diff --git a/.ci/generate-buildkite-pipeline-premerge b/.ci/generate-buildkite-pipeline-premerge
index 9d9ca32183944..e547afaeb722f 100755
--- a/.ci/generate-buildkite-pipeline-premerge
+++ b/.ci/generate-buildkite-pipeline-premerge
@@ -128,9 +128,8 @@ if [[ "${windows_projects}" != "" ]]; then
         limit: 2
   timeout_in_minutes: 150
   env:
-    CC: 'cl'
-    CXX: 'cl'
-    LD: 'link'
+    MAX_PARALLEL_COMPILE_JOBS: '16'
+    MAX_PARALLEL_LINK_JOBS: '4'
   commands:
   - 'C:\\BuildTools\\Common7\\Tools\\VsDevCmd.bat -arch=amd64 -host_arch=amd64'
   - 'bash .ci/monolithic-windows.sh "$(echo ${windows_projects} | tr ' ' ';')" "$(echo ${windows_check_targets})"'
diff --git a/.ci/metrics/metrics.py b/.ci/metrics/metrics.py
index 48d2aa2f330ec..70b787665a8b9 100644
--- a/.ci/metrics/metrics.py
+++ b/.ci/metrics/metrics.py
@@ -130,34 +130,6 @@ def get_per_workflow_metrics(
         workflow_jobs = workflow_run.jobs()
         if workflow_jobs.totalCount == 0:
             continue
-        if workflow_jobs.totalCount > 1:
-            raise ValueError(
-                f"Encountered an unexpected number of jobs: {workflow_jobs.totalCount}"
-            )
-
-        created_at = workflow_jobs[0].created_at
-        started_at = workflow_jobs[0].started_at
-        completed_at = workflow_jobs[0].completed_at
-
-        job_result = int(workflow_jobs[0].conclusion == "success")
-        if job_result:
-            # We still might want to mark the job as a failure if one of the steps
-            # failed. This is required due to use setting continue-on-error in
-            # the premerge pipeline to prevent sending emails while we are
-            # testing the infrastructure.
-            # TODO(boomanaiden154): Remove this once the premerge pipeline is no
-            # longer in a testing state and we can directly assert the workflow
-            # result.
-            for step in workflow_jobs[0].steps:
-                if step.conclusion != "success":
-                    job_result = 0
-                    break
-
-        queue_time = started_at - created_at
-        run_time = completed_at - started_at
-
-        if run_time.seconds == 0:
-            continue
 
         if (
             workflows_to_track[workflow_run.name] is None
@@ -170,20 +142,45 @@ def get_per_workflow_metrics(
         ):
             break
 
-        # The timestamp associated with the event is expected by Grafana to be
-        # in nanoseconds.
-        created_at_ns = int(created_at.timestamp()) * 10**9
-
-        workflow_metrics.append(
-            JobMetrics(
-                workflow_run.name,
-                queue_time.seconds,
-                run_time.seconds,
-                job_result,
-                created_at_ns,
-                workflow_run.id,
+        for workflow_job in workflow_jobs:
+            created_at = workflow_job.created_at
+            started_at = workflow_job.started_at
+            completed_at = workflow_job.completed_at
+
+            job_result = int(workflow_job.conclusion == "success")
+            if job_result:
+                # We still might want to mark the job as a failure if one of the steps
+                # failed. This is required due to use setting continue-on-error in
+                # the premerge pipeline to prevent sending emails while we are
+                # testing the infrastructure.
+                # TODO(boomanaiden154): Remove this once the premerge pipeline is no
+                # longer in a testing state and we can directly assert the workflow
+                # result.
+                for step in workflow_job.steps:
+                    if step.conclusion != "success":
+                        job_result = 0
+                        break
+
+            queue_time = started_at - created_at
+            run_time = completed_at - started_at
+
+            if run_time.seconds == 0:
+                continue
+
+            # The timestamp associated with the event is expected by Grafana to be
+            # in nanoseconds.
+            created_at_ns = int(created_at.timestamp()) * 10**9
+
+            workflow_metrics.append(
+                JobMetrics(
+                    workflow_run.name + "-" + workflow_job.name,
+                    queue_time.seconds,
+                    run_time.seconds,
+                    job_result,
+                    created_at_ns,
+                    workflow_run.id,
+                )
             )
-        )
 
     return workflow_metrics
 
diff --git a/.ci/monolithic-windows.sh b/.ci/monolithic-windows.sh
index 68303a3ea153a..57b276f3e1df0 100755
--- a/.ci/monolithic-windows.sh
+++ b/.ci/monolithic-windows.sh
@@ -50,6 +50,10 @@ echo "--- cmake"
 pip install -q -r "${MONOREPO_ROOT}"/mlir/python/requirements.txt
 pip install -q -r "${MONOREPO_ROOT}"/.ci/requirements.txt
 
+export CC=cl
+export CXX=cl
+export LD=link
+
 # The CMAKE_*_LINKER_FLAGS to disable the manifest come from research
 # on fixing a build reliability issue on the build server, please
 # see https://github.com/llvm/llvm-project/pull/82393 and
@@ -72,8 +76,8 @@ cmake -S "${MONOREPO_ROOT}"/llvm -B "${BUILD_DIR}" \
       -D CMAKE_EXE_LINKER_FLAGS="/MANIFEST:NO" \
       -D CMAKE_MODULE_LINKER_FLAGS="/MANIFEST:NO" \
       -D CMAKE_SHARED_LINKER_FLAGS="/MANIFEST:NO" \
-      -D LLVM_PARALLEL_COMPILE_JOBS=16 \
-      -D LLVM_PARALLEL_LINK_JOBS=4
+      -D LLVM_PARALLEL_COMPILE_JOBS=${MAX_PARALLEL_COMPILE_JOBS} \
+      -D LLVM_PARALLEL_LINK_JOBS=${MAX_PARALLEL_LINK_JOBS}
 
 echo "--- ninja"
 # Targets are not escaped as they are passed as separate arguments.
diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS
index 484b947bda402..dd4116fa16bc5 100644
--- a/.github/CODEOWNERS
+++ b/.github/CODEOWNERS
@@ -131,7 +131,7 @@
 /bolt/ @aaupov @maksfb @rafaelauler @ayermolo @dcci @yota9
 
 # Bazel build system.
-/utils/bazel/ @rupprecht @keith
+/utils/bazel/ @rupprecht @keith @aaronmondal
 
 # InstallAPI and TextAPI
 /llvm/**/TextAPI/ @cyndyishida
diff --git a/.github/new-prs-labeler.yml b/.github/new-prs-labeler.yml
index 566308bb3df8d..9863ff087ca86 100644
--- a/.github/new-prs-labeler.yml
+++ b/.github/new-prs-labeler.yml
@@ -730,6 +730,9 @@ llvm:regalloc:
 lldb:
   - lldb/**
 
+lldb-dap:
+  - lldb/tools/lldb-dap/**
+
 backend:AMDGPU:
   - '**/*amdgpu*'
   - '**/*AMDGPU*'
diff --git a/.github/workflows/build-ci-container.yml b/.github/workflows/build-ci-container.yml
index 8a81d47186469..8272c8f6e266f 100644
--- a/.github/workflows/build-ci-container.yml
+++ b/.github/workflows/build-ci-container.yml
@@ -20,24 +20,31 @@ on:
 jobs:
   build-ci-container:
     if: github.repository_owner == 'llvm'
-    runs-on: depot-ubuntu-22.04-16
-    outputs:
-      container-name: ${{ steps.vars.outputs.container-name }}
-      container-name-agent: ${{ steps.vars.outputs.container-name-agent }}
-      container-name-tag: ${{ steps.vars.outputs.container-name-tag }}
-      container-name-agent-tag: ${{ steps.vars.outputs.container-name-agent-tag }}
-      container-filename: ${{ steps.vars.outputs.container-filename }}
-      container-agent-filename: ${{ steps.vars.outputs.container-agent-filename }}
+    runs-on: ${{ matrix.runs-on }}
+    strategy:
+      matrix:
+        include:
+          # The arch names should match the names used on dockerhub.
+          # See https://github.com/docker-library/official-images#architectures-other-than-amd64
+          - arch: amd64
+            runs-on: depot-ubuntu-22.04-16
+          - arch: arm64v8
+            runs-on: depot-ubuntu-22.04-arm-16
     steps:
       - name: Checkout LLVM
         uses: actions/checkout@v4
         with:
           sparse-checkout: .github/workflows/containers/github-action-ci/
+      # podman is not installed by default on the ARM64 images.
+      - name: Install Podman
+        if: runner.arch == 'ARM64'
+        run: |
+          sudo apt-get install podman
       - name: Write Variables
         id: vars
         run: |
-          tag=`date +%s`
-          container_name="ghcr.io/$GITHUB_REPOSITORY_OWNER/ci-ubuntu-22.04"
+          tag=$(git rev-parse --short=12 HEAD)
+          container_name="ghcr.io/$GITHUB_REPOSITORY_OWNER/${{ matrix.arch }}/ci-ubuntu-22.04"
           echo "container-name=$container_name" >> $GITHUB_OUTPUT
           echo "container-name-agent=$container_name-agent" >> $GITHUB_OUTPUT
           echo "container-name-tag=$container_name:$tag" >> $GITHUB_OUTPUT
@@ -61,7 +68,7 @@ jobs:
       - name: Upload container image
         uses: actions/upload-artifact@v4
         with:
-          name: container
+          name: container-${{ matrix.arch }}
           path: "*.tar"
           retention-days: 14
 
@@ -84,18 +91,29 @@ jobs:
     steps:
       - name: Download container
         uses: actions/download-artifact@v4
-        with:
-          name: container
 
       - name: Push Container
         run: |
-          podman load -i ${{ needs.build-ci-container.outputs.container-filename }}
-          podman tag ${{ needs.build-ci-container.outputs.container-name-tag }} ${{ needs.build-ci-container.outputs.container-name }}:latest
+          function push_container {
+            image_name=$1
+            latest_name=$(echo $image_name | sed 's/:[a-f0-9]\+$/:latest/g')
+            podman tag $image_name $latest_name
+            echo "Pushing $image_name ..."
+            podman push $image_name
+            echo "Pushing $latest_name ..."
+            podman push $latest_name
+          }
+
           podman login -u ${{ github.actor }} -p $GITHUB_TOKEN ghcr.io
-          podman push ${{ needs.build-ci-container.outputs.container-name-tag }}
-          podman push ${{ needs.build-ci-container.outputs.container-name }}:latest
+          for f in $(find . -iname *.tar); do
+            image_name=$(podman load -q -i $f | sed 's/Loaded image: //g')
+            push_container $image_name
 
-          podman load -i ${{ needs.build-ci-container.outputs.container-agent-filename }}
-          podman tag ${{ needs.build-ci-container.outputs.container-name-agent-tag }} ${{ needs.build-ci-container.outputs.container-name-agent }}:latest
-          podman push ${{ needs.build-ci-container.outputs.container-name-agent-tag }}
-          podman push ${{ needs.build-ci-container.outputs.container-name-agent }}:latest
+            if echo $image_name | grep '/amd64/'; then
+              # For amd64, create an alias with the arch component removed.
+              # This matches the convention used on dockerhub.
+              default_image_name=$(echo $(dirname $(dirname $image_name))/$(basename $image_name))
+              podman tag $image_name $default_image_name
+              push_container $default_image_name
+            fi
+          done
diff --git a/.github/workflows/clang-tests.yml b/.github/workflows/clang-tests.yml
deleted file mode 100644
index 2569ce19518e3..0000000000000
--- a/.github/workflows/clang-tests.yml
+++ /dev/null
@@ -1,38 +0,0 @@
-name: Clang Tests
-
-permissions:
-  contents: read
-
-on:
-  workflow_dispatch:
-  push:
-    branches:
-      - 'release/**'
-    paths:
-      - 'clang/**'
-      - '.github/workflows/clang-tests.yml'
-      - '.github/workflows/llvm-project-tests.yml'
-      - '!llvm/**'
-  pull_request:
-    branches:
-      - 'release/**'
-    paths:
-      - 'clang/**'
-      - '.github/workflows/clang-tests.yml'
-      - '.github/workflows/llvm-project-tests.yml'
-      - '!llvm/**'
-
-concurrency:
-  # Skip intermediate builds: always.
-  # Cancel intermediate builds: only if it is a pull request build.
-  group: ${{ github.workflow }}-${{ github.ref }}
-  cancel-in-progress: ${{ startsWith(github.ref, 'refs/pull/') }}
-
-jobs:
-  check_clang:
-    if: github.repository_owner == 'llvm'
-    name: Test clang,lldb,libclc
-    uses: ./.github/workflows/llvm-project-tests.yml
-    with:
-      build_target: check-clang
-      projects: clang;lldb;libclc
diff --git a/.github/workflows/commit-access-greeter.yml b/.github/workflows/commit-access-greeter.yml
new file mode 100644
index 0000000000000..29a1b578f8af8
--- /dev/null
+++ b/.github/workflows/commit-access-greeter.yml
@@ -0,0 +1,39 @@
+name: Commit Access Greeter
+
+on:
+  issues:
+    types:
+      - labeled
+
+permissions:
+  contents: read
+
+jobs:
+  commit-access-greeter:
+    permissions:
+      issues: write
+    if: >-
+      github.repository_owner == 'llvm' &&
+      github.event.label.name == 'infra:commit-access-request'
+    runs-on: ubuntu-22.04
+    steps:
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683
+        with:
+          sparse-checkout: llvm/utils/git/
+
+      - name: Setup Automation Script
+        working-directory: ./llvm/utils/git/
+        run: |
+          pip install --require-hashes -r requirements.txt
+
+      - name: Add comments to issue
+        working-directory: ./llvm/utils/git/
+        env:
+          LABEL_NAME: ${{ github.event.label.name }}
+          GITHUB_TOKEN: ${{ github.token }}
+          ISSUE_NUMBER: ${{ github.event.issue.number }}
+        run: |
+          python3 ./github-automation.py \
+            --token $GITHUB_TOKEN \
+             commit-request-greeter \
+             --issue-number $ISSUE_NUMBER
diff --git a/.github/workflows/containers/github-action-ci-windows/Dockerfile b/.github/workflows/containers/github-action-ci-windows/Dockerfile
index 2295e39d62c30..9a1fab694c9df 100644
--- a/.github/workflows/containers/github-action-ci-windows/Dockerfile
+++ b/.github/workflows/containers/github-action-ci-windows/Dockerfile
@@ -108,7 +108,7 @@ RUN choco install -y handle
 
 RUN pip3 install pywin32 buildbot-worker==2.8.4
 
-ARG RUNNER_VERSION=2.321.0
+ARG RUNNER_VERSION=2.322.0
 ENV RUNNER_VERSION=$RUNNER_VERSION
 
 RUN powershell -Command \
diff --git a/.github/workflows/containers/github-action-ci/Dockerfile b/.github/workflows/containers/github-action-ci/Dockerfile
index 35a0f1f6020dc..377b8f14402ee 100644
--- a/.github/workflows/containers/github-action-ci/Dockerfile
+++ b/.github/workflows/containers/github-action-ci/Dockerfile
@@ -96,7 +96,7 @@ WORKDIR /home/gha
 
 FROM ci-container as ci-container-agent
 
-ENV GITHUB_RUNNER_VERSION=2.321.0
+ENV GITHUB_RUNNER_VERSION=2.322.0
 
 RUN mkdir actions-runner && \
     cd actions-runner && \
diff --git a/.github/workflows/libc-fullbuild-tests.yml b/.github/workflows/libc-fullbuild-tests.yml
index 2c88da653aae4..d93ac84116240 100644
--- a/.github/workflows/libc-fullbuild-tests.yml
+++ b/.github/workflows/libc-fullbuild-tests.yml
@@ -15,6 +15,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
+        build_type: [Debug, Release, MinSizeRel]
         include:
           - os: ubuntu-24.04
             ccache-variant: sccache
@@ -68,7 +69,7 @@ jobs:
         cmake -B ${{ steps.strings.outputs.build-output-dir }}
         -DCMAKE_CXX_COMPILER=${{ matrix.cpp_compiler }}
         -DCMAKE_C_COMPILER=${{ matrix.c_compiler }}
-        -DCMAKE_BUILD_TYPE=MinSizeRel
+        -DCMAKE_BUILD_TYPE=${{ matrix.build_type }}
         -DCMAKE_C_COMPILER_LAUNCHER=${{ matrix.ccache-variant }}
         -DCMAKE_CXX_COMPILER_LAUNCHER=${{ matrix.ccache-variant }}
         -DCMAKE_INSTALL_PREFIX=${{ steps.strings.outputs.build-install-dir }}
diff --git a/.github/workflows/libc-overlay-tests.yml b/.github/workflows/libc-overlay-tests.yml
index 0a0916084b18c..de4b58c008ee4 100644
--- a/.github/workflows/libc-overlay-tests.yml
+++ b/.github/workflows/libc-overlay-tests.yml
@@ -16,6 +16,7 @@ jobs:
       # Set fail-fast to false to ensure that feedback is delivered for all matrix combinations.
       fail-fast: false
       matrix:
+        build_type: [Debug, Release, MinSizeRel]
         include:
           # TODO: add linux gcc when it is fixed
           - os: ubuntu-24.04
@@ -95,7 +96,7 @@ jobs:
         cmake -B ${{ steps.strings.outputs.build-output-dir }}
         -DCMAKE_CXX_COMPILER=${{ matrix.compiler.cpp_compiler }}
         -DCMAKE_C_COMPILER=${{ matrix.compiler.c_compiler }}
-        -DCMAKE_BUILD_TYPE=MinSizeRel
+        -DCMAKE_BUILD_TYPE=${{ matrix.build_type }}
         -DCMAKE_C_COMPILER_LAUNCHER=${{ matrix.ccache-variant }}
         -DCMAKE_CXX_COMPILER_LAUNCHER=${{ matrix.ccache-variant }}
         -DCMAKE_POLICY_DEFAULT_CMP0141=NEW
diff --git a/.github/workflows/libclc-tests.yml b/.github/workflows/libclc-tests.yml
deleted file mode 100644
index 23192f776a985..0000000000000
--- a/.github/workflows/libclc-tests.yml
+++ /dev/null
@@ -1,39 +0,0 @@
-name: libclc Tests
-
-permissions:
-  contents: read
-
-on:
-  workflow_dispatch:
-  push:
-    branches:
-      - 'release/**'
-    paths:
-      - 'libclc/**'
-      - '.github/workflows/libclc-tests.yml'
-      - '.github/workflows/llvm-project-tests.yml'
-      - '!clang/**'
-      - '!llvm/**'
-  pull_request:
-    branches:
-      - 'release/**'
-    paths:
-      - 'libclc/**'
-      - '.github/workflows/libclc-tests.yml'
-      - '.github/workflows/llvm-project-tests.yml'
-      - '!clang/**'
-      - '!llvm/**'
-
-concurrency:
-  # Skip intermediate builds: always.
-  # Cancel intermediate builds: only if it is a pull request build.
-  group: ${{ github.workflow }}-${{ github.ref }}
-  cancel-in-progress: ${{ startsWith(github.ref, 'refs/pull/') }}
-
-jobs:
-  check_libclc:
-    if: github.repository_owner == 'llvm'
-    name: Test libclc
-    uses: ./.github/workflows/llvm-project-tests.yml
-    with:
-      projects: clang;libclc
diff --git a/.github/workflows/libcxx-build-and-test.yaml b/.github/workflows/libcxx-build-and-test.yaml
index a28bf4d5daf6d..ee77e83363d37 100644
--- a/.github/workflows/libcxx-build-and-test.yaml
+++ b/.github/workflows/libcxx-build-and-test.yaml
@@ -48,8 +48,8 @@ jobs:
           'generic-cxx26',
           'generic-modules'
         ]
-        cc: [  'clang-19' ]
-        cxx: [ 'clang++-19' ]
+        cc: [  'clang-20' ]
+        cxx: [ 'clang++-20' ]
         include:
           - config: 'generic-gcc'
             cc: 'gcc-14'
@@ -88,18 +88,18 @@ jobs:
           'generic-cxx20',
           'generic-cxx23'
         ]
-        cc: [ 'clang-19' ]
-        cxx: [ 'clang++-19' ]
+        cc: [ 'clang-20' ]
+        cxx: [ 'clang++-20' ]
         include:
           - config: 'generic-gcc-cxx11'
             cc: 'gcc-14'
             cxx: 'g++-14'
           - config: 'generic-cxx23'
-            cc: 'clang-17'
-            cxx: 'clang++-17'
-          - config: 'generic-cxx26'
             cc: 'clang-18'
             cxx: 'clang++-18'
+          - config: 'generic-cxx26'
+            cc: 'clang-19'
+            cxx: 'clang++-19'
     steps:
       - uses: actions/checkout@v4
       - name: ${{ matrix.config }}
@@ -169,8 +169,8 @@ jobs:
       - name: ${{ matrix.config }}
         run: libcxx/utils/ci/run-buildbot ${{ matrix.config }}
         env:
-          CC: clang-19
-          CXX: clang++-19
+          CC: clang-20
+          CXX: clang++-20
       - uses: actions/upload-artifact@26f96dfa697d77e81fd5907df203aa23a56210a8 # v4.3.0
         if: always()
         with:
diff --git a/.github/workflows/libcxx-build-containers.yml b/.github/workflows/libcxx-build-containers.yml
index 2d040f712ce59..bb4bd8843772f 100644
--- a/.github/workflows/libcxx-build-containers.yml
+++ b/.github/workflows/libcxx-build-containers.yml
@@ -9,7 +9,6 @@ name: Build Docker images for libc++ CI
 
 permissions:
   contents: read
-  packages: write
 
 on:
   push:
diff --git a/.github/workflows/lld-tests.yml b/.github/workflows/lld-tests.yml
deleted file mode 100644
index 599c0975fa685..0000000000000
--- a/.github/workflows/lld-tests.yml
+++ /dev/null
@@ -1,38 +0,0 @@
-name: LLD Tests
-
-permissions:
-  contents: read
-
-on:
-  workflow_dispatch:
-  push:
-    branches:
-      - 'release/**'
-    paths:
-      - 'lld/**'
-      - '.github/workflows/lld-tests.yml'
-      - '.github/workflows/llvm-project-tests.yml'
-      - '!llvm/**'
-  pull_request:
-    branches:
-      - 'release/**'
-    paths:
-      - 'lld/**'
-      - '.github/workflows/lld-tests.yml'
-      - '.github/workflows/llvm-project-tests.yml'
-      - '!llvm/**'
-
-concurrency:
-  # Skip intermediate builds: always.
-  # Cancel intermediate builds: only if it is a pull request build.
-  group: ${{ github.workflow }}-${{ github.ref }}
-  cancel-in-progress: ${{ startsWith(github.ref, 'refs/pull/') }}
-
-jobs:
-  check_lld:
-    if: github.repository_owner == 'llvm'
-    name: Test lld
-    uses: ./.github/workflows/llvm-project-tests.yml
-    with:
-      build_target: check-lld
-      projects: lld
diff --git a/.github/workflows/lldb-tests.yml b/.github/workflows/lldb-tests.yml
deleted file mode 100644
index 6bb9721956258..0000000000000
--- a/.github/workflows/lldb-tests.yml
+++ /dev/null
@@ -1,39 +0,0 @@
-name: lldb Tests
-
-permissions:
-  contents: read
-
-on:
-  workflow_dispatch:
-  push:
-    branches:
-      - 'release/**'
-    paths:
-      - 'lldb/**'
-      - '.github/workflows/lldb-tests.yml'
-      - '.github/workflows/llvm-project-tests.yml'
-      - '!clang/**'
-      - '!llvm/**'
-  pull_request:
-    branches:
-      - 'release/**'
-    paths:
-      - 'lldb/**'
-      - '.github/workflows/lldb-tests.yml'
-      - '.github/workflows/llvm-project-tests.yml'
-      - '!clang/**'
-      - '!llvm/**'
-
-concurrency:
-  # Skip intermediate builds: always.
-  # Cancel intermediate builds: only if it is a pull request build.
-  group: ${{ github.workflow }}-${{ github.ref }}
-  cancel-in-progress: ${{ startsWith(github.ref, 'refs/pull/') }}
-
-jobs:
-  build_lldb:
-    if: github.repository_owner == 'llvm'
-    name: Build lldb
-    uses: ./.github/workflows/llvm-project-tests.yml
-    with:
-      projects: clang;lldb
diff --git a/.github/workflows/llvm-tests.yml b/.github/workflows/llvm-tests.yml
index 4e570a7cb1455..9b3d49d4e99b9 100644
--- a/.github/workflows/llvm-tests.yml
+++ b/.github/workflows/llvm-tests.yml
@@ -11,14 +11,12 @@ on:
     paths:
       - 'llvm/**'
       - '.github/workflows/llvm-tests.yml'
-      - '.github/workflows/llvm-project-tests.yml'
   pull_request:
     branches:
       - 'release/**'
     paths:
       - 'llvm/**'
       - '.github/workflows/llvm-tests.yml'
-      - '.github/workflows/llvm-project-tests.yml'
 
 concurrency:
   # Skip intermediate builds: always.
@@ -27,14 +25,6 @@ concurrency:
   cancel-in-progress: ${{ startsWith(github.ref, 'refs/pull/') }}
 
 jobs:
-  check-all:
-    if: github.repository_owner == 'llvm'
-    name: Build and Test
-    uses: ./.github/workflows/llvm-project-tests.yml
-    with:
-      build_target: check-all
-      projects: clang;lld;libclc;lldb
-
   abi-dump-setup:
     if: github.repository_owner == 'llvm'
     runs-on: ubuntu-latest
diff --git a/.github/workflows/premerge.yaml b/.github/workflows/premerge.yaml
index 30f4fc807f3a5..178ab191a58be 100644
--- a/.github/workflows/premerge.yaml
+++ b/.github/workflows/premerge.yaml
@@ -5,19 +5,32 @@ permissions:
 
 on:
   pull_request:
+    types:
+      - opened
+      - synchronize
+      - reopened
+      # When a PR is closed, we still start this workflow, but then skip
+      # all the jobs, which makes it effectively a no-op.  The reason to
+      # do this is that it allows us to take advantage of concurrency groups
+      # to cancel in progress CI jobs whenever the PR is closed.
+      - closed
     paths:
       - .github/workflows/premerge.yaml
   push:
     branches:
       - 'main'
+      - 'release/**'
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}
+  cancel-in-progress: true
 
 jobs:
   premerge-checks-linux:
-    if: github.repository_owner == 'llvm'
+    if: >-
+        github.repository_owner == 'llvm' &&
+        (github.event_name != 'pull_request' || github.event.action != 'closed')
     runs-on: llvm-premerge-linux-runners
-    concurrency:
-      group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}
-      cancel-in-progress: true
     steps:
       - name: Checkout LLVM
         uses: actions/checkout@v4
@@ -70,3 +83,132 @@ jobs:
           export CXX=/opt/llvm/bin/clang++
 
           ./.ci/monolithic-linux.sh "$(echo ${linux_projects} | tr ' ' ';')" "$(echo ${linux_check_targets})" "$(echo ${linux_runtimes} | tr ' ' ';')" "$(echo ${linux_runtime_check_targets})"
+
+  premerge-checks-windows:
+    if: >-
+        github.repository_owner == 'llvm' &&
+        (github.event_name != 'pull_request' || github.event.action != 'closed')
+    runs-on: llvm-premerge-windows-runners
+    defaults:
+      run:
+        shell: bash
+    steps:
+      - name: Checkout LLVM
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 2
+      - name: Setup ccache
+        uses: hendrikmuhs/ccache-action@v1.2.14
+        with:
+          variant: "sccache"
+          max-size: "2000M"
+      - name: Compute Projects
+        id: vars
+        run: |
+          modified_files=$(git diff --name-only HEAD~1...HEAD)
+          modified_dirs=$(echo "$modified_files" | cut -d'/' -f1 | sort | uniq)
+
+          echo $modified_files
+          echo $modified_dirs
+
+          . ./.ci/compute-projects.sh
+
+          all_projects="bolt clang clang-tools-extra compiler-rt cross-project-tests flang libc libclc lld lldb llvm mlir openmp polly pstl"
+          modified_projects="$(keep-modified-projects ${all_projects})"
+
+          windows_projects_to_test=$(exclude-windows $(compute-projects-to-test 1 ${modified_projects}))
+          windows_check_targets=$(check-targets ${windows_projects_to_test} | sort | uniq | tr -d '\r' | tr '\n' ' ')
+          windows_projects=$(add-dependencies ${windows_projects_to_test} | sort | uniq | tr -d '\r' | tr '\n' ';')
+
+          if [[ "${windows_projects}" == "" ]]; then
+            echo "No projects to build"
+          fi
+
+          echo "Building projects: ${windows_projects}"
+          echo "Running project checks targets: ${windows_check_targets}"
+
+          echo "windows-projects=${windows_projects}" >> $GITHUB_OUTPUT
+          echo "windows-check-targets=${windows_check_targets}" >> $GITHUB_OUTPUT
+      - name: Build and Test
+        # Mark the job as a success even if the step fails so that people do
+        # not get notified while the new premerge pipeline is in an
+        # experimental state.
+        # TODO(boomanaiden154): Remove this once the pipeline is stable and we
+        # are ready for people to start recieving notifications.
+        continue-on-error: true
+        if: ${{ steps.vars.outputs.windows-projects != '' }}
+        shell: cmd
+        run: |
+          set MAX_PARALLEL_COMPILE_JOBS=64
+          set MAX_PARALLEL_LINK_JOBS=64
+          call C:\\BuildTools\\Common7\\Tools\\VsDevCmd.bat -arch=amd64 -host_arch=amd64
+          bash .ci/monolithic-windows.sh "${{ steps.vars.outputs.windows-projects }}" "${{ steps.vars.outputs.windows-check-targets }}"
+
+  permerge-check-macos:
+    runs-on: macos-14
+    if: >-
+      github.repository_owner == 'llvm' &&
+      (startswith(github.ref_name, 'release/') ||
+       startswith(github.base_ref, 'release/')) &&
+      (github.event_name != 'pull_request' || github.event.action != 'closed')
+    steps:
+      - name: Checkout LLVM
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 2
+      - name: Setup ccache
+        uses: hendrikmuhs/ccache-action@v1.2.14
+        with:
+          max-size: "2000M"
+      - name: Install Ninja
+        uses: llvm/actions/install-ninja@main
+      - name: Build and Test
+        run: |
+          modified_files=$(git diff --name-only HEAD~1...HEAD)
+          modified_dirs=$(echo "$modified_files" | cut -d'/' -f1 | sort -u)
+
+          echo $modified_files
+          echo $modified_dirs
+
+          . ./.ci/compute-projects.sh
+
+          all_projects="clang clang-tools-extra lld lldb llvm mlir"
+          modified_projects="$(keep-modified-projects ${all_projects})"
+
+          # We have to disable the runtimes builds due to https://github.com/llvm/llvm-project/issues/90568
+          # and the lldb tests depend on libcxx, so we need to skip them.
+          mac_check_targets=$(check-targets ${modified_projects} | sort | uniq | tr '\n' ' ' | sed -e 's/check-lldb //g')
+          mac_projects=$(add-dependencies ${modified_projects} | sort | uniq | tr '\n' ' ')
+
+          mac_runtimes_to_test=$(compute-runtimes-to-test ${modified_projects})
+          mac_runtime_check_targets=$(check-targets ${mac_runtimes_to_test} | sort | uniq | tr '\n' ' ')
+          mac_runtimes=$(echo ${mac_runtimes_to_test} | tr ' ' '\n' | sort | uniq | tr '\n' ' ')
+
+          if [[ "${mac_projects}" == "" ]]; then
+            echo "No projects to build"
+            exit 0
+          fi
+
+          echo "Projects to test: ${modified_projects}"
+          echo "Runtimes to test: ${mac_runtimes_to_test}"
+          echo "Building projects: ${mac_projects}"
+          echo "Running project checks targets: ${mac_check_targets}"
+          echo "Building runtimes: ${mac_runtimes}"
+          echo "Running runtimes checks targets: ${mac_runtime_check_targets}"
+
+          # -DLLVM_DISABLE_ASSEMBLY_FILES=ON is for
+          # https://github.com/llvm/llvm-project/issues/81967
+          # Disable sharding in lit so that the LIT_XFAIL environment var works.
+          cmake -G Ninja \
+                -B build \
+                -S llvm \
+                -DLLVM_ENABLE_PROJECTS="$(echo ${mac_projects} | tr ' ' ';')" \
+                -DLLVM_DISABLE_ASSEMBLY_FILES=ON \
+                -DCMAKE_BUILD_TYPE=Release \
+                -DLLDB_INCLUDE_TESTS=OFF \
+                -DLLVM_ENABLE_ASSERTIONS=ON \
+                -DCMAKE_C_COMPILER_LAUNCHER=ccache \
+                -DCMAKE_CXX_COMPILER_LAUNCHER=ccache
+
+          # The libcxx tests fail, so we are skipping the runtime targets.
+          ninja -C build  $mac_check_targets
diff --git a/.github/workflows/release-binaries-all.yml b/.github/workflows/release-binaries-all.yml
index d5b2d33286101..d18b9b0b5c2ff 100644
--- a/.github/workflows/release-binaries-all.yml
+++ b/.github/workflows/release-binaries-all.yml
@@ -83,6 +83,7 @@ jobs:
       matrix:
         runs-on:
           - ubuntu-22.04
+          - ubuntu-22.04-arm
           - macos-13
           - macos-14
 
diff --git a/.github/workflows/release-binaries.yml b/.github/workflows/release-binaries.yml
index f9a264e7cf48f..204ee6405382f 100644
--- a/.github/workflows/release-binaries.yml
+++ b/.github/workflows/release-binaries.yml
@@ -18,6 +18,7 @@ on:
         type: choice
         options:
           - ubuntu-22.04
+          - ubuntu-22.04-arm
           - macos-13
           - macos-14
 
@@ -55,8 +56,8 @@ jobs:
       ref: ${{ steps.vars.outputs.ref }}
       upload: ${{ steps.vars.outputs.upload }}
       target-cmake-flags: ${{ steps.vars.outputs.target-cmake-flags }}
+      ccache: ${{ steps.vars.outputs.ccache }}
       build-flang: ${{ steps.vars.outputs.build-flang }}
-      enable-pgo: ${{ steps.vars.outputs.enable-pgo }}
       release-binary-basename: ${{ steps.vars.outputs.release-binary-basename }}
       release-binary-filename: ${{ steps.vars.outputs.release-binary-filename }}
       build-runs-on: ${{ steps.vars.outputs.build-runs-on }}
@@ -119,10 +120,15 @@ jobs:
         echo "release-binary-basename=$release_binary_basename" >> $GITHUB_OUTPUT
         echo "release-binary-filename=$release_binary_basename.tar.xz" >> $GITHUB_OUTPUT
 
-        # Detect necessary CMake flags
         target="$RUNNER_OS-$RUNNER_ARCH"
-        echo "enable-pgo=false" >> $GITHUB_OUTPUT
-        target_cmake_flags="-DLLVM_RELEASE_ENABLE_PGO=OFF"
+        # The hendrikmuhs/ccache-action action does not support installing sccache
+        # on arm64 Linux.
+        if [ "$target" = "Linux-ARM64" ]; then
+          echo ccache=ccache >> $GITHUB_OUTPUT
+        else
+          echo ccache=sccache >> $GITHUB_OUTPUT
+        fi
+
         # The macOS builds try to cross compile some libraries so we need to
         # add extra CMake args to disable them.
         # See https://github.com/llvm/llvm-project/issues/99767
@@ -146,7 +152,7 @@ jobs:
         echo "target-cmake-flags=$target_cmake_flags" >> $GITHUB_OUTPUT
         echo "build-flang=$build_flang" >> $GITHUB_OUTPUT
         case "${{ inputs.runs-on }}" in
-          ubuntu-22.04)
+          ubuntu-22.04*)
             build_runs_on="depot-${{ inputs.runs-on }}-16"
             test_runs_on=$build_runs_on
             ;;
@@ -216,17 +222,11 @@ jobs:
       id: setup-stage
       uses: ./workflows-main/.github/workflows/release-binaries-setup-stage
 
-    - name: Setup sccache
-      uses: hendrikmuhs/ccache-action@ca3acd2731eef11f1572ccb126356c2f9298d35e # v1.2.9
-      with:
-        # Default to 2G to workaround: https://github.com/hendrikmuhs/ccache-action/issues/174
-        max-size: 2G
-        key: sccache-${{ runner.os }}-${{ runner.arch }}-release
-        variant: sccache
-
     - name: Configure
       id: build
       shell: bash
+      env:
+        CCACHE_BIN: ${{ needs.prepare.outputs.ccache }}
       run: |
         # There were some issues on the ARM64 MacOS runners with trying to build x86 object,
         # so we need to set some extra cmake flags to disable this.
@@ -234,14 +234,14 @@ jobs:
             ${{ needs.prepare.outputs.target-cmake-flags }} \
             -C clang/cmake/caches/Release.cmake \
             -DBOOTSTRAP_LLVM_PARALLEL_LINK_JOBS=1 \
-            -DBOOTSTRAP_CPACK_PACKAGE_FILE_NAME="${{ needs.prepare.outputs.release-binary-basename }}" \
-            -DCMAKE_C_COMPILER_LAUNCHER=sccache \
-            -DCMAKE_CXX_COMPILER_LAUNCHER=sccache
+            -DBOOTSTRAP_BOOTSTRAP_CPACK_PACKAGE_FILE_NAME="${{ needs.prepare.outputs.release-binary-basename }}"
+
     - name: Build
       shell: bash
       run: |
         ninja -v -C ${{ steps.setup-stage.outputs.build-prefix }}/build stage2-package
-        mv ${{ steps.setup-stage.outputs.build-prefix  }}/build/tools/clang/stage2-bins/${{ needs.prepare.outputs.release-binary-filename }} .
+        release_dir=`find ${{ steps.setup-stage.outputs.build-prefix }}/build -iname 'stage2-bins'`
+        mv $release_dir/${{ needs.prepare.outputs.release-binary-filename }} .
     
     - uses: actions/upload-artifact@26f96dfa697d77e81fd5907df203aa23a56210a8 #v4.3.0
       with:
@@ -256,7 +256,7 @@ jobs:
       shell: bash
       run: |
         find ${{ steps.setup-stage.outputs.build-prefix }}/build -iname ${{ needs.prepare.outputs.release-binary-filename }} -delete
-        rm -Rf ${{ steps.setup-stage.outputs.build-prefix }}/build/tools/clang/stage2-bins/_CPack_Packages
+        find ${{ steps.setup-stage.outputs.build-prefix }}/build -iname _CPack_Packages -prune -exec rm -r {} +
     
     - name: Save Stage
       uses: ./workflows-main/.github/workflows/release-binaries-save-stage
diff --git a/.github/workflows/release-tasks.yml b/.github/workflows/release-tasks.yml
index 780dd0ff6325c..52076ea1821b0 100644
--- a/.github/workflows/release-tasks.yml
+++ b/.github/workflows/release-tasks.yml
@@ -89,20 +89,10 @@ jobs:
     needs:
       - validate-tag
       - release-create
-    strategy:
-      fail-fast: false
-      matrix:
-        runs-on:
-          - ubuntu-22.04
-          - windows-2022
-          - macos-13
-          - macos-14
-
-    uses: ./.github/workflows/release-binaries.yml
+    uses: ./.github/workflows/release-binaries-all.yml
     with:
       release-version: ${{ needs.validate-tag.outputs.release-version }}
       upload: true
-      runs-on: ${{ matrix.runs-on }}
     # Called workflows don't have access to secrets by default, so we need to explicitly pass secrets that we use.
     secrets:
       RELEASE_TASKS_USER_TOKEN: ${{ secrets.RELEASE_TASKS_USER_TOKEN }}
diff --git a/bolt/include/bolt/Core/BinaryContext.h b/bolt/include/bolt/Core/BinaryContext.h
index 94fe4aa8aa0e5..8bec1db70e25a 100644
--- a/bolt/include/bolt/Core/BinaryContext.h
+++ b/bolt/include/bolt/Core/BinaryContext.h
@@ -1435,6 +1435,17 @@ class BinaryContext {
                         bool PrintRelocations = false,
                         StringRef Endl = "\n") const;
 
+  /// Print data when embedded in the instruction stream keeping the format
+  /// similar to printInstruction().
+  void printData(raw_ostream &OS, ArrayRef<uint8_t> Data,
+                 uint64_t Offset) const;
+
+  /// Extract data from the binary corresponding to [Address, Address + Size)
+  /// range. Return an empty ArrayRef if the address range does not belong to
+  /// any section in the binary, crosses a section boundary, or falls into a
+  /// virtual section.
+  ArrayRef<uint8_t> extractData(uint64_t Address, uint64_t Size) const;
+
   /// Print a range of instructions.
   template <typename Itr>
   uint64_t
diff --git a/bolt/include/bolt/Core/BinaryFunction.h b/bolt/include/bolt/Core/BinaryFunction.h
index e8b2757f7db21..942840a7621fd 100644
--- a/bolt/include/bolt/Core/BinaryFunction.h
+++ b/bolt/include/bolt/Core/BinaryFunction.h
@@ -2060,6 +2060,11 @@ class BinaryFunction {
     return Islands ? Islands->getAlignment() : 1;
   }
 
+  /// If there is a constant island in the range [StartOffset, EndOffset),
+  /// return its address.
+  std::optional<uint64_t> getIslandInRange(uint64_t StartOffset,
+                                           uint64_t EndOffset) const;
+
   uint64_t
   estimateConstantIslandSize(const BinaryFunction *OnBehalfOf = nullptr) const {
     if (!Islands)
diff --git a/bolt/include/bolt/Core/MCPlusBuilder.h b/bolt/include/bolt/Core/MCPlusBuilder.h
index 5d77e6faff2fc..c1460b2aac8a6 100644
--- a/bolt/include/bolt/Core/MCPlusBuilder.h
+++ b/bolt/include/bolt/Core/MCPlusBuilder.h
@@ -1426,11 +1426,12 @@ class MCPlusBuilder {
   }
 
   /// Creates an indirect call to the function within the \p DirectCall PLT
-  /// stub. The function's memory location is pointed by the \p TargetLocation
+  /// stub. The function's address location is pointed by the \p TargetLocation
   /// symbol.
+  /// Move instruction annotations from \p DirectCall to the indirect call.
   virtual InstructionListType
-  createIndirectPltCall(const MCInst &DirectCall,
-                        const MCSymbol *TargetLocation, MCContext *Ctx) {
+  createIndirectPLTCall(MCInst &&DirectCall, const MCSymbol *TargetLocation,
+                        MCContext *Ctx) {
     llvm_unreachable("not implemented");
     return {};
   }
diff --git a/bolt/lib/Core/BinaryContext.cpp b/bolt/lib/Core/BinaryContext.cpp
index f5e11358daaa3..f9fc536f3569a 100644
--- a/bolt/lib/Core/BinaryContext.cpp
+++ b/bolt/lib/Core/BinaryContext.cpp
@@ -1759,7 +1759,11 @@ void BinaryContext::preprocessDebugInfo() {
           dwarf::toString(CU->getUnitDIE().find(dwarf::DW_AT_name), nullptr);
       if (std::optional<uint64_t> DWOID = CU->getDWOId()) {
         auto Iter = DWOCUs.find(*DWOID);
-        assert(Iter != DWOCUs.end() && "DWO CU was not found.");
+        if (Iter == DWOCUs.end()) {
+          this->errs() << "BOLT-ERROR: DWO CU was not found for " << Name
+                       << '\n';
+          exit(1);
+        }
         Name = dwarf::toString(
             Iter->second->getUnitDIE().find(dwarf::DW_AT_name), nullptr);
       }
@@ -1942,6 +1946,43 @@ static void printDebugInfo(raw_ostream &OS, const MCInst &Instruction,
     OS << " discriminator:" << Row.Discriminator;
 }
 
+ArrayRef<uint8_t> BinaryContext::extractData(uint64_t Address,
+                                             uint64_t Size) const {
+  ArrayRef<uint8_t> Res;
+
+  const ErrorOr<const BinarySection &> Section = getSectionForAddress(Address);
+  if (!Section || Section->isVirtual())
+    return Res;
+
+  if (!Section->containsRange(Address, Size))
+    return Res;
+
+  auto *Bytes =
+      reinterpret_cast<const uint8_t *>(Section->getContents().data());
+  return ArrayRef<uint8_t>(Bytes + Address - Section->getAddress(), Size);
+}
+
+void BinaryContext::printData(raw_ostream &OS, ArrayRef<uint8_t> Data,
+                              uint64_t Offset) const {
+  DataExtractor DE(Data, AsmInfo->isLittleEndian(),
+                   AsmInfo->getCodePointerSize());
+  uint64_t DataOffset = 0;
+  while (DataOffset + 4 <= Data.size()) {
+    OS << format("    %08" PRIx64 ": \t.word\t0x", Offset + DataOffset);
+    const auto Word = DE.getUnsigned(&DataOffset, 4);
+    OS << Twine::utohexstr(Word) << '\n';
+  }
+  if (DataOffset + 2 <= Data.size()) {
+    OS << format("    %08" PRIx64 ": \t.short\t0x", Offset + DataOffset);
+    const auto Short = DE.getUnsigned(&DataOffset, 2);
+    OS << Twine::utohexstr(Short) << '\n';
+  }
+  if (DataOffset + 1 == Data.size()) {
+    OS << format("    %08" PRIx64 ": \t.byte\t0x%x\n", Offset + DataOffset,
+                 Data[DataOffset]);
+  }
+}
+
 void BinaryContext::printInstruction(raw_ostream &OS, const MCInst &Instruction,
                                      uint64_t Offset,
                                      const BinaryFunction *Function,
diff --git a/bolt/lib/Core/BinaryFunction.cpp b/bolt/lib/Core/BinaryFunction.cpp
index 1c5cd62a095b2..bc45caf3ec8b7 100644
--- a/bolt/lib/Core/BinaryFunction.cpp
+++ b/bolt/lib/Core/BinaryFunction.cpp
@@ -491,11 +491,27 @@ void BinaryFunction::print(raw_ostream &OS, std::string Annotation) {
   // Offset of the instruction in function.
   uint64_t Offset = 0;
 
+  auto printConstantIslandInRange = [&](uint64_t Start, uint64_t End) {
+    assert(Start <= End && "Invalid range");
+    std::optional<uint64_t> IslandOffset = getIslandInRange(Start, End);
+
+    if (!IslandOffset)
+      return;
+
+    const size_t IslandSize = getSizeOfDataInCodeAt(*IslandOffset);
+    BC.printData(OS, BC.extractData(getAddress() + *IslandOffset, IslandSize),
+                 *IslandOffset);
+  };
+
   if (BasicBlocks.empty() && !Instructions.empty()) {
     // Print before CFG was built.
+    uint64_t PrevOffset = 0;
     for (const std::pair<const uint32_t, MCInst> &II : Instructions) {
       Offset = II.first;
 
+      // Print any constant islands inbeetween the instructions.
+      printConstantIslandInRange(PrevOffset, Offset);
+
       // Print label if exists at this offset.
       auto LI = Labels.find(Offset);
       if (LI != Labels.end()) {
@@ -506,7 +522,12 @@ void BinaryFunction::print(raw_ostream &OS, std::string Annotation) {
       }
 
       BC.printInstruction(OS, II.second, Offset, this);
+
+      PrevOffset = Offset;
     }
+
+    // Print any data at the end of the function.
+    printConstantIslandInRange(PrevOffset, getMaxSize());
   }
 
   StringRef SplitPointMsg = "";
@@ -795,7 +816,6 @@ BinaryFunction::processIndirectBranch(MCInst &Instruction, unsigned Size,
 
   auto Begin = Instructions.begin();
   if (BC.isAArch64()) {
-    PreserveNops = BC.HasRelocations;
     // Start at the last label as an approximation of the current basic block.
     // This is a heuristic, since the full set of labels have yet to be
     // determined
@@ -1049,6 +1069,19 @@ size_t BinaryFunction::getSizeOfDataInCodeAt(uint64_t Offset) const {
   return getSize() - Offset;
 }
 
+std::optional<uint64_t>
+BinaryFunction::getIslandInRange(uint64_t StartOffset,
+                                 uint64_t EndOffset) const {
+  if (!Islands)
+    return std::nullopt;
+
+  auto Iter = llvm::lower_bound(Islands->DataOffsets, StartOffset);
+  if (Iter != Islands->DataOffsets.end() && *Iter < EndOffset)
+    return *Iter;
+
+  return std::nullopt;
+}
+
 bool BinaryFunction::isZeroPaddingAt(uint64_t Offset) const {
   ArrayRef<uint8_t> FunctionData = *getData();
   uint64_t EndOfCode = getSize();
@@ -2300,6 +2333,10 @@ Error BinaryFunction::buildCFG(MCPlusBuilder::AllocatorIdTy AllocatorId) {
       BC.errs() << "BOLT-WARNING: failed to post-process indirect branches for "
                 << *this << '\n';
     }
+
+    if (BC.isAArch64())
+      PreserveNops = BC.HasRelocations;
+
     // In relocation mode we want to keep processing the function but avoid
     // optimizing it.
     setSimple(false);
diff --git a/bolt/lib/Passes/PLTCall.cpp b/bolt/lib/Passes/PLTCall.cpp
index 2ed996fadbb99..31c2d92ebc204 100644
--- a/bolt/lib/Passes/PLTCall.cpp
+++ b/bolt/lib/Passes/PLTCall.cpp
@@ -70,8 +70,8 @@ Error PLTCall::runOnFunctions(BinaryContext &BC) {
         const BinaryFunction *CalleeBF = BC.getFunctionForSymbol(CallSymbol);
         if (!CalleeBF || !CalleeBF->isPLTFunction())
           continue;
-        const InstructionListType NewCode = BC.MIB->createIndirectPltCall(
-            *II, CalleeBF->getPLTSymbol(), BC.Ctx.get());
+        const InstructionListType NewCode = BC.MIB->createIndirectPLTCall(
+            std::move(*II), CalleeBF->getPLTSymbol(), BC.Ctx.get());
         II = BB.replaceInstruction(II, NewCode);
         assert(!NewCode.empty() && "PLT Call replacement must be non-empty");
         std::advance(II, NewCode.size() - 1);
diff --git a/bolt/lib/RuntimeLibs/RuntimeLibrary.cpp b/bolt/lib/RuntimeLibs/RuntimeLibrary.cpp
index 336c6768a7f71..8f5719e84ecea 100644
--- a/bolt/lib/RuntimeLibs/RuntimeLibrary.cpp
+++ b/bolt/lib/RuntimeLibs/RuntimeLibrary.cpp
@@ -18,6 +18,7 @@
 #include "llvm/Object/Archive.h"
 #include "llvm/Object/ObjectFile.h"
 #include "llvm/Support/Path.h"
+#include "llvm/Support/Program.h"
 
 #define DEBUG_TYPE "bolt-rtlib"
 
@@ -38,6 +39,23 @@ std::string RuntimeLibrary::getLibPathByToolPath(StringRef ToolPath,
     llvm::sys::path::append(LibPath, "lib" LLVM_LIBDIR_SUFFIX);
   }
   llvm::sys::path::append(LibPath, LibFileName);
+  if (!llvm::sys::fs::exists(LibPath)) {
+    // If it is a symlink, check the directory that the symlink points to.
+    if (llvm::sys::fs::is_symlink_file(ToolPath)) {
+      SmallString<256> RealPath;
+      llvm::sys::fs::real_path(ToolPath, RealPath);
+      if (llvm::ErrorOr<std::string> P =
+              llvm::sys::findProgramByName(RealPath)) {
+        outs() << "BOLT-INFO: library not found: " << LibPath << "\n"
+               << "BOLT-INFO: " << ToolPath << " is a symlink; will look up "
+               << LibFileName
+               << " at the target directory that the symlink points to\n";
+        return getLibPath(*P, LibFileName);
+      }
+    }
+    errs() << "BOLT-ERROR: library not found: " << LibPath << "\n";
+    exit(1);
+  }
   return std::string(LibPath);
 }
 
diff --git a/bolt/lib/Target/AArch64/AArch64MCPlusBuilder.cpp b/bolt/lib/Target/AArch64/AArch64MCPlusBuilder.cpp
index 0b6f21527f0ac..4b21ff719b3ab 100644
--- a/bolt/lib/Target/AArch64/AArch64MCPlusBuilder.cpp
+++ b/bolt/lib/Target/AArch64/AArch64MCPlusBuilder.cpp
@@ -834,6 +834,8 @@ class AArch64MCPlusBuilder : public MCPlusBuilder {
   ///                                   #  of this BB)
   ///   br      x0                      # Indirect jump instruction
   ///
+  /// Return true on successful jump table instruction sequence match, false
+  /// otherwise.
   bool analyzeIndirectBranchFragment(
       const MCInst &Inst,
       DenseMap<const MCInst *, SmallVector<MCInst *, 4>> &UDChain,
@@ -842,6 +844,8 @@ class AArch64MCPlusBuilder : public MCPlusBuilder {
     // Expect AArch64 BR
     assert(Inst.getOpcode() == AArch64::BR && "Unexpected opcode");
 
+    JumpTable = nullptr;
+
     // Match the indirect branch pattern for aarch64
     SmallVector<MCInst *, 4> &UsesRoot = UDChain[&Inst];
     if (UsesRoot.size() == 0 || UsesRoot[0] == nullptr)
@@ -879,8 +883,8 @@ class AArch64MCPlusBuilder : public MCPlusBuilder {
       // Parsed as ADDXrs reg:x8 reg:x8 reg:x12 imm:0
       return false;
     }
-    assert(DefAdd->getOpcode() == AArch64::ADDXrx &&
-           "Failed to match indirect branch!");
+    if (DefAdd->getOpcode() != AArch64::ADDXrx)
+      return false;
 
     // Validate ADD operands
     int64_t OperandExtension = DefAdd->getOperand(3).getImm();
@@ -897,8 +901,8 @@ class AArch64MCPlusBuilder : public MCPlusBuilder {
       //   ldr     w7, [x6]
       //   add     x6, x6, w7, sxtw => no shift amount
       //   br      x6
-      errs() << "BOLT-WARNING: "
-                "Failed to match indirect branch: ShiftVAL != 2 \n";
+      LLVM_DEBUG(dbgs() << "BOLT-DEBUG: "
+                           "failed to match indirect branch: ShiftVAL != 2\n");
       return false;
     }
 
@@ -909,7 +913,7 @@ class AArch64MCPlusBuilder : public MCPlusBuilder {
     else if (ExtendType == AArch64_AM::SXTW)
       ScaleValue = 4LL;
     else
-      llvm_unreachable("Failed to match indirect branch! (fragment 3)");
+      return false;
 
     // Match an ADR to load base address to be used when addressing JT targets
     SmallVector<MCInst *, 4> &UsesAdd = UDChain[DefAdd];
@@ -920,18 +924,15 @@ class AArch64MCPlusBuilder : public MCPlusBuilder {
       return false;
     }
     MCInst *DefBaseAddr = UsesAdd[1];
-    assert(DefBaseAddr->getOpcode() == AArch64::ADR &&
-           "Failed to match indirect branch pattern! (fragment 3)");
+    if (DefBaseAddr->getOpcode() != AArch64::ADR)
+      return false;
 
     PCRelBase = DefBaseAddr;
     // Match LOAD to load the jump table (relative) target
     const MCInst *DefLoad = UsesAdd[2];
-    assert(mayLoad(*DefLoad) &&
-           "Failed to match indirect branch load pattern! (1)");
-    assert((ScaleValue != 1LL || isLDRB(*DefLoad)) &&
-           "Failed to match indirect branch load pattern! (2)");
-    assert((ScaleValue != 2LL || isLDRH(*DefLoad)) &&
-           "Failed to match indirect branch load pattern! (3)");
+    if (!mayLoad(*DefLoad) || (ScaleValue == 1LL && !isLDRB(*DefLoad)) ||
+        (ScaleValue == 2LL && !isLDRH(*DefLoad)))
+      return false;
 
     // Match ADD that calculates the JumpTable Base Address (not the offset)
     SmallVector<MCInst *, 4> &UsesLoad = UDChain[DefLoad];
@@ -941,7 +942,6 @@ class AArch64MCPlusBuilder : public MCPlusBuilder {
         isRegToRegMove(*DefJTBaseAdd, From, To)) {
       // Sometimes base address may have been defined in another basic block
       // (hoisted). Return with no jump table info.
-      JumpTable = nullptr;
       return true;
     }
 
@@ -953,24 +953,27 @@ class AArch64MCPlusBuilder : public MCPlusBuilder {
       //  adr     x12, 0x247b30 <__gettextparse+0x5b0>
       //  add     x13, x12, w13, sxth #2
       //  br      x13
-      errs() << "BOLT-WARNING: Failed to match indirect branch: "
-                "nop/adr instead of adrp/add \n";
+      LLVM_DEBUG(dbgs() << "BOLT-DEBUG: failed to match indirect branch: "
+                           "nop/adr instead of adrp/add\n");
       return false;
     }
 
-    assert(DefJTBaseAdd->getOpcode() == AArch64::ADDXri &&
-           "Failed to match jump table base address pattern! (1)");
+    if (DefJTBaseAdd->getOpcode() != AArch64::ADDXri) {
+      LLVM_DEBUG(dbgs() << "BOLT-DEBUG: failed to match jump table base "
+                           "address pattern! (1)\n");
+      return false;
+    }
 
     if (DefJTBaseAdd->getOperand(2).isImm())
       Offset = DefJTBaseAdd->getOperand(2).getImm();
     SmallVector<MCInst *, 4> &UsesJTBaseAdd = UDChain[DefJTBaseAdd];
     const MCInst *DefJTBasePage = UsesJTBaseAdd[1];
     if (DefJTBasePage == nullptr || isLoadFromStack(*DefJTBasePage)) {
-      JumpTable = nullptr;
       return true;
     }
-    assert(DefJTBasePage->getOpcode() == AArch64::ADRP &&
-           "Failed to match jump table base page pattern! (2)");
+    if (DefJTBasePage->getOpcode() != AArch64::ADRP)
+      return false;
+
     if (DefJTBasePage->getOperand(1).isExpr())
       JumpTable = DefJTBasePage->getOperand(1).getExpr();
     return true;
@@ -1263,7 +1266,7 @@ class AArch64MCPlusBuilder : public MCPlusBuilder {
     return true;
   }
 
-  InstructionListType createIndirectPltCall(const MCInst &DirectCall,
+  InstructionListType createIndirectPLTCall(MCInst &&DirectCall,
                                             const MCSymbol *TargetLocation,
                                             MCContext *Ctx) override {
     const bool IsTailCall = isTailCall(DirectCall);
@@ -1297,8 +1300,7 @@ class AArch64MCPlusBuilder : public MCPlusBuilder {
     MCInst InstCall;
     InstCall.setOpcode(IsTailCall ? AArch64::BR : AArch64::BLR);
     InstCall.addOperand(MCOperand::createReg(AArch64::X17));
-    if (IsTailCall)
-      setTailCall(InstCall);
+    moveAnnotations(std::move(DirectCall), InstCall);
     Code.emplace_back(InstCall);
 
     return Code;
diff --git a/bolt/lib/Target/X86/X86MCPlusBuilder.cpp b/bolt/lib/Target/X86/X86MCPlusBuilder.cpp
index 63086c06d74fd..465533ee71f2b 100644
--- a/bolt/lib/Target/X86/X86MCPlusBuilder.cpp
+++ b/bolt/lib/Target/X86/X86MCPlusBuilder.cpp
@@ -1605,7 +1605,7 @@ class X86MCPlusBuilder : public MCPlusBuilder {
     return true;
   }
 
-  InstructionListType createIndirectPltCall(const MCInst &DirectCall,
+  InstructionListType createIndirectPLTCall(MCInst &&DirectCall,
                                             const MCSymbol *TargetLocation,
                                             MCContext *Ctx) override {
     assert((DirectCall.getOpcode() == X86::CALL64pcrel32 ||
diff --git a/bolt/test/AArch64/data-in-code.s b/bolt/test/AArch64/data-in-code.s
new file mode 100644
index 0000000000000..8d3179a0c3350
--- /dev/null
+++ b/bolt/test/AArch64/data-in-code.s
@@ -0,0 +1,31 @@
+## Check that llvm-bolt prints data embedded in code.
+
+# RUN: llvm-mc -filetype=obj -triple aarch64-unknown-unknown %s -o %t.o
+# RUN: %clang %cflags -fno-PIC -no-pie %t.o -o %t.exe -nostdlib \
+# RUN:    -fuse-ld=lld -Wl,-q
+
+## Check disassembly of BOLT input.
+# RUN: llvm-objdump %t.exe -d | FileCheck %s
+
+# RUN: llvm-bolt %t.exe -o %t.bolt --print-disasm | FileCheck %s
+
+.text
+.balign 4
+
+.global _start
+.type _start, %function
+_start:
+  mov x0, #0x0
+  .word 0x4f82e010
+  ret
+  .byte 0x0, 0xff, 0x42
+# CHECK-LABEL: _start
+# CHECK:        mov x0, #0x0
+# CHECK-NEXT:   .word 0x4f82e010
+# CHECK-NEXT:   ret
+# CHECK-NEXT:   .short 0xff00
+# CHECK-NEXT:   .byte 0x42
+.size _start, .-_start
+
+## Force relocation mode.
+  .reloc 0, R_AARCH64_NONE
diff --git a/bolt/test/AArch64/exceptions-plt.cpp b/bolt/test/AArch64/exceptions-plt.cpp
new file mode 100644
index 0000000000000..33c28406ca8d6
--- /dev/null
+++ b/bolt/test/AArch64/exceptions-plt.cpp
@@ -0,0 +1,23 @@
+// Verify that PLT optimization in BOLT preserves exception-handling info.
+
+// REQUIRES: system-linux
+
+// RUN: %clang %cflags -fpic -shared -xc /dev/null -o %t.so
+// Link against a DSO to ensure PLT entries.
+// RUN: %clangxx %cxxflags -O1 -Wl,-q,-znow %s %t.so -o %t.exe
+// RUN: llvm-bolt %t.exe -o %t.bolt.exe --plt=all --print-only=.*main.* \
+// RUN:   --print-finalized 2>&1 | FileCheck %s
+
+// CHECK-LABEL: Binary Function
+// CHECK:      adrp {{.*}}__cxa_throw
+// CHECK-NEXT: ldr {{.*}}__cxa_throw
+// CHECK-NEXT: blr x17 {{.*}} handler: {{.*}} PLTCall:
+
+int main() {
+  try {
+    throw new int;
+  } catch (...) {
+    return 0;
+  }
+  return 1;
+}
diff --git a/bolt/test/AArch64/jump-table-heuristic-fail.s b/bolt/test/AArch64/jump-table-heuristic-fail.s
new file mode 100644
index 0000000000000..724171ac39925
--- /dev/null
+++ b/bolt/test/AArch64/jump-table-heuristic-fail.s
@@ -0,0 +1,29 @@
+## Verify that BOLT does not crash while encountering instruction sequence that
+## does not perfectly match jump table pattern.
+
+# REQUIRES: system-linux
+
+# RUN: llvm-mc -filetype=obj -triple aarch64-unknown-unknown %s -o %t.o
+# RUN: %clang %cflags --target=aarch64-unknown-linux %t.o -o %t.exe -Wl,-q
+# RUN: llvm-bolt %t.exe -o %t.bolt --print-cfg 2>&1 | FileCheck %s
+
+  .section .text
+  .align 4
+  .globl _start
+  .type  _start, %function
+_start:
+  sub     w0, w0, #0x4a
+## The address loaded into x22 is undefined. However, the instructions that
+## follow ldr, use the x22 address as a regular jump table.
+  ldr     x22, [x29, #0x98]
+  ldrb    w0, [x22, w0, uxtw]
+  adr     x1, #12
+  add     x0, x1, w0, sxtb #2
+  br      x0
+# CHECK: br x0 # UNKNOWN
+.L0:
+  ret
+.size _start, .-_start
+
+## Force relocation mode.
+  .reloc 0, R_AARCH64_NONE
diff --git a/bolt/test/AArch64/pad-before-funcs.s b/bolt/test/AArch64/pad-before-funcs.s
index f3e8a23ddfdda..c1b92063d6fa4 100644
--- a/bolt/test/AArch64/pad-before-funcs.s
+++ b/bolt/test/AArch64/pad-before-funcs.s
@@ -15,7 +15,7 @@
 # RUN: llvm-bolt %t.exe -o %t.bolt.4.4 --pad-funcs-before=_start:4 --pad-funcs=_start:4
 # RUN: llvm-bolt %t.exe -o %t.bolt.4.8 --pad-funcs-before=_start:4 --pad-funcs=_start:8
 
-# RUN: not llvm-bolt %t.exe -o %t.bolt.8 --pad-funcs-before=_start:1 2>&1 | FileCheck --check-prefix=CHECK-BAD-ALIGN %s
+# RUN: not llvm-bolt %t.exe -o %t.bolt.1 --pad-funcs-before=_start:1 2>&1 | FileCheck --check-prefix=CHECK-BAD-ALIGN %s
 
 # CHECK-BAD-ALIGN: user-requested 1 padding bytes before function _start(*2) is not a multiple of the minimum function alignment (4).
 
diff --git a/bolt/test/AArch64/plt-call.test b/bolt/test/AArch64/plt-call.test
index da307d4a6c01e..1fa62c4a36aaf 100644
--- a/bolt/test/AArch64/plt-call.test
+++ b/bolt/test/AArch64/plt-call.test
@@ -1,6 +1,8 @@
 // Verify that PLTCall optimization works.
 
-RUN: %clang %cflags %p/../Inputs/plt-tailcall.c \
+RUN: %clang %cflags -fpic -shared -xc /dev/null -o %t.so
+// Link against a DSO to ensure PLT entries.
+RUN: %clang %cflags %p/../Inputs/plt-tailcall.c %t.so \
 RUN:    -o %t -Wl,-q
 RUN: llvm-bolt %t -o %t.bolt --plt=all --print-plt  --print-only=foo | FileCheck %s
 
diff --git a/bolt/test/AArch64/remove-nops.s b/bolt/test/AArch64/remove-nops.s
new file mode 100644
index 0000000000000..0f02a4b273dda
--- /dev/null
+++ b/bolt/test/AArch64/remove-nops.s
@@ -0,0 +1,28 @@
+## Verify that llvm-bolt removes nop instructions from functions with indirect
+## branches that have defined control flow.
+
+# REQUIRES: system-linux
+
+# RUN: llvm-mc -filetype=obj -triple aarch64-unknown-unknown %s -o %t.o
+# RUN: %clang %cflags --target=aarch64-unknown-linux %t.o -o %t.exe -Wl,-q
+# RUN: llvm-bolt %t.exe -o %t.bolt --print-normalized 2>&1 | FileCheck %s
+# RUN: llvm-objdump -d --disassemble-symbols=_start %t.bolt \
+# RUN:   | FileCheck %s --check-prefix=CHECK-OBJDUMP
+
+# CHECK-OBJDUMP-LABEL: _start
+# CHECK-OBJDUMP-NOT: nop
+
+  .section .text
+  .align 4
+  .globl _start
+  .type  _start, %function
+_start:
+# CHECK-LABEL: Binary Function "_start"
+  nop
+# CHECK-NOT: nop
+  br      x0
+# CHECK: br x0 # TAILCALL
+.size _start, .-_start
+
+## Force relocation mode.
+  .reloc 0, R_AARCH64_NONE
diff --git a/bolt/test/AArch64/test-indirect-branch.s b/bolt/test/AArch64/test-indirect-branch.s
index 168e50c8f47f5..1e16e76b11530 100644
--- a/bolt/test/AArch64/test-indirect-branch.s
+++ b/bolt/test/AArch64/test-indirect-branch.s
@@ -3,10 +3,11 @@
 
 // clang-format off
 
-// REQUIRES: system-linux
+// REQUIRES: system-linux, asserts
+
 // RUN: llvm-mc -filetype=obj -triple aarch64-unknown-unknown %s -o %t.o
 // RUN: %clang %cflags --target=aarch64-unknown-linux %t.o -o %t.exe -Wl,-q
-// RUN: llvm-bolt %t.exe -o %t.bolt --print-cfg --strict\
+// RUN: llvm-bolt %t.exe -o %t.bolt --print-cfg --strict --debug-only=mcplus \
 // RUN:  -v=1 2>&1 | FileCheck %s
 
 // Pattern 1: there is no shift amount after the 'add' instruction.
@@ -39,7 +40,7 @@ _start:
 // svc #0
 
 // Pattern 1
-// CHECK: BOLT-WARNING: Failed to match indirect branch: ShiftVAL != 2
+// CHECK: BOLT-DEBUG: failed to match indirect branch: ShiftVAL != 2
   .globl test1
   .type  test1, %function
 test1:
@@ -57,7 +58,7 @@ test1_2:
    ret
 
 // Pattern 2
-// CHECK: BOLT-WARNING: Failed to match indirect branch: nop/adr instead of adrp/add
+// CHECK: BOLT-DEBUG: failed to match indirect branch: nop/adr instead of adrp/add
   .globl test2
   .type  test2, %function
 test2:
diff --git a/bolt/test/X86/callcont-fallthru.s b/bolt/test/X86/callcont-fallthru.s
index 31a7910d7fa3f..d76f869c971fd 100644
--- a/bolt/test/X86/callcont-fallthru.s
+++ b/bolt/test/X86/callcont-fallthru.s
@@ -1,7 +1,9 @@
 ## Ensures that a call continuation fallthrough count is set when using
 ## pre-aggregated perf data.
 
-# RUN: %clangxx %cxxflags %s -o %t -Wl,-q -nostdlib
+# RUN: %clang %cflags -fpic -shared -xc /dev/null -o %t.so
+## Link against a DSO to ensure PLT entries.
+# RUN: %clangxx %cxxflags %s %t.so -o %t -Wl,-q -nostdlib
 # RUN: link_fdata %s %t %t.pa1 PREAGG
 # RUN: link_fdata %s %t %t.pa2 PREAGG2
 # RUN: link_fdata %s %t %t.pa3 PREAGG3
diff --git a/bolt/test/X86/cfi-instrs-reordered.s b/bolt/test/X86/cfi-instrs-reordered.s
index c325aaf1ad8b1..5173fa6c3c7d0 100644
--- a/bolt/test/X86/cfi-instrs-reordered.s
+++ b/bolt/test/X86/cfi-instrs-reordered.s
@@ -3,7 +3,9 @@
 
 # RUN: llvm-mc -filetype=obj -triple x86_64-unknown-unknown %s -o %t.o
 # RUN: llvm-strip --strip-unneeded %t.o
-# RUN: %clangxx %cflags %t.o -o %t.exe
+# RUN: %clang %cflags -fpic -shared -xc /dev/null -o %t.so
+## Link against a DSO to ensure PLT entries.
+# RUN: %clangxx %cflags %t.o %t.so -o %t.exe
 # RUN: llvm-bolt %t.exe -o %t --reorder-blocks=cache --print-after-lowering \
 # RUN:   --print-only=_Z10SolveCubicddddPiPd 2>&1 | FileCheck %s
 #
diff --git a/bolt/test/X86/dynamic-relocs-on-entry.s b/bolt/test/X86/dynamic-relocs-on-entry.s
index 2a29a43c4939a..4ec8ba4ad4460 100644
--- a/bolt/test/X86/dynamic-relocs-on-entry.s
+++ b/bolt/test/X86/dynamic-relocs-on-entry.s
@@ -4,12 +4,12 @@
 
 # RUN: %clang %cflags -fPIC -pie %s -o %t.exe -nostdlib -Wl,-q
 # RUN: llvm-bolt %t.exe -o %t.bolt > %t.out.txt
-# RUN: readelf -r %t.bolt >> %t.out.txt
+# RUN: llvm-readelf -r %t.bolt >> %t.out.txt
 # RUN: llvm-objdump --disassemble-symbols=chain %t.bolt >> %t.out.txt
 # RUN: FileCheck %s --input-file=%t.out.txt
 
 ## Check if the new address in `chain` is correctly updated by BOLT
-# CHECK: Relocation section '.rela.dyn' at offset 0x{{.*}} contains 1 entry:
+# CHECK: Relocation section '.rela.dyn' at offset 0x{{.*}} contains 1 entries:
 # CHECK: {{.*}} R_X86_64_RELATIVE [[#%x,ADDR:]]
 # CHECK: [[#ADDR]]: c3 retq
 	.text
@@ -29,4 +29,4 @@ _start:
 
 	.data
 .Lfoo:
-	.quad Label
\ No newline at end of file
+	.quad Label
diff --git a/bolt/test/X86/plt-call.test b/bolt/test/X86/plt-call.test
index e6ae86c179d27..aeee3024ac170 100644
--- a/bolt/test/X86/plt-call.test
+++ b/bolt/test/X86/plt-call.test
@@ -1,6 +1,8 @@
 // Verify that PLTCall optimization works.
 
-RUN: %clang %cflags %p/../Inputs/plt-tailcall.c \
+RUN: %clang %cflags -fpic -shared -xc /dev/null -o %t.so
+// Link against a DSO to ensure PLT entries.
+RUN: %clang %cflags %p/../Inputs/plt-tailcall.c %t.so \
 RUN:    -o %t -Wl,-q
 RUN: llvm-bolt %t -o %t.bolt --plt=all --print-plt  --print-only=foo | FileCheck %s
 
diff --git a/bolt/test/runtime/exceptions-plt.cpp b/bolt/test/runtime/exceptions-plt.cpp
new file mode 100644
index 0000000000000..3d8e7a5133e2c
--- /dev/null
+++ b/bolt/test/runtime/exceptions-plt.cpp
@@ -0,0 +1,18 @@
+// Verify that PLT optimization in BOLT preserves exception-handling info.
+
+// REQUIRES: system-linux
+
+// RUN: %clang %cflags -fpic -shared -xc /dev/null -o %t.so
+// Link against a DSO to ensure PLT entries.
+// RUN: %clangxx %cxxflags -O1 -Wl,-q,-znow %s %t.so -o %t.exe
+// RUN: llvm-bolt %t.exe -o %t.bolt.exe --plt=all
+// RUN: %t.bolt.exe
+
+int main() {
+  try {
+    throw new int;
+  } catch (...) {
+    return 0;
+  }
+  return 1;
+}
diff --git a/bolt/test/runtime/plt-lld.test b/bolt/test/runtime/plt-lld.test
index b505a191f90ab..3432e18bf4daf 100644
--- a/bolt/test/runtime/plt-lld.test
+++ b/bolt/test/runtime/plt-lld.test
@@ -1,14 +1,15 @@
 // This test checks that the pointers to PLT are properly updated.
-// The test is using lld linker.
+// The test uses lld and links against a DSO to ensure PLT entries.
+RUN: %clang %cflags -fpic -shared -xc /dev/null -o %t.so
 
 // Non-PIE:
-RUN: %clang %cflags -no-pie %p/../Inputs/plt.c -fuse-ld=lld \
+RUN: %clang %cflags -no-pie %p/../Inputs/plt.c %t.so -fuse-ld=lld \
 RUN:    -o %t.lld.exe -Wl,-q
 RUN: llvm-bolt %t.lld.exe -o %t.lld.bolt.exe --use-old-text=0 --lite=0
 RUN: %t.lld.bolt.exe | FileCheck %s
 
 // PIE:
-RUN: %clang %cflags -fPIC -pie %p/../Inputs/plt.c -fuse-ld=lld \
+RUN: %clang %cflags -fPIC -pie %p/../Inputs/plt.c %t.so -fuse-ld=lld \
 RUN:    -o %t.lld.pie.exe -Wl,-q
 RUN: llvm-bolt %t.lld.pie.exe -o %t.lld.bolt.pie.exe --use-old-text=0 --lite=0
 RUN: %t.lld.bolt.pie.exe | FileCheck %s
diff --git a/clang-tools-extra/clang-doc/HTMLGenerator.cpp b/clang-tools-extra/clang-doc/HTMLGenerator.cpp
index e3532559a32fc..18a0de826630c 100644
--- a/clang-tools-extra/clang-doc/HTMLGenerator.cpp
+++ b/clang-tools-extra/clang-doc/HTMLGenerator.cpp
@@ -494,18 +494,31 @@ genReferencesBlock(const std::vector<Reference> &References,
 static std::unique_ptr<TagNode>
 writeFileDefinition(const Location &L,
                     std::optional<StringRef> RepositoryUrl = std::nullopt) {
-  if (!L.IsFileInRootDir || !RepositoryUrl)
+  if (!L.IsFileInRootDir && !RepositoryUrl)
     return std::make_unique<TagNode>(
         HTMLTag::TAG_P, "Defined at line " + std::to_string(L.LineNumber) +
                             " of file " + L.Filename);
   SmallString<128> FileURL(*RepositoryUrl);
-  llvm::sys::path::append(FileURL, llvm::sys::path::Style::posix, L.Filename);
+  llvm::sys::path::append(
+      FileURL, llvm::sys::path::Style::posix,
+      // If we're on Windows, the file name will be in the wrong format, and
+      // append won't convert the full path being appended to the correct
+      // format, so we need to do that here.
+      llvm::sys::path::convert_to_slash(
+          L.Filename,
+          // The style here is the current style of the path, not the one we're
+          // targeting. If the string is already in the posix style, it will do
+          // nothing.
+          llvm::sys::path::Style::windows));
   auto Node = std::make_unique<TagNode>(HTMLTag::TAG_P);
   Node->Children.emplace_back(std::make_unique<TextNode>("Defined at line "));
   auto LocNumberNode =
       std::make_unique<TagNode>(HTMLTag::TAG_A, std::to_string(L.LineNumber));
   // The links to a specific line in the source code use the github /
   // googlesource notation so it won't work for all hosting pages.
+  // FIXME: we probably should have a configuration setting for line number
+  // rendering in the HTML. For example, GitHub uses #L22, while googlesource
+  // uses #22 for line numbers.
   LocNumberNode->Attributes.emplace_back(
       "href", (FileURL + "#" + std::to_string(L.LineNumber)).str());
   Node->Children.emplace_back(std::move(LocNumberNode));
@@ -964,7 +977,7 @@ HTMLGenerator::generateDocs(StringRef RootDir,
   for (const auto &Group : FileToInfos) {
     std::error_code FileErr;
     llvm::raw_fd_ostream InfoOS(Group.getKey(), FileErr,
-                                llvm::sys::fs::OF_None);
+                                llvm::sys::fs::OF_Text);
     if (FileErr) {
       return llvm::createStringError(FileErr, "Error opening file '%s'",
                                      Group.getKey().str().c_str());
@@ -1047,7 +1060,7 @@ static llvm::Error serializeIndex(ClangDocContext &CDCtx) {
   llvm::SmallString<128> FilePath;
   llvm::sys::path::native(CDCtx.OutDirectory, FilePath);
   llvm::sys::path::append(FilePath, "index_json.js");
-  llvm::raw_fd_ostream OS(FilePath, FileErr, llvm::sys::fs::OF_None);
+  llvm::raw_fd_ostream OS(FilePath, FileErr, llvm::sys::fs::OF_Text);
   if (FileErr != OK) {
     return llvm::createStringError(llvm::inconvertibleErrorCode(),
                                    "error creating index file: " +
@@ -1108,7 +1121,7 @@ static llvm::Error genIndex(const ClangDocContext &CDCtx) {
   llvm::SmallString<128> IndexPath;
   llvm::sys::path::native(CDCtx.OutDirectory, IndexPath);
   llvm::sys::path::append(IndexPath, "index.html");
-  llvm::raw_fd_ostream IndexOS(IndexPath, FileErr, llvm::sys::fs::OF_None);
+  llvm::raw_fd_ostream IndexOS(IndexPath, FileErr, llvm::sys::fs::OF_Text);
   if (FileErr != OK) {
     return llvm::createStringError(llvm::inconvertibleErrorCode(),
                                    "error creating main index: " +
diff --git a/clang-tools-extra/clang-doc/MDGenerator.cpp b/clang-tools-extra/clang-doc/MDGenerator.cpp
index 28b645cf021dd..7bef2c0db04d8 100644
--- a/clang-tools-extra/clang-doc/MDGenerator.cpp
+++ b/clang-tools-extra/clang-doc/MDGenerator.cpp
@@ -300,7 +300,7 @@ static llvm::Error serializeIndex(ClangDocContext &CDCtx) {
   llvm::SmallString<128> FilePath;
   llvm::sys::path::native(CDCtx.OutDirectory, FilePath);
   llvm::sys::path::append(FilePath, "all_files.md");
-  llvm::raw_fd_ostream OS(FilePath, FileErr, llvm::sys::fs::OF_None);
+  llvm::raw_fd_ostream OS(FilePath, FileErr, llvm::sys::fs::OF_Text);
   if (FileErr)
     return llvm::createStringError(llvm::inconvertibleErrorCode(),
                                    "error creating index file: " +
@@ -323,7 +323,7 @@ static llvm::Error genIndex(ClangDocContext &CDCtx) {
   llvm::SmallString<128> FilePath;
   llvm::sys::path::native(CDCtx.OutDirectory, FilePath);
   llvm::sys::path::append(FilePath, "index.md");
-  llvm::raw_fd_ostream OS(FilePath, FileErr, llvm::sys::fs::OF_None);
+  llvm::raw_fd_ostream OS(FilePath, FileErr, llvm::sys::fs::OF_Text);
   if (FileErr)
     return llvm::createStringError(llvm::inconvertibleErrorCode(),
                                    "error creating index file: " +
@@ -407,7 +407,7 @@ MDGenerator::generateDocs(StringRef RootDir,
   for (const auto &Group : FileToInfos) {
     std::error_code FileErr;
     llvm::raw_fd_ostream InfoOS(Group.getKey(), FileErr,
-                                llvm::sys::fs::OF_None);
+                                llvm::sys::fs::OF_Text);
     if (FileErr) {
       return llvm::createStringError(FileErr, "Error opening file '%s'",
                                      Group.getKey().str().c_str());
diff --git a/clang-tools-extra/clang-doc/YAMLGenerator.cpp b/clang-tools-extra/clang-doc/YAMLGenerator.cpp
index b612380a3cfa7..ffabd2fd82229 100644
--- a/clang-tools-extra/clang-doc/YAMLGenerator.cpp
+++ b/clang-tools-extra/clang-doc/YAMLGenerator.cpp
@@ -347,7 +347,7 @@ YAMLGenerator::generateDocs(StringRef RootDir,
     }
 
     std::error_code FileErr;
-    llvm::raw_fd_ostream InfoOS(Path, FileErr, llvm::sys::fs::OF_None);
+    llvm::raw_fd_ostream InfoOS(Path, FileErr, llvm::sys::fs::OF_Text);
     if (FileErr) {
       return llvm::createStringError(FileErr, "Error opening file '%s'",
                                      Path.c_str());
diff --git a/clang-tools-extra/clang-include-fixer/FuzzySymbolIndex.cpp b/clang-tools-extra/clang-include-fixer/FuzzySymbolIndex.cpp
index 91e2a9396a559..98987ad380196 100644
--- a/clang-tools-extra/clang-include-fixer/FuzzySymbolIndex.cpp
+++ b/clang-tools-extra/clang-include-fixer/FuzzySymbolIndex.cpp
@@ -131,7 +131,7 @@ FuzzySymbolIndex::queryRegexp(const std::vector<std::string> &Tokens) {
 
 llvm::Expected<std::unique_ptr<FuzzySymbolIndex>>
 FuzzySymbolIndex::createFromYAML(StringRef FilePath) {
-  auto Buffer = llvm::MemoryBuffer::getFile(FilePath);
+  auto Buffer = llvm::MemoryBuffer::getFile(FilePath, /*IsText=*/true);
   if (!Buffer)
     return llvm::errorCodeToError(Buffer.getError());
   return std::make_unique<MemSymbolIndex>(
diff --git a/clang-tools-extra/clang-include-fixer/YamlSymbolIndex.cpp b/clang-tools-extra/clang-include-fixer/YamlSymbolIndex.cpp
index 4271d9aa4e677..7f570ee917197 100644
--- a/clang-tools-extra/clang-include-fixer/YamlSymbolIndex.cpp
+++ b/clang-tools-extra/clang-include-fixer/YamlSymbolIndex.cpp
@@ -22,7 +22,7 @@ namespace include_fixer {
 
 llvm::ErrorOr<std::unique_ptr<YamlSymbolIndex>>
 YamlSymbolIndex::createFromFile(llvm::StringRef FilePath) {
-  auto Buffer = llvm::MemoryBuffer::getFile(FilePath);
+  auto Buffer = llvm::MemoryBuffer::getFile(FilePath, /*IsText=*/true);
   if (!Buffer)
     return Buffer.getError();
 
diff --git a/clang-tools-extra/clang-include-fixer/find-all-symbols/tool/FindAllSymbolsMain.cpp b/clang-tools-extra/clang-include-fixer/find-all-symbols/tool/FindAllSymbolsMain.cpp
index 298b02e77cb0a..6bac717bbb013 100644
--- a/clang-tools-extra/clang-include-fixer/find-all-symbols/tool/FindAllSymbolsMain.cpp
+++ b/clang-tools-extra/clang-include-fixer/find-all-symbols/tool/FindAllSymbolsMain.cpp
@@ -95,7 +95,7 @@ bool Merge(llvm::StringRef MergeDir, llvm::StringRef OutputFile) {
       // Parse YAML files in parallel.
       Pool.async(
           [&AddSymbols](std::string Path) {
-            auto Buffer = llvm::MemoryBuffer::getFile(Path);
+            auto Buffer = llvm::MemoryBuffer::getFile(Path, /*IsText=*/true);
             if (!Buffer) {
               llvm::errs() << "Can't open " << Path << "\n";
               return;
@@ -114,7 +114,7 @@ bool Merge(llvm::StringRef MergeDir, llvm::StringRef OutputFile) {
     }
   }
 
-  llvm::raw_fd_ostream OS(OutputFile, EC, llvm::sys::fs::OF_None);
+  llvm::raw_fd_ostream OS(OutputFile, EC, llvm::sys::fs::OF_Text);
   if (EC) {
     llvm::errs() << "Can't open '" << OutputFile << "': " << EC.message()
                  << '\n';
diff --git a/clang-tools-extra/clang-include-fixer/tool/ClangIncludeFixer.cpp b/clang-tools-extra/clang-include-fixer/tool/ClangIncludeFixer.cpp
index 3a11a22def194..6e51f25a66407 100644
--- a/clang-tools-extra/clang-include-fixer/tool/ClangIncludeFixer.cpp
+++ b/clang-tools-extra/clang-include-fixer/tool/ClangIncludeFixer.cpp
@@ -415,7 +415,7 @@ int includeFixerMain(int argc, const char **argv) {
       llvm::errs() << llvm::toString(InsertStyle.takeError()) << "\n";
       return 1;
     }
-    auto Buffer = llvm::MemoryBuffer::getFile(FilePath);
+    auto Buffer = llvm::MemoryBuffer::getFile(FilePath, /*IsText=*/true);
     if (!Buffer) {
       errs() << "Couldn't open file: " + FilePath.str() + ": "
              << Buffer.getError().message() + "\n";
diff --git a/clang-tools-extra/clang-tidy/bugprone/StringConstructorCheck.cpp b/clang-tools-extra/clang-tidy/bugprone/StringConstructorCheck.cpp
index 8ae4351ac2830..d1902b658061b 100644
--- a/clang-tools-extra/clang-tidy/bugprone/StringConstructorCheck.cpp
+++ b/clang-tools-extra/clang-tidy/bugprone/StringConstructorCheck.cpp
@@ -82,7 +82,7 @@ void StringConstructorCheck::registerMatchers(MatchFinder *Finder) {
   Finder->addMatcher(
       cxxConstructExpr(
           hasDeclaration(cxxMethodDecl(hasName("basic_string"))),
-          hasArgument(0, hasType(qualType(isInteger()))),
+          argumentCountIs(2), hasArgument(0, hasType(qualType(isInteger()))),
           hasArgument(1, hasType(qualType(isInteger()))),
           anyOf(
               // Detect the expression: string('x', 40);
@@ -102,7 +102,7 @@ void StringConstructorCheck::registerMatchers(MatchFinder *Finder) {
       cxxConstructExpr(
           hasDeclaration(cxxConstructorDecl(ofClass(
               cxxRecordDecl(hasAnyName(removeNamespaces(StringNames)))))),
-          hasArgument(0, hasType(CharPtrType)),
+          argumentCountIs(2), hasArgument(0, hasType(CharPtrType)),
           hasArgument(1, hasType(isInteger())),
           anyOf(
               // Detect the expression: string("...", 0);
@@ -114,7 +114,34 @@ void StringConstructorCheck::registerMatchers(MatchFinder *Finder) {
               // Detect the expression: string("lit", 5)
               allOf(hasArgument(0, ConstStrLiteral.bind("literal-with-length")),
                     hasArgument(1, ignoringParenImpCasts(
-                                       integerLiteral().bind("int"))))))
+                                       integerLiteral().bind("length"))))))
+          .bind("constructor"),
+      this);
+
+  // Check the literal string constructor with char pointer, start position and
+  // length parameters. [i.e. string (const char* s, size_t pos, size_t count);]
+  Finder->addMatcher(
+      cxxConstructExpr(
+          hasDeclaration(cxxConstructorDecl(ofClass(
+              cxxRecordDecl(hasAnyName(removeNamespaces(StringNames)))))),
+          argumentCountIs(3), hasArgument(0, hasType(CharPtrType)),
+          hasArgument(1, hasType(qualType(isInteger()))),
+          hasArgument(2, hasType(qualType(isInteger()))),
+          anyOf(
+              // Detect the expression: string("...", 1, 0);
+              hasArgument(2, ZeroExpr.bind("empty-string")),
+              // Detect the expression: string("...", -4, 1);
+              hasArgument(1, NegativeExpr.bind("negative-pos")),
+              // Detect the expression: string("...", 0, -4);
+              hasArgument(2, NegativeExpr.bind("negative-length")),
+              // Detect the expression: string("lit", 0, 0x1234567);
+              hasArgument(2, LargeLengthExpr.bind("large-length")),
+              // Detect the expression: string("lit", 1, 5)
+              allOf(hasArgument(0, ConstStrLiteral.bind("literal-with-length")),
+                    hasArgument(
+                        1, ignoringParenImpCasts(integerLiteral().bind("pos"))),
+                    hasArgument(2, ignoringParenImpCasts(
+                                       integerLiteral().bind("length"))))))
           .bind("constructor"),
       this);
 
@@ -155,14 +182,27 @@ void StringConstructorCheck::check(const MatchFinder::MatchResult &Result) {
     diag(Loc, "constructor creating an empty string");
   } else if (Result.Nodes.getNodeAs<Expr>("negative-length")) {
     diag(Loc, "negative value used as length parameter");
+  } else if (Result.Nodes.getNodeAs<Expr>("negative-pos")) {
+    diag(Loc, "negative value used as position of the "
+              "first character parameter");
   } else if (Result.Nodes.getNodeAs<Expr>("large-length")) {
     if (WarnOnLargeLength)
       diag(Loc, "suspicious large length parameter");
   } else if (Result.Nodes.getNodeAs<Expr>("literal-with-length")) {
     const auto *Str = Result.Nodes.getNodeAs<StringLiteral>("str");
-    const auto *Lit = Result.Nodes.getNodeAs<IntegerLiteral>("int");
-    if (Lit->getValue().ugt(Str->getLength())) {
+    const auto *Length = Result.Nodes.getNodeAs<IntegerLiteral>("length");
+    if (Length->getValue().ugt(Str->getLength())) {
       diag(Loc, "length is bigger than string literal size");
+      return;
+    }
+    if (const auto *Pos = Result.Nodes.getNodeAs<IntegerLiteral>("pos")) {
+      if (Pos->getValue().uge(Str->getLength())) {
+        diag(Loc, "position of the first character parameter is bigger than "
+                  "string literal character range");
+      } else if (Length->getValue().ugt(
+                     (Str->getLength() - Pos->getValue()).getZExtValue())) {
+        diag(Loc, "length is bigger than remaining string literal size");
+      }
     }
   } else if (const auto *Ptr = Result.Nodes.getNodeAs<Expr>("from-ptr")) {
     Expr::EvalResult ConstPtr;
diff --git a/clang-tools-extra/clang-tidy/bugprone/UnsafeFunctionsCheck.cpp b/clang-tools-extra/clang-tidy/bugprone/UnsafeFunctionsCheck.cpp
index 604a7cac0e490..a45949314a4ca 100644
--- a/clang-tools-extra/clang-tidy/bugprone/UnsafeFunctionsCheck.cpp
+++ b/clang-tools-extra/clang-tidy/bugprone/UnsafeFunctionsCheck.cpp
@@ -256,13 +256,32 @@ void UnsafeFunctionsCheck::registerMatchers(MatchFinder *Finder) {
                                           .bind(CustomFunctionNamesId)))
                            .bind(DeclRefId),
                        this);
+    // C++ member calls do not contain a DeclRefExpr to the function decl.
+    // Instead, they contain a MemberExpr that refers to the decl.
+    Finder->addMatcher(memberExpr(member(functionDecl(CustomFunctionsMatcher)
+                                             .bind(CustomFunctionNamesId)))
+                           .bind(DeclRefId),
+                       this);
   }
 }
 
 void UnsafeFunctionsCheck::check(const MatchFinder::MatchResult &Result) {
-  const auto *DeclRef = Result.Nodes.getNodeAs<DeclRefExpr>(DeclRefId);
-  const auto *FuncDecl = cast<FunctionDecl>(DeclRef->getDecl());
-  assert(DeclRef && FuncDecl && "No valid matched node in check()");
+  const Expr *SourceExpr;
+  const FunctionDecl *FuncDecl;
+
+  if (const auto *DeclRef = Result.Nodes.getNodeAs<DeclRefExpr>(DeclRefId)) {
+    SourceExpr = DeclRef;
+    FuncDecl = cast<FunctionDecl>(DeclRef->getDecl());
+  } else if (const auto *Member =
+                 Result.Nodes.getNodeAs<MemberExpr>(DeclRefId)) {
+    SourceExpr = Member;
+    FuncDecl = cast<FunctionDecl>(Member->getMemberDecl());
+  } else {
+    llvm_unreachable("No valid matched node in check()");
+    return;
+  }
+
+  assert(SourceExpr && FuncDecl && "No valid matched node in check()");
 
   // Only one of these are matched at a time.
   const auto *AnnexK = Result.Nodes.getNodeAs<FunctionDecl>(
@@ -286,14 +305,15 @@ void UnsafeFunctionsCheck::check(const MatchFinder::MatchResult &Result) {
             Entry.Reason.empty() ? "is marked as unsafe" : Entry.Reason.c_str();
 
         if (Entry.Replacement.empty()) {
-          diag(DeclRef->getExprLoc(), "function %0 %1; it should not be used")
+          diag(SourceExpr->getExprLoc(),
+               "function %0 %1; it should not be used")
               << FuncDecl << Reason << Entry.Replacement
-              << DeclRef->getSourceRange();
+              << SourceExpr->getSourceRange();
         } else {
-          diag(DeclRef->getExprLoc(),
+          diag(SourceExpr->getExprLoc(),
                "function %0 %1; '%2' should be used instead")
               << FuncDecl << Reason << Entry.Replacement
-              << DeclRef->getSourceRange();
+              << SourceExpr->getSourceRange();
         }
 
         return;
@@ -323,9 +343,9 @@ void UnsafeFunctionsCheck::check(const MatchFinder::MatchResult &Result) {
   if (!ReplacementFunctionName)
     return;
 
-  diag(DeclRef->getExprLoc(), "function %0 %1; '%2' should be used instead")
+  diag(SourceExpr->getExprLoc(), "function %0 %1; '%2' should be used instead")
       << FuncDecl << getRationaleFor(FunctionName)
-      << ReplacementFunctionName.value() << DeclRef->getSourceRange();
+      << ReplacementFunctionName.value() << SourceExpr->getSourceRange();
 }
 
 void UnsafeFunctionsCheck::registerPPCallbacks(
diff --git a/clang-tools-extra/clang-tidy/bugprone/UnsafeFunctionsCheck.h b/clang-tools-extra/clang-tidy/bugprone/UnsafeFunctionsCheck.h
index 63058c326ef29..9b2ec990be01f 100644
--- a/clang-tools-extra/clang-tidy/bugprone/UnsafeFunctionsCheck.h
+++ b/clang-tools-extra/clang-tidy/bugprone/UnsafeFunctionsCheck.h
@@ -43,7 +43,7 @@ class UnsafeFunctionsCheck : public ClangTidyCheck {
 private:
   const std::vector<CheckedFunction> CustomFunctions;
 
-  // If true, the default set of functions are reported.
+  /// If true, the default set of functions are reported.
   const bool ReportDefaultFunctions;
   /// If true, additional functions from widely used API-s (such as POSIX) are
   /// added to the list of reported functions.
diff --git a/clang-tools-extra/clang-tidy/misc/RedundantExpressionCheck.cpp b/clang-tools-extra/clang-tidy/misc/RedundantExpressionCheck.cpp
index fc35bc22c52e0..c249244b1a1b2 100644
--- a/clang-tools-extra/clang-tidy/misc/RedundantExpressionCheck.cpp
+++ b/clang-tools-extra/clang-tidy/misc/RedundantExpressionCheck.cpp
@@ -139,10 +139,8 @@ static bool areEquivalentExpr(const Expr *Left, const Expr *Right) {
     return cast<BinaryOperator>(Left)->getOpcode() ==
            cast<BinaryOperator>(Right)->getOpcode();
   case Stmt::UnaryExprOrTypeTraitExprClass:
-    const auto *LeftUnaryExpr =
-        cast<UnaryExprOrTypeTraitExpr>(Left);
-    const auto *RightUnaryExpr =
-        cast<UnaryExprOrTypeTraitExpr>(Right);
+    const auto *LeftUnaryExpr = cast<UnaryExprOrTypeTraitExpr>(Left);
+    const auto *RightUnaryExpr = cast<UnaryExprOrTypeTraitExpr>(Right);
     if (LeftUnaryExpr->isArgumentType() && RightUnaryExpr->isArgumentType())
       return LeftUnaryExpr->getKind() == RightUnaryExpr->getKind() &&
              LeftUnaryExpr->getArgumentType() ==
@@ -699,7 +697,8 @@ static bool retrieveRelationalIntegerConstantExpr(
 
     Symbol = OverloadedOperatorExpr->getArg(IntegerConstantIsFirstArg ? 1 : 0);
     OperandExpr = OverloadedOperatorExpr;
-    Opcode = BinaryOperator::getOverloadedOpcode(OverloadedOperatorExpr->getOperator());
+    Opcode = BinaryOperator::getOverloadedOpcode(
+        OverloadedOperatorExpr->getOperator());
 
     if (!retrieveIntegerConstantExpr(Result, Id, Value, ConstExpr))
       return false;
@@ -728,7 +727,8 @@ static bool retrieveRelationalIntegerConstantExpr(
 }
 
 // Checks for expressions like (X == 4) && (Y != 9)
-static bool areSidesBinaryConstExpressions(const BinaryOperator *&BinOp, const ASTContext *AstCtx) {
+static bool areSidesBinaryConstExpressions(const BinaryOperator *&BinOp,
+                                           const ASTContext *AstCtx) {
   const auto *LhsBinOp = dyn_cast<BinaryOperator>(BinOp->getLHS());
   const auto *RhsBinOp = dyn_cast<BinaryOperator>(BinOp->getRHS());
 
@@ -747,6 +747,28 @@ static bool areSidesBinaryConstExpressions(const BinaryOperator *&BinOp, const A
   return false;
 }
 
+static bool areSidesBinaryConstExpressionsOrDefinesOrIntegerConstant(
+    const BinaryOperator *&BinOp, const ASTContext *AstCtx) {
+  if (areSidesBinaryConstExpressions(BinOp, AstCtx))
+    return true;
+
+  const Expr *Lhs = BinOp->getLHS();
+  const Expr *Rhs = BinOp->getRHS();
+
+  if (!Lhs || !Rhs)
+    return false;
+
+  auto IsDefineExpr = [AstCtx](const Expr *E) {
+    const SourceRange Lsr = E->getSourceRange();
+    if (!Lsr.getBegin().isMacroID() || E->isValueDependent() ||
+        !E->isIntegerConstantExpr(*AstCtx))
+      return false;
+    return true;
+  };
+
+  return IsDefineExpr(Lhs) || IsDefineExpr(Rhs);
+}
+
 // Retrieves integer constant subexpressions from binary operator expressions
 // that have two equivalent sides.
 // E.g.: from (X == 5) && (X == 5) retrieves 5 and 5.
@@ -785,7 +807,7 @@ static bool retrieveConstExprFromBothSides(const BinaryOperator *&BinOp,
 }
 
 static bool isSameRawIdentifierToken(const Token &T1, const Token &T2,
-                        const SourceManager &SM) {
+                                     const SourceManager &SM) {
   if (T1.getKind() != T2.getKind())
     return false;
   if (T1.isNot(tok::raw_identifier))
@@ -808,8 +830,8 @@ static bool areExprsFromDifferentMacros(const Expr *LhsExpr,
                                         const ASTContext *AstCtx) {
   if (!LhsExpr || !RhsExpr)
     return false;
-  SourceRange Lsr = LhsExpr->getSourceRange();
-  SourceRange Rsr = RhsExpr->getSourceRange();
+  const SourceRange Lsr = LhsExpr->getSourceRange();
+  const SourceRange Rsr = RhsExpr->getSourceRange();
   if (!Lsr.getBegin().isMacroID() || !Rsr.getBegin().isMacroID())
     return false;
 
@@ -847,11 +869,83 @@ static bool areExprsMacroAndNonMacro(const Expr *&LhsExpr,
   if (!LhsExpr || !RhsExpr)
     return false;
 
-  SourceLocation LhsLoc = LhsExpr->getExprLoc();
-  SourceLocation RhsLoc = RhsExpr->getExprLoc();
+  const SourceLocation LhsLoc = LhsExpr->getExprLoc();
+  const SourceLocation RhsLoc = RhsExpr->getExprLoc();
 
   return LhsLoc.isMacroID() != RhsLoc.isMacroID();
 }
+
+static bool areStringsSameIgnoreSpaces(const llvm::StringRef Left,
+                                       const llvm::StringRef Right) {
+  if (Left == Right)
+    return true;
+
+  // Do running comparison ignoring spaces
+  llvm::StringRef L = Left.trim();
+  llvm::StringRef R = Right.trim();
+  while (!L.empty() && !R.empty()) {
+    L = L.ltrim();
+    R = R.ltrim();
+    if (L.empty() && R.empty())
+      return true;
+    // If symbol compared are different ==> strings are not the same
+    if (L.front() != R.front())
+      return false;
+    L = L.drop_front();
+    R = R.drop_front();
+  }
+  return L.empty() && R.empty();
+}
+
+static bool areExprsSameMacroOrLiteral(const BinaryOperator *BinOp,
+                                       const ASTContext *Context) {
+
+  if (!BinOp)
+    return false;
+
+  const Expr *Lhs = BinOp->getLHS();
+  const Expr *Rhs = BinOp->getRHS();
+  const SourceManager &SM = Context->getSourceManager();
+
+  const SourceRange Lsr = Lhs->getSourceRange();
+  const SourceRange Rsr = Rhs->getSourceRange();
+  if (Lsr.getBegin().isMacroID()) {
+    // Left is macro so right macro too
+    if (Rsr.getBegin().isMacroID()) {
+      // Both sides are macros so they are same macro or literal
+      const llvm::StringRef L = Lexer::getSourceText(
+          CharSourceRange::getTokenRange(Lsr), SM, Context->getLangOpts(), 0);
+      const llvm::StringRef R = Lexer::getSourceText(
+          CharSourceRange::getTokenRange(Rsr), SM, Context->getLangOpts(), 0);
+      return areStringsSameIgnoreSpaces(L, R);
+    }
+    // Left is macro but right is not so they are not same macro or literal
+    return false;
+  }
+  const auto *Lil = dyn_cast<IntegerLiteral>(Lhs);
+  const auto *Ril = dyn_cast<IntegerLiteral>(Rhs);
+  if (Lil && Ril)
+    return Lil->getValue() == Ril->getValue();
+
+  const auto *LStrl = dyn_cast<StringLiteral>(Lhs);
+  const auto *RStrl = dyn_cast<StringLiteral>(Rhs);
+  if (Lil && Ril) {
+    const llvm::StringRef L = Lexer::getSourceText(
+        CharSourceRange::getTokenRange(LStrl->getBeginLoc()), SM,
+        Context->getLangOpts(), 0);
+    const llvm::StringRef R = Lexer::getSourceText(
+        CharSourceRange::getTokenRange(RStrl->getBeginLoc()), SM,
+        Context->getLangOpts(), 0);
+    return L.compare(R) == 0;
+  }
+
+  const auto *Lbl = dyn_cast<CXXBoolLiteralExpr>(Lhs);
+  const auto *Rbl = dyn_cast<CXXBoolLiteralExpr>(Rhs);
+  if (Lbl && Rbl)
+    return Lbl->getValue() == Rbl->getValue();
+
+  return false;
+}
 } // namespace
 
 void RedundantExpressionCheck::registerMatchers(MatchFinder *Finder) {
@@ -1089,7 +1183,6 @@ static bool exprEvaluatesToSymbolic(BinaryOperatorKind Opcode, APSInt Value) {
          ((Opcode == BO_And || Opcode == BO_AndAssign) && ~Value == 0);
 }
 
-
 void RedundantExpressionCheck::checkBitwiseExpr(
     const MatchFinder::MatchResult &Result) {
   if (const auto *ComparisonOperator = Result.Nodes.getNodeAs<BinaryOperator>(
@@ -1134,8 +1227,8 @@ void RedundantExpressionCheck::checkBitwiseExpr(
                                      ConstExpr))
       return;
 
-    if((Value != 0 && ~Value != 0) || Sym->getExprLoc().isMacroID())
-        return;
+    if ((Value != 0 && ~Value != 0) || Sym->getExprLoc().isMacroID())
+      return;
 
     SourceLocation Loc = IneffectiveOperator->getOperatorLoc();
 
@@ -1276,19 +1369,23 @@ void RedundantExpressionCheck::check(const MatchFinder::MatchResult &Result) {
       return;
     }
 
-    if (areSidesBinaryConstExpressions(BinOp, Result.Context)) {
+    if (areSidesBinaryConstExpressionsOrDefinesOrIntegerConstant(
+            BinOp, Result.Context)) {
       const Expr *LhsConst = nullptr, *RhsConst = nullptr;
       BinaryOperatorKind MainOpcode{}, SideOpcode{};
-
-      if (!retrieveConstExprFromBothSides(BinOp, MainOpcode, SideOpcode,
-                                          LhsConst, RhsConst, Result.Context))
-        return;
-
-      if (areExprsFromDifferentMacros(LhsConst, RhsConst, Result.Context) ||
-          areExprsMacroAndNonMacro(LhsConst, RhsConst))
-        return;
+      if (areSidesBinaryConstExpressions(BinOp, Result.Context)) {
+        if (!retrieveConstExprFromBothSides(BinOp, MainOpcode, SideOpcode,
+                                            LhsConst, RhsConst, Result.Context))
+          return;
+
+        if (areExprsFromDifferentMacros(LhsConst, RhsConst, Result.Context) ||
+            areExprsMacroAndNonMacro(LhsConst, RhsConst))
+          return;
+      } else {
+        if (!areExprsSameMacroOrLiteral(BinOp, Result.Context))
+          return;
+      }
     }
-
     diag(BinOp->getOperatorLoc(), "both sides of operator are equivalent");
   }
 
diff --git a/clang-tools-extra/clangd/Diagnostics.cpp b/clang-tools-extra/clangd/Diagnostics.cpp
index a59d1e7ac8409..28bb994a9e99a 100644
--- a/clang-tools-extra/clangd/Diagnostics.cpp
+++ b/clang-tools-extra/clangd/Diagnostics.cpp
@@ -577,7 +577,17 @@ std::vector<Diag> StoreDiags::take(const clang::tidy::ClangTidyContext *Tidy) {
   for (auto &Diag : Output) {
     if (const char *ClangDiag = getDiagnosticCode(Diag.ID)) {
       // Warnings controlled by -Wfoo are better recognized by that name.
-      StringRef Warning = DiagnosticIDs::getWarningOptionForDiag(Diag.ID);
+      StringRef Warning = [&] {
+        if (OrigSrcMgr) {
+          return OrigSrcMgr->getDiagnostics()
+              .getDiagnosticIDs()
+              ->getWarningOptionForDiag(Diag.ID);
+        }
+        if (!DiagnosticIDs::IsCustomDiag(Diag.ID))
+          return DiagnosticIDs{}.getWarningOptionForDiag(Diag.ID);
+        return StringRef{};
+      }();
+
       if (!Warning.empty()) {
         Diag.Name = ("-W" + Warning).str();
       } else {
@@ -894,20 +904,23 @@ void StoreDiags::flushLastDiag() {
   Output.push_back(std::move(*LastDiag));
 }
 
-bool isBuiltinDiagnosticSuppressed(unsigned ID,
-                                   const llvm::StringSet<> &Suppress,
-                                   const LangOptions &LangOpts) {
+bool isDiagnosticSuppressed(const clang::Diagnostic &Diag,
+                            const llvm::StringSet<> &Suppress,
+                            const LangOptions &LangOpts) {
   // Don't complain about header-only stuff in mainfiles if it's a header.
   // FIXME: would be cleaner to suppress in clang, once we decide whether the
   //        behavior should be to silently-ignore or respect the pragma.
-  if (ID == diag::pp_pragma_sysheader_in_main_file && LangOpts.IsHeaderFile)
+  if (Diag.getID() == diag::pp_pragma_sysheader_in_main_file &&
+      LangOpts.IsHeaderFile)
     return true;
 
-  if (const char *CodePtr = getDiagnosticCode(ID)) {
+  if (const char *CodePtr = getDiagnosticCode(Diag.getID())) {
     if (Suppress.contains(normalizeSuppressedCode(CodePtr)))
       return true;
   }
-  StringRef Warning = DiagnosticIDs::getWarningOptionForDiag(ID);
+  StringRef Warning =
+      Diag.getDiags()->getDiagnosticIDs()->getWarningOptionForDiag(
+          Diag.getID());
   if (!Warning.empty() && Suppress.contains(Warning))
     return true;
   return false;
diff --git a/clang-tools-extra/clangd/Diagnostics.h b/clang-tools-extra/clangd/Diagnostics.h
index d4c0478c63a5c..c45d8dc3aa6ce 100644
--- a/clang-tools-extra/clangd/Diagnostics.h
+++ b/clang-tools-extra/clangd/Diagnostics.h
@@ -181,11 +181,11 @@ class StoreDiags : public DiagnosticConsumer {
 };
 
 /// Determine whether a (non-clang-tidy) diagnostic is suppressed by config.
-bool isBuiltinDiagnosticSuppressed(unsigned ID,
-                                   const llvm::StringSet<> &Suppressed,
-                                   const LangOptions &);
+bool isDiagnosticSuppressed(const clang::Diagnostic &Diag,
+                            const llvm::StringSet<> &Suppressed,
+                            const LangOptions &);
 /// Take a user-specified diagnostic code, and convert it to a normalized form
-/// stored in the config and consumed by isBuiltinDiagnosticsSuppressed.
+/// stored in the config and consumed by isDiagnosticsSuppressed.
 ///
 /// (This strips err_ and -W prefix so we can match with or without them.)
 llvm::StringRef normalizeSuppressedCode(llvm::StringRef);
diff --git a/clang-tools-extra/clangd/ParsedAST.cpp b/clang-tools-extra/clangd/ParsedAST.cpp
index 89d6f26d0f150..3f63daaf400db 100644
--- a/clang-tools-extra/clangd/ParsedAST.cpp
+++ b/clang-tools-extra/clangd/ParsedAST.cpp
@@ -342,7 +342,7 @@ void applyWarningOptions(llvm::ArrayRef<std::string> ExtraArgs,
       if (Enable) {
         if (Diags.getDiagnosticLevel(ID, SourceLocation()) <
             DiagnosticsEngine::Warning) {
-          auto Group = DiagnosticIDs::getGroupForDiag(ID);
+          auto Group = Diags.getDiagnosticIDs()->getGroupForDiag(ID);
           if (!Group || !EnabledGroups(*Group))
             continue;
           Diags.setSeverity(ID, diag::Severity::Warning, SourceLocation());
@@ -585,8 +585,8 @@ ParsedAST::build(llvm::StringRef Filename, const ParseInputs &Inputs,
     ASTDiags.setLevelAdjuster([&](DiagnosticsEngine::Level DiagLevel,
                                   const clang::Diagnostic &Info) {
       if (Cfg.Diagnostics.SuppressAll ||
-          isBuiltinDiagnosticSuppressed(Info.getID(), Cfg.Diagnostics.Suppress,
-                                        Clang->getLangOpts()))
+          isDiagnosticSuppressed(Info, Cfg.Diagnostics.Suppress,
+                                 Clang->getLangOpts()))
         return DiagnosticsEngine::Ignored;
 
       auto It = OverriddenSeverity.find(Info.getID());
diff --git a/clang-tools-extra/clangd/Preamble.cpp b/clang-tools-extra/clangd/Preamble.cpp
index ce88ec0eb88c1..b247e608eece3 100644
--- a/clang-tools-extra/clangd/Preamble.cpp
+++ b/clang-tools-extra/clangd/Preamble.cpp
@@ -622,8 +622,8 @@ buildPreamble(PathRef FileName, CompilerInvocation CI,
   PreambleDiagnostics.setLevelAdjuster([&](DiagnosticsEngine::Level DiagLevel,
                                            const clang::Diagnostic &Info) {
     if (Cfg.Diagnostics.SuppressAll ||
-        isBuiltinDiagnosticSuppressed(Info.getID(), Cfg.Diagnostics.Suppress,
-                                      CI.getLangOpts()))
+        isDiagnosticSuppressed(Info, Cfg.Diagnostics.Suppress,
+                               CI.getLangOpts()))
       return DiagnosticsEngine::Ignored;
     switch (Info.getID()) {
     case diag::warn_no_newline_eof:
diff --git a/clang-tools-extra/clangd/unittests/ConfigCompileTests.cpp b/clang-tools-extra/clangd/unittests/ConfigCompileTests.cpp
index 179960a02cade..c3e484a1a79c4 100644
--- a/clang-tools-extra/clangd/unittests/ConfigCompileTests.cpp
+++ b/clang-tools-extra/clangd/unittests/ConfigCompileTests.cpp
@@ -298,20 +298,41 @@ TEST_F(ConfigCompileTests, DiagnosticSuppression) {
                                    "unreachable-code", "unused-variable",
                                    "typecheck_bool_condition",
                                    "unexpected_friend", "warn_alloca"));
-  EXPECT_TRUE(isBuiltinDiagnosticSuppressed(
-      diag::warn_unreachable, Conf.Diagnostics.Suppress, LangOptions()));
+  clang::DiagnosticsEngine DiagEngine(new DiagnosticIDs, nullptr,
+                                      new clang::IgnoringDiagConsumer);
+
+  using Diag = clang::Diagnostic;
+  {
+    auto D = DiagEngine.Report(diag::warn_unreachable);
+    EXPECT_TRUE(isDiagnosticSuppressed(
+        Diag{&DiagEngine, D}, Conf.Diagnostics.Suppress, LangOptions()));
+  }
   // Subcategory not respected/suppressed.
-  EXPECT_FALSE(isBuiltinDiagnosticSuppressed(
-      diag::warn_unreachable_break, Conf.Diagnostics.Suppress, LangOptions()));
-  EXPECT_TRUE(isBuiltinDiagnosticSuppressed(
-      diag::warn_unused_variable, Conf.Diagnostics.Suppress, LangOptions()));
-  EXPECT_TRUE(isBuiltinDiagnosticSuppressed(diag::err_typecheck_bool_condition,
-                                            Conf.Diagnostics.Suppress,
-                                            LangOptions()));
-  EXPECT_TRUE(isBuiltinDiagnosticSuppressed(
-      diag::err_unexpected_friend, Conf.Diagnostics.Suppress, LangOptions()));
-  EXPECT_TRUE(isBuiltinDiagnosticSuppressed(
-      diag::warn_alloca, Conf.Diagnostics.Suppress, LangOptions()));
+  {
+    auto D = DiagEngine.Report(diag::warn_unreachable_break);
+    EXPECT_FALSE(isDiagnosticSuppressed(
+        Diag{&DiagEngine, D}, Conf.Diagnostics.Suppress, LangOptions()));
+  }
+  {
+    auto D = DiagEngine.Report(diag::warn_unused_variable);
+    EXPECT_TRUE(isDiagnosticSuppressed(
+        Diag{&DiagEngine, D}, Conf.Diagnostics.Suppress, LangOptions()));
+  }
+  {
+    auto D = DiagEngine.Report(diag::err_typecheck_bool_condition);
+    EXPECT_TRUE(isDiagnosticSuppressed(
+        Diag{&DiagEngine, D}, Conf.Diagnostics.Suppress, LangOptions()));
+  }
+  {
+    auto D = DiagEngine.Report(diag::err_unexpected_friend);
+    EXPECT_TRUE(isDiagnosticSuppressed(
+        Diag{&DiagEngine, D}, Conf.Diagnostics.Suppress, LangOptions()));
+  }
+  {
+    auto D = DiagEngine.Report(diag::warn_alloca);
+    EXPECT_TRUE(isDiagnosticSuppressed(
+        Diag{&DiagEngine, D}, Conf.Diagnostics.Suppress, LangOptions()));
+  }
 
   Frag.Diagnostics.Suppress.emplace_back("*");
   EXPECT_TRUE(compileAndApply());
diff --git a/clang-tools-extra/docs/ReleaseNotes.rst b/clang-tools-extra/docs/ReleaseNotes.rst
index cc5f64a3f9fa3..6b8fe22242417 100644
--- a/clang-tools-extra/docs/ReleaseNotes.rst
+++ b/clang-tools-extra/docs/ReleaseNotes.rst
@@ -43,21 +43,12 @@ infrastructure are described first, followed by tool-specific sections.
 Major New Features
 ------------------
 
-- The ``clang-pseudo`` tool is incomplete and does not have active maintainers,
-  so it has been removed. See
-  `the RFC <https://discourse.llvm.org/t/removing-pseudo-parser/71131/>`_ for
-  more details.
-
-...
-
 Improvements to clangd
 ----------------------
 
 Inlay hints
 ^^^^^^^^^^^
 
-- Added `DefaultArguments` Inlay Hints option.
-
 Diagnostics
 ^^^^^^^^^^^
 
@@ -73,16 +64,9 @@ Hover
 Code completion
 ^^^^^^^^^^^^^^^
 
-- Added completion for C++20 keywords.
-
 Code actions
 ^^^^^^^^^^^^
 
-- Added `Swap operands` tweak for certain binary operators.
-
-- Improved the extract-to-function code action to allow extracting statements
-  with overloaded operators like ``<<`` of ``std::ostream``.
-
 Signature help
 ^^^^^^^^^^^^^^
 
@@ -95,312 +79,36 @@ Objective-C
 Miscellaneous
 ^^^^^^^^^^^^^
 
-- The DefineOutline tweak now handles member functions of class templates.
-
 Improvements to clang-doc
 -------------------------
 
 Improvements to clang-query
 ---------------------------
 
-- Added `set enable-profile true/false` command for basic matcher profiling.
-
 Improvements to clang-tidy
 --------------------------
 
-- Improved :program:`clang-tidy-diff.py` script. Add the `-only-check-in-db`
-  option to exclude files not present in the compilation database, avoiding
-  false-negative results.
-
-- Improved :program:`run-clang-tidy.py` script. Fixed minor shutdown noise
-  happening on certain platforms when interrupting the script.
-
-- Improved :program:`clang-tidy`:
-
-  - add support for `--verify-config` flag to check the configuration options of
-    the `Clang Static Analyzer Checks <https://clang.llvm.org/docs/analyzer/checkers.html>`_.
-  - accept parameters file in command line.
-  - fix incorrect configuration file path resolving when file paths contain ``..``.
-  - remove global options for most of checks. All options are changed to local
-    options except `IncludeStyle`, `StrictMode` and `IgnoreMacros`. Global scoped
-    `StrictMode` and `IgnoreMacros` are deprecated and will be removed in further
-    releases.
-
-.. csv-table::
-  :header: "Check", "Options removed from global option"
-
-  :doc:`bugprone-reserved-identifier <clang-tidy/checks/bugprone/reserved-identifier>`, AggressiveDependentMemberLookup
-  :doc:`bugprone-unchecked-optional-access <clang-tidy/checks/bugprone/unchecked-optional-access>`, IgnoreSmartPointerDereference
-  :doc:`cppcoreguidelines-pro-type-member-init <clang-tidy/checks/cppcoreguidelines/pro-type-member-init>`, UseAssignment
-  :doc:`cppcoreguidelines-rvalue-reference-param-not-moved <clang-tidy/checks/cppcoreguidelines/rvalue-reference-param-not-moved>`, AllowPartialMove; IgnoreUnnamedParams; IgnoreNonDeducedTemplateTypes
-  :doc:`misc-include-cleaner <clang-tidy/checks/misc/include-cleaner>`, IgnoreHeaders; DeduplicateFindings
-  :doc:`performance-inefficient-vector-operation <clang-tidy/checks/performance/inefficient-vector-operation>`, EnableProto
-  :doc:`readability-identifier-naming <clang-tidy/checks/readability/identifier-naming>`, AggressiveDependentMemberLookup
-  :doc:`readability-inconsistent-declaration-parameter-name <clang-tidy/checks/readability/inconsistent-declaration-parameter-name>`, Strict
-  :doc:`readability-redundant-access-specifiers <clang-tidy/checks/readability/redundant-access-specifiers>`, CheckFirstDeclaration
-  :doc:`readability-redundant-casting <clang-tidy/checks/readability/redundant-casting>`, IgnoreTypeAliases
-
 New checks
 ^^^^^^^^^^
 
-- New :doc:`bugprone-bitwise-pointer-cast
-  <clang-tidy/checks/bugprone/bitwise-pointer-cast>` check.
-
-  Warns about code that tries to cast between pointers by means of
-  ``std::bit_cast`` or ``memcpy``.
-
-- New :doc:`bugprone-incorrect-enable-shared-from-this
-  <clang-tidy/checks/bugprone/incorrect-enable-shared-from-this>` check.
-
-  Detect classes or structs that do not publicly inherit from 
-  ``std::enable_shared_from_this``, because unintended behavior will 
-  otherwise occur when calling ``shared_from_this``.
-  
-- New :doc:`bugprone-nondeterministic-pointer-iteration-order
-  <clang-tidy/checks/bugprone/nondeterministic-pointer-iteration-order>`
-  check.
-
-  Finds nondeterministic usages of pointers in unordered containers.
-
-- New :doc:`bugprone-tagged-union-member-count
-  <clang-tidy/checks/bugprone/tagged-union-member-count>` check.
-
-  Gives warnings for tagged unions, where the number of tags is
-  different from the number of data members inside the union.
-
-- New :doc:`modernize-use-integer-sign-comparison
-  <clang-tidy/checks/modernize/use-integer-sign-comparison>` check.
-
-  Replace comparisons between signed and unsigned integers with their safe
-  C++20 ``std::cmp_*`` alternative, if available.
-
-- New :doc:`portability-template-virtual-member-function
-  <clang-tidy/checks/portability/template-virtual-member-function>` check.
-
-  Finds cases when an uninstantiated virtual member function in a template class
-  causes cross-compiler incompatibility.
-
 New check aliases
 ^^^^^^^^^^^^^^^^^
 
-- New alias :doc:`cert-arr39-c <clang-tidy/checks/cert/arr39-c>` to
-  :doc:`bugprone-sizeof-expression
-  <clang-tidy/checks/bugprone/sizeof-expression>` was added.
-
 Changes in existing checks
 ^^^^^^^^^^^^^^^^^^^^^^^^^^
 
-- Improved :doc:`altera-id-dependent-backward-branch
-  <clang-tidy/checks/altera/id-dependent-backward-branch>` check by fixing
-  crashes from invalid code.
-
-- Improved :doc:`bugprone-branch-clone
-  <clang-tidy/checks/bugprone/branch-clone>` check to improve detection of
-  branch clones by now detecting duplicate inner and outer if statements.
-
-- Improved :doc:`bugprone-casting-through-void
-  <clang-tidy/checks/bugprone/casting-through-void>` check to suggest replacing
-  the offending code with ``reinterpret_cast``, to more clearly express intent.
-
-- Improved :doc:`bugprone-dangling-handle
-  <clang-tidy/checks/bugprone/dangling-handle>` check to treat ``std::span`` as a
-  handle class.
-
-- Improved :doc:`bugprone-exception-escape
-  <clang-tidy/checks/bugprone/exception-escape>` by fixing false positives
-  when a consteval function with throw statements.
-
-- Improved :doc:`bugprone-forwarding-reference-overload
-  <clang-tidy/checks/bugprone/forwarding-reference-overload>` check by fixing
-  a crash when determining if an ``enable_if[_t]`` was found.
-
-- Improve :doc:`bugprone-narrowing-conversions
-  <clang-tidy/checks/bugprone/narrowing-conversions>` to avoid incorrect check
-  results when floating point type is not ``float``, ``double`` and
-  ``long double``.
-
-- Improved :doc:`bugprone-optional-value-conversion
-  <clang-tidy/checks/bugprone/optional-value-conversion>` to support detecting
-  conversion directly by ``std::make_unique`` and ``std::make_shared``.
-
-- Improved :doc:`bugprone-posix-return
-  <clang-tidy/checks/bugprone/posix-return>` check to support integer literals
-  as LHS and posix call as RHS of comparison.
-
-- Improved :doc:`bugprone-return-const-ref-from-parameter
-  <clang-tidy/checks/bugprone/return-const-ref-from-parameter>` check to
-  diagnose potential dangling references when returning a ``const &`` parameter
-  by using the conditional operator ``cond ? var1 : var2`` and fixing false
-  positives for functions which contain lambda and ignore parameters
-  with ``[[clang::lifetimebound]]`` attribute.
-  
-- Improved :doc:`bugprone-sizeof-expression
-  <clang-tidy/checks/bugprone/sizeof-expression>` check to find suspicious
-  usages of ``sizeof()``, ``alignof()``, and ``offsetof()`` when adding or
-  subtracting from a pointer directly or when used to scale a numeric value and
-  fix false positive when sizeof expression with template types.
-
-- Improved :doc:`bugprone-throw-keyword-missing
-  <clang-tidy/checks/bugprone/throw-keyword-missing>` by fixing a false positive
-  when using non-static member initializers and a constructor.
-
-- Improved :doc:`bugprone-unchecked-optional-access
-  <clang-tidy/checks/bugprone/unchecked-optional-access>` to support
-  ``bsl::optional`` and ``bdlb::NullableValue`` from
-  <https://github.com/bloomberg/bde>_.
-
-- Improved :doc:`bugprone-unhandled-self-assignment
-  <clang-tidy/checks/bugprone/unhandled-self-assignment>` check by fixing smart
-  pointer check against std::unique_ptr type.
+- Improved :doc:`bugprone-string-constructor
+  <clang-tidy/checks/bugprone/string-constructor>` check to find suspicious
+  calls of ``std::string`` constructor with char pointer, start position and
+  length parameters.
 
 - Improved :doc:`bugprone-unsafe-functions
   <clang-tidy/checks/bugprone/unsafe-functions>` check to allow specifying
-  additional functions to match.
-
-- Improved :doc:`bugprone-unused-local-non-trivial-variable
-  <clang-tidy/checks/bugprone/unused-local-non-trivial-variable>` check to avoid
-  false positives when using name-independent variables after C++26.
-
-- Improved :doc:`bugprone-use-after-move
-  <clang-tidy/checks/bugprone/use-after-move>` to avoid triggering on
-  ``reset()`` calls on moved-from ``std::optional`` and ``std::any`` objects,
-  similarly to smart pointers.
-
-- Improved :doc:`cert-flp30-c <clang-tidy/checks/cert/flp30-c>` check to
-  fix false positive that floating point variable is only used in increment
-  expression.
-
-- Improved :doc:`cppcoreguidelines-avoid-const-or-ref-data-members
-  <clang-tidy/checks/cppcoreguidelines/avoid-const-or-ref-data-members>` check to
-  avoid false positives when detecting a templated class with inheritance.
-
-- Improved :doc:`cppcoreguidelines-init-variables
-  <clang-tidy/checks/cppcoreguidelines/init-variables>` check by fixing the
-  insertion location for function pointers.
-
-- Improved :doc:`cppcoreguidelines-prefer-member-initializer
-  <clang-tidy/checks/cppcoreguidelines/prefer-member-initializer>` check to
-  avoid false positive when member initialization depends on a structured
-  binding variable.
-
-- Fixed :doc:`cppcoreguidelines-pro-type-union-access
-  <clang-tidy/checks/cppcoreguidelines/pro-type-union-access>` check to
-  report a location even when the member location is not valid.
-
-- Improved :doc:`misc-definitions-in-headers
-  <clang-tidy/checks/misc/definitions-in-headers>` check by rewording the
-  diagnostic note that suggests adding ``inline``.
+  additional C++ member functions to match.
 
 - Improved :doc:`misc-redundant-expression
-  <clang-tidy/checks/misc/redundant-expression>` check by extending the
-  checker to detect floating point and integer literals in redundant
-  expressions.
-
-- Improved :doc:`misc-unconventional-assign-operator
-  <clang-tidy/checks/misc/unconventional-assign-operator>` check to avoid
-  false positive for C++23 deducing this.
-
-- Improved :doc:`misc-use-internal-linkage
-  <clang-tidy/checks/misc/use-internal-linkage>` check to insert ``static``
-  keyword before type qualifiers such as ``const`` and ``volatile``. Also, fix
-  false positives for function declaration without body, C++20 consteval
-  functions, C++20 export declarations, and global scoped
-  overloaded ``operator new`` and ``operator delete``.
-
-- Improved :doc:`modernize-avoid-c-arrays
-  <clang-tidy/checks/modernize/avoid-c-arrays>` check to suggest using
-  ``std::span`` as a replacement for parameters of incomplete C array type in
-  C++20 and ``std::array`` or ``std::vector`` before C++20.
-
-- Improved :doc:`modernize-loop-convert
-  <clang-tidy/checks/modernize/loop-convert>` check to fix false positive when
-  using loop variable in initializer of lambda capture.
-
-- Improved :doc:`modernize-min-max-use-initializer-list
-  <clang-tidy/checks/modernize/min-max-use-initializer-list>` check by fixing
-  a false positive when only an implicit conversion happened inside an
-  initializer list.
-
-- Improved :doc:`modernize-raw-string-literal
-  <clang-tidy/checks/modernize/raw-string-literal>` check to fix incorrect
-  fix-it when the string contains a user-defined suffix.
-
-- Improved :doc:`modernize-use-designated-initializers
-  <clang-tidy/checks/modernize/use-designated-initializers>` check to fix a
-  crash when a class is declared but not defined.
-
-- Improved :doc:`modernize-use-integer-sign-comparison
-  <clang-tidy/checks/modernize/use-integer-sign-comparison>` check to
-  add an option ``EnableQtSupport``, that makes C++17 ``q20::cmp_*`` alternative
-  available for Qt-based applications.
-
-- Improved :doc:`modernize-use-nullptr
-  <clang-tidy/checks/modernize/use-nullptr>` check to also recognize
-  ``NULL``/``__null`` (but not ``0``) when used with a templated type.
-
-- Improved :doc:`modernize-use-starts-ends-with
-  <clang-tidy/checks/modernize/use-starts-ends-with>` check to handle two new
-  cases from ``rfind`` and ``compare`` to ``ends_with``, and one new case from
-  ``substr`` to ``starts_with``, and a small adjustment to the  diagnostic message.
-
-- Improved :doc:`modernize-use-std-format
-  <clang-tidy/checks/modernize/use-std-format>` check to support replacing
-  member function calls too and to only expand macros starting with ``PRI``
-  and ``__PRI`` from ``<inttypes.h>`` in the format string.
-
-- Improved :doc:`modernize-use-std-print
-  <clang-tidy/checks/modernize/use-std-print>` check to support replacing
-  member function calls too and to only expand macros starting with ``PRI``
-  and ``__PRI`` from ``<inttypes.h>`` in the format string.
-
-- Improved :doc:`modernize-use-using
-  <clang-tidy/checks/modernize/use-using>` check by not expanding macros.
-
-- Improved :doc:`performance-avoid-endl
-  <clang-tidy/checks/performance/avoid-endl>` check to use ``std::endl`` as
-  placeholder when lexer cannot get source text.
-
-- Improved :doc:`performance-move-const-arg
-  <clang-tidy/checks/performance/move-const-arg>` check to fix a crash when
-  an argument type is declared but not defined.
-
-- Improved :doc:`performance-unnecessary-copy-initialization
-  <clang-tidy/checks/performance/unnecessary-copy-initialization>` check
-  to consider static member functions the same way as free functions.
-
-- Improved :doc:`readability-container-contains
-  <clang-tidy/checks/readability/container-contains>` check to let it work on
-  any class that has a ``contains`` method. Fix some false negatives in the
-  ``find()`` case.
-
-- Improved :doc:`readability-enum-initial-value
-  <clang-tidy/checks/readability/enum-initial-value>` check by only issuing
-  diagnostics for the definition of an ``enum``, by not emitting a redundant
-  file path for anonymous enums in the diagnostic, and by fixing a typo in the
-  diagnostic.
-
-- Improved :doc:`readability-identifier-naming
-  <clang-tidy/checks/readability/identifier-naming>` check to
-  validate ``namespace`` aliases.
-
-- Improved :doc:`readability-implicit-bool-conversion
-  <clang-tidy/checks/readability/implicit-bool-conversion>` check
-  by adding the option `UseUpperCaseLiteralSuffix` to select the
-  case of the literal suffix in fixes and fixing false positive for implicit
-  conversion of comparison result in C23.
-
-- Improved :doc:`readability-redundant-casting
-  <clang-tidy/checks/readability/redundant-casting>` check
-  by addressing a false positive in aggregate initialization through
-  parenthesized list.
-
-- Improved :doc:`readability-redundant-smartptr-get
-  <clang-tidy/checks/readability/redundant-smartptr-get>` check to
-  remove ``->``, when redundant ``get()`` is removed.
-
-- Improved :doc:`readability-use-std-min-max
-  <clang-tidy/checks/readability/use-std-min-max>` check to use correct template
-  type in ``std::min`` and ``std::max`` when operand is integer literal.
+  <clang-tidy/checks/misc/redundant-expression>` check by providing additional
+  examples and fixing some macro related false positives.
 
 Removed checks
 ^^^^^^^^^^^^^^
@@ -408,13 +116,6 @@ Removed checks
 Miscellaneous
 ^^^^^^^^^^^^^
 
-- The :doc:`bugprone-narrowing-conversions <clang-tidy/checks/bugprone/narrowing-conversions>`
-  check is no longer an alias of :doc:`cppcoreguidelines-narrowing-conversions
-  <clang-tidy/checks/cppcoreguidelines/narrowing-conversions>`. Instead,
-  :doc:`cppcoreguidelines-narrowing-conversions
-  <clang-tidy/checks/cppcoreguidelines/narrowing-conversions>` is now an alias
-  of :doc:`bugprone-narrowing-conversions <clang-tidy/checks/bugprone/narrowing-conversions>`.
-
 Improvements to include-fixer
 -----------------------------
 
diff --git a/clang-tools-extra/docs/clang-tidy/checks/bugprone/assert-side-effect.rst b/clang-tools-extra/docs/clang-tidy/checks/bugprone/assert-side-effect.rst
index 8ba84ff61c6a9..1355afae92e4f 100644
--- a/clang-tools-extra/docs/clang-tidy/checks/bugprone/assert-side-effect.rst
+++ b/clang-tools-extra/docs/clang-tidy/checks/bugprone/assert-side-effect.rst
@@ -26,8 +26,8 @@ Options
 
    A semicolon-separated list of the names of functions or methods to be
    considered as not having side-effects. Regular expressions are accepted,
-   e.g. `[Rr]ef(erence)?$` matches every type with suffix `Ref`, `ref`,
-   `Reference` and `reference`. The default is empty. If a name in the list
-   contains the sequence `::` it is matched against the qualified typename
-   (i.e. `namespace::Type`, otherwise it is matched against only
-   the type name (i.e. `Type`).
+   e.g. ``[Rr]ef(erence)?$`` matches every type with suffix ``Ref``, ``ref``,
+   ``Reference`` and ``reference``. The default is empty. If a name in the list
+   contains the sequence `::` it is matched against the qualified type name
+   (i.e. ``namespace::Type``), otherwise it is matched against only
+   the type name (i.e. ``Type``).
diff --git a/clang-tools-extra/docs/clang-tidy/checks/bugprone/string-constructor.rst b/clang-tools-extra/docs/clang-tidy/checks/bugprone/string-constructor.rst
index b45c2ef4a9312..a0bd1d7c5bc15 100644
--- a/clang-tools-extra/docs/clang-tidy/checks/bugprone/string-constructor.rst
+++ b/clang-tools-extra/docs/clang-tidy/checks/bugprone/string-constructor.rst
@@ -21,6 +21,7 @@ Examples:
 .. code-block:: c++
 
   std::string("test", 200);   // Will include random characters after "test".
+  std::string("test", 2, 5);  // Will include random characters after "st".
   std::string_view("test", 200);
 
 Creating an empty string from constructors with parameters is considered
@@ -31,8 +32,19 @@ Examples:
 .. code-block:: c++
 
   std::string("test", 0);   // Creation of an empty string.
+  std::string("test", 1, 0);
   std::string_view("test", 0);
 
+Passing an invalid first character position parameter to constructor will
+cause ``std::out_of_range`` exception at runtime. 
+
+Examples:
+
+.. code-block:: c++
+
+  std::string("test", -1, 10); // Negative first character position.
+  std::string("test", 10, 10); // First character position is bigger than string literal character range".
+
 Options
 -------
 
diff --git a/clang-tools-extra/docs/clang-tidy/checks/bugprone/unsafe-functions.rst b/clang-tools-extra/docs/clang-tidy/checks/bugprone/unsafe-functions.rst
index fb070627e31b1..317db9c5564e2 100644
--- a/clang-tools-extra/docs/clang-tidy/checks/bugprone/unsafe-functions.rst
+++ b/clang-tools-extra/docs/clang-tidy/checks/bugprone/unsafe-functions.rst
@@ -114,6 +114,17 @@ qualified name (i.e. ``std::original``), otherwise the regex is matched against
 If the regular expression starts with `::` (or `^::`), it is matched against the
 fully qualified name (``::std::original``).
 
+.. note::
+
+   Fully qualified names can contain template parameters on certain C++ classes, but not on C++ functions.
+   Type aliases are resolved before matching.
+
+   As an example, the member function ``open`` in the class ``std::ifstream``
+   has a fully qualified name of ``::std::basic_ifstream<char>::open``.
+
+   The example could also be matched with the regex ``::std::basic_ifstream<[^>]*>::open``, which matches all potential
+   template parameters, but does not match nested template classes.
+
 Options
 -------
 
diff --git a/clang-tools-extra/docs/clang-tidy/checks/misc/redundant-expression.rst b/clang-tools-extra/docs/clang-tidy/checks/misc/redundant-expression.rst
index 44f0772d0bedf..cea998d39bd73 100644
--- a/clang-tools-extra/docs/clang-tidy/checks/misc/redundant-expression.rst
+++ b/clang-tools-extra/docs/clang-tidy/checks/misc/redundant-expression.rst
@@ -19,12 +19,14 @@ Examples:
 
 .. code-block:: c++
 
-  ((x+1) | (x+1))             // (x+1) is redundant
-  (p->x == p->x)              // always true
-  (p->x < p->x)               // always false
-  (speed - speed + 1 == 12)   // speed - speed is always zero
-  int b = a | 4 | a           // identical expr on both sides
-  ((x=1) | (x=1))             // expression is identical
+  ((x+1) | (x+1))                   // (x+1) is redundant
+  (p->x == p->x)                    // always true
+  (p->x < p->x)                     // always false
+  (speed - speed + 1 == 12)         // speed - speed is always zero
+  int b = a | 4 | a                 // identical expr on both sides
+  ((x=1) | (x=1))                   // expression is identical
+  (DEFINE_1 | DEFINE_1)             // same macro on the both sides
+  ((DEF_1 + DEF_2) | (DEF_1+DEF_2)) // expressions differ in spaces only
 
 Floats are handled except in the case that NaNs are checked like so:
 
diff --git a/clang-tools-extra/docs/clang-tidy/checks/performance/for-range-copy.rst b/clang-tools-extra/docs/clang-tidy/checks/performance/for-range-copy.rst
index 01fde9580e2a0..d740984029482 100644
--- a/clang-tools-extra/docs/clang-tidy/checks/performance/for-range-copy.rst
+++ b/clang-tools-extra/docs/clang-tidy/checks/performance/for-range-copy.rst
@@ -23,14 +23,15 @@ Options
 
 .. option:: WarnOnAllAutoCopies
 
-   When `true`, warns on any use of `auto` as the type of the range-based for
+   When `true`, warns on any use of ``auto`` as the type of the range-based for
    loop variable. Default is `false`.
 
 .. option:: AllowedTypes
 
    A semicolon-separated list of names of types allowed to be copied in each
-   iteration. Regular expressions are accepted, e.g. `[Rr]ef(erence)?$` matches
-   every type with suffix `Ref`, `ref`, `Reference` and `reference`. The default
-   is empty. If a name in the list contains the sequence `::` it is matched
-   against the qualified typename (i.e. `namespace::Type`, otherwise it is
-   matched against only the type name (i.e. `Type`).
+   iteration. Regular expressions are accepted, e.g. ``[Rr]ef(erence)?$``
+   matches every type with suffix ``Ref``, ``ref``, ``Reference`` and
+   ``reference``. The default is empty. If a name in the list contains the
+   sequence `::`, it is matched against the qualified type name
+   (i.e. ``namespace::Type``), otherwise it is matched against only the
+   type name (i.e. ``Type``).
diff --git a/clang-tools-extra/docs/clang-tidy/checks/performance/unnecessary-copy-initialization.rst b/clang-tools-extra/docs/clang-tidy/checks/performance/unnecessary-copy-initialization.rst
index 837283811ddcc..bb4f42c88d62f 100644
--- a/clang-tools-extra/docs/clang-tidy/checks/performance/unnecessary-copy-initialization.rst
+++ b/clang-tools-extra/docs/clang-tidy/checks/performance/unnecessary-copy-initialization.rst
@@ -42,11 +42,11 @@ Options
 .. option:: AllowedTypes
 
    A semicolon-separated list of names of types allowed to be initialized by
-   copying. Regular expressions are accepted, e.g. `[Rr]ef(erence)?$` matches
-   every type with suffix `Ref`, `ref`, `Reference` and `reference`. The default
-   is empty. If a name in the list contains the sequence `::` it is matched
-   against the qualified typename (i.e. `namespace::Type`, otherwise it is
-   matched against only the type name (i.e. `Type`).
+   copying. Regular expressions are accepted, e.g. ``[Rr]ef(erence)?$`` matches
+   every type with suffix ``Ref``, ``ref``, ``Reference`` and ``reference``.
+   The default is empty. If a name in the list contains the sequence `::`, it
+   is matched against the qualified type name (i.e. ``namespace::Type``),
+   otherwise it is matched against only the type name (i.e. ``Type``).
 
 .. option:: ExcludedContainerTypes
 
diff --git a/clang-tools-extra/docs/clang-tidy/checks/performance/unnecessary-value-param.rst b/clang-tools-extra/docs/clang-tidy/checks/performance/unnecessary-value-param.rst
index cc5e1ae73508c..d13c53eea994e 100644
--- a/clang-tools-extra/docs/clang-tidy/checks/performance/unnecessary-value-param.rst
+++ b/clang-tools-extra/docs/clang-tidy/checks/performance/unnecessary-value-param.rst
@@ -65,8 +65,8 @@ Options
 .. option:: AllowedTypes
 
    A semicolon-separated list of names of types allowed to be passed by value.
-   Regular expressions are accepted, e.g. `[Rr]ef(erence)?$` matches every type
-   with suffix `Ref`, `ref`, `Reference` and `reference`. The default is
-   empty. If a name in the list contains the sequence `::` it is matched against
-   the qualified typename (i.e. `namespace::Type`, otherwise it is matched
-   against only the type name (i.e. `Type`).
+   Regular expressions are accepted, e.g. ``[Rr]ef(erence)?$`` matches every
+   type with suffix ``Ref``, ``ref``, ``Reference`` and ``reference``. The
+   default is empty. If a name in the list contains the sequence `::`, it is
+   matched against the qualified type name (i.e. ``namespace::Type``),
+   otherwise it is matched against only the type name (i.e. ``Type``).
diff --git a/clang-tools-extra/include-cleaner/lib/WalkAST.cpp b/clang-tools-extra/include-cleaner/lib/WalkAST.cpp
index aae3eda519ffd..7a140c991925c 100644
--- a/clang-tools-extra/include-cleaner/lib/WalkAST.cpp
+++ b/clang-tools-extra/include-cleaner/lib/WalkAST.cpp
@@ -22,6 +22,7 @@
 #include "clang/AST/Type.h"
 #include "clang/AST/TypeLoc.h"
 #include "clang/Basic/IdentifierTable.h"
+#include "clang/Basic/OperatorKinds.h"
 #include "clang/Basic/SourceLocation.h"
 #include "clang/Basic/Specifiers.h"
 #include "llvm/ADT/STLExtras.h"
@@ -32,6 +33,11 @@
 
 namespace clang::include_cleaner {
 namespace {
+bool isOperatorNewDelete(OverloadedOperatorKind OpKind) {
+  return OpKind == OO_New || OpKind == OO_Delete || OpKind == OO_Array_New ||
+         OpKind == OO_Array_Delete;
+}
+
 using DeclCallback =
     llvm::function_ref<void(SourceLocation, NamedDecl &, RefType)>;
 
@@ -158,7 +164,15 @@ class ASTWalker : public RecursiveASTVisitor<ASTWalker> {
     // the container decl instead, which is preferred as it'll handle
     // aliases/exports properly.
     if (!FD->isCXXClassMember() && !llvm::isa<EnumConstantDecl>(FD)) {
-      report(DRE->getLocation(), FD);
+      // Global operator new/delete [] is available implicitly in every
+      // translation unit, even without including any explicit headers. So treat
+      // those as ambigious to not force inclusion in TUs that transitively
+      // depend on those.
+      RefType RT =
+          isOperatorNewDelete(FD->getDeclName().getCXXOverloadedOperator())
+              ? RefType::Ambiguous
+              : RefType::Explicit;
+      report(DRE->getLocation(), FD, RT);
       return true;
     }
     // If the ref is without a qualifier, and is a member, ignore it. As it is
diff --git a/clang-tools-extra/include-cleaner/unittests/AnalysisTest.cpp b/clang-tools-extra/include-cleaner/unittests/AnalysisTest.cpp
index d2d137a0dfb42..74321c312cb71 100644
--- a/clang-tools-extra/include-cleaner/unittests/AnalysisTest.cpp
+++ b/clang-tools-extra/include-cleaner/unittests/AnalysisTest.cpp
@@ -397,6 +397,55 @@ TEST_F(AnalyzeTest, SpellingIncludesWithSymlinks) {
   }
 }
 
+// Make sure that the references to implicit operator new/delete are reported as
+// ambigious.
+TEST_F(AnalyzeTest, ImplicitOperatorNewDeleteNotMissing) {
+  ExtraFS = llvm::makeIntrusiveRefCnt<llvm::vfs::InMemoryFileSystem>();
+  ExtraFS->addFile("header.h",
+                   /*ModificationTime=*/{},
+                   llvm::MemoryBuffer::getMemBufferCopy(guard(R"cpp(
+  void* operator new(decltype(sizeof(int)));
+  )cpp")));
+  ExtraFS->addFile("wrapper.h",
+                   /*ModificationTime=*/{},
+                   llvm::MemoryBuffer::getMemBufferCopy(guard(R"cpp(
+  #include "header.h"
+  )cpp")));
+
+  Inputs.Code = R"cpp(
+      #include "wrapper.h"
+      void bar() {
+        operator new(3);
+      })cpp";
+  TestAST AST(Inputs);
+  std::vector<Decl *> DeclsInTU;
+  for (auto *D : AST.context().getTranslationUnitDecl()->decls())
+    DeclsInTU.push_back(D);
+  auto Results = analyze(DeclsInTU, {}, PP.Includes, &PI, AST.preprocessor());
+  EXPECT_THAT(Results.Missing, testing::IsEmpty());
+}
+
+TEST_F(AnalyzeTest, ImplicitOperatorNewDeleteNotUnused) {
+  ExtraFS = llvm::makeIntrusiveRefCnt<llvm::vfs::InMemoryFileSystem>();
+  ExtraFS->addFile("header.h",
+                   /*ModificationTime=*/{},
+                   llvm::MemoryBuffer::getMemBufferCopy(guard(R"cpp(
+  void* operator new(decltype(sizeof(int)));
+  )cpp")));
+
+  Inputs.Code = R"cpp(
+      #include "header.h"
+      void bar() {
+        operator new(3);
+      })cpp";
+  TestAST AST(Inputs);
+  std::vector<Decl *> DeclsInTU;
+  for (auto *D : AST.context().getTranslationUnitDecl()->decls())
+    DeclsInTU.push_back(D);
+  auto Results = analyze(DeclsInTU, {}, PP.Includes, &PI, AST.preprocessor());
+  EXPECT_THAT(Results.Unused, testing::IsEmpty());
+}
+
 TEST(FixIncludes, Basic) {
   llvm::StringRef Code = R"cpp(#include "d.h"
 #include "a.h"
diff --git a/clang-tools-extra/test/clang-doc/Inputs/basic-project/src/Circle.cpp b/clang-tools-extra/test/clang-doc/Inputs/basic-project/src/Circle.cpp
index 823384a4d97e8..3ddb2fd9ff563 100644
--- a/clang-tools-extra/test/clang-doc/Inputs/basic-project/src/Circle.cpp
+++ b/clang-tools-extra/test/clang-doc/Inputs/basic-project/src/Circle.cpp
@@ -8,4 +8,5 @@ double Circle::area() const {
 
 double Circle::perimeter() const {
     return 3.141 * radius_;
-}
\ No newline at end of file
+}
+
diff --git a/clang-tools-extra/test/clang-doc/basic-project.test b/clang-tools-extra/test/clang-doc/basic-project.test
index 7c46c52bae687..1f5ba8bdc0703 100644
--- a/clang-tools-extra/test/clang-doc/basic-project.test
+++ b/clang-tools-extra/test/clang-doc/basic-project.test
@@ -1,6 +1,6 @@
 // RUN: rm -rf %t && mkdir -p %t/docs %t/build
 // RUN: sed 's|$test_dir|%/S|g' %S/Inputs/basic-project/database_template.json > %t/build/compile_commands.json
-// RUN: clang-doc --format=html --output=%t/docs --executor=all-TUs %t/build/compile_commands.json
+// RUN: clang-doc --format=html --output=%t/docs --executor=all-TUs %t/build/compile_commands.json --repository=https://repository.com
 // RUN: FileCheck %s -input-file=%t/docs/index_json.js -check-prefix=JSON-INDEX
 // RUN: FileCheck %s -input-file=%t/docs/GlobalNamespace/Shape.html -check-prefix=HTML-SHAPE
 // RUN: FileCheck %s -input-file=%t/docs/GlobalNamespace/Calculator.html -check-prefix=HTML-CALC
@@ -54,130 +54,183 @@
 // JSON-INDEX-NEXT: };
 // JSON-INDEX-NEXT: }
 
-// HTML-SHAPE: <h1>class Shape</h1>
-// HTML-SHAPE: <p>Defined at line 8 of file {{.*}}Shape.h</p>
-// HTML-SHAPE: <div>brief</div>
-// HTML-SHAPE: <p> Abstract base class for shapes.</p>
-// HTML-SHAPE: <p> Provides a common interface for different types of shapes.</p>
-// HTML-SHAPE: <h2 id="Functions">Functions</h2>
-// HTML-SHAPE: <h3 id="{{([0-9A-F]{40})}}">area</h3>
-// HTML-SHAPE: <p>public double area()</p>
-// HTML-SHAPE: <div>brief</div>
-// HTML-SHAPE: <p> Calculates the area of the shape.</p>
-// HTML-SHAPE: <h3 id="{{([0-9A-F]{40})}}">perimeter</h3>
-// HTML-SHAPE: <p>public double perimeter()</p>
-// HTML-SHAPE: <div>brief</div>
-// HTML-SHAPE: <p> Calculates the perimeter of the shape.</p>
-// HTML-SHAPE: <div>return</div>
-// HTML-SHAPE: <p> double The perimeter of the shape.</p>
-// HTML-SHAPE: <h3 id="{{([0-9A-F]{40})}}">~Shape</h3>
-// HTML-SHAPE: <p>public void ~Shape()</p>
-// HTML-SHAPE: <p>Defined at line 13 of file {{.*}}Shape.h</p>
-// HTML-SHAPE: <div>brief</div>
-// HTML-SHAPE: <p> Virtual destructor.</p>
+//      HTML-SHAPE: <h1>class Shape</h1>
+// HTML-SHAPE-NEXT: <p>
+// HTML-SHAPE-NEXT: Defined at line
+// HTML-SHAPE-NEXT: <a href="https://repository.com/./include/Shape.h#8">8</a>
+// HTML-SHAPE-NEXT: of file
+// HTML-SHAPE-NEXT: <a href="https://repository.com/./include/Shape.h">Shape.h</a>
+// HTML-SHAPE-NEXT: </p>
+//      HTML-SHAPE: <div>brief</div>
+//      HTML-SHAPE: <p> Abstract base class for shapes.</p>
+//      HTML-SHAPE: <p> Provides a common interface for different types of shapes.</p>
+//      HTML-SHAPE: <h2 id="Functions">Functions</h2>
+//      HTML-SHAPE: <h3 id="{{([0-9A-F]{40})}}">area</h3>
+//      HTML-SHAPE: <p>public double area()</p>
+//      HTML-SHAPE: <div>brief</div>
+//      HTML-SHAPE: <p> Calculates the area of the shape.</p>
+//      HTML-SHAPE: <h3 id="{{([0-9A-F]{40})}}">perimeter</h3>
+//      HTML-SHAPE: <p>public double perimeter()</p>
+//      HTML-SHAPE: <div>brief</div>
+//      HTML-SHAPE: <p> Calculates the perimeter of the shape.</p>
+//      HTML-SHAPE: <div>return</div>
+//      HTML-SHAPE: <p> double The perimeter of the shape.</p>
+//      HTML-SHAPE: <h3 id="{{([0-9A-F]{40})}}">~Shape</h3>
+//      HTML-SHAPE: <p>public void ~Shape()</p>
+//      HTML-SHAPE: Defined at line 
+// HTML-SHAPE-NEXT: <a href="https://repository.com/./include/Shape.h#13">13</a>
+// HTML-SHAPE-NEXT: of file 
+// HTML-SHAPE-NEXT: <a href="https://repository.com/./include/Shape.h">Shape.h</a>
+//      HTML-SHAPE: <div>brief</div>
+//      HTML-SHAPE: <p> Virtual destructor.</p>
 
-// HTML-CALC: <h1>class Calculator</h1>
-// HTML-CALC: <p>Defined at line 8 of file {{.*}}Calculator.h</p>
-// HTML-CALC: <div>brief</div>
-// HTML-CALC: <p> A simple calculator class.</p>
-// HTML-CALC: <p> Provides basic arithmetic operations.</p>
-// HTML-CALC: <h2 id="Functions">Functions</h2>
-// HTML-CALC: <h3 id="{{([0-9A-F]{40})}}">add</h3>
-// HTML-CALC: <p>public int add(int a, int b)</p>
-// HTML-CALC: <p>Defined at line 3 of file {{.*}}Calculator.cpp</p>
-// HTML-CALC: <div>brief</div>
-// HTML-CALC: <p> Adds two integers.</p>
-// HTML-CALC: <div>return</div>
-// HTML-CALC: <p> int The sum of a and b.</p>
-// HTML-CALC: <h3 id="{{([0-9A-F]{40})}}">subtract</h3>
-// HTML-CALC: <p>public int subtract(int a, int b)</p>
-// HTML-CALC: <p>Defined at line 7 of file {{.*}}Calculator.cpp</p>
-// HTML-CALC: <div>brief</div>
-// HTML-CALC: <p> Subtracts the second integer from the first.</p>
-// HTML-CALC: <div>return</div>
-// HTML-CALC: <p> int The result of a - b.</p>
-// HTML-CALC: <h3 id="{{([0-9A-F]{40})}}">multiply</h3>
-// HTML-CALC: <p>public int multiply(int a, int b)</p>
-// HTML-CALC: <p>Defined at line 11 of file {{.*}}Calculator.cpp</p>
-// HTML-CALC: <div>brief</div>
-// HTML-CALC: <p> Multiplies two integers.</p>
-// HTML-CALC: <div>return</div>
-// HTML-CALC: <p> int The product of a and b.</p>
-// HTML-CALC: <h3 id="{{([0-9A-F]{40})}}">divide</h3>
-// HTML-CALC: <p>public double divide(int a, int b)</p>
-// HTML-CALC: <p>Defined at line 15 of file {{.*}}Calculator.cpp</p>
-// HTML-CALC: <div>brief</div>
-// HTML-CALC: <p> Divides the first integer by the second.</p>
-// HTML-CALC: <div>return</div>
-// HTML-CALC: <p> double The result of a / b.</p>
-// HTML-CALC: <div>throw</div>
-// HTML-CALC: <p>if b is zero.</p>
+//      HTML-CALC: <h1>class Calculator</h1>
+// HTML-CALC-NEXT: <p>
+// HTML-CALC-NEXT: Defined at line 
+// HTML-CALC-NEXT: <a href="https://repository.com/./include/Calculator.h#8">8</a>
+// HTML-CALC-NEXT: of file 
+// HTML-CALC-NEXT: <a href="https://repository.com/./include/Calculator.h">Calculator.h</a>
+// HTML-CALC-NEXT: </p>
+//      HTML-CALC: <div>brief</div>
+//      HTML-CALC: <p> A simple calculator class.</p>
+//      HTML-CALC: <p> Provides basic arithmetic operations.</p>
+//      HTML-CALC: <h2 id="Functions">Functions</h2>
+//      HTML-CALC: <h3 id="{{([0-9A-F]{40})}}">add</h3>
+//      HTML-CALC: <p>public int add(int a, int b)</p>
+//      HTML-CALC: Defined at line 
+// HTML-CALC-NEXT: <a href="https://repository.com/./src/Calculator.cpp#3">3</a>
+// HTML-CALC-NEXT: of file 
+// HTML-CALC-NEXT: <a href="https://repository.com/./src/Calculator.cpp">Calculator.cpp</a>
+//      HTML-CALC: <div>brief</div>
+//      HTML-CALC: <p> Adds two integers.</p>
+//      HTML-CALC: <div>return</div>
+//      HTML-CALC: <p> int The sum of a and b.</p>
+//      HTML-CALC: <h3 id="{{([0-9A-F]{40})}}">subtract</h3>
+//      HTML-CALC: <p>public int subtract(int a, int b)</p>
+//      HTML-CALC: Defined at line 
+// HTML-CALC-NEXT: <a href="https://repository.com/./src/Calculator.cpp#7">7</a>
+// HTML-CALC-NEXT: of file 
+// HTML-CALC-NEXT: <a href="https://repository.com/./src/Calculator.cpp">Calculator.cpp</a>
+//      HTML-CALC: <div>brief</div>
+//      HTML-CALC: <p> Subtracts the second integer from the first.</p>
+//      HTML-CALC: <div>return</div>
+//      HTML-CALC: <p> int The result of a - b.</p>
+//      HTML-CALC: <h3 id="{{([0-9A-F]{40})}}">multiply</h3>
+//      HTML-CALC: <p>public int multiply(int a, int b)</p>
+//      HTML-CALC: Defined at line 
+// HTML-CALC-NEXT: <a href="https://repository.com/./src/Calculator.cpp#11">11</a>
+// HTML-CALC-NEXT: of file 
+// HTML-CALC-NEXT: <a href="https://repository.com/./src/Calculator.cpp">Calculator.cpp</a>
+//      HTML-CALC: <div>brief</div>
+//      HTML-CALC: <p> Multiplies two integers.</p>
+//      HTML-CALC: <div>return</div>
+//      HTML-CALC: <p> int The product of a and b.</p>
+//      HTML-CALC: <h3 id="{{([0-9A-F]{40})}}">divide</h3>
+//      HTML-CALC: <p>public double divide(int a, int b)</p>
+//      HTML-CALC: Defined at line 
+// HTML-CALC-NEXT: <a href="https://repository.com/./src/Calculator.cpp#15">15</a>
+// HTML-CALC-NEXT: of file 
+// HTML-CALC-NEXT: <a href="https://repository.com/./src/Calculator.cpp">Calculator.cpp</a>
+//      HTML-CALC: <div>brief</div>
+//      HTML-CALC: <p> Divides the first integer by the second.</p>
+//      HTML-CALC: <div>return</div>
+//      HTML-CALC: <p> double The result of a / b.</p>
+//      HTML-CALC: <div>throw</div>
+//      HTML-CALC: <p>if b is zero.</p>
 
-// HTML-RECTANGLE: <h1>class Rectangle</h1>
-// HTML-RECTANGLE: <p>Defined at line 10 of file {{.*}}Rectangle.h</p>
-// HTML-RECTANGLE: <p> Represents a rectangle with a given width and height.</p
-// HTML-RECTANGLE: <p>
-// HTML-RECTANGLE:   Inherits from
-// HTML-RECTANGLE:   <a href="Shape.html">Shape</a>
-// HTML-RECTANGLE: </p>
-// HTML-RECTANGLE: <h2 id="Members">Members</h2>
-// HTML-RECTANGLE: <p> Width of the rectangle.</p>
-// HTML-RECTANGLE: <div>private double width_</div>
-// HTML-RECTANGLE: <p> Height of the rectangle.</p>
-// HTML-RECTANGLE: <div>private double height_</div>
-// HTML-RECTANGLE: <h2 id="Functions">Functions</h2>
-// HTML-RECTANGLE: <h3 id="{{([0-9A-F]{40})}}">Rectangle</h3>
-// HTML-RECTANGLE: <p>public void Rectangle(double width, double height)</p>
-// HTML-RECTANGLE: <p>Defined at line 3 of file {{.*}}Rectangle.cpp</p>
-// HTML-RECTANGLE: <div>brief</div>
-// HTML-RECTANGLE: <p> Constructs a new Rectangle object.</p>
-// HTML-RECTANGLE: <h3 id="{{([0-9A-F]{40})}}">area</h3>
-// HTML-RECTANGLE: <p>public double area()</p>
-// HTML-RECTANGLE: <p>Defined at line 6 of file {{.*}}Rectangle.cpp</p>
-// HTML-RECTANGLE: <div>brief</div>
-// HTML-RECTANGLE: <p> Calculates the area of the rectangle.</p>
-// HTML-RECTANGLE: <div>return</div>
-// HTML-RECTANGLE: <p> double The area of the rectangle.</p>
-// HTML-RECTANGLE: <h3 id="{{([0-9A-F]{40})}}">perimeter</h3>
-// HTML-RECTANGLE: <p>public double perimeter()</p>
-// HTML-RECTANGLE: <p>Defined at line 10 of file {{.*}}Rectangle.cpp</p>
-// HTML-RECTANGLE: <div>brief</div>
-// HTML-RECTANGLE: <p> Calculates the perimeter of the rectangle.</p>
-// HTML-RECTANGLE: <div>return</div>
-// HTML-RECTANGLE: <p> double The perimeter of the rectangle.</p>
+//      HTML-RECTANGLE: <h1>class Rectangle</h1>
+// HTML-RECTANGLE-NEXT: <p>
+// HTML-RECTANGLE-NEXT: Defined at line
+// HTML-RECTANGLE-NEXT: <a href="https://repository.com/./include/Rectangle.h#10">10</a>
+// HTML-RECTANGLE-NEXT: of file 
+// HTML-RECTANGLE-NEXT: <a href="https://repository.com/./include/Rectangle.h">Rectangle.h</a>
+// HTML-RECTANGLE-NEXT: </p>
+//      HTML-RECTANGLE: <p> Represents a rectangle with a given width and height.</p>
+//      HTML-RECTANGLE: <p>
+//      HTML-RECTANGLE:   Inherits from
+//      HTML-RECTANGLE:   <a href="Shape.html">Shape</a>
+//      HTML-RECTANGLE: </p>
+//      HTML-RECTANGLE: <h2 id="Members">Members</h2>
+//      HTML-RECTANGLE: <p> Width of the rectangle.</p>
+//      HTML-RECTANGLE: <div>private double width_</div>
+//      HTML-RECTANGLE: <p> Height of the rectangle.</p>
+//      HTML-RECTANGLE: <div>private double height_</div>
+//      HTML-RECTANGLE: <h2 id="Functions">Functions</h2>
+//      HTML-RECTANGLE: <h3 id="{{([0-9A-F]{40})}}">Rectangle</h3>
+//      HTML-RECTANGLE: <p>public void Rectangle(double width, double height)</p>
+//      HTML-RECTANGLE: Defined at line
+// HTML-RECTANGLE-NEXT: <a href="https://repository.com/./src/Rectangle.cpp#3">3</a>
+// HTML-RECTANGLE-NEXT: of file 
+// HTML-RECTANGLE-NEXT: <a href="https://repository.com/./src/Rectangle.cpp">Rectangle.cpp</a>
+//      HTML-RECTANGLE: <div>brief</div>
+//      HTML-RECTANGLE: <p> Constructs a new Rectangle object.</p>
+//      HTML-RECTANGLE: <h3 id="{{([0-9A-F]{40})}}">area</h3>
+//      HTML-RECTANGLE: <p>public double area()</p>
+//      HTML-RECTANGLE: Defined at line
+// HTML-RECTANGLE-NEXT: <a href="https://repository.com/./src/Rectangle.cpp#6">6</a>
+// HTML-RECTANGLE-NEXT: of file 
+// HTML-RECTANGLE-NEXT: <a href="https://repository.com/./src/Rectangle.cpp">Rectangle.cpp</a>
+//      HTML-RECTANGLE: <div>brief</div>
+//      HTML-RECTANGLE: <p> Calculates the area of the rectangle.</p>
+//      HTML-RECTANGLE: <div>return</div>
+//      HTML-RECTANGLE: <p> double The area of the rectangle.</p>
+//      HTML-RECTANGLE: <h3 id="{{([0-9A-F]{40})}}">perimeter</h3>
+//      HTML-RECTANGLE: <p>public double perimeter()</p>
+//      HTML-RECTANGLE: Defined at line
+// HTML-RECTANGLE-NEXT: <a href="https://repository.com/./src/Rectangle.cpp#10">10</a>
+// HTML-RECTANGLE-NEXT: of file 
+// HTML-RECTANGLE-NEXT: <a href="https://repository.com/./src/Rectangle.cpp">Rectangle.cpp</a>
+//      HTML-RECTANGLE: <div>brief</div>
+//      HTML-RECTANGLE: <p> Calculates the perimeter of the rectangle.</p>
+//      HTML-RECTANGLE: <div>return</div>
+//      HTML-RECTANGLE: <p> double The perimeter of the rectangle.</p>
 
-// HTML-CIRCLE: <h1>class Circle</h1>
-// HTML-CIRCLE: <p>Defined at line 10 of file {{.*}}Circle.h</p>
-// HTML-CIRCLE: <div>brief</div>
-// HTML-CIRCLE: <p> Circle class derived from Shape.</p>
-// HTML-CIRCLE: <p> Represents a circle with a given radius.</p>
-// HTML-CIRCLE: <p>
-// HTML-CIRCLE:   Inherits from
-// HTML-CIRCLE:   <a href="Shape.html">Shape</a>
-// HTML-CIRCLE: </p>
-// HTML-CIRCLE: <h2 id="Members">Members</h2>
-// HTML-CIRCLE: <p> Radius of the circle.</p>
-// HTML-CIRCLE: <div>private double radius_</div>
-// HTML-CIRCLE: <h2 id="Functions">Functions</h2>
-// HTML-CIRCLE: <h3 id="{{([0-9A-F]{40})}}">Circle</h3>
-// HTML-CIRCLE: <p>public void Circle(double radius)</p>
-// HTML-CIRCLE: <p>Defined at line 3 of file {{.*}}Circle.cpp</p>
-// HTML-CIRCLE: <div>brief</div>
-// HTML-CIRCLE: <p> Constructs a new Circle object.</p>
-// HTML-CIRCLE: <h3 id="{{([0-9A-F]{40})}}">area</h3>
-// HTML-CIRCLE: <p>public double area()</p>
-// HTML-CIRCLE: <p>Defined at line 5 of file {{.*}}Circle.cpp</p>
-// HTML-CIRCLE: <div>brief</div>
-// HTML-CIRCLE: <p> Calculates the area of the circle.</p>
-// HTML-CIRCLE: <div>return</div>
-// HTML-CIRCLE: <p> double The area of the circle.</p>
-// HTML-CIRCLE: <h3 id="{{([0-9A-F]{40})}}">perimeter</h3>
-// HTML-CIRCLE: <p>public double perimeter()</p>
-// HTML-CIRCLE: <p>Defined at line 9 of file {{.*}}Circle.cpp</p>
-// HTML-CIRCLE: <div>brief</div>
-// HTML-CIRCLE: <p> Calculates the perimeter of the circle.</p>
-// HTML-CIRCLE: <div>return</div>
-// HTML-CIRCLE: <p> double The perimeter of the circle.</p>
+//      HTML-CIRCLE: <h1>class Circle</h1>
+// HTML-CIRCLE-NEXT: <p>
+// HTML-CIRCLE-NEXT: Defined at line 
+// HTML-CIRCLE-NEXT: <a href="https://repository.com/./include/Circle.h#10">10</a>
+// HTML-CIRCLE-NEXT: of file 
+// HTML-CIRCLE-NEXT: <a href="https://repository.com/./include/Circle.h">Circle.h</a>
+// HTML-CIRCLE-NEXT: </p>
+//      HTML-CIRCLE: <div>brief</div>
+//      HTML-CIRCLE: <p> Circle class derived from Shape.</p>
+//      HTML-CIRCLE: <p> Represents a circle with a given radius.</p>
+//      HTML-CIRCLE: <p>
+//      HTML-CIRCLE:   Inherits from
+//      HTML-CIRCLE:   <a href="Shape.html">Shape</a>
+//      HTML-CIRCLE: </p>
+//      HTML-CIRCLE: <h2 id="Members">Members</h2>
+//      HTML-CIRCLE: <p> Radius of the circle.</p>
+//      HTML-CIRCLE: <div>private double radius_</div>
+//      HTML-CIRCLE: <h2 id="Functions">Functions</h2>
+//      HTML-CIRCLE: <h3 id="{{([0-9A-F]{40})}}">Circle</h3>
+//      HTML-CIRCLE: <p>public void Circle(double radius)</p>
+//      HTML-CIRCLE: Defined at line 
+// HTML-CIRCLE-NEXT: <a href="https://repository.com/./src/Circle.cpp#3">3</a>
+// HTML-CIRCLE-NEXT:  of file 
+// HTML-CIRCLE-NEXT: <a href="https://repository.com/./src/Circle.cpp">Circle.cpp</a>
+//      HTML-CIRCLE: <div>brief</div>
+//      HTML-CIRCLE: <p> Constructs a new Circle object.</p>
+//      HTML-CIRCLE: <h3 id="{{([0-9A-F]{40})}}">area</h3>
+//      HTML-CIRCLE: <p>public double area()</p>
+//      HTML-CIRCLE: Defined at line 
+// HTML-CIRCLE-NEXT: <a href="https://repository.com/./src/Circle.cpp#5">5</a>
+// HTML-CIRCLE-NEXT:  of file 
+// HTML-CIRCLE-NEXT: <a href="https://repository.com/./src/Circle.cpp">Circle.cpp</a>
+//      HTML-CIRCLE: <div>brief</div>
+//      HTML-CIRCLE: <p> Calculates the area of the circle.</p>
+//      HTML-CIRCLE: <div>return</div>
+//      HTML-CIRCLE: <p> double The area of the circle.</p>
+//      HTML-CIRCLE: <h3 id="{{([0-9A-F]{40})}}">perimeter</h3>
+//      HTML-CIRCLE: <p>public double perimeter()</p>
+//      HTML-CIRCLE: Defined at line 
+// HTML-CIRCLE-NEXT: <a href="https://repository.com/./src/Circle.cpp#9">9</a>
+// HTML-CIRCLE-NEXT:  of file 
+// HTML-CIRCLE-NEXT: <a href="https://repository.com/./src/Circle.cpp">Circle.cpp</a>
+//      HTML-CIRCLE: <div>brief</div>
+//      HTML-CIRCLE: <p> Calculates the perimeter of the circle.</p>
+//      HTML-CIRCLE: <div>return</div>
+//      HTML-CIRCLE: <p> double The perimeter of the circle.</p>
 
 // MD-CALC: # class Calculator
 // MD-CALC: *Defined at .{{[\/]}}include{{[\/]}}Calculator.h#8*
@@ -286,4 +339,4 @@
 // MD-ALL-FILES: ## [GlobalNamespace](GlobalNamespace{{[\/]}}index.md)
 
 // MD-INDEX: #  C/C++ Reference
-// MD-INDEX: * Namespace: [GlobalNamespace](GlobalNamespace)
\ No newline at end of file
+// MD-INDEX: * Namespace: [GlobalNamespace](GlobalNamespace)
diff --git a/clang-tools-extra/test/clang-tidy/checkers/bugprone/string-constructor.cpp b/clang-tools-extra/test/clang-tidy/checkers/bugprone/string-constructor.cpp
index a5b6b240ddc66..2576d19916250 100644
--- a/clang-tools-extra/test/clang-tidy/checkers/bugprone/string-constructor.cpp
+++ b/clang-tools-extra/test/clang-tidy/checkers/bugprone/string-constructor.cpp
@@ -11,6 +11,7 @@ struct basic_string {
   basic_string(const C*, unsigned int size);
   basic_string(const C *, const A &allocator = A());
   basic_string(unsigned int size, C c);
+  basic_string(const C*, unsigned int pos, unsigned int size);
 };
 typedef basic_string<char> string;
 typedef basic_string<wchar_t> wstring;
@@ -61,6 +62,21 @@ void Test() {
   // CHECK-MESSAGES: [[@LINE-1]]:15: warning: constructing string from nullptr is undefined behaviour
   std::string q7 = 0;
   // CHECK-MESSAGES: [[@LINE-1]]:20: warning: constructing string from nullptr is undefined behaviour
+
+  std::string r1("test", 1, 0);
+  // CHECK-MESSAGES: [[@LINE-1]]:15: warning: constructor creating an empty string
+  std::string r2("test", 0, -4);
+  // CHECK-MESSAGES: [[@LINE-1]]:15: warning: negative value used as length parameter
+  std::string r3("test", -4, 1); 
+  // CHECK-MESSAGES: [[@LINE-1]]:15: warning: negative value used as position of the first character parameter
+  std::string r4("test", 0, 0x1000000);
+  // CHECK-MESSAGES: [[@LINE-1]]:15: warning: suspicious large length parameter
+  std::string r5("test", 0, 5);
+  // CHECK-MESSAGES: [[@LINE-1]]:15: warning: length is bigger than string literal size
+  std::string r6("test", 3, 2);
+  // CHECK-MESSAGES: [[@LINE-1]]:15: warning: length is bigger than remaining string literal size
+  std::string r7("test", 4, 1);
+  // CHECK-MESSAGES: [[@LINE-1]]:15: warning: position of the first character parameter is bigger than string literal character range
 }
 
 void TestView() {
@@ -82,6 +98,17 @@ void TestView() {
   // CHECK-MESSAGES: [[@LINE-1]]:25: warning: constructing string from nullptr is undefined behaviour
 }
 
+void TestUnsignedArguments() {
+  std::string s0("test", 0u);
+  // CHECK-MESSAGES: [[@LINE-1]]:15: warning: constructor creating an empty string
+  std::string s1(0x1000000ull, 'x');
+  // CHECK-MESSAGES: [[@LINE-1]]:15: warning: suspicious large length parameter
+  std::string s2("test", 3ull, 2u);
+  // CHECK-MESSAGES: [[@LINE-1]]:15: warning: length is bigger than remaining string literal size
+  std::string s3("test", 0u, 5ll);
+  // CHECK-MESSAGES: [[@LINE-1]]:15: warning: length is bigger than string literal size
+}
+
 std::string StringFromZero() {
   return 0;
   // CHECK-MESSAGES: [[@LINE-1]]:10: warning: constructing string from nullptr is undefined behaviour
@@ -101,6 +128,9 @@ void Valid() {
   std::string s3("test");
   std::string s4("test\000", 5);
   std::string s6("te" "st", 4);
+  std::string s7("test", 0, 4);
+  std::string s8("test", 3, 1);
+  std::string s9("te" "st", 1, 2);
 
   std::string_view emptyv();
   std::string_view sv1("test", 4);
diff --git a/clang-tools-extra/test/clang-tidy/checkers/bugprone/unsafe-functions-custom-regex.cpp b/clang-tools-extra/test/clang-tidy/checkers/bugprone/unsafe-functions-custom-regex.cpp
index fc97d1bc93bc5..ad0ba8739be2b 100644
--- a/clang-tools-extra/test/clang-tidy/checkers/bugprone/unsafe-functions-custom-regex.cpp
+++ b/clang-tools-extra/test/clang-tidy/checkers/bugprone/unsafe-functions-custom-regex.cpp
@@ -1,11 +1,19 @@
 // RUN: %check_clang_tidy -check-suffix=NON-STRICT-REGEX         %s bugprone-unsafe-functions %t --\
-// RUN:   -config="{CheckOptions: {bugprone-unsafe-functions.CustomFunctions: '::name_match,replacement,is a qualname match;^::prefix_match,,is matched on qualname prefix'}}"
+// RUN:   -config="{CheckOptions: {bugprone-unsafe-functions.CustomFunctions: '::name_match,replacement,is a qualname match;^::prefix_match,,is matched on qualname prefix;^::S::member_match_,,is matched on a C++ class member'}}"
 // RUN: %check_clang_tidy -check-suffix=STRICT-REGEX         %s bugprone-unsafe-functions %t --\
-// RUN:   -config="{CheckOptions: {bugprone-unsafe-functions.CustomFunctions: '^name_match$,replacement,is matched on function name only;^::prefix_match$,,is a full qualname match'}}"
+// RUN:   -config="{CheckOptions: {bugprone-unsafe-functions.CustomFunctions: '^name_match$,replacement,is matched on function name only;^::prefix_match$,,is a full qualname match;^::S::member_match_1$,,is matched on a C++ class member'}}"
 
 void name_match();
 void prefix_match();
 
+struct S {
+  static void member_match_1() {}
+  void member_match_2() {}
+};
+
+void member_match_1() {}
+void member_match_unmatched() {}
+
 namespace regex_test {
 void name_match();
 void prefix_match();
@@ -42,3 +50,25 @@ void f1() {
   // CHECK-MESSAGES-NON-STRICT-REGEX: :[[@LINE-1]]:3: warning: function 'prefix_match_regex' is matched on qualname prefix; it should not be used
   // no-warning STRICT-REGEX
 }
+
+void f2() {
+  S s;
+
+  S::member_match_1();
+  // CHECK-MESSAGES-NON-STRICT-REGEX: :[[@LINE-1]]:3: warning: function 'member_match_1' is matched on a C++ class member; it should not be used
+  // CHECK-MESSAGES-STRICT-REGEX: :[[@LINE-2]]:3: warning: function 'member_match_1' is matched on a C++ class member; it should not be used
+
+  s.member_match_1();
+  // CHECK-MESSAGES-NON-STRICT-REGEX: :[[@LINE-1]]:5: warning: function 'member_match_1' is matched on a C++ class member; it should not be used
+  // CHECK-MESSAGES-STRICT-REGEX: :[[@LINE-2]]:5: warning: function 'member_match_1' is matched on a C++ class member; it should not be used
+
+  s.member_match_2();
+  // CHECK-MESSAGES-NON-STRICT-REGEX: :[[@LINE-1]]:5: warning: function 'member_match_2' is matched on a C++ class member; it should not be used
+  // no-warning STRICT-REGEX
+
+  member_match_1();
+  // no-warning
+
+  member_match_unmatched();
+  // no-warning
+}
diff --git a/clang-tools-extra/test/clang-tidy/checkers/misc/redundant-expression.cpp b/clang-tools-extra/test/clang-tidy/checkers/misc/redundant-expression.cpp
index 7396d2dce76c4..95d8ecb15a4f1 100644
--- a/clang-tools-extra/test/clang-tidy/checkers/misc/redundant-expression.cpp
+++ b/clang-tools-extra/test/clang-tidy/checkers/misc/redundant-expression.cpp
@@ -1,4 +1,5 @@
 // RUN: %check_clang_tidy %s misc-redundant-expression %t -- -- -fno-delayed-template-parsing -Wno-array-compare-cxx26
+// RUN: %check_clang_tidy %s misc-redundant-expression %t -- -- -fno-delayed-template-parsing -Wno-array-compare-cxx26 -DTEST_MACRO
 
 typedef __INT64_TYPE__ I64;
 
@@ -91,6 +92,223 @@ int TestSimpleEquivalent(int X, int Y) {
   return 0;
 }
 
+#ifndef TEST_MACRO
+#define VAL_1 2
+#define VAL_3 3
+#else
+#define VAL_1 3
+#define VAL_3 2
+#endif
+
+#define VAL_2 2
+
+#ifndef TEST_MACRO
+#define VAL_4 2 + 1
+#define VAL_6 3 + 1
+#else
+#define VAL_4 3 + 1
+#define VAL_6 2 + 1
+#endif
+
+#define VAL_5 2 + 1
+
+struct TestStruct
+{
+  int mA;
+  int mB;
+  int mC[10];
+};
+
+int TestDefineEquivalent() {
+
+  int int_val1 = 3;
+  int int_val2 = 4;
+  int int_val = 0;
+  const int cint_val2 = 4;
+
+  // Cases which should not be reported
+  if (VAL_1 != VAL_2)  return 0;
+  if (VAL_3 != VAL_2)  return 0;
+  if (VAL_1 == VAL_2)  return 0;
+  if (VAL_3 == VAL_2)  return 0;
+  if (VAL_1 >= VAL_2)  return 0;
+  if (VAL_3 >= VAL_2)  return 0;
+  if (VAL_1 <= VAL_2)  return 0;
+  if (VAL_3 <= VAL_2)  return 0;
+  if (VAL_1 < VAL_2)  return 0;
+  if (VAL_3 < VAL_2)  return 0;
+  if (VAL_1 > VAL_2)  return 0;
+  if (VAL_3 > VAL_2)  return 0;
+
+  if (VAL_4 != VAL_5)  return 0;
+  if (VAL_6 != VAL_5)  return 0;
+  if (VAL_6 == VAL_5)  return 0;
+  if (VAL_4 >= VAL_5)  return 0;
+  if (VAL_6 >= VAL_5)  return 0;
+  if (VAL_4 <= VAL_5)  return 0;
+  if (VAL_6 <= VAL_5)  return 0;
+  if (VAL_4 > VAL_5)  return 0;
+  if (VAL_6 > VAL_5)  return 0;
+  if (VAL_4 < VAL_5)  return 0;
+  if (VAL_6 < VAL_5)  return 0;
+
+  if (VAL_1 != 2)  return 0;
+  if (VAL_3 == 3)  return 0;
+
+  if (VAL_1 >= VAL_1)  return 0;
+  // CHECK-MESSAGES: :[[@LINE-1]]:13: warning: both sides of operator are equivalent
+  if (VAL_2 <= VAL_2)  return 0;
+  // CHECK-MESSAGES: :[[@LINE-1]]:13: warning: both sides of operator are equivalent
+  if (VAL_3 > VAL_3)  return 0;
+  // CHECK-MESSAGES: :[[@LINE-1]]:13: warning: both sides of operator are equivalent
+  if (VAL_4 < VAL_4)  return 0;
+  // CHECK-MESSAGES: :[[@LINE-1]]:13: warning: both sides of operator are equivalent
+  if (VAL_6 == VAL_6) return 2;
+  // CHECK-MESSAGES: :[[@LINE-1]]:13: warning: both sides of operator are equivalent
+  if (VAL_5 != VAL_5) return 2;
+  // CHECK-MESSAGES: :[[@LINE-1]]:13: warning: both sides of operator are equivalent
+
+  // Test prefixes
+  if (+VAL_6 == +VAL_6) return 2;
+  // CHECK-MESSAGES: :[[@LINE-1]]:14: warning: both sides of operator are equivalent
+  if (-VAL_6 == -VAL_6) return 2;
+  // CHECK-MESSAGES: :[[@LINE-1]]:14: warning: both sides of operator are equivalent
+  if ((+VAL_6) == (+VAL_6)) return 2;
+  // CHECK-MESSAGES: :[[@LINE-1]]:16: warning: both sides of operator are equivalent
+  if ((-VAL_6) == (-VAL_6)) return 2;
+  // CHECK-MESSAGES: :[[@LINE-1]]:16: warning: both sides of operator are equivalent
+
+  if (1 >= 1)  return 0;
+  // CHECK-MESSAGES: :[[@LINE-1]]:9: warning: both sides of operator are equivalent
+  if (0xFF <= 0xFF)  return 0;
+  // CHECK-MESSAGES: :[[@LINE-1]]:12: warning: both sides of operator are equivalent
+  if (042 > 042)  return 0;
+  // CHECK-MESSAGES: :[[@LINE-1]]:11: warning: both sides of operator are equivalent
+
+  int_val = (VAL_6 == VAL_6)?int_val1: int_val2;
+  // CHECK-MESSAGES: :[[@LINE-1]]:20: warning: both sides of operator are equivalent
+  int_val = (042 > 042)?int_val1: int_val2;
+  // CHECK-MESSAGES: :[[@LINE-1]]:18: warning: both sides of operator are equivalent
+
+
+  // Ternary operator cases which should not be reported
+  int_val = (VAL_4 == VAL_5)? int_val1: int_val2;
+  int_val = (VAL_3 != VAL_2)? int_val1: int_val2;
+  int_val = (VAL_6 != 10)? int_val1: int_val2;
+  int_val = (VAL_6 != 3)? int_val1: int_val2;
+  int_val = (VAL_6 != 4)? int_val1: int_val2;
+  int_val = (VAL_6 == 3)? int_val1: int_val2;
+  int_val = (VAL_6 == 4)? int_val1: int_val2;
+
+  TestStruct tsVar1 = {
+    .mA = 3,
+    .mB = int_val,
+    .mC[0 ... VAL_2 - 2] = int_val + 1,
+  };
+
+  TestStruct tsVar2 = {
+    .mA = 3,
+    .mB = int_val,
+    .mC[0 ... cint_val2 - 2] = int_val + 1,
+  };
+
+  TestStruct tsVar3 = {
+    .mA = 3,
+    .mB = int_val,
+    .mC[0 ... VAL_3 - VAL_3] = int_val + 1,
+    // CHECK-MESSAGES: :[[@LINE-1]]:21: warning: both sides of operator are equivalent
+  };
+
+  TestStruct tsVar4 = {
+    .mA = 3,
+    .mB = int_val,
+    .mC[0 ... 5 - 5] = int_val + 1,
+    // CHECK-MESSAGES: :[[@LINE-1]]:17: warning: both sides of operator are equivalent
+  };
+
+  return 1 + int_val + sizeof(tsVar1) + sizeof(tsVar2) +
+         sizeof(tsVar3) + sizeof(tsVar4);
+}
+
+#define LOOP_DEFINE 1
+
+unsigned int testLoops(const unsigned int  arr1[LOOP_DEFINE])
+{
+  unsigned int localIndex;
+  for (localIndex = LOOP_DEFINE - 1; localIndex > 0; localIndex--)
+  {
+  }
+  for (localIndex = LOOP_DEFINE - 1; 10 > 10; localIndex--)
+  // CHECK-MESSAGES: :[[@LINE-1]]:41: warning: both sides of operator are equivalent
+  {
+  }
+
+  for (localIndex = LOOP_DEFINE - 1; LOOP_DEFINE > LOOP_DEFINE; localIndex--)
+  // CHECK-MESSAGES: :[[@LINE-1]]:50: warning: both sides of operator are equivalent
+  {
+  }
+
+  return localIndex;
+}
+
+#define getValue(a) a
+#define getValueM(a) a
+
+int TestParamDefine() {
+  int ret = 0;
+
+  // Negative cases
+  ret += getValue(VAL_6) == getValue(2);
+  ret += getValue(VAL_6) == getValue(3);
+  ret += getValue(VAL_5) == getValue(2);
+  ret += getValue(VAL_5) == getValue(3);
+  ret += getValue(1) > getValue( 2);
+  ret += getValue(VAL_1) == getValue(VAL_2);
+  ret += getValue(VAL_1) != getValue(VAL_2);
+  ret += getValue(VAL_1) == getValueM(VAL_1);
+  ret += getValue(VAL_1 + VAL_2) == getValueM(VAL_1 + VAL_2);
+  ret += getValue(1) == getValueM(1);
+  ret += getValue(false) == getValueM(false);
+  ret += -getValue(1) > +getValue( 1);
+
+  // Positive cases
+  ret += (+getValue(1)) > (+getValue( 1));
+  // CHECK-MESSAGES: :[[@LINE-1]]:25: warning: both sides of operator are equivalent
+  ret += (-getValue(1)) > (-getValue( 1));
+  // CHECK-MESSAGES: :[[@LINE-1]]:25: warning: both sides of operator are equivalent
+
+  ret += +getValue(1) > +getValue( 1);
+  // CHECK-MESSAGES: :[[@LINE-1]]:23: warning: both sides of operator are equivalent
+  ret += -getValue(1) > -getValue( 1);
+  // CHECK-MESSAGES: :[[@LINE-1]]:23: warning: both sides of operator are equivalent
+
+  ret += getValue(1) > getValue( 1);
+  // CHECK-MESSAGES: :[[@LINE-1]]:22: warning: both sides of operator are equivalent
+  ret += getValue(1) > getValue( 1  );
+  // CHECK-MESSAGES: :[[@LINE-1]]:22: warning: both sides of operator are equivalent
+  ret += getValue(1) > getValue(  1);
+  // CHECK-MESSAGES: :[[@LINE-1]]:22: warning: both sides of operator are equivalent
+  ret += getValue(     1) > getValue( 1);
+  // CHECK-MESSAGES: :[[@LINE-1]]:27: warning: both sides of operator are equivalent
+  ret += getValue( 1     ) > getValue( 1);
+  // CHECK-MESSAGES: :[[@LINE-1]]:28: warning: both sides of operator are equivalent
+  ret += getValue( VAL_5     ) > getValue(VAL_5);
+  // CHECK-MESSAGES: :[[@LINE-1]]:32: warning: both sides of operator are equivalent
+  ret += getValue( VAL_5     ) > getValue( VAL_5);
+  // CHECK-MESSAGES: :[[@LINE-1]]:32: warning: both sides of operator are equivalent
+  ret += getValue( VAL_5     ) > getValue( VAL_5  )  ;
+  // CHECK-MESSAGES: :[[@LINE-1]]:32: warning: both sides of operator are equivalent
+  ret += getValue(VAL_5) > getValue( VAL_5  )  ;
+  // CHECK-MESSAGES: :[[@LINE-1]]:26: warning: both sides of operator are equivalent
+  ret += getValue(VAL_1+VAL_2) > getValue(VAL_1 + VAL_2)  ;
+  // CHECK-MESSAGES: :[[@LINE-1]]:32: warning: both sides of operator are equivalent
+  ret += getValue(VAL_1)+getValue(VAL_2) > getValue(VAL_1) + getValue( VAL_2)  ;
+  // CHECK-MESSAGES: :[[@LINE-1]]:42: warning: both sides of operator are equivalent
+  ret += (getValue(VAL_1)+getValue(VAL_2)) > (getValue(VAL_1) + getValue( VAL_2) ) ;
+  // CHECK-MESSAGES: :[[@LINE-1]]:44: warning: both sides of operator are equivalent
+  return ret;
+}
+
 template <int DX>
 int TestSimpleEquivalentDependent() {
   if (DX > 0 && DX > 0) return 1;
diff --git a/clang-tools-extra/unittests/clang-doc/HTMLGeneratorTest.cpp b/clang-tools-extra/unittests/clang-doc/HTMLGeneratorTest.cpp
index dfd31e6578714..97afa12cab6d3 100644
--- a/clang-tools-extra/unittests/clang-doc/HTMLGeneratorTest.cpp
+++ b/clang-tools-extra/unittests/clang-doc/HTMLGeneratorTest.cpp
@@ -319,7 +319,12 @@ TEST(HTMLGeneratorTest, emitFunctionHTML) {
       <a href="path/to/int.html">int</a>
        P)
     </p>
-    <p>Defined at line 10 of file dir/test.cpp</p>
+    <p>
+      Defined at line 
+      <a href="https://www.repository.com/dir/test.cpp#10">10</a>
+       of file 
+      <a href="https://www.repository.com/dir/test.cpp">test.cpp</a>
+    </p>
   </div>
   <div id="sidebar-right" class="col-xs-6 col-sm-6 col-md-2 sidebar sidebar-offcanvas-right"></div>
 </main>
diff --git a/clang/CMakeLists.txt b/clang/CMakeLists.txt
index b79e570667b2c..c3f30e2a8e9c0 100644
--- a/clang/CMakeLists.txt
+++ b/clang/CMakeLists.txt
@@ -331,6 +331,7 @@ if(NOT DEFINED CLANG_VERSION_SUFFIX)
   set(CLANG_VERSION_SUFFIX ${LLVM_VERSION_SUFFIX})
 endif()
 set(CLANG_VERSION "${CLANG_VERSION_MAJOR}.${CLANG_VERSION_MINOR}.${CLANG_VERSION_PATCHLEVEL}${CLANG_VERSION_SUFFIX}")
+set(MAX_CLANG_ABI_COMPAT_VERSION "${LLVM_VERSION_MAJOR}")
 message(STATUS "Clang version: ${CLANG_VERSION}")
 
 # Configure the Version.inc file.
@@ -431,18 +432,16 @@ if(NOT LLVM_STATIC_LINK_CXX_STDLIB)
   set(HAVE_CLANG_REPL_SUPPORT ON)
 endif()
 
-option(CLANG_ENABLE_ARCMT "Build ARCMT." ON)
+option(CLANG_ENABLE_OBJC_REWRITER "Build the Objective-C rewriter tool" OFF)
+
 option(CLANG_ENABLE_STATIC_ANALYZER
   "Include static analyzer in clang binary." ON)
 
 option(CLANG_ENABLE_PROTO_FUZZER "Build Clang protobuf fuzzer." OFF)
 
-if(NOT CLANG_ENABLE_STATIC_ANALYZER AND CLANG_ENABLE_ARCMT)
-  message(FATAL_ERROR "Cannot disable static analyzer while enabling ARCMT or Z3")
-endif()
-
-if(CLANG_ENABLE_ARCMT)
-  set(CLANG_ENABLE_OBJC_REWRITER ON)
+if (DEFINED CLANG_ENABLE_ARCMT)
+  set(CLANG_ENABLE_OBJC_REWRITER ${CLANG_ENABLE_ARCMT})
+  message(DEPRECATION "'CLANG_ENABLE_ARCMT' is deprecated as ARCMigrate has been removed from Clang. Please use 'CLANG_ENABLE_OBJC_REWRITER' instead to enable or disable the Objective-C rewriter.")
 endif()
 
 # This option is a stop-gap, we should commit to removing this as
@@ -872,58 +871,6 @@ if (CLANG_ENABLE_BOOTSTRAP)
   endforeach()
 endif()
 
-set(CLANG_BOLT OFF CACHE STRING "Apply BOLT optimization to Clang. \
-  May be specified as Instrument or Perf or LBR to use a particular profiling \
-  mechanism.")
-string(TOUPPER "${CLANG_BOLT}" CLANG_BOLT)
-
-if (CLANG_BOLT AND NOT LLVM_BUILD_INSTRUMENTED)
-  set(CLANG_PATH ${LLVM_RUNTIME_OUTPUT_INTDIR}/clang)
-  set(CLANG_INSTRUMENTED ${LLVM_RUNTIME_OUTPUT_INTDIR}/${CLANG_BOLT_INSTRUMENTED})
-  set(BOLT_FDATA ${CMAKE_CURRENT_BINARY_DIR}/utils/perf-training/prof.fdata)
-
-  # Pass extra flag in no-LBR mode
-  if (CLANG_BOLT STREQUAL "PERF")
-    set(BOLT_NO_LBR "-nl")
-  endif()
-
-  if (CLANG_BOLT STREQUAL "INSTRUMENT")
-    # Instrument clang with BOLT
-    add_custom_target(clang-instrumented
-      DEPENDS ${CLANG_INSTRUMENTED}
-    )
-    add_custom_command(OUTPUT ${CLANG_INSTRUMENTED}
-      DEPENDS clang llvm-bolt
-      COMMAND llvm-bolt ${CLANG_PATH} -o ${CLANG_INSTRUMENTED}
-        -instrument --instrumentation-file-append-pid
-        --instrumentation-file=${BOLT_FDATA}
-      COMMENT "Instrumenting clang binary with BOLT"
-      USES_TERMINAL
-      VERBATIM
-    )
-    add_custom_target(clang-bolt-training-deps DEPENDS clang-instrumented)
-  else() # perf or LBR
-    add_custom_target(clang-bolt-training-deps DEPENDS clang)
-  endif()
-
-  # Optimize original (pre-bolt) Clang using the collected profile
-  add_custom_target(clang-bolt
-    DEPENDS clang-bolt-profile
-    COMMAND ${CMAKE_COMMAND} -E rename $<TARGET_FILE:clang> ${CLANG_PATH}-prebolt
-    COMMAND ${CMAKE_COMMAND} -E create_symlink ${CLANG_PATH}-prebolt ${CLANG_PATH}++-prebolt
-    COMMAND llvm-bolt ${CLANG_PATH}-prebolt
-      -o $<TARGET_FILE:clang>
-      -data ${BOLT_FDATA}
-      -reorder-blocks=ext-tsp -reorder-functions=cdsort -split-functions
-      -split-all-cold -split-eh -dyno-stats -use-gnu-stack
-      -update-debug-sections
-      ${BOLT_NO_LBR}
-    COMMENT "Optimizing Clang with BOLT"
-    USES_TERMINAL
-    VERBATIM
-  )
-endif()
-
 if (LLVM_ADD_NATIVE_VISUALIZERS_TO_SOLUTION)
   add_subdirectory(utils/ClangVisualizers)
 endif()
diff --git a/clang/cmake/caches/Android.cmake b/clang/cmake/caches/Android.cmake
index d5ca6b50d4ada..c89e31f67cc9b 100644
--- a/clang/cmake/caches/Android.cmake
+++ b/clang/cmake/caches/Android.cmake
@@ -2,7 +2,6 @@
 
 set(LLVM_TARGETS_TO_BUILD X86 CACHE STRING "")
 
-set(CLANG_ENABLE_ARCMT OFF CACHE BOOL "")
 set(CLANG_ENABLE_STATIC_ANALYZER OFF CACHE BOOL "")
 set(CLANG_TIDY_ENABLE_STATIC_ANALYZER OFF CACHE BOOL "")
 set(CLANG_VENDOR Android CACHE STRING "")
diff --git a/clang/cmake/caches/Fuchsia-stage2.cmake b/clang/cmake/caches/Fuchsia-stage2.cmake
index 1cbf691f29d58..99890b8246ad7 100644
--- a/clang/cmake/caches/Fuchsia-stage2.cmake
+++ b/clang/cmake/caches/Fuchsia-stage2.cmake
@@ -44,7 +44,6 @@ set(CLANG_DEFAULT_LINKER lld CACHE STRING "")
 set(CLANG_DEFAULT_OBJCOPY llvm-objcopy CACHE STRING "")
 set(CLANG_DEFAULT_RTLIB compiler-rt CACHE STRING "")
 set(CLANG_DEFAULT_UNWINDLIB libunwind CACHE STRING "")
-set(CLANG_ENABLE_ARCMT OFF CACHE BOOL "")
 set(CLANG_ENABLE_STATIC_ANALYZER ON CACHE BOOL "")
 set(CLANG_PLUGIN_SUPPORT OFF CACHE BOOL "")
 
@@ -190,6 +189,10 @@ foreach(target aarch64-unknown-linux-gnu;armv7-unknown-linux-gnueabihf;i386-unkn
     set(RUNTIMES_${target}_LLVM_TOOLS_DIR "${CMAKE_BINARY_DIR}/bin" CACHE BOOL "")
     set(RUNTIMES_${target}_LLVM_ENABLE_RUNTIMES "compiler-rt;libcxx;libcxxabi;libunwind" CACHE STRING "")
 
+    # Enable FatLTO for Linux and baremetal runtimes
+    set(RUNTIMES_${target}_LLVM_ENABLE_LTO OFF CACHE BOOL "")
+    set(RUNTIMES_${target}_LLVM_ENABLE_FATLTO OFF CACHE BOOL "")
+
     # Use .build-id link.
     list(APPEND RUNTIME_BUILD_ID_LINK "${target}")
   endif()
@@ -272,6 +275,10 @@ if(FUCHSIA_SDK)
     set(RUNTIMES_${target}+asan+noexcept_LIBCXXABI_ENABLE_EXCEPTIONS OFF CACHE BOOL "")
     set(RUNTIMES_${target}+asan+noexcept_LIBCXX_ENABLE_EXCEPTIONS OFF CACHE BOOL "")
 
+    # Enable FatLTO for Fuchsia runtimes
+    set(RUNTIMES_${target}_LLVM_ENABLE_LTO OFF CACHE BOOL "")
+    set(RUNTIMES_${target}_LLVM_ENABLE_FATLTO OFF CACHE BOOL "")
+
     # Use .build-id link.
     list(APPEND RUNTIME_BUILD_ID_LINK "${target}")
   endforeach()
@@ -369,6 +376,10 @@ foreach(target armv6m-none-eabi;armv7m-none-eabi;armv7em-none-eabi;armv8m.main-n
   set(RUNTIMES_${target}_LLVM_INCLUDE_TESTS OFF CACHE BOOL "")
   set(RUNTIMES_${target}_LLVM_ENABLE_ASSERTIONS OFF CACHE BOOL "")
   set(RUNTIMES_${target}_LLVM_ENABLE_RUNTIMES "libc;libcxx" CACHE STRING "")
+
+  # Enable FatLTO for baremetal runtimes
+  set(RUNTIMES_${target}_LLVM_ENABLE_LTO OFF CACHE BOOL "")
+  set(RUNTIMES_${target}_LLVM_ENABLE_FATLTO OFF CACHE BOOL "")
 endforeach()
 
 foreach(target riscv32-unknown-elf)
@@ -420,6 +431,10 @@ foreach(target riscv32-unknown-elf)
   set(RUNTIMES_${target}_LLVM_INCLUDE_TESTS OFF CACHE BOOL "")
   set(RUNTIMES_${target}_LLVM_ENABLE_ASSERTIONS OFF CACHE BOOL "")
   set(RUNTIMES_${target}_LLVM_ENABLE_RUNTIMES "libc;libcxx" CACHE STRING "")
+
+  # Enable FatLTO for baremetal runtimes
+  set(RUNTIMES_${target}_LLVM_ENABLE_LTO OFF CACHE BOOL "")
+  set(RUNTIMES_${target}_LLVM_ENABLE_FATLTO OFF CACHE BOOL "")
 endforeach()
 
 set(LLVM_BUILTIN_TARGETS "${BUILTIN_TARGETS}" CACHE STRING "")
diff --git a/clang/cmake/caches/Fuchsia.cmake b/clang/cmake/caches/Fuchsia.cmake
index 07637cd0ed08f..83336589da305 100644
--- a/clang/cmake/caches/Fuchsia.cmake
+++ b/clang/cmake/caches/Fuchsia.cmake
@@ -84,7 +84,6 @@ set(CLANG_DEFAULT_LINKER lld CACHE STRING "")
 set(CLANG_DEFAULT_OBJCOPY llvm-objcopy CACHE STRING "")
 set(CLANG_DEFAULT_RTLIB compiler-rt CACHE STRING "")
 set(CLANG_DEFAULT_UNWINDLIB libunwind CACHE STRING "")
-set(CLANG_ENABLE_ARCMT OFF CACHE BOOL "")
 set(CLANG_ENABLE_STATIC_ANALYZER OFF CACHE BOOL "")
 set(CLANG_PLUGIN_SUPPORT OFF CACHE BOOL "")
 
diff --git a/clang/docs/BoundsSafety.rst b/clang/docs/BoundsSafety.rst
index e24c69d8c7855..cf5b0c75c0387 100644
--- a/clang/docs/BoundsSafety.rst
+++ b/clang/docs/BoundsSafety.rst
@@ -777,13 +777,13 @@ the transformed pseudo code of function ``alloc_buf()`` in the example below.
       size_t count;
    } sized_buf_t;
 
-   void alloc_buf(sized_buf_t *sbuf, sized_t nelems) {
+   void alloc_buf(sized_buf_t *sbuf, size_t nelems) {
       sbuf->buf = (int *)malloc(sizeof(int) * nelems);
       sbuf->count = nelems;
    }
 
    // Transformed pseudo code:
-   void alloc_buf(sized_buf_t *sbuf, sized_t nelems) {
+   void alloc_buf(sized_buf_t *sbuf, size_t nelems) {
       // Materialize RHS values:
       int *tmp_ptr = (int *)malloc(sizeof(int) * nelems);
       int tmp_count = nelems;
@@ -959,7 +959,8 @@ that has the define.
    #if defined(__has_feature) && __has_feature(bounds_safety)
    #define __counted_by(T) __attribute__((__counted_by__(T)))
    // ... other bounds annotations
-   #else #define __counted_by(T) // defined as nothing
+   #else
+   #define __counted_by(T) // defined as nothing
    // ... other bounds annotations
    #endif
 
@@ -987,7 +988,7 @@ and it does not guarantee other types of memory safety properties. Consequently,
 it may not prevent some of the secondary bounds safety violations caused by
 other types of safety violations such as type confusion. For instance,
 ``-fbounds-safety`` does not perform type-safety checks on conversions between
-`__single`` pointers of different pointee types (e.g., ``char *__single`` →
+``__single`` pointers of different pointee types (e.g., ``char *__single`` →
 ``void *__single`` → ``int *__single``) beyond what the foundation languages
 (C/C++) already offer.
 
@@ -1003,4 +1004,4 @@ Try it out
 
 Your feedback on the programming model is valuable. You may want to follow the
 instruction in :doc:`BoundsSafetyAdoptionGuide` to play with ``-fbounds-safety``
-and please send your feedback to `Yeoul Na <mailto:yeoul_na@apple.com>`_.
\ No newline at end of file
+and please send your feedback to `Yeoul Na <mailto:yeoul_na@apple.com>`_.
diff --git a/clang/docs/BoundsSafetyImplPlans.rst b/clang/docs/BoundsSafetyImplPlans.rst
index 93c2ed7b43402..34276c920f31e 100644
--- a/clang/docs/BoundsSafetyImplPlans.rst
+++ b/clang/docs/BoundsSafetyImplPlans.rst
@@ -134,7 +134,7 @@ same basic block and without side effect in between.
       int *__counted_by(count) buf; size_t count;
    } sized_buf_t;
 
-   void alloc_buf(sized_buf_t *sbuf, sized_t nelems) {
+   void alloc_buf(sized_buf_t *sbuf, size_t nelems) {
       sbuf->buf = (int *)malloc(sizeof(int) * nelems);
       sbuf->count = nelems;
    }
diff --git a/clang/docs/ClangFormatStyleOptions.rst b/clang/docs/ClangFormatStyleOptions.rst
index 30a2325949f48..bf6dd9e13915f 100644
--- a/clang/docs/ClangFormatStyleOptions.rst
+++ b/clang/docs/ClangFormatStyleOptions.rst
@@ -2182,6 +2182,24 @@ the configuration (without a prefix: ``Auto``).
         aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa);
     }
 
+.. _BinPackLongBracedList:
+
+**BinPackLongBracedList** (``Boolean``) :versionbadge:`clang-format 21` :ref:`¶ <BinPackLongBracedList>`
+  If ``BinPackLongBracedList`` is ``true`` it overrides
+  ``BinPackArguments`` if there are 20 or more items in a braced
+  initializer list.
+
+  .. code-block:: c++
+
+     BinPackLongBracedList: false  vs.    BinPackLongBracedList: true
+     vector<int> x{                       vector<int> x{1, 2, ...,
+                                                        20, 21};
+                 1,
+                 2,
+                 ...,
+                 20,
+                 21};
+
 .. _BinPackParameters:
 
 **BinPackParameters** (``BinPackParametersStyle``) :versionbadge:`clang-format 3.7` :ref:`¶ <BinPackParameters>`
@@ -3421,6 +3439,35 @@ the configuration (without a prefix: ``Auto``).
 
 
 
+.. _BreakBeforeTemplateCloser:
+
+**BreakBeforeTemplateCloser** (``Boolean``) :versionbadge:`clang-format 21` :ref:`¶ <BreakBeforeTemplateCloser>`
+  If ``true``, break before a template closing bracket (``>``) when there is
+  a line break after the matching opening bracket (``<``).
+
+  .. code-block:: c++
+
+     true:
+     template <typename Foo, typename Bar>
+
+     template <typename Foo,
+               typename Bar>
+
+     template <
+         typename Foo,
+         typename Bar
+     >
+
+     false:
+     template <typename Foo, typename Bar>
+
+     template <typename Foo,
+               typename Bar>
+
+     template <
+         typename Foo,
+         typename Bar>
+
 .. _BreakBeforeTernaryOperators:
 
 **BreakBeforeTernaryOperators** (``Boolean``) :versionbadge:`clang-format 3.7` :ref:`¶ <BreakBeforeTernaryOperators>`
@@ -5198,6 +5245,11 @@ the configuration (without a prefix: ``Auto``).
 **PenaltyBreakBeforeFirstCallParameter** (``Unsigned``) :versionbadge:`clang-format 3.7` :ref:`¶ <PenaltyBreakBeforeFirstCallParameter>`
   The penalty for breaking a function call after ``call(``.
 
+.. _PenaltyBreakBeforeMemberAccess:
+
+**PenaltyBreakBeforeMemberAccess** (``Unsigned``) :versionbadge:`clang-format 20` :ref:`¶ <PenaltyBreakBeforeMemberAccess>`
+  The penalty for breaking before a member access operator (``.``, ``->``).
+
 .. _PenaltyBreakComment:
 
 **PenaltyBreakComment** (``Unsigned``) :versionbadge:`clang-format 3.7` :ref:`¶ <PenaltyBreakComment>`
diff --git a/clang/docs/ControlFlowIntegrity.rst b/clang/docs/ControlFlowIntegrity.rst
index 7de805e2154db..3c97f7da3d7d9 100644
--- a/clang/docs/ControlFlowIntegrity.rst
+++ b/clang/docs/ControlFlowIntegrity.rst
@@ -336,6 +336,15 @@ cross-DSO function address equality. These properties make KCFI easier to
 adopt in low-level software. KCFI is limited to checking only function
 pointers, and isn't compatible with executable-only memory.
 
+``-fsanitize-kcfi-arity``
+-----------------------------
+
+For supported targets, this feature extends kCFI by telling the compiler to
+record information about each indirect-callable function's arity (i.e., the
+number of arguments passed in registers) into the binary. Some kernel CFI
+techniques, such as FineIBT, may be able to use this information to provide
+enhanced security.
+
 Member Function Pointer Call Checking
 =====================================
 
diff --git a/clang/docs/HIPSupport.rst b/clang/docs/HIPSupport.rst
index e830acd8dd85c..481ed39230813 100644
--- a/clang/docs/HIPSupport.rst
+++ b/clang/docs/HIPSupport.rst
@@ -27,7 +27,8 @@ AMD GPU Support
 Clang provides HIP support on AMD GPUs via the ROCm platform `<https://rocm.docs.amd.com/en/latest/#>`_.
 The ROCm runtime forms the base for HIP host APIs, while HIP device APIs are realized through HIP header
 files and the ROCm device library. The Clang driver uses the HIPAMD toolchain to compile HIP device code
-to AMDGPU ISA via the AMDGPU backend. The compiled code is then bundled and embedded in the host executables.
+to AMDGPU ISA via the AMDGPU backend, or SPIR-V via the workflow outlined below.
+The compiled code is then bundled and embedded in the host executables.
 
 Intel GPU Support
 =================
@@ -285,6 +286,401 @@ Example Usage
       basePtr->virtualFunction(); // Allowed since obj is constructed in device code
    }
 
+C++ Standard Parallelism Offload Support: Compiler And Runtime
+==============================================================
+
+Introduction
+============
+
+This section describes the implementation of support for offloading the
+execution of standard C++ algorithms to accelerators that can be targeted via
+HIP. Furthermore, it enumerates restrictions on user defined code, as well as
+the interactions with runtimes.
+
+Algorithm Offload: What, Why, Where
+===================================
+
+C++17 introduced overloads
+`for most algorithms in the standard library <https://www.open-std.org/jtc1/sc22/wg21/docs/papers/2016/p0024r2.html>`_
+which allow the user to specify a desired
+`execution policy <https://en.cppreference.com/w/cpp/algorithm#Execution_policies>`_.
+The `parallel_unsequenced_policy <https://en.cppreference.com/w/cpp/algorithm/execution_policy_tag_t>`_
+maps relatively well to the execution model of AMD GPUs. This, coupled with the
+the availability and maturity of GPU accelerated algorithm libraries that
+implement most / all corresponding algorithms in the standard library
+(e.g. `rocThrust <https://github.com/ROCmSoftwarePlatform/rocThrust>`__), makes
+it feasible to provide seamless accelerator offload for supported algorithms,
+when an accelerated version exists. Thus, it becomes possible to easily access
+the computational resources of an AMD accelerator, via a well specified,
+familiar, algorithmic interface, without having to delve into low-level hardware
+specific details. Putting it all together:
+
+- **What**: standard library algorithms, when invoked with the
+  ``parallel_unsequenced_policy``
+- **Why**: democratise AMDGPU accelerator programming, without loss of user
+  familiarity
+- **Where**: only AMDGPU accelerators targeted by Clang/LLVM via HIP
+
+Small Example
+=============
+
+Given the following C++ code:
+
+.. code-block:: C++
+
+   bool has_the_answer(const std::vector<int>& v) {
+     return std::find(std::execution::par_unseq, std::cbegin(v), std::cend(v), 42) != std::cend(v);
+   }
+
+if Clang is invoked with the ``--hipstdpar --offload-arch=foo`` flags, the call
+to ``find`` will be offloaded to an accelerator that is part of the ``foo``
+target family. If either ``foo`` or its runtime environment do not support
+transparent on-demand paging (such as e.g. that provided in Linux via
+`HMM <https://docs.kernel.org/mm/hmm.html>`_), it is necessary to also include
+the ``--hipstdpar-interpose-alloc`` flag. If the accelerator specific algorithm
+library ``foo`` uses doesn't have an implementation of a particular algorithm,
+execution seamlessly falls back to the host CPU. It is legal to specify multiple
+``--offload-arch``\s. All the flags we introduce, as well as a thorough view of
+various restrictions an their implementations, will be provided below.
+
+Implementation - General View
+=============================
+
+We built support for Algorithm Offload support atop the pre-existing HIP
+infrastructure. More specifically, when one requests offload via ``--hipstdpar``,
+compilation is switched to HIP compilation, as if ``-x hip`` was specified.
+Similarly, linking is also switched to HIP linking, as if ``--hip-link`` was
+specified. Note that these are implicit, and one should not assume that any
+interop with HIP specific language constructs is available e.g. ``__device__``
+annotations are neither necessary nor guaranteed to work.
+
+Since there are no language restriction mechanisms in place, it is necessary to
+relax HIP language specific semantic checks performed by the FE; they would
+identify otherwise valid, offloadable code, as invalid HIP code. Given that we
+know that the user intended only for certain algorithms to be offloaded, and
+encoded this by specifying the ``parallel_unsequenced_policy``, we rely on a
+pass over IR to clean up any and all code that was not "meant" for offload. If
+requested, allocation interposition is also handled via a separate pass over IR.
+
+To interface with the client HIP runtime, and to forward offloaded algorithm
+invocations to the corresponding accelerator specific library implementation, an
+implementation detail forwarding header is implicitly included by the driver,
+when compiling with ``--hipstdpar``. In what follows, we will delve into each
+component that contributes to implementing Algorithm Offload support.
+
+Implementation - Driver
+=======================
+
+We augment the ``clang`` driver with the following flags:
+
+- ``--hipstdpar`` enables algorithm offload, which depending on phase, has the
+  following effects:
+
+  - when compiling:
+
+    - ``-x hip`` gets prepended to enable HIP support;
+    - the ``ROCmToolchain`` component checks for the ``hipstdpar_lib.hpp``
+      forwarding header,
+      `rocThrust <https://rocm.docs.amd.com/projects/rocThrust/en/latest/>`_ and
+      `rocPrim <https://rocm.docs.amd.com/projects/rocPRIM/en/latest/>`_ in
+      their canonical locations, which can be overriden via flags found below;
+      if all are found, the forwarding header gets implicitly included,
+      otherwise an error listing the missing component is generated;
+    - the ``LangOpts.HIPStdPar`` member is set.
+
+  - when linking:
+
+    - ``--hip-link`` and ``-frtlib-add-rpath`` gets appended to enable HIP
+      support.
+
+- ``--hipstdpar-interpose-alloc`` enables the interposition of standard
+  allocation / deallocation functions with accelerator aware equivalents; the
+  ``LangOpts.HIPStdParInterposeAlloc`` member is set;
+- ``--hipstdpar-path=`` specifies a non-canonical path for the forwarding
+  header; it must point to the folder where the header is located and not to the
+  header itself;
+- ``--hipstdpar-thrust-path=`` specifies a non-canonical path for
+  `rocThrust <https://rocm.docs.amd.com/projects/rocThrust/en/latest/>`_; it
+  must point to the folder where the library is installed / built under a
+  ``/thrust`` subfolder;
+- ``--hipstdpar-prim-path=`` specifies a non-canonical path for
+  `rocPrim <https://rocm.docs.amd.com/projects/rocPRIM/en/latest/>`_; it must
+  point to the folder where the library is installed / built under a
+  ``/rocprim`` subfolder;
+
+The `--offload-arch <https://llvm.org/docs/AMDGPUUsage.html#amdgpu-processors>`_
+flag can be used to specify the accelerator for which offload code is to be
+generated.
+
+Implementation - Front-End
+==========================
+
+When ``LangOpts.HIPStdPar`` is set, we relax some of the HIP language specific
+``Sema`` checks to account for the fact that we want to consume pure unannotated
+C++ code:
+
+1. ``__device__`` / ``__host__ __device__`` functions (which would originate in
+   the accelerator specific algorithm library) are allowed to call implicitly
+   ``__host__`` functions;
+2. ``__global__`` functions (which would originate in the accelerator specific
+   algorithm library) are allowed to call implicitly ``__host__`` functions;
+3. resolving ``__builtin`` availability is deferred, because it is possible that
+   a ``__builtin`` that is unavailable on the target accelerator is not
+   reachable from any offloaded algorithm, and thus will be safely removed in
+   the middle-end;
+4. ASM parsing / checking is deferred, because it is possible that an ASM block
+   that e.g. uses some constraints that are incompatible with the target
+   accelerator is not reachable from any offloaded algorithm, and thus will be
+   safely removed in the middle-end.
+
+``CodeGen`` is similarly relaxed, with implicitly ``__host__`` functions being
+emitted as well.
+
+Implementation - Middle-End
+===========================
+
+We add two ``opt`` passes:
+
+1. ``HipStdParAcceleratorCodeSelectionPass``
+
+   - For all kernels in a ``Module``, compute reachability, where a function
+     ``F`` is reachable from a kernel ``K`` if and only if there exists a direct
+     call-chain rooted in ``F`` that includes ``K``;
+   - Remove all functions that are not reachable from kernels;
+   - This pass is only run when compiling for the accelerator.
+
+The first pass assumes that the only code that the user intended to offload was
+that which was directly or transitively invocable as part of an algorithm
+execution. It also assumes that an accelerator aware algorithm implementation
+would rely on accelerator specific special functions (kernels), and that these
+effectively constitute the only roots for accelerator execution graphs. Both of
+these assumptions are based on observing how widespread accelerators,
+such as GPUs, work.
+
+1. ``HipStdParAllocationInterpositionPass``
+
+   - Iterate through all functions in a ``Module``, and replace standard
+     allocation / deallocation functions with accelerator-aware equivalents,
+     based on a pre-established table; the list of functions that can be
+     interposed is available
+     `here <https://github.com/ROCmSoftwarePlatform/roc-stdpar#allocation--deallocation-interposition-status>`__;
+   - This is only run when compiling for the host.
+
+The second pass is optional.
+
+Implementation - Forwarding Header
+==================================
+
+The forwarding header implements two pieces of functionality:
+
+1. It forwards algorithms to a target accelerator, which is done by relying on
+   C++ language rules around overloading:
+
+   - overloads taking an explicit argument of type
+     ``parallel_unsequenced_policy`` are introduced into the ``std`` namespace;
+   - these will get preferentially selected versus the master template;
+   - the body forwards to the equivalent algorithm from the accelerator specific
+     library
+
+2. It provides allocation / deallocation functions that are equivalent to the
+   standard ones, but obtain memory by invoking
+   `hipMallocManaged <https://rocm.docs.amd.com/projects/HIP/en/latest/.doxygen/docBin/html/group___memory_m.html#gab8cfa0e292193fa37e0cc2e4911fa90a>`_
+   and release it via `hipFree <https://rocm.docs.amd.com/projects/HIP/en/latest/.doxygen/docBin/html/group___memory.html#ga740d08da65cae1441ba32f8fedb863d1>`_.
+
+Predefined Macros
+=================
+
+.. list-table::
+   :header-rows: 1
+
+   * - Macro
+     - Description
+   * - ``__HIPSTDPAR__``
+     - Defined when Clang is compiling code in algorithm offload mode, enabled
+       with the ``--hipstdpar`` compiler option.
+   * - ``__HIPSTDPAR_INTERPOSE_ALLOC__``
+     - Defined only when compiling in algorithm offload mode, when the user
+       enables interposition mode with the ``--hipstdpar-interpose-alloc``
+       compiler option, indicating that all dynamic memory allocation /
+       deallocation functions should be replaced with accelerator aware
+       variants.
+
+Restrictions
+============
+
+We define two modes in which runtime execution can occur:
+
+1. **HMM Mode** - this assumes that the
+   `HMM <https://docs.kernel.org/mm/hmm.html>`_ subsystem of the Linux kernel
+   is used to provide transparent on-demand paging i.e. memory obtained from a
+   system / OS allocator such as via a call to ``malloc`` or ``operator new`` is
+   directly accessible to the accelerator and it follows the C++ memory model;
+2. **Interposition Mode** - this is a fallback mode for cases where transparent
+   on-demand paging is unavailable (e.g. in the Windows OS), which means that
+   memory must be allocated via an accelerator aware mechanism, and system
+   allocated memory is inaccessible for the accelerator.
+
+The following restrictions imposed on user code apply to both modes:
+
+1. Pointers to function, and all associated features, such as e.g. dynamic
+   polymorphism, cannot be used (directly or transitively) by the user provided
+   callable passed to an algorithm invocation;
+2. Global / namespace scope / ``static`` / ``thread`` storage duration variables
+   cannot be used (directly or transitively) in name by the user provided
+   callable;
+
+   - When executing in **HMM Mode** they can be used in address e.g.:
+
+     .. code-block:: C++
+
+        namespace { int foo = 42; }
+
+        bool never(const std::vector<int>& v) {
+          return std::any_of(std::execution::par_unseq, std::cbegin(v), std::cend(v), [](auto&& x) {
+            return x == foo;
+          });
+        }
+
+        bool only_in_hmm_mode(const std::vector<int>& v) {
+          return std::any_of(std::execution::par_unseq, std::cbegin(v), std::cend(v),
+                             [p = &foo](auto&& x) { return x == *p; });
+        }
+
+3. Only algorithms that are invoked with the ``parallel_unsequenced_policy`` are
+   candidates for offload;
+4. Only algorithms that are invoked with iterator arguments that model
+   `random_access_iterator <https://en.cppreference.com/w/cpp/iterator/random_access_iterator>`_
+   are candidates for offload;
+5. `Exceptions <https://en.cppreference.com/w/cpp/language/exceptions>`_ cannot
+   be used by the user provided callable;
+6. Dynamic memory allocation (e.g. ``operator new``) cannot be used by the user
+   provided callable;
+7. Selective offload is not possible i.e. it is not possible to indicate that
+   only some algorithms invoked with the ``parallel_unsequenced_policy`` are to
+   be executed on the accelerator.
+
+In addition to the above, using **Interposition Mode** imposes the following
+additional restrictions:
+
+1. All code that is expected to interoperate has to be recompiled with the
+   ``--hipstdpar-interpose-alloc`` flag i.e. it is not safe to compose libraries
+   that have been independently compiled;
+2. automatic storage duration (i.e. stack allocated) variables cannot be used
+   (directly or transitively) by the user provided callable e.g.
+
+   .. code-block:: c++
+
+      bool never(const std::vector<int>& v, int n) {
+        return std::any_of(std::execution::par_unseq, std::cbegin(v), std::cend(v),
+                           [p = &n](auto&& x) { return x == *p; });
+      }
+
+Current Support
+===============
+
+At the moment, C++ Standard Parallelism Offload is only available for AMD GPUs,
+when the `ROCm <https://rocm.docs.amd.com/en/latest/>`_ stack is used, on the
+Linux operating system. Support is synthesised in the following table:
+
+.. list-table::
+   :header-rows: 1
+
+   * - `Processor <https://llvm.org/docs/AMDGPUUsage.html#amdgpu-processors>`_
+     - HMM Mode
+     - Interposition Mode
+   * - GCN GFX9 (Vega)
+     - YES
+     - YES
+   * - GCN GFX10.1 (RDNA 1)
+     - *NO*
+     - YES
+   * - GCN GFX10.3 (RDNA 2)
+     - *NO*
+     - YES
+   * - GCN GFX11 (RDNA 3)
+     - *NO*
+     - YES
+   * - GCN GFX12 (RDNA 4)
+     - *NO*
+     - YES
+
+The minimum Linux kernel version for running in HMM mode is 6.4.
+
+The forwarding header can be obtained from
+`its GitHub repository <https://github.com/ROCmSoftwarePlatform/roc-stdpar>`_.
+It will be packaged with a future `ROCm <https://rocm.docs.amd.com/en/latest/>`_
+release. Because accelerated algorithms are provided via
+`rocThrust <https://rocm.docs.amd.com/projects/rocThrust/en/latest/>`_, a
+transitive dependency on
+`rocPrim <https://rocm.docs.amd.com/projects/rocPRIM/en/latest/>`_ exists. Both
+can be obtained either by installing their associated components of the
+`ROCm <https://rocm.docs.amd.com/en/latest/>`_ stack, or from their respective
+repositories. The list algorithms that can be offloaded is available
+`here <https://github.com/ROCmSoftwarePlatform/roc-stdpar#algorithm-support-status>`_.
+
+HIP Specific Elements
+---------------------
+
+1. There is no defined interop with the
+   `HIP kernel language <https://rocm.docs.amd.com/projects/HIP/en/latest/reference/kernel_language.html>`_;
+   whilst things like using `__device__` annotations might accidentally "work",
+   they are not guaranteed to, and thus cannot be relied upon by user code;
+
+   - A consequence of the above is that both bitcode linking and linking
+     relocatable object files will "work", but it is not guaranteed to remain
+     working or actively tested at the moment; this restriction might be relaxed
+     in the future.
+
+2. Combining explicit HIP, CUDA or OpenMP Offload compilation with
+   ``--hipstdpar`` based offloading is not allowed or supported in any way.
+3. There is no way to target different accelerators via a standard algorithm
+   invocation (`this might be addressed in future C++ standards <https://www.open-std.org/jtc1/sc22/wg21/docs/papers/2023/p2500r1.html>`_);
+   an unsafe (per the point above) way of achieving this is to spawn new threads
+   and invoke the `hipSetDevice <https://rocm.docs.amd.com/projects/HIP/en/latest/.doxygen/docBin/html/group___device.html#ga43c1e7f15925eeb762195ccb5e063eae>`_
+   interface e.g.:
+
+   .. code-block:: c++
+
+      int accelerator_0 = ...;
+      int accelerator_1 = ...;
+
+      bool multiple_accelerators(const std::vector<int>& u, const std::vector<int>& v) {
+        std::atomic<unsigned int> r{0u};
+
+        thread t0{[&]() {
+          hipSetDevice(accelerator_0);
+
+          r += std::count(std::execution::par_unseq, std::cbegin(u), std::cend(u), 42);
+        }};
+        thread t1{[&]() {
+          hitSetDevice(accelerator_1);
+
+          r += std::count(std::execution::par_unseq, std::cbegin(v), std::cend(v), 314152)
+        }};
+
+        t0.join();
+        t1.join();
+
+        return r;
+      }
+
+   Note that this is a temporary, unsafe workaround for a deficiency in the C++
+   Standard.
+
+Open Questions / Future Developments
+====================================
+
+1. The restriction on the use of global / namespace scope / ``static`` /
+   ``thread`` storage duration variables in offloaded algorithms will be lifted
+   in the future, when running in **HMM Mode**;
+2. The restriction on the use of dynamic memory allocation in offloaded
+   algorithms will be lifted in the future.
+3. The restriction on the use of pointers to function, and associated features
+   such as dynamic polymorphism might be lifted in the future, when running in
+   **HMM Mode**;
+4. Offload support might be extended to cases where the ``parallel_policy`` is
+   used for some or all targets.
+
 SPIR-V Support on HIPAMD ToolChain
 ==================================
 
@@ -316,13 +712,6 @@ Using ``--offload-arch=amdgcnspirv``
 - **Clang Offload Bundler**: The resulting SPIR-V is embedded in the Clang
   offload bundler with the bundle ID ``hip-spirv64-amd-amdhsa--amdgcnspirv``.
 
-Mixed with Normal ``--offload-arch``
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-
-**Mixing ``amdgcnspirv`` and concrete ``gfx###`` targets via ``--offload-arch``
-is not currently supported; this limitation is temporary and will be removed in
-a future release**
-
 Architecture Specific Macros
 ----------------------------
 
diff --git a/clang/docs/LanguageExtensions.rst b/clang/docs/LanguageExtensions.rst
index c42b88015e269..973cf8f9d091c 100644
--- a/clang/docs/LanguageExtensions.rst
+++ b/clang/docs/LanguageExtensions.rst
@@ -514,9 +514,7 @@ available in all language modes.
 __nullptr
 ---------
 
-``__nullptr`` is an alternate spelling for ``nullptr``, but is also available in
-C++ modes prior to C++11. Note that it's currently not availbale in C despite
-C23 having support for ``nullptr``.
+``__nullptr`` is an alternate spelling for ``nullptr``. It is available in all C and C++ language modes.
 
 __signed, __signed__
 --------------------
@@ -759,6 +757,9 @@ The integer elementwise intrinsics, including ``__builtin_elementwise_popcount``
 ``__builtin_elementwise_bitreverse``, ``__builtin_elementwise_add_sat``,
 ``__builtin_elementwise_sub_sat`` can be called in a ``constexpr`` context.
 
+No implicit promotion of integer types takes place. The mixing of integer types
+of different sizes and signs is forbidden in binary and ternary builtins.
+
 ============================================== ====================================================================== =========================================
          Name                                   Operation                                                             Supported element types
 ============================================== ====================================================================== =========================================
@@ -1802,10 +1803,6 @@ The following type trait primitives are supported by Clang. Those traits marked
 * ``__is_pointer_interconvertible_base_of`` (C++, GNU, Microsoft)
 * ``__is_polymorphic`` (C++, GNU, Microsoft, Embarcadero)
 * ``__is_reference`` (C++, Embarcadero)
-* ``__is_referenceable`` (C++, GNU, Microsoft, Embarcadero):
-  Returns true if a type is referenceable, and false otherwise. A referenceable
-  type is a type that's either an object type, a reference type, or an unqualified
-  function type. This trait is deprecated and will be removed in Clang 21.
 * ``__is_rvalue_reference`` (C++, Embarcadero)
 * ``__is_same`` (C++, Embarcadero)
 * ``__is_same_as`` (GCC): Synonym for ``__is_same``.
diff --git a/clang/docs/OpenMPSupport.rst b/clang/docs/OpenMPSupport.rst
index 673c34bf08a4a..725624ee8c66c 100644
--- a/clang/docs/OpenMPSupport.rst
+++ b/clang/docs/OpenMPSupport.rst
@@ -364,6 +364,8 @@ implementation.
 +=============================================================+===========================+===========================+==========================================================================+
 | free-agent threads                                          | :none:`unclaimed`         | :none:`unclaimed`         |                                                                          |
 +-------------------------------------------------------------+---------------------------+---------------------------+--------------------------------------------------------------------------+
+| threadset clause                                            | :`worked on`              | :none:`unclaimed`         |                                                                          |
++-------------------------------------------------------------+---------------------------+---------------------------+--------------------------------------------------------------------------+
 | Recording of task graphs                                    | :none:`unclaimed`         | :none:`unclaimed`         |                                                                          |
 +-------------------------------------------------------------+---------------------------+---------------------------+--------------------------------------------------------------------------+
 | Parallel inductions                                         | :none:`unclaimed`         | :none:`unclaimed`         |                                                                          |
@@ -410,13 +412,13 @@ implementation.
 +-------------------------------------------------------------+---------------------------+---------------------------+--------------------------------------------------------------------------+
 | Extensions to interop construct                             | :none:`unclaimed`         | :none:`unclaimed`         |                                                                          |
 +-------------------------------------------------------------+---------------------------+---------------------------+--------------------------------------------------------------------------+
-| no_openmp_constructs                                        | :none:`unclaimed`         | :none:`unclaimed`         |                                                                          |
+| no_openmp_constructs                                        | :good:`done`              | :none:`unclaimed`         | https://github.com/llvm/llvm-project/pull/125933                         |
 +-------------------------------------------------------------+---------------------------+---------------------------+--------------------------------------------------------------------------+
 | safe_sync and progress with identifier and API              | :none:`unclaimed`         | :none:`unclaimed`         |                                                                          |
 +-------------------------------------------------------------+---------------------------+---------------------------+--------------------------------------------------------------------------+
-| OpenMP directives in concurrent loop regions                | :none:`unclaimed`         | :none:`unclaimed`         |                                                                          |
+| OpenMP directives in concurrent loop regions                | :good:`done`              | :none:`unclaimed`         | https://github.com/llvm/llvm-project/pull/125621                         |
 +-------------------------------------------------------------+---------------------------+---------------------------+--------------------------------------------------------------------------+
-| atomics constructs on concurrent loop regions               | :none:`unclaimed`         | :none:`unclaimed`         |                                                                          |
+| atomics constructs on concurrent loop regions               | :good:`done`              | :none:`unclaimed`         | https://github.com/llvm/llvm-project/pull/125621                         |
 +-------------------------------------------------------------+---------------------------+---------------------------+--------------------------------------------------------------------------+
 | Loop construct with DO CONCURRENT                           | :none:`unclaimed`         | :none:`unclaimed`         |                                                                          |
 +-------------------------------------------------------------+---------------------------+---------------------------+--------------------------------------------------------------------------+
@@ -454,9 +456,7 @@ implementation.
 +-------------------------------------------------------------+---------------------------+---------------------------+--------------------------------------------------------------------------+
 | map-type modifiers in arbitrary position                    | :none:`unclaimed`         | :none:`unclaimed`         |                                                                          |
 +-------------------------------------------------------------+---------------------------+---------------------------+--------------------------------------------------------------------------+
-| atomic constructs in loop region                            | :none:`unclaimed`         | :none:`unclaimed`         |                                                                          |
-+-------------------------------------------------------------+---------------------------+---------------------------+--------------------------------------------------------------------------+
-| Lift nesting restriction on concurrent loop                 | :none:`unclaimed`         | :none:`unclaimed`         |                                                                          |
+| Lift nesting restriction on concurrent loop                 | :good:`done`              | :none:`unclaimed`         | https://github.com/llvm/llvm-project/pull/125621                         |
 +-------------------------------------------------------------+---------------------------+---------------------------+--------------------------------------------------------------------------+
 | priority clause for target constructs                       | :none:`unclaimed`         | :none:`unclaimed`         |                                                                          |
 +-------------------------------------------------------------+---------------------------+---------------------------+--------------------------------------------------------------------------+
diff --git a/clang/docs/ReleaseNotes.rst b/clang/docs/ReleaseNotes.rst
index f110b8cf76507..6344c4b36e357 100644
--- a/clang/docs/ReleaseNotes.rst
+++ b/clang/docs/ReleaseNotes.rst
@@ -33,218 +33,23 @@ latest release, please see the `Clang Web Site <https://clang.llvm.org>`_ or the
 
 Potentially Breaking Changes
 ============================
-These changes are ones which we think may surprise users when upgrading to
-Clang |release| because of the opportunity they pose for disruption to existing
-code bases.
-
-- The ``le32`` and ``le64`` targets have been removed.
-
-- ``clang -m32`` defaults to ``-mcpu=v9`` on SPARC Linux now.  Distros
-  still supporting SPARC V8 CPUs need to specify ``-mcpu=v8`` with a
-  `config file
-  <https://clang.llvm.org/docs/UsersManual.html#configuration-files>`_.
-
-- The ``clang-rename`` tool has been removed.
-
-- Removed support for RenderScript targets. This technology is
-  `officially deprecated <https://developer.android.com/guide/topics/renderscript/compute>`_
-  and users are encouraged to
-  `migrate to Vulkan <https://developer.android.com/guide/topics/renderscript/migrate>`_
-  or other options.
-
-- Clang now emits distinct type-based alias analysis tags for incompatible
-  pointers by default, enabling more powerful alias analysis when accessing
-  pointer types. This change may silently change code behavior for code
-  containing strict-aliasing violations. The new default behavior can be
-  disabled using ``-fno-pointer-tbaa``.
-
-- Clang will now more aggressively use undefined behavior on pointer addition
-  overflow for optimization purposes. For example, a check like
-  ``ptr + unsigned_offset < ptr`` will now optimize to ``false``, because
-  ``ptr + unsigned_offset`` will cause undefined behavior if it overflows (or
-  advances past the end of the object).
-
-  Previously, ``ptr + unsigned_offset < ptr`` was optimized (by both Clang and
-  GCC) to ``(ssize_t)unsigned_offset < 0``. This also results in an incorrect
-  overflow check, but in a way that is less apparent when only testing with
-  pointers in the low half of the address space.
-
-  To avoid pointer addition overflow, it is necessary to perform the addition
-  on integers, for example using
-  ``(uintptr_t)ptr + unsigned_offset < (uintptr_t)ptr``. Sometimes, it is also
-  possible to rewrite checks by only comparing the offset. For example,
-  ``ptr + offset < end_ptr && ptr + offset >= ptr`` can be written as
-  ``offset < (uintptr_t)(end_ptr - ptr)``.
-
-  Undefined behavior due to pointer addition overflow can be reliably detected
-  using ``-fsanitize=pointer-overflow``. It is also possible to use
-  ``-fno-strict-overflow`` to opt-in to a language dialect where signed integer
-  and pointer overflow are well-defined.
+
+- The Objective-C ARC migrator (ARCMigrate) has been removed.
 
 C/C++ Language Potentially Breaking Changes
 -------------------------------------------
 
-- Clang now rejects ``_Complex _BitInt`` types.
-
 C++ Specific Potentially Breaking Changes
 -----------------------------------------
 
-- The type trait builtin ``__is_nullptr`` has been removed, since it has very
-  few users and can be written as ``__is_same(__remove_cv(T), decltype(nullptr))``,
-  which GCC supports as well.
-
-- The type trait builtin ``__is_referenceable`` has been deprecated, since it has
+- The type trait builtin ``__is_referenceable`` has been removed, since it has
   very few users and all the type traits that could benefit from it in the
-  standard library already have their own bespoke builtins. It will be removed in
-  Clang 21.
-
-- Clang will now correctly diagnose as ill-formed a constant expression where an
-  enum without a fixed underlying type is set to a value outside the range of
-  the enumeration's values.
-
-  .. code-block:: c++
-
-    enum E { Zero, One, Two, Three, Four };
-    constexpr E Val1 = (E)3;  // Ok
-    constexpr E Val2 = (E)7;  // Ok
-    constexpr E Val3 = (E)8;  // Now ill-formed, out of the range [0, 7]
-    constexpr E Val4 = (E)-1; // Now ill-formed, out of the range [0, 7]
-
-  Since Clang 16, it has been possible to suppress the diagnostic via
-  `-Wno-enum-constexpr-conversion`, to allow for a transition period for users.
-  Now, in Clang 20, **it is no longer possible to suppress the diagnostic**.
-
-- Extraneous template headers are now ill-formed by default.
-  This error can be disable with ``-Wno-error=extraneous-template-head``.
-
-  .. code-block:: c++
-
-    template <> // error: extraneous template head
-    template <typename T>
-    void f();
-
-- During constant evaluation, comparisons between different evaluations of the
-  same string literal are now correctly treated as non-constant, and comparisons
-  between string literals that cannot possibly overlap in memory are now treated
-  as constant. This updates Clang to match the anticipated direction of open core
-  issue `CWG2765 <http://wg21.link/CWG2765>`, but is subject to change once that
-  issue is resolved.
-
-  .. code-block:: c++
-
-    constexpr const char *f() { return "hello"; }
-    constexpr const char *g() { return "world"; }
-    // Used to evaluate to false, now error: non-constant comparison.
-    constexpr bool a = f() == f();
-    // Might evaluate to true or false, as before.
-    bool at_runtime() { return f() == f(); }
-    // Was error, now evaluates to false.
-    constexpr bool b = f() == g();
-
-- Clang will now correctly not consider pointers to non classes for covariance
-  and disallow changing return type to a type that doesn't have the same or less cv-qualifications.
-
-  .. code-block:: c++
-
-    struct A {
-      virtual const int *f() const;
-      virtual const std::string *g() const;
-    };
-    struct B : A {
-      // Return type has less cv-qualification but doesn't point to a class.
-      // Error will be generated.
-      int *f() const override;
-
-      // Return type doesn't have more cv-qualification also not the same or
-      // less cv-qualification.
-      // Error will be generated.
-      volatile std::string *g() const override;
-    };
-
-- The warning ``-Wdeprecated-literal-operator`` is now on by default, as this is
-  something that WG21 has shown interest in removing from the language. The
-  result is that anyone who is compiling with ``-Werror`` should see this
-  diagnostic.  To fix this diagnostic, simply removing the space character from
-  between the ``operator""`` and the user defined literal name will make the
-  source no longer deprecated. This is consistent with `CWG2521 <https://cplusplus.github.io/CWG/issues/2521.html>_`.
-
-  .. code-block:: c++
-
-    // Now diagnoses by default.
-    unsigned operator"" _udl_name(unsigned long long);
-    // Fixed version:
-    unsigned operator""_udl_name(unsigned long long);
-
-- Clang will now produce an error diagnostic when ``[[clang::lifetimebound]]`` is
-  applied on a parameter or an implicit object parameter of a function that
-  returns void. This was previously ignored and had no effect. (#GH107556)
-
-  .. code-block:: c++
-
-    // Now diagnoses with an error.
-    void f(int& i [[clang::lifetimebound]]);
-
-- Clang will now produce an error diagnostic when ``[[clang::lifetimebound]]``
-  is applied on a type (instead of a function parameter or an implicit object
-  parameter); this includes the case when the attribute is specified for an
-  unnamed function parameter. These were previously ignored and had no effect.
-  (#GH118281)
-
-  .. code-block:: c++
-
-    // Now diagnoses with an error.
-    int* [[clang::lifetimebound]] x;
-    // Now diagnoses with an error.
-    void f(int* [[clang::lifetimebound]] i);
-    // Now diagnoses with an error.
-    void g(int* [[clang::lifetimebound]]);
-
-- Clang now rejects all field accesses on null pointers in constant expressions. The following code
-  used to work but will now be rejected:
-
-  .. code-block:: c++
-
-    struct S { int a; int b; };
-    constexpr const int *p = &((S*)nullptr)->b;
-
-  Previously, this code was erroneously accepted.
-
-- Clang will now consider the implicitly deleted destructor of a union or
-  a non-union class without virtual base class to be ``constexpr`` in C++20
-  mode (Clang 19 only did so in C++23 mode but the standard specification for
-  this changed in C++20). (#GH85550)
-
-  .. code-block:: c++
-
-    struct NonLiteral {
-      NonLiteral() {}
-      ~NonLiteral() {}
-    };
-
-    template <class T>
-    struct Opt {
-      union {
-        char c;
-        T data;
-      };
-      bool engaged = false;
-
-      constexpr Opt() {}
-      constexpr ~Opt() {
-        if (engaged)
-          data.~T();
-      }
-    };
-
-    // Previously only accepted in C++23 and later, now also accepted in C++20.
-    consteval void foo() { Opt<NonLiteral>{}; }
+  standard library already have their own bespoke builtins.
 
 ABI Changes in This Version
 ---------------------------
 
-- Fixed Microsoft name mangling of placeholder, auto and decltype(auto), return types for MSVC 1920+. This change resolves incompatibilities with code compiled by MSVC 1920+ but will introduce incompatibilities with code compiled by earlier versions of Clang unless such code is built with the compiler option -fms-compatibility-version=19.14 to imitate the MSVC 1914 mangling behavior.
-- Fixed the Itanium mangling of the construction vtable name. This change will introduce incompatibilities with code compiled by Clang 19 and earlier versions, unless the -fclang-abi-compat=19 option is used. (#GH108015)
-- Mangle member-like friend function templates as members of the enclosing class. (#GH110247, #GH110503)
+- Return larger CXX records in memory instead of using AVX registers. Code compiled with older clang will be incompatible with newer version of the clang unless -fclang-abi-compat=20 is provided. (#GH120670)
 
 AST Dumping Potentially Breaking Changes
 ----------------------------------------
@@ -254,569 +59,82 @@ Clang Frontend Potentially Breaking Changes
 
 Clang Python Bindings Potentially Breaking Changes
 --------------------------------------------------
-- Parts of the interface returning string results will now return
-  the empty string ``""`` when no result is available, instead of ``None``.
-- Calling a property on the ``CompletionChunk`` or ``CompletionString`` class
-  statically now leads to an error, instead of returning a ``CachedProperty`` object
-  that is used internally. Properties are only available on instances.
-- For a single-line ``SourceRange`` and a ``SourceLocation`` in the same line,
-  but after the end of the ``SourceRange``, ``SourceRange.__contains__``
-  used to incorrectly return ``True``. (#GH22617), (#GH52827)
 
 What's New in Clang |release|?
 ==============================
-Some of the major new features and improvements to Clang are listed
-here. Generic improvements to Clang as a whole or to its underlying
-infrastructure are described first, followed by language-specific
-sections with improvements to Clang's support for those languages.
 
 C++ Language Changes
 --------------------
-- Allow single element access of GCC vector/ext_vector_type object to be
-  constant expression. Supports the `V.xyzw` syntax and other tidbits
-  as seen in OpenCL. Selecting multiple elements is left as a future work.
-- Implement `CWG1815 <https://wg21.link/CWG1815>`_. Support lifetime extension
-  of temporary created by aggregate initialization using a default member
-  initializer.
-
-- Accept C++26 user-defined ``static_assert`` messages in C++11 as an extension.
-
-- Add ``__builtin_elementwise_popcount`` builtin for integer types only.
-
-- Add ``__builtin_elementwise_fmod`` builtin for floating point types only.
-
-- Add ``__builtin_elementwise_minimum`` and ``__builtin_elementwise_maximum``
-  builtin for floating point types only.
-
-- The builtin type alias ``__builtin_common_type`` has been added to improve the
-  performance of ``std::common_type``.
 
 C++2c Feature Support
 ^^^^^^^^^^^^^^^^^^^^^
 
-- Add ``__builtin_is_virtual_base_of`` intrinsic, which supports
-  `P2985R0 A type trait for detecting virtual base classes <https://wg21.link/p2985r0>`_
-
-- Implemented `P2893R3 Variadic Friends <https://wg21.link/P2893>`_
-
-- Implemented `P2747R2 constexpr placement new <https://wg21.link/P2747R2>`_.
-
-- Added the ``__builtin_is_within_lifetime`` builtin, which supports
-  `P2641R4 Checking if a union alternative is active <https://wg21.link/p2641r4>`_
-
-- Implemented `P3176R1 The Oxford variadic comma <https://wg21.link/P3176R1>`_
-
 C++23 Feature Support
 ^^^^^^^^^^^^^^^^^^^^^
-- Removed the restriction to literal types in constexpr functions in C++23 mode.
-
-- Extend lifetime of temporaries in mem-default-init for P2718R0. Clang now fully
-  supports `P2718R0 Lifetime extension in range-based for loops <https://wg21.link/P2718R0>`_.
-
-- ``__cpp_explicit_this_parameter`` is now defined. (#GH82780)
-
-- Add ``__builtin_is_implicit_lifetime`` intrinsic, which supports
-  `P2674R1 A trait for implicit lifetime types <https://wg21.link/p2674r1>`_
-
-- Add support for `P2280R4 Using unknown pointers and references in constant expressions <https://wg21.link/P2280R4>`_. (#GH63139)
 
 C++20 Feature Support
 ^^^^^^^^^^^^^^^^^^^^^
 
-- Implemented module level lookup for C++20 modules. (#GH90154)
-
 C++17 Feature Support
 ^^^^^^^^^^^^^^^^^^^^^
-- The implementation of the relaxed template template argument matching rules is
-  more complete and reliable, and should provide more accurate diagnostics.
 
 Resolutions to C++ Defect Reports
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
-- Allow calling initializer list constructors from initializer lists with
-  a single element of the same type instead of always copying.
-  (`CWG2137: List-initialization from object of same type <https://cplusplus.github.io/CWG/issues/2137.html>`)
-
-- Speculative resolution for CWG2311 implemented so that the implementation of CWG2137 doesn't remove
-  previous cases where guaranteed copy elision was done. Given a prvalue ``e`` of class type
-  ``T``, ``T{e}`` will try to resolve an initializer list constructor and will use it if successful.
-  Otherwise, if there is no initializer list constructor, the copy will be elided as if it was ``T(e)``.
-  (`CWG2311: Missed case for guaranteed copy elision <https://cplusplus.github.io/CWG/issues/2311.html>`)
-
-- Casts from a bit-field to an integral type is now not considered narrowing if the
-  width of the bit-field means that all potential values are in the range
-  of the target type, even if the type of the bit-field is larger.
-  (`CWG2627: Bit-fields and narrowing conversions <https://cplusplus.github.io/CWG/issues/2627.html>`_)
-
-- ``nullptr`` is now promoted to ``void*`` when passed to a C-style variadic function.
-  (`CWG722: Can nullptr be passed to an ellipsis? <https://cplusplus.github.io/CWG/issues/722.html>`_)
-
-- Allow ``void{}`` as a prvalue of type ``void``.
-  (`CWG2351: void{} <https://cplusplus.github.io/CWG/issues/2351.html>`_).
-
-- Clang now has improved resolution to CWG2398, allowing class templates to have
-  default arguments deduced when partial ordering, and better backwards compatibility
-  in overload resolution.
-
-- Clang now allows comparing unequal object pointers that have been cast to ``void *``
-  in constant expressions. These comparisons always worked in non-constant expressions.
-  (`CWG2749: Treatment of "pointer to void" for relational comparisons <https://cplusplus.github.io/CWG/issues/2749.html>`_).
-
-- Reject explicit object parameters with type ``void`` (``this void``).
-  (`CWG2915: Explicit object parameters of type void <https://cplusplus.github.io/CWG/issues/2915.html>`_).
-
-- Clang now allows trailing requires clause on explicit deduction guides.
-  (`CWG2707: Deduction guides cannot have a trailing requires-clause <https://cplusplus.github.io/CWG/issues/2707.html>`_).
-
-- Respect constructor constraints during CTAD.
-  (`CWG2628: Implicit deduction guides should propagate constraints <https://cplusplus.github.io/CWG/issues/2628.html>`_).
-
-- Clang now diagnoses a space in the first production of a ``literal-operator-id``
-  by default.
-  (`CWG2521: User-defined literals and reserved identifiers <https://cplusplus.github.io/CWG/issues/2521.html>`_).
-
-- Fix name lookup for a dependent base class that is the current instantiation.
-  (`CWG591: When a dependent base class is the current instantiation <https://cplusplus.github.io/CWG/issues/591.html>`_).
-
-- Clang now allows calling explicit object member functions directly with prvalues
-  instead of always materializing a temporary, meaning by-value explicit object parameters
-  do not need to move from a temporary.
-  (`CWG2813: Class member access with prvalues <https://cplusplus.github.io/CWG/issues/2813.html>`_).
+- The flag `-frelaxed-template-template-args`
+  and its negation have been removed, having been deprecated since the previous
+  two releases. The improvements to template template parameter matching implemented
+  in the previous release, as described in P3310 and P3579, made this flag unnecessary.
 
 C Language Changes
 ------------------
 
-- Extend clang's ``<limits.h>`` to define ``LONG_LONG_*`` macros for Android's bionic.
+- Clang now allows an ``inline`` specifier on a typedef declaration of a
+  function type in Microsoft compatibility mode. #GH124869
 
 C2y Feature Support
 ^^^^^^^^^^^^^^^^^^^
 
-- Updated conformance for `N3298 <https://www.open-std.org/jtc1/sc22/wg14/www/docs/n3298.htm>`_
-  which adds the ``i`` and ``j`` suffixes for the creation of a ``_Complex``
-  constant value. Clang has always supported these suffixes as a GNU extension,
-  so ``-Wgnu-imaginary-constant`` no longer has effect in C modes, as this is
-  now a C2y extension in C. ``-Wgnu-imaginary-constant`` still applies in C++
-  modes.
-
-- Clang updated conformance for `N3370 <https://www.open-std.org/jtc1/sc22/wg14/www/docs/n3370.htm>`_
-  case range expressions. This feature was previously supported by Clang as a
-  GNU extension, so ``-Wgnu-case-range`` no longer has effect in C modes, as
-  this is now a C2y extension in C. ``-Wgnu-case-range`` still applies in C++
-  modes.
-
-- Clang implemented support for `N3344 <https://www.open-std.org/jtc1/sc22/wg14/www/docs/n3344.pdf>`_
-  which disallows a ``void`` parameter from having a qualifier or storage class
-  specifier. Note that ``register void`` was previously accepted in all C
-  language modes but is now rejected (all of the other qualifiers and storage
-  class specifiers were previously rejected).
-
-- Updated conformance for `N3364 <https://www.open-std.org/jtc1/sc22/wg14/www/docs/n3364.pdf>`_
-  on floating-point translation-time initialization with signaling NaN. This
-  paper adopts Clang's existing practice, so there were no changes to compiler
-  behavior.
-
-- Implemented support for `N3341 <https://www.open-std.org/jtc1/sc22/wg14/www/docs/n3341.pdf>`_
-  which makes empty structure and union objects implementation-defined in C.
-  ``-Wgnu-empty-struct`` will be emitted in C23 and earlier modes because the
-  behavior is a conforming GNU extension in those modes, but will no longer
-  have an effect in C2y mode.
-
-- Updated conformance for `N3342 <https://www.open-std.org/jtc1/sc22/wg14/www/docs/n3342.pdf>`_
-  which made qualified function types implementation-defined rather than
-  undefined. Clang has always accepted ``const`` and ``volatile`` qualified
-  function types by ignoring the qualifiers.
-
-- Updated conformance for `N3346 <https://www.open-std.org/jtc1/sc22/wg14/www/docs/n3346.pdf>`_
-  which changes some undefined behavior around initialization to instead be
-  constraint violations. This paper adopts Clang's existing practice, so there
-  were no changes to compiler behavior.
-
 C23 Feature Support
 ^^^^^^^^^^^^^^^^^^^
 
-- Clang now supports `N3029 <https://www.open-std.org/jtc1/sc22/wg14/www/docs/n3029.htm>`_ Improved Normal Enumerations.
-- Clang now officially supports `N3030 <https://www.open-std.org/jtc1/sc22/wg14/www/docs/n3030.htm>`_ Enhancements to Enumerations. Clang already supported it as an extension, so there were no changes to compiler behavior.
-- Fixed the value of ``BOOL_WIDTH`` in ``<limits.h>`` to return ``1``
-  explicitly, as mandated by the standard. Fixes #GH117348
-
 Non-comprehensive list of changes in this release
 -------------------------------------------------
 
-- The floating point comparison builtins (``__builtin_isgreater``,
-  ``__builtin_isgreaterequal``, ``__builtin_isless``, etc.) and
-  ``__builtin_signbit`` can now be used in constant expressions.
-- Plugins can now define custom attributes that apply to statements
-  as well as declarations.
-- ``__builtin_abs`` function can now be used in constant expressions.
-
-- The new builtin ``__builtin_counted_by_ref`` was added. In contexts where the
-  programmer needs access to the ``counted_by`` attribute's field, but it's not
-  available --- e.g. in macros. For instance, it can be used to automatically
-  set the counter during allocation in the Linux kernel:
-
-  .. code-block:: c
-
-     /* A simplified version of Linux allocation macros */
-     #define alloc(PTR, FAM, COUNT) ({ \
-         sizeof_t __ignored_assignment;                             \
-         typeof(P) __p;                                             \
-         size_t __size = sizeof(*P) + sizeof(*P->FAM) * COUNT;      \
-         __p = malloc(__size);                                      \
-         *_Generic(                                                 \
-           __builtin_counted_by_ref(__p->FAM),                      \
-             void *: &__ignored_assignment,                         \
-             default: __builtin_counted_by_ref(__p->FAM)) = COUNT;  \
-         __p;                                                       \
-     })
-
-  The flexible array member (FAM) can now be accessed immediately without causing
-  issues with the sanitizer because the counter is automatically set.
-
-- The following builtins can now be used in constant expressions: ``__builtin_reduce_add``,
-  ``__builtin_reduce_mul``, ``__builtin_reduce_and``, ``__builtin_reduce_or``,
-  ``__builtin_reduce_xor``, ``__builtin_elementwise_popcount``,
-  ``__builtin_elementwise_bitreverse``, ``__builtin_elementwise_add_sat``,
-  ``__builtin_elementwise_sub_sat``, ``__builtin_reduce_min`` (For integral element type),
-  ``__builtin_reduce_max`` (For integral element type).
-
-- Clang now rejects ``_BitInt`` matrix element types if the bit width is less than ``CHAR_WIDTH`` or
-  not a power of two, matching preexisting behaviour for vector types.
-
-- Matrix types (a Clang extension) can now be used in pseudo-destructor expressions,
-  which allows them to be stored in STL containers.
-
-- In the ``-ftime-report`` output, the new "Clang time report" group replaces
-  the old "Clang front-end time report" and includes "Front end", "LLVM IR
-  generation", "Optimizer", and "Machine code generation".
-
 New Compiler Flags
 ------------------
 
-- The ``-fc++-static-destructors={all,thread-local,none}`` flag was
-  added to control which C++ variables have static destructors
-  registered: all (the default) does so for all variables, thread-local
-  only for thread-local variables, and none (which corresponds to the
-  existing ``-fno-c++-static-destructors`` flag) skips all static
-  destructors registration.
-
-- The ``-Warray-compare`` warning has been added to warn about array comparison
-  on versions older than C++20.
-
-- The ``-Warray-compare-cxx26`` warning has been added to warn about array comparison
-  starting from C++26, this warning is enabled as an error by default.
-
-- clang-cl and clang-dxc now support ``-fdiagnostics-color=[auto|never|always]``
-  in addition to ``-f[no-]color-diagnostics``.
+- New option ``-fprofile-continuous`` added to enable continuous profile syncing to file (#GH124353, `docs <https://clang.llvm.org/docs/UsersManual.html#cmdoption-fprofile-continuous>`_).
+  The feature has `existed <https://clang.llvm.org/docs/SourceBasedCodeCoverage.html#running-the-instrumented-program>`_)
+  for a while and this is just a user facing option.
 
 Deprecated Compiler Flags
 -------------------------
 
-- ``-fheinous-gnu-extensions`` is deprecated; it is now equivalent to
-  specifying ``-Wno-error=invalid-gnu-asm-cast`` and may be removed in the
-  future.
-
 Modified Compiler Flags
 -----------------------
 
-- The ``-ffp-model`` option has been updated to enable a more limited set of
-  optimizations when the ``fast`` argument is used and to accept a new argument,
-  ``aggressive``. The behavior of ``-ffp-model=aggressive`` is equivalent
-  to the previous behavior of ``-ffp-model=fast``. The updated
-  ``-ffp-model=fast`` behavior no longer assumes finite math only and uses
-  the ``promoted`` algorithm for complex division when possible rather than the
-  less basic (limited range) algorithm.
-
-- The ``-fveclib`` option has been updated to enable ``-fno-math-errno`` for
-  ``-fveclib=ArmPL`` and ``-fveclib=SLEEF``. This gives Clang more opportunities
-  to utilize these vector libraries. The behavior for all other vector function
-  libraries remains unchanged.
-
-- The ``-Wnontrivial-memcall`` warning has been added to warn about
-  passing non-trivially-copyable destination parameter to ``memcpy``,
-  ``memset`` and similar functions for which it is a documented undefined
-  behavior. It is implied by ``-Wnontrivial-memaccess``
-
-- Added ``-fmodules-reduced-bmi`` flag corresponding to
-  ``-fexperimental-modules-reduced-bmi`` flag. The ``-fmodules-reduced-bmi`` flag
-  is intended to be enabled by default in the future.
-
 Removed Compiler Flags
 -------------------------
 
-- The compiler flag `-Wenum-constexpr-conversion` (and the `Wno-`, `Wno-error-`
-  derivatives) is now removed, since it's no longer possible to suppress the
-  diagnostic (see above). Users can expect an `unknown warning` diagnostic if
-  it's still in use.
-
 Attribute Changes in Clang
 --------------------------
 
-- The ``swift_attr`` can now be applied to types. To make it possible to use imported APIs
-  in Swift safely there has to be a way to annotate individual parameters and result types
-  with relevant attributes that indicate that e.g. a block is called on a particular actor
-  or it accepts a Sendable or global-actor (i.e. ``@MainActor``) isolated parameter.
-
-  For example:
-
-  .. code-block:: objc
-
-     @interface MyService
-       -(void) handle: (void (^ __attribute__((swift_attr("@Sendable"))))(id)) handler;
-     @end
-
-- Clang now disallows more than one ``__attribute__((ownership_returns(class, idx)))`` with
-  different class names attached to one function.
-
-- Introduced a new format attribute ``__attribute__((format(syslog, 1, 2)))`` from OpenBSD.
-
-- The ``hybrid_patchable`` attribute is now supported on ARM64EC targets. It can be used to specify
-  that a function requires an additional x86-64 thunk, which may be patched at runtime.
-
-- The attribute ``[[clang::no_specializations]]`` has been added to warn
-  users that a specific template shouldn't be specialized. This is useful for
-  e.g. standard library type traits, where adding a specialization results in
-  undefined behaviour.
-
-- ``[[clang::lifetimebound]]`` is now explicitly disallowed on explicit object member functions
-  where they were previously silently ignored.
-
-- Clang now automatically adds ``[[clang::lifetimebound]]`` to the parameters of
-  ``std::span, std::string_view`` constructors, this enables Clang to capture
-  more cases where the returned reference outlives the object.
-  (#GH100567)
-
-- Clang now correctly diagnoses the use of ``btf_type_tag`` in C++ and ignores
-  it; this attribute is a C-only attribute, and caused crashes with template
-  instantiation by accidentally allowing it in C++ in some circumstances.
-  (#GH106864)
-
-- Introduced a new attribute ``[[clang::coro_await_elidable]]`` on coroutine return types
-  to express elideability at call sites where the coroutine is invoked under a safe elide context.
-
-- Introduced a new attribute ``[[clang::coro_await_elidable_argument]]`` on function parameters
-  to propagate safe elide context to arguments if such function is also under a safe elide context.
-
-- The documentation of the ``[[clang::musttail]]`` attribute was updated to
-  note that the lifetimes of all local variables end before the call. This does
-  not change the behaviour of the compiler, as this was true for previous
-  versions.
-
-- Fix a bug where clang doesn't automatically apply the ``[[gsl::Owner]]`` or
-  ``[[gsl::Pointer]]`` to STL explicit template specialization decls. (#GH109442)
-
-- Clang now supports ``[[clang::lifetime_capture_by(X)]]``. Similar to lifetimebound, this can be
-  used to specify when a reference to a function parameter is captured by another capturing entity ``X``.
-
-- The ``target_version`` attribute is now only supported for AArch64 and RISC-V architectures.
-
-- Clang now permits the usage of the placement new operator in ``[[msvc::constexpr]]``
-  context outside of the std namespace. (#GH74924)
-
-- Clang now disallows the use of attributes after the namespace name. (#GH121407)
+- The ``no_sanitize`` attribute now accepts both ``gnu`` and ``clang`` names.
+- Clang now diagnoses use of declaration attributes on void parameters. (#GH108819)
 
 Improvements to Clang's diagnostics
 -----------------------------------
 
-- Some template related diagnostics have been improved.
-
-  .. code-block:: c++
-
-     void foo() { template <typename> int i; } // error: templates can only be declared in namespace or class scope
-
-     struct S {
-      template <typename> int i; // error: non-static data member 'i' cannot be declared as a template
-     };
-
-- Clang now has improved diagnostics for functions with explicit 'this' parameters. Fixes #GH97878
-
-- Clang now diagnoses dangling references to fields of temporary objects. Fixes #GH81589.
-
-- Clang now diagnoses undefined behavior in constant expressions more consistently. This includes invalid shifts, and signed overflow in arithmetic.
-
-- -Wdangling-assignment-gsl is enabled by default.
-- Clang now always preserves the template arguments as written used
-  to specialize template type aliases.
-
-- Clang now diagnoses the use of ``main`` in an ``extern`` context as invalid according to [basic.start.main] p3. Fixes #GH101512.
-
-- Clang now diagnoses when the result of a [[nodiscard]] function is discarded after being cast in C. Fixes #GH104391.
-
-- Clang now properly explains the reason a template template argument failed to
-  match a template template parameter, in terms of the C++17 relaxed matching rules
-  instead of the old ones.
-
-- Don't emit duplicated dangling diagnostics. (#GH93386).
-
-- Improved diagnostic when trying to befriend a concept. (#GH45182).
-
-- Added the ``-Winvalid-gnu-asm-cast`` diagnostic group to control warnings
-  about use of "noop" casts for lvalues (a GNU extension). This diagnostic is
-  a warning which defaults to being an error, is enabled by default, and is
-  also controlled by the now-deprecated ``-fheinous-gnu-extensions`` flag.
-
-- Added the ``-Wdecls-in-multiple-modules`` option to assist users to identify
-  multiple declarations in different modules, which is the major reason of the slow
-  compilation speed with modules. This warning is disabled by default and it needs
-  to be explicitly enabled or by ``-Weverything``.
-
-- Improved diagnostic when trying to overload a function in an ``extern "C"`` context. (#GH80235)
-
-- Clang now respects lifetimebound attribute for the assignment operator parameter. (#GH106372).
-
-- The lifetimebound and GSL analysis in clang are coherent, allowing clang to
-  detect more use-after-free bugs. (#GH100549).
-
-- Clang now diagnoses dangling cases where a gsl-pointer is constructed from a gsl-owner object inside a container (#GH100384).
-
-- Clang now warns for u8 character literals used in C23 with ``-Wpre-c23-compat`` instead of ``-Wpre-c++17-compat``.
-
-- Clang now diagnose when importing module implementation partition units in module interface units.
-
-- Don't emit bogus dangling diagnostics when ``[[gsl::Owner]]`` and `[[clang::lifetimebound]]` are used together (#GH108272).
-
-- Don't emit bogus dignostic about an undefined behavior on ``reinterpret_cast<T>`` for non-instantiated template functions without sufficient knowledge whether it can actually lead to undefined behavior for ``T`` (#GH109430).
-
-- The ``-Wreturn-stack-address`` warning now also warns about addresses of
-  local variables passed to function calls using the ``[[clang::musttail]]``
-  attribute.
-
-- Clang now diagnoses cases where a dangling ``GSLOwner<GSLPointer>`` object is constructed, e.g. ``std::vector<string_view> v = {std::string()};`` (#GH100526).
-
-- Clang now diagnoses when a ``requires`` expression has a local parameter of void type, aligning with the function parameter (#GH109831).
-
-- Clang now emits a diagnostic note at the class declaration when the method definition does not match any declaration (#GH110638).
-
-- Clang now omits warnings for extra parentheses in fold expressions with single expansion (#GH101863).
-
-- The warning for an unsupported type for a named register variable is now phrased ``unsupported type for named register variable``,
-  instead of ``bad type for named register variable``. This makes it clear that the type is not supported at all, rather than being
-  suboptimal in some way the error fails to mention (#GH111550).
-
-- Clang now emits a ``-Wdepredcated-literal-operator`` diagnostic, even if the
-  name was a reserved name, which we improperly allowed to suppress the
-  diagnostic.
-
-- Clang now diagnoses ``[[deprecated]]`` attribute usage on local variables (#GH90073).
-
-- Fix false positives when `[[gsl::Owner/Pointer]]` and `[[clang::lifetimebound]]` are used together.
-
-- Improved diagnostic message for ``__builtin_bit_cast`` size mismatch (#GH115870).
-
-- Clang now omits shadow warnings for enum constants in separate class scopes (#GH62588).
-
-- When diagnosing an unused return value of a type declared ``[[nodiscard]]``, the type
-  itself is now included in the diagnostic.
-
-- Clang will now prefer the ``[[nodiscard]]`` declaration on function declarations over ``[[nodiscard]]``
-  declaration on the return type of a function. Previously, when both have a ``[[nodiscard]]`` declaration attached,
-  the one on the return type would be preferred. This may affect the generated warning message:
-
-  .. code-block:: c++
-
-    struct [[nodiscard("Reason 1")]] S {};
-    [[nodiscard("Reason 2")]] S getS();
-    void use()
-    {
-      getS(); // Now diagnoses "Reason 2", previously diagnoses "Reason 1"
-    }
-
-- Clang now diagnoses ``= delete("reason")`` extension warnings only in pedantic mode rather than on by default. (#GH109311).
-
-- Clang now diagnoses missing return value in functions containing ``if consteval`` (#GH116485).
-
-- Clang now correctly recognises code after a call to a ``[[noreturn]]`` constructor
-  as unreachable (#GH63009).
-
-- Clang now omits shadowing warnings for parameter names in explicit object member functions (#GH95707).
-
-- Improved error recovery for function call arguments with trailing commas (#GH100921).
-
-- For an rvalue reference bound to a temporary struct with an integer member, Clang will detect constant integer overflow
-  in the initializer for the integer member (#GH46755).
-
-- Fixed a false negative ``-Wunused-private-field`` diagnostic when a defaulted comparison operator is defined out of class (#GH116961).
-
-- Clang now diagnoses dangling references for C++20's parenthesized aggregate initialization (#101957).
-
-- Fixed a bug where Clang would not emit ``-Wunused-private-field`` warnings when an unrelated class
-  defined a defaulted comparison operator (#GH116270).
-
-  .. code-block:: c++
-
-    class A {
-    private:
-      int a; // warning: private field 'a' is not used, no diagnostic previously
-    };
-
-    class C {
-      bool operator==(const C&) = default;
-    };
-
-- Clang now emits `-Wdangling-capture` diangostic when a STL container captures a dangling reference.
-
-  .. code-block:: c++
-
-    void test() {
-      std::vector<std::string_view> views;
-      views.push_back(std::string("123")); // warning
-    }
-
-- Clang now emits a ``-Wtautological-compare`` diagnostic when a check for
-  pointer addition overflow is always true or false, because overflow would
-  be undefined behavior.
-
-  .. code-block:: c++
-
-    bool incorrect_overflow_check(const char *ptr, size_t index) {
-      return ptr + index < ptr; // warning
-    }
-
-- Clang now emits a ``-Wvarargs`` diagnostic when the second argument
-  to ``va_arg`` is of array type, which is an undefined behavior (#GH119360).
-
-  .. code-block:: c++
-
-    void test() {
-      va_list va;
-      va_arg(va, int[10]); // warning
-    }
-
-- Fix -Wdangling false positives on conditional operators (#120206).
-
-- Fixed a bug where Clang hung on an unsupported optional scope specifier ``::`` when parsing
-  Objective-C. Clang now emits a diagnostic message instead of hanging.
-
-- The :doc:`ThreadSafetyAnalysis` now supports passing scoped capabilities into functions:
-  an attribute on the scoped capability parameter indicates both the expected associated capabilities and,
-  like in the case of attributes on the function declaration itself, their state before and after the call.
-
-  .. code-block:: c++
-
-    #include "mutex.h"
-
-    Mutex mu1, mu2;
-    int a GUARDED_BY(mu1);
-
-    void require(MutexLocker& scope REQUIRES(mu1)) {
-      scope.Unlock();
-      a = 0; // Warning!  Requires mu1.
-      scope.Lock();
-    }
-
-    void testParameter() {
-      MutexLocker scope(&mu1), scope2(&mu2);
-      require(scope2); // Warning! Mutex managed by 'scope2' is 'mu2' instead of 'mu1'
-      require(scope); // OK.
-      scope.Unlock();
-      require(scope); // Warning!  Requires mu1.
-    }
-- Diagnose invalid declarators in the declaration of constructors and destructors (#GH121706).
-
-- Fix false positives warning for non-std functions with name `infinity` (#123231).
-
-- Clang now emits a ``-Wignored-qualifiers`` diagnostic when a base class includes cv-qualifiers (#GH55474).
-
-- Clang now diagnoses the use of attribute names reserved by the C++ standard (#GH92196).
+- Improve the diagnostics for deleted default constructor errors for C++ class
+  initializer lists that don't explicitly list a class member and thus attempt
+  to implicitly default construct that member.
+- The ``-Wunique-object-duplication`` warning has been added to warn about objects
+  which are supposed to only exist once per program, but may get duplicated when
+  built into a shared library.
+- Fixed a bug where Clang's Analysis did not correctly model the destructor behavior of ``union`` members (#GH119415).
+- A statement attribute applied to a ``case`` label no longer suppresses
+  'bypassing variable initialization' diagnostics (#84072).
 
 Improvements to Clang's time-trace
 ----------------------------------
@@ -827,365 +145,78 @@ Improvements to Coverage Mapping
 Bug Fixes in This Version
 -------------------------
 
-- Fixed the definition of ``ATOMIC_FLAG_INIT`` in ``<stdatomic.h>`` so it can
-  be used in C++.
-- Fixed a failed assertion when checking required literal types in C context. (#GH101304).
-- Fixed a crash when trying to transform a dependent address space type. Fixes #GH101685.
-- Fixed a crash when diagnosing format strings and encountering an empty
-  delimited escape sequence (e.g., ``"\o{}"``). #GH102218
-- Fixed a crash using ``__array_rank`` on 64-bit targets. (#GH113044).
-- The warning emitted for an unsupported register variable type now points to
-  the unsupported type instead of the ``register`` keyword (#GH109776).
-- Fixed a crash when emit ctor for global variant with flexible array init (#GH113187).
-- Fixed a crash when GNU statement expression contains invalid statement (#GH113468).
-- Fixed a crash when passing the variable length array type to ``va_arg`` (#GH119360).
-- Fixed a failed assertion when using ``__attribute__((noderef))`` on an
-  ``_Atomic``-qualified type (#GH116124).
-- No longer incorrectly diagnosing use of a deleted destructor when the
-  selected overload of ``operator delete`` for that type is a destroying delete
-  (#GH46818).
-- No longer return ``false`` for ``noexcept`` expressions involving a
-  ``delete`` which resolves to a destroying delete but the type of the object
-  being deleted has a potentially throwing destructor (#GH118660).
-
 Bug Fixes to Compiler Builtins
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
-- Fix crash when atomic builtins are called with pointer to zero-size struct (#GH90330)
-
-- Clang now allows pointee types of atomic builtin arguments to be complete template types
-  that was not instantiated elsewhere.
-
-- ``__noop`` can now be used in a constant expression. (#GH102064)
-
-- Fix ``__has_builtin`` incorrectly returning ``false`` for some C++ type traits. (#GH111477)
-
-- Fix ``__builtin_source_location`` incorrectly returning wrong column for method chains. (#GH119129)
+- The behvaiour of ``__add_pointer`` and ``__remove_pointer`` for Objective-C++'s ``id`` and interfaces has been fixed.
 
 Bug Fixes to Attribute Support
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+ - Fixed crash when a parameter to the ``clang::annotate`` attribute evaluates to ``void``. See #GH119125
 
 Bug Fixes to C++ Support
 ^^^^^^^^^^^^^^^^^^^^^^^^
 
-- Fixed a crash when an expression with a dependent ``__typeof__`` type is used as the operand of a unary operator. (#GH97646)
-- Fixed incorrect pack expansion of init-capture references in requires expresssions.
-- Fixed a failed assertion when checking invalid delete operator declaration. (#GH96191)
-- Fix a crash when checking destructor reference with an invalid initializer. (#GH97230)
-- Clang now correctly parses potentially declarative nested-name-specifiers in pointer-to-member declarators.
-- Fix a crash when checking the initializer of an object that was initialized
-  with a string literal. (#GH82167)
-- Fix a crash when matching template template parameters with templates which have
-  parameters of different class type. (#GH101394)
-- Clang now correctly recognizes the correct context for parameter
-  substitutions in concepts, so it doesn't incorrectly complain of missing
-  module imports in those situations. (#GH60336)
-- Fix init-capture packs having a size of one before being instantiated. (#GH63677)
-- Clang now preserves the unexpanded flag in a lambda transform used for pack expansion. (#GH56852), (#GH85667),
-  (#GH99877), (#GH122417).
-- Fixed a bug when diagnosing ambiguous explicit specializations of constrained member functions.
-- Fixed an assertion failure when selecting a function from an overload set that includes a
-  specialization of a conversion function template.
-- Correctly diagnose attempts to use a concept name in its own definition;
-  A concept name is introduced to its scope sooner to match the C++ standard. (#GH55875)
-- Properly reject defaulted relational operators with invalid types for explicit object parameters,
-  e.g., ``bool operator==(this int, const Foo&)`` (#GH100329), and rvalue reference parameters.
-- Properly reject defaulted copy/move assignment operators that have a non-reference explicit object parameter.
-- Clang now properly handles the order of attributes in `extern` blocks. (#GH101990).
-- Fixed an assertion failure by preventing null explicit object arguments from being deduced. (#GH102025).
-- Correctly check constraints of explicit instantiations of member functions. (#GH46029)
-- When performing partial ordering of function templates, clang now checks that
-  the deduction was consistent. Fixes (#GH18291).
-- Fixes to several issues in partial ordering of template template parameters, which
-  were documented in the test suite.
-- Fixed an assertion failure about a constraint of a friend function template references to a value with greater
-  template depth than the friend function template. (#GH98258)
-- Clang now rebuilds the template parameters of out-of-line declarations and specializations in the context
-  of the current instantiation in all cases.
-- Fix evaluation of the index of dependent pack indexing expressions/types specifiers (#GH105900)
-- Correctly handle subexpressions of an immediate invocation in the presence of implicit casts. (#GH105558)
-- Clang now correctly handles direct-list-initialization of a structured bindings from an array. (#GH31813)
-- Mangle placeholders for deduced types as a template-prefix, such that mangling
-  of template template parameters uses the correct production. (#GH106182)
-- Fixed an assertion failure when converting vectors to int/float with invalid expressions. (#GH105486)
-- Template parameter names are considered in the name lookup of out-of-line class template
-  specialization right before its declaration context. (#GH64082)
-- Fixed a constraint comparison bug for friend declarations. (#GH78101)
-- Fix handling of ``_`` as the name of a lambda's init capture variable. (#GH107024)
-- Fix an issue with dependent source location expressions (#GH106428), (#GH81155), (#GH80210), (#GH85373)
-- Fixed a bug in the substitution of empty pack indexing types. (#GH105903)
-- Clang no longer tries to capture non-odr used default arguments of template parameters of generic lambdas (#GH107048)
-- Fixed a bug where defaulted comparison operators would remove ``const`` from base classes. (#GH102588)
-- Fix a crash when using ``source_location`` in the trailing return type of a lambda expression. (#GH67134)
-- A follow-up fix was added for (#GH61460), as the previous fix was not entirely correct. (#GH86361), (#GH112352)
-- Fixed a crash in the typo correction of an invalid CTAD guide. (#GH107887)
-- Fixed a crash when clang tries to substitute parameter pack while retaining the parameter
-  pack. (#GH63819), (#GH107560)
-- Fix a crash when a static assert declaration has an invalid close location. (#GH108687)
-- Avoided a redundant friend declaration instantiation under a certain ``consteval`` context. (#GH107175)
-- Fixed an assertion failure in debug mode, and potential crashes in release mode, when
-  diagnosing a failed cast caused indirectly by a failed implicit conversion to the type of the constructor parameter.
-- Fixed an assertion failure by adjusting integral to boolean vector conversions (#GH108326)
-- Fixed a crash when mixture of designated and non-designated initializers in union. (#GH113855)
-- Fixed an issue deducing non-type template arguments of reference type. (#GH73460)
-- Fixed an issue in constraint evaluation, where type constraints on the lambda expression
-  containing outer unexpanded parameters were not correctly expanded. (#GH101754)
-- Fixes crashes with function template member specializations, and increases
-  conformance of explicit instantiation behaviour with MSVC. (#GH111266)
-- Fixed a bug in constraint expression comparison where the ``sizeof...`` expression was not handled properly
-  in certain friend declarations. (#GH93099)
-- Clang now instantiates the correct lambda call operator when a lambda's class type is
-  merged across modules. (#GH110401)
-- Fix a crash when parsing a pseudo destructor involving an invalid type. (#GH111460)
-- Fixed an assertion failure when invoking recovery call expressions with explicit attributes
-  and undeclared templates. (#GH107047), (#GH49093)
-- Clang no longer crashes when a lambda contains an invalid block declaration that contains an unexpanded
-  parameter pack. (#GH109148)
-- Fixed overload handling for object parameters with top-level cv-qualifiers in explicit member functions (#GH100394)
-- Fixed a bug in lambda captures where ``constexpr`` class-type objects were not properly considered ODR-used in
-  certain situations. (#GH47400), (#GH90896)
-- Fix erroneous templated array size calculation leading to crashes in generated code. (#GH41441)
-- During the lookup for a base class name, non-type names are ignored. (#GH16855)
-- Fix a crash when recovering an invalid expression involving an explicit object member conversion operator. (#GH112559)
-- Clang incorrectly considered a class with an anonymous union member to not be
-  const-default-constructible even if a union member has a default member initializer.
-  (#GH95854).
-- Fixed an assertion failure when evaluating an invalid expression in an array initializer. (#GH112140)
-- Fixed an assertion failure in range calculations for conditional throw expressions. (#GH111854)
-- Clang now correctly ignores previous partial specializations of member templates explicitly specialized for
-  an implicitly instantiated class template specialization. (#GH51051)
-- Fixed an assertion failure caused by invalid enum forward declarations. (#GH112208)
-- Name independent data members were not correctly initialized from default member initializers. (#GH114069)
-- Fixed expression transformation for ``[[assume(...)]]``, allowing using pack indexing expressions within the
-  assumption if they also occur inside of a dependent lambda. (#GH114787)
-- Lambdas now capture function types without considering top-level const qualifiers. (#GH84961)
-- Clang now uses valid deduced type locations when diagnosing functions with trailing return type
-  missing placeholder return type. (#GH78694)
-- Fixed a bug where bounds of partially expanded pack indexing expressions were checked too early. (#GH116105)
-- Fixed an assertion failure caused by using ``consteval`` in condition in consumed analyses. (#GH117385)
-- Fixed an assertion failure caused by invalid default argument substitutions in non-defining
-  friend declarations. (#GH113324)
-- Fix a crash caused by incorrect argument position in merging deduced template arguments. (#GH113659)
-- Fixed a parser crash when using pack indexing as a nested name specifier. (#GH119072)
-- Fixed a null pointer dereference issue when heuristically computing ``sizeof...(pack)`` expressions. (#GH81436)
-- Fixed an assertion failure caused by mangled names with invalid identifiers. (#GH112205)
-- Fixed an incorrect lambda scope of generic lambdas that caused Clang to crash when computing potential lambda
-  captures at the end of a full expression. (#GH115931)
-- Clang no longer rejects deleting a pointer of incomplete enumeration type. (#GH99278)
-- Fixed recognition of ``std::initializer_list`` when it's surrounded with ``extern "C++"`` and exported
-  out of a module (which is the case e.g. in MSVC's implementation of ``std`` module). (#GH118218)
-- Fixed a pack expansion issue in checking unexpanded parameter sizes. (#GH17042)
-- Fixed a bug where captured structured bindings were modifiable inside non-mutable lambda (#GH95081)
-- Passing incomplete types to ``__is_base_of`` and other builtin type traits for which the corresponding
-  standard type trait mandates a complete type is now a hard (non-sfinae-friendly) error
-  (`LWG3929 <https://wg21.link/LWG3929>`__.) (#GH121278)
-- Clang now identifies unexpanded parameter packs within the type constraint on a non-type template parameter. (#GH88866)
-- Fixed an issue while resolving type of expression indexing into a pack of values of non-dependent type (#GH121242)
-- Fixed a crash when __PRETTY_FUNCTION__ or __FUNCSIG__ (clang-cl) appears in the trailing return type of the lambda (#GH121274)
-- Fixed a crash caused by the incorrect construction of template arguments for CTAD alias guides when type
-  constraints are applied. (#GH122134)
-- Fixed canonicalization of pack indexing types - Clang did not always recognized identical pack indexing. (#GH123033)
-- Fixed a nested lambda substitution issue for constraint evaluation. (#GH123441)
-- Fixed various false diagnostics related to the use of immediate functions. (#GH123472)
-- Fix immediate escalation not propagating through inherited constructors.  (#GH112677)
-- Fixed assertions or false compiler diagnostics in the case of C++ modules for
-  lambda functions or inline friend functions defined inside templates (#GH122493).
+- Clang is now better at keeping track of friend function template instance contexts. (#GH55509)
+- The initialization kind of elements of structured bindings
+  direct-list-initialized from an array is corrected to direct-initialization.
 
 Bug Fixes to AST Handling
 ^^^^^^^^^^^^^^^^^^^^^^^^^
 
-- Fixed a crash that occurred when dividing by zero in complex integer division. (#GH55390).
-- Fixed a bug in ``ASTContext::getRawCommentForAnyRedecl()`` where the function could
-  sometimes incorrectly return null even if a comment was present. (#GH108145)
-- Clang now correctly parses the argument of the ``relates``, ``related``, ``relatesalso``,
-  and ``relatedalso`` comment commands.
-- Clang now uses the location of the begin of the member expression for ``CallExpr``
-  involving deduced ``this``. (#GH116928)
-- Fixed printout of AST that uses pack indexing expression. (#GH116486)
-
 Miscellaneous Bug Fixes
 ^^^^^^^^^^^^^^^^^^^^^^^
 
+- HTML tags in comments that span multiple lines are now parsed correctly by Clang's comment parser. (#GH120843)
+
 Miscellaneous Clang Crashes Fixed
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
-- Fixed a crash in C due to incorrect lookup that members in nested anonymous struct/union
-  can be found as ordinary identifiers in struct/union definition. (#GH31295)
-
-- Fixed a crash caused by long chains of ``sizeof`` and other similar operators
-  that can be followed by a non-parenthesized expression. (#GH45061)
-
-- Fixed an crash when compiling ``#pragma STDC FP_CONTRACT DEFAULT`` with
-  ``-ffp-contract=fast-honor-pragmas``. (#GH104830)
-
-- Fixed a crash when function has more than 65536 parameters.
-  Now a diagnostic is emitted. (#GH35741)
-
-- Fixed ``-ast-dump`` crashes on codes involving ``concept`` with ``-ast-dump-decl-types``. (#GH94928)
-
-- Fixed internal assertion firing when a declaration in the implicit global
-  module is found through ADL. (GH#109879)
-
-- Fixed a crash when an unscoped enumeration declared by an opaque-enum-declaration within a class template
-  with a dependent underlying type is subject to integral promotion. (#GH117960)
-
 OpenACC Specific Changes
 ------------------------
 
 Target Specific Changes
 -----------------------
 
-- Clang now implements the Solaris-specific mangling of ``std::tm`` as
-  ``tm``, same for ``std::div_t``, ``std::ldiv_t``, and
-  ``std::lconv``, for Solaris ABI compatibility. (#GH33114)
-
 AMDGPU Support
 ^^^^^^^^^^^^^^
 
-- Initial support for gfx950
-
-- Added headers ``gpuintrin.h`` and ``amdgpuintrin.h`` that contains common
-  definitions for GPU builtin functions. This header can be included for OpenMP,
-  CUDA, HIP, OpenCL, and C/C++.
-
 NVPTX Support
 ^^^^^^^^^^^^^^
 
-- Added headers ``gpuintrin.h`` and ``nvptxintrin.h`` that contains common
-  definitions for GPU builtin functions. This header can be included for OpenMP,
-  CUDA, HIP, OpenCL, and C/C++.
+Hexagon Support
+^^^^^^^^^^^^^^^
+
+-  The default compilation target has been changed from V60 to V68.
 
 X86 Support
 ^^^^^^^^^^^
 
-- The MMX vector intrinsic functions from ``*mmintrin.h`` which
-  operate on `__m64` vectors, such as ``_mm_add_pi8``, have been
-  reimplemented to use the SSE2 instruction-set and XMM registers
-  unconditionally. These intrinsics are therefore *no longer
-  supported* if MMX is enabled without SSE2 -- either from targeting
-  CPUs from the Pentium-MMX through the Pentium 3, or explicitly via
-  passing arguments such as ``-mmmx -mno-sse2``. MMX assembly code
-  remains supported without requiring SSE2, including inside
-  inline-assembly.
-
-- The compiler builtins such as ``__builtin_ia32_paddb`` which
-  formerly implemented the above MMX intrinsic functions have been
-  removed. Any uses of these removed functions should migrate to the
-  functions defined by the ``*mmintrin.h`` headers. A mapping can be
-  found in the file ``clang/www/builtins.py``.
-
-- Support ISA of ``AVX10.2``.
-  * Supported MINMAX intrinsics of ``*_(mask(z)))_minmax(ne)_p[s|d|h|bh]`` and
-  ``*_(mask(z)))_minmax_s[s|d|h]``.
-
-- Supported intrinsics for ``SM4 and AVX10.2``.
-  * Supported SM4 intrinsics of ``_mm512_sm4key4_epi32`` and
-  ``_mm512_sm4rnds4_epi32``.
-
-- All intrinsics in adcintrin.h can now be used in constant expressions.
-
-- All intrinsics in adxintrin.h can now be used in constant expressions.
-
-- All intrinsics in lzcntintrin.h can now be used in constant expressions.
-
-- All intrinsics in bmiintrin.h can now be used in constant expressions.
-
-- All intrinsics in bmi2intrin.h can now be used in constant expressions.
-
-- All intrinsics in tbmintrin.h can now be used in constant expressions.
-
-- Supported intrinsics for ``MOVRS AND AVX10.2``.
-  * Supported intrinsics of ``_mm(256|512)_(mask(z))_loadrs_epi(8|16|32|64)``.
-- Support ISA of ``AMX-FP8``.
-- Support ISA of ``AMX-TRANSPOSE``.
-- Support ISA of ``AMX-MOVRS``.
-- Support ISA of ``AMX-AVX512``.
-- Support ISA of ``AMX-TF32``.
-- Support ISA of ``MOVRS``.
-
-- Supported ``-march/tune=diamondrapids``
+- Disable ``-m[no-]avx10.1`` and switch ``-m[no-]avx10.2`` to alias of 512 bit
+  options.
+- Change ``-mno-avx10.1-512`` to alias of ``-mno-avx10.1-256`` to disable both
+  256 and 512 bit instructions.
 
 Arm and AArch64 Support
 ^^^^^^^^^^^^^^^^^^^^^^^
 
-- Implementation of SVE2.1 and SME2.1 in accordance with the Arm C Language
-  Extensions (ACLE) is now available.
-
-- In the ARM Target, the frame pointer (FP) of a leaf function can be retained
-  by using the ``-fno-omit-frame-pointer`` option. If you want to eliminate the FP
-  in leaf functions after enabling ``-fno-omit-frame-pointer``, you can do so by adding
-  the ``-momit-leaf-frame-pointer`` option.
-
-- SME keyword attributes which apply to function types are now represented in the
-  mangling of the type. This means that ``void foo(void (*f)() __arm_streaming);``
-  now has a different mangling from ``void foo(void (*f)());``.
-
-- The ``__arm_agnostic`` keyword attribute was added to let users describe
-  a function that preserves SME state enabled by PSTATE.ZA without having to share
-  this state with its callers and without making the assumption that this state
-  exists.
-
-- Support has been added for the following processors (-mcpu identifiers in parenthesis):
-
-  For AArch64:
-
-  * FUJITSU-MONAKA (fujitsu-monaka)
-
 Android Support
 ^^^^^^^^^^^^^^^
 
 Windows Support
 ^^^^^^^^^^^^^^^
 
-- clang-cl now supports ``/std:c++23preview`` which enables C++23 features.
-
-- Clang no longer allows references inside a union when emulating MSVC 1900+ even if `fms-extensions` is enabled.
-  Starting with VS2015, MSVC 1900, this Microsoft extension is no longer allowed and always results in an error.
-  Clang now follows the MSVC behavior in this scenario.
-  When `-fms-compatibility-version=18.00` or prior is set on the command line this Microsoft extension is still
-  allowed as VS2013 and prior allow it.
-
-- Clang now supports the ``#pragma clang section`` directive for COFF targets.
-
 LoongArch Support
 ^^^^^^^^^^^^^^^^^
 
-- Types of parameters and return value of ``__builtin_lsx_vorn_v`` and ``__builtin_lasx_xvorn_v``
-  are changed from ``signed char`` to ``unsigned char``. (#GH114514)
-
-- ``-mrelax`` and ``-mno-relax`` are supported now on LoongArch that can be used
-  to enable / disable the linker relaxation optimization. (#GH123587)
-
-- Fine-grained la64v1.1 options are added including ``-m{no-,}frecipe``, ``-m{no-,}lam-bh``,
-  ``-m{no-,}ld-seq-sa``, ``-m{no-,}div32``, ``-m{no-,}lamcas`` and ``-m{no-,}scq``.
-
-- Two options ``-m{no-,}annotate-tablejump`` are added to enable / disable
-  annotating table jump instruction to correlate it with the jump table. (#GH102411)
-
-- FreeBSD support is added for LoongArch64 and has been tested by building kernel-toolchain. (#GH119191)
-
 RISC-V Support
 ^^^^^^^^^^^^^^
 
-- The option ``-mcmodel=large`` for the large code model is supported.
-- Bump RVV intrinsic to version 1.0, the spec: https://github.com/riscv-non-isa/rvv-intrinsic-doc/releases/tag/v1.0.0-rc4
-
 CUDA/HIP Language Changes
 ^^^^^^^^^^^^^^^^^^^^^^^^^
-- Fixed a bug about overriding a constexpr pure-virtual member function with a non-constexpr virtual member function which causes compilation failure when including standard C++ header `format`.
-- Added initial support for version 3 of the compressed offload bundle format, which uses 64-bit fields for Total File Size and Uncompressed Binary Size. This enables support for files larger than 4GB. The support is currently experimental and can be enabled by setting the environment variable `COMPRESSED_BUNDLE_FORMAT_VERSION=3`.
 
 CUDA Support
 ^^^^^^^^^^^^
-- Clang now supports CUDA SDK up to 12.6
-- Added support for sm_100
-- Added support for `__grid_constant__` attribute.
-- CUDA now uses the new offloading driver by default. The new driver supports
-  device-side LTO, interoperability with OpenMP and other languages, and native ``-fgpu-rdc``
-  support with static libraries. The old behavior can be returned using the
-  ``--no-offload-new-driver`` flag. The binary format is no longer compatible
-  with the NVIDIA compiler's RDC-mode support. More information can be found at:
-  https://clang.llvm.org/docs/OffloadingDesign.html
 
 AIX Support
 ^^^^^^^^^^^
@@ -1196,276 +227,73 @@ NetBSD Support
 WebAssembly Support
 ^^^^^^^^^^^^^^^^^^^
 
-The default target CPU, "generic", now enables the `-mnontrapping-fptoint`
-and `-mbulk-memory` flags, which correspond to the [Bulk Memory Operations]
-and [Non-trapping float-to-int Conversions] language features, which are
-[widely implemented in engines].
-
-A new Lime1 target CPU is added, -mcpu=lime1. This CPU follows the definition of
-the Lime1 CPU [here], and enables -mmultivalue, -mmutable-globals,
--mcall-indirect-overlong, -msign-ext, -mbulk-memory-opt, -mnontrapping-fptoint,
-and -mextended-const.
-
-[Bulk Memory Operations]: https://github.com/WebAssembly/bulk-memory-operations/blob/master/proposals/bulk-memory-operations/Overview.md
-[Non-trapping float-to-int Conversions]: https://github.com/WebAssembly/spec/blob/master/proposals/nontrapping-float-to-int-conversion/Overview.md
-[widely implemented in engines]: https://webassembly.org/features/
-[here]: https://github.com/WebAssembly/tool-conventions/blob/main/Lime.md#lime1
-
 AVR Support
 ^^^^^^^^^^^
 
-- Reject C/C++ compilation for avr1 devices which have no SRAM.
-
 DWARF Support in Clang
 ----------------------
 
 Floating Point Support in Clang
 -------------------------------
 
-- Add ``__builtin_elementwise_atan2`` builtin for floating point types only.
-
 Fixed Point Support in Clang
 ----------------------------
 
 AST Matchers
 ------------
 
-- Fixed an issue with the `hasName` and `hasAnyName` matcher when matching
-  inline namespaces with an enclosing namespace of the same name.
-
-- Fixed an ordering issue with the `hasOperands` matcher occurring when setting a
-  binding in the first matcher and using it in the second matcher.
-
-- Fixed a crash when traverse lambda expr with invalid captures. (#GH106444)
-
-- Fixed ``isInstantiated`` and ``isInTemplateInstantiation`` to also match for variable templates. (#GH110666)
-
-- Ensure ``hasName`` matches template specializations across inline namespaces,
-  making `matchesNodeFullSlow` and `matchesNodeFullFast` consistent.
-
-- Improved the performance of the ``getExpansionLocOfMacro`` by tracking already processed macros during recursion.
-
-- Add ``exportDecl`` matcher to match export declaration.
-
-- Ensure ``hasType`` and ``hasDeclaration`` match Objective-C interface declarations.
-
-- Ensure ``pointee`` matches Objective-C pointer types.
-
-- Add ``dependentScopeDeclRefExpr`` matcher to match expressions that refer to dependent scope declarations.
-
-- Add ``dependentNameType`` matcher to match a dependent name type.
-
-- Add ``dependentTemplateSpecializationType`` matcher to match a dependent template specialization type.
-
-- Add ``hasDependentName`` matcher to match the dependent name of a DependentScopeDeclRefExpr or DependentNameType.
-
 clang-format
 ------------
 
-- Adds ``BreakBinaryOperations`` option.
-- Adds ``TemplateNames`` option.
-- Adds ``AlignFunctionDeclarations`` option to ``AlignConsecutiveDeclarations``.
-- Adds ``IndentOnly`` suboption to ``ReflowComments`` to fix the indentation of
-  multi-line comments without touching their contents, renames ``false`` to
-  ``Never``, and ``true`` to ``Always``.
-- Adds ``RemoveEmptyLinesInUnwrappedLines`` option.
-- Adds ``KeepFormFeed`` option and set it to ``true`` for ``GNU`` style.
-- Adds ``AllowShortNamespacesOnASingleLine`` option.
-- Adds ``VariableTemplates`` option.
-- Adds support for bash globstar in ``.clang-format-ignore``.
-- Adds ``WrapNamespaceBodyWithEmptyLines`` option.
-- Adds the ``IndentExportBlock`` option.
+- Adds ``BreakBeforeTemplateCloser`` option.
+- Adds ``BinPackLongBracedList`` option to override bin packing options in
+  long (20 item or more) braced list initializer lists.
 
 libclang
 --------
-- Add ``clang_isBeforeInTranslationUnit``. Given two source locations, it determines
-  whether the first one comes strictly before the second in the source code.
-- Add ``clang_getTypePrettyPrinted``.  It allows controlling the PrintingPolicy used
-  to pretty-print a type.
-- Added ``clang_visitCXXBaseClasses``, which allows visiting the base classes
-  of a class.
-- Added ``clang_getOffsetOfBase``, which allows computing the offset of a base
-  class in a class's layout.
-
 
 Code Completion
 ---------------
 
-- Use ``HeuristicResolver`` (upstreamed from clangd) to improve code completion results
-  in dependent code
-
 Static Analyzer
 ---------------
 
 New features
 ^^^^^^^^^^^^
 
-- Now CSA models `__builtin_*_overflow` functions. (#GH102602)
-
-- MallocChecker now checks for ``ownership_returns(class, idx)`` and ``ownership_takes(class, idx)``
-  attributes with class names different from "malloc". Clang static analyzer now reports an error
-  if class of allocation and deallocation function mismatches.
-  `Documentation <https://clang.llvm.org/docs/analyzer/checkers.html#unix-mismatcheddeallocator-c-c>`__.
-
-- Function effects, e.g. the ``nonblocking`` and ``nonallocating`` "performance constraint"
-  attributes, are now verified. For example, for functions declared with the ``nonblocking``
-  attribute, the compiler can generate warnings about the use of any language features, or calls to
-  other functions, which may block.
-
-- Introduced ``-warning-suppression-mappings`` flag to control diagnostic
-  suppressions per file. See `documentation <https://clang.llvm.org/docs/WarningSuppressionMappings.html>_` for details.
+A new flag - `-static-libclosure` was introduced to support statically linking
+the runtime for the Blocks extension on Windows. This flag currently only
+changes the code generation, and even then, only on Windows. This does not
+impact the linker behaviour like the other `-static-*` flags.
 
 Crash and bug fixes
 ^^^^^^^^^^^^^^^^^^^
 
-- In loops where the loop condition is opaque (i.e. the analyzer cannot
-  determine whether it's true or false), the analyzer will no longer assume
-  execution paths that perform more that two iterations. These unjustified
-  assumptions caused false positive reports (e.g. 100+ out-of-bounds reports in
-  the FFMPEG codebase) in loops where the programmer intended only two or three
-  steps but the analyzer wasn't able to understand that the loop is limited.
-
 Improvements
 ^^^^^^^^^^^^
 
-- Improved the handling of the ``ownership_returns`` attribute. Now, Clang reports an
-  error if the attribute is attached to a function that returns a non-pointer value.
-  Fixes (#GH99501)
-
 Moved checkers
 ^^^^^^^^^^^^^^
 
-- The checker ``alpha.core.IdenticalExpr`` was deleted because it was
-  duplicated in the clang-tidy checkers ``misc-redundant-expression`` and
-  ``bugprone-branch-clone``.
-
-- The checker ``alpha.security.MallocOverflow`` was deleted because it was
-  badly implemented and its aggressive logic produced too many false positives.
-  To detect too large arguments passed to malloc, consider using the checker
-  ``alpha.taint.TaintedAlloc``.
-
-- The checkers ``alpha.nondeterministic.PointerSorting`` and
-  ``alpha.nondeterministic.PointerIteration`` were moved to a new bugprone
-  checker named ``bugprone-nondeterministic-pointer-iteration-order``. The
-  original checkers were implemented only using AST matching and make more
-  sense as a single clang-tidy check.
-
-- The checker ``alpha.unix.Chroot`` was modernized, improved and moved to
-  ``unix.Chroot``. Testing was done on open source projects that use chroot(),
-  and false issues addressed in the improvements based on real use cases. Open
-  source projects used for testing include nsjail, lxroot, dive and ruri.
-  This checker conforms to SEI Cert C recommendation `POS05-C. Limit access to
-  files by creating a jail
-  <https://wiki.sei.cmu.edu/confluence/display/c/POS05-C.+Limit+access+to+files+by+creating+a+jail>`_.
-  Fixes (#GH34697).
-  (#GH117791) [Documentation](https://clang.llvm.org/docs/analyzer/checkers.html#unix-chroot-c).
+- After lots of improvements, the checker ``alpha.security.ArrayBoundV2`` is
+  renamed to ``security.ArrayBound``. As this checker is stable now, the old
+  checker ``alpha.security.ArrayBound`` (which was searching for the same kind
+  of bugs with an different, simpler and less accurate algorithm) is removed.
 
 .. _release-notes-sanitizers:
 
 Sanitizers
 ----------
-- Introduced Realtime Sanitizer, activated by using the -fsanitize=realtime
-  flag. This sanitizer detects unsafe system library calls, such as memory
-  allocations and mutex locks. If any such function is called during invocation
-  of a function marked with the ``[[clang::nonblocking]]`` attribute, an error
-  is printed to the console and the process exits non-zero.
-
-- Added the ``-fsanitize-undefined-ignore-overflow-pattern`` flag which can be
-  used to disable specific overflow-dependent code patterns. The supported
-  patterns are: ``add-signed-overflow-test``, ``add-unsigned-overflow-test``,
-  ``negated-unsigned-const``, and ``unsigned-post-decr-while``. The sanitizer
-  instrumentation can be toggled off for all available patterns by specifying
-  ``all``. Conversely, you may disable all exclusions with ``none`` which is
-  the default.
-
-  .. code-block:: c++
-
-     /// specified with ``-fsanitize-undefined-ignore-overflow-pattern=add-unsigned-overflow-test``
-     int common_overflow_check_pattern(unsigned base, unsigned offset) {
-       if (base + offset < base) { /* ... */ } // The pattern of `a + b < a`, and other re-orderings, won't be instrumented
-     }
-
-     /// specified with ``-fsanitize-undefined-ignore-overflow-pattern=add-signed-overflow-test``
-     int common_overflow_check_pattern_signed(signed int base, signed int offset) {
-       if (base + offset < base) { /* ... */ } // The pattern of `a + b < a`, and other re-orderings, won't be instrumented
-     }
-
-     /// specified with ``-fsanitize-undefined-ignore-overflow-pattern=negated-unsigned-const``
-     void negation_overflow() {
-       unsigned long foo = -1UL; // No longer causes a negation overflow warning
-       unsigned long bar = -2UL; // and so on...
-     }
-
-     /// specified with ``-fsanitize-undefined-ignore-overflow-pattern=unsigned-post-decr-while``
-     void while_post_decrement() {
-       unsigned char count = 16;
-       while (count--) { /* ... */ } // No longer causes unsigned-integer-overflow sanitizer to trip
-     }
-
-  Many existing projects have a large amount of these code patterns present.
-  This new flag should allow those projects to enable integer sanitizers with
-  less noise.
-
-- ``-fsanitize=signed-integer-overflow``, ``-fsanitize=unsigned-integer-overflow``,
-  ``-fsanitize=implicit-signed-integer-truncation``, ``-fsanitize=implicit-unsigned-integer-truncation``,
-  ``-fsanitize=enum`` now properly support the
-  "type" prefix within `Sanitizer Special Case Lists (SSCL)
-  <https://clang.llvm.org/docs/SanitizerSpecialCaseList.html>`_. See that link
-  for examples.
-
-- Introduced an experimental Type Sanitizer, activated by using the
-  ``-fsanitize=type`` flag. This sanitizer detects violations of C/C++ type-based
-  aliasing rules.
-
-- Implemented ``-f[no-]sanitize-trap=local-bounds``, and ``-f[no-]sanitize-recover=local-bounds``.
-
-- ``-fsanitize-merge`` (default) and ``-fno-sanitize-merge`` have been added for
-  fine-grained, unified control of which UBSan checks can potentially be merged
-  by the compiler (for example,
-  ``-fno-sanitize-merge=bool,enum,array-bounds,local-bounds``).
-
-- Changed ``-fsanitize=pointer-overflow`` to no longer report ``NULL + 0`` as
-  undefined behavior in C, in line with
-  `N3322 <https://www.open-std.org/jtc1/sc22/wg14/www/docs/n3322.pdf>`_,
-  and matching the previous behavior for C++.
-  ``NULL + non_zero`` continues to be reported as undefined behavior.
 
 Python Binding Changes
 ----------------------
-- Fixed an issue that led to crashes when calling ``Type.get_exception_specification_kind``.
-- Added ``Cursor.pretty_printed``, a binding for ``clang_getCursorPrettyPrinted``,
-  and related functions, which allow changing the formatting of pretty-printed code.
-- Added ``Cursor.is_anonymous_record_decl``, a binding for
-  ``clang_Cursor_isAnonymousRecordDecl``, which allows checking if a
-  declaration is an anonymous union or anonymous struct.
-- Added ``Type.pretty_printed`, a binding for ``clang_getTypePrettyPrinted``,
-  which allows changing the formatting of pretty-printed types.
-- Added ``Cursor.is_virtual_base``, a binding for ``clang_isVirtualBase``,
-  which checks whether a base class is virtual.
-- Added ``Type.get_bases``, a binding for ``clang_visitCXXBaseClasses``, which
-  allows visiting the base classes of a class.
-- Added ``Cursor.get_base_offsetof``, a binding for ``clang_getOffsetOfBase``,
-  which allows computing the offset of a base class in a class's layout.
 
 OpenMP Support
 --------------
-- Added support for 'omp assume' directive.
-- Added support for 'omp scope' directive.
-- Added support for allocator-modifier in 'allocate' clause.
-- Changed the OpenMP DeviceRTL to use 'generic' IR. The
-  ``LIBOMPTARGET_DEVICE_ARCHITECTURES`` CMake argument is now unused and will
-  always build support for AMDGPU and NVPTX targets.
-- Added support for combined masked constructs  'omp parallel masked taskloop',
-  'omp parallel masked taskloop simd','omp masked taskloop' and 'omp masked taskloop simd' directive.
-- Added support for align-modifier in 'allocate' clause.
+- Added support 'no_openmp_constructs' assumption clause.
 
 Improvements
 ^^^^^^^^^^^^
-- Improve the handling of mapping array-section for struct containing nested structs with user defined mappers
-
-- `num_teams` and `thead_limit` now accept multiple expressions when it is used
-  along in ``target teams ompx_bare`` construct. This allows the target region
-  to be launched with multi-dim grid on GPUs.
 
 Additional Information
 ======================
diff --git a/clang/docs/SourceBasedCodeCoverage.rst b/clang/docs/SourceBasedCodeCoverage.rst
index 73910e134a589..3e8642479a56d 100644
--- a/clang/docs/SourceBasedCodeCoverage.rst
+++ b/clang/docs/SourceBasedCodeCoverage.rst
@@ -94,6 +94,11 @@ directory structure will be created.  Additionally, the following special
   not specified (i.e the pattern is "%m"), it's assumed that ``N = 1``. The
   merge pool specifier can only occur once per filename pattern.
 
+* "%b" expands out to the binary ID (build ID). It can be used with "%Nm" to
+  avoid binary signature collisions. To use it, the program should be compiled
+  with the build ID linker option (``--build-id`` for GNU ld or LLD,
+  ``/build-id`` for lld-link on Windows). Linux, Windows and AIX are supported.
+
 * "%c" expands out to nothing, but enables a mode in which profile counter
   updates are continuously synced to a file. This means that if the
   instrumented program crashes, or is killed by a signal, perfect coverage
diff --git a/clang/docs/TypeSanitizer.rst b/clang/docs/TypeSanitizer.rst
new file mode 100644
index 0000000000000..3c683a6c24bb4
--- /dev/null
+++ b/clang/docs/TypeSanitizer.rst
@@ -0,0 +1,205 @@
+=============
+TypeSanitizer
+=============
+
+.. contents::
+   :local:
+
+Introduction
+============
+
+The TypeSanitizer is a detector for strict type aliasing violations. It consists of a compiler
+instrumentation module and a run-time library. C/C++ has type-based aliasing rules, and LLVM 
+can exploit these for optimizations given the TBAA metadata Clang emits. In general, a pointer 
+of a given type cannot access an object of a different type, with only a few exceptions. 
+
+These rules aren't always apparent to users, which leads to code that violates these rules
+(e.g. for type punning). This can lead to optimization passes introducing bugs unless the 
+code is build with ``-fno-strict-aliasing``, sacrificing performance.
+
+TypeSanitizer is built to catch when these strict aliasing rules have been violated, helping 
+users find where such bugs originate in their code despite the code looking valid at first glance.
+
+As TypeSanitizer is still experimental, it can currently have a large impact on runtime speed, 
+memory use, and code size. It also has a large compile-time overhead. Work is being done to 
+reduce these impacts.
+
+The TypeSanitizer Algorithm
+===========================
+For each TBAA type-access descriptor, encoded in LLVM IR using TBAA Metadata, the instrumentation 
+pass generates descriptor tables. Thus there is a unique pointer to each type (and access descriptor).
+These tables are comdat (except for anonymous-namespace types), so the pointer values are unique 
+across the program.
+
+The descriptors refer to other descriptors to form a type aliasing tree, like how LLVM's TBAA data 
+does.
+
+The runtime uses 8 bytes of shadow memory, the size of the pointer to the type descriptor, for 
+every byte of accessed data in the program. The first byte of a type will have its shadow memory 
+be set to the pointer to its type descriptor. Aside from that, there are some other values it may be.
+
+* 0 is used to represent an unknown type
+* Negative numbers represent an interior byte: A byte inside a type that is not the first one. As an 
+  example, a value of -2 means you are in the third byte of a type.
+
+The Instrumentation first checks for an exact match between the type of the current access and the 
+type for that address in the shadow memory. This can quickly be done by checking pointer values. If 
+it matches, it checks the remaining shadow memory of the type to ensure they are the correct negative 
+numbers. If this fails, it calls the "slow path" check. If the exact match fails, we check to see if 
+the value, and the remainder of the shadow bytes, is 0. If they are, we can set the shadow memory to 
+the correct type descriptor pointer for the first byte, and the correct negative numbers for the rest 
+of the type's shadow.
+
+If the type in shadow memory is neither an exact match nor 0, we call the slower runtime check. It 
+uses the full TBAA algorithm, just as the compiler does, to determine when two types are permitted to 
+alias.
+
+The instrumentation pass inserts calls to the memset intrinsic to set the memory updated by memset, 
+memcpy, and memmove, as well as allocas/byval (and for lifetime.start/end) to reset the shadow memory 
+to reflect that the type is now unknown. The runtime intercepts memset, memcpy, etc. to perform the 
+same function for the library calls.
+
+How to build
+============
+
+Build LLVM/Clang with `CMake <https://llvm.org/docs/CMake.html>`_ and enable
+the ``compiler-rt`` runtime. An example CMake configuration that will allow
+for the use/testing of TypeSanitizer:
+
+.. code-block:: console
+
+   $ cmake -DCMAKE_BUILD_TYPE=Release -DLLVM_ENABLE_PROJECTS="clang" -DLLVM_ENABLE_RUNTIMES="compiler-rt" <path to source>/llvm
+
+Usage
+=====
+
+Compile and link your program with ``-fsanitize=type`` flag. The
+TypeSanitizer run-time library should be linked to the final executable, so
+make sure to use ``clang`` (not ``ld``) for the final link step. To
+get a reasonable performance add ``-O1`` or higher.
+TypeSanitizer by default doesn't print the full stack trace in error messages. Use ``TYSAN_OPTIONS=print_stacktrace=1`` 
+to print the full trace. To get nicer stack traces in error messages add ``-fno-omit-frame-pointer`` and 
+``-g``.  To get perfect stack traces you may need to disable inlining (just use ``-O1``) and tail call elimination 
+(``-fno-optimize-sibling-calls``).
+
+.. code-block:: console
+
+    % cat example_AliasViolation.c
+    int main(int argc, char **argv) {
+      int x = 100;
+      float *y = (float*)&x;
+      *y += 2.0f;          // Strict aliasing violation
+      return 0;
+    }
+
+    # Compile and link
+    % clang++ -g -fsanitize=type example_AliasViolation.cc
+
+The program will print an error message to ``stderr`` each time a strict aliasing violation is detected. 
+The program won't terminate, which will allow you to detect many strict aliasing violations in one 
+run.
+
+.. code-block:: console
+
+    % ./a.out
+    ==1375532==ERROR: TypeSanitizer: type-aliasing-violation on address 0x7ffeebf1a72c (pc 0x5b3b1145ff41 bp 0x7ffeebf1a660 sp 0x7ffeebf19e08 tid 1375532)
+    READ of size 4 at 0x7ffeebf1a72c with type float accesses an existing object of type int
+        #0 0x5b3b1145ff40 in main example_AliasViolation.c:4:10
+
+    ==1375532==ERROR: TypeSanitizer: type-aliasing-violation on address 0x7ffeebf1a72c (pc 0x5b3b1146008a bp 0x7ffeebf1a660 sp 0x7ffeebf19e08 tid 1375532)
+    WRITE of size 4 at 0x7ffeebf1a72c with type float accesses an existing object of type int
+        #0 0x5b3b11460089 in main example_AliasViolation.c:4:10
+
+Error terminology
+------------------
+
+There are some terms that may appear in TypeSanitizer errors that are derived from 
+`TBAA Metadata <https://llvm.org/docs/LangRef.html#tbaa-metadata>`. This section hopes to provide a 
+brief dictionary of these terms.
+
+* ``omnipotent char``: This is a special type which can alias with anything. Its name comes from the C/C++ 
+  type ``char``.
+* ``type p[x]``: This signifies pointers to the type. ``x`` is the number of indirections to reach the final value.
+  As an example, a pointer to a pointer to an integer would be ``type p2 int``.
+
+TypeSanitizer is still experimental. User-facing error messages should be improved in the future to remove 
+references to LLVM IR specific terms.
+
+Sanitizer features
+==================
+
+``__has_feature(type_sanitizer)``
+------------------------------------
+
+In some cases one may need to execute different code depending on whether
+TypeSanitizer is enabled.
+:ref:`\_\_has\_feature <langext-__has_feature-__has_extension>` can be used for
+this purpose.
+
+.. code-block:: c
+
+    #if defined(__has_feature)
+    #  if __has_feature(type_sanitizer)
+    // code that builds only under TypeSanitizer
+    #  endif
+    #endif
+
+``__attribute__((no_sanitize("type")))``
+-----------------------------------------------
+
+Some code you may not want to be instrumented by TypeSanitizer.  One may use the
+function attribute ``no_sanitize("type")`` to disable instrumenting type aliasing. 
+It is possible, depending on what happens in non-instrumented code, that instrumented code 
+emits false-positives/ false-negatives. This attribute may not be supported by other 
+compilers, so we suggest to use it together with ``__has_feature(type_sanitizer)``.
+
+``__attribute__((disable_sanitizer_instrumentation))``
+--------------------------------------------------------
+
+The ``disable_sanitizer_instrumentation`` attribute can be applied to functions
+to prevent all kinds of instrumentation. As a result, it may introduce false
+positives and incorrect stack traces. Therefore, it should be used with care,
+and only if absolutely required; for example for certain code that cannot
+tolerate any instrumentation and resulting side-effects. This attribute
+overrides ``no_sanitize("type")``.
+
+Ignorelist
+----------
+
+TypeSanitizer supports ``src`` and ``fun`` entity types in
+:doc:`SanitizerSpecialCaseList`, that can be used to suppress aliasing 
+violation reports in the specified source files or functions. Like 
+with other methods of ignoring instrumentation, this can result in false 
+positives/ false-negatives.
+
+Limitations
+-----------
+
+* TypeSanitizer uses more real memory than a native run. It uses 8 bytes of
+  shadow memory for each byte of user memory.
+* There are transformation passes which run before TypeSanitizer. If these 
+  passes optimize out an aliasing violation, TypeSanitizer cannot catch it.
+* Currently, all instrumentation is inlined. This can result in a **15x** 
+  (on average) increase in generated file size, and **3x** to **7x** increase 
+  in compile time. In some documented cases this can cause the compiler to hang.
+  There are plans to improve this in the future.
+* Codebases that use unions and struct-initialized variables can see incorrect 
+  results, as TypeSanitizer doesn't yet instrument these reliably.
+* Since Clang & LLVM's TBAA system is used to generate the checks used by the 
+  instrumentation, TypeSanitizer follows Clang & LLVM's rules for type aliasing. 
+  There may be situations where that disagrees with the standard. However this 
+  does at least mean that TypeSanitizer will catch any aliasing violations that  
+  would cause bugs when compiling with Clang & LLVM.
+* TypeSanitizer cannot currently be run alongside other sanitizers such as 
+  AddressSanitizer, ThreadSanitizer or UndefinedBehaviourSanitizer.
+
+Current Status
+--------------
+
+TypeSanitizer is brand new, and still in development. There are some known 
+issues, especially in areas where Clang's emitted TBAA data isn't extensive 
+enough for TypeSanitizer's runtime.
+
+We are actively working on enhancing the tool --- stay tuned.  Any help, 
+issues, pull requests, ideas, is more than welcome. You can find the 
+`issue tracker here. <https://github.com/llvm/llvm-project/issues?q=is%3Aissue%20state%3Aopen%20TySan%20label%3Acompiler-rt%3Atysan>`_
diff --git a/clang/docs/UsersManual.rst b/clang/docs/UsersManual.rst
index 260e84910c6f7..d977868b8a2c6 100644
--- a/clang/docs/UsersManual.rst
+++ b/clang/docs/UsersManual.rst
@@ -2103,7 +2103,10 @@ are listed below.
 
       ``-fsanitize=undefined``: :doc:`UndefinedBehaviorSanitizer`,
       a fast and compatible undefined behavior checker.
+   -  .. _opt_fsanitize_type:
 
+      ``-fsanitize=type``: :doc:`TypeSanitizer`, a detector for strict
+      aliasing violations.
    -  ``-fsanitize=dataflow``: :doc:`DataFlowSanitizer`, a general data
       flow analysis.
    -  ``-fsanitize=cfi``: :doc:`control flow integrity <ControlFlowIntegrity>`
@@ -2220,6 +2223,12 @@ are listed below.
 
    This option is currently experimental.
 
+.. option:: -fsanitize-kcfi-arity
+
+   Extends kernel indirect call forward-edge control flow integrity with
+   additional function arity information (for supported targets). See
+   :doc:`ControlFlowIntegrity` for more details.
+
 .. option:: -fstrict-vtable-pointers
 
    Enable optimizations based on the strict rules for overwriting polymorphic
@@ -2486,6 +2495,82 @@ are listed below.
 
     $ clang -fuse-ld=lld -Oz -Wl,--icf=safe -fcodegen-data-use code.cc
 
+.. _strict_aliasing:
+
+Strict Aliasing
+---------------
+
+The C and C++ standards require accesses to objects in memory to use l-values of
+an appropriate type for the object. This is called *strict aliasing* or
+*type-based alias analysis*. Strict aliasing enhances a variety of powerful
+memory optimizations, including reordering, combining, and eliminating memory
+accesses. These optimizations can lead to unexpected behavior in code that
+violates the strict aliasing rules. For example:
+
+.. code-block:: c++
+
+    void advance(size_t *index, double *data) {
+      double value = data[*index];
+      /* Clang may assume that this store does not change the contents of `data`. */
+      *index += 1;
+      /* Clang may assume that this store does not change the contents of `index`. */
+      data[*index] = value;
+      /* Either of these facts may create significant optimization opportunities
+       if Clang is able to inline this function. */
+  }
+
+Strict aliasing can be explicitly enabled with ``-fstrict-aliasing`` and
+disabled with ``-fno-strict-aliasing``. ``clang-cl`` defaults to
+``-fno-strict-aliasing``; see . Otherwise, Clang defaults to ``-fstrict-aliasing``.
+
+C and C++ specify slightly different rules for strict aliasing. To improve
+language interoperability, Clang allows two types to alias if either language
+would permit it. This includes applying the C++ similar types rule to C,
+allowing ``int **`` to alias ``int const * const *``. Clang also relaxes the
+standard aliasing rules in the following ways:
+
+* All integer types of the same size are permitted to alias each other,
+  including signed and unsigned types.
+* ``void*`` is permitted to alias any pointer type, ``void**`` is permitted to
+  alias any pointer to pointer type, and so on.
+
+Code which violates strict aliasing has undefined behavior. A program that
+works in one version of Clang may not work in another because of changes to the
+optimizer. Clang provides a :doc:`TypeSanitizer` to help detect
+violations of the strict aliasing rules, but it is currently still experimental.
+Code that is known to violate strict aliasing should generally be built with
+``-fno-strict-aliasing`` if the violation cannot be fixed.
+
+Clang supports several ways to fix a violation of strict aliasing:
+
+* L-values of the character types ``char`` and ``unsigned char`` (as well as
+  other types, depending on the standard) are permitted to access objects of
+  any type.
+
+* Library functions such as ``memcpy`` and ``memset`` are specified as treating
+  memory as characters and therefore are not limited by strict aliasing. If a
+  value of one type must be reinterpreted as another (e.g. to read the bits of a
+  floating-point number), use ``memcpy`` to copy the representation to an object
+  of the destination type. This has no overhead over a direct l-value access
+  because Clang should reliably optimize calls to these functions to use simple
+  loads and stores when they are used with small constant sizes.
+
+* The attribute ``may_alias`` can be added to a ``typedef`` to give l-values of
+  that type the same aliasing power as the character types.
+
+Clang makes a best effort to avoid obvious miscompilations from strict aliasing
+by only considering type information when it cannot prove that two accesses must
+refer to the same memory. However, it is not recommended that programmers
+intentionally rely on this instead of using one of the solutions above because
+it is too easy for the compiler's analysis to be blocked in surprising ways.
+
+In Clang 20, Clang strengthened its implementation of strict aliasing for
+accesses of pointer type. Previously, all accesses of pointer type were
+permitted to alias each other, but Clang now distinguishes different pointers
+by their pointee type, except as limited by the relaxations around qualifiers
+and ``void*`` described above. The previous behavior of treating all pointers as
+aliasing can be restored using ``-fno-pointer-tbaa``.
+
 Profile Guided Optimization
 ---------------------------
 
@@ -2880,7 +2965,8 @@ instrumentation:
    environment variable to specify an alternate file. If non-default file name
    is specified by both the environment variable and the command line option,
    the environment variable takes precedence. The file name pattern specified
-   can include different modifiers: ``%p``, ``%h``, ``%m``, ``%t``, and ``%c``.
+   can include different modifiers: ``%p``, ``%h``, ``%m``, ``%b``, ``%t``, and
+   ``%c``.
 
    Any instance of ``%p`` in that file name will be replaced by the process
    ID, so that you can easily distinguish the profile output from multiple
@@ -2902,11 +2988,11 @@ instrumentation:
    ``%p`` is that the storage requirement for raw profile data files is greatly
    increased.  To avoid issues like this, the ``%m`` specifier can used in the profile
    name.  When this specifier is used, the profiler runtime will substitute ``%m``
-   with a unique integer identifier associated with the instrumented binary. Additionally,
+   with an integer identifier associated with the instrumented binary. Additionally,
    multiple raw profiles dumped from different processes that share a file system (can be
    on different hosts) will be automatically merged by the profiler runtime during the
    dumping. If the program links in multiple instrumented shared libraries, each library
-   will dump the profile data into its own profile data file (with its unique integer
+   will dump the profile data into its own profile data file (with its integer
    id embedded in the profile name). Note that the merging enabled by ``%m`` is for raw
    profile data generated by profiler runtime. The resulting merged "raw" profile data
    file still needs to be converted to a different format expected by the compiler (
@@ -2916,6 +3002,12 @@ instrumentation:
 
      $ LLVM_PROFILE_FILE="code-%m.profraw" ./code
 
+   Although rare, binary signatures used by the ``%m`` specifier can have
+   collisions. In this case, the ``%b`` specifier, which expands to the binary
+   ID (build ID in ELF and COFF), can be added. To use it, the program should be
+   compiled with the build ID linker option (``--build-id`` for GNU ld or LLD,
+   ``/build-id`` for lld-link on Windows). Linux, Windows and AIX are supported.
+
    See `this <SourceBasedCodeCoverage.html#running-the-instrumented-program>`_ section
    about the ``%t``, and ``%c`` modifiers.
 
@@ -3033,6 +3125,24 @@ indexed format, regardeless whether it is produced by frontend or the IR pass.
   overhead. ``prefer-atomic`` will be transformed to ``atomic`` when supported
   by the target, or ``single`` otherwise.
 
+.. option:: -fprofile-continuous
+
+  Enables the continuous instrumentation profiling where profile counter updates
+  are continuously synced to a file. This option sets any neccessary modifiers
+  (currently ``%c``) in the default profile filename and passes any necessary
+  flags to the middle-end to support this mode. Value profiling is not supported
+  in continuous mode.
+
+  .. code-block:: console
+
+    $ clang++ -O2 -fprofile-generate -fprofile-continuous code.cc -o code
+
+  Running ``./code`` will collect the profile and write it to the
+  ``default_xxxx.profraw`` file. However, if ``./code`` abruptly terminates or
+  does not call ``exit()``, in continuous mode the profile collected up to the
+  point of termination will be available in ``default_xxxx.profraw`` while in
+  the non-continuous mode, no profile file is generated.
+
 .. option:: -ftemporal-profile
 
   Enables the temporal profiling extension for IRPGO to improve startup time by
@@ -5269,12 +5379,6 @@ The Visual C++ Toolset has a slightly more elaborate mechanism for detection.
 Restrictions and Limitations compared to Clang
 ----------------------------------------------
 
-Strict Aliasing
-^^^^^^^^^^^^^^^
-
-Strict aliasing (TBAA) is always off by default in clang-cl. Whereas in clang,
-strict aliasing is turned on by default for all optimization levels.
-
-To enable LLVM optimizations based on strict aliasing rules (e.g., optimizations
-based on type of expressions in C/C++), user will need to explicitly pass
-`-fstrict-aliasing` to clang-cl.
+Strict aliasing (TBAA) is always off by default in clang-cl whereas in clang,
+strict aliasing is turned on by default for all optimization levels. For more
+details, see :ref:`Strict aliasing <strict_aliasing>`.
diff --git a/clang/docs/analyzer/checkers.rst b/clang/docs/analyzer/checkers.rst
index e093b2d672a74..707067358fdfe 100644
--- a/clang/docs/analyzer/checkers.rst
+++ b/clang/docs/analyzer/checkers.rst
@@ -1332,10 +1332,69 @@ security
 
 Security related checkers.
 
+.. _security-ArrayBound:
+
+security.ArrayBound (C, C++)
+""""""""""""""""""""""""""""
+Report out of bounds access to memory that is before the start or after the end
+of the accessed region (array, heap-allocated region, string literal etc.).
+This usually means incorrect indexing, but the checker also detects access via
+the operators ``*`` and ``->``.
+
+.. code-block:: c
+
+ void test_underflow(int x) {
+   int buf[100][100];
+   if (x < 0)
+     buf[0][x] = 1; // warn
+ }
+
+ void test_overflow() {
+   int buf[100];
+   int *p = buf + 100;
+   *p = 1; // warn
+ }
+
+If checkers like :ref:`unix-Malloc` or :ref:`cplusplus-NewDelete` are enabled
+to model the behavior of ``malloc()``, ``operator new`` and similar
+allocators), then this checker can also reports out of bounds access to
+dynamically allocated memory:
+
+.. code-block:: cpp
+
+ int *test_dynamic() {
+   int *mem = new int[100];
+   mem[-1] = 42; // warn
+   return mem;
+ }
+
+In uncertain situations (when the checker can neither prove nor disprove that
+overflow occurs), the checker assumes that the the index (more precisely, the
+memory offeset) is within bounds.
+
+However, if :ref:`optin-taint-GenericTaint` is enabled and the index/offset is
+tainted (i.e. it is influenced by an untrusted souce), then this checker
+reports the potential out of bounds access:
+
+.. code-block:: c
+
+ void test_with_tainted_index() {
+   char s[] = "abc";
+   int x = getchar();
+   char c = s[x]; // warn: potential out of bounds access with tainted index
+ }
+
+.. note::
+
+  This checker is an improved and renamed version of the checker that was
+  previously known as ``alpha.security.ArrayBoundV2``. The old checker
+  ``alpha.security.ArrayBound`` was removed when the (previously
+  "experimental") V2 variant became stable enough for regular use.
+
 .. _security-cert-env-InvalidPtr:
 
 security.cert.env.InvalidPtr
-""""""""""""""""""""""""""""""""""
+""""""""""""""""""""""""""""
 
 Corresponds to SEI CERT Rules `ENV31-C <https://wiki.sei.cmu.edu/confluence/display/c/ENV31-C.+Do+not+rely+on+an+environment+pointer+following+an+operation+that+may+invalidate+it>`_ and `ENV34-C <https://wiki.sei.cmu.edu/confluence/display/c/ENV34-C.+Do+not+store+pointers+returned+by+certain+functions>`_.
 
@@ -3216,78 +3275,6 @@ Warns against using one vs. many plural pattern in code when generating localize
 alpha.security
 ^^^^^^^^^^^^^^
 
-.. _alpha-security-ArrayBound:
-
-alpha.security.ArrayBound (C)
-"""""""""""""""""""""""""""""
-Warn about buffer overflows (older checker).
-
-.. code-block:: c
-
- void test() {
-   char *s = "";
-   char c = s[1]; // warn
- }
-
- struct seven_words {
-   int c[7];
- };
-
- void test() {
-   struct seven_words a, *p;
-   p = &a;
-   p[0] = a;
-   p[1] = a;
-   p[2] = a; // warn
- }
-
- // note: requires unix.Malloc or
- // alpha.unix.MallocWithAnnotations checks enabled.
- void test() {
-   int *p = malloc(12);
-   p[3] = 4; // warn
- }
-
- void test() {
-   char a[2];
-   int *b = (int*)a;
-   b[1] = 3; // warn
- }
-
-.. _alpha-security-ArrayBoundV2:
-
-alpha.security.ArrayBoundV2 (C)
-"""""""""""""""""""""""""""""""
-Warn about buffer overflows (newer checker).
-
-.. code-block:: c
-
- void test() {
-   char *s = "";
-   char c = s[1]; // warn
- }
-
- void test() {
-   int buf[100];
-   int *p = buf;
-   p = p + 99;
-   p[1] = 1; // warn
- }
-
- // note: compiler has internal check for this.
- // Use -Wno-array-bounds to suppress compiler warning.
- void test() {
-   int buf[100][100];
-   buf[0][-1] = 1; // warn
- }
-
- // note: requires optin.taint check turned on.
- void test() {
-   char s[] = "abc";
-   int x = getchar();
-   char c = s[x]; // warn: index is tainted
- }
-
 .. _alpha-security-ReturnPtrRange:
 
 alpha.security.ReturnPtrRange (C)
diff --git a/clang/docs/analyzer/developer-docs.rst b/clang/docs/analyzer/developer-docs.rst
index 1b8133ffa8920..60c0e71ad847c 100644
--- a/clang/docs/analyzer/developer-docs.rst
+++ b/clang/docs/analyzer/developer-docs.rst
@@ -11,3 +11,4 @@ Contents:
    developer-docs/InitializerLists
    developer-docs/nullability
    developer-docs/RegionStore
+   developer-docs/PerformanceInvestigation
diff --git a/clang/docs/analyzer/developer-docs/PerformanceInvestigation.rst b/clang/docs/analyzer/developer-docs/PerformanceInvestigation.rst
new file mode 100644
index 0000000000000..ca3a56828209b
--- /dev/null
+++ b/clang/docs/analyzer/developer-docs/PerformanceInvestigation.rst
@@ -0,0 +1,137 @@
+=========================
+Performance Investigation
+=========================
+
+Multiple factors contribute to the time it takes to analyze a file with Clang Static Analyzer.
+A translation unit contains multiple entry points, each of which take multiple steps to analyze.
+
+Performance analysis using ``-ftime-trace``
+===========================================
+
+You can add the ``-ftime-trace=file.json`` option to break down the analysis time into individual entry points and steps within each entry point.
+You can explore the generated JSON file in a Chromium browser using the ``chrome://tracing`` URL,
+or using `speedscope <https://speedscope.app>`_.
+Once you narrow down to specific analysis steps you are interested in, you can more effectively employ heavier profilers,
+such as `Perf <https://perfwiki.github.io/main/>`_ and `Callgrind <https://valgrind.org/docs/manual/cl-manual.html>`_.
+
+Each analysis step has a time scope in the trace, corresponds to processing of an exploded node, and is designated with a ``ProgramPoint``.
+If the ``ProgramPoint`` is associated with a location, you can see it on the scope metadata label.
+
+Here is an example of a time trace produced with
+
+.. code-block:: bash
+   :caption: Clang Static Analyzer invocation to generate a time trace of string.c analysis.
+
+   clang -cc1 -analyze -verify clang/test/Analysis/string.c \
+         -analyzer-checker=core,unix,alpha.unix.cstring,debug.ExprInspection \
+         -ftime-trace=trace.json -ftime-trace-granularity=1
+
+.. image:: ../images/speedscope.png
+
+On the speedscope screenshot above, under the first time ruler is the bird's-eye view of the entire trace that spans a little over 60 milliseconds.
+Under the second ruler (focused on the 18.09-18.13ms time point) you can see a narrowed-down portion.
+The second box ("HandleCode memset...") that spans entire screen (and actually extends beyond it) corresponds to the analysis of ``memset16_region_cast()`` entry point that is defined in the "string.c" test file on line 1627.
+Below it, you can find multiple sub-scopes each corresponding to processing of a single exploded node.
+
+- First: a ``PostStmt`` for some statement on line 1634. This scope has a selected subscope "CheckerManager::runCheckersForCallEvent (Pre)" that takes 5 microseconds.
+- Four other nodes, too small to be discernible at this zoom level
+- Last on this screenshot: another ``PostStmt`` for a statement on line 1635.
+
+In addition to the ``-ftime-trace`` option, you can use ``-ftime-trace-granularity`` to fine-tune the time trace.
+
+- ``-ftime-trace-granularity=NN`` dumps only time scopes that are longer than NN microseconds.
+- ``-ftime-trace-verbose`` enables some additional dumps in the frontend related to template instantiations.
+  At the moment, it has no effect on the traces from the static analyzer.
+
+Note: Both Chrome-tracing and speedscope tools might struggle with time traces above 100 MB in size.
+Luckily, in most cases the default max-steps boundary of 225 000 produces the traces of approximately that size
+for a single entry point.
+You can use ``-analyze-function=get_global_options`` together with ``-ftime-trace`` to narrow down analysis to a specific entry point.
+
+
+Performance analysis using ``perf``
+===================================
+
+`Perf <https://perfwiki.github.io/main/>`_ is a tool for conducting sampling-based profiling.
+It's easy to start profiling, you only have 2 prerequisites.
+Build with ``-fno-omit-frame-pointer`` and debug info (``-g``).
+You can use release builds, but probably the easiest is to set the ``CMAKE_BUILD_TYPE=RelWithDebInfo``
+along with ``CMAKE_CXX_FLAGS="-fno-omit-frame-pointer"`` when configuring ``llvm``.
+Here is how to `get started <https://llvm.org/docs/CMake.html#quick-start>`_ if you are in trouble.
+
+.. code-block:: bash
+   :caption: Running the Clang Static Analyzer through ``perf`` to gather samples of the execution.
+
+   # -F: Sampling frequency, use `-F max` for maximal frequency
+   # -g: Enable call-graph recording for both kernel and user space
+   perf record -F 99 -g --  clang -cc1 -analyze -verify clang/test/Analysis/string.c \
+         -analyzer-checker=core,unix,alpha.unix.cstring,debug.ExprInspection
+
+Once you have the profile data, you can use it to produce a Flame graph.
+A Flame graph is a visual representation of the stack frames of the samples.
+Common stack frame prefixes are squashed together, making up a wider bar.
+The wider the bar, the more time was spent under that particular stack frame,
+giving a sense of how the overall execution time was spent.
+
+Clone the `FlameGraph <https://github.com/brendangregg/FlameGraph>`_ git repository,
+as we will use some scripts from there to convert the ``perf`` samples into a Flame graph.
+It's also useful to check out Brendan Gregg's (the author of FlameGraph)
+`homepage <https://www.brendangregg.com/FlameGraphs/cpuflamegraphs.html>`_.
+
+
+.. code-block:: bash
+   :caption: Converting the ``perf`` profile into a Flamegraph, then opening it in Firefox.
+
+   perf script | /path/to/FlameGraph/stackcollapse-perf.pl > perf.folded
+   /path/to/FlameGraph/flamegraph.pl perf.folded  > perf.svg
+   firefox perf.svg
+
+.. image:: ../images/flamegraph.png
+
+
+Performance analysis using ``uftrace``
+======================================
+
+`uftrace <https://github.com/namhyung/uftrace/wiki/Tutorial#getting-started>`_ is a great tool to generate rich profile data
+that you can use to focus and drill down into the timeline of your application.
+We will use it to generate Chromium trace JSON.
+In contrast to ``perf``, this approach statically instruments every function, so it should be more precise and thorough than the sampling-based approaches like ``perf``.
+In contrast to using ``-ftime-trace``, functions don't need to opt-in to be profiled using ``llvm::TimeTraceScope``.
+All functions are profiled due to automatic static instrumentation.
+
+There is only one prerequisite to use this tool.
+You need to build the binary you are about to instrument using ``-pg`` or ``-finstrument-functions``.
+This will make it run substantially slower but allows rich instrumentation.
+It will also consume many gigabites of storage for a single trace unless filter flags are used during recording.
+
+.. code-block:: bash
+   :caption: Recording with ``uftrace``, then dumping the result as a Chrome trace JSON.
+
+   uftrace record  clang -cc1 -analyze -verify clang/test/Analysis/string.c \
+         -analyzer-checker=core,unix,alpha.unix.cstring,debug.ExprInspection
+   uftrace dump --filter=".*::AnalysisConsumer::HandleTranslationUnit" --time-filter=300 --chrome > trace.json
+
+.. image:: ../images/uftrace_detailed.png
+
+In this picture, you can see the functions below the Static Analyzer's entry point, which takes at least 300 nanoseconds to run, visualized by Chrome's ``about:tracing`` page
+You can also see how deep function calls we may have due to AST visitors.
+
+Using different filters can reduce the number of functions to record.
+For the common options, refer to the ``uftrace`` `documentation <https://github.com/namhyung/uftrace/blob/master/doc/uftrace-record.md#common-options>`_.
+
+Similar filters can be applied for dumping too. That way you can reuse the same (detailed)
+recording to selectively focus on some special part using a refinement of the filter flags.
+Remember, the trace JSON needs to fit into Chrome's ``about:tracing`` or `speedscope <https://speedscope.app>`_,
+thus it needs to be of a limited size.
+If you do not apply filters on recording, you will collect a large trace and every dump operation
+would need to sieve through the much larger recording which may be annoying if done repeatedly.
+
+If the trace JSON is still too large to load, have a look at the dump as plain text and look for frequent entries that refer to non-interesting parts.
+Once you have some of those, add them as ``--hide`` flags to the ``uftrace dump`` call.
+To see what functions appear frequently in the trace, use this command:
+
+.. code-block:: bash
+
+   cat trace.json | grep -Po '"name":"(.+)"' | sort | uniq -c | sort -nr | head -n 50
+
+``uftrace`` can also dump the report as a Flame graph using ``uftrace dump --framegraph``.
diff --git a/clang/docs/analyzer/images/flamegraph.png b/clang/docs/analyzer/images/flamegraph.png
new file mode 100644
index 0000000000000..b16ec90b9e600
Binary files /dev/null and b/clang/docs/analyzer/images/flamegraph.png differ
diff --git a/clang/docs/analyzer/images/speedscope.png b/clang/docs/analyzer/images/speedscope.png
new file mode 100644
index 0000000000000..767725d68f740
Binary files /dev/null and b/clang/docs/analyzer/images/speedscope.png differ
diff --git a/clang/docs/analyzer/images/uftrace_detailed.png b/clang/docs/analyzer/images/uftrace_detailed.png
new file mode 100644
index 0000000000000..fcf681909d070
Binary files /dev/null and b/clang/docs/analyzer/images/uftrace_detailed.png differ
diff --git a/clang/docs/analyzer/user-docs/Annotations.rst b/clang/docs/analyzer/user-docs/Annotations.rst
index d87e8f4df99c3..11f15939ecfaf 100644
--- a/clang/docs/analyzer/user-docs/Annotations.rst
+++ b/clang/docs/analyzer/user-docs/Annotations.rst
@@ -23,8 +23,8 @@ recognized by GCC. Their use can be conditioned using preprocessor macros
 .. contents::
    :local:
 
-Annotations to Enhance Generic Checks
-_____________________________________
+General Purpose Annotations
+___________________________
 
 Null Pointer Checking
 #####################
@@ -79,7 +79,7 @@ implemented with a macro, with the macro performing a check for the assertion
 condition and, when the check fails, calling an assertion handler.  For
 example, consider the following code fragment:
 
-.. code-block: c
+.. code-block:: c
 
   void foo(int *p) {
     assert(p != NULL);
@@ -87,7 +87,7 @@ example, consider the following code fragment:
 
 When this code is preprocessed on Mac OS X it expands to the following:
 
-.. code-block: c
+.. code-block:: c
 
   void foo(int *p) {
     (__builtin_expect(!(p != NULL), 0) ? __assert_rtn(__func__, "t.c", 4, "p != NULL") : (void)0);
@@ -131,7 +131,7 @@ return.
 On Mac OS X, the function prototype for ``__assert_rtn`` (declared in
 ``assert.h``) is specifically annotated with the 'noreturn' attribute:
 
-.. code-block: c
+.. code-block:: c
 
   void __assert_rtn(const char *, const char *, int, const char *) __attribute__((__noreturn__));
 
@@ -151,7 +151,7 @@ the use of preprocessor macros.
 
 **Example**
 
-.. code-block: c
+.. code-block:: c
 
   #ifndef CLANG_ANALYZER_NORETURN
   #if __has_feature(attribute_analyzer_noreturn)
@@ -163,6 +163,43 @@ the use of preprocessor macros.
 
   void my_assert_rtn(const char *, const char *, int, const char *) CLANG_ANALYZER_NORETURN;
 
+Dynamic Memory Modeling Annotations
+###################################
+
+If a project uses custom functions for dynamic memory management (that e.g. act as wrappers around ``malloc``/``free`` or ``new``/``delete`` in C++) and the analyzer cannot "see" the _definitions_ of these functions, it's possible to annotate their declarations to let the analyzer model their behavior. (Otherwise the analyzer cannot know that the opaque ``my_free()`` is basically equivalent to a standard ``free()`` call.)
+
+.. note::
+  **This page only provides a brief list of these annotations.** For a full documentation, see the main `Attributes in Clang <../../AttributeReference.html#ownership-holds-ownership-returns-ownership-takes-clang-static-analyzer>`_ page.
+
+Attribute 'ownership_returns' (Clang-specific)
+----------------------------------------------
+
+Use this attribute to mark functions that return dynamically allocated memory. Takes a single argument, the type of the allocation (e.g. ``malloc`` or ``new``).
+
+.. code-block:: c
+
+  void __attribute((ownership_returns(malloc))) *my_malloc(size_t);
+
+Attribute 'ownership_takes' (Clang-specific)
+--------------------------------------------
+
+Use this attribute to mark functions that deallocate memory. Takes two arguments: the type of the allocation (e.g. ``malloc`` or ``new``) and the index of the parameter that is being deallocated (counting from 1).
+
+.. code-block:: c
+
+  void __attribute((ownership_takes(malloc, 1))) my_free(void *);
+
+Attribute 'ownership_holds' (Clang-specific)
+--------------------------------------------
+
+Use this attribute to mark functions that take ownership of memory and will deallocate it at some unspecified point in the future. Takes two arguments: the type of the allocation (e.g. ``malloc`` or ``new``) and the index of the parameter that is being held (counting from 1).
+
+.. code-block:: c
+
+  void __attribute((ownership_holds(malloc, 2))) store_in_table(int key, record_t *val);
+
+The annotations ``ownership_takes`` and ``ownership_holds`` both prevent memory leak reports (concerning the specified argument); the difference between them is that using taken memory is a use-after-free error, while using held memory is assumed to be legitimate.
+
 Mac OS X API Annotations
 ________________________
 
@@ -207,7 +244,7 @@ functions allows the analyzer to perform extra checking.
 
 **Example**
 
-.. code-block: objc
+.. code-block:: objc
 
   #import <Foundation/Foundation.h>;
 
@@ -597,7 +634,6 @@ returned object.
     LIBKERN_RETURNS_NOT_RETAINED OSObject *myFieldGetter();
   }
 
-
   // Note that the annotation only has to be applied to the function declaration.
   OSObject * MyClass::myFieldGetter() {
     return f;
diff --git a/clang/docs/index.rst b/clang/docs/index.rst
index 349378b1efa21..6c792af66a62c 100644
--- a/clang/docs/index.rst
+++ b/clang/docs/index.rst
@@ -35,6 +35,7 @@ Using Clang as a Compiler
    UndefinedBehaviorSanitizer
    DataFlowSanitizer
    LeakSanitizer
+   TypeSanitizer
    RealtimeSanitizer
    SanitizerCoverage
    SanitizerStats
diff --git a/clang/include/clang-c/Index.h b/clang/include/clang-c/Index.h
index aac5d1fa8aa2e..61e361faabdaf 100644
--- a/clang/include/clang-c/Index.h
+++ b/clang/include/clang-c/Index.h
@@ -2206,7 +2206,11 @@ enum CXCursorKind {
    */
   CXCursor_OpenACCUpdateConstruct = 331,
 
-  CXCursor_LastStmt = CXCursor_OpenACCUpdateConstruct,
+  /** OpenACC atomic Construct.
+   */
+  CXCursor_OpenACCAtomicConstruct = 332,
+
+  CXCursor_LastStmt = CXCursor_OpenACCAtomicConstruct,
 
   /**
    * Cursor that represents the translation unit itself.
@@ -5905,66 +5909,6 @@ CINDEX_LINKAGE const char *clang_EvalResult_getAsStr(CXEvalResult E);
  * Disposes the created Eval memory.
  */
 CINDEX_LINKAGE void clang_EvalResult_dispose(CXEvalResult E);
-/**
- * @}
- */
-
-/** \defgroup CINDEX_REMAPPING Remapping functions
- *
- * @{
- */
-
-/**
- * A remapping of original source files and their translated files.
- */
-typedef void *CXRemapping;
-
-/**
- * Retrieve a remapping.
- *
- * \param path the path that contains metadata about remappings.
- *
- * \returns the requested remapping. This remapping must be freed
- * via a call to \c clang_remap_dispose(). Can return NULL if an error occurred.
- */
-CINDEX_LINKAGE CXRemapping clang_getRemappings(const char *path);
-
-/**
- * Retrieve a remapping.
- *
- * \param filePaths pointer to an array of file paths containing remapping info.
- *
- * \param numFiles number of file paths.
- *
- * \returns the requested remapping. This remapping must be freed
- * via a call to \c clang_remap_dispose(). Can return NULL if an error occurred.
- */
-CINDEX_LINKAGE
-CXRemapping clang_getRemappingsFromFileList(const char **filePaths,
-                                            unsigned numFiles);
-
-/**
- * Determine the number of remappings.
- */
-CINDEX_LINKAGE unsigned clang_remap_getNumFiles(CXRemapping);
-
-/**
- * Get the original and the associated filename from the remapping.
- *
- * \param original If non-NULL, will be set to the original filename.
- *
- * \param transformed If non-NULL, will be set to the filename that the original
- * is associated with.
- */
-CINDEX_LINKAGE void clang_remap_getFilenames(CXRemapping, unsigned index,
-                                             CXString *original,
-                                             CXString *transformed);
-
-/**
- * Dispose the remapping.
- */
-CINDEX_LINKAGE void clang_remap_dispose(CXRemapping);
-
 /**
  * @}
  */
diff --git a/clang/include/clang/ARCMigrate/ARCMT.h b/clang/include/clang/ARCMigrate/ARCMT.h
deleted file mode 100644
index 2b950e3d2cc2b..0000000000000
--- a/clang/include/clang/ARCMigrate/ARCMT.h
+++ /dev/null
@@ -1,130 +0,0 @@
-//===-- ARCMT.h - ARC Migration Rewriter ------------------------*- C++ -*-===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LLVM_CLANG_ARCMIGRATE_ARCMT_H
-#define LLVM_CLANG_ARCMIGRATE_ARCMT_H
-
-#include "clang/ARCMigrate/FileRemapper.h"
-#include "clang/Basic/SourceLocation.h"
-#include "clang/Frontend/CompilerInvocation.h"
-
-namespace clang {
-  class ASTContext;
-  class DiagnosticConsumer;
-  class PCHContainerOperations;
-
-namespace arcmt {
-  class MigrationPass;
-
-/// Creates an AST with the provided CompilerInvocation but with these
-/// changes:
-///   -if a PCH/PTH is set, the original header is used instead
-///   -Automatic Reference Counting mode is enabled
-///
-/// It then checks the AST and produces errors/warning for ARC migration issues
-/// that the user needs to handle manually.
-///
-/// \param emitPremigrationARCErrors if true all ARC errors will get emitted
-/// even if the migrator can fix them, but the function will still return false
-/// if all ARC errors can be fixed.
-///
-/// \param plistOut if non-empty, it is the file path to store the plist with
-/// the pre-migration ARC diagnostics.
-///
-/// \returns false if no error is produced, true otherwise.
-bool
-checkForManualIssues(CompilerInvocation &CI, const FrontendInputFile &Input,
-                     std::shared_ptr<PCHContainerOperations> PCHContainerOps,
-                     DiagnosticConsumer *DiagClient,
-                     bool emitPremigrationARCErrors = false,
-                     StringRef plistOut = StringRef());
-
-/// Works similar to checkForManualIssues but instead of checking, it
-/// applies automatic modifications to source files to conform to ARC.
-///
-/// \returns false if no error is produced, true otherwise.
-bool
-applyTransformations(CompilerInvocation &origCI,
-                     const FrontendInputFile &Input,
-                     std::shared_ptr<PCHContainerOperations> PCHContainerOps,
-                     DiagnosticConsumer *DiagClient);
-
-/// Applies automatic modifications and produces temporary files
-/// and metadata into the \p outputDir path.
-///
-/// \param emitPremigrationARCErrors if true all ARC errors will get emitted
-/// even if the migrator can fix them, but the function will still return false
-/// if all ARC errors can be fixed.
-///
-/// \param plistOut if non-empty, it is the file path to store the plist with
-/// the pre-migration ARC diagnostics.
-///
-/// \returns false if no error is produced, true otherwise.
-bool migrateWithTemporaryFiles(
-    CompilerInvocation &origCI, const FrontendInputFile &Input,
-    std::shared_ptr<PCHContainerOperations> PCHContainerOps,
-    DiagnosticConsumer *DiagClient, StringRef outputDir,
-    bool emitPremigrationARCErrors, StringRef plistOut);
-
-/// Get the set of file remappings from the \p outputDir path that
-/// migrateWithTemporaryFiles produced.
-///
-/// \returns false if no error is produced, true otherwise.
-bool getFileRemappings(std::vector<std::pair<std::string,std::string> > &remap,
-                       StringRef outputDir,
-                       DiagnosticConsumer *DiagClient);
-
-/// Get the set of file remappings from a list of files with remapping
-/// info.
-///
-/// \returns false if no error is produced, true otherwise.
-bool getFileRemappingsFromFileList(
-                        std::vector<std::pair<std::string,std::string> > &remap,
-                        ArrayRef<StringRef> remapFiles,
-                        DiagnosticConsumer *DiagClient);
-
-typedef void (*TransformFn)(MigrationPass &pass);
-
-std::vector<TransformFn> getAllTransformations(LangOptions::GCMode OrigGCMode,
-                                               bool NoFinalizeRemoval);
-
-class MigrationProcess {
-  CompilerInvocation OrigCI;
-  std::shared_ptr<PCHContainerOperations> PCHContainerOps;
-  DiagnosticConsumer *DiagClient;
-  FileRemapper Remapper;
-
-public:
-  bool HadARCErrors;
-
-  MigrationProcess(CompilerInvocation &CI,
-                   std::shared_ptr<PCHContainerOperations> PCHContainerOps,
-                   DiagnosticConsumer *diagClient,
-                   StringRef outputDir = StringRef());
-
-  class RewriteListener {
-  public:
-    virtual ~RewriteListener();
-
-    virtual void start(ASTContext &Ctx) { }
-    virtual void finish() { }
-
-    virtual void insert(SourceLocation loc, StringRef text) { }
-    virtual void remove(CharSourceRange range) { }
-  };
-
-  bool applyTransform(TransformFn trans, RewriteListener *listener = nullptr);
-
-  FileRemapper &getRemapper() { return Remapper; }
-};
-
-} // end namespace arcmt
-
-}  // end namespace clang
-
-#endif
diff --git a/clang/include/clang/ARCMigrate/ARCMTActions.h b/clang/include/clang/ARCMigrate/ARCMTActions.h
deleted file mode 100644
index 714f4b33db446..0000000000000
--- a/clang/include/clang/ARCMigrate/ARCMTActions.h
+++ /dev/null
@@ -1,76 +0,0 @@
-//===--- ARCMTActions.h - ARC Migrate Tool Frontend Actions -----*- C++ -*-===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LLVM_CLANG_ARCMIGRATE_ARCMTACTIONS_H
-#define LLVM_CLANG_ARCMIGRATE_ARCMTACTIONS_H
-
-#include "clang/ARCMigrate/FileRemapper.h"
-#include "clang/Frontend/FrontendAction.h"
-#include <memory>
-
-namespace clang {
-namespace arcmt {
-
-class CheckAction : public WrapperFrontendAction {
-protected:
-  bool BeginInvocation(CompilerInstance &CI) override;
-
-public:
-  CheckAction(std::unique_ptr<FrontendAction> WrappedAction);
-};
-
-class ModifyAction : public WrapperFrontendAction {
-protected:
-  bool BeginInvocation(CompilerInstance &CI) override;
-
-public:
-  ModifyAction(std::unique_ptr<FrontendAction> WrappedAction);
-};
-
-class MigrateSourceAction : public ASTFrontendAction {
-  FileRemapper Remapper;
-protected:
-  bool BeginInvocation(CompilerInstance &CI) override;
-  std::unique_ptr<ASTConsumer> CreateASTConsumer(CompilerInstance &CI,
-                                                 StringRef InFile) override;
-};
-
-class MigrateAction : public WrapperFrontendAction {
-  std::string MigrateDir;
-  std::string PlistOut;
-  bool EmitPremigrationARCErrors;
-protected:
-  bool BeginInvocation(CompilerInstance &CI) override;
-
-public:
-  MigrateAction(std::unique_ptr<FrontendAction> WrappedAction,
-                StringRef migrateDir,
-                StringRef plistOut,
-                bool emitPremigrationARCErrors);
-};
-
-/// Migrates to modern ObjC syntax.
-class ObjCMigrateAction : public WrapperFrontendAction {
-  std::string MigrateDir;
-  unsigned    ObjCMigAction;
-  FileRemapper Remapper;
-  CompilerInstance *CompInst;
-public:
-  ObjCMigrateAction(std::unique_ptr<FrontendAction> WrappedAction,
-                    StringRef migrateDir, unsigned migrateAction);
-
-protected:
-  std::unique_ptr<ASTConsumer> CreateASTConsumer(CompilerInstance &CI,
-                                                 StringRef InFile) override;
-  bool BeginInvocation(CompilerInstance &CI) override;
-};
-
-}
-}
-
-#endif
diff --git a/clang/include/clang/ARCMigrate/FileRemapper.h b/clang/include/clang/ARCMigrate/FileRemapper.h
deleted file mode 100644
index afcee363516a2..0000000000000
--- a/clang/include/clang/ARCMigrate/FileRemapper.h
+++ /dev/null
@@ -1,84 +0,0 @@
-//===-- FileRemapper.h - File Remapping Helper ------------------*- C++ -*-===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LLVM_CLANG_ARCMIGRATE_FILEREMAPPER_H
-#define LLVM_CLANG_ARCMIGRATE_FILEREMAPPER_H
-
-#include "clang/Basic/FileEntry.h"
-#include "clang/Basic/LLVM.h"
-#include "llvm/ADT/DenseMap.h"
-#include "llvm/ADT/STLExtras.h"
-#include "llvm/ADT/StringRef.h"
-#include <memory>
-#include <variant>
-
-namespace llvm {
-  class MemoryBuffer;
-  class MemoryBufferRef;
-}
-
-namespace clang {
-  class FileManager;
-  class DiagnosticsEngine;
-  class PreprocessorOptions;
-
-namespace arcmt {
-
-class FileRemapper {
-  // FIXME: Reuse the same FileManager for multiple ASTContexts.
-  std::unique_ptr<FileManager> FileMgr;
-
-  using Target = std::variant<FileEntryRef, llvm::MemoryBuffer *>;
-  using MappingsTy = llvm::DenseMap<FileEntryRef, Target>;
-  MappingsTy FromToMappings;
-
-  llvm::DenseMap<const FileEntry *, FileEntryRef> ToFromMappings;
-
-public:
-  FileRemapper();
-  ~FileRemapper();
-
-  bool initFromDisk(StringRef outputDir, DiagnosticsEngine &Diag,
-                    bool ignoreIfFilesChanged);
-  bool initFromFile(StringRef filePath, DiagnosticsEngine &Diag,
-                    bool ignoreIfFilesChanged);
-  bool flushToDisk(StringRef outputDir, DiagnosticsEngine &Diag);
-  bool flushToFile(StringRef outputPath, DiagnosticsEngine &Diag);
-
-  bool overwriteOriginal(DiagnosticsEngine &Diag,
-                         StringRef outputDir = StringRef());
-
-  void remap(StringRef filePath, std::unique_ptr<llvm::MemoryBuffer> memBuf);
-
-  void applyMappings(PreprocessorOptions &PPOpts) const;
-
-  /// Iterate through all the mappings.
-  void forEachMapping(
-      llvm::function_ref<void(StringRef, StringRef)> CaptureFile,
-      llvm::function_ref<void(StringRef, const llvm::MemoryBufferRef &)>
-          CaptureBuffer) const;
-
-  void clear(StringRef outputDir = StringRef());
-
-private:
-  void remap(FileEntryRef file, std::unique_ptr<llvm::MemoryBuffer> memBuf);
-  void remap(FileEntryRef file, FileEntryRef newfile);
-
-  OptionalFileEntryRef getOriginalFile(StringRef filePath);
-  void resetTarget(Target &targ);
-
-  bool report(const Twine &err, DiagnosticsEngine &Diag);
-
-  std::string getRemapInfoFile(StringRef outputDir);
-};
-
-} // end namespace arcmt
-
-}  // end namespace clang
-
-#endif
diff --git a/clang/include/clang/AST/APValue.h b/clang/include/clang/AST/APValue.h
index 833a78c77871d..9999a30c51ade 100644
--- a/clang/include/clang/AST/APValue.h
+++ b/clang/include/clang/AST/APValue.h
@@ -161,8 +161,9 @@ class APValue {
 
     template <class T> T get() const { return cast<T>(Ptr); }
 
-    template <class T>
-    T dyn_cast() const { return Ptr.dyn_cast<T>(); }
+    template <class T> T dyn_cast() const {
+      return dyn_cast_if_present<T>(Ptr);
+    }
 
     void *getOpaqueValue() const;
 
diff --git a/clang/include/clang/AST/ASTContext.h b/clang/include/clang/AST/ASTContext.h
index 4e9b961688d55..a96b9c0a17045 100644
--- a/clang/include/clang/AST/ASTContext.h
+++ b/clang/include/clang/AST/ASTContext.h
@@ -769,7 +769,7 @@ class ASTContext : public RefCountedBase<ASTContext> {
   /// pool.
   DeclListNode *AllocateDeclListNode(clang::NamedDecl *ND) {
     if (DeclListNode *Alloc = ListNodeFreeList) {
-      ListNodeFreeList = Alloc->Rest.dyn_cast<DeclListNode*>();
+      ListNodeFreeList = dyn_cast_if_present<DeclListNode *>(Alloc->Rest);
       Alloc->D = ND;
       Alloc->Rest = nullptr;
       return Alloc;
@@ -1733,6 +1733,47 @@ class ASTContext : public RefCountedBase<ASTContext> {
                             unsigned NumPositiveBits, QualType &BestType,
                             QualType &BestPromotionType);
 
+  /// Determine whether the given integral value is representable within
+  /// the given type T.
+  bool isRepresentableIntegerValue(llvm::APSInt &Value, QualType T);
+
+  /// Compute NumNegativeBits and NumPositiveBits for an enum based on
+  /// the constant values of its enumerators.
+  template <typename RangeT>
+  bool computeEnumBits(RangeT EnumConstants, unsigned &NumNegativeBits,
+                       unsigned &NumPositiveBits) {
+    NumNegativeBits = 0;
+    NumPositiveBits = 0;
+    bool MembersRepresentableByInt = true;
+    for (auto *Elem : EnumConstants) {
+      EnumConstantDecl *ECD = cast_or_null<EnumConstantDecl>(Elem);
+      if (!ECD)
+        continue; // Already issued a diagnostic.
+
+      llvm::APSInt InitVal = ECD->getInitVal();
+      if (InitVal.isUnsigned() || InitVal.isNonNegative()) {
+        // If the enumerator is zero that should still be counted as a positive
+        // bit since we need a bit to store the value zero.
+        unsigned ActiveBits = InitVal.getActiveBits();
+        NumPositiveBits = std::max({NumPositiveBits, ActiveBits, 1u});
+      } else {
+        NumNegativeBits =
+            std::max(NumNegativeBits, (unsigned)InitVal.getSignificantBits());
+      }
+
+      MembersRepresentableByInt &= isRepresentableIntegerValue(InitVal, IntTy);
+    }
+
+    // If we have an empty set of enumerators we still need one bit.
+    // From [dcl.enum]p8
+    // If the enumerator-list is empty, the values of the enumeration are as if
+    // the enumeration had a single enumerator with value 0
+    if (!NumPositiveBits && !NumNegativeBits)
+      NumPositiveBits = 1;
+
+    return MembersRepresentableByInt;
+  }
+
   QualType
   getUnresolvedUsingType(const UnresolvedUsingTypenameDecl *Decl) const;
 
diff --git a/clang/include/clang/AST/ASTLambda.h b/clang/include/clang/AST/ASTLambda.h
index 646cb574847fe..a1854b6a1a949 100644
--- a/clang/include/clang/AST/ASTLambda.h
+++ b/clang/include/clang/AST/ASTLambda.h
@@ -17,6 +17,7 @@
 
 #include "clang/AST/DeclCXX.h"
 #include "clang/AST/DeclTemplate.h"
+#include "llvm/Support/Casting.h"
 
 namespace clang {
 inline StringRef getLambdaStaticInvokerName() {
@@ -35,6 +36,12 @@ inline bool isLambdaCallOperator(const DeclContext *DC) {
   return isLambdaCallOperator(cast<CXXMethodDecl>(DC));
 }
 
+inline bool isLambdaMethod(const DeclContext *DC) {
+  if (const auto *MD = dyn_cast_if_present<CXXMethodDecl>(DC))
+    return MD->getParent()->isLambda();
+  return false;
+}
+
 inline bool isLambdaCallWithExplicitObjectParameter(const DeclContext *DC) {
   return isLambdaCallOperator(DC) &&
          cast<CXXMethodDecl>(DC)->isExplicitObjectMemberFunction();
diff --git a/clang/include/clang/AST/Decl.h b/clang/include/clang/AST/Decl.h
index d01681483a918..f305cbbce4c60 100644
--- a/clang/include/clang/AST/Decl.h
+++ b/clang/include/clang/AST/Decl.h
@@ -698,6 +698,10 @@ class ValueDecl : public NamedDecl {
     return const_cast<ValueDecl *>(this)->getPotentiallyDecomposedVarDecl();
   }
 
+  /// Determine whether this value is actually a function parameter pack,
+  /// init-capture pack, or structured binding pack
+  bool isParameterPack() const;
+
   // Implement isa/cast/dyncast/etc.
   static bool classof(const Decl *D) { return classofKind(D->getKind()); }
   static bool classofKind(Kind K) { return K >= firstValue && K <= lastValue; }
@@ -1527,10 +1531,6 @@ class VarDecl : public DeclaratorDecl, public Redeclarable<VarDecl> {
     NonParmVarDeclBits.IsInitCapture = IC;
   }
 
-  /// Determine whether this variable is actually a function parameter pack or
-  /// init-capture pack.
-  bool isParameterPack() const;
-
   /// Whether this local extern variable declaration's previous declaration
   /// was declared in the same block scope. Only correct in C++.
   bool isPreviousDeclInSameBlockScope() const {
@@ -2298,6 +2298,13 @@ class FunctionDecl : public DeclaratorDecl,
     FunctionDeclBits.IsLateTemplateParsed = ILT;
   }
 
+  bool isInstantiatedFromMemberTemplate() const {
+    return FunctionDeclBits.IsInstantiatedFromMemberTemplate;
+  }
+  void setInstantiatedFromMemberTemplate(bool Val = true) {
+    FunctionDeclBits.IsInstantiatedFromMemberTemplate = Val;
+  }
+
   /// Whether this function is "trivial" in some specialized C++ senses.
   /// Can only be true for default constructors, copy constructors,
   /// copy assignment operators, and destructors.  Not meaningful until
@@ -4035,7 +4042,7 @@ class EnumDecl : public TagDecl {
   /// Return the type source info for the underlying integer type,
   /// if no type source info exists, return 0.
   TypeSourceInfo *getIntegerTypeSourceInfo() const {
-    return IntegerType.dyn_cast<TypeSourceInfo*>();
+    return dyn_cast_if_present<TypeSourceInfo *>(IntegerType);
   }
 
   /// Retrieve the source range that covers the underlying type if
@@ -5139,6 +5146,12 @@ static constexpr StringRef getOpenMPVariantManglingSeparatorStr() {
 bool IsArmStreamingFunction(const FunctionDecl *FD,
                             bool IncludeLocallyStreaming);
 
+/// Returns whether the given FunctionDecl has Arm ZA state.
+bool hasArmZAState(const FunctionDecl *FD);
+
+/// Returns whether the given FunctionDecl has Arm ZT0 state.
+bool hasArmZT0State(const FunctionDecl *FD);
+
 } // namespace clang
 
 #endif // LLVM_CLANG_AST_DECL_H
diff --git a/clang/include/clang/AST/DeclBase.h b/clang/include/clang/AST/DeclBase.h
index 573b46a2321c5..648dae2838e03 100644
--- a/clang/include/clang/AST/DeclBase.h
+++ b/clang/include/clang/AST/DeclBase.h
@@ -1257,8 +1257,11 @@ class alignas(8) Decl {
   int64_t getID() const;
 
   /// Looks through the Decl's underlying type to extract a FunctionType
-  /// when possible. Will return null if the type underlying the Decl does not
-  /// have a FunctionType.
+  /// when possible. This includes direct FunctionDecls, along with various
+  /// function types and typedefs. This includes function pointers/references,
+  /// member function pointers, and optionally if \p BlocksToo is set
+  /// Objective-C block pointers. Returns nullptr if the type underlying the
+  /// Decl does not have a FunctionType.
   const FunctionType *getFunctionType(bool BlocksToo = true) const;
 
   // Looks through the Decl's underlying type to determine if it's a
@@ -1391,7 +1394,7 @@ class DeclContextLookupResult {
   const_iterator end() const { return iterator(); }
 
   bool empty() const { return Result.isNull();  }
-  bool isSingleResult() const { return Result.dyn_cast<NamedDecl*>(); }
+  bool isSingleResult() const { return isa_and_present<NamedDecl *>(Result); }
   reference front() const { return *begin(); }
 
   // Find the first declaration of the given type in the list. Note that this
@@ -1777,6 +1780,8 @@ class DeclContext {
     uint64_t HasImplicitReturnZero : 1;
     LLVM_PREFERRED_TYPE(bool)
     uint64_t IsLateTemplateParsed : 1;
+    LLVM_PREFERRED_TYPE(bool)
+    uint64_t IsInstantiatedFromMemberTemplate : 1;
 
     /// Kind of contexpr specifier as defined by ConstexprSpecKind.
     LLVM_PREFERRED_TYPE(ConstexprSpecKind)
@@ -1827,7 +1832,7 @@ class DeclContext {
   };
 
   /// Number of inherited and non-inherited bits in FunctionDeclBitfields.
-  enum { NumFunctionDeclBits = NumDeclContextBits + 31 };
+  enum { NumFunctionDeclBits = NumDeclContextBits + 32 };
 
   /// Stores the bits used by CXXConstructorDecl. If modified
   /// NumCXXConstructorDeclBits and the accessor
@@ -1838,12 +1843,12 @@ class DeclContext {
     LLVM_PREFERRED_TYPE(FunctionDeclBitfields)
     uint64_t : NumFunctionDeclBits;
 
-    /// 20 bits to fit in the remaining available space.
+    /// 19 bits to fit in the remaining available space.
     /// Note that this makes CXXConstructorDeclBitfields take
     /// exactly 64 bits and thus the width of NumCtorInitializers
     /// will need to be shrunk if some bit is added to NumDeclContextBitfields,
     /// NumFunctionDeclBitfields or CXXConstructorDeclBitfields.
-    uint64_t NumCtorInitializers : 17;
+    uint64_t NumCtorInitializers : 16;
     LLVM_PREFERRED_TYPE(bool)
     uint64_t IsInheritingConstructor : 1;
 
@@ -1857,7 +1862,7 @@ class DeclContext {
   };
 
   /// Number of inherited and non-inherited bits in CXXConstructorDeclBitfields.
-  enum { NumCXXConstructorDeclBits = NumFunctionDeclBits + 20 };
+  enum { NumCXXConstructorDeclBits = NumFunctionDeclBits + 19 };
 
   /// Stores the bits used by ObjCMethodDecl.
   /// If modified NumObjCMethodDeclBits and the accessor
diff --git a/clang/include/clang/AST/DeclCXX.h b/clang/include/clang/AST/DeclCXX.h
index fa3f4ec98eb36..766821b4fb25c 100644
--- a/clang/include/clang/AST/DeclCXX.h
+++ b/clang/include/clang/AST/DeclCXX.h
@@ -1967,17 +1967,29 @@ class ExplicitSpecifier {
 class CXXDeductionGuideDecl : public FunctionDecl {
   void anchor() override;
 
+public:
+  // Represents the relationship between this deduction guide and the
+  // deduction guide that it was generated from (or lack thereof).
+  // See the SourceDeductionGuide member for more details.
+  enum class SourceDeductionGuideKind : uint8_t {
+    None,
+    Alias,
+  };
+
 private:
   CXXDeductionGuideDecl(ASTContext &C, DeclContext *DC, SourceLocation StartLoc,
                         ExplicitSpecifier ES,
                         const DeclarationNameInfo &NameInfo, QualType T,
                         TypeSourceInfo *TInfo, SourceLocation EndLocation,
                         CXXConstructorDecl *Ctor, DeductionCandidate Kind,
-                        Expr *TrailingRequiresClause)
+                        Expr *TrailingRequiresClause,
+                        const CXXDeductionGuideDecl *GeneratedFrom,
+                        SourceDeductionGuideKind SourceKind)
       : FunctionDecl(CXXDeductionGuide, C, DC, StartLoc, NameInfo, T, TInfo,
                      SC_None, false, false, ConstexprSpecKind::Unspecified,
                      TrailingRequiresClause),
-        Ctor(Ctor), ExplicitSpec(ES) {
+        Ctor(Ctor), ExplicitSpec(ES),
+        SourceDeductionGuide(GeneratedFrom, SourceKind) {
     if (EndLocation.isValid())
       setRangeEnd(EndLocation);
     setDeductionCandidateKind(Kind);
@@ -1985,6 +1997,12 @@ class CXXDeductionGuideDecl : public FunctionDecl {
 
   CXXConstructorDecl *Ctor;
   ExplicitSpecifier ExplicitSpec;
+  // The deduction guide, if any, that this deduction guide was generated from,
+  // in the case of alias template deduction. The SourceDeductionGuideKind
+  // member indicates which of these sources applies, or is None otherwise.
+  llvm::PointerIntPair<const CXXDeductionGuideDecl *, 2,
+                       SourceDeductionGuideKind>
+      SourceDeductionGuide;
   void setExplicitSpecifier(ExplicitSpecifier ES) { ExplicitSpec = ES; }
 
 public:
@@ -1997,7 +2015,9 @@ class CXXDeductionGuideDecl : public FunctionDecl {
          TypeSourceInfo *TInfo, SourceLocation EndLocation,
          CXXConstructorDecl *Ctor = nullptr,
          DeductionCandidate Kind = DeductionCandidate::Normal,
-         Expr *TrailingRequiresClause = nullptr);
+         Expr *TrailingRequiresClause = nullptr,
+         const CXXDeductionGuideDecl *SourceDG = nullptr,
+         SourceDeductionGuideKind SK = SourceDeductionGuideKind::None);
 
   static CXXDeductionGuideDecl *CreateDeserialized(ASTContext &C,
                                                    GlobalDeclID ID);
@@ -2017,6 +2037,25 @@ class CXXDeductionGuideDecl : public FunctionDecl {
   /// this is an implicit deduction guide.
   CXXConstructorDecl *getCorrespondingConstructor() const { return Ctor; }
 
+  /// Get the deduction guide from which this deduction guide was generated,
+  /// if it was generated as part of alias template deduction or from an
+  /// inherited constructor.
+  const CXXDeductionGuideDecl *getSourceDeductionGuide() const {
+    return SourceDeductionGuide.getPointer();
+  }
+
+  void setSourceDeductionGuide(CXXDeductionGuideDecl *DG) {
+    SourceDeductionGuide.setPointer(DG);
+  }
+
+  SourceDeductionGuideKind getSourceDeductionGuideKind() const {
+    return SourceDeductionGuide.getInt();
+  }
+
+  void setSourceDeductionGuideKind(SourceDeductionGuideKind SK) {
+    SourceDeductionGuide.setInt(SK);
+  }
+
   void setDeductionCandidateKind(DeductionCandidate K) {
     FunctionDeclBits.DeductionCandidateKind = static_cast<unsigned char>(K);
   }
@@ -4136,8 +4175,9 @@ class BindingDecl : public ValueDecl {
   /// binding).
   Expr *Binding = nullptr;
 
-  BindingDecl(DeclContext *DC, SourceLocation IdLoc, IdentifierInfo *Id)
-      : ValueDecl(Decl::Binding, DC, IdLoc, Id, QualType()) {}
+  BindingDecl(DeclContext *DC, SourceLocation IdLoc, IdentifierInfo *Id,
+              QualType T)
+      : ValueDecl(Decl::Binding, DC, IdLoc, Id, T) {}
 
   void anchor() override;
 
@@ -4145,7 +4185,8 @@ class BindingDecl : public ValueDecl {
   friend class ASTDeclReader;
 
   static BindingDecl *Create(ASTContext &C, DeclContext *DC,
-                             SourceLocation IdLoc, IdentifierInfo *Id);
+                             SourceLocation IdLoc, IdentifierInfo *Id,
+                             QualType T);
   static BindingDecl *CreateDeserialized(ASTContext &C, GlobalDeclID ID);
 
   /// Get the expression to which this declaration is bound. This may be null
@@ -4153,14 +4194,13 @@ class BindingDecl : public ValueDecl {
   /// decomposition declaration, and when the initializer is type-dependent.
   Expr *getBinding() const { return Binding; }
 
+  // Get the array of Exprs when the binding represents a pack.
+  llvm::ArrayRef<Expr *> getBindingPackExprs() const;
+
   /// Get the decomposition declaration that this binding represents a
   /// decomposition of.
   ValueDecl *getDecomposedDecl() const { return Decomp; }
 
-  /// Get the variable (if any) that holds the value of evaluating the binding.
-  /// Only present for user-defined bindings for tuple-like types.
-  VarDecl *getHoldingVar() const;
-
   /// Set the binding for this BindingDecl, along with its declared type (which
   /// should be a possibly-cv-qualified form of the type of the binding, or a
   /// reference to such a type).
@@ -4172,6 +4212,10 @@ class BindingDecl : public ValueDecl {
   /// Set the decomposed variable for this BindingDecl.
   void setDecomposedDecl(ValueDecl *Decomposed) { Decomp = Decomposed; }
 
+  /// Get the variable (if any) that holds the value of evaluating the binding.
+  /// Only present for user-defined bindings for tuple-like types.
+  VarDecl *getHoldingVar() const;
+
   static bool classof(const Decl *D) { return classofKind(D->getKind()); }
   static bool classofKind(Kind K) { return K == Decl::Binding; }
 };
@@ -4199,8 +4243,16 @@ class DecompositionDecl final
         NumBindings(Bindings.size()) {
     std::uninitialized_copy(Bindings.begin(), Bindings.end(),
                             getTrailingObjects<BindingDecl *>());
-    for (auto *B : Bindings)
+    for (auto *B : Bindings) {
       B->setDecomposedDecl(this);
+      if (B->isParameterPack() && B->getBinding()) {
+        for (Expr *E : B->getBindingPackExprs()) {
+          auto *DRE = cast<DeclRefExpr>(E);
+          auto *NestedB = cast<BindingDecl>(DRE->getDecl());
+          NestedB->setDecomposedDecl(this);
+        }
+      }
+    }
   }
 
   void anchor() override;
@@ -4218,8 +4270,33 @@ class DecompositionDecl final
   static DecompositionDecl *CreateDeserialized(ASTContext &C, GlobalDeclID ID,
                                                unsigned NumBindings);
 
-  ArrayRef<BindingDecl *> bindings() const {
-    return llvm::ArrayRef(getTrailingObjects<BindingDecl *>(), NumBindings);
+  // Provide the range of bindings which may have a nested pack.
+  llvm::ArrayRef<BindingDecl *> bindings() const {
+    return {getTrailingObjects<BindingDecl *>(), NumBindings};
+  }
+
+  // Provide a flattened range to visit each binding.
+  auto flat_bindings() const {
+    llvm::ArrayRef<BindingDecl *> Bindings = bindings();
+    llvm::ArrayRef<Expr *> PackExprs;
+
+    // Split the bindings into subranges split by the pack.
+    auto S1 = Bindings.take_until(
+        [](BindingDecl *BD) { return BD->isParameterPack(); });
+
+    Bindings = Bindings.drop_front(S1.size());
+    if (!Bindings.empty()) {
+      PackExprs = Bindings.front()->getBindingPackExprs();
+      Bindings = Bindings.drop_front();
+    }
+
+    auto S2 = llvm::map_range(PackExprs, [](Expr *E) {
+      auto *DRE = cast<DeclRefExpr>(E);
+      return cast<BindingDecl>(DRE->getDecl());
+    });
+
+    return llvm::concat<BindingDecl *>(std::move(S1), std::move(S2),
+                                       std::move(Bindings));
   }
 
   void printName(raw_ostream &OS, const PrintingPolicy &Policy) const override;
diff --git a/clang/include/clang/AST/DeclTemplate.h b/clang/include/clang/AST/DeclTemplate.h
index 8c2da97c07a3b..b82f75dd63fa5 100644
--- a/clang/include/clang/AST/DeclTemplate.h
+++ b/clang/include/clang/AST/DeclTemplate.h
@@ -1011,6 +1011,26 @@ class FunctionTemplateDecl : public RedeclarableTemplateDecl {
     return getTemplatedDecl()->isThisDeclarationADefinition();
   }
 
+  bool isCompatibleWithDefinition() const {
+    return getTemplatedDecl()->isInstantiatedFromMemberTemplate() ||
+           isThisDeclarationADefinition();
+  }
+
+  // This bit closely tracks 'RedeclarableTemplateDecl::InstantiatedFromMember',
+  // except this is per declaration, while the redeclarable field is
+  // per chain. This indicates a template redeclaration which
+  // is compatible with the definition, in the non-trivial case
+  // where this is not already a definition.
+  // This is only really needed for instantiating the definition of friend
+  // function templates, which can have redeclarations in different template
+  // contexts.
+  // The bit is actually stored in the FunctionDecl for space efficiency
+  // reasons.
+  void setInstantiatedFromMemberTemplate(FunctionTemplateDecl *D) {
+    getTemplatedDecl()->setInstantiatedFromMemberTemplate();
+    RedeclarableTemplateDecl::setInstantiatedFromMemberTemplate(D);
+  }
+
   /// Return the specialization with the provided arguments if it exists,
   /// otherwise return the insertion point.
   FunctionDecl *findSpecialization(ArrayRef<TemplateArgument> Args,
@@ -1841,15 +1861,23 @@ class ClassTemplateSpecializationDecl : public CXXRecordDecl,
   LLVM_PREFERRED_TYPE(TemplateSpecializationKind)
   unsigned SpecializationKind : 3;
 
+  /// Indicate that we have matched a parameter pack with a non pack
+  /// argument, when the opposite match is also allowed.
+  /// This needs to be cached as deduction is performed during declaration,
+  /// and we need the information to be preserved so that it is consistent
+  /// during instantiation.
+  bool StrictPackMatch : 1;
+
 protected:
   ClassTemplateSpecializationDecl(ASTContext &Context, Kind DK, TagKind TK,
                                   DeclContext *DC, SourceLocation StartLoc,
                                   SourceLocation IdLoc,
                                   ClassTemplateDecl *SpecializedTemplate,
                                   ArrayRef<TemplateArgument> Args,
+                                  bool StrictPackMatch,
                                   ClassTemplateSpecializationDecl *PrevDecl);
 
-  explicit ClassTemplateSpecializationDecl(ASTContext &C, Kind DK);
+  ClassTemplateSpecializationDecl(ASTContext &C, Kind DK);
 
 public:
   friend class ASTDeclReader;
@@ -1859,7 +1887,7 @@ class ClassTemplateSpecializationDecl : public CXXRecordDecl,
   Create(ASTContext &Context, TagKind TK, DeclContext *DC,
          SourceLocation StartLoc, SourceLocation IdLoc,
          ClassTemplateDecl *SpecializedTemplate,
-         ArrayRef<TemplateArgument> Args,
+         ArrayRef<TemplateArgument> Args, bool StrictPackMatch,
          ClassTemplateSpecializationDecl *PrevDecl);
   static ClassTemplateSpecializationDecl *CreateDeserialized(ASTContext &C,
                                                              GlobalDeclID ID);
@@ -1930,6 +1958,10 @@ class ClassTemplateSpecializationDecl : public CXXRecordDecl,
     SpecializationKind = TSK;
   }
 
+  bool hasStrictPackMatch() const { return StrictPackMatch; }
+
+  void setStrictPackMatch(bool Val) { StrictPackMatch = Val; }
+
   /// Get the point of instantiation (if any), or null if none.
   SourceLocation getPointOfInstantiation() const {
     return PointOfInstantiation;
@@ -2009,7 +2041,8 @@ class ClassTemplateSpecializationDecl : public CXXRecordDecl,
   /// Retrieve the template argument list as written in the sources,
   /// if any.
   const ASTTemplateArgumentListInfo *getTemplateArgsAsWritten() const {
-    if (auto *Info = ExplicitInfo.dyn_cast<ExplicitInstantiationInfo *>())
+    if (auto *Info =
+            dyn_cast_if_present<ExplicitInstantiationInfo *>(ExplicitInfo))
       return Info->TemplateArgsAsWritten;
     return cast<const ASTTemplateArgumentListInfo *>(ExplicitInfo);
   }
@@ -2017,7 +2050,8 @@ class ClassTemplateSpecializationDecl : public CXXRecordDecl,
   /// Set the template argument list as written in the sources.
   void
   setTemplateArgsAsWritten(const ASTTemplateArgumentListInfo *ArgsWritten) {
-    if (auto *Info = ExplicitInfo.dyn_cast<ExplicitInstantiationInfo *>())
+    if (auto *Info =
+            dyn_cast_if_present<ExplicitInstantiationInfo *>(ExplicitInfo))
       Info->TemplateArgsAsWritten = ArgsWritten;
     else
       ExplicitInfo = ArgsWritten;
@@ -2031,7 +2065,8 @@ class ClassTemplateSpecializationDecl : public CXXRecordDecl,
 
   /// Gets the location of the extern keyword, if present.
   SourceLocation getExternKeywordLoc() const {
-    if (auto *Info = ExplicitInfo.dyn_cast<ExplicitInstantiationInfo *>())
+    if (auto *Info =
+            dyn_cast_if_present<ExplicitInstantiationInfo *>(ExplicitInfo))
       return Info->ExternKeywordLoc;
     return SourceLocation();
   }
@@ -2041,7 +2076,8 @@ class ClassTemplateSpecializationDecl : public CXXRecordDecl,
 
   /// Gets the location of the template keyword, if present.
   SourceLocation getTemplateKeywordLoc() const {
-    if (auto *Info = ExplicitInfo.dyn_cast<ExplicitInstantiationInfo *>())
+    if (auto *Info =
+            dyn_cast_if_present<ExplicitInstantiationInfo *>(ExplicitInfo))
       return Info->TemplateKeywordLoc;
     return SourceLocation();
   }
@@ -2778,7 +2814,8 @@ class VarTemplateSpecializationDecl : public VarDecl,
   /// Retrieve the template argument list as written in the sources,
   /// if any.
   const ASTTemplateArgumentListInfo *getTemplateArgsAsWritten() const {
-    if (auto *Info = ExplicitInfo.dyn_cast<ExplicitInstantiationInfo *>())
+    if (auto *Info =
+            dyn_cast_if_present<ExplicitInstantiationInfo *>(ExplicitInfo))
       return Info->TemplateArgsAsWritten;
     return cast<const ASTTemplateArgumentListInfo *>(ExplicitInfo);
   }
@@ -2786,7 +2823,8 @@ class VarTemplateSpecializationDecl : public VarDecl,
   /// Set the template argument list as written in the sources.
   void
   setTemplateArgsAsWritten(const ASTTemplateArgumentListInfo *ArgsWritten) {
-    if (auto *Info = ExplicitInfo.dyn_cast<ExplicitInstantiationInfo *>())
+    if (auto *Info =
+            dyn_cast_if_present<ExplicitInstantiationInfo *>(ExplicitInfo))
       Info->TemplateArgsAsWritten = ArgsWritten;
     else
       ExplicitInfo = ArgsWritten;
@@ -2800,7 +2838,8 @@ class VarTemplateSpecializationDecl : public VarDecl,
 
   /// Gets the location of the extern keyword, if present.
   SourceLocation getExternKeywordLoc() const {
-    if (auto *Info = ExplicitInfo.dyn_cast<ExplicitInstantiationInfo *>())
+    if (auto *Info =
+            dyn_cast_if_present<ExplicitInstantiationInfo *>(ExplicitInfo))
       return Info->ExternKeywordLoc;
     return SourceLocation();
   }
@@ -2810,7 +2849,8 @@ class VarTemplateSpecializationDecl : public VarDecl,
 
   /// Gets the location of the template keyword, if present.
   SourceLocation getTemplateKeywordLoc() const {
-    if (auto *Info = ExplicitInfo.dyn_cast<ExplicitInstantiationInfo *>())
+    if (auto *Info =
+            dyn_cast_if_present<ExplicitInstantiationInfo *>(ExplicitInfo))
       return Info->TemplateKeywordLoc;
     return SourceLocation();
   }
diff --git a/clang/include/clang/AST/DynamicRecursiveASTVisitor.h b/clang/include/clang/AST/DynamicRecursiveASTVisitor.h
index 4382d20990829..703cca22777ad 100644
--- a/clang/include/clang/AST/DynamicRecursiveASTVisitor.h
+++ b/clang/include/clang/AST/DynamicRecursiveASTVisitor.h
@@ -52,7 +52,11 @@ class ASTContext;
 /// WalkUpFromX or post-order traversal).
 ///
 /// \see RecursiveASTVisitor.
-class DynamicRecursiveASTVisitor {
+template <bool IsConst> class DynamicRecursiveASTVisitorBase {
+protected:
+  template <typename ASTNode>
+  using MaybeConst = std::conditional_t<IsConst, const ASTNode, ASTNode>;
+
 public:
   /// Whether this visitor should recurse into template instantiations.
   bool ShouldVisitTemplateInstantiations = false;
@@ -68,28 +72,29 @@ class DynamicRecursiveASTVisitor {
   bool ShouldVisitLambdaBody = true;
 
 protected:
-  DynamicRecursiveASTVisitor() = default;
-  DynamicRecursiveASTVisitor(DynamicRecursiveASTVisitor &&) = default;
-  DynamicRecursiveASTVisitor(const DynamicRecursiveASTVisitor &) = default;
-  DynamicRecursiveASTVisitor &
-  operator=(DynamicRecursiveASTVisitor &&) = default;
-  DynamicRecursiveASTVisitor &
-  operator=(const DynamicRecursiveASTVisitor &) = default;
+  DynamicRecursiveASTVisitorBase() = default;
+  DynamicRecursiveASTVisitorBase(DynamicRecursiveASTVisitorBase &&) = default;
+  DynamicRecursiveASTVisitorBase(const DynamicRecursiveASTVisitorBase &) =
+      default;
+  DynamicRecursiveASTVisitorBase &
+  operator=(DynamicRecursiveASTVisitorBase &&) = default;
+  DynamicRecursiveASTVisitorBase &
+  operator=(const DynamicRecursiveASTVisitorBase &) = default;
 
 public:
   virtual void anchor();
-  virtual ~DynamicRecursiveASTVisitor() = default;
+  virtual ~DynamicRecursiveASTVisitorBase() = default;
 
   /// Recursively visits an entire AST, starting from the TranslationUnitDecl.
   /// \returns false if visitation was terminated early.
-  virtual bool TraverseAST(ASTContext &AST);
+  virtual bool TraverseAST(MaybeConst<ASTContext> &AST);
 
   /// Recursively visit an attribute, by dispatching to
   /// Traverse*Attr() based on the argument's dynamic type.
   ///
   /// \returns false if the visitation was terminated early, true
   /// otherwise (including when the argument is a Null type location).
-  virtual bool TraverseAttr(Attr *At);
+  virtual bool TraverseAttr(MaybeConst<Attr> *At);
 
   /// Recursively visit a constructor initializer.  This
   /// automatically dispatches to another visitor for the initializer
@@ -97,7 +102,8 @@ class DynamicRecursiveASTVisitor {
   /// be overridden for clients that need access to the name.
   ///
   /// \returns false if the visitation was terminated early, true otherwise.
-  virtual bool TraverseConstructorInitializer(CXXCtorInitializer *Init);
+  virtual bool
+  TraverseConstructorInitializer(MaybeConst<CXXCtorInitializer> *Init);
 
   /// Recursively visit a base specifier. This can be overridden by a
   /// subclass.
@@ -110,7 +116,7 @@ class DynamicRecursiveASTVisitor {
   ///
   /// \returns false if the visitation was terminated early, true
   /// otherwise (including when the argument is NULL).
-  virtual bool TraverseDecl(Decl *D);
+  virtual bool TraverseDecl(MaybeConst<Decl> *D);
 
   /// Recursively visit a name with its location information.
   ///
@@ -121,13 +127,15 @@ class DynamicRecursiveASTVisitor {
   /// will be used to initialize the capture.
   ///
   /// \returns false if the visitation was terminated early, true otherwise.
-  virtual bool TraverseLambdaCapture(LambdaExpr *LE, const LambdaCapture *C,
-                                     Expr *Init);
+  virtual bool TraverseLambdaCapture(MaybeConst<LambdaExpr> *LE,
+                                     const LambdaCapture *C,
+                                     MaybeConst<Expr> *Init);
 
   /// Recursively visit a C++ nested-name-specifier.
   ///
   /// \returns false if the visitation was terminated early, true otherwise.
-  virtual bool TraverseNestedNameSpecifier(NestedNameSpecifier *NNS);
+  virtual bool
+  TraverseNestedNameSpecifier(MaybeConst<NestedNameSpecifier> *NNS);
 
   /// Recursively visit a C++ nested-name-specifier with location
   /// information.
@@ -140,7 +148,7 @@ class DynamicRecursiveASTVisitor {
   ///
   /// \returns false if the visitation was terminated early, true
   /// otherwise (including when the argument is nullptr).
-  virtual bool TraverseStmt(Stmt *S);
+  virtual bool TraverseStmt(MaybeConst<Stmt> *S);
 
   /// Recursively visit a template argument and dispatch to the
   /// appropriate method for the argument type.
@@ -190,41 +198,51 @@ class DynamicRecursiveASTVisitor {
 
   /// Traverse a concept (requirement).
   virtual bool TraverseTypeConstraint(const TypeConstraint *C);
-  virtual bool TraverseConceptRequirement(concepts::Requirement *R);
-  virtual bool TraverseConceptTypeRequirement(concepts::TypeRequirement *R);
-  virtual bool TraverseConceptExprRequirement(concepts::ExprRequirement *R);
-  virtual bool TraverseConceptNestedRequirement(concepts::NestedRequirement *R);
-  virtual bool TraverseConceptReference(ConceptReference *CR);
-  virtual bool VisitConceptReference(ConceptReference *CR) { return true; }
+  virtual bool TraverseConceptRequirement(MaybeConst<concepts::Requirement> *R);
+
+  virtual bool
+  TraverseConceptTypeRequirement(MaybeConst<concepts::TypeRequirement> *R);
+
+  virtual bool
+  TraverseConceptExprRequirement(MaybeConst<concepts::ExprRequirement> *R);
+
+  virtual bool
+  TraverseConceptNestedRequirement(MaybeConst<concepts::NestedRequirement> *R);
+
+  virtual bool TraverseConceptReference(MaybeConst<ConceptReference> *CR);
+  virtual bool VisitConceptReference(MaybeConst<ConceptReference> *CR) {
+    return true;
+  }
 
   /// Visit a node.
-  virtual bool VisitAttr(Attr *A) { return true; }
-  virtual bool VisitDecl(Decl *D) { return true; }
-  virtual bool VisitStmt(Stmt *S) { return true; }
-  virtual bool VisitType(Type *T) { return true; }
+  virtual bool VisitAttr(MaybeConst<Attr> *A) { return true; }
+  virtual bool VisitDecl(MaybeConst<Decl> *D) { return true; }
+  virtual bool VisitStmt(MaybeConst<Stmt> *S) { return true; }
+  virtual bool VisitType(MaybeConst<Type> *T) { return true; }
   virtual bool VisitTypeLoc(TypeLoc TL) { return true; }
 
   /// Walk up from a node.
-  bool WalkUpFromDecl(Decl *D) { return VisitDecl(D); }
-  bool WalkUpFromStmt(Stmt *S) { return VisitStmt(S); }
-  bool WalkUpFromType(Type *T) { return VisitType(T); }
+  bool WalkUpFromDecl(MaybeConst<Decl> *D) { return VisitDecl(D); }
+  bool WalkUpFromStmt(MaybeConst<Stmt> *S) { return VisitStmt(S); }
+  bool WalkUpFromType(MaybeConst<Type> *T) { return VisitType(T); }
   bool WalkUpFromTypeLoc(TypeLoc TL) { return VisitTypeLoc(TL); }
 
   /// Invoked before visiting a statement or expression via data recursion.
   ///
   /// \returns false to skip visiting the node, true otherwise.
-  virtual bool dataTraverseStmtPre(Stmt *S) { return true; }
+  virtual bool dataTraverseStmtPre(MaybeConst<Stmt> *S) { return true; }
 
   /// Invoked after visiting a statement or expression via data recursion.
   /// This is not invoked if the previously invoked \c dataTraverseStmtPre
   /// returned false.
   ///
   /// \returns false if the visitation was terminated early, true otherwise.
-  virtual bool dataTraverseStmtPost(Stmt *S) { return true; }
-  virtual bool dataTraverseNode(Stmt *S);
+  virtual bool dataTraverseStmtPost(MaybeConst<Stmt> *S) { return true; }
+  virtual bool dataTraverseNode(MaybeConst<Stmt> *S);
 
 #define DEF_TRAVERSE_TMPL_INST(kind)                                           \
-  virtual bool TraverseTemplateInstantiations(kind##TemplateDecl *D);
+  virtual bool TraverseTemplateInstantiations(                                 \
+      MaybeConst<kind##TemplateDecl> *D);
   DEF_TRAVERSE_TMPL_INST(Class)
   DEF_TRAVERSE_TMPL_INST(Var)
   DEF_TRAVERSE_TMPL_INST(Function)
@@ -232,32 +250,34 @@ class DynamicRecursiveASTVisitor {
 
   // Decls.
 #define ABSTRACT_DECL(DECL)
-#define DECL(CLASS, BASE) virtual bool Traverse##CLASS##Decl(CLASS##Decl *D);
+#define DECL(CLASS, BASE)                                                      \
+  bool WalkUpFrom##CLASS##Decl(MaybeConst<CLASS##Decl> *D);                    \
+  virtual bool Traverse##CLASS##Decl(MaybeConst<CLASS##Decl> *D);
 #include "clang/AST/DeclNodes.inc"
 
 #define DECL(CLASS, BASE)                                                      \
-  bool WalkUpFrom##CLASS##Decl(CLASS##Decl *D);                                \
-  virtual bool Visit##CLASS##Decl(CLASS##Decl *D) { return true; }
+  virtual bool Visit##CLASS##Decl(MaybeConst<CLASS##Decl> *D) { return true; }
 #include "clang/AST/DeclNodes.inc"
 
   // Stmts.
 #define ABSTRACT_STMT(STMT)
-#define STMT(CLASS, PARENT) virtual bool Traverse##CLASS(CLASS *S);
+#define STMT(CLASS, PARENT) virtual bool Traverse##CLASS(MaybeConst<CLASS> *S);
 #include "clang/AST/StmtNodes.inc"
 
 #define STMT(CLASS, PARENT)                                                    \
-  bool WalkUpFrom##CLASS(CLASS *S);                                            \
-  virtual bool Visit##CLASS(CLASS *S) { return true; }
+  bool WalkUpFrom##CLASS(MaybeConst<CLASS> *S);                                \
+  virtual bool Visit##CLASS(MaybeConst<CLASS> *S) { return true; }
 #include "clang/AST/StmtNodes.inc"
 
   // Types.
 #define ABSTRACT_TYPE(CLASS, BASE)
-#define TYPE(CLASS, BASE) virtual bool Traverse##CLASS##Type(CLASS##Type *T);
+#define TYPE(CLASS, BASE)                                                      \
+  bool WalkUpFrom##CLASS##Type(MaybeConst<CLASS##Type> *T);                    \
+  virtual bool Traverse##CLASS##Type(MaybeConst<CLASS##Type> *T);
 #include "clang/AST/TypeNodes.inc"
 
 #define TYPE(CLASS, BASE)                                                      \
-  bool WalkUpFrom##CLASS##Type(CLASS##Type *T);                                \
-  virtual bool Visit##CLASS##Type(CLASS##Type *T) { return true; }
+  virtual bool Visit##CLASS##Type(MaybeConst<CLASS##Type> *T) { return true; }
 #include "clang/AST/TypeNodes.inc"
 
   // TypeLocs.
@@ -271,6 +291,14 @@ class DynamicRecursiveASTVisitor {
   virtual bool Visit##CLASS##TypeLoc(CLASS##TypeLoc TL) { return true; }
 #include "clang/AST/TypeLocNodes.def"
 };
+
+extern template class DynamicRecursiveASTVisitorBase<false>;
+extern template class DynamicRecursiveASTVisitorBase<true>;
+
+using DynamicRecursiveASTVisitor =
+    DynamicRecursiveASTVisitorBase</*Const=*/false>;
+using ConstDynamicRecursiveASTVisitor =
+    DynamicRecursiveASTVisitorBase</*Const=*/true>;
 } // namespace clang
 
 #endif // LLVM_CLANG_AST_DYNAMIC_RECURSIVE_AST_VISITOR_H
diff --git a/clang/include/clang/AST/Expr.h b/clang/include/clang/AST/Expr.h
index 708c8656decbe..cd584d9621a22 100644
--- a/clang/include/clang/AST/Expr.h
+++ b/clang/include/clang/AST/Expr.h
@@ -5180,7 +5180,7 @@ class InitListExpr : public Expr {
   /// than there are initializers in the list, specifies an expression to be
   /// used for value initialization of the rest of the elements.
   Expr *getArrayFiller() {
-    return ArrayFillerOrUnionFieldInit.dyn_cast<Expr *>();
+    return dyn_cast_if_present<Expr *>(ArrayFillerOrUnionFieldInit);
   }
   const Expr *getArrayFiller() const {
     return const_cast<InitListExpr *>(this)->getArrayFiller();
@@ -5205,7 +5205,7 @@ class InitListExpr : public Expr {
   /// union. However, a designated initializer can specify the
   /// initialization of a different field within the union.
   FieldDecl *getInitializedFieldInUnion() {
-    return ArrayFillerOrUnionFieldInit.dyn_cast<FieldDecl *>();
+    return dyn_cast_if_present<FieldDecl *>(ArrayFillerOrUnionFieldInit);
   }
   const FieldDecl *getInitializedFieldInUnion() const {
     return const_cast<InitListExpr *>(this)->getInitializedFieldInUnion();
@@ -6678,7 +6678,6 @@ class PseudoObjectExpr final
 class AtomicExpr : public Expr {
 public:
   enum AtomicOp {
-#define BUILTIN(ID, TYPE, ATTRS)
 #define ATOMIC_BUILTIN(ID, TYPE, ATTRS) AO ## ID,
 #include "clang/Basic/Builtins.inc"
     // Avoid trailing comma
@@ -6742,7 +6741,6 @@ class AtomicExpr : public Expr {
   AtomicOp getOp() const { return Op; }
   StringRef getOpAsString() const {
     switch (Op) {
-#define BUILTIN(ID, TYPE, ATTRS)
 #define ATOMIC_BUILTIN(ID, TYPE, ATTRS)                                        \
   case AO##ID:                                                                 \
     return #ID;
diff --git a/clang/include/clang/AST/ExprCXX.h b/clang/include/clang/AST/ExprCXX.h
index 4cec89c979f77..98ba2bb41bb54 100644
--- a/clang/include/clang/AST/ExprCXX.h
+++ b/clang/include/clang/AST/ExprCXX.h
@@ -2847,8 +2847,8 @@ class TypeTraitExpr final
 ///
 /// Example:
 /// \code
-///   __array_rank(int[10][20]) == 2
-///   __array_extent(int, 1)    == 20
+///   __array_rank(int[10][20])      == 2
+///   __array_extent(int[10][20], 1) == 20
 /// \endcode
 class ArrayTypeTraitExpr : public Expr {
   /// The trait. An ArrayTypeTrait enum in MSVC compat unsigned.
@@ -4326,8 +4326,6 @@ class SizeOfPackExpr final
   /// Retrieve the parameter pack.
   NamedDecl *getPack() const { return Pack; }
 
-  void setPack(NamedDecl *NewPack) { Pack = NewPack; }
-
   /// Retrieve the length of the parameter pack.
   ///
   /// This routine may only be invoked when the expression is not
@@ -5026,11 +5024,11 @@ class CXXParenListInitExpr final
   void setArrayFiller(Expr *E) { ArrayFillerOrUnionFieldInit = E; }
 
   Expr *getArrayFiller() {
-    return ArrayFillerOrUnionFieldInit.dyn_cast<Expr *>();
+    return dyn_cast_if_present<Expr *>(ArrayFillerOrUnionFieldInit);
   }
 
   const Expr *getArrayFiller() const {
-    return ArrayFillerOrUnionFieldInit.dyn_cast<Expr *>();
+    return dyn_cast_if_present<Expr *>(ArrayFillerOrUnionFieldInit);
   }
 
   void setInitializedFieldInUnion(FieldDecl *FD) {
@@ -5038,11 +5036,11 @@ class CXXParenListInitExpr final
   }
 
   FieldDecl *getInitializedFieldInUnion() {
-    return ArrayFillerOrUnionFieldInit.dyn_cast<FieldDecl *>();
+    return dyn_cast_if_present<FieldDecl *>(ArrayFillerOrUnionFieldInit);
   }
 
   const FieldDecl *getInitializedFieldInUnion() const {
-    return ArrayFillerOrUnionFieldInit.dyn_cast<FieldDecl *>();
+    return dyn_cast_if_present<FieldDecl *>(ArrayFillerOrUnionFieldInit);
   }
 
   child_range children() {
@@ -5321,6 +5319,59 @@ class BuiltinBitCastExpr final
   }
 };
 
+// Represents an unexpanded pack where the list of expressions are
+// known. These are used when structured bindings introduce a pack.
+class ResolvedUnexpandedPackExpr final
+    : public Expr,
+      private llvm::TrailingObjects<ResolvedUnexpandedPackExpr, Expr *> {
+  friend class ASTStmtReader;
+  friend class ASTStmtWriter;
+  friend TrailingObjects;
+
+  SourceLocation BeginLoc;
+  unsigned NumExprs;
+
+  ResolvedUnexpandedPackExpr(SourceLocation BL, QualType QT, unsigned NumExprs);
+
+public:
+  static ResolvedUnexpandedPackExpr *CreateDeserialized(ASTContext &C,
+                                                        unsigned NumExprs);
+  static ResolvedUnexpandedPackExpr *
+  Create(ASTContext &C, SourceLocation BeginLoc, QualType T, unsigned NumExprs);
+  static ResolvedUnexpandedPackExpr *Create(ASTContext &C,
+                                            SourceLocation BeginLoc, QualType T,
+                                            llvm::ArrayRef<Expr *> Exprs);
+
+  unsigned getNumExprs() const { return NumExprs; }
+
+  llvm::MutableArrayRef<Expr *> getExprs() {
+    return {getTrailingObjects<Expr *>(), NumExprs};
+  }
+
+  llvm::ArrayRef<Expr *> getExprs() const {
+    return {getTrailingObjects<Expr *>(), NumExprs};
+  }
+
+  Expr *getExpansion(unsigned Idx) { return getExprs()[Idx]; }
+  Expr *getExpansion(unsigned Idx) const { return getExprs()[Idx]; }
+
+  // Iterators
+  child_range children() {
+    return child_range((Stmt **)getTrailingObjects<Expr *>(),
+                       (Stmt **)getTrailingObjects<Expr *>() + getNumExprs());
+  }
+
+  SourceLocation getBeginLoc() const LLVM_READONLY { return BeginLoc; }
+  SourceLocation getEndLoc() const LLVM_READONLY { return BeginLoc; }
+
+  // Returns the resolved pack of a decl or nullptr
+  static ResolvedUnexpandedPackExpr *getFromDecl(Decl *);
+
+  static bool classof(const Stmt *T) {
+    return T->getStmtClass() == ResolvedUnexpandedPackExprClass;
+  }
+};
+
 } // namespace clang
 
 #endif // LLVM_CLANG_AST_EXPRCXX_H
diff --git a/clang/include/clang/AST/OpenMPClause.h b/clang/include/clang/AST/OpenMPClause.h
index b9088eff3bb52..154ecfbaa4418 100644
--- a/clang/include/clang/AST/OpenMPClause.h
+++ b/clang/include/clang/AST/OpenMPClause.h
@@ -2423,6 +2423,28 @@ class OMPNoOpenMPRoutinesClause final
   OMPNoOpenMPRoutinesClause() : OMPNoChildClause() {}
 };
 
+/// This represents the 'no_openmp_constructs' clause in the
+//// '#pragma omp assume' directive.
+///
+/// \code
+/// #pragma omp assume no_openmp_constructs
+/// \endcode
+/// In this example directive '#pragma omp assume' has a 'no_openmp_constructs'
+/// clause.
+class OMPNoOpenMPConstructsClause final
+    : public OMPNoChildClause<llvm::omp::OMPC_no_openmp_constructs> {
+public:
+  /// Build 'no_openmp_constructs' clause.
+  ///
+  /// \param StartLoc Starting location of the clause.
+  /// \param EndLoc Ending location of the clause.
+  OMPNoOpenMPConstructsClause(SourceLocation StartLoc, SourceLocation EndLoc)
+      : OMPNoChildClause(StartLoc, EndLoc) {}
+
+  /// Build an empty clause.
+  OMPNoOpenMPConstructsClause() : OMPNoChildClause() {}
+};
+
 /// This represents the 'no_parallelism' clause in the '#pragma omp assume'
 /// directive.
 ///
@@ -9451,7 +9473,8 @@ struct TargetOMPContext final : public llvm::omp::OMPContext {
   TargetOMPContext(ASTContext &ASTCtx,
                    std::function<void(StringRef)> &&DiagUnknownTrait,
                    const FunctionDecl *CurrentFunctionDecl,
-                   ArrayRef<llvm::omp::TraitProperty> ConstructTraits);
+                   ArrayRef<llvm::omp::TraitProperty> ConstructTraits,
+                   int DeviceNum);
 
   virtual ~TargetOMPContext() = default;
 
diff --git a/clang/include/clang/AST/OperationKinds.def b/clang/include/clang/AST/OperationKinds.def
index 8788b8ff0ef0a..b3dc7c3d8dc77 100644
--- a/clang/include/clang/AST/OperationKinds.def
+++ b/clang/include/clang/AST/OperationKinds.def
@@ -367,6 +367,9 @@ CAST_OPERATION(HLSLVectorTruncation)
 // Non-decaying array RValue cast (HLSL only).
 CAST_OPERATION(HLSLArrayRValue)
 
+// Aggregate by Value cast (HLSL only).
+CAST_OPERATION(HLSLElementwiseCast)
+
 //===- Binary Operations  -------------------------------------------------===//
 // Operators listed in order of precedence.
 // Note that additions to this should also update the StmtVisitor class,
diff --git a/clang/include/clang/AST/RecursiveASTVisitor.h b/clang/include/clang/AST/RecursiveASTVisitor.h
index c4a1d03f1b3d1..06c762c080de0 100644
--- a/clang/include/clang/AST/RecursiveASTVisitor.h
+++ b/clang/include/clang/AST/RecursiveASTVisitor.h
@@ -2950,6 +2950,7 @@ DEF_TRAVERSE_STMT(FunctionParmPackExpr, {})
 DEF_TRAVERSE_STMT(CXXFoldExpr, {})
 DEF_TRAVERSE_STMT(AtomicExpr, {})
 DEF_TRAVERSE_STMT(CXXParenListInitExpr, {})
+DEF_TRAVERSE_STMT(ResolvedUnexpandedPackExpr, {})
 
 DEF_TRAVERSE_STMT(MaterializeTemporaryExpr, {
   if (S->getLifetimeExtendedTemporaryDecl()) {
@@ -3543,6 +3544,12 @@ bool RecursiveASTVisitor<Derived>::VisitOMPNoOpenMPRoutinesClause(
   return true;
 }
 
+template <typename Derived>
+bool RecursiveASTVisitor<Derived>::VisitOMPNoOpenMPConstructsClause(
+    OMPNoOpenMPConstructsClause *) {
+  return true;
+}
+
 template <typename Derived>
 bool RecursiveASTVisitor<Derived>::VisitOMPNoParallelismClause(
     OMPNoParallelismClause *) {
@@ -4098,6 +4105,8 @@ DEF_TRAVERSE_STMT(OpenACCSetConstruct,
                   { TRY_TO(VisitOpenACCClauseList(S->clauses())); })
 DEF_TRAVERSE_STMT(OpenACCUpdateConstruct,
                   { TRY_TO(VisitOpenACCClauseList(S->clauses())); })
+DEF_TRAVERSE_STMT(OpenACCAtomicConstruct,
+                  { TRY_TO(TraverseOpenACCAssociatedStmtConstruct(S)); })
 
 // Traverse HLSL: Out argument expression
 DEF_TRAVERSE_STMT(HLSLOutArgExpr, {})
diff --git a/clang/include/clang/AST/StmtOpenACC.h b/clang/include/clang/AST/StmtOpenACC.h
index ebbee152f918f..bd6c95d342ce2 100644
--- a/clang/include/clang/AST/StmtOpenACC.h
+++ b/clang/include/clang/AST/StmtOpenACC.h
@@ -751,5 +751,50 @@ class OpenACCUpdateConstruct final
   Create(const ASTContext &C, SourceLocation Start, SourceLocation DirectiveLoc,
          SourceLocation End, ArrayRef<const OpenACCClause *> Clauses);
 };
+
+// This class represents the 'atomic' construct, which has an associated
+// statement, but no clauses.
+class OpenACCAtomicConstruct final : public OpenACCAssociatedStmtConstruct {
+
+  friend class ASTStmtReader;
+  OpenACCAtomicKind AtomicKind = OpenACCAtomicKind::None;
+
+  OpenACCAtomicConstruct(EmptyShell)
+      : OpenACCAssociatedStmtConstruct(
+            OpenACCAtomicConstructClass, OpenACCDirectiveKind::Atomic,
+            SourceLocation{}, SourceLocation{}, SourceLocation{},
+            /*AssociatedStmt=*/nullptr) {}
+
+  OpenACCAtomicConstruct(SourceLocation Start, SourceLocation DirectiveLoc,
+                         OpenACCAtomicKind AtKind, SourceLocation End,
+                         Stmt *AssociatedStmt)
+      : OpenACCAssociatedStmtConstruct(OpenACCAtomicConstructClass,
+                                       OpenACCDirectiveKind::Atomic, Start,
+                                       DirectiveLoc, End, AssociatedStmt),
+        AtomicKind(AtKind) {}
+
+  void setAssociatedStmt(Stmt *S) {
+    OpenACCAssociatedStmtConstruct::setAssociatedStmt(S);
+  }
+
+public:
+  static bool classof(const Stmt *T) {
+    return T->getStmtClass() == OpenACCAtomicConstructClass;
+  }
+
+  static OpenACCAtomicConstruct *CreateEmpty(const ASTContext &C);
+  static OpenACCAtomicConstruct *
+  Create(const ASTContext &C, SourceLocation Start, SourceLocation DirectiveLoc,
+         OpenACCAtomicKind AtKind, SourceLocation End, Stmt *AssociatedStmt);
+
+  OpenACCAtomicKind getAtomicKind() const { return AtomicKind; }
+  const Stmt *getAssociatedStmt() const {
+    return OpenACCAssociatedStmtConstruct::getAssociatedStmt();
+  }
+  Stmt *getAssociatedStmt() {
+    return OpenACCAssociatedStmtConstruct::getAssociatedStmt();
+  }
+};
+
 } // namespace clang
 #endif // LLVM_CLANG_AST_STMTOPENACC_H
diff --git a/clang/include/clang/AST/TextNodeDumper.h b/clang/include/clang/AST/TextNodeDumper.h
index 4aaae48ba8b42..bfd205ffb0d99 100644
--- a/clang/include/clang/AST/TextNodeDumper.h
+++ b/clang/include/clang/AST/TextNodeDumper.h
@@ -420,6 +420,7 @@ class TextNodeDumper
   void VisitOpenACCSetConstruct(const OpenACCSetConstruct *S);
   void VisitOpenACCShutdownConstruct(const OpenACCShutdownConstruct *S);
   void VisitOpenACCUpdateConstruct(const OpenACCUpdateConstruct *S);
+  void VisitOpenACCAtomicConstruct(const OpenACCAtomicConstruct *S);
   void VisitOpenACCAsteriskSizeExpr(const OpenACCAsteriskSizeExpr *S);
   void VisitEmbedExpr(const EmbedExpr *S);
   void VisitAtomicExpr(const AtomicExpr *AE);
diff --git a/clang/include/clang/AST/Type.h b/clang/include/clang/AST/Type.h
index 3457d524c63aa..1d9743520654e 100644
--- a/clang/include/clang/AST/Type.h
+++ b/clang/include/clang/AST/Type.h
@@ -2518,6 +2518,7 @@ class alignas(TypeAlignment) Type : public ExtQualsTypeCommonBase {
   bool isFloat32Type() const;
   bool isDoubleType() const;
   bool isBFloat16Type() const;
+  bool isMFloat8Type() const;
   bool isFloat128Type() const;
   bool isIbm128Type() const;
   bool isRealType() const;         // C99 6.2.5p17 (real floating + integer)
@@ -8537,6 +8538,10 @@ inline bool Type::isBFloat16Type() const {
   return isSpecificBuiltinType(BuiltinType::BFloat16);
 }
 
+inline bool Type::isMFloat8Type() const {
+  return isSpecificBuiltinType(BuiltinType::MFloat8);
+}
+
 inline bool Type::isFloat128Type() const {
   return isSpecificBuiltinType(BuiltinType::Float128);
 }
diff --git a/clang/include/clang/ASTMatchers/ASTMatchers.h b/clang/include/clang/ASTMatchers/ASTMatchers.h
index 239fcba4e5e05..0f7e3a8a01762 100644
--- a/clang/include/clang/ASTMatchers/ASTMatchers.h
+++ b/clang/include/clang/ASTMatchers/ASTMatchers.h
@@ -2489,7 +2489,28 @@ extern const internal::VariadicDynCastAllOfMatcher<Stmt, FloatingLiteral>
 extern const internal::VariadicDynCastAllOfMatcher<Stmt, ImaginaryLiteral>
     imaginaryLiteral;
 
-/// Matches fixed point literals
+/// Matches fixed-point literals eg.
+/// 0.5r, 0.5hr, 0.5lr, 0.5uhr, 0.5ur, 0.5ulr
+/// 1.0k, 1.0hk, 1.0lk, 1.0uhk, 1.0uk, 1.0ulk
+/// Exponents 1.0e10k
+/// Hexadecimal numbers 0x0.2p2r
+///
+/// Does not match implicit conversions such as first two lines:
+/// \code
+///    short _Accum sa = 2;
+///    _Accum a = 12.5;
+///    _Accum b = 1.25hk;
+///    _Fract c = 0.25hr;
+///    _Fract v = 0.35uhr;
+///    _Accum g = 1.45uhk;
+///    _Accum decexp1 = 1.575e1k;
+/// \endcode
+/// \compile_args{-ffixed-point;-std=c99}
+///
+/// The matcher \matcher{fixedPointLiteral()} matches
+/// \match{1.25hk}, \match{0.25hr}, \match{0.35uhr},
+/// \match{1.45uhk}, \match{1.575e1k}, but does not
+/// match \nomatch{12.5} and \nomatch{2} from the code block.
 extern const internal::VariadicDynCastAllOfMatcher<Stmt, FixedPointLiteral>
     fixedPointLiteral;
 
diff --git a/clang/include/clang/Analysis/ProgramPoint.h b/clang/include/clang/Analysis/ProgramPoint.h
index b9339570e1ae7..1df1f1cb892e4 100644
--- a/clang/include/clang/Analysis/ProgramPoint.h
+++ b/clang/include/clang/Analysis/ProgramPoint.h
@@ -85,6 +85,9 @@ class ProgramPoint {
               LoopExitKind,
               EpsilonKind};
 
+  static StringRef getProgramPointKindName(Kind K);
+  std::optional<SourceLocation> getSourceLocation() const;
+
 private:
   const void *Data1;
   llvm::PointerIntPair<const void *, 2, unsigned> Data2;
diff --git a/clang/include/clang/Basic/AArch64SVEACLETypes.def b/clang/include/clang/Basic/AArch64SVEACLETypes.def
index 063cac1f4a58e..6a6f51c95ebd0 100644
--- a/clang/include/clang/Basic/AArch64SVEACLETypes.def
+++ b/clang/include/clang/Basic/AArch64SVEACLETypes.def
@@ -57,6 +57,11 @@
 //  - IsBF true for vector of brain float elements.
 //===----------------------------------------------------------------------===//
 
+#ifndef SVE_SCALAR_TYPE
+#define SVE_SCALAR_TYPE(Name, MangledName, Id, SingletonId, Bits) \
+  SVE_TYPE(Name, Id, SingletonId)
+#endif
+
 #ifndef SVE_VECTOR_TYPE
 #define SVE_VECTOR_TYPE(Name, MangledName, Id, SingletonId) \
   SVE_TYPE(Name, Id, SingletonId)
@@ -72,6 +77,11 @@
   SVE_VECTOR_TYPE_DETAILS(Name, MangledName, Id, SingletonId, NumEls, ElBits, NF, false, false, true)
 #endif
 
+#ifndef SVE_VECTOR_TYPE_MFLOAT
+#define SVE_VECTOR_TYPE_MFLOAT(Name, MangledName, Id, SingletonId, NumEls, ElBits, NF) \
+  SVE_VECTOR_TYPE_DETAILS(Name, MangledName, Id, SingletonId, NumEls, ElBits, NF, false, false, false)
+#endif
+
 #ifndef SVE_VECTOR_TYPE_FLOAT
 #define SVE_VECTOR_TYPE_FLOAT(Name, MangledName, Id, SingletonId, NumEls, ElBits, NF) \
   SVE_VECTOR_TYPE_DETAILS(Name, MangledName, Id, SingletonId, NumEls, ElBits, NF, false, true, false)
@@ -97,120 +107,107 @@
   SVE_TYPE(Name, Id, SingletonId)
 #endif
 
-#ifndef AARCH64_VECTOR_TYPE
-#define AARCH64_VECTOR_TYPE(Name, MangledName, Id, SingletonId) \
-  SVE_TYPE(Name, Id, SingletonId)
-#endif
-
-#ifndef AARCH64_VECTOR_TYPE_MFLOAT
-#define AARCH64_VECTOR_TYPE_MFLOAT(Name, MangledName, Id, SingletonId, NumEls, ElBits, NF) \
-  AARCH64_VECTOR_TYPE(Name, MangledName, Id, SingletonId)
-#endif
-
 //===- Vector point types -----------------------------------------------===//
 
-SVE_VECTOR_TYPE_INT("__SVInt8_t",  "__SVInt8_t",  SveInt8,  SveInt8Ty, 16,  8, 1, true)
-SVE_VECTOR_TYPE_INT("__SVInt16_t", "__SVInt16_t", SveInt16, SveInt16Ty, 8, 16, 1, true)
-SVE_VECTOR_TYPE_INT("__SVInt32_t", "__SVInt32_t", SveInt32, SveInt32Ty, 4, 32, 1, true)
-SVE_VECTOR_TYPE_INT("__SVInt64_t", "__SVInt64_t", SveInt64, SveInt64Ty, 2, 64, 1, true)
+SVE_VECTOR_TYPE_INT(__SVInt8_t,  __SVInt8_t,  SveInt8,  SveInt8Ty, 16,  8, 1, true)
+SVE_VECTOR_TYPE_INT(__SVInt16_t, __SVInt16_t, SveInt16, SveInt16Ty, 8, 16, 1, true)
+SVE_VECTOR_TYPE_INT(__SVInt32_t, __SVInt32_t, SveInt32, SveInt32Ty, 4, 32, 1, true)
+SVE_VECTOR_TYPE_INT(__SVInt64_t, __SVInt64_t, SveInt64, SveInt64Ty, 2, 64, 1, true)
 
-SVE_VECTOR_TYPE_INT("__SVUint8_t",  "__SVUint8_t",  SveUint8,  SveUint8Ty, 16, 8, 1, false)
-SVE_VECTOR_TYPE_INT("__SVUint16_t", "__SVUint16_t", SveUint16, SveUint16Ty, 8, 16, 1, false)
-SVE_VECTOR_TYPE_INT("__SVUint32_t", "__SVUint32_t", SveUint32, SveUint32Ty, 4, 32, 1, false)
-SVE_VECTOR_TYPE_INT("__SVUint64_t", "__SVUint64_t", SveUint64, SveUint64Ty, 2, 64, 1, false)
+SVE_VECTOR_TYPE_INT(__SVUint8_t,  __SVUint8_t,  SveUint8,  SveUint8Ty, 16, 8, 1, false)
+SVE_VECTOR_TYPE_INT(__SVUint16_t, __SVUint16_t, SveUint16, SveUint16Ty, 8, 16, 1, false)
+SVE_VECTOR_TYPE_INT(__SVUint32_t, __SVUint32_t, SveUint32, SveUint32Ty, 4, 32, 1, false)
+SVE_VECTOR_TYPE_INT(__SVUint64_t, __SVUint64_t, SveUint64, SveUint64Ty, 2, 64, 1, false)
 
-SVE_VECTOR_TYPE_FLOAT("__SVFloat16_t", "__SVFloat16_t", SveFloat16, SveFloat16Ty, 8, 16, 1)
-SVE_VECTOR_TYPE_FLOAT("__SVFloat32_t", "__SVFloat32_t", SveFloat32, SveFloat32Ty, 4, 32, 1)
-SVE_VECTOR_TYPE_FLOAT("__SVFloat64_t", "__SVFloat64_t", SveFloat64, SveFloat64Ty, 2, 64, 1)
+SVE_VECTOR_TYPE_FLOAT(__SVFloat16_t, __SVFloat16_t, SveFloat16, SveFloat16Ty, 8, 16, 1)
+SVE_VECTOR_TYPE_FLOAT(__SVFloat32_t, __SVFloat32_t, SveFloat32, SveFloat32Ty, 4, 32, 1)
+SVE_VECTOR_TYPE_FLOAT(__SVFloat64_t, __SVFloat64_t, SveFloat64, SveFloat64Ty, 2, 64, 1)
 
-SVE_VECTOR_TYPE_BFLOAT("__SVBfloat16_t", "__SVBfloat16_t", SveBFloat16, SveBFloat16Ty, 8, 16, 1)
+SVE_VECTOR_TYPE_BFLOAT(__SVBfloat16_t, __SVBfloat16_t, SveBFloat16, SveBFloat16Ty, 8, 16, 1)
 
-// This is a 8 bits opaque type.
-SVE_VECTOR_TYPE_INT("__SVMfloat8_t", "__SVMfloat8_t",  SveMFloat8, SveMFloat8Ty, 16, 8, 1, false)
+SVE_VECTOR_TYPE_MFLOAT(__SVMfloat8_t, __SVMfloat8_t,  SveMFloat8, SveMFloat8Ty, 16, 8, 1)
 
 //
 // x2
 //
 
-SVE_VECTOR_TYPE_INT("__clang_svint8x2_t",  "svint8x2_t",  SveInt8x2,  SveInt8x2Ty, 16, 8, 2, true)
-SVE_VECTOR_TYPE_INT("__clang_svint16x2_t", "svint16x2_t", SveInt16x2, SveInt16x2Ty, 8, 16, 2, true)
-SVE_VECTOR_TYPE_INT("__clang_svint32x2_t", "svint32x2_t", SveInt32x2, SveInt32x2Ty, 4, 32, 2, true)
-SVE_VECTOR_TYPE_INT("__clang_svint64x2_t", "svint64x2_t", SveInt64x2, SveInt64x2Ty, 2, 64, 2, true)
+SVE_VECTOR_TYPE_INT(__clang_svint8x2_t,  svint8x2_t,  SveInt8x2,  SveInt8x2Ty, 16, 8, 2, true)
+SVE_VECTOR_TYPE_INT(__clang_svint16x2_t, svint16x2_t, SveInt16x2, SveInt16x2Ty, 8, 16, 2, true)
+SVE_VECTOR_TYPE_INT(__clang_svint32x2_t, svint32x2_t, SveInt32x2, SveInt32x2Ty, 4, 32, 2, true)
+SVE_VECTOR_TYPE_INT(__clang_svint64x2_t, svint64x2_t, SveInt64x2, SveInt64x2Ty, 2, 64, 2, true)
 
-SVE_VECTOR_TYPE_INT("__clang_svuint8x2_t",  "svuint8x2_t",  SveUint8x2,  SveUint8x2Ty, 16 , 8, 2, false)
-SVE_VECTOR_TYPE_INT("__clang_svuint16x2_t", "svuint16x2_t", SveUint16x2, SveUint16x2Ty, 8, 16, 2, false)
-SVE_VECTOR_TYPE_INT("__clang_svuint32x2_t", "svuint32x2_t", SveUint32x2, SveUint32x2Ty, 4, 32, 2, false)
-SVE_VECTOR_TYPE_INT("__clang_svuint64x2_t", "svuint64x2_t", SveUint64x2, SveUint64x2Ty, 2, 64, 2, false)
+SVE_VECTOR_TYPE_INT(__clang_svuint8x2_t,  svuint8x2_t,  SveUint8x2,  SveUint8x2Ty, 16 , 8, 2, false)
+SVE_VECTOR_TYPE_INT(__clang_svuint16x2_t, svuint16x2_t, SveUint16x2, SveUint16x2Ty, 8, 16, 2, false)
+SVE_VECTOR_TYPE_INT(__clang_svuint32x2_t, svuint32x2_t, SveUint32x2, SveUint32x2Ty, 4, 32, 2, false)
+SVE_VECTOR_TYPE_INT(__clang_svuint64x2_t, svuint64x2_t, SveUint64x2, SveUint64x2Ty, 2, 64, 2, false)
 
-SVE_VECTOR_TYPE_FLOAT("__clang_svfloat16x2_t", "svfloat16x2_t", SveFloat16x2, SveFloat16x2Ty, 8, 16, 2)
-SVE_VECTOR_TYPE_FLOAT("__clang_svfloat32x2_t", "svfloat32x2_t", SveFloat32x2, SveFloat32x2Ty, 4, 32, 2)
-SVE_VECTOR_TYPE_FLOAT("__clang_svfloat64x2_t", "svfloat64x2_t", SveFloat64x2, SveFloat64x2Ty, 2, 64, 2)
+SVE_VECTOR_TYPE_FLOAT(__clang_svfloat16x2_t, svfloat16x2_t, SveFloat16x2, SveFloat16x2Ty, 8, 16, 2)
+SVE_VECTOR_TYPE_FLOAT(__clang_svfloat32x2_t, svfloat32x2_t, SveFloat32x2, SveFloat32x2Ty, 4, 32, 2)
+SVE_VECTOR_TYPE_FLOAT(__clang_svfloat64x2_t, svfloat64x2_t, SveFloat64x2, SveFloat64x2Ty, 2, 64, 2)
 
-SVE_VECTOR_TYPE_BFLOAT("__clang_svbfloat16x2_t", "svbfloat16x2_t", SveBFloat16x2, SveBFloat16x2Ty, 8, 16, 2)
+SVE_VECTOR_TYPE_BFLOAT(__clang_svbfloat16x2_t, svbfloat16x2_t, SveBFloat16x2, SveBFloat16x2Ty, 8, 16, 2)
 
-SVE_VECTOR_TYPE_INT("__clang_svmfloat8x2_t", "svmfloat8x2_t", SveMFloat8x2, SveMFloat8x2Ty, 16, 8, 2, false)
+SVE_VECTOR_TYPE_MFLOAT(__clang_svmfloat8x2_t, svmfloat8x2_t, SveMFloat8x2, SveMFloat8x2Ty, 16, 8, 2)
 
 //
 // x3
 //
 
-SVE_VECTOR_TYPE_INT("__clang_svint8x3_t",  "svint8x3_t",  SveInt8x3,  SveInt8x3Ty, 16,  8, 3, true)
-SVE_VECTOR_TYPE_INT("__clang_svint16x3_t", "svint16x3_t", SveInt16x3, SveInt16x3Ty, 8, 16, 3, true)
-SVE_VECTOR_TYPE_INT("__clang_svint32x3_t", "svint32x3_t", SveInt32x3, SveInt32x3Ty, 4, 32, 3, true)
-SVE_VECTOR_TYPE_INT("__clang_svint64x3_t", "svint64x3_t", SveInt64x3, SveInt64x3Ty, 2, 64, 3, true)
+SVE_VECTOR_TYPE_INT(__clang_svint8x3_t,  svint8x3_t,  SveInt8x3,  SveInt8x3Ty, 16,  8, 3, true)
+SVE_VECTOR_TYPE_INT(__clang_svint16x3_t, svint16x3_t, SveInt16x3, SveInt16x3Ty, 8, 16, 3, true)
+SVE_VECTOR_TYPE_INT(__clang_svint32x3_t, svint32x3_t, SveInt32x3, SveInt32x3Ty, 4, 32, 3, true)
+SVE_VECTOR_TYPE_INT(__clang_svint64x3_t, svint64x3_t, SveInt64x3, SveInt64x3Ty, 2, 64, 3, true)
 
-SVE_VECTOR_TYPE_INT("__clang_svuint8x3_t",  "svuint8x3_t",  SveUint8x3,  SveUint8x3Ty, 16,  8, 3, false)
-SVE_VECTOR_TYPE_INT("__clang_svuint16x3_t", "svuint16x3_t", SveUint16x3, SveUint16x3Ty, 8, 16, 3, false)
-SVE_VECTOR_TYPE_INT("__clang_svuint32x3_t", "svuint32x3_t", SveUint32x3, SveUint32x3Ty, 4, 32, 3, false)
-SVE_VECTOR_TYPE_INT("__clang_svuint64x3_t", "svuint64x3_t", SveUint64x3, SveUint64x3Ty, 2, 64, 3, false)
+SVE_VECTOR_TYPE_INT(__clang_svuint8x3_t,  svuint8x3_t,  SveUint8x3,  SveUint8x3Ty, 16,  8, 3, false)
+SVE_VECTOR_TYPE_INT(__clang_svuint16x3_t, svuint16x3_t, SveUint16x3, SveUint16x3Ty, 8, 16, 3, false)
+SVE_VECTOR_TYPE_INT(__clang_svuint32x3_t, svuint32x3_t, SveUint32x3, SveUint32x3Ty, 4, 32, 3, false)
+SVE_VECTOR_TYPE_INT(__clang_svuint64x3_t, svuint64x3_t, SveUint64x3, SveUint64x3Ty, 2, 64, 3, false)
 
-SVE_VECTOR_TYPE_FLOAT("__clang_svfloat16x3_t", "svfloat16x3_t", SveFloat16x3, SveFloat16x3Ty, 8, 16, 3)
-SVE_VECTOR_TYPE_FLOAT("__clang_svfloat32x3_t", "svfloat32x3_t", SveFloat32x3, SveFloat32x3Ty, 4, 32, 3)
-SVE_VECTOR_TYPE_FLOAT("__clang_svfloat64x3_t", "svfloat64x3_t", SveFloat64x3, SveFloat64x3Ty, 2, 64, 3)
+SVE_VECTOR_TYPE_FLOAT(__clang_svfloat16x3_t, svfloat16x3_t, SveFloat16x3, SveFloat16x3Ty, 8, 16, 3)
+SVE_VECTOR_TYPE_FLOAT(__clang_svfloat32x3_t, svfloat32x3_t, SveFloat32x3, SveFloat32x3Ty, 4, 32, 3)
+SVE_VECTOR_TYPE_FLOAT(__clang_svfloat64x3_t, svfloat64x3_t, SveFloat64x3, SveFloat64x3Ty, 2, 64, 3)
 
-SVE_VECTOR_TYPE_BFLOAT("__clang_svbfloat16x3_t", "svbfloat16x3_t", SveBFloat16x3, SveBFloat16x3Ty, 8, 16, 3)
+SVE_VECTOR_TYPE_BFLOAT(__clang_svbfloat16x3_t, svbfloat16x3_t, SveBFloat16x3, SveBFloat16x3Ty, 8, 16, 3)
 
-SVE_VECTOR_TYPE_INT("__clang_svmfloat8x3_t", "svmfloat8x3_t", SveMFloat8x3, SveMFloat8x3Ty, 16, 8, 3, false)
+SVE_VECTOR_TYPE_MFLOAT(__clang_svmfloat8x3_t, svmfloat8x3_t, SveMFloat8x3, SveMFloat8x3Ty, 16, 8, 3)
 
 //
 // x4
 //
 
-SVE_VECTOR_TYPE_INT("__clang_svint8x4_t",  "svint8x4_t",  SveInt8x4,  SveInt8x4Ty, 16,  8, 4, true)
-SVE_VECTOR_TYPE_INT("__clang_svint16x4_t", "svint16x4_t", SveInt16x4, SveInt16x4Ty, 8, 16, 4, true)
-SVE_VECTOR_TYPE_INT("__clang_svint32x4_t", "svint32x4_t", SveInt32x4, SveInt32x4Ty, 4, 32, 4, true)
-SVE_VECTOR_TYPE_INT("__clang_svint64x4_t", "svint64x4_t", SveInt64x4, SveInt64x4Ty, 2, 64, 4, true)
+SVE_VECTOR_TYPE_INT(__clang_svint8x4_t,  svint8x4_t,  SveInt8x4,  SveInt8x4Ty, 16,  8, 4, true)
+SVE_VECTOR_TYPE_INT(__clang_svint16x4_t, svint16x4_t, SveInt16x4, SveInt16x4Ty, 8, 16, 4, true)
+SVE_VECTOR_TYPE_INT(__clang_svint32x4_t, svint32x4_t, SveInt32x4, SveInt32x4Ty, 4, 32, 4, true)
+SVE_VECTOR_TYPE_INT(__clang_svint64x4_t, svint64x4_t, SveInt64x4, SveInt64x4Ty, 2, 64, 4, true)
 
-SVE_VECTOR_TYPE_INT("__clang_svuint8x4_t",  "svuint8x4_t",  SveUint8x4,  SveUint8x4Ty, 16,  8, 4, false)
-SVE_VECTOR_TYPE_INT("__clang_svuint16x4_t", "svuint16x4_t", SveUint16x4, SveUint16x4Ty, 8, 16, 4, false)
-SVE_VECTOR_TYPE_INT("__clang_svuint32x4_t", "svuint32x4_t", SveUint32x4, SveUint32x4Ty, 4, 32, 4, false)
-SVE_VECTOR_TYPE_INT("__clang_svuint64x4_t", "svuint64x4_t", SveUint64x4, SveUint64x4Ty, 2, 64, 4, false)
+SVE_VECTOR_TYPE_INT(__clang_svuint8x4_t,  svuint8x4_t,  SveUint8x4,  SveUint8x4Ty, 16,  8, 4, false)
+SVE_VECTOR_TYPE_INT(__clang_svuint16x4_t, svuint16x4_t, SveUint16x4, SveUint16x4Ty, 8, 16, 4, false)
+SVE_VECTOR_TYPE_INT(__clang_svuint32x4_t, svuint32x4_t, SveUint32x4, SveUint32x4Ty, 4, 32, 4, false)
+SVE_VECTOR_TYPE_INT(__clang_svuint64x4_t, svuint64x4_t, SveUint64x4, SveUint64x4Ty, 2, 64, 4, false)
 
-SVE_VECTOR_TYPE_FLOAT("__clang_svfloat16x4_t", "svfloat16x4_t", SveFloat16x4, SveFloat16x4Ty, 8, 16, 4)
-SVE_VECTOR_TYPE_FLOAT("__clang_svfloat32x4_t", "svfloat32x4_t", SveFloat32x4, SveFloat32x4Ty, 4, 32, 4)
-SVE_VECTOR_TYPE_FLOAT("__clang_svfloat64x4_t", "svfloat64x4_t", SveFloat64x4, SveFloat64x4Ty, 2, 64, 4)
+SVE_VECTOR_TYPE_FLOAT(__clang_svfloat16x4_t, svfloat16x4_t, SveFloat16x4, SveFloat16x4Ty, 8, 16, 4)
+SVE_VECTOR_TYPE_FLOAT(__clang_svfloat32x4_t, svfloat32x4_t, SveFloat32x4, SveFloat32x4Ty, 4, 32, 4)
+SVE_VECTOR_TYPE_FLOAT(__clang_svfloat64x4_t, svfloat64x4_t, SveFloat64x4, SveFloat64x4Ty, 2, 64, 4)
 
-SVE_VECTOR_TYPE_BFLOAT("__clang_svbfloat16x4_t", "svbfloat16x4_t", SveBFloat16x4, SveBFloat16x4Ty, 8, 16, 4)
+SVE_VECTOR_TYPE_BFLOAT(__clang_svbfloat16x4_t, svbfloat16x4_t, SveBFloat16x4, SveBFloat16x4Ty, 8, 16, 4)
 
-SVE_VECTOR_TYPE_INT("__clang_svmfloat8x4_t", "svmfloat8x4_t", SveMFloat8x4, SveMFloat8x4Ty, 16, 8, 4, false)
+SVE_VECTOR_TYPE_MFLOAT(__clang_svmfloat8x4_t, svmfloat8x4_t, SveMFloat8x4, SveMFloat8x4Ty, 16, 8, 4)
 
-SVE_PREDICATE_TYPE_ALL("__SVBool_t", "__SVBool_t", SveBool, SveBoolTy, 16, 1)
-SVE_PREDICATE_TYPE_ALL("__clang_svboolx2_t", "svboolx2_t", SveBoolx2, SveBoolx2Ty, 16, 2)
-SVE_PREDICATE_TYPE_ALL("__clang_svboolx4_t", "svboolx4_t", SveBoolx4, SveBoolx4Ty, 16, 4)
+SVE_PREDICATE_TYPE_ALL(__SVBool_t, __SVBool_t, SveBool, SveBoolTy, 16, 1)
+SVE_PREDICATE_TYPE_ALL(__clang_svboolx2_t, svboolx2_t, SveBoolx2, SveBoolx2Ty, 16, 2)
+SVE_PREDICATE_TYPE_ALL(__clang_svboolx4_t, svboolx4_t, SveBoolx4, SveBoolx4Ty, 16, 4)
 
-SVE_OPAQUE_TYPE("__SVCount_t", "__SVCount_t", SveCount, SveCountTy)
+SVE_OPAQUE_TYPE(__SVCount_t, __SVCount_t, SveCount, SveCountTy)
 
-AARCH64_VECTOR_TYPE_MFLOAT("__mfp8", "__mfp8", MFloat8, MFloat8Ty, 1, 8, 1)
-AARCH64_VECTOR_TYPE_MFLOAT("__MFloat8x8_t", "__MFloat8x8_t", MFloat8x8, MFloat8x8Ty, 8, 8, 1)
-AARCH64_VECTOR_TYPE_MFLOAT("__MFloat8x16_t", "__MFloat8x16_t", MFloat8x16, MFloat8x16Ty, 16, 8, 1)
+SVE_SCALAR_TYPE(__mfp8, __mfp8, MFloat8, MFloat8Ty, 8)
 
 #undef SVE_VECTOR_TYPE
+#undef SVE_VECTOR_TYPE_MFLOAT
 #undef SVE_VECTOR_TYPE_BFLOAT
 #undef SVE_VECTOR_TYPE_FLOAT
 #undef SVE_VECTOR_TYPE_INT
 #undef SVE_PREDICATE_TYPE
 #undef SVE_PREDICATE_TYPE_ALL
 #undef SVE_OPAQUE_TYPE
-#undef AARCH64_VECTOR_TYPE_MFLOAT
-#undef AARCH64_VECTOR_TYPE
+#undef SVE_SCALAR_TYPE
 #undef SVE_TYPE
diff --git a/clang/include/clang/Basic/AddressSpaces.h b/clang/include/clang/Basic/AddressSpaces.h
index 7b723d508fff1..d18bfe54931f9 100644
--- a/clang/include/clang/Basic/AddressSpaces.h
+++ b/clang/include/clang/Basic/AddressSpaces.h
@@ -58,6 +58,7 @@ enum class LangAS : unsigned {
 
   // HLSL specific address spaces.
   hlsl_groupshared,
+  hlsl_constant,
 
   // Wasm specific address spaces.
   wasm_funcref,
diff --git a/clang/include/clang/Basic/Attr.td b/clang/include/clang/Basic/Attr.td
index 408d3adf370c8..4384a98d63eb3 100644
--- a/clang/include/clang/Basic/Attr.td
+++ b/clang/include/clang/Basic/Attr.td
@@ -198,7 +198,7 @@ def OpenCLKernelFunction
 // inclusive nature of subject testing).
 def HasFunctionProto : SubsetSubject<DeclBase,
                                      [{(S->getFunctionType(true) != nullptr &&
-                              isa<FunctionProtoType>(S->getFunctionType())) ||
+                                       isa<FunctionProtoType>(S->getFunctionType())) ||
                                        isa<ObjCMethodDecl>(S) ||
                                        isa<BlockDecl>(S)}],
                                      "non-K&R-style functions">;
@@ -380,6 +380,13 @@ class Clang<string name, bit allowInC = 1, int version = 1>
   bit AllowInC = allowInC;
 }
 
+// This spelling combines the spellings of GCC and Clang for cases where the
+// spellings are equivalent for compile compatibility.
+class ClangGCC<string name, bit allowInC = 1, int version = 1>
+    : Spelling<name, "ClangGCC", version> {
+  bit AllowInC = allowInC;
+}
+
 // HLSL Annotation spellings
 class HLSLAnnotation<string name> : Spelling<name, "HLSLAnnotation">;
 
@@ -3458,18 +3465,16 @@ def DiagnoseIf : InheritableAttr {
   let Spellings = [GNU<"diagnose_if">];
   let Subjects = SubjectList<[Function, ObjCMethod, ObjCProperty]>;
   let Args = [ExprArgument<"Cond">, StringArgument<"Message">,
-              EnumArgument<"DiagnosticType", "DiagnosticType",
+              EnumArgument<"DefaultSeverity",
+                           "DefaultSeverity",
                            /*is_string=*/true,
-                           ["error", "warning"],
-                           ["DT_Error", "DT_Warning"]>,
+                           ["error",    "warning"],
+                           ["DS_error", "DS_warning"]>,
+              StringArgument<"WarningGroup", /*optional*/ 1>,
               BoolArgument<"ArgDependent", 0, /*fake*/ 1>,
               DeclArgument<Named, "Parent", 0, /*fake*/ 1>];
   let InheritEvenIfAlreadyPresent = 1;
   let LateParsed = LateAttrParseStandard;
-  let AdditionalMembers = [{
-    bool isError() const { return diagnosticType == DT_Error; }
-    bool isWarning() const { return diagnosticType == DT_Warning; }
-  }];
   let TemplateDependent = 1;
   let Documentation = [DiagnoseIfDocs];
 }
@@ -3679,7 +3684,7 @@ def X86ForceAlignArgPointer : InheritableAttr, TargetSpecificAttr<TargetAnyX86>
 }
 
 def NoSanitize : InheritableAttr {
-  let Spellings = [Clang<"no_sanitize">];
+  let Spellings = [ClangGCC<"no_sanitize">];
   let Args = [VariadicStringArgument<"Sanitizers">];
   let Subjects = SubjectList<[Function, ObjCMethod, GlobalVar], ErrorDiag>;
   let Documentation = [NoSanitizeDocs];
diff --git a/clang/include/clang/Basic/AttrDocs.td b/clang/include/clang/Basic/AttrDocs.td
index a8b588169725a..0ad4c958d0983 100644
--- a/clang/include/clang/Basic/AttrDocs.td
+++ b/clang/include/clang/Basic/AttrDocs.td
@@ -5274,6 +5274,7 @@ optimization passes are aware of the following assumptions:
     "omp_no_openmp"
     "omp_no_openmp_routines"
     "omp_no_parallelism"
+    "omp_no_openmp_constructs"
 
 The OpenMP standard defines the meaning of OpenMP assumptions ("omp_XYZ" is
 spelled "XYZ" in the `OpenMP 5.1 Standard`_).
diff --git a/clang/include/clang/Basic/Builtins.h b/clang/include/clang/Basic/Builtins.h
index 63559d977ce6b..6d29b4315e5a7 100644
--- a/clang/include/clang/Basic/Builtins.h
+++ b/clang/include/clang/Basic/Builtins.h
@@ -18,6 +18,7 @@
 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/StringMap.h"
 #include "llvm/ADT/StringRef.h"
+#include "llvm/ADT/StringTable.h"
 #include <cstring>
 
 // VC++ defines 'alloca' as an object-like macro, which interferes with our
@@ -55,6 +56,7 @@ struct HeaderDesc {
 #undef HEADER
   } ID;
 
+  constexpr HeaderDesc() : ID() {}
   constexpr HeaderDesc(HeaderID ID) : ID(ID) {}
 
   const char *getName() const;
@@ -62,20 +64,160 @@ struct HeaderDesc {
 
 namespace Builtin {
 enum ID {
-  NotBuiltin  = 0,      // This is not a builtin function.
-#define BUILTIN(ID, TYPE, ATTRS) BI##ID,
+  NotBuiltin = 0, // This is not a builtin function.
+#define GET_BUILTIN_ENUMERATORS
 #include "clang/Basic/Builtins.inc"
+#undef GET_BUILTIN_ENUMERATORS
   FirstTSBuiltin
 };
 
+struct InfosShard;
+
+/// The info used to represent each builtin.
 struct Info {
-  llvm::StringLiteral Name;
-  const char *Type, *Attributes;
-  const char *Features;
-  HeaderDesc Header;
-  LanguageID Langs;
+  // Rather than store pointers to the string literals describing these four
+  // aspects of builtins, we store offsets into a common string table.
+  struct StrOffsets {
+    llvm::StringTable::Offset Name = {};
+    llvm::StringTable::Offset Type = {};
+    llvm::StringTable::Offset Attributes = {};
+
+    // Defaults to the empty string offset.
+    llvm::StringTable::Offset Features = {};
+  } Offsets;
+
+  HeaderDesc Header = HeaderDesc::NO_HEADER;
+  LanguageID Langs = ALL_LANGUAGES;
+
+  /// Get the name for the builtin represented by this `Info` object.
+  ///
+  /// Must be provided the `Shard` for this `Info` object.
+  std::string getName(const InfosShard &Shard) const;
 };
 
+/// A constexpr function to construct an infos array from X-macros.
+///
+/// The input array uses the same data structure, but the offsets are actually
+/// _lengths_ when input. This is all we can compute from the X-macro approach
+/// to builtins. This function will convert these lengths into actual offsets to
+/// a string table built up through sequentially appending strings with the
+/// given lengths.
+template <size_t N>
+static constexpr std::array<Info, N> MakeInfos(std::array<Info, N> Infos) {
+  // Translate lengths to offsets. We start past the initial empty string at
+  // offset zero.
+  unsigned Offset = 1;
+  for (Info &I : Infos) {
+    Info::StrOffsets NewOffsets = {};
+    NewOffsets.Name = Offset;
+    Offset += I.Offsets.Name.value();
+    NewOffsets.Type = Offset;
+    Offset += I.Offsets.Type.value();
+    NewOffsets.Attributes = Offset;
+    Offset += I.Offsets.Attributes.value();
+    NewOffsets.Features = Offset;
+    Offset += I.Offsets.Features.value();
+    I.Offsets = NewOffsets;
+  }
+  return Infos;
+}
+
+/// A shard of a target's builtins string table and info.
+///
+/// Target builtins are sharded across multiple tables due to different
+/// structures, origins, and also to improve the overall scaling by avoiding a
+/// single table across all builtins.
+struct InfosShard {
+  const llvm::StringTable *Strings;
+  llvm::ArrayRef<Info> Infos;
+
+  llvm::StringLiteral NamePrefix = "";
+};
+
+// A detail macro used below to emit a string literal that, after string literal
+// concatenation, ends up triggering the `-Woverlength-strings` warning. While
+// the warning is useful in general to catch accidentally excessive strings,
+// here we are creating them intentionally.
+//
+// This relies on a subtle aspect of `_Pragma`: that the *diagnostic* ones don't
+// turn into actual tokens that would disrupt string literal concatenation.
+#ifdef __clang__
+#define CLANG_BUILTIN_DETAIL_STR_TABLE(S)                                      \
+  _Pragma("clang diagnostic push")                                             \
+      _Pragma("clang diagnostic ignored \"-Woverlength-strings\"")             \
+          S _Pragma("clang diagnostic pop")
+#else
+#define CLANG_BUILTIN_DETAIL_STR_TABLE(S) S
+#endif
+
+// We require string tables to start with an empty string so that a `0` offset
+// can always be used to refer to an empty string. To satisfy that when building
+// string tables with X-macros, we use this start macro prior to expanding the
+// X-macros.
+#define CLANG_BUILTIN_STR_TABLE_START CLANG_BUILTIN_DETAIL_STR_TABLE("\0")
+
+// A macro that can be used with `Builtins.def` and similar files as an X-macro
+// to add the string arguments to a builtin string table. This is typically the
+// target for the `BUILTIN`, `LANGBUILTIN`, or `LIBBUILTIN` macros in those
+// files.
+#define CLANG_BUILTIN_STR_TABLE(ID, TYPE, ATTRS)                               \
+  CLANG_BUILTIN_DETAIL_STR_TABLE(#ID "\0" TYPE "\0" ATTRS "\0" /*FEATURE*/ "\0")
+
+// A macro that can be used with target builtin `.def` and `.inc` files as an
+// X-macro to add the string arguments to a builtin string table. this is
+// typically the target for the `TARGET_BUILTIN` macro.
+#define CLANG_TARGET_BUILTIN_STR_TABLE(ID, TYPE, ATTRS, FEATURE)               \
+  CLANG_BUILTIN_DETAIL_STR_TABLE(#ID "\0" TYPE "\0" ATTRS "\0" FEATURE "\0")
+
+// A macro that can be used with target builtin `.def` and `.inc` files as an
+// X-macro to add the string arguments to a builtin string table. this is
+// typically the target for the `TARGET_HEADER_BUILTIN` macro. We can't delegate
+// to `TARGET_BUILTIN` because the `FEATURE` string changes position.
+#define CLANG_TARGET_HEADER_BUILTIN_STR_TABLE(ID, TYPE, ATTRS, HEADER, LANGS,  \
+                                              FEATURE)                         \
+  CLANG_BUILTIN_DETAIL_STR_TABLE(#ID "\0" TYPE "\0" ATTRS "\0" FEATURE "\0")
+
+// A detail macro used internally to compute the desired string table
+// `StrOffsets` struct for arguments to `MakeInfos`.
+#define CLANG_BUILTIN_DETAIL_STR_OFFSETS(ID, TYPE, ATTRS)                      \
+  Builtin::Info::StrOffsets {                                                  \
+    sizeof(#ID), sizeof(TYPE), sizeof(ATTRS), sizeof("")                       \
+  }
+
+// A detail macro used internally to compute the desired string table
+// `StrOffsets` struct for arguments to `Storage::Make`.
+#define CLANG_TARGET_BUILTIN_DETAIL_STR_OFFSETS(ID, TYPE, ATTRS, FEATURE)      \
+  Builtin::Info::StrOffsets {                                                  \
+    sizeof(#ID), sizeof(TYPE), sizeof(ATTRS), sizeof(FEATURE)                  \
+  }
+
+// A set of macros that can be used with builtin `.def' files as an X-macro to
+// create an `Info` struct for a particular builtin. It both computes the
+// `StrOffsets` value for the string table (the lengths here, translated to
+// offsets by the `MakeInfos` function), and the other metadata for each
+// builtin.
+//
+// There is a corresponding macro for each of `BUILTIN`, `LANGBUILTIN`,
+// `LIBBUILTIN`, `TARGET_BUILTIN`, and `TARGET_HEADER_BUILTIN`.
+#define CLANG_BUILTIN_ENTRY(ID, TYPE, ATTRS)                                   \
+  Builtin::Info{CLANG_BUILTIN_DETAIL_STR_OFFSETS(ID, TYPE, ATTRS),             \
+                HeaderDesc::NO_HEADER, ALL_LANGUAGES},
+#define CLANG_LANGBUILTIN_ENTRY(ID, TYPE, ATTRS, LANG)                         \
+  Builtin::Info{CLANG_BUILTIN_DETAIL_STR_OFFSETS(ID, TYPE, ATTRS),             \
+                HeaderDesc::NO_HEADER, LANG},
+#define CLANG_LIBBUILTIN_ENTRY(ID, TYPE, ATTRS, HEADER, LANG)                  \
+  Builtin::Info{CLANG_BUILTIN_DETAIL_STR_OFFSETS(ID, TYPE, ATTRS),             \
+                HeaderDesc::HEADER, LANG},
+#define CLANG_TARGET_BUILTIN_ENTRY(ID, TYPE, ATTRS, FEATURE)                   \
+  Builtin::Info{                                                               \
+      CLANG_TARGET_BUILTIN_DETAIL_STR_OFFSETS(ID, TYPE, ATTRS, FEATURE),       \
+      HeaderDesc::NO_HEADER, ALL_LANGUAGES},
+#define CLANG_TARGET_HEADER_BUILTIN_ENTRY(ID, TYPE, ATTRS, HEADER, LANG,       \
+                                          FEATURE)                             \
+  Builtin::Info{                                                               \
+      CLANG_TARGET_BUILTIN_DETAIL_STR_OFFSETS(ID, TYPE, ATTRS, FEATURE),       \
+      HeaderDesc::HEADER, LANG},
+
 /// Holds information about both target-independent and
 /// target-specific builtins, allowing easy queries by clients.
 ///
@@ -83,11 +225,16 @@ struct Info {
 /// AuxTSRecords. Their IDs are shifted up by TSRecords.size() and need to
 /// be translated back with getAuxBuiltinID() before use.
 class Context {
-  llvm::ArrayRef<Info> TSRecords;
-  llvm::ArrayRef<Info> AuxTSRecords;
+  llvm::SmallVector<InfosShard> BuiltinShards;
+
+  llvm::SmallVector<InfosShard> TargetShards;
+  llvm::SmallVector<InfosShard> AuxTargetShards;
+
+  unsigned NumTargetBuiltins = 0;
+  unsigned NumAuxTargetBuiltins = 0;
 
 public:
-  Context() = default;
+  Context();
 
   /// Perform target-specific initialization
   /// \param AuxTarget Target info to incorporate builtins from. May be nullptr.
@@ -100,13 +247,17 @@ class Context {
 
   /// Return the identifier name for the specified builtin,
   /// e.g. "__builtin_abs".
-  llvm::StringRef getName(unsigned ID) const { return getRecord(ID).Name; }
+  std::string getName(unsigned ID) const;
 
-  /// Return a quoted name for the specified builtin for use in diagnostics.
+  /// Return the identifier name for the specified builtin inside single quotes
+  /// for a diagnostic, e.g. "'__builtin_abs'".
   std::string getQuotedName(unsigned ID) const;
 
   /// Get the type descriptor string for the specified builtin.
-  const char *getTypeString(unsigned ID) const { return getRecord(ID).Type; }
+  const char *getTypeString(unsigned ID) const;
+
+  /// Get the attributes descriptor string for the specified builtin.
+  const char *getAttributesString(unsigned ID) const;
 
   /// Return true if this function is a target-specific builtin.
   bool isTSBuiltin(unsigned ID) const {
@@ -115,40 +266,40 @@ class Context {
 
   /// Return true if this function has no side effects.
   bool isPure(unsigned ID) const {
-    return strchr(getRecord(ID).Attributes, 'U') != nullptr;
+    return strchr(getAttributesString(ID), 'U') != nullptr;
   }
 
   /// Return true if this function has no side effects and doesn't
   /// read memory.
   bool isConst(unsigned ID) const {
-    return strchr(getRecord(ID).Attributes, 'c') != nullptr;
+    return strchr(getAttributesString(ID), 'c') != nullptr;
   }
 
   /// Return true if we know this builtin never throws an exception.
   bool isNoThrow(unsigned ID) const {
-    return strchr(getRecord(ID).Attributes, 'n') != nullptr;
+    return strchr(getAttributesString(ID), 'n') != nullptr;
   }
 
   /// Return true if we know this builtin never returns.
   bool isNoReturn(unsigned ID) const {
-    return strchr(getRecord(ID).Attributes, 'r') != nullptr;
+    return strchr(getAttributesString(ID), 'r') != nullptr;
   }
 
   /// Return true if we know this builtin can return twice.
   bool isReturnsTwice(unsigned ID) const {
-    return strchr(getRecord(ID).Attributes, 'j') != nullptr;
+    return strchr(getAttributesString(ID), 'j') != nullptr;
   }
 
   /// Returns true if this builtin does not perform the side-effects
   /// of its arguments.
   bool isUnevaluated(unsigned ID) const {
-    return strchr(getRecord(ID).Attributes, 'u') != nullptr;
+    return strchr(getAttributesString(ID), 'u') != nullptr;
   }
 
   /// Return true if this is a builtin for a libc/libm function,
   /// with a "__builtin_" prefix (e.g. __builtin_abs).
   bool isLibFunction(unsigned ID) const {
-    return strchr(getRecord(ID).Attributes, 'F') != nullptr;
+    return strchr(getAttributesString(ID), 'F') != nullptr;
   }
 
   /// Determines whether this builtin is a predefined libc/libm
@@ -159,21 +310,21 @@ class Context {
   /// they do not, but they are recognized as builtins once we see
   /// a declaration.
   bool isPredefinedLibFunction(unsigned ID) const {
-    return strchr(getRecord(ID).Attributes, 'f') != nullptr;
+    return strchr(getAttributesString(ID), 'f') != nullptr;
   }
 
   /// Returns true if this builtin requires appropriate header in other
   /// compilers. In Clang it will work even without including it, but we can emit
   /// a warning about missing header.
   bool isHeaderDependentFunction(unsigned ID) const {
-    return strchr(getRecord(ID).Attributes, 'h') != nullptr;
+    return strchr(getAttributesString(ID), 'h') != nullptr;
   }
 
   /// Determines whether this builtin is a predefined compiler-rt/libgcc
   /// function, such as "__clear_cache", where we know the signature a
   /// priori.
   bool isPredefinedRuntimeFunction(unsigned ID) const {
-    return strchr(getRecord(ID).Attributes, 'i') != nullptr;
+    return strchr(getAttributesString(ID), 'i') != nullptr;
   }
 
   /// Determines whether this builtin is a C++ standard library function
@@ -181,7 +332,7 @@ class Context {
   /// specialization, where the signature is determined by the standard library
   /// declaration.
   bool isInStdNamespace(unsigned ID) const {
-    return strchr(getRecord(ID).Attributes, 'z') != nullptr;
+    return strchr(getAttributesString(ID), 'z') != nullptr;
   }
 
   /// Determines whether this builtin can have its address taken with no
@@ -195,33 +346,33 @@ class Context {
 
   /// Determines whether this builtin has custom typechecking.
   bool hasCustomTypechecking(unsigned ID) const {
-    return strchr(getRecord(ID).Attributes, 't') != nullptr;
+    return strchr(getAttributesString(ID), 't') != nullptr;
   }
 
   /// Determines whether a declaration of this builtin should be recognized
   /// even if the type doesn't match the specified signature.
   bool allowTypeMismatch(unsigned ID) const {
-    return strchr(getRecord(ID).Attributes, 'T') != nullptr ||
+    return strchr(getAttributesString(ID), 'T') != nullptr ||
            hasCustomTypechecking(ID);
   }
 
   /// Determines whether this builtin has a result or any arguments which
   /// are pointer types.
   bool hasPtrArgsOrResult(unsigned ID) const {
-    return strchr(getRecord(ID).Type, '*') != nullptr;
+    return strchr(getTypeString(ID), '*') != nullptr;
   }
 
   /// Return true if this builtin has a result or any arguments which are
   /// reference types.
   bool hasReferenceArgsOrResult(unsigned ID) const {
-    return strchr(getRecord(ID).Type, '&') != nullptr ||
-           strchr(getRecord(ID).Type, 'A') != nullptr;
+    return strchr(getTypeString(ID), '&') != nullptr ||
+           strchr(getTypeString(ID), 'A') != nullptr;
   }
 
   /// If this is a library function that comes from a specific
   /// header, retrieve that header name.
   const char *getHeaderName(unsigned ID) const {
-    return getRecord(ID).Header.getName();
+    return getInfo(ID).Header.getName();
   }
 
   /// Determine whether this builtin is like printf in its
@@ -246,27 +397,25 @@ class Context {
   /// Such functions can be const when the MathErrno lang option and FP
   /// exceptions are disabled.
   bool isConstWithoutErrnoAndExceptions(unsigned ID) const {
-    return strchr(getRecord(ID).Attributes, 'e') != nullptr;
+    return strchr(getAttributesString(ID), 'e') != nullptr;
   }
 
   bool isConstWithoutExceptions(unsigned ID) const {
-    return strchr(getRecord(ID).Attributes, 'g') != nullptr;
+    return strchr(getAttributesString(ID), 'g') != nullptr;
   }
 
-  const char *getRequiredFeatures(unsigned ID) const {
-    return getRecord(ID).Features;
-  }
+  const char *getRequiredFeatures(unsigned ID) const;
 
   unsigned getRequiredVectorWidth(unsigned ID) const;
 
   /// Return true if builtin ID belongs to AuxTarget.
   bool isAuxBuiltinID(unsigned ID) const {
-    return ID >= (Builtin::FirstTSBuiltin + TSRecords.size());
+    return ID >= (Builtin::FirstTSBuiltin + NumTargetBuiltins);
   }
 
   /// Return real builtin ID (i.e. ID it would have during compilation
   /// for AuxTarget).
-  unsigned getAuxBuiltinID(unsigned ID) const { return ID - TSRecords.size(); }
+  unsigned getAuxBuiltinID(unsigned ID) const { return ID - NumTargetBuiltins; }
 
   /// Returns true if this is a libc/libm function without the '__builtin_'
   /// prefix.
@@ -278,16 +427,19 @@ class Context {
 
   /// Return true if this function can be constant evaluated by Clang frontend.
   bool isConstantEvaluated(unsigned ID) const {
-    return strchr(getRecord(ID).Attributes, 'E') != nullptr;
+    return strchr(getAttributesString(ID), 'E') != nullptr;
   }
 
   /// Returns true if this is an immediate (consteval) function
   bool isImmediate(unsigned ID) const {
-    return strchr(getRecord(ID).Attributes, 'G') != nullptr;
+    return strchr(getAttributesString(ID), 'G') != nullptr;
   }
 
 private:
-  const Info &getRecord(unsigned ID) const;
+  std::pair<const InfosShard &, const Info &>
+  getShardAndInfo(unsigned ID) const;
+
+  const Info &getInfo(unsigned ID) const { return getShardAndInfo(ID).second; }
 
   /// Helper function for isPrintfLike and isScanfLike.
   bool isLike(unsigned ID, unsigned &FormatIdx, bool &HasVAListArg,
diff --git a/clang/include/clang/Basic/Builtins.td b/clang/include/clang/Basic/Builtins.td
index 60c360d4a9e07..2433427a89429 100644
--- a/clang/include/clang/Basic/Builtins.td
+++ b/clang/include/clang/Basic/Builtins.td
@@ -4753,6 +4753,12 @@ def GetDeviceSideMangledName : LangBuiltin<"CUDA_LANG"> {
 }
 
 // HLSL
+def HLSLAddUint64: LangBuiltin<"HLSL_LANG"> {
+  let Spellings = ["__builtin_hlsl_adduint64"];
+  let Attributes = [NoThrow, Const];
+  let Prototype = "void(...)";
+}
+
 def HLSLResourceGetPointer : LangBuiltin<"HLSL_LANG"> {
   let Spellings = ["__builtin_hlsl_resource_getpointer"];
   let Attributes = [NoThrow];
@@ -4795,6 +4801,12 @@ def HLSLWaveActiveCountBits : LangBuiltin<"HLSL_LANG"> {
   let Prototype = "unsigned int(bool)";
 }
 
+def HLSLWaveActiveMax : LangBuiltin<"HLSL_LANG"> {
+  let Spellings = ["__builtin_hlsl_wave_active_max"];
+  let Attributes = [NoThrow, Const];
+  let Prototype = "void (...)";
+}
+
 def HLSLWaveActiveSum : LangBuiltin<"HLSL_LANG"> {
   let Spellings = ["__builtin_hlsl_wave_active_sum"];
   let Attributes = [NoThrow, Const];
diff --git a/clang/include/clang/Basic/BuiltinsAMDGPU.def b/clang/include/clang/Basic/BuiltinsAMDGPU.def
index 1b29a8e359c20..39e295aced96b 100644
--- a/clang/include/clang/Basic/BuiltinsAMDGPU.def
+++ b/clang/include/clang/Basic/BuiltinsAMDGPU.def
@@ -504,6 +504,8 @@ TARGET_BUILTIN(__builtin_amdgcn_global_load_tr_b128_v4i16, "V4sV4s*1", "nc", "gf
 TARGET_BUILTIN(__builtin_amdgcn_global_load_tr_b128_v4f16, "V4hV4h*1", "nc", "gfx12-insts,wavefrontsize64")
 TARGET_BUILTIN(__builtin_amdgcn_global_load_tr_b128_v4bf16, "V4yV4y*1", "nc", "gfx12-insts,wavefrontsize64")
 
+TARGET_BUILTIN(__builtin_amdgcn_ds_bpermute_fi_b32, "iii", "nc", "gfx12-insts")
+
 //===----------------------------------------------------------------------===//
 // WMMA builtins.
 // Postfix w32 indicates the builtin requires wavefront size of 32.
diff --git a/clang/include/clang/Basic/BuiltinsARM.def b/clang/include/clang/Basic/BuiltinsARM.def
index 5a7064a98045e..cbab87cecbc7d 100644
--- a/clang/include/clang/Basic/BuiltinsARM.def
+++ b/clang/include/clang/Basic/BuiltinsARM.def
@@ -206,13 +206,6 @@ BUILTIN(__builtin_arm_wsrp, "vcC*vC*", "nc")
 // Misc
 BUILTIN(__builtin_sponentry, "v*", "c")
 
-// Builtins for implementing ACLE MVE intrinsics. (Unlike NEON, these
-// don't need to live in a separate BuiltinsMVE.def, because they
-// aren't included from both here and BuiltinsAArch64.def.)
-#include "clang/Basic/arm_mve_builtins.inc"
-
-#include "clang/Basic/arm_cde_builtins.inc"
-
 // MSVC
 LANGBUILTIN(__emit, "vIUiC", "", ALL_MS_LANGUAGES)
 
diff --git a/clang/include/clang/Basic/BuiltinsBase.td b/clang/include/clang/Basic/BuiltinsBase.td
index 6180a94aa4b5c..cf15a31235e7e 100644
--- a/clang/include/clang/Basic/BuiltinsBase.td
+++ b/clang/include/clang/Basic/BuiltinsBase.td
@@ -86,6 +86,13 @@ def Consteval : Attribute<"EG">;
 // indicated by the remaining indices.
 class Callback<list<int> ArgIndices> : MultiIndexAttribute<"C", ArgIndices>;
 
+// Prefixes
+// ========
+
+class NamePrefix<string spelling> {
+  string Spelling = spelling;
+}
+
 // Builtin kinds
 // =============
 
@@ -99,6 +106,9 @@ class Builtin {
   bit RequiresUndef = 0;
   // Enables builtins to generate `long long` outside of OpenCL and `long` inside.
   bit EnableOpenCLLong = 0;
+  // Requires a common prefix to be prepended. Each generated set of builtins
+  // can optionally extract one common prefix that is handled separately.
+  NamePrefix RequiredNamePrefix;
 }
 
 class AtomicBuiltin : Builtin;
diff --git a/clang/include/clang/Basic/BuiltinsHexagon.def b/clang/include/clang/Basic/BuiltinsHexagon.def
deleted file mode 100644
index adff9f884c049..0000000000000
--- a/clang/include/clang/Basic/BuiltinsHexagon.def
+++ /dev/null
@@ -1,173 +0,0 @@
-//===-- BuiltinsHexagon.def - Hexagon Builtin function database --*- C++ -*-==//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-//
-// This file defines the Hexagon-specific builtin function database. Users of
-// this file must define the BUILTIN macro to make use of this information.
-//
-//===----------------------------------------------------------------------===//
-
-// The format of this database matches clang/Basic/Builtins.def.
-
-#if defined(BUILTIN) && !defined(TARGET_BUILTIN)
-#   define TARGET_BUILTIN(ID, TYPE, ATTRS, FEATURE) BUILTIN(ID, TYPE, ATTRS)
-#endif
-
-#pragma push_macro("V79")
-#define V79 "v79"
-#pragma push_macro("V75")
-#define V75 "v75|" V79
-#pragma push_macro("V73")
-#define V73 "v73|" V75
-#pragma push_macro("V71")
-#define V71 "v71|" V73
-#pragma push_macro("V69")
-#define V69 "v69|" V71
-#pragma push_macro("V68")
-#define V68 "v68|" V69
-#pragma push_macro("V67")
-#define V67 "v67|" V68
-#pragma push_macro("V66")
-#define V66 "v66|" V67
-#pragma push_macro("V65")
-#define V65 "v65|" V66
-#pragma push_macro("V62")
-#define V62 "v62|" V65
-#pragma push_macro("V60")
-#define V60 "v60|" V62
-#pragma push_macro("V55")
-#define V55 "v55|" V60
-#pragma push_macro("V5")
-#define V5 "v5|" V55
-
-#pragma push_macro("HVXV79")
-#define HVXV79 "hvxv79"
-#pragma push_macro("HVXV75")
-#define HVXV75 "hvxv75|" HVXV79
-#pragma push_macro("HVXV73")
-#define HVXV73 "hvxv73|" HVXV75
-#pragma push_macro("HVXV71")
-#define HVXV71 "hvxv71|" HVXV73
-#pragma push_macro("HVXV69")
-#define HVXV69 "hvxv69|" HVXV71
-#pragma push_macro("HVXV68")
-#define HVXV68 "hvxv68|" HVXV69
-#pragma push_macro("HVXV67")
-#define HVXV67 "hvxv67|" HVXV68
-#pragma push_macro("HVXV66")
-#define HVXV66 "hvxv66|" HVXV67
-#pragma push_macro("HVXV65")
-#define HVXV65 "hvxv65|" HVXV66
-#pragma push_macro("HVXV62")
-#define HVXV62 "hvxv62|" HVXV65
-#pragma push_macro("HVXV60")
-#define HVXV60 "hvxv60|" HVXV62
-
-
-// The builtins below are not autogenerated from iset.py.
-// Make sure you do not overwrite these.
-TARGET_BUILTIN(__builtin_SI_to_SXTHI_asrh, "ii", "", V5)
-TARGET_BUILTIN(__builtin_brev_ldd,   "v*LLi*CLLi*iC", "", V5)
-TARGET_BUILTIN(__builtin_brev_ldw,   "v*i*Ci*iC", "", V5)
-TARGET_BUILTIN(__builtin_brev_ldh,   "v*s*Cs*iC", "", V5)
-TARGET_BUILTIN(__builtin_brev_lduh,  "v*Us*CUs*iC", "", V5)
-TARGET_BUILTIN(__builtin_brev_ldb,   "v*Sc*CSc*iC", "", V5)
-TARGET_BUILTIN(__builtin_brev_ldub,  "v*Uc*CUc*iC", "", V5)
-TARGET_BUILTIN(__builtin_circ_ldd,   "LLi*LLi*LLi*iIi", "", V5)
-TARGET_BUILTIN(__builtin_circ_ldw,   "i*i*i*iIi", "", V5)
-TARGET_BUILTIN(__builtin_circ_ldh,   "s*s*s*iIi", "", V5)
-TARGET_BUILTIN(__builtin_circ_lduh,  "Us*Us*Us*iIi", "", V5)
-TARGET_BUILTIN(__builtin_circ_ldb,   "c*c*c*iIi", "", V5)
-TARGET_BUILTIN(__builtin_circ_ldub,  "Uc*Uc*Uc*iIi", "", V5)
-TARGET_BUILTIN(__builtin_brev_std,   "LLi*CLLi*LLiiC", "", V5)
-TARGET_BUILTIN(__builtin_brev_stw,   "i*Ci*iiC", "", V5)
-TARGET_BUILTIN(__builtin_brev_sth,   "s*Cs*iiC", "", V5)
-TARGET_BUILTIN(__builtin_brev_sthhi, "s*Cs*iiC", "", V5)
-TARGET_BUILTIN(__builtin_brev_stb,   "c*Cc*iiC", "", V5)
-TARGET_BUILTIN(__builtin_circ_std,   "LLi*LLi*LLiiIi", "", V5)
-TARGET_BUILTIN(__builtin_circ_stw,   "i*i*iiIi", "", V5)
-TARGET_BUILTIN(__builtin_circ_sth,   "s*s*iiIi", "", V5)
-TARGET_BUILTIN(__builtin_circ_sthhi, "s*s*iiIi", "", V5)
-TARGET_BUILTIN(__builtin_circ_stb,   "c*c*iiIi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_L2_loadrub_pci, "iv*IiivC*", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_L2_loadrb_pci, "iv*IiivC*", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_L2_loadruh_pci, "iv*IiivC*", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_L2_loadrh_pci, "iv*IiivC*", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_L2_loadri_pci, "iv*IiivC*", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_L2_loadrd_pci, "LLiv*IiivC*", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_L2_loadrub_pcr, "iv*ivC*", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_L2_loadrb_pcr, "iv*ivC*", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_L2_loadruh_pcr, "iv*ivC*", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_L2_loadrh_pcr, "iv*ivC*", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_L2_loadri_pcr, "iv*ivC*", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_L2_loadrd_pcr, "LLiv*ivC*", "", V5)
-
-TARGET_BUILTIN(__builtin_HEXAGON_S2_storerb_pci, "vv*IiiivC*", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_S2_storerh_pci, "vv*IiiivC*", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_S2_storerf_pci, "vv*IiiivC*", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_S2_storeri_pci, "vv*IiiivC*", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_S2_storerd_pci, "vv*IiiLLivC*", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_S2_storerb_pcr, "vv*iivC*", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_S2_storerh_pcr, "vv*iivC*", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_S2_storerf_pcr, "vv*iivC*", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_S2_storeri_pcr, "vv*iivC*", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_S2_storerd_pcr, "vv*iLLivC*", "", V5)
-
-TARGET_BUILTIN(__builtin_HEXAGON_prefetch,"vv*","", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_A6_vminub_RdP,"LLiLLiLLi","", V62)
-
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vmaskedstoreq,"vV64bv*V16i","", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vmaskedstorenq,"vV64bv*V16i","", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vmaskedstorentq,"vV64bv*V16i","", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vmaskedstorentnq,"vV64bv*V16i","", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vmaskedstoreq_128B,"vV128bv*V32i","", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vmaskedstorenq_128B,"vV128bv*V32i","", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vmaskedstorentq_128B,"vV128bv*V32i","", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vmaskedstorentnq_128B,"vV128bv*V32i","", HVXV60)
-
-
-// These are only valid on v65
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vrmpybub_rtt,"V32iV16iLLi","", "hvxv65")
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vrmpybub_rtt_128B,"V64iV32iLLi","", "hvxv65")
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vrmpybub_rtt_acc,"V32iV32iV16iLLi","", "hvxv65")
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vrmpybub_rtt_acc_128B,"V64iV64iV32iLLi","", "hvxv65")
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vrmpyub_rtt,"V32iV16iLLi","", "hvxv65")
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vrmpyub_rtt_128B,"V64iV32iLLi","", "hvxv65")
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vrmpyub_rtt_acc,"V32iV32iV16iLLi","", "hvxv65")
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vrmpyub_rtt_acc_128B,"V64iV64iV32iLLi","", "hvxv65")
-
-#include "clang/Basic/BuiltinsHexagonDep.def"
-
-#pragma pop_macro("HVXV60")
-#pragma pop_macro("HVXV62")
-#pragma pop_macro("HVXV65")
-#pragma pop_macro("HVXV66")
-#pragma pop_macro("HVXV67")
-#pragma pop_macro("HVXV68")
-#pragma pop_macro("HVXV69")
-#pragma pop_macro("HVXV71")
-#pragma pop_macro("HVXV73")
-#pragma pop_macro("HVXV75")
-#pragma pop_macro("HVXV79")
-
-#pragma pop_macro("V5")
-#pragma pop_macro("V55")
-#pragma pop_macro("V60")
-#pragma pop_macro("V62")
-#pragma pop_macro("V65")
-#pragma pop_macro("V66")
-#pragma pop_macro("V67")
-#pragma pop_macro("V68")
-#pragma pop_macro("V69")
-#pragma pop_macro("V71")
-#pragma pop_macro("V73")
-#pragma pop_macro("V75")
-#pragma pop_macro("V79")
-
-#undef BUILTIN
-#undef TARGET_BUILTIN
-
diff --git a/clang/include/clang/Basic/BuiltinsHexagon.td b/clang/include/clang/Basic/BuiltinsHexagon.td
new file mode 100644
index 0000000000000..0727c67346697
--- /dev/null
+++ b/clang/include/clang/Basic/BuiltinsHexagon.td
@@ -0,0 +1,2146 @@
+//===--- BuiltinsHexagon.td - Hexagon Builtin function defs -----*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file defines the Hexagon-specific builtin function database.
+//
+//===----------------------------------------------------------------------===//
+
+include "clang/Basic/BuiltinsBase.td"
+
+class VFeatures {
+  string Features;
+}
+
+class V<string version, VFeatures newer> : VFeatures {
+  let Features = !strconcat("v", version, "|", newer.Features);
+}
+
+let Features = "v79" in def V79 : VFeatures;
+
+def V75 : V<"75", V79>;
+def V73 : V<"73", V75>;
+def V71 : V<"71", V73>;
+def V69 : V<"69", V71>;
+def V68 : V<"68", V69>;
+def V67 : V<"67", V68>;
+def V66 : V<"66", V67>;
+def V65 : V<"65", V66>;
+def V62 : V<"62", V65>;
+def V60 : V<"60", V62>;
+def V55 : V<"55", V60>;
+def V5  : V<"5", V55>;
+
+class HVXVFeatures {
+  string Features;
+}
+
+class HVXV<string version, HVXVFeatures newer> : HVXVFeatures {
+  let Features = !strconcat("hvxv", version, "|", newer.Features);
+}
+
+let Features = "hvxv79" in def HVXV79 : HVXVFeatures;
+
+def HVXV75 : HVXV<"75", HVXV79>;
+def HVXV73 : HVXV<"73", HVXV75>;
+def HVXV71 : HVXV<"71", HVXV73>;
+def HVXV69 : HVXV<"69", HVXV71>;
+def HVXV68 : HVXV<"68", HVXV69>;
+def HVXV67 : HVXV<"67", HVXV68>;
+def HVXV66 : HVXV<"66", HVXV67>;
+def HVXV65 : HVXV<"65", HVXV66>;
+def HVXV62 : HVXV<"62", HVXV65>;
+def HVXV60 : HVXV<"60", HVXV62>;
+
+def HexagonPrefix : NamePrefix<"__builtin_HEXAGON_">;
+
+class HexagonBuiltin<string prototype> : TargetBuiltin {
+  let Spellings = [NAME];
+  let Prototype = prototype;
+  let Features = V5.Features;
+  let RequiredNamePrefix = HexagonPrefix; // Adds a prefix to the name.
+}
+
+class HexagonBuiltinNoPrefix<string prototype> : TargetBuiltin {
+  let Spellings = [NAME];
+  let Prototype = prototype;
+  let Features = V5.Features;
+}
+
+// The builtins below are not autogenerated from iset.py.
+// Make sure you do not overwrite these.
+def __builtin_SI_to_SXTHI_asrh : HexagonBuiltinNoPrefix<"int(int)">;
+def __builtin_brev_ldd : HexagonBuiltinNoPrefix<"void *(long long int * const, long long int *, int const)">;
+def __builtin_brev_ldw : HexagonBuiltinNoPrefix<"void *(int * const, int *, int const)">;
+def __builtin_brev_ldh : HexagonBuiltinNoPrefix<"void *(short * const, short *, int const)">;
+def __builtin_brev_lduh : HexagonBuiltinNoPrefix<"void *(unsigned short * const, unsigned short *, int const)">;
+def __builtin_brev_ldb : HexagonBuiltinNoPrefix<"void *(signed char * const, signed char *, int const)">;
+def __builtin_brev_ldub : HexagonBuiltinNoPrefix<"void *(unsigned char * const, unsigned char *, int const)">;
+def __builtin_circ_ldd : HexagonBuiltinNoPrefix<"long long int *(long long int *, long long int *, int, _Constant int)">;
+def __builtin_circ_ldw : HexagonBuiltinNoPrefix<"int *(int *, int *, int, _Constant int)">;
+def __builtin_circ_ldh : HexagonBuiltinNoPrefix<"short *(short *, short *, int, _Constant int)">;
+def __builtin_circ_lduh : HexagonBuiltinNoPrefix<"unsigned short *(unsigned short *, unsigned short *, int, _Constant int)">;
+def __builtin_circ_ldb : HexagonBuiltinNoPrefix<"char *(char *, char *, int, _Constant int)">;
+def __builtin_circ_ldub : HexagonBuiltinNoPrefix<"unsigned char *(unsigned char *, unsigned char *, int, _Constant int)">;
+def __builtin_brev_std : HexagonBuiltinNoPrefix<"long long int * const(long long int *, long long int, int const)">;
+def __builtin_brev_stw : HexagonBuiltinNoPrefix<"int * const(int *, int, int const)">;
+def __builtin_brev_sth : HexagonBuiltinNoPrefix<"short * const(short *, int, int const)">;
+def __builtin_brev_sthhi : HexagonBuiltinNoPrefix<"short * const(short *, int, int const)">;
+def __builtin_brev_stb : HexagonBuiltinNoPrefix<"char * const(char *, int, int const)">;
+def __builtin_circ_std : HexagonBuiltinNoPrefix<"long long int *(long long int *, long long int, int, _Constant int)">;
+def __builtin_circ_stw : HexagonBuiltinNoPrefix<"int *(int *, int, int, _Constant int)">;
+def __builtin_circ_sth : HexagonBuiltinNoPrefix<"short *(short *, int, int, _Constant int)">;
+def __builtin_circ_sthhi : HexagonBuiltinNoPrefix<"short *(short *, int, int, _Constant int)">;
+def __builtin_circ_stb : HexagonBuiltinNoPrefix<"char *(char *, int, int, _Constant int)">;
+def L2_loadrub_pci : HexagonBuiltin<"int(void *, _Constant int, int, void const *)">;
+def L2_loadrb_pci : HexagonBuiltin<"int(void *, _Constant int, int, void const *)">;
+def L2_loadruh_pci : HexagonBuiltin<"int(void *, _Constant int, int, void const *)">;
+def L2_loadrh_pci : HexagonBuiltin<"int(void *, _Constant int, int, void const *)">;
+def L2_loadri_pci : HexagonBuiltin<"int(void *, _Constant int, int, void const *)">;
+def L2_loadrd_pci : HexagonBuiltin<"long long int(void *, _Constant int, int, void const *)">;
+def L2_loadrub_pcr : HexagonBuiltin<"int(void *, int, void const *)">;
+def L2_loadrb_pcr : HexagonBuiltin<"int(void *, int, void const *)">;
+def L2_loadruh_pcr : HexagonBuiltin<"int(void *, int, void const *)">;
+def L2_loadrh_pcr : HexagonBuiltin<"int(void *, int, void const *)">;
+def L2_loadri_pcr : HexagonBuiltin<"int(void *, int, void const *)">;
+def L2_loadrd_pcr : HexagonBuiltin<"long long int(void *, int, void const *)">;
+
+def S2_storerb_pci : HexagonBuiltin<"void(void *, _Constant int, int, int, void const *)">;
+def S2_storerh_pci : HexagonBuiltin<"void(void *, _Constant int, int, int, void const *)">;
+def S2_storerf_pci : HexagonBuiltin<"void(void *, _Constant int, int, int, void const *)">;
+def S2_storeri_pci : HexagonBuiltin<"void(void *, _Constant int, int, int, void const *)">;
+def S2_storerd_pci : HexagonBuiltin<"void(void *, _Constant int, int, long long int, void const *)">;
+def S2_storerb_pcr : HexagonBuiltin<"void(void *, int, int, void const *)">;
+def S2_storerh_pcr : HexagonBuiltin<"void(void *, int, int, void const *)">;
+def S2_storerf_pcr : HexagonBuiltin<"void(void *, int, int, void const *)">;
+def S2_storeri_pcr : HexagonBuiltin<"void(void *, int, int, void const *)">;
+def S2_storerd_pcr : HexagonBuiltin<"void(void *, int, long long int, void const *)">;
+
+def prefetch : HexagonBuiltin<"void(void *)">;
+let Features = V62.Features in {
+  def A6_vminub_RdP : HexagonBuiltin<"long long int(long long int, long long int)">;
+}
+
+let Features = HVXV60.Features in {
+  def V6_vmaskedstoreq : HexagonBuiltin<"void(_Vector<64, bool>, void *, _Vector<16, int>)">;
+  def V6_vmaskedstorenq : HexagonBuiltin<"void(_Vector<64, bool>, void *, _Vector<16, int>)">;
+  def V6_vmaskedstorentq : HexagonBuiltin<"void(_Vector<64, bool>, void *, _Vector<16, int>)">;
+  def V6_vmaskedstorentnq : HexagonBuiltin<"void(_Vector<64, bool>, void *, _Vector<16, int>)">;
+  def V6_vmaskedstoreq_128B : HexagonBuiltin<"void(_Vector<128, bool>, void *, _Vector<32, int>)">;
+  def V6_vmaskedstorenq_128B : HexagonBuiltin<"void(_Vector<128, bool>, void *, _Vector<32, int>)">;
+  def V6_vmaskedstorentq_128B : HexagonBuiltin<"void(_Vector<128, bool>, void *, _Vector<32, int>)">;
+  def V6_vmaskedstorentnq_128B : HexagonBuiltin<"void(_Vector<128, bool>, void *, _Vector<32, int>)">;
+}
+
+
+// These are only valid on v65
+let Features = "hvxv65" in {
+  def V6_vrmpybub_rtt : HexagonBuiltin<"_Vector<32, int>(_Vector<16, int>, long long int)">;
+  def V6_vrmpybub_rtt_128B : HexagonBuiltin<"_Vector<64, int>(_Vector<32, int>, long long int)">;
+  def V6_vrmpybub_rtt_acc : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<16, int>, long long int)">;
+  def V6_vrmpybub_rtt_acc_128B : HexagonBuiltin<"_Vector<64, int>(_Vector<64, int>, _Vector<32, int>, long long int)">;
+  def V6_vrmpyub_rtt : HexagonBuiltin<"_Vector<32, int>(_Vector<16, int>, long long int)">;
+  def V6_vrmpyub_rtt_128B : HexagonBuiltin<"_Vector<64, int>(_Vector<32, int>, long long int)">;
+  def V6_vrmpyub_rtt_acc : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<16, int>, long long int)">;
+  def V6_vrmpyub_rtt_acc_128B : HexagonBuiltin<"_Vector<64, int>(_Vector<64, int>, _Vector<32, int>, long long int)">;
+}
+
+// V5 Scalar Instructions.
+
+def A2_abs : HexagonBuiltin<"int(int)">;
+def A2_absp : HexagonBuiltin<"long long int(long long int)">;
+def A2_abssat : HexagonBuiltin<"int(int)">;
+def A2_add : HexagonBuiltin<"int(int, int)">;
+def A2_addh_h16_hh : HexagonBuiltin<"int(int, int)">;
+def A2_addh_h16_hl : HexagonBuiltin<"int(int, int)">;
+def A2_addh_h16_lh : HexagonBuiltin<"int(int, int)">;
+def A2_addh_h16_ll : HexagonBuiltin<"int(int, int)">;
+def A2_addh_h16_sat_hh : HexagonBuiltin<"int(int, int)">;
+def A2_addh_h16_sat_hl : HexagonBuiltin<"int(int, int)">;
+def A2_addh_h16_sat_lh : HexagonBuiltin<"int(int, int)">;
+def A2_addh_h16_sat_ll : HexagonBuiltin<"int(int, int)">;
+def A2_addh_l16_hl : HexagonBuiltin<"int(int, int)">;
+def A2_addh_l16_ll : HexagonBuiltin<"int(int, int)">;
+def A2_addh_l16_sat_hl : HexagonBuiltin<"int(int, int)">;
+def A2_addh_l16_sat_ll : HexagonBuiltin<"int(int, int)">;
+def A2_addi : HexagonBuiltin<"int(int, _Constant int)">;
+def A2_addp : HexagonBuiltin<"long long int(long long int, long long int)">;
+def A2_addpsat : HexagonBuiltin<"long long int(long long int, long long int)">;
+def A2_addsat : HexagonBuiltin<"int(int, int)">;
+def A2_addsp : HexagonBuiltin<"long long int(int, long long int)">;
+def A2_and : HexagonBuiltin<"int(int, int)">;
+def A2_andir : HexagonBuiltin<"int(int, _Constant int)">;
+def A2_andp : HexagonBuiltin<"long long int(long long int, long long int)">;
+def A2_aslh : HexagonBuiltin<"int(int)">;
+def A2_asrh : HexagonBuiltin<"int(int)">;
+def A2_combine_hh : HexagonBuiltin<"int(int, int)">;
+def A2_combine_hl : HexagonBuiltin<"int(int, int)">;
+def A2_combine_lh : HexagonBuiltin<"int(int, int)">;
+def A2_combine_ll : HexagonBuiltin<"int(int, int)">;
+def A2_combineii : HexagonBuiltin<"long long int(_Constant int, _Constant int)">;
+def A2_combinew : HexagonBuiltin<"long long int(int, int)">;
+def A2_max : HexagonBuiltin<"int(int, int)">;
+def A2_maxp : HexagonBuiltin<"long long int(long long int, long long int)">;
+def A2_maxu : HexagonBuiltin<"unsigned int(int, int)">;
+def A2_maxup : HexagonBuiltin<"unsigned long long int(long long int, long long int)">;
+def A2_min : HexagonBuiltin<"int(int, int)">;
+def A2_minp : HexagonBuiltin<"long long int(long long int, long long int)">;
+def A2_minu : HexagonBuiltin<"unsigned int(int, int)">;
+def A2_minup : HexagonBuiltin<"unsigned long long int(long long int, long long int)">;
+def A2_neg : HexagonBuiltin<"int(int)">;
+def A2_negp : HexagonBuiltin<"long long int(long long int)">;
+def A2_negsat : HexagonBuiltin<"int(int)">;
+def A2_not : HexagonBuiltin<"int(int)">;
+def A2_notp : HexagonBuiltin<"long long int(long long int)">;
+def A2_or : HexagonBuiltin<"int(int, int)">;
+def A2_orir : HexagonBuiltin<"int(int, _Constant int)">;
+def A2_orp : HexagonBuiltin<"long long int(long long int, long long int)">;
+def A2_roundsat : HexagonBuiltin<"int(long long int)">;
+def A2_sat : HexagonBuiltin<"int(long long int)">;
+def A2_satb : HexagonBuiltin<"int(int)">;
+def A2_sath : HexagonBuiltin<"int(int)">;
+def A2_satub : HexagonBuiltin<"int(int)">;
+def A2_satuh : HexagonBuiltin<"int(int)">;
+def A2_sub : HexagonBuiltin<"int(int, int)">;
+def A2_subh_h16_hh : HexagonBuiltin<"int(int, int)">;
+def A2_subh_h16_hl : HexagonBuiltin<"int(int, int)">;
+def A2_subh_h16_lh : HexagonBuiltin<"int(int, int)">;
+def A2_subh_h16_ll : HexagonBuiltin<"int(int, int)">;
+def A2_subh_h16_sat_hh : HexagonBuiltin<"int(int, int)">;
+def A2_subh_h16_sat_hl : HexagonBuiltin<"int(int, int)">;
+def A2_subh_h16_sat_lh : HexagonBuiltin<"int(int, int)">;
+def A2_subh_h16_sat_ll : HexagonBuiltin<"int(int, int)">;
+def A2_subh_l16_hl : HexagonBuiltin<"int(int, int)">;
+def A2_subh_l16_ll : HexagonBuiltin<"int(int, int)">;
+def A2_subh_l16_sat_hl : HexagonBuiltin<"int(int, int)">;
+def A2_subh_l16_sat_ll : HexagonBuiltin<"int(int, int)">;
+def A2_subp : HexagonBuiltin<"long long int(long long int, long long int)">;
+def A2_subri : HexagonBuiltin<"int(_Constant int, int)">;
+def A2_subsat : HexagonBuiltin<"int(int, int)">;
+def A2_svaddh : HexagonBuiltin<"int(int, int)">;
+def A2_svaddhs : HexagonBuiltin<"int(int, int)">;
+def A2_svadduhs : HexagonBuiltin<"int(int, int)">;
+def A2_svavgh : HexagonBuiltin<"int(int, int)">;
+def A2_svavghs : HexagonBuiltin<"int(int, int)">;
+def A2_svnavgh : HexagonBuiltin<"int(int, int)">;
+def A2_svsubh : HexagonBuiltin<"int(int, int)">;
+def A2_svsubhs : HexagonBuiltin<"int(int, int)">;
+def A2_svsubuhs : HexagonBuiltin<"int(int, int)">;
+def A2_swiz : HexagonBuiltin<"int(int)">;
+def A2_sxtb : HexagonBuiltin<"int(int)">;
+def A2_sxth : HexagonBuiltin<"int(int)">;
+def A2_sxtw : HexagonBuiltin<"long long int(int)">;
+def A2_tfr : HexagonBuiltin<"int(int)">;
+def A2_tfrih : HexagonBuiltin<"int(int, unsigned _Constant int)">;
+def A2_tfril : HexagonBuiltin<"int(int, unsigned _Constant int)">;
+def A2_tfrp : HexagonBuiltin<"long long int(long long int)">;
+def A2_tfrpi : HexagonBuiltin<"long long int(_Constant int)">;
+def A2_tfrsi : HexagonBuiltin<"int(_Constant int)">;
+def A2_vabsh : HexagonBuiltin<"long long int(long long int)">;
+def A2_vabshsat : HexagonBuiltin<"long long int(long long int)">;
+def A2_vabsw : HexagonBuiltin<"long long int(long long int)">;
+def A2_vabswsat : HexagonBuiltin<"long long int(long long int)">;
+def A2_vaddb_map : HexagonBuiltin<"long long int(long long int, long long int)">;
+def A2_vaddh : HexagonBuiltin<"long long int(long long int, long long int)">;
+def A2_vaddhs : HexagonBuiltin<"long long int(long long int, long long int)">;
+def A2_vaddub : HexagonBuiltin<"long long int(long long int, long long int)">;
+def A2_vaddubs : HexagonBuiltin<"long long int(long long int, long long int)">;
+def A2_vadduhs : HexagonBuiltin<"long long int(long long int, long long int)">;
+def A2_vaddw : HexagonBuiltin<"long long int(long long int, long long int)">;
+def A2_vaddws : HexagonBuiltin<"long long int(long long int, long long int)">;
+def A2_vavgh : HexagonBuiltin<"long long int(long long int, long long int)">;
+def A2_vavghcr : HexagonBuiltin<"long long int(long long int, long long int)">;
+def A2_vavghr : HexagonBuiltin<"long long int(long long int, long long int)">;
+def A2_vavgub : HexagonBuiltin<"long long int(long long int, long long int)">;
+def A2_vavgubr : HexagonBuiltin<"long long int(long long int, long long int)">;
+def A2_vavguh : HexagonBuiltin<"long long int(long long int, long long int)">;
+def A2_vavguhr : HexagonBuiltin<"long long int(long long int, long long int)">;
+def A2_vavguw : HexagonBuiltin<"long long int(long long int, long long int)">;
+def A2_vavguwr : HexagonBuiltin<"long long int(long long int, long long int)">;
+def A2_vavgw : HexagonBuiltin<"long long int(long long int, long long int)">;
+def A2_vavgwcr : HexagonBuiltin<"long long int(long long int, long long int)">;
+def A2_vavgwr : HexagonBuiltin<"long long int(long long int, long long int)">;
+def A2_vcmpbeq : HexagonBuiltin<"int(long long int, long long int)">;
+def A2_vcmpbgtu : HexagonBuiltin<"int(long long int, long long int)">;
+def A2_vcmpheq : HexagonBuiltin<"int(long long int, long long int)">;
+def A2_vcmphgt : HexagonBuiltin<"int(long long int, long long int)">;
+def A2_vcmphgtu : HexagonBuiltin<"int(long long int, long long int)">;
+def A2_vcmpweq : HexagonBuiltin<"int(long long int, long long int)">;
+def A2_vcmpwgt : HexagonBuiltin<"int(long long int, long long int)">;
+def A2_vcmpwgtu : HexagonBuiltin<"int(long long int, long long int)">;
+def A2_vconj : HexagonBuiltin<"long long int(long long int)">;
+def A2_vmaxb : HexagonBuiltin<"long long int(long long int, long long int)">;
+def A2_vmaxh : HexagonBuiltin<"long long int(long long int, long long int)">;
+def A2_vmaxub : HexagonBuiltin<"long long int(long long int, long long int)">;
+def A2_vmaxuh : HexagonBuiltin<"long long int(long long int, long long int)">;
+def A2_vmaxuw : HexagonBuiltin<"long long int(long long int, long long int)">;
+def A2_vmaxw : HexagonBuiltin<"long long int(long long int, long long int)">;
+def A2_vminb : HexagonBuiltin<"long long int(long long int, long long int)">;
+def A2_vminh : HexagonBuiltin<"long long int(long long int, long long int)">;
+def A2_vminub : HexagonBuiltin<"long long int(long long int, long long int)">;
+def A2_vminuh : HexagonBuiltin<"long long int(long long int, long long int)">;
+def A2_vminuw : HexagonBuiltin<"long long int(long long int, long long int)">;
+def A2_vminw : HexagonBuiltin<"long long int(long long int, long long int)">;
+def A2_vnavgh : HexagonBuiltin<"long long int(long long int, long long int)">;
+def A2_vnavghcr : HexagonBuiltin<"long long int(long long int, long long int)">;
+def A2_vnavghr : HexagonBuiltin<"long long int(long long int, long long int)">;
+def A2_vnavgw : HexagonBuiltin<"long long int(long long int, long long int)">;
+def A2_vnavgwcr : HexagonBuiltin<"long long int(long long int, long long int)">;
+def A2_vnavgwr : HexagonBuiltin<"long long int(long long int, long long int)">;
+def A2_vraddub : HexagonBuiltin<"long long int(long long int, long long int)">;
+def A2_vraddub_acc : HexagonBuiltin<"long long int(long long int, long long int, long long int)">;
+def A2_vrsadub : HexagonBuiltin<"long long int(long long int, long long int)">;
+def A2_vrsadub_acc : HexagonBuiltin<"long long int(long long int, long long int, long long int)">;
+def A2_vsubb_map : HexagonBuiltin<"long long int(long long int, long long int)">;
+def A2_vsubh : HexagonBuiltin<"long long int(long long int, long long int)">;
+def A2_vsubhs : HexagonBuiltin<"long long int(long long int, long long int)">;
+def A2_vsubub : HexagonBuiltin<"long long int(long long int, long long int)">;
+def A2_vsububs : HexagonBuiltin<"long long int(long long int, long long int)">;
+def A2_vsubuhs : HexagonBuiltin<"long long int(long long int, long long int)">;
+def A2_vsubw : HexagonBuiltin<"long long int(long long int, long long int)">;
+def A2_vsubws : HexagonBuiltin<"long long int(long long int, long long int)">;
+def A2_xor : HexagonBuiltin<"int(int, int)">;
+def A2_xorp : HexagonBuiltin<"long long int(long long int, long long int)">;
+def A2_zxtb : HexagonBuiltin<"int(int)">;
+def A2_zxth : HexagonBuiltin<"int(int)">;
+def A4_andn : HexagonBuiltin<"int(int, int)">;
+def A4_andnp : HexagonBuiltin<"long long int(long long int, long long int)">;
+def A4_bitsplit : HexagonBuiltin<"long long int(int, int)">;
+def A4_bitspliti : HexagonBuiltin<"long long int(int, unsigned _Constant int)">;
+def A4_boundscheck : HexagonBuiltin<"int(int, long long int)">;
+def A4_cmpbeq : HexagonBuiltin<"int(int, int)">;
+def A4_cmpbeqi : HexagonBuiltin<"int(int, unsigned _Constant int)">;
+def A4_cmpbgt : HexagonBuiltin<"int(int, int)">;
+def A4_cmpbgti : HexagonBuiltin<"int(int, _Constant int)">;
+def A4_cmpbgtu : HexagonBuiltin<"int(int, int)">;
+def A4_cmpbgtui : HexagonBuiltin<"int(int, unsigned _Constant int)">;
+def A4_cmpheq : HexagonBuiltin<"int(int, int)">;
+def A4_cmpheqi : HexagonBuiltin<"int(int, _Constant int)">;
+def A4_cmphgt : HexagonBuiltin<"int(int, int)">;
+def A4_cmphgti : HexagonBuiltin<"int(int, _Constant int)">;
+def A4_cmphgtu : HexagonBuiltin<"int(int, int)">;
+def A4_cmphgtui : HexagonBuiltin<"int(int, unsigned _Constant int)">;
+def A4_combineir : HexagonBuiltin<"long long int(_Constant int, int)">;
+def A4_combineri : HexagonBuiltin<"long long int(int, _Constant int)">;
+def A4_cround_ri : HexagonBuiltin<"int(int, unsigned _Constant int)">;
+def A4_cround_rr : HexagonBuiltin<"int(int, int)">;
+def A4_modwrapu : HexagonBuiltin<"int(int, int)">;
+def A4_orn : HexagonBuiltin<"int(int, int)">;
+def A4_ornp : HexagonBuiltin<"long long int(long long int, long long int)">;
+def A4_rcmpeq : HexagonBuiltin<"int(int, int)">;
+def A4_rcmpeqi : HexagonBuiltin<"int(int, _Constant int)">;
+def A4_rcmpneq : HexagonBuiltin<"int(int, int)">;
+def A4_rcmpneqi : HexagonBuiltin<"int(int, _Constant int)">;
+def A4_round_ri : HexagonBuiltin<"int(int, unsigned _Constant int)">;
+def A4_round_ri_sat : HexagonBuiltin<"int(int, unsigned _Constant int)">;
+def A4_round_rr : HexagonBuiltin<"int(int, int)">;
+def A4_round_rr_sat : HexagonBuiltin<"int(int, int)">;
+def A4_tlbmatch : HexagonBuiltin<"int(long long int, int)">;
+def A4_vcmpbeq_any : HexagonBuiltin<"int(long long int, long long int)">;
+def A4_vcmpbeqi : HexagonBuiltin<"int(long long int, unsigned _Constant int)">;
+def A4_vcmpbgt : HexagonBuiltin<"int(long long int, long long int)">;
+def A4_vcmpbgti : HexagonBuiltin<"int(long long int, _Constant int)">;
+def A4_vcmpbgtui : HexagonBuiltin<"int(long long int, unsigned _Constant int)">;
+def A4_vcmpheqi : HexagonBuiltin<"int(long long int, _Constant int)">;
+def A4_vcmphgti : HexagonBuiltin<"int(long long int, _Constant int)">;
+def A4_vcmphgtui : HexagonBuiltin<"int(long long int, unsigned _Constant int)">;
+def A4_vcmpweqi : HexagonBuiltin<"int(long long int, _Constant int)">;
+def A4_vcmpwgti : HexagonBuiltin<"int(long long int, _Constant int)">;
+def A4_vcmpwgtui : HexagonBuiltin<"int(long long int, unsigned _Constant int)">;
+def A4_vrmaxh : HexagonBuiltin<"long long int(long long int, long long int, int)">;
+def A4_vrmaxuh : HexagonBuiltin<"long long int(long long int, long long int, int)">;
+def A4_vrmaxuw : HexagonBuiltin<"long long int(long long int, long long int, int)">;
+def A4_vrmaxw : HexagonBuiltin<"long long int(long long int, long long int, int)">;
+def A4_vrminh : HexagonBuiltin<"long long int(long long int, long long int, int)">;
+def A4_vrminuh : HexagonBuiltin<"long long int(long long int, long long int, int)">;
+def A4_vrminuw : HexagonBuiltin<"long long int(long long int, long long int, int)">;
+def A4_vrminw : HexagonBuiltin<"long long int(long long int, long long int, int)">;
+def A5_vaddhubs : HexagonBuiltin<"int(long long int, long long int)">;
+def C2_all8 : HexagonBuiltin<"int(int)">;
+def C2_and : HexagonBuiltin<"int(int, int)">;
+def C2_andn : HexagonBuiltin<"int(int, int)">;
+def C2_any8 : HexagonBuiltin<"int(int)">;
+def C2_bitsclr : HexagonBuiltin<"int(int, int)">;
+def C2_bitsclri : HexagonBuiltin<"int(int, unsigned _Constant int)">;
+def C2_bitsset : HexagonBuiltin<"int(int, int)">;
+def C2_cmpeq : HexagonBuiltin<"int(int, int)">;
+def C2_cmpeqi : HexagonBuiltin<"int(int, _Constant int)">;
+def C2_cmpeqp : HexagonBuiltin<"int(long long int, long long int)">;
+def C2_cmpgei : HexagonBuiltin<"int(int, _Constant int)">;
+def C2_cmpgeui : HexagonBuiltin<"int(int, unsigned _Constant int)">;
+def C2_cmpgt : HexagonBuiltin<"int(int, int)">;
+def C2_cmpgti : HexagonBuiltin<"int(int, _Constant int)">;
+def C2_cmpgtp : HexagonBuiltin<"int(long long int, long long int)">;
+def C2_cmpgtu : HexagonBuiltin<"int(int, int)">;
+def C2_cmpgtui : HexagonBuiltin<"int(int, unsigned _Constant int)">;
+def C2_cmpgtup : HexagonBuiltin<"int(long long int, long long int)">;
+def C2_cmplt : HexagonBuiltin<"int(int, int)">;
+def C2_cmpltu : HexagonBuiltin<"int(int, int)">;
+def C2_mask : HexagonBuiltin<"long long int(int)">;
+def C2_mux : HexagonBuiltin<"int(int, int, int)">;
+def C2_muxii : HexagonBuiltin<"int(int, _Constant int, _Constant int)">;
+def C2_muxir : HexagonBuiltin<"int(int, int, _Constant int)">;
+def C2_muxri : HexagonBuiltin<"int(int, _Constant int, int)">;
+def C2_not : HexagonBuiltin<"int(int)">;
+def C2_or : HexagonBuiltin<"int(int, int)">;
+def C2_orn : HexagonBuiltin<"int(int, int)">;
+def C2_pxfer_map : HexagonBuiltin<"int(int)">;
+def C2_tfrpr : HexagonBuiltin<"int(int)">;
+def C2_tfrrp : HexagonBuiltin<"int(int)">;
+def C2_vitpack : HexagonBuiltin<"int(int, int)">;
+def C2_vmux : HexagonBuiltin<"long long int(int, long long int, long long int)">;
+def C2_xor : HexagonBuiltin<"int(int, int)">;
+def C4_and_and : HexagonBuiltin<"int(int, int, int)">;
+def C4_and_andn : HexagonBuiltin<"int(int, int, int)">;
+def C4_and_or : HexagonBuiltin<"int(int, int, int)">;
+def C4_and_orn : HexagonBuiltin<"int(int, int, int)">;
+def C4_cmplte : HexagonBuiltin<"int(int, int)">;
+def C4_cmpltei : HexagonBuiltin<"int(int, _Constant int)">;
+def C4_cmplteu : HexagonBuiltin<"int(int, int)">;
+def C4_cmplteui : HexagonBuiltin<"int(int, unsigned _Constant int)">;
+def C4_cmpneq : HexagonBuiltin<"int(int, int)">;
+def C4_cmpneqi : HexagonBuiltin<"int(int, _Constant int)">;
+def C4_fastcorner9 : HexagonBuiltin<"int(int, int)">;
+def C4_fastcorner9_not : HexagonBuiltin<"int(int, int)">;
+def C4_nbitsclr : HexagonBuiltin<"int(int, int)">;
+def C4_nbitsclri : HexagonBuiltin<"int(int, unsigned _Constant int)">;
+def C4_nbitsset : HexagonBuiltin<"int(int, int)">;
+def C4_or_and : HexagonBuiltin<"int(int, int, int)">;
+def C4_or_andn : HexagonBuiltin<"int(int, int, int)">;
+def C4_or_or : HexagonBuiltin<"int(int, int, int)">;
+def C4_or_orn : HexagonBuiltin<"int(int, int, int)">;
+def F2_conv_d2df : HexagonBuiltin<"double(long long int)">;
+def F2_conv_d2sf : HexagonBuiltin<"float(long long int)">;
+def F2_conv_df2d : HexagonBuiltin<"long long int(double)">;
+def F2_conv_df2d_chop : HexagonBuiltin<"long long int(double)">;
+def F2_conv_df2sf : HexagonBuiltin<"float(double)">;
+def F2_conv_df2ud : HexagonBuiltin<"long long int(double)">;
+def F2_conv_df2ud_chop : HexagonBuiltin<"long long int(double)">;
+def F2_conv_df2uw : HexagonBuiltin<"int(double)">;
+def F2_conv_df2uw_chop : HexagonBuiltin<"int(double)">;
+def F2_conv_df2w : HexagonBuiltin<"int(double)">;
+def F2_conv_df2w_chop : HexagonBuiltin<"int(double)">;
+def F2_conv_sf2d : HexagonBuiltin<"long long int(float)">;
+def F2_conv_sf2d_chop : HexagonBuiltin<"long long int(float)">;
+def F2_conv_sf2df : HexagonBuiltin<"double(float)">;
+def F2_conv_sf2ud : HexagonBuiltin<"long long int(float)">;
+def F2_conv_sf2ud_chop : HexagonBuiltin<"long long int(float)">;
+def F2_conv_sf2uw : HexagonBuiltin<"int(float)">;
+def F2_conv_sf2uw_chop : HexagonBuiltin<"int(float)">;
+def F2_conv_sf2w : HexagonBuiltin<"int(float)">;
+def F2_conv_sf2w_chop : HexagonBuiltin<"int(float)">;
+def F2_conv_ud2df : HexagonBuiltin<"double(long long int)">;
+def F2_conv_ud2sf : HexagonBuiltin<"float(long long int)">;
+def F2_conv_uw2df : HexagonBuiltin<"double(int)">;
+def F2_conv_uw2sf : HexagonBuiltin<"float(int)">;
+def F2_conv_w2df : HexagonBuiltin<"double(int)">;
+def F2_conv_w2sf : HexagonBuiltin<"float(int)">;
+def F2_dfclass : HexagonBuiltin<"int(double, unsigned _Constant int)">;
+def F2_dfcmpeq : HexagonBuiltin<"int(double, double)">;
+def F2_dfcmpge : HexagonBuiltin<"int(double, double)">;
+def F2_dfcmpgt : HexagonBuiltin<"int(double, double)">;
+def F2_dfcmpuo : HexagonBuiltin<"int(double, double)">;
+def F2_dfimm_n : HexagonBuiltin<"double(unsigned _Constant int)">;
+def F2_dfimm_p : HexagonBuiltin<"double(unsigned _Constant int)">;
+def F2_sfadd : HexagonBuiltin<"float(float, float)">;
+def F2_sfclass : HexagonBuiltin<"int(float, unsigned _Constant int)">;
+def F2_sfcmpeq : HexagonBuiltin<"int(float, float)">;
+def F2_sfcmpge : HexagonBuiltin<"int(float, float)">;
+def F2_sfcmpgt : HexagonBuiltin<"int(float, float)">;
+def F2_sfcmpuo : HexagonBuiltin<"int(float, float)">;
+def F2_sffixupd : HexagonBuiltin<"float(float, float)">;
+def F2_sffixupn : HexagonBuiltin<"float(float, float)">;
+def F2_sffixupr : HexagonBuiltin<"float(float)">;
+def F2_sffma : HexagonBuiltin<"float(float, float, float)">;
+def F2_sffma_lib : HexagonBuiltin<"float(float, float, float)">;
+def F2_sffma_sc : HexagonBuiltin<"float(float, float, float, int)">;
+def F2_sffms : HexagonBuiltin<"float(float, float, float)">;
+def F2_sffms_lib : HexagonBuiltin<"float(float, float, float)">;
+def F2_sfimm_n : HexagonBuiltin<"float(unsigned _Constant int)">;
+def F2_sfimm_p : HexagonBuiltin<"float(unsigned _Constant int)">;
+def F2_sfmax : HexagonBuiltin<"float(float, float)">;
+def F2_sfmin : HexagonBuiltin<"float(float, float)">;
+def F2_sfmpy : HexagonBuiltin<"float(float, float)">;
+def F2_sfsub : HexagonBuiltin<"float(float, float)">;
+def M2_acci : HexagonBuiltin<"int(int, int, int)">;
+def M2_accii : HexagonBuiltin<"int(int, int, _Constant int)">;
+def M2_cmaci_s0 : HexagonBuiltin<"long long int(long long int, int, int)">;
+def M2_cmacr_s0 : HexagonBuiltin<"long long int(long long int, int, int)">;
+def M2_cmacs_s0 : HexagonBuiltin<"long long int(long long int, int, int)">;
+def M2_cmacs_s1 : HexagonBuiltin<"long long int(long long int, int, int)">;
+def M2_cmacsc_s0 : HexagonBuiltin<"long long int(long long int, int, int)">;
+def M2_cmacsc_s1 : HexagonBuiltin<"long long int(long long int, int, int)">;
+def M2_cmpyi_s0 : HexagonBuiltin<"long long int(int, int)">;
+def M2_cmpyr_s0 : HexagonBuiltin<"long long int(int, int)">;
+def M2_cmpyrs_s0 : HexagonBuiltin<"int(int, int)">;
+def M2_cmpyrs_s1 : HexagonBuiltin<"int(int, int)">;
+def M2_cmpyrsc_s0 : HexagonBuiltin<"int(int, int)">;
+def M2_cmpyrsc_s1 : HexagonBuiltin<"int(int, int)">;
+def M2_cmpys_s0 : HexagonBuiltin<"long long int(int, int)">;
+def M2_cmpys_s1 : HexagonBuiltin<"long long int(int, int)">;
+def M2_cmpysc_s0 : HexagonBuiltin<"long long int(int, int)">;
+def M2_cmpysc_s1 : HexagonBuiltin<"long long int(int, int)">;
+def M2_cnacs_s0 : HexagonBuiltin<"long long int(long long int, int, int)">;
+def M2_cnacs_s1 : HexagonBuiltin<"long long int(long long int, int, int)">;
+def M2_cnacsc_s0 : HexagonBuiltin<"long long int(long long int, int, int)">;
+def M2_cnacsc_s1 : HexagonBuiltin<"long long int(long long int, int, int)">;
+def M2_dpmpyss_acc_s0 : HexagonBuiltin<"long long int(long long int, int, int)">;
+def M2_dpmpyss_nac_s0 : HexagonBuiltin<"long long int(long long int, int, int)">;
+def M2_dpmpyss_rnd_s0 : HexagonBuiltin<"int(int, int)">;
+def M2_dpmpyss_s0 : HexagonBuiltin<"long long int(int, int)">;
+def M2_dpmpyuu_acc_s0 : HexagonBuiltin<"long long int(long long int, int, int)">;
+def M2_dpmpyuu_nac_s0 : HexagonBuiltin<"long long int(long long int, int, int)">;
+def M2_dpmpyuu_s0 : HexagonBuiltin<"unsigned long long int(int, int)">;
+def M2_hmmpyh_rs1 : HexagonBuiltin<"int(int, int)">;
+def M2_hmmpyh_s1 : HexagonBuiltin<"int(int, int)">;
+def M2_hmmpyl_rs1 : HexagonBuiltin<"int(int, int)">;
+def M2_hmmpyl_s1 : HexagonBuiltin<"int(int, int)">;
+def M2_maci : HexagonBuiltin<"int(int, int, int)">;
+def M2_macsin : HexagonBuiltin<"int(int, int, unsigned _Constant int)">;
+def M2_macsip : HexagonBuiltin<"int(int, int, unsigned _Constant int)">;
+def M2_mmachs_rs0 : HexagonBuiltin<"long long int(long long int, long long int, long long int)">;
+def M2_mmachs_rs1 : HexagonBuiltin<"long long int(long long int, long long int, long long int)">;
+def M2_mmachs_s0 : HexagonBuiltin<"long long int(long long int, long long int, long long int)">;
+def M2_mmachs_s1 : HexagonBuiltin<"long long int(long long int, long long int, long long int)">;
+def M2_mmacls_rs0 : HexagonBuiltin<"long long int(long long int, long long int, long long int)">;
+def M2_mmacls_rs1 : HexagonBuiltin<"long long int(long long int, long long int, long long int)">;
+def M2_mmacls_s0 : HexagonBuiltin<"long long int(long long int, long long int, long long int)">;
+def M2_mmacls_s1 : HexagonBuiltin<"long long int(long long int, long long int, long long int)">;
+def M2_mmacuhs_rs0 : HexagonBuiltin<"long long int(long long int, long long int, long long int)">;
+def M2_mmacuhs_rs1 : HexagonBuiltin<"long long int(long long int, long long int, long long int)">;
+def M2_mmacuhs_s0 : HexagonBuiltin<"long long int(long long int, long long int, long long int)">;
+def M2_mmacuhs_s1 : HexagonBuiltin<"long long int(long long int, long long int, long long int)">;
+def M2_mmaculs_rs0 : HexagonBuiltin<"long long int(long long int, long long int, long long int)">;
+def M2_mmaculs_rs1 : HexagonBuiltin<"long long int(long long int, long long int, long long int)">;
+def M2_mmaculs_s0 : HexagonBuiltin<"long long int(long long int, long long int, long long int)">;
+def M2_mmaculs_s1 : HexagonBuiltin<"long long int(long long int, long long int, long long int)">;
+def M2_mmpyh_rs0 : HexagonBuiltin<"long long int(long long int, long long int)">;
+def M2_mmpyh_rs1 : HexagonBuiltin<"long long int(long long int, long long int)">;
+def M2_mmpyh_s0 : HexagonBuiltin<"long long int(long long int, long long int)">;
+def M2_mmpyh_s1 : HexagonBuiltin<"long long int(long long int, long long int)">;
+def M2_mmpyl_rs0 : HexagonBuiltin<"long long int(long long int, long long int)">;
+def M2_mmpyl_rs1 : HexagonBuiltin<"long long int(long long int, long long int)">;
+def M2_mmpyl_s0 : HexagonBuiltin<"long long int(long long int, long long int)">;
+def M2_mmpyl_s1 : HexagonBuiltin<"long long int(long long int, long long int)">;
+def M2_mmpyuh_rs0 : HexagonBuiltin<"long long int(long long int, long long int)">;
+def M2_mmpyuh_rs1 : HexagonBuiltin<"long long int(long long int, long long int)">;
+def M2_mmpyuh_s0 : HexagonBuiltin<"long long int(long long int, long long int)">;
+def M2_mmpyuh_s1 : HexagonBuiltin<"long long int(long long int, long long int)">;
+def M2_mmpyul_rs0 : HexagonBuiltin<"long long int(long long int, long long int)">;
+def M2_mmpyul_rs1 : HexagonBuiltin<"long long int(long long int, long long int)">;
+def M2_mmpyul_s0 : HexagonBuiltin<"long long int(long long int, long long int)">;
+def M2_mmpyul_s1 : HexagonBuiltin<"long long int(long long int, long long int)">;
+def M2_mpy_acc_hh_s0 : HexagonBuiltin<"int(int, int, int)">;
+def M2_mpy_acc_hh_s1 : HexagonBuiltin<"int(int, int, int)">;
+def M2_mpy_acc_hl_s0 : HexagonBuiltin<"int(int, int, int)">;
+def M2_mpy_acc_hl_s1 : HexagonBuiltin<"int(int, int, int)">;
+def M2_mpy_acc_lh_s0 : HexagonBuiltin<"int(int, int, int)">;
+def M2_mpy_acc_lh_s1 : HexagonBuiltin<"int(int, int, int)">;
+def M2_mpy_acc_ll_s0 : HexagonBuiltin<"int(int, int, int)">;
+def M2_mpy_acc_ll_s1 : HexagonBuiltin<"int(int, int, int)">;
+def M2_mpy_acc_sat_hh_s0 : HexagonBuiltin<"int(int, int, int)">;
+def M2_mpy_acc_sat_hh_s1 : HexagonBuiltin<"int(int, int, int)">;
+def M2_mpy_acc_sat_hl_s0 : HexagonBuiltin<"int(int, int, int)">;
+def M2_mpy_acc_sat_hl_s1 : HexagonBuiltin<"int(int, int, int)">;
+def M2_mpy_acc_sat_lh_s0 : HexagonBuiltin<"int(int, int, int)">;
+def M2_mpy_acc_sat_lh_s1 : HexagonBuiltin<"int(int, int, int)">;
+def M2_mpy_acc_sat_ll_s0 : HexagonBuiltin<"int(int, int, int)">;
+def M2_mpy_acc_sat_ll_s1 : HexagonBuiltin<"int(int, int, int)">;
+def M2_mpy_hh_s0 : HexagonBuiltin<"int(int, int)">;
+def M2_mpy_hh_s1 : HexagonBuiltin<"int(int, int)">;
+def M2_mpy_hl_s0 : HexagonBuiltin<"int(int, int)">;
+def M2_mpy_hl_s1 : HexagonBuiltin<"int(int, int)">;
+def M2_mpy_lh_s0 : HexagonBuiltin<"int(int, int)">;
+def M2_mpy_lh_s1 : HexagonBuiltin<"int(int, int)">;
+def M2_mpy_ll_s0 : HexagonBuiltin<"int(int, int)">;
+def M2_mpy_ll_s1 : HexagonBuiltin<"int(int, int)">;
+def M2_mpy_nac_hh_s0 : HexagonBuiltin<"int(int, int, int)">;
+def M2_mpy_nac_hh_s1 : HexagonBuiltin<"int(int, int, int)">;
+def M2_mpy_nac_hl_s0 : HexagonBuiltin<"int(int, int, int)">;
+def M2_mpy_nac_hl_s1 : HexagonBuiltin<"int(int, int, int)">;
+def M2_mpy_nac_lh_s0 : HexagonBuiltin<"int(int, int, int)">;
+def M2_mpy_nac_lh_s1 : HexagonBuiltin<"int(int, int, int)">;
+def M2_mpy_nac_ll_s0 : HexagonBuiltin<"int(int, int, int)">;
+def M2_mpy_nac_ll_s1 : HexagonBuiltin<"int(int, int, int)">;
+def M2_mpy_nac_sat_hh_s0 : HexagonBuiltin<"int(int, int, int)">;
+def M2_mpy_nac_sat_hh_s1 : HexagonBuiltin<"int(int, int, int)">;
+def M2_mpy_nac_sat_hl_s0 : HexagonBuiltin<"int(int, int, int)">;
+def M2_mpy_nac_sat_hl_s1 : HexagonBuiltin<"int(int, int, int)">;
+def M2_mpy_nac_sat_lh_s0 : HexagonBuiltin<"int(int, int, int)">;
+def M2_mpy_nac_sat_lh_s1 : HexagonBuiltin<"int(int, int, int)">;
+def M2_mpy_nac_sat_ll_s0 : HexagonBuiltin<"int(int, int, int)">;
+def M2_mpy_nac_sat_ll_s1 : HexagonBuiltin<"int(int, int, int)">;
+def M2_mpy_rnd_hh_s0 : HexagonBuiltin<"int(int, int)">;
+def M2_mpy_rnd_hh_s1 : HexagonBuiltin<"int(int, int)">;
+def M2_mpy_rnd_hl_s0 : HexagonBuiltin<"int(int, int)">;
+def M2_mpy_rnd_hl_s1 : HexagonBuiltin<"int(int, int)">;
+def M2_mpy_rnd_lh_s0 : HexagonBuiltin<"int(int, int)">;
+def M2_mpy_rnd_lh_s1 : HexagonBuiltin<"int(int, int)">;
+def M2_mpy_rnd_ll_s0 : HexagonBuiltin<"int(int, int)">;
+def M2_mpy_rnd_ll_s1 : HexagonBuiltin<"int(int, int)">;
+def M2_mpy_sat_hh_s0 : HexagonBuiltin<"int(int, int)">;
+def M2_mpy_sat_hh_s1 : HexagonBuiltin<"int(int, int)">;
+def M2_mpy_sat_hl_s0 : HexagonBuiltin<"int(int, int)">;
+def M2_mpy_sat_hl_s1 : HexagonBuiltin<"int(int, int)">;
+def M2_mpy_sat_lh_s0 : HexagonBuiltin<"int(int, int)">;
+def M2_mpy_sat_lh_s1 : HexagonBuiltin<"int(int, int)">;
+def M2_mpy_sat_ll_s0 : HexagonBuiltin<"int(int, int)">;
+def M2_mpy_sat_ll_s1 : HexagonBuiltin<"int(int, int)">;
+def M2_mpy_sat_rnd_hh_s0 : HexagonBuiltin<"int(int, int)">;
+def M2_mpy_sat_rnd_hh_s1 : HexagonBuiltin<"int(int, int)">;
+def M2_mpy_sat_rnd_hl_s0 : HexagonBuiltin<"int(int, int)">;
+def M2_mpy_sat_rnd_hl_s1 : HexagonBuiltin<"int(int, int)">;
+def M2_mpy_sat_rnd_lh_s0 : HexagonBuiltin<"int(int, int)">;
+def M2_mpy_sat_rnd_lh_s1 : HexagonBuiltin<"int(int, int)">;
+def M2_mpy_sat_rnd_ll_s0 : HexagonBuiltin<"int(int, int)">;
+def M2_mpy_sat_rnd_ll_s1 : HexagonBuiltin<"int(int, int)">;
+def M2_mpy_up : HexagonBuiltin<"int(int, int)">;
+def M2_mpy_up_s1 : HexagonBuiltin<"int(int, int)">;
+def M2_mpy_up_s1_sat : HexagonBuiltin<"int(int, int)">;
+def M2_mpyd_acc_hh_s0 : HexagonBuiltin<"long long int(long long int, int, int)">;
+def M2_mpyd_acc_hh_s1 : HexagonBuiltin<"long long int(long long int, int, int)">;
+def M2_mpyd_acc_hl_s0 : HexagonBuiltin<"long long int(long long int, int, int)">;
+def M2_mpyd_acc_hl_s1 : HexagonBuiltin<"long long int(long long int, int, int)">;
+def M2_mpyd_acc_lh_s0 : HexagonBuiltin<"long long int(long long int, int, int)">;
+def M2_mpyd_acc_lh_s1 : HexagonBuiltin<"long long int(long long int, int, int)">;
+def M2_mpyd_acc_ll_s0 : HexagonBuiltin<"long long int(long long int, int, int)">;
+def M2_mpyd_acc_ll_s1 : HexagonBuiltin<"long long int(long long int, int, int)">;
+def M2_mpyd_hh_s0 : HexagonBuiltin<"long long int(int, int)">;
+def M2_mpyd_hh_s1 : HexagonBuiltin<"long long int(int, int)">;
+def M2_mpyd_hl_s0 : HexagonBuiltin<"long long int(int, int)">;
+def M2_mpyd_hl_s1 : HexagonBuiltin<"long long int(int, int)">;
+def M2_mpyd_lh_s0 : HexagonBuiltin<"long long int(int, int)">;
+def M2_mpyd_lh_s1 : HexagonBuiltin<"long long int(int, int)">;
+def M2_mpyd_ll_s0 : HexagonBuiltin<"long long int(int, int)">;
+def M2_mpyd_ll_s1 : HexagonBuiltin<"long long int(int, int)">;
+def M2_mpyd_nac_hh_s0 : HexagonBuiltin<"long long int(long long int, int, int)">;
+def M2_mpyd_nac_hh_s1 : HexagonBuiltin<"long long int(long long int, int, int)">;
+def M2_mpyd_nac_hl_s0 : HexagonBuiltin<"long long int(long long int, int, int)">;
+def M2_mpyd_nac_hl_s1 : HexagonBuiltin<"long long int(long long int, int, int)">;
+def M2_mpyd_nac_lh_s0 : HexagonBuiltin<"long long int(long long int, int, int)">;
+def M2_mpyd_nac_lh_s1 : HexagonBuiltin<"long long int(long long int, int, int)">;
+def M2_mpyd_nac_ll_s0 : HexagonBuiltin<"long long int(long long int, int, int)">;
+def M2_mpyd_nac_ll_s1 : HexagonBuiltin<"long long int(long long int, int, int)">;
+def M2_mpyd_rnd_hh_s0 : HexagonBuiltin<"long long int(int, int)">;
+def M2_mpyd_rnd_hh_s1 : HexagonBuiltin<"long long int(int, int)">;
+def M2_mpyd_rnd_hl_s0 : HexagonBuiltin<"long long int(int, int)">;
+def M2_mpyd_rnd_hl_s1 : HexagonBuiltin<"long long int(int, int)">;
+def M2_mpyd_rnd_lh_s0 : HexagonBuiltin<"long long int(int, int)">;
+def M2_mpyd_rnd_lh_s1 : HexagonBuiltin<"long long int(int, int)">;
+def M2_mpyd_rnd_ll_s0 : HexagonBuiltin<"long long int(int, int)">;
+def M2_mpyd_rnd_ll_s1 : HexagonBuiltin<"long long int(int, int)">;
+def M2_mpyi : HexagonBuiltin<"int(int, int)">;
+def M2_mpysmi : HexagonBuiltin<"int(int, _Constant int)">;
+def M2_mpysu_up : HexagonBuiltin<"int(int, int)">;
+def M2_mpyu_acc_hh_s0 : HexagonBuiltin<"int(int, int, int)">;
+def M2_mpyu_acc_hh_s1 : HexagonBuiltin<"int(int, int, int)">;
+def M2_mpyu_acc_hl_s0 : HexagonBuiltin<"int(int, int, int)">;
+def M2_mpyu_acc_hl_s1 : HexagonBuiltin<"int(int, int, int)">;
+def M2_mpyu_acc_lh_s0 : HexagonBuiltin<"int(int, int, int)">;
+def M2_mpyu_acc_lh_s1 : HexagonBuiltin<"int(int, int, int)">;
+def M2_mpyu_acc_ll_s0 : HexagonBuiltin<"int(int, int, int)">;
+def M2_mpyu_acc_ll_s1 : HexagonBuiltin<"int(int, int, int)">;
+def M2_mpyu_hh_s0 : HexagonBuiltin<"unsigned int(int, int)">;
+def M2_mpyu_hh_s1 : HexagonBuiltin<"unsigned int(int, int)">;
+def M2_mpyu_hl_s0 : HexagonBuiltin<"unsigned int(int, int)">;
+def M2_mpyu_hl_s1 : HexagonBuiltin<"unsigned int(int, int)">;
+def M2_mpyu_lh_s0 : HexagonBuiltin<"unsigned int(int, int)">;
+def M2_mpyu_lh_s1 : HexagonBuiltin<"unsigned int(int, int)">;
+def M2_mpyu_ll_s0 : HexagonBuiltin<"unsigned int(int, int)">;
+def M2_mpyu_ll_s1 : HexagonBuiltin<"unsigned int(int, int)">;
+def M2_mpyu_nac_hh_s0 : HexagonBuiltin<"int(int, int, int)">;
+def M2_mpyu_nac_hh_s1 : HexagonBuiltin<"int(int, int, int)">;
+def M2_mpyu_nac_hl_s0 : HexagonBuiltin<"int(int, int, int)">;
+def M2_mpyu_nac_hl_s1 : HexagonBuiltin<"int(int, int, int)">;
+def M2_mpyu_nac_lh_s0 : HexagonBuiltin<"int(int, int, int)">;
+def M2_mpyu_nac_lh_s1 : HexagonBuiltin<"int(int, int, int)">;
+def M2_mpyu_nac_ll_s0 : HexagonBuiltin<"int(int, int, int)">;
+def M2_mpyu_nac_ll_s1 : HexagonBuiltin<"int(int, int, int)">;
+def M2_mpyu_up : HexagonBuiltin<"unsigned int(int, int)">;
+def M2_mpyud_acc_hh_s0 : HexagonBuiltin<"long long int(long long int, int, int)">;
+def M2_mpyud_acc_hh_s1 : HexagonBuiltin<"long long int(long long int, int, int)">;
+def M2_mpyud_acc_hl_s0 : HexagonBuiltin<"long long int(long long int, int, int)">;
+def M2_mpyud_acc_hl_s1 : HexagonBuiltin<"long long int(long long int, int, int)">;
+def M2_mpyud_acc_lh_s0 : HexagonBuiltin<"long long int(long long int, int, int)">;
+def M2_mpyud_acc_lh_s1 : HexagonBuiltin<"long long int(long long int, int, int)">;
+def M2_mpyud_acc_ll_s0 : HexagonBuiltin<"long long int(long long int, int, int)">;
+def M2_mpyud_acc_ll_s1 : HexagonBuiltin<"long long int(long long int, int, int)">;
+def M2_mpyud_hh_s0 : HexagonBuiltin<"unsigned long long int(int, int)">;
+def M2_mpyud_hh_s1 : HexagonBuiltin<"unsigned long long int(int, int)">;
+def M2_mpyud_hl_s0 : HexagonBuiltin<"unsigned long long int(int, int)">;
+def M2_mpyud_hl_s1 : HexagonBuiltin<"unsigned long long int(int, int)">;
+def M2_mpyud_lh_s0 : HexagonBuiltin<"unsigned long long int(int, int)">;
+def M2_mpyud_lh_s1 : HexagonBuiltin<"unsigned long long int(int, int)">;
+def M2_mpyud_ll_s0 : HexagonBuiltin<"unsigned long long int(int, int)">;
+def M2_mpyud_ll_s1 : HexagonBuiltin<"unsigned long long int(int, int)">;
+def M2_mpyud_nac_hh_s0 : HexagonBuiltin<"long long int(long long int, int, int)">;
+def M2_mpyud_nac_hh_s1 : HexagonBuiltin<"long long int(long long int, int, int)">;
+def M2_mpyud_nac_hl_s0 : HexagonBuiltin<"long long int(long long int, int, int)">;
+def M2_mpyud_nac_hl_s1 : HexagonBuiltin<"long long int(long long int, int, int)">;
+def M2_mpyud_nac_lh_s0 : HexagonBuiltin<"long long int(long long int, int, int)">;
+def M2_mpyud_nac_lh_s1 : HexagonBuiltin<"long long int(long long int, int, int)">;
+def M2_mpyud_nac_ll_s0 : HexagonBuiltin<"long long int(long long int, int, int)">;
+def M2_mpyud_nac_ll_s1 : HexagonBuiltin<"long long int(long long int, int, int)">;
+def M2_mpyui : HexagonBuiltin<"int(int, int)">;
+def M2_nacci : HexagonBuiltin<"int(int, int, int)">;
+def M2_naccii : HexagonBuiltin<"int(int, int, _Constant int)">;
+def M2_subacc : HexagonBuiltin<"int(int, int, int)">;
+def M2_vabsdiffh : HexagonBuiltin<"long long int(long long int, long long int)">;
+def M2_vabsdiffw : HexagonBuiltin<"long long int(long long int, long long int)">;
+def M2_vcmac_s0_sat_i : HexagonBuiltin<"long long int(long long int, long long int, long long int)">;
+def M2_vcmac_s0_sat_r : HexagonBuiltin<"long long int(long long int, long long int, long long int)">;
+def M2_vcmpy_s0_sat_i : HexagonBuiltin<"long long int(long long int, long long int)">;
+def M2_vcmpy_s0_sat_r : HexagonBuiltin<"long long int(long long int, long long int)">;
+def M2_vcmpy_s1_sat_i : HexagonBuiltin<"long long int(long long int, long long int)">;
+def M2_vcmpy_s1_sat_r : HexagonBuiltin<"long long int(long long int, long long int)">;
+def M2_vdmacs_s0 : HexagonBuiltin<"long long int(long long int, long long int, long long int)">;
+def M2_vdmacs_s1 : HexagonBuiltin<"long long int(long long int, long long int, long long int)">;
+def M2_vdmpyrs_s0 : HexagonBuiltin<"int(long long int, long long int)">;
+def M2_vdmpyrs_s1 : HexagonBuiltin<"int(long long int, long long int)">;
+def M2_vdmpys_s0 : HexagonBuiltin<"long long int(long long int, long long int)">;
+def M2_vdmpys_s1 : HexagonBuiltin<"long long int(long long int, long long int)">;
+def M2_vmac2 : HexagonBuiltin<"long long int(long long int, int, int)">;
+def M2_vmac2es : HexagonBuiltin<"long long int(long long int, long long int, long long int)">;
+def M2_vmac2es_s0 : HexagonBuiltin<"long long int(long long int, long long int, long long int)">;
+def M2_vmac2es_s1 : HexagonBuiltin<"long long int(long long int, long long int, long long int)">;
+def M2_vmac2s_s0 : HexagonBuiltin<"long long int(long long int, int, int)">;
+def M2_vmac2s_s1 : HexagonBuiltin<"long long int(long long int, int, int)">;
+def M2_vmac2su_s0 : HexagonBuiltin<"long long int(long long int, int, int)">;
+def M2_vmac2su_s1 : HexagonBuiltin<"long long int(long long int, int, int)">;
+def M2_vmpy2es_s0 : HexagonBuiltin<"long long int(long long int, long long int)">;
+def M2_vmpy2es_s1 : HexagonBuiltin<"long long int(long long int, long long int)">;
+def M2_vmpy2s_s0 : HexagonBuiltin<"long long int(int, int)">;
+def M2_vmpy2s_s0pack : HexagonBuiltin<"int(int, int)">;
+def M2_vmpy2s_s1 : HexagonBuiltin<"long long int(int, int)">;
+def M2_vmpy2s_s1pack : HexagonBuiltin<"int(int, int)">;
+def M2_vmpy2su_s0 : HexagonBuiltin<"long long int(int, int)">;
+def M2_vmpy2su_s1 : HexagonBuiltin<"long long int(int, int)">;
+def M2_vraddh : HexagonBuiltin<"int(long long int, long long int)">;
+def M2_vradduh : HexagonBuiltin<"int(long long int, long long int)">;
+def M2_vrcmaci_s0 : HexagonBuiltin<"long long int(long long int, long long int, long long int)">;
+def M2_vrcmaci_s0c : HexagonBuiltin<"long long int(long long int, long long int, long long int)">;
+def M2_vrcmacr_s0 : HexagonBuiltin<"long long int(long long int, long long int, long long int)">;
+def M2_vrcmacr_s0c : HexagonBuiltin<"long long int(long long int, long long int, long long int)">;
+def M2_vrcmpyi_s0 : HexagonBuiltin<"long long int(long long int, long long int)">;
+def M2_vrcmpyi_s0c : HexagonBuiltin<"long long int(long long int, long long int)">;
+def M2_vrcmpyr_s0 : HexagonBuiltin<"long long int(long long int, long long int)">;
+def M2_vrcmpyr_s0c : HexagonBuiltin<"long long int(long long int, long long int)">;
+def M2_vrcmpys_acc_s1 : HexagonBuiltin<"long long int(long long int, long long int, int)">;
+def M2_vrcmpys_s1 : HexagonBuiltin<"long long int(long long int, int)">;
+def M2_vrcmpys_s1rp : HexagonBuiltin<"int(long long int, int)">;
+def M2_vrmac_s0 : HexagonBuiltin<"long long int(long long int, long long int, long long int)">;
+def M2_vrmpy_s0 : HexagonBuiltin<"long long int(long long int, long long int)">;
+def M2_xor_xacc : HexagonBuiltin<"int(int, int, int)">;
+def M4_and_and : HexagonBuiltin<"int(int, int, int)">;
+def M4_and_andn : HexagonBuiltin<"int(int, int, int)">;
+def M4_and_or : HexagonBuiltin<"int(int, int, int)">;
+def M4_and_xor : HexagonBuiltin<"int(int, int, int)">;
+def M4_cmpyi_wh : HexagonBuiltin<"int(long long int, int)">;
+def M4_cmpyi_whc : HexagonBuiltin<"int(long long int, int)">;
+def M4_cmpyr_wh : HexagonBuiltin<"int(long long int, int)">;
+def M4_cmpyr_whc : HexagonBuiltin<"int(long long int, int)">;
+def M4_mac_up_s1_sat : HexagonBuiltin<"int(int, int, int)">;
+def M4_mpyri_addi : HexagonBuiltin<"int(unsigned _Constant int, int, unsigned _Constant int)">;
+def M4_mpyri_addr : HexagonBuiltin<"int(int, int, unsigned _Constant int)">;
+def M4_mpyri_addr_u2 : HexagonBuiltin<"int(int, unsigned _Constant int, int)">;
+def M4_mpyrr_addi : HexagonBuiltin<"int(unsigned _Constant int, int, int)">;
+def M4_mpyrr_addr : HexagonBuiltin<"int(int, int, int)">;
+def M4_nac_up_s1_sat : HexagonBuiltin<"int(int, int, int)">;
+def M4_or_and : HexagonBuiltin<"int(int, int, int)">;
+def M4_or_andn : HexagonBuiltin<"int(int, int, int)">;
+def M4_or_or : HexagonBuiltin<"int(int, int, int)">;
+def M4_or_xor : HexagonBuiltin<"int(int, int, int)">;
+def M4_pmpyw : HexagonBuiltin<"long long int(int, int)">;
+def M4_pmpyw_acc : HexagonBuiltin<"long long int(long long int, int, int)">;
+def M4_vpmpyh : HexagonBuiltin<"long long int(int, int)">;
+def M4_vpmpyh_acc : HexagonBuiltin<"long long int(long long int, int, int)">;
+def M4_vrmpyeh_acc_s0 : HexagonBuiltin<"long long int(long long int, long long int, long long int)">;
+def M4_vrmpyeh_acc_s1 : HexagonBuiltin<"long long int(long long int, long long int, long long int)">;
+def M4_vrmpyeh_s0 : HexagonBuiltin<"long long int(long long int, long long int)">;
+def M4_vrmpyeh_s1 : HexagonBuiltin<"long long int(long long int, long long int)">;
+def M4_vrmpyoh_acc_s0 : HexagonBuiltin<"long long int(long long int, long long int, long long int)">;
+def M4_vrmpyoh_acc_s1 : HexagonBuiltin<"long long int(long long int, long long int, long long int)">;
+def M4_vrmpyoh_s0 : HexagonBuiltin<"long long int(long long int, long long int)">;
+def M4_vrmpyoh_s1 : HexagonBuiltin<"long long int(long long int, long long int)">;
+def M4_xor_and : HexagonBuiltin<"int(int, int, int)">;
+def M4_xor_andn : HexagonBuiltin<"int(int, int, int)">;
+def M4_xor_or : HexagonBuiltin<"int(int, int, int)">;
+def M4_xor_xacc : HexagonBuiltin<"long long int(long long int, long long int, long long int)">;
+def M5_vdmacbsu : HexagonBuiltin<"long long int(long long int, long long int, long long int)">;
+def M5_vdmpybsu : HexagonBuiltin<"long long int(long long int, long long int)">;
+def M5_vmacbsu : HexagonBuiltin<"long long int(long long int, int, int)">;
+def M5_vmacbuu : HexagonBuiltin<"long long int(long long int, int, int)">;
+def M5_vmpybsu : HexagonBuiltin<"long long int(int, int)">;
+def M5_vmpybuu : HexagonBuiltin<"long long int(int, int)">;
+def M5_vrmacbsu : HexagonBuiltin<"long long int(long long int, long long int, long long int)">;
+def M5_vrmacbuu : HexagonBuiltin<"long long int(long long int, long long int, long long int)">;
+def M5_vrmpybsu : HexagonBuiltin<"long long int(long long int, long long int)">;
+def M5_vrmpybuu : HexagonBuiltin<"long long int(long long int, long long int)">;
+def S2_addasl_rrri : HexagonBuiltin<"int(int, int, unsigned _Constant int)">;
+def S2_asl_i_p : HexagonBuiltin<"long long int(long long int, unsigned _Constant int)">;
+def S2_asl_i_p_acc : HexagonBuiltin<"long long int(long long int, long long int, unsigned _Constant int)">;
+def S2_asl_i_p_and : HexagonBuiltin<"long long int(long long int, long long int, unsigned _Constant int)">;
+def S2_asl_i_p_nac : HexagonBuiltin<"long long int(long long int, long long int, unsigned _Constant int)">;
+def S2_asl_i_p_or : HexagonBuiltin<"long long int(long long int, long long int, unsigned _Constant int)">;
+def S2_asl_i_p_xacc : HexagonBuiltin<"long long int(long long int, long long int, unsigned _Constant int)">;
+def S2_asl_i_r : HexagonBuiltin<"int(int, unsigned _Constant int)">;
+def S2_asl_i_r_acc : HexagonBuiltin<"int(int, int, unsigned _Constant int)">;
+def S2_asl_i_r_and : HexagonBuiltin<"int(int, int, unsigned _Constant int)">;
+def S2_asl_i_r_nac : HexagonBuiltin<"int(int, int, unsigned _Constant int)">;
+def S2_asl_i_r_or : HexagonBuiltin<"int(int, int, unsigned _Constant int)">;
+def S2_asl_i_r_sat : HexagonBuiltin<"int(int, unsigned _Constant int)">;
+def S2_asl_i_r_xacc : HexagonBuiltin<"int(int, int, unsigned _Constant int)">;
+def S2_asl_i_vh : HexagonBuiltin<"long long int(long long int, unsigned _Constant int)">;
+def S2_asl_i_vw : HexagonBuiltin<"long long int(long long int, unsigned _Constant int)">;
+def S2_asl_r_p : HexagonBuiltin<"long long int(long long int, int)">;
+def S2_asl_r_p_acc : HexagonBuiltin<"long long int(long long int, long long int, int)">;
+def S2_asl_r_p_and : HexagonBuiltin<"long long int(long long int, long long int, int)">;
+def S2_asl_r_p_nac : HexagonBuiltin<"long long int(long long int, long long int, int)">;
+def S2_asl_r_p_or : HexagonBuiltin<"long long int(long long int, long long int, int)">;
+def S2_asl_r_p_xor : HexagonBuiltin<"long long int(long long int, long long int, int)">;
+def S2_asl_r_r : HexagonBuiltin<"int(int, int)">;
+def S2_asl_r_r_acc : HexagonBuiltin<"int(int, int, int)">;
+def S2_asl_r_r_and : HexagonBuiltin<"int(int, int, int)">;
+def S2_asl_r_r_nac : HexagonBuiltin<"int(int, int, int)">;
+def S2_asl_r_r_or : HexagonBuiltin<"int(int, int, int)">;
+def S2_asl_r_r_sat : HexagonBuiltin<"int(int, int)">;
+def S2_asl_r_vh : HexagonBuiltin<"long long int(long long int, int)">;
+def S2_asl_r_vw : HexagonBuiltin<"long long int(long long int, int)">;
+def S2_asr_i_p : HexagonBuiltin<"long long int(long long int, unsigned _Constant int)">;
+def S2_asr_i_p_acc : HexagonBuiltin<"long long int(long long int, long long int, unsigned _Constant int)">;
+def S2_asr_i_p_and : HexagonBuiltin<"long long int(long long int, long long int, unsigned _Constant int)">;
+def S2_asr_i_p_nac : HexagonBuiltin<"long long int(long long int, long long int, unsigned _Constant int)">;
+def S2_asr_i_p_or : HexagonBuiltin<"long long int(long long int, long long int, unsigned _Constant int)">;
+def S2_asr_i_p_rnd : HexagonBuiltin<"long long int(long long int, unsigned _Constant int)">;
+def S2_asr_i_p_rnd_goodsyntax : HexagonBuiltin<"long long int(long long int, unsigned _Constant int)">;
+def S2_asr_i_r : HexagonBuiltin<"int(int, unsigned _Constant int)">;
+def S2_asr_i_r_acc : HexagonBuiltin<"int(int, int, unsigned _Constant int)">;
+def S2_asr_i_r_and : HexagonBuiltin<"int(int, int, unsigned _Constant int)">;
+def S2_asr_i_r_nac : HexagonBuiltin<"int(int, int, unsigned _Constant int)">;
+def S2_asr_i_r_or : HexagonBuiltin<"int(int, int, unsigned _Constant int)">;
+def S2_asr_i_r_rnd : HexagonBuiltin<"int(int, unsigned _Constant int)">;
+def S2_asr_i_r_rnd_goodsyntax : HexagonBuiltin<"int(int, unsigned _Constant int)">;
+def S2_asr_i_svw_trun : HexagonBuiltin<"int(long long int, unsigned _Constant int)">;
+def S2_asr_i_vh : HexagonBuiltin<"long long int(long long int, unsigned _Constant int)">;
+def S2_asr_i_vw : HexagonBuiltin<"long long int(long long int, unsigned _Constant int)">;
+def S2_asr_r_p : HexagonBuiltin<"long long int(long long int, int)">;
+def S2_asr_r_p_acc : HexagonBuiltin<"long long int(long long int, long long int, int)">;
+def S2_asr_r_p_and : HexagonBuiltin<"long long int(long long int, long long int, int)">;
+def S2_asr_r_p_nac : HexagonBuiltin<"long long int(long long int, long long int, int)">;
+def S2_asr_r_p_or : HexagonBuiltin<"long long int(long long int, long long int, int)">;
+def S2_asr_r_p_xor : HexagonBuiltin<"long long int(long long int, long long int, int)">;
+def S2_asr_r_r : HexagonBuiltin<"int(int, int)">;
+def S2_asr_r_r_acc : HexagonBuiltin<"int(int, int, int)">;
+def S2_asr_r_r_and : HexagonBuiltin<"int(int, int, int)">;
+def S2_asr_r_r_nac : HexagonBuiltin<"int(int, int, int)">;
+def S2_asr_r_r_or : HexagonBuiltin<"int(int, int, int)">;
+def S2_asr_r_r_sat : HexagonBuiltin<"int(int, int)">;
+def S2_asr_r_svw_trun : HexagonBuiltin<"int(long long int, int)">;
+def S2_asr_r_vh : HexagonBuiltin<"long long int(long long int, int)">;
+def S2_asr_r_vw : HexagonBuiltin<"long long int(long long int, int)">;
+def S2_brev : HexagonBuiltin<"int(int)">;
+def S2_brevp : HexagonBuiltin<"long long int(long long int)">;
+def S2_cl0 : HexagonBuiltin<"int(int)">;
+def S2_cl0p : HexagonBuiltin<"int(long long int)">;
+def S2_cl1 : HexagonBuiltin<"int(int)">;
+def S2_cl1p : HexagonBuiltin<"int(long long int)">;
+def S2_clb : HexagonBuiltin<"int(int)">;
+def S2_clbnorm : HexagonBuiltin<"int(int)">;
+def S2_clbp : HexagonBuiltin<"int(long long int)">;
+def S2_clrbit_i : HexagonBuiltin<"int(int, unsigned _Constant int)">;
+def S2_clrbit_r : HexagonBuiltin<"int(int, int)">;
+def S2_ct0 : HexagonBuiltin<"int(int)">;
+def S2_ct0p : HexagonBuiltin<"int(long long int)">;
+def S2_ct1 : HexagonBuiltin<"int(int)">;
+def S2_ct1p : HexagonBuiltin<"int(long long int)">;
+def S2_deinterleave : HexagonBuiltin<"long long int(long long int)">;
+def S2_extractu : HexagonBuiltin<"int(int, unsigned _Constant int, unsigned _Constant int)">;
+def S2_extractu_rp : HexagonBuiltin<"int(int, long long int)">;
+def S2_extractup : HexagonBuiltin<"long long int(long long int, unsigned _Constant int, unsigned _Constant int)">;
+def S2_extractup_rp : HexagonBuiltin<"long long int(long long int, long long int)">;
+def S2_insert : HexagonBuiltin<"int(int, int, unsigned _Constant int, unsigned _Constant int)">;
+def S2_insert_rp : HexagonBuiltin<"int(int, int, long long int)">;
+def S2_insertp : HexagonBuiltin<"long long int(long long int, long long int, unsigned _Constant int, unsigned _Constant int)">;
+def S2_insertp_rp : HexagonBuiltin<"long long int(long long int, long long int, long long int)">;
+def S2_interleave : HexagonBuiltin<"long long int(long long int)">;
+def S2_lfsp : HexagonBuiltin<"long long int(long long int, long long int)">;
+def S2_lsl_r_p : HexagonBuiltin<"long long int(long long int, int)">;
+def S2_lsl_r_p_acc : HexagonBuiltin<"long long int(long long int, long long int, int)">;
+def S2_lsl_r_p_and : HexagonBuiltin<"long long int(long long int, long long int, int)">;
+def S2_lsl_r_p_nac : HexagonBuiltin<"long long int(long long int, long long int, int)">;
+def S2_lsl_r_p_or : HexagonBuiltin<"long long int(long long int, long long int, int)">;
+def S2_lsl_r_p_xor : HexagonBuiltin<"long long int(long long int, long long int, int)">;
+def S2_lsl_r_r : HexagonBuiltin<"int(int, int)">;
+def S2_lsl_r_r_acc : HexagonBuiltin<"int(int, int, int)">;
+def S2_lsl_r_r_and : HexagonBuiltin<"int(int, int, int)">;
+def S2_lsl_r_r_nac : HexagonBuiltin<"int(int, int, int)">;
+def S2_lsl_r_r_or : HexagonBuiltin<"int(int, int, int)">;
+def S2_lsl_r_vh : HexagonBuiltin<"long long int(long long int, int)">;
+def S2_lsl_r_vw : HexagonBuiltin<"long long int(long long int, int)">;
+def S2_lsr_i_p : HexagonBuiltin<"long long int(long long int, unsigned _Constant int)">;
+def S2_lsr_i_p_acc : HexagonBuiltin<"long long int(long long int, long long int, unsigned _Constant int)">;
+def S2_lsr_i_p_and : HexagonBuiltin<"long long int(long long int, long long int, unsigned _Constant int)">;
+def S2_lsr_i_p_nac : HexagonBuiltin<"long long int(long long int, long long int, unsigned _Constant int)">;
+def S2_lsr_i_p_or : HexagonBuiltin<"long long int(long long int, long long int, unsigned _Constant int)">;
+def S2_lsr_i_p_xacc : HexagonBuiltin<"long long int(long long int, long long int, unsigned _Constant int)">;
+def S2_lsr_i_r : HexagonBuiltin<"int(int, unsigned _Constant int)">;
+def S2_lsr_i_r_acc : HexagonBuiltin<"int(int, int, unsigned _Constant int)">;
+def S2_lsr_i_r_and : HexagonBuiltin<"int(int, int, unsigned _Constant int)">;
+def S2_lsr_i_r_nac : HexagonBuiltin<"int(int, int, unsigned _Constant int)">;
+def S2_lsr_i_r_or : HexagonBuiltin<"int(int, int, unsigned _Constant int)">;
+def S2_lsr_i_r_xacc : HexagonBuiltin<"int(int, int, unsigned _Constant int)">;
+def S2_lsr_i_vh : HexagonBuiltin<"long long int(long long int, unsigned _Constant int)">;
+def S2_lsr_i_vw : HexagonBuiltin<"long long int(long long int, unsigned _Constant int)">;
+def S2_lsr_r_p : HexagonBuiltin<"long long int(long long int, int)">;
+def S2_lsr_r_p_acc : HexagonBuiltin<"long long int(long long int, long long int, int)">;
+def S2_lsr_r_p_and : HexagonBuiltin<"long long int(long long int, long long int, int)">;
+def S2_lsr_r_p_nac : HexagonBuiltin<"long long int(long long int, long long int, int)">;
+def S2_lsr_r_p_or : HexagonBuiltin<"long long int(long long int, long long int, int)">;
+def S2_lsr_r_p_xor : HexagonBuiltin<"long long int(long long int, long long int, int)">;
+def S2_lsr_r_r : HexagonBuiltin<"int(int, int)">;
+def S2_lsr_r_r_acc : HexagonBuiltin<"int(int, int, int)">;
+def S2_lsr_r_r_and : HexagonBuiltin<"int(int, int, int)">;
+def S2_lsr_r_r_nac : HexagonBuiltin<"int(int, int, int)">;
+def S2_lsr_r_r_or : HexagonBuiltin<"int(int, int, int)">;
+def S2_lsr_r_vh : HexagonBuiltin<"long long int(long long int, int)">;
+def S2_lsr_r_vw : HexagonBuiltin<"long long int(long long int, int)">;
+def S2_packhl : HexagonBuiltin<"long long int(int, int)">;
+def S2_parityp : HexagonBuiltin<"int(long long int, long long int)">;
+def S2_setbit_i : HexagonBuiltin<"int(int, unsigned _Constant int)">;
+def S2_setbit_r : HexagonBuiltin<"int(int, int)">;
+def S2_shuffeb : HexagonBuiltin<"long long int(long long int, long long int)">;
+def S2_shuffeh : HexagonBuiltin<"long long int(long long int, long long int)">;
+def S2_shuffob : HexagonBuiltin<"long long int(long long int, long long int)">;
+def S2_shuffoh : HexagonBuiltin<"long long int(long long int, long long int)">;
+def S2_svsathb : HexagonBuiltin<"int(int)">;
+def S2_svsathub : HexagonBuiltin<"int(int)">;
+def S2_tableidxb_goodsyntax : HexagonBuiltin<"int(int, int, unsigned _Constant int, unsigned _Constant int)">;
+def S2_tableidxd_goodsyntax : HexagonBuiltin<"int(int, int, unsigned _Constant int, unsigned _Constant int)">;
+def S2_tableidxh_goodsyntax : HexagonBuiltin<"int(int, int, unsigned _Constant int, unsigned _Constant int)">;
+def S2_tableidxw_goodsyntax : HexagonBuiltin<"int(int, int, unsigned _Constant int, unsigned _Constant int)">;
+def S2_togglebit_i : HexagonBuiltin<"int(int, unsigned _Constant int)">;
+def S2_togglebit_r : HexagonBuiltin<"int(int, int)">;
+def S2_tstbit_i : HexagonBuiltin<"int(int, unsigned _Constant int)">;
+def S2_tstbit_r : HexagonBuiltin<"int(int, int)">;
+def S2_valignib : HexagonBuiltin<"long long int(long long int, long long int, unsigned _Constant int)">;
+def S2_valignrb : HexagonBuiltin<"long long int(long long int, long long int, int)">;
+def S2_vcnegh : HexagonBuiltin<"long long int(long long int, int)">;
+def S2_vcrotate : HexagonBuiltin<"long long int(long long int, int)">;
+def S2_vrcnegh : HexagonBuiltin<"long long int(long long int, long long int, int)">;
+def S2_vrndpackwh : HexagonBuiltin<"int(long long int)">;
+def S2_vrndpackwhs : HexagonBuiltin<"int(long long int)">;
+def S2_vsathb : HexagonBuiltin<"int(long long int)">;
+def S2_vsathb_nopack : HexagonBuiltin<"long long int(long long int)">;
+def S2_vsathub : HexagonBuiltin<"int(long long int)">;
+def S2_vsathub_nopack : HexagonBuiltin<"long long int(long long int)">;
+def S2_vsatwh : HexagonBuiltin<"int(long long int)">;
+def S2_vsatwh_nopack : HexagonBuiltin<"long long int(long long int)">;
+def S2_vsatwuh : HexagonBuiltin<"int(long long int)">;
+def S2_vsatwuh_nopack : HexagonBuiltin<"long long int(long long int)">;
+def S2_vsplatrb : HexagonBuiltin<"int(int)">;
+def S2_vsplatrh : HexagonBuiltin<"long long int(int)">;
+def S2_vspliceib : HexagonBuiltin<"long long int(long long int, long long int, unsigned _Constant int)">;
+def S2_vsplicerb : HexagonBuiltin<"long long int(long long int, long long int, int)">;
+def S2_vsxtbh : HexagonBuiltin<"long long int(int)">;
+def S2_vsxthw : HexagonBuiltin<"long long int(int)">;
+def S2_vtrunehb : HexagonBuiltin<"int(long long int)">;
+def S2_vtrunewh : HexagonBuiltin<"long long int(long long int, long long int)">;
+def S2_vtrunohb : HexagonBuiltin<"int(long long int)">;
+def S2_vtrunowh : HexagonBuiltin<"long long int(long long int, long long int)">;
+def S2_vzxtbh : HexagonBuiltin<"long long int(int)">;
+def S2_vzxthw : HexagonBuiltin<"long long int(int)">;
+def S4_addaddi : HexagonBuiltin<"int(int, int, _Constant int)">;
+def S4_addi_asl_ri : HexagonBuiltin<"int(unsigned _Constant int, int, unsigned _Constant int)">;
+def S4_addi_lsr_ri : HexagonBuiltin<"int(unsigned _Constant int, int, unsigned _Constant int)">;
+def S4_andi_asl_ri : HexagonBuiltin<"int(unsigned _Constant int, int, unsigned _Constant int)">;
+def S4_andi_lsr_ri : HexagonBuiltin<"int(unsigned _Constant int, int, unsigned _Constant int)">;
+def S4_clbaddi : HexagonBuiltin<"int(int, _Constant int)">;
+def S4_clbpaddi : HexagonBuiltin<"int(long long int, _Constant int)">;
+def S4_clbpnorm : HexagonBuiltin<"int(long long int)">;
+def S4_extract : HexagonBuiltin<"int(int, unsigned _Constant int, unsigned _Constant int)">;
+def S4_extract_rp : HexagonBuiltin<"int(int, long long int)">;
+def S4_extractp : HexagonBuiltin<"long long int(long long int, unsigned _Constant int, unsigned _Constant int)">;
+def S4_extractp_rp : HexagonBuiltin<"long long int(long long int, long long int)">;
+def S4_lsli : HexagonBuiltin<"int(_Constant int, int)">;
+def S4_ntstbit_i : HexagonBuiltin<"int(int, unsigned _Constant int)">;
+def S4_ntstbit_r : HexagonBuiltin<"int(int, int)">;
+def S4_or_andi : HexagonBuiltin<"int(int, int, _Constant int)">;
+def S4_or_andix : HexagonBuiltin<"int(int, int, _Constant int)">;
+def S4_or_ori : HexagonBuiltin<"int(int, int, _Constant int)">;
+def S4_ori_asl_ri : HexagonBuiltin<"int(unsigned _Constant int, int, unsigned _Constant int)">;
+def S4_ori_lsr_ri : HexagonBuiltin<"int(unsigned _Constant int, int, unsigned _Constant int)">;
+def S4_parity : HexagonBuiltin<"int(int, int)">;
+def S4_subaddi : HexagonBuiltin<"int(int, _Constant int, int)">;
+def S4_subi_asl_ri : HexagonBuiltin<"int(unsigned _Constant int, int, unsigned _Constant int)">;
+def S4_subi_lsr_ri : HexagonBuiltin<"int(unsigned _Constant int, int, unsigned _Constant int)">;
+def S4_vrcrotate : HexagonBuiltin<"long long int(long long int, int, unsigned _Constant int)">;
+def S4_vrcrotate_acc : HexagonBuiltin<"long long int(long long int, long long int, int, unsigned _Constant int)">;
+def S4_vxaddsubh : HexagonBuiltin<"long long int(long long int, long long int)">;
+def S4_vxaddsubhr : HexagonBuiltin<"long long int(long long int, long long int)">;
+def S4_vxaddsubw : HexagonBuiltin<"long long int(long long int, long long int)">;
+def S4_vxsubaddh : HexagonBuiltin<"long long int(long long int, long long int)">;
+def S4_vxsubaddhr : HexagonBuiltin<"long long int(long long int, long long int)">;
+def S4_vxsubaddw : HexagonBuiltin<"long long int(long long int, long long int)">;
+def S5_asrhub_rnd_sat_goodsyntax : HexagonBuiltin<"int(long long int, unsigned _Constant int)">;
+def S5_asrhub_sat : HexagonBuiltin<"int(long long int, unsigned _Constant int)">;
+def S5_popcountp : HexagonBuiltin<"int(long long int)">;
+def S5_vasrhrnd_goodsyntax : HexagonBuiltin<"long long int(long long int, unsigned _Constant int)">;
+def Y2_dccleana : HexagonBuiltin<"void(void *)">;
+def Y2_dccleaninva : HexagonBuiltin<"void(void *)">;
+def Y2_dcfetch : HexagonBuiltin<"void(void *)">;
+def Y2_dcinva : HexagonBuiltin<"void(void *)">;
+def Y2_dczeroa : HexagonBuiltin<"void(void *)">;
+def Y4_l2fetch : HexagonBuiltin<"void(void *, int)">;
+def Y5_l2fetch : HexagonBuiltin<"void(void *, long long int)">;
+
+// V60 Scalar Instructions.
+
+let Features = V60.Features in {
+  def S6_rol_i_p : HexagonBuiltin<"long long int(long long int, unsigned _Constant int)">;
+  def S6_rol_i_p_acc : HexagonBuiltin<"long long int(long long int, long long int, unsigned _Constant int)">;
+  def S6_rol_i_p_and : HexagonBuiltin<"long long int(long long int, long long int, unsigned _Constant int)">;
+  def S6_rol_i_p_nac : HexagonBuiltin<"long long int(long long int, long long int, unsigned _Constant int)">;
+  def S6_rol_i_p_or : HexagonBuiltin<"long long int(long long int, long long int, unsigned _Constant int)">;
+  def S6_rol_i_p_xacc : HexagonBuiltin<"long long int(long long int, long long int, unsigned _Constant int)">;
+  def S6_rol_i_r : HexagonBuiltin<"int(int, unsigned _Constant int)">;
+  def S6_rol_i_r_acc : HexagonBuiltin<"int(int, int, unsigned _Constant int)">;
+  def S6_rol_i_r_and : HexagonBuiltin<"int(int, int, unsigned _Constant int)">;
+  def S6_rol_i_r_nac : HexagonBuiltin<"int(int, int, unsigned _Constant int)">;
+  def S6_rol_i_r_or : HexagonBuiltin<"int(int, int, unsigned _Constant int)">;
+  def S6_rol_i_r_xacc : HexagonBuiltin<"int(int, int, unsigned _Constant int)">;
+}
+
+// V62 Scalar Instructions.
+
+let Features = V62.Features in {
+  def M6_vabsdiffb : HexagonBuiltin<"long long int(long long int, long long int)">;
+  def M6_vabsdiffub : HexagonBuiltin<"long long int(long long int, long long int)">;
+  def S6_vsplatrbp : HexagonBuiltin<"long long int(int)">;
+  def S6_vtrunehb_ppp : HexagonBuiltin<"long long int(long long int, long long int)">;
+  def S6_vtrunohb_ppp : HexagonBuiltin<"long long int(long long int, long long int)">;
+}
+
+// V65 Scalar Instructions.
+
+let Features = V65.Features in {
+  def A6_vcmpbeq_notany : HexagonBuiltin<"int(long long int, long long int)">;
+}
+
+// V66 Scalar Instructions.
+
+let Features = V66.Features in {
+  def F2_dfadd : HexagonBuiltin<"double(double, double)">;
+  def F2_dfsub : HexagonBuiltin<"double(double, double)">;
+  def M2_mnaci : HexagonBuiltin<"int(int, int, int)">;
+  def S2_mask : HexagonBuiltin<"int(unsigned _Constant int, unsigned _Constant int)">;
+}
+
+// V67 Scalar Instructions.
+
+let Features = "audio" in {
+  def A7_clip : HexagonBuiltin<"int(int, unsigned _Constant int)">;
+  def A7_croundd_ri : HexagonBuiltin<"long long int(long long int, unsigned _Constant int)">;
+  def A7_croundd_rr : HexagonBuiltin<"long long int(long long int, int)">;
+  def A7_vclip : HexagonBuiltin<"long long int(long long int, unsigned _Constant int)">;
+}
+let Features = V67.Features in {
+  def F2_dfmax : HexagonBuiltin<"double(double, double)">;
+  def F2_dfmin : HexagonBuiltin<"double(double, double)">;
+  def F2_dfmpyfix : HexagonBuiltin<"double(double, double)">;
+  def F2_dfmpyhh : HexagonBuiltin<"double(double, double, double)">;
+  def F2_dfmpylh : HexagonBuiltin<"double(double, double, double)">;
+  def F2_dfmpyll : HexagonBuiltin<"double(double, double)">;
+}
+let Features = "audio" in {
+  def M7_dcmpyiw : HexagonBuiltin<"long long int(long long int, long long int)">;
+  def M7_dcmpyiw_acc : HexagonBuiltin<"long long int(long long int, long long int, long long int)">;
+  def M7_dcmpyiwc : HexagonBuiltin<"long long int(long long int, long long int)">;
+  def M7_dcmpyiwc_acc : HexagonBuiltin<"long long int(long long int, long long int, long long int)">;
+  def M7_dcmpyrw : HexagonBuiltin<"long long int(long long int, long long int)">;
+  def M7_dcmpyrw_acc : HexagonBuiltin<"long long int(long long int, long long int, long long int)">;
+  def M7_dcmpyrwc : HexagonBuiltin<"long long int(long long int, long long int)">;
+  def M7_dcmpyrwc_acc : HexagonBuiltin<"long long int(long long int, long long int, long long int)">;
+}
+let Features = V67.Features in {
+  def M7_vdmpy : HexagonBuiltin<"long long int(long long int, long long int)">;
+  def M7_vdmpy_acc : HexagonBuiltin<"long long int(long long int, long long int, long long int)">;
+}
+let Features = "audio" in {
+  def M7_wcmpyiw : HexagonBuiltin<"int(long long int, long long int)">;
+  def M7_wcmpyiw_rnd : HexagonBuiltin<"int(long long int, long long int)">;
+  def M7_wcmpyiwc : HexagonBuiltin<"int(long long int, long long int)">;
+  def M7_wcmpyiwc_rnd : HexagonBuiltin<"int(long long int, long long int)">;
+  def M7_wcmpyrw : HexagonBuiltin<"int(long long int, long long int)">;
+  def M7_wcmpyrw_rnd : HexagonBuiltin<"int(long long int, long long int)">;
+  def M7_wcmpyrwc : HexagonBuiltin<"int(long long int, long long int)">;
+  def M7_wcmpyrwc_rnd : HexagonBuiltin<"int(long long int, long long int)">;
+}
+
+// V68 Scalar Instructions.
+
+let Features = V68.Features in {
+  def Y6_dmlink : HexagonBuiltin<"void(void *, void *)">;
+  def Y6_dmpause : HexagonBuiltin<"int()">;
+  def Y6_dmpoll : HexagonBuiltin<"int()">;
+  def Y6_dmresume : HexagonBuiltin<"void(void *)">;
+  def Y6_dmstart : HexagonBuiltin<"void(void *)">;
+  def Y6_dmwait : HexagonBuiltin<"int()">;
+}
+
+// V60 HVX Instructions.
+
+let Features = HVXV60.Features in {
+  def V6_extractw : HexagonBuiltin<"int(_Vector<16, int>, int)">;
+  def V6_extractw_128B : HexagonBuiltin<"int(_Vector<32, int>, int)">;
+  def V6_hi : HexagonBuiltin<"_Vector<16, int>(_Vector<32, int>)">;
+  def V6_hi_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<64, int>)">;
+  def V6_lo : HexagonBuiltin<"_Vector<16, int>(_Vector<32, int>)">;
+  def V6_lo_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<64, int>)">;
+  def V6_lvsplatw : HexagonBuiltin<"_Vector<16, int>(int)">;
+  def V6_lvsplatw_128B : HexagonBuiltin<"_Vector<32, int>(int)">;
+  def V6_pred_and : HexagonBuiltin<"_Vector<64, bool>(_Vector<64, bool>, _Vector<64, bool>)">;
+  def V6_pred_and_128B : HexagonBuiltin<"_Vector<128, bool>(_Vector<128, bool>, _Vector<128, bool>)">;
+  def V6_pred_and_n : HexagonBuiltin<"_Vector<64, bool>(_Vector<64, bool>, _Vector<64, bool>)">;
+  def V6_pred_and_n_128B : HexagonBuiltin<"_Vector<128, bool>(_Vector<128, bool>, _Vector<128, bool>)">;
+  def V6_pred_not : HexagonBuiltin<"_Vector<64, bool>(_Vector<64, bool>)">;
+  def V6_pred_not_128B : HexagonBuiltin<"_Vector<128, bool>(_Vector<128, bool>)">;
+  def V6_pred_or : HexagonBuiltin<"_Vector<64, bool>(_Vector<64, bool>, _Vector<64, bool>)">;
+  def V6_pred_or_128B : HexagonBuiltin<"_Vector<128, bool>(_Vector<128, bool>, _Vector<128, bool>)">;
+  def V6_pred_or_n : HexagonBuiltin<"_Vector<64, bool>(_Vector<64, bool>, _Vector<64, bool>)">;
+  def V6_pred_or_n_128B : HexagonBuiltin<"_Vector<128, bool>(_Vector<128, bool>, _Vector<128, bool>)">;
+  def V6_pred_scalar2 : HexagonBuiltin<"_Vector<64, bool>(int)">;
+  def V6_pred_scalar2_128B : HexagonBuiltin<"_Vector<128, bool>(int)">;
+  def V6_pred_xor : HexagonBuiltin<"_Vector<64, bool>(_Vector<64, bool>, _Vector<64, bool>)">;
+  def V6_pred_xor_128B : HexagonBuiltin<"_Vector<128, bool>(_Vector<128, bool>, _Vector<128, bool>)">;
+  def V6_vS32b_nqpred_ai : HexagonBuiltin<"void(_Vector<64, bool>, void *, _Vector<16, int>)">;
+  def V6_vS32b_nqpred_ai_128B : HexagonBuiltin<"void(_Vector<128, bool>, void *, _Vector<32, int>)">;
+  def V6_vS32b_nt_nqpred_ai : HexagonBuiltin<"void(_Vector<64, bool>, void *, _Vector<16, int>)">;
+  def V6_vS32b_nt_nqpred_ai_128B : HexagonBuiltin<"void(_Vector<128, bool>, void *, _Vector<32, int>)">;
+  def V6_vS32b_nt_qpred_ai : HexagonBuiltin<"void(_Vector<64, bool>, void *, _Vector<16, int>)">;
+  def V6_vS32b_nt_qpred_ai_128B : HexagonBuiltin<"void(_Vector<128, bool>, void *, _Vector<32, int>)">;
+  def V6_vS32b_qpred_ai : HexagonBuiltin<"void(_Vector<64, bool>, void *, _Vector<16, int>)">;
+  def V6_vS32b_qpred_ai_128B : HexagonBuiltin<"void(_Vector<128, bool>, void *, _Vector<32, int>)">;
+  def V6_vabsdiffh : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>, _Vector<16, int>)">;
+  def V6_vabsdiffh_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<32, int>)">;
+  def V6_vabsdiffub : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>, _Vector<16, int>)">;
+  def V6_vabsdiffub_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<32, int>)">;
+  def V6_vabsdiffuh : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>, _Vector<16, int>)">;
+  def V6_vabsdiffuh_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<32, int>)">;
+  def V6_vabsdiffw : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>, _Vector<16, int>)">;
+  def V6_vabsdiffw_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<32, int>)">;
+  def V6_vabsh : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>)">;
+  def V6_vabsh_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>)">;
+  def V6_vabsh_sat : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>)">;
+  def V6_vabsh_sat_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>)">;
+  def V6_vabsw : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>)">;
+  def V6_vabsw_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>)">;
+  def V6_vabsw_sat : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>)">;
+  def V6_vabsw_sat_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>)">;
+  def V6_vaddb : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>, _Vector<16, int>)">;
+  def V6_vaddb_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<32, int>)">;
+  def V6_vaddb_dv : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<32, int>)">;
+  def V6_vaddb_dv_128B : HexagonBuiltin<"_Vector<64, int>(_Vector<64, int>, _Vector<64, int>)">;
+  def V6_vaddbnq : HexagonBuiltin<"_Vector<16, int>(_Vector<64, bool>, _Vector<16, int>, _Vector<16, int>)">;
+  def V6_vaddbnq_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<128, bool>, _Vector<32, int>, _Vector<32, int>)">;
+  def V6_vaddbq : HexagonBuiltin<"_Vector<16, int>(_Vector<64, bool>, _Vector<16, int>, _Vector<16, int>)">;
+  def V6_vaddbq_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<128, bool>, _Vector<32, int>, _Vector<32, int>)">;
+  def V6_vaddh : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>, _Vector<16, int>)">;
+  def V6_vaddh_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<32, int>)">;
+  def V6_vaddh_dv : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<32, int>)">;
+  def V6_vaddh_dv_128B : HexagonBuiltin<"_Vector<64, int>(_Vector<64, int>, _Vector<64, int>)">;
+  def V6_vaddhnq : HexagonBuiltin<"_Vector<16, int>(_Vector<64, bool>, _Vector<16, int>, _Vector<16, int>)">;
+  def V6_vaddhnq_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<128, bool>, _Vector<32, int>, _Vector<32, int>)">;
+  def V6_vaddhq : HexagonBuiltin<"_Vector<16, int>(_Vector<64, bool>, _Vector<16, int>, _Vector<16, int>)">;
+  def V6_vaddhq_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<128, bool>, _Vector<32, int>, _Vector<32, int>)">;
+  def V6_vaddhsat : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>, _Vector<16, int>)">;
+  def V6_vaddhsat_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<32, int>)">;
+  def V6_vaddhsat_dv : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<32, int>)">;
+  def V6_vaddhsat_dv_128B : HexagonBuiltin<"_Vector<64, int>(_Vector<64, int>, _Vector<64, int>)">;
+  def V6_vaddhw : HexagonBuiltin<"_Vector<32, int>(_Vector<16, int>, _Vector<16, int>)">;
+  def V6_vaddhw_128B : HexagonBuiltin<"_Vector<64, int>(_Vector<32, int>, _Vector<32, int>)">;
+  def V6_vaddubh : HexagonBuiltin<"_Vector<32, int>(_Vector<16, int>, _Vector<16, int>)">;
+  def V6_vaddubh_128B : HexagonBuiltin<"_Vector<64, int>(_Vector<32, int>, _Vector<32, int>)">;
+  def V6_vaddubsat : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>, _Vector<16, int>)">;
+  def V6_vaddubsat_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<32, int>)">;
+  def V6_vaddubsat_dv : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<32, int>)">;
+  def V6_vaddubsat_dv_128B : HexagonBuiltin<"_Vector<64, int>(_Vector<64, int>, _Vector<64, int>)">;
+  def V6_vadduhsat : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>, _Vector<16, int>)">;
+  def V6_vadduhsat_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<32, int>)">;
+  def V6_vadduhsat_dv : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<32, int>)">;
+  def V6_vadduhsat_dv_128B : HexagonBuiltin<"_Vector<64, int>(_Vector<64, int>, _Vector<64, int>)">;
+  def V6_vadduhw : HexagonBuiltin<"_Vector<32, int>(_Vector<16, int>, _Vector<16, int>)">;
+  def V6_vadduhw_128B : HexagonBuiltin<"_Vector<64, int>(_Vector<32, int>, _Vector<32, int>)">;
+  def V6_vaddw : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>, _Vector<16, int>)">;
+  def V6_vaddw_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<32, int>)">;
+  def V6_vaddw_dv : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<32, int>)">;
+  def V6_vaddw_dv_128B : HexagonBuiltin<"_Vector<64, int>(_Vector<64, int>, _Vector<64, int>)">;
+  def V6_vaddwnq : HexagonBuiltin<"_Vector<16, int>(_Vector<64, bool>, _Vector<16, int>, _Vector<16, int>)">;
+  def V6_vaddwnq_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<128, bool>, _Vector<32, int>, _Vector<32, int>)">;
+  def V6_vaddwq : HexagonBuiltin<"_Vector<16, int>(_Vector<64, bool>, _Vector<16, int>, _Vector<16, int>)">;
+  def V6_vaddwq_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<128, bool>, _Vector<32, int>, _Vector<32, int>)">;
+  def V6_vaddwsat : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>, _Vector<16, int>)">;
+  def V6_vaddwsat_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<32, int>)">;
+  def V6_vaddwsat_dv : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<32, int>)">;
+  def V6_vaddwsat_dv_128B : HexagonBuiltin<"_Vector<64, int>(_Vector<64, int>, _Vector<64, int>)">;
+  def V6_valignb : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>, _Vector<16, int>, int)">;
+  def V6_valignb_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<32, int>, int)">;
+  def V6_valignbi : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>, _Vector<16, int>, unsigned _Constant int)">;
+  def V6_valignbi_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<32, int>, unsigned _Constant int)">;
+  def V6_vand : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>, _Vector<16, int>)">;
+  def V6_vand_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<32, int>)">;
+  def V6_vandqrt : HexagonBuiltin<"_Vector<16, int>(_Vector<64, bool>, int)">;
+  def V6_vandqrt_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<128, bool>, int)">;
+  def V6_vandqrt_acc : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>, _Vector<64, bool>, int)">;
+  def V6_vandqrt_acc_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<128, bool>, int)">;
+  def V6_vandvrt : HexagonBuiltin<"_Vector<64, bool>(_Vector<16, int>, int)">;
+  def V6_vandvrt_128B : HexagonBuiltin<"_Vector<128, bool>(_Vector<32, int>, int)">;
+  def V6_vandvrt_acc : HexagonBuiltin<"_Vector<64, bool>(_Vector<64, bool>, _Vector<16, int>, int)">;
+  def V6_vandvrt_acc_128B : HexagonBuiltin<"_Vector<128, bool>(_Vector<128, bool>, _Vector<32, int>, int)">;
+  def V6_vaslh : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>, int)">;
+  def V6_vaslh_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, int)">;
+  def V6_vaslhv : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>, _Vector<16, int>)">;
+  def V6_vaslhv_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<32, int>)">;
+  def V6_vaslw : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>, int)">;
+  def V6_vaslw_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, int)">;
+  def V6_vaslw_acc : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>, _Vector<16, int>, int)">;
+  def V6_vaslw_acc_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<32, int>, int)">;
+  def V6_vaslwv : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>, _Vector<16, int>)">;
+  def V6_vaslwv_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<32, int>)">;
+  def V6_vasrh : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>, int)">;
+  def V6_vasrh_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, int)">;
+  def V6_vasrhbrndsat : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>, _Vector<16, int>, int)">;
+  def V6_vasrhbrndsat_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<32, int>, int)">;
+  def V6_vasrhubrndsat : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>, _Vector<16, int>, int)">;
+  def V6_vasrhubrndsat_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<32, int>, int)">;
+  def V6_vasrhubsat : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>, _Vector<16, int>, int)">;
+  def V6_vasrhubsat_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<32, int>, int)">;
+  def V6_vasrhv : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>, _Vector<16, int>)">;
+  def V6_vasrhv_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<32, int>)">;
+  def V6_vasrw : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>, int)">;
+  def V6_vasrw_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, int)">;
+  def V6_vasrw_acc : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>, _Vector<16, int>, int)">;
+  def V6_vasrw_acc_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<32, int>, int)">;
+  def V6_vasrwh : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>, _Vector<16, int>, int)">;
+  def V6_vasrwh_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<32, int>, int)">;
+  def V6_vasrwhrndsat : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>, _Vector<16, int>, int)">;
+  def V6_vasrwhrndsat_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<32, int>, int)">;
+  def V6_vasrwhsat : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>, _Vector<16, int>, int)">;
+  def V6_vasrwhsat_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<32, int>, int)">;
+  def V6_vasrwuhsat : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>, _Vector<16, int>, int)">;
+  def V6_vasrwuhsat_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<32, int>, int)">;
+  def V6_vasrwv : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>, _Vector<16, int>)">;
+  def V6_vasrwv_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<32, int>)">;
+  def V6_vassign : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>)">;
+  def V6_vassign_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>)">;
+  def V6_vassignp : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>)">;
+  def V6_vassignp_128B : HexagonBuiltin<"_Vector<64, int>(_Vector<64, int>)">;
+  def V6_vavgh : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>, _Vector<16, int>)">;
+  def V6_vavgh_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<32, int>)">;
+  def V6_vavghrnd : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>, _Vector<16, int>)">;
+  def V6_vavghrnd_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<32, int>)">;
+  def V6_vavgub : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>, _Vector<16, int>)">;
+  def V6_vavgub_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<32, int>)">;
+  def V6_vavgubrnd : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>, _Vector<16, int>)">;
+  def V6_vavgubrnd_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<32, int>)">;
+  def V6_vavguh : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>, _Vector<16, int>)">;
+  def V6_vavguh_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<32, int>)">;
+  def V6_vavguhrnd : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>, _Vector<16, int>)">;
+  def V6_vavguhrnd_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<32, int>)">;
+  def V6_vavgw : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>, _Vector<16, int>)">;
+  def V6_vavgw_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<32, int>)">;
+  def V6_vavgwrnd : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>, _Vector<16, int>)">;
+  def V6_vavgwrnd_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<32, int>)">;
+  def V6_vcl0h : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>)">;
+  def V6_vcl0h_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>)">;
+  def V6_vcl0w : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>)">;
+  def V6_vcl0w_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>)">;
+  def V6_vcombine : HexagonBuiltin<"_Vector<32, int>(_Vector<16, int>, _Vector<16, int>)">;
+  def V6_vcombine_128B : HexagonBuiltin<"_Vector<64, int>(_Vector<32, int>, _Vector<32, int>)">;
+  def V6_vd0 : HexagonBuiltin<"_Vector<16, int>()">;
+  def V6_vd0_128B : HexagonBuiltin<"_Vector<32, int>()">;
+  def V6_vdealb : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>)">;
+  def V6_vdealb_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>)">;
+  def V6_vdealb4w : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>, _Vector<16, int>)">;
+  def V6_vdealb4w_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<32, int>)">;
+  def V6_vdealh : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>)">;
+  def V6_vdealh_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>)">;
+  def V6_vdealvdd : HexagonBuiltin<"_Vector<32, int>(_Vector<16, int>, _Vector<16, int>, int)">;
+  def V6_vdealvdd_128B : HexagonBuiltin<"_Vector<64, int>(_Vector<32, int>, _Vector<32, int>, int)">;
+  def V6_vdelta : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>, _Vector<16, int>)">;
+  def V6_vdelta_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<32, int>)">;
+  def V6_vdmpybus : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>, int)">;
+  def V6_vdmpybus_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, int)">;
+  def V6_vdmpybus_acc : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>, _Vector<16, int>, int)">;
+  def V6_vdmpybus_acc_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<32, int>, int)">;
+  def V6_vdmpybus_dv : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, int)">;
+  def V6_vdmpybus_dv_128B : HexagonBuiltin<"_Vector<64, int>(_Vector<64, int>, int)">;
+  def V6_vdmpybus_dv_acc : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<32, int>, int)">;
+  def V6_vdmpybus_dv_acc_128B : HexagonBuiltin<"_Vector<64, int>(_Vector<64, int>, _Vector<64, int>, int)">;
+  def V6_vdmpyhb : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>, int)">;
+  def V6_vdmpyhb_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, int)">;
+  def V6_vdmpyhb_acc : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>, _Vector<16, int>, int)">;
+  def V6_vdmpyhb_acc_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<32, int>, int)">;
+  def V6_vdmpyhb_dv : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, int)">;
+  def V6_vdmpyhb_dv_128B : HexagonBuiltin<"_Vector<64, int>(_Vector<64, int>, int)">;
+  def V6_vdmpyhb_dv_acc : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<32, int>, int)">;
+  def V6_vdmpyhb_dv_acc_128B : HexagonBuiltin<"_Vector<64, int>(_Vector<64, int>, _Vector<64, int>, int)">;
+  def V6_vdmpyhisat : HexagonBuiltin<"_Vector<16, int>(_Vector<32, int>, int)">;
+  def V6_vdmpyhisat_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<64, int>, int)">;
+  def V6_vdmpyhisat_acc : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>, _Vector<32, int>, int)">;
+  def V6_vdmpyhisat_acc_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<64, int>, int)">;
+  def V6_vdmpyhsat : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>, int)">;
+  def V6_vdmpyhsat_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, int)">;
+  def V6_vdmpyhsat_acc : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>, _Vector<16, int>, int)">;
+  def V6_vdmpyhsat_acc_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<32, int>, int)">;
+  def V6_vdmpyhsuisat : HexagonBuiltin<"_Vector<16, int>(_Vector<32, int>, int)">;
+  def V6_vdmpyhsuisat_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<64, int>, int)">;
+  def V6_vdmpyhsuisat_acc : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>, _Vector<32, int>, int)">;
+  def V6_vdmpyhsuisat_acc_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<64, int>, int)">;
+  def V6_vdmpyhsusat : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>, int)">;
+  def V6_vdmpyhsusat_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, int)">;
+  def V6_vdmpyhsusat_acc : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>, _Vector<16, int>, int)">;
+  def V6_vdmpyhsusat_acc_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<32, int>, int)">;
+  def V6_vdmpyhvsat : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>, _Vector<16, int>)">;
+  def V6_vdmpyhvsat_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<32, int>)">;
+  def V6_vdmpyhvsat_acc : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>, _Vector<16, int>, _Vector<16, int>)">;
+  def V6_vdmpyhvsat_acc_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<32, int>, _Vector<32, int>)">;
+  def V6_vdsaduh : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, int)">;
+  def V6_vdsaduh_128B : HexagonBuiltin<"_Vector<64, int>(_Vector<64, int>, int)">;
+  def V6_vdsaduh_acc : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<32, int>, int)">;
+  def V6_vdsaduh_acc_128B : HexagonBuiltin<"_Vector<64, int>(_Vector<64, int>, _Vector<64, int>, int)">;
+  def V6_veqb : HexagonBuiltin<"_Vector<64, bool>(_Vector<16, int>, _Vector<16, int>)">;
+  def V6_veqb_128B : HexagonBuiltin<"_Vector<128, bool>(_Vector<32, int>, _Vector<32, int>)">;
+  def V6_veqb_and : HexagonBuiltin<"_Vector<64, bool>(_Vector<64, bool>, _Vector<16, int>, _Vector<16, int>)">;
+  def V6_veqb_and_128B : HexagonBuiltin<"_Vector<128, bool>(_Vector<128, bool>, _Vector<32, int>, _Vector<32, int>)">;
+  def V6_veqb_or : HexagonBuiltin<"_Vector<64, bool>(_Vector<64, bool>, _Vector<16, int>, _Vector<16, int>)">;
+  def V6_veqb_or_128B : HexagonBuiltin<"_Vector<128, bool>(_Vector<128, bool>, _Vector<32, int>, _Vector<32, int>)">;
+  def V6_veqb_xor : HexagonBuiltin<"_Vector<64, bool>(_Vector<64, bool>, _Vector<16, int>, _Vector<16, int>)">;
+  def V6_veqb_xor_128B : HexagonBuiltin<"_Vector<128, bool>(_Vector<128, bool>, _Vector<32, int>, _Vector<32, int>)">;
+  def V6_veqh : HexagonBuiltin<"_Vector<64, bool>(_Vector<16, int>, _Vector<16, int>)">;
+  def V6_veqh_128B : HexagonBuiltin<"_Vector<128, bool>(_Vector<32, int>, _Vector<32, int>)">;
+  def V6_veqh_and : HexagonBuiltin<"_Vector<64, bool>(_Vector<64, bool>, _Vector<16, int>, _Vector<16, int>)">;
+  def V6_veqh_and_128B : HexagonBuiltin<"_Vector<128, bool>(_Vector<128, bool>, _Vector<32, int>, _Vector<32, int>)">;
+  def V6_veqh_or : HexagonBuiltin<"_Vector<64, bool>(_Vector<64, bool>, _Vector<16, int>, _Vector<16, int>)">;
+  def V6_veqh_or_128B : HexagonBuiltin<"_Vector<128, bool>(_Vector<128, bool>, _Vector<32, int>, _Vector<32, int>)">;
+  def V6_veqh_xor : HexagonBuiltin<"_Vector<64, bool>(_Vector<64, bool>, _Vector<16, int>, _Vector<16, int>)">;
+  def V6_veqh_xor_128B : HexagonBuiltin<"_Vector<128, bool>(_Vector<128, bool>, _Vector<32, int>, _Vector<32, int>)">;
+  def V6_veqw : HexagonBuiltin<"_Vector<64, bool>(_Vector<16, int>, _Vector<16, int>)">;
+  def V6_veqw_128B : HexagonBuiltin<"_Vector<128, bool>(_Vector<32, int>, _Vector<32, int>)">;
+  def V6_veqw_and : HexagonBuiltin<"_Vector<64, bool>(_Vector<64, bool>, _Vector<16, int>, _Vector<16, int>)">;
+  def V6_veqw_and_128B : HexagonBuiltin<"_Vector<128, bool>(_Vector<128, bool>, _Vector<32, int>, _Vector<32, int>)">;
+  def V6_veqw_or : HexagonBuiltin<"_Vector<64, bool>(_Vector<64, bool>, _Vector<16, int>, _Vector<16, int>)">;
+  def V6_veqw_or_128B : HexagonBuiltin<"_Vector<128, bool>(_Vector<128, bool>, _Vector<32, int>, _Vector<32, int>)">;
+  def V6_veqw_xor : HexagonBuiltin<"_Vector<64, bool>(_Vector<64, bool>, _Vector<16, int>, _Vector<16, int>)">;
+  def V6_veqw_xor_128B : HexagonBuiltin<"_Vector<128, bool>(_Vector<128, bool>, _Vector<32, int>, _Vector<32, int>)">;
+  def V6_vgtb : HexagonBuiltin<"_Vector<64, bool>(_Vector<16, int>, _Vector<16, int>)">;
+  def V6_vgtb_128B : HexagonBuiltin<"_Vector<128, bool>(_Vector<32, int>, _Vector<32, int>)">;
+  def V6_vgtb_and : HexagonBuiltin<"_Vector<64, bool>(_Vector<64, bool>, _Vector<16, int>, _Vector<16, int>)">;
+  def V6_vgtb_and_128B : HexagonBuiltin<"_Vector<128, bool>(_Vector<128, bool>, _Vector<32, int>, _Vector<32, int>)">;
+  def V6_vgtb_or : HexagonBuiltin<"_Vector<64, bool>(_Vector<64, bool>, _Vector<16, int>, _Vector<16, int>)">;
+  def V6_vgtb_or_128B : HexagonBuiltin<"_Vector<128, bool>(_Vector<128, bool>, _Vector<32, int>, _Vector<32, int>)">;
+  def V6_vgtb_xor : HexagonBuiltin<"_Vector<64, bool>(_Vector<64, bool>, _Vector<16, int>, _Vector<16, int>)">;
+  def V6_vgtb_xor_128B : HexagonBuiltin<"_Vector<128, bool>(_Vector<128, bool>, _Vector<32, int>, _Vector<32, int>)">;
+  def V6_vgth : HexagonBuiltin<"_Vector<64, bool>(_Vector<16, int>, _Vector<16, int>)">;
+  def V6_vgth_128B : HexagonBuiltin<"_Vector<128, bool>(_Vector<32, int>, _Vector<32, int>)">;
+  def V6_vgth_and : HexagonBuiltin<"_Vector<64, bool>(_Vector<64, bool>, _Vector<16, int>, _Vector<16, int>)">;
+  def V6_vgth_and_128B : HexagonBuiltin<"_Vector<128, bool>(_Vector<128, bool>, _Vector<32, int>, _Vector<32, int>)">;
+  def V6_vgth_or : HexagonBuiltin<"_Vector<64, bool>(_Vector<64, bool>, _Vector<16, int>, _Vector<16, int>)">;
+  def V6_vgth_or_128B : HexagonBuiltin<"_Vector<128, bool>(_Vector<128, bool>, _Vector<32, int>, _Vector<32, int>)">;
+  def V6_vgth_xor : HexagonBuiltin<"_Vector<64, bool>(_Vector<64, bool>, _Vector<16, int>, _Vector<16, int>)">;
+  def V6_vgth_xor_128B : HexagonBuiltin<"_Vector<128, bool>(_Vector<128, bool>, _Vector<32, int>, _Vector<32, int>)">;
+  def V6_vgtub : HexagonBuiltin<"_Vector<64, bool>(_Vector<16, int>, _Vector<16, int>)">;
+  def V6_vgtub_128B : HexagonBuiltin<"_Vector<128, bool>(_Vector<32, int>, _Vector<32, int>)">;
+  def V6_vgtub_and : HexagonBuiltin<"_Vector<64, bool>(_Vector<64, bool>, _Vector<16, int>, _Vector<16, int>)">;
+  def V6_vgtub_and_128B : HexagonBuiltin<"_Vector<128, bool>(_Vector<128, bool>, _Vector<32, int>, _Vector<32, int>)">;
+  def V6_vgtub_or : HexagonBuiltin<"_Vector<64, bool>(_Vector<64, bool>, _Vector<16, int>, _Vector<16, int>)">;
+  def V6_vgtub_or_128B : HexagonBuiltin<"_Vector<128, bool>(_Vector<128, bool>, _Vector<32, int>, _Vector<32, int>)">;
+  def V6_vgtub_xor : HexagonBuiltin<"_Vector<64, bool>(_Vector<64, bool>, _Vector<16, int>, _Vector<16, int>)">;
+  def V6_vgtub_xor_128B : HexagonBuiltin<"_Vector<128, bool>(_Vector<128, bool>, _Vector<32, int>, _Vector<32, int>)">;
+  def V6_vgtuh : HexagonBuiltin<"_Vector<64, bool>(_Vector<16, int>, _Vector<16, int>)">;
+  def V6_vgtuh_128B : HexagonBuiltin<"_Vector<128, bool>(_Vector<32, int>, _Vector<32, int>)">;
+  def V6_vgtuh_and : HexagonBuiltin<"_Vector<64, bool>(_Vector<64, bool>, _Vector<16, int>, _Vector<16, int>)">;
+  def V6_vgtuh_and_128B : HexagonBuiltin<"_Vector<128, bool>(_Vector<128, bool>, _Vector<32, int>, _Vector<32, int>)">;
+  def V6_vgtuh_or : HexagonBuiltin<"_Vector<64, bool>(_Vector<64, bool>, _Vector<16, int>, _Vector<16, int>)">;
+  def V6_vgtuh_or_128B : HexagonBuiltin<"_Vector<128, bool>(_Vector<128, bool>, _Vector<32, int>, _Vector<32, int>)">;
+  def V6_vgtuh_xor : HexagonBuiltin<"_Vector<64, bool>(_Vector<64, bool>, _Vector<16, int>, _Vector<16, int>)">;
+  def V6_vgtuh_xor_128B : HexagonBuiltin<"_Vector<128, bool>(_Vector<128, bool>, _Vector<32, int>, _Vector<32, int>)">;
+  def V6_vgtuw : HexagonBuiltin<"_Vector<64, bool>(_Vector<16, int>, _Vector<16, int>)">;
+  def V6_vgtuw_128B : HexagonBuiltin<"_Vector<128, bool>(_Vector<32, int>, _Vector<32, int>)">;
+  def V6_vgtuw_and : HexagonBuiltin<"_Vector<64, bool>(_Vector<64, bool>, _Vector<16, int>, _Vector<16, int>)">;
+  def V6_vgtuw_and_128B : HexagonBuiltin<"_Vector<128, bool>(_Vector<128, bool>, _Vector<32, int>, _Vector<32, int>)">;
+  def V6_vgtuw_or : HexagonBuiltin<"_Vector<64, bool>(_Vector<64, bool>, _Vector<16, int>, _Vector<16, int>)">;
+  def V6_vgtuw_or_128B : HexagonBuiltin<"_Vector<128, bool>(_Vector<128, bool>, _Vector<32, int>, _Vector<32, int>)">;
+  def V6_vgtuw_xor : HexagonBuiltin<"_Vector<64, bool>(_Vector<64, bool>, _Vector<16, int>, _Vector<16, int>)">;
+  def V6_vgtuw_xor_128B : HexagonBuiltin<"_Vector<128, bool>(_Vector<128, bool>, _Vector<32, int>, _Vector<32, int>)">;
+  def V6_vgtw : HexagonBuiltin<"_Vector<64, bool>(_Vector<16, int>, _Vector<16, int>)">;
+  def V6_vgtw_128B : HexagonBuiltin<"_Vector<128, bool>(_Vector<32, int>, _Vector<32, int>)">;
+  def V6_vgtw_and : HexagonBuiltin<"_Vector<64, bool>(_Vector<64, bool>, _Vector<16, int>, _Vector<16, int>)">;
+  def V6_vgtw_and_128B : HexagonBuiltin<"_Vector<128, bool>(_Vector<128, bool>, _Vector<32, int>, _Vector<32, int>)">;
+  def V6_vgtw_or : HexagonBuiltin<"_Vector<64, bool>(_Vector<64, bool>, _Vector<16, int>, _Vector<16, int>)">;
+  def V6_vgtw_or_128B : HexagonBuiltin<"_Vector<128, bool>(_Vector<128, bool>, _Vector<32, int>, _Vector<32, int>)">;
+  def V6_vgtw_xor : HexagonBuiltin<"_Vector<64, bool>(_Vector<64, bool>, _Vector<16, int>, _Vector<16, int>)">;
+  def V6_vgtw_xor_128B : HexagonBuiltin<"_Vector<128, bool>(_Vector<128, bool>, _Vector<32, int>, _Vector<32, int>)">;
+  def V6_vinsertwr : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>, int)">;
+  def V6_vinsertwr_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, int)">;
+  def V6_vlalignb : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>, _Vector<16, int>, int)">;
+  def V6_vlalignb_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<32, int>, int)">;
+  def V6_vlalignbi : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>, _Vector<16, int>, unsigned _Constant int)">;
+  def V6_vlalignbi_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<32, int>, unsigned _Constant int)">;
+  def V6_vlsrh : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>, int)">;
+  def V6_vlsrh_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, int)">;
+  def V6_vlsrhv : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>, _Vector<16, int>)">;
+  def V6_vlsrhv_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<32, int>)">;
+  def V6_vlsrw : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>, int)">;
+  def V6_vlsrw_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, int)">;
+  def V6_vlsrwv : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>, _Vector<16, int>)">;
+  def V6_vlsrwv_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<32, int>)">;
+  def V6_vlutvvb : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>, _Vector<16, int>, int)">;
+  def V6_vlutvvb_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<32, int>, int)">;
+  def V6_vlutvvb_oracc : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>, _Vector<16, int>, _Vector<16, int>, int)">;
+  def V6_vlutvvb_oracc_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<32, int>, _Vector<32, int>, int)">;
+  def V6_vlutvwh : HexagonBuiltin<"_Vector<32, int>(_Vector<16, int>, _Vector<16, int>, int)">;
+  def V6_vlutvwh_128B : HexagonBuiltin<"_Vector<64, int>(_Vector<32, int>, _Vector<32, int>, int)">;
+  def V6_vlutvwh_oracc : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<16, int>, _Vector<16, int>, int)">;
+  def V6_vlutvwh_oracc_128B : HexagonBuiltin<"_Vector<64, int>(_Vector<64, int>, _Vector<32, int>, _Vector<32, int>, int)">;
+  def V6_vmaxh : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>, _Vector<16, int>)">;
+  def V6_vmaxh_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<32, int>)">;
+  def V6_vmaxub : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>, _Vector<16, int>)">;
+  def V6_vmaxub_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<32, int>)">;
+  def V6_vmaxuh : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>, _Vector<16, int>)">;
+  def V6_vmaxuh_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<32, int>)">;
+  def V6_vmaxw : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>, _Vector<16, int>)">;
+  def V6_vmaxw_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<32, int>)">;
+  def V6_vminh : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>, _Vector<16, int>)">;
+  def V6_vminh_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<32, int>)">;
+  def V6_vminub : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>, _Vector<16, int>)">;
+  def V6_vminub_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<32, int>)">;
+  def V6_vminuh : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>, _Vector<16, int>)">;
+  def V6_vminuh_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<32, int>)">;
+  def V6_vminw : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>, _Vector<16, int>)">;
+  def V6_vminw_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<32, int>)">;
+  def V6_vmpabus : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, int)">;
+  def V6_vmpabus_128B : HexagonBuiltin<"_Vector<64, int>(_Vector<64, int>, int)">;
+  def V6_vmpabus_acc : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<32, int>, int)">;
+  def V6_vmpabus_acc_128B : HexagonBuiltin<"_Vector<64, int>(_Vector<64, int>, _Vector<64, int>, int)">;
+  def V6_vmpabusv : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<32, int>)">;
+  def V6_vmpabusv_128B : HexagonBuiltin<"_Vector<64, int>(_Vector<64, int>, _Vector<64, int>)">;
+  def V6_vmpabuuv : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<32, int>)">;
+  def V6_vmpabuuv_128B : HexagonBuiltin<"_Vector<64, int>(_Vector<64, int>, _Vector<64, int>)">;
+  def V6_vmpahb : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, int)">;
+  def V6_vmpahb_128B : HexagonBuiltin<"_Vector<64, int>(_Vector<64, int>, int)">;
+  def V6_vmpahb_acc : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<32, int>, int)">;
+  def V6_vmpahb_acc_128B : HexagonBuiltin<"_Vector<64, int>(_Vector<64, int>, _Vector<64, int>, int)">;
+  def V6_vmpybus : HexagonBuiltin<"_Vector<32, int>(_Vector<16, int>, int)">;
+  def V6_vmpybus_128B : HexagonBuiltin<"_Vector<64, int>(_Vector<32, int>, int)">;
+  def V6_vmpybus_acc : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<16, int>, int)">;
+  def V6_vmpybus_acc_128B : HexagonBuiltin<"_Vector<64, int>(_Vector<64, int>, _Vector<32, int>, int)">;
+  def V6_vmpybusv : HexagonBuiltin<"_Vector<32, int>(_Vector<16, int>, _Vector<16, int>)">;
+  def V6_vmpybusv_128B : HexagonBuiltin<"_Vector<64, int>(_Vector<32, int>, _Vector<32, int>)">;
+  def V6_vmpybusv_acc : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<16, int>, _Vector<16, int>)">;
+  def V6_vmpybusv_acc_128B : HexagonBuiltin<"_Vector<64, int>(_Vector<64, int>, _Vector<32, int>, _Vector<32, int>)">;
+  def V6_vmpybv : HexagonBuiltin<"_Vector<32, int>(_Vector<16, int>, _Vector<16, int>)">;
+  def V6_vmpybv_128B : HexagonBuiltin<"_Vector<64, int>(_Vector<32, int>, _Vector<32, int>)">;
+  def V6_vmpybv_acc : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<16, int>, _Vector<16, int>)">;
+  def V6_vmpybv_acc_128B : HexagonBuiltin<"_Vector<64, int>(_Vector<64, int>, _Vector<32, int>, _Vector<32, int>)">;
+  def V6_vmpyewuh : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>, _Vector<16, int>)">;
+  def V6_vmpyewuh_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<32, int>)">;
+  def V6_vmpyh : HexagonBuiltin<"_Vector<32, int>(_Vector<16, int>, int)">;
+  def V6_vmpyh_128B : HexagonBuiltin<"_Vector<64, int>(_Vector<32, int>, int)">;
+  def V6_vmpyhsat_acc : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<16, int>, int)">;
+  def V6_vmpyhsat_acc_128B : HexagonBuiltin<"_Vector<64, int>(_Vector<64, int>, _Vector<32, int>, int)">;
+  def V6_vmpyhsrs : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>, int)">;
+  def V6_vmpyhsrs_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, int)">;
+  def V6_vmpyhss : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>, int)">;
+  def V6_vmpyhss_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, int)">;
+  def V6_vmpyhus : HexagonBuiltin<"_Vector<32, int>(_Vector<16, int>, _Vector<16, int>)">;
+  def V6_vmpyhus_128B : HexagonBuiltin<"_Vector<64, int>(_Vector<32, int>, _Vector<32, int>)">;
+  def V6_vmpyhus_acc : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<16, int>, _Vector<16, int>)">;
+  def V6_vmpyhus_acc_128B : HexagonBuiltin<"_Vector<64, int>(_Vector<64, int>, _Vector<32, int>, _Vector<32, int>)">;
+  def V6_vmpyhv : HexagonBuiltin<"_Vector<32, int>(_Vector<16, int>, _Vector<16, int>)">;
+  def V6_vmpyhv_128B : HexagonBuiltin<"_Vector<64, int>(_Vector<32, int>, _Vector<32, int>)">;
+  def V6_vmpyhv_acc : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<16, int>, _Vector<16, int>)">;
+  def V6_vmpyhv_acc_128B : HexagonBuiltin<"_Vector<64, int>(_Vector<64, int>, _Vector<32, int>, _Vector<32, int>)">;
+  def V6_vmpyhvsrs : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>, _Vector<16, int>)">;
+  def V6_vmpyhvsrs_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<32, int>)">;
+  def V6_vmpyieoh : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>, _Vector<16, int>)">;
+  def V6_vmpyieoh_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<32, int>)">;
+  def V6_vmpyiewh_acc : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>, _Vector<16, int>, _Vector<16, int>)">;
+  def V6_vmpyiewh_acc_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<32, int>, _Vector<32, int>)">;
+  def V6_vmpyiewuh : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>, _Vector<16, int>)">;
+  def V6_vmpyiewuh_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<32, int>)">;
+  def V6_vmpyiewuh_acc : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>, _Vector<16, int>, _Vector<16, int>)">;
+  def V6_vmpyiewuh_acc_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<32, int>, _Vector<32, int>)">;
+  def V6_vmpyih : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>, _Vector<16, int>)">;
+  def V6_vmpyih_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<32, int>)">;
+  def V6_vmpyih_acc : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>, _Vector<16, int>, _Vector<16, int>)">;
+  def V6_vmpyih_acc_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<32, int>, _Vector<32, int>)">;
+  def V6_vmpyihb : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>, int)">;
+  def V6_vmpyihb_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, int)">;
+  def V6_vmpyihb_acc : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>, _Vector<16, int>, int)">;
+  def V6_vmpyihb_acc_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<32, int>, int)">;
+  def V6_vmpyiowh : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>, _Vector<16, int>)">;
+  def V6_vmpyiowh_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<32, int>)">;
+  def V6_vmpyiwb : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>, int)">;
+  def V6_vmpyiwb_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, int)">;
+  def V6_vmpyiwb_acc : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>, _Vector<16, int>, int)">;
+  def V6_vmpyiwb_acc_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<32, int>, int)">;
+  def V6_vmpyiwh : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>, int)">;
+  def V6_vmpyiwh_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, int)">;
+  def V6_vmpyiwh_acc : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>, _Vector<16, int>, int)">;
+  def V6_vmpyiwh_acc_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<32, int>, int)">;
+  def V6_vmpyowh : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>, _Vector<16, int>)">;
+  def V6_vmpyowh_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<32, int>)">;
+  def V6_vmpyowh_rnd : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>, _Vector<16, int>)">;
+  def V6_vmpyowh_rnd_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<32, int>)">;
+  def V6_vmpyowh_rnd_sacc : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>, _Vector<16, int>, _Vector<16, int>)">;
+  def V6_vmpyowh_rnd_sacc_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<32, int>, _Vector<32, int>)">;
+  def V6_vmpyowh_sacc : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>, _Vector<16, int>, _Vector<16, int>)">;
+  def V6_vmpyowh_sacc_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<32, int>, _Vector<32, int>)">;
+  def V6_vmpyub : HexagonBuiltin<"_Vector<32, int>(_Vector<16, int>, int)">;
+  def V6_vmpyub_128B : HexagonBuiltin<"_Vector<64, int>(_Vector<32, int>, int)">;
+  def V6_vmpyub_acc : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<16, int>, int)">;
+  def V6_vmpyub_acc_128B : HexagonBuiltin<"_Vector<64, int>(_Vector<64, int>, _Vector<32, int>, int)">;
+  def V6_vmpyubv : HexagonBuiltin<"_Vector<32, int>(_Vector<16, int>, _Vector<16, int>)">;
+  def V6_vmpyubv_128B : HexagonBuiltin<"_Vector<64, int>(_Vector<32, int>, _Vector<32, int>)">;
+  def V6_vmpyubv_acc : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<16, int>, _Vector<16, int>)">;
+  def V6_vmpyubv_acc_128B : HexagonBuiltin<"_Vector<64, int>(_Vector<64, int>, _Vector<32, int>, _Vector<32, int>)">;
+  def V6_vmpyuh : HexagonBuiltin<"_Vector<32, int>(_Vector<16, int>, int)">;
+  def V6_vmpyuh_128B : HexagonBuiltin<"_Vector<64, int>(_Vector<32, int>, int)">;
+  def V6_vmpyuh_acc : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<16, int>, int)">;
+  def V6_vmpyuh_acc_128B : HexagonBuiltin<"_Vector<64, int>(_Vector<64, int>, _Vector<32, int>, int)">;
+  def V6_vmpyuhv : HexagonBuiltin<"_Vector<32, int>(_Vector<16, int>, _Vector<16, int>)">;
+  def V6_vmpyuhv_128B : HexagonBuiltin<"_Vector<64, int>(_Vector<32, int>, _Vector<32, int>)">;
+  def V6_vmpyuhv_acc : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<16, int>, _Vector<16, int>)">;
+  def V6_vmpyuhv_acc_128B : HexagonBuiltin<"_Vector<64, int>(_Vector<64, int>, _Vector<32, int>, _Vector<32, int>)">;
+  def V6_vmux : HexagonBuiltin<"_Vector<16, int>(_Vector<64, bool>, _Vector<16, int>, _Vector<16, int>)">;
+  def V6_vmux_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<128, bool>, _Vector<32, int>, _Vector<32, int>)">;
+  def V6_vnavgh : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>, _Vector<16, int>)">;
+  def V6_vnavgh_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<32, int>)">;
+  def V6_vnavgub : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>, _Vector<16, int>)">;
+  def V6_vnavgub_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<32, int>)">;
+  def V6_vnavgw : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>, _Vector<16, int>)">;
+  def V6_vnavgw_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<32, int>)">;
+  def V6_vnormamth : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>)">;
+  def V6_vnormamth_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>)">;
+  def V6_vnormamtw : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>)">;
+  def V6_vnormamtw_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>)">;
+  def V6_vnot : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>)">;
+  def V6_vnot_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>)">;
+  def V6_vor : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>, _Vector<16, int>)">;
+  def V6_vor_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<32, int>)">;
+  def V6_vpackeb : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>, _Vector<16, int>)">;
+  def V6_vpackeb_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<32, int>)">;
+  def V6_vpackeh : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>, _Vector<16, int>)">;
+  def V6_vpackeh_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<32, int>)">;
+  def V6_vpackhb_sat : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>, _Vector<16, int>)">;
+  def V6_vpackhb_sat_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<32, int>)">;
+  def V6_vpackhub_sat : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>, _Vector<16, int>)">;
+  def V6_vpackhub_sat_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<32, int>)">;
+  def V6_vpackob : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>, _Vector<16, int>)">;
+  def V6_vpackob_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<32, int>)">;
+  def V6_vpackoh : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>, _Vector<16, int>)">;
+  def V6_vpackoh_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<32, int>)">;
+  def V6_vpackwh_sat : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>, _Vector<16, int>)">;
+  def V6_vpackwh_sat_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<32, int>)">;
+  def V6_vpackwuh_sat : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>, _Vector<16, int>)">;
+  def V6_vpackwuh_sat_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<32, int>)">;
+  def V6_vpopcounth : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>)">;
+  def V6_vpopcounth_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>)">;
+  def V6_vrdelta : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>, _Vector<16, int>)">;
+  def V6_vrdelta_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<32, int>)">;
+  def V6_vrmpybus : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>, int)">;
+  def V6_vrmpybus_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, int)">;
+  def V6_vrmpybus_acc : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>, _Vector<16, int>, int)">;
+  def V6_vrmpybus_acc_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<32, int>, int)">;
+  def V6_vrmpybusi : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, int, unsigned _Constant int)">;
+  def V6_vrmpybusi_128B : HexagonBuiltin<"_Vector<64, int>(_Vector<64, int>, int, unsigned _Constant int)">;
+  def V6_vrmpybusi_acc : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<32, int>, int, unsigned _Constant int)">;
+  def V6_vrmpybusi_acc_128B : HexagonBuiltin<"_Vector<64, int>(_Vector<64, int>, _Vector<64, int>, int, unsigned _Constant int)">;
+  def V6_vrmpybusv : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>, _Vector<16, int>)">;
+  def V6_vrmpybusv_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<32, int>)">;
+  def V6_vrmpybusv_acc : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>, _Vector<16, int>, _Vector<16, int>)">;
+  def V6_vrmpybusv_acc_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<32, int>, _Vector<32, int>)">;
+  def V6_vrmpybv : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>, _Vector<16, int>)">;
+  def V6_vrmpybv_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<32, int>)">;
+  def V6_vrmpybv_acc : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>, _Vector<16, int>, _Vector<16, int>)">;
+  def V6_vrmpybv_acc_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<32, int>, _Vector<32, int>)">;
+  def V6_vrmpyub : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>, int)">;
+  def V6_vrmpyub_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, int)">;
+  def V6_vrmpyub_acc : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>, _Vector<16, int>, int)">;
+  def V6_vrmpyub_acc_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<32, int>, int)">;
+  def V6_vrmpyubi : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, int, unsigned _Constant int)">;
+  def V6_vrmpyubi_128B : HexagonBuiltin<"_Vector<64, int>(_Vector<64, int>, int, unsigned _Constant int)">;
+  def V6_vrmpyubi_acc : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<32, int>, int, unsigned _Constant int)">;
+  def V6_vrmpyubi_acc_128B : HexagonBuiltin<"_Vector<64, int>(_Vector<64, int>, _Vector<64, int>, int, unsigned _Constant int)">;
+  def V6_vrmpyubv : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>, _Vector<16, int>)">;
+  def V6_vrmpyubv_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<32, int>)">;
+  def V6_vrmpyubv_acc : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>, _Vector<16, int>, _Vector<16, int>)">;
+  def V6_vrmpyubv_acc_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<32, int>, _Vector<32, int>)">;
+  def V6_vror : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>, int)">;
+  def V6_vror_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, int)">;
+  def V6_vroundhb : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>, _Vector<16, int>)">;
+  def V6_vroundhb_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<32, int>)">;
+  def V6_vroundhub : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>, _Vector<16, int>)">;
+  def V6_vroundhub_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<32, int>)">;
+  def V6_vroundwh : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>, _Vector<16, int>)">;
+  def V6_vroundwh_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<32, int>)">;
+  def V6_vroundwuh : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>, _Vector<16, int>)">;
+  def V6_vroundwuh_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<32, int>)">;
+  def V6_vrsadubi : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, int, unsigned _Constant int)">;
+  def V6_vrsadubi_128B : HexagonBuiltin<"_Vector<64, int>(_Vector<64, int>, int, unsigned _Constant int)">;
+  def V6_vrsadubi_acc : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<32, int>, int, unsigned _Constant int)">;
+  def V6_vrsadubi_acc_128B : HexagonBuiltin<"_Vector<64, int>(_Vector<64, int>, _Vector<64, int>, int, unsigned _Constant int)">;
+  def V6_vsathub : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>, _Vector<16, int>)">;
+  def V6_vsathub_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<32, int>)">;
+  def V6_vsatwh : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>, _Vector<16, int>)">;
+  def V6_vsatwh_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<32, int>)">;
+  def V6_vsb : HexagonBuiltin<"_Vector<32, int>(_Vector<16, int>)">;
+  def V6_vsb_128B : HexagonBuiltin<"_Vector<64, int>(_Vector<32, int>)">;
+  def V6_vsh : HexagonBuiltin<"_Vector<32, int>(_Vector<16, int>)">;
+  def V6_vsh_128B : HexagonBuiltin<"_Vector<64, int>(_Vector<32, int>)">;
+  def V6_vshufeh : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>, _Vector<16, int>)">;
+  def V6_vshufeh_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<32, int>)">;
+  def V6_vshuffb : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>)">;
+  def V6_vshuffb_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>)">;
+  def V6_vshuffeb : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>, _Vector<16, int>)">;
+  def V6_vshuffeb_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<32, int>)">;
+  def V6_vshuffh : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>)">;
+  def V6_vshuffh_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>)">;
+  def V6_vshuffob : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>, _Vector<16, int>)">;
+  def V6_vshuffob_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<32, int>)">;
+  def V6_vshuffvdd : HexagonBuiltin<"_Vector<32, int>(_Vector<16, int>, _Vector<16, int>, int)">;
+  def V6_vshuffvdd_128B : HexagonBuiltin<"_Vector<64, int>(_Vector<32, int>, _Vector<32, int>, int)">;
+  def V6_vshufoeb : HexagonBuiltin<"_Vector<32, int>(_Vector<16, int>, _Vector<16, int>)">;
+  def V6_vshufoeb_128B : HexagonBuiltin<"_Vector<64, int>(_Vector<32, int>, _Vector<32, int>)">;
+  def V6_vshufoeh : HexagonBuiltin<"_Vector<32, int>(_Vector<16, int>, _Vector<16, int>)">;
+  def V6_vshufoeh_128B : HexagonBuiltin<"_Vector<64, int>(_Vector<32, int>, _Vector<32, int>)">;
+  def V6_vshufoh : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>, _Vector<16, int>)">;
+  def V6_vshufoh_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<32, int>)">;
+  def V6_vsubb : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>, _Vector<16, int>)">;
+  def V6_vsubb_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<32, int>)">;
+  def V6_vsubb_dv : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<32, int>)">;
+  def V6_vsubb_dv_128B : HexagonBuiltin<"_Vector<64, int>(_Vector<64, int>, _Vector<64, int>)">;
+  def V6_vsubbnq : HexagonBuiltin<"_Vector<16, int>(_Vector<64, bool>, _Vector<16, int>, _Vector<16, int>)">;
+  def V6_vsubbnq_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<128, bool>, _Vector<32, int>, _Vector<32, int>)">;
+  def V6_vsubbq : HexagonBuiltin<"_Vector<16, int>(_Vector<64, bool>, _Vector<16, int>, _Vector<16, int>)">;
+  def V6_vsubbq_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<128, bool>, _Vector<32, int>, _Vector<32, int>)">;
+  def V6_vsubh : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>, _Vector<16, int>)">;
+  def V6_vsubh_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<32, int>)">;
+  def V6_vsubh_dv : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<32, int>)">;
+  def V6_vsubh_dv_128B : HexagonBuiltin<"_Vector<64, int>(_Vector<64, int>, _Vector<64, int>)">;
+  def V6_vsubhnq : HexagonBuiltin<"_Vector<16, int>(_Vector<64, bool>, _Vector<16, int>, _Vector<16, int>)">;
+  def V6_vsubhnq_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<128, bool>, _Vector<32, int>, _Vector<32, int>)">;
+  def V6_vsubhq : HexagonBuiltin<"_Vector<16, int>(_Vector<64, bool>, _Vector<16, int>, _Vector<16, int>)">;
+  def V6_vsubhq_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<128, bool>, _Vector<32, int>, _Vector<32, int>)">;
+  def V6_vsubhsat : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>, _Vector<16, int>)">;
+  def V6_vsubhsat_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<32, int>)">;
+  def V6_vsubhsat_dv : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<32, int>)">;
+  def V6_vsubhsat_dv_128B : HexagonBuiltin<"_Vector<64, int>(_Vector<64, int>, _Vector<64, int>)">;
+  def V6_vsubhw : HexagonBuiltin<"_Vector<32, int>(_Vector<16, int>, _Vector<16, int>)">;
+  def V6_vsubhw_128B : HexagonBuiltin<"_Vector<64, int>(_Vector<32, int>, _Vector<32, int>)">;
+  def V6_vsububh : HexagonBuiltin<"_Vector<32, int>(_Vector<16, int>, _Vector<16, int>)">;
+  def V6_vsububh_128B : HexagonBuiltin<"_Vector<64, int>(_Vector<32, int>, _Vector<32, int>)">;
+  def V6_vsububsat : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>, _Vector<16, int>)">;
+  def V6_vsububsat_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<32, int>)">;
+  def V6_vsububsat_dv : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<32, int>)">;
+  def V6_vsububsat_dv_128B : HexagonBuiltin<"_Vector<64, int>(_Vector<64, int>, _Vector<64, int>)">;
+  def V6_vsubuhsat : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>, _Vector<16, int>)">;
+  def V6_vsubuhsat_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<32, int>)">;
+  def V6_vsubuhsat_dv : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<32, int>)">;
+  def V6_vsubuhsat_dv_128B : HexagonBuiltin<"_Vector<64, int>(_Vector<64, int>, _Vector<64, int>)">;
+  def V6_vsubuhw : HexagonBuiltin<"_Vector<32, int>(_Vector<16, int>, _Vector<16, int>)">;
+  def V6_vsubuhw_128B : HexagonBuiltin<"_Vector<64, int>(_Vector<32, int>, _Vector<32, int>)">;
+  def V6_vsubw : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>, _Vector<16, int>)">;
+  def V6_vsubw_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<32, int>)">;
+  def V6_vsubw_dv : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<32, int>)">;
+  def V6_vsubw_dv_128B : HexagonBuiltin<"_Vector<64, int>(_Vector<64, int>, _Vector<64, int>)">;
+  def V6_vsubwnq : HexagonBuiltin<"_Vector<16, int>(_Vector<64, bool>, _Vector<16, int>, _Vector<16, int>)">;
+  def V6_vsubwnq_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<128, bool>, _Vector<32, int>, _Vector<32, int>)">;
+  def V6_vsubwq : HexagonBuiltin<"_Vector<16, int>(_Vector<64, bool>, _Vector<16, int>, _Vector<16, int>)">;
+  def V6_vsubwq_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<128, bool>, _Vector<32, int>, _Vector<32, int>)">;
+  def V6_vsubwsat : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>, _Vector<16, int>)">;
+  def V6_vsubwsat_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<32, int>)">;
+  def V6_vsubwsat_dv : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<32, int>)">;
+  def V6_vsubwsat_dv_128B : HexagonBuiltin<"_Vector<64, int>(_Vector<64, int>, _Vector<64, int>)">;
+  def V6_vswap : HexagonBuiltin<"_Vector<32, int>(_Vector<64, bool>, _Vector<16, int>, _Vector<16, int>)">;
+  def V6_vswap_128B : HexagonBuiltin<"_Vector<64, int>(_Vector<128, bool>, _Vector<32, int>, _Vector<32, int>)">;
+  def V6_vtmpyb : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, int)">;
+  def V6_vtmpyb_128B : HexagonBuiltin<"_Vector<64, int>(_Vector<64, int>, int)">;
+  def V6_vtmpyb_acc : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<32, int>, int)">;
+  def V6_vtmpyb_acc_128B : HexagonBuiltin<"_Vector<64, int>(_Vector<64, int>, _Vector<64, int>, int)">;
+  def V6_vtmpybus : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, int)">;
+  def V6_vtmpybus_128B : HexagonBuiltin<"_Vector<64, int>(_Vector<64, int>, int)">;
+  def V6_vtmpybus_acc : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<32, int>, int)">;
+  def V6_vtmpybus_acc_128B : HexagonBuiltin<"_Vector<64, int>(_Vector<64, int>, _Vector<64, int>, int)">;
+  def V6_vtmpyhb : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, int)">;
+  def V6_vtmpyhb_128B : HexagonBuiltin<"_Vector<64, int>(_Vector<64, int>, int)">;
+  def V6_vtmpyhb_acc : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<32, int>, int)">;
+  def V6_vtmpyhb_acc_128B : HexagonBuiltin<"_Vector<64, int>(_Vector<64, int>, _Vector<64, int>, int)">;
+  def V6_vunpackb : HexagonBuiltin<"_Vector<32, int>(_Vector<16, int>)">;
+  def V6_vunpackb_128B : HexagonBuiltin<"_Vector<64, int>(_Vector<32, int>)">;
+  def V6_vunpackh : HexagonBuiltin<"_Vector<32, int>(_Vector<16, int>)">;
+  def V6_vunpackh_128B : HexagonBuiltin<"_Vector<64, int>(_Vector<32, int>)">;
+  def V6_vunpackob : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<16, int>)">;
+  def V6_vunpackob_128B : HexagonBuiltin<"_Vector<64, int>(_Vector<64, int>, _Vector<32, int>)">;
+  def V6_vunpackoh : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<16, int>)">;
+  def V6_vunpackoh_128B : HexagonBuiltin<"_Vector<64, int>(_Vector<64, int>, _Vector<32, int>)">;
+  def V6_vunpackub : HexagonBuiltin<"_Vector<32, int>(_Vector<16, int>)">;
+  def V6_vunpackub_128B : HexagonBuiltin<"_Vector<64, int>(_Vector<32, int>)">;
+  def V6_vunpackuh : HexagonBuiltin<"_Vector<32, int>(_Vector<16, int>)">;
+  def V6_vunpackuh_128B : HexagonBuiltin<"_Vector<64, int>(_Vector<32, int>)">;
+  def V6_vxor : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>, _Vector<16, int>)">;
+  def V6_vxor_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<32, int>)">;
+  def V6_vzb : HexagonBuiltin<"_Vector<32, int>(_Vector<16, int>)">;
+  def V6_vzb_128B : HexagonBuiltin<"_Vector<64, int>(_Vector<32, int>)">;
+  def V6_vzh : HexagonBuiltin<"_Vector<32, int>(_Vector<16, int>)">;
+  def V6_vzh_128B : HexagonBuiltin<"_Vector<64, int>(_Vector<32, int>)">;
+}
+
+// V62 HVX Instructions.
+
+let Features = HVXV62.Features in {
+  def V6_lvsplatb : HexagonBuiltin<"_Vector<16, int>(int)">;
+  def V6_lvsplatb_128B : HexagonBuiltin<"_Vector<32, int>(int)">;
+  def V6_lvsplath : HexagonBuiltin<"_Vector<16, int>(int)">;
+  def V6_lvsplath_128B : HexagonBuiltin<"_Vector<32, int>(int)">;
+  def V6_pred_scalar2v2 : HexagonBuiltin<"_Vector<64, bool>(int)">;
+  def V6_pred_scalar2v2_128B : HexagonBuiltin<"_Vector<128, bool>(int)">;
+  def V6_shuffeqh : HexagonBuiltin<"_Vector<64, bool>(_Vector<64, bool>, _Vector<64, bool>)">;
+  def V6_shuffeqh_128B : HexagonBuiltin<"_Vector<128, bool>(_Vector<128, bool>, _Vector<128, bool>)">;
+  def V6_shuffeqw : HexagonBuiltin<"_Vector<64, bool>(_Vector<64, bool>, _Vector<64, bool>)">;
+  def V6_shuffeqw_128B : HexagonBuiltin<"_Vector<128, bool>(_Vector<128, bool>, _Vector<128, bool>)">;
+  def V6_vaddbsat : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>, _Vector<16, int>)">;
+  def V6_vaddbsat_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<32, int>)">;
+  def V6_vaddbsat_dv : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<32, int>)">;
+  def V6_vaddbsat_dv_128B : HexagonBuiltin<"_Vector<64, int>(_Vector<64, int>, _Vector<64, int>)">;
+  def V6_vaddcarry : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>, _Vector<16, int>, void *)">;
+  def V6_vaddcarry_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<32, int>, void *)">;
+  def V6_vaddclbh : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>, _Vector<16, int>)">;
+  def V6_vaddclbh_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<32, int>)">;
+  def V6_vaddclbw : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>, _Vector<16, int>)">;
+  def V6_vaddclbw_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<32, int>)">;
+  def V6_vaddhw_acc : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<16, int>, _Vector<16, int>)">;
+  def V6_vaddhw_acc_128B : HexagonBuiltin<"_Vector<64, int>(_Vector<64, int>, _Vector<32, int>, _Vector<32, int>)">;
+  def V6_vaddubh_acc : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<16, int>, _Vector<16, int>)">;
+  def V6_vaddubh_acc_128B : HexagonBuiltin<"_Vector<64, int>(_Vector<64, int>, _Vector<32, int>, _Vector<32, int>)">;
+  def V6_vaddububb_sat : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>, _Vector<16, int>)">;
+  def V6_vaddububb_sat_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<32, int>)">;
+  def V6_vadduhw_acc : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<16, int>, _Vector<16, int>)">;
+  def V6_vadduhw_acc_128B : HexagonBuiltin<"_Vector<64, int>(_Vector<64, int>, _Vector<32, int>, _Vector<32, int>)">;
+  def V6_vadduwsat : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>, _Vector<16, int>)">;
+  def V6_vadduwsat_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<32, int>)">;
+  def V6_vadduwsat_dv : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<32, int>)">;
+  def V6_vadduwsat_dv_128B : HexagonBuiltin<"_Vector<64, int>(_Vector<64, int>, _Vector<64, int>)">;
+  def V6_vandnqrt : HexagonBuiltin<"_Vector<16, int>(_Vector<64, bool>, int)">;
+  def V6_vandnqrt_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<128, bool>, int)">;
+  def V6_vandnqrt_acc : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>, _Vector<64, bool>, int)">;
+  def V6_vandnqrt_acc_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<128, bool>, int)">;
+  def V6_vandvnqv : HexagonBuiltin<"_Vector<16, int>(_Vector<64, bool>, _Vector<16, int>)">;
+  def V6_vandvnqv_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<128, bool>, _Vector<32, int>)">;
+  def V6_vandvqv : HexagonBuiltin<"_Vector<16, int>(_Vector<64, bool>, _Vector<16, int>)">;
+  def V6_vandvqv_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<128, bool>, _Vector<32, int>)">;
+  def V6_vasrhbsat : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>, _Vector<16, int>, int)">;
+  def V6_vasrhbsat_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<32, int>, int)">;
+  def V6_vasruwuhrndsat : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>, _Vector<16, int>, int)">;
+  def V6_vasruwuhrndsat_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<32, int>, int)">;
+  def V6_vasrwuhrndsat : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>, _Vector<16, int>, int)">;
+  def V6_vasrwuhrndsat_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<32, int>, int)">;
+  def V6_vlsrb : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>, int)">;
+  def V6_vlsrb_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, int)">;
+  def V6_vlutvvb_nm : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>, _Vector<16, int>, int)">;
+  def V6_vlutvvb_nm_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<32, int>, int)">;
+  def V6_vlutvvb_oracci : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>, _Vector<16, int>, _Vector<16, int>, unsigned _Constant int)">;
+  def V6_vlutvvb_oracci_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<32, int>, _Vector<32, int>, unsigned _Constant int)">;
+  def V6_vlutvvbi : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>, _Vector<16, int>, unsigned _Constant int)">;
+  def V6_vlutvvbi_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<32, int>, unsigned _Constant int)">;
+  def V6_vlutvwh_nm : HexagonBuiltin<"_Vector<32, int>(_Vector<16, int>, _Vector<16, int>, int)">;
+  def V6_vlutvwh_nm_128B : HexagonBuiltin<"_Vector<64, int>(_Vector<32, int>, _Vector<32, int>, int)">;
+  def V6_vlutvwh_oracci : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<16, int>, _Vector<16, int>, unsigned _Constant int)">;
+  def V6_vlutvwh_oracci_128B : HexagonBuiltin<"_Vector<64, int>(_Vector<64, int>, _Vector<32, int>, _Vector<32, int>, unsigned _Constant int)">;
+  def V6_vlutvwhi : HexagonBuiltin<"_Vector<32, int>(_Vector<16, int>, _Vector<16, int>, unsigned _Constant int)">;
+  def V6_vlutvwhi_128B : HexagonBuiltin<"_Vector<64, int>(_Vector<32, int>, _Vector<32, int>, unsigned _Constant int)">;
+  def V6_vmaxb : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>, _Vector<16, int>)">;
+  def V6_vmaxb_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<32, int>)">;
+  def V6_vminb : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>, _Vector<16, int>)">;
+  def V6_vminb_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<32, int>)">;
+  def V6_vmpauhb : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, int)">;
+  def V6_vmpauhb_128B : HexagonBuiltin<"_Vector<64, int>(_Vector<64, int>, int)">;
+  def V6_vmpauhb_acc : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<32, int>, int)">;
+  def V6_vmpauhb_acc_128B : HexagonBuiltin<"_Vector<64, int>(_Vector<64, int>, _Vector<64, int>, int)">;
+  def V6_vmpyewuh_64 : HexagonBuiltin<"_Vector<32, int>(_Vector<16, int>, _Vector<16, int>)">;
+  def V6_vmpyewuh_64_128B : HexagonBuiltin<"_Vector<64, int>(_Vector<32, int>, _Vector<32, int>)">;
+  def V6_vmpyiwub : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>, int)">;
+  def V6_vmpyiwub_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, int)">;
+  def V6_vmpyiwub_acc : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>, _Vector<16, int>, int)">;
+  def V6_vmpyiwub_acc_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<32, int>, int)">;
+  def V6_vmpyowh_64_acc : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<16, int>, _Vector<16, int>)">;
+  def V6_vmpyowh_64_acc_128B : HexagonBuiltin<"_Vector<64, int>(_Vector<64, int>, _Vector<32, int>, _Vector<32, int>)">;
+  def V6_vrounduhub : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>, _Vector<16, int>)">;
+  def V6_vrounduhub_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<32, int>)">;
+  def V6_vrounduwuh : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>, _Vector<16, int>)">;
+  def V6_vrounduwuh_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<32, int>)">;
+  def V6_vsatuwuh : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>, _Vector<16, int>)">;
+  def V6_vsatuwuh_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<32, int>)">;
+  def V6_vsubbsat : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>, _Vector<16, int>)">;
+  def V6_vsubbsat_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<32, int>)">;
+  def V6_vsubbsat_dv : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<32, int>)">;
+  def V6_vsubbsat_dv_128B : HexagonBuiltin<"_Vector<64, int>(_Vector<64, int>, _Vector<64, int>)">;
+  def V6_vsubcarry : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>, _Vector<16, int>, void *)">;
+  def V6_vsubcarry_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<32, int>, void *)">;
+  def V6_vsubububb_sat : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>, _Vector<16, int>)">;
+  def V6_vsubububb_sat_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<32, int>)">;
+  def V6_vsubuwsat : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>, _Vector<16, int>)">;
+  def V6_vsubuwsat_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<32, int>)">;
+  def V6_vsubuwsat_dv : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<32, int>)">;
+  def V6_vsubuwsat_dv_128B : HexagonBuiltin<"_Vector<64, int>(_Vector<64, int>, _Vector<64, int>)">;
+}
+
+// V65 HVX Instructions.
+
+let Features = HVXV65.Features in {
+  def V6_vabsb : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>)">;
+  def V6_vabsb_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>)">;
+  def V6_vabsb_sat : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>)">;
+  def V6_vabsb_sat_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>)">;
+  def V6_vaslh_acc : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>, _Vector<16, int>, int)">;
+  def V6_vaslh_acc_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<32, int>, int)">;
+  def V6_vasrh_acc : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>, _Vector<16, int>, int)">;
+  def V6_vasrh_acc_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<32, int>, int)">;
+  def V6_vasruhubrndsat : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>, _Vector<16, int>, int)">;
+  def V6_vasruhubrndsat_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<32, int>, int)">;
+  def V6_vasruhubsat : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>, _Vector<16, int>, int)">;
+  def V6_vasruhubsat_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<32, int>, int)">;
+  def V6_vasruwuhsat : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>, _Vector<16, int>, int)">;
+  def V6_vasruwuhsat_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<32, int>, int)">;
+  def V6_vavgb : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>, _Vector<16, int>)">;
+  def V6_vavgb_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<32, int>)">;
+  def V6_vavgbrnd : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>, _Vector<16, int>)">;
+  def V6_vavgbrnd_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<32, int>)">;
+  def V6_vavguw : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>, _Vector<16, int>)">;
+  def V6_vavguw_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<32, int>)">;
+  def V6_vavguwrnd : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>, _Vector<16, int>)">;
+  def V6_vavguwrnd_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<32, int>)">;
+  def V6_vdd0 : HexagonBuiltin<"_Vector<32, int>()">;
+  def V6_vdd0_128B : HexagonBuiltin<"_Vector<64, int>()">;
+  def V6_vgathermh : HexagonBuiltin<"void(void *, int, int, _Vector<16, int>)">;
+  def V6_vgathermh_128B : HexagonBuiltin<"void(void *, int, int, _Vector<32, int>)">;
+  def V6_vgathermhq : HexagonBuiltin<"void(void *, _Vector<64, bool>, int, int, _Vector<16, int>)">;
+  def V6_vgathermhq_128B : HexagonBuiltin<"void(void *, _Vector<128, bool>, int, int, _Vector<32, int>)">;
+  def V6_vgathermhw : HexagonBuiltin<"void(void *, int, int, _Vector<32, int>)">;
+  def V6_vgathermhw_128B : HexagonBuiltin<"void(void *, int, int, _Vector<64, int>)">;
+  def V6_vgathermhwq : HexagonBuiltin<"void(void *, _Vector<64, bool>, int, int, _Vector<32, int>)">;
+  def V6_vgathermhwq_128B : HexagonBuiltin<"void(void *, _Vector<128, bool>, int, int, _Vector<64, int>)">;
+  def V6_vgathermw : HexagonBuiltin<"void(void *, int, int, _Vector<16, int>)">;
+  def V6_vgathermw_128B : HexagonBuiltin<"void(void *, int, int, _Vector<32, int>)">;
+  def V6_vgathermwq : HexagonBuiltin<"void(void *, _Vector<64, bool>, int, int, _Vector<16, int>)">;
+  def V6_vgathermwq_128B : HexagonBuiltin<"void(void *, _Vector<128, bool>, int, int, _Vector<32, int>)">;
+  def V6_vlut4 : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>, long long int)">;
+  def V6_vlut4_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, long long int)">;
+  def V6_vmpabuu : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, int)">;
+  def V6_vmpabuu_128B : HexagonBuiltin<"_Vector<64, int>(_Vector<64, int>, int)">;
+  def V6_vmpabuu_acc : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<32, int>, int)">;
+  def V6_vmpabuu_acc_128B : HexagonBuiltin<"_Vector<64, int>(_Vector<64, int>, _Vector<64, int>, int)">;
+  def V6_vmpahhsat : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>, _Vector<16, int>, long long int)">;
+  def V6_vmpahhsat_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<32, int>, long long int)">;
+  def V6_vmpauhuhsat : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>, _Vector<16, int>, long long int)">;
+  def V6_vmpauhuhsat_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<32, int>, long long int)">;
+  def V6_vmpsuhuhsat : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>, _Vector<16, int>, long long int)">;
+  def V6_vmpsuhuhsat_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<32, int>, long long int)">;
+  def V6_vmpyh_acc : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<16, int>, int)">;
+  def V6_vmpyh_acc_128B : HexagonBuiltin<"_Vector<64, int>(_Vector<64, int>, _Vector<32, int>, int)">;
+  def V6_vmpyuhe : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>, int)">;
+  def V6_vmpyuhe_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, int)">;
+  def V6_vmpyuhe_acc : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>, _Vector<16, int>, int)">;
+  def V6_vmpyuhe_acc_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<32, int>, int)">;
+  def V6_vnavgb : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>, _Vector<16, int>)">;
+  def V6_vnavgb_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<32, int>)">;
+  def V6_vprefixqb : HexagonBuiltin<"_Vector<16, int>(_Vector<64, bool>)">;
+  def V6_vprefixqb_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<128, bool>)">;
+  def V6_vprefixqh : HexagonBuiltin<"_Vector<16, int>(_Vector<64, bool>)">;
+  def V6_vprefixqh_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<128, bool>)">;
+  def V6_vprefixqw : HexagonBuiltin<"_Vector<16, int>(_Vector<64, bool>)">;
+  def V6_vprefixqw_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<128, bool>)">;
+  def V6_vscattermh : HexagonBuiltin<"void(int, int, _Vector<16, int>, _Vector<16, int>)">;
+  def V6_vscattermh_128B : HexagonBuiltin<"void(int, int, _Vector<32, int>, _Vector<32, int>)">;
+  def V6_vscattermh_add : HexagonBuiltin<"void(int, int, _Vector<16, int>, _Vector<16, int>)">;
+  def V6_vscattermh_add_128B : HexagonBuiltin<"void(int, int, _Vector<32, int>, _Vector<32, int>)">;
+  def V6_vscattermhq : HexagonBuiltin<"void(_Vector<64, bool>, int, int, _Vector<16, int>, _Vector<16, int>)">;
+  def V6_vscattermhq_128B : HexagonBuiltin<"void(_Vector<128, bool>, int, int, _Vector<32, int>, _Vector<32, int>)">;
+  def V6_vscattermhw : HexagonBuiltin<"void(int, int, _Vector<32, int>, _Vector<16, int>)">;
+  def V6_vscattermhw_128B : HexagonBuiltin<"void(int, int, _Vector<64, int>, _Vector<32, int>)">;
+  def V6_vscattermhw_add : HexagonBuiltin<"void(int, int, _Vector<32, int>, _Vector<16, int>)">;
+  def V6_vscattermhw_add_128B : HexagonBuiltin<"void(int, int, _Vector<64, int>, _Vector<32, int>)">;
+  def V6_vscattermhwq : HexagonBuiltin<"void(_Vector<64, bool>, int, int, _Vector<32, int>, _Vector<16, int>)">;
+  def V6_vscattermhwq_128B : HexagonBuiltin<"void(_Vector<128, bool>, int, int, _Vector<64, int>, _Vector<32, int>)">;
+  def V6_vscattermw : HexagonBuiltin<"void(int, int, _Vector<16, int>, _Vector<16, int>)">;
+  def V6_vscattermw_128B : HexagonBuiltin<"void(int, int, _Vector<32, int>, _Vector<32, int>)">;
+  def V6_vscattermw_add : HexagonBuiltin<"void(int, int, _Vector<16, int>, _Vector<16, int>)">;
+  def V6_vscattermw_add_128B : HexagonBuiltin<"void(int, int, _Vector<32, int>, _Vector<32, int>)">;
+  def V6_vscattermwq : HexagonBuiltin<"void(_Vector<64, bool>, int, int, _Vector<16, int>, _Vector<16, int>)">;
+  def V6_vscattermwq_128B : HexagonBuiltin<"void(_Vector<128, bool>, int, int, _Vector<32, int>, _Vector<32, int>)">;
+}
+
+// V66 HVX Instructions.
+
+let Features = HVXV66.Features in {
+  def V6_vaddcarryo : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>, _Vector<16, int>, void *)">;
+  def V6_vaddcarryo_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<32, int>, void *)">;
+  def V6_vaddcarrysat : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>, _Vector<16, int>, _Vector<64, bool>)">;
+  def V6_vaddcarrysat_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<32, int>, _Vector<128, bool>)">;
+  def V6_vasr_into : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<16, int>, _Vector<16, int>)">;
+  def V6_vasr_into_128B : HexagonBuiltin<"_Vector<64, int>(_Vector<64, int>, _Vector<32, int>, _Vector<32, int>)">;
+  def V6_vrotr : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>, _Vector<16, int>)">;
+  def V6_vrotr_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<32, int>)">;
+  def V6_vsatdw : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>, _Vector<16, int>)">;
+  def V6_vsatdw_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<32, int>)">;
+  def V6_vsubcarryo : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>, _Vector<16, int>, void *)">;
+  def V6_vsubcarryo_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<32, int>, void *)">;
+}
+
+// V68 HVX Instructions.
+
+let Features = HVXV68.Features in {
+  def V6_v6mpyhubs10 : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<32, int>, unsigned _Constant int)">;
+  def V6_v6mpyhubs10_128B : HexagonBuiltin<"_Vector<64, int>(_Vector<64, int>, _Vector<64, int>, unsigned _Constant int)">;
+  def V6_v6mpyhubs10_vxx : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<32, int>, _Vector<32, int>, unsigned _Constant int)">;
+  def V6_v6mpyhubs10_vxx_128B : HexagonBuiltin<"_Vector<64, int>(_Vector<64, int>, _Vector<64, int>, _Vector<64, int>, unsigned _Constant int)">;
+  def V6_v6mpyvubs10 : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<32, int>, unsigned _Constant int)">;
+  def V6_v6mpyvubs10_128B : HexagonBuiltin<"_Vector<64, int>(_Vector<64, int>, _Vector<64, int>, unsigned _Constant int)">;
+  def V6_v6mpyvubs10_vxx : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<32, int>, _Vector<32, int>, unsigned _Constant int)">;
+  def V6_v6mpyvubs10_vxx_128B : HexagonBuiltin<"_Vector<64, int>(_Vector<64, int>, _Vector<64, int>, _Vector<64, int>, unsigned _Constant int)">;
+  def V6_vabs_hf : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>)">;
+  def V6_vabs_hf_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>)">;
+  def V6_vabs_sf : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>)">;
+  def V6_vabs_sf_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>)">;
+  def V6_vadd_hf : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>, _Vector<16, int>)">;
+  def V6_vadd_hf_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<32, int>)">;
+  def V6_vadd_hf_hf : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>, _Vector<16, int>)">;
+  def V6_vadd_hf_hf_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<32, int>)">;
+  def V6_vadd_qf16 : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>, _Vector<16, int>)">;
+  def V6_vadd_qf16_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<32, int>)">;
+  def V6_vadd_qf16_mix : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>, _Vector<16, int>)">;
+  def V6_vadd_qf16_mix_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<32, int>)">;
+  def V6_vadd_qf32 : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>, _Vector<16, int>)">;
+  def V6_vadd_qf32_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<32, int>)">;
+  def V6_vadd_qf32_mix : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>, _Vector<16, int>)">;
+  def V6_vadd_qf32_mix_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<32, int>)">;
+  def V6_vadd_sf : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>, _Vector<16, int>)">;
+  def V6_vadd_sf_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<32, int>)">;
+  def V6_vadd_sf_hf : HexagonBuiltin<"_Vector<32, int>(_Vector<16, int>, _Vector<16, int>)">;
+  def V6_vadd_sf_hf_128B : HexagonBuiltin<"_Vector<64, int>(_Vector<32, int>, _Vector<32, int>)">;
+  def V6_vadd_sf_sf : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>, _Vector<16, int>)">;
+  def V6_vadd_sf_sf_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<32, int>)">;
+  def V6_vassign_fp : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>)">;
+  def V6_vassign_fp_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>)">;
+  def V6_vconv_hf_qf16 : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>)">;
+  def V6_vconv_hf_qf16_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>)">;
+  def V6_vconv_hf_qf32 : HexagonBuiltin<"_Vector<16, int>(_Vector<32, int>)">;
+  def V6_vconv_hf_qf32_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<64, int>)">;
+  def V6_vconv_sf_qf32 : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>)">;
+  def V6_vconv_sf_qf32_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>)">;
+  def V6_vcvt_b_hf : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>, _Vector<16, int>)">;
+  def V6_vcvt_b_hf_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<32, int>)">;
+  def V6_vcvt_h_hf : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>)">;
+  def V6_vcvt_h_hf_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>)">;
+  def V6_vcvt_hf_b : HexagonBuiltin<"_Vector<32, int>(_Vector<16, int>)">;
+  def V6_vcvt_hf_b_128B : HexagonBuiltin<"_Vector<64, int>(_Vector<32, int>)">;
+  def V6_vcvt_hf_h : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>)">;
+  def V6_vcvt_hf_h_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>)">;
+  def V6_vcvt_hf_sf : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>, _Vector<16, int>)">;
+  def V6_vcvt_hf_sf_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<32, int>)">;
+  def V6_vcvt_hf_ub : HexagonBuiltin<"_Vector<32, int>(_Vector<16, int>)">;
+  def V6_vcvt_hf_ub_128B : HexagonBuiltin<"_Vector<64, int>(_Vector<32, int>)">;
+  def V6_vcvt_hf_uh : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>)">;
+  def V6_vcvt_hf_uh_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>)">;
+  def V6_vcvt_sf_hf : HexagonBuiltin<"_Vector<32, int>(_Vector<16, int>)">;
+  def V6_vcvt_sf_hf_128B : HexagonBuiltin<"_Vector<64, int>(_Vector<32, int>)">;
+  def V6_vcvt_ub_hf : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>, _Vector<16, int>)">;
+  def V6_vcvt_ub_hf_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<32, int>)">;
+  def V6_vcvt_uh_hf : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>)">;
+  def V6_vcvt_uh_hf_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>)">;
+  def V6_vdmpy_sf_hf : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>, _Vector<16, int>)">;
+  def V6_vdmpy_sf_hf_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<32, int>)">;
+  def V6_vdmpy_sf_hf_acc : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>, _Vector<16, int>, _Vector<16, int>)">;
+  def V6_vdmpy_sf_hf_acc_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<32, int>, _Vector<32, int>)">;
+  def V6_vfmax_hf : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>, _Vector<16, int>)">;
+  def V6_vfmax_hf_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<32, int>)">;
+  def V6_vfmax_sf : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>, _Vector<16, int>)">;
+  def V6_vfmax_sf_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<32, int>)">;
+  def V6_vfmin_hf : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>, _Vector<16, int>)">;
+  def V6_vfmin_hf_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<32, int>)">;
+  def V6_vfmin_sf : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>, _Vector<16, int>)">;
+  def V6_vfmin_sf_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<32, int>)">;
+  def V6_vfneg_hf : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>)">;
+  def V6_vfneg_hf_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>)">;
+  def V6_vfneg_sf : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>)">;
+  def V6_vfneg_sf_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>)">;
+  def V6_vgthf : HexagonBuiltin<"_Vector<64, bool>(_Vector<16, int>, _Vector<16, int>)">;
+  def V6_vgthf_128B : HexagonBuiltin<"_Vector<128, bool>(_Vector<32, int>, _Vector<32, int>)">;
+  def V6_vgthf_and : HexagonBuiltin<"_Vector<64, bool>(_Vector<64, bool>, _Vector<16, int>, _Vector<16, int>)">;
+  def V6_vgthf_and_128B : HexagonBuiltin<"_Vector<128, bool>(_Vector<128, bool>, _Vector<32, int>, _Vector<32, int>)">;
+  def V6_vgthf_or : HexagonBuiltin<"_Vector<64, bool>(_Vector<64, bool>, _Vector<16, int>, _Vector<16, int>)">;
+  def V6_vgthf_or_128B : HexagonBuiltin<"_Vector<128, bool>(_Vector<128, bool>, _Vector<32, int>, _Vector<32, int>)">;
+  def V6_vgthf_xor : HexagonBuiltin<"_Vector<64, bool>(_Vector<64, bool>, _Vector<16, int>, _Vector<16, int>)">;
+  def V6_vgthf_xor_128B : HexagonBuiltin<"_Vector<128, bool>(_Vector<128, bool>, _Vector<32, int>, _Vector<32, int>)">;
+  def V6_vgtsf : HexagonBuiltin<"_Vector<64, bool>(_Vector<16, int>, _Vector<16, int>)">;
+  def V6_vgtsf_128B : HexagonBuiltin<"_Vector<128, bool>(_Vector<32, int>, _Vector<32, int>)">;
+  def V6_vgtsf_and : HexagonBuiltin<"_Vector<64, bool>(_Vector<64, bool>, _Vector<16, int>, _Vector<16, int>)">;
+  def V6_vgtsf_and_128B : HexagonBuiltin<"_Vector<128, bool>(_Vector<128, bool>, _Vector<32, int>, _Vector<32, int>)">;
+  def V6_vgtsf_or : HexagonBuiltin<"_Vector<64, bool>(_Vector<64, bool>, _Vector<16, int>, _Vector<16, int>)">;
+  def V6_vgtsf_or_128B : HexagonBuiltin<"_Vector<128, bool>(_Vector<128, bool>, _Vector<32, int>, _Vector<32, int>)">;
+  def V6_vgtsf_xor : HexagonBuiltin<"_Vector<64, bool>(_Vector<64, bool>, _Vector<16, int>, _Vector<16, int>)">;
+  def V6_vgtsf_xor_128B : HexagonBuiltin<"_Vector<128, bool>(_Vector<128, bool>, _Vector<32, int>, _Vector<32, int>)">;
+  def V6_vmax_hf : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>, _Vector<16, int>)">;
+  def V6_vmax_hf_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<32, int>)">;
+  def V6_vmax_sf : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>, _Vector<16, int>)">;
+  def V6_vmax_sf_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<32, int>)">;
+  def V6_vmin_hf : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>, _Vector<16, int>)">;
+  def V6_vmin_hf_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<32, int>)">;
+  def V6_vmin_sf : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>, _Vector<16, int>)">;
+  def V6_vmin_sf_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<32, int>)">;
+  def V6_vmpy_hf_hf : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>, _Vector<16, int>)">;
+  def V6_vmpy_hf_hf_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<32, int>)">;
+  def V6_vmpy_hf_hf_acc : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>, _Vector<16, int>, _Vector<16, int>)">;
+  def V6_vmpy_hf_hf_acc_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<32, int>, _Vector<32, int>)">;
+  def V6_vmpy_qf16 : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>, _Vector<16, int>)">;
+  def V6_vmpy_qf16_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<32, int>)">;
+  def V6_vmpy_qf16_hf : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>, _Vector<16, int>)">;
+  def V6_vmpy_qf16_hf_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<32, int>)">;
+  def V6_vmpy_qf16_mix_hf : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>, _Vector<16, int>)">;
+  def V6_vmpy_qf16_mix_hf_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<32, int>)">;
+  def V6_vmpy_qf32 : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>, _Vector<16, int>)">;
+  def V6_vmpy_qf32_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<32, int>)">;
+  def V6_vmpy_qf32_hf : HexagonBuiltin<"_Vector<32, int>(_Vector<16, int>, _Vector<16, int>)">;
+  def V6_vmpy_qf32_hf_128B : HexagonBuiltin<"_Vector<64, int>(_Vector<32, int>, _Vector<32, int>)">;
+  def V6_vmpy_qf32_mix_hf : HexagonBuiltin<"_Vector<32, int>(_Vector<16, int>, _Vector<16, int>)">;
+  def V6_vmpy_qf32_mix_hf_128B : HexagonBuiltin<"_Vector<64, int>(_Vector<32, int>, _Vector<32, int>)">;
+  def V6_vmpy_qf32_qf16 : HexagonBuiltin<"_Vector<32, int>(_Vector<16, int>, _Vector<16, int>)">;
+  def V6_vmpy_qf32_qf16_128B : HexagonBuiltin<"_Vector<64, int>(_Vector<32, int>, _Vector<32, int>)">;
+  def V6_vmpy_qf32_sf : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>, _Vector<16, int>)">;
+  def V6_vmpy_qf32_sf_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<32, int>)">;
+  def V6_vmpy_sf_hf : HexagonBuiltin<"_Vector<32, int>(_Vector<16, int>, _Vector<16, int>)">;
+  def V6_vmpy_sf_hf_128B : HexagonBuiltin<"_Vector<64, int>(_Vector<32, int>, _Vector<32, int>)">;
+  def V6_vmpy_sf_hf_acc : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<16, int>, _Vector<16, int>)">;
+  def V6_vmpy_sf_hf_acc_128B : HexagonBuiltin<"_Vector<64, int>(_Vector<64, int>, _Vector<32, int>, _Vector<32, int>)">;
+  def V6_vmpy_sf_sf : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>, _Vector<16, int>)">;
+  def V6_vmpy_sf_sf_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<32, int>)">;
+  def V6_vsub_hf : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>, _Vector<16, int>)">;
+  def V6_vsub_hf_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<32, int>)">;
+  def V6_vsub_hf_hf : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>, _Vector<16, int>)">;
+  def V6_vsub_hf_hf_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<32, int>)">;
+  def V6_vsub_qf16 : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>, _Vector<16, int>)">;
+  def V6_vsub_qf16_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<32, int>)">;
+  def V6_vsub_qf16_mix : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>, _Vector<16, int>)">;
+  def V6_vsub_qf16_mix_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<32, int>)">;
+  def V6_vsub_qf32 : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>, _Vector<16, int>)">;
+  def V6_vsub_qf32_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<32, int>)">;
+  def V6_vsub_qf32_mix : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>, _Vector<16, int>)">;
+  def V6_vsub_qf32_mix_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<32, int>)">;
+  def V6_vsub_sf : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>, _Vector<16, int>)">;
+  def V6_vsub_sf_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<32, int>)">;
+  def V6_vsub_sf_hf : HexagonBuiltin<"_Vector<32, int>(_Vector<16, int>, _Vector<16, int>)">;
+  def V6_vsub_sf_hf_128B : HexagonBuiltin<"_Vector<64, int>(_Vector<32, int>, _Vector<32, int>)">;
+  def V6_vsub_sf_sf : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>, _Vector<16, int>)">;
+  def V6_vsub_sf_sf_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<32, int>)">;
+}
+
+// V69 HVX Instructions.
+
+let Features = HVXV69.Features in {
+  def V6_vasrvuhubrndsat : HexagonBuiltin<"_Vector<16, int>(_Vector<32, int>, _Vector<16, int>)">;
+  def V6_vasrvuhubrndsat_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<64, int>, _Vector<32, int>)">;
+  def V6_vasrvuhubsat : HexagonBuiltin<"_Vector<16, int>(_Vector<32, int>, _Vector<16, int>)">;
+  def V6_vasrvuhubsat_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<64, int>, _Vector<32, int>)">;
+  def V6_vasrvwuhrndsat : HexagonBuiltin<"_Vector<16, int>(_Vector<32, int>, _Vector<16, int>)">;
+  def V6_vasrvwuhrndsat_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<64, int>, _Vector<32, int>)">;
+  def V6_vasrvwuhsat : HexagonBuiltin<"_Vector<16, int>(_Vector<32, int>, _Vector<16, int>)">;
+  def V6_vasrvwuhsat_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<64, int>, _Vector<32, int>)">;
+  def V6_vmpyuhvs : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>, _Vector<16, int>)">;
+  def V6_vmpyuhvs_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<32, int>)">;
+}
+
+// V73 HVX Instructions.
+
+let Features = HVXV73.Features in {
+  def V6_vadd_sf_bf : HexagonBuiltin<"_Vector<32, int>(_Vector<16, int>, _Vector<16, int>)">;
+  def V6_vadd_sf_bf_128B : HexagonBuiltin<"_Vector<64, int>(_Vector<32, int>, _Vector<32, int>)">;
+  def V6_vconv_h_hf : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>)">;
+  def V6_vconv_h_hf_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>)">;
+  def V6_vconv_hf_h : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>)">;
+  def V6_vconv_hf_h_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>)">;
+  def V6_vconv_sf_w : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>)">;
+  def V6_vconv_sf_w_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>)">;
+  def V6_vconv_w_sf : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>)">;
+  def V6_vconv_w_sf_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>)">;
+  def V6_vcvt_bf_sf : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>, _Vector<16, int>)">;
+  def V6_vcvt_bf_sf_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<32, int>)">;
+  def V6_vgtbf : HexagonBuiltin<"_Vector<64, bool>(_Vector<16, int>, _Vector<16, int>)">;
+  def V6_vgtbf_128B : HexagonBuiltin<"_Vector<128, bool>(_Vector<32, int>, _Vector<32, int>)">;
+  def V6_vgtbf_and : HexagonBuiltin<"_Vector<64, bool>(_Vector<64, bool>, _Vector<16, int>, _Vector<16, int>)">;
+  def V6_vgtbf_and_128B : HexagonBuiltin<"_Vector<128, bool>(_Vector<128, bool>, _Vector<32, int>, _Vector<32, int>)">;
+  def V6_vgtbf_or : HexagonBuiltin<"_Vector<64, bool>(_Vector<64, bool>, _Vector<16, int>, _Vector<16, int>)">;
+  def V6_vgtbf_or_128B : HexagonBuiltin<"_Vector<128, bool>(_Vector<128, bool>, _Vector<32, int>, _Vector<32, int>)">;
+  def V6_vgtbf_xor : HexagonBuiltin<"_Vector<64, bool>(_Vector<64, bool>, _Vector<16, int>, _Vector<16, int>)">;
+  def V6_vgtbf_xor_128B : HexagonBuiltin<"_Vector<128, bool>(_Vector<128, bool>, _Vector<32, int>, _Vector<32, int>)">;
+  def V6_vmax_bf : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>, _Vector<16, int>)">;
+  def V6_vmax_bf_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<32, int>)">;
+  def V6_vmin_bf : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>, _Vector<16, int>)">;
+  def V6_vmin_bf_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<32, int>)">;
+  def V6_vmpy_sf_bf : HexagonBuiltin<"_Vector<32, int>(_Vector<16, int>, _Vector<16, int>)">;
+  def V6_vmpy_sf_bf_128B : HexagonBuiltin<"_Vector<64, int>(_Vector<32, int>, _Vector<32, int>)">;
+  def V6_vmpy_sf_bf_acc : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<16, int>, _Vector<16, int>)">;
+  def V6_vmpy_sf_bf_acc_128B : HexagonBuiltin<"_Vector<64, int>(_Vector<64, int>, _Vector<32, int>, _Vector<32, int>)">;
+  def V6_vsub_sf_bf : HexagonBuiltin<"_Vector<32, int>(_Vector<16, int>, _Vector<16, int>)">;
+  def V6_vsub_sf_bf_128B : HexagonBuiltin<"_Vector<64, int>(_Vector<32, int>, _Vector<32, int>)">;
+}
+
+// V79 HVX Instructions.
+
+let Features = HVXV79.Features in {
+  def V6_get_qfext : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>, int)">;
+  def V6_get_qfext_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, int)">;
+  def V6_get_qfext_oracc : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>, _Vector<16, int>, int)">;
+  def V6_get_qfext_oracc_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<32, int>, int)">;
+  def V6_set_qfext : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>, int)">;
+  def V6_set_qfext_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, int)">;
+  def V6_vabs_f8 : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>)">;
+  def V6_vabs_f8_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>)">;
+  def V6_vadd_hf_f8 : HexagonBuiltin<"_Vector<32, int>(_Vector<16, int>, _Vector<16, int>)">;
+  def V6_vadd_hf_f8_128B : HexagonBuiltin<"_Vector<64, int>(_Vector<32, int>, _Vector<32, int>)">;
+  def V6_vcvt2_b_hf : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>, _Vector<16, int>)">;
+  def V6_vcvt2_b_hf_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<32, int>)">;
+  def V6_vcvt2_hf_b : HexagonBuiltin<"_Vector<32, int>(_Vector<16, int>)">;
+  def V6_vcvt2_hf_b_128B : HexagonBuiltin<"_Vector<64, int>(_Vector<32, int>)">;
+  def V6_vcvt2_hf_ub : HexagonBuiltin<"_Vector<32, int>(_Vector<16, int>)">;
+  def V6_vcvt2_hf_ub_128B : HexagonBuiltin<"_Vector<64, int>(_Vector<32, int>)">;
+  def V6_vcvt2_ub_hf : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>, _Vector<16, int>)">;
+  def V6_vcvt2_ub_hf_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<32, int>)">;
+  def V6_vcvt_f8_hf : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>, _Vector<16, int>)">;
+  def V6_vcvt_f8_hf_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<32, int>)">;
+  def V6_vcvt_hf_f8 : HexagonBuiltin<"_Vector<32, int>(_Vector<16, int>)">;
+  def V6_vcvt_hf_f8_128B : HexagonBuiltin<"_Vector<64, int>(_Vector<32, int>)">;
+  def V6_vfmax_f8 : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>, _Vector<16, int>)">;
+  def V6_vfmax_f8_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<32, int>)">;
+  def V6_vfmin_f8 : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>, _Vector<16, int>)">;
+  def V6_vfmin_f8_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<32, int>)">;
+  def V6_vfneg_f8 : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>)">;
+  def V6_vfneg_f8_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>)">;
+  def V6_vmerge_qf : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>, _Vector<16, int>)">;
+  def V6_vmerge_qf_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<32, int>)">;
+  def V6_vmpy_hf_f8 : HexagonBuiltin<"_Vector<32, int>(_Vector<16, int>, _Vector<16, int>)">;
+  def V6_vmpy_hf_f8_128B : HexagonBuiltin<"_Vector<64, int>(_Vector<32, int>, _Vector<32, int>)">;
+  def V6_vmpy_hf_f8_acc : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, _Vector<16, int>, _Vector<16, int>)">;
+  def V6_vmpy_hf_f8_acc_128B : HexagonBuiltin<"_Vector<64, int>(_Vector<64, int>, _Vector<32, int>, _Vector<32, int>)">;
+  def V6_vmpy_rt_hf : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>, int)">;
+  def V6_vmpy_rt_hf_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, int)">;
+  def V6_vmpy_rt_qf16 : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>, int)">;
+  def V6_vmpy_rt_qf16_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, int)">;
+  def V6_vmpy_rt_sf : HexagonBuiltin<"_Vector<16, int>(_Vector<16, int>, int)">;
+  def V6_vmpy_rt_sf_128B : HexagonBuiltin<"_Vector<32, int>(_Vector<32, int>, int)">;
+  def V6_vsub_hf_f8 : HexagonBuiltin<"_Vector<32, int>(_Vector<16, int>, _Vector<16, int>)">;
+  def V6_vsub_hf_f8_128B : HexagonBuiltin<"_Vector<64, int>(_Vector<32, int>, _Vector<32, int>)">;
+}
diff --git a/clang/include/clang/Basic/BuiltinsHexagonDep.def b/clang/include/clang/Basic/BuiltinsHexagonDep.def
deleted file mode 100644
index 616ff3ccf5b6b..0000000000000
--- a/clang/include/clang/Basic/BuiltinsHexagonDep.def
+++ /dev/null
@@ -1,1970 +0,0 @@
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-// Automatically generated file, do not edit!
-//===----------------------------------------------------------------------===//
-
-
-// V5 Scalar Instructions.
-
-TARGET_BUILTIN(__builtin_HEXAGON_A2_abs, "ii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_A2_absp, "LLiLLi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_A2_abssat, "ii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_A2_add, "iii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_A2_addh_h16_hh, "iii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_A2_addh_h16_hl, "iii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_A2_addh_h16_lh, "iii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_A2_addh_h16_ll, "iii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_A2_addh_h16_sat_hh, "iii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_A2_addh_h16_sat_hl, "iii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_A2_addh_h16_sat_lh, "iii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_A2_addh_h16_sat_ll, "iii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_A2_addh_l16_hl, "iii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_A2_addh_l16_ll, "iii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_A2_addh_l16_sat_hl, "iii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_A2_addh_l16_sat_ll, "iii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_A2_addi, "iiIi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_A2_addp, "LLiLLiLLi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_A2_addpsat, "LLiLLiLLi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_A2_addsat, "iii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_A2_addsp, "LLiiLLi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_A2_and, "iii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_A2_andir, "iiIi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_A2_andp, "LLiLLiLLi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_A2_aslh, "ii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_A2_asrh, "ii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_A2_combine_hh, "iii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_A2_combine_hl, "iii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_A2_combine_lh, "iii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_A2_combine_ll, "iii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_A2_combineii, "LLiIiIi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_A2_combinew, "LLiii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_A2_max, "iii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_A2_maxp, "LLiLLiLLi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_A2_maxu, "Uiii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_A2_maxup, "ULLiLLiLLi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_A2_min, "iii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_A2_minp, "LLiLLiLLi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_A2_minu, "Uiii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_A2_minup, "ULLiLLiLLi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_A2_neg, "ii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_A2_negp, "LLiLLi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_A2_negsat, "ii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_A2_not, "ii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_A2_notp, "LLiLLi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_A2_or, "iii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_A2_orir, "iiIi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_A2_orp, "LLiLLiLLi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_A2_roundsat, "iLLi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_A2_sat, "iLLi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_A2_satb, "ii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_A2_sath, "ii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_A2_satub, "ii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_A2_satuh, "ii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_A2_sub, "iii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_A2_subh_h16_hh, "iii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_A2_subh_h16_hl, "iii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_A2_subh_h16_lh, "iii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_A2_subh_h16_ll, "iii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_A2_subh_h16_sat_hh, "iii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_A2_subh_h16_sat_hl, "iii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_A2_subh_h16_sat_lh, "iii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_A2_subh_h16_sat_ll, "iii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_A2_subh_l16_hl, "iii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_A2_subh_l16_ll, "iii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_A2_subh_l16_sat_hl, "iii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_A2_subh_l16_sat_ll, "iii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_A2_subp, "LLiLLiLLi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_A2_subri, "iIii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_A2_subsat, "iii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_A2_svaddh, "iii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_A2_svaddhs, "iii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_A2_svadduhs, "iii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_A2_svavgh, "iii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_A2_svavghs, "iii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_A2_svnavgh, "iii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_A2_svsubh, "iii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_A2_svsubhs, "iii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_A2_svsubuhs, "iii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_A2_swiz, "ii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_A2_sxtb, "ii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_A2_sxth, "ii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_A2_sxtw, "LLii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_A2_tfr, "ii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_A2_tfrih, "iiUIi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_A2_tfril, "iiUIi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_A2_tfrp, "LLiLLi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_A2_tfrpi, "LLiIi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_A2_tfrsi, "iIi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_A2_vabsh, "LLiLLi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_A2_vabshsat, "LLiLLi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_A2_vabsw, "LLiLLi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_A2_vabswsat, "LLiLLi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_A2_vaddb_map, "LLiLLiLLi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_A2_vaddh, "LLiLLiLLi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_A2_vaddhs, "LLiLLiLLi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_A2_vaddub, "LLiLLiLLi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_A2_vaddubs, "LLiLLiLLi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_A2_vadduhs, "LLiLLiLLi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_A2_vaddw, "LLiLLiLLi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_A2_vaddws, "LLiLLiLLi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_A2_vavgh, "LLiLLiLLi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_A2_vavghcr, "LLiLLiLLi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_A2_vavghr, "LLiLLiLLi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_A2_vavgub, "LLiLLiLLi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_A2_vavgubr, "LLiLLiLLi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_A2_vavguh, "LLiLLiLLi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_A2_vavguhr, "LLiLLiLLi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_A2_vavguw, "LLiLLiLLi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_A2_vavguwr, "LLiLLiLLi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_A2_vavgw, "LLiLLiLLi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_A2_vavgwcr, "LLiLLiLLi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_A2_vavgwr, "LLiLLiLLi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_A2_vcmpbeq, "iLLiLLi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_A2_vcmpbgtu, "iLLiLLi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_A2_vcmpheq, "iLLiLLi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_A2_vcmphgt, "iLLiLLi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_A2_vcmphgtu, "iLLiLLi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_A2_vcmpweq, "iLLiLLi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_A2_vcmpwgt, "iLLiLLi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_A2_vcmpwgtu, "iLLiLLi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_A2_vconj, "LLiLLi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_A2_vmaxb, "LLiLLiLLi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_A2_vmaxh, "LLiLLiLLi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_A2_vmaxub, "LLiLLiLLi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_A2_vmaxuh, "LLiLLiLLi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_A2_vmaxuw, "LLiLLiLLi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_A2_vmaxw, "LLiLLiLLi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_A2_vminb, "LLiLLiLLi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_A2_vminh, "LLiLLiLLi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_A2_vminub, "LLiLLiLLi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_A2_vminuh, "LLiLLiLLi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_A2_vminuw, "LLiLLiLLi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_A2_vminw, "LLiLLiLLi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_A2_vnavgh, "LLiLLiLLi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_A2_vnavghcr, "LLiLLiLLi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_A2_vnavghr, "LLiLLiLLi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_A2_vnavgw, "LLiLLiLLi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_A2_vnavgwcr, "LLiLLiLLi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_A2_vnavgwr, "LLiLLiLLi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_A2_vraddub, "LLiLLiLLi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_A2_vraddub_acc, "LLiLLiLLiLLi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_A2_vrsadub, "LLiLLiLLi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_A2_vrsadub_acc, "LLiLLiLLiLLi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_A2_vsubb_map, "LLiLLiLLi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_A2_vsubh, "LLiLLiLLi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_A2_vsubhs, "LLiLLiLLi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_A2_vsubub, "LLiLLiLLi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_A2_vsububs, "LLiLLiLLi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_A2_vsubuhs, "LLiLLiLLi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_A2_vsubw, "LLiLLiLLi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_A2_vsubws, "LLiLLiLLi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_A2_xor, "iii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_A2_xorp, "LLiLLiLLi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_A2_zxtb, "ii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_A2_zxth, "ii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_A4_andn, "iii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_A4_andnp, "LLiLLiLLi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_A4_bitsplit, "LLiii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_A4_bitspliti, "LLiiUIi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_A4_boundscheck, "iiLLi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_A4_cmpbeq, "iii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_A4_cmpbeqi, "iiUIi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_A4_cmpbgt, "iii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_A4_cmpbgti, "iiIi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_A4_cmpbgtu, "iii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_A4_cmpbgtui, "iiUIi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_A4_cmpheq, "iii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_A4_cmpheqi, "iiIi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_A4_cmphgt, "iii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_A4_cmphgti, "iiIi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_A4_cmphgtu, "iii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_A4_cmphgtui, "iiUIi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_A4_combineir, "LLiIii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_A4_combineri, "LLiiIi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_A4_cround_ri, "iiUIi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_A4_cround_rr, "iii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_A4_modwrapu, "iii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_A4_orn, "iii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_A4_ornp, "LLiLLiLLi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_A4_rcmpeq, "iii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_A4_rcmpeqi, "iiIi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_A4_rcmpneq, "iii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_A4_rcmpneqi, "iiIi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_A4_round_ri, "iiUIi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_A4_round_ri_sat, "iiUIi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_A4_round_rr, "iii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_A4_round_rr_sat, "iii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_A4_tlbmatch, "iLLii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_A4_vcmpbeq_any, "iLLiLLi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_A4_vcmpbeqi, "iLLiUIi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_A4_vcmpbgt, "iLLiLLi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_A4_vcmpbgti, "iLLiIi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_A4_vcmpbgtui, "iLLiUIi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_A4_vcmpheqi, "iLLiIi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_A4_vcmphgti, "iLLiIi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_A4_vcmphgtui, "iLLiUIi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_A4_vcmpweqi, "iLLiIi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_A4_vcmpwgti, "iLLiIi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_A4_vcmpwgtui, "iLLiUIi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_A4_vrmaxh, "LLiLLiLLii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_A4_vrmaxuh, "LLiLLiLLii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_A4_vrmaxuw, "LLiLLiLLii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_A4_vrmaxw, "LLiLLiLLii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_A4_vrminh, "LLiLLiLLii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_A4_vrminuh, "LLiLLiLLii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_A4_vrminuw, "LLiLLiLLii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_A4_vrminw, "LLiLLiLLii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_A5_vaddhubs, "iLLiLLi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_C2_all8, "ii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_C2_and, "iii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_C2_andn, "iii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_C2_any8, "ii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_C2_bitsclr, "iii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_C2_bitsclri, "iiUIi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_C2_bitsset, "iii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_C2_cmpeq, "iii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_C2_cmpeqi, "iiIi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_C2_cmpeqp, "iLLiLLi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_C2_cmpgei, "iiIi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_C2_cmpgeui, "iiUIi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_C2_cmpgt, "iii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_C2_cmpgti, "iiIi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_C2_cmpgtp, "iLLiLLi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_C2_cmpgtu, "iii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_C2_cmpgtui, "iiUIi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_C2_cmpgtup, "iLLiLLi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_C2_cmplt, "iii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_C2_cmpltu, "iii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_C2_mask, "LLii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_C2_mux, "iiii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_C2_muxii, "iiIiIi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_C2_muxir, "iiiIi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_C2_muxri, "iiIii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_C2_not, "ii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_C2_or, "iii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_C2_orn, "iii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_C2_pxfer_map, "ii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_C2_tfrpr, "ii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_C2_tfrrp, "ii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_C2_vitpack, "iii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_C2_vmux, "LLiiLLiLLi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_C2_xor, "iii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_C4_and_and, "iiii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_C4_and_andn, "iiii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_C4_and_or, "iiii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_C4_and_orn, "iiii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_C4_cmplte, "iii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_C4_cmpltei, "iiIi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_C4_cmplteu, "iii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_C4_cmplteui, "iiUIi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_C4_cmpneq, "iii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_C4_cmpneqi, "iiIi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_C4_fastcorner9, "iii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_C4_fastcorner9_not, "iii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_C4_nbitsclr, "iii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_C4_nbitsclri, "iiUIi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_C4_nbitsset, "iii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_C4_or_and, "iiii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_C4_or_andn, "iiii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_C4_or_or, "iiii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_C4_or_orn, "iiii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_F2_conv_d2df, "dLLi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_F2_conv_d2sf, "fLLi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_F2_conv_df2d, "LLid", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_F2_conv_df2d_chop, "LLid", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_F2_conv_df2sf, "fd", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_F2_conv_df2ud, "LLid", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_F2_conv_df2ud_chop, "LLid", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_F2_conv_df2uw, "id", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_F2_conv_df2uw_chop, "id", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_F2_conv_df2w, "id", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_F2_conv_df2w_chop, "id", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_F2_conv_sf2d, "LLif", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_F2_conv_sf2d_chop, "LLif", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_F2_conv_sf2df, "df", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_F2_conv_sf2ud, "LLif", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_F2_conv_sf2ud_chop, "LLif", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_F2_conv_sf2uw, "if", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_F2_conv_sf2uw_chop, "if", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_F2_conv_sf2w, "if", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_F2_conv_sf2w_chop, "if", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_F2_conv_ud2df, "dLLi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_F2_conv_ud2sf, "fLLi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_F2_conv_uw2df, "di", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_F2_conv_uw2sf, "fi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_F2_conv_w2df, "di", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_F2_conv_w2sf, "fi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_F2_dfclass, "idUIi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_F2_dfcmpeq, "idd", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_F2_dfcmpge, "idd", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_F2_dfcmpgt, "idd", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_F2_dfcmpuo, "idd", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_F2_dfimm_n, "dUIi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_F2_dfimm_p, "dUIi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_F2_sfadd, "fff", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_F2_sfclass, "ifUIi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_F2_sfcmpeq, "iff", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_F2_sfcmpge, "iff", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_F2_sfcmpgt, "iff", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_F2_sfcmpuo, "iff", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_F2_sffixupd, "fff", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_F2_sffixupn, "fff", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_F2_sffixupr, "ff", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_F2_sffma, "ffff", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_F2_sffma_lib, "ffff", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_F2_sffma_sc, "ffffi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_F2_sffms, "ffff", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_F2_sffms_lib, "ffff", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_F2_sfimm_n, "fUIi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_F2_sfimm_p, "fUIi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_F2_sfmax, "fff", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_F2_sfmin, "fff", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_F2_sfmpy, "fff", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_F2_sfsub, "fff", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_acci, "iiii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_accii, "iiiIi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_cmaci_s0, "LLiLLiii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_cmacr_s0, "LLiLLiii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_cmacs_s0, "LLiLLiii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_cmacs_s1, "LLiLLiii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_cmacsc_s0, "LLiLLiii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_cmacsc_s1, "LLiLLiii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_cmpyi_s0, "LLiii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_cmpyr_s0, "LLiii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_cmpyrs_s0, "iii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_cmpyrs_s1, "iii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_cmpyrsc_s0, "iii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_cmpyrsc_s1, "iii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_cmpys_s0, "LLiii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_cmpys_s1, "LLiii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_cmpysc_s0, "LLiii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_cmpysc_s1, "LLiii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_cnacs_s0, "LLiLLiii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_cnacs_s1, "LLiLLiii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_cnacsc_s0, "LLiLLiii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_cnacsc_s1, "LLiLLiii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_dpmpyss_acc_s0, "LLiLLiii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_dpmpyss_nac_s0, "LLiLLiii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_dpmpyss_rnd_s0, "iii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_dpmpyss_s0, "LLiii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_dpmpyuu_acc_s0, "LLiLLiii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_dpmpyuu_nac_s0, "LLiLLiii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_dpmpyuu_s0, "ULLiii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_hmmpyh_rs1, "iii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_hmmpyh_s1, "iii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_hmmpyl_rs1, "iii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_hmmpyl_s1, "iii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_maci, "iiii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_macsin, "iiiUIi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_macsip, "iiiUIi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_mmachs_rs0, "LLiLLiLLiLLi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_mmachs_rs1, "LLiLLiLLiLLi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_mmachs_s0, "LLiLLiLLiLLi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_mmachs_s1, "LLiLLiLLiLLi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_mmacls_rs0, "LLiLLiLLiLLi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_mmacls_rs1, "LLiLLiLLiLLi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_mmacls_s0, "LLiLLiLLiLLi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_mmacls_s1, "LLiLLiLLiLLi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_mmacuhs_rs0, "LLiLLiLLiLLi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_mmacuhs_rs1, "LLiLLiLLiLLi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_mmacuhs_s0, "LLiLLiLLiLLi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_mmacuhs_s1, "LLiLLiLLiLLi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_mmaculs_rs0, "LLiLLiLLiLLi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_mmaculs_rs1, "LLiLLiLLiLLi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_mmaculs_s0, "LLiLLiLLiLLi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_mmaculs_s1, "LLiLLiLLiLLi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_mmpyh_rs0, "LLiLLiLLi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_mmpyh_rs1, "LLiLLiLLi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_mmpyh_s0, "LLiLLiLLi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_mmpyh_s1, "LLiLLiLLi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_mmpyl_rs0, "LLiLLiLLi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_mmpyl_rs1, "LLiLLiLLi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_mmpyl_s0, "LLiLLiLLi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_mmpyl_s1, "LLiLLiLLi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_mmpyuh_rs0, "LLiLLiLLi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_mmpyuh_rs1, "LLiLLiLLi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_mmpyuh_s0, "LLiLLiLLi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_mmpyuh_s1, "LLiLLiLLi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_mmpyul_rs0, "LLiLLiLLi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_mmpyul_rs1, "LLiLLiLLi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_mmpyul_s0, "LLiLLiLLi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_mmpyul_s1, "LLiLLiLLi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_mpy_acc_hh_s0, "iiii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_mpy_acc_hh_s1, "iiii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_mpy_acc_hl_s0, "iiii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_mpy_acc_hl_s1, "iiii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_mpy_acc_lh_s0, "iiii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_mpy_acc_lh_s1, "iiii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_mpy_acc_ll_s0, "iiii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_mpy_acc_ll_s1, "iiii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_mpy_acc_sat_hh_s0, "iiii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_mpy_acc_sat_hh_s1, "iiii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_mpy_acc_sat_hl_s0, "iiii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_mpy_acc_sat_hl_s1, "iiii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_mpy_acc_sat_lh_s0, "iiii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_mpy_acc_sat_lh_s1, "iiii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_mpy_acc_sat_ll_s0, "iiii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_mpy_acc_sat_ll_s1, "iiii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_mpy_hh_s0, "iii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_mpy_hh_s1, "iii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_mpy_hl_s0, "iii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_mpy_hl_s1, "iii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_mpy_lh_s0, "iii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_mpy_lh_s1, "iii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_mpy_ll_s0, "iii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_mpy_ll_s1, "iii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_mpy_nac_hh_s0, "iiii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_mpy_nac_hh_s1, "iiii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_mpy_nac_hl_s0, "iiii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_mpy_nac_hl_s1, "iiii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_mpy_nac_lh_s0, "iiii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_mpy_nac_lh_s1, "iiii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_mpy_nac_ll_s0, "iiii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_mpy_nac_ll_s1, "iiii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_mpy_nac_sat_hh_s0, "iiii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_mpy_nac_sat_hh_s1, "iiii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_mpy_nac_sat_hl_s0, "iiii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_mpy_nac_sat_hl_s1, "iiii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_mpy_nac_sat_lh_s0, "iiii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_mpy_nac_sat_lh_s1, "iiii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_mpy_nac_sat_ll_s0, "iiii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_mpy_nac_sat_ll_s1, "iiii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_mpy_rnd_hh_s0, "iii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_mpy_rnd_hh_s1, "iii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_mpy_rnd_hl_s0, "iii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_mpy_rnd_hl_s1, "iii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_mpy_rnd_lh_s0, "iii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_mpy_rnd_lh_s1, "iii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_mpy_rnd_ll_s0, "iii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_mpy_rnd_ll_s1, "iii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_mpy_sat_hh_s0, "iii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_mpy_sat_hh_s1, "iii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_mpy_sat_hl_s0, "iii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_mpy_sat_hl_s1, "iii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_mpy_sat_lh_s0, "iii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_mpy_sat_lh_s1, "iii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_mpy_sat_ll_s0, "iii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_mpy_sat_ll_s1, "iii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_mpy_sat_rnd_hh_s0, "iii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_mpy_sat_rnd_hh_s1, "iii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_mpy_sat_rnd_hl_s0, "iii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_mpy_sat_rnd_hl_s1, "iii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_mpy_sat_rnd_lh_s0, "iii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_mpy_sat_rnd_lh_s1, "iii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_mpy_sat_rnd_ll_s0, "iii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_mpy_sat_rnd_ll_s1, "iii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_mpy_up, "iii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_mpy_up_s1, "iii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_mpy_up_s1_sat, "iii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_mpyd_acc_hh_s0, "LLiLLiii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_mpyd_acc_hh_s1, "LLiLLiii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_mpyd_acc_hl_s0, "LLiLLiii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_mpyd_acc_hl_s1, "LLiLLiii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_mpyd_acc_lh_s0, "LLiLLiii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_mpyd_acc_lh_s1, "LLiLLiii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_mpyd_acc_ll_s0, "LLiLLiii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_mpyd_acc_ll_s1, "LLiLLiii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_mpyd_hh_s0, "LLiii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_mpyd_hh_s1, "LLiii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_mpyd_hl_s0, "LLiii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_mpyd_hl_s1, "LLiii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_mpyd_lh_s0, "LLiii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_mpyd_lh_s1, "LLiii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_mpyd_ll_s0, "LLiii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_mpyd_ll_s1, "LLiii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_mpyd_nac_hh_s0, "LLiLLiii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_mpyd_nac_hh_s1, "LLiLLiii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_mpyd_nac_hl_s0, "LLiLLiii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_mpyd_nac_hl_s1, "LLiLLiii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_mpyd_nac_lh_s0, "LLiLLiii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_mpyd_nac_lh_s1, "LLiLLiii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_mpyd_nac_ll_s0, "LLiLLiii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_mpyd_nac_ll_s1, "LLiLLiii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_mpyd_rnd_hh_s0, "LLiii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_mpyd_rnd_hh_s1, "LLiii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_mpyd_rnd_hl_s0, "LLiii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_mpyd_rnd_hl_s1, "LLiii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_mpyd_rnd_lh_s0, "LLiii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_mpyd_rnd_lh_s1, "LLiii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_mpyd_rnd_ll_s0, "LLiii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_mpyd_rnd_ll_s1, "LLiii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_mpyi, "iii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_mpysmi, "iiIi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_mpysu_up, "iii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_mpyu_acc_hh_s0, "iiii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_mpyu_acc_hh_s1, "iiii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_mpyu_acc_hl_s0, "iiii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_mpyu_acc_hl_s1, "iiii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_mpyu_acc_lh_s0, "iiii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_mpyu_acc_lh_s1, "iiii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_mpyu_acc_ll_s0, "iiii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_mpyu_acc_ll_s1, "iiii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_mpyu_hh_s0, "Uiii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_mpyu_hh_s1, "Uiii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_mpyu_hl_s0, "Uiii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_mpyu_hl_s1, "Uiii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_mpyu_lh_s0, "Uiii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_mpyu_lh_s1, "Uiii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_mpyu_ll_s0, "Uiii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_mpyu_ll_s1, "Uiii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_mpyu_nac_hh_s0, "iiii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_mpyu_nac_hh_s1, "iiii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_mpyu_nac_hl_s0, "iiii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_mpyu_nac_hl_s1, "iiii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_mpyu_nac_lh_s0, "iiii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_mpyu_nac_lh_s1, "iiii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_mpyu_nac_ll_s0, "iiii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_mpyu_nac_ll_s1, "iiii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_mpyu_up, "Uiii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_mpyud_acc_hh_s0, "LLiLLiii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_mpyud_acc_hh_s1, "LLiLLiii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_mpyud_acc_hl_s0, "LLiLLiii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_mpyud_acc_hl_s1, "LLiLLiii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_mpyud_acc_lh_s0, "LLiLLiii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_mpyud_acc_lh_s1, "LLiLLiii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_mpyud_acc_ll_s0, "LLiLLiii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_mpyud_acc_ll_s1, "LLiLLiii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_mpyud_hh_s0, "ULLiii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_mpyud_hh_s1, "ULLiii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_mpyud_hl_s0, "ULLiii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_mpyud_hl_s1, "ULLiii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_mpyud_lh_s0, "ULLiii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_mpyud_lh_s1, "ULLiii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_mpyud_ll_s0, "ULLiii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_mpyud_ll_s1, "ULLiii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_mpyud_nac_hh_s0, "LLiLLiii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_mpyud_nac_hh_s1, "LLiLLiii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_mpyud_nac_hl_s0, "LLiLLiii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_mpyud_nac_hl_s1, "LLiLLiii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_mpyud_nac_lh_s0, "LLiLLiii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_mpyud_nac_lh_s1, "LLiLLiii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_mpyud_nac_ll_s0, "LLiLLiii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_mpyud_nac_ll_s1, "LLiLLiii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_mpyui, "iii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_nacci, "iiii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_naccii, "iiiIi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_subacc, "iiii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_vabsdiffh, "LLiLLiLLi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_vabsdiffw, "LLiLLiLLi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_vcmac_s0_sat_i, "LLiLLiLLiLLi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_vcmac_s0_sat_r, "LLiLLiLLiLLi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_vcmpy_s0_sat_i, "LLiLLiLLi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_vcmpy_s0_sat_r, "LLiLLiLLi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_vcmpy_s1_sat_i, "LLiLLiLLi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_vcmpy_s1_sat_r, "LLiLLiLLi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_vdmacs_s0, "LLiLLiLLiLLi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_vdmacs_s1, "LLiLLiLLiLLi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_vdmpyrs_s0, "iLLiLLi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_vdmpyrs_s1, "iLLiLLi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_vdmpys_s0, "LLiLLiLLi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_vdmpys_s1, "LLiLLiLLi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_vmac2, "LLiLLiii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_vmac2es, "LLiLLiLLiLLi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_vmac2es_s0, "LLiLLiLLiLLi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_vmac2es_s1, "LLiLLiLLiLLi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_vmac2s_s0, "LLiLLiii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_vmac2s_s1, "LLiLLiii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_vmac2su_s0, "LLiLLiii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_vmac2su_s1, "LLiLLiii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_vmpy2es_s0, "LLiLLiLLi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_vmpy2es_s1, "LLiLLiLLi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_vmpy2s_s0, "LLiii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_vmpy2s_s0pack, "iii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_vmpy2s_s1, "LLiii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_vmpy2s_s1pack, "iii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_vmpy2su_s0, "LLiii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_vmpy2su_s1, "LLiii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_vraddh, "iLLiLLi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_vradduh, "iLLiLLi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_vrcmaci_s0, "LLiLLiLLiLLi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_vrcmaci_s0c, "LLiLLiLLiLLi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_vrcmacr_s0, "LLiLLiLLiLLi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_vrcmacr_s0c, "LLiLLiLLiLLi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_vrcmpyi_s0, "LLiLLiLLi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_vrcmpyi_s0c, "LLiLLiLLi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_vrcmpyr_s0, "LLiLLiLLi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_vrcmpyr_s0c, "LLiLLiLLi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_vrcmpys_acc_s1, "LLiLLiLLii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_vrcmpys_s1, "LLiLLii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_vrcmpys_s1rp, "iLLii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_vrmac_s0, "LLiLLiLLiLLi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_vrmpy_s0, "LLiLLiLLi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_xor_xacc, "iiii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M4_and_and, "iiii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M4_and_andn, "iiii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M4_and_or, "iiii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M4_and_xor, "iiii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M4_cmpyi_wh, "iLLii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M4_cmpyi_whc, "iLLii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M4_cmpyr_wh, "iLLii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M4_cmpyr_whc, "iLLii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M4_mac_up_s1_sat, "iiii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M4_mpyri_addi, "iUIiiUIi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M4_mpyri_addr, "iiiUIi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M4_mpyri_addr_u2, "iiUIii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M4_mpyrr_addi, "iUIiii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M4_mpyrr_addr, "iiii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M4_nac_up_s1_sat, "iiii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M4_or_and, "iiii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M4_or_andn, "iiii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M4_or_or, "iiii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M4_or_xor, "iiii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M4_pmpyw, "LLiii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M4_pmpyw_acc, "LLiLLiii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M4_vpmpyh, "LLiii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M4_vpmpyh_acc, "LLiLLiii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M4_vrmpyeh_acc_s0, "LLiLLiLLiLLi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M4_vrmpyeh_acc_s1, "LLiLLiLLiLLi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M4_vrmpyeh_s0, "LLiLLiLLi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M4_vrmpyeh_s1, "LLiLLiLLi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M4_vrmpyoh_acc_s0, "LLiLLiLLiLLi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M4_vrmpyoh_acc_s1, "LLiLLiLLiLLi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M4_vrmpyoh_s0, "LLiLLiLLi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M4_vrmpyoh_s1, "LLiLLiLLi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M4_xor_and, "iiii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M4_xor_andn, "iiii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M4_xor_or, "iiii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M4_xor_xacc, "LLiLLiLLiLLi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M5_vdmacbsu, "LLiLLiLLiLLi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M5_vdmpybsu, "LLiLLiLLi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M5_vmacbsu, "LLiLLiii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M5_vmacbuu, "LLiLLiii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M5_vmpybsu, "LLiii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M5_vmpybuu, "LLiii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M5_vrmacbsu, "LLiLLiLLiLLi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M5_vrmacbuu, "LLiLLiLLiLLi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M5_vrmpybsu, "LLiLLiLLi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_M5_vrmpybuu, "LLiLLiLLi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_S2_addasl_rrri, "iiiUIi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_S2_asl_i_p, "LLiLLiUIi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_S2_asl_i_p_acc, "LLiLLiLLiUIi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_S2_asl_i_p_and, "LLiLLiLLiUIi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_S2_asl_i_p_nac, "LLiLLiLLiUIi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_S2_asl_i_p_or, "LLiLLiLLiUIi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_S2_asl_i_p_xacc, "LLiLLiLLiUIi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_S2_asl_i_r, "iiUIi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_S2_asl_i_r_acc, "iiiUIi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_S2_asl_i_r_and, "iiiUIi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_S2_asl_i_r_nac, "iiiUIi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_S2_asl_i_r_or, "iiiUIi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_S2_asl_i_r_sat, "iiUIi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_S2_asl_i_r_xacc, "iiiUIi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_S2_asl_i_vh, "LLiLLiUIi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_S2_asl_i_vw, "LLiLLiUIi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_S2_asl_r_p, "LLiLLii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_S2_asl_r_p_acc, "LLiLLiLLii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_S2_asl_r_p_and, "LLiLLiLLii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_S2_asl_r_p_nac, "LLiLLiLLii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_S2_asl_r_p_or, "LLiLLiLLii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_S2_asl_r_p_xor, "LLiLLiLLii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_S2_asl_r_r, "iii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_S2_asl_r_r_acc, "iiii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_S2_asl_r_r_and, "iiii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_S2_asl_r_r_nac, "iiii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_S2_asl_r_r_or, "iiii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_S2_asl_r_r_sat, "iii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_S2_asl_r_vh, "LLiLLii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_S2_asl_r_vw, "LLiLLii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_S2_asr_i_p, "LLiLLiUIi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_S2_asr_i_p_acc, "LLiLLiLLiUIi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_S2_asr_i_p_and, "LLiLLiLLiUIi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_S2_asr_i_p_nac, "LLiLLiLLiUIi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_S2_asr_i_p_or, "LLiLLiLLiUIi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_S2_asr_i_p_rnd, "LLiLLiUIi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_S2_asr_i_p_rnd_goodsyntax, "LLiLLiUIi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_S2_asr_i_r, "iiUIi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_S2_asr_i_r_acc, "iiiUIi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_S2_asr_i_r_and, "iiiUIi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_S2_asr_i_r_nac, "iiiUIi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_S2_asr_i_r_or, "iiiUIi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_S2_asr_i_r_rnd, "iiUIi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_S2_asr_i_r_rnd_goodsyntax, "iiUIi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_S2_asr_i_svw_trun, "iLLiUIi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_S2_asr_i_vh, "LLiLLiUIi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_S2_asr_i_vw, "LLiLLiUIi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_S2_asr_r_p, "LLiLLii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_S2_asr_r_p_acc, "LLiLLiLLii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_S2_asr_r_p_and, "LLiLLiLLii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_S2_asr_r_p_nac, "LLiLLiLLii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_S2_asr_r_p_or, "LLiLLiLLii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_S2_asr_r_p_xor, "LLiLLiLLii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_S2_asr_r_r, "iii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_S2_asr_r_r_acc, "iiii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_S2_asr_r_r_and, "iiii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_S2_asr_r_r_nac, "iiii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_S2_asr_r_r_or, "iiii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_S2_asr_r_r_sat, "iii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_S2_asr_r_svw_trun, "iLLii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_S2_asr_r_vh, "LLiLLii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_S2_asr_r_vw, "LLiLLii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_S2_brev, "ii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_S2_brevp, "LLiLLi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_S2_cl0, "ii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_S2_cl0p, "iLLi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_S2_cl1, "ii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_S2_cl1p, "iLLi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_S2_clb, "ii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_S2_clbnorm, "ii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_S2_clbp, "iLLi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_S2_clrbit_i, "iiUIi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_S2_clrbit_r, "iii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_S2_ct0, "ii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_S2_ct0p, "iLLi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_S2_ct1, "ii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_S2_ct1p, "iLLi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_S2_deinterleave, "LLiLLi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_S2_extractu, "iiUIiUIi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_S2_extractu_rp, "iiLLi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_S2_extractup, "LLiLLiUIiUIi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_S2_extractup_rp, "LLiLLiLLi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_S2_insert, "iiiUIiUIi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_S2_insert_rp, "iiiLLi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_S2_insertp, "LLiLLiLLiUIiUIi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_S2_insertp_rp, "LLiLLiLLiLLi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_S2_interleave, "LLiLLi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_S2_lfsp, "LLiLLiLLi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_S2_lsl_r_p, "LLiLLii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_S2_lsl_r_p_acc, "LLiLLiLLii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_S2_lsl_r_p_and, "LLiLLiLLii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_S2_lsl_r_p_nac, "LLiLLiLLii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_S2_lsl_r_p_or, "LLiLLiLLii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_S2_lsl_r_p_xor, "LLiLLiLLii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_S2_lsl_r_r, "iii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_S2_lsl_r_r_acc, "iiii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_S2_lsl_r_r_and, "iiii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_S2_lsl_r_r_nac, "iiii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_S2_lsl_r_r_or, "iiii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_S2_lsl_r_vh, "LLiLLii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_S2_lsl_r_vw, "LLiLLii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_S2_lsr_i_p, "LLiLLiUIi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_S2_lsr_i_p_acc, "LLiLLiLLiUIi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_S2_lsr_i_p_and, "LLiLLiLLiUIi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_S2_lsr_i_p_nac, "LLiLLiLLiUIi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_S2_lsr_i_p_or, "LLiLLiLLiUIi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_S2_lsr_i_p_xacc, "LLiLLiLLiUIi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_S2_lsr_i_r, "iiUIi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_S2_lsr_i_r_acc, "iiiUIi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_S2_lsr_i_r_and, "iiiUIi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_S2_lsr_i_r_nac, "iiiUIi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_S2_lsr_i_r_or, "iiiUIi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_S2_lsr_i_r_xacc, "iiiUIi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_S2_lsr_i_vh, "LLiLLiUIi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_S2_lsr_i_vw, "LLiLLiUIi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_S2_lsr_r_p, "LLiLLii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_S2_lsr_r_p_acc, "LLiLLiLLii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_S2_lsr_r_p_and, "LLiLLiLLii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_S2_lsr_r_p_nac, "LLiLLiLLii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_S2_lsr_r_p_or, "LLiLLiLLii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_S2_lsr_r_p_xor, "LLiLLiLLii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_S2_lsr_r_r, "iii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_S2_lsr_r_r_acc, "iiii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_S2_lsr_r_r_and, "iiii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_S2_lsr_r_r_nac, "iiii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_S2_lsr_r_r_or, "iiii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_S2_lsr_r_vh, "LLiLLii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_S2_lsr_r_vw, "LLiLLii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_S2_packhl, "LLiii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_S2_parityp, "iLLiLLi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_S2_setbit_i, "iiUIi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_S2_setbit_r, "iii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_S2_shuffeb, "LLiLLiLLi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_S2_shuffeh, "LLiLLiLLi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_S2_shuffob, "LLiLLiLLi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_S2_shuffoh, "LLiLLiLLi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_S2_svsathb, "ii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_S2_svsathub, "ii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_S2_tableidxb_goodsyntax, "iiiUIiUIi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_S2_tableidxd_goodsyntax, "iiiUIiUIi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_S2_tableidxh_goodsyntax, "iiiUIiUIi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_S2_tableidxw_goodsyntax, "iiiUIiUIi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_S2_togglebit_i, "iiUIi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_S2_togglebit_r, "iii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_S2_tstbit_i, "iiUIi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_S2_tstbit_r, "iii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_S2_valignib, "LLiLLiLLiUIi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_S2_valignrb, "LLiLLiLLii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_S2_vcnegh, "LLiLLii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_S2_vcrotate, "LLiLLii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_S2_vrcnegh, "LLiLLiLLii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_S2_vrndpackwh, "iLLi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_S2_vrndpackwhs, "iLLi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_S2_vsathb, "iLLi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_S2_vsathb_nopack, "LLiLLi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_S2_vsathub, "iLLi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_S2_vsathub_nopack, "LLiLLi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_S2_vsatwh, "iLLi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_S2_vsatwh_nopack, "LLiLLi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_S2_vsatwuh, "iLLi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_S2_vsatwuh_nopack, "LLiLLi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_S2_vsplatrb, "ii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_S2_vsplatrh, "LLii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_S2_vspliceib, "LLiLLiLLiUIi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_S2_vsplicerb, "LLiLLiLLii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_S2_vsxtbh, "LLii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_S2_vsxthw, "LLii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_S2_vtrunehb, "iLLi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_S2_vtrunewh, "LLiLLiLLi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_S2_vtrunohb, "iLLi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_S2_vtrunowh, "LLiLLiLLi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_S2_vzxtbh, "LLii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_S2_vzxthw, "LLii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_S4_addaddi, "iiiIi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_S4_addi_asl_ri, "iUIiiUIi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_S4_addi_lsr_ri, "iUIiiUIi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_S4_andi_asl_ri, "iUIiiUIi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_S4_andi_lsr_ri, "iUIiiUIi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_S4_clbaddi, "iiIi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_S4_clbpaddi, "iLLiIi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_S4_clbpnorm, "iLLi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_S4_extract, "iiUIiUIi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_S4_extract_rp, "iiLLi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_S4_extractp, "LLiLLiUIiUIi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_S4_extractp_rp, "LLiLLiLLi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_S4_lsli, "iIii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_S4_ntstbit_i, "iiUIi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_S4_ntstbit_r, "iii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_S4_or_andi, "iiiIi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_S4_or_andix, "iiiIi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_S4_or_ori, "iiiIi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_S4_ori_asl_ri, "iUIiiUIi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_S4_ori_lsr_ri, "iUIiiUIi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_S4_parity, "iii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_S4_subaddi, "iiIii", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_S4_subi_asl_ri, "iUIiiUIi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_S4_subi_lsr_ri, "iUIiiUIi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_S4_vrcrotate, "LLiLLiiUIi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_S4_vrcrotate_acc, "LLiLLiLLiiUIi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_S4_vxaddsubh, "LLiLLiLLi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_S4_vxaddsubhr, "LLiLLiLLi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_S4_vxaddsubw, "LLiLLiLLi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_S4_vxsubaddh, "LLiLLiLLi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_S4_vxsubaddhr, "LLiLLiLLi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_S4_vxsubaddw, "LLiLLiLLi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_S5_asrhub_rnd_sat_goodsyntax, "iLLiUIi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_S5_asrhub_sat, "iLLiUIi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_S5_popcountp, "iLLi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_S5_vasrhrnd_goodsyntax, "LLiLLiUIi", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_Y2_dccleana, "vv*", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_Y2_dccleaninva, "vv*", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_Y2_dcfetch, "vv*", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_Y2_dcinva, "vv*", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_Y2_dczeroa, "vv*", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_Y4_l2fetch, "vv*i", "", V5)
-TARGET_BUILTIN(__builtin_HEXAGON_Y5_l2fetch, "vv*LLi", "", V5)
-
-// V60 Scalar Instructions.
-
-TARGET_BUILTIN(__builtin_HEXAGON_S6_rol_i_p, "LLiLLiUIi", "", V60)
-TARGET_BUILTIN(__builtin_HEXAGON_S6_rol_i_p_acc, "LLiLLiLLiUIi", "", V60)
-TARGET_BUILTIN(__builtin_HEXAGON_S6_rol_i_p_and, "LLiLLiLLiUIi", "", V60)
-TARGET_BUILTIN(__builtin_HEXAGON_S6_rol_i_p_nac, "LLiLLiLLiUIi", "", V60)
-TARGET_BUILTIN(__builtin_HEXAGON_S6_rol_i_p_or, "LLiLLiLLiUIi", "", V60)
-TARGET_BUILTIN(__builtin_HEXAGON_S6_rol_i_p_xacc, "LLiLLiLLiUIi", "", V60)
-TARGET_BUILTIN(__builtin_HEXAGON_S6_rol_i_r, "iiUIi", "", V60)
-TARGET_BUILTIN(__builtin_HEXAGON_S6_rol_i_r_acc, "iiiUIi", "", V60)
-TARGET_BUILTIN(__builtin_HEXAGON_S6_rol_i_r_and, "iiiUIi", "", V60)
-TARGET_BUILTIN(__builtin_HEXAGON_S6_rol_i_r_nac, "iiiUIi", "", V60)
-TARGET_BUILTIN(__builtin_HEXAGON_S6_rol_i_r_or, "iiiUIi", "", V60)
-TARGET_BUILTIN(__builtin_HEXAGON_S6_rol_i_r_xacc, "iiiUIi", "", V60)
-
-// V62 Scalar Instructions.
-
-TARGET_BUILTIN(__builtin_HEXAGON_M6_vabsdiffb, "LLiLLiLLi", "", V62)
-TARGET_BUILTIN(__builtin_HEXAGON_M6_vabsdiffub, "LLiLLiLLi", "", V62)
-TARGET_BUILTIN(__builtin_HEXAGON_S6_vsplatrbp, "LLii", "", V62)
-TARGET_BUILTIN(__builtin_HEXAGON_S6_vtrunehb_ppp, "LLiLLiLLi", "", V62)
-TARGET_BUILTIN(__builtin_HEXAGON_S6_vtrunohb_ppp, "LLiLLiLLi", "", V62)
-
-// V65 Scalar Instructions.
-
-TARGET_BUILTIN(__builtin_HEXAGON_A6_vcmpbeq_notany, "iLLiLLi", "", V65)
-
-// V66 Scalar Instructions.
-
-TARGET_BUILTIN(__builtin_HEXAGON_F2_dfadd, "ddd", "", V66)
-TARGET_BUILTIN(__builtin_HEXAGON_F2_dfsub, "ddd", "", V66)
-TARGET_BUILTIN(__builtin_HEXAGON_M2_mnaci, "iiii", "", V66)
-TARGET_BUILTIN(__builtin_HEXAGON_S2_mask, "iUIiUIi", "", V66)
-
-// V67 Scalar Instructions.
-
-TARGET_BUILTIN(__builtin_HEXAGON_A7_clip, "iiUIi", "", "audio")
-TARGET_BUILTIN(__builtin_HEXAGON_A7_croundd_ri, "LLiLLiUIi", "", "audio")
-TARGET_BUILTIN(__builtin_HEXAGON_A7_croundd_rr, "LLiLLii", "", "audio")
-TARGET_BUILTIN(__builtin_HEXAGON_A7_vclip, "LLiLLiUIi", "", "audio")
-TARGET_BUILTIN(__builtin_HEXAGON_F2_dfmax, "ddd", "", V67)
-TARGET_BUILTIN(__builtin_HEXAGON_F2_dfmin, "ddd", "", V67)
-TARGET_BUILTIN(__builtin_HEXAGON_F2_dfmpyfix, "ddd", "", V67)
-TARGET_BUILTIN(__builtin_HEXAGON_F2_dfmpyhh, "dddd", "", V67)
-TARGET_BUILTIN(__builtin_HEXAGON_F2_dfmpylh, "dddd", "", V67)
-TARGET_BUILTIN(__builtin_HEXAGON_F2_dfmpyll, "ddd", "", V67)
-TARGET_BUILTIN(__builtin_HEXAGON_M7_dcmpyiw, "LLiLLiLLi", "", "audio")
-TARGET_BUILTIN(__builtin_HEXAGON_M7_dcmpyiw_acc, "LLiLLiLLiLLi", "", "audio")
-TARGET_BUILTIN(__builtin_HEXAGON_M7_dcmpyiwc, "LLiLLiLLi", "", "audio")
-TARGET_BUILTIN(__builtin_HEXAGON_M7_dcmpyiwc_acc, "LLiLLiLLiLLi", "", "audio")
-TARGET_BUILTIN(__builtin_HEXAGON_M7_dcmpyrw, "LLiLLiLLi", "", "audio")
-TARGET_BUILTIN(__builtin_HEXAGON_M7_dcmpyrw_acc, "LLiLLiLLiLLi", "", "audio")
-TARGET_BUILTIN(__builtin_HEXAGON_M7_dcmpyrwc, "LLiLLiLLi", "", "audio")
-TARGET_BUILTIN(__builtin_HEXAGON_M7_dcmpyrwc_acc, "LLiLLiLLiLLi", "", "audio")
-TARGET_BUILTIN(__builtin_HEXAGON_M7_vdmpy, "LLiLLiLLi", "", V67)
-TARGET_BUILTIN(__builtin_HEXAGON_M7_vdmpy_acc, "LLiLLiLLiLLi", "", V67)
-TARGET_BUILTIN(__builtin_HEXAGON_M7_wcmpyiw, "iLLiLLi", "", "audio")
-TARGET_BUILTIN(__builtin_HEXAGON_M7_wcmpyiw_rnd, "iLLiLLi", "", "audio")
-TARGET_BUILTIN(__builtin_HEXAGON_M7_wcmpyiwc, "iLLiLLi", "", "audio")
-TARGET_BUILTIN(__builtin_HEXAGON_M7_wcmpyiwc_rnd, "iLLiLLi", "", "audio")
-TARGET_BUILTIN(__builtin_HEXAGON_M7_wcmpyrw, "iLLiLLi", "", "audio")
-TARGET_BUILTIN(__builtin_HEXAGON_M7_wcmpyrw_rnd, "iLLiLLi", "", "audio")
-TARGET_BUILTIN(__builtin_HEXAGON_M7_wcmpyrwc, "iLLiLLi", "", "audio")
-TARGET_BUILTIN(__builtin_HEXAGON_M7_wcmpyrwc_rnd, "iLLiLLi", "", "audio")
-
-// V68 Scalar Instructions.
-
-TARGET_BUILTIN(__builtin_HEXAGON_Y6_dmlink, "vv*v*", "", V68)
-TARGET_BUILTIN(__builtin_HEXAGON_Y6_dmpause, "i", "", V68)
-TARGET_BUILTIN(__builtin_HEXAGON_Y6_dmpoll, "i", "", V68)
-TARGET_BUILTIN(__builtin_HEXAGON_Y6_dmresume, "vv*", "", V68)
-TARGET_BUILTIN(__builtin_HEXAGON_Y6_dmstart, "vv*", "", V68)
-TARGET_BUILTIN(__builtin_HEXAGON_Y6_dmwait, "i", "", V68)
-
-// V60 HVX Instructions.
-
-TARGET_BUILTIN(__builtin_HEXAGON_V6_extractw, "iV16ii", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_extractw_128B, "iV32ii", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_hi, "V16iV32i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_hi_128B, "V32iV64i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_lo, "V16iV32i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_lo_128B, "V32iV64i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_lvsplatw, "V16ii", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_lvsplatw_128B, "V32ii", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_pred_and, "V64bV64bV64b", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_pred_and_128B, "V128bV128bV128b", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_pred_and_n, "V64bV64bV64b", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_pred_and_n_128B, "V128bV128bV128b", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_pred_not, "V64bV64b", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_pred_not_128B, "V128bV128b", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_pred_or, "V64bV64bV64b", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_pred_or_128B, "V128bV128bV128b", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_pred_or_n, "V64bV64bV64b", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_pred_or_n_128B, "V128bV128bV128b", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_pred_scalar2, "V64bi", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_pred_scalar2_128B, "V128bi", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_pred_xor, "V64bV64bV64b", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_pred_xor_128B, "V128bV128bV128b", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vS32b_nqpred_ai, "vV64bv*V16i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vS32b_nqpred_ai_128B, "vV128bv*V32i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vS32b_nt_nqpred_ai, "vV64bv*V16i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vS32b_nt_nqpred_ai_128B, "vV128bv*V32i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vS32b_nt_qpred_ai, "vV64bv*V16i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vS32b_nt_qpred_ai_128B, "vV128bv*V32i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vS32b_qpred_ai, "vV64bv*V16i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vS32b_qpred_ai_128B, "vV128bv*V32i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vabsdiffh, "V16iV16iV16i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vabsdiffh_128B, "V32iV32iV32i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vabsdiffub, "V16iV16iV16i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vabsdiffub_128B, "V32iV32iV32i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vabsdiffuh, "V16iV16iV16i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vabsdiffuh_128B, "V32iV32iV32i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vabsdiffw, "V16iV16iV16i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vabsdiffw_128B, "V32iV32iV32i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vabsh, "V16iV16i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vabsh_128B, "V32iV32i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vabsh_sat, "V16iV16i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vabsh_sat_128B, "V32iV32i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vabsw, "V16iV16i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vabsw_128B, "V32iV32i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vabsw_sat, "V16iV16i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vabsw_sat_128B, "V32iV32i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vaddb, "V16iV16iV16i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vaddb_128B, "V32iV32iV32i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vaddb_dv, "V32iV32iV32i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vaddb_dv_128B, "V64iV64iV64i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vaddbnq, "V16iV64bV16iV16i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vaddbnq_128B, "V32iV128bV32iV32i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vaddbq, "V16iV64bV16iV16i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vaddbq_128B, "V32iV128bV32iV32i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vaddh, "V16iV16iV16i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vaddh_128B, "V32iV32iV32i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vaddh_dv, "V32iV32iV32i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vaddh_dv_128B, "V64iV64iV64i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vaddhnq, "V16iV64bV16iV16i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vaddhnq_128B, "V32iV128bV32iV32i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vaddhq, "V16iV64bV16iV16i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vaddhq_128B, "V32iV128bV32iV32i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vaddhsat, "V16iV16iV16i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vaddhsat_128B, "V32iV32iV32i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vaddhsat_dv, "V32iV32iV32i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vaddhsat_dv_128B, "V64iV64iV64i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vaddhw, "V32iV16iV16i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vaddhw_128B, "V64iV32iV32i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vaddubh, "V32iV16iV16i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vaddubh_128B, "V64iV32iV32i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vaddubsat, "V16iV16iV16i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vaddubsat_128B, "V32iV32iV32i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vaddubsat_dv, "V32iV32iV32i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vaddubsat_dv_128B, "V64iV64iV64i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vadduhsat, "V16iV16iV16i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vadduhsat_128B, "V32iV32iV32i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vadduhsat_dv, "V32iV32iV32i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vadduhsat_dv_128B, "V64iV64iV64i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vadduhw, "V32iV16iV16i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vadduhw_128B, "V64iV32iV32i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vaddw, "V16iV16iV16i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vaddw_128B, "V32iV32iV32i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vaddw_dv, "V32iV32iV32i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vaddw_dv_128B, "V64iV64iV64i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vaddwnq, "V16iV64bV16iV16i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vaddwnq_128B, "V32iV128bV32iV32i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vaddwq, "V16iV64bV16iV16i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vaddwq_128B, "V32iV128bV32iV32i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vaddwsat, "V16iV16iV16i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vaddwsat_128B, "V32iV32iV32i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vaddwsat_dv, "V32iV32iV32i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vaddwsat_dv_128B, "V64iV64iV64i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_valignb, "V16iV16iV16ii", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_valignb_128B, "V32iV32iV32ii", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_valignbi, "V16iV16iV16iUIi", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_valignbi_128B, "V32iV32iV32iUIi", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vand, "V16iV16iV16i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vand_128B, "V32iV32iV32i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vandqrt, "V16iV64bi", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vandqrt_128B, "V32iV128bi", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vandqrt_acc, "V16iV16iV64bi", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vandqrt_acc_128B, "V32iV32iV128bi", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vandvrt, "V64bV16ii", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vandvrt_128B, "V128bV32ii", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vandvrt_acc, "V64bV64bV16ii", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vandvrt_acc_128B, "V128bV128bV32ii", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vaslh, "V16iV16ii", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vaslh_128B, "V32iV32ii", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vaslhv, "V16iV16iV16i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vaslhv_128B, "V32iV32iV32i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vaslw, "V16iV16ii", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vaslw_128B, "V32iV32ii", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vaslw_acc, "V16iV16iV16ii", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vaslw_acc_128B, "V32iV32iV32ii", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vaslwv, "V16iV16iV16i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vaslwv_128B, "V32iV32iV32i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vasrh, "V16iV16ii", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vasrh_128B, "V32iV32ii", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vasrhbrndsat, "V16iV16iV16ii", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vasrhbrndsat_128B, "V32iV32iV32ii", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vasrhubrndsat, "V16iV16iV16ii", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vasrhubrndsat_128B, "V32iV32iV32ii", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vasrhubsat, "V16iV16iV16ii", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vasrhubsat_128B, "V32iV32iV32ii", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vasrhv, "V16iV16iV16i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vasrhv_128B, "V32iV32iV32i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vasrw, "V16iV16ii", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vasrw_128B, "V32iV32ii", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vasrw_acc, "V16iV16iV16ii", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vasrw_acc_128B, "V32iV32iV32ii", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vasrwh, "V16iV16iV16ii", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vasrwh_128B, "V32iV32iV32ii", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vasrwhrndsat, "V16iV16iV16ii", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vasrwhrndsat_128B, "V32iV32iV32ii", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vasrwhsat, "V16iV16iV16ii", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vasrwhsat_128B, "V32iV32iV32ii", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vasrwuhsat, "V16iV16iV16ii", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vasrwuhsat_128B, "V32iV32iV32ii", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vasrwv, "V16iV16iV16i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vasrwv_128B, "V32iV32iV32i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vassign, "V16iV16i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vassign_128B, "V32iV32i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vassignp, "V32iV32i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vassignp_128B, "V64iV64i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vavgh, "V16iV16iV16i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vavgh_128B, "V32iV32iV32i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vavghrnd, "V16iV16iV16i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vavghrnd_128B, "V32iV32iV32i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vavgub, "V16iV16iV16i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vavgub_128B, "V32iV32iV32i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vavgubrnd, "V16iV16iV16i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vavgubrnd_128B, "V32iV32iV32i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vavguh, "V16iV16iV16i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vavguh_128B, "V32iV32iV32i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vavguhrnd, "V16iV16iV16i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vavguhrnd_128B, "V32iV32iV32i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vavgw, "V16iV16iV16i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vavgw_128B, "V32iV32iV32i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vavgwrnd, "V16iV16iV16i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vavgwrnd_128B, "V32iV32iV32i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vcl0h, "V16iV16i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vcl0h_128B, "V32iV32i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vcl0w, "V16iV16i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vcl0w_128B, "V32iV32i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vcombine, "V32iV16iV16i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vcombine_128B, "V64iV32iV32i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vd0, "V16i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vd0_128B, "V32i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vdealb, "V16iV16i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vdealb_128B, "V32iV32i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vdealb4w, "V16iV16iV16i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vdealb4w_128B, "V32iV32iV32i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vdealh, "V16iV16i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vdealh_128B, "V32iV32i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vdealvdd, "V32iV16iV16ii", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vdealvdd_128B, "V64iV32iV32ii", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vdelta, "V16iV16iV16i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vdelta_128B, "V32iV32iV32i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vdmpybus, "V16iV16ii", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vdmpybus_128B, "V32iV32ii", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vdmpybus_acc, "V16iV16iV16ii", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vdmpybus_acc_128B, "V32iV32iV32ii", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vdmpybus_dv, "V32iV32ii", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vdmpybus_dv_128B, "V64iV64ii", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vdmpybus_dv_acc, "V32iV32iV32ii", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vdmpybus_dv_acc_128B, "V64iV64iV64ii", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vdmpyhb, "V16iV16ii", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vdmpyhb_128B, "V32iV32ii", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vdmpyhb_acc, "V16iV16iV16ii", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vdmpyhb_acc_128B, "V32iV32iV32ii", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vdmpyhb_dv, "V32iV32ii", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vdmpyhb_dv_128B, "V64iV64ii", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vdmpyhb_dv_acc, "V32iV32iV32ii", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vdmpyhb_dv_acc_128B, "V64iV64iV64ii", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vdmpyhisat, "V16iV32ii", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vdmpyhisat_128B, "V32iV64ii", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vdmpyhisat_acc, "V16iV16iV32ii", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vdmpyhisat_acc_128B, "V32iV32iV64ii", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vdmpyhsat, "V16iV16ii", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vdmpyhsat_128B, "V32iV32ii", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vdmpyhsat_acc, "V16iV16iV16ii", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vdmpyhsat_acc_128B, "V32iV32iV32ii", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vdmpyhsuisat, "V16iV32ii", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vdmpyhsuisat_128B, "V32iV64ii", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vdmpyhsuisat_acc, "V16iV16iV32ii", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vdmpyhsuisat_acc_128B, "V32iV32iV64ii", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vdmpyhsusat, "V16iV16ii", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vdmpyhsusat_128B, "V32iV32ii", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vdmpyhsusat_acc, "V16iV16iV16ii", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vdmpyhsusat_acc_128B, "V32iV32iV32ii", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vdmpyhvsat, "V16iV16iV16i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vdmpyhvsat_128B, "V32iV32iV32i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vdmpyhvsat_acc, "V16iV16iV16iV16i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vdmpyhvsat_acc_128B, "V32iV32iV32iV32i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vdsaduh, "V32iV32ii", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vdsaduh_128B, "V64iV64ii", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vdsaduh_acc, "V32iV32iV32ii", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vdsaduh_acc_128B, "V64iV64iV64ii", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_veqb, "V64bV16iV16i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_veqb_128B, "V128bV32iV32i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_veqb_and, "V64bV64bV16iV16i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_veqb_and_128B, "V128bV128bV32iV32i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_veqb_or, "V64bV64bV16iV16i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_veqb_or_128B, "V128bV128bV32iV32i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_veqb_xor, "V64bV64bV16iV16i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_veqb_xor_128B, "V128bV128bV32iV32i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_veqh, "V64bV16iV16i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_veqh_128B, "V128bV32iV32i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_veqh_and, "V64bV64bV16iV16i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_veqh_and_128B, "V128bV128bV32iV32i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_veqh_or, "V64bV64bV16iV16i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_veqh_or_128B, "V128bV128bV32iV32i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_veqh_xor, "V64bV64bV16iV16i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_veqh_xor_128B, "V128bV128bV32iV32i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_veqw, "V64bV16iV16i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_veqw_128B, "V128bV32iV32i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_veqw_and, "V64bV64bV16iV16i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_veqw_and_128B, "V128bV128bV32iV32i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_veqw_or, "V64bV64bV16iV16i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_veqw_or_128B, "V128bV128bV32iV32i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_veqw_xor, "V64bV64bV16iV16i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_veqw_xor_128B, "V128bV128bV32iV32i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vgtb, "V64bV16iV16i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vgtb_128B, "V128bV32iV32i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vgtb_and, "V64bV64bV16iV16i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vgtb_and_128B, "V128bV128bV32iV32i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vgtb_or, "V64bV64bV16iV16i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vgtb_or_128B, "V128bV128bV32iV32i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vgtb_xor, "V64bV64bV16iV16i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vgtb_xor_128B, "V128bV128bV32iV32i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vgth, "V64bV16iV16i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vgth_128B, "V128bV32iV32i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vgth_and, "V64bV64bV16iV16i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vgth_and_128B, "V128bV128bV32iV32i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vgth_or, "V64bV64bV16iV16i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vgth_or_128B, "V128bV128bV32iV32i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vgth_xor, "V64bV64bV16iV16i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vgth_xor_128B, "V128bV128bV32iV32i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vgtub, "V64bV16iV16i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vgtub_128B, "V128bV32iV32i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vgtub_and, "V64bV64bV16iV16i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vgtub_and_128B, "V128bV128bV32iV32i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vgtub_or, "V64bV64bV16iV16i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vgtub_or_128B, "V128bV128bV32iV32i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vgtub_xor, "V64bV64bV16iV16i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vgtub_xor_128B, "V128bV128bV32iV32i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vgtuh, "V64bV16iV16i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vgtuh_128B, "V128bV32iV32i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vgtuh_and, "V64bV64bV16iV16i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vgtuh_and_128B, "V128bV128bV32iV32i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vgtuh_or, "V64bV64bV16iV16i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vgtuh_or_128B, "V128bV128bV32iV32i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vgtuh_xor, "V64bV64bV16iV16i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vgtuh_xor_128B, "V128bV128bV32iV32i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vgtuw, "V64bV16iV16i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vgtuw_128B, "V128bV32iV32i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vgtuw_and, "V64bV64bV16iV16i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vgtuw_and_128B, "V128bV128bV32iV32i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vgtuw_or, "V64bV64bV16iV16i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vgtuw_or_128B, "V128bV128bV32iV32i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vgtuw_xor, "V64bV64bV16iV16i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vgtuw_xor_128B, "V128bV128bV32iV32i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vgtw, "V64bV16iV16i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vgtw_128B, "V128bV32iV32i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vgtw_and, "V64bV64bV16iV16i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vgtw_and_128B, "V128bV128bV32iV32i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vgtw_or, "V64bV64bV16iV16i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vgtw_or_128B, "V128bV128bV32iV32i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vgtw_xor, "V64bV64bV16iV16i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vgtw_xor_128B, "V128bV128bV32iV32i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vinsertwr, "V16iV16ii", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vinsertwr_128B, "V32iV32ii", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vlalignb, "V16iV16iV16ii", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vlalignb_128B, "V32iV32iV32ii", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vlalignbi, "V16iV16iV16iUIi", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vlalignbi_128B, "V32iV32iV32iUIi", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vlsrh, "V16iV16ii", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vlsrh_128B, "V32iV32ii", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vlsrhv, "V16iV16iV16i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vlsrhv_128B, "V32iV32iV32i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vlsrw, "V16iV16ii", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vlsrw_128B, "V32iV32ii", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vlsrwv, "V16iV16iV16i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vlsrwv_128B, "V32iV32iV32i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vlutvvb, "V16iV16iV16ii", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vlutvvb_128B, "V32iV32iV32ii", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vlutvvb_oracc, "V16iV16iV16iV16ii", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vlutvvb_oracc_128B, "V32iV32iV32iV32ii", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vlutvwh, "V32iV16iV16ii", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vlutvwh_128B, "V64iV32iV32ii", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vlutvwh_oracc, "V32iV32iV16iV16ii", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vlutvwh_oracc_128B, "V64iV64iV32iV32ii", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vmaxh, "V16iV16iV16i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vmaxh_128B, "V32iV32iV32i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vmaxub, "V16iV16iV16i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vmaxub_128B, "V32iV32iV32i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vmaxuh, "V16iV16iV16i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vmaxuh_128B, "V32iV32iV32i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vmaxw, "V16iV16iV16i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vmaxw_128B, "V32iV32iV32i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vminh, "V16iV16iV16i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vminh_128B, "V32iV32iV32i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vminub, "V16iV16iV16i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vminub_128B, "V32iV32iV32i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vminuh, "V16iV16iV16i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vminuh_128B, "V32iV32iV32i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vminw, "V16iV16iV16i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vminw_128B, "V32iV32iV32i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vmpabus, "V32iV32ii", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vmpabus_128B, "V64iV64ii", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vmpabus_acc, "V32iV32iV32ii", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vmpabus_acc_128B, "V64iV64iV64ii", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vmpabusv, "V32iV32iV32i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vmpabusv_128B, "V64iV64iV64i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vmpabuuv, "V32iV32iV32i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vmpabuuv_128B, "V64iV64iV64i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vmpahb, "V32iV32ii", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vmpahb_128B, "V64iV64ii", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vmpahb_acc, "V32iV32iV32ii", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vmpahb_acc_128B, "V64iV64iV64ii", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vmpybus, "V32iV16ii", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vmpybus_128B, "V64iV32ii", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vmpybus_acc, "V32iV32iV16ii", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vmpybus_acc_128B, "V64iV64iV32ii", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vmpybusv, "V32iV16iV16i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vmpybusv_128B, "V64iV32iV32i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vmpybusv_acc, "V32iV32iV16iV16i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vmpybusv_acc_128B, "V64iV64iV32iV32i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vmpybv, "V32iV16iV16i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vmpybv_128B, "V64iV32iV32i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vmpybv_acc, "V32iV32iV16iV16i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vmpybv_acc_128B, "V64iV64iV32iV32i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vmpyewuh, "V16iV16iV16i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vmpyewuh_128B, "V32iV32iV32i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vmpyh, "V32iV16ii", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vmpyh_128B, "V64iV32ii", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vmpyhsat_acc, "V32iV32iV16ii", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vmpyhsat_acc_128B, "V64iV64iV32ii", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vmpyhsrs, "V16iV16ii", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vmpyhsrs_128B, "V32iV32ii", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vmpyhss, "V16iV16ii", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vmpyhss_128B, "V32iV32ii", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vmpyhus, "V32iV16iV16i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vmpyhus_128B, "V64iV32iV32i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vmpyhus_acc, "V32iV32iV16iV16i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vmpyhus_acc_128B, "V64iV64iV32iV32i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vmpyhv, "V32iV16iV16i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vmpyhv_128B, "V64iV32iV32i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vmpyhv_acc, "V32iV32iV16iV16i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vmpyhv_acc_128B, "V64iV64iV32iV32i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vmpyhvsrs, "V16iV16iV16i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vmpyhvsrs_128B, "V32iV32iV32i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vmpyieoh, "V16iV16iV16i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vmpyieoh_128B, "V32iV32iV32i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vmpyiewh_acc, "V16iV16iV16iV16i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vmpyiewh_acc_128B, "V32iV32iV32iV32i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vmpyiewuh, "V16iV16iV16i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vmpyiewuh_128B, "V32iV32iV32i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vmpyiewuh_acc, "V16iV16iV16iV16i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vmpyiewuh_acc_128B, "V32iV32iV32iV32i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vmpyih, "V16iV16iV16i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vmpyih_128B, "V32iV32iV32i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vmpyih_acc, "V16iV16iV16iV16i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vmpyih_acc_128B, "V32iV32iV32iV32i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vmpyihb, "V16iV16ii", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vmpyihb_128B, "V32iV32ii", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vmpyihb_acc, "V16iV16iV16ii", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vmpyihb_acc_128B, "V32iV32iV32ii", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vmpyiowh, "V16iV16iV16i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vmpyiowh_128B, "V32iV32iV32i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vmpyiwb, "V16iV16ii", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vmpyiwb_128B, "V32iV32ii", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vmpyiwb_acc, "V16iV16iV16ii", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vmpyiwb_acc_128B, "V32iV32iV32ii", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vmpyiwh, "V16iV16ii", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vmpyiwh_128B, "V32iV32ii", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vmpyiwh_acc, "V16iV16iV16ii", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vmpyiwh_acc_128B, "V32iV32iV32ii", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vmpyowh, "V16iV16iV16i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vmpyowh_128B, "V32iV32iV32i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vmpyowh_rnd, "V16iV16iV16i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vmpyowh_rnd_128B, "V32iV32iV32i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vmpyowh_rnd_sacc, "V16iV16iV16iV16i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vmpyowh_rnd_sacc_128B, "V32iV32iV32iV32i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vmpyowh_sacc, "V16iV16iV16iV16i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vmpyowh_sacc_128B, "V32iV32iV32iV32i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vmpyub, "V32iV16ii", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vmpyub_128B, "V64iV32ii", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vmpyub_acc, "V32iV32iV16ii", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vmpyub_acc_128B, "V64iV64iV32ii", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vmpyubv, "V32iV16iV16i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vmpyubv_128B, "V64iV32iV32i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vmpyubv_acc, "V32iV32iV16iV16i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vmpyubv_acc_128B, "V64iV64iV32iV32i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vmpyuh, "V32iV16ii", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vmpyuh_128B, "V64iV32ii", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vmpyuh_acc, "V32iV32iV16ii", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vmpyuh_acc_128B, "V64iV64iV32ii", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vmpyuhv, "V32iV16iV16i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vmpyuhv_128B, "V64iV32iV32i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vmpyuhv_acc, "V32iV32iV16iV16i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vmpyuhv_acc_128B, "V64iV64iV32iV32i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vmux, "V16iV64bV16iV16i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vmux_128B, "V32iV128bV32iV32i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vnavgh, "V16iV16iV16i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vnavgh_128B, "V32iV32iV32i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vnavgub, "V16iV16iV16i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vnavgub_128B, "V32iV32iV32i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vnavgw, "V16iV16iV16i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vnavgw_128B, "V32iV32iV32i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vnormamth, "V16iV16i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vnormamth_128B, "V32iV32i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vnormamtw, "V16iV16i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vnormamtw_128B, "V32iV32i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vnot, "V16iV16i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vnot_128B, "V32iV32i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vor, "V16iV16iV16i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vor_128B, "V32iV32iV32i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vpackeb, "V16iV16iV16i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vpackeb_128B, "V32iV32iV32i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vpackeh, "V16iV16iV16i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vpackeh_128B, "V32iV32iV32i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vpackhb_sat, "V16iV16iV16i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vpackhb_sat_128B, "V32iV32iV32i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vpackhub_sat, "V16iV16iV16i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vpackhub_sat_128B, "V32iV32iV32i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vpackob, "V16iV16iV16i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vpackob_128B, "V32iV32iV32i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vpackoh, "V16iV16iV16i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vpackoh_128B, "V32iV32iV32i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vpackwh_sat, "V16iV16iV16i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vpackwh_sat_128B, "V32iV32iV32i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vpackwuh_sat, "V16iV16iV16i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vpackwuh_sat_128B, "V32iV32iV32i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vpopcounth, "V16iV16i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vpopcounth_128B, "V32iV32i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vrdelta, "V16iV16iV16i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vrdelta_128B, "V32iV32iV32i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vrmpybus, "V16iV16ii", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vrmpybus_128B, "V32iV32ii", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vrmpybus_acc, "V16iV16iV16ii", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vrmpybus_acc_128B, "V32iV32iV32ii", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vrmpybusi, "V32iV32iiUIi", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vrmpybusi_128B, "V64iV64iiUIi", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vrmpybusi_acc, "V32iV32iV32iiUIi", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vrmpybusi_acc_128B, "V64iV64iV64iiUIi", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vrmpybusv, "V16iV16iV16i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vrmpybusv_128B, "V32iV32iV32i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vrmpybusv_acc, "V16iV16iV16iV16i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vrmpybusv_acc_128B, "V32iV32iV32iV32i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vrmpybv, "V16iV16iV16i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vrmpybv_128B, "V32iV32iV32i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vrmpybv_acc, "V16iV16iV16iV16i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vrmpybv_acc_128B, "V32iV32iV32iV32i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vrmpyub, "V16iV16ii", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vrmpyub_128B, "V32iV32ii", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vrmpyub_acc, "V16iV16iV16ii", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vrmpyub_acc_128B, "V32iV32iV32ii", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vrmpyubi, "V32iV32iiUIi", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vrmpyubi_128B, "V64iV64iiUIi", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vrmpyubi_acc, "V32iV32iV32iiUIi", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vrmpyubi_acc_128B, "V64iV64iV64iiUIi", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vrmpyubv, "V16iV16iV16i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vrmpyubv_128B, "V32iV32iV32i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vrmpyubv_acc, "V16iV16iV16iV16i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vrmpyubv_acc_128B, "V32iV32iV32iV32i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vror, "V16iV16ii", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vror_128B, "V32iV32ii", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vroundhb, "V16iV16iV16i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vroundhb_128B, "V32iV32iV32i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vroundhub, "V16iV16iV16i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vroundhub_128B, "V32iV32iV32i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vroundwh, "V16iV16iV16i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vroundwh_128B, "V32iV32iV32i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vroundwuh, "V16iV16iV16i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vroundwuh_128B, "V32iV32iV32i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vrsadubi, "V32iV32iiUIi", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vrsadubi_128B, "V64iV64iiUIi", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vrsadubi_acc, "V32iV32iV32iiUIi", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vrsadubi_acc_128B, "V64iV64iV64iiUIi", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vsathub, "V16iV16iV16i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vsathub_128B, "V32iV32iV32i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vsatwh, "V16iV16iV16i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vsatwh_128B, "V32iV32iV32i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vsb, "V32iV16i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vsb_128B, "V64iV32i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vsh, "V32iV16i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vsh_128B, "V64iV32i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vshufeh, "V16iV16iV16i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vshufeh_128B, "V32iV32iV32i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vshuffb, "V16iV16i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vshuffb_128B, "V32iV32i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vshuffeb, "V16iV16iV16i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vshuffeb_128B, "V32iV32iV32i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vshuffh, "V16iV16i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vshuffh_128B, "V32iV32i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vshuffob, "V16iV16iV16i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vshuffob_128B, "V32iV32iV32i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vshuffvdd, "V32iV16iV16ii", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vshuffvdd_128B, "V64iV32iV32ii", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vshufoeb, "V32iV16iV16i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vshufoeb_128B, "V64iV32iV32i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vshufoeh, "V32iV16iV16i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vshufoeh_128B, "V64iV32iV32i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vshufoh, "V16iV16iV16i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vshufoh_128B, "V32iV32iV32i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vsubb, "V16iV16iV16i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vsubb_128B, "V32iV32iV32i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vsubb_dv, "V32iV32iV32i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vsubb_dv_128B, "V64iV64iV64i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vsubbnq, "V16iV64bV16iV16i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vsubbnq_128B, "V32iV128bV32iV32i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vsubbq, "V16iV64bV16iV16i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vsubbq_128B, "V32iV128bV32iV32i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vsubh, "V16iV16iV16i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vsubh_128B, "V32iV32iV32i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vsubh_dv, "V32iV32iV32i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vsubh_dv_128B, "V64iV64iV64i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vsubhnq, "V16iV64bV16iV16i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vsubhnq_128B, "V32iV128bV32iV32i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vsubhq, "V16iV64bV16iV16i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vsubhq_128B, "V32iV128bV32iV32i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vsubhsat, "V16iV16iV16i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vsubhsat_128B, "V32iV32iV32i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vsubhsat_dv, "V32iV32iV32i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vsubhsat_dv_128B, "V64iV64iV64i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vsubhw, "V32iV16iV16i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vsubhw_128B, "V64iV32iV32i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vsububh, "V32iV16iV16i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vsububh_128B, "V64iV32iV32i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vsububsat, "V16iV16iV16i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vsububsat_128B, "V32iV32iV32i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vsububsat_dv, "V32iV32iV32i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vsububsat_dv_128B, "V64iV64iV64i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vsubuhsat, "V16iV16iV16i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vsubuhsat_128B, "V32iV32iV32i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vsubuhsat_dv, "V32iV32iV32i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vsubuhsat_dv_128B, "V64iV64iV64i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vsubuhw, "V32iV16iV16i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vsubuhw_128B, "V64iV32iV32i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vsubw, "V16iV16iV16i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vsubw_128B, "V32iV32iV32i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vsubw_dv, "V32iV32iV32i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vsubw_dv_128B, "V64iV64iV64i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vsubwnq, "V16iV64bV16iV16i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vsubwnq_128B, "V32iV128bV32iV32i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vsubwq, "V16iV64bV16iV16i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vsubwq_128B, "V32iV128bV32iV32i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vsubwsat, "V16iV16iV16i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vsubwsat_128B, "V32iV32iV32i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vsubwsat_dv, "V32iV32iV32i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vsubwsat_dv_128B, "V64iV64iV64i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vswap, "V32iV64bV16iV16i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vswap_128B, "V64iV128bV32iV32i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vtmpyb, "V32iV32ii", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vtmpyb_128B, "V64iV64ii", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vtmpyb_acc, "V32iV32iV32ii", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vtmpyb_acc_128B, "V64iV64iV64ii", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vtmpybus, "V32iV32ii", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vtmpybus_128B, "V64iV64ii", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vtmpybus_acc, "V32iV32iV32ii", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vtmpybus_acc_128B, "V64iV64iV64ii", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vtmpyhb, "V32iV32ii", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vtmpyhb_128B, "V64iV64ii", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vtmpyhb_acc, "V32iV32iV32ii", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vtmpyhb_acc_128B, "V64iV64iV64ii", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vunpackb, "V32iV16i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vunpackb_128B, "V64iV32i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vunpackh, "V32iV16i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vunpackh_128B, "V64iV32i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vunpackob, "V32iV32iV16i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vunpackob_128B, "V64iV64iV32i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vunpackoh, "V32iV32iV16i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vunpackoh_128B, "V64iV64iV32i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vunpackub, "V32iV16i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vunpackub_128B, "V64iV32i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vunpackuh, "V32iV16i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vunpackuh_128B, "V64iV32i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vxor, "V16iV16iV16i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vxor_128B, "V32iV32iV32i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vzb, "V32iV16i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vzb_128B, "V64iV32i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vzh, "V32iV16i", "", HVXV60)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vzh_128B, "V64iV32i", "", HVXV60)
-
-// V62 HVX Instructions.
-
-TARGET_BUILTIN(__builtin_HEXAGON_V6_lvsplatb, "V16ii", "", HVXV62)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_lvsplatb_128B, "V32ii", "", HVXV62)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_lvsplath, "V16ii", "", HVXV62)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_lvsplath_128B, "V32ii", "", HVXV62)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_pred_scalar2v2, "V64bi", "", HVXV62)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_pred_scalar2v2_128B, "V128bi", "", HVXV62)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_shuffeqh, "V64bV64bV64b", "", HVXV62)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_shuffeqh_128B, "V128bV128bV128b", "", HVXV62)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_shuffeqw, "V64bV64bV64b", "", HVXV62)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_shuffeqw_128B, "V128bV128bV128b", "", HVXV62)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vaddbsat, "V16iV16iV16i", "", HVXV62)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vaddbsat_128B, "V32iV32iV32i", "", HVXV62)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vaddbsat_dv, "V32iV32iV32i", "", HVXV62)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vaddbsat_dv_128B, "V64iV64iV64i", "", HVXV62)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vaddcarry, "V16iV16iV16iv*", "", HVXV62)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vaddcarry_128B, "V32iV32iV32iv*", "", HVXV62)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vaddclbh, "V16iV16iV16i", "", HVXV62)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vaddclbh_128B, "V32iV32iV32i", "", HVXV62)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vaddclbw, "V16iV16iV16i", "", HVXV62)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vaddclbw_128B, "V32iV32iV32i", "", HVXV62)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vaddhw_acc, "V32iV32iV16iV16i", "", HVXV62)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vaddhw_acc_128B, "V64iV64iV32iV32i", "", HVXV62)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vaddubh_acc, "V32iV32iV16iV16i", "", HVXV62)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vaddubh_acc_128B, "V64iV64iV32iV32i", "", HVXV62)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vaddububb_sat, "V16iV16iV16i", "", HVXV62)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vaddububb_sat_128B, "V32iV32iV32i", "", HVXV62)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vadduhw_acc, "V32iV32iV16iV16i", "", HVXV62)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vadduhw_acc_128B, "V64iV64iV32iV32i", "", HVXV62)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vadduwsat, "V16iV16iV16i", "", HVXV62)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vadduwsat_128B, "V32iV32iV32i", "", HVXV62)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vadduwsat_dv, "V32iV32iV32i", "", HVXV62)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vadduwsat_dv_128B, "V64iV64iV64i", "", HVXV62)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vandnqrt, "V16iV64bi", "", HVXV62)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vandnqrt_128B, "V32iV128bi", "", HVXV62)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vandnqrt_acc, "V16iV16iV64bi", "", HVXV62)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vandnqrt_acc_128B, "V32iV32iV128bi", "", HVXV62)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vandvnqv, "V16iV64bV16i", "", HVXV62)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vandvnqv_128B, "V32iV128bV32i", "", HVXV62)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vandvqv, "V16iV64bV16i", "", HVXV62)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vandvqv_128B, "V32iV128bV32i", "", HVXV62)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vasrhbsat, "V16iV16iV16ii", "", HVXV62)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vasrhbsat_128B, "V32iV32iV32ii", "", HVXV62)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vasruwuhrndsat, "V16iV16iV16ii", "", HVXV62)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vasruwuhrndsat_128B, "V32iV32iV32ii", "", HVXV62)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vasrwuhrndsat, "V16iV16iV16ii", "", HVXV62)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vasrwuhrndsat_128B, "V32iV32iV32ii", "", HVXV62)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vlsrb, "V16iV16ii", "", HVXV62)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vlsrb_128B, "V32iV32ii", "", HVXV62)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vlutvvb_nm, "V16iV16iV16ii", "", HVXV62)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vlutvvb_nm_128B, "V32iV32iV32ii", "", HVXV62)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vlutvvb_oracci, "V16iV16iV16iV16iUIi", "", HVXV62)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vlutvvb_oracci_128B, "V32iV32iV32iV32iUIi", "", HVXV62)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vlutvvbi, "V16iV16iV16iUIi", "", HVXV62)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vlutvvbi_128B, "V32iV32iV32iUIi", "", HVXV62)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vlutvwh_nm, "V32iV16iV16ii", "", HVXV62)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vlutvwh_nm_128B, "V64iV32iV32ii", "", HVXV62)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vlutvwh_oracci, "V32iV32iV16iV16iUIi", "", HVXV62)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vlutvwh_oracci_128B, "V64iV64iV32iV32iUIi", "", HVXV62)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vlutvwhi, "V32iV16iV16iUIi", "", HVXV62)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vlutvwhi_128B, "V64iV32iV32iUIi", "", HVXV62)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vmaxb, "V16iV16iV16i", "", HVXV62)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vmaxb_128B, "V32iV32iV32i", "", HVXV62)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vminb, "V16iV16iV16i", "", HVXV62)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vminb_128B, "V32iV32iV32i", "", HVXV62)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vmpauhb, "V32iV32ii", "", HVXV62)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vmpauhb_128B, "V64iV64ii", "", HVXV62)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vmpauhb_acc, "V32iV32iV32ii", "", HVXV62)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vmpauhb_acc_128B, "V64iV64iV64ii", "", HVXV62)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vmpyewuh_64, "V32iV16iV16i", "", HVXV62)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vmpyewuh_64_128B, "V64iV32iV32i", "", HVXV62)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vmpyiwub, "V16iV16ii", "", HVXV62)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vmpyiwub_128B, "V32iV32ii", "", HVXV62)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vmpyiwub_acc, "V16iV16iV16ii", "", HVXV62)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vmpyiwub_acc_128B, "V32iV32iV32ii", "", HVXV62)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vmpyowh_64_acc, "V32iV32iV16iV16i", "", HVXV62)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vmpyowh_64_acc_128B, "V64iV64iV32iV32i", "", HVXV62)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vrounduhub, "V16iV16iV16i", "", HVXV62)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vrounduhub_128B, "V32iV32iV32i", "", HVXV62)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vrounduwuh, "V16iV16iV16i", "", HVXV62)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vrounduwuh_128B, "V32iV32iV32i", "", HVXV62)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vsatuwuh, "V16iV16iV16i", "", HVXV62)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vsatuwuh_128B, "V32iV32iV32i", "", HVXV62)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vsubbsat, "V16iV16iV16i", "", HVXV62)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vsubbsat_128B, "V32iV32iV32i", "", HVXV62)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vsubbsat_dv, "V32iV32iV32i", "", HVXV62)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vsubbsat_dv_128B, "V64iV64iV64i", "", HVXV62)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vsubcarry, "V16iV16iV16iv*", "", HVXV62)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vsubcarry_128B, "V32iV32iV32iv*", "", HVXV62)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vsubububb_sat, "V16iV16iV16i", "", HVXV62)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vsubububb_sat_128B, "V32iV32iV32i", "", HVXV62)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vsubuwsat, "V16iV16iV16i", "", HVXV62)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vsubuwsat_128B, "V32iV32iV32i", "", HVXV62)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vsubuwsat_dv, "V32iV32iV32i", "", HVXV62)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vsubuwsat_dv_128B, "V64iV64iV64i", "", HVXV62)
-
-// V65 HVX Instructions.
-
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vabsb, "V16iV16i", "", HVXV65)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vabsb_128B, "V32iV32i", "", HVXV65)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vabsb_sat, "V16iV16i", "", HVXV65)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vabsb_sat_128B, "V32iV32i", "", HVXV65)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vaslh_acc, "V16iV16iV16ii", "", HVXV65)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vaslh_acc_128B, "V32iV32iV32ii", "", HVXV65)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vasrh_acc, "V16iV16iV16ii", "", HVXV65)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vasrh_acc_128B, "V32iV32iV32ii", "", HVXV65)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vasruhubrndsat, "V16iV16iV16ii", "", HVXV65)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vasruhubrndsat_128B, "V32iV32iV32ii", "", HVXV65)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vasruhubsat, "V16iV16iV16ii", "", HVXV65)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vasruhubsat_128B, "V32iV32iV32ii", "", HVXV65)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vasruwuhsat, "V16iV16iV16ii", "", HVXV65)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vasruwuhsat_128B, "V32iV32iV32ii", "", HVXV65)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vavgb, "V16iV16iV16i", "", HVXV65)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vavgb_128B, "V32iV32iV32i", "", HVXV65)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vavgbrnd, "V16iV16iV16i", "", HVXV65)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vavgbrnd_128B, "V32iV32iV32i", "", HVXV65)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vavguw, "V16iV16iV16i", "", HVXV65)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vavguw_128B, "V32iV32iV32i", "", HVXV65)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vavguwrnd, "V16iV16iV16i", "", HVXV65)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vavguwrnd_128B, "V32iV32iV32i", "", HVXV65)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vdd0, "V32i", "", HVXV65)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vdd0_128B, "V64i", "", HVXV65)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vgathermh, "vv*iiV16i", "", HVXV65)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vgathermh_128B, "vv*iiV32i", "", HVXV65)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vgathermhq, "vv*V64biiV16i", "", HVXV65)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vgathermhq_128B, "vv*V128biiV32i", "", HVXV65)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vgathermhw, "vv*iiV32i", "", HVXV65)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vgathermhw_128B, "vv*iiV64i", "", HVXV65)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vgathermhwq, "vv*V64biiV32i", "", HVXV65)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vgathermhwq_128B, "vv*V128biiV64i", "", HVXV65)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vgathermw, "vv*iiV16i", "", HVXV65)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vgathermw_128B, "vv*iiV32i", "", HVXV65)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vgathermwq, "vv*V64biiV16i", "", HVXV65)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vgathermwq_128B, "vv*V128biiV32i", "", HVXV65)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vlut4, "V16iV16iLLi", "", HVXV65)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vlut4_128B, "V32iV32iLLi", "", HVXV65)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vmpabuu, "V32iV32ii", "", HVXV65)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vmpabuu_128B, "V64iV64ii", "", HVXV65)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vmpabuu_acc, "V32iV32iV32ii", "", HVXV65)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vmpabuu_acc_128B, "V64iV64iV64ii", "", HVXV65)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vmpahhsat, "V16iV16iV16iLLi", "", HVXV65)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vmpahhsat_128B, "V32iV32iV32iLLi", "", HVXV65)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vmpauhuhsat, "V16iV16iV16iLLi", "", HVXV65)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vmpauhuhsat_128B, "V32iV32iV32iLLi", "", HVXV65)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vmpsuhuhsat, "V16iV16iV16iLLi", "", HVXV65)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vmpsuhuhsat_128B, "V32iV32iV32iLLi", "", HVXV65)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vmpyh_acc, "V32iV32iV16ii", "", HVXV65)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vmpyh_acc_128B, "V64iV64iV32ii", "", HVXV65)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vmpyuhe, "V16iV16ii", "", HVXV65)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vmpyuhe_128B, "V32iV32ii", "", HVXV65)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vmpyuhe_acc, "V16iV16iV16ii", "", HVXV65)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vmpyuhe_acc_128B, "V32iV32iV32ii", "", HVXV65)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vnavgb, "V16iV16iV16i", "", HVXV65)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vnavgb_128B, "V32iV32iV32i", "", HVXV65)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vprefixqb, "V16iV64b", "", HVXV65)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vprefixqb_128B, "V32iV128b", "", HVXV65)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vprefixqh, "V16iV64b", "", HVXV65)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vprefixqh_128B, "V32iV128b", "", HVXV65)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vprefixqw, "V16iV64b", "", HVXV65)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vprefixqw_128B, "V32iV128b", "", HVXV65)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vscattermh, "viiV16iV16i", "", HVXV65)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vscattermh_128B, "viiV32iV32i", "", HVXV65)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vscattermh_add, "viiV16iV16i", "", HVXV65)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vscattermh_add_128B, "viiV32iV32i", "", HVXV65)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vscattermhq, "vV64biiV16iV16i", "", HVXV65)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vscattermhq_128B, "vV128biiV32iV32i", "", HVXV65)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vscattermhw, "viiV32iV16i", "", HVXV65)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vscattermhw_128B, "viiV64iV32i", "", HVXV65)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vscattermhw_add, "viiV32iV16i", "", HVXV65)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vscattermhw_add_128B, "viiV64iV32i", "", HVXV65)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vscattermhwq, "vV64biiV32iV16i", "", HVXV65)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vscattermhwq_128B, "vV128biiV64iV32i", "", HVXV65)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vscattermw, "viiV16iV16i", "", HVXV65)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vscattermw_128B, "viiV32iV32i", "", HVXV65)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vscattermw_add, "viiV16iV16i", "", HVXV65)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vscattermw_add_128B, "viiV32iV32i", "", HVXV65)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vscattermwq, "vV64biiV16iV16i", "", HVXV65)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vscattermwq_128B, "vV128biiV32iV32i", "", HVXV65)
-
-// V66 HVX Instructions.
-
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vaddcarryo, "V16iV16iV16iv*", "", HVXV66)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vaddcarryo_128B, "V32iV32iV32iv*", "", HVXV66)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vaddcarrysat, "V16iV16iV16iV64b", "", HVXV66)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vaddcarrysat_128B, "V32iV32iV32iV128b", "", HVXV66)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vasr_into, "V32iV32iV16iV16i", "", HVXV66)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vasr_into_128B, "V64iV64iV32iV32i", "", HVXV66)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vrotr, "V16iV16iV16i", "", HVXV66)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vrotr_128B, "V32iV32iV32i", "", HVXV66)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vsatdw, "V16iV16iV16i", "", HVXV66)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vsatdw_128B, "V32iV32iV32i", "", HVXV66)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vsubcarryo, "V16iV16iV16iv*", "", HVXV66)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vsubcarryo_128B, "V32iV32iV32iv*", "", HVXV66)
-
-// V68 HVX Instructions.
-
-TARGET_BUILTIN(__builtin_HEXAGON_V6_v6mpyhubs10, "V32iV32iV32iUIi", "", HVXV68)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_v6mpyhubs10_128B, "V64iV64iV64iUIi", "", HVXV68)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_v6mpyhubs10_vxx, "V32iV32iV32iV32iUIi", "", HVXV68)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_v6mpyhubs10_vxx_128B, "V64iV64iV64iV64iUIi", "", HVXV68)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_v6mpyvubs10, "V32iV32iV32iUIi", "", HVXV68)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_v6mpyvubs10_128B, "V64iV64iV64iUIi", "", HVXV68)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_v6mpyvubs10_vxx, "V32iV32iV32iV32iUIi", "", HVXV68)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_v6mpyvubs10_vxx_128B, "V64iV64iV64iV64iUIi", "", HVXV68)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vabs_hf, "V16iV16i", "", HVXV68)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vabs_hf_128B, "V32iV32i", "", HVXV68)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vabs_sf, "V16iV16i", "", HVXV68)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vabs_sf_128B, "V32iV32i", "", HVXV68)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vadd_hf, "V16iV16iV16i", "", HVXV68)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vadd_hf_128B, "V32iV32iV32i", "", HVXV68)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vadd_hf_hf, "V16iV16iV16i", "", HVXV68)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vadd_hf_hf_128B, "V32iV32iV32i", "", HVXV68)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vadd_qf16, "V16iV16iV16i", "", HVXV68)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vadd_qf16_128B, "V32iV32iV32i", "", HVXV68)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vadd_qf16_mix, "V16iV16iV16i", "", HVXV68)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vadd_qf16_mix_128B, "V32iV32iV32i", "", HVXV68)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vadd_qf32, "V16iV16iV16i", "", HVXV68)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vadd_qf32_128B, "V32iV32iV32i", "", HVXV68)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vadd_qf32_mix, "V16iV16iV16i", "", HVXV68)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vadd_qf32_mix_128B, "V32iV32iV32i", "", HVXV68)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vadd_sf, "V16iV16iV16i", "", HVXV68)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vadd_sf_128B, "V32iV32iV32i", "", HVXV68)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vadd_sf_hf, "V32iV16iV16i", "", HVXV68)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vadd_sf_hf_128B, "V64iV32iV32i", "", HVXV68)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vadd_sf_sf, "V16iV16iV16i", "", HVXV68)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vadd_sf_sf_128B, "V32iV32iV32i", "", HVXV68)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vassign_fp, "V16iV16i", "", HVXV68)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vassign_fp_128B, "V32iV32i", "", HVXV68)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vconv_hf_qf16, "V16iV16i", "", HVXV68)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vconv_hf_qf16_128B, "V32iV32i", "", HVXV68)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vconv_hf_qf32, "V16iV32i", "", HVXV68)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vconv_hf_qf32_128B, "V32iV64i", "", HVXV68)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vconv_sf_qf32, "V16iV16i", "", HVXV68)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vconv_sf_qf32_128B, "V32iV32i", "", HVXV68)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vcvt_b_hf, "V16iV16iV16i", "", HVXV68)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vcvt_b_hf_128B, "V32iV32iV32i", "", HVXV68)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vcvt_h_hf, "V16iV16i", "", HVXV68)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vcvt_h_hf_128B, "V32iV32i", "", HVXV68)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vcvt_hf_b, "V32iV16i", "", HVXV68)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vcvt_hf_b_128B, "V64iV32i", "", HVXV68)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vcvt_hf_h, "V16iV16i", "", HVXV68)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vcvt_hf_h_128B, "V32iV32i", "", HVXV68)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vcvt_hf_sf, "V16iV16iV16i", "", HVXV68)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vcvt_hf_sf_128B, "V32iV32iV32i", "", HVXV68)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vcvt_hf_ub, "V32iV16i", "", HVXV68)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vcvt_hf_ub_128B, "V64iV32i", "", HVXV68)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vcvt_hf_uh, "V16iV16i", "", HVXV68)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vcvt_hf_uh_128B, "V32iV32i", "", HVXV68)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vcvt_sf_hf, "V32iV16i", "", HVXV68)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vcvt_sf_hf_128B, "V64iV32i", "", HVXV68)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vcvt_ub_hf, "V16iV16iV16i", "", HVXV68)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vcvt_ub_hf_128B, "V32iV32iV32i", "", HVXV68)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vcvt_uh_hf, "V16iV16i", "", HVXV68)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vcvt_uh_hf_128B, "V32iV32i", "", HVXV68)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vdmpy_sf_hf, "V16iV16iV16i", "", HVXV68)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vdmpy_sf_hf_128B, "V32iV32iV32i", "", HVXV68)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vdmpy_sf_hf_acc, "V16iV16iV16iV16i", "", HVXV68)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vdmpy_sf_hf_acc_128B, "V32iV32iV32iV32i", "", HVXV68)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vfmax_hf, "V16iV16iV16i", "", HVXV68)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vfmax_hf_128B, "V32iV32iV32i", "", HVXV68)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vfmax_sf, "V16iV16iV16i", "", HVXV68)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vfmax_sf_128B, "V32iV32iV32i", "", HVXV68)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vfmin_hf, "V16iV16iV16i", "", HVXV68)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vfmin_hf_128B, "V32iV32iV32i", "", HVXV68)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vfmin_sf, "V16iV16iV16i", "", HVXV68)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vfmin_sf_128B, "V32iV32iV32i", "", HVXV68)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vfneg_hf, "V16iV16i", "", HVXV68)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vfneg_hf_128B, "V32iV32i", "", HVXV68)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vfneg_sf, "V16iV16i", "", HVXV68)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vfneg_sf_128B, "V32iV32i", "", HVXV68)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vgthf, "V64bV16iV16i", "", HVXV68)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vgthf_128B, "V128bV32iV32i", "", HVXV68)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vgthf_and, "V64bV64bV16iV16i", "", HVXV68)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vgthf_and_128B, "V128bV128bV32iV32i", "", HVXV68)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vgthf_or, "V64bV64bV16iV16i", "", HVXV68)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vgthf_or_128B, "V128bV128bV32iV32i", "", HVXV68)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vgthf_xor, "V64bV64bV16iV16i", "", HVXV68)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vgthf_xor_128B, "V128bV128bV32iV32i", "", HVXV68)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vgtsf, "V64bV16iV16i", "", HVXV68)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vgtsf_128B, "V128bV32iV32i", "", HVXV68)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vgtsf_and, "V64bV64bV16iV16i", "", HVXV68)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vgtsf_and_128B, "V128bV128bV32iV32i", "", HVXV68)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vgtsf_or, "V64bV64bV16iV16i", "", HVXV68)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vgtsf_or_128B, "V128bV128bV32iV32i", "", HVXV68)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vgtsf_xor, "V64bV64bV16iV16i", "", HVXV68)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vgtsf_xor_128B, "V128bV128bV32iV32i", "", HVXV68)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vmax_hf, "V16iV16iV16i", "", HVXV68)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vmax_hf_128B, "V32iV32iV32i", "", HVXV68)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vmax_sf, "V16iV16iV16i", "", HVXV68)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vmax_sf_128B, "V32iV32iV32i", "", HVXV68)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vmin_hf, "V16iV16iV16i", "", HVXV68)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vmin_hf_128B, "V32iV32iV32i", "", HVXV68)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vmin_sf, "V16iV16iV16i", "", HVXV68)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vmin_sf_128B, "V32iV32iV32i", "", HVXV68)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vmpy_hf_hf, "V16iV16iV16i", "", HVXV68)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vmpy_hf_hf_128B, "V32iV32iV32i", "", HVXV68)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vmpy_hf_hf_acc, "V16iV16iV16iV16i", "", HVXV68)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vmpy_hf_hf_acc_128B, "V32iV32iV32iV32i", "", HVXV68)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vmpy_qf16, "V16iV16iV16i", "", HVXV68)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vmpy_qf16_128B, "V32iV32iV32i", "", HVXV68)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vmpy_qf16_hf, "V16iV16iV16i", "", HVXV68)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vmpy_qf16_hf_128B, "V32iV32iV32i", "", HVXV68)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vmpy_qf16_mix_hf, "V16iV16iV16i", "", HVXV68)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vmpy_qf16_mix_hf_128B, "V32iV32iV32i", "", HVXV68)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vmpy_qf32, "V16iV16iV16i", "", HVXV68)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vmpy_qf32_128B, "V32iV32iV32i", "", HVXV68)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vmpy_qf32_hf, "V32iV16iV16i", "", HVXV68)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vmpy_qf32_hf_128B, "V64iV32iV32i", "", HVXV68)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vmpy_qf32_mix_hf, "V32iV16iV16i", "", HVXV68)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vmpy_qf32_mix_hf_128B, "V64iV32iV32i", "", HVXV68)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vmpy_qf32_qf16, "V32iV16iV16i", "", HVXV68)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vmpy_qf32_qf16_128B, "V64iV32iV32i", "", HVXV68)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vmpy_qf32_sf, "V16iV16iV16i", "", HVXV68)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vmpy_qf32_sf_128B, "V32iV32iV32i", "", HVXV68)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vmpy_sf_hf, "V32iV16iV16i", "", HVXV68)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vmpy_sf_hf_128B, "V64iV32iV32i", "", HVXV68)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vmpy_sf_hf_acc, "V32iV32iV16iV16i", "", HVXV68)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vmpy_sf_hf_acc_128B, "V64iV64iV32iV32i", "", HVXV68)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vmpy_sf_sf, "V16iV16iV16i", "", HVXV68)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vmpy_sf_sf_128B, "V32iV32iV32i", "", HVXV68)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vsub_hf, "V16iV16iV16i", "", HVXV68)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vsub_hf_128B, "V32iV32iV32i", "", HVXV68)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vsub_hf_hf, "V16iV16iV16i", "", HVXV68)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vsub_hf_hf_128B, "V32iV32iV32i", "", HVXV68)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vsub_qf16, "V16iV16iV16i", "", HVXV68)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vsub_qf16_128B, "V32iV32iV32i", "", HVXV68)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vsub_qf16_mix, "V16iV16iV16i", "", HVXV68)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vsub_qf16_mix_128B, "V32iV32iV32i", "", HVXV68)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vsub_qf32, "V16iV16iV16i", "", HVXV68)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vsub_qf32_128B, "V32iV32iV32i", "", HVXV68)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vsub_qf32_mix, "V16iV16iV16i", "", HVXV68)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vsub_qf32_mix_128B, "V32iV32iV32i", "", HVXV68)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vsub_sf, "V16iV16iV16i", "", HVXV68)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vsub_sf_128B, "V32iV32iV32i", "", HVXV68)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vsub_sf_hf, "V32iV16iV16i", "", HVXV68)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vsub_sf_hf_128B, "V64iV32iV32i", "", HVXV68)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vsub_sf_sf, "V16iV16iV16i", "", HVXV68)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vsub_sf_sf_128B, "V32iV32iV32i", "", HVXV68)
-
-// V69 HVX Instructions.
-
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vasrvuhubrndsat, "V16iV32iV16i", "", HVXV69)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vasrvuhubrndsat_128B, "V32iV64iV32i", "", HVXV69)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vasrvuhubsat, "V16iV32iV16i", "", HVXV69)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vasrvuhubsat_128B, "V32iV64iV32i", "", HVXV69)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vasrvwuhrndsat, "V16iV32iV16i", "", HVXV69)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vasrvwuhrndsat_128B, "V32iV64iV32i", "", HVXV69)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vasrvwuhsat, "V16iV32iV16i", "", HVXV69)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vasrvwuhsat_128B, "V32iV64iV32i", "", HVXV69)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vmpyuhvs, "V16iV16iV16i", "", HVXV69)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vmpyuhvs_128B, "V32iV32iV32i", "", HVXV69)
-
-// V73 HVX Instructions.
-
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vadd_sf_bf, "V32iV16iV16i", "", HVXV73)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vadd_sf_bf_128B, "V64iV32iV32i", "", HVXV73)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vconv_h_hf, "V16iV16i", "", HVXV73)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vconv_h_hf_128B, "V32iV32i", "", HVXV73)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vconv_hf_h, "V16iV16i", "", HVXV73)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vconv_hf_h_128B, "V32iV32i", "", HVXV73)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vconv_sf_w, "V16iV16i", "", HVXV73)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vconv_sf_w_128B, "V32iV32i", "", HVXV73)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vconv_w_sf, "V16iV16i", "", HVXV73)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vconv_w_sf_128B, "V32iV32i", "", HVXV73)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vcvt_bf_sf, "V16iV16iV16i", "", HVXV73)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vcvt_bf_sf_128B, "V32iV32iV32i", "", HVXV73)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vgtbf, "V64bV16iV16i", "", HVXV73)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vgtbf_128B, "V128bV32iV32i", "", HVXV73)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vgtbf_and, "V64bV64bV16iV16i", "", HVXV73)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vgtbf_and_128B, "V128bV128bV32iV32i", "", HVXV73)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vgtbf_or, "V64bV64bV16iV16i", "", HVXV73)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vgtbf_or_128B, "V128bV128bV32iV32i", "", HVXV73)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vgtbf_xor, "V64bV64bV16iV16i", "", HVXV73)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vgtbf_xor_128B, "V128bV128bV32iV32i", "", HVXV73)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vmax_bf, "V16iV16iV16i", "", HVXV73)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vmax_bf_128B, "V32iV32iV32i", "", HVXV73)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vmin_bf, "V16iV16iV16i", "", HVXV73)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vmin_bf_128B, "V32iV32iV32i", "", HVXV73)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vmpy_sf_bf, "V32iV16iV16i", "", HVXV73)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vmpy_sf_bf_128B, "V64iV32iV32i", "", HVXV73)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vmpy_sf_bf_acc, "V32iV32iV16iV16i", "", HVXV73)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vmpy_sf_bf_acc_128B, "V64iV64iV32iV32i", "", HVXV73)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vsub_sf_bf, "V32iV16iV16i", "", HVXV73)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vsub_sf_bf_128B, "V64iV32iV32i", "", HVXV73)
-
-// V79 HVX Instructions.
-
-TARGET_BUILTIN(__builtin_HEXAGON_V6_get_qfext, "V16iV16ii", "", HVXV79)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_get_qfext_128B, "V32iV32ii", "", HVXV79)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_get_qfext_oracc, "V16iV16iV16ii", "", HVXV79)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_get_qfext_oracc_128B, "V32iV32iV32ii", "", HVXV79)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_set_qfext, "V16iV16ii", "", HVXV79)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_set_qfext_128B, "V32iV32ii", "", HVXV79)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vabs_f8, "V16iV16i", "", HVXV79)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vabs_f8_128B, "V32iV32i", "", HVXV79)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vadd_hf_f8, "V32iV16iV16i", "", HVXV79)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vadd_hf_f8_128B, "V64iV32iV32i", "", HVXV79)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vcvt2_b_hf, "V16iV16iV16i", "", HVXV79)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vcvt2_b_hf_128B, "V32iV32iV32i", "", HVXV79)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vcvt2_hf_b, "V32iV16i", "", HVXV79)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vcvt2_hf_b_128B, "V64iV32i", "", HVXV79)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vcvt2_hf_ub, "V32iV16i", "", HVXV79)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vcvt2_hf_ub_128B, "V64iV32i", "", HVXV79)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vcvt2_ub_hf, "V16iV16iV16i", "", HVXV79)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vcvt2_ub_hf_128B, "V32iV32iV32i", "", HVXV79)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vcvt_f8_hf, "V16iV16iV16i", "", HVXV79)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vcvt_f8_hf_128B, "V32iV32iV32i", "", HVXV79)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vcvt_hf_f8, "V32iV16i", "", HVXV79)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vcvt_hf_f8_128B, "V64iV32i", "", HVXV79)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vfmax_f8, "V16iV16iV16i", "", HVXV79)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vfmax_f8_128B, "V32iV32iV32i", "", HVXV79)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vfmin_f8, "V16iV16iV16i", "", HVXV79)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vfmin_f8_128B, "V32iV32iV32i", "", HVXV79)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vfneg_f8, "V16iV16i", "", HVXV79)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vfneg_f8_128B, "V32iV32i", "", HVXV79)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vmerge_qf, "V16iV16iV16i", "", HVXV79)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vmerge_qf_128B, "V32iV32iV32i", "", HVXV79)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vmpy_hf_f8, "V32iV16iV16i", "", HVXV79)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vmpy_hf_f8_128B, "V64iV32iV32i", "", HVXV79)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vmpy_hf_f8_acc, "V32iV32iV16iV16i", "", HVXV79)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vmpy_hf_f8_acc_128B, "V64iV64iV32iV32i", "", HVXV79)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vmpy_rt_hf, "V16iV16ii", "", HVXV79)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vmpy_rt_hf_128B, "V32iV32ii", "", HVXV79)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vmpy_rt_qf16, "V16iV16ii", "", HVXV79)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vmpy_rt_qf16_128B, "V32iV32ii", "", HVXV79)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vmpy_rt_sf, "V16iV16ii", "", HVXV79)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vmpy_rt_sf_128B, "V32iV32ii", "", HVXV79)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vsub_hf_f8, "V32iV16iV16i", "", HVXV79)
-TARGET_BUILTIN(__builtin_HEXAGON_V6_vsub_hf_f8_128B, "V64iV32iV32i", "", HVXV79)
diff --git a/clang/include/clang/Basic/BuiltinsLoongArch.def b/clang/include/clang/Basic/BuiltinsLoongArch.def
deleted file mode 100644
index 95359a3fdc711..0000000000000
--- a/clang/include/clang/Basic/BuiltinsLoongArch.def
+++ /dev/null
@@ -1,28 +0,0 @@
-//==- BuiltinsLoongArch.def - LoongArch Builtin function database -- C++ -*-==//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-//
-// This file defines the LoongArch-specific builtin function database.  Users of
-// this file must define the BUILTIN macro to make use of this information.
-//
-//===----------------------------------------------------------------------===//
-
-#if defined(BUILTIN) && !defined(TARGET_BUILTIN)
-#   define TARGET_BUILTIN(ID, TYPE, ATTRS, FEATURE) BUILTIN(ID, TYPE, ATTRS)
-#endif
-
-// Definition of LoongArch basic builtins.
-#include "clang/Basic/BuiltinsLoongArchBase.def"
-
-// Definition of LSX builtins.
-#include "clang/Basic/BuiltinsLoongArchLSX.def"
-
-// Definition of LASX builtins.
-#include "clang/Basic/BuiltinsLoongArchLASX.def"
-
-#undef BUILTIN
-#undef TARGET_BUILTIN
diff --git a/clang/include/clang/Basic/BuiltinsNEON.def b/clang/include/clang/Basic/BuiltinsNEON.def
deleted file mode 100644
index 9627005ba9824..0000000000000
--- a/clang/include/clang/Basic/BuiltinsNEON.def
+++ /dev/null
@@ -1,22 +0,0 @@
-//===--- BuiltinsNEON.def - NEON Builtin function database ------*- C++ -*-===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-//
-// This file defines the NEON-specific builtin function database.  Users of
-// this file must define the BUILTIN macro to make use of this information.
-//
-//===----------------------------------------------------------------------===//
-
-// The format of this database matches clang/Basic/Builtins.def.
-
-#define GET_NEON_BUILTINS
-#include "clang/Basic/arm_neon.inc"
-#include "clang/Basic/arm_fp16.inc"
-#undef GET_NEON_BUILTINS
-
-#undef BUILTIN
-#undef TARGET_BUILTIN
diff --git a/clang/include/clang/Basic/BuiltinsNVPTX.def b/clang/include/clang/Basic/BuiltinsNVPTX.def
deleted file mode 100644
index 37b4e6ff77fda..0000000000000
--- a/clang/include/clang/Basic/BuiltinsNVPTX.def
+++ /dev/null
@@ -1,1119 +0,0 @@
-//===--- BuiltinsPTX.def - PTX Builtin function database ----*- C++ -*-===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-//
-// This file defines the PTX-specific builtin function database.  Users of
-// this file must define the BUILTIN macro to make use of this information.
-//
-//===----------------------------------------------------------------------===//
-
-// The format of this database matches clang/Basic/Builtins.def.
-
-#if defined(BUILTIN) && !defined(TARGET_BUILTIN)
-#   define TARGET_BUILTIN(ID, TYPE, ATTRS, FEATURE) BUILTIN(ID, TYPE, ATTRS)
-#endif
-
-#pragma push_macro("SM_53")
-#pragma push_macro("SM_70")
-#pragma push_macro("SM_72")
-#pragma push_macro("SM_75")
-#pragma push_macro("SM_80")
-#pragma push_macro("SM_86")
-#pragma push_macro("SM_87")
-#pragma push_macro("SM_89")
-#pragma push_macro("SM_90")
-#pragma push_macro("SM_90a")
-#pragma push_macro("SM_100")
-#pragma push_macro("SM_100a")
-#define SM_100a "sm_100a"
-#define SM_100 "sm_100|" SM_100a
-#define SM_90a "sm_90a"
-#define SM_90 "sm_90|" SM_90a "|" SM_100
-#define SM_89 "sm_89|" SM_90
-#define SM_87 "sm_87|" SM_89
-#define SM_86 "sm_86|" SM_87
-#define SM_80 "sm_80|" SM_86
-#define SM_75 "sm_75|" SM_80
-#define SM_72 "sm_72|" SM_75
-#define SM_70 "sm_70|" SM_72
-
-#pragma push_macro("SM_60")
-#define SM_60 "sm_60|sm_61|sm_62|" SM_70
-#define SM_53 "sm_53|" SM_60
-
-#pragma push_macro("PTX42")
-#pragma push_macro("PTX60")
-#pragma push_macro("PTX61")
-#pragma push_macro("PTX62")
-#pragma push_macro("PTX63")
-#pragma push_macro("PTX64")
-#pragma push_macro("PTX65")
-#pragma push_macro("PTX70")
-#pragma push_macro("PTX71")
-#pragma push_macro("PTX72")
-#pragma push_macro("PTX73")
-#pragma push_macro("PTX74")
-#pragma push_macro("PTX75")
-#pragma push_macro("PTX76")
-#pragma push_macro("PTX77")
-#pragma push_macro("PTX78")
-#pragma push_macro("PTX80")
-#pragma push_macro("PTX81")
-#pragma push_macro("PTX82")
-#pragma push_macro("PTX83")
-#pragma push_macro("PTX84")
-#pragma push_macro("PTX85")
-#pragma push_macro("PTX86")
-#define PTX86 "ptx86"
-#define PTX85 "ptx85|" PTX86
-#define PTX84 "ptx84|" PTX85
-#define PTX83 "ptx83|" PTX84
-#define PTX82 "ptx82|" PTX83
-#define PTX81 "ptx81|" PTX82
-#define PTX80 "ptx80|" PTX81
-#define PTX78 "ptx78|" PTX80
-#define PTX77 "ptx77|" PTX78
-#define PTX76 "ptx76|" PTX77
-#define PTX75 "ptx75|" PTX76
-#define PTX74 "ptx74|" PTX75
-#define PTX73 "ptx73|" PTX74
-#define PTX72 "ptx72|" PTX73
-#define PTX71 "ptx71|" PTX72
-#define PTX70 "ptx70|" PTX71
-#define PTX65 "ptx65|" PTX70
-#define PTX64 "ptx64|" PTX65
-#define PTX63 "ptx63|" PTX64
-#define PTX62 "ptx62|" PTX63
-#define PTX61 "ptx61|" PTX62
-#define PTX60 "ptx60|" PTX61
-#define PTX42 "ptx42|" PTX60
-
-#pragma push_macro("AND")
-#define AND(a, b) "(" a "),(" b ")"
-
-// Special Registers
-
-BUILTIN(__nvvm_read_ptx_sreg_tid_x, "i", "nc")
-BUILTIN(__nvvm_read_ptx_sreg_tid_y, "i", "nc")
-BUILTIN(__nvvm_read_ptx_sreg_tid_z, "i", "nc")
-BUILTIN(__nvvm_read_ptx_sreg_tid_w, "i", "nc")
-
-BUILTIN(__nvvm_read_ptx_sreg_ntid_x, "i", "nc")
-BUILTIN(__nvvm_read_ptx_sreg_ntid_y, "i", "nc")
-BUILTIN(__nvvm_read_ptx_sreg_ntid_z, "i", "nc")
-BUILTIN(__nvvm_read_ptx_sreg_ntid_w, "i", "nc")
-
-BUILTIN(__nvvm_read_ptx_sreg_ctaid_x, "i", "nc")
-BUILTIN(__nvvm_read_ptx_sreg_ctaid_y, "i", "nc")
-BUILTIN(__nvvm_read_ptx_sreg_ctaid_z, "i", "nc")
-BUILTIN(__nvvm_read_ptx_sreg_ctaid_w, "i", "nc")
-
-BUILTIN(__nvvm_read_ptx_sreg_nctaid_x, "i", "nc")
-BUILTIN(__nvvm_read_ptx_sreg_nctaid_y, "i", "nc")
-BUILTIN(__nvvm_read_ptx_sreg_nctaid_z, "i", "nc")
-BUILTIN(__nvvm_read_ptx_sreg_nctaid_w, "i", "nc")
-
-TARGET_BUILTIN(__nvvm_read_ptx_sreg_clusterid_x, "i", "nc", AND(SM_90, PTX78))
-TARGET_BUILTIN(__nvvm_read_ptx_sreg_clusterid_y, "i", "nc", AND(SM_90, PTX78))
-TARGET_BUILTIN(__nvvm_read_ptx_sreg_clusterid_z, "i", "nc", AND(SM_90, PTX78))
-TARGET_BUILTIN(__nvvm_read_ptx_sreg_clusterid_w, "i", "nc", AND(SM_90, PTX78))
-
-TARGET_BUILTIN(__nvvm_read_ptx_sreg_nclusterid_x, "i", "nc", AND(SM_90, PTX78))
-TARGET_BUILTIN(__nvvm_read_ptx_sreg_nclusterid_y, "i", "nc", AND(SM_90, PTX78))
-TARGET_BUILTIN(__nvvm_read_ptx_sreg_nclusterid_z, "i", "nc", AND(SM_90, PTX78))
-TARGET_BUILTIN(__nvvm_read_ptx_sreg_nclusterid_w, "i", "nc", AND(SM_90, PTX78))
-
-TARGET_BUILTIN(__nvvm_read_ptx_sreg_cluster_ctaid_x, "i", "nc", AND(SM_90, PTX78))
-TARGET_BUILTIN(__nvvm_read_ptx_sreg_cluster_ctaid_y, "i", "nc", AND(SM_90, PTX78))
-TARGET_BUILTIN(__nvvm_read_ptx_sreg_cluster_ctaid_z, "i", "nc", AND(SM_90, PTX78))
-TARGET_BUILTIN(__nvvm_read_ptx_sreg_cluster_ctaid_w, "i", "nc", AND(SM_90, PTX78))
-
-TARGET_BUILTIN(__nvvm_read_ptx_sreg_cluster_nctaid_x, "i", "nc", AND(SM_90, PTX78))
-TARGET_BUILTIN(__nvvm_read_ptx_sreg_cluster_nctaid_y, "i", "nc", AND(SM_90, PTX78))
-TARGET_BUILTIN(__nvvm_read_ptx_sreg_cluster_nctaid_z, "i", "nc", AND(SM_90, PTX78))
-TARGET_BUILTIN(__nvvm_read_ptx_sreg_cluster_nctaid_w, "i", "nc", AND(SM_90, PTX78))
-
-TARGET_BUILTIN(__nvvm_read_ptx_sreg_cluster_ctarank, "i", "nc", AND(SM_90, PTX78))
-TARGET_BUILTIN(__nvvm_read_ptx_sreg_cluster_nctarank, "i", "nc", AND(SM_90, PTX78))
-
-TARGET_BUILTIN(__nvvm_is_explicit_cluster, "b", "nc", AND(SM_90, PTX78))
-
-BUILTIN(__nvvm_read_ptx_sreg_laneid, "i", "nc")
-BUILTIN(__nvvm_read_ptx_sreg_warpid, "i", "nc")
-BUILTIN(__nvvm_read_ptx_sreg_nwarpid, "i", "nc")
-BUILTIN(__nvvm_read_ptx_sreg_warpsize, "i", "nc")
-
-BUILTIN(__nvvm_read_ptx_sreg_smid, "i", "nc")
-BUILTIN(__nvvm_read_ptx_sreg_nsmid, "i", "nc")
-BUILTIN(__nvvm_read_ptx_sreg_gridid, "i", "nc")
-
-BUILTIN(__nvvm_read_ptx_sreg_lanemask_eq, "i", "nc")
-BUILTIN(__nvvm_read_ptx_sreg_lanemask_le, "i", "nc")
-BUILTIN(__nvvm_read_ptx_sreg_lanemask_lt, "i", "nc")
-BUILTIN(__nvvm_read_ptx_sreg_lanemask_ge, "i", "nc")
-BUILTIN(__nvvm_read_ptx_sreg_lanemask_gt, "i", "nc")
-
-BUILTIN(__nvvm_read_ptx_sreg_clock, "i", "n")
-BUILTIN(__nvvm_read_ptx_sreg_clock64, "LLi", "n")
-BUILTIN(__nvvm_read_ptx_sreg_globaltimer, "LLi", "n")
-
-BUILTIN(__nvvm_read_ptx_sreg_pm0, "i", "n")
-BUILTIN(__nvvm_read_ptx_sreg_pm1, "i", "n")
-BUILTIN(__nvvm_read_ptx_sreg_pm2, "i", "n")
-BUILTIN(__nvvm_read_ptx_sreg_pm3, "i", "n")
-
-// MISC
-
-BUILTIN(__nvvm_prmt, "UiUiUiUi", "")
-BUILTIN(__nvvm_exit, "v", "r")
-BUILTIN(__nvvm_reflect, "UicC*", "r")
-TARGET_BUILTIN(__nvvm_nanosleep, "vUi", "n", AND(SM_70, PTX63))
-
-// Min Max
-
-TARGET_BUILTIN(__nvvm_fmin_f16, "hhh", "", AND(SM_80, PTX70))
-TARGET_BUILTIN(__nvvm_fmin_ftz_f16, "hhh", "", AND(SM_80, PTX70))
-TARGET_BUILTIN(__nvvm_fmin_nan_f16, "hhh", "", AND(SM_80, PTX70))
-TARGET_BUILTIN(__nvvm_fmin_ftz_nan_f16, "hhh", "", AND(SM_80, PTX70))
-TARGET_BUILTIN(__nvvm_fmin_xorsign_abs_f16, "hhh", "", AND(SM_86, PTX72))
-TARGET_BUILTIN(__nvvm_fmin_ftz_xorsign_abs_f16, "hhh", "", AND(SM_86, PTX72))
-TARGET_BUILTIN(__nvvm_fmin_nan_xorsign_abs_f16, "hhh", "", AND(SM_86, PTX72))
-TARGET_BUILTIN(__nvvm_fmin_ftz_nan_xorsign_abs_f16, "hhh", "",
-               AND(SM_86, PTX72))
-TARGET_BUILTIN(__nvvm_fmin_f16x2, "V2hV2hV2h", "", AND(SM_80, PTX70))
-TARGET_BUILTIN(__nvvm_fmin_ftz_f16x2, "V2hV2hV2h", "", AND(SM_80, PTX70))
-TARGET_BUILTIN(__nvvm_fmin_nan_f16x2, "V2hV2hV2h", "", AND(SM_80, PTX70))
-TARGET_BUILTIN(__nvvm_fmin_ftz_nan_f16x2, "V2hV2hV2h", "", AND(SM_80, PTX70))
-TARGET_BUILTIN(__nvvm_fmin_xorsign_abs_f16x2, "V2hV2hV2h", "",
-               AND(SM_86, PTX72))
-TARGET_BUILTIN(__nvvm_fmin_ftz_xorsign_abs_f16x2, "V2hV2hV2h", "",
-               AND(SM_86, PTX72))
-TARGET_BUILTIN(__nvvm_fmin_nan_xorsign_abs_f16x2, "V2hV2hV2h", "",
-               AND(SM_86, PTX72))
-TARGET_BUILTIN(__nvvm_fmin_ftz_nan_xorsign_abs_f16x2, "V2hV2hV2h", "",
-               AND(SM_86, PTX72))
-TARGET_BUILTIN(__nvvm_fmin_bf16, "yyy", "", AND(SM_80, PTX70))
-TARGET_BUILTIN(__nvvm_fmin_ftz_bf16, "yyy", "", AND(SM_80, PTX70))
-TARGET_BUILTIN(__nvvm_fmin_nan_bf16, "yyy", "", AND(SM_80, PTX70))
-TARGET_BUILTIN(__nvvm_fmin_ftz_nan_bf16, "yyy", "", AND(SM_80, PTX70))
-TARGET_BUILTIN(__nvvm_fmin_xorsign_abs_bf16, "yyy", "", AND(SM_86, PTX72))
-TARGET_BUILTIN(__nvvm_fmin_nan_xorsign_abs_bf16, "yyy", "",
-               AND(SM_86, PTX72))
-TARGET_BUILTIN(__nvvm_fmin_bf16x2, "V2yV2yV2y", "", AND(SM_80, PTX70))
-TARGET_BUILTIN(__nvvm_fmin_ftz_bf16x2, "V2yV2yV2y", "", AND(SM_80, PTX70))
-TARGET_BUILTIN(__nvvm_fmin_nan_bf16x2, "V2yV2yV2y", "", AND(SM_80, PTX70))
-TARGET_BUILTIN(__nvvm_fmin_ftz_nan_bf16x2, "V2yV2yV2y", "", AND(SM_80, PTX70))
-TARGET_BUILTIN(__nvvm_fmin_xorsign_abs_bf16x2, "V2yV2yV2y", "",
-               AND(SM_86, PTX72))
-TARGET_BUILTIN(__nvvm_fmin_nan_xorsign_abs_bf16x2, "V2yV2yV2y", "",
-               AND(SM_86, PTX72))
-BUILTIN(__nvvm_fmin_f, "fff", "")
-BUILTIN(__nvvm_fmin_ftz_f, "fff", "")
-TARGET_BUILTIN(__nvvm_fmin_nan_f, "fff", "", AND(SM_80, PTX70))
-TARGET_BUILTIN(__nvvm_fmin_ftz_nan_f, "fff", "", AND(SM_80, PTX70))
-TARGET_BUILTIN(__nvvm_fmin_xorsign_abs_f, "fff", "", AND(SM_86, PTX72))
-TARGET_BUILTIN(__nvvm_fmin_ftz_xorsign_abs_f, "fff", "", AND(SM_86, PTX72))
-TARGET_BUILTIN(__nvvm_fmin_nan_xorsign_abs_f, "fff", "", AND(SM_86, PTX72))
-TARGET_BUILTIN(__nvvm_fmin_ftz_nan_xorsign_abs_f, "fff", "", AND(SM_86, PTX72))
-BUILTIN(__nvvm_fmin_d, "ddd", "")
-
-TARGET_BUILTIN(__nvvm_fmax_f16, "hhh", "", AND(SM_80, PTX70))
-TARGET_BUILTIN(__nvvm_fmax_ftz_f16, "hhh", "", AND(SM_80, PTX70))
-TARGET_BUILTIN(__nvvm_fmax_nan_f16, "hhh", "", AND(SM_80, PTX70))
-TARGET_BUILTIN(__nvvm_fmax_ftz_nan_f16, "hhh", "", AND(SM_80, PTX70))
-TARGET_BUILTIN(__nvvm_fmax_xorsign_abs_f16, "hhh", "", AND(SM_86, PTX72))
-TARGET_BUILTIN(__nvvm_fmax_ftz_xorsign_abs_f16, "hhh", "", AND(SM_86, PTX72))
-TARGET_BUILTIN(__nvvm_fmax_nan_xorsign_abs_f16, "hhh", "", AND(SM_86, PTX72))
-TARGET_BUILTIN(__nvvm_fmax_ftz_nan_xorsign_abs_f16, "hhh", "",
-               AND(SM_86, PTX72))
-TARGET_BUILTIN(__nvvm_fmax_f16x2, "V2hV2hV2h", "", AND(SM_80, PTX70))
-TARGET_BUILTIN(__nvvm_fmax_ftz_f16x2, "V2hV2hV2h", "", AND(SM_80, PTX70))
-TARGET_BUILTIN(__nvvm_fmax_nan_f16x2, "V2hV2hV2h", "", AND(SM_80, PTX70))
-TARGET_BUILTIN(__nvvm_fmax_ftz_nan_f16x2, "V2hV2hV2h", "", AND(SM_80, PTX70))
-TARGET_BUILTIN(__nvvm_fmax_xorsign_abs_f16x2, "V2hV2hV2h", "",
-               AND(SM_86, PTX72))
-TARGET_BUILTIN(__nvvm_fmax_ftz_xorsign_abs_f16x2, "V2hV2hV2h", "",
-               AND(SM_86, PTX72))
-TARGET_BUILTIN(__nvvm_fmax_nan_xorsign_abs_f16x2, "V2hV2hV2h", "",
-               AND(SM_86, PTX72))
-TARGET_BUILTIN(__nvvm_fmax_ftz_nan_xorsign_abs_f16x2, "V2hV2hV2h", "",
-               AND(SM_86, PTX72))
-TARGET_BUILTIN(__nvvm_fmax_bf16, "yyy", "", AND(SM_80, PTX70))
-TARGET_BUILTIN(__nvvm_fmax_ftz_bf16, "yyy", "", AND(SM_80, PTX70))
-TARGET_BUILTIN(__nvvm_fmax_nan_bf16, "yyy", "", AND(SM_80, PTX70))
-TARGET_BUILTIN(__nvvm_fmax_ftz_nan_bf16, "yyy", "", AND(SM_80, PTX70))
-TARGET_BUILTIN(__nvvm_fmax_xorsign_abs_bf16, "yyy", "", AND(SM_86, PTX72))
-TARGET_BUILTIN(__nvvm_fmax_nan_xorsign_abs_bf16, "yyy", "",
-               AND(SM_86, PTX72))
-TARGET_BUILTIN(__nvvm_fmax_bf16x2, "V2yV2yV2y", "", AND(SM_80, PTX70))
-TARGET_BUILTIN(__nvvm_fmax_ftz_bf16x2, "V2yV2yV2y", "", AND(SM_80, PTX70))
-TARGET_BUILTIN(__nvvm_fmax_nan_bf16x2, "V2yV2yV2y", "", AND(SM_80, PTX70))
-TARGET_BUILTIN(__nvvm_fmax_ftz_nan_bf16x2, "V2yV2yV2y", "", AND(SM_80, PTX70))
-TARGET_BUILTIN(__nvvm_fmax_xorsign_abs_bf16x2, "V2yV2yV2y", "",
-               AND(SM_86, PTX72))
-TARGET_BUILTIN(__nvvm_fmax_nan_xorsign_abs_bf16x2, "V2yV2yV2y", "",
-               AND(SM_86, PTX72))
-BUILTIN(__nvvm_fmax_f, "fff", "")
-BUILTIN(__nvvm_fmax_ftz_f, "fff", "")
-TARGET_BUILTIN(__nvvm_fmax_nan_f, "fff", "", AND(SM_80, PTX70))
-TARGET_BUILTIN(__nvvm_fmax_ftz_nan_f, "fff", "", AND(SM_80, PTX70))
-TARGET_BUILTIN(__nvvm_fmax_xorsign_abs_f, "fff", "", AND(SM_86, PTX72))
-TARGET_BUILTIN(__nvvm_fmax_ftz_xorsign_abs_f, "fff", "", AND(SM_86, PTX72))
-TARGET_BUILTIN(__nvvm_fmax_nan_xorsign_abs_f, "fff", "", AND(SM_86, PTX72))
-TARGET_BUILTIN(__nvvm_fmax_ftz_nan_xorsign_abs_f, "fff", "", AND(SM_86, PTX72))
-BUILTIN(__nvvm_fmax_d, "ddd", "")
-
-// Multiplication
-
-BUILTIN(__nvvm_mulhi_i, "iii", "")
-BUILTIN(__nvvm_mulhi_ui, "UiUiUi", "")
-BUILTIN(__nvvm_mulhi_ll, "LLiLLiLLi", "")
-BUILTIN(__nvvm_mulhi_ull, "ULLiULLiULLi", "")
-
-BUILTIN(__nvvm_mul_rn_ftz_f,  "fff", "")
-BUILTIN(__nvvm_mul_rn_f,  "fff", "")
-BUILTIN(__nvvm_mul_rz_ftz_f,  "fff", "")
-BUILTIN(__nvvm_mul_rz_f,  "fff", "")
-BUILTIN(__nvvm_mul_rm_ftz_f,  "fff", "")
-BUILTIN(__nvvm_mul_rm_f,  "fff", "")
-BUILTIN(__nvvm_mul_rp_ftz_f,  "fff", "")
-BUILTIN(__nvvm_mul_rp_f,  "fff", "")
-
-BUILTIN(__nvvm_mul_rn_d,  "ddd", "")
-BUILTIN(__nvvm_mul_rz_d,  "ddd", "")
-BUILTIN(__nvvm_mul_rm_d,  "ddd", "")
-BUILTIN(__nvvm_mul_rp_d,  "ddd", "")
-
-BUILTIN(__nvvm_mul24_i,  "iii", "")
-BUILTIN(__nvvm_mul24_ui,  "UiUiUi", "")
-
-// Div
-
-BUILTIN(__nvvm_div_approx_ftz_f,  "fff", "")
-BUILTIN(__nvvm_div_approx_f,  "fff", "")
-
-BUILTIN(__nvvm_div_rn_ftz_f,  "fff", "")
-BUILTIN(__nvvm_div_rn_f,  "fff", "")
-BUILTIN(__nvvm_div_rz_ftz_f,  "fff", "")
-BUILTIN(__nvvm_div_rz_f,  "fff", "")
-BUILTIN(__nvvm_div_rm_ftz_f,  "fff", "")
-BUILTIN(__nvvm_div_rm_f,  "fff", "")
-BUILTIN(__nvvm_div_rp_ftz_f,  "fff", "")
-BUILTIN(__nvvm_div_rp_f,  "fff", "")
-
-BUILTIN(__nvvm_div_rn_d,  "ddd", "")
-BUILTIN(__nvvm_div_rz_d,  "ddd", "")
-BUILTIN(__nvvm_div_rm_d,  "ddd", "")
-BUILTIN(__nvvm_div_rp_d,  "ddd", "")
-
-// Sad
-
-BUILTIN(__nvvm_sad_i, "iiii", "")
-BUILTIN(__nvvm_sad_ui, "UiUiUiUi", "")
-
-// Floor, Ceil
-
-BUILTIN(__nvvm_floor_ftz_f, "ff", "")
-BUILTIN(__nvvm_floor_f, "ff", "")
-BUILTIN(__nvvm_floor_d, "dd", "")
-
-BUILTIN(__nvvm_ceil_ftz_f, "ff", "")
-BUILTIN(__nvvm_ceil_f, "ff", "")
-BUILTIN(__nvvm_ceil_d, "dd", "")
-
-// Abs
-
-BUILTIN(__nvvm_fabs_ftz_f, "ff", "")
-BUILTIN(__nvvm_fabs_f, "ff", "")
-BUILTIN(__nvvm_fabs_d, "dd", "")
-
-// Round
-
-BUILTIN(__nvvm_round_ftz_f, "ff", "")
-BUILTIN(__nvvm_round_f, "ff", "")
-BUILTIN(__nvvm_round_d, "dd", "")
-
-// Trunc
-
-BUILTIN(__nvvm_trunc_ftz_f, "ff", "")
-BUILTIN(__nvvm_trunc_f, "ff", "")
-BUILTIN(__nvvm_trunc_d, "dd", "")
-
-// Saturate
-
-BUILTIN(__nvvm_saturate_ftz_f, "ff", "")
-BUILTIN(__nvvm_saturate_f, "ff", "")
-BUILTIN(__nvvm_saturate_d, "dd", "")
-
-// Exp2, Log2
-
-BUILTIN(__nvvm_ex2_approx_ftz_f, "ff", "")
-BUILTIN(__nvvm_ex2_approx_f, "ff", "")
-BUILTIN(__nvvm_ex2_approx_d, "dd", "")
-TARGET_BUILTIN(__nvvm_ex2_approx_f16, "hh", "", AND(SM_75, PTX70))
-TARGET_BUILTIN(__nvvm_ex2_approx_f16x2, "V2hV2h", "", AND(SM_75, PTX70))
-
-BUILTIN(__nvvm_lg2_approx_ftz_f, "ff", "")
-BUILTIN(__nvvm_lg2_approx_f, "ff", "")
-BUILTIN(__nvvm_lg2_approx_d, "dd", "")
-
-// Sin, Cos
-
-BUILTIN(__nvvm_sin_approx_ftz_f, "ff", "")
-BUILTIN(__nvvm_sin_approx_f, "ff", "")
-
-BUILTIN(__nvvm_cos_approx_ftz_f, "ff", "")
-BUILTIN(__nvvm_cos_approx_f, "ff", "")
-
-// Fma
-
-TARGET_BUILTIN(__nvvm_fma_rn_f16, "hhhh", "", AND(SM_53, PTX42))
-TARGET_BUILTIN(__nvvm_fma_rn_ftz_f16, "hhhh", "", AND(SM_53, PTX42))
-TARGET_BUILTIN(__nvvm_fma_rn_sat_f16, "hhhh", "", AND(SM_53, PTX42))
-TARGET_BUILTIN(__nvvm_fma_rn_ftz_sat_f16, "hhhh", "", AND(SM_53, PTX42))
-TARGET_BUILTIN(__nvvm_fma_rn_relu_f16, "hhhh", "", AND(SM_80, PTX70))
-TARGET_BUILTIN(__nvvm_fma_rn_ftz_relu_f16, "hhhh", "", AND(SM_80, PTX70))
-TARGET_BUILTIN(__nvvm_fma_rn_f16x2, "V2hV2hV2hV2h", "", AND(SM_53, PTX42))
-TARGET_BUILTIN(__nvvm_fma_rn_ftz_f16x2, "V2hV2hV2hV2h", "", AND(SM_53, PTX42))
-TARGET_BUILTIN(__nvvm_fma_rn_sat_f16x2, "V2hV2hV2hV2h", "", AND(SM_53, PTX42))
-TARGET_BUILTIN(__nvvm_fma_rn_ftz_sat_f16x2, "V2hV2hV2hV2h", "", AND(SM_53, PTX42))
-TARGET_BUILTIN(__nvvm_fma_rn_relu_f16x2, "V2hV2hV2hV2h", "", AND(SM_80, PTX70))
-TARGET_BUILTIN(__nvvm_fma_rn_ftz_relu_f16x2, "V2hV2hV2hV2h", "", AND(SM_80, PTX70))
-TARGET_BUILTIN(__nvvm_fma_rn_bf16, "yyyy", "", AND(SM_80, PTX70))
-TARGET_BUILTIN(__nvvm_fma_rn_relu_bf16, "yyyy", "", AND(SM_80, PTX70))
-TARGET_BUILTIN(__nvvm_fma_rn_bf16x2, "V2yV2yV2yV2y", "", AND(SM_80, PTX70))
-TARGET_BUILTIN(__nvvm_fma_rn_relu_bf16x2, "V2yV2yV2yV2y", "", AND(SM_80, PTX70))
-BUILTIN(__nvvm_fma_rn_ftz_f, "ffff", "")
-BUILTIN(__nvvm_fma_rn_f, "ffff", "")
-BUILTIN(__nvvm_fma_rz_ftz_f, "ffff", "")
-BUILTIN(__nvvm_fma_rz_f, "ffff", "")
-BUILTIN(__nvvm_fma_rm_ftz_f, "ffff", "")
-BUILTIN(__nvvm_fma_rm_f, "ffff", "")
-BUILTIN(__nvvm_fma_rp_ftz_f, "ffff", "")
-BUILTIN(__nvvm_fma_rp_f, "ffff", "")
-BUILTIN(__nvvm_fma_rn_d, "dddd", "")
-BUILTIN(__nvvm_fma_rz_d, "dddd", "")
-BUILTIN(__nvvm_fma_rm_d, "dddd", "")
-BUILTIN(__nvvm_fma_rp_d, "dddd", "")
-
-// Rcp
-
-BUILTIN(__nvvm_rcp_rn_ftz_f, "ff", "")
-BUILTIN(__nvvm_rcp_rn_f, "ff", "")
-BUILTIN(__nvvm_rcp_rz_ftz_f, "ff", "")
-BUILTIN(__nvvm_rcp_rz_f, "ff", "")
-BUILTIN(__nvvm_rcp_rm_ftz_f, "ff", "")
-BUILTIN(__nvvm_rcp_rm_f, "ff", "")
-BUILTIN(__nvvm_rcp_rp_ftz_f, "ff", "")
-BUILTIN(__nvvm_rcp_rp_f, "ff", "")
-
-BUILTIN(__nvvm_rcp_rn_d, "dd", "")
-BUILTIN(__nvvm_rcp_rz_d, "dd", "")
-BUILTIN(__nvvm_rcp_rm_d, "dd", "")
-BUILTIN(__nvvm_rcp_rp_d, "dd", "")
-
-BUILTIN(__nvvm_rcp_approx_ftz_f, "ff", "")
-BUILTIN(__nvvm_rcp_approx_ftz_d, "dd", "")
-
-// Sqrt
-
-BUILTIN(__nvvm_sqrt_rn_ftz_f, "ff", "")
-BUILTIN(__nvvm_sqrt_rn_f, "ff", "")
-BUILTIN(__nvvm_sqrt_rz_ftz_f, "ff", "")
-BUILTIN(__nvvm_sqrt_rz_f, "ff", "")
-BUILTIN(__nvvm_sqrt_rm_ftz_f, "ff", "")
-BUILTIN(__nvvm_sqrt_rm_f, "ff", "")
-BUILTIN(__nvvm_sqrt_rp_ftz_f, "ff", "")
-BUILTIN(__nvvm_sqrt_rp_f, "ff", "")
-BUILTIN(__nvvm_sqrt_approx_ftz_f, "ff", "")
-BUILTIN(__nvvm_sqrt_approx_f, "ff", "")
-
-BUILTIN(__nvvm_sqrt_rn_d, "dd", "")
-BUILTIN(__nvvm_sqrt_rz_d, "dd", "")
-BUILTIN(__nvvm_sqrt_rm_d, "dd", "")
-BUILTIN(__nvvm_sqrt_rp_d, "dd", "")
-
-// Rsqrt
-
-BUILTIN(__nvvm_rsqrt_approx_ftz_f, "ff", "")
-BUILTIN(__nvvm_rsqrt_approx_f, "ff", "")
-BUILTIN(__nvvm_rsqrt_approx_d, "dd", "")
-
-// Add
-
-BUILTIN(__nvvm_add_rn_ftz_f, "fff", "")
-BUILTIN(__nvvm_add_rn_f, "fff", "")
-BUILTIN(__nvvm_add_rz_ftz_f, "fff", "")
-BUILTIN(__nvvm_add_rz_f, "fff", "")
-BUILTIN(__nvvm_add_rm_ftz_f, "fff", "")
-BUILTIN(__nvvm_add_rm_f, "fff", "")
-BUILTIN(__nvvm_add_rp_ftz_f, "fff", "")
-BUILTIN(__nvvm_add_rp_f, "fff", "")
-
-BUILTIN(__nvvm_add_rn_d, "ddd", "")
-BUILTIN(__nvvm_add_rz_d, "ddd", "")
-BUILTIN(__nvvm_add_rm_d, "ddd", "")
-BUILTIN(__nvvm_add_rp_d, "ddd", "")
-
-// Convert
-
-BUILTIN(__nvvm_d2f_rn_ftz, "fd", "")
-BUILTIN(__nvvm_d2f_rn, "fd", "")
-BUILTIN(__nvvm_d2f_rz_ftz, "fd", "")
-BUILTIN(__nvvm_d2f_rz, "fd", "")
-BUILTIN(__nvvm_d2f_rm_ftz, "fd", "")
-BUILTIN(__nvvm_d2f_rm, "fd", "")
-BUILTIN(__nvvm_d2f_rp_ftz, "fd", "")
-BUILTIN(__nvvm_d2f_rp, "fd", "")
-
-BUILTIN(__nvvm_d2i_rn, "id", "")
-BUILTIN(__nvvm_d2i_rz, "id", "")
-BUILTIN(__nvvm_d2i_rm, "id", "")
-BUILTIN(__nvvm_d2i_rp, "id", "")
-
-BUILTIN(__nvvm_d2ui_rn, "Uid", "")
-BUILTIN(__nvvm_d2ui_rz, "Uid", "")
-BUILTIN(__nvvm_d2ui_rm, "Uid", "")
-BUILTIN(__nvvm_d2ui_rp, "Uid", "")
-
-BUILTIN(__nvvm_i2d_rn, "di", "")
-BUILTIN(__nvvm_i2d_rz, "di", "")
-BUILTIN(__nvvm_i2d_rm, "di", "")
-BUILTIN(__nvvm_i2d_rp, "di", "")
-
-BUILTIN(__nvvm_ui2d_rn, "dUi", "")
-BUILTIN(__nvvm_ui2d_rz, "dUi", "")
-BUILTIN(__nvvm_ui2d_rm, "dUi", "")
-BUILTIN(__nvvm_ui2d_rp, "dUi", "")
-
-BUILTIN(__nvvm_f2i_rn_ftz, "if", "")
-BUILTIN(__nvvm_f2i_rn, "if", "")
-BUILTIN(__nvvm_f2i_rz_ftz, "if", "")
-BUILTIN(__nvvm_f2i_rz, "if", "")
-BUILTIN(__nvvm_f2i_rm_ftz, "if", "")
-BUILTIN(__nvvm_f2i_rm, "if", "")
-BUILTIN(__nvvm_f2i_rp_ftz, "if", "")
-BUILTIN(__nvvm_f2i_rp, "if", "")
-
-BUILTIN(__nvvm_f2ui_rn_ftz, "Uif", "")
-BUILTIN(__nvvm_f2ui_rn, "Uif", "")
-BUILTIN(__nvvm_f2ui_rz_ftz, "Uif", "")
-BUILTIN(__nvvm_f2ui_rz, "Uif", "")
-BUILTIN(__nvvm_f2ui_rm_ftz, "Uif", "")
-BUILTIN(__nvvm_f2ui_rm, "Uif", "")
-BUILTIN(__nvvm_f2ui_rp_ftz, "Uif", "")
-BUILTIN(__nvvm_f2ui_rp, "Uif", "")
-
-BUILTIN(__nvvm_i2f_rn, "fi", "")
-BUILTIN(__nvvm_i2f_rz, "fi", "")
-BUILTIN(__nvvm_i2f_rm, "fi", "")
-BUILTIN(__nvvm_i2f_rp, "fi", "")
-
-BUILTIN(__nvvm_ui2f_rn, "fUi", "")
-BUILTIN(__nvvm_ui2f_rz, "fUi", "")
-BUILTIN(__nvvm_ui2f_rm, "fUi", "")
-BUILTIN(__nvvm_ui2f_rp, "fUi", "")
-
-BUILTIN(__nvvm_lohi_i2d, "dii", "")
-
-BUILTIN(__nvvm_d2i_lo, "id", "")
-BUILTIN(__nvvm_d2i_hi, "id", "")
-
-BUILTIN(__nvvm_f2ll_rn_ftz, "LLif", "")
-BUILTIN(__nvvm_f2ll_rn, "LLif", "")
-BUILTIN(__nvvm_f2ll_rz_ftz, "LLif", "")
-BUILTIN(__nvvm_f2ll_rz, "LLif", "")
-BUILTIN(__nvvm_f2ll_rm_ftz, "LLif", "")
-BUILTIN(__nvvm_f2ll_rm, "LLif", "")
-BUILTIN(__nvvm_f2ll_rp_ftz, "LLif", "")
-BUILTIN(__nvvm_f2ll_rp, "LLif", "")
-
-BUILTIN(__nvvm_f2ull_rn_ftz, "ULLif", "")
-BUILTIN(__nvvm_f2ull_rn, "ULLif", "")
-BUILTIN(__nvvm_f2ull_rz_ftz, "ULLif", "")
-BUILTIN(__nvvm_f2ull_rz, "ULLif", "")
-BUILTIN(__nvvm_f2ull_rm_ftz, "ULLif", "")
-BUILTIN(__nvvm_f2ull_rm, "ULLif", "")
-BUILTIN(__nvvm_f2ull_rp_ftz, "ULLif", "")
-BUILTIN(__nvvm_f2ull_rp, "ULLif", "")
-
-BUILTIN(__nvvm_d2ll_rn, "LLid", "")
-BUILTIN(__nvvm_d2ll_rz, "LLid", "")
-BUILTIN(__nvvm_d2ll_rm, "LLid", "")
-BUILTIN(__nvvm_d2ll_rp, "LLid", "")
-
-BUILTIN(__nvvm_d2ull_rn, "ULLid", "")
-BUILTIN(__nvvm_d2ull_rz, "ULLid", "")
-BUILTIN(__nvvm_d2ull_rm, "ULLid", "")
-BUILTIN(__nvvm_d2ull_rp, "ULLid", "")
-
-BUILTIN(__nvvm_ll2f_rn, "fLLi", "")
-BUILTIN(__nvvm_ll2f_rz, "fLLi", "")
-BUILTIN(__nvvm_ll2f_rm, "fLLi", "")
-BUILTIN(__nvvm_ll2f_rp, "fLLi", "")
-
-BUILTIN(__nvvm_ull2f_rn, "fULLi", "")
-BUILTIN(__nvvm_ull2f_rz, "fULLi", "")
-BUILTIN(__nvvm_ull2f_rm, "fULLi", "")
-BUILTIN(__nvvm_ull2f_rp, "fULLi", "")
-
-BUILTIN(__nvvm_ll2d_rn, "dLLi", "")
-BUILTIN(__nvvm_ll2d_rz, "dLLi", "")
-BUILTIN(__nvvm_ll2d_rm, "dLLi", "")
-BUILTIN(__nvvm_ll2d_rp, "dLLi", "")
-
-BUILTIN(__nvvm_ull2d_rn, "dULLi", "")
-BUILTIN(__nvvm_ull2d_rz, "dULLi", "")
-BUILTIN(__nvvm_ull2d_rm, "dULLi", "")
-BUILTIN(__nvvm_ull2d_rp, "dULLi", "")
-
-BUILTIN(__nvvm_f2h_rn_ftz, "Usf", "")
-BUILTIN(__nvvm_f2h_rn, "Usf", "")
-
-TARGET_BUILTIN(__nvvm_ff2bf16x2_rn, "V2yff", "", AND(SM_80,PTX70))
-TARGET_BUILTIN(__nvvm_ff2bf16x2_rn_relu, "V2yff", "", AND(SM_80,PTX70))
-TARGET_BUILTIN(__nvvm_ff2bf16x2_rz, "V2yff", "", AND(SM_80,PTX70))
-TARGET_BUILTIN(__nvvm_ff2bf16x2_rz_relu, "V2yff", "", AND(SM_80,PTX70))
-
-TARGET_BUILTIN(__nvvm_ff2f16x2_rn, "V2hff", "", AND(SM_80,PTX70))
-TARGET_BUILTIN(__nvvm_ff2f16x2_rn_relu, "V2hff", "", AND(SM_80,PTX70))
-TARGET_BUILTIN(__nvvm_ff2f16x2_rz, "V2hff", "", AND(SM_80,PTX70))
-TARGET_BUILTIN(__nvvm_ff2f16x2_rz_relu, "V2hff", "", AND(SM_80,PTX70))
-
-TARGET_BUILTIN(__nvvm_f2bf16_rn, "yf", "", AND(SM_80,PTX70))
-TARGET_BUILTIN(__nvvm_f2bf16_rn_relu, "yf", "", AND(SM_80,PTX70))
-TARGET_BUILTIN(__nvvm_f2bf16_rz, "yf", "", AND(SM_80,PTX70))
-TARGET_BUILTIN(__nvvm_f2bf16_rz_relu, "yf", "", AND(SM_80,PTX70))
-
-TARGET_BUILTIN(__nvvm_f2tf32_rna, "ZUif", "", AND(SM_80,PTX70))
-
-TARGET_BUILTIN(__nvvm_ff_to_e4m3x2_rn, "sff", "", AND(SM_89,PTX81))
-TARGET_BUILTIN(__nvvm_ff_to_e4m3x2_rn_relu, "sff", "", AND(SM_89,PTX81))
-TARGET_BUILTIN(__nvvm_ff_to_e5m2x2_rn, "sff", "", AND(SM_89,PTX81))
-TARGET_BUILTIN(__nvvm_ff_to_e5m2x2_rn_relu, "sff", "", AND(SM_89,PTX81))
-
-TARGET_BUILTIN(__nvvm_f16x2_to_e4m3x2_rn, "sV2h", "", AND(SM_89,PTX81))
-TARGET_BUILTIN(__nvvm_f16x2_to_e4m3x2_rn_relu, "sV2h", "", AND(SM_89,PTX81))
-TARGET_BUILTIN(__nvvm_f16x2_to_e5m2x2_rn, "sV2h", "", AND(SM_89,PTX81))
-TARGET_BUILTIN(__nvvm_f16x2_to_e5m2x2_rn_relu, "sV2h", "", AND(SM_89,PTX81))
-
-TARGET_BUILTIN(__nvvm_e4m3x2_to_f16x2_rn, "V2hs", "", AND(SM_89,PTX81))
-TARGET_BUILTIN(__nvvm_e4m3x2_to_f16x2_rn_relu, "V2hs", "", AND(SM_89,PTX81))
-TARGET_BUILTIN(__nvvm_e5m2x2_to_f16x2_rn, "V2hs", "", AND(SM_89,PTX81))
-TARGET_BUILTIN(__nvvm_e5m2x2_to_f16x2_rn_relu, "V2hs", "", AND(SM_89,PTX81))
-
-// FNS
-TARGET_BUILTIN(__nvvm_fns, "UiUiUii", "n", PTX60)
-
-// Sync
-
-BUILTIN(__syncthreads, "v", "")
-BUILTIN(__nvvm_bar0_popc, "ii", "")
-BUILTIN(__nvvm_bar0_and, "ii", "")
-BUILTIN(__nvvm_bar0_or, "ii", "")
-BUILTIN(__nvvm_bar_sync, "vi", "n")
-TARGET_BUILTIN(__nvvm_bar_warp_sync, "vUi", "n", PTX60)
-TARGET_BUILTIN(__nvvm_barrier_sync, "vUi", "n", PTX60)
-TARGET_BUILTIN(__nvvm_barrier_sync_cnt, "vUiUi", "n", PTX60)
-
-TARGET_BUILTIN(__nvvm_barrier_cluster_arrive, "v", "n", AND(SM_90,PTX78))
-TARGET_BUILTIN(__nvvm_barrier_cluster_arrive_relaxed, "v", "n", AND(SM_90,PTX80))
-TARGET_BUILTIN(__nvvm_barrier_cluster_wait, "v", "n", AND(SM_90,PTX78))
-TARGET_BUILTIN(__nvvm_fence_sc_cluster, "v", "n", AND(SM_90,PTX78))
-
-// Shuffle
-
-BUILTIN(__nvvm_shfl_down_i32, "iiii", "")
-BUILTIN(__nvvm_shfl_down_f32, "ffii", "")
-BUILTIN(__nvvm_shfl_up_i32, "iiii", "")
-BUILTIN(__nvvm_shfl_up_f32, "ffii", "")
-BUILTIN(__nvvm_shfl_bfly_i32, "iiii", "")
-BUILTIN(__nvvm_shfl_bfly_f32, "ffii", "")
-BUILTIN(__nvvm_shfl_idx_i32, "iiii", "")
-BUILTIN(__nvvm_shfl_idx_f32, "ffii", "")
-
-TARGET_BUILTIN(__nvvm_shfl_sync_down_i32, "iUiiii", "", PTX60)
-TARGET_BUILTIN(__nvvm_shfl_sync_down_f32, "fUifii", "", PTX60)
-TARGET_BUILTIN(__nvvm_shfl_sync_up_i32, "iUiiii", "", PTX60)
-TARGET_BUILTIN(__nvvm_shfl_sync_up_f32, "fUifii", "", PTX60)
-TARGET_BUILTIN(__nvvm_shfl_sync_bfly_i32, "iUiiii", "", PTX60)
-TARGET_BUILTIN(__nvvm_shfl_sync_bfly_f32, "fUifii", "", PTX60)
-TARGET_BUILTIN(__nvvm_shfl_sync_idx_i32, "iUiiii", "", PTX60)
-TARGET_BUILTIN(__nvvm_shfl_sync_idx_f32, "fUifii", "", PTX60)
-
-// Vote
-BUILTIN(__nvvm_vote_all, "bb", "")
-BUILTIN(__nvvm_vote_any, "bb", "")
-BUILTIN(__nvvm_vote_uni, "bb", "")
-BUILTIN(__nvvm_vote_ballot, "Uib", "")
-
-TARGET_BUILTIN(__nvvm_vote_all_sync, "bUib", "", PTX60)
-TARGET_BUILTIN(__nvvm_vote_any_sync, "bUib", "", PTX60)
-TARGET_BUILTIN(__nvvm_vote_uni_sync, "bUib", "", PTX60)
-TARGET_BUILTIN(__nvvm_vote_ballot_sync, "UiUib", "", PTX60)
-
-// Mask
-TARGET_BUILTIN(__nvvm_activemask, "Ui", "n", PTX62)
-
-// Match
-TARGET_BUILTIN(__nvvm_match_any_sync_i32, "UiUiUi", "", AND(SM_70,PTX60))
-TARGET_BUILTIN(__nvvm_match_any_sync_i64, "UiUiWi", "", AND(SM_70,PTX60))
-// These return a pair {value, predicate}, which requires custom lowering.
-TARGET_BUILTIN(__nvvm_match_all_sync_i32p, "UiUiUii*", "", AND(SM_70,PTX60))
-TARGET_BUILTIN(__nvvm_match_all_sync_i64p, "UiUiWii*", "", AND(SM_70,PTX60))
-
-// Redux
-TARGET_BUILTIN(__nvvm_redux_sync_add, "iii", "", AND(SM_80,PTX70))
-TARGET_BUILTIN(__nvvm_redux_sync_min, "iii", "", AND(SM_80,PTX70))
-TARGET_BUILTIN(__nvvm_redux_sync_max, "iii", "", AND(SM_80,PTX70))
-TARGET_BUILTIN(__nvvm_redux_sync_umin, "UiUii", "", AND(SM_80,PTX70))
-TARGET_BUILTIN(__nvvm_redux_sync_umax, "UiUii", "", AND(SM_80,PTX70))
-TARGET_BUILTIN(__nvvm_redux_sync_and, "iii", "", AND(SM_80,PTX70))
-TARGET_BUILTIN(__nvvm_redux_sync_xor, "iii", "", AND(SM_80,PTX70))
-TARGET_BUILTIN(__nvvm_redux_sync_or, "iii", "", AND(SM_80,PTX70))
-
-// Membar
-
-BUILTIN(__nvvm_membar_cta, "v", "")
-BUILTIN(__nvvm_membar_gl, "v", "")
-BUILTIN(__nvvm_membar_sys, "v", "")
-
-// mbarrier
-
-TARGET_BUILTIN(__nvvm_mbarrier_init, "vWi*i", "", AND(SM_80,PTX70))
-TARGET_BUILTIN(__nvvm_mbarrier_init_shared, "vWi*3i", "", AND(SM_80,PTX70))
-
-TARGET_BUILTIN(__nvvm_mbarrier_inval, "vWi*", "", AND(SM_80,PTX70))
-TARGET_BUILTIN(__nvvm_mbarrier_inval_shared, "vWi*3", "", AND(SM_80,PTX70))
-
-TARGET_BUILTIN(__nvvm_mbarrier_arrive, "WiWi*", "", AND(SM_80,PTX70))
-TARGET_BUILTIN(__nvvm_mbarrier_arrive_shared, "WiWi*3", "", AND(SM_80,PTX70))
-TARGET_BUILTIN(__nvvm_mbarrier_arrive_noComplete, "WiWi*i", "", AND(SM_80,PTX70))
-TARGET_BUILTIN(__nvvm_mbarrier_arrive_noComplete_shared, "WiWi*3i", "", AND(SM_80,PTX70))
-
-TARGET_BUILTIN(__nvvm_mbarrier_arrive_drop, "WiWi*", "", AND(SM_80,PTX70))
-TARGET_BUILTIN(__nvvm_mbarrier_arrive_drop_shared, "WiWi*3", "", AND(SM_80,PTX70))
-TARGET_BUILTIN(__nvvm_mbarrier_arrive_drop_noComplete, "WiWi*i", "", AND(SM_80,PTX70))
-TARGET_BUILTIN(__nvvm_mbarrier_arrive_drop_noComplete_shared, "WiWi*3i", "", AND(SM_80,PTX70))
-
-TARGET_BUILTIN(__nvvm_mbarrier_test_wait, "bWi*Wi", "", AND(SM_80,PTX70))
-TARGET_BUILTIN(__nvvm_mbarrier_test_wait_shared, "bWi*3Wi", "", AND(SM_80,PTX70))
-
-TARGET_BUILTIN(__nvvm_mbarrier_pending_count, "iWi", "", AND(SM_80,PTX70))
-
-// Memcpy, Memset
-
-BUILTIN(__nvvm_memcpy, "vUc*Uc*zi","")
-BUILTIN(__nvvm_memset, "vUc*Uczi","")
-
-// Image
-
-BUILTIN(__builtin_ptx_read_image2Dfi_, "V4fiiii", "")
-BUILTIN(__builtin_ptx_read_image2Dff_, "V4fiiff", "")
-BUILTIN(__builtin_ptx_read_image2Dii_, "V4iiiii", "")
-BUILTIN(__builtin_ptx_read_image2Dif_, "V4iiiff", "")
-
-BUILTIN(__builtin_ptx_read_image3Dfi_, "V4fiiiiii", "")
-BUILTIN(__builtin_ptx_read_image3Dff_, "V4fiiffff", "")
-BUILTIN(__builtin_ptx_read_image3Dii_, "V4iiiiiii", "")
-BUILTIN(__builtin_ptx_read_image3Dif_, "V4iiiffff", "")
-
-BUILTIN(__builtin_ptx_write_image2Df_, "viiiffff", "")
-BUILTIN(__builtin_ptx_write_image2Di_, "viiiiiii", "")
-BUILTIN(__builtin_ptx_write_image2Dui_, "viiiUiUiUiUi", "")
-BUILTIN(__builtin_ptx_get_image_depthi_, "ii", "")
-BUILTIN(__builtin_ptx_get_image_heighti_, "ii", "")
-BUILTIN(__builtin_ptx_get_image_widthi_, "ii", "")
-BUILTIN(__builtin_ptx_get_image_channel_data_typei_, "ii", "")
-BUILTIN(__builtin_ptx_get_image_channel_orderi_, "ii", "")
-
-// Atomic
-//
-// We need the atom intrinsics because
-// - they are used in converging analysis
-// - they are used in address space analysis and optimization
-// So it does not hurt to expose them as builtins.
-//
-BUILTIN(__nvvm_atom_add_gen_i, "iiD*i", "n")
-TARGET_BUILTIN(__nvvm_atom_cta_add_gen_i, "iiD*i", "n", SM_60)
-TARGET_BUILTIN(__nvvm_atom_sys_add_gen_i, "iiD*i", "n", SM_60)
-BUILTIN(__nvvm_atom_add_gen_l, "LiLiD*Li", "n")
-TARGET_BUILTIN(__nvvm_atom_cta_add_gen_l, "LiLiD*Li", "n", SM_60)
-TARGET_BUILTIN(__nvvm_atom_sys_add_gen_l, "LiLiD*Li", "n", SM_60)
-BUILTIN(__nvvm_atom_add_gen_ll, "LLiLLiD*LLi", "n")
-TARGET_BUILTIN(__nvvm_atom_cta_add_gen_ll, "LLiLLiD*LLi", "n", SM_60)
-TARGET_BUILTIN(__nvvm_atom_sys_add_gen_ll, "LLiLLiD*LLi", "n", SM_60)
-BUILTIN(__nvvm_atom_add_gen_f, "ffD*f", "n")
-TARGET_BUILTIN(__nvvm_atom_cta_add_gen_f, "ffD*f", "n", SM_60)
-TARGET_BUILTIN(__nvvm_atom_sys_add_gen_f, "ffD*f", "n", SM_60)
-TARGET_BUILTIN(__nvvm_atom_add_gen_d, "ddD*d", "n", SM_60)
-TARGET_BUILTIN(__nvvm_atom_cta_add_gen_d, "ddD*d", "n", SM_60)
-TARGET_BUILTIN(__nvvm_atom_sys_add_gen_d, "ddD*d", "n", SM_60)
-
-BUILTIN(__nvvm_atom_sub_gen_i, "iiD*i", "n")
-BUILTIN(__nvvm_atom_sub_gen_l, "LiLiD*Li", "n")
-BUILTIN(__nvvm_atom_sub_gen_ll, "LLiLLiD*LLi", "n")
-
-BUILTIN(__nvvm_atom_xchg_gen_i, "iiD*i", "n")
-TARGET_BUILTIN(__nvvm_atom_cta_xchg_gen_i, "iiD*i", "n", SM_60)
-TARGET_BUILTIN(__nvvm_atom_sys_xchg_gen_i, "iiD*i", "n", SM_60)
-BUILTIN(__nvvm_atom_xchg_gen_l, "LiLiD*Li", "n")
-TARGET_BUILTIN(__nvvm_atom_cta_xchg_gen_l, "LiLiD*Li", "n", SM_60)
-TARGET_BUILTIN(__nvvm_atom_sys_xchg_gen_l, "LiLiD*Li", "n", SM_60)
-BUILTIN(__nvvm_atom_xchg_gen_ll, "LLiLLiD*LLi", "n")
-TARGET_BUILTIN(__nvvm_atom_cta_xchg_gen_ll, "LLiLLiD*LLi", "n", SM_60)
-TARGET_BUILTIN(__nvvm_atom_sys_xchg_gen_ll, "LLiLLiD*LLi", "n", SM_60)
-
-BUILTIN(__nvvm_atom_max_gen_i, "iiD*i", "n")
-TARGET_BUILTIN(__nvvm_atom_cta_max_gen_i, "iiD*i", "n", SM_60)
-TARGET_BUILTIN(__nvvm_atom_sys_max_gen_i, "iiD*i", "n", SM_60)
-BUILTIN(__nvvm_atom_max_gen_ui, "UiUiD*Ui", "n")
-TARGET_BUILTIN(__nvvm_atom_cta_max_gen_ui, "UiUiD*Ui", "n", SM_60)
-TARGET_BUILTIN(__nvvm_atom_sys_max_gen_ui, "UiUiD*Ui", "n", SM_60)
-BUILTIN(__nvvm_atom_max_gen_l, "LiLiD*Li", "n")
-TARGET_BUILTIN(__nvvm_atom_cta_max_gen_l, "LiLiD*Li", "n", SM_60)
-TARGET_BUILTIN(__nvvm_atom_sys_max_gen_l, "LiLiD*Li", "n", SM_60)
-BUILTIN(__nvvm_atom_max_gen_ul, "ULiULiD*ULi", "n")
-TARGET_BUILTIN(__nvvm_atom_cta_max_gen_ul, "ULiULiD*ULi", "n", SM_60)
-TARGET_BUILTIN(__nvvm_atom_sys_max_gen_ul, "ULiULiD*ULi", "n", SM_60)
-BUILTIN(__nvvm_atom_max_gen_ll, "LLiLLiD*LLi", "n")
-TARGET_BUILTIN(__nvvm_atom_cta_max_gen_ll, "LLiLLiD*LLi", "n", SM_60)
-TARGET_BUILTIN(__nvvm_atom_sys_max_gen_ll, "LLiLLiD*LLi", "n", SM_60)
-BUILTIN(__nvvm_atom_max_gen_ull, "ULLiULLiD*ULLi", "n")
-TARGET_BUILTIN(__nvvm_atom_cta_max_gen_ull, "ULLiULLiD*ULLi", "n", SM_60)
-TARGET_BUILTIN(__nvvm_atom_sys_max_gen_ull, "ULLiULLiD*ULLi", "n", SM_60)
-
-BUILTIN(__nvvm_atom_min_gen_i, "iiD*i", "n")
-TARGET_BUILTIN(__nvvm_atom_cta_min_gen_i, "iiD*i", "n", SM_60)
-TARGET_BUILTIN(__nvvm_atom_sys_min_gen_i, "iiD*i", "n", SM_60)
-BUILTIN(__nvvm_atom_min_gen_ui, "UiUiD*Ui", "n")
-TARGET_BUILTIN(__nvvm_atom_cta_min_gen_ui, "UiUiD*Ui", "n", SM_60)
-TARGET_BUILTIN(__nvvm_atom_sys_min_gen_ui, "UiUiD*Ui", "n", SM_60)
-BUILTIN(__nvvm_atom_min_gen_l, "LiLiD*Li", "n")
-TARGET_BUILTIN(__nvvm_atom_cta_min_gen_l, "LiLiD*Li", "n", SM_60)
-TARGET_BUILTIN(__nvvm_atom_sys_min_gen_l, "LiLiD*Li", "n", SM_60)
-BUILTIN(__nvvm_atom_min_gen_ul, "ULiULiD*ULi", "n")
-TARGET_BUILTIN(__nvvm_atom_cta_min_gen_ul, "ULiULiD*ULi", "n", SM_60)
-TARGET_BUILTIN(__nvvm_atom_sys_min_gen_ul, "ULiULiD*ULi", "n", SM_60)
-BUILTIN(__nvvm_atom_min_gen_ll, "LLiLLiD*LLi", "n")
-TARGET_BUILTIN(__nvvm_atom_cta_min_gen_ll, "LLiLLiD*LLi", "n", SM_60)
-TARGET_BUILTIN(__nvvm_atom_sys_min_gen_ll, "LLiLLiD*LLi", "n", SM_60)
-BUILTIN(__nvvm_atom_min_gen_ull, "ULLiULLiD*ULLi", "n")
-TARGET_BUILTIN(__nvvm_atom_cta_min_gen_ull, "ULLiULLiD*ULLi", "n", SM_60)
-TARGET_BUILTIN(__nvvm_atom_sys_min_gen_ull, "ULLiULLiD*ULLi", "n", SM_60)
-
-BUILTIN(__nvvm_atom_inc_gen_ui, "UiUiD*Ui", "n")
-TARGET_BUILTIN(__nvvm_atom_cta_inc_gen_ui, "UiUiD*Ui", "n", SM_60)
-TARGET_BUILTIN(__nvvm_atom_sys_inc_gen_ui, "UiUiD*Ui", "n", SM_60)
-BUILTIN(__nvvm_atom_dec_gen_ui, "UiUiD*Ui", "n")
-TARGET_BUILTIN(__nvvm_atom_cta_dec_gen_ui, "UiUiD*Ui", "n", SM_60)
-TARGET_BUILTIN(__nvvm_atom_sys_dec_gen_ui, "UiUiD*Ui", "n", SM_60)
-
-BUILTIN(__nvvm_atom_and_gen_i, "iiD*i", "n")
-TARGET_BUILTIN(__nvvm_atom_cta_and_gen_i, "iiD*i", "n", SM_60)
-TARGET_BUILTIN(__nvvm_atom_sys_and_gen_i, "iiD*i", "n", SM_60)
-BUILTIN(__nvvm_atom_and_gen_l, "LiLiD*Li", "n")
-TARGET_BUILTIN(__nvvm_atom_cta_and_gen_l, "LiLiD*Li", "n", SM_60)
-TARGET_BUILTIN(__nvvm_atom_sys_and_gen_l, "LiLiD*Li", "n", SM_60)
-BUILTIN(__nvvm_atom_and_gen_ll, "LLiLLiD*LLi", "n")
-TARGET_BUILTIN(__nvvm_atom_cta_and_gen_ll, "LLiLLiD*LLi", "n", SM_60)
-TARGET_BUILTIN(__nvvm_atom_sys_and_gen_ll, "LLiLLiD*LLi", "n", SM_60)
-
-BUILTIN(__nvvm_atom_or_gen_i, "iiD*i", "n")
-TARGET_BUILTIN(__nvvm_atom_cta_or_gen_i, "iiD*i", "n", SM_60)
-TARGET_BUILTIN(__nvvm_atom_sys_or_gen_i, "iiD*i", "n", SM_60)
-BUILTIN(__nvvm_atom_or_gen_l, "LiLiD*Li", "n")
-TARGET_BUILTIN(__nvvm_atom_cta_or_gen_l, "LiLiD*Li", "n", SM_60)
-TARGET_BUILTIN(__nvvm_atom_sys_or_gen_l, "LiLiD*Li", "n", SM_60)
-BUILTIN(__nvvm_atom_or_gen_ll, "LLiLLiD*LLi", "n")
-TARGET_BUILTIN(__nvvm_atom_cta_or_gen_ll, "LLiLLiD*LLi", "n", SM_60)
-TARGET_BUILTIN(__nvvm_atom_sys_or_gen_ll, "LLiLLiD*LLi", "n", SM_60)
-
-BUILTIN(__nvvm_atom_xor_gen_i, "iiD*i", "n")
-TARGET_BUILTIN(__nvvm_atom_cta_xor_gen_i, "iiD*i", "n", SM_60)
-TARGET_BUILTIN(__nvvm_atom_sys_xor_gen_i, "iiD*i", "n", SM_60)
-BUILTIN(__nvvm_atom_xor_gen_l, "LiLiD*Li", "n")
-TARGET_BUILTIN(__nvvm_atom_cta_xor_gen_l, "LiLiD*Li", "n", SM_60)
-TARGET_BUILTIN(__nvvm_atom_sys_xor_gen_l, "LiLiD*Li", "n", SM_60)
-BUILTIN(__nvvm_atom_xor_gen_ll, "LLiLLiD*LLi", "n")
-TARGET_BUILTIN(__nvvm_atom_cta_xor_gen_ll, "LLiLLiD*LLi", "n", SM_60)
-TARGET_BUILTIN(__nvvm_atom_sys_xor_gen_ll, "LLiLLiD*LLi", "n", SM_60)
-
-TARGET_BUILTIN(__nvvm_atom_cas_gen_us, "UsUsD*UsUs", "n", SM_70)
-TARGET_BUILTIN(__nvvm_atom_cta_cas_gen_us, "UsUsD*UsUs", "n", SM_70)
-TARGET_BUILTIN(__nvvm_atom_sys_cas_gen_us, "UsUsD*UsUs", "n", SM_70)
-BUILTIN(__nvvm_atom_cas_gen_i, "iiD*ii", "n")
-TARGET_BUILTIN(__nvvm_atom_cta_cas_gen_i, "iiD*ii", "n", SM_60)
-TARGET_BUILTIN(__nvvm_atom_sys_cas_gen_i, "iiD*ii", "n", SM_60)
-BUILTIN(__nvvm_atom_cas_gen_l, "LiLiD*LiLi", "n")
-TARGET_BUILTIN(__nvvm_atom_cta_cas_gen_l, "LiLiD*LiLi", "n", SM_60)
-TARGET_BUILTIN(__nvvm_atom_sys_cas_gen_l, "LiLiD*LiLi", "n", SM_60)
-BUILTIN(__nvvm_atom_cas_gen_ll, "LLiLLiD*LLiLLi", "n")
-TARGET_BUILTIN(__nvvm_atom_cta_cas_gen_ll, "LLiLLiD*LLiLLi", "n", SM_60)
-TARGET_BUILTIN(__nvvm_atom_sys_cas_gen_ll, "LLiLLiD*LLiLLi", "n", SM_60)
-
-// Compiler Error Warn
-BUILTIN(__nvvm_compiler_error, "vcC*4", "n")
-BUILTIN(__nvvm_compiler_warn, "vcC*4", "n")
-
-BUILTIN(__nvvm_ldu_c, "ccC*", "")
-BUILTIN(__nvvm_ldu_sc, "ScScC*", "")
-BUILTIN(__nvvm_ldu_s, "ssC*", "")
-BUILTIN(__nvvm_ldu_i, "iiC*", "")
-BUILTIN(__nvvm_ldu_l, "LiLiC*", "")
-BUILTIN(__nvvm_ldu_ll, "LLiLLiC*", "")
-
-BUILTIN(__nvvm_ldu_uc, "UcUcC*", "")
-BUILTIN(__nvvm_ldu_us, "UsUsC*", "")
-BUILTIN(__nvvm_ldu_ui, "UiUiC*", "")
-BUILTIN(__nvvm_ldu_ul, "ULiULiC*", "")
-BUILTIN(__nvvm_ldu_ull, "ULLiULLiC*", "")
-
-BUILTIN(__nvvm_ldu_h, "hhC*", "")
-BUILTIN(__nvvm_ldu_f, "ffC*", "")
-BUILTIN(__nvvm_ldu_d, "ddC*", "")
-
-BUILTIN(__nvvm_ldu_c2, "E2cE2cC*", "")
-BUILTIN(__nvvm_ldu_sc2, "E2ScE2ScC*", "")
-BUILTIN(__nvvm_ldu_c4, "E4cE4cC*", "")
-BUILTIN(__nvvm_ldu_sc4, "E4ScE4ScC*", "")
-BUILTIN(__nvvm_ldu_s2, "E2sE2sC*", "")
-BUILTIN(__nvvm_ldu_s4, "E4sE4sC*", "")
-BUILTIN(__nvvm_ldu_i2, "E2iE2iC*", "")
-BUILTIN(__nvvm_ldu_i4, "E4iE4iC*", "")
-BUILTIN(__nvvm_ldu_l2, "E2LiE2LiC*", "")
-BUILTIN(__nvvm_ldu_ll2, "E2LLiE2LLiC*", "")
-
-BUILTIN(__nvvm_ldu_uc2, "E2UcE2UcC*", "")
-BUILTIN(__nvvm_ldu_uc4, "E4UcE4UcC*", "")
-BUILTIN(__nvvm_ldu_us2, "E2UsE2UsC*", "")
-BUILTIN(__nvvm_ldu_us4, "E4UsE4UsC*", "")
-BUILTIN(__nvvm_ldu_ui2, "E2UiE2UiC*", "")
-BUILTIN(__nvvm_ldu_ui4, "E4UiE4UiC*", "")
-BUILTIN(__nvvm_ldu_ul2, "E2ULiE2ULiC*", "")
-BUILTIN(__nvvm_ldu_ull2, "E2ULLiE2ULLiC*", "")
-
-BUILTIN(__nvvm_ldu_h2, "E2hE2hC*", "")
-BUILTIN(__nvvm_ldu_f2, "E2fE2fC*", "")
-BUILTIN(__nvvm_ldu_f4, "E4fE4fC*", "")
-BUILTIN(__nvvm_ldu_d2, "E2dE2dC*", "")
-
-BUILTIN(__nvvm_ldg_c, "ccC*", "")
-BUILTIN(__nvvm_ldg_sc, "ScScC*", "")
-BUILTIN(__nvvm_ldg_s, "ssC*", "")
-BUILTIN(__nvvm_ldg_i, "iiC*", "")
-BUILTIN(__nvvm_ldg_l, "LiLiC*", "")
-BUILTIN(__nvvm_ldg_ll, "LLiLLiC*", "")
-
-BUILTIN(__nvvm_ldg_uc, "UcUcC*", "")
-BUILTIN(__nvvm_ldg_us, "UsUsC*", "")
-BUILTIN(__nvvm_ldg_ui, "UiUiC*", "")
-BUILTIN(__nvvm_ldg_ul, "ULiULiC*", "")
-BUILTIN(__nvvm_ldg_ull, "ULLiULLiC*", "")
-
-BUILTIN(__nvvm_ldg_h, "hhC*", "")
-BUILTIN(__nvvm_ldg_f, "ffC*", "")
-BUILTIN(__nvvm_ldg_d, "ddC*", "")
-
-BUILTIN(__nvvm_ldg_c2, "E2cE2cC*", "")
-BUILTIN(__nvvm_ldg_sc2, "E2ScE2ScC*", "")
-BUILTIN(__nvvm_ldg_c4, "E4cE4cC*", "")
-BUILTIN(__nvvm_ldg_sc4, "E4ScE4ScC*", "")
-BUILTIN(__nvvm_ldg_s2, "E2sE2sC*", "")
-BUILTIN(__nvvm_ldg_s4, "E4sE4sC*", "")
-BUILTIN(__nvvm_ldg_i2, "E2iE2iC*", "")
-BUILTIN(__nvvm_ldg_i4, "E4iE4iC*", "")
-BUILTIN(__nvvm_ldg_l2, "E2LiE2LiC*", "")
-BUILTIN(__nvvm_ldg_ll2, "E2LLiE2LLiC*", "")
-
-BUILTIN(__nvvm_ldg_uc2, "E2UcE2UcC*", "")
-BUILTIN(__nvvm_ldg_uc4, "E4UcE4UcC*", "")
-BUILTIN(__nvvm_ldg_us2, "E2UsE2UsC*", "")
-BUILTIN(__nvvm_ldg_us4, "E4UsE4UsC*", "")
-BUILTIN(__nvvm_ldg_ui2, "E2UiE2UiC*", "")
-BUILTIN(__nvvm_ldg_ui4, "E4UiE4UiC*", "")
-BUILTIN(__nvvm_ldg_ul2, "E2ULiE2ULiC*", "")
-BUILTIN(__nvvm_ldg_ull2, "E2ULLiE2ULLiC*", "")
-
-BUILTIN(__nvvm_ldg_h2, "E2hE2hC*", "")
-BUILTIN(__nvvm_ldg_f2, "E2fE2fC*", "")
-BUILTIN(__nvvm_ldg_f4, "E4fE4fC*", "")
-BUILTIN(__nvvm_ldg_d2, "E2dE2dC*", "")
-
-// Address space predicates.
-BUILTIN(__nvvm_isspacep_const, "bvC*", "nc")
-BUILTIN(__nvvm_isspacep_global, "bvC*", "nc")
-BUILTIN(__nvvm_isspacep_local, "bvC*", "nc")
-BUILTIN(__nvvm_isspacep_shared, "bvC*", "nc")
-TARGET_BUILTIN(__nvvm_isspacep_shared_cluster,"bvC*", "nc", AND(SM_90,PTX78))
-
-// Builtins to support WMMA instructions on sm_70
-TARGET_BUILTIN(__hmma_m16n16k16_ld_a, "vi*iC*UiIi", "", AND(SM_70,PTX60))
-TARGET_BUILTIN(__hmma_m16n16k16_ld_b, "vi*iC*UiIi", "", AND(SM_70,PTX60))
-TARGET_BUILTIN(__hmma_m16n16k16_ld_c_f16, "vi*iC*UiIi", "", AND(SM_70,PTX60))
-TARGET_BUILTIN(__hmma_m16n16k16_ld_c_f32, "vf*fC*UiIi", "", AND(SM_70,PTX60))
-TARGET_BUILTIN(__hmma_m16n16k16_st_c_f16, "vi*iC*UiIi", "", AND(SM_70,PTX60))
-TARGET_BUILTIN(__hmma_m16n16k16_st_c_f32, "vf*fC*UiIi", "", AND(SM_70,PTX60))
-
-TARGET_BUILTIN(__hmma_m32n8k16_ld_a, "vi*iC*UiIi", "", AND(SM_70,PTX61))
-TARGET_BUILTIN(__hmma_m32n8k16_ld_b, "vi*iC*UiIi", "", AND(SM_70,PTX61))
-TARGET_BUILTIN(__hmma_m32n8k16_ld_c_f16, "vi*iC*UiIi", "", AND(SM_70,PTX61))
-TARGET_BUILTIN(__hmma_m32n8k16_ld_c_f32, "vf*fC*UiIi", "", AND(SM_70,PTX61))
-TARGET_BUILTIN(__hmma_m32n8k16_st_c_f16, "vi*iC*UiIi", "", AND(SM_70,PTX61))
-TARGET_BUILTIN(__hmma_m32n8k16_st_c_f32, "vf*fC*UiIi", "", AND(SM_70,PTX61))
-
-TARGET_BUILTIN(__hmma_m8n32k16_ld_a, "vi*iC*UiIi", "", AND(SM_70,PTX61))
-TARGET_BUILTIN(__hmma_m8n32k16_ld_b, "vi*iC*UiIi", "", AND(SM_70,PTX61))
-TARGET_BUILTIN(__hmma_m8n32k16_ld_c_f16, "vi*iC*UiIi", "", AND(SM_70,PTX61))
-TARGET_BUILTIN(__hmma_m8n32k16_ld_c_f32, "vf*fC*UiIi", "", AND(SM_70,PTX61))
-TARGET_BUILTIN(__hmma_m8n32k16_st_c_f16, "vi*iC*UiIi", "", AND(SM_70,PTX61))
-TARGET_BUILTIN(__hmma_m8n32k16_st_c_f32, "vf*fC*UiIi", "", AND(SM_70,PTX61))
-
-TARGET_BUILTIN(__hmma_m16n16k16_mma_f16f16, "vi*iC*iC*iC*IiIi", "", AND(SM_70,PTX60))
-TARGET_BUILTIN(__hmma_m16n16k16_mma_f32f16, "vf*iC*iC*iC*IiIi", "", AND(SM_70,PTX60))
-TARGET_BUILTIN(__hmma_m16n16k16_mma_f32f32, "vf*iC*iC*fC*IiIi", "", AND(SM_70,PTX60))
-TARGET_BUILTIN(__hmma_m16n16k16_mma_f16f32, "vi*iC*iC*fC*IiIi", "", AND(SM_70,PTX60))
-
-TARGET_BUILTIN(__hmma_m32n8k16_mma_f16f16, "vi*iC*iC*iC*IiIi", "", AND(SM_70,PTX61))
-TARGET_BUILTIN(__hmma_m32n8k16_mma_f32f16, "vf*iC*iC*iC*IiIi", "", AND(SM_70,PTX61))
-TARGET_BUILTIN(__hmma_m32n8k16_mma_f32f32, "vf*iC*iC*fC*IiIi", "", AND(SM_70,PTX61))
-TARGET_BUILTIN(__hmma_m32n8k16_mma_f16f32, "vi*iC*iC*fC*IiIi", "", AND(SM_70,PTX61))
-
-TARGET_BUILTIN(__hmma_m8n32k16_mma_f16f16, "vi*iC*iC*iC*IiIi", "", AND(SM_70,PTX61))
-TARGET_BUILTIN(__hmma_m8n32k16_mma_f32f16, "vf*iC*iC*iC*IiIi", "", AND(SM_70,PTX61))
-TARGET_BUILTIN(__hmma_m8n32k16_mma_f32f32, "vf*iC*iC*fC*IiIi", "", AND(SM_70,PTX61))
-TARGET_BUILTIN(__hmma_m8n32k16_mma_f16f32, "vi*iC*iC*fC*IiIi", "", AND(SM_70,PTX61))
-
-// Builtins to support integer and sub-integer WMMA instructions on sm_72/sm_75
-TARGET_BUILTIN(__bmma_m8n8k128_ld_a_b1, "vi*iC*UiIi", "", AND(SM_75,PTX63))
-TARGET_BUILTIN(__bmma_m8n8k128_ld_b_b1, "vi*iC*UiIi", "", AND(SM_75,PTX63))
-TARGET_BUILTIN(__bmma_m8n8k128_ld_c, "vi*iC*UiIi", "", AND(SM_75,PTX63))
-TARGET_BUILTIN(__bmma_m8n8k128_mma_and_popc_b1, "vi*iC*iC*iC*Ii", "", AND(SM_80,PTX71))
-TARGET_BUILTIN(__bmma_m8n8k128_mma_xor_popc_b1, "vi*iC*iC*iC*Ii", "", AND(SM_75,PTX63))
-TARGET_BUILTIN(__bmma_m8n8k128_st_c_i32, "vi*iC*UiIi", "", AND(SM_75,PTX63))
-TARGET_BUILTIN(__imma_m16n16k16_ld_a_s8, "vi*iC*UiIi", "", AND(SM_72,PTX63))
-TARGET_BUILTIN(__imma_m16n16k16_ld_a_u8, "vi*iC*UiIi", "", AND(SM_72,PTX63))
-TARGET_BUILTIN(__imma_m16n16k16_ld_b_s8, "vi*iC*UiIi", "", AND(SM_72,PTX63))
-TARGET_BUILTIN(__imma_m16n16k16_ld_b_u8, "vi*iC*UiIi", "", AND(SM_72,PTX63))
-TARGET_BUILTIN(__imma_m16n16k16_ld_c, "vi*iC*UiIi", "", AND(SM_72,PTX63))
-TARGET_BUILTIN(__imma_m16n16k16_mma_s8, "vi*iC*iC*iC*IiIi", "", AND(SM_72,PTX63))
-TARGET_BUILTIN(__imma_m16n16k16_mma_u8, "vi*iC*iC*iC*IiIi", "", AND(SM_72,PTX63))
-TARGET_BUILTIN(__imma_m16n16k16_st_c_i32, "vi*iC*UiIi", "", AND(SM_72,PTX63))
-TARGET_BUILTIN(__imma_m32n8k16_ld_a_s8, "vi*iC*UiIi", "", AND(SM_72,PTX63))
-TARGET_BUILTIN(__imma_m32n8k16_ld_a_u8, "vi*iC*UiIi", "", AND(SM_72,PTX63))
-TARGET_BUILTIN(__imma_m32n8k16_ld_b_s8, "vi*iC*UiIi", "", AND(SM_72,PTX63))
-TARGET_BUILTIN(__imma_m32n8k16_ld_b_u8, "vi*iC*UiIi", "", AND(SM_72,PTX63))
-TARGET_BUILTIN(__imma_m32n8k16_ld_c, "vi*iC*UiIi", "", AND(SM_72,PTX63))
-TARGET_BUILTIN(__imma_m32n8k16_mma_s8, "vi*iC*iC*iC*IiIi", "", AND(SM_72,PTX63))
-TARGET_BUILTIN(__imma_m32n8k16_mma_u8, "vi*iC*iC*iC*IiIi", "", AND(SM_72,PTX63))
-TARGET_BUILTIN(__imma_m32n8k16_st_c_i32, "vi*iC*UiIi", "", AND(SM_72,PTX63))
-TARGET_BUILTIN(__imma_m8n32k16_ld_a_s8, "vi*iC*UiIi", "", AND(SM_72,PTX63))
-TARGET_BUILTIN(__imma_m8n32k16_ld_a_u8, "vi*iC*UiIi", "", AND(SM_72,PTX63))
-TARGET_BUILTIN(__imma_m8n32k16_ld_b_s8, "vi*iC*UiIi", "", AND(SM_72,PTX63))
-TARGET_BUILTIN(__imma_m8n32k16_ld_b_u8, "vi*iC*UiIi", "", AND(SM_72,PTX63))
-TARGET_BUILTIN(__imma_m8n32k16_ld_c, "vi*iC*UiIi", "", AND(SM_72,PTX63))
-TARGET_BUILTIN(__imma_m8n32k16_mma_s8, "vi*iC*iC*iC*IiIi", "", AND(SM_72,PTX63))
-TARGET_BUILTIN(__imma_m8n32k16_mma_u8, "vi*iC*iC*iC*IiIi", "", AND(SM_72,PTX63))
-TARGET_BUILTIN(__imma_m8n32k16_st_c_i32, "vi*iC*UiIi", "", AND(SM_72,PTX63))
-TARGET_BUILTIN(__imma_m8n8k32_ld_a_s4, "vi*iC*UiIi", "", AND(SM_75,PTX63))
-TARGET_BUILTIN(__imma_m8n8k32_ld_a_u4, "vi*iC*UiIi", "", AND(SM_75,PTX63))
-TARGET_BUILTIN(__imma_m8n8k32_ld_b_s4, "vi*iC*UiIi", "", AND(SM_75,PTX63))
-TARGET_BUILTIN(__imma_m8n8k32_ld_b_u4, "vi*iC*UiIi", "", AND(SM_75,PTX63))
-TARGET_BUILTIN(__imma_m8n8k32_ld_c, "vi*iC*UiIi", "", AND(SM_75,PTX63))
-TARGET_BUILTIN(__imma_m8n8k32_mma_s4, "vi*iC*iC*iC*IiIi", "", AND(SM_75,PTX63))
-TARGET_BUILTIN(__imma_m8n8k32_mma_u4, "vi*iC*iC*iC*IiIi", "", AND(SM_75,PTX63))
-TARGET_BUILTIN(__imma_m8n8k32_st_c_i32, "vi*iC*UiIi", "", AND(SM_75,PTX63))
-
-// Builtins to support double and alternate float WMMA instructions on sm_80
-TARGET_BUILTIN(__dmma_m8n8k4_ld_a, "vd*dC*UiIi", "", AND(SM_80,PTX70))
-TARGET_BUILTIN(__dmma_m8n8k4_ld_b, "vd*dC*UiIi", "", AND(SM_80,PTX70))
-TARGET_BUILTIN(__dmma_m8n8k4_ld_c, "vd*dC*UiIi", "", AND(SM_80,PTX70))
-TARGET_BUILTIN(__dmma_m8n8k4_st_c_f64, "vd*dC*UiIi", "", AND(SM_80,PTX70))
-TARGET_BUILTIN(__dmma_m8n8k4_mma_f64, "vd*dC*dC*dC*IiIi", "", AND(SM_80,PTX70))
-
-TARGET_BUILTIN(__mma_bf16_m16n16k16_ld_a, "vi*iC*UiIi", "", AND(SM_80,PTX70))
-TARGET_BUILTIN(__mma_bf16_m16n16k16_ld_b, "vi*iC*UiIi", "", AND(SM_80,PTX70))
-TARGET_BUILTIN(__mma_bf16_m16n16k16_mma_f32, "vf*iC*iC*fC*IiIi", "", AND(SM_80,PTX70))
-TARGET_BUILTIN(__mma_bf16_m8n32k16_ld_a, "vi*iC*UiIi", "", AND(SM_80,PTX70))
-TARGET_BUILTIN(__mma_bf16_m8n32k16_ld_b, "vi*iC*UiIi", "", AND(SM_80,PTX70))
-TARGET_BUILTIN(__mma_bf16_m8n32k16_mma_f32, "vf*iC*iC*fC*IiIi", "", AND(SM_80,PTX70))
-TARGET_BUILTIN(__mma_bf16_m32n8k16_ld_a, "vi*iC*UiIi", "", AND(SM_80,PTX70))
-TARGET_BUILTIN(__mma_bf16_m32n8k16_ld_b, "vi*iC*UiIi", "", AND(SM_80,PTX70))
-TARGET_BUILTIN(__mma_bf16_m32n8k16_mma_f32, "vf*iC*iC*fC*IiIi", "", AND(SM_80,PTX70))
-
-TARGET_BUILTIN(__mma_tf32_m16n16k8_ld_a, "vi*iC*UiIi", "", AND(SM_80,PTX70))
-TARGET_BUILTIN(__mma_tf32_m16n16k8_ld_b, "vi*iC*UiIi", "", AND(SM_80,PTX70))
-TARGET_BUILTIN(__mma_tf32_m16n16k8_ld_c, "vf*fC*UiIi", "", AND(SM_80,PTX70))
-TARGET_BUILTIN(__mma_m16n16k8_st_c_f32, "vf*fC*UiIi", "", AND(SM_80,PTX70))
-TARGET_BUILTIN(__mma_tf32_m16n16k8_mma_f32, "vf*iC*iC*fC*IiIi", "", AND(SM_80,PTX70))
-
-// Async Copy
-TARGET_BUILTIN(__nvvm_cp_async_mbarrier_arrive, "vWi*", "", AND(SM_80,PTX70))
-TARGET_BUILTIN(__nvvm_cp_async_mbarrier_arrive_shared, "vWi*3", "", AND(SM_80,PTX70))
-TARGET_BUILTIN(__nvvm_cp_async_mbarrier_arrive_noinc, "vWi*", "", AND(SM_80,PTX70))
-TARGET_BUILTIN(__nvvm_cp_async_mbarrier_arrive_noinc_shared, "vWi*3", "", AND(SM_80,PTX70))
-
-TARGET_BUILTIN(__nvvm_cp_async_ca_shared_global_4, "vv*3vC*1.", "", AND(SM_80,PTX70))
-TARGET_BUILTIN(__nvvm_cp_async_ca_shared_global_8, "vv*3vC*1.", "", AND(SM_80,PTX70))
-TARGET_BUILTIN(__nvvm_cp_async_ca_shared_global_16, "vv*3vC*1.", "", AND(SM_80,PTX70))
-TARGET_BUILTIN(__nvvm_cp_async_cg_shared_global_16, "vv*3vC*1.", "", AND(SM_80,PTX70))
-
-TARGET_BUILTIN(__nvvm_cp_async_commit_group, "v", "", AND(SM_80,PTX70))
-TARGET_BUILTIN(__nvvm_cp_async_wait_group, "vIi", "", AND(SM_80,PTX70))
-TARGET_BUILTIN(__nvvm_cp_async_wait_all, "v", "", AND(SM_80,PTX70))
-
-
-// bf16, bf16x2 abs, neg
-TARGET_BUILTIN(__nvvm_abs_bf16, "yy", "", AND(SM_80,PTX70))
-TARGET_BUILTIN(__nvvm_abs_bf16x2, "V2yV2y", "", AND(SM_80,PTX70))
-TARGET_BUILTIN(__nvvm_neg_bf16, "yy", "", AND(SM_80,PTX70))
-TARGET_BUILTIN(__nvvm_neg_bf16x2, "V2yV2y", "", AND(SM_80,PTX70))
-
-TARGET_BUILTIN(__nvvm_mapa, "v*v*i", "", AND(SM_90, PTX78))
-TARGET_BUILTIN(__nvvm_mapa_shared_cluster, "v*3v*3i", "", AND(SM_90, PTX78))
-TARGET_BUILTIN(__nvvm_getctarank, "iv*", "", AND(SM_90, PTX78))
-TARGET_BUILTIN(__nvvm_getctarank_shared_cluster, "iv*3", "", AND(SM_90,PTX78))
-
-#undef BUILTIN
-#undef TARGET_BUILTIN
-#pragma pop_macro("AND")
-#pragma pop_macro("SM_53")
-#pragma pop_macro("SM_60")
-#pragma pop_macro("SM_70")
-#pragma pop_macro("SM_72")
-#pragma pop_macro("SM_75")
-#pragma pop_macro("SM_80")
-#pragma pop_macro("SM_86")
-#pragma pop_macro("SM_87")
-#pragma pop_macro("SM_89")
-#pragma pop_macro("SM_90")
-#pragma pop_macro("SM_90a")
-#pragma pop_macro("SM_100")
-#pragma pop_macro("SM_100a")
-#pragma pop_macro("PTX42")
-#pragma pop_macro("PTX60")
-#pragma pop_macro("PTX61")
-#pragma pop_macro("PTX62")
-#pragma pop_macro("PTX63")
-#pragma pop_macro("PTX64")
-#pragma pop_macro("PTX65")
-#pragma pop_macro("PTX70")
-#pragma pop_macro("PTX71")
-#pragma pop_macro("PTX72")
-#pragma pop_macro("PTX73")
-#pragma pop_macro("PTX74")
-#pragma pop_macro("PTX75")
-#pragma pop_macro("PTX76")
-#pragma pop_macro("PTX77")
-#pragma pop_macro("PTX78")
-#pragma pop_macro("PTX80")
-#pragma pop_macro("PTX81")
-#pragma pop_macro("PTX82")
-#pragma pop_macro("PTX83")
-#pragma pop_macro("PTX84")
-#pragma pop_macro("PTX85")
-#pragma pop_macro("PTX86")
diff --git a/clang/include/clang/Basic/BuiltinsNVPTX.td b/clang/include/clang/Basic/BuiltinsNVPTX.td
new file mode 100644
index 0000000000000..9d24a992563a4
--- /dev/null
+++ b/clang/include/clang/Basic/BuiltinsNVPTX.td
@@ -0,0 +1,1079 @@
+//===--- BuiltinsNVPTX.td - NVPTX Builtin function defs ---------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file defines the PTX-specific builtin function database.
+//
+//===----------------------------------------------------------------------===//
+
+include "clang/Basic/BuiltinsBase.td"
+
+class SMFeatures {
+  string Features;
+}
+
+class SM<string version, list<SMFeatures> newer_list> : SMFeatures {
+  let Features = !foldl(!strconcat("sm_", version), newer_list, f, newer,
+                        !strconcat(f, "|", newer.Features));
+}
+
+let Features = "sm_100a" in def SM_100a : SMFeatures;
+
+def SM_100 : SM<"100", [SM_100a]>;
+
+let Features = "sm_90a" in def SM_90a : SMFeatures;
+
+def SM_90 : SM<"90", [SM_90a, SM_100]>;
+def SM_89 : SM<"89", [SM_90]>;
+def SM_87 : SM<"87", [SM_89]>;
+def SM_86 : SM<"86", [SM_87]>;
+def SM_80 : SM<"80", [SM_86]>;
+def SM_75 : SM<"75", [SM_80]>;
+def SM_72 : SM<"72", [SM_75]>;
+def SM_70 : SM<"70", [SM_72]>;
+def SM_62 : SM<"62", [SM_70]>;
+def SM_61 : SM<"61", [SM_62]>;
+def SM_60 : SM<"60", [SM_61]>;
+def SM_53 : SM<"53", [SM_60]>;
+
+class PTXFeatures {
+  string Features;
+}
+
+class PTX<string version, PTXFeatures newer> : PTXFeatures {
+  let Features = !strconcat("ptx", version, "|", newer.Features);
+}
+
+let Features = "ptx87" in def PTX87 : PTXFeatures;
+
+def PTX86 : PTX<"86", PTX87>;
+def PTX85 : PTX<"85", PTX86>;
+def PTX84 : PTX<"84", PTX85>;
+def PTX83 : PTX<"83", PTX84>;
+def PTX82 : PTX<"82", PTX83>;
+def PTX81 : PTX<"81", PTX82>;
+def PTX80 : PTX<"80", PTX81>;
+def PTX78 : PTX<"78", PTX80>;
+def PTX77 : PTX<"77", PTX78>;
+def PTX76 : PTX<"76", PTX77>;
+def PTX75 : PTX<"75", PTX76>;
+def PTX74 : PTX<"74", PTX75>;
+def PTX73 : PTX<"73", PTX74>;
+def PTX72 : PTX<"72", PTX73>;
+def PTX71 : PTX<"71", PTX72>;
+def PTX70 : PTX<"70", PTX71>;
+def PTX65 : PTX<"65", PTX70>;
+def PTX64 : PTX<"64", PTX65>;
+def PTX63 : PTX<"63", PTX64>;
+def PTX62 : PTX<"62", PTX63>;
+def PTX61 : PTX<"61", PTX62>;
+def PTX60 : PTX<"60", PTX61>;
+def PTX42 : PTX<"42", PTX60>;
+
+class NVPTXBuiltin<string prototype> : TargetBuiltin {
+  let Spellings = [NAME];
+  let Prototype = prototype;
+}
+
+class NVPTXBuiltinSM<string prototype, SMFeatures sm> : NVPTXBuiltin<prototype> {
+  let Features = sm.Features;
+}
+
+class NVPTXBuiltinPTX<string prototype, PTXFeatures ptx> : NVPTXBuiltin<prototype> {
+  let Features = ptx.Features;
+}
+
+class NVPTXBuiltinSMAndPTX<string prototype, SMFeatures sm, PTXFeatures ptx> : NVPTXBuiltin<prototype> {
+  let Features = !strconcat("(", sm.Features, "),(", ptx.Features, ")");
+}
+
+// Special Registers
+
+let Attributes = [NoThrow, Const] in {
+  def __nvvm_read_ptx_sreg_tid_x : NVPTXBuiltin<"int()">;
+  def __nvvm_read_ptx_sreg_tid_y : NVPTXBuiltin<"int()">;
+  def __nvvm_read_ptx_sreg_tid_z : NVPTXBuiltin<"int()">;
+  def __nvvm_read_ptx_sreg_tid_w : NVPTXBuiltin<"int()">;
+
+  def __nvvm_read_ptx_sreg_ntid_x : NVPTXBuiltin<"int()">;
+  def __nvvm_read_ptx_sreg_ntid_y : NVPTXBuiltin<"int()">;
+  def __nvvm_read_ptx_sreg_ntid_z : NVPTXBuiltin<"int()">;
+  def __nvvm_read_ptx_sreg_ntid_w : NVPTXBuiltin<"int()">;
+
+  def __nvvm_read_ptx_sreg_ctaid_x : NVPTXBuiltin<"int()">;
+  def __nvvm_read_ptx_sreg_ctaid_y : NVPTXBuiltin<"int()">;
+  def __nvvm_read_ptx_sreg_ctaid_z : NVPTXBuiltin<"int()">;
+  def __nvvm_read_ptx_sreg_ctaid_w : NVPTXBuiltin<"int()">;
+
+  def __nvvm_read_ptx_sreg_nctaid_x : NVPTXBuiltin<"int()">;
+  def __nvvm_read_ptx_sreg_nctaid_y : NVPTXBuiltin<"int()">;
+  def __nvvm_read_ptx_sreg_nctaid_z : NVPTXBuiltin<"int()">;
+  def __nvvm_read_ptx_sreg_nctaid_w : NVPTXBuiltin<"int()">;
+
+  def __nvvm_read_ptx_sreg_clusterid_x : NVPTXBuiltinSMAndPTX<"int()", SM_90, PTX78>;
+  def __nvvm_read_ptx_sreg_clusterid_y : NVPTXBuiltinSMAndPTX<"int()", SM_90, PTX78>;
+  def __nvvm_read_ptx_sreg_clusterid_z : NVPTXBuiltinSMAndPTX<"int()", SM_90, PTX78>;
+  def __nvvm_read_ptx_sreg_clusterid_w : NVPTXBuiltinSMAndPTX<"int()", SM_90, PTX78>;
+
+  def __nvvm_read_ptx_sreg_nclusterid_x : NVPTXBuiltinSMAndPTX<"int()", SM_90, PTX78>;
+  def __nvvm_read_ptx_sreg_nclusterid_y : NVPTXBuiltinSMAndPTX<"int()", SM_90, PTX78>;
+  def __nvvm_read_ptx_sreg_nclusterid_z : NVPTXBuiltinSMAndPTX<"int()", SM_90, PTX78>;
+  def __nvvm_read_ptx_sreg_nclusterid_w : NVPTXBuiltinSMAndPTX<"int()", SM_90, PTX78>;
+
+  def __nvvm_read_ptx_sreg_cluster_ctaid_x : NVPTXBuiltinSMAndPTX<"int()", SM_90, PTX78>;
+  def __nvvm_read_ptx_sreg_cluster_ctaid_y : NVPTXBuiltinSMAndPTX<"int()", SM_90, PTX78>;
+  def __nvvm_read_ptx_sreg_cluster_ctaid_z : NVPTXBuiltinSMAndPTX<"int()", SM_90, PTX78>;
+  def __nvvm_read_ptx_sreg_cluster_ctaid_w : NVPTXBuiltinSMAndPTX<"int()", SM_90, PTX78>;
+
+  def __nvvm_read_ptx_sreg_cluster_nctaid_x : NVPTXBuiltinSMAndPTX<"int()", SM_90, PTX78>;
+  def __nvvm_read_ptx_sreg_cluster_nctaid_y : NVPTXBuiltinSMAndPTX<"int()", SM_90, PTX78>;
+  def __nvvm_read_ptx_sreg_cluster_nctaid_z : NVPTXBuiltinSMAndPTX<"int()", SM_90, PTX78>;
+  def __nvvm_read_ptx_sreg_cluster_nctaid_w : NVPTXBuiltinSMAndPTX<"int()", SM_90, PTX78>;
+
+  def __nvvm_read_ptx_sreg_cluster_ctarank : NVPTXBuiltinSMAndPTX<"int()", SM_90, PTX78>;
+  def __nvvm_read_ptx_sreg_cluster_nctarank : NVPTXBuiltinSMAndPTX<"int()", SM_90, PTX78>;
+
+  def __nvvm_is_explicit_cluster : NVPTXBuiltinSMAndPTX<"bool()", SM_90, PTX78>;
+
+  def __nvvm_read_ptx_sreg_laneid : NVPTXBuiltin<"int()">;
+  def __nvvm_read_ptx_sreg_warpid : NVPTXBuiltin<"int()">;
+  def __nvvm_read_ptx_sreg_nwarpid : NVPTXBuiltin<"int()">;
+  def __nvvm_read_ptx_sreg_warpsize : NVPTXBuiltin<"int()">;
+
+  def __nvvm_read_ptx_sreg_smid : NVPTXBuiltin<"int()">;
+  def __nvvm_read_ptx_sreg_nsmid : NVPTXBuiltin<"int()">;
+  def __nvvm_read_ptx_sreg_gridid : NVPTXBuiltin<"int()">;
+
+  def __nvvm_read_ptx_sreg_lanemask_eq : NVPTXBuiltin<"int()">;
+  def __nvvm_read_ptx_sreg_lanemask_le : NVPTXBuiltin<"int()">;
+  def __nvvm_read_ptx_sreg_lanemask_lt : NVPTXBuiltin<"int()">;
+  def __nvvm_read_ptx_sreg_lanemask_ge : NVPTXBuiltin<"int()">;
+  def __nvvm_read_ptx_sreg_lanemask_gt : NVPTXBuiltin<"int()">;
+}
+
+let Attributes = [NoThrow] in {
+  def __nvvm_read_ptx_sreg_clock : NVPTXBuiltin<"int()">;
+  def __nvvm_read_ptx_sreg_clock64 : NVPTXBuiltin<"long long int()">;
+  def __nvvm_read_ptx_sreg_globaltimer : NVPTXBuiltin<"long long int()">;
+
+  def __nvvm_read_ptx_sreg_pm0 : NVPTXBuiltin<"int()">;
+  def __nvvm_read_ptx_sreg_pm1 : NVPTXBuiltin<"int()">;
+  def __nvvm_read_ptx_sreg_pm2 : NVPTXBuiltin<"int()">;
+  def __nvvm_read_ptx_sreg_pm3 : NVPTXBuiltin<"int()">;
+}
+
+// MISC
+
+def __nvvm_prmt : NVPTXBuiltin<"unsigned int(unsigned int, unsigned int, unsigned int)">;
+let Attributes = [NoReturn] in {
+  def __nvvm_exit : NVPTXBuiltin<"void()">;
+  def __nvvm_reflect : NVPTXBuiltin<"unsigned int(char const *)">;
+}
+let Attributes = [NoThrow] in {
+  def __nvvm_nanosleep : NVPTXBuiltinSMAndPTX<"void(unsigned int)", SM_70, PTX63>;
+}
+
+// Min Max
+
+def __nvvm_fmin_f16 : NVPTXBuiltinSMAndPTX<"__fp16(__fp16, __fp16)", SM_80, PTX70>;
+def __nvvm_fmin_ftz_f16 : NVPTXBuiltinSMAndPTX<"__fp16(__fp16, __fp16)", SM_80, PTX70>;
+def __nvvm_fmin_nan_f16 : NVPTXBuiltinSMAndPTX<"__fp16(__fp16, __fp16)", SM_80, PTX70>;
+def __nvvm_fmin_ftz_nan_f16 : NVPTXBuiltinSMAndPTX<"__fp16(__fp16, __fp16)", SM_80, PTX70>;
+def __nvvm_fmin_xorsign_abs_f16 : NVPTXBuiltinSMAndPTX<"__fp16(__fp16, __fp16)", SM_86, PTX72>;
+def __nvvm_fmin_ftz_xorsign_abs_f16 : NVPTXBuiltinSMAndPTX<"__fp16(__fp16, __fp16)", SM_86, PTX72>;
+def __nvvm_fmin_nan_xorsign_abs_f16 : NVPTXBuiltinSMAndPTX<"__fp16(__fp16, __fp16)", SM_86, PTX72>;
+def __nvvm_fmin_ftz_nan_xorsign_abs_f16 : NVPTXBuiltinSMAndPTX<"__fp16(__fp16, __fp16)", SM_86, PTX72>;
+def __nvvm_fmin_f16x2 : NVPTXBuiltinSMAndPTX<"_Vector<2, __fp16>(_Vector<2, __fp16>, _Vector<2, __fp16>)", SM_80, PTX70>;
+def __nvvm_fmin_ftz_f16x2 : NVPTXBuiltinSMAndPTX<"_Vector<2, __fp16>(_Vector<2, __fp16>, _Vector<2, __fp16>)", SM_80, PTX70>;
+def __nvvm_fmin_nan_f16x2 : NVPTXBuiltinSMAndPTX<"_Vector<2, __fp16>(_Vector<2, __fp16>, _Vector<2, __fp16>)", SM_80, PTX70>;
+def __nvvm_fmin_ftz_nan_f16x2 : NVPTXBuiltinSMAndPTX<"_Vector<2, __fp16>(_Vector<2, __fp16>, _Vector<2, __fp16>)", SM_80, PTX70>;
+def __nvvm_fmin_xorsign_abs_f16x2 : NVPTXBuiltinSMAndPTX<"_Vector<2, __fp16>(_Vector<2, __fp16>, _Vector<2, __fp16>)", SM_86, PTX72>;
+def __nvvm_fmin_ftz_xorsign_abs_f16x2 : NVPTXBuiltinSMAndPTX<"_Vector<2, __fp16>(_Vector<2, __fp16>, _Vector<2, __fp16>)", SM_86, PTX72>;
+def __nvvm_fmin_nan_xorsign_abs_f16x2 : NVPTXBuiltinSMAndPTX<"_Vector<2, __fp16>(_Vector<2, __fp16>, _Vector<2, __fp16>)", SM_86, PTX72>;
+def __nvvm_fmin_ftz_nan_xorsign_abs_f16x2 : NVPTXBuiltinSMAndPTX<"_Vector<2, __fp16>(_Vector<2, __fp16>, _Vector<2, __fp16>)", SM_86, PTX72>;
+def __nvvm_fmin_bf16 : NVPTXBuiltinSMAndPTX<"__bf16(__bf16, __bf16)", SM_80, PTX70>;
+def __nvvm_fmin_ftz_bf16 : NVPTXBuiltinSMAndPTX<"__bf16(__bf16, __bf16)", SM_80, PTX70>;
+def __nvvm_fmin_nan_bf16 : NVPTXBuiltinSMAndPTX<"__bf16(__bf16, __bf16)", SM_80, PTX70>;
+def __nvvm_fmin_ftz_nan_bf16 : NVPTXBuiltinSMAndPTX<"__bf16(__bf16, __bf16)", SM_80, PTX70>;
+def __nvvm_fmin_xorsign_abs_bf16 : NVPTXBuiltinSMAndPTX<"__bf16(__bf16, __bf16)", SM_86, PTX72>;
+def __nvvm_fmin_nan_xorsign_abs_bf16 : NVPTXBuiltinSMAndPTX<"__bf16(__bf16, __bf16)", SM_86, PTX72>;
+def __nvvm_fmin_bf16x2 : NVPTXBuiltinSMAndPTX<"_Vector<2, __bf16>(_Vector<2, __bf16>, _Vector<2, __bf16>)", SM_80, PTX70>;
+def __nvvm_fmin_ftz_bf16x2 : NVPTXBuiltinSMAndPTX<"_Vector<2, __bf16>(_Vector<2, __bf16>, _Vector<2, __bf16>)", SM_80, PTX70>;
+def __nvvm_fmin_nan_bf16x2 : NVPTXBuiltinSMAndPTX<"_Vector<2, __bf16>(_Vector<2, __bf16>, _Vector<2, __bf16>)", SM_80, PTX70>;
+def __nvvm_fmin_ftz_nan_bf16x2 : NVPTXBuiltinSMAndPTX<"_Vector<2, __bf16>(_Vector<2, __bf16>, _Vector<2, __bf16>)", SM_80, PTX70>;
+def __nvvm_fmin_xorsign_abs_bf16x2 : NVPTXBuiltinSMAndPTX<"_Vector<2, __bf16>(_Vector<2, __bf16>, _Vector<2, __bf16>)", SM_86, PTX72>;
+def __nvvm_fmin_nan_xorsign_abs_bf16x2 : NVPTXBuiltinSMAndPTX<"_Vector<2, __bf16>(_Vector<2, __bf16>, _Vector<2, __bf16>)", SM_86, PTX72>;
+def __nvvm_fmin_f : NVPTXBuiltin<"float(float, float)">;
+def __nvvm_fmin_ftz_f : NVPTXBuiltin<"float(float, float)">;
+def __nvvm_fmin_nan_f : NVPTXBuiltinSMAndPTX<"float(float, float)", SM_80, PTX70>;
+def __nvvm_fmin_ftz_nan_f : NVPTXBuiltinSMAndPTX<"float(float, float)", SM_80, PTX70>;
+def __nvvm_fmin_xorsign_abs_f : NVPTXBuiltinSMAndPTX<"float(float, float)", SM_86, PTX72>;
+def __nvvm_fmin_ftz_xorsign_abs_f : NVPTXBuiltinSMAndPTX<"float(float, float)", SM_86, PTX72>;
+def __nvvm_fmin_nan_xorsign_abs_f : NVPTXBuiltinSMAndPTX<"float(float, float)", SM_86, PTX72>;
+def __nvvm_fmin_ftz_nan_xorsign_abs_f : NVPTXBuiltinSMAndPTX<"float(float, float)", SM_86, PTX72>;
+def __nvvm_fmin_d : NVPTXBuiltin<"double(double, double)">;
+
+def __nvvm_fmax_f16 : NVPTXBuiltinSMAndPTX<"__fp16(__fp16, __fp16)", SM_80, PTX70>;
+def __nvvm_fmax_ftz_f16 : NVPTXBuiltinSMAndPTX<"__fp16(__fp16, __fp16)", SM_80, PTX70>;
+def __nvvm_fmax_nan_f16 : NVPTXBuiltinSMAndPTX<"__fp16(__fp16, __fp16)", SM_80, PTX70>;
+def __nvvm_fmax_ftz_nan_f16 : NVPTXBuiltinSMAndPTX<"__fp16(__fp16, __fp16)", SM_80, PTX70>;
+def __nvvm_fmax_xorsign_abs_f16 : NVPTXBuiltinSMAndPTX<"__fp16(__fp16, __fp16)", SM_86, PTX72>;
+def __nvvm_fmax_ftz_xorsign_abs_f16 : NVPTXBuiltinSMAndPTX<"__fp16(__fp16, __fp16)", SM_86, PTX72>;
+def __nvvm_fmax_nan_xorsign_abs_f16 : NVPTXBuiltinSMAndPTX<"__fp16(__fp16, __fp16)", SM_86, PTX72>;
+def __nvvm_fmax_ftz_nan_xorsign_abs_f16 : NVPTXBuiltinSMAndPTX<"__fp16(__fp16, __fp16)", SM_86, PTX72>;
+def __nvvm_fmax_f16x2 : NVPTXBuiltinSMAndPTX<"_Vector<2, __fp16>(_Vector<2, __fp16>, _Vector<2, __fp16>)", SM_80, PTX70>;
+def __nvvm_fmax_ftz_f16x2 : NVPTXBuiltinSMAndPTX<"_Vector<2, __fp16>(_Vector<2, __fp16>, _Vector<2, __fp16>)", SM_80, PTX70>;
+def __nvvm_fmax_nan_f16x2 : NVPTXBuiltinSMAndPTX<"_Vector<2, __fp16>(_Vector<2, __fp16>, _Vector<2, __fp16>)", SM_80, PTX70>;
+def __nvvm_fmax_ftz_nan_f16x2 : NVPTXBuiltinSMAndPTX<"_Vector<2, __fp16>(_Vector<2, __fp16>, _Vector<2, __fp16>)", SM_80, PTX70>;
+def __nvvm_fmax_xorsign_abs_f16x2 : NVPTXBuiltinSMAndPTX<"_Vector<2, __fp16>(_Vector<2, __fp16>, _Vector<2, __fp16>)", SM_86, PTX72>;
+def __nvvm_fmax_ftz_xorsign_abs_f16x2 : NVPTXBuiltinSMAndPTX<"_Vector<2, __fp16>(_Vector<2, __fp16>, _Vector<2, __fp16>)", SM_86, PTX72>;
+def __nvvm_fmax_nan_xorsign_abs_f16x2 : NVPTXBuiltinSMAndPTX<"_Vector<2, __fp16>(_Vector<2, __fp16>, _Vector<2, __fp16>)", SM_86, PTX72>;
+def __nvvm_fmax_ftz_nan_xorsign_abs_f16x2 : NVPTXBuiltinSMAndPTX<"_Vector<2, __fp16>(_Vector<2, __fp16>, _Vector<2, __fp16>)", SM_86, PTX72>;
+def __nvvm_fmax_bf16 : NVPTXBuiltinSMAndPTX<"__bf16(__bf16, __bf16)", SM_80, PTX70>;
+def __nvvm_fmax_ftz_bf16 : NVPTXBuiltinSMAndPTX<"__bf16(__bf16, __bf16)", SM_80, PTX70>;
+def __nvvm_fmax_nan_bf16 : NVPTXBuiltinSMAndPTX<"__bf16(__bf16, __bf16)", SM_80, PTX70>;
+def __nvvm_fmax_ftz_nan_bf16 : NVPTXBuiltinSMAndPTX<"__bf16(__bf16, __bf16)", SM_80, PTX70>;
+def __nvvm_fmax_xorsign_abs_bf16 : NVPTXBuiltinSMAndPTX<"__bf16(__bf16, __bf16)", SM_86, PTX72>;
+def __nvvm_fmax_nan_xorsign_abs_bf16 : NVPTXBuiltinSMAndPTX<"__bf16(__bf16, __bf16)", SM_86, PTX72>;
+def __nvvm_fmax_bf16x2 : NVPTXBuiltinSMAndPTX<"_Vector<2, __bf16>(_Vector<2, __bf16>, _Vector<2, __bf16>)", SM_80, PTX70>;
+def __nvvm_fmax_ftz_bf16x2 : NVPTXBuiltinSMAndPTX<"_Vector<2, __bf16>(_Vector<2, __bf16>, _Vector<2, __bf16>)", SM_80, PTX70>;
+def __nvvm_fmax_nan_bf16x2 : NVPTXBuiltinSMAndPTX<"_Vector<2, __bf16>(_Vector<2, __bf16>, _Vector<2, __bf16>)", SM_80, PTX70>;
+def __nvvm_fmax_ftz_nan_bf16x2 : NVPTXBuiltinSMAndPTX<"_Vector<2, __bf16>(_Vector<2, __bf16>, _Vector<2, __bf16>)", SM_80, PTX70>;
+def __nvvm_fmax_xorsign_abs_bf16x2 : NVPTXBuiltinSMAndPTX<"_Vector<2, __bf16>(_Vector<2, __bf16>, _Vector<2, __bf16>)", SM_86, PTX72>;
+def __nvvm_fmax_nan_xorsign_abs_bf16x2 : NVPTXBuiltinSMAndPTX<"_Vector<2, __bf16>(_Vector<2, __bf16>, _Vector<2, __bf16>)", SM_86, PTX72>;
+def __nvvm_fmax_f : NVPTXBuiltin<"float(float, float)">;
+def __nvvm_fmax_ftz_f : NVPTXBuiltin<"float(float, float)">;
+def __nvvm_fmax_nan_f : NVPTXBuiltinSMAndPTX<"float(float, float)", SM_80, PTX70>;
+def __nvvm_fmax_ftz_nan_f : NVPTXBuiltinSMAndPTX<"float(float, float)", SM_80, PTX70>;
+def __nvvm_fmax_xorsign_abs_f : NVPTXBuiltinSMAndPTX<"float(float, float)", SM_86, PTX72>;
+def __nvvm_fmax_ftz_xorsign_abs_f : NVPTXBuiltinSMAndPTX<"float(float, float)", SM_86, PTX72>;
+def __nvvm_fmax_nan_xorsign_abs_f : NVPTXBuiltinSMAndPTX<"float(float, float)", SM_86, PTX72>;
+def __nvvm_fmax_ftz_nan_xorsign_abs_f : NVPTXBuiltinSMAndPTX<"float(float, float)", SM_86, PTX72>;
+def __nvvm_fmax_d : NVPTXBuiltin<"double(double, double)">;
+
+// Multiplication
+
+def __nvvm_mulhi_i : NVPTXBuiltin<"int(int, int)">;
+def __nvvm_mulhi_ui : NVPTXBuiltin<"unsigned int(unsigned int, unsigned int)">;
+def __nvvm_mulhi_ll : NVPTXBuiltin<"long long int(long long int, long long int)">;
+def __nvvm_mulhi_ull : NVPTXBuiltin<"unsigned long long int(unsigned long long int, unsigned long long int)">;
+
+def __nvvm_mul_rn_ftz_f : NVPTXBuiltin<"float(float, float)">;
+def __nvvm_mul_rn_f : NVPTXBuiltin<"float(float, float)">;
+def __nvvm_mul_rz_ftz_f : NVPTXBuiltin<"float(float, float)">;
+def __nvvm_mul_rz_f : NVPTXBuiltin<"float(float, float)">;
+def __nvvm_mul_rm_ftz_f : NVPTXBuiltin<"float(float, float)">;
+def __nvvm_mul_rm_f : NVPTXBuiltin<"float(float, float)">;
+def __nvvm_mul_rp_ftz_f : NVPTXBuiltin<"float(float, float)">;
+def __nvvm_mul_rp_f : NVPTXBuiltin<"float(float, float)">;
+
+def __nvvm_mul_rn_d : NVPTXBuiltin<"double(double, double)">;
+def __nvvm_mul_rz_d : NVPTXBuiltin<"double(double, double)">;
+def __nvvm_mul_rm_d : NVPTXBuiltin<"double(double, double)">;
+def __nvvm_mul_rp_d : NVPTXBuiltin<"double(double, double)">;
+
+def __nvvm_mul24_i : NVPTXBuiltin<"int(int, int)">;
+def __nvvm_mul24_ui : NVPTXBuiltin<"unsigned int(unsigned int, unsigned int)">;
+
+// Div
+
+def __nvvm_div_approx_ftz_f : NVPTXBuiltin<"float(float, float)">;
+def __nvvm_div_approx_f : NVPTXBuiltin<"float(float, float)">;
+
+def __nvvm_div_rn_ftz_f : NVPTXBuiltin<"float(float, float)">;
+def __nvvm_div_rn_f : NVPTXBuiltin<"float(float, float)">;
+def __nvvm_div_rz_ftz_f : NVPTXBuiltin<"float(float, float)">;
+def __nvvm_div_rz_f : NVPTXBuiltin<"float(float, float)">;
+def __nvvm_div_rm_ftz_f : NVPTXBuiltin<"float(float, float)">;
+def __nvvm_div_rm_f : NVPTXBuiltin<"float(float, float)">;
+def __nvvm_div_rp_ftz_f : NVPTXBuiltin<"float(float, float)">;
+def __nvvm_div_rp_f : NVPTXBuiltin<"float(float, float)">;
+
+def __nvvm_div_rn_d : NVPTXBuiltin<"double(double, double)">;
+def __nvvm_div_rz_d : NVPTXBuiltin<"double(double, double)">;
+def __nvvm_div_rm_d : NVPTXBuiltin<"double(double, double)">;
+def __nvvm_div_rp_d : NVPTXBuiltin<"double(double, double)">;
+
+// Sad
+
+def __nvvm_sad_i : NVPTXBuiltin<"int(int, int, int)">;
+def __nvvm_sad_ui : NVPTXBuiltin<"unsigned int(unsigned int, unsigned int, unsigned int)">;
+
+// Floor, Ceil
+
+def __nvvm_floor_ftz_f : NVPTXBuiltin<"float(float)">;
+def __nvvm_floor_f : NVPTXBuiltin<"float(float)">;
+def __nvvm_floor_d : NVPTXBuiltin<"double(double)">;
+
+def __nvvm_ceil_ftz_f : NVPTXBuiltin<"float(float)">;
+def __nvvm_ceil_f : NVPTXBuiltin<"float(float)">;
+def __nvvm_ceil_d : NVPTXBuiltin<"double(double)">;
+
+// Abs
+
+def __nvvm_fabs_ftz_f : NVPTXBuiltin<"float(float)">;
+def __nvvm_fabs_f : NVPTXBuiltin<"float(float)">;
+def __nvvm_fabs_d : NVPTXBuiltin<"double(double)">;
+
+// Round
+
+def __nvvm_round_ftz_f : NVPTXBuiltin<"float(float)">;
+def __nvvm_round_f : NVPTXBuiltin<"float(float)">;
+def __nvvm_round_d : NVPTXBuiltin<"double(double)">;
+
+// Trunc
+
+def __nvvm_trunc_ftz_f : NVPTXBuiltin<"float(float)">;
+def __nvvm_trunc_f : NVPTXBuiltin<"float(float)">;
+def __nvvm_trunc_d : NVPTXBuiltin<"double(double)">;
+
+// Saturate
+
+def __nvvm_saturate_ftz_f : NVPTXBuiltin<"float(float)">;
+def __nvvm_saturate_f : NVPTXBuiltin<"float(float)">;
+def __nvvm_saturate_d : NVPTXBuiltin<"double(double)">;
+
+// Exp2, Log2
+
+def __nvvm_ex2_approx_ftz_f : NVPTXBuiltin<"float(float)">;
+def __nvvm_ex2_approx_f : NVPTXBuiltin<"float(float)">;
+def __nvvm_ex2_approx_d : NVPTXBuiltin<"double(double)">;
+def __nvvm_ex2_approx_f16 : NVPTXBuiltinSMAndPTX<"__fp16(__fp16)", SM_75, PTX70>;
+def __nvvm_ex2_approx_f16x2 : NVPTXBuiltinSMAndPTX<"_Vector<2, __fp16>(_Vector<2, __fp16>)", SM_75, PTX70>;
+
+def __nvvm_lg2_approx_ftz_f : NVPTXBuiltin<"float(float)">;
+def __nvvm_lg2_approx_f : NVPTXBuiltin<"float(float)">;
+def __nvvm_lg2_approx_d : NVPTXBuiltin<"double(double)">;
+
+// Sin, Cos
+
+def __nvvm_sin_approx_ftz_f : NVPTXBuiltin<"float(float)">;
+def __nvvm_sin_approx_f : NVPTXBuiltin<"float(float)">;
+
+def __nvvm_cos_approx_ftz_f : NVPTXBuiltin<"float(float)">;
+def __nvvm_cos_approx_f : NVPTXBuiltin<"float(float)">;
+
+// Fma
+
+def __nvvm_fma_rn_f16 : NVPTXBuiltinSMAndPTX<"__fp16(__fp16, __fp16, __fp16)", SM_53, PTX42>;
+def __nvvm_fma_rn_ftz_f16 : NVPTXBuiltinSMAndPTX<"__fp16(__fp16, __fp16, __fp16)", SM_53, PTX42>;
+def __nvvm_fma_rn_sat_f16 : NVPTXBuiltinSMAndPTX<"__fp16(__fp16, __fp16, __fp16)", SM_53, PTX42>;
+def __nvvm_fma_rn_ftz_sat_f16 : NVPTXBuiltinSMAndPTX<"__fp16(__fp16, __fp16, __fp16)", SM_53, PTX42>;
+def __nvvm_fma_rn_relu_f16 : NVPTXBuiltinSMAndPTX<"__fp16(__fp16, __fp16, __fp16)", SM_80, PTX70>;
+def __nvvm_fma_rn_ftz_relu_f16 : NVPTXBuiltinSMAndPTX<"__fp16(__fp16, __fp16, __fp16)", SM_80, PTX70>;
+def __nvvm_fma_rn_f16x2 : NVPTXBuiltinSMAndPTX<"_Vector<2, __fp16>(_Vector<2, __fp16>, _Vector<2, __fp16>, _Vector<2, __fp16>)", SM_53, PTX42>;
+def __nvvm_fma_rn_ftz_f16x2 : NVPTXBuiltinSMAndPTX<"_Vector<2, __fp16>(_Vector<2, __fp16>, _Vector<2, __fp16>, _Vector<2, __fp16>)", SM_53, PTX42>;
+def __nvvm_fma_rn_sat_f16x2 : NVPTXBuiltinSMAndPTX<"_Vector<2, __fp16>(_Vector<2, __fp16>, _Vector<2, __fp16>, _Vector<2, __fp16>)", SM_53, PTX42>;
+def __nvvm_fma_rn_ftz_sat_f16x2 : NVPTXBuiltinSMAndPTX<"_Vector<2, __fp16>(_Vector<2, __fp16>, _Vector<2, __fp16>, _Vector<2, __fp16>)", SM_53, PTX42>;
+def __nvvm_fma_rn_relu_f16x2 : NVPTXBuiltinSMAndPTX<"_Vector<2, __fp16>(_Vector<2, __fp16>, _Vector<2, __fp16>, _Vector<2, __fp16>)", SM_80, PTX70>;
+def __nvvm_fma_rn_ftz_relu_f16x2 : NVPTXBuiltinSMAndPTX<"_Vector<2, __fp16>(_Vector<2, __fp16>, _Vector<2, __fp16>, _Vector<2, __fp16>)", SM_80, PTX70>;
+def __nvvm_fma_rn_bf16 : NVPTXBuiltinSMAndPTX<"__bf16(__bf16, __bf16, __bf16)", SM_80, PTX70>;
+def __nvvm_fma_rn_relu_bf16 : NVPTXBuiltinSMAndPTX<"__bf16(__bf16, __bf16, __bf16)", SM_80, PTX70>;
+def __nvvm_fma_rn_bf16x2 : NVPTXBuiltinSMAndPTX<"_Vector<2, __bf16>(_Vector<2, __bf16>, _Vector<2, __bf16>, _Vector<2, __bf16>)", SM_80, PTX70>;
+def __nvvm_fma_rn_relu_bf16x2 : NVPTXBuiltinSMAndPTX<"_Vector<2, __bf16>(_Vector<2, __bf16>, _Vector<2, __bf16>, _Vector<2, __bf16>)", SM_80, PTX70>;
+def __nvvm_fma_rn_ftz_f : NVPTXBuiltin<"float(float, float, float)">;
+def __nvvm_fma_rn_f : NVPTXBuiltin<"float(float, float, float)">;
+def __nvvm_fma_rz_ftz_f : NVPTXBuiltin<"float(float, float, float)">;
+def __nvvm_fma_rz_f : NVPTXBuiltin<"float(float, float, float)">;
+def __nvvm_fma_rm_ftz_f : NVPTXBuiltin<"float(float, float, float)">;
+def __nvvm_fma_rm_f : NVPTXBuiltin<"float(float, float, float)">;
+def __nvvm_fma_rp_ftz_f : NVPTXBuiltin<"float(float, float, float)">;
+def __nvvm_fma_rp_f : NVPTXBuiltin<"float(float, float, float)">;
+def __nvvm_fma_rn_d : NVPTXBuiltin<"double(double, double, double)">;
+def __nvvm_fma_rz_d : NVPTXBuiltin<"double(double, double, double)">;
+def __nvvm_fma_rm_d : NVPTXBuiltin<"double(double, double, double)">;
+def __nvvm_fma_rp_d : NVPTXBuiltin<"double(double, double, double)">;
+
+// Rcp
+
+def __nvvm_rcp_rn_ftz_f : NVPTXBuiltin<"float(float)">;
+def __nvvm_rcp_rn_f : NVPTXBuiltin<"float(float)">;
+def __nvvm_rcp_rz_ftz_f : NVPTXBuiltin<"float(float)">;
+def __nvvm_rcp_rz_f : NVPTXBuiltin<"float(float)">;
+def __nvvm_rcp_rm_ftz_f : NVPTXBuiltin<"float(float)">;
+def __nvvm_rcp_rm_f : NVPTXBuiltin<"float(float)">;
+def __nvvm_rcp_rp_ftz_f : NVPTXBuiltin<"float(float)">;
+def __nvvm_rcp_rp_f : NVPTXBuiltin<"float(float)">;
+
+def __nvvm_rcp_rn_d : NVPTXBuiltin<"double(double)">;
+def __nvvm_rcp_rz_d : NVPTXBuiltin<"double(double)">;
+def __nvvm_rcp_rm_d : NVPTXBuiltin<"double(double)">;
+def __nvvm_rcp_rp_d : NVPTXBuiltin<"double(double)">;
+
+def __nvvm_rcp_approx_ftz_f : NVPTXBuiltin<"float(float)">;
+def __nvvm_rcp_approx_ftz_d : NVPTXBuiltin<"double(double)">;
+
+// Sqrt
+
+def __nvvm_sqrt_rn_ftz_f : NVPTXBuiltin<"float(float)">;
+def __nvvm_sqrt_rn_f : NVPTXBuiltin<"float(float)">;
+def __nvvm_sqrt_rz_ftz_f : NVPTXBuiltin<"float(float)">;
+def __nvvm_sqrt_rz_f : NVPTXBuiltin<"float(float)">;
+def __nvvm_sqrt_rm_ftz_f : NVPTXBuiltin<"float(float)">;
+def __nvvm_sqrt_rm_f : NVPTXBuiltin<"float(float)">;
+def __nvvm_sqrt_rp_ftz_f : NVPTXBuiltin<"float(float)">;
+def __nvvm_sqrt_rp_f : NVPTXBuiltin<"float(float)">;
+def __nvvm_sqrt_approx_ftz_f : NVPTXBuiltin<"float(float)">;
+def __nvvm_sqrt_approx_f : NVPTXBuiltin<"float(float)">;
+
+def __nvvm_sqrt_rn_d : NVPTXBuiltin<"double(double)">;
+def __nvvm_sqrt_rz_d : NVPTXBuiltin<"double(double)">;
+def __nvvm_sqrt_rm_d : NVPTXBuiltin<"double(double)">;
+def __nvvm_sqrt_rp_d : NVPTXBuiltin<"double(double)">;
+
+// Rsqrt
+
+def __nvvm_rsqrt_approx_ftz_f : NVPTXBuiltin<"float(float)">;
+def __nvvm_rsqrt_approx_f : NVPTXBuiltin<"float(float)">;
+def __nvvm_rsqrt_approx_d : NVPTXBuiltin<"double(double)">;
+
+// Add
+
+def __nvvm_add_rn_ftz_f : NVPTXBuiltin<"float(float, float)">;
+def __nvvm_add_rn_f : NVPTXBuiltin<"float(float, float)">;
+def __nvvm_add_rz_ftz_f : NVPTXBuiltin<"float(float, float)">;
+def __nvvm_add_rz_f : NVPTXBuiltin<"float(float, float)">;
+def __nvvm_add_rm_ftz_f : NVPTXBuiltin<"float(float, float)">;
+def __nvvm_add_rm_f : NVPTXBuiltin<"float(float, float)">;
+def __nvvm_add_rp_ftz_f : NVPTXBuiltin<"float(float, float)">;
+def __nvvm_add_rp_f : NVPTXBuiltin<"float(float, float)">;
+
+def __nvvm_add_rn_d : NVPTXBuiltin<"double(double, double)">;
+def __nvvm_add_rz_d : NVPTXBuiltin<"double(double, double)">;
+def __nvvm_add_rm_d : NVPTXBuiltin<"double(double, double)">;
+def __nvvm_add_rp_d : NVPTXBuiltin<"double(double, double)">;
+
+// Convert
+
+def __nvvm_d2f_rn_ftz : NVPTXBuiltin<"float(double)">;
+def __nvvm_d2f_rn : NVPTXBuiltin<"float(double)">;
+def __nvvm_d2f_rz_ftz : NVPTXBuiltin<"float(double)">;
+def __nvvm_d2f_rz : NVPTXBuiltin<"float(double)">;
+def __nvvm_d2f_rm_ftz : NVPTXBuiltin<"float(double)">;
+def __nvvm_d2f_rm : NVPTXBuiltin<"float(double)">;
+def __nvvm_d2f_rp_ftz : NVPTXBuiltin<"float(double)">;
+def __nvvm_d2f_rp : NVPTXBuiltin<"float(double)">;
+
+def __nvvm_d2i_rn : NVPTXBuiltin<"int(double)">;
+def __nvvm_d2i_rz : NVPTXBuiltin<"int(double)">;
+def __nvvm_d2i_rm : NVPTXBuiltin<"int(double)">;
+def __nvvm_d2i_rp : NVPTXBuiltin<"int(double)">;
+
+def __nvvm_d2ui_rn : NVPTXBuiltin<"unsigned int(double)">;
+def __nvvm_d2ui_rz : NVPTXBuiltin<"unsigned int(double)">;
+def __nvvm_d2ui_rm : NVPTXBuiltin<"unsigned int(double)">;
+def __nvvm_d2ui_rp : NVPTXBuiltin<"unsigned int(double)">;
+
+def __nvvm_i2d_rn : NVPTXBuiltin<"double(int)">;
+def __nvvm_i2d_rz : NVPTXBuiltin<"double(int)">;
+def __nvvm_i2d_rm : NVPTXBuiltin<"double(int)">;
+def __nvvm_i2d_rp : NVPTXBuiltin<"double(int)">;
+
+def __nvvm_ui2d_rn : NVPTXBuiltin<"double(unsigned int)">;
+def __nvvm_ui2d_rz : NVPTXBuiltin<"double(unsigned int)">;
+def __nvvm_ui2d_rm : NVPTXBuiltin<"double(unsigned int)">;
+def __nvvm_ui2d_rp : NVPTXBuiltin<"double(unsigned int)">;
+
+def __nvvm_f2i_rn_ftz : NVPTXBuiltin<"int(float)">;
+def __nvvm_f2i_rn : NVPTXBuiltin<"int(float)">;
+def __nvvm_f2i_rz_ftz : NVPTXBuiltin<"int(float)">;
+def __nvvm_f2i_rz : NVPTXBuiltin<"int(float)">;
+def __nvvm_f2i_rm_ftz : NVPTXBuiltin<"int(float)">;
+def __nvvm_f2i_rm : NVPTXBuiltin<"int(float)">;
+def __nvvm_f2i_rp_ftz : NVPTXBuiltin<"int(float)">;
+def __nvvm_f2i_rp : NVPTXBuiltin<"int(float)">;
+
+def __nvvm_f2ui_rn_ftz : NVPTXBuiltin<"unsigned int(float)">;
+def __nvvm_f2ui_rn : NVPTXBuiltin<"unsigned int(float)">;
+def __nvvm_f2ui_rz_ftz : NVPTXBuiltin<"unsigned int(float)">;
+def __nvvm_f2ui_rz : NVPTXBuiltin<"unsigned int(float)">;
+def __nvvm_f2ui_rm_ftz : NVPTXBuiltin<"unsigned int(float)">;
+def __nvvm_f2ui_rm : NVPTXBuiltin<"unsigned int(float)">;
+def __nvvm_f2ui_rp_ftz : NVPTXBuiltin<"unsigned int(float)">;
+def __nvvm_f2ui_rp : NVPTXBuiltin<"unsigned int(float)">;
+
+def __nvvm_i2f_rn : NVPTXBuiltin<"float(int)">;
+def __nvvm_i2f_rz : NVPTXBuiltin<"float(int)">;
+def __nvvm_i2f_rm : NVPTXBuiltin<"float(int)">;
+def __nvvm_i2f_rp : NVPTXBuiltin<"float(int)">;
+
+def __nvvm_ui2f_rn : NVPTXBuiltin<"float(unsigned int)">;
+def __nvvm_ui2f_rz : NVPTXBuiltin<"float(unsigned int)">;
+def __nvvm_ui2f_rm : NVPTXBuiltin<"float(unsigned int)">;
+def __nvvm_ui2f_rp : NVPTXBuiltin<"float(unsigned int)">;
+
+def __nvvm_lohi_i2d : NVPTXBuiltin<"double(int, int)">;
+
+def __nvvm_d2i_lo : NVPTXBuiltin<"int(double)">;
+def __nvvm_d2i_hi : NVPTXBuiltin<"int(double)">;
+
+def __nvvm_f2ll_rn_ftz : NVPTXBuiltin<"long long int(float)">;
+def __nvvm_f2ll_rn : NVPTXBuiltin<"long long int(float)">;
+def __nvvm_f2ll_rz_ftz : NVPTXBuiltin<"long long int(float)">;
+def __nvvm_f2ll_rz : NVPTXBuiltin<"long long int(float)">;
+def __nvvm_f2ll_rm_ftz : NVPTXBuiltin<"long long int(float)">;
+def __nvvm_f2ll_rm : NVPTXBuiltin<"long long int(float)">;
+def __nvvm_f2ll_rp_ftz : NVPTXBuiltin<"long long int(float)">;
+def __nvvm_f2ll_rp : NVPTXBuiltin<"long long int(float)">;
+
+def __nvvm_f2ull_rn_ftz : NVPTXBuiltin<"unsigned long long int(float)">;
+def __nvvm_f2ull_rn : NVPTXBuiltin<"unsigned long long int(float)">;
+def __nvvm_f2ull_rz_ftz : NVPTXBuiltin<"unsigned long long int(float)">;
+def __nvvm_f2ull_rz : NVPTXBuiltin<"unsigned long long int(float)">;
+def __nvvm_f2ull_rm_ftz : NVPTXBuiltin<"unsigned long long int(float)">;
+def __nvvm_f2ull_rm : NVPTXBuiltin<"unsigned long long int(float)">;
+def __nvvm_f2ull_rp_ftz : NVPTXBuiltin<"unsigned long long int(float)">;
+def __nvvm_f2ull_rp : NVPTXBuiltin<"unsigned long long int(float)">;
+
+def __nvvm_d2ll_rn : NVPTXBuiltin<"long long int(double)">;
+def __nvvm_d2ll_rz : NVPTXBuiltin<"long long int(double)">;
+def __nvvm_d2ll_rm : NVPTXBuiltin<"long long int(double)">;
+def __nvvm_d2ll_rp : NVPTXBuiltin<"long long int(double)">;
+
+def __nvvm_d2ull_rn : NVPTXBuiltin<"unsigned long long int(double)">;
+def __nvvm_d2ull_rz : NVPTXBuiltin<"unsigned long long int(double)">;
+def __nvvm_d2ull_rm : NVPTXBuiltin<"unsigned long long int(double)">;
+def __nvvm_d2ull_rp : NVPTXBuiltin<"unsigned long long int(double)">;
+
+def __nvvm_ll2f_rn : NVPTXBuiltin<"float(long long int)">;
+def __nvvm_ll2f_rz : NVPTXBuiltin<"float(long long int)">;
+def __nvvm_ll2f_rm : NVPTXBuiltin<"float(long long int)">;
+def __nvvm_ll2f_rp : NVPTXBuiltin<"float(long long int)">;
+
+def __nvvm_ull2f_rn : NVPTXBuiltin<"float(unsigned long long int)">;
+def __nvvm_ull2f_rz : NVPTXBuiltin<"float(unsigned long long int)">;
+def __nvvm_ull2f_rm : NVPTXBuiltin<"float(unsigned long long int)">;
+def __nvvm_ull2f_rp : NVPTXBuiltin<"float(unsigned long long int)">;
+
+def __nvvm_ll2d_rn : NVPTXBuiltin<"double(long long int)">;
+def __nvvm_ll2d_rz : NVPTXBuiltin<"double(long long int)">;
+def __nvvm_ll2d_rm : NVPTXBuiltin<"double(long long int)">;
+def __nvvm_ll2d_rp : NVPTXBuiltin<"double(long long int)">;
+
+def __nvvm_ull2d_rn : NVPTXBuiltin<"double(unsigned long long int)">;
+def __nvvm_ull2d_rz : NVPTXBuiltin<"double(unsigned long long int)">;
+def __nvvm_ull2d_rm : NVPTXBuiltin<"double(unsigned long long int)">;
+def __nvvm_ull2d_rp : NVPTXBuiltin<"double(unsigned long long int)">;
+
+def __nvvm_f2h_rn_ftz : NVPTXBuiltin<"unsigned short(float)">;
+def __nvvm_f2h_rn : NVPTXBuiltin<"unsigned short(float)">;
+
+def __nvvm_ff2bf16x2_rn : NVPTXBuiltinSMAndPTX<"_Vector<2, __bf16>(float, float)", SM_80, PTX70>;
+def __nvvm_ff2bf16x2_rn_relu : NVPTXBuiltinSMAndPTX<"_Vector<2, __bf16>(float, float)", SM_80, PTX70>;
+def __nvvm_ff2bf16x2_rz : NVPTXBuiltinSMAndPTX<"_Vector<2, __bf16>(float, float)", SM_80, PTX70>;
+def __nvvm_ff2bf16x2_rz_relu : NVPTXBuiltinSMAndPTX<"_Vector<2, __bf16>(float, float)", SM_80, PTX70>;
+
+def __nvvm_ff2f16x2_rn : NVPTXBuiltinSMAndPTX<"_Vector<2, __fp16>(float, float)", SM_80, PTX70>;
+def __nvvm_ff2f16x2_rn_relu : NVPTXBuiltinSMAndPTX<"_Vector<2, __fp16>(float, float)", SM_80, PTX70>;
+def __nvvm_ff2f16x2_rz : NVPTXBuiltinSMAndPTX<"_Vector<2, __fp16>(float, float)", SM_80, PTX70>;
+def __nvvm_ff2f16x2_rz_relu : NVPTXBuiltinSMAndPTX<"_Vector<2, __fp16>(float, float)", SM_80, PTX70>;
+
+def __nvvm_f2bf16_rn : NVPTXBuiltinSMAndPTX<"__bf16(float)", SM_80, PTX70>;
+def __nvvm_f2bf16_rn_relu : NVPTXBuiltinSMAndPTX<"__bf16(float)", SM_80, PTX70>;
+def __nvvm_f2bf16_rz : NVPTXBuiltinSMAndPTX<"__bf16(float)", SM_80, PTX70>;
+def __nvvm_f2bf16_rz_relu : NVPTXBuiltinSMAndPTX<"__bf16(float)", SM_80, PTX70>;
+
+def __nvvm_f2tf32_rna : NVPTXBuiltinSMAndPTX<"int32_t(float)", SM_80, PTX70>;
+
+def __nvvm_ff_to_e4m3x2_rn : NVPTXBuiltinSMAndPTX<"short(float, float)", SM_89, PTX81>;
+def __nvvm_ff_to_e4m3x2_rn_relu : NVPTXBuiltinSMAndPTX<"short(float, float)", SM_89, PTX81>;
+def __nvvm_ff_to_e5m2x2_rn : NVPTXBuiltinSMAndPTX<"short(float, float)", SM_89, PTX81>;
+def __nvvm_ff_to_e5m2x2_rn_relu : NVPTXBuiltinSMAndPTX<"short(float, float)", SM_89, PTX81>;
+
+def __nvvm_f16x2_to_e4m3x2_rn : NVPTXBuiltinSMAndPTX<"short(_Vector<2, __fp16>)", SM_89, PTX81>;
+def __nvvm_f16x2_to_e4m3x2_rn_relu : NVPTXBuiltinSMAndPTX<"short(_Vector<2, __fp16>)", SM_89, PTX81>;
+def __nvvm_f16x2_to_e5m2x2_rn : NVPTXBuiltinSMAndPTX<"short(_Vector<2, __fp16>)", SM_89, PTX81>;
+def __nvvm_f16x2_to_e5m2x2_rn_relu : NVPTXBuiltinSMAndPTX<"short(_Vector<2, __fp16>)", SM_89, PTX81>;
+
+def __nvvm_e4m3x2_to_f16x2_rn : NVPTXBuiltinSMAndPTX<"_Vector<2, __fp16>(short)", SM_89, PTX81>;
+def __nvvm_e4m3x2_to_f16x2_rn_relu : NVPTXBuiltinSMAndPTX<"_Vector<2, __fp16>(short)", SM_89, PTX81>;
+def __nvvm_e5m2x2_to_f16x2_rn : NVPTXBuiltinSMAndPTX<"_Vector<2, __fp16>(short)", SM_89, PTX81>;
+def __nvvm_e5m2x2_to_f16x2_rn_relu : NVPTXBuiltinSMAndPTX<"_Vector<2, __fp16>(short)", SM_89, PTX81>;
+
+// FNS
+let Attributes = [NoThrow] in {
+  def __nvvm_fns : NVPTXBuiltinPTX<"unsigned int(unsigned int, unsigned int, int)", PTX60>;
+}
+
+// Sync
+
+def __syncthreads : NVPTXBuiltin<"void()">;
+def __nvvm_bar0_popc : NVPTXBuiltin<"int(int)">;
+def __nvvm_bar0_and : NVPTXBuiltin<"int(int)">;
+def __nvvm_bar0_or : NVPTXBuiltin<"int(int)">;
+let Attributes = [NoThrow] in {
+  def __nvvm_bar_sync : NVPTXBuiltin<"void(int)">;
+  def __nvvm_bar_warp_sync : NVPTXBuiltinPTX<"void(unsigned int)", PTX60>;
+  def __nvvm_barrier_sync : NVPTXBuiltinPTX<"void(unsigned int)", PTX60>;
+  def __nvvm_barrier_sync_cnt : NVPTXBuiltinPTX<"void(unsigned int, unsigned int)", PTX60>;
+
+  def __nvvm_barrier_cluster_arrive : NVPTXBuiltinSMAndPTX<"void()", SM_90, PTX78>;
+  def __nvvm_barrier_cluster_arrive_relaxed : NVPTXBuiltinSMAndPTX<"void()", SM_90, PTX80>;
+  def __nvvm_barrier_cluster_wait : NVPTXBuiltinSMAndPTX<"void()", SM_90, PTX78>;
+  def __nvvm_fence_sc_cluster : NVPTXBuiltinSMAndPTX<"void()", SM_90, PTX78>;
+}
+
+// Shuffle
+
+def __nvvm_shfl_down_i32 : NVPTXBuiltin<"int(int, int, int)">;
+def __nvvm_shfl_down_f32 : NVPTXBuiltin<"float(float, int, int)">;
+def __nvvm_shfl_up_i32 : NVPTXBuiltin<"int(int, int, int)">;
+def __nvvm_shfl_up_f32 : NVPTXBuiltin<"float(float, int, int)">;
+def __nvvm_shfl_bfly_i32 : NVPTXBuiltin<"int(int, int, int)">;
+def __nvvm_shfl_bfly_f32 : NVPTXBuiltin<"float(float, int, int)">;
+def __nvvm_shfl_idx_i32 : NVPTXBuiltin<"int(int, int, int)">;
+def __nvvm_shfl_idx_f32 : NVPTXBuiltin<"float(float, int, int)">;
+
+def __nvvm_shfl_sync_down_i32 : NVPTXBuiltinPTX<"int(unsigned int, int, int, int)", PTX60>;
+def __nvvm_shfl_sync_down_f32 : NVPTXBuiltinPTX<"float(unsigned int, float, int, int)", PTX60>;
+def __nvvm_shfl_sync_up_i32 : NVPTXBuiltinPTX<"int(unsigned int, int, int, int)", PTX60>;
+def __nvvm_shfl_sync_up_f32 : NVPTXBuiltinPTX<"float(unsigned int, float, int, int)", PTX60>;
+def __nvvm_shfl_sync_bfly_i32 : NVPTXBuiltinPTX<"int(unsigned int, int, int, int)", PTX60>;
+def __nvvm_shfl_sync_bfly_f32 : NVPTXBuiltinPTX<"float(unsigned int, float, int, int)", PTX60>;
+def __nvvm_shfl_sync_idx_i32 : NVPTXBuiltinPTX<"int(unsigned int, int, int, int)", PTX60>;
+def __nvvm_shfl_sync_idx_f32 : NVPTXBuiltinPTX<"float(unsigned int, float, int, int)", PTX60>;
+
+// Vote
+def __nvvm_vote_all : NVPTXBuiltin<"bool(bool)">;
+def __nvvm_vote_any : NVPTXBuiltin<"bool(bool)">;
+def __nvvm_vote_uni : NVPTXBuiltin<"bool(bool)">;
+def __nvvm_vote_ballot : NVPTXBuiltin<"unsigned int(bool)">;
+
+def __nvvm_vote_all_sync : NVPTXBuiltinPTX<"bool(unsigned int, bool)", PTX60>;
+def __nvvm_vote_any_sync : NVPTXBuiltinPTX<"bool(unsigned int, bool)", PTX60>;
+def __nvvm_vote_uni_sync : NVPTXBuiltinPTX<"bool(unsigned int, bool)", PTX60>;
+def __nvvm_vote_ballot_sync : NVPTXBuiltinPTX<"unsigned int(unsigned int, bool)", PTX60>;
+
+// Mask
+let Attributes = [NoThrow] in {
+  def __nvvm_activemask : NVPTXBuiltinPTX<"unsigned int()", PTX62>;
+}
+
+// Match
+def __nvvm_match_any_sync_i32 : NVPTXBuiltinSMAndPTX<"unsigned int(unsigned int, unsigned int)", SM_70, PTX60>;
+def __nvvm_match_any_sync_i64 : NVPTXBuiltinSMAndPTX<"unsigned int(unsigned int, int64_t)", SM_70, PTX60>;
+// These return a pair {value, predicate}, which requires custom lowering.
+def __nvvm_match_all_sync_i32p : NVPTXBuiltinSMAndPTX<"unsigned int(unsigned int, unsigned int, int *)", SM_70, PTX60>;
+def __nvvm_match_all_sync_i64p : NVPTXBuiltinSMAndPTX<"unsigned int(unsigned int, int64_t, int *)", SM_70, PTX60>;
+
+// Redux
+def __nvvm_redux_sync_add : NVPTXBuiltinSMAndPTX<"int(int, int)", SM_80, PTX70>;
+def __nvvm_redux_sync_min : NVPTXBuiltinSMAndPTX<"int(int, int)", SM_80, PTX70>;
+def __nvvm_redux_sync_max : NVPTXBuiltinSMAndPTX<"int(int, int)", SM_80, PTX70>;
+def __nvvm_redux_sync_umin : NVPTXBuiltinSMAndPTX<"unsigned int(unsigned int, int)", SM_80, PTX70>;
+def __nvvm_redux_sync_umax : NVPTXBuiltinSMAndPTX<"unsigned int(unsigned int, int)", SM_80, PTX70>;
+def __nvvm_redux_sync_and : NVPTXBuiltinSMAndPTX<"int(int, int)", SM_80, PTX70>;
+def __nvvm_redux_sync_xor : NVPTXBuiltinSMAndPTX<"int(int, int)", SM_80, PTX70>;
+def __nvvm_redux_sync_or : NVPTXBuiltinSMAndPTX<"int(int, int)", SM_80, PTX70>;
+
+// Membar
+
+def __nvvm_membar_cta : NVPTXBuiltin<"void()">;
+def __nvvm_membar_gl : NVPTXBuiltin<"void()">;
+def __nvvm_membar_sys : NVPTXBuiltin<"void()">;
+
+// mbarrier
+
+def __nvvm_mbarrier_init : NVPTXBuiltinSMAndPTX<"void(int64_t *, int)", SM_80, PTX70>;
+def __nvvm_mbarrier_init_shared : NVPTXBuiltinSMAndPTX<"void(int64_t address_space<3> *, int)", SM_80, PTX70>;
+
+def __nvvm_mbarrier_inval : NVPTXBuiltinSMAndPTX<"void(int64_t *)", SM_80, PTX70>;
+def __nvvm_mbarrier_inval_shared : NVPTXBuiltinSMAndPTX<"void(int64_t address_space<3> *)", SM_80, PTX70>;
+
+def __nvvm_mbarrier_arrive : NVPTXBuiltinSMAndPTX<"int64_t(int64_t *)", SM_80, PTX70>;
+def __nvvm_mbarrier_arrive_shared : NVPTXBuiltinSMAndPTX<"int64_t(int64_t address_space<3> *)", SM_80, PTX70>;
+def __nvvm_mbarrier_arrive_noComplete : NVPTXBuiltinSMAndPTX<"int64_t(int64_t *, int)", SM_80, PTX70>;
+def __nvvm_mbarrier_arrive_noComplete_shared : NVPTXBuiltinSMAndPTX<"int64_t(int64_t address_space<3> *, int)", SM_80, PTX70>;
+
+def __nvvm_mbarrier_arrive_drop : NVPTXBuiltinSMAndPTX<"int64_t(int64_t *)", SM_80, PTX70>;
+def __nvvm_mbarrier_arrive_drop_shared : NVPTXBuiltinSMAndPTX<"int64_t(int64_t address_space<3> *)", SM_80, PTX70>;
+def __nvvm_mbarrier_arrive_drop_noComplete : NVPTXBuiltinSMAndPTX<"int64_t(int64_t *, int)", SM_80, PTX70>;
+def __nvvm_mbarrier_arrive_drop_noComplete_shared : NVPTXBuiltinSMAndPTX<"int64_t(int64_t address_space<3> *, int)", SM_80, PTX70>;
+
+def __nvvm_mbarrier_test_wait : NVPTXBuiltinSMAndPTX<"bool(int64_t *, int64_t)", SM_80, PTX70>;
+def __nvvm_mbarrier_test_wait_shared : NVPTXBuiltinSMAndPTX<"bool(int64_t address_space<3> *, int64_t)", SM_80, PTX70>;
+
+def __nvvm_mbarrier_pending_count : NVPTXBuiltinSMAndPTX<"int(int64_t)", SM_80, PTX70>;
+
+// Memcpy, Memset
+
+def __nvvm_memcpy : NVPTXBuiltin<"void(unsigned char *, unsigned char *, size_t, int)">;
+def __nvvm_memset : NVPTXBuiltin<"void(unsigned char *, unsigned char, size_t, int)">;
+
+// Image
+
+def __builtin_ptx_read_image2Dfi_ : NVPTXBuiltin<"_Vector<4, float>(int, int, int, int)">;
+def __builtin_ptx_read_image2Dff_ : NVPTXBuiltin<"_Vector<4, float>(int, int, float, float)">;
+def __builtin_ptx_read_image2Dii_ : NVPTXBuiltin<"_Vector<4, int>(int, int, int, int)">;
+def __builtin_ptx_read_image2Dif_ : NVPTXBuiltin<"_Vector<4, int>(int, int, float, float)">;
+
+def __builtin_ptx_read_image3Dfi_ : NVPTXBuiltin<"_Vector<4, float>(int, int, int, int, int, int)">;
+def __builtin_ptx_read_image3Dff_ : NVPTXBuiltin<"_Vector<4, float>(int, int, float, float, float, float)">;
+def __builtin_ptx_read_image3Dii_ : NVPTXBuiltin<"_Vector<4, int>(int, int, int, int, int, int)">;
+def __builtin_ptx_read_image3Dif_ : NVPTXBuiltin<"_Vector<4, int>(int, int, float, float, float, float)">;
+
+def __builtin_ptx_write_image2Df_ : NVPTXBuiltin<"void(int, int, int, float, float, float, float)">;
+def __builtin_ptx_write_image2Di_ : NVPTXBuiltin<"void(int, int, int, int, int, int, int)">;
+def __builtin_ptx_write_image2Dui_ : NVPTXBuiltin<"void(int, int, int, unsigned int, unsigned int, unsigned int, unsigned int)">;
+def __builtin_ptx_get_image_depthi_ : NVPTXBuiltin<"int(int)">;
+def __builtin_ptx_get_image_heighti_ : NVPTXBuiltin<"int(int)">;
+def __builtin_ptx_get_image_widthi_ : NVPTXBuiltin<"int(int)">;
+def __builtin_ptx_get_image_channel_data_typei_ : NVPTXBuiltin<"int(int)">;
+def __builtin_ptx_get_image_channel_orderi_ : NVPTXBuiltin<"int(int)">;
+
+// Atomic
+//
+// We need the atom intrinsics because
+// - they are used in converging analysis
+// - they are used in address space analysis and optimization
+// So it does not hurt to expose them as builtins.
+//
+let Attributes = [NoThrow] in {
+  def __nvvm_atom_add_gen_i : NVPTXBuiltin<"int(int volatile *, int)">;
+  def __nvvm_atom_cta_add_gen_i : NVPTXBuiltinSM<"int(int volatile *, int)", SM_60>;
+  def __nvvm_atom_sys_add_gen_i : NVPTXBuiltinSM<"int(int volatile *, int)", SM_60>;
+  def __nvvm_atom_add_gen_l : NVPTXBuiltin<"long int(long int volatile *, long int)">;
+  def __nvvm_atom_cta_add_gen_l : NVPTXBuiltinSM<"long int(long int volatile *, long int)", SM_60>;
+  def __nvvm_atom_sys_add_gen_l : NVPTXBuiltinSM<"long int(long int volatile *, long int)", SM_60>;
+  def __nvvm_atom_add_gen_ll : NVPTXBuiltin<"long long int(long long int volatile *, long long int)">;
+  def __nvvm_atom_cta_add_gen_ll : NVPTXBuiltinSM<"long long int(long long int volatile *, long long int)", SM_60>;
+  def __nvvm_atom_sys_add_gen_ll : NVPTXBuiltinSM<"long long int(long long int volatile *, long long int)", SM_60>;
+  def __nvvm_atom_add_gen_f : NVPTXBuiltin<"float(float volatile *, float)">;
+  def __nvvm_atom_cta_add_gen_f : NVPTXBuiltinSM<"float(float volatile *, float)", SM_60>;
+  def __nvvm_atom_sys_add_gen_f : NVPTXBuiltinSM<"float(float volatile *, float)", SM_60>;
+  def __nvvm_atom_add_gen_d : NVPTXBuiltinSM<"double(double volatile *, double)", SM_60>;
+  def __nvvm_atom_cta_add_gen_d : NVPTXBuiltinSM<"double(double volatile *, double)", SM_60>;
+  def __nvvm_atom_sys_add_gen_d : NVPTXBuiltinSM<"double(double volatile *, double)", SM_60>;
+
+  def __nvvm_atom_sub_gen_i : NVPTXBuiltin<"int(int volatile *, int)">;
+  def __nvvm_atom_sub_gen_l : NVPTXBuiltin<"long int(long int volatile *, long int)">;
+  def __nvvm_atom_sub_gen_ll : NVPTXBuiltin<"long long int(long long int volatile *, long long int)">;
+
+  def __nvvm_atom_xchg_gen_i : NVPTXBuiltin<"int(int volatile *, int)">;
+  def __nvvm_atom_cta_xchg_gen_i : NVPTXBuiltinSM<"int(int volatile *, int)", SM_60>;
+  def __nvvm_atom_sys_xchg_gen_i : NVPTXBuiltinSM<"int(int volatile *, int)", SM_60>;
+  def __nvvm_atom_xchg_gen_l : NVPTXBuiltin<"long int(long int volatile *, long int)">;
+  def __nvvm_atom_cta_xchg_gen_l : NVPTXBuiltinSM<"long int(long int volatile *, long int)", SM_60>;
+  def __nvvm_atom_sys_xchg_gen_l : NVPTXBuiltinSM<"long int(long int volatile *, long int)", SM_60>;
+  def __nvvm_atom_xchg_gen_ll : NVPTXBuiltin<"long long int(long long int volatile *, long long int)">;
+  def __nvvm_atom_cta_xchg_gen_ll : NVPTXBuiltinSM<"long long int(long long int volatile *, long long int)", SM_60>;
+  def __nvvm_atom_sys_xchg_gen_ll : NVPTXBuiltinSM<"long long int(long long int volatile *, long long int)", SM_60>;
+
+  def __nvvm_atom_max_gen_i : NVPTXBuiltin<"int(int volatile *, int)">;
+  def __nvvm_atom_cta_max_gen_i : NVPTXBuiltinSM<"int(int volatile *, int)", SM_60>;
+  def __nvvm_atom_sys_max_gen_i : NVPTXBuiltinSM<"int(int volatile *, int)", SM_60>;
+  def __nvvm_atom_max_gen_ui : NVPTXBuiltin<"unsigned int(unsigned int volatile *, unsigned int)">;
+  def __nvvm_atom_cta_max_gen_ui : NVPTXBuiltinSM<"unsigned int(unsigned int volatile *, unsigned int)", SM_60>;
+  def __nvvm_atom_sys_max_gen_ui : NVPTXBuiltinSM<"unsigned int(unsigned int volatile *, unsigned int)", SM_60>;
+  def __nvvm_atom_max_gen_l : NVPTXBuiltin<"long int(long int volatile *, long int)">;
+  def __nvvm_atom_cta_max_gen_l : NVPTXBuiltinSM<"long int(long int volatile *, long int)", SM_60>;
+  def __nvvm_atom_sys_max_gen_l : NVPTXBuiltinSM<"long int(long int volatile *, long int)", SM_60>;
+  def __nvvm_atom_max_gen_ul : NVPTXBuiltin<"unsigned long int(unsigned long int volatile *, unsigned long int)">;
+  def __nvvm_atom_cta_max_gen_ul : NVPTXBuiltinSM<"unsigned long int(unsigned long int volatile *, unsigned long int)", SM_60>;
+  def __nvvm_atom_sys_max_gen_ul : NVPTXBuiltinSM<"unsigned long int(unsigned long int volatile *, unsigned long int)", SM_60>;
+  def __nvvm_atom_max_gen_ll : NVPTXBuiltin<"long long int(long long int volatile *, long long int)">;
+  def __nvvm_atom_cta_max_gen_ll : NVPTXBuiltinSM<"long long int(long long int volatile *, long long int)", SM_60>;
+  def __nvvm_atom_sys_max_gen_ll : NVPTXBuiltinSM<"long long int(long long int volatile *, long long int)", SM_60>;
+  def __nvvm_atom_max_gen_ull : NVPTXBuiltin<"unsigned long long int(unsigned long long int volatile *, unsigned long long int)">;
+  def __nvvm_atom_cta_max_gen_ull : NVPTXBuiltinSM<"unsigned long long int(unsigned long long int volatile *, unsigned long long int)", SM_60>;
+  def __nvvm_atom_sys_max_gen_ull : NVPTXBuiltinSM<"unsigned long long int(unsigned long long int volatile *, unsigned long long int)", SM_60>;
+
+  def __nvvm_atom_min_gen_i : NVPTXBuiltin<"int(int volatile *, int)">;
+  def __nvvm_atom_cta_min_gen_i : NVPTXBuiltinSM<"int(int volatile *, int)", SM_60>;
+  def __nvvm_atom_sys_min_gen_i : NVPTXBuiltinSM<"int(int volatile *, int)", SM_60>;
+  def __nvvm_atom_min_gen_ui : NVPTXBuiltin<"unsigned int(unsigned int volatile *, unsigned int)">;
+  def __nvvm_atom_cta_min_gen_ui : NVPTXBuiltinSM<"unsigned int(unsigned int volatile *, unsigned int)", SM_60>;
+  def __nvvm_atom_sys_min_gen_ui : NVPTXBuiltinSM<"unsigned int(unsigned int volatile *, unsigned int)", SM_60>;
+  def __nvvm_atom_min_gen_l : NVPTXBuiltin<"long int(long int volatile *, long int)">;
+  def __nvvm_atom_cta_min_gen_l : NVPTXBuiltinSM<"long int(long int volatile *, long int)", SM_60>;
+  def __nvvm_atom_sys_min_gen_l : NVPTXBuiltinSM<"long int(long int volatile *, long int)", SM_60>;
+  def __nvvm_atom_min_gen_ul : NVPTXBuiltin<"unsigned long int(unsigned long int volatile *, unsigned long int)">;
+  def __nvvm_atom_cta_min_gen_ul : NVPTXBuiltinSM<"unsigned long int(unsigned long int volatile *, unsigned long int)", SM_60>;
+  def __nvvm_atom_sys_min_gen_ul : NVPTXBuiltinSM<"unsigned long int(unsigned long int volatile *, unsigned long int)", SM_60>;
+  def __nvvm_atom_min_gen_ll : NVPTXBuiltin<"long long int(long long int volatile *, long long int)">;
+  def __nvvm_atom_cta_min_gen_ll : NVPTXBuiltinSM<"long long int(long long int volatile *, long long int)", SM_60>;
+  def __nvvm_atom_sys_min_gen_ll : NVPTXBuiltinSM<"long long int(long long int volatile *, long long int)", SM_60>;
+  def __nvvm_atom_min_gen_ull : NVPTXBuiltin<"unsigned long long int(unsigned long long int volatile *, unsigned long long int)">;
+  def __nvvm_atom_cta_min_gen_ull : NVPTXBuiltinSM<"unsigned long long int(unsigned long long int volatile *, unsigned long long int)", SM_60>;
+  def __nvvm_atom_sys_min_gen_ull : NVPTXBuiltinSM<"unsigned long long int(unsigned long long int volatile *, unsigned long long int)", SM_60>;
+
+  def __nvvm_atom_inc_gen_ui : NVPTXBuiltin<"unsigned int(unsigned int volatile *, unsigned int)">;
+  def __nvvm_atom_cta_inc_gen_ui : NVPTXBuiltinSM<"unsigned int(unsigned int volatile *, unsigned int)", SM_60>;
+  def __nvvm_atom_sys_inc_gen_ui : NVPTXBuiltinSM<"unsigned int(unsigned int volatile *, unsigned int)", SM_60>;
+  def __nvvm_atom_dec_gen_ui : NVPTXBuiltin<"unsigned int(unsigned int volatile *, unsigned int)">;
+  def __nvvm_atom_cta_dec_gen_ui : NVPTXBuiltinSM<"unsigned int(unsigned int volatile *, unsigned int)", SM_60>;
+  def __nvvm_atom_sys_dec_gen_ui : NVPTXBuiltinSM<"unsigned int(unsigned int volatile *, unsigned int)", SM_60>;
+
+  def __nvvm_atom_and_gen_i : NVPTXBuiltin<"int(int volatile *, int)">;
+  def __nvvm_atom_cta_and_gen_i : NVPTXBuiltinSM<"int(int volatile *, int)", SM_60>;
+  def __nvvm_atom_sys_and_gen_i : NVPTXBuiltinSM<"int(int volatile *, int)", SM_60>;
+  def __nvvm_atom_and_gen_l : NVPTXBuiltin<"long int(long int volatile *, long int)">;
+  def __nvvm_atom_cta_and_gen_l : NVPTXBuiltinSM<"long int(long int volatile *, long int)", SM_60>;
+  def __nvvm_atom_sys_and_gen_l : NVPTXBuiltinSM<"long int(long int volatile *, long int)", SM_60>;
+  def __nvvm_atom_and_gen_ll : NVPTXBuiltin<"long long int(long long int volatile *, long long int)">;
+  def __nvvm_atom_cta_and_gen_ll : NVPTXBuiltinSM<"long long int(long long int volatile *, long long int)", SM_60>;
+  def __nvvm_atom_sys_and_gen_ll : NVPTXBuiltinSM<"long long int(long long int volatile *, long long int)", SM_60>;
+
+  def __nvvm_atom_or_gen_i : NVPTXBuiltin<"int(int volatile *, int)">;
+  def __nvvm_atom_cta_or_gen_i : NVPTXBuiltinSM<"int(int volatile *, int)", SM_60>;
+  def __nvvm_atom_sys_or_gen_i : NVPTXBuiltinSM<"int(int volatile *, int)", SM_60>;
+  def __nvvm_atom_or_gen_l : NVPTXBuiltin<"long int(long int volatile *, long int)">;
+  def __nvvm_atom_cta_or_gen_l : NVPTXBuiltinSM<"long int(long int volatile *, long int)", SM_60>;
+  def __nvvm_atom_sys_or_gen_l : NVPTXBuiltinSM<"long int(long int volatile *, long int)", SM_60>;
+  def __nvvm_atom_or_gen_ll : NVPTXBuiltin<"long long int(long long int volatile *, long long int)">;
+  def __nvvm_atom_cta_or_gen_ll : NVPTXBuiltinSM<"long long int(long long int volatile *, long long int)", SM_60>;
+  def __nvvm_atom_sys_or_gen_ll : NVPTXBuiltinSM<"long long int(long long int volatile *, long long int)", SM_60>;
+
+  def __nvvm_atom_xor_gen_i : NVPTXBuiltin<"int(int volatile *, int)">;
+  def __nvvm_atom_cta_xor_gen_i : NVPTXBuiltinSM<"int(int volatile *, int)", SM_60>;
+  def __nvvm_atom_sys_xor_gen_i : NVPTXBuiltinSM<"int(int volatile *, int)", SM_60>;
+  def __nvvm_atom_xor_gen_l : NVPTXBuiltin<"long int(long int volatile *, long int)">;
+  def __nvvm_atom_cta_xor_gen_l : NVPTXBuiltinSM<"long int(long int volatile *, long int)", SM_60>;
+  def __nvvm_atom_sys_xor_gen_l : NVPTXBuiltinSM<"long int(long int volatile *, long int)", SM_60>;
+  def __nvvm_atom_xor_gen_ll : NVPTXBuiltin<"long long int(long long int volatile *, long long int)">;
+  def __nvvm_atom_cta_xor_gen_ll : NVPTXBuiltinSM<"long long int(long long int volatile *, long long int)", SM_60>;
+  def __nvvm_atom_sys_xor_gen_ll : NVPTXBuiltinSM<"long long int(long long int volatile *, long long int)", SM_60>;
+
+  def __nvvm_atom_cas_gen_us : NVPTXBuiltinSM<"unsigned short(unsigned short volatile *, unsigned short, unsigned short)", SM_70>;
+  def __nvvm_atom_cta_cas_gen_us : NVPTXBuiltinSM<"unsigned short(unsigned short volatile *, unsigned short, unsigned short)", SM_70>;
+  def __nvvm_atom_sys_cas_gen_us : NVPTXBuiltinSM<"unsigned short(unsigned short volatile *, unsigned short, unsigned short)", SM_70>;
+  def __nvvm_atom_cas_gen_i : NVPTXBuiltin<"int(int volatile *, int, int)">;
+  def __nvvm_atom_cta_cas_gen_i : NVPTXBuiltinSM<"int(int volatile *, int, int)", SM_60>;
+  def __nvvm_atom_sys_cas_gen_i : NVPTXBuiltinSM<"int(int volatile *, int, int)", SM_60>;
+  def __nvvm_atom_cas_gen_l : NVPTXBuiltin<"long int(long int volatile *, long int, long int)">;
+  def __nvvm_atom_cta_cas_gen_l : NVPTXBuiltinSM<"long int(long int volatile *, long int, long int)", SM_60>;
+  def __nvvm_atom_sys_cas_gen_l : NVPTXBuiltinSM<"long int(long int volatile *, long int, long int)", SM_60>;
+  def __nvvm_atom_cas_gen_ll : NVPTXBuiltin<"long long int(long long int volatile *, long long int, long long int)">;
+  def __nvvm_atom_cta_cas_gen_ll : NVPTXBuiltinSM<"long long int(long long int volatile *, long long int, long long int)", SM_60>;
+  def __nvvm_atom_sys_cas_gen_ll : NVPTXBuiltinSM<"long long int(long long int volatile *, long long int, long long int)", SM_60>;
+}
+
+// Compiler Error Warn
+let Attributes = [NoThrow] in {
+  def __nvvm_compiler_error : NVPTXBuiltin<"void(char const address_space<4> *)">;
+  def __nvvm_compiler_warn : NVPTXBuiltin<"void(char const address_space<4> *)">;
+}
+
+def __nvvm_ldu_c : NVPTXBuiltin<"char(char const *)">;
+def __nvvm_ldu_sc : NVPTXBuiltin<"signed char(signed char const *)">;
+def __nvvm_ldu_s : NVPTXBuiltin<"short(short const *)">;
+def __nvvm_ldu_i : NVPTXBuiltin<"int(int const *)">;
+def __nvvm_ldu_l : NVPTXBuiltin<"long int(long int const *)">;
+def __nvvm_ldu_ll : NVPTXBuiltin<"long long int(long long int const *)">;
+
+def __nvvm_ldu_uc : NVPTXBuiltin<"unsigned char(unsigned char const *)">;
+def __nvvm_ldu_us : NVPTXBuiltin<"unsigned short(unsigned short const *)">;
+def __nvvm_ldu_ui : NVPTXBuiltin<"unsigned int(unsigned int const *)">;
+def __nvvm_ldu_ul : NVPTXBuiltin<"unsigned long int(unsigned long int const *)">;
+def __nvvm_ldu_ull : NVPTXBuiltin<"unsigned long long int(unsigned long long int const *)">;
+
+def __nvvm_ldu_h : NVPTXBuiltin<"__fp16(__fp16 const *)">;
+def __nvvm_ldu_f : NVPTXBuiltin<"float(float const *)">;
+def __nvvm_ldu_d : NVPTXBuiltin<"double(double const *)">;
+
+def __nvvm_ldu_c2 : NVPTXBuiltin<"_ExtVector<2, char>(_ExtVector<2, char const *>)">;
+def __nvvm_ldu_sc2 : NVPTXBuiltin<"_ExtVector<2, signed char>(_ExtVector<2, signed char const *>)">;
+def __nvvm_ldu_c4 : NVPTXBuiltin<"_ExtVector<4, char>(_ExtVector<4, char const *>)">;
+def __nvvm_ldu_sc4 : NVPTXBuiltin<"_ExtVector<4, signed char>(_ExtVector<4, signed char const *>)">;
+def __nvvm_ldu_s2 : NVPTXBuiltin<"_ExtVector<2, short>(_ExtVector<2, short const *>)">;
+def __nvvm_ldu_s4 : NVPTXBuiltin<"_ExtVector<4, short>(_ExtVector<4, short const *>)">;
+def __nvvm_ldu_i2 : NVPTXBuiltin<"_ExtVector<2, int>(_ExtVector<2, int const *>)">;
+def __nvvm_ldu_i4 : NVPTXBuiltin<"_ExtVector<4, int>(_ExtVector<4, int const *>)">;
+def __nvvm_ldu_l2 : NVPTXBuiltin<"_ExtVector<2, long int>(_ExtVector<2, long int const *>)">;
+def __nvvm_ldu_ll2 : NVPTXBuiltin<"_ExtVector<2, long long int>(_ExtVector<2, long long int const *>)">;
+
+def __nvvm_ldu_uc2 : NVPTXBuiltin<"_ExtVector<2, unsigned char>(_ExtVector<2, unsigned char const *>)">;
+def __nvvm_ldu_uc4 : NVPTXBuiltin<"_ExtVector<4, unsigned char>(_ExtVector<4, unsigned char const *>)">;
+def __nvvm_ldu_us2 : NVPTXBuiltin<"_ExtVector<2, unsigned short>(_ExtVector<2, unsigned short const *>)">;
+def __nvvm_ldu_us4 : NVPTXBuiltin<"_ExtVector<4, unsigned short>(_ExtVector<4, unsigned short const *>)">;
+def __nvvm_ldu_ui2 : NVPTXBuiltin<"_ExtVector<2, unsigned int>(_ExtVector<2, unsigned int const *>)">;
+def __nvvm_ldu_ui4 : NVPTXBuiltin<"_ExtVector<4, unsigned int>(_ExtVector<4, unsigned int const *>)">;
+def __nvvm_ldu_ul2 : NVPTXBuiltin<"_ExtVector<2, unsigned long int>(_ExtVector<2, unsigned long int const *>)">;
+def __nvvm_ldu_ull2 : NVPTXBuiltin<"_ExtVector<2, unsigned long long int>(_ExtVector<2, unsigned long long int const *>)">;
+
+def __nvvm_ldu_h2 : NVPTXBuiltin<"_ExtVector<2, __fp16>(_ExtVector<2, __fp16 const *>)">;
+def __nvvm_ldu_f2 : NVPTXBuiltin<"_ExtVector<2, float>(_ExtVector<2, float const *>)">;
+def __nvvm_ldu_f4 : NVPTXBuiltin<"_ExtVector<4, float>(_ExtVector<4, float const *>)">;
+def __nvvm_ldu_d2 : NVPTXBuiltin<"_ExtVector<2, double>(_ExtVector<2, double const *>)">;
+
+def __nvvm_ldg_c : NVPTXBuiltin<"char(char const *)">;
+def __nvvm_ldg_sc : NVPTXBuiltin<"signed char(signed char const *)">;
+def __nvvm_ldg_s : NVPTXBuiltin<"short(short const *)">;
+def __nvvm_ldg_i : NVPTXBuiltin<"int(int const *)">;
+def __nvvm_ldg_l : NVPTXBuiltin<"long int(long int const *)">;
+def __nvvm_ldg_ll : NVPTXBuiltin<"long long int(long long int const *)">;
+
+def __nvvm_ldg_uc : NVPTXBuiltin<"unsigned char(unsigned char const *)">;
+def __nvvm_ldg_us : NVPTXBuiltin<"unsigned short(unsigned short const *)">;
+def __nvvm_ldg_ui : NVPTXBuiltin<"unsigned int(unsigned int const *)">;
+def __nvvm_ldg_ul : NVPTXBuiltin<"unsigned long int(unsigned long int const *)">;
+def __nvvm_ldg_ull : NVPTXBuiltin<"unsigned long long int(unsigned long long int const *)">;
+
+def __nvvm_ldg_h : NVPTXBuiltin<"__fp16(__fp16 const *)">;
+def __nvvm_ldg_f : NVPTXBuiltin<"float(float const *)">;
+def __nvvm_ldg_d : NVPTXBuiltin<"double(double const *)">;
+
+def __nvvm_ldg_c2 : NVPTXBuiltin<"_ExtVector<2, char>(_ExtVector<2, char const *>)">;
+def __nvvm_ldg_sc2 : NVPTXBuiltin<"_ExtVector<2, signed char>(_ExtVector<2, signed char const *>)">;
+def __nvvm_ldg_c4 : NVPTXBuiltin<"_ExtVector<4, char>(_ExtVector<4, char const *>)">;
+def __nvvm_ldg_sc4 : NVPTXBuiltin<"_ExtVector<4, signed char>(_ExtVector<4, signed char const *>)">;
+def __nvvm_ldg_s2 : NVPTXBuiltin<"_ExtVector<2, short>(_ExtVector<2, short const *>)">;
+def __nvvm_ldg_s4 : NVPTXBuiltin<"_ExtVector<4, short>(_ExtVector<4, short const *>)">;
+def __nvvm_ldg_i2 : NVPTXBuiltin<"_ExtVector<2, int>(_ExtVector<2, int const *>)">;
+def __nvvm_ldg_i4 : NVPTXBuiltin<"_ExtVector<4, int>(_ExtVector<4, int const *>)">;
+def __nvvm_ldg_l2 : NVPTXBuiltin<"_ExtVector<2, long int>(_ExtVector<2, long int const *>)">;
+def __nvvm_ldg_ll2 : NVPTXBuiltin<"_ExtVector<2, long long int>(_ExtVector<2, long long int const *>)">;
+
+def __nvvm_ldg_uc2 : NVPTXBuiltin<"_ExtVector<2, unsigned char>(_ExtVector<2, unsigned char const *>)">;
+def __nvvm_ldg_uc4 : NVPTXBuiltin<"_ExtVector<4, unsigned char>(_ExtVector<4, unsigned char const *>)">;
+def __nvvm_ldg_us2 : NVPTXBuiltin<"_ExtVector<2, unsigned short>(_ExtVector<2, unsigned short const *>)">;
+def __nvvm_ldg_us4 : NVPTXBuiltin<"_ExtVector<4, unsigned short>(_ExtVector<4, unsigned short const *>)">;
+def __nvvm_ldg_ui2 : NVPTXBuiltin<"_ExtVector<2, unsigned int>(_ExtVector<2, unsigned int const *>)">;
+def __nvvm_ldg_ui4 : NVPTXBuiltin<"_ExtVector<4, unsigned int>(_ExtVector<4, unsigned int const *>)">;
+def __nvvm_ldg_ul2 : NVPTXBuiltin<"_ExtVector<2, unsigned long int>(_ExtVector<2, unsigned long int const *>)">;
+def __nvvm_ldg_ull2 : NVPTXBuiltin<"_ExtVector<2, unsigned long long int>(_ExtVector<2, unsigned long long int const *>)">;
+
+def __nvvm_ldg_h2 : NVPTXBuiltin<"_ExtVector<2, __fp16>(_ExtVector<2, __fp16 const *>)">;
+def __nvvm_ldg_f2 : NVPTXBuiltin<"_ExtVector<2, float>(_ExtVector<2, float const *>)">;
+def __nvvm_ldg_f4 : NVPTXBuiltin<"_ExtVector<4, float>(_ExtVector<4, float const *>)">;
+def __nvvm_ldg_d2 : NVPTXBuiltin<"_ExtVector<2, double>(_ExtVector<2, double const *>)">;
+
+// Address space predicates.
+let Attributes = [NoThrow, Const] in {
+  def __nvvm_isspacep_const : NVPTXBuiltin<"bool(void const *)">;
+  def __nvvm_isspacep_global : NVPTXBuiltin<"bool(void const *)">;
+  def __nvvm_isspacep_local : NVPTXBuiltin<"bool(void const *)">;
+  def __nvvm_isspacep_shared : NVPTXBuiltin<"bool(void const *)">;
+  def __nvvm_isspacep_shared_cluster : NVPTXBuiltinSMAndPTX<"bool(void const *)", SM_90, PTX78>;
+}
+
+// Builtins to support WMMA instructions on sm_70
+def __hmma_m16n16k16_ld_a : NVPTXBuiltinSMAndPTX<"void(int *, int const *, unsigned int, _Constant int)", SM_70, PTX60>;
+def __hmma_m16n16k16_ld_b : NVPTXBuiltinSMAndPTX<"void(int *, int const *, unsigned int, _Constant int)", SM_70, PTX60>;
+def __hmma_m16n16k16_ld_c_f16 : NVPTXBuiltinSMAndPTX<"void(int *, int const *, unsigned int, _Constant int)", SM_70, PTX60>;
+def __hmma_m16n16k16_ld_c_f32 : NVPTXBuiltinSMAndPTX<"void(float *, float const *, unsigned int, _Constant int)", SM_70, PTX60>;
+def __hmma_m16n16k16_st_c_f16 : NVPTXBuiltinSMAndPTX<"void(int *, int const *, unsigned int, _Constant int)", SM_70, PTX60>;
+def __hmma_m16n16k16_st_c_f32 : NVPTXBuiltinSMAndPTX<"void(float *, float const *, unsigned int, _Constant int)", SM_70, PTX60>;
+
+def __hmma_m32n8k16_ld_a : NVPTXBuiltinSMAndPTX<"void(int *, int const *, unsigned int, _Constant int)", SM_70, PTX61>;
+def __hmma_m32n8k16_ld_b : NVPTXBuiltinSMAndPTX<"void(int *, int const *, unsigned int, _Constant int)", SM_70, PTX61>;
+def __hmma_m32n8k16_ld_c_f16 : NVPTXBuiltinSMAndPTX<"void(int *, int const *, unsigned int, _Constant int)", SM_70, PTX61>;
+def __hmma_m32n8k16_ld_c_f32 : NVPTXBuiltinSMAndPTX<"void(float *, float const *, unsigned int, _Constant int)", SM_70, PTX61>;
+def __hmma_m32n8k16_st_c_f16 : NVPTXBuiltinSMAndPTX<"void(int *, int const *, unsigned int, _Constant int)", SM_70, PTX61>;
+def __hmma_m32n8k16_st_c_f32 : NVPTXBuiltinSMAndPTX<"void(float *, float const *, unsigned int, _Constant int)", SM_70, PTX61>;
+
+def __hmma_m8n32k16_ld_a : NVPTXBuiltinSMAndPTX<"void(int *, int const *, unsigned int, _Constant int)", SM_70, PTX61>;
+def __hmma_m8n32k16_ld_b : NVPTXBuiltinSMAndPTX<"void(int *, int const *, unsigned int, _Constant int)", SM_70, PTX61>;
+def __hmma_m8n32k16_ld_c_f16 : NVPTXBuiltinSMAndPTX<"void(int *, int const *, unsigned int, _Constant int)", SM_70, PTX61>;
+def __hmma_m8n32k16_ld_c_f32 : NVPTXBuiltinSMAndPTX<"void(float *, float const *, unsigned int, _Constant int)", SM_70, PTX61>;
+def __hmma_m8n32k16_st_c_f16 : NVPTXBuiltinSMAndPTX<"void(int *, int const *, unsigned int, _Constant int)", SM_70, PTX61>;
+def __hmma_m8n32k16_st_c_f32 : NVPTXBuiltinSMAndPTX<"void(float *, float const *, unsigned int, _Constant int)", SM_70, PTX61>;
+
+def __hmma_m16n16k16_mma_f16f16 : NVPTXBuiltinSMAndPTX<"void(int *, int const *, int const *, int const *, _Constant int, _Constant int)", SM_70, PTX60>;
+def __hmma_m16n16k16_mma_f32f16 : NVPTXBuiltinSMAndPTX<"void(float *, int const *, int const *, int const *, _Constant int, _Constant int)", SM_70, PTX60>;
+def __hmma_m16n16k16_mma_f32f32 : NVPTXBuiltinSMAndPTX<"void(float *, int const *, int const *, float const *, _Constant int, _Constant int)", SM_70, PTX60>;
+def __hmma_m16n16k16_mma_f16f32 : NVPTXBuiltinSMAndPTX<"void(int *, int const *, int const *, float const *, _Constant int, _Constant int)", SM_70, PTX60>;
+
+def __hmma_m32n8k16_mma_f16f16 : NVPTXBuiltinSMAndPTX<"void(int *, int const *, int const *, int const *, _Constant int, _Constant int)", SM_70, PTX61>;
+def __hmma_m32n8k16_mma_f32f16 : NVPTXBuiltinSMAndPTX<"void(float *, int const *, int const *, int const *, _Constant int, _Constant int)", SM_70, PTX61>;
+def __hmma_m32n8k16_mma_f32f32 : NVPTXBuiltinSMAndPTX<"void(float *, int const *, int const *, float const *, _Constant int, _Constant int)", SM_70, PTX61>;
+def __hmma_m32n8k16_mma_f16f32 : NVPTXBuiltinSMAndPTX<"void(int *, int const *, int const *, float const *, _Constant int, _Constant int)", SM_70, PTX61>;
+
+def __hmma_m8n32k16_mma_f16f16 : NVPTXBuiltinSMAndPTX<"void(int *, int const *, int const *, int const *, _Constant int, _Constant int)", SM_70, PTX61>;
+def __hmma_m8n32k16_mma_f32f16 : NVPTXBuiltinSMAndPTX<"void(float *, int const *, int const *, int const *, _Constant int, _Constant int)", SM_70, PTX61>;
+def __hmma_m8n32k16_mma_f32f32 : NVPTXBuiltinSMAndPTX<"void(float *, int const *, int const *, float const *, _Constant int, _Constant int)", SM_70, PTX61>;
+def __hmma_m8n32k16_mma_f16f32 : NVPTXBuiltinSMAndPTX<"void(int *, int const *, int const *, float const *, _Constant int, _Constant int)", SM_70, PTX61>;
+
+// Builtins to support integer and sub-integer WMMA instructions on sm_72/sm_75
+def __bmma_m8n8k128_ld_a_b1 : NVPTXBuiltinSMAndPTX<"void(int *, int const *, unsigned int, _Constant int)", SM_75, PTX63>;
+def __bmma_m8n8k128_ld_b_b1 : NVPTXBuiltinSMAndPTX<"void(int *, int const *, unsigned int, _Constant int)", SM_75, PTX63>;
+def __bmma_m8n8k128_ld_c : NVPTXBuiltinSMAndPTX<"void(int *, int const *, unsigned int, _Constant int)", SM_75, PTX63>;
+def __bmma_m8n8k128_mma_and_popc_b1 : NVPTXBuiltinSMAndPTX<"void(int *, int const *, int const *, int const *, _Constant int)", SM_80, PTX71>;
+def __bmma_m8n8k128_mma_xor_popc_b1 : NVPTXBuiltinSMAndPTX<"void(int *, int const *, int const *, int const *, _Constant int)", SM_75, PTX63>;
+def __bmma_m8n8k128_st_c_i32 : NVPTXBuiltinSMAndPTX<"void(int *, int const *, unsigned int, _Constant int)", SM_75, PTX63>;
+def __imma_m16n16k16_ld_a_s8 : NVPTXBuiltinSMAndPTX<"void(int *, int const *, unsigned int, _Constant int)", SM_72, PTX63>;
+def __imma_m16n16k16_ld_a_u8 : NVPTXBuiltinSMAndPTX<"void(int *, int const *, unsigned int, _Constant int)", SM_72, PTX63>;
+def __imma_m16n16k16_ld_b_s8 : NVPTXBuiltinSMAndPTX<"void(int *, int const *, unsigned int, _Constant int)", SM_72, PTX63>;
+def __imma_m16n16k16_ld_b_u8 : NVPTXBuiltinSMAndPTX<"void(int *, int const *, unsigned int, _Constant int)", SM_72, PTX63>;
+def __imma_m16n16k16_ld_c : NVPTXBuiltinSMAndPTX<"void(int *, int const *, unsigned int, _Constant int)", SM_72, PTX63>;
+def __imma_m16n16k16_mma_s8 : NVPTXBuiltinSMAndPTX<"void(int *, int const *, int const *, int const *, _Constant int, _Constant int)", SM_72, PTX63>;
+def __imma_m16n16k16_mma_u8 : NVPTXBuiltinSMAndPTX<"void(int *, int const *, int const *, int const *, _Constant int, _Constant int)", SM_72, PTX63>;
+def __imma_m16n16k16_st_c_i32 : NVPTXBuiltinSMAndPTX<"void(int *, int const *, unsigned int, _Constant int)", SM_72, PTX63>;
+def __imma_m32n8k16_ld_a_s8 : NVPTXBuiltinSMAndPTX<"void(int *, int const *, unsigned int, _Constant int)", SM_72, PTX63>;
+def __imma_m32n8k16_ld_a_u8 : NVPTXBuiltinSMAndPTX<"void(int *, int const *, unsigned int, _Constant int)", SM_72, PTX63>;
+def __imma_m32n8k16_ld_b_s8 : NVPTXBuiltinSMAndPTX<"void(int *, int const *, unsigned int, _Constant int)", SM_72, PTX63>;
+def __imma_m32n8k16_ld_b_u8 : NVPTXBuiltinSMAndPTX<"void(int *, int const *, unsigned int, _Constant int)", SM_72, PTX63>;
+def __imma_m32n8k16_ld_c : NVPTXBuiltinSMAndPTX<"void(int *, int const *, unsigned int, _Constant int)", SM_72, PTX63>;
+def __imma_m32n8k16_mma_s8 : NVPTXBuiltinSMAndPTX<"void(int *, int const *, int const *, int const *, _Constant int, _Constant int)", SM_72, PTX63>;
+def __imma_m32n8k16_mma_u8 : NVPTXBuiltinSMAndPTX<"void(int *, int const *, int const *, int const *, _Constant int, _Constant int)", SM_72, PTX63>;
+def __imma_m32n8k16_st_c_i32 : NVPTXBuiltinSMAndPTX<"void(int *, int const *, unsigned int, _Constant int)", SM_72, PTX63>;
+def __imma_m8n32k16_ld_a_s8 : NVPTXBuiltinSMAndPTX<"void(int *, int const *, unsigned int, _Constant int)", SM_72, PTX63>;
+def __imma_m8n32k16_ld_a_u8 : NVPTXBuiltinSMAndPTX<"void(int *, int const *, unsigned int, _Constant int)", SM_72, PTX63>;
+def __imma_m8n32k16_ld_b_s8 : NVPTXBuiltinSMAndPTX<"void(int *, int const *, unsigned int, _Constant int)", SM_72, PTX63>;
+def __imma_m8n32k16_ld_b_u8 : NVPTXBuiltinSMAndPTX<"void(int *, int const *, unsigned int, _Constant int)", SM_72, PTX63>;
+def __imma_m8n32k16_ld_c : NVPTXBuiltinSMAndPTX<"void(int *, int const *, unsigned int, _Constant int)", SM_72, PTX63>;
+def __imma_m8n32k16_mma_s8 : NVPTXBuiltinSMAndPTX<"void(int *, int const *, int const *, int const *, _Constant int, _Constant int)", SM_72, PTX63>;
+def __imma_m8n32k16_mma_u8 : NVPTXBuiltinSMAndPTX<"void(int *, int const *, int const *, int const *, _Constant int, _Constant int)", SM_72, PTX63>;
+def __imma_m8n32k16_st_c_i32 : NVPTXBuiltinSMAndPTX<"void(int *, int const *, unsigned int, _Constant int)", SM_72, PTX63>;
+def __imma_m8n8k32_ld_a_s4 : NVPTXBuiltinSMAndPTX<"void(int *, int const *, unsigned int, _Constant int)", SM_75, PTX63>;
+def __imma_m8n8k32_ld_a_u4 : NVPTXBuiltinSMAndPTX<"void(int *, int const *, unsigned int, _Constant int)", SM_75, PTX63>;
+def __imma_m8n8k32_ld_b_s4 : NVPTXBuiltinSMAndPTX<"void(int *, int const *, unsigned int, _Constant int)", SM_75, PTX63>;
+def __imma_m8n8k32_ld_b_u4 : NVPTXBuiltinSMAndPTX<"void(int *, int const *, unsigned int, _Constant int)", SM_75, PTX63>;
+def __imma_m8n8k32_ld_c : NVPTXBuiltinSMAndPTX<"void(int *, int const *, unsigned int, _Constant int)", SM_75, PTX63>;
+def __imma_m8n8k32_mma_s4 : NVPTXBuiltinSMAndPTX<"void(int *, int const *, int const *, int const *, _Constant int, _Constant int)", SM_75, PTX63>;
+def __imma_m8n8k32_mma_u4 : NVPTXBuiltinSMAndPTX<"void(int *, int const *, int const *, int const *, _Constant int, _Constant int)", SM_75, PTX63>;
+def __imma_m8n8k32_st_c_i32 : NVPTXBuiltinSMAndPTX<"void(int *, int const *, unsigned int, _Constant int)", SM_75, PTX63>;
+
+// Builtins to support double and alternate float WMMA instructions on sm_80
+def __dmma_m8n8k4_ld_a : NVPTXBuiltinSMAndPTX<"void(double *, double const *, unsigned int, _Constant int)", SM_80, PTX70>;
+def __dmma_m8n8k4_ld_b : NVPTXBuiltinSMAndPTX<"void(double *, double const *, unsigned int, _Constant int)", SM_80, PTX70>;
+def __dmma_m8n8k4_ld_c : NVPTXBuiltinSMAndPTX<"void(double *, double const *, unsigned int, _Constant int)", SM_80, PTX70>;
+def __dmma_m8n8k4_st_c_f64 : NVPTXBuiltinSMAndPTX<"void(double *, double const *, unsigned int, _Constant int)", SM_80, PTX70>;
+def __dmma_m8n8k4_mma_f64 : NVPTXBuiltinSMAndPTX<"void(double *, double const *, double const *, double const *, _Constant int, _Constant int)", SM_80, PTX70>;
+
+def __mma_bf16_m16n16k16_ld_a : NVPTXBuiltinSMAndPTX<"void(int *, int const *, unsigned int, _Constant int)", SM_80, PTX70>;
+def __mma_bf16_m16n16k16_ld_b : NVPTXBuiltinSMAndPTX<"void(int *, int const *, unsigned int, _Constant int)", SM_80, PTX70>;
+def __mma_bf16_m16n16k16_mma_f32 : NVPTXBuiltinSMAndPTX<"void(float *, int const *, int const *, float const *, _Constant int, _Constant int)", SM_80, PTX70>;
+def __mma_bf16_m8n32k16_ld_a : NVPTXBuiltinSMAndPTX<"void(int *, int const *, unsigned int, _Constant int)", SM_80, PTX70>;
+def __mma_bf16_m8n32k16_ld_b : NVPTXBuiltinSMAndPTX<"void(int *, int const *, unsigned int, _Constant int)", SM_80, PTX70>;
+def __mma_bf16_m8n32k16_mma_f32 : NVPTXBuiltinSMAndPTX<"void(float *, int const *, int const *, float const *, _Constant int, _Constant int)", SM_80, PTX70>;
+def __mma_bf16_m32n8k16_ld_a : NVPTXBuiltinSMAndPTX<"void(int *, int const *, unsigned int, _Constant int)", SM_80, PTX70>;
+def __mma_bf16_m32n8k16_ld_b : NVPTXBuiltinSMAndPTX<"void(int *, int const *, unsigned int, _Constant int)", SM_80, PTX70>;
+def __mma_bf16_m32n8k16_mma_f32 : NVPTXBuiltinSMAndPTX<"void(float *, int const *, int const *, float const *, _Constant int, _Constant int)", SM_80, PTX70>;
+
+def __mma_tf32_m16n16k8_ld_a : NVPTXBuiltinSMAndPTX<"void(int *, int const *, unsigned int, _Constant int)", SM_80, PTX70>;
+def __mma_tf32_m16n16k8_ld_b : NVPTXBuiltinSMAndPTX<"void(int *, int const *, unsigned int, _Constant int)", SM_80, PTX70>;
+def __mma_tf32_m16n16k8_ld_c : NVPTXBuiltinSMAndPTX<"void(float *, float const *, unsigned int, _Constant int)", SM_80, PTX70>;
+def __mma_m16n16k8_st_c_f32 : NVPTXBuiltinSMAndPTX<"void(float *, float const *, unsigned int, _Constant int)", SM_80, PTX70>;
+def __mma_tf32_m16n16k8_mma_f32 : NVPTXBuiltinSMAndPTX<"void(float *, int const *, int const *, float const *, _Constant int, _Constant int)", SM_80, PTX70>;
+
+// Async Copy
+def __nvvm_cp_async_mbarrier_arrive : NVPTXBuiltinSMAndPTX<"void(int64_t *)", SM_80, PTX70>;
+def __nvvm_cp_async_mbarrier_arrive_shared : NVPTXBuiltinSMAndPTX<"void(int64_t address_space<3> *)", SM_80, PTX70>;
+def __nvvm_cp_async_mbarrier_arrive_noinc : NVPTXBuiltinSMAndPTX<"void(int64_t *)", SM_80, PTX70>;
+def __nvvm_cp_async_mbarrier_arrive_noinc_shared : NVPTXBuiltinSMAndPTX<"void(int64_t address_space<3> *)", SM_80, PTX70>;
+
+def __nvvm_cp_async_ca_shared_global_4 : NVPTXBuiltinSMAndPTX<"void(void address_space<3> *, void const address_space<1> *, ...)", SM_80, PTX70>;
+def __nvvm_cp_async_ca_shared_global_8 : NVPTXBuiltinSMAndPTX<"void(void address_space<3> *, void const address_space<1> *, ...)", SM_80, PTX70>;
+def __nvvm_cp_async_ca_shared_global_16 : NVPTXBuiltinSMAndPTX<"void(void address_space<3> *, void const address_space<1> *, ...)", SM_80, PTX70>;
+def __nvvm_cp_async_cg_shared_global_16 : NVPTXBuiltinSMAndPTX<"void(void address_space<3> *, void const address_space<1> *, ...)", SM_80, PTX70>;
+
+def __nvvm_cp_async_commit_group : NVPTXBuiltinSMAndPTX<"void()", SM_80, PTX70>;
+def __nvvm_cp_async_wait_group : NVPTXBuiltinSMAndPTX<"void(_Constant int)", SM_80, PTX70>;
+def __nvvm_cp_async_wait_all : NVPTXBuiltinSMAndPTX<"void()", SM_80, PTX70>;
+
+
+// bf16, bf16x2 abs, neg
+def __nvvm_abs_bf16 : NVPTXBuiltinSMAndPTX<"__bf16(__bf16)", SM_80, PTX70>;
+def __nvvm_abs_bf16x2 : NVPTXBuiltinSMAndPTX<"_Vector<2, __bf16>(_Vector<2, __bf16>)", SM_80, PTX70>;
+def __nvvm_neg_bf16 : NVPTXBuiltinSMAndPTX<"__bf16(__bf16)", SM_80, PTX70>;
+def __nvvm_neg_bf16x2 : NVPTXBuiltinSMAndPTX<"_Vector<2, __bf16>(_Vector<2, __bf16>)", SM_80, PTX70>;
+
+def __nvvm_mapa : NVPTXBuiltinSMAndPTX<"void *(void *, int)", SM_90, PTX78>;
+def __nvvm_mapa_shared_cluster : NVPTXBuiltinSMAndPTX<"void address_space<3> *(void address_space<3> *, int)", SM_90, PTX78>;
+def __nvvm_getctarank : NVPTXBuiltinSMAndPTX<"int(void *)", SM_90, PTX78>;
+def __nvvm_getctarank_shared_cluster : NVPTXBuiltinSMAndPTX<"int(void address_space<3> *)", SM_90, PTX78>;
diff --git a/clang/include/clang/Basic/BuiltinsPPC.def b/clang/include/clang/Basic/BuiltinsPPC.def
index 161df386f00f0..bb7d54bbb793e 100644
--- a/clang/include/clang/Basic/BuiltinsPPC.def
+++ b/clang/include/clang/Basic/BuiltinsPPC.def
@@ -1138,5 +1138,6 @@ UNALIASED_CUSTOM_BUILTIN(mma_pmxvbf16ger2nn, "vW512*VVi15i15i3", true,
 // FIXME: Obviously incomplete.
 
 #undef BUILTIN
+#undef TARGET_BUILTIN
 #undef CUSTOM_BUILTIN
 #undef UNALIASED_CUSTOM_BUILTIN
diff --git a/clang/include/clang/Basic/BuiltinsRISCVVector.def b/clang/include/clang/Basic/BuiltinsRISCVVector.def
deleted file mode 100644
index 6dfa87a1a1d31..0000000000000
--- a/clang/include/clang/Basic/BuiltinsRISCVVector.def
+++ /dev/null
@@ -1,22 +0,0 @@
-//==- BuiltinsRISCVVector.def - RISC-V Vector Builtin Database ---*- C++ -*-==//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-//
-// This file defines the RISC-V-specific builtin function database.  Users of
-// this file must define the BUILTIN macro to make use of this information.
-//
-//===----------------------------------------------------------------------===//
-
-#if defined(BUILTIN) && !defined(TARGET_BUILTIN)
-#   define TARGET_BUILTIN(ID, TYPE, ATTRS, FEATURE) BUILTIN(ID, TYPE, ATTRS)
-#endif
-
-#include "clang/Basic/riscv_vector_builtins.inc"
-#include "clang/Basic/riscv_sifive_vector_builtins.inc"
-
-#undef BUILTIN
-#undef TARGET_BUILTIN
diff --git a/clang/include/clang/Basic/BuiltinsSME.def b/clang/include/clang/Basic/BuiltinsSME.def
deleted file mode 100644
index 180ee20295ccd..0000000000000
--- a/clang/include/clang/Basic/BuiltinsSME.def
+++ /dev/null
@@ -1,21 +0,0 @@
-//===--- BuiltinsSME.def - SME Builtin function database --------*- C++ -*-===//
-//
-//  Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-//  See https://llvm.org/LICENSE.txt for license information.
-//  SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-//
-// This file defines the SME-specific builtin function database.  Users of
-// this file must define the BUILTIN macro to make use of this information.
-//
-//===----------------------------------------------------------------------===//
-
-// The format of this database matches clang/Basic/Builtins.def.
-
-#define GET_SME_BUILTINS
-#include "clang/Basic/arm_sme_builtins.inc"
-#undef GET_SME_BUILTINS
-
-#undef BUILTIN
-#undef TARGET_BUILTIN
diff --git a/clang/include/clang/Basic/BuiltinsSVE.def b/clang/include/clang/Basic/BuiltinsSVE.def
deleted file mode 100644
index a83f1c8f82dd0..0000000000000
--- a/clang/include/clang/Basic/BuiltinsSVE.def
+++ /dev/null
@@ -1,22 +0,0 @@
-//===--- BuiltinsSVE.def - SVE Builtin function database --------*- C++ -*-===//
-//
-//  Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-//  See https://llvm.org/LICENSE.txt for license information.
-//  SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-//
-// This file defines the SVE-specific builtin function database.  Users of
-// this file must define the BUILTIN macro to make use of this information.
-//
-//===----------------------------------------------------------------------===//
-
-// The format of this database matches clang/Basic/Builtins.def.
-
-#define GET_SVE_BUILTINS
-#include "clang/Basic/arm_sve_builtins.inc"
-#include "clang/Basic/BuiltinsAArch64NeonSVEBridge.def"
-#undef GET_SVE_BUILTINS
-
-#undef BUILTIN
-#undef TARGET_BUILTIN
diff --git a/clang/include/clang/Basic/BuiltinsX86Base.td b/clang/include/clang/Basic/BuiltinsX86Base.td
index aca39c204516a..0d739ee0b0b69 100644
--- a/clang/include/clang/Basic/BuiltinsX86Base.td
+++ b/clang/include/clang/Basic/BuiltinsX86Base.td
@@ -12,10 +12,13 @@
 
 include "clang/Basic/BuiltinsBase.td"
 
+def X86Prefix : NamePrefix<"__builtin_ia32_">;
+
 class X86Builtin<string prototype> : TargetBuiltin {
-  let Spellings = ["__builtin_ia32_" # NAME];
+  let Spellings = [NAME];
   let Prototype = prototype;
   let EnableOpenCLLong = 1;
+  let RequiredNamePrefix = X86Prefix; // Adds a prefix to the name.
 }
 
 class X86NoPrefixBuiltin<string prototype> : TargetBuiltin {
diff --git a/clang/include/clang/Basic/CMakeLists.txt b/clang/include/clang/Basic/CMakeLists.txt
index 4103d2753abc5..93dbc9c8ca62f 100644
--- a/clang/include/clang/Basic/CMakeLists.txt
+++ b/clang/include/clang/Basic/CMakeLists.txt
@@ -72,6 +72,14 @@ clang_tablegen(BuiltinsBPF.inc -gen-clang-builtins
   SOURCE BuiltinsBPF.td
   TARGET ClangBuiltinsBPF)
 
+clang_tablegen(BuiltinsHexagon.inc -gen-clang-builtins
+  SOURCE BuiltinsHexagon.td
+  TARGET ClangBuiltinsHexagon)
+
+clang_tablegen(BuiltinsNVPTX.inc -gen-clang-builtins
+  SOURCE BuiltinsNVPTX.td
+  TARGET ClangBuiltinsNVPTX)
+
 clang_tablegen(BuiltinsRISCV.inc -gen-clang-builtins
   SOURCE BuiltinsRISCV.td
   TARGET ClangBuiltinsRISCV)
diff --git a/clang/include/clang/Basic/CodeGenOptions.def b/clang/include/clang/Basic/CodeGenOptions.def
index 1ab8c7fb4d3c3..a7f5f1abbb825 100644
--- a/clang/include/clang/Basic/CodeGenOptions.def
+++ b/clang/include/clang/Basic/CodeGenOptions.def
@@ -221,6 +221,7 @@ AFFECTING_VALUE_CODEGENOPT(OptimizationLevel, 2, 0) ///< The -O[0-3] option spec
 AFFECTING_VALUE_CODEGENOPT(OptimizeSize, 2, 0) ///< If -Os (==1) or -Oz (==2) is specified.
 
 CODEGENOPT(AtomicProfileUpdate , 1, 0) ///< Set -fprofile-update=atomic
+CODEGENOPT(ContinuousProfileSync, 1, 0) ///< Enable continuous instrumentation profiling
 /// Choose profile instrumenation kind or no instrumentation.
 ENUM_CODEGENOPT(ProfileInstr, ProfileInstrKind, 2, ProfileNone)
 /// Choose profile kind for PGO use compilation.
@@ -277,6 +278,7 @@ CODEGENOPT(SanitizeCfiICallNormalizeIntegers, 1, 0) ///< Normalize integer types
                                                     ///< CFI icall function signatures
 CODEGENOPT(SanitizeCfiCanonicalJumpTables, 1, 0) ///< Make jump table symbols canonical
                                                  ///< instead of creating a local jump table.
+CODEGENOPT(SanitizeKcfiArity, 1, 0) ///< Embed arity in KCFI patchable function prefix
 CODEGENOPT(SanitizeCoverageType, 2, 0) ///< Type of sanitizer coverage
                                        ///< instrumentation.
 CODEGENOPT(SanitizeCoverageIndirectCalls, 1, 0) ///< Enable sanitizer coverage
@@ -319,6 +321,7 @@ CODEGENOPT(TimePassesPerRun  , 1, 0) ///< Set when -ftime-report=per-pass-run is
 CODEGENOPT(TimeTrace         , 1, 0) ///< Set when -ftime-trace is enabled.
 VALUE_CODEGENOPT(TimeTraceGranularity, 32, 500) ///< Minimum time granularity (in microseconds),
                                                ///< traced by time profiler
+CODEGENOPT(InterchangeLoops  , 1, 0) ///< Run loop-interchange.
 CODEGENOPT(UnrollLoops       , 1, 0) ///< Control whether loops are unrolled.
 CODEGENOPT(RerollLoops       , 1, 0) ///< Control whether loops are rerolled.
 CODEGENOPT(NoUseJumpTables   , 1, 0) ///< Set when -fno-jump-tables is enabled.
@@ -393,6 +396,9 @@ CODEGENOPT(EnableTLSDESC, 1, 0)
 /// Bit size of immediate TLS offsets (0 == use the default).
 VALUE_CODEGENOPT(TLSSize, 8, 0)
 
+/// The types of variables that we will extend the live ranges of.
+ENUM_CODEGENOPT(ExtendVariableLiveness, ExtendVariableLivenessKind, 2, ExtendVariableLivenessKind::None)
+
 /// The default stack protector guard offset to use.
 VALUE_CODEGENOPT(StackProtectorGuardOffset, 32, INT_MAX)
 
@@ -462,6 +468,14 @@ ENUM_CODEGENOPT(ZeroCallUsedRegs, llvm::ZeroCallUsedRegs::ZeroCallUsedRegsKind,
 /// non-deleting destructors. (No effect on Microsoft ABI.)
 CODEGENOPT(CtorDtorReturnThis, 1, 0)
 
+/// Enables emitting Import Call sections on supported targets that can be used
+/// by the Windows kernel to enable import call optimization.
+CODEGENOPT(ImportCallOptimization, 1, 0)
+
+/// Controls whether we generate code for static linking of libclosure
+/// (BlocksRuntime) on Windows.
+CODEGENOPT(StaticClosure, 1, 0)
+
 /// FIXME: Make DebugOptions its own top-level .def file.
 #include "DebugOptions.def"
 
diff --git a/clang/include/clang/Basic/CodeGenOptions.h b/clang/include/clang/Basic/CodeGenOptions.h
index b64ad74d711c6..c531c656f42b7 100644
--- a/clang/include/clang/Basic/CodeGenOptions.h
+++ b/clang/include/clang/Basic/CodeGenOptions.h
@@ -95,6 +95,12 @@ class CodeGenOptions : public CodeGenOptionsBase {
     Embed_Marker    // Embed a marker as a placeholder for bitcode.
   };
 
+  enum class ExtendVariableLivenessKind {
+    None,
+    This,
+    All,
+  };
+
   enum InlineAsmDialectKind {
     IAD_ATT,
     IAD_Intel,
diff --git a/clang/include/clang/Basic/Diagnostic.h b/clang/include/clang/Basic/Diagnostic.h
index 510b782e35d06..848acce3c4f13 100644
--- a/clang/include/clang/Basic/Diagnostic.h
+++ b/clang/include/clang/Basic/Diagnostic.h
@@ -375,10 +375,12 @@ class DiagnosticsEngine : public RefCountedBase<DiagnosticsEngine> {
     // Map extensions to warnings or errors?
     diag::Severity ExtBehavior = diag::Severity::Ignored;
 
-    DiagState()
+    DiagnosticIDs &DiagIDs;
+
+    DiagState(DiagnosticIDs &DiagIDs)
         : IgnoreAllWarnings(false), EnableAllWarnings(false),
           WarningsAsErrors(false), ErrorsAsFatal(false),
-          SuppressSystemWarnings(false) {}
+          SuppressSystemWarnings(false), DiagIDs(DiagIDs) {}
 
     using iterator = llvm::DenseMap<unsigned, DiagnosticMapping>::iterator;
     using const_iterator =
@@ -893,6 +895,8 @@ class DiagnosticsEngine : public RefCountedBase<DiagnosticsEngine> {
   /// \param FormatString A fixed diagnostic format string that will be hashed
   /// and mapped to a unique DiagID.
   template <unsigned N>
+  // TODO: Deprecate this once all uses are removed from Clang.
+  // [[deprecated("Use a CustomDiagDesc instead of a Level")]]
   unsigned getCustomDiagID(Level L, const char (&FormatString)[N]) {
     return Diags->getCustomDiagID((DiagnosticIDs::Level)L,
                                   StringRef(FormatString, N - 1));
diff --git a/clang/include/clang/Basic/DiagnosticCategories.h b/clang/include/clang/Basic/DiagnosticCategories.h
index 14be326f7515f..839f8dee3ca89 100644
--- a/clang/include/clang/Basic/DiagnosticCategories.h
+++ b/clang/include/clang/Basic/DiagnosticCategories.h
@@ -21,11 +21,12 @@ namespace clang {
     };
 
     enum class Group {
-#define DIAG_ENTRY(GroupName, FlagNameOffset, Members, SubGroups, Docs)        \
-  GroupName,
+#define DIAG_ENTRY(GroupName, FlagNameOffset, Members, SubGroups, Docs)    \
+      GroupName,
 #include "clang/Basic/DiagnosticGroups.inc"
 #undef CATEGORY
 #undef DIAG_ENTRY
+      NUM_GROUPS
     };
   }  // end namespace diag
 }  // end namespace clang
diff --git a/clang/include/clang/Basic/DiagnosticCommonKinds.td b/clang/include/clang/Basic/DiagnosticCommonKinds.td
index f4a155bb00bb3..f26c906b46447 100644
--- a/clang/include/clang/Basic/DiagnosticCommonKinds.td
+++ b/clang/include/clang/Basic/DiagnosticCommonKinds.td
@@ -402,16 +402,6 @@ def note_file_misc_sloc_usage : Note<
 def err_module_format_unhandled : Error<
   "no handler registered for module format '%0'">, DefaultFatal;
 
-// TransformActions
-// TODO: Use a custom category name to distinguish rewriter errors.
-def err_mt_message : Error<"[rewriter] %0">, SuppressInSystemHeader;
-def warn_mt_message : Warning<"[rewriter] %0">;
-def note_mt_message : Note<"[rewriter] %0">;
-
-// ARCMigrate
-def warn_arcmt_nsalloc_realloc : Warning<"[rewriter] call returns pointer to GC managed memory; it will become unmanaged in ARC">;
-def err_arcmt_nsinvocation_ownership : Error<"NSInvocation's %0 is not safe to be used with an object with ownership other than __unsafe_unretained">;
-
 // API notes
 def err_apinotes_message : Error<"%0">;
 def warn_apinotes_message : Warning<"%0">, InGroup<DiagGroup<"apinotes">>;
diff --git a/clang/include/clang/Basic/DiagnosticDriverKinds.td b/clang/include/clang/Basic/DiagnosticDriverKinds.td
index 288786b8ce939..2e571ee33cafa 100644
--- a/clang/include/clang/Basic/DiagnosticDriverKinds.td
+++ b/clang/include/clang/Basic/DiagnosticDriverKinds.td
@@ -209,6 +209,8 @@ def err_drv_cannot_open_randomize_layout_seed_file : Error<
   "cannot read randomize layout seed file '%0'">;
 def err_drv_invalid_version_number : Error<
   "invalid version number in '%0'">;
+def err_drv_kcfi_arity_unsupported_target : Error<
+  "target '%0' is unsupported by -fsanitize-kcfi-arity">;
 def err_drv_no_linker_llvm_support : Error<
   "'%0': unable to pass LLVM bit-code files to linker">;
 def err_drv_no_ast_support : Error<
@@ -445,9 +447,6 @@ def warn_drv_clang_unsupported : Warning<
   "the clang compiler does not support '%0'">;
 def warn_drv_deprecated_arg : Warning<
   "argument '%0' is deprecated%select{|, use '%2' instead}1">, InGroup<Deprecated>;
-def warn_drv_deprecated_arg_no_relaxed_template_template_args : Warning<
-  "argument '-fno-relaxed-template-template-args' is deprecated">,
-  InGroup<DeprecatedNoRelaxedTemplateTemplateArgs>;
 def warn_drv_deprecated_arg_ofast : Warning<
   "argument '-Ofast' is deprecated; use '-O3 -ffast-math' for the same behavior,"
   " or '-O3' to enable only conforming optimizations">,
diff --git a/clang/include/clang/Basic/DiagnosticGroups.td b/clang/include/clang/Basic/DiagnosticGroups.td
index 209792f851b6a..05e39899e6f25 100644
--- a/clang/include/clang/Basic/DiagnosticGroups.td
+++ b/clang/include/clang/Basic/DiagnosticGroups.td
@@ -102,7 +102,6 @@ def EnumConversion : DiagGroup<"enum-conversion",
                                [EnumEnumConversion,
                                 EnumFloatConversion,
                                 EnumCompareConditional]>;
-def DeprecatedNoRelaxedTemplateTemplateArgs : DiagGroup<"deprecated-no-relaxed-template-template-args">;
 def DeprecatedOFast : DiagGroup<"deprecated-ofast">;
 def ObjCSignedCharBoolImplicitIntConversion :
   DiagGroup<"objc-signed-char-bool-implicit-int-conversion">;
@@ -230,7 +229,6 @@ def Deprecated : DiagGroup<"deprecated", [DeprecatedAnonEnumEnumConversion,
                                           DeprecatedLiteralOperator,
                                           DeprecatedPragma,
                                           DeprecatedRegister,
-                                          DeprecatedNoRelaxedTemplateTemplateArgs,
                                           DeprecatedOFast,
                                           DeprecatedThisCapture,
                                           DeprecatedType,
@@ -696,6 +694,36 @@ def SuspiciousMemaccess : DiagGroup<"suspicious-memaccess",
    NonTrivialMemaccess, MemsetTransposedArgs, SuspiciousBzero]>;
 def StaticInInline : DiagGroup<"static-in-inline">;
 def StaticLocalInInline : DiagGroup<"static-local-in-inline">;
+def UniqueObjectDuplication : DiagGroup<"unique-object-duplication"> {
+  code Documentation = [{
+Warns when objects which are supposed to be globally unique might get duplicated
+when built into a shared library.
+
+If an object with hidden visibility is built into a shared library, each instance
+of the library will get its own copy. This can cause very subtle bugs if there was
+only supposed to be one copy of the object in question: singletons aren't single,
+changes to one object won't affect the others, the object's initializer will run
+once per copy, etc.
+
+Specifically, this warning fires when it detects an object which:
+  1. Appears in a header file (so it might get compiled into multiple libaries), and
+  2. Has external linkage (otherwise it's supposed to be duplicated), and
+  3. Has hidden visibility.
+
+As well as one of the following:
+  1. The object is mutable, or
+  2. The object's initializer definitely has side effects.
+
+The warning is best resolved by making the object ``const`` (if possible), or by explicitly
+giving the object non-hidden visibility, e.g. using ``__attribute((visibility("default")))``.
+Note that all levels of a pointer variable must be constant; ``const int*`` will
+trigger the warning because the pointer itself is mutable.
+
+This warning is currently disabled on Windows since it uses import/export rules
+instead of visibility.
+}];
+}
+
 def GNUStaticFloatInit : DiagGroup<"gnu-static-float-init">;
 def StaticFloatInit : DiagGroup<"static-float-init", [GNUStaticFloatInit]>;
 // Allow differentiation between GNU statement expressions in a macro versus
@@ -1306,6 +1334,8 @@ def MicrosoftStaticAssert : DiagGroup<"microsoft-static-assert">;
 def MicrosoftInitFromPredefined : DiagGroup<"microsoft-init-from-predefined">;
 def MicrosoftStringLiteralFromPredefined : DiagGroup<
     "microsoft-string-literal-from-predefined">;
+def MicrosoftInlineOnNonFunction : DiagGroup<
+    "microsoft-inline-on-non-function">;
 
 // Aliases.
 def : DiagGroup<"msvc-include", [MicrosoftInclude]>;
@@ -1324,7 +1354,7 @@ def Microsoft : DiagGroup<"microsoft",
      MicrosoftConstInit, MicrosoftVoidPseudoDtor, MicrosoftAnonTag,
      MicrosoftCommentPaste, MicrosoftEndOfFile, MicrosoftStaticAssert,
      MicrosoftInitFromPredefined, MicrosoftStringLiteralFromPredefined,
-     MicrosoftInconsistentDllImport]>;
+     MicrosoftInconsistentDllImport, MicrosoftInlineOnNonFunction]>;
 
 def ClangClPch : DiagGroup<"clang-cl-pch">;
 
diff --git a/clang/include/clang/Basic/DiagnosticIDs.h b/clang/include/clang/Basic/DiagnosticIDs.h
index a051af327de28..b49185c3335d8 100644
--- a/clang/include/clang/Basic/DiagnosticIDs.h
+++ b/clang/include/clang/Basic/DiagnosticIDs.h
@@ -14,6 +14,7 @@
 #ifndef LLVM_CLANG_BASIC_DIAGNOSTICIDS_H
 #define LLVM_CLANG_BASIC_DIAGNOSTICIDS_H
 
+#include "clang/Basic/DiagnosticCategories.h"
 #include "clang/Basic/LLVM.h"
 #include "llvm/ADT/IntrusiveRefCntPtr.h"
 #include "llvm/ADT/StringRef.h"
@@ -84,7 +85,7 @@ namespace clang {
     /// to either Ignore (nothing), Remark (emit a remark), Warning
     /// (emit a warning) or Error (emit as an error).  It allows clients to
     /// map ERRORs to Error or Fatal (stop emitting diagnostics after this one).
-    enum class Severity {
+    enum class Severity : uint8_t {
       // NOTE: 0 means "uncomputed".
       Ignored = 1, ///< Do not present this diagnostic, ignore it.
       Remark = 2,  ///< Present this diagnostic as a remark.
@@ -181,13 +182,96 @@ class DiagnosticMapping {
 class DiagnosticIDs : public RefCountedBase<DiagnosticIDs> {
 public:
   /// The level of the diagnostic, after it has been through mapping.
-  enum Level {
-    Ignored, Note, Remark, Warning, Error, Fatal
+  enum Level : uint8_t { Ignored, Note, Remark, Warning, Error, Fatal };
+
+  // Diagnostic classes.
+  enum Class {
+    CLASS_INVALID = 0x00,
+    CLASS_NOTE = 0x01,
+    CLASS_REMARK = 0x02,
+    CLASS_WARNING = 0x03,
+    CLASS_EXTENSION = 0x04,
+    CLASS_ERROR = 0x05
+  };
+
+  static bool IsCustomDiag(diag::kind Diag) {
+    return Diag >= diag::DIAG_UPPER_LIMIT;
+  }
+
+  class CustomDiagDesc {
+    LLVM_PREFERRED_TYPE(diag::Severity)
+    unsigned DefaultSeverity : 3;
+    LLVM_PREFERRED_TYPE(Class)
+    unsigned DiagClass : 3;
+    LLVM_PREFERRED_TYPE(bool)
+    unsigned ShowInSystemHeader : 1;
+    LLVM_PREFERRED_TYPE(bool)
+    unsigned ShowInSystemMacro : 1;
+    LLVM_PREFERRED_TYPE(bool)
+    unsigned HasGroup : 1;
+    diag::Group Group;
+    std::string Description;
+
+    auto get_as_tuple() const {
+      return std::tuple(DefaultSeverity, DiagClass, ShowInSystemHeader,
+                        ShowInSystemMacro, HasGroup, Group,
+                        std::string_view{Description});
+    }
+
+  public:
+    CustomDiagDesc(diag::Severity DefaultSeverity, std::string Description,
+                   unsigned Class = CLASS_WARNING,
+                   bool ShowInSystemHeader = false,
+                   bool ShowInSystemMacro = false,
+                   std::optional<diag::Group> Group = std::nullopt)
+        : DefaultSeverity(static_cast<unsigned>(DefaultSeverity)),
+          DiagClass(Class), ShowInSystemHeader(ShowInSystemHeader),
+          ShowInSystemMacro(ShowInSystemMacro), HasGroup(Group != std::nullopt),
+          Group(Group.value_or(diag::Group{})),
+          Description(std::move(Description)) {}
+
+    std::optional<diag::Group> GetGroup() const {
+      if (HasGroup)
+        return Group;
+      return std::nullopt;
+    }
+
+    diag::Severity GetDefaultSeverity() const {
+      return static_cast<diag::Severity>(DefaultSeverity);
+    }
+
+    Class GetClass() const { return static_cast<Class>(DiagClass); }
+    std::string_view GetDescription() const { return Description; }
+    bool ShouldShowInSystemHeader() const { return ShowInSystemHeader; }
+
+    friend bool operator==(const CustomDiagDesc &lhs,
+                           const CustomDiagDesc &rhs) {
+      return lhs.get_as_tuple() == rhs.get_as_tuple();
+    }
+
+    friend bool operator<(const CustomDiagDesc &lhs,
+                          const CustomDiagDesc &rhs) {
+      return lhs.get_as_tuple() < rhs.get_as_tuple();
+    }
+  };
+
+  struct GroupInfo {
+    LLVM_PREFERRED_TYPE(diag::Severity)
+    unsigned Severity : 3;
+    LLVM_PREFERRED_TYPE(bool)
+    unsigned HasNoWarningAsError : 1;
   };
 
 private:
   /// Information for uniquing and looking up custom diags.
   std::unique_ptr<diag::CustomDiagInfo> CustomDiagInfo;
+  std::unique_ptr<GroupInfo[]> GroupInfos = []() {
+    auto GIs = std::make_unique<GroupInfo[]>(
+        static_cast<size_t>(diag::Group::NUM_GROUPS));
+    for (size_t i = 0; i != static_cast<size_t>(diag::Group::NUM_GROUPS); ++i)
+      GIs[i] = {{}, false};
+    return GIs;
+  }();
 
 public:
   DiagnosticIDs();
@@ -202,7 +286,35 @@ class DiagnosticIDs : public RefCountedBase<DiagnosticIDs> {
   // FIXME: Replace this function with a create-only facilty like
   // createCustomDiagIDFromFormatString() to enforce safe usage. At the time of
   // writing, nearly all callers of this function were invalid.
-  unsigned getCustomDiagID(Level L, StringRef FormatString);
+  unsigned getCustomDiagID(CustomDiagDesc Diag);
+
+  // TODO: Deprecate this once all uses are removed from LLVM
+  // [[deprecated("Use a CustomDiagDesc instead of a Level")]]
+  unsigned getCustomDiagID(Level Level, StringRef Message) {
+    return getCustomDiagID([&]() -> CustomDiagDesc {
+      switch (Level) {
+      case DiagnosticIDs::Level::Ignored:
+        return {diag::Severity::Ignored, std::string(Message), CLASS_WARNING,
+                /*ShowInSystemHeader*/ true, /*ShowInSystemMacro=*/true};
+      case DiagnosticIDs::Level::Note:
+        return {diag::Severity::Fatal, std::string(Message), CLASS_NOTE,
+                /*ShowInSystemHeader*/ true, /*ShowInSystemMacro=*/true};
+      case DiagnosticIDs::Level::Remark:
+        return {diag::Severity::Remark, std::string(Message), CLASS_REMARK,
+                /*ShowInSystemHeader*/ true, /*ShowInSystemMacro=*/true};
+      case DiagnosticIDs::Level::Warning:
+        return {diag::Severity::Warning, std::string(Message), CLASS_WARNING,
+                /*ShowInSystemHeader*/ true, /*ShowInSystemMacro=*/true};
+      case DiagnosticIDs::Level::Error:
+        return {diag::Severity::Error, std::string(Message), CLASS_ERROR,
+                /*ShowInSystemHeader*/ true, /*ShowInSystemMacro=*/true};
+      case DiagnosticIDs::Level::Fatal:
+        return {diag::Severity::Fatal, std::string(Message), CLASS_ERROR,
+                /*ShowInSystemHeader*/ true, /*ShowInSystemMacro=*/true};
+      }
+      llvm_unreachable("Fully covered switch above!");
+    }());
+  }
 
   //===--------------------------------------------------------------------===//
   // Diagnostic classification and reporting interfaces.
@@ -214,35 +326,36 @@ class DiagnosticIDs : public RefCountedBase<DiagnosticIDs> {
   /// Return true if the unmapped diagnostic levelof the specified
   /// diagnostic ID is a Warning or Extension.
   ///
-  /// This only works on builtin diagnostics, not custom ones, and is not
-  /// legal to call on NOTEs.
-  static bool isBuiltinWarningOrExtension(unsigned DiagID);
+  /// This is not legal to call on NOTEs.
+  bool isWarningOrExtension(unsigned DiagID) const;
 
   /// Return true if the specified diagnostic is mapped to errors by
   /// default.
-  static bool isDefaultMappingAsError(unsigned DiagID);
+  bool isDefaultMappingAsError(unsigned DiagID) const;
 
   /// Get the default mapping for this diagnostic.
-  static DiagnosticMapping getDefaultMapping(unsigned DiagID);
+  DiagnosticMapping getDefaultMapping(unsigned DiagID) const;
+
+  void initCustomDiagMapping(DiagnosticMapping &, unsigned DiagID);
 
-  /// Determine whether the given built-in diagnostic ID is a Note.
-  static bool isBuiltinNote(unsigned DiagID);
+  /// Determine whether the given diagnostic ID is a Note.
+  bool isNote(unsigned DiagID) const;
 
-  /// Determine whether the given built-in diagnostic ID is for an
+  /// Determine whether the given diagnostic ID is for an
   /// extension of some sort.
-  static bool isBuiltinExtensionDiag(unsigned DiagID) {
+  bool isExtensionDiag(unsigned DiagID) const {
     bool ignored;
-    return isBuiltinExtensionDiag(DiagID, ignored);
+    return isExtensionDiag(DiagID, ignored);
   }
 
-  /// Determine whether the given built-in diagnostic ID is for an
+  /// Determine whether the given diagnostic ID is for an
   /// extension of some sort, and whether it is enabled by default.
   ///
   /// This also returns EnabledByDefault, which is set to indicate whether the
   /// diagnostic is ignored by default (in which case -pedantic enables it) or
   /// treated as a warning/error by default.
   ///
-  static bool isBuiltinExtensionDiag(unsigned DiagID, bool &EnabledByDefault);
+  bool isExtensionDiag(unsigned DiagID, bool &EnabledByDefault) const;
 
   /// Given a group ID, returns the flag that toggles the group.
   /// For example, for Group::DeprecatedDeclarations, returns
@@ -252,19 +365,22 @@ class DiagnosticIDs : public RefCountedBase<DiagnosticIDs> {
   /// Given a diagnostic group ID, return its documentation.
   static StringRef getWarningOptionDocumentation(diag::Group GroupID);
 
+  void setGroupSeverity(StringRef Group, diag::Severity);
+  void setGroupNoWarningsAsError(StringRef Group, bool);
+
   /// Given a group ID, returns the flag that toggles the group.
   /// For example, for "deprecated-declarations", returns
   /// Group::DeprecatedDeclarations.
   static std::optional<diag::Group> getGroupForWarningOption(StringRef);
 
   /// Return the lowest-level group that contains the specified diagnostic.
-  static std::optional<diag::Group> getGroupForDiag(unsigned DiagID);
+  std::optional<diag::Group> getGroupForDiag(unsigned DiagID) const;
 
   /// Return the lowest-level warning option that enables the specified
   /// diagnostic.
   ///
   /// If there is no -Wfoo flag that controls the diagnostic, this returns null.
-  static StringRef getWarningOptionForDiag(unsigned DiagID);
+  StringRef getWarningOptionForDiag(unsigned DiagID);
 
   /// Return the category number that a specified \p DiagID belongs to,
   /// or 0 if no category.
@@ -365,6 +481,8 @@ class DiagnosticIDs : public RefCountedBase<DiagnosticIDs> {
   getDiagnosticSeverity(unsigned DiagID, SourceLocation Loc,
                         const DiagnosticsEngine &Diag) const LLVM_READONLY;
 
+  Class getDiagClass(unsigned DiagID) const;
+
   /// Used to report a diagnostic that is finally fully formed.
   ///
   /// \returns \c true if the diagnostic was emitted, \c false if it was
diff --git a/clang/include/clang/Basic/DiagnosticParseKinds.td b/clang/include/clang/Basic/DiagnosticParseKinds.td
index 3309f59a981fc..c513dab810d1f 100644
--- a/clang/include/clang/Basic/DiagnosticParseKinds.td
+++ b/clang/include/clang/Basic/DiagnosticParseKinds.td
@@ -1099,6 +1099,16 @@ def err_lambda_capture_misplaced_ellipsis : Error<
   "the name of the capture">;
 def err_lambda_capture_multiple_ellipses : Error<
   "multiple ellipses in pack capture">;
+def err_binding_multiple_ellipses : Error<
+  "multiple packs in structured binding declaration">;
+def note_previous_ellipsis : Note<
+  "previous binding pack specified here">;
+def ext_cxx_binding_pack : ExtWarn<
+  "structured binding packs are a C++2c extension ">,
+  InGroup<CXX26>;
+def warn_cxx23_compat_binding_pack : Warning<
+  "structured binding packs are incompatible with C++ standards before C++2c">,
+  InGroup<CXXPre26Compat>, DefaultIgnore;
 def err_capture_default_first : Error<
   "capture default must be first">;
 def ext_decl_attrs_on_lambda : ExtWarn<
diff --git a/clang/include/clang/Basic/DiagnosticSemaKinds.td b/clang/include/clang/Basic/DiagnosticSemaKinds.td
index 774e5484cfa0e..e78339ee924ff 100644
--- a/clang/include/clang/Basic/DiagnosticSemaKinds.td
+++ b/clang/include/clang/Basic/DiagnosticSemaKinds.td
@@ -82,11 +82,11 @@ def err_typecheck_converted_constant_expression_indirect : Error<
   "conversion from %0 to %1 in converted constant expression would "
   "bind reference to a temporary">;
 def err_expr_not_cce : Error<
-  "%select{case value|enumerator value|non-type template argument|"
+  "%select{case value|enumerator value|non-type template argument|non-type parameter of template template parameter|"
   "array size|explicit specifier argument|noexcept specifier argument|"
   "call to 'size()'|call to 'data()'}0 is not a constant expression">;
 def ext_cce_narrowing : ExtWarn<
-  "%select{case value|enumerator value|non-type template argument|"
+  "%select{case value|enumerator value|non-type template argument|non-type parameter of template template parameter|"
   "array size|explicit specifier argument|noexcept specifier argument|"
   "call to 'size()'|call to 'data()'}0 %select{cannot be narrowed from "
   "type %2 to %3|evaluates to %2, which cannot be narrowed to type %3}1">,
@@ -350,10 +350,11 @@ def warn_arm_interrupt_vfp_clobber : Warning<
   InGroup<DiagGroup<"arm-interrupt-vfp-clobber">>;
 def err_arm_interrupt_called : Error<
   "interrupt service routine cannot be called directly">;
-def warn_interrupt_attribute_invalid : Warning<
-   "%select{MIPS|MSP430|RISC-V}0 'interrupt' attribute only applies to "
-   "functions that have %select{no parameters|a 'void' return type}1">,
-   InGroup<IgnoredAttributes>;
+def warn_interrupt_signal_attribute_invalid : Warning<
+  "%select{MIPS|MSP430|RISC-V|AVR}0 '%select{interrupt|signal}1' "
+  "attribute only applies to functions that have "
+  "%select{no parameters|a 'void' return type}2">,
+  InGroup<IgnoredAttributes>;
 def warn_riscv_repeated_interrupt_attribute : Warning<
   "repeated RISC-V 'interrupt' attribute">, InGroup<IgnoredAttributes>;
 def note_riscv_repeated_interrupt_attribute : Note<
@@ -482,13 +483,15 @@ def ext_use_out_of_scope_declaration : ExtWarn<
   InGroup<DiagGroup<"out-of-scope-function">>;
 def err_inline_non_function : Error<
   "'inline' can only appear on functions%select{| and non-local variables}0">;
+def warn_ms_inline_non_function : ExtWarn<err_inline_non_function.Summary>,
+  InGroup<MicrosoftInlineOnNonFunction>;
 def err_noreturn_non_function : Error<
   "'_Noreturn' can only appear on functions">;
 def warn_qual_return_type : Warning<
   "'%0' type qualifier%s1 on return type %plural{1:has|:have}1 no effect">,
   InGroup<IgnoredQualifiers>, DefaultIgnore;
 def warn_qual_base_type : Warning<
-  "'%0' qualifier%s1 on base class type %2 have no effect">,
+  "'%0' qualifier%s1 on base class type %2 %plural{1:has|:have}1 no effect">,
   InGroup<IgnoredQualifiers>, DefaultIgnore;
 
 def warn_deprecated_redundant_constexpr_static_def : Warning<
@@ -2953,9 +2956,15 @@ def ext_constexpr_function_never_constant_expr : ExtWarn<
   "constant expression">, InGroup<DiagGroup<"invalid-constexpr">>, DefaultError;
 def err_attr_cond_never_constant_expr : Error<
   "%0 attribute expression never produces a constant expression">;
+def err_diagnose_if_unknown_warning : Error<"unknown warning group '%0'">;
 def err_diagnose_if_invalid_diagnostic_type : Error<
   "invalid diagnostic type for 'diagnose_if'; use \"error\" or \"warning\" "
   "instead">;
+def err_diagnose_if_unknown_option : Error<"unknown diagnostic option">;
+def err_diagnose_if_expected_equals : Error<
+  "expected '=' after diagnostic option">;
+def err_diagnose_if_unexpected_value : Error<
+  "unexpected value; use 'true' or 'false'">;
 def err_constexpr_body_no_return : Error<
   "no return statement in %select{constexpr|consteval}0 function">;
 def err_constexpr_return_missing_expr : Error<
@@ -3824,6 +3833,9 @@ def warn_type_attribute_wrong_type : Warning<
   "'%0' only applies to %select{function|pointer|"
   "Objective-C object or block pointer}1 types; type here is %2">,
   InGroup<IgnoredAttributes>;
+def warn_attribute_on_void_param: Warning<
+  "attribute %0 cannot be applied to a 'void' parameter">,
+  InGroup<IgnoredAttributes>;
 def err_type_attribute_wrong_type : Error<
   warn_type_attribute_wrong_type.Summary>;
 def warn_incomplete_encoded_type : Warning<
@@ -3864,6 +3876,9 @@ def err_sme_definition_using_za_in_non_sme_target : Error<
   "function using ZA state requires 'sme'">;
 def err_sme_definition_using_zt0_in_non_sme2_target : Error<
   "function using ZT0 state requires 'sme2'">;
+def err_sme_openmp_captured_region : Error<
+  "OpenMP captured regions are not yet supported in "
+  "%select{streaming functions|functions with ZA state|functions with ZT0 state}0">;
 def warn_sme_streaming_pass_return_vl_to_non_streaming : Warning<
   "%select{returning|passing}0 a VL-dependent argument %select{from|to}0 a function with a different"
   " streaming-mode is undefined behaviour when the streaming and non-streaming vector lengths are different at runtime">,
@@ -5936,6 +5951,9 @@ def warn_cxx23_pack_indexing : Warning<
   "pack indexing is incompatible with C++ standards before C++2c">,
   DefaultIgnore, InGroup<CXXPre26Compat>;
 
+def err_pack_outside_template : Error<
+  "pack declaration outside of template">;
+
 def err_fold_expression_packs_both_sides : Error<
   "binary fold expression has unexpanded parameter packs in both operands">;
 def err_fold_expression_empty : Error<
@@ -6099,6 +6117,8 @@ def note_deleted_special_member_class_subobject : Note<
   "destructor}5"
   "%select{||s||}4"
   "|is an ObjC pointer}6">;
+def note_default_constructed_field
+    : Note<"default constructed field %0 declared here">;
 def note_deleted_default_ctor_uninit_field : Note<
   "%select{default constructor of|constructor inherited by}0 "
   "%1 is implicitly deleted because field %2 of "
@@ -6153,6 +6173,15 @@ def warn_static_local_in_extern_inline : Warning<
 def note_convert_inline_to_static : Note<
   "use 'static' to give inline function %0 internal linkage">;
 
+def warn_possible_object_duplication_mutable : Warning<
+  "%0 may be duplicated when built into a shared library: "
+  "it is mutable, has hidden visibility, and external linkage">,
+  InGroup<UniqueObjectDuplication>, DefaultIgnore;
+def warn_possible_object_duplication_init : Warning<
+  "initializeation of %0 may run twice when built into a shared library: "
+  "it has hidden visibility and external linkage">,
+  InGroup<UniqueObjectDuplication>, DefaultIgnore;
+
 def ext_redefinition_of_typedef : ExtWarn<
   "redefinition of typedef %0 is a C11 feature">,
   InGroup<DiagGroup<"typedef-redefinition"> >;
@@ -9168,6 +9197,8 @@ def err_cuda_device_exceptions : Error<
 def err_dynamic_var_init : Error<
     "dynamic initialization is not supported for "
     "__device__, __constant__, __shared__, and __managed__ variables">;
+def err_cuda_ctor_dtor_attrs
+    : Error<"CUDA does not support global %0 for __device__ functions">;
 def err_shared_var_init : Error<
     "initialization is not supported for __shared__ variables">;
 def err_cuda_vla : Error<
@@ -10624,6 +10655,8 @@ def err_second_argument_to_cwsc_not_pointer : Error<
 
 def err_vector_incorrect_num_elements : Error<
   "%select{too many|too few}0 elements in vector %select{initialization|operand}3 (expected %1 elements, have %2)">;
+def err_invalid_even_odd_vector_element_count : Error<
+  "invalid element count of %0 in vector %select{initialization|operand}4 (expected an %select{even|odd}3 element count in the range of %1 and %2)">;
 def err_altivec_empty_initializer : Error<"expected initializer">;
 
 def err_invalid_neon_type_code : Error<
@@ -12889,6 +12922,48 @@ def err_acc_update_as_body
     : Error<"OpenACC 'update' construct may not appear in place of the "
             "statement following a%select{n if statement| while statement| do "
             "statement| switch statement| label statement}0">;
+def err_acc_invalid_atomic
+    : Error<"statement associated with OpenACC 'atomic%select{| "
+            "%1}0' directive is invalid">;
+def note_acc_atomic_expr_must_be
+    : Note<"expected "
+           "%enum_select<OACCAtomicExpr>{%Assign{assignment}|%UnaryCompAssign{"
+           "assignment, compound assignment, increment, or decrement}}0 "
+           "expression">;
+def note_acc_atomic_unsupported_unary_operator
+    : Note<"unary operator not supported, only increment and decrement "
+           "operations permitted">;
+def note_acc_atomic_unsupported_binary_operator
+    : Note<"binary operator not supported, only +, *, -, /, &, ^, |, <<, or >> "
+           "are permitted">;
+def note_acc_atomic_unsupported_compound_binary_operator
+    : Note<"compound binary operator not supported, only +=, *=, -=, /=, &=, "
+           "^=, |=, <<=, or >>= are permitted">;
+
+def note_acc_atomic_operand_lvalue_scalar
+    : Note<"%select{left |right |}0operand to "
+           "%enum_select<OACCAtomicOpKind>{%Assign{assignment}|%CompoundAssign{"
+           "compound assignment}|%Inc{increment}|"
+           "%Dec{decrement}}1 "
+           "expression must be "
+           "%enum_select<OACCLValScalar>{%LVal{an l-value}|%Scalar{of scalar "
+           "type (was %3)}}2">;
+def note_acc_atomic_too_many_stmts
+    : Note<"'atomic capture' with a compound statement only supports two "
+           "statements">;
+def note_acc_atomic_expected_binop : Note<"expected binary operation on right "
+                                          "hand side of assignment operator">;
+def note_acc_atomic_mismatch_operand
+    : Note<"left hand side of assignment operation('%0') must match one side "
+           "of the sub-operation on the right hand side('%1' and '%2')">;
+def note_acc_atomic_mismatch_compound_operand
+    : Note<"variable %select{|in unary expression|on right hand side of "
+           "assignment|on left hand side of assignment|on left hand side of "
+           "compound assignment|on left hand side of assignment}2('%3') must "
+           "match variable used %select{|in unary expression|on right hand "
+           "side of assignment|<not possible>|on left hand side of compound "
+           "assignment|on left hand side of assignment}0('%1') from the first "
+           "statement">;
 
 // AMDGCN builtins diagnostics
 def err_amdgcn_global_load_lds_size_invalid_value : Error<"invalid size value">;
diff --git a/clang/include/clang/Basic/FPOptions.def b/clang/include/clang/Basic/FPOptions.def
index 79f04c89c9fed..90428c3c73c8b 100644
--- a/clang/include/clang/Basic/FPOptions.def
+++ b/clang/include/clang/Basic/FPOptions.def
@@ -28,5 +28,5 @@ OPTION(FPEvalMethod, LangOptions::FPEvalMethodKind, 2, AllowApproxFunc)
 OPTION(Float16ExcessPrecision, LangOptions::ExcessPrecisionKind, 2, FPEvalMethod)
 OPTION(BFloat16ExcessPrecision, LangOptions::ExcessPrecisionKind, 2, Float16ExcessPrecision)
 OPTION(MathErrno, bool, 1, BFloat16ExcessPrecision)
-OPTION(ComplexRange, LangOptions::ComplexRangeKind, 2, MathErrno)
+OPTION(ComplexRange, LangOptions::ComplexRangeKind, 3, MathErrno)
 #undef OPTION
diff --git a/clang/include/clang/Basic/Features.def b/clang/include/clang/Basic/Features.def
index c82b6d9b5f6c1..e736b46411ed1 100644
--- a/clang/include/clang/Basic/Features.def
+++ b/clang/include/clang/Basic/Features.def
@@ -254,6 +254,7 @@ FEATURE(is_trivially_constructible, LangOpts.CPlusPlus)
 FEATURE(is_trivially_copyable, LangOpts.CPlusPlus)
 FEATURE(is_union, LangOpts.CPlusPlus)
 FEATURE(kcfi, LangOpts.Sanitize.has(SanitizerKind::KCFI))
+FEATURE(kcfi_arity, LangOpts.Sanitize.has(SanitizerKind::KCFI))
 FEATURE(modules, LangOpts.Modules)
 FEATURE(safe_stack, LangOpts.Sanitize.has(SanitizerKind::SafeStack))
 FEATURE(shadow_call_stack,
diff --git a/clang/include/clang/Basic/IdentifierTable.h b/clang/include/clang/Basic/IdentifierTable.h
index 33d1cdb46f108..0347880244a40 100644
--- a/clang/include/clang/Basic/IdentifierTable.h
+++ b/clang/include/clang/Basic/IdentifierTable.h
@@ -101,8 +101,9 @@ enum class InterestingIdentifier {
   NUM_OBJC_KEYWORDS_AND_NOTABLE_IDENTIFIERS,
 
   NotBuiltin,
-#define BUILTIN(ID, TYPE, ATTRS) BI##ID,
+#define GET_BUILTIN_ENUMERATORS
 #include "clang/Basic/Builtins.inc"
+#undef GET_BUILTIN_ENUMERATORS
   FirstTSBuiltin,
 
   NotInterestingIdentifier = 65534
@@ -1008,7 +1009,7 @@ class Selector {
   }
 
   const IdentifierInfo *getAsIdentifierInfo() const {
-    return InfoPtr.getPointer().dyn_cast<const IdentifierInfo *>();
+    return dyn_cast_if_present<const IdentifierInfo *>(InfoPtr.getPointer());
   }
 
   MultiKeywordSelector *getMultiKeywordSelector() const {
diff --git a/clang/include/clang/Basic/LangOptions.def b/clang/include/clang/Basic/LangOptions.def
index a980be853d53e..bfab0baa089cf 100644
--- a/clang/include/clang/Basic/LangOptions.def
+++ b/clang/include/clang/Basic/LangOptions.def
@@ -160,7 +160,6 @@ LANGOPT(GNUAsm            , 1, 1, "GNU-style inline assembly")
 LANGOPT(Coroutines        , 1, 0, "C++20 coroutines")
 LANGOPT(CoroAlignedAllocation, 1, 0, "prefer Aligned Allocation according to P2014 Option 2")
 LANGOPT(DllExportInlines  , 1, 1, "dllexported classes dllexport inline methods")
-LANGOPT(RelaxedTemplateTemplateArgs, 1, 1, "C++17 relaxed matching of template template arguments")
 LANGOPT(ExperimentalLibrary, 1, 0, "enable unstable and experimental library features")
 LANGOPT(RetainSubstTemplateTypeParmTypeAstNodes, 1, 0, "retain SubstTemplateTypeParmType nodes in the AST's representation of alias template specializations")
 
@@ -239,7 +238,7 @@ BENIGN_LANGOPT(NoSignedZero      , 1, 0, "Permit Floating Point optimization wit
 BENIGN_LANGOPT(AllowRecip        , 1, 0, "Permit Floating Point reciprocal")
 BENIGN_LANGOPT(ApproxFunc        , 1, 0, "Permit Floating Point approximation")
 
-ENUM_LANGOPT(ComplexRange, ComplexRangeKind, 2, CX_None, "Enable use of range reduction for complex arithmetics.")
+ENUM_LANGOPT(ComplexRange, ComplexRangeKind, 3, CX_None, "Enable use of range reduction for complex arithmetics.")
 
 BENIGN_LANGOPT(ObjCGCBitmapPrint , 1, 0, "printing of GC's bitmap layout for __weak/__strong ivars")
 
@@ -407,6 +406,7 @@ VALUE_LANGOPT(TrivialAutoVarInitMaxSize, 32, 0,
              "stop trivial automatic variable initialization if var size exceeds the specified size (in bytes). Must be greater than 0.")
 ENUM_LANGOPT(SignedOverflowBehavior, SignedOverflowBehaviorTy, 2, SOB_Undefined,
              "signed integer overflow handling")
+LANGOPT(PointerOverflowDefined, 1, 0, "make pointer overflow defined")
 ENUM_LANGOPT(ThreadModel  , ThreadModelKind, 2, ThreadModelKind::POSIX, "Thread Model")
 
 BENIGN_LANGOPT(ArrowDepth, 32, 256,
diff --git a/clang/include/clang/Basic/LangOptions.h b/clang/include/clang/Basic/LangOptions.h
index 949c8f5d448bc..651b3b67b1058 100644
--- a/clang/include/clang/Basic/LangOptions.h
+++ b/clang/include/clang/Basic/LangOptions.h
@@ -144,6 +144,7 @@ class LangOptionsBase {
     MSVC2019_5 = 1925,
     MSVC2019_8 = 1928,
     MSVC2022_3 = 1933,
+    MSVC2022_9 = 1939,
   };
 
   enum SYCLMajorVersion {
@@ -245,8 +246,15 @@ class LangOptionsBase {
     ///   construction vtable because it hasn't added 'type' as a substitution.
     ///   - Skip mangling enclosing class templates of member-like friend
     ///   function templates.
+    ///   - Ignore empty struct arguments in C++ mode for ARM, instead of
+    ///   passing them as if they had a size of 1 byte.
     Ver19,
 
+    /// Attempt to be ABI-compatible with code generated by Clang 20.0.x.
+    /// This causes clang to:
+    ///   - Incorrectly return C++ records in AVX registers on x86_64.
+    Ver20,
+
     /// Conform to the underlying platform's C and C++ ABIs as closely
     /// as we can.
     Latest
@@ -640,9 +648,12 @@ class LangOptions : public LangOptionsBase {
 
   // Define accessors/mutators for language options of enumeration type.
 #define LANGOPT(Name, Bits, Default, Description)
-#define ENUM_LANGOPT(Name, Type, Bits, Default, Description) \
-  Type get##Name() const { return static_cast<Type>(Name); } \
-  void set##Name(Type Value) { Name = static_cast<unsigned>(Value); }
+#define ENUM_LANGOPT(Name, Type, Bits, Default, Description)                   \
+  Type get##Name() const { return static_cast<Type>(Name); }                   \
+  void set##Name(Type Value) {                                                 \
+    assert(static_cast<unsigned>(Value) < (1u << Bits));                       \
+    Name = static_cast<unsigned>(Value);                                       \
+  }
 #include "clang/Basic/LangOptions.def"
 
   /// Are we compiling a module?
@@ -951,11 +962,14 @@ class FPOptions {
   void applyChanges(FPOptionsOverride FPO);
 
   // We can define most of the accessors automatically:
+  // TODO: consider enforcing the assertion that value fits within bits
+  // statically.
 #define OPTION(NAME, TYPE, WIDTH, PREVIOUS)                                    \
   TYPE get##NAME() const {                                                     \
     return static_cast<TYPE>((Value & NAME##Mask) >> NAME##Shift);             \
   }                                                                            \
   void set##NAME(TYPE value) {                                                 \
+    assert(storage_type(value) < (1u << WIDTH));                               \
     Value = (Value & ~NAME##Mask) | (storage_type(value) << NAME##Shift);      \
   }
 #include "clang/Basic/FPOptions.def"
diff --git a/clang/include/clang/Basic/OpenACCKinds.h b/clang/include/clang/Basic/OpenACCKinds.h
index 7fb76271826a6..c2d7732123ef2 100644
--- a/clang/include/clang/Basic/OpenACCKinds.h
+++ b/clang/include/clang/Basic/OpenACCKinds.h
@@ -171,9 +171,34 @@ enum class OpenACCAtomicKind : uint8_t {
   Write,
   Update,
   Capture,
-  Invalid,
+  None,
 };
 
+template <typename StreamTy>
+inline StreamTy &printOpenACCAtomicKind(StreamTy &Out, OpenACCAtomicKind AK) {
+  switch (AK) {
+  case OpenACCAtomicKind::Read:
+    return Out << "read";
+  case OpenACCAtomicKind::Write:
+    return Out << "write";
+  case OpenACCAtomicKind::Update:
+    return Out << "update";
+  case OpenACCAtomicKind::Capture:
+    return Out << "capture";
+  case OpenACCAtomicKind::None:
+    return Out << "<none>";
+  }
+  llvm_unreachable("unknown atomic kind");
+}
+inline const StreamingDiagnostic &operator<<(const StreamingDiagnostic &Out,
+                                             OpenACCAtomicKind AK) {
+  return printOpenACCAtomicKind(Out, AK);
+}
+inline llvm::raw_ostream &operator<<(llvm::raw_ostream &Out,
+                                     OpenACCAtomicKind AK) {
+  return printOpenACCAtomicKind(Out, AK);
+}
+
 /// Represents the kind of an OpenACC clause.
 enum class OpenACCClauseKind : uint8_t {
   /// 'finalize' clause, allowed on 'exit data' directive.
diff --git a/clang/include/clang/Basic/OpenMPKinds.h b/clang/include/clang/Basic/OpenMPKinds.h
index 3e5da2a6abc01..e80bce34a97e0 100644
--- a/clang/include/clang/Basic/OpenMPKinds.h
+++ b/clang/include/clang/Basic/OpenMPKinds.h
@@ -399,6 +399,14 @@ bool isOpenMPInformationalDirective(OpenMPDirectiveKind DKind);
 /// \return true - if the above condition is met for this directive
 /// otherwise - false.
 bool isOpenMPCapturingDirective(OpenMPDirectiveKind DKind);
+
+/// Checks if the specified directive is an order concurrent nestable
+/// directive that can be nested within region corresponding to construct
+/// on which order clause was specified with concurrent as ordering argument.
+/// \param DKind Specified directive.
+/// \return true - if the above condition is met for this directive
+/// otherwise - false.
+bool isOpenMPOrderConcurrentNestableDirective(OpenMPDirectiveKind DKind);
 }
 
 template <>
diff --git a/clang/include/clang/Basic/Sanitizers.h b/clang/include/clang/Basic/Sanitizers.h
index fc0576d452b17..3782deb5962bc 100644
--- a/clang/include/clang/Basic/Sanitizers.h
+++ b/clang/include/clang/Basic/Sanitizers.h
@@ -162,6 +162,11 @@ class SanitizerMaskCutoffs {
 
   void set(SanitizerMask K, double V);
   void clear(SanitizerMask K = SanitizerKind::All);
+
+  // Returns nullopt if all the values are zero.
+  // Otherwise, return value contains a vector of all the scaled values.
+  std::optional<std::vector<unsigned>>
+  getAllScaled(unsigned ScalingFactor) const;
 };
 
 struct SanitizerSet {
diff --git a/clang/include/clang/Basic/StmtNodes.td b/clang/include/clang/Basic/StmtNodes.td
index 53fc77bbbcecc..d47e0a8157fc6 100644
--- a/clang/include/clang/Basic/StmtNodes.td
+++ b/clang/include/clang/Basic/StmtNodes.td
@@ -163,6 +163,7 @@ def MaterializeTemporaryExpr : StmtNode<Expr>;
 def LambdaExpr : StmtNode<Expr>;
 def CXXFoldExpr : StmtNode<Expr>;
 def CXXParenListInitExpr: StmtNode<Expr>;
+def ResolvedUnexpandedPackExpr : StmtNode<Expr>;
 
 // C++ Coroutines expressions
 def CoroutineSuspendExpr : StmtNode<Expr, 1>;
@@ -318,6 +319,7 @@ def OpenACCInitConstruct : StmtNode<OpenACCConstructStmt>;
 def OpenACCShutdownConstruct : StmtNode<OpenACCConstructStmt>;
 def OpenACCSetConstruct : StmtNode<OpenACCConstructStmt>;
 def OpenACCUpdateConstruct : StmtNode<OpenACCConstructStmt>;
+def OpenACCAtomicConstruct : StmtNode<OpenACCAssociatedStmtConstruct>;
 
 // OpenACC Additional Expressions.
 def OpenACCAsteriskSizeExpr : StmtNode<Expr>;
diff --git a/clang/include/clang/Basic/TargetBuiltins.h b/clang/include/clang/Basic/TargetBuiltins.h
index 4dc8b24ed8ae6..4781054240b5b 100644
--- a/clang/include/clang/Basic/TargetBuiltins.h
+++ b/clang/include/clang/Basic/TargetBuiltins.h
@@ -26,30 +26,50 @@ namespace clang {
   namespace NEON {
   enum {
     LastTIBuiltin = clang::Builtin::FirstTSBuiltin - 1,
-#define BUILTIN(ID, TYPE, ATTRS) BI##ID,
-#define TARGET_BUILTIN(ID, TYPE, ATTRS, FEATURE) BI##ID,
-#include "clang/Basic/BuiltinsNEON.def"
+#define GET_NEON_BUILTIN_ENUMERATORS
+#include "clang/Basic/arm_neon.inc"
+    FirstFp16Builtin,
+    LastNeonBuiltin = FirstFp16Builtin - 1,
+#include "clang/Basic/arm_fp16.inc"
+#undef GET_NEON_BUILTIN_ENUMERATORS
     FirstTSBuiltin
   };
   }
 
   /// ARM builtins
   namespace ARM {
-    enum {
-      LastTIBuiltin = clang::Builtin::FirstTSBuiltin-1,
-      LastNEONBuiltin = NEON::FirstTSBuiltin - 1,
+  enum {
+    LastTIBuiltin = clang::Builtin::FirstTSBuiltin - 1,
+    LastNEONBuiltin = NEON::FirstTSBuiltin - 1,
+#define GET_MVE_BUILTIN_ENUMERATORS
+#include "clang/Basic/arm_mve_builtins.inc"
+#undef GET_MVE_BUILTIN_ENUMERATORS
+    FirstCDEBuiltin,
+    LastMVEBuiltin = FirstCDEBuiltin - 1,
+#define GET_CDE_BUILTIN_ENUMERATORS
+#include "clang/Basic/arm_cde_builtins.inc"
+#undef GET_CDE_BUILTIN_ENUMERATORS
+    FirstARMBuiltin,
+    LastCDEBuiltin = FirstARMBuiltin - 1,
 #define BUILTIN(ID, TYPE, ATTRS) BI##ID,
 #include "clang/Basic/BuiltinsARM.def"
-      LastTSBuiltin
-    };
+    LastTSBuiltin
+  };
   }
 
   namespace SVE {
   enum {
     LastNEONBuiltin = NEON::FirstTSBuiltin - 1,
-#define BUILTIN(ID, TYPE, ATTRS) BI##ID,
+#define GET_SVE_BUILTIN_ENUMERATORS
+#include "clang/Basic/arm_sve_builtins.inc"
+#undef GET_SVE_BUILTIN_ENUMERATORS
+    FirstNeonBridgeBuiltin,
+    LastSveBuiltin = FirstNeonBridgeBuiltin - 1,
+#define GET_SVE_BUILTINS
 #define TARGET_BUILTIN(ID, TYPE, ATTRS, FEATURE) BI##ID,
-#include "clang/Basic/BuiltinsSVE.def"
+#include "clang/Basic/BuiltinsAArch64NeonSVEBridge.def"
+#undef TARGET_BUILTIN
+#undef GET_SVE_BUILTINS
     FirstTSBuiltin,
   };
   }
@@ -57,9 +77,9 @@ namespace clang {
   namespace SME {
   enum {
     LastSVEBuiltin = SVE::FirstTSBuiltin - 1,
-#define BUILTIN(ID, TYPE, ATTRS) BI##ID,
-#define TARGET_BUILTIN(ID, TYPE, ATTRS, FEATURE) BI##ID,
-#include "clang/Basic/BuiltinsSME.def"
+#define GET_SME_BUILTIN_ENUMERATORS
+#include "clang/Basic/arm_sme_builtins.inc"
+#undef GET_SME_BUILTIN_ENUMERATORS
     FirstTSBuiltin,
   };
   }
@@ -83,8 +103,9 @@ namespace clang {
   namespace BPF {
   enum {
     LastTIBuiltin = clang::Builtin::FirstTSBuiltin - 1,
-  #define BUILTIN(ID, TYPE, ATTRS) BI##ID,
-  #include "clang/Basic/BuiltinsBPF.inc"
+#define GET_BUILTIN_ENUMERATORS
+#include "clang/Basic/BuiltinsBPF.inc"
+#undef GET_BUILTIN_ENUMERATORS
     LastTSBuiltin
   };
   }
@@ -101,12 +122,13 @@ namespace clang {
 
   /// NVPTX builtins
   namespace NVPTX {
-    enum {
-        LastTIBuiltin = clang::Builtin::FirstTSBuiltin-1,
-#define BUILTIN(ID, TYPE, ATTRS) BI##ID,
-#include "clang/Basic/BuiltinsNVPTX.def"
-        LastTSBuiltin
-    };
+  enum {
+    LastTIBuiltin = clang::Builtin::FirstTSBuiltin - 1,
+#define GET_BUILTIN_ENUMERATORS
+#include "clang/Basic/BuiltinsNVPTX.inc"
+#undef GET_BUILTIN_ENUMERATORS
+    LastTSBuiltin
+  };
   }
 
   /// AMDGPU builtins
@@ -123,8 +145,9 @@ namespace clang {
   namespace SPIRV {
   enum {
     LastTIBuiltin = clang::Builtin::FirstTSBuiltin - 1,
-#define BUILTIN(ID, TYPE, ATTRS) BI##ID,
+#define GET_BUILTIN_ENUMERATORS
 #include "clang/Basic/BuiltinsSPIRV.inc"
+#undef GET_BUILTIN_ENUMERATORS
     LastTSBuiltin
   };
   } // namespace SPIRV
@@ -133,12 +156,14 @@ namespace clang {
   namespace X86 {
   enum {
     LastTIBuiltin = clang::Builtin::FirstTSBuiltin - 1,
-#define BUILTIN(ID, TYPE, ATTRS) BI##ID,
+#define GET_BUILTIN_ENUMERATORS
 #include "clang/Basic/BuiltinsX86.inc"
+#undef GET_BUILTIN_ENUMERATORS
     FirstX86_64Builtin,
     LastX86CommonBuiltin = FirstX86_64Builtin - 1,
-#define BUILTIN(ID, TYPE, ATTRS) BI##ID,
+#define GET_BUILTIN_ENUMERATORS
 #include "clang/Basic/BuiltinsX86_64.inc"
+#undef GET_BUILTIN_ENUMERATORS
     LastTSBuiltin
   };
   }
@@ -156,8 +181,12 @@ namespace clang {
   namespace RISCVVector {
   enum {
     LastTIBuiltin = clang::Builtin::FirstTSBuiltin - 1,
-#define BUILTIN(ID, TYPE, ATTRS) BI##ID,
-#include "clang/Basic/BuiltinsRISCVVector.def"
+#define GET_RISCVV_BUILTIN_ENUMERATORS
+#include "clang/Basic/riscv_vector_builtins.inc"
+    FirstSiFiveBuiltin,
+    LastRVVBuiltin = FirstSiFiveBuiltin - 1,
+#include "clang/Basic/riscv_sifive_vector_builtins.inc"
+#undef GET_RISCVV_BUILTIN_ENUMERATORS
     FirstTSBuiltin,
   };
   }
@@ -168,8 +197,9 @@ namespace clang {
     LastTIBuiltin = clang::Builtin::FirstTSBuiltin - 1,
     FirstRVVBuiltin = clang::Builtin::FirstTSBuiltin,
     LastRVVBuiltin = RISCVVector::FirstTSBuiltin - 1,
-#define BUILTIN(ID, TYPE, ATTRS) BI##ID,
+#define GET_BUILTIN_ENUMERATORS
 #include "clang/Basic/BuiltinsRISCV.inc"
+#undef GET_BUILTIN_ENUMERATORS
     LastTSBuiltin
   };
   } // namespace RISCV
@@ -178,8 +208,16 @@ namespace clang {
   namespace LoongArch {
   enum {
     LastTIBuiltin = clang::Builtin::FirstTSBuiltin - 1,
-#define BUILTIN(ID, TYPE, ATTRS) BI##ID,
-#include "clang/Basic/BuiltinsLoongArch.def"
+#define TARGET_BUILTIN(ID, TYPE, ATTRS, FEATURE) BI##ID,
+#include "clang/Basic/BuiltinsLoongArchBase.def"
+    FirstLSXBuiltin,
+    LastBaseBuiltin = FirstLSXBuiltin - 1,
+#define TARGET_BUILTIN(ID, TYPE, ATTRS, FEATURE) BI##ID,
+#include "clang/Basic/BuiltinsLoongArchLSX.def"
+    FirstLASXBuiltin,
+    LastLSXBuiltin = FirstLASXBuiltin - 1,
+#define TARGET_BUILTIN(ID, TYPE, ATTRS, FEATURE) BI##ID,
+#include "clang/Basic/BuiltinsLoongArchLASX.def"
     LastTSBuiltin
   };
   } // namespace LoongArch
@@ -208,7 +246,8 @@ namespace clang {
       Float16,
       Float32,
       Float64,
-      BFloat16
+      BFloat16,
+      MFloat8
     };
 
     NeonTypeFlags(unsigned F) : Flags(F) {}
@@ -230,6 +269,7 @@ namespace clang {
       switch (getEltType()) {
       case Int8:
       case Poly8:
+      case MFloat8:
         return 8;
       case Int16:
       case Float16:
@@ -352,12 +392,13 @@ namespace clang {
 
   /// Hexagon builtins
   namespace Hexagon {
-    enum {
-        LastTIBuiltin = clang::Builtin::FirstTSBuiltin-1,
-#define BUILTIN(ID, TYPE, ATTRS) BI##ID,
-#include "clang/Basic/BuiltinsHexagon.def"
-        LastTSBuiltin
-    };
+  enum {
+    LastTIBuiltin = clang::Builtin::FirstTSBuiltin - 1,
+#define GET_BUILTIN_ENUMERATORS
+#include "clang/Basic/BuiltinsHexagon.inc"
+#undef GET_BUILTIN_ENUMERATORS
+    LastTSBuiltin
+  };
   }
 
   /// MIPS builtins
diff --git a/clang/include/clang/Basic/TargetInfo.h b/clang/include/clang/Basic/TargetInfo.h
index 43c09cf1f973e..070cc792ca7db 100644
--- a/clang/include/clang/Basic/TargetInfo.h
+++ b/clang/include/clang/Basic/TargetInfo.h
@@ -16,6 +16,7 @@
 
 #include "clang/Basic/AddressSpaces.h"
 #include "clang/Basic/BitmaskEnum.h"
+#include "clang/Basic/Builtins.h"
 #include "clang/Basic/CFProtectionOptions.h"
 #include "clang/Basic/CodeGenOptions.h"
 #include "clang/Basic/LLVM.h"
@@ -32,6 +33,7 @@
 #include "llvm/ADT/StringMap.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/ADT/StringSet.h"
+#include "llvm/ADT/StringTable.h"
 #include "llvm/Frontend/OpenMP/OMPGridValues.h"
 #include "llvm/IR/DerivedTypes.h"
 #include "llvm/Support/DataTypes.h"
@@ -1016,14 +1018,15 @@ class TargetInfo : public TransferrableTargetInfo,
   virtual void getTargetDefines(const LangOptions &Opts,
                                 MacroBuilder &Builder) const = 0;
 
-  /// Return information about target-specific builtins for
-  /// the current primary target, and info about which builtins are non-portable
-  /// across the current set of primary and secondary targets.
-  virtual ArrayRef<Builtin::Info> getTargetBuiltins() const = 0;
+  /// Return information about target-specific builtins for the current primary
+  /// target, and info about which builtins are non-portable across the current
+  /// set of primary and secondary targets.
+  virtual llvm::SmallVector<Builtin::InfosShard> getTargetBuiltins() const = 0;
 
   /// Returns target-specific min and max values VScale_Range.
   virtual std::optional<std::pair<unsigned, unsigned>>
-  getVScaleRange(const LangOptions &LangOpts) const {
+  getVScaleRange(const LangOptions &LangOpts,
+                 bool IsArmStreamingFunction) const {
     return std::nullopt;
   }
   /// The __builtin_clz* and __builtin_ctz* built-in
@@ -1468,6 +1471,7 @@ class TargetInfo : public TransferrableTargetInfo,
   /// specification
   virtual bool validateBranchProtection(StringRef Spec, StringRef Arch,
                                         BranchProtectionInfo &BPI,
+                                        const LangOptions &LO,
                                         StringRef &Err) const {
     Err = "";
     return false;
@@ -1658,7 +1662,7 @@ class TargetInfo : public TransferrableTargetInfo,
   // access target-specific GPU grid values that must be consistent between
   // host RTL (plugin), deviceRTL and clang.
   virtual const llvm::omp::GV &getGridValue() const {
-    llvm_unreachable("getGridValue not implemented on this target");
+    return llvm::omp::SPIRVGridValues;
   }
 
   /// Retrieve the name of the platform as it is used in the
diff --git a/clang/include/clang/Basic/TokenKinds.def b/clang/include/clang/Basic/TokenKinds.def
index 2c692c999bdff..397a5d95709fb 100644
--- a/clang/include/clang/Basic/TokenKinds.def
+++ b/clang/include/clang/Basic/TokenKinds.def
@@ -546,7 +546,6 @@ TYPE_TRAIT_1(__is_trivially_equality_comparable, IsTriviallyEqualityComparable,
 TYPE_TRAIT_1(__is_bounded_array, IsBoundedArray, KEYCXX)
 TYPE_TRAIT_1(__is_unbounded_array, IsUnboundedArray, KEYCXX)
 TYPE_TRAIT_1(__is_scoped_enum, IsScopedEnum, KEYCXX)
-TYPE_TRAIT_1(__is_referenceable, IsReferenceable, KEYCXX)
 TYPE_TRAIT_1(__can_pass_in_regs, CanPassInRegs, KEYCXX)
 TYPE_TRAIT_2(__reference_binds_to_temporary, ReferenceBindsToTemporary, KEYCXX)
 TYPE_TRAIT_2(__reference_constructs_from_temporary, ReferenceConstructsFromTemporary, KEYCXX)
@@ -707,7 +706,7 @@ ALIAS("__decltype"       , decltype     , KEYCXX)
 ALIAS("__imag__"         , __imag       , KEYALL)
 ALIAS("__inline"         , inline       , KEYALL)
 ALIAS("__inline__"       , inline       , KEYALL)
-ALIAS("__nullptr"        , nullptr      , KEYCXX)
+ALIAS("__nullptr"        , nullptr      , KEYALL)
 ALIAS("__real__"         , __real       , KEYALL)
 ALIAS("__restrict"       , restrict     , KEYALL)
 ALIAS("__restrict__"     , restrict     , KEYALL)
diff --git a/clang/include/clang/Basic/Version.inc.in b/clang/include/clang/Basic/Version.inc.in
index 370da8506d142..69c79d03861cd 100644
--- a/clang/include/clang/Basic/Version.inc.in
+++ b/clang/include/clang/Basic/Version.inc.in
@@ -4,3 +4,4 @@
 #define CLANG_VERSION_MAJOR_STRING "@CLANG_VERSION_MAJOR@"
 #define CLANG_VERSION_MINOR @CLANG_VERSION_MINOR@
 #define CLANG_VERSION_PATCHLEVEL @CLANG_VERSION_PATCHLEVEL@
+#define MAX_CLANG_ABI_COMPAT_VERSION @MAX_CLANG_ABI_COMPAT_VERSION@
diff --git a/clang/include/clang/Basic/arm_neon.td b/clang/include/clang/Basic/arm_neon.td
index ddc5391eb3fa2..3e73dd054933f 100644
--- a/clang/include/clang/Basic/arm_neon.td
+++ b/clang/include/clang/Basic/arm_neon.td
@@ -1299,7 +1299,7 @@ def VQTBX4_A64 : WInst<"vqtbx4", "..(4Q)U", "UccPcQUcQcQPc">;
 // NeonEmitter implicitly takes the cartesian product of the type string with
 // itself during generation so, unlike all other intrinsics, this one should
 // include *all* types, not just additional ones.
-def VVREINTERPRET : REINTERPRET_CROSS_SELF<"csilUcUsUiUlhfdPcPsPlQcQsQiQlQUcQUsQUiQUlQhQfQdQPcQPsQPlQPk"> {
+def VVREINTERPRET : REINTERPRET_CROSS_SELF<"csilUcUsUiUlmhfdPcPsPlQcQsQiQlQUcQUsQUiQUlQmQhQfQdQPcQPsQPlQPk"> {
   let ArchGuard = "defined(__aarch64__) || defined(__arm64ec__)";
   let BigEndianSafe = 1;
 }
@@ -2119,6 +2119,72 @@ let ArchGuard = "defined(__aarch64__)", TargetGuard = "lut" in {
   }
 }
 
+let ArchGuard = "defined(__aarch64__)", TargetGuard = "fp8,neon" in {
+  def VBF1CVT_BF16_MF8        : VInst<"vcvt1_bf16_mf8_fpm",      "(QB).V", "m">;
+  def VBF1CVT_LOW_BF16_MF8    : VInst<"vcvt1_low_bf16_mf8_fpm",  "B.V",    "Hm">;
+  def VBF2CVTL_BF16_MF8       : VInst<"vcvt2_bf16_mf8_fpm",      "(QB).V", "m">;
+  def VBF2CVTL_LOW_BF16_MF8   : VInst<"vcvt2_low_bf16_mf8_fpm",  "B.V",    "Hm">;
+  def VBF1CVTL2_HIGH_BF16_MF8 : VInst<"vcvt1_high_bf16_mf8_fpm", "B.V",    "Hm">;
+  def VBF2CVTL2_HIGH_BF16_MF8 : VInst<"vcvt2_high_bf16_mf8_fpm", "B.V",    "Hm">;
+}
+
+let ArchGuard = "defined(__aarch64__)", TargetGuard = "fp8,neon" in {
+  def VF1CVT_F16_MF8        : VInst<"vcvt1_f16_mf8_fpm",      "(>QF).V", "m">;
+  def VF1CVT_LOW_F16_MF8    : VInst<"vcvt1_low_f16_mf8_fpm",  "(>F).V",  "Hm">;
+  def VF2CVTL_F16_MF8       : VInst<"vcvt2_f16_mf8_fpm",      "(>QF).V", "m">;
+  def VF2CVTL_LOW_F16_MF8   : VInst<"vcvt2_low_f16_mf8_fpm",  "(>F).V",  "Hm">;
+  def VF1CVTL2_HIGH_F16_MF8 : VInst<"vcvt1_high_f16_mf8_fpm", "(>F).V",  "Hm">;
+  def VF2CVTL2_HIGH_F16_MF8 : VInst<"vcvt2_high_f16_mf8_fpm", "(>F).V",  "Hm">;
+
+  def VCVTN_LOW_F8_F32  : VInst<"vcvt_mf8_f32_fpm",      ".(>>QF)(>>QF)V",  "m">;
+  def VCVTN_HIGH_F8_F32 : VInst<"vcvt_high_mf8_f32_fpm", ".(q)(>>F)(>>F)V", "Hm">;
+  def VCVTN_F8_F16      : VInst<"vcvt_mf8_f16_fpm",      ".(>F)(>F)V",      "mQm">;
+}
+
+let ArchGuard = "defined(__aarch64__)", TargetGuard = "fp8dot2,neon" in {
+  def VDOT_F16_MF8 : VInst<"vdot_f16_mf8_fpm", "(>F)(>F)..V", "mQm">;
+
+  def VDOT_LANE_F16_MF8  : VInst<"vdot_lane_f16_mf8_fpm",  "(>F)(>F)..IV", "m", [ImmCheck<3, ImmCheck0_3, 0>]>;
+  def VDOT_LANEQ_F16_MF8 : VInst<"vdot_laneq_f16_mf8_fpm", "(>F)(>F).QIV", "m", [ImmCheck<3, ImmCheck0_7, 0>]>;
+
+  def VDOTQ_LANE_F16_MF8  : VInst<"vdot_lane_f16_mf8_fpm",  "(>F)(>F).qIV", "Qm", [ImmCheck<3, ImmCheck0_3, 0>]>;
+  def VDOTQ_LANEQ_F16_MF8 : VInst<"vdot_laneq_f16_mf8_fpm", "(>F)(>F)..IV", "Qm", [ImmCheck<3, ImmCheck0_7, 0>]>;
+}
+
+let ArchGuard = "defined(__aarch64__)", TargetGuard = "fp8dot4,neon" in {
+  def VDOT_F32_MF8 : VInst<"vdot_f32_mf8_fpm", "(>>F)(>>F)..V", "mQm">;
+
+  def VDOT_LANE_F32_MF8  : VInst<"vdot_lane_f32_mf8_fpm",  "(>>F)(>>F)..IV", "m", [ImmCheck<3, ImmCheck0_1, 0>]>;
+  def VDOT_LANEQ_F32_MF8 : VInst<"vdot_laneq_f32_mf8_fpm", "(>>F)(>>F).QIV", "m", [ImmCheck<3, ImmCheck0_3, 0>]>;
+
+  def VDOTQ_LANE_F32_MF8  : VInst<"vdot_lane_f32_mf8_fpm",  "(>>F)(>>F).qIV", "Qm", [ImmCheck<3, ImmCheck0_1, 0>]>;
+  def VDOTQ_LANEQ_F32_MF8 : VInst<"vdot_laneq_f32_mf8_fpm", "(>>F)(>>F)..IV", "Qm", [ImmCheck<3, ImmCheck0_3, 0>]>;
+}
+
+let ArchGuard = "defined(__aarch64__)", TargetGuard = "fp8fma,neon" in {
+  def VMLALB_F16_F8 : VInst<"vmlalb_f16_mf8_fpm", "(>F)(>F)..V", "Qm">;
+  def VMLALT_F16_F8 : VInst<"vmlalt_f16_mf8_fpm", "(>F)(>F)..V", "Qm">;
+
+  def VMLALLBB_F32_F8 : VInst<"vmlallbb_f32_mf8_fpm", "(>>F)(>>F)..V", "Qm">;
+  def VMLALLBT_F32_F8 : VInst<"vmlallbt_f32_mf8_fpm", "(>>F)(>>F)..V", "Qm">;
+  def VMLALLTB_F32_F8 : VInst<"vmlalltb_f32_mf8_fpm", "(>>F)(>>F)..V", "Qm">;
+  def VMLALLTT_F32_F8 : VInst<"vmlalltt_f32_mf8_fpm", "(>>F)(>>F)..V", "Qm">;
+
+  def VMLALB_F16_F8_LANE  : VInst<"vmlalb_lane_f16_mf8_fpm",  "(>F)(>F).qIV", "Qm", [ImmCheck<3, ImmCheck0_7,  0>]>;
+  def VMLALB_F16_F8_LANEQ : VInst<"vmlalb_laneq_f16_mf8_fpm", "(>F)(>F)..IV", "Qm", [ImmCheck<3, ImmCheck0_15, 0>]>;
+  def VMLALT_F16_F8_LANE  : VInst<"vmlalt_lane_f16_mf8_fpm",  "(>F)(>F).qIV", "Qm", [ImmCheck<3, ImmCheck0_7,  0>]>;
+  def VMLALT_F16_F8_LANEQ : VInst<"vmlalt_laneq_f16_mf8_fpm", "(>F)(>F)..IV", "Qm", [ImmCheck<3, ImmCheck0_15, 0>]>;
+
+  def VMLALLBB_F32_F8_LANE  : VInst<"vmlallbb_lane_f32_mf8_fpm",  "(>>F)(>>F).qIV", "Qm", [ImmCheck<3, ImmCheck0_7,  0>]>;
+  def VMLALLBB_F32_F8_LANEQ : VInst<"vmlallbb_laneq_f32_mf8_fpm", "(>>F)(>>F)..IV", "Qm", [ImmCheck<3, ImmCheck0_15, 0>]>;
+  def VMLALLBT_F32_F8_LANE  : VInst<"vmlallbt_lane_f32_mf8_fpm",  "(>>F)(>>F).qIV", "Qm", [ImmCheck<3, ImmCheck0_7,  0>]>;
+  def VMLALLBT_F32_F8_LANEQ : VInst<"vmlallbt_laneq_f32_mf8_fpm", "(>>F)(>>F)..IV", "Qm", [ImmCheck<3, ImmCheck0_15, 0>]>;
+  def VMLALLTB_F32_F8_LANE  : VInst<"vmlalltb_lane_f32_mf8_fpm",  "(>>F)(>>F).qIV", "Qm", [ImmCheck<3, ImmCheck0_7,  0>]>;
+  def VMLALLTB_F32_F8_LANEQ : VInst<"vmlalltb_laneq_f32_mf8_fpm", "(>>F)(>>F)..IV", "Qm", [ImmCheck<3, ImmCheck0_15, 0>]>;
+  def VMLALLTT_F32_F8_LANE  : VInst<"vmlalltt_lane_f32_mf8_fpm",  "(>>F)(>>F).qIV", "Qm", [ImmCheck<3, ImmCheck0_7,  0>]>;
+  def VMLALLTT_F32_F8_LANEQ : VInst<"vmlalltt_laneq_f32_mf8_fpm", "(>>F)(>>F)..IV", "Qm", [ImmCheck<3, ImmCheck0_15, 0>]>;
+}
+
 let ArchGuard = "defined(__aarch64__)", TargetGuard = "neon,faminmax" in {
   def FAMIN : WInst<"vamin", "...", "fhQdQfQh">;
   def FAMAX : WInst<"vamax", "...", "fhQdQfQh">;
diff --git a/clang/include/clang/Basic/arm_neon_incl.td b/clang/include/clang/Basic/arm_neon_incl.td
index fd800e5a6278e..b9b9d509c2251 100644
--- a/clang/include/clang/Basic/arm_neon_incl.td
+++ b/clang/include/clang/Basic/arm_neon_incl.td
@@ -243,6 +243,7 @@ def OP_UNAVAILABLE : Operation {
 // B: change to BFloat16
 // P: change to polynomial category.
 // p: change polynomial to equivalent integer category. Otherwise nop.
+// V: change to fpm_t
 //
 // >: double element width (vector size unchanged).
 // <: half element width (vector size unchanged).
@@ -301,6 +302,7 @@ class Inst <string n, string p, string t, Operation o, list<ImmCheck> ch = []>{
 class SInst<string n, string p, string t, list<ImmCheck> ch = []> : Inst<n, p, t, OP_NONE, ch> {}
 class IInst<string n, string p, string t, list<ImmCheck> ch = []> : Inst<n, p, t, OP_NONE, ch> {}
 class WInst<string n, string p, string t, list<ImmCheck> ch = []> : Inst<n, p, t, OP_NONE, ch> {}
+class VInst<string n, string p, string t, list<ImmCheck> ch = []> : Inst<n, p, t, OP_NONE, ch> {}
 
 // The following instruction classes are implemented via operators
 // instead of builtins. As such these declarations are only used for
diff --git a/clang/include/clang/Basic/arm_sme.td b/clang/include/clang/Basic/arm_sme.td
index 891ed9874bb3d..288a8c04c217f 100644
--- a/clang/include/clang/Basic/arm_sme.td
+++ b/clang/include/clang/Basic/arm_sme.td
@@ -110,11 +110,11 @@ multiclass ZARead<string n_suffix, string t, string i_prefix, list<ImmCheck> ch>
   }
 }
 
-defm SVREAD_ZA8 : ZARead<"za8", "cUc", "aarch64_sme_read", [ImmCheck<2, ImmCheck0_0>]>;
+defm SVREAD_ZA8 : ZARead<"za8", "cUcm", "aarch64_sme_read", [ImmCheck<2, ImmCheck0_0>]>;
 defm SVREAD_ZA16 : ZARead<"za16", "sUshb", "aarch64_sme_read", [ImmCheck<2, ImmCheck0_1>]>;
 defm SVREAD_ZA32 : ZARead<"za32", "iUif", "aarch64_sme_read", [ImmCheck<2, ImmCheck0_3>]>;
 defm SVREAD_ZA64 : ZARead<"za64", "lUld", "aarch64_sme_read", [ImmCheck<2, ImmCheck0_7>]>;
-defm SVREAD_ZA128 : ZARead<"za128", "csilUcUsUiUlhbfd", "aarch64_sme_readq", [ImmCheck<2, ImmCheck0_15>]>;
+defm SVREAD_ZA128 : ZARead<"za128", "csilUcUsUiUlmhbfd", "aarch64_sme_readq", [ImmCheck<2, ImmCheck0_15>]>;
 
 ////////////////////////////////////////////////////////////////////////////////
 // Write horizontal/vertical ZA slices
@@ -131,11 +131,11 @@ multiclass ZAWrite<string n_suffix, string t, string i_prefix, list<ImmCheck> ch
   }
 }
 
-defm SVWRITE_ZA8 : ZAWrite<"za8", "cUc", "aarch64_sme_write", [ImmCheck<0, ImmCheck0_0>]>;
+defm SVWRITE_ZA8 : ZAWrite<"za8", "cUcm", "aarch64_sme_write", [ImmCheck<0, ImmCheck0_0>]>;
 defm SVWRITE_ZA16 : ZAWrite<"za16", "sUshb", "aarch64_sme_write", [ImmCheck<0, ImmCheck0_1>]>;
 defm SVWRITE_ZA32 : ZAWrite<"za32", "iUif", "aarch64_sme_write", [ImmCheck<0, ImmCheck0_3>]>;
 defm SVWRITE_ZA64 : ZAWrite<"za64", "lUld", "aarch64_sme_write", [ImmCheck<0, ImmCheck0_7>]>;
-defm SVWRITE_ZA128 : ZAWrite<"za128", "csilUcUsUiUlhbfd", "aarch64_sme_writeq", [ImmCheck<0, ImmCheck0_15>]>;
+defm SVWRITE_ZA128 : ZAWrite<"za128", "csilUcUsUiUlmhbfd", "aarch64_sme_writeq", [ImmCheck<0, ImmCheck0_15>]>;
 
 ////////////////////////////////////////////////////////////////////////////////
 // SME - Zero
@@ -350,7 +350,7 @@ multiclass ZAWrite_VG<string n, string t, string i, list<ImmCheck> checks> {
 }
 
 let SMETargetGuard = "sme2" in {
-  defm SVWRITE_ZA8  : ZAWrite_VG<"za8",  "cUc",   "aarch64_sme_write", [ImmCheck<0, ImmCheck0_0>]>;
+  defm SVWRITE_ZA8  : ZAWrite_VG<"za8",  "cUcm",   "aarch64_sme_write", [ImmCheck<0, ImmCheck0_0>]>;
   defm SVWRITE_ZA16 : ZAWrite_VG<"za16", "sUshb", "aarch64_sme_write", [ImmCheck<0, ImmCheck0_1>]>;
   defm SVWRITE_ZA32 : ZAWrite_VG<"za32", "iUif",  "aarch64_sme_write", [ImmCheck<0, ImmCheck0_3>]>;
   defm SVWRITE_ZA64 : ZAWrite_VG<"za64", "lUld",  "aarch64_sme_write", [ImmCheck<0, ImmCheck0_7>]>;
@@ -366,7 +366,7 @@ multiclass ZARead_VG<string n, string t, string i, list<ImmCheck> checks> {
 }
 
 let SMETargetGuard = "sme2" in {
-  defm SVREAD_ZA8  : ZARead_VG<"za8",  "cUc",   "aarch64_sme_read", [ImmCheck<0, ImmCheck0_0>]>;
+  defm SVREAD_ZA8  : ZARead_VG<"za8",  "cUcm",   "aarch64_sme_read", [ImmCheck<0, ImmCheck0_0>]>;
   defm SVREAD_ZA16 : ZARead_VG<"za16", "sUshb", "aarch64_sme_read", [ImmCheck<0, ImmCheck0_1>]>;
   defm SVREAD_ZA32 : ZARead_VG<"za32", "iUif",  "aarch64_sme_read", [ImmCheck<0, ImmCheck0_3>]>;
   defm SVREAD_ZA64 : ZARead_VG<"za64", "lUld",  "aarch64_sme_read", [ImmCheck<0, ImmCheck0_7>]>;
@@ -722,7 +722,7 @@ def IN_STREAMING_MODE :  Inst<"__arm_in_streaming_mode", "sv", "Pc", MergeNone,
 // lookup table expand four contiguous registers
 //
 let SMETargetGuard = "sme2" in {
-  def SVLUTI2_LANE_ZT_X4 : Inst<"svluti2_lane_zt_{d}_x4", "4.di[i", "cUcsUsiUibhf", MergeNone, "aarch64_sme_luti2_lane_zt_x4", [IsStreaming, IsInZT0], [ImmCheck<0, ImmCheck0_0>, ImmCheck<2, ImmCheck0_3>]>;
+  def SVLUTI2_LANE_ZT_X4 : Inst<"svluti2_lane_zt_{d}_x4", "4.di[i", "cUcsUsiUimbhf", MergeNone, "aarch64_sme_luti2_lane_zt_x4", [IsStreaming, IsInZT0], [ImmCheck<0, ImmCheck0_0>, ImmCheck<2, ImmCheck0_3>]>;
   def SVLUTI4_LANE_ZT_X4 : Inst<"svluti4_lane_zt_{d}_x4", "4.di[i", "sUsiUibhf", MergeNone, "aarch64_sme_luti4_lane_zt_x4", [IsStreaming, IsInZT0], [ImmCheck<0, ImmCheck0_0>, ImmCheck<2, ImmCheck0_1>]>;
 }
 
@@ -730,16 +730,16 @@ let SMETargetGuard = "sme2" in {
 // lookup table expand one register
 //
 let SMETargetGuard = "sme2" in {
-  def SVLUTI2_LANE_ZT : Inst<"svluti2_lane_zt_{d}", "di[i", "cUcsUsiUibhf", MergeNone, "aarch64_sme_luti2_lane_zt", [IsStreaming, IsInZT0], [ImmCheck<0, ImmCheck0_0>, ImmCheck<2, ImmCheck0_15>]>;
-  def SVLUTI4_LANE_ZT : Inst<"svluti4_lane_zt_{d}", "di[i", "cUcsUsiUibhf", MergeNone, "aarch64_sme_luti4_lane_zt", [IsStreaming, IsInZT0], [ImmCheck<0, ImmCheck0_0>, ImmCheck<2, ImmCheck0_7>]>;
+  def SVLUTI2_LANE_ZT : Inst<"svluti2_lane_zt_{d}", "di[i", "cUcsUsiUimbhf", MergeNone, "aarch64_sme_luti2_lane_zt", [IsStreaming, IsInZT0], [ImmCheck<0, ImmCheck0_0>, ImmCheck<2, ImmCheck0_15>]>;
+  def SVLUTI4_LANE_ZT : Inst<"svluti4_lane_zt_{d}", "di[i", "cUcsUsiUimbhf", MergeNone, "aarch64_sme_luti4_lane_zt", [IsStreaming, IsInZT0], [ImmCheck<0, ImmCheck0_0>, ImmCheck<2, ImmCheck0_7>]>;
 }
 
 //
 // lookup table expand two contiguous registers
 //
 let SMETargetGuard = "sme2" in {
-  def SVLUTI2_LANE_ZT_X2 : Inst<"svluti2_lane_zt_{d}_x2", "2.di[i", "cUcsUsiUibhf", MergeNone, "aarch64_sme_luti2_lane_zt_x2", [IsStreaming, IsInZT0], [ImmCheck<0, ImmCheck0_0>, ImmCheck<2, ImmCheck0_7>]>;
-  def SVLUTI4_LANE_ZT_X2 : Inst<"svluti4_lane_zt_{d}_x2", "2.di[i", "cUcsUsiUibhf", MergeNone, "aarch64_sme_luti4_lane_zt_x2", [IsStreaming, IsInZT0], [ImmCheck<0, ImmCheck0_0>, ImmCheck<2, ImmCheck0_3>]>;
+  def SVLUTI2_LANE_ZT_X2 : Inst<"svluti2_lane_zt_{d}_x2", "2.di[i", "cUcsUsiUimbhf", MergeNone, "aarch64_sme_luti2_lane_zt_x2", [IsStreaming, IsInZT0], [ImmCheck<0, ImmCheck0_0>, ImmCheck<2, ImmCheck0_7>]>;
+  def SVLUTI4_LANE_ZT_X2 : Inst<"svluti4_lane_zt_{d}_x2", "2.di[i", "cUcsUsiUimbhf", MergeNone, "aarch64_sme_luti4_lane_zt_x2", [IsStreaming, IsInZT0], [ImmCheck<0, ImmCheck0_0>, ImmCheck<2, ImmCheck0_3>]>;
 }
 
 //
@@ -748,30 +748,30 @@ let SMETargetGuard = "sme2" in {
 
 // FDOT
 let SMETargetGuard = "sme-f8f32" in {
-  def SVDOT_LANE_FP8_ZA32_VG1x2 : Inst<"svdot_lane_za32[_mf8]_vg1x2_fpm", "vm2di>", "m", MergeNone, "aarch64_sme_fp8_fdot_lane_za32_vg1x2", [IsStreaming, IsInOutZA, SetsFPMR, IsOverloadNone], [ImmCheck<3, ImmCheck0_3>]>;
-  def SVDOT_LANE_FP8_ZA32_VG1x4 : Inst<"svdot_lane_za32[_mf8]_vg1x4_fpm", "vm4di>", "m", MergeNone, "aarch64_sme_fp8_fdot_lane_za32_vg1x4", [IsStreaming, IsInOutZA, SetsFPMR, IsOverloadNone], [ImmCheck<3, ImmCheck0_3>]>;
+  def SVDOT_LANE_FP8_ZA32_VG1x2 : Inst<"svdot_lane_za32[_mf8]_vg1x2", "vm2di>", "m", MergeNone, "aarch64_sme_fp8_fdot_lane_za32_vg1x2", [IsStreaming, IsInOutZA, IsOverloadNone], [ImmCheck<3, ImmCheck0_3>]>;
+  def SVDOT_LANE_FP8_ZA32_VG1x4 : Inst<"svdot_lane_za32[_mf8]_vg1x4", "vm4di>", "m", MergeNone, "aarch64_sme_fp8_fdot_lane_za32_vg1x4", [IsStreaming, IsInOutZA, IsOverloadNone], [ImmCheck<3, ImmCheck0_3>]>;
 
-  def SVVDOTB_LANE_FP8_ZA32_VG1x4 : Inst<"svvdotb_lane_za32[_mf8]_vg1x4_fpm", "vm2di>", "m", MergeNone, "aarch64_sme_fp8_fvdotb_lane_za32_vg1x4", [IsOverloadNone, IsStreaming, IsInOutZA, SetsFPMR], [ImmCheck<3, ImmCheck0_3>]>;
-  def SVVDOTT_LANE_FP8_ZA32_VG1x4 : Inst<"svvdott_lane_za32[_mf8]_vg1x4_fpm", "vm2di>", "m", MergeNone, "aarch64_sme_fp8_fvdott_lane_za32_vg1x4", [IsOverloadNone, IsStreaming, IsInOutZA, SetsFPMR], [ImmCheck<3, ImmCheck0_3>]>;
+  def SVVDOTB_LANE_FP8_ZA32_VG1x4 : Inst<"svvdotb_lane_za32[_mf8]_vg1x4", "vm2di>", "m", MergeNone, "aarch64_sme_fp8_fvdotb_lane_za32_vg1x4", [IsOverloadNone, IsStreaming, IsInOutZA], [ImmCheck<3, ImmCheck0_3>]>;
+  def SVVDOTT_LANE_FP8_ZA32_VG1x4 : Inst<"svvdott_lane_za32[_mf8]_vg1x4", "vm2di>", "m", MergeNone, "aarch64_sme_fp8_fvdott_lane_za32_vg1x4", [IsOverloadNone, IsStreaming, IsInOutZA], [ImmCheck<3, ImmCheck0_3>]>;
 
-  def SVDOT_SINGLE_FP8_ZA32_VG1x2 : Inst<"svdot[_single]_za32[_mf8]_vg1x2_fpm", "vm2d>", "m", MergeNone, "aarch64_sme_fp8_fdot_single_za32_vg1x2", [IsStreaming, IsInOutZA, SetsFPMR, IsOverloadNone], []>;
-  def SVDOT_SINGLE_FP8_ZA32_VG1x4 : Inst<"svdot[_single]_za32[_mf8]_vg1x4_fpm", "vm4d>", "m", MergeNone, "aarch64_sme_fp8_fdot_single_za32_vg1x4", [IsStreaming, IsInOutZA, SetsFPMR, IsOverloadNone], []>;
+  def SVDOT_SINGLE_FP8_ZA32_VG1x2 : Inst<"svdot[_single]_za32[_mf8]_vg1x2", "vm2d>", "m", MergeNone, "aarch64_sme_fp8_fdot_single_za32_vg1x2", [IsStreaming, IsInOutZA, IsOverloadNone], []>;
+  def SVDOT_SINGLE_FP8_ZA32_VG1x4 : Inst<"svdot[_single]_za32[_mf8]_vg1x4", "vm4d>", "m", MergeNone, "aarch64_sme_fp8_fdot_single_za32_vg1x4", [IsStreaming, IsInOutZA, IsOverloadNone], []>;
 
-  def SVDOT_MULTI_FP8_ZA32_VG1x2 : Inst<"svdot_za32[_mf8]_vg1x2_fpm", "vm22>", "m", MergeNone, "aarch64_sme_fp8_fdot_multi_za32_vg1x2", [IsStreaming, IsInOutZA, SetsFPMR, IsOverloadNone], []>;
-  def SVDOT_MULTI_FP8_ZA32_VG1x4 : Inst<"svdot_za32[_mf8]_vg1x4_fpm", "vm44>", "m", MergeNone, "aarch64_sme_fp8_fdot_multi_za32_vg1x4", [IsStreaming, IsInOutZA, SetsFPMR, IsOverloadNone], []>;
+  def SVDOT_MULTI_FP8_ZA32_VG1x2 : Inst<"svdot_za32[_mf8]_vg1x2", "vm22>", "m", MergeNone, "aarch64_sme_fp8_fdot_multi_za32_vg1x2", [IsStreaming, IsInOutZA, IsOverloadNone], []>;
+  def SVDOT_MULTI_FP8_ZA32_VG1x4 : Inst<"svdot_za32[_mf8]_vg1x4", "vm44>", "m", MergeNone, "aarch64_sme_fp8_fdot_multi_za32_vg1x4", [IsStreaming, IsInOutZA, IsOverloadNone], []>;
 }
 
 let SMETargetGuard = "sme-f8f16" in {
-  def SVDOT_LANE_FP8_ZA16_VG1x2 : Inst<"svdot_lane_za16[_mf8]_vg1x2_fpm", "vm2di>", "m", MergeNone, "aarch64_sme_fp8_fdot_lane_za16_vg1x2", [IsStreaming, IsInOutZA, SetsFPMR, IsOverloadNone], [ImmCheck<3, ImmCheck0_7>]>;
-  def SVDOT_LANE_FP8_ZA16_VG1x4 : Inst<"svdot_lane_za16[_mf8]_vg1x4_fpm", "vm4di>", "m", MergeNone, "aarch64_sme_fp8_fdot_lane_za16_vg1x4", [IsStreaming, IsInOutZA, SetsFPMR, IsOverloadNone], [ImmCheck<3, ImmCheck0_7>]>;
+  def SVDOT_LANE_FP8_ZA16_VG1x2 : Inst<"svdot_lane_za16[_mf8]_vg1x2", "vm2di>", "m", MergeNone, "aarch64_sme_fp8_fdot_lane_za16_vg1x2", [IsStreaming, IsInOutZA, IsOverloadNone], [ImmCheck<3, ImmCheck0_7>]>;
+  def SVDOT_LANE_FP8_ZA16_VG1x4 : Inst<"svdot_lane_za16[_mf8]_vg1x4", "vm4di>", "m", MergeNone, "aarch64_sme_fp8_fdot_lane_za16_vg1x4", [IsStreaming, IsInOutZA, IsOverloadNone], [ImmCheck<3, ImmCheck0_7>]>;
 
-  def SVVDOT_LANE_FP8_ZA16_VG1x2 : Inst<"svvdot_lane_za16[_mf8]_vg1x2_fpm", "vm2di>", "m", MergeNone, "aarch64_sme_fp8_fvdot_lane_za16_vg1x2", [IsOverloadNone, IsStreaming, IsInOutZA, SetsFPMR], [ImmCheck<3, ImmCheck0_7>]>;
+  def SVVDOT_LANE_FP8_ZA16_VG1x2 : Inst<"svvdot_lane_za16[_mf8]_vg1x2", "vm2di>", "m", MergeNone, "aarch64_sme_fp8_fvdot_lane_za16_vg1x2", [IsOverloadNone, IsStreaming, IsInOutZA], [ImmCheck<3, ImmCheck0_7>]>;
 
-  def SVDOT_SINGLE_FP8_ZA16_VG1x2 : Inst<"svdot[_single]_za16[_mf8]_vg1x2_fpm", "vm2d>", "m", MergeNone, "aarch64_sme_fp8_fdot_single_za16_vg1x2", [IsStreaming, IsInOutZA, SetsFPMR, IsOverloadNone], []>;
-  def SVDOT_SINGLE_FP8_ZA16_VG1x4 : Inst<"svdot[_single]_za16[_mf8]_vg1x4_fpm", "vm4d>", "m", MergeNone, "aarch64_sme_fp8_fdot_single_za16_vg1x4", [IsStreaming, IsInOutZA, SetsFPMR, IsOverloadNone], []>;
+  def SVDOT_SINGLE_FP8_ZA16_VG1x2 : Inst<"svdot[_single]_za16[_mf8]_vg1x2", "vm2d>", "m", MergeNone, "aarch64_sme_fp8_fdot_single_za16_vg1x2", [IsStreaming, IsInOutZA, IsOverloadNone], []>;
+  def SVDOT_SINGLE_FP8_ZA16_VG1x4 : Inst<"svdot[_single]_za16[_mf8]_vg1x4", "vm4d>", "m", MergeNone, "aarch64_sme_fp8_fdot_single_za16_vg1x4", [IsStreaming, IsInOutZA, IsOverloadNone], []>;
 
-  def SVDOT_MULTI_FP8_ZA16_VG1x2 : Inst<"svdot_za16[_mf8]_vg1x2_fpm", "vm22>", "m", MergeNone, "aarch64_sme_fp8_fdot_multi_za16_vg1x2", [IsStreaming, IsInOutZA, SetsFPMR, IsOverloadNone], []>;
-  def SVDOT_MULTI_FP8_ZA16_VG1x4 : Inst<"svdot_za16[_mf8]_vg1x4_fpm", "vm44>", "m", MergeNone, "aarch64_sme_fp8_fdot_multi_za16_vg1x4", [IsStreaming, IsInOutZA, SetsFPMR, IsOverloadNone], []>;
+  def SVDOT_MULTI_FP8_ZA16_VG1x2 : Inst<"svdot_za16[_mf8]_vg1x2", "vm22>", "m", MergeNone, "aarch64_sme_fp8_fdot_multi_za16_vg1x2", [IsStreaming, IsInOutZA, IsOverloadNone], []>;
+  def SVDOT_MULTI_FP8_ZA16_VG1x4 : Inst<"svdot_za16[_mf8]_vg1x4", "vm44>", "m", MergeNone, "aarch64_sme_fp8_fdot_multi_za16_vg1x4", [IsStreaming, IsInOutZA, IsOverloadNone], []>;
 }
 
 ////////////////////////////////////////////////////////////////////////////////
@@ -811,12 +811,12 @@ multiclass ZAReadz<string n_suffix, string vg_num, string t, string i_prefix, li
   }
 }
 
-defm SVREADZ_ZA8_X2  : ZAReadz<"za8",  "2", "cUc",   "aarch64_sme_readz", [ImmCheck<0, ImmCheck0_0>]>;
+defm SVREADZ_ZA8_X2  : ZAReadz<"za8",  "2", "cUcm",   "aarch64_sme_readz", [ImmCheck<0, ImmCheck0_0>]>;
 defm SVREADZ_ZA16_X2 : ZAReadz<"za16", "2", "sUshb", "aarch64_sme_readz", [ImmCheck<0, ImmCheck0_1>]>;
 defm SVREADZ_ZA32_X2 : ZAReadz<"za32", "2", "iUif",  "aarch64_sme_readz", [ImmCheck<0, ImmCheck0_3>]>;
 defm SVREADZ_ZA64_X2 : ZAReadz<"za64", "2", "lUld",  "aarch64_sme_readz", [ImmCheck<0, ImmCheck0_7>]>;
 
-defm SVREADZ_ZA8_X4  : ZAReadz<"za8",  "4", "cUc",   "aarch64_sme_readz", [ImmCheck<0, ImmCheck0_0>]>;
+defm SVREADZ_ZA8_X4  : ZAReadz<"za8",  "4", "cUcm",   "aarch64_sme_readz", [ImmCheck<0, ImmCheck0_0>]>;
 defm SVREADZ_ZA16_X4 : ZAReadz<"za16", "4", "sUshb", "aarch64_sme_readz", [ImmCheck<0, ImmCheck0_1>]>;
 defm SVREADZ_ZA32_X4 : ZAReadz<"za32", "4", "iUif",  "aarch64_sme_readz", [ImmCheck<0, ImmCheck0_3>]>;
 defm SVREADZ_ZA64_X4 : ZAReadz<"za64", "4", "lUld",  "aarch64_sme_readz", [ImmCheck<0, ImmCheck0_7>]>;
@@ -834,15 +834,15 @@ multiclass ZAReadzSingle<string n_suffix, string t, string i_prefix, list<ImmChe
   }
 }
 
-defm SVREADZ_ZA8   : ZAReadzSingle<"za8", "cUc", "aarch64_sme_readz", [ImmCheck<0, ImmCheck0_0>]>;
+defm SVREADZ_ZA8   : ZAReadzSingle<"za8", "cUcm", "aarch64_sme_readz", [ImmCheck<0, ImmCheck0_0>]>;
 defm SVREADZ_ZA16  : ZAReadzSingle<"za16", "sUshb", "aarch64_sme_readz", [ImmCheck<0, ImmCheck0_1>]>;
 defm SVREADZ_ZA32  : ZAReadzSingle<"za32", "iUif", "aarch64_sme_readz", [ImmCheck<0, ImmCheck0_3>]>;
 defm SVREADZ_ZA64  : ZAReadzSingle<"za64", "lUld", "aarch64_sme_readz", [ImmCheck<0, ImmCheck0_7>]>;
-defm SVREADZ_ZA128 : ZAReadzSingle<"za128", "csilUcUiUsUlbhfd", "aarch64_sme_readz_q", [ImmCheck<0, ImmCheck0_15>]>;
+defm SVREADZ_ZA128 : ZAReadzSingle<"za128", "csilUcUiUsUlmbhfd", "aarch64_sme_readz_q", [ImmCheck<0, ImmCheck0_15>]>;
 
 multiclass ZAReadzArray<string vg_num>{
   let SMETargetGuard = "sme2p1" in {
-    def NAME # _B : SInst<"svreadz_za8_{d}_vg1x" # vg_num, vg_num # "m", "cUc", MergeNone, "aarch64_sme_readz_x" # vg_num, [IsStreaming, IsInOutZA]>;
+    def NAME # _B : SInst<"svreadz_za8_{d}_vg1x" # vg_num, vg_num # "m", "cUcm", MergeNone, "aarch64_sme_readz_x" # vg_num, [IsStreaming, IsInOutZA]>;
     def NAME # _H : SInst<"svreadz_za16_{d}_vg1x" # vg_num, vg_num # "m", "sUsbh", MergeNone, "aarch64_sme_readz_x" # vg_num, [IsStreaming, IsInOutZA]>;
     def NAME # _S : SInst<"svreadz_za32_{d}_vg1x" # vg_num, vg_num # "m", "iUif", MergeNone, "aarch64_sme_readz_x" # vg_num, [IsStreaming, IsInOutZA]>;
     def NAME # _D : SInst<"svreadz_za64_{d}_vg1x" # vg_num, vg_num # "m", "lUld", MergeNone, "aarch64_sme_readz_x" # vg_num, [IsStreaming, IsInOutZA]>;
@@ -859,51 +859,51 @@ let SMETargetGuard = "sme-lutv2" in {
 }
 
 let SMETargetGuard = "sme-f8f32" in {
-  def SVMOPA_FP8_ZA32 : Inst<"svmopa_za32[_mf8]_m_fpm", "viPPdd>", "m", MergeNone, "aarch64_sme_fp8_fmopa_za32",
-                             [IsStreaming, IsInOutZA, SetsFPMR, IsOverloadNone], [ImmCheck<0, ImmCheck0_3>]>;
+  def SVMOPA_FP8_ZA32 : Inst<"svmopa_za32[_mf8]_m", "viPPdd>", "m", MergeNone, "aarch64_sme_fp8_fmopa_za32",
+                             [IsStreaming, IsInOutZA, IsOverloadNone], [ImmCheck<0, ImmCheck0_3>]>;
    // FMLALL (indexed)
-  def SVMLA_FP8_LANE_ZA32_VG4x1 : Inst<"svmla_lane_za32[_mf8]_vg4x1_fpm", "vmddi>", "m", MergeNone, "aarch64_sme_fp8_fmlall_lane_za32_vg4x1",
-                                       [IsStreaming, IsInOutZA, SetsFPMR, IsOverloadNone], [ImmCheck<3, ImmCheck0_15>]>;
-  def SVMLA_FP8_LANE_ZA32_VG4x2 : Inst<"svmla_lane_za32[_mf8]_vg4x2_fpm", "vm2di>", "m", MergeNone, "aarch64_sme_fp8_fmlall_lane_za32_vg4x2",
-                                       [IsStreaming, IsInOutZA, SetsFPMR, IsOverloadNone], [ImmCheck<3, ImmCheck0_15>]>;
-  def SVMLA_FP8_LANE_ZA16_VG4x4 : Inst<"svmla_lane_za32[_mf8]_vg4x4_fpm", "vm4di>", "m", MergeNone, "aarch64_sme_fp8_fmlall_lane_za32_vg4x4",
-                                       [IsStreaming, IsInOutZA, SetsFPMR, IsOverloadNone], [ImmCheck<3, ImmCheck0_15>]>;
+  def SVMLA_FP8_LANE_ZA32_VG4x1 : Inst<"svmla_lane_za32[_mf8]_vg4x1", "vmddi>", "m", MergeNone, "aarch64_sme_fp8_fmlall_lane_za32_vg4x1",
+                                       [IsStreaming, IsInOutZA, IsOverloadNone], [ImmCheck<3, ImmCheck0_15>]>;
+  def SVMLA_FP8_LANE_ZA32_VG4x2 : Inst<"svmla_lane_za32[_mf8]_vg4x2", "vm2di>", "m", MergeNone, "aarch64_sme_fp8_fmlall_lane_za32_vg4x2",
+                                       [IsStreaming, IsInOutZA, IsOverloadNone], [ImmCheck<3, ImmCheck0_15>]>;
+  def SVMLA_FP8_LANE_ZA16_VG4x4 : Inst<"svmla_lane_za32[_mf8]_vg4x4", "vm4di>", "m", MergeNone, "aarch64_sme_fp8_fmlall_lane_za32_vg4x4",
+                                       [IsStreaming, IsInOutZA, IsOverloadNone], [ImmCheck<3, ImmCheck0_15>]>;
   // FMLALL (single)
-  def SVMLA_FP8_SINGLE_ZA32_VG4x1 : Inst<"svmla[_single]_za32[_mf8]_vg4x1_fpm", "vmdd>", "m", MergeNone, "aarch64_sme_fp8_fmlall_single_za32_vg4x1",
-                                         [IsStreaming, IsInOutZA, SetsFPMR, IsOverloadNone], []>;
-  def SVMLA_FP8_SINGLE_ZA32_VG4x2 : Inst<"svmla[_single]_za32[_mf8]_vg4x2_fpm", "vm2d>", "m", MergeNone, "aarch64_sme_fp8_fmlall_single_za32_vg4x2",
-                                         [IsStreaming, IsInOutZA, SetsFPMR, IsOverloadNone], []>;
-  def SVMLA_FP8_SINGLE_ZA32_VG4x4 : Inst<"svmla[_single]_za32[_mf8]_vg4x4_fpm", "vm4d>", "m", MergeNone, "aarch64_sme_fp8_fmlall_single_za32_vg4x4",
-                                         [IsStreaming, IsInOutZA, SetsFPMR, IsOverloadNone], []>;
+  def SVMLA_FP8_SINGLE_ZA32_VG4x1 : Inst<"svmla[_single]_za32[_mf8]_vg4x1", "vmdd>", "m", MergeNone, "aarch64_sme_fp8_fmlall_single_za32_vg4x1",
+                                         [IsStreaming, IsInOutZA, IsOverloadNone], []>;
+  def SVMLA_FP8_SINGLE_ZA32_VG4x2 : Inst<"svmla[_single]_za32[_mf8]_vg4x2", "vm2d>", "m", MergeNone, "aarch64_sme_fp8_fmlall_single_za32_vg4x2",
+                                         [IsStreaming, IsInOutZA, IsOverloadNone], []>;
+  def SVMLA_FP8_SINGLE_ZA32_VG4x4 : Inst<"svmla[_single]_za32[_mf8]_vg4x4", "vm4d>", "m", MergeNone, "aarch64_sme_fp8_fmlall_single_za32_vg4x4",
+                                         [IsStreaming, IsInOutZA, IsOverloadNone], []>;
   // FMLALL (multiple)
-  def SVMLA_FP8_MULTI_ZA32_VG4x2 : Inst<"svmla_za32[_mf8]_vg4x2_fpm", "vm22>", "m", MergeNone, "aarch64_sme_fp8_fmlall_multi_za32_vg4x2",
-                                        [IsStreaming, IsInOutZA, SetsFPMR, IsOverloadNone], []>;
-  def SVMLA_FP8_MULTI_ZA32_VG4x4 : Inst<"svmla_za32[_mf8]_vg4x4_fpm", "vm44>", "m", MergeNone, "aarch64_sme_fp8_fmlall_multi_za32_vg4x4",
-                                        [IsStreaming, IsInOutZA, SetsFPMR, IsOverloadNone], []>;
+  def SVMLA_FP8_MULTI_ZA32_VG4x2 : Inst<"svmla_za32[_mf8]_vg4x2", "vm22>", "m", MergeNone, "aarch64_sme_fp8_fmlall_multi_za32_vg4x2",
+                                        [IsStreaming, IsInOutZA, IsOverloadNone], []>;
+  def SVMLA_FP8_MULTI_ZA32_VG4x4 : Inst<"svmla_za32[_mf8]_vg4x4", "vm44>", "m", MergeNone, "aarch64_sme_fp8_fmlall_multi_za32_vg4x4",
+                                        [IsStreaming, IsInOutZA, IsOverloadNone], []>;
 }
 
 let SMETargetGuard = "sme-f8f16" in {
-  def SVMOPA_FP8_ZA16 : Inst<"svmopa_za16[_mf8]_m_fpm", "viPPdd>", "m", MergeNone, "aarch64_sme_fp8_fmopa_za16",
-                             [IsStreaming, IsInOutZA, SetsFPMR, IsOverloadNone], [ImmCheck<0, ImmCheck0_1>]>;
+  def SVMOPA_FP8_ZA16 : Inst<"svmopa_za16[_mf8]_m", "viPPdd>", "m", MergeNone, "aarch64_sme_fp8_fmopa_za16",
+                             [IsStreaming, IsInOutZA, IsOverloadNone], [ImmCheck<0, ImmCheck0_1>]>;
   // FMLAL (indexed)
-  def SVMLA_FP8_LANE_ZA16_VG2x1 : Inst<"svmla_lane_za16[_mf8]_vg2x1_fpm", "vmddi>", "m", MergeNone, "aarch64_sme_fp8_fmlal_lane_za16_vg2x1",
-                                       [IsStreaming, IsInOutZA, SetsFPMR, IsOverloadNone], [ImmCheck<3, ImmCheck0_15>]>;
-  def SVMLA_FP8_LANE_ZA16_VG2x2 : Inst<"svmla_lane_za16[_mf8]_vg2x2_fpm", "vm2di>", "m", MergeNone, "aarch64_sme_fp8_fmlal_lane_za16_vg2x2",
-                                       [IsStreaming, IsInOutZA, SetsFPMR, IsOverloadNone], [ImmCheck<3, ImmCheck0_15>]>;
-  def SVMLA_FP8_LANE_ZA16_VG2x4 : Inst<"svmla_lane_za16[_mf8]_vg2x4_fpm", "vm4di>", "m", MergeNone, "aarch64_sme_fp8_fmlal_lane_za16_vg2x4",
-                                       [IsStreaming, IsInOutZA, SetsFPMR, IsOverloadNone], [ImmCheck<3, ImmCheck0_15>]>;
+  def SVMLA_FP8_LANE_ZA16_VG2x1 : Inst<"svmla_lane_za16[_mf8]_vg2x1", "vmddi>", "m", MergeNone, "aarch64_sme_fp8_fmlal_lane_za16_vg2x1",
+                                       [IsStreaming, IsInOutZA, IsOverloadNone], [ImmCheck<3, ImmCheck0_15>]>;
+  def SVMLA_FP8_LANE_ZA16_VG2x2 : Inst<"svmla_lane_za16[_mf8]_vg2x2", "vm2di>", "m", MergeNone, "aarch64_sme_fp8_fmlal_lane_za16_vg2x2",
+                                       [IsStreaming, IsInOutZA, IsOverloadNone], [ImmCheck<3, ImmCheck0_15>]>;
+  def SVMLA_FP8_LANE_ZA16_VG2x4 : Inst<"svmla_lane_za16[_mf8]_vg2x4", "vm4di>", "m", MergeNone, "aarch64_sme_fp8_fmlal_lane_za16_vg2x4",
+                                       [IsStreaming, IsInOutZA, IsOverloadNone], [ImmCheck<3, ImmCheck0_15>]>;
   // FMLAL (single)
-  def SVMLA_FP8_SINGLE_ZA16_VG2x1 : Inst<"svmla[_single]_za16[_mf8]_vg2x1_fpm", "vmdd>", "m", MergeNone, "aarch64_sme_fp8_fmlal_single_za16_vg2x1",
-                                         [IsStreaming, IsInOutZA, SetsFPMR, IsOverloadNone], []>;
-  def SVMLA_FP8_SINGLE_ZA16_VG2x2 : Inst<"svmla[_single]_za16[_mf8]_vg2x2_fpm", "vm2d>", "m", MergeNone, "aarch64_sme_fp8_fmlal_single_za16_vg2x2",
-                                         [IsStreaming, IsInOutZA, SetsFPMR, IsOverloadNone], []>;
-  def SVMLA_FP8_SINGLE_ZA16_VG2x4 : Inst<"svmla[_single]_za16[_mf8]_vg2x4_fpm", "vm4d>", "m", MergeNone, "aarch64_sme_fp8_fmlal_single_za16_vg2x4",
-                                         [IsStreaming, IsInOutZA, SetsFPMR, IsOverloadNone], []>;
+  def SVMLA_FP8_SINGLE_ZA16_VG2x1 : Inst<"svmla[_single]_za16[_mf8]_vg2x1", "vmdd>", "m", MergeNone, "aarch64_sme_fp8_fmlal_single_za16_vg2x1",
+                                         [IsStreaming, IsInOutZA, IsOverloadNone], []>;
+  def SVMLA_FP8_SINGLE_ZA16_VG2x2 : Inst<"svmla[_single]_za16[_mf8]_vg2x2", "vm2d>", "m", MergeNone, "aarch64_sme_fp8_fmlal_single_za16_vg2x2",
+                                         [IsStreaming, IsInOutZA, IsOverloadNone], []>;
+  def SVMLA_FP8_SINGLE_ZA16_VG2x4 : Inst<"svmla[_single]_za16[_mf8]_vg2x4", "vm4d>", "m", MergeNone, "aarch64_sme_fp8_fmlal_single_za16_vg2x4",
+                                         [IsStreaming, IsInOutZA, IsOverloadNone], []>;
   // FMLAL (multiple)
-  def SVMLA_FP8_MULTI_ZA16_VG2x2 : Inst<"svmla_za16[_mf8]_vg2x2_fpm", "vm22>", "m", MergeNone, "aarch64_sme_fp8_fmlal_multi_za16_vg2x2",
-                                        [IsStreaming, IsInOutZA, SetsFPMR, IsOverloadNone], []>;
-  def SVMLA_FP8_MULTI_ZA16_VG2x4 : Inst<"svmla_za16[_mf8]_vg2x4_fpm", "vm44>", "m", MergeNone, "aarch64_sme_fp8_fmlal_multi_za16_vg2x4",
-                                        [IsStreaming, IsInOutZA, SetsFPMR, IsOverloadNone], []>;
+  def SVMLA_FP8_MULTI_ZA16_VG2x2 : Inst<"svmla_za16[_mf8]_vg2x2", "vm22>", "m", MergeNone, "aarch64_sme_fp8_fmlal_multi_za16_vg2x2",
+                                        [IsStreaming, IsInOutZA, IsOverloadNone], []>;
+  def SVMLA_FP8_MULTI_ZA16_VG2x4 : Inst<"svmla_za16[_mf8]_vg2x4", "vm44>", "m", MergeNone, "aarch64_sme_fp8_fmlal_multi_za16_vg2x4",
+                                        [IsStreaming, IsInOutZA, IsOverloadNone], []>;
 }
 
 } // let SVETargetGuard = InvalidMode
diff --git a/clang/include/clang/Basic/arm_sve.td b/clang/include/clang/Basic/arm_sve.td
index e7001bac450e8..b20383e72e66a 100644
--- a/clang/include/clang/Basic/arm_sve.td
+++ b/clang/include/clang/Basic/arm_sve.td
@@ -2104,7 +2104,7 @@ let SVETargetGuard = "sve2p1", SMETargetGuard = "sme" in {
 def SVSCLAMP : SInst<"svclamp[_{d}]", "dddd", "csil",     MergeNone, "aarch64_sve_sclamp", [VerifyRuntimeMode], []>;
 def SVUCLAMP : SInst<"svclamp[_{d}]", "dddd", "UcUsUiUl", MergeNone, "aarch64_sve_uclamp", [VerifyRuntimeMode], []>;
 
-defm SVREVD : SInstZPZ<"svrevd", "csilUcUsUiUlbhfd", "aarch64_sve_revd">;
+defm SVREVD : SInstZPZ<"svrevd", "csilUcUsUiUlmbhfd", "aarch64_sve_revd">;
 }
 
 let SVETargetGuard = "sve2p1", SMETargetGuard = "sme2" in {
@@ -2223,8 +2223,8 @@ let SVETargetGuard = InvalidMode, SMETargetGuard = "sme2" in {
   def SVADD_SINGLE_X4 : SInst<"svadd[_single_{d}_x4]", "44d", "cUcsUsiUilUl", MergeNone, "aarch64_sve_add_single_x4", [IsStreaming], []>;
 
   // 2-way and 4-way selects
-  def SVSEL_X2  : SInst<"svsel[_{d}_x2]", "2}22", "cUcsUsiUilUlbhfd", MergeNone, "aarch64_sve_sel_x2", [IsStreaming], []>;
-  def SVSEL_X4  : SInst<"svsel[_{d}_x4]", "4}44", "cUcsUsiUilUlbhfd", MergeNone, "aarch64_sve_sel_x4", [IsStreaming], []>;
+  def SVSEL_X2  : SInst<"svsel[_{d}_x2]", "2}22", "cUcsUsiUilUlmbhfd", MergeNone, "aarch64_sve_sel_x2", [IsStreaming], []>;
+  def SVSEL_X4  : SInst<"svsel[_{d}_x4]", "4}44", "cUcsUsiUilUlmbhfd", MergeNone, "aarch64_sve_sel_x4", [IsStreaming], []>;
 
   // SRSHL / URSHL
   def SVSRSHL_SINGLE_X2 : SInst<"svrshl[_single_{d}_x2]", "22d", "csil",     MergeNone, "aarch64_sve_srshl_single_x2", [IsStreaming], []>;
@@ -2402,15 +2402,15 @@ let SVETargetGuard = InvalidMode, SMETargetGuard = "sme2" in {
 //
 
 let SVETargetGuard = InvalidMode, SMETargetGuard = "sme2" in {
-  def SVZIP_X2  : SInst<"svzip[_{d}_x2]",  "22", "cUcsUsiUilUlbhfd", MergeNone, "aarch64_sve_zip_x2",  [IsStreaming], []>;
-  def SVZIPQ_X2 : SInst<"svzipq[_{d}_x2]", "22", "cUcsUsiUilUlbhfd", MergeNone, "aarch64_sve_zipq_x2", [IsStreaming], []>;
-  def SVZIP_X4  : SInst<"svzip[_{d}_x4]",  "44", "cUcsUsiUilUlbhfd", MergeNone, "aarch64_sve_zip_x4",  [IsStreaming], []>;
-  def SVZIPQ_X4 : SInst<"svzipq[_{d}_x4]", "44", "cUcsUsiUilUlbhfd", MergeNone, "aarch64_sve_zipq_x4", [IsStreaming], []>;
+  def SVZIP_X2  : SInst<"svzip[_{d}_x2]",  "22", "cUcsUsiUilUlmbhfd", MergeNone, "aarch64_sve_zip_x2",  [IsStreaming], []>;
+  def SVZIPQ_X2 : SInst<"svzipq[_{d}_x2]", "22", "cUcsUsiUilUlmbhfd", MergeNone, "aarch64_sve_zipq_x2", [IsStreaming], []>;
+  def SVZIP_X4  : SInst<"svzip[_{d}_x4]",  "44", "cUcsUsiUilUlmbhfd", MergeNone, "aarch64_sve_zip_x4",  [IsStreaming], []>;
+  def SVZIPQ_X4 : SInst<"svzipq[_{d}_x4]", "44", "cUcsUsiUilUlmbhfd", MergeNone, "aarch64_sve_zipq_x4", [IsStreaming], []>;
 
-  def SVUZP_X2  : SInst<"svuzp[_{d}_x2]",  "22", "cUcsUsiUilUlbhfd", MergeNone, "aarch64_sve_uzp_x2",  [IsStreaming], []>;
-  def SVUZPQ_X2 : SInst<"svuzpq[_{d}_x2]", "22", "cUcsUsiUilUlbhfd", MergeNone, "aarch64_sve_uzpq_x2", [IsStreaming], []>;
-  def SVUZP_X4  : SInst<"svuzp[_{d}_x4]",  "44", "cUcsUsiUilUlbhfd", MergeNone, "aarch64_sve_uzp_x4",  [IsStreaming], []>;
-  def SVUZPQ_X4 : SInst<"svuzpq[_{d}_x4]", "44", "cUcsUsiUilUlbhfd", MergeNone, "aarch64_sve_uzpq_x4", [IsStreaming], []>;
+  def SVUZP_X2  : SInst<"svuzp[_{d}_x2]",  "22", "cUcsUsiUilUlmbhfd", MergeNone, "aarch64_sve_uzp_x2",  [IsStreaming], []>;
+  def SVUZPQ_X2 : SInst<"svuzpq[_{d}_x2]", "22", "cUcsUsiUilUlmbhfd", MergeNone, "aarch64_sve_uzpq_x2", [IsStreaming], []>;
+  def SVUZP_X4  : SInst<"svuzp[_{d}_x4]",  "44", "cUcsUsiUilUlmbhfd", MergeNone, "aarch64_sve_uzp_x4",  [IsStreaming], []>;
+  def SVUZPQ_X4 : SInst<"svuzpq[_{d}_x4]", "44", "cUcsUsiUilUlmbhfd", MergeNone, "aarch64_sve_uzpq_x4", [IsStreaming], []>;
 }
 
 //
@@ -2432,18 +2432,18 @@ let SVETargetGuard = InvalidMode, SMETargetGuard = "sme2,fp8" in {
   def FSCALE_X4 : Inst<"svscale[_{d}_x4]", "444.x", "fhd", MergeNone, "aarch64_sme_fp8_scale_x4", [IsStreaming],[]>;
 
   // Convert from FP8 to half-precision/BFloat16 multi-vector
-  def SVF1CVT_X2 : Inst<"svcvt1_{d}[_mf8]_x2_fpm", "2~>", "bh", MergeNone, "aarch64_sve_fp8_cvt1_x2", [IsStreaming, SetsFPMR], []>;
-  def SVF2CVT_X2 : Inst<"svcvt2_{d}[_mf8]_x2_fpm", "2~>", "bh", MergeNone, "aarch64_sve_fp8_cvt2_x2", [IsStreaming, SetsFPMR], []>;
+  def SVF1CVT_X2 : Inst<"svcvt1_{d}[_mf8]_x2", "2~>", "bh", MergeNone, "aarch64_sve_fp8_cvt1_x2", [IsStreaming], []>;
+  def SVF2CVT_X2 : Inst<"svcvt2_{d}[_mf8]_x2", "2~>", "bh", MergeNone, "aarch64_sve_fp8_cvt2_x2", [IsStreaming], []>;
 
   // Convert from FP8 to deinterleaved half-precision/BFloat16 multi-vector
-  def SVF1CVTL_X2 : Inst<"svcvtl1_{d}[_mf8]_x2_fpm",  "2~>", "bh", MergeNone, "aarch64_sve_fp8_cvtl1_x2",  [IsStreaming, SetsFPMR], []>;
-  def SVF2CVTL_X2 : Inst<"svcvtl2_{d}[_mf8]_x2_fpm",  "2~>", "bh", MergeNone, "aarch64_sve_fp8_cvtl2_x2",  [IsStreaming, SetsFPMR], []>;
+  def SVF1CVTL_X2 : Inst<"svcvtl1_{d}[_mf8]_x2",  "2~>", "bh", MergeNone, "aarch64_sve_fp8_cvtl1_x2",  [IsStreaming], []>;
+  def SVF2CVTL_X2 : Inst<"svcvtl2_{d}[_mf8]_x2",  "2~>", "bh", MergeNone, "aarch64_sve_fp8_cvtl2_x2",  [IsStreaming], []>;
 
   // Convert from single/half/bfloat multivector to FP8
-  def SVFCVT_X2 : Inst<"svcvt_mf8[_{d}_x2]_fpm", "~2>", "bh", MergeNone, "aarch64_sve_fp8_cvt_x2", [IsStreaming, SetsFPMR], []>;
-  def SVFCVT_X4 : Inst<"svcvt_mf8[_{d}_x4]_fpm", "~4>", "f",  MergeNone, "aarch64_sve_fp8_cvt_x4", [IsOverloadNone, IsStreaming, SetsFPMR], []>;
+  def SVFCVT_X2 : Inst<"svcvt_mf8[_{d}_x2]", "~2>", "bh", MergeNone, "aarch64_sve_fp8_cvt_x2", [IsStreaming], []>;
+  def SVFCVT_X4 : Inst<"svcvt_mf8[_{d}_x4]", "~4>", "f",  MergeNone, "aarch64_sve_fp8_cvt_x4", [IsOverloadNone, IsStreaming], []>;
   // interleaved
-  def SVFCVTN_X4 : Inst<"svcvtn_mf8[_{d}_x4]_fpm", "~4>", "f", MergeNone, "aarch64_sve_fp8_cvtn_x4", [IsOverloadNone, IsStreaming, SetsFPMR], []>;
+  def SVFCVTN_X4 : Inst<"svcvtn_mf8[_{d}_x4]", "~4>", "f", MergeNone, "aarch64_sve_fp8_cvtn_x4", [IsOverloadNone, IsStreaming], []>;
 }
 
 let SVETargetGuard = "sve2p1", SMETargetGuard = "sme2" in {
@@ -2464,67 +2464,67 @@ let SVETargetGuard = "sve2,fp8", SMETargetGuard = "sme2,fp8" in {
   // SVE FP8 widening conversions
 
   // 8-bit floating-point convert to BFloat16/Float16
-  def SVF1CVT : SInst<"svcvt1_{d}[_mf8]_fpm", "d~>", "bh", MergeNone, "aarch64_sve_fp8_cvt1", [VerifyRuntimeMode, SetsFPMR]>;
-  def SVF2CVT : SInst<"svcvt2_{d}[_mf8]_fpm", "d~>", "bh", MergeNone, "aarch64_sve_fp8_cvt2", [VerifyRuntimeMode, SetsFPMR]>;
+  def SVF1CVT : SInst<"svcvt1_{d}[_mf8]", "d~>", "bh", MergeNone, "aarch64_sve_fp8_cvt1", [VerifyRuntimeMode]>;
+  def SVF2CVT : SInst<"svcvt2_{d}[_mf8]", "d~>", "bh", MergeNone, "aarch64_sve_fp8_cvt2", [VerifyRuntimeMode]>;
 
   // 8-bit floating-point convert to BFloat16/Float16 (top)
-  def SVF1CVTLT : SInst<"svcvtlt1_{d}[_mf8]_fpm", "d~>", "bh", MergeNone, "aarch64_sve_fp8_cvtlt1", [VerifyRuntimeMode, SetsFPMR]>;
-  def SVF2CVTLT : SInst<"svcvtlt2_{d}[_mf8]_fpm", "d~>", "bh", MergeNone, "aarch64_sve_fp8_cvtlt2", [VerifyRuntimeMode, SetsFPMR]>;
+  def SVF1CVTLT : SInst<"svcvtlt1_{d}[_mf8]", "d~>", "bh", MergeNone, "aarch64_sve_fp8_cvtlt1", [VerifyRuntimeMode]>;
+  def SVF2CVTLT : SInst<"svcvtlt2_{d}[_mf8]", "d~>", "bh", MergeNone, "aarch64_sve_fp8_cvtlt2", [VerifyRuntimeMode]>;
 
   // BFloat16/Float16 convert, narrow and interleave to 8-bit floating-point
-  def SVFCVTN : SInst<"svcvtn_mf8[_{d}_x2]_fpm", "~2>", "bh", MergeNone, "aarch64_sve_fp8_cvtn", [VerifyRuntimeMode, SetsFPMR]>;
+  def SVFCVTN : SInst<"svcvtn_mf8[_{d}_x2]", "~2>", "bh", MergeNone, "aarch64_sve_fp8_cvtn", [VerifyRuntimeMode]>;
 
   // Single-precision convert, narrow and interleave to 8-bit floating-point (top and bottom)
-  def SVFCVTNB : SInst<"svcvtnb_mf8[_f32_x2]_fpm", "~2>",  "f", MergeNone, "aarch64_sve_fp8_cvtnb", [VerifyRuntimeMode, SetsFPMR]>;
-  def SVFCVTNT : SInst<"svcvtnt_mf8[_f32_x2]_fpm", "~~2>", "f", MergeNone, "aarch64_sve_fp8_cvtnt", [VerifyRuntimeMode, SetsFPMR]>;
+  def SVFCVTNB : SInst<"svcvtnb_mf8[_f32_x2]", "~2>",  "f", MergeNone, "aarch64_sve_fp8_cvtnb", [VerifyRuntimeMode]>;
+  def SVFCVTNT : SInst<"svcvtnt_mf8[_f32_x2]", "~~2>", "f", MergeNone, "aarch64_sve_fp8_cvtnt", [VerifyRuntimeMode]>;
 }
 
 let SVETargetGuard = "sve2,fp8dot2", SMETargetGuard ="sme,ssve-fp8dot2" in {
   // 8-bit floating-point dot product to half-precision (vectors)
-  def SVFDOT_2WAY   :  SInst<"svdot[_f16_mf8]_fpm",   "dd~~>", "h", MergeNone, "aarch64_sve_fp8_fdot", [VerifyRuntimeMode, SetsFPMR]>;
-  def SVFDOT_N_2WAY :  SInst<"svdot[_n_f16_mf8]_fpm", "dd~!>", "h", MergeNone, "aarch64_sve_fp8_fdot", [VerifyRuntimeMode, SetsFPMR]>;
+  def SVFDOT_2WAY   :  SInst<"svdot[_f16_mf8]",   "dd~~>", "h", MergeNone, "aarch64_sve_fp8_fdot", [VerifyRuntimeMode]>;
+  def SVFDOT_N_2WAY :  SInst<"svdot[_n_f16_mf8]", "dd~!>", "h", MergeNone, "aarch64_sve_fp8_fdot", [VerifyRuntimeMode]>;
 
   // 8-bit floating-point dot product to half-precision (indexed)
-  def SVFDOT_LANE_2WAY :  SInst<"svdot_lane[_f16_mf8]_fpm", "dd~~i>", "h", MergeNone, "aarch64_sve_fp8_fdot_lane", [VerifyRuntimeMode, SetsFPMR], [ImmCheck<3, ImmCheck0_7>]>;
+  def SVFDOT_LANE_2WAY :  SInst<"svdot_lane[_f16_mf8]", "dd~~i>", "h", MergeNone, "aarch64_sve_fp8_fdot_lane", [VerifyRuntimeMode], [ImmCheck<3, ImmCheck0_7>]>;
 }
 
 let SVETargetGuard = "sve2,fp8dot4", SMETargetGuard ="sme,ssve-fp8dot4" in {
   // 8-bit floating-point dot product to single-precision (vectors)
-  def SVFDOT_4WAY   : SInst<"svdot[_f32_mf8]_fpm",   "dd~~>", "f", MergeNone, "aarch64_sve_fp8_fdot", [VerifyRuntimeMode, SetsFPMR]>;
-  def SVFDOT_N_4WAY : SInst<"svdot[_n_f32_mf8]_fpm", "dd~!>", "f", MergeNone, "aarch64_sve_fp8_fdot", [VerifyRuntimeMode, SetsFPMR]>;
+  def SVFDOT_4WAY   : SInst<"svdot[_f32_mf8]",   "dd~~>", "f", MergeNone, "aarch64_sve_fp8_fdot", [VerifyRuntimeMode]>;
+  def SVFDOT_N_4WAY : SInst<"svdot[_n_f32_mf8]", "dd~!>", "f", MergeNone, "aarch64_sve_fp8_fdot", [VerifyRuntimeMode]>;
 
   // 8-bit floating-point dot product to single-precision (indexed)
-  def SVFDOT_LANE_4WAY :  SInst<"svdot_lane[_f32_mf8]_fpm", "dd~~i>", "f", MergeNone, "aarch64_sve_fp8_fdot_lane", [VerifyRuntimeMode, SetsFPMR], [ImmCheck<3, ImmCheck0_3>]>;
+  def SVFDOT_LANE_4WAY :  SInst<"svdot_lane[_f32_mf8]", "dd~~i>", "f", MergeNone, "aarch64_sve_fp8_fdot_lane", [VerifyRuntimeMode], [ImmCheck<3, ImmCheck0_3>]>;
 }
 
 let SVETargetGuard = "sve2,fp8fma", SMETargetGuard = "sme,ssve-fp8fma" in {
   // 8-bit floating-point multiply-add long to half-precision (bottom)
-  def SVFMLALB   : SInst<"svmlalb[_f16_mf8]_fpm",   "dd~~>", "h", MergeNone, "aarch64_sve_fp8_fmlalb", [VerifyRuntimeMode, SetsFPMR]>;
-  def SVFMLALB_N : SInst<"svmlalb[_n_f16_mf8]_fpm", "dd~!>", "h", MergeNone, "aarch64_sve_fp8_fmlalb", [VerifyRuntimeMode, SetsFPMR]>;
+  def SVFMLALB   : SInst<"svmlalb[_f16_mf8]",   "dd~~>", "h", MergeNone, "aarch64_sve_fp8_fmlalb", [VerifyRuntimeMode]>;
+  def SVFMLALB_N : SInst<"svmlalb[_n_f16_mf8]", "dd~!>", "h", MergeNone, "aarch64_sve_fp8_fmlalb", [VerifyRuntimeMode]>;
 
-  // 8-bit floating-point multiply-add long to ha_fpmlf-precision (bottom, indexed)
-  def SVFMLALB_LANE : SInst<"svmlalb_lane[_f16_mf8]_fpm", "dd~~i>", "h", MergeNone, "aarch64_sve_fp8_fmlalb_lane", [VerifyRuntimeMode, SetsFPMR], [ImmCheck<3, ImmCheck0_15>]>;
+  // 8-bit floating-point multiply-add long to half-precision (bottom, indexed)
+  def SVFMLALB_LANE : SInst<"svmlalb_lane[_f16_mf8]", "dd~~i>", "h", MergeNone, "aarch64_sve_fp8_fmlalb_lane", [VerifyRuntimeMode], [ImmCheck<3, ImmCheck0_15>]>;
 
   // 8-bit floating-point multiply-add long to half-precision (top)
-  def SVFMLALT   : SInst<"svmlalt[_f16_mf8]_fpm",   "dd~~>", "h", MergeNone, "aarch64_sve_fp8_fmlalt", [VerifyRuntimeMode, SetsFPMR]>;
-  def SVFMLALT_N : SInst<"svmlalt[_n_f16_mf8]_fpm", "dd~!>", "h", MergeNone, "aarch64_sve_fp8_fmlalt", [VerifyRuntimeMode, SetsFPMR]>;
+  def SVFMLALT   : SInst<"svmlalt[_f16_mf8]",   "dd~~>", "h", MergeNone, "aarch64_sve_fp8_fmlalt", [VerifyRuntimeMode]>;
+  def SVFMLALT_N : SInst<"svmlalt[_n_f16_mf8]", "dd~!>", "h", MergeNone, "aarch64_sve_fp8_fmlalt", [VerifyRuntimeMode]>;
 
   // 8-bit floating-point multiply-add long to half-precision (top, indexed)
-  def SVFMLALT_LANE : SInst<"svmlalt_lane[_f16_mf8]_fpm", "dd~~i>", "h", MergeNone, "aarch64_sve_fp8_fmlalt_lane", [VerifyRuntimeMode, SetsFPMR], [ImmCheck<3, ImmCheck0_15>]>;
+  def SVFMLALT_LANE : SInst<"svmlalt_lane[_f16_mf8]", "dd~~i>", "h", MergeNone, "aarch64_sve_fp8_fmlalt_lane", [VerifyRuntimeMode], [ImmCheck<3, ImmCheck0_15>]>;
 
   // 8-bit floating-point multiply-add long long to single-precision (all top/bottom variants)
-  def SVFMLALLBB   : SInst<"svmlallbb[_f32_mf8]_fpm",   "dd~~>", "f", MergeNone, "aarch64_sve_fp8_fmlallbb", [VerifyRuntimeMode, SetsFPMR]>;
-  def SVFMLALLBB_N : SInst<"svmlallbb[_n_f32_mf8]_fpm", "dd~!>", "f", MergeNone, "aarch64_sve_fp8_fmlallbb", [VerifyRuntimeMode, SetsFPMR]>;
-  def SVFMLALLBT   : SInst<"svmlallbt[_f32_mf8]_fpm",   "dd~~>", "f", MergeNone, "aarch64_sve_fp8_fmlallbt", [VerifyRuntimeMode, SetsFPMR]>;
-  def SVFMLALLBT_N : SInst<"svmlallbt[_n_f32_mf8]_fpm", "dd~!>", "f", MergeNone, "aarch64_sve_fp8_fmlallbt", [VerifyRuntimeMode, SetsFPMR]>;
-  def SVFMLALLTB   : SInst<"svmlalltb[_f32_mf8]_fpm",   "dd~~>", "f", MergeNone, "aarch64_sve_fp8_fmlalltb", [VerifyRuntimeMode, SetsFPMR]>;
-  def SVFMLALLTB_N : SInst<"svmlalltb[_n_f32_mf8]_fpm", "dd~!>", "f", MergeNone, "aarch64_sve_fp8_fmlalltb", [VerifyRuntimeMode, SetsFPMR]>;
-  def SVFMLALLTT   : SInst<"svmlalltt[_f32_mf8]_fpm",   "dd~~>", "f", MergeNone, "aarch64_sve_fp8_fmlalltt", [VerifyRuntimeMode, SetsFPMR]>;
-  def SVFMLALLTT_N : SInst<"svmlalltt[_n_f32_mf8]_fpm", "dd~!>", "f", MergeNone, "aarch64_sve_fp8_fmlalltt", [VerifyRuntimeMode, SetsFPMR]>;
+  def SVFMLALLBB   : SInst<"svmlallbb[_f32_mf8]",   "dd~~>", "f", MergeNone, "aarch64_sve_fp8_fmlallbb", [VerifyRuntimeMode]>;
+  def SVFMLALLBB_N : SInst<"svmlallbb[_n_f32_mf8]", "dd~!>", "f", MergeNone, "aarch64_sve_fp8_fmlallbb", [VerifyRuntimeMode]>;
+  def SVFMLALLBT   : SInst<"svmlallbt[_f32_mf8]",   "dd~~>", "f", MergeNone, "aarch64_sve_fp8_fmlallbt", [VerifyRuntimeMode]>;
+  def SVFMLALLBT_N : SInst<"svmlallbt[_n_f32_mf8]", "dd~!>", "f", MergeNone, "aarch64_sve_fp8_fmlallbt", [VerifyRuntimeMode]>;
+  def SVFMLALLTB   : SInst<"svmlalltb[_f32_mf8]",   "dd~~>", "f", MergeNone, "aarch64_sve_fp8_fmlalltb", [VerifyRuntimeMode]>;
+  def SVFMLALLTB_N : SInst<"svmlalltb[_n_f32_mf8]", "dd~!>", "f", MergeNone, "aarch64_sve_fp8_fmlalltb", [VerifyRuntimeMode]>;
+  def SVFMLALLTT   : SInst<"svmlalltt[_f32_mf8]",   "dd~~>", "f", MergeNone, "aarch64_sve_fp8_fmlalltt", [VerifyRuntimeMode]>;
+  def SVFMLALLTT_N : SInst<"svmlalltt[_n_f32_mf8]", "dd~!>", "f", MergeNone, "aarch64_sve_fp8_fmlalltt", [VerifyRuntimeMode]>;
 
   // 8-bit floating-point multiply-add long long to single-precision (indexed, all top/bottom variants)
-  def SVFMLALLBB_LANE : SInst<"svmlallbb_lane[_f32_mf8]_fpm", "dd~~i>", "f", MergeNone, "aarch64_sve_fp8_fmlallbb_lane", [VerifyRuntimeMode, SetsFPMR], [ImmCheck<3, ImmCheck0_7>]>;
-  def SVFMLALLBT_LANE : SInst<"svmlallbt_lane[_f32_mf8]_fpm", "dd~~i>", "f", MergeNone, "aarch64_sve_fp8_fmlallbt_lane", [VerifyRuntimeMode, SetsFPMR], [ImmCheck<3, ImmCheck0_7>]>;
-  def SVFMLALLTB_LANE : SInst<"svmlalltb_lane[_f32_mf8]_fpm", "dd~~i>", "f", MergeNone, "aarch64_sve_fp8_fmlalltb_lane", [VerifyRuntimeMode, SetsFPMR], [ImmCheck<3, ImmCheck0_7>]>;
-  def SVFMLALLTT_LANE : SInst<"svmlalltt_lane[_f32_mf8]_fpm", "dd~~i>", "f", MergeNone, "aarch64_sve_fp8_fmlalltt_lane", [VerifyRuntimeMode, SetsFPMR], [ImmCheck<3, ImmCheck0_7>]>;
+  def SVFMLALLBB_LANE : SInst<"svmlallbb_lane[_f32_mf8]", "dd~~i>", "f", MergeNone, "aarch64_sve_fp8_fmlallbb_lane", [VerifyRuntimeMode], [ImmCheck<3, ImmCheck0_7>]>;
+  def SVFMLALLBT_LANE : SInst<"svmlallbt_lane[_f32_mf8]", "dd~~i>", "f", MergeNone, "aarch64_sve_fp8_fmlallbt_lane", [VerifyRuntimeMode], [ImmCheck<3, ImmCheck0_7>]>;
+  def SVFMLALLTB_LANE : SInst<"svmlalltb_lane[_f32_mf8]", "dd~~i>", "f", MergeNone, "aarch64_sve_fp8_fmlalltb_lane", [VerifyRuntimeMode], [ImmCheck<3, ImmCheck0_7>]>;
+  def SVFMLALLTT_LANE : SInst<"svmlalltt_lane[_f32_mf8]", "dd~~i>", "f", MergeNone, "aarch64_sve_fp8_fmlalltt_lane", [VerifyRuntimeMode], [ImmCheck<3, ImmCheck0_7>]>;
 }
diff --git a/clang/include/clang/CIR/Dialect/IR/CIRAttrVisitor.h b/clang/include/clang/CIR/Dialect/IR/CIRAttrVisitor.h
new file mode 100644
index 0000000000000..bbba89cb7e3fd
--- /dev/null
+++ b/clang/include/clang/CIR/Dialect/IR/CIRAttrVisitor.h
@@ -0,0 +1,52 @@
+//===- CIRAttrVisitor.h - Visitor for CIR attributes ------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file defines the CirAttrVisitor interface.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CLANG_CIR_DIALECT_IR_CIRATTRVISITOR_H
+#define LLVM_CLANG_CIR_DIALECT_IR_CIRATTRVISITOR_H
+
+#include "clang/CIR/Dialect/IR/CIRAttrs.h"
+
+namespace cir {
+
+template <typename ImplClass, typename RetTy> class CirAttrVisitor {
+public:
+  // FIXME: Create a TableGen list to automatically handle new attributes
+  RetTy visit(mlir::Attribute attr) {
+    if (const auto intAttr = mlir::dyn_cast<cir::IntAttr>(attr))
+      return getImpl().visitCirIntAttr(intAttr);
+    if (const auto fltAttr = mlir::dyn_cast<cir::FPAttr>(attr))
+      return getImpl().visitCirFPAttr(fltAttr);
+    if (const auto ptrAttr = mlir::dyn_cast<cir::ConstPtrAttr>(attr))
+      return getImpl().visitCirConstPtrAttr(ptrAttr);
+    llvm_unreachable("unhandled attribute type");
+  }
+
+  // If the implementation chooses not to implement a certain visit
+  // method, fall back to the parent.
+  RetTy visitCirIntAttr(cir::IntAttr attr) {
+    return getImpl().visitCirAttr(attr);
+  }
+  RetTy visitCirFPAttr(cir::FPAttr attr) {
+    return getImpl().visitCirAttr(attr);
+  }
+  RetTy visitCirConstPtrAttr(cir::ConstPtrAttr attr) {
+    return getImpl().visitCirAttr(attr);
+  }
+
+  RetTy visitCirAttr(mlir::Attribute attr) { return RetTy(); }
+
+  ImplClass &getImpl() { return *static_cast<ImplClass *>(this); }
+};
+
+} // namespace cir
+
+#endif // LLVM_CLANG_CIR_DIALECT_IR_CIRATTRVISITOR_H
diff --git a/clang/include/clang/CIR/FrontendAction/.clang-tidy b/clang/include/clang/CIR/FrontendAction/.clang-tidy
new file mode 100644
index 0000000000000..1a5dfe1418063
--- /dev/null
+++ b/clang/include/clang/CIR/FrontendAction/.clang-tidy
@@ -0,0 +1,68 @@
+InheritParentConfig: true
+Checks: >
+        -misc-const-correctness,
+        -llvm-header-guard,
+        bugprone-argument-comment,
+        bugprone-assert-side-effect,
+        bugprone-branch-clone,
+        bugprone-copy-constructor-init,
+        bugprone-dangling-handle,
+        bugprone-dynamic-static-initializers,
+        bugprone-macro-parentheses,
+        bugprone-macro-repeated-side-effects,
+        bugprone-misplaced-widening-cast,
+        bugprone-move-forwarding-reference,
+        bugprone-multiple-statement-macro,
+        bugprone-suspicious-semicolon,
+        bugprone-swapped-arguments,
+        bugprone-terminating-continue,
+        bugprone-unused-raii,
+        bugprone-unused-return-value,
+        misc-redundant-expression,
+        misc-static-assert,
+        misc-unused-using-decls,
+        modernize-use-bool-literals,
+        modernize-loop-convert,
+        modernize-make-unique,
+        modernize-raw-string-literal,
+        modernize-use-equals-default,
+        modernize-use-default-member-init,
+        modernize-use-emplace,
+        modernize-use-nullptr,
+        modernize-use-override,
+        modernize-use-using,
+        performance-for-range-copy,
+        performance-implicit-conversion-in-loop,
+        performance-inefficient-algorithm,
+        performance-inefficient-vector-operation,
+        performance-move-const-arg,
+        performance-no-automatic-move,
+        performance-trivially-destructible,
+        performance-unnecessary-copy-initialization,
+        performance-unnecessary-value-param,
+        readability-avoid-const-params-in-decls,
+        readability-const-return-type,
+        readability-container-size-empty,
+        readability-identifier-naming,
+        readability-inconsistent-declaration-parameter-name,
+        readability-misleading-indentation,
+        readability-redundant-control-flow,
+        readability-redundant-smartptr-get,
+        readability-simplify-boolean-expr,
+        readability-simplify-subscript-expr,
+        readability-use-anyofallof
+CheckOptions:
+  - key:             readability-identifier-naming.ClassCase
+    value:           CamelCase
+  - key:             readability-identifier-naming.EnumCase
+    value:           CamelCase
+  - key:             readability-identifier-naming.FunctionCase
+    value:           camelBack
+  - key:             readability-identifier-naming.MemberCase
+    value:           CamelCase
+  - key:             readability-identifier-naming.ParameterCase
+    value:           CamelCase
+  - key:             readability-identifier-naming.UnionCase
+    value:           CamelCase
+  - key:             readability-identifier-naming.VariableCase
+    value:           CamelCase
diff --git a/clang/include/clang/CIR/FrontendAction/CIRGenAction.h b/clang/include/clang/CIR/FrontendAction/CIRGenAction.h
index 2ab612613b73d..5f9110bc83b89 100644
--- a/clang/include/clang/CIR/FrontendAction/CIRGenAction.h
+++ b/clang/include/clang/CIR/FrontendAction/CIRGenAction.h
@@ -26,6 +26,7 @@ class CIRGenAction : public clang::ASTFrontendAction {
 public:
   enum class OutputType {
     EmitCIR,
+    EmitLLVM,
   };
 
 private:
@@ -55,6 +56,13 @@ class EmitCIRAction : public CIRGenAction {
   EmitCIRAction(mlir::MLIRContext *MLIRCtx = nullptr);
 };
 
+class EmitLLVMAction : public CIRGenAction {
+  virtual void anchor();
+
+public:
+  EmitLLVMAction(mlir::MLIRContext *MLIRCtx = nullptr);
+};
+
 } // namespace cir
 
 #endif
diff --git a/clang/include/clang/CIR/LowerToLLVM.h b/clang/include/clang/CIR/LowerToLLVM.h
new file mode 100644
index 0000000000000..6e1b0270fcd2b
--- /dev/null
+++ b/clang/include/clang/CIR/LowerToLLVM.h
@@ -0,0 +1,35 @@
+//====- LowerToLLVM.h- Lowering from CIR to LLVM --------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file declares an interface for converting CIR modules to LLVM IR.
+//
+//===----------------------------------------------------------------------===//
+#ifndef CLANG_CIR_LOWERTOLLVM_H
+#define CLANG_CIR_LOWERTOLLVM_H
+
+#include <memory>
+
+namespace llvm {
+class LLVMContext;
+class Module;
+} // namespace llvm
+
+namespace mlir {
+class ModuleOp;
+} // namespace mlir
+
+namespace cir {
+
+namespace direct {
+std::unique_ptr<llvm::Module>
+lowerDirectlyFromCIRToLLVMIR(mlir::ModuleOp mlirModule,
+                             llvm::LLVMContext &llvmCtx);
+} // namespace direct
+} // namespace cir
+
+#endif // CLANG_CIR_LOWERTOLLVM_H
diff --git a/clang/include/clang/CIR/MissingFeatures.h b/clang/include/clang/CIR/MissingFeatures.h
new file mode 100644
index 0000000000000..d4fcd52e7e6e3
--- /dev/null
+++ b/clang/include/clang/CIR/MissingFeatures.h
@@ -0,0 +1,43 @@
+//===---- MissingFeatures.h - Checks for unimplemented features -*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file introduces some helper classes to guard against features that
+// CIR dialect supports that we do not have and also do not have great ways to
+// assert against.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef CLANG_CIR_MISSINGFEATURES_H
+#define CLANG_CIR_MISSINGFEATURES_H
+
+namespace cir {
+
+// As a way to track features that haven't yet been implemented this class
+// explicitly contains a list of static fns that will return false that you
+// can guard against. If and when a feature becomes implemented simply changing
+// this return to true will cause compilation to fail at all the points in which
+// we noted that we needed to address. This is a much more explicit way to
+// handle "TODO"s.
+struct MissingFeatures {
+  // Address space related
+  static bool addressSpace() { return false; }
+
+  // This isn't needed until we add support for bools.
+  static bool convertTypeForMemory() { return false; }
+
+  // Unhandled global/linkage information.
+  static bool opGlobalDSOLocal() { return false; }
+  static bool opGlobalThreadLocal() { return false; }
+  static bool opGlobalConstant() { return false; }
+  static bool opGlobalAlignment() { return false; }
+  static bool opGlobalLinkage() { return false; }
+};
+
+} // namespace cir
+
+#endif // CLANG_CIR_MISSINGFEATURES_H
diff --git a/clang/include/clang/CIRFrontendAction/.clang-tidy b/clang/include/clang/CIRFrontendAction/.clang-tidy
deleted file mode 100644
index ef88dbcec488c..0000000000000
--- a/clang/include/clang/CIRFrontendAction/.clang-tidy
+++ /dev/null
@@ -1,53 +0,0 @@
-InheritParentConfig: true
-Checks: >
-        -misc-const-correctness,
-        -llvm-header-guard,
-        bugprone-argument-comment,
-        bugprone-assert-side-effect,
-        bugprone-branch-clone,
-        bugprone-copy-constructor-init,
-        bugprone-dangling-handle,
-        bugprone-dynamic-static-initializers,
-        bugprone-macro-parentheses,
-        bugprone-macro-repeated-side-effects,
-        bugprone-misplaced-widening-cast,
-        bugprone-move-forwarding-reference,
-        bugprone-multiple-statement-macro,
-        bugprone-suspicious-semicolon,
-        bugprone-swapped-arguments,
-        bugprone-terminating-continue,
-        bugprone-unused-raii,
-        bugprone-unused-return-value,
-        misc-redundant-expression,
-        misc-static-assert,
-        misc-unused-using-decls,
-        modernize-use-bool-literals,
-        modernize-loop-convert,
-        modernize-make-unique,
-        modernize-raw-string-literal,
-        modernize-use-equals-default,
-        modernize-use-default-member-init,
-        modernize-use-emplace,
-        modernize-use-nullptr,
-        modernize-use-override,
-        modernize-use-using,
-        performance-for-range-copy,
-        performance-implicit-conversion-in-loop,
-        performance-inefficient-algorithm,
-        performance-inefficient-vector-operation,
-        performance-move-const-arg,
-        performance-no-automatic-move,
-        performance-trivially-destructible,
-        performance-unnecessary-copy-initialization,
-        performance-unnecessary-value-param,
-        readability-avoid-const-params-in-decls,
-        readability-const-return-type,
-        readability-container-size-empty,
-        readability-identifier-naming,
-        readability-inconsistent-declaration-parameter-name,
-        readability-misleading-indentation,
-        readability-redundant-control-flow,
-        readability-redundant-smartptr-get,
-        readability-simplify-boolean-expr,
-        readability-simplify-subscript-expr,
-        readability-use-anyofallof
diff --git a/clang/include/clang/Config/config.h.cmake b/clang/include/clang/Config/config.h.cmake
index 27ed69e21562b..00c352b458c34 100644
--- a/clang/include/clang/Config/config.h.cmake
+++ b/clang/include/clang/Config/config.h.cmake
@@ -76,7 +76,6 @@
 #cmakedefine01 PPC_LINUX_DEFAULT_IEEELONGDOUBLE
 
 /* Enable each functionality of modules */
-#cmakedefine01 CLANG_ENABLE_ARCMT
 #cmakedefine01 CLANG_ENABLE_OBJC_REWRITER
 #cmakedefine01 CLANG_ENABLE_STATIC_ANALYZER
 
diff --git a/clang/include/clang/Driver/Action.h b/clang/include/clang/Driver/Action.h
index feeabae89d6b1..e5307b0fcedd5 100644
--- a/clang/include/clang/Driver/Action.h
+++ b/clang/include/clang/Driver/Action.h
@@ -60,7 +60,6 @@ class Action {
     PrecompileJobClass,
     ExtractAPIJobClass,
     AnalyzeJobClass,
-    MigrateJobClass,
     CompileJobClass,
     BackendJobClass,
     AssembleJobClass,
@@ -460,17 +459,6 @@ class AnalyzeJobAction : public JobAction {
   }
 };
 
-class MigrateJobAction : public JobAction {
-  void anchor() override;
-
-public:
-  MigrateJobAction(Action *Input, types::ID OutputType);
-
-  static bool classof(const Action *A) {
-    return A->getKind() == MigrateJobClass;
-  }
-};
-
 class CompileJobAction : public JobAction {
   void anchor() override;
 
diff --git a/clang/include/clang/Driver/Driver.h b/clang/include/clang/Driver/Driver.h
index f4a52cc529b79..b463dc2a93550 100644
--- a/clang/include/clang/Driver/Driver.h
+++ b/clang/include/clang/Driver/Driver.h
@@ -797,22 +797,14 @@ class Driver {
   const ToolChain &getToolChain(const llvm::opt::ArgList &Args,
                                 const llvm::Triple &Target) const;
 
-  /// @}
-
-  /// Retrieves a ToolChain for a particular device \p Target triple
-  ///
-  /// \param[in] HostTC is the host ToolChain paired with the device
-  ///
-  /// \param[in] TargetDeviceOffloadKind (e.g. OFK_Cuda/OFK_OpenMP/OFK_SYCL) is
-  /// an Offloading action that is optionally passed to a ToolChain (used by
-  /// CUDA, to specify if it's used in conjunction with OpenMP)
+  /// Retrieves a ToolChain for a particular \p Target triple for offloading.
   ///
   /// Will cache ToolChains for the life of the driver object, and create them
   /// on-demand.
-  const ToolChain &getOffloadingDeviceToolChain(
-      const llvm::opt::ArgList &Args, const llvm::Triple &Target,
-      const ToolChain &HostTC,
-      const Action::OffloadKind &TargetDeviceOffloadKind) const;
+  const ToolChain &getOffloadToolChain(const llvm::opt::ArgList &Args,
+                                       const Action::OffloadKind Kind,
+                                       const llvm::Triple &Target,
+                                       const llvm::Triple &AuxTarget) const;
 
   /// Get bitmasks for which option flags to include and exclude based on
   /// the driver mode.
diff --git a/clang/include/clang/Driver/Options.td b/clang/include/clang/Driver/Options.td
index c5b7fcb7c7f09..1cf62ab466134 100644
--- a/clang/include/clang/Driver/Options.td
+++ b/clang/include/clang/Driver/Options.td
@@ -694,20 +694,6 @@ def ccc_print_phases : Flag<["-"], "ccc-print-phases">,
 def ccc_print_bindings : Flag<["-"], "ccc-print-bindings">, InternalDebugOpt,
   HelpText<"Show bindings of tools to actions">;
 
-def ccc_arcmt_check : Flag<["-"], "ccc-arcmt-check">, InternalDriverOpt,
-  HelpText<"Check for ARC migration issues that need manual handling">;
-def ccc_arcmt_modify : Flag<["-"], "ccc-arcmt-modify">, InternalDriverOpt,
-  HelpText<"Apply modifications to files to conform to ARC">;
-def ccc_arcmt_migrate : Separate<["-"], "ccc-arcmt-migrate">, InternalDriverOpt,
-  HelpText<"Apply modifications and produces temporary files that conform to ARC">;
-def arcmt_migrate_report_output : Separate<["-"], "arcmt-migrate-report-output">,
-  HelpText<"Output path for the plist report">,
-  Visibility<[ClangOption, CC1Option]>,
-  MarshallingInfoString<FrontendOpts<"ARCMTMigrateReportOut">>;
-def arcmt_migrate_emit_arc_errors : Flag<["-"], "arcmt-migrate-emit-errors">,
-  HelpText<"Emit ARC errors even if the migrator can fix them">,
-  Visibility<[ClangOption, CC1Option]>,
-  MarshallingInfoFlag<FrontendOpts<"ARCMTMigrateEmitARCErrors">>;
 def gen_reproducer_eq: Joined<["-"], "gen-reproducer=">,
   Flags<[NoArgumentUnused]>, Visibility<[ClangOption, CLOption, DXCOption]>,
   HelpText<"Emit reproducer on (option: off, crash (default), error, always)">;
@@ -723,87 +709,6 @@ def no_round_trip_args : Flag<["-"], "no-round-trip-args">,
   Visibility<[CC1Option]>,
   HelpText<"Disable command line arguments round-trip.">;
 
-def _migrate : Flag<["--"], "migrate">, Flags<[NoXarchOption]>,
-  HelpText<"Run the migrator">;
-def ccc_objcmt_migrate : Separate<["-"], "ccc-objcmt-migrate">,
-  InternalDriverOpt,
-  HelpText<"Apply modifications and produces temporary files to migrate to "
-   "modern ObjC syntax">;
-
-def objcmt_migrate_literals : Flag<["-"], "objcmt-migrate-literals">,
-  Visibility<[ClangOption, CC1Option]>,
-  HelpText<"Enable migration to modern ObjC literals">,
-  MarshallingInfoBitfieldFlag<FrontendOpts<"ObjCMTAction">, "FrontendOptions::ObjCMT_Literals">;
-def objcmt_migrate_subscripting : Flag<["-"], "objcmt-migrate-subscripting">,
-  Visibility<[ClangOption, CC1Option]>,
-  HelpText<"Enable migration to modern ObjC subscripting">,
-  MarshallingInfoBitfieldFlag<FrontendOpts<"ObjCMTAction">, "FrontendOptions::ObjCMT_Subscripting">;
-def objcmt_migrate_property : Flag<["-"], "objcmt-migrate-property">,
-  Visibility<[ClangOption, CC1Option]>,
-  HelpText<"Enable migration to modern ObjC property">,
-  MarshallingInfoBitfieldFlag<FrontendOpts<"ObjCMTAction">, "FrontendOptions::ObjCMT_Property">;
-def objcmt_migrate_all : Flag<["-"], "objcmt-migrate-all">,
-  Visibility<[ClangOption, CC1Option]>,
-  HelpText<"Enable migration to modern ObjC">,
-  MarshallingInfoBitfieldFlag<FrontendOpts<"ObjCMTAction">, "FrontendOptions::ObjCMT_MigrateDecls">;
-def objcmt_migrate_readonly_property : Flag<["-"], "objcmt-migrate-readonly-property">,
-  Visibility<[ClangOption, CC1Option]>,
-  HelpText<"Enable migration to modern ObjC readonly property">,
-  MarshallingInfoBitfieldFlag<FrontendOpts<"ObjCMTAction">, "FrontendOptions::ObjCMT_ReadonlyProperty">;
-def objcmt_migrate_readwrite_property : Flag<["-"], "objcmt-migrate-readwrite-property">,
-  Visibility<[ClangOption, CC1Option]>,
-  HelpText<"Enable migration to modern ObjC readwrite property">,
-  MarshallingInfoBitfieldFlag<FrontendOpts<"ObjCMTAction">, "FrontendOptions::ObjCMT_ReadwriteProperty">;
-def objcmt_migrate_property_dot_syntax : Flag<["-"], "objcmt-migrate-property-dot-syntax">,
-  Visibility<[ClangOption, CC1Option]>,
-  HelpText<"Enable migration of setter/getter messages to property-dot syntax">,
-  MarshallingInfoBitfieldFlag<FrontendOpts<"ObjCMTAction">, "FrontendOptions::ObjCMT_PropertyDotSyntax">;
-def objcmt_migrate_annotation : Flag<["-"], "objcmt-migrate-annotation">,
-  Visibility<[ClangOption, CC1Option]>,
-  HelpText<"Enable migration to property and method annotations">,
-  MarshallingInfoBitfieldFlag<FrontendOpts<"ObjCMTAction">, "FrontendOptions::ObjCMT_Annotation">;
-def objcmt_migrate_instancetype : Flag<["-"], "objcmt-migrate-instancetype">,
-  Visibility<[ClangOption, CC1Option]>,
-  HelpText<"Enable migration to infer instancetype for method result type">,
-  MarshallingInfoBitfieldFlag<FrontendOpts<"ObjCMTAction">, "FrontendOptions::ObjCMT_Instancetype">;
-def objcmt_migrate_nsmacros : Flag<["-"], "objcmt-migrate-ns-macros">,
-  Visibility<[ClangOption, CC1Option]>,
-  HelpText<"Enable migration to NS_ENUM/NS_OPTIONS macros">,
-  MarshallingInfoBitfieldFlag<FrontendOpts<"ObjCMTAction">, "FrontendOptions::ObjCMT_NsMacros">;
-def objcmt_migrate_protocol_conformance : Flag<["-"], "objcmt-migrate-protocol-conformance">,
-  Visibility<[ClangOption, CC1Option]>,
-  HelpText<"Enable migration to add protocol conformance on classes">,
-  MarshallingInfoBitfieldFlag<FrontendOpts<"ObjCMTAction">, "FrontendOptions::ObjCMT_ProtocolConformance">;
-def objcmt_atomic_property : Flag<["-"], "objcmt-atomic-property">,
-  Visibility<[ClangOption, CC1Option]>,
-  HelpText<"Make migration to 'atomic' properties">,
-  MarshallingInfoBitfieldFlag<FrontendOpts<"ObjCMTAction">, "FrontendOptions::ObjCMT_AtomicProperty">;
-def objcmt_returns_innerpointer_property : Flag<["-"], "objcmt-returns-innerpointer-property">,
-  Visibility<[ClangOption, CC1Option]>,
-  HelpText<"Enable migration to annotate property with NS_RETURNS_INNER_POINTER">,
-  MarshallingInfoBitfieldFlag<FrontendOpts<"ObjCMTAction">, "FrontendOptions::ObjCMT_ReturnsInnerPointerProperty">;
-def objcmt_ns_nonatomic_iosonly: Flag<["-"], "objcmt-ns-nonatomic-iosonly">,
-  Visibility<[ClangOption, CC1Option]>,
-  HelpText<"Enable migration to use NS_NONATOMIC_IOSONLY macro for setting property's 'atomic' attribute">,
-  MarshallingInfoBitfieldFlag<FrontendOpts<"ObjCMTAction">, "FrontendOptions::ObjCMT_NsAtomicIOSOnlyProperty">;
-def objcmt_migrate_designated_init : Flag<["-"], "objcmt-migrate-designated-init">,
-  Visibility<[ClangOption, CC1Option]>,
-  HelpText<"Enable migration to infer NS_DESIGNATED_INITIALIZER for initializer methods">,
-  MarshallingInfoBitfieldFlag<FrontendOpts<"ObjCMTAction">, "FrontendOptions::ObjCMT_DesignatedInitializer">;
-
-def objcmt_allowlist_dir_path: Joined<["-"], "objcmt-allowlist-dir-path=">,
-  Visibility<[ClangOption, CC1Option]>,
-  HelpText<"Only modify files with a filename contained in the provided directory path">,
-  MarshallingInfoString<FrontendOpts<"ObjCMTAllowListPath">>;
-def : Joined<["-"], "objcmt-whitelist-dir-path=">,
-  Visibility<[ClangOption, CC1Option]>,
-  HelpText<"Alias for -objcmt-allowlist-dir-path">,
-  Alias<objcmt_allowlist_dir_path>;
-// The misspelt "white-list" [sic] alias is due for removal.
-def : Joined<["-"], "objcmt-white-list-dir-path=">,
-  Visibility<[ClangOption, CC1Option]>,
-  Alias<objcmt_allowlist_dir_path>;
-
 // Make sure all other -ccc- options are rejected.
 def ccc_ : Joined<["-"], "ccc-">, Group<internal_Group>, Flags<[Unsupported]>;
 
@@ -1027,7 +932,9 @@ def W_Joined : Joined<["-"], "W">, Group<W_Group>,
 def Xanalyzer : Separate<["-"], "Xanalyzer">,
   HelpText<"Pass <arg> to the static analyzer">, MetaVarName<"<arg>">,
   Group<StaticAnalyzer_Group>;
-def Xarch__ : JoinedAndSeparate<["-"], "Xarch_">, Flags<[NoXarchOption]>;
+def Xarch__ : JoinedAndSeparate<["-"], "Xarch_">, Flags<[NoXarchOption]>,
+  HelpText<"Pass <arg> to the compiliation if the target matches <arch>">,
+  MetaVarName<"<arch> <arg>">;
 def Xarch_host : Separate<["-"], "Xarch_host">, Flags<[NoXarchOption]>,
   HelpText<"Pass <arg> to the CUDA/HIP host compilation">, MetaVarName<"<arg>">;
 def Xarch_device : Separate<["-"], "Xarch_device">, Flags<[NoXarchOption]>,
@@ -1210,8 +1117,8 @@ def fno_convergent_functions : Flag<["-"], "fno-convergent-functions">,
 
 // Common offloading options
 let Group = offload_Group in {
-def offload_arch_EQ : Joined<["--"], "offload-arch=">, Flags<[NoXarchOption]>,
-  Visibility<[ClangOption, FlangOption]>,
+def offload_arch_EQ : Joined<["--"], "offload-arch=">,
+  Visibility<[ClangOption, FlangOption]>, Flags<[NoXarchOption]>,
   HelpText<"Specify an offloading device architecture for CUDA, HIP, or OpenMP. (e.g. sm_35). "
            "If 'native' is used the compiler will detect locally installed architectures. "
            "For HIP offloading, the device architecture can be followed by target ID features "
@@ -1888,6 +1795,11 @@ def fprofile_update_EQ : Joined<["-"], "fprofile-update=">,
     Values<"atomic,prefer-atomic,single">,
     MetaVarName<"<method>">, HelpText<"Set update method of profile counters">,
     MarshallingInfoFlag<CodeGenOpts<"AtomicProfileUpdate">>;
+def fprofile_continuous : Flag<["-"], "fprofile-continuous">,
+    Group<f_Group>, Visibility<[ClangOption, CC1Option]>,
+    HelpText<"Enable continuous instrumentation profiling mode">,
+    MarshallingInfoFlag<CodeGenOpts<"ContinuousProfileSync">>;
+
 defm pseudo_probe_for_profiling : BoolFOption<"pseudo-probe-for-profiling",
   CodeGenOpts<"PseudoProbeForProfiling">, DefaultFalse,
   PosFlag<SetTrue, [], [ClangOption], "Emit">,
@@ -2619,6 +2531,10 @@ defm sanitize_cfi_canonical_jump_tables : BoolOption<"f", "sanitize-cfi-canonica
           "Do not make">,
   BothFlags<[], [ClangOption], " the jump table addresses canonical in the symbol table">>,
   Group<f_clang_Group>;
+def fsanitize_kcfi_arity : Flag<["-"], "fsanitize-kcfi-arity">,
+                           Group<f_clang_Group>,
+                           HelpText<"Embed function arity information into the KCFI patchable function prefix">,
+                           MarshallingInfoFlag<CodeGenOpts<"SanitizeKcfiArity">>;
 defm sanitize_stats : BoolOption<"f", "sanitize-stats",
   CodeGenOpts<"SanitizeStats">, DefaultFalse,
   PosFlag<SetTrue, [], [ClangOption], "Enable">,
@@ -3505,8 +3421,6 @@ def fno_strict_aliasing : Flag<["-"], "fno-strict-aliasing">, Group<f_Group>,
 def fstruct_path_tbaa : Flag<["-"], "fstruct-path-tbaa">, Group<f_Group>;
 def fno_struct_path_tbaa : Flag<["-"], "fno-struct-path-tbaa">, Group<f_Group>;
 def fno_strict_enums : Flag<["-"], "fno-strict-enums">, Group<f_Group>;
-def fno_strict_overflow : Flag<["-"], "fno-strict-overflow">, Group<f_Group>,
-  Visibility<[ClangOption, FlangOption]>;
 defm init_global_zero : BoolOptionWithoutMarshalling<"f", "init-global-zero",
   PosFlag<SetTrue, [], [FlangOption, FC1Option],
           "Zero initialize globals without default initialization (default)">,
@@ -3557,11 +3471,6 @@ defm application_extension : BoolFOption<"application-extension",
   PosFlag<SetTrue, [], [ClangOption, CC1Option],
           "Restrict code to those available for App Extensions">,
   NegFlag<SetFalse>>;
-defm relaxed_template_template_args : BoolFOption<"relaxed-template-template-args",
-  LangOpts<"RelaxedTemplateTemplateArgs">, DefaultTrue,
-  PosFlag<SetTrue, [], [], "Enable">,
-  NegFlag<SetFalse, [], [CC1Option], "Disable">,
-  BothFlags<[], [ClangOption], " C++17 relaxed template template argument matching">>;
 defm retain_subst_template_type_parm_type_ast_nodes : BoolFOption<"retain-subst-template-type-parm-type-ast-nodes",
   LangOpts<"RetainSubstTemplateTypeParmTypeAstNodes">, DefaultFalse,
   PosFlag<SetTrue, [], [CC1Option], "Enable">,
@@ -4023,7 +3932,9 @@ defm strict_vtable_pointers : BoolFOption<"strict-vtable-pointers",
             " overwriting polymorphic C++ objects">,
   NegFlag<SetFalse>>;
 def fstrict_overflow : Flag<["-"], "fstrict-overflow">, Group<f_Group>,
-  Visibility<[ClangOption, FlangOption]>;
+  Visibility<[ClangOption, CLOption, FlangOption]>;
+def fno_strict_overflow : Flag<["-"], "fno-strict-overflow">, Group<f_Group>,
+  Visibility<[ClangOption, CLOption, FlangOption]>;
 def fpointer_tbaa : Flag<["-"], "fpointer-tbaa">, Group<f_Group>;
 def fdriver_only : Flag<["-"], "fdriver-only">, Flags<[NoXarchOption]>,
   Visibility<[ClangOption, CLOption, DXCOption]>,
@@ -4163,6 +4074,10 @@ def ftrap_function_EQ : Joined<["-"], "ftrap-function=">, Group<f_Group>,
   Visibility<[ClangOption, CC1Option]>,
   HelpText<"Issue call to specified function rather than a trap instruction">,
   MarshallingInfoString<CodeGenOpts<"TrapFuncName">>;
+def floop_interchange : Flag<["-"], "floop-interchange">, Group<f_Group>,
+  HelpText<"Enable the loop interchange pass">, Visibility<[ClangOption, CC1Option]>;
+def fno_loop_interchange: Flag<["-"], "fno-loop-interchange">, Group<f_Group>,
+  HelpText<"Disable the loop interchange pass">, Visibility<[ClangOption, CC1Option]>;
 def funroll_loops : Flag<["-"], "funroll-loops">, Group<f_Group>,
   HelpText<"Turn on loop unroller">, Visibility<[ClangOption, CC1Option, FlangOption, FC1Option]>;
 def fno_unroll_loops : Flag<["-"], "fno-unroll-loops">, Group<f_Group>,
@@ -4301,6 +4216,11 @@ def fwrapv : Flag<["-"], "fwrapv">, Group<f_Group>,
   HelpText<"Treat signed integer overflow as two's complement">;
 def fno_wrapv : Flag<["-"], "fno-wrapv">, Group<f_Group>,
   Visibility<[ClangOption, CLOption, FlangOption]>;
+def fwrapv_pointer : Flag<["-"], "fwrapv-pointer">, Group<f_Group>,
+  Visibility<[ClangOption, CLOption, CC1Option, FlangOption, FC1Option]>,
+  HelpText<"Treat pointer overflow as two's complement">;
+def fno_wrapv_pointer : Flag<["-"], "fno-wrapv-pointer">, Group<f_Group>,
+  Visibility<[ClangOption, CLOption, FlangOption]>;
 def fwritable_strings : Flag<["-"], "fwritable-strings">, Group<f_Group>,
   Visibility<[ClangOption, CC1Option]>,
   HelpText<"Store string literals as writable data">,
@@ -4344,6 +4264,18 @@ def stack_usage_file : Separate<["-"], "stack-usage-file">,
   Visibility<[CC1Option]>,
   HelpText<"Filename (or -) to write stack usage output to">,
   MarshallingInfoString<CodeGenOpts<"StackUsageOutput">>;
+def fextend_variable_liveness_EQ : Joined<["-"], "fextend-variable-liveness=">,
+  Group<f_Group>, Visibility<[ClangOption, CC1Option]>,
+  HelpText<"Extend the liveness of user variables through optimizations to "
+           "prevent stale or optimized-out variable values when debugging.">,
+  Values<"all,this,none">,
+  NormalizedValues<["All", "This", "None"]>,
+  NormalizedValuesScope<"CodeGenOptions::ExtendVariableLivenessKind">,
+  MarshallingInfoEnum<CodeGenOpts<"ExtendVariableLiveness">, "None">;
+def fextend_variable_liveness : Flag<["-"], "fextend-variable-liveness">,
+  Visibility<[ClangOption, CC1Option]>,
+  Alias<fextend_variable_liveness_EQ>, AliasArgs<["all"]>,
+  HelpText<"Alias for -fextend-variable-liveness=all.">;
 
 defm unique_basic_block_section_names : BoolFOption<"unique-basic-block-section-names",
   CodeGenOpts<"UniqueBasicBlockSectionNames">, DefaultFalse,
@@ -5908,6 +5840,9 @@ def start_no_unused_arguments : Flag<["--"], "start-no-unused-arguments">,
   HelpText<"Don't emit warnings about unused arguments for the following arguments">;
 def static_libgcc : Flag<["-"], "static-libgcc">;
 def static_libstdcxx : Flag<["-"], "static-libstdc++">;
+def static_libclosure : Flag<["-"], "static-libclosure">,
+    Visibility<[ClangOption, CC1Option]>,
+    HelpText<"Generate code for statically linking libclosure (BlocksRuntime)">;
 def static : Flag<["-", "--"], "static">, Group<Link_Group>,
   Visibility<[ClangOption, FlangOption]>,
   Flags<[NoArgumentUnused]>;
@@ -6424,15 +6359,13 @@ def mno_avx : Flag<["-"], "mno-avx">, Group<m_x86_Features_Group>;
 def mavx10_1_256 : Flag<["-"], "mavx10.1-256">, Group<m_x86_AVX10_Features_Group>;
 def mno_avx10_1_256 : Flag<["-"], "mno-avx10.1-256">, Group<m_x86_AVX10_Features_Group>;
 def mavx10_1_512 : Flag<["-"], "mavx10.1-512">, Group<m_x86_AVX10_Features_Group>;
-def mno_avx10_1_512 : Flag<["-"], "mno-avx10.1-512">, Group<m_x86_AVX10_Features_Group>;
-def mavx10_1 : Flag<["-"], "mavx10.1">, Alias<mavx10_1_256>;
-def mno_avx10_1 : Flag<["-"], "mno-avx10.1">, Alias<mno_avx10_1_256>;
+def mno_avx10_1_512 : Flag<["-"], "mno-avx10.1-512">, Alias<mno_avx10_1_256>;
+def mavx10_1 : Flag<["-"], "mavx10.1">, Flags<[Unsupported]>;
+def mno_avx10_1 : Flag<["-"], "mno-avx10.1">, Flags<[Unsupported]>;
 def mavx10_2_256 : Flag<["-"], "mavx10.2-256">, Group<m_x86_AVX10_Features_Group>;
-def mno_avx10_2_256 : Flag<["-"], "mno-avx10.2-256">, Group<m_x86_AVX10_Features_Group>;
 def mavx10_2_512 : Flag<["-"], "mavx10.2-512">, Group<m_x86_AVX10_Features_Group>;
-def mno_avx10_2_512 : Flag<["-"], "mno-avx10.2-512">, Group<m_x86_AVX10_Features_Group>;
-def mavx10_2 : Flag<["-"], "mavx10.2">, Alias<mavx10_2_256>;
-def mno_avx10_2 : Flag<["-"], "mno-avx10.2">, Alias<mno_avx10_2_256>;
+def mavx10_2 : Flag<["-"], "mavx10.2">, Alias<mavx10_2_512>;
+def mno_avx10_2 : Flag<["-"], "mno-avx10.2">, Group<m_x86_AVX10_Features_Group>;
 def mavx2 : Flag<["-"], "mavx2">, Group<m_x86_Features_Group>;
 def mno_avx2 : Flag<["-"], "mno-avx2">, Group<m_x86_Features_Group>;
 def mavx512f : Flag<["-"], "mavx512f">, Group<m_x86_Features_Group>;
@@ -6961,6 +6894,7 @@ defm backslash : OptInFC1FFlag<"backslash", "Specify that backslash in string in
 defm xor_operator : OptInFC1FFlag<"xor-operator", "Enable .XOR. as a synonym of .NEQV.">;
 defm logical_abbreviations : OptInFC1FFlag<"logical-abbreviations", "Enable logical abbreviations">;
 defm implicit_none : OptInFC1FFlag<"implicit-none", "No implicit typing allowed unless overridden by IMPLICIT statements">;
+defm implicit_none_ext : OptInFC1FFlag<"implicit-none-ext", "No implicit externals allowed">;
 defm underscoring : OptInFC1FFlag<"underscoring", "Appends one trailing underscore to external names">;
 defm ppc_native_vec_elem_order: BoolOptionWithoutMarshalling<"f", "ppc-native-vector-element-order",
   PosFlag<SetTrue, [], [ClangOption], "Specifies PowerPC native vector element order (default)">,
@@ -6970,8 +6904,11 @@ defm unsigned : OptInFC1FFlag<"unsigned", "Enables UNSIGNED type">;
 def fno_automatic : Flag<["-"], "fno-automatic">, Group<f_Group>,
   HelpText<"Implies the SAVE attribute for non-automatic local objects in subprograms unless RECURSIVE">;
 
-def fsave_main_program : Flag<["-"], "fsave-main-program">, Group<f_Group>,
-  HelpText<"Place all variables from the main program in static memory (otherwise scalars may be placed on the stack)">;
+defm save_main_program : BoolOptionWithoutMarshalling<"f", "save-main-program",
+  PosFlag<SetTrue, [], [],
+    "Place all main program variables in static memory (otherwise scalars may be placed on the stack)">,
+  NegFlag<SetFalse, [], [],
+    "Allow placing main program variables on the stack (default)">>;
 
 defm stack_arrays : BoolOptionWithoutMarshalling<"f", "stack-arrays",
   PosFlag<SetTrue, [], [ClangOption], "Attempt to allocate array temporaries on the stack, no matter their size">,
@@ -7384,6 +7321,9 @@ let Visibility = [CC1Option, FC1Option] in {
 def mlink_builtin_bitcode : Separate<["-"], "mlink-builtin-bitcode">,
   HelpText<"Link and internalize needed symbols from the given bitcode file "
            "before performing optimizations.">;
+def mlink_bitcode_file
+    : Separate<["-"], "mlink-bitcode-file">,
+      HelpText<"Link the given bitcode file before performing optimizations.">;
 } // let Visibility = [CC1Option, FC1Option]
 
 let Visibility = [CC1Option] in {
@@ -7487,14 +7427,13 @@ def msmall_data_limit : Separate<["-"], "msmall-data-limit">,
 def funwind_tables_EQ : Joined<["-"], "funwind-tables=">,
   HelpText<"Generate unwinding tables for all functions">,
   MarshallingInfoInt<CodeGenOpts<"UnwindTables">>;
-defm constructor_aliases : BoolMOption<"constructor-aliases",
-  CodeGenOpts<"CXXCtorDtorAliases">, DefaultFalse,
-  PosFlag<SetTrue, [], [ClangOption], "Enable">,
-  NegFlag<SetFalse, [], [ClangOption], "Disable">,
-  BothFlags<[], [ClangOption, CC1Option],
-          " emitting complete constructors and destructors as aliases when possible">>;
-def mlink_bitcode_file : Separate<["-"], "mlink-bitcode-file">,
-  HelpText<"Link the given bitcode file before performing optimizations.">;
+defm constructor_aliases
+    : BoolMOption<"constructor-aliases", CodeGenOpts<"CXXCtorDtorAliases">,
+                  DefaultFalse, PosFlag<SetTrue, [], [ClangOption], "Enable">,
+                  NegFlag<SetFalse, [], [ClangOption], "Disable">,
+                  BothFlags<[], [ClangOption, CC1Option],
+                            " emitting complete constructors and destructors "
+                            "as aliases when possible">>;
 defm link_builtin_bitcode_postopt: BoolMOption<"link-builtin-bitcode-postopt",
   CodeGenOpts<"LinkBitcodePostopt">, DefaultFalse,
   PosFlag<SetTrue, [], [ClangOption], "Link builtin bitcodes after the "
@@ -7669,6 +7608,11 @@ def fexperimental_assignment_tracking_EQ : Joined<["-"], "fexperimental-assignme
 def enable_tlsdesc : Flag<["-"], "enable-tlsdesc">,
   MarshallingInfoFlag<CodeGenOpts<"EnableTLSDESC">>;
 
+def import_call_optimization : Flag<["-"], "import-call-optimization">,
+    HelpText<"Emit Import Call sections on supported targets that can be used "
+             "by the Windows kernel to enable import call optimization">,
+    MarshallingInfoFlag<CodeGenOpts<"ImportCallOptimization">>;
+
 } // let Visibility = [CC1Option]
 
 //===----------------------------------------------------------------------===//
@@ -7960,8 +7904,6 @@ def rewrite_test : Flag<["-"], "rewrite-test">,
   HelpText<"Rewriter playground">;
 def rewrite_macros : Flag<["-"], "rewrite-macros">,
   HelpText<"Expand macros without full preprocessing">;
-def migrate : Flag<["-"], "migrate">,
-  HelpText<"Migrate source code">;
 def compiler_options_dump : Flag<["-"], "compiler-options-dump">,
   HelpText<"Dump the compiler configuration options">;
 def print_dependency_directives_minimized_source : Flag<["-"],
@@ -7975,17 +7917,6 @@ defm emit_llvm_uselists : BoolOption<"", "emit-llvm-uselists",
   NegFlag<SetFalse, [], [ClangOption], "Don't preserve">,
   BothFlags<[], [ClangOption], " order of LLVM use-lists when serializing">>;
 
-def mt_migrate_directory : Separate<["-"], "mt-migrate-directory">,
-  HelpText<"Directory for temporary files produced during ARC or ObjC migration">,
-  MarshallingInfoString<FrontendOpts<"MTMigrateDir">>;
-
-def arcmt_action_EQ : Joined<["-"], "arcmt-action=">, Visibility<[CC1Option]>,
-  HelpText<"The ARC migration action to take">,
-  Values<"check,modify,migrate">,
-  NormalizedValuesScope<"FrontendOptions">,
-  NormalizedValues<["ARCMT_Check", "ARCMT_Modify", "ARCMT_Migrate"]>,
-  MarshallingInfoEnum<FrontendOpts<"ARCMTAction">, "ARCMT_None">;
-
 def print_stats : Flag<["-"], "print-stats">,
   HelpText<"Print performance metrics and statistics">,
   MarshallingInfoFlag<FrontendOpts<"ShowStats">>;
diff --git a/clang/include/clang/Driver/SanitizerArgs.h b/clang/include/clang/Driver/SanitizerArgs.h
index a54995e2b153b..6a866ded0e75c 100644
--- a/clang/include/clang/Driver/SanitizerArgs.h
+++ b/clang/include/clang/Driver/SanitizerArgs.h
@@ -43,6 +43,7 @@ class SanitizerArgs {
   bool CfiICallGeneralizePointers = false;
   bool CfiICallNormalizeIntegers = false;
   bool CfiCanonicalJumpTables = false;
+  bool KcfiArity = false;
   int AsanFieldPadding = 0;
   bool SharedRuntime = false;
   bool StableABI = false;
diff --git a/clang/include/clang/ExtractAPI/ExtractAPIVisitor.h b/clang/include/clang/ExtractAPI/ExtractAPIVisitor.h
index b09b8b44d9aba..e60440e14a9fe 100644
--- a/clang/include/clang/ExtractAPI/ExtractAPIVisitor.h
+++ b/clang/include/clang/ExtractAPI/ExtractAPIVisitor.h
@@ -252,6 +252,11 @@ class ExtractAPIVisitorBase : public RecursiveASTVisitor<Derived> {
     if (!NewRecordContext)
       return;
     auto *Tag = D.getType()->getAsTagDecl();
+    if (!Tag) {
+      if (const auto *AT = D.getASTContext().getAsArrayType(D.getType())) {
+        Tag = AT->getElementType()->getAsTagDecl();
+      }
+    }
     SmallString<128> TagUSR;
     clang::index::generateUSRForDecl(Tag, TagUSR);
     if (auto *Record = llvm::dyn_cast_if_present<TagRecord>(
@@ -1141,11 +1146,29 @@ bool ExtractAPIVisitorBase<Derived>::VisitTypedefNameDecl(
 
   StringRef Name = Decl->getName();
 
+  auto nameMatches = [&Name](TagDecl *TagDecl) {
+    StringRef TagName = TagDecl->getName();
+
+    if (TagName == Name)
+      return true;
+
+    // Also check whether the tag decl's name is the same as the typedef name
+    // with prefixed underscores
+    if (TagName.starts_with('_')) {
+      StringRef StrippedName = TagName.ltrim('_');
+
+      if (StrippedName == Name)
+        return true;
+    }
+
+    return false;
+  };
+
   // If the underlying type was defined as part of the typedef modify it's
   // fragments directly and pretend the typedef doesn't exist.
   if (auto *TagDecl = Decl->getUnderlyingType()->getAsTagDecl()) {
     if (TagDecl->isEmbeddedInDeclarator() && TagDecl->isCompleteDefinition() &&
-        Decl->getName() == TagDecl->getName()) {
+        nameMatches(TagDecl)) {
       SmallString<128> TagUSR;
       index::generateUSRForDecl(TagDecl, TagUSR);
       if (auto *Record = API.findRecordForUSR(TagUSR)) {
@@ -1159,6 +1182,11 @@ bool ExtractAPIVisitorBase<Derived>::VisitTypedefNameDecl(
             .append(Name, DeclarationFragments::FragmentKind::Identifier)
             .appendSemicolon();
 
+        // Replace the name and subheading in case it's underscored so we can
+        // use the non-underscored version
+        Record->Name = Name;
+        Record->SubHeading = DeclarationFragmentsBuilder::getSubHeading(Decl);
+
         return true;
       }
     }
diff --git a/clang/include/clang/Format/Format.h b/clang/include/clang/Format/Format.h
index fd526f189ec83..16956b4e0fbd4 100644
--- a/clang/include/clang/Format/Format.h
+++ b/clang/include/clang/Format/Format.h
@@ -1212,6 +1212,22 @@ struct FormatStyle {
   /// \version 3.7
   bool BinPackArguments;
 
+  /// If ``BinPackLongBracedList`` is ``true`` it overrides
+  /// ``BinPackArguments`` if there are 20 or more items in a braced
+  /// initializer list.
+  /// \code
+  ///    BinPackLongBracedList: false  vs.    BinPackLongBracedList: true
+  ///    vector<int> x{                       vector<int> x{1, 2, ...,
+  ///                                                       20, 21};
+  ///                1,
+  ///                2,
+  ///                ...,
+  ///                20,
+  ///                21};
+  /// \endcode
+  /// \version 21
+  bool BinPackLongBracedList;
+
   /// Different way to try to fit all parameters on a line.
   enum BinPackParametersStyle : int8_t {
     /// Bin-pack parameters.
@@ -2252,6 +2268,33 @@ struct FormatStyle {
   /// \version 16
   BreakBeforeInlineASMColonStyle BreakBeforeInlineASMColon;
 
+  /// If ``true``, break before a template closing bracket (``>``) when there is
+  /// a line break after the matching opening bracket (``<``).
+  /// \code
+  ///    true:
+  ///    template <typename Foo, typename Bar>
+  ///
+  ///    template <typename Foo,
+  ///              typename Bar>
+  ///
+  ///    template <
+  ///        typename Foo,
+  ///        typename Bar
+  ///    >
+  ///
+  ///    false:
+  ///    template <typename Foo, typename Bar>
+  ///
+  ///    template <typename Foo,
+  ///              typename Bar>
+  ///
+  ///    template <
+  ///        typename Foo,
+  ///        typename Bar>
+  /// \endcode
+  /// \version 21
+  bool BreakBeforeTemplateCloser;
+
   /// If ``true``, ternary operators will be placed after line breaks.
   /// \code
   ///    true:
@@ -3639,6 +3682,10 @@ struct FormatStyle {
   /// \version 3.7
   unsigned PenaltyBreakBeforeFirstCallParameter;
 
+  /// The penalty for breaking before a member access operator (``.``, ``->``).
+  /// \version 20
+  unsigned PenaltyBreakBeforeMemberAccess;
+
   /// The penalty for each line break introduced inside a comment.
   /// \version 3.7
   unsigned PenaltyBreakComment;
@@ -5235,6 +5282,7 @@ struct FormatStyle {
                R.AlwaysBreakBeforeMultilineStrings &&
            AttributeMacros == R.AttributeMacros &&
            BinPackArguments == R.BinPackArguments &&
+           BinPackLongBracedList == R.BinPackLongBracedList &&
            BinPackParameters == R.BinPackParameters &&
            BitFieldColonSpacing == R.BitFieldColonSpacing &&
            BracedInitializerIndentWidth == R.BracedInitializerIndentWidth &&
@@ -5247,6 +5295,7 @@ struct FormatStyle {
            BreakBeforeBraces == R.BreakBeforeBraces &&
            BreakBeforeConceptDeclarations == R.BreakBeforeConceptDeclarations &&
            BreakBeforeInlineASMColon == R.BreakBeforeInlineASMColon &&
+           BreakBeforeTemplateCloser == R.BreakBeforeTemplateCloser &&
            BreakBeforeTernaryOperators == R.BreakBeforeTernaryOperators &&
            BreakBinaryOperations == R.BreakBinaryOperations &&
            BreakConstructorInitializers == R.BreakConstructorInitializers &&
@@ -5311,6 +5360,7 @@ struct FormatStyle {
            PenaltyBreakAssignment == R.PenaltyBreakAssignment &&
            PenaltyBreakBeforeFirstCallParameter ==
                R.PenaltyBreakBeforeFirstCallParameter &&
+           PenaltyBreakBeforeMemberAccess == R.PenaltyBreakBeforeMemberAccess &&
            PenaltyBreakComment == R.PenaltyBreakComment &&
            PenaltyBreakFirstLessLess == R.PenaltyBreakFirstLessLess &&
            PenaltyBreakOpenParenthesis == R.PenaltyBreakOpenParenthesis &&
diff --git a/clang/include/clang/Frontend/FrontendOptions.h b/clang/include/clang/Frontend/FrontendOptions.h
index 8241925c98476..99b2d9a98ed1f 100644
--- a/clang/include/clang/Frontend/FrontendOptions.h
+++ b/clang/include/clang/Frontend/FrontendOptions.h
@@ -141,9 +141,6 @@ enum ActionKind {
   /// Dump template instantiations
   TemplightDump,
 
-  /// Run migrator.
-  MigrateSource,
-
   /// Just lex, no output.
   RunPreprocessorOnly,
 
@@ -330,10 +327,6 @@ class FrontendOptions {
   LLVM_PREFERRED_TYPE(bool)
   unsigned FixToTemporaries : 1;
 
-  /// Emit ARC errors even if the migrator can fix them.
-  LLVM_PREFERRED_TYPE(bool)
-  unsigned ARCMTMigrateEmitARCErrors : 1;
-
   /// Skip over function bodies to speed up parsing in cases you do not need
   /// them (e.g. with code completion).
   LLVM_PREFERRED_TYPE(bool)
@@ -424,72 +417,6 @@ class FrontendOptions {
   /// Specifies the output format of the AST.
   ASTDumpOutputFormat ASTDumpFormat = ADOF_Default;
 
-  enum {
-    ARCMT_None,
-    ARCMT_Check,
-    ARCMT_Modify,
-    ARCMT_Migrate
-  } ARCMTAction = ARCMT_None;
-
-  enum {
-    ObjCMT_None = 0,
-
-    /// Enable migration to modern ObjC literals.
-    ObjCMT_Literals = 0x1,
-
-    /// Enable migration to modern ObjC subscripting.
-    ObjCMT_Subscripting = 0x2,
-
-    /// Enable migration to modern ObjC readonly property.
-    ObjCMT_ReadonlyProperty = 0x4,
-
-    /// Enable migration to modern ObjC readwrite property.
-    ObjCMT_ReadwriteProperty = 0x8,
-
-    /// Enable migration to modern ObjC property.
-    ObjCMT_Property = (ObjCMT_ReadonlyProperty | ObjCMT_ReadwriteProperty),
-
-    /// Enable annotation of ObjCMethods of all kinds.
-    ObjCMT_Annotation = 0x10,
-
-    /// Enable migration of ObjC methods to 'instancetype'.
-    ObjCMT_Instancetype = 0x20,
-
-    /// Enable migration to NS_ENUM/NS_OPTIONS macros.
-    ObjCMT_NsMacros = 0x40,
-
-    /// Enable migration to add conforming protocols.
-    ObjCMT_ProtocolConformance = 0x80,
-
-    /// prefer 'atomic' property over 'nonatomic'.
-    ObjCMT_AtomicProperty = 0x100,
-
-    /// annotate property with NS_RETURNS_INNER_POINTER
-    ObjCMT_ReturnsInnerPointerProperty = 0x200,
-
-    /// use NS_NONATOMIC_IOSONLY for property 'atomic' attribute
-    ObjCMT_NsAtomicIOSOnlyProperty = 0x400,
-
-    /// Enable inferring NS_DESIGNATED_INITIALIZER for ObjC methods.
-    ObjCMT_DesignatedInitializer = 0x800,
-
-    /// Enable converting setter/getter expressions to property-dot syntx.
-    ObjCMT_PropertyDotSyntax = 0x1000,
-
-    ObjCMT_MigrateDecls = (ObjCMT_ReadonlyProperty | ObjCMT_ReadwriteProperty |
-                           ObjCMT_Annotation | ObjCMT_Instancetype |
-                           ObjCMT_NsMacros | ObjCMT_ProtocolConformance |
-                           ObjCMT_NsAtomicIOSOnlyProperty |
-                           ObjCMT_DesignatedInitializer),
-    ObjCMT_MigrateAll = (ObjCMT_Literals | ObjCMT_Subscripting |
-                         ObjCMT_MigrateDecls | ObjCMT_PropertyDotSyntax)
-  };
-  unsigned ObjCMTAction = ObjCMT_None;
-  std::string ObjCMTAllowListPath;
-
-  std::string MTMigrateDir;
-  std::string ARCMTMigrateReportOut;
-
   /// The input kind, either specified via -x argument or deduced from the input
   /// file name.
   InputKind DashX;
@@ -596,14 +523,14 @@ class FrontendOptions {
       : DisableFree(false), RelocatablePCH(false), ShowHelp(false),
         ShowStats(false), AppendStats(false), ShowVersion(false),
         FixWhatYouCan(false), FixOnlyWarnings(false), FixAndRecompile(false),
-        FixToTemporaries(false), ARCMTMigrateEmitARCErrors(false),
-        SkipFunctionBodies(false), UseGlobalModuleIndex(true),
-        GenerateGlobalModuleIndex(true), ASTDumpDecls(false),
-        ASTDumpLookups(false), BuildingImplicitModule(false),
-        BuildingImplicitModuleUsesLock(true), ModulesEmbedAllFiles(false),
-        IncludeTimestamps(true), UseTemporary(true),
-        AllowPCMWithCompilerErrors(false), ModulesShareFileManager(true),
-        EmitSymbolGraph(false), EmitExtensionSymbolGraphs(false),
+        FixToTemporaries(false), SkipFunctionBodies(false),
+        UseGlobalModuleIndex(true), GenerateGlobalModuleIndex(true),
+        ASTDumpDecls(false), ASTDumpLookups(false),
+        BuildingImplicitModule(false), BuildingImplicitModuleUsesLock(true),
+        ModulesEmbedAllFiles(false), IncludeTimestamps(true),
+        UseTemporary(true), AllowPCMWithCompilerErrors(false),
+        ModulesShareFileManager(true), EmitSymbolGraph(false),
+        EmitExtensionSymbolGraphs(false),
         EmitSymbolGraphSymbolLabelsForTesting(false),
         EmitPrettySymbolGraphs(false), GenReducedBMI(false),
         UseClangIRPipeline(false), TimeTraceGranularity(500),
diff --git a/clang/include/clang/Lex/Preprocessor.h b/clang/include/clang/Lex/Preprocessor.h
index 8ddc5b56eedbd..2bf4d1a166994 100644
--- a/clang/include/clang/Lex/Preprocessor.h
+++ b/clang/include/clang/Lex/Preprocessor.h
@@ -856,7 +856,7 @@ class Preprocessor {
           !PP.CurSubmoduleState->VisibleModules.getGeneration())
         return nullptr;
 
-      auto *Info = State.dyn_cast<ModuleMacroInfo*>();
+      auto *Info = dyn_cast_if_present<ModuleMacroInfo *>(State);
       if (!Info) {
         Info = new (PP.getPreprocessorAllocator())
             ModuleMacroInfo(cast<MacroDirective *>(State));
@@ -885,18 +885,18 @@ class Preprocessor {
     }
 
     ~MacroState() {
-      if (auto *Info = State.dyn_cast<ModuleMacroInfo*>())
+      if (auto *Info = dyn_cast_if_present<ModuleMacroInfo *>(State))
         Info->~ModuleMacroInfo();
     }
 
     MacroDirective *getLatest() const {
-      if (auto *Info = State.dyn_cast<ModuleMacroInfo*>())
+      if (auto *Info = dyn_cast_if_present<ModuleMacroInfo *>(State))
         return Info->MD;
       return cast<MacroDirective *>(State);
     }
 
     void setLatest(MacroDirective *MD) {
-      if (auto *Info = State.dyn_cast<ModuleMacroInfo*>())
+      if (auto *Info = dyn_cast_if_present<ModuleMacroInfo *>(State))
         Info->MD = MD;
       else
         State = MD;
@@ -933,14 +933,14 @@ class Preprocessor {
     }
 
     ArrayRef<ModuleMacro*> getOverriddenMacros() const {
-      if (auto *Info = State.dyn_cast<ModuleMacroInfo*>())
+      if (auto *Info = dyn_cast_if_present<ModuleMacroInfo *>(State))
         return Info->OverriddenMacros;
       return {};
     }
 
     void setOverriddenMacros(Preprocessor &PP,
                              ArrayRef<ModuleMacro *> Overrides) {
-      auto *Info = State.dyn_cast<ModuleMacroInfo*>();
+      auto *Info = dyn_cast_if_present<ModuleMacroInfo *>(State);
       if (!Info) {
         if (Overrides.empty())
           return;
diff --git a/clang/include/clang/Parse/Parser.h b/clang/include/clang/Parse/Parser.h
index e99d2cf2eaa40..335258d597028 100644
--- a/clang/include/clang/Parse/Parser.h
+++ b/clang/include/clang/Parse/Parser.h
@@ -3710,6 +3710,7 @@ class Parser : public CodeCompletionHandler {
     SourceLocation RParenLoc;
     SourceLocation EndLoc;
     SourceLocation MiscLoc;
+    OpenACCAtomicKind AtomicKind;
     SmallVector<Expr *> Exprs;
     SmallVector<OpenACCClause *> Clauses;
     // TODO OpenACC: As we implement support for the Atomic, Routine, and Cache
diff --git a/clang/include/clang/Sema/DeclSpec.h b/clang/include/clang/Sema/DeclSpec.h
index 06243f2624876..5f5df3a45d41d 100644
--- a/clang/include/clang/Sema/DeclSpec.h
+++ b/clang/include/clang/Sema/DeclSpec.h
@@ -1795,6 +1795,7 @@ class DecompositionDeclarator {
     IdentifierInfo *Name;
     SourceLocation NameLoc;
     std::optional<ParsedAttributes> Attrs;
+    SourceLocation EllipsisLoc;
   };
 
 private:
diff --git a/clang/include/clang/Sema/Overload.h b/clang/include/clang/Sema/Overload.h
index c7f2422b542dd..c03ec00d03dc5 100644
--- a/clang/include/clang/Sema/Overload.h
+++ b/clang/include/clang/Sema/Overload.h
@@ -933,7 +933,7 @@ class Sema;
     /// Have we matched any packs on the parameter side, versus any non-packs on
     /// the argument side, in a context where the opposite matching is also
     /// allowed?
-    bool HasMatchedPackOnParmToNonPackOnArg : 1;
+    bool StrictPackMatch : 1;
 
     /// True if the candidate was found using ADL.
     LLVM_PREFERRED_TYPE(CallExpr::ADLCallKind)
@@ -1010,8 +1010,7 @@ class Sema;
     friend class OverloadCandidateSet;
     OverloadCandidate()
         : IsSurrogate(false), IgnoreObjectArgument(false),
-          TookAddressOfOverload(false),
-          HasMatchedPackOnParmToNonPackOnArg(false),
+          TookAddressOfOverload(false), StrictPackMatch(false),
           IsADLCandidate(llvm::to_underlying(CallExpr::NotADL)),
           RewriteKind(CRK_None) {}
   };
diff --git a/clang/include/clang/Sema/Sema.h b/clang/include/clang/Sema/Sema.h
index 4d6e02fe2956e..1870d1271c556 100644
--- a/clang/include/clang/Sema/Sema.h
+++ b/clang/include/clang/Sema/Sema.h
@@ -16,6 +16,7 @@
 
 #include "clang/APINotes/APINotesManager.h"
 #include "clang/AST/ASTFwd.h"
+#include "clang/AST/ASTLambda.h"
 #include "clang/AST/Attr.h"
 #include "clang/AST/AttrIterator.h"
 #include "clang/AST/CharUnits.h"
@@ -231,7 +232,8 @@ void threadSafetyCleanup(BeforeSet *Cache);
 
 // FIXME: No way to easily map from TemplateTypeParmTypes to
 // TemplateTypeParmDecls, so we have this horrible PointerUnion.
-typedef std::pair<llvm::PointerUnion<const TemplateTypeParmType *, NamedDecl *>,
+typedef std::pair<llvm::PointerUnion<const TemplateTypeParmType *, NamedDecl *,
+                                     ResolvedUnexpandedPackExpr *>,
                   SourceLocation>
     UnexpandedParameterPack;
 
@@ -2330,7 +2332,8 @@ class Sema final : public SemaBase {
                          const FunctionProtoType *Proto);
 
   /// \param FPOnly restricts the arguments to floating-point types.
-  bool BuiltinVectorMath(CallExpr *TheCall, QualType &Res, bool FPOnly = false);
+  std::optional<QualType> BuiltinVectorMath(CallExpr *TheCall,
+                                            bool FPOnly = false);
   bool BuiltinVectorToScalarMath(CallExpr *TheCall);
 
   void checkLifetimeCaptureBy(FunctionDecl *FDecl, bool IsMemberFunction,
@@ -3661,6 +3664,12 @@ class Sema final : public SemaBase {
                              NonTrivialCUnionContext UseContext,
                              unsigned NonTrivialKind);
 
+  /// Certain globally-unique variables might be accidentally duplicated if
+  /// built into multiple shared libraries with hidden visibility. This can
+  /// cause problems if the variable is mutable, its initialization is
+  /// effectful, or its address is taken.
+  bool GloballyUniqueObjectMightBeAccidentallyDuplicated(const VarDecl *Dcl);
+
   /// AddInitializerToDecl - Adds the initializer Init to the
   /// declaration dcl. If DirectInit is true, this is C++ direct
   /// initialization rather than copy initialization.
@@ -6019,6 +6028,7 @@ class Sema final : public SemaBase {
                                          RecordDecl *ClassDecl,
                                          const IdentifierInfo *Name);
 
+  unsigned GetDecompositionElementCount(QualType DecompType);
   void CheckCompleteDecompositionDeclaration(DecompositionDecl *DD);
 
   /// Stack containing information needed when in C++2a an 'auto' is encountered
@@ -7498,10 +7508,15 @@ class Sema final : public SemaBase {
     return K == ConditionKind::Switch ? Context.IntTy : Context.BoolTy;
   }
 
-  // UsualUnaryConversions - promotes integers (C99 6.3.1.1p2) and converts
-  // functions and arrays to their respective pointers (C99 6.3.2.1).
+  // UsualUnaryConversions - promotes integers (C99 6.3.1.1p2), converts
+  // functions and arrays to their respective pointers (C99 6.3.2.1), and
+  // promotes floating-piont types according to the language semantics.
   ExprResult UsualUnaryConversions(Expr *E);
 
+  // UsualUnaryFPConversions - promotes floating-point types according to the
+  // current language semantics.
+  ExprResult UsualUnaryFPConversions(Expr *E);
+
   /// CallExprUnaryConversions - a special case of an unary conversion
   /// performed on a function designator of a call expression.
   ExprResult CallExprUnaryConversions(Expr *E);
@@ -7564,6 +7579,11 @@ class Sema final : public SemaBase {
   ExprResult DefaultVariadicArgumentPromotion(Expr *E, VariadicCallType CT,
                                               FunctionDecl *FDecl);
 
+  // Check that the usual arithmetic conversions can be performed on this pair
+  // of expressions that might be of enumeration type.
+  void checkEnumArithmeticConversions(Expr *LHS, Expr *RHS, SourceLocation Loc,
+                                      Sema::ArithConvKind ACK);
+
   // UsualArithmeticConversions - performs the UsualUnaryConversions on it's
   // operands and then handles various conversions that are common to binary
   // operators (C99 6.3.1.8). If both operands aren't arithmetic, this
@@ -9999,6 +10019,7 @@ class Sema final : public SemaBase {
     CCEK_CaseValue,    ///< Expression in a case label.
     CCEK_Enumerator,   ///< Enumerator value with fixed underlying type.
     CCEK_TemplateArg,  ///< Value of a non-type template parameter.
+    CCEK_InjectedTTP,  ///< Injected parameter of a template template parameter.
     CCEK_ArrayBound,   ///< Array bound in array declarator or new-expression.
     CCEK_ExplicitBool, ///< Condition in an explicit(bool) specifier.
     CCEK_Noexcept,     ///< Condition in a noexcept(bool) specifier.
@@ -10159,18 +10180,15 @@ class Sema final : public SemaBase {
   /// \param PartialOverloading true if we are performing "partial" overloading
   /// based on an incomplete set of function arguments. This feature is used by
   /// code completion.
-  void AddOverloadCandidate(FunctionDecl *Function, DeclAccessPair FoundDecl,
-                            ArrayRef<Expr *> Args,
-                            OverloadCandidateSet &CandidateSet,
-                            bool SuppressUserConversions = false,
-                            bool PartialOverloading = false,
-                            bool AllowExplicit = true,
-                            bool AllowExplicitConversion = false,
-                            ADLCallKind IsADLCandidate = ADLCallKind::NotADL,
-                            ConversionSequenceList EarlyConversions = {},
-                            OverloadCandidateParamOrder PO = {},
-                            bool AggregateCandidateDeduction = false,
-                            bool HasMatchedPackOnParmToNonPackOnArg = false);
+  void AddOverloadCandidate(
+      FunctionDecl *Function, DeclAccessPair FoundDecl, ArrayRef<Expr *> Args,
+      OverloadCandidateSet &CandidateSet, bool SuppressUserConversions = false,
+      bool PartialOverloading = false, bool AllowExplicit = true,
+      bool AllowExplicitConversion = false,
+      ADLCallKind IsADLCandidate = ADLCallKind::NotADL,
+      ConversionSequenceList EarlyConversions = {},
+      OverloadCandidateParamOrder PO = {},
+      bool AggregateCandidateDeduction = false, bool StrictPackMatch = false);
 
   /// Add all of the function declarations in the given function set to
   /// the overload candidate set.
@@ -10206,7 +10224,7 @@ class Sema final : public SemaBase {
                           bool PartialOverloading = false,
                           ConversionSequenceList EarlyConversions = {},
                           OverloadCandidateParamOrder PO = {},
-                          bool HasMatchedPackOnParmToNonPackOnArg = false);
+                          bool StrictPackMatch = false);
 
   /// Add a C++ member function template as a candidate to the candidate
   /// set, using template argument deduction to produce an appropriate member
@@ -10253,7 +10271,7 @@ class Sema final : public SemaBase {
       CXXRecordDecl *ActingContext, Expr *From, QualType ToType,
       OverloadCandidateSet &CandidateSet, bool AllowObjCConversionOnExplicit,
       bool AllowExplicit, bool AllowResultConversion = true,
-      bool HasMatchedPackOnParmToNonPackOnArg = false);
+      bool StrictPackMatch = false);
 
   /// Adds a conversion function template specialization
   /// candidate to the overload set, using template argument deduction
@@ -11649,6 +11667,33 @@ class Sema final : public SemaBase {
     CTAK_DeducedFromArrayBound
   };
 
+  struct CheckTemplateArgumentInfo {
+    explicit CheckTemplateArgumentInfo(bool PartialOrdering = false,
+                                       bool MatchingTTP = false)
+        : PartialOrdering(PartialOrdering), MatchingTTP(MatchingTTP) {}
+    CheckTemplateArgumentInfo(const CheckTemplateArgumentInfo &) = delete;
+    CheckTemplateArgumentInfo &
+    operator=(const CheckTemplateArgumentInfo &) = delete;
+
+    /// The checked, converted argument will be added to the
+    /// end of these vectors.
+    SmallVector<TemplateArgument, 4> SugaredConverted, CanonicalConverted;
+
+    /// The check is being performed in the context of partial ordering.
+    bool PartialOrdering;
+
+    /// If true, assume these template arguments are
+    /// the injected template arguments for a template template parameter.
+    /// This will relax the requirement that all its possible uses are valid:
+    /// TTP checking is loose, and assumes that invalid uses will be diagnosed
+    /// during instantiation.
+    bool MatchingTTP;
+
+    /// Is set to true when, in the context of TTP matching, a pack parameter
+    /// matches non-pack arguments.
+    bool StrictPackMatch = false;
+  };
+
   /// Check that the given template argument corresponds to the given
   /// template parameter.
   ///
@@ -11668,21 +11713,16 @@ class Sema final : public SemaBase {
   /// \param ArgumentPackIndex The index into the argument pack where this
   /// argument will be placed. Only valid if the parameter is a parameter pack.
   ///
-  /// \param Converted The checked, converted argument will be added to the
-  /// end of this small vector.
-  ///
   /// \param CTAK Describes how we arrived at this particular template argument:
   /// explicitly written, deduced, etc.
   ///
   /// \returns true on error, false otherwise.
-  bool
-  CheckTemplateArgument(NamedDecl *Param, TemplateArgumentLoc &Arg,
-                        NamedDecl *Template, SourceLocation TemplateLoc,
-                        SourceLocation RAngleLoc, unsigned ArgumentPackIndex,
-                        SmallVectorImpl<TemplateArgument> &SugaredConverted,
-                        SmallVectorImpl<TemplateArgument> &CanonicalConverted,
-                        CheckTemplateArgumentKind CTAK, bool PartialOrdering,
-                        bool *MatchedPackOnParmToNonPackOnArg);
+  bool CheckTemplateArgument(NamedDecl *Param, TemplateArgumentLoc &Arg,
+                             NamedDecl *Template, SourceLocation TemplateLoc,
+                             SourceLocation RAngleLoc,
+                             unsigned ArgumentPackIndex,
+                             CheckTemplateArgumentInfo &CTAI,
+                             CheckTemplateArgumentKind CTAK);
 
   /// Check that the given template arguments can be provided to
   /// the given template, converting the arguments along the way.
@@ -11715,22 +11755,15 @@ class Sema final : public SemaBase {
   /// \param DefaultArgs any default arguments from template specialization
   /// deduction.
   ///
-  /// \param PartialOrderingTTP If true, assume these template arguments are
-  /// the injected template arguments for a template template parameter.
-  /// This will relax the requirement that all its possible uses are valid:
-  /// TTP checking is loose, and assumes that invalid uses will be diagnosed
-  /// during instantiation.
-  ///
   /// \returns true if an error occurred, false otherwise.
-  bool CheckTemplateArgumentList(
-      TemplateDecl *Template, SourceLocation TemplateLoc,
-      TemplateArgumentListInfo &TemplateArgs,
-      const DefaultArguments &DefaultArgs, bool PartialTemplateArgs,
-      SmallVectorImpl<TemplateArgument> &SugaredConverted,
-      SmallVectorImpl<TemplateArgument> &CanonicalConverted,
-      bool UpdateArgsWithConversions = true,
-      bool *ConstraintsNotSatisfied = nullptr, bool PartialOrderingTTP = false,
-      bool *MatchedPackOnParmToNonPackOnArg = nullptr);
+  bool CheckTemplateArgumentList(TemplateDecl *Template,
+                                 SourceLocation TemplateLoc,
+                                 TemplateArgumentListInfo &TemplateArgs,
+                                 const DefaultArguments &DefaultArgs,
+                                 bool PartialTemplateArgs,
+                                 CheckTemplateArgumentInfo &CTAI,
+                                 bool UpdateArgsWithConversions = true,
+                                 bool *ConstraintsNotSatisfied = nullptr);
 
   bool CheckTemplateTypeArgument(
       TemplateTypeParmDecl *Param, TemplateArgumentLoc &Arg,
@@ -11755,6 +11788,7 @@ class Sema final : public SemaBase {
                                    QualType InstantiatedParamType, Expr *Arg,
                                    TemplateArgument &SugaredConverted,
                                    TemplateArgument &CanonicalConverted,
+                                   bool MatchingTTP,
                                    CheckTemplateArgumentKind CTAK);
 
   /// Check a template argument against its corresponding
@@ -11766,7 +11800,7 @@ class Sema final : public SemaBase {
                                      TemplateParameterList *Params,
                                      TemplateArgumentLoc &Arg,
                                      bool PartialOrdering,
-                                     bool *MatchedPackOnParmToNonPackOnArg);
+                                     bool *StrictPackMatch);
 
   void NoteTemplateLocation(const NamedDecl &Decl,
                             std::optional<SourceRange> ParamRange = {});
@@ -11776,6 +11810,9 @@ class Sema final : public SemaBase {
   /// declaration and the type of its corresponding non-type template
   /// parameter, produce an expression that properly refers to that
   /// declaration.
+  /// FIXME: This is used in some contexts where the resulting expression
+  /// doesn't need to live too long. It would be useful if this function
+  /// could return a temporary expression.
   ExprResult BuildExpressionFromDeclTemplateArgument(
       const TemplateArgument &Arg, QualType ParamType, SourceLocation Loc,
       NamedDecl *TemplateParam = nullptr);
@@ -11805,17 +11842,6 @@ class Sema final : public SemaBase {
     /// \endcode
     TPL_TemplateTemplateParmMatch,
 
-    /// We are matching the template parameter lists of a template
-    /// template argument against the template parameter lists of a template
-    /// template parameter.
-    ///
-    /// \code
-    /// template<template<int Value> class Metafun> struct X;
-    /// template<int Value> struct integer_c;
-    /// X<integer_c> xic;
-    /// \endcode
-    TPL_TemplateTemplateArgumentMatch,
-
     /// We are determining whether the template-parameters are equivalent
     /// according to C++ [temp.over.link]/6. This comparison does not consider
     /// constraints.
@@ -12468,7 +12494,7 @@ class Sema final : public SemaBase {
   bool isTemplateTemplateParameterAtLeastAsSpecializedAs(
       TemplateParameterList *PParam, TemplateDecl *PArg, TemplateDecl *AArg,
       const DefaultArguments &DefaultArgs, SourceLocation ArgLoc,
-      bool PartialOrdering, bool *MatchedPackOnParmToNonPackOnArg);
+      bool PartialOrdering, bool *StrictPackMatch);
 
   /// Mark which template parameters are used in a given expression.
   ///
@@ -13108,14 +13134,16 @@ class Sema final : public SemaBase {
               ? ExpressionEvaluationContext::ImmediateFunctionContext
               : ExpressionEvaluationContext::PotentiallyEvaluated);
       if (FD) {
+        auto &Current = S.currentEvaluationContext();
+        const auto &Parent = S.parentEvaluationContext();
+
         FD->setWillHaveBody(true);
-        S.ExprEvalContexts.back().InImmediateFunctionContext =
+        Current.InImmediateFunctionContext =
             FD->isImmediateFunction() ||
-            S.ExprEvalContexts[S.ExprEvalContexts.size() - 2]
-                .isConstantEvaluated() ||
-            S.ExprEvalContexts[S.ExprEvalContexts.size() - 2]
-                .isImmediateFunctionContext();
-        S.ExprEvalContexts.back().InImmediateEscalatingFunctionContext =
+            (isLambdaMethod(FD) && (Parent.isConstantEvaluated() ||
+                                    Parent.isImmediateFunctionContext()));
+
+        Current.InImmediateEscalatingFunctionContext =
             S.getLangOpts().CPlusPlus20 && FD->isImmediateEscalating();
       } else
         assert(isa<ObjCMethodDecl>(DC));
@@ -13468,8 +13496,8 @@ class Sema final : public SemaBase {
   bool InstantiateClassTemplateSpecialization(
       SourceLocation PointOfInstantiation,
       ClassTemplateSpecializationDecl *ClassTemplateSpec,
-      TemplateSpecializationKind TSK, bool Complain = true,
-      bool PrimaryHasMatchedPackOnParmToNonPackOnArg = false);
+      TemplateSpecializationKind TSK, bool Complain,
+      bool PrimaryStrictPackMatch);
 
   /// Instantiates the definitions of all of the member
   /// of the given class, which is an instantiation of a class template
diff --git a/clang/include/clang/Sema/SemaHLSL.h b/clang/include/clang/Sema/SemaHLSL.h
index f4cd11f423a84..6e8ca2e4710de 100644
--- a/clang/include/clang/Sema/SemaHLSL.h
+++ b/clang/include/clang/Sema/SemaHLSL.h
@@ -141,6 +141,9 @@ class SemaHLSL : public SemaBase {
   // Diagnose whether the input ID is uint/unit2/uint3 type.
   bool diagnoseInputIDType(QualType T, const ParsedAttr &AL);
 
+  bool CanPerformScalarCast(QualType SrcTy, QualType DestTy);
+  bool ContainsBitField(QualType BaseTy);
+  bool CanPerformElementwiseCast(Expr *Src, QualType DestType);
   ExprResult ActOnOutParamExpr(ParmVarDecl *Param, Expr *Arg);
 
   QualType getInoutParameterType(QualType Ty);
@@ -160,9 +163,9 @@ class SemaHLSL : public SemaBase {
   ResourceBindings Bindings;
 
 private:
-  void collectResourcesOnVarDecl(VarDecl *D);
-  void collectResourcesOnUserRecordDecl(const VarDecl *VD,
-                                        const RecordType *RT);
+  void collectResourceBindingsOnVarDecl(VarDecl *D);
+  void collectResourceBindingsOnUserRecordDecl(const VarDecl *VD,
+                                               const RecordType *RT);
   void processExplicitBindingsOnDecl(VarDecl *D);
 };
 
diff --git a/clang/include/clang/Sema/SemaOpenACC.h b/clang/include/clang/Sema/SemaOpenACC.h
index 2e5a0ea0aaac6..3004b98760a98 100644
--- a/clang/include/clang/Sema/SemaOpenACC.h
+++ b/clang/include/clang/Sema/SemaOpenACC.h
@@ -695,24 +695,53 @@ class SemaOpenACC : public SemaBase {
   /// should check legality of the statement as it appertains to this Construct.
   StmtResult ActOnAssociatedStmt(SourceLocation DirectiveLoc,
                                  OpenACCDirectiveKind K,
+                                 OpenACCAtomicKind AtKind,
                                  ArrayRef<const OpenACCClause *> Clauses,
                                  StmtResult AssocStmt);
 
+  StmtResult ActOnAssociatedStmt(SourceLocation DirectiveLoc,
+                                 OpenACCDirectiveKind K,
+                                 ArrayRef<const OpenACCClause *> Clauses,
+                                 StmtResult AssocStmt) {
+    return ActOnAssociatedStmt(DirectiveLoc, K, OpenACCAtomicKind::None,
+                               Clauses, AssocStmt);
+  }
+  /// Called to check the form of the `atomic` construct which has some fairly
+  /// sizable restrictions.
+  StmtResult CheckAtomicAssociatedStmt(SourceLocation AtomicDirLoc,
+                                       OpenACCAtomicKind AtKind,
+                                       StmtResult AssocStmt);
+
   /// Called after the directive has been completely parsed, including the
   /// declaration group or associated statement.
+  /// DirLoc: Location of the actual directive keyword.
   /// LParenLoc: Location of the left paren, if it exists (not on all
   /// constructs).
   /// MiscLoc: First misc location, if necessary (not all constructs).
   /// Exprs: List of expressions on the construct itself, if necessary (not all
   /// constructs).
+  /// AK: The atomic kind of the directive, if necessary (atomic only)
   /// RParenLoc: Location of the right paren, if it exists (not on all
   /// constructs).
+  /// EndLoc: The last source location of the driective.
+  /// Clauses: The list of clauses for the directive, if present.
+  /// AssocStmt: The associated statement for this construct, if necessary.
   StmtResult ActOnEndStmtDirective(
       OpenACCDirectiveKind K, SourceLocation StartLoc, SourceLocation DirLoc,
       SourceLocation LParenLoc, SourceLocation MiscLoc, ArrayRef<Expr *> Exprs,
-      SourceLocation RParenLoc, SourceLocation EndLoc,
+      OpenACCAtomicKind AK, SourceLocation RParenLoc, SourceLocation EndLoc,
       ArrayRef<OpenACCClause *> Clauses, StmtResult AssocStmt);
 
+  StmtResult ActOnEndStmtDirective(
+      OpenACCDirectiveKind K, SourceLocation StartLoc, SourceLocation DirLoc,
+      SourceLocation LParenLoc, SourceLocation MiscLoc, ArrayRef<Expr *> Exprs,
+      SourceLocation RParenLoc, SourceLocation EndLoc,
+      ArrayRef<OpenACCClause *> Clauses, StmtResult AssocStmt) {
+    return ActOnEndStmtDirective(K, StartLoc, DirLoc, LParenLoc, MiscLoc, Exprs,
+                                 OpenACCAtomicKind::None, RParenLoc, EndLoc,
+                                 Clauses, AssocStmt);
+  }
+
   /// Called after the directive has been completely parsed, including the
   /// declaration group or associated statement.
   DeclGroupRef ActOnEndDeclDirective();
diff --git a/clang/include/clang/Sema/SemaOpenMP.h b/clang/include/clang/Sema/SemaOpenMP.h
index a056a96f50233..fa244da36a322 100644
--- a/clang/include/clang/Sema/SemaOpenMP.h
+++ b/clang/include/clang/Sema/SemaOpenMP.h
@@ -849,6 +849,9 @@ class SemaOpenMP : public SemaBase {
       ArrayRef<OMPInteropInfo> AppendArgs, SourceLocation AdjustArgsLoc,
       SourceLocation AppendArgsLoc, SourceRange SR);
 
+  /// Called on device_num selector in context selectors.
+  void ActOnOpenMPDeviceNum(Expr *DeviceNumExpr);
+
   OMPClause *ActOnOpenMPSingleExprClause(OpenMPClauseKind Kind, Expr *Expr,
                                          SourceLocation StartLoc,
                                          SourceLocation LParenLoc,
@@ -1410,6 +1413,13 @@ class SemaOpenMP : public SemaBase {
 
   void handleOMPAssumeAttr(Decl *D, const ParsedAttr &AL);
 
+  /// Setter and getter functions for device_num.
+  void setOpenMPDeviceNum(int Num);
+
+  int getOpenMPDeviceNum() const;
+
+  void setOpenMPDeviceNumID(StringRef ID);
+
 private:
   void *VarDataSharingAttributesStack;
 
@@ -1480,6 +1490,12 @@ class SemaOpenMP : public SemaBase {
 
   /// All `omp assumes` we encountered so far.
   SmallVector<OMPAssumeAttr *, 4> OMPAssumeGlobal;
+
+  /// Device number specified by the context selector.
+  int DeviceNum = -1;
+
+  /// Device number identifier specified by the context selector.
+  StringRef DeviceNumID;
 };
 
 } // namespace clang
diff --git a/clang/include/clang/Sema/TemplateDeduction.h b/clang/include/clang/Sema/TemplateDeduction.h
index 9c12eef5c42a0..020e19bc7a608 100644
--- a/clang/include/clang/Sema/TemplateDeduction.h
+++ b/clang/include/clang/Sema/TemplateDeduction.h
@@ -54,7 +54,7 @@ class TemplateDeductionInfo {
   /// Have we matched any packs on the parameter side, versus any non-packs on
   /// the argument side, in a context where the opposite matching is also
   /// allowed?
-  bool MatchedPackOnParmToNonPackOnArg = false;
+  bool StrictPackMatch = false;
 
   /// The template parameter depth for which we're performing deduction.
   unsigned DeducedDepth;
@@ -92,13 +92,9 @@ class TemplateDeductionInfo {
     return DeducedDepth;
   }
 
-  bool hasMatchedPackOnParmToNonPackOnArg() const {
-    return MatchedPackOnParmToNonPackOnArg;
-  }
+  bool hasStrictPackMatch() const { return StrictPackMatch; }
 
-  void setMatchedPackOnParmToNonPackOnArg() {
-    MatchedPackOnParmToNonPackOnArg = true;
-  }
+  void setStrictPackMatch() { StrictPackMatch = true; }
 
   /// Get the number of explicitly-specified arguments.
   unsigned getNumExplicitArgs() const {
diff --git a/clang/include/clang/Serialization/ASTBitCodes.h b/clang/include/clang/Serialization/ASTBitCodes.h
index 1b56ed2c9776b..3c184db5b2adf 100644
--- a/clang/include/clang/Serialization/ASTBitCodes.h
+++ b/clang/include/clang/Serialization/ASTBitCodes.h
@@ -1908,6 +1908,7 @@ enum StmtCode {
   EXPR_PACK_EXPANSION,                    // PackExpansionExpr
   EXPR_PACK_INDEXING,                     // PackIndexingExpr
   EXPR_SIZEOF_PACK,                       // SizeOfPackExpr
+  EXPR_RESOLVED_UNEXPANDED_PACK,          // ResolvedUnexpandedPackExpr
   EXPR_SUBST_NON_TYPE_TEMPLATE_PARM,      // SubstNonTypeTemplateParmExpr
   EXPR_SUBST_NON_TYPE_TEMPLATE_PARM_PACK, // SubstNonTypeTemplateParmPackExpr
   EXPR_FUNCTION_PARM_PACK,                // FunctionParmPackExpr
@@ -2044,6 +2045,7 @@ enum StmtCode {
   STMT_OPENACC_SHUTDOWN_CONSTRUCT,
   STMT_OPENACC_SET_CONSTRUCT,
   STMT_OPENACC_UPDATE_CONSTRUCT,
+  STMT_OPENACC_ATOMIC_CONSTRUCT,
 
   // HLSL Constructs
   EXPR_HLSL_OUT_ARG,
diff --git a/clang/include/clang/StaticAnalyzer/Checkers/Checkers.td b/clang/include/clang/StaticAnalyzer/Checkers/Checkers.td
index 1361da46c3c81..9bf491eac1e0e 100644
--- a/clang/include/clang/StaticAnalyzer/Checkers/Checkers.td
+++ b/clang/include/clang/StaticAnalyzer/Checkers/Checkers.td
@@ -989,30 +989,41 @@ def decodeValueOfObjCType : Checker<"decodeValueOfObjCType">,
 
 let ParentPackage = Security in {
 
-def FloatLoopCounter : Checker<"FloatLoopCounter">,
-  HelpText<"Warn on using a floating point value as a loop counter (CERT: "
-           "FLP30-C, FLP30-CPP)">,
-  Dependencies<[SecuritySyntaxChecker]>,
-  Documentation<HasDocumentation>;
-
-def MmapWriteExecChecker : Checker<"MmapWriteExec">,
-  HelpText<"Warn on mmap() calls with both writable and executable access">,
-  Documentation<HasDocumentation>;
-
-def PointerSubChecker : Checker<"PointerSub">,
-  HelpText<"Check for pointer subtractions on two pointers pointing to "
-           "different memory chunks">,
-  Documentation<HasDocumentation>;
-
-def PutenvStackArray : Checker<"PutenvStackArray">,
-  HelpText<"Finds calls to the function 'putenv' which pass a pointer to "
-           "an automatic (stack-allocated) array as the argument.">,
-  Documentation<HasDocumentation>;
-
-def SetgidSetuidOrderChecker : Checker<"SetgidSetuidOrder">,
-  HelpText<"Warn on possible reversed order of 'setgid(getgid()))' and "
-           "'setuid(getuid())' (CERT: POS36-C)">,
-  Documentation<HasDocumentation>;
+  def ArrayBoundChecker : Checker<"ArrayBound">,
+                          HelpText<"Warn about out of bounds access to memory">,
+                          Documentation<HasDocumentation>;
+
+  def FloatLoopCounter
+      : Checker<"FloatLoopCounter">,
+        HelpText<
+            "Warn on using a floating point value as a loop counter (CERT: "
+            "FLP30-C, FLP30-CPP)">,
+        Dependencies<[SecuritySyntaxChecker]>,
+        Documentation<HasDocumentation>;
+
+  def MmapWriteExecChecker
+      : Checker<"MmapWriteExec">,
+        HelpText<
+            "Warn on mmap() calls with both writable and executable access">,
+        Documentation<HasDocumentation>;
+
+  def PointerSubChecker
+      : Checker<"PointerSub">,
+        HelpText<"Check for pointer subtractions on two pointers pointing to "
+                 "different memory chunks">,
+        Documentation<HasDocumentation>;
+
+  def PutenvStackArray
+      : Checker<"PutenvStackArray">,
+        HelpText<"Finds calls to the function 'putenv' which pass a pointer to "
+                 "an automatic (stack-allocated) array as the argument.">,
+        Documentation<HasDocumentation>;
+
+  def SetgidSetuidOrderChecker
+      : Checker<"SetgidSetuidOrder">,
+        HelpText<"Warn on possible reversed order of 'setgid(getgid()))' and "
+                 "'setuid(getuid())' (CERT: POS36-C)">,
+        Documentation<HasDocumentation>;
 
 } // end "security"
 
@@ -1035,14 +1046,6 @@ let ParentPackage = ENV in {
 
 let ParentPackage = SecurityAlpha in {
 
-def ArrayBoundChecker : Checker<"ArrayBound">,
-  HelpText<"Warn about buffer overflows (older checker)">,
-  Documentation<HasDocumentation>;
-
-def ArrayBoundCheckerV2 : Checker<"ArrayBoundV2">,
-  HelpText<"Warn about buffer overflows (newer checker)">,
-  Documentation<HasDocumentation>;
-
 def ReturnPointerRangeChecker : Checker<"ReturnPtrRange">,
   HelpText<"Check for an out-of-bound pointer being returned to callers">,
   Documentation<HasDocumentation>;
diff --git a/clang/include/clang/StaticAnalyzer/Core/BugReporter/BugReporter.h b/clang/include/clang/StaticAnalyzer/Core/BugReporter/BugReporter.h
index 8974342166fad..7563d8bbd1d27 100644
--- a/clang/include/clang/StaticAnalyzer/Core/BugReporter/BugReporter.h
+++ b/clang/include/clang/StaticAnalyzer/Core/BugReporter/BugReporter.h
@@ -748,8 +748,8 @@ class BugReporterContext {
 /// It can be valuable to produce tags with some bits of information and later
 /// reuse them for a better diagnostic.
 ///
-/// Please make sure that derived class' constuctor is private and that the user
-/// can only create objects using DataTag::Factory.  This also means that
+/// Please make sure that derived class' constructor is private and that the
+/// user can only create objects using DataTag::Factory.  This also means that
 /// DataTag::Factory should be friend for every derived class.
 class DataTag : public ProgramPointTag {
 public:
diff --git a/clang/include/clang/StaticAnalyzer/Core/PathSensitive/CallDescription.h b/clang/include/clang/StaticAnalyzer/Core/PathSensitive/CallDescription.h
index a99c11766f110..3c242898ef6cd 100644
--- a/clang/include/clang/StaticAnalyzer/Core/PathSensitive/CallDescription.h
+++ b/clang/include/clang/StaticAnalyzer/Core/PathSensitive/CallDescription.h
@@ -35,7 +35,7 @@ class CallDescription {
     /// Match calls to functions from the C standard library. This also
     /// recognizes builtin variants whose name is derived by adding
     /// "__builtin", "__inline" or similar prefixes or suffixes; but only
-    /// matches functions than are externally visible and are declared either
+    /// matches functions that are externally visible and are declared either
     /// directly within a TU or in the namespace 'std'.
     /// For the exact heuristics, see CheckerContext::isCLibraryFunction().
     CLibrary,
@@ -152,7 +152,7 @@ class CallDescription {
   /// exists only when that is not available, for example, when _only_
   /// syntactic check is done on a piece of code.
   ///
-  /// Also, StdLibraryFunctionsChecker::Signature is likely a better candicade
+  /// Also, StdLibraryFunctionsChecker::Signature is likely a better candidate
   /// for syntactic only matching if you are writing a new checker. This is
   /// handy if a CallDescriptionMap is already there.
   ///
@@ -233,7 +233,7 @@ template <typename T> class CallDescriptionMap {
   /// exists only when that is not available, for example, when _only_
   /// syntactic check is done on a piece of code.
   ///
-  /// Also, StdLibraryFunctionsChecker::Signature is likely a better candicade
+  /// Also, StdLibraryFunctionsChecker::Signature is likely a better candidate
   /// for syntactic only matching if you are writing a new checker. This is
   /// handy if a CallDescriptionMap is already there.
   ///
@@ -274,7 +274,7 @@ class CallDescriptionSet {
   /// exists only when that is not available, for example, when _only_
   /// syntactic check is done on a piece of code.
   ///
-  /// Also, StdLibraryFunctionsChecker::Signature is likely a better candicade
+  /// Also, StdLibraryFunctionsChecker::Signature is likely a better candidate
   /// for syntactic only matching if you are writing a new checker. This is
   /// handy if a CallDescriptionMap is already there.
   ///
diff --git a/clang/include/clang/StaticAnalyzer/Core/PathSensitive/MemRegion.h b/clang/include/clang/StaticAnalyzer/Core/PathSensitive/MemRegion.h
index f88bf70d72398..796fb43a2fc66 100644
--- a/clang/include/clang/StaticAnalyzer/Core/PathSensitive/MemRegion.h
+++ b/clang/include/clang/StaticAnalyzer/Core/PathSensitive/MemRegion.h
@@ -344,7 +344,7 @@ class GlobalImmutableSpaceRegion : public NonStaticGlobalSpaceRegion {
 };
 
 /// The region containing globals which can be modified by calls to
-/// "internally" defined functions - (for now just) functions other then system
+/// "internally" defined functions - (for now just) functions other than system
 /// calls.
 class GlobalInternalSpaceRegion : public NonStaticGlobalSpaceRegion {
   friend class MemRegionManager;
@@ -1021,7 +1021,7 @@ class NonParamVarRegion : public VarRegion {
   }
 };
 
-/// ParamVarRegion - Represents a region for paremters. Only parameters of the
+/// ParamVarRegion - Represents a region for parameters. Only parameters of the
 /// function in the current stack frame are represented as `ParamVarRegion`s.
 /// Parameters of top-level analyzed functions as well as captured paremeters
 /// by lambdas and blocks are repesented as `VarRegion`s.
diff --git a/clang/include/clang/StaticAnalyzer/Core/PathSensitive/SMTConstraintManager.h b/clang/include/clang/StaticAnalyzer/Core/PathSensitive/SMTConstraintManager.h
index 7cfb24e5e649d..e084a13995306 100644
--- a/clang/include/clang/StaticAnalyzer/Core/PathSensitive/SMTConstraintManager.h
+++ b/clang/include/clang/StaticAnalyzer/Core/PathSensitive/SMTConstraintManager.h
@@ -353,12 +353,7 @@ class SMTConstraintManager : public clang::ento::SimpleConstraintManager {
     addStateConstraints(NewState);
 
     std::optional<bool> res = Solver->check();
-    if (!res)
-      Cached[hash] = ConditionTruthVal();
-    else
-      Cached[hash] = ConditionTruthVal(*res);
-
-    return Cached[hash];
+    return Cached[hash] = res ? ConditionTruthVal(*res) : ConditionTruthVal();
   }
 
   // Cache the result of an SMT query (true, false, unknown). The key is the
diff --git a/clang/include/clang/Tooling/DependencyScanning/DependencyScanningService.h b/clang/include/clang/Tooling/DependencyScanning/DependencyScanningService.h
index 4a343f2872d8d..f002f8645d3f6 100644
--- a/clang/include/clang/Tooling/DependencyScanning/DependencyScanningService.h
+++ b/clang/include/clang/Tooling/DependencyScanning/DependencyScanningService.h
@@ -55,15 +55,18 @@ enum class ScanningOptimizations {
   HeaderSearch = 1,
 
   /// Remove warnings from system modules.
-  SystemWarnings = 2,
+  SystemWarnings = (1 << 1),
 
   /// Remove unused -ivfsoverlay arguments.
-  VFS = 4,
+  VFS = (1 << 2),
 
   /// Canonicalize -D and -U options.
-  Macros = 8,
+  Macros = (1 << 3),
 
-  DSS_LAST_BITMASK_ENUM(Macros),
+  /// Ignore the compiler's working directory if it is safe.
+  IgnoreCWD = (1 << 4),
+
+  DSS_LAST_BITMASK_ENUM(IgnoreCWD),
   Default = All
 };
 
diff --git a/clang/include/clang/Tooling/DependencyScanning/DependencyScanningTool.h b/clang/include/clang/Tooling/DependencyScanning/DependencyScanningTool.h
index ddb078dc16e3c..bcc9ea17e2588 100644
--- a/clang/include/clang/Tooling/DependencyScanning/DependencyScanningTool.h
+++ b/clang/include/clang/Tooling/DependencyScanning/DependencyScanningTool.h
@@ -128,14 +128,17 @@ class DependencyScanningTool {
   /// \param LookupModuleOutput This function is called to fill in
   ///                           "-fmodule-file=", "-o" and other output
   ///                           arguments for dependencies.
+  /// \param TUBuffer Optional memory buffer for translation unit input. If
+  ///                 TUBuffer is nullopt, the input should be included in the
+  ///                 Commandline already.
   ///
   /// \returns a \c StringError with the diagnostic output if clang errors
   /// occurred, \c TranslationUnitDeps otherwise.
-  llvm::Expected<TranslationUnitDeps>
-  getTranslationUnitDependencies(const std::vector<std::string> &CommandLine,
-                                 StringRef CWD,
-                                 const llvm::DenseSet<ModuleID> &AlreadySeen,
-                                 LookupModuleOutputCallback LookupModuleOutput);
+  llvm::Expected<TranslationUnitDeps> getTranslationUnitDependencies(
+      const std::vector<std::string> &CommandLine, StringRef CWD,
+      const llvm::DenseSet<ModuleID> &AlreadySeen,
+      LookupModuleOutputCallback LookupModuleOutput,
+      std::optional<llvm::MemoryBufferRef> TUBuffer = std::nullopt);
 
   /// Given a compilation context specified via the Clang driver command-line,
   /// gather modular dependencies of module with the given name, and return the
diff --git a/clang/include/clang/Tooling/DependencyScanning/DependencyScanningWorker.h b/clang/include/clang/Tooling/DependencyScanning/DependencyScanningWorker.h
index da6e0401411a3..ee7582b851020 100644
--- a/clang/include/clang/Tooling/DependencyScanning/DependencyScanningWorker.h
+++ b/clang/include/clang/Tooling/DependencyScanning/DependencyScanningWorker.h
@@ -17,6 +17,7 @@
 #include "clang/Tooling/DependencyScanning/ModuleDepCollector.h"
 #include "llvm/Support/Error.h"
 #include "llvm/Support/FileSystem.h"
+#include "llvm/Support/MemoryBufferRef.h"
 #include <optional>
 #include <string>
 
@@ -83,9 +84,21 @@ class DependencyScanningWorker {
                            llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> FS);
 
   /// Run the dependency scanning tool for a given clang driver command-line,
-  /// and report the discovered dependencies to the provided consumer. If \p
-  /// ModuleName isn't empty, this function reports the dependencies of module
-  /// \p ModuleName.
+  /// and report the discovered dependencies to the provided consumer. If
+  /// TUBuffer is not nullopt, it is used as TU input for the dependency
+  /// scanning. Otherwise, the input should be included as part of the
+  /// command-line.
+  ///
+  /// \returns false if clang errors occurred (with diagnostics reported to
+  /// \c DiagConsumer), true otherwise.
+  bool computeDependencies(
+      StringRef WorkingDirectory, const std::vector<std::string> &CommandLine,
+      DependencyConsumer &DepConsumer, DependencyActionController &Controller,
+      DiagnosticConsumer &DiagConsumer,
+      std::optional<llvm::MemoryBufferRef> TUBuffer = std::nullopt);
+
+  /// Run the dependency scanning tool for a given clang driver command-line
+  /// for a specific module.
   ///
   /// \returns false if clang errors occurred (with diagnostics reported to
   /// \c DiagConsumer), true otherwise.
@@ -94,13 +107,28 @@ class DependencyScanningWorker {
                            DependencyConsumer &DepConsumer,
                            DependencyActionController &Controller,
                            DiagnosticConsumer &DiagConsumer,
-                           std::optional<StringRef> ModuleName = std::nullopt);
+                           StringRef ModuleName);
+
+  /// Run the dependency scanning tool for a given clang driver command-line
+  /// for a specific translation unit via file system or memory buffer.
+  ///
   /// \returns A \c StringError with the diagnostic output if clang errors
   /// occurred, success otherwise.
   llvm::Error computeDependencies(
       StringRef WorkingDirectory, const std::vector<std::string> &CommandLine,
       DependencyConsumer &Consumer, DependencyActionController &Controller,
-      std::optional<StringRef> ModuleName = std::nullopt);
+      std::optional<llvm::MemoryBufferRef> TUBuffer = std::nullopt);
+
+  /// Run the dependency scanning tool for a given clang driver command-line
+  /// for a specific module.
+  ///
+  /// \returns A \c StringError with the diagnostic output if clang errors
+  /// occurred, success otherwise.
+  llvm::Error computeDependencies(StringRef WorkingDirectory,
+                                  const std::vector<std::string> &CommandLine,
+                                  DependencyConsumer &Consumer,
+                                  DependencyActionController &Controller,
+                                  StringRef ModuleName);
 
   bool shouldEagerLoadModules() const { return EagerLoadModules; }
 
@@ -121,6 +149,15 @@ class DependencyScanningWorker {
   ScanningOptimizations OptimizeArgs;
   /// Whether to set up command-lines to load PCM files eagerly.
   bool EagerLoadModules;
+
+  /// Private helper functions.
+  bool scanDependencies(StringRef WorkingDirectory,
+                        const std::vector<std::string> &CommandLine,
+                        DependencyConsumer &Consumer,
+                        DependencyActionController &Controller,
+                        DiagnosticConsumer &DC,
+                        llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> FS,
+                        std::optional<StringRef> ModuleName);
 };
 
 } // end namespace dependencies
diff --git a/clang/include/module.modulemap b/clang/include/module.modulemap
index f00dede7fd526..fb8e445cb4b72 100644
--- a/clang/include/module.modulemap
+++ b/clang/include/module.modulemap
@@ -44,20 +44,12 @@ module Clang_Basic {
   textual header "clang/Basic/BuiltinsAArch64NeonSVEBridge_cg.def"
   textual header "clang/Basic/BuiltinsAMDGPU.def"
   textual header "clang/Basic/BuiltinsARM.def"
-  textual header "clang/Basic/BuiltinsHexagon.def"
-  textual header "clang/Basic/BuiltinsHexagonDep.def"
   textual header "clang/Basic/BuiltinsHexagonMapCustomDep.def"
-  textual header "clang/Basic/BuiltinsLoongArch.def"
   textual header "clang/Basic/BuiltinsLoongArchBase.def"
   textual header "clang/Basic/BuiltinsLoongArchLASX.def"
   textual header "clang/Basic/BuiltinsLoongArchLSX.def"
   textual header "clang/Basic/BuiltinsMips.def"
-  textual header "clang/Basic/BuiltinsNEON.def"
-  textual header "clang/Basic/BuiltinsNVPTX.def"
   textual header "clang/Basic/BuiltinsPPC.def"
-  textual header "clang/Basic/BuiltinsRISCVVector.def"
-  textual header "clang/Basic/BuiltinsSME.def"
-  textual header "clang/Basic/BuiltinsSVE.def"
   textual header "clang/Basic/BuiltinsSystemZ.def"
   textual header "clang/Basic/BuiltinsVE.def"
   textual header "clang/Basic/BuiltinsVEVL.gen.def"
@@ -116,6 +108,8 @@ module Clang_Diagnostics {
   module Parse { header "clang/Basic/DiagnosticParse.h" export * }
   module Serialization { header "clang/Serialization/SerializationDiagnostic.h" export * }
   module Refactoring { header "clang/Tooling/Refactoring/RefactoringDiagnostic.h" export * }
+
+  textual header "clang/Basic/AllDiagnosticKinds.inc"
 }
 
 module Clang_Driver {
diff --git a/clang/lib/APINotes/APINotesManager.cpp b/clang/lib/APINotes/APINotesManager.cpp
index 70d96c735503f..7f8a126ffaa03 100644
--- a/clang/lib/APINotes/APINotesManager.cpp
+++ b/clang/lib/APINotes/APINotesManager.cpp
@@ -56,7 +56,7 @@ APINotesManager::APINotesManager(SourceManager &SM, const LangOptions &LangOpts)
 APINotesManager::~APINotesManager() {
   // Free the API notes readers.
   for (const auto &Entry : Readers) {
-    if (auto Reader = Entry.second.dyn_cast<APINotesReader *>())
+    if (auto Reader = dyn_cast_if_present<APINotesReader *>(Entry.second))
       delete Reader;
   }
 
@@ -381,7 +381,7 @@ APINotesManager::findAPINotes(SourceLocation Loc) {
       }
 
       // We have the answer.
-      if (auto Reader = Known->second.dyn_cast<APINotesReader *>())
+      if (auto Reader = dyn_cast_if_present<APINotesReader *>(Known->second))
         Results.push_back(Reader);
       break;
     }
diff --git a/clang/lib/ARCMigrate/ARCMT.cpp b/clang/lib/ARCMigrate/ARCMT.cpp
deleted file mode 100644
index 1a8a200f2fc19..0000000000000
--- a/clang/lib/ARCMigrate/ARCMT.cpp
+++ /dev/null
@@ -1,616 +0,0 @@
-//===--- ARCMT.cpp - Migration to ARC mode --------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "clang/ARCMigrate/ARCMT.h"
-#include "Internals.h"
-#include "clang/AST/ASTConsumer.h"
-#include "clang/Basic/DiagnosticCategories.h"
-#include "clang/Frontend/ASTUnit.h"
-#include "clang/Frontend/CompilerInstance.h"
-#include "clang/Frontend/FrontendAction.h"
-#include "clang/Frontend/TextDiagnosticPrinter.h"
-#include "clang/Frontend/Utils.h"
-#include "clang/Lex/Preprocessor.h"
-#include "clang/Lex/PreprocessorOptions.h"
-#include "clang/Rewrite/Core/Rewriter.h"
-#include "clang/Sema/SemaDiagnostic.h"
-#include "clang/Serialization/ASTReader.h"
-#include "llvm/Support/MemoryBuffer.h"
-#include "llvm/TargetParser/Triple.h"
-#include <utility>
-using namespace clang;
-using namespace arcmt;
-
-bool CapturedDiagList::clearDiagnostic(ArrayRef<unsigned> IDs,
-                                       SourceRange range) {
-  if (range.isInvalid())
-    return false;
-
-  bool cleared = false;
-  ListTy::iterator I = List.begin();
-  while (I != List.end()) {
-    FullSourceLoc diagLoc = I->getLocation();
-    if ((IDs.empty() || // empty means clear all diagnostics in the range.
-         llvm::is_contained(IDs, I->getID())) &&
-        !diagLoc.isBeforeInTranslationUnitThan(range.getBegin()) &&
-        (diagLoc == range.getEnd() ||
-         diagLoc.isBeforeInTranslationUnitThan(range.getEnd()))) {
-      cleared = true;
-      ListTy::iterator eraseS = I++;
-      if (eraseS->getLevel() != DiagnosticsEngine::Note)
-        while (I != List.end() && I->getLevel() == DiagnosticsEngine::Note)
-          ++I;
-      // Clear the diagnostic and any notes following it.
-      I = List.erase(eraseS, I);
-      continue;
-    }
-
-    ++I;
-  }
-
-  return cleared;
-}
-
-bool CapturedDiagList::hasDiagnostic(ArrayRef<unsigned> IDs,
-                                     SourceRange range) const {
-  if (range.isInvalid())
-    return false;
-
-  ListTy::const_iterator I = List.begin();
-  while (I != List.end()) {
-    FullSourceLoc diagLoc = I->getLocation();
-    if ((IDs.empty() || // empty means any diagnostic in the range.
-         llvm::is_contained(IDs, I->getID())) &&
-        !diagLoc.isBeforeInTranslationUnitThan(range.getBegin()) &&
-        (diagLoc == range.getEnd() ||
-         diagLoc.isBeforeInTranslationUnitThan(range.getEnd()))) {
-      return true;
-    }
-
-    ++I;
-  }
-
-  return false;
-}
-
-void CapturedDiagList::reportDiagnostics(DiagnosticsEngine &Diags) const {
-  for (ListTy::const_iterator I = List.begin(), E = List.end(); I != E; ++I)
-    Diags.Report(*I);
-}
-
-bool CapturedDiagList::hasErrors() const {
-  for (ListTy::const_iterator I = List.begin(), E = List.end(); I != E; ++I)
-    if (I->getLevel() >= DiagnosticsEngine::Error)
-      return true;
-
-  return false;
-}
-
-namespace {
-
-class CaptureDiagnosticConsumer : public DiagnosticConsumer {
-  DiagnosticsEngine &Diags;
-  DiagnosticConsumer &DiagClient;
-  CapturedDiagList &CapturedDiags;
-  bool HasBegunSourceFile;
-public:
-  CaptureDiagnosticConsumer(DiagnosticsEngine &diags,
-                            DiagnosticConsumer &client,
-                            CapturedDiagList &capturedDiags)
-    : Diags(diags), DiagClient(client), CapturedDiags(capturedDiags),
-      HasBegunSourceFile(false) { }
-
-  void BeginSourceFile(const LangOptions &Opts,
-                       const Preprocessor *PP) override {
-    // Pass BeginSourceFile message onto DiagClient on first call.
-    // The corresponding EndSourceFile call will be made from an
-    // explicit call to FinishCapture.
-    if (!HasBegunSourceFile) {
-      DiagClient.BeginSourceFile(Opts, PP);
-      HasBegunSourceFile = true;
-    }
-  }
-
-  void FinishCapture() {
-    // Call EndSourceFile on DiagClient on completion of capture to
-    // enable VerifyDiagnosticConsumer to check diagnostics *after*
-    // it has received the diagnostic list.
-    if (HasBegunSourceFile) {
-      DiagClient.EndSourceFile();
-      HasBegunSourceFile = false;
-    }
-  }
-
-  ~CaptureDiagnosticConsumer() override {
-    assert(!HasBegunSourceFile && "FinishCapture not called!");
-  }
-
-  void HandleDiagnostic(DiagnosticsEngine::Level level,
-                        const Diagnostic &Info) override {
-    if (DiagnosticIDs::isARCDiagnostic(Info.getID()) ||
-        level >= DiagnosticsEngine::Error || level == DiagnosticsEngine::Note) {
-      if (Info.getLocation().isValid())
-        CapturedDiags.push_back(StoredDiagnostic(level, Info));
-      return;
-    }
-
-    // Non-ARC warnings are ignored.
-    Diags.setLastDiagnosticIgnored(true);
-  }
-};
-
-} // end anonymous namespace
-
-static bool HasARCRuntime(CompilerInvocation &origCI) {
-  // This duplicates some functionality from Darwin::AddDeploymentTarget
-  // but this function is well defined, so keep it decoupled from the driver
-  // and avoid unrelated complications.
-  llvm::Triple triple(origCI.getTargetOpts().Triple);
-
-  if (triple.isiOS())
-    return triple.getOSMajorVersion() >= 5;
-
-  if (triple.isWatchOS())
-    return true;
-
-  if (triple.getOS() == llvm::Triple::Darwin)
-    return triple.getOSMajorVersion() >= 11;
-
-  if (triple.getOS() == llvm::Triple::MacOSX) {
-    return triple.getOSVersion() >= VersionTuple(10, 7);
-  }
-
-  return false;
-}
-
-static CompilerInvocation *
-createInvocationForMigration(CompilerInvocation &origCI,
-                             const PCHContainerReader &PCHContainerRdr) {
-  std::unique_ptr<CompilerInvocation> CInvok;
-  CInvok.reset(new CompilerInvocation(origCI));
-  PreprocessorOptions &PPOpts = CInvok->getPreprocessorOpts();
-  if (!PPOpts.ImplicitPCHInclude.empty()) {
-    // We can't use a PCH because it was likely built in non-ARC mode and we
-    // want to parse in ARC. Include the original header.
-    FileManager FileMgr(origCI.getFileSystemOpts());
-    IntrusiveRefCntPtr<DiagnosticIDs> DiagID(new DiagnosticIDs());
-    IntrusiveRefCntPtr<DiagnosticsEngine> Diags(
-        new DiagnosticsEngine(DiagID, &origCI.getDiagnosticOpts(),
-                              new IgnoringDiagConsumer()));
-    std::string OriginalFile = ASTReader::getOriginalSourceFile(
-        PPOpts.ImplicitPCHInclude, FileMgr, PCHContainerRdr, *Diags);
-    if (!OriginalFile.empty())
-      PPOpts.Includes.insert(PPOpts.Includes.begin(), OriginalFile);
-    PPOpts.ImplicitPCHInclude.clear();
-  }
-  std::string define = std::string(getARCMTMacroName());
-  define += '=';
-  CInvok->getPreprocessorOpts().addMacroDef(define);
-  CInvok->getLangOpts().ObjCAutoRefCount = true;
-  CInvok->getLangOpts().setGC(LangOptions::NonGC);
-  CInvok->getDiagnosticOpts().ErrorLimit = 0;
-  CInvok->getDiagnosticOpts().PedanticErrors = 0;
-
-  // Ignore -Werror flags when migrating.
-  std::vector<std::string> WarnOpts;
-  for (std::vector<std::string>::iterator
-         I = CInvok->getDiagnosticOpts().Warnings.begin(),
-         E = CInvok->getDiagnosticOpts().Warnings.end(); I != E; ++I) {
-    if (!StringRef(*I).starts_with("error"))
-      WarnOpts.push_back(*I);
-  }
-  WarnOpts.push_back("error=arc-unsafe-retained-assign");
-  CInvok->getDiagnosticOpts().Warnings = std::move(WarnOpts);
-
-  CInvok->getLangOpts().ObjCWeakRuntime = HasARCRuntime(origCI);
-  CInvok->getLangOpts().ObjCWeak = CInvok->getLangOpts().ObjCWeakRuntime;
-
-  return CInvok.release();
-}
-
-static void emitPremigrationErrors(const CapturedDiagList &arcDiags,
-                                   DiagnosticOptions *diagOpts,
-                                   Preprocessor &PP) {
-  TextDiagnosticPrinter printer(llvm::errs(), diagOpts);
-  IntrusiveRefCntPtr<DiagnosticIDs> DiagID(new DiagnosticIDs());
-  IntrusiveRefCntPtr<DiagnosticsEngine> Diags(
-      new DiagnosticsEngine(DiagID, diagOpts, &printer,
-                            /*ShouldOwnClient=*/false));
-  Diags->setSourceManager(&PP.getSourceManager());
-
-  printer.BeginSourceFile(PP.getLangOpts(), &PP);
-  arcDiags.reportDiagnostics(*Diags);
-  printer.EndSourceFile();
-}
-
-//===----------------------------------------------------------------------===//
-// checkForManualIssues.
-//===----------------------------------------------------------------------===//
-
-bool arcmt::checkForManualIssues(
-    CompilerInvocation &origCI, const FrontendInputFile &Input,
-    std::shared_ptr<PCHContainerOperations> PCHContainerOps,
-    DiagnosticConsumer *DiagClient, bool emitPremigrationARCErrors,
-    StringRef plistOut) {
-  if (!origCI.getLangOpts().ObjC)
-    return false;
-
-  LangOptions::GCMode OrigGCMode = origCI.getLangOpts().getGC();
-  bool NoNSAllocReallocError = origCI.getMigratorOpts().NoNSAllocReallocError;
-  bool NoFinalizeRemoval = origCI.getMigratorOpts().NoFinalizeRemoval;
-
-  std::vector<TransformFn> transforms = arcmt::getAllTransformations(OrigGCMode,
-                                                                     NoFinalizeRemoval);
-  assert(!transforms.empty());
-
-  std::unique_ptr<CompilerInvocation> CInvok;
-  CInvok.reset(
-      createInvocationForMigration(origCI, PCHContainerOps->getRawReader()));
-  CInvok->getFrontendOpts().Inputs.clear();
-  CInvok->getFrontendOpts().Inputs.push_back(Input);
-
-  CapturedDiagList capturedDiags;
-
-  assert(DiagClient);
-  IntrusiveRefCntPtr<DiagnosticIDs> DiagID(new DiagnosticIDs());
-  IntrusiveRefCntPtr<DiagnosticsEngine> Diags(
-      new DiagnosticsEngine(DiagID, &origCI.getDiagnosticOpts(),
-                            DiagClient, /*ShouldOwnClient=*/false));
-
-  // Filter of all diagnostics.
-  CaptureDiagnosticConsumer errRec(*Diags, *DiagClient, capturedDiags);
-  Diags->setClient(&errRec, /*ShouldOwnClient=*/false);
-
-  std::unique_ptr<ASTUnit> Unit(ASTUnit::LoadFromCompilerInvocationAction(
-      std::move(CInvok), PCHContainerOps, Diags));
-  if (!Unit) {
-    errRec.FinishCapture();
-    return true;
-  }
-
-  // Don't filter diagnostics anymore.
-  Diags->setClient(DiagClient, /*ShouldOwnClient=*/false);
-
-  ASTContext &Ctx = Unit->getASTContext();
-
-  if (Diags->hasFatalErrorOccurred()) {
-    Diags->Reset();
-    DiagClient->BeginSourceFile(Ctx.getLangOpts(), &Unit->getPreprocessor());
-    capturedDiags.reportDiagnostics(*Diags);
-    DiagClient->EndSourceFile();
-    errRec.FinishCapture();
-    return true;
-  }
-
-  if (emitPremigrationARCErrors)
-    emitPremigrationErrors(capturedDiags, &origCI.getDiagnosticOpts(),
-                           Unit->getPreprocessor());
-  if (!plistOut.empty()) {
-    SmallVector<StoredDiagnostic, 8> arcDiags;
-    for (CapturedDiagList::iterator
-           I = capturedDiags.begin(), E = capturedDiags.end(); I != E; ++I)
-      arcDiags.push_back(*I);
-    writeARCDiagsToPlist(std::string(plistOut), arcDiags,
-                         Ctx.getSourceManager(), Ctx.getLangOpts());
-  }
-
-  // After parsing of source files ended, we want to reuse the
-  // diagnostics objects to emit further diagnostics.
-  // We call BeginSourceFile because DiagnosticConsumer requires that
-  // diagnostics with source range information are emitted only in between
-  // BeginSourceFile() and EndSourceFile().
-  DiagClient->BeginSourceFile(Ctx.getLangOpts(), &Unit->getPreprocessor());
-
-  // No macros will be added since we are just checking and we won't modify
-  // source code.
-  std::vector<SourceLocation> ARCMTMacroLocs;
-
-  TransformActions testAct(*Diags, capturedDiags, Ctx, Unit->getPreprocessor());
-  MigrationPass pass(Ctx, OrigGCMode, Unit->getSema(), testAct, capturedDiags,
-                     ARCMTMacroLocs);
-  pass.setNoFinalizeRemoval(NoFinalizeRemoval);
-  if (!NoNSAllocReallocError)
-    Diags->setSeverity(diag::warn_arcmt_nsalloc_realloc, diag::Severity::Error,
-                       SourceLocation());
-
-  for (unsigned i=0, e = transforms.size(); i != e; ++i)
-    transforms[i](pass);
-
-  capturedDiags.reportDiagnostics(*Diags);
-
-  DiagClient->EndSourceFile();
-  errRec.FinishCapture();
-
-  return capturedDiags.hasErrors() || testAct.hasReportedErrors();
-}
-
-//===----------------------------------------------------------------------===//
-// applyTransformations.
-//===----------------------------------------------------------------------===//
-
-static bool
-applyTransforms(CompilerInvocation &origCI, const FrontendInputFile &Input,
-                std::shared_ptr<PCHContainerOperations> PCHContainerOps,
-                DiagnosticConsumer *DiagClient, StringRef outputDir,
-                bool emitPremigrationARCErrors, StringRef plistOut) {
-  if (!origCI.getLangOpts().ObjC)
-    return false;
-
-  LangOptions::GCMode OrigGCMode = origCI.getLangOpts().getGC();
-
-  // Make sure checking is successful first.
-  CompilerInvocation CInvokForCheck(origCI);
-  if (arcmt::checkForManualIssues(CInvokForCheck, Input, PCHContainerOps,
-                                  DiagClient, emitPremigrationARCErrors,
-                                  plistOut))
-    return true;
-
-  CompilerInvocation CInvok(origCI);
-  CInvok.getFrontendOpts().Inputs.clear();
-  CInvok.getFrontendOpts().Inputs.push_back(Input);
-
-  MigrationProcess migration(CInvok, PCHContainerOps, DiagClient, outputDir);
-  bool NoFinalizeRemoval = origCI.getMigratorOpts().NoFinalizeRemoval;
-
-  std::vector<TransformFn> transforms = arcmt::getAllTransformations(OrigGCMode,
-                                                                     NoFinalizeRemoval);
-  assert(!transforms.empty());
-
-  for (unsigned i=0, e = transforms.size(); i != e; ++i) {
-    bool err = migration.applyTransform(transforms[i]);
-    if (err) return true;
-  }
-
-  IntrusiveRefCntPtr<DiagnosticIDs> DiagID(new DiagnosticIDs());
-  IntrusiveRefCntPtr<DiagnosticsEngine> Diags(
-      new DiagnosticsEngine(DiagID, &origCI.getDiagnosticOpts(),
-                            DiagClient, /*ShouldOwnClient=*/false));
-
-  if (outputDir.empty()) {
-    origCI.getLangOpts().ObjCAutoRefCount = true;
-    return migration.getRemapper().overwriteOriginal(*Diags);
-  } else {
-    return migration.getRemapper().flushToDisk(outputDir, *Diags);
-  }
-}
-
-bool arcmt::applyTransformations(
-    CompilerInvocation &origCI, const FrontendInputFile &Input,
-    std::shared_ptr<PCHContainerOperations> PCHContainerOps,
-    DiagnosticConsumer *DiagClient) {
-  return applyTransforms(origCI, Input, PCHContainerOps, DiagClient,
-                         StringRef(), false, StringRef());
-}
-
-bool arcmt::migrateWithTemporaryFiles(
-    CompilerInvocation &origCI, const FrontendInputFile &Input,
-    std::shared_ptr<PCHContainerOperations> PCHContainerOps,
-    DiagnosticConsumer *DiagClient, StringRef outputDir,
-    bool emitPremigrationARCErrors, StringRef plistOut) {
-  assert(!outputDir.empty() && "Expected output directory path");
-  return applyTransforms(origCI, Input, PCHContainerOps, DiagClient, outputDir,
-                         emitPremigrationARCErrors, plistOut);
-}
-
-bool arcmt::getFileRemappings(std::vector<std::pair<std::string,std::string> > &
-                                  remap,
-                              StringRef outputDir,
-                              DiagnosticConsumer *DiagClient) {
-  assert(!outputDir.empty());
-
-  IntrusiveRefCntPtr<DiagnosticIDs> DiagID(new DiagnosticIDs());
-  IntrusiveRefCntPtr<DiagnosticsEngine> Diags(
-      new DiagnosticsEngine(DiagID, new DiagnosticOptions,
-                            DiagClient, /*ShouldOwnClient=*/false));
-
-  FileRemapper remapper;
-  bool err = remapper.initFromDisk(outputDir, *Diags,
-                                   /*ignoreIfFilesChanged=*/true);
-  if (err)
-    return true;
-
-  remapper.forEachMapping(
-      [&](StringRef From, StringRef To) {
-        remap.push_back(std::make_pair(From.str(), To.str()));
-      },
-      [](StringRef, const llvm::MemoryBufferRef &) {});
-
-  return false;
-}
-
-
-//===----------------------------------------------------------------------===//
-// CollectTransformActions.
-//===----------------------------------------------------------------------===//
-
-namespace {
-
-class ARCMTMacroTrackerPPCallbacks : public PPCallbacks {
-  std::vector<SourceLocation> &ARCMTMacroLocs;
-
-public:
-  ARCMTMacroTrackerPPCallbacks(std::vector<SourceLocation> &ARCMTMacroLocs)
-    : ARCMTMacroLocs(ARCMTMacroLocs) { }
-
-  void MacroExpands(const Token &MacroNameTok, const MacroDefinition &MD,
-                    SourceRange Range, const MacroArgs *Args) override {
-    if (MacroNameTok.getIdentifierInfo()->getName() == getARCMTMacroName())
-      ARCMTMacroLocs.push_back(MacroNameTok.getLocation());
-  }
-};
-
-class ARCMTMacroTrackerAction : public ASTFrontendAction {
-  std::vector<SourceLocation> &ARCMTMacroLocs;
-
-public:
-  ARCMTMacroTrackerAction(std::vector<SourceLocation> &ARCMTMacroLocs)
-    : ARCMTMacroLocs(ARCMTMacroLocs) { }
-
-  std::unique_ptr<ASTConsumer> CreateASTConsumer(CompilerInstance &CI,
-                                                 StringRef InFile) override {
-    CI.getPreprocessor().addPPCallbacks(
-               std::make_unique<ARCMTMacroTrackerPPCallbacks>(ARCMTMacroLocs));
-    return std::make_unique<ASTConsumer>();
-  }
-};
-
-class RewritesApplicator : public TransformActions::RewriteReceiver {
-  Rewriter &rewriter;
-  MigrationProcess::RewriteListener *Listener;
-
-public:
-  RewritesApplicator(Rewriter &rewriter, ASTContext &ctx,
-                     MigrationProcess::RewriteListener *listener)
-    : rewriter(rewriter), Listener(listener) {
-    if (Listener)
-      Listener->start(ctx);
-  }
-  ~RewritesApplicator() override {
-    if (Listener)
-      Listener->finish();
-  }
-
-  void insert(SourceLocation loc, StringRef text) override {
-    bool err = rewriter.InsertText(loc, text, /*InsertAfter=*/true,
-                                   /*indentNewLines=*/true);
-    if (!err && Listener)
-      Listener->insert(loc, text);
-  }
-
-  void remove(CharSourceRange range) override {
-    Rewriter::RewriteOptions removeOpts;
-    removeOpts.IncludeInsertsAtBeginOfRange = false;
-    removeOpts.IncludeInsertsAtEndOfRange = false;
-    removeOpts.RemoveLineIfEmpty = true;
-
-    bool err = rewriter.RemoveText(range, removeOpts);
-    if (!err && Listener)
-      Listener->remove(range);
-  }
-
-  void increaseIndentation(CharSourceRange range,
-                            SourceLocation parentIndent) override {
-    rewriter.IncreaseIndentation(range, parentIndent);
-  }
-};
-
-} // end anonymous namespace.
-
-/// Anchor for VTable.
-MigrationProcess::RewriteListener::~RewriteListener() { }
-
-MigrationProcess::MigrationProcess(
-    CompilerInvocation &CI,
-    std::shared_ptr<PCHContainerOperations> PCHContainerOps,
-    DiagnosticConsumer *diagClient, StringRef outputDir)
-    : OrigCI(CI), PCHContainerOps(std::move(PCHContainerOps)),
-      DiagClient(diagClient), HadARCErrors(false) {
-  if (!outputDir.empty()) {
-    IntrusiveRefCntPtr<DiagnosticIDs> DiagID(new DiagnosticIDs());
-    IntrusiveRefCntPtr<DiagnosticsEngine> Diags(
-      new DiagnosticsEngine(DiagID, &CI.getDiagnosticOpts(),
-                            DiagClient, /*ShouldOwnClient=*/false));
-    Remapper.initFromDisk(outputDir, *Diags, /*ignoreIfFilesChanged=*/true);
-  }
-}
-
-bool MigrationProcess::applyTransform(TransformFn trans,
-                                      RewriteListener *listener) {
-  std::unique_ptr<CompilerInvocation> CInvok;
-  CInvok.reset(
-      createInvocationForMigration(OrigCI, PCHContainerOps->getRawReader()));
-  CInvok->getDiagnosticOpts().IgnoreWarnings = true;
-
-  Remapper.applyMappings(CInvok->getPreprocessorOpts());
-
-  CapturedDiagList capturedDiags;
-  std::vector<SourceLocation> ARCMTMacroLocs;
-
-  assert(DiagClient);
-  IntrusiveRefCntPtr<DiagnosticIDs> DiagID(new DiagnosticIDs());
-  IntrusiveRefCntPtr<DiagnosticsEngine> Diags(
-      new DiagnosticsEngine(DiagID, new DiagnosticOptions,
-                            DiagClient, /*ShouldOwnClient=*/false));
-
-  // Filter of all diagnostics.
-  CaptureDiagnosticConsumer errRec(*Diags, *DiagClient, capturedDiags);
-  Diags->setClient(&errRec, /*ShouldOwnClient=*/false);
-
-  std::unique_ptr<ARCMTMacroTrackerAction> ASTAction;
-  ASTAction.reset(new ARCMTMacroTrackerAction(ARCMTMacroLocs));
-
-  std::unique_ptr<ASTUnit> Unit(ASTUnit::LoadFromCompilerInvocationAction(
-      std::move(CInvok), PCHContainerOps, Diags, ASTAction.get()));
-  if (!Unit) {
-    errRec.FinishCapture();
-    return true;
-  }
-  Unit->setOwnsRemappedFileBuffers(false); // FileRemapper manages that.
-
-  HadARCErrors = HadARCErrors || capturedDiags.hasErrors();
-
-  // Don't filter diagnostics anymore.
-  Diags->setClient(DiagClient, /*ShouldOwnClient=*/false);
-
-  ASTContext &Ctx = Unit->getASTContext();
-
-  if (Diags->hasFatalErrorOccurred()) {
-    Diags->Reset();
-    DiagClient->BeginSourceFile(Ctx.getLangOpts(), &Unit->getPreprocessor());
-    capturedDiags.reportDiagnostics(*Diags);
-    DiagClient->EndSourceFile();
-    errRec.FinishCapture();
-    return true;
-  }
-
-  // After parsing of source files ended, we want to reuse the
-  // diagnostics objects to emit further diagnostics.
-  // We call BeginSourceFile because DiagnosticConsumer requires that
-  // diagnostics with source range information are emitted only in between
-  // BeginSourceFile() and EndSourceFile().
-  DiagClient->BeginSourceFile(Ctx.getLangOpts(), &Unit->getPreprocessor());
-
-  Rewriter rewriter(Ctx.getSourceManager(), Ctx.getLangOpts());
-  TransformActions TA(*Diags, capturedDiags, Ctx, Unit->getPreprocessor());
-  MigrationPass pass(Ctx, OrigCI.getLangOpts().getGC(),
-                     Unit->getSema(), TA, capturedDiags, ARCMTMacroLocs);
-
-  trans(pass);
-
-  {
-    RewritesApplicator applicator(rewriter, Ctx, listener);
-    TA.applyRewrites(applicator);
-  }
-
-  DiagClient->EndSourceFile();
-  errRec.FinishCapture();
-
-  if (DiagClient->getNumErrors())
-    return true;
-
-  for (Rewriter::buffer_iterator
-        I = rewriter.buffer_begin(), E = rewriter.buffer_end(); I != E; ++I) {
-    FileID FID = I->first;
-    llvm::RewriteBuffer &buf = I->second;
-    OptionalFileEntryRef file =
-        Ctx.getSourceManager().getFileEntryRefForID(FID);
-    assert(file);
-    std::string newFname = std::string(file->getName());
-    newFname += "-trans";
-    SmallString<512> newText;
-    llvm::raw_svector_ostream vecOS(newText);
-    buf.write(vecOS);
-    std::unique_ptr<llvm::MemoryBuffer> memBuf(
-        llvm::MemoryBuffer::getMemBufferCopy(newText.str(), newFname));
-    SmallString<64> filePath(file->getName());
-    Unit->getFileManager().FixupRelativePath(filePath);
-    Remapper.remap(filePath.str(), std::move(memBuf));
-  }
-
-  return false;
-}
diff --git a/clang/lib/ARCMigrate/ARCMTActions.cpp b/clang/lib/ARCMigrate/ARCMTActions.cpp
deleted file mode 100644
index 0805d90d25aa6..0000000000000
--- a/clang/lib/ARCMigrate/ARCMTActions.cpp
+++ /dev/null
@@ -1,59 +0,0 @@
-//===--- ARCMTActions.cpp - ARC Migrate Tool Frontend Actions ---*- C++ -*-===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "clang/ARCMigrate/ARCMTActions.h"
-#include "clang/ARCMigrate/ARCMT.h"
-#include "clang/Frontend/CompilerInstance.h"
-
-using namespace clang;
-using namespace arcmt;
-
-bool CheckAction::BeginInvocation(CompilerInstance &CI) {
-  if (arcmt::checkForManualIssues(CI.getInvocation(), getCurrentInput(),
-                                  CI.getPCHContainerOperations(),
-                                  CI.getDiagnostics().getClient()))
-    return false; // errors, stop the action.
-
-  // We only want to see warnings reported from arcmt::checkForManualIssues.
-  CI.getDiagnostics().setIgnoreAllWarnings(true);
-  return true;
-}
-
-CheckAction::CheckAction(std::unique_ptr<FrontendAction> WrappedAction)
-  : WrapperFrontendAction(std::move(WrappedAction)) {}
-
-bool ModifyAction::BeginInvocation(CompilerInstance &CI) {
-  return !arcmt::applyTransformations(CI.getInvocation(), getCurrentInput(),
-                                      CI.getPCHContainerOperations(),
-                                      CI.getDiagnostics().getClient());
-}
-
-ModifyAction::ModifyAction(std::unique_ptr<FrontendAction> WrappedAction)
-  : WrapperFrontendAction(std::move(WrappedAction)) {}
-
-bool MigrateAction::BeginInvocation(CompilerInstance &CI) {
-  if (arcmt::migrateWithTemporaryFiles(
-          CI.getInvocation(), getCurrentInput(), CI.getPCHContainerOperations(),
-          CI.getDiagnostics().getClient(), MigrateDir, EmitPremigrationARCErrors,
-          PlistOut))
-    return false; // errors, stop the action.
-
-  // We only want to see diagnostics emitted by migrateWithTemporaryFiles.
-  CI.getDiagnostics().setIgnoreAllWarnings(true);
-  return true;
-}
-
-MigrateAction::MigrateAction(std::unique_ptr<FrontendAction> WrappedAction,
-                             StringRef migrateDir,
-                             StringRef plistOut,
-                             bool emitPremigrationARCErrors)
-  : WrapperFrontendAction(std::move(WrappedAction)), MigrateDir(migrateDir),
-    PlistOut(plistOut), EmitPremigrationARCErrors(emitPremigrationARCErrors) {
-  if (MigrateDir.empty())
-    MigrateDir = "."; // user current directory if none is given.
-}
diff --git a/clang/lib/ARCMigrate/CMakeLists.txt b/clang/lib/ARCMigrate/CMakeLists.txt
deleted file mode 100644
index 515d0960920af..0000000000000
--- a/clang/lib/ARCMigrate/CMakeLists.txt
+++ /dev/null
@@ -1,48 +0,0 @@
-set(LLVM_LINK_COMPONENTS
-  Support
-  TargetParser
-  )
-
-# By default MSVC has a 2^16 limit on the number of sections in an object
-# file, and Transforms.cpp needs more than that.
-if (MSVC)
-  set_source_files_properties(Transforms.cpp PROPERTIES COMPILE_FLAGS /bigobj)
-endif()
-
-add_clang_library(clangARCMigrate
-  ARCMT.cpp
-  ARCMTActions.cpp
-  FileRemapper.cpp
-  ObjCMT.cpp
-  PlistReporter.cpp
-  TransAPIUses.cpp
-  TransARCAssign.cpp
-  TransAutoreleasePool.cpp
-  TransBlockObjCVariable.cpp
-  TransEmptyStatementsAndDealloc.cpp
-  TransGCAttrs.cpp
-  TransGCCalls.cpp
-  TransProperties.cpp
-  TransProtectedScope.cpp
-  TransRetainReleaseDealloc.cpp
-  TransUnbridgedCasts.cpp
-  TransUnusedInitDelegate.cpp
-  TransZeroOutPropsInDealloc.cpp
-  TransformActions.cpp
-  Transforms.cpp
-
-  LINK_LIBS
-  clangAST
-  clangAnalysis
-  clangBasic
-  clangEdit
-  clangFrontend
-  clangLex
-  clangRewrite
-  clangSema
-  clangSerialization
-
-  DEPENDS
-  omp_gen
-  ClangDriverOptions
-  )
diff --git a/clang/lib/ARCMigrate/FileRemapper.cpp b/clang/lib/ARCMigrate/FileRemapper.cpp
deleted file mode 100644
index 84024c3bafdca..0000000000000
--- a/clang/lib/ARCMigrate/FileRemapper.cpp
+++ /dev/null
@@ -1,274 +0,0 @@
-//===--- FileRemapper.cpp - File Remapping Helper -------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "clang/ARCMigrate/FileRemapper.h"
-#include "clang/Basic/Diagnostic.h"
-#include "clang/Basic/FileManager.h"
-#include "clang/Lex/PreprocessorOptions.h"
-#include "llvm/Support/FileSystem.h"
-#include "llvm/Support/MemoryBuffer.h"
-#include "llvm/Support/Path.h"
-#include "llvm/Support/raw_ostream.h"
-#include <fstream>
-
-using namespace clang;
-using namespace arcmt;
-
-FileRemapper::FileRemapper() {
-  FileMgr.reset(new FileManager(FileSystemOptions()));
-}
-
-FileRemapper::~FileRemapper() {
-  clear();
-}
-
-void FileRemapper::clear(StringRef outputDir) {
-  for (MappingsTy::iterator
-         I = FromToMappings.begin(), E = FromToMappings.end(); I != E; ++I)
-    resetTarget(I->second);
-  FromToMappings.clear();
-  assert(ToFromMappings.empty());
-  if (!outputDir.empty()) {
-    std::string infoFile = getRemapInfoFile(outputDir);
-    llvm::sys::fs::remove(infoFile);
-  }
-}
-
-std::string FileRemapper::getRemapInfoFile(StringRef outputDir) {
-  assert(!outputDir.empty());
-  SmallString<128> InfoFile = outputDir;
-  llvm::sys::path::append(InfoFile, "remap");
-  return std::string(InfoFile);
-}
-
-bool FileRemapper::initFromDisk(StringRef outputDir, DiagnosticsEngine &Diag,
-                                bool ignoreIfFilesChanged) {
-  std::string infoFile = getRemapInfoFile(outputDir);
-  return initFromFile(infoFile, Diag, ignoreIfFilesChanged);
-}
-
-bool FileRemapper::initFromFile(StringRef filePath, DiagnosticsEngine &Diag,
-                                bool ignoreIfFilesChanged) {
-  assert(FromToMappings.empty() &&
-         "initFromDisk should be called before any remap calls");
-  std::string infoFile = std::string(filePath);
-  if (!llvm::sys::fs::exists(infoFile))
-    return false;
-
-  std::vector<std::pair<FileEntryRef, FileEntryRef>> pairs;
-
-  llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>> fileBuf =
-      llvm::MemoryBuffer::getFile(infoFile, /*IsText=*/true);
-  if (!fileBuf)
-    return report("Error opening file: " + infoFile, Diag);
-
-  SmallVector<StringRef, 64> lines;
-  fileBuf.get()->getBuffer().split(lines, "\n");
-
-  for (unsigned idx = 0; idx+3 <= lines.size(); idx += 3) {
-    StringRef fromFilename = lines[idx];
-    unsigned long long timeModified;
-    if (lines[idx+1].getAsInteger(10, timeModified))
-      return report("Invalid file data: '" + lines[idx+1] + "' not a number",
-                    Diag);
-    StringRef toFilename = lines[idx+2];
-
-    auto origFE = FileMgr->getOptionalFileRef(fromFilename);
-    if (!origFE) {
-      if (ignoreIfFilesChanged)
-        continue;
-      return report("File does not exist: " + fromFilename, Diag);
-    }
-    auto newFE = FileMgr->getOptionalFileRef(toFilename);
-    if (!newFE) {
-      if (ignoreIfFilesChanged)
-        continue;
-      return report("File does not exist: " + toFilename, Diag);
-    }
-
-    if ((uint64_t)origFE->getModificationTime() != timeModified) {
-      if (ignoreIfFilesChanged)
-        continue;
-      return report("File was modified: " + fromFilename, Diag);
-    }
-
-    pairs.push_back(std::make_pair(*origFE, *newFE));
-  }
-
-  for (unsigned i = 0, e = pairs.size(); i != e; ++i)
-    remap(pairs[i].first, pairs[i].second);
-
-  return false;
-}
-
-bool FileRemapper::flushToDisk(StringRef outputDir, DiagnosticsEngine &Diag) {
-  using namespace llvm::sys;
-
-  if (fs::create_directory(outputDir))
-    return report("Could not create directory: " + outputDir, Diag);
-
-  std::string infoFile = getRemapInfoFile(outputDir);
-  return flushToFile(infoFile, Diag);
-}
-
-bool FileRemapper::flushToFile(StringRef outputPath, DiagnosticsEngine &Diag) {
-  using namespace llvm::sys;
-
-  std::error_code EC;
-  std::string infoFile = std::string(outputPath);
-  llvm::raw_fd_ostream infoOut(infoFile, EC, llvm::sys::fs::OF_Text);
-  if (EC)
-    return report(EC.message(), Diag);
-
-  for (MappingsTy::iterator
-         I = FromToMappings.begin(), E = FromToMappings.end(); I != E; ++I) {
-
-    FileEntryRef origFE = I->first;
-    SmallString<200> origPath = StringRef(origFE.getName());
-    fs::make_absolute(origPath);
-    infoOut << origPath << '\n';
-    infoOut << (uint64_t)origFE.getModificationTime() << '\n';
-
-    if (const auto *FE = std::get_if<FileEntryRef>(&I->second)) {
-      SmallString<200> newPath = StringRef(FE->getName());
-      fs::make_absolute(newPath);
-      infoOut << newPath << '\n';
-    } else {
-
-      SmallString<64> tempPath;
-      int fd;
-      if (fs::createTemporaryFile(
-              path::filename(origFE.getName()),
-              path::extension(origFE.getName()).drop_front(), fd, tempPath,
-              llvm::sys::fs::OF_Text))
-        return report("Could not create file: " + tempPath.str(), Diag);
-
-      llvm::raw_fd_ostream newOut(fd, /*shouldClose=*/true);
-      llvm::MemoryBuffer *mem = std::get<llvm::MemoryBuffer *>(I->second);
-      newOut.write(mem->getBufferStart(), mem->getBufferSize());
-      newOut.close();
-
-      auto newE = FileMgr->getOptionalFileRef(tempPath);
-      if (newE) {
-        remap(origFE, *newE);
-        infoOut << newE->getName() << '\n';
-      }
-    }
-  }
-
-  infoOut.close();
-  return false;
-}
-
-bool FileRemapper::overwriteOriginal(DiagnosticsEngine &Diag,
-                                     StringRef outputDir) {
-  using namespace llvm::sys;
-
-  for (MappingsTy::iterator
-         I = FromToMappings.begin(), E = FromToMappings.end(); I != E; ++I) {
-    FileEntryRef origFE = I->first;
-    assert(std::holds_alternative<llvm::MemoryBuffer *>(I->second));
-    if (!fs::exists(origFE.getName()))
-      return report(StringRef("File does not exist: ") + origFE.getName(),
-                    Diag);
-
-    std::error_code EC;
-    llvm::raw_fd_ostream Out(origFE.getName(), EC, llvm::sys::fs::OF_None);
-    if (EC)
-      return report(EC.message(), Diag);
-
-    llvm::MemoryBuffer *mem = std::get<llvm::MemoryBuffer *>(I->second);
-    Out.write(mem->getBufferStart(), mem->getBufferSize());
-    Out.close();
-  }
-
-  clear(outputDir);
-  return false;
-}
-
-void FileRemapper::forEachMapping(
-    llvm::function_ref<void(StringRef, StringRef)> CaptureFile,
-    llvm::function_ref<void(StringRef, const llvm::MemoryBufferRef &)>
-        CaptureBuffer) const {
-  for (auto &Mapping : FromToMappings) {
-    if (const auto *FE = std::get_if<FileEntryRef>(&Mapping.second)) {
-      CaptureFile(Mapping.first.getName(), FE->getName());
-      continue;
-    }
-    CaptureBuffer(
-        Mapping.first.getName(),
-        std::get<llvm::MemoryBuffer *>(Mapping.second)->getMemBufferRef());
-  }
-}
-
-void FileRemapper::applyMappings(PreprocessorOptions &PPOpts) const {
-  for (MappingsTy::const_iterator
-         I = FromToMappings.begin(), E = FromToMappings.end(); I != E; ++I) {
-    if (const auto *FE = std::get_if<FileEntryRef>(&I->second)) {
-      PPOpts.addRemappedFile(I->first.getName(), FE->getName());
-    } else {
-      llvm::MemoryBuffer *mem = std::get<llvm::MemoryBuffer *>(I->second);
-      PPOpts.addRemappedFile(I->first.getName(), mem);
-    }
-  }
-
-  PPOpts.RetainRemappedFileBuffers = true;
-}
-
-void FileRemapper::remap(StringRef filePath,
-                         std::unique_ptr<llvm::MemoryBuffer> memBuf) {
-  OptionalFileEntryRef File = getOriginalFile(filePath);
-  assert(File);
-  remap(*File, std::move(memBuf));
-}
-
-void FileRemapper::remap(FileEntryRef File,
-                         std::unique_ptr<llvm::MemoryBuffer> MemBuf) {
-  auto [It, New] = FromToMappings.insert({File, nullptr});
-  if (!New)
-    resetTarget(It->second);
-  It->second = MemBuf.release();
-}
-
-void FileRemapper::remap(FileEntryRef File, FileEntryRef NewFile) {
-  auto [It, New] = FromToMappings.insert({File, nullptr});
-  if (!New)
-    resetTarget(It->second);
-  It->second = NewFile;
-  ToFromMappings.insert({NewFile, File});
-}
-
-OptionalFileEntryRef FileRemapper::getOriginalFile(StringRef filePath) {
-  OptionalFileEntryRef File = FileMgr->getOptionalFileRef(filePath);
-  if (!File)
-    return std::nullopt;
-  // If we are updating a file that overridden an original file,
-  // actually update the original file.
-  auto I = ToFromMappings.find(*File);
-  if (I != ToFromMappings.end()) {
-    *File = I->second;
-    assert(FromToMappings.contains(*File) && "Original file not in mappings!");
-  }
-  return File;
-}
-
-void FileRemapper::resetTarget(Target &targ) {
-  if (std::holds_alternative<llvm::MemoryBuffer *>(targ)) {
-    llvm::MemoryBuffer *oldmem = std::get<llvm::MemoryBuffer *>(targ);
-    delete oldmem;
-  } else {
-    FileEntryRef toFE = std::get<FileEntryRef>(targ);
-    ToFromMappings.erase(toFE);
-  }
-}
-
-bool FileRemapper::report(const Twine &err, DiagnosticsEngine &Diag) {
-  Diag.Report(Diag.getCustomDiagID(DiagnosticsEngine::Error, "%0"))
-      << err.str();
-  return true;
-}
diff --git a/clang/lib/ARCMigrate/Internals.h b/clang/lib/ARCMigrate/Internals.h
deleted file mode 100644
index de6ebdce1bea1..0000000000000
--- a/clang/lib/ARCMigrate/Internals.h
+++ /dev/null
@@ -1,180 +0,0 @@
-//===-- Internals.h - Implementation Details---------------------*- C++ -*-===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LLVM_CLANG_LIB_ARCMIGRATE_INTERNALS_H
-#define LLVM_CLANG_LIB_ARCMIGRATE_INTERNALS_H
-
-#include "clang/Basic/LangOptions.h"
-#include "clang/Basic/Diagnostic.h"
-#include "clang/Frontend/MigratorOptions.h"
-#include "llvm/ADT/ArrayRef.h"
-#include <list>
-#include <optional>
-
-namespace clang {
-  class ASTContext;
-  class Sema;
-  class Stmt;
-
-namespace arcmt {
-
-class CapturedDiagList {
-  typedef std::list<StoredDiagnostic> ListTy;
-  ListTy List;
-
-public:
-  void push_back(const StoredDiagnostic &diag) { List.push_back(diag); }
-
-  bool clearDiagnostic(ArrayRef<unsigned> IDs, SourceRange range);
-  bool hasDiagnostic(ArrayRef<unsigned> IDs, SourceRange range) const;
-
-  void reportDiagnostics(DiagnosticsEngine &diags) const;
-
-  bool hasErrors() const;
-
-  typedef ListTy::const_iterator iterator;
-  iterator begin() const { return List.begin(); }
-  iterator end()   const { return List.end();   }
-};
-
-void writeARCDiagsToPlist(const std::string &outPath,
-                          ArrayRef<StoredDiagnostic> diags,
-                          SourceManager &SM, const LangOptions &LangOpts);
-
-class TransformActions {
-  DiagnosticsEngine &Diags;
-  CapturedDiagList &CapturedDiags;
-  void *Impl; // TransformActionsImpl.
-
-public:
-  TransformActions(DiagnosticsEngine &diag, CapturedDiagList &capturedDiags,
-                   ASTContext &ctx, Preprocessor &PP);
-  ~TransformActions();
-
-  void startTransaction();
-  bool commitTransaction();
-  void abortTransaction();
-
-  void insert(SourceLocation loc, StringRef text);
-  void insertAfterToken(SourceLocation loc, StringRef text);
-  void remove(SourceRange range);
-  void removeStmt(Stmt *S);
-  void replace(SourceRange range, StringRef text);
-  void replace(SourceRange range, SourceRange replacementRange);
-  void replaceStmt(Stmt *S, StringRef text);
-  void replaceText(SourceLocation loc, StringRef text,
-                   StringRef replacementText);
-  void increaseIndentation(SourceRange range,
-                           SourceLocation parentIndent);
-
-  bool clearDiagnostic(ArrayRef<unsigned> IDs, SourceRange range);
-  bool clearAllDiagnostics(SourceRange range) {
-    return clearDiagnostic({}, range);
-  }
-  bool clearDiagnostic(unsigned ID1, unsigned ID2, SourceRange range) {
-    unsigned IDs[] = { ID1, ID2 };
-    return clearDiagnostic(IDs, range);
-  }
-  bool clearDiagnostic(unsigned ID1, unsigned ID2, unsigned ID3,
-                       SourceRange range) {
-    unsigned IDs[] = { ID1, ID2, ID3 };
-    return clearDiagnostic(IDs, range);
-  }
-
-  bool hasDiagnostic(unsigned ID, SourceRange range) {
-    return CapturedDiags.hasDiagnostic(ID, range);
-  }
-
-  bool hasDiagnostic(unsigned ID1, unsigned ID2, SourceRange range) {
-    unsigned IDs[] = { ID1, ID2 };
-    return CapturedDiags.hasDiagnostic(IDs, range);
-  }
-
-  DiagnosticBuilder report(SourceLocation loc, unsigned diagId,
-                           SourceRange range = SourceRange());
-  void reportError(StringRef error, SourceLocation loc,
-                   SourceRange range = SourceRange());
-  void reportWarning(StringRef warning, SourceLocation loc,
-                   SourceRange range = SourceRange());
-  void reportNote(StringRef note, SourceLocation loc,
-                  SourceRange range = SourceRange());
-
-  bool hasReportedErrors() const {
-    return Diags.hasUnrecoverableErrorOccurred();
-  }
-
-  class RewriteReceiver {
-  public:
-    virtual ~RewriteReceiver();
-
-    virtual void insert(SourceLocation loc, StringRef text) = 0;
-    virtual void remove(CharSourceRange range) = 0;
-    virtual void increaseIndentation(CharSourceRange range,
-                                     SourceLocation parentIndent) = 0;
-  };
-
-  void applyRewrites(RewriteReceiver &receiver);
-};
-
-class Transaction {
-  TransformActions &TA;
-  bool Aborted;
-
-public:
-  Transaction(TransformActions &TA) : TA(TA), Aborted(false) {
-    TA.startTransaction();
-  }
-
-  ~Transaction() {
-    if (!isAborted())
-      TA.commitTransaction();
-  }
-
-  void abort() {
-    TA.abortTransaction();
-    Aborted = true;
-  }
-
-  bool isAborted() const { return Aborted; }
-};
-
-class MigrationPass {
-public:
-  ASTContext &Ctx;
-  LangOptions::GCMode OrigGCMode;
-  MigratorOptions MigOptions;
-  Sema &SemaRef;
-  TransformActions &TA;
-  const CapturedDiagList &CapturedDiags;
-  std::vector<SourceLocation> &ARCMTMacroLocs;
-  std::optional<bool> EnableCFBridgeFns;
-
-  MigrationPass(ASTContext &Ctx, LangOptions::GCMode OrigGCMode, Sema &sema,
-                TransformActions &TA, const CapturedDiagList &capturedDiags,
-                std::vector<SourceLocation> &ARCMTMacroLocs)
-      : Ctx(Ctx), OrigGCMode(OrigGCMode), SemaRef(sema), TA(TA),
-        CapturedDiags(capturedDiags), ARCMTMacroLocs(ARCMTMacroLocs) {}
-
-  const CapturedDiagList &getDiags() const { return CapturedDiags; }
-
-  bool isGCMigration() const { return OrigGCMode != LangOptions::NonGC; }
-  bool noFinalizeRemoval() const { return MigOptions.NoFinalizeRemoval; }
-  void setNoFinalizeRemoval(bool val) {MigOptions.NoFinalizeRemoval = val; }
-
-  bool CFBridgingFunctionsDefined();
-};
-
-static inline StringRef getARCMTMacroName() {
-  return "__IMPL_ARCMT_REMOVED_EXPR__";
-}
-
-} // end namespace arcmt
-
-} // end namespace clang
-
-#endif
diff --git a/clang/lib/ARCMigrate/ObjCMT.cpp b/clang/lib/ARCMigrate/ObjCMT.cpp
deleted file mode 100644
index c1bc7c762088f..0000000000000
--- a/clang/lib/ARCMigrate/ObjCMT.cpp
+++ /dev/null
@@ -1,2262 +0,0 @@
-//===--- ObjCMT.cpp - ObjC Migrate Tool -----------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "Transforms.h"
-#include "clang/Analysis/RetainSummaryManager.h"
-#include "clang/ARCMigrate/ARCMT.h"
-#include "clang/ARCMigrate/ARCMTActions.h"
-#include "clang/AST/ASTConsumer.h"
-#include "clang/AST/ASTContext.h"
-#include "clang/AST/Attr.h"
-#include "clang/AST/NSAPI.h"
-#include "clang/AST/ParentMap.h"
-#include "clang/AST/RecursiveASTVisitor.h"
-#include "clang/Analysis/DomainSpecific/CocoaConventions.h"
-#include "clang/Basic/FileManager.h"
-#include "clang/Edit/Commit.h"
-#include "clang/Edit/EditedSource.h"
-#include "clang/Edit/EditsReceiver.h"
-#include "clang/Edit/Rewriters.h"
-#include "clang/Frontend/CompilerInstance.h"
-#include "clang/Frontend/MultiplexConsumer.h"
-#include "clang/Lex/PPConditionalDirectiveRecord.h"
-#include "clang/Lex/Preprocessor.h"
-#include "clang/Rewrite/Core/Rewriter.h"
-#include "llvm/ADT/SmallString.h"
-#include "llvm/ADT/StringSet.h"
-#include "llvm/Support/Path.h"
-#include "llvm/Support/SourceMgr.h"
-#include "llvm/Support/YAMLParser.h"
-
-using namespace clang;
-using namespace arcmt;
-using namespace ento;
-using llvm::RewriteBuffer;
-
-namespace {
-
-class ObjCMigrateASTConsumer : public ASTConsumer {
-  enum CF_BRIDGING_KIND {
-    CF_BRIDGING_NONE,
-    CF_BRIDGING_ENABLE,
-    CF_BRIDGING_MAY_INCLUDE
-  };
-
-  void migrateDecl(Decl *D);
-  void migrateObjCContainerDecl(ASTContext &Ctx, ObjCContainerDecl *D);
-  void migrateProtocolConformance(ASTContext &Ctx,
-                                  const ObjCImplementationDecl *ImpDecl);
-  void CacheObjCNSIntegerTypedefed(const TypedefDecl *TypedefDcl);
-  bool migrateNSEnumDecl(ASTContext &Ctx, const EnumDecl *EnumDcl,
-                     const TypedefDecl *TypedefDcl);
-  void migrateAllMethodInstaceType(ASTContext &Ctx, ObjCContainerDecl *CDecl);
-  void migrateMethodInstanceType(ASTContext &Ctx, ObjCContainerDecl *CDecl,
-                                 ObjCMethodDecl *OM);
-  bool migrateProperty(ASTContext &Ctx, ObjCContainerDecl *D, ObjCMethodDecl *OM);
-  void migrateNsReturnsInnerPointer(ASTContext &Ctx, ObjCMethodDecl *OM);
-  void migratePropertyNsReturnsInnerPointer(ASTContext &Ctx, ObjCPropertyDecl *P);
-  void migrateFactoryMethod(ASTContext &Ctx, ObjCContainerDecl *CDecl,
-                            ObjCMethodDecl *OM,
-                            ObjCInstanceTypeFamily OIT_Family = OIT_None);
-
-  void migrateCFAnnotation(ASTContext &Ctx, const Decl *Decl);
-  void AddCFAnnotations(ASTContext &Ctx,
-                        const RetainSummary *RS,
-                        const FunctionDecl *FuncDecl, bool ResultAnnotated);
-  void AddCFAnnotations(ASTContext &Ctx,
-                        const RetainSummary *RS,
-                        const ObjCMethodDecl *MethodDecl, bool ResultAnnotated);
-
-  void AnnotateImplicitBridging(ASTContext &Ctx);
-
-  CF_BRIDGING_KIND migrateAddFunctionAnnotation(ASTContext &Ctx,
-                                                const FunctionDecl *FuncDecl);
-
-  void migrateARCSafeAnnotation(ASTContext &Ctx, ObjCContainerDecl *CDecl);
-
-  void migrateAddMethodAnnotation(ASTContext &Ctx,
-                                  const ObjCMethodDecl *MethodDecl);
-
-  void inferDesignatedInitializers(ASTContext &Ctx,
-                                   const ObjCImplementationDecl *ImplD);
-
-  bool InsertFoundation(ASTContext &Ctx, SourceLocation Loc);
-
-  std::unique_ptr<RetainSummaryManager> Summaries;
-
-public:
-  std::string MigrateDir;
-  unsigned ASTMigrateActions;
-  FileID FileId;
-  const TypedefDecl *NSIntegerTypedefed;
-  const TypedefDecl *NSUIntegerTypedefed;
-  std::unique_ptr<NSAPI> NSAPIObj;
-  std::unique_ptr<edit::EditedSource> Editor;
-  FileRemapper &Remapper;
-  FileManager &FileMgr;
-  const PPConditionalDirectiveRecord *PPRec;
-  Preprocessor &PP;
-  bool IsOutputFile;
-  bool FoundationIncluded;
-  llvm::SmallPtrSet<ObjCProtocolDecl *, 32> ObjCProtocolDecls;
-  llvm::SmallVector<const Decl *, 8> CFFunctionIBCandidates;
-  llvm::StringSet<> AllowListFilenames;
-
-  RetainSummaryManager &getSummaryManager(ASTContext &Ctx) {
-    if (!Summaries)
-      Summaries.reset(new RetainSummaryManager(Ctx,
-                                               /*TrackNSCFObjects=*/true,
-                                               /*trackOSObjects=*/false));
-    return *Summaries;
-  }
-
-  ObjCMigrateASTConsumer(StringRef migrateDir, unsigned astMigrateActions,
-                         FileRemapper &remapper, FileManager &fileMgr,
-                         const PPConditionalDirectiveRecord *PPRec,
-                         Preprocessor &PP, bool isOutputFile,
-                         ArrayRef<std::string> AllowList)
-      : MigrateDir(migrateDir), ASTMigrateActions(astMigrateActions),
-        NSIntegerTypedefed(nullptr), NSUIntegerTypedefed(nullptr),
-        Remapper(remapper), FileMgr(fileMgr), PPRec(PPRec), PP(PP),
-        IsOutputFile(isOutputFile), FoundationIncluded(false) {
-    AllowListFilenames.insert(AllowList.begin(), AllowList.end());
-  }
-
-protected:
-  void Initialize(ASTContext &Context) override {
-    NSAPIObj.reset(new NSAPI(Context));
-    Editor.reset(new edit::EditedSource(Context.getSourceManager(),
-                                        Context.getLangOpts(),
-                                        PPRec));
-  }
-
-  bool HandleTopLevelDecl(DeclGroupRef DG) override {
-    for (DeclGroupRef::iterator I = DG.begin(), E = DG.end(); I != E; ++I)
-      migrateDecl(*I);
-    return true;
-  }
-  void HandleInterestingDecl(DeclGroupRef DG) override {
-    // Ignore decls from the PCH.
-  }
-  void HandleTopLevelDeclInObjCContainer(DeclGroupRef DG) override {
-    ObjCMigrateASTConsumer::HandleTopLevelDecl(DG);
-  }
-
-  void HandleTranslationUnit(ASTContext &Ctx) override;
-
-  bool canModifyFile(StringRef Path) {
-    if (AllowListFilenames.empty())
-      return true;
-    return AllowListFilenames.contains(llvm::sys::path::filename(Path));
-  }
-  bool canModifyFile(OptionalFileEntryRef FE) {
-    if (!FE)
-      return false;
-    return canModifyFile(FE->getName());
-  }
-  bool canModifyFile(FileID FID) {
-    if (FID.isInvalid())
-      return false;
-    return canModifyFile(PP.getSourceManager().getFileEntryRefForID(FID));
-  }
-
-  bool canModify(const Decl *D) {
-    if (!D)
-      return false;
-    if (const ObjCCategoryImplDecl *CatImpl = dyn_cast<ObjCCategoryImplDecl>(D))
-      return canModify(CatImpl->getCategoryDecl());
-    if (const ObjCImplementationDecl *Impl = dyn_cast<ObjCImplementationDecl>(D))
-      return canModify(Impl->getClassInterface());
-    if (const ObjCMethodDecl *MD = dyn_cast<ObjCMethodDecl>(D))
-      return canModify(cast<Decl>(MD->getDeclContext()));
-
-    FileID FID = PP.getSourceManager().getFileID(D->getLocation());
-    return canModifyFile(FID);
-  }
-};
-
-} // end anonymous namespace
-
-ObjCMigrateAction::ObjCMigrateAction(
-    std::unique_ptr<FrontendAction> WrappedAction, StringRef migrateDir,
-    unsigned migrateAction)
-    : WrapperFrontendAction(std::move(WrappedAction)), MigrateDir(migrateDir),
-      ObjCMigAction(migrateAction), CompInst(nullptr) {
-  if (MigrateDir.empty())
-    MigrateDir = "."; // user current directory if none is given.
-}
-
-std::unique_ptr<ASTConsumer>
-ObjCMigrateAction::CreateASTConsumer(CompilerInstance &CI, StringRef InFile) {
-  PPConditionalDirectiveRecord *
-    PPRec = new PPConditionalDirectiveRecord(CompInst->getSourceManager());
-  CI.getPreprocessor().addPPCallbacks(std::unique_ptr<PPCallbacks>(PPRec));
-  std::vector<std::unique_ptr<ASTConsumer>> Consumers;
-  Consumers.push_back(WrapperFrontendAction::CreateASTConsumer(CI, InFile));
-  Consumers.push_back(std::make_unique<ObjCMigrateASTConsumer>(
-      MigrateDir, ObjCMigAction, Remapper, CompInst->getFileManager(), PPRec,
-      CompInst->getPreprocessor(), false, std::nullopt));
-  return std::make_unique<MultiplexConsumer>(std::move(Consumers));
-}
-
-bool ObjCMigrateAction::BeginInvocation(CompilerInstance &CI) {
-  Remapper.initFromDisk(MigrateDir, CI.getDiagnostics(),
-                        /*ignoreIfFilesChanged=*/true);
-  CompInst = &CI;
-  CI.getDiagnostics().setIgnoreAllWarnings(true);
-  return true;
-}
-
-namespace {
-  // FIXME. This duplicates one in RewriteObjCFoundationAPI.cpp
-  bool subscriptOperatorNeedsParens(const Expr *FullExpr) {
-    const Expr* Expr = FullExpr->IgnoreImpCasts();
-    return !(isa<ArraySubscriptExpr>(Expr) || isa<CallExpr>(Expr) ||
-             isa<DeclRefExpr>(Expr) || isa<CXXNamedCastExpr>(Expr) ||
-             isa<CXXConstructExpr>(Expr) || isa<CXXThisExpr>(Expr) ||
-             isa<CXXTypeidExpr>(Expr) ||
-             isa<CXXUnresolvedConstructExpr>(Expr) ||
-             isa<ObjCMessageExpr>(Expr) || isa<ObjCPropertyRefExpr>(Expr) ||
-             isa<ObjCProtocolExpr>(Expr) || isa<MemberExpr>(Expr) ||
-             isa<ObjCIvarRefExpr>(Expr) || isa<ParenExpr>(FullExpr) ||
-             isa<ParenListExpr>(Expr) || isa<SizeOfPackExpr>(Expr));
-  }
-
-  /// - Rewrite message expression for Objective-C setter and getters into
-  /// property-dot syntax.
-  bool rewriteToPropertyDotSyntax(const ObjCMessageExpr *Msg,
-                                  Preprocessor &PP,
-                                  const NSAPI &NS, edit::Commit &commit,
-                                  const ParentMap *PMap) {
-    if (!Msg || Msg->isImplicit() ||
-        (Msg->getReceiverKind() != ObjCMessageExpr::Instance &&
-         Msg->getReceiverKind() != ObjCMessageExpr::SuperInstance))
-      return false;
-    if (const Expr *Receiver = Msg->getInstanceReceiver())
-      if (Receiver->getType()->isObjCBuiltinType())
-        return false;
-
-    const ObjCMethodDecl *Method = Msg->getMethodDecl();
-    if (!Method)
-      return false;
-    if (!Method->isPropertyAccessor())
-      return false;
-
-    const ObjCPropertyDecl *Prop = Method->findPropertyDecl();
-    if (!Prop)
-      return false;
-
-    SourceRange MsgRange = Msg->getSourceRange();
-    bool ReceiverIsSuper =
-      (Msg->getReceiverKind() == ObjCMessageExpr::SuperInstance);
-    // for 'super' receiver is nullptr.
-    const Expr *receiver = Msg->getInstanceReceiver();
-    bool NeedsParen =
-      ReceiverIsSuper ? false : subscriptOperatorNeedsParens(receiver);
-    bool IsGetter = (Msg->getNumArgs() == 0);
-    if (IsGetter) {
-      // Find space location range between receiver expression and getter method.
-      SourceLocation BegLoc =
-          ReceiverIsSuper ? Msg->getSuperLoc() : receiver->getEndLoc();
-      BegLoc = PP.getLocForEndOfToken(BegLoc);
-      SourceLocation EndLoc = Msg->getSelectorLoc(0);
-      SourceRange SpaceRange(BegLoc, EndLoc);
-      std::string PropertyDotString;
-      // rewrite getter method expression into: receiver.property or
-      // (receiver).property
-      if (NeedsParen) {
-        commit.insertBefore(receiver->getBeginLoc(), "(");
-        PropertyDotString = ").";
-      }
-      else
-        PropertyDotString = ".";
-      PropertyDotString += Prop->getName();
-      commit.replace(SpaceRange, PropertyDotString);
-
-      // remove '[' ']'
-      commit.replace(SourceRange(MsgRange.getBegin(), MsgRange.getBegin()), "");
-      commit.replace(SourceRange(MsgRange.getEnd(), MsgRange.getEnd()), "");
-    } else {
-      if (NeedsParen)
-        commit.insertWrap("(", receiver->getSourceRange(), ")");
-      std::string PropertyDotString = ".";
-      PropertyDotString += Prop->getName();
-      PropertyDotString += " =";
-      const Expr*const* Args = Msg->getArgs();
-      const Expr *RHS = Args[0];
-      if (!RHS)
-        return false;
-      SourceLocation BegLoc =
-          ReceiverIsSuper ? Msg->getSuperLoc() : receiver->getEndLoc();
-      BegLoc = PP.getLocForEndOfToken(BegLoc);
-      SourceLocation EndLoc = RHS->getBeginLoc();
-      EndLoc = EndLoc.getLocWithOffset(-1);
-      const char *colon = PP.getSourceManager().getCharacterData(EndLoc);
-      // Add a space after '=' if there is no space between RHS and '='
-      if (colon && colon[0] == ':')
-        PropertyDotString += " ";
-      SourceRange Range(BegLoc, EndLoc);
-      commit.replace(Range, PropertyDotString);
-      // remove '[' ']'
-      commit.replace(SourceRange(MsgRange.getBegin(), MsgRange.getBegin()), "");
-      commit.replace(SourceRange(MsgRange.getEnd(), MsgRange.getEnd()), "");
-    }
-    return true;
-  }
-
-class ObjCMigrator : public RecursiveASTVisitor<ObjCMigrator> {
-  ObjCMigrateASTConsumer &Consumer;
-  ParentMap &PMap;
-
-public:
-  ObjCMigrator(ObjCMigrateASTConsumer &consumer, ParentMap &PMap)
-    : Consumer(consumer), PMap(PMap) { }
-
-  bool shouldVisitTemplateInstantiations() const { return false; }
-  bool shouldWalkTypesOfTypeLocs() const { return false; }
-
-  bool VisitObjCMessageExpr(ObjCMessageExpr *E) {
-    if (Consumer.ASTMigrateActions & FrontendOptions::ObjCMT_Literals) {
-      edit::Commit commit(*Consumer.Editor);
-      edit::rewriteToObjCLiteralSyntax(E, *Consumer.NSAPIObj, commit, &PMap);
-      Consumer.Editor->commit(commit);
-    }
-
-    if (Consumer.ASTMigrateActions & FrontendOptions::ObjCMT_Subscripting) {
-      edit::Commit commit(*Consumer.Editor);
-      edit::rewriteToObjCSubscriptSyntax(E, *Consumer.NSAPIObj, commit);
-      Consumer.Editor->commit(commit);
-    }
-
-    if (Consumer.ASTMigrateActions & FrontendOptions::ObjCMT_PropertyDotSyntax) {
-      edit::Commit commit(*Consumer.Editor);
-      rewriteToPropertyDotSyntax(E, Consumer.PP, *Consumer.NSAPIObj,
-                                 commit, &PMap);
-      Consumer.Editor->commit(commit);
-    }
-
-    return true;
-  }
-
-  bool TraverseObjCMessageExpr(ObjCMessageExpr *E) {
-    // Do depth first; we want to rewrite the subexpressions first so that if
-    // we have to move expressions we will move them already rewritten.
-    for (Stmt *SubStmt : E->children())
-      if (!TraverseStmt(SubStmt))
-        return false;
-
-    return WalkUpFromObjCMessageExpr(E);
-  }
-};
-
-class BodyMigrator : public RecursiveASTVisitor<BodyMigrator> {
-  ObjCMigrateASTConsumer &Consumer;
-  std::unique_ptr<ParentMap> PMap;
-
-public:
-  BodyMigrator(ObjCMigrateASTConsumer &consumer) : Consumer(consumer) { }
-
-  bool shouldVisitTemplateInstantiations() const { return false; }
-  bool shouldWalkTypesOfTypeLocs() const { return false; }
-
-  bool TraverseStmt(Stmt *S) {
-    PMap.reset(new ParentMap(S));
-    ObjCMigrator(Consumer, *PMap).TraverseStmt(S);
-    return true;
-  }
-};
-} // end anonymous namespace
-
-void ObjCMigrateASTConsumer::migrateDecl(Decl *D) {
-  if (!D)
-    return;
-  if (isa<ObjCMethodDecl>(D))
-    return; // Wait for the ObjC container declaration.
-
-  BodyMigrator(*this).TraverseDecl(D);
-}
-
-static void append_attr(std::string &PropertyString, const char *attr,
-                        bool &LParenAdded) {
-  if (!LParenAdded) {
-    PropertyString += "(";
-    LParenAdded = true;
-  }
-  else
-    PropertyString += ", ";
-  PropertyString += attr;
-}
-
-static
-void MigrateBlockOrFunctionPointerTypeVariable(std::string & PropertyString,
-                                               const std::string& TypeString,
-                                               const char *name) {
-  const char *argPtr = TypeString.c_str();
-  int paren = 0;
-  while (*argPtr) {
-    switch (*argPtr) {
-      case '(':
-        PropertyString += *argPtr;
-        paren++;
-        break;
-      case ')':
-        PropertyString += *argPtr;
-        paren--;
-        break;
-      case '^':
-      case '*':
-        PropertyString += (*argPtr);
-        if (paren == 1) {
-          PropertyString += name;
-          name = "";
-        }
-        break;
-      default:
-        PropertyString += *argPtr;
-        break;
-    }
-    argPtr++;
-  }
-}
-
-static const char *PropertyMemoryAttribute(ASTContext &Context, QualType ArgType) {
-  Qualifiers::ObjCLifetime propertyLifetime = ArgType.getObjCLifetime();
-  bool RetainableObject = ArgType->isObjCRetainableType();
-  if (RetainableObject &&
-      (propertyLifetime == Qualifiers::OCL_Strong
-       || propertyLifetime == Qualifiers::OCL_None)) {
-    if (const ObjCObjectPointerType *ObjPtrTy =
-        ArgType->getAs<ObjCObjectPointerType>()) {
-      ObjCInterfaceDecl *IDecl = ObjPtrTy->getObjectType()->getInterface();
-      if (IDecl &&
-          IDecl->lookupNestedProtocol(&Context.Idents.get("NSCopying")))
-        return "copy";
-      else
-        return "strong";
-    }
-    else if (ArgType->isBlockPointerType())
-      return "copy";
-  } else if (propertyLifetime == Qualifiers::OCL_Weak)
-    // TODO. More precise determination of 'weak' attribute requires
-    // looking into setter's implementation for backing weak ivar.
-    return "weak";
-  else if (RetainableObject)
-    return ArgType->isBlockPointerType() ? "copy" : "strong";
-  return nullptr;
-}
-
-static void rewriteToObjCProperty(const ObjCMethodDecl *Getter,
-                                  const ObjCMethodDecl *Setter,
-                                  const NSAPI &NS, edit::Commit &commit,
-                                  unsigned LengthOfPrefix,
-                                  bool Atomic, bool UseNsIosOnlyMacro,
-                                  bool AvailabilityArgsMatch) {
-  ASTContext &Context = NS.getASTContext();
-  bool LParenAdded = false;
-  std::string PropertyString = "@property ";
-  if (UseNsIosOnlyMacro && NS.isMacroDefined("NS_NONATOMIC_IOSONLY")) {
-    PropertyString += "(NS_NONATOMIC_IOSONLY";
-    LParenAdded = true;
-  } else if (!Atomic) {
-    PropertyString += "(nonatomic";
-    LParenAdded = true;
-  }
-
-  std::string PropertyNameString = Getter->getNameAsString();
-  StringRef PropertyName(PropertyNameString);
-  if (LengthOfPrefix > 0) {
-    if (!LParenAdded) {
-      PropertyString += "(getter=";
-      LParenAdded = true;
-    }
-    else
-      PropertyString += ", getter=";
-    PropertyString += PropertyNameString;
-  }
-  // Property with no setter may be suggested as a 'readonly' property.
-  if (!Setter)
-    append_attr(PropertyString, "readonly", LParenAdded);
-
-
-  // Short circuit 'delegate' properties that contain the name "delegate" or
-  // "dataSource", or have exact name "target" to have 'assign' attribute.
-  if (PropertyName == "target" || PropertyName.contains("delegate") ||
-      PropertyName.contains("dataSource")) {
-    QualType QT = Getter->getReturnType();
-    if (!QT->isRealType())
-      append_attr(PropertyString, "assign", LParenAdded);
-  } else if (!Setter) {
-    QualType ResType = Context.getCanonicalType(Getter->getReturnType());
-    if (const char *MemoryManagementAttr = PropertyMemoryAttribute(Context, ResType))
-      append_attr(PropertyString, MemoryManagementAttr, LParenAdded);
-  } else {
-    const ParmVarDecl *argDecl = *Setter->param_begin();
-    QualType ArgType = Context.getCanonicalType(argDecl->getType());
-    if (const char *MemoryManagementAttr = PropertyMemoryAttribute(Context, ArgType))
-      append_attr(PropertyString, MemoryManagementAttr, LParenAdded);
-  }
-  if (LParenAdded)
-    PropertyString += ')';
-  QualType RT = Getter->getReturnType();
-  if (!RT->getAs<TypedefType>()) {
-    // strip off any ARC lifetime qualifier.
-    QualType CanResultTy = Context.getCanonicalType(RT);
-    if (CanResultTy.getQualifiers().hasObjCLifetime()) {
-      Qualifiers Qs = CanResultTy.getQualifiers();
-      Qs.removeObjCLifetime();
-      RT = Context.getQualifiedType(CanResultTy.getUnqualifiedType(), Qs);
-    }
-  }
-  PropertyString += " ";
-  PrintingPolicy SubPolicy(Context.getPrintingPolicy());
-  SubPolicy.SuppressStrongLifetime = true;
-  SubPolicy.SuppressLifetimeQualifiers = true;
-  std::string TypeString = RT.getAsString(SubPolicy);
-  if (LengthOfPrefix > 0) {
-    // property name must strip off "is" and lower case the first character
-    // after that; e.g. isContinuous will become continuous.
-    StringRef PropertyNameStringRef(PropertyNameString);
-    PropertyNameStringRef = PropertyNameStringRef.drop_front(LengthOfPrefix);
-    PropertyNameString = std::string(PropertyNameStringRef);
-    bool NoLowering = (isUppercase(PropertyNameString[0]) &&
-                       PropertyNameString.size() > 1 &&
-                       isUppercase(PropertyNameString[1]));
-    if (!NoLowering)
-      PropertyNameString[0] = toLowercase(PropertyNameString[0]);
-  }
-  if (RT->isBlockPointerType() || RT->isFunctionPointerType())
-    MigrateBlockOrFunctionPointerTypeVariable(PropertyString,
-                                              TypeString,
-                                              PropertyNameString.c_str());
-  else {
-    char LastChar = TypeString[TypeString.size()-1];
-    PropertyString += TypeString;
-    if (LastChar != '*')
-      PropertyString += ' ';
-    PropertyString += PropertyNameString;
-  }
-  SourceLocation StartGetterSelectorLoc = Getter->getSelectorStartLoc();
-  Selector GetterSelector = Getter->getSelector();
-
-  SourceLocation EndGetterSelectorLoc =
-    StartGetterSelectorLoc.getLocWithOffset(GetterSelector.getNameForSlot(0).size());
-  commit.replace(CharSourceRange::getCharRange(Getter->getBeginLoc(),
-                                               EndGetterSelectorLoc),
-                 PropertyString);
-  if (Setter && AvailabilityArgsMatch) {
-    SourceLocation EndLoc = Setter->getDeclaratorEndLoc();
-    // Get location past ';'
-    EndLoc = EndLoc.getLocWithOffset(1);
-    SourceLocation BeginOfSetterDclLoc = Setter->getBeginLoc();
-    // FIXME. This assumes that setter decl; is immediately preceded by eoln.
-    // It is trying to remove the setter method decl. line entirely.
-    BeginOfSetterDclLoc = BeginOfSetterDclLoc.getLocWithOffset(-1);
-    commit.remove(SourceRange(BeginOfSetterDclLoc, EndLoc));
-  }
-}
-
-static bool IsCategoryNameWithDeprecatedSuffix(ObjCContainerDecl *D) {
-  if (ObjCCategoryDecl *CatDecl = dyn_cast<ObjCCategoryDecl>(D)) {
-    StringRef Name = CatDecl->getName();
-    return Name.ends_with("Deprecated");
-  }
-  return false;
-}
-
-void ObjCMigrateASTConsumer::migrateObjCContainerDecl(ASTContext &Ctx,
-                                                      ObjCContainerDecl *D) {
-  if (D->isDeprecated() || IsCategoryNameWithDeprecatedSuffix(D))
-    return;
-
-  for (auto *Method : D->methods()) {
-    if (Method->isDeprecated())
-      continue;
-    bool PropertyInferred = migrateProperty(Ctx, D, Method);
-    // If a property is inferred, do not attempt to attach NS_RETURNS_INNER_POINTER to
-    // the getter method as it ends up on the property itself which we don't want
-    // to do unless -objcmt-returns-innerpointer-property  option is on.
-    if (!PropertyInferred ||
-        (ASTMigrateActions & FrontendOptions::ObjCMT_ReturnsInnerPointerProperty))
-      if (ASTMigrateActions & FrontendOptions::ObjCMT_Annotation)
-        migrateNsReturnsInnerPointer(Ctx, Method);
-  }
-  if (!(ASTMigrateActions & FrontendOptions::ObjCMT_ReturnsInnerPointerProperty))
-    return;
-
-  for (auto *Prop : D->instance_properties()) {
-    if ((ASTMigrateActions & FrontendOptions::ObjCMT_Annotation) &&
-        !Prop->isDeprecated())
-      migratePropertyNsReturnsInnerPointer(Ctx, Prop);
-  }
-}
-
-static bool
-ClassImplementsAllMethodsAndProperties(ASTContext &Ctx,
-                                      const ObjCImplementationDecl *ImpDecl,
-                                       const ObjCInterfaceDecl *IDecl,
-                                      ObjCProtocolDecl *Protocol) {
-  // In auto-synthesis, protocol properties are not synthesized. So,
-  // a conforming protocol must have its required properties declared
-  // in class interface.
-  bool HasAtleastOneRequiredProperty = false;
-  if (const ObjCProtocolDecl *PDecl = Protocol->getDefinition())
-    for (const auto *Property : PDecl->instance_properties()) {
-      if (Property->getPropertyImplementation() == ObjCPropertyDecl::Optional)
-        continue;
-      HasAtleastOneRequiredProperty = true;
-      DeclContext::lookup_result R = IDecl->lookup(Property->getDeclName());
-      if (R.empty()) {
-        // Relax the rule and look into class's implementation for a synthesize
-        // or dynamic declaration. Class is implementing a property coming from
-        // another protocol. This still makes the target protocol as conforming.
-        if (!ImpDecl->FindPropertyImplDecl(
-                                  Property->getDeclName().getAsIdentifierInfo(),
-                                  Property->getQueryKind()))
-          return false;
-      } else if (auto *ClassProperty = R.find_first<ObjCPropertyDecl>()) {
-        if ((ClassProperty->getPropertyAttributes() !=
-             Property->getPropertyAttributes()) ||
-            !Ctx.hasSameType(ClassProperty->getType(), Property->getType()))
-          return false;
-      } else
-        return false;
-    }
-
-  // At this point, all required properties in this protocol conform to those
-  // declared in the class.
-  // Check that class implements the required methods of the protocol too.
-  bool HasAtleastOneRequiredMethod = false;
-  if (const ObjCProtocolDecl *PDecl = Protocol->getDefinition()) {
-    if (PDecl->meth_begin() == PDecl->meth_end())
-      return HasAtleastOneRequiredProperty;
-    for (const auto *MD : PDecl->methods()) {
-      if (MD->isImplicit())
-        continue;
-      if (MD->getImplementationControl() == ObjCImplementationControl::Optional)
-        continue;
-      DeclContext::lookup_result R = ImpDecl->lookup(MD->getDeclName());
-      if (R.empty())
-        return false;
-      bool match = false;
-      HasAtleastOneRequiredMethod = true;
-      for (NamedDecl *ND : R)
-        if (ObjCMethodDecl *ImpMD = dyn_cast<ObjCMethodDecl>(ND))
-          if (Ctx.ObjCMethodsAreEqual(MD, ImpMD)) {
-            match = true;
-            break;
-          }
-      if (!match)
-        return false;
-    }
-  }
-  return HasAtleastOneRequiredProperty || HasAtleastOneRequiredMethod;
-}
-
-static bool rewriteToObjCInterfaceDecl(const ObjCInterfaceDecl *IDecl,
-                    llvm::SmallVectorImpl<ObjCProtocolDecl*> &ConformingProtocols,
-                    const NSAPI &NS, edit::Commit &commit) {
-  const ObjCList<ObjCProtocolDecl> &Protocols = IDecl->getReferencedProtocols();
-  std::string ClassString;
-  SourceLocation EndLoc =
-  IDecl->getSuperClass() ? IDecl->getSuperClassLoc() : IDecl->getLocation();
-
-  if (Protocols.empty()) {
-    ClassString = '<';
-    for (unsigned i = 0, e = ConformingProtocols.size(); i != e; i++) {
-      ClassString += ConformingProtocols[i]->getNameAsString();
-      if (i != (e-1))
-        ClassString += ", ";
-    }
-    ClassString += "> ";
-  }
-  else {
-    ClassString = ", ";
-    for (unsigned i = 0, e = ConformingProtocols.size(); i != e; i++) {
-      ClassString += ConformingProtocols[i]->getNameAsString();
-      if (i != (e-1))
-        ClassString += ", ";
-    }
-    ObjCInterfaceDecl::protocol_loc_iterator PL = IDecl->protocol_loc_end() - 1;
-    EndLoc = *PL;
-  }
-
-  commit.insertAfterToken(EndLoc, ClassString);
-  return true;
-}
-
-static StringRef GetUnsignedName(StringRef NSIntegerName) {
-  StringRef UnsignedName = llvm::StringSwitch<StringRef>(NSIntegerName)
-    .Case("int8_t", "uint8_t")
-    .Case("int16_t", "uint16_t")
-    .Case("int32_t", "uint32_t")
-    .Case("NSInteger", "NSUInteger")
-    .Case("int64_t", "uint64_t")
-    .Default(NSIntegerName);
-  return UnsignedName;
-}
-
-static bool rewriteToNSEnumDecl(const EnumDecl *EnumDcl,
-                                const TypedefDecl *TypedefDcl,
-                                const NSAPI &NS, edit::Commit &commit,
-                                StringRef NSIntegerName,
-                                bool NSOptions) {
-  std::string ClassString;
-  if (NSOptions) {
-    ClassString = "typedef NS_OPTIONS(";
-    ClassString += GetUnsignedName(NSIntegerName);
-  }
-  else {
-    ClassString = "typedef NS_ENUM(";
-    ClassString += NSIntegerName;
-  }
-  ClassString += ", ";
-
-  ClassString += TypedefDcl->getIdentifier()->getName();
-  ClassString += ')';
-  SourceRange R(EnumDcl->getBeginLoc(), EnumDcl->getBeginLoc());
-  commit.replace(R, ClassString);
-  SourceLocation EndOfEnumDclLoc = EnumDcl->getEndLoc();
-  EndOfEnumDclLoc = trans::findSemiAfterLocation(EndOfEnumDclLoc,
-                                                 NS.getASTContext(), /*IsDecl*/true);
-  if (EndOfEnumDclLoc.isValid()) {
-    SourceRange EnumDclRange(EnumDcl->getBeginLoc(), EndOfEnumDclLoc);
-    commit.insertFromRange(TypedefDcl->getBeginLoc(), EnumDclRange);
-  }
-  else
-    return false;
-
-  SourceLocation EndTypedefDclLoc = TypedefDcl->getEndLoc();
-  EndTypedefDclLoc = trans::findSemiAfterLocation(EndTypedefDclLoc,
-                                                 NS.getASTContext(), /*IsDecl*/true);
-  if (EndTypedefDclLoc.isValid()) {
-    SourceRange TDRange(TypedefDcl->getBeginLoc(), EndTypedefDclLoc);
-    commit.remove(TDRange);
-  }
-  else
-    return false;
-
-  EndOfEnumDclLoc =
-      trans::findLocationAfterSemi(EnumDcl->getEndLoc(), NS.getASTContext(),
-                                   /*IsDecl*/ true);
-  if (EndOfEnumDclLoc.isValid()) {
-    SourceLocation BeginOfEnumDclLoc = EnumDcl->getBeginLoc();
-    // FIXME. This assumes that enum decl; is immediately preceded by eoln.
-    // It is trying to remove the enum decl. lines entirely.
-    BeginOfEnumDclLoc = BeginOfEnumDclLoc.getLocWithOffset(-1);
-    commit.remove(SourceRange(BeginOfEnumDclLoc, EndOfEnumDclLoc));
-    return true;
-  }
-  return false;
-}
-
-static void rewriteToNSMacroDecl(ASTContext &Ctx,
-                                 const EnumDecl *EnumDcl,
-                                const TypedefDecl *TypedefDcl,
-                                const NSAPI &NS, edit::Commit &commit,
-                                 bool IsNSIntegerType) {
-  QualType DesignatedEnumType = EnumDcl->getIntegerType();
-  assert(!DesignatedEnumType.isNull()
-         && "rewriteToNSMacroDecl - underlying enum type is null");
-
-  PrintingPolicy Policy(Ctx.getPrintingPolicy());
-  std::string TypeString = DesignatedEnumType.getAsString(Policy);
-  std::string ClassString = IsNSIntegerType ? "NS_ENUM(" : "NS_OPTIONS(";
-  ClassString += TypeString;
-  ClassString += ", ";
-
-  ClassString += TypedefDcl->getIdentifier()->getName();
-  ClassString += ") ";
-  SourceLocation EndLoc = EnumDcl->getBraceRange().getBegin();
-  if (EndLoc.isInvalid())
-    return;
-  CharSourceRange R =
-      CharSourceRange::getCharRange(EnumDcl->getBeginLoc(), EndLoc);
-  commit.replace(R, ClassString);
-  // This is to remove spaces between '}' and typedef name.
-  SourceLocation StartTypedefLoc = EnumDcl->getEndLoc();
-  StartTypedefLoc = StartTypedefLoc.getLocWithOffset(+1);
-  SourceLocation EndTypedefLoc = TypedefDcl->getEndLoc();
-
-  commit.remove(SourceRange(StartTypedefLoc, EndTypedefLoc));
-}
-
-static bool UseNSOptionsMacro(Preprocessor &PP, ASTContext &Ctx,
-                              const EnumDecl *EnumDcl) {
-  bool PowerOfTwo = true;
-  bool AllHexdecimalEnumerator = true;
-  uint64_t MaxPowerOfTwoVal = 0;
-  for (auto *Enumerator : EnumDcl->enumerators()) {
-    const Expr *InitExpr = Enumerator->getInitExpr();
-    if (!InitExpr) {
-      PowerOfTwo = false;
-      AllHexdecimalEnumerator = false;
-      continue;
-    }
-    InitExpr = InitExpr->IgnoreParenCasts();
-    if (const BinaryOperator *BO = dyn_cast<BinaryOperator>(InitExpr))
-      if (BO->isShiftOp() || BO->isBitwiseOp())
-        return true;
-
-    uint64_t EnumVal = Enumerator->getInitVal().getZExtValue();
-    if (PowerOfTwo && EnumVal) {
-      if (!llvm::isPowerOf2_64(EnumVal))
-        PowerOfTwo = false;
-      else if (EnumVal > MaxPowerOfTwoVal)
-        MaxPowerOfTwoVal = EnumVal;
-    }
-    if (AllHexdecimalEnumerator && EnumVal) {
-      bool FoundHexdecimalEnumerator = false;
-      SourceLocation EndLoc = Enumerator->getEndLoc();
-      Token Tok;
-      if (!PP.getRawToken(EndLoc, Tok, /*IgnoreWhiteSpace=*/true))
-        if (Tok.isLiteral() && Tok.getLength() > 2) {
-          if (const char *StringLit = Tok.getLiteralData())
-            FoundHexdecimalEnumerator =
-              (StringLit[0] == '0' && (toLowercase(StringLit[1]) == 'x'));
-        }
-      if (!FoundHexdecimalEnumerator)
-        AllHexdecimalEnumerator = false;
-    }
-  }
-  return AllHexdecimalEnumerator || (PowerOfTwo && (MaxPowerOfTwoVal > 2));
-}
-
-void ObjCMigrateASTConsumer::migrateProtocolConformance(ASTContext &Ctx,
-                                            const ObjCImplementationDecl *ImpDecl) {
-  const ObjCInterfaceDecl *IDecl = ImpDecl->getClassInterface();
-  if (!IDecl || ObjCProtocolDecls.empty() || IDecl->isDeprecated())
-    return;
-  // Find all implicit conforming protocols for this class
-  // and make them explicit.
-  llvm::SmallPtrSet<ObjCProtocolDecl *, 8> ExplicitProtocols;
-  Ctx.CollectInheritedProtocols(IDecl, ExplicitProtocols);
-  llvm::SmallVector<ObjCProtocolDecl *, 8> PotentialImplicitProtocols;
-
-  for (ObjCProtocolDecl *ProtDecl : ObjCProtocolDecls)
-    if (!ExplicitProtocols.count(ProtDecl))
-      PotentialImplicitProtocols.push_back(ProtDecl);
-
-  if (PotentialImplicitProtocols.empty())
-    return;
-
-  // go through list of non-optional methods and properties in each protocol
-  // in the PotentialImplicitProtocols list. If class implements every one of the
-  // methods and properties, then this class conforms to this protocol.
-  llvm::SmallVector<ObjCProtocolDecl*, 8> ConformingProtocols;
-  for (unsigned i = 0, e = PotentialImplicitProtocols.size(); i != e; i++)
-    if (ClassImplementsAllMethodsAndProperties(Ctx, ImpDecl, IDecl,
-                                              PotentialImplicitProtocols[i]))
-      ConformingProtocols.push_back(PotentialImplicitProtocols[i]);
-
-  if (ConformingProtocols.empty())
-    return;
-
-  // Further reduce number of conforming protocols. If protocol P1 is in the list
-  // protocol P2 (P2<P1>), No need to include P1.
-  llvm::SmallVector<ObjCProtocolDecl*, 8> MinimalConformingProtocols;
-  for (unsigned i = 0, e = ConformingProtocols.size(); i != e; i++) {
-    bool DropIt = false;
-    ObjCProtocolDecl *TargetPDecl = ConformingProtocols[i];
-    for (unsigned i1 = 0, e1 = ConformingProtocols.size(); i1 != e1; i1++) {
-      ObjCProtocolDecl *PDecl = ConformingProtocols[i1];
-      if (PDecl == TargetPDecl)
-        continue;
-      if (PDecl->lookupProtocolNamed(
-            TargetPDecl->getDeclName().getAsIdentifierInfo())) {
-        DropIt = true;
-        break;
-      }
-    }
-    if (!DropIt)
-      MinimalConformingProtocols.push_back(TargetPDecl);
-  }
-  if (MinimalConformingProtocols.empty())
-    return;
-  edit::Commit commit(*Editor);
-  rewriteToObjCInterfaceDecl(IDecl, MinimalConformingProtocols,
-                             *NSAPIObj, commit);
-  Editor->commit(commit);
-}
-
-void ObjCMigrateASTConsumer::CacheObjCNSIntegerTypedefed(
-                                          const TypedefDecl *TypedefDcl) {
-
-  QualType qt = TypedefDcl->getTypeSourceInfo()->getType();
-  if (NSAPIObj->isObjCNSIntegerType(qt))
-    NSIntegerTypedefed = TypedefDcl;
-  else if (NSAPIObj->isObjCNSUIntegerType(qt))
-    NSUIntegerTypedefed = TypedefDcl;
-}
-
-bool ObjCMigrateASTConsumer::migrateNSEnumDecl(ASTContext &Ctx,
-                                           const EnumDecl *EnumDcl,
-                                           const TypedefDecl *TypedefDcl) {
-  if (!EnumDcl->isCompleteDefinition() || EnumDcl->getIdentifier() ||
-      EnumDcl->isDeprecated())
-    return false;
-  if (!TypedefDcl) {
-    if (NSIntegerTypedefed) {
-      TypedefDcl = NSIntegerTypedefed;
-      NSIntegerTypedefed = nullptr;
-    }
-    else if (NSUIntegerTypedefed) {
-      TypedefDcl = NSUIntegerTypedefed;
-      NSUIntegerTypedefed = nullptr;
-    }
-    else
-      return false;
-    FileID FileIdOfTypedefDcl =
-      PP.getSourceManager().getFileID(TypedefDcl->getLocation());
-    FileID FileIdOfEnumDcl =
-      PP.getSourceManager().getFileID(EnumDcl->getLocation());
-    if (FileIdOfTypedefDcl != FileIdOfEnumDcl)
-      return false;
-  }
-  if (TypedefDcl->isDeprecated())
-    return false;
-
-  QualType qt = TypedefDcl->getTypeSourceInfo()->getType();
-  StringRef NSIntegerName = NSAPIObj->GetNSIntegralKind(qt);
-
-  if (NSIntegerName.empty()) {
-    // Also check for typedef enum {...} TD;
-    if (const EnumType *EnumTy = qt->getAs<EnumType>()) {
-      if (EnumTy->getDecl() == EnumDcl) {
-        bool NSOptions = UseNSOptionsMacro(PP, Ctx, EnumDcl);
-        if (!InsertFoundation(Ctx, TypedefDcl->getBeginLoc()))
-          return false;
-        edit::Commit commit(*Editor);
-        rewriteToNSMacroDecl(Ctx, EnumDcl, TypedefDcl, *NSAPIObj, commit, !NSOptions);
-        Editor->commit(commit);
-        return true;
-      }
-    }
-    return false;
-  }
-
-  // We may still use NS_OPTIONS based on what we find in the enumertor list.
-  bool NSOptions = UseNSOptionsMacro(PP, Ctx, EnumDcl);
-  if (!InsertFoundation(Ctx, TypedefDcl->getBeginLoc()))
-    return false;
-  edit::Commit commit(*Editor);
-  bool Res = rewriteToNSEnumDecl(EnumDcl, TypedefDcl, *NSAPIObj,
-                                 commit, NSIntegerName, NSOptions);
-  Editor->commit(commit);
-  return Res;
-}
-
-static void ReplaceWithInstancetype(ASTContext &Ctx,
-                                    const ObjCMigrateASTConsumer &ASTC,
-                                    ObjCMethodDecl *OM) {
-  if (OM->getReturnType() == Ctx.getObjCInstanceType())
-    return; // already has instancetype.
-
-  SourceRange R;
-  std::string ClassString;
-  if (TypeSourceInfo *TSInfo = OM->getReturnTypeSourceInfo()) {
-    TypeLoc TL = TSInfo->getTypeLoc();
-    R = SourceRange(TL.getBeginLoc(), TL.getEndLoc());
-    ClassString = "instancetype";
-  }
-  else {
-    R = SourceRange(OM->getBeginLoc(), OM->getBeginLoc());
-    ClassString = OM->isInstanceMethod() ? '-' : '+';
-    ClassString += " (instancetype)";
-  }
-  edit::Commit commit(*ASTC.Editor);
-  commit.replace(R, ClassString);
-  ASTC.Editor->commit(commit);
-}
-
-static void ReplaceWithClasstype(const ObjCMigrateASTConsumer &ASTC,
-                                    ObjCMethodDecl *OM) {
-  ObjCInterfaceDecl *IDecl = OM->getClassInterface();
-  SourceRange R;
-  std::string ClassString;
-  if (TypeSourceInfo *TSInfo = OM->getReturnTypeSourceInfo()) {
-    TypeLoc TL = TSInfo->getTypeLoc();
-    R = SourceRange(TL.getBeginLoc(), TL.getEndLoc()); {
-      ClassString = std::string(IDecl->getName());
-      ClassString += "*";
-    }
-  }
-  else {
-    R = SourceRange(OM->getBeginLoc(), OM->getBeginLoc());
-    ClassString = "+ (";
-    ClassString += IDecl->getName(); ClassString += "*)";
-  }
-  edit::Commit commit(*ASTC.Editor);
-  commit.replace(R, ClassString);
-  ASTC.Editor->commit(commit);
-}
-
-void ObjCMigrateASTConsumer::migrateMethodInstanceType(ASTContext &Ctx,
-                                                       ObjCContainerDecl *CDecl,
-                                                       ObjCMethodDecl *OM) {
-  ObjCInstanceTypeFamily OIT_Family =
-    Selector::getInstTypeMethodFamily(OM->getSelector());
-
-  std::string ClassName;
-  switch (OIT_Family) {
-    case OIT_None:
-      migrateFactoryMethod(Ctx, CDecl, OM);
-      return;
-    case OIT_Array:
-      ClassName = "NSArray";
-      break;
-    case OIT_Dictionary:
-      ClassName = "NSDictionary";
-      break;
-    case OIT_Singleton:
-      migrateFactoryMethod(Ctx, CDecl, OM, OIT_Singleton);
-      return;
-    case OIT_Init:
-      if (OM->getReturnType()->isObjCIdType())
-        ReplaceWithInstancetype(Ctx, *this, OM);
-      return;
-    case OIT_ReturnsSelf:
-      migrateFactoryMethod(Ctx, CDecl, OM, OIT_ReturnsSelf);
-      return;
-  }
-  if (!OM->getReturnType()->isObjCIdType())
-    return;
-
-  ObjCInterfaceDecl *IDecl = dyn_cast<ObjCInterfaceDecl>(CDecl);
-  if (!IDecl) {
-    if (ObjCCategoryDecl *CatDecl = dyn_cast<ObjCCategoryDecl>(CDecl))
-      IDecl = CatDecl->getClassInterface();
-    else if (ObjCImplDecl *ImpDecl = dyn_cast<ObjCImplDecl>(CDecl))
-      IDecl = ImpDecl->getClassInterface();
-  }
-  if (!IDecl ||
-      !IDecl->lookupInheritedClass(&Ctx.Idents.get(ClassName))) {
-    migrateFactoryMethod(Ctx, CDecl, OM);
-    return;
-  }
-  ReplaceWithInstancetype(Ctx, *this, OM);
-}
-
-static bool TypeIsInnerPointer(QualType T) {
-  if (!T->isAnyPointerType())
-    return false;
-  if (T->isObjCObjectPointerType() || T->isObjCBuiltinType() ||
-      T->isBlockPointerType() || T->isFunctionPointerType() ||
-      ento::coreFoundation::isCFObjectRef(T))
-    return false;
-  // Also, typedef-of-pointer-to-incomplete-struct is something that we assume
-  // is not an innter pointer type.
-  QualType OrigT = T;
-  while (const auto *TD = T->getAs<TypedefType>())
-    T = TD->getDecl()->getUnderlyingType();
-  if (OrigT == T || !T->isPointerType())
-    return true;
-  const PointerType* PT = T->getAs<PointerType>();
-  QualType UPointeeT = PT->getPointeeType().getUnqualifiedType();
-  if (UPointeeT->isRecordType()) {
-    const RecordType *RecordTy = UPointeeT->getAs<RecordType>();
-    if (!RecordTy->getDecl()->isCompleteDefinition())
-      return false;
-  }
-  return true;
-}
-
-/// Check whether the two versions match.
-static bool versionsMatch(const VersionTuple &X, const VersionTuple &Y) {
-  return (X == Y);
-}
-
-/// AvailabilityAttrsMatch - This routine checks that if comparing two
-/// availability attributes, all their components match. It returns
-/// true, if not dealing with availability or when all components of
-/// availability attributes match. This routine is only called when
-/// the attributes are of the same kind.
-static bool AvailabilityAttrsMatch(Attr *At1, Attr *At2) {
-  const AvailabilityAttr *AA1 = dyn_cast<AvailabilityAttr>(At1);
-  if (!AA1)
-    return true;
-  const AvailabilityAttr *AA2 = cast<AvailabilityAttr>(At2);
-
-  VersionTuple Introduced1 = AA1->getIntroduced();
-  VersionTuple Deprecated1 = AA1->getDeprecated();
-  VersionTuple Obsoleted1 = AA1->getObsoleted();
-  bool IsUnavailable1 = AA1->getUnavailable();
-  VersionTuple Introduced2 = AA2->getIntroduced();
-  VersionTuple Deprecated2 = AA2->getDeprecated();
-  VersionTuple Obsoleted2 = AA2->getObsoleted();
-  bool IsUnavailable2 = AA2->getUnavailable();
-  return (versionsMatch(Introduced1, Introduced2) &&
-          versionsMatch(Deprecated1, Deprecated2) &&
-          versionsMatch(Obsoleted1, Obsoleted2) &&
-          IsUnavailable1 == IsUnavailable2);
-}
-
-static bool MatchTwoAttributeLists(const AttrVec &Attrs1, const AttrVec &Attrs2,
-                                   bool &AvailabilityArgsMatch) {
-  // This list is very small, so this need not be optimized.
-  for (unsigned i = 0, e = Attrs1.size(); i != e; i++) {
-    bool match = false;
-    for (unsigned j = 0, f = Attrs2.size(); j != f; j++) {
-      // Matching attribute kind only. Except for Availability attributes,
-      // we are not getting into details of the attributes. For all practical purposes
-      // this is sufficient.
-      if (Attrs1[i]->getKind() == Attrs2[j]->getKind()) {
-        if (AvailabilityArgsMatch)
-          AvailabilityArgsMatch = AvailabilityAttrsMatch(Attrs1[i], Attrs2[j]);
-        match = true;
-        break;
-      }
-    }
-    if (!match)
-      return false;
-  }
-  return true;
-}
-
-/// AttributesMatch - This routine checks list of attributes for two
-/// decls. It returns false, if there is a mismatch in kind of
-/// attributes seen in the decls. It returns true if the two decls
-/// have list of same kind of attributes. Furthermore, when there
-/// are availability attributes in the two decls, it sets the
-/// AvailabilityArgsMatch to false if availability attributes have
-/// different versions, etc.
-static bool AttributesMatch(const Decl *Decl1, const Decl *Decl2,
-                            bool &AvailabilityArgsMatch) {
-  if (!Decl1->hasAttrs() || !Decl2->hasAttrs()) {
-    AvailabilityArgsMatch = (Decl1->hasAttrs() == Decl2->hasAttrs());
-    return true;
-  }
-  AvailabilityArgsMatch = true;
-  const AttrVec &Attrs1 = Decl1->getAttrs();
-  const AttrVec &Attrs2 = Decl2->getAttrs();
-  bool match = MatchTwoAttributeLists(Attrs1, Attrs2, AvailabilityArgsMatch);
-  if (match && (Attrs2.size() > Attrs1.size()))
-    return MatchTwoAttributeLists(Attrs2, Attrs1, AvailabilityArgsMatch);
-  return match;
-}
-
-static bool IsValidIdentifier(ASTContext &Ctx,
-                              const char *Name) {
-  if (!isAsciiIdentifierStart(Name[0]))
-    return false;
-  std::string NameString = Name;
-  NameString[0] = toLowercase(NameString[0]);
-  const IdentifierInfo *II = &Ctx.Idents.get(NameString);
-  return II->getTokenID() ==  tok::identifier;
-}
-
-bool ObjCMigrateASTConsumer::migrateProperty(ASTContext &Ctx,
-                             ObjCContainerDecl *D,
-                             ObjCMethodDecl *Method) {
-  if (Method->isPropertyAccessor() || !Method->isInstanceMethod() ||
-      Method->param_size() != 0)
-    return false;
-  // Is this method candidate to be a getter?
-  QualType GRT = Method->getReturnType();
-  if (GRT->isVoidType())
-    return false;
-
-  Selector GetterSelector = Method->getSelector();
-  ObjCInstanceTypeFamily OIT_Family =
-    Selector::getInstTypeMethodFamily(GetterSelector);
-
-  if (OIT_Family != OIT_None)
-    return false;
-
-  const IdentifierInfo *getterName = GetterSelector.getIdentifierInfoForSlot(0);
-  Selector SetterSelector =
-  SelectorTable::constructSetterSelector(PP.getIdentifierTable(),
-                                         PP.getSelectorTable(),
-                                         getterName);
-  ObjCMethodDecl *SetterMethod = D->getInstanceMethod(SetterSelector);
-  unsigned LengthOfPrefix = 0;
-  if (!SetterMethod) {
-    // try a different naming convention for getter: isXxxxx
-    StringRef getterNameString = getterName->getName();
-    bool IsPrefix = getterNameString.starts_with("is");
-    // Note that we don't want to change an isXXX method of retainable object
-    // type to property (readonly or otherwise).
-    if (IsPrefix && GRT->isObjCRetainableType())
-      return false;
-    if (IsPrefix || getterNameString.starts_with("get")) {
-      LengthOfPrefix = (IsPrefix ? 2 : 3);
-      const char *CGetterName = getterNameString.data() + LengthOfPrefix;
-      // Make sure that first character after "is" or "get" prefix can
-      // start an identifier.
-      if (!IsValidIdentifier(Ctx, CGetterName))
-        return false;
-      if (CGetterName[0] && isUppercase(CGetterName[0])) {
-        getterName = &Ctx.Idents.get(CGetterName);
-        SetterSelector =
-        SelectorTable::constructSetterSelector(PP.getIdentifierTable(),
-                                               PP.getSelectorTable(),
-                                               getterName);
-        SetterMethod = D->getInstanceMethod(SetterSelector);
-      }
-    }
-  }
-
-  if (SetterMethod) {
-    if ((ASTMigrateActions & FrontendOptions::ObjCMT_ReadwriteProperty) == 0)
-      return false;
-    bool AvailabilityArgsMatch;
-    if (SetterMethod->isDeprecated() ||
-        !AttributesMatch(Method, SetterMethod, AvailabilityArgsMatch))
-      return false;
-
-    // Is this a valid setter, matching the target getter?
-    QualType SRT = SetterMethod->getReturnType();
-    if (!SRT->isVoidType())
-      return false;
-    const ParmVarDecl *argDecl = *SetterMethod->param_begin();
-    QualType ArgType = argDecl->getType();
-    if (!Ctx.hasSameUnqualifiedType(ArgType, GRT))
-      return false;
-    edit::Commit commit(*Editor);
-    rewriteToObjCProperty(Method, SetterMethod, *NSAPIObj, commit,
-                          LengthOfPrefix,
-                          (ASTMigrateActions &
-                           FrontendOptions::ObjCMT_AtomicProperty) != 0,
-                          (ASTMigrateActions &
-                           FrontendOptions::ObjCMT_NsAtomicIOSOnlyProperty) != 0,
-                          AvailabilityArgsMatch);
-    Editor->commit(commit);
-    return true;
-  }
-  else if (ASTMigrateActions & FrontendOptions::ObjCMT_ReadonlyProperty) {
-    // Try a non-void method with no argument (and no setter or property of same name
-    // as a 'readonly' property.
-    edit::Commit commit(*Editor);
-    rewriteToObjCProperty(Method, nullptr /*SetterMethod*/, *NSAPIObj, commit,
-                          LengthOfPrefix,
-                          (ASTMigrateActions &
-                           FrontendOptions::ObjCMT_AtomicProperty) != 0,
-                          (ASTMigrateActions &
-                           FrontendOptions::ObjCMT_NsAtomicIOSOnlyProperty) != 0,
-                          /*AvailabilityArgsMatch*/false);
-    Editor->commit(commit);
-    return true;
-  }
-  return false;
-}
-
-void ObjCMigrateASTConsumer::migrateNsReturnsInnerPointer(ASTContext &Ctx,
-                                                          ObjCMethodDecl *OM) {
-  if (OM->isImplicit() ||
-      !OM->isInstanceMethod() ||
-      OM->hasAttr<ObjCReturnsInnerPointerAttr>())
-    return;
-
-  QualType RT = OM->getReturnType();
-  if (!TypeIsInnerPointer(RT) ||
-      !NSAPIObj->isMacroDefined("NS_RETURNS_INNER_POINTER"))
-    return;
-
-  edit::Commit commit(*Editor);
-  commit.insertBefore(OM->getEndLoc(), " NS_RETURNS_INNER_POINTER");
-  Editor->commit(commit);
-}
-
-void ObjCMigrateASTConsumer::migratePropertyNsReturnsInnerPointer(ASTContext &Ctx,
-                                                                  ObjCPropertyDecl *P) {
-  QualType T = P->getType();
-
-  if (!TypeIsInnerPointer(T) ||
-      !NSAPIObj->isMacroDefined("NS_RETURNS_INNER_POINTER"))
-    return;
-  edit::Commit commit(*Editor);
-  commit.insertBefore(P->getEndLoc(), " NS_RETURNS_INNER_POINTER ");
-  Editor->commit(commit);
-}
-
-void ObjCMigrateASTConsumer::migrateAllMethodInstaceType(ASTContext &Ctx,
-                                                 ObjCContainerDecl *CDecl) {
-  if (CDecl->isDeprecated() || IsCategoryNameWithDeprecatedSuffix(CDecl))
-    return;
-
-  // migrate methods which can have instancetype as their result type.
-  for (auto *Method : CDecl->methods()) {
-    if (Method->isDeprecated())
-      continue;
-    migrateMethodInstanceType(Ctx, CDecl, Method);
-  }
-}
-
-void ObjCMigrateASTConsumer::migrateFactoryMethod(ASTContext &Ctx,
-                                                  ObjCContainerDecl *CDecl,
-                                                  ObjCMethodDecl *OM,
-                                                  ObjCInstanceTypeFamily OIT_Family) {
-  if (OM->isInstanceMethod() ||
-      OM->getReturnType() == Ctx.getObjCInstanceType() ||
-      !OM->getReturnType()->isObjCIdType())
-    return;
-
-  // Candidate factory methods are + (id) NaMeXXX : ... which belong to a class
-  // NSYYYNamE with matching names be at least 3 characters long.
-  ObjCInterfaceDecl *IDecl = dyn_cast<ObjCInterfaceDecl>(CDecl);
-  if (!IDecl) {
-    if (ObjCCategoryDecl *CatDecl = dyn_cast<ObjCCategoryDecl>(CDecl))
-      IDecl = CatDecl->getClassInterface();
-    else if (ObjCImplDecl *ImpDecl = dyn_cast<ObjCImplDecl>(CDecl))
-      IDecl = ImpDecl->getClassInterface();
-  }
-  if (!IDecl)
-    return;
-
-  std::string StringClassName = std::string(IDecl->getName());
-  StringRef LoweredClassName(StringClassName);
-  std::string StringLoweredClassName = LoweredClassName.lower();
-  LoweredClassName = StringLoweredClassName;
-
-  const IdentifierInfo *MethodIdName =
-      OM->getSelector().getIdentifierInfoForSlot(0);
-  // Handle method with no name at its first selector slot; e.g. + (id):(int)x.
-  if (!MethodIdName)
-    return;
-
-  std::string MethodName = std::string(MethodIdName->getName());
-  if (OIT_Family == OIT_Singleton || OIT_Family == OIT_ReturnsSelf) {
-    StringRef STRefMethodName(MethodName);
-    size_t len = 0;
-    if (STRefMethodName.starts_with("standard"))
-      len = strlen("standard");
-    else if (STRefMethodName.starts_with("shared"))
-      len = strlen("shared");
-    else if (STRefMethodName.starts_with("default"))
-      len = strlen("default");
-    else
-      return;
-    MethodName = std::string(STRefMethodName.substr(len));
-  }
-  std::string MethodNameSubStr = MethodName.substr(0, 3);
-  StringRef MethodNamePrefix(MethodNameSubStr);
-  std::string StringLoweredMethodNamePrefix = MethodNamePrefix.lower();
-  MethodNamePrefix = StringLoweredMethodNamePrefix;
-  size_t Ix = LoweredClassName.rfind(MethodNamePrefix);
-  if (Ix == StringRef::npos)
-    return;
-  std::string ClassNamePostfix = std::string(LoweredClassName.substr(Ix));
-  StringRef LoweredMethodName(MethodName);
-  std::string StringLoweredMethodName = LoweredMethodName.lower();
-  LoweredMethodName = StringLoweredMethodName;
-  if (!LoweredMethodName.starts_with(ClassNamePostfix))
-    return;
-  if (OIT_Family == OIT_ReturnsSelf)
-    ReplaceWithClasstype(*this, OM);
-  else
-    ReplaceWithInstancetype(Ctx, *this, OM);
-}
-
-static bool IsVoidStarType(QualType Ty) {
-  if (!Ty->isPointerType())
-    return false;
-
-  // Is the type void*?
-  const PointerType* PT = Ty->castAs<PointerType>();
-  if (PT->getPointeeType().getUnqualifiedType()->isVoidType())
-    return true;
-  return IsVoidStarType(PT->getPointeeType());
-}
-
-/// AuditedType - This routine audits the type AT and returns false if it is one of known
-/// CF object types or of the "void *" variety. It returns true if we don't care about the type
-/// such as a non-pointer or pointers which have no ownership issues (such as "int *").
-static bool AuditedType (QualType AT) {
-  if (!AT->isAnyPointerType() && !AT->isBlockPointerType())
-    return true;
-  // FIXME. There isn't much we can say about CF pointer type; or is there?
-  if (ento::coreFoundation::isCFObjectRef(AT) ||
-      IsVoidStarType(AT) ||
-      // If an ObjC object is type, assuming that it is not a CF function and
-      // that it is an un-audited function.
-      AT->isObjCObjectPointerType() || AT->isObjCBuiltinType())
-    return false;
-  // All other pointers are assumed audited as harmless.
-  return true;
-}
-
-void ObjCMigrateASTConsumer::AnnotateImplicitBridging(ASTContext &Ctx) {
-  if (CFFunctionIBCandidates.empty())
-    return;
-  if (!NSAPIObj->isMacroDefined("CF_IMPLICIT_BRIDGING_ENABLED")) {
-    CFFunctionIBCandidates.clear();
-    FileId = FileID();
-    return;
-  }
-  // Insert CF_IMPLICIT_BRIDGING_ENABLE/CF_IMPLICIT_BRIDGING_DISABLED
-  const Decl *FirstFD = CFFunctionIBCandidates[0];
-  const Decl *LastFD  =
-    CFFunctionIBCandidates[CFFunctionIBCandidates.size()-1];
-  const char *PragmaString = "\nCF_IMPLICIT_BRIDGING_ENABLED\n\n";
-  edit::Commit commit(*Editor);
-  commit.insertBefore(FirstFD->getBeginLoc(), PragmaString);
-  PragmaString = "\n\nCF_IMPLICIT_BRIDGING_DISABLED\n";
-  SourceLocation EndLoc = LastFD->getEndLoc();
-  // get location just past end of function location.
-  EndLoc = PP.getLocForEndOfToken(EndLoc);
-  if (isa<FunctionDecl>(LastFD)) {
-    // For Methods, EndLoc points to the ending semcolon. So,
-    // not of these extra work is needed.
-    Token Tok;
-    // get locaiton of token that comes after end of function.
-    bool Failed = PP.getRawToken(EndLoc, Tok, /*IgnoreWhiteSpace=*/true);
-    if (!Failed)
-      EndLoc = Tok.getLocation();
-  }
-  commit.insertAfterToken(EndLoc, PragmaString);
-  Editor->commit(commit);
-  FileId = FileID();
-  CFFunctionIBCandidates.clear();
-}
-
-void ObjCMigrateASTConsumer::migrateCFAnnotation(ASTContext &Ctx, const Decl *Decl) {
-  if (Decl->isDeprecated())
-    return;
-
-  if (Decl->hasAttr<CFAuditedTransferAttr>()) {
-    assert(CFFunctionIBCandidates.empty() &&
-           "Cannot have audited functions/methods inside user "
-           "provided CF_IMPLICIT_BRIDGING_ENABLE");
-    return;
-  }
-
-  // Finction must be annotated first.
-  if (const FunctionDecl *FuncDecl = dyn_cast<FunctionDecl>(Decl)) {
-    CF_BRIDGING_KIND AuditKind = migrateAddFunctionAnnotation(Ctx, FuncDecl);
-    if (AuditKind == CF_BRIDGING_ENABLE) {
-      CFFunctionIBCandidates.push_back(Decl);
-      if (FileId.isInvalid())
-        FileId = PP.getSourceManager().getFileID(Decl->getLocation());
-    }
-    else if (AuditKind == CF_BRIDGING_MAY_INCLUDE) {
-      if (!CFFunctionIBCandidates.empty()) {
-        CFFunctionIBCandidates.push_back(Decl);
-        if (FileId.isInvalid())
-          FileId = PP.getSourceManager().getFileID(Decl->getLocation());
-      }
-    }
-    else
-      AnnotateImplicitBridging(Ctx);
-  }
-  else {
-    migrateAddMethodAnnotation(Ctx, cast<ObjCMethodDecl>(Decl));
-    AnnotateImplicitBridging(Ctx);
-  }
-}
-
-void ObjCMigrateASTConsumer::AddCFAnnotations(ASTContext &Ctx,
-                                              const RetainSummary *RS,
-                                              const FunctionDecl *FuncDecl,
-                                              bool ResultAnnotated) {
-  // Annotate function.
-  if (!ResultAnnotated) {
-    RetEffect Ret = RS->getRetEffect();
-    const char *AnnotationString = nullptr;
-    if (Ret.getObjKind() == ObjKind::CF) {
-      if (Ret.isOwned() && NSAPIObj->isMacroDefined("CF_RETURNS_RETAINED"))
-        AnnotationString = " CF_RETURNS_RETAINED";
-      else if (Ret.notOwned() &&
-               NSAPIObj->isMacroDefined("CF_RETURNS_NOT_RETAINED"))
-        AnnotationString = " CF_RETURNS_NOT_RETAINED";
-    }
-    else if (Ret.getObjKind() == ObjKind::ObjC) {
-      if (Ret.isOwned() && NSAPIObj->isMacroDefined("NS_RETURNS_RETAINED"))
-        AnnotationString = " NS_RETURNS_RETAINED";
-    }
-
-    if (AnnotationString) {
-      edit::Commit commit(*Editor);
-      commit.insertAfterToken(FuncDecl->getEndLoc(), AnnotationString);
-      Editor->commit(commit);
-    }
-  }
-  unsigned i = 0;
-  for (FunctionDecl::param_const_iterator pi = FuncDecl->param_begin(),
-       pe = FuncDecl->param_end(); pi != pe; ++pi, ++i) {
-    const ParmVarDecl *pd = *pi;
-    ArgEffect AE = RS->getArg(i);
-    if (AE.getKind() == DecRef && AE.getObjKind() == ObjKind::CF &&
-        !pd->hasAttr<CFConsumedAttr>() &&
-        NSAPIObj->isMacroDefined("CF_CONSUMED")) {
-      edit::Commit commit(*Editor);
-      commit.insertBefore(pd->getLocation(), "CF_CONSUMED ");
-      Editor->commit(commit);
-    } else if (AE.getKind() == DecRef && AE.getObjKind() == ObjKind::ObjC &&
-               !pd->hasAttr<NSConsumedAttr>() &&
-               NSAPIObj->isMacroDefined("NS_CONSUMED")) {
-      edit::Commit commit(*Editor);
-      commit.insertBefore(pd->getLocation(), "NS_CONSUMED ");
-      Editor->commit(commit);
-    }
-  }
-}
-
-ObjCMigrateASTConsumer::CF_BRIDGING_KIND
-  ObjCMigrateASTConsumer::migrateAddFunctionAnnotation(
-                                                  ASTContext &Ctx,
-                                                  const FunctionDecl *FuncDecl) {
-  if (FuncDecl->hasBody())
-    return CF_BRIDGING_NONE;
-
-  const RetainSummary *RS =
-      getSummaryManager(Ctx).getSummary(AnyCall(FuncDecl));
-  bool FuncIsReturnAnnotated = (FuncDecl->hasAttr<CFReturnsRetainedAttr>() ||
-                                FuncDecl->hasAttr<CFReturnsNotRetainedAttr>() ||
-                                FuncDecl->hasAttr<NSReturnsRetainedAttr>() ||
-                                FuncDecl->hasAttr<NSReturnsNotRetainedAttr>() ||
-                                FuncDecl->hasAttr<NSReturnsAutoreleasedAttr>());
-
-  // Trivial case of when function is annotated and has no argument.
-  if (FuncIsReturnAnnotated && FuncDecl->getNumParams() == 0)
-    return CF_BRIDGING_NONE;
-
-  bool ReturnCFAudited = false;
-  if (!FuncIsReturnAnnotated) {
-    RetEffect Ret = RS->getRetEffect();
-    if (Ret.getObjKind() == ObjKind::CF &&
-        (Ret.isOwned() || Ret.notOwned()))
-      ReturnCFAudited = true;
-    else if (!AuditedType(FuncDecl->getReturnType()))
-      return CF_BRIDGING_NONE;
-  }
-
-  // At this point result type is audited for potential inclusion.
-  unsigned i = 0;
-  bool ArgCFAudited = false;
-  for (FunctionDecl::param_const_iterator pi = FuncDecl->param_begin(),
-       pe = FuncDecl->param_end(); pi != pe; ++pi, ++i) {
-    const ParmVarDecl *pd = *pi;
-    ArgEffect AE = RS->getArg(i);
-    if ((AE.getKind() == DecRef /*CFConsumed annotated*/ ||
-         AE.getKind() == IncRef) && AE.getObjKind() == ObjKind::CF) {
-      if (AE.getKind() == DecRef && !pd->hasAttr<CFConsumedAttr>())
-        ArgCFAudited = true;
-      else if (AE.getKind() == IncRef)
-        ArgCFAudited = true;
-    } else {
-      QualType AT = pd->getType();
-      if (!AuditedType(AT)) {
-        AddCFAnnotations(Ctx, RS, FuncDecl, FuncIsReturnAnnotated);
-        return CF_BRIDGING_NONE;
-      }
-    }
-  }
-  if (ReturnCFAudited || ArgCFAudited)
-    return CF_BRIDGING_ENABLE;
-
-  return CF_BRIDGING_MAY_INCLUDE;
-}
-
-void ObjCMigrateASTConsumer::migrateARCSafeAnnotation(ASTContext &Ctx,
-                                                 ObjCContainerDecl *CDecl) {
-  if (!isa<ObjCInterfaceDecl>(CDecl) || CDecl->isDeprecated())
-    return;
-
-  // migrate methods which can have instancetype as their result type.
-  for (const auto *Method : CDecl->methods())
-    migrateCFAnnotation(Ctx, Method);
-}
-
-void ObjCMigrateASTConsumer::AddCFAnnotations(ASTContext &Ctx,
-                                              const RetainSummary *RS,
-                                              const ObjCMethodDecl *MethodDecl,
-                                              bool ResultAnnotated) {
-  // Annotate function.
-  if (!ResultAnnotated) {
-    RetEffect Ret = RS->getRetEffect();
-    const char *AnnotationString = nullptr;
-    if (Ret.getObjKind() == ObjKind::CF) {
-      if (Ret.isOwned() && NSAPIObj->isMacroDefined("CF_RETURNS_RETAINED"))
-        AnnotationString = " CF_RETURNS_RETAINED";
-      else if (Ret.notOwned() &&
-               NSAPIObj->isMacroDefined("CF_RETURNS_NOT_RETAINED"))
-        AnnotationString = " CF_RETURNS_NOT_RETAINED";
-    }
-    else if (Ret.getObjKind() == ObjKind::ObjC) {
-      ObjCMethodFamily OMF = MethodDecl->getMethodFamily();
-      switch (OMF) {
-        case clang::OMF_alloc:
-        case clang::OMF_new:
-        case clang::OMF_copy:
-        case clang::OMF_init:
-        case clang::OMF_mutableCopy:
-          break;
-
-        default:
-          if (Ret.isOwned() && NSAPIObj->isMacroDefined("NS_RETURNS_RETAINED"))
-            AnnotationString = " NS_RETURNS_RETAINED";
-          break;
-      }
-    }
-
-    if (AnnotationString) {
-      edit::Commit commit(*Editor);
-      commit.insertBefore(MethodDecl->getEndLoc(), AnnotationString);
-      Editor->commit(commit);
-    }
-  }
-  unsigned i = 0;
-  for (ObjCMethodDecl::param_const_iterator pi = MethodDecl->param_begin(),
-       pe = MethodDecl->param_end(); pi != pe; ++pi, ++i) {
-    const ParmVarDecl *pd = *pi;
-    ArgEffect AE = RS->getArg(i);
-    if (AE.getKind() == DecRef
-        && AE.getObjKind() == ObjKind::CF
-        && !pd->hasAttr<CFConsumedAttr>() &&
-        NSAPIObj->isMacroDefined("CF_CONSUMED")) {
-      edit::Commit commit(*Editor);
-      commit.insertBefore(pd->getLocation(), "CF_CONSUMED ");
-      Editor->commit(commit);
-    }
-  }
-}
-
-void ObjCMigrateASTConsumer::migrateAddMethodAnnotation(
-                                            ASTContext &Ctx,
-                                            const ObjCMethodDecl *MethodDecl) {
-  if (MethodDecl->hasBody() || MethodDecl->isImplicit())
-    return;
-
-  const RetainSummary *RS =
-      getSummaryManager(Ctx).getSummary(AnyCall(MethodDecl));
-
-  bool MethodIsReturnAnnotated =
-      (MethodDecl->hasAttr<CFReturnsRetainedAttr>() ||
-       MethodDecl->hasAttr<CFReturnsNotRetainedAttr>() ||
-       MethodDecl->hasAttr<NSReturnsRetainedAttr>() ||
-       MethodDecl->hasAttr<NSReturnsNotRetainedAttr>() ||
-       MethodDecl->hasAttr<NSReturnsAutoreleasedAttr>());
-
-  if (RS->getReceiverEffect().getKind() == DecRef &&
-      !MethodDecl->hasAttr<NSConsumesSelfAttr>() &&
-      MethodDecl->getMethodFamily() != OMF_init &&
-      MethodDecl->getMethodFamily() != OMF_release &&
-      NSAPIObj->isMacroDefined("NS_CONSUMES_SELF")) {
-    edit::Commit commit(*Editor);
-    commit.insertBefore(MethodDecl->getEndLoc(), " NS_CONSUMES_SELF");
-    Editor->commit(commit);
-  }
-
-  // Trivial case of when function is annotated and has no argument.
-  if (MethodIsReturnAnnotated &&
-      (MethodDecl->param_begin() == MethodDecl->param_end()))
-    return;
-
-  if (!MethodIsReturnAnnotated) {
-    RetEffect Ret = RS->getRetEffect();
-    if ((Ret.getObjKind() == ObjKind::CF ||
-         Ret.getObjKind() == ObjKind::ObjC) &&
-        (Ret.isOwned() || Ret.notOwned())) {
-      AddCFAnnotations(Ctx, RS, MethodDecl, false);
-      return;
-    } else if (!AuditedType(MethodDecl->getReturnType()))
-      return;
-  }
-
-  // At this point result type is either annotated or audited.
-  unsigned i = 0;
-  for (ObjCMethodDecl::param_const_iterator pi = MethodDecl->param_begin(),
-       pe = MethodDecl->param_end(); pi != pe; ++pi, ++i) {
-    const ParmVarDecl *pd = *pi;
-    ArgEffect AE = RS->getArg(i);
-    if ((AE.getKind() == DecRef && !pd->hasAttr<CFConsumedAttr>()) ||
-        AE.getKind() == IncRef || !AuditedType(pd->getType())) {
-      AddCFAnnotations(Ctx, RS, MethodDecl, MethodIsReturnAnnotated);
-      return;
-    }
-  }
-}
-
-namespace {
-class SuperInitChecker : public RecursiveASTVisitor<SuperInitChecker> {
-public:
-  bool shouldVisitTemplateInstantiations() const { return false; }
-  bool shouldWalkTypesOfTypeLocs() const { return false; }
-
-  bool VisitObjCMessageExpr(ObjCMessageExpr *E) {
-    if (E->getReceiverKind() == ObjCMessageExpr::SuperInstance) {
-      if (E->getMethodFamily() == OMF_init)
-        return false;
-    }
-    return true;
-  }
-};
-} // end anonymous namespace
-
-static bool hasSuperInitCall(const ObjCMethodDecl *MD) {
-  return !SuperInitChecker().TraverseStmt(MD->getBody());
-}
-
-void ObjCMigrateASTConsumer::inferDesignatedInitializers(
-    ASTContext &Ctx,
-    const ObjCImplementationDecl *ImplD) {
-
-  const ObjCInterfaceDecl *IFace = ImplD->getClassInterface();
-  if (!IFace || IFace->hasDesignatedInitializers())
-    return;
-  if (!NSAPIObj->isMacroDefined("NS_DESIGNATED_INITIALIZER"))
-    return;
-
-  for (const auto *MD : ImplD->instance_methods()) {
-    if (MD->isDeprecated() ||
-        MD->getMethodFamily() != OMF_init ||
-        MD->isDesignatedInitializerForTheInterface())
-      continue;
-    const ObjCMethodDecl *IFaceM = IFace->getMethod(MD->getSelector(),
-                                                    /*isInstance=*/true);
-    if (!IFaceM)
-      continue;
-    if (hasSuperInitCall(MD)) {
-      edit::Commit commit(*Editor);
-      commit.insert(IFaceM->getEndLoc(), " NS_DESIGNATED_INITIALIZER");
-      Editor->commit(commit);
-    }
-  }
-}
-
-bool ObjCMigrateASTConsumer::InsertFoundation(ASTContext &Ctx,
-                                              SourceLocation  Loc) {
-  if (FoundationIncluded)
-    return true;
-  if (Loc.isInvalid())
-    return false;
-  auto *nsEnumId = &Ctx.Idents.get("NS_ENUM");
-  if (PP.getMacroDefinitionAtLoc(nsEnumId, Loc)) {
-    FoundationIncluded = true;
-    return true;
-  }
-  edit::Commit commit(*Editor);
-  if (Ctx.getLangOpts().Modules)
-    commit.insert(Loc, "#ifndef NS_ENUM\n@import Foundation;\n#endif\n");
-  else
-    commit.insert(Loc, "#ifndef NS_ENUM\n#import <Foundation/Foundation.h>\n#endif\n");
-  Editor->commit(commit);
-  FoundationIncluded = true;
-  return true;
-}
-
-namespace {
-
-class RewritesReceiver : public edit::EditsReceiver {
-  Rewriter &Rewrite;
-
-public:
-  RewritesReceiver(Rewriter &Rewrite) : Rewrite(Rewrite) { }
-
-  void insert(SourceLocation loc, StringRef text) override {
-    Rewrite.InsertText(loc, text);
-  }
-  void replace(CharSourceRange range, StringRef text) override {
-    Rewrite.ReplaceText(range.getBegin(), Rewrite.getRangeSize(range), text);
-  }
-};
-
-class JSONEditWriter : public edit::EditsReceiver {
-  SourceManager &SourceMgr;
-  llvm::raw_ostream &OS;
-
-public:
-  JSONEditWriter(SourceManager &SM, llvm::raw_ostream &OS)
-    : SourceMgr(SM), OS(OS) {
-    OS << "[\n";
-  }
-  ~JSONEditWriter() override { OS << "]\n"; }
-
-private:
-  struct EntryWriter {
-    SourceManager &SourceMgr;
-    llvm::raw_ostream &OS;
-
-    EntryWriter(SourceManager &SM, llvm::raw_ostream &OS)
-      : SourceMgr(SM), OS(OS) {
-      OS << " {\n";
-    }
-    ~EntryWriter() {
-      OS << " },\n";
-    }
-
-    void writeLoc(SourceLocation Loc) {
-      FileID FID;
-      unsigned Offset;
-      std::tie(FID, Offset) = SourceMgr.getDecomposedLoc(Loc);
-      assert(FID.isValid());
-      SmallString<200> Path =
-          StringRef(SourceMgr.getFileEntryRefForID(FID)->getName());
-      llvm::sys::fs::make_absolute(Path);
-      OS << "  \"file\": \"";
-      OS.write_escaped(Path.str()) << "\",\n";
-      OS << "  \"offset\": " << Offset << ",\n";
-    }
-
-    void writeRemove(CharSourceRange Range) {
-      assert(Range.isCharRange());
-      std::pair<FileID, unsigned> Begin =
-          SourceMgr.getDecomposedLoc(Range.getBegin());
-      std::pair<FileID, unsigned> End =
-          SourceMgr.getDecomposedLoc(Range.getEnd());
-      assert(Begin.first == End.first);
-      assert(Begin.second <= End.second);
-      unsigned Length = End.second - Begin.second;
-
-      OS << "  \"remove\": " << Length << ",\n";
-    }
-
-    void writeText(StringRef Text) {
-      OS << "  \"text\": \"";
-      OS.write_escaped(Text) << "\",\n";
-    }
-  };
-
-  void insert(SourceLocation Loc, StringRef Text) override {
-    EntryWriter Writer(SourceMgr, OS);
-    Writer.writeLoc(Loc);
-    Writer.writeText(Text);
-  }
-
-  void replace(CharSourceRange Range, StringRef Text) override {
-    EntryWriter Writer(SourceMgr, OS);
-    Writer.writeLoc(Range.getBegin());
-    Writer.writeRemove(Range);
-    Writer.writeText(Text);
-  }
-
-  void remove(CharSourceRange Range) override {
-    EntryWriter Writer(SourceMgr, OS);
-    Writer.writeLoc(Range.getBegin());
-    Writer.writeRemove(Range);
-  }
-};
-
-} // end anonymous namespace
-
-void ObjCMigrateASTConsumer::HandleTranslationUnit(ASTContext &Ctx) {
-
-  TranslationUnitDecl *TU = Ctx.getTranslationUnitDecl();
-  if (ASTMigrateActions & FrontendOptions::ObjCMT_MigrateDecls) {
-    for (DeclContext::decl_iterator D = TU->decls_begin(), DEnd = TU->decls_end();
-         D != DEnd; ++D) {
-      FileID FID = PP.getSourceManager().getFileID((*D)->getLocation());
-      if (FID.isValid())
-        if (FileId.isValid() && FileId != FID) {
-          if (ASTMigrateActions & FrontendOptions::ObjCMT_Annotation)
-            AnnotateImplicitBridging(Ctx);
-        }
-
-      if (ObjCInterfaceDecl *CDecl = dyn_cast<ObjCInterfaceDecl>(*D))
-        if (canModify(CDecl))
-          migrateObjCContainerDecl(Ctx, CDecl);
-      if (ObjCCategoryDecl *CatDecl = dyn_cast<ObjCCategoryDecl>(*D)) {
-        if (canModify(CatDecl))
-          migrateObjCContainerDecl(Ctx, CatDecl);
-      }
-      else if (ObjCProtocolDecl *PDecl = dyn_cast<ObjCProtocolDecl>(*D)) {
-        ObjCProtocolDecls.insert(PDecl->getCanonicalDecl());
-        if (canModify(PDecl))
-          migrateObjCContainerDecl(Ctx, PDecl);
-      }
-      else if (const ObjCImplementationDecl *ImpDecl =
-               dyn_cast<ObjCImplementationDecl>(*D)) {
-        if ((ASTMigrateActions & FrontendOptions::ObjCMT_ProtocolConformance) &&
-            canModify(ImpDecl))
-          migrateProtocolConformance(Ctx, ImpDecl);
-      }
-      else if (const EnumDecl *ED = dyn_cast<EnumDecl>(*D)) {
-        if (!(ASTMigrateActions & FrontendOptions::ObjCMT_NsMacros))
-          continue;
-        if (!canModify(ED))
-          continue;
-        DeclContext::decl_iterator N = D;
-        if (++N != DEnd) {
-          const TypedefDecl *TD = dyn_cast<TypedefDecl>(*N);
-          if (migrateNSEnumDecl(Ctx, ED, TD) && TD)
-            D++;
-        }
-        else
-          migrateNSEnumDecl(Ctx, ED, /*TypedefDecl */nullptr);
-      }
-      else if (const TypedefDecl *TD = dyn_cast<TypedefDecl>(*D)) {
-        if (!(ASTMigrateActions & FrontendOptions::ObjCMT_NsMacros))
-          continue;
-        if (!canModify(TD))
-          continue;
-        DeclContext::decl_iterator N = D;
-        if (++N == DEnd)
-          continue;
-        if (const EnumDecl *ED = dyn_cast<EnumDecl>(*N)) {
-          if (canModify(ED)) {
-            if (++N != DEnd)
-              if (const TypedefDecl *TDF = dyn_cast<TypedefDecl>(*N)) {
-                // prefer typedef-follows-enum to enum-follows-typedef pattern.
-                if (migrateNSEnumDecl(Ctx, ED, TDF)) {
-                  ++D; ++D;
-                  CacheObjCNSIntegerTypedefed(TD);
-                  continue;
-                }
-              }
-            if (migrateNSEnumDecl(Ctx, ED, TD)) {
-              ++D;
-              continue;
-            }
-          }
-        }
-        CacheObjCNSIntegerTypedefed(TD);
-      }
-      else if (const FunctionDecl *FD = dyn_cast<FunctionDecl>(*D)) {
-        if ((ASTMigrateActions & FrontendOptions::ObjCMT_Annotation) &&
-            canModify(FD))
-          migrateCFAnnotation(Ctx, FD);
-      }
-
-      if (ObjCContainerDecl *CDecl = dyn_cast<ObjCContainerDecl>(*D)) {
-        bool CanModify = canModify(CDecl);
-        // migrate methods which can have instancetype as their result type.
-        if ((ASTMigrateActions & FrontendOptions::ObjCMT_Instancetype) &&
-            CanModify)
-          migrateAllMethodInstaceType(Ctx, CDecl);
-        // annotate methods with CF annotations.
-        if ((ASTMigrateActions & FrontendOptions::ObjCMT_Annotation) &&
-            CanModify)
-          migrateARCSafeAnnotation(Ctx, CDecl);
-      }
-
-      if (const ObjCImplementationDecl *
-            ImplD = dyn_cast<ObjCImplementationDecl>(*D)) {
-        if ((ASTMigrateActions & FrontendOptions::ObjCMT_DesignatedInitializer) &&
-            canModify(ImplD))
-          inferDesignatedInitializers(Ctx, ImplD);
-      }
-    }
-    if (ASTMigrateActions & FrontendOptions::ObjCMT_Annotation)
-      AnnotateImplicitBridging(Ctx);
-  }
-
- if (IsOutputFile) {
-   std::error_code EC;
-   llvm::raw_fd_ostream OS(MigrateDir, EC, llvm::sys::fs::OF_None);
-   if (EC) {
-      DiagnosticsEngine &Diags = Ctx.getDiagnostics();
-      Diags.Report(Diags.getCustomDiagID(DiagnosticsEngine::Error, "%0"))
-          << EC.message();
-      return;
-    }
-
-   JSONEditWriter Writer(Ctx.getSourceManager(), OS);
-   Editor->applyRewrites(Writer);
-   return;
- }
-
-  Rewriter rewriter(Ctx.getSourceManager(), Ctx.getLangOpts());
-  RewritesReceiver Rec(rewriter);
-  Editor->applyRewrites(Rec);
-
-  for (Rewriter::buffer_iterator
-        I = rewriter.buffer_begin(), E = rewriter.buffer_end(); I != E; ++I) {
-    FileID FID = I->first;
-    RewriteBuffer &buf = I->second;
-    OptionalFileEntryRef file =
-        Ctx.getSourceManager().getFileEntryRefForID(FID);
-    assert(file);
-    SmallString<512> newText;
-    llvm::raw_svector_ostream vecOS(newText);
-    buf.write(vecOS);
-    std::unique_ptr<llvm::MemoryBuffer> memBuf(
-        llvm::MemoryBuffer::getMemBufferCopy(newText.str(), file->getName()));
-    SmallString<64> filePath(file->getName());
-    FileMgr.FixupRelativePath(filePath);
-    Remapper.remap(filePath.str(), std::move(memBuf));
-  }
-
-  if (IsOutputFile) {
-    Remapper.flushToFile(MigrateDir, Ctx.getDiagnostics());
-  } else {
-    Remapper.flushToDisk(MigrateDir, Ctx.getDiagnostics());
-  }
-}
-
-bool MigrateSourceAction::BeginInvocation(CompilerInstance &CI) {
-  CI.getDiagnostics().setIgnoreAllWarnings(true);
-  return true;
-}
-
-static std::vector<std::string> getAllowListFilenames(StringRef DirPath) {
-  using namespace llvm::sys::fs;
-  using namespace llvm::sys::path;
-
-  std::vector<std::string> Filenames;
-  if (DirPath.empty() || !is_directory(DirPath))
-    return Filenames;
-
-  std::error_code EC;
-  directory_iterator DI = directory_iterator(DirPath, EC);
-  directory_iterator DE;
-  for (; !EC && DI != DE; DI = DI.increment(EC)) {
-    if (is_regular_file(DI->path()))
-      Filenames.push_back(std::string(filename(DI->path())));
-  }
-
-  return Filenames;
-}
-
-std::unique_ptr<ASTConsumer>
-MigrateSourceAction::CreateASTConsumer(CompilerInstance &CI, StringRef InFile) {
-  PPConditionalDirectiveRecord *
-    PPRec = new PPConditionalDirectiveRecord(CI.getSourceManager());
-  unsigned ObjCMTAction = CI.getFrontendOpts().ObjCMTAction;
-  unsigned ObjCMTOpts = ObjCMTAction;
-  // These are companion flags, they do not enable transformations.
-  ObjCMTOpts &= ~(FrontendOptions::ObjCMT_AtomicProperty |
-                  FrontendOptions::ObjCMT_NsAtomicIOSOnlyProperty);
-  if (ObjCMTOpts == FrontendOptions::ObjCMT_None) {
-    // If no specific option was given, enable literals+subscripting transforms
-    // by default.
-    ObjCMTAction |=
-        FrontendOptions::ObjCMT_Literals | FrontendOptions::ObjCMT_Subscripting;
-  }
-  CI.getPreprocessor().addPPCallbacks(std::unique_ptr<PPCallbacks>(PPRec));
-  std::vector<std::string> AllowList =
-      getAllowListFilenames(CI.getFrontendOpts().ObjCMTAllowListPath);
-  return std::make_unique<ObjCMigrateASTConsumer>(
-      CI.getFrontendOpts().OutputFile, ObjCMTAction, Remapper,
-      CI.getFileManager(), PPRec, CI.getPreprocessor(),
-      /*isOutputFile=*/true, AllowList);
-}
-
-namespace {
-struct EditEntry {
-  OptionalFileEntryRef File;
-  unsigned Offset = 0;
-  unsigned RemoveLen = 0;
-  std::string Text;
-};
-} // end anonymous namespace
-
-namespace llvm {
-template<> struct DenseMapInfo<EditEntry> {
-  static inline EditEntry getEmptyKey() {
-    EditEntry Entry;
-    Entry.Offset = unsigned(-1);
-    return Entry;
-  }
-  static inline EditEntry getTombstoneKey() {
-    EditEntry Entry;
-    Entry.Offset = unsigned(-2);
-    return Entry;
-  }
-  static unsigned getHashValue(const EditEntry& Val) {
-    return (unsigned)llvm::hash_combine(Val.File, Val.Offset, Val.RemoveLen,
-                                        Val.Text);
-  }
-  static bool isEqual(const EditEntry &LHS, const EditEntry &RHS) {
-    return LHS.File == RHS.File &&
-        LHS.Offset == RHS.Offset &&
-        LHS.RemoveLen == RHS.RemoveLen &&
-        LHS.Text == RHS.Text;
-  }
-};
-} // end namespace llvm
-
-namespace {
-class RemapFileParser {
-  FileManager &FileMgr;
-
-public:
-  RemapFileParser(FileManager &FileMgr) : FileMgr(FileMgr) { }
-
-  bool parse(StringRef File, SmallVectorImpl<EditEntry> &Entries) {
-    using namespace llvm::yaml;
-
-    llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>> FileBufOrErr =
-        llvm::MemoryBuffer::getFile(File);
-    if (!FileBufOrErr)
-      return true;
-
-    llvm::SourceMgr SM;
-    Stream YAMLStream(FileBufOrErr.get()->getMemBufferRef(), SM);
-    document_iterator I = YAMLStream.begin();
-    if (I == YAMLStream.end())
-      return true;
-    Node *Root = I->getRoot();
-    if (!Root)
-      return true;
-
-    SequenceNode *SeqNode = dyn_cast<SequenceNode>(Root);
-    if (!SeqNode)
-      return true;
-
-    for (SequenceNode::iterator
-           AI = SeqNode->begin(), AE = SeqNode->end(); AI != AE; ++AI) {
-      MappingNode *MapNode = dyn_cast<MappingNode>(&*AI);
-      if (!MapNode)
-        continue;
-      parseEdit(MapNode, Entries);
-    }
-
-    return false;
-  }
-
-private:
-  void parseEdit(llvm::yaml::MappingNode *Node,
-                 SmallVectorImpl<EditEntry> &Entries) {
-    using namespace llvm::yaml;
-    EditEntry Entry;
-    bool Ignore = false;
-
-    for (MappingNode::iterator
-           KVI = Node->begin(), KVE = Node->end(); KVI != KVE; ++KVI) {
-      ScalarNode *KeyString = dyn_cast<ScalarNode>((*KVI).getKey());
-      if (!KeyString)
-        continue;
-      SmallString<10> KeyStorage;
-      StringRef Key = KeyString->getValue(KeyStorage);
-
-      ScalarNode *ValueString = dyn_cast<ScalarNode>((*KVI).getValue());
-      if (!ValueString)
-        continue;
-      SmallString<64> ValueStorage;
-      StringRef Val = ValueString->getValue(ValueStorage);
-
-      if (Key == "file") {
-        if (auto File = FileMgr.getOptionalFileRef(Val))
-          Entry.File = File;
-        else
-          Ignore = true;
-      } else if (Key == "offset") {
-        if (Val.getAsInteger(10, Entry.Offset))
-          Ignore = true;
-      } else if (Key == "remove") {
-        if (Val.getAsInteger(10, Entry.RemoveLen))
-          Ignore = true;
-      } else if (Key == "text") {
-        Entry.Text = std::string(Val);
-      }
-    }
-
-    if (!Ignore)
-      Entries.push_back(Entry);
-  }
-};
-} // end anonymous namespace
-
-static bool reportDiag(const Twine &Err, DiagnosticsEngine &Diag) {
-  Diag.Report(Diag.getCustomDiagID(DiagnosticsEngine::Error, "%0"))
-      << Err.str();
-  return true;
-}
-
-static std::string applyEditsToTemp(FileEntryRef FE,
-                                    ArrayRef<EditEntry> Edits,
-                                    FileManager &FileMgr,
-                                    DiagnosticsEngine &Diag) {
-  using namespace llvm::sys;
-
-  SourceManager SM(Diag, FileMgr);
-  FileID FID = SM.createFileID(FE, SourceLocation(), SrcMgr::C_User);
-  LangOptions LangOpts;
-  edit::EditedSource Editor(SM, LangOpts);
-  for (ArrayRef<EditEntry>::iterator
-        I = Edits.begin(), E = Edits.end(); I != E; ++I) {
-    const EditEntry &Entry = *I;
-    assert(Entry.File == FE);
-    SourceLocation Loc =
-        SM.getLocForStartOfFile(FID).getLocWithOffset(Entry.Offset);
-    CharSourceRange Range;
-    if (Entry.RemoveLen != 0) {
-      Range = CharSourceRange::getCharRange(Loc,
-                                         Loc.getLocWithOffset(Entry.RemoveLen));
-    }
-
-    edit::Commit commit(Editor);
-    if (Range.isInvalid()) {
-      commit.insert(Loc, Entry.Text);
-    } else if (Entry.Text.empty()) {
-      commit.remove(Range);
-    } else {
-      commit.replace(Range, Entry.Text);
-    }
-    Editor.commit(commit);
-  }
-
-  Rewriter rewriter(SM, LangOpts);
-  RewritesReceiver Rec(rewriter);
-  Editor.applyRewrites(Rec, /*adjustRemovals=*/false);
-
-  const RewriteBuffer *Buf = rewriter.getRewriteBufferFor(FID);
-  SmallString<512> NewText;
-  llvm::raw_svector_ostream OS(NewText);
-  Buf->write(OS);
-
-  SmallString<64> TempPath;
-  int FD;
-  if (fs::createTemporaryFile(path::filename(FE.getName()),
-                              path::extension(FE.getName()).drop_front(), FD,
-                              TempPath)) {
-    reportDiag("Could not create file: " + TempPath.str(), Diag);
-    return std::string();
-  }
-
-  llvm::raw_fd_ostream TmpOut(FD, /*shouldClose=*/true);
-  TmpOut.write(NewText.data(), NewText.size());
-  TmpOut.close();
-
-  return std::string(TempPath);
-}
-
-bool arcmt::getFileRemappingsFromFileList(
-                        std::vector<std::pair<std::string,std::string> > &remap,
-                        ArrayRef<StringRef> remapFiles,
-                        DiagnosticConsumer *DiagClient) {
-  bool hasErrorOccurred = false;
-
-  FileSystemOptions FSOpts;
-  FileManager FileMgr(FSOpts);
-  RemapFileParser Parser(FileMgr);
-
-  IntrusiveRefCntPtr<DiagnosticIDs> DiagID(new DiagnosticIDs());
-  IntrusiveRefCntPtr<DiagnosticsEngine> Diags(
-      new DiagnosticsEngine(DiagID, new DiagnosticOptions,
-                            DiagClient, /*ShouldOwnClient=*/false));
-
-  typedef llvm::DenseMap<FileEntryRef, std::vector<EditEntry> >
-      FileEditEntriesTy;
-  FileEditEntriesTy FileEditEntries;
-
-  llvm::DenseSet<EditEntry> EntriesSet;
-
-  for (ArrayRef<StringRef>::iterator
-         I = remapFiles.begin(), E = remapFiles.end(); I != E; ++I) {
-    SmallVector<EditEntry, 16> Entries;
-    if (Parser.parse(*I, Entries))
-      continue;
-
-    for (SmallVectorImpl<EditEntry>::iterator
-           EI = Entries.begin(), EE = Entries.end(); EI != EE; ++EI) {
-      EditEntry &Entry = *EI;
-      if (!Entry.File)
-        continue;
-      std::pair<llvm::DenseSet<EditEntry>::iterator, bool>
-        Insert = EntriesSet.insert(Entry);
-      if (!Insert.second)
-        continue;
-
-      FileEditEntries[*Entry.File].push_back(Entry);
-    }
-  }
-
-  for (FileEditEntriesTy::iterator
-         I = FileEditEntries.begin(), E = FileEditEntries.end(); I != E; ++I) {
-    std::string TempFile = applyEditsToTemp(I->first, I->second,
-                                            FileMgr, *Diags);
-    if (TempFile.empty()) {
-      hasErrorOccurred = true;
-      continue;
-    }
-
-    remap.emplace_back(std::string(I->first.getName()), TempFile);
-  }
-
-  return hasErrorOccurred;
-}
diff --git a/clang/lib/ARCMigrate/PlistReporter.cpp b/clang/lib/ARCMigrate/PlistReporter.cpp
deleted file mode 100644
index f78ca5e1c9bdd..0000000000000
--- a/clang/lib/ARCMigrate/PlistReporter.cpp
+++ /dev/null
@@ -1,124 +0,0 @@
-//===--- PlistReporter.cpp - ARC Migrate Tool Plist Reporter ----*- C++ -*-===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "Internals.h"
-#include "clang/Basic/FileManager.h"
-#include "clang/Basic/PlistSupport.h"
-#include "clang/Basic/SourceManager.h"
-#include "clang/Lex/Lexer.h"
-using namespace clang;
-using namespace arcmt;
-using namespace markup;
-
-static StringRef getLevelName(DiagnosticsEngine::Level Level) {
-  switch (Level) {
-  case DiagnosticsEngine::Ignored:
-    llvm_unreachable("ignored");
-  case DiagnosticsEngine::Note:
-    return "note";
-  case DiagnosticsEngine::Remark:
-  case DiagnosticsEngine::Warning:
-    return "warning";
-  case DiagnosticsEngine::Fatal:
-  case DiagnosticsEngine::Error:
-    return "error";
-  }
-  llvm_unreachable("Invalid DiagnosticsEngine level!");
-}
-
-void arcmt::writeARCDiagsToPlist(const std::string &outPath,
-                                 ArrayRef<StoredDiagnostic> diags,
-                                 SourceManager &SM,
-                                 const LangOptions &LangOpts) {
-  DiagnosticIDs DiagIDs;
-
-  // Build up a set of FIDs that we use by scanning the locations and
-  // ranges of the diagnostics.
-  FIDMap FM;
-  SmallVector<FileID, 10> Fids;
-
-  for (ArrayRef<StoredDiagnostic>::iterator
-         I = diags.begin(), E = diags.end(); I != E; ++I) {
-    const StoredDiagnostic &D = *I;
-
-    AddFID(FM, Fids, SM, D.getLocation());
-
-    for (StoredDiagnostic::range_iterator
-           RI = D.range_begin(), RE = D.range_end(); RI != RE; ++RI) {
-      AddFID(FM, Fids, SM, RI->getBegin());
-      AddFID(FM, Fids, SM, RI->getEnd());
-    }
-  }
-
-  std::error_code EC;
-  llvm::raw_fd_ostream o(outPath, EC, llvm::sys::fs::OF_TextWithCRLF);
-  if (EC) {
-    llvm::errs() << "error: could not create file: " << outPath << '\n';
-    return;
-  }
-
-  EmitPlistHeader(o);
-
-  // Write the root object: a <dict> containing...
-  //  - "files", an <array> mapping from FIDs to file names
-  //  - "diagnostics", an <array> containing the diagnostics
-  o << "<dict>\n"
-       " <key>files</key>\n"
-       " <array>\n";
-
-  for (FileID FID : Fids)
-    EmitString(o << "  ", SM.getFileEntryRefForID(FID)->getName()) << '\n';
-
-  o << " </array>\n"
-       " <key>diagnostics</key>\n"
-       " <array>\n";
-
-  for (ArrayRef<StoredDiagnostic>::iterator
-         DI = diags.begin(), DE = diags.end(); DI != DE; ++DI) {
-
-    const StoredDiagnostic &D = *DI;
-
-    if (D.getLevel() == DiagnosticsEngine::Ignored)
-      continue;
-
-    o << "  <dict>\n";
-
-    // Output the diagnostic.
-    o << "   <key>description</key>";
-    EmitString(o, D.getMessage()) << '\n';
-    o << "   <key>category</key>";
-    EmitString(o, DiagIDs.getCategoryNameFromID(
-                          DiagIDs.getCategoryNumberForDiag(D.getID()))) << '\n';
-    o << "   <key>type</key>";
-    EmitString(o, getLevelName(D.getLevel())) << '\n';
-
-    // Output the location of the bug.
-    o << "  <key>location</key>\n";
-    EmitLocation(o, SM, D.getLocation(), FM, 2);
-
-    // Output the ranges (if any).
-    if (!D.getRanges().empty()) {
-      o << "   <key>ranges</key>\n";
-      o << "   <array>\n";
-      for (auto &R : D.getRanges()) {
-        CharSourceRange ExpansionRange = SM.getExpansionRange(R);
-        EmitRange(o, SM, Lexer::getAsCharRange(ExpansionRange, SM, LangOpts),
-                  FM, 4);
-      }
-      o << "   </array>\n";
-    }
-
-    // Close up the entry.
-    o << "  </dict>\n";
-  }
-
-  o << " </array>\n";
-
-  // Finish.
-  o << "</dict>\n</plist>\n";
-}
diff --git a/clang/lib/ARCMigrate/TransAPIUses.cpp b/clang/lib/ARCMigrate/TransAPIUses.cpp
deleted file mode 100644
index 8f5d4f4bde06c..0000000000000
--- a/clang/lib/ARCMigrate/TransAPIUses.cpp
+++ /dev/null
@@ -1,107 +0,0 @@
-//===--- TransAPIUses.cpp - Transformations to ARC mode -------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-//
-// checkAPIUses:
-//
-// Emits error/fix with some API uses that are obsolete or not safe in ARC mode:
-//
-// - NSInvocation's [get/set]ReturnValue and [get/set]Argument are only safe
-//   with __unsafe_unretained objects.
-// - Calling -zone gets replaced with 'nil'.
-//
-//===----------------------------------------------------------------------===//
-
-#include "Transforms.h"
-#include "Internals.h"
-#include "clang/AST/ASTContext.h"
-#include "clang/Sema/SemaDiagnostic.h"
-
-using namespace clang;
-using namespace arcmt;
-using namespace trans;
-
-namespace {
-
-class APIChecker : public RecursiveASTVisitor<APIChecker> {
-  MigrationPass &Pass;
-
-  Selector getReturnValueSel, setReturnValueSel;
-  Selector getArgumentSel, setArgumentSel;
-
-  Selector zoneSel;
-public:
-  APIChecker(MigrationPass &pass) : Pass(pass) {
-    SelectorTable &sels = Pass.Ctx.Selectors;
-    IdentifierTable &ids = Pass.Ctx.Idents;
-    getReturnValueSel = sels.getUnarySelector(&ids.get("getReturnValue"));
-    setReturnValueSel = sels.getUnarySelector(&ids.get("setReturnValue"));
-
-    const IdentifierInfo *selIds[2];
-    selIds[0] = &ids.get("getArgument");
-    selIds[1] = &ids.get("atIndex");
-    getArgumentSel = sels.getSelector(2, selIds);
-    selIds[0] = &ids.get("setArgument");
-    setArgumentSel = sels.getSelector(2, selIds);
-
-    zoneSel = sels.getNullarySelector(&ids.get("zone"));
-  }
-
-  bool VisitObjCMessageExpr(ObjCMessageExpr *E) {
-    // NSInvocation.
-    if (E->isInstanceMessage() &&
-        E->getReceiverInterface() &&
-        E->getReceiverInterface()->getName() == "NSInvocation") {
-      StringRef selName;
-      if (E->getSelector() == getReturnValueSel)
-        selName = "getReturnValue";
-      else if (E->getSelector() == setReturnValueSel)
-        selName = "setReturnValue";
-      else if (E->getSelector() == getArgumentSel)
-        selName = "getArgument";
-      else if (E->getSelector() == setArgumentSel)
-        selName = "setArgument";
-      else
-        return true;
-
-      Expr *parm = E->getArg(0)->IgnoreParenCasts();
-      QualType pointee = parm->getType()->getPointeeType();
-      if (pointee.isNull())
-        return true;
-
-      if (pointee.getObjCLifetime() > Qualifiers::OCL_ExplicitNone)
-        Pass.TA.report(parm->getBeginLoc(),
-                       diag::err_arcmt_nsinvocation_ownership,
-                       parm->getSourceRange())
-            << selName;
-
-      return true;
-    }
-
-    // -zone.
-    if (E->isInstanceMessage() &&
-        E->getInstanceReceiver() &&
-        E->getSelector() == zoneSel &&
-        Pass.TA.hasDiagnostic(diag::err_unavailable,
-                              diag::err_unavailable_message,
-                              E->getSelectorLoc(0))) {
-      // Calling -zone is meaningless in ARC, change it to nil.
-      Transaction Trans(Pass.TA);
-      Pass.TA.clearDiagnostic(diag::err_unavailable,
-                              diag::err_unavailable_message,
-                              E->getSelectorLoc(0));
-      Pass.TA.replace(E->getSourceRange(), getNilString(Pass));
-    }
-    return true;
-  }
-};
-
-} // anonymous namespace
-
-void trans::checkAPIUses(MigrationPass &pass) {
-  APIChecker(pass).TraverseDecl(pass.Ctx.getTranslationUnitDecl());
-}
diff --git a/clang/lib/ARCMigrate/TransARCAssign.cpp b/clang/lib/ARCMigrate/TransARCAssign.cpp
deleted file mode 100644
index d1d5b9e014b17..0000000000000
--- a/clang/lib/ARCMigrate/TransARCAssign.cpp
+++ /dev/null
@@ -1,77 +0,0 @@
-//===--- TransARCAssign.cpp - Transformations to ARC mode -----------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-//
-// makeAssignARCSafe:
-//
-// Add '__strong' where appropriate.
-//
-//  for (id x in collection) {
-//    x = 0;
-//  }
-// ---->
-//  for (__strong id x in collection) {
-//    x = 0;
-//  }
-//
-//===----------------------------------------------------------------------===//
-
-#include "Transforms.h"
-#include "Internals.h"
-#include "clang/AST/ASTContext.h"
-#include "clang/Sema/SemaDiagnostic.h"
-
-using namespace clang;
-using namespace arcmt;
-using namespace trans;
-
-namespace {
-
-class ARCAssignChecker : public RecursiveASTVisitor<ARCAssignChecker> {
-  MigrationPass &Pass;
-  llvm::DenseSet<VarDecl *> ModifiedVars;
-
-public:
-  ARCAssignChecker(MigrationPass &pass) : Pass(pass) { }
-
-  bool VisitBinaryOperator(BinaryOperator *Exp) {
-    if (Exp->getType()->isDependentType())
-      return true;
-
-    Expr *E = Exp->getLHS();
-    SourceLocation OrigLoc = E->getExprLoc();
-    SourceLocation Loc = OrigLoc;
-    DeclRefExpr *declRef = dyn_cast<DeclRefExpr>(E->IgnoreParenCasts());
-    if (declRef && isa<VarDecl>(declRef->getDecl())) {
-      ASTContext &Ctx = Pass.Ctx;
-      Expr::isModifiableLvalueResult IsLV = E->isModifiableLvalue(Ctx, &Loc);
-      if (IsLV != Expr::MLV_ConstQualified)
-        return true;
-      VarDecl *var = cast<VarDecl>(declRef->getDecl());
-      if (var->isARCPseudoStrong()) {
-        Transaction Trans(Pass.TA);
-        if (Pass.TA.clearDiagnostic(diag::err_typecheck_arr_assign_enumeration,
-                                    Exp->getOperatorLoc())) {
-          if (!ModifiedVars.count(var)) {
-            TypeLoc TLoc = var->getTypeSourceInfo()->getTypeLoc();
-            Pass.TA.insert(TLoc.getBeginLoc(), "__strong ");
-            ModifiedVars.insert(var);
-          }
-        }
-      }
-    }
-
-    return true;
-  }
-};
-
-} // anonymous namespace
-
-void trans::makeAssignARCSafe(MigrationPass &pass) {
-  ARCAssignChecker assignCheck(pass);
-  assignCheck.TraverseDecl(pass.Ctx.getTranslationUnitDecl());
-}
diff --git a/clang/lib/ARCMigrate/TransAutoreleasePool.cpp b/clang/lib/ARCMigrate/TransAutoreleasePool.cpp
deleted file mode 100644
index 6d501228e712b..0000000000000
--- a/clang/lib/ARCMigrate/TransAutoreleasePool.cpp
+++ /dev/null
@@ -1,435 +0,0 @@
-//===--- TransAutoreleasePool.cpp - Transformations to ARC mode -----------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-//
-// rewriteAutoreleasePool:
-//
-// Calls to NSAutoreleasePools will be rewritten as an @autorelease scope.
-//
-//  NSAutoreleasePool *pool = [[NSAutoreleasePool alloc] init];
-//  ...
-//  [pool release];
-// ---->
-//  @autorelease {
-//  ...
-//  }
-//
-// An NSAutoreleasePool will not be touched if:
-// - There is not a corresponding -release/-drain in the same scope
-// - Not all references of the NSAutoreleasePool variable can be removed
-// - There is a variable that is declared inside the intended @autorelease scope
-//   which is also used outside it.
-//
-//===----------------------------------------------------------------------===//
-
-#include "Transforms.h"
-#include "Internals.h"
-#include "clang/AST/ASTContext.h"
-#include "clang/Basic/SourceManager.h"
-#include "clang/Sema/SemaDiagnostic.h"
-#include <map>
-
-using namespace clang;
-using namespace arcmt;
-using namespace trans;
-
-namespace {
-
-class ReleaseCollector : public RecursiveASTVisitor<ReleaseCollector> {
-  Decl *Dcl;
-  SmallVectorImpl<ObjCMessageExpr *> &Releases;
-
-public:
-  ReleaseCollector(Decl *D, SmallVectorImpl<ObjCMessageExpr *> &releases)
-    : Dcl(D), Releases(releases) { }
-
-  bool VisitObjCMessageExpr(ObjCMessageExpr *E) {
-    if (!E->isInstanceMessage())
-      return true;
-    if (E->getMethodFamily() != OMF_release)
-      return true;
-    Expr *instance = E->getInstanceReceiver()->IgnoreParenCasts();
-    if (DeclRefExpr *DE = dyn_cast<DeclRefExpr>(instance)) {
-      if (DE->getDecl() == Dcl)
-        Releases.push_back(E);
-    }
-    return true;
-  }
-};
-
-}
-
-namespace {
-
-class AutoreleasePoolRewriter
-                         : public RecursiveASTVisitor<AutoreleasePoolRewriter> {
-public:
-  AutoreleasePoolRewriter(MigrationPass &pass)
-    : Body(nullptr), Pass(pass) {
-    PoolII = &pass.Ctx.Idents.get("NSAutoreleasePool");
-    DrainSel = pass.Ctx.Selectors.getNullarySelector(
-                                                 &pass.Ctx.Idents.get("drain"));
-  }
-
-  void transformBody(Stmt *body, Decl *ParentD) {
-    Body = body;
-    TraverseStmt(body);
-  }
-
-  ~AutoreleasePoolRewriter() {
-    SmallVector<VarDecl *, 8> VarsToHandle;
-
-    for (std::map<VarDecl *, PoolVarInfo>::iterator
-           I = PoolVars.begin(), E = PoolVars.end(); I != E; ++I) {
-      VarDecl *var = I->first;
-      PoolVarInfo &info = I->second;
-
-      // Check that we can handle/rewrite all references of the pool.
-
-      clearRefsIn(info.Dcl, info.Refs);
-      for (SmallVectorImpl<PoolScope>::iterator
-             scpI = info.Scopes.begin(),
-             scpE = info.Scopes.end(); scpI != scpE; ++scpI) {
-        PoolScope &scope = *scpI;
-        clearRefsIn(*scope.Begin, info.Refs);
-        clearRefsIn(*scope.End, info.Refs);
-        clearRefsIn(scope.Releases.begin(), scope.Releases.end(), info.Refs);
-      }
-
-      // Even if one reference is not handled we will not do anything about that
-      // pool variable.
-      if (info.Refs.empty())
-        VarsToHandle.push_back(var);
-    }
-
-    for (unsigned i = 0, e = VarsToHandle.size(); i != e; ++i) {
-      PoolVarInfo &info = PoolVars[VarsToHandle[i]];
-
-      Transaction Trans(Pass.TA);
-
-      clearUnavailableDiags(info.Dcl);
-      Pass.TA.removeStmt(info.Dcl);
-
-      // Add "@autoreleasepool { }"
-      for (SmallVectorImpl<PoolScope>::iterator
-             scpI = info.Scopes.begin(),
-             scpE = info.Scopes.end(); scpI != scpE; ++scpI) {
-        PoolScope &scope = *scpI;
-        clearUnavailableDiags(*scope.Begin);
-        clearUnavailableDiags(*scope.End);
-        if (scope.IsFollowedBySimpleReturnStmt) {
-          // Include the return in the scope.
-          Pass.TA.replaceStmt(*scope.Begin, "@autoreleasepool {");
-          Pass.TA.removeStmt(*scope.End);
-          Stmt::child_iterator retI = scope.End;
-          ++retI;
-          SourceLocation afterSemi =
-              findLocationAfterSemi((*retI)->getEndLoc(), Pass.Ctx);
-          assert(afterSemi.isValid() &&
-                 "Didn't we check before setting IsFollowedBySimpleReturnStmt "
-                 "to true?");
-          Pass.TA.insertAfterToken(afterSemi, "\n}");
-          Pass.TA.increaseIndentation(
-              SourceRange(scope.getIndentedRange().getBegin(),
-                          (*retI)->getEndLoc()),
-              scope.CompoundParent->getBeginLoc());
-        } else {
-          Pass.TA.replaceStmt(*scope.Begin, "@autoreleasepool {");
-          Pass.TA.replaceStmt(*scope.End, "}");
-          Pass.TA.increaseIndentation(scope.getIndentedRange(),
-                                      scope.CompoundParent->getBeginLoc());
-        }
-      }
-
-      // Remove rest of pool var references.
-      for (SmallVectorImpl<PoolScope>::iterator
-             scpI = info.Scopes.begin(),
-             scpE = info.Scopes.end(); scpI != scpE; ++scpI) {
-        PoolScope &scope = *scpI;
-        for (SmallVectorImpl<ObjCMessageExpr *>::iterator
-               relI = scope.Releases.begin(),
-               relE = scope.Releases.end(); relI != relE; ++relI) {
-          clearUnavailableDiags(*relI);
-          Pass.TA.removeStmt(*relI);
-        }
-      }
-    }
-  }
-
-  bool VisitCompoundStmt(CompoundStmt *S) {
-    SmallVector<PoolScope, 4> Scopes;
-
-    for (Stmt::child_iterator
-           I = S->body_begin(), E = S->body_end(); I != E; ++I) {
-      Stmt *child = getEssential(*I);
-      if (DeclStmt *DclS = dyn_cast<DeclStmt>(child)) {
-        if (DclS->isSingleDecl()) {
-          if (VarDecl *VD = dyn_cast<VarDecl>(DclS->getSingleDecl())) {
-            if (isNSAutoreleasePool(VD->getType())) {
-              PoolVarInfo &info = PoolVars[VD];
-              info.Dcl = DclS;
-              collectRefs(VD, S, info.Refs);
-              // Does this statement follow the pattern:
-              // NSAutoreleasePool * pool = [NSAutoreleasePool  new];
-              if (isPoolCreation(VD->getInit())) {
-                Scopes.push_back(PoolScope());
-                Scopes.back().PoolVar = VD;
-                Scopes.back().CompoundParent = S;
-                Scopes.back().Begin = I;
-              }
-            }
-          }
-        }
-      } else if (BinaryOperator *bop = dyn_cast<BinaryOperator>(child)) {
-        if (DeclRefExpr *dref = dyn_cast<DeclRefExpr>(bop->getLHS())) {
-          if (VarDecl *VD = dyn_cast<VarDecl>(dref->getDecl())) {
-            // Does this statement follow the pattern:
-            // pool = [NSAutoreleasePool  new];
-            if (isNSAutoreleasePool(VD->getType()) &&
-                isPoolCreation(bop->getRHS())) {
-              Scopes.push_back(PoolScope());
-              Scopes.back().PoolVar = VD;
-              Scopes.back().CompoundParent = S;
-              Scopes.back().Begin = I;
-            }
-          }
-        }
-      }
-
-      if (Scopes.empty())
-        continue;
-
-      if (isPoolDrain(Scopes.back().PoolVar, child)) {
-        PoolScope &scope = Scopes.back();
-        scope.End = I;
-        handlePoolScope(scope, S);
-        Scopes.pop_back();
-      }
-    }
-    return true;
-  }
-
-private:
-  void clearUnavailableDiags(Stmt *S) {
-    if (S)
-      Pass.TA.clearDiagnostic(diag::err_unavailable,
-                              diag::err_unavailable_message,
-                              S->getSourceRange());
-  }
-
-  struct PoolScope {
-    VarDecl *PoolVar;
-    CompoundStmt *CompoundParent;
-    Stmt::child_iterator Begin;
-    Stmt::child_iterator End;
-    bool IsFollowedBySimpleReturnStmt;
-    SmallVector<ObjCMessageExpr *, 4> Releases;
-
-    PoolScope()
-        : PoolVar(nullptr), CompoundParent(nullptr),
-          IsFollowedBySimpleReturnStmt(false) {}
-
-    SourceRange getIndentedRange() const {
-      Stmt::child_iterator rangeS = Begin;
-      ++rangeS;
-      if (rangeS == End)
-        return SourceRange();
-      Stmt::child_iterator rangeE = Begin;
-      for (Stmt::child_iterator I = rangeS; I != End; ++I)
-        ++rangeE;
-      return SourceRange((*rangeS)->getBeginLoc(), (*rangeE)->getEndLoc());
-    }
-  };
-
-  class NameReferenceChecker : public RecursiveASTVisitor<NameReferenceChecker>{
-    ASTContext &Ctx;
-    SourceRange ScopeRange;
-    SourceLocation &referenceLoc, &declarationLoc;
-
-  public:
-    NameReferenceChecker(ASTContext &ctx, PoolScope &scope,
-                         SourceLocation &referenceLoc,
-                         SourceLocation &declarationLoc)
-      : Ctx(ctx), referenceLoc(referenceLoc),
-        declarationLoc(declarationLoc) {
-      ScopeRange = SourceRange((*scope.Begin)->getBeginLoc(),
-                               (*scope.End)->getBeginLoc());
-    }
-
-    bool VisitDeclRefExpr(DeclRefExpr *E) {
-      return checkRef(E->getLocation(), E->getDecl()->getLocation());
-    }
-
-    bool VisitTypedefTypeLoc(TypedefTypeLoc TL) {
-      return checkRef(TL.getBeginLoc(), TL.getTypedefNameDecl()->getLocation());
-    }
-
-    bool VisitTagTypeLoc(TagTypeLoc TL) {
-      return checkRef(TL.getBeginLoc(), TL.getDecl()->getLocation());
-    }
-
-  private:
-    bool checkRef(SourceLocation refLoc, SourceLocation declLoc) {
-      if (isInScope(declLoc)) {
-        referenceLoc = refLoc;
-        declarationLoc = declLoc;
-        return false;
-      }
-      return true;
-    }
-
-    bool isInScope(SourceLocation loc) {
-      if (loc.isInvalid())
-        return false;
-
-      SourceManager &SM = Ctx.getSourceManager();
-      if (SM.isBeforeInTranslationUnit(loc, ScopeRange.getBegin()))
-        return false;
-      return SM.isBeforeInTranslationUnit(loc, ScopeRange.getEnd());
-    }
-  };
-
-  void handlePoolScope(PoolScope &scope, CompoundStmt *compoundS) {
-    // Check that all names declared inside the scope are not used
-    // outside the scope.
-    {
-      bool nameUsedOutsideScope = false;
-      SourceLocation referenceLoc, declarationLoc;
-      Stmt::child_iterator SI = scope.End, SE = compoundS->body_end();
-      ++SI;
-      // Check if the autoreleasepool scope is followed by a simple return
-      // statement, in which case we will include the return in the scope.
-      if (SI != SE)
-        if (ReturnStmt *retS = dyn_cast<ReturnStmt>(*SI))
-          if ((retS->getRetValue() == nullptr ||
-               isa<DeclRefExpr>(retS->getRetValue()->IgnoreParenCasts())) &&
-              findLocationAfterSemi(retS->getEndLoc(), Pass.Ctx).isValid()) {
-            scope.IsFollowedBySimpleReturnStmt = true;
-            ++SI; // the return will be included in scope, don't check it.
-          }
-
-      for (; SI != SE; ++SI) {
-        nameUsedOutsideScope = !NameReferenceChecker(Pass.Ctx, scope,
-                                                     referenceLoc,
-                                              declarationLoc).TraverseStmt(*SI);
-        if (nameUsedOutsideScope)
-          break;
-      }
-
-      // If not all references were cleared it means some variables/typenames/etc
-      // declared inside the pool scope are used outside of it.
-      // We won't try to rewrite the pool.
-      if (nameUsedOutsideScope) {
-        Pass.TA.reportError("a name is referenced outside the "
-            "NSAutoreleasePool scope that it was declared in", referenceLoc);
-        Pass.TA.reportNote("name declared here", declarationLoc);
-        Pass.TA.reportNote("intended @autoreleasepool scope begins here",
-                           (*scope.Begin)->getBeginLoc());
-        Pass.TA.reportNote("intended @autoreleasepool scope ends here",
-                           (*scope.End)->getBeginLoc());
-        return;
-      }
-    }
-
-    // Collect all releases of the pool; they will be removed.
-    {
-      ReleaseCollector releaseColl(scope.PoolVar, scope.Releases);
-      Stmt::child_iterator I = scope.Begin;
-      ++I;
-      for (; I != scope.End; ++I)
-        releaseColl.TraverseStmt(*I);
-    }
-
-    PoolVars[scope.PoolVar].Scopes.push_back(scope);
-  }
-
-  bool isPoolCreation(Expr *E) {
-    if (!E) return false;
-    E = getEssential(E);
-    ObjCMessageExpr *ME = dyn_cast<ObjCMessageExpr>(E);
-    if (!ME) return false;
-    if (ME->getMethodFamily() == OMF_new &&
-        ME->getReceiverKind() == ObjCMessageExpr::Class &&
-        isNSAutoreleasePool(ME->getReceiverInterface()))
-      return true;
-    if (ME->getReceiverKind() == ObjCMessageExpr::Instance &&
-        ME->getMethodFamily() == OMF_init) {
-      Expr *rec = getEssential(ME->getInstanceReceiver());
-      if (ObjCMessageExpr *recME = dyn_cast_or_null<ObjCMessageExpr>(rec)) {
-        if (recME->getMethodFamily() == OMF_alloc &&
-            recME->getReceiverKind() == ObjCMessageExpr::Class &&
-            isNSAutoreleasePool(recME->getReceiverInterface()))
-          return true;
-      }
-    }
-
-    return false;
-  }
-
-  bool isPoolDrain(VarDecl *poolVar, Stmt *S) {
-    if (!S) return false;
-    S = getEssential(S);
-    ObjCMessageExpr *ME = dyn_cast<ObjCMessageExpr>(S);
-    if (!ME) return false;
-    if (ME->getReceiverKind() == ObjCMessageExpr::Instance) {
-      Expr *rec = getEssential(ME->getInstanceReceiver());
-      if (DeclRefExpr *dref = dyn_cast<DeclRefExpr>(rec))
-        if (dref->getDecl() == poolVar)
-          return ME->getMethodFamily() == OMF_release ||
-                 ME->getSelector() == DrainSel;
-    }
-
-    return false;
-  }
-
-  bool isNSAutoreleasePool(ObjCInterfaceDecl *IDecl) {
-    return IDecl && IDecl->getIdentifier() == PoolII;
-  }
-
-  bool isNSAutoreleasePool(QualType Ty) {
-    QualType pointee = Ty->getPointeeType();
-    if (pointee.isNull())
-      return false;
-    if (const ObjCInterfaceType *interT = pointee->getAs<ObjCInterfaceType>())
-      return isNSAutoreleasePool(interT->getDecl());
-    return false;
-  }
-
-  static Expr *getEssential(Expr *E) {
-    return cast<Expr>(getEssential((Stmt*)E));
-  }
-  static Stmt *getEssential(Stmt *S) {
-    if (FullExpr *FE = dyn_cast<FullExpr>(S))
-      S = FE->getSubExpr();
-    if (Expr *E = dyn_cast<Expr>(S))
-      S = E->IgnoreParenCasts();
-    return S;
-  }
-
-  Stmt *Body;
-  MigrationPass &Pass;
-
-  IdentifierInfo *PoolII;
-  Selector DrainSel;
-
-  struct PoolVarInfo {
-    DeclStmt *Dcl = nullptr;
-    ExprSet Refs;
-    SmallVector<PoolScope, 2> Scopes;
-
-    PoolVarInfo() = default;
-  };
-
-  std::map<VarDecl *, PoolVarInfo> PoolVars;
-};
-
-} // anonymous namespace
-
-void trans::rewriteAutoreleasePool(MigrationPass &pass) {
-  BodyTransform<AutoreleasePoolRewriter> trans(pass);
-  trans.TraverseDecl(pass.Ctx.getTranslationUnitDecl());
-}
diff --git a/clang/lib/ARCMigrate/TransBlockObjCVariable.cpp b/clang/lib/ARCMigrate/TransBlockObjCVariable.cpp
deleted file mode 100644
index 1e4db33135b6a..0000000000000
--- a/clang/lib/ARCMigrate/TransBlockObjCVariable.cpp
+++ /dev/null
@@ -1,146 +0,0 @@
-//===--- TransBlockObjCVariable.cpp - Transformations to ARC mode ---------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-//
-// rewriteBlockObjCVariable:
-//
-// Adding __block to an obj-c variable could be either because the variable
-// is used for output storage or the user wanted to break a retain cycle.
-// This transformation checks whether a reference of the variable for the block
-// is actually needed (it is assigned to or its address is taken) or not.
-// If the reference is not needed it will assume __block was added to break a
-// cycle so it will remove '__block' and add __weak/__unsafe_unretained.
-// e.g
-//
-//   __block Foo *x;
-//   bar(^ { [x cake]; });
-// ---->
-//   __weak Foo *x;
-//   bar(^ { [x cake]; });
-//
-//===----------------------------------------------------------------------===//
-
-#include "Transforms.h"
-#include "Internals.h"
-#include "clang/AST/ASTContext.h"
-#include "clang/AST/Attr.h"
-#include "clang/Basic/SourceManager.h"
-
-using namespace clang;
-using namespace arcmt;
-using namespace trans;
-
-namespace {
-
-class RootBlockObjCVarRewriter :
-                          public RecursiveASTVisitor<RootBlockObjCVarRewriter> {
-  llvm::DenseSet<VarDecl *> &VarsToChange;
-
-  class BlockVarChecker : public RecursiveASTVisitor<BlockVarChecker> {
-    VarDecl *Var;
-
-    typedef RecursiveASTVisitor<BlockVarChecker> base;
-  public:
-    BlockVarChecker(VarDecl *var) : Var(var) { }
-
-    bool TraverseImplicitCastExpr(ImplicitCastExpr *castE) {
-      if (DeclRefExpr *
-            ref = dyn_cast<DeclRefExpr>(castE->getSubExpr())) {
-        if (ref->getDecl() == Var) {
-          if (castE->getCastKind() == CK_LValueToRValue)
-            return true; // Using the value of the variable.
-          if (castE->getCastKind() == CK_NoOp && castE->isLValue() &&
-              Var->getASTContext().getLangOpts().CPlusPlus)
-            return true; // Binding to const C++ reference.
-        }
-      }
-
-      return base::TraverseImplicitCastExpr(castE);
-    }
-
-    bool VisitDeclRefExpr(DeclRefExpr *E) {
-      if (E->getDecl() == Var)
-        return false; // The reference of the variable, and not just its value,
-                      //  is needed.
-      return true;
-    }
-  };
-
-public:
-  RootBlockObjCVarRewriter(llvm::DenseSet<VarDecl *> &VarsToChange)
-    : VarsToChange(VarsToChange) { }
-
-  bool VisitBlockDecl(BlockDecl *block) {
-    SmallVector<VarDecl *, 4> BlockVars;
-
-    for (const auto &I : block->captures()) {
-      VarDecl *var = I.getVariable();
-      if (I.isByRef() &&
-          var->getType()->isObjCObjectPointerType() &&
-          isImplicitStrong(var->getType())) {
-        BlockVars.push_back(var);
-      }
-    }
-
-    for (unsigned i = 0, e = BlockVars.size(); i != e; ++i) {
-      VarDecl *var = BlockVars[i];
-
-      BlockVarChecker checker(var);
-      bool onlyValueOfVarIsNeeded = checker.TraverseStmt(block->getBody());
-      if (onlyValueOfVarIsNeeded)
-        VarsToChange.insert(var);
-      else
-        VarsToChange.erase(var);
-    }
-
-    return true;
-  }
-
-private:
-  bool isImplicitStrong(QualType ty) {
-    if (isa<AttributedType>(ty.getTypePtr()))
-      return false;
-    return ty.getLocalQualifiers().getObjCLifetime() == Qualifiers::OCL_Strong;
-  }
-};
-
-class BlockObjCVarRewriter : public RecursiveASTVisitor<BlockObjCVarRewriter> {
-  llvm::DenseSet<VarDecl *> &VarsToChange;
-
-public:
-  BlockObjCVarRewriter(llvm::DenseSet<VarDecl *> &VarsToChange)
-    : VarsToChange(VarsToChange) { }
-
-  bool TraverseBlockDecl(BlockDecl *block) {
-    RootBlockObjCVarRewriter(VarsToChange).TraverseDecl(block);
-    return true;
-  }
-};
-
-} // anonymous namespace
-
-void BlockObjCVariableTraverser::traverseBody(BodyContext &BodyCtx) {
-  MigrationPass &Pass = BodyCtx.getMigrationContext().Pass;
-  llvm::DenseSet<VarDecl *> VarsToChange;
-
-  BlockObjCVarRewriter trans(VarsToChange);
-  trans.TraverseStmt(BodyCtx.getTopStmt());
-
-  for (llvm::DenseSet<VarDecl *>::iterator
-         I = VarsToChange.begin(), E = VarsToChange.end(); I != E; ++I) {
-    VarDecl *var = *I;
-    BlocksAttr *attr = var->getAttr<BlocksAttr>();
-    if(!attr)
-      continue;
-    bool useWeak = canApplyWeak(Pass.Ctx, var->getType());
-    SourceManager &SM = Pass.Ctx.getSourceManager();
-    Transaction Trans(Pass.TA);
-    Pass.TA.replaceText(SM.getExpansionLoc(attr->getLocation()),
-                        "__block",
-                        useWeak ? "__weak" : "__unsafe_unretained");
-  }
-}
diff --git a/clang/lib/ARCMigrate/TransEmptyStatementsAndDealloc.cpp b/clang/lib/ARCMigrate/TransEmptyStatementsAndDealloc.cpp
deleted file mode 100644
index e9c21b8106d7d..0000000000000
--- a/clang/lib/ARCMigrate/TransEmptyStatementsAndDealloc.cpp
+++ /dev/null
@@ -1,249 +0,0 @@
-//===-- TransEmptyStatementsAndDealloc.cpp - Transformations to ARC mode --===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-//
-// removeEmptyStatementsAndDealloc:
-//
-// Removes empty statements that are leftovers from previous transformations.
-// e.g for
-//
-//  [x retain];
-//
-// removeRetainReleaseDealloc will leave an empty ";" that removeEmptyStatements
-// will remove.
-//
-//===----------------------------------------------------------------------===//
-
-#include "Transforms.h"
-#include "Internals.h"
-#include "clang/AST/ASTContext.h"
-#include "clang/AST/StmtVisitor.h"
-#include "clang/Basic/SourceManager.h"
-
-using namespace clang;
-using namespace arcmt;
-using namespace trans;
-
-static bool isEmptyARCMTMacroStatement(NullStmt *S,
-                                       std::vector<SourceLocation> &MacroLocs,
-                                       ASTContext &Ctx) {
-  if (!S->hasLeadingEmptyMacro())
-    return false;
-
-  SourceLocation SemiLoc = S->getSemiLoc();
-  if (SemiLoc.isInvalid() || SemiLoc.isMacroID())
-    return false;
-
-  if (MacroLocs.empty())
-    return false;
-
-  SourceManager &SM = Ctx.getSourceManager();
-  std::vector<SourceLocation>::iterator I = llvm::upper_bound(
-      MacroLocs, SemiLoc, BeforeThanCompare<SourceLocation>(SM));
-  --I;
-  SourceLocation
-      AfterMacroLoc = I->getLocWithOffset(getARCMTMacroName().size());
-  assert(AfterMacroLoc.isFileID());
-
-  if (AfterMacroLoc == SemiLoc)
-    return true;
-
-  SourceLocation::IntTy RelOffs = 0;
-  if (!SM.isInSameSLocAddrSpace(AfterMacroLoc, SemiLoc, &RelOffs))
-    return false;
-  if (RelOffs < 0)
-    return false;
-
-  // We make the reasonable assumption that a semicolon after 100 characters
-  // means that it is not the next token after our macro. If this assumption
-  // fails it is not critical, we will just fail to clear out, e.g., an empty
-  // 'if'.
-  if (RelOffs - getARCMTMacroName().size() > 100)
-    return false;
-
-  SourceLocation AfterMacroSemiLoc = findSemiAfterLocation(AfterMacroLoc, Ctx);
-  return AfterMacroSemiLoc == SemiLoc;
-}
-
-namespace {
-
-/// Returns true if the statement became empty due to previous
-/// transformations.
-class EmptyChecker : public StmtVisitor<EmptyChecker, bool> {
-  ASTContext &Ctx;
-  std::vector<SourceLocation> &MacroLocs;
-
-public:
-  EmptyChecker(ASTContext &ctx, std::vector<SourceLocation> &macroLocs)
-    : Ctx(ctx), MacroLocs(macroLocs) { }
-
-  bool VisitNullStmt(NullStmt *S) {
-    return isEmptyARCMTMacroStatement(S, MacroLocs, Ctx);
-  }
-  bool VisitCompoundStmt(CompoundStmt *S) {
-    if (S->body_empty())
-      return false; // was already empty, not because of transformations.
-    for (auto *I : S->body())
-      if (!Visit(I))
-        return false;
-    return true;
-  }
-  bool VisitIfStmt(IfStmt *S) {
-    if (S->getConditionVariable())
-      return false;
-    Expr *condE = S->getCond();
-    if (!condE)
-      return false;
-    if (hasSideEffects(condE, Ctx))
-      return false;
-    if (!S->getThen() || !Visit(S->getThen()))
-      return false;
-    return !S->getElse() || Visit(S->getElse());
-  }
-  bool VisitWhileStmt(WhileStmt *S) {
-    if (S->getConditionVariable())
-      return false;
-    Expr *condE = S->getCond();
-    if (!condE)
-      return false;
-    if (hasSideEffects(condE, Ctx))
-      return false;
-    if (!S->getBody())
-      return false;
-    return Visit(S->getBody());
-  }
-  bool VisitDoStmt(DoStmt *S) {
-    Expr *condE = S->getCond();
-    if (!condE)
-      return false;
-    if (hasSideEffects(condE, Ctx))
-      return false;
-    if (!S->getBody())
-      return false;
-    return Visit(S->getBody());
-  }
-  bool VisitObjCForCollectionStmt(ObjCForCollectionStmt *S) {
-    Expr *Exp = S->getCollection();
-    if (!Exp)
-      return false;
-    if (hasSideEffects(Exp, Ctx))
-      return false;
-    if (!S->getBody())
-      return false;
-    return Visit(S->getBody());
-  }
-  bool VisitObjCAutoreleasePoolStmt(ObjCAutoreleasePoolStmt *S) {
-    if (!S->getSubStmt())
-      return false;
-    return Visit(S->getSubStmt());
-  }
-};
-
-class EmptyStatementsRemover :
-                            public RecursiveASTVisitor<EmptyStatementsRemover> {
-  MigrationPass &Pass;
-
-public:
-  EmptyStatementsRemover(MigrationPass &pass) : Pass(pass) { }
-
-  bool TraverseStmtExpr(StmtExpr *E) {
-    CompoundStmt *S = E->getSubStmt();
-    for (CompoundStmt::body_iterator
-           I = S->body_begin(), E = S->body_end(); I != E; ++I) {
-      if (I != E - 1)
-        check(*I);
-      TraverseStmt(*I);
-    }
-    return true;
-  }
-
-  bool VisitCompoundStmt(CompoundStmt *S) {
-    for (auto *I : S->body())
-      check(I);
-    return true;
-  }
-
-  ASTContext &getContext() { return Pass.Ctx; }
-
-private:
-  void check(Stmt *S) {
-    if (!S) return;
-    if (EmptyChecker(Pass.Ctx, Pass.ARCMTMacroLocs).Visit(S)) {
-      Transaction Trans(Pass.TA);
-      Pass.TA.removeStmt(S);
-    }
-  }
-};
-
-} // anonymous namespace
-
-static bool isBodyEmpty(CompoundStmt *body, ASTContext &Ctx,
-                        std::vector<SourceLocation> &MacroLocs) {
-  for (auto *I : body->body())
-    if (!EmptyChecker(Ctx, MacroLocs).Visit(I))
-      return false;
-
-  return true;
-}
-
-static void cleanupDeallocOrFinalize(MigrationPass &pass) {
-  ASTContext &Ctx = pass.Ctx;
-  TransformActions &TA = pass.TA;
-  DeclContext *DC = Ctx.getTranslationUnitDecl();
-  Selector FinalizeSel =
-      Ctx.Selectors.getNullarySelector(&pass.Ctx.Idents.get("finalize"));
-
-  typedef DeclContext::specific_decl_iterator<ObjCImplementationDecl>
-    impl_iterator;
-  for (impl_iterator I = impl_iterator(DC->decls_begin()),
-                     E = impl_iterator(DC->decls_end()); I != E; ++I) {
-    ObjCMethodDecl *DeallocM = nullptr;
-    ObjCMethodDecl *FinalizeM = nullptr;
-    for (auto *MD : I->instance_methods()) {
-      if (!MD->hasBody())
-        continue;
-
-      if (MD->getMethodFamily() == OMF_dealloc) {
-        DeallocM = MD;
-      } else if (MD->isInstanceMethod() && MD->getSelector() == FinalizeSel) {
-        FinalizeM = MD;
-      }
-    }
-
-    if (DeallocM) {
-      if (isBodyEmpty(DeallocM->getCompoundBody(), Ctx, pass.ARCMTMacroLocs)) {
-        Transaction Trans(TA);
-        TA.remove(DeallocM->getSourceRange());
-      }
-
-      if (FinalizeM) {
-        Transaction Trans(TA);
-        TA.remove(FinalizeM->getSourceRange());
-      }
-
-    } else if (FinalizeM) {
-      if (isBodyEmpty(FinalizeM->getCompoundBody(), Ctx, pass.ARCMTMacroLocs)) {
-        Transaction Trans(TA);
-        TA.remove(FinalizeM->getSourceRange());
-      } else {
-        Transaction Trans(TA);
-        TA.replaceText(FinalizeM->getSelectorStartLoc(), "finalize", "dealloc");
-      }
-    }
-  }
-}
-
-void trans::removeEmptyStatementsAndDeallocFinalize(MigrationPass &pass) {
-  EmptyStatementsRemover(pass).TraverseDecl(pass.Ctx.getTranslationUnitDecl());
-
-  cleanupDeallocOrFinalize(pass);
-
-  for (unsigned i = 0, e = pass.ARCMTMacroLocs.size(); i != e; ++i) {
-    Transaction Trans(pass.TA);
-    pass.TA.remove(pass.ARCMTMacroLocs[i]);
-  }
-}
diff --git a/clang/lib/ARCMigrate/TransGCAttrs.cpp b/clang/lib/ARCMigrate/TransGCAttrs.cpp
deleted file mode 100644
index 85e3fe77660b5..0000000000000
--- a/clang/lib/ARCMigrate/TransGCAttrs.cpp
+++ /dev/null
@@ -1,350 +0,0 @@
-//===--- TransGCAttrs.cpp - Transformations to ARC mode -------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "Transforms.h"
-#include "Internals.h"
-#include "clang/AST/ASTContext.h"
-#include "clang/Basic/SourceManager.h"
-#include "clang/Lex/Lexer.h"
-#include "clang/Sema/SemaDiagnostic.h"
-#include "llvm/ADT/SmallString.h"
-#include "llvm/ADT/TinyPtrVector.h"
-#include "llvm/Support/SaveAndRestore.h"
-
-using namespace clang;
-using namespace arcmt;
-using namespace trans;
-
-namespace {
-
-/// Collects all the places where GC attributes __strong/__weak occur.
-class GCAttrsCollector : public RecursiveASTVisitor<GCAttrsCollector> {
-  MigrationContext &MigrateCtx;
-  bool FullyMigratable;
-  std::vector<ObjCPropertyDecl *> &AllProps;
-
-  typedef RecursiveASTVisitor<GCAttrsCollector> base;
-public:
-  GCAttrsCollector(MigrationContext &ctx,
-                   std::vector<ObjCPropertyDecl *> &AllProps)
-    : MigrateCtx(ctx), FullyMigratable(false),
-      AllProps(AllProps) { }
-
-  bool shouldWalkTypesOfTypeLocs() const { return false; }
-
-  bool VisitAttributedTypeLoc(AttributedTypeLoc TL) {
-    handleAttr(TL);
-    return true;
-  }
-
-  bool TraverseDecl(Decl *D) {
-    if (!D || D->isImplicit())
-      return true;
-
-    SaveAndRestore Save(FullyMigratable, isMigratable(D));
-
-    if (ObjCPropertyDecl *PropD = dyn_cast<ObjCPropertyDecl>(D)) {
-      lookForAttribute(PropD, PropD->getTypeSourceInfo());
-      AllProps.push_back(PropD);
-    } else if (DeclaratorDecl *DD = dyn_cast<DeclaratorDecl>(D)) {
-      lookForAttribute(DD, DD->getTypeSourceInfo());
-    }
-    return base::TraverseDecl(D);
-  }
-
-  void lookForAttribute(Decl *D, TypeSourceInfo *TInfo) {
-    if (!TInfo)
-      return;
-    TypeLoc TL = TInfo->getTypeLoc();
-    while (TL) {
-      if (QualifiedTypeLoc QL = TL.getAs<QualifiedTypeLoc>()) {
-        TL = QL.getUnqualifiedLoc();
-      } else if (AttributedTypeLoc Attr = TL.getAs<AttributedTypeLoc>()) {
-        if (handleAttr(Attr, D))
-          break;
-        TL = Attr.getModifiedLoc();
-      } else if (MacroQualifiedTypeLoc MDTL =
-                     TL.getAs<MacroQualifiedTypeLoc>()) {
-        TL = MDTL.getInnerLoc();
-      } else if (ArrayTypeLoc Arr = TL.getAs<ArrayTypeLoc>()) {
-        TL = Arr.getElementLoc();
-      } else if (PointerTypeLoc PT = TL.getAs<PointerTypeLoc>()) {
-        TL = PT.getPointeeLoc();
-      } else if (ReferenceTypeLoc RT = TL.getAs<ReferenceTypeLoc>())
-        TL = RT.getPointeeLoc();
-      else
-        break;
-    }
-  }
-
-  bool handleAttr(AttributedTypeLoc TL, Decl *D = nullptr) {
-    auto *OwnershipAttr = TL.getAttrAs<ObjCOwnershipAttr>();
-    if (!OwnershipAttr)
-      return false;
-
-    SourceLocation Loc = OwnershipAttr->getLocation();
-    SourceLocation OrigLoc = Loc;
-    if (MigrateCtx.AttrSet.count(OrigLoc))
-      return true;
-
-    ASTContext &Ctx = MigrateCtx.Pass.Ctx;
-    SourceManager &SM = Ctx.getSourceManager();
-    if (Loc.isMacroID())
-      Loc = SM.getImmediateExpansionRange(Loc).getBegin();
-    StringRef Spell = OwnershipAttr->getKind()->getName();
-    MigrationContext::GCAttrOccurrence::AttrKind Kind;
-    if (Spell == "strong")
-      Kind = MigrationContext::GCAttrOccurrence::Strong;
-    else if (Spell == "weak")
-      Kind = MigrationContext::GCAttrOccurrence::Weak;
-    else
-      return false;
-
-    MigrateCtx.AttrSet.insert(OrigLoc);
-    MigrateCtx.GCAttrs.push_back(MigrationContext::GCAttrOccurrence());
-    MigrationContext::GCAttrOccurrence &Attr = MigrateCtx.GCAttrs.back();
-
-    Attr.Kind = Kind;
-    Attr.Loc = Loc;
-    Attr.ModifiedType = TL.getModifiedLoc().getType();
-    Attr.Dcl = D;
-    Attr.FullyMigratable = FullyMigratable;
-    return true;
-  }
-
-  bool isMigratable(Decl *D) {
-    if (isa<TranslationUnitDecl>(D))
-      return false;
-
-    if (isInMainFile(D))
-      return true;
-
-    if (FunctionDecl *FD = dyn_cast<FunctionDecl>(D))
-      return FD->hasBody();
-
-    if (ObjCContainerDecl *ContD = dyn_cast<ObjCContainerDecl>(D))
-      return hasObjCImpl(ContD);
-
-    if (CXXRecordDecl *RD = dyn_cast<CXXRecordDecl>(D)) {
-      for (const auto *MI : RD->methods()) {
-        if (MI->isOutOfLine())
-          return true;
-      }
-      return false;
-    }
-
-    return isMigratable(cast<Decl>(D->getDeclContext()));
-  }
-
-  static bool hasObjCImpl(Decl *D) {
-    if (!D)
-      return false;
-    if (ObjCContainerDecl *ContD = dyn_cast<ObjCContainerDecl>(D)) {
-      if (ObjCInterfaceDecl *ID = dyn_cast<ObjCInterfaceDecl>(ContD))
-        return ID->getImplementation() != nullptr;
-      if (ObjCCategoryDecl *CD = dyn_cast<ObjCCategoryDecl>(ContD))
-        return CD->getImplementation() != nullptr;
-      return isa<ObjCImplDecl>(ContD);
-    }
-    return false;
-  }
-
-  bool isInMainFile(Decl *D) {
-    if (!D)
-      return false;
-
-    for (auto *I : D->redecls())
-      if (!isInMainFile(I->getLocation()))
-        return false;
-
-    return true;
-  }
-
-  bool isInMainFile(SourceLocation Loc) {
-    if (Loc.isInvalid())
-      return false;
-
-    SourceManager &SM = MigrateCtx.Pass.Ctx.getSourceManager();
-    return SM.isInFileID(SM.getExpansionLoc(Loc), SM.getMainFileID());
-  }
-};
-
-} // anonymous namespace
-
-static void errorForGCAttrsOnNonObjC(MigrationContext &MigrateCtx) {
-  TransformActions &TA = MigrateCtx.Pass.TA;
-
-  for (unsigned i = 0, e = MigrateCtx.GCAttrs.size(); i != e; ++i) {
-    MigrationContext::GCAttrOccurrence &Attr = MigrateCtx.GCAttrs[i];
-    if (Attr.FullyMigratable && Attr.Dcl) {
-      if (Attr.ModifiedType.isNull())
-        continue;
-      if (!Attr.ModifiedType->isObjCRetainableType()) {
-        TA.reportError("GC managed memory will become unmanaged in ARC",
-                       Attr.Loc);
-      }
-    }
-  }
-}
-
-static void checkWeakGCAttrs(MigrationContext &MigrateCtx) {
-  TransformActions &TA = MigrateCtx.Pass.TA;
-
-  for (unsigned i = 0, e = MigrateCtx.GCAttrs.size(); i != e; ++i) {
-    MigrationContext::GCAttrOccurrence &Attr = MigrateCtx.GCAttrs[i];
-    if (Attr.Kind == MigrationContext::GCAttrOccurrence::Weak) {
-      if (Attr.ModifiedType.isNull() ||
-          !Attr.ModifiedType->isObjCRetainableType())
-        continue;
-      if (!canApplyWeak(MigrateCtx.Pass.Ctx, Attr.ModifiedType,
-                        /*AllowOnUnknownClass=*/true)) {
-        Transaction Trans(TA);
-        if (!MigrateCtx.RemovedAttrSet.count(Attr.Loc))
-          TA.replaceText(Attr.Loc, "__weak", "__unsafe_unretained");
-        TA.clearDiagnostic(diag::err_arc_weak_no_runtime,
-                           diag::err_arc_unsupported_weak_class,
-                           Attr.Loc);
-      }
-    }
-  }
-}
-
-typedef llvm::TinyPtrVector<ObjCPropertyDecl *> IndivPropsTy;
-
-static void checkAllAtProps(MigrationContext &MigrateCtx,
-                            SourceLocation AtLoc,
-                            IndivPropsTy &IndProps) {
-  if (IndProps.empty())
-    return;
-
-  for (IndivPropsTy::iterator
-         PI = IndProps.begin(), PE = IndProps.end(); PI != PE; ++PI) {
-    QualType T = (*PI)->getType();
-    if (T.isNull() || !T->isObjCRetainableType())
-      return;
-  }
-
-  SmallVector<std::pair<AttributedTypeLoc, ObjCPropertyDecl *>, 4> ATLs;
-  bool hasWeak = false, hasStrong = false;
-  ObjCPropertyAttribute::Kind Attrs = ObjCPropertyAttribute::kind_noattr;
-  for (IndivPropsTy::iterator
-         PI = IndProps.begin(), PE = IndProps.end(); PI != PE; ++PI) {
-    ObjCPropertyDecl *PD = *PI;
-    Attrs = PD->getPropertyAttributesAsWritten();
-    TypeSourceInfo *TInfo = PD->getTypeSourceInfo();
-    if (!TInfo)
-      return;
-    TypeLoc TL = TInfo->getTypeLoc();
-    if (AttributedTypeLoc ATL =
-            TL.getAs<AttributedTypeLoc>()) {
-      ATLs.push_back(std::make_pair(ATL, PD));
-      if (TInfo->getType().getObjCLifetime() == Qualifiers::OCL_Weak) {
-        hasWeak = true;
-      } else if (TInfo->getType().getObjCLifetime() == Qualifiers::OCL_Strong)
-        hasStrong = true;
-      else
-        return;
-    }
-  }
-  if (ATLs.empty())
-    return;
-  if (hasWeak && hasStrong)
-    return;
-
-  TransformActions &TA = MigrateCtx.Pass.TA;
-  Transaction Trans(TA);
-
-  if (GCAttrsCollector::hasObjCImpl(
-                              cast<Decl>(IndProps.front()->getDeclContext()))) {
-    if (hasWeak)
-      MigrateCtx.AtPropsWeak.insert(AtLoc);
-
-  } else {
-    StringRef toAttr = "strong";
-    if (hasWeak) {
-      if (canApplyWeak(MigrateCtx.Pass.Ctx, IndProps.front()->getType(),
-                       /*AllowOnUnknownClass=*/true))
-        toAttr = "weak";
-      else
-        toAttr = "unsafe_unretained";
-    }
-    if (Attrs & ObjCPropertyAttribute::kind_assign)
-      MigrateCtx.rewritePropertyAttribute("assign", toAttr, AtLoc);
-    else
-      MigrateCtx.addPropertyAttribute(toAttr, AtLoc);
-  }
-
-  for (unsigned i = 0, e = ATLs.size(); i != e; ++i) {
-    SourceLocation Loc = ATLs[i].first.getAttr()->getLocation();
-    if (Loc.isMacroID())
-      Loc = MigrateCtx.Pass.Ctx.getSourceManager()
-                .getImmediateExpansionRange(Loc)
-                .getBegin();
-    TA.remove(Loc);
-    TA.clearDiagnostic(diag::err_objc_property_attr_mutually_exclusive, AtLoc);
-    TA.clearDiagnostic(diag::err_arc_inconsistent_property_ownership,
-                       ATLs[i].second->getLocation());
-    MigrateCtx.RemovedAttrSet.insert(Loc);
-  }
-}
-
-static void checkAllProps(MigrationContext &MigrateCtx,
-                          std::vector<ObjCPropertyDecl *> &AllProps) {
-  typedef llvm::TinyPtrVector<ObjCPropertyDecl *> IndivPropsTy;
-  llvm::DenseMap<SourceLocation, IndivPropsTy> AtProps;
-
-  for (unsigned i = 0, e = AllProps.size(); i != e; ++i) {
-    ObjCPropertyDecl *PD = AllProps[i];
-    if (PD->getPropertyAttributesAsWritten() &
-        (ObjCPropertyAttribute::kind_assign |
-         ObjCPropertyAttribute::kind_readonly)) {
-      SourceLocation AtLoc = PD->getAtLoc();
-      if (AtLoc.isInvalid())
-        continue;
-      AtProps[AtLoc].push_back(PD);
-    }
-  }
-
-  for (auto I = AtProps.begin(), E = AtProps.end(); I != E; ++I) {
-    SourceLocation AtLoc = I->first;
-    IndivPropsTy &IndProps = I->second;
-    checkAllAtProps(MigrateCtx, AtLoc, IndProps);
-  }
-}
-
-void GCAttrsTraverser::traverseTU(MigrationContext &MigrateCtx) {
-  std::vector<ObjCPropertyDecl *> AllProps;
-  GCAttrsCollector(MigrateCtx, AllProps).TraverseDecl(
-                                  MigrateCtx.Pass.Ctx.getTranslationUnitDecl());
-
-  errorForGCAttrsOnNonObjC(MigrateCtx);
-  checkAllProps(MigrateCtx, AllProps);
-  checkWeakGCAttrs(MigrateCtx);
-}
-
-void MigrationContext::dumpGCAttrs() {
-  llvm::errs() << "\n################\n";
-  for (unsigned i = 0, e = GCAttrs.size(); i != e; ++i) {
-    GCAttrOccurrence &Attr = GCAttrs[i];
-    llvm::errs() << "KIND: "
-        << (Attr.Kind == GCAttrOccurrence::Strong ? "strong" : "weak");
-    llvm::errs() << "\nLOC: ";
-    Attr.Loc.print(llvm::errs(), Pass.Ctx.getSourceManager());
-    llvm::errs() << "\nTYPE: ";
-    Attr.ModifiedType.dump();
-    if (Attr.Dcl) {
-      llvm::errs() << "DECL:\n";
-      Attr.Dcl->dump();
-    } else {
-      llvm::errs() << "DECL: NONE";
-    }
-    llvm::errs() << "\nMIGRATABLE: " << Attr.FullyMigratable;
-    llvm::errs() << "\n----------------\n";
-  }
-  llvm::errs() << "\n################\n";
-}
diff --git a/clang/lib/ARCMigrate/TransGCCalls.cpp b/clang/lib/ARCMigrate/TransGCCalls.cpp
deleted file mode 100644
index 43233e2d0b456..0000000000000
--- a/clang/lib/ARCMigrate/TransGCCalls.cpp
+++ /dev/null
@@ -1,76 +0,0 @@
-//===--- TransGCCalls.cpp - Transformations to ARC mode -------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "Transforms.h"
-#include "Internals.h"
-#include "clang/AST/ASTContext.h"
-#include "clang/Sema/SemaDiagnostic.h"
-
-using namespace clang;
-using namespace arcmt;
-using namespace trans;
-
-namespace {
-
-class GCCollectableCallsChecker :
-                         public RecursiveASTVisitor<GCCollectableCallsChecker> {
-  MigrationContext &MigrateCtx;
-  IdentifierInfo *NSMakeCollectableII;
-  IdentifierInfo *CFMakeCollectableII;
-
-public:
-  GCCollectableCallsChecker(MigrationContext &ctx)
-    : MigrateCtx(ctx) {
-    IdentifierTable &Ids = MigrateCtx.Pass.Ctx.Idents;
-    NSMakeCollectableII = &Ids.get("NSMakeCollectable");
-    CFMakeCollectableII = &Ids.get("CFMakeCollectable");
-  }
-
-  bool shouldWalkTypesOfTypeLocs() const { return false; }
-
-  bool VisitCallExpr(CallExpr *E) {
-    TransformActions &TA = MigrateCtx.Pass.TA;
-
-    if (MigrateCtx.isGCOwnedNonObjC(E->getType())) {
-      TA.report(E->getBeginLoc(), diag::warn_arcmt_nsalloc_realloc,
-                E->getSourceRange());
-      return true;
-    }
-
-    Expr *CEE = E->getCallee()->IgnoreParenImpCasts();
-    if (DeclRefExpr *DRE = dyn_cast<DeclRefExpr>(CEE)) {
-      if (FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(DRE->getDecl())) {
-        if (!FD->getDeclContext()->getRedeclContext()->isFileContext())
-          return true;
-
-        if (FD->getIdentifier() == NSMakeCollectableII) {
-          Transaction Trans(TA);
-          TA.clearDiagnostic(diag::err_unavailable,
-                             diag::err_unavailable_message,
-                             diag::err_ovl_deleted_call, // ObjC++
-                             DRE->getSourceRange());
-          TA.replace(DRE->getSourceRange(), "CFBridgingRelease");
-
-        } else if (FD->getIdentifier() == CFMakeCollectableII) {
-          TA.reportError("CFMakeCollectable will leak the object that it "
-                         "receives in ARC", DRE->getLocation(),
-                         DRE->getSourceRange());
-        }
-      }
-    }
-
-    return true;
-  }
-};
-
-} // anonymous namespace
-
-void GCCollectableCallsTraverser::traverseBody(BodyContext &BodyCtx) {
-  GCCollectableCallsChecker(BodyCtx.getMigrationContext())
-                                            .TraverseStmt(BodyCtx.getTopStmt());
-}
diff --git a/clang/lib/ARCMigrate/TransProperties.cpp b/clang/lib/ARCMigrate/TransProperties.cpp
deleted file mode 100644
index 6d1d950821a07..0000000000000
--- a/clang/lib/ARCMigrate/TransProperties.cpp
+++ /dev/null
@@ -1,379 +0,0 @@
-//===--- TransProperties.cpp - Transformations to ARC mode ----------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-//
-// rewriteProperties:
-//
-// - Adds strong/weak/unsafe_unretained ownership specifier to properties that
-//   are missing one.
-// - Migrates properties from (retain) to (strong) and (assign) to
-//   (unsafe_unretained/weak).
-// - If a property is synthesized, adds the ownership specifier in the ivar
-//   backing the property.
-//
-//  @interface Foo : NSObject {
-//      NSObject *x;
-//  }
-//  @property (assign) id x;
-//  @end
-// ---->
-//  @interface Foo : NSObject {
-//      NSObject *__weak x;
-//  }
-//  @property (weak) id x;
-//  @end
-//
-//===----------------------------------------------------------------------===//
-
-#include "Transforms.h"
-#include "Internals.h"
-#include "clang/Basic/SourceManager.h"
-#include "clang/Lex/Lexer.h"
-#include "clang/Sema/SemaDiagnostic.h"
-#include <map>
-
-using namespace clang;
-using namespace arcmt;
-using namespace trans;
-
-namespace {
-
-class PropertiesRewriter {
-  MigrationContext &MigrateCtx;
-  MigrationPass &Pass;
-  ObjCImplementationDecl *CurImplD = nullptr;
-
-  enum PropActionKind {
-    PropAction_None,
-    PropAction_RetainReplacedWithStrong,
-    PropAction_AssignRemoved,
-    PropAction_AssignRewritten,
-    PropAction_MaybeAddWeakOrUnsafe
-  };
-
-  struct PropData {
-    ObjCPropertyDecl *PropD;
-    ObjCIvarDecl *IvarD;
-    ObjCPropertyImplDecl *ImplD;
-
-    PropData(ObjCPropertyDecl *propD)
-      : PropD(propD), IvarD(nullptr), ImplD(nullptr) {}
-  };
-
-  typedef SmallVector<PropData, 2> PropsTy;
-  typedef std::map<SourceLocation, PropsTy> AtPropDeclsTy;
-  AtPropDeclsTy AtProps;
-  llvm::DenseMap<IdentifierInfo *, PropActionKind> ActionOnProp;
-
-public:
-  explicit PropertiesRewriter(MigrationContext &MigrateCtx)
-    : MigrateCtx(MigrateCtx), Pass(MigrateCtx.Pass) { }
-
-  static void collectProperties(ObjCContainerDecl *D, AtPropDeclsTy &AtProps,
-                                AtPropDeclsTy *PrevAtProps = nullptr) {
-    for (auto *Prop : D->instance_properties()) {
-      SourceLocation Loc = Prop->getAtLoc();
-      if (Loc.isInvalid())
-        continue;
-      if (PrevAtProps)
-        if (PrevAtProps->find(Loc) != PrevAtProps->end())
-          continue;
-      PropsTy &props = AtProps[Loc];
-      props.push_back(Prop);
-    }
-  }
-
-  void doTransform(ObjCImplementationDecl *D) {
-    CurImplD = D;
-    ObjCInterfaceDecl *iface = D->getClassInterface();
-    if (!iface)
-      return;
-
-    collectProperties(iface, AtProps);
-
-    // Look through extensions.
-    for (auto *Ext : iface->visible_extensions())
-      collectProperties(Ext, AtProps);
-
-    typedef DeclContext::specific_decl_iterator<ObjCPropertyImplDecl>
-        prop_impl_iterator;
-    for (prop_impl_iterator
-           I = prop_impl_iterator(D->decls_begin()),
-           E = prop_impl_iterator(D->decls_end()); I != E; ++I) {
-      ObjCPropertyImplDecl *implD = *I;
-      if (implD->getPropertyImplementation() != ObjCPropertyImplDecl::Synthesize)
-        continue;
-      ObjCPropertyDecl *propD = implD->getPropertyDecl();
-      if (!propD || propD->isInvalidDecl())
-        continue;
-      ObjCIvarDecl *ivarD = implD->getPropertyIvarDecl();
-      if (!ivarD || ivarD->isInvalidDecl())
-        continue;
-      AtPropDeclsTy::iterator findAtLoc = AtProps.find(propD->getAtLoc());
-      if (findAtLoc == AtProps.end())
-        continue;
-
-      PropsTy &props = findAtLoc->second;
-      for (PropsTy::iterator I = props.begin(), E = props.end(); I != E; ++I) {
-        if (I->PropD == propD) {
-          I->IvarD = ivarD;
-          I->ImplD = implD;
-          break;
-        }
-      }
-    }
-
-    for (AtPropDeclsTy::iterator
-           I = AtProps.begin(), E = AtProps.end(); I != E; ++I) {
-      SourceLocation atLoc = I->first;
-      PropsTy &props = I->second;
-      if (!getPropertyType(props)->isObjCRetainableType())
-        continue;
-      if (hasIvarWithExplicitARCOwnership(props))
-        continue;
-
-      Transaction Trans(Pass.TA);
-      rewriteProperty(props, atLoc);
-    }
-  }
-
-private:
-  void doPropAction(PropActionKind kind,
-                    PropsTy &props, SourceLocation atLoc,
-                    bool markAction = true) {
-    if (markAction)
-      for (PropsTy::iterator I = props.begin(), E = props.end(); I != E; ++I)
-        ActionOnProp[I->PropD->getIdentifier()] = kind;
-
-    switch (kind) {
-    case PropAction_None:
-      return;
-    case PropAction_RetainReplacedWithStrong: {
-      StringRef toAttr = "strong";
-      MigrateCtx.rewritePropertyAttribute("retain", toAttr, atLoc);
-      return;
-    }
-    case PropAction_AssignRemoved:
-      return removeAssignForDefaultStrong(props, atLoc);
-    case PropAction_AssignRewritten:
-      return rewriteAssign(props, atLoc);
-    case PropAction_MaybeAddWeakOrUnsafe:
-      return maybeAddWeakOrUnsafeUnretainedAttr(props, atLoc);
-    }
-  }
-
-  void rewriteProperty(PropsTy &props, SourceLocation atLoc) {
-    ObjCPropertyAttribute::Kind propAttrs = getPropertyAttrs(props);
-
-    if (propAttrs &
-        (ObjCPropertyAttribute::kind_copy |
-         ObjCPropertyAttribute::kind_unsafe_unretained |
-         ObjCPropertyAttribute::kind_strong | ObjCPropertyAttribute::kind_weak))
-      return;
-
-    if (propAttrs & ObjCPropertyAttribute::kind_retain) {
-      // strong is the default.
-      return doPropAction(PropAction_RetainReplacedWithStrong, props, atLoc);
-    }
-
-    bool HasIvarAssignedAPlusOneObject = hasIvarAssignedAPlusOneObject(props);
-
-    if (propAttrs & ObjCPropertyAttribute::kind_assign) {
-      if (HasIvarAssignedAPlusOneObject)
-        return doPropAction(PropAction_AssignRemoved, props, atLoc);
-      return doPropAction(PropAction_AssignRewritten, props, atLoc);
-    }
-
-    if (HasIvarAssignedAPlusOneObject ||
-        (Pass.isGCMigration() && !hasGCWeak(props, atLoc)))
-      return; // 'strong' by default.
-
-    return doPropAction(PropAction_MaybeAddWeakOrUnsafe, props, atLoc);
-  }
-
-  void removeAssignForDefaultStrong(PropsTy &props,
-                                    SourceLocation atLoc) const {
-    removeAttribute("retain", atLoc);
-    if (!removeAttribute("assign", atLoc))
-      return;
-
-    for (PropsTy::iterator I = props.begin(), E = props.end(); I != E; ++I) {
-      if (I->ImplD)
-        Pass.TA.clearDiagnostic(diag::err_arc_strong_property_ownership,
-                                diag::err_arc_assign_property_ownership,
-                                diag::err_arc_inconsistent_property_ownership,
-                                I->IvarD->getLocation());
-    }
-  }
-
-  void rewriteAssign(PropsTy &props, SourceLocation atLoc) const {
-    bool canUseWeak = canApplyWeak(Pass.Ctx, getPropertyType(props),
-                                  /*AllowOnUnknownClass=*/Pass.isGCMigration());
-    const char *toWhich =
-      (Pass.isGCMigration() && !hasGCWeak(props, atLoc)) ? "strong" :
-      (canUseWeak ? "weak" : "unsafe_unretained");
-
-    bool rewroteAttr = rewriteAttribute("assign", toWhich, atLoc);
-    if (!rewroteAttr)
-      canUseWeak = false;
-
-    for (PropsTy::iterator I = props.begin(), E = props.end(); I != E; ++I) {
-      if (isUserDeclared(I->IvarD)) {
-        if (I->IvarD &&
-            I->IvarD->getType().getObjCLifetime() != Qualifiers::OCL_Weak) {
-          const char *toWhich =
-            (Pass.isGCMigration() && !hasGCWeak(props, atLoc)) ? "__strong " :
-              (canUseWeak ? "__weak " : "__unsafe_unretained ");
-          Pass.TA.insert(I->IvarD->getLocation(), toWhich);
-        }
-      }
-      if (I->ImplD)
-        Pass.TA.clearDiagnostic(diag::err_arc_strong_property_ownership,
-                                diag::err_arc_assign_property_ownership,
-                                diag::err_arc_inconsistent_property_ownership,
-                                I->IvarD->getLocation());
-    }
-  }
-
-  void maybeAddWeakOrUnsafeUnretainedAttr(PropsTy &props,
-                                          SourceLocation atLoc) const {
-    bool canUseWeak = canApplyWeak(Pass.Ctx, getPropertyType(props),
-                                  /*AllowOnUnknownClass=*/Pass.isGCMigration());
-
-    bool addedAttr = addAttribute(canUseWeak ? "weak" : "unsafe_unretained",
-                                  atLoc);
-    if (!addedAttr)
-      canUseWeak = false;
-
-    for (PropsTy::iterator I = props.begin(), E = props.end(); I != E; ++I) {
-      if (isUserDeclared(I->IvarD)) {
-        if (I->IvarD &&
-            I->IvarD->getType().getObjCLifetime() != Qualifiers::OCL_Weak)
-          Pass.TA.insert(I->IvarD->getLocation(),
-                         canUseWeak ? "__weak " : "__unsafe_unretained ");
-      }
-      if (I->ImplD) {
-        Pass.TA.clearDiagnostic(diag::err_arc_strong_property_ownership,
-                                diag::err_arc_assign_property_ownership,
-                                diag::err_arc_inconsistent_property_ownership,
-                                I->IvarD->getLocation());
-        Pass.TA.clearDiagnostic(
-                           diag::err_arc_objc_property_default_assign_on_object,
-                           I->ImplD->getLocation());
-      }
-    }
-  }
-
-  bool removeAttribute(StringRef fromAttr, SourceLocation atLoc) const {
-    return MigrateCtx.removePropertyAttribute(fromAttr, atLoc);
-  }
-
-  bool rewriteAttribute(StringRef fromAttr, StringRef toAttr,
-                        SourceLocation atLoc) const {
-    return MigrateCtx.rewritePropertyAttribute(fromAttr, toAttr, atLoc);
-  }
-
-  bool addAttribute(StringRef attr, SourceLocation atLoc) const {
-    return MigrateCtx.addPropertyAttribute(attr, atLoc);
-  }
-
-  class PlusOneAssign : public RecursiveASTVisitor<PlusOneAssign> {
-    ObjCIvarDecl *Ivar;
-  public:
-    PlusOneAssign(ObjCIvarDecl *D) : Ivar(D) {}
-
-    bool VisitBinaryOperator(BinaryOperator *E) {
-      if (E->getOpcode() != BO_Assign)
-        return true;
-
-      Expr *lhs = E->getLHS()->IgnoreParenImpCasts();
-      if (ObjCIvarRefExpr *RE = dyn_cast<ObjCIvarRefExpr>(lhs)) {
-        if (RE->getDecl() != Ivar)
-          return true;
-
-        if (isPlusOneAssign(E))
-          return false;
-      }
-
-      return true;
-    }
-  };
-
-  bool hasIvarAssignedAPlusOneObject(PropsTy &props) const {
-    for (PropsTy::iterator I = props.begin(), E = props.end(); I != E; ++I) {
-      PlusOneAssign oneAssign(I->IvarD);
-      bool notFound = oneAssign.TraverseDecl(CurImplD);
-      if (!notFound)
-        return true;
-    }
-
-    return false;
-  }
-
-  bool hasIvarWithExplicitARCOwnership(PropsTy &props) const {
-    if (Pass.isGCMigration())
-      return false;
-
-    for (PropsTy::iterator I = props.begin(), E = props.end(); I != E; ++I) {
-      if (isUserDeclared(I->IvarD)) {
-        if (isa<AttributedType>(I->IvarD->getType()))
-          return true;
-        if (I->IvarD->getType().getLocalQualifiers().getObjCLifetime()
-              != Qualifiers::OCL_Strong)
-          return true;
-      }
-    }
-
-    return false;
-  }
-
-  // Returns true if all declarations in the @property have GC __weak.
-  bool hasGCWeak(PropsTy &props, SourceLocation atLoc) const {
-    if (!Pass.isGCMigration())
-      return false;
-    if (props.empty())
-      return false;
-    return MigrateCtx.AtPropsWeak.count(atLoc);
-  }
-
-  bool isUserDeclared(ObjCIvarDecl *ivarD) const {
-    return ivarD && !ivarD->getSynthesize();
-  }
-
-  QualType getPropertyType(PropsTy &props) const {
-    assert(!props.empty());
-    QualType ty = props[0].PropD->getType().getUnqualifiedType();
-
-#ifndef NDEBUG
-    for (PropsTy::iterator I = props.begin(), E = props.end(); I != E; ++I)
-      assert(ty == I->PropD->getType().getUnqualifiedType());
-#endif
-
-    return ty;
-  }
-
-  ObjCPropertyAttribute::Kind getPropertyAttrs(PropsTy &props) const {
-    assert(!props.empty());
-    ObjCPropertyAttribute::Kind attrs =
-        props[0].PropD->getPropertyAttributesAsWritten();
-
-#ifndef NDEBUG
-    for (PropsTy::iterator I = props.begin(), E = props.end(); I != E; ++I)
-      assert(attrs == I->PropD->getPropertyAttributesAsWritten());
-#endif
-
-    return attrs;
-  }
-};
-
-} // anonymous namespace
-
-void PropertyRewriteTraverser::traverseObjCImplementation(
-                                           ObjCImplementationContext &ImplCtx) {
-  PropertiesRewriter(ImplCtx.getMigrationContext())
-                                  .doTransform(ImplCtx.getImplementationDecl());
-}
diff --git a/clang/lib/ARCMigrate/TransProtectedScope.cpp b/clang/lib/ARCMigrate/TransProtectedScope.cpp
deleted file mode 100644
index 154e0b54800f9..0000000000000
--- a/clang/lib/ARCMigrate/TransProtectedScope.cpp
+++ /dev/null
@@ -1,203 +0,0 @@
-//===--- TransProtectedScope.cpp - Transformations to ARC mode ------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-//
-// Adds brackets in case statements that "contain" initialization of retaining
-// variable, thus emitting the "switch case is in protected scope" error.
-//
-//===----------------------------------------------------------------------===//
-
-#include "Internals.h"
-#include "Transforms.h"
-#include "clang/AST/ASTContext.h"
-#include "clang/Basic/SourceManager.h"
-#include "clang/Sema/SemaDiagnostic.h"
-
-using namespace clang;
-using namespace arcmt;
-using namespace trans;
-
-namespace {
-
-class LocalRefsCollector : public RecursiveASTVisitor<LocalRefsCollector> {
-  SmallVectorImpl<DeclRefExpr *> &Refs;
-
-public:
-  LocalRefsCollector(SmallVectorImpl<DeclRefExpr *> &refs)
-    : Refs(refs) { }
-
-  bool VisitDeclRefExpr(DeclRefExpr *E) {
-    if (ValueDecl *D = E->getDecl())
-      if (D->getDeclContext()->getRedeclContext()->isFunctionOrMethod())
-        Refs.push_back(E);
-    return true;
-  }
-};
-
-struct CaseInfo {
-  SwitchCase *SC;
-  SourceRange Range;
-  enum {
-    St_Unchecked,
-    St_CannotFix,
-    St_Fixed
-  } State;
-
-  CaseInfo() : SC(nullptr), State(St_Unchecked) {}
-  CaseInfo(SwitchCase *S, SourceRange Range)
-    : SC(S), Range(Range), State(St_Unchecked) {}
-};
-
-class CaseCollector : public RecursiveASTVisitor<CaseCollector> {
-  ParentMap &PMap;
-  SmallVectorImpl<CaseInfo> &Cases;
-
-public:
-  CaseCollector(ParentMap &PMap, SmallVectorImpl<CaseInfo> &Cases)
-    : PMap(PMap), Cases(Cases) { }
-
-  bool VisitSwitchStmt(SwitchStmt *S) {
-    SwitchCase *Curr = S->getSwitchCaseList();
-    if (!Curr)
-      return true;
-    Stmt *Parent = getCaseParent(Curr);
-    Curr = Curr->getNextSwitchCase();
-    // Make sure all case statements are in the same scope.
-    while (Curr) {
-      if (getCaseParent(Curr) != Parent)
-        return true;
-      Curr = Curr->getNextSwitchCase();
-    }
-
-    SourceLocation NextLoc = S->getEndLoc();
-    Curr = S->getSwitchCaseList();
-    // We iterate over case statements in reverse source-order.
-    while (Curr) {
-      Cases.push_back(
-          CaseInfo(Curr, SourceRange(Curr->getBeginLoc(), NextLoc)));
-      NextLoc = Curr->getBeginLoc();
-      Curr = Curr->getNextSwitchCase();
-    }
-    return true;
-  }
-
-  Stmt *getCaseParent(SwitchCase *S) {
-    Stmt *Parent = PMap.getParent(S);
-    while (Parent && (isa<SwitchCase>(Parent) || isa<LabelStmt>(Parent)))
-      Parent = PMap.getParent(Parent);
-    return Parent;
-  }
-};
-
-class ProtectedScopeFixer {
-  MigrationPass &Pass;
-  SourceManager &SM;
-  SmallVector<CaseInfo, 16> Cases;
-  SmallVector<DeclRefExpr *, 16> LocalRefs;
-
-public:
-  ProtectedScopeFixer(BodyContext &BodyCtx)
-    : Pass(BodyCtx.getMigrationContext().Pass),
-      SM(Pass.Ctx.getSourceManager()) {
-
-    CaseCollector(BodyCtx.getParentMap(), Cases)
-        .TraverseStmt(BodyCtx.getTopStmt());
-    LocalRefsCollector(LocalRefs).TraverseStmt(BodyCtx.getTopStmt());
-
-    SourceRange BodyRange = BodyCtx.getTopStmt()->getSourceRange();
-    const CapturedDiagList &DiagList = Pass.getDiags();
-    // Copy the diagnostics so we don't have to worry about invaliding iterators
-    // from the diagnostic list.
-    SmallVector<StoredDiagnostic, 16> StoredDiags;
-    StoredDiags.append(DiagList.begin(), DiagList.end());
-    SmallVectorImpl<StoredDiagnostic>::iterator
-        I = StoredDiags.begin(), E = StoredDiags.end();
-    while (I != E) {
-      if (I->getID() == diag::err_switch_into_protected_scope &&
-          isInRange(I->getLocation(), BodyRange)) {
-        handleProtectedScopeError(I, E);
-        continue;
-      }
-      ++I;
-    }
-  }
-
-  void handleProtectedScopeError(
-                             SmallVectorImpl<StoredDiagnostic>::iterator &DiagI,
-                             SmallVectorImpl<StoredDiagnostic>::iterator DiagE){
-    Transaction Trans(Pass.TA);
-    assert(DiagI->getID() == diag::err_switch_into_protected_scope);
-    SourceLocation ErrLoc = DiagI->getLocation();
-    bool handledAllNotes = true;
-    ++DiagI;
-    for (; DiagI != DiagE && DiagI->getLevel() == DiagnosticsEngine::Note;
-         ++DiagI) {
-      if (!handleProtectedNote(*DiagI))
-        handledAllNotes = false;
-    }
-
-    if (handledAllNotes)
-      Pass.TA.clearDiagnostic(diag::err_switch_into_protected_scope, ErrLoc);
-  }
-
-  bool handleProtectedNote(const StoredDiagnostic &Diag) {
-    assert(Diag.getLevel() == DiagnosticsEngine::Note);
-
-    for (unsigned i = 0; i != Cases.size(); i++) {
-      CaseInfo &info = Cases[i];
-      if (isInRange(Diag.getLocation(), info.Range)) {
-
-        if (info.State == CaseInfo::St_Unchecked)
-          tryFixing(info);
-        assert(info.State != CaseInfo::St_Unchecked);
-
-        if (info.State == CaseInfo::St_Fixed) {
-          Pass.TA.clearDiagnostic(Diag.getID(), Diag.getLocation());
-          return true;
-        }
-        return false;
-      }
-    }
-
-    return false;
-  }
-
-  void tryFixing(CaseInfo &info) {
-    assert(info.State == CaseInfo::St_Unchecked);
-    if (hasVarReferencedOutside(info)) {
-      info.State = CaseInfo::St_CannotFix;
-      return;
-    }
-
-    Pass.TA.insertAfterToken(info.SC->getColonLoc(), " {");
-    Pass.TA.insert(info.Range.getEnd(), "}\n");
-    info.State = CaseInfo::St_Fixed;
-  }
-
-  bool hasVarReferencedOutside(CaseInfo &info) {
-    for (unsigned i = 0, e = LocalRefs.size(); i != e; ++i) {
-      DeclRefExpr *DRE = LocalRefs[i];
-      if (isInRange(DRE->getDecl()->getLocation(), info.Range) &&
-          !isInRange(DRE->getLocation(), info.Range))
-        return true;
-    }
-    return false;
-  }
-
-  bool isInRange(SourceLocation Loc, SourceRange R) {
-    if (Loc.isInvalid())
-      return false;
-    return !SM.isBeforeInTranslationUnit(Loc, R.getBegin()) &&
-            SM.isBeforeInTranslationUnit(Loc, R.getEnd());
-  }
-};
-
-} // anonymous namespace
-
-void ProtectedScopeTraverser::traverseBody(BodyContext &BodyCtx) {
-  ProtectedScopeFixer Fix(BodyCtx);
-}
diff --git a/clang/lib/ARCMigrate/TransRetainReleaseDealloc.cpp b/clang/lib/ARCMigrate/TransRetainReleaseDealloc.cpp
deleted file mode 100644
index baa503d8a39dc..0000000000000
--- a/clang/lib/ARCMigrate/TransRetainReleaseDealloc.cpp
+++ /dev/null
@@ -1,459 +0,0 @@
-//===--- TransRetainReleaseDealloc.cpp - Transformations to ARC mode ------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-//
-// removeRetainReleaseDealloc:
-//
-// Removes retain/release/autorelease/dealloc messages.
-//
-//  return [[foo retain] autorelease];
-// ---->
-//  return foo;
-//
-//===----------------------------------------------------------------------===//
-
-#include "Transforms.h"
-#include "Internals.h"
-#include "clang/AST/ASTContext.h"
-#include "clang/AST/ParentMap.h"
-#include "clang/Basic/SourceManager.h"
-#include "clang/Lex/Lexer.h"
-#include "clang/Sema/SemaDiagnostic.h"
-#include "llvm/ADT/StringSwitch.h"
-
-using namespace clang;
-using namespace arcmt;
-using namespace trans;
-
-namespace {
-
-class RetainReleaseDeallocRemover :
-                       public RecursiveASTVisitor<RetainReleaseDeallocRemover> {
-  Stmt *Body;
-  MigrationPass &Pass;
-
-  ExprSet Removables;
-  std::unique_ptr<ParentMap> StmtMap;
-
-  Selector DelegateSel, FinalizeSel;
-
-public:
-  RetainReleaseDeallocRemover(MigrationPass &pass)
-    : Body(nullptr), Pass(pass) {
-    DelegateSel =
-        Pass.Ctx.Selectors.getNullarySelector(&Pass.Ctx.Idents.get("delegate"));
-    FinalizeSel =
-        Pass.Ctx.Selectors.getNullarySelector(&Pass.Ctx.Idents.get("finalize"));
-  }
-
-  void transformBody(Stmt *body, Decl *ParentD) {
-    Body = body;
-    collectRemovables(body, Removables);
-    StmtMap.reset(new ParentMap(body));
-    TraverseStmt(body);
-  }
-
-  bool VisitObjCMessageExpr(ObjCMessageExpr *E) {
-    switch (E->getMethodFamily()) {
-    default:
-      if (E->isInstanceMessage() && E->getSelector() == FinalizeSel)
-        break;
-      return true;
-    case OMF_autorelease:
-      if (isRemovable(E)) {
-        if (!isCommonUnusedAutorelease(E)) {
-          // An unused autorelease is badness. If we remove it the receiver
-          // will likely die immediately while previously it was kept alive
-          // by the autorelease pool. This is bad practice in general, leave it
-          // and emit an error to force the user to restructure their code.
-          Pass.TA.reportError(
-              "it is not safe to remove an unused 'autorelease' "
-              "message; its receiver may be destroyed immediately",
-              E->getBeginLoc(), E->getSourceRange());
-          return true;
-        }
-      }
-      // Pass through.
-      [[fallthrough]];
-    case OMF_retain:
-    case OMF_release:
-      if (E->getReceiverKind() == ObjCMessageExpr::Instance)
-        if (Expr *rec = E->getInstanceReceiver()) {
-          rec = rec->IgnoreParenImpCasts();
-          if (rec->getType().getObjCLifetime() == Qualifiers::OCL_ExplicitNone &&
-              (E->getMethodFamily() != OMF_retain || isRemovable(E))) {
-            std::string err = "it is not safe to remove '";
-            err += E->getSelector().getAsString() + "' message on "
-                "an __unsafe_unretained type";
-            Pass.TA.reportError(err, rec->getBeginLoc());
-            return true;
-          }
-
-          if (isGlobalVar(rec) &&
-              (E->getMethodFamily() != OMF_retain || isRemovable(E))) {
-            std::string err = "it is not safe to remove '";
-            err += E->getSelector().getAsString() + "' message on "
-                "a global variable";
-            Pass.TA.reportError(err, rec->getBeginLoc());
-            return true;
-          }
-
-          if (E->getMethodFamily() == OMF_release && isDelegateMessage(rec)) {
-            Pass.TA.reportError(
-                "it is not safe to remove 'retain' "
-                "message on the result of a 'delegate' message; "
-                "the object that was passed to 'setDelegate:' may not be "
-                "properly retained",
-                rec->getBeginLoc());
-            return true;
-          }
-        }
-      break;
-    case OMF_dealloc:
-      break;
-    }
-
-    switch (E->getReceiverKind()) {
-    default:
-      return true;
-    case ObjCMessageExpr::SuperInstance: {
-      Transaction Trans(Pass.TA);
-      clearDiagnostics(E->getSelectorLoc(0));
-      if (tryRemoving(E))
-        return true;
-      Pass.TA.replace(E->getSourceRange(), "self");
-      return true;
-    }
-    case ObjCMessageExpr::Instance:
-      break;
-    }
-
-    Expr *rec = E->getInstanceReceiver();
-    if (!rec) return true;
-
-    Transaction Trans(Pass.TA);
-    clearDiagnostics(E->getSelectorLoc(0));
-
-    ObjCMessageExpr *Msg = E;
-    Expr *RecContainer = Msg;
-    SourceRange RecRange = rec->getSourceRange();
-    checkForGCDOrXPC(Msg, RecContainer, rec, RecRange);
-
-    if (Msg->getMethodFamily() == OMF_release &&
-        isRemovable(RecContainer) && isInAtFinally(RecContainer)) {
-      // Change the -release to "receiver = nil" in a finally to avoid a leak
-      // when an exception is thrown.
-      Pass.TA.replace(RecContainer->getSourceRange(), RecRange);
-      std::string str = " = ";
-      str += getNilString(Pass);
-      Pass.TA.insertAfterToken(RecRange.getEnd(), str);
-      return true;
-    }
-
-    if (hasSideEffects(rec, Pass.Ctx) || !tryRemoving(RecContainer))
-      Pass.TA.replace(RecContainer->getSourceRange(), RecRange);
-
-    return true;
-  }
-
-private:
-  /// Checks for idioms where an unused -autorelease is common.
-  ///
-  /// Returns true for this idiom which is common in property
-  /// setters:
-  ///
-  ///   [backingValue autorelease];
-  ///   backingValue = [newValue retain]; // in general a +1 assign
-  ///
-  /// For these as well:
-  ///
-  ///   [[var retain] autorelease];
-  ///   return var;
-  ///
-  bool isCommonUnusedAutorelease(ObjCMessageExpr *E) {
-    return isPlusOneAssignBeforeOrAfterAutorelease(E) ||
-           isReturnedAfterAutorelease(E);
-  }
-
-  bool isReturnedAfterAutorelease(ObjCMessageExpr *E) {
-    Expr *Rec = E->getInstanceReceiver();
-    if (!Rec)
-      return false;
-
-    Decl *RefD = getReferencedDecl(Rec);
-    if (!RefD)
-      return false;
-
-    Stmt *nextStmt = getNextStmt(E);
-    if (!nextStmt)
-      return false;
-
-    // Check for "return <variable>;".
-
-    if (ReturnStmt *RetS = dyn_cast<ReturnStmt>(nextStmt))
-      return RefD == getReferencedDecl(RetS->getRetValue());
-
-    return false;
-  }
-
-  bool isPlusOneAssignBeforeOrAfterAutorelease(ObjCMessageExpr *E) {
-    Expr *Rec = E->getInstanceReceiver();
-    if (!Rec)
-      return false;
-
-    Decl *RefD = getReferencedDecl(Rec);
-    if (!RefD)
-      return false;
-
-    Stmt *prevStmt, *nextStmt;
-    std::tie(prevStmt, nextStmt) = getPreviousAndNextStmt(E);
-
-    return isPlusOneAssignToVar(prevStmt, RefD) ||
-           isPlusOneAssignToVar(nextStmt, RefD);
-  }
-
-  bool isPlusOneAssignToVar(Stmt *S, Decl *RefD) {
-    if (!S)
-      return false;
-
-    // Check for "RefD = [+1 retained object];".
-
-    if (BinaryOperator *Bop = dyn_cast<BinaryOperator>(S)) {
-      return (RefD == getReferencedDecl(Bop->getLHS())) && isPlusOneAssign(Bop);
-    }
-
-    if (DeclStmt *DS = dyn_cast<DeclStmt>(S)) {
-      if (DS->isSingleDecl() && DS->getSingleDecl() == RefD) {
-        if (VarDecl *VD = dyn_cast<VarDecl>(RefD))
-          return isPlusOne(VD->getInit());
-      }
-      return false;
-    }
-
-    return false;
-  }
-
-  Stmt *getNextStmt(Expr *E) {
-    return getPreviousAndNextStmt(E).second;
-  }
-
-  std::pair<Stmt *, Stmt *> getPreviousAndNextStmt(Expr *E) {
-    Stmt *prevStmt = nullptr, *nextStmt = nullptr;
-    if (!E)
-      return std::make_pair(prevStmt, nextStmt);
-
-    Stmt *OuterS = E, *InnerS;
-    do {
-      InnerS = OuterS;
-      OuterS = StmtMap->getParent(InnerS);
-    }
-    while (OuterS && (isa<ParenExpr>(OuterS) ||
-                      isa<CastExpr>(OuterS) ||
-                      isa<FullExpr>(OuterS)));
-
-    if (!OuterS)
-      return std::make_pair(prevStmt, nextStmt);
-
-    Stmt::child_iterator currChildS = OuterS->child_begin();
-    Stmt::child_iterator childE = OuterS->child_end();
-    Stmt::child_iterator prevChildS = childE;
-    for (; currChildS != childE; ++currChildS) {
-      if (*currChildS == InnerS)
-        break;
-      prevChildS = currChildS;
-    }
-
-    if (prevChildS != childE) {
-      prevStmt = *prevChildS;
-      if (auto *E = dyn_cast_or_null<Expr>(prevStmt))
-        prevStmt = E->IgnoreImplicit();
-    }
-
-    if (currChildS == childE)
-      return std::make_pair(prevStmt, nextStmt);
-    ++currChildS;
-    if (currChildS == childE)
-      return std::make_pair(prevStmt, nextStmt);
-
-    nextStmt = *currChildS;
-    if (auto *E = dyn_cast_or_null<Expr>(nextStmt))
-      nextStmt = E->IgnoreImplicit();
-
-    return std::make_pair(prevStmt, nextStmt);
-  }
-
-  Decl *getReferencedDecl(Expr *E) {
-    if (!E)
-      return nullptr;
-
-    E = E->IgnoreParenCasts();
-    if (ObjCMessageExpr *ME = dyn_cast<ObjCMessageExpr>(E)) {
-      switch (ME->getMethodFamily()) {
-      case OMF_copy:
-      case OMF_autorelease:
-      case OMF_release:
-      case OMF_retain:
-        return getReferencedDecl(ME->getInstanceReceiver());
-      default:
-        return nullptr;
-      }
-    }
-    if (DeclRefExpr *DRE = dyn_cast<DeclRefExpr>(E))
-      return DRE->getDecl();
-    if (MemberExpr *ME = dyn_cast<MemberExpr>(E))
-      return ME->getMemberDecl();
-    if (ObjCIvarRefExpr *IRE = dyn_cast<ObjCIvarRefExpr>(E))
-      return IRE->getDecl();
-
-    return nullptr;
-  }
-
-  /// Check if the retain/release is due to a GCD/XPC macro that are
-  /// defined as:
-  ///
-  /// #define dispatch_retain(object) ({ dispatch_object_t _o = (object); _dispatch_object_validate(_o); (void)[_o retain]; })
-  /// #define dispatch_release(object) ({ dispatch_object_t _o = (object); _dispatch_object_validate(_o); [_o release]; })
-  /// #define xpc_retain(object) ({ xpc_object_t _o = (object); _xpc_object_validate(_o); [_o retain]; })
-  /// #define xpc_release(object) ({ xpc_object_t _o = (object); _xpc_object_validate(_o); [_o release]; })
-  ///
-  /// and return the top container which is the StmtExpr and the macro argument
-  /// expression.
-  void checkForGCDOrXPC(ObjCMessageExpr *Msg, Expr *&RecContainer,
-                        Expr *&Rec, SourceRange &RecRange) {
-    SourceLocation Loc = Msg->getExprLoc();
-    if (!Loc.isMacroID())
-      return;
-    SourceManager &SM = Pass.Ctx.getSourceManager();
-    StringRef MacroName = Lexer::getImmediateMacroName(Loc, SM,
-                                                     Pass.Ctx.getLangOpts());
-    bool isGCDOrXPC = llvm::StringSwitch<bool>(MacroName)
-        .Case("dispatch_retain", true)
-        .Case("dispatch_release", true)
-        .Case("xpc_retain", true)
-        .Case("xpc_release", true)
-        .Default(false);
-    if (!isGCDOrXPC)
-      return;
-
-    StmtExpr *StmtE = nullptr;
-    Stmt *S = Msg;
-    while (S) {
-      if (StmtExpr *SE = dyn_cast<StmtExpr>(S)) {
-        StmtE = SE;
-        break;
-      }
-      S = StmtMap->getParent(S);
-    }
-
-    if (!StmtE)
-      return;
-
-    Stmt::child_range StmtExprChild = StmtE->children();
-    if (StmtExprChild.begin() == StmtExprChild.end())
-      return;
-    auto *CompS = dyn_cast_or_null<CompoundStmt>(*StmtExprChild.begin());
-    if (!CompS)
-      return;
-
-    Stmt::child_range CompStmtChild = CompS->children();
-    if (CompStmtChild.begin() == CompStmtChild.end())
-      return;
-    auto *DeclS = dyn_cast_or_null<DeclStmt>(*CompStmtChild.begin());
-    if (!DeclS)
-      return;
-    if (!DeclS->isSingleDecl())
-      return;
-    VarDecl *VD = dyn_cast_or_null<VarDecl>(DeclS->getSingleDecl());
-    if (!VD)
-      return;
-    Expr *Init = VD->getInit();
-    if (!Init)
-      return;
-
-    RecContainer = StmtE;
-    Rec = Init->IgnoreParenImpCasts();
-    if (FullExpr *FE = dyn_cast<FullExpr>(Rec))
-      Rec = FE->getSubExpr()->IgnoreParenImpCasts();
-    RecRange = Rec->getSourceRange();
-    if (SM.isMacroArgExpansion(RecRange.getBegin()))
-      RecRange.setBegin(SM.getImmediateSpellingLoc(RecRange.getBegin()));
-    if (SM.isMacroArgExpansion(RecRange.getEnd()))
-      RecRange.setEnd(SM.getImmediateSpellingLoc(RecRange.getEnd()));
-  }
-
-  void clearDiagnostics(SourceLocation loc) const {
-    Pass.TA.clearDiagnostic(diag::err_arc_illegal_explicit_message,
-                            diag::err_unavailable,
-                            diag::err_unavailable_message,
-                            loc);
-  }
-
-  bool isDelegateMessage(Expr *E) const {
-    if (!E) return false;
-
-    E = E->IgnoreParenCasts();
-
-    // Also look through property-getter sugar.
-    if (PseudoObjectExpr *pseudoOp = dyn_cast<PseudoObjectExpr>(E))
-      E = pseudoOp->getResultExpr()->IgnoreImplicit();
-
-    if (ObjCMessageExpr *ME = dyn_cast<ObjCMessageExpr>(E))
-      return (ME->isInstanceMessage() && ME->getSelector() == DelegateSel);
-
-    return false;
-  }
-
-  bool isInAtFinally(Expr *E) const {
-    assert(E);
-    Stmt *S = E;
-    while (S) {
-      if (isa<ObjCAtFinallyStmt>(S))
-        return true;
-      S = StmtMap->getParent(S);
-    }
-
-    return false;
-  }
-
-  bool isRemovable(Expr *E) const {
-    return Removables.count(E);
-  }
-
-  bool tryRemoving(Expr *E) const {
-    if (isRemovable(E)) {
-      Pass.TA.removeStmt(E);
-      return true;
-    }
-
-    Stmt *parent = StmtMap->getParent(E);
-
-    if (ImplicitCastExpr *castE = dyn_cast_or_null<ImplicitCastExpr>(parent))
-      return tryRemoving(castE);
-
-    if (ParenExpr *parenE = dyn_cast_or_null<ParenExpr>(parent))
-      return tryRemoving(parenE);
-
-    if (BinaryOperator *
-          bopE = dyn_cast_or_null<BinaryOperator>(parent)) {
-      if (bopE->getOpcode() == BO_Comma && bopE->getLHS() == E &&
-          isRemovable(bopE)) {
-        Pass.TA.replace(bopE->getSourceRange(), bopE->getRHS()->getSourceRange());
-        return true;
-      }
-    }
-
-    return false;
-  }
-
-};
-
-} // anonymous namespace
-
-void trans::removeRetainReleaseDeallocFinalize(MigrationPass &pass) {
-  BodyTransform<RetainReleaseDeallocRemover> trans(pass);
-  trans.TraverseDecl(pass.Ctx.getTranslationUnitDecl());
-}
diff --git a/clang/lib/ARCMigrate/TransUnbridgedCasts.cpp b/clang/lib/ARCMigrate/TransUnbridgedCasts.cpp
deleted file mode 100644
index 7390ea17c8a4b..0000000000000
--- a/clang/lib/ARCMigrate/TransUnbridgedCasts.cpp
+++ /dev/null
@@ -1,466 +0,0 @@
-//===--- TransUnbridgedCasts.cpp - Transformations to ARC mode ------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-//
-// rewriteUnbridgedCasts:
-//
-// A cast of non-objc pointer to an objc one is checked. If the non-objc pointer
-// is from a file-level variable, __bridge cast is used to convert it.
-// For the result of a function call that we know is +1/+0,
-// __bridge/CFBridgingRelease is used.
-//
-//  NSString *str = (NSString *)kUTTypePlainText;
-//  str = b ? kUTTypeRTF : kUTTypePlainText;
-//  NSString *_uuidString = (NSString *)CFUUIDCreateString(kCFAllocatorDefault,
-//                                                         _uuid);
-// ---->
-//  NSString *str = (__bridge NSString *)kUTTypePlainText;
-//  str = (__bridge NSString *)(b ? kUTTypeRTF : kUTTypePlainText);
-// NSString *_uuidString = (NSString *)
-//            CFBridgingRelease(CFUUIDCreateString(kCFAllocatorDefault, _uuid));
-//
-// For a C pointer to ObjC, for casting 'self', __bridge is used.
-//
-//  CFStringRef str = (CFStringRef)self;
-// ---->
-//  CFStringRef str = (__bridge CFStringRef)self;
-//
-// Uses of Block_copy/Block_release macros are rewritten:
-//
-//  c = Block_copy(b);
-//  Block_release(c);
-// ---->
-//  c = [b copy];
-//  <removed>
-//
-//===----------------------------------------------------------------------===//
-
-#include "Transforms.h"
-#include "Internals.h"
-#include "clang/AST/ASTContext.h"
-#include "clang/AST/Attr.h"
-#include "clang/AST/ParentMap.h"
-#include "clang/Analysis/DomainSpecific/CocoaConventions.h"
-#include "clang/Basic/SourceManager.h"
-#include "clang/Lex/Lexer.h"
-#include "clang/Sema/SemaDiagnostic.h"
-#include "llvm/ADT/SmallString.h"
-
-using namespace clang;
-using namespace arcmt;
-using namespace trans;
-
-namespace {
-
-class UnbridgedCastRewriter : public RecursiveASTVisitor<UnbridgedCastRewriter>{
-  MigrationPass &Pass;
-  IdentifierInfo *SelfII;
-  std::unique_ptr<ParentMap> StmtMap;
-  Decl *ParentD;
-  Stmt *Body;
-  mutable std::unique_ptr<ExprSet> Removables;
-
-public:
-  UnbridgedCastRewriter(MigrationPass &pass)
-    : Pass(pass), ParentD(nullptr), Body(nullptr) {
-    SelfII = &Pass.Ctx.Idents.get("self");
-  }
-
-  void transformBody(Stmt *body, Decl *ParentD) {
-    this->ParentD = ParentD;
-    Body = body;
-    StmtMap.reset(new ParentMap(body));
-    TraverseStmt(body);
-  }
-
-  bool TraverseBlockDecl(BlockDecl *D) {
-    // ParentMap does not enter into a BlockDecl to record its stmts, so use a
-    // new UnbridgedCastRewriter to handle the block.
-    UnbridgedCastRewriter(Pass).transformBody(D->getBody(), D);
-    return true;
-  }
-
-  bool VisitCastExpr(CastExpr *E) {
-    if (E->getCastKind() != CK_CPointerToObjCPointerCast &&
-        E->getCastKind() != CK_BitCast &&
-        E->getCastKind() != CK_AnyPointerToBlockPointerCast)
-      return true;
-
-    QualType castType = E->getType();
-    Expr *castExpr = E->getSubExpr();
-    QualType castExprType = castExpr->getType();
-
-    if (castType->isObjCRetainableType() == castExprType->isObjCRetainableType())
-      return true;
-
-    bool exprRetainable = castExprType->isObjCIndirectLifetimeType();
-    bool castRetainable = castType->isObjCIndirectLifetimeType();
-    if (exprRetainable == castRetainable) return true;
-
-    if (castExpr->isNullPointerConstant(Pass.Ctx,
-                                        Expr::NPC_ValueDependentIsNull))
-      return true;
-
-    SourceLocation loc = castExpr->getExprLoc();
-    if (loc.isValid() && Pass.Ctx.getSourceManager().isInSystemHeader(loc))
-      return true;
-
-    if (castType->isObjCRetainableType())
-      transformNonObjCToObjCCast(E);
-    else
-      transformObjCToNonObjCCast(E);
-
-    return true;
-  }
-
-private:
-  void transformNonObjCToObjCCast(CastExpr *E) {
-    if (!E) return;
-
-    // Global vars are assumed that are cast as unretained.
-    if (isGlobalVar(E))
-      if (E->getSubExpr()->getType()->isPointerType()) {
-        castToObjCObject(E, /*retained=*/false);
-        return;
-      }
-
-    // If the cast is directly over the result of a Core Foundation function
-    // try to figure out whether it should be cast as retained or unretained.
-    Expr *inner = E->IgnoreParenCasts();
-    if (CallExpr *callE = dyn_cast<CallExpr>(inner)) {
-      if (FunctionDecl *FD = callE->getDirectCallee()) {
-        if (FD->hasAttr<CFReturnsRetainedAttr>()) {
-          castToObjCObject(E, /*retained=*/true);
-          return;
-        }
-        if (FD->hasAttr<CFReturnsNotRetainedAttr>()) {
-          castToObjCObject(E, /*retained=*/false);
-          return;
-        }
-        if (FD->isGlobal() &&
-            FD->getIdentifier() &&
-            ento::cocoa::isRefType(E->getSubExpr()->getType(), "CF",
-                                   FD->getIdentifier()->getName())) {
-          StringRef fname = FD->getIdentifier()->getName();
-          if (fname.ends_with("Retain") || fname.contains("Create") ||
-              fname.contains("Copy")) {
-            // Do not migrate to couple of bridge transfer casts which
-            // cancel each other out. Leave it unchanged so error gets user
-            // attention instead.
-            if (FD->getName() == "CFRetain" &&
-                FD->getNumParams() == 1 &&
-                FD->getParent()->isTranslationUnit() &&
-                FD->isExternallyVisible()) {
-              Expr *Arg = callE->getArg(0);
-              if (const ImplicitCastExpr *ICE = dyn_cast<ImplicitCastExpr>(Arg)) {
-                const Expr *sub = ICE->getSubExpr();
-                QualType T = sub->getType();
-                if (T->isObjCObjectPointerType())
-                  return;
-              }
-            }
-            castToObjCObject(E, /*retained=*/true);
-            return;
-          }
-
-          if (fname.contains("Get")) {
-            castToObjCObject(E, /*retained=*/false);
-            return;
-          }
-        }
-      }
-    }
-
-    // If returning an ivar or a member of an ivar from a +0 method, use
-    // a __bridge cast.
-    Expr *base = inner->IgnoreParenImpCasts();
-    while (isa<MemberExpr>(base))
-      base = cast<MemberExpr>(base)->getBase()->IgnoreParenImpCasts();
-    if (isa<ObjCIvarRefExpr>(base) &&
-        isa<ReturnStmt>(StmtMap->getParentIgnoreParenCasts(E))) {
-      if (ObjCMethodDecl *method = dyn_cast_or_null<ObjCMethodDecl>(ParentD)) {
-        if (!method->hasAttr<NSReturnsRetainedAttr>()) {
-          castToObjCObject(E, /*retained=*/false);
-          return;
-        }
-      }
-    }
-  }
-
-  void castToObjCObject(CastExpr *E, bool retained) {
-    rewriteToBridgedCast(E, retained ? OBC_BridgeTransfer : OBC_Bridge);
-  }
-
-  void rewriteToBridgedCast(CastExpr *E, ObjCBridgeCastKind Kind) {
-    Transaction Trans(Pass.TA);
-    rewriteToBridgedCast(E, Kind, Trans);
-  }
-
-  void rewriteToBridgedCast(CastExpr *E, ObjCBridgeCastKind Kind,
-                            Transaction &Trans) {
-    TransformActions &TA = Pass.TA;
-
-    // We will remove the compiler diagnostic.
-    if (!TA.hasDiagnostic(diag::err_arc_mismatched_cast,
-                          diag::err_arc_cast_requires_bridge,
-                          E->getBeginLoc())) {
-      Trans.abort();
-      return;
-    }
-
-    StringRef bridge;
-    switch(Kind) {
-    case OBC_Bridge:
-      bridge = "__bridge "; break;
-    case OBC_BridgeTransfer:
-      bridge = "__bridge_transfer "; break;
-    case OBC_BridgeRetained:
-      bridge = "__bridge_retained "; break;
-    }
-
-    TA.clearDiagnostic(diag::err_arc_mismatched_cast,
-                       diag::err_arc_cast_requires_bridge, E->getBeginLoc());
-    if (Kind == OBC_Bridge || !Pass.CFBridgingFunctionsDefined()) {
-      if (CStyleCastExpr *CCE = dyn_cast<CStyleCastExpr>(E)) {
-        TA.insertAfterToken(CCE->getLParenLoc(), bridge);
-      } else {
-        SourceLocation insertLoc = E->getSubExpr()->getBeginLoc();
-        SmallString<128> newCast;
-        newCast += '(';
-        newCast += bridge;
-        newCast += E->getType().getAsString(Pass.Ctx.getPrintingPolicy());
-        newCast += ')';
-
-        if (isa<ParenExpr>(E->getSubExpr())) {
-          TA.insert(insertLoc, newCast.str());
-        } else {
-          newCast += '(';
-          TA.insert(insertLoc, newCast.str());
-          TA.insertAfterToken(E->getEndLoc(), ")");
-        }
-      }
-    } else {
-      assert(Kind == OBC_BridgeTransfer || Kind == OBC_BridgeRetained);
-      SmallString<32> BridgeCall;
-
-      Expr *WrapE = E->getSubExpr();
-      SourceLocation InsertLoc = WrapE->getBeginLoc();
-
-      SourceManager &SM = Pass.Ctx.getSourceManager();
-      char PrevChar = *SM.getCharacterData(InsertLoc.getLocWithOffset(-1));
-      if (Lexer::isAsciiIdentifierContinueChar(PrevChar,
-                                               Pass.Ctx.getLangOpts()))
-        BridgeCall += ' ';
-
-      if (Kind == OBC_BridgeTransfer)
-        BridgeCall += "CFBridgingRelease";
-      else
-        BridgeCall += "CFBridgingRetain";
-
-      if (isa<ParenExpr>(WrapE)) {
-        TA.insert(InsertLoc, BridgeCall);
-      } else {
-        BridgeCall += '(';
-        TA.insert(InsertLoc, BridgeCall);
-        TA.insertAfterToken(WrapE->getEndLoc(), ")");
-      }
-    }
-  }
-
-  void rewriteCastForCFRetain(CastExpr *castE, CallExpr *callE) {
-    Transaction Trans(Pass.TA);
-    Pass.TA.replace(callE->getSourceRange(), callE->getArg(0)->getSourceRange());
-    rewriteToBridgedCast(castE, OBC_BridgeRetained, Trans);
-  }
-
-  void getBlockMacroRanges(CastExpr *E, SourceRange &Outer, SourceRange &Inner) {
-    SourceManager &SM = Pass.Ctx.getSourceManager();
-    SourceLocation Loc = E->getExprLoc();
-    assert(Loc.isMacroID());
-    CharSourceRange MacroRange = SM.getImmediateExpansionRange(Loc);
-    SourceRange SubRange = E->getSubExpr()->IgnoreParenImpCasts()->getSourceRange();
-    SourceLocation InnerBegin = SM.getImmediateMacroCallerLoc(SubRange.getBegin());
-    SourceLocation InnerEnd = SM.getImmediateMacroCallerLoc(SubRange.getEnd());
-
-    Outer = MacroRange.getAsRange();
-    Inner = SourceRange(InnerBegin, InnerEnd);
-  }
-
-  void rewriteBlockCopyMacro(CastExpr *E) {
-    SourceRange OuterRange, InnerRange;
-    getBlockMacroRanges(E, OuterRange, InnerRange);
-
-    Transaction Trans(Pass.TA);
-    Pass.TA.replace(OuterRange, InnerRange);
-    Pass.TA.insert(InnerRange.getBegin(), "[");
-    Pass.TA.insertAfterToken(InnerRange.getEnd(), " copy]");
-    Pass.TA.clearDiagnostic(diag::err_arc_mismatched_cast,
-                            diag::err_arc_cast_requires_bridge,
-                            OuterRange);
-  }
-
-  void removeBlockReleaseMacro(CastExpr *E) {
-    SourceRange OuterRange, InnerRange;
-    getBlockMacroRanges(E, OuterRange, InnerRange);
-
-    Transaction Trans(Pass.TA);
-    Pass.TA.clearDiagnostic(diag::err_arc_mismatched_cast,
-                            diag::err_arc_cast_requires_bridge,
-                            OuterRange);
-    if (!hasSideEffects(E, Pass.Ctx)) {
-      if (tryRemoving(cast<Expr>(StmtMap->getParentIgnoreParenCasts(E))))
-        return;
-    }
-    Pass.TA.replace(OuterRange, InnerRange);
-  }
-
-  bool tryRemoving(Expr *E) const {
-    if (!Removables) {
-      Removables.reset(new ExprSet);
-      collectRemovables(Body, *Removables);
-    }
-
-    if (Removables->count(E)) {
-      Pass.TA.removeStmt(E);
-      return true;
-    }
-
-    return false;
-  }
-
-  void transformObjCToNonObjCCast(CastExpr *E) {
-    SourceLocation CastLoc = E->getExprLoc();
-    if (CastLoc.isMacroID()) {
-      StringRef MacroName = Lexer::getImmediateMacroName(CastLoc,
-                                                    Pass.Ctx.getSourceManager(),
-                                                    Pass.Ctx.getLangOpts());
-      if (MacroName == "Block_copy") {
-        rewriteBlockCopyMacro(E);
-        return;
-      }
-      if (MacroName == "Block_release") {
-        removeBlockReleaseMacro(E);
-        return;
-      }
-    }
-
-    if (isSelf(E->getSubExpr()))
-      return rewriteToBridgedCast(E, OBC_Bridge);
-
-    CallExpr *callE;
-    if (isPassedToCFRetain(E, callE))
-      return rewriteCastForCFRetain(E, callE);
-
-    ObjCMethodFamily family = getFamilyOfMessage(E->getSubExpr());
-    if (family == OMF_retain)
-      return rewriteToBridgedCast(E, OBC_BridgeRetained);
-
-    if (family == OMF_autorelease || family == OMF_release) {
-      std::string err = "it is not safe to cast to '";
-      err += E->getType().getAsString(Pass.Ctx.getPrintingPolicy());
-      err += "' the result of '";
-      err += family == OMF_autorelease ? "autorelease" : "release";
-      err += "' message; a __bridge cast may result in a pointer to a "
-          "destroyed object and a __bridge_retained may leak the object";
-      Pass.TA.reportError(err, E->getBeginLoc(),
-                          E->getSubExpr()->getSourceRange());
-      Stmt *parent = E;
-      do {
-        parent = StmtMap->getParentIgnoreParenImpCasts(parent);
-      } while (isa_and_nonnull<FullExpr>(parent));
-
-      if (ReturnStmt *retS = dyn_cast_or_null<ReturnStmt>(parent)) {
-        std::string note = "remove the cast and change return type of function "
-            "to '";
-        note += E->getSubExpr()->getType().getAsString(Pass.Ctx.getPrintingPolicy());
-        note += "' to have the object automatically autoreleased";
-        Pass.TA.reportNote(note, retS->getBeginLoc());
-      }
-    }
-
-    Expr *subExpr = E->getSubExpr();
-
-    // Look through pseudo-object expressions.
-    if (PseudoObjectExpr *pseudo = dyn_cast<PseudoObjectExpr>(subExpr)) {
-      subExpr = pseudo->getResultExpr();
-      assert(subExpr && "no result for pseudo-object of non-void type?");
-    }
-
-    if (ImplicitCastExpr *implCE = dyn_cast<ImplicitCastExpr>(subExpr)) {
-      if (implCE->getCastKind() == CK_ARCConsumeObject)
-        return rewriteToBridgedCast(E, OBC_BridgeRetained);
-      if (implCE->getCastKind() == CK_ARCReclaimReturnedObject)
-        return rewriteToBridgedCast(E, OBC_Bridge);
-    }
-
-    bool isConsumed = false;
-    if (isPassedToCParamWithKnownOwnership(E, isConsumed))
-      return rewriteToBridgedCast(E, isConsumed ? OBC_BridgeRetained
-                                                : OBC_Bridge);
-  }
-
-  static ObjCMethodFamily getFamilyOfMessage(Expr *E) {
-    E = E->IgnoreParenCasts();
-    if (ObjCMessageExpr *ME = dyn_cast<ObjCMessageExpr>(E))
-      return ME->getMethodFamily();
-
-    return OMF_None;
-  }
-
-  bool isPassedToCFRetain(Expr *E, CallExpr *&callE) const {
-    if ((callE = dyn_cast_or_null<CallExpr>(
-                                     StmtMap->getParentIgnoreParenImpCasts(E))))
-      if (FunctionDecl *
-            FD = dyn_cast_or_null<FunctionDecl>(callE->getCalleeDecl()))
-        if (FD->getName() == "CFRetain" && FD->getNumParams() == 1 &&
-            FD->getParent()->isTranslationUnit() &&
-            FD->isExternallyVisible())
-          return true;
-
-    return false;
-  }
-
-  bool isPassedToCParamWithKnownOwnership(Expr *E, bool &isConsumed) const {
-    if (CallExpr *callE = dyn_cast_or_null<CallExpr>(
-                                     StmtMap->getParentIgnoreParenImpCasts(E)))
-      if (FunctionDecl *
-            FD = dyn_cast_or_null<FunctionDecl>(callE->getCalleeDecl())) {
-        unsigned i = 0;
-        for (unsigned e = callE->getNumArgs(); i != e; ++i) {
-          Expr *arg = callE->getArg(i);
-          if (arg == E || arg->IgnoreParenImpCasts() == E)
-            break;
-        }
-        if (i < callE->getNumArgs() && i < FD->getNumParams()) {
-          ParmVarDecl *PD = FD->getParamDecl(i);
-          if (PD->hasAttr<CFConsumedAttr>()) {
-            isConsumed = true;
-            return true;
-          }
-        }
-      }
-
-    return false;
-  }
-
-  bool isSelf(Expr *E) const {
-    E = E->IgnoreParenLValueCasts();
-    if (DeclRefExpr *DRE = dyn_cast<DeclRefExpr>(E))
-      if (ImplicitParamDecl *IPD = dyn_cast<ImplicitParamDecl>(DRE->getDecl()))
-        if (IPD->getIdentifier() == SelfII)
-          return true;
-
-    return false;
-  }
-};
-
-} // end anonymous namespace
-
-void trans::rewriteUnbridgedCasts(MigrationPass &pass) {
-  BodyTransform<UnbridgedCastRewriter> trans(pass);
-  trans.TraverseDecl(pass.Ctx.getTranslationUnitDecl());
-}
diff --git a/clang/lib/ARCMigrate/TransUnusedInitDelegate.cpp b/clang/lib/ARCMigrate/TransUnusedInitDelegate.cpp
deleted file mode 100644
index bac8dfac9b415..0000000000000
--- a/clang/lib/ARCMigrate/TransUnusedInitDelegate.cpp
+++ /dev/null
@@ -1,77 +0,0 @@
-//===--- TransUnusedInitDelegate.cpp - Transformations to ARC mode --------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-// Transformations:
-//===----------------------------------------------------------------------===//
-//
-// rewriteUnusedInitDelegate:
-//
-// Rewrites an unused result of calling a delegate initialization, to assigning
-// the result to self.
-// e.g
-//  [self init];
-// ---->
-//  self = [self init];
-//
-//===----------------------------------------------------------------------===//
-
-#include "Transforms.h"
-#include "Internals.h"
-#include "clang/AST/ASTContext.h"
-#include "clang/Sema/SemaDiagnostic.h"
-
-using namespace clang;
-using namespace arcmt;
-using namespace trans;
-
-namespace {
-
-class UnusedInitRewriter : public RecursiveASTVisitor<UnusedInitRewriter> {
-  Stmt *Body;
-  MigrationPass &Pass;
-
-  ExprSet Removables;
-
-public:
-  UnusedInitRewriter(MigrationPass &pass)
-    : Body(nullptr), Pass(pass) { }
-
-  void transformBody(Stmt *body, Decl *ParentD) {
-    Body = body;
-    collectRemovables(body, Removables);
-    TraverseStmt(body);
-  }
-
-  bool VisitObjCMessageExpr(ObjCMessageExpr *ME) {
-    if (ME->isDelegateInitCall() &&
-        isRemovable(ME) &&
-        Pass.TA.hasDiagnostic(diag::err_arc_unused_init_message,
-                              ME->getExprLoc())) {
-      Transaction Trans(Pass.TA);
-      Pass.TA.clearDiagnostic(diag::err_arc_unused_init_message,
-                              ME->getExprLoc());
-      SourceRange ExprRange = ME->getSourceRange();
-      Pass.TA.insert(ExprRange.getBegin(), "if (!(self = ");
-      std::string retStr = ")) return ";
-      retStr += getNilString(Pass);
-      Pass.TA.insertAfterToken(ExprRange.getEnd(), retStr);
-    }
-    return true;
-  }
-
-private:
-  bool isRemovable(Expr *E) const {
-    return Removables.count(E);
-  }
-};
-
-} // anonymous namespace
-
-void trans::rewriteUnusedInitDelegate(MigrationPass &pass) {
-  BodyTransform<UnusedInitRewriter> trans(pass);
-  trans.TraverseDecl(pass.Ctx.getTranslationUnitDecl());
-}
diff --git a/clang/lib/ARCMigrate/TransZeroOutPropsInDealloc.cpp b/clang/lib/ARCMigrate/TransZeroOutPropsInDealloc.cpp
deleted file mode 100644
index 81e67628fb1f4..0000000000000
--- a/clang/lib/ARCMigrate/TransZeroOutPropsInDealloc.cpp
+++ /dev/null
@@ -1,224 +0,0 @@
-//===--- TransZeroOutPropsInDealloc.cpp - Transformations to ARC mode -----===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-//
-// removeZeroOutPropsInDealloc:
-//
-// Removes zero'ing out "strong" @synthesized properties in a -dealloc method.
-//
-//===----------------------------------------------------------------------===//
-
-#include "Transforms.h"
-#include "Internals.h"
-#include "clang/AST/ASTContext.h"
-
-using namespace clang;
-using namespace arcmt;
-using namespace trans;
-
-namespace {
-
-class ZeroOutInDeallocRemover :
-                           public RecursiveASTVisitor<ZeroOutInDeallocRemover> {
-  typedef RecursiveASTVisitor<ZeroOutInDeallocRemover> base;
-
-  MigrationPass &Pass;
-
-  llvm::DenseMap<ObjCPropertyDecl*, ObjCPropertyImplDecl*> SynthesizedProperties;
-  ImplicitParamDecl *SelfD;
-  ExprSet Removables;
-  Selector FinalizeSel;
-
-public:
-  ZeroOutInDeallocRemover(MigrationPass &pass) : Pass(pass), SelfD(nullptr) {
-    FinalizeSel =
-        Pass.Ctx.Selectors.getNullarySelector(&Pass.Ctx.Idents.get("finalize"));
-  }
-
-  bool VisitObjCMessageExpr(ObjCMessageExpr *ME) {
-    ASTContext &Ctx = Pass.Ctx;
-    TransformActions &TA = Pass.TA;
-
-    if (ME->getReceiverKind() != ObjCMessageExpr::Instance)
-      return true;
-    Expr *receiver = ME->getInstanceReceiver();
-    if (!receiver)
-      return true;
-
-    DeclRefExpr *refE = dyn_cast<DeclRefExpr>(receiver->IgnoreParenCasts());
-    if (!refE || refE->getDecl() != SelfD)
-      return true;
-
-    bool BackedBySynthesizeSetter = false;
-    for (llvm::DenseMap<ObjCPropertyDecl*, ObjCPropertyImplDecl*>::iterator
-         P = SynthesizedProperties.begin(),
-         E = SynthesizedProperties.end(); P != E; ++P) {
-      ObjCPropertyDecl *PropDecl = P->first;
-      if (PropDecl->getSetterName() == ME->getSelector()) {
-        BackedBySynthesizeSetter = true;
-        break;
-      }
-    }
-    if (!BackedBySynthesizeSetter)
-      return true;
-
-    // Remove the setter message if RHS is null
-    Transaction Trans(TA);
-    Expr *RHS = ME->getArg(0);
-    bool RHSIsNull =
-      RHS->isNullPointerConstant(Ctx,
-                                 Expr::NPC_ValueDependentIsNull);
-    if (RHSIsNull && isRemovable(ME))
-      TA.removeStmt(ME);
-
-    return true;
-  }
-
-  bool VisitPseudoObjectExpr(PseudoObjectExpr *POE) {
-    if (isZeroingPropIvar(POE) && isRemovable(POE)) {
-      Transaction Trans(Pass.TA);
-      Pass.TA.removeStmt(POE);
-    }
-
-    return true;
-  }
-
-  bool VisitBinaryOperator(BinaryOperator *BOE) {
-    if (isZeroingPropIvar(BOE) && isRemovable(BOE)) {
-      Transaction Trans(Pass.TA);
-      Pass.TA.removeStmt(BOE);
-    }
-
-    return true;
-  }
-
-  bool TraverseObjCMethodDecl(ObjCMethodDecl *D) {
-    if (D->getMethodFamily() != OMF_dealloc &&
-        !(D->isInstanceMethod() && D->getSelector() == FinalizeSel))
-      return true;
-    if (!D->hasBody())
-      return true;
-
-    ObjCImplDecl *IMD = dyn_cast<ObjCImplDecl>(D->getDeclContext());
-    if (!IMD)
-      return true;
-
-    SelfD = D->getSelfDecl();
-    collectRemovables(D->getBody(), Removables);
-
-    // For a 'dealloc' method use, find all property implementations in
-    // this class implementation.
-    for (auto *PID : IMD->property_impls()) {
-      if (PID->getPropertyImplementation() ==
-          ObjCPropertyImplDecl::Synthesize) {
-        ObjCPropertyDecl *PD = PID->getPropertyDecl();
-        ObjCMethodDecl *setterM = PD->getSetterMethodDecl();
-        if (!(setterM && setterM->isDefined())) {
-          ObjCPropertyAttribute::Kind AttrKind = PD->getPropertyAttributes();
-          if (AttrKind & (ObjCPropertyAttribute::kind_retain |
-                          ObjCPropertyAttribute::kind_copy |
-                          ObjCPropertyAttribute::kind_strong))
-            SynthesizedProperties[PD] = PID;
-        }
-      }
-    }
-
-    // Now, remove all zeroing of ivars etc.
-    base::TraverseObjCMethodDecl(D);
-
-    // clear out for next method.
-    SynthesizedProperties.clear();
-    SelfD = nullptr;
-    Removables.clear();
-    return true;
-  }
-
-  bool TraverseFunctionDecl(FunctionDecl *D) { return true; }
-  bool TraverseBlockDecl(BlockDecl *block) { return true; }
-  bool TraverseBlockExpr(BlockExpr *block) { return true; }
-
-private:
-  bool isRemovable(Expr *E) const {
-    return Removables.count(E);
-  }
-
-  bool isZeroingPropIvar(Expr *E) {
-    E = E->IgnoreParens();
-    if (BinaryOperator *BO = dyn_cast<BinaryOperator>(E))
-      return isZeroingPropIvar(BO);
-    if (PseudoObjectExpr *PO = dyn_cast<PseudoObjectExpr>(E))
-      return isZeroingPropIvar(PO);
-    return false;
-  }
-
-  bool isZeroingPropIvar(BinaryOperator *BOE) {
-    if (BOE->getOpcode() == BO_Comma)
-      return isZeroingPropIvar(BOE->getLHS()) &&
-             isZeroingPropIvar(BOE->getRHS());
-
-    if (BOE->getOpcode() != BO_Assign)
-      return false;
-
-    Expr *LHS = BOE->getLHS();
-    if (ObjCIvarRefExpr *IV = dyn_cast<ObjCIvarRefExpr>(LHS)) {
-      ObjCIvarDecl *IVDecl = IV->getDecl();
-      if (!IVDecl->getType()->isObjCObjectPointerType())
-        return false;
-      bool IvarBacksPropertySynthesis = false;
-      for (llvm::DenseMap<ObjCPropertyDecl*, ObjCPropertyImplDecl*>::iterator
-           P = SynthesizedProperties.begin(),
-           E = SynthesizedProperties.end(); P != E; ++P) {
-        ObjCPropertyImplDecl *PropImpDecl = P->second;
-        if (PropImpDecl && PropImpDecl->getPropertyIvarDecl() == IVDecl) {
-          IvarBacksPropertySynthesis = true;
-          break;
-        }
-      }
-      if (!IvarBacksPropertySynthesis)
-        return false;
-    }
-    else
-        return false;
-
-    return isZero(BOE->getRHS());
-  }
-
-  bool isZeroingPropIvar(PseudoObjectExpr *PO) {
-    BinaryOperator *BO = dyn_cast<BinaryOperator>(PO->getSyntacticForm());
-    if (!BO) return false;
-    if (BO->getOpcode() != BO_Assign) return false;
-
-    ObjCPropertyRefExpr *PropRefExp =
-      dyn_cast<ObjCPropertyRefExpr>(BO->getLHS()->IgnoreParens());
-    if (!PropRefExp) return false;
-
-    // TODO: Using implicit property decl.
-    if (PropRefExp->isImplicitProperty())
-      return false;
-
-    if (ObjCPropertyDecl *PDecl = PropRefExp->getExplicitProperty()) {
-      if (!SynthesizedProperties.count(PDecl))
-        return false;
-    }
-
-    return isZero(cast<OpaqueValueExpr>(BO->getRHS())->getSourceExpr());
-  }
-
-  bool isZero(Expr *E) {
-    if (E->isNullPointerConstant(Pass.Ctx, Expr::NPC_ValueDependentIsNull))
-      return true;
-
-    return isZeroingPropIvar(E);
-  }
-};
-
-} // anonymous namespace
-
-void trans::removeZeroOutPropsInDeallocFinalize(MigrationPass &pass) {
-  ZeroOutInDeallocRemover trans(pass);
-  trans.TraverseDecl(pass.Ctx.getTranslationUnitDecl());
-}
diff --git a/clang/lib/ARCMigrate/TransformActions.cpp b/clang/lib/ARCMigrate/TransformActions.cpp
deleted file mode 100644
index 6bc6fed1a9032..0000000000000
--- a/clang/lib/ARCMigrate/TransformActions.cpp
+++ /dev/null
@@ -1,700 +0,0 @@
-//===-- TransformActions.cpp - Migration to ARC mode ----------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "Internals.h"
-#include "clang/AST/ASTContext.h"
-#include "clang/AST/Expr.h"
-#include "clang/Basic/SourceManager.h"
-#include "clang/Lex/Preprocessor.h"
-#include "llvm/ADT/DenseSet.h"
-#include <map>
-using namespace clang;
-using namespace arcmt;
-
-namespace {
-
-/// Collects transformations and merges them before applying them with
-/// with applyRewrites(). E.g. if the same source range
-/// is requested to be removed twice, only one rewriter remove will be invoked.
-/// Rewrites happen in "transactions"; if one rewrite in the transaction cannot
-/// be done (e.g. it resides in a macro) all rewrites in the transaction are
-/// aborted.
-/// FIXME: "Transactional" rewrites support should be baked in the Rewriter.
-class TransformActionsImpl {
-  CapturedDiagList &CapturedDiags;
-  ASTContext &Ctx;
-  Preprocessor &PP;
-
-  bool IsInTransaction;
-
-  enum ActionKind {
-    Act_Insert, Act_InsertAfterToken,
-    Act_Remove, Act_RemoveStmt,
-    Act_Replace, Act_ReplaceText,
-    Act_IncreaseIndentation,
-    Act_ClearDiagnostic
-  };
-
-  struct ActionData {
-    ActionKind Kind;
-    SourceLocation Loc;
-    SourceRange R1, R2;
-    StringRef Text1, Text2;
-    Stmt *S;
-    SmallVector<unsigned, 2> DiagIDs;
-  };
-
-  std::vector<ActionData> CachedActions;
-
-  enum RangeComparison {
-    Range_Before,
-    Range_After,
-    Range_Contains,
-    Range_Contained,
-    Range_ExtendsBegin,
-    Range_ExtendsEnd
-  };
-
-  /// A range to remove. It is a character range.
-  struct CharRange {
-    FullSourceLoc Begin, End;
-
-    CharRange(CharSourceRange range, SourceManager &srcMgr, Preprocessor &PP) {
-      SourceLocation beginLoc = range.getBegin(), endLoc = range.getEnd();
-      assert(beginLoc.isValid() && endLoc.isValid());
-      if (range.isTokenRange()) {
-        Begin = FullSourceLoc(srcMgr.getExpansionLoc(beginLoc), srcMgr);
-        End = FullSourceLoc(getLocForEndOfToken(endLoc, srcMgr, PP), srcMgr);
-      } else {
-        Begin = FullSourceLoc(srcMgr.getExpansionLoc(beginLoc), srcMgr);
-        End = FullSourceLoc(srcMgr.getExpansionLoc(endLoc), srcMgr);
-      }
-      assert(Begin.isValid() && End.isValid());
-    }
-
-    RangeComparison compareWith(const CharRange &RHS) const {
-      if (End.isBeforeInTranslationUnitThan(RHS.Begin))
-        return Range_Before;
-      if (RHS.End.isBeforeInTranslationUnitThan(Begin))
-        return Range_After;
-      if (!Begin.isBeforeInTranslationUnitThan(RHS.Begin) &&
-          !RHS.End.isBeforeInTranslationUnitThan(End))
-        return Range_Contained;
-      if (Begin.isBeforeInTranslationUnitThan(RHS.Begin) &&
-          RHS.End.isBeforeInTranslationUnitThan(End))
-        return Range_Contains;
-      if (Begin.isBeforeInTranslationUnitThan(RHS.Begin))
-        return Range_ExtendsBegin;
-      else
-        return Range_ExtendsEnd;
-    }
-
-    static RangeComparison compare(SourceRange LHS, SourceRange RHS,
-                                   SourceManager &SrcMgr, Preprocessor &PP) {
-      return CharRange(CharSourceRange::getTokenRange(LHS), SrcMgr, PP)
-                  .compareWith(CharRange(CharSourceRange::getTokenRange(RHS),
-                                            SrcMgr, PP));
-    }
-  };
-
-  typedef SmallVector<StringRef, 2> TextsVec;
-  typedef std::map<FullSourceLoc, TextsVec, FullSourceLoc::BeforeThanCompare>
-      InsertsMap;
-  InsertsMap Inserts;
-  /// A list of ranges to remove. They are always sorted and they never
-  /// intersect with each other.
-  std::list<CharRange> Removals;
-
-  llvm::DenseSet<Stmt *> StmtRemovals;
-
-  std::vector<std::pair<CharRange, SourceLocation> > IndentationRanges;
-
-  /// Keeps text passed to transformation methods.
-  llvm::StringMap<bool> UniqueText;
-
-public:
-  TransformActionsImpl(CapturedDiagList &capturedDiags,
-                       ASTContext &ctx, Preprocessor &PP)
-    : CapturedDiags(capturedDiags), Ctx(ctx), PP(PP), IsInTransaction(false) { }
-
-  ASTContext &getASTContext() { return Ctx; }
-
-  void startTransaction();
-  bool commitTransaction();
-  void abortTransaction();
-
-  bool isInTransaction() const { return IsInTransaction; }
-
-  void insert(SourceLocation loc, StringRef text);
-  void insertAfterToken(SourceLocation loc, StringRef text);
-  void remove(SourceRange range);
-  void removeStmt(Stmt *S);
-  void replace(SourceRange range, StringRef text);
-  void replace(SourceRange range, SourceRange replacementRange);
-  void replaceStmt(Stmt *S, StringRef text);
-  void replaceText(SourceLocation loc, StringRef text,
-                   StringRef replacementText);
-  void increaseIndentation(SourceRange range,
-                           SourceLocation parentIndent);
-
-  bool clearDiagnostic(ArrayRef<unsigned> IDs, SourceRange range);
-
-  void applyRewrites(TransformActions::RewriteReceiver &receiver);
-
-private:
-  bool canInsert(SourceLocation loc);
-  bool canInsertAfterToken(SourceLocation loc);
-  bool canRemoveRange(SourceRange range);
-  bool canReplaceRange(SourceRange range, SourceRange replacementRange);
-  bool canReplaceText(SourceLocation loc, StringRef text);
-
-  void commitInsert(SourceLocation loc, StringRef text);
-  void commitInsertAfterToken(SourceLocation loc, StringRef text);
-  void commitRemove(SourceRange range);
-  void commitRemoveStmt(Stmt *S);
-  void commitReplace(SourceRange range, SourceRange replacementRange);
-  void commitReplaceText(SourceLocation loc, StringRef text,
-                         StringRef replacementText);
-  void commitIncreaseIndentation(SourceRange range,SourceLocation parentIndent);
-  void commitClearDiagnostic(ArrayRef<unsigned> IDs, SourceRange range);
-
-  void addRemoval(CharSourceRange range);
-  void addInsertion(SourceLocation loc, StringRef text);
-
-  /// Stores text passed to the transformation methods to keep the string
-  /// "alive". Since the vast majority of text will be the same, we also unique
-  /// the strings using a StringMap.
-  StringRef getUniqueText(StringRef text);
-
-  /// Computes the source location just past the end of the token at
-  /// the given source location. If the location points at a macro, the whole
-  /// macro expansion is skipped.
-  static SourceLocation getLocForEndOfToken(SourceLocation loc,
-                                            SourceManager &SM,Preprocessor &PP);
-};
-
-} // anonymous namespace
-
-void TransformActionsImpl::startTransaction() {
-  assert(!IsInTransaction &&
-         "Cannot start a transaction in the middle of another one");
-  IsInTransaction = true;
-}
-
-bool TransformActionsImpl::commitTransaction() {
-  assert(IsInTransaction && "No transaction started");
-
-  if (CachedActions.empty()) {
-    IsInTransaction = false;
-    return false;
-  }
-
-  // Verify that all actions are possible otherwise abort the whole transaction.
-  bool AllActionsPossible = true;
-  for (unsigned i = 0, e = CachedActions.size(); i != e; ++i) {
-    ActionData &act = CachedActions[i];
-    switch (act.Kind) {
-    case Act_Insert:
-      if (!canInsert(act.Loc))
-        AllActionsPossible = false;
-      break;
-    case Act_InsertAfterToken:
-      if (!canInsertAfterToken(act.Loc))
-        AllActionsPossible = false;
-      break;
-    case Act_Remove:
-      if (!canRemoveRange(act.R1))
-        AllActionsPossible = false;
-      break;
-    case Act_RemoveStmt:
-      assert(act.S);
-      if (!canRemoveRange(act.S->getSourceRange()))
-        AllActionsPossible = false;
-      break;
-    case Act_Replace:
-      if (!canReplaceRange(act.R1, act.R2))
-        AllActionsPossible = false;
-      break;
-    case Act_ReplaceText:
-      if (!canReplaceText(act.Loc, act.Text1))
-        AllActionsPossible = false;
-      break;
-    case Act_IncreaseIndentation:
-      // This is not important, we don't care if it will fail.
-      break;
-    case Act_ClearDiagnostic:
-      // We are just checking source rewrites.
-      break;
-    }
-    if (!AllActionsPossible)
-      break;
-  }
-
-  if (!AllActionsPossible) {
-    abortTransaction();
-    return true;
-  }
-
-  for (unsigned i = 0, e = CachedActions.size(); i != e; ++i) {
-    ActionData &act = CachedActions[i];
-    switch (act.Kind) {
-    case Act_Insert:
-      commitInsert(act.Loc, act.Text1);
-      break;
-    case Act_InsertAfterToken:
-      commitInsertAfterToken(act.Loc, act.Text1);
-      break;
-    case Act_Remove:
-      commitRemove(act.R1);
-      break;
-    case Act_RemoveStmt:
-      commitRemoveStmt(act.S);
-      break;
-    case Act_Replace:
-      commitReplace(act.R1, act.R2);
-      break;
-    case Act_ReplaceText:
-      commitReplaceText(act.Loc, act.Text1, act.Text2);
-      break;
-    case Act_IncreaseIndentation:
-      commitIncreaseIndentation(act.R1, act.Loc);
-      break;
-    case Act_ClearDiagnostic:
-      commitClearDiagnostic(act.DiagIDs, act.R1);
-      break;
-    }
-  }
-
-  CachedActions.clear();
-  IsInTransaction = false;
-  return false;
-}
-
-void TransformActionsImpl::abortTransaction() {
-  assert(IsInTransaction && "No transaction started");
-  CachedActions.clear();
-  IsInTransaction = false;
-}
-
-void TransformActionsImpl::insert(SourceLocation loc, StringRef text) {
-  assert(IsInTransaction && "Actions only allowed during a transaction");
-  text = getUniqueText(text);
-  ActionData data;
-  data.Kind = Act_Insert;
-  data.Loc = loc;
-  data.Text1 = text;
-  CachedActions.push_back(data);
-}
-
-void TransformActionsImpl::insertAfterToken(SourceLocation loc, StringRef text) {
-  assert(IsInTransaction && "Actions only allowed during a transaction");
-  text = getUniqueText(text);
-  ActionData data;
-  data.Kind = Act_InsertAfterToken;
-  data.Loc = loc;
-  data.Text1 = text;
-  CachedActions.push_back(data);
-}
-
-void TransformActionsImpl::remove(SourceRange range) {
-  assert(IsInTransaction && "Actions only allowed during a transaction");
-  ActionData data;
-  data.Kind = Act_Remove;
-  data.R1 = range;
-  CachedActions.push_back(data);
-}
-
-void TransformActionsImpl::removeStmt(Stmt *S) {
-  assert(IsInTransaction && "Actions only allowed during a transaction");
-  ActionData data;
-  data.Kind = Act_RemoveStmt;
-  if (auto *E = dyn_cast<Expr>(S))
-    S = E->IgnoreImplicit(); // important for uniquing
-  data.S = S;
-  CachedActions.push_back(data);
-}
-
-void TransformActionsImpl::replace(SourceRange range, StringRef text) {
-  assert(IsInTransaction && "Actions only allowed during a transaction");
-  text = getUniqueText(text);
-  remove(range);
-  insert(range.getBegin(), text);
-}
-
-void TransformActionsImpl::replace(SourceRange range,
-                                   SourceRange replacementRange) {
-  assert(IsInTransaction && "Actions only allowed during a transaction");
-  ActionData data;
-  data.Kind = Act_Replace;
-  data.R1 = range;
-  data.R2 = replacementRange;
-  CachedActions.push_back(data);
-}
-
-void TransformActionsImpl::replaceText(SourceLocation loc, StringRef text,
-                                       StringRef replacementText) {
-  text = getUniqueText(text);
-  replacementText = getUniqueText(replacementText);
-  ActionData data;
-  data.Kind = Act_ReplaceText;
-  data.Loc = loc;
-  data.Text1 = text;
-  data.Text2 = replacementText;
-  CachedActions.push_back(data);
-}
-
-void TransformActionsImpl::replaceStmt(Stmt *S, StringRef text) {
-  assert(IsInTransaction && "Actions only allowed during a transaction");
-  text = getUniqueText(text);
-  insert(S->getBeginLoc(), text);
-  removeStmt(S);
-}
-
-void TransformActionsImpl::increaseIndentation(SourceRange range,
-                                               SourceLocation parentIndent) {
-  if (range.isInvalid()) return;
-  assert(IsInTransaction && "Actions only allowed during a transaction");
-  ActionData data;
-  data.Kind = Act_IncreaseIndentation;
-  data.R1 = range;
-  data.Loc = parentIndent;
-  CachedActions.push_back(data);
-}
-
-bool TransformActionsImpl::clearDiagnostic(ArrayRef<unsigned> IDs,
-                                           SourceRange range) {
-  assert(IsInTransaction && "Actions only allowed during a transaction");
-  if (!CapturedDiags.hasDiagnostic(IDs, range))
-    return false;
-
-  ActionData data;
-  data.Kind = Act_ClearDiagnostic;
-  data.R1 = range;
-  data.DiagIDs.append(IDs.begin(), IDs.end());
-  CachedActions.push_back(data);
-  return true;
-}
-
-bool TransformActionsImpl::canInsert(SourceLocation loc) {
-  if (loc.isInvalid())
-    return false;
-
-  SourceManager &SM = Ctx.getSourceManager();
-  if (SM.isInSystemHeader(SM.getExpansionLoc(loc)))
-    return false;
-
-  if (loc.isFileID())
-    return true;
-  return PP.isAtStartOfMacroExpansion(loc);
-}
-
-bool TransformActionsImpl::canInsertAfterToken(SourceLocation loc) {
-  if (loc.isInvalid())
-    return false;
-
-  SourceManager &SM = Ctx.getSourceManager();
-  if (SM.isInSystemHeader(SM.getExpansionLoc(loc)))
-    return false;
-
-  if (loc.isFileID())
-    return true;
-  return PP.isAtEndOfMacroExpansion(loc);
-}
-
-bool TransformActionsImpl::canRemoveRange(SourceRange range) {
-  return canInsert(range.getBegin()) && canInsertAfterToken(range.getEnd());
-}
-
-bool TransformActionsImpl::canReplaceRange(SourceRange range,
-                                           SourceRange replacementRange) {
-  return canRemoveRange(range) && canRemoveRange(replacementRange);
-}
-
-bool TransformActionsImpl::canReplaceText(SourceLocation loc, StringRef text) {
-  if (!canInsert(loc))
-    return false;
-
-  SourceManager &SM = Ctx.getSourceManager();
-  loc = SM.getExpansionLoc(loc);
-
-  // Break down the source location.
-  std::pair<FileID, unsigned> locInfo = SM.getDecomposedLoc(loc);
-
-  // Try to load the file buffer.
-  bool invalidTemp = false;
-  StringRef file = SM.getBufferData(locInfo.first, &invalidTemp);
-  if (invalidTemp)
-    return false;
-
-  return file.substr(locInfo.second).starts_with(text);
-}
-
-void TransformActionsImpl::commitInsert(SourceLocation loc, StringRef text) {
-  addInsertion(loc, text);
-}
-
-void TransformActionsImpl::commitInsertAfterToken(SourceLocation loc,
-                                                  StringRef text) {
-  addInsertion(getLocForEndOfToken(loc, Ctx.getSourceManager(), PP), text);
-}
-
-void TransformActionsImpl::commitRemove(SourceRange range) {
-  addRemoval(CharSourceRange::getTokenRange(range));
-}
-
-void TransformActionsImpl::commitRemoveStmt(Stmt *S) {
-  assert(S);
-  if (StmtRemovals.count(S))
-    return; // already removed.
-
-  if (Expr *E = dyn_cast<Expr>(S)) {
-    commitRemove(E->getSourceRange());
-    commitInsert(E->getSourceRange().getBegin(), getARCMTMacroName());
-  } else
-    commitRemove(S->getSourceRange());
-
-  StmtRemovals.insert(S);
-}
-
-void TransformActionsImpl::commitReplace(SourceRange range,
-                                         SourceRange replacementRange) {
-  RangeComparison comp = CharRange::compare(replacementRange, range,
-                                               Ctx.getSourceManager(), PP);
-  assert(comp == Range_Contained);
-  if (comp != Range_Contained)
-    return; // Although we asserted, be extra safe for release build.
-  if (range.getBegin() != replacementRange.getBegin())
-    addRemoval(CharSourceRange::getCharRange(range.getBegin(),
-                                             replacementRange.getBegin()));
-  if (replacementRange.getEnd() != range.getEnd())
-    addRemoval(CharSourceRange::getTokenRange(
-                                  getLocForEndOfToken(replacementRange.getEnd(),
-                                                      Ctx.getSourceManager(), PP),
-                                  range.getEnd()));
-}
-void TransformActionsImpl::commitReplaceText(SourceLocation loc,
-                                             StringRef text,
-                                             StringRef replacementText) {
-  SourceManager &SM = Ctx.getSourceManager();
-  loc = SM.getExpansionLoc(loc);
-  // canReplaceText already checked if loc points at text.
-  SourceLocation afterText = loc.getLocWithOffset(text.size());
-
-  addRemoval(CharSourceRange::getCharRange(loc, afterText));
-  commitInsert(loc, replacementText);
-}
-
-void TransformActionsImpl::commitIncreaseIndentation(SourceRange range,
-                                                  SourceLocation parentIndent) {
-  SourceManager &SM = Ctx.getSourceManager();
-  IndentationRanges.push_back(
-                 std::make_pair(CharRange(CharSourceRange::getTokenRange(range),
-                                          SM, PP),
-                                SM.getExpansionLoc(parentIndent)));
-}
-
-void TransformActionsImpl::commitClearDiagnostic(ArrayRef<unsigned> IDs,
-                                                 SourceRange range) {
-  CapturedDiags.clearDiagnostic(IDs, range);
-}
-
-void TransformActionsImpl::addInsertion(SourceLocation loc, StringRef text) {
-  SourceManager &SM = Ctx.getSourceManager();
-  loc = SM.getExpansionLoc(loc);
-  for (const CharRange &I : llvm::reverse(Removals)) {
-    if (!SM.isBeforeInTranslationUnit(loc, I.End))
-      break;
-    if (I.Begin.isBeforeInTranslationUnitThan(loc))
-      return;
-  }
-
-  Inserts[FullSourceLoc(loc, SM)].push_back(text);
-}
-
-void TransformActionsImpl::addRemoval(CharSourceRange range) {
-  CharRange newRange(range, Ctx.getSourceManager(), PP);
-  if (newRange.Begin == newRange.End)
-    return;
-
-  Inserts.erase(Inserts.upper_bound(newRange.Begin),
-                Inserts.lower_bound(newRange.End));
-
-  std::list<CharRange>::iterator I = Removals.end();
-  while (I != Removals.begin()) {
-    std::list<CharRange>::iterator RI = I;
-    --RI;
-    RangeComparison comp = newRange.compareWith(*RI);
-    switch (comp) {
-    case Range_Before:
-      --I;
-      break;
-    case Range_After:
-      Removals.insert(I, newRange);
-      return;
-    case Range_Contained:
-      return;
-    case Range_Contains:
-      RI->End = newRange.End;
-      [[fallthrough]];
-    case Range_ExtendsBegin:
-      newRange.End = RI->End;
-      Removals.erase(RI);
-      break;
-    case Range_ExtendsEnd:
-      RI->End = newRange.End;
-      return;
-    }
-  }
-
-  Removals.insert(Removals.begin(), newRange);
-}
-
-void TransformActionsImpl::applyRewrites(
-                                  TransformActions::RewriteReceiver &receiver) {
-  for (InsertsMap::iterator I = Inserts.begin(), E = Inserts.end(); I!=E; ++I) {
-    SourceLocation loc = I->first;
-    for (TextsVec::iterator
-           TI = I->second.begin(), TE = I->second.end(); TI != TE; ++TI) {
-      receiver.insert(loc, *TI);
-    }
-  }
-
-  for (std::vector<std::pair<CharRange, SourceLocation> >::iterator
-       I = IndentationRanges.begin(), E = IndentationRanges.end(); I!=E; ++I) {
-    CharSourceRange range = CharSourceRange::getCharRange(I->first.Begin,
-                                                          I->first.End);
-    receiver.increaseIndentation(range, I->second);
-  }
-
-  for (std::list<CharRange>::iterator
-         I = Removals.begin(), E = Removals.end(); I != E; ++I) {
-    CharSourceRange range = CharSourceRange::getCharRange(I->Begin, I->End);
-    receiver.remove(range);
-  }
-}
-
-/// Stores text passed to the transformation methods to keep the string
-/// "alive". Since the vast majority of text will be the same, we also unique
-/// the strings using a StringMap.
-StringRef TransformActionsImpl::getUniqueText(StringRef text) {
-  return UniqueText.insert(std::make_pair(text, false)).first->first();
-}
-
-/// Computes the source location just past the end of the token at
-/// the given source location. If the location points at a macro, the whole
-/// macro expansion is skipped.
-SourceLocation TransformActionsImpl::getLocForEndOfToken(SourceLocation loc,
-                                                         SourceManager &SM,
-                                                         Preprocessor &PP) {
-  if (loc.isMacroID()) {
-    CharSourceRange Exp = SM.getExpansionRange(loc);
-    if (Exp.isCharRange())
-      return Exp.getEnd();
-    loc = Exp.getEnd();
-  }
-  return PP.getLocForEndOfToken(loc);
-}
-
-TransformActions::RewriteReceiver::~RewriteReceiver() { }
-
-TransformActions::TransformActions(DiagnosticsEngine &diag,
-                                   CapturedDiagList &capturedDiags,
-                                   ASTContext &ctx, Preprocessor &PP)
-    : Diags(diag), CapturedDiags(capturedDiags) {
-  Impl = new TransformActionsImpl(capturedDiags, ctx, PP);
-}
-
-TransformActions::~TransformActions() {
-  delete static_cast<TransformActionsImpl*>(Impl);
-}
-
-void TransformActions::startTransaction() {
-  static_cast<TransformActionsImpl*>(Impl)->startTransaction();
-}
-
-bool TransformActions::commitTransaction() {
-  return static_cast<TransformActionsImpl*>(Impl)->commitTransaction();
-}
-
-void TransformActions::abortTransaction() {
-  static_cast<TransformActionsImpl*>(Impl)->abortTransaction();
-}
-
-
-void TransformActions::insert(SourceLocation loc, StringRef text) {
-  static_cast<TransformActionsImpl*>(Impl)->insert(loc, text);
-}
-
-void TransformActions::insertAfterToken(SourceLocation loc,
-                                        StringRef text) {
-  static_cast<TransformActionsImpl*>(Impl)->insertAfterToken(loc, text);
-}
-
-void TransformActions::remove(SourceRange range) {
-  static_cast<TransformActionsImpl*>(Impl)->remove(range);
-}
-
-void TransformActions::removeStmt(Stmt *S) {
-  static_cast<TransformActionsImpl*>(Impl)->removeStmt(S);
-}
-
-void TransformActions::replace(SourceRange range, StringRef text) {
-  static_cast<TransformActionsImpl*>(Impl)->replace(range, text);
-}
-
-void TransformActions::replace(SourceRange range,
-                               SourceRange replacementRange) {
-  static_cast<TransformActionsImpl*>(Impl)->replace(range, replacementRange);
-}
-
-void TransformActions::replaceStmt(Stmt *S, StringRef text) {
-  static_cast<TransformActionsImpl*>(Impl)->replaceStmt(S, text);
-}
-
-void TransformActions::replaceText(SourceLocation loc, StringRef text,
-                                   StringRef replacementText) {
-  static_cast<TransformActionsImpl*>(Impl)->replaceText(loc, text,
-                                                        replacementText);
-}
-
-void TransformActions::increaseIndentation(SourceRange range,
-                                           SourceLocation parentIndent) {
-  static_cast<TransformActionsImpl*>(Impl)->increaseIndentation(range,
-                                                                parentIndent);
-}
-
-bool TransformActions::clearDiagnostic(ArrayRef<unsigned> IDs,
-                                       SourceRange range) {
-  return static_cast<TransformActionsImpl*>(Impl)->clearDiagnostic(IDs, range);
-}
-
-void TransformActions::applyRewrites(RewriteReceiver &receiver) {
-  static_cast<TransformActionsImpl*>(Impl)->applyRewrites(receiver);
-}
-
-DiagnosticBuilder TransformActions::report(SourceLocation loc, unsigned diagId,
-                                           SourceRange range) {
-  assert(!static_cast<TransformActionsImpl *>(Impl)->isInTransaction() &&
-         "Errors should be emitted out of a transaction");
-  return Diags.Report(loc, diagId) << range;
-}
-
-void TransformActions::reportError(StringRef message, SourceLocation loc,
-                                   SourceRange range) {
-  report(loc, diag::err_mt_message, range) << message;
-}
-
-void TransformActions::reportWarning(StringRef message, SourceLocation loc,
-                                     SourceRange range) {
-  report(loc, diag::warn_mt_message, range) << message;
-}
-
-void TransformActions::reportNote(StringRef message, SourceLocation loc,
-                                  SourceRange range) {
-  report(loc, diag::note_mt_message, range) << message;
-}
diff --git a/clang/lib/ARCMigrate/Transforms.cpp b/clang/lib/ARCMigrate/Transforms.cpp
deleted file mode 100644
index fda0e1c932fc0..0000000000000
--- a/clang/lib/ARCMigrate/Transforms.cpp
+++ /dev/null
@@ -1,594 +0,0 @@
-//===--- Transforms.cpp - Transformations to ARC mode ---------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "Transforms.h"
-#include "Internals.h"
-#include "clang/ARCMigrate/ARCMT.h"
-#include "clang/AST/ASTContext.h"
-#include "clang/AST/RecursiveASTVisitor.h"
-#include "clang/Analysis/DomainSpecific/CocoaConventions.h"
-#include "clang/Basic/SourceManager.h"
-#include "clang/Basic/TargetInfo.h"
-#include "clang/Lex/Lexer.h"
-#include "clang/Lex/Preprocessor.h"
-#include "clang/Sema/Sema.h"
-#include "clang/Sema/SemaObjC.h"
-
-using namespace clang;
-using namespace arcmt;
-using namespace trans;
-
-ASTTraverser::~ASTTraverser() { }
-
-bool MigrationPass::CFBridgingFunctionsDefined() {
-  if (!EnableCFBridgeFns)
-    EnableCFBridgeFns = SemaRef.ObjC().isKnownName("CFBridgingRetain") &&
-                        SemaRef.ObjC().isKnownName("CFBridgingRelease");
-  return *EnableCFBridgeFns;
-}
-
-//===----------------------------------------------------------------------===//
-// Helpers.
-//===----------------------------------------------------------------------===//
-
-bool trans::canApplyWeak(ASTContext &Ctx, QualType type,
-                         bool AllowOnUnknownClass) {
-  if (!Ctx.getLangOpts().ObjCWeakRuntime)
-    return false;
-
-  QualType T = type;
-  if (T.isNull())
-    return false;
-
-  // iOS is always safe to use 'weak'.
-  if (Ctx.getTargetInfo().getTriple().isiOS() ||
-      Ctx.getTargetInfo().getTriple().isWatchOS())
-    AllowOnUnknownClass = true;
-
-  while (const PointerType *ptr = T->getAs<PointerType>())
-    T = ptr->getPointeeType();
-  if (const ObjCObjectPointerType *ObjT = T->getAs<ObjCObjectPointerType>()) {
-    ObjCInterfaceDecl *Class = ObjT->getInterfaceDecl();
-    if (!AllowOnUnknownClass && (!Class || Class->getName() == "NSObject"))
-      return false; // id/NSObject is not safe for weak.
-    if (!AllowOnUnknownClass && !Class->hasDefinition())
-      return false; // forward classes are not verifiable, therefore not safe.
-    if (Class && Class->isArcWeakrefUnavailable())
-      return false;
-  }
-
-  return true;
-}
-
-bool trans::isPlusOneAssign(const BinaryOperator *E) {
-  if (E->getOpcode() != BO_Assign)
-    return false;
-
-  return isPlusOne(E->getRHS());
-}
-
-bool trans::isPlusOne(const Expr *E) {
-  if (!E)
-    return false;
-  if (const FullExpr *FE = dyn_cast<FullExpr>(E))
-    E = FE->getSubExpr();
-
-  if (const ObjCMessageExpr *
-        ME = dyn_cast<ObjCMessageExpr>(E->IgnoreParenCasts()))
-    if (ME->getMethodFamily() == OMF_retain)
-      return true;
-
-  if (const CallExpr *
-        callE = dyn_cast<CallExpr>(E->IgnoreParenCasts())) {
-    if (const FunctionDecl *FD = callE->getDirectCallee()) {
-      if (FD->hasAttr<CFReturnsRetainedAttr>())
-        return true;
-
-      if (FD->isGlobal() &&
-          FD->getIdentifier() &&
-          FD->getParent()->isTranslationUnit() &&
-          FD->isExternallyVisible() &&
-          ento::cocoa::isRefType(callE->getType(), "CF",
-                                 FD->getIdentifier()->getName())) {
-        StringRef fname = FD->getIdentifier()->getName();
-        if (fname.ends_with("Retain") || fname.contains("Create") ||
-            fname.contains("Copy"))
-          return true;
-      }
-    }
-  }
-
-  const ImplicitCastExpr *implCE = dyn_cast<ImplicitCastExpr>(E);
-  while (implCE && implCE->getCastKind() ==  CK_BitCast)
-    implCE = dyn_cast<ImplicitCastExpr>(implCE->getSubExpr());
-
-  return implCE && implCE->getCastKind() == CK_ARCConsumeObject;
-}
-
-/// 'Loc' is the end of a statement range. This returns the location
-/// immediately after the semicolon following the statement.
-/// If no semicolon is found or the location is inside a macro, the returned
-/// source location will be invalid.
-SourceLocation trans::findLocationAfterSemi(SourceLocation loc,
-                                            ASTContext &Ctx, bool IsDecl) {
-  SourceLocation SemiLoc = findSemiAfterLocation(loc, Ctx, IsDecl);
-  if (SemiLoc.isInvalid())
-    return SourceLocation();
-  return SemiLoc.getLocWithOffset(1);
-}
-
-/// \arg Loc is the end of a statement range. This returns the location
-/// of the semicolon following the statement.
-/// If no semicolon is found or the location is inside a macro, the returned
-/// source location will be invalid.
-SourceLocation trans::findSemiAfterLocation(SourceLocation loc,
-                                            ASTContext &Ctx,
-                                            bool IsDecl) {
-  SourceManager &SM = Ctx.getSourceManager();
-  if (loc.isMacroID()) {
-    if (!Lexer::isAtEndOfMacroExpansion(loc, SM, Ctx.getLangOpts(), &loc))
-      return SourceLocation();
-  }
-  loc = Lexer::getLocForEndOfToken(loc, /*Offset=*/0, SM, Ctx.getLangOpts());
-
-  // Break down the source location.
-  std::pair<FileID, unsigned> locInfo = SM.getDecomposedLoc(loc);
-
-  // Try to load the file buffer.
-  bool invalidTemp = false;
-  StringRef file = SM.getBufferData(locInfo.first, &invalidTemp);
-  if (invalidTemp)
-    return SourceLocation();
-
-  const char *tokenBegin = file.data() + locInfo.second;
-
-  // Lex from the start of the given location.
-  Lexer lexer(SM.getLocForStartOfFile(locInfo.first),
-              Ctx.getLangOpts(),
-              file.begin(), tokenBegin, file.end());
-  Token tok;
-  lexer.LexFromRawLexer(tok);
-  if (tok.isNot(tok::semi)) {
-    if (!IsDecl)
-      return SourceLocation();
-    // Declaration may be followed with other tokens; such as an __attribute,
-    // before ending with a semicolon.
-    return findSemiAfterLocation(tok.getLocation(), Ctx, /*IsDecl*/true);
-  }
-
-  return tok.getLocation();
-}
-
-bool trans::hasSideEffects(Expr *E, ASTContext &Ctx) {
-  if (!E || !E->HasSideEffects(Ctx))
-    return false;
-
-  E = E->IgnoreParenCasts();
-  ObjCMessageExpr *ME = dyn_cast<ObjCMessageExpr>(E);
-  if (!ME)
-    return true;
-  switch (ME->getMethodFamily()) {
-  case OMF_autorelease:
-  case OMF_dealloc:
-  case OMF_release:
-  case OMF_retain:
-    switch (ME->getReceiverKind()) {
-    case ObjCMessageExpr::SuperInstance:
-      return false;
-    case ObjCMessageExpr::Instance:
-      return hasSideEffects(ME->getInstanceReceiver(), Ctx);
-    default:
-      break;
-    }
-    break;
-  default:
-    break;
-  }
-
-  return true;
-}
-
-bool trans::isGlobalVar(Expr *E) {
-  E = E->IgnoreParenCasts();
-  if (DeclRefExpr *DRE = dyn_cast<DeclRefExpr>(E))
-    return DRE->getDecl()->getDeclContext()->isFileContext() &&
-           DRE->getDecl()->isExternallyVisible();
-  if (ConditionalOperator *condOp = dyn_cast<ConditionalOperator>(E))
-    return isGlobalVar(condOp->getTrueExpr()) &&
-           isGlobalVar(condOp->getFalseExpr());
-
-  return false;
-}
-
-StringRef trans::getNilString(MigrationPass &Pass) {
-  return Pass.SemaRef.PP.isMacroDefined("nil") ? "nil" : "0";
-}
-
-namespace {
-
-class ReferenceClear : public RecursiveASTVisitor<ReferenceClear> {
-  ExprSet &Refs;
-public:
-  ReferenceClear(ExprSet &refs) : Refs(refs) { }
-  bool VisitDeclRefExpr(DeclRefExpr *E) { Refs.erase(E); return true; }
-};
-
-class ReferenceCollector : public RecursiveASTVisitor<ReferenceCollector> {
-  ValueDecl *Dcl;
-  ExprSet &Refs;
-
-public:
-  ReferenceCollector(ValueDecl *D, ExprSet &refs)
-    : Dcl(D), Refs(refs) { }
-
-  bool VisitDeclRefExpr(DeclRefExpr *E) {
-    if (E->getDecl() == Dcl)
-      Refs.insert(E);
-    return true;
-  }
-};
-
-class RemovablesCollector : public RecursiveASTVisitor<RemovablesCollector> {
-  ExprSet &Removables;
-
-public:
-  RemovablesCollector(ExprSet &removables)
-  : Removables(removables) { }
-
-  bool shouldWalkTypesOfTypeLocs() const { return false; }
-
-  bool TraverseStmtExpr(StmtExpr *E) {
-    CompoundStmt *S = E->getSubStmt();
-    for (CompoundStmt::body_iterator
-        I = S->body_begin(), E = S->body_end(); I != E; ++I) {
-      if (I != E - 1)
-        mark(*I);
-      TraverseStmt(*I);
-    }
-    return true;
-  }
-
-  bool VisitCompoundStmt(CompoundStmt *S) {
-    for (auto *I : S->body())
-      mark(I);
-    return true;
-  }
-
-  bool VisitIfStmt(IfStmt *S) {
-    mark(S->getThen());
-    mark(S->getElse());
-    return true;
-  }
-
-  bool VisitWhileStmt(WhileStmt *S) {
-    mark(S->getBody());
-    return true;
-  }
-
-  bool VisitDoStmt(DoStmt *S) {
-    mark(S->getBody());
-    return true;
-  }
-
-  bool VisitForStmt(ForStmt *S) {
-    mark(S->getInit());
-    mark(S->getInc());
-    mark(S->getBody());
-    return true;
-  }
-
-private:
-  void mark(Stmt *S) {
-    if (!S) return;
-
-    while (auto *Label = dyn_cast<LabelStmt>(S))
-      S = Label->getSubStmt();
-    if (auto *E = dyn_cast<Expr>(S))
-      S = E->IgnoreImplicit();
-    if (auto *E = dyn_cast<Expr>(S))
-      Removables.insert(E);
-  }
-};
-
-} // end anonymous namespace
-
-void trans::clearRefsIn(Stmt *S, ExprSet &refs) {
-  ReferenceClear(refs).TraverseStmt(S);
-}
-
-void trans::collectRefs(ValueDecl *D, Stmt *S, ExprSet &refs) {
-  ReferenceCollector(D, refs).TraverseStmt(S);
-}
-
-void trans::collectRemovables(Stmt *S, ExprSet &exprs) {
-  RemovablesCollector(exprs).TraverseStmt(S);
-}
-
-//===----------------------------------------------------------------------===//
-// MigrationContext
-//===----------------------------------------------------------------------===//
-
-namespace {
-
-class ASTTransform : public RecursiveASTVisitor<ASTTransform> {
-  MigrationContext &MigrateCtx;
-  typedef RecursiveASTVisitor<ASTTransform> base;
-
-public:
-  ASTTransform(MigrationContext &MigrateCtx) : MigrateCtx(MigrateCtx) { }
-
-  bool shouldWalkTypesOfTypeLocs() const { return false; }
-
-  bool TraverseObjCImplementationDecl(ObjCImplementationDecl *D) {
-    ObjCImplementationContext ImplCtx(MigrateCtx, D);
-    for (MigrationContext::traverser_iterator
-           I = MigrateCtx.traversers_begin(),
-           E = MigrateCtx.traversers_end(); I != E; ++I)
-      (*I)->traverseObjCImplementation(ImplCtx);
-
-    return base::TraverseObjCImplementationDecl(D);
-  }
-
-  bool TraverseStmt(Stmt *rootS) {
-    if (!rootS)
-      return true;
-
-    BodyContext BodyCtx(MigrateCtx, rootS);
-    for (MigrationContext::traverser_iterator
-           I = MigrateCtx.traversers_begin(),
-           E = MigrateCtx.traversers_end(); I != E; ++I)
-      (*I)->traverseBody(BodyCtx);
-
-    return true;
-  }
-};
-
-}
-
-MigrationContext::~MigrationContext() {
-  for (traverser_iterator
-         I = traversers_begin(), E = traversers_end(); I != E; ++I)
-    delete *I;
-}
-
-bool MigrationContext::isGCOwnedNonObjC(QualType T) {
-  while (!T.isNull()) {
-    if (const AttributedType *AttrT = T->getAs<AttributedType>()) {
-      if (AttrT->getAttrKind() == attr::ObjCOwnership)
-        return !AttrT->getModifiedType()->isObjCRetainableType();
-    }
-
-    if (T->isArrayType())
-      T = Pass.Ctx.getBaseElementType(T);
-    else if (const PointerType *PT = T->getAs<PointerType>())
-      T = PT->getPointeeType();
-    else if (const ReferenceType *RT = T->getAs<ReferenceType>())
-      T = RT->getPointeeType();
-    else
-      break;
-  }
-
-  return false;
-}
-
-bool MigrationContext::rewritePropertyAttribute(StringRef fromAttr,
-                                                StringRef toAttr,
-                                                SourceLocation atLoc) {
-  if (atLoc.isMacroID())
-    return false;
-
-  SourceManager &SM = Pass.Ctx.getSourceManager();
-
-  // Break down the source location.
-  std::pair<FileID, unsigned> locInfo = SM.getDecomposedLoc(atLoc);
-
-  // Try to load the file buffer.
-  bool invalidTemp = false;
-  StringRef file = SM.getBufferData(locInfo.first, &invalidTemp);
-  if (invalidTemp)
-    return false;
-
-  const char *tokenBegin = file.data() + locInfo.second;
-
-  // Lex from the start of the given location.
-  Lexer lexer(SM.getLocForStartOfFile(locInfo.first),
-              Pass.Ctx.getLangOpts(),
-              file.begin(), tokenBegin, file.end());
-  Token tok;
-  lexer.LexFromRawLexer(tok);
-  if (tok.isNot(tok::at)) return false;
-  lexer.LexFromRawLexer(tok);
-  if (tok.isNot(tok::raw_identifier)) return false;
-  if (tok.getRawIdentifier() != "property")
-    return false;
-  lexer.LexFromRawLexer(tok);
-  if (tok.isNot(tok::l_paren)) return false;
-
-  Token BeforeTok = tok;
-  Token AfterTok;
-  AfterTok.startToken();
-  SourceLocation AttrLoc;
-
-  lexer.LexFromRawLexer(tok);
-  if (tok.is(tok::r_paren))
-    return false;
-
-  while (true) {
-    if (tok.isNot(tok::raw_identifier)) return false;
-    if (tok.getRawIdentifier() == fromAttr) {
-      if (!toAttr.empty()) {
-        Pass.TA.replaceText(tok.getLocation(), fromAttr, toAttr);
-        return true;
-      }
-      // We want to remove the attribute.
-      AttrLoc = tok.getLocation();
-    }
-
-    do {
-      lexer.LexFromRawLexer(tok);
-      if (AttrLoc.isValid() && AfterTok.is(tok::unknown))
-        AfterTok = tok;
-    } while (tok.isNot(tok::comma) && tok.isNot(tok::r_paren));
-    if (tok.is(tok::r_paren))
-      break;
-    if (AttrLoc.isInvalid())
-      BeforeTok = tok;
-    lexer.LexFromRawLexer(tok);
-  }
-
-  if (toAttr.empty() && AttrLoc.isValid() && AfterTok.isNot(tok::unknown)) {
-    // We want to remove the attribute.
-    if (BeforeTok.is(tok::l_paren) && AfterTok.is(tok::r_paren)) {
-      Pass.TA.remove(SourceRange(BeforeTok.getLocation(),
-                                 AfterTok.getLocation()));
-    } else if (BeforeTok.is(tok::l_paren) && AfterTok.is(tok::comma)) {
-      Pass.TA.remove(SourceRange(AttrLoc, AfterTok.getLocation()));
-    } else {
-      Pass.TA.remove(SourceRange(BeforeTok.getLocation(), AttrLoc));
-    }
-
-    return true;
-  }
-
-  return false;
-}
-
-bool MigrationContext::addPropertyAttribute(StringRef attr,
-                                            SourceLocation atLoc) {
-  if (atLoc.isMacroID())
-    return false;
-
-  SourceManager &SM = Pass.Ctx.getSourceManager();
-
-  // Break down the source location.
-  std::pair<FileID, unsigned> locInfo = SM.getDecomposedLoc(atLoc);
-
-  // Try to load the file buffer.
-  bool invalidTemp = false;
-  StringRef file = SM.getBufferData(locInfo.first, &invalidTemp);
-  if (invalidTemp)
-    return false;
-
-  const char *tokenBegin = file.data() + locInfo.second;
-
-  // Lex from the start of the given location.
-  Lexer lexer(SM.getLocForStartOfFile(locInfo.first),
-              Pass.Ctx.getLangOpts(),
-              file.begin(), tokenBegin, file.end());
-  Token tok;
-  lexer.LexFromRawLexer(tok);
-  if (tok.isNot(tok::at)) return false;
-  lexer.LexFromRawLexer(tok);
-  if (tok.isNot(tok::raw_identifier)) return false;
-  if (tok.getRawIdentifier() != "property")
-    return false;
-  lexer.LexFromRawLexer(tok);
-
-  if (tok.isNot(tok::l_paren)) {
-    Pass.TA.insert(tok.getLocation(), std::string("(") + attr.str() + ") ");
-    return true;
-  }
-
-  lexer.LexFromRawLexer(tok);
-  if (tok.is(tok::r_paren)) {
-    Pass.TA.insert(tok.getLocation(), attr);
-    return true;
-  }
-
-  if (tok.isNot(tok::raw_identifier)) return false;
-
-  Pass.TA.insert(tok.getLocation(), std::string(attr) + ", ");
-  return true;
-}
-
-void MigrationContext::traverse(TranslationUnitDecl *TU) {
-  for (traverser_iterator
-         I = traversers_begin(), E = traversers_end(); I != E; ++I)
-    (*I)->traverseTU(*this);
-
-  ASTTransform(*this).TraverseDecl(TU);
-}
-
-static void GCRewriteFinalize(MigrationPass &pass) {
-  ASTContext &Ctx = pass.Ctx;
-  TransformActions &TA = pass.TA;
-  DeclContext *DC = Ctx.getTranslationUnitDecl();
-  Selector FinalizeSel =
-   Ctx.Selectors.getNullarySelector(&pass.Ctx.Idents.get("finalize"));
-
-  typedef DeclContext::specific_decl_iterator<ObjCImplementationDecl>
-  impl_iterator;
-  for (impl_iterator I = impl_iterator(DC->decls_begin()),
-       E = impl_iterator(DC->decls_end()); I != E; ++I) {
-    for (const auto *MD : I->instance_methods()) {
-      if (!MD->hasBody())
-        continue;
-
-      if (MD->isInstanceMethod() && MD->getSelector() == FinalizeSel) {
-        const ObjCMethodDecl *FinalizeM = MD;
-        Transaction Trans(TA);
-        TA.insert(FinalizeM->getSourceRange().getBegin(),
-                  "#if !__has_feature(objc_arc)\n");
-        CharSourceRange::getTokenRange(FinalizeM->getSourceRange());
-        const SourceManager &SM = pass.Ctx.getSourceManager();
-        const LangOptions &LangOpts = pass.Ctx.getLangOpts();
-        bool Invalid;
-        std::string str = "\n#endif\n";
-        str += Lexer::getSourceText(
-                  CharSourceRange::getTokenRange(FinalizeM->getSourceRange()),
-                                    SM, LangOpts, &Invalid);
-        TA.insertAfterToken(FinalizeM->getSourceRange().getEnd(), str);
-
-        break;
-      }
-    }
-  }
-}
-
-//===----------------------------------------------------------------------===//
-// getAllTransformations.
-//===----------------------------------------------------------------------===//
-
-static void traverseAST(MigrationPass &pass) {
-  MigrationContext MigrateCtx(pass);
-
-  if (pass.isGCMigration()) {
-    MigrateCtx.addTraverser(new GCCollectableCallsTraverser);
-    MigrateCtx.addTraverser(new GCAttrsTraverser());
-  }
-  MigrateCtx.addTraverser(new PropertyRewriteTraverser());
-  MigrateCtx.addTraverser(new BlockObjCVariableTraverser());
-  MigrateCtx.addTraverser(new ProtectedScopeTraverser());
-
-  MigrateCtx.traverse(pass.Ctx.getTranslationUnitDecl());
-}
-
-static void independentTransforms(MigrationPass &pass) {
-  rewriteAutoreleasePool(pass);
-  removeRetainReleaseDeallocFinalize(pass);
-  rewriteUnusedInitDelegate(pass);
-  removeZeroOutPropsInDeallocFinalize(pass);
-  makeAssignARCSafe(pass);
-  rewriteUnbridgedCasts(pass);
-  checkAPIUses(pass);
-  traverseAST(pass);
-}
-
-std::vector<TransformFn> arcmt::getAllTransformations(
-                                               LangOptions::GCMode OrigGCMode,
-                                               bool NoFinalizeRemoval) {
-  std::vector<TransformFn> transforms;
-
-  if (OrigGCMode ==  LangOptions::GCOnly && NoFinalizeRemoval)
-    transforms.push_back(GCRewriteFinalize);
-  transforms.push_back(independentTransforms);
-  // This depends on previous transformations removing various expressions.
-  transforms.push_back(removeEmptyStatementsAndDeallocFinalize);
-
-  return transforms;
-}
diff --git a/clang/lib/ARCMigrate/Transforms.h b/clang/lib/ARCMigrate/Transforms.h
deleted file mode 100644
index 37e2d6b2a7e12..0000000000000
--- a/clang/lib/ARCMigrate/Transforms.h
+++ /dev/null
@@ -1,224 +0,0 @@
-//===-- Transforms.h - Transformations to ARC mode --------------*- C++ -*-===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LLVM_CLANG_LIB_ARCMIGRATE_TRANSFORMS_H
-#define LLVM_CLANG_LIB_ARCMIGRATE_TRANSFORMS_H
-
-#include "clang/AST/ParentMap.h"
-#include "clang/AST/RecursiveASTVisitor.h"
-#include "llvm/ADT/DenseSet.h"
-#include "llvm/Support/SaveAndRestore.h"
-
-namespace clang {
-  class Decl;
-  class Stmt;
-  class BlockDecl;
-  class ObjCMethodDecl;
-  class FunctionDecl;
-
-namespace arcmt {
-  class MigrationPass;
-
-namespace trans {
-
-  class MigrationContext;
-
-//===----------------------------------------------------------------------===//
-// Transformations.
-//===----------------------------------------------------------------------===//
-
-void rewriteAutoreleasePool(MigrationPass &pass);
-void rewriteUnbridgedCasts(MigrationPass &pass);
-void makeAssignARCSafe(MigrationPass &pass);
-void removeRetainReleaseDeallocFinalize(MigrationPass &pass);
-void removeZeroOutPropsInDeallocFinalize(MigrationPass &pass);
-void rewriteUnusedInitDelegate(MigrationPass &pass);
-void checkAPIUses(MigrationPass &pass);
-
-void removeEmptyStatementsAndDeallocFinalize(MigrationPass &pass);
-
-class BodyContext {
-  MigrationContext &MigrateCtx;
-  ParentMap PMap;
-  Stmt *TopStmt;
-
-public:
-  BodyContext(MigrationContext &MigrateCtx, Stmt *S)
-    : MigrateCtx(MigrateCtx), PMap(S), TopStmt(S) {}
-
-  MigrationContext &getMigrationContext() { return MigrateCtx; }
-  ParentMap &getParentMap() { return PMap; }
-  Stmt *getTopStmt() { return TopStmt; }
-};
-
-class ObjCImplementationContext {
-  MigrationContext &MigrateCtx;
-  ObjCImplementationDecl *ImpD;
-
-public:
-  ObjCImplementationContext(MigrationContext &MigrateCtx,
-                            ObjCImplementationDecl *D)
-    : MigrateCtx(MigrateCtx), ImpD(D) {}
-
-  MigrationContext &getMigrationContext() { return MigrateCtx; }
-  ObjCImplementationDecl *getImplementationDecl() { return ImpD; }
-};
-
-class ASTTraverser {
-public:
-  virtual ~ASTTraverser();
-  virtual void traverseTU(MigrationContext &MigrateCtx) { }
-  virtual void traverseBody(BodyContext &BodyCtx) { }
-  virtual void traverseObjCImplementation(ObjCImplementationContext &ImplCtx) {}
-};
-
-class MigrationContext {
-  std::vector<ASTTraverser *> Traversers;
-
-public:
-  MigrationPass &Pass;
-
-  struct GCAttrOccurrence {
-    enum AttrKind { Weak, Strong } Kind;
-    SourceLocation Loc;
-    QualType ModifiedType;
-    Decl *Dcl;
-    /// true if the attribute is owned, e.g. it is in a body and not just
-    /// in an interface.
-    bool FullyMigratable;
-  };
-  std::vector<GCAttrOccurrence> GCAttrs;
-  llvm::DenseSet<SourceLocation> AttrSet;
-  llvm::DenseSet<SourceLocation> RemovedAttrSet;
-
-  /// Set of raw '@' locations for 'assign' properties group that contain
-  /// GC __weak.
-  llvm::DenseSet<SourceLocation> AtPropsWeak;
-
-  explicit MigrationContext(MigrationPass &pass) : Pass(pass) {}
-  ~MigrationContext();
-
-  typedef std::vector<ASTTraverser *>::iterator traverser_iterator;
-  traverser_iterator traversers_begin() { return Traversers.begin(); }
-  traverser_iterator traversers_end() { return Traversers.end(); }
-
-  void addTraverser(ASTTraverser *traverser) {
-    Traversers.push_back(traverser);
-  }
-
-  bool isGCOwnedNonObjC(QualType T);
-  bool removePropertyAttribute(StringRef fromAttr, SourceLocation atLoc) {
-    return rewritePropertyAttribute(fromAttr, StringRef(), atLoc);
-  }
-  bool rewritePropertyAttribute(StringRef fromAttr, StringRef toAttr,
-                                SourceLocation atLoc);
-  bool addPropertyAttribute(StringRef attr, SourceLocation atLoc);
-
-  void traverse(TranslationUnitDecl *TU);
-
-  void dumpGCAttrs();
-};
-
-class PropertyRewriteTraverser : public ASTTraverser {
-public:
-  void traverseObjCImplementation(ObjCImplementationContext &ImplCtx) override;
-};
-
-class BlockObjCVariableTraverser : public ASTTraverser {
-public:
-  void traverseBody(BodyContext &BodyCtx) override;
-};
-
-class ProtectedScopeTraverser : public ASTTraverser {
-public:
-  void traverseBody(BodyContext &BodyCtx) override;
-};
-
-// GC transformations
-
-class GCAttrsTraverser : public ASTTraverser {
-public:
-  void traverseTU(MigrationContext &MigrateCtx) override;
-};
-
-class GCCollectableCallsTraverser : public ASTTraverser {
-public:
-  void traverseBody(BodyContext &BodyCtx) override;
-};
-
-//===----------------------------------------------------------------------===//
-// Helpers.
-//===----------------------------------------------------------------------===//
-
-/// Determine whether we can add weak to the given type.
-bool canApplyWeak(ASTContext &Ctx, QualType type,
-                  bool AllowOnUnknownClass = false);
-
-bool isPlusOneAssign(const BinaryOperator *E);
-bool isPlusOne(const Expr *E);
-
-/// 'Loc' is the end of a statement range. This returns the location
-/// immediately after the semicolon following the statement.
-/// If no semicolon is found or the location is inside a macro, the returned
-/// source location will be invalid.
-SourceLocation findLocationAfterSemi(SourceLocation loc, ASTContext &Ctx,
-                                     bool IsDecl = false);
-
-/// 'Loc' is the end of a statement range. This returns the location
-/// of the semicolon following the statement.
-/// If no semicolon is found or the location is inside a macro, the returned
-/// source location will be invalid.
-SourceLocation findSemiAfterLocation(SourceLocation loc, ASTContext &Ctx,
-                                     bool IsDecl = false);
-
-bool hasSideEffects(Expr *E, ASTContext &Ctx);
-bool isGlobalVar(Expr *E);
-/// Returns "nil" or "0" if 'nil' macro is not actually defined.
-StringRef getNilString(MigrationPass &Pass);
-
-template <typename BODY_TRANS>
-class BodyTransform : public RecursiveASTVisitor<BodyTransform<BODY_TRANS> > {
-  MigrationPass &Pass;
-  Decl *ParentD;
-
-  typedef RecursiveASTVisitor<BodyTransform<BODY_TRANS> > base;
-public:
-  BodyTransform(MigrationPass &pass) : Pass(pass), ParentD(nullptr) { }
-
-  bool TraverseStmt(Stmt *rootS) {
-    if (rootS)
-      BODY_TRANS(Pass).transformBody(rootS, ParentD);
-    return true;
-  }
-
-  bool TraverseObjCMethodDecl(ObjCMethodDecl *D) {
-    SaveAndRestore<Decl *> SetParent(ParentD, D);
-    return base::TraverseObjCMethodDecl(D);
-  }
-};
-
-typedef llvm::DenseSet<Expr *> ExprSet;
-
-void clearRefsIn(Stmt *S, ExprSet &refs);
-template <typename iterator>
-void clearRefsIn(iterator begin, iterator end, ExprSet &refs) {
-  for (; begin != end; ++begin)
-    clearRefsIn(*begin, refs);
-}
-
-void collectRefs(ValueDecl *D, Stmt *S, ExprSet &refs);
-
-void collectRemovables(Stmt *S, ExprSet &exprs);
-
-} // end namespace trans
-
-} // end namespace arcmt
-
-} // end namespace clang
-
-#endif
diff --git a/clang/lib/AST/ASTContext.cpp b/clang/lib/AST/ASTContext.cpp
index a4ba9fd055346..b1b9d56ccca9f 100644
--- a/clang/lib/AST/ASTContext.cpp
+++ b/clang/lib/AST/ASTContext.cpp
@@ -1055,7 +1055,8 @@ void ASTContext::PrintStats() const {
 void ASTContext::mergeDefinitionIntoModule(NamedDecl *ND, Module *M,
                                            bool NotifyListeners) {
   if (NotifyListeners)
-    if (auto *Listener = getASTMutationListener())
+    if (auto *Listener = getASTMutationListener();
+        Listener && !ND->isUnconditionallyVisible())
       Listener->RedefinedHiddenDefinition(ND, M);
 
   MergedDefModules[cast<NamedDecl>(ND->getCanonicalDecl())].push_back(M);
@@ -2269,11 +2270,10 @@ TypeInfo ASTContext::getTypeInfoImpl(const Type *T) const {
     Width = 0;                                                                 \
     Align = 16;                                                                \
     break;
-#define AARCH64_VECTOR_TYPE_MFLOAT(Name, MangledName, Id, SingletonId, NumEls, \
-                                   ElBits, NF)                                 \
+#define SVE_SCALAR_TYPE(Name, MangledName, Id, SingletonId, Bits)              \
   case BuiltinType::Id:                                                        \
-    Width = NumEls * ElBits * NF;                                              \
-    Align = NumEls * ElBits;                                                   \
+    Width = Bits;                                                              \
+    Align = Bits;                                                              \
     break;
 #include "clang/Basic/AArch64SVEACLETypes.def"
 #define PPC_VECTOR_TYPE(Name, Id, Size)                                        \
@@ -3902,7 +3902,7 @@ QualType ASTContext::getArrayParameterType(QualType Ty) const {
   if (Ty->isArrayParameterType())
     return Ty;
   assert(Ty->isConstantArrayType() && "Ty must be an array type.");
-  const auto *ATy = cast<ConstantArrayType>(Ty);
+  const auto *ATy = cast<ConstantArrayType>(Ty.getDesugaredType(*this));
   llvm::FoldingSetNodeID ID;
   ATy->Profile(ID, *this, ATy->getElementType(), ATy->getZExtSize(),
                ATy->getSizeExpr(), ATy->getSizeModifier(),
@@ -4423,15 +4423,14 @@ ASTContext::getBuiltinVectorTypeInfo(const BuiltinType *Ty) const {
                                ElBits, NF)                                     \
   case BuiltinType::Id:                                                        \
     return {BFloat16Ty, llvm::ElementCount::getScalable(NumEls), NF};
+#define SVE_VECTOR_TYPE_MFLOAT(Name, MangledName, Id, SingletonId, NumEls,     \
+                               ElBits, NF)                                     \
+  case BuiltinType::Id:                                                        \
+    return {MFloat8Ty, llvm::ElementCount::getScalable(NumEls), NF};
 #define SVE_PREDICATE_TYPE_ALL(Name, MangledName, Id, SingletonId, NumEls, NF) \
   case BuiltinType::Id:                                                        \
     return {BoolTy, llvm::ElementCount::getScalable(NumEls), NF};
-#define AARCH64_VECTOR_TYPE_MFLOAT(Name, MangledName, Id, SingletonId, NumEls, \
-                                   ElBits, NF)                                 \
-  case BuiltinType::Id:                                                        \
-    return {getIntTypeForBitwidth(ElBits, false),                              \
-            llvm::ElementCount::getFixed(NumEls), NF};
-#define SVE_OPAQUE_TYPE(Name, MangledName, Id, SingletonId)
+#define SVE_TYPE(Name, Id, SingletonId)
 #include "clang/Basic/AArch64SVEACLETypes.def"
 
 #define RVV_VECTOR_TYPE_INT(Name, Id, SingletonId, NumEls, ElBits, NF,         \
@@ -4493,11 +4492,16 @@ QualType ASTContext::getScalableVectorType(QualType EltTy, unsigned NumElts,
       EltTySize == ElBits && NumElts == (NumEls * NF) && NumFields == 1) {     \
     return SingletonId;                                                        \
   }
+#define SVE_VECTOR_TYPE_MFLOAT(Name, MangledName, Id, SingletonId, NumEls,     \
+                               ElBits, NF)                                     \
+  if (EltTy->isMFloat8Type() && EltTySize == ElBits &&                         \
+      NumElts == (NumEls * NF) && NumFields == 1) {                            \
+    return SingletonId;                                                        \
+  }
 #define SVE_PREDICATE_TYPE_ALL(Name, MangledName, Id, SingletonId, NumEls, NF) \
   if (EltTy->isBooleanType() && NumElts == (NumEls * NF) && NumFields == 1)    \
     return SingletonId;
-#define SVE_OPAQUE_TYPE(Name, MangledName, Id, SingletonId)
-#define AARCH64_VECTOR_TYPE(Name, MangledName, Id, SingletonId)
+#define SVE_TYPE(Name, Id, SingletonId)
 #include "clang/Basic/AArch64SVEACLETypes.def"
   } else if (Target->hasRISCVVTypes()) {
     uint64_t EltTySize = getTypeSize(EltTy);
@@ -5316,6 +5320,19 @@ bool ASTContext::computeBestEnumTypes(bool IsPacked, unsigned NumNegativeBits,
   return EnumTooLarge;
 }
 
+bool ASTContext::isRepresentableIntegerValue(llvm::APSInt &Value, QualType T) {
+  assert((T->isIntegralType(*this) || T->isEnumeralType()) &&
+         "Integral type required!");
+  unsigned BitWidth = getIntWidth(T);
+
+  if (Value.isUnsigned() || Value.isNonNegative()) {
+    if (T->isSignedIntegerOrEnumerationType())
+      --BitWidth;
+    return Value.getActiveBits() <= BitWidth;
+  }
+  return Value.getSignificantBits() <= BitWidth;
+}
+
 QualType ASTContext::getUnresolvedUsingType(
     const UnresolvedUsingTypenameDecl *Decl) const {
   if (Decl->TypeForDecl)
@@ -7221,6 +7238,16 @@ static bool isSameQualifier(const NestedNameSpecifier *X,
   return !PX && !PY;
 }
 
+static bool hasSameCudaAttrs(const FunctionDecl *A, const FunctionDecl *B) {
+  if (!A->getASTContext().getLangOpts().CUDA)
+    return true; // Target attributes are overloadable in CUDA compilation only.
+  if (A->hasAttr<CUDADeviceAttr>() != B->hasAttr<CUDADeviceAttr>())
+    return false;
+  if (A->hasAttr<CUDADeviceAttr>() && B->hasAttr<CUDADeviceAttr>())
+    return A->hasAttr<CUDAHostAttr>() == B->hasAttr<CUDAHostAttr>();
+  return true; // unattributed and __host__ functions are the same.
+}
+
 /// Determine whether the attributes we can overload on are identical for A and
 /// B. Will ignore any overloadable attrs represented in the type of A and B.
 static bool hasSameOverloadableAttrs(const FunctionDecl *A,
@@ -7251,7 +7278,7 @@ static bool hasSameOverloadableAttrs(const FunctionDecl *A,
     if (Cand1ID != Cand2ID)
       return false;
   }
-  return true;
+  return hasSameCudaAttrs(A, B);
 }
 
 bool ASTContext::isSameEntity(const NamedDecl *X, const NamedDecl *Y) const {
@@ -10360,7 +10387,8 @@ bool ASTContext::areLaxCompatibleSveTypes(QualType FirstType,
 /// getRVVTypeSize - Return RVV vector register size.
 static uint64_t getRVVTypeSize(ASTContext &Context, const BuiltinType *Ty) {
   assert(Ty->isRVVVLSBuiltinType() && "Invalid RVV Type");
-  auto VScale = Context.getTargetInfo().getVScaleRange(Context.getLangOpts());
+  auto VScale =
+      Context.getTargetInfo().getVScaleRange(Context.getLangOpts(), false);
   if (!VScale)
     return 0;
 
@@ -12382,6 +12410,9 @@ static QualType DecodeTypeFromStr(const char *&Str, const ASTContext &Context,
   case 'p':
     Type = Context.getProcessIDType();
     break;
+  case 'm':
+    Type = Context.MFloat8Ty;
+    break;
   }
 
   // If there are modifiers and if we're allowed to parse them, go for it.
@@ -12835,11 +12866,12 @@ bool ASTContext::DeclMustBeEmitted(const Decl *D) {
 
   // Likewise, variables with tuple-like bindings are required if their
   // bindings have side-effects.
-  if (const auto *DD = dyn_cast<DecompositionDecl>(VD))
-    for (const auto *BD : DD->bindings())
+  if (const auto *DD = dyn_cast<DecompositionDecl>(VD)) {
+    for (const auto *BD : DD->flat_bindings())
       if (const auto *BindingVD = BD->getHoldingVar())
         if (DeclMustBeEmitted(BindingVD))
           return true;
+  }
 
   return false;
 }
diff --git a/clang/lib/AST/ASTImporter.cpp b/clang/lib/AST/ASTImporter.cpp
index 0669aa1b809c3..c27ebbf838ad1 100644
--- a/clang/lib/AST/ASTImporter.cpp
+++ b/clang/lib/AST/ASTImporter.cpp
@@ -15,6 +15,7 @@
 #include "clang/AST/ASTContext.h"
 #include "clang/AST/ASTDiagnostic.h"
 #include "clang/AST/ASTImporterSharedState.h"
+#include "clang/AST/ASTLambda.h"
 #include "clang/AST/ASTStructuralEquivalence.h"
 #include "clang/AST/Attr.h"
 #include "clang/AST/Decl.h"
@@ -2552,7 +2553,7 @@ ExpectedDecl ASTNodeImporter::VisitBindingDecl(BindingDecl *D) {
 
   BindingDecl *ToD;
   if (GetImportedOrCreateDecl(ToD, D, Importer.getToContext(), DC, Loc,
-                              Name.getAsIdentifierInfo()))
+                              Name.getAsIdentifierInfo(), D->getType()))
     return ToD;
 
   Error Err = Error::success();
@@ -3729,10 +3730,7 @@ bool ASTNodeImporter::hasReturnTypeDeclaredInside(FunctionDecl *D) {
     if (Importer.FromContext.getLangOpts().CPlusPlus14) // C++14 or later
       return false;
 
-    if (const auto *MD = dyn_cast<CXXMethodDecl>(D))
-      return cast<CXXRecordDecl>(MD->getDeclContext())->isLambda();
-
-    return false;
+    return isLambdaMethod(D);
   };
 
   QualType RetT = FromFPT->getReturnType();
@@ -3999,14 +3997,16 @@ ExpectedDecl ASTNodeImporter::VisitFunctionDecl(FunctionDecl *D) {
         importExplicitSpecifier(Err, Guide->getExplicitSpecifier());
     CXXConstructorDecl *Ctor =
         importChecked(Err, Guide->getCorrespondingConstructor());
+    const CXXDeductionGuideDecl *SourceDG =
+        importChecked(Err, Guide->getSourceDeductionGuide());
     if (Err)
       return std::move(Err);
     if (GetImportedOrCreateDecl<CXXDeductionGuideDecl>(
             ToFunction, D, Importer.getToContext(), DC, ToInnerLocStart, ESpec,
-            NameInfo, T, TInfo, ToEndLoc, Ctor))
+            NameInfo, T, TInfo, ToEndLoc, Ctor,
+            Guide->getDeductionCandidateKind(), TrailingRequiresClause,
+            SourceDG, Guide->getSourceDeductionGuideKind()))
       return ToFunction;
-    cast<CXXDeductionGuideDecl>(ToFunction)
-        ->setDeductionCandidateKind(Guide->getDeductionCandidateKind());
   } else {
     if (GetImportedOrCreateDecl(
             ToFunction, D, Importer.getToContext(), DC, ToInnerLocStart,
@@ -4701,9 +4701,13 @@ ExpectedDecl ASTNodeImporter::VisitImplicitParamDecl(ImplicitParamDecl *D) {
 
 Error ASTNodeImporter::ImportDefaultArgOfParmVarDecl(
     const ParmVarDecl *FromParam, ParmVarDecl *ToParam) {
+
+  if (auto LocOrErr = import(FromParam->getExplicitObjectParamThisLoc()))
+    ToParam->setExplicitObjectParameterLoc(*LocOrErr);
+  else
+    return LocOrErr.takeError();
+
   ToParam->setHasInheritedDefaultArg(FromParam->hasInheritedDefaultArg());
-  ToParam->setExplicitObjectParameterLoc(
-      FromParam->getExplicitObjectParamThisLoc());
   ToParam->setKNRPromoted(FromParam->isKNRPromoted());
 
   if (FromParam->hasUninstantiatedDefaultArg()) {
@@ -6316,10 +6320,10 @@ ExpectedDecl ASTNodeImporter::VisitClassTemplateSpecializationDecl(
 
     updateLookupTableForTemplateParameters(*ToTPList);
   } else { // Not a partial specialization.
-    if (GetImportedOrCreateDecl(
-            D2, D, Importer.getToContext(), D->getTagKind(), DC,
-            *BeginLocOrErr, *IdLocOrErr, ClassTemplate, TemplateArgs,
-            PrevDecl))
+    if (GetImportedOrCreateDecl(D2, D, Importer.getToContext(), D->getTagKind(),
+                                DC, *BeginLocOrErr, *IdLocOrErr, ClassTemplate,
+                                TemplateArgs, D->hasStrictPackMatch(),
+                                PrevDecl))
       return D2;
 
     // Update InsertPos, because preceding import calls may have invalidated
@@ -6755,6 +6759,14 @@ ASTNodeImporter::VisitFunctionTemplateDecl(FunctionTemplateDecl *D) {
                               Params, TemplatedFD))
     return ToFunc;
 
+  // Fail if TemplatedFD is already part of a template.
+  // The template should have been found by structural equivalence check before,
+  // or ToFunc should be already imported.
+  // If not, there is AST incompatibility that can be caused by previous import
+  // errors. (NameConflict is not exact here.)
+  if (TemplatedFD->getDescribedTemplate())
+    return make_error<ASTImportError>(ASTImportError::NameConflict);
+
   TemplatedFD->setDescribedFunctionTemplate(ToFunc);
 
   ToFunc->setAccess(D->getAccess());
diff --git a/clang/lib/AST/ByteCode/ByteCodeEmitter.cpp b/clang/lib/AST/ByteCode/ByteCodeEmitter.cpp
index 19e2416c4c942..5bd1b73133d65 100644
--- a/clang/lib/AST/ByteCode/ByteCodeEmitter.cpp
+++ b/clang/lib/AST/ByteCode/ByteCodeEmitter.cpp
@@ -135,11 +135,9 @@ Function *ByteCodeEmitter::compileFunc(const FunctionDecl *FuncDecl) {
   // Create a handle over the emitted code.
   Function *Func = P.getFunction(FuncDecl);
   if (!Func) {
-    unsigned BuiltinID = FuncDecl->getBuiltinID();
-    Func =
-        P.createFunction(FuncDecl, ParamOffset, std::move(ParamTypes),
-                         std::move(ParamDescriptors), std::move(ParamOffsets),
-                         HasThisPointer, HasRVO, BuiltinID);
+    Func = P.createFunction(FuncDecl, ParamOffset, std::move(ParamTypes),
+                            std::move(ParamDescriptors),
+                            std::move(ParamOffsets), HasThisPointer, HasRVO);
   }
 
   assert(Func);
@@ -212,8 +210,7 @@ Function *ByteCodeEmitter::compileObjCBlock(const BlockExpr *BE) {
   Function *Func =
       P.createFunction(BE, ParamOffset, std::move(ParamTypes),
                        std::move(ParamDescriptors), std::move(ParamOffsets),
-                       /*HasThisPointer=*/false, /*HasRVO=*/false,
-                       /*IsUnevaluatedBuiltin=*/false);
+                       /*HasThisPointer=*/false, /*HasRVO=*/false);
 
   assert(Func);
   Func->setDefined(true);
diff --git a/clang/lib/AST/ByteCode/Compiler.cpp b/clang/lib/AST/ByteCode/Compiler.cpp
index 66ab27bdd13da..cf39209819ade 100644
--- a/clang/lib/AST/ByteCode/Compiler.cpp
+++ b/clang/lib/AST/ByteCode/Compiler.cpp
@@ -194,12 +194,12 @@ template <class Emitter> class StmtExprScope final {
 template <class Emitter>
 bool Compiler<Emitter>::VisitCastExpr(const CastExpr *CE) {
   const Expr *SubExpr = CE->getSubExpr();
-  switch (CE->getCastKind()) {
 
-  case CK_LValueToRValue: {
-    if (DiscardResult)
-      return this->discard(SubExpr);
+  if (DiscardResult)
+    return this->delegate(SubExpr);
 
+  switch (CE->getCastKind()) {
+  case CK_LValueToRValue: {
     std::optional<PrimType> SubExprT = classify(SubExpr->getType());
     // Prepare storage for the result.
     if (!Initializing && !SubExprT) {
@@ -253,9 +253,6 @@ bool Compiler<Emitter>::VisitCastExpr(const CastExpr *CE) {
 
   case CK_UncheckedDerivedToBase:
   case CK_DerivedToBase: {
-    if (DiscardResult)
-      return this->discard(SubExpr);
-
     if (!this->delegate(SubExpr))
       return false;
 
@@ -285,9 +282,6 @@ bool Compiler<Emitter>::VisitCastExpr(const CastExpr *CE) {
   }
 
   case CK_BaseToDerived: {
-    if (DiscardResult)
-      return this->discard(SubExpr);
-
     if (!this->delegate(SubExpr))
       return false;
 
@@ -302,8 +296,6 @@ bool Compiler<Emitter>::VisitCastExpr(const CastExpr *CE) {
     if (!SubExpr->getType()->isFloatingType() ||
         !CE->getType()->isFloatingType())
       return false;
-    if (DiscardResult)
-      return this->discard(SubExpr);
     if (!this->visit(SubExpr))
       return false;
     const auto *TargetSemantics = &Ctx.getFloatSemantics(CE->getType());
@@ -311,8 +303,6 @@ bool Compiler<Emitter>::VisitCastExpr(const CastExpr *CE) {
   }
 
   case CK_IntegralToFloating: {
-    if (DiscardResult)
-      return this->discard(SubExpr);
     std::optional<PrimType> FromT = classify(SubExpr->getType());
     if (!FromT)
       return false;
@@ -327,8 +317,6 @@ bool Compiler<Emitter>::VisitCastExpr(const CastExpr *CE) {
 
   case CK_FloatingToBoolean:
   case CK_FloatingToIntegral: {
-    if (DiscardResult)
-      return this->discard(SubExpr);
 
     std::optional<PrimType> ToT = classify(CE->getType());
 
@@ -352,9 +340,6 @@ bool Compiler<Emitter>::VisitCastExpr(const CastExpr *CE) {
   case CK_NullToMemberPointer: {
     if (!this->discard(SubExpr))
       return false;
-    if (DiscardResult)
-      return true;
-
     const Descriptor *Desc = nullptr;
     const QualType PointeeType = CE->getType()->getPointeeType();
     if (!PointeeType.isNull()) {
@@ -371,9 +356,6 @@ bool Compiler<Emitter>::VisitCastExpr(const CastExpr *CE) {
   }
 
   case CK_PointerToIntegral: {
-    if (DiscardResult)
-      return this->discard(SubExpr);
-
     if (!this->visit(SubExpr))
       return false;
 
@@ -399,8 +381,6 @@ bool Compiler<Emitter>::VisitCastExpr(const CastExpr *CE) {
       return false;
     if (!this->emitArrayDecay(CE))
       return false;
-    if (DiscardResult)
-      return this->emitPopPtr(CE);
     return true;
   }
 
@@ -412,9 +392,6 @@ bool Compiler<Emitter>::VisitCastExpr(const CastExpr *CE) {
     // FIXME: I think the discard is wrong since the int->ptr cast might cause a
     // diagnostic.
     PrimType T = classifyPrim(IntType);
-    if (DiscardResult)
-      return this->emitPop(T, CE);
-
     QualType PtrType = CE->getType();
     const Descriptor *Desc;
     if (std::optional<PrimType> T = classify(PtrType->getPointeeType()))
@@ -454,10 +431,6 @@ bool Compiler<Emitter>::VisitCastExpr(const CastExpr *CE) {
         return false;
       return this->emitInvalidCast(CastKind::Reinterpret, /*Fatal=*/true, CE);
     }
-
-    if (DiscardResult)
-      return this->discard(SubExpr);
-
     QualType SubExprTy = SubExpr->getType();
     std::optional<PrimType> FromT = classify(SubExprTy);
     // Casts from integer/vector to vector.
@@ -493,8 +466,6 @@ bool Compiler<Emitter>::VisitCastExpr(const CastExpr *CE) {
   case CK_FixedPointToBoolean:
   case CK_BooleanToSignedIntegral:
   case CK_IntegralCast: {
-    if (DiscardResult)
-      return this->discard(SubExpr);
     std::optional<PrimType> FromT = classify(SubExpr->getType());
     std::optional<PrimType> ToT = classify(CE->getType());
 
@@ -546,8 +517,6 @@ bool Compiler<Emitter>::VisitCastExpr(const CastExpr *CE) {
 
   case CK_IntegralComplexToBoolean:
   case CK_FloatingComplexToBoolean: {
-    if (DiscardResult)
-      return this->discard(SubExpr);
     if (!this->visit(SubExpr))
       return false;
     return this->emitComplexBoolCast(SubExpr);
@@ -562,8 +531,10 @@ bool Compiler<Emitter>::VisitCastExpr(const CastExpr *CE) {
     // We're creating a complex value here, so we need to
     // allocate storage for it.
     if (!Initializing) {
-      unsigned LocalIndex = allocateTemporary(CE);
-      if (!this->emitGetPtrLocal(LocalIndex, CE))
+      std::optional<unsigned> LocalIndex = allocateTemporary(CE);
+      if (!LocalIndex)
+        return false;
+      if (!this->emitGetPtrLocal(*LocalIndex, CE))
         return false;
     }
 
@@ -583,9 +554,6 @@ bool Compiler<Emitter>::VisitCastExpr(const CastExpr *CE) {
   case CK_FloatingComplexToIntegralComplex: {
     assert(CE->getType()->isAnyComplexType());
     assert(SubExpr->getType()->isAnyComplexType());
-    if (DiscardResult)
-      return this->discard(SubExpr);
-
     if (!Initializing) {
       std::optional<unsigned> LocalIndex = allocateLocal(CE);
       if (!LocalIndex)
@@ -631,9 +599,6 @@ bool Compiler<Emitter>::VisitCastExpr(const CastExpr *CE) {
     assert(classify(SubExpr->getType()));
     assert(CE->getType()->isVectorType());
 
-    if (DiscardResult)
-      return this->discard(SubExpr);
-
     if (!Initializing) {
       std::optional<unsigned> LocalIndex = allocateLocal(CE);
       if (!LocalIndex)
@@ -679,8 +644,10 @@ bool Compiler<Emitter>::VisitCastExpr(const CastExpr *CE) {
     assert(CE->getType()->isVectorType());
 
     if (!Initializing) {
-      unsigned LocalIndex = allocateTemporary(CE);
-      if (!this->emitGetPtrLocal(LocalIndex, CE))
+      std::optional<unsigned> LocalIndex = allocateTemporary(CE);
+      if (!LocalIndex)
+        return false;
+      if (!this->emitGetPtrLocal(*LocalIndex, CE))
         return false;
     }
     unsigned ToSize = CE->getType()->getAs<VectorType>()->getNumElements();
@@ -759,8 +726,10 @@ bool Compiler<Emitter>::VisitImaginaryLiteral(const ImaginaryLiteral *E) {
     return true;
 
   if (!Initializing) {
-    unsigned LocalIndex = allocateTemporary(E);
-    if (!this->emitGetPtrLocal(LocalIndex, E))
+    std::optional<unsigned> LocalIndex = allocateTemporary(E);
+    if (!LocalIndex)
+      return false;
+    if (!this->emitGetPtrLocal(*LocalIndex, E))
       return false;
   }
 
@@ -1118,8 +1087,10 @@ template <class Emitter>
 bool Compiler<Emitter>::VisitComplexBinOp(const BinaryOperator *E) {
   // Prepare storage for result.
   if (!Initializing) {
-    unsigned LocalIndex = allocateTemporary(E);
-    if (!this->emitGetPtrLocal(LocalIndex, E))
+    std::optional<unsigned> LocalIndex = allocateTemporary(E);
+    if (!LocalIndex)
+      return false;
+    if (!this->emitGetPtrLocal(*LocalIndex, E))
       return false;
   }
 
@@ -1175,7 +1146,10 @@ bool Compiler<Emitter>::VisitComplexBinOp(const BinaryOperator *E) {
 
     if (!LHSIsComplex) {
       // This is using the RHS type for the fake-complex LHS.
-      LHSOffset = allocateTemporary(RHS);
+      std::optional<unsigned> LocalIndex = allocateTemporary(RHS);
+      if (!LocalIndex)
+        return false;
+      LHSOffset = *LocalIndex;
 
       if (!this->emitGetPtrLocal(LHSOffset, E))
         return false;
@@ -1347,8 +1321,10 @@ bool Compiler<Emitter>::VisitVectorBinOp(const BinaryOperator *E) {
 
   // Prepare storage for result.
   if (!Initializing && !E->isCompoundAssignmentOp()) {
-    unsigned LocalIndex = allocateTemporary(E);
-    if (!this->emitGetPtrLocal(LocalIndex, E))
+    std::optional<unsigned> LocalIndex = allocateTemporary(E);
+    if (!LocalIndex)
+      return false;
+    if (!this->emitGetPtrLocal(*LocalIndex, E))
       return false;
   }
 
@@ -3357,15 +3333,23 @@ bool Compiler<Emitter>::VisitCXXNewExpr(const CXXNewExpr *E) {
 
     PrimType SizeT = classifyPrim(Stripped->getType());
 
+    // Save evaluated array size to a variable.
+    unsigned ArrayLen = allocateLocalPrimitive(
+        Stripped, SizeT, /*IsConst=*/false, /*IsExtended=*/false);
+    if (!this->visit(Stripped))
+      return false;
+    if (!this->emitSetLocal(SizeT, ArrayLen, E))
+      return false;
+
     if (PlacementDest) {
       if (!this->visit(PlacementDest))
         return false;
-      if (!this->visit(Stripped))
+      if (!this->emitGetLocal(SizeT, ArrayLen, E))
         return false;
       if (!this->emitCheckNewTypeMismatchArray(SizeT, E, E))
         return false;
     } else {
-      if (!this->visit(Stripped))
+      if (!this->emitGetLocal(SizeT, ArrayLen, E))
         return false;
 
       if (ElemT) {
@@ -3379,10 +3363,113 @@ bool Compiler<Emitter>::VisitCXXNewExpr(const CXXNewExpr *E) {
       }
     }
 
-    if (Init && !this->visitInitializer(Init))
-      return false;
+    if (Init) {
+      QualType InitType = Init->getType();
+      size_t StaticInitElems = 0;
+      const Expr *DynamicInit = nullptr;
+      if (const ConstantArrayType *CAT =
+              Ctx.getASTContext().getAsConstantArrayType(InitType)) {
+        StaticInitElems = CAT->getZExtSize();
+        if (!this->visitInitializer(Init))
+          return false;
 
-  } else {
+        if (const auto *ILE = dyn_cast<InitListExpr>(Init);
+            ILE && ILE->hasArrayFiller())
+          DynamicInit = ILE->getArrayFiller();
+      }
+
+      // The initializer initializes a certain number of elements, S.
+      // However, the complete number of elements, N, might be larger than that.
+      // In this case, we need to get an initializer for the remaining elements.
+      // There are to cases:
+      //   1) For the form 'new Struct[n];', the initializer is a
+      //      CXXConstructExpr and its type is an IncompleteArrayType.
+      //   2) For the form 'new Struct[n]{1,2,3}', the initializer is an
+      //      InitListExpr and the initializer for the remaining elements
+      //      is the array filler.
+
+      if (DynamicInit || InitType->isIncompleteArrayType()) {
+        const Function *CtorFunc = nullptr;
+        if (const auto *CE = dyn_cast<CXXConstructExpr>(Init)) {
+          CtorFunc = getFunction(CE->getConstructor());
+          if (!CtorFunc)
+            return false;
+        }
+
+        LabelTy EndLabel = this->getLabel();
+        LabelTy StartLabel = this->getLabel();
+
+        // In the nothrow case, the alloc above might have returned nullptr.
+        // Don't call any constructors that case.
+        if (IsNoThrow) {
+          if (!this->emitDupPtr(E))
+            return false;
+          if (!this->emitNullPtr(0, nullptr, E))
+            return false;
+          if (!this->emitEQPtr(E))
+            return false;
+          if (!this->jumpTrue(EndLabel))
+            return false;
+        }
+
+        // Create loop variables.
+        unsigned Iter = allocateLocalPrimitive(
+            Stripped, SizeT, /*IsConst=*/false, /*IsExtended=*/false);
+        if (!this->emitConst(StaticInitElems, SizeT, E))
+          return false;
+        if (!this->emitSetLocal(SizeT, Iter, E))
+          return false;
+
+        this->fallthrough(StartLabel);
+        this->emitLabel(StartLabel);
+        // Condition. Iter < ArrayLen?
+        if (!this->emitGetLocal(SizeT, Iter, E))
+          return false;
+        if (!this->emitGetLocal(SizeT, ArrayLen, E))
+          return false;
+        if (!this->emitLT(SizeT, E))
+          return false;
+        if (!this->jumpFalse(EndLabel))
+          return false;
+
+        // Pointer to the allocated array is already on the stack.
+        if (!this->emitGetLocal(SizeT, Iter, E))
+          return false;
+        if (!this->emitArrayElemPtr(SizeT, E))
+          return false;
+
+        if (DynamicInit) {
+          if (std::optional<PrimType> InitT = classify(DynamicInit)) {
+            if (!this->visit(DynamicInit))
+              return false;
+            if (!this->emitStorePop(*InitT, E))
+              return false;
+          } else {
+            if (!this->visitInitializer(DynamicInit))
+              return false;
+            if (!this->emitPopPtr(E))
+              return false;
+          }
+        } else {
+          assert(CtorFunc);
+          if (!this->emitCall(CtorFunc, 0, E))
+            return false;
+        }
+
+        // ++Iter;
+        if (!this->emitGetPtrLocal(Iter, E))
+          return false;
+        if (!this->emitIncPop(SizeT, E))
+          return false;
+
+        if (!this->jump(StartLabel))
+          return false;
+
+        this->fallthrough(EndLabel);
+        this->emitLabel(EndLabel);
+      }
+    }
+  } else { // Non-array.
     if (PlacementDest) {
       if (!this->visit(PlacementDest))
         return false;
@@ -4170,14 +4257,16 @@ Compiler<Emitter>::allocateLocal(DeclTy &&Src, QualType Ty,
 }
 
 template <class Emitter>
-unsigned Compiler<Emitter>::allocateTemporary(const Expr *E) {
+std::optional<unsigned> Compiler<Emitter>::allocateTemporary(const Expr *E) {
   QualType Ty = E->getType();
   assert(!Ty->isRecordType());
 
   Descriptor *D = P.createDescriptor(
       E, Ty.getTypePtr(), Descriptor::InlineDescMD, Ty.isConstQualified(),
       /*IsTemporary=*/true, /*IsMutable=*/false, /*Init=*/nullptr);
-  assert(D);
+
+  if (!D)
+    return std::nullopt;
 
   Scope::Local Local = this->createLocal(D);
   VariableScope<Emitter> *S = VarScope;
@@ -4661,7 +4750,7 @@ bool Compiler<Emitter>::VisitCallExpr(const CallExpr *E) {
       OCE && OCE->isAssignmentOp()) {
     // Just like with regular assignments, we need to special-case assignment
     // operators here and evaluate the RHS (the second arg) before the LHS (the
-    // first arg. We fix this by using a Flip op later.
+    // first arg). We fix this by using a Flip op later.
     assert(Args.size() == 2);
     IsAssignmentOperatorCall = true;
     std::reverse(Args.begin(), Args.end());
@@ -4700,6 +4789,14 @@ bool Compiler<Emitter>::VisitCallExpr(const CallExpr *E) {
     } else if (!this->visit(MC->getImplicitObjectArgument())) {
       return false;
     }
+  } else if (const auto *PD =
+                 dyn_cast<CXXPseudoDestructorExpr>(E->getCallee())) {
+    const Expr *Base = PD->getBase();
+    if (!Base->isGLValue())
+      return this->discard(Base);
+    if (!this->visit(Base))
+      return false;
+    return this->emitKill(E);
   } else if (!FuncDecl) {
     const Expr *Callee = E->getCallee();
     CalleeOffset = this->allocateLocalPrimitive(Callee, PT_FnPtr, true, false);
@@ -4976,8 +5073,8 @@ bool Compiler<Emitter>::visitCompoundStmt(const CompoundStmt *S) {
 template <class Emitter>
 bool Compiler<Emitter>::visitDeclStmt(const DeclStmt *DS) {
   for (const auto *D : DS->decls()) {
-    if (isa<StaticAssertDecl, TagDecl, TypedefNameDecl, UsingEnumDecl,
-            FunctionDecl>(D))
+    if (isa<StaticAssertDecl, TagDecl, TypedefNameDecl, BaseUsingDecl,
+            FunctionDecl, NamespaceAliasDecl>(D))
       continue;
 
     const auto *VD = dyn_cast<VarDecl>(D);
@@ -5581,6 +5678,22 @@ bool Compiler<Emitter>::compileConstructor(const CXXConstructorDecl *Ctor) {
 
       if (!emitFieldInitializer(NestedField, NestedFieldOffset, InitExpr))
         return false;
+
+      // Mark all chain links as initialized.
+      unsigned InitFieldOffset = 0;
+      for (const NamedDecl *ND : IFD->chain().drop_back()) {
+        const auto *FD = cast<FieldDecl>(ND);
+        const Record *FieldRecord = this->P.getOrCreateRecord(FD->getParent());
+        assert(FieldRecord);
+        NestedField = FieldRecord->getField(FD);
+        InitFieldOffset += NestedField->Offset;
+        assert(NestedField);
+        if (!this->emitGetPtrThisField(InitFieldOffset, InitExpr))
+          return false;
+        if (!this->emitFinishInitPop(InitExpr))
+          return false;
+      }
+
     } else {
       assert(Init->isDelegatingInitializer());
       if (!this->emitThis(InitExpr))
@@ -5649,6 +5762,21 @@ bool Compiler<Emitter>::compileDestructor(const CXXDestructorDecl *Dtor) {
   return this->emitPopPtr(Dtor) && this->emitRetVoid(Dtor);
 }
 
+template <class Emitter>
+bool Compiler<Emitter>::compileUnionAssignmentOperator(
+    const CXXMethodDecl *MD) {
+  if (!this->emitThis(MD))
+    return false;
+
+  auto PVD = MD->getParamDecl(0);
+  ParamOffset PO = this->Params[PVD]; // Must exist.
+
+  if (!this->emitGetParam(PT_Ptr, PO.Offset, MD))
+    return false;
+
+  return this->emitMemcpy(MD) && this->emitRet(PT_Ptr, MD);
+}
+
 template <class Emitter>
 bool Compiler<Emitter>::visitFunc(const FunctionDecl *F) {
   // Classify the return type.
@@ -5660,9 +5788,16 @@ bool Compiler<Emitter>::visitFunc(const FunctionDecl *F) {
     return this->compileDestructor(Dtor);
 
   // Emit custom code if this is a lambda static invoker.
-  if (const auto *MD = dyn_cast<CXXMethodDecl>(F);
-      MD && MD->isLambdaStaticInvoker())
-    return this->emitLambdaStaticInvokerBody(MD);
+  if (const auto *MD = dyn_cast<CXXMethodDecl>(F)) {
+    const RecordDecl *RD = MD->getParent();
+
+    if (RD->isUnion() &&
+        (MD->isCopyAssignmentOperator() || MD->isMoveAssignmentOperator()))
+      return this->compileUnionAssignmentOperator(MD);
+
+    if (MD->isLambdaStaticInvoker())
+      return this->emitLambdaStaticInvokerBody(MD);
+  }
 
   // Regular functions.
   if (const auto *Body = F->getBody())
@@ -6173,9 +6308,6 @@ bool Compiler<Emitter>::visitDeclRef(const ValueDecl *D, const Expr *E) {
 
       return this->emitGetPtrParam(It->second.Offset, E);
     }
-
-    if (D->getType()->isReferenceType())
-      return false; // FIXME: Do we need to emit InvalidDeclRef?
   }
 
   // In case we need to re-visit a declaration.
diff --git a/clang/lib/AST/ByteCode/Compiler.h b/clang/lib/AST/ByteCode/Compiler.h
index f9a597a16ef4a..0a93c46a40ef5 100644
--- a/clang/lib/AST/ByteCode/Compiler.h
+++ b/clang/lib/AST/ByteCode/Compiler.h
@@ -309,7 +309,7 @@ class Compiler : public ConstStmtVisitor<Compiler<Emitter>, bool>,
   std::optional<unsigned>
   allocateLocal(DeclTy &&Decl, QualType Ty = QualType(),
                 const ValueDecl *ExtendingDecl = nullptr);
-  unsigned allocateTemporary(const Expr *E);
+  std::optional<unsigned> allocateTemporary(const Expr *E);
 
 private:
   friend class VariableScope<Emitter>;
@@ -383,6 +383,7 @@ class Compiler : public ConstStmtVisitor<Compiler<Emitter>, bool>,
   bool emitBuiltinBitCast(const CastExpr *E);
   bool compileConstructor(const CXXConstructorDecl *Ctor);
   bool compileDestructor(const CXXDestructorDecl *Dtor);
+  bool compileUnionAssignmentOperator(const CXXMethodDecl *MD);
 
   bool checkLiteralType(const Expr *E);
 
diff --git a/clang/lib/AST/ByteCode/Context.cpp b/clang/lib/AST/ByteCode/Context.cpp
index b52892cf69bfb..aa434d5c85921 100644
--- a/clang/lib/AST/ByteCode/Context.cpp
+++ b/clang/lib/AST/ByteCode/Context.cpp
@@ -27,10 +27,7 @@ Context::~Context() {}
 
 bool Context::isPotentialConstantExpr(State &Parent, const FunctionDecl *FD) {
   assert(Stk.empty());
-  Function *Func = P->getFunction(FD);
-  if (!Func || !Func->hasBody())
-    Func = Compiler<ByteCodeEmitter>(*this, *P).compileFunc(FD);
-
+  const Function *Func = getOrCreateFunction(FD);
   if (!Func)
     return false;
 
@@ -215,14 +212,11 @@ const llvm::fltSemantics &Context::getFloatSemantics(QualType T) const {
 bool Context::Run(State &Parent, const Function *Func) {
 
   {
-    InterpState State(Parent, *P, Stk, *this);
-    State.Current = new InterpFrame(State, Func, /*Caller=*/nullptr, CodePtr(),
-                                    Func->getArgSize());
+    InterpState State(Parent, *P, Stk, *this, Func);
     if (Interpret(State)) {
       assert(Stk.empty());
       return true;
     }
-
     // State gets destroyed here, so the Stk.clear() below doesn't accidentally
     // remove values the State's destructor might access.
   }
@@ -271,6 +265,7 @@ Context::getOverridingFunction(const CXXRecordDecl *DynamicDecl,
 
 const Function *Context::getOrCreateFunction(const FunctionDecl *FD) {
   assert(FD);
+  FD = FD->getMostRecentDecl();
   const Function *Func = P->getFunction(FD);
   bool IsBeingCompiled = Func && Func->isDefined() && !Func->isFullyCompiled();
   bool WasNotDefined = Func && !Func->isConstexpr() && !Func->isDefined();
diff --git a/clang/lib/AST/ByteCode/Descriptor.cpp b/clang/lib/AST/ByteCode/Descriptor.cpp
index 1c16c2022dd02..319d1690c1cd0 100644
--- a/clang/lib/AST/ByteCode/Descriptor.cpp
+++ b/clang/lib/AST/ByteCode/Descriptor.cpp
@@ -409,7 +409,8 @@ QualType Descriptor::getElemQualType() const {
   assert(isArray());
   QualType T = getType();
   if (T->isPointerOrReferenceType())
-    return T->getPointeeType();
+    T = T->getPointeeType();
+
   if (const auto *AT = T->getAsArrayTypeUnsafe()) {
     // For primitive arrays, we don't save a QualType at all,
     // just a PrimType. Try to figure out the QualType here.
@@ -424,7 +425,8 @@ QualType Descriptor::getElemQualType() const {
     return CT->getElementType();
   if (const auto *CT = T->getAs<VectorType>())
     return CT->getElementType();
-  llvm_unreachable("Array that's not an array/complex/vector type?");
+
+  return T;
 }
 
 SourceLocation Descriptor::getLocation() const {
diff --git a/clang/lib/AST/ByteCode/Descriptor.h b/clang/lib/AST/ByteCode/Descriptor.h
index a73e28d2e600e..96c82a18913e0 100644
--- a/clang/lib/AST/ByteCode/Descriptor.h
+++ b/clang/lib/AST/ByteCode/Descriptor.h
@@ -61,6 +61,11 @@ struct alignas(void *) GlobalInlineDescriptor {
 };
 static_assert(sizeof(GlobalInlineDescriptor) == sizeof(void *), "");
 
+enum class Lifetime : uint8_t {
+  Started,
+  Ended,
+};
+
 /// Inline descriptor embedded in structures and arrays.
 ///
 /// Such descriptors precede all composite array elements and structure fields.
@@ -100,12 +105,14 @@ struct InlineDescriptor {
   LLVM_PREFERRED_TYPE(bool)
   unsigned IsArrayElement : 1;
 
+  Lifetime LifeState;
+
   const Descriptor *Desc;
 
   InlineDescriptor(const Descriptor *D)
       : Offset(sizeof(InlineDescriptor)), IsConst(false), IsInitialized(false),
         IsBase(false), IsActive(false), IsFieldMutable(false),
-        IsArrayElement(false), Desc(D) {}
+        IsArrayElement(false), LifeState(Lifetime::Started), Desc(D) {}
 
   void dump() const { dump(llvm::errs()); }
   void dump(llvm::raw_ostream &OS) const;
diff --git a/clang/lib/AST/ByteCode/Disasm.cpp b/clang/lib/AST/ByteCode/Disasm.cpp
index 3c55c884a3507..92a169a37c365 100644
--- a/clang/lib/AST/ByteCode/Disasm.cpp
+++ b/clang/lib/AST/ByteCode/Disasm.cpp
@@ -240,7 +240,7 @@ LLVM_DUMP_METHOD void Descriptor::dump(llvm::raw_ostream &OS) const {
   else if (isRecord())
     OS << " record";
   else if (isPrimitive())
-    OS << " primitive";
+    OS << " primitive " << primTypeToString(getPrimType());
 
   if (isZeroSizeArray())
     OS << " zero-size-array";
diff --git a/clang/lib/AST/ByteCode/EvalEmitter.cpp b/clang/lib/AST/ByteCode/EvalEmitter.cpp
index 9763fe89b7374..95149efbea992 100644
--- a/clang/lib/AST/ByteCode/EvalEmitter.cpp
+++ b/clang/lib/AST/ByteCode/EvalEmitter.cpp
@@ -17,11 +17,7 @@ using namespace clang::interp;
 
 EvalEmitter::EvalEmitter(Context &Ctx, Program &P, State &Parent,
                          InterpStack &Stk)
-    : Ctx(Ctx), P(P), S(Parent, P, Stk, Ctx, this), EvalResult(&Ctx) {
-  // Create a dummy frame for the interpreter which does not have locals.
-  S.Current =
-      new InterpFrame(S, /*Func=*/nullptr, /*Caller=*/nullptr, CodePtr(), 0);
-}
+    : Ctx(Ctx), P(P), S(Parent, P, Stk, Ctx, this), EvalResult(&Ctx) {}
 
 EvalEmitter::~EvalEmitter() {
   for (auto &[K, V] : Locals) {
@@ -174,6 +170,9 @@ template <> bool EvalEmitter::emitRet<PT_Ptr>(const SourceInfo &Info) {
       return false;
     }
   } else {
+    if (!Ptr.isLive() && !Ptr.isTemporary())
+      return false;
+
     EvalResult.setValue(Ptr.toAPValue(Ctx.getASTContext()));
   }
 
diff --git a/clang/lib/AST/ByteCode/EvalEmitter.h b/clang/lib/AST/ByteCode/EvalEmitter.h
index e7c9e80d75d93..2cac2ba2ef221 100644
--- a/clang/lib/AST/ByteCode/EvalEmitter.h
+++ b/clang/lib/AST/ByteCode/EvalEmitter.h
@@ -17,7 +17,6 @@
 #include "InterpState.h"
 #include "PrimType.h"
 #include "Source.h"
-#include "llvm/Support/Error.h"
 
 namespace clang {
 namespace interp {
@@ -42,8 +41,6 @@ class EvalEmitter : public SourceMapper {
   /// Clean up all resources.
   void cleanup();
 
-  InterpState &getState() { return S; }
-
 protected:
   EvalEmitter(Context &Ctx, Program &P, State &Parent, InterpStack &Stk);
 
diff --git a/clang/lib/AST/ByteCode/Function.cpp b/clang/lib/AST/ByteCode/Function.cpp
index 896a4fb3f9469..6b892dfd616c1 100644
--- a/clang/lib/AST/ByteCode/Function.cpp
+++ b/clang/lib/AST/ByteCode/Function.cpp
@@ -19,12 +19,28 @@ Function::Function(Program &P, FunctionDeclTy Source, unsigned ArgSize,
                    llvm::SmallVectorImpl<PrimType> &&ParamTypes,
                    llvm::DenseMap<unsigned, ParamDescriptor> &&Params,
                    llvm::SmallVectorImpl<unsigned> &&ParamOffsets,
-                   bool HasThisPointer, bool HasRVO, unsigned BuiltinID)
-    : P(P), Source(Source), ArgSize(ArgSize), ParamTypes(std::move(ParamTypes)),
-      Params(std::move(Params)), ParamOffsets(std::move(ParamOffsets)),
-      HasThisPointer(HasThisPointer), HasRVO(HasRVO), BuiltinID(BuiltinID) {
-  if (const auto *F = Source.dyn_cast<const FunctionDecl *>())
+                   bool HasThisPointer, bool HasRVO)
+    : P(P), Kind(FunctionKind::Normal), Source(Source), ArgSize(ArgSize),
+      ParamTypes(std::move(ParamTypes)), Params(std::move(Params)),
+      ParamOffsets(std::move(ParamOffsets)), HasThisPointer(HasThisPointer),
+      HasRVO(HasRVO) {
+  if (const auto *F = dyn_cast<const FunctionDecl *>(Source)) {
     Variadic = F->isVariadic();
+    BuiltinID = F->getBuiltinID();
+    if (const auto *CD = dyn_cast<CXXConstructorDecl>(F)) {
+      Virtual = CD->isVirtual();
+      Kind = FunctionKind::Ctor;
+    } else if (const auto *CD = dyn_cast<CXXDestructorDecl>(F)) {
+      Virtual = CD->isVirtual();
+      Kind = FunctionKind::Dtor;
+    } else if (const auto *MD = dyn_cast<CXXMethodDecl>(F)) {
+      Virtual = MD->isVirtual();
+      if (MD->isLambdaStaticInvoker())
+        Kind = FunctionKind::LambdaStaticInvoker;
+      else if (clang::isLambdaCallOperator(F))
+        Kind = FunctionKind::LambdaCallOperator;
+    }
+  }
 }
 
 Function::ParamDescriptor Function::getParamDescriptor(unsigned Offset) const {
@@ -45,13 +61,6 @@ SourceInfo Function::getSource(CodePtr PC) const {
   return It->second;
 }
 
-bool Function::isVirtual() const {
-  if (const auto *M = dyn_cast_if_present<CXXMethodDecl>(
-          Source.dyn_cast<const FunctionDecl *>()))
-    return M->isVirtual();
-  return false;
-}
-
 /// Unevaluated builtins don't get their arguments put on the stack
 /// automatically. They instead operate on the AST of their Call
 /// Expression.
diff --git a/clang/lib/AST/ByteCode/Function.h b/clang/lib/AST/ByteCode/Function.h
index 409a80f59f1e9..e17183eef9eac 100644
--- a/clang/lib/AST/ByteCode/Function.h
+++ b/clang/lib/AST/ByteCode/Function.h
@@ -51,6 +51,11 @@ class Scope final {
     return llvm::make_range(Descriptors.begin(), Descriptors.end());
   }
 
+  llvm::iterator_range<LocalVectorTy::const_reverse_iterator>
+  locals_reverse() const {
+    return llvm::reverse(Descriptors);
+  }
+
 private:
   /// Object descriptors in this block.
   LocalVectorTy Descriptors;
@@ -80,6 +85,13 @@ using FunctionDeclTy =
 ///
 class Function final {
 public:
+  enum class FunctionKind {
+    Normal,
+    Ctor,
+    Dtor,
+    LambdaStaticInvoker,
+    LambdaCallOperator,
+  };
   using ParamDescriptor = std::pair<PrimType, Descriptor *>;
 
   /// Returns the size of the function's local stack.
@@ -141,43 +153,31 @@ class Function final {
   bool isConstexpr() const { return IsValid || isLambdaStaticInvoker(); }
 
   /// Checks if the function is virtual.
-  bool isVirtual() const;
+  bool isVirtual() const { return Virtual; };
 
   /// Checks if the function is a constructor.
-  bool isConstructor() const {
-    return isa_and_nonnull<CXXConstructorDecl>(
-        dyn_cast<const FunctionDecl *>(Source));
-  }
+  bool isConstructor() const { return Kind == FunctionKind::Ctor; }
   /// Checks if the function is a destructor.
-  bool isDestructor() const {
-    return isa_and_nonnull<CXXDestructorDecl>(
-        dyn_cast<const FunctionDecl *>(Source));
-  }
-
-  /// Returns the parent record decl, if any.
-  const CXXRecordDecl *getParentDecl() const {
-    if (const auto *MD = dyn_cast_if_present<CXXMethodDecl>(
-            dyn_cast<const FunctionDecl *>(Source)))
-      return MD->getParent();
-    return nullptr;
-  }
+  bool isDestructor() const { return Kind == FunctionKind::Dtor; }
 
   /// Returns whether this function is a lambda static invoker,
   /// which we generate custom byte code for.
   bool isLambdaStaticInvoker() const {
-    if (const auto *MD = dyn_cast_if_present<CXXMethodDecl>(
-            dyn_cast<const FunctionDecl *>(Source)))
-      return MD->isLambdaStaticInvoker();
-    return false;
+    return Kind == FunctionKind::LambdaStaticInvoker;
   }
 
   /// Returns whether this function is the call operator
   /// of a lambda record decl.
   bool isLambdaCallOperator() const {
+    return Kind == FunctionKind::LambdaCallOperator;
+  }
+
+  /// Returns the parent record decl, if any.
+  const CXXRecordDecl *getParentDecl() const {
     if (const auto *MD = dyn_cast_if_present<CXXMethodDecl>(
             dyn_cast<const FunctionDecl *>(Source)))
-      return clang::isLambdaCallOperator(MD);
-    return false;
+      return MD->getParent();
+    return nullptr;
   }
 
   /// Checks if the function is fully done compiling.
@@ -213,7 +213,7 @@ class Function final {
 
   bool isThisPointerExplicit() const {
     if (const auto *MD = dyn_cast_if_present<CXXMethodDecl>(
-            Source.dyn_cast<const FunctionDecl *>()))
+            dyn_cast<const FunctionDecl *>(Source)))
       return MD->isExplicitObjectMemberFunction();
     return false;
   }
@@ -232,7 +232,7 @@ class Function final {
            llvm::SmallVectorImpl<PrimType> &&ParamTypes,
            llvm::DenseMap<unsigned, ParamDescriptor> &&Params,
            llvm::SmallVectorImpl<unsigned> &&ParamOffsets, bool HasThisPointer,
-           bool HasRVO, unsigned BuiltinID);
+           bool HasRVO);
 
   /// Sets the code of a function.
   void setCode(unsigned NewFrameSize, std::vector<std::byte> &&NewCode,
@@ -255,6 +255,8 @@ class Function final {
 
   /// Program reference.
   Program &P;
+  /// Function Kind.
+  FunctionKind Kind;
   /// Declaration this function was compiled from.
   FunctionDeclTy Source;
   /// Local area size: storage + metadata.
@@ -289,6 +291,7 @@ class Function final {
   bool HasBody = false;
   bool Defined = false;
   bool Variadic = false;
+  bool Virtual = false;
   unsigned BuiltinID = 0;
 
 public:
diff --git a/clang/lib/AST/ByteCode/Interp.cpp b/clang/lib/AST/ByteCode/Interp.cpp
index 40fe7147a18a3..c80be094856b0 100644
--- a/clang/lib/AST/ByteCode/Interp.cpp
+++ b/clang/lib/AST/ByteCode/Interp.cpp
@@ -65,14 +65,17 @@ static void diagnoseNonConstVariable(InterpState &S, CodePtr OpPC,
                                      const ValueDecl *VD);
 static bool diagnoseUnknownDecl(InterpState &S, CodePtr OpPC,
                                 const ValueDecl *D) {
-  const SourceInfo &E = S.Current->getSource(OpPC);
 
   if (isa<ParmVarDecl>(D)) {
+    if (D->getType()->isReferenceType())
+      return false;
+
+    const SourceInfo &Loc = S.Current->getSource(OpPC);
     if (S.getLangOpts().CPlusPlus11) {
-      S.FFDiag(E, diag::note_constexpr_function_param_value_unknown) << D;
+      S.FFDiag(Loc, diag::note_constexpr_function_param_value_unknown) << D;
       S.Note(D->getLocation(), diag::note_declared_at) << D->getSourceRange();
     } else {
-      S.FFDiag(E);
+      S.FFDiag(Loc);
     }
     return false;
   }
@@ -558,6 +561,18 @@ bool CheckInitialized(InterpState &S, CodePtr OpPC, const Pointer &Ptr,
   return false;
 }
 
+static bool CheckLifetime(InterpState &S, CodePtr OpPC, const Pointer &Ptr,
+                          AccessKinds AK) {
+  if (Ptr.getLifetime() == Lifetime::Started)
+    return true;
+
+  if (!S.checkingPotentialConstantExpression()) {
+    S.FFDiag(S.Current->getSource(OpPC), diag::note_constexpr_access_uninit)
+        << AK << /*uninitialized=*/false << S.Current->getRange(OpPC);
+  }
+  return false;
+}
+
 bool CheckGlobalInitialized(InterpState &S, CodePtr OpPC, const Pointer &Ptr) {
   if (Ptr.isInitialized())
     return true;
@@ -602,6 +617,8 @@ bool CheckLoad(InterpState &S, CodePtr OpPC, const Pointer &Ptr,
     return false;
   if (!CheckActive(S, OpPC, Ptr, AK))
     return false;
+  if (!CheckLifetime(S, OpPC, Ptr, AK))
+    return false;
   if (!CheckInitialized(S, OpPC, Ptr, AK))
     return false;
   if (!CheckTemporary(S, OpPC, Ptr, AK))
@@ -631,6 +648,8 @@ bool CheckFinalLoad(InterpState &S, CodePtr OpPC, const Pointer &Ptr) {
     return false;
   if (!CheckActive(S, OpPC, Ptr, AK_Read))
     return false;
+  if (!CheckLifetime(S, OpPC, Ptr, AK_Read))
+    return false;
   if (!CheckInitialized(S, OpPC, Ptr, AK_Read))
     return false;
   if (!CheckTemporary(S, OpPC, Ptr, AK_Read))
@@ -647,6 +666,8 @@ bool CheckStore(InterpState &S, CodePtr OpPC, const Pointer &Ptr) {
     return false;
   if (!CheckDummy(S, OpPC, Ptr, AK_Assign))
     return false;
+  if (!CheckLifetime(S, OpPC, Ptr, AK_Assign))
+    return false;
   if (!CheckExtern(S, OpPC, Ptr))
     return false;
   if (!CheckRange(S, OpPC, Ptr, AK_Assign))
@@ -1287,6 +1308,12 @@ bool Call(InterpState &S, CodePtr OpPC, const Function *Func,
 
     const Pointer &ThisPtr = S.Stk.peek<Pointer>(ThisOffset);
 
+    // C++23 [expr.const]p5.6
+    // an invocation of a virtual function ([class.virtual]) for an object whose
+    // dynamic type is constexpr-unknown;
+    if (ThisPtr.isDummy() && Func->isVirtual())
+      return false;
+
     // If the current function is a lambda static invoker and
     // the function we're about to call is a lambda call operator,
     // skip the CheckInvoke, since the ThisPtr is a null pointer
@@ -1537,34 +1564,38 @@ bool CheckNewTypeMismatch(InterpState &S, CodePtr OpPC, const Expr *E,
 bool InvalidNewDeleteExpr(InterpState &S, CodePtr OpPC, const Expr *E) {
   assert(E);
 
-  if (S.getLangOpts().CPlusPlus26)
-    return true;
-
-  const auto &Loc = S.Current->getSource(OpPC);
-
   if (const auto *NewExpr = dyn_cast<CXXNewExpr>(E)) {
     const FunctionDecl *OperatorNew = NewExpr->getOperatorNew();
 
-    if (!S.getLangOpts().CPlusPlus26 && NewExpr->getNumPlacementArgs() > 0) {
+    if (NewExpr->getNumPlacementArgs() > 0) {
       // This is allowed pre-C++26, but only an std function.
-      if (S.Current->isStdFunction())
+      if (S.getLangOpts().CPlusPlus26 || S.Current->isStdFunction())
         return true;
-      S.FFDiag(Loc, diag::note_constexpr_new_placement)
+      S.FFDiag(S.Current->getSource(OpPC), diag::note_constexpr_new_placement)
           << /*C++26 feature*/ 1 << E->getSourceRange();
-    } else if (NewExpr->getNumPlacementArgs() == 1 &&
-               !OperatorNew->isReservedGlobalPlacementOperator()) {
-      S.FFDiag(Loc, diag::note_constexpr_new_placement)
-          << /*Unsupported*/ 0 << E->getSourceRange();
     } else if (!OperatorNew->isReplaceableGlobalAllocationFunction()) {
-      S.FFDiag(Loc, diag::note_constexpr_new_non_replaceable)
+      S.FFDiag(S.Current->getSource(OpPC),
+               diag::note_constexpr_new_non_replaceable)
           << isa<CXXMethodDecl>(OperatorNew) << OperatorNew;
+      return false;
+    } else if (!S.getLangOpts().CPlusPlus26 &&
+               NewExpr->getNumPlacementArgs() == 1 &&
+               !OperatorNew->isReservedGlobalPlacementOperator()) {
+      if (!S.getLangOpts().CPlusPlus26) {
+        S.FFDiag(S.Current->getSource(OpPC), diag::note_constexpr_new_placement)
+            << /*Unsupported*/ 0 << E->getSourceRange();
+        return false;
+      }
+      return true;
     }
   } else {
     const auto *DeleteExpr = cast<CXXDeleteExpr>(E);
     const FunctionDecl *OperatorDelete = DeleteExpr->getOperatorDelete();
     if (!OperatorDelete->isReplaceableGlobalAllocationFunction()) {
-      S.FFDiag(Loc, diag::note_constexpr_new_non_replaceable)
+      S.FFDiag(S.Current->getSource(OpPC),
+               diag::note_constexpr_new_non_replaceable)
           << isa<CXXMethodDecl>(OperatorDelete) << OperatorDelete;
+      return false;
     }
   }
 
@@ -1661,17 +1692,6 @@ bool GetTypeidPtr(InterpState &S, CodePtr OpPC, const Type *TypeInfoType) {
   if (!P.isBlockPointer())
     return false;
 
-  if (P.isDummy()) {
-    QualType StarThisType =
-        S.getASTContext().getLValueReferenceType(P.getType());
-    S.FFDiag(S.Current->getSource(OpPC),
-             diag::note_constexpr_polymorphic_unknown_dynamic_type)
-        << AK_TypeId
-        << P.toAPValue(S.getASTContext())
-               .getAsString(S.getASTContext(), StarThisType);
-    return false;
-  }
-
   S.Stk.push<Pointer>(P.getType().getTypePtr(), TypeInfoType);
   return true;
 }
diff --git a/clang/lib/AST/ByteCode/Interp.h b/clang/lib/AST/ByteCode/Interp.h
index 063970afec9e3..5cc371c7ee495 100644
--- a/clang/lib/AST/ByteCode/Interp.h
+++ b/clang/lib/AST/ByteCode/Interp.h
@@ -325,11 +325,11 @@ bool Ret(InterpState &S, CodePtr &PC) {
 
   if (InterpFrame *Caller = S.Current->Caller) {
     PC = S.Current->getRetPC();
-    delete S.Current;
+    InterpFrame::free(S.Current);
     S.Current = Caller;
     S.Stk.push<T>(Ret);
   } else {
-    delete S.Current;
+    InterpFrame::free(S.Current);
     S.Current = nullptr;
     // The topmost frame should come from an EvalEmitter,
     // which has its own implementation of the Ret<> instruction.
@@ -345,10 +345,10 @@ inline bool RetVoid(InterpState &S, CodePtr &PC) {
 
   if (InterpFrame *Caller = S.Current->Caller) {
     PC = S.Current->getRetPC();
-    delete S.Current;
+    InterpFrame::free(S.Current);
     S.Current = Caller;
   } else {
-    delete S.Current;
+    InterpFrame::free(S.Current);
     S.Current = nullptr;
   }
   return true;
@@ -379,15 +379,14 @@ bool AddSubMulHelper(InterpState &S, CodePtr OpPC, unsigned Bits, const T &LHS,
   APSInt Value = OpAP<APSInt>()(LHS.toAPSInt(Bits), RHS.toAPSInt(Bits));
 
   // Report undefined behaviour, stopping if required.
-  const Expr *E = S.Current->getExpr(OpPC);
-  QualType Type = E->getType();
   if (S.checkingForUndefinedBehavior()) {
+    const Expr *E = S.Current->getExpr(OpPC);
+    QualType Type = E->getType();
     SmallString<32> Trunc;
     Value.trunc(Result.bitWidth())
         .toString(Trunc, 10, Result.isSigned(), /*formatAsCLiteral=*/false,
                   /*UpperCase=*/true, /*InsertSeparators=*/true);
-    auto Loc = E->getExprLoc();
-    S.report(Loc, diag::warn_integer_constant_overflow)
+    S.report(E->getExprLoc(), diag::warn_integer_constant_overflow)
         << Trunc << Type << E->getSourceRange();
   }
 
@@ -737,16 +736,14 @@ bool Neg(InterpState &S, CodePtr OpPC) {
   S.Stk.push<T>(Result);
 
   APSInt NegatedValue = -Value.toAPSInt(Value.bitWidth() + 1);
-  const Expr *E = S.Current->getExpr(OpPC);
-  QualType Type = E->getType();
-
   if (S.checkingForUndefinedBehavior()) {
+    const Expr *E = S.Current->getExpr(OpPC);
+    QualType Type = E->getType();
     SmallString<32> Trunc;
     NegatedValue.trunc(Result.bitWidth())
         .toString(Trunc, 10, Result.isSigned(), /*formatAsCLiteral=*/false,
                   /*UpperCase=*/true, /*InsertSeparators=*/true);
-    auto Loc = E->getExprLoc();
-    S.report(Loc, diag::warn_integer_constant_overflow)
+    S.report(E->getExprLoc(), diag::warn_integer_constant_overflow)
         << Trunc << Type << E->getSourceRange();
     return true;
   }
@@ -800,15 +797,14 @@ bool IncDecHelper(InterpState &S, CodePtr OpPC, const Pointer &Ptr) {
     APResult = --Value.toAPSInt(Bits);
 
   // Report undefined behaviour, stopping if required.
-  const Expr *E = S.Current->getExpr(OpPC);
-  QualType Type = E->getType();
   if (S.checkingForUndefinedBehavior()) {
+    const Expr *E = S.Current->getExpr(OpPC);
+    QualType Type = E->getType();
     SmallString<32> Trunc;
     APResult.trunc(Result.bitWidth())
         .toString(Trunc, 10, Result.isSigned(), /*formatAsCLiteral=*/false,
                   /*UpperCase=*/true, /*InsertSeparators=*/true);
-    auto Loc = E->getExprLoc();
-    S.report(Loc, diag::warn_integer_constant_overflow)
+    S.report(E->getExprLoc(), diag::warn_integer_constant_overflow)
         << Trunc << Type << E->getSourceRange();
     return true;
   }
@@ -1258,6 +1254,12 @@ bool GetLocal(InterpState &S, CodePtr OpPC, uint32_t I) {
   return true;
 }
 
+static inline bool Kill(InterpState &S, CodePtr OpPC) {
+  const auto &Ptr = S.Stk.pop<Pointer>();
+  Ptr.endLifetime();
+  return true;
+}
+
 /// 1) Pops the value from the stack.
 /// 2) Writes the value to the local variable with the
 ///    given offset.
@@ -1482,7 +1484,10 @@ bool InitThisBitField(InterpState &S, CodePtr OpPC, const Record::Field *F,
 template <PrimType Name, class T = typename PrimConv<Name>::T>
 bool InitField(InterpState &S, CodePtr OpPC, uint32_t I) {
   const T &Value = S.Stk.pop<T>();
-  const Pointer &Field = S.Stk.peek<Pointer>().atField(I);
+  const Pointer &Ptr = S.Stk.peek<Pointer>();
+  if (!CheckRange(S, OpPC, Ptr, CSK_Field))
+    return false;
+  const Pointer &Field = Ptr.atField(I);
   Field.deref<T>() = Value;
   Field.activate();
   Field.initialize();
diff --git a/clang/lib/AST/ByteCode/InterpBuiltin.cpp b/clang/lib/AST/ByteCode/InterpBuiltin.cpp
index e657dbd2f9c73..0e586725b5869 100644
--- a/clang/lib/AST/ByteCode/InterpBuiltin.cpp
+++ b/clang/lib/AST/ByteCode/InterpBuiltin.cpp
@@ -1538,9 +1538,12 @@ static bool interp__builtin_constant_p(InterpState &S, CodePtr OpPC,
   if (ArgType->isIntegralOrEnumerationType() || ArgType->isFloatingType() ||
       ArgType->isAnyComplexType() || ArgType->isPointerType() ||
       ArgType->isNullPtrType()) {
+    auto PrevDiags = S.getEvalStatus().Diag;
+    S.getEvalStatus().Diag = nullptr;
     InterpStack Stk;
     Compiler<EvalEmitter> C(S.Ctx, S.P, S, Stk);
     auto Res = C.interpretExpr(Arg, /*ConvertResultToRValue=*/Arg->isGLValue());
+    S.getEvalStatus().Diag = PrevDiags;
     if (Res.isInvalid()) {
       C.cleanup();
       Stk.clear();
diff --git a/clang/lib/AST/ByteCode/InterpFrame.cpp b/clang/lib/AST/ByteCode/InterpFrame.cpp
index 20f67d9b1fd42..c383b2bc7f95c 100644
--- a/clang/lib/AST/ByteCode/InterpFrame.cpp
+++ b/clang/lib/AST/ByteCode/InterpFrame.cpp
@@ -23,11 +23,15 @@
 using namespace clang;
 using namespace clang::interp;
 
+InterpFrame::InterpFrame(InterpState &S)
+    : Caller(nullptr), S(S), Depth(0), Func(nullptr), RetPC(CodePtr()),
+      ArgSize(0), Args(nullptr), FrameOffset(0), IsBottom(true) {}
+
 InterpFrame::InterpFrame(InterpState &S, const Function *Func,
                          InterpFrame *Caller, CodePtr RetPC, unsigned ArgSize)
     : Caller(Caller), S(S), Depth(Caller ? Caller->Depth + 1 : 0), Func(Func),
       RetPC(RetPC), ArgSize(ArgSize), Args(static_cast<char *>(S.Stk.top())),
-      FrameOffset(S.Stk.size()) {
+      FrameOffset(S.Stk.size()), IsBottom(!Caller) {
   if (!Func)
     return;
 
@@ -73,11 +77,15 @@ InterpFrame::~InterpFrame() {
   // When destroying the InterpFrame, call the Dtor for all block
   // that haven't been destroyed via a destroy() op yet.
   // This happens when the execution is interruped midway-through.
-  if (Func) {
-    for (auto &Scope : Func->scopes()) {
-      for (auto &Local : Scope.locals()) {
-        S.deallocate(localBlock(Local.Offset));
-      }
+  destroyScopes();
+}
+
+void InterpFrame::destroyScopes() {
+  if (!Func)
+    return;
+  for (auto &Scope : Func->scopes()) {
+    for (auto &Local : Scope.locals()) {
+      S.deallocate(localBlock(Local.Offset));
     }
   }
 }
@@ -91,7 +99,7 @@ void InterpFrame::initScope(unsigned Idx) {
 }
 
 void InterpFrame::destroy(unsigned Idx) {
-  for (auto &Local : Func->getScope(Idx).locals()) {
+  for (auto &Local : Func->getScope(Idx).locals_reverse()) {
     S.deallocate(localBlock(Local.Offset));
   }
 }
@@ -244,7 +252,7 @@ SourceInfo InterpFrame::getSource(CodePtr PC) const {
 
 const Expr *InterpFrame::getExpr(CodePtr PC) const {
   if (Func && !funcHasUsableBody(Func) && Caller)
-    return Caller->getExpr(PC);
+    return Caller->getExpr(RetPC);
 
   return S.getExpr(Func, PC);
 }
diff --git a/clang/lib/AST/ByteCode/InterpFrame.h b/clang/lib/AST/ByteCode/InterpFrame.h
index 7cfc3ac68b4f3..360e6bff12327 100644
--- a/clang/lib/AST/ByteCode/InterpFrame.h
+++ b/clang/lib/AST/ByteCode/InterpFrame.h
@@ -28,6 +28,9 @@ class InterpFrame final : public Frame {
   /// The frame of the previous function.
   InterpFrame *Caller;
 
+  /// Bottom Frame.
+  InterpFrame(InterpState &S);
+
   /// Creates a new frame for a method call.
   InterpFrame(InterpState &S, const Function *Func, InterpFrame *Caller,
               CodePtr RetPC, unsigned ArgSize);
@@ -42,9 +45,15 @@ class InterpFrame final : public Frame {
   /// Destroys the frame, killing all live pointers to stack slots.
   ~InterpFrame();
 
+  static void free(InterpFrame *F) {
+    if (!F->isBottomFrame())
+      delete F;
+  }
+
   /// Invokes the destructors for a scope.
   void destroy(unsigned Idx);
   void initScope(unsigned Idx);
+  void destroyScopes();
 
   /// Describes the frame with arguments for diagnostic purposes.
   void describe(llvm::raw_ostream &OS) const override;
@@ -119,6 +128,8 @@ class InterpFrame final : public Frame {
 
   bool isStdFunction() const;
 
+  bool isBottomFrame() const { return IsBottom; }
+
   void dump() const { dump(llvm::errs(), 0); }
   void dump(llvm::raw_ostream &OS, unsigned Indent = 0) const;
 
@@ -167,6 +178,7 @@ class InterpFrame final : public Frame {
   const size_t FrameOffset;
   /// Mapping from arg offsets to their argument blocks.
   llvm::DenseMap<unsigned, std::unique_ptr<char[]>> Params;
+  bool IsBottom = false;
 };
 
 } // namespace interp
diff --git a/clang/lib/AST/ByteCode/InterpState.cpp b/clang/lib/AST/ByteCode/InterpState.cpp
index 287c3bd3bca3a..70a2e9b62fc3a 100644
--- a/clang/lib/AST/ByteCode/InterpState.cpp
+++ b/clang/lib/AST/ByteCode/InterpState.cpp
@@ -17,7 +17,14 @@ using namespace clang::interp;
 
 InterpState::InterpState(State &Parent, Program &P, InterpStack &Stk,
                          Context &Ctx, SourceMapper *M)
-    : Parent(Parent), M(M), P(P), Stk(Stk), Ctx(Ctx), Current(nullptr) {}
+    : Parent(Parent), M(M), P(P), Stk(Stk), Ctx(Ctx), BottomFrame(*this),
+      Current(&BottomFrame) {}
+
+InterpState::InterpState(State &Parent, Program &P, InterpStack &Stk,
+                         Context &Ctx, const Function *Func)
+    : Parent(Parent), M(nullptr), P(P), Stk(Stk), Ctx(Ctx),
+      BottomFrame(*this, Func, nullptr, CodePtr(), Func->getArgSize()),
+      Current(&BottomFrame) {}
 
 bool InterpState::inConstantContext() const {
   if (ConstantContextOverride)
@@ -27,11 +34,12 @@ bool InterpState::inConstantContext() const {
 }
 
 InterpState::~InterpState() {
-  while (Current) {
+  while (Current && !Current->isBottomFrame()) {
     InterpFrame *Next = Current->Caller;
     delete Current;
     Current = Next;
   }
+  BottomFrame.destroyScopes();
 
   while (DeadBlocks) {
     DeadBlock *Next = DeadBlocks->Next;
diff --git a/clang/lib/AST/ByteCode/InterpState.h b/clang/lib/AST/ByteCode/InterpState.h
index 2a1311c86a2f2..d6adfff1a713a 100644
--- a/clang/lib/AST/ByteCode/InterpState.h
+++ b/clang/lib/AST/ByteCode/InterpState.h
@@ -37,6 +37,8 @@ class InterpState final : public State, public SourceMapper {
 public:
   InterpState(State &Parent, Program &P, InterpStack &Stk, Context &Ctx,
               SourceMapper *M = nullptr);
+  InterpState(State &Parent, Program &P, InterpStack &Stk, Context &Ctx,
+              const Function *Func);
 
   ~InterpState();
 
@@ -134,6 +136,8 @@ class InterpState final : public State, public SourceMapper {
   InterpStack &Stk;
   /// Interpreter Context.
   Context &Ctx;
+  /// Bottom function frame.
+  InterpFrame BottomFrame;
   /// The current frame.
   InterpFrame *Current = nullptr;
   /// Source location of the evaluating expression
diff --git a/clang/lib/AST/ByteCode/Opcodes.td b/clang/lib/AST/ByteCode/Opcodes.td
index 4b0c902ab2926..088a3e40fe2a7 100644
--- a/clang/lib/AST/ByteCode/Opcodes.td
+++ b/clang/lib/AST/ByteCode/Opcodes.td
@@ -394,6 +394,11 @@ def GetLocal : AccessOpcode { let HasCustomEval = 1; }
 // [] -> [Pointer]
 def SetLocal : AccessOpcode { let HasCustomEval = 1; }
 
+def Kill : Opcode {
+  let Types = [];
+  let Args = [];
+}
+
 def CheckDecl : Opcode {
   let Args = [ArgVarDecl];
 }
diff --git a/clang/lib/AST/ByteCode/Pointer.cpp b/clang/lib/AST/ByteCode/Pointer.cpp
index ec4756fe4f87d..3033bd47adf75 100644
--- a/clang/lib/AST/ByteCode/Pointer.cpp
+++ b/clang/lib/AST/ByteCode/Pointer.cpp
@@ -209,6 +209,10 @@ APValue Pointer::toAPValue(const ASTContext &ASTCtx) const {
     return ASTCtx.toCharUnitsFromBits(Layout.getFieldOffset(FieldIndex));
   };
 
+  bool UsePath = true;
+  if (getType()->isLValueReferenceType())
+    UsePath = false;
+
   // Build the path into the object.
   Pointer Ptr = *this;
   while (Ptr.isField() || Ptr.isArrayElement()) {
@@ -217,38 +221,42 @@ APValue Pointer::toAPValue(const ASTContext &ASTCtx) const {
       // An array root may still be an array element itself.
       if (Ptr.isArrayElement()) {
         Ptr = Ptr.expand();
+        const Descriptor *Desc = Ptr.getFieldDesc();
         unsigned Index = Ptr.getIndex();
-        Path.push_back(APValue::LValuePathEntry::ArrayIndex(Index));
-        QualType ElemType = Ptr.getFieldDesc()->getElemQualType();
+        QualType ElemType = Desc->getElemQualType();
         Offset += (Index * ASTCtx.getTypeSizeInChars(ElemType));
+        if (Ptr.getArray().getType()->isArrayType())
+          Path.push_back(APValue::LValuePathEntry::ArrayIndex(Index));
         Ptr = Ptr.getArray();
       } else {
-        Path.push_back(APValue::LValuePathEntry(
-            {Ptr.getFieldDesc()->asDecl(), /*IsVirtual=*/false}));
+        const Descriptor *Desc = Ptr.getFieldDesc();
+        const auto *Dcl = Desc->asDecl();
+        Path.push_back(APValue::LValuePathEntry({Dcl, /*IsVirtual=*/false}));
 
-        if (const auto *FD =
-                dyn_cast_if_present<FieldDecl>(Ptr.getFieldDesc()->asDecl()))
+        if (const auto *FD = dyn_cast_if_present<FieldDecl>(Dcl))
           Offset += getFieldOffset(FD);
 
         Ptr = Ptr.getBase();
       }
     } else if (Ptr.isArrayElement()) {
       Ptr = Ptr.expand();
+      const Descriptor *Desc = Ptr.getFieldDesc();
       unsigned Index;
       if (Ptr.isOnePastEnd())
         Index = Ptr.getArray().getNumElems();
       else
         Index = Ptr.getIndex();
 
-      QualType ElemType = Ptr.getFieldDesc()->getElemQualType();
+      QualType ElemType = Desc->getElemQualType();
       Offset += (Index * ASTCtx.getTypeSizeInChars(ElemType));
-      Path.push_back(APValue::LValuePathEntry::ArrayIndex(Index));
+      if (Ptr.getArray().getType()->isArrayType())
+        Path.push_back(APValue::LValuePathEntry::ArrayIndex(Index));
       Ptr = Ptr.getArray();
     } else {
+      const Descriptor *Desc = Ptr.getFieldDesc();
       bool IsVirtual = false;
 
       // Create a path entry for the field.
-      const Descriptor *Desc = Ptr.getFieldDesc();
       if (const auto *BaseOrMember = Desc->asDecl()) {
         if (const auto *FD = dyn_cast<FieldDecl>(BaseOrMember)) {
           Ptr = Ptr.getBase();
@@ -281,8 +289,11 @@ APValue Pointer::toAPValue(const ASTContext &ASTCtx) const {
   // Just invert the order of the elements.
   std::reverse(Path.begin(), Path.end());
 
-  return APValue(Base, Offset, Path, /*IsOnePastEnd=*/isOnePastEnd(),
-                 /*IsNullPtr=*/false);
+  if (UsePath)
+    return APValue(Base, Offset, Path,
+                   /*IsOnePastEnd=*/!isElementPastEnd() && isOnePastEnd());
+
+  return APValue(Base, Offset, APValue::NoLValuePath());
 }
 
 void Pointer::print(llvm::raw_ostream &OS) const {
diff --git a/clang/lib/AST/ByteCode/Pointer.h b/clang/lib/AST/ByteCode/Pointer.h
index ef03c12e86c10..3970d5833fcdc 100644
--- a/clang/lib/AST/ByteCode/Pointer.h
+++ b/clang/lib/AST/ByteCode/Pointer.h
@@ -630,8 +630,7 @@ class Pointer {
     if (isUnknownSizeArray())
       return false;
 
-    return isElementPastEnd() || isPastEnd() ||
-           (getSize() == getOffset() && !isZeroSizeArray());
+    return isPastEnd() || (getSize() == getOffset() && !isZeroSizeArray());
   }
 
   /// Checks if the pointer points past the end of the object.
@@ -688,6 +687,22 @@ class Pointer {
   /// Deactivates an entire strurcutre.
   void deactivate() const;
 
+  Lifetime getLifetime() const {
+    if (!isBlockPointer())
+      return Lifetime::Started;
+    if (asBlockPointer().Base < sizeof(InlineDescriptor))
+      return Lifetime::Started;
+    return getInlineDesc()->LifeState;
+  }
+
+  void endLifetime() const {
+    if (!isBlockPointer())
+      return;
+    if (asBlockPointer().Base < sizeof(InlineDescriptor))
+      return;
+    getInlineDesc()->LifeState = Lifetime::Ended;
+  }
+
   /// Compare two pointers.
   ComparisonCategoryResult compare(const Pointer &Other) const {
     if (!hasSameBase(*this, Other))
diff --git a/clang/lib/AST/ByteCode/Program.cpp b/clang/lib/AST/ByteCode/Program.cpp
index 7d8862d606ba3..833c9ef88d770 100644
--- a/clang/lib/AST/ByteCode/Program.cpp
+++ b/clang/lib/AST/ByteCode/Program.cpp
@@ -18,14 +18,12 @@ using namespace clang;
 using namespace clang::interp;
 
 unsigned Program::getOrCreateNativePointer(const void *Ptr) {
-  auto It = NativePointerIndices.find(Ptr);
-  if (It != NativePointerIndices.end())
-    return It->second;
+  auto [It, Inserted] =
+      NativePointerIndices.try_emplace(Ptr, NativePointers.size());
+  if (Inserted)
+    NativePointers.push_back(Ptr);
 
-  unsigned Idx = NativePointers.size();
-  NativePointers.push_back(Ptr);
-  NativePointerIndices[Ptr] = Idx;
-  return Idx;
+  return It->second;
 }
 
 const void *Program::getNativePointer(unsigned Idx) {
@@ -35,6 +33,7 @@ const void *Program::getNativePointer(unsigned Idx) {
 unsigned Program::createGlobalString(const StringLiteral *S, const Expr *Base) {
   const size_t CharWidth = S->getCharByteWidth();
   const size_t BitWidth = CharWidth * Ctx.getCharBit();
+  unsigned StringLength = S->getLength();
 
   PrimType CharType;
   switch (CharWidth) {
@@ -55,15 +54,15 @@ unsigned Program::createGlobalString(const StringLiteral *S, const Expr *Base) {
     Base = S;
 
   // Create a descriptor for the string.
-  Descriptor *Desc = allocateDescriptor(Base, CharType, Descriptor::GlobalMD,
-                                        S->getLength() + 1,
-                                        /*isConst=*/true,
-                                        /*isTemporary=*/false,
-                                        /*isMutable=*/false);
+  Descriptor *Desc =
+      allocateDescriptor(Base, CharType, Descriptor::GlobalMD, StringLength + 1,
+                         /*isConst=*/true,
+                         /*isTemporary=*/false,
+                         /*isMutable=*/false);
 
   // Allocate storage for the string.
   // The byte length does not include the null terminator.
-  unsigned I = Globals.size();
+  unsigned GlobalIndex = Globals.size();
   unsigned Sz = Desc->getAllocSize();
   auto *G = new (Allocator, Sz) Global(Ctx.getEvalID(), Desc, /*isStatic=*/true,
                                        /*isExtern=*/false);
@@ -74,33 +73,32 @@ unsigned Program::createGlobalString(const StringLiteral *S, const Expr *Base) {
 
   // Construct the string in storage.
   const Pointer Ptr(G->block());
-  for (unsigned I = 0, N = S->getLength(); I <= N; ++I) {
-    Pointer Field = Ptr.atIndex(I).narrow();
-    const uint32_t CodePoint = I == N ? 0 : S->getCodeUnit(I);
+  for (unsigned I = 0; I <= StringLength; ++I) {
+    Pointer Field = Ptr.atIndex(I);
+    const uint32_t CodePoint = I == StringLength ? 0 : S->getCodeUnit(I);
     switch (CharType) {
     case PT_Sint8: {
       using T = PrimConv<PT_Sint8>::T;
       Field.deref<T>() = T::from(CodePoint, BitWidth);
-      Field.initialize();
       break;
     }
     case PT_Uint16: {
       using T = PrimConv<PT_Uint16>::T;
       Field.deref<T>() = T::from(CodePoint, BitWidth);
-      Field.initialize();
       break;
     }
     case PT_Uint32: {
       using T = PrimConv<PT_Uint32>::T;
       Field.deref<T>() = T::from(CodePoint, BitWidth);
-      Field.initialize();
       break;
     }
     default:
       llvm_unreachable("unsupported character type");
     }
   }
-  return I;
+  Ptr.initialize();
+
+  return GlobalIndex;
 }
 
 Pointer Program::getPtrGlobal(unsigned Idx) const {
@@ -453,15 +451,21 @@ Descriptor *Program::createDescriptor(const DeclTy &D, const Type *Ty,
 
   // Complex types - represented as arrays of elements.
   if (const auto *CT = Ty->getAs<ComplexType>()) {
-    PrimType ElemTy = *Ctx.classify(CT->getElementType());
-    return allocateDescriptor(D, ElemTy, MDSize, 2, IsConst, IsTemporary,
+    std::optional<PrimType> ElemTy = Ctx.classify(CT->getElementType());
+    if (!ElemTy)
+      return nullptr;
+
+    return allocateDescriptor(D, *ElemTy, MDSize, 2, IsConst, IsTemporary,
                               IsMutable);
   }
 
   // Same with vector types.
   if (const auto *VT = Ty->getAs<VectorType>()) {
-    PrimType ElemTy = *Ctx.classify(VT->getElementType());
-    return allocateDescriptor(D, ElemTy, MDSize, VT->getNumElements(), IsConst,
+    std::optional<PrimType> ElemTy = Ctx.classify(VT->getElementType());
+    if (!ElemTy)
+      return nullptr;
+
+    return allocateDescriptor(D, *ElemTy, MDSize, VT->getNumElements(), IsConst,
                               IsTemporary, IsMutable);
   }
 
diff --git a/clang/lib/AST/CommentLexer.cpp b/clang/lib/AST/CommentLexer.cpp
index ec9a5b480aa29..804be89a8d4dd 100644
--- a/clang/lib/AST/CommentLexer.cpp
+++ b/clang/lib/AST/CommentLexer.cpp
@@ -196,6 +196,15 @@ const char *skipWhitespace(const char *BufferPtr, const char *BufferEnd) {
   return BufferEnd;
 }
 
+const char *skipHorizontalWhitespace(const char *BufferPtr,
+                                     const char *BufferEnd) {
+  for (; BufferPtr != BufferEnd; ++BufferPtr) {
+    if (!isHorizontalWhitespace(*BufferPtr))
+      return BufferPtr;
+  }
+  return BufferEnd;
+}
+
 bool isWhitespace(const char *BufferPtr, const char *BufferEnd) {
   return skipWhitespace(BufferPtr, BufferEnd) == BufferEnd;
 }
@@ -637,17 +646,41 @@ void Lexer::setupAndLexHTMLStartTag(Token &T) {
   formTokenWithChars(T, TagNameEnd, tok::html_start_tag);
   T.setHTMLTagStartName(Name);
 
-  BufferPtr = skipWhitespace(BufferPtr, CommentEnd);
+  BufferPtr = skipHorizontalWhitespace(BufferPtr, CommentEnd);
+  if (BufferPtr == CommentEnd) { // in BCPL comments
+    State = LS_HTMLStartTag;
+    return;
+  }
 
   const char C = *BufferPtr;
   if (BufferPtr != CommentEnd &&
-      (C == '>' || C == '/' || isHTMLIdentifierStartingCharacter(C)))
+      (C == '>' || C == '/' || isVerticalWhitespace(C) ||
+       isHTMLIdentifierStartingCharacter(C)))
     State = LS_HTMLStartTag;
 }
 
 void Lexer::lexHTMLStartTag(Token &T) {
   assert(State == LS_HTMLStartTag);
 
+  // Skip leading whitespace and comment decorations
+  while (isVerticalWhitespace(*BufferPtr)) {
+    BufferPtr = skipNewline(BufferPtr, CommentEnd);
+
+    if (CommentState == LCS_InsideCComment)
+      skipLineStartingDecorations();
+
+    BufferPtr = skipHorizontalWhitespace(BufferPtr, CommentEnd);
+    if (BufferPtr == CommentEnd) {
+      // HTML starting tags must be defined in a single comment block.
+      // It's likely a user-error where they forgot to terminate the comment.
+      State = LS_Normal;
+      // Since at least one newline was skipped and one token needs to be lexed,
+      // return a newline.
+      formTokenWithChars(T, BufferPtr, tok::newline);
+      return;
+    }
+  }
+
   const char *TokenPtr = BufferPtr;
   char C = *TokenPtr;
   if (isHTMLIdentifierCharacter(C)) {
@@ -693,14 +726,13 @@ void Lexer::lexHTMLStartTag(Token &T) {
 
   // Now look ahead and return to normal state if we don't see any HTML tokens
   // ahead.
-  BufferPtr = skipWhitespace(BufferPtr, CommentEnd);
+  BufferPtr = skipHorizontalWhitespace(BufferPtr, CommentEnd);
   if (BufferPtr == CommentEnd) {
-    State = LS_Normal;
     return;
   }
 
   C = *BufferPtr;
-  if (!isHTMLIdentifierStartingCharacter(C) &&
+  if (!isHTMLIdentifierStartingCharacter(C) && !isVerticalWhitespace(C) &&
       C != '=' && C != '\"' && C != '\'' && C != '>' && C != '/') {
     State = LS_Normal;
     return;
@@ -774,8 +806,17 @@ void Lexer::lex(Token &T) {
         BufferPtr++;
 
       CommentState = LCS_InsideBCPLComment;
-      if (State != LS_VerbatimBlockBody && State != LS_VerbatimBlockFirstLine)
+      switch (State) {
+      case LS_VerbatimBlockFirstLine:
+      case LS_VerbatimBlockBody:
+        break;
+      case LS_HTMLStartTag:
+        BufferPtr = skipHorizontalWhitespace(BufferPtr, BufferEnd);
+        break;
+      default:
         State = LS_Normal;
+        break;
+      }
       CommentEnd = findBCPLCommentEnd(BufferPtr, BufferEnd);
       goto again;
     }
@@ -807,6 +848,14 @@ void Lexer::lex(Token &T) {
     while(EndWhitespace != BufferEnd && *EndWhitespace != '/')
       EndWhitespace++;
 
+    // When lexing the start of an HTML tag (i.e. going through the attributes)
+    // there won't be any newlines generated.
+    if (State == LS_HTMLStartTag && EndWhitespace != BufferEnd) {
+      CommentState = LCS_BeforeComment;
+      BufferPtr = EndWhitespace;
+      goto again;
+    }
+
     // Turn any whitespace between comments (and there is only whitespace
     // between them -- guaranteed by comment extraction) into a newline.  We
     // have two newlines between C comments in total (first one was synthesized
@@ -829,6 +878,14 @@ void Lexer::lex(Token &T) {
         BufferPtr += 2;
         assert(BufferPtr <= BufferEnd);
 
+        // When lexing the start of an HTML tag (i.e. going through the
+        // attributes) there won't be any newlines generated - whitespace still
+        // needs to be skipped.
+        if (State == LS_HTMLStartTag && BufferPtr != BufferEnd) {
+          CommentState = LCS_BetweenComments;
+          goto again;
+        }
+
         // Synthenize newline just after the C comment, regardless if there is
         // actually a newline.
         formTokenWithChars(T, BufferPtr, tok::newline);
diff --git a/clang/lib/AST/Decl.cpp b/clang/lib/AST/Decl.cpp
index 5ce03ce20d284..610207cf8b9a4 100644
--- a/clang/lib/AST/Decl.cpp
+++ b/clang/lib/AST/Decl.cpp
@@ -2447,7 +2447,7 @@ bool VarDecl::isOutOfLine() const {
 }
 
 void VarDecl::setInit(Expr *I) {
-  if (auto *Eval = Init.dyn_cast<EvaluatedStmt *>()) {
+  if (auto *Eval = dyn_cast_if_present<EvaluatedStmt *>(Init)) {
     Eval->~EvaluatedStmt();
     getASTContext().Deallocate(Eval);
   }
@@ -2527,7 +2527,7 @@ bool VarDecl::isUsableInConstantExpressions(const ASTContext &Context) const {
 /// form, which contains extra information on the evaluated value of the
 /// initializer.
 EvaluatedStmt *VarDecl::ensureEvaluatedStmt() const {
-  auto *Eval = Init.dyn_cast<EvaluatedStmt *>();
+  auto *Eval = dyn_cast_if_present<EvaluatedStmt *>(Init);
   if (!Eval) {
     // Note: EvaluatedStmt contains an APValue, which usually holds
     // resources not allocated from the ASTContext.  We need to do some
@@ -2541,7 +2541,7 @@ EvaluatedStmt *VarDecl::ensureEvaluatedStmt() const {
 }
 
 EvaluatedStmt *VarDecl::getEvaluatedStmt() const {
-  return Init.dyn_cast<EvaluatedStmt *>();
+  return dyn_cast_if_present<EvaluatedStmt *>(Init);
 }
 
 APValue *VarDecl::evaluateValue() const {
@@ -2659,10 +2659,6 @@ bool VarDecl::checkForConstantInitialization(
   return Eval->HasConstantInitialization;
 }
 
-bool VarDecl::isParameterPack() const {
-  return isa<PackExpansionType>(getType());
-}
-
 template<typename DeclT>
 static DeclT *getDefinitionOrSelf(DeclT *D) {
   assert(D);
@@ -2784,8 +2780,8 @@ SourceLocation VarDecl::getPointOfInstantiation() const {
 }
 
 VarTemplateDecl *VarDecl::getDescribedVarTemplate() const {
-  return getASTContext().getTemplateOrSpecializationInfo(this)
-      .dyn_cast<VarTemplateDecl *>();
+  return dyn_cast_if_present<VarTemplateDecl *>(
+      getASTContext().getTemplateOrSpecializationInfo(this));
 }
 
 void VarDecl::setDescribedVarTemplate(VarTemplateDecl *Template) {
@@ -2875,8 +2871,8 @@ MemberSpecializationInfo *VarDecl::getMemberSpecializationInfo() const {
   if (isStaticDataMember())
     // FIXME: Remove ?
     // return getASTContext().getInstantiatedFromStaticDataMember(this);
-    return getASTContext().getTemplateOrSpecializationInfo(this)
-        .dyn_cast<MemberSpecializationInfo *>();
+    return dyn_cast_if_present<MemberSpecializationInfo *>(
+        getASTContext().getTemplateOrSpecializationInfo(this));
   return nullptr;
 }
 
@@ -3069,6 +3065,7 @@ FunctionDecl::FunctionDecl(Kind DK, ASTContext &C, DeclContext *DC,
   FunctionDeclBits.IsIneligibleOrNotSelected = false;
   FunctionDeclBits.HasImplicitReturnZero = false;
   FunctionDeclBits.IsLateTemplateParsed = false;
+  FunctionDeclBits.IsInstantiatedFromMemberTemplate = false;
   FunctionDeclBits.ConstexprKind = static_cast<uint64_t>(ConstexprKind);
   FunctionDeclBits.BodyContainsImmediateEscalatingExpression = false;
   FunctionDeclBits.InstantiationIsPending = false;
@@ -3314,6 +3311,10 @@ bool FunctionDecl::isImmediateFunction() const {
         .getConstructor()
         ->isImmediateFunction();
 
+  if (FunctionDecl *P = getTemplateInstantiationPattern();
+      P && P->isImmediateFunction())
+    return true;
+
   if (const auto *MD = dyn_cast<CXXMethodDecl>(this);
       MD && MD->isLambdaStaticInvoker())
     return MD->getParent()->getLambdaCallOperator()->isImmediateFunction();
@@ -4040,11 +4041,11 @@ FunctionDecl *FunctionDecl::getInstantiatedFromMemberFunction() const {
 }
 
 MemberSpecializationInfo *FunctionDecl::getMemberSpecializationInfo() const {
-  if (auto *MSI =
-          TemplateOrSpecialization.dyn_cast<MemberSpecializationInfo *>())
+  if (auto *MSI = dyn_cast_if_present<MemberSpecializationInfo *>(
+          TemplateOrSpecialization))
     return MSI;
-  if (auto *FTSI = TemplateOrSpecialization
-                       .dyn_cast<FunctionTemplateSpecializationInfo *>())
+  if (auto *FTSI = dyn_cast_if_present<FunctionTemplateSpecializationInfo *>(
+          TemplateOrSpecialization))
     return FTSI->getMemberSpecializationInfo();
   return nullptr;
 }
@@ -4062,7 +4063,7 @@ FunctionDecl::setInstantiationOfMemberFunction(ASTContext &C,
 
 FunctionTemplateDecl *FunctionDecl::getDescribedFunctionTemplate() const {
   return dyn_cast_if_present<FunctionTemplateDecl>(
-      TemplateOrSpecialization.dyn_cast<NamedDecl *>());
+      dyn_cast_if_present<NamedDecl *>(TemplateOrSpecialization));
 }
 
 void FunctionDecl::setDescribedFunctionTemplate(
@@ -4181,9 +4182,9 @@ FunctionDecl::getTemplateInstantiationPattern(bool ForDefinition) const {
 }
 
 FunctionTemplateDecl *FunctionDecl::getPrimaryTemplate() const {
-  if (FunctionTemplateSpecializationInfo *Info
-        = TemplateOrSpecialization
-            .dyn_cast<FunctionTemplateSpecializationInfo*>()) {
+  if (FunctionTemplateSpecializationInfo *Info =
+          dyn_cast_if_present<FunctionTemplateSpecializationInfo *>(
+              TemplateOrSpecialization)) {
     return Info->getTemplate();
   }
   return nullptr;
@@ -4191,15 +4192,15 @@ FunctionTemplateDecl *FunctionDecl::getPrimaryTemplate() const {
 
 FunctionTemplateSpecializationInfo *
 FunctionDecl::getTemplateSpecializationInfo() const {
-  return TemplateOrSpecialization
-      .dyn_cast<FunctionTemplateSpecializationInfo *>();
+  return dyn_cast_if_present<FunctionTemplateSpecializationInfo *>(
+      TemplateOrSpecialization);
 }
 
 const TemplateArgumentList *
 FunctionDecl::getTemplateSpecializationArgs() const {
-  if (FunctionTemplateSpecializationInfo *Info
-        = TemplateOrSpecialization
-            .dyn_cast<FunctionTemplateSpecializationInfo*>()) {
+  if (FunctionTemplateSpecializationInfo *Info =
+          dyn_cast_if_present<FunctionTemplateSpecializationInfo *>(
+              TemplateOrSpecialization)) {
     return Info->TemplateArguments;
   }
   return nullptr;
@@ -4207,14 +4208,14 @@ FunctionDecl::getTemplateSpecializationArgs() const {
 
 const ASTTemplateArgumentListInfo *
 FunctionDecl::getTemplateSpecializationArgsAsWritten() const {
-  if (FunctionTemplateSpecializationInfo *Info
-        = TemplateOrSpecialization
-            .dyn_cast<FunctionTemplateSpecializationInfo*>()) {
+  if (FunctionTemplateSpecializationInfo *Info =
+          dyn_cast_if_present<FunctionTemplateSpecializationInfo *>(
+              TemplateOrSpecialization)) {
     return Info->TemplateArgumentsAsWritten;
   }
   if (DependentFunctionTemplateSpecializationInfo *Info =
-          TemplateOrSpecialization
-              .dyn_cast<DependentFunctionTemplateSpecializationInfo *>()) {
+          dyn_cast_if_present<DependentFunctionTemplateSpecializationInfo *>(
+              TemplateOrSpecialization)) {
     return Info->TemplateArgumentsAsWritten;
   }
   return nullptr;
@@ -4239,7 +4240,8 @@ void FunctionDecl::setFunctionTemplateSpecialization(
       FunctionTemplateSpecializationInfo::Create(
           C, this, Template, TSK, TemplateArgs, TemplateArgsAsWritten,
           PointOfInstantiation,
-          TemplateOrSpecialization.dyn_cast<MemberSpecializationInfo *>());
+          dyn_cast_if_present<MemberSpecializationInfo *>(
+              TemplateOrSpecialization));
   TemplateOrSpecialization = Info;
   Template->addSpecialization(Info, InsertPos);
 }
@@ -4256,8 +4258,8 @@ void FunctionDecl::setDependentTemplateSpecialization(
 
 DependentFunctionTemplateSpecializationInfo *
 FunctionDecl::getDependentSpecializationInfo() const {
-  return TemplateOrSpecialization
-      .dyn_cast<DependentFunctionTemplateSpecializationInfo *>();
+  return dyn_cast_if_present<DependentFunctionTemplateSpecializationInfo *>(
+      TemplateOrSpecialization);
 }
 
 DependentFunctionTemplateSpecializationInfo *
@@ -4288,12 +4290,13 @@ TemplateSpecializationKind FunctionDecl::getTemplateSpecializationKind() const {
   // For a function template specialization, query the specialization
   // information object.
   if (FunctionTemplateSpecializationInfo *FTSInfo =
-          TemplateOrSpecialization
-              .dyn_cast<FunctionTemplateSpecializationInfo *>())
+          dyn_cast_if_present<FunctionTemplateSpecializationInfo *>(
+              TemplateOrSpecialization))
     return FTSInfo->getTemplateSpecializationKind();
 
   if (MemberSpecializationInfo *MSInfo =
-          TemplateOrSpecialization.dyn_cast<MemberSpecializationInfo *>())
+          dyn_cast_if_present<MemberSpecializationInfo *>(
+              TemplateOrSpecialization))
     return MSInfo->getTemplateSpecializationKind();
 
   // A dependent function template specialization is an explicit specialization,
@@ -4331,15 +4334,16 @@ FunctionDecl::getTemplateSpecializationKindForInstantiation() const {
   // of A<int>::f, and that A<int>::f<int> should be implicitly instantiated
   // from A::f<int> if a definition is needed.
   if (FunctionTemplateSpecializationInfo *FTSInfo =
-          TemplateOrSpecialization
-              .dyn_cast<FunctionTemplateSpecializationInfo *>()) {
+          dyn_cast_if_present<FunctionTemplateSpecializationInfo *>(
+              TemplateOrSpecialization)) {
     if (auto *MSInfo = FTSInfo->getMemberSpecializationInfo())
       return MSInfo->getTemplateSpecializationKind();
     return FTSInfo->getTemplateSpecializationKind();
   }
 
   if (MemberSpecializationInfo *MSInfo =
-          TemplateOrSpecialization.dyn_cast<MemberSpecializationInfo *>())
+          dyn_cast_if_present<MemberSpecializationInfo *>(
+              TemplateOrSpecialization))
     return MSInfo->getTemplateSpecializationKind();
 
   if (isa<DependentFunctionTemplateSpecializationInfo *>(
@@ -4353,9 +4357,9 @@ FunctionDecl::getTemplateSpecializationKindForInstantiation() const {
 void
 FunctionDecl::setTemplateSpecializationKind(TemplateSpecializationKind TSK,
                                           SourceLocation PointOfInstantiation) {
-  if (FunctionTemplateSpecializationInfo *FTSInfo
-        = TemplateOrSpecialization.dyn_cast<
-                                    FunctionTemplateSpecializationInfo*>()) {
+  if (FunctionTemplateSpecializationInfo *FTSInfo =
+          dyn_cast<FunctionTemplateSpecializationInfo *>(
+              TemplateOrSpecialization)) {
     FTSInfo->setTemplateSpecializationKind(TSK);
     if (TSK != TSK_ExplicitSpecialization &&
         PointOfInstantiation.isValid() &&
@@ -4364,8 +4368,9 @@ FunctionDecl::setTemplateSpecializationKind(TemplateSpecializationKind TSK,
       if (ASTMutationListener *L = getASTContext().getASTMutationListener())
         L->InstantiationRequested(this);
     }
-  } else if (MemberSpecializationInfo *MSInfo
-             = TemplateOrSpecialization.dyn_cast<MemberSpecializationInfo*>()) {
+  } else if (MemberSpecializationInfo *MSInfo =
+                 dyn_cast<MemberSpecializationInfo *>(
+                     TemplateOrSpecialization)) {
     MSInfo->setTemplateSpecializationKind(TSK);
     if (TSK != TSK_ExplicitSpecialization &&
         PointOfInstantiation.isValid() &&
@@ -5412,6 +5417,13 @@ bool ValueDecl::isInitCapture() const {
   return false;
 }
 
+bool ValueDecl::isParameterPack() const {
+  if (const auto *NTTP = dyn_cast<NonTypeTemplateParmDecl>(this))
+    return NTTP->isParameterPack();
+
+  return isa_and_nonnull<PackExpansionType>(getType().getTypePtrOrNull());
+}
+
 void ImplicitParamDecl::anchor() {}
 
 ImplicitParamDecl *ImplicitParamDecl::Create(ASTContext &C, DeclContext *DC,
@@ -5837,3 +5849,17 @@ bool clang::IsArmStreamingFunction(const FunctionDecl *FD,
 
   return false;
 }
+
+bool clang::hasArmZAState(const FunctionDecl *FD) {
+  const auto *T = FD->getType()->getAs<FunctionProtoType>();
+  return (T && FunctionType::getArmZAState(T->getAArch64SMEAttributes()) !=
+                   FunctionType::ARM_None) ||
+         (FD->hasAttr<ArmNewAttr>() && FD->getAttr<ArmNewAttr>()->isNewZA());
+}
+
+bool clang::hasArmZT0State(const FunctionDecl *FD) {
+  const auto *T = FD->getType()->getAs<FunctionProtoType>();
+  return (T && FunctionType::getArmZT0State(T->getAArch64SMEAttributes()) !=
+                   FunctionType::ARM_None) ||
+         (FD->hasAttr<ArmNewAttr>() && FD->getAttr<ArmNewAttr>()->isNewZT0());
+}
diff --git a/clang/lib/AST/DeclBase.cpp b/clang/lib/AST/DeclBase.cpp
index 8506b95f761fe..fc16448cf9e90 100644
--- a/clang/lib/AST/DeclBase.cpp
+++ b/clang/lib/AST/DeclBase.cpp
@@ -245,7 +245,7 @@ bool Decl::isTemplateParameterPack() const {
 }
 
 bool Decl::isParameterPack() const {
-  if (const auto *Var = dyn_cast<VarDecl>(this))
+  if (const auto *Var = dyn_cast<ValueDecl>(this))
     return Var->isParameterPack();
 
   return isTemplateParameterPack();
@@ -1203,6 +1203,8 @@ const FunctionType *Decl::getFunctionType(bool BlocksToo) const {
 
   if (Ty->isFunctionPointerType())
     Ty = Ty->castAs<PointerType>()->getPointeeType();
+  else if (Ty->isMemberFunctionPointerType())
+    Ty = Ty->castAs<MemberPointerType>()->getPointeeType();
   else if (Ty->isFunctionReferenceType())
     Ty = Ty->castAs<ReferenceType>()->getPointeeType();
   else if (BlocksToo && Ty->isBlockPointerType())
diff --git a/clang/lib/AST/DeclCXX.cpp b/clang/lib/AST/DeclCXX.cpp
index 44f45898fb483..e394e0515e599 100644
--- a/clang/lib/AST/DeclCXX.cpp
+++ b/clang/lib/AST/DeclCXX.cpp
@@ -1987,7 +1987,8 @@ CXXRecordDecl *CXXRecordDecl::getInstantiatedFromMemberClass() const {
 }
 
 MemberSpecializationInfo *CXXRecordDecl::getMemberSpecializationInfo() const {
-  return TemplateOrInstantiation.dyn_cast<MemberSpecializationInfo *>();
+  return dyn_cast_if_present<MemberSpecializationInfo *>(
+      TemplateOrInstantiation);
 }
 
 void
@@ -2001,7 +2002,7 @@ CXXRecordDecl::setInstantiationOfMemberClass(CXXRecordDecl *RD,
 }
 
 ClassTemplateDecl *CXXRecordDecl::getDescribedClassTemplate() const {
-  return TemplateOrInstantiation.dyn_cast<ClassTemplateDecl *>();
+  return dyn_cast_if_present<ClassTemplateDecl *>(TemplateOrInstantiation);
 }
 
 void CXXRecordDecl::setDescribedClassTemplate(ClassTemplateDecl *Template) {
@@ -2045,7 +2046,7 @@ const CXXRecordDecl *CXXRecordDecl::getTemplateInstantiationPattern() const {
   // specialization from which it was instantiated.
   if (auto *TD = dyn_cast<ClassTemplateSpecializationDecl>(this)) {
     auto From = TD->getInstantiatedFrom();
-    if (auto *CTD = From.dyn_cast<ClassTemplateDecl *>()) {
+    if (auto *CTD = dyn_cast_if_present<ClassTemplateDecl *>(From)) {
       while (auto *NewCTD = CTD->getInstantiatedFromMemberTemplate()) {
         if (NewCTD->isMemberSpecialization())
           break;
@@ -2054,7 +2055,8 @@ const CXXRecordDecl *CXXRecordDecl::getTemplateInstantiationPattern() const {
       return GetDefinitionOrSelf(CTD->getTemplatedDecl());
     }
     if (auto *CTPSD =
-            From.dyn_cast<ClassTemplatePartialSpecializationDecl *>()) {
+            dyn_cast_if_present<ClassTemplatePartialSpecializationDecl *>(
+                From)) {
       while (auto *NewCTPSD = CTPSD->getInstantiatedFromMember()) {
         if (NewCTPSD->isMemberSpecialization())
           break;
@@ -2290,18 +2292,22 @@ CXXDeductionGuideDecl *CXXDeductionGuideDecl::Create(
     ASTContext &C, DeclContext *DC, SourceLocation StartLoc,
     ExplicitSpecifier ES, const DeclarationNameInfo &NameInfo, QualType T,
     TypeSourceInfo *TInfo, SourceLocation EndLocation, CXXConstructorDecl *Ctor,
-    DeductionCandidate Kind, Expr *TrailingRequiresClause) {
-  return new (C, DC)
-      CXXDeductionGuideDecl(C, DC, StartLoc, ES, NameInfo, T, TInfo,
-                            EndLocation, Ctor, Kind, TrailingRequiresClause);
+    DeductionCandidate Kind, Expr *TrailingRequiresClause,
+    const CXXDeductionGuideDecl *GeneratedFrom,
+    SourceDeductionGuideKind SourceKind) {
+  return new (C, DC) CXXDeductionGuideDecl(
+      C, DC, StartLoc, ES, NameInfo, T, TInfo, EndLocation, Ctor, Kind,
+      TrailingRequiresClause, GeneratedFrom, SourceKind);
 }
 
 CXXDeductionGuideDecl *
 CXXDeductionGuideDecl::CreateDeserialized(ASTContext &C, GlobalDeclID ID) {
   return new (C, ID) CXXDeductionGuideDecl(
-      C, nullptr, SourceLocation(), ExplicitSpecifier(), DeclarationNameInfo(),
-      QualType(), nullptr, SourceLocation(), nullptr,
-      DeductionCandidate::Normal, nullptr);
+      C, /*DC=*/nullptr, SourceLocation(), ExplicitSpecifier(),
+      DeclarationNameInfo(), QualType(), /*TInfo=*/nullptr, SourceLocation(),
+      /*Ctor=*/nullptr, DeductionCandidate::Normal,
+      /*TrailingRequiresClause=*/nullptr,
+      /*GeneratedFrom=*/nullptr, SourceDeductionGuideKind::None);
 }
 
 RequiresExprBodyDecl *RequiresExprBodyDecl::Create(
@@ -3456,19 +3462,21 @@ VarDecl *ValueDecl::getPotentiallyDecomposedVarDecl() {
   if (auto *Var = llvm::dyn_cast<VarDecl>(this))
     return Var;
   if (auto *BD = llvm::dyn_cast<BindingDecl>(this))
-    return llvm::dyn_cast<VarDecl>(BD->getDecomposedDecl());
+    return llvm::dyn_cast_if_present<VarDecl>(BD->getDecomposedDecl());
   return nullptr;
 }
 
 void BindingDecl::anchor() {}
 
 BindingDecl *BindingDecl::Create(ASTContext &C, DeclContext *DC,
-                                 SourceLocation IdLoc, IdentifierInfo *Id) {
-  return new (C, DC) BindingDecl(DC, IdLoc, Id);
+                                 SourceLocation IdLoc, IdentifierInfo *Id,
+                                 QualType T) {
+  return new (C, DC) BindingDecl(DC, IdLoc, Id, T);
 }
 
 BindingDecl *BindingDecl::CreateDeserialized(ASTContext &C, GlobalDeclID ID) {
-  return new (C, ID) BindingDecl(nullptr, SourceLocation(), nullptr);
+  return new (C, ID)
+      BindingDecl(nullptr, SourceLocation(), nullptr, QualType());
 }
 
 VarDecl *BindingDecl::getHoldingVar() const {
@@ -3484,6 +3492,12 @@ VarDecl *BindingDecl::getHoldingVar() const {
   return VD;
 }
 
+llvm::ArrayRef<Expr *> BindingDecl::getBindingPackExprs() const {
+  assert(Binding && "expecting a pack expr");
+  auto *RP = cast<ResolvedUnexpandedPackExpr>(Binding);
+  return RP->getExprs();
+}
+
 void DecompositionDecl::anchor() {}
 
 DecompositionDecl *DecompositionDecl::Create(ASTContext &C, DeclContext *DC,
diff --git a/clang/lib/AST/DeclTemplate.cpp b/clang/lib/AST/DeclTemplate.cpp
index 40ee3753c2422..7fb89bf5b499f 100644
--- a/clang/lib/AST/DeclTemplate.cpp
+++ b/clang/lib/AST/DeclTemplate.cpp
@@ -957,18 +957,17 @@ FunctionTemplateSpecializationInfo *FunctionTemplateSpecializationInfo::Create(
 // ClassTemplateSpecializationDecl Implementation
 //===----------------------------------------------------------------------===//
 
-ClassTemplateSpecializationDecl::
-ClassTemplateSpecializationDecl(ASTContext &Context, Kind DK, TagKind TK,
-                                DeclContext *DC, SourceLocation StartLoc,
-                                SourceLocation IdLoc,
-                                ClassTemplateDecl *SpecializedTemplate,
-                                ArrayRef<TemplateArgument> Args,
-                                ClassTemplateSpecializationDecl *PrevDecl)
+ClassTemplateSpecializationDecl::ClassTemplateSpecializationDecl(
+    ASTContext &Context, Kind DK, TagKind TK, DeclContext *DC,
+    SourceLocation StartLoc, SourceLocation IdLoc,
+    ClassTemplateDecl *SpecializedTemplate, ArrayRef<TemplateArgument> Args,
+    bool StrictPackMatch, ClassTemplateSpecializationDecl *PrevDecl)
     : CXXRecordDecl(DK, TK, Context, DC, StartLoc, IdLoc,
                     SpecializedTemplate->getIdentifier(), PrevDecl),
-    SpecializedTemplate(SpecializedTemplate),
-    TemplateArgs(TemplateArgumentList::CreateCopy(Context, Args)),
-    SpecializationKind(TSK_Undeclared) {
+      SpecializedTemplate(SpecializedTemplate),
+      TemplateArgs(TemplateArgumentList::CreateCopy(Context, Args)),
+      SpecializationKind(TSK_Undeclared), StrictPackMatch(StrictPackMatch) {
+  assert(DK == Kind::ClassTemplateSpecialization || StrictPackMatch == false);
 }
 
 ClassTemplateSpecializationDecl::ClassTemplateSpecializationDecl(ASTContext &C,
@@ -977,18 +976,14 @@ ClassTemplateSpecializationDecl::ClassTemplateSpecializationDecl(ASTContext &C,
                     SourceLocation(), nullptr, nullptr),
       SpecializationKind(TSK_Undeclared) {}
 
-ClassTemplateSpecializationDecl *
-ClassTemplateSpecializationDecl::Create(ASTContext &Context, TagKind TK,
-                                        DeclContext *DC,
-                                        SourceLocation StartLoc,
-                                        SourceLocation IdLoc,
-                                        ClassTemplateDecl *SpecializedTemplate,
-                                        ArrayRef<TemplateArgument> Args,
-                                   ClassTemplateSpecializationDecl *PrevDecl) {
-  auto *Result =
-      new (Context, DC) ClassTemplateSpecializationDecl(
-          Context, ClassTemplateSpecialization, TK, DC, StartLoc, IdLoc,
-          SpecializedTemplate, Args, PrevDecl);
+ClassTemplateSpecializationDecl *ClassTemplateSpecializationDecl::Create(
+    ASTContext &Context, TagKind TK, DeclContext *DC, SourceLocation StartLoc,
+    SourceLocation IdLoc, ClassTemplateDecl *SpecializedTemplate,
+    ArrayRef<TemplateArgument> Args, bool StrictPackMatch,
+    ClassTemplateSpecializationDecl *PrevDecl) {
+  auto *Result = new (Context, DC) ClassTemplateSpecializationDecl(
+      Context, ClassTemplateSpecialization, TK, DC, StartLoc, IdLoc,
+      SpecializedTemplate, Args, StrictPackMatch, PrevDecl);
   Result->setMayHaveOutOfDateDef(false);
 
   // If the template decl is incomplete, copy the external lexical storage from
@@ -1049,7 +1044,7 @@ ClassTemplateSpecializationDecl::getSourceRange() const {
     assert(!Pattern.isNull() &&
            "Class template specialization without pattern?");
     if (const auto *CTPSD =
-            Pattern.dyn_cast<ClassTemplatePartialSpecializationDecl *>())
+            dyn_cast<ClassTemplatePartialSpecializationDecl *>(Pattern))
       return CTPSD->getSourceRange();
     return cast<ClassTemplateDecl *>(Pattern)->getSourceRange();
   }
@@ -1077,7 +1072,7 @@ ClassTemplateSpecializationDecl::getSourceRange() const {
 }
 
 void ClassTemplateSpecializationDecl::setExternKeywordLoc(SourceLocation Loc) {
-  auto *Info = ExplicitInfo.dyn_cast<ExplicitInstantiationInfo *>();
+  auto *Info = dyn_cast_if_present<ExplicitInstantiationInfo *>(ExplicitInfo);
   if (!Info) {
     // Don't allocate if the location is invalid.
     if (Loc.isInvalid())
@@ -1091,7 +1086,7 @@ void ClassTemplateSpecializationDecl::setExternKeywordLoc(SourceLocation Loc) {
 
 void ClassTemplateSpecializationDecl::setTemplateKeywordLoc(
     SourceLocation Loc) {
-  auto *Info = ExplicitInfo.dyn_cast<ExplicitInstantiationInfo *>();
+  auto *Info = dyn_cast_if_present<ExplicitInstantiationInfo *>(ExplicitInfo);
   if (!Info) {
     // Don't allocate if the location is invalid.
     if (Loc.isInvalid())
@@ -1175,7 +1170,9 @@ ClassTemplatePartialSpecializationDecl::ClassTemplatePartialSpecializationDecl(
     ClassTemplatePartialSpecializationDecl *PrevDecl)
     : ClassTemplateSpecializationDecl(
           Context, ClassTemplatePartialSpecialization, TK, DC, StartLoc, IdLoc,
-          SpecializedTemplate, Args, PrevDecl),
+          // Tracking StrictPackMatch for Partial
+          // Specializations is not needed.
+          SpecializedTemplate, Args, /*StrictPackMatch=*/false, PrevDecl),
       TemplateParams(Params), InstantiatedFromMember(nullptr, false) {
   if (AdoptTemplateParameterList(Params, this))
     setInvalidDecl();
@@ -1463,7 +1460,7 @@ SourceRange VarTemplateSpecializationDecl::getSourceRange() const {
     assert(!Pattern.isNull() &&
            "Variable template specialization without pattern?");
     if (const auto *VTPSD =
-            Pattern.dyn_cast<VarTemplatePartialSpecializationDecl *>())
+            dyn_cast<VarTemplatePartialSpecializationDecl *>(Pattern))
       return VTPSD->getSourceRange();
     VarTemplateDecl *VTD = cast<VarTemplateDecl *>(Pattern);
     if (hasInit()) {
@@ -1496,7 +1493,7 @@ SourceRange VarTemplateSpecializationDecl::getSourceRange() const {
 }
 
 void VarTemplateSpecializationDecl::setExternKeywordLoc(SourceLocation Loc) {
-  auto *Info = ExplicitInfo.dyn_cast<ExplicitInstantiationInfo *>();
+  auto *Info = dyn_cast_if_present<ExplicitInstantiationInfo *>(ExplicitInfo);
   if (!Info) {
     // Don't allocate if the location is invalid.
     if (Loc.isInvalid())
@@ -1509,7 +1506,7 @@ void VarTemplateSpecializationDecl::setExternKeywordLoc(SourceLocation Loc) {
 }
 
 void VarTemplateSpecializationDecl::setTemplateKeywordLoc(SourceLocation Loc) {
-  auto *Info = ExplicitInfo.dyn_cast<ExplicitInstantiationInfo *>();
+  auto *Info = dyn_cast_if_present<ExplicitInstantiationInfo *>(ExplicitInfo);
   if (!Info) {
     // Don't allocate if the location is invalid.
     if (Loc.isInvalid())
@@ -1773,7 +1770,7 @@ TemplateParameterList *clang::getReplacedTemplateParameterList(Decl *D) {
     const auto *CTSD = cast<ClassTemplateSpecializationDecl>(D);
     auto P = CTSD->getSpecializedTemplateOrPartial();
     if (const auto *CTPSD =
-            P.dyn_cast<ClassTemplatePartialSpecializationDecl *>())
+            dyn_cast<ClassTemplatePartialSpecializationDecl *>(P))
       return CTPSD->getTemplateParameters();
     return cast<ClassTemplateDecl *>(P)->getTemplateParameters();
   }
@@ -1801,8 +1798,7 @@ TemplateParameterList *clang::getReplacedTemplateParameterList(Decl *D) {
   case Decl::Kind::VarTemplateSpecialization: {
     const auto *VTSD = cast<VarTemplateSpecializationDecl>(D);
     auto P = VTSD->getSpecializedTemplateOrPartial();
-    if (const auto *VTPSD =
-            P.dyn_cast<VarTemplatePartialSpecializationDecl *>())
+    if (const auto *VTPSD = dyn_cast<VarTemplatePartialSpecializationDecl *>(P))
       return VTPSD->getTemplateParameters();
     return cast<VarTemplateDecl *>(P)->getTemplateParameters();
   }
diff --git a/clang/lib/AST/DynamicRecursiveASTVisitor.cpp b/clang/lib/AST/DynamicRecursiveASTVisitor.cpp
index 8cfabd9f3e93f..b478e7a39ea18 100644
--- a/clang/lib/AST/DynamicRecursiveASTVisitor.cpp
+++ b/clang/lib/AST/DynamicRecursiveASTVisitor.cpp
@@ -89,9 +89,9 @@ using namespace clang;
 //
 //   End result: RAV::TraverseCallExpr() is executed,
 namespace {
-struct Impl : RecursiveASTVisitor<Impl> {
-  DynamicRecursiveASTVisitor &Visitor;
-  Impl(DynamicRecursiveASTVisitor &Visitor) : Visitor(Visitor) {}
+template <bool Const> struct Impl : RecursiveASTVisitor<Impl<Const>> {
+  DynamicRecursiveASTVisitorBase<Const> &Visitor;
+  Impl(DynamicRecursiveASTVisitorBase<Const> &Visitor) : Visitor(Visitor) {}
 
   bool shouldVisitTemplateInstantiations() const {
     return Visitor.ShouldVisitTemplateInstantiations;
@@ -189,8 +189,10 @@ struct Impl : RecursiveASTVisitor<Impl> {
 
   // TraverseStmt() always passes in a queue, so we have no choice but to
   // accept it as a parameter here.
-  bool dataTraverseNode(Stmt *S, DataRecursionQueue * = nullptr) {
-    // But since don't support postorder traversal, we don't need it, so
+  bool dataTraverseNode(
+      Stmt *S,
+      typename RecursiveASTVisitor<Impl>::DataRecursionQueue * = nullptr) {
+    // But since we don't support postorder traversal, we don't need it, so
     // simply discard it here. This way, derived classes don't need to worry
     // about including it as a parameter that they never use.
     return Visitor.dataTraverseNode(S);
@@ -266,187 +268,106 @@ struct Impl : RecursiveASTVisitor<Impl> {
 };
 } // namespace
 
-void DynamicRecursiveASTVisitor::anchor() {}
-
-bool DynamicRecursiveASTVisitor::TraverseAST(ASTContext &AST) {
-  return Impl(*this).RecursiveASTVisitor<Impl>::TraverseAST(AST);
-}
-
-bool DynamicRecursiveASTVisitor::TraverseAttr(Attr *At) {
-  return Impl(*this).RecursiveASTVisitor<Impl>::TraverseAttr(At);
-}
-
-bool DynamicRecursiveASTVisitor::TraverseConstructorInitializer(
-    CXXCtorInitializer *Init) {
-  return Impl(*this).RecursiveASTVisitor<Impl>::TraverseConstructorInitializer(
-      Init);
-}
-
-bool DynamicRecursiveASTVisitor::TraverseDecl(Decl *D) {
-  return Impl(*this).RecursiveASTVisitor<Impl>::TraverseDecl(D);
-}
-
-bool DynamicRecursiveASTVisitor::TraverseLambdaCapture(LambdaExpr *LE,
-                                                       const LambdaCapture *C,
-                                                       Expr *Init) {
-  return Impl(*this).RecursiveASTVisitor<Impl>::TraverseLambdaCapture(LE, C,
-                                                                      Init);
-}
-
-bool DynamicRecursiveASTVisitor::TraverseStmt(Stmt *S) {
-  return Impl(*this).RecursiveASTVisitor<Impl>::TraverseStmt(S);
-}
-
-bool DynamicRecursiveASTVisitor::TraverseTemplateArgument(
-    const TemplateArgument &Arg) {
-  return Impl(*this).RecursiveASTVisitor<Impl>::TraverseTemplateArgument(Arg);
-}
-
-bool DynamicRecursiveASTVisitor::TraverseTemplateArguments(
-    ArrayRef<TemplateArgument> Args) {
-  return Impl(*this).RecursiveASTVisitor<Impl>::TraverseTemplateArguments(Args);
-}
-
-bool DynamicRecursiveASTVisitor::TraverseTemplateArgumentLoc(
-    const TemplateArgumentLoc &ArgLoc) {
-  return Impl(*this).RecursiveASTVisitor<Impl>::TraverseTemplateArgumentLoc(
-      ArgLoc);
-}
-
-bool DynamicRecursiveASTVisitor::TraverseTemplateName(TemplateName Template) {
-  return Impl(*this).RecursiveASTVisitor<Impl>::TraverseTemplateName(Template);
-}
-
-bool DynamicRecursiveASTVisitor::TraverseType(QualType T) {
-  return Impl(*this).RecursiveASTVisitor<Impl>::TraverseType(T);
-}
-
-bool DynamicRecursiveASTVisitor::TraverseTypeLoc(TypeLoc TL) {
-  return Impl(*this).RecursiveASTVisitor<Impl>::TraverseTypeLoc(TL);
-}
-
-bool DynamicRecursiveASTVisitor::TraverseTypeConstraint(
-    const TypeConstraint *C) {
-  return Impl(*this).RecursiveASTVisitor<Impl>::TraverseTypeConstraint(C);
-}
-bool DynamicRecursiveASTVisitor::TraverseObjCProtocolLoc(
-    ObjCProtocolLoc ProtocolLoc) {
-  return Impl(*this).RecursiveASTVisitor<Impl>::TraverseObjCProtocolLoc(
-      ProtocolLoc);
-}
-
-bool DynamicRecursiveASTVisitor::TraverseConceptRequirement(
-    concepts::Requirement *R) {
-  return Impl(*this).RecursiveASTVisitor<Impl>::TraverseConceptRequirement(R);
-}
-bool DynamicRecursiveASTVisitor::TraverseConceptTypeRequirement(
-    concepts::TypeRequirement *R) {
-  return Impl(*this).RecursiveASTVisitor<Impl>::TraverseConceptTypeRequirement(
-      R);
-}
-bool DynamicRecursiveASTVisitor::TraverseConceptExprRequirement(
-    concepts::ExprRequirement *R) {
-  return Impl(*this).RecursiveASTVisitor<Impl>::TraverseConceptExprRequirement(
-      R);
-}
-bool DynamicRecursiveASTVisitor::TraverseConceptNestedRequirement(
-    concepts::NestedRequirement *R) {
-  return Impl(*this)
-      .RecursiveASTVisitor<Impl>::TraverseConceptNestedRequirement(R);
-}
-
-bool DynamicRecursiveASTVisitor::TraverseConceptReference(
-    ConceptReference *CR) {
-  return Impl(*this).RecursiveASTVisitor<Impl>::TraverseConceptReference(CR);
-}
-
-bool DynamicRecursiveASTVisitor::TraverseCXXBaseSpecifier(
-    const CXXBaseSpecifier &Base) {
-  return Impl(*this).RecursiveASTVisitor<Impl>::TraverseCXXBaseSpecifier(Base);
-}
-
-bool DynamicRecursiveASTVisitor::TraverseDeclarationNameInfo(
-    DeclarationNameInfo NameInfo) {
-  return Impl(*this).RecursiveASTVisitor<Impl>::TraverseDeclarationNameInfo(
-      NameInfo);
-}
-
-bool DynamicRecursiveASTVisitor::TraverseNestedNameSpecifier(
-    NestedNameSpecifier *NNS) {
-  return Impl(*this).RecursiveASTVisitor<Impl>::TraverseNestedNameSpecifier(
-      NNS);
-}
-
-bool DynamicRecursiveASTVisitor::TraverseNestedNameSpecifierLoc(
-    NestedNameSpecifierLoc NNS) {
-  return Impl(*this).RecursiveASTVisitor<Impl>::TraverseNestedNameSpecifierLoc(
-      NNS);
+template <bool Const> void DynamicRecursiveASTVisitorBase<Const>::anchor() {}
+
+// Helper macros to forward a call to the base implementation since that
+// ends up getting very verbose otherwise.
+
+// This calls the RecursiveASTVisitor implementation of the same function,
+// stripping any 'const' that the DRAV implementation may have added since
+// the RAV implementation largely doesn't use 'const'.
+#define FORWARD_TO_BASE(Function, Type, RefOrPointer)                          \
+  template <bool Const>                                                        \
+  bool DynamicRecursiveASTVisitorBase<Const>::Function(                        \
+      MaybeConst<Type> RefOrPointer Param) {                                   \
+    return Impl<Const>(*this).RecursiveASTVisitor<Impl<Const>>::Function(      \
+        const_cast<Type RefOrPointer>(Param));                                 \
+  }
+
+// Same as 'FORWARD_TO_BASE', but doesn't change the parameter type in any way.
+#define FORWARD_TO_BASE_EXACT(Function, Type)                                  \
+  template <bool Const>                                                        \
+  bool DynamicRecursiveASTVisitorBase<Const>::Function(Type Param) {           \
+    return Impl<Const>(*this).RecursiveASTVisitor<Impl<Const>>::Function(      \
+        Param);                                                                \
+  }
+
+FORWARD_TO_BASE(TraverseAST, ASTContext, &)
+FORWARD_TO_BASE(TraverseAttr, Attr, *)
+FORWARD_TO_BASE(TraverseConstructorInitializer, CXXCtorInitializer, *)
+FORWARD_TO_BASE(TraverseDecl, Decl, *)
+FORWARD_TO_BASE(TraverseStmt, Stmt, *)
+FORWARD_TO_BASE(TraverseNestedNameSpecifier, NestedNameSpecifier, *)
+FORWARD_TO_BASE(TraverseTemplateInstantiations, ClassTemplateDecl, *)
+FORWARD_TO_BASE(TraverseTemplateInstantiations, VarTemplateDecl, *)
+FORWARD_TO_BASE(TraverseTemplateInstantiations, FunctionTemplateDecl, *)
+FORWARD_TO_BASE(TraverseConceptRequirement, concepts::Requirement, *)
+FORWARD_TO_BASE(TraverseConceptTypeRequirement, concepts::TypeRequirement, *)
+FORWARD_TO_BASE(TraverseConceptExprRequirement, concepts::ExprRequirement, *)
+FORWARD_TO_BASE(TraverseConceptReference, ConceptReference, *)
+FORWARD_TO_BASE(TraverseConceptNestedRequirement,
+                concepts::NestedRequirement, *)
+
+FORWARD_TO_BASE_EXACT(TraverseCXXBaseSpecifier, const CXXBaseSpecifier &)
+FORWARD_TO_BASE_EXACT(TraverseDeclarationNameInfo, DeclarationNameInfo)
+FORWARD_TO_BASE_EXACT(TraverseTemplateArgument, const TemplateArgument &)
+FORWARD_TO_BASE_EXACT(TraverseTemplateArguments, ArrayRef<TemplateArgument>)
+FORWARD_TO_BASE_EXACT(TraverseTemplateArgumentLoc, const TemplateArgumentLoc &)
+FORWARD_TO_BASE_EXACT(TraverseTemplateName, TemplateName)
+FORWARD_TO_BASE_EXACT(TraverseType, QualType)
+FORWARD_TO_BASE_EXACT(TraverseTypeLoc, TypeLoc)
+FORWARD_TO_BASE_EXACT(TraverseTypeConstraint, const TypeConstraint *)
+FORWARD_TO_BASE_EXACT(TraverseObjCProtocolLoc, ObjCProtocolLoc)
+FORWARD_TO_BASE_EXACT(TraverseNestedNameSpecifierLoc, NestedNameSpecifierLoc)
+
+template <bool Const>
+bool DynamicRecursiveASTVisitorBase<Const>::TraverseLambdaCapture(
+    MaybeConst<LambdaExpr> *LE, const LambdaCapture *C,
+    MaybeConst<Expr> *Init) {
+  return Impl<Const>(*this)
+      .RecursiveASTVisitor<Impl<Const>>::TraverseLambdaCapture(
+          const_cast<LambdaExpr *>(LE), C, const_cast<Expr *>(Init));
 }
 
-bool DynamicRecursiveASTVisitor::dataTraverseNode(Stmt *S) {
-  return Impl(*this).RecursiveASTVisitor<Impl>::dataTraverseNode(S, nullptr);
+template <bool Const>
+bool DynamicRecursiveASTVisitorBase<Const>::dataTraverseNode(
+    MaybeConst<Stmt> *S) {
+  return Impl<Const>(*this).RecursiveASTVisitor<Impl<Const>>::dataTraverseNode(
+      const_cast<Stmt *>(S), nullptr);
 }
 
-#define DEF_TRAVERSE_TMPL_INST(kind)                                           \
-  bool DynamicRecursiveASTVisitor::TraverseTemplateInstantiations(             \
-      kind##TemplateDecl *D) {                                                 \
-    return Impl(*this)                                                         \
-        .RecursiveASTVisitor<Impl>::TraverseTemplateInstantiations(D);         \
-  }
-DEF_TRAVERSE_TMPL_INST(Class)
-DEF_TRAVERSE_TMPL_INST(Var)
-DEF_TRAVERSE_TMPL_INST(Function)
-#undef DEF_TRAVERSE_TMPL_INST
-
 // Declare Traverse*() for and friends all concrete Decl classes.
 #define ABSTRACT_DECL(DECL)
 #define DECL(CLASS, BASE)                                                      \
-  bool DynamicRecursiveASTVisitor::Traverse##CLASS##Decl(CLASS##Decl *D) {     \
-    return Impl(*this).RecursiveASTVisitor<Impl>::Traverse##CLASS##Decl(D);    \
-  }                                                                            \
-  bool DynamicRecursiveASTVisitor::WalkUpFrom##CLASS##Decl(CLASS##Decl *D) {   \
-    return Impl(*this).RecursiveASTVisitor<Impl>::WalkUpFrom##CLASS##Decl(D);  \
-  }
+  FORWARD_TO_BASE(Traverse##CLASS##Decl, CLASS##Decl, *)                       \
+  FORWARD_TO_BASE(WalkUpFrom##CLASS##Decl, CLASS##Decl, *)
 #include "clang/AST/DeclNodes.inc"
 
 // Declare Traverse*() and friends for all concrete Stmt classes.
 #define ABSTRACT_STMT(STMT)
-#define STMT(CLASS, PARENT)                                                    \
-  bool DynamicRecursiveASTVisitor::Traverse##CLASS(CLASS *S) {                 \
-    return Impl(*this).RecursiveASTVisitor<Impl>::Traverse##CLASS(S);          \
-  }
+#define STMT(CLASS, PARENT) FORWARD_TO_BASE(Traverse##CLASS, CLASS, *)
 #include "clang/AST/StmtNodes.inc"
 
-#define STMT(CLASS, PARENT)                                                    \
-  bool DynamicRecursiveASTVisitor::WalkUpFrom##CLASS(CLASS *S) {               \
-    return Impl(*this).RecursiveASTVisitor<Impl>::WalkUpFrom##CLASS(S);        \
-  }
+#define STMT(CLASS, PARENT) FORWARD_TO_BASE(WalkUpFrom##CLASS, CLASS, *)
 #include "clang/AST/StmtNodes.inc"
 
-// Declare Traverse*() and friends for all concrete Typeclasses.
+// Declare Traverse*() and friends for all concrete Type classes.
 #define ABSTRACT_TYPE(CLASS, BASE)
 #define TYPE(CLASS, BASE)                                                      \
-  bool DynamicRecursiveASTVisitor::Traverse##CLASS##Type(CLASS##Type *T) {     \
-    return Impl(*this).RecursiveASTVisitor<Impl>::Traverse##CLASS##Type(T);    \
-  }                                                                            \
-  bool DynamicRecursiveASTVisitor::WalkUpFrom##CLASS##Type(CLASS##Type *T) {   \
-    return Impl(*this).RecursiveASTVisitor<Impl>::WalkUpFrom##CLASS##Type(T);  \
-  }
+  FORWARD_TO_BASE(Traverse##CLASS##Type, CLASS##Type, *)                       \
+  FORWARD_TO_BASE(WalkUpFrom##CLASS##Type, CLASS##Type, *)
 #include "clang/AST/TypeNodes.inc"
 
 #define ABSTRACT_TYPELOC(CLASS, BASE)
 #define TYPELOC(CLASS, BASE)                                                   \
-  bool DynamicRecursiveASTVisitor::Traverse##CLASS##TypeLoc(                   \
-      CLASS##TypeLoc TL) {                                                     \
-    return Impl(*this).RecursiveASTVisitor<Impl>::Traverse##CLASS##TypeLoc(    \
-        TL);                                                                   \
-  }
+  FORWARD_TO_BASE_EXACT(Traverse##CLASS##TypeLoc, CLASS##TypeLoc)
 #include "clang/AST/TypeLocNodes.def"
 
 #define TYPELOC(CLASS, BASE)                                                   \
-  bool DynamicRecursiveASTVisitor::WalkUpFrom##CLASS##TypeLoc(                 \
-      CLASS##TypeLoc TL) {                                                     \
-    return Impl(*this).RecursiveASTVisitor<Impl>::WalkUpFrom##CLASS##TypeLoc(  \
-        TL);                                                                   \
-  }
+  FORWARD_TO_BASE_EXACT(WalkUpFrom##CLASS##TypeLoc, CLASS##TypeLoc)
 #include "clang/AST/TypeLocNodes.def"
+
+namespace clang {
+template class DynamicRecursiveASTVisitorBase<false>;
+template class DynamicRecursiveASTVisitorBase<true>;
+} // namespace clang
diff --git a/clang/lib/AST/Expr.cpp b/clang/lib/AST/Expr.cpp
index 31b95bca613c2..c22aa66ba2cfb 100644
--- a/clang/lib/AST/Expr.cpp
+++ b/clang/lib/AST/Expr.cpp
@@ -895,8 +895,7 @@ std::string PredefinedExpr::ComputeName(PredefinedIdentKind IK,
     // type deduction and lambdas. For trailing return types resolve the
     // decltype expression. Otherwise print the real type when this is
     // not a constructor or destructor.
-    if (isa<CXXMethodDecl>(FD) &&
-         cast<CXXMethodDecl>(FD)->getParent()->isLambda())
+    if (isLambdaMethod(FD))
       Proto = "auto " + Proto;
     else if (FT && FT->getReturnType()->getAs<DecltypeType>())
       FT->getReturnType()
@@ -1957,6 +1956,7 @@ bool CastExpr::CastConsistency() const {
   case CK_FixedPointToBoolean:
   case CK_HLSLArrayRValue:
   case CK_HLSLVectorTruncation:
+  case CK_HLSLElementwiseCast:
   CheckNoBasePath:
     assert(path_empty() && "Cast kind should not have a base path!");
     break;
@@ -3660,6 +3660,7 @@ bool Expr::HasSideEffects(const ASTContext &Ctx,
   case PackIndexingExprClass:
   case HLSLOutArgExprClass:
   case OpenACCAsteriskSizeExprClass:
+  case ResolvedUnexpandedPackExprClass:
     // These never have a side-effect.
     return false;
 
diff --git a/clang/lib/AST/ExprCXX.cpp b/clang/lib/AST/ExprCXX.cpp
index 5bf5d6adf525a..d900af895b42a 100644
--- a/clang/lib/AST/ExprCXX.cpp
+++ b/clang/lib/AST/ExprCXX.cpp
@@ -1965,3 +1965,52 @@ CXXFoldExpr::CXXFoldExpr(QualType T, UnresolvedLookupExpr *Callee,
   SubExprs[SubExpr::RHS] = RHS;
   setDependence(computeDependence(this));
 }
+
+ResolvedUnexpandedPackExpr::ResolvedUnexpandedPackExpr(SourceLocation BL,
+                                                       QualType QT,
+                                                       unsigned NumExprs)
+    : Expr(ResolvedUnexpandedPackExprClass, QT, VK_PRValue, OK_Ordinary),
+      BeginLoc(BL), NumExprs(NumExprs) {
+  // C++ [temp.dep.expr]p3
+  // An id-expression is type-dependent if it is
+  //    - associated by name lookup with a pack
+  setDependence(ExprDependence::TypeValueInstantiation |
+                ExprDependence::UnexpandedPack);
+}
+
+ResolvedUnexpandedPackExpr *
+ResolvedUnexpandedPackExpr::CreateDeserialized(ASTContext &Ctx,
+                                               unsigned NumExprs) {
+  void *Mem = Ctx.Allocate(totalSizeToAlloc<Expr *>(NumExprs),
+                           alignof(ResolvedUnexpandedPackExpr));
+  return new (Mem)
+      ResolvedUnexpandedPackExpr(SourceLocation(), QualType(), NumExprs);
+}
+
+ResolvedUnexpandedPackExpr *
+ResolvedUnexpandedPackExpr::Create(ASTContext &Ctx, SourceLocation BL,
+                                   QualType T, unsigned NumExprs) {
+  void *Mem = Ctx.Allocate(totalSizeToAlloc<Expr *>(NumExprs),
+                           alignof(ResolvedUnexpandedPackExpr));
+  ResolvedUnexpandedPackExpr *New =
+      new (Mem) ResolvedUnexpandedPackExpr(BL, T, NumExprs);
+
+  auto Exprs = New->getExprs();
+  std::uninitialized_fill(Exprs.begin(), Exprs.end(), nullptr);
+
+  return New;
+}
+
+ResolvedUnexpandedPackExpr *
+ResolvedUnexpandedPackExpr::Create(ASTContext &Ctx, SourceLocation BL,
+                                   QualType T, ArrayRef<Expr *> Exprs) {
+  auto *New = Create(Ctx, BL, T, Exprs.size());
+  std::uninitialized_copy(Exprs.begin(), Exprs.end(), New->getExprs().begin());
+  return New;
+}
+
+ResolvedUnexpandedPackExpr *ResolvedUnexpandedPackExpr::getFromDecl(Decl *D) {
+  if (auto *BD = dyn_cast<BindingDecl>(D))
+    return dyn_cast_if_present<ResolvedUnexpandedPackExpr>(BD->getBinding());
+  return nullptr;
+}
diff --git a/clang/lib/AST/ExprClassification.cpp b/clang/lib/AST/ExprClassification.cpp
index 3f37d06cc8f3a..5225c3ca773ad 100644
--- a/clang/lib/AST/ExprClassification.cpp
+++ b/clang/lib/AST/ExprClassification.cpp
@@ -451,6 +451,13 @@ static Cl::Kinds ClassifyInternal(ASTContext &Ctx, const Expr *E) {
   case Expr::PackExpansionExprClass:
     return ClassifyInternal(Ctx, cast<PackExpansionExpr>(E)->getPattern());
 
+  case Expr::ResolvedUnexpandedPackExprClass: {
+    if (cast<ResolvedUnexpandedPackExpr>(E)->getNumExprs() > 0)
+      return ClassifyInternal(
+          Ctx, cast<ResolvedUnexpandedPackExpr>(E)->getExpansion(0));
+    return Cl::CL_LValue;
+  }
+
   case Expr::MaterializeTemporaryExprClass:
     return cast<MaterializeTemporaryExpr>(E)->isBoundToLvalueReference()
               ? Cl::CL_LValue
diff --git a/clang/lib/AST/ExprConstant.cpp b/clang/lib/AST/ExprConstant.cpp
index 3b5ab839c6cf7..5c6ca4c9ee4de 100644
--- a/clang/lib/AST/ExprConstant.cpp
+++ b/clang/lib/AST/ExprConstant.cpp
@@ -1961,7 +1961,7 @@ APValue &
 CallStackFrame::createConstexprUnknownAPValues(const VarDecl *Key,
                                                APValue::LValueBase Base) {
   APValue &Result = ConstexprUnknownAPValues[MapKeyTy(Key, Base.getVersion())];
-  Result = APValue(Base, CharUnits::One(), APValue::ConstexprUnknown{});
+  Result = APValue(Base, CharUnits::Zero(), APValue::ConstexprUnknown{});
 
   return Result;
 }
@@ -3600,8 +3600,12 @@ static bool evaluateVarDeclInit(EvalInfo &Info, const Expr *E,
        VD->mightBeUsableInConstantExpressions(Info.Ctx)) ||
       ((Info.getLangOpts().CPlusPlus || Info.getLangOpts().OpenCL) &&
        !Info.getLangOpts().CPlusPlus11 && !VD->hasICEInitializer(Info.Ctx))) {
-    Info.CCEDiag(E, diag::note_constexpr_var_init_non_constant, 1) << VD;
-    NoteLValueLocation(Info, Base);
+    if (Init) {
+      Info.CCEDiag(E, diag::note_constexpr_var_init_non_constant, 1) << VD;
+      NoteLValueLocation(Info, Base);
+    } else {
+      Info.CCEDiag(E);
+    }
   }
 
   // Never use the initializer of a weak variable, not even for constant
@@ -5221,7 +5225,7 @@ static bool EvaluateDecl(EvalInfo &Info, const Decl *D) {
     OK &= EvaluateVarDecl(Info, VD);
 
   if (const DecompositionDecl *DD = dyn_cast<DecompositionDecl>(D))
-    for (auto *BD : DD->bindings())
+    for (auto *BD : DD->flat_bindings())
       if (auto *VD = BD->getHoldingVar())
         OK &= EvaluateDecl(Info, VD);
 
@@ -12532,10 +12536,9 @@ static const Expr *ignorePointerCastsAndParens(const Expr *E) {
 static bool isDesignatorAtObjectEnd(const ASTContext &Ctx, const LValue &LVal) {
   assert(!LVal.Designator.Invalid);
 
-  auto IsLastOrInvalidFieldDecl = [&Ctx](const FieldDecl *FD, bool &Invalid) {
+  auto IsLastOrInvalidFieldDecl = [&Ctx](const FieldDecl *FD) {
     const RecordDecl *Parent = FD->getParent();
-    Invalid = Parent->isInvalidDecl();
-    if (Invalid || Parent->isUnion())
+    if (Parent->isInvalidDecl() || Parent->isUnion())
       return true;
     const ASTRecordLayout &Layout = Ctx.getASTRecordLayout(Parent);
     return FD->getFieldIndex() + 1 == Layout.getFieldCount();
@@ -12544,14 +12547,12 @@ static bool isDesignatorAtObjectEnd(const ASTContext &Ctx, const LValue &LVal) {
   auto &Base = LVal.getLValueBase();
   if (auto *ME = dyn_cast_or_null<MemberExpr>(Base.dyn_cast<const Expr *>())) {
     if (auto *FD = dyn_cast<FieldDecl>(ME->getMemberDecl())) {
-      bool Invalid;
-      if (!IsLastOrInvalidFieldDecl(FD, Invalid))
-        return Invalid;
+      if (!IsLastOrInvalidFieldDecl(FD))
+        return false;
     } else if (auto *IFD = dyn_cast<IndirectFieldDecl>(ME->getMemberDecl())) {
       for (auto *FD : IFD->chain()) {
-        bool Invalid;
-        if (!IsLastOrInvalidFieldDecl(cast<FieldDecl>(FD), Invalid))
-          return Invalid;
+        if (!IsLastOrInvalidFieldDecl(cast<FieldDecl>(FD)))
+          return false;
       }
     }
   }
@@ -12587,9 +12588,8 @@ static bool isDesignatorAtObjectEnd(const ASTContext &Ctx, const LValue &LVal) {
         return false;
       BaseType = CT->getElementType();
     } else if (auto *FD = getAsField(Entry)) {
-      bool Invalid;
-      if (!IsLastOrInvalidFieldDecl(FD, Invalid))
-        return Invalid;
+      if (!IsLastOrInvalidFieldDecl(FD))
+        return false;
       BaseType = FD->getType();
     } else {
       assert(getAsBaseClass(Entry) && "Expecting cast to a base class");
@@ -15043,6 +15043,7 @@ bool IntExprEvaluator::VisitCastExpr(const CastExpr *E) {
   case CK_NoOp:
   case CK_LValueToRValueBitCast:
   case CK_HLSLArrayRValue:
+  case CK_HLSLElementwiseCast:
     return ExprEvaluatorBaseTy::VisitCastExpr(E);
 
   case CK_MemberPointerToBoolean:
@@ -15901,6 +15902,7 @@ bool ComplexExprEvaluator::VisitCastExpr(const CastExpr *E) {
   case CK_IntegralToFixedPoint:
   case CK_MatrixCast:
   case CK_HLSLVectorTruncation:
+  case CK_HLSLElementwiseCast:
     llvm_unreachable("invalid cast kind for complex value");
 
   case CK_LValueToRValue:
@@ -17249,6 +17251,7 @@ static ICEDiag CheckICE(const Expr* E, const ASTContext &Ctx) {
   case Expr::SYCLUniqueStableNameExprClass:
   case Expr::CXXParenListInitExprClass:
   case Expr::HLSLOutArgExprClass:
+  case Expr::ResolvedUnexpandedPackExprClass:
     return ICEDiag(IK_NotICE, E->getBeginLoc());
 
   case Expr::InitListExprClass: {
diff --git a/clang/lib/AST/ItaniumMangle.cpp b/clang/lib/AST/ItaniumMangle.cpp
index 1dd936cf4fb51..e5eb22eae7dd1 100644
--- a/clang/lib/AST/ItaniumMangle.cpp
+++ b/clang/lib/AST/ItaniumMangle.cpp
@@ -3419,24 +3419,24 @@ void CXXNameMangler::mangleType(const BuiltinType *T) {
       /* Prior to Clang 18.0 we used this incorrect mangled name */            \
       mangleVendorType("__SVBFloat16_t");                                      \
     } else {                                                                   \
-      type_name = MangledName;                                                 \
-      Out << (type_name == Name ? "u" : "") << type_name.size() << type_name;  \
+      type_name = #MangledName;                                                \
+      Out << (type_name == #Name ? "u" : "") << type_name.size() << type_name; \
     }                                                                          \
     break;
 #define SVE_PREDICATE_TYPE(Name, MangledName, Id, SingletonId)                 \
   case BuiltinType::Id:                                                        \
-    type_name = MangledName;                                                   \
-    Out << (type_name == Name ? "u" : "") << type_name.size() << type_name;    \
+    type_name = #MangledName;                                                  \
+    Out << (type_name == #Name ? "u" : "") << type_name.size() << type_name;   \
     break;
 #define SVE_OPAQUE_TYPE(Name, MangledName, Id, SingletonId)                    \
   case BuiltinType::Id:                                                        \
-    type_name = MangledName;                                                   \
-    Out << (type_name == Name ? "u" : "") << type_name.size() << type_name;    \
+    type_name = #MangledName;                                                  \
+    Out << (type_name == #Name ? "u" : "") << type_name.size() << type_name;   \
     break;
-#define AARCH64_VECTOR_TYPE(Name, MangledName, Id, SingletonId)                \
+#define SVE_SCALAR_TYPE(Name, MangledName, Id, SingletonId, Bits)              \
   case BuiltinType::Id:                                                        \
-    type_name = MangledName;                                                   \
-    Out << (type_name == Name ? "u" : "") << type_name.size() << type_name;    \
+    type_name = #MangledName;                                                  \
+    Out << (type_name == #Name ? "u" : "") << type_name.size() << type_name;   \
     break;
 #include "clang/Basic/AArch64SVEACLETypes.def"
 #define PPC_VECTOR_TYPE(Name, Id, Size)                                        \
@@ -3919,6 +3919,9 @@ void CXXNameMangler::mangleNeonVectorType(const VectorType *T) {
     case BuiltinType::Float:     EltName = "float32_t"; break;
     case BuiltinType::Half:      EltName = "float16_t"; break;
     case BuiltinType::BFloat16:  EltName = "bfloat16_t"; break;
+    case BuiltinType::MFloat8:
+      EltName = "mfloat8_t";
+      break;
     default:
       llvm_unreachable("unexpected Neon vector element type");
     }
@@ -3972,6 +3975,8 @@ static StringRef mangleAArch64VectorBase(const BuiltinType *EltType) {
     return "Float64";
   case BuiltinType::BFloat16:
     return "Bfloat16";
+  case BuiltinType::MFloat8:
+    return "Mfloat8";
   default:
     llvm_unreachable("Unexpected vector element base type");
   }
@@ -4193,7 +4198,7 @@ void CXXNameMangler::mangleRISCVFixedRVVVectorType(const VectorType *T) {
 
   // Apend the LMUL suffix.
   auto VScale = getASTContext().getTargetInfo().getVScaleRange(
-      getASTContext().getLangOpts());
+      getASTContext().getLangOpts(), false);
   unsigned VLen = VScale->first * llvm::RISCV::RVVBitsPerBlock;
 
   if (T->getVectorKind() == VectorKind::RVVFixedLengthData) {
@@ -4928,7 +4933,7 @@ void CXXNameMangler::mangleExpression(const Expr *E, unsigned Arity,
   case Expr::SourceLocExprClass:
   case Expr::EmbedExprClass:
   case Expr::BuiltinBitCastExprClass:
-  {
+  case Expr::ResolvedUnexpandedPackExprClass: {
     NotPrimaryExpr();
     if (!NullOut) {
       // As bad as this diagnostic is, it's better than crashing.
diff --git a/clang/lib/AST/JSONNodeDumper.cpp b/clang/lib/AST/JSONNodeDumper.cpp
index 36ef1fc8c79db..169e3ee94c221 100644
--- a/clang/lib/AST/JSONNodeDumper.cpp
+++ b/clang/lib/AST/JSONNodeDumper.cpp
@@ -1003,6 +1003,11 @@ void JSONNodeDumper::VisitRecordDecl(const RecordDecl *RD) {
 void JSONNodeDumper::VisitCXXRecordDecl(const CXXRecordDecl *RD) {
   VisitRecordDecl(RD);
 
+  if (const auto *CTSD = dyn_cast<ClassTemplateSpecializationDecl>(RD)) {
+    if (CTSD->hasStrictPackMatch())
+      JOS.attribute("strict-pack-match", true);
+  }
+
   // All other information requires a complete definition.
   if (!RD->isCompleteDefinition())
     return;
diff --git a/clang/lib/AST/MicrosoftMangle.cpp b/clang/lib/AST/MicrosoftMangle.cpp
index edeeaeaa9ae17..fe34251688a98 100644
--- a/clang/lib/AST/MicrosoftMangle.cpp
+++ b/clang/lib/AST/MicrosoftMangle.cpp
@@ -2792,6 +2792,10 @@ void MicrosoftCXXNameMangler::mangleType(const BuiltinType *T, Qualifiers,
     mangleArtificialTagType(TagTypeKind::Struct, "__bf16", {"__clang"});
     break;
 
+  case BuiltinType::MFloat8:
+    mangleArtificialTagType(TagTypeKind::Struct, "__mfp8", {"__clang"});
+    break;
+
 #define WASM_REF_TYPE(InternalName, MangledName, Id, SingletonId, AS)          \
   case BuiltinType::Id:                                                        \
     mangleArtificialTagType(TagTypeKind::Struct, MangledName);                 \
@@ -2806,48 +2810,13 @@ void MicrosoftCXXNameMangler::mangleType(const BuiltinType *T, Qualifiers,
     break;
 #include "clang/Basic/HLSLIntangibleTypes.def"
 
-#define SVE_TYPE(Name, Id, SingletonId) \
-  case BuiltinType::Id:
-#include "clang/Basic/AArch64SVEACLETypes.def"
-#define PPC_VECTOR_TYPE(Name, Id, Size) \
-  case BuiltinType::Id:
-#include "clang/Basic/PPCTypes.def"
-#define RVV_TYPE(Name, Id, SingletonId) case BuiltinType::Id:
-#include "clang/Basic/RISCVVTypes.def"
-#define AMDGPU_TYPE(Name, Id, SingletonId, Width, Align) case BuiltinType::Id:
-#include "clang/Basic/AMDGPUTypes.def"
-  case BuiltinType::ShortAccum:
-  case BuiltinType::Accum:
-  case BuiltinType::LongAccum:
-  case BuiltinType::UShortAccum:
-  case BuiltinType::UAccum:
-  case BuiltinType::ULongAccum:
-  case BuiltinType::ShortFract:
-  case BuiltinType::Fract:
-  case BuiltinType::LongFract:
-  case BuiltinType::UShortFract:
-  case BuiltinType::UFract:
-  case BuiltinType::ULongFract:
-  case BuiltinType::SatShortAccum:
-  case BuiltinType::SatAccum:
-  case BuiltinType::SatLongAccum:
-  case BuiltinType::SatUShortAccum:
-  case BuiltinType::SatUAccum:
-  case BuiltinType::SatULongAccum:
-  case BuiltinType::SatShortFract:
-  case BuiltinType::SatFract:
-  case BuiltinType::SatLongFract:
-  case BuiltinType::SatUShortFract:
-  case BuiltinType::SatUFract:
-  case BuiltinType::SatULongFract:
-  case BuiltinType::Ibm128:
-  case BuiltinType::Float128: {
+    // Issue an error for any type not explicitly handled.
+  default:
     Error(Range.getBegin(), "built-in type: ",
           T->getName(Context.getASTContext().getPrintingPolicy()))
         << Range;
     break;
   }
-  }
 }
 
 // <type>          ::= <function-type>
diff --git a/clang/lib/AST/OpenMPClause.cpp b/clang/lib/AST/OpenMPClause.cpp
index 532933d6183ce..424cab3a7de35 100644
--- a/clang/lib/AST/OpenMPClause.cpp
+++ b/clang/lib/AST/OpenMPClause.cpp
@@ -2073,6 +2073,11 @@ void OMPClausePrinter::VisitOMPNoOpenMPRoutinesClause(
   OS << "no_openmp_routines";
 }
 
+void OMPClausePrinter::VisitOMPNoOpenMPConstructsClause(
+    OMPNoOpenMPConstructsClause *) {
+  OS << "no_openmp_constructs";
+}
+
 void OMPClausePrinter::VisitOMPNoParallelismClause(OMPNoParallelismClause *) {
   OS << "no_parallelism";
 }
@@ -2927,9 +2932,13 @@ llvm::raw_ostream &clang::operator<<(llvm::raw_ostream &OS,
 TargetOMPContext::TargetOMPContext(
     ASTContext &ASTCtx, std::function<void(StringRef)> &&DiagUnknownTrait,
     const FunctionDecl *CurrentFunctionDecl,
-    ArrayRef<llvm::omp::TraitProperty> ConstructTraits)
+    ArrayRef<llvm::omp::TraitProperty> ConstructTraits, int DeviceNum)
     : OMPContext(ASTCtx.getLangOpts().OpenMPIsTargetDevice,
-                 ASTCtx.getTargetInfo().getTriple()),
+                 ASTCtx.getTargetInfo().getTriple(),
+                 ASTCtx.getLangOpts().OMPTargetTriples.empty()
+                     ? llvm::Triple()
+                     : ASTCtx.getLangOpts().OMPTargetTriples[0],
+                 DeviceNum),
       FeatureValidityCheck([&](StringRef FeatureName) {
         return ASTCtx.getTargetInfo().isValidFeatureName(FeatureName);
       }),
diff --git a/clang/lib/AST/ParentMapContext.cpp b/clang/lib/AST/ParentMapContext.cpp
index af7d9fcdc638b..e9387ec79c373 100644
--- a/clang/lib/AST/ParentMapContext.cpp
+++ b/clang/lib/AST/ParentMapContext.cpp
@@ -103,9 +103,9 @@ class ParentMapContext::ParentMap {
 
   static DynTypedNode
   getSingleDynTypedNodeFromParentMap(ParentMapPointers::mapped_type U) {
-    if (const auto *D = U.dyn_cast<const Decl *>())
+    if (const auto *D = dyn_cast<const Decl *>(U))
       return DynTypedNode::create(*D);
-    if (const auto *S = U.dyn_cast<const Stmt *>())
+    if (const auto *S = dyn_cast<const Stmt *>(U))
       return DynTypedNode::create(*S);
     return *cast<DynTypedNode *>(U);
   }
@@ -117,7 +117,7 @@ class ParentMapContext::ParentMap {
     if (I == Map.end()) {
       return llvm::ArrayRef<DynTypedNode>();
     }
-    if (const auto *V = I->second.template dyn_cast<ParentVector *>()) {
+    if (const auto *V = dyn_cast<ParentVector *>(I->second)) {
       return V->view();
     }
     return getSingleDynTypedNodeFromParentMap(I->second);
@@ -268,9 +268,9 @@ class ParentMapContext::ParentMap {
       auto It = PointerParents.find(E);
       if (It == PointerParents.end())
         break;
-      const auto *S = It->second.dyn_cast<const Stmt *>();
+      const auto *S = dyn_cast<const Stmt *>(It->second);
       if (!S) {
-        if (auto *Vec = It->second.dyn_cast<ParentVector *>())
+        if (auto *Vec = dyn_cast<ParentVector *>(It->second))
           return Vec->view();
         return getSingleDynTypedNodeFromParentMap(It->second);
       }
@@ -395,7 +395,7 @@ class ParentMapContext::ParentMap::ASTVisitor
       if (!isa<ParentVector *>(NodeOrVector)) {
         auto *Vector = new ParentVector(
             1, getSingleDynTypedNodeFromParentMap(NodeOrVector));
-        delete NodeOrVector.template dyn_cast<DynTypedNode *>();
+        delete dyn_cast<DynTypedNode *>(NodeOrVector);
         NodeOrVector = Vector;
       }
 
diff --git a/clang/lib/AST/RawCommentList.cpp b/clang/lib/AST/RawCommentList.cpp
index fddedd3a31856..9658c6ab3d39d 100644
--- a/clang/lib/AST/RawCommentList.cpp
+++ b/clang/lib/AST/RawCommentList.cpp
@@ -287,13 +287,13 @@ void RawCommentList::addComment(const RawComment &RC,
 
   // If this is the first Doxygen comment, save it (because there isn't
   // anything to merge it with).
-  if (OrderedComments[CommentFile].empty()) {
-    OrderedComments[CommentFile][CommentOffset] =
-        new (Allocator) RawComment(RC);
+  auto &OC = OrderedComments[CommentFile];
+  if (OC.empty()) {
+    OC[CommentOffset] = new (Allocator) RawComment(RC);
     return;
   }
 
-  const RawComment &C1 = *OrderedComments[CommentFile].rbegin()->second;
+  const RawComment &C1 = *OC.rbegin()->second;
   const RawComment &C2 = RC;
 
   // Merge comments only if there is only whitespace between them.
diff --git a/clang/lib/AST/StmtOpenACC.cpp b/clang/lib/AST/StmtOpenACC.cpp
index 2b0ac716bab56..11eab0c27579d 100644
--- a/clang/lib/AST/StmtOpenACC.cpp
+++ b/clang/lib/AST/StmtOpenACC.cpp
@@ -305,3 +305,19 @@ OpenACCUpdateConstruct::Create(const ASTContext &C, SourceLocation Start,
       new (Mem) OpenACCUpdateConstruct(Start, DirectiveLoc, End, Clauses);
   return Inst;
 }
+
+OpenACCAtomicConstruct *
+OpenACCAtomicConstruct::CreateEmpty(const ASTContext &C) {
+  void *Mem = C.Allocate(sizeof(OpenACCAtomicConstruct));
+  auto *Inst = new (Mem) OpenACCAtomicConstruct(EmptyShell{});
+  return Inst;
+}
+
+OpenACCAtomicConstruct *OpenACCAtomicConstruct::Create(
+    const ASTContext &C, SourceLocation Start, SourceLocation DirectiveLoc,
+    OpenACCAtomicKind AtKind, SourceLocation End, Stmt *AssociatedStmt) {
+  void *Mem = C.Allocate(sizeof(OpenACCAtomicConstruct));
+  auto *Inst = new (Mem)
+      OpenACCAtomicConstruct(Start, DirectiveLoc, AtKind, End, AssociatedStmt);
+  return Inst;
+}
diff --git a/clang/lib/AST/StmtPrinter.cpp b/clang/lib/AST/StmtPrinter.cpp
index b5def6fbe525c..3ce932a9dd352 100644
--- a/clang/lib/AST/StmtPrinter.cpp
+++ b/clang/lib/AST/StmtPrinter.cpp
@@ -45,9 +45,11 @@
 #include "clang/Basic/TypeTraits.h"
 #include "clang/Lex/Lexer.h"
 #include "llvm/ADT/ArrayRef.h"
+#include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/StringExtras.h"
 #include "llvm/ADT/StringRef.h"
+#include "llvm/Support/Casting.h"
 #include "llvm/Support/Compiler.h"
 #include "llvm/Support/ErrorHandling.h"
 #include "llvm/Support/raw_ostream.h"
@@ -1240,6 +1242,16 @@ void StmtPrinter::VisitOpenACCWaitConstruct(OpenACCWaitConstruct *S) {
   OS << '\n';
 }
 
+void StmtPrinter::VisitOpenACCAtomicConstruct(OpenACCAtomicConstruct *S) {
+  Indent() << "#pragma acc atomic";
+
+  if (S->getAtomicKind() != OpenACCAtomicKind::None)
+    OS << " " << S->getAtomicKind();
+
+  OS << '\n';
+  PrintStmt(S->getAssociatedStmt());
+}
+
 //===----------------------------------------------------------------------===//
 //  Expr printing methods.
 //===----------------------------------------------------------------------===//
@@ -1257,11 +1269,12 @@ void StmtPrinter::VisitConstantExpr(ConstantExpr *Node) {
 }
 
 void StmtPrinter::VisitDeclRefExpr(DeclRefExpr *Node) {
-  if (const auto *OCED = dyn_cast<OMPCapturedExprDecl>(Node->getDecl())) {
+  ValueDecl *VD = Node->getDecl();
+  if (const auto *OCED = dyn_cast<OMPCapturedExprDecl>(VD)) {
     OCED->getInit()->IgnoreImpCasts()->printPretty(OS, nullptr, Policy);
     return;
   }
-  if (const auto *TPOD = dyn_cast<TemplateParamObjectDecl>(Node->getDecl())) {
+  if (const auto *TPOD = dyn_cast<TemplateParamObjectDecl>(VD)) {
     TPOD->printAsExpr(OS, Policy);
     return;
   }
@@ -1269,16 +1282,41 @@ void StmtPrinter::VisitDeclRefExpr(DeclRefExpr *Node) {
     Qualifier->print(OS, Policy);
   if (Node->hasTemplateKeyword())
     OS << "template ";
-  if (Policy.CleanUglifiedParameters &&
-      isa<ParmVarDecl, NonTypeTemplateParmDecl>(Node->getDecl()) &&
-      Node->getDecl()->getIdentifier())
-    OS << Node->getDecl()->getIdentifier()->deuglifiedName();
-  else
-    Node->getNameInfo().printName(OS, Policy);
+  DeclarationNameInfo NameInfo = Node->getNameInfo();
+  if (IdentifierInfo *ID = NameInfo.getName().getAsIdentifierInfo();
+      ID || NameInfo.getName().getNameKind() != DeclarationName::Identifier) {
+    if (Policy.CleanUglifiedParameters &&
+        isa<ParmVarDecl, NonTypeTemplateParmDecl>(VD) && ID)
+      OS << ID->deuglifiedName();
+    else
+      NameInfo.printName(OS, Policy);
+  } else {
+    switch (VD->getKind()) {
+    case Decl::NonTypeTemplateParm: {
+      auto *TD = cast<NonTypeTemplateParmDecl>(VD);
+      OS << "value-parameter-" << TD->getDepth() << '-' << TD->getIndex() << "";
+      break;
+    }
+    case Decl::ParmVar: {
+      auto *PD = cast<ParmVarDecl>(VD);
+      OS << "function-parameter-" << PD->getFunctionScopeDepth() << '-'
+         << PD->getFunctionScopeIndex();
+      break;
+    }
+    case Decl::Decomposition:
+      OS << "decomposition";
+      for (const auto &I : cast<DecompositionDecl>(VD)->bindings())
+        OS << '-' << I->getName();
+      break;
+    default:
+      OS << "unhandled-anonymous-" << VD->getDeclKindName();
+      break;
+    }
+  }
   if (Node->hasExplicitTemplateArgs()) {
     const TemplateParameterList *TPL = nullptr;
     if (!Node->hadMultipleCandidates())
-      if (auto *TD = dyn_cast<TemplateDecl>(Node->getDecl()))
+      if (auto *TD = dyn_cast<TemplateDecl>(VD))
         TPL = TD->getTemplateParameters();
     printTemplateArgumentList(OS, Node->template_arguments(), Policy, TPL);
   }
@@ -1939,7 +1977,6 @@ void StmtPrinter::VisitPseudoObjectExpr(PseudoObjectExpr *Node) {
 void StmtPrinter::VisitAtomicExpr(AtomicExpr *Node) {
   const char *Name = nullptr;
   switch (Node->getOp()) {
-#define BUILTIN(ID, TYPE, ATTRS)
 #define ATOMIC_BUILTIN(ID, TYPE, ATTRS) \
   case AtomicExpr::AO ## ID: \
     Name = #ID "("; \
@@ -2567,6 +2604,15 @@ void StmtPrinter::VisitPackIndexingExpr(PackIndexingExpr *E) {
   OS << "]";
 }
 
+void StmtPrinter::VisitResolvedUnexpandedPackExpr(
+    ResolvedUnexpandedPackExpr *E) {
+  OS << "<<resolved pack(";
+  llvm::interleave(
+      E->getExprs().begin(), E->getExprs().end(),
+      [this](auto *X) { PrintExpr(X); }, [this] { OS << ", "; });
+  OS << ")>>";
+}
+
 void StmtPrinter::VisitSubstNonTypeTemplateParmPackExpr(
                                        SubstNonTypeTemplateParmPackExpr *Node) {
   OS << *Node->getParameterPack();
diff --git a/clang/lib/AST/StmtProfile.cpp b/clang/lib/AST/StmtProfile.cpp
index 85b59f714ba84..8b4b8ba19f75b 100644
--- a/clang/lib/AST/StmtProfile.cpp
+++ b/clang/lib/AST/StmtProfile.cpp
@@ -606,6 +606,9 @@ void OMPClauseProfiler::VisitOMPNoOpenMPClause(const OMPNoOpenMPClause *) {}
 void OMPClauseProfiler::VisitOMPNoOpenMPRoutinesClause(
     const OMPNoOpenMPRoutinesClause *) {}
 
+void OMPClauseProfiler::VisitOMPNoOpenMPConstructsClause(
+    const OMPNoOpenMPConstructsClause *) {}
+
 void OMPClauseProfiler::VisitOMPNoParallelismClause(
     const OMPNoParallelismClause *) {}
 
@@ -2270,16 +2273,20 @@ void StmtProfiler::VisitPackExpansionExpr(const PackExpansionExpr *S) {
 
 void StmtProfiler::VisitSizeOfPackExpr(const SizeOfPackExpr *S) {
   VisitExpr(S);
-  VisitDecl(S->getPack());
   if (S->isPartiallySubstituted()) {
     auto Args = S->getPartialArguments();
     ID.AddInteger(Args.size());
     for (const auto &TA : Args)
       VisitTemplateArgument(TA);
   } else {
+    VisitDecl(S->getPack());
     ID.AddInteger(0);
   }
 }
+void StmtProfiler::VisitResolvedUnexpandedPackExpr(
+    const ResolvedUnexpandedPackExpr *S) {
+  VisitExpr(S);
+}
 
 void StmtProfiler::VisitPackIndexingExpr(const PackIndexingExpr *E) {
   VisitExpr(E);
@@ -2805,6 +2812,11 @@ void StmtProfiler::VisitOpenACCUpdateConstruct(
   P.VisitOpenACCClauseList(S->clauses());
 }
 
+void StmtProfiler::VisitOpenACCAtomicConstruct(
+    const OpenACCAtomicConstruct *S) {
+  VisitStmt(S);
+}
+
 void StmtProfiler::VisitHLSLOutArgExpr(const HLSLOutArgExpr *S) {
   VisitStmt(S);
 }
diff --git a/clang/lib/AST/TemplateBase.cpp b/clang/lib/AST/TemplateBase.cpp
index 3625b6e435a55..0eef8f305fcb3 100644
--- a/clang/lib/AST/TemplateBase.cpp
+++ b/clang/lib/AST/TemplateBase.cpp
@@ -515,19 +515,17 @@ void TemplateArgument::print(const PrintingPolicy &Policy, raw_ostream &Out,
   }
 
   case Declaration: {
-    NamedDecl *ND = getAsDecl();
+    ValueDecl *VD = getAsDecl();
     if (getParamTypeForDecl()->isRecordType()) {
-      if (auto *TPO = dyn_cast<TemplateParamObjectDecl>(ND)) {
+      if (auto *TPO = dyn_cast<TemplateParamObjectDecl>(VD)) {
         TPO->getType().getUnqualifiedType().print(Out, Policy);
         TPO->printAsInit(Out, Policy);
         break;
       }
     }
-    if (auto *VD = dyn_cast<ValueDecl>(ND)) {
-      if (needsAmpersandOnTemplateArg(getParamTypeForDecl(), VD->getType()))
-        Out << "&";
-    }
-    ND->printQualifiedName(Out);
+    if (needsAmpersandOnTemplateArg(getParamTypeForDecl(), VD->getType()))
+      Out << "&";
+    VD->printQualifiedName(Out);
     break;
   }
 
diff --git a/clang/lib/AST/TemplateName.cpp b/clang/lib/AST/TemplateName.cpp
index 7d6275caedc4f..9e0a7dc2b8cdc 100644
--- a/clang/lib/AST/TemplateName.cpp
+++ b/clang/lib/AST/TemplateName.cpp
@@ -144,7 +144,7 @@ TemplateName::TemplateName(DeducedTemplateStorage *Deduced)
 bool TemplateName::isNull() const { return Storage.isNull(); }
 
 TemplateName::NameKind TemplateName::getKind() const {
-  if (auto *ND = Storage.dyn_cast<Decl *>()) {
+  if (auto *ND = dyn_cast<Decl *>(Storage)) {
     if (isa<UsingShadowDecl>(ND))
       return UsingTemplate;
     assert(isa<TemplateDecl>(ND));
@@ -182,7 +182,8 @@ TemplateDecl *TemplateName::getAsTemplateDecl(bool IgnoreDeduced) const {
            "Unexpected canonical DeducedTemplateName; Did you mean to use "
            "getTemplateDeclAndDefaultArgs instead?");
 
-  return cast_if_present<TemplateDecl>(Name.Storage.dyn_cast<Decl *>());
+  return cast_if_present<TemplateDecl>(
+      dyn_cast_if_present<Decl *>(Name.Storage));
 }
 
 std::pair<TemplateDecl *, DefaultArguments>
@@ -208,7 +209,7 @@ TemplateName::getTemplateDeclAndDefaultArgs() const {
 }
 
 std::optional<TemplateName> TemplateName::desugar(bool IgnoreDeduced) const {
-  if (Decl *D = Storage.dyn_cast<Decl *>()) {
+  if (Decl *D = dyn_cast_if_present<Decl *>(Storage)) {
     if (auto *USD = dyn_cast<UsingShadowDecl>(D))
       return TemplateName(USD->getTargetDecl());
     return std::nullopt;
@@ -242,7 +243,7 @@ AssumedTemplateStorage *TemplateName::getAsAssumedTemplateName() const {
 SubstTemplateTemplateParmStorage *
 TemplateName::getAsSubstTemplateTemplateParm() const {
   if (UncommonTemplateNameStorage *uncommon =
-          Storage.dyn_cast<UncommonTemplateNameStorage *>())
+          dyn_cast_if_present<UncommonTemplateNameStorage *>(Storage))
     return uncommon->getAsSubstTemplateTemplateParm();
 
   return nullptr;
@@ -258,7 +259,7 @@ TemplateName::getAsSubstTemplateTemplateParmPack() const {
 }
 
 QualifiedTemplateName *TemplateName::getAsQualifiedTemplateName() const {
-  return Storage.dyn_cast<QualifiedTemplateName *>();
+  return dyn_cast_if_present<QualifiedTemplateName *>(Storage);
 }
 
 DependentTemplateName *TemplateName::getAsDependentTemplateName() const {
@@ -276,7 +277,7 @@ UsingShadowDecl *TemplateName::getAsUsingShadowDecl() const {
 
 DeducedTemplateStorage *TemplateName::getAsDeducedTemplateName() const {
   if (UncommonTemplateNameStorage *Uncommon =
-          Storage.dyn_cast<UncommonTemplateNameStorage *>())
+          dyn_cast_if_present<UncommonTemplateNameStorage *>(Storage))
     return Uncommon->getAsDeducedTemplateName();
 
   return nullptr;
diff --git a/clang/lib/AST/TextNodeDumper.cpp b/clang/lib/AST/TextNodeDumper.cpp
index 670641242cae2..6da1f776b4b63 100644
--- a/clang/lib/AST/TextNodeDumper.cpp
+++ b/clang/lib/AST/TextNodeDumper.cpp
@@ -710,10 +710,36 @@ void TextNodeDumper::Visit(const APValue &Value, QualType Ty) {
          << GetApproxValue(Value.getComplexFloatImag()) << 'i';
     }
     return;
-  case APValue::LValue:
+  case APValue::LValue: {
     (void)Context;
-    OS << "LValue <todo>";
+    OS << "LValue Base=";
+    APValue::LValueBase B = Value.getLValueBase();
+    if (B.isNull())
+      OS << "null";
+    else if (const auto *BE = B.dyn_cast<const Expr *>()) {
+      OS << BE->getStmtClassName() << ' ';
+      dumpPointer(BE);
+    } else {
+      const auto *VDB = B.get<const ValueDecl *>();
+      OS << VDB->getDeclKindName() << "Decl";
+      dumpPointer(VDB);
+    }
+    OS << ", Null=" << Value.isNullPointer()
+       << ", Offset=" << Value.getLValueOffset().getQuantity()
+       << ", HasPath=" << Value.hasLValuePath();
+    if (Value.hasLValuePath()) {
+      OS << ", PathLength=" << Value.getLValuePath().size();
+      OS << ", Path=(";
+      llvm::ListSeparator Sep;
+      for (const auto &PathEntry : Value.getLValuePath()) {
+        // We're printing all entries as array indices because don't have the
+        // type information here to do anything else.
+        OS << Sep << PathEntry.getAsArrayIndex();
+      }
+      OS << ")";
+    }
     return;
+  }
   case APValue::Array: {
     unsigned ArraySize = Value.getArraySize();
     unsigned NumInitializedElements = Value.getArrayInitializedElts();
@@ -875,7 +901,41 @@ void TextNodeDumper::dumpBareDeclRef(const Decl *D) {
 
   if (const NamedDecl *ND = dyn_cast<NamedDecl>(D)) {
     ColorScope Color(OS, ShowColors, DeclNameColor);
-    OS << " '" << ND->getDeclName() << '\'';
+    if (DeclarationName Name = ND->getDeclName())
+      OS << " '" << Name << '\'';
+    else
+      switch (ND->getKind()) {
+      case Decl::Decomposition: {
+        auto *DD = cast<DecompositionDecl>(ND);
+        OS << " first_binding '" << DD->bindings()[0]->getDeclName() << '\'';
+        break;
+      }
+      case Decl::Field: {
+        auto *FD = cast<FieldDecl>(ND);
+        OS << " field_index " << FD->getFieldIndex();
+        break;
+      }
+      case Decl::ParmVar: {
+        auto *PD = cast<ParmVarDecl>(ND);
+        OS << " depth " << PD->getFunctionScopeDepth() << " index "
+           << PD->getFunctionScopeIndex();
+        break;
+      }
+      case Decl::TemplateTypeParm: {
+        auto *TD = cast<TemplateTypeParmDecl>(ND);
+        OS << " depth " << TD->getDepth() << " index " << TD->getIndex();
+        break;
+      }
+      case Decl::NonTypeTemplateParm: {
+        auto *TD = cast<NonTypeTemplateParmDecl>(ND);
+        OS << " depth " << TD->getDepth() << " index " << TD->getIndex();
+        break;
+      }
+      default:
+        // Var, Namespace, (CXX)Record: Nothing else besides source location.
+        dumpSourceRange(ND->getSourceRange());
+        break;
+      }
   }
 
   if (const ValueDecl *VD = dyn_cast<ValueDecl>(D))
@@ -898,9 +958,9 @@ void TextNodeDumper::dumpAccessSpecifier(AccessSpecifier AS) {
 
 void TextNodeDumper::dumpCleanupObject(
     const ExprWithCleanups::CleanupObject &C) {
-  if (auto *BD = C.dyn_cast<BlockDecl *>())
+  if (auto *BD = dyn_cast<BlockDecl *>(C))
     dumpDeclRef(BD, "cleanup");
-  else if (auto *CLE = C.dyn_cast<CompoundLiteralExpr *>())
+  else if (auto *CLE = dyn_cast<CompoundLiteralExpr *>(C))
     AddChild([=] {
       OS << "cleanup ";
       {
@@ -2465,8 +2525,11 @@ void TextNodeDumper::VisitCXXRecordDecl(const CXXRecordDecl *D) {
     OS << " instantiated_from";
     dumpPointer(Instance);
   }
-  if (const auto *CTSD = dyn_cast<ClassTemplateSpecializationDecl>(D))
+  if (const auto *CTSD = dyn_cast<ClassTemplateSpecializationDecl>(D)) {
     dumpTemplateSpecializationKind(CTSD->getSpecializationKind());
+    if (CTSD->hasStrictPackMatch())
+      OS << " strict-pack-match";
+  }
 
   dumpNestedNameSpecifier(D->getQualifier());
 
@@ -2981,6 +3044,12 @@ void TextNodeDumper::VisitOpenACCUpdateConstruct(
   VisitOpenACCConstructStmt(S);
 }
 
+void TextNodeDumper::VisitOpenACCAtomicConstruct(
+    const OpenACCAtomicConstruct *S) {
+  VisitOpenACCConstructStmt(S);
+  OS << ' ' << S->getAtomicKind();
+}
+
 void TextNodeDumper::VisitEmbedExpr(const EmbedExpr *S) {
   AddChild("begin", [=] { OS << S->getStartingElementPos(); });
   AddChild("number of elements", [=] { OS << S->getDataElementCount(); });
diff --git a/clang/lib/AST/Type.cpp b/clang/lib/AST/Type.cpp
index caa0ac858a1be..8c11ec2e1fe24 100644
--- a/clang/lib/AST/Type.cpp
+++ b/clang/lib/AST/Type.cpp
@@ -2527,9 +2527,7 @@ bool Type::isSVESizelessBuiltinType() const {
 #define SVE_PREDICATE_TYPE(Name, MangledName, Id, SingletonId)                 \
   case BuiltinType::Id:                                                        \
     return true;
-#define AARCH64_VECTOR_TYPE(Name, MangledName, Id, SingletonId)                \
-  case BuiltinType::Id:                                                        \
-    return false;
+#define SVE_TYPE(Name, Id, SingletonId)
 #include "clang/Basic/AArch64SVEACLETypes.def"
     default:
       return false;
@@ -3482,9 +3480,9 @@ StringRef BuiltinType::getName(const PrintingPolicy &Policy) const {
   case Id: \
     return #ExtType;
 #include "clang/Basic/OpenCLExtensionTypes.def"
-#define SVE_TYPE(Name, Id, SingletonId) \
-  case Id: \
-    return Name;
+#define SVE_TYPE(Name, Id, SingletonId)                                        \
+  case Id:                                                                     \
+    return #Name;
 #include "clang/Basic/AArch64SVEACLETypes.def"
 #define PPC_VECTOR_TYPE(Name, Id, Size) \
   case Id: \
diff --git a/clang/lib/AST/TypePrinter.cpp b/clang/lib/AST/TypePrinter.cpp
index a850410ffc846..31695374cb52b 100644
--- a/clang/lib/AST/TypePrinter.cpp
+++ b/clang/lib/AST/TypePrinter.cpp
@@ -2552,10 +2552,12 @@ std::string Qualifiers::getAddrSpaceAsString(LangAS AS) {
     return "__uptr __ptr32";
   case LangAS::ptr64:
     return "__ptr64";
-  case LangAS::wasm_funcref:
-    return "__funcref";
   case LangAS::hlsl_groupshared:
     return "groupshared";
+  case LangAS::hlsl_constant:
+    return "hlsl_constant";
+  case LangAS::wasm_funcref:
+    return "__funcref";
   default:
     return std::to_string(toTargetAddressSpace(AS));
   }
diff --git a/clang/lib/AST/VTableBuilder.cpp b/clang/lib/AST/VTableBuilder.cpp
index fa3055dd1206f..19d76df99dbe3 100644
--- a/clang/lib/AST/VTableBuilder.cpp
+++ b/clang/lib/AST/VTableBuilder.cpp
@@ -1169,12 +1169,13 @@ void ItaniumVTableBuilder::ComputeThisAdjustments() {
       //
       // Do not set ThunkInfo::Method if Idx is already in VTableThunks. This
       // can happen when covariant return adjustment is required too.
-      if (!VTableThunks.count(Idx)) {
+      auto [It, Inserted] = VTableThunks.try_emplace(Idx);
+      if (Inserted) {
         const CXXMethodDecl *Method = VTables.findOriginalMethodInMap(MD);
-        VTableThunks[Idx].Method = Method;
-        VTableThunks[Idx].ThisType = Method->getThisType().getTypePtr();
+        It->second.Method = Method;
+        It->second.ThisType = Method->getThisType().getTypePtr();
       }
-      VTableThunks[Idx].This = ThisAdjustment;
+      It->second.This = ThisAdjustment;
     };
 
     SetThisAdjustmentThunk(VTableIndex);
@@ -1653,8 +1654,9 @@ void ItaniumVTableBuilder::AddMethods(
     // findOriginalMethod to find the method that created the entry if the
     // method in the entry requires adjustment.
     if (!ReturnAdjustment.isEmpty()) {
-      VTableThunks[Components.size()].Method = MD;
-      VTableThunks[Components.size()].ThisType = MD->getThisType().getTypePtr();
+      auto &VTT = VTableThunks[Components.size()];
+      VTT.Method = MD;
+      VTT.ThisType = MD->getThisType().getTypePtr();
     }
 
     AddMethod(Overrider.Method, ReturnAdjustment);
diff --git a/clang/lib/Analysis/CFG.cpp b/clang/lib/Analysis/CFG.cpp
index 304bbb2b422c6..3e144395cffc6 100644
--- a/clang/lib/Analysis/CFG.cpp
+++ b/clang/lib/Analysis/CFG.cpp
@@ -2041,6 +2041,8 @@ void CFGBuilder::addImplicitDtorsForDestructor(const CXXDestructorDecl *DD) {
   }
 
   // First destroy member objects.
+  if (RD->isUnion())
+    return;
   for (auto *FI : RD->fields()) {
     // Check for constant size array. Set type to array element type.
     QualType QT = FI->getType();
diff --git a/clang/lib/Analysis/ExprMutationAnalyzer.cpp b/clang/lib/Analysis/ExprMutationAnalyzer.cpp
index d7b44149d0fc4..8944343484e58 100644
--- a/clang/lib/Analysis/ExprMutationAnalyzer.cpp
+++ b/clang/lib/Analysis/ExprMutationAnalyzer.cpp
@@ -806,17 +806,15 @@ FunctionParmMutationAnalyzer::FunctionParmMutationAnalyzer(
 
 const Stmt *
 FunctionParmMutationAnalyzer::findMutation(const ParmVarDecl *Parm) {
-  const auto Memoized = Results.find(Parm);
-  if (Memoized != Results.end())
-    return Memoized->second;
+  auto [Place, Inserted] = Results.try_emplace(Parm);
+  if (!Inserted)
+    return Place->second;
+
   // To handle call A -> call B -> call A. Assume parameters of A is not mutated
   // before analyzing parameters of A. Then when analyzing the second "call A",
   // FunctionParmMutationAnalyzer can use this memoized value to avoid infinite
   // recursion.
-  Results[Parm] = nullptr;
-  if (const Stmt *S = BodyAnalyzer.findMutation(Parm))
-    return Results[Parm] = S;
-  return Results[Parm];
+  return Place->second = BodyAnalyzer.findMutation(Parm);
 }
 
 } // namespace clang
diff --git a/clang/lib/Analysis/FlowSensitive/DataflowAnalysisContext.cpp b/clang/lib/Analysis/FlowSensitive/DataflowAnalysisContext.cpp
index 4b86daa56d7b5..1c4fe5c6d5019 100644
--- a/clang/lib/Analysis/FlowSensitive/DataflowAnalysisContext.cpp
+++ b/clang/lib/Analysis/FlowSensitive/DataflowAnalysisContext.cpp
@@ -160,8 +160,9 @@ Atom
 DataflowAnalysisContext::joinFlowConditions(Atom FirstToken,
                                             Atom SecondToken) {
   Atom Token = arena().makeFlowConditionToken();
-  FlowConditionDeps[Token].insert(FirstToken);
-  FlowConditionDeps[Token].insert(SecondToken);
+  auto &TokenDeps = FlowConditionDeps[Token];
+  TokenDeps.insert(FirstToken);
+  TokenDeps.insert(SecondToken);
   addFlowConditionConstraint(Token,
                              arena().makeOr(arena().makeAtomRef(FirstToken),
                                             arena().makeAtomRef(SecondToken)));
diff --git a/clang/lib/Analysis/FlowSensitive/SmartPointerAccessorCaching.cpp b/clang/lib/Analysis/FlowSensitive/SmartPointerAccessorCaching.cpp
index c58bd309545db..4f5d18225c414 100644
--- a/clang/lib/Analysis/FlowSensitive/SmartPointerAccessorCaching.cpp
+++ b/clang/lib/Analysis/FlowSensitive/SmartPointerAccessorCaching.cpp
@@ -14,6 +14,7 @@ using ast_matchers::callee;
 using ast_matchers::cxxMemberCallExpr;
 using ast_matchers::cxxMethodDecl;
 using ast_matchers::cxxOperatorCallExpr;
+using ast_matchers::hasCanonicalType;
 using ast_matchers::hasName;
 using ast_matchers::hasOverloadedOperatorName;
 using ast_matchers::ofClass;
@@ -41,7 +42,8 @@ bool hasSmartPointerClassShape(const CXXRecordDecl &RD, bool &HasGet,
         HasStar = true;
         StarReturnType = MD->getReturnType()
                              .getNonReferenceType()
-                             ->getCanonicalTypeUnqualified();
+                             ->getCanonicalTypeUnqualified()
+                             .getUnqualifiedType();
       }
       break;
     case OO_Arrow:
@@ -49,7 +51,8 @@ bool hasSmartPointerClassShape(const CXXRecordDecl &RD, bool &HasGet,
         HasArrow = true;
         ArrowReturnType = MD->getReturnType()
                               ->getPointeeType()
-                              ->getCanonicalTypeUnqualified();
+                              ->getCanonicalTypeUnqualified()
+                              .getUnqualifiedType();
       }
       break;
     case OO_None: {
@@ -61,14 +64,16 @@ bool hasSmartPointerClassShape(const CXXRecordDecl &RD, bool &HasGet,
           HasGet = true;
           GetReturnType = MD->getReturnType()
                               ->getPointeeType()
-                              ->getCanonicalTypeUnqualified();
+                              ->getCanonicalTypeUnqualified()
+                              .getUnqualifiedType();
         }
       } else if (II->isStr("value")) {
         if (MD->getReturnType()->isReferenceType()) {
           HasValue = true;
           ValueReturnType = MD->getReturnType()
                                 .getNonReferenceType()
-                                ->getCanonicalTypeUnqualified();
+                                ->getCanonicalTypeUnqualified()
+                                .getUnqualifiedType();
         }
       }
     } break;
@@ -122,27 +127,29 @@ namespace clang::dataflow {
 ast_matchers::StatementMatcher isSmartPointerLikeOperatorStar() {
   return cxxOperatorCallExpr(
       hasOverloadedOperatorName("*"),
-      callee(cxxMethodDecl(parameterCountIs(0), returns(referenceType()),
+      callee(cxxMethodDecl(parameterCountIs(0),
+                           returns(hasCanonicalType(referenceType())),
                            ofClass(smartPointerClassWithGetOrValue()))));
 }
 
 ast_matchers::StatementMatcher isSmartPointerLikeOperatorArrow() {
   return cxxOperatorCallExpr(
       hasOverloadedOperatorName("->"),
-      callee(cxxMethodDecl(parameterCountIs(0), returns(pointerType()),
+      callee(cxxMethodDecl(parameterCountIs(0),
+                           returns(hasCanonicalType(pointerType())),
                            ofClass(smartPointerClassWithGetOrValue()))));
 }
 
 ast_matchers::StatementMatcher isSmartPointerLikeValueMethodCall() {
-  return cxxMemberCallExpr(callee(
-      cxxMethodDecl(parameterCountIs(0), returns(referenceType()),
-                    hasName("value"), ofClass(smartPointerClassWithValue()))));
+  return cxxMemberCallExpr(callee(cxxMethodDecl(
+      parameterCountIs(0), returns(hasCanonicalType(referenceType())),
+      hasName("value"), ofClass(smartPointerClassWithValue()))));
 }
 
 ast_matchers::StatementMatcher isSmartPointerLikeGetMethodCall() {
-  return cxxMemberCallExpr(callee(
-      cxxMethodDecl(parameterCountIs(0), returns(pointerType()), hasName("get"),
-                    ofClass(smartPointerClassWithGet()))));
+  return cxxMemberCallExpr(callee(cxxMethodDecl(
+      parameterCountIs(0), returns(hasCanonicalType(pointerType())),
+      hasName("get"), ofClass(smartPointerClassWithGet()))));
 }
 
 const FunctionDecl *
diff --git a/clang/lib/Analysis/ProgramPoint.cpp b/clang/lib/Analysis/ProgramPoint.cpp
index 7945c5c2fc27f..e508681410b0b 100644
--- a/clang/lib/Analysis/ProgramPoint.cpp
+++ b/clang/lib/Analysis/ProgramPoint.cpp
@@ -49,6 +49,121 @@ LLVM_DUMP_METHOD void ProgramPoint::dump() const {
   return printJson(llvm::errs());
 }
 
+StringRef ProgramPoint::getProgramPointKindName(Kind K) {
+  switch (K) {
+  case BlockEdgeKind:
+    return "BlockEdge";
+  case BlockEntranceKind:
+    return "BlockEntrance";
+  case BlockExitKind:
+    return "BlockExit";
+  case PreStmtKind:
+    return "PreStmt";
+  case PreStmtPurgeDeadSymbolsKind:
+    return "PreStmtPurgeDeadSymbols";
+  case PostStmtPurgeDeadSymbolsKind:
+    return "PostStmtPurgeDeadSymbols";
+  case PostStmtKind:
+    return "PostStmt";
+  case PreLoadKind:
+    return "PreLoad";
+  case PostLoadKind:
+    return "PostLoad";
+  case PreStoreKind:
+    return "PreStore";
+  case PostStoreKind:
+    return "PostStore";
+  case PostConditionKind:
+    return "PostCondition";
+  case PostLValueKind:
+    return "PostLValue";
+  case PostAllocatorCallKind:
+    return "PostAllocatorCall";
+  case PostInitializerKind:
+    return "PostInitializer";
+  case CallEnterKind:
+    return "CallEnter";
+  case CallExitBeginKind:
+    return "CallExitBegin";
+  case CallExitEndKind:
+    return "CallExitEnd";
+  case FunctionExitKind:
+    return "FunctionExit";
+  case PreImplicitCallKind:
+    return "PreImplicitCall";
+  case PostImplicitCallKind:
+    return "PostImplicitCall";
+  case LoopExitKind:
+    return "LoopExit";
+  case EpsilonKind:
+    return "Epsilon";
+  }
+  llvm_unreachable("Unknown ProgramPoint kind");
+}
+
+std::optional<SourceLocation> ProgramPoint::getSourceLocation() const {
+  switch (getKind()) {
+  case BlockEdgeKind:
+    // If needed, the source and or destination beginning can be used to get
+    // source location.
+    return std::nullopt;
+  case BlockEntranceKind:
+    // If needed, first statement of the block can be used.
+    return std::nullopt;
+  case BlockExitKind:
+    if (const auto *B = castAs<BlockExit>().getBlock()) {
+      if (const auto *T = B->getTerminatorStmt()) {
+        return T->getBeginLoc();
+      }
+    }
+    return std::nullopt;
+  case PreStmtKind:
+  case PreStmtPurgeDeadSymbolsKind:
+  case PostStmtPurgeDeadSymbolsKind:
+  case PostStmtKind:
+  case PreLoadKind:
+  case PostLoadKind:
+  case PreStoreKind:
+  case PostStoreKind:
+  case PostConditionKind:
+  case PostLValueKind:
+  case PostAllocatorCallKind:
+    if (const Stmt *S = castAs<StmtPoint>().getStmt())
+      return S->getBeginLoc();
+    return std::nullopt;
+  case PostInitializerKind:
+    if (const auto *Init = castAs<PostInitializer>().getInitializer())
+      return Init->getSourceLocation();
+    return std::nullopt;
+  case CallEnterKind:
+    if (const Stmt *S = castAs<CallEnter>().getCallExpr())
+      return S->getBeginLoc();
+    return std::nullopt;
+  case CallExitBeginKind:
+    if (const Stmt *S = castAs<CallExitBegin>().getReturnStmt())
+      return S->getBeginLoc();
+    return std::nullopt;
+  case CallExitEndKind:
+    return std::nullopt;
+  case FunctionExitKind:
+    if (const auto *B = castAs<FunctionExitPoint>().getBlock();
+        B && B->getTerminatorStmt())
+      return B->getTerminatorStmt()->getBeginLoc();
+    return std::nullopt;
+  case PreImplicitCallKind:
+    return castAs<ImplicitCallPoint>().getLocation();
+  case PostImplicitCallKind:
+    return castAs<ImplicitCallPoint>().getLocation();
+  case LoopExitKind:
+    if (const Stmt *S = castAs<LoopExit>().getLoopStmt())
+      return S->getBeginLoc();
+    return std::nullopt;
+  case EpsilonKind:
+    return std::nullopt;
+  }
+  llvm_unreachable("Unknown ProgramPoint kind");
+}
+
 void ProgramPoint::printJson(llvm::raw_ostream &Out, const char *NL) const {
   const ASTContext &Context =
       getLocationContext()->getAnalysisDeclContext()->getASTContext();
diff --git a/clang/lib/Analysis/UninitializedValues.cpp b/clang/lib/Analysis/UninitializedValues.cpp
index bf2f730618650..3a052eb27a444 100644
--- a/clang/lib/Analysis/UninitializedValues.cpp
+++ b/clang/lib/Analysis/UninitializedValues.cpp
@@ -379,8 +379,10 @@ void ClassifyRefs::classify(const Expr *E, Class C) {
   }
 
   FindVarResult Var = findVar(E, DC);
-  if (const DeclRefExpr *DRE = Var.getDeclRefExpr())
-    Classification[DRE] = std::max(Classification[DRE], C);
+  if (const DeclRefExpr *DRE = Var.getDeclRefExpr()) {
+    auto &Class = Classification[DRE];
+    Class = std::max(Class, C);
+  }
 }
 
 void ClassifyRefs::VisitDeclStmt(DeclStmt *DS) {
diff --git a/clang/lib/Analysis/UnsafeBufferUsage.cpp b/clang/lib/Analysis/UnsafeBufferUsage.cpp
index c064aa30e8aed..c51398698922b 100644
--- a/clang/lib/Analysis/UnsafeBufferUsage.cpp
+++ b/clang/lib/Analysis/UnsafeBufferUsage.cpp
@@ -930,7 +930,7 @@ AST_MATCHER(CallExpr, hasUnsafeSnprintfBuffer) {
       // The array element type must be compatible with `char` otherwise an
       // explicit cast will be needed, which will make this check unreachable.
       // Therefore, the array extent is same as its' bytewise size.
-      if (Size->EvaluateAsConstantExpr(ER, Ctx)) {
+      if (Size->EvaluateAsInt(ER, Ctx)) {
         APSInt EVal = ER.Val.getInt(); // Size must have integer type
 
         return APSInt::compareValues(EVal, APSInt(CAT->getSize(), true)) != 0;
diff --git a/clang/lib/Basic/Builtins.cpp b/clang/lib/Basic/Builtins.cpp
index 588183788de32..e7829a461bbc5 100644
--- a/clang/lib/Basic/Builtins.cpp
+++ b/clang/lib/Basic/Builtins.cpp
@@ -29,54 +29,120 @@ const char *HeaderDesc::getName() const {
   llvm_unreachable("Unknown HeaderDesc::HeaderID enum");
 }
 
-static constexpr Builtin::Info BuiltinInfo[] = {
-    {"not a builtin function", nullptr, nullptr, nullptr, HeaderDesc::NO_HEADER,
-     ALL_LANGUAGES},
-#define BUILTIN(ID, TYPE, ATTRS)                                               \
-  {#ID, TYPE, ATTRS, nullptr, HeaderDesc::NO_HEADER, ALL_LANGUAGES},
-#define LANGBUILTIN(ID, TYPE, ATTRS, LANGS)                                    \
-  {#ID, TYPE, ATTRS, nullptr, HeaderDesc::NO_HEADER, LANGS},
-#define LIBBUILTIN(ID, TYPE, ATTRS, HEADER, LANGS)                             \
-  {#ID, TYPE, ATTRS, nullptr, HeaderDesc::HEADER, LANGS},
+static constexpr unsigned NumBuiltins = Builtin::FirstTSBuiltin;
+
+#define GET_BUILTIN_STR_TABLE
+#include "clang/Basic/Builtins.inc"
+#undef GET_BUILTIN_STR_TABLE
+
+static constexpr Builtin::Info BuiltinInfos[] = {
+    Builtin::Info{}, // No-builtin info entry.
+#define GET_BUILTIN_INFOS
 #include "clang/Basic/Builtins.inc"
+#undef GET_BUILTIN_INFOS
 };
+static_assert(std::size(BuiltinInfos) == NumBuiltins);
 
-const Builtin::Info &Builtin::Context::getRecord(unsigned ID) const {
-  if (ID < Builtin::FirstTSBuiltin)
-    return BuiltinInfo[ID];
-  assert(((ID - Builtin::FirstTSBuiltin) <
-          (TSRecords.size() + AuxTSRecords.size())) &&
+std::pair<const Builtin::InfosShard &, const Builtin::Info &>
+Builtin::Context::getShardAndInfo(unsigned ID) const {
+  assert((ID < (Builtin::FirstTSBuiltin + NumTargetBuiltins +
+                NumAuxTargetBuiltins)) &&
          "Invalid builtin ID!");
-  if (isAuxBuiltinID(ID))
-    return AuxTSRecords[getAuxBuiltinID(ID) - Builtin::FirstTSBuiltin];
-  return TSRecords[ID - Builtin::FirstTSBuiltin];
+
+  ArrayRef<InfosShard> Shards = BuiltinShards;
+  if (isAuxBuiltinID(ID)) {
+    Shards = AuxTargetShards;
+    ID = getAuxBuiltinID(ID) - Builtin::FirstTSBuiltin;
+  } else if (ID >= Builtin::FirstTSBuiltin) {
+    Shards = TargetShards;
+    ID -= Builtin::FirstTSBuiltin;
+  }
+
+  // Loop over the shards to find the one matching this ID. We don't expect to
+  // have many shards and so its better to search linearly than with a binary
+  // search.
+  for (const auto &Shard : Shards) {
+    if (ID < Shard.Infos.size()) {
+      return {Shard, Shard.Infos[ID]};
+    }
+
+    ID -= Shard.Infos.size();
+  }
+  llvm_unreachable("Invalid target builtin shard structure!");
+}
+
+std::string Builtin::Info::getName(const Builtin::InfosShard &Shard) const {
+  return (Twine(Shard.NamePrefix) + (*Shard.Strings)[Offsets.Name]).str();
+}
+
+/// Return the identifier name for the specified builtin,
+/// e.g. "__builtin_abs".
+std::string Builtin::Context::getName(unsigned ID) const {
+  const auto &[Shard, I] = getShardAndInfo(ID);
+  return I.getName(Shard);
+}
+
+std::string Builtin::Context::getQuotedName(unsigned ID) const {
+  const auto &[Shard, I] = getShardAndInfo(ID);
+  return (Twine("'") + Shard.NamePrefix + (*Shard.Strings)[I.Offsets.Name] +
+          "'")
+      .str();
+}
+
+const char *Builtin::Context::getTypeString(unsigned ID) const {
+  const auto &[Shard, I] = getShardAndInfo(ID);
+  return (*Shard.Strings)[I.Offsets.Type].data();
 }
 
+const char *Builtin::Context::getAttributesString(unsigned ID) const {
+  const auto &[Shard, I] = getShardAndInfo(ID);
+  return (*Shard.Strings)[I.Offsets.Attributes].data();
+}
+
+const char *Builtin::Context::getRequiredFeatures(unsigned ID) const {
+  const auto &[Shard, I] = getShardAndInfo(ID);
+  return (*Shard.Strings)[I.Offsets.Features].data();
+}
+
+Builtin::Context::Context() : BuiltinShards{{&BuiltinStrings, BuiltinInfos}} {}
+
 void Builtin::Context::InitializeTarget(const TargetInfo &Target,
                                         const TargetInfo *AuxTarget) {
-  assert(TSRecords.empty() && "Already initialized target?");
-  TSRecords = Target.getTargetBuiltins();
-  if (AuxTarget)
-    AuxTSRecords = AuxTarget->getTargetBuiltins();
+  assert(TargetShards.empty() && "Already initialized target?");
+  assert(NumTargetBuiltins == 0 && "Already initialized target?");
+  TargetShards = Target.getTargetBuiltins();
+  for (const auto &Shard : TargetShards)
+    NumTargetBuiltins += Shard.Infos.size();
+  if (AuxTarget) {
+    AuxTargetShards = AuxTarget->getTargetBuiltins();
+    for (const auto &Shard : AuxTargetShards)
+      NumAuxTargetBuiltins += Shard.Infos.size();
+  }
 }
 
 bool Builtin::Context::isBuiltinFunc(llvm::StringRef FuncName) {
   bool InStdNamespace = FuncName.consume_front("std-");
-  for (unsigned i = Builtin::NotBuiltin + 1; i != Builtin::FirstTSBuiltin;
-       ++i) {
-    if (FuncName == BuiltinInfo[i].Name &&
-        (bool)strchr(BuiltinInfo[i].Attributes, 'z') == InStdNamespace)
-      return strchr(BuiltinInfo[i].Attributes, 'f') != nullptr;
-  }
+  for (const auto &Shard : {InfosShard{&BuiltinStrings, BuiltinInfos}})
+    if (llvm::StringRef FuncNameSuffix = FuncName;
+        FuncNameSuffix.consume_front(Shard.NamePrefix))
+      for (const auto &I : Shard.Infos)
+        if (FuncNameSuffix == (*Shard.Strings)[I.Offsets.Name] &&
+            (bool)strchr((*Shard.Strings)[I.Offsets.Attributes].data(), 'z') ==
+                InStdNamespace)
+          return strchr((*Shard.Strings)[I.Offsets.Attributes].data(), 'f') !=
+                 nullptr;
 
   return false;
 }
 
 /// Is this builtin supported according to the given language options?
-static bool builtinIsSupported(const Builtin::Info &BuiltinInfo,
+static bool builtinIsSupported(const llvm::StringTable &Strings,
+                               const Builtin::Info &BuiltinInfo,
                                const LangOptions &LangOpts) {
+  auto AttributesStr = Strings[BuiltinInfo.Offsets.Attributes];
+
   /* Builtins Unsupported */
-  if (LangOpts.NoBuiltin && strchr(BuiltinInfo.Attributes, 'f') != nullptr)
+  if (LangOpts.NoBuiltin && strchr(AttributesStr.data(), 'f') != nullptr)
     return false;
   /* CorBuiltins Unsupported */
   if (!LangOpts.Coroutines && (BuiltinInfo.Langs & COR_LANG))
@@ -123,7 +189,7 @@ static bool builtinIsSupported(const Builtin::Info &BuiltinInfo,
   if (!LangOpts.CPlusPlus && BuiltinInfo.Langs == CXX_LANG)
     return false;
   /* consteval Unsupported */
-  if (!LangOpts.CPlusPlus20 && strchr(BuiltinInfo.Attributes, 'G') != nullptr)
+  if (!LangOpts.CPlusPlus20 && strchr(AttributesStr.data(), 'G') != nullptr)
     return false;
   return true;
 }
@@ -132,22 +198,34 @@ static bool builtinIsSupported(const Builtin::Info &BuiltinInfo,
 /// appropriate builtin ID # and mark any non-portable builtin identifiers as
 /// such.
 void Builtin::Context::initializeBuiltins(IdentifierTable &Table,
-                                          const LangOptions& LangOpts) {
-  // Step #1: mark all target-independent builtins with their ID's.
-  for (unsigned i = Builtin::NotBuiltin + 1; i != Builtin::FirstTSBuiltin; ++i)
-    if (builtinIsSupported(BuiltinInfo[i], LangOpts)) {
-      Table.get(BuiltinInfo[i].Name).setBuiltinID(i);
-    }
-
-  // Step #2: Register target-specific builtins.
-  for (unsigned i = 0, e = TSRecords.size(); i != e; ++i)
-    if (builtinIsSupported(TSRecords[i], LangOpts))
-      Table.get(TSRecords[i].Name).setBuiltinID(i + Builtin::FirstTSBuiltin);
+                                          const LangOptions &LangOpts) {
+  {
+    unsigned ID = 0;
+    // Step #1: mark all target-independent builtins with their ID's.
+    for (const auto &Shard : BuiltinShards)
+      for (const auto &I : Shard.Infos) {
+        // If this is a real builtin (ID != 0) and is supported, add it.
+        if (ID != 0 && builtinIsSupported(*Shard.Strings, I, LangOpts))
+          Table.get(I.getName(Shard)).setBuiltinID(ID);
+        ++ID;
+      }
+    assert(ID == FirstTSBuiltin && "Should have added all non-target IDs!");
+
+    // Step #2: Register target-specific builtins.
+    for (const auto &Shard : TargetShards)
+      for (const auto &I : Shard.Infos) {
+        if (builtinIsSupported(*Shard.Strings, I, LangOpts))
+          Table.get(I.getName(Shard)).setBuiltinID(ID);
+        ++ID;
+      }
 
-  // Step #3: Register target-specific builtins for AuxTarget.
-  for (unsigned i = 0, e = AuxTSRecords.size(); i != e; ++i)
-    Table.get(AuxTSRecords[i].Name)
-        .setBuiltinID(i + Builtin::FirstTSBuiltin + TSRecords.size());
+    // Step #3: Register target-specific builtins for AuxTarget.
+    for (const auto &Shard : AuxTargetShards)
+      for (const auto &I : Shard.Infos) {
+        Table.get(I.getName(Shard)).setBuiltinID(ID);
+        ++ID;
+      }
+  }
 
   // Step #4: Unregister any builtins specified by -fno-builtin-foo.
   for (llvm::StringRef Name : LangOpts.NoBuiltinFuncs) {
@@ -163,12 +241,8 @@ void Builtin::Context::initializeBuiltins(IdentifierTable &Table,
   }
 }
 
-std::string Builtin::Context::getQuotedName(unsigned ID) const {
-  return (llvm::Twine("'") + getName(ID) + "'").str();
-}
-
 unsigned Builtin::Context::getRequiredVectorWidth(unsigned ID) const {
-  const char *WidthPos = ::strchr(getRecord(ID).Attributes, 'V');
+  const char *WidthPos = ::strchr(getAttributesString(ID), 'V');
   if (!WidthPos)
     return 0;
 
@@ -191,7 +265,7 @@ bool Builtin::Context::isLike(unsigned ID, unsigned &FormatIdx,
   assert(::toupper(Fmt[0]) == Fmt[1] &&
          "Format string is not in the form \"xX\"");
 
-  const char *Like = ::strpbrk(getRecord(ID).Attributes, Fmt);
+  const char *Like = ::strpbrk(getAttributesString(ID), Fmt);
   if (!Like)
     return false;
 
@@ -218,7 +292,7 @@ bool Builtin::Context::isScanfLike(unsigned ID, unsigned &FormatIdx,
 
 bool Builtin::Context::performsCallback(unsigned ID,
                                         SmallVectorImpl<int> &Encoding) const {
-  const char *CalleePos = ::strchr(getRecord(ID).Attributes, 'C');
+  const char *CalleePos = ::strchr(getAttributesString(ID), 'C');
   if (!CalleePos)
     return false;
 
diff --git a/clang/lib/Basic/Diagnostic.cpp b/clang/lib/Basic/Diagnostic.cpp
index ae71758bc81e0..9e2f134135647 100644
--- a/clang/lib/Basic/Diagnostic.cpp
+++ b/clang/lib/Basic/Diagnostic.cpp
@@ -145,7 +145,7 @@ void DiagnosticsEngine::Reset(bool soft /*=false*/) {
 
     // Create a DiagState and DiagStatePoint representing diagnostic changes
     // through command-line.
-    DiagStates.emplace_back();
+    DiagStates.emplace_back(*Diags);
     DiagStatesByLoc.appendFirst(&DiagStates.back());
   }
 }
@@ -156,8 +156,11 @@ DiagnosticsEngine::DiagState::getOrAddMapping(diag::kind Diag) {
       DiagMap.insert(std::make_pair(Diag, DiagnosticMapping()));
 
   // Initialize the entry if we added it.
-  if (Result.second)
-    Result.first->second = DiagnosticIDs::getDefaultMapping(Diag);
+  if (Result.second) {
+    Result.first->second = DiagIDs.getDefaultMapping(Diag);
+    if (DiagnosticIDs::IsCustomDiag(Diag))
+      DiagIDs.initCustomDiagMapping(Result.first->second, Diag);
+  }
 
   return Result.first->second;
 }
@@ -299,7 +302,8 @@ void DiagnosticsEngine::DiagStateMap::dump(SourceManager &SrcMgr,
 
       for (auto &Mapping : *Transition.State) {
         StringRef Option =
-            DiagnosticIDs::getWarningOptionForDiag(Mapping.first);
+            SrcMgr.getDiagnostics().Diags->getWarningOptionForDiag(
+                Mapping.first);
         if (!DiagName.empty() && DiagName != Option)
           continue;
 
@@ -343,9 +347,7 @@ void DiagnosticsEngine::PushDiagStatePoint(DiagState *State,
 
 void DiagnosticsEngine::setSeverity(diag::kind Diag, diag::Severity Map,
                                     SourceLocation L) {
-  assert(Diag < diag::DIAG_UPPER_LIMIT &&
-         "Can only map builtin diagnostics");
-  assert((Diags->isBuiltinWarningOrExtension(Diag) ||
+  assert((Diags->isWarningOrExtension(Diag) ||
           (Map == diag::Severity::Fatal || Map == diag::Severity::Error)) &&
          "Cannot map errors into warnings!");
   assert((L.isInvalid() || SourceMgr) && "No SourceMgr for valid location");
@@ -397,6 +399,8 @@ bool DiagnosticsEngine::setSeverityForGroup(diag::Flavor Flavor,
   if (Diags->getDiagnosticsInGroup(Flavor, Group, GroupDiags))
     return true;
 
+  Diags->setGroupSeverity(Group, Map);
+
   // Set the mapping.
   for (diag::kind Diag : GroupDiags)
     setSeverity(Diag, Map, Loc);
@@ -419,6 +423,7 @@ bool DiagnosticsEngine::setDiagnosticGroupWarningAsError(StringRef Group,
   if (Enabled)
     return setSeverityForGroup(diag::Flavor::WarningOrError, Group,
                                diag::Severity::Error);
+  Diags->setGroupSeverity(Group, diag::Severity::Warning);
 
   // Otherwise, we want to set the diagnostic mapping's "no Werror" bit, and
   // potentially downgrade anything already mapped to be a warning.
@@ -450,6 +455,7 @@ bool DiagnosticsEngine::setDiagnosticGroupErrorAsFatal(StringRef Group,
   if (Enabled)
     return setSeverityForGroup(diag::Flavor::WarningOrError, Group,
                                diag::Severity::Fatal);
+  Diags->setGroupSeverity(Group, diag::Severity::Error);
 
   // Otherwise, we want to set the diagnostic mapping's "no Wfatal-errors" bit,
   // and potentially downgrade anything already mapped to be a fatal error.
@@ -482,7 +488,7 @@ void DiagnosticsEngine::setSeverityForAll(diag::Flavor Flavor,
 
   // Set the mapping.
   for (diag::kind Diag : AllDiags)
-    if (Diags->isBuiltinWarningOrExtension(Diag))
+    if (Diags->isWarningOrExtension(Diag))
       setSeverity(Diag, Map, Loc);
 }
 
diff --git a/clang/lib/Basic/DiagnosticIDs.cpp b/clang/lib/Basic/DiagnosticIDs.cpp
index de1de6f61f3a1..ca5b8d2da769e 100644
--- a/clang/lib/Basic/DiagnosticIDs.cpp
+++ b/clang/lib/Basic/DiagnosticIDs.cpp
@@ -62,13 +62,12 @@ const uint32_t StaticDiagInfoDescriptionOffsets[] = {
 #undef DIAG
 };
 
-// Diagnostic classes.
 enum DiagnosticClass {
-  CLASS_NOTE       = 0x01,
-  CLASS_REMARK     = 0x02,
-  CLASS_WARNING    = 0x03,
-  CLASS_EXTENSION  = 0x04,
-  CLASS_ERROR      = 0x05
+  CLASS_NOTE = DiagnosticIDs::CLASS_NOTE,
+  CLASS_REMARK = DiagnosticIDs::CLASS_REMARK,
+  CLASS_WARNING = DiagnosticIDs::CLASS_WARNING,
+  CLASS_EXTENSION = DiagnosticIDs::CLASS_EXTENSION,
+  CLASS_ERROR = DiagnosticIDs::CLASS_ERROR,
 };
 
 struct StaticDiagInfoRec {
@@ -229,11 +228,60 @@ CATEGORY(INSTALLAPI, REFACTORING)
   return Found;
 }
 
-DiagnosticMapping DiagnosticIDs::getDefaultMapping(unsigned DiagID) {
+//===----------------------------------------------------------------------===//
+// Custom Diagnostic information
+//===----------------------------------------------------------------------===//
+
+namespace clang {
+namespace diag {
+using CustomDiagDesc = DiagnosticIDs::CustomDiagDesc;
+class CustomDiagInfo {
+  std::vector<CustomDiagDesc> DiagInfo;
+  std::map<CustomDiagDesc, unsigned> DiagIDs;
+  std::map<diag::Group, std::vector<unsigned>> GroupToDiags;
+
+public:
+  /// getDescription - Return the description of the specified custom
+  /// diagnostic.
+  const CustomDiagDesc &getDescription(unsigned DiagID) const {
+    assert(DiagID - DIAG_UPPER_LIMIT < DiagInfo.size() &&
+           "Invalid diagnostic ID");
+    return DiagInfo[DiagID - DIAG_UPPER_LIMIT];
+  }
+
+  unsigned getOrCreateDiagID(DiagnosticIDs::CustomDiagDesc D) {
+    // Check to see if it already exists.
+    std::map<CustomDiagDesc, unsigned>::iterator I = DiagIDs.lower_bound(D);
+    if (I != DiagIDs.end() && I->first == D)
+      return I->second;
+
+    // If not, assign a new ID.
+    unsigned ID = DiagInfo.size() + DIAG_UPPER_LIMIT;
+    DiagIDs.insert(std::make_pair(D, ID));
+    DiagInfo.push_back(D);
+    if (auto Group = D.GetGroup())
+      GroupToDiags[*Group].emplace_back(ID);
+    return ID;
+  }
+
+  ArrayRef<unsigned> getDiagsInGroup(diag::Group G) const {
+    if (auto Diags = GroupToDiags.find(G); Diags != GroupToDiags.end())
+      return Diags->second;
+    return {};
+  }
+};
+
+} // namespace diag
+} // namespace clang
+
+DiagnosticMapping DiagnosticIDs::getDefaultMapping(unsigned DiagID) const {
   DiagnosticMapping Info = DiagnosticMapping::Make(
       diag::Severity::Fatal, /*IsUser=*/false, /*IsPragma=*/false);
 
-  if (const StaticDiagInfoRec *StaticInfo = GetDiagInfo(DiagID)) {
+  if (IsCustomDiag(DiagID)) {
+    Info.setSeverity(
+        CustomDiagInfo->getDescription(DiagID).GetDefaultSeverity());
+  } else if (const StaticDiagInfoRec *StaticInfo = GetDiagInfo(DiagID)) {
     Info.setSeverity((diag::Severity)StaticInfo->DefaultSeverity);
 
     if (StaticInfo->WarnNoWerror) {
@@ -246,6 +294,22 @@ DiagnosticMapping DiagnosticIDs::getDefaultMapping(unsigned DiagID) {
   return Info;
 }
 
+void DiagnosticIDs::initCustomDiagMapping(DiagnosticMapping &Mapping,
+                                          unsigned DiagID) {
+  assert(IsCustomDiag(DiagID));
+  const auto &Diag = CustomDiagInfo->getDescription(DiagID);
+  if (auto Group = Diag.GetGroup()) {
+    GroupInfo GroupInfo = GroupInfos[static_cast<size_t>(*Group)];
+    if (static_cast<diag::Severity>(GroupInfo.Severity) != diag::Severity())
+      Mapping.setSeverity(static_cast<diag::Severity>(GroupInfo.Severity));
+    Mapping.setNoWarningAsError(GroupInfo.HasNoWarningAsError);
+  } else {
+    Mapping.setSeverity(Diag.GetDefaultSeverity());
+    Mapping.setNoWarningAsError(true);
+    Mapping.setNoErrorAsFatal(true);
+  }
+}
+
 /// getCategoryNumberForDiag - Return the category number that a specified
 /// DiagID belongs to, or 0 if no category.
 unsigned DiagnosticIDs::getCategoryNumberForDiag(unsigned DiagID) {
@@ -303,61 +367,6 @@ bool DiagnosticIDs::isDeferrable(unsigned DiagID) {
   return false;
 }
 
-/// getBuiltinDiagClass - Return the class field of the diagnostic.
-///
-static unsigned getBuiltinDiagClass(unsigned DiagID) {
-  if (const StaticDiagInfoRec *Info = GetDiagInfo(DiagID))
-    return Info->Class;
-  return ~0U;
-}
-
-//===----------------------------------------------------------------------===//
-// Custom Diagnostic information
-//===----------------------------------------------------------------------===//
-
-namespace clang {
-  namespace diag {
-    class CustomDiagInfo {
-      typedef std::pair<DiagnosticIDs::Level, std::string> DiagDesc;
-      std::vector<DiagDesc> DiagInfo;
-      std::map<DiagDesc, unsigned> DiagIDs;
-    public:
-
-      /// getDescription - Return the description of the specified custom
-      /// diagnostic.
-      StringRef getDescription(unsigned DiagID) const {
-        assert(DiagID - DIAG_UPPER_LIMIT < DiagInfo.size() &&
-               "Invalid diagnostic ID");
-        return DiagInfo[DiagID-DIAG_UPPER_LIMIT].second;
-      }
-
-      /// getLevel - Return the level of the specified custom diagnostic.
-      DiagnosticIDs::Level getLevel(unsigned DiagID) const {
-        assert(DiagID - DIAG_UPPER_LIMIT < DiagInfo.size() &&
-               "Invalid diagnostic ID");
-        return DiagInfo[DiagID-DIAG_UPPER_LIMIT].first;
-      }
-
-      unsigned getOrCreateDiagID(DiagnosticIDs::Level L, StringRef Message,
-                                 DiagnosticIDs &Diags) {
-        DiagDesc D(L, std::string(Message));
-        // Check to see if it already exists.
-        std::map<DiagDesc, unsigned>::iterator I = DiagIDs.lower_bound(D);
-        if (I != DiagIDs.end() && I->first == D)
-          return I->second;
-
-        // If not, assign a new ID.
-        unsigned ID = DiagInfo.size()+DIAG_UPPER_LIMIT;
-        DiagIDs.insert(std::make_pair(D, ID));
-        DiagInfo.push_back(D);
-        return ID;
-      }
-    };
-
-  } // end diag namespace
-} // end clang namespace
-
-
 //===----------------------------------------------------------------------===//
 // Common Diagnostic implementation
 //===----------------------------------------------------------------------===//
@@ -372,38 +381,32 @@ DiagnosticIDs::~DiagnosticIDs() {}
 ///
 /// \param FormatString A fixed diagnostic format string that will be hashed and
 /// mapped to a unique DiagID.
-unsigned DiagnosticIDs::getCustomDiagID(Level L, StringRef FormatString) {
+unsigned DiagnosticIDs::getCustomDiagID(CustomDiagDesc Diag) {
   if (!CustomDiagInfo)
     CustomDiagInfo.reset(new diag::CustomDiagInfo());
-  return CustomDiagInfo->getOrCreateDiagID(L, FormatString, *this);
+  return CustomDiagInfo->getOrCreateDiagID(Diag);
 }
 
-
-/// isBuiltinWarningOrExtension - Return true if the unmapped diagnostic
-/// level of the specified diagnostic ID is a Warning or Extension.
-/// This only works on builtin diagnostics, not custom ones, and is not legal to
-/// call on NOTEs.
-bool DiagnosticIDs::isBuiltinWarningOrExtension(unsigned DiagID) {
-  return DiagID < diag::DIAG_UPPER_LIMIT &&
-         getBuiltinDiagClass(DiagID) != CLASS_ERROR;
+bool DiagnosticIDs::isWarningOrExtension(unsigned DiagID) const {
+  return DiagID < diag::DIAG_UPPER_LIMIT
+             ? getDiagClass(DiagID) != CLASS_ERROR
+             : CustomDiagInfo->getDescription(DiagID).GetClass() != CLASS_ERROR;
 }
 
 /// Determine whether the given built-in diagnostic ID is a
 /// Note.
-bool DiagnosticIDs::isBuiltinNote(unsigned DiagID) {
-  return DiagID < diag::DIAG_UPPER_LIMIT &&
-    getBuiltinDiagClass(DiagID) == CLASS_NOTE;
+bool DiagnosticIDs::isNote(unsigned DiagID) const {
+  return DiagID < diag::DIAG_UPPER_LIMIT && getDiagClass(DiagID) == CLASS_NOTE;
 }
 
-/// isBuiltinExtensionDiag - Determine whether the given built-in diagnostic
+/// isExtensionDiag - Determine whether the given built-in diagnostic
 /// ID is for an extension of some sort.  This also returns EnabledByDefault,
 /// which is set to indicate whether the diagnostic is ignored by default (in
 /// which case -pedantic enables it) or treated as a warning/error by default.
 ///
-bool DiagnosticIDs::isBuiltinExtensionDiag(unsigned DiagID,
-                                        bool &EnabledByDefault) {
-  if (DiagID >= diag::DIAG_UPPER_LIMIT ||
-      getBuiltinDiagClass(DiagID) != CLASS_EXTENSION)
+bool DiagnosticIDs::isExtensionDiag(unsigned DiagID,
+                                    bool &EnabledByDefault) const {
+  if (IsCustomDiag(DiagID) || getDiagClass(DiagID) != CLASS_EXTENSION)
     return false;
 
   EnabledByDefault =
@@ -411,10 +414,7 @@ bool DiagnosticIDs::isBuiltinExtensionDiag(unsigned DiagID,
   return true;
 }
 
-bool DiagnosticIDs::isDefaultMappingAsError(unsigned DiagID) {
-  if (DiagID >= diag::DIAG_UPPER_LIMIT)
-    return false;
-
+bool DiagnosticIDs::isDefaultMappingAsError(unsigned DiagID) const {
   return getDefaultMapping(DiagID).getSeverity() >= diag::Severity::Error;
 }
 
@@ -424,7 +424,7 @@ StringRef DiagnosticIDs::getDescription(unsigned DiagID) const {
   if (const StaticDiagInfoRec *Info = GetDiagInfo(DiagID))
     return Info->getDescription();
   assert(CustomDiagInfo && "Invalid CustomDiagInfo");
-  return CustomDiagInfo->getDescription(DiagID);
+  return CustomDiagInfo->getDescription(DiagID).GetDescription();
 }
 
 static DiagnosticIDs::Level toLevel(diag::Severity SV) {
@@ -449,13 +449,7 @@ static DiagnosticIDs::Level toLevel(diag::Severity SV) {
 DiagnosticIDs::Level
 DiagnosticIDs::getDiagnosticLevel(unsigned DiagID, SourceLocation Loc,
                                   const DiagnosticsEngine &Diag) const {
-  // Handle custom diagnostics, which cannot be mapped.
-  if (DiagID >= diag::DIAG_UPPER_LIMIT) {
-    assert(CustomDiagInfo && "Invalid CustomDiagInfo");
-    return CustomDiagInfo->getLevel(DiagID);
-  }
-
-  unsigned DiagClass = getBuiltinDiagClass(DiagID);
+  unsigned DiagClass = getDiagClass(DiagID);
   if (DiagClass == CLASS_NOTE) return DiagnosticIDs::Note;
   return toLevel(getDiagnosticSeverity(DiagID, Loc, Diag));
 }
@@ -469,7 +463,8 @@ DiagnosticIDs::getDiagnosticLevel(unsigned DiagID, SourceLocation Loc,
 diag::Severity
 DiagnosticIDs::getDiagnosticSeverity(unsigned DiagID, SourceLocation Loc,
                                      const DiagnosticsEngine &Diag) const {
-  assert(getBuiltinDiagClass(DiagID) != CLASS_NOTE);
+  bool IsCustomDiag = DiagnosticIDs::IsCustomDiag(DiagID);
+  assert(getDiagClass(DiagID) != CLASS_NOTE);
 
   // Specific non-error diagnostics may be mapped to various levels from ignored
   // to error.  Errors can only be mapped to fatal.
@@ -477,7 +472,7 @@ DiagnosticIDs::getDiagnosticSeverity(unsigned DiagID, SourceLocation Loc,
 
   // Get the mapping information, or compute it lazily.
   DiagnosticsEngine::DiagState *State = Diag.GetDiagStateForLoc(Loc);
-  DiagnosticMapping &Mapping = State->getOrAddMapping((diag::kind)DiagID);
+  DiagnosticMapping Mapping = State->getOrAddMapping((diag::kind)DiagID);
 
   // TODO: Can a null severity really get here?
   if (Mapping.getSeverity() != diag::Severity())
@@ -485,14 +480,15 @@ DiagnosticIDs::getDiagnosticSeverity(unsigned DiagID, SourceLocation Loc,
 
   // Upgrade ignored diagnostics if -Weverything is enabled.
   if (State->EnableAllWarnings && Result == diag::Severity::Ignored &&
-      !Mapping.isUser() && getBuiltinDiagClass(DiagID) != CLASS_REMARK)
+      !Mapping.isUser() &&
+      (IsCustomDiag || getDiagClass(DiagID) != CLASS_REMARK))
     Result = diag::Severity::Warning;
 
   // Ignore -pedantic diagnostics inside __extension__ blocks.
   // (The diagnostics controlled by -pedantic are the extension diagnostics
   // that are not enabled by default.)
   bool EnabledByDefault = false;
-  bool IsExtensionDiag = isBuiltinExtensionDiag(DiagID, EnabledByDefault);
+  bool IsExtensionDiag = isExtensionDiag(DiagID, EnabledByDefault);
   if (Diag.AllExtensionsSilenced && IsExtensionDiag && !EnabledByDefault)
     return diag::Severity::Ignored;
 
@@ -510,10 +506,12 @@ DiagnosticIDs::getDiagnosticSeverity(unsigned DiagID, SourceLocation Loc,
   // as well as disabling all messages which are currently mapped to Warning
   // (whether by default or downgraded from Error via e.g. -Wno-error or #pragma
   // diagnostic.)
+  // FIXME: Should -w be ignored for custom warnings without a group?
   if (State->IgnoreAllWarnings) {
-    if (Result == diag::Severity::Warning ||
-        (Result >= diag::Severity::Error &&
-         !isDefaultMappingAsError((diag::kind)DiagID)))
+    if ((!IsCustomDiag || CustomDiagInfo->getDescription(DiagID).GetGroup()) &&
+        (Result == diag::Severity::Warning ||
+         (Result >= diag::Severity::Error &&
+          !isDefaultMappingAsError((diag::kind)DiagID))))
       return diag::Severity::Ignored;
   }
 
@@ -541,9 +539,11 @@ DiagnosticIDs::getDiagnosticSeverity(unsigned DiagID, SourceLocation Loc,
     return Result;
 
   const auto &SM = Diag.getSourceManager();
-  // Custom diagnostics always are emitted in system headers.
+
   bool ShowInSystemHeader =
-      !GetDiagInfo(DiagID) || GetDiagInfo(DiagID)->WarnShowInSystemHeader;
+      IsCustomDiag
+          ? CustomDiagInfo->getDescription(DiagID).ShouldShowInSystemHeader()
+          : !GetDiagInfo(DiagID) || GetDiagInfo(DiagID)->WarnShowInSystemHeader;
 
   // If we are in a system header, we ignore it. We look at the diagnostic class
   // because we also want to ignore extensions and warnings in -Werror and
@@ -566,6 +566,15 @@ DiagnosticIDs::getDiagnosticSeverity(unsigned DiagID, SourceLocation Loc,
   return Result;
 }
 
+DiagnosticIDs::Class DiagnosticIDs::getDiagClass(unsigned DiagID) const {
+  if (IsCustomDiag(DiagID))
+    return Class(CustomDiagInfo->getDescription(DiagID).GetClass());
+
+  if (const StaticDiagInfoRec *Info = GetDiagInfo(DiagID))
+    return Class(Info->Class);
+  return CLASS_INVALID;
+}
+
 #define GET_DIAG_ARRAYS
 #include "clang/Basic/DiagnosticGroups.inc"
 #undef GET_DIAG_ARRAYS
@@ -607,7 +616,12 @@ DiagnosticIDs::getGroupForWarningOption(StringRef Name) {
   return static_cast<diag::Group>(Found - OptionTable);
 }
 
-std::optional<diag::Group> DiagnosticIDs::getGroupForDiag(unsigned DiagID) {
+std::optional<diag::Group>
+DiagnosticIDs::getGroupForDiag(unsigned DiagID) const {
+  if (IsCustomDiag(DiagID)) {
+    assert(CustomDiagInfo);
+    return CustomDiagInfo->getDescription(DiagID).GetGroup();
+  }
   if (const StaticDiagInfoRec *Info = GetDiagInfo(DiagID))
     return static_cast<diag::Group>(Info->getOptionGroupIndex());
   return std::nullopt;
@@ -639,7 +653,8 @@ std::vector<std::string> DiagnosticIDs::getDiagnosticFlags() {
 /// were filtered out due to having the wrong flavor.
 static bool getDiagnosticsInGroup(diag::Flavor Flavor,
                                   const WarningOption *Group,
-                                  SmallVectorImpl<diag::kind> &Diags) {
+                                  SmallVectorImpl<diag::kind> &Diags,
+                                  diag::CustomDiagInfo *CustomDiagInfo) {
   // An empty group is considered to be a warning group: we have empty groups
   // for GCC compatibility, and GCC does not have remarks.
   if (!Group->Members && !Group->SubGroups)
@@ -658,9 +673,14 @@ static bool getDiagnosticsInGroup(diag::Flavor Flavor,
 
   // Add the members of the subgroups.
   const int16_t *SubGroups = DiagSubGroups + Group->SubGroups;
-  for (; *SubGroups != (int16_t)-1; ++SubGroups)
+  for (; *SubGroups != (int16_t)-1; ++SubGroups) {
+    if (CustomDiagInfo)
+      llvm::copy(
+          CustomDiagInfo->getDiagsInGroup(static_cast<diag::Group>(*SubGroups)),
+          std::back_inserter(Diags));
     NotFound &= getDiagnosticsInGroup(Flavor, &OptionTable[(short)*SubGroups],
-                                      Diags);
+                                      Diags, CustomDiagInfo);
+  }
 
   return NotFound;
 }
@@ -668,12 +688,49 @@ static bool getDiagnosticsInGroup(diag::Flavor Flavor,
 bool
 DiagnosticIDs::getDiagnosticsInGroup(diag::Flavor Flavor, StringRef Group,
                                      SmallVectorImpl<diag::kind> &Diags) const {
-  if (std::optional<diag::Group> G = getGroupForWarningOption(Group))
-    return ::getDiagnosticsInGroup(
-        Flavor, &OptionTable[static_cast<unsigned>(*G)], Diags);
+  if (std::optional<diag::Group> G = getGroupForWarningOption(Group)) {
+    if (CustomDiagInfo)
+      llvm::copy(CustomDiagInfo->getDiagsInGroup(*G),
+                 std::back_inserter(Diags));
+    return ::getDiagnosticsInGroup(Flavor,
+                                   &OptionTable[static_cast<unsigned>(*G)],
+                                   Diags, CustomDiagInfo.get());
+  }
   return true;
 }
 
+template <class Func>
+static void forEachSubGroupImpl(const WarningOption *Group, Func func) {
+  for (const int16_t *SubGroups = DiagSubGroups + Group->SubGroups;
+       *SubGroups != -1; ++SubGroups) {
+    func(static_cast<size_t>(*SubGroups));
+    forEachSubGroupImpl(&OptionTable[*SubGroups], std::move(func));
+  }
+}
+
+template <class Func>
+static void forEachSubGroup(diag::Group Group, Func func) {
+  const WarningOption *WarningOpt = &OptionTable[static_cast<size_t>(Group)];
+  func(static_cast<size_t>(Group));
+  ::forEachSubGroupImpl(WarningOpt, std::move(func));
+}
+
+void DiagnosticIDs::setGroupSeverity(StringRef Group, diag::Severity Sev) {
+  if (std::optional<diag::Group> G = getGroupForWarningOption(Group)) {
+    ::forEachSubGroup(*G, [&](size_t SubGroup) {
+      GroupInfos[SubGroup].Severity = static_cast<unsigned>(Sev);
+    });
+  }
+}
+
+void DiagnosticIDs::setGroupNoWarningsAsError(StringRef Group, bool Val) {
+  if (std::optional<diag::Group> G = getGroupForWarningOption(Group)) {
+    ::forEachSubGroup(*G, [&](size_t SubGroup) {
+      GroupInfos[static_cast<size_t>(*G)].HasNoWarningAsError = Val;
+    });
+  }
+}
+
 void DiagnosticIDs::getAllDiagnostics(diag::Flavor Flavor,
                                       std::vector<diag::kind> &Diags) {
   for (unsigned i = 0; i != StaticDiagInfoSize; ++i)
@@ -696,7 +753,7 @@ StringRef DiagnosticIDs::getNearestOption(diag::Flavor Flavor,
 
     // Don't suggest groups that are not of this kind.
     llvm::SmallVector<diag::kind, 8> Diags;
-    if (::getDiagnosticsInGroup(Flavor, &O, Diags) || Diags.empty())
+    if (::getDiagnosticsInGroup(Flavor, &O, Diags, nullptr) || Diags.empty())
       continue;
 
     if (Distance == BestDistance) {
@@ -810,14 +867,8 @@ void DiagnosticIDs::EmitDiag(DiagnosticsEngine &Diag,
 }
 
 bool DiagnosticIDs::isUnrecoverable(unsigned DiagID) const {
-  if (DiagID >= diag::DIAG_UPPER_LIMIT) {
-    assert(CustomDiagInfo && "Invalid CustomDiagInfo");
-    // Custom diagnostics.
-    return CustomDiagInfo->getLevel(DiagID) >= DiagnosticIDs::Error;
-  }
-
   // Only errors may be unrecoverable.
-  if (getBuiltinDiagClass(DiagID) < CLASS_ERROR)
+  if (getDiagClass(DiagID) < CLASS_ERROR)
     return false;
 
   if (DiagID == diag::err_unavailable ||
diff --git a/clang/lib/Basic/OpenMPKinds.cpp b/clang/lib/Basic/OpenMPKinds.cpp
index 62a13f01481b2..956d92a7e95f0 100644
--- a/clang/lib/Basic/OpenMPKinds.cpp
+++ b/clang/lib/Basic/OpenMPKinds.cpp
@@ -765,6 +765,12 @@ bool clang::isOpenMPCapturingDirective(OpenMPDirectiveKind DKind) {
   return false;
 }
 
+bool clang::isOpenMPOrderConcurrentNestableDirective(
+    OpenMPDirectiveKind DKind) {
+  return DKind == OMPD_atomic || DKind == OMPD_loop || DKind == OMPD_simd ||
+         DKind == OMPD_parallel || isOpenMPLoopTransformationDirective(DKind);
+}
+
 void clang::getOpenMPCaptureRegions(
     SmallVectorImpl<OpenMPDirectiveKind> &CaptureRegions,
     OpenMPDirectiveKind DKind) {
diff --git a/clang/lib/Basic/Sanitizers.cpp b/clang/lib/Basic/Sanitizers.cpp
index 5b9b88d032702..e0eff2a37b8a0 100644
--- a/clang/lib/Basic/Sanitizers.cpp
+++ b/clang/lib/Basic/Sanitizers.cpp
@@ -18,6 +18,7 @@
 #include "llvm/Support/MathExtras.h"
 #include "llvm/Support/raw_ostream.h"
 #include <algorithm>
+#include <cmath>
 #include <optional>
 
 using namespace clang;
@@ -43,6 +44,27 @@ std::optional<double> SanitizerMaskCutoffs::operator[](unsigned Kind) const {
 
 void SanitizerMaskCutoffs::clear(SanitizerMask K) { set(K, 0); }
 
+std::optional<std::vector<unsigned>>
+SanitizerMaskCutoffs::getAllScaled(unsigned ScalingFactor) const {
+  std::vector<unsigned> ScaledCutoffs;
+
+  bool AnyCutoff = false;
+  for (unsigned int i = 0; i < SanitizerKind::SO_Count; ++i) {
+    auto C = (*this)[i];
+    if (C.has_value()) {
+      ScaledCutoffs.push_back(lround(std::clamp(*C, 0.0, 1.0) * ScalingFactor));
+      AnyCutoff = true;
+    } else {
+      ScaledCutoffs.push_back(0);
+    }
+  }
+
+  if (AnyCutoff)
+    return ScaledCutoffs;
+
+  return std::nullopt;
+}
+
 // Once LLVM switches to C++17, the constexpr variables can be inline and we
 // won't need this.
 #define SANITIZER(NAME, ID) constexpr SanitizerMask SanitizerKind::ID;
diff --git a/clang/lib/Basic/Targets/AArch64.cpp b/clang/lib/Basic/Targets/AArch64.cpp
index 0b899137bbb5c..fad8d773bfc52 100644
--- a/clang/lib/Basic/Targets/AArch64.cpp
+++ b/clang/lib/Basic/Targets/AArch64.cpp
@@ -26,36 +26,106 @@
 using namespace clang;
 using namespace clang::targets;
 
-static constexpr Builtin::Info BuiltinInfo[] = {
-#define BUILTIN(ID, TYPE, ATTRS)                                               \
-  {#ID, TYPE, ATTRS, nullptr, HeaderDesc::NO_HEADER, ALL_LANGUAGES},
-#define TARGET_BUILTIN(ID, TYPE, ATTRS, FEATURE)                               \
-  {#ID, TYPE, ATTRS, FEATURE, HeaderDesc::NO_HEADER, ALL_LANGUAGES},
-#include "clang/Basic/BuiltinsNEON.def"
-
-#define BUILTIN(ID, TYPE, ATTRS)                                               \
-  {#ID, TYPE, ATTRS, nullptr, HeaderDesc::NO_HEADER, ALL_LANGUAGES},
-#define TARGET_BUILTIN(ID, TYPE, ATTRS, FEATURE)                               \
-  {#ID, TYPE, ATTRS, FEATURE, HeaderDesc::NO_HEADER, ALL_LANGUAGES},
-#include "clang/Basic/BuiltinsSVE.def"
-
-#define BUILTIN(ID, TYPE, ATTRS)                                               \
-  {#ID, TYPE, ATTRS, nullptr, HeaderDesc::NO_HEADER, ALL_LANGUAGES},
-#define TARGET_BUILTIN(ID, TYPE, ATTRS, FEATURE)                               \
-  {#ID, TYPE, ATTRS, FEATURE, HeaderDesc::NO_HEADER, ALL_LANGUAGES},
-#include "clang/Basic/BuiltinsSME.def"
-
-#define BUILTIN(ID, TYPE, ATTRS)                                               \
-  {#ID, TYPE, ATTRS, nullptr, HeaderDesc::NO_HEADER, ALL_LANGUAGES},
-#define LANGBUILTIN(ID, TYPE, ATTRS, LANG)                                     \
-  {#ID, TYPE, ATTRS, nullptr, HeaderDesc::NO_HEADER, LANG},
-#define TARGET_BUILTIN(ID, TYPE, ATTRS, FEATURE)                               \
-  {#ID, TYPE, ATTRS, FEATURE, HeaderDesc::NO_HEADER, ALL_LANGUAGES},
-#define TARGET_HEADER_BUILTIN(ID, TYPE, ATTRS, HEADER, LANGS, FEATURE)         \
-  {#ID, TYPE, ATTRS, FEATURE, HeaderDesc::HEADER, LANGS},
-#include "clang/Basic/BuiltinsAArch64.def"
+static constexpr int NumNeonBuiltins =
+    NEON::FirstFp16Builtin - Builtin::FirstTSBuiltin;
+static constexpr int NumFp16Builtins =
+    NEON::FirstTSBuiltin - NEON::FirstFp16Builtin;
+static constexpr int NumSVEBuiltins =
+    SVE::FirstNeonBridgeBuiltin - NEON::FirstTSBuiltin;
+static constexpr int NumSVENeonBridgeBuiltins =
+    SVE::FirstTSBuiltin - SVE::FirstNeonBridgeBuiltin;
+static constexpr int NumSMEBuiltins = SME::FirstTSBuiltin - SVE::FirstTSBuiltin;
+static constexpr int NumAArch64Builtins =
+    AArch64::LastTSBuiltin - SME::FirstTSBuiltin;
+static constexpr int NumBuiltins =
+    AArch64::LastTSBuiltin - Builtin::FirstTSBuiltin;
+static_assert(NumBuiltins ==
+              (NumNeonBuiltins + NumFp16Builtins + NumSVEBuiltins +
+               NumSVENeonBridgeBuiltins + NumSMEBuiltins + NumAArch64Builtins));
+
+namespace clang {
+namespace NEON {
+#define GET_NEON_BUILTIN_STR_TABLE
+#include "clang/Basic/arm_neon.inc"
+#undef GET_NEON_BUILTIN_STR_TABLE
+
+static constexpr std::array<Builtin::Info, NumNeonBuiltins> BuiltinInfos = {
+#define GET_NEON_BUILTIN_INFOS
+#include "clang/Basic/arm_neon.inc"
+#undef GET_NEON_BUILTIN_INFOS
 };
 
+namespace FP16 {
+#define GET_NEON_BUILTIN_STR_TABLE
+#include "clang/Basic/arm_fp16.inc"
+#undef GET_NEON_BUILTIN_STR_TABLE
+
+static constexpr std::array<Builtin::Info, NumFp16Builtins> BuiltinInfos = {
+#define GET_NEON_BUILTIN_INFOS
+#include "clang/Basic/arm_fp16.inc"
+#undef GET_NEON_BUILTIN_INFOS
+};
+} // namespace FP16
+} // namespace NEON
+
+namespace SVE {
+#define GET_SVE_BUILTIN_STR_TABLE
+#include "clang/Basic/arm_sve_builtins.inc"
+#undef GET_SVE_BUILTIN_STR_TABLE
+
+static constexpr std::array<Builtin::Info, NumSVEBuiltins> BuiltinInfos = {
+#define GET_SVE_BUILTIN_INFOS
+#include "clang/Basic/arm_sve_builtins.inc"
+#undef GET_SVE_BUILTIN_INFOS
+};
+} // namespace SVE
+
+namespace SME {
+#define GET_SME_BUILTIN_STR_TABLE
+#include "clang/Basic/arm_sme_builtins.inc"
+#undef GET_SME_BUILTIN_STR_TABLE
+
+static constexpr std::array<Builtin::Info, NumSMEBuiltins> BuiltinInfos = {
+#define GET_SME_BUILTIN_INFOS
+#include "clang/Basic/arm_sme_builtins.inc"
+#undef GET_SME_BUILTIN_INFOS
+};
+} // namespace SME
+} // namespace clang
+
+static constexpr llvm::StringTable BuiltinSVENeonBridgeStrings =
+    CLANG_BUILTIN_STR_TABLE_START
+#define TARGET_BUILTIN CLANG_TARGET_BUILTIN_STR_TABLE
+#define GET_SVE_BUILTINS
+#include "clang/Basic/BuiltinsAArch64NeonSVEBridge.def"
+#undef GET_SVE_BUILTINS
+#undef TARGET_BUILTIN
+    ;
+static constexpr llvm::StringTable BuiltinAArch64Strings =
+    CLANG_BUILTIN_STR_TABLE_START
+#define BUILTIN CLANG_BUILTIN_STR_TABLE
+#define TARGET_BUILTIN CLANG_TARGET_BUILTIN_STR_TABLE
+#define TARGET_HEADER_BUILTIN CLANG_TARGET_HEADER_BUILTIN_STR_TABLE
+#include "clang/Basic/BuiltinsAArch64.def"
+    ;
+
+static constexpr auto BuiltinSVENeonBridgeInfos =
+    Builtin::MakeInfos<NumSVENeonBridgeBuiltins>({
+#define TARGET_BUILTIN CLANG_TARGET_BUILTIN_ENTRY
+#define GET_SVE_BUILTINS
+#include "clang/Basic/BuiltinsAArch64NeonSVEBridge.def"
+#undef GET_SVE_BUILTINS
+#undef TARGET_BUILTIN
+    });
+static constexpr auto BuiltinAArch64Infos =
+    Builtin::MakeInfos<NumAArch64Builtins>({
+#define BUILTIN CLANG_BUILTIN_ENTRY
+#define TARGET_BUILTIN CLANG_TARGET_BUILTIN_ENTRY
+#define LANGBUILTIN CLANG_LANGBUILTIN_ENTRY
+#define TARGET_HEADER_BUILTIN CLANG_TARGET_HEADER_BUILTIN_ENTRY
+#include "clang/Basic/BuiltinsAArch64.def"
+    });
+
 void AArch64TargetInfo::setArchFeatures() {
   if (*ArchInfo == llvm::AArch64::ARMV8R) {
     HasDotProd = true;
@@ -253,11 +323,19 @@ bool AArch64TargetInfo::validateGlobalRegisterVariable(
 
 bool AArch64TargetInfo::validateBranchProtection(StringRef Spec, StringRef,
                                                  BranchProtectionInfo &BPI,
+                                                 const LangOptions &LO,
                                                  StringRef &Err) const {
   llvm::ARM::ParsedBranchProtection PBP;
   if (!llvm::ARM::parseBranchProtection(Spec, PBP, Err, HasPAuthLR))
     return false;
 
+  // GCS is currently untested with ptrauth-returns, but enabling this could be
+  // allowed in future after testing with a suitable system.
+  if (LO.PointerAuthReturns &&
+      (PBP.Scope != "none" || PBP.BranchProtectionPAuthLR ||
+       PBP.GuardedControlStack))
+    return false;
+
   BPI.SignReturnAddr =
       llvm::StringSwitch<LangOptions::SignReturnAddressScopeKind>(PBP.Scope)
           .Case("non-leaf", LangOptions::SignReturnAddressScopeKind::NonLeaf)
@@ -697,18 +775,27 @@ void AArch64TargetInfo::getTargetDefines(const LangOptions &Opts,
   }
 }
 
-ArrayRef<Builtin::Info> AArch64TargetInfo::getTargetBuiltins() const {
-  return llvm::ArrayRef(BuiltinInfo, clang::AArch64::LastTSBuiltin -
-                                         Builtin::FirstTSBuiltin);
+llvm::SmallVector<Builtin::InfosShard>
+AArch64TargetInfo::getTargetBuiltins() const {
+  return {
+      {&NEON::BuiltinStrings, NEON::BuiltinInfos, "__builtin_neon_"},
+      {&NEON::FP16::BuiltinStrings, NEON::FP16::BuiltinInfos,
+       "__builtin_neon_"},
+      {&SVE::BuiltinStrings, SVE::BuiltinInfos, "__builtin_sve_"},
+      {&BuiltinSVENeonBridgeStrings, BuiltinSVENeonBridgeInfos},
+      {&SME::BuiltinStrings, SME::BuiltinInfos, "__builtin_sme_"},
+      {&BuiltinAArch64Strings, BuiltinAArch64Infos},
+  };
 }
 
 std::optional<std::pair<unsigned, unsigned>>
-AArch64TargetInfo::getVScaleRange(const LangOptions &LangOpts) const {
+AArch64TargetInfo::getVScaleRange(const LangOptions &LangOpts,
+                                  bool IsArmStreamingFunction) const {
   if (LangOpts.VScaleMin || LangOpts.VScaleMax)
     return std::pair<unsigned, unsigned>(
         LangOpts.VScaleMin ? LangOpts.VScaleMin : 1, LangOpts.VScaleMax);
 
-  if (hasFeature("sve"))
+  if (hasFeature("sve") || (IsArmStreamingFunction && hasFeature("sme")))
     return std::pair<unsigned, unsigned>(1, 16);
 
   return std::nullopt;
diff --git a/clang/lib/Basic/Targets/AArch64.h b/clang/lib/Basic/Targets/AArch64.h
index ecf80b23a508c..9b6451bd06316 100644
--- a/clang/lib/Basic/Targets/AArch64.h
+++ b/clang/lib/Basic/Targets/AArch64.h
@@ -44,6 +44,7 @@ static const unsigned ARM64AddrSpaceMap[] = {
     static_cast<unsigned>(AArch64AddrSpace::ptr32_uptr),
     static_cast<unsigned>(AArch64AddrSpace::ptr64),
     0, // hlsl_groupshared
+    0, // hlsl_constant
     // Wasm address space values for this target are dummy values,
     // as it is only enabled for Wasm targets.
     20, // wasm_funcref
@@ -131,6 +132,7 @@ class LLVM_LIBRARY_VISIBILITY AArch64TargetInfo : public TargetInfo {
 
   bool validateBranchProtection(StringRef Spec, StringRef Arch,
                                 BranchProtectionInfo &BPI,
+                                const LangOptions &LO,
                                 StringRef &Err) const override;
 
   bool isValidCPUName(StringRef Name) const override;
@@ -180,10 +182,11 @@ class LLVM_LIBRARY_VISIBILITY AArch64TargetInfo : public TargetInfo {
   void getTargetDefines(const LangOptions &Opts,
                         MacroBuilder &Builder) const override;
 
-  ArrayRef<Builtin::Info> getTargetBuiltins() const override;
+  llvm::SmallVector<Builtin::InfosShard> getTargetBuiltins() const override;
 
   std::optional<std::pair<unsigned, unsigned>>
-  getVScaleRange(const LangOptions &LangOpts) const override;
+  getVScaleRange(const LangOptions &LangOpts,
+                 bool IsArmStreamingFunction) const override;
   bool doesFeatureAffectCodeGen(StringRef Name) const override;
   bool validateCpuSupports(StringRef FeatureStr) const override;
   bool hasFeature(StringRef Feature) const override;
@@ -226,6 +229,11 @@ class LLVM_LIBRARY_VISIBILITY AArch64TargetInfo : public TargetInfo {
   bool validatePointerAuthKey(const llvm::APSInt &value) const override;
 
   const char *getBFloat16Mangling() const override { return "u6__bf16"; };
+
+  std::pair<unsigned, unsigned> hardwareInterferenceSizes() const override {
+    return std::make_pair(256, 64);
+  }
+
   bool hasInt128Type() const override;
 
   bool hasBitIntType() const override { return true; }
diff --git a/clang/lib/Basic/Targets/AMDGPU.cpp b/clang/lib/Basic/Targets/AMDGPU.cpp
index 99f8f2944e279..228f967caf2f1 100644
--- a/clang/lib/Basic/Targets/AMDGPU.cpp
+++ b/clang/lib/Basic/Targets/AMDGPU.cpp
@@ -59,6 +59,7 @@ const LangASMap AMDGPUTargetInfo::AMDGPUDefIsGenMap = {
     llvm::AMDGPUAS::FLAT_ADDRESS,     // ptr32_uptr
     llvm::AMDGPUAS::FLAT_ADDRESS,     // ptr64
     llvm::AMDGPUAS::FLAT_ADDRESS,     // hlsl_groupshared
+    llvm::AMDGPUAS::CONSTANT_ADDRESS, // hlsl_constant
 };
 
 const LangASMap AMDGPUTargetInfo::AMDGPUDefIsPrivMap = {
@@ -74,27 +75,35 @@ const LangASMap AMDGPUTargetInfo::AMDGPUDefIsPrivMap = {
     llvm::AMDGPUAS::CONSTANT_ADDRESS, // cuda_constant
     llvm::AMDGPUAS::LOCAL_ADDRESS,    // cuda_shared
     // SYCL address space values for this map are dummy
-    llvm::AMDGPUAS::FLAT_ADDRESS, // sycl_global
-    llvm::AMDGPUAS::FLAT_ADDRESS, // sycl_global_device
-    llvm::AMDGPUAS::FLAT_ADDRESS, // sycl_global_host
-    llvm::AMDGPUAS::FLAT_ADDRESS, // sycl_local
-    llvm::AMDGPUAS::FLAT_ADDRESS, // sycl_private
-    llvm::AMDGPUAS::FLAT_ADDRESS, // ptr32_sptr
-    llvm::AMDGPUAS::FLAT_ADDRESS, // ptr32_uptr
-    llvm::AMDGPUAS::FLAT_ADDRESS, // ptr64
-    llvm::AMDGPUAS::FLAT_ADDRESS, // hlsl_groupshared
-
+    llvm::AMDGPUAS::FLAT_ADDRESS,     // sycl_global
+    llvm::AMDGPUAS::FLAT_ADDRESS,     // sycl_global_device
+    llvm::AMDGPUAS::FLAT_ADDRESS,     // sycl_global_host
+    llvm::AMDGPUAS::FLAT_ADDRESS,     // sycl_local
+    llvm::AMDGPUAS::FLAT_ADDRESS,     // sycl_private
+    llvm::AMDGPUAS::FLAT_ADDRESS,     // ptr32_sptr
+    llvm::AMDGPUAS::FLAT_ADDRESS,     // ptr32_uptr
+    llvm::AMDGPUAS::FLAT_ADDRESS,     // ptr64
+    llvm::AMDGPUAS::FLAT_ADDRESS,     // hlsl_groupshared
+    llvm::AMDGPUAS::CONSTANT_ADDRESS, // hlsl_constant
 };
 } // namespace targets
 } // namespace clang
 
-static constexpr Builtin::Info BuiltinInfo[] = {
-#define BUILTIN(ID, TYPE, ATTRS)                                               \
-  {#ID, TYPE, ATTRS, nullptr, HeaderDesc::NO_HEADER, ALL_LANGUAGES},
-#define TARGET_BUILTIN(ID, TYPE, ATTRS, FEATURE)                               \
-  {#ID, TYPE, ATTRS, FEATURE, HeaderDesc::NO_HEADER, ALL_LANGUAGES},
+static constexpr int NumBuiltins =
+    clang::AMDGPU::LastTSBuiltin - Builtin::FirstTSBuiltin;
+
+static constexpr llvm::StringTable BuiltinStrings =
+    CLANG_BUILTIN_STR_TABLE_START
+#define BUILTIN CLANG_BUILTIN_STR_TABLE
+#define TARGET_BUILTIN CLANG_TARGET_BUILTIN_STR_TABLE
 #include "clang/Basic/BuiltinsAMDGPU.def"
-};
+    ;
+
+static constexpr auto BuiltinInfos = Builtin::MakeInfos<NumBuiltins>({
+#define BUILTIN CLANG_BUILTIN_ENTRY
+#define TARGET_BUILTIN CLANG_TARGET_BUILTIN_ENTRY
+#include "clang/Basic/BuiltinsAMDGPU.def"
+});
 
 const char *const AMDGPUTargetInfo::GCCRegNames[] = {
   "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8",
@@ -266,9 +275,9 @@ void AMDGPUTargetInfo::adjust(DiagnosticsEngine &Diags, LangOptions &Opts) {
                      !isAMDGCN(getTriple()));
 }
 
-ArrayRef<Builtin::Info> AMDGPUTargetInfo::getTargetBuiltins() const {
-  return llvm::ArrayRef(BuiltinInfo,
-                        clang::AMDGPU::LastTSBuiltin - Builtin::FirstTSBuiltin);
+llvm::SmallVector<Builtin::InfosShard>
+AMDGPUTargetInfo::getTargetBuiltins() const {
+  return {{&BuiltinStrings, BuiltinInfos}};
 }
 
 void AMDGPUTargetInfo::getTargetDefines(const LangOptions &Opts,
diff --git a/clang/lib/Basic/Targets/AMDGPU.h b/clang/lib/Basic/Targets/AMDGPU.h
index ea4189cdea47d..3d6778fb5a76f 100644
--- a/clang/lib/Basic/Targets/AMDGPU.h
+++ b/clang/lib/Basic/Targets/AMDGPU.h
@@ -257,7 +257,7 @@ class LLVM_LIBRARY_VISIBILITY AMDGPUTargetInfo final : public TargetInfo {
                  StringRef CPU,
                  const std::vector<std::string> &FeatureVec) const override;
 
-  ArrayRef<Builtin::Info> getTargetBuiltins() const override;
+  llvm::SmallVector<Builtin::InfosShard> getTargetBuiltins() const override;
 
   bool useFP16ConversionIntrinsics() const override { return false; }
 
diff --git a/clang/lib/Basic/Targets/ARC.h b/clang/lib/Basic/Targets/ARC.h
index 7f3d0aa15ab81..2b69f95591fa1 100644
--- a/clang/lib/Basic/Targets/ARC.h
+++ b/clang/lib/Basic/Targets/ARC.h
@@ -40,7 +40,9 @@ class LLVM_LIBRARY_VISIBILITY ARCTargetInfo : public TargetInfo {
   void getTargetDefines(const LangOptions &Opts,
                         MacroBuilder &Builder) const override;
 
-  ArrayRef<Builtin::Info> getTargetBuiltins() const override { return {}; }
+  llvm::SmallVector<Builtin::InfosShard> getTargetBuiltins() const override {
+    return {};
+  }
 
   BuiltinVaListKind getBuiltinVaListKind() const override {
     return TargetInfo::VoidPtrBuiltinVaList;
diff --git a/clang/lib/Basic/Targets/ARM.cpp b/clang/lib/Basic/Targets/ARM.cpp
index 0fd5433a76402..ca2c1ffbb0eb7 100644
--- a/clang/lib/Basic/Targets/ARM.cpp
+++ b/clang/lib/Basic/Targets/ARM.cpp
@@ -405,6 +405,7 @@ bool ARMTargetInfo::isBranchProtectionSupportedArch(StringRef Arch) const {
 
 bool ARMTargetInfo::validateBranchProtection(StringRef Spec, StringRef Arch,
                                              BranchProtectionInfo &BPI,
+                                             const LangOptions &LO,
                                              StringRef &Err) const {
   llvm::ARM::ParsedBranchProtection PBP;
   if (!llvm::ARM::parseBranchProtection(Spec, PBP, Err))
@@ -608,6 +609,8 @@ bool ARMTargetInfo::handleTargetFeatures(std::vector<std::string> &Features,
       HasBTI = 1;
     } else if (Feature == "+fullbf16") {
       HasFullBFloat16 = true;
+    } else if (Feature == "+execute-only") {
+      TLSSupported = false;
     }
   }
 
@@ -1072,31 +1075,99 @@ void ARMTargetInfo::getTargetDefines(const LangOptions &Opts,
   }
 }
 
-static constexpr Builtin::Info BuiltinInfo[] = {
-#define BUILTIN(ID, TYPE, ATTRS)                                               \
-  {#ID, TYPE, ATTRS, nullptr, HeaderDesc::NO_HEADER, ALL_LANGUAGES},
-#define LIBBUILTIN(ID, TYPE, ATTRS, HEADER)                                    \
-  {#ID, TYPE, ATTRS, nullptr, HeaderDesc::HEADER, ALL_LANGUAGES},
-#define TARGET_BUILTIN(ID, TYPE, ATTRS, FEATURE)                               \
-  {#ID, TYPE, ATTRS, FEATURE, HeaderDesc::NO_HEADER, ALL_LANGUAGES},
-#include "clang/Basic/BuiltinsNEON.def"
-
-#define BUILTIN(ID, TYPE, ATTRS)                                               \
-  {#ID, TYPE, ATTRS, nullptr, HeaderDesc::NO_HEADER, ALL_LANGUAGES},
-#define LANGBUILTIN(ID, TYPE, ATTRS, LANG)                                     \
-  {#ID, TYPE, ATTRS, nullptr, HeaderDesc::NO_HEADER, LANG},
-#define LIBBUILTIN(ID, TYPE, ATTRS, HEADER)                                    \
-  {#ID, TYPE, ATTRS, nullptr, HeaderDesc::HEADER, ALL_LANGUAGES},
-#define TARGET_BUILTIN(ID, TYPE, ATTRS, FEATURE)                               \
-  {#ID, TYPE, ATTRS, FEATURE, HeaderDesc::NO_HEADER, ALL_LANGUAGES},
-#define TARGET_HEADER_BUILTIN(ID, TYPE, ATTRS, HEADER, LANGS, FEATURE)         \
-  {#ID, TYPE, ATTRS, FEATURE, HeaderDesc::HEADER, LANGS},
-#include "clang/Basic/BuiltinsARM.def"
+static constexpr int NumBuiltins = ARM::LastTSBuiltin - Builtin::FirstTSBuiltin;
+static constexpr int NumNeonBuiltins =
+    NEON::FirstFp16Builtin - Builtin::FirstTSBuiltin;
+static constexpr int NumFp16Builtins =
+    NEON::FirstTSBuiltin - NEON::FirstFp16Builtin;
+static constexpr int NumMVEBuiltins =
+    ARM::FirstCDEBuiltin - NEON::FirstTSBuiltin;
+static constexpr int NumCDEBuiltins =
+    ARM::FirstARMBuiltin - ARM::FirstCDEBuiltin;
+static constexpr int NumARMBuiltins = ARM::LastTSBuiltin - ARM::FirstARMBuiltin;
+static_assert(NumBuiltins ==
+              (NumNeonBuiltins + NumFp16Builtins + NumMVEBuiltins +
+               NumCDEBuiltins + NumARMBuiltins));
+
+namespace clang {
+namespace NEON {
+#define GET_NEON_BUILTIN_STR_TABLE
+#include "clang/Basic/arm_neon.inc"
+#undef GET_NEON_BUILTIN_STR_TABLE
+
+static constexpr std::array<Builtin::Info, NumNeonBuiltins> BuiltinInfos = {
+#define GET_NEON_BUILTIN_INFOS
+#include "clang/Basic/arm_neon.inc"
+#undef GET_NEON_BUILTIN_INFOS
+};
+
+namespace FP16 {
+#define GET_NEON_BUILTIN_STR_TABLE
+#include "clang/Basic/arm_fp16.inc"
+#undef GET_NEON_BUILTIN_STR_TABLE
+
+static constexpr std::array<Builtin::Info, NumFp16Builtins> BuiltinInfos = {
+#define GET_NEON_BUILTIN_INFOS
+#include "clang/Basic/arm_fp16.inc"
+#undef GET_NEON_BUILTIN_INFOS
+};
+} // namespace FP16
+} // namespace NEON
+} // namespace clang
+
+namespace {
+namespace MVE {
+#define GET_MVE_BUILTIN_STR_TABLE
+#include "clang/Basic/arm_mve_builtins.inc"
+#undef GET_MVE_BUILTIN_STR_TABLE
+
+static constexpr std::array<Builtin::Info, NumMVEBuiltins> BuiltinInfos = {
+#define GET_MVE_BUILTIN_INFOS
+#include "clang/Basic/arm_mve_builtins.inc"
+#undef GET_MVE_BUILTIN_INFOS
 };
+} // namespace MVE
 
-ArrayRef<Builtin::Info> ARMTargetInfo::getTargetBuiltins() const {
-  return llvm::ArrayRef(BuiltinInfo,
-                        clang::ARM::LastTSBuiltin - Builtin::FirstTSBuiltin);
+namespace CDE {
+#define GET_CDE_BUILTIN_STR_TABLE
+#include "clang/Basic/arm_cde_builtins.inc"
+#undef GET_CDE_BUILTIN_STR_TABLE
+
+static constexpr std::array<Builtin::Info, NumCDEBuiltins> BuiltinInfos = {
+#define GET_CDE_BUILTIN_INFOS
+#include "clang/Basic/arm_cde_builtins.inc"
+#undef GET_CDE_BUILTIN_INFOS
+};
+} // namespace CDE
+} // namespace
+
+static constexpr llvm::StringTable BuiltinStrings =
+    CLANG_BUILTIN_STR_TABLE_START
+#define BUILTIN CLANG_BUILTIN_STR_TABLE
+#define TARGET_BUILTIN CLANG_TARGET_BUILTIN_STR_TABLE
+#define TARGET_HEADER_BUILTIN CLANG_TARGET_HEADER_BUILTIN_STR_TABLE
+#include "clang/Basic/BuiltinsARM.def"
+    ; // namespace clang
+
+static constexpr auto BuiltinInfos = Builtin::MakeInfos<NumARMBuiltins>({
+#define BUILTIN CLANG_BUILTIN_ENTRY
+#define LANGBUILTIN CLANG_LANGBUILTIN_ENTRY
+#define LIBBUILTIN CLANG_LIBBUILTIN_ENTRY
+#define TARGET_BUILTIN CLANG_TARGET_BUILTIN_ENTRY
+#define TARGET_HEADER_BUILTIN CLANG_TARGET_HEADER_BUILTIN_ENTRY
+#include "clang/Basic/BuiltinsARM.def"
+});
+
+llvm::SmallVector<Builtin::InfosShard>
+ARMTargetInfo::getTargetBuiltins() const {
+  return {
+      {&NEON::BuiltinStrings, NEON::BuiltinInfos, "__builtin_neon_"},
+      {&NEON::FP16::BuiltinStrings, NEON::FP16::BuiltinInfos,
+       "__builtin_neon_"},
+      {&MVE::BuiltinStrings, MVE::BuiltinInfos, "__builtin_arm_mve_"},
+      {&CDE::BuiltinStrings, CDE::BuiltinInfos, "__builtin_arm_cde_"},
+      {&BuiltinStrings, BuiltinInfos},
+  };
 }
 
 bool ARMTargetInfo::isCLZForZeroUndef() const { return false; }
diff --git a/clang/lib/Basic/Targets/ARM.h b/clang/lib/Basic/Targets/ARM.h
index fdb40c3d41918..1719217ca25b7 100644
--- a/clang/lib/Basic/Targets/ARM.h
+++ b/clang/lib/Basic/Targets/ARM.h
@@ -155,6 +155,7 @@ class LLVM_LIBRARY_VISIBILITY ARMTargetInfo : public TargetInfo {
   bool isBranchProtectionSupportedArch(StringRef Arch) const override;
   bool validateBranchProtection(StringRef Spec, StringRef Arch,
                                 BranchProtectionInfo &BPI,
+                                const LangOptions &LO,
                                 StringRef &Err) const override;
 
   // FIXME: This should be based on Arch attributes, not CPU names.
@@ -196,7 +197,7 @@ class LLVM_LIBRARY_VISIBILITY ARMTargetInfo : public TargetInfo {
   void getTargetDefines(const LangOptions &Opts,
                         MacroBuilder &Builder) const override;
 
-  ArrayRef<Builtin::Info> getTargetBuiltins() const override;
+  llvm::SmallVector<Builtin::InfosShard> getTargetBuiltins() const override;
 
   bool isCLZForZeroUndef() const override;
   BuiltinVaListKind getBuiltinVaListKind() const override;
@@ -227,7 +228,7 @@ class LLVM_LIBRARY_VISIBILITY ARMTargetInfo : public TargetInfo {
   const char *getBFloat16Mangling() const override { return "u6__bf16"; };
 
   std::pair<unsigned, unsigned> hardwareInterferenceSizes() const override {
-    return std::make_pair(getTriple().isArch64Bit() ? 256 : 64, 64);
+    return std::make_pair(64, 64);
   }
 };
 
diff --git a/clang/lib/Basic/Targets/AVR.h b/clang/lib/Basic/Targets/AVR.h
index df1f8d171efba..2117ab58e6f30 100644
--- a/clang/lib/Basic/Targets/AVR.h
+++ b/clang/lib/Basic/Targets/AVR.h
@@ -63,7 +63,9 @@ class LLVM_LIBRARY_VISIBILITY AVRTargetInfo : public TargetInfo {
   void getTargetDefines(const LangOptions &Opts,
                         MacroBuilder &Builder) const override;
 
-  ArrayRef<Builtin::Info> getTargetBuiltins() const override { return {}; }
+  llvm::SmallVector<Builtin::InfosShard> getTargetBuiltins() const override {
+    return {};
+  }
 
   bool allowsLargerPreferedTypeAlignment() const override { return false; }
 
diff --git a/clang/lib/Basic/Targets/BPF.cpp b/clang/lib/Basic/Targets/BPF.cpp
index f4684765b7ffb..a463de0884020 100644
--- a/clang/lib/Basic/Targets/BPF.cpp
+++ b/clang/lib/Basic/Targets/BPF.cpp
@@ -19,11 +19,19 @@
 using namespace clang;
 using namespace clang::targets;
 
-static constexpr Builtin::Info BuiltinInfo[] = {
-#define BUILTIN(ID, TYPE, ATTRS)                                               \
-  {#ID, TYPE, ATTRS, nullptr, HeaderDesc::NO_HEADER, ALL_LANGUAGES},
+static constexpr int NumBuiltins =
+    clang::BPF::LastTSBuiltin - Builtin::FirstTSBuiltin;
+
+#define GET_BUILTIN_STR_TABLE
+#include "clang/Basic/BuiltinsBPF.inc"
+#undef GET_BUILTIN_STR_TABLE
+
+static constexpr Builtin::Info BuiltinInfos[] = {
+#define GET_BUILTIN_INFOS
 #include "clang/Basic/BuiltinsBPF.inc"
+#undef GET_BUILTIN_INFOS
 };
+static_assert(std::size(BuiltinInfos) == NumBuiltins);
 
 void BPFTargetInfo::getTargetDefines(const LangOptions &Opts,
                                      MacroBuilder &Builder) const {
@@ -81,9 +89,9 @@ void BPFTargetInfo::fillValidCPUList(SmallVectorImpl<StringRef> &Values) const {
   Values.append(std::begin(ValidCPUNames), std::end(ValidCPUNames));
 }
 
-ArrayRef<Builtin::Info> BPFTargetInfo::getTargetBuiltins() const {
-  return llvm::ArrayRef(BuiltinInfo,
-                        clang::BPF::LastTSBuiltin - Builtin::FirstTSBuiltin);
+llvm::SmallVector<Builtin::InfosShard>
+BPFTargetInfo::getTargetBuiltins() const {
+  return {{&BuiltinStrings, BuiltinInfos}};
 }
 
 bool BPFTargetInfo::handleTargetFeatures(std::vector<std::string> &Features,
diff --git a/clang/lib/Basic/Targets/BPF.h b/clang/lib/Basic/Targets/BPF.h
index 27a4b5f314970..d1f68b842348e 100644
--- a/clang/lib/Basic/Targets/BPF.h
+++ b/clang/lib/Basic/Targets/BPF.h
@@ -58,7 +58,7 @@ class LLVM_LIBRARY_VISIBILITY BPFTargetInfo : public TargetInfo {
   bool handleTargetFeatures(std::vector<std::string> &Features,
                             DiagnosticsEngine &Diags) override;
 
-  ArrayRef<Builtin::Info> getTargetBuiltins() const override;
+  llvm::SmallVector<Builtin::InfosShard> getTargetBuiltins() const override;
 
   std::string_view getClobbers() const override { return ""; }
 
diff --git a/clang/lib/Basic/Targets/CSKY.cpp b/clang/lib/Basic/Targets/CSKY.cpp
index c8bf8b9234d24..e698508a2370c 100644
--- a/clang/lib/Basic/Targets/CSKY.cpp
+++ b/clang/lib/Basic/Targets/CSKY.cpp
@@ -139,10 +139,6 @@ bool CSKYTargetInfo::handleTargetFeatures(std::vector<std::string> &Features,
   return true;
 }
 
-ArrayRef<Builtin::Info> CSKYTargetInfo::getTargetBuiltins() const {
-  return ArrayRef<Builtin::Info>();
-}
-
 ArrayRef<const char *> CSKYTargetInfo::getGCCRegNames() const {
   static const char *const GCCRegNames[] = {
       // Integer registers
diff --git a/clang/lib/Basic/Targets/CSKY.h b/clang/lib/Basic/Targets/CSKY.h
index 94d4eeb9a1fff..ddfbe4794daad 100644
--- a/clang/lib/Basic/Targets/CSKY.h
+++ b/clang/lib/Basic/Targets/CSKY.h
@@ -73,7 +73,9 @@ class LLVM_LIBRARY_VISIBILITY CSKYTargetInfo : public TargetInfo {
 
   unsigned getMinGlobalAlign(uint64_t, bool HasNonWeakDef) const override;
 
-  ArrayRef<Builtin::Info> getTargetBuiltins() const override;
+  llvm::SmallVector<Builtin::InfosShard> getTargetBuiltins() const override {
+    return {};
+  }
 
   BuiltinVaListKind getBuiltinVaListKind() const override {
     return VoidPtrBuiltinVaList;
diff --git a/clang/lib/Basic/Targets/DirectX.h b/clang/lib/Basic/Targets/DirectX.h
index ab22d1281a4df..6e3ddad626341 100644
--- a/clang/lib/Basic/Targets/DirectX.h
+++ b/clang/lib/Basic/Targets/DirectX.h
@@ -42,6 +42,7 @@ static const unsigned DirectXAddrSpaceMap[] = {
     0, // ptr32_uptr
     0, // ptr64
     3, // hlsl_groupshared
+    2, // hlsl_constant
     // Wasm address space values for this target are dummy values,
     // as it is only enabled for Wasm targets.
     20, // wasm_funcref
@@ -72,7 +73,9 @@ class LLVM_LIBRARY_VISIBILITY DirectXTargetInfo : public TargetInfo {
     return Feature == "directx";
   }
 
-  ArrayRef<Builtin::Info> getTargetBuiltins() const override { return {}; }
+  llvm::SmallVector<Builtin::InfosShard> getTargetBuiltins() const override {
+    return {};
+  }
 
   std::string_view getClobbers() const override { return ""; }
 
diff --git a/clang/lib/Basic/Targets/Hexagon.cpp b/clang/lib/Basic/Targets/Hexagon.cpp
index b5e06b679ece7..c19c2e76c511a 100644
--- a/clang/lib/Basic/Targets/Hexagon.cpp
+++ b/clang/lib/Basic/Targets/Hexagon.cpp
@@ -204,15 +204,26 @@ ArrayRef<TargetInfo::GCCRegAlias> HexagonTargetInfo::getGCCRegAliases() const {
   return llvm::ArrayRef(GCCRegAliases);
 }
 
-static constexpr Builtin::Info BuiltinInfo[] = {
-#define BUILTIN(ID, TYPE, ATTRS)                                               \
-  {#ID, TYPE, ATTRS, nullptr, HeaderDesc::NO_HEADER, ALL_LANGUAGES},
-#define LIBBUILTIN(ID, TYPE, ATTRS, HEADER)                                    \
-  {#ID, TYPE, ATTRS, nullptr, HEADER, ALL_LANGUAGES},
-#define TARGET_BUILTIN(ID, TYPE, ATTRS, FEATURE)                               \
-  {#ID, TYPE, ATTRS, FEATURE, HeaderDesc::NO_HEADER, ALL_LANGUAGES},
-#include "clang/Basic/BuiltinsHexagon.def"
+static constexpr int NumBuiltins =
+    clang::Hexagon::LastTSBuiltin - Builtin::FirstTSBuiltin;
+
+#define GET_BUILTIN_STR_TABLE
+#include "clang/Basic/BuiltinsHexagon.inc"
+#undef GET_BUILTIN_STR_TABLE
+
+static constexpr Builtin::Info BuiltinInfos[] = {
+#define GET_BUILTIN_INFOS
+#include "clang/Basic/BuiltinsHexagon.inc"
+#undef GET_BUILTIN_INFOS
+};
+
+static constexpr Builtin::Info PrefixedBuiltinInfos[] = {
+#define GET_BUILTIN_PREFIXED_INFOS
+#include "clang/Basic/BuiltinsHexagon.inc"
+#undef GET_BUILTIN_PREFIXED_INFOS
 };
+static_assert((std::size(BuiltinInfos) + std::size(PrefixedBuiltinInfos)) ==
+              NumBuiltins);
 
 bool HexagonTargetInfo::hasFeature(StringRef Feature) const {
   std::string VS = "hvxv" + HVXVersion;
@@ -271,7 +282,8 @@ void HexagonTargetInfo::fillValidCPUList(
     Values.push_back(Suffix.Name);
 }
 
-ArrayRef<Builtin::Info> HexagonTargetInfo::getTargetBuiltins() const {
-  return llvm::ArrayRef(BuiltinInfo, clang::Hexagon::LastTSBuiltin -
-                                         Builtin::FirstTSBuiltin);
+llvm::SmallVector<Builtin::InfosShard>
+HexagonTargetInfo::getTargetBuiltins() const {
+  return {{&BuiltinStrings, BuiltinInfos},
+          {&BuiltinStrings, PrefixedBuiltinInfos, "__builtin_HEXAGON_"}};
 }
diff --git a/clang/lib/Basic/Targets/Hexagon.h b/clang/lib/Basic/Targets/Hexagon.h
index 7f053ab7e4888..a65663ca09eee 100644
--- a/clang/lib/Basic/Targets/Hexagon.h
+++ b/clang/lib/Basic/Targets/Hexagon.h
@@ -66,7 +66,7 @@ class LLVM_LIBRARY_VISIBILITY HexagonTargetInfo : public TargetInfo {
     BoolWidth = BoolAlign = 8;
   }
 
-  ArrayRef<Builtin::Info> getTargetBuiltins() const override;
+  llvm::SmallVector<Builtin::InfosShard> getTargetBuiltins() const override;
 
   bool validateAsmConstraint(const char *&Name,
                              TargetInfo::ConstraintInfo &Info) const override {
diff --git a/clang/lib/Basic/Targets/Lanai.h b/clang/lib/Basic/Targets/Lanai.h
index f7e439c7c9e1c..e32ef9d7d40da 100644
--- a/clang/lib/Basic/Targets/Lanai.h
+++ b/clang/lib/Basic/Targets/Lanai.h
@@ -78,7 +78,9 @@ class LLVM_LIBRARY_VISIBILITY LanaiTargetInfo : public TargetInfo {
     return TargetInfo::VoidPtrBuiltinVaList;
   }
 
-  ArrayRef<Builtin::Info> getTargetBuiltins() const override { return {}; }
+  llvm::SmallVector<Builtin::InfosShard> getTargetBuiltins() const override {
+    return {};
+  }
 
   bool validateAsmConstraint(const char *&Name,
                              TargetInfo::ConstraintInfo &info) const override {
diff --git a/clang/lib/Basic/Targets/LoongArch.cpp b/clang/lib/Basic/Targets/LoongArch.cpp
index bb0d0b68cfcb0..ca742797d7a3b 100644
--- a/clang/lib/Basic/Targets/LoongArch.cpp
+++ b/clang/lib/Basic/Targets/LoongArch.cpp
@@ -273,13 +273,55 @@ void LoongArchTargetInfo::getTargetDefines(const LangOptions &Opts,
     Builder.defineMacro("__GCC_HAVE_SYNC_COMPARE_AND_SWAP_8");
 }
 
-static constexpr Builtin::Info BuiltinInfo[] = {
-#define BUILTIN(ID, TYPE, ATTRS)                                               \
-  {#ID, TYPE, ATTRS, nullptr, HeaderDesc::NO_HEADER, ALL_LANGUAGES},
-#define TARGET_BUILTIN(ID, TYPE, ATTRS, FEATURE)                               \
-  {#ID, TYPE, ATTRS, FEATURE, HeaderDesc::NO_HEADER, ALL_LANGUAGES},
-#include "clang/Basic/BuiltinsLoongArch.def"
-};
+static constexpr int NumBaseBuiltins =
+    LoongArch::FirstLSXBuiltin - Builtin::FirstTSBuiltin;
+static constexpr int NumLSXBuiltins =
+    LoongArch::FirstLASXBuiltin - LoongArch::FirstLSXBuiltin;
+static constexpr int NumLASXBuiltins =
+    LoongArch::LastTSBuiltin - LoongArch::FirstLASXBuiltin;
+static constexpr int NumBuiltins =
+    LoongArch::LastTSBuiltin - Builtin::FirstTSBuiltin;
+static_assert(NumBuiltins ==
+              (NumBaseBuiltins + NumLSXBuiltins + NumLASXBuiltins));
+
+static constexpr llvm::StringTable BuiltinBaseStrings =
+    CLANG_BUILTIN_STR_TABLE_START
+#define TARGET_BUILTIN CLANG_TARGET_BUILTIN_STR_TABLE
+#include "clang/Basic/BuiltinsLoongArchBase.def"
+#undef TARGET_BUILTIN
+    ;
+
+static constexpr auto BuiltinBaseInfos = Builtin::MakeInfos<NumBaseBuiltins>({
+#define TARGET_BUILTIN CLANG_TARGET_BUILTIN_ENTRY
+#include "clang/Basic/BuiltinsLoongArchBase.def"
+#undef TARGET_BUILTIN
+});
+
+static constexpr llvm::StringTable BuiltinLSXStrings =
+    CLANG_BUILTIN_STR_TABLE_START
+#define TARGET_BUILTIN CLANG_TARGET_BUILTIN_STR_TABLE
+#include "clang/Basic/BuiltinsLoongArchLSX.def"
+#undef TARGET_BUILTIN
+    ;
+
+static constexpr auto BuiltinLSXInfos = Builtin::MakeInfos<NumLSXBuiltins>({
+#define TARGET_BUILTIN CLANG_TARGET_BUILTIN_ENTRY
+#include "clang/Basic/BuiltinsLoongArchLSX.def"
+#undef TARGET_BUILTIN
+});
+
+static constexpr llvm::StringTable BuiltinLASXStrings =
+    CLANG_BUILTIN_STR_TABLE_START
+#define TARGET_BUILTIN CLANG_TARGET_BUILTIN_STR_TABLE
+#include "clang/Basic/BuiltinsLoongArchLASX.def"
+#undef TARGET_BUILTIN
+    ;
+
+static constexpr auto BuiltinLASXInfos = Builtin::MakeInfos<NumLASXBuiltins>({
+#define TARGET_BUILTIN CLANG_TARGET_BUILTIN_ENTRY
+#include "clang/Basic/BuiltinsLoongArchLASX.def"
+#undef TARGET_BUILTIN
+});
 
 bool LoongArchTargetInfo::initFeatureMap(
     llvm::StringMap<bool> &Features, DiagnosticsEngine &Diags, StringRef CPU,
@@ -306,9 +348,13 @@ bool LoongArchTargetInfo::hasFeature(StringRef Feature) const {
       .Default(false);
 }
 
-ArrayRef<Builtin::Info> LoongArchTargetInfo::getTargetBuiltins() const {
-  return llvm::ArrayRef(BuiltinInfo, clang::LoongArch::LastTSBuiltin -
-                                         Builtin::FirstTSBuiltin);
+llvm::SmallVector<Builtin::InfosShard>
+LoongArchTargetInfo::getTargetBuiltins() const {
+  return {
+      {&BuiltinBaseStrings, BuiltinBaseInfos},
+      {&BuiltinLSXStrings, BuiltinLSXInfos},
+      {&BuiltinLASXStrings, BuiltinLASXInfos},
+  };
 }
 
 bool LoongArchTargetInfo::handleTargetFeatures(
diff --git a/clang/lib/Basic/Targets/LoongArch.h b/clang/lib/Basic/Targets/LoongArch.h
index 5c34c84ff8d3e..4c7b53abfef9b 100644
--- a/clang/lib/Basic/Targets/LoongArch.h
+++ b/clang/lib/Basic/Targets/LoongArch.h
@@ -72,7 +72,7 @@ class LLVM_LIBRARY_VISIBILITY LoongArchTargetInfo : public TargetInfo {
   void getTargetDefines(const LangOptions &Opts,
                         MacroBuilder &Builder) const override;
 
-  ArrayRef<Builtin::Info> getTargetBuiltins() const override;
+  llvm::SmallVector<Builtin::InfosShard> getTargetBuiltins() const override;
 
   BuiltinVaListKind getBuiltinVaListKind() const override {
     return TargetInfo::VoidPtrBuiltinVaList;
diff --git a/clang/lib/Basic/Targets/M68k.cpp b/clang/lib/Basic/Targets/M68k.cpp
index b5b29fd867563..e5b7f06829cd9 100644
--- a/clang/lib/Basic/Targets/M68k.cpp
+++ b/clang/lib/Basic/Targets/M68k.cpp
@@ -115,7 +115,8 @@ void M68kTargetInfo::getTargetDefines(const LangOptions &Opts,
     Builder.defineMacro("__HAVE_68881__");
 }
 
-ArrayRef<Builtin::Info> M68kTargetInfo::getTargetBuiltins() const {
+llvm::SmallVector<Builtin::InfosShard>
+M68kTargetInfo::getTargetBuiltins() const {
   // FIXME: Implement.
   return {};
 }
diff --git a/clang/lib/Basic/Targets/M68k.h b/clang/lib/Basic/Targets/M68k.h
index b732add77e034..729d79ff77fbf 100644
--- a/clang/lib/Basic/Targets/M68k.h
+++ b/clang/lib/Basic/Targets/M68k.h
@@ -44,7 +44,7 @@ class LLVM_LIBRARY_VISIBILITY M68kTargetInfo : public TargetInfo {
 
   void getTargetDefines(const LangOptions &Opts,
                         MacroBuilder &Builder) const override;
-  ArrayRef<Builtin::Info> getTargetBuiltins() const override;
+  llvm::SmallVector<Builtin::InfosShard> getTargetBuiltins() const override;
   bool hasFeature(StringRef Feature) const override;
   ArrayRef<const char *> getGCCRegNames() const override;
   ArrayRef<TargetInfo::GCCRegAlias> getGCCRegAliases() const override;
diff --git a/clang/lib/Basic/Targets/MSP430.h b/clang/lib/Basic/Targets/MSP430.h
index 2266ada25c1dd..d7d05f992f4f6 100644
--- a/clang/lib/Basic/Targets/MSP430.h
+++ b/clang/lib/Basic/Targets/MSP430.h
@@ -50,7 +50,7 @@ class LLVM_LIBRARY_VISIBILITY MSP430TargetInfo : public TargetInfo {
   void getTargetDefines(const LangOptions &Opts,
                         MacroBuilder &Builder) const override;
 
-  ArrayRef<Builtin::Info> getTargetBuiltins() const override {
+  llvm::SmallVector<Builtin::InfosShard> getTargetBuiltins() const override {
     // FIXME: Implement.
     return {};
   }
diff --git a/clang/lib/Basic/Targets/Mips.cpp b/clang/lib/Basic/Targets/Mips.cpp
index d56995e3ccc48..866be53c8a363 100644
--- a/clang/lib/Basic/Targets/Mips.cpp
+++ b/clang/lib/Basic/Targets/Mips.cpp
@@ -20,13 +20,20 @@
 using namespace clang;
 using namespace clang::targets;
 
-static constexpr Builtin::Info BuiltinInfo[] = {
-#define BUILTIN(ID, TYPE, ATTRS)                                               \
-  {#ID, TYPE, ATTRS, nullptr, HeaderDesc::NO_HEADER, ALL_LANGUAGES},
-#define LIBBUILTIN(ID, TYPE, ATTRS, HEADER)                                    \
-  {#ID, TYPE, ATTRS, nullptr, HeaderDesc::HEADER, ALL_LANGUAGES},
+static constexpr int NumBuiltins =
+    clang::Mips::LastTSBuiltin - Builtin::FirstTSBuiltin;
+
+static constexpr llvm::StringTable BuiltinStrings =
+    CLANG_BUILTIN_STR_TABLE_START
+#define BUILTIN CLANG_BUILTIN_STR_TABLE
+#include "clang/Basic/BuiltinsMips.def"
+    ;
+
+static constexpr auto BuiltinInfos = Builtin::MakeInfos<NumBuiltins>({
+#define BUILTIN CLANG_BUILTIN_ENTRY
+#define LIBBUILTIN CLANG_LIBBUILTIN_ENTRY
 #include "clang/Basic/BuiltinsMips.def"
-};
+});
 
 bool MipsTargetInfo::processorSupportsGPR64() const {
   return llvm::StringSwitch<bool>(CPU)
@@ -223,9 +230,9 @@ bool MipsTargetInfo::hasFeature(StringRef Feature) const {
       .Default(false);
 }
 
-ArrayRef<Builtin::Info> MipsTargetInfo::getTargetBuiltins() const {
-  return llvm::ArrayRef(BuiltinInfo,
-                        clang::Mips::LastTSBuiltin - Builtin::FirstTSBuiltin);
+llvm::SmallVector<Builtin::InfosShard>
+MipsTargetInfo::getTargetBuiltins() const {
+  return {{&BuiltinStrings, BuiltinInfos}};
 }
 
 unsigned MipsTargetInfo::getUnwindWordWidth() const {
diff --git a/clang/lib/Basic/Targets/Mips.h b/clang/lib/Basic/Targets/Mips.h
index 7ddcd57053cb2..35501ed44ccd7 100644
--- a/clang/lib/Basic/Targets/Mips.h
+++ b/clang/lib/Basic/Targets/Mips.h
@@ -198,7 +198,7 @@ class LLVM_LIBRARY_VISIBILITY MipsTargetInfo : public TargetInfo {
   void getTargetDefines(const LangOptions &Opts,
                         MacroBuilder &Builder) const override;
 
-  ArrayRef<Builtin::Info> getTargetBuiltins() const override;
+  llvm::SmallVector<Builtin::InfosShard> getTargetBuiltins() const override;
 
   bool hasFeature(StringRef Feature) const override;
 
diff --git a/clang/lib/Basic/Targets/NVPTX.cpp b/clang/lib/Basic/Targets/NVPTX.cpp
index 56efad90cb7c8..7d13c1f145440 100644
--- a/clang/lib/Basic/Targets/NVPTX.cpp
+++ b/clang/lib/Basic/Targets/NVPTX.cpp
@@ -20,15 +20,19 @@
 using namespace clang;
 using namespace clang::targets;
 
-static constexpr Builtin::Info BuiltinInfo[] = {
-#define BUILTIN(ID, TYPE, ATTRS)                                               \
-  {#ID, TYPE, ATTRS, nullptr, HeaderDesc::NO_HEADER, ALL_LANGUAGES},
-#define LIBBUILTIN(ID, TYPE, ATTRS, HEADER)                                    \
-  {#ID, TYPE, ATTRS, nullptr, HeaderDesc::HEADER, ALL_LANGUAGES},
-#define TARGET_BUILTIN(ID, TYPE, ATTRS, FEATURE)                               \
-  {#ID, TYPE, ATTRS, FEATURE, HeaderDesc::NO_HEADER, ALL_LANGUAGES},
-#include "clang/Basic/BuiltinsNVPTX.def"
+static constexpr int NumBuiltins =
+    clang::NVPTX::LastTSBuiltin - Builtin::FirstTSBuiltin;
+
+#define GET_BUILTIN_STR_TABLE
+#include "clang/Basic/BuiltinsNVPTX.inc"
+#undef GET_BUILTIN_STR_TABLE
+
+static constexpr Builtin::Info BuiltinInfos[] = {
+#define GET_BUILTIN_INFOS
+#include "clang/Basic/BuiltinsNVPTX.inc"
+#undef GET_BUILTIN_INFOS
 };
+static_assert(std::size(BuiltinInfos) == NumBuiltins);
 
 const char *const NVPTXTargetInfo::GCCRegNames[] = {"r0"};
 
@@ -66,12 +70,13 @@ NVPTXTargetInfo::NVPTXTargetInfo(const llvm::Triple &Triple,
   HasFloat16 = true;
 
   if (TargetPointerWidth == 32)
-    resetDataLayout("e-p:32:32-i64:64-i128:128-v16:16-v32:32-n16:32:64");
-  else if (Opts.NVPTXUseShortPointers)
     resetDataLayout(
-        "e-p3:32:32-p4:32:32-p5:32:32-i64:64-i128:128-v16:16-v32:32-n16:32:64");
+        "e-p:32:32-p6:32:32-i64:64-i128:128-v16:16-v32:32-n16:32:64");
+  else if (Opts.NVPTXUseShortPointers)
+    resetDataLayout("e-p3:32:32-p4:32:32-p5:32:32-p6:32:32-i64:64-i128:128-v16:"
+                    "16-v32:32-n16:32:64");
   else
-    resetDataLayout("e-i64:64-i128:128-v16:16-v32:32-n16:32:64");
+    resetDataLayout("e-p6:32:32-i64:64-i128:128-v16:16-v32:32-n16:32:64");
 
   // If possible, get a TargetInfo for our host triple, so we can match its
   // types.
@@ -298,7 +303,7 @@ void NVPTXTargetInfo::getTargetDefines(const LangOptions &Opts,
   }
 }
 
-ArrayRef<Builtin::Info> NVPTXTargetInfo::getTargetBuiltins() const {
-  return llvm::ArrayRef(BuiltinInfo,
-                        clang::NVPTX::LastTSBuiltin - Builtin::FirstTSBuiltin);
+llvm::SmallVector<Builtin::InfosShard>
+NVPTXTargetInfo::getTargetBuiltins() const {
+  return {{&BuiltinStrings, BuiltinInfos}};
 }
diff --git a/clang/lib/Basic/Targets/NVPTX.h b/clang/lib/Basic/Targets/NVPTX.h
index d81b89a7f24ac..6a868c42e1265 100644
--- a/clang/lib/Basic/Targets/NVPTX.h
+++ b/clang/lib/Basic/Targets/NVPTX.h
@@ -46,6 +46,7 @@ static const unsigned NVPTXAddrSpaceMap[] = {
     0, // ptr32_uptr
     0, // ptr64
     0, // hlsl_groupshared
+    0, // hlsl_constant
     // Wasm address space values for this target are dummy values,
     // as it is only enabled for Wasm targets.
     20, // wasm_funcref
@@ -74,7 +75,7 @@ class LLVM_LIBRARY_VISIBILITY NVPTXTargetInfo : public TargetInfo {
   void getTargetDefines(const LangOptions &Opts,
                         MacroBuilder &Builder) const override;
 
-  ArrayRef<Builtin::Info> getTargetBuiltins() const override;
+  llvm::SmallVector<Builtin::InfosShard> getTargetBuiltins() const override;
 
   bool useFP16ConversionIntrinsics() const override { return false; }
 
diff --git a/clang/lib/Basic/Targets/OSTargets.cpp b/clang/lib/Basic/Targets/OSTargets.cpp
index bf10f9a725567..8af6623e5cb15 100644
--- a/clang/lib/Basic/Targets/OSTargets.cpp
+++ b/clang/lib/Basic/Targets/OSTargets.cpp
@@ -259,8 +259,10 @@ static void addVisualCDefines(const LangOptions &Opts, MacroBuilder &Builder) {
     Builder.defineMacro("_KERNEL_MODE");
 
   Builder.defineMacro("_INTEGRAL_MAX_BITS", "64");
-  Builder.defineMacro("__STDC_NO_THREADS__");
-
+  // Define __STDC_NO_THREADS__ based on MSVC version, threads.h availability,
+  // and language standard.
+  if (!(Opts.isCompatibleWithMSVC(LangOptions::MSVC2022_9) && Opts.C11))
+    Builder.defineMacro("__STDC_NO_THREADS__");
   // Starting with VS 2022 17.1, MSVC predefines the below macro to inform
   // users of the execution character set defined at compile time.
   // The value given is the Windows Code Page Identifier:
diff --git a/clang/lib/Basic/Targets/PNaCl.h b/clang/lib/Basic/Targets/PNaCl.h
index 7e0e10aa362d8..d162776b5a0d6 100644
--- a/clang/lib/Basic/Targets/PNaCl.h
+++ b/clang/lib/Basic/Targets/PNaCl.h
@@ -52,7 +52,9 @@ class LLVM_LIBRARY_VISIBILITY PNaClTargetInfo : public TargetInfo {
     return Feature == "pnacl";
   }
 
-  ArrayRef<Builtin::Info> getTargetBuiltins() const override { return {}; }
+  llvm::SmallVector<Builtin::InfosShard> getTargetBuiltins() const override {
+    return {};
+  }
 
   BuiltinVaListKind getBuiltinVaListKind() const override {
     return TargetInfo::PNaClABIBuiltinVaList;
diff --git a/clang/lib/Basic/Targets/PPC.cpp b/clang/lib/Basic/Targets/PPC.cpp
index 1448069173b5f..2d8891a739ca3 100644
--- a/clang/lib/Basic/Targets/PPC.cpp
+++ b/clang/lib/Basic/Targets/PPC.cpp
@@ -19,15 +19,22 @@
 using namespace clang;
 using namespace clang::targets;
 
-static constexpr Builtin::Info BuiltinInfo[] = {
-#define BUILTIN(ID, TYPE, ATTRS)                                               \
-  {#ID, TYPE, ATTRS, nullptr, HeaderDesc::NO_HEADER, ALL_LANGUAGES},
-#define TARGET_BUILTIN(ID, TYPE, ATTRS, FEATURE)                               \
-  {#ID, TYPE, ATTRS, FEATURE, HeaderDesc::NO_HEADER, ALL_LANGUAGES},
-#define LIBBUILTIN(ID, TYPE, ATTRS, HEADER)                                    \
-  {#ID, TYPE, ATTRS, nullptr, HeaderDesc::HEADER, ALL_LANGUAGES},
+static constexpr int NumBuiltins =
+    clang::PPC::LastTSBuiltin - Builtin::FirstTSBuiltin;
+
+static constexpr llvm::StringTable BuiltinStrings =
+    CLANG_BUILTIN_STR_TABLE_START
+#define BUILTIN CLANG_BUILTIN_STR_TABLE
+#define TARGET_BUILTIN CLANG_TARGET_BUILTIN_STR_TABLE
 #include "clang/Basic/BuiltinsPPC.def"
-};
+    ;
+
+static constexpr auto BuiltinInfos = Builtin::MakeInfos<NumBuiltins>({
+#define BUILTIN CLANG_BUILTIN_ENTRY
+#define TARGET_BUILTIN CLANG_TARGET_BUILTIN_ENTRY
+#define LIBBUILTIN CLANG_LIBBUILTIN_ENTRY
+#include "clang/Basic/BuiltinsPPC.def"
+});
 
 /// handleTargetFeatures - Perform initialization based on the user
 /// configured set of features.
@@ -927,9 +934,9 @@ void PPCTargetInfo::adjust(DiagnosticsEngine &Diags, LangOptions &Opts) {
     MaxAtomicInlineWidth = 128;
 }
 
-ArrayRef<Builtin::Info> PPCTargetInfo::getTargetBuiltins() const {
-  return llvm::ArrayRef(BuiltinInfo,
-                        clang::PPC::LastTSBuiltin - Builtin::FirstTSBuiltin);
+llvm::SmallVector<Builtin::InfosShard>
+PPCTargetInfo::getTargetBuiltins() const {
+  return {{&BuiltinStrings, BuiltinInfos}};
 }
 
 bool PPCTargetInfo::validateCpuSupports(StringRef FeatureStr) const {
diff --git a/clang/lib/Basic/Targets/PPC.h b/clang/lib/Basic/Targets/PPC.h
index 3cd0fcad17293..db6ac6f0bd338 100644
--- a/clang/lib/Basic/Targets/PPC.h
+++ b/clang/lib/Basic/Targets/PPC.h
@@ -187,7 +187,7 @@ class LLVM_LIBRARY_VISIBILITY PPCTargetInfo : public TargetInfo {
 
   StringRef getABI() const override { return ABI; }
 
-  ArrayRef<Builtin::Info> getTargetBuiltins() const override;
+  llvm::SmallVector<Builtin::InfosShard> getTargetBuiltins() const override;
 
   bool isCLZForZeroUndef() const override { return false; }
 
diff --git a/clang/lib/Basic/Targets/RISCV.cpp b/clang/lib/Basic/Targets/RISCV.cpp
index 8167d7603b0e1..b4aa3206fcfab 100644
--- a/clang/lib/Basic/Targets/RISCV.cpp
+++ b/clang/lib/Basic/Targets/RISCV.cpp
@@ -222,7 +222,7 @@ void RISCVTargetInfo::getTargetDefines(const LangOptions &Opts,
   // Currently we support the v1.0 RISC-V V intrinsics.
   Builder.defineMacro("__riscv_v_intrinsic", Twine(getVersionValue(1, 0)));
 
-  auto VScale = getVScaleRange(Opts);
+  auto VScale = getVScaleRange(Opts, false);
   if (VScale && VScale->first && VScale->first == VScale->second)
     Builder.defineMacro("__riscv_v_fixed_vlen",
                         Twine(VScale->first * llvm::RISCV::RVVBitsPerBlock));
@@ -240,22 +240,61 @@ void RISCVTargetInfo::getTargetDefines(const LangOptions &Opts,
   }
 }
 
-static constexpr Builtin::Info BuiltinInfo[] = {
-#define BUILTIN(ID, TYPE, ATTRS)                                               \
-  {#ID, TYPE, ATTRS, nullptr, HeaderDesc::NO_HEADER, ALL_LANGUAGES},
-#define TARGET_BUILTIN(ID, TYPE, ATTRS, FEATURE)                               \
-  {#ID, TYPE, ATTRS, FEATURE, HeaderDesc::NO_HEADER, ALL_LANGUAGES},
-#include "clang/Basic/BuiltinsRISCVVector.def"
-#define BUILTIN(ID, TYPE, ATTRS)                                               \
-  {#ID, TYPE, ATTRS, nullptr, HeaderDesc::NO_HEADER, ALL_LANGUAGES},
-#define TARGET_BUILTIN(ID, TYPE, ATTRS, FEATURE)                               \
-  {#ID, TYPE, ATTRS, FEATURE, HeaderDesc::NO_HEADER, ALL_LANGUAGES},
-#include "clang/Basic/BuiltinsRISCV.inc"
+static constexpr int NumRVVBuiltins =
+    RISCVVector::FirstSiFiveBuiltin - Builtin::FirstTSBuiltin;
+static constexpr int NumRVVSiFiveBuiltins =
+    RISCVVector::FirstTSBuiltin - RISCVVector::FirstSiFiveBuiltin;
+static constexpr int NumRISCVBuiltins =
+    RISCV::LastTSBuiltin - RISCVVector::FirstTSBuiltin;
+static constexpr int NumBuiltins =
+    RISCV::LastTSBuiltin - Builtin::FirstTSBuiltin;
+static_assert(NumBuiltins ==
+              (NumRVVBuiltins + NumRVVSiFiveBuiltins + NumRISCVBuiltins));
+
+namespace RVV {
+#define GET_RISCVV_BUILTIN_STR_TABLE
+#include "clang/Basic/riscv_vector_builtins.inc"
+#undef GET_RISCVV_BUILTIN_STR_TABLE
+static_assert(BuiltinStrings.size() < 100'000);
+
+static constexpr std::array<Builtin::Info, NumRVVBuiltins> BuiltinInfos = {
+#define GET_RISCVV_BUILTIN_INFOS
+#include "clang/Basic/riscv_vector_builtins.inc"
+#undef GET_RISCVV_BUILTIN_INFOS
+};
+} // namespace RVV
+
+namespace RVVSiFive {
+#define GET_RISCVV_BUILTIN_STR_TABLE
+#include "clang/Basic/riscv_sifive_vector_builtins.inc"
+#undef GET_RISCVV_BUILTIN_STR_TABLE
+
+static constexpr std::array<Builtin::Info, NumRVVSiFiveBuiltins> BuiltinInfos =
+    {
+#define GET_RISCVV_BUILTIN_INFOS
+#include "clang/Basic/riscv_sifive_vector_builtins.inc"
+#undef GET_RISCVV_BUILTIN_INFOS
 };
+} // namespace RVVSiFive
+
+#define GET_BUILTIN_STR_TABLE
+#include "clang/Basic/BuiltinsRISCV.inc"
+#undef GET_BUILTIN_STR_TABLE
 
-ArrayRef<Builtin::Info> RISCVTargetInfo::getTargetBuiltins() const {
-  return llvm::ArrayRef(BuiltinInfo,
-                        clang::RISCV::LastTSBuiltin - Builtin::FirstTSBuiltin);
+static constexpr Builtin::Info BuiltinInfos[] = {
+#define GET_BUILTIN_INFOS
+#include "clang/Basic/BuiltinsRISCV.inc"
+#undef GET_BUILTIN_INFOS
+};
+static_assert(std::size(BuiltinInfos) == NumRISCVBuiltins);
+
+llvm::SmallVector<Builtin::InfosShard>
+RISCVTargetInfo::getTargetBuiltins() const {
+  return {
+      {&RVV::BuiltinStrings, RVV::BuiltinInfos, "__builtin_rvv_"},
+      {&RVVSiFive::BuiltinStrings, RVVSiFive::BuiltinInfos, "__builtin_rvv_"},
+      {&BuiltinStrings, BuiltinInfos},
+  };
 }
 
 bool RISCVTargetInfo::initFeatureMap(
@@ -289,7 +328,8 @@ bool RISCVTargetInfo::initFeatureMap(
 }
 
 std::optional<std::pair<unsigned, unsigned>>
-RISCVTargetInfo::getVScaleRange(const LangOptions &LangOpts) const {
+RISCVTargetInfo::getVScaleRange(const LangOptions &LangOpts,
+                                bool IsArmStreamingFunction) const {
   // RISCV::RVVBitsPerBlock is 64.
   unsigned VScaleMin = ISAInfo->getMinVLen() / llvm::RISCV::RVVBitsPerBlock;
 
diff --git a/clang/lib/Basic/Targets/RISCV.h b/clang/lib/Basic/Targets/RISCV.h
index bb3f3a5cda7c6..c26aa19080162 100644
--- a/clang/lib/Basic/Targets/RISCV.h
+++ b/clang/lib/Basic/Targets/RISCV.h
@@ -62,7 +62,7 @@ class RISCVTargetInfo : public TargetInfo {
   void getTargetDefines(const LangOptions &Opts,
                         MacroBuilder &Builder) const override;
 
-  ArrayRef<Builtin::Info> getTargetBuiltins() const override;
+  llvm::SmallVector<Builtin::InfosShard> getTargetBuiltins() const override;
 
   BuiltinVaListKind getBuiltinVaListKind() const override {
     return TargetInfo::VoidPtrBuiltinVaList;
@@ -99,7 +99,8 @@ class RISCVTargetInfo : public TargetInfo {
                  const std::vector<std::string> &FeaturesVec) const override;
 
   std::optional<std::pair<unsigned, unsigned>>
-  getVScaleRange(const LangOptions &LangOpts) const override;
+  getVScaleRange(const LangOptions &LangOpts,
+                 bool IsArmStreamingFunction) const override;
 
   bool hasFeature(StringRef Feature) const override;
 
diff --git a/clang/lib/Basic/Targets/SPIR.cpp b/clang/lib/Basic/Targets/SPIR.cpp
index f242fedc1ad66..5c076f694dfa4 100644
--- a/clang/lib/Basic/Targets/SPIR.cpp
+++ b/clang/lib/Basic/Targets/SPIR.cpp
@@ -20,15 +20,23 @@
 using namespace clang;
 using namespace clang::targets;
 
-static constexpr Builtin::Info BuiltinInfo[] = {
-#define BUILTIN(ID, TYPE, ATTRS)                                               \
-  {#ID, TYPE, ATTRS, nullptr, HeaderDesc::NO_HEADER, ALL_LANGUAGES},
+static constexpr int NumBuiltins =
+    clang::SPIRV::LastTSBuiltin - Builtin::FirstTSBuiltin;
+
+#define GET_BUILTIN_STR_TABLE
+#include "clang/Basic/BuiltinsSPIRV.inc"
+#undef GET_BUILTIN_STR_TABLE
+
+static constexpr Builtin::Info BuiltinInfos[] = {
+#define GET_BUILTIN_INFOS
 #include "clang/Basic/BuiltinsSPIRV.inc"
+#undef GET_BUILTIN_INFOS
 };
+static_assert(std::size(BuiltinInfos) == NumBuiltins);
 
-ArrayRef<Builtin::Info> SPIRVTargetInfo::getTargetBuiltins() const {
-  return llvm::ArrayRef(BuiltinInfo,
-                        clang::SPIRV::LastTSBuiltin - Builtin::FirstTSBuiltin);
+llvm::SmallVector<Builtin::InfosShard>
+SPIRVTargetInfo::getTargetBuiltins() const {
+  return {{&BuiltinStrings, BuiltinInfos}};
 }
 
 void SPIRTargetInfo::getTargetDefines(const LangOptions &Opts,
@@ -94,7 +102,8 @@ SPIRV64AMDGCNTargetInfo::convertConstraint(const char *&Constraint) const {
   return AMDGPUTI.convertConstraint(Constraint);
 }
 
-ArrayRef<Builtin::Info> SPIRV64AMDGCNTargetInfo::getTargetBuiltins() const {
+llvm::SmallVector<Builtin::InfosShard>
+SPIRV64AMDGCNTargetInfo::getTargetBuiltins() const {
   return AMDGPUTI.getTargetBuiltins();
 }
 
diff --git a/clang/lib/Basic/Targets/SPIR.h b/clang/lib/Basic/Targets/SPIR.h
index 5a328b9ceeb1d..61f9ef7e3e361 100644
--- a/clang/lib/Basic/Targets/SPIR.h
+++ b/clang/lib/Basic/Targets/SPIR.h
@@ -47,6 +47,7 @@ static const unsigned SPIRDefIsPrivMap[] = {
     0, // ptr32_uptr
     0, // ptr64
     0, // hlsl_groupshared
+    2, // hlsl_constant
     // Wasm address space values for this target are dummy values,
     // as it is only enabled for Wasm targets.
     20, // wasm_funcref
@@ -80,6 +81,7 @@ static const unsigned SPIRDefIsGenMap[] = {
     0, // ptr32_uptr
     0, // ptr64
     0, // hlsl_groupshared
+    0, // hlsl_constant
     // Wasm address space values for this target are dummy values,
     // as it is only enabled for Wasm targets.
     20, // wasm_funcref
@@ -159,7 +161,9 @@ class LLVM_LIBRARY_VISIBILITY BaseSPIRTargetInfo : public TargetInfo {
   // memcpy as per section 3 of the SPIR spec.
   bool useFP16ConversionIntrinsics() const override { return false; }
 
-  ArrayRef<Builtin::Info> getTargetBuiltins() const override { return {}; }
+  llvm::SmallVector<Builtin::InfosShard> getTargetBuiltins() const override {
+    return {};
+  }
 
   std::string_view getClobbers() const override { return ""; }
 
@@ -313,7 +317,9 @@ class LLVM_LIBRARY_VISIBILITY SPIRVTargetInfo : public BaseSPIRVTargetInfo {
     resetDataLayout("e-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-"
                     "v256:256-v512:512-v1024:1024-n8:16:32:64-G1");
   }
-  ArrayRef<Builtin::Info> getTargetBuiltins() const override;
+
+  llvm::SmallVector<Builtin::InfosShard> getTargetBuiltins() const override;
+
   void getTargetDefines(const LangOptions &Opts,
                         MacroBuilder &Builder) const override;
 };
@@ -408,7 +414,7 @@ class LLVM_LIBRARY_VISIBILITY SPIRV64AMDGCNTargetInfo final
 
   std::string convertConstraint(const char *&Constraint) const override;
 
-  ArrayRef<Builtin::Info> getTargetBuiltins() const override;
+  llvm::SmallVector<Builtin::InfosShard> getTargetBuiltins() const override;
 
   void getTargetDefines(const LangOptions &Opts,
                         MacroBuilder &Builder) const override;
diff --git a/clang/lib/Basic/Targets/Sparc.h b/clang/lib/Basic/Targets/Sparc.h
index 9c529a5bc5e7f..3215e648ba6c3 100644
--- a/clang/lib/Basic/Targets/Sparc.h
+++ b/clang/lib/Basic/Targets/Sparc.h
@@ -48,7 +48,7 @@ class LLVM_LIBRARY_VISIBILITY SparcTargetInfo : public TargetInfo {
 
   bool hasFeature(StringRef Feature) const override;
 
-  ArrayRef<Builtin::Info> getTargetBuiltins() const override {
+  llvm::SmallVector<Builtin::InfosShard> getTargetBuiltins() const override {
     // FIXME: Implement!
     return {};
   }
diff --git a/clang/lib/Basic/Targets/SystemZ.cpp b/clang/lib/Basic/Targets/SystemZ.cpp
index c836d110d26d5..0e9a61c6a01c8 100644
--- a/clang/lib/Basic/Targets/SystemZ.cpp
+++ b/clang/lib/Basic/Targets/SystemZ.cpp
@@ -20,13 +20,21 @@
 using namespace clang;
 using namespace clang::targets;
 
-static constexpr Builtin::Info BuiltinInfo[] = {
-#define BUILTIN(ID, TYPE, ATTRS)                                               \
-  {#ID, TYPE, ATTRS, nullptr, HeaderDesc::NO_HEADER, ALL_LANGUAGES},
-#define TARGET_BUILTIN(ID, TYPE, ATTRS, FEATURE)                               \
-  {#ID, TYPE, ATTRS, FEATURE, HeaderDesc::NO_HEADER, ALL_LANGUAGES},
+static constexpr int NumBuiltins =
+    clang::SystemZ::LastTSBuiltin - Builtin::FirstTSBuiltin;
+
+static constexpr llvm::StringTable BuiltinStrings =
+    CLANG_BUILTIN_STR_TABLE_START
+#define BUILTIN CLANG_BUILTIN_STR_TABLE
+#define TARGET_BUILTIN CLANG_TARGET_BUILTIN_STR_TABLE
 #include "clang/Basic/BuiltinsSystemZ.def"
-};
+    ;
+
+static constexpr auto BuiltinInfos = Builtin::MakeInfos<NumBuiltins>({
+#define BUILTIN CLANG_BUILTIN_ENTRY
+#define TARGET_BUILTIN CLANG_TARGET_BUILTIN_ENTRY
+#include "clang/Basic/BuiltinsSystemZ.def"
+});
 
 const char *const SystemZTargetInfo::GCCRegNames[] = {
     "r0",  "r1",  "r2",  "r3",  "r4",  "r5",  "r6",  "r7",
@@ -172,7 +180,7 @@ void SystemZTargetInfo::getTargetDefines(const LangOptions &Opts,
     Builder.defineMacro("__VEC__", "10305");
 }
 
-ArrayRef<Builtin::Info> SystemZTargetInfo::getTargetBuiltins() const {
-  return llvm::ArrayRef(BuiltinInfo, clang::SystemZ::LastTSBuiltin -
-                                         Builtin::FirstTSBuiltin);
+llvm::SmallVector<Builtin::InfosShard>
+SystemZTargetInfo::getTargetBuiltins() const {
+  return {{&BuiltinStrings, BuiltinInfos}};
 }
diff --git a/clang/lib/Basic/Targets/SystemZ.h b/clang/lib/Basic/Targets/SystemZ.h
index d05948586c467..4ca3f53f83cba 100644
--- a/clang/lib/Basic/Targets/SystemZ.h
+++ b/clang/lib/Basic/Targets/SystemZ.h
@@ -42,6 +42,7 @@ static const unsigned ZOSAddressMap[] = {
     1, // ptr32_uptr
     0, // ptr64
     0, // hlsl_groupshared
+    0, // hlsl_constant
     0  // wasm_funcref
 };
 
@@ -99,7 +100,7 @@ class LLVM_LIBRARY_VISIBILITY SystemZTargetInfo : public TargetInfo {
   void getTargetDefines(const LangOptions &Opts,
                         MacroBuilder &Builder) const override;
 
-  ArrayRef<Builtin::Info> getTargetBuiltins() const override;
+  llvm::SmallVector<Builtin::InfosShard> getTargetBuiltins() const override;
 
   ArrayRef<const char *> getGCCRegNames() const override;
 
diff --git a/clang/lib/Basic/Targets/TCE.h b/clang/lib/Basic/Targets/TCE.h
index d6280b02f07b2..46c70de8f9ec1 100644
--- a/clang/lib/Basic/Targets/TCE.h
+++ b/clang/lib/Basic/Targets/TCE.h
@@ -51,6 +51,7 @@ static const unsigned TCEOpenCLAddrSpaceMap[] = {
     0, // ptr32_uptr
     0, // ptr64
     0, // hlsl_groupshared
+    0, // hlsl_constant
     // Wasm address space values for this target are dummy values,
     // as it is only enabled for Wasm targets.
     20, // wasm_funcref
@@ -95,7 +96,9 @@ class LLVM_LIBRARY_VISIBILITY TCETargetInfo : public TargetInfo {
 
   bool hasFeature(StringRef Feature) const override { return Feature == "tce"; }
 
-  ArrayRef<Builtin::Info> getTargetBuiltins() const override { return {}; }
+  llvm::SmallVector<Builtin::InfosShard> getTargetBuiltins() const override {
+    return {};
+  }
 
   std::string_view getClobbers() const override { return ""; }
 
diff --git a/clang/lib/Basic/Targets/VE.cpp b/clang/lib/Basic/Targets/VE.cpp
index 67cae8faf6052..5451f3c303637 100644
--- a/clang/lib/Basic/Targets/VE.cpp
+++ b/clang/lib/Basic/Targets/VE.cpp
@@ -18,11 +18,19 @@
 using namespace clang;
 using namespace clang::targets;
 
-static constexpr Builtin::Info BuiltinInfo[] = {
-#define BUILTIN(ID, TYPE, ATTRS)                                               \
-  {#ID, TYPE, ATTRS, nullptr, HeaderDesc::NO_HEADER, ALL_LANGUAGES},
+static constexpr int NumBuiltins =
+    clang::VE::LastTSBuiltin - Builtin::FirstTSBuiltin;
+
+static constexpr llvm::StringTable BuiltinStrings =
+    CLANG_BUILTIN_STR_TABLE_START
+#define BUILTIN CLANG_BUILTIN_STR_TABLE
+#include "clang/Basic/BuiltinsVE.def"
+    ;
+
+static constexpr auto BuiltinInfos = Builtin::MakeInfos<NumBuiltins>({
+#define BUILTIN CLANG_BUILTIN_ENTRY
 #include "clang/Basic/BuiltinsVE.def"
-};
+});
 
 void VETargetInfo::getTargetDefines(const LangOptions &Opts,
                                     MacroBuilder &Builder) const {
@@ -39,7 +47,6 @@ void VETargetInfo::getTargetDefines(const LangOptions &Opts,
   Builder.defineMacro("__GCC_HAVE_SYNC_COMPARE_AND_SWAP_8");
 }
 
-ArrayRef<Builtin::Info> VETargetInfo::getTargetBuiltins() const {
-  return llvm::ArrayRef(BuiltinInfo,
-                        clang::VE::LastTSBuiltin - Builtin::FirstTSBuiltin);
+llvm::SmallVector<Builtin::InfosShard> VETargetInfo::getTargetBuiltins() const {
+  return {{&BuiltinStrings, BuiltinInfos}};
 }
diff --git a/clang/lib/Basic/Targets/VE.h b/clang/lib/Basic/Targets/VE.h
index 7e8fdf6096ef2..e9b7e92f3f850 100644
--- a/clang/lib/Basic/Targets/VE.h
+++ b/clang/lib/Basic/Targets/VE.h
@@ -55,7 +55,7 @@ class LLVM_LIBRARY_VISIBILITY VETargetInfo : public TargetInfo {
 
   bool hasSjLjLowering() const override { return true; }
 
-  ArrayRef<Builtin::Info> getTargetBuiltins() const override;
+  llvm::SmallVector<Builtin::InfosShard> getTargetBuiltins() const override;
 
   BuiltinVaListKind getBuiltinVaListKind() const override {
     return TargetInfo::VoidPtrBuiltinVaList;
diff --git a/clang/lib/Basic/Targets/WebAssembly.cpp b/clang/lib/Basic/Targets/WebAssembly.cpp
index 7b0fd0c841ba2..f19c57f1a3a50 100644
--- a/clang/lib/Basic/Targets/WebAssembly.cpp
+++ b/clang/lib/Basic/Targets/WebAssembly.cpp
@@ -20,15 +20,22 @@
 using namespace clang;
 using namespace clang::targets;
 
-static constexpr Builtin::Info BuiltinInfo[] = {
-#define BUILTIN(ID, TYPE, ATTRS)                                               \
-  {#ID, TYPE, ATTRS, nullptr, HeaderDesc::NO_HEADER, ALL_LANGUAGES},
-#define TARGET_BUILTIN(ID, TYPE, ATTRS, FEATURE)                               \
-  {#ID, TYPE, ATTRS, FEATURE, HeaderDesc::NO_HEADER, ALL_LANGUAGES},
-#define LIBBUILTIN(ID, TYPE, ATTRS, HEADER)                                    \
-  {#ID, TYPE, ATTRS, nullptr, HeaderDesc::HEADER, ALL_LANGUAGES},
+static constexpr int NumBuiltins =
+    clang::WebAssembly::LastTSBuiltin - Builtin::FirstTSBuiltin;
+
+static constexpr llvm::StringTable BuiltinStrings =
+    CLANG_BUILTIN_STR_TABLE_START
+#define BUILTIN CLANG_BUILTIN_STR_TABLE
+#define TARGET_BUILTIN CLANG_TARGET_BUILTIN_STR_TABLE
+#include "clang/Basic/BuiltinsWebAssembly.def"
+    ;
+
+static constexpr auto BuiltinInfos = Builtin::MakeInfos<NumBuiltins>({
+#define BUILTIN CLANG_BUILTIN_ENTRY
+#define TARGET_BUILTIN CLANG_TARGET_BUILTIN_ENTRY
+#define LIBBUILTIN CLANG_LIBBUILTIN_ENTRY
 #include "clang/Basic/BuiltinsWebAssembly.def"
-};
+});
 
 static constexpr llvm::StringLiteral ValidCPUNames[] = {
     {"mvp"}, {"bleeding-edge"}, {"generic"}, {"lime1"}};
@@ -360,9 +367,9 @@ bool WebAssemblyTargetInfo::handleTargetFeatures(
   return true;
 }
 
-ArrayRef<Builtin::Info> WebAssemblyTargetInfo::getTargetBuiltins() const {
-  return llvm::ArrayRef(BuiltinInfo, clang::WebAssembly::LastTSBuiltin -
-                                         Builtin::FirstTSBuiltin);
+llvm::SmallVector<Builtin::InfosShard>
+WebAssemblyTargetInfo::getTargetBuiltins() const {
+  return {{&BuiltinStrings, BuiltinInfos}};
 }
 
 void WebAssemblyTargetInfo::adjust(DiagnosticsEngine &Diags,
diff --git a/clang/lib/Basic/Targets/WebAssembly.h b/clang/lib/Basic/Targets/WebAssembly.h
index 0a14da6a277b8..fb48c786a7edb 100644
--- a/clang/lib/Basic/Targets/WebAssembly.h
+++ b/clang/lib/Basic/Targets/WebAssembly.h
@@ -42,6 +42,7 @@ static const unsigned WebAssemblyAddrSpaceMap[] = {
     0,  // ptr32_uptr
     0,  // ptr64
     0,  // hlsl_groupshared
+    0,  // hlsl_constant
     20, // wasm_funcref
 };
 
@@ -120,7 +121,7 @@ class LLVM_LIBRARY_VISIBILITY WebAssemblyTargetInfo : public TargetInfo {
 
   bool setCPU(const std::string &Name) final { return isValidCPUName(Name); }
 
-  ArrayRef<Builtin::Info> getTargetBuiltins() const final;
+  llvm::SmallVector<Builtin::InfosShard> getTargetBuiltins() const final;
 
   BuiltinVaListKind getBuiltinVaListKind() const final {
     return VoidPtrBuiltinVaList;
diff --git a/clang/lib/Basic/Targets/X86.cpp b/clang/lib/Basic/Targets/X86.cpp
index 40ad8fd9a0967..84a05cec04e7f 100644
--- a/clang/lib/Basic/Targets/X86.cpp
+++ b/clang/lib/Basic/Targets/X86.cpp
@@ -23,23 +23,53 @@
 namespace clang {
 namespace targets {
 
-static constexpr Builtin::Info BuiltinInfoX86[] = {
-#define BUILTIN(ID, TYPE, ATTRS)                                               \
-  {#ID, TYPE, ATTRS, nullptr, HeaderDesc::NO_HEADER, ALL_LANGUAGES},
-#define TARGET_BUILTIN(ID, TYPE, ATTRS, FEATURE)                               \
-  {#ID, TYPE, ATTRS, FEATURE, HeaderDesc::NO_HEADER, ALL_LANGUAGES},
-#define TARGET_HEADER_BUILTIN(ID, TYPE, ATTRS, HEADER, LANGS, FEATURE)         \
-  {#ID, TYPE, ATTRS, FEATURE, HeaderDesc::HEADER, LANGS},
+// The x86-32 builtins are a subset and prefix of the x86-64 builtins.
+static constexpr int NumX86Builtins =
+    X86::LastX86CommonBuiltin - Builtin::FirstTSBuiltin + 1;
+static constexpr int NumX86_64Builtins =
+    X86::LastTSBuiltin - X86::FirstX86_64Builtin;
+static constexpr int NumBuiltins = X86::LastTSBuiltin - Builtin::FirstTSBuiltin;
+static_assert(NumBuiltins == (NumX86Builtins + NumX86_64Builtins));
+
+namespace X86 {
+#define GET_BUILTIN_STR_TABLE
 #include "clang/Basic/BuiltinsX86.inc"
+#undef GET_BUILTIN_STR_TABLE
 
-#define BUILTIN(ID, TYPE, ATTRS)                                               \
-  {#ID, TYPE, ATTRS, nullptr, HeaderDesc::NO_HEADER, ALL_LANGUAGES},
-#define TARGET_BUILTIN(ID, TYPE, ATTRS, FEATURE)                               \
-  {#ID, TYPE, ATTRS, FEATURE, HeaderDesc::NO_HEADER, ALL_LANGUAGES},
-#define TARGET_HEADER_BUILTIN(ID, TYPE, ATTRS, HEADER, LANGS, FEATURE)         \
-  {#ID, TYPE, ATTRS, FEATURE, HeaderDesc::HEADER, LANGS},
+static constexpr Builtin::Info BuiltinInfos[] = {
+#define GET_BUILTIN_INFOS
+#include "clang/Basic/BuiltinsX86.inc"
+#undef GET_BUILTIN_INFOS
+};
+
+static constexpr Builtin::Info PrefixedBuiltinInfos[] = {
+#define GET_BUILTIN_PREFIXED_INFOS
+#include "clang/Basic/BuiltinsX86.inc"
+#undef GET_BUILTIN_PREFIXED_INFOS
+};
+static_assert((std::size(BuiltinInfos) + std::size(PrefixedBuiltinInfos)) ==
+              NumX86Builtins);
+} // namespace X86
+
+namespace X86_64 {
+#define GET_BUILTIN_STR_TABLE
 #include "clang/Basic/BuiltinsX86_64.inc"
+#undef GET_BUILTIN_STR_TABLE
+
+static constexpr Builtin::Info BuiltinInfos[] = {
+#define GET_BUILTIN_INFOS
+#include "clang/Basic/BuiltinsX86_64.inc"
+#undef GET_BUILTIN_INFOS
+};
+
+static constexpr Builtin::Info PrefixedBuiltinInfos[] = {
+#define GET_BUILTIN_PREFIXED_INFOS
+#include "clang/Basic/BuiltinsX86_64.inc"
+#undef GET_BUILTIN_PREFIXED_INFOS
 };
+static_assert((std::size(BuiltinInfos) + std::size(PrefixedBuiltinInfos)) ==
+              NumX86_64Builtins);
+} // namespace X86_64
 
 static const char *const GCCRegNames[] = {
     "ax",    "dx",    "cx",    "bx",    "si",      "di",    "bp",    "sp",
@@ -1856,12 +1886,21 @@ ArrayRef<TargetInfo::AddlRegName> X86TargetInfo::getGCCAddlRegNames() const {
   return llvm::ArrayRef(AddlRegNames);
 }
 
-ArrayRef<Builtin::Info> X86_32TargetInfo::getTargetBuiltins() const {
-  return llvm::ArrayRef(BuiltinInfoX86, clang::X86::LastX86CommonBuiltin -
-                                            Builtin::FirstTSBuiltin + 1);
+llvm::SmallVector<Builtin::InfosShard>
+X86_32TargetInfo::getTargetBuiltins() const {
+  return {
+      {&X86::BuiltinStrings, X86::BuiltinInfos},
+      {&X86::BuiltinStrings, X86::PrefixedBuiltinInfos, "__builtin_ia32_"},
+  };
 }
 
-ArrayRef<Builtin::Info> X86_64TargetInfo::getTargetBuiltins() const {
-  return llvm::ArrayRef(BuiltinInfoX86,
-                        X86::LastTSBuiltin - Builtin::FirstTSBuiltin);
+llvm::SmallVector<Builtin::InfosShard>
+X86_64TargetInfo::getTargetBuiltins() const {
+  return {
+      {&X86::BuiltinStrings, X86::BuiltinInfos},
+      {&X86::BuiltinStrings, X86::PrefixedBuiltinInfos, "__builtin_ia32_"},
+      {&X86_64::BuiltinStrings, X86_64::BuiltinInfos},
+      {&X86_64::BuiltinStrings, X86_64::PrefixedBuiltinInfos,
+       "__builtin_ia32_"},
+  };
 }
diff --git a/clang/lib/Basic/Targets/X86.h b/clang/lib/Basic/Targets/X86.h
index 2c200e64a3d84..205edcab9ccb3 100644
--- a/clang/lib/Basic/Targets/X86.h
+++ b/clang/lib/Basic/Targets/X86.h
@@ -46,6 +46,7 @@ static const unsigned X86AddrSpaceMap[] = {
     271, // ptr32_uptr
     272, // ptr64
     0,   // hlsl_groupshared
+    0,   // hlsl_constant
     // Wasm address space values for this target are dummy values,
     // as it is only enabled for Wasm targets.
     20, // wasm_funcref
@@ -508,7 +509,7 @@ class LLVM_LIBRARY_VISIBILITY X86_32TargetInfo : public X86TargetInfo {
       MaxAtomicInlineWidth = 64;
   }
 
-  ArrayRef<Builtin::Info> getTargetBuiltins() const override;
+  llvm::SmallVector<Builtin::InfosShard> getTargetBuiltins() const override;
 
   bool hasBitIntType() const override { return true; }
   size_t getMaxBitIntWidth() const override {
@@ -820,7 +821,7 @@ class LLVM_LIBRARY_VISIBILITY X86_64TargetInfo : public X86TargetInfo {
       MaxAtomicInlineWidth = 128;
   }
 
-  ArrayRef<Builtin::Info> getTargetBuiltins() const override;
+  llvm::SmallVector<Builtin::InfosShard> getTargetBuiltins() const override;
 
   bool hasBitIntType() const override { return true; }
   size_t getMaxBitIntWidth() const override {
diff --git a/clang/lib/Basic/Targets/XCore.cpp b/clang/lib/Basic/Targets/XCore.cpp
index fd377bbfb90e1..c725703ede5b0 100644
--- a/clang/lib/Basic/Targets/XCore.cpp
+++ b/clang/lib/Basic/Targets/XCore.cpp
@@ -18,13 +18,20 @@
 using namespace clang;
 using namespace clang::targets;
 
-static constexpr Builtin::Info BuiltinInfo[] = {
-#define BUILTIN(ID, TYPE, ATTRS)                                               \
-  {#ID, TYPE, ATTRS, nullptr, HeaderDesc::NO_HEADER, ALL_LANGUAGES},
-#define LIBBUILTIN(ID, TYPE, ATTRS, HEADER)                                    \
-  {#ID, TYPE, ATTRS, nullptr, HeaderDesc::HEADER, ALL_LANGUAGES},
+static constexpr int NumBuiltins =
+    XCore::LastTSBuiltin - Builtin::FirstTSBuiltin;
+
+static constexpr llvm::StringTable BuiltinStrings =
+    CLANG_BUILTIN_STR_TABLE_START
+#define BUILTIN CLANG_BUILTIN_STR_TABLE
+#include "clang/Basic/BuiltinsXCore.def"
+    ;
+
+static constexpr auto BuiltinInfos = Builtin::MakeInfos<NumBuiltins>({
+#define BUILTIN CLANG_BUILTIN_ENTRY
+#define LIBBUILTIN CLANG_LIBBUILTIN_ENTRY
 #include "clang/Basic/BuiltinsXCore.def"
-};
+});
 
 void XCoreTargetInfo::getTargetDefines(const LangOptions &Opts,
                                        MacroBuilder &Builder) const {
@@ -32,7 +39,7 @@ void XCoreTargetInfo::getTargetDefines(const LangOptions &Opts,
   Builder.defineMacro("__XS1B__");
 }
 
-ArrayRef<Builtin::Info> XCoreTargetInfo::getTargetBuiltins() const {
-  return llvm::ArrayRef(BuiltinInfo,
-                        clang::XCore::LastTSBuiltin - Builtin::FirstTSBuiltin);
+llvm::SmallVector<Builtin::InfosShard>
+XCoreTargetInfo::getTargetBuiltins() const {
+  return {{&BuiltinStrings, BuiltinInfos}};
 }
diff --git a/clang/lib/Basic/Targets/XCore.h b/clang/lib/Basic/Targets/XCore.h
index 84fd59d1a71e4..9af9e0658f629 100644
--- a/clang/lib/Basic/Targets/XCore.h
+++ b/clang/lib/Basic/Targets/XCore.h
@@ -43,7 +43,7 @@ class LLVM_LIBRARY_VISIBILITY XCoreTargetInfo : public TargetInfo {
   void getTargetDefines(const LangOptions &Opts,
                         MacroBuilder &Builder) const override;
 
-  ArrayRef<Builtin::Info> getTargetBuiltins() const override;
+  llvm::SmallVector<Builtin::InfosShard> getTargetBuiltins() const override;
 
   BuiltinVaListKind getBuiltinVaListKind() const override {
     return TargetInfo::VoidPtrBuiltinVaList;
diff --git a/clang/lib/Basic/Targets/Xtensa.h b/clang/lib/Basic/Targets/Xtensa.h
index a440ba8aa3c6d..470835aacff52 100644
--- a/clang/lib/Basic/Targets/Xtensa.h
+++ b/clang/lib/Basic/Targets/Xtensa.h
@@ -56,8 +56,8 @@ class LLVM_LIBRARY_VISIBILITY XtensaTargetInfo : public TargetInfo {
   void getTargetDefines(const LangOptions &Opts,
                         MacroBuilder &Builder) const override;
 
-  ArrayRef<Builtin::Info> getTargetBuiltins() const override {
-    return std::nullopt;
+  llvm::SmallVector<Builtin::InfosShard> getTargetBuiltins() const override {
+    return {};
   }
 
   BuiltinVaListKind getBuiltinVaListKind() const override {
diff --git a/clang/lib/Basic/Warnings.cpp b/clang/lib/Basic/Warnings.cpp
index da0304463007b..5f48e0ec81554 100644
--- a/clang/lib/Basic/Warnings.cpp
+++ b/clang/lib/Basic/Warnings.cpp
@@ -73,16 +73,6 @@ void clang::ProcessWarningOptions(DiagnosticsEngine &Diags,
   else
     Diags.setExtensionHandlingBehavior(diag::Severity::Ignored);
 
-  if (!Opts.DiagnosticSuppressionMappingsFile.empty()) {
-    if (auto FileContents =
-            VFS.getBufferForFile(Opts.DiagnosticSuppressionMappingsFile)) {
-      Diags.setDiagSuppressionMapping(**FileContents);
-    } else if (ReportDiags) {
-      Diags.Report(diag::err_drv_no_such_file)
-          << Opts.DiagnosticSuppressionMappingsFile;
-    }
-  }
-
   SmallVector<diag::kind, 10> _Diags;
   const IntrusiveRefCntPtr< DiagnosticIDs > DiagIDs =
     Diags.getDiagnosticIDs();
@@ -237,4 +227,17 @@ void clang::ProcessWarningOptions(DiagnosticsEngine &Diags,
       }
     }
   }
+
+  // Process suppression mappings file after processing other warning flags
+  // (like -Wno-unknown-warning-option) as we can emit extra warnings during
+  // processing.
+  if (!Opts.DiagnosticSuppressionMappingsFile.empty()) {
+    if (auto FileContents =
+            VFS.getBufferForFile(Opts.DiagnosticSuppressionMappingsFile)) {
+      Diags.setDiagSuppressionMapping(**FileContents);
+    } else if (ReportDiags) {
+      Diags.Report(diag::err_drv_no_such_file)
+          << Opts.DiagnosticSuppressionMappingsFile;
+    }
+  }
 }
diff --git a/clang/lib/CIR/CMakeLists.txt b/clang/lib/CIR/CMakeLists.txt
index f3ef8525e15c2..4a99ecb33dfb2 100644
--- a/clang/lib/CIR/CMakeLists.txt
+++ b/clang/lib/CIR/CMakeLists.txt
@@ -5,3 +5,4 @@ add_subdirectory(Dialect)
 add_subdirectory(CodeGen)
 add_subdirectory(FrontendAction)
 add_subdirectory(Interfaces)
+add_subdirectory(Lowering)
diff --git a/clang/lib/CIR/FrontendAction/.clang-tidy b/clang/lib/CIR/FrontendAction/.clang-tidy
new file mode 100644
index 0000000000000..cfb5bdb4bd1fe
--- /dev/null
+++ b/clang/lib/CIR/FrontendAction/.clang-tidy
@@ -0,0 +1,16 @@
+InheritParentConfig: true
+CheckOptions:
+  - key:             readability-identifier-naming.ClassCase
+    value:           CamelCase
+  - key:             readability-identifier-naming.EnumCase
+    value:           CamelCase
+  - key:             readability-identifier-naming.FunctionCase
+    value:           camelBack
+  - key:             readability-identifier-naming.MemberCase
+    value:           CamelCase
+  - key:             readability-identifier-naming.ParameterCase
+    value:           CamelCase
+  - key:             readability-identifier-naming.UnionCase
+    value:           CamelCase
+  - key:             readability-identifier-naming.VariableCase
+    value:           CamelCase
diff --git a/clang/lib/CIR/FrontendAction/CIRGenAction.cpp b/clang/lib/CIR/FrontendAction/CIRGenAction.cpp
index 21b6bc56ed050..eab6958ac8f6d 100644
--- a/clang/lib/CIR/FrontendAction/CIRGenAction.cpp
+++ b/clang/lib/CIR/FrontendAction/CIRGenAction.cpp
@@ -7,23 +7,47 @@
 //===----------------------------------------------------------------------===//
 
 #include "clang/CIR/FrontendAction/CIRGenAction.h"
-#include "clang/CIR/CIRGenerator.h"
-#include "clang/Frontend/CompilerInstance.h"
-
 #include "mlir/IR/MLIRContext.h"
 #include "mlir/IR/OwningOpRef.h"
+#include "clang/CIR/CIRGenerator.h"
+#include "clang/CIR/LowerToLLVM.h"
+#include "clang/CodeGen/BackendUtil.h"
+#include "clang/Frontend/CompilerInstance.h"
+#include "llvm/IR/Module.h"
 
 using namespace cir;
 using namespace clang;
 
 namespace cir {
 
+static BackendAction
+getBackendActionFromOutputType(CIRGenAction::OutputType Action) {
+  switch (Action) {
+  case CIRGenAction::OutputType::EmitCIR:
+    assert(false &&
+           "Unsupported output type for getBackendActionFromOutputType!");
+    break; // Unreachable, but fall through to report that
+  case CIRGenAction::OutputType::EmitLLVM:
+    return BackendAction::Backend_EmitLL;
+  }
+  // We should only get here if a non-enum value is passed in or we went through
+  // the assert(false) case above
+  llvm_unreachable("Unsupported output type!");
+}
+
+static std::unique_ptr<llvm::Module>
+lowerFromCIRToLLVMIR(mlir::ModuleOp MLIRModule, llvm::LLVMContext &LLVMCtx) {
+  return direct::lowerDirectlyFromCIRToLLVMIR(MLIRModule, LLVMCtx);
+}
+
 class CIRGenConsumer : public clang::ASTConsumer {
 
   virtual void anchor();
 
   CIRGenAction::OutputType Action;
 
+  CompilerInstance &CI;
+
   std::unique_ptr<raw_pwrite_stream> OutputStream;
 
   ASTContext *Context{nullptr};
@@ -31,18 +55,12 @@ class CIRGenConsumer : public clang::ASTConsumer {
   std::unique_ptr<CIRGenerator> Gen;
 
 public:
-  CIRGenConsumer(CIRGenAction::OutputType Action,
-                 DiagnosticsEngine &DiagnosticsEngine,
-                 IntrusiveRefCntPtr<llvm::vfs::FileSystem> VFS,
-                 const HeaderSearchOptions &HeaderSearchOptions,
-                 const CodeGenOptions &CodeGenOptions,
-                 const TargetOptions &TargetOptions,
-                 const LangOptions &LangOptions,
-                 const FrontendOptions &FEOptions,
+  CIRGenConsumer(CIRGenAction::OutputType Action, CompilerInstance &CI,
                  std::unique_ptr<raw_pwrite_stream> OS)
-      : Action(Action), OutputStream(std::move(OS)), FS(VFS),
-        Gen(std::make_unique<CIRGenerator>(DiagnosticsEngine, std::move(VFS),
-                                           CodeGenOptions)) {}
+      : Action(Action), CI(CI), OutputStream(std::move(OS)),
+        FS(&CI.getVirtualFileSystem()),
+        Gen(std::make_unique<CIRGenerator>(CI.getDiagnostics(), std::move(FS),
+                                           CI.getCodeGenOpts())) {}
 
   void Initialize(ASTContext &Ctx) override {
     assert(!Context && "initialized multiple times");
@@ -66,6 +84,17 @@ class CIRGenConsumer : public clang::ASTConsumer {
         MlirModule->print(*OutputStream, Flags);
       }
       break;
+    case CIRGenAction::OutputType::EmitLLVM: {
+      llvm::LLVMContext LLVMCtx;
+      std::unique_ptr<llvm::Module> LLVMModule =
+          lowerFromCIRToLLVMIR(MlirModule, LLVMCtx);
+
+      BackendAction BEAction = getBackendActionFromOutputType(Action);
+      emitBackendOutput(
+          CI, CI.getCodeGenOpts(), C.getTargetInfo().getDataLayoutString(),
+          LLVMModule.get(), BEAction, FS, std::move(OutputStream));
+      break;
+    }
     }
   }
 };
@@ -84,6 +113,8 @@ getOutputStream(CompilerInstance &CI, StringRef InFile,
   switch (Action) {
   case CIRGenAction::OutputType::EmitCIR:
     return CI.createDefaultOutputFile(false, InFile, "cir");
+  case CIRGenAction::OutputType::EmitLLVM:
+    return CI.createDefaultOutputFile(false, InFile, "ll");
   }
   llvm_unreachable("Invalid CIRGenAction::OutputType");
 }
@@ -95,10 +126,8 @@ CIRGenAction::CreateASTConsumer(CompilerInstance &CI, StringRef InFile) {
   if (!Out)
     Out = getOutputStream(CI, InFile, Action);
 
-  auto Result = std::make_unique<cir::CIRGenConsumer>(
-      Action, CI.getDiagnostics(), &CI.getVirtualFileSystem(),
-      CI.getHeaderSearchOpts(), CI.getCodeGenOpts(), CI.getTargetOpts(),
-      CI.getLangOpts(), CI.getFrontendOpts(), std::move(Out));
+  auto Result =
+      std::make_unique<cir::CIRGenConsumer>(Action, CI, std::move(Out));
 
   return Result;
 }
@@ -106,3 +135,7 @@ CIRGenAction::CreateASTConsumer(CompilerInstance &CI, StringRef InFile) {
 void EmitCIRAction::anchor() {}
 EmitCIRAction::EmitCIRAction(mlir::MLIRContext *MLIRCtx)
     : CIRGenAction(OutputType::EmitCIR, MLIRCtx) {}
+
+void EmitLLVMAction::anchor() {}
+EmitLLVMAction::EmitLLVMAction(mlir::MLIRContext *MLIRCtx)
+    : CIRGenAction(OutputType::EmitLLVM, MLIRCtx) {}
diff --git a/clang/lib/CIR/FrontendAction/CMakeLists.txt b/clang/lib/CIR/FrontendAction/CMakeLists.txt
index b0616ab5d64b0..ac2b857239d07 100644
--- a/clang/lib/CIR/FrontendAction/CMakeLists.txt
+++ b/clang/lib/CIR/FrontendAction/CMakeLists.txt
@@ -12,6 +12,8 @@ add_clang_library(clangCIRFrontendAction
   clangAST
   clangFrontend
   clangCIR
+  clangCIRLoweringDirectToLLVM
+  clangCodeGen
   MLIRCIR
   MLIRIR
   )
diff --git a/clang/lib/CIR/Lowering/CMakeLists.txt b/clang/lib/CIR/Lowering/CMakeLists.txt
new file mode 100644
index 0000000000000..95c304ded9183
--- /dev/null
+++ b/clang/lib/CIR/Lowering/CMakeLists.txt
@@ -0,0 +1 @@
+add_subdirectory(DirectToLLVM)
diff --git a/clang/lib/CIR/Lowering/DirectToLLVM/CMakeLists.txt b/clang/lib/CIR/Lowering/DirectToLLVM/CMakeLists.txt
new file mode 100644
index 0000000000000..3f74c79249a27
--- /dev/null
+++ b/clang/lib/CIR/Lowering/DirectToLLVM/CMakeLists.txt
@@ -0,0 +1,17 @@
+set(LLVM_LINK_COMPONENTS
+  Core
+  Support
+  )
+
+get_property(dialect_libs GLOBAL PROPERTY MLIR_DIALECT_LIBS)
+
+add_clang_library(clangCIRLoweringDirectToLLVM
+  LowerToLLVM.cpp
+
+  LINK_LIBS
+  MLIRIR
+  ${dialect_libs}
+  MLIRCIR
+  MLIRBuiltinToLLVMIRTranslation
+  MLIRLLVMToLLVMIRTranslation
+  )
diff --git a/clang/lib/CIR/Lowering/DirectToLLVM/LowerToLLVM.cpp b/clang/lib/CIR/Lowering/DirectToLLVM/LowerToLLVM.cpp
new file mode 100644
index 0000000000000..d60a6b38b0c12
--- /dev/null
+++ b/clang/lib/CIR/Lowering/DirectToLLVM/LowerToLLVM.cpp
@@ -0,0 +1,336 @@
+//====- LowerToLLVM.cpp - Lowering from CIR to LLVMIR ---------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file implements lowering of CIR operations to LLVMIR.
+//
+//===----------------------------------------------------------------------===//
+
+#include "LowerToLLVM.h"
+
+#include "mlir/Conversion/LLVMCommon/TypeConverter.h"
+#include "mlir/Dialect/DLTI/DLTI.h"
+#include "mlir/Dialect/Func/IR/FuncOps.h"
+#include "mlir/Dialect/LLVMIR/LLVMDialect.h"
+#include "mlir/IR/BuiltinDialect.h"
+#include "mlir/IR/BuiltinOps.h"
+#include "mlir/Pass/Pass.h"
+#include "mlir/Pass/PassManager.h"
+#include "mlir/Target/LLVMIR/Dialect/Builtin/BuiltinToLLVMIRTranslation.h"
+#include "mlir/Target/LLVMIR/Dialect/LLVMIR/LLVMToLLVMIRTranslation.h"
+#include "mlir/Target/LLVMIR/Export.h"
+#include "mlir/Transforms/DialectConversion.h"
+#include "clang/CIR/Dialect/IR/CIRAttrVisitor.h"
+#include "clang/CIR/Dialect/IR/CIRDialect.h"
+#include "clang/CIR/MissingFeatures.h"
+#include "llvm/IR/Module.h"
+#include "llvm/Support/TimeProfiler.h"
+
+using namespace cir;
+using namespace llvm;
+
+namespace cir {
+namespace direct {
+
+class CIRAttrToValue : public CirAttrVisitor<CIRAttrToValue, mlir::Value> {
+public:
+  CIRAttrToValue(mlir::Operation *parentOp,
+                 mlir::ConversionPatternRewriter &rewriter,
+                 const mlir::TypeConverter *converter)
+      : parentOp(parentOp), rewriter(rewriter), converter(converter) {}
+
+  mlir::Value lowerCirAttrAsValue(mlir::Attribute attr) { return visit(attr); }
+
+  mlir::Value visitCirIntAttr(cir::IntAttr intAttr) {
+    mlir::Location loc = parentOp->getLoc();
+    return rewriter.create<mlir::LLVM::ConstantOp>(
+        loc, converter->convertType(intAttr.getType()), intAttr.getValue());
+  }
+
+  mlir::Value visitCirFPAttr(cir::FPAttr fltAttr) {
+    mlir::Location loc = parentOp->getLoc();
+    return rewriter.create<mlir::LLVM::ConstantOp>(
+        loc, converter->convertType(fltAttr.getType()), fltAttr.getValue());
+  }
+
+  mlir::Value visitCirConstPtrAttr(cir::ConstPtrAttr ptrAttr) {
+    mlir::Location loc = parentOp->getLoc();
+    if (ptrAttr.isNullValue()) {
+      return rewriter.create<mlir::LLVM::ZeroOp>(
+          loc, converter->convertType(ptrAttr.getType()));
+    }
+    mlir::DataLayout layout(parentOp->getParentOfType<mlir::ModuleOp>());
+    mlir::Value ptrVal = rewriter.create<mlir::LLVM::ConstantOp>(
+        loc,
+        rewriter.getIntegerType(layout.getTypeSizeInBits(ptrAttr.getType())),
+        ptrAttr.getValue().getInt());
+    return rewriter.create<mlir::LLVM::IntToPtrOp>(
+        loc, converter->convertType(ptrAttr.getType()), ptrVal);
+  }
+
+private:
+  mlir::Operation *parentOp;
+  mlir::ConversionPatternRewriter &rewriter;
+  const mlir::TypeConverter *converter;
+};
+
+// This class handles rewriting initializer attributes for types that do not
+// require region initialization.
+class GlobalInitAttrRewriter
+    : public CirAttrVisitor<GlobalInitAttrRewriter, mlir::Attribute> {
+public:
+  GlobalInitAttrRewriter(mlir::Type type,
+                         mlir::ConversionPatternRewriter &rewriter)
+      : llvmType(type), rewriter(rewriter) {}
+
+  mlir::Attribute rewriteInitAttr(mlir::Attribute attr) { return visit(attr); }
+
+  mlir::Attribute visitCirIntAttr(cir::IntAttr attr) {
+    return rewriter.getIntegerAttr(llvmType, attr.getValue());
+  }
+  mlir::Attribute visitCirFPAttr(cir::FPAttr attr) {
+    return rewriter.getFloatAttr(llvmType, attr.getValue());
+  }
+
+private:
+  mlir::Type llvmType;
+  mlir::ConversionPatternRewriter &rewriter;
+};
+
+// This pass requires the CIR to be in a "flat" state. All blocks in each
+// function must belong to the parent region. Once scopes and control flow
+// are implemented in CIR, a pass will be run before this one to flatten
+// the CIR and get it into the state that this pass requires.
+struct ConvertCIRToLLVMPass
+    : public mlir::PassWrapper<ConvertCIRToLLVMPass,
+                               mlir::OperationPass<mlir::ModuleOp>> {
+  void getDependentDialects(mlir::DialectRegistry &registry) const override {
+    registry.insert<mlir::BuiltinDialect, mlir::DLTIDialect,
+                    mlir::LLVM::LLVMDialect, mlir::func::FuncDialect>();
+  }
+  void runOnOperation() final;
+
+  StringRef getDescription() const override {
+    return "Convert the prepared CIR dialect module to LLVM dialect";
+  }
+
+  StringRef getArgument() const override { return "cir-flat-to-llvm"; }
+};
+
+bool CIRToLLVMGlobalOpLowering::attrRequiresRegionInitialization(
+    mlir::Attribute attr) const {
+  // There will be more cases added later.
+  return isa<cir::ConstPtrAttr>(attr);
+}
+
+/// Replace CIR global with a region initialized LLVM global and update
+/// insertion point to the end of the initializer block.
+void CIRToLLVMGlobalOpLowering::setupRegionInitializedLLVMGlobalOp(
+    cir::GlobalOp op, mlir::ConversionPatternRewriter &rewriter) const {
+  assert(!cir::MissingFeatures::convertTypeForMemory());
+  const mlir::Type llvmType = getTypeConverter()->convertType(op.getSymType());
+
+  // FIXME: These default values are placeholders until the the equivalent
+  //        attributes are available on cir.global ops. This duplicates code
+  //        in CIRToLLVMGlobalOpLowering::matchAndRewrite() but that will go
+  //        away when the placeholders are no longer needed.
+  assert(!cir::MissingFeatures::opGlobalConstant());
+  const bool isConst = false;
+  assert(!cir::MissingFeatures::addressSpace());
+  const unsigned addrSpace = 0;
+  assert(!cir::MissingFeatures::opGlobalDSOLocal());
+  const bool isDsoLocal = true;
+  assert(!cir::MissingFeatures::opGlobalThreadLocal());
+  const bool isThreadLocal = false;
+  assert(!cir::MissingFeatures::opGlobalAlignment());
+  const uint64_t alignment = 0;
+  assert(!cir::MissingFeatures::opGlobalLinkage());
+  const mlir::LLVM::Linkage linkage = mlir::LLVM::Linkage::External;
+  const StringRef symbol = op.getSymName();
+
+  SmallVector<mlir::NamedAttribute> attributes;
+  auto newGlobalOp = rewriter.replaceOpWithNewOp<mlir::LLVM::GlobalOp>(
+      op, llvmType, isConst, linkage, symbol, nullptr, alignment, addrSpace,
+      isDsoLocal, isThreadLocal,
+      /*comdat=*/mlir::SymbolRefAttr(), attributes);
+  newGlobalOp.getRegion().push_back(new mlir::Block());
+  rewriter.setInsertionPointToEnd(newGlobalOp.getInitializerBlock());
+};
+
+mlir::LogicalResult
+CIRToLLVMGlobalOpLowering::matchAndRewriteRegionInitializedGlobal(
+    cir::GlobalOp op, mlir::Attribute init,
+    mlir::ConversionPatternRewriter &rewriter) const {
+  // TODO: Generalize this handling when more types are needed here.
+  assert(isa<cir::ConstPtrAttr>(init));
+
+  // TODO(cir): once LLVM's dialect has proper equivalent attributes this
+  // should be updated. For now, we use a custom op to initialize globals
+  // to the appropriate value.
+  const mlir::Location loc = op.getLoc();
+  setupRegionInitializedLLVMGlobalOp(op, rewriter);
+  CIRAttrToValue attrVisitor(op, rewriter, typeConverter);
+  mlir::Value value = attrVisitor.lowerCirAttrAsValue(init);
+  rewriter.create<mlir::LLVM::ReturnOp>(loc, value);
+  return mlir::success();
+}
+
+mlir::LogicalResult CIRToLLVMGlobalOpLowering::matchAndRewrite(
+    cir::GlobalOp op, OpAdaptor adaptor,
+    mlir::ConversionPatternRewriter &rewriter) const {
+
+  std::optional<mlir::Attribute> init = op.getInitialValue();
+
+  // If we have an initializer and it requires region initialization, handle
+  // that separately
+  if (init.has_value() && attrRequiresRegionInitialization(init.value())) {
+    return matchAndRewriteRegionInitializedGlobal(op, init.value(), rewriter);
+  }
+
+  // Fetch required values to create LLVM op.
+  const mlir::Type cirSymType = op.getSymType();
+
+  // This is the LLVM dialect type.
+  assert(!cir::MissingFeatures::convertTypeForMemory());
+  const mlir::Type llvmType = getTypeConverter()->convertType(cirSymType);
+  // FIXME: These default values are placeholders until the the equivalent
+  //        attributes are available on cir.global ops.
+  assert(!cir::MissingFeatures::opGlobalConstant());
+  const bool isConst = false;
+  assert(!cir::MissingFeatures::addressSpace());
+  const unsigned addrSpace = 0;
+  assert(!cir::MissingFeatures::opGlobalDSOLocal());
+  const bool isDsoLocal = true;
+  assert(!cir::MissingFeatures::opGlobalThreadLocal());
+  const bool isThreadLocal = false;
+  assert(!cir::MissingFeatures::opGlobalAlignment());
+  const uint64_t alignment = 0;
+  assert(!cir::MissingFeatures::opGlobalLinkage());
+  const mlir::LLVM::Linkage linkage = mlir::LLVM::Linkage::External;
+  const StringRef symbol = op.getSymName();
+  SmallVector<mlir::NamedAttribute> attributes;
+
+  if (init.has_value()) {
+    GlobalInitAttrRewriter initRewriter(llvmType, rewriter);
+    init = initRewriter.rewriteInitAttr(init.value());
+    // If initRewriter returned a null attribute, init will have a value but
+    // the value will be null. If that happens, initRewriter didn't handle the
+    // attribute type. It probably needs to be added to GlobalInitAttrRewriter.
+    if (!init.value()) {
+      op.emitError() << "unsupported initializer '" << init.value() << "'";
+      return mlir::failure();
+    }
+  }
+
+  // Rewrite op.
+  rewriter.replaceOpWithNewOp<mlir::LLVM::GlobalOp>(
+      op, llvmType, isConst, linkage, symbol, init.value_or(mlir::Attribute()),
+      alignment, addrSpace, isDsoLocal, isThreadLocal,
+      /*comdat=*/mlir::SymbolRefAttr(), attributes);
+
+  return mlir::success();
+}
+
+static void prepareTypeConverter(mlir::LLVMTypeConverter &converter,
+                                 mlir::DataLayout &dataLayout) {
+  converter.addConversion([&](cir::PointerType type) -> mlir::Type {
+    // Drop pointee type since LLVM dialect only allows opaque pointers.
+    assert(!cir::MissingFeatures::addressSpace());
+    unsigned targetAS = 0;
+
+    return mlir::LLVM::LLVMPointerType::get(type.getContext(), targetAS);
+  });
+  converter.addConversion([&](cir::IntType type) -> mlir::Type {
+    // LLVM doesn't work with signed types, so we drop the CIR signs here.
+    return mlir::IntegerType::get(type.getContext(), type.getWidth());
+  });
+  converter.addConversion([&](cir::SingleType type) -> mlir::Type {
+    return mlir::Float32Type::get(type.getContext());
+  });
+  converter.addConversion([&](cir::DoubleType type) -> mlir::Type {
+    return mlir::Float64Type::get(type.getContext());
+  });
+  converter.addConversion([&](cir::FP80Type type) -> mlir::Type {
+    return mlir::Float80Type::get(type.getContext());
+  });
+  converter.addConversion([&](cir::FP128Type type) -> mlir::Type {
+    return mlir::Float128Type::get(type.getContext());
+  });
+  converter.addConversion([&](cir::LongDoubleType type) -> mlir::Type {
+    return converter.convertType(type.getUnderlying());
+  });
+  converter.addConversion([&](cir::FP16Type type) -> mlir::Type {
+    return mlir::Float16Type::get(type.getContext());
+  });
+  converter.addConversion([&](cir::BF16Type type) -> mlir::Type {
+    return mlir::BFloat16Type::get(type.getContext());
+  });
+}
+
+void ConvertCIRToLLVMPass::runOnOperation() {
+  llvm::TimeTraceScope scope("Convert CIR to LLVM Pass");
+
+  mlir::ModuleOp module = getOperation();
+  mlir::DataLayout dl(module);
+  mlir::LLVMTypeConverter converter(&getContext());
+  prepareTypeConverter(converter, dl);
+
+  mlir::RewritePatternSet patterns(&getContext());
+
+  patterns.add<CIRToLLVMGlobalOpLowering>(converter, patterns.getContext(), dl);
+
+  mlir::ConversionTarget target(getContext());
+  target.addLegalOp<mlir::ModuleOp>();
+  target.addLegalDialect<mlir::LLVM::LLVMDialect>();
+  target.addIllegalDialect<mlir::BuiltinDialect, cir::CIRDialect,
+                           mlir::func::FuncDialect>();
+
+  if (failed(applyPartialConversion(module, target, std::move(patterns))))
+    signalPassFailure();
+}
+
+static std::unique_ptr<mlir::Pass> createConvertCIRToLLVMPass() {
+  return std::make_unique<ConvertCIRToLLVMPass>();
+}
+
+static void populateCIRToLLVMPasses(mlir::OpPassManager &pm) {
+  pm.addPass(createConvertCIRToLLVMPass());
+}
+
+std::unique_ptr<llvm::Module>
+lowerDirectlyFromCIRToLLVMIR(mlir::ModuleOp mlirModule, LLVMContext &llvmCtx) {
+  llvm::TimeTraceScope scope("lower from CIR to LLVM directly");
+
+  mlir::MLIRContext *mlirCtx = mlirModule.getContext();
+
+  mlir::PassManager pm(mlirCtx);
+  populateCIRToLLVMPasses(pm);
+
+  if (mlir::failed(pm.run(mlirModule))) {
+    // FIXME: Handle any errors where they occurs and return a nullptr here.
+    report_fatal_error(
+        "The pass manager failed to lower CIR to LLVMIR dialect!");
+  }
+
+  mlir::registerBuiltinDialectTranslation(*mlirCtx);
+  mlir::registerLLVMDialectTranslation(*mlirCtx);
+
+  llvm::TimeTraceScope translateScope("translateModuleToLLVMIR");
+
+  StringRef moduleName = mlirModule.getName().value_or("CIRToLLVMModule");
+  std::unique_ptr<llvm::Module> llvmModule =
+      mlir::translateModuleToLLVMIR(mlirModule, llvmCtx, moduleName);
+
+  if (!llvmModule) {
+    // FIXME: Handle any errors where they occurs and return a nullptr here.
+    report_fatal_error("Lowering from LLVMIR dialect to llvm IR failed!");
+  }
+
+  return llvmModule;
+}
+} // namespace direct
+} // namespace cir
diff --git a/clang/lib/CIR/Lowering/DirectToLLVM/LowerToLLVM.h b/clang/lib/CIR/Lowering/DirectToLLVM/LowerToLLVM.h
new file mode 100644
index 0000000000000..b3366c1fb9337
--- /dev/null
+++ b/clang/lib/CIR/Lowering/DirectToLLVM/LowerToLLVM.h
@@ -0,0 +1,52 @@
+//====- LowerToLLVM.h- Lowering from CIR to LLVM --------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file declares an interface for converting CIR modules to LLVM IR.
+//
+//===----------------------------------------------------------------------===//
+#ifndef CLANG_CIR_LOWERTOLLVM_H
+#define CLANG_CIR_LOWERTOLLVM_H
+
+#include "mlir/Transforms/DialectConversion.h"
+#include "clang/CIR/Dialect/IR/CIRDialect.h"
+
+namespace cir {
+
+namespace direct {
+
+class CIRToLLVMGlobalOpLowering
+    : public mlir::OpConversionPattern<cir::GlobalOp> {
+  const mlir::DataLayout &dataLayout;
+
+public:
+  CIRToLLVMGlobalOpLowering(const mlir::TypeConverter &typeConverter,
+                            mlir::MLIRContext *context,
+                            const mlir::DataLayout &dataLayout)
+      : OpConversionPattern(typeConverter, context), dataLayout(dataLayout) {
+    setHasBoundedRewriteRecursion();
+  }
+
+  mlir::LogicalResult
+  matchAndRewrite(cir::GlobalOp op, OpAdaptor adaptor,
+                  mlir::ConversionPatternRewriter &rewriter) const override;
+
+private:
+  bool attrRequiresRegionInitialization(mlir::Attribute attr) const;
+
+  mlir::LogicalResult matchAndRewriteRegionInitializedGlobal(
+      cir::GlobalOp op, mlir::Attribute init,
+      mlir::ConversionPatternRewriter &rewriter) const;
+
+  void setupRegionInitializedLLVMGlobalOp(
+      cir::GlobalOp op, mlir::ConversionPatternRewriter &rewriter) const;
+};
+
+} // namespace direct
+} // namespace cir
+
+#endif // CLANG_CIR_LOWERTOLLVM_H
diff --git a/clang/lib/CMakeLists.txt b/clang/lib/CMakeLists.txt
index 14ba55360fe05..4f2218b583e41 100644
--- a/clang/lib/CMakeLists.txt
+++ b/clang/lib/CMakeLists.txt
@@ -12,9 +12,6 @@ add_subdirectory(Analysis)
 add_subdirectory(Edit)
 add_subdirectory(ExtractAPI)
 add_subdirectory(Rewrite)
-if(CLANG_ENABLE_ARCMT)
-  add_subdirectory(ARCMigrate)
-endif()
 add_subdirectory(Driver)
 add_subdirectory(Serialization)
 add_subdirectory(Frontend)
diff --git a/clang/lib/CodeGen/Address.h b/clang/lib/CodeGen/Address.h
index a18c7169af1eb..a748ddaa110a5 100644
--- a/clang/lib/CodeGen/Address.h
+++ b/clang/lib/CodeGen/Address.h
@@ -197,10 +197,7 @@ class Address {
 
   /// Return the type of the pointer value.
   llvm::PointerType *getType() const {
-    return llvm::PointerType::get(
-        ElementType,
-        llvm::cast<llvm::PointerType>(Pointer.getPointer()->getType())
-            ->getAddressSpace());
+    return llvm::cast<llvm::PointerType>(Pointer.getPointer()->getType());
   }
 
   /// Return the type of the values stored in this address.
diff --git a/clang/lib/CodeGen/BackendUtil.cpp b/clang/lib/CodeGen/BackendUtil.cpp
index 3e65eeb3755d2..1750719e17670 100644
--- a/clang/lib/CodeGen/BackendUtil.cpp
+++ b/clang/lib/CodeGen/BackendUtil.cpp
@@ -124,15 +124,25 @@ namespace clang {
 extern llvm::cl::opt<bool> ClSanitizeGuardChecks;
 }
 
-namespace {
-
 // Default filename used for profile generation.
-std::string getDefaultProfileGenName() {
+static std::string getDefaultProfileGenName() {
   return DebugInfoCorrelate || ProfileCorrelate != InstrProfCorrelator::NONE
              ? "default_%m.proflite"
              : "default_%m.profraw";
 }
 
+// Path and name of file used for profile generation
+static std::string getProfileGenName(const CodeGenOptions &CodeGenOpts) {
+  std::string FileName = CodeGenOpts.InstrProfileOutput.empty()
+                             ? getDefaultProfileGenName()
+                             : CodeGenOpts.InstrProfileOutput;
+  if (CodeGenOpts.ContinuousProfileSync)
+    FileName = "%c" + FileName;
+  return FileName;
+}
+
+namespace {
+
 class EmitAssemblyHelper {
   CompilerInstance &CI;
   DiagnosticsEngine &Diags;
@@ -551,7 +561,9 @@ getInstrProfOptions(const CodeGenOptions &CodeGenOpts,
     return std::nullopt;
   InstrProfOptions Options;
   Options.NoRedZone = CodeGenOpts.DisableRedZone;
-  Options.InstrProfileOutput = CodeGenOpts.InstrProfileOutput;
+  Options.InstrProfileOutput = CodeGenOpts.ContinuousProfileSync
+                                   ? ("%c" + CodeGenOpts.InstrProfileOutput)
+                                   : CodeGenOpts.InstrProfileOutput;
   Options.Atomic = CodeGenOpts.AtomicProfileUpdate;
   return Options;
 }
@@ -795,14 +807,23 @@ static void addSanitizers(const Triple &TargetTriple,
     PB.registerOptimizerLastEPCallback(SanitizersCallback);
   }
 
-  if (LowerAllowCheckPass::IsRequested()) {
+  // SanitizeSkipHotCutoffs: doubles with range [0, 1]
+  // Opts.cutoffs: unsigned ints with range [0, 1000000]
+  auto ScaledCutoffs = CodeGenOpts.SanitizeSkipHotCutoffs.getAllScaled(1000000);
+
+  // TODO: remove IsRequested()
+  if (LowerAllowCheckPass::IsRequested() || ScaledCutoffs.has_value()) {
     // We want to call it after inline, which is about OptimizerEarlyEPCallback.
-    PB.registerOptimizerEarlyEPCallback([&](ModulePassManager &MPM,
-                                            OptimizationLevel Level,
-                                            ThinOrFullLTOPhase Phase) {
-      LowerAllowCheckPass::Options Opts;
-      MPM.addPass(createModuleToFunctionPassAdaptor(LowerAllowCheckPass(Opts)));
-    });
+    PB.registerOptimizerEarlyEPCallback(
+        [ScaledCutoffs](ModulePassManager &MPM, OptimizationLevel Level,
+                        ThinOrFullLTOPhase Phase) {
+          LowerAllowCheckPass::Options Opts;
+          // TODO: after removing IsRequested(), make this unconditional
+          if (ScaledCutoffs.has_value())
+            Opts.cutoffs = ScaledCutoffs.value();
+          MPM.addPass(
+              createModuleToFunctionPassAdaptor(LowerAllowCheckPass(Opts)));
+        });
   }
 }
 
@@ -813,13 +834,12 @@ void EmitAssemblyHelper::RunOptimizationPipeline(
 
   if (CodeGenOpts.hasProfileIRInstr())
     // -fprofile-generate.
-    PGOOpt = PGOOptions(
-        CodeGenOpts.InstrProfileOutput.empty() ? getDefaultProfileGenName()
-                                               : CodeGenOpts.InstrProfileOutput,
-        "", "", CodeGenOpts.MemoryProfileUsePath, nullptr, PGOOptions::IRInstr,
-        PGOOptions::NoCSAction, ClPGOColdFuncAttr,
-        CodeGenOpts.DebugInfoForProfiling,
-        /*PseudoProbeForProfiling=*/false, CodeGenOpts.AtomicProfileUpdate);
+    PGOOpt = PGOOptions(getProfileGenName(CodeGenOpts), "", "",
+                        CodeGenOpts.MemoryProfileUsePath, nullptr,
+                        PGOOptions::IRInstr, PGOOptions::NoCSAction,
+                        ClPGOColdFuncAttr, CodeGenOpts.DebugInfoForProfiling,
+                        /*PseudoProbeForProfiling=*/false,
+                        CodeGenOpts.AtomicProfileUpdate);
   else if (CodeGenOpts.hasProfileIRUse()) {
     // -fprofile-use.
     auto CSAction = CodeGenOpts.hasProfileCSIRUse() ? PGOOptions::CSIRUse
@@ -863,24 +883,20 @@ void EmitAssemblyHelper::RunOptimizationPipeline(
              PGOOpt->Action != PGOOptions::SampleUse &&
              "Cannot run CSProfileGen pass with ProfileGen or SampleUse "
              " pass");
-      PGOOpt->CSProfileGenFile = CodeGenOpts.InstrProfileOutput.empty()
-                                     ? getDefaultProfileGenName()
-                                     : CodeGenOpts.InstrProfileOutput;
+      PGOOpt->CSProfileGenFile = getProfileGenName(CodeGenOpts);
       PGOOpt->CSAction = PGOOptions::CSIRInstr;
     } else
-      PGOOpt = PGOOptions("",
-                          CodeGenOpts.InstrProfileOutput.empty()
-                              ? getDefaultProfileGenName()
-                              : CodeGenOpts.InstrProfileOutput,
-                          "", /*MemoryProfile=*/"", nullptr,
-                          PGOOptions::NoAction, PGOOptions::CSIRInstr,
-                          ClPGOColdFuncAttr, CodeGenOpts.DebugInfoForProfiling);
+      PGOOpt = PGOOptions("", getProfileGenName(CodeGenOpts), "",
+                          /*MemoryProfile=*/"", nullptr, PGOOptions::NoAction,
+                          PGOOptions::CSIRInstr, ClPGOColdFuncAttr,
+                          CodeGenOpts.DebugInfoForProfiling);
   }
   if (TM)
     TM->setPGOOption(PGOOpt);
 
   PipelineTuningOptions PTO;
   PTO.LoopUnrolling = CodeGenOpts.UnrollLoops;
+  PTO.LoopInterchange = CodeGenOpts.InterchangeLoops;
   // For historical reasons, loop interleaving is set to mirror setting for loop
   // unrolling.
   PTO.LoopInterleaving = CodeGenOpts.UnrollLoops;
@@ -1305,6 +1321,7 @@ runThinLTOBackend(CompilerInstance &CI, ModuleSummaryIndex *CombinedIndex,
   initTargetOptions(CI, Diags, Conf.Options);
   Conf.SampleProfile = std::move(SampleProfile);
   Conf.PTO.LoopUnrolling = CGOpts.UnrollLoops;
+  Conf.PTO.LoopInterchange = CGOpts.InterchangeLoops;
   // For historical reasons, loop interleaving is set to mirror setting for loop
   // unrolling.
   Conf.PTO.LoopInterleaving = CGOpts.UnrollLoops;
diff --git a/clang/lib/CodeGen/CGBlocks.cpp b/clang/lib/CodeGen/CGBlocks.cpp
index a7584a95c8ca7..faef6a5fbe1f5 100644
--- a/clang/lib/CodeGen/CGBlocks.cpp
+++ b/clang/lib/CodeGen/CGBlocks.cpp
@@ -1097,31 +1097,10 @@ llvm::Type *CodeGenModule::getBlockDescriptorType() {
   if (BlockDescriptorType)
     return BlockDescriptorType;
 
-  llvm::Type *UnsignedLongTy =
-    getTypes().ConvertType(getContext().UnsignedLongTy);
-
-  // struct __block_descriptor {
-  //   unsigned long reserved;
-  //   unsigned long block_size;
-  //
-  //   // later, the following will be added
-  //
-  //   struct {
-  //     void (*copyHelper)();
-  //     void (*copyHelper)();
-  //   } helpers;                // !!! optional
-  //
-  //   const char *signature;   // the block signature
-  //   const char *layout;      // reserved
-  // };
-  BlockDescriptorType = llvm::StructType::create(
-      "struct.__block_descriptor", UnsignedLongTy, UnsignedLongTy);
-
-  // Now form a pointer to that.
   unsigned AddrSpace = 0;
   if (getLangOpts().OpenCL)
     AddrSpace = getContext().getTargetAddressSpace(LangAS::opencl_constant);
-  BlockDescriptorType = llvm::PointerType::get(BlockDescriptorType, AddrSpace);
+  BlockDescriptorType = llvm::PointerType::get(getLLVMContext(), AddrSpace);
   return BlockDescriptorType;
 }
 
@@ -2800,7 +2779,8 @@ static void configureBlocksRuntimeObject(CodeGenModule &CGM,
                                          llvm::Constant *C) {
   auto *GV = cast<llvm::GlobalValue>(C->stripPointerCasts());
 
-  if (CGM.getTarget().getTriple().isOSBinFormatCOFF()) {
+  if (!CGM.getCodeGenOpts().StaticClosure &&
+      CGM.getTarget().getTriple().isOSBinFormatCOFF()) {
     const IdentifierInfo &II = CGM.getContext().Idents.get(C->getName());
     TranslationUnitDecl *TUDecl = CGM.getContext().getTranslationUnitDecl();
     DeclContext *DC = TranslationUnitDecl::castToDeclContext(TUDecl);
@@ -2815,7 +2795,6 @@ static void configureBlocksRuntimeObject(CodeGenModule &CGM,
           (ND = dyn_cast<VarDecl>(Result)))
         break;
 
-    // TODO: support static blocks runtime
     if (GV->isDeclaration() && (!ND || !ND->hasAttr<DLLExportAttr>())) {
       GV->setDLLStorageClass(llvm::GlobalValue::DLLImportStorageClass);
       GV->setLinkage(llvm::GlobalValue::ExternalLinkage);
diff --git a/clang/lib/CodeGen/CGBuiltin.cpp b/clang/lib/CodeGen/CGBuiltin.cpp
index 26bccccdc5e36..0fe8cf5179b53 100644
--- a/clang/lib/CodeGen/CGBuiltin.cpp
+++ b/clang/lib/CodeGen/CGBuiltin.cpp
@@ -29,6 +29,7 @@
 #include "clang/AST/Expr.h"
 #include "clang/AST/OSLog.h"
 #include "clang/AST/OperationKinds.h"
+#include "clang/AST/StmtVisitor.h"
 #include "clang/AST/Type.h"
 #include "clang/Basic/TargetBuiltins.h"
 #include "clang/Basic/TargetInfo.h"
@@ -264,8 +265,10 @@ llvm::Constant *CodeGenModule::getBuiltinLibFunction(const FunctionDecl *FD,
                                                      unsigned BuiltinID) {
   assert(Context.BuiltinInfo.isLibFunction(BuiltinID));
 
-  // Get the name, skip over the __builtin_ prefix (if necessary).
-  StringRef Name;
+  // Get the name, skip over the __builtin_ prefix (if necessary). We may have
+  // to build this up so provide a small stack buffer to handle the vast
+  // majority of names.
+  llvm::SmallString<64> Name;
   GlobalDecl D(FD);
 
   // TODO: This list should be expanded or refactored after all GCC-compatible
@@ -1049,33 +1052,102 @@ CodeGenFunction::evaluateOrEmitBuiltinObjectSize(const Expr *E, unsigned Type,
   return ConstantInt::get(ResType, ObjectSize, /*isSigned=*/true);
 }
 
-const FieldDecl *CodeGenFunction::FindFlexibleArrayMemberFieldAndOffset(
-    ASTContext &Ctx, const RecordDecl *RD, const FieldDecl *FAMDecl,
-    uint64_t &Offset) {
+namespace {
+
+/// StructFieldAccess is a simple visitor class to grab the first MemberExpr
+/// from an Expr. It records any ArraySubscriptExpr we meet along the way.
+class StructFieldAccess
+    : public ConstStmtVisitor<StructFieldAccess, const Expr *> {
+  bool AddrOfSeen = false;
+
+public:
+  const ArraySubscriptExpr *ASE = nullptr;
+
+  const Expr *VisitMemberExpr(const MemberExpr *E) {
+    if (AddrOfSeen && E->getType()->isArrayType())
+      // Avoid forms like '&ptr->array'.
+      return nullptr;
+    return E;
+  }
+
+  const Expr *VisitArraySubscriptExpr(const ArraySubscriptExpr *E) {
+    if (ASE)
+      // We don't support multiple subscripts.
+      return nullptr;
+
+    AddrOfSeen = false; // '&ptr->array[idx]' is okay.
+    ASE = E;
+    return Visit(E->getBase());
+  }
+  const Expr *VisitCastExpr(const CastExpr *E) {
+    if (E->getCastKind() == CK_LValueToRValue)
+      return E;
+    return Visit(E->getSubExpr());
+  }
+  const Expr *VisitParenExpr(const ParenExpr *E) {
+    return Visit(E->getSubExpr());
+  }
+  const Expr *VisitUnaryAddrOf(const clang::UnaryOperator *E) {
+    AddrOfSeen = true;
+    return Visit(E->getSubExpr());
+  }
+  const Expr *VisitUnaryDeref(const clang::UnaryOperator *E) {
+    AddrOfSeen = false;
+    return Visit(E->getSubExpr());
+  }
+};
+
+} // end anonymous namespace
+
+/// Find a struct's flexible array member. It may be embedded inside multiple
+/// sub-structs, but must still be the last field.
+static const FieldDecl *FindFlexibleArrayMemberField(CodeGenFunction &CGF,
+                                                     ASTContext &Ctx,
+                                                     const RecordDecl *RD) {
   const LangOptions::StrictFlexArraysLevelKind StrictFlexArraysLevel =
-      getLangOpts().getStrictFlexArraysLevel();
-  uint32_t FieldNo = 0;
+      CGF.getLangOpts().getStrictFlexArraysLevel();
 
   if (RD->isImplicit())
     return nullptr;
 
   for (const FieldDecl *FD : RD->fields()) {
-    if ((!FAMDecl || FD == FAMDecl) &&
-        Decl::isFlexibleArrayMemberLike(
+    if (Decl::isFlexibleArrayMemberLike(
             Ctx, FD, FD->getType(), StrictFlexArraysLevel,
-            /*IgnoreTemplateOrMacroSubstitution=*/true)) {
-      const ASTRecordLayout &Layout = Ctx.getASTRecordLayout(RD);
-      Offset += Layout.getFieldOffset(FieldNo);
+            /*IgnoreTemplateOrMacroSubstitution=*/true))
       return FD;
+
+    if (auto RT = FD->getType()->getAs<RecordType>())
+      if (const FieldDecl *FD =
+              FindFlexibleArrayMemberField(CGF, Ctx, RT->getAsRecordDecl()))
+        return FD;
+  }
+
+  return nullptr;
+}
+
+/// Calculate the offset of a struct field. It may be embedded inside multiple
+/// sub-structs.
+static bool GetFieldOffset(ASTContext &Ctx, const RecordDecl *RD,
+                           const FieldDecl *FD, int64_t &Offset) {
+  if (RD->isImplicit())
+    return false;
+
+  // Keep track of the field number ourselves, because the other methods
+  // (CGRecordLayout::getLLVMFieldNo) aren't always equivalent to how the AST
+  // is laid out.
+  uint32_t FieldNo = 0;
+  const ASTRecordLayout &Layout = Ctx.getASTRecordLayout(RD);
+
+  for (const FieldDecl *Field : RD->fields()) {
+    if (Field == FD) {
+      Offset += Layout.getFieldOffset(FieldNo);
+      return true;
     }
 
-    QualType Ty = FD->getType();
-    if (Ty->isRecordType()) {
-      if (const FieldDecl *Field = FindFlexibleArrayMemberFieldAndOffset(
-              Ctx, Ty->getAsRecordDecl(), FAMDecl, Offset)) {
-        const ASTRecordLayout &Layout = Ctx.getASTRecordLayout(RD);
+    if (auto RT = Field->getType()->getAs<RecordType>()) {
+      if (GetFieldOffset(Ctx, RT->getAsRecordDecl(), FD, Offset)) {
         Offset += Layout.getFieldOffset(FieldNo);
-        return Field;
+        return true;
       }
     }
 
@@ -1083,202 +1155,255 @@ const FieldDecl *CodeGenFunction::FindFlexibleArrayMemberFieldAndOffset(
       ++FieldNo;
   }
 
-  return nullptr;
+  return false;
 }
 
-static unsigned CountCountedByAttrs(const RecordDecl *RD) {
-  unsigned Num = 0;
-
-  for (const FieldDecl *FD : RD->fields()) {
-    if (FD->getType()->isCountAttributedType())
-      return ++Num;
+static std::optional<int64_t>
+GetFieldOffset(ASTContext &Ctx, const RecordDecl *RD, const FieldDecl *FD) {
+  int64_t Offset = 0;
 
-    QualType Ty = FD->getType();
-    if (Ty->isRecordType())
-      Num += CountCountedByAttrs(Ty->getAsRecordDecl());
-  }
+  if (GetFieldOffset(Ctx, RD, FD, Offset))
+    return std::optional<int64_t>(Offset);
 
-  return Num;
+  return std::nullopt;
 }
 
 llvm::Value *
-CodeGenFunction::emitFlexibleArrayMemberSize(const Expr *E, unsigned Type,
-                                             llvm::IntegerType *ResType) {
-  // The code generated here calculates the size of a struct with a flexible
-  // array member that uses the counted_by attribute. There are two instances
-  // we handle:
-  //
-  //       struct s {
-  //         unsigned long flags;
-  //         int count;
-  //         int array[] __attribute__((counted_by(count)));
-  //       }
-  //
-  //   1) bdos of the flexible array itself:
-  //
-  //     __builtin_dynamic_object_size(p->array, 1) ==
-  //         p->count * sizeof(*p->array)
-  //
-  //   2) bdos of a pointer into the flexible array:
-  //
-  //     __builtin_dynamic_object_size(&p->array[42], 1) ==
-  //         (p->count - 42) * sizeof(*p->array)
+CodeGenFunction::emitCountedByMemberSize(const Expr *E, llvm::Value *EmittedE,
+                                         unsigned Type,
+                                         llvm::IntegerType *ResType) {
+  ASTContext &Ctx = getContext();
+
+  // Note: If the whole struct is specificed in the __bdos (i.e. Visitor
+  // returns a DeclRefExpr). The calculation of the whole size of the structure
+  // with a flexible array member can be done in two ways:
   //
-  //   2) bdos of the whole struct, including the flexible array:
+  //     1) sizeof(struct S) + count * sizeof(typeof(fam))
+  //     2) offsetof(struct S, fam) + count * sizeof(typeof(fam))
   //
-  //     __builtin_dynamic_object_size(p, 1) ==
-  //        max(sizeof(struct s),
-  //            offsetof(struct s, array) + p->count * sizeof(*p->array))
+  // The first will add additional padding after the end of the array
+  // allocation while the second method is more precise, but not quite expected
+  // from programmers. See
+  // https://lore.kernel.org/lkml/ZvV6X5FPBBW7CO1f@archlinux/ for a discussion
+  // of the topic.
   //
-  ASTContext &Ctx = getContext();
-  const Expr *Base = E->IgnoreParenImpCasts();
-  const Expr *Idx = nullptr;
-
-  if (const auto *UO = dyn_cast<UnaryOperator>(Base);
-      UO && UO->getOpcode() == UO_AddrOf) {
-    Expr *SubExpr = UO->getSubExpr()->IgnoreParenImpCasts();
-    if (const auto *ASE = dyn_cast<ArraySubscriptExpr>(SubExpr)) {
-      Base = ASE->getBase()->IgnoreParenImpCasts();
-      Idx = ASE->getIdx()->IgnoreParenImpCasts();
-
-      if (const auto *IL = dyn_cast<IntegerLiteral>(Idx)) {
-        int64_t Val = IL->getValue().getSExtValue();
-        if (Val < 0)
-          return getDefaultBuiltinObjectSizeResult(Type, ResType);
-
-        if (Val == 0)
-          // The index is 0, so we don't need to take it into account.
-          Idx = nullptr;
-      }
-    } else {
-      // Potential pointer to another element in the struct.
-      Base = SubExpr;
-    }
-  }
+  // GCC isn't (currently) able to calculate __bdos on a pointer to the whole
+  // structure. Therefore, because of the above issue, we choose to match what
+  // GCC does for consistency's sake.
 
-  // Get the flexible array member Decl.
-  const RecordDecl *OuterRD = nullptr;
-  const FieldDecl *FAMDecl = nullptr;
-  if (const auto *ME = dyn_cast<MemberExpr>(Base)) {
-    // Check if \p Base is referencing the FAM itself.
-    const ValueDecl *VD = ME->getMemberDecl();
-    OuterRD = VD->getDeclContext()->getOuterLexicalRecordContext();
-    FAMDecl = dyn_cast<FieldDecl>(VD);
-    if (!FAMDecl)
-      return nullptr;
-  } else if (const auto *DRE = dyn_cast<DeclRefExpr>(Base)) {
-    // Check if we're pointing to the whole struct.
-    QualType Ty = DRE->getDecl()->getType();
-    if (Ty->isPointerType())
-      Ty = Ty->getPointeeType();
-    OuterRD = Ty->getAsRecordDecl();
-
-    // If we have a situation like this:
-    //
-    //     struct union_of_fams {
-    //         int flags;
-    //         union {
-    //             signed char normal_field;
-    //             struct {
-    //                 int count1;
-    //                 int arr1[] __counted_by(count1);
-    //             };
-    //             struct {
-    //                 signed char count2;
-    //                 int arr2[] __counted_by(count2);
-    //             };
-    //         };
-    //    };
-    //
-    // We don't know which 'count' to use in this scenario:
-    //
-    //     size_t get_size(struct union_of_fams *p) {
-    //         return __builtin_dynamic_object_size(p, 1);
-    //     }
-    //
-    // Instead of calculating a wrong number, we give up.
-    if (OuterRD && CountCountedByAttrs(OuterRD) > 1)
-      return nullptr;
-  }
+  StructFieldAccess Visitor;
+  const MemberExpr *ME = dyn_cast_if_present<MemberExpr>(Visitor.Visit(E));
+  if (!ME)
+    return nullptr;
 
-  if (!OuterRD)
+  const auto *FD = dyn_cast<FieldDecl>(ME->getMemberDecl());
+  if (!FD)
     return nullptr;
 
-  // We call FindFlexibleArrayMemberAndOffset even if FAMDecl is non-null to
-  // get its offset.
-  uint64_t Offset = 0;
-  FAMDecl =
-      FindFlexibleArrayMemberFieldAndOffset(Ctx, OuterRD, FAMDecl, Offset);
-  Offset = Ctx.toCharUnitsFromBits(Offset).getQuantity();
+  const RecordDecl *RD = FD->getDeclContext()->getOuterLexicalRecordContext();
+  const FieldDecl *FlexibleArrayMemberFD = nullptr;
 
-  if (!FAMDecl || !FAMDecl->getType()->isCountAttributedType())
-    // No flexible array member found or it doesn't have the "counted_by"
-    // attribute.
-    return nullptr;
+  if (Decl::isFlexibleArrayMemberLike(
+          Ctx, FD, FD->getType(), getLangOpts().getStrictFlexArraysLevel(),
+          /*IgnoreTemplateOrMacroSubstitution=*/true))
+    FlexibleArrayMemberFD = FD;
+  else
+    FlexibleArrayMemberFD = FindFlexibleArrayMemberField(*this, Ctx, RD);
 
-  const FieldDecl *CountedByFD = FAMDecl->findCountedByField();
-  if (!CountedByFD)
-    // Can't find the field referenced by the "counted_by" attribute.
+  if (!FlexibleArrayMemberFD ||
+      !FlexibleArrayMemberFD->getType()->isCountAttributedType())
     return nullptr;
 
-  if (isa<DeclRefExpr>(Base))
-    // The whole struct is specificed in the __bdos. The calculation of the
-    // whole size of the structure can be done in two ways:
-    //
-    //     1) sizeof(struct S) + count * sizeof(typeof(fam))
-    //     2) offsetof(struct S, fam) + count * sizeof(typeof(fam))
-    //
-    // The first will add additional padding after the end of the array,
-    // allocation while the second method is more precise, but not quite
-    // expected from programmers. See
-    // https://lore.kernel.org/lkml/ZvV6X5FPBBW7CO1f@archlinux/ for a
-    // discussion of the topic.
-    //
-    // GCC isn't (currently) able to calculate __bdos on a pointer to the whole
-    // structure. Therefore, because of the above issue, we'll choose to match
-    // what GCC does for consistency's sake.
+  const FieldDecl *CountFD = FlexibleArrayMemberFD->findCountedByField();
+  if (!CountFD)
+    // Can't find the field referenced by the "counted_by" attribute.
     return nullptr;
 
-  // Build a load of the counted_by field.
-  bool IsSigned = CountedByFD->getType()->isSignedIntegerType();
-  Value *CountedByInst = EmitLoadOfCountedByField(Base, FAMDecl, CountedByFD);
-  if (!CountedByInst)
-    return getDefaultBuiltinObjectSizeResult(Type, ResType);
-
-  CountedByInst = Builder.CreateIntCast(CountedByInst, ResType, IsSigned);
+  const Expr *Idx = nullptr;
+  if (Visitor.ASE) {
+    Idx = Visitor.ASE->getIdx();
 
-  // Build a load of the index and subtract it from the count.
-  Value *IdxInst = nullptr;
-  if (Idx) {
-    if (Idx->HasSideEffects(getContext()))
+    if (Idx->HasSideEffects(Ctx))
       // We can't have side-effects.
       return getDefaultBuiltinObjectSizeResult(Type, ResType);
 
+    if (const auto *IL = dyn_cast<IntegerLiteral>(Idx)) {
+      int64_t Val = IL->getValue().getSExtValue();
+      if (Val < 0)
+        return getDefaultBuiltinObjectSizeResult(Type, ResType);
+
+      // The index is 0, so we don't need to take it into account.
+      if (Val == 0)
+        Idx = nullptr;
+    }
+  }
+
+  // Calculate the flexible array member's object size using these formulae
+  // (note: if the calculation is negative, we return 0.):
+  //
+  //      struct p;
+  //      struct s {
+  //          /* ... */
+  //          int count;
+  //          struct p *array[] __attribute__((counted_by(count)));
+  //      };
+  //
+  // 1) 'ptr->array':
+  //
+  //    count = ptr->count;
+  //
+  //    flexible_array_member_base_size = sizeof (*ptr->array);
+  //    flexible_array_member_size =
+  //            count * flexible_array_member_base_size;
+  //
+  //    if (flexible_array_member_size < 0)
+  //        return 0;
+  //    return flexible_array_member_size;
+  //
+  // 2) '&ptr->array[idx]':
+  //
+  //    count = ptr->count;
+  //    index = idx;
+  //
+  //    flexible_array_member_base_size = sizeof (*ptr->array);
+  //    flexible_array_member_size =
+  //            count * flexible_array_member_base_size;
+  //
+  //    index_size = index * flexible_array_member_base_size;
+  //
+  //    if (flexible_array_member_size < 0 || index < 0)
+  //        return 0;
+  //    return flexible_array_member_size - index_size;
+  //
+  // 3) '&ptr->field':
+  //
+  //    count = ptr->count;
+  //    sizeof_struct = sizeof (struct s);
+  //
+  //    flexible_array_member_base_size = sizeof (*ptr->array);
+  //    flexible_array_member_size =
+  //            count * flexible_array_member_base_size;
+  //
+  //    field_offset = offsetof (struct s, field);
+  //    offset_diff = sizeof_struct - field_offset;
+  //
+  //    if (flexible_array_member_size < 0)
+  //        return 0;
+  //    return offset_diff + flexible_array_member_size;
+  //
+  // 4) '&ptr->field_array[idx]':
+  //
+  //    count = ptr->count;
+  //    index = idx;
+  //    sizeof_struct = sizeof (struct s);
+  //
+  //    flexible_array_member_base_size = sizeof (*ptr->array);
+  //    flexible_array_member_size =
+  //            count * flexible_array_member_base_size;
+  //
+  //    field_base_size = sizeof (*ptr->field_array);
+  //    field_offset = offsetof (struct s, field)
+  //    field_offset += index * field_base_size;
+  //
+  //    offset_diff = sizeof_struct - field_offset;
+  //
+  //    if (flexible_array_member_size < 0 || index < 0)
+  //        return 0;
+  //    return offset_diff + flexible_array_member_size;
+
+  QualType CountTy = CountFD->getType();
+  bool IsSigned = CountTy->isSignedIntegerType();
+
+  QualType FlexibleArrayMemberTy = FlexibleArrayMemberFD->getType();
+  QualType FieldTy = FD->getType();
+
+  // Explicit cast because otherwise the CharWidth will promote an i32's into
+  // u64's leading to overflows..
+  int64_t CharWidth = static_cast<int64_t>(CGM.getContext().getCharWidth());
+
+  //  size_t field_offset = offsetof (struct s, field);
+  Value *FieldOffset = nullptr;
+  if (FlexibleArrayMemberFD != FD) {
+    std::optional<int64_t> Offset = GetFieldOffset(Ctx, RD, FD);
+    if (!Offset)
+      return nullptr;
+    FieldOffset =
+        llvm::ConstantInt::get(ResType, *Offset / CharWidth, IsSigned);
+  }
+
+  //  size_t count = (size_t) ptr->count;
+  Value *Count = EmitLoadOfCountedByField(ME, FlexibleArrayMemberFD, CountFD);
+  if (!Count)
+    return nullptr;
+  Count = Builder.CreateIntCast(Count, ResType, IsSigned, "count");
+
+  //  size_t index = (size_t) ptr->index;
+  Value *Index = nullptr;
+  if (Idx) {
     bool IdxSigned = Idx->getType()->isSignedIntegerType();
-    IdxInst = EmitAnyExprToTemp(Idx).getScalarVal();
-    IdxInst = Builder.CreateIntCast(IdxInst, ResType, IdxSigned);
-
-    // We go ahead with the calculation here. If the index turns out to be
-    // negative, we'll catch it at the end.
-    CountedByInst =
-        Builder.CreateSub(CountedByInst, IdxInst, "", !IsSigned, IsSigned);
-  }
-
-  // Calculate how large the flexible array member is in bytes.
-  const ArrayType *ArrayTy = Ctx.getAsArrayType(FAMDecl->getType());
-  CharUnits Size = Ctx.getTypeSizeInChars(ArrayTy->getElementType());
-  llvm::Constant *ElemSize =
-      llvm::ConstantInt::get(ResType, Size.getQuantity(), IsSigned);
-  Value *Res =
-      Builder.CreateMul(CountedByInst, ElemSize, "", !IsSigned, IsSigned);
-  Res = Builder.CreateIntCast(Res, ResType, IsSigned);
-
-  // A negative \p IdxInst or \p CountedByInst means that the index lands
-  // outside of the flexible array member. If that's the case, we want to
-  // return 0.
-  Value *Cmp = Builder.CreateIsNotNeg(CountedByInst);
-  if (IdxInst)
-    Cmp = Builder.CreateAnd(Builder.CreateIsNotNeg(IdxInst), Cmp);
+    Index = EmitScalarExpr(Idx);
+    Index = Builder.CreateIntCast(Index, ResType, IdxSigned, "index");
+  }
+
+  //  size_t flexible_array_member_base_size = sizeof (*ptr->array);
+  const ArrayType *ArrayTy = Ctx.getAsArrayType(FlexibleArrayMemberTy);
+  CharUnits BaseSize = Ctx.getTypeSizeInChars(ArrayTy->getElementType());
+  auto *FlexibleArrayMemberBaseSize =
+      llvm::ConstantInt::get(ResType, BaseSize.getQuantity(), IsSigned);
+
+  //  size_t flexible_array_member_size =
+  //          count * flexible_array_member_base_size;
+  Value *FlexibleArrayMemberSize =
+      Builder.CreateMul(Count, FlexibleArrayMemberBaseSize,
+                        "flexible_array_member_size", !IsSigned, IsSigned);
+
+  Value *Res = nullptr;
+  if (FlexibleArrayMemberFD == FD) {
+    if (Idx) { // Option (2) '&ptr->array[idx]'
+      //  size_t index_size = index * flexible_array_member_base_size;
+      Value *IndexSize = Builder.CreateMul(FlexibleArrayMemberBaseSize, Index,
+                                           "index_size", !IsSigned, IsSigned);
+
+      //  return flexible_array_member_size - index_size;
+      Res = Builder.CreateSub(FlexibleArrayMemberSize, IndexSize, "result",
+                              !IsSigned, IsSigned);
+    } else { // Option (1) 'ptr->array'
+      //  return flexible_array_member_size;
+      Res = FlexibleArrayMemberSize;
+    }
+  } else {
+    //  size_t sizeof_struct = sizeof (struct s);
+    llvm::StructType *StructTy = getTypes().getCGRecordLayout(RD).getLLVMType();
+    const llvm::DataLayout &Layout = CGM.getDataLayout();
+    TypeSize Size = Layout.getTypeSizeInBits(StructTy);
+    Value *SizeofStruct =
+        llvm::ConstantInt::get(ResType, Size.getKnownMinValue() / CharWidth);
+
+    if (Idx) { // Option (4) '&ptr->field_array[idx]'
+      //  size_t field_base_size = sizeof (*ptr->field_array);
+      const ArrayType *ArrayTy = Ctx.getAsArrayType(FieldTy);
+      CharUnits BaseSize = Ctx.getTypeSizeInChars(ArrayTy->getElementType());
+      auto *FieldBaseSize =
+          llvm::ConstantInt::get(ResType, BaseSize.getQuantity(), IsSigned);
+
+      //  field_offset += index * field_base_size;
+      Value *Mul = Builder.CreateMul(Index, FieldBaseSize, "field_offset",
+                                     !IsSigned, IsSigned);
+      FieldOffset = Builder.CreateAdd(FieldOffset, Mul);
+    }
+    // Option (3) '&ptr->field', and Option (4) continuation.
+
+    //  size_t offset_diff = flexible_array_member_offset - field_offset;
+    Value *OffsetDiff = Builder.CreateSub(SizeofStruct, FieldOffset,
+                                          "offset_diff", !IsSigned, IsSigned);
+
+    //  return offset_diff + flexible_array_member_size;
+    Res = Builder.CreateAdd(FlexibleArrayMemberSize, OffsetDiff, "result");
+  }
+
+  Value *Cmp = Builder.CreateIsNotNeg(Res);
+  if (Idx)
+    Cmp = Builder.CreateAnd(Builder.CreateIsNotNeg(Index), Cmp);
 
   return Builder.CreateSelect(Cmp, Res, ConstantInt::get(ResType, 0, IsSigned));
 }
@@ -1315,13 +1440,6 @@ CodeGenFunction::emitBuiltinObjectSize(const Expr *E, unsigned Type,
     }
   }
 
-  if (IsDynamic) {
-    // Emit special code for a flexible array member with the "counted_by"
-    // attribute.
-    if (Value *V = emitFlexibleArrayMemberSize(E, Type, ResType))
-      return V;
-  }
-
   // LLVM can't handle Type=3 appropriately, and __builtin_object_size shouldn't
   // evaluate E for side-effects. In either case, we shouldn't lower to
   // @llvm.objectsize.
@@ -1332,6 +1450,12 @@ CodeGenFunction::emitBuiltinObjectSize(const Expr *E, unsigned Type,
   assert(Ptr->getType()->isPointerTy() &&
          "Non-pointer passed to __builtin_object_size?");
 
+  if (IsDynamic)
+    // Emit special code for a flexible array member with the "counted_by"
+    // attribute.
+    if (Value *V = emitCountedByMemberSize(E, Ptr, Type, ResType))
+      return V;
+
   Function *F =
       CGM.getIntrinsic(Intrinsic::objectsize, {ResType, Ptr->getType()});
 
@@ -6454,7 +6578,7 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
     LargestVectorWidth = std::max(LargestVectorWidth, VectorWidth);
 
   // See if we have a target specific intrinsic.
-  StringRef Name = getContext().BuiltinInfo.getName(BuiltinID);
+  std::string Name = getContext().BuiltinInfo.getName(BuiltinID);
   Intrinsic::ID IntrinsicID = Intrinsic::not_intrinsic;
   StringRef Prefix =
       llvm::Triple::getArchTypePrefix(getTarget().getTriple().getArch());
@@ -6679,6 +6803,7 @@ static llvm::FixedVectorType *GetNeonType(CodeGenFunction *CGF,
   switch (TypeFlags.getEltType()) {
   case NeonTypeFlags::Int8:
   case NeonTypeFlags::Poly8:
+  case NeonTypeFlags::MFloat8:
     return llvm::FixedVectorType::get(CGF->Int8Ty, V1Ty ? 1 : (8 << IsQuad));
   case NeonTypeFlags::Int16:
   case NeonTypeFlags::Poly16:
@@ -6758,12 +6883,68 @@ Value *CodeGenFunction::EmitNeonCall(Function *F, SmallVectorImpl<Value*> &Ops,
     return Builder.CreateCall(F, Ops, name);
 }
 
+Value *CodeGenFunction::EmitFP8NeonCall(unsigned IID,
+                                        ArrayRef<llvm::Type *> Tys,
+                                        SmallVectorImpl<Value *> &Ops,
+                                        const CallExpr *E, const char *name) {
+  llvm::Value *FPM =
+      EmitScalarOrConstFoldImmArg(/* ICEArguments */ 0, E->getNumArgs() - 1, E);
+  Builder.CreateCall(CGM.getIntrinsic(Intrinsic::aarch64_set_fpmr), FPM);
+  return EmitNeonCall(CGM.getIntrinsic(IID, Tys), Ops, name);
+}
+
+llvm::Value *CodeGenFunction::EmitFP8NeonFDOTCall(
+    unsigned IID, bool ExtendLaneArg, llvm::Type *RetTy,
+    SmallVectorImpl<llvm::Value *> &Ops, const CallExpr *E, const char *name) {
+
+  const unsigned ElemCount = Ops[0]->getType()->getPrimitiveSizeInBits() /
+                             RetTy->getPrimitiveSizeInBits();
+  llvm::Type *Tys[] = {llvm::FixedVectorType::get(RetTy, ElemCount),
+                       Ops[1]->getType()};
+  if (ExtendLaneArg) {
+    auto *VT = llvm::FixedVectorType::get(Int8Ty, 16);
+    Ops[2] = Builder.CreateInsertVector(VT, PoisonValue::get(VT), Ops[2],
+                                        Builder.getInt64(0));
+  }
+  return EmitFP8NeonCall(IID, Tys, Ops, E, name);
+}
+
+llvm::Value *CodeGenFunction::EmitFP8NeonFMLACall(
+    unsigned IID, bool ExtendLaneArg, llvm::Type *RetTy,
+    SmallVectorImpl<llvm::Value *> &Ops, const CallExpr *E, const char *name) {
+
+  if (ExtendLaneArg) {
+    auto *VT = llvm::FixedVectorType::get(Int8Ty, 16);
+    Ops[2] = Builder.CreateInsertVector(VT, PoisonValue::get(VT), Ops[2],
+                                        Builder.getInt64(0));
+  }
+  const unsigned ElemCount = Ops[0]->getType()->getPrimitiveSizeInBits() /
+                             RetTy->getPrimitiveSizeInBits();
+  return EmitFP8NeonCall(IID, {llvm::FixedVectorType::get(RetTy, ElemCount)},
+                         Ops, E, name);
+}
+
 Value *CodeGenFunction::EmitNeonShiftVector(Value *V, llvm::Type *Ty,
                                             bool neg) {
   int SV = cast<ConstantInt>(V)->getSExtValue();
   return ConstantInt::get(Ty, neg ? -SV : SV);
 }
 
+Value *CodeGenFunction::EmitFP8NeonCvtCall(unsigned IID, llvm::Type *Ty0,
+                                           llvm::Type *Ty1, bool Extract,
+                                           SmallVectorImpl<llvm::Value *> &Ops,
+                                           const CallExpr *E,
+                                           const char *name) {
+  llvm::Type *Tys[] = {Ty0, Ty1};
+  if (Extract) {
+    // Op[0] is mfloat8x16_t, but the intrinsic converts only the lower part of
+    // the vector.
+    Tys[1] = llvm::FixedVectorType::get(Int8Ty, 8);
+    Ops[0] = Builder.CreateExtractVector(Tys[1], Ops[0], Builder.getInt64(0));
+  }
+  return EmitFP8NeonCall(IID, Tys, Ops, E, name);
+}
+
 // Right-shift a vector by a constant.
 Value *CodeGenFunction::EmitNeonRShiftImm(Value *Vec, Value *Shift,
                                           llvm::Type *Ty, bool usgn,
@@ -12735,6 +12916,8 @@ Value *CodeGenFunction::EmitAArch64BuiltinExpr(unsigned BuiltinID,
     return V;
 
   unsigned Int;
+  bool ExtractLow = false;
+  bool ExtendLaneArg = false;
   switch (BuiltinID) {
   default: return nullptr;
   case NEON::BI__builtin_neon_vbsl_v:
@@ -13949,7 +14132,145 @@ Value *CodeGenFunction::EmitAArch64BuiltinExpr(unsigned BuiltinID,
     Int = Intrinsic::aarch64_neon_vluti4q_laneq_x2;
     return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vluti4q_laneq_x2");
   }
-
+  case NEON::BI__builtin_neon_vcvt1_low_bf16_mf8_fpm:
+    ExtractLow = true;
+    LLVM_FALLTHROUGH;
+  case NEON::BI__builtin_neon_vcvt1_bf16_mf8_fpm:
+  case NEON::BI__builtin_neon_vcvt1_high_bf16_mf8_fpm:
+    return EmitFP8NeonCvtCall(Intrinsic::aarch64_neon_fp8_cvtl1,
+                              llvm::FixedVectorType::get(BFloatTy, 8),
+                              Ops[0]->getType(), ExtractLow, Ops, E, "vbfcvt1");
+  case NEON::BI__builtin_neon_vcvt2_low_bf16_mf8_fpm:
+    ExtractLow = true;
+    LLVM_FALLTHROUGH;
+  case NEON::BI__builtin_neon_vcvt2_bf16_mf8_fpm:
+  case NEON::BI__builtin_neon_vcvt2_high_bf16_mf8_fpm:
+    return EmitFP8NeonCvtCall(Intrinsic::aarch64_neon_fp8_cvtl2,
+                              llvm::FixedVectorType::get(BFloatTy, 8),
+                              Ops[0]->getType(), ExtractLow, Ops, E, "vbfcvt2");
+  case NEON::BI__builtin_neon_vcvt1_low_f16_mf8_fpm:
+    ExtractLow = true;
+    LLVM_FALLTHROUGH;
+  case NEON::BI__builtin_neon_vcvt1_f16_mf8_fpm:
+  case NEON::BI__builtin_neon_vcvt1_high_f16_mf8_fpm:
+    return EmitFP8NeonCvtCall(Intrinsic::aarch64_neon_fp8_cvtl1,
+                              llvm::FixedVectorType::get(HalfTy, 8),
+                              Ops[0]->getType(), ExtractLow, Ops, E, "vbfcvt1");
+  case NEON::BI__builtin_neon_vcvt2_low_f16_mf8_fpm:
+    ExtractLow = true;
+    LLVM_FALLTHROUGH;
+  case NEON::BI__builtin_neon_vcvt2_f16_mf8_fpm:
+  case NEON::BI__builtin_neon_vcvt2_high_f16_mf8_fpm:
+    return EmitFP8NeonCvtCall(Intrinsic::aarch64_neon_fp8_cvtl2,
+                              llvm::FixedVectorType::get(HalfTy, 8),
+                              Ops[0]->getType(), ExtractLow, Ops, E, "vbfcvt2");
+  case NEON::BI__builtin_neon_vcvt_mf8_f32_fpm:
+    return EmitFP8NeonCvtCall(Intrinsic::aarch64_neon_fp8_fcvtn,
+                              llvm::FixedVectorType::get(Int8Ty, 8),
+                              Ops[0]->getType(), false, Ops, E, "vfcvtn");
+  case NEON::BI__builtin_neon_vcvt_mf8_f16_fpm:
+    return EmitFP8NeonCvtCall(Intrinsic::aarch64_neon_fp8_fcvtn,
+                              llvm::FixedVectorType::get(Int8Ty, 8),
+                              llvm::FixedVectorType::get(HalfTy, 4), false, Ops,
+                              E, "vfcvtn");
+  case NEON::BI__builtin_neon_vcvtq_mf8_f16_fpm:
+    return EmitFP8NeonCvtCall(Intrinsic::aarch64_neon_fp8_fcvtn,
+                              llvm::FixedVectorType::get(Int8Ty, 16),
+                              llvm::FixedVectorType::get(HalfTy, 8), false, Ops,
+                              E, "vfcvtn");
+  case NEON::BI__builtin_neon_vcvt_high_mf8_f32_fpm: {
+    llvm::Type *Ty = llvm::FixedVectorType::get(Int8Ty, 16);
+    Ops[0] = Builder.CreateInsertVector(Ty, PoisonValue::get(Ty), Ops[0],
+                                        Builder.getInt64(0));
+    return EmitFP8NeonCvtCall(Intrinsic::aarch64_neon_fp8_fcvtn2, Ty,
+                              Ops[1]->getType(), false, Ops, E, "vfcvtn2");
+  }
+
+  case NEON::BI__builtin_neon_vdot_f16_mf8_fpm:
+  case NEON::BI__builtin_neon_vdotq_f16_mf8_fpm:
+    return EmitFP8NeonFDOTCall(Intrinsic::aarch64_neon_fp8_fdot2, false, HalfTy,
+                               Ops, E, "fdot2");
+  case NEON::BI__builtin_neon_vdot_lane_f16_mf8_fpm:
+  case NEON::BI__builtin_neon_vdotq_lane_f16_mf8_fpm:
+    ExtendLaneArg = true;
+    LLVM_FALLTHROUGH;
+  case NEON::BI__builtin_neon_vdot_laneq_f16_mf8_fpm:
+  case NEON::BI__builtin_neon_vdotq_laneq_f16_mf8_fpm:
+    return EmitFP8NeonFDOTCall(Intrinsic::aarch64_neon_fp8_fdot2_lane,
+                               ExtendLaneArg, HalfTy, Ops, E, "fdot2_lane");
+  case NEON::BI__builtin_neon_vdot_f32_mf8_fpm:
+  case NEON::BI__builtin_neon_vdotq_f32_mf8_fpm:
+    return EmitFP8NeonFDOTCall(Intrinsic::aarch64_neon_fp8_fdot4, false,
+                               FloatTy, Ops, E, "fdot4");
+  case NEON::BI__builtin_neon_vdot_lane_f32_mf8_fpm:
+  case NEON::BI__builtin_neon_vdotq_lane_f32_mf8_fpm:
+    ExtendLaneArg = true;
+    LLVM_FALLTHROUGH;
+  case NEON::BI__builtin_neon_vdot_laneq_f32_mf8_fpm:
+  case NEON::BI__builtin_neon_vdotq_laneq_f32_mf8_fpm:
+    return EmitFP8NeonFDOTCall(Intrinsic::aarch64_neon_fp8_fdot4_lane,
+                               ExtendLaneArg, FloatTy, Ops, E, "fdot4_lane");
+
+  case NEON::BI__builtin_neon_vmlalbq_f16_mf8_fpm:
+    return EmitFP8NeonCall(Intrinsic::aarch64_neon_fp8_fmlalb,
+                           {llvm::FixedVectorType::get(HalfTy, 8)}, Ops, E,
+                           "vmlal");
+  case NEON::BI__builtin_neon_vmlaltq_f16_mf8_fpm:
+    return EmitFP8NeonCall(Intrinsic::aarch64_neon_fp8_fmlalt,
+                           {llvm::FixedVectorType::get(HalfTy, 8)}, Ops, E,
+                           "vmlal");
+  case NEON::BI__builtin_neon_vmlallbbq_f32_mf8_fpm:
+    return EmitFP8NeonCall(Intrinsic::aarch64_neon_fp8_fmlallbb,
+                           {llvm::FixedVectorType::get(FloatTy, 4)}, Ops, E,
+                           "vmlall");
+  case NEON::BI__builtin_neon_vmlallbtq_f32_mf8_fpm:
+    return EmitFP8NeonCall(Intrinsic::aarch64_neon_fp8_fmlallbt,
+                           {llvm::FixedVectorType::get(FloatTy, 4)}, Ops, E,
+                           "vmlall");
+  case NEON::BI__builtin_neon_vmlalltbq_f32_mf8_fpm:
+    return EmitFP8NeonCall(Intrinsic::aarch64_neon_fp8_fmlalltb,
+                           {llvm::FixedVectorType::get(FloatTy, 4)}, Ops, E,
+                           "vmlall");
+  case NEON::BI__builtin_neon_vmlallttq_f32_mf8_fpm:
+    return EmitFP8NeonCall(Intrinsic::aarch64_neon_fp8_fmlalltt,
+                           {llvm::FixedVectorType::get(FloatTy, 4)}, Ops, E,
+                           "vmlall");
+  case NEON::BI__builtin_neon_vmlalbq_lane_f16_mf8_fpm:
+    ExtendLaneArg = true;
+    LLVM_FALLTHROUGH;
+  case NEON::BI__builtin_neon_vmlalbq_laneq_f16_mf8_fpm:
+    return EmitFP8NeonFMLACall(Intrinsic::aarch64_neon_fp8_fmlalb_lane,
+                               ExtendLaneArg, HalfTy, Ops, E, "vmlal_lane");
+  case NEON::BI__builtin_neon_vmlaltq_lane_f16_mf8_fpm:
+    ExtendLaneArg = true;
+    LLVM_FALLTHROUGH;
+  case NEON::BI__builtin_neon_vmlaltq_laneq_f16_mf8_fpm:
+    return EmitFP8NeonFMLACall(Intrinsic::aarch64_neon_fp8_fmlalt_lane,
+                               ExtendLaneArg, HalfTy, Ops, E, "vmlal_lane");
+  case NEON::BI__builtin_neon_vmlallbbq_lane_f32_mf8_fpm:
+    ExtendLaneArg = true;
+    LLVM_FALLTHROUGH;
+  case NEON::BI__builtin_neon_vmlallbbq_laneq_f32_mf8_fpm:
+    return EmitFP8NeonFMLACall(Intrinsic::aarch64_neon_fp8_fmlallbb_lane,
+                               ExtendLaneArg, FloatTy, Ops, E, "vmlall_lane");
+  case NEON::BI__builtin_neon_vmlallbtq_lane_f32_mf8_fpm:
+    ExtendLaneArg = true;
+    LLVM_FALLTHROUGH;
+  case NEON::BI__builtin_neon_vmlallbtq_laneq_f32_mf8_fpm:
+    return EmitFP8NeonFMLACall(Intrinsic::aarch64_neon_fp8_fmlallbt_lane,
+                               ExtendLaneArg, FloatTy, Ops, E, "vmlall_lane");
+  case NEON::BI__builtin_neon_vmlalltbq_lane_f32_mf8_fpm:
+    ExtendLaneArg = true;
+    LLVM_FALLTHROUGH;
+  case NEON::BI__builtin_neon_vmlalltbq_laneq_f32_mf8_fpm:
+    return EmitFP8NeonFMLACall(Intrinsic::aarch64_neon_fp8_fmlalltb_lane,
+                               ExtendLaneArg, FloatTy, Ops, E, "vmlall_lane");
+  case NEON::BI__builtin_neon_vmlallttq_lane_f32_mf8_fpm:
+    ExtendLaneArg = true;
+    LLVM_FALLTHROUGH;
+  case NEON::BI__builtin_neon_vmlallttq_laneq_f32_mf8_fpm:
+    return EmitFP8NeonFMLACall(Intrinsic::aarch64_neon_fp8_fmlalltt_lane,
+                               ExtendLaneArg, FloatTy, Ops, E, "vmlall_lane");
   case NEON::BI__builtin_neon_vamin_f16:
   case NEON::BI__builtin_neon_vaminq_f16:
   case NEON::BI__builtin_neon_vamin_f32:
@@ -19098,6 +19419,25 @@ static Intrinsic::ID getWaveActiveSumIntrinsic(llvm::Triple::ArchType Arch,
   }
 }
 
+// Return wave active sum that corresponds to the QT scalar type
+static Intrinsic::ID getWaveActiveMaxIntrinsic(llvm::Triple::ArchType Arch,
+                                               CGHLSLRuntime &RT, QualType QT) {
+  switch (Arch) {
+  case llvm::Triple::spirv:
+    if (QT->isUnsignedIntegerType())
+      return llvm::Intrinsic::spv_wave_reduce_umax;
+    return llvm::Intrinsic::spv_wave_reduce_max;
+  case llvm::Triple::dxil: {
+    if (QT->isUnsignedIntegerType())
+      return llvm::Intrinsic::dx_wave_reduce_umax;
+    return llvm::Intrinsic::dx_wave_reduce_max;
+  }
+  default:
+    llvm_unreachable("Intrinsic WaveActiveMax"
+                     " not supported by target architecture");
+  }
+}
+
 Value *CodeGenFunction::EmitHLSLBuiltinExpr(unsigned BuiltinID,
                                             const CallExpr *E,
                                             ReturnValueSlot ReturnValue) {
@@ -19105,6 +19445,70 @@ Value *CodeGenFunction::EmitHLSLBuiltinExpr(unsigned BuiltinID,
     return nullptr;
 
   switch (BuiltinID) {
+  case Builtin::BI__builtin_hlsl_adduint64: {
+    Value *OpA = EmitScalarExpr(E->getArg(0));
+    Value *OpB = EmitScalarExpr(E->getArg(1));
+    assert(E->getArg(0)->getType()->hasIntegerRepresentation() &&
+           E->getArg(1)->getType()->hasIntegerRepresentation() &&
+           "AddUint64 operands must have an integer representation");
+    assert(((E->getArg(0)->getType()->castAs<VectorType>()->getNumElements() ==
+                 2 &&
+             E->getArg(1)->getType()->castAs<VectorType>()->getNumElements() ==
+                 2) ||
+            (E->getArg(0)->getType()->castAs<VectorType>()->getNumElements() ==
+                 4 &&
+             E->getArg(1)->getType()->castAs<VectorType>()->getNumElements() ==
+                 4)) &&
+           "input vectors must have 2 or 4 elements each");
+
+    uint64_t NumElements =
+        E->getArg(0)->getType()->castAs<VectorType>()->getNumElements();
+
+    llvm::Value *Result = PoisonValue::get(OpA->getType());
+    llvm::Value *LowA;
+    llvm::Value *HighA;
+    llvm::Value *LowB;
+    llvm::Value *HighB;
+
+    // Obtain low and high words of inputs A and B
+    if (NumElements == 2) {
+      LowA = Builder.CreateExtractElement(OpA, (uint64_t)0, "LowA");
+      HighA = Builder.CreateExtractElement(OpA, (uint64_t)1, "HighA");
+      LowB = Builder.CreateExtractElement(OpB, (uint64_t)0, "LowB");
+      HighB = Builder.CreateExtractElement(OpB, (uint64_t)1, "HighB");
+    } else {
+      LowA = Builder.CreateShuffleVector(OpA, ArrayRef<int>{0, 2}, "LowA");
+      HighA = Builder.CreateShuffleVector(OpA, ArrayRef<int>{1, 3}, "HighA");
+      LowB = Builder.CreateShuffleVector(OpB, ArrayRef<int>{0, 2}, "LowB");
+      HighB = Builder.CreateShuffleVector(OpB, ArrayRef<int>{1, 3}, "HighB");
+    }
+
+    // Use an uadd_with_overflow to compute the sum of low words and obtain a
+    // carry value
+    llvm::Value *Carry;
+    llvm::Value *LowSum = EmitOverflowIntrinsic(
+        *this, llvm::Intrinsic::uadd_with_overflow, LowA, LowB, Carry);
+    llvm::Value *ZExtCarry =
+        Builder.CreateZExt(Carry, HighA->getType(), "CarryZExt");
+
+    // Sum the high words and the carry
+    llvm::Value *HighSum = Builder.CreateAdd(HighA, HighB, "HighSum");
+    llvm::Value *HighSumPlusCarry =
+        Builder.CreateAdd(HighSum, ZExtCarry, "HighSumPlusCarry");
+
+    // Insert the low and high word sums into the result vector
+    if (NumElements == 2) {
+      Result = Builder.CreateInsertElement(Result, LowSum, (uint64_t)0,
+                                           "hlsl.AddUint64.upto0");
+      Result = Builder.CreateInsertElement(Result, HighSumPlusCarry,
+                                           (uint64_t)1, "hlsl.AddUint64");
+    } else { /* NumElements == 4 */
+      Result = Builder.CreateShuffleVector(LowSum, HighSumPlusCarry,
+                                           ArrayRef<int>{0, 2, 1, 3},
+                                           "hlsl.AddUint64");
+    }
+    return Result;
+  }
   case Builtin::BI__builtin_hlsl_resource_getpointer: {
     Value *HandleOp = EmitScalarExpr(E->getArg(0));
     Value *IndexOp = EmitScalarExpr(E->getArg(1));
@@ -19427,6 +19831,23 @@ case Builtin::BI__builtin_hlsl_elementwise_isinf: {
                                                      /*AssumeConvergent=*/true),
                            ArrayRef{OpExpr}, "hlsl.wave.active.sum");
   }
+  case Builtin::BI__builtin_hlsl_wave_active_max: {
+    // Due to the use of variadic arguments, explicitly retreive argument
+    Value *OpExpr = EmitScalarExpr(E->getArg(0));
+    llvm::FunctionType *FT = llvm::FunctionType::get(
+        OpExpr->getType(), ArrayRef{OpExpr->getType()}, false);
+    Intrinsic::ID IID = getWaveActiveMaxIntrinsic(
+        getTarget().getTriple().getArch(), CGM.getHLSLRuntime(),
+        E->getArg(0)->getType());
+
+    // Get overloaded name
+    std::string Name =
+        Intrinsic::getName(IID, ArrayRef{OpExpr->getType()}, &CGM.getModule());
+    return EmitRuntimeCall(CGM.CreateRuntimeFunction(FT, Name, {},
+                                                     /*Local=*/false,
+                                                     /*AssumeConvergent=*/true),
+                           ArrayRef{OpExpr}, "hlsl.wave.active.max");
+  }
   case Builtin::BI__builtin_hlsl_wave_get_lane_index: {
     // We don't define a SPIR-V intrinsic, instead it is a SPIR-V built-in
     // defined in SPIRVBuiltins.td. So instead we manually get the matching name
@@ -21216,7 +21637,7 @@ static Value *MakeHalfType(unsigned IntrinsicID, unsigned BuiltinID,
   auto &C = CGF.CGM.getContext();
   if (!(C.getLangOpts().NativeHalfType ||
         !C.getTargetInfo().useFP16ConversionIntrinsics())) {
-    CGF.CGM.Error(E->getExprLoc(), C.BuiltinInfo.getName(BuiltinID).str() +
+    CGF.CGM.Error(E->getExprLoc(), C.BuiltinInfo.getQuotedName(BuiltinID) +
                                        " requires native half type support.");
     return nullptr;
   }
@@ -22051,7 +22472,7 @@ RValue CodeGenFunction::EmitBuiltinAlignTo(const CallExpr *E, bool AlignUp) {
     // By adding the mask, we ensure that align_up on an already aligned
     // value will not change the value.
     if (Args.Src->getType()->isPointerTy()) {
-      if (getLangOpts().isSignedOverflowDefined())
+      if (getLangOpts().PointerOverflowDefined)
         SrcForMask =
             Builder.CreateGEP(Int8Ty, SrcForMask, Args.Mask, "over_boundary");
       else
diff --git a/clang/lib/CodeGen/CGCUDANV.cpp b/clang/lib/CodeGen/CGCUDANV.cpp
index 23a40b8f7c32a..5445a9278596d 100644
--- a/clang/lib/CodeGen/CGCUDANV.cpp
+++ b/clang/lib/CodeGen/CGCUDANV.cpp
@@ -40,8 +40,6 @@ class CGNVCUDARuntime : public CGCUDARuntime {
 
   /// The prefix used for function calls and section names (CUDA, HIP, LLVM)
   StringRef Prefix;
-  /// TODO: We should transition the OpenMP section to LLVM/Offload
-  StringRef SectionPrefix;
 
 private:
   llvm::IntegerType *IntTy, *SizeTy;
@@ -234,13 +232,12 @@ CGNVCUDARuntime::CGNVCUDARuntime(CodeGenModule &CGM)
   VoidTy = CGM.VoidTy;
   PtrTy = CGM.UnqualPtrTy;
 
-  if (CGM.getLangOpts().OffloadViaLLVM) {
+  if (CGM.getLangOpts().OffloadViaLLVM)
     Prefix = "llvm";
-    SectionPrefix = "omp";
-  } else if (CGM.getLangOpts().HIP)
-    SectionPrefix = Prefix = "hip";
+  else if (CGM.getLangOpts().HIP)
+    Prefix = "hip";
   else
-    SectionPrefix = Prefix = "cuda";
+    Prefix = "cuda";
 }
 
 llvm::FunctionCallee CGNVCUDARuntime::getSetupArgumentFn() const {
@@ -1198,14 +1195,19 @@ void CGNVCUDARuntime::transformManagedVars() {
 // register the symbols with the linked device image.
 void CGNVCUDARuntime::createOffloadingEntries() {
   SmallVector<char, 32> Out;
-  StringRef Section = (SectionPrefix + "_offloading_entries").toStringRef(Out);
+  llvm::object::OffloadKind Kind = CGM.getLangOpts().HIP
+                                       ? llvm::object::OffloadKind::OFK_HIP
+                                       : llvm::object::OffloadKind::OFK_Cuda;
+  // For now, just spoof this as OpenMP because that's the runtime it uses.
+  if (CGM.getLangOpts().OffloadViaLLVM)
+    Kind = llvm::object::OffloadKind::OFK_OpenMP;
 
   llvm::Module &M = CGM.getModule();
   for (KernelInfo &I : EmittedKernels)
     llvm::offloading::emitOffloadingEntry(
-        M, KernelHandles[I.Kernel->getName()],
+        M, Kind, KernelHandles[I.Kernel->getName()],
         getDeviceSideName(cast<NamedDecl>(I.D)), /*Flags=*/0, /*Data=*/0,
-        llvm::offloading::OffloadGlobalEntry, Section);
+        llvm::offloading::OffloadGlobalEntry);
 
   for (VarInfo &I : DeviceVars) {
     uint64_t VarSize =
@@ -1221,44 +1223,32 @@ void CGNVCUDARuntime::createOffloadingEntries() {
              ? static_cast<int32_t>(llvm::offloading::OffloadGlobalNormalized)
              : 0);
     if (I.Flags.getKind() == DeviceVarFlags::Variable) {
-      // TODO: Update the offloading entries struct to avoid this indirection.
       if (I.Flags.isManaged()) {
         assert(I.Var->getName().ends_with(".managed") &&
                "HIP managed variables not transformed");
 
-        // Create a struct to contain the two variables.
         auto *ManagedVar = M.getNamedGlobal(
             I.Var->getName().drop_back(StringRef(".managed").size()));
-        llvm::Constant *StructData[] = {ManagedVar, I.Var};
-        llvm::Constant *Initializer = llvm::ConstantStruct::get(
-            llvm::offloading::getManagedTy(M), StructData);
-        auto *Struct = new llvm::GlobalVariable(
-            M, llvm::offloading::getManagedTy(M),
-            /*IsConstant=*/true, llvm::GlobalValue::PrivateLinkage, Initializer,
-            I.Var->getName(), /*InsertBefore=*/nullptr,
-            llvm::GlobalVariable::NotThreadLocal,
-            M.getDataLayout().getDefaultGlobalsAddressSpace());
-
         llvm::offloading::emitOffloadingEntry(
-            M, Struct, getDeviceSideName(I.D), VarSize,
+            M, Kind, I.Var, getDeviceSideName(I.D), VarSize,
             llvm::offloading::OffloadGlobalManagedEntry | Flags,
-            /*Data=*/static_cast<uint32_t>(I.Var->getAlignment()), Section);
+            /*Data=*/I.Var->getAlignment(), ManagedVar);
       } else {
         llvm::offloading::emitOffloadingEntry(
-            M, I.Var, getDeviceSideName(I.D), VarSize,
+            M, Kind, I.Var, getDeviceSideName(I.D), VarSize,
             llvm::offloading::OffloadGlobalEntry | Flags,
-            /*Data=*/0, Section);
+            /*Data=*/0);
       }
     } else if (I.Flags.getKind() == DeviceVarFlags::Surface) {
       llvm::offloading::emitOffloadingEntry(
-          M, I.Var, getDeviceSideName(I.D), VarSize,
+          M, Kind, I.Var, getDeviceSideName(I.D), VarSize,
           llvm::offloading::OffloadGlobalSurfaceEntry | Flags,
-          I.Flags.getSurfTexType(), Section);
+          I.Flags.getSurfTexType());
     } else if (I.Flags.getKind() == DeviceVarFlags::Texture) {
       llvm::offloading::emitOffloadingEntry(
-          M, I.Var, getDeviceSideName(I.D), VarSize,
+          M, Kind, I.Var, getDeviceSideName(I.D), VarSize,
           llvm::offloading::OffloadGlobalTextureEntry | Flags,
-          I.Flags.getSurfTexType(), Section);
+          I.Flags.getSurfTexType());
     }
   }
 }
diff --git a/clang/lib/CodeGen/CGCall.cpp b/clang/lib/CodeGen/CGCall.cpp
index e0cf6ca69f0df..2dce86410db85 100644
--- a/clang/lib/CodeGen/CGCall.cpp
+++ b/clang/lib/CodeGen/CGCall.cpp
@@ -2881,7 +2881,7 @@ void CodeGenModule::ConstructAttributeList(StringRef Name,
     }
 
     if (FI.getExtParameterInfo(ArgNo).isNoEscape())
-      Attrs.addAttribute(llvm::Attribute::NoCapture);
+      Attrs.addCapturesAttr(llvm::CaptureInfo::none());
 
     if (Attrs.hasAttributes()) {
       unsigned FirstIRArg, NumIRArgs;
@@ -3581,15 +3581,26 @@ static llvm::StoreInst *findDominatingStoreToReturnValue(CodeGenFunction &CGF) {
     llvm::BasicBlock *IP = CGF.Builder.GetInsertBlock();
     if (IP->empty()) return nullptr;
 
-    // Look at directly preceding instruction, skipping bitcasts and lifetime
-    // markers.
+    // Look at directly preceding instruction, skipping bitcasts, lifetime
+    // markers, and fake uses and their operands.
+    const llvm::Instruction *LoadIntoFakeUse = nullptr;
     for (llvm::Instruction &I : make_range(IP->rbegin(), IP->rend())) {
+      // Ignore instructions that are just loads for fake uses; the load should
+      // immediately precede the fake use, so we only need to remember the
+      // operand for the last fake use seen.
+      if (LoadIntoFakeUse == &I)
+        continue;
       if (isa<llvm::BitCastInst>(&I))
         continue;
-      if (auto *II = dyn_cast<llvm::IntrinsicInst>(&I))
+      if (auto *II = dyn_cast<llvm::IntrinsicInst>(&I)) {
         if (II->getIntrinsicID() == llvm::Intrinsic::lifetime_end)
           continue;
 
+        if (II->getIntrinsicID() == llvm::Intrinsic::fake_use) {
+          LoadIntoFakeUse = dyn_cast<llvm::Instruction>(II->getArgOperand(0));
+          continue;
+        }
+      }
       return GetStoreIfValid(&I);
     }
     return nullptr;
diff --git a/clang/lib/CodeGen/CGCleanup.cpp b/clang/lib/CodeGen/CGCleanup.cpp
index d9c0dbe45d6cf..7e1c5b7da9552 100644
--- a/clang/lib/CodeGen/CGCleanup.cpp
+++ b/clang/lib/CodeGen/CGCleanup.cpp
@@ -112,11 +112,15 @@ void EHScopeStack::deallocate(size_t Size) {
   StartOfData += llvm::alignTo(Size, ScopeStackAlignment);
 }
 
-bool EHScopeStack::containsOnlyLifetimeMarkers(
+bool EHScopeStack::containsOnlyNoopCleanups(
     EHScopeStack::stable_iterator Old) const {
   for (EHScopeStack::iterator it = begin(); stabilize(it) != Old; it++) {
     EHCleanupScope *cleanup = dyn_cast<EHCleanupScope>(&*it);
-    if (!cleanup || !cleanup->isLifetimeMarker())
+    // If this is anything other than a lifetime marker or fake use cleanup,
+    // then the scope stack does not contain only noop cleanups.
+    if (!cleanup)
+      return false;
+    if (!cleanup->isLifetimeMarker() && !cleanup->isFakeUse())
       return false;
   }
 
@@ -154,6 +158,7 @@ void *EHScopeStack::pushCleanup(CleanupKind Kind, size_t Size) {
   bool IsNormalCleanup = Kind & NormalCleanup;
   bool IsEHCleanup = Kind & EHCleanup;
   bool IsLifetimeMarker = Kind & LifetimeMarker;
+  bool IsFakeUse = Kind & FakeUse;
 
   // Per C++ [except.terminate], it is implementation-defined whether none,
   // some, or all cleanups are called before std::terminate. Thus, when
@@ -176,6 +181,8 @@ void *EHScopeStack::pushCleanup(CleanupKind Kind, size_t Size) {
     InnermostEHScope = stable_begin();
   if (IsLifetimeMarker)
     Scope->setLifetimeMarker();
+  if (IsFakeUse)
+    Scope->setFakeUse();
 
   // With Windows -EHa, Invoke llvm.seh.scope.begin() for EHCleanup
   // If exceptions are disabled/ignored and SEH is not in use, then there is no
diff --git a/clang/lib/CodeGen/CGCleanup.h b/clang/lib/CodeGen/CGCleanup.h
index c73c97146abc4..ba78e5478ac37 100644
--- a/clang/lib/CodeGen/CGCleanup.h
+++ b/clang/lib/CodeGen/CGCleanup.h
@@ -87,6 +87,10 @@ class EHScope {
     LLVM_PREFERRED_TYPE(bool)
     unsigned IsLifetimeMarker : 1;
 
+    /// Whether this cleanup is a fake use
+    LLVM_PREFERRED_TYPE(bool)
+    unsigned IsFakeUse : 1;
+
     /// Whether the normal cleanup should test the activation flag.
     LLVM_PREFERRED_TYPE(bool)
     unsigned TestFlagInNormalCleanup : 1;
@@ -352,6 +356,7 @@ class alignas(8) EHCleanupScope : public EHScope {
     CleanupBits.IsEHCleanup = isEH;
     CleanupBits.IsActive = true;
     CleanupBits.IsLifetimeMarker = false;
+    CleanupBits.IsFakeUse = false;
     CleanupBits.TestFlagInNormalCleanup = false;
     CleanupBits.TestFlagInEHCleanup = false;
     CleanupBits.CleanupSize = cleanupSize;
@@ -384,6 +389,9 @@ class alignas(8) EHCleanupScope : public EHScope {
   bool isLifetimeMarker() const { return CleanupBits.IsLifetimeMarker; }
   void setLifetimeMarker() { CleanupBits.IsLifetimeMarker = true; }
 
+  bool isFakeUse() const { return CleanupBits.IsFakeUse; }
+  void setFakeUse() { CleanupBits.IsFakeUse = true; }
+
   bool hasActiveFlag() const { return ActiveFlag.isValid(); }
   Address getActiveFlag() const {
     return ActiveFlag;
diff --git a/clang/lib/CodeGen/CGDebugInfo.cpp b/clang/lib/CodeGen/CGDebugInfo.cpp
index 6cbcaf0384410..d5b584ec0f2e9 100644
--- a/clang/lib/CodeGen/CGDebugInfo.cpp
+++ b/clang/lib/CodeGen/CGDebugInfo.cpp
@@ -2815,7 +2815,7 @@ static bool shouldOmitDefinition(llvm::codegenoptions::DebugInfoKind DebugKind,
   // without any dllimport methods can be used in one DLL and constructed in
   // another, but it is the current behavior of LimitedDebugInfo.
   if (CXXDecl->hasDefinition() && CXXDecl->isDynamicClass() &&
-      !isClassOrMethodDLLImport(CXXDecl))
+      !isClassOrMethodDLLImport(CXXDecl) && !CXXDecl->hasAttr<MSNoVTableAttr>())
     return true;
 
   TemplateSpecializationKind Spec = TSK_Undeclared;
@@ -3567,6 +3567,10 @@ llvm::DIType *CGDebugInfo::CreateTypeDefinition(const EnumType *Ty) {
         DBuilder.createEnumerator(Enum->getName(), Enum->getInitVal()));
   }
 
+  std::optional<EnumExtensibilityAttr::Kind> EnumKind;
+  if (auto *Attr = ED->getAttr<EnumExtensibilityAttr>())
+    EnumKind = Attr->getExtensibility();
+
   // Return a CompositeType for the enum itself.
   llvm::DINodeArray EltArray = DBuilder.getOrCreateArray(Enumerators);
 
@@ -3576,7 +3580,7 @@ llvm::DIType *CGDebugInfo::CreateTypeDefinition(const EnumType *Ty) {
   llvm::DIType *ClassTy = getOrCreateType(ED->getIntegerType(), DefUnit);
   return DBuilder.createEnumerationType(
       EnumContext, ED->getName(), DefUnit, Line, Size, Align, EltArray, ClassTy,
-      /*RunTimeLang=*/0, Identifier, ED->isScoped());
+      /*RunTimeLang=*/0, Identifier, ED->isScoped(), EnumKind);
 }
 
 llvm::DIMacro *CGDebugInfo::CreateMacro(llvm::DIMacroFile *Parent,
@@ -5083,10 +5087,9 @@ CGDebugInfo::EmitDeclareOfAutoVariable(const VarDecl *VD, llvm::Value *Storage,
   assert(CGM.getCodeGenOpts().hasReducedDebugInfo());
 
   if (auto *DD = dyn_cast<DecompositionDecl>(VD)) {
-    for (auto *B : DD->bindings()) {
+    for (BindingDecl *B : DD->flat_bindings())
       EmitDeclare(B, Storage, std::nullopt, Builder,
                   VD->getType()->isReferenceType());
-    }
     // Don't emit an llvm.dbg.declare for the composite storage as it doesn't
     // correspond to a user variable.
     return nullptr;
diff --git a/clang/lib/CodeGen/CGDecl.cpp b/clang/lib/CodeGen/CGDecl.cpp
index 60f67d4640370..668282a6ab1a8 100644
--- a/clang/lib/CodeGen/CGDecl.cpp
+++ b/clang/lib/CodeGen/CGDecl.cpp
@@ -164,9 +164,10 @@ void CodeGenFunction::EmitDecl(const Decl &D) {
            "Should not see file-scope variables inside a function!");
     EmitVarDecl(VD);
     if (auto *DD = dyn_cast<DecompositionDecl>(&VD))
-      for (auto *B : DD->bindings())
+      for (auto *B : DD->flat_bindings())
         if (auto *HD = B->getHoldingVar())
           EmitVarDecl(*HD);
+
     return;
   }
 
@@ -1355,6 +1356,14 @@ void CodeGenFunction::EmitLifetimeEnd(llvm::Value *Size, llvm::Value *Addr) {
   C->setDoesNotThrow();
 }
 
+void CodeGenFunction::EmitFakeUse(Address Addr) {
+  auto NL = ApplyDebugLocation::CreateEmpty(*this);
+  llvm::Value *V = Builder.CreateLoad(Addr, "fake.use");
+  llvm::CallInst *C = Builder.CreateCall(CGM.getLLVMFakeUseFn(), {V});
+  C->setDoesNotThrow();
+  C->setTailCallKind(llvm::CallInst::TCK_NoTail);
+}
+
 void CodeGenFunction::EmitAndRegisterVariableArrayDimensions(
     CGDebugInfo *DI, const VarDecl &D, bool EmitDebugInfo) {
   // For each dimension stores its QualType and corresponding
@@ -1414,6 +1423,39 @@ void CodeGenFunction::EmitAndRegisterVariableArrayDimensions(
   }
 }
 
+/// Return the maximum size of an aggregate for which we generate a fake use
+/// intrinsic when -fextend-variable-liveness is in effect.
+static uint64_t maxFakeUseAggregateSize(const ASTContext &C) {
+  return 4 * C.getTypeSize(C.UnsignedIntTy);
+}
+
+// Helper function to determine whether a variable's or parameter's lifetime
+// should be extended.
+static bool shouldExtendLifetime(const ASTContext &Context,
+                                 const Decl *FuncDecl, const VarDecl &D,
+                                 ImplicitParamDecl *CXXABIThisDecl) {
+  // When we're not inside a valid function it is unlikely that any
+  // lifetime extension is useful.
+  if (!FuncDecl)
+    return false;
+  if (FuncDecl->isImplicit())
+    return false;
+  // Do not extend compiler-created variables except for the this pointer.
+  if (D.isImplicit() && &D != CXXABIThisDecl)
+    return false;
+  QualType Ty = D.getType();
+  // No need to extend volatiles, they have a memory location.
+  if (Ty.isVolatileQualified())
+    return false;
+  // Don't extend variables that exceed a certain size.
+  if (Context.getTypeSize(Ty) > maxFakeUseAggregateSize(Context))
+    return false;
+  // Do not extend variables in nodebug or optnone functions.
+  if (FuncDecl->hasAttr<NoDebugAttr>() || FuncDecl->hasAttr<OptimizeNoneAttr>())
+    return false;
+  return true;
+}
+
 /// EmitAutoVarAlloca - Emit the alloca and debug information for a
 /// local variable.  Does not emit initialization or destruction.
 CodeGenFunction::AutoVarEmission
@@ -1666,6 +1708,18 @@ CodeGenFunction::EmitAutoVarAlloca(const VarDecl &D) {
                                          emission.getOriginalAllocatedAddress(),
                                          emission.getSizeForLifetimeMarkers());
 
+  // Analogous to lifetime markers, we use a 'cleanup' to emit fake.use
+  // calls for local variables. We are exempting volatile variables and
+  // non-scalars larger than 4 times the size of an unsigned int. Larger
+  // non-scalars are often allocated in memory and may create unnecessary
+  // overhead.
+  if (CGM.getCodeGenOpts().getExtendVariableLiveness() ==
+      CodeGenOptions::ExtendVariableLivenessKind::All) {
+    if (shouldExtendLifetime(getContext(), CurCodeDecl, D, CXXABIThisDecl))
+      EHStack.pushCleanup<FakeUse>(NormalFakeUse,
+                                   emission.getAllocatedAddress());
+  }
+
   return emission;
 }
 
@@ -2532,6 +2586,15 @@ llvm::Function *CodeGenModule::getLLVMLifetimeEndFn() {
   return LifetimeEndFn;
 }
 
+/// Lazily declare the @llvm.fake.use intrinsic.
+llvm::Function *CodeGenModule::getLLVMFakeUseFn() {
+  if (FakeUseFn)
+    return FakeUseFn;
+  FakeUseFn = llvm::Intrinsic::getOrInsertDeclaration(
+      &getModule(), llvm::Intrinsic::fake_use);
+  return FakeUseFn;
+}
+
 namespace {
   /// A cleanup to perform a release of an object at the end of a
   /// function.  This is used to balance out the incoming +1 of a
@@ -2725,6 +2788,18 @@ void CodeGenFunction::EmitParmDecl(const VarDecl &D, ParamValue Arg,
 
   setAddrOfLocalVar(&D, DeclPtr);
 
+  // Push a FakeUse 'cleanup' object onto the EHStack for the parameter,
+  // which may be the 'this' pointer. This causes the emission of a fake.use
+  // call with the parameter as argument at the end of the function.
+  if (CGM.getCodeGenOpts().getExtendVariableLiveness() ==
+          CodeGenOptions::ExtendVariableLivenessKind::All ||
+      (CGM.getCodeGenOpts().getExtendVariableLiveness() ==
+           CodeGenOptions::ExtendVariableLivenessKind::This &&
+       &D == CXXABIThisDecl)) {
+    if (shouldExtendLifetime(getContext(), CurCodeDecl, D, CXXABIThisDecl))
+      EHStack.pushCleanup<FakeUse>(NormalFakeUse, DeclPtr);
+  }
+
   // Emit debug info for param declarations in non-thunk functions.
   if (CGDebugInfo *DI = getDebugInfo()) {
     if (CGM.getCodeGenOpts().hasReducedDebugInfo() && !CurFuncIsThunk &&
@@ -2795,15 +2870,12 @@ void CodeGenModule::EmitOMPAllocateDecl(const OMPAllocateDecl *D) {
 
     // We can also keep the existing global if the address space is what we
     // expect it to be, if not, it is replaced.
-    QualType ASTTy = VD->getType();
     clang::LangAS GVAS = GetGlobalVarAddressSpace(VD);
     auto TargetAS = getContext().getTargetAddressSpace(GVAS);
     if (Entry->getType()->getAddressSpace() == TargetAS)
       continue;
 
-    // Make a new global with the correct type / address space.
-    llvm::Type *Ty = getTypes().ConvertTypeForMem(ASTTy);
-    llvm::PointerType *PTy = llvm::PointerType::get(Ty, TargetAS);
+    llvm::PointerType *PTy = llvm::PointerType::get(getLLVMContext(), TargetAS);
 
     // Replace all uses of the old global with a cast. Since we mutate the type
     // in place we neeed an intermediate that takes the spot of the old entry
@@ -2816,8 +2888,7 @@ void CodeGenModule::EmitOMPAllocateDecl(const OMPAllocateDecl *D) {
 
     Entry->mutateType(PTy);
     llvm::Constant *NewPtrForOldDecl =
-        llvm::ConstantExpr::getPointerBitCastOrAddrSpaceCast(
-            Entry, DummyGV->getType());
+        llvm::ConstantExpr::getAddrSpaceCast(Entry, DummyGV->getType());
 
     // Now we have a casted version of the changed global, the dummy can be
     // replaced and deleted.
diff --git a/clang/lib/CodeGen/CGDeclCXX.cpp b/clang/lib/CodeGen/CGDeclCXX.cpp
index 1c2fecea1a6ac..f5950f03673a1 100644
--- a/clang/lib/CodeGen/CGDeclCXX.cpp
+++ b/clang/lib/CodeGen/CGDeclCXX.cpp
@@ -345,10 +345,7 @@ void CodeGenFunction::registerGlobalDtorWithLLVM(const VarDecl &VD,
 
 void CodeGenFunction::registerGlobalDtorWithAtExit(llvm::Constant *dtorStub) {
   // extern "C" int atexit(void (*f)(void));
-  assert(dtorStub->getType() ==
-             llvm::PointerType::get(
-                 llvm::FunctionType::get(CGM.VoidTy, false),
-                 dtorStub->getType()->getPointerAddressSpace()) &&
+  assert(dtorStub->getType()->isPointerTy() &&
          "Argument to atexit has a wrong type.");
 
   llvm::FunctionType *atexitTy =
@@ -372,10 +369,7 @@ CodeGenFunction::unregisterGlobalDtorWithUnAtExit(llvm::Constant *dtorStub) {
   // value is returned.
   //
   // extern "C" int unatexit(void (*f)(void));
-  assert(dtorStub->getType() ==
-             llvm::PointerType::get(
-                 llvm::FunctionType::get(CGM.VoidTy, false),
-                 dtorStub->getType()->getPointerAddressSpace()) &&
+  assert(dtorStub->getType()->isPointerTy() &&
          "Argument to unatexit has a wrong type.");
 
   llvm::FunctionType *unatexitTy =
diff --git a/clang/lib/CodeGen/CGExpr.cpp b/clang/lib/CodeGen/CGExpr.cpp
index 054f8d1eadb8c..2bbc0791c6587 100644
--- a/clang/lib/CodeGen/CGExpr.cpp
+++ b/clang/lib/CodeGen/CGExpr.cpp
@@ -24,6 +24,7 @@
 #include "ConstantEmitter.h"
 #include "TargetInfo.h"
 #include "clang/AST/ASTContext.h"
+#include "clang/AST/ASTLambda.h"
 #include "clang/AST/Attr.h"
 #include "clang/AST/DeclObjC.h"
 #include "clang/AST/NSAPI.h"
@@ -871,7 +872,7 @@ void CodeGenFunction::EmitTypeCheck(TypeCheckKind TCK, SourceLocation Loc,
       llvm::Value *TypeHash =
           llvm::ConstantInt::get(Int64Ty, xxh3_64bits(Out.str()));
 
-      llvm::Type *VPtrTy = llvm::PointerType::get(IntPtrTy, 0);
+      llvm::Type *VPtrTy = llvm::PointerType::get(getLLVMContext(), 0);
       Address VPtrAddr(Ptr, IntPtrTy, getPointerAlign());
       llvm::Value *VPtrVal = GetVTablePtr(VPtrAddr, VPtrTy,
                                           Ty->getAsCXXRecordDecl(),
@@ -1809,8 +1810,7 @@ CodeGenFunction::tryEmitAsConstant(DeclRefExpr *refExpr) {
   if (CGM.getLangOpts().CUDAIsDevice && result.Val.isLValue() &&
       refExpr->refersToEnclosingVariableOrCapture()) {
     auto *MD = dyn_cast_or_null<CXXMethodDecl>(CurCodeDecl);
-    if (MD && MD->getParent()->isLambda() &&
-        MD->getOverloadedOperator() == OO_Call) {
+    if (isLambdaMethod(MD) && MD->getOverloadedOperator() == OO_Call) {
       const APValue::LValueBase &base = result.Val.getLValueBase();
       if (const ValueDecl *D = base.dyn_cast<const ValueDecl *>()) {
         if (const VarDecl *VD = dyn_cast<const VarDecl>(D)) {
@@ -2414,8 +2414,15 @@ void CodeGenFunction::EmitStoreThroughLValue(RValue Src, LValue Dst,
         Vec = Builder.CreateBitCast(Vec, IRVecTy);
         // iN --> <N x i1>.
       }
-      Vec = Builder.CreateInsertElement(Vec, Src.getScalarVal(),
-                                        Dst.getVectorIdx(), "vecins");
+      llvm::Value *SrcVal = Src.getScalarVal();
+      // Allow inserting `<1 x T>` into an `<N x T>`. It can happen with scalar
+      // types which are mapped to vector LLVM IR types (e.g. for implementing
+      // an ABI).
+      if (auto *EltTy = dyn_cast<llvm::FixedVectorType>(SrcVal->getType());
+          EltTy && EltTy->getNumElements() == 1)
+        SrcVal = Builder.CreateBitCast(SrcVal, EltTy->getElementType());
+      Vec = Builder.CreateInsertElement(Vec, SrcVal, Dst.getVectorIdx(),
+                                        "vecins");
       if (IRStoreTy) {
         // <N x i1> --> <iN>.
         Vec = Builder.CreateBitCast(Vec, IRStoreTy);
@@ -3047,7 +3054,7 @@ LValue CodeGenFunction::EmitDeclRefLValue(const DeclRefExpr *E) {
                                            getContext().getDeclAlign(VD));
         llvm::Type *VarTy = getTypes().ConvertTypeForMem(VD->getType());
         auto *PTy = llvm::PointerType::get(
-            VarTy, getTypes().getTargetAddressSpace(VD->getType()));
+            getLLVMContext(), getTypes().getTargetAddressSpace(VD->getType()));
         Addr = Builder.CreatePointerBitCastOrAddrSpaceCast(Addr, PTy, VarTy);
       } else {
         // Should we be using the alignment of the constant pointer we emitted?
@@ -3607,29 +3614,33 @@ void CodeGenFunction::EmitCheck(
   llvm::Value *RecoverableCond = nullptr;
   llvm::Value *TrapCond = nullptr;
   bool NoMerge = false;
+  // Expand checks into:
+  //   (Check1 || !allow_ubsan_check) && (Check2 || !allow_ubsan_check) ...
+  // We need separate allow_ubsan_check intrinsics because they have separately
+  // specified cutoffs.
+  // This expression looks expensive but will be simplified after
+  // LowerAllowCheckPass.
   for (auto &[Check, Ord] : Checked) {
+    llvm::Value *GuardedCheck = Check;
+    if (ClSanitizeGuardChecks ||
+        (CGM.getCodeGenOpts().SanitizeSkipHotCutoffs[Ord] > 0)) {
+      llvm::Value *Allow = Builder.CreateCall(
+          CGM.getIntrinsic(llvm::Intrinsic::allow_ubsan_check),
+          llvm::ConstantInt::get(CGM.Int8Ty, Ord));
+      GuardedCheck = Builder.CreateOr(Check, Builder.CreateNot(Allow));
+    }
+
     // -fsanitize-trap= overrides -fsanitize-recover=.
     llvm::Value *&Cond = CGM.getCodeGenOpts().SanitizeTrap.has(Ord) ? TrapCond
                          : CGM.getCodeGenOpts().SanitizeRecover.has(Ord)
                              ? RecoverableCond
                              : FatalCond;
-    Cond = Cond ? Builder.CreateAnd(Cond, Check) : Check;
+    Cond = Cond ? Builder.CreateAnd(Cond, GuardedCheck) : GuardedCheck;
 
     if (!CGM.getCodeGenOpts().SanitizeMergeHandlers.has(Ord))
       NoMerge = true;
   }
 
-  if (ClSanitizeGuardChecks) {
-    llvm::Value *Allow =
-        Builder.CreateCall(CGM.getIntrinsic(llvm::Intrinsic::allow_ubsan_check),
-                           llvm::ConstantInt::get(CGM.Int8Ty, CheckHandler));
-
-    for (llvm::Value **Cond : {&FatalCond, &RecoverableCond, &TrapCond}) {
-      if (*Cond)
-        *Cond = Builder.CreateOr(*Cond, Builder.CreateNot(Allow));
-    }
-  }
-
   if (TrapCond)
     EmitTrapCheck(TrapCond, CheckHandler, NoMerge);
   if (!FatalCond && !RecoverableCond)
@@ -4311,14 +4322,14 @@ LValue CodeGenFunction::EmitArraySubscriptExpr(const ArraySubscriptExpr *E,
     // GEP indexes are signed, and scaling an index isn't permitted to
     // signed-overflow, so we use the same semantics for our explicit
     // multiply.  We suppress this if overflow is not undefined behavior.
-    if (getLangOpts().isSignedOverflowDefined()) {
+    if (getLangOpts().PointerOverflowDefined) {
       Idx = Builder.CreateMul(Idx, numElements);
     } else {
       Idx = Builder.CreateNSWMul(Idx, numElements);
     }
 
     Addr = emitArraySubscriptGEP(*this, Addr, Idx, vla->getElementType(),
-                                 !getLangOpts().isSignedOverflowDefined(),
+                                 !getLangOpts().PointerOverflowDefined,
                                  SignedIndices, E->getExprLoc());
 
   } else if (const ObjCObjectType *OIT = E->getType()->getAs<ObjCObjectType>()){
@@ -4408,7 +4419,7 @@ LValue CodeGenFunction::EmitArraySubscriptExpr(const ArraySubscriptExpr *E,
     QualType arrayType = Array->getType();
     Addr = emitArraySubscriptGEP(
         *this, ArrayLV.getAddress(), {CGM.getSize(CharUnits::Zero()), Idx},
-        E->getType(), !getLangOpts().isSignedOverflowDefined(), SignedIndices,
+        E->getType(), !getLangOpts().PointerOverflowDefined, SignedIndices,
         E->getExprLoc(), &arrayType, E->getBase());
     EltBaseInfo = ArrayLV.getBaseInfo();
     EltTBAAInfo = CGM.getTBAAInfoForSubobject(ArrayLV, E->getType());
@@ -4417,10 +4428,9 @@ LValue CodeGenFunction::EmitArraySubscriptExpr(const ArraySubscriptExpr *E,
     Addr = EmitPointerWithAlignment(E->getBase(), &EltBaseInfo, &EltTBAAInfo);
     auto *Idx = EmitIdxAfterBase(/*Promote*/true);
     QualType ptrType = E->getBase()->getType();
-    Addr = emitArraySubscriptGEP(*this, Addr, Idx, E->getType(),
-                                 !getLangOpts().isSignedOverflowDefined(),
-                                 SignedIndices, E->getExprLoc(), &ptrType,
-                                 E->getBase());
+    Addr = emitArraySubscriptGEP(
+        *this, Addr, Idx, E->getType(), !getLangOpts().PointerOverflowDefined,
+        SignedIndices, E->getExprLoc(), &ptrType, E->getBase());
   }
 
   LValue LV = MakeAddrLValue(Addr, E->getType(), EltBaseInfo, EltTBAAInfo);
@@ -4565,11 +4575,11 @@ LValue CodeGenFunction::EmitArraySectionExpr(const ArraySectionExpr *E,
                 : llvm::ConstantInt::get(IntPtrTy, ConstLength);
         Idx = Builder.CreateAdd(LowerBoundVal, LengthVal, "lb_add_len",
                                 /*HasNUW=*/false,
-                                !getLangOpts().isSignedOverflowDefined());
+                                !getLangOpts().PointerOverflowDefined);
         if (Length && LowerBound) {
           Idx = Builder.CreateSub(
               Idx, llvm::ConstantInt::get(IntPtrTy, /*V=*/1), "idx_sub_1",
-              /*HasNUW=*/false, !getLangOpts().isSignedOverflowDefined());
+              /*HasNUW=*/false, !getLangOpts().PointerOverflowDefined);
         }
       } else
         Idx = llvm::ConstantInt::get(IntPtrTy, ConstLength + ConstLowerBound);
@@ -4595,7 +4605,7 @@ LValue CodeGenFunction::EmitArraySectionExpr(const ArraySectionExpr *E,
             Length->getType()->hasSignedIntegerRepresentation());
         Idx = Builder.CreateSub(
             LengthVal, llvm::ConstantInt::get(IntPtrTy, /*V=*/1), "len_sub_1",
-            /*HasNUW=*/false, !getLangOpts().isSignedOverflowDefined());
+            /*HasNUW=*/false, !getLangOpts().PointerOverflowDefined);
       } else {
         ConstLength = ConstLength.zextOrTrunc(PointerWidthInBits);
         --ConstLength;
@@ -4622,12 +4632,12 @@ LValue CodeGenFunction::EmitArraySectionExpr(const ArraySectionExpr *E,
     // GEP indexes are signed, and scaling an index isn't permitted to
     // signed-overflow, so we use the same semantics for our explicit
     // multiply.  We suppress this if overflow is not undefined behavior.
-    if (getLangOpts().isSignedOverflowDefined())
+    if (getLangOpts().PointerOverflowDefined)
       Idx = Builder.CreateMul(Idx, NumElements);
     else
       Idx = Builder.CreateNSWMul(Idx, NumElements);
     EltPtr = emitArraySubscriptGEP(*this, Base, Idx, VLA->getElementType(),
-                                   !getLangOpts().isSignedOverflowDefined(),
+                                   !getLangOpts().PointerOverflowDefined,
                                    /*signedIndices=*/false, E->getExprLoc());
   } else if (const Expr *Array = isSimpleArrayDecayOperand(E->getBase())) {
     // If this is A[i] where A is an array, the frontend will have decayed the
@@ -4647,7 +4657,7 @@ LValue CodeGenFunction::EmitArraySectionExpr(const ArraySectionExpr *E,
     // Propagate the alignment from the array itself to the result.
     EltPtr = emitArraySubscriptGEP(
         *this, ArrayLV.getAddress(), {CGM.getSize(CharUnits::Zero()), Idx},
-        ResultExprTy, !getLangOpts().isSignedOverflowDefined(),
+        ResultExprTy, !getLangOpts().PointerOverflowDefined,
         /*signedIndices=*/false, E->getExprLoc());
     BaseInfo = ArrayLV.getBaseInfo();
     TBAAInfo = CGM.getTBAAInfoForSubobject(ArrayLV, ResultExprTy);
@@ -4656,7 +4666,7 @@ LValue CodeGenFunction::EmitArraySectionExpr(const ArraySectionExpr *E,
         emitOMPArraySectionBase(*this, E->getBase(), BaseInfo, TBAAInfo, BaseTy,
                                 ResultExprTy, IsLowerBound);
     EltPtr = emitArraySubscriptGEP(*this, Base, Idx, ResultExprTy,
-                                   !getLangOpts().isSignedOverflowDefined(),
+                                   !getLangOpts().PointerOverflowDefined,
                                    /*signedIndices=*/false, E->getExprLoc());
   }
 
@@ -5328,6 +5338,7 @@ LValue CodeGenFunction::EmitCastLValue(const CastExpr *E) {
   case CK_MatrixCast:
   case CK_HLSLVectorTruncation:
   case CK_HLSLArrayRValue:
+  case CK_HLSLElementwiseCast:
     return EmitUnsupportedLValue(E, "unexpected cast lvalue");
 
   case CK_Dependent:
@@ -6366,3 +6377,75 @@ RValue CodeGenFunction::EmitPseudoObjectRValue(const PseudoObjectExpr *E,
 LValue CodeGenFunction::EmitPseudoObjectLValue(const PseudoObjectExpr *E) {
   return emitPseudoObjectExpr(*this, E, true, AggValueSlot::ignored()).LV;
 }
+
+void CodeGenFunction::FlattenAccessAndType(
+    Address Addr, QualType AddrType,
+    SmallVectorImpl<std::pair<Address, llvm::Value *>> &AccessList,
+    SmallVectorImpl<QualType> &FlatTypes) {
+  // WorkList is list of type we are processing + the Index List to access
+  // the field of that type in Addr for use in a GEP
+  llvm::SmallVector<std::pair<QualType, llvm::SmallVector<llvm::Value *, 4>>,
+                    16>
+      WorkList;
+  llvm::IntegerType *IdxTy = llvm::IntegerType::get(getLLVMContext(), 32);
+  // Addr should be a pointer so we need to 'dereference' it
+  WorkList.push_back({AddrType, {llvm::ConstantInt::get(IdxTy, 0)}});
+
+  while (!WorkList.empty()) {
+    auto [T, IdxList] = WorkList.pop_back_val();
+    T = T.getCanonicalType().getUnqualifiedType();
+    assert(!isa<MatrixType>(T) && "Matrix types not yet supported in HLSL");
+    if (const auto *CAT = dyn_cast<ConstantArrayType>(T)) {
+      uint64_t Size = CAT->getZExtSize();
+      for (int64_t I = Size - 1; I > -1; I--) {
+        llvm::SmallVector<llvm::Value *, 4> IdxListCopy = IdxList;
+        IdxListCopy.push_back(llvm::ConstantInt::get(IdxTy, I));
+        WorkList.emplace_back(CAT->getElementType(), IdxListCopy);
+      }
+    } else if (const auto *RT = dyn_cast<RecordType>(T)) {
+      const RecordDecl *Record = RT->getDecl();
+      assert(!Record->isUnion() && "Union types not supported in flat cast.");
+
+      const CXXRecordDecl *CXXD = dyn_cast<CXXRecordDecl>(Record);
+
+      llvm::SmallVector<QualType, 16> FieldTypes;
+      if (CXXD && CXXD->isStandardLayout())
+        Record = CXXD->getStandardLayoutBaseWithFields();
+
+      // deal with potential base classes
+      if (CXXD && !CXXD->isStandardLayout()) {
+        for (auto &Base : CXXD->bases())
+          FieldTypes.push_back(Base.getType());
+      }
+
+      for (auto *FD : Record->fields())
+        FieldTypes.push_back(FD->getType());
+
+      for (int64_t I = FieldTypes.size() - 1; I > -1; I--) {
+        llvm::SmallVector<llvm::Value *, 4> IdxListCopy = IdxList;
+        IdxListCopy.push_back(llvm::ConstantInt::get(IdxTy, I));
+        WorkList.insert(WorkList.end(), {FieldTypes[I], IdxListCopy});
+      }
+    } else if (const auto *VT = dyn_cast<VectorType>(T)) {
+      llvm::Type *LLVMT = ConvertTypeForMem(T);
+      CharUnits Align = getContext().getTypeAlignInChars(T);
+      Address GEP =
+          Builder.CreateInBoundsGEP(Addr, IdxList, LLVMT, Align, "vector.gep");
+      for (unsigned I = 0, E = VT->getNumElements(); I < E; I++) {
+        llvm::Value *Idx = llvm::ConstantInt::get(IdxTy, I);
+        // gep on vector fields is not recommended so combine gep with
+        // extract/insert
+        AccessList.emplace_back(GEP, Idx);
+        FlatTypes.push_back(VT->getElementType());
+      }
+    } else {
+      // a scalar/builtin type
+      llvm::Type *LLVMT = ConvertTypeForMem(T);
+      CharUnits Align = getContext().getTypeAlignInChars(T);
+      Address GEP =
+          Builder.CreateInBoundsGEP(Addr, IdxList, LLVMT, Align, "gep");
+      AccessList.emplace_back(GEP, nullptr);
+      FlatTypes.push_back(T);
+    }
+  }
+}
diff --git a/clang/lib/CodeGen/CGExprAgg.cpp b/clang/lib/CodeGen/CGExprAgg.cpp
index 2ad6587089f10..c3f1cbed6b39f 100644
--- a/clang/lib/CodeGen/CGExprAgg.cpp
+++ b/clang/lib/CodeGen/CGExprAgg.cpp
@@ -491,6 +491,79 @@ static bool isTrivialFiller(Expr *E) {
   return false;
 }
 
+// emit a flat cast where the RHS is a scalar, including vector
+static void EmitHLSLScalarFlatCast(CodeGenFunction &CGF, Address DestVal,
+                                   QualType DestTy, llvm::Value *SrcVal,
+                                   QualType SrcTy, SourceLocation Loc) {
+  // Flatten our destination
+  SmallVector<QualType, 16> DestTypes; // Flattened type
+  SmallVector<std::pair<Address, llvm::Value *>, 16> StoreGEPList;
+  // ^^ Flattened accesses to DestVal we want to store into
+  CGF.FlattenAccessAndType(DestVal, DestTy, StoreGEPList, DestTypes);
+
+  assert(SrcTy->isVectorType() && "HLSL Flat cast doesn't handle splatting.");
+  const VectorType *VT = SrcTy->getAs<VectorType>();
+  SrcTy = VT->getElementType();
+  assert(StoreGEPList.size() <= VT->getNumElements() &&
+         "Cannot perform HLSL flat cast when vector source \
+         object has less elements than flattened destination \
+         object.");
+  for (unsigned I = 0, Size = StoreGEPList.size(); I < Size; I++) {
+    llvm::Value *Load = CGF.Builder.CreateExtractElement(SrcVal, I, "vec.load");
+    llvm::Value *Cast =
+        CGF.EmitScalarConversion(Load, SrcTy, DestTypes[I], Loc);
+
+    // store back
+    llvm::Value *Idx = StoreGEPList[I].second;
+    if (Idx) {
+      llvm::Value *V =
+          CGF.Builder.CreateLoad(StoreGEPList[I].first, "load.for.insert");
+      Cast = CGF.Builder.CreateInsertElement(V, Cast, Idx);
+    }
+    CGF.Builder.CreateStore(Cast, StoreGEPList[I].first);
+  }
+  return;
+}
+
+// emit a flat cast where the RHS is an aggregate
+static void EmitHLSLElementwiseCast(CodeGenFunction &CGF, Address DestVal,
+                                    QualType DestTy, Address SrcVal,
+                                    QualType SrcTy, SourceLocation Loc) {
+  // Flatten our destination
+  SmallVector<QualType, 16> DestTypes; // Flattened type
+  SmallVector<std::pair<Address, llvm::Value *>, 16> StoreGEPList;
+  // ^^ Flattened accesses to DestVal we want to store into
+  CGF.FlattenAccessAndType(DestVal, DestTy, StoreGEPList, DestTypes);
+  // Flatten our src
+  SmallVector<QualType, 16> SrcTypes; // Flattened type
+  SmallVector<std::pair<Address, llvm::Value *>, 16> LoadGEPList;
+  // ^^ Flattened accesses to SrcVal we want to load from
+  CGF.FlattenAccessAndType(SrcVal, SrcTy, LoadGEPList, SrcTypes);
+
+  assert(StoreGEPList.size() <= LoadGEPList.size() &&
+         "Cannot perform HLSL flat cast when flattened source object \
+          has less elements than flattened destination object.");
+  // apply casts to what we load from LoadGEPList
+  // and store result in Dest
+  for (unsigned I = 0, E = StoreGEPList.size(); I < E; I++) {
+    llvm::Value *Idx = LoadGEPList[I].second;
+    llvm::Value *Load = CGF.Builder.CreateLoad(LoadGEPList[I].first, "load");
+    Load =
+        Idx ? CGF.Builder.CreateExtractElement(Load, Idx, "vec.extract") : Load;
+    llvm::Value *Cast =
+        CGF.EmitScalarConversion(Load, SrcTypes[I], DestTypes[I], Loc);
+
+    // store back
+    Idx = StoreGEPList[I].second;
+    if (Idx) {
+      llvm::Value *V =
+          CGF.Builder.CreateLoad(StoreGEPList[I].first, "load.for.insert");
+      Cast = CGF.Builder.CreateInsertElement(V, Cast, Idx);
+    }
+    CGF.Builder.CreateStore(Cast, StoreGEPList[I].first);
+  }
+}
+
 /// Emit initialization of an array from an initializer list. ExprToVisit must
 /// be either an InitListEpxr a CXXParenInitListExpr.
 void AggExprEmitter::EmitArrayInit(Address DestPtr, llvm::ArrayType *AType,
@@ -890,7 +963,25 @@ void AggExprEmitter::VisitCastExpr(CastExpr *E) {
   case CK_HLSLArrayRValue:
     Visit(E->getSubExpr());
     break;
-
+  case CK_HLSLElementwiseCast: {
+    Expr *Src = E->getSubExpr();
+    QualType SrcTy = Src->getType();
+    RValue RV = CGF.EmitAnyExpr(Src);
+    QualType DestTy = E->getType();
+    Address DestVal = Dest.getAddress();
+    SourceLocation Loc = E->getExprLoc();
+
+    if (RV.isScalar()) {
+      llvm::Value *SrcVal = RV.getScalarVal();
+      EmitHLSLScalarFlatCast(CGF, DestVal, DestTy, SrcVal, SrcTy, Loc);
+    } else {
+      assert(RV.isAggregate() &&
+             "Can't perform HLSL Aggregate cast on a complex type.");
+      Address SrcVal = RV.getAggregateAddress();
+      EmitHLSLElementwiseCast(CGF, DestVal, DestTy, SrcVal, SrcTy, Loc);
+    }
+    break;
+  }
   case CK_NoOp:
   case CK_UserDefinedConversion:
   case CK_ConstructorConversion:
@@ -1461,6 +1552,7 @@ static bool castPreservesZero(const CastExpr *CE) {
   case CK_NonAtomicToAtomic:
   case CK_AtomicToNonAtomic:
   case CK_HLSLVectorTruncation:
+  case CK_HLSLElementwiseCast:
     return true;
 
   case CK_BaseToDerivedMemberPointer:
diff --git a/clang/lib/CodeGen/CGExprComplex.cpp b/clang/lib/CodeGen/CGExprComplex.cpp
index ac31dff11b585..c2679ea92dc97 100644
--- a/clang/lib/CodeGen/CGExprComplex.cpp
+++ b/clang/lib/CodeGen/CGExprComplex.cpp
@@ -610,6 +610,7 @@ ComplexPairTy ComplexExprEmitter::EmitCast(CastKind CK, Expr *Op,
   case CK_MatrixCast:
   case CK_HLSLVectorTruncation:
   case CK_HLSLArrayRValue:
+  case CK_HLSLElementwiseCast:
     llvm_unreachable("invalid cast kind for complex value");
 
   case CK_FloatingRealToComplex:
diff --git a/clang/lib/CodeGen/CGExprConstant.cpp b/clang/lib/CodeGen/CGExprConstant.cpp
index 655fc3dc954c8..ef11798869d3b 100644
--- a/clang/lib/CodeGen/CGExprConstant.cpp
+++ b/clang/lib/CodeGen/CGExprConstant.cpp
@@ -1335,6 +1335,7 @@ class ConstExprEmitter
     case CK_MatrixCast:
     case CK_HLSLVectorTruncation:
     case CK_HLSLArrayRValue:
+    case CK_HLSLElementwiseCast:
       return nullptr;
     }
     llvm_unreachable("Invalid CastKind");
diff --git a/clang/lib/CodeGen/CGExprScalar.cpp b/clang/lib/CodeGen/CGExprScalar.cpp
index ac499e490ee87..80daed7e53951 100644
--- a/clang/lib/CodeGen/CGExprScalar.cpp
+++ b/clang/lib/CodeGen/CGExprScalar.cpp
@@ -2269,6 +2269,42 @@ bool CodeGenFunction::ShouldNullCheckClassCastValue(const CastExpr *CE) {
   return true;
 }
 
+// RHS is an aggregate type
+static Value *EmitHLSLElementwiseCast(CodeGenFunction &CGF, Address RHSVal,
+                                      QualType RHSTy, QualType LHSTy,
+                                      SourceLocation Loc) {
+  SmallVector<std::pair<Address, llvm::Value *>, 16> LoadGEPList;
+  SmallVector<QualType, 16> SrcTypes; // Flattened type
+  CGF.FlattenAccessAndType(RHSVal, RHSTy, LoadGEPList, SrcTypes);
+  // LHS is either a vector or a builtin?
+  // if its a vector create a temp alloca to store into and return that
+  if (auto *VecTy = LHSTy->getAs<VectorType>()) {
+    assert(SrcTypes.size() >= VecTy->getNumElements() &&
+           "Flattened type on RHS must have more elements than vector on LHS.");
+    llvm::Value *V =
+        CGF.Builder.CreateLoad(CGF.CreateIRTemp(LHSTy, "flatcast.tmp"));
+    // write to V.
+    for (unsigned I = 0, E = VecTy->getNumElements(); I < E; I++) {
+      llvm::Value *Load = CGF.Builder.CreateLoad(LoadGEPList[I].first, "load");
+      llvm::Value *Idx = LoadGEPList[I].second;
+      Load = Idx ? CGF.Builder.CreateExtractElement(Load, Idx, "vec.extract")
+                 : Load;
+      llvm::Value *Cast = CGF.EmitScalarConversion(
+          Load, SrcTypes[I], VecTy->getElementType(), Loc);
+      V = CGF.Builder.CreateInsertElement(V, Cast, I);
+    }
+    return V;
+  }
+  // i its a builtin just do an extract element or load.
+  assert(LHSTy->isBuiltinType() &&
+         "Destination type must be a vector or builtin type.");
+  llvm::Value *Load = CGF.Builder.CreateLoad(LoadGEPList[0].first, "load");
+  llvm::Value *Idx = LoadGEPList[0].second;
+  Load =
+      Idx ? CGF.Builder.CreateExtractElement(Load, Idx, "vec.extract") : Load;
+  return CGF.EmitScalarConversion(Load, LHSTy, SrcTypes[0], Loc);
+}
+
 // VisitCastExpr - Emit code for an explicit or implicit cast.  Implicit casts
 // have to handle a more broad range of conversions than explicit casts, as they
 // handle things like function to ptr-to-function decay etc.
@@ -2759,7 +2795,16 @@ Value *ScalarExprEmitter::VisitCastExpr(CastExpr *CE) {
     llvm::Value *Zero = llvm::Constant::getNullValue(CGF.SizeTy);
     return Builder.CreateExtractElement(Vec, Zero, "cast.vtrunc");
   }
+  case CK_HLSLElementwiseCast: {
+    RValue RV = CGF.EmitAnyExpr(E);
+    SourceLocation Loc = CE->getExprLoc();
+    QualType SrcTy = E->getType();
 
+    assert(RV.isAggregate() && "Not a valid HLSL Flat Cast.");
+    // RHS is an aggregate
+    Address SrcVal = RV.getAggregateAddress();
+    return EmitHLSLElementwiseCast(CGF, SrcVal, SrcTy, DestTy, Loc);
+  }
   } // end of switch
 
   llvm_unreachable("unknown scalar cast");
@@ -3043,7 +3088,7 @@ ScalarExprEmitter::EmitScalarPrePostIncDec(const UnaryOperator *E, LValue LV,
       llvm::Value *numElts = CGF.getVLASize(vla).NumElts;
       if (!isInc) numElts = Builder.CreateNSWNeg(numElts, "vla.negsize");
       llvm::Type *elemTy = CGF.ConvertTypeForMem(vla->getElementType());
-      if (CGF.getLangOpts().isSignedOverflowDefined())
+      if (CGF.getLangOpts().PointerOverflowDefined)
         value = Builder.CreateGEP(elemTy, value, numElts, "vla.inc");
       else
         value = CGF.EmitCheckedInBoundsGEP(
@@ -3054,7 +3099,7 @@ ScalarExprEmitter::EmitScalarPrePostIncDec(const UnaryOperator *E, LValue LV,
     } else if (type->isFunctionType()) {
       llvm::Value *amt = Builder.getInt32(amount);
 
-      if (CGF.getLangOpts().isSignedOverflowDefined())
+      if (CGF.getLangOpts().PointerOverflowDefined)
         value = Builder.CreateGEP(CGF.Int8Ty, value, amt, "incdec.funcptr");
       else
         value =
@@ -3066,7 +3111,7 @@ ScalarExprEmitter::EmitScalarPrePostIncDec(const UnaryOperator *E, LValue LV,
     } else {
       llvm::Value *amt = Builder.getInt32(amount);
       llvm::Type *elemTy = CGF.ConvertTypeForMem(type);
-      if (CGF.getLangOpts().isSignedOverflowDefined())
+      if (CGF.getLangOpts().PointerOverflowDefined)
         value = Builder.CreateGEP(elemTy, value, amt, "incdec.ptr");
       else
         value = CGF.EmitCheckedInBoundsGEP(
@@ -3179,7 +3224,7 @@ ScalarExprEmitter::EmitScalarPrePostIncDec(const UnaryOperator *E, LValue LV,
     llvm::Value *sizeValue =
       llvm::ConstantInt::get(CGF.SizeTy, size.getQuantity());
 
-    if (CGF.getLangOpts().isSignedOverflowDefined())
+    if (CGF.getLangOpts().PointerOverflowDefined)
       value = Builder.CreateGEP(CGF.Int8Ty, value, sizeValue, "incdec.objptr");
     else
       value = CGF.EmitCheckedInBoundsGEP(
@@ -4075,7 +4120,7 @@ static Value *emitPointerArithmetic(CodeGenFunction &CGF,
     // signed-overflow, so we use the same semantics for our explicit
     // multiply.  We suppress this if overflow is not undefined behavior.
     llvm::Type *elemTy = CGF.ConvertTypeForMem(vla->getElementType());
-    if (CGF.getLangOpts().isSignedOverflowDefined()) {
+    if (CGF.getLangOpts().PointerOverflowDefined) {
       index = CGF.Builder.CreateMul(index, numElements, "vla.index");
       pointer = CGF.Builder.CreateGEP(elemTy, pointer, index, "add.ptr");
     } else {
@@ -4096,7 +4141,7 @@ static Value *emitPointerArithmetic(CodeGenFunction &CGF,
   else
     elemTy = CGF.ConvertTypeForMem(elementType);
 
-  if (CGF.getLangOpts().isSignedOverflowDefined())
+  if (CGF.getLangOpts().PointerOverflowDefined)
     return CGF.Builder.CreateGEP(elemTy, pointer, index, "add.ptr");
 
   return CGF.EmitCheckedInBoundsGEP(
diff --git a/clang/lib/CodeGen/CGHLSLRuntime.cpp b/clang/lib/CodeGen/CGHLSLRuntime.cpp
index 345e218f42451..6cccd353cff96 100644
--- a/clang/lib/CodeGen/CGHLSLRuntime.cpp
+++ b/clang/lib/CodeGen/CGHLSLRuntime.cpp
@@ -100,22 +100,6 @@ GlobalVariable *replaceBuffer(CGHLSLRuntime::Buffer &Buf) {
       llvm::formatv("{0}{1}", Buf.Name, Buf.IsCBuffer ? ".cb." : ".tb."),
       GlobalValue::NotThreadLocal);
 
-  IRBuilder<> B(CBGV->getContext());
-  Value *ZeroIdx = B.getInt32(0);
-  // Replace Const use with CB use.
-  for (auto &[GV, Offset] : Buf.Constants) {
-    Value *GEP =
-        B.CreateGEP(Buf.LayoutStruct, CBGV, {ZeroIdx, B.getInt32(Offset)});
-
-    assert(Buf.LayoutStruct->getElementType(Offset) == GV->getValueType() &&
-           "constant type mismatch");
-
-    // Replace.
-    GV->replaceAllUsesWith(GEP);
-    // Erase GV.
-    GV->removeDeadConstantUsers();
-    GV->eraseFromParent();
-  }
   return CBGV;
 }
 
@@ -144,6 +128,7 @@ void CGHLSLRuntime::addConstant(VarDecl *D, Buffer &CB) {
   }
 
   auto *GV = cast<GlobalVariable>(CGM.GetAddrOfGlobalVar(D));
+  GV->setExternallyInitialized(true);
   // Add debug info for constVal.
   if (CGDebugInfo *DI = CGM.getModuleDebugInfo())
     if (CGM.getCodeGenOpts().getDebugInfo() >=
@@ -360,6 +345,13 @@ void clang::CodeGen::CGHLSLRuntime::setHLSLEntryAttributes(
                 WaveSizeAttr->getPreferred());
     Fn->addFnAttr(WaveSizeKindStr, WaveSizeStr);
   }
+  // HLSL entry functions are materialized for module functions with
+  // HLSLShaderAttr attribute. SetLLVMFunctionAttributesForDefinition called
+  // later in the compiler-flow for such module functions is not aware of and
+  // hence not able to set attributes of the newly materialized entry functions.
+  // So, set attributes of entry function here, as appropriate.
+  if (CGM.getCodeGenOpts().OptimizationLevel == 0)
+    Fn->addFnAttr(llvm::Attribute::OptimizeNone);
   Fn->addFnAttr(llvm::Attribute::NoInline);
 }
 
diff --git a/clang/lib/CodeGen/CGObjCGNU.cpp b/clang/lib/CodeGen/CGObjCGNU.cpp
index cfc92be393940..ebd88bb38849e 100644
--- a/clang/lib/CodeGen/CGObjCGNU.cpp
+++ b/clang/lib/CodeGen/CGObjCGNU.cpp
@@ -773,7 +773,9 @@ class CGObjCGNUstep : public CGObjCGNU {
 
       // The lookup function is guaranteed not to capture the receiver pointer.
       if (auto *LookupFn2 = dyn_cast<llvm::Function>(LookupFn.getCallee()))
-        LookupFn2->addParamAttr(0, llvm::Attribute::NoCapture);
+        LookupFn2->addParamAttr(
+            0, llvm::Attribute::getWithCaptureInfo(CGF.getLLVMContext(),
+                                                   llvm::CaptureInfo::none()));
 
       llvm::Value *args[] = {
           EnforceType(Builder, ReceiverPtr.getPointer(), PtrToIdTy),
diff --git a/clang/lib/CodeGen/CGObjCMac.cpp b/clang/lib/CodeGen/CGObjCMac.cpp
index dd900f9b32fb7..6c929a6431c0f 100644
--- a/clang/lib/CodeGen/CGObjCMac.cpp
+++ b/clang/lib/CodeGen/CGObjCMac.cpp
@@ -5717,7 +5717,7 @@ ObjCCommonTypesHelper::ObjCCommonTypesHelper(CodeGen::CodeGenModule &cgm)
   IntTy = CGM.IntTy;
   LongTy = cast<llvm::IntegerType>(Types.ConvertType(Ctx.LongTy));
   Int8PtrTy = CGM.Int8PtrTy;
-  Int8PtrProgramASTy = llvm::PointerType::get(CGM.Int8Ty, ProgramAS);
+  Int8PtrProgramASTy = llvm::PointerType::get(CGM.getLLVMContext(), ProgramAS);
   Int8PtrPtrTy = CGM.Int8PtrPtrTy;
 
   // arm64 targets use "int" ivar offset variables. All others,
diff --git a/clang/lib/CodeGen/CGObjCRuntime.cpp b/clang/lib/CodeGen/CGObjCRuntime.cpp
index b438a92a4fd62..a7f5c913f42fc 100644
--- a/clang/lib/CodeGen/CGObjCRuntime.cpp
+++ b/clang/lib/CodeGen/CGObjCRuntime.cpp
@@ -230,11 +230,14 @@ void CGObjCRuntime::EmitTryCatchStmt(CodeGenFunction &CGF,
     CodeGenFunction::LexicalScope Cleanups(CGF, Handler.Body->getSourceRange());
     SaveAndRestore RevertAfterScope(CGF.CurrentFuncletPad);
     if (useFunclets) {
-      llvm::Instruction *CPICandidate = Handler.Block->getFirstNonPHI();
-      if (auto *CPI = dyn_cast_or_null<llvm::CatchPadInst>(CPICandidate)) {
-        CGF.CurrentFuncletPad = CPI;
-        CPI->setOperand(2, CGF.getExceptionSlot().emitRawPointer(CGF));
-        CGF.EHStack.pushCleanup<CatchRetScope>(NormalCleanup, CPI);
+      llvm::BasicBlock::iterator CPICandidate =
+          Handler.Block->getFirstNonPHIIt();
+      if (CPICandidate != Handler.Block->end()) {
+        if (auto *CPI = dyn_cast_or_null<llvm::CatchPadInst>(CPICandidate)) {
+          CGF.CurrentFuncletPad = CPI;
+          CPI->setOperand(2, CGF.getExceptionSlot().emitRawPointer(CGF));
+          CGF.EHStack.pushCleanup<CatchRetScope>(NormalCleanup, CPI);
+        }
       }
     }
 
diff --git a/clang/lib/CodeGen/CGOpenCLRuntime.cpp b/clang/lib/CodeGen/CGOpenCLRuntime.cpp
index 115b618056a44..9f8ff488755ec 100644
--- a/clang/lib/CodeGen/CGOpenCLRuntime.cpp
+++ b/clang/lib/CodeGen/CGOpenCLRuntime.cpp
@@ -130,10 +130,11 @@ void CGOpenCLRuntime::recordBlockInfo(const BlockExpr *E,
   assert(!EnqueuedBlockMap.contains(E) && "Block expression emitted twice");
   assert(isa<llvm::Function>(InvokeF) && "Invalid invoke function");
   assert(Block->getType()->isPointerTy() && "Invalid block literal type");
-  EnqueuedBlockMap[E].InvokeFunc = InvokeF;
-  EnqueuedBlockMap[E].BlockArg = Block;
-  EnqueuedBlockMap[E].BlockTy = BlockTy;
-  EnqueuedBlockMap[E].KernelHandle = nullptr;
+  EnqueuedBlockInfo &BlockInfo = EnqueuedBlockMap[E];
+  BlockInfo.InvokeFunc = InvokeF;
+  BlockInfo.BlockArg = Block;
+  BlockInfo.BlockTy = BlockTy;
+  BlockInfo.KernelHandle = nullptr;
 }
 
 llvm::Function *CGOpenCLRuntime::getInvokeFunction(const Expr *E) {
@@ -148,17 +149,19 @@ CGOpenCLRuntime::emitOpenCLEnqueuedBlock(CodeGenFunction &CGF, const Expr *E) {
   // to get the block literal.
   const BlockExpr *Block = getBlockExpr(E);
 
-  assert(EnqueuedBlockMap.contains(Block) && "Block expression not emitted");
+  auto It = EnqueuedBlockMap.find(Block);
+  assert(It != EnqueuedBlockMap.end() && "Block expression not emitted");
+  EnqueuedBlockInfo &BlockInfo = It->second;
 
   // Do not emit the block wrapper again if it has been emitted.
-  if (EnqueuedBlockMap[Block].KernelHandle) {
-    return EnqueuedBlockMap[Block];
+  if (BlockInfo.KernelHandle) {
+    return BlockInfo;
   }
 
   auto *F = CGF.getTargetHooks().createEnqueuedBlockKernel(
-      CGF, EnqueuedBlockMap[Block].InvokeFunc, EnqueuedBlockMap[Block].BlockTy);
+      CGF, BlockInfo.InvokeFunc, BlockInfo.BlockTy);
 
   // The common part of the post-processing of the kernel goes here.
-  EnqueuedBlockMap[Block].KernelHandle = F;
-  return EnqueuedBlockMap[Block];
+  BlockInfo.KernelHandle = F;
+  return BlockInfo;
 }
diff --git a/clang/lib/CodeGen/CGOpenMPRuntime.cpp b/clang/lib/CodeGen/CGOpenMPRuntime.cpp
index cafaaa364cb76..b679d63874b3b 100644
--- a/clang/lib/CodeGen/CGOpenMPRuntime.cpp
+++ b/clang/lib/CodeGen/CGOpenMPRuntime.cpp
@@ -1457,14 +1457,13 @@ void CGOpenMPRuntime::functionFinished(CodeGenFunction &CGF) {
     clearLocThreadIdInsertPt(CGF);
     OpenMPLocThreadIDMap.erase(CGF.CurFn);
   }
-  if (FunctionUDRMap.count(CGF.CurFn) > 0) {
-    for(const auto *D : FunctionUDRMap[CGF.CurFn])
+  if (auto I = FunctionUDRMap.find(CGF.CurFn); I != FunctionUDRMap.end()) {
+    for (const auto *D : I->second)
       UDRMap.erase(D);
-    FunctionUDRMap.erase(CGF.CurFn);
+    FunctionUDRMap.erase(I);
   }
-  auto I = FunctionUDMMap.find(CGF.CurFn);
-  if (I != FunctionUDMMap.end()) {
-    for(const auto *D : I->second)
+  if (auto I = FunctionUDMMap.find(CGF.CurFn); I != FunctionUDMMap.end()) {
+    for (const auto *D : I->second)
       UDMMap.erase(D);
     FunctionUDMMap.erase(I);
   }
diff --git a/clang/lib/CodeGen/CGStmt.cpp b/clang/lib/CodeGen/CGStmt.cpp
index 7c944fe85a352..c96301c306d41 100644
--- a/clang/lib/CodeGen/CGStmt.cpp
+++ b/clang/lib/CodeGen/CGStmt.cpp
@@ -489,6 +489,8 @@ void CodeGenFunction::EmitStmt(const Stmt *S, ArrayRef<const Attr *> Attrs) {
   case Stmt::OpenACCUpdateConstructClass:
     EmitOpenACCUpdateConstruct(cast<OpenACCUpdateConstruct>(*S));
     break;
+  case Stmt::OpenACCAtomicConstructClass:
+    EmitOpenACCAtomicConstruct(cast<OpenACCAtomicConstruct>(*S));
   }
 }
 
@@ -3303,18 +3305,9 @@ CodeGenFunction::addConvergenceControlToken(llvm::CallBase *Input) {
 
 llvm::ConvergenceControlInst *
 CodeGenFunction::emitConvergenceLoopToken(llvm::BasicBlock *BB) {
-  CGBuilderTy::InsertPoint IP = Builder.saveIP();
-  if (BB->empty())
-    Builder.SetInsertPoint(BB);
-  else
-    Builder.SetInsertPoint(BB->getFirstInsertionPt());
-
-  llvm::CallBase *CB = Builder.CreateIntrinsic(
-      llvm::Intrinsic::experimental_convergence_loop, {}, {});
-  Builder.restoreIP(IP);
-
-  CB = addConvergenceControlToken(CB);
-  return cast<llvm::ConvergenceControlInst>(CB);
+  llvm::ConvergenceControlInst *ParentToken = ConvergenceTokenStack.back();
+  assert(ParentToken);
+  return llvm::ConvergenceControlInst::CreateLoop(*BB, ParentToken);
 }
 
 llvm::ConvergenceControlInst *
@@ -3327,13 +3320,5 @@ CodeGenFunction::getOrEmitConvergenceEntryToken(llvm::Function *F) {
   // Adding a convergence token requires the function to be marked as
   // convergent.
   F->setConvergent();
-
-  CGBuilderTy::InsertPoint IP = Builder.saveIP();
-  Builder.SetInsertPoint(&BB->front());
-  llvm::CallBase *I = Builder.CreateIntrinsic(
-      llvm::Intrinsic::experimental_convergence_entry, {}, {});
-  assert(isa<llvm::IntrinsicInst>(I));
-  Builder.restoreIP(IP);
-
-  return cast<llvm::ConvergenceControlInst>(I);
+  return llvm::ConvergenceControlInst::CreateEntry(*BB);
 }
diff --git a/clang/lib/CodeGen/CodeGenFunction.cpp b/clang/lib/CodeGen/CodeGenFunction.cpp
index 11fdddba1144b..08165e0b28406 100644
--- a/clang/lib/CodeGen/CodeGenFunction.cpp
+++ b/clang/lib/CodeGen/CodeGenFunction.cpp
@@ -404,9 +404,9 @@ void CodeGenFunction::FinishFunction(SourceLocation EndLoc) {
   // important to do this before we enter the return block or return
   // edges will be *really* confused.
   bool HasCleanups = EHStack.stable_begin() != PrologueCleanupDepth;
-  bool HasOnlyLifetimeMarkers =
-      HasCleanups && EHStack.containsOnlyLifetimeMarkers(PrologueCleanupDepth);
-  bool EmitRetDbgLoc = !HasCleanups || HasOnlyLifetimeMarkers;
+  bool HasOnlyNoopCleanups =
+      HasCleanups && EHStack.containsOnlyNoopCleanups(PrologueCleanupDepth);
+  bool EmitRetDbgLoc = !HasCleanups || HasOnlyNoopCleanups;
 
   std::optional<ApplyDebugLocation> OAL;
   if (HasCleanups) {
@@ -551,14 +551,6 @@ void CodeGenFunction::FinishFunction(SourceLocation EndLoc) {
     CurFn->addFnAttr("min-legal-vector-width",
                      llvm::utostr(LargestVectorWidth));
 
-  // Add vscale_range attribute if appropriate.
-  std::optional<std::pair<unsigned, unsigned>> VScaleRange =
-      getContext().getTargetInfo().getVScaleRange(getLangOpts());
-  if (VScaleRange) {
-    CurFn->addFnAttr(llvm::Attribute::getWithVScaleRangeArgs(
-        getLLVMContext(), VScaleRange->first, VScaleRange->second));
-  }
-
   // If we generated an unreachable return block, delete it now.
   if (ReturnBlock.isValid() && ReturnBlock.getBlock()->use_empty()) {
     Builder.ClearInsertionPoint();
@@ -1110,6 +1102,15 @@ void CodeGenFunction::StartFunction(GlobalDecl GD, QualType RetTy,
   if (FD && FD->isMain())
     Fn->removeFnAttr("zero-call-used-regs");
 
+  // Add vscale_range attribute if appropriate.
+  std::optional<std::pair<unsigned, unsigned>> VScaleRange =
+      getContext().getTargetInfo().getVScaleRange(
+          getLangOpts(), FD ? IsArmStreamingFunction(FD, true) : false);
+  if (VScaleRange) {
+    CurFn->addFnAttr(llvm::Attribute::getWithVScaleRangeArgs(
+        getLLVMContext(), VScaleRange->first, VScaleRange->second));
+  }
+
   llvm::BasicBlock *EntryBB = createBasicBlock("entry", CurFn);
 
   // Create a marker to make it easy to insert allocas into the entryblock
diff --git a/clang/lib/CodeGen/CodeGenFunction.h b/clang/lib/CodeGen/CodeGenFunction.h
index fab27d4c22ed8..e7a5100a9fa29 100644
--- a/clang/lib/CodeGen/CodeGenFunction.h
+++ b/clang/lib/CodeGen/CodeGenFunction.h
@@ -451,7 +451,7 @@ class CodeGenFunction : public CodeGenTypeCache {
              "EBB should be entry block of the current code gen function");
       PostAllocaInsertPt = AllocaInsertPt->clone();
       PostAllocaInsertPt->setName("postallocapt");
-      PostAllocaInsertPt->insertAfter(AllocaInsertPt);
+      PostAllocaInsertPt->insertAfter(AllocaInsertPt->getIterator());
     }
 
     return PostAllocaInsertPt;
@@ -723,6 +723,20 @@ class CodeGenFunction : public CodeGenTypeCache {
     }
   };
 
+  // We are using objects of this 'cleanup' class to emit fake.use calls
+  // for -fextend-variable-liveness. They are placed at the end of a variable's
+  // scope analogous to lifetime markers.
+  class FakeUse final : public EHScopeStack::Cleanup {
+    Address Addr;
+
+  public:
+    FakeUse(Address addr) : Addr(addr) {}
+
+    void Emit(CodeGenFunction &CGF, Flags flags) override {
+      CGF.EmitFakeUse(Addr);
+    }
+  };
+
   /// Header for data within LifetimeExtendedCleanupStack.
   struct LifetimeExtendedCleanupHeader {
     /// The size of the following cleanup object.
@@ -3310,20 +3324,11 @@ class CodeGenFunction : public CodeGenTypeCache {
                            llvm::Value *Index, QualType IndexType,
                            QualType IndexedType, bool Accessed);
 
-  // Find a struct's flexible array member and get its offset. It may be
-  // embedded inside multiple sub-structs, but must still be the last field.
-  const FieldDecl *
-  FindFlexibleArrayMemberFieldAndOffset(ASTContext &Ctx, const RecordDecl *RD,
-                                        const FieldDecl *FAMDecl,
-                                        uint64_t &Offset);
-
-  llvm::Value *GetCountedByFieldExprGEP(const Expr *Base,
-                                        const FieldDecl *FAMDecl,
+  llvm::Value *GetCountedByFieldExprGEP(const Expr *Base, const FieldDecl *FD,
                                         const FieldDecl *CountDecl);
 
   /// Build an expression accessing the "counted_by" field.
-  llvm::Value *EmitLoadOfCountedByField(const Expr *Base,
-                                        const FieldDecl *FAMDecl,
+  llvm::Value *EmitLoadOfCountedByField(const Expr *Base, const FieldDecl *FD,
                                         const FieldDecl *CountDecl);
 
   llvm::Value *EmitScalarPrePostIncDec(const UnaryOperator *E, LValue LV,
@@ -4162,6 +4167,13 @@ class CodeGenFunction : public CodeGenTypeCache {
     // but in the future we will implement some sort of IR.
   }
 
+  void EmitOpenACCAtomicConstruct(const OpenACCAtomicConstruct &S) {
+    // TODO OpenACC: Implement this.  It is currently implemented as a 'no-op',
+    // simply emitting its associated stmt, but in the future we will implement
+    // some sort of IR.
+    EmitStmt(S.getAssociatedStmt());
+  }
+
   //===--------------------------------------------------------------------===//
   //                         LValue Expression Emission
   //===--------------------------------------------------------------------===//
@@ -4427,6 +4439,11 @@ class CodeGenFunction : public CodeGenTypeCache {
                                 AggValueSlot slot = AggValueSlot::ignored());
   LValue EmitPseudoObjectLValue(const PseudoObjectExpr *e);
 
+  void FlattenAccessAndType(
+      Address Addr, QualType AddrTy,
+      SmallVectorImpl<std::pair<Address, llvm::Value *>> &AccessList,
+      SmallVectorImpl<QualType> &FlatTypes);
+
   llvm::Value *EmitIvarOffset(const ObjCInterfaceDecl *Interface,
                               const ObjCIvarDecl *Ivar);
   llvm::Value *EmitIvarOffsetAsPointerDiff(const ObjCInterfaceDecl *Interface,
@@ -4692,6 +4709,21 @@ class CodeGenFunction : public CodeGenTypeCache {
                             SmallVectorImpl<llvm::Value*> &O,
                             const char *name,
                             unsigned shift = 0, bool rightshift = false);
+  llvm::Value *EmitFP8NeonCall(unsigned IID, ArrayRef<llvm::Type *> Tys,
+                               SmallVectorImpl<llvm::Value *> &O,
+                               const CallExpr *E, const char *name);
+  llvm::Value *EmitFP8NeonCvtCall(unsigned IID, llvm::Type *Ty0,
+                                  llvm::Type *Ty1, bool Extract,
+                                  SmallVectorImpl<llvm::Value *> &Ops,
+                                  const CallExpr *E, const char *name);
+  llvm::Value *EmitFP8NeonFDOTCall(unsigned IID, bool ExtendLaneArg,
+                                   llvm::Type *RetTy,
+                                   SmallVectorImpl<llvm::Value *> &Ops,
+                                   const CallExpr *E, const char *name);
+  llvm::Value *EmitFP8NeonFMLACall(unsigned IID, bool ExtendLaneArg,
+                                   llvm::Type *RetTy,
+                                   SmallVectorImpl<llvm::Value *> &Ops,
+                                   const CallExpr *E, const char *name);
   llvm::Value *EmitNeonSplat(llvm::Value *V, llvm::Constant *Idx,
                              const llvm::ElementCount &Count);
   llvm::Value *EmitNeonSplat(llvm::Value *V, llvm::Constant *Idx);
@@ -5060,6 +5092,8 @@ class CodeGenFunction : public CodeGenTypeCache {
 
   RValue EmitAtomicExpr(AtomicExpr *E);
 
+  void EmitFakeUse(Address Addr);
+
   //===--------------------------------------------------------------------===//
   //                         Annotations Emission
   //===--------------------------------------------------------------------===//
@@ -5338,8 +5372,9 @@ class CodeGenFunction : public CodeGenTypeCache {
                                      llvm::Value *EmittedE,
                                      bool IsDynamic);
 
-  llvm::Value *emitFlexibleArrayMemberSize(const Expr *E, unsigned Type,
-                                           llvm::IntegerType *ResType);
+  llvm::Value *emitCountedByMemberSize(const Expr *E, llvm::Value *EmittedE,
+                                       unsigned Type,
+                                       llvm::IntegerType *ResType);
 
   void emitZeroOrPatternForAutoVarInit(QualType type, const VarDecl &D,
                                        Address Loc);
diff --git a/clang/lib/CodeGen/CodeGenModule.cpp b/clang/lib/CodeGen/CodeGenModule.cpp
index eb8d3ceeeba4c..7924c32fcf633 100644
--- a/clang/lib/CodeGen/CodeGenModule.cpp
+++ b/clang/lib/CodeGen/CodeGenModule.cpp
@@ -486,8 +486,10 @@ void CodeGenModule::createOpenMPRuntime() {
   case llvm::Triple::nvptx:
   case llvm::Triple::nvptx64:
   case llvm::Triple::amdgcn:
-    assert(getLangOpts().OpenMPIsTargetDevice &&
-           "OpenMP AMDGPU/NVPTX is only prepared to deal with device code.");
+  case llvm::Triple::spirv64:
+    assert(
+        getLangOpts().OpenMPIsTargetDevice &&
+        "OpenMP AMDGPU/NVPTX/SPIRV is only prepared to deal with device code.");
     OpenMPRuntime.reset(new CGOpenMPRuntimeGPU(*this));
     break;
   default:
@@ -1149,6 +1151,8 @@ void CodeGenModule::Release() {
     if (CodeGenOpts.PatchableFunctionEntryOffset)
       getModule().addModuleFlag(llvm::Module::Override, "kcfi-offset",
                                 CodeGenOpts.PatchableFunctionEntryOffset);
+    if (CodeGenOpts.SanitizeKcfiArity)
+      getModule().addModuleFlag(llvm::Module::Override, "kcfi-arity", 1);
   }
 
   if (CodeGenOpts.CFProtectionReturn &&
@@ -1293,6 +1297,11 @@ void CodeGenModule::Release() {
   if (LangOpts.EHAsynch)
     getModule().addModuleFlag(llvm::Module::Warning, "eh-asynch", 1);
 
+  // Emit Import Call section.
+  if (CodeGenOpts.ImportCallOptimization)
+    getModule().addModuleFlag(llvm::Module::Warning, "import-call-optimization",
+                              1);
+
   // Indicate whether this Module was compiled with -fopenmp
   if (getLangOpts().OpenMP && !getLangOpts().OpenMPSimd)
     getModule().addModuleFlag(llvm::Module::Max, "openmp", LangOpts.OpenMP);
@@ -3760,7 +3769,7 @@ ConstantAddress CodeGenModule::GetAddrOfTemplateParamObject(
   auto *GV = new llvm::GlobalVariable(getModule(), Init->getType(),
                                       /*isConstant=*/true, Linkage, Init, Name);
   setGVProperties(GV, TPO);
-  if (supportsCOMDAT())
+  if (supportsCOMDAT() && Linkage == llvm::GlobalValue::LinkOnceODRLinkage)
     GV->setComdat(TheModule.getOrInsertComdat(GV->getName()));
   Emitter.finalize(GV);
 
@@ -4010,7 +4019,8 @@ namespace {
       unsigned BuiltinID = FD->getBuiltinID();
       if (!BuiltinID || !BI.isLibFunction(BuiltinID))
         return false;
-      StringRef BuiltinName = BI.getName(BuiltinID);
+      std::string BuiltinNameStr = BI.getName(BuiltinID);
+      StringRef BuiltinName = BuiltinNameStr;
       if (BuiltinName.starts_with("__builtin_") &&
           Name == BuiltinName.slice(strlen("__builtin_"), StringRef::npos)) {
         return true;
@@ -4424,7 +4434,7 @@ void CodeGenModule::emitCPUDispatchDefinition(GlobalDecl GD) {
   GlobalDecl ResolverGD;
   if (getTarget().supportsIFunc()) {
     ResolverType = llvm::FunctionType::get(
-        llvm::PointerType::get(DeclTy,
+        llvm::PointerType::get(getLLVMContext(),
                                getTypes().getTargetAddressSpace(FD->getType())),
         false);
   }
@@ -4596,8 +4606,8 @@ llvm::Constant *CodeGenModule::GetOrCreateMultiVersionResolver(GlobalDecl GD) {
   // cpu_dispatch will be emitted in this translation unit.
   if (ShouldReturnIFunc) {
     unsigned AS = getTypes().getTargetAddressSpace(FD->getType());
-    llvm::Type *ResolverType =
-        llvm::FunctionType::get(llvm::PointerType::get(DeclTy, AS), false);
+    llvm::Type *ResolverType = llvm::FunctionType::get(
+        llvm::PointerType::get(getLLVMContext(), AS), false);
     llvm::Constant *Resolver = GetOrCreateLLVMFunction(
         MangledName + ".resolver", ResolverType, GlobalDecl{},
         /*ForVTable=*/false);
@@ -7014,9 +7024,10 @@ void CodeGenModule::EmitTopLevelDecl(Decl *D) {
   case Decl::VarTemplateSpecialization:
     EmitGlobal(cast<VarDecl>(D));
     if (auto *DD = dyn_cast<DecompositionDecl>(D))
-      for (auto *B : DD->bindings())
+      for (auto *B : DD->flat_bindings())
         if (auto *HD = B->getHoldingVar())
           EmitGlobal(HD);
+
     break;
 
   // Indirect fields from global anonymous structs and unions can be
diff --git a/clang/lib/CodeGen/CodeGenModule.h b/clang/lib/CodeGen/CodeGenModule.h
index 1aa5d483d49c0..0956296e2d5d8 100644
--- a/clang/lib/CodeGen/CodeGenModule.h
+++ b/clang/lib/CodeGen/CodeGenModule.h
@@ -647,6 +647,9 @@ class CodeGenModule : public CodeGenTypeCache {
   /// void @llvm.lifetime.end(i64 %size, i8* nocapture <ptr>)
   llvm::Function *LifetimeEndFn = nullptr;
 
+  /// void @llvm.fake.use(...)
+  llvm::Function *FakeUseFn = nullptr;
+
   std::unique_ptr<SanitizerMetadata> SanitizerMD;
 
   llvm::MapVector<const Decl *, bool> DeferredEmptyCoverageMappingDecls;
@@ -1326,6 +1329,7 @@ class CodeGenModule : public CodeGenTypeCache {
 
   llvm::Function *getLLVMLifetimeStartFn();
   llvm::Function *getLLVMLifetimeEndFn();
+  llvm::Function *getLLVMFakeUseFn();
 
   // Make sure that this type is translated.
   void UpdateCompletedType(const TagDecl *TD);
diff --git a/clang/lib/CodeGen/CodeGenTBAA.cpp b/clang/lib/CodeGen/CodeGenTBAA.cpp
index 75e66bae79afd..3f1a24791ddd8 100644
--- a/clang/lib/CodeGen/CodeGenTBAA.cpp
+++ b/clang/lib/CodeGen/CodeGenTBAA.cpp
@@ -226,6 +226,14 @@ llvm::MDNode *CodeGenTBAA::getTypeInfoHelper(const Type *Ty) {
       PtrDepth++;
       Ty = Ty->getPointeeType()->getBaseElementTypeUnsafe();
     } while (Ty->isPointerType());
+
+    // While there are no special rules in the standards regarding void pointers
+    // and strict aliasing, emitting distinct tags for void pointers break some
+    // common idioms and there is no good alternative to re-write the code
+    // without strict-aliasing violations.
+    if (Ty->isVoidType())
+      return AnyPtr;
+
     assert(!isa<VariableArrayType>(Ty));
     // When the underlying type is a builtin type, we compute the pointee type
     // string recursively, which is implicitly more forgiving than the standards
diff --git a/clang/lib/CodeGen/CodeGenTypes.cpp b/clang/lib/CodeGen/CodeGenTypes.cpp
index 09191a4901f49..405242e97e75c 100644
--- a/clang/lib/CodeGen/CodeGenTypes.cpp
+++ b/clang/lib/CodeGen/CodeGenTypes.cpp
@@ -505,15 +505,18 @@ llvm::Type *CodeGenTypes::ConvertType(QualType T) {
   case BuiltinType::Id:
 #define SVE_PREDICATE_TYPE(Name, MangledName, Id, SingletonId)                 \
   case BuiltinType::Id:
-#define AARCH64_VECTOR_TYPE(Name, MangledName, Id, SingletonId)                \
-  case BuiltinType::Id:
-#define SVE_OPAQUE_TYPE(Name, MangledName, Id, SingletonId)
+#define SVE_TYPE(Name, Id, SingletonId)
 #include "clang/Basic/AArch64SVEACLETypes.def"
       {
         ASTContext::BuiltinVectorTypeInfo Info =
             Context.getBuiltinVectorTypeInfo(cast<BuiltinType>(Ty));
-        auto VTy =
-            llvm::VectorType::get(ConvertType(Info.ElementType), Info.EC);
+        // The `__mfp8` type maps to `<1 x i8>` which can't be used to build
+        // a <N x i8> vector type, hence bypass the call to `ConvertType` for
+        // the element type and create the vector type directly.
+        auto *EltTy = Info.ElementType->isMFloat8Type()
+                          ? llvm::Type::getInt8Ty(getLLVMContext())
+                          : ConvertType(Info.ElementType);
+        auto *VTy = llvm::VectorType::get(EltTy, Info.EC);
         switch (Info.NumVectors) {
         default:
           llvm_unreachable("Expected 1, 2, 3 or 4 vectors!");
@@ -529,6 +532,9 @@ llvm::Type *CodeGenTypes::ConvertType(QualType T) {
       }
     case BuiltinType::SveCount:
       return llvm::TargetExtType::get(getLLVMContext(), "aarch64.svcount");
+    case BuiltinType::MFloat8:
+      return llvm::VectorType::get(llvm::Type::getInt8Ty(getLLVMContext()), 1,
+                                   false);
 #define PPC_VECTOR_TYPE(Name, Id, Size) \
     case BuiltinType::Id: \
       ResultType = \
@@ -650,6 +656,8 @@ llvm::Type *CodeGenTypes::ConvertType(QualType T) {
     // An ext_vector_type of Bool is really a vector of bits.
     llvm::Type *IRElemTy = VT->isExtVectorBoolType()
                                ? llvm::Type::getInt1Ty(getLLVMContext())
+                           : VT->getElementType()->isMFloat8Type()
+                               ? llvm::Type::getInt8Ty(getLLVMContext())
                                : ConvertType(VT->getElementType());
     ResultType = llvm::FixedVectorType::get(IRElemTy, VT->getNumElements());
     break;
diff --git a/clang/lib/CodeGen/EHScopeStack.h b/clang/lib/CodeGen/EHScopeStack.h
index 0c667e80bb6d8..ed11dc2bb05d7 100644
--- a/clang/lib/CodeGen/EHScopeStack.h
+++ b/clang/lib/CodeGen/EHScopeStack.h
@@ -87,6 +87,11 @@ enum CleanupKind : unsigned {
 
   LifetimeMarker = 0x8,
   NormalEHLifetimeMarker = LifetimeMarker | NormalAndEHCleanup,
+
+  // FakeUse needs to be recognized as a special cleanup similar to lifetime
+  // markers chiefly to be ignored in most contexts.
+  FakeUse = 0x10,
+  NormalFakeUse = FakeUse | NormalCleanup,
 };
 
 /// A stack of scopes which respond to exceptions, including cleanups
@@ -352,8 +357,8 @@ class EHScopeStack {
   void popTerminate();
 
   // Returns true iff the current scope is either empty or contains only
-  // lifetime markers, i.e. no real cleanup code
-  bool containsOnlyLifetimeMarkers(stable_iterator Old) const;
+  // noop cleanups, i.e. lifetime markers and fake uses.
+  bool containsOnlyNoopCleanups(stable_iterator Old) const;
 
   /// Determines whether the exception-scopes stack is empty.
   bool empty() const { return StartOfData == EndOfBuffer; }
diff --git a/clang/lib/CodeGen/ItaniumCXXABI.cpp b/clang/lib/CodeGen/ItaniumCXXABI.cpp
index 7c463f51f63dc..7375a511809b9 100644
--- a/clang/lib/CodeGen/ItaniumCXXABI.cpp
+++ b/clang/lib/CodeGen/ItaniumCXXABI.cpp
@@ -2239,8 +2239,8 @@ CGCallee ItaniumCXXABI::getVirtualFunctionPointer(CodeGenFunction &CGF,
 llvm::Value *ItaniumCXXABI::EmitVirtualDestructorCall(
     CodeGenFunction &CGF, const CXXDestructorDecl *Dtor, CXXDtorType DtorType,
     Address This, DeleteOrMemberCallExpr E, llvm::CallBase **CallOrInvoke) {
-  auto *CE = E.dyn_cast<const CXXMemberCallExpr *>();
-  auto *D = E.dyn_cast<const CXXDeleteExpr *>();
+  auto *CE = dyn_cast<const CXXMemberCallExpr *>(E);
+  auto *D = dyn_cast<const CXXDeleteExpr *>(E);
   assert((CE != nullptr) ^ (D != nullptr));
   assert(CE == nullptr || CE->arg_begin() == CE->arg_end());
   assert(DtorType == Dtor_Deleting || DtorType == Dtor_Complete);
diff --git a/clang/lib/CodeGen/MicrosoftCXXABI.cpp b/clang/lib/CodeGen/MicrosoftCXXABI.cpp
index 0d53e8cb45fe7..4a2630e83b62d 100644
--- a/clang/lib/CodeGen/MicrosoftCXXABI.cpp
+++ b/clang/lib/CodeGen/MicrosoftCXXABI.cpp
@@ -1996,8 +1996,8 @@ CGCallee MicrosoftCXXABI::getVirtualFunctionPointer(CodeGenFunction &CGF,
 llvm::Value *MicrosoftCXXABI::EmitVirtualDestructorCall(
     CodeGenFunction &CGF, const CXXDestructorDecl *Dtor, CXXDtorType DtorType,
     Address This, DeleteOrMemberCallExpr E, llvm::CallBase **CallOrInvoke) {
-  auto *CE = E.dyn_cast<const CXXMemberCallExpr *>();
-  auto *D = E.dyn_cast<const CXXDeleteExpr *>();
+  auto *CE = dyn_cast<const CXXMemberCallExpr *>(E);
+  auto *D = dyn_cast<const CXXDeleteExpr *>(E);
   assert((CE != nullptr) ^ (D != nullptr));
   assert(CE == nullptr || CE->arg_begin() == CE->arg_end());
   assert(DtorType == Dtor_Deleting || DtorType == Dtor_Complete);
diff --git a/clang/lib/CodeGen/Targets/AArch64.cpp b/clang/lib/CodeGen/Targets/AArch64.cpp
index 7db67ecba07c8..dc3a1d4287be1 100644
--- a/clang/lib/CodeGen/Targets/AArch64.cpp
+++ b/clang/lib/CodeGen/Targets/AArch64.cpp
@@ -147,8 +147,8 @@ class AArch64TargetCodeGenInfo : public TargetCodeGenInfo {
           CGM.getTarget().parseTargetAttr(TA->getFeaturesStr());
       if (!Attr.BranchProtection.empty()) {
         StringRef Error;
-        (void)CGM.getTarget().validateBranchProtection(Attr.BranchProtection,
-                                                       Attr.CPU, BPI, Error);
+        (void)CGM.getTarget().validateBranchProtection(
+            Attr.BranchProtection, Attr.CPU, BPI, CGM.getLangOpts(), Error);
         assert(Error.empty());
       }
     }
@@ -244,6 +244,7 @@ AArch64ABIInfo::convertFixedToScalableVectorType(const VectorType *VT) const {
 
     case BuiltinType::SChar:
     case BuiltinType::UChar:
+    case BuiltinType::MFloat8:
       return llvm::ScalableVectorType::get(
           llvm::Type::getInt8Ty(getVMContext()), 16);
 
@@ -383,10 +384,6 @@ ABIArgInfo AArch64ABIInfo::classifyArgumentType(QualType Ty, bool IsVariadicFn,
         NSRN = std::min(NSRN + 1, 8u);
       else {
         switch (BT->getKind()) {
-        case BuiltinType::MFloat8x8:
-        case BuiltinType::MFloat8x16:
-          NSRN = std::min(NSRN + 1, 8u);
-          break;
         case BuiltinType::SveBool:
         case BuiltinType::SveCount:
           NPRN = std::min(NPRN + 1, 4u);
@@ -418,18 +415,21 @@ ABIArgInfo AArch64ABIInfo::classifyArgumentType(QualType Ty, bool IsVariadicFn,
                                      CGCXXABI::RAA_DirectInMemory);
   }
 
-  // Empty records are always ignored on Darwin, but actually passed in C++ mode
-  // elsewhere for GNU compatibility.
+  // Empty records:
   uint64_t Size = getContext().getTypeSize(Ty);
   bool IsEmpty = isEmptyRecord(getContext(), Ty, true);
   if (!Ty->isSVESizelessBuiltinType() && (IsEmpty || Size == 0)) {
+    // Empty records are ignored in C mode, and in C++ on Darwin.
     if (!getContext().getLangOpts().CPlusPlus || isDarwinPCS())
       return ABIArgInfo::getIgnore();
 
-    // GNU C mode. The only argument that gets ignored is an empty one with size
-    // 0.
-    if (IsEmpty && Size == 0)
+    // In C++ mode, arguments which have sizeof() == 0 (which are non-standard
+    // C++) are ignored. This isn't defined by any standard, so we copy GCC's
+    // behaviour here.
+    if (Size == 0)
       return ABIArgInfo::getIgnore();
+
+    // Otherwise, they are passed as if they have a size of 1 byte.
     return ABIArgInfo::getDirect(llvm::Type::getInt8Ty(getVMContext()));
   }
 
@@ -629,8 +629,7 @@ bool AArch64ABIInfo::isHomogeneousAggregateBaseType(QualType Ty) const {
   // but with the difference that any floating-point type is allowed,
   // including __fp16.
   if (const BuiltinType *BT = Ty->getAs<BuiltinType>()) {
-    if (BT->isFloatingPoint() || BT->getKind() == BuiltinType::MFloat8x16 ||
-        BT->getKind() == BuiltinType::MFloat8x8)
+    if (BT->isFloatingPoint())
       return true;
   } else if (const VectorType *VT = Ty->getAs<VectorType>()) {
     if (auto Kind = VT->getVectorKind();
@@ -781,8 +780,10 @@ bool AArch64ABIInfo::passAsPureScalableType(
     NPred += Info.NumVectors;
   else
     NVec += Info.NumVectors;
-  auto VTy = llvm::ScalableVectorType::get(CGT.ConvertType(Info.ElementType),
-                                           Info.EC.getKnownMinValue());
+  llvm::Type *EltTy = Info.ElementType->isMFloat8Type()
+                          ? llvm::Type::getInt8Ty(getVMContext())
+                          : CGT.ConvertType(Info.ElementType);
+  auto *VTy = llvm::ScalableVectorType::get(EltTy, Info.EC.getKnownMinValue());
 
   if (CoerceToSeq.size() + Info.NumVectors > 12)
     return false;
@@ -842,7 +843,7 @@ RValue AArch64ABIInfo::EmitAAPCSVAArg(Address VAListAddr, QualType Ty,
 
   llvm::Type *BaseTy = CGF.ConvertType(Ty);
   if (IsIndirect)
-    BaseTy = llvm::PointerType::getUnqual(BaseTy);
+    BaseTy = llvm::PointerType::getUnqual(BaseTy->getContext());
   else if (AI.getCoerceToType())
     BaseTy = AI.getCoerceToType();
 
@@ -960,7 +961,7 @@ RValue AArch64ABIInfo::EmitAAPCSVAArg(Address VAListAddr, QualType Ty,
   if (IsIndirect) {
     // If it's been passed indirectly (actually a struct), whatever we find from
     // stored registers or on the stack will actually be a struct **.
-    MemTy = llvm::PointerType::getUnqual(MemTy);
+    MemTy = llvm::PointerType::getUnqual(MemTy->getContext());
   }
 
   const Type *Base = nullptr;
diff --git a/clang/lib/CodeGen/Targets/ARM.cpp b/clang/lib/CodeGen/Targets/ARM.cpp
index 2d858fa2f3c3a..77641ce10e3d3 100644
--- a/clang/lib/CodeGen/Targets/ARM.cpp
+++ b/clang/lib/CodeGen/Targets/ARM.cpp
@@ -71,6 +71,7 @@ class ARMABIInfo : public ABIInfo {
                                   unsigned functionCallConv) const;
   ABIArgInfo classifyHomogeneousAggregate(QualType Ty, const Type *Base,
                                           uint64_t Members) const;
+  bool shouldIgnoreEmptyArg(QualType Ty) const;
   ABIArgInfo coerceIllegalVector(QualType Ty) const;
   bool isIllegalVectorType(QualType Ty) const;
   bool containsAnyFP16Vectors(QualType Ty) const;
@@ -148,8 +149,8 @@ class ARMTargetCodeGenInfo : public TargetCodeGenInfo {
         StringRef DiagMsg;
         StringRef Arch =
             Attr.CPU.empty() ? CGM.getTarget().getTargetOpts().CPU : Attr.CPU;
-        if (!CGM.getTarget().validateBranchProtection(Attr.BranchProtection,
-                                                      Arch, BPI, DiagMsg)) {
+        if (!CGM.getTarget().validateBranchProtection(
+                Attr.BranchProtection, Arch, BPI, CGM.getLangOpts(), DiagMsg)) {
           CGM.getDiags().Report(
               D->getLocation(),
               diag::warn_target_unsupported_branch_protection_attribute)
@@ -328,6 +329,31 @@ ABIArgInfo ARMABIInfo::classifyHomogeneousAggregate(QualType Ty,
   return ABIArgInfo::getDirect(nullptr, 0, nullptr, false, Align);
 }
 
+bool ARMABIInfo::shouldIgnoreEmptyArg(QualType Ty) const {
+  uint64_t Size = getContext().getTypeSize(Ty);
+  assert((isEmptyRecord(getContext(), Ty, true) || Size == 0) &&
+         "Arg is not empty");
+
+  // Empty records are ignored in C mode, and in C++ on WatchOS.
+  if (!getContext().getLangOpts().CPlusPlus ||
+      getABIKind() == ARMABIKind::AAPCS16_VFP)
+    return true;
+
+  // In C++ mode, arguments which have sizeof() == 0 are ignored. This is not a
+  // situation which is defined by any C++ standard or ABI, but this matches
+  // GCC's de facto ABI.
+  if (Size == 0)
+    return true;
+
+  // Clang 19.0 and earlier always ignored empty struct arguments in C++ mode.
+  if (getContext().getLangOpts().getClangABICompat() <=
+      LangOptions::ClangABI::Ver19)
+    return true;
+
+  // Otherwise, they are passed as if they have a size of 1 byte.
+  return false;
+}
+
 ABIArgInfo ARMABIInfo::classifyArgumentType(QualType Ty, bool isVariadic,
                                             unsigned functionCallConv) const {
   // 6.1.2.1 The following argument types are VFP CPRCs:
@@ -366,9 +392,15 @@ ABIArgInfo ARMABIInfo::classifyArgumentType(QualType Ty, bool isVariadic,
     return getNaturalAlignIndirect(Ty, RAA == CGCXXABI::RAA_DirectInMemory);
   }
 
-  // Ignore empty records.
-  if (isEmptyRecord(getContext(), Ty, true))
-    return ABIArgInfo::getIgnore();
+  // Empty records are either ignored completely or passed as if they were a
+  // 1-byte object, depending on the ABI and language standard.
+  if (isEmptyRecord(getContext(), Ty, true) ||
+      getContext().getTypeSize(Ty) == 0) {
+    if (shouldIgnoreEmptyArg(Ty))
+      return ABIArgInfo::getIgnore();
+    else
+      return ABIArgInfo::getDirect(llvm::Type::getInt8Ty(getVMContext()));
+  }
 
   if (IsAAPCS_VFP) {
     // Homogeneous Aggregates need to be expanded when we can fit the aggregate
@@ -588,7 +620,8 @@ ABIArgInfo ARMABIInfo::classifyReturnType(QualType RetTy, bool isVariadic,
 
   // Otherwise this is an AAPCS variant.
 
-  if (isEmptyRecord(getContext(), RetTy, true))
+  if (isEmptyRecord(getContext(), RetTy, true) ||
+      getContext().getTypeSize(RetTy) == 0)
     return ABIArgInfo::getIgnore();
 
   // Check for homogeneous aggregates with AAPCS-VFP.
@@ -752,7 +785,9 @@ RValue ARMABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,
   CharUnits SlotSize = CharUnits::fromQuantity(4);
 
   // Empty records are ignored for parameter passing purposes.
-  if (isEmptyRecord(getContext(), Ty, true))
+  uint64_t Size = getContext().getTypeSize(Ty);
+  bool IsEmpty = isEmptyRecord(getContext(), Ty, true);
+  if ((IsEmpty || Size == 0) && shouldIgnoreEmptyArg(Ty))
     return Slot.asRValue();
 
   CharUnits TySize = getContext().getTypeSizeInChars(Ty);
diff --git a/clang/lib/CodeGen/Targets/Hexagon.cpp b/clang/lib/CodeGen/Targets/Hexagon.cpp
index 8fd2a81494d99..aada8d0d61303 100644
--- a/clang/lib/CodeGen/Targets/Hexagon.cpp
+++ b/clang/lib/CodeGen/Targets/Hexagon.cpp
@@ -336,10 +336,6 @@ Address HexagonABIInfo::EmitVAArgForHexagonLinux(CodeGenFunction &CGF,
   // Implement the block where argument is in register saved area
   CGF.EmitBlock(InRegBlock);
 
-  llvm::Type *PTy = CGF.ConvertType(Ty);
-  llvm::Value *__saved_reg_area_p = CGF.Builder.CreateBitCast(
-      __current_saved_reg_area_pointer, llvm::PointerType::getUnqual(PTy));
-
   CGF.Builder.CreateStore(__new_saved_reg_area_pointer,
                           __current_saved_reg_area_pointer_p);
 
@@ -388,22 +384,16 @@ Address HexagonABIInfo::EmitVAArgForHexagonLinux(CodeGenFunction &CGF,
   CGF.Builder.CreateStore(__new_overflow_area_pointer,
                           __current_saved_reg_area_pointer_p);
 
-  // Bitcast the overflow area pointer to the type of argument.
-  llvm::Type *OverflowPTy = CGF.ConvertTypeForMem(Ty);
-  llvm::Value *__overflow_area_p = CGF.Builder.CreateBitCast(
-      __overflow_area_pointer, llvm::PointerType::getUnqual(OverflowPTy));
-
   CGF.EmitBranch(ContBlock);
-
   // Get the correct pointer to load the variable argument
   // Implement the ContBlock
   CGF.EmitBlock(ContBlock);
 
   llvm::Type *MemTy = CGF.ConvertTypeForMem(Ty);
-  llvm::Type *MemPTy = llvm::PointerType::getUnqual(MemTy);
-  llvm::PHINode *ArgAddr = CGF.Builder.CreatePHI(MemPTy, 2, "vaarg.addr");
-  ArgAddr->addIncoming(__saved_reg_area_p, InRegBlock);
-  ArgAddr->addIncoming(__overflow_area_p, OnStackBlock);
+  llvm::PHINode *ArgAddr = CGF.Builder.CreatePHI(
+      llvm::PointerType::getUnqual(MemTy->getContext()), 2, "vaarg.addr");
+  ArgAddr->addIncoming(__current_saved_reg_area_pointer, InRegBlock);
+  ArgAddr->addIncoming(__overflow_area_pointer, OnStackBlock);
 
   return Address(ArgAddr, MemTy, CharUnits::fromQuantity(ArgAlign));
 }
diff --git a/clang/lib/CodeGen/Targets/RISCV.cpp b/clang/lib/CodeGen/Targets/RISCV.cpp
index 2b70f2bd3f38b..2c48ba37fd206 100644
--- a/clang/lib/CodeGen/Targets/RISCV.cpp
+++ b/clang/lib/CodeGen/Targets/RISCV.cpp
@@ -367,8 +367,8 @@ ABIArgInfo RISCVABIInfo::coerceVLSVector(QualType Ty) const {
   const auto *VT = Ty->castAs<VectorType>();
   assert(VT->getElementType()->isBuiltinType() && "expected builtin type!");
 
-  auto VScale =
-      getContext().getTargetInfo().getVScaleRange(getContext().getLangOpts());
+  auto VScale = getContext().getTargetInfo().getVScaleRange(
+      getContext().getLangOpts(), false);
 
   unsigned NumElts = VT->getNumElements();
   llvm::Type *EltType = llvm::Type::getInt1Ty(getVMContext());
diff --git a/clang/lib/CodeGen/Targets/SystemZ.cpp b/clang/lib/CodeGen/Targets/SystemZ.cpp
index 23c96fa5cf98c..9bb8ddbc548d2 100644
--- a/clang/lib/CodeGen/Targets/SystemZ.cpp
+++ b/clang/lib/CodeGen/Targets/SystemZ.cpp
@@ -272,7 +272,7 @@ RValue SystemZABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,
   SZCGI.handleExternallyVisibleObjABI(Ty.getTypePtr(), CGT.getCGM(),
                                       /*IsParam*/true);
   if (IsIndirect) {
-    DirectTy = llvm::PointerType::getUnqual(DirectTy);
+    DirectTy = llvm::PointerType::getUnqual(DirectTy->getContext());
     UnpaddedSize = DirectAlign = CharUnits::fromQuantity(8);
   } else {
     if (AI.getCoerceToType())
diff --git a/clang/lib/CodeGen/Targets/X86.cpp b/clang/lib/CodeGen/Targets/X86.cpp
index 5ee5179dd0f3e..7e470abe078f5 100644
--- a/clang/lib/CodeGen/Targets/X86.cpp
+++ b/clang/lib/CodeGen/Targets/X86.cpp
@@ -1334,6 +1334,15 @@ class X86_64ABIInfo : public ABIInfo {
     return T.isOSLinux() || T.isOSNetBSD();
   }
 
+  bool returnCXXRecordGreaterThan128InMem() const {
+    // Clang <= 20.0 did not do this.
+    if (getContext().getLangOpts().getClangABICompat() <=
+        LangOptions::ClangABI::Ver20)
+      return false;
+
+    return true;
+  }
+
   X86AVXABILevel AVXLevel;
   // Some ABIs (e.g. X32 ABI and Native Client OS) use 32 bit pointers on
   // 64-bit hardware.
@@ -2067,6 +2076,13 @@ void X86_64ABIInfo::classify(QualType Ty, uint64_t OffsetBase, Class &Lo,
         classify(I.getType(), Offset, FieldLo, FieldHi, isNamedArg);
         Lo = merge(Lo, FieldLo);
         Hi = merge(Hi, FieldHi);
+        if (returnCXXRecordGreaterThan128InMem() &&
+            (Size > 128 && (Size != getContext().getTypeSize(I.getType()) ||
+                            Size > getNativeVectorSizeForAVXABI(AVXLevel)))) {
+          // The only case a 256(or 512)-bit wide vector could be used to return
+          // is when CXX record contains a single 256(or 512)-bit element.
+          Lo = Memory;
+        }
         if (Lo == Memory || Hi == Memory) {
           postMerge(Size, Lo, Hi);
           return;
diff --git a/clang/lib/CodeGen/Targets/XCore.cpp b/clang/lib/CodeGen/Targets/XCore.cpp
index ced4981fd124f..b7824bde5f55a 100644
--- a/clang/lib/CodeGen/Targets/XCore.cpp
+++ b/clang/lib/CodeGen/Targets/XCore.cpp
@@ -149,7 +149,7 @@ RValue XCoreABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,
   llvm::Type *ArgTy = CGT.ConvertType(Ty);
   if (AI.canHaveCoerceToType() && !AI.getCoerceToType())
     AI.setCoerceToType(ArgTy);
-  llvm::Type *ArgPtrTy = llvm::PointerType::getUnqual(ArgTy);
+  llvm::Type *ArgPtrTy = llvm::PointerType::getUnqual(ArgTy->getContext());
 
   Address Val = Address::invalid();
   CharUnits ArgSize = CharUnits::Zero();
diff --git a/clang/lib/CrossTU/CrossTranslationUnit.cpp b/clang/lib/CrossTU/CrossTranslationUnit.cpp
index 9faf2a8a17341..ad2ebb6cd6e6c 100644
--- a/clang/lib/CrossTU/CrossTranslationUnit.cpp
+++ b/clang/lib/CrossTU/CrossTranslationUnit.cpp
@@ -453,7 +453,8 @@ CrossTranslationUnitContext::ASTUnitStorage::getASTUnitForFunction(
       return std::move(IndexLoadError);
 
     // Check if there is an entry in the index for the function.
-    if (!NameFileMap.count(FunctionName)) {
+    auto It = NameFileMap.find(FunctionName);
+    if (It == NameFileMap.end()) {
       ++NumNotInOtherTU;
       return llvm::make_error<IndexError>(index_error_code::missing_definition);
     }
@@ -461,7 +462,7 @@ CrossTranslationUnitContext::ASTUnitStorage::getASTUnitForFunction(
     // Search in the index for the filename where the definition of FunctionName
     // resides.
     if (llvm::Expected<ASTUnit *> FoundForFile =
-            getASTUnitForFile(NameFileMap[FunctionName], DisplayCTUProgress)) {
+            getASTUnitForFile(It->second, DisplayCTUProgress)) {
 
       // Update the cache.
       NameASTUnitMap[FunctionName] = *FoundForFile;
diff --git a/clang/lib/Driver/Action.cpp b/clang/lib/Driver/Action.cpp
index 23dbcebc9a1ca..0899b8ef00152 100644
--- a/clang/lib/Driver/Action.cpp
+++ b/clang/lib/Driver/Action.cpp
@@ -27,8 +27,8 @@ const char *Action::getClassName(ActionClass AC) {
   case PrecompileJobClass: return "precompiler";
   case ExtractAPIJobClass:
     return "api-extractor";
-  case AnalyzeJobClass: return "analyzer";
-  case MigrateJobClass: return "migrator";
+  case AnalyzeJobClass:
+    return "analyzer";
   case CompileJobClass: return "compiler";
   case BackendJobClass: return "backend";
   case AssembleJobClass: return "assembler";
@@ -373,11 +373,6 @@ void AnalyzeJobAction::anchor() {}
 AnalyzeJobAction::AnalyzeJobAction(Action *Input, types::ID OutputType)
     : JobAction(AnalyzeJobClass, Input, OutputType) {}
 
-void MigrateJobAction::anchor() {}
-
-MigrateJobAction::MigrateJobAction(Action *Input, types::ID OutputType)
-    : JobAction(MigrateJobClass, Input, OutputType) {}
-
 void CompileJobAction::anchor() {}
 
 CompileJobAction::CompileJobAction(Action *Input, types::ID OutputType)
diff --git a/clang/lib/Driver/Driver.cpp b/clang/lib/Driver/Driver.cpp
index 87855fdb79971..50941d2aaa429 100644
--- a/clang/lib/Driver/Driver.cpp
+++ b/clang/lib/Driver/Driver.cpp
@@ -413,12 +413,12 @@ phases::ID Driver::getFinalPhase(const DerivedArgList &DAL,
     // -{fsyntax-only,-analyze,emit-ast} only run up to the compiler.
   } else if ((PhaseArg = DAL.getLastArg(options::OPT_fsyntax_only)) ||
              (PhaseArg = DAL.getLastArg(options::OPT_print_supported_cpus)) ||
-             (PhaseArg = DAL.getLastArg(options::OPT_print_enabled_extensions)) ||
+             (PhaseArg =
+                  DAL.getLastArg(options::OPT_print_enabled_extensions)) ||
              (PhaseArg = DAL.getLastArg(options::OPT_module_file_info)) ||
              (PhaseArg = DAL.getLastArg(options::OPT_verify_pch)) ||
              (PhaseArg = DAL.getLastArg(options::OPT_rewrite_objc)) ||
              (PhaseArg = DAL.getLastArg(options::OPT_rewrite_legacy_objc)) ||
-             (PhaseArg = DAL.getLastArg(options::OPT__migrate)) ||
              (PhaseArg = DAL.getLastArg(options::OPT__analyze)) ||
              (PhaseArg = DAL.getLastArg(options::OPT_emit_cir)) ||
              (PhaseArg = DAL.getLastArg(options::OPT_emit_ast))) {
@@ -637,6 +637,10 @@ static llvm::Triple computeTargetTriple(const Driver &D,
     }
   }
 
+  // Currently the only architecture supported by *-uefi triples are x86_64.
+  if (Target.isUEFI() && Target.getArch() != llvm::Triple::x86_64)
+    D.Diag(diag::err_target_unknown_triple) << Target.str();
+
   // The `-maix[32|64]` flags are only valid for AIX targets.
   if (Arg *A = Args.getLastArgNoClaim(options::OPT_maix32, options::OPT_maix64);
       A && !Target.isOSAIX())
@@ -882,27 +886,21 @@ void Driver::CreateOffloadingDeviceToolChains(Compilation &C,
     return;
   }
   if (IsCuda && !UseLLVMOffload) {
-    const ToolChain *HostTC = C.getSingleOffloadToolChain<Action::OFK_Host>();
-    const llvm::Triple &HostTriple = HostTC->getTriple();
-    auto OFK = Action::OFK_Cuda;
-    auto CudaTriple =
-        getNVIDIAOffloadTargetTriple(*this, C.getInputArgs(), HostTriple);
+    auto CudaTriple = getNVIDIAOffloadTargetTriple(
+        *this, C.getInputArgs(), C.getDefaultToolChain().getTriple());
     if (!CudaTriple)
       return;
-    // Use the CUDA and host triples as the key into the ToolChains map,
-    // because the device toolchain we create depends on both.
-    auto &CudaTC = ToolChains[CudaTriple->str() + "/" + HostTriple.str()];
-    if (!CudaTC) {
-      CudaTC = std::make_unique<toolchains::CudaToolChain>(
-          *this, *CudaTriple, *HostTC, C.getInputArgs());
-
-      // Emit a warning if the detected CUDA version is too new.
-      CudaInstallationDetector &CudaInstallation =
-          static_cast<toolchains::CudaToolChain &>(*CudaTC).CudaInstallation;
-      if (CudaInstallation.isValid())
-        CudaInstallation.WarnIfUnsupportedVersion();
-    }
-    C.addOffloadDeviceToolChain(CudaTC.get(), OFK);
+
+    auto &TC =
+        getOffloadToolChain(C.getInputArgs(), Action::OFK_Cuda, *CudaTriple,
+                            C.getDefaultToolChain().getTriple());
+
+    // Emit a warning if the detected CUDA version is too new.
+    const CudaInstallationDetector &CudaInstallation =
+        static_cast<const toolchains::CudaToolChain &>(TC).CudaInstallation;
+    if (CudaInstallation.isValid())
+      CudaInstallation.WarnIfUnsupportedVersion();
+    C.addOffloadDeviceToolChain(&TC, Action::OFK_Cuda);
   } else if (IsHIP && !UseLLVMOffload) {
     if (auto *OMPTargetArg =
             C.getInputArgs().getLastArg(options::OPT_fopenmp_targets_EQ)) {
@@ -910,14 +908,15 @@ void Driver::CreateOffloadingDeviceToolChains(Compilation &C,
           << OMPTargetArg->getSpelling() << "HIP";
       return;
     }
-    const ToolChain *HostTC = C.getSingleOffloadToolChain<Action::OFK_Host>();
-    auto OFK = Action::OFK_HIP;
+
     auto HIPTriple = getHIPOffloadTargetTriple(*this, C.getInputArgs());
     if (!HIPTriple)
       return;
-    auto *HIPTC = &getOffloadingDeviceToolChain(C.getInputArgs(), *HIPTriple,
-                                                *HostTC, OFK);
-    C.addOffloadDeviceToolChain(HIPTC, OFK);
+
+    auto &TC =
+        getOffloadToolChain(C.getInputArgs(), Action::OFK_HIP, *HIPTriple,
+                            C.getDefaultToolChain().getTriple());
+    C.addOffloadDeviceToolChain(&TC, Action::OFK_HIP);
   }
 
   if (IsCuda || IsHIP)
@@ -1034,40 +1033,17 @@ void Driver::CreateOffloadingDeviceToolChains(Compilation &C,
       FoundNormalizedTriples[NormalizedName] = Val;
 
       // If the specified target is invalid, emit a diagnostic.
-      if (TT.getArch() == llvm::Triple::UnknownArch)
+      if (TT.getArch() == llvm::Triple::UnknownArch) {
         Diag(clang::diag::err_drv_invalid_omp_target) << Val;
-      else {
-        const ToolChain *TC;
-        // Device toolchains have to be selected differently. They pair host
-        // and device in their implementation.
-        if (TT.isNVPTX() || TT.isAMDGCN() || TT.isSPIRV()) {
-          const ToolChain *HostTC =
-              C.getSingleOffloadToolChain<Action::OFK_Host>();
-          assert(HostTC && "Host toolchain should be always defined.");
-          auto &DeviceTC =
-              ToolChains[TT.str() + "/" + HostTC->getTriple().normalize()];
-          if (!DeviceTC) {
-            if (TT.isNVPTX())
-              DeviceTC = std::make_unique<toolchains::CudaToolChain>(
-                  *this, TT, *HostTC, C.getInputArgs());
-            else if (TT.isAMDGCN())
-              DeviceTC = std::make_unique<toolchains::AMDGPUOpenMPToolChain>(
-                  *this, TT, *HostTC, C.getInputArgs());
-            else if (TT.isSPIRV())
-              DeviceTC = std::make_unique<toolchains::SPIRVOpenMPToolChain>(
-                  *this, TT, *HostTC, C.getInputArgs());
-            else
-              assert(DeviceTC && "Device toolchain not defined.");
-          }
-
-          TC = DeviceTC.get();
-        } else
-          TC = &getToolChain(C.getInputArgs(), TT);
-        C.addOffloadDeviceToolChain(TC, Action::OFK_OpenMP);
-        auto It = DerivedArchs.find(TT.getTriple());
-        if (It != DerivedArchs.end())
-          KnownArchs[TC] = It->second;
+        continue;
       }
+
+      auto &TC = getOffloadToolChain(C.getInputArgs(), Action::OFK_OpenMP, TT,
+                                     C.getDefaultToolChain().getTriple());
+      C.addOffloadDeviceToolChain(&TC, Action::OFK_OpenMP);
+      auto It = DerivedArchs.find(TT.getTriple());
+      if (It != DerivedArchs.end())
+        KnownArchs[&TC] = It->second;
     }
   } else if (C.getInputArgs().hasArg(options::OPT_fopenmp_targets_EQ)) {
     Diag(clang::diag::err_drv_expecting_fopenmp_with_fopenmp_targets);
@@ -1099,9 +1075,9 @@ void Driver::CreateOffloadingDeviceToolChains(Compilation &C,
     // getOffloadingDeviceToolChain, because the device toolchains we're
     // going to create will depend on both.
     const ToolChain *HostTC = C.getSingleOffloadToolChain<Action::OFK_Host>();
-    for (const auto &TargetTriple : UniqueSYCLTriplesVec) {
-      auto SYCLTC = &getOffloadingDeviceToolChain(
-          C.getInputArgs(), TargetTriple, *HostTC, Action::OFK_SYCL);
+    for (const auto &TT : UniqueSYCLTriplesVec) {
+      auto SYCLTC = &getOffloadToolChain(C.getInputArgs(), Action::OFK_SYCL, TT,
+                                         HostTC->getTriple());
       C.addOffloadDeviceToolChain(SYCLTC, Action::OFK_SYCL);
     }
   }
@@ -3405,7 +3381,9 @@ class OffloadingActionBuilder final {
       // Collect all offload arch parameters, removing duplicates.
       std::set<StringRef> GpuArchs;
       bool Error = false;
-      for (Arg *A : Args) {
+      const ToolChain &TC = *ToolChains.front();
+      for (Arg *A : C.getArgsForToolChain(&TC, /*BoundArch=*/"",
+                                          AssociatedOffloadKind)) {
         if (!(A->getOption().matches(options::OPT_offload_arch_EQ) ||
               A->getOption().matches(options::OPT_no_offload_arch_EQ)))
           continue;
@@ -3416,7 +3394,6 @@ class OffloadingActionBuilder final {
               ArchStr == "all") {
             GpuArchs.clear();
           } else if (ArchStr == "native") {
-            const ToolChain &TC = *ToolChains.front();
             auto GPUsOrErr = ToolChains.front()->getSystemGPUArchs(Args);
             if (!GPUsOrErr) {
               TC.getDriver().Diag(diag::err_drv_undetermined_gpu_arch)
@@ -4708,23 +4685,7 @@ Driver::getOffloadArchs(Compilation &C, const llvm::opt::DerivedArgList &Args,
     return KnownArchs.lookup(TC);
 
   llvm::DenseSet<StringRef> Archs;
-  for (auto *Arg : Args) {
-    // Extract any '--[no-]offload-arch' arguments intended for this toolchain.
-    std::unique_ptr<llvm::opt::Arg> ExtractedArg = nullptr;
-    if (Arg->getOption().matches(options::OPT_Xopenmp_target_EQ) &&
-        ToolChain::getOpenMPTriple(Arg->getValue(0)) == TC->getTriple()) {
-      Arg->claim();
-      unsigned Index = Args.getBaseArgs().MakeIndex(Arg->getValue(1));
-      unsigned Prev = Index;
-      ExtractedArg = getOpts().ParseOneArg(Args, Index);
-      if (!ExtractedArg || Index > Prev + 1) {
-        TC->getDriver().Diag(diag::err_drv_invalid_Xopenmp_target_with_args)
-            << Arg->getAsString(Args);
-        continue;
-      }
-      Arg = ExtractedArg.get();
-    }
-
+  for (auto *Arg : C.getArgsForToolChain(TC, /*BoundArch=*/"", Kind)) {
     // Add or remove the seen architectures in order of appearance. If an
     // invalid architecture is given we simply exit.
     if (Arg->getOption().matches(options::OPT_offload_arch_EQ)) {
@@ -4781,14 +4742,31 @@ Driver::getOffloadArchs(Compilation &C, const llvm::opt::DerivedArgList &Args,
     return Archs;
 
   if (Archs.empty()) {
-    if (Kind == Action::OFK_Cuda)
+    if (Kind == Action::OFK_Cuda) {
       Archs.insert(OffloadArchToString(OffloadArch::CudaDefault));
-    else if (Kind == Action::OFK_HIP)
+    } else if (Kind == Action::OFK_HIP) {
       Archs.insert(OffloadArchToString(OffloadArch::HIPDefault));
-    else if (Kind == Action::OFK_OpenMP)
-      Archs.insert(StringRef());
-    else if (Kind == Action::OFK_SYCL)
+    } else if (Kind == Action::OFK_SYCL) {
       Archs.insert(StringRef());
+    } else if (Kind == Action::OFK_OpenMP) {
+      // Accept legacy `-march` device arguments for OpenMP.
+      if (auto *Arg = C.getArgsForToolChain(TC, /*BoundArch=*/"", Kind)
+                          .getLastArg(options::OPT_march_EQ)) {
+        Archs.insert(Arg->getValue());
+      } else {
+        auto ArchsOrErr = TC->getSystemGPUArchs(Args);
+        if (!ArchsOrErr) {
+          TC->getDriver().Diag(diag::err_drv_undetermined_gpu_arch)
+              << llvm::Triple::getArchTypeName(TC->getArch())
+              << llvm::toString(ArchsOrErr.takeError()) << "--offload-arch";
+        } else if (!ArchsOrErr->empty()) {
+          for (auto Arch : *ArchsOrErr)
+            Archs.insert(Args.MakeArgStringRef(Arch));
+        } else {
+          Archs.insert(StringRef());
+        }
+      }
+    }
   } else {
     Args.ClaimAllArgs(options::OPT_offload_arch_EQ);
     Args.ClaimAllArgs(options::OPT_no_offload_arch_EQ);
@@ -5078,8 +5056,6 @@ Action *Driver::ConstructPhaseAction(
                                             types::TY_RewrittenLegacyObjC);
     if (Args.hasArg(options::OPT__analyze))
       return C.MakeAction<AnalyzeJobAction>(Input, types::TY_Plist);
-    if (Args.hasArg(options::OPT__migrate))
-      return C.MakeAction<MigrateJobAction>(Input, types::TY_Remap);
     if (Args.hasArg(options::OPT_emit_ast))
       return C.MakeAction<CompileJobAction>(Input, types::TY_AST);
     if (Args.hasArg(options::OPT_emit_cir))
@@ -6601,6 +6577,73 @@ std::string Driver::GetClPchPath(Compilation &C, StringRef BaseName) const {
   return std::string(Output);
 }
 
+const ToolChain &Driver::getOffloadToolChain(
+    const llvm::opt::ArgList &Args, const Action::OffloadKind Kind,
+    const llvm::Triple &Target, const llvm::Triple &AuxTarget) const {
+  std::unique_ptr<ToolChain> &TC =
+      ToolChains[Target.str() + "/" + AuxTarget.str()];
+  std::unique_ptr<ToolChain> &HostTC = ToolChains[AuxTarget.str()];
+
+  assert(HostTC && "Host toolchain for offloading doesn't exit?");
+  if (!TC) {
+    // Detect the toolchain based off of the target operating system.
+    switch (Target.getOS()) {
+    case llvm::Triple::CUDA:
+      TC = std::make_unique<toolchains::CudaToolChain>(*this, Target, *HostTC,
+                                                       Args);
+      break;
+    case llvm::Triple::AMDHSA:
+      if (Kind == Action::OFK_HIP)
+        TC = std::make_unique<toolchains::HIPAMDToolChain>(*this, Target,
+                                                           *HostTC, Args);
+      else if (Kind == Action::OFK_OpenMP)
+        TC = std::make_unique<toolchains::AMDGPUOpenMPToolChain>(*this, Target,
+                                                                 *HostTC, Args);
+      break;
+    default:
+      break;
+    }
+  }
+  if (!TC) {
+    // Detect the toolchain based off of the target architecture if that failed.
+    switch (Target.getArch()) {
+    case llvm::Triple::spir:
+    case llvm::Triple::spir64:
+    case llvm::Triple::spirv:
+    case llvm::Triple::spirv32:
+    case llvm::Triple::spirv64:
+      switch (Kind) {
+      case Action::OFK_SYCL:
+        TC = std::make_unique<toolchains::SYCLToolChain>(*this, Target, *HostTC,
+                                                         Args);
+        break;
+      case Action::OFK_HIP:
+        TC = std::make_unique<toolchains::HIPSPVToolChain>(*this, Target,
+                                                           *HostTC, Args);
+        break;
+      case Action::OFK_OpenMP:
+        TC = std::make_unique<toolchains::SPIRVOpenMPToolChain>(*this, Target,
+                                                                *HostTC, Args);
+        break;
+      case Action::OFK_Cuda:
+        TC = std::make_unique<toolchains::CudaToolChain>(*this, Target, *HostTC,
+                                                         Args);
+        break;
+      default:
+        break;
+      }
+      break;
+    default:
+      break;
+    }
+  }
+
+  // If all else fails, just look up the normal toolchain for the target.
+  if (!TC)
+    return getToolChain(Args, Target);
+  return *TC;
+}
+
 const ToolChain &Driver::getToolChain(const ArgList &Args,
                                       const llvm::Triple &Target) const {
 
@@ -6794,45 +6837,6 @@ const ToolChain &Driver::getToolChain(const ArgList &Args,
   return *TC;
 }
 
-const ToolChain &Driver::getOffloadingDeviceToolChain(
-    const ArgList &Args, const llvm::Triple &Target, const ToolChain &HostTC,
-    const Action::OffloadKind &TargetDeviceOffloadKind) const {
-  // Use device / host triples as the key into the ToolChains map because the
-  // device ToolChain we create depends on both.
-  auto &TC = ToolChains[Target.str() + "/" + HostTC.getTriple().str()];
-  if (!TC) {
-    // Categorized by offload kind > arch rather than OS > arch like
-    // the normal getToolChain call, as it seems a reasonable way to categorize
-    // things.
-    switch (TargetDeviceOffloadKind) {
-    case Action::OFK_HIP: {
-      if (((Target.getArch() == llvm::Triple::amdgcn ||
-            Target.getArch() == llvm::Triple::spirv64) &&
-           Target.getVendor() == llvm::Triple::AMD &&
-           Target.getOS() == llvm::Triple::AMDHSA) ||
-          !Args.hasArgNoClaim(options::OPT_offload_EQ))
-        TC = std::make_unique<toolchains::HIPAMDToolChain>(*this, Target,
-                                                           HostTC, Args);
-      else if (Target.getArch() == llvm::Triple::spirv64 &&
-               Target.getVendor() == llvm::Triple::UnknownVendor &&
-               Target.getOS() == llvm::Triple::UnknownOS)
-        TC = std::make_unique<toolchains::HIPSPVToolChain>(*this, Target,
-                                                           HostTC, Args);
-      break;
-    }
-    case Action::OFK_SYCL:
-      if (Target.isSPIROrSPIRV())
-        TC = std::make_unique<toolchains::SYCLToolChain>(*this, Target, HostTC,
-                                                         Args);
-      break;
-    default:
-      break;
-    }
-  }
-  assert(TC && "Could not create offloading device tool chain.");
-  return *TC;
-}
-
 bool Driver::ShouldUseClangCompiler(const JobAction &JA) const {
   // Say "no" if there is not exactly one input of a type clang understands.
   if (JA.size() != 1 ||
diff --git a/clang/lib/Driver/SanitizerArgs.cpp b/clang/lib/Driver/SanitizerArgs.cpp
index a0d6919c6dc8d..294f637ef6515 100644
--- a/clang/lib/Driver/SanitizerArgs.cpp
+++ b/clang/lib/Driver/SanitizerArgs.cpp
@@ -575,6 +575,9 @@ SanitizerArgs::SanitizerArgs(const ToolChain &TC,
                                      options::OPT_fstrict_overflow, false);
         if (Args.hasFlagNoClaim(options::OPT_fwrapv, options::OPT_fno_wrapv, S))
           Add &= ~SanitizerKind::SignedIntegerOverflow;
+        if (Args.hasFlagNoClaim(options::OPT_fwrapv_pointer,
+                                options::OPT_fno_wrapv_pointer, S))
+          Add &= ~SanitizerKind::PointerOverflow;
       }
       Add &= Supported;
 
@@ -829,6 +832,8 @@ SanitizerArgs::SanitizerArgs(const ToolChain &TC,
     CfiICallNormalizeIntegers =
         Args.hasArg(options::OPT_fsanitize_cfi_icall_normalize_integers);
 
+    KcfiArity = Args.hasArg(options::OPT_fsanitize_kcfi_arity);
+
     if (AllAddedKinds & SanitizerKind::CFI && DiagnoseErrors)
       D.Diag(diag::err_drv_argument_not_allowed_with)
           << "-fsanitize=kcfi"
@@ -1383,6 +1388,14 @@ void SanitizerArgs::addArgs(const ToolChain &TC, const llvm::opt::ArgList &Args,
   if (CfiICallNormalizeIntegers)
     CmdArgs.push_back("-fsanitize-cfi-icall-experimental-normalize-integers");
 
+  if (KcfiArity) {
+    if (!TC.getTriple().isOSLinux() || !TC.getTriple().isArch64Bit()) {
+      TC.getDriver().Diag(clang::diag::err_drv_kcfi_arity_unsupported_target)
+          << TC.getTriple().str();
+    }
+    CmdArgs.push_back("-fsanitize-kcfi-arity");
+  }
+
   if (CfiCanonicalJumpTables)
     CmdArgs.push_back("-fsanitize-cfi-canonical-jump-tables");
 
diff --git a/clang/lib/Driver/ToolChain.cpp b/clang/lib/Driver/ToolChain.cpp
index acf9d264d631b..56bc500da66b9 100644
--- a/clang/lib/Driver/ToolChain.cpp
+++ b/clang/lib/Driver/ToolChain.cpp
@@ -646,7 +646,6 @@ Tool *ToolChain::getTool(Action::ActionClass AC) const {
   case Action::PreprocessJobClass:
   case Action::ExtractAPIJobClass:
   case Action::AnalyzeJobClass:
-  case Action::MigrateJobClass:
   case Action::VerifyPCHJobClass:
   case Action::BackendJobClass:
     return getClang();
@@ -1649,9 +1648,11 @@ void ToolChain::TranslateXarchArgs(
       A->getOption().matches(options::OPT_Xarch_host))
     ValuePos = 0;
 
-  unsigned Index = Args.getBaseArgs().MakeIndex(A->getValue(ValuePos));
+  const InputArgList &BaseArgs = Args.getBaseArgs();
+  unsigned Index = BaseArgs.MakeIndex(A->getValue(ValuePos));
   unsigned Prev = Index;
-  std::unique_ptr<llvm::opt::Arg> XarchArg(Opts.ParseOneArg(Args, Index));
+  std::unique_ptr<llvm::opt::Arg> XarchArg(Opts.ParseOneArg(
+      Args, Index, llvm::opt::Visibility(clang::driver::options::ClangOption)));
 
   // If the argument parsing failed or more than one argument was
   // consumed, the -Xarch_ argument's parameter tried to consume
@@ -1673,8 +1674,31 @@ void ToolChain::TranslateXarchArgs(
     Diags.Report(DiagID) << A->getAsString(Args);
     return;
   }
+
   XarchArg->setBaseArg(A);
   A = XarchArg.release();
+
+  // Linker input arguments require custom handling. The problem is that we
+  // have already constructed the phase actions, so we can not treat them as
+  // "input arguments".
+  if (A->getOption().hasFlag(options::LinkerInput)) {
+    // Convert the argument into individual Zlinker_input_args. Need to do this
+    // manually to avoid memory leaks with the allocated arguments.
+    for (const char *Value : A->getValues()) {
+      auto Opt = Opts.getOption(options::OPT_Zlinker_input);
+      unsigned Index = BaseArgs.MakeIndex(Opt.getName(), Value);
+      auto NewArg =
+          new Arg(Opt, BaseArgs.MakeArgString(Opt.getPrefix() + Opt.getName()),
+                  Index, BaseArgs.getArgString(Index + 1), A);
+
+      DAL->append(NewArg);
+      if (!AllocatedArgs)
+        DAL->AddSynthesizedArg(NewArg);
+      else
+        AllocatedArgs->push_back(NewArg);
+    }
+  }
+
   if (!AllocatedArgs)
     DAL->AddSynthesizedArg(A);
   else
@@ -1698,19 +1722,17 @@ llvm::opt::DerivedArgList *ToolChain::TranslateXarchArgs(
     } else if (A->getOption().matches(options::OPT_Xarch_host)) {
       NeedTrans = !IsDevice;
       Skip = IsDevice;
-    } else if (A->getOption().matches(options::OPT_Xarch__) && IsDevice) {
-      // Do not translate -Xarch_ options for non CUDA/HIP toolchain since
-      // they may need special translation.
-      // Skip this argument unless the architecture matches BoundArch
-      if (BoundArch.empty() || A->getValue(0) != BoundArch)
-        Skip = true;
-      else
-        NeedTrans = true;
+    } else if (A->getOption().matches(options::OPT_Xarch__)) {
+      NeedTrans = A->getValue() == getArchName() ||
+                  (!BoundArch.empty() && A->getValue() == BoundArch);
+      Skip = !NeedTrans;
     }
     if (NeedTrans || Skip)
       Modified = true;
-    if (NeedTrans)
+    if (NeedTrans) {
+      A->claim();
       TranslateXarchArgs(Args, A, DAL, AllocatedArgs);
+    }
     if (!Skip)
       DAL->append(A);
   }
diff --git a/clang/lib/Driver/ToolChains/AMDGPU.cpp b/clang/lib/Driver/ToolChains/AMDGPU.cpp
index a8061ffd9321f..e66e5a32e58ac 100644
--- a/clang/lib/Driver/ToolChains/AMDGPU.cpp
+++ b/clang/lib/Driver/ToolChains/AMDGPU.cpp
@@ -950,6 +950,11 @@ void ROCMToolChain::addClangTargetOptions(
                                                 ABIVer))
     return;
 
+  std::tuple<bool, const SanitizerArgs> GPUSan(
+      DriverArgs.hasFlag(options::OPT_fgpu_sanitize,
+                         options::OPT_fno_gpu_sanitize, true),
+      getSanitizerArgs(DriverArgs));
+
   bool Wave64 = isWave64(DriverArgs, Kind);
 
   // TODO: There are way too many flags that change this. Do we need to check
@@ -965,21 +970,19 @@ void ROCMToolChain::addClangTargetOptions(
       DriverArgs.hasArg(options::OPT_cl_fp32_correctly_rounded_divide_sqrt);
 
   // Add the OpenCL specific bitcode library.
-  llvm::SmallVector<std::string, 12> BCLibs;
-  BCLibs.push_back(RocmInstallation->getOpenCLPath().str());
+  llvm::SmallVector<BitCodeLibraryInfo, 12> BCLibs;
+  BCLibs.emplace_back(RocmInstallation->getOpenCLPath().str());
 
   // Add the generic set of libraries.
   BCLibs.append(RocmInstallation->getCommonBitcodeLibs(
       DriverArgs, LibDeviceFile, Wave64, DAZ, FiniteOnly, UnsafeMathOpt,
-      FastRelaxedMath, CorrectSqrt, ABIVer, false));
+      FastRelaxedMath, CorrectSqrt, ABIVer, GPUSan, false));
 
-  if (getSanitizerArgs(DriverArgs).needsAsanRt()) {
-    CC1Args.push_back("-mlink-bitcode-file");
-    CC1Args.push_back(
-        DriverArgs.MakeArgString(RocmInstallation->getAsanRTLPath()));
-  }
-  for (StringRef BCFile : BCLibs) {
-    CC1Args.push_back("-mlink-builtin-bitcode");
+  for (auto [BCFile, Internalize] : BCLibs) {
+    if (Internalize)
+      CC1Args.push_back("-mlink-builtin-bitcode");
+    else
+      CC1Args.push_back("-mlink-bitcode-file");
     CC1Args.push_back(DriverArgs.MakeArgString(BCFile));
   }
 }
@@ -1002,18 +1005,35 @@ bool RocmInstallationDetector::checkCommonBitcodeLibs(
   return true;
 }
 
-llvm::SmallVector<std::string, 12>
+llvm::SmallVector<ToolChain::BitCodeLibraryInfo, 12>
 RocmInstallationDetector::getCommonBitcodeLibs(
     const llvm::opt::ArgList &DriverArgs, StringRef LibDeviceFile, bool Wave64,
     bool DAZ, bool FiniteOnly, bool UnsafeMathOpt, bool FastRelaxedMath,
-    bool CorrectSqrt, DeviceLibABIVersion ABIVer, bool isOpenMP = false) const {
-  llvm::SmallVector<std::string, 12> BCLibs;
-
-  auto AddBCLib = [&](StringRef BCFile) { BCLibs.push_back(BCFile.str()); };
+    bool CorrectSqrt, DeviceLibABIVersion ABIVer,
+    const std::tuple<bool, const SanitizerArgs> &GPUSan,
+    bool isOpenMP = false) const {
+  llvm::SmallVector<ToolChain::BitCodeLibraryInfo, 12> BCLibs;
+
+  auto GPUSanEnabled = [GPUSan]() { return std::get<bool>(GPUSan); };
+  auto AddBCLib = [&](ToolChain::BitCodeLibraryInfo BCLib,
+                      bool Internalize = true) {
+    BCLib.ShouldInternalize = Internalize;
+    BCLibs.emplace_back(BCLib);
+  };
+  auto AddSanBCLibs = [&]() {
+    if (GPUSanEnabled()) {
+      auto SanArgs = std::get<const SanitizerArgs>(GPUSan);
+      if (SanArgs.needsAsanRt())
+        AddBCLib(getAsanRTLPath(), false);
+    }
+  };
 
+  AddSanBCLibs();
   AddBCLib(getOCMLPath());
   if (!isOpenMP)
     AddBCLib(getOCKLPath());
+  else if (GPUSanEnabled() && isOpenMP)
+    AddBCLib(getOCKLPath(), false);
   AddBCLib(getDenormalsAreZeroPath(DAZ));
   AddBCLib(getUnsafeMathPath(UnsafeMathOpt || FastRelaxedMath));
   AddBCLib(getFiniteOnlyPath(FiniteOnly || FastRelaxedMath));
@@ -1027,7 +1047,7 @@ RocmInstallationDetector::getCommonBitcodeLibs(
   return BCLibs;
 }
 
-llvm::SmallVector<std::string, 12>
+llvm::SmallVector<ToolChain::BitCodeLibraryInfo, 12>
 ROCMToolChain::getCommonDeviceLibNames(const llvm::opt::ArgList &DriverArgs,
                                        const std::string &GPUArch,
                                        bool isOpenMP) const {
@@ -1044,6 +1064,10 @@ ROCMToolChain::getCommonDeviceLibNames(const llvm::opt::ArgList &DriverArgs,
   // If --hip-device-lib is not set, add the default bitcode libraries.
   // TODO: There are way too many flags that change this. Do we need to check
   // them all?
+  std::tuple<bool, const SanitizerArgs> GPUSan(
+      DriverArgs.hasFlag(options::OPT_fgpu_sanitize,
+                         options::OPT_fno_gpu_sanitize, false),
+      getSanitizerArgs(DriverArgs));
   bool DAZ = DriverArgs.hasFlag(options::OPT_fgpu_flush_denormals_to_zero,
                                 options::OPT_fno_gpu_flush_denormals_to_zero,
                                 getDefaultDenormsAreZeroForTarget(Kind));
@@ -1061,7 +1085,7 @@ ROCMToolChain::getCommonDeviceLibNames(const llvm::opt::ArgList &DriverArgs,
 
   return RocmInstallation->getCommonBitcodeLibs(
       DriverArgs, LibDeviceFile, Wave64, DAZ, FiniteOnly, UnsafeMathOpt,
-      FastRelaxedMath, CorrectSqrt, ABIVer, isOpenMP);
+      FastRelaxedMath, CorrectSqrt, ABIVer, GPUSan, isOpenMP);
 }
 
 bool AMDGPUToolChain::shouldSkipSanitizeOption(
@@ -1075,7 +1099,7 @@ bool AMDGPUToolChain::shouldSkipSanitizeOption(
     return false;
 
   if (!DriverArgs.hasFlag(options::OPT_fgpu_sanitize,
-                          options::OPT_fno_gpu_sanitize, true))
+                          options::OPT_fno_gpu_sanitize, false))
     return true;
 
   auto &Diags = TC.getDriver().getDiags();
diff --git a/clang/lib/Driver/ToolChains/AMDGPU.h b/clang/lib/Driver/ToolChains/AMDGPU.h
index a9b4552a1f91a..bc941a40445ad 100644
--- a/clang/lib/Driver/ToolChains/AMDGPU.h
+++ b/clang/lib/Driver/ToolChains/AMDGPU.h
@@ -142,13 +142,28 @@ class LLVM_LIBRARY_VISIBILITY ROCMToolChain : public AMDGPUToolChain {
                         Action::OffloadKind DeviceOffloadKind) const override;
 
   // Returns a list of device library names shared by different languages
-  llvm::SmallVector<std::string, 12>
+  llvm::SmallVector<BitCodeLibraryInfo, 12>
   getCommonDeviceLibNames(const llvm::opt::ArgList &DriverArgs,
                           const std::string &GPUArch,
                           bool isOpenMP = false) const;
+
   SanitizerMask getSupportedSanitizers() const override {
     return SanitizerKind::Address;
   }
+
+  void diagnoseUnsupportedSanitizers(const llvm::opt::ArgList &Args) const {
+    if (!Args.hasFlag(options::OPT_fgpu_sanitize, options::OPT_fno_gpu_sanitize,
+                      true))
+      return;
+    auto &Diags = getDriver().getDiags();
+    for (auto *A : Args.filtered(options::OPT_fsanitize_EQ)) {
+      SanitizerMask K =
+          parseSanitizerValue(A->getValue(), /*Allow Groups*/ false);
+      if (K != SanitizerKind::Address)
+        Diags.Report(clang::diag::warn_drv_unsupported_option_for_target)
+            << A->getAsString(Args) << getTriple().str();
+    }
+  }
 };
 
 } // end namespace toolchains
diff --git a/clang/lib/Driver/ToolChains/AMDGPUOpenMP.cpp b/clang/lib/Driver/ToolChains/AMDGPUOpenMP.cpp
index 3f0b3f2d86b3e..00bf9c7338edd 100644
--- a/clang/lib/Driver/ToolChains/AMDGPUOpenMP.cpp
+++ b/clang/lib/Driver/ToolChains/AMDGPUOpenMP.cpp
@@ -9,7 +9,7 @@
 #include "AMDGPUOpenMP.h"
 #include "AMDGPU.h"
 #include "CommonArgs.h"
-#include "ToolChains/ROCm.h"
+#include "ROCm.h"
 #include "clang/Basic/DiagnosticDriver.h"
 #include "clang/Driver/Compilation.h"
 #include "clang/Driver/Driver.h"
@@ -37,6 +37,8 @@ AMDGPUOpenMPToolChain::AMDGPUOpenMPToolChain(const Driver &D,
   // Lookup binaries into the driver directory, this is used to
   // discover the 'amdgpu-arch' executable.
   getProgramPaths().push_back(getDriver().Dir);
+  // Diagnose unsupported sanitizer options only once.
+  diagnoseUnsupportedSanitizers(Args);
 }
 
 void AMDGPUOpenMPToolChain::addClangTargetOptions(
@@ -71,34 +73,10 @@ llvm::opt::DerivedArgList *AMDGPUOpenMPToolChain::TranslateArgs(
 
   const OptTable &Opts = getDriver().getOpts();
 
-  if (DeviceOffloadKind == Action::OFK_OpenMP) {
-    for (Arg *A : Args)
-      if (!llvm::is_contained(*DAL, A))
-        DAL->append(A);
-
-    if (!DAL->hasArg(options::OPT_march_EQ)) {
-      StringRef Arch = BoundArch;
-      if (Arch.empty()) {
-        auto ArchsOrErr = getSystemGPUArchs(Args);
-        if (!ArchsOrErr) {
-          std::string ErrMsg =
-              llvm::formatv("{0}", llvm::fmt_consume(ArchsOrErr.takeError()));
-          getDriver().Diag(diag::err_drv_undetermined_gpu_arch)
-              << llvm::Triple::getArchTypeName(getArch()) << ErrMsg << "-march";
-          Arch = OffloadArchToString(OffloadArch::HIPDefault);
-        } else {
-          Arch = Args.MakeArgString(ArchsOrErr->front());
-        }
-      }
-      DAL->AddJoinedArg(nullptr, Opts.getOption(options::OPT_march_EQ), Arch);
-    }
-
-    return DAL;
-  }
-
-  for (Arg *A : Args) {
-    DAL->append(A);
-  }
+  for (Arg *A : Args)
+    if (!shouldSkipSanitizeOption(*this, Args, BoundArch, A) &&
+        !llvm::is_contained(*DAL, A))
+      DAL->append(A);
 
   if (!BoundArch.empty()) {
     DAL->eraseArg(options::OPT_march_EQ);
@@ -159,11 +137,6 @@ AMDGPUOpenMPToolChain::getDeviceLibs(const llvm::opt::ArgList &Args) const {
   if (Args.hasArg(options::OPT_nogpulib))
     return {};
 
-  if (!RocmInstallation->hasDeviceLibrary()) {
-    getDriver().Diag(diag::err_drv_no_rocm_device_lib) << 0;
-    return {};
-  }
-
   StringRef GpuArch = getProcessorFromTargetID(
       getTriple(), Args.getLastArgValue(options::OPT_march_EQ));
 
diff --git a/clang/lib/Driver/ToolChains/Arch/X86.cpp b/clang/lib/Driver/ToolChains/Arch/X86.cpp
index b2109e11038fe..47c2c3e23f9fd 100644
--- a/clang/lib/Driver/ToolChains/Arch/X86.cpp
+++ b/clang/lib/Driver/ToolChains/Arch/X86.cpp
@@ -237,15 +237,18 @@ void x86::getX86TargetFeatures(const Driver &D, const llvm::Triple &Triple,
 
     bool IsNegative = Name.consume_front("no-");
 
-#ifndef NDEBUG
-    assert(Name.starts_with("avx10.") && "Invalid AVX10 feature name.");
     StringRef Version, Width;
     std::tie(Version, Width) = Name.substr(6).split('-');
+    assert(Name.starts_with("avx10.") && "Invalid AVX10 feature name.");
     assert((Version == "1" || Version == "2") && "Invalid AVX10 feature name.");
-    assert((Width == "256" || Width == "512") && "Invalid AVX10 feature name.");
-#endif
 
-    Features.push_back(Args.MakeArgString((IsNegative ? "-" : "+") + Name));
+    if (Width == "") {
+      assert(IsNegative && "Only negative options can omit width.");
+      Features.push_back(Args.MakeArgString("-" + Name + "-256"));
+    } else {
+      assert((Width == "256" || Width == "512") && "Invalid vector length.");
+      Features.push_back(Args.MakeArgString((IsNegative ? "-" : "+") + Name));
+    }
   }
 
   // Now add any that the user explicitly requested on the command line,
diff --git a/clang/lib/Driver/ToolChains/Clang.cpp b/clang/lib/Driver/ToolChains/Clang.cpp
index 518113e20cb06..5deafa2ad0f4a 100644
--- a/clang/lib/Driver/ToolChains/Clang.cpp
+++ b/clang/lib/Driver/ToolChains/Clang.cpp
@@ -580,6 +580,7 @@ static void addPGOAndCoverageFlags(const ToolChain &TC, Compilation &C,
                                    const ArgList &Args, SanitizerArgs &SanArgs,
                                    ArgStringList &CmdArgs) {
   const Driver &D = TC.getDriver();
+  const llvm::Triple &T = TC.getTriple();
   auto *PGOGenerateArg = Args.getLastArg(options::OPT_fprofile_generate,
                                          options::OPT_fprofile_generate_EQ,
                                          options::OPT_fno_profile_generate);
@@ -785,6 +786,34 @@ static void addPGOAndCoverageFlags(const ToolChain &TC, Compilation &C,
       D.Diag(diag::err_drv_unsupported_option_argument)
           << A->getSpelling() << Val;
   }
+  if (const auto *A = Args.getLastArg(options::OPT_fprofile_continuous)) {
+    if (!PGOGenerateArg && !CSPGOGenerateArg && !ProfileGenerateArg)
+      D.Diag(clang::diag::err_drv_argument_only_allowed_with)
+          << A->getSpelling()
+          << "-fprofile-generate, -fprofile-instr-generate, or "
+             "-fcs-profile-generate";
+    else {
+      CmdArgs.push_back("-fprofile-continuous");
+      // Platforms that require a bias variable:
+      if (T.isOSBinFormatELF() || T.isOSAIX()) {
+        CmdArgs.push_back("-mllvm");
+        CmdArgs.push_back("-runtime-counter-relocation");
+      }
+      // -fprofile-instr-generate does not decide the profile file name in the
+      // FE, and so it does not define the filename symbol
+      // (__llvm_profile_filename). Instead, the runtime uses the name
+      // "default.profraw" for the profile file. When continuous mode is ON, we
+      // will create the filename symbol so that we can insert the "%c"
+      // modifier.
+      if (ProfileGenerateArg &&
+          (ProfileGenerateArg->getOption().matches(
+               options::OPT_fprofile_instr_generate) ||
+           (ProfileGenerateArg->getOption().matches(
+                options::OPT_fprofile_instr_generate_EQ) &&
+            strlen(ProfileGenerateArg->getValue()) == 0)))
+        CmdArgs.push_back("-fprofile-instrument-path=default.profraw");
+    }
+  }
 
   int FunctionGroups = 1;
   int SelectedFunctionGroup = 0;
@@ -1013,21 +1042,23 @@ void Clang::AddPreprocessingOptions(Compilation &C, const JobAction &JA,
     ArgM = ArgMD;
 
   if (ArgM) {
-    // Determine the output location.
-    const char *DepFile;
-    if (Arg *MF = Args.getLastArg(options::OPT_MF)) {
-      DepFile = MF->getValue();
-      C.addFailureResultFile(DepFile, &JA);
-    } else if (Output.getType() == types::TY_Dependencies) {
-      DepFile = Output.getFilename();
-    } else if (!ArgMD) {
-      DepFile = "-";
-    } else {
-      DepFile = getDependencyFileName(Args, Inputs);
-      C.addFailureResultFile(DepFile, &JA);
+    if (!JA.isDeviceOffloading(Action::OFK_HIP)) {
+      // Determine the output location.
+      const char *DepFile;
+      if (Arg *MF = Args.getLastArg(options::OPT_MF)) {
+        DepFile = MF->getValue();
+        C.addFailureResultFile(DepFile, &JA);
+      } else if (Output.getType() == types::TY_Dependencies) {
+        DepFile = Output.getFilename();
+      } else if (!ArgMD) {
+        DepFile = "-";
+      } else {
+        DepFile = getDependencyFileName(Args, Inputs);
+        C.addFailureResultFile(DepFile, &JA);
+      }
+      CmdArgs.push_back("-dependency-file");
+      CmdArgs.push_back(DepFile);
     }
-    CmdArgs.push_back("-dependency-file");
-    CmdArgs.push_back(DepFile);
 
     bool HasTarget = false;
     for (const Arg *A : Args.filtered(options::OPT_MT, options::OPT_MQ)) {
@@ -1358,6 +1389,7 @@ static bool isSignedCharDefault(const llvm::Triple &Triple) {
       return true;
     return false;
 
+  case llvm::Triple::csky:
   case llvm::Triple::hexagon:
   case llvm::Triple::msp430:
   case llvm::Triple::ppcle:
@@ -1560,15 +1592,21 @@ static void handlePAuthABI(const ArgList &DriverArgs, ArgStringList &CC1Args) {
 
 static void CollectARMPACBTIOptions(const ToolChain &TC, const ArgList &Args,
                                     ArgStringList &CmdArgs, bool isAArch64) {
+  const llvm::Triple &Triple = TC.getEffectiveTriple();
   const Arg *A = isAArch64
                      ? Args.getLastArg(options::OPT_msign_return_address_EQ,
                                        options::OPT_mbranch_protection_EQ)
                      : Args.getLastArg(options::OPT_mbranch_protection_EQ);
-  if (!A)
+  if (!A) {
+    if (Triple.isOSOpenBSD() && isAArch64) {
+      CmdArgs.push_back("-msign-return-address=non-leaf");
+      CmdArgs.push_back("-msign-return-address-key=a_key");
+      CmdArgs.push_back("-mbranch-target-enforce");
+    }
     return;
+  }
 
   const Driver &D = TC.getDriver();
-  const llvm::Triple &Triple = TC.getEffectiveTriple();
   if (!(isAArch64 || (Triple.isArmT32() && Triple.isArmMClass())))
     D.Diag(diag::warn_incompatible_branch_protection_option)
         << Triple.getArchName();
@@ -1582,7 +1620,7 @@ static void CollectARMPACBTIOptions(const ToolChain &TC, const ArgList &Args,
       D.Diag(diag::err_drv_unsupported_option_argument)
           << A->getSpelling() << Scope;
     Key = "a_key";
-    IndirectBranches = false;
+    IndirectBranches = Triple.isOSOpenBSD() && isAArch64;
     BranchProtectionPAuthLR = false;
     GuardedControlStack = false;
   } else {
@@ -1618,32 +1656,34 @@ static void CollectARMPACBTIOptions(const ToolChain &TC, const ArgList &Args,
     GuardedControlStack = PBP.GuardedControlStack;
   }
 
-  CmdArgs.push_back(
-      Args.MakeArgString(Twine("-msign-return-address=") + Scope));
-  if (Scope != "none") {
+  bool HasPtrauthReturns = llvm::any_of(CmdArgs, [](const char *Arg) {
+    return StringRef(Arg) == "-fptrauth-returns";
+  });
+  // GCS is currently untested with ptrauth-returns, but enabling this could be
+  // allowed in future after testing with a suitable system.
+  if (HasPtrauthReturns &&
+      (Scope != "none" || BranchProtectionPAuthLR || GuardedControlStack)) {
     if (Triple.getEnvironment() == llvm::Triple::PAuthTest)
       D.Diag(diag::err_drv_unsupported_opt_for_target)
           << A->getAsString(Args) << Triple.getTriple();
+    else
+      D.Diag(diag::err_drv_incompatible_options)
+          << A->getAsString(Args) << "-fptrauth-returns";
+  }
+
+  CmdArgs.push_back(
+      Args.MakeArgString(Twine("-msign-return-address=") + Scope));
+  if (Scope != "none")
     CmdArgs.push_back(
         Args.MakeArgString(Twine("-msign-return-address-key=") + Key));
-  }
-  if (BranchProtectionPAuthLR) {
-    if (Triple.getEnvironment() == llvm::Triple::PAuthTest)
-      D.Diag(diag::err_drv_unsupported_opt_for_target)
-          << A->getAsString(Args) << Triple.getTriple();
+  if (BranchProtectionPAuthLR)
     CmdArgs.push_back(
         Args.MakeArgString(Twine("-mbranch-protection-pauth-lr")));
-  }
   if (IndirectBranches)
     CmdArgs.push_back("-mbranch-target-enforce");
-  // GCS is currently untested with PAuthABI, but enabling this could be allowed
-  // in future after testing with a suitable system.
-  if (GuardedControlStack) {
-    if (Triple.getEnvironment() == llvm::Triple::PAuthTest)
-      D.Diag(diag::err_drv_unsupported_opt_for_target)
-          << A->getAsString(Args) << Triple.getTriple();
+
+  if (GuardedControlStack)
     CmdArgs.push_back("-mguarded-control-stack");
-  }
 }
 
 void Clang::AddARMTargetArgs(const llvm::Triple &Triple, const ArgList &Args,
@@ -1822,12 +1862,6 @@ void Clang::AddAArch64TargetArgs(const ArgList &Args,
       CmdArgs.push_back("-aarch64-enable-global-merge=true");
   }
 
-  // Enable/disable return address signing and indirect branch targets.
-  CollectARMPACBTIOptions(getToolChain(), Args, CmdArgs, true /*isAArch64*/);
-
-  if (Triple.getEnvironment() == llvm::Triple::PAuthTest)
-    handlePAuthABI(Args, CmdArgs);
-
   // Handle -msve_vector_bits=<bits>
   if (Arg *A = Args.getLastArg(options::OPT_msve_vector_bits_EQ)) {
     StringRef Val = A->getValue();
@@ -1896,6 +1930,12 @@ void Clang::AddAArch64TargetArgs(const ArgList &Args,
                     options::OPT_fno_ptrauth_init_fini_address_discrimination);
   Args.addOptInFlag(CmdArgs, options::OPT_faarch64_jump_table_hardening,
                     options::OPT_fno_aarch64_jump_table_hardening);
+
+  if (Triple.getEnvironment() == llvm::Triple::PAuthTest)
+    handlePAuthABI(Args, CmdArgs);
+
+  // Enable/disable return address signing and indirect branch targets.
+  CollectARMPACBTIOptions(getToolChain(), Args, CmdArgs, true /*isAArch64*/);
 }
 
 void Clang::AddLoongArchTargetArgs(const ArgList &Args,
@@ -3958,78 +3998,6 @@ static void RenderOpenACCOptions(const Driver &D, const ArgList &Args,
   }
 }
 
-static void RenderARCMigrateToolOptions(const Driver &D, const ArgList &Args,
-                                        ArgStringList &CmdArgs) {
-  bool ARCMTEnabled = false;
-  if (!Args.hasArg(options::OPT_fno_objc_arc, options::OPT_fobjc_arc)) {
-    if (const Arg *A = Args.getLastArg(options::OPT_ccc_arcmt_check,
-                                       options::OPT_ccc_arcmt_modify,
-                                       options::OPT_ccc_arcmt_migrate)) {
-      ARCMTEnabled = true;
-      switch (A->getOption().getID()) {
-      default: llvm_unreachable("missed a case");
-      case options::OPT_ccc_arcmt_check:
-        CmdArgs.push_back("-arcmt-action=check");
-        break;
-      case options::OPT_ccc_arcmt_modify:
-        CmdArgs.push_back("-arcmt-action=modify");
-        break;
-      case options::OPT_ccc_arcmt_migrate:
-        CmdArgs.push_back("-arcmt-action=migrate");
-        CmdArgs.push_back("-mt-migrate-directory");
-        CmdArgs.push_back(A->getValue());
-
-        Args.AddLastArg(CmdArgs, options::OPT_arcmt_migrate_report_output);
-        Args.AddLastArg(CmdArgs, options::OPT_arcmt_migrate_emit_arc_errors);
-        break;
-      }
-    }
-  } else {
-    Args.ClaimAllArgs(options::OPT_ccc_arcmt_check);
-    Args.ClaimAllArgs(options::OPT_ccc_arcmt_modify);
-    Args.ClaimAllArgs(options::OPT_ccc_arcmt_migrate);
-  }
-
-  if (const Arg *A = Args.getLastArg(options::OPT_ccc_objcmt_migrate)) {
-    if (ARCMTEnabled)
-      D.Diag(diag::err_drv_argument_not_allowed_with)
-          << A->getAsString(Args) << "-ccc-arcmt-migrate";
-
-    CmdArgs.push_back("-mt-migrate-directory");
-    CmdArgs.push_back(A->getValue());
-
-    if (!Args.hasArg(options::OPT_objcmt_migrate_literals,
-                     options::OPT_objcmt_migrate_subscripting,
-                     options::OPT_objcmt_migrate_property)) {
-      // None specified, means enable them all.
-      CmdArgs.push_back("-objcmt-migrate-literals");
-      CmdArgs.push_back("-objcmt-migrate-subscripting");
-      CmdArgs.push_back("-objcmt-migrate-property");
-    } else {
-      Args.AddLastArg(CmdArgs, options::OPT_objcmt_migrate_literals);
-      Args.AddLastArg(CmdArgs, options::OPT_objcmt_migrate_subscripting);
-      Args.AddLastArg(CmdArgs, options::OPT_objcmt_migrate_property);
-    }
-  } else {
-    Args.AddLastArg(CmdArgs, options::OPT_objcmt_migrate_literals);
-    Args.AddLastArg(CmdArgs, options::OPT_objcmt_migrate_subscripting);
-    Args.AddLastArg(CmdArgs, options::OPT_objcmt_migrate_property);
-    Args.AddLastArg(CmdArgs, options::OPT_objcmt_migrate_all);
-    Args.AddLastArg(CmdArgs, options::OPT_objcmt_migrate_readonly_property);
-    Args.AddLastArg(CmdArgs, options::OPT_objcmt_migrate_readwrite_property);
-    Args.AddLastArg(CmdArgs, options::OPT_objcmt_migrate_property_dot_syntax);
-    Args.AddLastArg(CmdArgs, options::OPT_objcmt_migrate_annotation);
-    Args.AddLastArg(CmdArgs, options::OPT_objcmt_migrate_instancetype);
-    Args.AddLastArg(CmdArgs, options::OPT_objcmt_migrate_nsmacros);
-    Args.AddLastArg(CmdArgs, options::OPT_objcmt_migrate_protocol_conformance);
-    Args.AddLastArg(CmdArgs, options::OPT_objcmt_atomic_property);
-    Args.AddLastArg(CmdArgs, options::OPT_objcmt_returns_innerpointer_property);
-    Args.AddLastArg(CmdArgs, options::OPT_objcmt_ns_nonatomic_iosonly);
-    Args.AddLastArg(CmdArgs, options::OPT_objcmt_migrate_designated_init);
-    Args.AddLastArg(CmdArgs, options::OPT_objcmt_allowlist_dir_path);
-  }
-}
-
 static void RenderBuiltinOptions(const ToolChain &TC, const llvm::Triple &T,
                                  const ArgList &Args, ArgStringList &CmdArgs) {
   // -fbuiltin is default unless -mkernel is used.
@@ -5319,8 +5287,6 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA,
   if (isa<AnalyzeJobAction>(JA)) {
     assert(JA.getType() == types::TY_Plist && "Invalid output type.");
     CmdArgs.push_back("-analyze");
-  } else if (isa<MigrateJobAction>(JA)) {
-    CmdArgs.push_back("-migrate");
   } else if (isa<PreprocessJobAction>(JA)) {
     if (Output.getType() == types::TY_Dependencies)
       CmdArgs.push_back("-Eonly");
@@ -5654,6 +5620,8 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA,
   if (Args.hasArg(options::OPT_static))
     CmdArgs.push_back("-static-define");
 
+  Args.AddLastArg(CmdArgs, options::OPT_static_libclosure);
+
   if (Args.hasArg(options::OPT_municode))
     CmdArgs.push_back("-DUNICODE");
 
@@ -6445,8 +6413,6 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA,
 
   Args.AddLastArg(CmdArgs, options::OPT_working_directory);
 
-  RenderARCMigrateToolOptions(D, Args, CmdArgs);
-
   // Add preprocessing options like -I, -D, etc. if we are using the
   // preprocessor.
   //
@@ -7040,6 +7006,8 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA,
   Args.AddLastArg(CmdArgs, options::OPT_fwritable_strings);
   Args.AddLastArg(CmdArgs, options::OPT_funroll_loops,
                   options::OPT_fno_unroll_loops);
+  Args.AddLastArg(CmdArgs, options::OPT_floop_interchange,
+                  options::OPT_fno_loop_interchange);
 
   Args.AddLastArg(CmdArgs, options::OPT_fstrict_flex_arrays_EQ);
 
@@ -7130,6 +7098,11 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA,
     if (Arg *SA = Args.getLastArg(options::OPT_mcf_branch_label_scheme_EQ))
       CmdArgs.push_back(Args.MakeArgString(Twine("-mcf-branch-label-scheme=") +
                                            SA->getValue()));
+  } else if (Triple.isOSOpenBSD() && Triple.getArch() == llvm::Triple::x86_64) {
+    // Emit IBT endbr64 instructions by default
+    CmdArgs.push_back("-fcf-protection=branch");
+    // jump-table can generate indirect jumps, which are not permitted
+    CmdArgs.push_back("-fno-jump-tables");
   }
 
   if (Arg *A = Args.getLastArg(options::OPT_mfunction_return_EQ))
@@ -7502,20 +7475,6 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA,
   Args.addOptOutFlag(CmdArgs, options::OPT_fassume_unique_vtables,
                      options::OPT_fno_assume_unique_vtables);
 
-  // -frelaxed-template-template-args is deprecated.
-  if (Arg *A =
-          Args.getLastArg(options::OPT_frelaxed_template_template_args,
-                          options::OPT_fno_relaxed_template_template_args)) {
-    if (A->getOption().matches(
-            options::OPT_fno_relaxed_template_template_args)) {
-      D.Diag(diag::warn_drv_deprecated_arg_no_relaxed_template_template_args);
-      CmdArgs.push_back("-fno-relaxed-template-template-args");
-    } else {
-      D.Diag(diag::warn_drv_deprecated_arg)
-          << A->getAsString(Args) << /*hasReplacement=*/false;
-    }
-  }
-
   // -fsized-deallocation is on by default in C++14 onwards and otherwise off
   // by default.
   Args.addLastArg(CmdArgs, options::OPT_fsized_deallocation,
@@ -7706,6 +7665,8 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA,
   if (Args.hasArg(options::OPT_fretain_comments_from_system_headers))
     CmdArgs.push_back("-fretain-comments-from-system-headers");
 
+  Args.AddLastArg(CmdArgs, options::OPT_fextend_variable_liveness_EQ);
+
   // Forward -fcomment-block-commands to -cc1.
   Args.AddAllArgs(CmdArgs, options::OPT_fcomment_block_commands);
   // Forward -fparse-all-comments to -cc1.
@@ -9202,7 +9163,7 @@ void OffloadPackager::ConstructJob(Compilation &C, const JobAction &JA,
     SmallVector<std::string> Parts{
         "file=" + File.str(),
         "triple=" + TC->getTripleString(),
-        "arch=" + Arch.str(),
+        "arch=" + (Arch.empty() ? "generic" : Arch.str()),
         "kind=" + Kind.str(),
     };
 
@@ -9224,81 +9185,81 @@ void LinkerWrapper::ConstructJob(Compilation &C, const JobAction &JA,
                                  const InputInfoList &Inputs,
                                  const ArgList &Args,
                                  const char *LinkingOutput) const {
-  const Driver &D = getToolChain().getDriver();
-  const llvm::Triple TheTriple = getToolChain().getTriple();
-  ArgStringList CmdArgs;
+  using namespace options;
+
+  // A list of permitted options that will be forwarded to the embedded device
+  // compilation job.
+  const llvm::DenseSet<unsigned> CompilerOptions{
+      OPT_v,
+      OPT_cuda_path_EQ,
+      OPT_rocm_path_EQ,
+      OPT_O_Group,
+      OPT_g_Group,
+      OPT_g_flags_Group,
+      OPT_R_value_Group,
+      OPT_R_Group,
+      OPT_Xcuda_ptxas,
+      OPT_ftime_report,
+      OPT_ftime_trace,
+      OPT_ftime_trace_EQ,
+      OPT_ftime_trace_granularity_EQ,
+      OPT_ftime_trace_verbose,
+      OPT_opt_record_file,
+      OPT_opt_record_format,
+      OPT_opt_record_passes,
+      OPT_fsave_optimization_record,
+      OPT_fsave_optimization_record_EQ,
+      OPT_fno_save_optimization_record,
+      OPT_foptimization_record_file_EQ,
+      OPT_foptimization_record_passes_EQ,
+      OPT_save_temps,
+      OPT_save_temps_EQ,
+      OPT_mcode_object_version_EQ,
+      OPT_load,
+      OPT_fno_lto,
+      OPT_flto,
+      OPT_flto_EQ};
+  const llvm::DenseSet<unsigned> LinkerOptions{OPT_mllvm, OPT_Zlinker_input};
+  auto ShouldForward = [&](const llvm::DenseSet<unsigned> &Set, Arg *A) {
+    return Set.contains(A->getOption().getID()) ||
+           (A->getOption().getGroup().isValid() &&
+            Set.contains(A->getOption().getGroup().getID()));
+  };
 
-  // Pass the CUDA path to the linker wrapper tool.
+  ArgStringList CmdArgs;
   for (Action::OffloadKind Kind : {Action::OFK_Cuda, Action::OFK_OpenMP}) {
     auto TCRange = C.getOffloadToolChains(Kind);
     for (auto &I : llvm::make_range(TCRange)) {
       const ToolChain *TC = I.second;
-      if (TC->getTriple().isNVPTX()) {
-        CudaInstallationDetector CudaInstallation(D, TheTriple, Args);
-        if (CudaInstallation.isValid())
-          CmdArgs.push_back(Args.MakeArgString(
-              "--cuda-path=" + CudaInstallation.getInstallPath()));
-        break;
+
+      // We do not use a bound architecture here so options passed only to a
+      // specific architecture via -Xarch_<cpu> will not be forwarded.
+      ArgStringList CompilerArgs;
+      ArgStringList LinkerArgs;
+      for (Arg *A : C.getArgsForToolChain(TC, /*BoundArch=*/"", Kind)) {
+        if (A->getOption().matches(OPT_Zlinker_input))
+          LinkerArgs.emplace_back(A->getValue());
+        else if (ShouldForward(CompilerOptions, A))
+          A->render(Args, CompilerArgs);
+        else if (ShouldForward(LinkerOptions, A))
+          A->render(Args, LinkerArgs);
       }
-    }
-  }
 
-  // Pass in the optimization level to use for LTO.
-  if (const Arg *A = Args.getLastArg(options::OPT_O_Group)) {
-    StringRef OOpt;
-    if (A->getOption().matches(options::OPT_O4) ||
-        A->getOption().matches(options::OPT_Ofast))
-      OOpt = "3";
-    else if (A->getOption().matches(options::OPT_O)) {
-      OOpt = A->getValue();
-      if (OOpt == "g")
-        OOpt = "1";
-      else if (OOpt == "s" || OOpt == "z")
-        OOpt = "2";
-    } else if (A->getOption().matches(options::OPT_O0))
-      OOpt = "0";
-    if (!OOpt.empty())
-      CmdArgs.push_back(Args.MakeArgString(Twine("--opt-level=O") + OOpt));
+      // Forward all of these to the appropriate toolchain.
+      for (StringRef Arg : CompilerArgs)
+        CmdArgs.push_back(Args.MakeArgString(
+            "--device-compiler=" + TC->getTripleString() + "=" + Arg));
+      for (StringRef Arg : LinkerArgs)
+        CmdArgs.push_back(Args.MakeArgString(
+            "--device-linker=" + TC->getTripleString() + "=" + Arg));
+    }
   }
 
   CmdArgs.push_back(
-      Args.MakeArgString("--host-triple=" + TheTriple.getTriple()));
+      Args.MakeArgString("--host-triple=" + getToolChain().getTripleString()));
   if (Args.hasArg(options::OPT_v))
     CmdArgs.push_back("--wrapper-verbose");
 
-  if (const Arg *A = Args.getLastArg(options::OPT_g_Group)) {
-    if (!A->getOption().matches(options::OPT_g0))
-      CmdArgs.push_back("--device-debug");
-  }
-
-  // code-object-version=X needs to be passed to clang-linker-wrapper to ensure
-  // that it is used by lld.
-  if (const Arg *A = Args.getLastArg(options::OPT_mcode_object_version_EQ)) {
-    CmdArgs.push_back(Args.MakeArgString("-mllvm"));
-    CmdArgs.push_back(Args.MakeArgString(
-        Twine("--amdhsa-code-object-version=") + A->getValue()));
-  }
-
-  for (const auto &A : Args.getAllArgValues(options::OPT_Xcuda_ptxas))
-    CmdArgs.push_back(Args.MakeArgString("--ptxas-arg=" + A));
-
-  // Forward remarks passes to the LLVM backend in the wrapper.
-  if (const Arg *A = Args.getLastArg(options::OPT_Rpass_EQ))
-    CmdArgs.push_back(Args.MakeArgString(Twine("--offload-opt=-pass-remarks=") +
-                                         A->getValue()));
-  if (const Arg *A = Args.getLastArg(options::OPT_Rpass_missed_EQ))
-    CmdArgs.push_back(Args.MakeArgString(
-        Twine("--offload-opt=-pass-remarks-missed=") + A->getValue()));
-  if (const Arg *A = Args.getLastArg(options::OPT_Rpass_analysis_EQ))
-    CmdArgs.push_back(Args.MakeArgString(
-        Twine("--offload-opt=-pass-remarks-analysis=") + A->getValue()));
-
-  if (Args.getLastArg(options::OPT_ftime_report))
-    CmdArgs.push_back("--device-compiler=-ftime-report");
-
-  if (Args.getLastArg(options::OPT_save_temps_EQ))
-    CmdArgs.push_back("--save-temps");
-
   // Construct the link job so we can wrap around it.
   Linker->ConstructJob(C, JA, Output, Inputs, Args, LinkingOutput);
   const auto &LinkCommand = C.getJobs().getJobs().back();
@@ -9322,12 +9283,10 @@ void LinkerWrapper::ConstructJob(Compilation &C, const JobAction &JA,
                    options::OPT_fno_openmp_target_jit, false))
     CmdArgs.push_back("--embed-bitcode");
 
-  // Forward `-mllvm` arguments to the LLVM invocations if present.
-  for (Arg *A : Args.filtered(options::OPT_mllvm)) {
-    CmdArgs.push_back("-mllvm");
-    CmdArgs.push_back(A->getValue());
-    A->claim();
-  }
+  // Save temporary files created by the linker wrapper.
+  if (Args.hasArg(options::OPT_save_temps_EQ) ||
+      Args.hasArg(options::OPT_save_temps))
+    CmdArgs.push_back("--save-temps");
 
   // Pass in the C library for GPUs if present and not disabled.
   if (!Args.hasArg(options::OPT_nostdlib, options::OPT_r, options::OPT_nogpulib,
@@ -9354,11 +9313,6 @@ void LinkerWrapper::ConstructJob(Compilation &C, const JobAction &JA,
     });
   }
 
-  // If we disable the GPU C library support it needs to be forwarded to the
-  // link job.
-  if (!Args.hasFlag(options::OPT_gpulibc, options::OPT_nogpulibc, true))
-    CmdArgs.push_back("--device-compiler=-nolibc");
-
   // Add the linker arguments to be forwarded by the wrapper.
   CmdArgs.push_back(Args.MakeArgString(Twine("--linker-path=") +
                                        LinkCommand->getExecutable()));
diff --git a/clang/lib/Driver/ToolChains/CommonArgs.cpp b/clang/lib/Driver/ToolChains/CommonArgs.cpp
index c045069c34424..9a4d3f55c911c 100644
--- a/clang/lib/Driver/ToolChains/CommonArgs.cpp
+++ b/clang/lib/Driver/ToolChains/CommonArgs.cpp
@@ -1321,7 +1321,7 @@ void tools::addOpenMPHostOffloadingArgs(const Compilation &C,
 /// Add Fortran runtime libs
 void tools::addFortranRuntimeLibs(const ToolChain &TC, const ArgList &Args,
                                   llvm::opt::ArgStringList &CmdArgs) {
-  // Link FortranRuntime and FortranDecimal
+  // Link flang_rt.runtime
   // These are handled earlier on Windows by telling the frontend driver to
   // add the correct libraries to link against as dependents in the object
   // file.
@@ -1330,16 +1330,20 @@ void tools::addFortranRuntimeLibs(const ToolChain &TC, const ArgList &Args,
     F128LibName.consume_front_insensitive("lib");
     if (!F128LibName.empty()) {
       bool AsNeeded = !TC.getTriple().isOSAIX();
-      CmdArgs.push_back("-lFortranFloat128Math");
+      CmdArgs.push_back("-lflang_rt.quadmath");
       if (AsNeeded)
         addAsNeededOption(TC, Args, CmdArgs, /*as_needed=*/true);
       CmdArgs.push_back(Args.MakeArgString("-l" + F128LibName));
       if (AsNeeded)
         addAsNeededOption(TC, Args, CmdArgs, /*as_needed=*/false);
     }
-    CmdArgs.push_back("-lFortranRuntime");
-    CmdArgs.push_back("-lFortranDecimal");
+    CmdArgs.push_back("-lflang_rt.runtime");
     addArchSpecificRPath(TC, Args, CmdArgs);
+
+    // needs libexecinfo for backtrace functions
+    if (TC.getTriple().isOSFreeBSD() || TC.getTriple().isOSNetBSD() ||
+        TC.getTriple().isOSOpenBSD() || TC.getTriple().isOSDragonFly())
+      CmdArgs.push_back("-lexecinfo");
   }
 
   // libomp needs libatomic for atomic operations if using libgcc
@@ -1440,13 +1444,12 @@ void tools::linkSanitizerRuntimeDeps(const ToolChain &TC,
   CmdArgs.push_back("-lm");
   // There's no libdl on all OSes.
   if (!TC.getTriple().isOSFreeBSD() && !TC.getTriple().isOSNetBSD() &&
-      !TC.getTriple().isOSOpenBSD() &&
+      !TC.getTriple().isOSOpenBSD() && !TC.getTriple().isOSDragonFly() &&
       TC.getTriple().getOS() != llvm::Triple::RTEMS)
     CmdArgs.push_back("-ldl");
   // Required for backtrace on some OSes
-  if (TC.getTriple().isOSFreeBSD() ||
-      TC.getTriple().isOSNetBSD() ||
-      TC.getTriple().isOSOpenBSD())
+  if (TC.getTriple().isOSFreeBSD() || TC.getTriple().isOSNetBSD() ||
+      TC.getTriple().isOSOpenBSD() || TC.getTriple().isOSDragonFly())
     CmdArgs.push_back("-lexecinfo");
   // There is no libresolv on Android, FreeBSD, OpenBSD, etc. On musl
   // libresolv.a, even if exists, is an empty archive to satisfy POSIX -lresolv
@@ -3093,14 +3096,39 @@ bool tools::shouldRecordCommandLine(const ToolChain &TC,
 
 void tools::renderCommonIntegerOverflowOptions(const ArgList &Args,
                                                ArgStringList &CmdArgs) {
-  // -fno-strict-overflow implies -fwrapv if it isn't disabled, but
-  // -fstrict-overflow won't turn off an explicitly enabled -fwrapv.
-  if (Arg *A = Args.getLastArg(options::OPT_fwrapv, options::OPT_fno_wrapv)) {
-    if (A->getOption().matches(options::OPT_fwrapv))
-      CmdArgs.push_back("-fwrapv");
-  } else if (Arg *A = Args.getLastArg(options::OPT_fstrict_overflow,
-                                      options::OPT_fno_strict_overflow)) {
-    if (A->getOption().matches(options::OPT_fno_strict_overflow))
-      CmdArgs.push_back("-fwrapv");
+  bool use_fwrapv = false;
+  bool use_fwrapv_pointer = false;
+  for (const Arg *A : Args.filtered(
+           options::OPT_fstrict_overflow, options::OPT_fno_strict_overflow,
+           options::OPT_fwrapv, options::OPT_fno_wrapv,
+           options::OPT_fwrapv_pointer, options::OPT_fno_wrapv_pointer)) {
+    A->claim();
+    switch (A->getOption().getID()) {
+    case options::OPT_fstrict_overflow:
+      use_fwrapv = false;
+      use_fwrapv_pointer = false;
+      break;
+    case options::OPT_fno_strict_overflow:
+      use_fwrapv = true;
+      use_fwrapv_pointer = true;
+      break;
+    case options::OPT_fwrapv:
+      use_fwrapv = true;
+      break;
+    case options::OPT_fno_wrapv:
+      use_fwrapv = false;
+      break;
+    case options::OPT_fwrapv_pointer:
+      use_fwrapv_pointer = true;
+      break;
+    case options::OPT_fno_wrapv_pointer:
+      use_fwrapv_pointer = false;
+      break;
+    }
   }
+
+  if (use_fwrapv)
+    CmdArgs.push_back("-fwrapv");
+  if (use_fwrapv_pointer)
+    CmdArgs.push_back("-fwrapv-pointer");
 }
diff --git a/clang/lib/Driver/ToolChains/Cuda.cpp b/clang/lib/Driver/ToolChains/Cuda.cpp
index 0922a97ed7c19..d6487d4bc274d 100644
--- a/clang/lib/Driver/ToolChains/Cuda.cpp
+++ b/clang/lib/Driver/ToolChains/Cuda.cpp
@@ -123,7 +123,7 @@ CudaVersion parseCudaHFile(llvm::StringRef Input) {
 }
 } // namespace
 
-void CudaInstallationDetector::WarnIfUnsupportedVersion() {
+void CudaInstallationDetector::WarnIfUnsupportedVersion() const {
   if (Version > CudaVersion::PARTIALLY_SUPPORTED) {
     std::string VersionString = CudaVersionToString(Version);
     if (!VersionString.empty())
@@ -639,9 +639,6 @@ void NVPTX::Linker::ConstructJob(Compilation &C, const JobAction &JA,
   CmdArgs.push_back(
       Args.MakeArgString("--plugin-opt=-mattr=" + llvm::join(Features, ",")));
 
-  // Enable ctor / dtor lowering for the direct / freestanding NVPTX target.
-  CmdArgs.append({"-mllvm", "--nvptx-lower-global-ctor-dtor"});
-
   // Add paths for the default clang library path.
   SmallString<256> DefaultLibPath =
       llvm::sys::path::parent_path(TC.getDriver().Dir);
@@ -726,9 +723,8 @@ void NVPTX::getNVPTXTargetFeatures(const Driver &D, const llvm::Triple &Triple,
 /// toolchain.
 NVPTXToolChain::NVPTXToolChain(const Driver &D, const llvm::Triple &Triple,
                                const llvm::Triple &HostTriple,
-                               const ArgList &Args, bool Freestanding = false)
-    : ToolChain(D, Triple, Args), CudaInstallation(D, HostTriple, Args),
-      Freestanding(Freestanding) {
+                               const ArgList &Args)
+    : ToolChain(D, Triple, Args), CudaInstallation(D, HostTriple, Args) {
   if (CudaInstallation.isValid())
     getProgramPaths().push_back(std::string(CudaInstallation.getBinPath()));
   // Lookup binaries into the driver directory, this is used to
@@ -740,8 +736,7 @@ NVPTXToolChain::NVPTXToolChain(const Driver &D, const llvm::Triple &Triple,
 /// system's default triple if not provided.
 NVPTXToolChain::NVPTXToolChain(const Driver &D, const llvm::Triple &Triple,
                                const ArgList &Args)
-    : NVPTXToolChain(D, Triple, llvm::Triple(LLVM_HOST_TRIPLE), Args,
-                     /*Freestanding=*/true) {}
+    : NVPTXToolChain(D, Triple, llvm::Triple(LLVM_HOST_TRIPLE), Args) {}
 
 llvm::opt::DerivedArgList *
 NVPTXToolChain::TranslateArgs(const llvm::opt::DerivedArgList &Args,
@@ -782,13 +777,7 @@ NVPTXToolChain::TranslateArgs(const llvm::opt::DerivedArgList &Args,
 
 void NVPTXToolChain::addClangTargetOptions(
     const llvm::opt::ArgList &DriverArgs, llvm::opt::ArgStringList &CC1Args,
-    Action::OffloadKind DeviceOffloadingKind) const {
-  // If we are compiling with a standalone NVPTX toolchain we want to try to
-  // mimic a standard environment as much as possible. So we enable lowering
-  // ctor / dtor functions to global symbols that can be registered.
-  if (Freestanding && !getDriver().isUsingLTO())
-    CC1Args.append({"-mllvm", "--nvptx-lower-global-ctor-dtor"});
-}
+    Action::OffloadKind DeviceOffloadingKind) const {}
 
 bool NVPTXToolChain::supportsDebugInfoOption(const llvm::opt::Arg *A) const {
   const Option &O = A->getOption();
@@ -969,34 +958,6 @@ CudaToolChain::TranslateArgs(const llvm::opt::DerivedArgList &Args,
 
   const OptTable &Opts = getDriver().getOpts();
 
-  // For OpenMP device offloading, append derived arguments. Make sure
-  // flags are not duplicated.
-  // Also append the compute capability.
-  if (DeviceOffloadKind == Action::OFK_OpenMP) {
-    for (Arg *A : Args)
-      if (!llvm::is_contained(*DAL, A))
-        DAL->append(A);
-
-    if (!DAL->hasArg(options::OPT_march_EQ)) {
-      StringRef Arch = BoundArch;
-      if (Arch.empty()) {
-        auto ArchsOrErr = getSystemGPUArchs(Args);
-        if (!ArchsOrErr) {
-          std::string ErrMsg =
-              llvm::formatv("{0}", llvm::fmt_consume(ArchsOrErr.takeError()));
-          getDriver().Diag(diag::err_drv_undetermined_gpu_arch)
-              << llvm::Triple::getArchTypeName(getArch()) << ErrMsg << "-march";
-          Arch = OffloadArchToString(OffloadArch::CudaDefault);
-        } else {
-          Arch = Args.MakeArgString(ArchsOrErr->front());
-        }
-      }
-      DAL->AddJoinedArg(nullptr, Opts.getOption(options::OPT_march_EQ), Arch);
-    }
-
-    return DAL;
-  }
-
   for (Arg *A : Args) {
     // Make sure flags are not duplicated.
     if (!llvm::is_contained(*DAL, A)) {
diff --git a/clang/lib/Driver/ToolChains/Cuda.h b/clang/lib/Driver/ToolChains/Cuda.h
index 7a6a6fb209012..259eda6ebcadf 100644
--- a/clang/lib/Driver/ToolChains/Cuda.h
+++ b/clang/lib/Driver/ToolChains/Cuda.h
@@ -74,7 +74,7 @@ class CudaInstallationDetector {
   std::string getLibDeviceFile(StringRef Gpu) const {
     return LibDeviceMap.lookup(Gpu);
   }
-  void WarnIfUnsupportedVersion();
+  void WarnIfUnsupportedVersion() const;
 };
 
 namespace tools {
@@ -132,8 +132,8 @@ namespace toolchains {
 class LLVM_LIBRARY_VISIBILITY NVPTXToolChain : public ToolChain {
 public:
   NVPTXToolChain(const Driver &D, const llvm::Triple &Triple,
-                 const llvm::Triple &HostTriple, const llvm::opt::ArgList &Args,
-                 bool Freestanding);
+                 const llvm::Triple &HostTriple,
+                 const llvm::opt::ArgList &Args);
 
   NVPTXToolChain(const Driver &D, const llvm::Triple &Triple,
                  const llvm::opt::ArgList &Args);
@@ -179,9 +179,6 @@ class LLVM_LIBRARY_VISIBILITY NVPTXToolChain : public ToolChain {
 protected:
   Tool *buildAssembler() const override; // ptxas.
   Tool *buildLinker() const override;    // nvlink.
-
-private:
-  bool Freestanding = false;
 };
 
 class LLVM_LIBRARY_VISIBILITY CudaToolChain : public NVPTXToolChain {
diff --git a/clang/lib/Driver/ToolChains/Darwin.cpp b/clang/lib/Driver/ToolChains/Darwin.cpp
index 55c55bad73934..b26c5bf1a909e 100644
--- a/clang/lib/Driver/ToolChains/Darwin.cpp
+++ b/clang/lib/Driver/ToolChains/Darwin.cpp
@@ -588,20 +588,6 @@ void darwin::Linker::ConstructJob(Compilation &C, const JobAction &JA,
   // more information.
   ArgStringList CmdArgs;
 
-  /// Hack(tm) to ignore linking errors when we are doing ARC migration.
-  if (Args.hasArg(options::OPT_ccc_arcmt_check,
-                  options::OPT_ccc_arcmt_migrate)) {
-    for (const auto &Arg : Args)
-      Arg->claim();
-    const char *Exec =
-        Args.MakeArgString(getToolChain().GetProgramPath("touch"));
-    CmdArgs.push_back(Output.getFilename());
-    C.addCommand(std::make_unique<Command>(JA, *this,
-                                           ResponseFileSupport::None(), Exec,
-                                           CmdArgs, std::nullopt, Output));
-    return;
-  }
-
   VersionTuple Version = getMachOToolChain().getLinkerVersion(Args);
 
   bool LinkerIsLLD;
@@ -2791,30 +2777,6 @@ DerivedArgList *MachO::TranslateArgs(const DerivedArgList &Args,
   // and try to push it down into tool specific logic.
 
   for (Arg *A : Args) {
-    if (A->getOption().matches(options::OPT_Xarch__)) {
-      // Skip this argument unless the architecture matches either the toolchain
-      // triple arch, or the arch being bound.
-      StringRef XarchArch = A->getValue(0);
-      if (!(XarchArch == getArchName() ||
-            (!BoundArch.empty() && XarchArch == BoundArch)))
-        continue;
-
-      Arg *OriginalArg = A;
-      TranslateXarchArgs(Args, A, DAL);
-
-      // Linker input arguments require custom handling. The problem is that we
-      // have already constructed the phase actions, so we can not treat them as
-      // "input arguments".
-      if (A->getOption().hasFlag(options::LinkerInput)) {
-        // Convert the argument into individual Zlinker_input_args.
-        for (const char *Value : A->getValues()) {
-          DAL->AddSeparateArg(
-              OriginalArg, Opts.getOption(options::OPT_Zlinker_input), Value);
-        }
-        continue;
-      }
-    }
-
     // Sob. These is strictly gcc compatible for the time being. Apple
     // gcc translates options twice, which means that self-expanding
     // options add duplicates.
diff --git a/clang/lib/Driver/ToolChains/Flang.cpp b/clang/lib/Driver/ToolChains/Flang.cpp
index 45d05ed3e2485..591003f56e8bb 100644
--- a/clang/lib/Driver/ToolChains/Flang.cpp
+++ b/clang/lib/Driver/ToolChains/Flang.cpp
@@ -42,6 +42,7 @@ void Flang::addFortranDialectOptions(const ArgList &Args,
                             options::OPT_fopenacc,
                             options::OPT_finput_charset_EQ,
                             options::OPT_fimplicit_none,
+                            options::OPT_fimplicit_none_ext,
                             options::OPT_fno_implicit_none,
                             options::OPT_fbackslash,
                             options::OPT_fno_backslash,
@@ -58,7 +59,8 @@ void Flang::addFortranDialectOptions(const ArgList &Args,
                             options::OPT_fhermetic_module_files,
                             options::OPT_frealloc_lhs,
                             options::OPT_fno_realloc_lhs,
-                            options::OPT_fsave_main_program});
+                            options::OPT_fsave_main_program,
+                            options::OPT_fno_save_main_program});
 }
 
 void Flang::addPreprocessingOptions(const ArgList &Args,
@@ -344,11 +346,15 @@ static void processVSRuntimeLibrary(const ToolChain &TC, const ArgList &Args,
                                     ArgStringList &CmdArgs) {
   assert(TC.getTriple().isKnownWindowsMSVCEnvironment() &&
          "can only add VS runtime library on Windows!");
-  // if -fno-fortran-main has been passed, skip linking Fortran_main.a
-  if (TC.getTriple().isKnownWindowsMSVCEnvironment()) {
-    CmdArgs.push_back(Args.MakeArgString(
-        "--dependent-lib=" + TC.getCompilerRTBasename(Args, "builtins")));
-  }
+
+  // Flang/Clang (including clang-cl) -compiled programs targeting the MSVC ABI
+  // should only depend on msv(u)crt. LLVM still emits libgcc/compiler-rt
+  // functions in some cases like 128-bit integer math (__udivti3, __modti3,
+  // __fixsfti, __floattidf, ...) that msvc does not support. We are injecting a
+  // dependency to Compiler-RT's builtin library where these are implemented.
+  CmdArgs.push_back(Args.MakeArgString(
+      "--dependent-lib=" + TC.getCompilerRTBasename(Args, "builtins")));
+
   unsigned RTOptionID = options::OPT__SLASH_MT;
   if (auto *rtl = Args.getLastArg(options::OPT_fms_runtime_lib_EQ)) {
     RTOptionID = llvm::StringSwitch<unsigned>(rtl->getValue())
@@ -362,30 +368,26 @@ static void processVSRuntimeLibrary(const ToolChain &TC, const ArgList &Args,
   case options::OPT__SLASH_MT:
     CmdArgs.push_back("-D_MT");
     CmdArgs.push_back("--dependent-lib=libcmt");
-    CmdArgs.push_back("--dependent-lib=FortranRuntime.static.lib");
-    CmdArgs.push_back("--dependent-lib=FortranDecimal.static.lib");
+    CmdArgs.push_back("--dependent-lib=flang_rt.runtime.static.lib");
     break;
   case options::OPT__SLASH_MTd:
     CmdArgs.push_back("-D_MT");
     CmdArgs.push_back("-D_DEBUG");
     CmdArgs.push_back("--dependent-lib=libcmtd");
-    CmdArgs.push_back("--dependent-lib=FortranRuntime.static_dbg.lib");
-    CmdArgs.push_back("--dependent-lib=FortranDecimal.static_dbg.lib");
+    CmdArgs.push_back("--dependent-lib=flang_rt.runtime.static_dbg.lib");
     break;
   case options::OPT__SLASH_MD:
     CmdArgs.push_back("-D_MT");
     CmdArgs.push_back("-D_DLL");
     CmdArgs.push_back("--dependent-lib=msvcrt");
-    CmdArgs.push_back("--dependent-lib=FortranRuntime.dynamic.lib");
-    CmdArgs.push_back("--dependent-lib=FortranDecimal.dynamic.lib");
+    CmdArgs.push_back("--dependent-lib=flang_rt.runtime.dynamic.lib");
     break;
   case options::OPT__SLASH_MDd:
     CmdArgs.push_back("-D_MT");
     CmdArgs.push_back("-D_DEBUG");
     CmdArgs.push_back("-D_DLL");
     CmdArgs.push_back("--dependent-lib=msvcrtd");
-    CmdArgs.push_back("--dependent-lib=FortranRuntime.dynamic_dbg.lib");
-    CmdArgs.push_back("--dependent-lib=FortranDecimal.dynamic_dbg.lib");
+    CmdArgs.push_back("--dependent-lib=flang_rt.runtime.dynamic_dbg.lib");
     break;
   }
 }
diff --git a/clang/lib/Driver/ToolChains/HIPAMD.cpp b/clang/lib/Driver/ToolChains/HIPAMD.cpp
index ccee065b59064..0e50eddd6b3d2 100644
--- a/clang/lib/Driver/ToolChains/HIPAMD.cpp
+++ b/clang/lib/Driver/ToolChains/HIPAMD.cpp
@@ -216,17 +216,8 @@ HIPAMDToolChain::HIPAMDToolChain(const Driver &D, const llvm::Triple &Triple,
   // Lookup binaries into the driver directory, this is used to
   // discover the clang-offload-bundler executable.
   getProgramPaths().push_back(getDriver().Dir);
-
   // Diagnose unsupported sanitizer options only once.
-  if (!Args.hasFlag(options::OPT_fgpu_sanitize, options::OPT_fno_gpu_sanitize,
-                    true))
-    return;
-  for (auto *A : Args.filtered(options::OPT_fsanitize_EQ)) {
-    SanitizerMask K = parseSanitizerValue(A->getValue(), /*AllowGroups=*/false);
-    if (K != SanitizerKind::Address)
-      D.getDiags().Report(clang::diag::warn_drv_unsupported_option_for_target)
-          << A->getAsString(Args) << getTriple().str();
-  }
+  diagnoseUnsupportedSanitizers(Args);
 }
 
 void HIPAMDToolChain::addClangTargetOptions(
@@ -382,7 +373,7 @@ HIPAMDToolChain::getDeviceLibs(const llvm::opt::ArgList &DriverArgs) const {
         llvm::sys::path::append(Path, BCName);
         FullName = Path;
         if (llvm::sys::fs::exists(FullName)) {
-          BCLibs.push_back(FullName);
+          BCLibs.emplace_back(FullName);
           return;
         }
       }
@@ -396,28 +387,11 @@ HIPAMDToolChain::getDeviceLibs(const llvm::opt::ArgList &DriverArgs) const {
     StringRef GpuArch = getGPUArch(DriverArgs);
     assert(!GpuArch.empty() && "Must have an explicit GPU arch.");
 
-    // If --hip-device-lib is not set, add the default bitcode libraries.
-    if (DriverArgs.hasFlag(options::OPT_fgpu_sanitize,
-                           options::OPT_fno_gpu_sanitize, true) &&
-        getSanitizerArgs(DriverArgs).needsAsanRt()) {
-      auto AsanRTL = RocmInstallation->getAsanRTLPath();
-      if (AsanRTL.empty()) {
-        unsigned DiagID = getDriver().getDiags().getCustomDiagID(
-            DiagnosticsEngine::Error,
-            "AMDGPU address sanitizer runtime library (asanrtl) is not found. "
-            "Please install ROCm device library which supports address "
-            "sanitizer");
-        getDriver().Diag(DiagID);
-        return {};
-      } else
-        BCLibs.emplace_back(AsanRTL, /*ShouldInternalize=*/false);
-    }
-
     // Add the HIP specific bitcode library.
-    BCLibs.push_back(RocmInstallation->getHIPPath());
+    BCLibs.emplace_back(RocmInstallation->getHIPPath());
 
     // Add common device libraries like ocml etc.
-    for (StringRef N : getCommonDeviceLibNames(DriverArgs, GpuArch.str()))
+    for (auto N : getCommonDeviceLibNames(DriverArgs, GpuArch.str()))
       BCLibs.emplace_back(N);
 
     // Add instrument lib.
@@ -426,7 +400,7 @@ HIPAMDToolChain::getDeviceLibs(const llvm::opt::ArgList &DriverArgs) const {
     if (InstLib.empty())
       return BCLibs;
     if (llvm::sys::fs::exists(InstLib))
-      BCLibs.push_back(InstLib);
+      BCLibs.emplace_back(InstLib);
     else
       getDriver().Diag(diag::err_drv_no_such_file) << InstLib;
   }
diff --git a/clang/lib/Driver/ToolChains/Hexagon.cpp b/clang/lib/Driver/ToolChains/Hexagon.cpp
index 76cedf312d68a..7ca5ab9af8810 100644
--- a/clang/lib/Driver/ToolChains/Hexagon.cpp
+++ b/clang/lib/Driver/ToolChains/Hexagon.cpp
@@ -802,9 +802,7 @@ bool HexagonToolChain::isAutoHVXEnabled(const llvm::opt::ArgList &Args) {
 // Returns the default CPU for Hexagon. This is the default compilation target
 // if no Hexagon processor is selected at the command-line.
 //
-StringRef HexagonToolChain::GetDefaultCPU() {
-  return "hexagonv60";
-}
+StringRef HexagonToolChain::GetDefaultCPU() { return "hexagonv68"; }
 
 StringRef HexagonToolChain::GetTargetCPUVersion(const ArgList &Args) {
   Arg *CpuArg = nullptr;
diff --git a/clang/lib/Driver/ToolChains/ROCm.h b/clang/lib/Driver/ToolChains/ROCm.h
index dceb0ab036693..681c242b0678e 100644
--- a/clang/lib/Driver/ToolChains/ROCm.h
+++ b/clang/lib/Driver/ToolChains/ROCm.h
@@ -13,6 +13,7 @@
 #include "clang/Basic/LLVM.h"
 #include "clang/Driver/Driver.h"
 #include "clang/Driver/Options.h"
+#include "clang/Driver/SanitizerArgs.h"
 #include "llvm/ADT/SmallString.h"
 #include "llvm/ADT/StringMap.h"
 #include "llvm/Option/ArgList.h"
@@ -173,12 +174,11 @@ class RocmInstallationDetector {
 
   /// Get file paths of default bitcode libraries common to AMDGPU based
   /// toolchains.
-  llvm::SmallVector<std::string, 12>
-  getCommonBitcodeLibs(const llvm::opt::ArgList &DriverArgs,
-                       StringRef LibDeviceFile, bool Wave64, bool DAZ,
-                       bool FiniteOnly, bool UnsafeMathOpt,
-                       bool FastRelaxedMath, bool CorrectSqrt,
-                       DeviceLibABIVersion ABIVer, bool isOpenMP) const;
+  llvm::SmallVector<ToolChain::BitCodeLibraryInfo, 12> getCommonBitcodeLibs(
+      const llvm::opt::ArgList &DriverArgs, StringRef LibDeviceFile,
+      bool Wave64, bool DAZ, bool FiniteOnly, bool UnsafeMathOpt,
+      bool FastRelaxedMath, bool CorrectSqrt, DeviceLibABIVersion ABIVer,
+      const std::tuple<bool, const SanitizerArgs> &GPUSan, bool isOpenMP) const;
   /// Check file paths of default bitcode libraries common to AMDGPU based
   /// toolchains. \returns false if there are invalid or missing files.
   bool checkCommonBitcodeLibs(StringRef GPUArch, StringRef LibDeviceFile,
diff --git a/clang/lib/Driver/ToolChains/UEFI.h b/clang/lib/Driver/ToolChains/UEFI.h
index 7e038b5cb8b18..4474c2fcc6939 100644
--- a/clang/lib/Driver/ToolChains/UEFI.h
+++ b/clang/lib/Driver/ToolChains/UEFI.h
@@ -55,6 +55,10 @@ class LLVM_LIBRARY_VISIBILITY UEFI : public ToolChain {
   void
   AddClangSystemIncludeArgs(const llvm::opt::ArgList &DriverArgs,
                             llvm::opt::ArgStringList &CC1Args) const override;
+
+  llvm::codegenoptions::DebugInfoFormat getDefaultDebugFormat() const override {
+    return llvm::codegenoptions::DIF_CodeView;
+  }
 };
 
 } // namespace toolchains
diff --git a/clang/lib/Driver/ToolChains/WebAssembly.cpp b/clang/lib/Driver/ToolChains/WebAssembly.cpp
index eebe3becada65..bd25fd1a8933a 100644
--- a/clang/lib/Driver/ToolChains/WebAssembly.cpp
+++ b/clang/lib/Driver/ToolChains/WebAssembly.cpp
@@ -344,12 +344,15 @@ void WebAssembly::addClangTargetOptions(const ArgList &DriverArgs,
     }
   }
 
+  bool HasBannedIncompatibleOptionsForWasmEHSjLj = false;
+  bool HasEnabledFeaturesForWasmEHSjLj = false;
+
   // Bans incompatible options for Wasm EH / SjLj. We don't allow using
   // different modes for EH and SjLj.
   auto BanIncompatibleOptionsForWasmEHSjLj = [&](StringRef CurOption) {
-    static bool HasRun = false;
-    if (HasRun)
+    if (HasBannedIncompatibleOptionsForWasmEHSjLj)
       return;
+    HasBannedIncompatibleOptionsForWasmEHSjLj = true;
     if (DriverArgs.hasFlag(options::OPT_mno_exception_handing,
                            options::OPT_mexception_handing, false))
       getDriver().Diag(diag::err_drv_argument_not_allowed_with)
@@ -373,14 +376,13 @@ void WebAssembly::addClangTargetOptions(const ArgList &DriverArgs,
               << CurOption << Option;
       }
     }
-    HasRun = true;
   };
 
   // Enable necessary features for Wasm EH / SjLj in the backend.
   auto EnableFeaturesForWasmEHSjLj = [&]() {
-    static bool HasRun = false;
-    if (HasRun)
+    if (HasEnabledFeaturesForWasmEHSjLj)
       return;
+    HasEnabledFeaturesForWasmEHSjLj = true;
     CC1Args.push_back("-target-feature");
     CC1Args.push_back("+exception-handling");
     // The standardized Wasm EH spec requires multivalue and reference-types.
@@ -390,7 +392,6 @@ void WebAssembly::addClangTargetOptions(const ArgList &DriverArgs,
     CC1Args.push_back("+reference-types");
     // Backend needs '-exception-model=wasm' to use Wasm EH instructions
     CC1Args.push_back("-exception-model=wasm");
-    HasRun = true;
   };
 
   if (DriverArgs.getLastArg(options::OPT_fwasm_exceptions)) {
diff --git a/clang/lib/Edit/RewriteObjCFoundationAPI.cpp b/clang/lib/Edit/RewriteObjCFoundationAPI.cpp
index 81797c8c4dc75..32f5ebb55155e 100644
--- a/clang/lib/Edit/RewriteObjCFoundationAPI.cpp
+++ b/clang/lib/Edit/RewriteObjCFoundationAPI.cpp
@@ -1085,6 +1085,7 @@ static bool rewriteToNumericBoxedExpression(const ObjCMessageExpr *Msg,
       llvm_unreachable("OpenCL-specific cast in Objective-C?");
 
     case CK_HLSLVectorTruncation:
+    case CK_HLSLElementwiseCast:
       llvm_unreachable("HLSL-specific cast in Objective-C?");
       break;
 
diff --git a/clang/lib/Format/ContinuationIndenter.cpp b/clang/lib/Format/ContinuationIndenter.cpp
index c311deaa17bb0..3e51b4aab1082 100644
--- a/clang/lib/Format/ContinuationIndenter.cpp
+++ b/clang/lib/Format/ContinuationIndenter.cpp
@@ -356,6 +356,16 @@ bool ContinuationIndenter::canBreak(const LineState &State) {
     return CurrentState.BreakBeforeClosingBrace;
   }
 
+  // Allow breaking before the right parens with block indentation if there was
+  // a break after the left parens, which is tracked by BreakBeforeClosingParen.
+  if (Style.AlignAfterOpenBracket == FormatStyle::BAS_BlockIndent &&
+      Current.is(tok::r_paren)) {
+    return CurrentState.BreakBeforeClosingParen;
+  }
+
+  if (Style.BreakBeforeTemplateCloser && Current.is(TT_TemplateCloser))
+    return CurrentState.BreakBeforeClosingAngle;
+
   // If binary operators are moved to the next line (including commas for some
   // styles of constructor initializers), that's always ok.
   if (!Current.isOneOf(TT_BinaryOperator, tok::comma) &&
@@ -407,6 +417,8 @@ bool ContinuationIndenter::mustBreak(const LineState &State) {
   }
   if (CurrentState.BreakBeforeClosingParen && Current.is(tok::r_paren))
     return true;
+  if (CurrentState.BreakBeforeClosingAngle && Current.is(TT_TemplateCloser))
+    return true;
   if (Style.Language == FormatStyle::LK_ObjC &&
       Style.ObjCBreakBeforeNestedBlockParam &&
       Current.ObjCSelectorNameParts > 1 &&
@@ -1236,6 +1248,9 @@ unsigned ContinuationIndenter::addTokenOnNewLine(LineState &State,
         Style.AlignAfterOpenBracket == FormatStyle::BAS_BlockIndent;
   }
 
+  if (PreviousNonComment && PreviousNonComment->is(TT_TemplateOpener))
+    CurrentState.BreakBeforeClosingAngle = Style.BreakBeforeTemplateCloser;
+
   if (CurrentState.AvoidBinPacking) {
     // If we are breaking after '(', '{', '<', or this is the break after a ':'
     // to start a member initializer list in a constructor, this should not
@@ -1372,6 +1387,10 @@ unsigned ContinuationIndenter::getNewLineColumn(const LineState &State) {
       State.Stack.size() > 1) {
     return State.Stack[State.Stack.size() - 2].LastSpace;
   }
+  if (Style.BreakBeforeTemplateCloser && Current.is(TT_TemplateCloser) &&
+      State.Stack.size() > 1) {
+    return State.Stack[State.Stack.size() - 2].LastSpace;
+  }
   if (NextNonComment->is(TT_TemplateString) && NextNonComment->closesScope())
     return State.Stack[State.Stack.size() - 2].LastSpace;
   // Field labels in a nested type should be aligned to the brace. For example
diff --git a/clang/lib/Format/ContinuationIndenter.h b/clang/lib/Format/ContinuationIndenter.h
index 18441e10a1249..ac354aa96f86e 100644
--- a/clang/lib/Format/ContinuationIndenter.h
+++ b/clang/lib/Format/ContinuationIndenter.h
@@ -200,14 +200,15 @@ struct ParenState {
       : Tok(Tok), Indent(Indent), LastSpace(LastSpace),
         NestedBlockIndent(Indent), IsAligned(false),
         BreakBeforeClosingBrace(false), BreakBeforeClosingParen(false),
-        AvoidBinPacking(AvoidBinPacking), BreakBeforeParameter(false),
-        NoLineBreak(NoLineBreak), NoLineBreakInOperand(false),
-        LastOperatorWrapped(true), ContainsLineBreak(false),
-        ContainsUnwrappedBuilder(false), AlignColons(true),
-        ObjCSelectorNameFound(false), HasMultipleNestedBlocks(false),
-        NestedBlockInlined(false), IsInsideObjCArrayLiteral(false),
-        IsCSharpGenericTypeConstraint(false), IsChainedConditional(false),
-        IsWrappedConditional(false), UnindentOperator(false) {}
+        BreakBeforeClosingAngle(false), AvoidBinPacking(AvoidBinPacking),
+        BreakBeforeParameter(false), NoLineBreak(NoLineBreak),
+        NoLineBreakInOperand(false), LastOperatorWrapped(true),
+        ContainsLineBreak(false), ContainsUnwrappedBuilder(false),
+        AlignColons(true), ObjCSelectorNameFound(false),
+        HasMultipleNestedBlocks(false), NestedBlockInlined(false),
+        IsInsideObjCArrayLiteral(false), IsCSharpGenericTypeConstraint(false),
+        IsChainedConditional(false), IsWrappedConditional(false),
+        UnindentOperator(false) {}
 
   /// \brief The token opening this parenthesis level, or nullptr if this level
   /// is opened by fake parenthesis.
@@ -280,6 +281,9 @@ struct ParenState {
   /// was a newline after the beginning left paren.
   bool BreakBeforeClosingParen : 1;
 
+  /// Whether a newline needs to be inserted before a closing angle `>`.
+  bool BreakBeforeClosingAngle : 1;
+
   /// Avoid bin packing, i.e. multiple parameters/elements on multiple
   /// lines, in this context.
   bool AvoidBinPacking : 1;
@@ -367,6 +371,8 @@ struct ParenState {
       return BreakBeforeClosingBrace;
     if (BreakBeforeClosingParen != Other.BreakBeforeClosingParen)
       return BreakBeforeClosingParen;
+    if (BreakBeforeClosingAngle != Other.BreakBeforeClosingAngle)
+      return BreakBeforeClosingAngle;
     if (QuestionColumn != Other.QuestionColumn)
       return QuestionColumn < Other.QuestionColumn;
     if (AvoidBinPacking != Other.AvoidBinPacking)
diff --git a/clang/lib/Format/Format.cpp b/clang/lib/Format/Format.cpp
index c25d9bf7c2251..0898b69528ebc 100644
--- a/clang/lib/Format/Format.cpp
+++ b/clang/lib/Format/Format.cpp
@@ -995,6 +995,7 @@ template <> struct MappingTraits<FormatStyle> {
                    Style.AlwaysBreakBeforeMultilineStrings);
     IO.mapOptional("AttributeMacros", Style.AttributeMacros);
     IO.mapOptional("BinPackArguments", Style.BinPackArguments);
+    IO.mapOptional("BinPackLongBracedList", Style.BinPackLongBracedList);
     IO.mapOptional("BinPackParameters", Style.BinPackParameters);
     IO.mapOptional("BitFieldColonSpacing", Style.BitFieldColonSpacing);
     IO.mapOptional("BracedInitializerIndentWidth",
@@ -1014,6 +1015,8 @@ template <> struct MappingTraits<FormatStyle> {
     IO.mapOptional("BreakBeforeBraces", Style.BreakBeforeBraces);
     IO.mapOptional("BreakBeforeInlineASMColon",
                    Style.BreakBeforeInlineASMColon);
+    IO.mapOptional("BreakBeforeTemplateCloser",
+                   Style.BreakBeforeTemplateCloser);
     IO.mapOptional("BreakBeforeTernaryOperators",
                    Style.BreakBeforeTernaryOperators);
     IO.mapOptional("BreakBinaryOperations", Style.BreakBinaryOperations);
@@ -1091,6 +1094,8 @@ template <> struct MappingTraits<FormatStyle> {
     IO.mapOptional("PenaltyBreakAssignment", Style.PenaltyBreakAssignment);
     IO.mapOptional("PenaltyBreakBeforeFirstCallParameter",
                    Style.PenaltyBreakBeforeFirstCallParameter);
+    IO.mapOptional("PenaltyBreakBeforeMemberAccess",
+                   Style.PenaltyBreakBeforeMemberAccess);
     IO.mapOptional("PenaltyBreakComment", Style.PenaltyBreakComment);
     IO.mapOptional("PenaltyBreakFirstLessLess",
                    Style.PenaltyBreakFirstLessLess);
@@ -1503,6 +1508,7 @@ FormatStyle getLLVMStyle(FormatStyle::LanguageKind Language) {
   LLVMStyle.AlwaysBreakBeforeMultilineStrings = false;
   LLVMStyle.AttributeMacros.push_back("__capability");
   LLVMStyle.BinPackArguments = true;
+  LLVMStyle.BinPackLongBracedList = true;
   LLVMStyle.BinPackParameters = FormatStyle::BPPS_BinPack;
   LLVMStyle.BitFieldColonSpacing = FormatStyle::BFCS_Both;
   LLVMStyle.BracedInitializerIndentWidth = std::nullopt;
@@ -1533,6 +1539,7 @@ FormatStyle getLLVMStyle(FormatStyle::LanguageKind Language) {
   LLVMStyle.BreakBeforeBraces = FormatStyle::BS_Attach;
   LLVMStyle.BreakBeforeConceptDeclarations = FormatStyle::BBCDS_Always;
   LLVMStyle.BreakBeforeInlineASMColon = FormatStyle::BBIAS_OnlyMultiline;
+  LLVMStyle.BreakBeforeTemplateCloser = false;
   LLVMStyle.BreakBeforeTernaryOperators = true;
   LLVMStyle.BreakBinaryOperations = FormatStyle::BBO_Never;
   LLVMStyle.BreakConstructorInitializers = FormatStyle::BCIS_BeforeColon;
@@ -1659,6 +1666,7 @@ FormatStyle getLLVMStyle(FormatStyle::LanguageKind Language) {
 
   LLVMStyle.PenaltyBreakAssignment = prec::Assignment;
   LLVMStyle.PenaltyBreakBeforeFirstCallParameter = 19;
+  LLVMStyle.PenaltyBreakBeforeMemberAccess = 150;
   LLVMStyle.PenaltyBreakComment = 300;
   LLVMStyle.PenaltyBreakFirstLessLess = 120;
   LLVMStyle.PenaltyBreakOpenParenthesis = 0;
diff --git a/clang/lib/Format/FormatToken.cpp b/clang/lib/Format/FormatToken.cpp
index 963e8f87793fa..fb040a0043602 100644
--- a/clang/lib/Format/FormatToken.cpp
+++ b/clang/lib/Format/FormatToken.cpp
@@ -42,11 +42,12 @@ static SmallVector<StringRef> CppNonKeywordTypes = {
 };
 
 bool FormatToken::isTypeName(const LangOptions &LangOpts) const {
+  if (is(TT_TypeName) || Tok.isSimpleTypeSpecifier(LangOpts))
+    return true;
   const bool IsCpp = LangOpts.CXXOperatorNames;
-  return is(TT_TypeName) || Tok.isSimpleTypeSpecifier(LangOpts) ||
-         (IsCpp && is(tok::identifier) &&
-          std::binary_search(CppNonKeywordTypes.begin(),
-                             CppNonKeywordTypes.end(), TokenText));
+  return IsCpp && is(tok::identifier) &&
+         std::binary_search(CppNonKeywordTypes.begin(),
+                            CppNonKeywordTypes.end(), TokenText);
 }
 
 bool FormatToken::isTypeOrIdentifier(const LangOptions &LangOpts) const {
@@ -174,7 +175,7 @@ void CommaSeparatedList::precomputeFormattingInfos(const FormatToken *Token) {
   // have many items (20 or more) or we allow bin-packing of function call
   // arguments.
   if (Style.Cpp11BracedListStyle && !Style.BinPackArguments &&
-      Commas.size() < 19) {
+      (Commas.size() < 19 || !Style.BinPackLongBracedList)) {
     return;
   }
 
diff --git a/clang/lib/Format/FormatToken.h b/clang/lib/Format/FormatToken.h
index d97b6522f1fef..29aba281ae103 100644
--- a/clang/lib/Format/FormatToken.h
+++ b/clang/lib/Format/FormatToken.h
@@ -44,6 +44,8 @@ namespace format {
   TYPE(CaseLabelColon)                                                         \
   TYPE(CastRParen)                                                             \
   TYPE(ClassLBrace)                                                            \
+  /* Name of class/struct/union/interface definition. */                       \
+  TYPE(ClassHeadName)                                                          \
   TYPE(ClassRBrace)                                                            \
   TYPE(CompoundRequirementLBrace)                                              \
   /* ternary ?: expression */                                                  \
diff --git a/clang/lib/Format/QualifierAlignmentFixer.cpp b/clang/lib/Format/QualifierAlignmentFixer.cpp
index 21fb5074b4928..23e8b44eee15c 100644
--- a/clang/lib/Format/QualifierAlignmentFixer.cpp
+++ b/clang/lib/Format/QualifierAlignmentFixer.cpp
@@ -132,8 +132,10 @@ static void rotateTokens(const SourceManager &SourceMgr,
   // Then move through the other tokens.
   auto *Tok = Begin;
   while (Tok != End) {
-    if (!NewText.empty() && !endsWithSpace(NewText))
+    if (!NewText.empty() && !endsWithSpace(NewText) &&
+        Tok->isNot(tok::coloncolon)) {
       NewText += " ";
+    }
 
     NewText += Tok->TokenText;
     Tok = Tok->Next;
@@ -412,6 +414,14 @@ const FormatToken *LeftRightQualifierAlignmentFixer::analyzeLeft(
   // The case `const long long volatile int` -> `const volatile long long int`
   // The case `long volatile long int const` -> `const volatile long long int`
   if (TypeToken->isTypeName(LangOpts)) {
+    for (const auto *Prev = TypeToken->Previous;
+         Prev && Prev->is(tok::coloncolon); Prev = Prev->Previous) {
+      TypeToken = Prev;
+      Prev = Prev->Previous;
+      if (!(Prev && Prev->is(tok::identifier)))
+        break;
+      TypeToken = Prev;
+    }
     const FormatToken *LastSimpleTypeSpecifier = TypeToken;
     while (isConfiguredQualifierOrType(
         LastSimpleTypeSpecifier->getPreviousNonComment(),
diff --git a/clang/lib/Format/TokenAnnotator.cpp b/clang/lib/Format/TokenAnnotator.cpp
index f36cf7b638e0d..b3540f39e6f69 100644
--- a/clang/lib/Format/TokenAnnotator.cpp
+++ b/clang/lib/Format/TokenAnnotator.cpp
@@ -252,10 +252,10 @@ class AnnotatingParser {
       // parameters.
       // FIXME: This is getting out of hand, write a decent parser.
       if (MaybeAngles && InExpr && !Line.startsWith(tok::kw_template) &&
-          Prev.is(TT_BinaryOperator)) {
-        const auto Precedence = Prev.getPrecedence();
-        if (Precedence > prec::Conditional && Precedence < prec::Relational)
-          MaybeAngles = false;
+          Prev.is(TT_BinaryOperator) &&
+          (Prev.isOneOf(tok::pipepipe, tok::ampamp) ||
+           Prev.getPrecedence() == prec::Equality)) {
+        MaybeAngles = false;
       }
       if (Prev.isOneOf(tok::question, tok::colon) && !Style.isProto())
         SeenTernaryOperator = true;
@@ -477,8 +477,9 @@ class AnnotatingParser {
     FormatToken *PossibleObjCForInToken = nullptr;
     while (CurrentToken) {
       const auto &Prev = *CurrentToken->Previous;
+      const auto *PrevPrev = Prev.Previous;
       if (Prev.is(TT_PointerOrReference) &&
-          Prev.Previous->isOneOf(tok::l_paren, tok::coloncolon)) {
+          PrevPrev->isOneOf(tok::l_paren, tok::coloncolon)) {
         ProbablyFunctionType = true;
       }
       if (CurrentToken->is(tok::comma))
@@ -486,8 +487,10 @@ class AnnotatingParser {
       if (Prev.is(TT_BinaryOperator))
         Contexts.back().IsExpression = true;
       if (CurrentToken->is(tok::r_paren)) {
-        if (Prev.is(TT_PointerOrReference) && Prev.Previous == &OpeningParen)
+        if (Prev.is(TT_PointerOrReference) &&
+            (PrevPrev == &OpeningParen || PrevPrev->is(tok::coloncolon))) {
           MightBeFunctionType = true;
+        }
         if (OpeningParen.isNot(TT_CppCastLParen) && MightBeFunctionType &&
             ProbablyFunctionType && CurrentToken->Next &&
             (CurrentToken->Next->is(tok::l_paren) ||
@@ -1115,7 +1118,7 @@ class AnnotatingParser {
       }
       if (!CurrentToken || CurrentToken->isNot(tok::l_paren))
         return false;
-      skipToNextNonComment();
+      next();
       // FIXME: Hack using inheritance to child context
       Contexts.back().IsTableGenBangOpe = true;
       bool Result = parseParens();
@@ -1124,12 +1127,10 @@ class AnnotatingParser {
     }
     // SimpleValue 9: Cond operator
     if (Tok->is(TT_TableGenCondOperator)) {
-      Tok = CurrentToken;
-      skipToNextNonComment();
-      if (!Tok || Tok->isNot(tok::l_paren))
+      if (!CurrentToken || CurrentToken->isNot(tok::l_paren))
         return false;
-      bool Result = parseParens();
-      return Result;
+      next();
+      return parseParens();
     }
     // We have to check identifier at the last because the kind of bang/cond
     // operators are also identifier.
@@ -1567,7 +1568,8 @@ class AnnotatingParser {
         if (const auto *Previous = Tok->Previous;
             !Previous ||
             (!Previous->isAttribute() &&
-             !Previous->isOneOf(TT_RequiresClause, TT_LeadingJavaAnnotation))) {
+             !Previous->isOneOf(TT_RequiresClause, TT_LeadingJavaAnnotation,
+                                TT_BinaryOperator))) {
           Line.MightBeFunctionDecl = true;
           Tok->MightBeFunctionDeclParen = true;
         }
@@ -2582,9 +2584,14 @@ class AnnotatingParser {
     if (Style.isVerilog())
       return false;
 
-    if (Tok.isNot(tok::identifier) || !Tok.Previous)
+    if (!Tok.Previous || Tok.isNot(tok::identifier) || Tok.is(TT_ClassHeadName))
       return false;
 
+    if ((Style.isJavaScript() || Style.Language == FormatStyle::LK_Java) &&
+        Tok.is(Keywords.kw_extends)) {
+      return false;
+    }
+
     if (const auto *NextNonComment = Tok.getNextNonComment();
         (!NextNonComment && !Line.InMacroBody) ||
         (NextNonComment &&
@@ -3784,7 +3791,7 @@ static bool isFunctionDeclarationName(const LangOptions &LangOpts,
         return Next;
       if (Next->is(TT_OverloadedOperator))
         continue;
-      if (Next->isOneOf(tok::kw_new, tok::kw_delete)) {
+      if (Next->isOneOf(tok::kw_new, tok::kw_delete, tok::kw_co_await)) {
         // For 'new[]' and 'delete[]'.
         if (Next->Next &&
             Next->Next->startsSequence(tok::l_square, tok::r_square)) {
@@ -4315,9 +4322,11 @@ unsigned TokenAnnotator::splitPenalty(const AnnotatedLine &Line,
     //
     //   aaaaaaa
     //       .aaaaaaaaa.bbbbbbbb(cccccccc);
-    return !Right.NextOperator || !Right.NextOperator->Previous->closesScope()
-               ? 150
-               : 35;
+    const auto *NextOperator = Right.NextOperator;
+    const auto Penalty = Style.PenaltyBreakBeforeMemberAccess;
+    return NextOperator && NextOperator->Previous->closesScope()
+               ? std::min(Penalty, 35u)
+               : Penalty;
   }
 
   if (Right.is(TT_TrailingAnnotation) &&
@@ -6174,6 +6183,9 @@ bool TokenAnnotator::canBreakBefore(const AnnotatedLine &Line,
     return false;
   }
 
+  if (Right.is(TT_TemplateCloser))
+    return Style.BreakBeforeTemplateCloser;
+
   if (Left.is(tok::at))
     return false;
   if (Left.Tok.getObjCKeywordID() == tok::objc_interface)
@@ -6322,8 +6334,6 @@ bool TokenAnnotator::canBreakBefore(const AnnotatedLine &Line,
   if (Right.is(TT_ImplicitStringLiteral))
     return false;
 
-  if (Right.is(TT_TemplateCloser))
-    return false;
   if (Right.is(tok::r_square) && Right.MatchingParen &&
       Right.MatchingParen->is(TT_LambdaLSquare)) {
     return false;
diff --git a/clang/lib/Format/UnwrappedLineParser.cpp b/clang/lib/Format/UnwrappedLineParser.cpp
index 4258329136348..4e040183f2f0a 100644
--- a/clang/lib/Format/UnwrappedLineParser.cpp
+++ b/clang/lib/Format/UnwrappedLineParser.cpp
@@ -3632,7 +3632,7 @@ void UnwrappedLineParser::parseRequiresClause(FormatToken *RequiresToken) {
   // It could be inlined into here.
   parseConstraintExpression();
 
-  if (!InRequiresExpression)
+  if (!InRequiresExpression && FormatTok->Previous)
     FormatTok->Previous->ClosesRequiresClause = true;
 }
 
@@ -4029,7 +4029,7 @@ void UnwrappedLineParser::parseRecord(bool ParseAsExpr) {
   const FormatToken &InitialToken = *FormatTok;
   nextToken();
 
-  const FormatToken *ClassName = nullptr;
+  FormatToken *ClassName = nullptr;
   bool IsDerived = false;
   auto IsNonMacroIdentifier = [](const FormatToken *Tok) {
     return Tok->is(tok::identifier) && Tok->TokenText != Tok->TokenText.upper();
@@ -4059,7 +4059,7 @@ void UnwrappedLineParser::parseRecord(bool ParseAsExpr) {
     }
     if (FormatTok->is(tok::l_square) && handleCppAttributes())
       continue;
-    const auto *Previous = FormatTok;
+    auto *Previous = FormatTok;
     nextToken();
     switch (FormatTok->Tok.getKind()) {
     case tok::l_paren:
@@ -4074,8 +4074,12 @@ void UnwrappedLineParser::parseRecord(bool ParseAsExpr) {
     case tok::hashhash:
       break;
     default:
-      if (!JSPastExtendsOrImplements && !ClassName &&
-          Previous->is(tok::identifier) && Previous->isNot(TT_AttributeMacro)) {
+      if (JSPastExtendsOrImplements || ClassName ||
+          Previous->isNot(tok::identifier) || Previous->is(TT_AttributeMacro)) {
+        break;
+      }
+      if (const auto Text = Previous->TokenText;
+          Text.size() == 1 || Text != Text.upper()) {
         ClassName = Previous;
       }
     }
@@ -4102,7 +4106,7 @@ void UnwrappedLineParser::parseRecord(bool ParseAsExpr) {
       if (AngleNestingLevel == 0) {
         if (FormatTok->is(tok::colon)) {
           IsDerived = true;
-        } else if (FormatTok->is(tok::identifier) &&
+        } else if (!IsDerived && FormatTok->is(tok::identifier) &&
                    FormatTok->Previous->is(tok::coloncolon)) {
           ClassName = FormatTok;
         } else if (FormatTok->is(tok::l_paren) &&
@@ -4159,6 +4163,8 @@ void UnwrappedLineParser::parseRecord(bool ParseAsExpr) {
   if (FormatTok->is(tok::l_brace)) {
     if (IsListInitialization())
       return;
+    if (ClassName)
+      ClassName->setFinalizedType(TT_ClassHeadName);
     auto [OpenBraceType, ClosingBraceType] = GetBraceTypes(InitialToken);
     FormatTok->setFinalizedType(OpenBraceType);
     if (ParseAsExpr) {
diff --git a/clang/lib/Frontend/CompilerInvocation.cpp b/clang/lib/Frontend/CompilerInvocation.cpp
index 3bf124e4827be..014e629c959e2 100644
--- a/clang/lib/Frontend/CompilerInvocation.cpp
+++ b/clang/lib/Frontend/CompilerInvocation.cpp
@@ -1665,6 +1665,11 @@ void CompilerInvocationBase::GenerateCodeGenArgs(const CodeGenOptions &Opts,
   else if (!Opts.UnrollLoops && Opts.OptimizationLevel > 1)
     GenerateArg(Consumer, OPT_fno_unroll_loops);
 
+  if (Opts.InterchangeLoops)
+    GenerateArg(Consumer, OPT_floop_interchange);
+  else
+    GenerateArg(Consumer, OPT_fno_loop_interchange);
+
   if (!Opts.BinutilsVersion.empty())
     GenerateArg(Consumer, OPT_fbinutils_version_EQ, Opts.BinutilsVersion);
 
@@ -1843,6 +1848,9 @@ void CompilerInvocationBase::GenerateCodeGenArgs(const CodeGenOptions &Opts,
     GenerateArg(Consumer, OPT_fno_finite_loops);
     break;
   }
+
+  if (Opts.StaticClosure)
+    GenerateArg(Consumer, OPT_static_libclosure);
 }
 
 bool CompilerInvocation::ParseCodeGenArgs(CodeGenOptions &Opts, ArgList &Args,
@@ -1968,6 +1976,8 @@ bool CompilerInvocation::ParseCodeGenArgs(CodeGenOptions &Opts, ArgList &Args,
   Opts.UnrollLoops =
       Args.hasFlag(OPT_funroll_loops, OPT_fno_unroll_loops,
                    (Opts.OptimizationLevel > 1));
+  Opts.InterchangeLoops =
+      Args.hasFlag(OPT_floop_interchange, OPT_fno_loop_interchange, false);
   Opts.BinutilsVersion =
       std::string(Args.getLastArgValue(OPT_fbinutils_version_EQ));
 
@@ -2330,6 +2340,8 @@ bool CompilerInvocation::ParseCodeGenArgs(CodeGenOptions &Opts, ArgList &Args,
   if (!Opts.EmitIEEENaNCompliantInsts && !LangOptsRef.NoHonorNaNs)
     Diags.Report(diag::err_drv_amdgpu_ieee_without_no_honor_nans);
 
+  Opts.StaticClosure = Args.hasArg(options::OPT_static_libclosure);
+
   return Diags.getNumErrors() == NumErrorsBefore;
 }
 
@@ -2756,7 +2768,6 @@ static const auto &getFrontendActionTable() {
       {frontend::RewriteObjC, OPT_rewrite_objc},
       {frontend::RewriteTest, OPT_rewrite_test},
       {frontend::RunAnalysis, OPT_analyze},
-      {frontend::MigrateSource, OPT_migrate},
       {frontend::RunPreprocessorOnly, OPT_Eonly},
       {frontend::PrintDependencyDirectivesSourceMinimizerOutput,
        OPT_print_dependency_directives_minimized_source},
@@ -3099,12 +3110,6 @@ static bool ParseFrontendArgs(FrontendOptions &Opts, ArgList &Args,
   if (Args.hasArg(OPT_aux_target_feature))
     Opts.AuxTargetFeatures = Args.getAllArgValues(OPT_aux_target_feature);
 
-  if (Opts.ARCMTAction != FrontendOptions::ARCMT_None &&
-      Opts.ObjCMTAction != FrontendOptions::ObjCMT_None) {
-    Diags.Report(diag::err_drv_argument_not_allowed_with)
-      << "ARC migration" << "ObjC migration";
-  }
-
   InputKind DashX(Language::Unknown);
   if (const Arg *A = Args.getLastArg(OPT_x)) {
     StringRef XValue = A->getValue();
@@ -3721,6 +3726,8 @@ void CompilerInvocationBase::GenerateLangArgs(const LangOptions &Opts,
   } else if (Opts.SignedOverflowBehavior == LangOptions::SOB_Defined) {
     GenerateArg(Consumer, OPT_fwrapv);
   }
+  if (Opts.PointerOverflowDefined)
+    GenerateArg(Consumer, OPT_fwrapv_pointer);
 
   if (Opts.MSCompatibilityVersion != 0) {
     unsigned Major = Opts.MSCompatibilityVersion / 10000000;
@@ -3893,6 +3900,9 @@ void CompilerInvocationBase::GenerateLangArgs(const LangOptions &Opts,
   case LangOptions::ClangABI::Ver19:
     GenerateArg(Consumer, OPT_fclang_abi_compat_EQ, "19.0");
     break;
+  case LangOptions::ClangABI::Ver20:
+    GenerateArg(Consumer, OPT_fclang_abi_compat_EQ, "20.0");
+    break;
   case LangOptions::ClangABI::Latest:
     break;
   }
@@ -4138,6 +4148,8 @@ bool CompilerInvocation::ParseLangArgs(LangOptions &Opts, ArgList &Args,
   }
   else if (Args.hasArg(OPT_fwrapv))
     Opts.setSignedOverflowBehavior(LangOptions::SOB_Defined);
+  if (Args.hasArg(OPT_fwrapv_pointer))
+    Opts.PointerOverflowDefined = true;
 
   Opts.MSCompatibilityVersion = 0;
   if (const Arg *A = Args.getLastArg(OPT_fms_compatibility_version)) {
@@ -4211,7 +4223,7 @@ bool CompilerInvocation::ParseLangArgs(LangOptions &Opts, ArgList &Args,
     }
   }
 
-  // Check if -fopenmp is specified and set default version to 5.0.
+  // Check if -fopenmp is specified and set default version to 5.1.
   Opts.OpenMP = Args.hasArg(OPT_fopenmp) ? 51 : 0;
   // Check if -fopenmp-simd is specified.
   bool IsSimdSpecified =
@@ -4409,7 +4421,7 @@ bool CompilerInvocation::ParseLangArgs(LangOptions &Opts, ArgList &Args,
     // y or y.0 (4 <= y <= current version).
     if (!VerParts.first.starts_with("0") &&
         !VerParts.first.getAsInteger(10, Major) && 3 <= Major &&
-        Major <= CLANG_VERSION_MAJOR &&
+        Major <= MAX_CLANG_ABI_COMPAT_VERSION &&
         (Major == 3
              ? VerParts.second.size() == 1 &&
                    !VerParts.second.getAsInteger(10, Minor)
@@ -4635,7 +4647,6 @@ static bool isStrictlyPreprocessorAction(frontend::ActionKind Action) {
   case frontend::RewriteTest:
   case frontend::RunAnalysis:
   case frontend::TemplightDump:
-  case frontend::MigrateSource:
     return false;
 
   case frontend::DumpCompilerOptions:
diff --git a/clang/lib/Frontend/InitPreprocessor.cpp b/clang/lib/Frontend/InitPreprocessor.cpp
index 29723b573e771..77833f5d1defb 100644
--- a/clang/lib/Frontend/InitPreprocessor.cpp
+++ b/clang/lib/Frontend/InitPreprocessor.cpp
@@ -253,6 +253,8 @@ static void DefineExactWidthIntType(const LangOptions &LangOpts,
 
   StringRef ConstSuffix(TI.getTypeConstantSuffix(Ty));
   Builder.defineMacro(Prefix + Twine(TypeWidth) + "_C_SUFFIX__", ConstSuffix);
+  Builder.defineMacro(Prefix + Twine(TypeWidth) + "_C(c)",
+                      ConstSuffix.size() ? Twine("c##") + ConstSuffix : "c");
 }
 
 static void DefineExactWidthIntTypeSize(TargetInfo::IntType Ty,
@@ -727,8 +729,8 @@ static void InitializeCPlusPlusFeatureTestMacros(const LangOptions &LangOpts,
   }
   if (LangOpts.AlignedAllocation && !LangOpts.AlignedAllocationUnavailable)
     Builder.defineMacro("__cpp_aligned_new", "201606L");
-  if (LangOpts.RelaxedTemplateTemplateArgs)
-    Builder.defineMacro("__cpp_template_template_args", "201611L");
+
+  Builder.defineMacro("__cpp_template_template_args", "201611L");
 
   // C++20 features.
   if (LangOpts.CPlusPlus20) {
@@ -1164,12 +1166,16 @@ static void InitializePredefinedMacros(const TargetInfo &TI,
 
   DefineType("__INTMAX_TYPE__", TI.getIntMaxType(), Builder);
   DefineFmt(LangOpts, "__INTMAX", TI.getIntMaxType(), TI, Builder);
-  Builder.defineMacro("__INTMAX_C_SUFFIX__",
-                      TI.getTypeConstantSuffix(TI.getIntMaxType()));
+  StringRef ConstSuffix(TI.getTypeConstantSuffix(TI.getIntMaxType()));
+  Builder.defineMacro("__INTMAX_C_SUFFIX__", ConstSuffix);
+  Builder.defineMacro("__INTMAX_C(c)",
+                      ConstSuffix.size() ? Twine("c##") + ConstSuffix : "c");
   DefineType("__UINTMAX_TYPE__", TI.getUIntMaxType(), Builder);
   DefineFmt(LangOpts, "__UINTMAX", TI.getUIntMaxType(), TI, Builder);
-  Builder.defineMacro("__UINTMAX_C_SUFFIX__",
-                      TI.getTypeConstantSuffix(TI.getUIntMaxType()));
+  ConstSuffix = TI.getTypeConstantSuffix(TI.getUIntMaxType());
+  Builder.defineMacro("__UINTMAX_C_SUFFIX__", ConstSuffix);
+  Builder.defineMacro("__UINTMAX_C(c)",
+                      ConstSuffix.size() ? Twine("c##") + ConstSuffix : "c");
   DefineType("__PTRDIFF_TYPE__", TI.getPtrDiffType(LangAS::Default), Builder);
   DefineFmt(LangOpts, "__PTRDIFF", TI.getPtrDiffType(LangAS::Default), TI,
             Builder);
@@ -1460,9 +1466,15 @@ static void InitializePredefinedMacros(const TargetInfo &TI,
     case 50:
       Builder.defineMacro("_OPENMP", "201811");
       break;
+    case 51:
+      Builder.defineMacro("_OPENMP", "202011");
+      break;
     case 52:
       Builder.defineMacro("_OPENMP", "202111");
       break;
+    case 60:
+      Builder.defineMacro("_OPENMP", "202411");
+      break;
     default: // case 51:
       // Default version is OpenMP 5.1
       Builder.defineMacro("_OPENMP", "202011");
diff --git a/clang/lib/Frontend/LogDiagnosticPrinter.cpp b/clang/lib/Frontend/LogDiagnosticPrinter.cpp
index 469d1c22633aa..4e963af837f01 100644
--- a/clang/lib/Frontend/LogDiagnosticPrinter.cpp
+++ b/clang/lib/Frontend/LogDiagnosticPrinter.cpp
@@ -129,7 +129,8 @@ void LogDiagnosticPrinter::HandleDiagnostic(DiagnosticsEngine::Level Level,
   DE.DiagnosticLevel = Level;
 
   DE.WarningOption =
-      std::string(DiagnosticIDs::getWarningOptionForDiag(DE.DiagnosticID));
+      std::string(Info.getDiags()->getDiagnosticIDs()->getWarningOptionForDiag(
+          DE.DiagnosticID));
 
   // Format the message.
   SmallString<100> MessageStr;
@@ -160,4 +161,3 @@ void LogDiagnosticPrinter::HandleDiagnostic(DiagnosticsEngine::Level Level,
   // Record the diagnostic entry.
   Entries.push_back(DE);
 }
-
diff --git a/clang/lib/Frontend/SerializedDiagnosticPrinter.cpp b/clang/lib/Frontend/SerializedDiagnosticPrinter.cpp
index 0887b5a504f05..02aa3e8e4d984 100644
--- a/clang/lib/Frontend/SerializedDiagnosticPrinter.cpp
+++ b/clang/lib/Frontend/SerializedDiagnosticPrinter.cpp
@@ -202,7 +202,7 @@ class SDiagsWriter : public DiagnosticConsumer {
 
   /// Emit the string information for diagnostic flags.
   unsigned getEmitDiagnosticFlag(DiagnosticsEngine::Level DiagLevel,
-                                 unsigned DiagID = 0);
+                                 const Diagnostic *Diag = nullptr);
 
   unsigned getEmitDiagnosticFlag(StringRef DiagName);
 
@@ -536,11 +536,13 @@ unsigned SDiagsWriter::getEmitCategory(unsigned int category) {
 }
 
 unsigned SDiagsWriter::getEmitDiagnosticFlag(DiagnosticsEngine::Level DiagLevel,
-                                             unsigned DiagID) {
-  if (DiagLevel == DiagnosticsEngine::Note)
+                                             const Diagnostic *Diag) {
+  if (!Diag || DiagLevel == DiagnosticsEngine::Note)
     return 0; // No flag for notes.
 
-  StringRef FlagName = DiagnosticIDs::getWarningOptionForDiag(DiagID);
+  StringRef FlagName =
+      Diag->getDiags()->getDiagnosticIDs()->getWarningOptionForDiag(
+          Diag->getID());
   return getEmitDiagnosticFlag(FlagName);
 }
 
@@ -650,12 +652,12 @@ void SDiagsWriter::EmitDiagnosticMessage(FullSourceLoc Loc, PresumedLoc PLoc,
   Record.push_back(getStableLevel(Level));
   AddLocToRecord(Loc, PLoc, Record);
 
-  if (const Diagnostic *Info = D.dyn_cast<const Diagnostic*>()) {
+  if (const Diagnostic *Info = dyn_cast_if_present<const Diagnostic *>(D)) {
     // Emit the category string lazily and get the category ID.
     unsigned DiagID = DiagnosticIDs::getCategoryNumberForDiag(Info->getID());
     Record.push_back(getEmitCategory(DiagID));
     // Emit the diagnostic flag string lazily and get the mapped ID.
-    Record.push_back(getEmitDiagnosticFlag(Level, Info->getID()));
+    Record.push_back(getEmitDiagnosticFlag(Level, Info));
   } else {
     Record.push_back(getEmitCategory());
     Record.push_back(getEmitDiagnosticFlag(Level));
diff --git a/clang/lib/Frontend/TextDiagnosticPrinter.cpp b/clang/lib/Frontend/TextDiagnosticPrinter.cpp
index dac5c44fe9256..28f7218dc23f5 100644
--- a/clang/lib/Frontend/TextDiagnosticPrinter.cpp
+++ b/clang/lib/Frontend/TextDiagnosticPrinter.cpp
@@ -70,13 +70,17 @@ static void printDiagnosticOptions(raw_ostream &OS,
     // flag it as such. Note that diagnostics could also have been mapped by a
     // pragma, but we don't currently have a way to distinguish this.
     if (Level == DiagnosticsEngine::Error &&
-        DiagnosticIDs::isBuiltinWarningOrExtension(Info.getID()) &&
-        !DiagnosticIDs::isDefaultMappingAsError(Info.getID())) {
+        Info.getDiags()->getDiagnosticIDs()->isWarningOrExtension(
+            Info.getID()) &&
+        !Info.getDiags()->getDiagnosticIDs()->isDefaultMappingAsError(
+            Info.getID())) {
       OS << " [-Werror";
       Started = true;
     }
 
-    StringRef Opt = DiagnosticIDs::getWarningOptionForDiag(Info.getID());
+    StringRef Opt =
+        Info.getDiags()->getDiagnosticIDs()->getWarningOptionForDiag(
+            Info.getID());
     if (!Opt.empty()) {
       OS << (Started ? "," : " [")
          << (Level == DiagnosticsEngine::Remark ? "-R" : "-W") << Opt;
diff --git a/clang/lib/FrontendTool/CMakeLists.txt b/clang/lib/FrontendTool/CMakeLists.txt
index bfc7652b4c118..d7a3699361f0a 100644
--- a/clang/lib/FrontendTool/CMakeLists.txt
+++ b/clang/lib/FrontendTool/CMakeLists.txt
@@ -21,12 +21,6 @@ if(CLANG_ENABLE_CIR)
     )
 endif()
 
-if(CLANG_ENABLE_ARCMT)
-  list(APPEND link_libs
-    clangARCMigrate
-    )
-endif()
-
 if(CLANG_ENABLE_STATIC_ANALYZER)
   list(APPEND link_libs
     clangStaticAnalyzerFrontend
diff --git a/clang/lib/FrontendTool/ExecuteCompilerInvocation.cpp b/clang/lib/FrontendTool/ExecuteCompilerInvocation.cpp
index 3f95a1efb2eed..c8d004163b96d 100644
--- a/clang/lib/FrontendTool/ExecuteCompilerInvocation.cpp
+++ b/clang/lib/FrontendTool/ExecuteCompilerInvocation.cpp
@@ -11,7 +11,6 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "clang/ARCMigrate/ARCMTActions.h"
 #include "clang/CodeGen/CodeGenAction.h"
 #include "clang/Config/config.h"
 #include "clang/Driver/Options.h"
@@ -72,7 +71,13 @@ CreateFrontendBaseAction(CompilerInstance &CI) {
     llvm_unreachable("CIR suppport not built into clang");
 #endif
   case EmitHTML:               return std::make_unique<HTMLPrintAction>();
-  case EmitLLVM:               return std::make_unique<EmitLLVMAction>();
+  case EmitLLVM: {
+#if CLANG_ENABLE_CIR
+    if (UseCIR)
+      return std::make_unique<cir::EmitLLVMAction>();
+#endif
+    return std::make_unique<EmitLLVMAction>();
+  }
   case EmitLLVMOnly:           return std::make_unique<EmitLLVMOnlyAction>();
   case EmitCodeGenOnly:        return std::make_unique<EmitCodeGenOnlyAction>();
   case EmitObj:                return std::make_unique<EmitObjAction>();
@@ -131,12 +136,6 @@ CreateFrontendBaseAction(CompilerInstance &CI) {
 #else
   case RewriteObjC:            Action = "RewriteObjC"; break;
 #endif
-#if CLANG_ENABLE_ARCMT
-  case MigrateSource:
-    return std::make_unique<arcmt::MigrateSourceAction>();
-#else
-  case MigrateSource:          Action = "MigrateSource"; break;
-#endif
 #if CLANG_ENABLE_STATIC_ANALYZER
   case RunAnalysis:            return std::make_unique<ento::AnalysisAction>();
 #else
@@ -147,8 +146,7 @@ CreateFrontendBaseAction(CompilerInstance &CI) {
     return std::make_unique<PrintDependencyDirectivesSourceMinimizerAction>();
   }
 
-#if !CLANG_ENABLE_ARCMT || !CLANG_ENABLE_STATIC_ANALYZER \
-  || !CLANG_ENABLE_OBJC_REWRITER
+#if !CLANG_ENABLE_STATIC_ANALYZER || !CLANG_ENABLE_OBJC_REWRITER
   CI.getDiagnostics().Report(diag::err_fe_action_not_available) << Action;
   return 0;
 #else
@@ -169,35 +167,6 @@ CreateFrontendAction(CompilerInstance &CI) {
     Act = std::make_unique<FixItRecompile>(std::move(Act));
   }
 
-#if CLANG_ENABLE_ARCMT
-  if (CI.getFrontendOpts().ProgramAction != frontend::MigrateSource &&
-      CI.getFrontendOpts().ProgramAction != frontend::GeneratePCH) {
-    // Potentially wrap the base FE action in an ARC Migrate Tool action.
-    switch (FEOpts.ARCMTAction) {
-    case FrontendOptions::ARCMT_None:
-      break;
-    case FrontendOptions::ARCMT_Check:
-      Act = std::make_unique<arcmt::CheckAction>(std::move(Act));
-      break;
-    case FrontendOptions::ARCMT_Modify:
-      Act = std::make_unique<arcmt::ModifyAction>(std::move(Act));
-      break;
-    case FrontendOptions::ARCMT_Migrate:
-      Act = std::make_unique<arcmt::MigrateAction>(std::move(Act),
-                                     FEOpts.MTMigrateDir,
-                                     FEOpts.ARCMTMigrateReportOut,
-                                     FEOpts.ARCMTMigrateEmitARCErrors);
-      break;
-    }
-
-    if (FEOpts.ObjCMTAction != FrontendOptions::ObjCMT_None) {
-      Act = std::make_unique<arcmt::ObjCMigrateAction>(std::move(Act),
-                                                        FEOpts.MTMigrateDir,
-                                                        FEOpts.ObjCMTAction);
-    }
-  }
-#endif
-
   // Wrap the base FE action in an extract api action to generate
   // symbol graph as a biproduct of compilation (enabled with
   // --emit-symbol-graph option)
diff --git a/clang/lib/Headers/amdgpuintrin.h b/clang/lib/Headers/amdgpuintrin.h
index 720674a85f52c..9dad99ffe9439 100644
--- a/clang/lib/Headers/amdgpuintrin.h
+++ b/clang/lib/Headers/amdgpuintrin.h
@@ -145,29 +145,33 @@ _DEFAULT_FN_ATTRS static __inline__ void __gpu_sync_lane(uint64_t __lane_mask) {
 
 // Shuffles the the lanes inside the wavefront according to the given index.
 _DEFAULT_FN_ATTRS static __inline__ uint32_t
-__gpu_shuffle_idx_u32(uint64_t __lane_mask, uint32_t __idx, uint32_t __x) {
-  return __builtin_amdgcn_ds_bpermute(__idx << 2, __x);
+__gpu_shuffle_idx_u32(uint64_t __lane_mask, uint32_t __idx, uint32_t __x,
+                      uint32_t __width) {
+  uint32_t __lane = __idx + (__gpu_lane_id() & ~(__width - 1));
+  return __builtin_amdgcn_ds_bpermute(__lane << 2, __x);
 }
 
 // Shuffles the the lanes inside the wavefront according to the given index.
 _DEFAULT_FN_ATTRS static __inline__ uint64_t
-__gpu_shuffle_idx_u64(uint64_t __lane_mask, uint32_t __idx, uint64_t __x) {
+__gpu_shuffle_idx_u64(uint64_t __lane_mask, uint32_t __idx, uint64_t __x,
+                      uint32_t __width) {
   uint32_t __hi = (uint32_t)(__x >> 32ull);
   uint32_t __lo = (uint32_t)(__x & 0xFFFFFFFF);
-  return ((uint64_t)__builtin_amdgcn_ds_bpermute(__idx << 2, __hi) << 32ull) |
-         ((uint64_t)__builtin_amdgcn_ds_bpermute(__idx << 2, __lo));
+  return ((uint64_t)__gpu_shuffle_idx_u32(__lane_mask, __idx, __hi, __width)
+          << 32ull) |
+         ((uint64_t)__gpu_shuffle_idx_u32(__lane_mask, __idx, __lo, __width));
 }
 
-// Returns true if the flat pointer points to CUDA 'shared' memory.
+// Returns true if the flat pointer points to AMDGPU 'shared' memory.
 _DEFAULT_FN_ATTRS static __inline__ bool __gpu_is_ptr_local(void *ptr) {
-  return __builtin_amdgcn_is_shared((void __attribute__((address_space(0))) *)((
+  return __builtin_amdgcn_is_shared((void [[clang::address_space(0)]] *)((
       void [[clang::opencl_generic]] *)ptr));
 }
 
-// Returns true if the flat pointer points to CUDA 'local' memory.
+// Returns true if the flat pointer points to AMDGPU 'private' memory.
 _DEFAULT_FN_ATTRS static __inline__ bool __gpu_is_ptr_private(void *ptr) {
-  return __builtin_amdgcn_is_private((void __attribute__((
-      address_space(0))) *)((void [[clang::opencl_generic]] *)ptr));
+  return __builtin_amdgcn_is_private((void [[clang::address_space(0)]] *)((
+      void [[clang::opencl_generic]] *)ptr));
 }
 
 // Terminates execution of the associated wavefront.
diff --git a/clang/lib/Headers/amxtf32transposeintrin.h b/clang/lib/Headers/amxtf32transposeintrin.h
index 60336f953ecb7..e1b90c1adfb22 100644
--- a/clang/lib/Headers/amxtf32transposeintrin.h
+++ b/clang/lib/Headers/amxtf32transposeintrin.h
@@ -8,7 +8,7 @@
  */
 #ifndef __IMMINTRIN_H
 #error                                                                         \
-    "Never use <amxtf32tranposeintrin.h> directly; include <immintrin.h> instead."
+    "Never use <amxtf32transposeintrin.h> directly; include <immintrin.h> instead."
 #endif // __IMMINTRIN_H
 
 #ifndef __AMX_TF32TRANSPOSEINTRIN_H
diff --git a/clang/lib/Headers/avx10_2_512convertintrin.h b/clang/lib/Headers/avx10_2_512convertintrin.h
index 0b5fca5cda522..516ccc68672d6 100644
--- a/clang/lib/Headers/avx10_2_512convertintrin.h
+++ b/clang/lib/Headers/avx10_2_512convertintrin.h
@@ -213,19 +213,19 @@ _mm512_maskz_cvts2ph_hf8(__mmask64 __U, __m512h __A, __m512h __B) {
       (__v64qi)(__m512i)_mm512_setzero_si512());
 }
 
-static __inline__ __m512h __DEFAULT_FN_ATTRS512 _mm512_cvthf8(__m256i __A) {
+static __inline__ __m512h __DEFAULT_FN_ATTRS512 _mm512_cvthf8_ph(__m256i __A) {
   return (__m512h)__builtin_ia32_vcvthf8_2ph512_mask(
       (__v32qi)__A, (__v32hf)(__m512h)_mm512_undefined_ph(), (__mmask32)-1);
 }
 
 static __inline__ __m512h __DEFAULT_FN_ATTRS512
-_mm512_mask_cvthf8(__m512h __W, __mmask32 __U, __m256i __A) {
+_mm512_mask_cvthf8_ph(__m512h __W, __mmask32 __U, __m256i __A) {
   return (__m512h)__builtin_ia32_vcvthf8_2ph512_mask(
       (__v32qi)__A, (__v32hf)(__m512h)__W, (__mmask32)__U);
 }
 
 static __inline__ __m512h __DEFAULT_FN_ATTRS512
-_mm512_maskz_cvthf8(__mmask32 __U, __m256i __A) {
+_mm512_maskz_cvthf8_ph(__mmask32 __U, __m256i __A) {
   return (__m512h)__builtin_ia32_vcvthf8_2ph512_mask(
       (__v32qi)__A, (__v32hf)(__m512h)_mm512_setzero_ph(), (__mmask32)__U);
 }
diff --git a/clang/lib/Headers/avx10_2convertintrin.h b/clang/lib/Headers/avx10_2convertintrin.h
index c67a5b890f195..07722090c30ee 100644
--- a/clang/lib/Headers/avx10_2convertintrin.h
+++ b/clang/lib/Headers/avx10_2convertintrin.h
@@ -260,13 +260,13 @@ static __inline__ __m256i __DEFAULT_FN_ATTRS256 _mm256_cvt2ph_bf8(__m256h __A,
 static __inline__ __m256i __DEFAULT_FN_ATTRS256
 _mm256_mask_cvt2ph_bf8(__m256i __W, __mmask32 __U, __m256h __A, __m256h __B) {
   return (__m256i)__builtin_ia32_selectb_256(
-      (__mmask16)__U, (__v32qi)_mm256_cvt2ph_bf8(__A, __B), (__v32qi)__W);
+      (__mmask32)__U, (__v32qi)_mm256_cvt2ph_bf8(__A, __B), (__v32qi)__W);
 }
 
 static __inline__ __m256i __DEFAULT_FN_ATTRS256
 _mm256_maskz_cvt2ph_bf8(__mmask32 __U, __m256h __A, __m256h __B) {
   return (__m256i)__builtin_ia32_selectb_256(
-      (__mmask16)__U, (__v32qi)_mm256_cvt2ph_bf8(__A, __B),
+      (__mmask32)__U, (__v32qi)_mm256_cvt2ph_bf8(__A, __B),
       (__v32qi)(__m256i)_mm256_setzero_si256());
 }
 
@@ -297,13 +297,13 @@ _mm256_cvts2ph_bf8(__m256h __A, __m256h __B) {
 static __inline__ __m256i __DEFAULT_FN_ATTRS256
 _mm256_mask_cvts2ph_bf8(__m256i __W, __mmask32 __U, __m256h __A, __m256h __B) {
   return (__m256i)__builtin_ia32_selectb_256(
-      (__mmask16)__U, (__v32qi)_mm256_cvts2ph_bf8(__A, __B), (__v32qi)__W);
+      (__mmask32)__U, (__v32qi)_mm256_cvts2ph_bf8(__A, __B), (__v32qi)__W);
 }
 
 static __inline__ __m256i __DEFAULT_FN_ATTRS256
 _mm256_maskz_cvts2ph_bf8(__mmask32 __U, __m256h __A, __m256h __B) {
   return (__m256i)__builtin_ia32_selectb_256(
-      (__mmask16)__U, (__v32qi)_mm256_cvts2ph_bf8(__A, __B),
+      (__mmask32)__U, (__v32qi)_mm256_cvts2ph_bf8(__A, __B),
       (__v32qi)(__m256i)_mm256_setzero_si256());
 }
 
@@ -334,13 +334,13 @@ static __inline__ __m256i __DEFAULT_FN_ATTRS256 _mm256_cvt2ph_hf8(__m256h __A,
 static __inline__ __m256i __DEFAULT_FN_ATTRS256
 _mm256_mask_cvt2ph_hf8(__m256i __W, __mmask32 __U, __m256h __A, __m256h __B) {
   return (__m256i)__builtin_ia32_selectb_256(
-      (__mmask16)__U, (__v32qi)_mm256_cvt2ph_hf8(__A, __B), (__v32qi)__W);
+      (__mmask32)__U, (__v32qi)_mm256_cvt2ph_hf8(__A, __B), (__v32qi)__W);
 }
 
 static __inline__ __m256i __DEFAULT_FN_ATTRS256
 _mm256_maskz_cvt2ph_hf8(__mmask32 __U, __m256h __A, __m256h __B) {
   return (__m256i)__builtin_ia32_selectb_256(
-      (__mmask16)__U, (__v32qi)_mm256_cvt2ph_hf8(__A, __B),
+      (__mmask32)__U, (__v32qi)_mm256_cvt2ph_hf8(__A, __B),
       (__v32qi)(__m256i)_mm256_setzero_si256());
 }
 
@@ -371,47 +371,46 @@ _mm256_cvts2ph_hf8(__m256h __A, __m256h __B) {
 static __inline__ __m256i __DEFAULT_FN_ATTRS256
 _mm256_mask_cvts2ph_hf8(__m256i __W, __mmask32 __U, __m256h __A, __m256h __B) {
   return (__m256i)__builtin_ia32_selectb_256(
-      (__mmask16)__U, (__v32qi)_mm256_cvts2ph_hf8(__A, __B), (__v32qi)__W);
+      (__mmask32)__U, (__v32qi)_mm256_cvts2ph_hf8(__A, __B), (__v32qi)__W);
 }
 
 static __inline__ __m256i __DEFAULT_FN_ATTRS256
 _mm256_maskz_cvts2ph_hf8(__mmask32 __U, __m256h __A, __m256h __B) {
   return (__m256i)__builtin_ia32_selectb_256(
-      (__mmask16)__U, (__v32qi)_mm256_cvts2ph_hf8(__A, __B),
+      (__mmask32)__U, (__v32qi)_mm256_cvts2ph_hf8(__A, __B),
       (__v32qi)(__m256i)_mm256_setzero_si256());
 }
 
-static __inline__ __m128h __DEFAULT_FN_ATTRS128 _mm_cvthf8(__m128i __A) {
+static __inline__ __m128h __DEFAULT_FN_ATTRS128 _mm_cvthf8_ph(__m128i __A) {
   return (__m128h)__builtin_ia32_vcvthf8_2ph128_mask(
       (__v16qi)__A, (__v8hf)(__m128h)_mm_undefined_ph(), (__mmask8)-1);
 }
 
-static __inline__ __m128h __DEFAULT_FN_ATTRS128 _mm_mask_cvthf8(__m128h __W,
-                                                                __mmask8 __U,
-                                                                __m128i __A) {
+static __inline__ __m128h __DEFAULT_FN_ATTRS128
+_mm_mask_cvthf8_ph(__m128h __W, __mmask8 __U, __m128i __A) {
   return (__m128h)__builtin_ia32_vcvthf8_2ph128_mask(
       (__v16qi)__A, (__v8hf)(__m128h)__W, (__mmask8)__U);
 }
 
-static __inline__ __m128h __DEFAULT_FN_ATTRS128 _mm_maskz_cvthf8(__mmask8 __U,
-                                                                 __m128i __A) {
+static __inline__ __m128h __DEFAULT_FN_ATTRS128
+_mm_maskz_cvthf8_ph(__mmask8 __U, __m128i __A) {
   return (__m128h)__builtin_ia32_vcvthf8_2ph128_mask(
       (__v16qi)__A, (__v8hf)(__m128h)_mm_setzero_ph(), (__mmask8)__U);
 }
 
-static __inline__ __m256h __DEFAULT_FN_ATTRS256 _mm256_cvthf8(__m128i __A) {
+static __inline__ __m256h __DEFAULT_FN_ATTRS256 _mm256_cvthf8_ph(__m128i __A) {
   return (__m256h)__builtin_ia32_vcvthf8_2ph256_mask(
       (__v16qi)__A, (__v16hf)(__m256h)_mm256_undefined_ph(), (__mmask16)-1);
 }
 
 static __inline__ __m256h __DEFAULT_FN_ATTRS256
-_mm256_mask_cvthf8(__m256h __W, __mmask16 __U, __m128i __A) {
+_mm256_mask_cvthf8_ph(__m256h __W, __mmask16 __U, __m128i __A) {
   return (__m256h)__builtin_ia32_vcvthf8_2ph256_mask(
       (__v16qi)__A, (__v16hf)(__m256h)__W, (__mmask16)__U);
 }
 
 static __inline__ __m256h __DEFAULT_FN_ATTRS256
-_mm256_maskz_cvthf8(__mmask16 __U, __m128i __A) {
+_mm256_maskz_cvthf8_ph(__mmask16 __U, __m128i __A) {
   return (__m256h)__builtin_ia32_vcvthf8_2ph256_mask(
       (__v16qi)__A, (__v16hf)(__m256h)_mm256_setzero_ph(), (__mmask16)__U);
 }
diff --git a/clang/lib/Headers/gpuintrin.h b/clang/lib/Headers/gpuintrin.h
index 4c463c333308f..11c87e85cd497 100644
--- a/clang/lib/Headers/gpuintrin.h
+++ b/clang/lib/Headers/gpuintrin.h
@@ -133,18 +133,21 @@ __gpu_read_first_lane_f64(uint64_t __lane_mask, double __x) {
 
 // Shuffles the the lanes according to the given index.
 _DEFAULT_FN_ATTRS static __inline__ float
-__gpu_shuffle_idx_f32(uint64_t __lane_mask, uint32_t __idx, float __x) {
+__gpu_shuffle_idx_f32(uint64_t __lane_mask, uint32_t __idx, float __x,
+                      uint32_t __width) {
   return __builtin_bit_cast(
       float, __gpu_shuffle_idx_u32(__lane_mask, __idx,
-                                   __builtin_bit_cast(uint32_t, __x)));
+                                   __builtin_bit_cast(uint32_t, __x), __width));
 }
 
 // Shuffles the the lanes according to the given index.
 _DEFAULT_FN_ATTRS static __inline__ double
-__gpu_shuffle_idx_f64(uint64_t __lane_mask, uint32_t __idx, double __x) {
+__gpu_shuffle_idx_f64(uint64_t __lane_mask, uint32_t __idx, double __x,
+                      uint32_t __width) {
   return __builtin_bit_cast(
-      double, __gpu_shuffle_idx_u64(__lane_mask, __idx,
-                                    __builtin_bit_cast(uint64_t, __x)));
+      double,
+      __gpu_shuffle_idx_u64(__lane_mask, __idx,
+                            __builtin_bit_cast(uint64_t, __x), __width));
 }
 
 // Gets the sum of all lanes inside the warp or wavefront.
@@ -153,7 +156,8 @@ __gpu_shuffle_idx_f64(uint64_t __lane_mask, uint32_t __idx, double __x) {
       uint64_t __lane_mask, __type __x) {                                      \
     for (uint32_t __step = __gpu_num_lanes() / 2; __step > 0; __step /= 2) {   \
       uint32_t __index = __step + __gpu_lane_id();                             \
-      __x += __gpu_shuffle_idx_##__suffix(__lane_mask, __index, __x);          \
+      __x += __gpu_shuffle_idx_##__suffix(__lane_mask, __index, __x,           \
+                                          __gpu_num_lanes());                  \
     }                                                                          \
     return __gpu_read_first_lane_##__suffix(__lane_mask, __x);                 \
   }
@@ -171,10 +175,10 @@ __DO_LANE_SUM(double, f64);   // double __gpu_lane_sum_f64(m, x)
       uint32_t __index = __gpu_lane_id() - __step;                             \
       __bitmask_type bitmask = __gpu_lane_id() >= __step;                      \
       __x += __builtin_bit_cast(                                               \
-          __type,                                                              \
-          -bitmask & __builtin_bit_cast(__bitmask_type,                        \
-                                        __gpu_shuffle_idx_##__suffix(          \
-                                            __lane_mask, __index, __x)));      \
+          __type, -bitmask & __builtin_bit_cast(__bitmask_type,                \
+                                                __gpu_shuffle_idx_##__suffix(  \
+                                                    __lane_mask, __index, __x, \
+                                                    __gpu_num_lanes())));      \
     }                                                                          \
     return __x;                                                                \
   }
diff --git a/clang/lib/Headers/hlsl/hlsl_intrinsics.h b/clang/lib/Headers/hlsl/hlsl_intrinsics.h
index d1e4eb08aa764..513639ed1b81d 100644
--- a/clang/lib/Headers/hlsl/hlsl_intrinsics.h
+++ b/clang/lib/Headers/hlsl/hlsl_intrinsics.h
@@ -138,6 +138,27 @@ _HLSL_BUILTIN_ALIAS(__builtin_elementwise_acos)
 float4 acos(float4);
 
 //===----------------------------------------------------------------------===//
+// AddUint64 builtins
+//===----------------------------------------------------------------------===//
+
+/// \fn T AddUint64(T a, T b)
+/// \brief Implements unsigned 64-bit integer addition using pairs of unsigned
+/// 32-bit integers.
+/// \param x [in] The first unsigned 32-bit integer pair(s)
+/// \param y [in] The second unsigned 32-bit integer pair(s)
+///
+/// This function takes one or two pairs (low, high) of unsigned 32-bit integer
+/// values and returns pairs (low, high) of unsigned 32-bit integer
+/// values representing the result of unsigned 64-bit integer addition.
+
+_HLSL_AVAILABILITY(shadermodel, 6.0)
+_HLSL_BUILTIN_ALIAS(__builtin_hlsl_adduint64)
+uint32_t2 AddUint64(uint32_t2, uint32_t2);
+_HLSL_AVAILABILITY(shadermodel, 6.0)
+_HLSL_BUILTIN_ALIAS(__builtin_hlsl_adduint64)
+uint32_t4 AddUint64(uint32_t4, uint32_t4);
+
+// //===----------------------------------------------------------------------===//
 // all builtins
 //===----------------------------------------------------------------------===//
 
@@ -2468,6 +2489,105 @@ __attribute__((convergent)) double3 WaveReadLaneAt(double3, int32_t);
 _HLSL_BUILTIN_ALIAS(__builtin_hlsl_wave_read_lane_at)
 __attribute__((convergent)) double4 WaveReadLaneAt(double4, int32_t);
 
+//===----------------------------------------------------------------------===//
+// WaveActiveMax builtins
+//===----------------------------------------------------------------------===//
+
+_HLSL_16BIT_AVAILABILITY(shadermodel, 6.0)
+_HLSL_BUILTIN_ALIAS(__builtin_hlsl_wave_active_max)
+__attribute__((convergent)) half WaveActiveMax(half);
+_HLSL_16BIT_AVAILABILITY(shadermodel, 6.0)
+_HLSL_BUILTIN_ALIAS(__builtin_hlsl_wave_active_max)
+__attribute__((convergent)) half2 WaveActiveMax(half2);
+_HLSL_16BIT_AVAILABILITY(shadermodel, 6.0)
+_HLSL_BUILTIN_ALIAS(__builtin_hlsl_wave_active_max)
+__attribute__((convergent)) half3 WaveActiveMax(half3);
+_HLSL_16BIT_AVAILABILITY(shadermodel, 6.0)
+_HLSL_BUILTIN_ALIAS(__builtin_hlsl_wave_active_max)
+__attribute__((convergent)) half4 WaveActiveMax(half4);
+
+#ifdef __HLSL_ENABLE_16_BIT
+_HLSL_AVAILABILITY(shadermodel, 6.0)
+_HLSL_BUILTIN_ALIAS(__builtin_hlsl_wave_active_max)
+__attribute__((convergent)) int16_t WaveActiveMax(int16_t);
+_HLSL_AVAILABILITY(shadermodel, 6.0)
+_HLSL_BUILTIN_ALIAS(__builtin_hlsl_wave_active_max)
+__attribute__((convergent)) int16_t2 WaveActiveMax(int16_t2);
+_HLSL_AVAILABILITY(shadermodel, 6.0)
+_HLSL_BUILTIN_ALIAS(__builtin_hlsl_wave_active_max)
+__attribute__((convergent)) int16_t3 WaveActiveMax(int16_t3);
+_HLSL_AVAILABILITY(shadermodel, 6.0)
+_HLSL_BUILTIN_ALIAS(__builtin_hlsl_wave_active_max)
+__attribute__((convergent)) int16_t4 WaveActiveMax(int16_t4);
+
+_HLSL_AVAILABILITY(shadermodel, 6.0)
+_HLSL_BUILTIN_ALIAS(__builtin_hlsl_wave_active_max)
+__attribute__((convergent)) uint16_t WaveActiveMax(uint16_t);
+_HLSL_AVAILABILITY(shadermodel, 6.0)
+_HLSL_BUILTIN_ALIAS(__builtin_hlsl_wave_active_max)
+__attribute__((convergent)) uint16_t2 WaveActiveMax(uint16_t2);
+_HLSL_AVAILABILITY(shadermodel, 6.0)
+_HLSL_BUILTIN_ALIAS(__builtin_hlsl_wave_active_max)
+__attribute__((convergent)) uint16_t3 WaveActiveMax(uint16_t3);
+_HLSL_AVAILABILITY(shadermodel, 6.0)
+_HLSL_BUILTIN_ALIAS(__builtin_hlsl_wave_active_max)
+__attribute__((convergent)) uint16_t4 WaveActiveMax(uint16_t4);
+#endif
+
+_HLSL_BUILTIN_ALIAS(__builtin_hlsl_wave_active_max)
+__attribute__((convergent)) int WaveActiveMax(int);
+_HLSL_BUILTIN_ALIAS(__builtin_hlsl_wave_active_max)
+__attribute__((convergent)) int2 WaveActiveMax(int2);
+_HLSL_BUILTIN_ALIAS(__builtin_hlsl_wave_active_max)
+__attribute__((convergent)) int3 WaveActiveMax(int3);
+_HLSL_BUILTIN_ALIAS(__builtin_hlsl_wave_active_max)
+__attribute__((convergent)) int4 WaveActiveMax(int4);
+
+_HLSL_BUILTIN_ALIAS(__builtin_hlsl_wave_active_max)
+__attribute__((convergent)) uint WaveActiveMax(uint);
+_HLSL_BUILTIN_ALIAS(__builtin_hlsl_wave_active_max)
+__attribute__((convergent)) uint2 WaveActiveMax(uint2);
+_HLSL_BUILTIN_ALIAS(__builtin_hlsl_wave_active_max)
+__attribute__((convergent)) uint3 WaveActiveMax(uint3);
+_HLSL_BUILTIN_ALIAS(__builtin_hlsl_wave_active_max)
+__attribute__((convergent)) uint4 WaveActiveMax(uint4);
+
+_HLSL_BUILTIN_ALIAS(__builtin_hlsl_wave_active_max)
+__attribute__((convergent)) int64_t WaveActiveMax(int64_t);
+_HLSL_BUILTIN_ALIAS(__builtin_hlsl_wave_active_max)
+__attribute__((convergent)) int64_t2 WaveActiveMax(int64_t2);
+_HLSL_BUILTIN_ALIAS(__builtin_hlsl_wave_active_max)
+__attribute__((convergent)) int64_t3 WaveActiveMax(int64_t3);
+_HLSL_BUILTIN_ALIAS(__builtin_hlsl_wave_active_max)
+__attribute__((convergent)) int64_t4 WaveActiveMax(int64_t4);
+
+_HLSL_BUILTIN_ALIAS(__builtin_hlsl_wave_active_max)
+__attribute__((convergent)) uint64_t WaveActiveMax(uint64_t);
+_HLSL_BUILTIN_ALIAS(__builtin_hlsl_wave_active_max)
+__attribute__((convergent)) uint64_t2 WaveActiveMax(uint64_t2);
+_HLSL_BUILTIN_ALIAS(__builtin_hlsl_wave_active_max)
+__attribute__((convergent)) uint64_t3 WaveActiveMax(uint64_t3);
+_HLSL_BUILTIN_ALIAS(__builtin_hlsl_wave_active_max)
+__attribute__((convergent)) uint64_t4 WaveActiveMax(uint64_t4);
+
+_HLSL_BUILTIN_ALIAS(__builtin_hlsl_wave_active_max)
+__attribute__((convergent)) float WaveActiveMax(float);
+_HLSL_BUILTIN_ALIAS(__builtin_hlsl_wave_active_max)
+__attribute__((convergent)) float2 WaveActiveMax(float2);
+_HLSL_BUILTIN_ALIAS(__builtin_hlsl_wave_active_max)
+__attribute__((convergent)) float3 WaveActiveMax(float3);
+_HLSL_BUILTIN_ALIAS(__builtin_hlsl_wave_active_max)
+__attribute__((convergent)) float4 WaveActiveMax(float4);
+
+_HLSL_BUILTIN_ALIAS(__builtin_hlsl_wave_active_max)
+__attribute__((convergent)) double WaveActiveMax(double);
+_HLSL_BUILTIN_ALIAS(__builtin_hlsl_wave_active_max)
+__attribute__((convergent)) double2 WaveActiveMax(double2);
+_HLSL_BUILTIN_ALIAS(__builtin_hlsl_wave_active_max)
+__attribute__((convergent)) double3 WaveActiveMax(double3);
+_HLSL_BUILTIN_ALIAS(__builtin_hlsl_wave_active_max)
+__attribute__((convergent)) double4 WaveActiveMax(double4);
+
 //===----------------------------------------------------------------------===//
 // WaveActiveSum builtins
 //===----------------------------------------------------------------------===//
diff --git a/clang/lib/Headers/nvptxintrin.h b/clang/lib/Headers/nvptxintrin.h
index 962dca9cf0312..40fa2edebe975 100644
--- a/clang/lib/Headers/nvptxintrin.h
+++ b/clang/lib/Headers/nvptxintrin.h
@@ -149,25 +149,23 @@ _DEFAULT_FN_ATTRS static __inline__ void __gpu_sync_lane(uint64_t __lane_mask) {
 
 // Shuffles the the lanes inside the warp according to the given index.
 _DEFAULT_FN_ATTRS static __inline__ uint32_t
-__gpu_shuffle_idx_u32(uint64_t __lane_mask, uint32_t __idx, uint32_t __x) {
+__gpu_shuffle_idx_u32(uint64_t __lane_mask, uint32_t __idx, uint32_t __x,
+                      uint32_t __width) {
   uint32_t __mask = (uint32_t)__lane_mask;
-  uint32_t __bitmask = (__mask >> __idx) & 1u;
-  return -__bitmask &
-         __nvvm_shfl_sync_idx_i32(__mask, __x, __idx, __gpu_num_lanes() - 1u);
+  return __nvvm_shfl_sync_idx_i32(__mask, __x, __idx,
+                                  ((__gpu_num_lanes() - __width) << 8u) | 0x1f);
 }
 
 // Shuffles the the lanes inside the warp according to the given index.
 _DEFAULT_FN_ATTRS static __inline__ uint64_t
-__gpu_shuffle_idx_u64(uint64_t __lane_mask, uint32_t __idx, uint64_t __x) {
+__gpu_shuffle_idx_u64(uint64_t __lane_mask, uint32_t __idx, uint64_t __x,
+                      uint32_t __width) {
   uint32_t __hi = (uint32_t)(__x >> 32ull);
   uint32_t __lo = (uint32_t)(__x & 0xFFFFFFFF);
   uint32_t __mask = (uint32_t)__lane_mask;
-  uint64_t __bitmask = (__mask >> __idx) & 1u;
-  return -__bitmask & ((uint64_t)__nvvm_shfl_sync_idx_i32(
-                           __mask, __hi, __idx, __gpu_num_lanes() - 1u)
-                       << 32ull) |
-         ((uint64_t)__nvvm_shfl_sync_idx_i32(__mask, __lo, __idx,
-                                             __gpu_num_lanes() - 1u));
+  return ((uint64_t)__gpu_shuffle_idx_u32(__mask, __idx, __hi, __width)
+          << 32ull) |
+         ((uint64_t)__gpu_shuffle_idx_u32(__mask, __idx, __lo, __width));
 }
 
 // Returns true if the flat pointer points to CUDA 'shared' memory.
diff --git a/clang/lib/Index/FileIndexRecord.cpp b/clang/lib/Index/FileIndexRecord.cpp
index 449c33637eb7e..cf40a596f5094 100644
--- a/clang/lib/Index/FileIndexRecord.cpp
+++ b/clang/lib/Index/FileIndexRecord.cpp
@@ -55,7 +55,7 @@ void FileIndexRecord::removeHeaderGuardMacros() {
 void FileIndexRecord::print(llvm::raw_ostream &OS, SourceManager &SM) const {
   OS << "DECLS BEGIN ---\n";
   for (auto &DclInfo : Decls) {
-    if (const auto *D = DclInfo.DeclOrMacro.dyn_cast<const Decl *>()) {
+    if (const auto *D = dyn_cast<const Decl *>(DclInfo.DeclOrMacro)) {
       SourceLocation Loc = SM.getFileLoc(D->getLocation());
       PresumedLoc PLoc = SM.getPresumedLoc(Loc);
       OS << llvm::sys::path::filename(PLoc.getFilename()) << ':'
diff --git a/clang/lib/Index/IndexBody.cpp b/clang/lib/Index/IndexBody.cpp
index c18daf7faa749..f1dc4d5831ce7 100644
--- a/clang/lib/Index/IndexBody.cpp
+++ b/clang/lib/Index/IndexBody.cpp
@@ -130,6 +130,9 @@ class BodyIndexer : public RecursiveASTVisitor<BodyIndexer> {
 
   void addCallRole(SymbolRoleSet &Roles,
                    SmallVectorImpl<SymbolRelation> &Relations) {
+    if (isa<CXXDeductionGuideDecl>(ParentDC))
+      return;
+
     Roles |= (unsigned)SymbolRole::Call;
     if (auto *FD = dyn_cast<FunctionDecl>(ParentDC))
       Relations.emplace_back((unsigned)SymbolRole::RelationCalledBy, FD);
diff --git a/clang/lib/Index/USRGeneration.cpp b/clang/lib/Index/USRGeneration.cpp
index 493123459a5a4..1e54b413dc59c 100644
--- a/clang/lib/Index/USRGeneration.cpp
+++ b/clang/lib/Index/USRGeneration.cpp
@@ -763,9 +763,10 @@ void USRGenerator::VisitType(QualType T) {
           Out << "@BT@OCLReserveID"; break;
         case BuiltinType::OCLSampler:
           Out << "@BT@OCLSampler"; break;
-#define SVE_TYPE(Name, Id, SingletonId) \
-        case BuiltinType::Id: \
-          Out << "@BT@" << Name; break;
+#define SVE_TYPE(Name, Id, SingletonId)                                        \
+  case BuiltinType::Id:                                                        \
+    Out << "@BT@" << #Name;                                                    \
+    break;
 #include "clang/Basic/AArch64SVEACLETypes.def"
 #define PPC_VECTOR_TYPE(Name, Id, Size) \
         case BuiltinType::Id: \
diff --git a/clang/lib/Lex/ModuleMap.cpp b/clang/lib/Lex/ModuleMap.cpp
index ccf94f6345ff2..998e2b977d109 100644
--- a/clang/lib/Lex/ModuleMap.cpp
+++ b/clang/lib/Lex/ModuleMap.cpp
@@ -2928,9 +2928,10 @@ void ModuleMapParser::parseInferredModuleDecl(bool Framework, bool Explicit) {
     ActiveModule->InferExplicitSubmodules = Explicit;
   } else {
     // We'll be inferring framework modules for this directory.
-    Map.InferredDirectories[Directory].InferModules = true;
-    Map.InferredDirectories[Directory].Attrs = Attrs;
-    Map.InferredDirectories[Directory].ModuleMapFID = ModuleMapFID;
+    auto &InfDir = Map.InferredDirectories[Directory];
+    InfDir.InferModules = true;
+    InfDir.Attrs = Attrs;
+    InfDir.ModuleMapFID = ModuleMapFID;
     // FIXME: Handle the 'framework' keyword.
   }
 
diff --git a/clang/lib/Lex/PPMacroExpansion.cpp b/clang/lib/Lex/PPMacroExpansion.cpp
index 9cf29668f251f..944966a791add 100644
--- a/clang/lib/Lex/PPMacroExpansion.cpp
+++ b/clang/lib/Lex/PPMacroExpansion.cpp
@@ -1798,58 +1798,55 @@ void Preprocessor::ExpandBuiltinMacro(Token &Tok) {
         return II && HasExtension(*this, II->getName());
       });
   } else if (II == Ident__has_builtin) {
-    EvaluateFeatureLikeBuiltinMacro(OS, Tok, II, *this, false,
-      [this](Token &Tok, bool &HasLexedNextToken) -> int {
-        IdentifierInfo *II = ExpectFeatureIdentifierInfo(Tok, *this,
-                                           diag::err_feature_check_malformed);
-        if (!II)
-          return false;
-        auto BuiltinID = II->getBuiltinID();
-        if (BuiltinID != 0) {
-          switch (BuiltinID) {
-          case Builtin::BI__builtin_cpu_is:
-            return getTargetInfo().supportsCpuIs();
-          case Builtin::BI__builtin_cpu_init:
-            return getTargetInfo().supportsCpuInit();
-          case Builtin::BI__builtin_cpu_supports:
-            return getTargetInfo().supportsCpuSupports();
-          case Builtin::BI__builtin_operator_new:
-          case Builtin::BI__builtin_operator_delete:
-            // denotes date of behavior change to support calling arbitrary
-            // usual allocation and deallocation functions. Required by libc++
-            return 201802;
-          default:
-            // __has_builtin should return false for aux builtins.
-            if (getBuiltinInfo().isAuxBuiltinID(BuiltinID))
-              return false;
-            return Builtin::evaluateRequiredTargetFeatures(
-                getBuiltinInfo().getRequiredFeatures(BuiltinID),
-                getTargetInfo().getTargetOpts().FeatureMap);
+    EvaluateFeatureLikeBuiltinMacro(
+        OS, Tok, II, *this, false,
+        [this](Token &Tok, bool &HasLexedNextToken) -> int {
+          IdentifierInfo *II = ExpectFeatureIdentifierInfo(
+              Tok, *this, diag::err_feature_check_malformed);
+          if (!II)
+            return false;
+          else if (II->getBuiltinID() != 0) {
+            switch (II->getBuiltinID()) {
+            case Builtin::BI__builtin_cpu_is:
+              return getTargetInfo().supportsCpuIs();
+            case Builtin::BI__builtin_cpu_init:
+              return getTargetInfo().supportsCpuInit();
+            case Builtin::BI__builtin_cpu_supports:
+              return getTargetInfo().supportsCpuSupports();
+            case Builtin::BI__builtin_operator_new:
+            case Builtin::BI__builtin_operator_delete:
+              // denotes date of behavior change to support calling arbitrary
+              // usual allocation and deallocation functions. Required by libc++
+              return 201802;
+            default:
+              return Builtin::evaluateRequiredTargetFeatures(
+                  getBuiltinInfo().getRequiredFeatures(II->getBuiltinID()),
+                  getTargetInfo().getTargetOpts().FeatureMap);
+            }
+            return true;
+          } else if (IsBuiltinTrait(Tok)) {
+            return true;
+          } else if (II->getTokenID() != tok::identifier &&
+                     II->getName().starts_with("__builtin_")) {
+            return true;
+          } else {
+            return llvm::StringSwitch<bool>(II->getName())
+                // Report builtin templates as being builtins.
+                .Case("__make_integer_seq", getLangOpts().CPlusPlus)
+                .Case("__type_pack_element", getLangOpts().CPlusPlus)
+                .Case("__builtin_common_type", getLangOpts().CPlusPlus)
+                // Likewise for some builtin preprocessor macros.
+                // FIXME: This is inconsistent; we usually suggest detecting
+                // builtin macros via #ifdef. Don't add more cases here.
+                .Case("__is_target_arch", true)
+                .Case("__is_target_vendor", true)
+                .Case("__is_target_os", true)
+                .Case("__is_target_environment", true)
+                .Case("__is_target_variant_os", true)
+                .Case("__is_target_variant_environment", true)
+                .Default(false);
           }
-          return true;
-        } else if (IsBuiltinTrait(Tok)) {
-          return true;
-        } else if (II->getTokenID() != tok::identifier &&
-                   II->getName().starts_with("__builtin_")) {
-          return true;
-        } else {
-          return llvm::StringSwitch<bool>(II->getName())
-              // Report builtin templates as being builtins.
-              .Case("__make_integer_seq", getLangOpts().CPlusPlus)
-              .Case("__type_pack_element", getLangOpts().CPlusPlus)
-              .Case("__builtin_common_type", getLangOpts().CPlusPlus)
-              // Likewise for some builtin preprocessor macros.
-              // FIXME: This is inconsistent; we usually suggest detecting
-              // builtin macros via #ifdef. Don't add more cases here.
-              .Case("__is_target_arch", true)
-              .Case("__is_target_vendor", true)
-              .Case("__is_target_os", true)
-              .Case("__is_target_environment", true)
-              .Case("__is_target_variant_os", true)
-              .Case("__is_target_variant_environment", true)
-              .Default(false);
-        }
-      });
+        });
   } else if (II == Ident__has_constexpr_builtin) {
     EvaluateFeatureLikeBuiltinMacro(
         OS, Tok, II, *this, false,
diff --git a/clang/lib/Parse/ParseDecl.cpp b/clang/lib/Parse/ParseDecl.cpp
index f136d5007e8a5..75b5e11f8327c 100644
--- a/clang/lib/Parse/ParseDecl.cpp
+++ b/clang/lib/Parse/ParseDecl.cpp
@@ -7315,15 +7315,16 @@ void Parser::ParseDecompositionDeclarator(Declarator &D) {
 
   // If this doesn't look like a structured binding, maybe it's a misplaced
   // array declarator.
-  if (!(Tok.is(tok::identifier) &&
+  if (!(Tok.isOneOf(tok::identifier, tok::ellipsis) &&
         NextToken().isOneOf(tok::comma, tok::r_square, tok::kw_alignas,
-                            tok::l_square)) &&
+                            tok::identifier, tok::l_square, tok::ellipsis)) &&
       !(Tok.is(tok::r_square) &&
         NextToken().isOneOf(tok::equal, tok::l_brace))) {
     PA.Revert();
     return ParseMisplacedBracketDeclarator(D);
   }
 
+  SourceLocation PrevEllipsisLoc;
   SmallVector<DecompositionDeclarator::Binding, 32> Bindings;
   while (Tok.isNot(tok::r_square)) {
     if (!Bindings.empty()) {
@@ -7338,11 +7339,11 @@ void Parser::ParseDecompositionDeclarator(Declarator &D) {
           Diag(Tok, diag::err_expected_comma_or_rsquare);
         }
 
-        SkipUntil(tok::r_square, tok::comma, tok::identifier,
+        SkipUntil({tok::r_square, tok::comma, tok::identifier, tok::ellipsis},
                   StopAtSemi | StopBeforeMatch);
         if (Tok.is(tok::comma))
           ConsumeToken();
-        else if (Tok.isNot(tok::identifier))
+        else if (Tok.is(tok::r_square))
           break;
       }
     }
@@ -7350,6 +7351,21 @@ void Parser::ParseDecompositionDeclarator(Declarator &D) {
     if (isCXX11AttributeSpecifier())
       DiagnoseAndSkipCXX11Attributes();
 
+    SourceLocation EllipsisLoc;
+
+    if (Tok.is(tok::ellipsis)) {
+      Diag(Tok, getLangOpts().CPlusPlus26 ? diag::warn_cxx23_compat_binding_pack
+                                          : diag::ext_cxx_binding_pack);
+      if (PrevEllipsisLoc.isValid()) {
+        Diag(Tok, diag::err_binding_multiple_ellipses);
+        Diag(PrevEllipsisLoc, diag::note_previous_ellipsis);
+        break;
+      }
+      EllipsisLoc = Tok.getLocation();
+      PrevEllipsisLoc = EllipsisLoc;
+      ConsumeToken();
+    }
+
     if (Tok.isNot(tok::identifier)) {
       Diag(Tok, diag::err_expected) << tok::identifier;
       break;
@@ -7359,6 +7375,13 @@ void Parser::ParseDecompositionDeclarator(Declarator &D) {
     SourceLocation Loc = Tok.getLocation();
     ConsumeToken();
 
+    if (Tok.is(tok::ellipsis) && !PrevEllipsisLoc.isValid()) {
+      DiagnoseMisplacedEllipsis(Tok.getLocation(), Loc, EllipsisLoc.isValid(),
+                                true);
+      EllipsisLoc = Tok.getLocation();
+      ConsumeToken();
+    }
+
     ParsedAttributes Attrs(AttrFactory);
     if (isCXX11AttributeSpecifier()) {
       Diag(Tok, getLangOpts().CPlusPlus26
@@ -7367,7 +7390,7 @@ void Parser::ParseDecompositionDeclarator(Declarator &D) {
       MaybeParseCXX11Attributes(Attrs);
     }
 
-    Bindings.push_back({II, Loc, std::move(Attrs)});
+    Bindings.push_back({II, Loc, std::move(Attrs), EllipsisLoc});
   }
 
   if (Tok.isNot(tok::r_square))
diff --git a/clang/lib/Parse/ParseDeclCXX.cpp b/clang/lib/Parse/ParseDeclCXX.cpp
index fddb8a5729ee8..43db715ac6d70 100644
--- a/clang/lib/Parse/ParseDeclCXX.cpp
+++ b/clang/lib/Parse/ParseDeclCXX.cpp
@@ -1813,7 +1813,6 @@ void Parser::ParseClassSpecifier(tok::TokenKind TagTokKind,
           tok::kw___is_pointer,
           tok::kw___is_polymorphic,
           tok::kw___is_reference,
-          tok::kw___is_referenceable,
           tok::kw___is_rvalue_expr,
           tok::kw___is_rvalue_reference,
           tok::kw___is_same,
diff --git a/clang/lib/Parse/ParseExpr.cpp b/clang/lib/Parse/ParseExpr.cpp
index aa8b3870a188c..da5f2ccf3a0c6 100644
--- a/clang/lib/Parse/ParseExpr.cpp
+++ b/clang/lib/Parse/ParseExpr.cpp
@@ -824,7 +824,6 @@ bool Parser::isRevertibleTypeTrait(const IdentifierInfo *II,
     REVERTIBLE_TYPE_TRAIT(__is_pointer);
     REVERTIBLE_TYPE_TRAIT(__is_polymorphic);
     REVERTIBLE_TYPE_TRAIT(__is_reference);
-    REVERTIBLE_TYPE_TRAIT(__is_referenceable);
     REVERTIBLE_TYPE_TRAIT(__is_rvalue_expr);
     REVERTIBLE_TYPE_TRAIT(__is_rvalue_reference);
     REVERTIBLE_TYPE_TRAIT(__is_same);
diff --git a/clang/lib/Parse/ParseOpenACC.cpp b/clang/lib/Parse/ParseOpenACC.cpp
index 98fd61913e5a4..d036971d2fc31 100644
--- a/clang/lib/Parse/ParseOpenACC.cpp
+++ b/clang/lib/Parse/ParseOpenACC.cpp
@@ -156,14 +156,14 @@ OpenACCClauseKind getOpenACCClauseKind(Token Tok) {
 // second part of the directive.
 OpenACCAtomicKind getOpenACCAtomicKind(Token Tok) {
   if (!Tok.is(tok::identifier))
-    return OpenACCAtomicKind::Invalid;
+    return OpenACCAtomicKind::None;
   return llvm::StringSwitch<OpenACCAtomicKind>(
              Tok.getIdentifierInfo()->getName())
       .Case("read", OpenACCAtomicKind::Read)
       .Case("write", OpenACCAtomicKind::Write)
       .Case("update", OpenACCAtomicKind::Update)
       .Case("capture", OpenACCAtomicKind::Capture)
-      .Default(OpenACCAtomicKind::Invalid);
+      .Default(OpenACCAtomicKind::None);
 }
 
 OpenACCDefaultClauseKind getOpenACCDefaultClauseKind(Token Tok) {
@@ -398,17 +398,16 @@ OpenACCAtomicKind ParseOpenACCAtomicKind(Parser &P) {
 
   // #pragma acc atomic is equivilent to update:
   if (AtomicClauseToken.isAnnotation())
-    return OpenACCAtomicKind::Update;
+    return OpenACCAtomicKind::None;
 
   OpenACCAtomicKind AtomicKind = getOpenACCAtomicKind(AtomicClauseToken);
 
-  // If we don't know what this is, treat it as 'nothing', and treat the rest of
-  // this as a clause list, which, despite being invalid, is likely what the
-  // user was trying to do.
-  if (AtomicKind == OpenACCAtomicKind::Invalid)
-    return OpenACCAtomicKind::Update;
+  // If this isn't a valid atomic-kind, don't consume the token, and treat the
+  // rest as a clause list, which despite there being no permissible clauses,
+  // will diagnose as a clause.
+  if (AtomicKind != OpenACCAtomicKind::None)
+    P.ConsumeToken();
 
-  P.ConsumeToken();
   return AtomicKind;
 }
 
@@ -570,12 +569,19 @@ void SkipUntilEndOfDirective(Parser &P) {
 
 bool doesDirectiveHaveAssociatedStmt(OpenACCDirectiveKind DirKind) {
   switch (DirKind) {
-  default:
+  case OpenACCDirectiveKind::Routine:
+    // FIXME: Routine MIGHT end up needing to be 'true' here, as it needs a way
+    // to capture a lambda-expression on the next line.
+  case OpenACCDirectiveKind::Cache:
+  case OpenACCDirectiveKind::Declare:
+  case OpenACCDirectiveKind::Set:
   case OpenACCDirectiveKind::EnterData:
   case OpenACCDirectiveKind::ExitData:
   case OpenACCDirectiveKind::Wait:
   case OpenACCDirectiveKind::Init:
   case OpenACCDirectiveKind::Shutdown:
+  case OpenACCDirectiveKind::Update:
+  case OpenACCDirectiveKind::Invalid:
     return false;
   case OpenACCDirectiveKind::Parallel:
   case OpenACCDirectiveKind::Serial:
@@ -586,6 +592,7 @@ bool doesDirectiveHaveAssociatedStmt(OpenACCDirectiveKind DirKind) {
   case OpenACCDirectiveKind::Loop:
   case OpenACCDirectiveKind::Data:
   case OpenACCDirectiveKind::HostData:
+  case OpenACCDirectiveKind::Atomic:
     return true;
   }
   llvm_unreachable("Unhandled directive->assoc stmt");
@@ -1428,6 +1435,7 @@ Parser::ParseOpenACCDirective() {
   SourceLocation DirLoc = getCurToken().getLocation();
   OpenACCDirectiveKind DirKind = ParseOpenACCDirectiveKind(*this);
   Parser::OpenACCWaitParseInfo WaitInfo;
+  OpenACCAtomicKind AtomicKind = OpenACCAtomicKind::None;
 
   getActions().OpenACC().ActOnConstruct(DirKind, DirLoc);
 
@@ -1435,7 +1443,7 @@ Parser::ParseOpenACCDirective() {
   // specifiers that need to be taken care of. Atomic has an 'atomic-clause'
   // that needs to be parsed.
   if (DirKind == OpenACCDirectiveKind::Atomic)
-    ParseOpenACCAtomicKind(*this);
+    AtomicKind = ParseOpenACCAtomicKind(*this);
 
   // We've successfully parsed the construct/directive name, however a few of
   // the constructs have optional parens that contain further details.
@@ -1490,6 +1498,7 @@ Parser::ParseOpenACCDirective() {
                                       T.getCloseLocation(),
                                       /*EndLoc=*/SourceLocation{},
                                       WaitInfo.QueuesLoc,
+                                      AtomicKind,
                                       WaitInfo.getAllExprs(),
                                       ParseOpenACCClauseList(DirKind)};
 
@@ -1538,11 +1547,12 @@ StmtResult Parser::ParseOpenACCDirectiveStmt() {
     ParseScope ACCScope(this, getOpenACCScopeFlags(DirInfo.DirKind));
 
     AssocStmt = getActions().OpenACC().ActOnAssociatedStmt(
-        DirInfo.StartLoc, DirInfo.DirKind, DirInfo.Clauses, ParseStatement());
+        DirInfo.StartLoc, DirInfo.DirKind, DirInfo.AtomicKind, DirInfo.Clauses,
+        ParseStatement());
   }
 
   return getActions().OpenACC().ActOnEndStmtDirective(
       DirInfo.DirKind, DirInfo.StartLoc, DirInfo.DirLoc, DirInfo.LParenLoc,
-      DirInfo.MiscLoc, DirInfo.Exprs, DirInfo.RParenLoc, DirInfo.EndLoc,
-      DirInfo.Clauses, AssocStmt);
+      DirInfo.MiscLoc, DirInfo.Exprs, DirInfo.AtomicKind, DirInfo.RParenLoc,
+      DirInfo.EndLoc, DirInfo.Clauses, AssocStmt);
 }
diff --git a/clang/lib/Parse/ParseOpenMP.cpp b/clang/lib/Parse/ParseOpenMP.cpp
index 89b83938f352d..a455659ca8f2c 100644
--- a/clang/lib/Parse/ParseOpenMP.cpp
+++ b/clang/lib/Parse/ParseOpenMP.cpp
@@ -888,7 +888,18 @@ void Parser::parseOMPTraitPropertyKind(OMPTraitProperty &TIProperty,
   TIProperty.Kind = TraitProperty::invalid;
 
   SourceLocation NameLoc = Tok.getLocation();
-  StringRef Name = getNameFromIdOrString(*this, Tok, CONTEXT_TRAIT_LVL);
+  StringRef Name;
+  if (Selector == llvm::omp::TraitSelector::target_device_device_num) {
+    Name = "number";
+    TIProperty.Kind = getOpenMPContextTraitPropertyKind(Set, Selector, Name);
+    ExprResult DeviceNumExprResult = ParseExpression();
+    if (DeviceNumExprResult.isUsable()) {
+      Expr *DeviceNumExpr = DeviceNumExprResult.get();
+      Actions.OpenMP().ActOnOpenMPDeviceNum(DeviceNumExpr);
+    }
+    return;
+  }
+  Name = getNameFromIdOrString(*this, Tok, CONTEXT_TRAIT_LVL);
   if (Name.empty()) {
     Diag(Tok.getLocation(), diag::note_omp_declare_variant_ctx_options)
         << CONTEXT_TRAIT_LVL << listOpenMPContextTraitProperties(Set, Selector);
@@ -918,7 +929,8 @@ void Parser::parseOMPTraitPropertyKind(OMPTraitProperty &TIProperty,
         << "(<property-name>)";
     return;
   }
-  TraitSelector SelectorForName = getOpenMPContextTraitSelectorKind(Name);
+  TraitSelector SelectorForName =
+      getOpenMPContextTraitSelectorKind(Name, SetForName);
   if (SelectorForName != TraitSelector::invalid) {
     Diag(NameLoc, diag::note_omp_declare_variant_ctx_is_a)
         << Name << CONTEXT_SELECTOR_LVL << CONTEXT_TRAIT_LVL;
@@ -935,7 +947,7 @@ void Parser::parseOMPTraitPropertyKind(OMPTraitProperty &TIProperty,
   }
   for (const auto &PotentialSet :
        {TraitSet::construct, TraitSet::user, TraitSet::implementation,
-        TraitSet::device}) {
+        TraitSet::device, TraitSet::target_device}) {
     TraitProperty PropertyForName =
         getOpenMPContextTraitPropertyKind(PotentialSet, Selector, Name);
     if (PropertyForName == TraitProperty::invalid)
@@ -1062,7 +1074,7 @@ void Parser::parseOMPTraitSelectorKind(OMPTraitSelector &TISelector,
     return;
   }
 
-  TISelector.Kind = getOpenMPContextTraitSelectorKind(Name);
+  TISelector.Kind = getOpenMPContextTraitSelectorKind(Name, Set);
   if (TISelector.Kind != TraitSelector::invalid) {
     if (checkForDuplicates(*this, Name, NameLoc, Seen, CONTEXT_SELECTOR_LVL))
       TISelector.Kind = TraitSelector::invalid;
@@ -1084,7 +1096,7 @@ void Parser::parseOMPTraitSelectorKind(OMPTraitSelector &TISelector,
   }
   for (const auto &PotentialSet :
        {TraitSet::construct, TraitSet::user, TraitSet::implementation,
-        TraitSet::device}) {
+        TraitSet::device, TraitSet::target_device}) {
     TraitProperty PropertyForName = getOpenMPContextTraitPropertyKind(
         PotentialSet, TraitSelector::invalid, Name);
     if (PropertyForName == TraitProperty::invalid)
@@ -1259,7 +1271,8 @@ void Parser::parseOMPTraitSetKind(OMPTraitSet &TISet,
   // It follows diagnosis and helping notes.
   Diag(NameLoc, diag::warn_omp_declare_variant_ctx_not_a_set) << Name;
 
-  TraitSelector SelectorForName = getOpenMPContextTraitSelectorKind(Name);
+  TraitSelector SelectorForName =
+      getOpenMPContextTraitSelectorKind(Name, TISet.Kind);
   if (SelectorForName != TraitSelector::invalid) {
     Diag(NameLoc, diag::note_omp_declare_variant_ctx_is_a)
         << Name << CONTEXT_SELECTOR_LVL << CONTEXT_SELECTOR_SET_LVL;
@@ -1276,7 +1289,7 @@ void Parser::parseOMPTraitSetKind(OMPTraitSet &TISet,
   }
   for (const auto &PotentialSet :
        {TraitSet::construct, TraitSet::user, TraitSet::implementation,
-        TraitSet::device}) {
+        TraitSet::device, TraitSet::target_device}) {
     TraitProperty PropertyForName = getOpenMPContextTraitPropertyKind(
         PotentialSet, TraitSelector::invalid, Name);
     if (PropertyForName == TraitProperty::invalid)
@@ -1680,6 +1693,7 @@ void Parser::ParseOpenMPClauses(OpenMPDirectiveKind DKind,
 ///     'holds' '(' scalar-expression ')'
 ///     'no_openmp'
 ///     'no_openmp_routines'
+///     'no_openmp_constructs' (OpenMP 6.0)
 ///     'no_parallelism'
 ///
 void Parser::ParseOpenMPAssumesDirective(OpenMPDirectiveKind DKind,
@@ -2253,7 +2267,8 @@ Parser::DeclGroupPtrTy Parser::ParseOpenMPDeclarativeDirectiveWithExtDecl(
     TargetOMPContext OMPCtx(
         ASTCtx, std::move(DiagUnknownTrait),
         /* CurrentFunctionDecl */ nullptr,
-        /* ConstructTraits */ ArrayRef<llvm::omp::TraitProperty>());
+        /* ConstructTraits */ ArrayRef<llvm::omp::TraitProperty>(),
+        Actions.OpenMP().getOpenMPDeviceNum());
 
     if (isVariantApplicableInContext(VMI, OMPCtx, /* DeviceSetOnly */ true)) {
       Actions.OpenMP().ActOnOpenMPBeginDeclareVariant(Loc, TI);
@@ -2805,7 +2820,8 @@ StmtResult Parser::ParseOpenMPDeclarativeOrExecutableDirective(
         };
     TargetOMPContext OMPCtx(ASTContext, std::move(DiagUnknownTrait),
                             /* CurrentFunctionDecl */ nullptr,
-                            ArrayRef<llvm::omp::TraitProperty>());
+                            ArrayRef<llvm::omp::TraitProperty>(),
+                            Actions.OpenMP().getOpenMPDeviceNum());
 
     // A single match is returned for OpenMP 5.0
     int BestIdx = getBestVariantMatchForContext(VMIs, OMPCtx);
@@ -3459,6 +3475,7 @@ OMPClause *Parser::ParseOpenMPClause(OpenMPDirectiveKind DKind,
   }
   case OMPC_no_openmp:
   case OMPC_no_openmp_routines:
+  case OMPC_no_openmp_constructs:
   case OMPC_no_parallelism: {
     if (!FirstClause) {
       Diag(Tok, diag::err_omp_more_one_clause)
diff --git a/clang/lib/Sema/CMakeLists.txt b/clang/lib/Sema/CMakeLists.txt
index 19cf3a2db00fd..1a351684d133e 100644
--- a/clang/lib/Sema/CMakeLists.txt
+++ b/clang/lib/Sema/CMakeLists.txt
@@ -71,6 +71,7 @@ add_clang_library(clangSema
   SemaObjC.cpp
   SemaObjCProperty.cpp
   SemaOpenACC.cpp
+  SemaOpenACCAtomic.cpp
   SemaOpenACCClause.cpp
   SemaOpenCL.cpp
   SemaOpenMP.cpp
diff --git a/clang/lib/Sema/HeuristicResolver.cpp b/clang/lib/Sema/HeuristicResolver.cpp
index 2a726fe51d355..3af4d001d6c1a 100644
--- a/clang/lib/Sema/HeuristicResolver.cpp
+++ b/clang/lib/Sema/HeuristicResolver.cpp
@@ -71,8 +71,17 @@ class HeuristicResolverImpl {
 
   // Helper function for HeuristicResolver::resolveDependentMember()
   // which takes a possibly-dependent type `T` and heuristically
-  // resolves it to a CXXRecordDecl in which we can try name lookup.
-  CXXRecordDecl *resolveTypeToRecordDecl(const Type *T);
+  // resolves it to a TagDecl in which we can try name lookup.
+  TagDecl *resolveTypeToTagDecl(const Type *T);
+
+  // Helper function for simplifying a type.
+  // `Type` is the type to simplify.
+  // `E` is the expression whose type `Type` is, if known. This sometimes
+  // contains information relevant to the type that's not stored in `Type`
+  // itself.
+  // If `UnwrapPointer` is true, exactly only pointer type will be unwrapped
+  // during simplification, and the operation fails if no pointer type is found.
+  QualType simplifyType(QualType Type, const Expr *E, bool UnwrapPointer);
 
   // This is a reimplementation of CXXRecordDecl::lookupDependentName()
   // so that the implementation can call into other HeuristicResolver helpers.
@@ -130,7 +139,7 @@ TemplateName getReferencedTemplateName(const Type *T) {
 // Helper function for HeuristicResolver::resolveDependentMember()
 // which takes a possibly-dependent type `T` and heuristically
 // resolves it to a CXXRecordDecl in which we can try name lookup.
-CXXRecordDecl *HeuristicResolverImpl::resolveTypeToRecordDecl(const Type *T) {
+TagDecl *HeuristicResolverImpl::resolveTypeToTagDecl(const Type *T) {
   assert(T);
 
   // Unwrap type sugar such as type aliases.
@@ -144,8 +153,9 @@ CXXRecordDecl *HeuristicResolverImpl::resolveTypeToRecordDecl(const Type *T) {
     T = T->getCanonicalTypeInternal().getTypePtr();
   }
 
-  if (const auto *RT = T->getAs<RecordType>())
-    return dyn_cast<CXXRecordDecl>(RT->getDecl());
+  if (auto *TT = T->getAs<TagType>()) {
+    return TT->getDecl();
+  }
 
   if (const auto *ICNT = T->getAs<InjectedClassNameType>())
     T = ICNT->getInjectedSpecializationType().getTypePtrOrNull();
@@ -197,6 +207,70 @@ QualType HeuristicResolverImpl::getPointeeType(QualType T) {
   return FirstArg.getAsType();
 }
 
+QualType HeuristicResolverImpl::simplifyType(QualType Type, const Expr *E,
+                                             bool UnwrapPointer) {
+  bool DidUnwrapPointer = false;
+  // A type, together with an optional expression whose type it represents
+  // which may have additional information about the expression's type
+  // not stored in the QualType itself.
+  struct TypeExprPair {
+    QualType Type;
+    const Expr *E = nullptr;
+  };
+  TypeExprPair Current{Type, E};
+  auto SimplifyOneStep = [UnwrapPointer, &DidUnwrapPointer,
+                          this](TypeExprPair T) -> TypeExprPair {
+    if (UnwrapPointer) {
+      if (QualType Pointee = getPointeeType(T.Type); !Pointee.isNull()) {
+        DidUnwrapPointer = true;
+        return {Pointee};
+      }
+    }
+    if (const auto *RT = T.Type->getAs<ReferenceType>()) {
+      // Does not count as "unwrap pointer".
+      return {RT->getPointeeType()};
+    }
+    if (const auto *BT = T.Type->getAs<BuiltinType>()) {
+      // If BaseType is the type of a dependent expression, it's just
+      // represented as BuiltinType::Dependent which gives us no information. We
+      // can get further by analyzing the dependent expression.
+      if (T.E && BT->getKind() == BuiltinType::Dependent) {
+        return {resolveExprToType(T.E), T.E};
+      }
+    }
+    if (const auto *AT = T.Type->getContainedAutoType()) {
+      // If T contains a dependent `auto` type, deduction will not have
+      // been performed on it yet. In simple cases (e.g. `auto` variable with
+      // initializer), get the approximate type that would result from
+      // deduction.
+      // FIXME: A more accurate implementation would propagate things like the
+      // `const` in `const auto`.
+      if (T.E && AT->isUndeducedAutoType()) {
+        if (const auto *DRE = dyn_cast<DeclRefExpr>(T.E)) {
+          if (const auto *VD = dyn_cast<VarDecl>(DRE->getDecl())) {
+            if (auto *Init = VD->getInit())
+              return {resolveExprToType(Init), Init};
+          }
+        }
+      }
+    }
+    return T;
+  };
+  // As an additional protection against infinite loops, bound the number of
+  // simplification steps.
+  size_t StepCount = 0;
+  const size_t MaxSteps = 64;
+  while (!Current.Type.isNull() && StepCount++ < MaxSteps) {
+    TypeExprPair New = SimplifyOneStep(Current);
+    if (New.Type == Current.Type)
+      break;
+    Current = New;
+  }
+  if (UnwrapPointer && !DidUnwrapPointer)
+    return QualType();
+  return Current.Type;
+}
+
 std::vector<const NamedDecl *> HeuristicResolverImpl::resolveMemberExpr(
     const CXXDependentScopeMemberExpr *ME) {
   // If the expression has a qualifier, try resolving the member inside the
@@ -227,28 +301,17 @@ std::vector<const NamedDecl *> HeuristicResolverImpl::resolveMemberExpr(
   }
 
   // Try resolving the member inside the expression's base type.
+  Expr *Base = ME->isImplicitAccess() ? nullptr : ME->getBase();
   QualType BaseType = ME->getBaseType();
-  if (ME->isArrow()) {
-    BaseType = getPointeeType(BaseType);
-  }
-  if (BaseType.isNull())
-    return {};
-  if (const auto *BT = BaseType->getAs<BuiltinType>()) {
-    // If BaseType is the type of a dependent expression, it's just
-    // represented as BuiltinType::Dependent which gives us no information. We
-    // can get further by analyzing the dependent expression.
-    Expr *Base = ME->isImplicitAccess() ? nullptr : ME->getBase();
-    if (Base && BT->getKind() == BuiltinType::Dependent) {
-      BaseType = resolveExprToType(Base);
-    }
-  }
+  BaseType = simplifyType(BaseType, Base, ME->isArrow());
   return resolveDependentMember(BaseType, ME->getMember(), NoFilter);
 }
 
 std::vector<const NamedDecl *>
 HeuristicResolverImpl::resolveDeclRefExpr(const DependentScopeDeclRefExpr *RE) {
-  return resolveDependentMember(QualType(RE->getQualifier()->getAsType(), 0),
-                                RE->getDeclName(), StaticFilter);
+  return resolveDependentMember(
+      resolveNestedNameSpecifierToType(RE->getQualifier()), RE->getDeclName(),
+      StaticFilter);
 }
 
 std::vector<const NamedDecl *>
@@ -260,7 +323,7 @@ HeuristicResolverImpl::resolveTypeOfCallExpr(const CallExpr *CE) {
     CalleeType = FnTypePtr->getPointeeType();
   if (const FunctionType *FnType = CalleeType->getAs<FunctionType>()) {
     if (const auto *D =
-            resolveTypeToRecordDecl(FnType->getReturnType().getTypePtr())) {
+            resolveTypeToTagDecl(FnType->getReturnType().getTypePtr())) {
       return {D};
     }
   }
@@ -371,11 +434,11 @@ bool findOrdinaryMember(const CXXRecordDecl *RD, CXXBasePath &Path,
 bool HeuristicResolverImpl::findOrdinaryMemberInDependentClasses(
     const CXXBaseSpecifier *Specifier, CXXBasePath &Path,
     DeclarationName Name) {
-  CXXRecordDecl *RD =
-      resolveTypeToRecordDecl(Specifier->getType().getTypePtr());
-  if (!RD)
-    return false;
-  return findOrdinaryMember(RD, Path, Name);
+  TagDecl *TD = resolveTypeToTagDecl(Specifier->getType().getTypePtr());
+  if (const auto *RD = dyn_cast_if_present<CXXRecordDecl>(TD)) {
+    return findOrdinaryMember(RD, Path, Name);
+  }
+  return false;
 }
 
 std::vector<const NamedDecl *> HeuristicResolverImpl::lookupDependentName(
@@ -417,11 +480,14 @@ std::vector<const NamedDecl *> HeuristicResolverImpl::resolveDependentMember(
   const Type *T = QT.getTypePtrOrNull();
   if (!T)
     return {};
-  if (auto *ET = T->getAs<EnumType>()) {
-    auto Result = ET->getDecl()->lookup(Name);
+  TagDecl *TD = resolveTypeToTagDecl(T);
+  if (!TD)
+    return {};
+  if (auto *ED = dyn_cast<EnumDecl>(TD)) {
+    auto Result = ED->lookup(Name);
     return {Result.begin(), Result.end()};
   }
-  if (auto *RD = resolveTypeToRecordDecl(T)) {
+  if (auto *RD = dyn_cast<CXXRecordDecl>(TD)) {
     if (!RD->hasDefinition())
       return {};
     RD = RD->getDefinition();
@@ -429,7 +495,8 @@ std::vector<const NamedDecl *> HeuristicResolverImpl::resolveDependentMember(
       if (!Filter(ND))
         return false;
       if (const auto *MD = dyn_cast<CXXMethodDecl>(ND)) {
-        return MD->getMethodQualifiers().compatiblyIncludes(QT.getQualifiers(),
+        return !MD->isInstance() ||
+               MD->getMethodQualifiers().compatiblyIncludes(QT.getQualifiers(),
                                                             Ctx);
       }
       return true;
diff --git a/clang/lib/Sema/JumpDiagnostics.cpp b/clang/lib/Sema/JumpDiagnostics.cpp
index d465599450e7f..edcfffa2b3894 100644
--- a/clang/lib/Sema/JumpDiagnostics.cpp
+++ b/clang/lib/Sema/JumpDiagnostics.cpp
@@ -561,12 +561,12 @@ void JumpScopeChecker::BuildScopeInformation(Stmt *S,
     // implementable but a lot of work which we haven't felt up to doing.
     ExprWithCleanups *EWC = cast<ExprWithCleanups>(S);
     for (unsigned i = 0, e = EWC->getNumObjects(); i != e; ++i) {
-      if (auto *BDecl = EWC->getObject(i).dyn_cast<BlockDecl *>())
+      if (auto *BDecl = dyn_cast<BlockDecl *>(EWC->getObject(i)))
         for (const auto &CI : BDecl->captures()) {
           VarDecl *variable = CI.getVariable();
           BuildScopeInformation(variable, BDecl, origParentScope);
         }
-      else if (auto *CLE = EWC->getObject(i).dyn_cast<CompoundLiteralExpr *>())
+      else if (auto *CLE = dyn_cast<CompoundLiteralExpr *>(EWC->getObject(i)))
         BuildScopeInformation(CLE, origParentScope);
       else
         llvm_unreachable("unexpected cleanup object type");
@@ -597,15 +597,6 @@ void JumpScopeChecker::BuildScopeInformation(Stmt *S,
     LabelAndGotoScopes[S] = ParentScope;
     break;
 
-  case Stmt::AttributedStmtClass: {
-    AttributedStmt *AS = cast<AttributedStmt>(S);
-    if (GetMustTailAttr(AS)) {
-      LabelAndGotoScopes[AS] = ParentScope;
-      MustTailStmts.push_back(AS);
-    }
-    break;
-  }
-
   case Stmt::OpenACCComputeConstructClass: {
     unsigned NewParentScope = Scopes.size();
     OpenACCComputeConstruct *CC = cast<OpenACCComputeConstruct>(S);
@@ -649,7 +640,7 @@ void JumpScopeChecker::BuildScopeInformation(Stmt *S,
       continue;
     }
 
-    // Cases, labels, and defaults aren't "scope parents".  It's also
+    // Cases, labels, attributes, and defaults aren't "scope parents".  It's also
     // important to handle these iteratively instead of recursively in
     // order to avoid blowing out the stack.
     while (true) {
@@ -658,7 +649,13 @@ void JumpScopeChecker::BuildScopeInformation(Stmt *S,
         Next = SC->getSubStmt();
       else if (LabelStmt *LS = dyn_cast<LabelStmt>(SubStmt))
         Next = LS->getSubStmt();
-      else
+      else if (AttributedStmt *AS = dyn_cast<AttributedStmt>(SubStmt)) {
+        if (GetMustTailAttr(AS)) {
+          LabelAndGotoScopes[AS] = ParentScope;
+          MustTailStmts.push_back(AS);
+        }
+        Next = AS->getSubStmt();
+      } else
         break;
 
       LabelAndGotoScopes[SubStmt] = ParentScope;
@@ -786,8 +783,7 @@ void JumpScopeChecker::VerifyIndirectJumps() {
     if (CHECK_PERMISSIVE(!LabelAndGotoScopes.count(TheLabel->getStmt())))
       continue;
     unsigned LabelScope = LabelAndGotoScopes[TheLabel->getStmt()];
-    if (!TargetScopes.contains(LabelScope))
-      TargetScopes[LabelScope] = TheLabel;
+    TargetScopes.try_emplace(LabelScope, TheLabel);
   }
 
   // For each target scope, make sure it's trivially reachable from
diff --git a/clang/lib/Sema/Sema.cpp b/clang/lib/Sema/Sema.cpp
index abb46d3a84e74..15c18f9a4525b 100644
--- a/clang/lib/Sema/Sema.cpp
+++ b/clang/lib/Sema/Sema.cpp
@@ -478,8 +478,8 @@ void Sema::Initialize() {
   if (Context.getTargetInfo().hasAArch64SVETypes() ||
       (Context.getAuxTargetInfo() &&
        Context.getAuxTargetInfo()->hasAArch64SVETypes())) {
-#define SVE_TYPE(Name, Id, SingletonId) \
-    addImplicitTypedef(Name, Context.SingletonId);
+#define SVE_TYPE(Name, Id, SingletonId)                                        \
+  addImplicitTypedef(#Name, Context.SingletonId);
 #include "clang/Basic/AArch64SVEACLETypes.def"
   }
 
@@ -1679,7 +1679,7 @@ void Sema::EmitDiagnostic(unsigned DiagID, const DiagnosticBuilder &DB) {
   // that is different from the last template instantiation where
   // we emitted an error, print a template instantiation
   // backtrace.
-  if (!DiagnosticIDs::isBuiltinNote(DiagID))
+  if (!Diags.getDiagnosticIDs()->isNote(DiagID))
     PrintContextStack();
 }
 
@@ -1693,7 +1693,8 @@ bool Sema::hasUncompilableErrorOccurred() const {
   if (Loc == DeviceDeferredDiags.end())
     return false;
   for (auto PDAt : Loc->second) {
-    if (DiagnosticIDs::isDefaultMappingAsError(PDAt.second.getDiagID()))
+    if (Diags.getDiagnosticIDs()->isDefaultMappingAsError(
+            PDAt.second.getDiagID()))
       return true;
   }
   return false;
diff --git a/clang/lib/Sema/SemaAPINotes.cpp b/clang/lib/Sema/SemaAPINotes.cpp
index 4f79775bc5e91..b354bb7b06435 100644
--- a/clang/lib/Sema/SemaAPINotes.cpp
+++ b/clang/lib/Sema/SemaAPINotes.cpp
@@ -478,7 +478,7 @@ static void ProcessAPINotes(Sema &S, FunctionOrMethod AnyFunc,
                             const api_notes::FunctionInfo &Info,
                             VersionedInfoMetadata Metadata) {
   // Find the declaration itself.
-  FunctionDecl *FD = AnyFunc.dyn_cast<FunctionDecl *>();
+  FunctionDecl *FD = dyn_cast<FunctionDecl *>(AnyFunc);
   Decl *D = FD;
   ObjCMethodDecl *MD = nullptr;
   if (!D) {
diff --git a/clang/lib/Sema/SemaARM.cpp b/clang/lib/Sema/SemaARM.cpp
index db418d80e0e09..b4b40f293b28e 100644
--- a/clang/lib/Sema/SemaARM.cpp
+++ b/clang/lib/Sema/SemaARM.cpp
@@ -352,6 +352,8 @@ static QualType getNeonEltType(NeonTypeFlags Flags, ASTContext &Context,
     return Context.DoubleTy;
   case NeonTypeFlags::BFloat16:
     return Context.BFloat16Ty;
+  case NeonTypeFlags::MFloat8:
+    return Context.MFloat8Ty;
   }
   llvm_unreachable("Invalid NeonTypeFlag!");
 }
@@ -622,20 +624,6 @@ static bool checkArmStreamingBuiltin(Sema &S, CallExpr *TheCall,
   return true;
 }
 
-static bool hasArmZAState(const FunctionDecl *FD) {
-  const auto *T = FD->getType()->getAs<FunctionProtoType>();
-  return (T && FunctionType::getArmZAState(T->getAArch64SMEAttributes()) !=
-                   FunctionType::ARM_None) ||
-         (FD->hasAttr<ArmNewAttr>() && FD->getAttr<ArmNewAttr>()->isNewZA());
-}
-
-static bool hasArmZT0State(const FunctionDecl *FD) {
-  const auto *T = FD->getType()->getAs<FunctionProtoType>();
-  return (T && FunctionType::getArmZT0State(T->getAArch64SMEAttributes()) !=
-                   FunctionType::ARM_None) ||
-         (FD->hasAttr<ArmNewAttr>() && FD->getAttr<ArmNewAttr>()->isNewZT0());
-}
-
 static ArmSMEState getSMEState(unsigned BuiltinID) {
   switch (BuiltinID) {
   default:
@@ -648,7 +636,8 @@ static ArmSMEState getSMEState(unsigned BuiltinID) {
 
 bool SemaARM::CheckSMEBuiltinFunctionCall(unsigned BuiltinID,
                                           CallExpr *TheCall) {
-  if (const FunctionDecl *FD = SemaRef.getCurFunctionDecl()) {
+  if (const FunctionDecl *FD =
+          SemaRef.getCurFunctionDecl(/*AllowLambda=*/true)) {
     std::optional<ArmStreamingType> BuiltinType;
 
     switch (BuiltinID) {
@@ -688,7 +677,8 @@ bool SemaARM::CheckSMEBuiltinFunctionCall(unsigned BuiltinID,
 
 bool SemaARM::CheckSVEBuiltinFunctionCall(unsigned BuiltinID,
                                           CallExpr *TheCall) {
-  if (const FunctionDecl *FD = SemaRef.getCurFunctionDecl()) {
+  if (const FunctionDecl *FD =
+          SemaRef.getCurFunctionDecl(/*AllowLambda=*/true)) {
     std::optional<ArmStreamingType> BuiltinType;
 
     switch (BuiltinID) {
@@ -717,23 +707,20 @@ bool SemaARM::CheckSVEBuiltinFunctionCall(unsigned BuiltinID,
 bool SemaARM::CheckNeonBuiltinFunctionCall(const TargetInfo &TI,
                                            unsigned BuiltinID,
                                            CallExpr *TheCall) {
-  if (const FunctionDecl *FD = SemaRef.getCurFunctionDecl()) {
+  if (const FunctionDecl *FD =
+          SemaRef.getCurFunctionDecl(/*AllowLambda=*/true)) {
+    std::optional<ArmStreamingType> BuiltinType;
 
     switch (BuiltinID) {
     default:
       break;
-#define GET_NEON_BUILTINS
-#define TARGET_BUILTIN(id, ...) case NEON::BI##id:
-#define BUILTIN(id, ...) case NEON::BI##id:
+#define GET_NEON_STREAMING_COMPAT_FLAG
 #include "clang/Basic/arm_neon.inc"
-      if (checkArmStreamingBuiltin(SemaRef, TheCall, FD, ArmNonStreaming,
-                                   BuiltinID))
-        return true;
-      break;
-#undef TARGET_BUILTIN
-#undef BUILTIN
-#undef GET_NEON_BUILTINS
+#undef GET_NEON_STREAMING_COMPAT_FLAG
     }
+    if (BuiltinType &&
+        checkArmStreamingBuiltin(SemaRef, TheCall, FD, *BuiltinType, BuiltinID))
+      return true;
   }
 
   llvm::APSInt Result;
diff --git a/clang/lib/Sema/SemaAVR.cpp b/clang/lib/Sema/SemaAVR.cpp
index 47368780b6203..389c55404fde3 100644
--- a/clang/lib/Sema/SemaAVR.cpp
+++ b/clang/lib/Sema/SemaAVR.cpp
@@ -30,6 +30,18 @@ void SemaAVR::handleInterruptAttr(Decl *D, const ParsedAttr &AL) {
   if (!AL.checkExactlyNumArgs(SemaRef, 0))
     return;
 
+  // AVR interrupt handlers must have no parameter and be void type.
+  if (hasFunctionProto(D) && getFunctionOrMethodNumParams(D) != 0) {
+    Diag(D->getLocation(), diag::warn_interrupt_signal_attribute_invalid)
+        << /*AVR*/ 3 << /*interrupt*/ 0 << 0;
+    return;
+  }
+  if (!getFunctionOrMethodResultType(D)->isVoidType()) {
+    Diag(D->getLocation(), diag::warn_interrupt_signal_attribute_invalid)
+        << /*AVR*/ 3 << /*interrupt*/ 0 << 1;
+    return;
+  }
+
   handleSimpleAttribute<AVRInterruptAttr>(*this, D, AL);
 }
 
@@ -43,6 +55,18 @@ void SemaAVR::handleSignalAttr(Decl *D, const ParsedAttr &AL) {
   if (!AL.checkExactlyNumArgs(SemaRef, 0))
     return;
 
+  // AVR signal handlers must have no parameter and be void type.
+  if (hasFunctionProto(D) && getFunctionOrMethodNumParams(D) != 0) {
+    Diag(D->getLocation(), diag::warn_interrupt_signal_attribute_invalid)
+        << /*AVR*/ 3 << /*signal*/ 1 << 0;
+    return;
+  }
+  if (!getFunctionOrMethodResultType(D)->isVoidType()) {
+    Diag(D->getLocation(), diag::warn_interrupt_signal_attribute_invalid)
+        << /*AVR*/ 3 << /*signal*/ 1 << 1;
+    return;
+  }
+
   handleSimpleAttribute<AVRSignalAttr>(*this, D, AL);
 }
 
diff --git a/clang/lib/Sema/SemaAttr.cpp b/clang/lib/Sema/SemaAttr.cpp
index 6907fa91e28c2..37c0546e321d9 100644
--- a/clang/lib/Sema/SemaAttr.cpp
+++ b/clang/lib/Sema/SemaAttr.cpp
@@ -537,7 +537,6 @@ bool Sema::ConstantFoldAttrArgs(const AttributeCommonInfo &CI,
         Diag(Note.first, Note.second);
       return false;
     }
-    assert(Eval.Val.hasValue());
     E = ConstantExpr::Create(Context, E, Eval.Val);
   }
 
@@ -1190,6 +1189,11 @@ void Sema::ActOnPragmaAttributePop(SourceLocation PragmaLoc,
 void Sema::AddPragmaAttributes(Scope *S, Decl *D) {
   if (PragmaAttributeStack.empty())
     return;
+
+  if (const auto *P = dyn_cast<ParmVarDecl>(D))
+    if (P->getType()->isVoidType())
+      return;
+
   for (auto &Group : PragmaAttributeStack) {
     for (auto &Entry : Group.Entries) {
       ParsedAttr *Attribute = Entry.Attribute;
diff --git a/clang/lib/Sema/SemaCUDA.cpp b/clang/lib/Sema/SemaCUDA.cpp
index 35f28bf1bd61a..0e1bf727d72d2 100644
--- a/clang/lib/Sema/SemaCUDA.cpp
+++ b/clang/lib/Sema/SemaCUDA.cpp
@@ -833,7 +833,7 @@ SemaBase::SemaDiagnosticBuilder SemaCUDA::DiagIfDeviceCode(SourceLocation Loc,
       if (!getLangOpts().CUDAIsDevice)
         return SemaDiagnosticBuilder::K_Nop;
       if (SemaRef.IsLastErrorImmediate &&
-          getDiagnostics().getDiagnosticIDs()->isBuiltinNote(DiagID))
+          getDiagnostics().getDiagnosticIDs()->isNote(DiagID))
         return SemaDiagnosticBuilder::K_Immediate;
       return (SemaRef.getEmissionStatus(CurFunContext) ==
               Sema::FunctionEmissionStatus::Emitted)
@@ -864,7 +864,7 @@ Sema::SemaDiagnosticBuilder SemaCUDA::DiagIfHostCode(SourceLocation Loc,
       if (getLangOpts().CUDAIsDevice)
         return SemaDiagnosticBuilder::K_Nop;
       if (SemaRef.IsLastErrorImmediate &&
-          getDiagnostics().getDiagnosticIDs()->isBuiltinNote(DiagID))
+          getDiagnostics().getDiagnosticIDs()->isNote(DiagID))
         return SemaDiagnosticBuilder::K_Immediate;
       return (SemaRef.getEmissionStatus(CurFunContext) ==
               Sema::FunctionEmissionStatus::Emitted)
diff --git a/clang/lib/Sema/SemaCast.cpp b/clang/lib/Sema/SemaCast.cpp
index 54bc52fa2ac40..23be71ad8e2ae 100644
--- a/clang/lib/Sema/SemaCast.cpp
+++ b/clang/lib/Sema/SemaCast.cpp
@@ -23,6 +23,7 @@
 #include "clang/Basic/TargetInfo.h"
 #include "clang/Lex/Preprocessor.h"
 #include "clang/Sema/Initialization.h"
+#include "clang/Sema/SemaHLSL.h"
 #include "clang/Sema/SemaObjC.h"
 #include "clang/Sema/SemaRISCV.h"
 #include "llvm/ADT/SmallVector.h"
@@ -2772,6 +2773,22 @@ void CastOperation::CheckCXXCStyleCast(bool FunctionalStyle,
     return;
   }
 
+  CheckedConversionKind CCK = FunctionalStyle
+                                  ? CheckedConversionKind::FunctionalCast
+                                  : CheckedConversionKind::CStyleCast;
+  // This case should not trigger on regular vector splat
+  // vector cast, vector truncation, or special hlsl splat cases
+  QualType SrcTy = SrcExpr.get()->getType();
+  if (Self.getLangOpts().HLSL &&
+      Self.HLSL().CanPerformElementwiseCast(SrcExpr.get(), DestType)) {
+    if (SrcTy->isConstantArrayType())
+      SrcExpr = Self.ImpCastExprToType(
+          SrcExpr.get(), Self.Context.getArrayParameterType(SrcTy),
+          CK_HLSLArrayRValue, VK_PRValue, nullptr, CCK);
+    Kind = CK_HLSLElementwiseCast;
+    return;
+  }
+
   if (ValueKind == VK_PRValue && !DestType->isRecordType() &&
       !isPlaceholder(BuiltinType::Overload)) {
     SrcExpr = Self.DefaultFunctionArrayLvalueConversion(SrcExpr.get());
@@ -2824,9 +2841,6 @@ void CastOperation::CheckCXXCStyleCast(bool FunctionalStyle,
   if (isValidCast(tcr))
     Kind = CK_NoOp;
 
-  CheckedConversionKind CCK = FunctionalStyle
-                                  ? CheckedConversionKind::FunctionalCast
-                                  : CheckedConversionKind::CStyleCast;
   if (tcr == TC_NotApplicable) {
     tcr = TryAddressSpaceCast(Self, SrcExpr, DestType, /*CStyle*/ true, msg,
                               Kind);
diff --git a/clang/lib/Sema/SemaChecking.cpp b/clang/lib/Sema/SemaChecking.cpp
index e440f60526bb3..66c233de4ef30 100644
--- a/clang/lib/Sema/SemaChecking.cpp
+++ b/clang/lib/Sema/SemaChecking.cpp
@@ -1236,7 +1236,9 @@ void Sema::checkFortifiedBuiltinMemoryFunction(FunctionDecl *FD,
   bool IsChkVariant = false;
 
   auto GetFunctionName = [&]() {
-    StringRef FunctionName = getASTContext().BuiltinInfo.getName(BuiltinID);
+    std::string FunctionNameStr =
+        getASTContext().BuiltinInfo.getName(BuiltinID);
+    llvm::StringRef FunctionName = FunctionNameStr;
     // Skim off the details of whichever builtin was called to produce a better
     // diagnostic, as it's unlikely that the user wrote the __builtin
     // explicitly.
@@ -1246,7 +1248,7 @@ void Sema::checkFortifiedBuiltinMemoryFunction(FunctionDecl *FD,
     } else {
       FunctionName.consume_front("__builtin_");
     }
-    return FunctionName;
+    return FunctionName.str();
   };
 
   switch (BuiltinID) {
@@ -1290,7 +1292,7 @@ void Sema::checkFortifiedBuiltinMemoryFunction(FunctionDecl *FD,
                         unsigned SourceSize) {
       DiagID = diag::warn_fortify_scanf_overflow;
       unsigned Index = ArgIndex + DataIndex;
-      StringRef FunctionName = GetFunctionName();
+      std::string FunctionName = GetFunctionName();
       DiagRuntimeBehavior(TheCall->getArg(Index)->getBeginLoc(), TheCall,
                           PDiag(DiagID) << FunctionName << (Index + 1)
                                         << DestSize << SourceSize);
@@ -1439,7 +1441,7 @@ void Sema::checkFortifiedBuiltinMemoryFunction(FunctionDecl *FD,
       llvm::APSInt::compareValues(*SourceSize, *DestinationSize) <= 0)
     return;
 
-  StringRef FunctionName = GetFunctionName();
+  std::string FunctionName = GetFunctionName();
 
   SmallString<16> DestinationStr;
   SmallString<16> SourceStr;
@@ -2449,7 +2451,6 @@ Sema::CheckBuiltinFunctionCall(FunctionDecl *FDecl, unsigned BuiltinID,
       CheckNonNullArgument(*this, TheCall->getArg(0), TheCall->getExprLoc());
     break;
   }
-#define BUILTIN(ID, TYPE, ATTRS)
 #define ATOMIC_BUILTIN(ID, TYPE, ATTRS)                                        \
   case Builtin::BI##ID:                                                        \
     return AtomicOpsOverloaded(TheCallResult, AtomicExpr::AO##ID);
@@ -4584,7 +4585,7 @@ ExprResult Sema::BuiltinAtomicOverloaded(ExprResult TheCallResult) {
   // Get the decl for the concrete builtin from this, we can tell what the
   // concrete integer type we should convert to is.
   unsigned NewBuiltinID = BuiltinIndices[BuiltinIndex][SizeIndex];
-  StringRef NewBuiltinName = Context.BuiltinInfo.getName(NewBuiltinID);
+  std::string NewBuiltinName = Context.BuiltinInfo.getName(NewBuiltinID);
   FunctionDecl *NewBuiltinDecl;
   if (NewBuiltinID == BuiltinID)
     NewBuiltinDecl = FDecl;
@@ -8379,7 +8380,7 @@ static void emitReplacement(Sema &S, SourceLocation Loc, SourceRange Range,
                             unsigned AbsKind, QualType ArgType) {
   bool EmitHeaderHint = true;
   const char *HeaderName = nullptr;
-  StringRef FunctionName;
+  std::string FunctionName;
   if (S.getLangOpts().CPlusPlus && !ArgType->isAnyComplexType()) {
     FunctionName = "std::abs";
     if (ArgType->isIntegralOrEnumerationType()) {
@@ -8545,7 +8546,7 @@ void Sema::CheckAbsoluteValueFunction(const CallExpr *Call,
   // Unsigned types cannot be negative.  Suggest removing the absolute value
   // function call.
   if (ArgType->isUnsignedIntegerType()) {
-    StringRef FunctionName =
+    std::string FunctionName =
         IsStdAbs ? "std::abs" : Context.BuiltinInfo.getName(AbsKind);
     Diag(Call->getExprLoc(), diag::warn_unsigned_abs) << ArgType << ParamType;
     Diag(Call->getExprLoc(), diag::note_remove_abs)
@@ -14649,11 +14650,23 @@ void Sema::CheckAddressOfPackedMember(Expr *rhs) {
                      _2, _3, _4));
 }
 
+// Performs a similar job to Sema::UsualUnaryConversions, but without any
+// implicit promotion of integral/enumeration types.
+static ExprResult BuiltinVectorMathConversions(Sema &S, Expr *E) {
+  // First, convert to an r-value.
+  ExprResult Res = S.DefaultFunctionArrayLvalueConversion(E);
+  if (Res.isInvalid())
+    return ExprError();
+
+  // Promote floating-point types.
+  return S.UsualUnaryFPConversions(Res.get());
+}
+
 bool Sema::PrepareBuiltinElementwiseMathOneArgCall(CallExpr *TheCall) {
   if (checkArgCount(TheCall, 1))
     return true;
 
-  ExprResult A = UsualUnaryConversions(TheCall->getArg(0));
+  ExprResult A = BuiltinVectorMathConversions(*this, TheCall->getArg(0));
   if (A.isInvalid())
     return true;
 
@@ -14668,57 +14681,77 @@ bool Sema::PrepareBuiltinElementwiseMathOneArgCall(CallExpr *TheCall) {
 }
 
 bool Sema::BuiltinElementwiseMath(CallExpr *TheCall, bool FPOnly) {
-  QualType Res;
-  if (BuiltinVectorMath(TheCall, Res, FPOnly))
-    return true;
-  TheCall->setType(Res);
-  return false;
+  if (auto Res = BuiltinVectorMath(TheCall, FPOnly); Res.has_value()) {
+    TheCall->setType(*Res);
+    return false;
+  }
+  return true;
 }
 
 bool Sema::BuiltinVectorToScalarMath(CallExpr *TheCall) {
-  QualType Res;
-  if (BuiltinVectorMath(TheCall, Res))
+  std::optional<QualType> Res = BuiltinVectorMath(TheCall);
+  if (!Res)
     return true;
 
-  if (auto *VecTy0 = Res->getAs<VectorType>())
+  if (auto *VecTy0 = (*Res)->getAs<VectorType>())
     TheCall->setType(VecTy0->getElementType());
   else
-    TheCall->setType(Res);
+    TheCall->setType(*Res);
 
   return false;
 }
 
-bool Sema::BuiltinVectorMath(CallExpr *TheCall, QualType &Res, bool FPOnly) {
+static bool checkBuiltinVectorMathMixedEnums(Sema &S, Expr *LHS, Expr *RHS,
+                                             SourceLocation Loc) {
+  QualType L = LHS->getEnumCoercedType(S.Context),
+           R = RHS->getEnumCoercedType(S.Context);
+  if (L->isUnscopedEnumerationType() && R->isUnscopedEnumerationType() &&
+      !S.Context.hasSameUnqualifiedType(L, R)) {
+    return S.Diag(Loc, diag::err_conv_mixed_enum_types_cxx26)
+           << LHS->getSourceRange() << RHS->getSourceRange()
+           << /*Arithmetic Between*/ 0 << L << R;
+  }
+  return false;
+}
+
+std::optional<QualType> Sema::BuiltinVectorMath(CallExpr *TheCall,
+                                                bool FPOnly) {
   if (checkArgCount(TheCall, 2))
-    return true;
+    return std::nullopt;
 
-  ExprResult A = TheCall->getArg(0);
-  ExprResult B = TheCall->getArg(1);
-  // Do standard promotions between the two arguments, returning their common
-  // type.
-  Res = UsualArithmeticConversions(A, B, TheCall->getExprLoc(), ACK_Comparison);
-  if (A.isInvalid() || B.isInvalid())
-    return true;
+  if (checkBuiltinVectorMathMixedEnums(
+          *this, TheCall->getArg(0), TheCall->getArg(1), TheCall->getExprLoc()))
+    return std::nullopt;
 
-  QualType TyA = A.get()->getType();
-  QualType TyB = B.get()->getType();
+  Expr *Args[2];
+  for (int I = 0; I < 2; ++I) {
+    ExprResult Converted =
+        BuiltinVectorMathConversions(*this, TheCall->getArg(I));
+    if (Converted.isInvalid())
+      return std::nullopt;
+    Args[I] = Converted.get();
+  }
 
-  if (Res.isNull() || TyA.getCanonicalType() != TyB.getCanonicalType())
-    return Diag(A.get()->getBeginLoc(),
-                diag::err_typecheck_call_different_arg_types)
-           << TyA << TyB;
+  SourceLocation LocA = Args[0]->getBeginLoc();
+  QualType TyA = Args[0]->getType();
+  QualType TyB = Args[1]->getType();
+
+  if (TyA.getCanonicalType() != TyB.getCanonicalType()) {
+    Diag(LocA, diag::err_typecheck_call_different_arg_types) << TyA << TyB;
+    return std::nullopt;
+  }
 
   if (FPOnly) {
-    if (checkFPMathBuiltinElementType(*this, A.get()->getBeginLoc(), TyA, 1))
-      return true;
+    if (checkFPMathBuiltinElementType(*this, LocA, TyA, 1))
+      return std::nullopt;
   } else {
-    if (checkMathBuiltinElementType(*this, A.get()->getBeginLoc(), TyA, 1))
-      return true;
+    if (checkMathBuiltinElementType(*this, LocA, TyA, 1))
+      return std::nullopt;
   }
 
-  TheCall->setArg(0, A.get());
-  TheCall->setArg(1, B.get());
-  return false;
+  TheCall->setArg(0, Args[0]);
+  TheCall->setArg(1, Args[1]);
+  return TyA;
 }
 
 bool Sema::BuiltinElementwiseTernaryMath(CallExpr *TheCall,
@@ -14726,9 +14759,17 @@ bool Sema::BuiltinElementwiseTernaryMath(CallExpr *TheCall,
   if (checkArgCount(TheCall, 3))
     return true;
 
+  SourceLocation Loc = TheCall->getExprLoc();
+  if (checkBuiltinVectorMathMixedEnums(*this, TheCall->getArg(0),
+                                       TheCall->getArg(1), Loc) ||
+      checkBuiltinVectorMathMixedEnums(*this, TheCall->getArg(1),
+                                       TheCall->getArg(2), Loc))
+    return true;
+
   Expr *Args[3];
   for (int I = 0; I < 3; ++I) {
-    ExprResult Converted = UsualUnaryConversions(TheCall->getArg(I));
+    ExprResult Converted =
+        BuiltinVectorMathConversions(*this, TheCall->getArg(I));
     if (Converted.isInvalid())
       return true;
     Args[I] = Converted.get();
diff --git a/clang/lib/Sema/SemaCodeComplete.cpp b/clang/lib/Sema/SemaCodeComplete.cpp
index 58f3efbe0daf8..80ae87e7c5725 100644
--- a/clang/lib/Sema/SemaCodeComplete.cpp
+++ b/clang/lib/Sema/SemaCodeComplete.cpp
@@ -137,7 +137,7 @@ class ResultBuilder {
 
     ~ShadowMapEntry() {
       if (DeclIndexPairVector *Vec =
-              DeclOrVector.dyn_cast<DeclIndexPairVector *>()) {
+              dyn_cast_if_present<DeclIndexPairVector *>(DeclOrVector)) {
         delete Vec;
         DeclOrVector = ((NamedDecl *)nullptr);
       }
@@ -678,7 +678,7 @@ class ResultBuilder::ShadowMapEntry::iterator {
   }*/
 
   reference operator*() const {
-    if (const NamedDecl *ND = DeclOrIterator.dyn_cast<const NamedDecl *>())
+    if (const NamedDecl *ND = dyn_cast<const NamedDecl *>(DeclOrIterator))
       return reference(ND, SingleDeclIndex);
 
     return *cast<const DeclIndexPair *>(DeclOrIterator);
@@ -5796,24 +5796,11 @@ QualType getApproximateType(const Expr *E, HeuristicResolver &Resolver) {
         return QualType(Common, 0);
     }
   }
-  // A dependent member: approximate-resolve the base, then lookup.
+  // A dependent member: resolve using HeuristicResolver.
   if (const auto *CDSME = llvm::dyn_cast<CXXDependentScopeMemberExpr>(E)) {
-    QualType Base = CDSME->isImplicitAccess()
-                        ? CDSME->getBaseType()
-                        : getApproximateType(CDSME->getBase(), Resolver);
-    if (CDSME->isArrow() && !Base.isNull())
-      Base = Base->getPointeeType(); // could handle unique_ptr etc here?
-    auto *RD =
-        Base.isNull()
-            ? nullptr
-            : llvm::dyn_cast_or_null<CXXRecordDecl>(getAsRecordDecl(Base));
-    if (RD && RD->isCompleteDefinition()) {
-      // Look up member heuristically, including in bases.
-      for (const auto *Member : RD->lookupDependentName(
-               CDSME->getMember(), [](const NamedDecl *Member) {
-                 return llvm::isa<ValueDecl>(Member);
-               })) {
-        return llvm::cast<ValueDecl>(Member)->getType().getNonReferenceType();
+    for (const auto *Member : Resolver.resolveMemberExpr(CDSME)) {
+      if (const auto *VD = dyn_cast<ValueDecl>(Member)) {
+        return VD->getType().getNonReferenceType();
       }
     }
   }
diff --git a/clang/lib/Sema/SemaDecl.cpp b/clang/lib/Sema/SemaDecl.cpp
index ad49eac66e98e..6eedc77ed20a0 100644
--- a/clang/lib/Sema/SemaDecl.cpp
+++ b/clang/lib/Sema/SemaDecl.cpp
@@ -6681,7 +6681,10 @@ Sema::ActOnTypedefDeclarator(Scope* S, Declarator& D, DeclContext* DC,
   DiagnoseFunctionSpecifiers(D.getDeclSpec());
 
   if (D.getDeclSpec().isInlineSpecified())
-    Diag(D.getDeclSpec().getInlineSpecLoc(), diag::err_inline_non_function)
+    Diag(D.getDeclSpec().getInlineSpecLoc(),
+         (getLangOpts().MSVCCompat && !getLangOpts().CPlusPlus)
+             ? diag::warn_ms_inline_non_function
+             : diag::err_inline_non_function)
         << getLangOpts().CPlusPlus17;
   if (D.getDeclSpec().hasConstexprSpecifier())
     Diag(D.getDeclSpec().getConstexprSpecLoc(), diag::err_invalid_constexpr)
@@ -11380,8 +11383,11 @@ static bool CheckMultiVersionAdditionalRules(Sema &S, const FunctionDecl *OldFD,
     return true;
 
   // Only allow transition to MultiVersion if it hasn't been used.
-  if (OldFD && CausesMV && OldFD->isUsed(false))
-    return S.Diag(NewFD->getLocation(), diag::err_multiversion_after_used);
+  if (OldFD && CausesMV && OldFD->isUsed(false)) {
+    S.Diag(NewFD->getLocation(), diag::err_multiversion_after_used);
+    S.Diag(OldFD->getLocation(), diag::note_previous_declaration);
+    return true;
+  }
 
   return S.areMultiversionVariantFunctionsCompatible(
       OldFD, NewFD, S.PDiag(diag::err_multiversion_noproto),
@@ -13374,6 +13380,62 @@ void Sema::checkNonTrivialCUnion(QualType QT, SourceLocation Loc,
         .visit(QT, nullptr, false);
 }
 
+bool Sema::GloballyUniqueObjectMightBeAccidentallyDuplicated(
+    const VarDecl *Dcl) {
+  if (!getLangOpts().CPlusPlus)
+    return false;
+
+  // We only need to warn if the definition is in a header file, so wait to
+  // diagnose until we've seen the definition.
+  if (!Dcl->isThisDeclarationADefinition())
+    return false;
+
+  // If an object is defined in a source file, its definition can't get
+  // duplicated since it will never appear in more than one TU.
+  if (Dcl->getASTContext().getSourceManager().isInMainFile(Dcl->getLocation()))
+    return false;
+
+  // If the variable we're looking at is a static local, then we actually care
+  // about the properties of the function containing it.
+  const ValueDecl *Target = Dcl;
+  // VarDecls and FunctionDecls have different functions for checking
+  // inline-ness, so we have to do it manually.
+  bool TargetIsInline = Dcl->isInline();
+
+  // Update the Target and TargetIsInline property if necessary
+  if (Dcl->isStaticLocal()) {
+    const DeclContext *Ctx = Dcl->getDeclContext();
+    if (!Ctx)
+      return false;
+
+    const FunctionDecl *FunDcl =
+        dyn_cast_if_present<FunctionDecl>(Ctx->getNonClosureAncestor());
+    if (!FunDcl)
+      return false;
+
+    Target = FunDcl;
+    // IsInlined() checks for the C++ inline property
+    TargetIsInline = FunDcl->isInlined();
+  }
+
+  // Non-inline variables can only legally appear in one TU
+  // FIXME: This also applies to templated variables, but that can rarely lead
+  // to false positives so templates are disabled for now.
+  if (!TargetIsInline)
+    return false;
+
+  // If the object isn't hidden, the dynamic linker will prevent duplication.
+  clang::LinkageInfo Lnk = Target->getLinkageAndVisibility();
+  if (Lnk.getVisibility() != HiddenVisibility)
+    return false;
+
+  // If the obj doesn't have external linkage, it's supposed to be duplicated.
+  if (!isExternalFormalLinkage(Lnk.getLinkage()))
+    return false;
+
+  return true;
+}
+
 void Sema::AddInitializerToDecl(Decl *RealDecl, Expr *Init, bool DirectInit) {
   // If there is no declaration, there was an error parsing it.  Just ignore
   // the initializer.
@@ -14780,6 +14842,51 @@ void Sema::FinalizeDeclaration(Decl *ThisDecl) {
   if (DC->getRedeclContext()->isFileContext() && VD->isExternallyVisible())
     AddPushedVisibilityAttribute(VD);
 
+  // If this object has external linkage and hidden visibility, it might be
+  // duplicated when built into a shared library, which causes problems if it's
+  // mutable (since the copies won't be in sync) or its initialization has side
+  // effects (since it will run once per copy instead of once globally)
+  // FIXME: Windows uses dllexport/dllimport instead of visibility, and we don't
+  // handle that yet. Disable the warning on Windows for now.
+  // FIXME: Checking templates can cause false positives if the template in
+  // question is never instantiated (e.g. only specialized templates are used).
+  if (!Context.getTargetInfo().shouldDLLImportComdatSymbols() &&
+      !VD->isTemplated() &&
+      GloballyUniqueObjectMightBeAccidentallyDuplicated(VD)) {
+    // Check mutability. For pointers, ensure that both the pointer and the
+    // pointee are (recursively) const.
+    QualType Type = VD->getType().getNonReferenceType();
+    if (!Type.isConstant(VD->getASTContext())) {
+      Diag(VD->getLocation(), diag::warn_possible_object_duplication_mutable)
+          << VD;
+    } else {
+      while (Type->isPointerType()) {
+        Type = Type->getPointeeType();
+        if (Type->isFunctionType())
+          break;
+        if (!Type.isConstant(VD->getASTContext())) {
+          Diag(VD->getLocation(),
+               diag::warn_possible_object_duplication_mutable)
+              << VD;
+          break;
+        }
+      }
+    }
+
+    // To keep false positives low, only warn if we're certain that the
+    // initializer has side effects. Don't warn on operator new, since a mutable
+    // pointer will trigger the previous warning, and an immutable pointer
+    // getting duplicated just results in a little extra memory usage.
+    const Expr *Init = VD->getAnyInitializer();
+    if (Init &&
+        Init->HasSideEffects(VD->getASTContext(),
+                             /*IncludePossibleEffects=*/false) &&
+        !isa<CXXNewExpr>(Init->IgnoreParenImpCasts())) {
+      Diag(Init->getExprLoc(), diag::warn_possible_object_duplication_init)
+          << VD;
+    }
+  }
+
   // FIXME: Warn on unused var template partial specializations.
   if (VD->isFileVarDecl() && !isa<VarTemplatePartialSpecializationDecl>(VD))
     MarkUnusedFileScopedDecl(VD);
@@ -16016,7 +16123,6 @@ Decl *Sema::ActOnFinishFunctionBody(Decl *dcl, Stmt *Body,
       if (!FD->isDeletedAsWritten())
         FD->setBody(Body);
       FD->setWillHaveBody(false);
-      CheckImmediateEscalatingFunctionDefinition(FD, FSI);
 
       if (getLangOpts().CPlusPlus14) {
         if (!FD->isInvalidDecl() && Body && !FD->isDependentContext() &&
@@ -16394,6 +16500,9 @@ Decl *Sema::ActOnFinishFunctionBody(Decl *dcl, Stmt *Body,
     // the declaration context below. Otherwise, we're unable to transform
     // 'this' expressions when transforming immediate context functions.
 
+  if (FD)
+    CheckImmediateEscalatingFunctionDefinition(FD, getCurFunction());
+
   if (!IsInstantiation)
     PopDeclContext();
 
@@ -17700,9 +17809,11 @@ Sema::ActOnTag(Scope *S, unsigned TagSpec, TagUseKind TUK, SourceLocation KWLoc,
             return PrevTagDecl;
 
           QualType EnumUnderlyingTy;
-          if (TypeSourceInfo *TI = EnumUnderlying.dyn_cast<TypeSourceInfo*>())
+          if (TypeSourceInfo *TI =
+                  dyn_cast_if_present<TypeSourceInfo *>(EnumUnderlying))
             EnumUnderlyingTy = TI->getType().getUnqualifiedType();
-          else if (const Type *T = EnumUnderlying.dyn_cast<const Type*>())
+          else if (const Type *T =
+                       dyn_cast_if_present<const Type *>(EnumUnderlying))
             EnumUnderlyingTy = QualType(T, 0);
 
           // All conflicts with previous declarations are recovered by
@@ -19522,23 +19633,6 @@ void Sema::ActOnFields(Scope *S, SourceLocation RecLoc, Decl *EnclosingDecl,
   ProcessAPINotes(Record);
 }
 
-/// Determine whether the given integral value is representable within
-/// the given type T.
-static bool isRepresentableIntegerValue(ASTContext &Context,
-                                        llvm::APSInt &Value,
-                                        QualType T) {
-  assert((T->isIntegralType(Context) || T->isEnumeralType()) &&
-         "Integral type required!");
-  unsigned BitWidth = Context.getIntWidth(T);
-
-  if (Value.isUnsigned() || Value.isNonNegative()) {
-    if (T->isSignedIntegerOrEnumerationType())
-      --BitWidth;
-    return Value.getActiveBits() <= BitWidth;
-  }
-  return Value.getSignificantBits() <= BitWidth;
-}
-
 // Given an integral type, return the next larger integral type
 // (or a NULL type of no such type exists).
 static QualType getNextLargerIntegralType(ASTContext &Context, QualType T) {
@@ -19612,7 +19706,7 @@ EnumConstantDecl *Sema::CheckEnumConstant(EnumDecl *Enum,
           // representable in the underlying type of the enumeration. In C++11,
           // we perform a non-narrowing conversion as part of converted constant
           // expression checking.
-          if (!isRepresentableIntegerValue(Context, EnumVal, EltTy)) {
+          if (!Context.isRepresentableIntegerValue(EnumVal, EltTy)) {
             if (Context.getTargetInfo()
                     .getTriple()
                     .isWindowsMSVCEnvironment()) {
@@ -19641,7 +19735,7 @@ EnumConstantDecl *Sema::CheckEnumConstant(EnumDecl *Enum,
           //   representable as an int.
 
           // Complain if the value is not representable in an int.
-          if (!isRepresentableIntegerValue(Context, EnumVal, Context.IntTy)) {
+          if (!Context.isRepresentableIntegerValue(EnumVal, Context.IntTy)) {
             Diag(IdLoc, getLangOpts().C23
                             ? diag::warn_c17_compat_enum_value_not_int
                             : diag::ext_c23_enum_value_not_int)
@@ -19733,7 +19827,7 @@ EnumConstantDecl *Sema::CheckEnumConstant(EnumDecl *Enum,
                           : diag::ext_c23_enum_value_not_int)
               << 1 << toString(EnumVal, 10) << 1;
       } else if (!getLangOpts().CPlusPlus && !EltTy->isDependentType() &&
-                 !isRepresentableIntegerValue(Context, EnumVal, EltTy)) {
+                 !Context.isRepresentableIntegerValue(EnumVal, EltTy)) {
         // Enforce C99 6.7.2.2p2 even when we compute the next value.
         Diag(IdLoc, getLangOpts().C23 ? diag::warn_c17_compat_enum_value_not_int
                                       : diag::ext_c23_enum_value_not_int)
@@ -19959,7 +20053,7 @@ static void CheckForDuplicateEnumValues(Sema &S, ArrayRef<Decl *> Elements,
       continue;
 
     DeclOrVector& Entry = Iter->second;
-    if (EnumConstantDecl *D = Entry.dyn_cast<EnumConstantDecl*>()) {
+    if (EnumConstantDecl *D = dyn_cast<EnumConstantDecl *>(Entry)) {
       // Ensure constants are different.
       if (D == ECD)
         continue;
@@ -20060,35 +20154,8 @@ void Sema::ActOnEnumBody(SourceLocation EnumLoc, SourceRange BraceRange,
   // reverse the list.
   unsigned NumNegativeBits = 0;
   unsigned NumPositiveBits = 0;
-  bool MembersRepresentableByInt = true;
-
-  for (unsigned i = 0, e = Elements.size(); i != e; ++i) {
-    EnumConstantDecl *ECD =
-      cast_or_null<EnumConstantDecl>(Elements[i]);
-    if (!ECD) continue;  // Already issued a diagnostic.
-
-    llvm::APSInt InitVal = ECD->getInitVal();
-
-    // Keep track of the size of positive and negative values.
-    if (InitVal.isUnsigned() || InitVal.isNonNegative()) {
-      // If the enumerator is zero that should still be counted as a positive
-      // bit since we need a bit to store the value zero.
-      unsigned ActiveBits = InitVal.getActiveBits();
-      NumPositiveBits = std::max({NumPositiveBits, ActiveBits, 1u});
-    } else {
-      NumNegativeBits =
-          std::max(NumNegativeBits, (unsigned)InitVal.getSignificantBits());
-    }
-    MembersRepresentableByInt &=
-        isRepresentableIntegerValue(Context, InitVal, Context.IntTy);
-  }
-
-  // If we have an empty set of enumerators we still need one bit.
-  // From [dcl.enum]p8
-  // If the enumerator-list is empty, the values of the enumeration are as if
-  // the enumeration had a single enumerator with value 0
-  if (!NumPositiveBits && !NumNegativeBits)
-    NumPositiveBits = 1;
+  bool MembersRepresentableByInt =
+      Context.computeEnumBits(Elements, NumNegativeBits, NumPositiveBits);
 
   // Figure out the type that should be used for this enum.
   QualType BestType;
diff --git a/clang/lib/Sema/SemaDeclAttr.cpp b/clang/lib/Sema/SemaDeclAttr.cpp
index c2d82b9aa9b32..527db176cf8dd 100644
--- a/clang/lib/Sema/SemaDeclAttr.cpp
+++ b/clang/lib/Sema/SemaDeclAttr.cpp
@@ -879,22 +879,38 @@ static void handleDiagnoseIfAttr(Sema &S, Decl *D, const ParsedAttr &AL) {
   if (!checkFunctionConditionAttr(S, D, AL, Cond, Msg))
     return;
 
-  StringRef DiagTypeStr;
-  if (!S.checkStringLiteralArgumentAttr(AL, 2, DiagTypeStr))
+  StringRef DefaultSevStr;
+  if (!S.checkStringLiteralArgumentAttr(AL, 2, DefaultSevStr))
     return;
 
-  DiagnoseIfAttr::DiagnosticType DiagType;
-  if (!DiagnoseIfAttr::ConvertStrToDiagnosticType(DiagTypeStr, DiagType)) {
+  DiagnoseIfAttr::DefaultSeverity DefaultSev;
+  if (!DiagnoseIfAttr::ConvertStrToDefaultSeverity(DefaultSevStr, DefaultSev)) {
     S.Diag(AL.getArgAsExpr(2)->getBeginLoc(),
            diag::err_diagnose_if_invalid_diagnostic_type);
     return;
   }
 
+  StringRef WarningGroup;
+  SmallVector<StringRef, 2> Options;
+  if (AL.getNumArgs() > 3) {
+    if (!S.checkStringLiteralArgumentAttr(AL, 3, WarningGroup))
+      return;
+    if (WarningGroup.empty() ||
+        !S.getDiagnostics().getDiagnosticIDs()->getGroupForWarningOption(
+            WarningGroup)) {
+      S.Diag(AL.getArgAsExpr(3)->getBeginLoc(),
+             diag::err_diagnose_if_unknown_warning)
+          << WarningGroup;
+      return;
+    }
+  }
+
   bool ArgDependent = false;
   if (const auto *FD = dyn_cast<FunctionDecl>(D))
     ArgDependent = ArgumentDependenceChecker(FD).referencesArgs(Cond);
   D->addAttr(::new (S.Context) DiagnoseIfAttr(
-      S.Context, AL, Cond, Msg, DiagType, ArgDependent, cast<NamedDecl>(D)));
+      S.Context, AL, Cond, Msg, DefaultSev, WarningGroup, ArgDependent,
+      cast<NamedDecl>(D)));
 }
 
 static void handleNoBuiltinAttr(Sema &S, Decl *D, const ParsedAttr &AL) {
@@ -3057,7 +3073,8 @@ bool Sema::checkTargetAttr(SourceLocation LiteralLoc, StringRef AttrStr) {
   if (ParsedAttrs.BranchProtection.empty())
     return false;
   if (!Context.getTargetInfo().validateBranchProtection(
-          ParsedAttrs.BranchProtection, ParsedAttrs.CPU, BPI, DiagMsg)) {
+          ParsedAttrs.BranchProtection, ParsedAttrs.CPU, BPI,
+          Context.getLangOpts(), DiagMsg)) {
     if (DiagMsg.empty())
       return Diag(LiteralLoc, diag::warn_unsupported_target_attribute)
              << Unsupported << None << "branch-protection" << Target;
@@ -7467,6 +7484,15 @@ void Sema::ProcessDeclAttributeList(
     }
   }
 
+  // Do not permit 'constructor' or 'destructor' attributes on __device__ code.
+  if (getLangOpts().CUDAIsDevice && D->hasAttr<CUDADeviceAttr>() &&
+      (D->hasAttr<ConstructorAttr>() || D->hasAttr<DestructorAttr>()) &&
+      !getLangOpts().GPUAllowDeviceInit) {
+    Diag(D->getLocation(), diag::err_cuda_ctor_dtor_attrs)
+        << (D->hasAttr<ConstructorAttr>() ? "constructors" : "destructors");
+    D->setInvalidDecl();
+  }
+
   // Do this check after processing D's attributes because the attribute
   // objc_method_family can change whether the given method is in the init
   // family, and it can be applied after objc_designated_initializer. This is a
diff --git a/clang/lib/Sema/SemaDeclCXX.cpp b/clang/lib/Sema/SemaDeclCXX.cpp
index 839b3a1cccdcc..0cf02fe6407c2 100644
--- a/clang/lib/Sema/SemaDeclCXX.cpp
+++ b/clang/lib/Sema/SemaDeclCXX.cpp
@@ -888,7 +888,15 @@ Sema::ActOnDecompositionDeclarator(Scope *S, Declarator &D,
       Previous.clear();
     }
 
-    auto *BD = BindingDecl::Create(Context, DC, B.NameLoc, VarName);
+    QualType QT;
+    if (B.EllipsisLoc.isValid()) {
+      if (!cast<Decl>(DC)->isTemplated())
+        Diag(B.EllipsisLoc, diag::err_pack_outside_template);
+      QT = Context.getPackExpansionType(Context.DependentTy, std::nullopt,
+                                        /*ExpectsPackInType=*/false);
+    }
+
+    auto *BD = BindingDecl::Create(Context, DC, B.NameLoc, B.Name, QT);
 
     ProcessDeclAttributeList(S, BD, *B.Attrs);
 
@@ -951,20 +959,68 @@ Sema::ActOnDecompositionDeclarator(Scope *S, Declarator &D,
   return New;
 }
 
+// Check the arity of the structured bindings.
+// Create the resolved pack expr if needed.
+static bool CheckBindingsCount(Sema &S, DecompositionDecl *DD,
+                               QualType DecompType,
+                               ArrayRef<BindingDecl *> Bindings,
+                               unsigned MemberCount) {
+  auto BindingWithPackItr =
+      std::find_if(Bindings.begin(), Bindings.end(),
+                   [](BindingDecl *D) -> bool { return D->isParameterPack(); });
+  bool HasPack = BindingWithPackItr != Bindings.end();
+  bool IsValid;
+  if (!HasPack) {
+    IsValid = Bindings.size() == MemberCount;
+  } else {
+    // There may not be more members than non-pack bindings.
+    IsValid = MemberCount >= Bindings.size() - 1;
+  }
+
+  if (IsValid && HasPack) {
+    // Create the pack expr and assign it to the binding.
+    unsigned PackSize = MemberCount - Bindings.size() + 1;
+    QualType PackType = S.Context.getPackExpansionType(
+        S.Context.DependentTy, std::nullopt, /*ExpectsPackInType=*/false);
+    BindingDecl *BD = (*BindingWithPackItr);
+    auto *RP = ResolvedUnexpandedPackExpr::Create(S.Context, DD->getBeginLoc(),
+                                                  DecompType, PackSize);
+    BD->setDecomposedDecl(DD);
+    BD->setBinding(PackType, RP);
+
+    BindingDecl *BPack = *BindingWithPackItr;
+    // Create the nested BindingDecls.
+    for (Expr *&E : RP->getExprs()) {
+      auto *NestedBD = BindingDecl::Create(S.Context, BPack->getDeclContext(),
+                                           BPack->getLocation(),
+                                           BPack->getIdentifier(), QualType());
+      NestedBD->setDecomposedDecl(DD);
+      E = S.BuildDeclRefExpr(NestedBD, S.Context.DependentTy, VK_LValue,
+                             BPack->getLocation());
+    }
+  }
+
+  if (IsValid)
+    return false;
+
+  S.Diag(DD->getLocation(), diag::err_decomp_decl_wrong_number_bindings)
+      << DecompType << (unsigned)Bindings.size() << MemberCount << MemberCount
+      << (MemberCount < Bindings.size());
+  return true;
+}
+
 static bool checkSimpleDecomposition(
     Sema &S, ArrayRef<BindingDecl *> Bindings, ValueDecl *Src,
-    QualType DecompType, const llvm::APSInt &NumElems, QualType ElemType,
+    QualType DecompType, const llvm::APSInt &NumElemsAPS, QualType ElemType,
     llvm::function_ref<ExprResult(SourceLocation, Expr *, unsigned)> GetInit) {
-  if ((int64_t)Bindings.size() != NumElems) {
-    S.Diag(Src->getLocation(), diag::err_decomp_decl_wrong_number_bindings)
-        << DecompType << (unsigned)Bindings.size()
-        << (unsigned)NumElems.getLimitedValue(UINT_MAX)
-        << toString(NumElems, 10) << (NumElems < Bindings.size());
+  unsigned NumElems = (unsigned)NumElemsAPS.getLimitedValue(UINT_MAX);
+  auto *DD = cast<DecompositionDecl>(Src);
+
+  if (CheckBindingsCount(S, DD, DecompType, Bindings, NumElems))
     return true;
-  }
 
   unsigned I = 0;
-  for (auto *B : Bindings) {
+  for (auto *B : DD->flat_bindings()) {
     SourceLocation Loc = B->getLocation();
     ExprResult E = S.BuildDeclRefExpr(Src, DecompType, VK_LValue, Loc);
     if (E.isInvalid())
@@ -1210,13 +1266,10 @@ static bool checkTupleLikeDecomposition(Sema &S,
                                         ArrayRef<BindingDecl *> Bindings,
                                         VarDecl *Src, QualType DecompType,
                                         const llvm::APSInt &TupleSize) {
-  if ((int64_t)Bindings.size() != TupleSize) {
-    S.Diag(Src->getLocation(), diag::err_decomp_decl_wrong_number_bindings)
-        << DecompType << (unsigned)Bindings.size()
-        << (unsigned)TupleSize.getLimitedValue(UINT_MAX)
-        << toString(TupleSize, 10) << (TupleSize < Bindings.size());
+  auto *DD = cast<DecompositionDecl>(Src);
+  unsigned NumElems = (unsigned)TupleSize.getLimitedValue(UINT_MAX);
+  if (CheckBindingsCount(S, DD, DecompType, Bindings, NumElems))
     return true;
-  }
 
   if (Bindings.empty())
     return false;
@@ -1250,7 +1303,7 @@ static bool checkTupleLikeDecomposition(Sema &S,
   }
 
   unsigned I = 0;
-  for (auto *B : Bindings) {
+  for (auto *B : DD->flat_bindings()) {
     InitializingBinding InitContext(S, B);
     SourceLocation Loc = B->getLocation();
 
@@ -1433,20 +1486,18 @@ static bool checkMemberDecomposition(Sema &S, ArrayRef<BindingDecl*> Bindings,
   QualType BaseType = S.Context.getQualifiedType(S.Context.getRecordType(RD),
                                                  DecompType.getQualifiers());
 
-  auto DiagnoseBadNumberOfBindings = [&]() -> bool {
-    unsigned NumFields = llvm::count_if(
-        RD->fields(), [](FieldDecl *FD) { return !FD->isUnnamedBitField(); });
-    assert(Bindings.size() != NumFields);
-    S.Diag(Src->getLocation(), diag::err_decomp_decl_wrong_number_bindings)
-        << DecompType << (unsigned)Bindings.size() << NumFields << NumFields
-        << (NumFields < Bindings.size());
+  auto *DD = cast<DecompositionDecl>(Src);
+  unsigned NumFields = llvm::count_if(
+      RD->fields(), [](FieldDecl *FD) { return !FD->isUnnamedBitField(); });
+  if (CheckBindingsCount(S, DD, DecompType, Bindings, NumFields))
     return true;
-  };
 
   //   all of E's non-static data members shall be [...] well-formed
   //   when named as e.name in the context of the structured binding,
   //   E shall not have an anonymous union member, ...
-  unsigned I = 0;
+  auto FlatBindings = DD->flat_bindings();
+  assert(llvm::range_size(FlatBindings) == NumFields);
+  auto FlatBindingsItr = FlatBindings.begin();
   for (auto *FD : RD->fields()) {
     if (FD->isUnnamedBitField())
       continue;
@@ -1471,9 +1522,8 @@ static bool checkMemberDecomposition(Sema &S, ArrayRef<BindingDecl*> Bindings,
     }
 
     // We have a real field to bind.
-    if (I >= Bindings.size())
-      return DiagnoseBadNumberOfBindings();
-    auto *B = Bindings[I++];
+    assert(FlatBindingsItr != FlatBindings.end());
+    BindingDecl *B = *(FlatBindingsItr++);
     SourceLocation Loc = B->getLocation();
 
     // The field must be accessible in the context of the structured binding.
@@ -1511,9 +1561,6 @@ static bool checkMemberDecomposition(Sema &S, ArrayRef<BindingDecl*> Bindings,
     B->setBinding(S.BuildQualifiedType(FD->getType(), Loc, Q), E.get());
   }
 
-  if (I != Bindings.size())
-    return DiagnoseBadNumberOfBindings();
-
   return false;
 }
 
@@ -1523,8 +1570,12 @@ void Sema::CheckCompleteDecompositionDeclaration(DecompositionDecl *DD) {
   // If the type of the decomposition is dependent, then so is the type of
   // each binding.
   if (DecompType->isDependentType()) {
-    for (auto *B : DD->bindings())
-      B->setType(Context.DependentTy);
+    // Note that all of the types are still Null or PackExpansionType.
+    for (auto *B : DD->bindings()) {
+      // Do not overwrite any pack type.
+      if (B->getType().isNull())
+        B->setType(Context.DependentTy);
+    }
     return;
   }
 
@@ -3316,6 +3367,29 @@ void Sema::CheckShadowInheritedFields(const SourceLocation &Loc,
   }
 }
 
+template <typename AttrType>
+inline static bool HasAttribute(const QualType &T) {
+  if (const TagDecl *TD = T->getAsTagDecl())
+    return TD->hasAttr<AttrType>();
+  if (const TypedefType *TDT = T->getAs<TypedefType>())
+    return TDT->getDecl()->hasAttr<AttrType>();
+  return false;
+}
+
+static bool IsUnusedPrivateField(const FieldDecl *FD) {
+  if (FD->getAccess() == AS_private && FD->getDeclName()) {
+    QualType FieldType = FD->getType();
+    if (HasAttribute<WarnUnusedAttr>(FieldType))
+      return true;
+
+    return !FD->isImplicit() && !FD->hasAttr<UnusedAttr>() &&
+           !FD->getParent()->isDependentContext() &&
+           !HasAttribute<UnusedAttr>(FieldType) &&
+           !InitializationHasSideEffects(*FD);
+  }
+  return false;
+}
+
 NamedDecl *
 Sema::ActOnCXXMemberDeclarator(Scope *S, AccessSpecifier AS, Declarator &D,
                                MultiTemplateParamsArg TemplateParameterLists,
@@ -3598,25 +3672,11 @@ Sema::ActOnCXXMemberDeclarator(Scope *S, AccessSpecifier AS, Declarator &D,
     FieldDecl *FD = cast<FieldDecl>(Member);
     FieldCollector->Add(FD);
 
-    if (!Diags.isIgnored(diag::warn_unused_private_field, FD->getLocation())) {
+    if (!Diags.isIgnored(diag::warn_unused_private_field, FD->getLocation()) &&
+        IsUnusedPrivateField(FD)) {
       // Remember all explicit private FieldDecls that have a name, no side
       // effects and are not part of a dependent type declaration.
-
-      auto DeclHasUnusedAttr = [](const QualType &T) {
-        if (const TagDecl *TD = T->getAsTagDecl())
-          return TD->hasAttr<UnusedAttr>();
-        if (const TypedefType *TDT = T->getAs<TypedefType>())
-          return TDT->getDecl()->hasAttr<UnusedAttr>();
-        return false;
-      };
-
-      if (!FD->isImplicit() && FD->getDeclName() &&
-          FD->getAccess() == AS_private &&
-          !FD->hasAttr<UnusedAttr>() &&
-          !FD->getParent()->isDependentContext() &&
-          !DeclHasUnusedAttr(FD->getType()) &&
-          !InitializationHasSideEffects(*FD))
-        UnusedPrivateFields.insert(FD);
+      UnusedPrivateFields.insert(FD);
     }
   }
 
@@ -9235,7 +9295,7 @@ struct SpecialMemberVisitor {
   static SourceLocation getSubobjectLoc(Subobject Subobj) {
     // FIXME: For an indirect virtual base, the direct base leading to
     // the indirect virtual base would be a more useful choice.
-    if (auto *B = Subobj.dyn_cast<CXXBaseSpecifier*>())
+    if (auto *B = dyn_cast<CXXBaseSpecifier *>(Subobj))
       return B->getBaseTypeLoc();
     else
       return cast<FieldDecl *>(Subobj)->getLocation();
@@ -13406,8 +13466,6 @@ Decl *Sema::ActOnAliasDeclaration(Scope *S, AccessSpecifier AS,
                                   SourceLocation UsingLoc, UnqualifiedId &Name,
                                   const ParsedAttributesView &AttrList,
                                   TypeResult Type, Decl *DeclFromDeclSpec) {
-  // Get the innermost enclosing declaration scope.
-  S = S->getDeclParent();
 
   if (Type.isInvalid())
     return nullptr;
@@ -13458,6 +13516,9 @@ Decl *Sema::ActOnAliasDeclaration(Scope *S, AccessSpecifier AS,
   CheckTypedefForVariablyModifiedType(S, NewTD);
   Invalid |= NewTD->isInvalidDecl();
 
+  // Get the innermost enclosing declaration scope.
+  S = S->getDeclParent();
+
   bool Redeclaration = false;
 
   NamedDecl *NewND;
@@ -13854,8 +13915,7 @@ void Sema::setupImplicitSpecialMemberType(CXXMethodDecl *SpecialMem,
   // During template instantiation of implicit special member functions we need
   // a reliable TypeSourceInfo for the function prototype in order to allow
   // functions to be substituted.
-  if (inTemplateInstantiation() &&
-      cast<CXXRecordDecl>(SpecialMem->getParent())->isLambda()) {
+  if (inTemplateInstantiation() && isLambdaMethod(SpecialMem)) {
     TypeSourceInfo *TSI =
         Context.getTrivialTypeSourceInfo(SpecialMem->getType());
     SpecialMem->setTypeSourceInfo(TSI);
@@ -17524,7 +17584,7 @@ DeclResult Sema::ActOnTemplatedFriendTag(
   unsigned FriendDeclDepth = TempParamLists.front()->getDepth();
   for (UnexpandedParameterPack &U : Unexpanded) {
     if (getDepthAndIndex(U).first >= FriendDeclDepth) {
-      auto *ND = U.first.dyn_cast<NamedDecl *>();
+      auto *ND = dyn_cast<NamedDecl *>(U.first);
       if (!ND)
         ND = cast<const TemplateTypeParmType *>(U.first)->getDecl();
       Diag(U.second, diag::friend_template_decl_malformed_pack_expansion)
diff --git a/clang/lib/Sema/SemaDeclObjC.cpp b/clang/lib/Sema/SemaDeclObjC.cpp
index f97f17e8c9658..e665d0293dc84 100644
--- a/clang/lib/Sema/SemaDeclObjC.cpp
+++ b/clang/lib/Sema/SemaDeclObjC.cpp
@@ -1584,7 +1584,7 @@ void SemaObjC::actOnObjCTypeArgsOrProtocolQualifiers(
     const char* prevSpec; // unused
     unsigned diagID; // unused
     QualType type;
-    if (auto *actualTypeDecl = typeDecl.dyn_cast<TypeDecl *>())
+    if (auto *actualTypeDecl = dyn_cast<TypeDecl *>(typeDecl))
       type = Context.getTypeDeclType(actualTypeDecl);
     else
       type = Context.getObjCInterfaceType(cast<ObjCInterfaceDecl *>(typeDecl));
diff --git a/clang/lib/Sema/SemaExceptionSpec.cpp b/clang/lib/Sema/SemaExceptionSpec.cpp
index 470d0d753b558..77a1bbcc74e50 100644
--- a/clang/lib/Sema/SemaExceptionSpec.cpp
+++ b/clang/lib/Sema/SemaExceptionSpec.cpp
@@ -1068,7 +1068,7 @@ static CanThrowResult canVarDeclThrow(Sema &Self, const VarDecl *VD) {
 
   // If this is a decomposition declaration, bindings might throw.
   if (auto *DD = dyn_cast<DecompositionDecl>(VD))
-    for (auto *B : DD->bindings())
+    for (auto *B : DD->flat_bindings())
       if (auto *HD = B->getHoldingVar())
         CT = mergeCanThrow(CT, canVarDeclThrow(Self, HD));
 
@@ -1286,6 +1286,7 @@ CanThrowResult Sema::canThrow(const Stmt *S) {
   case Expr::ConvertVectorExprClass:
   case Expr::VAArgExprClass:
   case Expr::CXXParenListInitExprClass:
+  case Expr::ResolvedUnexpandedPackExprClass:
     return canSubStmtsThrow(*this, S);
 
   case Expr::CompoundLiteralExprClass:
@@ -1424,6 +1425,7 @@ CanThrowResult Sema::canThrow(const Stmt *S) {
   case Stmt::OpenACCCombinedConstructClass:
   case Stmt::OpenACCDataConstructClass:
   case Stmt::OpenACCHostDataConstructClass:
+  case Stmt::OpenACCAtomicConstructClass:
   case Stmt::AttributedStmtClass:
   case Stmt::BreakStmtClass:
   case Stmt::CapturedStmtClass:
diff --git a/clang/lib/Sema/SemaExpr.cpp b/clang/lib/Sema/SemaExpr.cpp
index d5273d463d7c0..3cd4010740d19 100644
--- a/clang/lib/Sema/SemaExpr.cpp
+++ b/clang/lib/Sema/SemaExpr.cpp
@@ -776,20 +776,11 @@ ExprResult Sema::CallExprUnaryConversions(Expr *E) {
   return Res.get();
 }
 
-/// UsualUnaryConversions - Performs various conversions that are common to most
-/// operators (C99 6.3). The conversions of array and function types are
-/// sometimes suppressed. For example, the array->pointer conversion doesn't
-/// apply if the array is an argument to the sizeof or address (&) operators.
-/// In these instances, this routine should *not* be called.
-ExprResult Sema::UsualUnaryConversions(Expr *E) {
-  // First, convert to an r-value.
-  ExprResult Res = DefaultFunctionArrayLvalueConversion(E);
-  if (Res.isInvalid())
-    return ExprError();
-  E = Res.get();
-
+/// UsualUnaryFPConversions - Promotes floating-point types according to the
+/// current language semantics.
+ExprResult Sema::UsualUnaryFPConversions(Expr *E) {
   QualType Ty = E->getType();
-  assert(!Ty.isNull() && "UsualUnaryConversions - missing type");
+  assert(!Ty.isNull() && "UsualUnaryFPConversions - missing type");
 
   LangOptions::FPEvalMethodKind EvalMethod = CurFPFeatures.getFPEvalMethod();
   if (EvalMethod != LangOptions::FEM_Source && Ty->isFloatingType() &&
@@ -827,7 +818,30 @@ ExprResult Sema::UsualUnaryConversions(Expr *E) {
 
   // Half FP have to be promoted to float unless it is natively supported
   if (Ty->isHalfType() && !getLangOpts().NativeHalfType)
-    return ImpCastExprToType(Res.get(), Context.FloatTy, CK_FloatingCast);
+    return ImpCastExprToType(E, Context.FloatTy, CK_FloatingCast);
+
+  return E;
+}
+
+/// UsualUnaryConversions - Performs various conversions that are common to most
+/// operators (C99 6.3). The conversions of array and function types are
+/// sometimes suppressed. For example, the array->pointer conversion doesn't
+/// apply if the array is an argument to the sizeof or address (&) operators.
+/// In these instances, this routine should *not* be called.
+ExprResult Sema::UsualUnaryConversions(Expr *E) {
+  // First, convert to an r-value.
+  ExprResult Res = DefaultFunctionArrayLvalueConversion(E);
+  if (Res.isInvalid())
+    return ExprError();
+
+  // Promote floating-point types.
+  Res = UsualUnaryFPConversions(Res.get());
+  if (Res.isInvalid())
+    return ExprError();
+  E = Res.get();
+
+  QualType Ty = E->getType();
+  assert(!Ty.isNull() && "UsualUnaryConversions - missing type");
 
   // Try to perform integral promotions if the object has a theoretically
   // promotable type.
@@ -1489,9 +1503,9 @@ static QualType handleFixedPointConversion(Sema &S, QualType LHSTy,
 
 /// Check that the usual arithmetic conversions can be performed on this pair of
 /// expressions that might be of enumeration type.
-static void checkEnumArithmeticConversions(Sema &S, Expr *LHS, Expr *RHS,
-                                           SourceLocation Loc,
-                                           Sema::ArithConvKind ACK) {
+void Sema::checkEnumArithmeticConversions(Expr *LHS, Expr *RHS,
+                                          SourceLocation Loc,
+                                          Sema::ArithConvKind ACK) {
   // C++2a [expr.arith.conv]p1:
   //   If one operand is of enumeration type and the other operand is of a
   //   different enumeration type or a floating-point type, this behavior is
@@ -1499,54 +1513,53 @@ static void checkEnumArithmeticConversions(Sema &S, Expr *LHS, Expr *RHS,
   //
   // Warn on this in all language modes. Produce a deprecation warning in C++20.
   // Eventually we will presumably reject these cases (in C++23 onwards?).
-  QualType L = LHS->getEnumCoercedType(S.Context),
-           R = RHS->getEnumCoercedType(S.Context);
+  QualType L = LHS->getEnumCoercedType(Context),
+           R = RHS->getEnumCoercedType(Context);
   bool LEnum = L->isUnscopedEnumerationType(),
        REnum = R->isUnscopedEnumerationType();
   bool IsCompAssign = ACK == Sema::ACK_CompAssign;
   if ((!IsCompAssign && LEnum && R->isFloatingType()) ||
       (REnum && L->isFloatingType())) {
-    S.Diag(Loc, S.getLangOpts().CPlusPlus26
-                    ? diag::err_arith_conv_enum_float_cxx26
-                : S.getLangOpts().CPlusPlus20
-                    ? diag::warn_arith_conv_enum_float_cxx20
-                    : diag::warn_arith_conv_enum_float)
+    Diag(Loc, getLangOpts().CPlusPlus26 ? diag::err_arith_conv_enum_float_cxx26
+              : getLangOpts().CPlusPlus20
+                  ? diag::warn_arith_conv_enum_float_cxx20
+                  : diag::warn_arith_conv_enum_float)
         << LHS->getSourceRange() << RHS->getSourceRange() << (int)ACK << LEnum
         << L << R;
   } else if (!IsCompAssign && LEnum && REnum &&
-             !S.Context.hasSameUnqualifiedType(L, R)) {
+             !Context.hasSameUnqualifiedType(L, R)) {
     unsigned DiagID;
     // In C++ 26, usual arithmetic conversions between 2 different enum types
     // are ill-formed.
-    if (S.getLangOpts().CPlusPlus26)
+    if (getLangOpts().CPlusPlus26)
       DiagID = diag::err_conv_mixed_enum_types_cxx26;
     else if (!L->castAs<EnumType>()->getDecl()->hasNameForLinkage() ||
              !R->castAs<EnumType>()->getDecl()->hasNameForLinkage()) {
       // If either enumeration type is unnamed, it's less likely that the
       // user cares about this, but this situation is still deprecated in
       // C++2a. Use a different warning group.
-      DiagID = S.getLangOpts().CPlusPlus20
-                    ? diag::warn_arith_conv_mixed_anon_enum_types_cxx20
-                    : diag::warn_arith_conv_mixed_anon_enum_types;
+      DiagID = getLangOpts().CPlusPlus20
+                   ? diag::warn_arith_conv_mixed_anon_enum_types_cxx20
+                   : diag::warn_arith_conv_mixed_anon_enum_types;
     } else if (ACK == Sema::ACK_Conditional) {
       // Conditional expressions are separated out because they have
       // historically had a different warning flag.
-      DiagID = S.getLangOpts().CPlusPlus20
+      DiagID = getLangOpts().CPlusPlus20
                    ? diag::warn_conditional_mixed_enum_types_cxx20
                    : diag::warn_conditional_mixed_enum_types;
     } else if (ACK == Sema::ACK_Comparison) {
       // Comparison expressions are separated out because they have
       // historically had a different warning flag.
-      DiagID = S.getLangOpts().CPlusPlus20
+      DiagID = getLangOpts().CPlusPlus20
                    ? diag::warn_comparison_mixed_enum_types_cxx20
                    : diag::warn_comparison_mixed_enum_types;
     } else {
-      DiagID = S.getLangOpts().CPlusPlus20
+      DiagID = getLangOpts().CPlusPlus20
                    ? diag::warn_arith_conv_mixed_enum_types_cxx20
                    : diag::warn_arith_conv_mixed_enum_types;
     }
-    S.Diag(Loc, DiagID) << LHS->getSourceRange() << RHS->getSourceRange()
-                        << (int)ACK << L << R;
+    Diag(Loc, DiagID) << LHS->getSourceRange() << RHS->getSourceRange()
+                      << (int)ACK << L << R;
   }
 }
 
@@ -1557,7 +1570,7 @@ static void checkEnumArithmeticConversions(Sema &S, Expr *LHS, Expr *RHS,
 QualType Sema::UsualArithmeticConversions(ExprResult &LHS, ExprResult &RHS,
                                           SourceLocation Loc,
                                           ArithConvKind ACK) {
-  checkEnumArithmeticConversions(*this, LHS.get(), RHS.get(), Loc, ACK);
+  checkEnumArithmeticConversions(LHS.get(), RHS.get(), Loc, ACK);
 
   if (ACK != ACK_CompAssign) {
     LHS = UsualUnaryConversions(LHS.get());
@@ -6682,7 +6695,7 @@ ExprResult Sema::BuildCallExpr(Scope *Scope, Expr *Fn, SourceLocation LParenLoc,
 
 Expr *Sema::BuildBuiltinCallExpr(SourceLocation Loc, Builtin::ID Id,
                                  MultiExprArg CallArgs) {
-  StringRef Name = Context.BuiltinInfo.getName(Id);
+  std::string Name = Context.BuiltinInfo.getName(Id);
   LookupResult R(*this, &Context.Idents.get(Name), Loc,
                  Sema::LookupOrdinaryName);
   LookupName(R, TUScope, /*AllowBuiltinCreation=*/true);
@@ -7503,7 +7516,7 @@ static bool breakDownVectorType(QualType type, uint64_t &len,
   if (const VectorType *vecType = type->getAs<VectorType>()) {
     len = vecType->getNumElements();
     eltType = vecType->getElementType();
-    assert(eltType->isScalarType());
+    assert(eltType->isScalarType() || eltType->isMFloat8Type());
     return true;
   }
 
@@ -10174,6 +10187,11 @@ QualType Sema::CheckVectorOperands(ExprResult &LHS, ExprResult &RHS,
     return HLSL().handleVectorBinOpConversion(LHS, RHS, LHSType, RHSType,
                                               IsCompAssign);
 
+  // Any operation with MFloat8 type is only possible with C intrinsics
+  if ((LHSVecType && LHSVecType->getElementType()->isMFloat8Type()) ||
+      (RHSVecType && RHSVecType->getElementType()->isMFloat8Type()))
+    return InvalidOperands(Loc, LHS, RHS);
+
   // AltiVec-style "vector bool op vector bool" combinations are allowed
   // for some operators but not others.
   if (!AllowBothBool && LHSVecType &&
@@ -11793,7 +11811,7 @@ static std::optional<bool> isTautologicalBoundsCheck(Sema &S, const Expr *LHS,
                                                      const Expr *RHS,
                                                      BinaryOperatorKind Opc) {
   if (!LHS->getType()->isPointerType() ||
-      S.getLangOpts().isSignedOverflowDefined())
+      S.getLangOpts().PointerOverflowDefined)
     return std::nullopt;
 
   // Canonicalize to >= or < predicate.
diff --git a/clang/lib/Sema/SemaExprCXX.cpp b/clang/lib/Sema/SemaExprCXX.cpp
index 1e39d69e8b230..34219e0235a74 100644
--- a/clang/lib/Sema/SemaExprCXX.cpp
+++ b/clang/lib/Sema/SemaExprCXX.cpp
@@ -5032,7 +5032,6 @@ static bool CheckUnaryTypeTraitTypeCompleteness(Sema &S, TypeTrait UTT,
   case UTT_IsArray:
   case UTT_IsBoundedArray:
   case UTT_IsPointer:
-  case UTT_IsReferenceable:
   case UTT_IsLvalueReference:
   case UTT_IsRvalueReference:
   case UTT_IsMemberFunctionPointer:
@@ -5679,8 +5678,6 @@ static bool EvaluateUnaryTypeTrait(Sema &Self, TypeTrait UTT,
     return T.isTriviallyRelocatableType(C);
   case UTT_IsBitwiseCloneable:
     return T.isBitwiseCloneableType(C);
-  case UTT_IsReferenceable:
-    return T.isReferenceable();
   case UTT_CanPassInRegs:
     if (CXXRecordDecl *RD = T->getAsCXXRecordDecl(); RD && !T.hasQualifiers())
       return RD->canPassInRegisters();
diff --git a/clang/lib/Sema/SemaHLSL.cpp b/clang/lib/Sema/SemaHLSL.cpp
index f26469e6a2f1d..99eb5360ec356 100644
--- a/clang/lib/Sema/SemaHLSL.cpp
+++ b/clang/lib/Sema/SemaHLSL.cpp
@@ -25,6 +25,7 @@
 #include "clang/Basic/IdentifierTable.h"
 #include "clang/Basic/LLVM.h"
 #include "clang/Basic/SourceLocation.h"
+#include "clang/Basic/Specifiers.h"
 #include "clang/Basic/TargetInfo.h"
 #include "clang/Sema/Initialization.h"
 #include "clang/Sema/ParsedAttr.h"
@@ -268,8 +269,11 @@ static bool isZeroSizedArray(const ConstantArrayType *CAT) {
   return CAT != nullptr;
 }
 
-// Returns true if the record type is an HLSL resource class
-static bool isResourceRecordType(const Type *Ty) {
+// Returns true if the record type is an HLSL resource class or an array of
+// resource classes
+static bool isResourceRecordTypeOrArrayOf(const Type *Ty) {
+  while (const ConstantArrayType *CAT = dyn_cast<ConstantArrayType>(Ty))
+    Ty = CAT->getArrayElementTypeNoTypeQual();
   return HLSLAttributedResourceType::findHandleTypeOnResource(Ty) != nullptr;
 }
 
@@ -278,15 +282,15 @@ static bool isResourceRecordType(const Type *Ty) {
 // array, or a builtin intangible type. Returns false it is a valid leaf element
 // type or if it is a record type that needs to be inspected further.
 static bool isInvalidConstantBufferLeafElementType(const Type *Ty) {
-  if (Ty->isRecordType()) {
-    if (isResourceRecordType(Ty) || Ty->getAsCXXRecordDecl()->isEmpty())
-      return true;
-    return false;
-  }
+  Ty = Ty->getUnqualifiedDesugaredType();
+  if (isResourceRecordTypeOrArrayOf(Ty))
+    return true;
+  if (Ty->isRecordType())
+    return Ty->getAsCXXRecordDecl()->isEmpty();
   if (Ty->isConstantArrayType() &&
       isZeroSizedArray(cast<ConstantArrayType>(Ty)))
     return true;
-  if (Ty->isHLSLBuiltinIntangibleType())
+  if (Ty->isHLSLBuiltinIntangibleType() || Ty->isHLSLAttributedResourceType())
     return true;
   return false;
 }
@@ -338,7 +342,7 @@ static IdentifierInfo *getHostLayoutStructName(Sema &S, NamedDecl *BaseDecl,
   ASTContext &AST = S.getASTContext();
 
   IdentifierInfo *NameBaseII = BaseDecl->getIdentifier();
-  llvm::SmallString<64> Name("__layout_");
+  llvm::SmallString<64> Name("__cblayout_");
   if (NameBaseII) {
     Name.append(NameBaseII->getName());
   } else {
@@ -392,7 +396,7 @@ static FieldDecl *createFieldForHostLayoutStruct(Sema &S, const Type *Ty,
   auto *Field = FieldDecl::Create(AST, LayoutStruct, SourceLocation(),
                                   SourceLocation(), II, QT, TSI, nullptr, false,
                                   InClassInitStyle::ICIS_NoInit);
-  Field->setAccess(AccessSpecifier::AS_private);
+  Field->setAccess(AccessSpecifier::AS_public);
   return Field;
 }
 
@@ -416,14 +420,17 @@ static CXXRecordDecl *createHostLayoutStruct(Sema &S,
   if (CXXRecordDecl *RD = findRecordDeclInContext(II, DC))
     return RD;
 
-  CXXRecordDecl *LS = CXXRecordDecl::Create(
-      AST, TagDecl::TagKind::Class, DC, SourceLocation(), SourceLocation(), II);
+  CXXRecordDecl *LS =
+      CXXRecordDecl::Create(AST, TagDecl::TagKind::Struct, DC, SourceLocation(),
+                            SourceLocation(), II);
   LS->setImplicit(true);
+  LS->addAttr(PackedAttr::CreateImplicit(AST));
   LS->startDefinition();
 
   // copy base struct, create HLSL Buffer compatible version if needed
   if (unsigned NumBases = StructDecl->getNumBases()) {
     assert(NumBases == 1 && "HLSL supports only one base type");
+    (void)NumBases;
     CXXBaseSpecifier Base = *StructDecl->bases_begin();
     CXXRecordDecl *BaseDecl = Base.getType()->getAsCXXRecordDecl();
     if (requiresImplicitBufferLayoutStructure(BaseDecl)) {
@@ -459,30 +466,38 @@ static CXXRecordDecl *createHostLayoutStruct(Sema &S,
 
 // Creates host layout struct for HLSL Buffer. The struct will include only
 // fields of types that are allowed in HLSL buffer and it will filter out:
-// - static variable declarations
+// - static or groupshared variable declarations
 // - resource classes
 // - empty structs
 // - zero-sized arrays
 // - non-variable declarations
-// The layour struct will be added to the HLSLBufferDecl declarations.
+// The layout struct will be added to the HLSLBufferDecl declarations.
 void createHostLayoutStructForBuffer(Sema &S, HLSLBufferDecl *BufDecl) {
   ASTContext &AST = S.getASTContext();
   IdentifierInfo *II = getHostLayoutStructName(S, BufDecl, true);
 
   CXXRecordDecl *LS =
-      CXXRecordDecl::Create(AST, TagDecl::TagKind::Class, BufDecl,
+      CXXRecordDecl::Create(AST, TagDecl::TagKind::Struct, BufDecl,
                             SourceLocation(), SourceLocation(), II);
+  LS->addAttr(PackedAttr::CreateImplicit(AST));
   LS->setImplicit(true);
   LS->startDefinition();
 
-  for (const Decl *D : BufDecl->decls()) {
-    const VarDecl *VD = dyn_cast<VarDecl>(D);
-    if (!VD || VD->getStorageClass() == SC_Static)
+  for (Decl *D : BufDecl->decls()) {
+    VarDecl *VD = dyn_cast<VarDecl>(D);
+    if (!VD || VD->getStorageClass() == SC_Static ||
+        VD->getType().getAddressSpace() == LangAS::hlsl_groupshared)
       continue;
     const Type *Ty = VD->getType()->getUnqualifiedDesugaredType();
     if (FieldDecl *FD =
-            createFieldForHostLayoutStruct(S, Ty, VD->getIdentifier(), LS))
+            createFieldForHostLayoutStruct(S, Ty, VD->getIdentifier(), LS)) {
+      // add the field decl to the layout struct
       LS->addDecl(FD);
+      // update address space of the original decl to hlsl_constant
+      QualType NewTy =
+          AST.getAddrSpaceQualType(VD->getType(), LangAS::hlsl_constant);
+      VD->setType(NewTy);
+    }
   }
   LS->completeDefinition();
   BufDecl->addDecl(LS);
@@ -1315,8 +1330,8 @@ SemaHLSL::TakeLocForHLSLAttribute(const HLSLAttributedResourceType *RT) {
 
 // Walks though the global variable declaration, collects all resource binding
 // requirements and adds them to Bindings
-void SemaHLSL::collectResourcesOnUserRecordDecl(const VarDecl *VD,
-                                                const RecordType *RT) {
+void SemaHLSL::collectResourceBindingsOnUserRecordDecl(const VarDecl *VD,
+                                                       const RecordType *RT) {
   const RecordDecl *RD = RT->getDecl();
   for (FieldDecl *FD : RD->fields()) {
     const Type *Ty = FD->getType()->getUnqualifiedDesugaredType();
@@ -1346,15 +1361,15 @@ void SemaHLSL::collectResourcesOnUserRecordDecl(const VarDecl *VD,
       // binding, which is something we are probably going to need to do later
       // on. Hopefully nesting of structs in structs too many levels is
       // unlikely.
-      collectResourcesOnUserRecordDecl(VD, RT);
+      collectResourceBindingsOnUserRecordDecl(VD, RT);
     }
   }
 }
 
-// Diagnore localized register binding errors for a single binding; does not
+// Diagnose localized register binding errors for a single binding; does not
 // diagnose resource binding on user record types, that will be done later
 // in processResourceBindingOnDecl based on the information collected in
-// collectResourcesOnVarDecl.
+// collectResourceBindingsOnVarDecl.
 // Returns false if the register binding is not valid.
 static bool DiagnoseLocalRegisterBinding(Sema &S, SourceLocation &ArgLoc,
                                          Decl *D, RegisterType RegType,
@@ -2023,6 +2038,18 @@ static bool CheckAllArgsHaveFloatRepresentation(Sema *S, CallExpr *TheCall) {
                                     checkAllFloatTypes);
 }
 
+static bool CheckUnsignedIntRepresentations(Sema *S, CallExpr *TheCall) {
+  auto checkUnsignedInteger = [](clang::QualType PassedType) -> bool {
+    clang::QualType BaseType =
+        PassedType->isVectorType()
+            ? PassedType->getAs<clang::VectorType>()->getElementType()
+            : PassedType;
+    return !BaseType->isUnsignedIntegerType();
+  };
+  return CheckAllArgTypesAreCorrect(S, TheCall, S->Context.UnsignedIntTy,
+                                    checkUnsignedInteger);
+}
+
 static bool CheckFloatOrHalfRepresentations(Sema *S, CallExpr *TheCall) {
   auto checkFloatorHalf = [](clang::QualType PassedType) -> bool {
     clang::QualType BaseType =
@@ -2214,6 +2241,41 @@ static bool CheckResourceHandle(
 // returning an ExprError
 bool SemaHLSL::CheckBuiltinFunctionCall(unsigned BuiltinID, CallExpr *TheCall) {
   switch (BuiltinID) {
+  case Builtin::BI__builtin_hlsl_adduint64: {
+    if (SemaRef.checkArgCount(TheCall, 2))
+      return true;
+    if (CheckVectorElementCallArgs(&SemaRef, TheCall))
+      return true;
+    if (CheckUnsignedIntRepresentations(&SemaRef, TheCall))
+      return true;
+
+    // CheckVectorElementCallArgs(...) guarantees both args are the same type.
+    assert(TheCall->getArg(0)->getType() == TheCall->getArg(1)->getType() &&
+           "Both args must be of the same type");
+
+    // ensure both args are vectors
+    auto *VTy = TheCall->getArg(0)->getType()->getAs<VectorType>();
+    if (!VTy) {
+      SemaRef.Diag(TheCall->getBeginLoc(), diag::err_vec_builtin_non_vector)
+          << "AddUint64" << /*all*/ 1;
+      return true;
+    }
+
+    // ensure both args have 2 elements, or both args have 4 elements
+    int NumElementsArg = VTy->getNumElements();
+    if (NumElementsArg != 2 && NumElementsArg != 4) {
+      SemaRef.Diag(TheCall->getBeginLoc(),
+                   diag::err_invalid_even_odd_vector_element_count)
+          << NumElementsArg << 2 << 4 << /*even*/ 0 << /*operand*/ 1;
+      return true;
+    }
+
+    ExprResult A = TheCall->getArg(0);
+    QualType ArgTyA = A.get()->getType();
+    // return type is the same as the input type
+    TheCall->setType(ArgTyA);
+    break;
+  }
   case Builtin::BI__builtin_hlsl_resource_getpointer: {
     if (SemaRef.checkArgCount(TheCall, 2) ||
         CheckResourceHandle(&SemaRef, TheCall, 0) ||
@@ -2422,6 +2484,7 @@ bool SemaHLSL::CheckBuiltinFunctionCall(unsigned BuiltinID, CallExpr *TheCall) {
     TheCall->setType(ArgTyA);
     break;
   }
+  case Builtin::BI__builtin_hlsl_wave_active_max:
   case Builtin::BI__builtin_hlsl_wave_active_sum: {
     if (SemaRef.checkArgCount(TheCall, 1))
       return true;
@@ -2697,6 +2760,150 @@ bool SemaHLSL::CheckCompatibleParameterABI(FunctionDecl *New,
   return HadError;
 }
 
+// Generally follows PerformScalarCast, with cases reordered for
+// clarity of what types are supported
+bool SemaHLSL::CanPerformScalarCast(QualType SrcTy, QualType DestTy) {
+
+  if (SemaRef.getASTContext().hasSameUnqualifiedType(SrcTy, DestTy))
+    return true;
+
+  switch (SrcTy->getScalarTypeKind()) {
+  case Type::STK_Bool: // casting from bool is like casting from an integer
+  case Type::STK_Integral:
+    switch (DestTy->getScalarTypeKind()) {
+    case Type::STK_Bool:
+    case Type::STK_Integral:
+    case Type::STK_Floating:
+      return true;
+    case Type::STK_CPointer:
+    case Type::STK_ObjCObjectPointer:
+    case Type::STK_BlockPointer:
+    case Type::STK_MemberPointer:
+      llvm_unreachable("HLSL doesn't support pointers.");
+    case Type::STK_IntegralComplex:
+    case Type::STK_FloatingComplex:
+      llvm_unreachable("HLSL doesn't support complex types.");
+    case Type::STK_FixedPoint:
+      llvm_unreachable("HLSL doesn't support fixed point types.");
+    }
+    llvm_unreachable("Should have returned before this");
+
+  case Type::STK_Floating:
+    switch (DestTy->getScalarTypeKind()) {
+    case Type::STK_Floating:
+    case Type::STK_Bool:
+    case Type::STK_Integral:
+      return true;
+    case Type::STK_FloatingComplex:
+    case Type::STK_IntegralComplex:
+      llvm_unreachable("HLSL doesn't support complex types.");
+    case Type::STK_FixedPoint:
+      llvm_unreachable("HLSL doesn't support fixed point types.");
+    case Type::STK_CPointer:
+    case Type::STK_ObjCObjectPointer:
+    case Type::STK_BlockPointer:
+    case Type::STK_MemberPointer:
+      llvm_unreachable("HLSL doesn't support pointers.");
+    }
+    llvm_unreachable("Should have returned before this");
+
+  case Type::STK_MemberPointer:
+  case Type::STK_CPointer:
+  case Type::STK_BlockPointer:
+  case Type::STK_ObjCObjectPointer:
+    llvm_unreachable("HLSL doesn't support pointers.");
+
+  case Type::STK_FixedPoint:
+    llvm_unreachable("HLSL doesn't support fixed point types.");
+
+  case Type::STK_FloatingComplex:
+  case Type::STK_IntegralComplex:
+    llvm_unreachable("HLSL doesn't support complex types.");
+  }
+
+  llvm_unreachable("Unhandled scalar cast");
+}
+
+// Detect if a type contains a bitfield. Will be removed when
+// bitfield support is added to HLSLElementwiseCast
+bool SemaHLSL::ContainsBitField(QualType BaseTy) {
+  llvm::SmallVector<QualType, 16> WorkList;
+  WorkList.push_back(BaseTy);
+  while (!WorkList.empty()) {
+    QualType T = WorkList.pop_back_val();
+    T = T.getCanonicalType().getUnqualifiedType();
+    // only check aggregate types
+    if (const auto *AT = dyn_cast<ConstantArrayType>(T)) {
+      WorkList.push_back(AT->getElementType());
+      continue;
+    }
+    if (const auto *RT = dyn_cast<RecordType>(T)) {
+      const RecordDecl *RD = RT->getDecl();
+      if (RD->isUnion())
+        continue;
+
+      const CXXRecordDecl *CXXD = dyn_cast<CXXRecordDecl>(RD);
+
+      if (CXXD && CXXD->isStandardLayout())
+        RD = CXXD->getStandardLayoutBaseWithFields();
+
+      for (const auto *FD : RD->fields()) {
+        if (FD->isBitField())
+          return true;
+        WorkList.push_back(FD->getType());
+      }
+      continue;
+    }
+  }
+  return false;
+}
+
+// Can we perform an HLSL Elementwise cast?
+// TODO: update this code when matrices are added; see issue #88060
+bool SemaHLSL::CanPerformElementwiseCast(Expr *Src, QualType DestTy) {
+
+  // Don't handle casts where LHS and RHS are any combination of scalar/vector
+  // There must be an aggregate somewhere
+  QualType SrcTy = Src->getType();
+  if (SrcTy->isScalarType()) // always a splat and this cast doesn't handle that
+    return false;
+
+  if (SrcTy->isVectorType() &&
+      (DestTy->isScalarType() || DestTy->isVectorType()))
+    return false;
+
+  if (ContainsBitField(DestTy) || ContainsBitField(SrcTy))
+    return false;
+
+  llvm::SmallVector<QualType> DestTypes;
+  BuildFlattenedTypeList(DestTy, DestTypes);
+  llvm::SmallVector<QualType> SrcTypes;
+  BuildFlattenedTypeList(SrcTy, SrcTypes);
+
+  // Usually the size of SrcTypes must be greater than or equal to the size of
+  // DestTypes.
+  if (SrcTypes.size() < DestTypes.size())
+    return false;
+
+  unsigned SrcSize = SrcTypes.size();
+  unsigned DstSize = DestTypes.size();
+  unsigned I;
+  for (I = 0; I < DstSize && I < SrcSize; I++) {
+    if (SrcTypes[I]->isUnionType() || DestTypes[I]->isUnionType())
+      return false;
+    if (!CanPerformScalarCast(SrcTypes[I], DestTypes[I])) {
+      return false;
+    }
+  }
+
+  // check the rest of the source type for unions.
+  for (; I < SrcSize; I++) {
+    if (SrcTypes[I]->isUnionType())
+      return false;
+  }
+  return true;
+}
+
 ExprResult SemaHLSL::ActOnOutParamExpr(ParmVarDecl *Param, Expr *Arg) {
   assert(Param->hasAttr<HLSLParamModifierAttr>() &&
          "We should not get here without a parameter modifier expression");
@@ -2779,7 +2986,7 @@ void SemaHLSL::ActOnVariableDeclarator(VarDecl *VD) {
 
     // find all resources on decl
     if (VD->getType()->isHLSLIntangibleType())
-      collectResourcesOnVarDecl(VD);
+      collectResourceBindingsOnVarDecl(VD);
 
     // process explicit bindings
     processExplicitBindingsOnDecl(VD);
@@ -2788,7 +2995,7 @@ void SemaHLSL::ActOnVariableDeclarator(VarDecl *VD) {
 
 // Walks though the global variable declaration, collects all resource binding
 // requirements and adds them to Bindings
-void SemaHLSL::collectResourcesOnVarDecl(VarDecl *VD) {
+void SemaHLSL::collectResourceBindingsOnVarDecl(VarDecl *VD) {
   assert(VD->hasGlobalStorage() && VD->getType()->isHLSLIntangibleType() &&
          "expected global variable that contains HLSL resource");
 
@@ -2817,7 +3024,7 @@ void SemaHLSL::collectResourcesOnVarDecl(VarDecl *VD) {
 
   // User defined record type
   if (const RecordType *RT = dyn_cast<RecordType>(Ty))
-    collectResourcesOnUserRecordDecl(VD, RT);
+    collectResourceBindingsOnUserRecordDecl(VD, RT);
 }
 
 // Walks though the explicit resource binding attributes on the declaration,
diff --git a/clang/lib/Sema/SemaInit.cpp b/clang/lib/Sema/SemaInit.cpp
index b95cbbf422205..18090eb1c9e9a 100644
--- a/clang/lib/Sema/SemaInit.cpp
+++ b/clang/lib/Sema/SemaInit.cpp
@@ -4573,8 +4573,12 @@ static void TryConstructorInitialization(Sema &S,
 
   CXXConstructorDecl *CtorDecl = cast<CXXConstructorDecl>(Best->Function);
   if (Result != OR_Deleted) {
-    if (!IsListInit && Kind.getKind() == InitializationKind::IK_Default &&
-        DestRecordDecl != nullptr && DestRecordDecl->isAggregate() &&
+    if (!IsListInit &&
+        (Kind.getKind() == InitializationKind::IK_Default ||
+         Kind.getKind() == InitializationKind::IK_Direct) &&
+        DestRecordDecl != nullptr &&
+        !(CtorDecl->isCopyOrMoveConstructor() && CtorDecl->isImplicit()) &&
+        DestRecordDecl->isAggregate() &&
         DestRecordDecl->hasUninitializedExplicitInitFields()) {
       S.Diag(Kind.getLocation(), diag::warn_field_requires_explicit_init)
           << /* Var-in-Record */ 1 << DestRecordDecl;
@@ -4860,9 +4864,13 @@ static void TryListInitialization(Sema &S,
         assert(
             S.Context.hasSameUnqualifiedType(SubInit[0]->getType(), DestType) &&
             "Deduced to other type?");
+        assert(Kind.getKind() == clang::InitializationKind::IK_DirectList &&
+               "List-initialize structured bindings but not "
+               "direct-list-initialization?");
         TryArrayCopy(S,
-                     InitializationKind::CreateCopy(Kind.getLocation(),
-                                                    InitList->getLBraceLoc()),
+                     InitializationKind::CreateDirect(Kind.getLocation(),
+                                                      InitList->getLBraceLoc(),
+                                                      InitList->getRBraceLoc()),
                      Entity, SubInit[0], DestType, Sequence,
                      TreatUnavailableAsInvalid);
         if (Sequence)
@@ -9146,6 +9154,17 @@ bool InitializationSequence::Diagnose(Sema &S,
               << (Msg ? Msg->getString() : StringRef()) << ArgsRange;
         }
 
+        // If it's a default constructed member, but it's not in the
+        // constructor's initializer list, explicitly note where the member is
+        // declared so the user can see which member is erroneously initialized
+        // with a deleted default constructor.
+        if (Kind.getKind() == InitializationKind::IK_Default &&
+            (Entity.getKind() == InitializedEntity::EK_Member ||
+             Entity.getKind() == InitializedEntity::EK_ParenAggInitMember)) {
+          S.Diag(Entity.getDecl()->getLocation(),
+                 diag::note_default_constructed_field)
+              << Entity.getDecl();
+        }
         S.NoteDeletedFunction(Best->Function);
         break;
       }
diff --git a/clang/lib/Sema/SemaLambda.cpp b/clang/lib/Sema/SemaLambda.cpp
index 87b3ca53cefaf..ceb32ee15dfa3 100644
--- a/clang/lib/Sema/SemaLambda.cpp
+++ b/clang/lib/Sema/SemaLambda.cpp
@@ -2239,18 +2239,18 @@ ExprResult Sema::BuildLambdaExpr(SourceLocation StartLoc, SourceLocation EndLoc,
 
   Cleanup.mergeFrom(LambdaCleanup);
 
-  LambdaExpr *Lambda = LambdaExpr::Create(Context, Class, IntroducerRange,
-                                          CaptureDefault, CaptureDefaultLoc,
-                                          ExplicitParams, ExplicitResultType,
-                                          CaptureInits, EndLoc,
-                                          ContainsUnexpandedParameterPack);
+  LambdaExpr *Lambda =
+      LambdaExpr::Create(Context, Class, IntroducerRange, CaptureDefault,
+                         CaptureDefaultLoc, ExplicitParams, ExplicitResultType,
+                         CaptureInits, EndLoc, ContainsUnexpandedParameterPack);
+
   // If the lambda expression's call operator is not explicitly marked constexpr
-  // and we are not in a dependent context, analyze the call operator to infer
+  // and is not dependent, analyze the call operator to infer
   // its constexpr-ness, suppressing diagnostics while doing so.
   if (getLangOpts().CPlusPlus17 && !CallOperator->isInvalidDecl() &&
       !CallOperator->isConstexpr() &&
       !isa<CoroutineBodyStmt>(CallOperator->getBody()) &&
-      !Class->getDeclContext()->isDependentContext()) {
+      !Class->isDependentContext()) {
     CallOperator->setConstexprKind(
         CheckConstexprFunctionDefinition(CallOperator,
                                          CheckConstexprKind::CheckValid)
diff --git a/clang/lib/Sema/SemaLookup.cpp b/clang/lib/Sema/SemaLookup.cpp
index 5f8ffa71607bb..0f5b7426e743e 100644
--- a/clang/lib/Sema/SemaLookup.cpp
+++ b/clang/lib/Sema/SemaLookup.cpp
@@ -3721,13 +3721,11 @@ Sema::LookupLiteralOperator(Scope *S, LookupResult &R,
         // is a well-formed template argument for the template parameter.
         if (StringLit) {
           SFINAETrap Trap(*this);
-          SmallVector<TemplateArgument, 1> SugaredChecked, CanonicalChecked;
+          CheckTemplateArgumentInfo CTAI;
           TemplateArgumentLoc Arg(TemplateArgument(StringLit), StringLit);
           if (CheckTemplateArgument(
                   Params->getParam(0), Arg, FD, R.getNameLoc(), R.getNameLoc(),
-                  0, SugaredChecked, CanonicalChecked, CTAK_Specified,
-                  /*PartialOrdering=*/false,
-                  /*MatchedPackOnParmToNonPackOnArg=*/nullptr) ||
+                  /*ArgumentPackIndex=*/0, CTAI, CTAK_Specified) ||
               Trap.hasErrorOccurred())
             IsTemplate = false;
         }
diff --git a/clang/lib/Sema/SemaMIPS.cpp b/clang/lib/Sema/SemaMIPS.cpp
index 50d210e5b3814..ce6ad6d9cc3f3 100644
--- a/clang/lib/Sema/SemaMIPS.cpp
+++ b/clang/lib/Sema/SemaMIPS.cpp
@@ -271,14 +271,14 @@ void SemaMIPS::handleInterruptAttr(Decl *D, const ParsedAttr &AL) {
   }
 
   if (hasFunctionProto(D) && getFunctionOrMethodNumParams(D) != 0) {
-    Diag(D->getLocation(), diag::warn_interrupt_attribute_invalid)
-        << /*MIPS*/ 0 << 0;
+    Diag(D->getLocation(), diag::warn_interrupt_signal_attribute_invalid)
+        << /*MIPS*/ 0 << /*interrupt*/ 0 << 0;
     return;
   }
 
   if (!getFunctionOrMethodResultType(D)->isVoidType()) {
-    Diag(D->getLocation(), diag::warn_interrupt_attribute_invalid)
-        << /*MIPS*/ 0 << 1;
+    Diag(D->getLocation(), diag::warn_interrupt_signal_attribute_invalid)
+        << /*MIPS*/ 0 << /*interrupt*/ 0 << 1;
     return;
   }
 
diff --git a/clang/lib/Sema/SemaMSP430.cpp b/clang/lib/Sema/SemaMSP430.cpp
index 4038a1ff61d63..5bf931e388f03 100644
--- a/clang/lib/Sema/SemaMSP430.cpp
+++ b/clang/lib/Sema/SemaMSP430.cpp
@@ -32,14 +32,14 @@ void SemaMSP430::handleInterruptAttr(Decl *D, const ParsedAttr &AL) {
   }
 
   if (hasFunctionProto(D) && getFunctionOrMethodNumParams(D) != 0) {
-    Diag(D->getLocation(), diag::warn_interrupt_attribute_invalid)
-        << /*MSP430*/ 1 << 0;
+    Diag(D->getLocation(), diag::warn_interrupt_signal_attribute_invalid)
+        << /*MSP430*/ 1 << /*interrupt*/ 0 << 0;
     return;
   }
 
   if (!getFunctionOrMethodResultType(D)->isVoidType()) {
-    Diag(D->getLocation(), diag::warn_interrupt_attribute_invalid)
-        << /*MSP430*/ 1 << 1;
+    Diag(D->getLocation(), diag::warn_interrupt_signal_attribute_invalid)
+        << /*MSP430*/ 1 << /*interrupt*/ 0 << 1;
     return;
   }
 
diff --git a/clang/lib/Sema/SemaOpenACC.cpp b/clang/lib/Sema/SemaOpenACC.cpp
index f5edc0ed36a9a..2d2f8ddf4652b 100644
--- a/clang/lib/Sema/SemaOpenACC.cpp
+++ b/clang/lib/Sema/SemaOpenACC.cpp
@@ -30,18 +30,23 @@ bool diagnoseConstructAppertainment(SemaOpenACC &S, OpenACCDirectiveKind K,
     // Nothing to do here, both invalid and unimplemented don't really need to
     // do anything.
     break;
-  case OpenACCDirectiveKind::ParallelLoop:
-  case OpenACCDirectiveKind::SerialLoop:
-  case OpenACCDirectiveKind::KernelsLoop:
   case OpenACCDirectiveKind::Parallel:
+  case OpenACCDirectiveKind::ParallelLoop:
   case OpenACCDirectiveKind::Serial:
+  case OpenACCDirectiveKind::SerialLoop:
   case OpenACCDirectiveKind::Kernels:
+  case OpenACCDirectiveKind::KernelsLoop:
   case OpenACCDirectiveKind::Loop:
   case OpenACCDirectiveKind::Data:
   case OpenACCDirectiveKind::EnterData:
   case OpenACCDirectiveKind::ExitData:
   case OpenACCDirectiveKind::HostData:
   case OpenACCDirectiveKind::Wait:
+  case OpenACCDirectiveKind::Update:
+  case OpenACCDirectiveKind::Init:
+  case OpenACCDirectiveKind::Shutdown:
+  case OpenACCDirectiveKind::Cache:
+  case OpenACCDirectiveKind::Atomic:
     if (!IsStmt)
       return S.Diag(StartLoc, diag::err_acc_construct_appertainment) << K;
     break;
@@ -73,6 +78,7 @@ bool PreserveLoopRAIIDepthInAssociatedStmtRAII(OpenACCDirectiveKind DK) {
     return false;
   case OpenACCDirectiveKind::Data:
   case OpenACCDirectiveKind::HostData:
+  case OpenACCDirectiveKind::Atomic:
     return true;
   case OpenACCDirectiveKind::EnterData:
   case OpenACCDirectiveKind::ExitData:
@@ -327,6 +333,7 @@ void SemaOpenACC::ActOnConstruct(OpenACCDirectiveKind K,
   case OpenACCDirectiveKind::Shutdown:
   case OpenACCDirectiveKind::Set:
   case OpenACCDirectiveKind::Update:
+  case OpenACCDirectiveKind::Atomic:
     // Nothing to do here, there is no real legalization that needs to happen
     // here as these constructs do not take any arguments.
     break;
@@ -1518,8 +1525,9 @@ bool SemaOpenACC::ActOnStartStmtDirective(
 StmtResult SemaOpenACC::ActOnEndStmtDirective(
     OpenACCDirectiveKind K, SourceLocation StartLoc, SourceLocation DirLoc,
     SourceLocation LParenLoc, SourceLocation MiscLoc, ArrayRef<Expr *> Exprs,
-    SourceLocation RParenLoc, SourceLocation EndLoc,
-    ArrayRef<OpenACCClause *> Clauses, StmtResult AssocStmt) {
+    OpenACCAtomicKind AtomicKind, SourceLocation RParenLoc,
+    SourceLocation EndLoc, ArrayRef<OpenACCClause *> Clauses,
+    StmtResult AssocStmt) {
   switch (K) {
   default:
     return StmtEmpty();
@@ -1583,13 +1591,20 @@ StmtResult SemaOpenACC::ActOnEndStmtDirective(
     return OpenACCUpdateConstruct::Create(getASTContext(), StartLoc, DirLoc,
                                           EndLoc, Clauses);
   }
+  case OpenACCDirectiveKind::Atomic: {
+    assert(Clauses.empty() && "Atomic doesn't allow clauses");
+    return OpenACCAtomicConstruct::Create(
+        getASTContext(), StartLoc, DirLoc, AtomicKind, EndLoc,
+        AssocStmt.isUsable() ? AssocStmt.get() : nullptr);
+  }
   }
   llvm_unreachable("Unhandled case in directive handling?");
 }
 
 StmtResult SemaOpenACC::ActOnAssociatedStmt(
     SourceLocation DirectiveLoc, OpenACCDirectiveKind K,
-    ArrayRef<const OpenACCClause *> Clauses, StmtResult AssocStmt) {
+    OpenACCAtomicKind AtKind, ArrayRef<const OpenACCClause *> Clauses,
+    StmtResult AssocStmt) {
   switch (K) {
   default:
     llvm_unreachable("Unimplemented associated statement application");
@@ -1601,6 +1616,8 @@ StmtResult SemaOpenACC::ActOnAssociatedStmt(
   case OpenACCDirectiveKind::Set:
     llvm_unreachable(
         "these don't have associated statements, so shouldn't get here");
+  case OpenACCDirectiveKind::Atomic:
+    return CheckAtomicAssociatedStmt(DirectiveLoc, AtKind, AssocStmt);
   case OpenACCDirectiveKind::Parallel:
   case OpenACCDirectiveKind::Serial:
   case OpenACCDirectiveKind::Kernels:
diff --git a/clang/lib/Sema/SemaOpenACCAtomic.cpp b/clang/lib/Sema/SemaOpenACCAtomic.cpp
new file mode 100644
index 0000000000000..68cf338c07115
--- /dev/null
+++ b/clang/lib/Sema/SemaOpenACCAtomic.cpp
@@ -0,0 +1,736 @@
+//== SemaOpenACCAtomic.cpp - Semantic Analysis for OpenACC Atomic Construct===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+/// \file
+/// This file implements semantic analysis for the OpenACC atomic construct.
+///
+//===----------------------------------------------------------------------===//
+
+#include "clang/AST/ExprCXX.h"
+#include "clang/Basic/DiagnosticSema.h"
+#include "clang/Sema/SemaOpenACC.h"
+
+#include <optional>
+#include <variant>
+
+using namespace clang;
+
+namespace {
+
+class AtomicOperandChecker {
+  SemaOpenACC &SemaRef;
+  OpenACCAtomicKind AtKind;
+  SourceLocation AtomicDirLoc;
+  StmtResult AssocStmt;
+
+  // Do a diagnostic, which sets the correct error, then displays passed note.
+  bool DiagnoseInvalidAtomic(SourceLocation Loc, PartialDiagnostic NoteDiag) {
+    SemaRef.Diag(AtomicDirLoc, diag::err_acc_invalid_atomic)
+        << (AtKind != OpenACCAtomicKind::None) << AtKind;
+    SemaRef.Diag(Loc, NoteDiag);
+    return true;
+  }
+
+  // Create a replacement recovery expr in case we find an error here.  This
+  // allows us to ignore this during template instantiation so we only get a
+  // single error.
+  StmtResult getRecoveryExpr() {
+    if (!AssocStmt.isUsable())
+      return AssocStmt;
+
+    if (!SemaRef.getASTContext().getLangOpts().RecoveryAST)
+      return StmtError();
+
+    Expr *E = dyn_cast<Expr>(AssocStmt.get());
+    QualType T = E ? E->getType() : SemaRef.getASTContext().DependentTy;
+
+    return RecoveryExpr::Create(SemaRef.getASTContext(), T,
+                                AssocStmt.get()->getBeginLoc(),
+                                AssocStmt.get()->getEndLoc(),
+                                E ? ArrayRef<Expr *>{E} : ArrayRef<Expr *>{});
+  }
+
+  // OpenACC 3.3 2.12: 'expr' is an expression with scalar type.
+  bool CheckOperandExpr(const Expr *E, PartialDiagnostic PD) {
+    QualType ExprTy = E->getType();
+
+    // Scalar allowed, plus we allow instantiation dependent to support
+    // templates.
+    if (ExprTy->isInstantiationDependentType() || ExprTy->isScalarType())
+      return false;
+
+    return DiagnoseInvalidAtomic(E->getExprLoc(),
+                                 PD << diag::OACCLValScalar::Scalar << ExprTy);
+  }
+
+  // OpenACC 3.3 2.12: 'x' and 'v' (as applicable) are boht l-value expressoins
+  // with scalar type.
+  bool CheckOperandVariable(const Expr *E, PartialDiagnostic PD) {
+    if (CheckOperandExpr(E, PD))
+      return true;
+
+    if (E->isLValue())
+      return false;
+
+    return DiagnoseInvalidAtomic(E->getExprLoc(),
+                                 PD << diag::OACCLValScalar::LVal);
+  }
+
+  Expr *RequireExpr(Stmt *Stmt, PartialDiagnostic ExpectedNote) {
+    if (Expr *E = dyn_cast<Expr>(Stmt))
+      return E->IgnoreImpCasts();
+
+    DiagnoseInvalidAtomic(Stmt->getBeginLoc(), ExpectedNote);
+    return nullptr;
+  }
+
+  // A struct to hold the return the inner components of any operands, which
+  // allows for compound checking.
+  struct BinaryOpInfo {
+    const Expr *FoundExpr = nullptr;
+    const Expr *LHS = nullptr;
+    const Expr *RHS = nullptr;
+    BinaryOperatorKind Operator;
+  };
+
+  struct UnaryOpInfo {
+    const Expr *FoundExpr = nullptr;
+    const Expr *SubExpr = nullptr;
+    UnaryOperatorKind Operator;
+
+    bool IsIncrementOp() {
+      return Operator == UO_PostInc || Operator == UO_PreInc;
+    }
+  };
+
+  std::optional<UnaryOpInfo> GetUnaryOperatorInfo(const Expr *E) {
+    // If this is a simple unary operator, just return its details.
+    if (const auto *UO = dyn_cast<UnaryOperator>(E))
+      return UnaryOpInfo{UO, UO->getSubExpr()->IgnoreImpCasts(),
+                         UO->getOpcode()};
+
+    // This might be an overloaded operator or a dependent context, so make sure
+    // we can get as many details out of this as we can.
+    if (const auto *OpCall = dyn_cast<CXXOperatorCallExpr>(E)) {
+      UnaryOpInfo Inf;
+      Inf.FoundExpr = OpCall;
+
+      switch (OpCall->getOperator()) {
+      default:
+        return std::nullopt;
+      case OO_PlusPlus:
+        Inf.Operator = OpCall->getNumArgs() == 1 ? UO_PreInc : UO_PostInc;
+        break;
+      case OO_MinusMinus:
+        Inf.Operator = OpCall->getNumArgs() == 1 ? UO_PreDec : UO_PostDec;
+        break;
+      case OO_Amp:
+        Inf.Operator = UO_AddrOf;
+        break;
+      case OO_Star:
+        Inf.Operator = UO_Deref;
+        break;
+      case OO_Plus:
+        Inf.Operator = UO_Plus;
+        break;
+      case OO_Minus:
+        Inf.Operator = UO_Minus;
+        break;
+      case OO_Tilde:
+        Inf.Operator = UO_Not;
+        break;
+      case OO_Exclaim:
+        Inf.Operator = UO_LNot;
+        break;
+      case OO_Coawait:
+        Inf.Operator = UO_Coawait;
+        break;
+      }
+
+      // Some of the above can be both binary and unary operations, so make sure
+      // we get the right one.
+      if (Inf.Operator != UO_PostInc && Inf.Operator != UO_PostDec &&
+          OpCall->getNumArgs() != 1)
+        return std::nullopt;
+
+      Inf.SubExpr = OpCall->getArg(0);
+      return Inf;
+    }
+    return std::nullopt;
+  }
+
+  // Get a normalized version of a binary operator.
+  std::optional<BinaryOpInfo> GetBinaryOperatorInfo(const Expr *E) {
+    if (const auto *BO = dyn_cast<BinaryOperator>(E))
+      return BinaryOpInfo{BO, BO->getLHS()->IgnoreImpCasts(),
+                          BO->getRHS()->IgnoreImpCasts(), BO->getOpcode()};
+
+    // In case this is an operator-call, which allows us to support overloaded
+    // operators and dependent expression.
+    if (const auto *OpCall = dyn_cast<CXXOperatorCallExpr>(E)) {
+      BinaryOpInfo Inf;
+      Inf.FoundExpr = OpCall;
+
+      switch (OpCall->getOperator()) {
+      default:
+        return std::nullopt;
+      case OO_Plus:
+        Inf.Operator = BO_Add;
+        break;
+      case OO_Minus:
+        Inf.Operator = BO_Sub;
+        break;
+      case OO_Star:
+        Inf.Operator = BO_Mul;
+        break;
+      case OO_Slash:
+        Inf.Operator = BO_Div;
+        break;
+      case OO_Percent:
+        Inf.Operator = BO_Rem;
+        break;
+      case OO_Caret:
+        Inf.Operator = BO_Xor;
+        break;
+      case OO_Amp:
+        Inf.Operator = BO_And;
+        break;
+      case OO_Pipe:
+        Inf.Operator = BO_Or;
+        break;
+      case OO_Equal:
+        Inf.Operator = BO_Assign;
+        break;
+      case OO_Spaceship:
+        Inf.Operator = BO_Cmp;
+        break;
+      case OO_Less:
+        Inf.Operator = BO_LT;
+        break;
+      case OO_Greater:
+        Inf.Operator = BO_GT;
+        break;
+      case OO_PlusEqual:
+        Inf.Operator = BO_AddAssign;
+        break;
+      case OO_MinusEqual:
+        Inf.Operator = BO_SubAssign;
+        break;
+      case OO_StarEqual:
+        Inf.Operator = BO_MulAssign;
+        break;
+      case OO_SlashEqual:
+        Inf.Operator = BO_DivAssign;
+        break;
+      case OO_PercentEqual:
+        Inf.Operator = BO_RemAssign;
+        break;
+      case OO_CaretEqual:
+        Inf.Operator = BO_XorAssign;
+        break;
+      case OO_AmpEqual:
+        Inf.Operator = BO_AndAssign;
+        break;
+      case OO_PipeEqual:
+        Inf.Operator = BO_OrAssign;
+        break;
+      case OO_LessLess:
+        Inf.Operator = BO_Shl;
+        break;
+      case OO_GreaterGreater:
+        Inf.Operator = BO_Shr;
+        break;
+      case OO_LessLessEqual:
+        Inf.Operator = BO_ShlAssign;
+        break;
+      case OO_GreaterGreaterEqual:
+        Inf.Operator = BO_ShrAssign;
+        break;
+      case OO_EqualEqual:
+        Inf.Operator = BO_EQ;
+        break;
+      case OO_ExclaimEqual:
+        Inf.Operator = BO_NE;
+        break;
+      case OO_LessEqual:
+        Inf.Operator = BO_LE;
+        break;
+      case OO_GreaterEqual:
+        Inf.Operator = BO_GE;
+        break;
+      case OO_AmpAmp:
+        Inf.Operator = BO_LAnd;
+        break;
+      case OO_PipePipe:
+        Inf.Operator = BO_LOr;
+        break;
+      case OO_Comma:
+        Inf.Operator = BO_Comma;
+        break;
+      case OO_ArrowStar:
+        Inf.Operator = BO_PtrMemI;
+        break;
+      }
+
+      // This isn't a binary operator unless there are two arguments.
+      if (OpCall->getNumArgs() != 2)
+        return std::nullopt;
+
+      // Callee is the call-operator, so we only need to extract the two
+      // arguments here.
+      Inf.LHS = OpCall->getArg(0)->IgnoreImpCasts();
+      Inf.RHS = OpCall->getArg(1)->IgnoreImpCasts();
+      return Inf;
+    }
+
+    return std::nullopt;
+  }
+
+  // Checks a required assignment operation, but don't check the LHS or RHS,
+  // callers have to do that here.
+  std::optional<BinaryOpInfo> CheckAssignment(const Expr *E) {
+    std::optional<BinaryOpInfo> Inf = GetBinaryOperatorInfo(E);
+
+    if (!Inf) {
+      DiagnoseInvalidAtomic(E->getExprLoc(),
+                            SemaRef.PDiag(diag::note_acc_atomic_expr_must_be)
+                                << diag::OACCAtomicExpr::Assign);
+      return std::nullopt;
+    }
+
+    if (Inf->Operator != BO_Assign) {
+      DiagnoseInvalidAtomic(Inf->FoundExpr->getExprLoc(),
+                            SemaRef.PDiag(diag::note_acc_atomic_expr_must_be)
+                                << diag::OACCAtomicExpr::Assign);
+      return std::nullopt;
+    }
+
+    // Assignment always requires an lvalue/scalar on the LHS.
+    if (CheckOperandVariable(
+            Inf->LHS, SemaRef.PDiag(diag::note_acc_atomic_operand_lvalue_scalar)
+                          << /*left=*/0 << diag::OACCAtomicOpKind::Assign))
+      return std::nullopt;
+
+    return Inf;
+  }
+
+  struct IDACInfo {
+    bool Failed = false;
+    enum ExprKindTy {
+      Invalid,
+      // increment/decrement ops.
+      Unary,
+      // v = x
+      SimpleAssign,
+      // x = expr
+      ExprAssign,
+      // x binop= expr
+      CompoundAssign,
+      // x = x binop expr
+      // x = expr binop x
+      AssignBinOp
+    } ExprKind;
+
+    // The variable referred to as 'x' in all of the grammar, such that it is
+    // needed in compound statement checking of capture to check between the two
+    // expressions.
+    const Expr *X_Var = nullptr;
+
+    static IDACInfo Fail() { return IDACInfo{true, Invalid, nullptr}; };
+  };
+
+  // Helper for CheckIncDecAssignCompoundAssign, does checks for inc/dec.
+  IDACInfo CheckIncDec(UnaryOpInfo Inf) {
+
+    if (!UnaryOperator::isIncrementDecrementOp(Inf.Operator)) {
+      DiagnoseInvalidAtomic(
+          Inf.FoundExpr->getExprLoc(),
+          SemaRef.PDiag(diag::note_acc_atomic_unsupported_unary_operator));
+      return IDACInfo::Fail();
+    }
+    bool Failed = CheckOperandVariable(
+        Inf.SubExpr,
+        SemaRef.PDiag(diag::note_acc_atomic_operand_lvalue_scalar)
+            << /*none=*/2
+            << (Inf.IsIncrementOp() ? diag::OACCAtomicOpKind::Inc
+                                    : diag::OACCAtomicOpKind::Dec));
+    // For increment/decrements, the subexpr is the 'x' (x++, ++x, etc).
+    return IDACInfo{Failed, IDACInfo::Unary, Inf.SubExpr};
+  }
+
+  enum class SimpleAssignKind { None, Var, Expr };
+
+  // Check an assignment, and ensure the RHS is either x binop expr or expr
+  // binop x.
+  // If AllowSimpleAssign, also allows v = x;
+  IDACInfo CheckAssignmentWithBinOpOnRHS(BinaryOpInfo AssignInf,
+                                         SimpleAssignKind SAK) {
+    PartialDiagnostic PD =
+        SemaRef.PDiag(diag::note_acc_atomic_operand_lvalue_scalar)
+        << /*left=*/0 << diag::OACCAtomicOpKind::Assign;
+    if (CheckOperandVariable(AssignInf.LHS, PD))
+      return IDACInfo::Fail();
+
+    std::optional<BinaryOpInfo> BinInf = GetBinaryOperatorInfo(AssignInf.RHS);
+
+    if (!BinInf) {
+
+      // Capture in a compound statement allows v = x assignment.  So make sure
+      // we permit that here.
+      if (SAK != SimpleAssignKind::None) {
+        PartialDiagnostic PD =
+            SemaRef.PDiag(diag::note_acc_atomic_operand_lvalue_scalar)
+            << /*right=*/1 << diag::OACCAtomicOpKind::Assign;
+        if (SAK == SimpleAssignKind::Var) {
+          // In the var version, everywhere we allow v = x;, X is the RHS.
+          return IDACInfo{CheckOperandVariable(AssignInf.RHS, PD),
+                          IDACInfo::SimpleAssign, AssignInf.RHS};
+        }
+        assert(SAK == SimpleAssignKind::Expr);
+        // In the expression version, supported by v=x; x = expr;, we need to
+        // set to the LHS here.
+        return IDACInfo{CheckOperandExpr(AssignInf.RHS, PD),
+                        IDACInfo::ExprAssign, AssignInf.LHS};
+      }
+
+      DiagnoseInvalidAtomic(
+          AssignInf.RHS->getExprLoc(),
+          SemaRef.PDiag(diag::note_acc_atomic_expected_binop));
+
+      return IDACInfo::Fail();
+    }
+    switch (BinInf->Operator) {
+    default:
+      DiagnoseInvalidAtomic(
+          BinInf->FoundExpr->getExprLoc(),
+          SemaRef.PDiag(diag::note_acc_atomic_unsupported_binary_operator));
+      return IDACInfo::Fail();
+      // binop is one of +, *, -, /, &, ^, |, <<, or >>
+    case BO_Add:
+    case BO_Mul:
+    case BO_Sub:
+    case BO_Div:
+    case BO_And:
+    case BO_Xor:
+    case BO_Or:
+    case BO_Shl:
+    case BO_Shr:
+      // Handle these outside of the switch.
+      break;
+    }
+
+    llvm::FoldingSetNodeID LHS_ID, InnerLHS_ID, InnerRHS_ID;
+    AssignInf.LHS->Profile(LHS_ID, SemaRef.getASTContext(),
+                           /*Canonical=*/true);
+    BinInf->LHS->Profile(InnerLHS_ID, SemaRef.getASTContext(),
+                         /*Canonical=*/true);
+
+    // This is X = X binop expr;
+    // Check the RHS is an expression.
+    if (LHS_ID == InnerLHS_ID)
+      return IDACInfo{
+          CheckOperandExpr(
+              BinInf->RHS,
+              SemaRef.PDiag(diag::note_acc_atomic_operand_lvalue_scalar
+                            << /*right=*/1
+                            << diag::OACCAtomicOpKind::CompoundAssign)),
+          IDACInfo::AssignBinOp, AssignInf.LHS};
+
+    BinInf->RHS->Profile(InnerRHS_ID, SemaRef.getASTContext(),
+                         /*Canonical=*/true);
+    // This is X = expr binop X;
+    // Check the LHS is an expression
+    if (LHS_ID == InnerRHS_ID)
+      return IDACInfo{
+          CheckOperandExpr(
+              BinInf->LHS,
+              SemaRef.PDiag(diag::note_acc_atomic_operand_lvalue_scalar)
+                  << /*left=*/0 << diag::OACCAtomicOpKind::CompoundAssign),
+          IDACInfo::AssignBinOp, AssignInf.LHS};
+
+    // If nothing matches, error out.
+    DiagnoseInvalidAtomic(BinInf->FoundExpr->getExprLoc(),
+                          SemaRef.PDiag(diag::note_acc_atomic_mismatch_operand)
+                              << const_cast<Expr *>(AssignInf.LHS)
+                              << const_cast<Expr *>(BinInf->LHS)
+                              << const_cast<Expr *>(BinInf->RHS));
+    return IDACInfo::Fail();
+  }
+
+  // Ensures that the expression is an increment/decrement, an assignment, or a
+  // compound assignment. If its an assignment, allows the x binop expr/x binop
+  // expr syntax. If it is a compound-assignment, allows any expr on the RHS.
+  IDACInfo CheckIncDecAssignCompoundAssign(const Expr *E,
+                                           SimpleAssignKind SAK) {
+    std::optional<UnaryOpInfo> UInf = GetUnaryOperatorInfo(E);
+
+    // If this is a unary operator, only increment/decrement are allowed, so get
+    // unary operator, then check everything we can.
+    if (UInf)
+      return CheckIncDec(*UInf);
+
+    std::optional<BinaryOpInfo> BinInf = GetBinaryOperatorInfo(E);
+
+    // Unary or binary operator were the only choices, so error here.
+    if (!BinInf) {
+      DiagnoseInvalidAtomic(E->getExprLoc(),
+                            SemaRef.PDiag(diag::note_acc_atomic_expr_must_be)
+                                << diag::OACCAtomicExpr::UnaryCompAssign);
+      return IDACInfo::Fail();
+    }
+
+    switch (BinInf->Operator) {
+    default:
+      DiagnoseInvalidAtomic(
+          BinInf->FoundExpr->getExprLoc(),
+          SemaRef.PDiag(
+              diag::note_acc_atomic_unsupported_compound_binary_operator));
+      return IDACInfo::Fail();
+    case BO_Assign:
+      return CheckAssignmentWithBinOpOnRHS(*BinInf, SAK);
+    case BO_AddAssign:
+    case BO_MulAssign:
+    case BO_SubAssign:
+    case BO_DivAssign:
+    case BO_AndAssign:
+    case BO_XorAssign:
+    case BO_OrAssign:
+    case BO_ShlAssign:
+    case BO_ShrAssign: {
+      PartialDiagnostic LPD =
+          SemaRef.PDiag(diag::note_acc_atomic_operand_lvalue_scalar)
+          << /*left=*/0 << diag::OACCAtomicOpKind::CompoundAssign;
+      PartialDiagnostic RPD =
+          SemaRef.PDiag(diag::note_acc_atomic_operand_lvalue_scalar)
+          << /*right=*/1 << diag::OACCAtomicOpKind::CompoundAssign;
+      // nothing to do other than check the variable expressions.
+      // success or failure
+      bool Failed = CheckOperandVariable(BinInf->LHS, LPD) ||
+                    CheckOperandExpr(BinInf->RHS, RPD);
+
+      return IDACInfo{Failed, IDACInfo::CompoundAssign, BinInf->LHS};
+    }
+    }
+    llvm_unreachable("all binary operator kinds should be checked above");
+  }
+
+  StmtResult CheckRead() {
+    Expr *AssocExpr = RequireExpr(
+        AssocStmt.get(), SemaRef.PDiag(diag::note_acc_atomic_expr_must_be)
+                             << diag::OACCAtomicExpr::Assign);
+
+    if (!AssocExpr)
+      return getRecoveryExpr();
+
+    std::optional<BinaryOpInfo> AssignRes = CheckAssignment(AssocExpr);
+    if (!AssignRes)
+      return getRecoveryExpr();
+
+    PartialDiagnostic PD =
+        SemaRef.PDiag(diag::note_acc_atomic_operand_lvalue_scalar)
+        << /*right=*/1 << diag::OACCAtomicOpKind::Assign;
+
+    // Finally, check the RHS.
+    if (CheckOperandVariable(AssignRes->RHS, PD))
+      return getRecoveryExpr();
+
+    return AssocStmt;
+  }
+
+  StmtResult CheckWrite() {
+    Expr *AssocExpr = RequireExpr(
+        AssocStmt.get(), SemaRef.PDiag(diag::note_acc_atomic_expr_must_be)
+                             << diag::OACCAtomicExpr::Assign);
+
+    if (!AssocExpr)
+      return getRecoveryExpr();
+
+    std::optional<BinaryOpInfo> AssignRes = CheckAssignment(AssocExpr);
+    if (!AssignRes)
+      return getRecoveryExpr();
+
+    PartialDiagnostic PD =
+        SemaRef.PDiag(diag::note_acc_atomic_operand_lvalue_scalar)
+        << /*right=*/1 << diag::OACCAtomicOpKind::Assign;
+
+    // Finally, check the RHS.
+    if (CheckOperandExpr(AssignRes->RHS, PD))
+      return getRecoveryExpr();
+
+    return AssocStmt;
+  }
+
+  StmtResult CheckUpdate() {
+    Expr *AssocExpr = RequireExpr(
+        AssocStmt.get(), SemaRef.PDiag(diag::note_acc_atomic_expr_must_be)
+                             << diag::OACCAtomicExpr::UnaryCompAssign);
+
+    if (!AssocExpr ||
+        CheckIncDecAssignCompoundAssign(AssocExpr, SimpleAssignKind::None)
+            .Failed)
+      return getRecoveryExpr();
+
+    return AssocStmt;
+  }
+
+  bool CheckVarRefsSame(IDACInfo::ExprKindTy FirstKind, const Expr *FirstX,
+                        IDACInfo::ExprKindTy SecondKind, const Expr *SecondX) {
+    llvm::FoldingSetNodeID First_ID, Second_ID;
+    FirstX->Profile(First_ID, SemaRef.getASTContext(), /*Canonical=*/true);
+    SecondX->Profile(Second_ID, SemaRef.getASTContext(), /*Canonical=*/true);
+
+    if (First_ID == Second_ID)
+      return false;
+
+    PartialDiagnostic PD =
+        SemaRef.PDiag(diag::note_acc_atomic_mismatch_compound_operand)
+        << FirstKind << const_cast<Expr *>(FirstX) << SecondKind
+        << const_cast<Expr *>(SecondX);
+
+    return DiagnoseInvalidAtomic(SecondX->getExprLoc(), PD);
+  }
+
+  StmtResult CheckCapture() {
+    if (const auto *CmpdStmt = dyn_cast<CompoundStmt>(AssocStmt.get())) {
+      auto *const *BodyItr = CmpdStmt->body().begin();
+      PartialDiagnostic PD = SemaRef.PDiag(diag::note_acc_atomic_expr_must_be)
+                             << diag::OACCAtomicExpr::UnaryCompAssign;
+      // If we don't have at least 1 statement, error.
+      if (BodyItr == CmpdStmt->body().end()) {
+        DiagnoseInvalidAtomic(CmpdStmt->getBeginLoc(), PD);
+        return getRecoveryExpr();
+      }
+
+      // First Expr can be inc/dec, assign, or compound assign.
+      Expr *FirstExpr = RequireExpr(*BodyItr, PD);
+      if (!FirstExpr)
+        return getRecoveryExpr();
+
+      IDACInfo FirstExprResults =
+          CheckIncDecAssignCompoundAssign(FirstExpr, SimpleAssignKind::Var);
+      if (FirstExprResults.Failed)
+        return getRecoveryExpr();
+
+      ++BodyItr;
+
+      // If we don't have second statement, error.
+      if (BodyItr == CmpdStmt->body().end()) {
+        DiagnoseInvalidAtomic(CmpdStmt->getEndLoc(), PD);
+        return getRecoveryExpr();
+      }
+
+      Expr *SecondExpr = RequireExpr(*BodyItr, PD);
+      if (!SecondExpr)
+        return getRecoveryExpr();
+
+      assert(FirstExprResults.ExprKind != IDACInfo::Invalid);
+
+      switch (FirstExprResults.ExprKind) {
+      case IDACInfo::Invalid:
+      case IDACInfo::ExprAssign:
+        llvm_unreachable("Should have error'ed out by now");
+      case IDACInfo::Unary:
+      case IDACInfo::CompoundAssign:
+      case IDACInfo::AssignBinOp: {
+        // Everything but simple-assign can only be followed by a simple
+        // assignment.
+        std::optional<BinaryOpInfo> AssignRes = CheckAssignment(SecondExpr);
+        if (!AssignRes)
+          return getRecoveryExpr();
+
+        PartialDiagnostic PD =
+            SemaRef.PDiag(diag::note_acc_atomic_operand_lvalue_scalar)
+            << /*right=*/1 << diag::OACCAtomicOpKind::Assign;
+
+        if (CheckOperandVariable(AssignRes->RHS, PD))
+          return getRecoveryExpr();
+
+        if (CheckVarRefsSame(FirstExprResults.ExprKind, FirstExprResults.X_Var,
+                             IDACInfo::SimpleAssign, AssignRes->RHS))
+          return getRecoveryExpr();
+        break;
+      }
+      case IDACInfo::SimpleAssign: {
+        // If the first was v = x, anything but simple expression is allowed.
+        IDACInfo SecondExprResults =
+            CheckIncDecAssignCompoundAssign(SecondExpr, SimpleAssignKind::Expr);
+        if (SecondExprResults.Failed)
+          return getRecoveryExpr();
+
+        if (CheckVarRefsSame(FirstExprResults.ExprKind, FirstExprResults.X_Var,
+                             SecondExprResults.ExprKind,
+                             SecondExprResults.X_Var))
+          return getRecoveryExpr();
+        break;
+      }
+      }
+      ++BodyItr;
+      if (BodyItr != CmpdStmt->body().end()) {
+        DiagnoseInvalidAtomic(
+            (*BodyItr)->getBeginLoc(),
+            SemaRef.PDiag(diag::note_acc_atomic_too_many_stmts));
+        return getRecoveryExpr();
+      }
+    } else {
+      // This check doesn't need to happen if it is a compound stmt.
+      Expr *AssocExpr = RequireExpr(
+          AssocStmt.get(), SemaRef.PDiag(diag::note_acc_atomic_expr_must_be)
+                               << diag::OACCAtomicExpr::Assign);
+      if (!AssocExpr)
+        return getRecoveryExpr();
+
+      // First, we require an assignment.
+      std::optional<BinaryOpInfo> AssignRes = CheckAssignment(AssocExpr);
+
+      if (!AssignRes)
+        return getRecoveryExpr();
+
+      if (CheckIncDecAssignCompoundAssign(AssignRes->RHS,
+                                          SimpleAssignKind::None)
+              .Failed)
+        return getRecoveryExpr();
+    }
+
+    return AssocStmt;
+  }
+
+public:
+  AtomicOperandChecker(SemaOpenACC &S, OpenACCAtomicKind AtKind,
+                       SourceLocation DirLoc, StmtResult AssocStmt)
+      : SemaRef(S), AtKind(AtKind), AtomicDirLoc(DirLoc), AssocStmt(AssocStmt) {
+  }
+
+  StmtResult Check() {
+
+    switch (AtKind) {
+    case OpenACCAtomicKind::Read:
+      return CheckRead();
+    case OpenACCAtomicKind::Write:
+      return CheckWrite();
+    case OpenACCAtomicKind::None:
+    case OpenACCAtomicKind::Update:
+      return CheckUpdate();
+    case OpenACCAtomicKind::Capture:
+      return CheckCapture();
+    }
+    llvm_unreachable("Unhandled atomic kind?");
+  }
+};
+} // namespace
+
+StmtResult SemaOpenACC::CheckAtomicAssociatedStmt(SourceLocation AtomicDirLoc,
+                                                  OpenACCAtomicKind AtKind,
+                                                  StmtResult AssocStmt) {
+  if (!AssocStmt.isUsable())
+    return AssocStmt;
+
+  if (isa<RecoveryExpr>(AssocStmt.get()))
+    return AssocStmt;
+
+  AtomicOperandChecker Checker{*this, AtKind, AtomicDirLoc, AssocStmt};
+  return Checker.Check();
+}
diff --git a/clang/lib/Sema/SemaOpenACCClause.cpp b/clang/lib/Sema/SemaOpenACCClause.cpp
index 000934225402a..1e74f126c31ce 100644
--- a/clang/lib/Sema/SemaOpenACCClause.cpp
+++ b/clang/lib/Sema/SemaOpenACCClause.cpp
@@ -589,7 +589,6 @@ bool checkValidAfterDeviceType(
 // construct has been implemented.
 bool isDirectiveKindImplemented(OpenACCDirectiveKind DK) {
   return DK != OpenACCDirectiveKind::Declare &&
-         DK != OpenACCDirectiveKind::Atomic &&
          DK != OpenACCDirectiveKind::Routine;
 }
 
diff --git a/clang/lib/Sema/SemaOpenMP.cpp b/clang/lib/Sema/SemaOpenMP.cpp
index b83b2b12f4a23..39ce65381a98c 100644
--- a/clang/lib/Sema/SemaOpenMP.cpp
+++ b/clang/lib/Sema/SemaOpenMP.cpp
@@ -4788,13 +4788,26 @@ static bool checkNestingOfRegions(Sema &SemaRef, const DSAStackTy *Stack,
       getLeafOrCompositeConstructs(ParentRegion, LeafOrComposite);
   OpenMPDirectiveKind EnclosingConstruct = ParentLOC.back();
 
-  if (SemaRef.LangOpts.OpenMP >= 51 && Stack->isParentOrderConcurrent() &&
-      CurrentRegion != OMPD_simd && CurrentRegion != OMPD_loop &&
-      CurrentRegion != OMPD_parallel &&
-      !isOpenMPCombinedParallelADirective(CurrentRegion)) {
-    SemaRef.Diag(StartLoc, diag::err_omp_prohibited_region_order)
-        << getOpenMPDirectiveName(CurrentRegion);
-    return true;
+  if (Stack->isParentOrderConcurrent()) {
+    bool InvalidOrderNesting = false;
+    if ((SemaRef.LangOpts.OpenMP == 51 || SemaRef.LangOpts.OpenMP == 52) &&
+        CurrentRegion != OMPD_simd && CurrentRegion != OMPD_loop &&
+        CurrentRegion != OMPD_parallel &&
+        !isOpenMPCombinedParallelADirective(CurrentRegion)) {
+      InvalidOrderNesting = true;
+    } else if (SemaRef.LangOpts.OpenMP >= 60 &&
+               !isOpenMPOrderConcurrentNestableDirective(CurrentRegion)) {
+      // OpenMP 6.0 [12.3 order Clause, Restrictions]
+      // Only regions that correspond to order-concurrent-nestable constructs
+      // or order-concurrent-nestable routines may be strictly nested regions
+      // of regions that correspond to constructs on which the order clause is
+      // specified with concurrent as the ordering argument.
+      InvalidOrderNesting = true;
+    }
+    if (InvalidOrderNesting) {
+      SemaRef.Diag(StartLoc, diag::err_omp_prohibited_region_order)
+          << getOpenMPDirectiveName(CurrentRegion);
+    }
   }
   if (isOpenMPSimdDirective(ParentRegion) &&
       ((SemaRef.LangOpts.OpenMP <= 45 && CurrentRegion != OMPD_ordered) ||
@@ -5065,7 +5078,8 @@ static bool checkIfClauses(Sema &S, OpenMPDirectiveKind Kind,
       // At most one if clause without a directive-name-modifier can appear on
       // the directive.
       OpenMPDirectiveKind CurNM = IC->getNameModifier();
-      if (FoundNameModifiers[CurNM]) {
+      auto &FNM = FoundNameModifiers[CurNM];
+      if (FNM) {
         S.Diag(C->getBeginLoc(), diag::err_omp_more_one_clause)
             << getOpenMPDirectiveName(Kind) << getOpenMPClauseName(OMPC_if)
             << (CurNM != OMPD_unknown) << getOpenMPDirectiveName(CurNM);
@@ -5074,7 +5088,7 @@ static bool checkIfClauses(Sema &S, OpenMPDirectiveKind Kind,
         NameModifierLoc.push_back(IC->getNameModifierLoc());
         ++NamedModifiersNumber;
       }
-      FoundNameModifiers[CurNM] = IC;
+      FNM = IC;
       if (CurNM == OMPD_unknown)
         continue;
       // Check if the specified name modifier is allowed for the current
@@ -6746,16 +6760,15 @@ SemaOpenMP::DeclGroupPtrTy SemaOpenMP::ActOnOpenMPDeclareSimdDirective(
                     ->getCanonicalDecl() == CanonPVD) {
           // OpenMP  [2.8.1, simd construct, Restrictions]
           // A list-item cannot appear in more than one aligned clause.
-          if (AlignedArgs.count(CanonPVD) > 0) {
+          auto [It, Inserted] = AlignedArgs.try_emplace(CanonPVD, E);
+          if (!Inserted) {
             Diag(E->getExprLoc(), diag::err_omp_used_in_clause_twice)
                 << 1 << getOpenMPClauseName(OMPC_aligned)
                 << E->getSourceRange();
-            Diag(AlignedArgs[CanonPVD]->getExprLoc(),
-                 diag::note_omp_explicit_dsa)
+            Diag(It->second->getExprLoc(), diag::note_omp_explicit_dsa)
                 << getOpenMPClauseName(OMPC_aligned);
             continue;
           }
-          AlignedArgs[CanonPVD] = E;
           QualType QTy = PVD->getType()
                              .getNonReferenceType()
                              .getUnqualifiedType()
@@ -7114,7 +7127,8 @@ ExprResult SemaOpenMP::ActOnOpenMPCall(ExprResult Call, Scope *Scope,
   if (!CalleeFnDecl)
     return Call;
 
-  if (getLangOpts().OpenMP >= 51 && CalleeFnDecl->getIdentifier() &&
+  if (getLangOpts().OpenMP >= 51 && getLangOpts().OpenMP < 60 &&
+      CalleeFnDecl->getIdentifier() &&
       CalleeFnDecl->getName().starts_with_insensitive("omp_")) {
     // checking for any calls inside an Order region
     if (Scope && Scope->isOpenMPOrderClauseScope())
@@ -7134,7 +7148,7 @@ ExprResult SemaOpenMP::ActOnOpenMPCall(ExprResult Call, Scope *Scope,
   };
   TargetOMPContext OMPCtx(Context, std::move(DiagUnknownTrait),
                           SemaRef.getCurFunctionDecl(),
-                          DSAStack->getConstructTraits());
+                          DSAStack->getConstructTraits(), getOpenMPDeviceNum());
 
   QualType CalleeFnType = CalleeFnDecl->getType();
 
@@ -15631,6 +15645,38 @@ ExprResult SemaOpenMP::VerifyPositiveIntegerConstantInClause(
   return ICE;
 }
 
+void SemaOpenMP::setOpenMPDeviceNum(int Num) { DeviceNum = Num; }
+
+void SemaOpenMP::setOpenMPDeviceNumID(StringRef ID) { DeviceNumID = ID; }
+
+int SemaOpenMP::getOpenMPDeviceNum() const { return DeviceNum; }
+
+void SemaOpenMP::ActOnOpenMPDeviceNum(Expr *DeviceNumExpr) {
+  llvm::APSInt Result;
+  Expr::EvalResult EvalResult;
+  // Evaluate the expression to an integer value
+  if (!DeviceNumExpr->isValueDependent() &&
+      DeviceNumExpr->EvaluateAsInt(EvalResult, SemaRef.Context)) {
+    // The device expression must evaluate to a non-negative integer value.
+    Result = EvalResult.Val.getInt();
+    if (Result.isNonNegative()) {
+      setOpenMPDeviceNum(Result.getZExtValue());
+    } else {
+      Diag(DeviceNumExpr->getExprLoc(),
+           diag::err_omp_negative_expression_in_clause)
+          << "device_num" << 0 << DeviceNumExpr->getSourceRange();
+    }
+  } else if (auto *DeclRef = dyn_cast<DeclRefExpr>(DeviceNumExpr)) {
+    // Check if the expression is an identifier
+    IdentifierInfo *IdInfo = DeclRef->getDecl()->getIdentifier();
+    if (IdInfo) {
+      setOpenMPDeviceNumID(IdInfo->getName());
+    }
+  } else {
+    Diag(DeviceNumExpr->getExprLoc(), diag::err_expected_expression);
+  }
+}
+
 OMPClause *SemaOpenMP::ActOnOpenMPSafelenClause(Expr *Len,
                                                 SourceLocation StartLoc,
                                                 SourceLocation LParenLoc,
@@ -22773,8 +22819,12 @@ class GlobalDeclRefChecker final : public StmtVisitor<GlobalDeclRefChecker> {
   void declareTargetInitializer(Decl *TD) {
     A = TD->getAttr<OMPDeclareTargetDeclAttr>();
     DeclVector.push_back(cast<VarDecl>(TD));
+    llvm::SmallDenseSet<Decl *> Visited;
     while (!DeclVector.empty()) {
       VarDecl *TargetVarDecl = DeclVector.pop_back_val();
+      if (!Visited.insert(TargetVarDecl).second)
+        continue;
+
       if (TargetVarDecl->hasAttr<OMPDeclareTargetDeclAttr>() &&
           TargetVarDecl->hasInit() && TargetVarDecl->hasGlobalStorage()) {
         if (Expr *Ex = TargetVarDecl->getInit())
@@ -23717,6 +23767,8 @@ OMPClause *SemaOpenMP::ActOnOpenMPNullaryAssumptionClause(OpenMPClauseKind CK,
     return new (getASTContext()) OMPNoOpenMPRoutinesClause(Loc, RLoc);
   case OMPC_no_parallelism:
     return new (getASTContext()) OMPNoParallelismClause(Loc, RLoc);
+  case OMPC_no_openmp_constructs:
+    return new (getASTContext()) OMPNoOpenMPConstructsClause(Loc, RLoc);
   default:
     llvm_unreachable("Unexpected OpenMP clause");
   }
diff --git a/clang/lib/Sema/SemaOverload.cpp b/clang/lib/Sema/SemaOverload.cpp
index 6ae9c51c06b31..8d5b5ac190b5b 100644
--- a/clang/lib/Sema/SemaOverload.cpp
+++ b/clang/lib/Sema/SemaOverload.cpp
@@ -6151,8 +6151,8 @@ static ExprResult BuildConvertedConstantExpression(Sema &S, Expr *From,
                                                    Sema::CCEKind CCE,
                                                    NamedDecl *Dest,
                                                    APValue &PreNarrowingValue) {
-  assert(S.getLangOpts().CPlusPlus11 &&
-         "converted constant expression outside C++11");
+  assert((S.getLangOpts().CPlusPlus11 || CCE == Sema::CCEK_InjectedTTP) &&
+         "converted constant expression outside C++11 or TTP matching");
 
   if (checkPlaceholderForOverload(S, From))
     return ExprError();
@@ -6221,8 +6221,10 @@ static ExprResult BuildConvertedConstantExpression(Sema &S, Expr *From,
   // earlier, but that's not guaranteed to work when initializing an object of
   // class type.
   ExprResult Result;
+  bool IsTemplateArgument =
+      CCE == Sema::CCEK_TemplateArg || CCE == Sema::CCEK_InjectedTTP;
   if (T->isRecordType()) {
-    assert(CCE == Sema::CCEK_TemplateArg &&
+    assert(IsTemplateArgument &&
            "unexpected class type converted constant expr");
     Result = S.PerformCopyInitialization(
         InitializedEntity::InitializeTemplateParameter(
@@ -6239,7 +6241,7 @@ static ExprResult BuildConvertedConstantExpression(Sema &S, Expr *From,
   //   A full-expression is [...] a constant-expression [...]
   Result = S.ActOnFinishFullExpr(Result.get(), From->getExprLoc(),
                                  /*DiscardedValue=*/false, /*IsConstexpr=*/true,
-                                 CCE == Sema::CCEKind::CCEK_TemplateArg);
+                                 IsTemplateArgument);
   if (Result.isInvalid())
     return Result;
 
@@ -6248,9 +6250,6 @@ static ExprResult BuildConvertedConstantExpression(Sema &S, Expr *From,
   QualType PreNarrowingType;
   switch (SCS->getNarrowingKind(S.Context, Result.get(), PreNarrowingValue,
                                 PreNarrowingType)) {
-  case NK_Dependent_Narrowing:
-    // Implicit conversion to a narrower type, but the expression is
-    // value-dependent so we can't tell whether it's actually narrowing.
   case NK_Variable_Narrowing:
     // Implicit conversion to a narrower type, and the value is not a constant
     // expression. We'll diagnose this in a moment.
@@ -6271,6 +6270,14 @@ static ExprResult BuildConvertedConstantExpression(Sema &S, Expr *From,
         << PreNarrowingValue.getAsString(S.Context, PreNarrowingType) << T;
     break;
 
+  case NK_Dependent_Narrowing:
+    // Implicit conversion to a narrower type, but the expression is
+    // value-dependent so we can't tell whether it's actually narrowing.
+    // For matching the parameters of a TTP, the conversion is ill-formed
+    // if it may narrow.
+    if (CCE != Sema::CCEK_InjectedTTP)
+      break;
+    [[fallthrough]];
   case NK_Type_Narrowing:
     // FIXME: It would be better to diagnose that the expression is not a
     // constant expression.
@@ -6343,6 +6350,8 @@ Sema::EvaluateConvertedConstantExpression(Expr *E, QualType T, APValue &Value,
   Expr::EvalResult Eval;
   Eval.Diag = &Notes;
 
+  assert(CCE != Sema::CCEK_InjectedTTP && "unnexpected CCE Kind");
+
   ConstantExprKind Kind;
   if (CCE == Sema::CCEK_TemplateArg && T->isRecordType())
     Kind = ConstantExprKind::ClassTemplateArgument;
@@ -6918,7 +6927,7 @@ void Sema::AddOverloadCandidate(
     bool PartialOverloading, bool AllowExplicit, bool AllowExplicitConversions,
     ADLCallKind IsADLCandidate, ConversionSequenceList EarlyConversions,
     OverloadCandidateParamOrder PO, bool AggregateCandidateDeduction,
-    bool HasMatchedPackOnParmToNonPackOnArg) {
+    bool StrictPackMatch) {
   const FunctionProtoType *Proto
     = dyn_cast<FunctionProtoType>(Function->getType()->getAs<FunctionType>());
   assert(Proto && "Functions without a prototype cannot be overloaded");
@@ -6938,7 +6947,7 @@ void Sema::AddOverloadCandidate(
                          Expr::Classification::makeSimpleLValue(), Args,
                          CandidateSet, SuppressUserConversions,
                          PartialOverloading, EarlyConversions, PO,
-                         HasMatchedPackOnParmToNonPackOnArg);
+                         StrictPackMatch);
       return;
     }
     // We treat a constructor like a non-member function, since its object
@@ -6981,8 +6990,7 @@ void Sema::AddOverloadCandidate(
       CandidateSet.getRewriteInfo().getRewriteKind(Function, PO);
   Candidate.IsADLCandidate = llvm::to_underlying(IsADLCandidate);
   Candidate.ExplicitCallArguments = Args.size();
-  Candidate.HasMatchedPackOnParmToNonPackOnArg =
-      HasMatchedPackOnParmToNonPackOnArg;
+  Candidate.StrictPackMatch = StrictPackMatch;
 
   // Explicit functions are not actually candidates at all if we're not
   // allowing them in this context, but keep them around so we can point
@@ -7377,8 +7385,10 @@ static bool diagnoseDiagnoseIfAttrsWith(Sema &S, const NamedDecl *ND,
     return false;
 
   auto WarningBegin = std::stable_partition(
-      Attrs.begin(), Attrs.end(),
-      [](const DiagnoseIfAttr *DIA) { return DIA->isError(); });
+      Attrs.begin(), Attrs.end(), [](const DiagnoseIfAttr *DIA) {
+        return DIA->getDefaultSeverity() == DiagnoseIfAttr::DS_error &&
+               DIA->getWarningGroup().empty();
+      });
 
   // Note that diagnose_if attributes are late-parsed, so they appear in the
   // correct order (unlike enable_if attributes).
@@ -7392,11 +7402,32 @@ static bool diagnoseDiagnoseIfAttrsWith(Sema &S, const NamedDecl *ND,
     return true;
   }
 
+  auto ToSeverity = [](DiagnoseIfAttr::DefaultSeverity Sev) {
+    switch (Sev) {
+    case DiagnoseIfAttr::DS_warning:
+      return diag::Severity::Warning;
+    case DiagnoseIfAttr::DS_error:
+      return diag::Severity::Error;
+    }
+    llvm_unreachable("Fully covered switch above!");
+  };
+
   for (const auto *DIA : llvm::make_range(WarningBegin, Attrs.end()))
     if (IsSuccessful(DIA)) {
-      S.Diag(Loc, diag::warn_diagnose_if_succeeded) << DIA->getMessage();
-      S.Diag(DIA->getLocation(), diag::note_from_diagnose_if)
-          << DIA->getParent() << DIA->getCond()->getSourceRange();
+      if (DIA->getWarningGroup().empty() &&
+          DIA->getDefaultSeverity() == DiagnoseIfAttr::DS_warning) {
+        S.Diag(Loc, diag::warn_diagnose_if_succeeded) << DIA->getMessage();
+        S.Diag(DIA->getLocation(), diag::note_from_diagnose_if)
+            << DIA->getParent() << DIA->getCond()->getSourceRange();
+      } else {
+        auto DiagGroup = S.Diags.getDiagnosticIDs()->getGroupForWarningOption(
+            DIA->getWarningGroup());
+        assert(DiagGroup);
+        auto DiagID = S.Diags.getDiagnosticIDs()->getCustomDiagID(
+            {ToSeverity(DIA->getDefaultSeverity()), "%0",
+             DiagnosticIDs::CLASS_WARNING, false, false, *DiagGroup});
+        S.Diag(Loc, DiagID) << DIA->getMessage();
+      }
     }
 
   return false;
@@ -7531,7 +7562,7 @@ void Sema::AddMethodCandidate(
     Expr::Classification ObjectClassification, ArrayRef<Expr *> Args,
     OverloadCandidateSet &CandidateSet, bool SuppressUserConversions,
     bool PartialOverloading, ConversionSequenceList EarlyConversions,
-    OverloadCandidateParamOrder PO, bool HasMatchedPackOnParmToNonPackOnArg) {
+    OverloadCandidateParamOrder PO, bool StrictPackMatch) {
   const FunctionProtoType *Proto
     = dyn_cast<FunctionProtoType>(Method->getType()->getAs<FunctionType>());
   assert(Proto && "Methods without a prototype cannot be overloaded");
@@ -7562,8 +7593,7 @@ void Sema::AddMethodCandidate(
   Candidate.TookAddressOfOverload =
       CandidateSet.getKind() == OverloadCandidateSet::CSK_AddressOfOverloadSet;
   Candidate.ExplicitCallArguments = Args.size();
-  Candidate.HasMatchedPackOnParmToNonPackOnArg =
-      HasMatchedPackOnParmToNonPackOnArg;
+  Candidate.StrictPackMatch = StrictPackMatch;
 
   bool IgnoreExplicitObject =
       (Method->isExplicitObjectMemberFunction() &&
@@ -7773,8 +7803,7 @@ void Sema::AddMethodTemplateCandidate(
   AddMethodCandidate(cast<CXXMethodDecl>(Specialization), FoundDecl,
                      ActingContext, ObjectType, ObjectClassification, Args,
                      CandidateSet, SuppressUserConversions, PartialOverloading,
-                     Conversions, PO,
-                     Info.hasMatchedPackOnParmToNonPackOnArg());
+                     Conversions, PO, Info.hasStrictPackMatch());
 }
 
 /// Determine whether a given function template has a simple explicit specifier
@@ -7862,7 +7891,7 @@ void Sema::AddTemplateOverloadCandidate(
       PartialOverloading, AllowExplicit,
       /*AllowExplicitConversions=*/false, IsADLCandidate, Conversions, PO,
       Info.AggregateDeductionCandidateHasMismatchedArity,
-      Info.hasMatchedPackOnParmToNonPackOnArg());
+      Info.hasStrictPackMatch());
 }
 
 bool Sema::CheckNonDependentConversions(
@@ -7984,8 +8013,7 @@ void Sema::AddConversionCandidate(
     CXXConversionDecl *Conversion, DeclAccessPair FoundDecl,
     CXXRecordDecl *ActingContext, Expr *From, QualType ToType,
     OverloadCandidateSet &CandidateSet, bool AllowObjCConversionOnExplicit,
-    bool AllowExplicit, bool AllowResultConversion,
-    bool HasMatchedPackOnParmToNonPackOnArg) {
+    bool AllowExplicit, bool AllowResultConversion, bool StrictPackMatch) {
   assert(!Conversion->getDescribedFunctionTemplate() &&
          "Conversion function templates use AddTemplateConversionCandidate");
   QualType ConvType = Conversion->getConversionType().getNonReferenceType();
@@ -8030,8 +8058,7 @@ void Sema::AddConversionCandidate(
   Candidate.FinalConversion.setAllToTypes(ToType);
   Candidate.Viable = true;
   Candidate.ExplicitCallArguments = 1;
-  Candidate.HasMatchedPackOnParmToNonPackOnArg =
-      HasMatchedPackOnParmToNonPackOnArg;
+  Candidate.StrictPackMatch = StrictPackMatch;
 
   // Explicit functions are not actually candidates at all if we're not
   // allowing them in this context, but keep them around so we can point
@@ -8234,7 +8261,7 @@ void Sema::AddTemplateConversionCandidate(
   AddConversionCandidate(Specialization, FoundDecl, ActingDC, From, ToType,
                          CandidateSet, AllowObjCConversionOnExplicit,
                          AllowExplicit, AllowResultConversion,
-                         Info.hasMatchedPackOnParmToNonPackOnArg());
+                         Info.hasStrictPackMatch());
 }
 
 void Sema::AddSurrogateCandidate(CXXConversionDecl *Conversion,
@@ -10586,9 +10613,8 @@ bool clang::isBetterOverloadCandidate(
           isa<CXXConstructorDecl>(Cand2.Function))
     return isa<CXXConstructorDecl>(Cand1.Function);
 
-  if (Cand1.HasMatchedPackOnParmToNonPackOnArg !=
-      Cand2.HasMatchedPackOnParmToNonPackOnArg)
-    return Cand2.HasMatchedPackOnParmToNonPackOnArg;
+  if (Cand1.StrictPackMatch != Cand2.StrictPackMatch)
+    return Cand2.StrictPackMatch;
 
   //    -- F1 is a non-template function and F2 is a function template
   //       specialization, or, if not that,
diff --git a/clang/lib/Sema/SemaRISCV.cpp b/clang/lib/Sema/SemaRISCV.cpp
index 163f7129a7b42..8a5037d045125 100644
--- a/clang/lib/Sema/SemaRISCV.cpp
+++ b/clang/lib/Sema/SemaRISCV.cpp
@@ -1457,14 +1457,14 @@ void SemaRISCV::handleInterruptAttr(Decl *D, const ParsedAttr &AL) {
   }
 
   if (hasFunctionProto(D) && getFunctionOrMethodNumParams(D) != 0) {
-    Diag(D->getLocation(), diag::warn_interrupt_attribute_invalid)
-        << /*RISC-V*/ 2 << 0;
+    Diag(D->getLocation(), diag::warn_interrupt_signal_attribute_invalid)
+        << /*RISC-V*/ 2 << /*interrupt*/ 0 << 0;
     return;
   }
 
   if (!getFunctionOrMethodResultType(D)->isVoidType()) {
-    Diag(D->getLocation(), diag::warn_interrupt_attribute_invalid)
-        << /*RISC-V*/ 2 << 1;
+    Diag(D->getLocation(), diag::warn_interrupt_signal_attribute_invalid)
+        << /*RISC-V*/ 2 << /*interrupt*/ 0 << 1;
     return;
   }
 
diff --git a/clang/lib/Sema/SemaStmt.cpp b/clang/lib/Sema/SemaStmt.cpp
index 25a07d0315eac..ec38674a2c3e7 100644
--- a/clang/lib/Sema/SemaStmt.cpp
+++ b/clang/lib/Sema/SemaStmt.cpp
@@ -2716,8 +2716,10 @@ StmtResult Sema::BuildCXXForRangeStmt(
     // them in properly when we instantiate the loop.
     if (!LoopVar->isInvalidDecl() && Kind != BFRK_Check) {
       if (auto *DD = dyn_cast<DecompositionDecl>(LoopVar))
-        for (auto *Binding : DD->bindings())
-          Binding->setType(Context.DependentTy);
+        for (auto *Binding : DD->bindings()) {
+          if (!Binding->isParameterPack())
+            Binding->setType(Context.DependentTy);
+        }
       LoopVar->setType(SubstAutoTypeDependent(LoopVar->getType()));
     }
   } else if (!BeginDeclStmt.get()) {
@@ -4568,9 +4570,27 @@ buildCapturedStmtCaptureList(Sema &S, CapturedRegionScopeInfo *RSI,
   return false;
 }
 
+static std::optional<int>
+isOpenMPCapturedRegionInArmSMEFunction(Sema const &S, CapturedRegionKind Kind) {
+  if (!S.getLangOpts().OpenMP || Kind != CR_OpenMP)
+    return {};
+  if (const FunctionDecl *FD = S.getCurFunctionDecl(/*AllowLambda=*/true)) {
+    if (IsArmStreamingFunction(FD, /*IncludeLocallyStreaming=*/true))
+      return /* in streaming functions */ 0;
+    if (hasArmZAState(FD))
+      return /* in functions with ZA state */ 1;
+    if (hasArmZT0State(FD))
+      return /* in fuctions with ZT0 state */ 2;
+  }
+  return {};
+}
+
 void Sema::ActOnCapturedRegionStart(SourceLocation Loc, Scope *CurScope,
                                     CapturedRegionKind Kind,
                                     unsigned NumParams) {
+  if (auto ErrorIndex = isOpenMPCapturedRegionInArmSMEFunction(*this, Kind))
+    Diag(Loc, diag::err_sme_openmp_captured_region) << *ErrorIndex;
+
   CapturedDecl *CD = nullptr;
   RecordDecl *RD = CreateCapturedStmtRecordDecl(CD, Loc, NumParams);
 
@@ -4602,6 +4622,9 @@ void Sema::ActOnCapturedRegionStart(SourceLocation Loc, Scope *CurScope,
                                     CapturedRegionKind Kind,
                                     ArrayRef<CapturedParamNameType> Params,
                                     unsigned OpenMPCaptureLevel) {
+  if (auto ErrorIndex = isOpenMPCapturedRegionInArmSMEFunction(*this, Kind))
+    Diag(Loc, diag::err_sme_openmp_captured_region) << *ErrorIndex;
+
   CapturedDecl *CD = nullptr;
   RecordDecl *RD = CreateCapturedStmtRecordDecl(CD, Loc, Params.size());
 
diff --git a/clang/lib/Sema/SemaTemplate.cpp b/clang/lib/Sema/SemaTemplate.cpp
index 38196c5c2bc12..9e68972b33f0a 100644
--- a/clang/lib/Sema/SemaTemplate.cpp
+++ b/clang/lib/Sema/SemaTemplate.cpp
@@ -38,6 +38,7 @@
 #include "clang/Sema/TemplateDeduction.h"
 #include "llvm/ADT/SmallBitVector.h"
 #include "llvm/ADT/StringExtras.h"
+#include "llvm/Support/SaveAndRestore.h"
 
 #include <optional>
 using namespace clang;
@@ -3497,10 +3498,10 @@ QualType Sema::CheckTemplateIdType(TemplateName Name,
 
   // Check that the template argument list is well-formed for this
   // template.
-  SmallVector<TemplateArgument, 4> SugaredConverted, CanonicalConverted;
+  CheckTemplateArgumentInfo CTAI;
   if (CheckTemplateArgumentList(Template, TemplateLoc, TemplateArgs,
-                                DefaultArgs, false, SugaredConverted,
-                                CanonicalConverted,
+                                DefaultArgs, /*PartialTemplateArgs=*/false,
+                                CTAI,
                                 /*UpdateArgsWithConversions=*/true))
     return QualType();
 
@@ -3522,7 +3523,7 @@ QualType Sema::CheckTemplateIdType(TemplateName Name,
     // template type alias specializations apart.
     MultiLevelTemplateArgumentList TemplateArgLists;
     TemplateArgLists.addOuterTemplateArguments(
-        Template, SugaredConverted,
+        Template, CTAI.SugaredConverted,
         /*Final=*/!getLangOpts().RetainSubstTemplateTypeParmTypeAstNodes);
     TemplateArgLists.addOuterRetainedLevels(
         AliasTemplate->getTemplateParameters()->getDepth());
@@ -3582,11 +3583,11 @@ QualType Sema::CheckTemplateIdType(TemplateName Name,
       return QualType();
     }
   } else if (auto *BTD = dyn_cast<BuiltinTemplateDecl>(Template)) {
-    CanonType = checkBuiltinTemplateIdType(*this, BTD, SugaredConverted,
+    CanonType = checkBuiltinTemplateIdType(*this, BTD, CTAI.SugaredConverted,
                                            TemplateLoc, TemplateArgs);
   } else if (Name.isDependent() ||
              TemplateSpecializationType::anyDependentTemplateArguments(
-                 TemplateArgs, CanonicalConverted)) {
+                 TemplateArgs, CTAI.CanonicalConverted)) {
     // This class template specialization is a dependent
     // type. Therefore, its canonical type is another class template
     // specialization type that contains all of the converted
@@ -3595,7 +3596,7 @@ QualType Sema::CheckTemplateIdType(TemplateName Name,
     //
     //   template<typename T, typename U = T> struct A;
     CanonType = Context.getCanonicalTemplateSpecializationType(
-        Name, CanonicalConverted);
+        Name, CTAI.CanonicalConverted);
 
     // This might work out to be a current instantiation, in which
     // case the canonical type needs to be the InjectedClassNameType.
@@ -3640,7 +3641,7 @@ QualType Sema::CheckTemplateIdType(TemplateName Name,
     // corresponds to these arguments.
     void *InsertPos = nullptr;
     ClassTemplateSpecializationDecl *Decl =
-        ClassTemplate->findSpecialization(CanonicalConverted, InsertPos);
+        ClassTemplate->findSpecialization(CTAI.CanonicalConverted, InsertPos);
     if (!Decl) {
       // This is the first time we have referenced this class template
       // specialization. Create the canonical declaration and add it to
@@ -3649,8 +3650,8 @@ QualType Sema::CheckTemplateIdType(TemplateName Name,
           Context, ClassTemplate->getTemplatedDecl()->getTagKind(),
           ClassTemplate->getDeclContext(),
           ClassTemplate->getTemplatedDecl()->getBeginLoc(),
-          ClassTemplate->getLocation(), ClassTemplate, CanonicalConverted,
-          nullptr);
+          ClassTemplate->getLocation(), ClassTemplate, CTAI.CanonicalConverted,
+          CTAI.StrictPackMatch, nullptr);
       ClassTemplate->AddSpecialization(Decl, InsertPos);
       if (ClassTemplate->isOutOfLine())
         Decl->setLexicalDeclContext(ClassTemplate->getLexicalDeclContext());
@@ -3661,7 +3662,7 @@ QualType Sema::CheckTemplateIdType(TemplateName Name,
       InstantiatingTemplate Inst(*this, TemplateLoc, Decl);
       if (!Inst.isInvalid()) {
         MultiLevelTemplateArgumentList TemplateArgLists(Template,
-                                                        CanonicalConverted,
+                                                        CTAI.CanonicalConverted,
                                                         /*Final=*/false);
         InstantiateAttrsForDecl(TemplateArgLists,
                                 ClassTemplate->getTemplatedDecl(), Decl);
@@ -4183,10 +4184,10 @@ DeclResult Sema::ActOnVarTemplateSpecialization(
 
   // Check that the template argument list is well-formed for this
   // template.
-  SmallVector<TemplateArgument, 4> SugaredConverted, CanonicalConverted;
+  CheckTemplateArgumentInfo CTAI;
   if (CheckTemplateArgumentList(VarTemplate, TemplateNameLoc, TemplateArgs,
-                                /*DefaultArgs=*/{}, false, SugaredConverted,
-                                CanonicalConverted,
+                                /*DefaultArgs=*/{},
+                                /*PartialTemplateArgs=*/false, CTAI,
                                 /*UpdateArgsWithConversions=*/true))
     return true;
 
@@ -4195,21 +4196,21 @@ DeclResult Sema::ActOnVarTemplateSpecialization(
   if (IsPartialSpecialization) {
     if (CheckTemplatePartialSpecializationArgs(TemplateNameLoc, VarTemplate,
                                                TemplateArgs.size(),
-                                               CanonicalConverted))
+                                               CTAI.CanonicalConverted))
       return true;
 
-    // FIXME: Move these checks to CheckTemplatePartialSpecializationArgs so we
-    // also do them during instantiation.
+    // FIXME: Move these checks to CheckTemplatePartialSpecializationArgs so
+    // we also do them during instantiation.
     if (!Name.isDependent() &&
         !TemplateSpecializationType::anyDependentTemplateArguments(
-            TemplateArgs, CanonicalConverted)) {
+            TemplateArgs, CTAI.CanonicalConverted)) {
       Diag(TemplateNameLoc, diag::err_partial_spec_fully_specialized)
           << VarTemplate->getDeclName();
       IsPartialSpecialization = false;
     }
 
     if (isSameAsPrimaryTemplate(VarTemplate->getTemplateParameters(),
-                                CanonicalConverted) &&
+                                CTAI.CanonicalConverted) &&
         (!Context.getLangOpts().CPlusPlus20 ||
          !TemplateParams->hasAssociatedConstraints())) {
       // C++ [temp.class.spec]p9b3:
@@ -4217,11 +4218,11 @@ DeclResult Sema::ActOnVarTemplateSpecialization(
       //   -- The argument list of the specialization shall not be identical
       //      to the implicit argument list of the primary template.
       Diag(TemplateNameLoc, diag::err_partial_spec_args_match_primary_template)
-        << /*variable template*/ 1
-        << /*is definition*/(SC != SC_Extern && !CurContext->isRecord())
-        << FixItHint::CreateRemoval(SourceRange(LAngleLoc, RAngleLoc));
-      // FIXME: Recover from this by treating the declaration as a redeclaration
-      // of the primary template.
+          << /*variable template*/ 1
+          << /*is definition*/ (SC != SC_Extern && !CurContext->isRecord())
+          << FixItHint::CreateRemoval(SourceRange(LAngleLoc, RAngleLoc));
+      // FIXME: Recover from this by treating the declaration as a
+      // redeclaration of the primary template.
       return true;
     }
   }
@@ -4231,9 +4232,10 @@ DeclResult Sema::ActOnVarTemplateSpecialization(
 
   if (IsPartialSpecialization)
     PrevDecl = VarTemplate->findPartialSpecialization(
-        CanonicalConverted, TemplateParams, InsertPos);
+        CTAI.CanonicalConverted, TemplateParams, InsertPos);
   else
-    PrevDecl = VarTemplate->findSpecialization(CanonicalConverted, InsertPos);
+    PrevDecl =
+        VarTemplate->findSpecialization(CTAI.CanonicalConverted, InsertPos);
 
   VarTemplateSpecializationDecl *Specialization = nullptr;
 
@@ -4260,7 +4262,7 @@ DeclResult Sema::ActOnVarTemplateSpecialization(
         VarTemplatePartialSpecializationDecl::Create(
             Context, VarTemplate->getDeclContext(), TemplateKWLoc,
             TemplateNameLoc, TemplateParams, VarTemplate, DI->getType(), DI, SC,
-            CanonicalConverted);
+            CTAI.CanonicalConverted);
     Partial->setTemplateArgsAsWritten(TemplateArgs);
 
     if (!PrevPartial)
@@ -4278,7 +4280,7 @@ DeclResult Sema::ActOnVarTemplateSpecialization(
     // this explicit specialization or friend declaration.
     Specialization = VarTemplateSpecializationDecl::Create(
         Context, VarTemplate->getDeclContext(), TemplateKWLoc, TemplateNameLoc,
-        VarTemplate, DI->getType(), DI, SC, CanonicalConverted);
+        VarTemplate, DI->getType(), DI, SC, CTAI.CanonicalConverted);
     Specialization->setTemplateArgsAsWritten(TemplateArgs);
 
     if (!PrevDecl)
@@ -4350,25 +4352,25 @@ Sema::CheckVarTemplateId(VarTemplateDecl *Template, SourceLocation TemplateLoc,
   assert(Template && "A variable template id without template?");
 
   // Check that the template argument list is well-formed for this template.
-  SmallVector<TemplateArgument, 4> SugaredConverted, CanonicalConverted;
+  CheckTemplateArgumentInfo CTAI;
   if (CheckTemplateArgumentList(
           Template, TemplateNameLoc,
           const_cast<TemplateArgumentListInfo &>(TemplateArgs),
-          /*DefaultArgs=*/{}, false, SugaredConverted, CanonicalConverted,
+          /*DefaultArgs=*/{}, /*PartialTemplateArgs=*/false, CTAI,
           /*UpdateArgsWithConversions=*/true))
     return true;
 
   // Produce a placeholder value if the specialization is dependent.
   if (Template->getDeclContext()->isDependentContext() ||
       TemplateSpecializationType::anyDependentTemplateArguments(
-          TemplateArgs, CanonicalConverted))
+          TemplateArgs, CTAI.CanonicalConverted))
     return DeclResult();
 
   // Find the variable template specialization declaration that
   // corresponds to these arguments.
   void *InsertPos = nullptr;
   if (VarTemplateSpecializationDecl *Spec =
-          Template->findSpecialization(CanonicalConverted, InsertPos)) {
+          Template->findSpecialization(CTAI.CanonicalConverted, InsertPos)) {
     checkSpecializationReachability(TemplateNameLoc, Spec);
     // If we already have a variable template specialization, return it.
     return Spec;
@@ -4412,7 +4414,7 @@ Sema::CheckVarTemplateId(VarTemplateDecl *Template, SourceLocation TemplateLoc,
     TemplateDeductionInfo Info(FailedCandidates.getLocation());
 
     if (TemplateDeductionResult Result =
-            DeduceTemplateArguments(Partial, SugaredConverted, Info);
+            DeduceTemplateArguments(Partial, CTAI.SugaredConverted, Info);
         Result != TemplateDeductionResult::Success) {
       // Store the failed-deduction information for use in diagnostics, later.
       // TODO: Actually use the failed-deduction info?
@@ -4479,7 +4481,7 @@ Sema::CheckVarTemplateId(VarTemplateDecl *Template, SourceLocation TemplateLoc,
   // FIXME: LateAttrs et al.?
   VarTemplateSpecializationDecl *Decl = BuildVarTemplateInstantiation(
       Template, InstantiationPattern, PartialSpecArgs, TemplateArgs,
-      CanonicalConverted, TemplateNameLoc /*, LateAttrs, StartingScope*/);
+      CTAI.CanonicalConverted, TemplateNameLoc /*, LateAttrs, StartingScope*/);
   if (!Decl)
     return true;
 
@@ -4559,12 +4561,12 @@ Sema::CheckConceptTemplateId(const CXXScopeSpec &SS,
   if (NamedConcept->isInvalidDecl())
     return ExprError();
 
-  llvm::SmallVector<TemplateArgument, 4> SugaredConverted, CanonicalConverted;
+  CheckTemplateArgumentInfo CTAI;
   if (CheckTemplateArgumentList(
           NamedConcept, ConceptNameInfo.getLoc(),
           const_cast<TemplateArgumentListInfo &>(*TemplateArgs),
           /*DefaultArgs=*/{},
-          /*PartialTemplateArgs=*/false, SugaredConverted, CanonicalConverted,
+          /*PartialTemplateArgs=*/false, CTAI,
           /*UpdateArgsWithConversions=*/false))
     return ExprError();
 
@@ -4572,12 +4574,12 @@ Sema::CheckConceptTemplateId(const CXXScopeSpec &SS,
 
   auto *CSD = ImplicitConceptSpecializationDecl::Create(
       Context, NamedConcept->getDeclContext(), NamedConcept->getLocation(),
-      CanonicalConverted);
+      CTAI.CanonicalConverted);
   ConstraintSatisfaction Satisfaction;
   bool AreArgsDependent =
       TemplateSpecializationType::anyDependentTemplateArguments(
-          *TemplateArgs, CanonicalConverted);
-  MultiLevelTemplateArgumentList MLTAL(NamedConcept, CanonicalConverted,
+          *TemplateArgs, CTAI.CanonicalConverted);
+  MultiLevelTemplateArgumentList MLTAL(NamedConcept, CTAI.CanonicalConverted,
                                        /*Final=*/false);
   LocalInstantiationScope Scope(*this);
 
@@ -5198,19 +5200,19 @@ convertTypeTemplateArgumentToTemplate(ASTContext &Context, TypeLoc TLoc) {
   return TemplateArgumentLoc();
 }
 
-bool Sema::CheckTemplateArgument(
-    NamedDecl *Param, TemplateArgumentLoc &Arg, NamedDecl *Template,
-    SourceLocation TemplateLoc, SourceLocation RAngleLoc,
-    unsigned ArgumentPackIndex,
-    SmallVectorImpl<TemplateArgument> &SugaredConverted,
-    SmallVectorImpl<TemplateArgument> &CanonicalConverted,
-    CheckTemplateArgumentKind CTAK, bool PartialOrdering,
-    bool *MatchedPackOnParmToNonPackOnArg) {
+bool Sema::CheckTemplateArgument(NamedDecl *Param, TemplateArgumentLoc &ArgLoc,
+                                 NamedDecl *Template,
+                                 SourceLocation TemplateLoc,
+                                 SourceLocation RAngleLoc,
+                                 unsigned ArgumentPackIndex,
+                                 CheckTemplateArgumentInfo &CTAI,
+                                 CheckTemplateArgumentKind CTAK) {
   // Check template type parameters.
   if (TemplateTypeParmDecl *TTP = dyn_cast<TemplateTypeParmDecl>(Param))
-    return CheckTemplateTypeArgument(TTP, Arg, SugaredConverted,
-                                     CanonicalConverted);
+    return CheckTemplateTypeArgument(TTP, ArgLoc, CTAI.SugaredConverted,
+                                     CTAI.CanonicalConverted);
 
+  const TemplateArgument &Arg = ArgLoc.getArgument();
   // Check non-type template parameters.
   if (NonTypeTemplateParmDecl *NTTP =dyn_cast<NonTypeTemplateParmDecl>(Param)) {
     // Do substitution on the type of the non-type template parameter
@@ -5225,12 +5227,12 @@ bool Sema::CheckTemplateArgument(
         !Template->getDeclContext()->isDependentContext()) {
       // Do substitution on the type of the non-type template parameter.
       InstantiatingTemplate Inst(*this, TemplateLoc, Template, NTTP,
-                                 SugaredConverted,
+                                 CTAI.SugaredConverted,
                                  SourceRange(TemplateLoc, RAngleLoc));
       if (Inst.isInvalid())
         return true;
 
-      MultiLevelTemplateArgumentList MLTAL(Template, SugaredConverted,
+      MultiLevelTemplateArgumentList MLTAL(Template, CTAI.SugaredConverted,
                                            /*Final=*/true);
       // If the parameter is a pack expansion, expand this slice of the pack.
       if (auto *PET = NTTPType->getAs<PackExpansionType>()) {
@@ -5252,62 +5254,73 @@ bool Sema::CheckTemplateArgument(
         return true;
     }
 
-    switch (Arg.getArgument().getKind()) {
+    auto checkExpr = [&](Expr *E) -> Expr * {
+      TemplateArgument SugaredResult, CanonicalResult;
+      unsigned CurSFINAEErrors = NumSFINAEErrors;
+      ExprResult Res =
+          CheckTemplateArgument(NTTP, NTTPType, E, SugaredResult,
+                                CanonicalResult, CTAI.MatchingTTP, CTAK);
+      // If the current template argument causes an error, give up now.
+      if (Res.isInvalid() || CurSFINAEErrors < NumSFINAEErrors)
+        return nullptr;
+      CTAI.SugaredConverted.push_back(SugaredResult);
+      CTAI.CanonicalConverted.push_back(CanonicalResult);
+      return Res.get();
+    };
+
+    switch (Arg.getKind()) {
     case TemplateArgument::Null:
       llvm_unreachable("Should never see a NULL template argument here");
 
     case TemplateArgument::Expression: {
-      Expr *E = Arg.getArgument().getAsExpr();
-      TemplateArgument SugaredResult, CanonicalResult;
-      unsigned CurSFINAEErrors = NumSFINAEErrors;
-      ExprResult Res = CheckTemplateArgument(NTTP, NTTPType, E, SugaredResult,
-                                             CanonicalResult, CTAK);
-      if (Res.isInvalid())
+      Expr *E = Arg.getAsExpr();
+      Expr *R = checkExpr(E);
+      if (!R)
         return true;
-      // If the current template argument causes an error, give up now.
-      if (CurSFINAEErrors < NumSFINAEErrors)
-        return true;
-
       // If the resulting expression is new, then use it in place of the
       // old expression in the template argument.
-      if (Res.get() != E) {
-        TemplateArgument TA(Res.get());
-        Arg = TemplateArgumentLoc(TA, Res.get());
+      if (R != E) {
+        TemplateArgument TA(R);
+        ArgLoc = TemplateArgumentLoc(TA, R);
       }
-
-      SugaredConverted.push_back(SugaredResult);
-      CanonicalConverted.push_back(CanonicalResult);
       break;
     }
 
-    case TemplateArgument::Declaration:
+    // As for the converted NTTP kinds, they still might need another
+    // conversion, as the new corresponding parameter might be different.
+    // Ideally, we would always perform substitution starting with sugared types
+    // and never need these, as we would still have expressions. Since these are
+    // needed so rarely, it's probably a better tradeoff to just convert them
+    // back to expressions.
     case TemplateArgument::Integral:
-    case TemplateArgument::StructuralValue:
+    case TemplateArgument::Declaration:
     case TemplateArgument::NullPtr:
-      // We've already checked this template argument, so just copy
-      // it to the list of converted arguments.
-      SugaredConverted.push_back(Arg.getArgument());
-      CanonicalConverted.push_back(
-          Context.getCanonicalTemplateArgument(Arg.getArgument()));
+    case TemplateArgument::StructuralValue: {
+      // FIXME: StructuralValue is untested here.
+      ExprResult R =
+          BuildExpressionFromNonTypeTemplateArgument(Arg, SourceLocation());
+      assert(R.isUsable());
+      if (!checkExpr(R.get()))
+        return true;
       break;
+    }
 
     case TemplateArgument::Template:
     case TemplateArgument::TemplateExpansion:
       // We were given a template template argument. It may not be ill-formed;
       // see below.
-      if (DependentTemplateName *DTN
-            = Arg.getArgument().getAsTemplateOrTemplatePattern()
-                                              .getAsDependentTemplateName()) {
+      if (DependentTemplateName *DTN = Arg.getAsTemplateOrTemplatePattern()
+                                           .getAsDependentTemplateName()) {
         // We have a template argument such as \c T::template X, which we
         // parsed as a template template argument. However, since we now
         // know that we need a non-type template argument, convert this
         // template name into an expression.
 
         DeclarationNameInfo NameInfo(DTN->getIdentifier(),
-                                     Arg.getTemplateNameLoc());
+                                     ArgLoc.getTemplateNameLoc());
 
         CXXScopeSpec SS;
-        SS.Adopt(Arg.getTemplateQualifierLoc());
+        SS.Adopt(ArgLoc.getTemplateQualifierLoc());
         // FIXME: the template-template arg was a DependentTemplateName,
         // so it was provided with a template keyword. However, its source
         // location is not stored in the template argument structure.
@@ -5318,28 +5331,29 @@ bool Sema::CheckTemplateArgument(
 
         // If we parsed the template argument as a pack expansion, create a
         // pack expansion expression.
-        if (Arg.getArgument().getKind() == TemplateArgument::TemplateExpansion){
-          E = ActOnPackExpansion(E.get(), Arg.getTemplateEllipsisLoc());
+        if (Arg.getKind() == TemplateArgument::TemplateExpansion) {
+          E = ActOnPackExpansion(E.get(), ArgLoc.getTemplateEllipsisLoc());
           if (E.isInvalid())
             return true;
         }
 
         TemplateArgument SugaredResult, CanonicalResult;
         E = CheckTemplateArgument(NTTP, NTTPType, E.get(), SugaredResult,
-                                  CanonicalResult, CTAK_Specified);
+                                  CanonicalResult, /*PartialOrderingTTP=*/false,
+                                  CTAK_Specified);
         if (E.isInvalid())
           return true;
 
-        SugaredConverted.push_back(SugaredResult);
-        CanonicalConverted.push_back(CanonicalResult);
+        CTAI.SugaredConverted.push_back(SugaredResult);
+        CTAI.CanonicalConverted.push_back(CanonicalResult);
         break;
       }
 
       // We have a template argument that actually does refer to a class
       // template, alias template, or template template parameter, and
       // therefore cannot be a non-type template argument.
-      Diag(Arg.getLocation(), diag::err_template_arg_must_be_expr)
-        << Arg.getSourceRange();
+      Diag(ArgLoc.getLocation(), diag::err_template_arg_must_be_expr)
+          << ArgLoc.getSourceRange();
       NoteTemplateParameterLocation(*Param);
 
       return true;
@@ -5355,8 +5369,8 @@ bool Sema::CheckTemplateArgument(
       //
       // We warn specifically about this case, since it can be rather
       // confusing for users.
-      QualType T = Arg.getArgument().getAsType();
-      SourceRange SR = Arg.getSourceRange();
+      QualType T = Arg.getAsType();
+      SourceRange SR = ArgLoc.getSourceRange();
       if (T->isFunctionType())
         Diag(SR.getBegin(), diag::err_template_arg_nontype_ambig) << SR << T;
       else
@@ -5389,16 +5403,16 @@ bool Sema::CheckTemplateArgument(
     // Set up a template instantiation context.
     LocalInstantiationScope Scope(*this);
     InstantiatingTemplate Inst(*this, TemplateLoc, Template, TempParm,
-                               SugaredConverted,
+                               CTAI.SugaredConverted,
                                SourceRange(TemplateLoc, RAngleLoc));
     if (Inst.isInvalid())
       return true;
 
-    Params =
-        SubstTemplateParams(Params, CurContext,
-                            MultiLevelTemplateArgumentList(
-                                Template, SugaredConverted, /*Final=*/true),
-                            /*EvaluateConstraints=*/false);
+    Params = SubstTemplateParams(
+        Params, CurContext,
+        MultiLevelTemplateArgumentList(Template, CTAI.SugaredConverted,
+                                       /*Final=*/true),
+        /*EvaluateConstraints=*/false);
     if (!Params)
       return true;
   }
@@ -5407,34 +5421,35 @@ bool Sema::CheckTemplateArgument(
   //   When [the injected-class-name] is used [...] as a template-argument for
   //   a template template-parameter [...] it refers to the class template
   //   itself.
-  if (Arg.getArgument().getKind() == TemplateArgument::Type) {
+  if (Arg.getKind() == TemplateArgument::Type) {
     TemplateArgumentLoc ConvertedArg = convertTypeTemplateArgumentToTemplate(
-        Context, Arg.getTypeSourceInfo()->getTypeLoc());
+        Context, ArgLoc.getTypeSourceInfo()->getTypeLoc());
     if (!ConvertedArg.getArgument().isNull())
-      Arg = ConvertedArg;
+      ArgLoc = ConvertedArg;
   }
 
-  switch (Arg.getArgument().getKind()) {
+  switch (Arg.getKind()) {
   case TemplateArgument::Null:
     llvm_unreachable("Should never see a NULL template argument here");
 
   case TemplateArgument::Template:
   case TemplateArgument::TemplateExpansion:
-    if (CheckTemplateTemplateArgument(TempParm, Params, Arg, PartialOrdering,
-                                      MatchedPackOnParmToNonPackOnArg))
+    if (CheckTemplateTemplateArgument(TempParm, Params, ArgLoc,
+                                      CTAI.PartialOrdering,
+                                      &CTAI.StrictPackMatch))
       return true;
 
-    SugaredConverted.push_back(Arg.getArgument());
-    CanonicalConverted.push_back(
-        Context.getCanonicalTemplateArgument(Arg.getArgument()));
+    CTAI.SugaredConverted.push_back(Arg);
+    CTAI.CanonicalConverted.push_back(
+        Context.getCanonicalTemplateArgument(Arg));
     break;
 
   case TemplateArgument::Expression:
   case TemplateArgument::Type:
     // We have a template template parameter but the template
     // argument does not refer to a template.
-    Diag(Arg.getLocation(), diag::err_template_arg_must_be_template)
-      << getLangOpts().CPlusPlus11;
+    Diag(ArgLoc.getLocation(), diag::err_template_arg_must_be_template)
+        << getLangOpts().CPlusPlus11;
     return true;
 
   case TemplateArgument::Declaration:
@@ -5491,11 +5506,8 @@ static bool diagnoseMissingArgument(Sema &S, SourceLocation Loc,
 bool Sema::CheckTemplateArgumentList(
     TemplateDecl *Template, SourceLocation TemplateLoc,
     TemplateArgumentListInfo &TemplateArgs, const DefaultArguments &DefaultArgs,
-    bool PartialTemplateArgs,
-    SmallVectorImpl<TemplateArgument> &SugaredConverted,
-    SmallVectorImpl<TemplateArgument> &CanonicalConverted,
-    bool UpdateArgsWithConversions, bool *ConstraintsNotSatisfied,
-    bool PartialOrderingTTP, bool *MatchedPackOnParmToNonPackOnArg) {
+    bool PartialTemplateArgs, CheckTemplateArgumentInfo &CTAI,
+    bool UpdateArgsWithConversions, bool *ConstraintsNotSatisfied) {
 
   if (ConstraintsNotSatisfied)
     *ConstraintsNotSatisfied = false;
@@ -5532,8 +5544,8 @@ bool Sema::CheckTemplateArgumentList(
         assert(Param + DefaultArgs.Args.size() <= ParamEnd);
         // Default arguments from a DeducedTemplateName are already converted.
         for (const TemplateArgument &DefArg : DefaultArgs.Args) {
-          SugaredConverted.push_back(DefArg);
-          CanonicalConverted.push_back(
+          CTAI.SugaredConverted.push_back(DefArg);
+          CTAI.CanonicalConverted.push_back(
               Context.getCanonicalTemplateArgument(DefArg));
           ++Param;
         }
@@ -5547,11 +5559,11 @@ bool Sema::CheckTemplateArgumentList(
       if (*Expansions == SugaredArgumentPack.size()) {
         // We're done with this parameter pack. Pack up its arguments and add
         // them to the list.
-        SugaredConverted.push_back(
+        CTAI.SugaredConverted.push_back(
             TemplateArgument::CreatePackCopy(Context, SugaredArgumentPack));
         SugaredArgumentPack.clear();
 
-        CanonicalConverted.push_back(
+        CTAI.CanonicalConverted.push_back(
             TemplateArgument::CreatePackCopy(Context, CanonicalArgumentPack));
         CanonicalArgumentPack.clear();
 
@@ -5575,7 +5587,7 @@ bool Sema::CheckTemplateArgumentList(
           !(*Param)->isTemplateParameterPack() || getExpandedPackSize(*Param);
       bool ArgIsExpansion = ArgLoc.getArgument().isPackExpansion();
 
-      if (ArgIsExpansion && PartialOrderingTTP) {
+      if (ArgIsExpansion && CTAI.MatchingTTP) {
         SmallVector<TemplateArgument, 4> Args(ParamEnd - Param);
         for (TemplateParameterList::iterator First = Param; Param != ParamEnd;
              ++Param) {
@@ -5585,31 +5597,30 @@ bool Sema::CheckTemplateArgumentList(
               getExpandedPackSize(*Param))
             Arg = Arg.getPackExpansionPattern();
           TemplateArgumentLoc NewArgLoc(Arg, ArgLoc.getLocInfo());
+          SaveAndRestore _1(CTAI.PartialOrdering, false);
+          SaveAndRestore _2(CTAI.MatchingTTP, true);
           if (CheckTemplateArgument(*Param, NewArgLoc, Template, TemplateLoc,
-                                    RAngleLoc, SugaredArgumentPack.size(),
-                                    SugaredConverted, CanonicalConverted,
-                                    CTAK_Specified, /*PartialOrdering=*/false,
-                                    MatchedPackOnParmToNonPackOnArg))
+                                    RAngleLoc, SugaredArgumentPack.size(), CTAI,
+                                    CTAK_Specified))
             return true;
           Arg = NewArgLoc.getArgument();
-          CanonicalConverted.back().setIsDefaulted(
+          CTAI.CanonicalConverted.back().setIsDefaulted(
               clang::isSubstitutedDefaultArgument(Context, Arg, *Param,
-                                                  CanonicalConverted,
+                                                  CTAI.CanonicalConverted,
                                                   Params->getDepth()));
         }
         ArgLoc =
             TemplateArgumentLoc(TemplateArgument::CreatePackCopy(Context, Args),
                                 ArgLoc.getLocInfo());
       } else {
+        SaveAndRestore _1(CTAI.PartialOrdering, false);
         if (CheckTemplateArgument(*Param, ArgLoc, Template, TemplateLoc,
-                                  RAngleLoc, SugaredArgumentPack.size(),
-                                  SugaredConverted, CanonicalConverted,
-                                  CTAK_Specified, /*PartialOrdering=*/false,
-                                  MatchedPackOnParmToNonPackOnArg))
+                                  RAngleLoc, SugaredArgumentPack.size(), CTAI,
+                                  CTAK_Specified))
           return true;
-        CanonicalConverted.back().setIsDefaulted(
+        CTAI.CanonicalConverted.back().setIsDefaulted(
             clang::isSubstitutedDefaultArgument(Context, ArgLoc.getArgument(),
-                                                *Param, CanonicalConverted,
+                                                *Param, CTAI.CanonicalConverted,
                                                 Params->getDepth()));
         if (ArgIsExpansion && NonPackParameter) {
           // CWG1430/CWG2686: we have a pack expansion as an argument to an
@@ -5629,28 +5640,28 @@ bool Sema::CheckTemplateArgumentList(
       // We're now done with this argument.
       ++ArgIdx;
 
-      if (ArgIsExpansion && (PartialOrderingTTP || NonPackParameter)) {
+      if (ArgIsExpansion && (CTAI.MatchingTTP || NonPackParameter)) {
         // Directly convert the remaining arguments, because we don't know what
         // parameters they'll match up with.
 
         if (!SugaredArgumentPack.empty()) {
           // If we were part way through filling in an expanded parameter pack,
           // fall back to just producing individual arguments.
-          SugaredConverted.insert(SugaredConverted.end(),
-                                  SugaredArgumentPack.begin(),
-                                  SugaredArgumentPack.end());
+          CTAI.SugaredConverted.insert(CTAI.SugaredConverted.end(),
+                                       SugaredArgumentPack.begin(),
+                                       SugaredArgumentPack.end());
           SugaredArgumentPack.clear();
 
-          CanonicalConverted.insert(CanonicalConverted.end(),
-                                    CanonicalArgumentPack.begin(),
-                                    CanonicalArgumentPack.end());
+          CTAI.CanonicalConverted.insert(CTAI.CanonicalConverted.end(),
+                                         CanonicalArgumentPack.begin(),
+                                         CanonicalArgumentPack.end());
           CanonicalArgumentPack.clear();
         }
 
         while (ArgIdx < NumArgs) {
           const TemplateArgument &Arg = NewArgs[ArgIdx].getArgument();
-          SugaredConverted.push_back(Arg);
-          CanonicalConverted.push_back(
+          CTAI.SugaredConverted.push_back(Arg);
+          CTAI.CanonicalConverted.push_back(
               Context.getCanonicalTemplateArgument(Arg));
           ++ArgIdx;
         }
@@ -5663,8 +5674,8 @@ bool Sema::CheckTemplateArgumentList(
         // deduced argument and place it on the argument pack. Note that we
         // stay on the same template parameter so that we can deduce more
         // arguments.
-        SugaredArgumentPack.push_back(SugaredConverted.pop_back_val());
-        CanonicalArgumentPack.push_back(CanonicalConverted.pop_back_val());
+        SugaredArgumentPack.push_back(CTAI.SugaredConverted.pop_back_val());
+        CanonicalArgumentPack.push_back(CTAI.CanonicalConverted.pop_back_val());
       } else {
         // Move to the next template parameter.
         ++Param;
@@ -5675,9 +5686,9 @@ bool Sema::CheckTemplateArgumentList(
     // If we're checking a partial template argument list, we're done.
     if (PartialTemplateArgs) {
       if ((*Param)->isTemplateParameterPack() && !SugaredArgumentPack.empty()) {
-        SugaredConverted.push_back(
+        CTAI.SugaredConverted.push_back(
             TemplateArgument::CreatePackCopy(Context, SugaredArgumentPack));
-        CanonicalConverted.push_back(
+        CTAI.CanonicalConverted.push_back(
             TemplateArgument::CreatePackCopy(Context, CanonicalArgumentPack));
       }
       return false;
@@ -5699,11 +5710,11 @@ bool Sema::CheckTemplateArgumentList(
         return true;
       }
 
-      SugaredConverted.push_back(
+      CTAI.SugaredConverted.push_back(
           TemplateArgument::CreatePackCopy(Context, SugaredArgumentPack));
       SugaredArgumentPack.clear();
 
-      CanonicalConverted.push_back(
+      CTAI.CanonicalConverted.push_back(
           TemplateArgument::CreatePackCopy(Context, CanonicalArgumentPack));
       CanonicalArgumentPack.clear();
 
@@ -5720,8 +5731,8 @@ bool Sema::CheckTemplateArgumentList(
     // (when the template parameter was part of a nested template) into
     // the default argument.
     TemplateArgumentLoc Arg = SubstDefaultTemplateArgumentIfAvailable(
-        Template, TemplateLoc, RAngleLoc, *Param, SugaredConverted,
-        CanonicalConverted, HasDefaultArg);
+        Template, TemplateLoc, RAngleLoc, *Param, CTAI.SugaredConverted,
+        CTAI.CanonicalConverted, HasDefaultArg);
 
     if (Arg.getArgument().isNull()) {
       if (!HasDefaultArg) {
@@ -5744,20 +5755,21 @@ bool Sema::CheckTemplateArgumentList(
     // template here, we just create this object to put a note into the
     // context stack.
     InstantiatingTemplate Inst(*this, RAngleLoc, Template, *Param,
-                               SugaredConverted,
+                               CTAI.SugaredConverted,
                                SourceRange(TemplateLoc, RAngleLoc));
     if (Inst.isInvalid())
       return true;
 
+    SaveAndRestore _1(CTAI.PartialOrdering, false);
+    SaveAndRestore _2(CTAI.MatchingTTP, false);
+    SaveAndRestore _3(CTAI.StrictPackMatch, {});
     // Check the default template argument.
     if (CheckTemplateArgument(*Param, Arg, Template, TemplateLoc, RAngleLoc, 0,
-                              SugaredConverted, CanonicalConverted,
-                              CTAK_Specified, /*PartialOrdering=*/false,
-                              /*MatchedPackOnParmToNonPackOnArg=*/nullptr))
+                              CTAI, CTAK_Specified))
       return true;
 
-    SugaredConverted.back().setIsDefaulted(true);
-    CanonicalConverted.back().setIsDefaulted(true);
+    CTAI.SugaredConverted.back().setIsDefaulted(true);
+    CTAI.CanonicalConverted.back().setIsDefaulted(true);
 
     // Core issue 150 (assumed resolution): if this is a template template
     // parameter, keep track of the default template arguments from the
@@ -5774,14 +5786,15 @@ bool Sema::CheckTemplateArgumentList(
   // pack expansions; they might be empty. This can happen even if
   // PartialTemplateArgs is false (the list of arguments is complete but
   // still dependent).
-  if (PartialOrderingTTP ||
+  if (CTAI.MatchingTTP ||
       (CurrentInstantiationScope &&
        CurrentInstantiationScope->getPartiallySubstitutedPack())) {
     while (ArgIdx < NumArgs &&
            NewArgs[ArgIdx].getArgument().isPackExpansion()) {
       const TemplateArgument &Arg = NewArgs[ArgIdx++].getArgument();
-      SugaredConverted.push_back(Arg);
-      CanonicalConverted.push_back(Context.getCanonicalTemplateArgument(Arg));
+      CTAI.SugaredConverted.push_back(Arg);
+      CTAI.CanonicalConverted.push_back(
+          Context.getCanonicalTemplateArgument(Arg));
     }
   }
 
@@ -5822,7 +5835,7 @@ bool Sema::CheckTemplateArgumentList(
     CXXThisScopeRAII(*this, RD, ThisQuals, RD != nullptr);
 
     MultiLevelTemplateArgumentList MLTAL = getTemplateInstantiationArgs(
-        Template, NewContext, /*Final=*/false, CanonicalConverted,
+        Template, NewContext, /*Final=*/false, CTAI.CanonicalConverted,
         /*RelativeToPrimary=*/true,
         /*Pattern=*/nullptr,
         /*ForConceptInstantiation=*/true);
@@ -6740,6 +6753,7 @@ ExprResult Sema::CheckTemplateArgument(NonTypeTemplateParmDecl *Param,
                                        QualType ParamType, Expr *Arg,
                                        TemplateArgument &SugaredConverted,
                                        TemplateArgument &CanonicalConverted,
+                                       bool PartialOrderingTTP,
                                        CheckTemplateArgumentKind CTAK) {
   SourceLocation StartLoc = Arg->getBeginLoc();
 
@@ -6930,17 +6944,21 @@ ExprResult Sema::CheckTemplateArgument(NonTypeTemplateParmDecl *Param,
     IsConvertedConstantExpression = false;
   }
 
-  if (getLangOpts().CPlusPlus17) {
+  if (getLangOpts().CPlusPlus17 || PartialOrderingTTP) {
     // C++17 [temp.arg.nontype]p1:
     //   A template-argument for a non-type template parameter shall be
     //   a converted constant expression of the type of the template-parameter.
     APValue Value;
     ExprResult ArgResult;
     if (IsConvertedConstantExpression) {
-      ArgResult = BuildConvertedConstantExpression(Arg, ParamType,
-                                                   CCEK_TemplateArg, Param);
-      if (ArgResult.isInvalid())
+      ArgResult = BuildConvertedConstantExpression(
+          Arg, ParamType,
+          PartialOrderingTTP ? CCEK_InjectedTTP : CCEK_TemplateArg, Param);
+      assert(!ArgResult.isUnset());
+      if (ArgResult.isInvalid()) {
+        NoteTemplateParameterLocation(*Param);
         return ExprError();
+      }
     } else {
       ArgResult = Arg;
     }
@@ -7343,10 +7361,11 @@ static void DiagnoseTemplateParameterListArityMismatch(
     Sema &S, TemplateParameterList *New, TemplateParameterList *Old,
     Sema::TemplateParameterListEqualKind Kind, SourceLocation TemplateArgLoc);
 
-bool Sema::CheckTemplateTemplateArgument(
-    TemplateTemplateParmDecl *Param, TemplateParameterList *Params,
-    TemplateArgumentLoc &Arg, bool PartialOrdering,
-    bool *MatchedPackOnParmToNonPackOnArg) {
+bool Sema::CheckTemplateTemplateArgument(TemplateTemplateParmDecl *Param,
+                                         TemplateParameterList *Params,
+                                         TemplateArgumentLoc &Arg,
+                                         bool PartialOrdering,
+                                         bool *StrictPackMatch) {
   TemplateName Name = Arg.getArgument().getAsTemplateOrTemplatePattern();
   auto [Template, DefaultArgs] = Name.getTemplateDeclAndDefaultArgs();
   if (!Template) {
@@ -7381,17 +7400,12 @@ bool Sema::CheckTemplateTemplateArgument(
       << Template;
   }
 
-  if (!getLangOpts().RelaxedTemplateTemplateArgs)
-    return !TemplateParameterListsAreEqual(
-        Template->getTemplateParameters(), Params, /*Complain=*/true,
-        TPL_TemplateTemplateArgumentMatch, Arg.getLocation());
-
   // C++1z [temp.arg.template]p3: (DR 150)
   //   A template-argument matches a template template-parameter P when P
   //   is at least as specialized as the template-argument A.
   if (!isTemplateTemplateParameterAtLeastAsSpecializedAs(
           Params, Param, Template, DefaultArgs, Arg.getLocation(),
-          PartialOrdering, MatchedPackOnParmToNonPackOnArg))
+          PartialOrdering, StrictPackMatch))
     return true;
   // P2113
   // C++20[temp.func.order]p2
@@ -7757,9 +7771,7 @@ static bool MatchTemplateParameterKind(
   // However, if we are matching a template template argument to a
   // template template parameter, the template template parameter can have
   // a parameter pack where the template template argument does not.
-  if (Old->isTemplateParameterPack() != New->isTemplateParameterPack() &&
-      !(Kind == Sema::TPL_TemplateTemplateArgumentMatch &&
-        Old->isTemplateParameterPack())) {
+  if (Old->isTemplateParameterPack() != New->isTemplateParameterPack()) {
     if (Complain) {
       unsigned NextDiag = diag::err_template_parameter_pack_non_pack;
       if (TemplateArgLoc.isValid()) {
@@ -7785,37 +7797,28 @@ static bool MatchTemplateParameterKind(
                                     = dyn_cast<NonTypeTemplateParmDecl>(Old)) {
     NonTypeTemplateParmDecl *NewNTTP = cast<NonTypeTemplateParmDecl>(New);
 
-    // If we are matching a template template argument to a template
-    // template parameter and one of the non-type template parameter types
-    // is dependent, then we must wait until template instantiation time
-    // to actually compare the arguments.
-    if (Kind != Sema::TPL_TemplateTemplateArgumentMatch ||
-        (!OldNTTP->getType()->isDependentType() &&
-         !NewNTTP->getType()->isDependentType())) {
-      // C++20 [temp.over.link]p6:
-      //   Two [non-type] template-parameters are equivalent [if] they have
-      //   equivalent types ignoring the use of type-constraints for
-      //   placeholder types
-      QualType OldType = S.Context.getUnconstrainedType(OldNTTP->getType());
-      QualType NewType = S.Context.getUnconstrainedType(NewNTTP->getType());
-      if (!S.Context.hasSameType(OldType, NewType)) {
-        if (Complain) {
-          unsigned NextDiag = diag::err_template_nontype_parm_different_type;
-          if (TemplateArgLoc.isValid()) {
-            S.Diag(TemplateArgLoc,
-                   diag::err_template_arg_template_params_mismatch);
-            NextDiag = diag::note_template_nontype_parm_different_type;
-          }
-          S.Diag(NewNTTP->getLocation(), NextDiag)
-            << NewNTTP->getType()
-            << (Kind != Sema::TPL_TemplateMatch);
-          S.Diag(OldNTTP->getLocation(),
-                 diag::note_template_nontype_parm_prev_declaration)
-            << OldNTTP->getType();
+    // C++20 [temp.over.link]p6:
+    //   Two [non-type] template-parameters are equivalent [if] they have
+    //   equivalent types ignoring the use of type-constraints for
+    //   placeholder types
+    QualType OldType = S.Context.getUnconstrainedType(OldNTTP->getType());
+    QualType NewType = S.Context.getUnconstrainedType(NewNTTP->getType());
+    if (!S.Context.hasSameType(OldType, NewType)) {
+      if (Complain) {
+        unsigned NextDiag = diag::err_template_nontype_parm_different_type;
+        if (TemplateArgLoc.isValid()) {
+          S.Diag(TemplateArgLoc,
+                 diag::err_template_arg_template_params_mismatch);
+          NextDiag = diag::note_template_nontype_parm_different_type;
         }
-
-        return false;
+        S.Diag(NewNTTP->getLocation(), NextDiag)
+            << NewNTTP->getType() << (Kind != Sema::TPL_TemplateMatch);
+        S.Diag(OldNTTP->getLocation(),
+               diag::note_template_nontype_parm_prev_declaration)
+            << OldNTTP->getType();
       }
+
+      return false;
     }
   }
   // For template template parameters, check the template parameter types.
@@ -7835,7 +7838,6 @@ static bool MatchTemplateParameterKind(
   }
 
   if (Kind != Sema::TPL_TemplateParamsEquivalent &&
-      Kind != Sema::TPL_TemplateTemplateArgumentMatch &&
       !isa<TemplateTemplateParmDecl>(Old)) {
     const Expr *NewC = nullptr, *OldC = nullptr;
 
@@ -7906,7 +7908,7 @@ bool Sema::TemplateParameterListsAreEqual(
     const TemplateCompareNewDeclInfo &NewInstFrom, TemplateParameterList *New,
     const NamedDecl *OldInstFrom, TemplateParameterList *Old, bool Complain,
     TemplateParameterListEqualKind Kind, SourceLocation TemplateArgLoc) {
-  if (Old->size() != New->size() && Kind != TPL_TemplateTemplateArgumentMatch) {
+  if (Old->size() != New->size()) {
     if (Complain)
       DiagnoseTemplateParameterListArityMismatch(*this, New, Old, Kind,
                                                  TemplateArgLoc);
@@ -7923,40 +7925,18 @@ bool Sema::TemplateParameterListsAreEqual(
   TemplateParameterList::iterator NewParm = New->begin();
   TemplateParameterList::iterator NewParmEnd = New->end();
   for (TemplateParameterList::iterator OldParm = Old->begin(),
-                                    OldParmEnd = Old->end();
-       OldParm != OldParmEnd; ++OldParm) {
-    if (Kind != TPL_TemplateTemplateArgumentMatch ||
-        !(*OldParm)->isTemplateParameterPack()) {
-      if (NewParm == NewParmEnd) {
-        if (Complain)
-          DiagnoseTemplateParameterListArityMismatch(*this, New, Old, Kind,
-                                                     TemplateArgLoc);
-
-        return false;
-      }
-
-      if (!MatchTemplateParameterKind(*this, *NewParm, NewInstFrom, *OldParm,
-                                      OldInstFrom, Complain, Kind,
-                                      TemplateArgLoc))
-        return false;
-
-      ++NewParm;
-      continue;
-    }
-
-    // C++0x [temp.arg.template]p3:
-    //   [...] When P's template- parameter-list contains a template parameter
-    //   pack (14.5.3), the template parameter pack will match zero or more
-    //   template parameters or template parameter packs in the
-    //   template-parameter-list of A with the same type and form as the
-    //   template parameter pack in P (ignoring whether those template
-    //   parameters are template parameter packs).
-    for (; NewParm != NewParmEnd; ++NewParm) {
-      if (!MatchTemplateParameterKind(*this, *NewParm, NewInstFrom, *OldParm,
-                                      OldInstFrom, Complain, Kind,
-                                      TemplateArgLoc))
-        return false;
+                                       OldParmEnd = Old->end();
+       OldParm != OldParmEnd; ++OldParm, ++NewParm) {
+    if (NewParm == NewParmEnd) {
+      if (Complain)
+        DiagnoseTemplateParameterListArityMismatch(*this, New, Old, Kind,
+                                                   TemplateArgLoc);
+      return false;
     }
+    if (!MatchTemplateParameterKind(*this, *NewParm, NewInstFrom, *OldParm,
+                                    OldInstFrom, Complain, Kind,
+                                    TemplateArgLoc))
+      return false;
   }
 
   // Make sure we exhausted all of the arguments.
@@ -7968,8 +7948,7 @@ bool Sema::TemplateParameterListsAreEqual(
     return false;
   }
 
-  if (Kind != TPL_TemplateTemplateArgumentMatch &&
-      Kind != TPL_TemplateParamsEquivalent) {
+  if (Kind != TPL_TemplateParamsEquivalent) {
     const Expr *NewRC = New->getRequiresClause();
     const Expr *OldRC = Old->getRequiresClause();
 
@@ -8441,11 +8420,10 @@ DeclResult Sema::ActOnClassTemplateSpecialization(
 
   // Check that the template argument list is well-formed for this
   // template.
-  SmallVector<TemplateArgument, 4> SugaredConverted, CanonicalConverted;
+  CheckTemplateArgumentInfo CTAI;
   if (CheckTemplateArgumentList(ClassTemplate, TemplateNameLoc, TemplateArgs,
                                 /*DefaultArgs=*/{},
-                                /*PartialTemplateArgs=*/false, SugaredConverted,
-                                CanonicalConverted,
+                                /*PartialTemplateArgs=*/false, CTAI,
                                 /*UpdateArgsWithConversions=*/true))
     return true;
 
@@ -8454,14 +8432,14 @@ DeclResult Sema::ActOnClassTemplateSpecialization(
   if (isPartialSpecialization) {
     if (CheckTemplatePartialSpecializationArgs(TemplateNameLoc, ClassTemplate,
                                                TemplateArgs.size(),
-                                               CanonicalConverted))
+                                               CTAI.CanonicalConverted))
       return true;
 
     // FIXME: Move this to CheckTemplatePartialSpecializationArgs so we
     // also do it during instantiation.
     if (!Name.isDependent() &&
         !TemplateSpecializationType::anyDependentTemplateArguments(
-            TemplateArgs, CanonicalConverted)) {
+            TemplateArgs, CTAI.CanonicalConverted)) {
       Diag(TemplateNameLoc, diag::err_partial_spec_fully_specialized)
         << ClassTemplate->getDeclName();
       isPartialSpecialization = false;
@@ -8474,9 +8452,10 @@ DeclResult Sema::ActOnClassTemplateSpecialization(
 
   if (isPartialSpecialization)
     PrevDecl = ClassTemplate->findPartialSpecialization(
-        CanonicalConverted, TemplateParams, InsertPos);
+        CTAI.CanonicalConverted, TemplateParams, InsertPos);
   else
-    PrevDecl = ClassTemplate->findSpecialization(CanonicalConverted, InsertPos);
+    PrevDecl =
+        ClassTemplate->findSpecialization(CTAI.CanonicalConverted, InsertPos);
 
   ClassTemplateSpecializationDecl *Specialization = nullptr;
 
@@ -8495,7 +8474,7 @@ DeclResult Sema::ActOnClassTemplateSpecialization(
     // arguments of the class template partial specialization.
     TemplateName CanonTemplate = Context.getCanonicalTemplateName(Name);
     CanonType = Context.getTemplateSpecializationType(CanonTemplate,
-                                                      CanonicalConverted);
+                                                      CTAI.CanonicalConverted);
 
     if (Context.hasSameType(CanonType,
                         ClassTemplate->getInjectedClassNameSpecialization()) &&
@@ -8525,7 +8504,7 @@ DeclResult Sema::ActOnClassTemplateSpecialization(
     ClassTemplatePartialSpecializationDecl *Partial =
         ClassTemplatePartialSpecializationDecl::Create(
             Context, Kind, DC, KWLoc, TemplateNameLoc, TemplateParams,
-            ClassTemplate, CanonicalConverted, CanonType, PrevPartial);
+            ClassTemplate, CTAI.CanonicalConverted, CanonType, PrevPartial);
     Partial->setTemplateArgsAsWritten(TemplateArgs);
     SetNestedNameSpecifier(*this, Partial, SS);
     if (TemplateParameterLists.size() > 1 && SS.isSet()) {
@@ -8548,7 +8527,7 @@ DeclResult Sema::ActOnClassTemplateSpecialization(
     // this explicit specialization or friend declaration.
     Specialization = ClassTemplateSpecializationDecl::Create(
         Context, Kind, DC, KWLoc, TemplateNameLoc, ClassTemplate,
-        CanonicalConverted, PrevDecl);
+        CTAI.CanonicalConverted, CTAI.StrictPackMatch, PrevDecl);
     Specialization->setTemplateArgsAsWritten(TemplateArgs);
     SetNestedNameSpecifier(*this, Specialization, SS);
     if (TemplateParameterLists.size() > 0) {
@@ -8561,8 +8540,8 @@ DeclResult Sema::ActOnClassTemplateSpecialization(
 
     if (CurContext->isDependentContext()) {
       TemplateName CanonTemplate = Context.getCanonicalTemplateName(Name);
-      CanonType = Context.getTemplateSpecializationType(CanonTemplate,
-                                                        CanonicalConverted);
+      CanonType = Context.getTemplateSpecializationType(
+          CanonTemplate, CTAI.CanonicalConverted);
     } else {
       CanonType = Context.getTypeDeclType(Specialization);
     }
@@ -9821,21 +9800,18 @@ DeclResult Sema::ActOnExplicitInstantiation(
 
   // Check that the template argument list is well-formed for this
   // template.
-  bool PrimaryHasMatchedPackOnParmToNonPackOnArg = false;
-  SmallVector<TemplateArgument, 4> SugaredConverted, CanonicalConverted;
-  if (CheckTemplateArgumentList(
-          ClassTemplate, TemplateNameLoc, TemplateArgs,
-          /*DefaultArgs=*/{}, false, SugaredConverted, CanonicalConverted,
-          /*UpdateArgsWithConversions=*/true,
-          /*ConstraintsNotSatisfied=*/nullptr, /*PartialOrderingTTP=*/false,
-          &PrimaryHasMatchedPackOnParmToNonPackOnArg))
+  CheckTemplateArgumentInfo CTAI;
+  if (CheckTemplateArgumentList(ClassTemplate, TemplateNameLoc, TemplateArgs,
+                                /*DefaultArgs=*/{}, false, CTAI,
+                                /*UpdateArgsWithConversions=*/true,
+                                /*ConstraintsNotSatisfied=*/nullptr))
     return true;
 
   // Find the class template specialization declaration that
   // corresponds to these arguments.
   void *InsertPos = nullptr;
   ClassTemplateSpecializationDecl *PrevDecl =
-      ClassTemplate->findSpecialization(CanonicalConverted, InsertPos);
+      ClassTemplate->findSpecialization(CTAI.CanonicalConverted, InsertPos);
 
   TemplateSpecializationKind PrevDecl_TSK
     = PrevDecl ? PrevDecl->getTemplateSpecializationKind() : TSK_Undeclared;
@@ -9894,7 +9870,7 @@ DeclResult Sema::ActOnExplicitInstantiation(
     // this explicit specialization.
     Specialization = ClassTemplateSpecializationDecl::Create(
         Context, Kind, ClassTemplate->getDeclContext(), KWLoc, TemplateNameLoc,
-        ClassTemplate, CanonicalConverted, PrevDecl);
+        ClassTemplate, CTAI.CanonicalConverted, CTAI.StrictPackMatch, PrevDecl);
     SetNestedNameSpecifier(*this, Specialization, SS);
 
     // A MSInheritanceAttr attached to the previous declaration must be
@@ -9949,9 +9925,9 @@ DeclResult Sema::ActOnExplicitInstantiation(
     = cast_or_null<ClassTemplateSpecializationDecl>(
                                               Specialization->getDefinition());
   if (!Def)
-    InstantiateClassTemplateSpecialization(
-        TemplateNameLoc, Specialization, TSK,
-        /*Complain=*/true, PrimaryHasMatchedPackOnParmToNonPackOnArg);
+    InstantiateClassTemplateSpecialization(TemplateNameLoc, Specialization, TSK,
+                                           /*Complain=*/true,
+                                           CTAI.StrictPackMatch);
   else if (TSK == TSK_ExplicitInstantiationDefinition) {
     MarkVTableUsed(TemplateNameLoc, Specialization, true);
     Specialization->setPointOfInstantiation(Def->getPointOfInstantiation());
diff --git a/clang/lib/Sema/SemaTemplateDeduction.cpp b/clang/lib/Sema/SemaTemplateDeduction.cpp
index 7882d7a755d34..eb70dd743c8be 100644
--- a/clang/lib/Sema/SemaTemplateDeduction.cpp
+++ b/clang/lib/Sema/SemaTemplateDeduction.cpp
@@ -53,6 +53,7 @@
 #include "llvm/Support/Casting.h"
 #include "llvm/Support/Compiler.h"
 #include "llvm/Support/ErrorHandling.h"
+#include "llvm/Support/SaveAndRestore.h"
 #include <algorithm>
 #include <cassert>
 #include <optional>
@@ -2541,10 +2542,9 @@ DeduceTemplateArguments(Sema &S, TemplateParameterList *TemplateParams,
     return TemplateDeductionResult::NonDeducedMismatch;
 
   case TemplateArgument::NullPtr:
-    if (A.getKind() == TemplateArgument::NullPtr &&
-        S.Context.hasSameType(P.getNullPtrType(), A.getNullPtrType()))
+    // 'nullptr' has only one possible value, so it always matches.
+    if (A.getKind() == TemplateArgument::NullPtr)
       return TemplateDeductionResult::Success;
-
     Info.FirstArg = P;
     Info.SecondArg = A;
     return TemplateDeductionResult::NonDeducedMismatch;
@@ -2559,6 +2559,8 @@ DeduceTemplateArguments(Sema &S, TemplateParameterList *TemplateParams,
     return TemplateDeductionResult::NonDeducedMismatch;
 
   case TemplateArgument::StructuralValue:
+    // FIXME: structural equality will also compare types,
+    // but they should match iff they have the same value.
     if (A.getKind() == TemplateArgument::StructuralValue &&
         A.structurallyEquals(P))
       return TemplateDeductionResult::Success;
@@ -2774,7 +2776,7 @@ DeduceTemplateArguments(Sema &S, TemplateParameterList *TemplateParams,
         if (!FoldPackParameter)
           return TemplateDeductionResult::MiscellaneousDeductionFailure;
         if (FoldPackArgument)
-          Info.setMatchedPackOnParmToNonPackOnArg();
+          Info.setStrictPackMatch();
       }
       // Deduce template arguments from the pattern.
       if (auto Result = DeduceTemplateArguments(
@@ -2956,11 +2958,11 @@ Sema::getIdentityTemplateArgumentLoc(NamedDecl *TemplateParm,
 
 /// Convert the given deduced template argument and add it to the set of
 /// fully-converted template arguments.
-static bool ConvertDeducedTemplateArgument(
-    Sema &S, NamedDecl *Param, DeducedTemplateArgument Arg, NamedDecl *Template,
-    TemplateDeductionInfo &Info, bool IsDeduced, bool PartialOrdering,
-    SmallVectorImpl<TemplateArgument> &SugaredOutput,
-    SmallVectorImpl<TemplateArgument> &CanonicalOutput) {
+static bool
+ConvertDeducedTemplateArgument(Sema &S, NamedDecl *Param,
+                               DeducedTemplateArgument Arg, NamedDecl *Template,
+                               TemplateDeductionInfo &Info, bool IsDeduced,
+                               Sema::CheckTemplateArgumentInfo &CTAI) {
   auto ConvertArg = [&](DeducedTemplateArgument Arg,
                         unsigned ArgumentPackIndex) {
     // Convert the deduced template argument into a template
@@ -2969,19 +2971,18 @@ static bool ConvertDeducedTemplateArgument(
     TemplateArgumentLoc ArgLoc = S.getTrivialTemplateArgumentLoc(
         Arg, QualType(), Info.getLocation(), Param);
 
-    bool MatchedPackOnParmToNonPackOnArg = false;
+    SaveAndRestore _1(CTAI.MatchingTTP, false);
+    SaveAndRestore _2(CTAI.StrictPackMatch, false);
     // Check the template argument, converting it as necessary.
     auto Res = S.CheckTemplateArgument(
         Param, ArgLoc, Template, Template->getLocation(),
-        Template->getSourceRange().getEnd(), ArgumentPackIndex, SugaredOutput,
-        CanonicalOutput,
+        Template->getSourceRange().getEnd(), ArgumentPackIndex, CTAI,
         IsDeduced
             ? (Arg.wasDeducedFromArrayBound() ? Sema::CTAK_DeducedFromArrayBound
                                               : Sema::CTAK_Deduced)
-            : Sema::CTAK_Specified,
-        PartialOrdering, &MatchedPackOnParmToNonPackOnArg);
-    if (MatchedPackOnParmToNonPackOnArg)
-      Info.setMatchedPackOnParmToNonPackOnArg();
+            : Sema::CTAK_Specified);
+    if (CTAI.StrictPackMatch)
+      Info.setStrictPackMatch();
     return Res;
   };
 
@@ -3012,20 +3013,21 @@ static bool ConvertDeducedTemplateArgument(
         return true;
 
       // Move the converted template argument into our argument pack.
-      SugaredPackedArgsBuilder.push_back(SugaredOutput.pop_back_val());
-      CanonicalPackedArgsBuilder.push_back(CanonicalOutput.pop_back_val());
+      SugaredPackedArgsBuilder.push_back(CTAI.SugaredConverted.pop_back_val());
+      CanonicalPackedArgsBuilder.push_back(
+          CTAI.CanonicalConverted.pop_back_val());
     }
 
     // If the pack is empty, we still need to substitute into the parameter
     // itself, in case that substitution fails.
     if (SugaredPackedArgsBuilder.empty()) {
       LocalInstantiationScope Scope(S);
-      MultiLevelTemplateArgumentList Args(Template, SugaredOutput,
+      MultiLevelTemplateArgumentList Args(Template, CTAI.SugaredConverted,
                                           /*Final=*/true);
 
       if (auto *NTTP = dyn_cast<NonTypeTemplateParmDecl>(Param)) {
         Sema::InstantiatingTemplate Inst(S, Template->getLocation(), Template,
-                                         NTTP, SugaredOutput,
+                                         NTTP, CTAI.SugaredConverted,
                                          Template->getSourceRange());
         if (Inst.isInvalid() ||
             S.SubstType(NTTP->getType(), Args, NTTP->getLocation(),
@@ -3033,7 +3035,7 @@ static bool ConvertDeducedTemplateArgument(
           return true;
       } else if (auto *TTP = dyn_cast<TemplateTemplateParmDecl>(Param)) {
         Sema::InstantiatingTemplate Inst(S, Template->getLocation(), Template,
-                                         TTP, SugaredOutput,
+                                         TTP, CTAI.SugaredConverted,
                                          Template->getSourceRange());
         if (Inst.isInvalid() || !S.SubstDecl(TTP, S.CurContext, Args))
           return true;
@@ -3042,9 +3044,9 @@ static bool ConvertDeducedTemplateArgument(
     }
 
     // Create the resulting argument pack.
-    SugaredOutput.push_back(
+    CTAI.SugaredConverted.push_back(
         TemplateArgument::CreatePackCopy(S.Context, SugaredPackedArgsBuilder));
-    CanonicalOutput.push_back(TemplateArgument::CreatePackCopy(
+    CTAI.CanonicalConverted.push_back(TemplateArgument::CreatePackCopy(
         S.Context, CanonicalPackedArgsBuilder));
     return false;
   }
@@ -3063,9 +3065,7 @@ template <typename TemplateDeclT>
 static TemplateDeductionResult ConvertDeducedTemplateArguments(
     Sema &S, TemplateDeclT *Template, bool IsDeduced,
     SmallVectorImpl<DeducedTemplateArgument> &Deduced,
-    TemplateDeductionInfo &Info,
-    SmallVectorImpl<TemplateArgument> &SugaredBuilder,
-    SmallVectorImpl<TemplateArgument> &CanonicalBuilder, bool PartialOrdering,
+    TemplateDeductionInfo &Info, Sema::CheckTemplateArgumentInfo &CTAI,
     LocalInstantiationScope *CurrentInstantiationScope,
     unsigned NumAlreadyConverted, bool *IsIncomplete) {
   TemplateParameterList *TemplateParams = Template->getTemplateParameters();
@@ -3100,8 +3100,8 @@ static TemplateDeductionResult ConvertDeducedTemplateArguments(
           // We have already fully type-checked and converted this
           // argument, because it was explicitly-specified. Just record the
           // presence of this argument.
-          SugaredBuilder.push_back(Deduced[I]);
-          CanonicalBuilder.push_back(
+          CTAI.SugaredConverted.push_back(Deduced[I]);
+          CTAI.CanonicalConverted.push_back(
               S.Context.getCanonicalTemplateArgument(Deduced[I]));
           continue;
         }
@@ -3110,13 +3110,13 @@ static TemplateDeductionResult ConvertDeducedTemplateArguments(
       // We may have deduced this argument, so it still needs to be
       // checked and converted.
       if (ConvertDeducedTemplateArgument(S, Param, Deduced[I], Template, Info,
-                                         IsDeduced, PartialOrdering,
-                                         SugaredBuilder, CanonicalBuilder)) {
+                                         IsDeduced, CTAI)) {
         Info.Param = makeTemplateParameter(Param);
         // FIXME: These template arguments are temporary. Free them!
         Info.reset(
-            TemplateArgumentList::CreateCopy(S.Context, SugaredBuilder),
-            TemplateArgumentList::CreateCopy(S.Context, CanonicalBuilder));
+            TemplateArgumentList::CreateCopy(S.Context, CTAI.SugaredConverted),
+            TemplateArgumentList::CreateCopy(S.Context,
+                                             CTAI.CanonicalConverted));
         return TemplateDeductionResult::SubstitutionFailure;
       }
 
@@ -3130,8 +3130,8 @@ static TemplateDeductionResult ConvertDeducedTemplateArguments(
     // arguments.
     if (IsIncomplete) {
       *IsIncomplete = true;
-      SugaredBuilder.push_back({});
-      CanonicalBuilder.push_back({});
+      CTAI.SugaredConverted.push_back({});
+      CTAI.CanonicalConverted.push_back({});
       continue;
     }
 
@@ -3160,31 +3160,34 @@ static TemplateDeductionResult ConvertDeducedTemplateArguments(
 
       DefArg = S.SubstDefaultTemplateArgumentIfAvailable(
           TD, TD->getLocation(), TD->getSourceRange().getEnd(), Param,
-          SugaredBuilder, CanonicalBuilder, HasDefaultArg);
+          CTAI.SugaredConverted, CTAI.CanonicalConverted, HasDefaultArg);
     }
 
     // If there was no default argument, deduction is incomplete.
     if (DefArg.getArgument().isNull()) {
       Info.Param = makeTemplateParameter(
           const_cast<NamedDecl *>(TemplateParams->getParam(I)));
-      Info.reset(TemplateArgumentList::CreateCopy(S.Context, SugaredBuilder),
-                 TemplateArgumentList::CreateCopy(S.Context, CanonicalBuilder));
+      Info.reset(
+          TemplateArgumentList::CreateCopy(S.Context, CTAI.SugaredConverted),
+          TemplateArgumentList::CreateCopy(S.Context, CTAI.CanonicalConverted));
 
       return HasDefaultArg ? TemplateDeductionResult::SubstitutionFailure
                            : TemplateDeductionResult::Incomplete;
     }
 
+    SaveAndRestore _1(CTAI.PartialOrdering, false);
+    SaveAndRestore _2(CTAI.MatchingTTP, false);
+    SaveAndRestore _3(CTAI.StrictPackMatch, false);
     // Check whether we can actually use the default argument.
     if (S.CheckTemplateArgument(
             Param, DefArg, TD, TD->getLocation(), TD->getSourceRange().getEnd(),
-            /*ArgumentPackIndex=*/0, SugaredBuilder, CanonicalBuilder,
-            Sema::CTAK_Specified, /*PartialOrdering=*/false,
-            /*MatchedPackOnParmToNonPackOnArg=*/nullptr)) {
+            /*ArgumentPackIndex=*/0, CTAI, Sema::CTAK_Specified)) {
       Info.Param = makeTemplateParameter(
                          const_cast<NamedDecl *>(TemplateParams->getParam(I)));
       // FIXME: These template arguments are temporary. Free them!
-      Info.reset(TemplateArgumentList::CreateCopy(S.Context, SugaredBuilder),
-                 TemplateArgumentList::CreateCopy(S.Context, CanonicalBuilder));
+      Info.reset(
+          TemplateArgumentList::CreateCopy(S.Context, CTAI.SugaredConverted),
+          TemplateArgumentList::CreateCopy(S.Context, CTAI.CanonicalConverted));
       return TemplateDeductionResult::SubstitutionFailure;
     }
 
@@ -3284,10 +3287,9 @@ FinishTemplateArgumentDeduction(
   // C++ [temp.deduct.type]p2:
   //   [...] or if any template argument remains neither deduced nor
   //   explicitly specified, template argument deduction fails.
-  SmallVector<TemplateArgument, 4> SugaredBuilder, CanonicalBuilder;
+  Sema::CheckTemplateArgumentInfo CTAI(IsPartialOrdering);
   if (auto Result = ConvertDeducedTemplateArguments(
-          S, Partial, IsPartialOrdering, Deduced, Info, SugaredBuilder,
-          CanonicalBuilder, IsPartialOrdering,
+          S, Partial, IsPartialOrdering, Deduced, Info, CTAI,
           /*CurrentInstantiationScope=*/nullptr, /*NumAlreadyConverted=*/0,
           /*IsIncomplete=*/nullptr);
       Result != TemplateDeductionResult::Success)
@@ -3295,9 +3297,9 @@ FinishTemplateArgumentDeduction(
 
   // Form the template argument list from the deduced template arguments.
   TemplateArgumentList *SugaredDeducedArgumentList =
-      TemplateArgumentList::CreateCopy(S.Context, SugaredBuilder);
+      TemplateArgumentList::CreateCopy(S.Context, CTAI.SugaredConverted);
   TemplateArgumentList *CanonicalDeducedArgumentList =
-      TemplateArgumentList::CreateCopy(S.Context, CanonicalBuilder);
+      TemplateArgumentList::CreateCopy(S.Context, CTAI.CanonicalConverted);
 
   Info.reset(SugaredDeducedArgumentList, CanonicalDeducedArgumentList);
 
@@ -3314,11 +3316,11 @@ FinishTemplateArgumentDeduction(
   TemplateArgumentListInfo InstArgs(PartialTemplArgInfo->LAngleLoc,
                                     PartialTemplArgInfo->RAngleLoc);
 
-  if (S.SubstTemplateArguments(PartialTemplArgInfo->arguments(),
-                               MultiLevelTemplateArgumentList(Partial,
-                                                              SugaredBuilder,
-                                                              /*Final=*/true),
-                               InstArgs)) {
+  if (S.SubstTemplateArguments(
+          PartialTemplArgInfo->arguments(),
+          MultiLevelTemplateArgumentList(Partial, CTAI.SugaredConverted,
+                                         /*Final=*/true),
+          InstArgs)) {
     unsigned ArgIdx = InstArgs.size(), ParamIdx = ArgIdx;
     if (ParamIdx >= Partial->getTemplateParameters()->size())
       ParamIdx = Partial->getTemplateParameters()->size() - 1;
@@ -3330,24 +3332,19 @@ FinishTemplateArgumentDeduction(
     return TemplateDeductionResult::SubstitutionFailure;
   }
 
-  bool MatchedPackOnParmToNonPackOnArg = false;
   bool ConstraintsNotSatisfied;
-  SmallVector<TemplateArgument, 4> SugaredConvertedInstArgs,
-      CanonicalConvertedInstArgs;
-  if (S.CheckTemplateArgumentList(
-          Template, Partial->getLocation(), InstArgs, /*DefaultArgs=*/{}, false,
-          SugaredConvertedInstArgs, CanonicalConvertedInstArgs,
-          /*UpdateArgsWithConversions=*/true, &ConstraintsNotSatisfied,
-          /*PartialOrderingTTP=*/false, &MatchedPackOnParmToNonPackOnArg))
+  Sema::CheckTemplateArgumentInfo InstCTAI;
+  if (S.CheckTemplateArgumentList(Template, Partial->getLocation(), InstArgs,
+                                  /*DefaultArgs=*/{}, false, InstCTAI,
+                                  /*UpdateArgsWithConversions=*/true,
+                                  &ConstraintsNotSatisfied))
     return ConstraintsNotSatisfied
                ? TemplateDeductionResult::ConstraintsNotSatisfied
                : TemplateDeductionResult::SubstitutionFailure;
-  if (MatchedPackOnParmToNonPackOnArg)
-    Info.setMatchedPackOnParmToNonPackOnArg();
 
   TemplateParameterList *TemplateParams = Template->getTemplateParameters();
   for (unsigned I = 0, E = TemplateParams->size(); I != E; ++I) {
-    TemplateArgument InstArg = SugaredConvertedInstArgs.data()[I];
+    TemplateArgument InstArg = InstCTAI.SugaredConverted.data()[I];
     if (!isSameTemplateArg(S.Context, TemplateArgs[I], InstArg,
                            IsPartialOrdering)) {
       Info.Param = makeTemplateParameter(TemplateParams->getParam(I));
@@ -3362,7 +3359,7 @@ FinishTemplateArgumentDeduction(
 
   if (!IsPartialOrdering) {
     if (auto Result = CheckDeducedArgumentConstraints(
-            S, Partial, SugaredBuilder, CanonicalBuilder, Info);
+            S, Partial, CTAI.SugaredConverted, CTAI.CanonicalConverted, Info);
         Result != TemplateDeductionResult::Success)
       return Result;
   }
@@ -3387,10 +3384,9 @@ static TemplateDeductionResult FinishTemplateArgumentDeduction(
   // C++ [temp.deduct.type]p2:
   //   [...] or if any template argument remains neither deduced nor
   //   explicitly specified, template argument deduction fails.
-  SmallVector<TemplateArgument, 4> SugaredBuilder, CanonicalBuilder;
+  Sema::CheckTemplateArgumentInfo CTAI(PartialOrdering);
   if (auto Result = ConvertDeducedTemplateArguments(
-          S, Template, /*IsDeduced=*/PartialOrdering, Deduced, Info,
-          SugaredBuilder, CanonicalBuilder, PartialOrdering,
+          S, Template, /*IsDeduced=*/PartialOrdering, Deduced, Info, CTAI,
           /*CurrentInstantiationScope=*/nullptr,
           /*NumAlreadyConverted=*/0U, /*IsIncomplete=*/nullptr);
       Result != TemplateDeductionResult::Success)
@@ -3398,7 +3394,7 @@ static TemplateDeductionResult FinishTemplateArgumentDeduction(
 
   // Check that we produced the correct argument list.
   SmallVector<ArrayRef<TemplateArgument>, 4> PsStack{TemplateArgs},
-      AsStack{CanonicalBuilder};
+      AsStack{CTAI.CanonicalConverted};
   for (;;) {
     auto take = [](SmallVectorImpl<ArrayRef<TemplateArgument>> &Stack)
         -> std::tuple<ArrayRef<TemplateArgument> &, TemplateArgument> {
@@ -3455,7 +3451,7 @@ static TemplateDeductionResult FinishTemplateArgumentDeduction(
 
   if (!PartialOrdering) {
     if (auto Result = CheckDeducedArgumentConstraints(
-            S, Template, SugaredBuilder, CanonicalBuilder, Info);
+            S, Template, CTAI.SugaredConverted, CTAI.CanonicalConverted, Info);
         Result != TemplateDeductionResult::Success)
       return Result;
   }
@@ -3479,17 +3475,16 @@ static TemplateDeductionResult FinishTemplateArgumentDeduction(
   // C++ [temp.deduct.type]p2:
   //   [...] or if any template argument remains neither deduced nor
   //   explicitly specified, template argument deduction fails.
-  SmallVector<TemplateArgument, 4> SugaredBuilder, CanonicalBuilder;
+  Sema::CheckTemplateArgumentInfo CTAI;
   if (auto Result = ConvertDeducedTemplateArguments(
-          S, TD, /*IsDeduced=*/false, Deduced, Info, SugaredBuilder,
-          CanonicalBuilder, /*PartialOrdering=*/false,
+          S, TD, /*IsDeduced=*/false, Deduced, Info, CTAI,
           /*CurrentInstantiationScope=*/nullptr, /*NumAlreadyConverted=*/0,
           /*IsIncomplete=*/nullptr);
       Result != TemplateDeductionResult::Success)
     return Result;
 
-  return ::CheckDeducedArgumentConstraints(S, TD, SugaredBuilder,
-                                           CanonicalBuilder, Info);
+  return ::CheckDeducedArgumentConstraints(S, TD, CTAI.SugaredConverted,
+                                           CTAI.CanonicalConverted, Info);
 }
 
 /// Perform template argument deduction to determine whether the given template
@@ -3670,7 +3665,6 @@ TemplateDeductionResult Sema::SubstituteExplicitTemplateArguments(
   //   declaration order of their corresponding template-parameters. The
   //   template argument list shall not specify more template-arguments than
   //   there are corresponding template-parameters.
-  SmallVector<TemplateArgument, 4> SugaredBuilder, CanonicalBuilder;
 
   // Enter a new template instantiation context where we check the
   // explicitly-specified template arguments against this function template,
@@ -3682,12 +3676,13 @@ TemplateDeductionResult Sema::SubstituteExplicitTemplateArguments(
   if (Inst.isInvalid())
     return TemplateDeductionResult::InstantiationDepth;
 
+  CheckTemplateArgumentInfo CTAI;
   if (CheckTemplateArgumentList(FunctionTemplate, SourceLocation(),
-                                ExplicitTemplateArgs, /*DefaultArgs=*/{}, true,
-                                SugaredBuilder, CanonicalBuilder,
+                                ExplicitTemplateArgs, /*DefaultArgs=*/{},
+                                /*PartialTemplateArgs=*/true, CTAI,
                                 /*UpdateArgsWithConversions=*/false) ||
       Trap.hasErrorOccurred()) {
-    unsigned Index = SugaredBuilder.size();
+    unsigned Index = CTAI.SugaredConverted.size();
     if (Index >= TemplateParams->size())
       return TemplateDeductionResult::SubstitutionFailure;
     Info.Param = makeTemplateParameter(TemplateParams->getParam(Index));
@@ -3697,9 +3692,9 @@ TemplateDeductionResult Sema::SubstituteExplicitTemplateArguments(
   // Form the template argument list from the explicitly-specified
   // template arguments.
   TemplateArgumentList *SugaredExplicitArgumentList =
-      TemplateArgumentList::CreateCopy(Context, SugaredBuilder);
+      TemplateArgumentList::CreateCopy(Context, CTAI.SugaredConverted);
   TemplateArgumentList *CanonicalExplicitArgumentList =
-      TemplateArgumentList::CreateCopy(Context, CanonicalBuilder);
+      TemplateArgumentList::CreateCopy(Context, CTAI.CanonicalConverted);
   Info.setExplicitArgs(SugaredExplicitArgumentList,
                        CanonicalExplicitArgumentList);
 
@@ -3714,15 +3709,15 @@ TemplateDeductionResult Sema::SubstituteExplicitTemplateArguments(
   // the explicit template arguments. They'll be used as part of deduction
   // for this template parameter pack.
   unsigned PartiallySubstitutedPackIndex = -1u;
-  if (!SugaredBuilder.empty()) {
-    const TemplateArgument &Arg = SugaredBuilder.back();
+  if (!CTAI.SugaredConverted.empty()) {
+    const TemplateArgument &Arg = CTAI.SugaredConverted.back();
     if (Arg.getKind() == TemplateArgument::Pack) {
-      auto *Param = TemplateParams->getParam(SugaredBuilder.size() - 1);
+      auto *Param = TemplateParams->getParam(CTAI.SugaredConverted.size() - 1);
       // If this is a fully-saturated fixed-size pack, it should be
       // fully-substituted, not partially-substituted.
       std::optional<unsigned> Expansions = getExpandedPackSize(Param);
       if (!Expansions || Arg.pack_size() < *Expansions) {
-        PartiallySubstitutedPackIndex = SugaredBuilder.size() - 1;
+        PartiallySubstitutedPackIndex = CTAI.SugaredConverted.size() - 1;
         CurrentInstantiationScope->SetPartiallySubstitutedPack(
             Param, Arg.pack_begin(), Arg.pack_size());
       }
@@ -4044,10 +4039,9 @@ TemplateDeductionResult Sema::FinishTemplateArgumentDeduction(
   //   [...] or if any template argument remains neither deduced nor
   //   explicitly specified, template argument deduction fails.
   bool IsIncomplete = false;
-  SmallVector<TemplateArgument, 4> SugaredBuilder, CanonicalBuilder;
+  CheckTemplateArgumentInfo CTAI(PartialOrdering);
   if (auto Result = ConvertDeducedTemplateArguments(
-          *this, FunctionTemplate, /*IsDeduced=*/true, Deduced, Info,
-          SugaredBuilder, CanonicalBuilder, PartialOrdering,
+          *this, FunctionTemplate, /*IsDeduced=*/true, Deduced, Info, CTAI,
           CurrentInstantiationScope, NumExplicitlySpecified,
           PartialOverloading ? &IsIncomplete : nullptr);
       Result != TemplateDeductionResult::Success)
@@ -4067,9 +4061,9 @@ TemplateDeductionResult Sema::FinishTemplateArgumentDeduction(
 
   // Form the template argument list from the deduced template arguments.
   TemplateArgumentList *SugaredDeducedArgumentList =
-      TemplateArgumentList::CreateCopy(Context, SugaredBuilder);
+      TemplateArgumentList::CreateCopy(Context, CTAI.SugaredConverted);
   TemplateArgumentList *CanonicalDeducedArgumentList =
-      TemplateArgumentList::CreateCopy(Context, CanonicalBuilder);
+      TemplateArgumentList::CreateCopy(Context, CTAI.CanonicalConverted);
   Info.reset(SugaredDeducedArgumentList, CanonicalDeducedArgumentList);
 
   // Substitute the deduced template arguments into the function template
@@ -4078,22 +4072,7 @@ TemplateDeductionResult Sema::FinishTemplateArgumentDeduction(
   if (FunctionTemplate->getFriendObjectKind())
     Owner = FunctionTemplate->getLexicalDeclContext();
   FunctionDecl *FD = FunctionTemplate->getTemplatedDecl();
-  // additional check for inline friend,
-  // ```
-  //   template <class F1> int foo(F1 X);
-  //   template <int A1> struct A {
-  //     template <class F1> friend int foo(F1 X) { return A1; }
-  //   };
-  //   template struct A<1>;
-  //   int a = foo(1.0);
-  // ```
-  const FunctionDecl *FDFriend;
-  if (FD->getFriendObjectKind() == Decl::FriendObjectKind::FOK_None &&
-      FD->isDefined(FDFriend, /*CheckForPendingFriendDefinition*/ true) &&
-      FDFriend->getFriendObjectKind() != Decl::FriendObjectKind::FOK_None) {
-    FD = const_cast<FunctionDecl *>(FDFriend);
-    Owner = FD->getLexicalDeclContext();
-  }
+
   MultiLevelTemplateArgumentList SubstArgs(
       FunctionTemplate, CanonicalDeducedArgumentList->asArray(),
       /*Final=*/false);
@@ -4130,13 +4109,13 @@ TemplateDeductionResult Sema::FinishTemplateArgumentDeduction(
   //   deduction fails.
   if (!IsIncomplete) {
     if (CheckInstantiatedFunctionTemplateConstraints(
-            Info.getLocation(), Specialization, CanonicalBuilder,
+            Info.getLocation(), Specialization, CTAI.CanonicalConverted,
             Info.AssociatedConstraintsSatisfaction))
       return TemplateDeductionResult::MiscellaneousDeductionFailure;
 
     if (!Info.AssociatedConstraintsSatisfaction.IsSatisfied) {
-      Info.reset(Info.takeSugared(),
-                 TemplateArgumentList::CreateCopy(Context, CanonicalBuilder));
+      Info.reset(Info.takeSugared(), TemplateArgumentList::CreateCopy(
+                                         Context, CTAI.CanonicalConverted));
       return TemplateDeductionResult::ConstraintsNotSatisfied;
     }
   }
@@ -5219,12 +5198,12 @@ static bool CheckDeducedPlaceholderConstraints(Sema &S, const AutoType &Type,
   for (unsigned I = 0, C = TypeLoc.getNumArgs(); I != C; ++I)
     TemplateArgs.addArgument(TypeLoc.getArgLoc(I));
 
-  llvm::SmallVector<TemplateArgument, 4> SugaredConverted, CanonicalConverted;
-  if (S.CheckTemplateArgumentList(
-          Concept, SourceLocation(), TemplateArgs, /*DefaultArgs=*/{},
-          /*PartialTemplateArgs=*/false, SugaredConverted, CanonicalConverted))
+  Sema::CheckTemplateArgumentInfo CTAI;
+  if (S.CheckTemplateArgumentList(Concept, SourceLocation(), TemplateArgs,
+                                  /*DefaultArgs=*/{},
+                                  /*PartialTemplateArgs=*/false, CTAI))
     return true;
-  MultiLevelTemplateArgumentList MLTAL(Concept, CanonicalConverted,
+  MultiLevelTemplateArgumentList MLTAL(Concept, CTAI.CanonicalConverted,
                                        /*Final=*/false);
   // Build up an EvaluationContext with an ImplicitConceptSpecializationDecl so
   // that the template arguments of the constraint can be preserved. For
@@ -5239,7 +5218,7 @@ static bool CheckDeducedPlaceholderConstraints(Sema &S, const AutoType &Type,
       S, Sema::ExpressionEvaluationContext::Unevaluated,
       ImplicitConceptSpecializationDecl::Create(
           S.getASTContext(), Concept->getDeclContext(), Concept->getLocation(),
-          CanonicalConverted));
+          CTAI.CanonicalConverted));
   if (S.CheckConstraintSatisfaction(Concept, {Concept->getConstraintExpr()},
                                     MLTAL, TypeLoc.getLocalSourceRange(),
                                     Satisfaction))
@@ -5669,10 +5648,9 @@ static TemplateDeductionResult FinishTemplateArgumentDeduction(
   //   [...] or if any template argument remains neither deduced nor
   //   explicitly specified, template argument deduction fails.
   bool IsIncomplete = false;
-  SmallVector<TemplateArgument, 4> SugaredBuilder, CanonicalBuilder;
+  Sema::CheckTemplateArgumentInfo CTAI(/*PartialOrdering=*/true);
   if (auto Result = ConvertDeducedTemplateArguments(
-          S, FTD, /*IsDeduced=*/true, Deduced, Info, SugaredBuilder,
-          CanonicalBuilder, /*PartialOrdering=*/true,
+          S, FTD, /*IsDeduced=*/true, Deduced, Info, CTAI,
           /*CurrentInstantiationScope=*/nullptr,
           /*NumAlreadyConverted=*/0, &IsIncomplete);
       Result != TemplateDeductionResult::Success)
@@ -5680,9 +5658,9 @@ static TemplateDeductionResult FinishTemplateArgumentDeduction(
 
   // Form the template argument list from the deduced template arguments.
   TemplateArgumentList *SugaredDeducedArgumentList =
-      TemplateArgumentList::CreateCopy(S.Context, SugaredBuilder);
+      TemplateArgumentList::CreateCopy(S.Context, CTAI.SugaredConverted);
   TemplateArgumentList *CanonicalDeducedArgumentList =
-      TemplateArgumentList::CreateCopy(S.Context, CanonicalBuilder);
+      TemplateArgumentList::CreateCopy(S.Context, CTAI.CanonicalConverted);
 
   Info.reset(SugaredDeducedArgumentList, CanonicalDeducedArgumentList);
 
@@ -5691,7 +5669,7 @@ static TemplateDeductionResult FinishTemplateArgumentDeduction(
   // and equivalent to the parameter.
   LocalInstantiationScope InstScope(S);
 
-  if (auto TDR = CheckDeductionConsistency(S, FTD, SugaredBuilder);
+  if (auto TDR = CheckDeductionConsistency(S, FTD, CTAI.SugaredConverted);
       TDR != TemplateDeductionResult::Success)
     return TDR;
 
@@ -6519,7 +6497,7 @@ bool Sema::isMoreSpecializedThanPrimary(
 bool Sema::isTemplateTemplateParameterAtLeastAsSpecializedAs(
     TemplateParameterList *P, TemplateDecl *PArg, TemplateDecl *AArg,
     const DefaultArguments &DefaultArgs, SourceLocation ArgLoc,
-    bool PartialOrdering, bool *MatchedPackOnParmToNonPackOnArg) {
+    bool PartialOrdering, bool *StrictPackMatch) {
   // C++1z [temp.arg.template]p4: (DR 150)
   //   A template template-parameter P is at least as specialized as a
   //   template template-argument A if, given the following rewrite to two
@@ -6568,13 +6546,17 @@ bool Sema::isTemplateTemplateParameterAtLeastAsSpecializedAs(
     // C++1z [temp.arg.template]p3:
     //   If the rewrite produces an invalid type, then P is not at least as
     //   specialized as A.
-    SmallVector<TemplateArgument, 4> CanonicalPArgs;
-    if (CheckTemplateArgumentList(
-            AArg, ArgLoc, PArgList, DefaultArgs, false, PArgs, CanonicalPArgs,
-            /*UpdateArgsWithConversions=*/true,
-            /*ConstraintsNotSatisfied=*/nullptr,
-            /*PartialOrderingTTP=*/true, MatchedPackOnParmToNonPackOnArg))
+    CheckTemplateArgumentInfo CTAI(
+        /*PartialOrdering=*/false, /*MatchingTTP=*/true);
+    CTAI.SugaredConverted = std::move(PArgs);
+    if (CheckTemplateArgumentList(AArg, ArgLoc, PArgList, DefaultArgs,
+                                  /*PartialTemplateArgs=*/false, CTAI,
+                                  /*UpdateArgsWithConversions=*/true,
+                                  /*ConstraintsNotSatisfied=*/nullptr))
       return false;
+    PArgs = std::move(CTAI.SugaredConverted);
+    if (StrictPackMatch)
+      *StrictPackMatch |= CTAI.StrictPackMatch;
   }
 
   // Determine whether P1 is at least as specialized as P2.
@@ -6599,9 +6581,8 @@ bool Sema::isTemplateTemplateParameterAtLeastAsSpecializedAs(
       PartialOrdering ? PackFold::ArgumentToParameter : PackFold::Both,
       /*HasDeducedAnyParam=*/nullptr)) {
   case clang::TemplateDeductionResult::Success:
-    if (MatchedPackOnParmToNonPackOnArg &&
-        Info.hasMatchedPackOnParmToNonPackOnArg())
-      *MatchedPackOnParmToNonPackOnArg = true;
+    if (StrictPackMatch && Info.hasStrictPackMatch())
+      *StrictPackMatch = true;
     break;
 
   case TemplateDeductionResult::MiscellaneousDeductionFailure:
diff --git a/clang/lib/Sema/SemaTemplateDeductionGuide.cpp b/clang/lib/Sema/SemaTemplateDeductionGuide.cpp
index 950783303efb3..e5931f4684a57 100644
--- a/clang/lib/Sema/SemaTemplateDeductionGuide.cpp
+++ b/clang/lib/Sema/SemaTemplateDeductionGuide.cpp
@@ -740,6 +740,28 @@ bool hasDeclaredDeductionGuides(DeclarationName Name, DeclContext *DC) {
   return false;
 }
 
+// Returns all source deduction guides associated with the declared
+// deduction guides that have the specified deduction guide name.
+llvm::DenseSet<const NamedDecl *> getSourceDeductionGuides(DeclarationName Name,
+                                                           DeclContext *DC) {
+  assert(Name.getNameKind() ==
+             DeclarationName::NameKind::CXXDeductionGuideName &&
+         "name must be a deduction guide name");
+  llvm::DenseSet<const NamedDecl *> Result;
+  for (auto *D : DC->lookup(Name)) {
+    if (const auto *FTD = dyn_cast<FunctionTemplateDecl>(D))
+      D = FTD->getTemplatedDecl();
+
+    if (const auto *GD = dyn_cast<CXXDeductionGuideDecl>(D)) {
+      assert(GD->getSourceDeductionGuide() &&
+             "deduction guide for alias template must have a source deduction "
+             "guide");
+      Result.insert(GD->getSourceDeductionGuide());
+    }
+  }
+  return Result;
+}
+
 // Build the associated constraints for the alias deduction guides.
 // C++ [over.match.class.deduct]p3.3:
 //   The associated constraints ([temp.constr.decl]) are the conjunction of the
@@ -1175,8 +1197,12 @@ BuildDeductionGuideForTypeAlias(Sema &SemaRef,
         GG->getTypeSourceInfo(), AliasTemplate->getBeginLoc(),
         AliasTemplate->getLocation(), AliasTemplate->getEndLoc(),
         F->isImplicit()));
-    cast<CXXDeductionGuideDecl>(Result->getTemplatedDecl())
-        ->setDeductionCandidateKind(GG->getDeductionCandidateKind());
+    auto *DGuide = cast<CXXDeductionGuideDecl>(Result->getTemplatedDecl());
+    DGuide->setDeductionCandidateKind(GG->getDeductionCandidateKind());
+    DGuide->setSourceDeductionGuide(
+        cast<CXXDeductionGuideDecl>(F->getTemplatedDecl()));
+    DGuide->setSourceDeductionGuideKind(
+        CXXDeductionGuideDecl::SourceDeductionGuideKind::Alias);
     return Result;
   }
   return nullptr;
@@ -1187,17 +1213,14 @@ void DeclareImplicitDeductionGuidesForTypeAlias(
   if (AliasTemplate->isInvalidDecl())
     return;
   auto &Context = SemaRef.Context;
-  // FIXME: if there is an explicit deduction guide after the first use of the
-  // type alias usage, we will not cover this explicit deduction guide. fix this
-  // case.
-  if (hasDeclaredDeductionGuides(
-          Context.DeclarationNames.getCXXDeductionGuideName(AliasTemplate),
-          AliasTemplate->getDeclContext()))
-    return;
   auto [Template, AliasRhsTemplateArgs] =
       getRHSTemplateDeclAndArgs(SemaRef, AliasTemplate);
   if (!Template)
     return;
+  auto SourceDeductionGuides = getSourceDeductionGuides(
+      Context.DeclarationNames.getCXXDeductionGuideName(AliasTemplate),
+      AliasTemplate->getDeclContext());
+
   DeclarationNameInfo NameInfo(
       Context.DeclarationNames.getCXXDeductionGuideName(Template), Loc);
   LookupResult Guides(SemaRef, NameInfo, clang::Sema::LookupOrdinaryName);
@@ -1206,6 +1229,8 @@ void DeclareImplicitDeductionGuidesForTypeAlias(
 
   for (auto *G : Guides) {
     if (auto *DG = dyn_cast<CXXDeductionGuideDecl>(G)) {
+      if (SourceDeductionGuides.contains(DG))
+        continue;
       // The deduction guide is a non-template function decl, we just clone it.
       auto *FunctionType =
           SemaRef.Context.getTrivialTypeSourceInfo(DG->getType());
@@ -1223,11 +1248,14 @@ void DeclareImplicitDeductionGuidesForTypeAlias(
         NewParam->setScopeInfo(0, I);
         FPTL.setParam(I, NewParam);
       }
-      auto *Transformed = cast<FunctionDecl>(buildDeductionGuide(
+      auto *Transformed = cast<CXXDeductionGuideDecl>(buildDeductionGuide(
           SemaRef, AliasTemplate, /*TemplateParams=*/nullptr,
           /*Constructor=*/nullptr, DG->getExplicitSpecifier(), FunctionType,
           AliasTemplate->getBeginLoc(), AliasTemplate->getLocation(),
           AliasTemplate->getEndLoc(), DG->isImplicit()));
+      Transformed->setSourceDeductionGuide(DG);
+      Transformed->setSourceDeductionGuideKind(
+          CXXDeductionGuideDecl::SourceDeductionGuideKind::Alias);
 
       // FIXME: Here the synthesized deduction guide is not a templated
       // function. Per [dcl.decl]p4, the requires-clause shall be present only
@@ -1242,9 +1270,10 @@ void DeclareImplicitDeductionGuidesForTypeAlias(
           Constraint = Conjunction.getAs<Expr>();
       }
       Transformed->setTrailingRequiresClause(Constraint);
+      continue;
     }
     FunctionTemplateDecl *F = dyn_cast<FunctionTemplateDecl>(G);
-    if (!F)
+    if (!F || SourceDeductionGuides.contains(F->getTemplatedDecl()))
       continue;
     // The **aggregate** deduction guides are handled in a different code path
     // (DeclareAggregateDeductionGuideFromInitList), which involves the tricky
diff --git a/clang/lib/Sema/SemaTemplateInstantiate.cpp b/clang/lib/Sema/SemaTemplateInstantiate.cpp
index 3dc5696bd3821..d9a47ca3daa20 100644
--- a/clang/lib/Sema/SemaTemplateInstantiate.cpp
+++ b/clang/lib/Sema/SemaTemplateInstantiate.cpp
@@ -190,7 +190,7 @@ HandleVarTemplateSpec(const VarTemplateSpecializationDecl *VarTemplSpec,
   llvm::PointerUnion<VarTemplateDecl *, VarTemplatePartialSpecializationDecl *>
       Specialized = VarTemplSpec->getSpecializedTemplateOrPartial();
   if (VarTemplatePartialSpecializationDecl *Partial =
-          Specialized.dyn_cast<VarTemplatePartialSpecializationDecl *>()) {
+          dyn_cast<VarTemplatePartialSpecializationDecl *>(Specialized)) {
     if (!SkipForSpecialization)
       Result.addOuterTemplateArguments(
           Partial, VarTemplSpec->getTemplateInstantiationArgs().asArray(),
@@ -479,9 +479,6 @@ MultiLevelTemplateArgumentList Sema::getTemplateInstantiationArgs(
   using namespace TemplateInstArgsHelpers;
   const Decl *CurDecl = ND;
 
-  if (!CurDecl)
-    CurDecl = Decl::castFromDeclContext(DC);
-
   if (Innermost) {
     Result.addOuterTemplateArguments(const_cast<NamedDecl *>(ND), *Innermost,
                                      Final);
@@ -495,8 +492,10 @@ MultiLevelTemplateArgumentList Sema::getTemplateInstantiationArgs(
     // has a depth of 0.
     if (const auto *TTP = dyn_cast<TemplateTemplateParmDecl>(CurDecl))
       HandleDefaultTempArgIntoTempTempParam(TTP, Result);
-    CurDecl = Response::UseNextDecl(CurDecl).NextDecl;
-  }
+    CurDecl = DC ? Decl::castFromDeclContext(DC)
+                 : Response::UseNextDecl(CurDecl).NextDecl;
+  } else if (!CurDecl)
+    CurDecl = Decl::castFromDeclContext(DC);
 
   while (!CurDecl->isFileContextDecl()) {
     Response R;
@@ -1584,6 +1583,10 @@ namespace {
     /// pack.
     ExprResult TransformFunctionParmPackExpr(FunctionParmPackExpr *E);
 
+    // Transform a ResolvedUnexpandedPackExpr
+    ExprResult
+    TransformResolvedUnexpandedPackExpr(ResolvedUnexpandedPackExpr *E);
+
     QualType TransformFunctionProtoType(TypeLocBuilder &TLB,
                                         FunctionProtoTypeLoc TL) {
       // Call the base version; it will forward to our overridden version below.
@@ -1762,23 +1765,6 @@ namespace {
       return inherited::TransformLambdaBody(E, Body);
     }
 
-    ExprResult TransformSizeOfPackExpr(SizeOfPackExpr *E) {
-      ExprResult Transformed = inherited::TransformSizeOfPackExpr(E);
-      if (!Transformed.isUsable())
-        return Transformed;
-      auto *TransformedExpr = cast<SizeOfPackExpr>(Transformed.get());
-      if (SemaRef.CodeSynthesisContexts.back().Kind ==
-              Sema::CodeSynthesisContext::ConstraintNormalization &&
-          TransformedExpr->getPack() == E->getPack()) {
-        Decl *NewPack =
-            TransformDecl(E->getPackLoc(), TransformedExpr->getPack());
-        if (!NewPack)
-          return ExprError();
-        TransformedExpr->setPack(cast<NamedDecl>(NewPack));
-      }
-      return TransformedExpr;
-    }
-
     ExprResult TransformRequiresExpr(RequiresExpr *E) {
       LocalInstantiationScope Scope(SemaRef, /*CombineWithOuterScope=*/true);
       ExprResult TransReq = inherited::TransformRequiresExpr(E);
@@ -1866,7 +1852,8 @@ bool TemplateInstantiator::AlreadyTransformed(QualType T) {
   if (T.isNull())
     return true;
 
-  if (T->isInstantiationDependentType() || T->isVariablyModifiedType())
+  if (T->isInstantiationDependentType() || T->isVariablyModifiedType() ||
+      T->containsUnexpandedParameterPack())
     return false;
 
   getSema().MarkDeclarationsReferencedInType(Loc, T);
@@ -1902,15 +1889,6 @@ Decl *TemplateInstantiator::TransformDecl(SourceLocation Loc, Decl *D) {
       TemplateArgument Arg = TemplateArgs(TTP->getDepth(), TTP->getPosition());
 
       if (TTP->isParameterPack()) {
-        // We might not have an index for pack expansion when normalizing
-        // constraint expressions. In that case, resort to instantiation scopes
-        // for the transformed declarations.
-        if (SemaRef.ArgumentPackSubstitutionIndex == -1 &&
-            SemaRef.CodeSynthesisContexts.back().Kind ==
-                Sema::CodeSynthesisContext::ConstraintNormalization) {
-          return SemaRef.FindInstantiatedDecl(Loc, cast<NamedDecl>(D),
-                                              TemplateArgs);
-        }
         assert(Arg.getKind() == TemplateArgument::Pack &&
                "Missing argument pack");
         Arg = getPackSubstitutedTemplateArgument(getSema(), Arg);
@@ -2399,9 +2377,10 @@ TemplateInstantiator::TransformSubstNonTypeTemplateParmExpr(
   // The call to CheckTemplateArgument here produces the ImpCast.
   TemplateArgument SugaredConverted, CanonicalConverted;
   if (SemaRef
-          .CheckTemplateArgument(E->getParameter(), SubstType,
-                                 SubstReplacement.get(), SugaredConverted,
-                                 CanonicalConverted, Sema::CTAK_Specified)
+          .CheckTemplateArgument(
+              E->getParameter(), SubstType, SubstReplacement.get(),
+              SugaredConverted, CanonicalConverted,
+              /*PartialOrderingTTP=*/false, Sema::CTAK_Specified)
           .isInvalid())
     return true;
   return transformNonTypeTemplateParmRef(E->getAssociatedDecl(),
@@ -2458,7 +2437,7 @@ TemplateInstantiator::TransformFunctionParmPackRefExpr(DeclRefExpr *E,
   assert(Found && "no instantiation for parameter pack");
 
   Decl *TransformedDecl;
-  if (DeclArgumentPack *Pack = Found->dyn_cast<DeclArgumentPack *>()) {
+  if (DeclArgumentPack *Pack = dyn_cast<DeclArgumentPack *>(*Found)) {
     // If this is a reference to a function parameter pack which we can
     // substitute but can't yet expand, build a FunctionParmPackExpr for it.
     if (getSema().ArgumentPackSubstitutionIndex == -1) {
@@ -2499,6 +2478,15 @@ TemplateInstantiator::TransformDeclRefExpr(DeclRefExpr *E) {
     if (PD->isParameterPack())
       return TransformFunctionParmPackRefExpr(E, PD);
 
+  if (BindingDecl *BD = dyn_cast<BindingDecl>(D); BD && BD->isParameterPack()) {
+    BD = cast_or_null<BindingDecl>(TransformDecl(BD->getLocation(), BD));
+    if (!BD)
+      return ExprError();
+    if (auto *RP =
+            dyn_cast_if_present<ResolvedUnexpandedPackExpr>(BD->getBinding()))
+      return TransformResolvedUnexpandedPackExpr(RP);
+  }
+
   return inherited::TransformDeclRefExpr(E);
 }
 
@@ -2663,6 +2651,19 @@ TemplateInstantiator::TransformTemplateTypeParmType(TypeLocBuilder &TLB,
   return Result;
 }
 
+ExprResult TemplateInstantiator::TransformResolvedUnexpandedPackExpr(
+    ResolvedUnexpandedPackExpr *E) {
+  if (getSema().ArgumentPackSubstitutionIndex != -1) {
+    assert(static_cast<unsigned>(getSema().ArgumentPackSubstitutionIndex) <
+               E->getNumExprs() &&
+           "ArgumentPackSubstitutionIndex is out of range");
+    return TransformExpr(
+        E->getExpansion(getSema().ArgumentPackSubstitutionIndex));
+  }
+
+  return inherited::TransformResolvedUnexpandedPackExpr(E);
+}
+
 QualType TemplateInstantiator::TransformSubstTemplateTypeParmPackType(
     TypeLocBuilder &TLB, SubstTemplateTypeParmPackTypeLoc TL,
     bool SuppressObjCLifetime) {
@@ -4057,8 +4058,7 @@ bool Sema::usesPartialOrExplicitSpecialization(
 static ActionResult<CXXRecordDecl *> getPatternForClassTemplateSpecialization(
     Sema &S, SourceLocation PointOfInstantiation,
     ClassTemplateSpecializationDecl *ClassTemplateSpec,
-    TemplateSpecializationKind TSK,
-    bool PrimaryHasMatchedPackOnParmToNonPackOnArg) {
+    TemplateSpecializationKind TSK, bool PrimaryStrictPackMatch) {
   Sema::InstantiatingTemplate Inst(S, PointOfInstantiation, ClassTemplateSpec);
   if (Inst.isInvalid())
     return {/*Invalid=*/true};
@@ -4111,12 +4111,11 @@ static ActionResult<CXXRecordDecl *> getPatternForClassTemplateSpecialization(
             MakeDeductionFailureInfo(S.Context, Result, Info));
         (void)Result;
       } else {
-        auto &List =
-            Info.hasMatchedPackOnParmToNonPackOnArg() ? ExtraMatched : Matched;
+        auto &List = Info.hasStrictPackMatch() ? ExtraMatched : Matched;
         List.push_back(MatchResult{Partial, Info.takeCanonical()});
       }
     }
-    if (Matched.empty() && PrimaryHasMatchedPackOnParmToNonPackOnArg)
+    if (Matched.empty() && PrimaryStrictPackMatch)
       Matched = std::move(ExtraMatched);
 
     // If we're dealing with a member template where the template parameters
@@ -4221,7 +4220,7 @@ bool Sema::InstantiateClassTemplateSpecialization(
     SourceLocation PointOfInstantiation,
     ClassTemplateSpecializationDecl *ClassTemplateSpec,
     TemplateSpecializationKind TSK, bool Complain,
-    bool PrimaryHasMatchedPackOnParmToNonPackOnArg) {
+    bool PrimaryStrictPackMatch) {
   // Perform the actual instantiation on the canonical declaration.
   ClassTemplateSpec = cast<ClassTemplateSpecializationDecl>(
       ClassTemplateSpec->getCanonicalDecl());
@@ -4229,9 +4228,9 @@ bool Sema::InstantiateClassTemplateSpecialization(
     return true;
 
   ActionResult<CXXRecordDecl *> Pattern =
-      getPatternForClassTemplateSpecialization(
-          *this, PointOfInstantiation, ClassTemplateSpec, TSK,
-          PrimaryHasMatchedPackOnParmToNonPackOnArg);
+      getPatternForClassTemplateSpecialization(*this, PointOfInstantiation,
+                                               ClassTemplateSpec, TSK,
+                                               PrimaryStrictPackMatch);
   if (!Pattern.isUsable())
     return Pattern.isInvalid();
 
diff --git a/clang/lib/Sema/SemaTemplateInstantiateDecl.cpp b/clang/lib/Sema/SemaTemplateInstantiateDecl.cpp
index 6a2331e59477a..1f42f9500959e 100644
--- a/clang/lib/Sema/SemaTemplateInstantiateDecl.cpp
+++ b/clang/lib/Sema/SemaTemplateInstantiateDecl.cpp
@@ -12,6 +12,7 @@
 #include "TreeTransform.h"
 #include "clang/AST/ASTConsumer.h"
 #include "clang/AST/ASTContext.h"
+#include "clang/AST/ASTLambda.h"
 #include "clang/AST/ASTMutationListener.h"
 #include "clang/AST/DeclTemplate.h"
 #include "clang/AST/DependentDiagnostic.h"
@@ -284,7 +285,8 @@ static void instantiateDependentDiagnoseIfAttr(
   if (Cond)
     New->addAttr(new (S.getASTContext()) DiagnoseIfAttr(
         S.getASTContext(), *DIA, Cond, DIA->getMessage(),
-        DIA->getDiagnosticType(), DIA->getArgDependent(), New));
+        DIA->getDefaultSeverity(), DIA->getWarningGroup(),
+        DIA->getArgDependent(), New));
 }
 
 // Constructs and adds to New a new instance of CUDALaunchBoundsAttr using
@@ -1166,26 +1168,57 @@ TemplateDeclInstantiator::VisitTypeAliasTemplateDecl(TypeAliasTemplateDecl *D) {
 
 Decl *TemplateDeclInstantiator::VisitBindingDecl(BindingDecl *D) {
   auto *NewBD = BindingDecl::Create(SemaRef.Context, Owner, D->getLocation(),
-                                    D->getIdentifier());
+                                    D->getIdentifier(), D->getType());
   NewBD->setReferenced(D->isReferenced());
   SemaRef.CurrentInstantiationScope->InstantiatedLocal(D, NewBD);
+
   return NewBD;
 }
 
 Decl *TemplateDeclInstantiator::VisitDecompositionDecl(DecompositionDecl *D) {
   // Transform the bindings first.
+  // The transformed DD will have all of the concrete BindingDecls.
   SmallVector<BindingDecl*, 16> NewBindings;
-  for (auto *OldBD : D->bindings())
+  ResolvedUnexpandedPackExpr *OldResolvedPack = nullptr;
+  for (auto *OldBD : D->bindings()) {
+    Expr *BindingExpr = OldBD->getBinding();
+    if (auto *RP =
+            dyn_cast_if_present<ResolvedUnexpandedPackExpr>(BindingExpr)) {
+      assert(!OldResolvedPack && "no more than one pack is allowed");
+      OldResolvedPack = RP;
+    }
     NewBindings.push_back(cast<BindingDecl>(VisitBindingDecl(OldBD)));
+  }
   ArrayRef<BindingDecl*> NewBindingArray = NewBindings;
 
-  auto *NewDD = cast_or_null<DecompositionDecl>(
+  auto *NewDD = cast_if_present<DecompositionDecl>(
       VisitVarDecl(D, /*InstantiatingVarTemplate=*/false, &NewBindingArray));
 
   if (!NewDD || NewDD->isInvalidDecl())
     for (auto *NewBD : NewBindings)
       NewBD->setInvalidDecl();
 
+  if (OldResolvedPack) {
+    // Mark the holding vars (if any) in the pack as instantiated since
+    // they are created implicitly.
+    auto Bindings = NewDD->bindings();
+    auto BPack = llvm::find_if(
+        Bindings, [](BindingDecl *D) -> bool { return D->isParameterPack(); });
+    auto *NewResolvedPack =
+        cast<ResolvedUnexpandedPackExpr>((*BPack)->getBinding());
+    auto OldExprs = OldResolvedPack->getExprs();
+    auto NewExprs = NewResolvedPack->getExprs();
+    assert(OldExprs.size() == NewExprs.size());
+    for (unsigned I = 0; I < OldResolvedPack->getNumExprs(); I++) {
+      DeclRefExpr *OldDRE = cast<DeclRefExpr>(OldExprs[I]);
+      BindingDecl *OldNestedBD = cast<BindingDecl>(OldDRE->getDecl());
+      DeclRefExpr *NewDRE = cast<DeclRefExpr>(NewExprs[I]);
+      BindingDecl *NewNestedBD = cast<BindingDecl>(NewDRE->getDecl());
+      SemaRef.CurrentInstantiationScope->InstantiatedLocal(OldNestedBD,
+                                                           NewNestedBD);
+    }
+  }
+
   return NewDD;
 }
 
@@ -1627,12 +1660,17 @@ Decl *TemplateDeclInstantiator::VisitEnumDecl(EnumDecl *D) {
   // specialization causes the implicit instantiation of the declarations, but
   // not the definitions of scoped member enumerations.
   //
-  // DR1484 clarifies that enumeration definitions inside of a template
+  // DR1484 clarifies that enumeration definitions inside a template
   // declaration aren't considered entities that can be separately instantiated
-  // from the rest of the entity they are declared inside of.
+  // from the rest of the entity they are declared inside.
   if (isDeclWithinFunction(D) ? D == Def : Def && !Enum->isScoped()) {
-    SemaRef.CurrentInstantiationScope->InstantiatedLocal(D, Enum);
-    InstantiateEnumDefinition(Enum, Def);
+    // Prevent redundant instantiation of the enumerator-definition if the
+    // definition has already been instantiated due to a prior
+    // opaque-enum-declaration.
+    if (PrevDecl == nullptr) {
+      SemaRef.CurrentInstantiationScope->InstantiatedLocal(D, Enum);
+      InstantiateEnumDefinition(Enum, Def);
+    }
   }
 
   return Enum;
@@ -2264,7 +2302,9 @@ Decl *TemplateDeclInstantiator::VisitFunctionDecl(
         SemaRef.Context, DC, D->getInnerLocStart(),
         InstantiatedExplicitSpecifier, NameInfo, T, TInfo,
         D->getSourceRange().getEnd(), DGuide->getCorrespondingConstructor(),
-        DGuide->getDeductionCandidateKind(), TrailingRequiresClause);
+        DGuide->getDeductionCandidateKind(), TrailingRequiresClause,
+        DGuide->getSourceDeductionGuide(),
+        DGuide->getSourceDeductionGuideKind());
     Function->setAccess(D->getAccess());
   } else {
     Function = FunctionDecl::Create(
@@ -2606,8 +2646,7 @@ Decl *TemplateDeclInstantiator::VisitCXXMethodDecl(
   // conditionally populate the TSI without breaking non-template related use
   // cases. Populate TSIs prior to calling SubstFunctionType to make sure we get
   // a proper transformation.
-  if (cast<CXXRecordDecl>(D->getParent())->isLambda() &&
-      !D->getTypeSourceInfo() &&
+  if (isLambdaMethod(D) && !D->getTypeSourceInfo() &&
       isa<CXXConstructorDecl, CXXDestructorDecl>(D)) {
     TypeSourceInfo *TSI =
         SemaRef.Context.getTrivialTypeSourceInfo(D->getType());
@@ -3950,10 +3989,10 @@ TemplateDeclInstantiator::VisitClassTemplateSpecializationDecl(
 
   // Check that the template argument list is well-formed for this
   // class template.
-  SmallVector<TemplateArgument, 4> SugaredConverted, CanonicalConverted;
+  Sema::CheckTemplateArgumentInfo CTAI;
   if (SemaRef.CheckTemplateArgumentList(
           InstClassTemplate, D->getLocation(), InstTemplateArgs,
-          /*DefaultArgs=*/{}, false, SugaredConverted, CanonicalConverted,
+          /*DefaultArgs=*/{}, /*PartialTemplateArgs=*/false, CTAI,
           /*UpdateArgsWithConversions=*/true))
     return nullptr;
 
@@ -3961,7 +4000,7 @@ TemplateDeclInstantiator::VisitClassTemplateSpecializationDecl(
   // in the member template's set of class template explicit specializations.
   void *InsertPos = nullptr;
   ClassTemplateSpecializationDecl *PrevDecl =
-      InstClassTemplate->findSpecialization(CanonicalConverted, InsertPos);
+      InstClassTemplate->findSpecialization(CTAI.CanonicalConverted, InsertPos);
 
   // Check whether we've already seen a conflicting instantiation of this
   // declaration (for instance, if there was a prior implicit instantiation).
@@ -3999,7 +4038,8 @@ TemplateDeclInstantiator::VisitClassTemplateSpecializationDecl(
   ClassTemplateSpecializationDecl *InstD =
       ClassTemplateSpecializationDecl::Create(
           SemaRef.Context, D->getTagKind(), Owner, D->getBeginLoc(),
-          D->getLocation(), InstClassTemplate, CanonicalConverted, PrevDecl);
+          D->getLocation(), InstClassTemplate, CTAI.CanonicalConverted,
+          CTAI.StrictPackMatch, PrevDecl);
   InstD->setTemplateArgsAsWritten(InstTemplateArgs);
 
   // Add this partial specialization to the set of class template partial
@@ -4058,17 +4098,17 @@ Decl *TemplateDeclInstantiator::VisitVarTemplateSpecializationDecl(
   }
 
   // Check that the template argument list is well-formed for this template.
-  SmallVector<TemplateArgument, 4> SugaredConverted, CanonicalConverted;
+  Sema::CheckTemplateArgumentInfo CTAI;
   if (SemaRef.CheckTemplateArgumentList(
           InstVarTemplate, D->getLocation(), VarTemplateArgsInfo,
-          /*DefaultArgs=*/{}, false, SugaredConverted, CanonicalConverted,
+          /*DefaultArgs=*/{}, /*PartialTemplateArgs=*/false, CTAI,
           /*UpdateArgsWithConversions=*/true))
     return nullptr;
 
   // Check whether we've already seen a declaration of this specialization.
   void *InsertPos = nullptr;
   VarTemplateSpecializationDecl *PrevDecl =
-      InstVarTemplate->findSpecialization(CanonicalConverted, InsertPos);
+      InstVarTemplate->findSpecialization(CTAI.CanonicalConverted, InsertPos);
 
   // Check whether we've already seen a conflicting instantiation of this
   // declaration (for instance, if there was a prior implicit instantiation).
@@ -4079,8 +4119,9 @@ Decl *TemplateDeclInstantiator::VisitVarTemplateSpecializationDecl(
                       PrevDecl->getPointOfInstantiation(), Ignored))
     return nullptr;
 
-  return VisitVarTemplateSpecializationDecl(
-      InstVarTemplate, D, VarTemplateArgsInfo, CanonicalConverted, PrevDecl);
+  return VisitVarTemplateSpecializationDecl(InstVarTemplate, D,
+                                            VarTemplateArgsInfo,
+                                            CTAI.CanonicalConverted, PrevDecl);
 }
 
 Decl *TemplateDeclInstantiator::VisitVarTemplateSpecializationDecl(
@@ -4324,37 +4365,37 @@ TemplateDeclInstantiator::InstantiateClassTemplatePartialSpecialization(
 
   // Check that the template argument list is well-formed for this
   // class template.
-  SmallVector<TemplateArgument, 4> SugaredConverted, CanonicalConverted;
+  Sema::CheckTemplateArgumentInfo CTAI;
   if (SemaRef.CheckTemplateArgumentList(
           ClassTemplate, PartialSpec->getLocation(), InstTemplateArgs,
           /*DefaultArgs=*/{},
-          /*PartialTemplateArgs=*/false, SugaredConverted, CanonicalConverted))
+          /*PartialTemplateArgs=*/false, CTAI))
     return nullptr;
 
   // Check these arguments are valid for a template partial specialization.
   if (SemaRef.CheckTemplatePartialSpecializationArgs(
           PartialSpec->getLocation(), ClassTemplate, InstTemplateArgs.size(),
-          CanonicalConverted))
+          CTAI.CanonicalConverted))
     return nullptr;
 
   // Figure out where to insert this class template partial specialization
   // in the member template's set of class template partial specializations.
   void *InsertPos = nullptr;
   ClassTemplateSpecializationDecl *PrevDecl =
-      ClassTemplate->findPartialSpecialization(CanonicalConverted, InstParams,
-                                               InsertPos);
+      ClassTemplate->findPartialSpecialization(CTAI.CanonicalConverted,
+                                               InstParams, InsertPos);
 
   // Build the canonical type that describes the converted template
   // arguments of the class template partial specialization.
   QualType CanonType = SemaRef.Context.getTemplateSpecializationType(
-      TemplateName(ClassTemplate), CanonicalConverted);
+      TemplateName(ClassTemplate), CTAI.CanonicalConverted);
 
   // Create the class template partial specialization declaration.
   ClassTemplatePartialSpecializationDecl *InstPartialSpec =
       ClassTemplatePartialSpecializationDecl::Create(
           SemaRef.Context, PartialSpec->getTagKind(), Owner,
           PartialSpec->getBeginLoc(), PartialSpec->getLocation(), InstParams,
-          ClassTemplate, CanonicalConverted, CanonType,
+          ClassTemplate, CTAI.CanonicalConverted, CanonType,
           /*PrevDecl=*/nullptr);
 
   InstPartialSpec->setTemplateArgsAsWritten(InstTemplateArgs);
@@ -4438,25 +4479,24 @@ TemplateDeclInstantiator::InstantiateVarTemplatePartialSpecialization(
 
   // Check that the template argument list is well-formed for this
   // class template.
-  SmallVector<TemplateArgument, 4> SugaredConverted, CanonicalConverted;
+  Sema::CheckTemplateArgumentInfo CTAI;
   if (SemaRef.CheckTemplateArgumentList(VarTemplate, PartialSpec->getLocation(),
                                         InstTemplateArgs, /*DefaultArgs=*/{},
-                                        /*PartialTemplateArgs=*/false,
-                                        SugaredConverted, CanonicalConverted))
+                                        /*PartialTemplateArgs=*/false, CTAI))
     return nullptr;
 
   // Check these arguments are valid for a template partial specialization.
   if (SemaRef.CheckTemplatePartialSpecializationArgs(
           PartialSpec->getLocation(), VarTemplate, InstTemplateArgs.size(),
-          CanonicalConverted))
+          CTAI.CanonicalConverted))
     return nullptr;
 
   // Figure out where to insert this variable template partial specialization
   // in the member template's set of variable template partial specializations.
   void *InsertPos = nullptr;
   VarTemplateSpecializationDecl *PrevDecl =
-      VarTemplate->findPartialSpecialization(CanonicalConverted, InstParams,
-                                             InsertPos);
+      VarTemplate->findPartialSpecialization(CTAI.CanonicalConverted,
+                                             InstParams, InsertPos);
 
   // Do substitution on the type of the declaration
   TypeSourceInfo *DI = SemaRef.SubstType(
@@ -4477,7 +4517,7 @@ TemplateDeclInstantiator::InstantiateVarTemplatePartialSpecialization(
       VarTemplatePartialSpecializationDecl::Create(
           SemaRef.Context, Owner, PartialSpec->getInnerLocStart(),
           PartialSpec->getLocation(), InstParams, VarTemplate, DI->getType(),
-          DI, PartialSpec->getStorageClass(), CanonicalConverted);
+          DI, PartialSpec->getStorageClass(), CTAI.CanonicalConverted);
 
   InstPartialSpec->setTemplateArgsAsWritten(InstTemplateArgs);
 
@@ -5237,9 +5277,31 @@ void Sema::InstantiateFunctionDefinition(SourceLocation PointOfInstantiation,
     RebuildTypeSourceInfoForDefaultSpecialMembers();
     SetDeclDefaulted(Function, PatternDecl->getLocation());
   } else {
+    NamedDecl *ND = Function;
+    DeclContext *DC = ND->getLexicalDeclContext();
+    std::optional<ArrayRef<TemplateArgument>> Innermost;
+    if (auto *Primary = Function->getPrimaryTemplate();
+        Primary &&
+        !isGenericLambdaCallOperatorOrStaticInvokerSpecialization(Function) &&
+        Function->getTemplateSpecializationKind() !=
+            TSK_ExplicitSpecialization) {
+      auto It = llvm::find_if(Primary->redecls(),
+                              [](const RedeclarableTemplateDecl *RTD) {
+                                return cast<FunctionTemplateDecl>(RTD)
+                                    ->isCompatibleWithDefinition();
+                              });
+      assert(It != Primary->redecls().end() &&
+             "Should't get here without a definition");
+      if (FunctionDecl *Def = cast<FunctionTemplateDecl>(*It)
+                                  ->getTemplatedDecl()
+                                  ->getDefinition())
+        DC = Def->getLexicalDeclContext();
+      else
+        DC = (*It)->getLexicalDeclContext();
+      Innermost.emplace(Function->getTemplateSpecializationArgs()->asArray());
+    }
     MultiLevelTemplateArgumentList TemplateArgs = getTemplateInstantiationArgs(
-        Function, Function->getLexicalDeclContext(), /*Final=*/false,
-        /*Innermost=*/std::nullopt, false, PatternDecl);
+        Function, DC, /*Final=*/false, Innermost, false, PatternDecl);
 
     // Substitute into the qualifier; we can get a substitution failure here
     // through evil use of alias templates.
@@ -6214,8 +6276,16 @@ NamedDecl *Sema::FindInstantiatedDecl(SourceLocation Loc, NamedDecl *D,
     // declarations to their instantiations.
     if (CurrentInstantiationScope) {
       if (auto Found = CurrentInstantiationScope->findInstantiationOf(D)) {
-        if (Decl *FD = Found->dyn_cast<Decl *>())
+        if (Decl *FD = Found->dyn_cast<Decl *>()) {
+          if (auto *BD = dyn_cast<BindingDecl>(FD);
+              BD && BD->isParameterPack() &&
+              ArgumentPackSubstitutionIndex != -1) {
+            auto *DRE = cast<DeclRefExpr>(
+                BD->getBindingPackExprs()[ArgumentPackSubstitutionIndex]);
+            return cast<NamedDecl>(DRE->getDecl());
+          }
           return cast<NamedDecl>(FD);
+        }
 
         int PackIdx = ArgumentPackSubstitutionIndex;
         assert(PackIdx != -1 &&
diff --git a/clang/lib/Sema/SemaTemplateVariadic.cpp b/clang/lib/Sema/SemaTemplateVariadic.cpp
index c8452db6bc901..3c56794722dcc 100644
--- a/clang/lib/Sema/SemaTemplateVariadic.cpp
+++ b/clang/lib/Sema/SemaTemplateVariadic.cpp
@@ -50,17 +50,29 @@ class CollectUnexpandedParameterPacksVisitor
         auto *FTD = FD ? FD->getDescribedFunctionTemplate() : nullptr;
         if (FTD && FTD->getTemplateParameters()->getDepth() >= DepthLimit)
           return;
-      } else if (getDepthAndIndex(ND).first >= DepthLimit)
+      } else if (auto *BD = dyn_cast<BindingDecl>(ND)) {
+        Expr *E = BD->getBinding();
+        if (auto *RP = cast_if_present<ResolvedUnexpandedPackExpr>(E)) {
+          addUnexpanded(RP);
+          return;
+        }
+      } else if (getDepthAndIndex(ND).first >= DepthLimit) {
         return;
+      }
 
       Unexpanded.push_back({ND, Loc});
     }
+
     void addUnexpanded(const TemplateTypeParmType *T,
                        SourceLocation Loc = SourceLocation()) {
       if (T->getDepth() < DepthLimit)
         Unexpanded.push_back({T, Loc});
     }
 
+    void addUnexpanded(ResolvedUnexpandedPackExpr *E) {
+      Unexpanded.push_back({E, E->getBeginLoc()});
+    }
+
   public:
     explicit CollectUnexpandedParameterPacksVisitor(
         SmallVectorImpl<UnexpandedParameterPack> &Unexpanded)
@@ -103,6 +115,12 @@ class CollectUnexpandedParameterPacksVisitor
       return true;
     }
 
+    bool
+    VisitResolvedUnexpandedPackExpr(ResolvedUnexpandedPackExpr *E) override {
+      addUnexpanded(E);
+      return true;
+    }
+
     /// Record occurrences of template template parameter packs.
     bool TraverseTemplateName(TemplateName Template) override {
       if (auto *TTP = dyn_cast_or_null<TemplateTemplateParmDecl>(
@@ -422,8 +440,8 @@ Sema::DiagnoseUnexpandedParameterPacks(SourceLocation Loc,
     if (const TemplateTypeParmType *TTP
           = Unexpanded[I].first.dyn_cast<const TemplateTypeParmType *>())
       Name = TTP->getIdentifier();
-    else
-      Name = cast<NamedDecl *>(Unexpanded[I].first)->getIdentifier();
+    else if (NamedDecl *ND = Unexpanded[I].first.dyn_cast<NamedDecl *>())
+      Name = ND->getIdentifier();
 
     if (Name && NamesKnown.insert(Name).second)
       Names.push_back(Name);
@@ -757,23 +775,39 @@ bool Sema::CheckParameterPacksForExpansion(
   bool HaveFirstPack = false;
   std::optional<unsigned> NumPartialExpansions;
   SourceLocation PartiallySubstitutedPackLoc;
+  typedef LocalInstantiationScope::DeclArgumentPack DeclArgumentPack;
 
   for (UnexpandedParameterPack ParmPack : Unexpanded) {
     // Compute the depth and index for this parameter pack.
     unsigned Depth = 0, Index = 0;
     IdentifierInfo *Name;
     bool IsVarDeclPack = false;
+    ResolvedUnexpandedPackExpr *ResolvedPack = nullptr;
 
     if (const TemplateTypeParmType *TTP =
             ParmPack.first.dyn_cast<const TemplateTypeParmType *>()) {
       Depth = TTP->getDepth();
       Index = TTP->getIndex();
       Name = TTP->getIdentifier();
+    } else if (auto *RP =
+                   ParmPack.first.dyn_cast<ResolvedUnexpandedPackExpr *>()) {
+      ResolvedPack = RP;
     } else {
       NamedDecl *ND = cast<NamedDecl *>(ParmPack.first);
       if (isa<VarDecl>(ND))
         IsVarDeclPack = true;
-      else
+      else if (isa<BindingDecl>(ND)) {
+        // Find the instantiated BindingDecl and check it for a resolved pack.
+        llvm::PointerUnion<Decl *, DeclArgumentPack *> *Instantiation =
+            CurrentInstantiationScope->findInstantiationOf(ND);
+        Decl *B = cast<Decl *>(*Instantiation);
+        Expr *BindingExpr = cast<BindingDecl>(B)->getBinding();
+        ResolvedPack = cast_if_present<ResolvedUnexpandedPackExpr>(BindingExpr);
+        if (!ResolvedPack) {
+          ShouldExpand = false;
+          continue;
+        }
+      } else
         std::tie(Depth, Index) = getDepthAndIndex(ND);
 
       Name = ND->getIdentifier();
@@ -783,8 +817,6 @@ bool Sema::CheckParameterPacksForExpansion(
     unsigned NewPackSize, PendingPackExpansionSize = 0;
     if (IsVarDeclPack) {
       // Figure out whether we're instantiating to an argument pack or not.
-      typedef LocalInstantiationScope::DeclArgumentPack DeclArgumentPack;
-
       llvm::PointerUnion<Decl *, DeclArgumentPack *> *Instantiation =
           CurrentInstantiationScope->findInstantiationOf(
               cast<NamedDecl *>(ParmPack.first));
@@ -797,6 +829,8 @@ bool Sema::CheckParameterPacksForExpansion(
         ShouldExpand = false;
         continue;
       }
+    } else if (ResolvedPack) {
+      NewPackSize = ResolvedPack->getNumExprs();
     } else {
       // If we don't have a template argument at this depth/index, then we
       // cannot expand the pack expansion. Make a note of this, but we still
@@ -833,7 +867,7 @@ bool Sema::CheckParameterPacksForExpansion(
     //   Template argument deduction can extend the sequence of template
     //   arguments corresponding to a template parameter pack, even when the
     //   sequence contains explicitly specified template arguments.
-    if (!IsVarDeclPack && CurrentInstantiationScope) {
+    if (!IsVarDeclPack && !ResolvedPack && CurrentInstantiationScope) {
       if (NamedDecl *PartialPack =
               CurrentInstantiationScope->getPartiallySubstitutedPack()) {
         unsigned PartialDepth, PartialIndex;
@@ -939,6 +973,12 @@ std::optional<unsigned> Sema::getNumArgumentsInExpansionFromUnexpanded(
             Unexpanded[I].first.dyn_cast<const TemplateTypeParmType *>()) {
       Depth = TTP->getDepth();
       Index = TTP->getIndex();
+    } else if (auto *PE = Unexpanded[I]
+                              .first.dyn_cast<ResolvedUnexpandedPackExpr *>()) {
+      unsigned Size = PE->getNumExprs();
+      assert((!Result || *Result == Size) && "inconsistent pack sizes");
+      Result = Size;
+      continue;
     } else {
       NamedDecl *ND = cast<NamedDecl *>(Unexpanded[I].first);
       if (isa<VarDecl>(ND)) {
@@ -1167,8 +1207,12 @@ ExprResult Sema::ActOnSizeofParameterPackExpr(Scope *S,
 
   MarkAnyDeclReferenced(OpLoc, ParameterPack, true);
 
+  std::optional<unsigned> Length;
+  if (auto *RP = ResolvedUnexpandedPackExpr::getFromDecl(ParameterPack))
+    Length = RP->getNumExprs();
+
   return SizeOfPackExpr::Create(Context, OpLoc, ParameterPack, NameLoc,
-                                RParenLoc);
+                                RParenLoc, Length);
 }
 
 static bool isParameterPack(Expr *PackExpression) {
diff --git a/clang/lib/Sema/SemaType.cpp b/clang/lib/Sema/SemaType.cpp
index 2ccf5a8e1d6f3..6f30a36621601 100644
--- a/clang/lib/Sema/SemaType.cpp
+++ b/clang/lib/Sema/SemaType.cpp
@@ -1826,7 +1826,8 @@ QualType Sema::BuildPointerType(QualType T,
   if (checkQualifiedFunction(*this, T, Loc, QFK_Pointer))
     return QualType();
 
-  assert(!T->isObjCObjectType() && "Should build ObjCObjectPointerType");
+  if (T->isObjCObjectType())
+    return Context.getObjCObjectPointerType(T);
 
   // In ARC, it is forbidden to build pointers to unqualified pointers.
   if (getLangOpts().ObjCAutoRefCount)
@@ -5182,6 +5183,11 @@ static TypeSourceInfo *GetFullTypeForDeclarator(TypeProcessingState &state,
               if (ParamTy.hasQualifiers())
                 S.Diag(DeclType.Loc, diag::err_void_param_qualified);
 
+              for (const auto *A : Param->attrs()) {
+                S.Diag(A->getLoc(), diag::warn_attribute_on_void_param)
+                    << A << A->getRange();
+              }
+
               // Reject, but continue to parse 'float(this void)' as
               // 'float(void)'.
               if (Param->isExplicitObjectParameter()) {
@@ -8306,7 +8312,8 @@ static bool isPermittedNeonBaseType(QualType &Ty, VectorKind VecKind, Sema &S) {
          BTy->getKind() == BuiltinType::ULongLong ||
          BTy->getKind() == BuiltinType::Float ||
          BTy->getKind() == BuiltinType::Half ||
-         BTy->getKind() == BuiltinType::BFloat16;
+         BTy->getKind() == BuiltinType::BFloat16 ||
+         BTy->getKind() == BuiltinType::MFloat8;
 }
 
 static bool verifyValidIntegerConstantExpr(Sema &S, const ParsedAttr &Attr,
@@ -8490,7 +8497,8 @@ static void HandleRISCVRVVVectorBitsTypeAttr(QualType &CurType,
     return;
   }
 
-  auto VScale = S.Context.getTargetInfo().getVScaleRange(S.getLangOpts());
+  auto VScale =
+      S.Context.getTargetInfo().getVScaleRange(S.getLangOpts(), false);
   if (!VScale || !VScale->first || VScale->first != VScale->second) {
     S.Diag(Attr.getLoc(), diag::err_attribute_riscv_rvv_bits_unsupported)
         << Attr;
@@ -9396,7 +9404,7 @@ bool Sema::RequireCompleteTypeImpl(SourceLocation Loc, QualType T,
         runWithSufficientStackSpace(Loc, [&] {
           Diagnosed = InstantiateClassTemplateSpecialization(
               Loc, ClassTemplateSpec, TSK_ImplicitInstantiation,
-              /*Complain=*/Diagnoser);
+              /*Complain=*/Diagnoser, ClassTemplateSpec->hasStrictPackMatch());
         });
         Instantiated = true;
       }
@@ -9806,8 +9814,7 @@ QualType Sema::BuiltinAddPointer(QualType BaseType, SourceLocation Loc) {
 }
 
 QualType Sema::BuiltinRemovePointer(QualType BaseType, SourceLocation Loc) {
-  // We don't want block pointers or ObjectiveC's id type.
-  if (!BaseType->isAnyPointerType() || BaseType->isObjCIdType())
+  if (!BaseType->isAnyPointerType())
     return BaseType;
 
   return BaseType->getPointeeType();
diff --git a/clang/lib/Sema/TreeTransform.h b/clang/lib/Sema/TreeTransform.h
index 12680843a434a..813b172c4d89e 100644
--- a/clang/lib/Sema/TreeTransform.h
+++ b/clang/lib/Sema/TreeTransform.h
@@ -3680,6 +3680,13 @@ class TreeTransform {
                                            FullySubstituted);
   }
 
+  ExprResult RebuildResolvedUnexpandedPackExpr(SourceLocation BeginLoc,
+                                               QualType T,
+                                               ArrayRef<Expr *> Exprs) {
+    return ResolvedUnexpandedPackExpr::Create(SemaRef.Context, BeginLoc, T,
+                                              Exprs);
+  }
+
   /// Build a new expression representing a call to a source location
   ///  builtin.
   ///
@@ -4204,6 +4211,17 @@ class TreeTransform {
         Exprs, RParenLoc, EndLoc, Clauses, {});
   }
 
+  StmtResult RebuildOpenACCAtomicConstruct(SourceLocation BeginLoc,
+                                           SourceLocation DirLoc,
+                                           OpenACCAtomicKind AtKind,
+                                           SourceLocation EndLoc,
+                                           StmtResult AssociatedStmt) {
+    return getSema().OpenACC().ActOnEndStmtDirective(
+        OpenACCDirectiveKind::Atomic, BeginLoc, DirLoc, SourceLocation{},
+        SourceLocation{}, {}, AtKind, SourceLocation{}, EndLoc, {},
+        AssociatedStmt);
+  }
+
   ExprResult RebuildOpenACCAsteriskSizeExpr(SourceLocation AsteriskLoc) {
     return getSema().OpenACC().ActOnOpenACCAsteriskSizeExpr(AsteriskLoc);
   }
@@ -7335,8 +7353,10 @@ QualType TreeTransform<Derived>::TransformTemplateSpecializationType(
                                               NewTemplateArgs))
     return QualType();
 
-  // FIXME: maybe don't rebuild if all the template arguments are the same.
-
+  // This needs to be rebuilt if either the arguments changed, or if the
+  // original template changed. If the template changed, and even if the
+  // arguments didn't change, these arguments might not correspond to their
+  // respective parameters, therefore needing conversions.
   QualType Result =
     getDerived().RebuildTemplateSpecializationType(Template,
                                                    TL.getTemplateNameLoc(),
@@ -10607,6 +10627,11 @@ OMPClause *TreeTransform<Derived>::TransformOMPNoOpenMPRoutinesClause(
   return C;
 }
 template <typename Derived>
+OMPClause *TreeTransform<Derived>::TransformOMPNoOpenMPConstructsClause(
+    OMPNoOpenMPConstructsClause *C) {
+  return C;
+}
+template <typename Derived>
 OMPClause *TreeTransform<Derived>::TransformOMPNoParallelismClause(
     OMPNoParallelismClause *C) {
   return C;
@@ -12604,6 +12629,29 @@ TreeTransform<Derived>::TransformOpenACCWaitConstruct(OpenACCWaitConstruct *C) {
       QueueIdExprs, C->getRParenLoc(), C->getEndLoc(), TransformedClauses);
 }
 
+template <typename Derived>
+StmtResult TreeTransform<Derived>::TransformOpenACCAtomicConstruct(
+    OpenACCAtomicConstruct *C) {
+  getSema().OpenACC().ActOnConstruct(C->getDirectiveKind(), C->getBeginLoc());
+
+  if (getSema().OpenACC().ActOnStartStmtDirective(C->getDirectiveKind(),
+                                                  C->getBeginLoc(), {}))
+    return StmtError();
+
+  // Transform Associated Stmt.
+  SemaOpenACC::AssociatedStmtRAII AssocStmtRAII(
+      getSema().OpenACC(), C->getDirectiveKind(), C->getDirectiveLoc(), {}, {});
+
+  StmtResult AssocStmt = getDerived().TransformStmt(C->getAssociatedStmt());
+  AssocStmt = getSema().OpenACC().ActOnAssociatedStmt(
+      C->getBeginLoc(), C->getDirectiveKind(), C->getAtomicKind(), {},
+      AssocStmt);
+
+  return getDerived().RebuildOpenACCAtomicConstruct(
+      C->getBeginLoc(), C->getDirectiveLoc(), C->getAtomicKind(),
+      C->getEndLoc(), AssocStmt);
+}
+
 template <typename Derived>
 ExprResult TreeTransform<Derived>::TransformOpenACCAsteriskSizeExpr(
     OpenACCAsteriskSizeExpr *E) {
@@ -14903,7 +14951,7 @@ TreeTransform<Derived>::TransformExprRequirement(concepts::ExprRequirement *Req)
     TransRetReq.emplace(TPL);
   }
   assert(TransRetReq && "All code paths leading here must set TransRetReq");
-  if (Expr *E = TransExpr.dyn_cast<Expr *>())
+  if (Expr *E = dyn_cast<Expr *>(TransExpr))
     return getDerived().RebuildExprRequirement(E, Req->isSimple(),
                                                Req->getNoexceptLoc(),
                                                std::move(*TransRetReq));
@@ -14947,9 +14995,6 @@ TreeTransform<Derived>::TransformArrayTypeTraitExpr(ArrayTypeTraitExpr *E) {
     SubExpr = getDerived().TransformExpr(E->getDimensionExpression());
     if (SubExpr.isInvalid())
       return ExprError();
-
-    if (!getDerived().AlwaysRebuild() && SubExpr.get() == E->getDimensionExpression())
-      return E;
   }
 
   return getDerived().RebuildArrayTypeTrait(E->getTrait(), E->getBeginLoc(), T,
@@ -15428,12 +15473,11 @@ TreeTransform<Derived>::TransformLambdaExpr(LambdaExpr *E) {
         // The transform has determined that we should perform an expansion;
         // transform and capture each of the arguments.
         // expansion of the pattern. Do so.
-        auto *Pack = cast<VarDecl>(C->getCapturedVar());
+        auto *Pack = cast<ValueDecl>(C->getCapturedVar());
         for (unsigned I = 0; I != *NumExpansions; ++I) {
           Sema::ArgumentPackSubstitutionIndexRAII SubstIndex(getSema(), I);
-          VarDecl *CapturedVar
-            = cast_or_null<VarDecl>(getDerived().TransformDecl(C->getLocation(),
-                                                               Pack));
+          ValueDecl *CapturedVar = cast_if_present<ValueDecl>(
+              getDerived().TransformDecl(C->getLocation(), Pack));
           if (!CapturedVar) {
             Invalid = true;
             continue;
@@ -16128,6 +16172,24 @@ TreeTransform<Derived>::TransformFunctionParmPackExpr(FunctionParmPackExpr *E) {
   return E;
 }
 
+template <typename Derived>
+ExprResult TreeTransform<Derived>::TransformResolvedUnexpandedPackExpr(
+    ResolvedUnexpandedPackExpr *E) {
+  bool ArgumentChanged = false;
+  SmallVector<Expr *, 12> NewExprs;
+  if (TransformExprs(E->getExprs().begin(), E->getNumExprs(),
+                     /*IsCall=*/false, NewExprs, &ArgumentChanged))
+    return ExprError();
+
+  if (!AlwaysRebuild() && !ArgumentChanged)
+    return E;
+
+  // NOTE: The type is just a superficial PackExpansionType
+  //       that needs no substitution.
+  return RebuildResolvedUnexpandedPackExpr(E->getBeginLoc(), E->getType(),
+                                           NewExprs);
+}
+
 template<typename Derived>
 ExprResult
 TreeTransform<Derived>::TransformMaterializeTemporaryExpr(
diff --git a/clang/lib/Serialization/ASTReader.cpp b/clang/lib/Serialization/ASTReader.cpp
index 08801d22fdca8..3c64b67503195 100644
--- a/clang/lib/Serialization/ASTReader.cpp
+++ b/clang/lib/Serialization/ASTReader.cpp
@@ -645,10 +645,11 @@ collectMacroDefinitions(const PreprocessorOptions &PPOpts,
 
     // For an #undef'd macro, we only care about the name.
     if (IsUndef) {
-      if (MacroNames && !Macros.count(MacroName))
+      auto [It, Inserted] = Macros.try_emplace(MacroName);
+      if (MacroNames && Inserted)
         MacroNames->push_back(MacroName);
 
-      Macros[MacroName] = std::make_pair("", true);
+      It->second = std::make_pair("", true);
       continue;
     }
 
@@ -661,9 +662,10 @@ collectMacroDefinitions(const PreprocessorOptions &PPOpts,
       MacroBody = MacroBody.substr(0, End);
     }
 
-    if (MacroNames && !Macros.count(MacroName))
+    auto [It, Inserted] = Macros.try_emplace(MacroName);
+    if (MacroNames && Inserted)
       MacroNames->push_back(MacroName);
-    Macros[MacroName] = std::make_pair(MacroBody, false);
+    It->second = std::make_pair(MacroBody, false);
   }
 }
 
@@ -6864,7 +6866,7 @@ void ASTReader::ReadPragmaDiagnosticMappings(DiagnosticsEngine &Diag) {
       // command line (-w, -Weverything, -Werror, ...) along with any explicit
       // -Wblah flags.
       unsigned Flags = Record[Idx++];
-      DiagState Initial;
+      DiagState Initial(*Diag.getDiagnosticIDs());
       Initial.SuppressSystemWarnings = Flags & 1; Flags >>= 1;
       Initial.ErrorsAsFatal = Flags & 1; Flags >>= 1;
       Initial.WarningsAsErrors = Flags & 1; Flags >>= 1;
@@ -10186,12 +10188,12 @@ void ASTReader::visitTopLevelModuleMaps(
 }
 
 void ASTReader::finishPendingActions() {
-  while (
-      !PendingIdentifierInfos.empty() || !PendingDeducedFunctionTypes.empty() ||
-      !PendingDeducedVarTypes.empty() || !PendingIncompleteDeclChains.empty() ||
-      !PendingDeclChains.empty() || !PendingMacroIDs.empty() ||
-      !PendingDeclContextInfos.empty() || !PendingUpdateRecords.empty() ||
-      !PendingObjCExtensionIvarRedeclarations.empty()) {
+  while (!PendingIdentifierInfos.empty() ||
+         !PendingDeducedFunctionTypes.empty() ||
+         !PendingDeducedVarTypes.empty() || !PendingDeclChains.empty() ||
+         !PendingMacroIDs.empty() || !PendingDeclContextInfos.empty() ||
+         !PendingUpdateRecords.empty() ||
+         !PendingObjCExtensionIvarRedeclarations.empty()) {
     // If any identifiers with corresponding top-level declarations have
     // been loaded, load those declarations now.
     using TopLevelDeclsMap =
@@ -10239,13 +10241,6 @@ void ASTReader::finishPendingActions() {
     }
     PendingDeducedVarTypes.clear();
 
-    // For each decl chain that we wanted to complete while deserializing, mark
-    // it as "still needs to be completed".
-    for (unsigned I = 0; I != PendingIncompleteDeclChains.size(); ++I) {
-      markIncompleteDeclChain(PendingIncompleteDeclChains[I]);
-    }
-    PendingIncompleteDeclChains.clear();
-
     // Load pending declaration chains.
     for (unsigned I = 0; I != PendingDeclChains.size(); ++I)
       loadPendingDeclChain(PendingDeclChains[I].first,
@@ -10483,6 +10478,12 @@ void ASTReader::finishPendingActions() {
   for (auto *ND : PendingMergedDefinitionsToDeduplicate)
     getContext().deduplicateMergedDefinitonsFor(ND);
   PendingMergedDefinitionsToDeduplicate.clear();
+
+  // For each decl chain that we wanted to complete while deserializing, mark
+  // it as "still needs to be completed".
+  for (Decl *D : PendingIncompleteDeclChains)
+    markIncompleteDeclChain(D);
+  PendingIncompleteDeclChains.clear();
 }
 
 void ASTReader::diagnoseOdrViolations() {
@@ -11060,6 +11061,9 @@ OMPClause *OMPClauseReader::readClause() {
   case llvm::omp::OMPC_no_openmp_routines:
     C = new (Context) OMPNoOpenMPRoutinesClause();
     break;
+  case llvm::omp::OMPC_no_openmp_constructs:
+    C = new (Context) OMPNoOpenMPConstructsClause();
+    break;
   case llvm::omp::OMPC_no_parallelism:
     C = new (Context) OMPNoParallelismClause();
     break;
@@ -11501,6 +11505,9 @@ void OMPClauseReader::VisitOMPNoOpenMPClause(OMPNoOpenMPClause *) {}
 void OMPClauseReader::VisitOMPNoOpenMPRoutinesClause(
     OMPNoOpenMPRoutinesClause *) {}
 
+void OMPClauseReader::VisitOMPNoOpenMPConstructsClause(
+    OMPNoOpenMPConstructsClause *) {}
+
 void OMPClauseReader::VisitOMPNoParallelismClause(OMPNoParallelismClause *) {}
 
 void OMPClauseReader::VisitOMPSeqCstClause(OMPSeqCstClause *) {}
diff --git a/clang/lib/Serialization/ASTReaderDecl.cpp b/clang/lib/Serialization/ASTReaderDecl.cpp
index 0b75468a94103..17a41fff2267c 100644
--- a/clang/lib/Serialization/ASTReaderDecl.cpp
+++ b/clang/lib/Serialization/ASTReaderDecl.cpp
@@ -1064,6 +1064,7 @@ void ASTDeclReader::VisitFunctionDecl(FunctionDecl *FD) {
   FD->setHasImplicitReturnZero(FunctionDeclBits.getNextBit());
   FD->setIsMultiVersion(FunctionDeclBits.getNextBit());
   FD->setLateTemplateParsed(FunctionDeclBits.getNextBit());
+  FD->setInstantiatedFromMemberTemplate(FunctionDeclBits.getNextBit());
   FD->setFriendConstraintRefersToEnclosingTemplate(
       FunctionDeclBits.getNextBit());
   FD->setUsesSEHTry(FunctionDeclBits.getNextBit());
@@ -2295,6 +2296,10 @@ void ASTDeclReader::VisitCXXDeductionGuideDecl(CXXDeductionGuideDecl *D) {
   VisitFunctionDecl(D);
   D->setDeductionCandidateKind(
       static_cast<DeductionCandidate>(Record.readInt()));
+  D->setSourceDeductionGuide(readDeclAs<CXXDeductionGuideDecl>());
+  D->setSourceDeductionGuideKind(
+      static_cast<CXXDeductionGuideDecl::SourceDeductionGuideKind>(
+          Record.readInt()));
 }
 
 void ASTDeclReader::VisitCXXMethodDecl(CXXMethodDecl *D) {
@@ -2528,6 +2533,7 @@ RedeclarableResult ASTDeclReader::VisitClassTemplateSpecializationDeclImpl(
   D->TemplateArgs = TemplateArgumentList::CreateCopy(C, TemplArgs);
   D->PointOfInstantiation = readSourceLocation();
   D->SpecializationKind = (TemplateSpecializationKind)Record.readInt();
+  D->StrictPackMatch = Record.readBool();
 
   bool writtenAsCanonicalDecl = Record.readInt();
   if (writtenAsCanonicalDecl) {
@@ -3742,6 +3748,18 @@ void ASTDeclReader::checkMultipleDefinitionInNamedModules(ASTReader &Reader,
       Func && Func->getTemplateSpecializationInfo())
     return;
 
+  // The module ownership of in-class friend declaration is not straightforward.
+  // Avoid diagnosing such cases.
+  if (D->getFriendObjectKind() || Previous->getFriendObjectKind())
+    return;
+
+  // Skip diagnosing in-class declarations.
+  if (!Previous->getLexicalDeclContext()
+           ->getNonTransparentContext()
+           ->isFileContext() ||
+      !D->getLexicalDeclContext()->getNonTransparentContext()->isFileContext())
+    return;
+
   Module *M = Previous->getOwningModule();
   if (!M)
     return;
@@ -4678,8 +4696,8 @@ void ASTDeclReader::UpdateDecl(Decl *D) {
         MSInfo->setPointOfInstantiation(POI);
       } else {
         auto *FD = cast<FunctionDecl>(D);
-        if (auto *FTSInfo = FD->TemplateOrSpecialization
-                    .dyn_cast<FunctionTemplateSpecializationInfo *>())
+        if (auto *FTSInfo = dyn_cast<FunctionTemplateSpecializationInfo *>(
+                FD->TemplateOrSpecialization))
           FTSInfo->setPointOfInstantiation(POI);
         else
           cast<MemberSpecializationInfo *>(FD->TemplateOrSpecialization)
diff --git a/clang/lib/Serialization/ASTReaderStmt.cpp b/clang/lib/Serialization/ASTReaderStmt.cpp
index 990235a310d90..dc953ddeee85c 100644
--- a/clang/lib/Serialization/ASTReaderStmt.cpp
+++ b/clang/lib/Serialization/ASTReaderStmt.cpp
@@ -2208,6 +2208,16 @@ void ASTStmtReader::VisitPackIndexingExpr(PackIndexingExpr *E) {
     Exprs[I] = Record.readExpr();
 }
 
+void ASTStmtReader::VisitResolvedUnexpandedPackExpr(
+    ResolvedUnexpandedPackExpr *E) {
+  VisitExpr(E);
+  E->NumExprs = Record.readInt();
+  E->BeginLoc = readSourceLocation();
+  auto **Exprs = E->getTrailingObjects<Expr *>();
+  for (unsigned I = 0; I < E->NumExprs; ++I)
+    Exprs[I] = Record.readExpr();
+}
+
 void ASTStmtReader::VisitSubstNonTypeTemplateParmExpr(
                                               SubstNonTypeTemplateParmExpr *E) {
   VisitExpr(E);
@@ -2913,6 +2923,15 @@ void ASTStmtReader::VisitOpenACCWaitConstruct(OpenACCWaitConstruct *S) {
   }
 }
 
+void ASTStmtReader::VisitOpenACCAtomicConstruct(OpenACCAtomicConstruct *S) {
+  VisitStmt(S);
+  S->Kind = Record.readEnum<OpenACCDirectiveKind>();
+  S->Range = Record.readSourceRange();
+  S->DirectiveLoc = Record.readSourceLocation();
+  S->AtomicKind = Record.readEnum<OpenACCAtomicKind>();
+  S->setAssociatedStmt(Record.readSubStmt());
+}
+
 //===----------------------------------------------------------------------===//
 // HLSL Constructs/Directives.
 //===----------------------------------------------------------------------===//
@@ -4291,6 +4310,12 @@ Stmt *ASTReader::ReadStmtFromStream(ModuleFile &F) {
           /*TransformedExprs=*/Record[ASTStmtReader::NumExprFields]);
       break;
 
+    case EXPR_RESOLVED_UNEXPANDED_PACK:
+      S = ResolvedUnexpandedPackExpr::CreateDeserialized(
+          Context,
+          /*NumExprs=*/Record[ASTStmtReader::NumExprFields]);
+      break;
+
     case EXPR_SUBST_NON_TYPE_TEMPLATE_PARM:
       S = new (Context) SubstNonTypeTemplateParmExpr(Empty);
       break;
@@ -4438,6 +4463,10 @@ Stmt *ASTReader::ReadStmtFromStream(ModuleFile &F) {
       S = OpenACCUpdateConstruct::CreateEmpty(Context, NumClauses);
       break;
     }
+    case STMT_OPENACC_ATOMIC_CONSTRUCT: {
+      S = OpenACCAtomicConstruct::CreateEmpty(Context);
+      break;
+    }
     case EXPR_REQUIRES: {
       unsigned numLocalParameters = Record[ASTStmtReader::NumExprFields];
       unsigned numRequirement = Record[ASTStmtReader::NumExprFields + 1];
diff --git a/clang/lib/Serialization/ASTWriter.cpp b/clang/lib/Serialization/ASTWriter.cpp
index a580f375aee35..903a165ee75c6 100644
--- a/clang/lib/Serialization/ASTWriter.cpp
+++ b/clang/lib/Serialization/ASTWriter.cpp
@@ -874,6 +874,7 @@ static void AddStmtsExprs(llvm::BitstreamWriter &Stream,
   RECORD(EXPR_PACK_EXPANSION);
   RECORD(EXPR_SIZEOF_PACK);
   RECORD(EXPR_PACK_INDEXING);
+  RECORD(EXPR_RESOLVED_UNEXPANDED_PACK);
   RECORD(EXPR_SUBST_NON_TYPE_TEMPLATE_PARM);
   RECORD(EXPR_SUBST_NON_TYPE_TEMPLATE_PARM_PACK);
   RECORD(EXPR_FUNCTION_PARM_PACK);
@@ -3264,7 +3265,7 @@ void ASTWriter::WritePragmaDiagnosticMappings(const DiagnosticsEngine &Diag,
         // Skip default mappings. We have a mapping for every diagnostic ever
         // emitted, regardless of whether it was customized.
         if (!I.second.isPragma() &&
-            I.second == DiagnosticIDs::getDefaultMapping(I.first))
+            I.second == Diag.getDiagnosticIDs()->getDefaultMapping(I.first))
           continue;
         Mappings.push_back(I);
       }
@@ -5358,7 +5359,7 @@ ASTWriter::WriteAST(llvm::PointerUnion<Sema *, Preprocessor *> Subject,
   llvm::TimeTraceScope scope("WriteAST", OutputFile);
   WritingAST = true;
 
-  Sema *SemaPtr = Subject.dyn_cast<Sema *>();
+  Sema *SemaPtr = dyn_cast<Sema *>(Subject);
   Preprocessor &PPRef =
       SemaPtr ? SemaPtr->getPreprocessor() : *cast<Preprocessor *>(Subject);
 
@@ -7885,6 +7886,9 @@ void OMPClauseWriter::VisitOMPNoOpenMPClause(OMPNoOpenMPClause *) {}
 void OMPClauseWriter::VisitOMPNoOpenMPRoutinesClause(
     OMPNoOpenMPRoutinesClause *) {}
 
+void OMPClauseWriter::VisitOMPNoOpenMPConstructsClause(
+    OMPNoOpenMPConstructsClause *) {}
+
 void OMPClauseWriter::VisitOMPNoParallelismClause(OMPNoParallelismClause *) {}
 
 void OMPClauseWriter::VisitOMPAcquireClause(OMPAcquireClause *) {}
diff --git a/clang/lib/Serialization/ASTWriterDecl.cpp b/clang/lib/Serialization/ASTWriterDecl.cpp
index 8b9ba04dce91c..b25dadab656b0 100644
--- a/clang/lib/Serialization/ASTWriterDecl.cpp
+++ b/clang/lib/Serialization/ASTWriterDecl.cpp
@@ -679,7 +679,7 @@ void ASTDeclWriter::VisitDeclaratorDecl(DeclaratorDecl *D) {
 }
 
 void ASTDeclWriter::VisitFunctionDecl(FunctionDecl *D) {
-  static_assert(DeclContext::NumFunctionDeclBits == 44,
+  static_assert(DeclContext::NumFunctionDeclBits == 45,
                 "You need to update the serializer after you change the "
                 "FunctionDeclBits");
 
@@ -785,6 +785,7 @@ void ASTDeclWriter::VisitFunctionDecl(FunctionDecl *D) {
   FunctionDeclBits.addBit(D->hasImplicitReturnZero());
   FunctionDeclBits.addBit(D->isMultiVersion());
   FunctionDeclBits.addBit(D->isLateTemplateParsed());
+  FunctionDeclBits.addBit(D->isInstantiatedFromMemberTemplate());
   FunctionDeclBits.addBit(D->FriendConstraintRefersToEnclosingTemplate());
   FunctionDeclBits.addBit(D->usesSEHTry());
   Record.push_back(FunctionDeclBits);
@@ -846,6 +847,9 @@ void ASTDeclWriter::VisitCXXDeductionGuideDecl(CXXDeductionGuideDecl *D) {
   Record.AddDeclRef(D->Ctor);
   VisitFunctionDecl(D);
   Record.push_back(static_cast<unsigned char>(D->getDeductionCandidateKind()));
+  Record.AddDeclRef(D->getSourceDeductionGuide());
+  Record.push_back(
+      static_cast<unsigned char>(D->getSourceDeductionGuideKind()));
   Code = serialization::DECL_CXX_DEDUCTION_GUIDE;
 }
 
@@ -1840,6 +1844,7 @@ void ASTDeclWriter::VisitClassTemplateSpecializationDecl(
   Record.AddTemplateArgumentList(&D->getTemplateArgs());
   Record.AddSourceLocation(D->getPointOfInstantiation());
   Record.push_back(D->getSpecializationKind());
+  Record.push_back(D->hasStrictPackMatch());
   Record.push_back(D->isCanonicalDecl());
 
   if (D->isCanonicalDecl()) {
diff --git a/clang/lib/Serialization/ASTWriterStmt.cpp b/clang/lib/Serialization/ASTWriterStmt.cpp
index 651553244812f..e5caf3debc023 100644
--- a/clang/lib/Serialization/ASTWriterStmt.cpp
+++ b/clang/lib/Serialization/ASTWriterStmt.cpp
@@ -475,7 +475,7 @@ addConstraintSatisfaction(ASTRecordWriter &Record,
   if (!Satisfaction.IsSatisfied) {
     Record.push_back(Satisfaction.NumRecords);
     for (const auto &DetailRecord : Satisfaction) {
-      auto *E = DetailRecord.dyn_cast<Expr *>();
+      auto *E = dyn_cast<Expr *>(DetailRecord);
       Record.push_back(/* IsDiagnostic */ E == nullptr);
       if (E)
         Record.AddStmt(E);
@@ -2210,6 +2210,16 @@ void ASTStmtWriter::VisitPackIndexingExpr(PackIndexingExpr *E) {
   Code = serialization::EXPR_PACK_INDEXING;
 }
 
+void ASTStmtWriter::VisitResolvedUnexpandedPackExpr(
+    ResolvedUnexpandedPackExpr *E) {
+  VisitExpr(E);
+  Record.push_back(E->getNumExprs());
+  Record.AddSourceLocation(E->getBeginLoc());
+  for (Expr *Sub : E->getExprs())
+    Record.AddStmt(Sub);
+  Code = serialization::EXPR_RESOLVED_UNEXPANDED_PACK;
+}
+
 void ASTStmtWriter::VisitSubstNonTypeTemplateParmExpr(
                                               SubstNonTypeTemplateParmExpr *E) {
   VisitExpr(E);
@@ -2997,6 +3007,17 @@ void ASTStmtWriter::VisitOpenACCWaitConstruct(OpenACCWaitConstruct *S) {
   Code = serialization::STMT_OPENACC_WAIT_CONSTRUCT;
 }
 
+void ASTStmtWriter::VisitOpenACCAtomicConstruct(OpenACCAtomicConstruct *S) {
+  VisitStmt(S);
+  Record.writeEnum(S->Kind);
+  Record.AddSourceRange(S->Range);
+  Record.AddSourceLocation(S->DirectiveLoc);
+  Record.writeEnum(S->getAtomicKind());
+  Record.AddStmt(S->getAssociatedStmt());
+
+  Code = serialization::STMT_OPENACC_ATOMIC_CONSTRUCT;
+}
+
 //===----------------------------------------------------------------------===//
 // HLSL Constructs/Directives.
 //===----------------------------------------------------------------------===//
diff --git a/clang/lib/StaticAnalyzer/Checkers/ArrayBoundChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/ArrayBoundChecker.cpp
index c990ad138f890..109faacf1726a 100644
--- a/clang/lib/StaticAnalyzer/Checkers/ArrayBoundChecker.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/ArrayBoundChecker.cpp
@@ -1,4 +1,4 @@
-//== ArrayBoundChecker.cpp ------------------------------*- C++ -*--==//
+//== ArrayBoundChecker.cpp -------------------------------------------------==//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
@@ -6,81 +6,767 @@
 //
 //===----------------------------------------------------------------------===//
 //
-// This file defines ArrayBoundChecker, which is a path-sensitive check
-// which looks for an out-of-bound array element access.
+// This file defines security.ArrayBound, which is a path-sensitive checker
+// that looks for out of bounds access of memory regions.
 //
 //===----------------------------------------------------------------------===//
 
+#include "clang/AST/CharUnits.h"
+#include "clang/AST/ParentMapContext.h"
 #include "clang/StaticAnalyzer/Checkers/BuiltinCheckerRegistration.h"
+#include "clang/StaticAnalyzer/Checkers/Taint.h"
 #include "clang/StaticAnalyzer/Core/BugReporter/BugType.h"
 #include "clang/StaticAnalyzer/Core/Checker.h"
 #include "clang/StaticAnalyzer/Core/CheckerManager.h"
+#include "clang/StaticAnalyzer/Core/PathSensitive/APSIntType.h"
 #include "clang/StaticAnalyzer/Core/PathSensitive/CheckerContext.h"
 #include "clang/StaticAnalyzer/Core/PathSensitive/DynamicExtent.h"
 #include "clang/StaticAnalyzer/Core/PathSensitive/ExprEngine.h"
+#include "llvm/ADT/APSInt.h"
+#include "llvm/ADT/SmallString.h"
+#include "llvm/Support/FormatVariadic.h"
+#include "llvm/Support/raw_ostream.h"
+#include <optional>
 
 using namespace clang;
 using namespace ento;
+using namespace taint;
+using llvm::formatv;
 
 namespace {
-class ArrayBoundChecker :
-    public Checker<check::Location> {
-  const BugType BT{this, "Out-of-bound array access"};
+/// If `E` is a "clean" array subscript expression, return the type of the
+/// accessed element. If the base of the subscript expression is modified by
+/// pointer arithmetic (and not the beginning of a "full" memory region), this
+/// always returns nullopt because that's the right (or the least bad) thing to
+/// do for the diagnostic output that's relying on this.
+static std::optional<QualType> determineElementType(const Expr *E,
+                                                    const CheckerContext &C) {
+  const auto *ASE = dyn_cast<ArraySubscriptExpr>(E);
+  if (!ASE)
+    return std::nullopt;
+
+  const MemRegion *SubscriptBaseReg = C.getSVal(ASE->getBase()).getAsRegion();
+  if (!SubscriptBaseReg)
+    return std::nullopt;
+
+  // The base of the subscript expression is affected by pointer arithmetics,
+  // so we want to report byte offsets instead of indices.
+  if (isa<ElementRegion>(SubscriptBaseReg->StripCasts()))
+    return std::nullopt;
+
+  return ASE->getType();
+}
+
+static std::optional<int64_t>
+determineElementSize(const std::optional<QualType> T, const CheckerContext &C) {
+  if (!T)
+    return std::nullopt;
+  return C.getASTContext().getTypeSizeInChars(*T).getQuantity();
+}
+
+class StateUpdateReporter {
+  const SubRegion *Reg;
+  const NonLoc ByteOffsetVal;
+  const std::optional<QualType> ElementType;
+  const std::optional<int64_t> ElementSize;
+  bool AssumedNonNegative = false;
+  std::optional<NonLoc> AssumedUpperBound = std::nullopt;
 
 public:
-  void checkLocation(SVal l, bool isLoad, const Stmt* S,
-                     CheckerContext &C) const;
+  StateUpdateReporter(const SubRegion *R, NonLoc ByteOffsVal, const Expr *E,
+                      CheckerContext &C)
+      : Reg(R), ByteOffsetVal(ByteOffsVal),
+        ElementType(determineElementType(E, C)),
+        ElementSize(determineElementSize(ElementType, C)) {}
+
+  void recordNonNegativeAssumption() { AssumedNonNegative = true; }
+  void recordUpperBoundAssumption(NonLoc UpperBoundVal) {
+    AssumedUpperBound = UpperBoundVal;
+  }
+
+  bool assumedNonNegative() { return AssumedNonNegative; }
+
+  const NoteTag *createNoteTag(CheckerContext &C) const;
+
+private:
+  std::string getMessage(PathSensitiveBugReport &BR) const;
+
+  /// Return true if information about the value of `Sym` can put constraints
+  /// on some symbol which is interesting within the bug report `BR`.
+  /// In particular, this returns true when `Sym` is interesting within `BR`;
+  /// but it also returns true if `Sym` is an expression that contains integer
+  /// constants and a single symbolic operand which is interesting (in `BR`).
+  /// We need to use this instead of plain `BR.isInteresting()` because if we
+  /// are analyzing code like
+  ///   int array[10];
+  ///   int f(int arg) {
+  ///     return array[arg] && array[arg + 10];
+  ///   }
+  /// then the byte offsets are `arg * 4` and `(arg + 10) * 4`, which are not
+  /// sub-expressions of each other (but `getSimplifiedOffsets` is smart enough
+  /// to detect this out of bounds access).
+  static bool providesInformationAboutInteresting(SymbolRef Sym,
+                                                  PathSensitiveBugReport &BR);
+  static bool providesInformationAboutInteresting(SVal SV,
+                                                  PathSensitiveBugReport &BR) {
+    return providesInformationAboutInteresting(SV.getAsSymbol(), BR);
+  }
 };
+
+struct Messages {
+  std::string Short, Full;
+};
+
+// NOTE: The `ArraySubscriptExpr` and `UnaryOperator` callbacks are `PostStmt`
+// instead of `PreStmt` because the current implementation passes the whole
+// expression to `CheckerContext::getSVal()` which only works after the
+// symbolic evaluation of the expression. (To turn them into `PreStmt`
+// callbacks, we'd need to duplicate the logic that evaluates these
+// expressions.) The `MemberExpr` callback would work as `PreStmt` but it's
+// defined as `PostStmt` for the sake of consistency with the other callbacks.
+class ArrayBoundChecker : public Checker<check::PostStmt<ArraySubscriptExpr>,
+                                         check::PostStmt<UnaryOperator>,
+                                         check::PostStmt<MemberExpr>> {
+  BugType BT{this, "Out-of-bound access"};
+  BugType TaintBT{this, "Out-of-bound access", categories::TaintedData};
+
+  void performCheck(const Expr *E, CheckerContext &C) const;
+
+  void reportOOB(CheckerContext &C, ProgramStateRef ErrorState, Messages Msgs,
+                 NonLoc Offset, std::optional<NonLoc> Extent,
+                 bool IsTaintBug = false) const;
+
+  static void markPartsInteresting(PathSensitiveBugReport &BR,
+                                   ProgramStateRef ErrorState, NonLoc Val,
+                                   bool MarkTaint);
+
+  static bool isFromCtypeMacro(const Stmt *S, ASTContext &AC);
+
+  static bool isIdiomaticPastTheEndPtr(const Expr *E, ProgramStateRef State,
+                                       NonLoc Offset, NonLoc Limit,
+                                       CheckerContext &C);
+  static bool isInAddressOf(const Stmt *S, ASTContext &AC);
+
+public:
+  void checkPostStmt(const ArraySubscriptExpr *E, CheckerContext &C) const {
+    performCheck(E, C);
+  }
+  void checkPostStmt(const UnaryOperator *E, CheckerContext &C) const {
+    if (E->getOpcode() == UO_Deref)
+      performCheck(E, C);
+  }
+  void checkPostStmt(const MemberExpr *E, CheckerContext &C) const {
+    if (E->isArrow())
+      performCheck(E->getBase(), C);
+  }
+};
+
+} // anonymous namespace
+
+/// For a given Location that can be represented as a symbolic expression
+/// Arr[Idx] (or perhaps Arr[Idx1][Idx2] etc.), return the parent memory block
+/// Arr and the distance of Location from the beginning of Arr (expressed in a
+/// NonLoc that specifies the number of CharUnits). Returns nullopt when these
+/// cannot be determined.
+static std::optional<std::pair<const SubRegion *, NonLoc>>
+computeOffset(ProgramStateRef State, SValBuilder &SVB, SVal Location) {
+  QualType T = SVB.getArrayIndexType();
+  auto EvalBinOp = [&SVB, State, T](BinaryOperatorKind Op, NonLoc L, NonLoc R) {
+    // We will use this utility to add and multiply values.
+    return SVB.evalBinOpNN(State, Op, L, R, T).getAs<NonLoc>();
+  };
+
+  const SubRegion *OwnerRegion = nullptr;
+  std::optional<NonLoc> Offset = SVB.makeZeroArrayIndex();
+
+  const ElementRegion *CurRegion =
+      dyn_cast_or_null<ElementRegion>(Location.getAsRegion());
+
+  while (CurRegion) {
+    const auto Index = CurRegion->getIndex().getAs<NonLoc>();
+    if (!Index)
+      return std::nullopt;
+
+    QualType ElemType = CurRegion->getElementType();
+
+    // FIXME: The following early return was presumably added to safeguard the
+    // getTypeSizeInChars() call (which doesn't accept an incomplete type), but
+    // it seems that `ElemType` cannot be incomplete at this point.
+    if (ElemType->isIncompleteType())
+      return std::nullopt;
+
+    // Calculate Delta = Index * sizeof(ElemType).
+    NonLoc Size = SVB.makeArrayIndex(
+        SVB.getContext().getTypeSizeInChars(ElemType).getQuantity());
+    auto Delta = EvalBinOp(BO_Mul, *Index, Size);
+    if (!Delta)
+      return std::nullopt;
+
+    // Perform Offset += Delta.
+    Offset = EvalBinOp(BO_Add, *Offset, *Delta);
+    if (!Offset)
+      return std::nullopt;
+
+    OwnerRegion = CurRegion->getSuperRegion()->getAs<SubRegion>();
+    // When this is just another ElementRegion layer, we need to continue the
+    // offset calculations:
+    CurRegion = dyn_cast_or_null<ElementRegion>(OwnerRegion);
+  }
+
+  if (OwnerRegion)
+    return std::make_pair(OwnerRegion, *Offset);
+
+  return std::nullopt;
 }
 
-void ArrayBoundChecker::checkLocation(SVal l, bool isLoad, const Stmt* LoadS,
-                                      CheckerContext &C) const {
-  // Check for out of bound array element access.
-  const MemRegion *R = l.getAsRegion();
-  if (!R)
-    return;
+// NOTE: This function is the "heart" of this checker. It simplifies
+// inequalities with transformations that are valid (and very elementary) in
+// pure mathematics, but become invalid if we use them in C++ number model
+// where the calculations may overflow.
+// Due to the overflow issues I think it's impossible (or at least not
+// practical) to integrate this kind of simplification into the resolution of
+// arbitrary inequalities (i.e. the code of `evalBinOp`); but this function
+// produces valid results when the calculations are handling memory offsets
+// and every value is well below SIZE_MAX.
+// TODO: This algorithm should be moved to a central location where it's
+// available for other checkers that need to compare memory offsets.
+// NOTE: the simplification preserves the order of the two operands in a
+// mathematical sense, but it may change the result produced by a C++
+// comparison operator (and the automatic type conversions).
+// For example, consider a comparison "X+1 < 0", where the LHS is stored as a
+// size_t and the RHS is stored in an int. (As size_t is unsigned, this
+// comparison is false for all values of "X".) However, the simplification may
+// turn it into "X < -1", which is still always false in a mathematical sense,
+// but can produce a true result when evaluated by `evalBinOp` (which follows
+// the rules of C++ and casts -1 to SIZE_MAX).
+static std::pair<NonLoc, nonloc::ConcreteInt>
+getSimplifiedOffsets(NonLoc offset, nonloc::ConcreteInt extent,
+                     SValBuilder &svalBuilder) {
+  const llvm::APSInt &extentVal = extent.getValue();
+  std::optional<nonloc::SymbolVal> SymVal = offset.getAs<nonloc::SymbolVal>();
+  if (SymVal && SymVal->isExpression()) {
+    if (const SymIntExpr *SIE = dyn_cast<SymIntExpr>(SymVal->getSymbol())) {
+      llvm::APSInt constant = APSIntType(extentVal).convert(SIE->getRHS());
+      switch (SIE->getOpcode()) {
+      case BO_Mul:
+        // The constant should never be 0 here, becasue multiplication by zero
+        // is simplified by the engine.
+        if ((extentVal % constant) != 0)
+          return std::pair<NonLoc, nonloc::ConcreteInt>(offset, extent);
+        else
+          return getSimplifiedOffsets(
+              nonloc::SymbolVal(SIE->getLHS()),
+              svalBuilder.makeIntVal(extentVal / constant), svalBuilder);
+      case BO_Add:
+        return getSimplifiedOffsets(
+            nonloc::SymbolVal(SIE->getLHS()),
+            svalBuilder.makeIntVal(extentVal - constant), svalBuilder);
+      default:
+        break;
+      }
+    }
+  }
+
+  return std::pair<NonLoc, nonloc::ConcreteInt>(offset, extent);
+}
+
+static bool isNegative(SValBuilder &SVB, ProgramStateRef State, NonLoc Value) {
+  const llvm::APSInt *MaxV = SVB.getMaxValue(State, Value);
+  return MaxV && MaxV->isNegative();
+}
+
+static bool isUnsigned(SValBuilder &SVB, NonLoc Value) {
+  QualType T = Value.getType(SVB.getContext());
+  return T->isUnsignedIntegerType();
+}
+
+// Evaluate the comparison Value < Threshold with the help of the custom
+// simplification algorithm defined for this checker. Return a pair of states,
+// where the first one corresponds to "value below threshold" and the second
+// corresponds to "value at or above threshold". Returns {nullptr, nullptr} in
+// the case when the evaluation fails.
+// If the optional argument CheckEquality is true, then use BO_EQ instead of
+// the default BO_LT after consistently applying the same simplification steps.
+static std::pair<ProgramStateRef, ProgramStateRef>
+compareValueToThreshold(ProgramStateRef State, NonLoc Value, NonLoc Threshold,
+                        SValBuilder &SVB, bool CheckEquality = false) {
+  if (auto ConcreteThreshold = Threshold.getAs<nonloc::ConcreteInt>()) {
+    std::tie(Value, Threshold) =
+        getSimplifiedOffsets(Value, *ConcreteThreshold, SVB);
+  }
+
+  // We want to perform a _mathematical_ comparison between the numbers `Value`
+  // and `Threshold`; but `evalBinOpNN` evaluates a C/C++ operator that may
+  // perform automatic conversions. For example the number -1 is less than the
+  // number 1000, but -1 < `1000ull` will evaluate to `false` because the `int`
+  // -1 is converted to ULONGLONG_MAX.
+  // To avoid automatic conversions, we evaluate the "obvious" cases without
+  // calling `evalBinOpNN`:
+  if (isNegative(SVB, State, Value) && isUnsigned(SVB, Threshold)) {
+    if (CheckEquality) {
+      // negative_value == unsigned_threshold is always false
+      return {nullptr, State};
+    }
+    // negative_value < unsigned_threshold is always true
+    return {State, nullptr};
+  }
+  if (isUnsigned(SVB, Value) && isNegative(SVB, State, Threshold)) {
+    // unsigned_value == negative_threshold and
+    // unsigned_value < negative_threshold are both always false
+    return {nullptr, State};
+  }
+  // FIXME: These special cases are sufficient for handling real-world
+  // comparisons, but in theory there could be contrived situations where
+  // automatic conversion of a symbolic value (which can be negative and can be
+  // positive) leads to incorrect results.
+  // NOTE: We NEED to use the `evalBinOpNN` call in the "common" case, because
+  // we want to ensure that assumptions coming from this precondition and
+  // assumptions coming from regular C/C++ operator calls are represented by
+  // constraints on the same symbolic expression. A solution that would
+  // evaluate these "mathematical" compariosns through a separate pathway would
+  // be a step backwards in this sense.
+
+  const BinaryOperatorKind OpKind = CheckEquality ? BO_EQ : BO_LT;
+  auto BelowThreshold =
+      SVB.evalBinOpNN(State, OpKind, Value, Threshold, SVB.getConditionType())
+          .getAs<NonLoc>();
+
+  if (BelowThreshold)
+    return State->assume(*BelowThreshold);
+
+  return {nullptr, nullptr};
+}
+
+static std::string getRegionName(const SubRegion *Region) {
+  if (std::string RegName = Region->getDescriptiveName(); !RegName.empty())
+    return RegName;
+
+  // Field regions only have descriptive names when their parent has a
+  // descriptive name; so we provide a fallback representation for them:
+  if (const auto *FR = Region->getAs<FieldRegion>()) {
+    if (StringRef Name = FR->getDecl()->getName(); !Name.empty())
+      return formatv("the field '{0}'", Name);
+    return "the unnamed field";
+  }
+
+  if (isa<AllocaRegion>(Region))
+    return "the memory returned by 'alloca'";
+
+  if (isa<SymbolicRegion>(Region) &&
+      isa<HeapSpaceRegion>(Region->getMemorySpace()))
+    return "the heap area";
+
+  if (isa<StringRegion>(Region))
+    return "the string literal";
+
+  return "the region";
+}
+
+static std::optional<int64_t> getConcreteValue(NonLoc SV) {
+  if (auto ConcreteVal = SV.getAs<nonloc::ConcreteInt>()) {
+    return ConcreteVal->getValue()->tryExtValue();
+  }
+  return std::nullopt;
+}
 
-  const ElementRegion *ER = dyn_cast<ElementRegion>(R);
-  if (!ER)
+static std::optional<int64_t> getConcreteValue(std::optional<NonLoc> SV) {
+  return SV ? getConcreteValue(*SV) : std::nullopt;
+}
+
+static Messages getPrecedesMsgs(const SubRegion *Region, NonLoc Offset) {
+  std::string RegName = getRegionName(Region), OffsetStr = "";
+
+  if (auto ConcreteOffset = getConcreteValue(Offset))
+    OffsetStr = formatv(" {0}", ConcreteOffset);
+
+  return {
+      formatv("Out of bound access to memory preceding {0}", RegName),
+      formatv("Access of {0} at negative byte offset{1}", RegName, OffsetStr)};
+}
+
+/// Try to divide `Val1` and `Val2` (in place) by `Divisor` and return true if
+/// it can be performed (`Divisor` is nonzero and there is no remainder). The
+/// values `Val1` and `Val2` may be nullopt and in that case the corresponding
+/// division is considered to be successful.
+static bool tryDividePair(std::optional<int64_t> &Val1,
+                          std::optional<int64_t> &Val2, int64_t Divisor) {
+  if (!Divisor)
+    return false;
+  const bool Val1HasRemainder = Val1 && *Val1 % Divisor;
+  const bool Val2HasRemainder = Val2 && *Val2 % Divisor;
+  if (!Val1HasRemainder && !Val2HasRemainder) {
+    if (Val1)
+      *Val1 /= Divisor;
+    if (Val2)
+      *Val2 /= Divisor;
+    return true;
+  }
+  return false;
+}
+
+static Messages getExceedsMsgs(ASTContext &ACtx, const SubRegion *Region,
+                               NonLoc Offset, NonLoc Extent, SVal Location,
+                               bool AlsoMentionUnderflow) {
+  std::string RegName = getRegionName(Region);
+  const auto *EReg = Location.getAsRegion()->getAs<ElementRegion>();
+  assert(EReg && "this checker only handles element access");
+  QualType ElemType = EReg->getElementType();
+
+  std::optional<int64_t> OffsetN = getConcreteValue(Offset);
+  std::optional<int64_t> ExtentN = getConcreteValue(Extent);
+
+  int64_t ElemSize = ACtx.getTypeSizeInChars(ElemType).getQuantity();
+
+  bool UseByteOffsets = !tryDividePair(OffsetN, ExtentN, ElemSize);
+  const char *OffsetOrIndex = UseByteOffsets ? "byte offset" : "index";
+
+  SmallString<256> Buf;
+  llvm::raw_svector_ostream Out(Buf);
+  Out << "Access of ";
+  if (!ExtentN && !UseByteOffsets)
+    Out << "'" << ElemType.getAsString() << "' element in ";
+  Out << RegName << " at ";
+  if (AlsoMentionUnderflow) {
+    Out << "a negative or overflowing " << OffsetOrIndex;
+  } else if (OffsetN) {
+    Out << OffsetOrIndex << " " << *OffsetN;
+  } else {
+    Out << "an overflowing " << OffsetOrIndex;
+  }
+  if (ExtentN) {
+    Out << ", while it holds only ";
+    if (*ExtentN != 1)
+      Out << *ExtentN;
+    else
+      Out << "a single";
+    if (UseByteOffsets)
+      Out << " byte";
+    else
+      Out << " '" << ElemType.getAsString() << "' element";
+
+    if (*ExtentN > 1)
+      Out << "s";
+  }
+
+  return {formatv("Out of bound access to memory {0} {1}",
+                  AlsoMentionUnderflow ? "around" : "after the end of",
+                  RegName),
+          std::string(Buf)};
+}
+
+static Messages getTaintMsgs(const SubRegion *Region, const char *OffsetName,
+                             bool AlsoMentionUnderflow) {
+  std::string RegName = getRegionName(Region);
+  return {formatv("Potential out of bound access to {0} with tainted {1}",
+                  RegName, OffsetName),
+          formatv("Access of {0} with a tainted {1} that may be {2}too large",
+                  RegName, OffsetName,
+                  AlsoMentionUnderflow ? "negative or " : "")};
+}
+
+const NoteTag *StateUpdateReporter::createNoteTag(CheckerContext &C) const {
+  // Don't create a note tag if we didn't assume anything:
+  if (!AssumedNonNegative && !AssumedUpperBound)
+    return nullptr;
+
+  return C.getNoteTag([*this](PathSensitiveBugReport &BR) -> std::string {
+    return getMessage(BR);
+  });
+}
+
+std::string StateUpdateReporter::getMessage(PathSensitiveBugReport &BR) const {
+  bool ShouldReportNonNegative = AssumedNonNegative;
+  if (!providesInformationAboutInteresting(ByteOffsetVal, BR)) {
+    if (AssumedUpperBound &&
+        providesInformationAboutInteresting(*AssumedUpperBound, BR)) {
+      // Even if the byte offset isn't interesting (e.g. it's a constant value),
+      // the assumption can still be interesting if it provides information
+      // about an interesting symbolic upper bound.
+      ShouldReportNonNegative = false;
+    } else {
+      // We don't have anything interesting, don't report the assumption.
+      return "";
+    }
+  }
+
+  std::optional<int64_t> OffsetN = getConcreteValue(ByteOffsetVal);
+  std::optional<int64_t> ExtentN = getConcreteValue(AssumedUpperBound);
+
+  const bool UseIndex =
+      ElementSize && tryDividePair(OffsetN, ExtentN, *ElementSize);
+
+  SmallString<256> Buf;
+  llvm::raw_svector_ostream Out(Buf);
+  Out << "Assuming ";
+  if (UseIndex) {
+    Out << "index ";
+    if (OffsetN)
+      Out << "'" << OffsetN << "' ";
+  } else if (AssumedUpperBound) {
+    Out << "byte offset ";
+    if (OffsetN)
+      Out << "'" << OffsetN << "' ";
+  } else {
+    Out << "offset ";
+  }
+
+  Out << "is";
+  if (ShouldReportNonNegative) {
+    Out << " non-negative";
+  }
+  if (AssumedUpperBound) {
+    if (ShouldReportNonNegative)
+      Out << " and";
+    Out << " less than ";
+    if (ExtentN)
+      Out << *ExtentN << ", ";
+    if (UseIndex && ElementType)
+      Out << "the number of '" << ElementType->getAsString()
+          << "' elements in ";
+    else
+      Out << "the extent of ";
+    Out << getRegionName(Reg);
+  }
+  return std::string(Out.str());
+}
+
+bool StateUpdateReporter::providesInformationAboutInteresting(
+    SymbolRef Sym, PathSensitiveBugReport &BR) {
+  if (!Sym)
+    return false;
+  for (SymbolRef PartSym : Sym->symbols()) {
+    // The interestingess mark may appear on any layer as we're stripping off
+    // the SymIntExpr, UnarySymExpr etc. layers...
+    if (BR.isInteresting(PartSym))
+      return true;
+    // ...but if both sides of the expression are symbolic, then there is no
+    // practical algorithm to produce separate constraints for the two
+    // operands (from the single combined result).
+    if (isa<SymSymExpr>(PartSym))
+      return false;
+  }
+  return false;
+}
+
+void ArrayBoundChecker::performCheck(const Expr *E, CheckerContext &C) const {
+  const SVal Location = C.getSVal(E);
+
+  // The header ctype.h (from e.g. glibc) implements the isXXXXX() macros as
+  //   #define isXXXXX(arg) (LOOKUP_TABLE[arg] & BITMASK_FOR_XXXXX)
+  // and incomplete analysis of these leads to false positives. As even
+  // accurate reports would be confusing for the users, just disable reports
+  // from these macros:
+  if (isFromCtypeMacro(E, C.getASTContext()))
     return;
 
-  // Get the index of the accessed element.
-  DefinedOrUnknownSVal Idx = ER->getIndex().castAs<DefinedOrUnknownSVal>();
+  ProgramStateRef State = C.getState();
+  SValBuilder &SVB = C.getSValBuilder();
 
-  // Zero index is always in bound, this also passes ElementRegions created for
-  // pointer casts.
-  if (Idx.isZeroConstant())
+  const std::optional<std::pair<const SubRegion *, NonLoc>> &RawOffset =
+      computeOffset(State, SVB, Location);
+
+  if (!RawOffset)
     return;
 
-  ProgramStateRef state = C.getState();
+  auto [Reg, ByteOffset] = *RawOffset;
 
-  // Get the size of the array.
-  DefinedOrUnknownSVal ElementCount = getDynamicElementCount(
-      state, ER->getSuperRegion(), C.getSValBuilder(), ER->getValueType());
+  // The state updates will be reported as a single note tag, which will be
+  // composed by this helper class.
+  StateUpdateReporter SUR(Reg, ByteOffset, E, C);
 
-  ProgramStateRef StInBound, StOutBound;
-  std::tie(StInBound, StOutBound) = state->assumeInBoundDual(Idx, ElementCount);
-  if (StOutBound && !StInBound) {
-    ExplodedNode *N = C.generateErrorNode(StOutBound);
-    if (!N)
-      return;
+  // CHECK LOWER BOUND
+  const MemSpaceRegion *Space = Reg->getMemorySpace();
+  if (!(isa<SymbolicRegion>(Reg) && isa<UnknownSpaceRegion>(Space))) {
+    // A symbolic region in unknown space represents an unknown pointer that
+    // may point into the middle of an array, so we don't look for underflows.
+    // Both conditions are significant because we want to check underflows in
+    // symbolic regions on the heap (which may be introduced by checkers like
+    // MallocChecker that call SValBuilder::getConjuredHeapSymbolVal()) and
+    // non-symbolic regions (e.g. a field subregion of a symbolic region) in
+    // unknown space.
+    auto [PrecedesLowerBound, WithinLowerBound] = compareValueToThreshold(
+        State, ByteOffset, SVB.makeZeroArrayIndex(), SVB);
 
-    // FIXME: It would be nice to eventually make this diagnostic more clear,
-    // e.g., by referencing the original declaration or by saying *why* this
-    // reference is outside the range.
+    if (PrecedesLowerBound) {
+      // The offset may be invalid (negative)...
+      if (!WithinLowerBound) {
+        // ...and it cannot be valid (>= 0), so report an error.
+        Messages Msgs = getPrecedesMsgs(Reg, ByteOffset);
+        reportOOB(C, PrecedesLowerBound, Msgs, ByteOffset, std::nullopt);
+        return;
+      }
+      // ...but it can be valid as well, so the checker will (optimistically)
+      // assume that it's valid and mention this in the note tag.
+      SUR.recordNonNegativeAssumption();
+    }
 
-    // Generate a report for this bug.
-    auto report = std::make_unique<PathSensitiveBugReport>(
-        BT, "Access out-of-bound array element (buffer overflow)", N);
+    // Actually update the state. The "if" only fails in the extremely unlikely
+    // case when compareValueToThreshold returns {nullptr, nullptr} becasue
+    // evalBinOpNN fails to evaluate the less-than operator.
+    if (WithinLowerBound)
+      State = WithinLowerBound;
+  }
 
-    report->addRange(LoadS->getSourceRange());
-    C.emitReport(std::move(report));
-    return;
+  // CHECK UPPER BOUND
+  DefinedOrUnknownSVal Size = getDynamicExtent(State, Reg, SVB);
+  if (auto KnownSize = Size.getAs<NonLoc>()) {
+    // In a situation where both underflow and overflow are possible (but the
+    // index is either tainted or known to be invalid), the logic of this
+    // checker will first assume that the offset is non-negative, and then
+    // (with this additional assumption) it will detect an overflow error.
+    // In this situation the warning message should mention both possibilities.
+    bool AlsoMentionUnderflow = SUR.assumedNonNegative();
+
+    auto [WithinUpperBound, ExceedsUpperBound] =
+        compareValueToThreshold(State, ByteOffset, *KnownSize, SVB);
+
+    if (ExceedsUpperBound) {
+      // The offset may be invalid (>= Size)...
+      if (!WithinUpperBound) {
+        // ...and it cannot be within bounds, so report an error, unless we can
+        // definitely determine that this is an idiomatic `&array[size]`
+        // expression that calculates the past-the-end pointer.
+        if (isIdiomaticPastTheEndPtr(E, ExceedsUpperBound, ByteOffset,
+                                     *KnownSize, C)) {
+          C.addTransition(ExceedsUpperBound, SUR.createNoteTag(C));
+          return;
+        }
+
+        Messages Msgs =
+            getExceedsMsgs(C.getASTContext(), Reg, ByteOffset, *KnownSize,
+                           Location, AlsoMentionUnderflow);
+        reportOOB(C, ExceedsUpperBound, Msgs, ByteOffset, KnownSize);
+        return;
+      }
+      // ...and it can be valid as well...
+      if (isTainted(State, ByteOffset)) {
+        // ...but it's tainted, so report an error.
+
+        // Diagnostic detail: saying "tainted offset" is always correct, but
+        // the common case is that 'idx' is tainted in 'arr[idx]' and then it's
+        // nicer to say "tainted index".
+        const char *OffsetName = "offset";
+        if (const auto *ASE = dyn_cast<ArraySubscriptExpr>(E))
+          if (isTainted(State, ASE->getIdx(), C.getLocationContext()))
+            OffsetName = "index";
+
+        Messages Msgs = getTaintMsgs(Reg, OffsetName, AlsoMentionUnderflow);
+        reportOOB(C, ExceedsUpperBound, Msgs, ByteOffset, KnownSize,
+                  /*IsTaintBug=*/true);
+        return;
+      }
+      // ...and it isn't tainted, so the checker will (optimistically) assume
+      // that the offset is in bounds and mention this in the note tag.
+      SUR.recordUpperBoundAssumption(*KnownSize);
+    }
+
+    // Actually update the state. The "if" only fails in the extremely unlikely
+    // case when compareValueToThreshold returns {nullptr, nullptr} becasue
+    // evalBinOpNN fails to evaluate the less-than operator.
+    if (WithinUpperBound)
+      State = WithinUpperBound;
+  }
+
+  // Add a transition, reporting the state updates that we accumulated.
+  C.addTransition(State, SUR.createNoteTag(C));
+}
+
+void ArrayBoundChecker::markPartsInteresting(PathSensitiveBugReport &BR,
+                                             ProgramStateRef ErrorState,
+                                             NonLoc Val, bool MarkTaint) {
+  if (SymbolRef Sym = Val.getAsSymbol()) {
+    // If the offset is a symbolic value, iterate over its "parts" with
+    // `SymExpr::symbols()` and mark each of them as interesting.
+    // For example, if the offset is `x*4 + y` then we put interestingness onto
+    // the SymSymExpr `x*4 + y`, the SymIntExpr `x*4` and the two data symbols
+    // `x` and `y`.
+    for (SymbolRef PartSym : Sym->symbols())
+      BR.markInteresting(PartSym);
+  }
+
+  if (MarkTaint) {
+    // If the issue that we're reporting depends on the taintedness of the
+    // offset, then put interestingness onto symbols that could be the origin
+    // of the taint. Note that this may find symbols that did not appear in
+    // `Sym->symbols()` (because they're only loosely connected to `Val`).
+    for (SymbolRef Sym : getTaintedSymbols(ErrorState, Val))
+      BR.markInteresting(Sym);
   }
+}
+
+void ArrayBoundChecker::reportOOB(CheckerContext &C, ProgramStateRef ErrorState,
+                                  Messages Msgs, NonLoc Offset,
+                                  std::optional<NonLoc> Extent,
+                                  bool IsTaintBug /*=false*/) const {
+
+  ExplodedNode *ErrorNode = C.generateErrorNode(ErrorState);
+  if (!ErrorNode)
+    return;
+
+  auto BR = std::make_unique<PathSensitiveBugReport>(
+      IsTaintBug ? TaintBT : BT, Msgs.Short, Msgs.Full, ErrorNode);
+
+  // FIXME: ideally we would just call trackExpressionValue() and that would
+  // "do the right thing": mark the relevant symbols as interesting, track the
+  // control dependencies and statements storing the relevant values and add
+  // helpful diagnostic pieces. However, right now trackExpressionValue() is
+  // a heap of unreliable heuristics, so it would cause several issues:
+  // - Interestingness is not applied consistently, e.g. if `array[x+10]`
+  //   causes an overflow, then `x` is not marked as interesting.
+  // - We get irrelevant diagnostic pieces, e.g. in the code
+  //   `int *p = (int*)malloc(2*sizeof(int)); p[3] = 0;`
+  //   it places a "Storing uninitialized value" note on the `malloc` call
+  //   (which is technically true, but irrelevant).
+  // If trackExpressionValue() becomes reliable, it should be applied instead
+  // of this custom markPartsInteresting().
+  markPartsInteresting(*BR, ErrorState, Offset, IsTaintBug);
+  if (Extent)
+    markPartsInteresting(*BR, ErrorState, *Extent, IsTaintBug);
+
+  C.emitReport(std::move(BR));
+}
 
-  // Array bound check succeeded.  From this point forward the array bound
-  // should always succeed.
-  C.addTransition(StInBound);
+bool ArrayBoundChecker::isFromCtypeMacro(const Stmt *S, ASTContext &ACtx) {
+  SourceLocation Loc = S->getBeginLoc();
+  if (!Loc.isMacroID())
+    return false;
+
+  StringRef MacroName = Lexer::getImmediateMacroName(
+      Loc, ACtx.getSourceManager(), ACtx.getLangOpts());
+
+  if (MacroName.size() < 7 || MacroName[0] != 'i' || MacroName[1] != 's')
+    return false;
+
+  return ((MacroName == "isalnum") || (MacroName == "isalpha") ||
+          (MacroName == "isblank") || (MacroName == "isdigit") ||
+          (MacroName == "isgraph") || (MacroName == "islower") ||
+          (MacroName == "isnctrl") || (MacroName == "isprint") ||
+          (MacroName == "ispunct") || (MacroName == "isspace") ||
+          (MacroName == "isupper") || (MacroName == "isxdigit"));
+}
+
+bool ArrayBoundChecker::isInAddressOf(const Stmt *S, ASTContext &ACtx) {
+  ParentMapContext &ParentCtx = ACtx.getParentMapContext();
+  do {
+    const DynTypedNodeList Parents = ParentCtx.getParents(*S);
+    if (Parents.empty())
+      return false;
+    S = Parents[0].get<Stmt>();
+  } while (isa_and_nonnull<ParenExpr, ImplicitCastExpr>(S));
+  const auto *UnaryOp = dyn_cast_or_null<UnaryOperator>(S);
+  return UnaryOp && UnaryOp->getOpcode() == UO_AddrOf;
+}
+
+bool ArrayBoundChecker::isIdiomaticPastTheEndPtr(const Expr *E,
+                                                 ProgramStateRef State,
+                                                 NonLoc Offset, NonLoc Limit,
+                                                 CheckerContext &C) {
+  if (isa<ArraySubscriptExpr>(E) && isInAddressOf(E, C.getASTContext())) {
+    auto [EqualsToThreshold, NotEqualToThreshold] = compareValueToThreshold(
+        State, Offset, Limit, C.getSValBuilder(), /*CheckEquality=*/true);
+    return EqualsToThreshold && !NotEqualToThreshold;
+  }
+  return false;
 }
 
 void ento::registerArrayBoundChecker(CheckerManager &mgr) {
diff --git a/clang/lib/StaticAnalyzer/Checkers/ArrayBoundCheckerV2.cpp b/clang/lib/StaticAnalyzer/Checkers/ArrayBoundCheckerV2.cpp
deleted file mode 100644
index 6422933c8828a..0000000000000
--- a/clang/lib/StaticAnalyzer/Checkers/ArrayBoundCheckerV2.cpp
+++ /dev/null
@@ -1,777 +0,0 @@
-//== ArrayBoundCheckerV2.cpp ------------------------------------*- C++ -*--==//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-//
-// This file defines ArrayBoundCheckerV2, which is a path-sensitive check
-// which looks for an out-of-bound array element access.
-//
-//===----------------------------------------------------------------------===//
-
-#include "clang/AST/CharUnits.h"
-#include "clang/AST/ParentMapContext.h"
-#include "clang/StaticAnalyzer/Checkers/BuiltinCheckerRegistration.h"
-#include "clang/StaticAnalyzer/Checkers/Taint.h"
-#include "clang/StaticAnalyzer/Core/BugReporter/BugType.h"
-#include "clang/StaticAnalyzer/Core/Checker.h"
-#include "clang/StaticAnalyzer/Core/CheckerManager.h"
-#include "clang/StaticAnalyzer/Core/PathSensitive/APSIntType.h"
-#include "clang/StaticAnalyzer/Core/PathSensitive/CheckerContext.h"
-#include "clang/StaticAnalyzer/Core/PathSensitive/DynamicExtent.h"
-#include "clang/StaticAnalyzer/Core/PathSensitive/ExprEngine.h"
-#include "llvm/ADT/APSInt.h"
-#include "llvm/ADT/SmallString.h"
-#include "llvm/Support/FormatVariadic.h"
-#include "llvm/Support/raw_ostream.h"
-#include <optional>
-
-using namespace clang;
-using namespace ento;
-using namespace taint;
-using llvm::formatv;
-
-namespace {
-/// If `E` is a "clean" array subscript expression, return the type of the
-/// accessed element. If the base of the subscript expression is modified by
-/// pointer arithmetic (and not the beginning of a "full" memory region), this
-/// always returns nullopt because that's the right (or the least bad) thing to
-/// do for the diagnostic output that's relying on this.
-static std::optional<QualType> determineElementType(const Expr *E,
-                                                    const CheckerContext &C) {
-  const auto *ASE = dyn_cast<ArraySubscriptExpr>(E);
-  if (!ASE)
-    return std::nullopt;
-
-  const MemRegion *SubscriptBaseReg = C.getSVal(ASE->getBase()).getAsRegion();
-  if (!SubscriptBaseReg)
-    return std::nullopt;
-
-  // The base of the subscript expression is affected by pointer arithmetics,
-  // so we want to report byte offsets instead of indices.
-  if (isa<ElementRegion>(SubscriptBaseReg->StripCasts()))
-    return std::nullopt;
-
-  return ASE->getType();
-}
-
-static std::optional<int64_t>
-determineElementSize(const std::optional<QualType> T, const CheckerContext &C) {
-  if (!T)
-    return std::nullopt;
-  return C.getASTContext().getTypeSizeInChars(*T).getQuantity();
-}
-
-class StateUpdateReporter {
-  const SubRegion *Reg;
-  const NonLoc ByteOffsetVal;
-  const std::optional<QualType> ElementType;
-  const std::optional<int64_t> ElementSize;
-  bool AssumedNonNegative = false;
-  std::optional<NonLoc> AssumedUpperBound = std::nullopt;
-
-public:
-  StateUpdateReporter(const SubRegion *R, NonLoc ByteOffsVal, const Expr *E,
-                      CheckerContext &C)
-      : Reg(R), ByteOffsetVal(ByteOffsVal),
-        ElementType(determineElementType(E, C)),
-        ElementSize(determineElementSize(ElementType, C)) {}
-
-  void recordNonNegativeAssumption() { AssumedNonNegative = true; }
-  void recordUpperBoundAssumption(NonLoc UpperBoundVal) {
-    AssumedUpperBound = UpperBoundVal;
-  }
-
-  bool assumedNonNegative() { return AssumedNonNegative; }
-
-  const NoteTag *createNoteTag(CheckerContext &C) const;
-
-private:
-  std::string getMessage(PathSensitiveBugReport &BR) const;
-
-  /// Return true if information about the value of `Sym` can put constraints
-  /// on some symbol which is interesting within the bug report `BR`.
-  /// In particular, this returns true when `Sym` is interesting within `BR`;
-  /// but it also returns true if `Sym` is an expression that contains integer
-  /// constants and a single symbolic operand which is interesting (in `BR`).
-  /// We need to use this instead of plain `BR.isInteresting()` because if we
-  /// are analyzing code like
-  ///   int array[10];
-  ///   int f(int arg) {
-  ///     return array[arg] && array[arg + 10];
-  ///   }
-  /// then the byte offsets are `arg * 4` and `(arg + 10) * 4`, which are not
-  /// sub-expressions of each other (but `getSimplifiedOffsets` is smart enough
-  /// to detect this out of bounds access).
-  static bool providesInformationAboutInteresting(SymbolRef Sym,
-                                                  PathSensitiveBugReport &BR);
-  static bool providesInformationAboutInteresting(SVal SV,
-                                                  PathSensitiveBugReport &BR) {
-    return providesInformationAboutInteresting(SV.getAsSymbol(), BR);
-  }
-};
-
-struct Messages {
-  std::string Short, Full;
-};
-
-// NOTE: The `ArraySubscriptExpr` and `UnaryOperator` callbacks are `PostStmt`
-// instead of `PreStmt` because the current implementation passes the whole
-// expression to `CheckerContext::getSVal()` which only works after the
-// symbolic evaluation of the expression. (To turn them into `PreStmt`
-// callbacks, we'd need to duplicate the logic that evaluates these
-// expressions.) The `MemberExpr` callback would work as `PreStmt` but it's
-// defined as `PostStmt` for the sake of consistency with the other callbacks.
-class ArrayBoundCheckerV2 : public Checker<check::PostStmt<ArraySubscriptExpr>,
-                                           check::PostStmt<UnaryOperator>,
-                                           check::PostStmt<MemberExpr>> {
-  BugType BT{this, "Out-of-bound access"};
-  BugType TaintBT{this, "Out-of-bound access", categories::TaintedData};
-
-  void performCheck(const Expr *E, CheckerContext &C) const;
-
-  void reportOOB(CheckerContext &C, ProgramStateRef ErrorState, Messages Msgs,
-                 NonLoc Offset, std::optional<NonLoc> Extent,
-                 bool IsTaintBug = false) const;
-
-  static void markPartsInteresting(PathSensitiveBugReport &BR,
-                                   ProgramStateRef ErrorState, NonLoc Val,
-                                   bool MarkTaint);
-
-  static bool isFromCtypeMacro(const Stmt *S, ASTContext &AC);
-
-  static bool isIdiomaticPastTheEndPtr(const Expr *E, ProgramStateRef State,
-                                       NonLoc Offset, NonLoc Limit,
-                                       CheckerContext &C);
-  static bool isInAddressOf(const Stmt *S, ASTContext &AC);
-
-public:
-  void checkPostStmt(const ArraySubscriptExpr *E, CheckerContext &C) const {
-    performCheck(E, C);
-  }
-  void checkPostStmt(const UnaryOperator *E, CheckerContext &C) const {
-    if (E->getOpcode() == UO_Deref)
-      performCheck(E, C);
-  }
-  void checkPostStmt(const MemberExpr *E, CheckerContext &C) const {
-    if (E->isArrow())
-      performCheck(E->getBase(), C);
-  }
-};
-
-} // anonymous namespace
-
-/// For a given Location that can be represented as a symbolic expression
-/// Arr[Idx] (or perhaps Arr[Idx1][Idx2] etc.), return the parent memory block
-/// Arr and the distance of Location from the beginning of Arr (expressed in a
-/// NonLoc that specifies the number of CharUnits). Returns nullopt when these
-/// cannot be determined.
-static std::optional<std::pair<const SubRegion *, NonLoc>>
-computeOffset(ProgramStateRef State, SValBuilder &SVB, SVal Location) {
-  QualType T = SVB.getArrayIndexType();
-  auto EvalBinOp = [&SVB, State, T](BinaryOperatorKind Op, NonLoc L, NonLoc R) {
-    // We will use this utility to add and multiply values.
-    return SVB.evalBinOpNN(State, Op, L, R, T).getAs<NonLoc>();
-  };
-
-  const SubRegion *OwnerRegion = nullptr;
-  std::optional<NonLoc> Offset = SVB.makeZeroArrayIndex();
-
-  const ElementRegion *CurRegion =
-      dyn_cast_or_null<ElementRegion>(Location.getAsRegion());
-
-  while (CurRegion) {
-    const auto Index = CurRegion->getIndex().getAs<NonLoc>();
-    if (!Index)
-      return std::nullopt;
-
-    QualType ElemType = CurRegion->getElementType();
-
-    // FIXME: The following early return was presumably added to safeguard the
-    // getTypeSizeInChars() call (which doesn't accept an incomplete type), but
-    // it seems that `ElemType` cannot be incomplete at this point.
-    if (ElemType->isIncompleteType())
-      return std::nullopt;
-
-    // Calculate Delta = Index * sizeof(ElemType).
-    NonLoc Size = SVB.makeArrayIndex(
-        SVB.getContext().getTypeSizeInChars(ElemType).getQuantity());
-    auto Delta = EvalBinOp(BO_Mul, *Index, Size);
-    if (!Delta)
-      return std::nullopt;
-
-    // Perform Offset += Delta.
-    Offset = EvalBinOp(BO_Add, *Offset, *Delta);
-    if (!Offset)
-      return std::nullopt;
-
-    OwnerRegion = CurRegion->getSuperRegion()->getAs<SubRegion>();
-    // When this is just another ElementRegion layer, we need to continue the
-    // offset calculations:
-    CurRegion = dyn_cast_or_null<ElementRegion>(OwnerRegion);
-  }
-
-  if (OwnerRegion)
-    return std::make_pair(OwnerRegion, *Offset);
-
-  return std::nullopt;
-}
-
-// NOTE: This function is the "heart" of this checker. It simplifies
-// inequalities with transformations that are valid (and very elementary) in
-// pure mathematics, but become invalid if we use them in C++ number model
-// where the calculations may overflow.
-// Due to the overflow issues I think it's impossible (or at least not
-// practical) to integrate this kind of simplification into the resolution of
-// arbitrary inequalities (i.e. the code of `evalBinOp`); but this function
-// produces valid results when the calculations are handling memory offsets
-// and every value is well below SIZE_MAX.
-// TODO: This algorithm should be moved to a central location where it's
-// available for other checkers that need to compare memory offsets.
-// NOTE: the simplification preserves the order of the two operands in a
-// mathematical sense, but it may change the result produced by a C++
-// comparison operator (and the automatic type conversions).
-// For example, consider a comparison "X+1 < 0", where the LHS is stored as a
-// size_t and the RHS is stored in an int. (As size_t is unsigned, this
-// comparison is false for all values of "X".) However, the simplification may
-// turn it into "X < -1", which is still always false in a mathematical sense,
-// but can produce a true result when evaluated by `evalBinOp` (which follows
-// the rules of C++ and casts -1 to SIZE_MAX).
-static std::pair<NonLoc, nonloc::ConcreteInt>
-getSimplifiedOffsets(NonLoc offset, nonloc::ConcreteInt extent,
-                     SValBuilder &svalBuilder) {
-  const llvm::APSInt &extentVal = extent.getValue();
-  std::optional<nonloc::SymbolVal> SymVal = offset.getAs<nonloc::SymbolVal>();
-  if (SymVal && SymVal->isExpression()) {
-    if (const SymIntExpr *SIE = dyn_cast<SymIntExpr>(SymVal->getSymbol())) {
-      llvm::APSInt constant = APSIntType(extentVal).convert(SIE->getRHS());
-      switch (SIE->getOpcode()) {
-      case BO_Mul:
-        // The constant should never be 0 here, becasue multiplication by zero
-        // is simplified by the engine.
-        if ((extentVal % constant) != 0)
-          return std::pair<NonLoc, nonloc::ConcreteInt>(offset, extent);
-        else
-          return getSimplifiedOffsets(
-              nonloc::SymbolVal(SIE->getLHS()),
-              svalBuilder.makeIntVal(extentVal / constant), svalBuilder);
-      case BO_Add:
-        return getSimplifiedOffsets(
-            nonloc::SymbolVal(SIE->getLHS()),
-            svalBuilder.makeIntVal(extentVal - constant), svalBuilder);
-      default:
-        break;
-      }
-    }
-  }
-
-  return std::pair<NonLoc, nonloc::ConcreteInt>(offset, extent);
-}
-
-static bool isNegative(SValBuilder &SVB, ProgramStateRef State, NonLoc Value) {
-  const llvm::APSInt *MaxV = SVB.getMaxValue(State, Value);
-  return MaxV && MaxV->isNegative();
-}
-
-static bool isUnsigned(SValBuilder &SVB, NonLoc Value) {
-  QualType T = Value.getType(SVB.getContext());
-  return T->isUnsignedIntegerType();
-}
-
-// Evaluate the comparison Value < Threshold with the help of the custom
-// simplification algorithm defined for this checker. Return a pair of states,
-// where the first one corresponds to "value below threshold" and the second
-// corresponds to "value at or above threshold". Returns {nullptr, nullptr} in
-// the case when the evaluation fails.
-// If the optional argument CheckEquality is true, then use BO_EQ instead of
-// the default BO_LT after consistently applying the same simplification steps.
-static std::pair<ProgramStateRef, ProgramStateRef>
-compareValueToThreshold(ProgramStateRef State, NonLoc Value, NonLoc Threshold,
-                        SValBuilder &SVB, bool CheckEquality = false) {
-  if (auto ConcreteThreshold = Threshold.getAs<nonloc::ConcreteInt>()) {
-    std::tie(Value, Threshold) = getSimplifiedOffsets(Value, *ConcreteThreshold, SVB);
-  }
-
-  // We want to perform a _mathematical_ comparison between the numbers `Value`
-  // and `Threshold`; but `evalBinOpNN` evaluates a C/C++ operator that may
-  // perform automatic conversions. For example the number -1 is less than the
-  // number 1000, but -1 < `1000ull` will evaluate to `false` because the `int`
-  // -1 is converted to ULONGLONG_MAX.
-  // To avoid automatic conversions, we evaluate the "obvious" cases without
-  // calling `evalBinOpNN`:
-  if (isNegative(SVB, State, Value) && isUnsigned(SVB, Threshold)) {
-    if (CheckEquality) {
-      // negative_value == unsigned_threshold is always false
-      return {nullptr, State};
-    }
-    // negative_value < unsigned_threshold is always true
-    return {State, nullptr};
-  }
-  if (isUnsigned(SVB, Value) && isNegative(SVB, State, Threshold)) {
-    // unsigned_value == negative_threshold and
-    // unsigned_value < negative_threshold are both always false
-    return {nullptr, State};
-  }
-  // FIXME: These special cases are sufficient for handling real-world
-  // comparisons, but in theory there could be contrived situations where
-  // automatic conversion of a symbolic value (which can be negative and can be
-  // positive) leads to incorrect results.
-  // NOTE: We NEED to use the `evalBinOpNN` call in the "common" case, because
-  // we want to ensure that assumptions coming from this precondition and
-  // assumptions coming from regular C/C++ operator calls are represented by
-  // constraints on the same symbolic expression. A solution that would
-  // evaluate these "mathematical" compariosns through a separate pathway would
-  // be a step backwards in this sense.
-
-  const BinaryOperatorKind OpKind = CheckEquality ? BO_EQ : BO_LT;
-  auto BelowThreshold =
-      SVB.evalBinOpNN(State, OpKind, Value, Threshold, SVB.getConditionType())
-          .getAs<NonLoc>();
-
-  if (BelowThreshold)
-    return State->assume(*BelowThreshold);
-
-  return {nullptr, nullptr};
-}
-
-static std::string getRegionName(const SubRegion *Region) {
-  if (std::string RegName = Region->getDescriptiveName(); !RegName.empty())
-    return RegName;
-
-  // Field regions only have descriptive names when their parent has a
-  // descriptive name; so we provide a fallback representation for them:
-  if (const auto *FR = Region->getAs<FieldRegion>()) {
-    if (StringRef Name = FR->getDecl()->getName(); !Name.empty())
-      return formatv("the field '{0}'", Name);
-    return "the unnamed field";
-  }
-
-  if (isa<AllocaRegion>(Region))
-    return "the memory returned by 'alloca'";
-
-  if (isa<SymbolicRegion>(Region) &&
-      isa<HeapSpaceRegion>(Region->getMemorySpace()))
-    return "the heap area";
-
-  if (isa<StringRegion>(Region))
-    return "the string literal";
-
-  return "the region";
-}
-
-static std::optional<int64_t> getConcreteValue(NonLoc SV) {
-  if (auto ConcreteVal = SV.getAs<nonloc::ConcreteInt>()) {
-    return ConcreteVal->getValue()->tryExtValue();
-  }
-  return std::nullopt;
-}
-
-static std::optional<int64_t> getConcreteValue(std::optional<NonLoc> SV) {
-  return SV ? getConcreteValue(*SV) : std::nullopt;
-}
-
-static Messages getPrecedesMsgs(const SubRegion *Region, NonLoc Offset) {
-  std::string RegName = getRegionName(Region), OffsetStr = "";
-
-  if (auto ConcreteOffset = getConcreteValue(Offset))
-    OffsetStr = formatv(" {0}", ConcreteOffset);
-
-  return {
-      formatv("Out of bound access to memory preceding {0}", RegName),
-      formatv("Access of {0} at negative byte offset{1}", RegName, OffsetStr)};
-}
-
-/// Try to divide `Val1` and `Val2` (in place) by `Divisor` and return true if
-/// it can be performed (`Divisor` is nonzero and there is no remainder). The
-/// values `Val1` and `Val2` may be nullopt and in that case the corresponding
-/// division is considered to be successful.
-static bool tryDividePair(std::optional<int64_t> &Val1,
-                          std::optional<int64_t> &Val2, int64_t Divisor) {
-  if (!Divisor)
-    return false;
-  const bool Val1HasRemainder = Val1 && *Val1 % Divisor;
-  const bool Val2HasRemainder = Val2 && *Val2 % Divisor;
-  if (!Val1HasRemainder && !Val2HasRemainder) {
-    if (Val1)
-      *Val1 /= Divisor;
-    if (Val2)
-      *Val2 /= Divisor;
-    return true;
-  }
-  return false;
-}
-
-static Messages getExceedsMsgs(ASTContext &ACtx, const SubRegion *Region,
-                               NonLoc Offset, NonLoc Extent, SVal Location,
-                               bool AlsoMentionUnderflow) {
-  std::string RegName = getRegionName(Region);
-  const auto *EReg = Location.getAsRegion()->getAs<ElementRegion>();
-  assert(EReg && "this checker only handles element access");
-  QualType ElemType = EReg->getElementType();
-
-  std::optional<int64_t> OffsetN = getConcreteValue(Offset);
-  std::optional<int64_t> ExtentN = getConcreteValue(Extent);
-
-  int64_t ElemSize = ACtx.getTypeSizeInChars(ElemType).getQuantity();
-
-  bool UseByteOffsets = !tryDividePair(OffsetN, ExtentN, ElemSize);
-  const char *OffsetOrIndex = UseByteOffsets ? "byte offset" : "index";
-
-  SmallString<256> Buf;
-  llvm::raw_svector_ostream Out(Buf);
-  Out << "Access of ";
-  if (!ExtentN && !UseByteOffsets)
-    Out << "'" << ElemType.getAsString() << "' element in ";
-  Out << RegName << " at ";
-  if (AlsoMentionUnderflow) {
-    Out << "a negative or overflowing " << OffsetOrIndex;
-  } else if (OffsetN) {
-    Out << OffsetOrIndex << " " << *OffsetN;
-  } else {
-    Out << "an overflowing " << OffsetOrIndex;
-  }
-  if (ExtentN) {
-    Out << ", while it holds only ";
-    if (*ExtentN != 1)
-      Out << *ExtentN;
-    else
-      Out << "a single";
-    if (UseByteOffsets)
-      Out << " byte";
-    else
-      Out << " '" << ElemType.getAsString() << "' element";
-
-    if (*ExtentN > 1)
-      Out << "s";
-  }
-
-  return {formatv("Out of bound access to memory {0} {1}",
-                  AlsoMentionUnderflow ? "around" : "after the end of",
-                  RegName),
-          std::string(Buf)};
-}
-
-static Messages getTaintMsgs(const SubRegion *Region, const char *OffsetName,
-                             bool AlsoMentionUnderflow) {
-  std::string RegName = getRegionName(Region);
-  return {formatv("Potential out of bound access to {0} with tainted {1}",
-                  RegName, OffsetName),
-          formatv("Access of {0} with a tainted {1} that may be {2}too large",
-                  RegName, OffsetName,
-                  AlsoMentionUnderflow ? "negative or " : "")};
-}
-
-const NoteTag *StateUpdateReporter::createNoteTag(CheckerContext &C) const {
-  // Don't create a note tag if we didn't assume anything:
-  if (!AssumedNonNegative && !AssumedUpperBound)
-    return nullptr;
-
-  return C.getNoteTag([*this](PathSensitiveBugReport &BR) -> std::string {
-    return getMessage(BR);
-  });
-}
-
-std::string StateUpdateReporter::getMessage(PathSensitiveBugReport &BR) const {
-  bool ShouldReportNonNegative = AssumedNonNegative;
-  if (!providesInformationAboutInteresting(ByteOffsetVal, BR)) {
-    if (AssumedUpperBound &&
-        providesInformationAboutInteresting(*AssumedUpperBound, BR)) {
-      // Even if the byte offset isn't interesting (e.g. it's a constant value),
-      // the assumption can still be interesting if it provides information
-      // about an interesting symbolic upper bound.
-      ShouldReportNonNegative = false;
-    } else {
-      // We don't have anything interesting, don't report the assumption.
-      return "";
-    }
-  }
-
-  std::optional<int64_t> OffsetN = getConcreteValue(ByteOffsetVal);
-  std::optional<int64_t> ExtentN = getConcreteValue(AssumedUpperBound);
-
-  const bool UseIndex =
-      ElementSize && tryDividePair(OffsetN, ExtentN, *ElementSize);
-
-  SmallString<256> Buf;
-  llvm::raw_svector_ostream Out(Buf);
-  Out << "Assuming ";
-  if (UseIndex) {
-    Out << "index ";
-    if (OffsetN)
-      Out << "'" << OffsetN << "' ";
-  } else if (AssumedUpperBound) {
-    Out << "byte offset ";
-    if (OffsetN)
-      Out << "'" << OffsetN << "' ";
-  } else {
-    Out << "offset ";
-  }
-
-  Out << "is";
-  if (ShouldReportNonNegative) {
-    Out << " non-negative";
-  }
-  if (AssumedUpperBound) {
-    if (ShouldReportNonNegative)
-      Out << " and";
-    Out << " less than ";
-    if (ExtentN)
-      Out << *ExtentN << ", ";
-    if (UseIndex && ElementType)
-      Out << "the number of '" << ElementType->getAsString()
-          << "' elements in ";
-    else
-      Out << "the extent of ";
-    Out << getRegionName(Reg);
-  }
-  return std::string(Out.str());
-}
-
-bool StateUpdateReporter::providesInformationAboutInteresting(
-    SymbolRef Sym, PathSensitiveBugReport &BR) {
-  if (!Sym)
-    return false;
-  for (SymbolRef PartSym : Sym->symbols()) {
-    // The interestingess mark may appear on any layer as we're stripping off
-    // the SymIntExpr, UnarySymExpr etc. layers...
-    if (BR.isInteresting(PartSym))
-      return true;
-    // ...but if both sides of the expression are symbolic, then there is no
-    // practical algorithm to produce separate constraints for the two
-    // operands (from the single combined result).
-    if (isa<SymSymExpr>(PartSym))
-      return false;
-  }
-  return false;
-}
-
-void ArrayBoundCheckerV2::performCheck(const Expr *E, CheckerContext &C) const {
-  const SVal Location = C.getSVal(E);
-
-  // The header ctype.h (from e.g. glibc) implements the isXXXXX() macros as
-  //   #define isXXXXX(arg) (LOOKUP_TABLE[arg] & BITMASK_FOR_XXXXX)
-  // and incomplete analysis of these leads to false positives. As even
-  // accurate reports would be confusing for the users, just disable reports
-  // from these macros:
-  if (isFromCtypeMacro(E, C.getASTContext()))
-    return;
-
-  ProgramStateRef State = C.getState();
-  SValBuilder &SVB = C.getSValBuilder();
-
-  const std::optional<std::pair<const SubRegion *, NonLoc>> &RawOffset =
-      computeOffset(State, SVB, Location);
-
-  if (!RawOffset)
-    return;
-
-  auto [Reg, ByteOffset] = *RawOffset;
-
-  // The state updates will be reported as a single note tag, which will be
-  // composed by this helper class.
-  StateUpdateReporter SUR(Reg, ByteOffset, E, C);
-
-  // CHECK LOWER BOUND
-  const MemSpaceRegion *Space = Reg->getMemorySpace();
-  if (!(isa<SymbolicRegion>(Reg) && isa<UnknownSpaceRegion>(Space))) {
-    // A symbolic region in unknown space represents an unknown pointer that
-    // may point into the middle of an array, so we don't look for underflows.
-    // Both conditions are significant because we want to check underflows in
-    // symbolic regions on the heap (which may be introduced by checkers like
-    // MallocChecker that call SValBuilder::getConjuredHeapSymbolVal()) and
-    // non-symbolic regions (e.g. a field subregion of a symbolic region) in
-    // unknown space.
-    auto [PrecedesLowerBound, WithinLowerBound] = compareValueToThreshold(
-        State, ByteOffset, SVB.makeZeroArrayIndex(), SVB);
-
-    if (PrecedesLowerBound) {
-      // The offset may be invalid (negative)...
-      if (!WithinLowerBound) {
-        // ...and it cannot be valid (>= 0), so report an error.
-        Messages Msgs = getPrecedesMsgs(Reg, ByteOffset);
-        reportOOB(C, PrecedesLowerBound, Msgs, ByteOffset, std::nullopt);
-        return;
-      }
-      // ...but it can be valid as well, so the checker will (optimistically)
-      // assume that it's valid and mention this in the note tag.
-      SUR.recordNonNegativeAssumption();
-    }
-
-    // Actually update the state. The "if" only fails in the extremely unlikely
-    // case when compareValueToThreshold returns {nullptr, nullptr} becasue
-    // evalBinOpNN fails to evaluate the less-than operator.
-    if (WithinLowerBound)
-      State = WithinLowerBound;
-  }
-
-  // CHECK UPPER BOUND
-  DefinedOrUnknownSVal Size = getDynamicExtent(State, Reg, SVB);
-  if (auto KnownSize = Size.getAs<NonLoc>()) {
-    // In a situation where both underflow and overflow are possible (but the
-    // index is either tainted or known to be invalid), the logic of this
-    // checker will first assume that the offset is non-negative, and then
-    // (with this additional assumption) it will detect an overflow error.
-    // In this situation the warning message should mention both possibilities.
-    bool AlsoMentionUnderflow = SUR.assumedNonNegative();
-
-    auto [WithinUpperBound, ExceedsUpperBound] =
-        compareValueToThreshold(State, ByteOffset, *KnownSize, SVB);
-
-    if (ExceedsUpperBound) {
-      // The offset may be invalid (>= Size)...
-      if (!WithinUpperBound) {
-        // ...and it cannot be within bounds, so report an error, unless we can
-        // definitely determine that this is an idiomatic `&array[size]`
-        // expression that calculates the past-the-end pointer.
-        if (isIdiomaticPastTheEndPtr(E, ExceedsUpperBound, ByteOffset,
-                                     *KnownSize, C)) {
-          C.addTransition(ExceedsUpperBound, SUR.createNoteTag(C));
-          return;
-        }
-
-        Messages Msgs =
-            getExceedsMsgs(C.getASTContext(), Reg, ByteOffset, *KnownSize,
-                           Location, AlsoMentionUnderflow);
-        reportOOB(C, ExceedsUpperBound, Msgs, ByteOffset, KnownSize);
-        return;
-      }
-      // ...and it can be valid as well...
-      if (isTainted(State, ByteOffset)) {
-        // ...but it's tainted, so report an error.
-
-        // Diagnostic detail: saying "tainted offset" is always correct, but
-        // the common case is that 'idx' is tainted in 'arr[idx]' and then it's
-        // nicer to say "tainted index".
-        const char *OffsetName = "offset";
-        if (const auto *ASE = dyn_cast<ArraySubscriptExpr>(E))
-          if (isTainted(State, ASE->getIdx(), C.getLocationContext()))
-            OffsetName = "index";
-
-        Messages Msgs = getTaintMsgs(Reg, OffsetName, AlsoMentionUnderflow);
-        reportOOB(C, ExceedsUpperBound, Msgs, ByteOffset, KnownSize,
-                  /*IsTaintBug=*/true);
-        return;
-      }
-      // ...and it isn't tainted, so the checker will (optimistically) assume
-      // that the offset is in bounds and mention this in the note tag.
-      SUR.recordUpperBoundAssumption(*KnownSize);
-    }
-
-    // Actually update the state. The "if" only fails in the extremely unlikely
-    // case when compareValueToThreshold returns {nullptr, nullptr} becasue
-    // evalBinOpNN fails to evaluate the less-than operator.
-    if (WithinUpperBound)
-      State = WithinUpperBound;
-  }
-
-  // Add a transition, reporting the state updates that we accumulated.
-  C.addTransition(State, SUR.createNoteTag(C));
-}
-
-void ArrayBoundCheckerV2::markPartsInteresting(PathSensitiveBugReport &BR,
-                                               ProgramStateRef ErrorState,
-                                               NonLoc Val, bool MarkTaint) {
-  if (SymbolRef Sym = Val.getAsSymbol()) {
-    // If the offset is a symbolic value, iterate over its "parts" with
-    // `SymExpr::symbols()` and mark each of them as interesting.
-    // For example, if the offset is `x*4 + y` then we put interestingness onto
-    // the SymSymExpr `x*4 + y`, the SymIntExpr `x*4` and the two data symbols
-    // `x` and `y`.
-    for (SymbolRef PartSym : Sym->symbols())
-      BR.markInteresting(PartSym);
-  }
-
-  if (MarkTaint) {
-    // If the issue that we're reporting depends on the taintedness of the
-    // offset, then put interestingness onto symbols that could be the origin
-    // of the taint. Note that this may find symbols that did not appear in
-    // `Sym->symbols()` (because they're only loosely connected to `Val`).
-    for (SymbolRef Sym : getTaintedSymbols(ErrorState, Val))
-      BR.markInteresting(Sym);
-  }
-}
-
-void ArrayBoundCheckerV2::reportOOB(CheckerContext &C,
-                                    ProgramStateRef ErrorState, Messages Msgs,
-                                    NonLoc Offset, std::optional<NonLoc> Extent,
-                                    bool IsTaintBug /*=false*/) const {
-
-  ExplodedNode *ErrorNode = C.generateErrorNode(ErrorState);
-  if (!ErrorNode)
-    return;
-
-  auto BR = std::make_unique<PathSensitiveBugReport>(
-      IsTaintBug ? TaintBT : BT, Msgs.Short, Msgs.Full, ErrorNode);
-
-  // FIXME: ideally we would just call trackExpressionValue() and that would
-  // "do the right thing": mark the relevant symbols as interesting, track the
-  // control dependencies and statements storing the relevant values and add
-  // helpful diagnostic pieces. However, right now trackExpressionValue() is
-  // a heap of unreliable heuristics, so it would cause several issues:
-  // - Interestingness is not applied consistently, e.g. if `array[x+10]`
-  //   causes an overflow, then `x` is not marked as interesting.
-  // - We get irrelevant diagnostic pieces, e.g. in the code
-  //   `int *p = (int*)malloc(2*sizeof(int)); p[3] = 0;`
-  //   it places a "Storing uninitialized value" note on the `malloc` call
-  //   (which is technically true, but irrelevant).
-  // If trackExpressionValue() becomes reliable, it should be applied instead
-  // of this custom markPartsInteresting().
-  markPartsInteresting(*BR, ErrorState, Offset, IsTaintBug);
-  if (Extent)
-    markPartsInteresting(*BR, ErrorState, *Extent, IsTaintBug);
-
-  C.emitReport(std::move(BR));
-}
-
-bool ArrayBoundCheckerV2::isFromCtypeMacro(const Stmt *S, ASTContext &ACtx) {
-  SourceLocation Loc = S->getBeginLoc();
-  if (!Loc.isMacroID())
-    return false;
-
-  StringRef MacroName = Lexer::getImmediateMacroName(
-      Loc, ACtx.getSourceManager(), ACtx.getLangOpts());
-
-  if (MacroName.size() < 7 || MacroName[0] != 'i' || MacroName[1] != 's')
-    return false;
-
-  return ((MacroName == "isalnum") || (MacroName == "isalpha") ||
-          (MacroName == "isblank") || (MacroName == "isdigit") ||
-          (MacroName == "isgraph") || (MacroName == "islower") ||
-          (MacroName == "isnctrl") || (MacroName == "isprint") ||
-          (MacroName == "ispunct") || (MacroName == "isspace") ||
-          (MacroName == "isupper") || (MacroName == "isxdigit"));
-}
-
-bool ArrayBoundCheckerV2::isInAddressOf(const Stmt *S, ASTContext &ACtx) {
-  ParentMapContext &ParentCtx = ACtx.getParentMapContext();
-  do {
-    const DynTypedNodeList Parents = ParentCtx.getParents(*S);
-    if (Parents.empty())
-      return false;
-    S = Parents[0].get<Stmt>();
-  } while (isa_and_nonnull<ParenExpr, ImplicitCastExpr>(S));
-  const auto *UnaryOp = dyn_cast_or_null<UnaryOperator>(S);
-  return UnaryOp && UnaryOp->getOpcode() == UO_AddrOf;
-}
-
-bool ArrayBoundCheckerV2::isIdiomaticPastTheEndPtr(const Expr *E,
-                                                   ProgramStateRef State,
-                                                   NonLoc Offset, NonLoc Limit,
-                                                   CheckerContext &C) {
-  if (isa<ArraySubscriptExpr>(E) && isInAddressOf(E, C.getASTContext())) {
-    auto [EqualsToThreshold, NotEqualToThreshold] = compareValueToThreshold(
-        State, Offset, Limit, C.getSValBuilder(), /*CheckEquality=*/true);
-    return EqualsToThreshold && !NotEqualToThreshold;
-  }
-  return false;
-}
-
-void ento::registerArrayBoundCheckerV2(CheckerManager &mgr) {
-  mgr.registerChecker<ArrayBoundCheckerV2>();
-}
-
-bool ento::shouldRegisterArrayBoundCheckerV2(const CheckerManager &mgr) {
-  return true;
-}
diff --git a/clang/lib/StaticAnalyzer/Checkers/CMakeLists.txt b/clang/lib/StaticAnalyzer/Checkers/CMakeLists.txt
index fcbe8b864b6e4..5910043440987 100644
--- a/clang/lib/StaticAnalyzer/Checkers/CMakeLists.txt
+++ b/clang/lib/StaticAnalyzer/Checkers/CMakeLists.txt
@@ -8,7 +8,6 @@ add_clang_library(clangStaticAnalyzerCheckers
   AnalysisOrderChecker.cpp
   AnalyzerStatsChecker.cpp
   ArrayBoundChecker.cpp
-  ArrayBoundCheckerV2.cpp
   BasicObjCFoundationChecks.cpp
   BitwiseShiftChecker.cpp
   BlockInCriticalSectionChecker.cpp
diff --git a/clang/lib/StaticAnalyzer/Checkers/CStringChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/CStringChecker.cpp
index 1a14f38e34f0e..39dcaf02dbe25 100644
--- a/clang/lib/StaticAnalyzer/Checkers/CStringChecker.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/CStringChecker.cpp
@@ -547,8 +547,10 @@ ProgramStateRef CStringChecker::checkInit(CheckerContext &C,
   }
   return State;
 }
-
-// FIXME: This was originally copied from ArrayBoundChecker.cpp. Refactor?
+// FIXME: The root of this logic was copied from the old checker
+// alpha.security.ArrayBound (which is removed within this commit).
+// It should be refactored to use the different, more sophisticated bounds
+// checking logic used by the new checker ``security.ArrayBound``.
 ProgramStateRef CStringChecker::CheckLocation(CheckerContext &C,
                                               ProgramStateRef state,
                                               AnyArgExpr Buffer, SVal Element,
diff --git a/clang/lib/StaticAnalyzer/Checkers/ExprInspectionChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/ExprInspectionChecker.cpp
index 5534ef86a7bef..28898bb370823 100644
--- a/clang/lib/StaticAnalyzer/Checkers/ExprInspectionChecker.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/ExprInspectionChecker.cpp
@@ -227,10 +227,11 @@ void ExprInspectionChecker::analyzerWarnIfReached(const CallExpr *CE,
 
 void ExprInspectionChecker::analyzerNumTimesReached(const CallExpr *CE,
                                                     CheckerContext &C) const {
-  ++ReachedStats[CE].NumTimesReached;
-  if (!ReachedStats[CE].ExampleNode) {
+  ReachedStat &Stat = ReachedStats[CE];
+  ++Stat.NumTimesReached;
+  if (!Stat.ExampleNode) {
     // Later, in checkEndAnalysis, we'd throw a report against it.
-    ReachedStats[CE].ExampleNode = C.generateNonFatalErrorNode();
+    Stat.ExampleNode = C.generateNonFatalErrorNode();
   }
 }
 
diff --git a/clang/lib/StaticAnalyzer/Checkers/StackAddrEscapeChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/StackAddrEscapeChecker.cpp
index f4de3b500499c..c9df15ceb3b40 100644
--- a/clang/lib/StaticAnalyzer/Checkers/StackAddrEscapeChecker.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/StackAddrEscapeChecker.cpp
@@ -54,8 +54,8 @@ class StackAddrEscapeChecker
                                   CheckerContext &C) const;
   void checkAsyncExecutedBlockCaptures(const BlockDataRegion &B,
                                        CheckerContext &C) const;
-  void EmitStackError(CheckerContext &C, const MemRegion *R,
-                      const Expr *RetE) const;
+  void EmitReturnLeakError(CheckerContext &C, const MemRegion *LeakedRegion,
+                           const Expr *RetE) const;
   bool isSemaphoreCaptured(const BlockDecl &B) const;
   static SourceRange genName(raw_ostream &os, const MemRegion *R,
                              ASTContext &Ctx);
@@ -147,9 +147,22 @@ StackAddrEscapeChecker::getCapturedStackRegions(const BlockDataRegion &B,
   return Regions;
 }
 
-void StackAddrEscapeChecker::EmitStackError(CheckerContext &C,
-                                            const MemRegion *R,
-                                            const Expr *RetE) const {
+static void EmitReturnedAsPartOfError(llvm::raw_ostream &OS, SVal ReturnedVal,
+                                      const MemRegion *LeakedRegion) {
+  if (const MemRegion *ReturnedRegion = ReturnedVal.getAsRegion()) {
+    if (isa<BlockDataRegion>(ReturnedRegion)) {
+      OS << " is captured by a returned block";
+      return;
+    }
+  }
+
+  // Generic message
+  OS << " returned to caller";
+}
+
+void StackAddrEscapeChecker::EmitReturnLeakError(CheckerContext &C,
+                                                 const MemRegion *R,
+                                                 const Expr *RetE) const {
   ExplodedNode *N = C.generateNonFatalErrorNode();
   if (!N)
     return;
@@ -157,11 +170,15 @@ void StackAddrEscapeChecker::EmitStackError(CheckerContext &C,
     BT_returnstack = std::make_unique<BugType>(
         CheckNames[CK_StackAddrEscapeChecker],
         "Return of address to stack-allocated memory");
+
   // Generate a report for this bug.
   SmallString<128> buf;
   llvm::raw_svector_ostream os(buf);
+
+  // Error message formatting
   SourceRange range = genName(os, R, C.getASTContext());
-  os << " returned to caller";
+  EmitReturnedAsPartOfError(os, C.getSVal(RetE), R);
+
   auto report =
       std::make_unique<PathSensitiveBugReport>(*BT_returnstack, os.str(), N);
   report->addRange(RetE->getSourceRange());
@@ -209,30 +226,6 @@ void StackAddrEscapeChecker::checkAsyncExecutedBlockCaptures(
   }
 }
 
-void StackAddrEscapeChecker::checkReturnedBlockCaptures(
-    const BlockDataRegion &B, CheckerContext &C) const {
-  for (const MemRegion *Region : getCapturedStackRegions(B, C)) {
-    if (isNotInCurrentFrame(Region, C))
-      continue;
-    ExplodedNode *N = C.generateNonFatalErrorNode();
-    if (!N)
-      continue;
-    if (!BT_capturedstackret)
-      BT_capturedstackret = std::make_unique<BugType>(
-          CheckNames[CK_StackAddrEscapeChecker],
-          "Address of stack-allocated memory is captured");
-    SmallString<128> Buf;
-    llvm::raw_svector_ostream Out(Buf);
-    SourceRange Range = genName(Out, Region, C.getASTContext());
-    Out << " is captured by a returned block";
-    auto Report = std::make_unique<PathSensitiveBugReport>(*BT_capturedstackret,
-                                                           Out.str(), N);
-    if (Range.isValid())
-      Report->addRange(Range);
-    C.emitReport(std::move(Report));
-  }
-}
-
 void StackAddrEscapeChecker::checkPreCall(const CallEvent &Call,
                                           CheckerContext &C) const {
   if (!ChecksEnabled[CK_StackAddrAsyncEscapeChecker])
@@ -247,45 +240,128 @@ void StackAddrEscapeChecker::checkPreCall(const CallEvent &Call,
   }
 }
 
-void StackAddrEscapeChecker::checkPreStmt(const ReturnStmt *RS,
-                                          CheckerContext &C) const {
-  if (!ChecksEnabled[CK_StackAddrEscapeChecker])
-    return;
+/// A visitor made for use with a ScanReachableSymbols scanner, used
+/// for finding stack regions within an SVal that live on the current
+/// stack frame of the given checker context. This visitor excludes
+/// NonParamVarRegion that data is bound to in a BlockDataRegion's
+/// bindings, since these are likely uninteresting, e.g., in case a
+/// temporary is constructed on the stack, but it captures values
+/// that would leak.
+class FindStackRegionsSymbolVisitor final : public SymbolVisitor {
+  CheckerContext &Ctxt;
+  const StackFrameContext *PoppedStackFrame;
+  SmallVectorImpl<const MemRegion *> &EscapingStackRegions;
 
-  const Expr *RetE = RS->getRetValue();
-  if (!RetE)
-    return;
-  RetE = RetE->IgnoreParens();
+public:
+  explicit FindStackRegionsSymbolVisitor(
+      CheckerContext &Ctxt,
+      SmallVectorImpl<const MemRegion *> &StorageForStackRegions)
+      : Ctxt(Ctxt), PoppedStackFrame(Ctxt.getStackFrame()),
+        EscapingStackRegions(StorageForStackRegions) {}
 
-  SVal V = C.getSVal(RetE);
-  const MemRegion *R = V.getAsRegion();
-  if (!R)
-    return;
+  bool VisitSymbol(SymbolRef sym) override { return true; }
 
-  if (const BlockDataRegion *B = dyn_cast<BlockDataRegion>(R))
-    checkReturnedBlockCaptures(*B, C);
+  bool VisitMemRegion(const MemRegion *MR) override {
+    SaveIfEscapes(MR);
 
-  if (!isa<StackSpaceRegion>(R->getMemorySpace()) || isNotInCurrentFrame(R, C))
-    return;
+    if (const BlockDataRegion *BDR = MR->getAs<BlockDataRegion>())
+      return VisitBlockDataRegionCaptures(BDR);
+
+    return true;
+  }
+
+private:
+  void SaveIfEscapes(const MemRegion *MR) {
+    const StackSpaceRegion *SSR =
+        MR->getMemorySpace()->getAs<StackSpaceRegion>();
+    if (SSR && SSR->getStackFrame() == PoppedStackFrame)
+      EscapingStackRegions.push_back(MR);
+  }
+
+  bool VisitBlockDataRegionCaptures(const BlockDataRegion *BDR) {
+    for (auto Var : BDR->referenced_vars()) {
+      SVal Val = Ctxt.getState()->getSVal(Var.getCapturedRegion());
+      const MemRegion *Region = Val.getAsRegion();
+      if (Region) {
+        SaveIfEscapes(Region);
+        VisitMemRegion(Region);
+      }
+    }
+
+    return false;
+  }
+};
+
+/// Given some memory regions that are flagged by FindStackRegionsSymbolVisitor,
+/// this function filters out memory regions that are being returned that are
+/// likely not true leaks:
+/// 1. If returning a block data region that has stack memory space
+/// 2. If returning a constructed object that has stack memory space
+static SmallVector<const MemRegion *> FilterReturnExpressionLeaks(
+    const SmallVectorImpl<const MemRegion *> &MaybeEscaped, CheckerContext &C,
+    const Expr *RetE, SVal &RetVal) {
+
+  SmallVector<const MemRegion *> WillEscape;
+
+  const MemRegion *RetRegion = RetVal.getAsRegion();
 
   // Returning a record by value is fine. (In this case, the returned
   // expression will be a copy-constructor, possibly wrapped in an
   // ExprWithCleanups node.)
   if (const ExprWithCleanups *Cleanup = dyn_cast<ExprWithCleanups>(RetE))
     RetE = Cleanup->getSubExpr();
-  if (isa<CXXConstructExpr>(RetE) && RetE->getType()->isRecordType())
-    return;
+  bool IsConstructExpr =
+      isa<CXXConstructExpr>(RetE) && RetE->getType()->isRecordType();
 
   // The CK_CopyAndAutoreleaseBlockObject cast causes the block to be copied
   // so the stack address is not escaping here.
+  bool IsCopyAndAutoreleaseBlockObj = false;
   if (const auto *ICE = dyn_cast<ImplicitCastExpr>(RetE)) {
-    if (isa<BlockDataRegion>(R) &&
-        ICE->getCastKind() == CK_CopyAndAutoreleaseBlockObject) {
-      return;
-    }
+    IsCopyAndAutoreleaseBlockObj =
+        isa_and_nonnull<BlockDataRegion>(RetRegion) &&
+        ICE->getCastKind() == CK_CopyAndAutoreleaseBlockObject;
+  }
+
+  for (const MemRegion *MR : MaybeEscaped) {
+    if (RetRegion == MR && (IsCopyAndAutoreleaseBlockObj || IsConstructExpr))
+      continue;
+
+    WillEscape.push_back(MR);
   }
 
-  EmitStackError(C, R, RetE);
+  return WillEscape;
+}
+
+/// For use in finding regions that live on the checker context's current
+/// stack frame, deep in the SVal representing the return value.
+static SmallVector<const MemRegion *>
+FindEscapingStackRegions(CheckerContext &C, const Expr *RetE, SVal RetVal) {
+  SmallVector<const MemRegion *> FoundStackRegions;
+
+  FindStackRegionsSymbolVisitor Finder(C, FoundStackRegions);
+  ScanReachableSymbols Scanner(C.getState(), Finder);
+  Scanner.scan(RetVal);
+
+  return FilterReturnExpressionLeaks(FoundStackRegions, C, RetE, RetVal);
+}
+
+void StackAddrEscapeChecker::checkPreStmt(const ReturnStmt *RS,
+                                          CheckerContext &C) const {
+  if (!ChecksEnabled[CK_StackAddrEscapeChecker])
+    return;
+
+  const Expr *RetE = RS->getRetValue();
+  if (!RetE)
+    return;
+  RetE = RetE->IgnoreParens();
+
+  SVal V = C.getSVal(RetE);
+
+  SmallVector<const MemRegion *> EscapedStackRegions =
+      FindEscapingStackRegions(C, RetE, V);
+
+  for (const MemRegion *ER : EscapedStackRegions)
+    EmitReturnLeakError(C, ER, RetE);
 }
 
 static const MemSpaceRegion *getStackOrGlobalSpaceRegion(const MemRegion *R) {
diff --git a/clang/lib/StaticAnalyzer/Checkers/WebKit/ASTUtils.cpp b/clang/lib/StaticAnalyzer/Checkers/WebKit/ASTUtils.cpp
index abf5d3ec193a4..a12853d01819f 100644
--- a/clang/lib/StaticAnalyzer/Checkers/WebKit/ASTUtils.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/WebKit/ASTUtils.cpp
@@ -154,19 +154,20 @@ bool isConstOwnerPtrMemberExpr(const clang::Expr *E) {
   if (auto *MCE = dyn_cast<CXXMemberCallExpr>(E)) {
     if (auto *Callee = MCE->getDirectCallee()) {
       auto Name = safeGetName(Callee);
-      if (Name == "get" || Name == "ptr") {
-        auto *ThisArg = MCE->getImplicitObjectArgument();
-        E = ThisArg;
-      }
+      if (Name == "get" || Name == "ptr")
+        E = MCE->getImplicitObjectArgument();
+      if (isa<CXXConversionDecl>(Callee))
+        E = MCE->getImplicitObjectArgument();
     }
   } else if (auto *OCE = dyn_cast<CXXOperatorCallExpr>(E)) {
     if (OCE->getOperator() == OO_Star && OCE->getNumArgs() == 1)
       E = OCE->getArg(0);
   }
-  auto *ME = dyn_cast<MemberExpr>(E);
-  if (!ME)
-    return false;
-  auto *D = ME->getMemberDecl();
+  const ValueDecl *D = nullptr;
+  if (auto *ME = dyn_cast<MemberExpr>(E))
+    D = ME->getMemberDecl();
+  else if (auto *IVR = dyn_cast<ObjCIvarRefExpr>(E))
+    D = IVR->getDecl();
   if (!D)
     return false;
   auto T = D->getType();
diff --git a/clang/lib/StaticAnalyzer/Checkers/WebKit/PtrTypesSemantics.cpp b/clang/lib/StaticAnalyzer/Checkers/WebKit/PtrTypesSemantics.cpp
index 5487fea1b956c..d40b4b4dbb560 100644
--- a/clang/lib/StaticAnalyzer/Checkers/WebKit/PtrTypesSemantics.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/WebKit/PtrTypesSemantics.cpp
@@ -636,6 +636,11 @@ class TrivialFunctionAnalysisVisitor
     return true;
   }
 
+  bool VisitImplicitValueInitExpr(const ImplicitValueInitExpr *IVIE) {
+    // An implicit value initialization is trvial.
+    return true;
+  }
+
 private:
   CacheTy &Cache;
   CacheTy RecursiveFn;
diff --git a/clang/lib/StaticAnalyzer/Checkers/WebKit/RawPtrRefMemberChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/WebKit/RawPtrRefMemberChecker.cpp
index d7e5ebee9a9b4..79f88553feb95 100644
--- a/clang/lib/StaticAnalyzer/Checkers/WebKit/RawPtrRefMemberChecker.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/WebKit/RawPtrRefMemberChecker.cpp
@@ -48,7 +48,7 @@ class RawPtrRefMemberChecker
     // The calls to checkAST* from AnalysisConsumer don't
     // visit template instantiations or lambda classes. We
     // want to visit those, so we make our own RecursiveASTVisitor.
-    struct LocalVisitor : DynamicRecursiveASTVisitor {
+    struct LocalVisitor : ConstDynamicRecursiveASTVisitor {
       const RawPtrRefMemberChecker *Checker;
       explicit LocalVisitor(const RawPtrRefMemberChecker *Checker)
           : Checker(Checker) {
@@ -57,14 +57,14 @@ class RawPtrRefMemberChecker
         ShouldVisitImplicitCode = false;
       }
 
-      bool VisitRecordDecl(RecordDecl *RD) override {
+      bool VisitRecordDecl(const RecordDecl *RD) override {
         Checker->visitRecordDecl(RD);
         return true;
       }
     };
 
     LocalVisitor visitor(this);
-    visitor.TraverseDecl(const_cast<TranslationUnitDecl *>(TUD));
+    visitor.TraverseDecl(TUD);
   }
 
   void visitRecordDecl(const RecordDecl *RD) const {
diff --git a/clang/lib/StaticAnalyzer/Checkers/WebKit/UncountedLambdaCapturesChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/WebKit/UncountedLambdaCapturesChecker.cpp
index a57499d52acd0..a56f48c83c660 100644
--- a/clang/lib/StaticAnalyzer/Checkers/WebKit/UncountedLambdaCapturesChecker.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/WebKit/UncountedLambdaCapturesChecker.cpp
@@ -109,8 +109,8 @@ class UncountedLambdaCapturesChecker
       bool VisitCallExpr(CallExpr *CE) override {
         checkCalleeLambda(CE);
         if (auto *Callee = CE->getDirectCallee()) {
+          unsigned ArgIndex = isa<CXXOperatorCallExpr>(CE);
           bool TreatAllArgsAsNoEscape = shouldTreatAllArgAsNoEscape(Callee);
-          unsigned ArgIndex = 0;
           for (auto *Param : Callee->parameters()) {
             if (ArgIndex >= CE->getNumArgs())
               return true;
diff --git a/clang/lib/StaticAnalyzer/Core/BugReporter.cpp b/clang/lib/StaticAnalyzer/Core/BugReporter.cpp
index 2904eab0097dc..13677ed341d0c 100644
--- a/clang/lib/StaticAnalyzer/Core/BugReporter.cpp
+++ b/clang/lib/StaticAnalyzer/Core/BugReporter.cpp
@@ -62,6 +62,7 @@
 #include "llvm/Support/Compiler.h"
 #include "llvm/Support/ErrorHandling.h"
 #include "llvm/Support/MemoryBuffer.h"
+#include "llvm/Support/TimeProfiler.h"
 #include "llvm/Support/raw_ostream.h"
 #include <algorithm>
 #include <cassert>
@@ -287,6 +288,33 @@ class PathDiagnosticBuilder : public BugReporterContext {
   const PathSensitiveBugReport *getBugReport() const { return R; }
 };
 
+std::string timeTraceName(const BugReportEquivClass &EQ) {
+  if (!llvm::timeTraceProfilerEnabled())
+    return "";
+  const auto &BugReports = EQ.getReports();
+  if (BugReports.empty())
+    return "Empty Equivalence Class";
+  const BugReport *R = BugReports.front().get();
+  const auto &BT = R->getBugType();
+  return ("Flushing EQC " + BT.getDescription()).str();
+}
+
+llvm::TimeTraceMetadata timeTraceMetadata(const BugReportEquivClass &EQ,
+                                          const SourceManager &SM) {
+  // Must be called only when constructing non-bogus TimeTraceScope
+  assert(llvm::timeTraceProfilerEnabled());
+
+  const auto &BugReports = EQ.getReports();
+  if (BugReports.empty())
+    return {};
+  const BugReport *R = BugReports.front().get();
+  const auto &BT = R->getBugType();
+  auto Loc = R->getLocation().asLocation();
+  std::string File = SM.getFilename(Loc).str();
+  return {BT.getCheckerName().str(), std::move(File),
+          static_cast<int>(Loc.getLineNumber())};
+}
+
 } // namespace
 
 //===----------------------------------------------------------------------===//
@@ -2892,6 +2920,7 @@ std::optional<PathDiagnosticBuilder> PathDiagnosticBuilder::findValidReport(
 
     if (R->isValid()) {
       if (Reporter.getAnalyzerOptions().ShouldCrosscheckWithZ3) {
+        llvm::TimeTraceScope TCS{"Crosscheck with Z3"};
         // If crosscheck is enabled, remove all visitors, add the refutation
         // visitor and check again
         R->clearVisitors();
@@ -3119,7 +3148,10 @@ BugReport *PathSensitiveBugReporter::findReportInEquivalenceClass(
   return exampleReport;
 }
 
-void BugReporter::FlushReport(BugReportEquivClass& EQ) {
+void BugReporter::FlushReport(BugReportEquivClass &EQ) {
+  llvm::TimeTraceScope TCS{timeTraceName(EQ), [&]() {
+                             return timeTraceMetadata(EQ, getSourceManager());
+                           }};
   SmallVector<BugReport*, 10> bugReports;
   BugReport *report = findReportInEquivalenceClass(EQ, bugReports);
   if (!report)
diff --git a/clang/lib/StaticAnalyzer/Core/BugReporterVisitors.cpp b/clang/lib/StaticAnalyzer/Core/BugReporterVisitors.cpp
index a9b4dbb39b5bd..a6142063895db 100644
--- a/clang/lib/StaticAnalyzer/Core/BugReporterVisitors.cpp
+++ b/clang/lib/StaticAnalyzer/Core/BugReporterVisitors.cpp
@@ -1198,7 +1198,10 @@ static bool isInitializationOfVar(const ExplodedNode *N, const VarRegion *VR) {
     // If we ever directly evaluate global DeclStmts, this assertion will be
     // invalid, but this still seems preferable to silently accepting an
     // initialization that may be for a path-sensitive variable.
-    assert(VR->getDecl()->isStaticLocal() && "non-static stackless VarRegion");
+    [[maybe_unused]] bool IsLocalStaticOrLocalExtern =
+        VR->getDecl()->isStaticLocal() || VR->getDecl()->isLocalExternDecl();
+    assert(IsLocalStaticOrLocalExtern &&
+           "Declared a variable on the stack without Stack memspace?");
     return true;
   }
 
diff --git a/clang/lib/StaticAnalyzer/Core/BugSuppression.cpp b/clang/lib/StaticAnalyzer/Core/BugSuppression.cpp
index 05c99c4a844e9..5b5f9df9cb0dc 100644
--- a/clang/lib/StaticAnalyzer/Core/BugSuppression.cpp
+++ b/clang/lib/StaticAnalyzer/Core/BugSuppression.cpp
@@ -9,6 +9,8 @@
 #include "clang/StaticAnalyzer/Core/BugReporter/BugSuppression.h"
 #include "clang/AST/DynamicRecursiveASTVisitor.h"
 #include "clang/StaticAnalyzer/Core/BugReporter/BugReporter.h"
+#include "llvm/Support/FormatVariadic.h"
+#include "llvm/Support/TimeProfiler.h"
 
 using namespace clang;
 using namespace ento;
@@ -119,6 +121,29 @@ class CacheInitializer : public DynamicRecursiveASTVisitor {
   Ranges &Result;
 };
 
+std::string timeScopeName(const Decl *DeclWithIssue) {
+  if (!llvm::timeTraceProfilerEnabled())
+    return "";
+  return llvm::formatv(
+             "BugSuppression::isSuppressed init suppressions cache for {0}",
+             DeclWithIssue->getDeclKindName())
+      .str();
+}
+
+llvm::TimeTraceMetadata getDeclTimeTraceMetadata(const Decl *DeclWithIssue) {
+  assert(DeclWithIssue);
+  assert(llvm::timeTraceProfilerEnabled());
+  std::string Name = "<noname>";
+  if (const auto *ND = dyn_cast<NamedDecl>(DeclWithIssue)) {
+    Name = ND->getNameAsString();
+  }
+  const auto &SM = DeclWithIssue->getASTContext().getSourceManager();
+  auto Line = SM.getPresumedLineNumber(DeclWithIssue->getBeginLoc());
+  auto Fname = SM.getFilename(DeclWithIssue->getBeginLoc());
+  return llvm::TimeTraceMetadata{std::move(Name), Fname.str(),
+                                 static_cast<int>(Line)};
+}
+
 } // end anonymous namespace
 
 // TODO: Introduce stable IDs for checkers and check for those here
@@ -177,6 +202,9 @@ bool BugSuppression::isSuppressed(const PathDiagnosticLocation &Location,
       std::make_pair(DeclWithIssue, CachedRanges{}));
   Ranges &SuppressionRanges = InsertionResult.first->second;
   if (InsertionResult.second) {
+    llvm::TimeTraceScope TimeScope(
+        timeScopeName(DeclWithIssue),
+        [DeclWithIssue]() { return getDeclTimeTraceMetadata(DeclWithIssue); });
     // We haven't checked this declaration for suppressions yet!
     CacheInitializer::initialize(DeclWithIssue, SuppressionRanges);
   }
diff --git a/clang/lib/StaticAnalyzer/Core/CheckerContext.cpp b/clang/lib/StaticAnalyzer/Core/CheckerContext.cpp
index 96464b30c078f..d0145293fa3e5 100644
--- a/clang/lib/StaticAnalyzer/Core/CheckerContext.cpp
+++ b/clang/lib/StaticAnalyzer/Core/CheckerContext.cpp
@@ -55,7 +55,7 @@ bool CheckerContext::isCLibraryFunction(const FunctionDecl *FD,
   if (BId != 0) {
     if (Name.empty())
       return true;
-    StringRef BName = FD->getASTContext().BuiltinInfo.getName(BId);
+    std::string BName = FD->getASTContext().BuiltinInfo.getName(BId);
     size_t start = BName.find(Name);
     if (start != StringRef::npos) {
       // Accept exact match.
diff --git a/clang/lib/StaticAnalyzer/Core/CheckerManager.cpp b/clang/lib/StaticAnalyzer/Core/CheckerManager.cpp
index 9f7a0fcc2edb3..53929d370e2fe 100644
--- a/clang/lib/StaticAnalyzer/Core/CheckerManager.cpp
+++ b/clang/lib/StaticAnalyzer/Core/CheckerManager.cpp
@@ -27,6 +27,7 @@
 #include "llvm/Support/Casting.h"
 #include "llvm/Support/ErrorHandling.h"
 #include "llvm/Support/FormatVariadic.h"
+#include "llvm/Support/TimeProfiler.h"
 #include <cassert>
 #include <optional>
 #include <vector>
@@ -134,6 +135,14 @@ static void expandGraphWithCheckers(CHECK_CTX checkCtx,
 
 namespace {
 
+std::string checkerScopeName(StringRef Name, const CheckerBase *Checker) {
+  if (!llvm::timeTraceProfilerEnabled())
+    return "";
+  StringRef CheckerName =
+      Checker ? Checker->getCheckerName().getName() : "<unknown>";
+  return (Name + ":" + CheckerName).str();
+}
+
   struct CheckStmtContext {
     using CheckersTy = SmallVectorImpl<CheckerManager::CheckStmtFunc>;
 
@@ -153,6 +162,7 @@ namespace {
 
     void runChecker(CheckerManager::CheckStmtFunc checkFn,
                     NodeBuilder &Bldr, ExplodedNode *Pred) {
+      llvm::TimeTraceScope TimeScope(checkerScopeName("Stmt", checkFn.Checker));
       // FIXME: Remove respondsToCallback from CheckerContext;
       ProgramPoint::Kind K =  IsPreVisit ? ProgramPoint::PreStmtKind :
                                            ProgramPoint::PostStmtKind;
@@ -174,6 +184,9 @@ void CheckerManager::runCheckersForStmt(bool isPreVisit,
                                         bool WasInlined) {
   CheckStmtContext C(isPreVisit, getCachedStmtCheckersFor(S, isPreVisit),
                      S, Eng, WasInlined);
+  llvm::TimeTraceScope TimeScope(
+      isPreVisit ? "CheckerManager::runCheckersForStmt (Pre)"
+                 : "CheckerManager::runCheckersForStmt (Post)");
   expandGraphWithCheckers(C, Dst, Src);
 }
 
@@ -200,6 +213,8 @@ namespace {
 
     void runChecker(CheckerManager::CheckObjCMessageFunc checkFn,
                     NodeBuilder &Bldr, ExplodedNode *Pred) {
+      llvm::TimeTraceScope TimeScope(
+          checkerScopeName("ObjCMsg", checkFn.Checker));
       bool IsPreVisit;
 
       switch (Kind) {
@@ -230,6 +245,7 @@ void CheckerManager::runCheckersForObjCMessage(ObjCMessageVisitKind visitKind,
                                                bool WasInlined) {
   const auto &checkers = getObjCMessageCheckers(visitKind);
   CheckObjCMessageContext C(visitKind, checkers, msg, Eng, WasInlined);
+  llvm::TimeTraceScope TimeScope("CheckerManager::runCheckersForObjCMessage");
   expandGraphWithCheckers(C, Dst, Src);
 }
 
@@ -270,6 +286,7 @@ namespace {
 
     void runChecker(CheckerManager::CheckCallFunc checkFn,
                     NodeBuilder &Bldr, ExplodedNode *Pred) {
+      llvm::TimeTraceScope TimeScope(checkerScopeName("Call", checkFn.Checker));
       const ProgramPoint &L = Call.getProgramPoint(IsPreVisit,checkFn.Checker);
       CheckerContext C(Bldr, Eng, Pred, L, WasInlined);
 
@@ -290,6 +307,9 @@ void CheckerManager::runCheckersForCallEvent(bool isPreVisit,
                      isPreVisit ? PreCallCheckers
                                 : PostCallCheckers,
                      Call, Eng, WasInlined);
+  llvm::TimeTraceScope TimeScope(
+      isPreVisit ? "CheckerManager::runCheckersForCallEvent (Pre)"
+                 : "CheckerManager::runCheckersForCallEvent (Post)");
   expandGraphWithCheckers(C, Dst, Src);
 }
 
@@ -317,6 +337,7 @@ namespace {
 
     void runChecker(CheckerManager::CheckLocationFunc checkFn,
                     NodeBuilder &Bldr, ExplodedNode *Pred) {
+      llvm::TimeTraceScope TimeScope(checkerScopeName("Loc", checkFn.Checker));
       ProgramPoint::Kind K =  IsLoad ? ProgramPoint::PreLoadKind :
                                        ProgramPoint::PreStoreKind;
       const ProgramPoint &L =
@@ -340,6 +361,9 @@ void CheckerManager::runCheckersForLocation(ExplodedNodeSet &Dst,
                                             ExprEngine &Eng) {
   CheckLocationContext C(LocationCheckers, location, isLoad, NodeEx,
                          BoundEx, Eng);
+  llvm::TimeTraceScope TimeScope(
+      isLoad ? "CheckerManager::runCheckersForLocation (Load)"
+             : "CheckerManager::runCheckersForLocation (Store)");
   expandGraphWithCheckers(C, Dst, Src);
 }
 
@@ -365,6 +389,7 @@ namespace {
 
     void runChecker(CheckerManager::CheckBindFunc checkFn,
                     NodeBuilder &Bldr, ExplodedNode *Pred) {
+      llvm::TimeTraceScope TimeScope(checkerScopeName("Bind", checkFn.Checker));
       const ProgramPoint &L = PP.withTag(checkFn.Checker);
       CheckerContext C(Bldr, Eng, Pred, L);
 
@@ -372,6 +397,14 @@ namespace {
     }
   };
 
+  llvm::TimeTraceMetadata getTimeTraceBindMetadata(SVal Val) {
+    assert(llvm::timeTraceProfilerEnabled());
+    std::string Name;
+    llvm::raw_string_ostream OS(Name);
+    Val.dumpToStream(OS);
+    return llvm::TimeTraceMetadata{OS.str(), ""};
+  }
+
 } // namespace
 
 /// Run checkers for binding of a value to a location.
@@ -381,6 +414,9 @@ void CheckerManager::runCheckersForBind(ExplodedNodeSet &Dst,
                                         const Stmt *S, ExprEngine &Eng,
                                         const ProgramPoint &PP) {
   CheckBindContext C(BindCheckers, location, val, S, Eng, PP);
+  llvm::TimeTraceScope TimeScope{
+      "CheckerManager::runCheckersForBind",
+      [&val]() { return getTimeTraceBindMetadata(val); }};
   expandGraphWithCheckers(C, Dst, Src);
 }
 
@@ -409,6 +445,7 @@ struct CheckBeginFunctionContext {
 
   void runChecker(CheckerManager::CheckBeginFunctionFunc checkFn,
                   NodeBuilder &Bldr, ExplodedNode *Pred) {
+    llvm::TimeTraceScope TimeScope(checkerScopeName("Begin", checkFn.Checker));
     const ProgramPoint &L = PP.withTag(checkFn.Checker);
     CheckerContext C(Bldr, Eng, Pred, L);
 
@@ -425,6 +462,7 @@ void CheckerManager::runCheckersForBeginFunction(ExplodedNodeSet &Dst,
   ExplodedNodeSet Src;
   Src.insert(Pred);
   CheckBeginFunctionContext C(BeginFunctionCheckers, Eng, L);
+  llvm::TimeTraceScope TimeScope("CheckerManager::runCheckersForBeginFunction");
   expandGraphWithCheckers(C, Dst, Src);
 }
 
@@ -444,6 +482,7 @@ void CheckerManager::runCheckersForEndFunction(NodeBuilderContext &BC,
     const ProgramPoint &L =
         FunctionExitPoint(RS, Pred->getLocationContext(), checkFn.Checker);
     CheckerContext C(Bldr, Eng, Pred, L);
+    llvm::TimeTraceScope TimeScope(checkerScopeName("End", checkFn.Checker));
     checkFn(RS, C);
   }
 }
@@ -466,6 +505,8 @@ namespace {
 
     void runChecker(CheckerManager::CheckBranchConditionFunc checkFn,
                     NodeBuilder &Bldr, ExplodedNode *Pred) {
+      llvm::TimeTraceScope TimeScope(
+          checkerScopeName("BranchCond", checkFn.Checker));
       ProgramPoint L = PostCondition(Condition, Pred->getLocationContext(),
                                      checkFn.Checker);
       CheckerContext C(Bldr, Eng, Pred, L);
@@ -483,6 +524,8 @@ void CheckerManager::runCheckersForBranchCondition(const Stmt *Condition,
   ExplodedNodeSet Src;
   Src.insert(Pred);
   CheckBranchConditionContext C(BranchConditionCheckers, Condition, Eng);
+  llvm::TimeTraceScope TimeScope(
+      "CheckerManager::runCheckersForBranchCondition");
   expandGraphWithCheckers(C, Dst, Src);
 }
 
@@ -506,6 +549,8 @@ namespace {
 
     void runChecker(CheckerManager::CheckNewAllocatorFunc checkFn,
                     NodeBuilder &Bldr, ExplodedNode *Pred) {
+      llvm::TimeTraceScope TimeScope(
+          checkerScopeName("Allocator", checkFn.Checker));
       ProgramPoint L =
           PostAllocatorCall(Call.getOriginExpr(), Pred->getLocationContext());
       CheckerContext C(Bldr, Eng, Pred, L, WasInlined);
@@ -524,6 +569,7 @@ void CheckerManager::runCheckersForNewAllocator(const CXXAllocatorCall &Call,
   ExplodedNodeSet Src;
   Src.insert(Pred);
   CheckNewAllocatorContext C(NewAllocatorCheckers, Call, WasInlined, Eng);
+  llvm::TimeTraceScope TimeScope("CheckerManager::runCheckersForNewAllocator");
   expandGraphWithCheckers(C, Dst, Src);
 }
 
@@ -555,6 +601,8 @@ namespace {
 
     void runChecker(CheckerManager::CheckDeadSymbolsFunc checkFn,
                     NodeBuilder &Bldr, ExplodedNode *Pred) {
+      llvm::TimeTraceScope TimeScope(
+          checkerScopeName("DeadSymbols", checkFn.Checker));
       const ProgramPoint &L = ProgramPoint::getProgramPoint(S, ProgarmPointKind,
                                 Pred->getLocationContext(), checkFn.Checker);
       CheckerContext C(Bldr, Eng, Pred, L);
@@ -576,6 +624,7 @@ void CheckerManager::runCheckersForDeadSymbols(ExplodedNodeSet &Dst,
                                                ExprEngine &Eng,
                                                ProgramPoint::Kind K) {
   CheckDeadSymbolsContext C(DeadSymbolsCheckers, SymReaper, S, Eng, K);
+  llvm::TimeTraceScope TimeScope("CheckerManager::runCheckersForDeadSymbols");
   expandGraphWithCheckers(C, Dst, Src);
 }
 
diff --git a/clang/lib/StaticAnalyzer/Core/CoreEngine.cpp b/clang/lib/StaticAnalyzer/Core/CoreEngine.cpp
index 775a22e18c619..bf1fd7c2356dc 100644
--- a/clang/lib/StaticAnalyzer/Core/CoreEngine.cpp
+++ b/clang/lib/StaticAnalyzer/Core/CoreEngine.cpp
@@ -30,6 +30,8 @@
 #include "llvm/ADT/Statistic.h"
 #include "llvm/Support/Casting.h"
 #include "llvm/Support/ErrorHandling.h"
+#include "llvm/Support/FormatVariadic.h"
+#include "llvm/Support/TimeProfiler.h"
 #include <algorithm>
 #include <cassert>
 #include <memory>
@@ -179,8 +181,41 @@ bool CoreEngine::ExecuteWorkList(const LocationContext *L, unsigned MaxSteps,
   return WList->hasWork();
 }
 
-void CoreEngine::dispatchWorkItem(ExplodedNode* Pred, ProgramPoint Loc,
-                                  const WorkListUnit& WU) {
+static std::string timeTraceScopeName(const ProgramPoint &Loc) {
+  if (llvm::timeTraceProfilerEnabled()) {
+    return llvm::formatv("Loc {0}",
+                         ProgramPoint::getProgramPointKindName(Loc.getKind()))
+        .str();
+  }
+  return "";
+}
+
+static llvm::TimeTraceMetadata timeTraceMetadata(const ExplodedNode *Pred,
+                                                 const ProgramPoint &Loc) {
+  // If time-trace profiler is not enabled, this function is never called.
+  assert(llvm::timeTraceProfilerEnabled());
+  std::string Detail = "";
+  if (const auto SP = Loc.getAs<StmtPoint>()) {
+    if (const Stmt *S = SP->getStmt())
+      Detail = S->getStmtClassName();
+  }
+  auto SLoc = Loc.getSourceLocation();
+  if (!SLoc)
+    return llvm::TimeTraceMetadata{Detail, ""};
+  const auto &SM = Pred->getLocationContext()
+                       ->getAnalysisDeclContext()
+                       ->getASTContext()
+                       .getSourceManager();
+  auto Line = SM.getPresumedLineNumber(*SLoc);
+  auto Fname = SM.getFilename(*SLoc);
+  return llvm::TimeTraceMetadata{Detail, Fname.str(), static_cast<int>(Line)};
+}
+
+void CoreEngine::dispatchWorkItem(ExplodedNode *Pred, ProgramPoint Loc,
+                                  const WorkListUnit &WU) {
+  llvm::TimeTraceScope tcs{timeTraceScopeName(Loc), [Loc, Pred]() {
+                             return timeTraceMetadata(Pred, Loc);
+                           }};
   // Dispatch on the location type.
   switch (Loc.getKind()) {
     case ProgramPoint::BlockEdgeKind:
diff --git a/clang/lib/StaticAnalyzer/Core/ExplodedGraph.cpp b/clang/lib/StaticAnalyzer/Core/ExplodedGraph.cpp
index 55bcb6e220e1e..7b2cccce93cfe 100644
--- a/clang/lib/StaticAnalyzer/Core/ExplodedGraph.cpp
+++ b/clang/lib/StaticAnalyzer/Core/ExplodedGraph.cpp
@@ -488,15 +488,17 @@ ExplodedGraph::trim(ArrayRef<const NodeTy *> Sinks,
   while (!WL2.empty()) {
     const ExplodedNode *N = WL2.pop_back_val();
 
+    auto [Place, Inserted] = Pass2.try_emplace(N);
+
     // Skip this node if we have already processed it.
-    if (Pass2.contains(N))
+    if (!Inserted)
       continue;
 
     // Create the corresponding node in the new graph and record the mapping
     // from the old node to the new node.
     ExplodedNode *NewN = G->createUncachedNode(N->getLocation(), N->State,
                                                N->getID(), N->isSink());
-    Pass2[N] = NewN;
+    Place->second = NewN;
 
     // Also record the reverse mapping from the new node to the old node.
     if (InverseMap) (*InverseMap)[NewN] = N;
diff --git a/clang/lib/StaticAnalyzer/Core/ExprEngine.cpp b/clang/lib/StaticAnalyzer/Core/ExprEngine.cpp
index 140c77790496d..e3ec7c57571c8 100644
--- a/clang/lib/StaticAnalyzer/Core/ExprEngine.cpp
+++ b/clang/lib/StaticAnalyzer/Core/ExprEngine.cpp
@@ -74,6 +74,7 @@
 #include "llvm/Support/ErrorHandling.h"
 #include "llvm/Support/GraphWriter.h"
 #include "llvm/Support/SaveAndRestore.h"
+#include "llvm/Support/TimeProfiler.h"
 #include "llvm/Support/raw_ostream.h"
 #include <cassert>
 #include <cstdint>
@@ -1031,6 +1032,7 @@ void ExprEngine::removeDead(ExplodedNode *Pred, ExplodedNodeSet &Out,
                             const LocationContext *LC,
                             const Stmt *DiagnosticStmt,
                             ProgramPoint::Kind K) {
+  llvm::TimeTraceScope TimeScope("ExprEngine::removeDead");
   assert((K == ProgramPoint::PreStmtPurgeDeadSymbolsKind ||
           ReferenceStmt == nullptr || isa<ReturnStmt>(ReferenceStmt))
           && "PostStmt is not generally supported by the SymbolReaper yet");
@@ -1743,6 +1745,7 @@ void ExprEngine::Visit(const Stmt *S, ExplodedNode *Pred,
     case Stmt::DependentCoawaitExprClass:
     case Stmt::CoreturnStmtClass:
     case Stmt::CoyieldExprClass:
+    case Stmt::ResolvedUnexpandedPackExprClass:
     case Stmt::SEHTryStmtClass:
     case Stmt::SEHExceptStmtClass:
     case Stmt::SEHLeaveStmtClass:
@@ -1835,6 +1838,7 @@ void ExprEngine::Visit(const Stmt *S, ExplodedNode *Pred,
     case Stmt::OpenACCShutdownConstructClass:
     case Stmt::OpenACCSetConstructClass:
     case Stmt::OpenACCUpdateConstructClass:
+    case Stmt::OpenACCAtomicConstructClass:
     case Stmt::OMPUnrollDirectiveClass:
     case Stmt::OMPMetaDirectiveClass:
     case Stmt::HLSLOutArgExprClass: {
diff --git a/clang/lib/StaticAnalyzer/Core/ExprEngineC.cpp b/clang/lib/StaticAnalyzer/Core/ExprEngineC.cpp
index 7a900780384a9..3a983421358c7 100644
--- a/clang/lib/StaticAnalyzer/Core/ExprEngineC.cpp
+++ b/clang/lib/StaticAnalyzer/Core/ExprEngineC.cpp
@@ -522,6 +522,7 @@ void ExprEngine::VisitCast(const CastExpr *CastE, const Expr *Ex,
       case CK_ToUnion:
       case CK_MatrixCast:
       case CK_VectorSplat:
+      case CK_HLSLElementwiseCast:
       case CK_HLSLVectorTruncation: {
         QualType resultType = CastE->getType();
         if (CastE->isGLValue())
diff --git a/clang/lib/StaticAnalyzer/Core/ExprEngineObjC.cpp b/clang/lib/StaticAnalyzer/Core/ExprEngineObjC.cpp
index f075df3ab5e4d..9426e0afd65a0 100644
--- a/clang/lib/StaticAnalyzer/Core/ExprEngineObjC.cpp
+++ b/clang/lib/StaticAnalyzer/Core/ExprEngineObjC.cpp
@@ -124,24 +124,26 @@ void ExprEngine::VisitObjCForCollectionStmt(const ObjCForCollectionStmt *S,
 
   bool isContainerNull = state->isNull(collectionV).isConstrainedTrue();
 
-  ExplodedNodeSet dstLocation;
-  evalLocation(dstLocation, S, elem, Pred, state, elementV, false);
+  ExplodedNodeSet DstLocation; // states in `DstLocation` may differ from `Pred`
+  evalLocation(DstLocation, S, elem, Pred, state, elementV, false);
 
-  ExplodedNodeSet Tmp;
-  StmtNodeBuilder Bldr(Pred, Tmp, *currBldrCtx);
+  for (ExplodedNode *dstLocation : DstLocation) {
+    ExplodedNodeSet DstLocationSingleton{dstLocation}, Tmp;
+    StmtNodeBuilder Bldr(dstLocation, Tmp, *currBldrCtx);
 
-  if (!isContainerNull)
-    populateObjCForDestinationSet(dstLocation, svalBuilder, S, elem, elementV,
-                                  SymMgr, currBldrCtx, Bldr,
-                                  /*hasElements=*/true);
+    if (!isContainerNull)
+      populateObjCForDestinationSet(DstLocationSingleton, svalBuilder, S, elem,
+                                    elementV, SymMgr, currBldrCtx, Bldr,
+                                    /*hasElements=*/true);
 
-  populateObjCForDestinationSet(dstLocation, svalBuilder, S, elem, elementV,
-                                SymMgr, currBldrCtx, Bldr,
-                                /*hasElements=*/false);
+    populateObjCForDestinationSet(DstLocationSingleton, svalBuilder, S, elem,
+                                  elementV, SymMgr, currBldrCtx, Bldr,
+                                  /*hasElements=*/false);
 
-  // Finally, run any custom checkers.
-  // FIXME: Eventually all pre- and post-checks should live in VisitStmt.
-  getCheckerManager().runCheckersForPostStmt(Dst, Tmp, S, *this);
+    // Finally, run any custom checkers.
+    // FIXME: Eventually all pre- and post-checks should live in VisitStmt.
+    getCheckerManager().runCheckersForPostStmt(Dst, Tmp, S, *this);
+  }
 }
 
 void ExprEngine::VisitObjCMessage(const ObjCMessageExpr *ME,
diff --git a/clang/lib/StaticAnalyzer/Core/MemRegion.cpp b/clang/lib/StaticAnalyzer/Core/MemRegion.cpp
index 2c5cd2cf7630f..c81bb632b83e9 100644
--- a/clang/lib/StaticAnalyzer/Core/MemRegion.cpp
+++ b/clang/lib/StaticAnalyzer/Core/MemRegion.cpp
@@ -891,9 +891,8 @@ DefinedOrUnknownSVal MemRegionManager::getStaticSize(const MemRegion *MR,
 
     return Size;
   }
-    // FIXME: The following are being used in 'SimpleSValBuilder' and in
-    // 'ArrayBoundChecker::checkLocation' because there is no symbol to
-    // represent the regions more appropriately.
+    // FIXME: The following are being used in 'SimpleSValBuilder' because there
+    // is no symbol to represent the regions more appropriately.
   case MemRegion::BlockDataRegionKind:
   case MemRegion::BlockCodeRegionKind:
   case MemRegion::FunctionCodeRegionKind:
diff --git a/clang/lib/StaticAnalyzer/Core/RegionStore.cpp b/clang/lib/StaticAnalyzer/Core/RegionStore.cpp
index 6266878565c52..d01b6ae55f611 100644
--- a/clang/lib/StaticAnalyzer/Core/RegionStore.cpp
+++ b/clang/lib/StaticAnalyzer/Core/RegionStore.cpp
@@ -29,6 +29,7 @@
 #include "clang/StaticAnalyzer/Core/PathSensitive/ProgramStateTrait.h"
 #include "llvm/ADT/ImmutableMap.h"
 #include "llvm/ADT/STLExtras.h"
+#include "llvm/Support/TimeProfiler.h"
 #include "llvm/Support/raw_ostream.h"
 #include <optional>
 #include <utility>
@@ -112,6 +113,13 @@ class BindingKey {
 
   LLVM_DUMP_METHOD void dump() const;
 };
+
+std::string locDescr(Loc L) {
+  std::string S;
+  llvm::raw_string_ostream OS(S);
+  L.dumpToStream(OS);
+  return OS.str();
+}
 } // end anonymous namespace
 
 BindingKey BindingKey::Make(const MemRegion *R, Kind k) {
@@ -2408,6 +2416,8 @@ StoreRef RegionStoreManager::killBinding(Store ST, Loc L) {
 
 RegionBindingsRef
 RegionStoreManager::bind(RegionBindingsConstRef B, Loc L, SVal V) {
+  llvm::TimeTraceScope TimeScope("RegionStoreManager::bind",
+                                 [&L]() { return locDescr(L); });
   // We only care about region locations.
   auto MemRegVal = L.getAs<loc::MemRegionVal>();
   if (!MemRegVal)
@@ -2514,6 +2524,8 @@ RegionBindingsRef
 RegionStoreManager::bindArray(RegionBindingsConstRef B,
                               const TypedValueRegion* R,
                               SVal Init) {
+  llvm::TimeTraceScope TimeScope("RegionStoreManager::bindArray",
+                                 [R]() { return R->getDescriptiveName(); });
 
   const ArrayType *AT =cast<ArrayType>(Ctx.getCanonicalType(R->getValueType()));
   QualType ElementTy = AT->getElementType();
@@ -2578,6 +2590,8 @@ RegionStoreManager::bindArray(RegionBindingsConstRef B,
 RegionBindingsRef RegionStoreManager::bindVector(RegionBindingsConstRef B,
                                                  const TypedValueRegion* R,
                                                  SVal V) {
+  llvm::TimeTraceScope TimeScope("RegionStoreManager::bindVector",
+                                 [R]() { return R->getDescriptiveName(); });
   QualType T = R->getValueType();
   const VectorType *VT = T->castAs<VectorType>(); // Use castAs for typedefs.
 
@@ -2700,6 +2714,8 @@ std::optional<RegionBindingsRef> RegionStoreManager::tryBindSmallStruct(
 RegionBindingsRef RegionStoreManager::bindStruct(RegionBindingsConstRef B,
                                                  const TypedValueRegion *R,
                                                  SVal V) {
+  llvm::TimeTraceScope TimeScope("RegionStoreManager::bindStruct",
+                                 [R]() { return R->getDescriptiveName(); });
   QualType T = R->getValueType();
   assert(T->isStructureOrClassType());
 
@@ -2818,6 +2834,8 @@ RegionBindingsRef
 RegionStoreManager::bindAggregate(RegionBindingsConstRef B,
                                   const TypedRegion *R,
                                   SVal Val) {
+  llvm::TimeTraceScope TimeScope("RegionStoreManager::bindAggregate",
+                                 [R]() { return R->getDescriptiveName(); });
   // Remove the old bindings, using 'R' as the root of all regions
   // we will invalidate. Then add the new binding.
   return removeSubRegionBindings(B, R).addBinding(R, BindingKey::Default, Val);
diff --git a/clang/lib/StaticAnalyzer/Core/TextDiagnostics.cpp b/clang/lib/StaticAnalyzer/Core/TextDiagnostics.cpp
index e8cf367b83346..79ee430988b7c 100644
--- a/clang/lib/StaticAnalyzer/Core/TextDiagnostics.cpp
+++ b/clang/lib/StaticAnalyzer/Core/TextDiagnostics.cpp
@@ -91,7 +91,6 @@ class TextDiagnostics : public PathDiagnosticConsumer {
                                     ? " [" + PD->getCheckerName() + "]"
                                     : "")
                                    .str();
-
       reportPiece(WarnID, PD->getLocation().asLocation(),
                   (PD->getShortDescription() + WarningMsg).str(),
                   PD->path.back()->getRanges(), PD->path.back()->getFixits());
diff --git a/clang/lib/StaticAnalyzer/Frontend/AnalysisConsumer.cpp b/clang/lib/StaticAnalyzer/Frontend/AnalysisConsumer.cpp
index 91c9b085f6829..189d7d6bede8e 100644
--- a/clang/lib/StaticAnalyzer/Frontend/AnalysisConsumer.cpp
+++ b/clang/lib/StaticAnalyzer/Frontend/AnalysisConsumer.cpp
@@ -39,6 +39,7 @@
 #include "llvm/Support/FileSystem.h"
 #include "llvm/Support/Path.h"
 #include "llvm/Support/Program.h"
+#include "llvm/Support/TimeProfiler.h"
 #include "llvm/Support/Timer.h"
 #include "llvm/Support/raw_ostream.h"
 #include <memory>
@@ -358,9 +359,40 @@ class AnalysisConsumer : public AnalysisASTConsumer,
 
   /// Print \p S to stderr if \c Opts.AnalyzerDisplayProgress is set.
   void reportAnalyzerProgress(StringRef S);
-}; // namespace
-} // end anonymous namespace
+};
 
+std::string timeTraceScopeDeclName(StringRef FunName, const Decl *D) {
+  if (llvm::timeTraceProfilerEnabled()) {
+    if (const NamedDecl *ND = dyn_cast<NamedDecl>(D))
+      return (FunName + " " + ND->getQualifiedNameAsString()).str();
+    return (FunName + " <anonymous> ").str();
+  }
+  return "";
+}
+
+llvm::TimeTraceMetadata timeTraceScopeDeclMetadata(const Decl *D) {
+  // If time-trace profiler is not enabled, this function is never called.
+  assert(llvm::timeTraceProfilerEnabled());
+  if (const auto &Loc = D->getBeginLoc(); Loc.isValid()) {
+    const auto &SM = D->getASTContext().getSourceManager();
+    std::string DeclName = AnalysisDeclContext::getFunctionName(D);
+    return llvm::TimeTraceMetadata{
+        std::move(DeclName), SM.getFilename(Loc).str(),
+        static_cast<int>(SM.getExpansionLineNumber(Loc))};
+  }
+  return llvm::TimeTraceMetadata{"", ""};
+}
+
+void flushReports(llvm::Timer *BugReporterTimer, BugReporter &BR) {
+  llvm::TimeTraceScope TCS{"Flushing reports"};
+  // Display warnings.
+  if (BugReporterTimer)
+    BugReporterTimer->startTimer();
+  BR.FlushReports();
+  if (BugReporterTimer)
+    BugReporterTimer->stopTimer();
+}
+} // namespace
 
 //===----------------------------------------------------------------------===//
 // AnalysisConsumer implementation.
@@ -658,6 +690,8 @@ AnalysisConsumer::getModeForDecl(Decl *D, AnalysisMode Mode) {
 void AnalysisConsumer::HandleCode(Decl *D, AnalysisMode Mode,
                                   ExprEngine::InliningModes IMode,
                                   SetOfConstDecls *VisitedCallees) {
+  llvm::TimeTraceScope TCS(timeTraceScopeDeclName("HandleCode", D),
+                           [D]() { return timeTraceScopeDeclMetadata(D); });
   if (!D->hasBody())
     return;
   Mode = getModeForDecl(D, Mode);
@@ -742,12 +776,7 @@ void AnalysisConsumer::RunPathSensitiveChecks(Decl *D,
   if (Mgr->options.visualizeExplodedGraphWithGraphViz)
     Eng.ViewGraph(Mgr->options.TrimGraph);
 
-  // Display warnings.
-  if (BugReporterTimer)
-    BugReporterTimer->startTimer();
-  Eng.getBugReporter().FlushReports();
-  if (BugReporterTimer)
-    BugReporterTimer->stopTimer();
+  flushReports(BugReporterTimer.get(), Eng.getBugReporter());
 }
 
 //===----------------------------------------------------------------------===//
diff --git a/clang/lib/Tooling/DependencyScanning/DependencyScanningTool.cpp b/clang/lib/Tooling/DependencyScanning/DependencyScanningTool.cpp
index 4219f67165861..2b4c2bb76434a 100644
--- a/clang/lib/Tooling/DependencyScanning/DependencyScanningTool.cpp
+++ b/clang/lib/Tooling/DependencyScanning/DependencyScanningTool.cpp
@@ -142,11 +142,13 @@ llvm::Expected<TranslationUnitDeps>
 DependencyScanningTool::getTranslationUnitDependencies(
     const std::vector<std::string> &CommandLine, StringRef CWD,
     const llvm::DenseSet<ModuleID> &AlreadySeen,
-    LookupModuleOutputCallback LookupModuleOutput) {
+    LookupModuleOutputCallback LookupModuleOutput,
+    std::optional<llvm::MemoryBufferRef> TUBuffer) {
   FullDependencyConsumer Consumer(AlreadySeen);
   CallbackActionController Controller(LookupModuleOutput);
-  llvm::Error Result =
-      Worker.computeDependencies(CWD, CommandLine, Consumer, Controller);
+  llvm::Error Result = Worker.computeDependencies(CWD, CommandLine, Consumer,
+                                                  Controller, TUBuffer);
+
   if (Result)
     return std::move(Result);
   return Consumer.takeTranslationUnitDeps();
diff --git a/clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp b/clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp
index 5a648df05e4fd..d15b74a28ab24 100644
--- a/clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp
+++ b/clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp
@@ -24,9 +24,11 @@
 #include "clang/Tooling/DependencyScanning/DependencyScanningService.h"
 #include "clang/Tooling/DependencyScanning/ModuleDepCollector.h"
 #include "clang/Tooling/Tooling.h"
+#include "llvm/ADT/IntrusiveRefCntPtr.h"
 #include "llvm/ADT/ScopeExit.h"
 #include "llvm/Support/Allocator.h"
 #include "llvm/Support/Error.h"
+#include "llvm/Support/MemoryBuffer.h"
 #include "llvm/TargetParser/Host.h"
 #include <optional>
 
@@ -521,20 +523,43 @@ DependencyScanningWorker::DependencyScanningWorker(
   }
 }
 
-llvm::Error DependencyScanningWorker::computeDependencies(
-    StringRef WorkingDirectory, const std::vector<std::string> &CommandLine,
-    DependencyConsumer &Consumer, DependencyActionController &Controller,
-    std::optional<StringRef> ModuleName) {
+static std::unique_ptr<DiagnosticOptions>
+createDiagOptions(const std::vector<std::string> &CommandLine) {
   std::vector<const char *> CLI;
   for (const std::string &Arg : CommandLine)
     CLI.push_back(Arg.c_str());
   auto DiagOpts = CreateAndPopulateDiagOpts(CLI);
   sanitizeDiagOpts(*DiagOpts);
+  return DiagOpts;
+}
+
+llvm::Error DependencyScanningWorker::computeDependencies(
+    StringRef WorkingDirectory, const std::vector<std::string> &CommandLine,
+    DependencyConsumer &Consumer, DependencyActionController &Controller,
+    std::optional<llvm::MemoryBufferRef> TUBuffer) {
+  // Capture the emitted diagnostics and report them to the client
+  // in the case of a failure.
+  std::string DiagnosticOutput;
+  llvm::raw_string_ostream DiagnosticsOS(DiagnosticOutput);
+  auto DiagOpts = createDiagOptions(CommandLine);
+  TextDiagnosticPrinter DiagPrinter(DiagnosticsOS, DiagOpts.release());
 
+  if (computeDependencies(WorkingDirectory, CommandLine, Consumer, Controller,
+                          DiagPrinter, TUBuffer))
+    return llvm::Error::success();
+  return llvm::make_error<llvm::StringError>(DiagnosticsOS.str(),
+                                             llvm::inconvertibleErrorCode());
+}
+
+llvm::Error DependencyScanningWorker::computeDependencies(
+    StringRef WorkingDirectory, const std::vector<std::string> &CommandLine,
+    DependencyConsumer &Consumer, DependencyActionController &Controller,
+    StringRef ModuleName) {
   // Capture the emitted diagnostics and report them to the client
   // in the case of a failure.
   std::string DiagnosticOutput;
   llvm::raw_string_ostream DiagnosticsOS(DiagnosticOutput);
+  auto DiagOpts = createDiagOptions(CommandLine);
   TextDiagnosticPrinter DiagPrinter(DiagnosticsOS, DiagOpts.release());
 
   if (computeDependencies(WorkingDirectory, CommandLine, Consumer, Controller,
@@ -604,54 +629,22 @@ static bool createAndRunToolInvocation(
   return true;
 }
 
-bool DependencyScanningWorker::computeDependencies(
+bool DependencyScanningWorker::scanDependencies(
     StringRef WorkingDirectory, const std::vector<std::string> &CommandLine,
     DependencyConsumer &Consumer, DependencyActionController &Controller,
-    DiagnosticConsumer &DC, std::optional<StringRef> ModuleName) {
-  // Reset what might have been modified in the previous worker invocation.
-  BaseFS->setCurrentWorkingDirectory(WorkingDirectory);
-
-  std::optional<std::vector<std::string>> ModifiedCommandLine;
-  llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> ModifiedFS;
-
-  // If we're scanning based on a module name alone, we don't expect the client
-  // to provide us with an input file. However, the driver really wants to have
-  // one. Let's just make it up to make the driver happy.
-  if (ModuleName) {
-    auto OverlayFS =
-        llvm::makeIntrusiveRefCnt<llvm::vfs::OverlayFileSystem>(BaseFS);
-    auto InMemoryFS =
-        llvm::makeIntrusiveRefCnt<llvm::vfs::InMemoryFileSystem>();
-    InMemoryFS->setCurrentWorkingDirectory(WorkingDirectory);
-    OverlayFS->pushOverlay(InMemoryFS);
-    ModifiedFS = OverlayFS;
-
-    SmallString<128> FakeInputPath;
-    // TODO: We should retry the creation if the path already exists.
-    llvm::sys::fs::createUniquePath(*ModuleName + "-%%%%%%%%.input",
-                                    FakeInputPath,
-                                    /*MakeAbsolute=*/false);
-    InMemoryFS->addFile(FakeInputPath, 0, llvm::MemoryBuffer::getMemBuffer(""));
-
-    ModifiedCommandLine = CommandLine;
-    ModifiedCommandLine->emplace_back(FakeInputPath);
-  }
-
-  const std::vector<std::string> &FinalCommandLine =
-      ModifiedCommandLine ? *ModifiedCommandLine : CommandLine;
-  auto &FinalFS = ModifiedFS ? ModifiedFS : BaseFS;
-
+    DiagnosticConsumer &DC, llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> FS,
+    std::optional<StringRef> ModuleName) {
   auto FileMgr =
-      llvm::makeIntrusiveRefCnt<FileManager>(FileSystemOptions{}, FinalFS);
+      llvm::makeIntrusiveRefCnt<FileManager>(FileSystemOptions{}, FS);
 
-  std::vector<const char *> FinalCCommandLine(FinalCommandLine.size(), nullptr);
-  llvm::transform(FinalCommandLine, FinalCCommandLine.begin(),
+  std::vector<const char *> CCommandLine(CommandLine.size(), nullptr);
+  llvm::transform(CommandLine, CCommandLine.begin(),
                   [](const std::string &Str) { return Str.c_str(); });
-
-  auto DiagOpts = CreateAndPopulateDiagOpts(FinalCCommandLine);
+  auto DiagOpts = CreateAndPopulateDiagOpts(CCommandLine);
   sanitizeDiagOpts(*DiagOpts);
   IntrusiveRefCntPtr<DiagnosticsEngine> Diags =
-      CompilerInstance::createDiagnostics(*FinalFS, DiagOpts.release(), &DC,
+      CompilerInstance::createDiagnostics(FileMgr->getVirtualFileSystem(),
+                                          DiagOpts.release(), &DC,
                                           /*ShouldOwnClient=*/false);
 
   // Although `Diagnostics` are used only for command-line parsing, the
@@ -667,12 +660,12 @@ bool DependencyScanningWorker::computeDependencies(
                                   DisableFree, ModuleName);
 
   bool Success = false;
-  if (FinalCommandLine[1] == "-cc1") {
-    Success = createAndRunToolInvocation(FinalCommandLine, Action, *FileMgr,
+  if (CommandLine[1] == "-cc1") {
+    Success = createAndRunToolInvocation(CommandLine, Action, *FileMgr,
                                          PCHContainerOps, *Diags, Consumer);
   } else {
     Success = forEachDriverJob(
-        FinalCommandLine, *Diags, *FileMgr, [&](const driver::Command &Cmd) {
+        CommandLine, *Diags, *FileMgr, [&](const driver::Command &Cmd) {
           if (StringRef(Cmd.getCreator().getName()) != "clang") {
             // Non-clang command. Just pass through to the dependency
             // consumer.
@@ -699,8 +692,77 @@ bool DependencyScanningWorker::computeDependencies(
 
   if (Success && !Action.hasScanned())
     Diags->Report(diag::err_fe_expected_compiler_job)
-        << llvm::join(FinalCommandLine, " ");
+        << llvm::join(CommandLine, " ");
   return Success && Action.hasScanned();
 }
 
+bool DependencyScanningWorker::computeDependencies(
+    StringRef WorkingDirectory, const std::vector<std::string> &CommandLine,
+    DependencyConsumer &Consumer, DependencyActionController &Controller,
+    DiagnosticConsumer &DC, std::optional<llvm::MemoryBufferRef> TUBuffer) {
+  // Reset what might have been modified in the previous worker invocation.
+  BaseFS->setCurrentWorkingDirectory(WorkingDirectory);
+
+  std::optional<std::vector<std::string>> ModifiedCommandLine;
+  llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> ModifiedFS;
+
+  // If we're scanning based on a module name alone, we don't expect the client
+  // to provide us with an input file. However, the driver really wants to have
+  // one. Let's just make it up to make the driver happy.
+  if (TUBuffer) {
+    auto OverlayFS =
+        llvm::makeIntrusiveRefCnt<llvm::vfs::OverlayFileSystem>(BaseFS);
+    auto InMemoryFS =
+        llvm::makeIntrusiveRefCnt<llvm::vfs::InMemoryFileSystem>();
+    InMemoryFS->setCurrentWorkingDirectory(WorkingDirectory);
+    auto InputPath = TUBuffer->getBufferIdentifier();
+    InMemoryFS->addFile(
+        InputPath, 0,
+        llvm::MemoryBuffer::getMemBufferCopy(TUBuffer->getBuffer()));
+    llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> InMemoryOverlay =
+        InMemoryFS;
+
+    OverlayFS->pushOverlay(InMemoryOverlay);
+    ModifiedFS = OverlayFS;
+    ModifiedCommandLine = CommandLine;
+    ModifiedCommandLine->emplace_back(InputPath);
+  }
+
+  const std::vector<std::string> &FinalCommandLine =
+      ModifiedCommandLine ? *ModifiedCommandLine : CommandLine;
+  auto &FinalFS = ModifiedFS ? ModifiedFS : BaseFS;
+
+  return scanDependencies(WorkingDirectory, FinalCommandLine, Consumer,
+                          Controller, DC, FinalFS, /*ModuleName=*/std::nullopt);
+}
+
+bool DependencyScanningWorker::computeDependencies(
+    StringRef WorkingDirectory, const std::vector<std::string> &CommandLine,
+    DependencyConsumer &Consumer, DependencyActionController &Controller,
+    DiagnosticConsumer &DC, StringRef ModuleName) {
+  // Reset what might have been modified in the previous worker invocation.
+  BaseFS->setCurrentWorkingDirectory(WorkingDirectory);
+
+  // If we're scanning based on a module name alone, we don't expect the client
+  // to provide us with an input file. However, the driver really wants to have
+  // one. Let's just make it up to make the driver happy.
+  auto OverlayFS =
+      llvm::makeIntrusiveRefCnt<llvm::vfs::OverlayFileSystem>(BaseFS);
+  auto InMemoryFS = llvm::makeIntrusiveRefCnt<llvm::vfs::InMemoryFileSystem>();
+  InMemoryFS->setCurrentWorkingDirectory(WorkingDirectory);
+  SmallString<128> FakeInputPath;
+  // TODO: We should retry the creation if the path already exists.
+  llvm::sys::fs::createUniquePath(ModuleName + "-%%%%%%%%.input", FakeInputPath,
+                                  /*MakeAbsolute=*/false);
+  InMemoryFS->addFile(FakeInputPath, 0, llvm::MemoryBuffer::getMemBuffer(""));
+  llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> InMemoryOverlay = InMemoryFS;
+
+  OverlayFS->pushOverlay(InMemoryOverlay);
+  auto ModifiedCommandLine = CommandLine;
+  ModifiedCommandLine.emplace_back(FakeInputPath);
+
+  return scanDependencies(WorkingDirectory, ModifiedCommandLine, Consumer,
+                          Controller, DC, OverlayFS, ModuleName);
+}
+
 DependencyActionController::~DependencyActionController() {}
diff --git a/clang/lib/Tooling/DependencyScanning/ModuleDepCollector.cpp b/clang/lib/Tooling/DependencyScanning/ModuleDepCollector.cpp
index 2e97cac0796ce..1c5f4c4b50ab6 100644
--- a/clang/lib/Tooling/DependencyScanning/ModuleDepCollector.cpp
+++ b/clang/lib/Tooling/DependencyScanning/ModuleDepCollector.cpp
@@ -214,9 +214,6 @@ makeCommonInvocationForModuleBuild(CompilerInvocation CI) {
   CI.getDependencyOutputOpts().Targets.clear();
 
   CI.getFrontendOpts().ProgramAction = frontend::GenerateModule;
-  CI.getFrontendOpts().ARCMTAction = FrontendOptions::ARCMT_None;
-  CI.getFrontendOpts().ObjCMTAction = FrontendOptions::ObjCMT_None;
-  CI.getFrontendOpts().MTMigrateDir.clear();
   CI.getLangOpts().ModuleName.clear();
 
   // Remove any macro definitions that are explicitly ignored.
@@ -397,9 +394,91 @@ void ModuleDepCollector::applyDiscoveredDependencies(CompilerInvocation &CI) {
   }
 }
 
+static bool isSafeToIgnoreCWD(const CowCompilerInvocation &CI) {
+  // Check if the command line input uses relative paths.
+  // It is not safe to ignore the current working directory if any of the
+  // command line inputs use relative paths.
+#define IF_RELATIVE_RETURN_FALSE(PATH)                                         \
+  do {                                                                         \
+    if (!PATH.empty() && !llvm::sys::path::is_absolute(PATH))                  \
+      return false;                                                            \
+  } while (0)
+
+#define IF_ANY_RELATIVE_RETURN_FALSE(PATHS)                                    \
+  do {                                                                         \
+    if (llvm::any_of(PATHS, [](const auto &P) {                                \
+          return !P.empty() && !llvm::sys::path::is_absolute(P);               \
+        }))                                                                    \
+      return false;                                                            \
+  } while (0)
+
+  // Header search paths.
+  const auto &HeaderSearchOpts = CI.getHeaderSearchOpts();
+  IF_RELATIVE_RETURN_FALSE(HeaderSearchOpts.Sysroot);
+  for (auto &Entry : HeaderSearchOpts.UserEntries)
+    if (Entry.IgnoreSysRoot)
+      IF_RELATIVE_RETURN_FALSE(Entry.Path);
+  IF_RELATIVE_RETURN_FALSE(HeaderSearchOpts.ResourceDir);
+  IF_RELATIVE_RETURN_FALSE(HeaderSearchOpts.ModuleCachePath);
+  IF_RELATIVE_RETURN_FALSE(HeaderSearchOpts.ModuleUserBuildPath);
+  for (auto I = HeaderSearchOpts.PrebuiltModuleFiles.begin(),
+            E = HeaderSearchOpts.PrebuiltModuleFiles.end();
+       I != E;) {
+    auto Current = I++;
+    IF_RELATIVE_RETURN_FALSE(Current->second);
+  }
+  IF_ANY_RELATIVE_RETURN_FALSE(HeaderSearchOpts.PrebuiltModulePaths);
+  IF_ANY_RELATIVE_RETURN_FALSE(HeaderSearchOpts.VFSOverlayFiles);
+
+  // Preprocessor options.
+  const auto &PPOpts = CI.getPreprocessorOpts();
+  IF_ANY_RELATIVE_RETURN_FALSE(PPOpts.MacroIncludes);
+  IF_ANY_RELATIVE_RETURN_FALSE(PPOpts.Includes);
+  IF_RELATIVE_RETURN_FALSE(PPOpts.ImplicitPCHInclude);
+
+  // Frontend options.
+  const auto &FrontendOpts = CI.getFrontendOpts();
+  for (const FrontendInputFile &Input : FrontendOpts.Inputs) {
+    if (Input.isBuffer())
+      continue; // FIXME: Can this happen when parsing command-line?
+
+    IF_RELATIVE_RETURN_FALSE(Input.getFile());
+  }
+  IF_RELATIVE_RETURN_FALSE(FrontendOpts.CodeCompletionAt.FileName);
+  IF_ANY_RELATIVE_RETURN_FALSE(FrontendOpts.ModuleMapFiles);
+  IF_ANY_RELATIVE_RETURN_FALSE(FrontendOpts.ModuleFiles);
+  IF_ANY_RELATIVE_RETURN_FALSE(FrontendOpts.ModulesEmbedFiles);
+  IF_ANY_RELATIVE_RETURN_FALSE(FrontendOpts.ASTMergeFiles);
+  IF_RELATIVE_RETURN_FALSE(FrontendOpts.OverrideRecordLayoutsFile);
+  IF_RELATIVE_RETURN_FALSE(FrontendOpts.StatsFile);
+
+  // Filesystem options.
+  const auto &FileSystemOpts = CI.getFileSystemOpts();
+  IF_RELATIVE_RETURN_FALSE(FileSystemOpts.WorkingDir);
+
+  // Codegen options.
+  const auto &CodeGenOpts = CI.getCodeGenOpts();
+  IF_RELATIVE_RETURN_FALSE(CodeGenOpts.DebugCompilationDir);
+  IF_RELATIVE_RETURN_FALSE(CodeGenOpts.CoverageCompilationDir);
+
+  // Sanitizer options.
+  IF_ANY_RELATIVE_RETURN_FALSE(CI.getLangOpts().NoSanitizeFiles);
+
+  // Coverage mappings.
+  IF_RELATIVE_RETURN_FALSE(CodeGenOpts.ProfileInstrumentUsePath);
+  IF_RELATIVE_RETURN_FALSE(CodeGenOpts.SampleProfileFile);
+  IF_RELATIVE_RETURN_FALSE(CodeGenOpts.ProfileRemappingFile);
+
+  // Dependency output options.
+  for (auto &ExtraDep : CI.getDependencyOutputOpts().ExtraDeps)
+    IF_RELATIVE_RETURN_FALSE(ExtraDep.first);
+
+  return true;
+}
+
 static std::string getModuleContextHash(const ModuleDeps &MD,
                                         const CowCompilerInvocation &CI,
-                                        bool EagerLoadModules,
+                                        bool EagerLoadModules, bool IgnoreCWD,
                                         llvm::vfs::FileSystem &VFS) {
   llvm::HashBuilder<llvm::TruncatedBLAKE3<16>, llvm::endianness::native>
       HashBuilder;
@@ -410,8 +489,11 @@ static std::string getModuleContextHash(const ModuleDeps &MD,
   HashBuilder.add(getClangFullRepositoryVersion());
   HashBuilder.add(serialization::VERSION_MAJOR, serialization::VERSION_MINOR);
   llvm::ErrorOr<std::string> CWD = VFS.getCurrentWorkingDirectory();
-  if (CWD)
+  auto &FSOpts = const_cast<FileSystemOptions &>(CI.getFileSystemOpts());
+  if (CWD && !IgnoreCWD)
     HashBuilder.add(*CWD);
+  else
+    FSOpts.WorkingDir.clear();
 
   // Hash the BuildInvocation without any input files.
   SmallString<0> ArgVec;
@@ -443,8 +525,11 @@ static std::string getModuleContextHash(const ModuleDeps &MD,
 
 void ModuleDepCollector::associateWithContextHash(
     const CowCompilerInvocation &CI, ModuleDeps &Deps) {
-  Deps.ID.ContextHash = getModuleContextHash(
-      Deps, CI, EagerLoadModules, ScanInstance.getVirtualFileSystem());
+  bool IgnoreCWD = any(OptimizeArgs & ScanningOptimizations::IgnoreCWD) &&
+                   isSafeToIgnoreCWD(CI);
+  Deps.ID.ContextHash =
+      getModuleContextHash(Deps, CI, EagerLoadModules, IgnoreCWD,
+                           ScanInstance.getVirtualFileSystem());
   bool Inserted = ModuleDepsByID.insert({Deps.ID, &Deps}).second;
   (void)Inserted;
   assert(Inserted && "duplicate module mapping");
diff --git a/clang/test/ARCMT/Common.h b/clang/test/ARCMT/Common.h
deleted file mode 100644
index b388ecab74109..0000000000000
--- a/clang/test/ARCMT/Common.h
+++ /dev/null
@@ -1,110 +0,0 @@
-#if __has_feature(objc_arr)
-#define NS_AUTOMATED_REFCOUNT_UNAVAILABLE __attribute__((unavailable("not available in automatic reference counting mode")))
-#else
-#define NS_AUTOMATED_REFCOUNT_UNAVAILABLE
-#endif
-
-#define NS_RETURNS_RETAINED __attribute__((ns_returns_retained))
-#define CF_CONSUMED __attribute__((cf_consumed))
-#define CF_RETURNS_RETAINED __attribute__((cf_returns_retained))
-
-#define NS_INLINE static __inline__ __attribute__((always_inline))
-#define nil ((void*) 0)
-#define NULL ((void*)0)
-
-typedef int BOOL;
-typedef unsigned NSUInteger;
-typedef int int32_t;
-typedef unsigned char uint8_t;
-typedef int32_t UChar32;
-typedef unsigned char UChar;
-
-typedef struct _NSZone NSZone;
-
-typedef const void * CFTypeRef;
-CFTypeRef CFRetain(CFTypeRef cf);
-CFTypeRef CFMakeCollectable(CFTypeRef cf) NS_AUTOMATED_REFCOUNT_UNAVAILABLE;
-
-NS_INLINE NS_RETURNS_RETAINED id NSMakeCollectable(CFTypeRef CF_CONSUMED cf) NS_AUTOMATED_REFCOUNT_UNAVAILABLE;
-
-@protocol NSObject
-- (BOOL)isEqual:(id)object;
-- (NSZone *)zone NS_AUTOMATED_REFCOUNT_UNAVAILABLE;
-- (id)retain NS_AUTOMATED_REFCOUNT_UNAVAILABLE;
-- (NSUInteger)retainCount NS_AUTOMATED_REFCOUNT_UNAVAILABLE;
-- (oneway void)release NS_AUTOMATED_REFCOUNT_UNAVAILABLE;
-- (id)autorelease NS_AUTOMATED_REFCOUNT_UNAVAILABLE;
-@end
-
-@interface NSObject <NSObject> {}
-- (id)init;
-
-+ (id)new;
-+ (id)alloc;
-- (void)dealloc;
-
-- (void)finalize;
-
-- (id)copy;
-- (id)mutableCopy;
-@end
-
-NS_AUTOMATED_REFCOUNT_UNAVAILABLE
-@interface NSAutoreleasePool : NSObject {
-@private
-    void    *_token;
-    void    *_reserved3;
-    void    *_reserved2;
-    void    *_reserved;
-}
-
-+ (void)addObject:(id)anObject;
-
-- (void)addObject:(id)anObject;
-
-- (void)drain;
-
-@end
-
-typedef const void* objc_objectptr_t; 
-extern __attribute__((ns_returns_retained)) id objc_retainedObject(objc_objectptr_t __attribute__((cf_consumed)) pointer);
-extern __attribute__((ns_returns_not_retained)) id objc_unretainedObject(objc_objectptr_t pointer);
-extern objc_objectptr_t objc_unretainedPointer(id object);
-
-#define dispatch_retain(object) ({ dispatch_object_t _o = (object); _dispatch_object_validate(_o); (void)[_o retain]; })
-#define dispatch_release(object) ({ dispatch_object_t _o = (object); _dispatch_object_validate(_o); [_o release]; })
-#define xpc_retain(object) ({ xpc_object_t _o = (object); _xpc_object_validate(_o); [_o retain]; })
-#define xpc_release(object) ({ xpc_object_t _o = (object); _xpc_object_validate(_o); [_o release]; })
-
-typedef id dispatch_object_t;
-typedef id xpc_object_t;
-
-void _dispatch_object_validate(dispatch_object_t object);
-void _xpc_object_validate(xpc_object_t object);
-
-#if __has_feature(objc_arc)
-
-NS_INLINE CF_RETURNS_RETAINED CFTypeRef CFBridgingRetain(id X) {
-    return (__bridge_retained CFTypeRef)X;
-}
-
-NS_INLINE id CFBridgingRelease(CFTypeRef CF_CONSUMED X) {
-    return (__bridge_transfer id)X;
-}
-
-#else
-
-NS_INLINE CF_RETURNS_RETAINED CFTypeRef CFBridgingRetain(id X) {
-    return X ? CFRetain((CFTypeRef)X) : NULL;
-}
-
-NS_INLINE id CFBridgingRelease(CFTypeRef CF_CONSUMED X) {
-    return [(id)CFMakeCollectable(X) autorelease];
-}
-
-#endif
-
-void *_Block_copy(const void *aBlock);
-void _Block_release(const void *aBlock);
-#define Block_copy(...) ((__typeof(__VA_ARGS__))_Block_copy((const void *)(__VA_ARGS__)))
-#define Block_release(...) _Block_release((const void *)(__VA_ARGS__))
diff --git a/clang/test/ARCMT/GC-check-warn-nsalloc.m b/clang/test/ARCMT/GC-check-warn-nsalloc.m
deleted file mode 100644
index 1c33de5b65b25..0000000000000
--- a/clang/test/ARCMT/GC-check-warn-nsalloc.m
+++ /dev/null
@@ -1,10 +0,0 @@
-// RUN: %clang_cc1 -arcmt-action=check -no-ns-alloc-error -triple x86_64-apple-darwin10 -fobjc-gc-only %s 2>&1 | grep 'warning: \[rewriter\] call returns pointer to GC managed memory'
-// RUN: %clang_cc1 -arcmt-action=check -no-ns-alloc-error -triple x86_64-apple-darwin10 -fobjc-gc-only -x objective-c++ %s 2>&1 | grep 'warning: \[rewriter\] call returns pointer to GC managed memory'
-// TODO: Investigate VerifyDiagnosticConsumer failures on these tests when using -verify.
-
-typedef unsigned NSUInteger;
-void *__strong NSAllocateCollectable(NSUInteger size, NSUInteger options);
-
-void test1(void) {
-  NSAllocateCollectable(100, 0);
-}
diff --git a/clang/test/ARCMT/GC-check.m b/clang/test/ARCMT/GC-check.m
deleted file mode 100644
index e95e285432e0f..0000000000000
--- a/clang/test/ARCMT/GC-check.m
+++ /dev/null
@@ -1,19 +0,0 @@
-// RUN: %clang_cc1 -arcmt-action=check -verify -triple x86_64-apple-darwin10 -fobjc-gc-only %s
-// RUN: %clang_cc1 -arcmt-action=check -verify -triple x86_64-apple-darwin10 -fobjc-gc-only -x objective-c++ %s
-
-#define CF_AUTOMATED_REFCOUNT_UNAVAILABLE __attribute__((unavailable("not available in automatic reference counting mode")))
-typedef unsigned NSUInteger;
-typedef const void * CFTypeRef;
-CFTypeRef CFMakeCollectable(CFTypeRef cf) CF_AUTOMATED_REFCOUNT_UNAVAILABLE; // expected-note {{unavailable}}
-void *__strong NSAllocateCollectable(NSUInteger size, NSUInteger options);
-
-void test1(CFTypeRef *cft) {
-  CFTypeRef c = CFMakeCollectable(cft); // expected-error {{CFMakeCollectable will leak the object that it receives in ARC}} \
-                // expected-error {{unavailable}}
-  NSAllocateCollectable(100, 0); // expected-error {{call returns pointer to GC managed memory; it will become unmanaged in ARC}}
-}
-
-@interface I1 {
-  __strong void *gcVar; // expected-error {{GC managed memory will become unmanaged in ARC}}
-}
-@end;
diff --git a/clang/test/ARCMT/GC-no-arc-runtime.m b/clang/test/ARCMT/GC-no-arc-runtime.m
deleted file mode 100644
index 99ba2eb5f7aef..0000000000000
--- a/clang/test/ARCMT/GC-no-arc-runtime.m
+++ /dev/null
@@ -1,80 +0,0 @@
-// RUN: %clang_cc1 -triple x86_64-apple-macosx10.6 -fsyntax-only -fobjc-runtime-has-weak -fobjc-arc -x objective-c %s.result
-// RUN: arcmt-test --args -triple x86_64-apple-macosx10.6 -fsyntax-only -fobjc-gc-only -x objective-c %s > %t
-// RUN: diff %t %s.result
-// RUN: arcmt-test --args -triple x86_64-apple-macosx10.6 -fsyntax-only -fobjc-gc-only -x objective-c++ %s > %t
-// RUN: diff %t %s.result
-
-// MRC __weak broke this test somehow.
-// XFAIL: *
-
-#include "Common.h"
-#include "GC.h"
-
-void test1(CFTypeRef *cft) {
-  id x = NSMakeCollectable(cft);
-}
-
-@interface I1
-@end
-
-@implementation I1
--(void)dealloc {
-  // dealloc
-  test1(0);
-}
-
--(void)finalize {
-  // finalize
-  test1(0);
-}
-@end
-
-@interface I2
-@property (retain) id prop;
-@end
-
-@implementation I2
-@synthesize prop;
-
--(void)finalize {
-  self.prop = 0;
-  // finalize
-  test1(0);
-}
-@end
-
-__attribute__((objc_arc_weak_reference_unavailable))
-@interface QQ {
-  __weak id s;
-  __weak QQ *q;
-}
-@end
-
-@interface I3
-@property (assign) I3 *__weak pw1, *__weak pw2;
-@property (assign) I3 *__strong ps;
-@property (assign) I3 * pds;
-@end
-
-@interface I4Impl {
-  I4Impl *pds2;
-}
-@property (assign) I4Impl *__weak pw1, *__weak pw2;
-@property (assign) I4Impl *__strong ps;
-@property (assign) I4Impl * pds;
-@property (assign) I4Impl * pds2;
-@end
-
-@implementation I4Impl
-@synthesize pw1, pw2, ps, pds, pds2;
-
--(void)test1:(CFTypeRef *)cft {
-  id x = NSMakeCollectable(cft);
-}
-@end
-
-@interface I5 {
-  __weak id prop;
-}
-@property (readonly) __weak id prop;
-@end
diff --git a/clang/test/ARCMT/GC-no-arc-runtime.m.result b/clang/test/ARCMT/GC-no-arc-runtime.m.result
deleted file mode 100644
index c338bdb2ed471..0000000000000
--- a/clang/test/ARCMT/GC-no-arc-runtime.m.result
+++ /dev/null
@@ -1,72 +0,0 @@
-// RUN: %clang_cc1 -triple x86_64-apple-macosx10.6 -fsyntax-only -fobjc-runtime-has-weak -fobjc-arc -x objective-c %s.result
-// RUN: arcmt-test --args -triple x86_64-apple-macosx10.6 -fsyntax-only -fobjc-gc-only -x objective-c %s > %t
-// RUN: diff %t %s.result
-// RUN: arcmt-test --args -triple x86_64-apple-macosx10.6 -fsyntax-only -fobjc-gc-only -x objective-c++ %s > %t
-// RUN: diff %t %s.result
-
-#include "Common.h"
-#include "GC.h"
-
-void test1(CFTypeRef *cft) {
-  id x = CFBridgingRelease(cft);
-}
-
-@interface I1
-@end
-
-@implementation I1
--(void)dealloc {
-  // dealloc
-  test1(0);
-}
-
-@end
-
-@interface I2
-@property (strong) id prop;
-@end
-
-@implementation I2
-@synthesize prop;
-
--(void)dealloc {
-  // finalize
-  test1(0);
-}
-@end
-
-__attribute__((objc_arc_weak_reference_unavailable))
-@interface QQ {
-  __unsafe_unretained id s;
-  __unsafe_unretained QQ *q;
-}
-@end
-
-@interface I3
-@property (unsafe_unretained) I3 * pw1, * pw2;
-@property (strong) I3 * ps;
-@property (assign) I3 * pds;
-@end
-
-@interface I4Impl {
-  I4Impl *__strong pds2;
-}
-@property (unsafe_unretained) I4Impl * pw1, * pw2;
-@property (strong) I4Impl * ps;
-@property (strong) I4Impl * pds;
-@property (strong) I4Impl * pds2;
-@end
-
-@implementation I4Impl
-@synthesize pw1, pw2, ps, pds, pds2;
-
--(void)test1:(CFTypeRef *)cft {
-  id x = CFBridgingRelease(cft);
-}
-@end
-
-@interface I5 {
-  __unsafe_unretained id prop;
-}
-@property (unsafe_unretained, readonly)  id prop;
-@end
diff --git a/clang/test/ARCMT/GC-no-finalize-removal.m b/clang/test/ARCMT/GC-no-finalize-removal.m
deleted file mode 100644
index 07a737c1b7757..0000000000000
--- a/clang/test/ARCMT/GC-no-finalize-removal.m
+++ /dev/null
@@ -1,88 +0,0 @@
-// RUN: %clang_cc1 -triple x86_64-apple-macosx10.7 -fsyntax-only -fobjc-runtime-has-weak -fobjc-arc -x objective-c %s.result
-// RUN: arcmt-test --args -triple x86_64-apple-macosx10.7 -fsyntax-only -fobjc-gc-only -no-finalize-removal -x objective-c %s > %t
-// RUN: diff %t %s.result
-// RUN: arcmt-test --args -triple x86_64-apple-macosx10.7 -fsyntax-only -fobjc-gc-only -no-finalize-removal -x objective-c++ %s > %t
-// RUN: diff %t %s.result
-
-#include "Common.h"
-#include "GC.h"
-
-void test1(CFTypeRef *cft) {
-  id x = NSMakeCollectable(cft);
-}
-
-@interface I1
-@end
-
-@implementation I1
--(void)dealloc {
-  // dealloc
-  test1(0);
-}
-
--(void)finalize {
-  // finalize
-  test1(0);
-}
-@end
-
-@interface I2
-@property (retain) id prop;
-@end
-
-@implementation I2
-@synthesize prop;
-
--(void)finalize {
-  self.prop = 0;
-  // finalize
-  test1(0);
-}
-@end
-
-__attribute__((objc_arc_weak_reference_unavailable))
-@interface QQ {
-  __weak id s;
-  __weak QQ *q;
-}
-@end
-
-@interface I3
-@property (assign) I3 *__weak pw1, *__weak pw2;
-@property (assign) I3 *__strong ps;
-@property (assign) I3 * pds;
-@end
-
-@interface I4Impl {
-  I4Impl *pds2;
-  I4Impl *pds3;
-  __weak I4Impl *pw3;
-  __weak I4Impl *pw4;
-}
-@property (assign) I4Impl *__weak pw1, *__weak pw2;
-@property (assign) I4Impl *__strong ps;
-@property (assign) I4Impl * pds;
-@property (assign) I4Impl * pds2;
-@property (readwrite) I4Impl * pds3;
-@property (readonly) I4Impl * pds4;
-@property (readonly) __weak I4Impl *pw3;
-@property (assign) __weak I4Impl *pw4;
-@end
-
-@implementation I4Impl
-@synthesize pw1, pw2, pw3, pw4, ps, pds, pds2, pds3, pds4;
-
--(void)test1:(CFTypeRef *)cft {
-  id x = NSMakeCollectable(cft);
-}
-@end
-
-@interface rdar10532449
-@property (assign) id assign_prop;
-@property (assign, readonly) id __strong strong_readonly_prop;
-@property (assign) id __weak weak_prop;
-@end
-
-@implementation rdar10532449
-@synthesize assign_prop, strong_readonly_prop, weak_prop;
-@end
diff --git a/clang/test/ARCMT/GC-no-finalize-removal.m.result b/clang/test/ARCMT/GC-no-finalize-removal.m.result
deleted file mode 100644
index a2105b32fc755..0000000000000
--- a/clang/test/ARCMT/GC-no-finalize-removal.m.result
+++ /dev/null
@@ -1,96 +0,0 @@
-// RUN: %clang_cc1 -triple x86_64-apple-macosx10.7 -fsyntax-only -fobjc-runtime-has-weak -fobjc-arc -x objective-c %s.result
-// RUN: arcmt-test --args -triple x86_64-apple-macosx10.7 -fsyntax-only -fobjc-gc-only -no-finalize-removal -x objective-c %s > %t
-// RUN: diff %t %s.result
-// RUN: arcmt-test --args -triple x86_64-apple-macosx10.7 -fsyntax-only -fobjc-gc-only -no-finalize-removal -x objective-c++ %s > %t
-// RUN: diff %t %s.result
-
-#include "Common.h"
-#include "GC.h"
-
-void test1(CFTypeRef *cft) {
-  id x = CFBridgingRelease(cft);
-}
-
-@interface I1
-@end
-
-@implementation I1
--(void)dealloc {
-  // dealloc
-  test1(0);
-}
-
-#if !__has_feature(objc_arc)
--(void)finalize {
-  // finalize
-  test1(0);
-}
-#endif
-@end
-
-@interface I2
-@property (strong) id prop;
-@end
-
-@implementation I2
-@synthesize prop;
-
-#if !__has_feature(objc_arc)
--(void)finalize {
-  self.prop = 0;
-  // finalize
-  test1(0);
-}
-#endif
--(void)dealloc {
-  // finalize
-  test1(0);
-}
-@end
-
-__attribute__((objc_arc_weak_reference_unavailable))
-@interface QQ {
-  __weak id s;
-  __unsafe_unretained QQ *q;
-}
-@end
-
-@interface I3
-@property (weak) I3 * pw1, * pw2;
-@property (strong) I3 * ps;
-@property (assign) I3 * pds;
-@end
-
-@interface I4Impl {
-  I4Impl *__strong pds2;
-  I4Impl *pds3;
-  __weak I4Impl *pw3;
-  __weak I4Impl *pw4;
-}
-@property (weak) I4Impl * pw1, * pw2;
-@property (strong) I4Impl * ps;
-@property (strong) I4Impl * pds;
-@property (strong) I4Impl * pds2;
-@property (readwrite) I4Impl * pds3;
-@property (readonly) I4Impl * pds4;
-@property (weak, readonly)  I4Impl *pw3;
-@property (weak)  I4Impl *pw4;
-@end
-
-@implementation I4Impl
-@synthesize pw1, pw2, pw3, pw4, ps, pds, pds2, pds3, pds4;
-
--(void)test1:(CFTypeRef *)cft {
-  id x = CFBridgingRelease(cft);
-}
-@end
-
-@interface rdar10532449
-@property (strong) id assign_prop;
-@property (strong, readonly) id  strong_readonly_prop;
-@property (weak) id  weak_prop;
-@end
-
-@implementation rdar10532449
-@synthesize assign_prop, strong_readonly_prop, weak_prop;
-@end
diff --git a/clang/test/ARCMT/GC.h b/clang/test/ARCMT/GC.h
deleted file mode 100644
index 4301baf272465..0000000000000
--- a/clang/test/ARCMT/GC.h
+++ /dev/null
@@ -1,6 +0,0 @@
-
-@interface ExtInterface {
-  __strong ExtInterface *myivar;
-  __strong void *gcVar;
-}
-@end
diff --git a/clang/test/ARCMT/GC.m b/clang/test/ARCMT/GC.m
deleted file mode 100644
index 97723e89ed369..0000000000000
--- a/clang/test/ARCMT/GC.m
+++ /dev/null
@@ -1,93 +0,0 @@
-// RUN: %clang_cc1 -triple x86_64-apple-macosx10.7 -fsyntax-only -fobjc-runtime-has-weak -fobjc-arc -x objective-c %s.result
-// RUN: arcmt-test --args -triple x86_64-apple-macosx10.7 -fsyntax-only -fobjc-gc-only -x objective-c %s > %t
-// RUN: diff %t %s.result
-// RUN: arcmt-test --args -triple x86_64-apple-macosx10.7 -fsyntax-only -fobjc-gc-only -x objective-c++ %s > %t
-// RUN: diff %t %s.result
-
-#include "Common.h"
-#include "GC.h"
-
-void test1(CFTypeRef *cft) {
-  id x = NSMakeCollectable(cft);
-}
-
-@interface I1
-@end
-
-@implementation I1
--(void)dealloc {
-  // dealloc
-  test1(0);
-}
-
--(void)finalize {
-  // finalize
-  test1(0);
-}
-@end
-
-@interface I2
-@property (retain) id prop;
-@end
-
-@implementation I2
-@synthesize prop;
-
--(void)finalize {
-  self.prop = 0;
-  // finalize
-  test1(0);
-}
-@end
-
-__attribute__((objc_arc_weak_reference_unavailable))
-@interface QQ {
-  __weak id s;
-  __weak QQ *q;
-}
-@end
-
-@interface I3
-@property (assign) I3 *__weak pw1, *__weak pw2;
-@property (assign) I3 *__strong ps;
-@property (assign) I3 * pds;
-@end
-
-@interface I4Impl {
-  I4Impl *pds2;
-  I4Impl *pds3;
-  __weak I4Impl *pw3;
-  __weak I4Impl *pw4;
-}
-@property (assign) I4Impl *__weak pw1, *__weak pw2;
-@property (assign) I4Impl *__strong ps;
-@property (assign) I4Impl * pds;
-@property (assign) I4Impl * pds2;
-@property (readwrite) I4Impl * pds3;
-@property (readonly) I4Impl * pds4;
-@property (readonly) __weak I4Impl *pw3;
-@property (assign) __weak I4Impl *pw4;
-@end
-
-@implementation I4Impl
-@synthesize pw1, pw2, pw3, pw4, ps, pds, pds2, pds3, pds4;
-
--(void)test1:(CFTypeRef *)cft {
-  id x = NSMakeCollectable(cft);
-}
-@end
-
-@interface rdar10532449
-@property (assign) id assign_prop;
-@property (assign, readonly) id __strong strong_readonly_prop;
-@property (assign) id __weak weak_prop;
-@end
-
-@implementation rdar10532449
-@synthesize assign_prop, strong_readonly_prop, weak_prop;
-@end
-
-void test2(id p, __strong I1 *ap[]) {
-  for (__strong I1 *specRule in p) {
-  }
-}
diff --git a/clang/test/ARCMT/GC.m.result b/clang/test/ARCMT/GC.m.result
deleted file mode 100644
index b60b07ac9c110..0000000000000
--- a/clang/test/ARCMT/GC.m.result
+++ /dev/null
@@ -1,88 +0,0 @@
-// RUN: %clang_cc1 -triple x86_64-apple-macosx10.7 -fsyntax-only -fobjc-runtime-has-weak -fobjc-arc -x objective-c %s.result
-// RUN: arcmt-test --args -triple x86_64-apple-macosx10.7 -fsyntax-only -fobjc-gc-only -x objective-c %s > %t
-// RUN: diff %t %s.result
-// RUN: arcmt-test --args -triple x86_64-apple-macosx10.7 -fsyntax-only -fobjc-gc-only -x objective-c++ %s > %t
-// RUN: diff %t %s.result
-
-#include "Common.h"
-#include "GC.h"
-
-void test1(CFTypeRef *cft) {
-  id x = CFBridgingRelease(cft);
-}
-
-@interface I1
-@end
-
-@implementation I1
--(void)dealloc {
-  // dealloc
-  test1(0);
-}
-
-@end
-
-@interface I2
-@property (strong) id prop;
-@end
-
-@implementation I2
-@synthesize prop;
-
--(void)dealloc {
-  // finalize
-  test1(0);
-}
-@end
-
-__attribute__((objc_arc_weak_reference_unavailable))
-@interface QQ {
-  __weak id s;
-  __unsafe_unretained QQ *q;
-}
-@end
-
-@interface I3
-@property (weak) I3 * pw1, * pw2;
-@property (strong) I3 * ps;
-@property (assign) I3 * pds;
-@end
-
-@interface I4Impl {
-  I4Impl *__strong pds2;
-  I4Impl *pds3;
-  __weak I4Impl *pw3;
-  __weak I4Impl *pw4;
-}
-@property (weak) I4Impl * pw1, * pw2;
-@property (strong) I4Impl * ps;
-@property (strong) I4Impl * pds;
-@property (strong) I4Impl * pds2;
-@property (readwrite) I4Impl * pds3;
-@property (readonly) I4Impl * pds4;
-@property (weak, readonly)  I4Impl *pw3;
-@property (weak)  I4Impl *pw4;
-@end
-
-@implementation I4Impl
-@synthesize pw1, pw2, pw3, pw4, ps, pds, pds2, pds3, pds4;
-
--(void)test1:(CFTypeRef *)cft {
-  id x = CFBridgingRelease(cft);
-}
-@end
-
-@interface rdar10532449
-@property (strong) id assign_prop;
-@property (strong, readonly) id  strong_readonly_prop;
-@property (weak) id  weak_prop;
-@end
-
-@implementation rdar10532449
-@synthesize assign_prop, strong_readonly_prop, weak_prop;
-@end
-
-void test2(id p, __strong I1 *ap[]) {
-  for (__strong I1 *specRule in p) {
-  }
-}
diff --git a/clang/test/ARCMT/Inputs/Module.framework/Frameworks/SubFramework.framework/Headers/SubFramework.h b/clang/test/ARCMT/Inputs/Module.framework/Frameworks/SubFramework.framework/Headers/SubFramework.h
deleted file mode 100644
index f7f9fb66c9e94..0000000000000
--- a/clang/test/ARCMT/Inputs/Module.framework/Frameworks/SubFramework.framework/Headers/SubFramework.h
+++ /dev/null
@@ -1,5 +0,0 @@
-#ifndef MODULE_SUBFRAMEWORK_H
-#define MODULE_SUBFRAMEWORK_H
-#__private_macro MODULE_SUBFRAMEWORK_H
-char *module_subframework;
-#endif
diff --git a/clang/test/ARCMT/Inputs/Module.framework/Headers/Buried/Treasure.h b/clang/test/ARCMT/Inputs/Module.framework/Headers/Buried/Treasure.h
deleted file mode 100644
index 6e81adcb2b60c..0000000000000
--- a/clang/test/ARCMT/Inputs/Module.framework/Headers/Buried/Treasure.h
+++ /dev/null
@@ -1 +0,0 @@
-unsigned *Buried_Treasure;
diff --git a/clang/test/ARCMT/Inputs/Module.framework/Headers/Module.h b/clang/test/ARCMT/Inputs/Module.framework/Headers/Module.h
deleted file mode 100644
index 3d2476b20431d..0000000000000
--- a/clang/test/ARCMT/Inputs/Module.framework/Headers/Module.h
+++ /dev/null
@@ -1,28 +0,0 @@
-// expected-warning 0-1 {{umbrella header}}
-
-// FIXME: The "umbrella header" warning should be moved to a separate test.
-// This "0-1" is only here because the warning is only emitted when the
-// module is (otherwise) successfully included.
-
-#ifndef MODULE_H
-#define MODULE_H
-const char *getModuleVersion(void);
-
-#ifdef FOO
-#  error Module should have been built without -DFOO
-#endif
-
-@interface Module
-+(const char *)version; // retrieve module version
-+alloc;
-@end
-
-#define MODULE_H_MACRO 1
-#__private_macro MODULE_H_MACRO
-
-#include <Module/Sub.h>
-#include <Module/Buried/Treasure.h>
-
-__asm("foo");
-
-#endif // MODULE_H
diff --git a/clang/test/ARCMT/Inputs/Module.framework/Headers/NotInModule.h b/clang/test/ARCMT/Inputs/Module.framework/Headers/NotInModule.h
deleted file mode 100644
index 6b15791eb2c70..0000000000000
--- a/clang/test/ARCMT/Inputs/Module.framework/Headers/NotInModule.h
+++ /dev/null
@@ -1 +0,0 @@
-int not_in_module;
diff --git a/clang/test/ARCMT/Inputs/Module.framework/Headers/Sub.h b/clang/test/ARCMT/Inputs/Module.framework/Headers/Sub.h
deleted file mode 100644
index dea76e7646176..0000000000000
--- a/clang/test/ARCMT/Inputs/Module.framework/Headers/Sub.h
+++ /dev/null
@@ -1,3 +0,0 @@
-#include <Module/Sub2.h>
-int *Module_Sub;
-
diff --git a/clang/test/ARCMT/Inputs/Module.framework/Headers/Sub2.h b/clang/test/ARCMT/Inputs/Module.framework/Headers/Sub2.h
deleted file mode 100644
index beed4a862dcaf..0000000000000
--- a/clang/test/ARCMT/Inputs/Module.framework/Headers/Sub2.h
+++ /dev/null
@@ -1 +0,0 @@
-int *Module_Sub2;
diff --git a/clang/test/ARCMT/Inputs/Module.framework/Module b/clang/test/ARCMT/Inputs/Module.framework/Module
deleted file mode 100644
index e69de29bb2d1d..0000000000000
diff --git a/clang/test/ARCMT/Inputs/Module.framework/PrivateHeaders/ModulePrivate.h b/clang/test/ARCMT/Inputs/Module.framework/PrivateHeaders/ModulePrivate.h
deleted file mode 100644
index 0782336df9dec..0000000000000
--- a/clang/test/ARCMT/Inputs/Module.framework/PrivateHeaders/ModulePrivate.h
+++ /dev/null
@@ -1 +0,0 @@
-int module_private;
diff --git a/clang/test/ARCMT/Inputs/module.modulemap b/clang/test/ARCMT/Inputs/module.modulemap
deleted file mode 100644
index 061abbd24d570..0000000000000
--- a/clang/test/ARCMT/Inputs/module.modulemap
+++ /dev/null
@@ -1,309 +0,0 @@
-module c_library [extern_c] { module inner { header "c-header.h" } }
-module cxx_library { header "cxx-header.h" requires cplusplus }
-module c_library_bad [extern_c] { header "c-header-bad.h" }
-module diamond_top { header "diamond_top.h" }
-module diamond_left { 
-  header "diamond_left.h" 
-  export diamond_top
-}
-module diamond_right { 
-  header "diamond_right.h" 
-  export diamond_top
-}
-module diamond_bottom { 
-  header "diamond_bottom.h" 
-  export *
-}
-module irgen { header "irgen.h" }
-module cxx_irgen_top { header "cxx-irgen-top.h" }
-module cxx_irgen_left { header "cxx-irgen-left.h" }
-module cxx_irgen_right { header "cxx-irgen-right.h" }
-module lookup_left_objc { header "lookup_left.h" }
-module lookup_right_objc { header "lookup_right.h" }
-module lookup_left_cxx { header "lookup_left.hpp" }
-module lookup_right_cxx { header "lookup_right.hpp" }
-module module_private_left { header "module_private_left.h" }
-module module_private_right { header "module_private_right.h" }
-module macros_top { 
-  header "macros_top.h" 
-  explicit module b { header "macros_top_b.h" }
-  explicit module c { header "macros_top_c.h" }
-}
-module macros_left { 
-  header "macros_left.h" 
-  export *
-}
-module macros_right { 
-  header "macros_right.h" 
-  export *
-  explicit module undef {
-    header "macros_right_undef.h"
-  }
-}
-module macros { header "macros.h" }
-module macros_other { header "macros_other.h" }
-module category_top { header "category_top.h" }
-module category_left { 
-  header "category_left.h" 
-  export category_top
-
-  explicit module sub {
-    header "category_left_sub.h"
-  }
-}
-module category_right { 
-  header "category_right.h" 
-  export category_top
-
-  explicit module sub {
-    header "category_right_sub.h"
-  }
-}
-module category_bottom { 
-  header "category_bottom.h" 
-  export category_left
-  export category_right
-}
-module category_other { header "category_other.h" }
-module redeclarations_left { header "redeclarations_left.h" }
-module redeclarations_right { header "redeclarations_right.h" }
-module redecl_namespaces_left { header "redecl_namespaces_left.h" }
-module redecl_namespaces_right { header "redecl_namespaces_right.h" }
-module redecl_add_after_load_top { header "redecl-add-after-load-top.h" }
-module redecl_add_after_load { header "redecl-add-after-load.h" }
-module load_failure { header "load_failure.h" }
-
-module decldef {
-  explicit module Decl { header "decl.h" }
-  explicit module Decl2 { header "decl2.h" }
-  explicit module Def { header "def.h" }
-}
-
-module redecl_merge_top { 
-  header "redecl-merge-top.h"
-  explicit module Explicit { header "redecl-merge-top-explicit.h" }
-  exclude header "nonexistent.h"
-}
-module redecl_merge_left { 
-  header "redecl-merge-left.h" 
-  export *
-}
-module redecl_merge_left_left { 
-  header "redecl-merge-left-left.h" 
-  export *
-}
-module redecl_merge_right { 
-  header "redecl-merge-right.h" 
-  export *
-}
-module redecl_merge_bottom { 
-  explicit module prefix {
-    header "redecl-merge-bottom-prefix.h"
-  }
-
-  header "redecl-merge-bottom.h" 
-  export *
-}
-module namespaces_top { 
-  header "namespaces-top.h"
-  export *
-}
-module namespaces_left { 
-  header "namespaces-left.h"
-  export *
-}
-module namespaces_right { 
-  header "namespaces-right.h"
-  export *
-}
-module templates_top { 
-  header "templates-top.h"
-  export *
-}
-module templates_left { 
-  header "templates-left.h"
-  export *
-}
-module templates_right { 
-  header "templates-right.h"
-  export *
-}
-module MethodPoolA {
-  header "MethodPoolA.h"
-
-  explicit module Sub2 {
-    header "MethodPoolASub2.h"
-  }
-
-  explicit module Sub {
-    header "MethodPoolASub.h"
-  }
-}
-module MethodPoolB {
-  header "MethodPoolB.h"
-
-  explicit module Sub2 {
-    header "MethodPoolBSub2.h"
-  }
-
-  explicit module Sub {
-    header "MethodPoolBSub.h"
-  }
-}
-module import_decl {
-  header "import-decl.h"
-}
-
-framework module * { 
-  exclude NotAModule
-}
-
-module linkage_merge_left {
-  explicit module sub {
-    header "linkage-merge-sub.h"
-  }
-}
-
-module autolink {
-  header "autolink.h"
-  link "autolink"
-
-  explicit module sub {
-    header "autolink-sub.h"
-    link "autolink_sub"
-  }
-
-  explicit module sub2 {
-    header "autolink-sub2.h"
-    link framework "autolink_framework"
-  }
-
-  explicit module sub3 {
-    header "autolink-sub3.h"
-    link "autolink_from_pch"
-  }
-}
-
-module weird_objc {
-  header "weird_objc.h"
-}
-
-module ignored_macros {
-  header "ignored_macros.h"
-}
-
-module cxx_many_overloads {
-  header "cxx-many-overloads.h"
-}
-
-module cxx_inline_namespace {
-  header "cxx-inline-namespace.h"
-}
-
-module cxx_inline_namespace_b {
-  header "cxx-inline-namespace-b.h"
-}
-
-module cxx_linkage_cache {
-  header "cxx-linkage-cache.h"
-}
-
-module cxx_templates_common {
-  header "cxx-templates-common.h"
-}
-
-module cxx_templates_a {
-  header "cxx-templates-a.h"
-}
-
-module cxx_templates_b_impl {
-  header "cxx-templates-b-impl.h"
-}
-
-module cxx_templates_b {
-  header "cxx-templates-b.h"
-}
-
-module cxx_templates_c {
-  header "cxx-templates-c.h"
-}
-
-module cxx_decls {
-  module unimported {
-    header "cxx-decls-unimported.h"
-  }
-  module imported {
-    header "cxx-decls-imported.h"
-  }
-}
-
-module config {
-  header "config.h"
-  config_macros [exhaustive] WANT_FOO, WANT_BAR
-}
-
-module diag_pragma {
-  header "diag_pragma.h"
-}
-
-module dummy {
-  header "dummy.h"
-}
-
-module builtin {
-  header "builtin.h"
-  explicit module sub {
-    header "builtin_sub.h"
-  }
-}
-
-module linkage_merge {
-  explicit module foo {
-    header "linkage-merge-foo.h"
-  }
-  explicit module bar {
-    header "linkage-merge-bar.h"
-  }
-
-}
-
-module incomplete_mod {
-  header "incomplete_mod.h"
-}
-
-module warning {
-  header "warning.h"
-}
-
-module initializer_list {
-  header "initializer_list"
-}
-
-module using_decl {
-  module a { header "using-decl-a.h" export * }
-  module b { header "using-decl-b.h" export * }
-}
-
-module recursive_visibility_a1 {
-  module inner { header "recursive_visibility_a1_inner.h" }
-}
-module recursive_visibility_a2 {
-  module inner {
-    module more_inner {
-      header "recursive_visibility_a2_more_inner.h"
-    }
-  }
-}
-module recursive_visibility_b {
-  header "recursive_visibility_b.h"
-  export *
-}
-module recursive_visibility_c {
-  header "recursive_visibility_c.h"
-}
-module recursive1 {
-  header "recursive1.h"
-}
-module recursive2 {
-  header "recursive2.h"
-}
diff --git a/clang/test/ARCMT/Inputs/test.h b/clang/test/ARCMT/Inputs/test.h
deleted file mode 100644
index 756295f27e698..0000000000000
--- a/clang/test/ARCMT/Inputs/test.h
+++ /dev/null
@@ -1,15 +0,0 @@
-@protocol NSObject
-- (oneway void)release;
-@end
-
-#ifdef PART1
-static inline void part1(id p) {
-  [p release];
-}
-#endif
-
-#ifdef PART2
-static inline void part2(id p) {
-  [p release];
-}
-#endif
diff --git a/clang/test/ARCMT/Inputs/test.h.result b/clang/test/ARCMT/Inputs/test.h.result
deleted file mode 100644
index 0638a3378c1c0..0000000000000
--- a/clang/test/ARCMT/Inputs/test.h.result
+++ /dev/null
@@ -1,13 +0,0 @@
-@protocol NSObject
-- (oneway void)release;
-@end
-
-#ifdef PART1
-static inline void part1(id p) {
-}
-#endif
-
-#ifdef PART2
-static inline void part2(id p) {
-}
-#endif
diff --git a/clang/test/ARCMT/Inputs/test1.m.in b/clang/test/ARCMT/Inputs/test1.m.in
deleted file mode 100644
index 44a3c4cf3d93f..0000000000000
--- a/clang/test/ARCMT/Inputs/test1.m.in
+++ /dev/null
@@ -1,16 +0,0 @@
-#define PART1
-#include "test.h"
-
-void test1(id p) {
-  [p release];
-}
-
-@interface Test2
-@property (strong) id prop;
-@end
-
-@implementation Test2
--(id)init {
-  _prop = 0;
-}
-@end
diff --git a/clang/test/ARCMT/Inputs/test1.m.in.result b/clang/test/ARCMT/Inputs/test1.m.in.result
deleted file mode 100644
index 1db9bf7ad65c7..0000000000000
--- a/clang/test/ARCMT/Inputs/test1.m.in.result
+++ /dev/null
@@ -1,15 +0,0 @@
-#define PART1
-#include "test.h"
-
-void test1(id p) {
-}
-
-@interface Test2
-@property (strong) id prop;
-@end
-
-@implementation Test2
--(id)init {
-  _prop = 0;
-}
-@end
diff --git a/clang/test/ARCMT/Inputs/test2.m.in b/clang/test/ARCMT/Inputs/test2.m.in
deleted file mode 100644
index 99f87b0721716..0000000000000
--- a/clang/test/ARCMT/Inputs/test2.m.in
+++ /dev/null
@@ -1,6 +0,0 @@
-#define PART2
-#include "test.h"
-
-void test2(id p) {
-  [p release];
-}
diff --git a/clang/test/ARCMT/Inputs/test2.m.in.result b/clang/test/ARCMT/Inputs/test2.m.in.result
deleted file mode 100644
index f8e918ce2598e..0000000000000
--- a/clang/test/ARCMT/Inputs/test2.m.in.result
+++ /dev/null
@@ -1,5 +0,0 @@
-#define PART2
-#include "test.h"
-
-void test2(id p) {
-}
diff --git a/clang/test/ARCMT/Inputs/with space/test.h b/clang/test/ARCMT/Inputs/with space/test.h
deleted file mode 100644
index 756295f27e698..0000000000000
--- a/clang/test/ARCMT/Inputs/with space/test.h	
+++ /dev/null
@@ -1,15 +0,0 @@
-@protocol NSObject
-- (oneway void)release;
-@end
-
-#ifdef PART1
-static inline void part1(id p) {
-  [p release];
-}
-#endif
-
-#ifdef PART2
-static inline void part2(id p) {
-  [p release];
-}
-#endif
diff --git a/clang/test/ARCMT/Inputs/with space/test.h.result b/clang/test/ARCMT/Inputs/with space/test.h.result
deleted file mode 100644
index 0638a3378c1c0..0000000000000
--- a/clang/test/ARCMT/Inputs/with space/test.h.result	
+++ /dev/null
@@ -1,13 +0,0 @@
-@protocol NSObject
-- (oneway void)release;
-@end
-
-#ifdef PART1
-static inline void part1(id p) {
-}
-#endif
-
-#ifdef PART2
-static inline void part2(id p) {
-}
-#endif
diff --git a/clang/test/ARCMT/Inputs/with space/test1.m.in b/clang/test/ARCMT/Inputs/with space/test1.m.in
deleted file mode 100644
index 8416a88965696..0000000000000
--- a/clang/test/ARCMT/Inputs/with space/test1.m.in	
+++ /dev/null
@@ -1,6 +0,0 @@
-#define PART1
-#include "test.h"
-
-void test1(id p) {
-  [p release];
-}
diff --git a/clang/test/ARCMT/Inputs/with space/test1.m.in.result b/clang/test/ARCMT/Inputs/with space/test1.m.in.result
deleted file mode 100644
index f351fe6c83552..0000000000000
--- a/clang/test/ARCMT/Inputs/with space/test1.m.in.result	
+++ /dev/null
@@ -1,5 +0,0 @@
-#define PART1
-#include "test.h"
-
-void test1(id p) {
-}
diff --git a/clang/test/ARCMT/Inputs/with space/test2.m.in b/clang/test/ARCMT/Inputs/with space/test2.m.in
deleted file mode 100644
index 99f87b0721716..0000000000000
--- a/clang/test/ARCMT/Inputs/with space/test2.m.in	
+++ /dev/null
@@ -1,6 +0,0 @@
-#define PART2
-#include "test.h"
-
-void test2(id p) {
-  [p release];
-}
diff --git a/clang/test/ARCMT/Inputs/with space/test2.m.in.result b/clang/test/ARCMT/Inputs/with space/test2.m.in.result
deleted file mode 100644
index f8e918ce2598e..0000000000000
--- a/clang/test/ARCMT/Inputs/with space/test2.m.in.result	
+++ /dev/null
@@ -1,5 +0,0 @@
-#define PART2
-#include "test.h"
-
-void test2(id p) {
-}
diff --git a/clang/test/ARCMT/allowlisted/Inputs/header1.h b/clang/test/ARCMT/allowlisted/Inputs/header1.h
deleted file mode 100644
index 44430f3b77e32..0000000000000
--- a/clang/test/ARCMT/allowlisted/Inputs/header1.h
+++ /dev/null
@@ -1 +0,0 @@
-// the contents are not important
diff --git a/clang/test/ARCMT/allowlisted/header1.h b/clang/test/ARCMT/allowlisted/header1.h
deleted file mode 100644
index 33f77aa5dd152..0000000000000
--- a/clang/test/ARCMT/allowlisted/header1.h
+++ /dev/null
@@ -1,8 +0,0 @@
-
-@interface I1 : NSObject
--(int)prop;
--(void)setProp:(int)p;
-+(id)i1;
-@end
-
-typedef long NSInteger;
diff --git a/clang/test/ARCMT/allowlisted/header1.h.result b/clang/test/ARCMT/allowlisted/header1.h.result
deleted file mode 100644
index c7cf109a27e92..0000000000000
--- a/clang/test/ARCMT/allowlisted/header1.h.result
+++ /dev/null
@@ -1,7 +0,0 @@
-
-@interface I1 : NSObject
-@property (nonatomic) int prop;
-+(instancetype)i1;
-@end
-
-typedef long NSInteger;
diff --git a/clang/test/ARCMT/allowlisted/header2.h b/clang/test/ARCMT/allowlisted/header2.h
deleted file mode 100644
index ac3888ccdf8c1..0000000000000
--- a/clang/test/ARCMT/allowlisted/header2.h
+++ /dev/null
@@ -1,8 +0,0 @@
-
-#define NS_ENUM(_type, _name) enum _name : _type _name; enum _name : _type
-typedef enum : NSInteger {five} ApplicableEnum;
-
-@interface I2 : NSObject
--(int)prop;
--(void)setProp:(int)p;
-@end
diff --git a/clang/test/ARCMT/allowlisted/header2.h.result b/clang/test/ARCMT/allowlisted/header2.h.result
deleted file mode 100644
index 3226e711b98ac..0000000000000
--- a/clang/test/ARCMT/allowlisted/header2.h.result
+++ /dev/null
@@ -1,7 +0,0 @@
-
-#define NS_ENUM(_type, _name) enum _name : _type _name; enum _name : _type
-typedef NS_ENUM(NSInteger, ApplicableEnum) {five};
-
-@interface I2 : NSObject
-@property (nonatomic) int prop;
-@end
diff --git a/clang/test/ARCMT/allowlisted/objcmt-with-allowlist-impl.m b/clang/test/ARCMT/allowlisted/objcmt-with-allowlist-impl.m
deleted file mode 100644
index 36797eb1c843b..0000000000000
--- a/clang/test/ARCMT/allowlisted/objcmt-with-allowlist-impl.m
+++ /dev/null
@@ -1,18 +0,0 @@
-// RUN: rm -rf %t
-// RUN: %clang_cc1 -objcmt-migrate-readwrite-property -objcmt-migrate-instancetype -objcmt-allowlist-dir-path=%S/Inputs %s -triple x86_64-apple-darwin11 -migrate -o %t.remap
-// RUN: c-arcmt-test %t.remap | arcmt-test -verify-transformed-files %S/header1.h.result %s.result
-
-@interface NSObject
-+ (id)alloc;
-@end
-
-#include "header1.h"
-#include "header2.h"
-
-@interface I2(cat)
--(id)initInCat;
-@end
-
-@implementation I1
-+(id)i1 {}
-@end
diff --git a/clang/test/ARCMT/allowlisted/objcmt-with-allowlist-impl.m.result b/clang/test/ARCMT/allowlisted/objcmt-with-allowlist-impl.m.result
deleted file mode 100644
index 69e7b767ef2b7..0000000000000
--- a/clang/test/ARCMT/allowlisted/objcmt-with-allowlist-impl.m.result
+++ /dev/null
@@ -1,18 +0,0 @@
-// RUN: rm -rf %t
-// RUN: %clang_cc1 -objcmt-migrate-readwrite-property -objcmt-migrate-instancetype -objcmt-allowlist-dir-path=%S/Inputs %s -triple x86_64-apple-darwin11 -migrate -o %t.remap
-// RUN: c-arcmt-test %t.remap | arcmt-test -verify-transformed-files %S/header1.h.result %s.result
-
-@interface NSObject
-+ (id)alloc;
-@end
-
-#include "header1.h"
-#include "header2.h"
-
-@interface I2(cat)
--(id)initInCat;
-@end
-
-@implementation I1
-+(instancetype)i1 {}
-@end
diff --git a/clang/test/ARCMT/allowlisted/objcmt-with-allowlist.m b/clang/test/ARCMT/allowlisted/objcmt-with-allowlist.m
deleted file mode 100644
index c22c02d3bf2e4..0000000000000
--- a/clang/test/ARCMT/allowlisted/objcmt-with-allowlist.m
+++ /dev/null
@@ -1,12 +0,0 @@
-// RUN: rm -rf %t
-// RUN: %clang_cc1 -objcmt-migrate-readwrite-property -objcmt-migrate-instancetype -objcmt-migrate-ns-macros %s -triple x86_64-apple-darwin11 -migrate -o %t.remap
-// RUN: c-arcmt-test %t.remap | arcmt-test -verify-transformed-files %S/header1.h.result %S/header2.h.result
-// RUN: %clang_cc1 -objcmt-migrate-readwrite-property -objcmt-migrate-instancetype -objcmt-migrate-ns-macros -objcmt-allowlist-dir-path=%S/Inputs %s -triple x86_64-apple-darwin11 -migrate -o %t.remap
-// RUN: c-arcmt-test %t.remap | arcmt-test -verify-transformed-files %S/header1.h.result
-
-@interface NSObject
-+ (id)alloc;
-@end
-
-#include "header1.h"
-#include "header2.h"
diff --git a/clang/test/ARCMT/api.m b/clang/test/ARCMT/api.m
deleted file mode 100644
index b186ec724745c..0000000000000
--- a/clang/test/ARCMT/api.m
+++ /dev/null
@@ -1,9 +0,0 @@
-// RUN: %clang_cc1 -triple x86_64-apple-darwin10 -fsyntax-only -fobjc-arc -x objective-c %s.result
-// RUN: arcmt-test --args -triple x86_64-apple-darwin10 -fsyntax-only -x objective-c %s > %t
-// RUN: diff %t %s.result
-
-#include "Common.h"
-
-void test(NSObject *o) {
-  NSZone *z = [o zone];
-}
diff --git a/clang/test/ARCMT/api.m.result b/clang/test/ARCMT/api.m.result
deleted file mode 100644
index e3093751b626b..0000000000000
--- a/clang/test/ARCMT/api.m.result
+++ /dev/null
@@ -1,9 +0,0 @@
-// RUN: %clang_cc1 -triple x86_64-apple-darwin10 -fsyntax-only -fobjc-arc -x objective-c %s.result
-// RUN: arcmt-test --args -triple x86_64-apple-darwin10 -fsyntax-only -x objective-c %s > %t
-// RUN: diff %t %s.result
-
-#include "Common.h"
-
-void test(NSObject *o) {
-  NSZone *z = nil;
-}
diff --git a/clang/test/ARCMT/assign-prop-no-arc-runtime.m b/clang/test/ARCMT/assign-prop-no-arc-runtime.m
deleted file mode 100644
index de1c456b3d19f..0000000000000
--- a/clang/test/ARCMT/assign-prop-no-arc-runtime.m
+++ /dev/null
@@ -1,15 +0,0 @@
-// RUN: %clang_cc1 -triple x86_64-apple-darwin10 -fsyntax-only -fobjc-arc -x objective-c %s.result
-// RUN: arcmt-test --args -triple x86_64-apple-macosx10.6 -fsyntax-only %s > %t
-// RUN: diff %t %s.result
-
-#include "Common.h"
-
-@interface Foo : NSObject {
-  NSObject *x;
-}
-@property (readonly,assign) id x;
-@end
-
-@implementation Foo
-@synthesize x;
-@end
diff --git a/clang/test/ARCMT/assign-prop-no-arc-runtime.m.result b/clang/test/ARCMT/assign-prop-no-arc-runtime.m.result
deleted file mode 100644
index 23848d357268f..0000000000000
--- a/clang/test/ARCMT/assign-prop-no-arc-runtime.m.result
+++ /dev/null
@@ -1,15 +0,0 @@
-// RUN: %clang_cc1 -triple x86_64-apple-darwin10 -fsyntax-only -fobjc-arc -x objective-c %s.result
-// RUN: arcmt-test --args -triple x86_64-apple-macosx10.6 -fsyntax-only %s > %t
-// RUN: diff %t %s.result
-
-#include "Common.h"
-
-@interface Foo : NSObject {
-  NSObject *__unsafe_unretained x;
-}
-@property (readonly,unsafe_unretained) id x;
-@end
-
-@implementation Foo
-@synthesize x;
-@end
diff --git a/clang/test/ARCMT/assign-prop-with-arc-runtime.m b/clang/test/ARCMT/assign-prop-with-arc-runtime.m
deleted file mode 100644
index a00538cd6197d..0000000000000
--- a/clang/test/ARCMT/assign-prop-with-arc-runtime.m
+++ /dev/null
@@ -1,72 +0,0 @@
-// RUN: %clang_cc1 -triple x86_64-apple-darwin10 -fsyntax-only -fobjc-arc -fobjc-runtime-has-weak -x objective-c %s.result
-// RUN: arcmt-test --args -triple x86_64-apple-macosx10.7 -fsyntax-only %s > %t
-// RUN: diff %t %s.result
-
-#include "Common.h"
-
-__attribute__((objc_arc_weak_reference_unavailable))
-@interface WeakOptOut
-@end
-
-@class _NSCachedAttributedString;
-typedef _NSCachedAttributedString *BadClassForWeak;
-
-@class Forw;
-
-@interface Foo : NSObject {
-  Foo *x, *w, *q1, *q2;
-  WeakOptOut *oo;
-  BadClassForWeak bcw;
-  id not_safe1;
-  NSObject *not_safe2;
-  Forw *not_safe3;
-  Foo *assign_plus1;
-}
-@property (readonly) Foo *x;
-@property (assign) Foo *w;
-@property Foo *q1, *q2;
-@property (assign) WeakOptOut *oo;
-@property (assign) BadClassForWeak bcw;
-@property (assign) id not_safe1;
-@property () NSObject *not_safe2;
-@property Forw *not_safe3;
-@property (readonly) Foo *assign_plus1;
-@property (readonly) Foo *assign_plus2;
-@property (readonly) Foo *assign_plus3;
-
-@property (assign) Foo *no_user_ivar1;
-@property (readonly) Foo *no_user_ivar2;
-
-@property (retain) id def1;
-@property (atomic,retain) id def2;
-@property (retain,atomic) id def3;
-
-@end
-
-@implementation Foo
-@synthesize x,w,q1,q2,oo,bcw,not_safe1,not_safe2,not_safe3;
-@synthesize no_user_ivar1, no_user_ivar2;
-@synthesize assign_plus1, assign_plus2, assign_plus3;
-@synthesize def1, def2, def3;
-
--(void)test:(Foo *)parm {
-  assign_plus1 = [[Foo alloc] init];
-  assign_plus2 = [Foo new];
-  assign_plus3 = [parm retain];
-}
-@end
-
-@interface TestExt
-@property (retain,readonly) TestExt *x1;
-@property (readonly) TestExt *x2;
-@end
-
-@interface TestExt()
-@property (retain,readwrite) TestExt *x1;
-@property (readwrite) TestExt *x2;
-@property (retain) TestExt *x3;
-@end
-
-@implementation TestExt
-@synthesize x1, x2, x3;
-@end
diff --git a/clang/test/ARCMT/assign-prop-with-arc-runtime.m.result b/clang/test/ARCMT/assign-prop-with-arc-runtime.m.result
deleted file mode 100644
index 8bb684f79a339..0000000000000
--- a/clang/test/ARCMT/assign-prop-with-arc-runtime.m.result
+++ /dev/null
@@ -1,72 +0,0 @@
-// RUN: %clang_cc1 -triple x86_64-apple-darwin10 -fsyntax-only -fobjc-arc -fobjc-runtime-has-weak -x objective-c %s.result
-// RUN: arcmt-test --args -triple x86_64-apple-macosx10.7 -fsyntax-only %s > %t
-// RUN: diff %t %s.result
-
-#include "Common.h"
-
-__attribute__((objc_arc_weak_reference_unavailable))
-@interface WeakOptOut
-@end
-
-@class _NSCachedAttributedString;
-typedef _NSCachedAttributedString *BadClassForWeak;
-
-@class Forw;
-
-@interface Foo : NSObject {
-  Foo *__weak x, *__weak w, *__weak q1, *__weak q2;
-  WeakOptOut *__unsafe_unretained oo;
-  BadClassForWeak __unsafe_unretained bcw;
-  id __unsafe_unretained not_safe1;
-  NSObject *__unsafe_unretained not_safe2;
-  Forw *__unsafe_unretained not_safe3;
-  Foo *assign_plus1;
-}
-@property (weak, readonly) Foo *x;
-@property (weak) Foo *w;
-@property (weak) Foo *q1, *q2;
-@property (unsafe_unretained) WeakOptOut *oo;
-@property (unsafe_unretained) BadClassForWeak bcw;
-@property (unsafe_unretained) id not_safe1;
-@property (unsafe_unretained) NSObject *not_safe2;
-@property (unsafe_unretained) Forw *not_safe3;
-@property (readonly) Foo *assign_plus1;
-@property (readonly) Foo *assign_plus2;
-@property (readonly) Foo *assign_plus3;
-
-@property (weak) Foo *no_user_ivar1;
-@property (weak, readonly) Foo *no_user_ivar2;
-
-@property (strong) id def1;
-@property (atomic,strong) id def2;
-@property (strong,atomic) id def3;
-
-@end
-
-@implementation Foo
-@synthesize x,w,q1,q2,oo,bcw,not_safe1,not_safe2,not_safe3;
-@synthesize no_user_ivar1, no_user_ivar2;
-@synthesize assign_plus1, assign_plus2, assign_plus3;
-@synthesize def1, def2, def3;
-
--(void)test:(Foo *)parm {
-  assign_plus1 = [[Foo alloc] init];
-  assign_plus2 = [Foo new];
-  assign_plus3 = parm;
-}
-@end
-
-@interface TestExt
-@property (strong,readonly) TestExt *x1;
-@property (weak, readonly) TestExt *x2;
-@end
-
-@interface TestExt()
-@property (strong,readwrite) TestExt *x1;
-@property (weak, readwrite) TestExt *x2;
-@property (strong) TestExt *x3;
-@end
-
-@implementation TestExt
-@synthesize x1, x2, x3;
-@end
diff --git a/clang/test/ARCMT/atautorelease-2.m b/clang/test/ARCMT/atautorelease-2.m
deleted file mode 100644
index b9bc106553259..0000000000000
--- a/clang/test/ARCMT/atautorelease-2.m
+++ /dev/null
@@ -1,29 +0,0 @@
-// RUN: %clang_cc1 -triple x86_64-apple-darwin10 -fsyntax-only -fobjc-arc -x objective-c %s.result
-// RUN: arcmt-test --args -triple x86_64-apple-darwin10 -fsyntax-only -x objective-c %s > %t
-// RUN: diff %t %s.result
-
-@interface NSAutoreleasePool
-- drain;
-+new;
-+alloc;
--init;
--autorelease;
--release;
-@end
-
-void NSLog(id, ...);
-
-int main (int argc, const char * argv[]) {
-    NSAutoreleasePool * pool = [[NSAutoreleasePool alloc] init];
-    NSAutoreleasePool *chunkPool = [[NSAutoreleasePool alloc] init];
-
-    while (argc) {
-      [chunkPool release];
-      return 0;
-    }
-
-    [chunkPool drain];
-    [pool drain];
-
-    return 0;
-}
diff --git a/clang/test/ARCMT/atautorelease-2.m.result b/clang/test/ARCMT/atautorelease-2.m.result
deleted file mode 100644
index 205473380b739..0000000000000
--- a/clang/test/ARCMT/atautorelease-2.m.result
+++ /dev/null
@@ -1,28 +0,0 @@
-// RUN: %clang_cc1 -triple x86_64-apple-darwin10 -fsyntax-only -fobjc-arc -x objective-c %s.result
-// RUN: arcmt-test --args -triple x86_64-apple-darwin10 -fsyntax-only -x objective-c %s > %t
-// RUN: diff %t %s.result
-
-@interface NSAutoreleasePool
-- drain;
-+new;
-+alloc;
--init;
--autorelease;
--release;
-@end
-
-void NSLog(id, ...);
-
-int main (int argc, const char * argv[]) {
-    @autoreleasepool {
-        @autoreleasepool {
-
-            while (argc) {
-              return 0;
-            }
-
-        }
-    }
-
-    return 0;
-}
diff --git a/clang/test/ARCMT/atautorelease-3.m b/clang/test/ARCMT/atautorelease-3.m
deleted file mode 100644
index 87b80af9350eb..0000000000000
--- a/clang/test/ARCMT/atautorelease-3.m
+++ /dev/null
@@ -1,40 +0,0 @@
-// RUN: %clang_cc1 -triple x86_64-apple-darwin10 -fsyntax-only -fobjc-arc -x objective-c %s.result
-// RUN: arcmt-test --args -triple x86_64-apple-darwin10 -fsyntax-only -x objective-c %s > %t
-// RUN: diff %t %s.result
-
-@interface NSAutoreleasePool
-- drain;
-+new;
-+alloc;
--init;
--autorelease;
-- release;
-@end
-
-void NSLog(id, ...);
-
-void test1(int x) {
-  // All this stuff get removed since nothing is happening inside.
-  NSAutoreleasePool * pool = [[NSAutoreleasePool alloc] init];
-  NSAutoreleasePool *chunkPool = [[NSAutoreleasePool alloc] init];
-  while (x) {
-    chunkPool = [[NSAutoreleasePool alloc] init];
-    [chunkPool release];
-  }
-
-  [chunkPool drain];
-  [pool drain];
-}
-
-void test2(int x) {
-  NSAutoreleasePool * pool = [[NSAutoreleasePool alloc] init];
-  NSAutoreleasePool *chunkPool = [[NSAutoreleasePool alloc] init];
-  while (x) {
-    chunkPool = [[NSAutoreleasePool alloc] init];
-    ++x;
-    [chunkPool release];
-  }
-
-  [chunkPool drain];
-  [pool drain];
-}
diff --git a/clang/test/ARCMT/atautorelease-3.m.result b/clang/test/ARCMT/atautorelease-3.m.result
deleted file mode 100644
index 801376a7e82bd..0000000000000
--- a/clang/test/ARCMT/atautorelease-3.m.result
+++ /dev/null
@@ -1,31 +0,0 @@
-// RUN: %clang_cc1 -triple x86_64-apple-darwin10 -fsyntax-only -fobjc-arc -x objective-c %s.result
-// RUN: arcmt-test --args -triple x86_64-apple-darwin10 -fsyntax-only -x objective-c %s > %t
-// RUN: diff %t %s.result
-
-@interface NSAutoreleasePool
-- drain;
-+new;
-+alloc;
--init;
--autorelease;
-- release;
-@end
-
-void NSLog(id, ...);
-
-void test1(int x) {
-  // All this stuff get removed since nothing is happening inside.
-}
-
-void test2(int x) {
-  @autoreleasepool {
-    @autoreleasepool {
-      while (x) {
-        @autoreleasepool {
-          ++x;
-        }
-      }
-
-    }
-  }
-}
diff --git a/clang/test/ARCMT/atautorelease-check.m b/clang/test/ARCMT/atautorelease-check.m
deleted file mode 100644
index 5f8ffa8bc40ee..0000000000000
--- a/clang/test/ARCMT/atautorelease-check.m
+++ /dev/null
@@ -1,144 +0,0 @@
-// RUN: %clang_cc1 -arcmt-action=check -verify -triple x86_64-apple-darwin10 %s
-
-#if __has_feature(objc_arr)
-#define NS_AUTOMATED_REFCOUNT_UNAVAILABLE __attribute__((unavailable("not available in automatic reference counting mode")))
-#else
-#define NS_AUTOMATED_REFCOUNT_UNAVAILABLE
-#endif
-
-typedef struct _NSZone NSZone;
-typedef int BOOL;
-typedef unsigned NSUInteger;
-
-@protocol NSObject
-- (BOOL)isEqual:(id)object;
-- (id)retain NS_AUTOMATED_REFCOUNT_UNAVAILABLE;
-- (NSUInteger)retainCount NS_AUTOMATED_REFCOUNT_UNAVAILABLE;
-- (oneway void)release NS_AUTOMATED_REFCOUNT_UNAVAILABLE;
-- (id)autorelease NS_AUTOMATED_REFCOUNT_UNAVAILABLE;
-
-- (NSZone *)zone NS_AUTOMATED_REFCOUNT_UNAVAILABLE;
-@end
-
-@protocol NSCopying
-- (id)copyWithZone:(NSZone *)zone;
-@end
-
-@protocol NSMutableCopying
-- (id)mutableCopyWithZone:(NSZone *)zone;
-@end
-
-@interface NSObject <NSObject> {}
-- (id)init;
-
-+ (id)new;
-+ (id)allocWithZone:(NSZone *)zone NS_AUTOMATED_REFCOUNT_UNAVAILABLE;
-+ (id)alloc;
-- (void)dealloc;
-
-- (void)finalize;
-
-- (id)copy;
-- (id)mutableCopy;
-
-+ (id)copyWithZone:(NSZone *)zone NS_AUTOMATED_REFCOUNT_UNAVAILABLE;
-+ (id)mutableCopyWithZone:(NSZone *)zone NS_AUTOMATED_REFCOUNT_UNAVAILABLE;
-@end
-
-extern void NSRecycleZone(NSZone *zone);
-
-NS_AUTOMATED_REFCOUNT_UNAVAILABLE
-@interface NSAutoreleasePool : NSObject { // expected-note 13 {{marked unavailable here}}
-@private
-    void    *_token;
-    void    *_reserved3;
-    void    *_reserved2;
-    void    *_reserved;
-}
-
-+ (void)addObject:(id)anObject;
-
-- (void)addObject:(id)anObject;
-
-- (void)drain;
-
-@end
-
-
-void NSLog(id, ...);
-
-int main (int argc, const char * argv[]) {
-    NSAutoreleasePool * pool = [[NSAutoreleasePool alloc] init]; 
-    NSAutoreleasePool *chunkPool = [[NSAutoreleasePool alloc] init]; // expected-error 2 {{'NSAutoreleasePool' is unavailable}}
-
-    while (argc) {
-      [chunkPool release];
-      // the following pool was not released in this scope, don't touch it. 
-      chunkPool = [[NSAutoreleasePool alloc] init]; // expected-error {{'NSAutoreleasePool' is unavailable}}
-    }
-
-    [chunkPool drain];
-    [pool drain];
-
-    return 0;
-}
-
-void f(void) {
-    NSAutoreleasePool * pool;  // expected-error {{'NSAutoreleasePool' is unavailable}}
-
-    for (int i=0; i != 10; ++i) {
-      id x = pool; // We won't touch a NSAutoreleasePool if we can't safely
-                   // remove all the references to it.
-    }
-
-    pool = [[NSAutoreleasePool alloc] init];  // expected-error {{'NSAutoreleasePool' is unavailable}}
-    NSLog(@"%s", "YES");
-    [pool release];
-}
-
-void f2(void) {
-    NSAutoreleasePool *pool = [[NSAutoreleasePool alloc] init]; // expected-error 2 {{'NSAutoreleasePool' is unavailable}} \
-                                            // expected-note {{scope begins here}}
-
-    // 'x' is declared inside the "pool scope" but used outside it, if we create
-    // a @autorelease scope it will be undefined outside it so don't touch the pool.
-    int x = 0; // expected-note {{declared here}}
-
-    [pool release]; // expected-note {{scope ends here}}
-    
-    ++x; // expected-error {{a name is referenced outside the NSAutoreleasePool scope that it was declared in}}
-}
-
-void f3(void) {
-    NSAutoreleasePool *pool = [[NSAutoreleasePool alloc] init]; // expected-error 2 {{'NSAutoreleasePool' is unavailable}} \
-                                            // expected-note {{scope begins here}}
-
-    struct S { int x; }; // expected-note {{declared here}}
-
-    [pool release]; // expected-note {{scope ends here}}
-
-    struct S *var; // expected-error {{a name is referenced outside the NSAutoreleasePool scope that it was declared in}}
-    var->x = 0;
-}
-
-void f4(void) {
-    NSAutoreleasePool *pool = [[NSAutoreleasePool alloc] init]; // expected-error 2 {{'NSAutoreleasePool' is unavailable}} \
-                                            // expected-note {{scope begins here}}
-
-    enum { Bar }; // expected-note {{declared here}}
-
-    [pool release]; // expected-note {{scope ends here}}
-
-    int x = Bar; // expected-error {{a name is referenced outside the NSAutoreleasePool scope that it was declared in}}
-}
-
-void f5(void) {
-    NSAutoreleasePool *pool = [[NSAutoreleasePool alloc] init]; // expected-error 2 {{'NSAutoreleasePool' is unavailable}} \
-                                            // expected-note {{scope begins here}}
-
-    typedef int Bar; // expected-note {{declared here}}
-
-    [pool release]; // expected-note {{scope ends here}}
-
-    Bar x; // expected-error {{a name is referenced outside the NSAutoreleasePool scope that it was declared in}}
-}
diff --git a/clang/test/ARCMT/atautorelease.m b/clang/test/ARCMT/atautorelease.m
deleted file mode 100644
index a6aed146497b4..0000000000000
--- a/clang/test/ARCMT/atautorelease.m
+++ /dev/null
@@ -1,61 +0,0 @@
-// RUN: %clang_cc1 -triple x86_64-apple-darwin10 -fsyntax-only -fobjc-arc -x objective-c %s.result
-// RUN: arcmt-test --args -triple x86_64-apple-darwin10 -fsyntax-only -x objective-c %s > %t
-// RUN: diff %t %s.result
-
-#include "Common.h"
-
-void NSLog(id, ...);
-
-int main (int argc, const char * argv[]) {
-
-    NSAutoreleasePool * pool = [[NSAutoreleasePool alloc] init];
-
-    if (argc) {
-        NSAutoreleasePool * pool = [NSAutoreleasePool  new];
-        NSLog(@"%s", "YES");
-        [pool drain];
-    }
-    [pool drain];
-
-    NSAutoreleasePool * pool1 = [[NSAutoreleasePool alloc] init];
-    NSLog(@"%s", "YES");
-    [pool1 release];
-
-    return 0;
-}
-
-void f(void) {
-  NSAutoreleasePool *pool1;
-
-  pool1 = [NSAutoreleasePool new];
-  int x = 4;
-
-  NSAutoreleasePool *pool2 = [[NSAutoreleasePool alloc] init];
-  ++x;
-  [pool2 drain];
-
-  [pool1 release];
-}
-
-int UIApplicationMain(int argc, char *argv[]);
-
-int main2(int argc, char *argv[]) {
-    NSAutoreleasePool * pool = [[NSAutoreleasePool alloc] init];
-    int result = UIApplicationMain(argc, argv);
-    [pool release];
-    return result;
-}
-
-@interface Foo : NSObject
-@property (assign) id myProp;
-@end
-
-@implementation Foo
-@synthesize myProp;
-
--(void)test:(id)p {
-  NSAutoreleasePool *pool = [[NSAutoreleasePool alloc] init];
-  [pool drain];
-  self.myProp = p;
-}
-@end
diff --git a/clang/test/ARCMT/atautorelease.m.result b/clang/test/ARCMT/atautorelease.m.result
deleted file mode 100644
index e24339a3b9e36..0000000000000
--- a/clang/test/ARCMT/atautorelease.m.result
+++ /dev/null
@@ -1,60 +0,0 @@
-// RUN: %clang_cc1 -triple x86_64-apple-darwin10 -fsyntax-only -fobjc-arc -x objective-c %s.result
-// RUN: arcmt-test --args -triple x86_64-apple-darwin10 -fsyntax-only -x objective-c %s > %t
-// RUN: diff %t %s.result
-
-#include "Common.h"
-
-void NSLog(id, ...);
-
-int main (int argc, const char * argv[]) {
-
-    @autoreleasepool {
-
-        if (argc) {
-            @autoreleasepool {
-                NSLog(@"%s", "YES");
-            }
-        }
-    }
-
-    @autoreleasepool {
-        NSLog(@"%s", "YES");
-    }
-
-    return 0;
-}
-
-void f(void) {
-
-  @autoreleasepool {
-    int x = 4;
-
-    @autoreleasepool {
-      ++x;
-    }
-
-  }
-}
-
-int UIApplicationMain(int argc, char *argv[]);
-
-int main2(int argc, char *argv[]) {
-    @autoreleasepool {
-        int result = UIApplicationMain(argc, argv);
-        return result;
-    }
-}
-
-@interface Foo : NSObject
-@property (unsafe_unretained) id myProp;
-@end
-
-@implementation Foo
-@synthesize myProp;
-
--(void)test:(id)p {
-  @autoreleasepool {
-  }
-  self.myProp = p;
-}
-@end
diff --git a/clang/test/ARCMT/autoreleases.m b/clang/test/ARCMT/autoreleases.m
deleted file mode 100644
index 7c046dc227a06..0000000000000
--- a/clang/test/ARCMT/autoreleases.m
+++ /dev/null
@@ -1,75 +0,0 @@
-// RUN: %clang_cc1 -triple x86_64-apple-darwin10 -fsyntax-only -fobjc-arc -x objective-c %s.result
-// RUN: arcmt-test --args -triple x86_64-apple-darwin10 -fsyntax-only -x objective-c %s > %t
-// RUN: diff %t %s.result
-
-#include "Common.h"
-
-@interface A : NSObject {
-@package
-    id object;
-}
-@end
-
-@interface B : NSObject {
-  id _prop;
-  xpc_object_t _xpc_prop;
-}
-- (BOOL)containsSelf:(A*)a;
-@property (retain) id prop;
-@property (retain) xpc_object_t xpc_prop;
-@end
-
-@implementation A
-@end
-
-@implementation B
-- (BOOL)containsSelf:(A*)a {
-    return a->object == self;
-}
-
--(id) prop {
-  return _prop;
-}
--(void) setProp:(id) newVal {
-  [_prop autorelease];
-  _prop = [newVal retain];
-}
--(void) setProp2:(CFTypeRef) newVal {
-  [_prop autorelease];
-  _prop = (id)CFRetain(newVal);
-}
-
--(id) xpc_prop {
-  return _xpc_prop;
-}
--(void) setXpc_prop:(xpc_object_t) newVal {
-  [_xpc_prop autorelease];
-  _xpc_prop = xpc_retain(newVal);
-}
-@end
-
-void NSLog(id, ...);
-
-int main (int argc, const char * argv[]) {
-    NSAutoreleasePool * pool = [[NSAutoreleasePool alloc] init];
-    A *a = [[A new] autorelease];
-    B *b = [[B new] autorelease];
-    NSLog(@"%s", [b containsSelf:a] ? "YES" : "NO");
-    [pool drain];
-    return 0;
-}
-
-void test(A *prevVal, A *newVal) {
-  [prevVal autorelease];
-  prevVal = [newVal retain];
-}
-
-id test2(A* val) {
-  [[val retain] autorelease];
-  return val;
-}
-
-void test3(void) {
-  id a = [[A alloc] init];
-  [a autorelease];
-}
diff --git a/clang/test/ARCMT/autoreleases.m.result b/clang/test/ARCMT/autoreleases.m.result
deleted file mode 100644
index 29d00ea60deed..0000000000000
--- a/clang/test/ARCMT/autoreleases.m.result
+++ /dev/null
@@ -1,69 +0,0 @@
-// RUN: %clang_cc1 -triple x86_64-apple-darwin10 -fsyntax-only -fobjc-arc -x objective-c %s.result
-// RUN: arcmt-test --args -triple x86_64-apple-darwin10 -fsyntax-only -x objective-c %s > %t
-// RUN: diff %t %s.result
-
-#include "Common.h"
-
-@interface A : NSObject {
-@package
-    id object;
-}
-@end
-
-@interface B : NSObject {
-  id _prop;
-  xpc_object_t _xpc_prop;
-}
-- (BOOL)containsSelf:(A*)a;
-@property (strong) id prop;
-@property (strong) xpc_object_t xpc_prop;
-@end
-
-@implementation A
-@end
-
-@implementation B
-- (BOOL)containsSelf:(A*)a {
-    return a->object == self;
-}
-
--(id) prop {
-  return _prop;
-}
--(void) setProp:(id) newVal {
-  _prop = newVal;
-}
--(void) setProp2:(CFTypeRef) newVal {
-  _prop = (id)CFBridgingRelease(CFRetain(newVal));
-}
-
--(id) xpc_prop {
-  return _xpc_prop;
-}
--(void) setXpc_prop:(xpc_object_t) newVal {
-  _xpc_prop = newVal;
-}
-@end
-
-void NSLog(id, ...);
-
-int main (int argc, const char * argv[]) {
-    @autoreleasepool {
-        A *a = [A new];
-        B *b = [B new];
-        NSLog(@"%s", [b containsSelf:a] ? "YES" : "NO");
-    }
-    return 0;
-}
-
-void test(A *prevVal, A *newVal) {
-  prevVal = newVal;
-}
-
-id test2(A* val) {
-  return val;
-}
-
-void test3(void) {
-  id a = [[A alloc] init];
-}
diff --git a/clang/test/ARCMT/block_copy_release.m b/clang/test/ARCMT/block_copy_release.m
deleted file mode 100644
index ae3b82660a8e4..0000000000000
--- a/clang/test/ARCMT/block_copy_release.m
+++ /dev/null
@@ -1,17 +0,0 @@
-// RUN: %clang_cc1 -fblocks -fsyntax-only -fobjc-arc -x objective-c %s.result
-// RUN: arcmt-test --args -triple x86_64-apple-darwin10 -fblocks -fsyntax-only -x objective-c %s > %t
-// RUN: diff %t %s.result
-
-#include "Common.h"
-
-typedef void (^blk)(int);
-
-void func(blk b) {
-  blk c = Block_copy(b);
-  Block_release(c);
-}
-
-void func2(id b) {
-  id c = Block_copy(b);
-  Block_release(c);
-}
diff --git a/clang/test/ARCMT/block_copy_release.m.result b/clang/test/ARCMT/block_copy_release.m.result
deleted file mode 100644
index b292b64f17d6b..0000000000000
--- a/clang/test/ARCMT/block_copy_release.m.result
+++ /dev/null
@@ -1,15 +0,0 @@
-// RUN: %clang_cc1 -fblocks -fsyntax-only -fobjc-arc -x objective-c %s.result
-// RUN: arcmt-test --args -triple x86_64-apple-darwin10 -fblocks -fsyntax-only -x objective-c %s > %t
-// RUN: diff %t %s.result
-
-#include "Common.h"
-
-typedef void (^blk)(int);
-
-void func(blk b) {
-  blk c = [b copy];
-}
-
-void func2(id b) {
-  id c = [b copy];
-}
diff --git a/clang/test/ARCMT/check-api.m b/clang/test/ARCMT/check-api.m
deleted file mode 100644
index b395f0b4a4b97..0000000000000
--- a/clang/test/ARCMT/check-api.m
+++ /dev/null
@@ -1,43 +0,0 @@
-// RUN: %clang_cc1 -arcmt-action=check -verify -triple x86_64-apple-macosx10.7 %s
-
-#include "Common.h"
-
-@interface NSInvocation : NSObject
-- (void)getReturnValue:(void *)retLoc;
-- (void)setReturnValue:(void *)retLoc;
-
-- (void)getArgument:(void *)argumentLocation atIndex:(int)idx;
-- (void)setArgument:(void *)argumentLocation atIndex:(int)idx;
-@end
-
-@interface Test
-@end
-
-@implementation Test {
-  id strong_id;
-  __weak id weak_id;
-  __unsafe_unretained id unsafe_id;
-  int arg;
-}
-- (void) test:(NSInvocation *)invok {
-  [invok getReturnValue:&strong_id]; // expected-error {{NSInvocation's getReturnValue is not safe to be used with an object with ownership other than __unsafe_unretained}}
-  [invok getReturnValue:&weak_id]; // expected-error {{NSInvocation's getReturnValue is not safe to be used with an object with ownership other than __unsafe_unretained}}
-  [invok getReturnValue:&unsafe_id];
-  [invok getReturnValue:&arg];
-
-  [invok setReturnValue:&strong_id]; // expected-error {{NSInvocation's setReturnValue is not safe to be used with an object with ownership other than __unsafe_unretained}}
-  [invok setReturnValue:&weak_id]; // expected-error {{NSInvocation's setReturnValue is not safe to be used with an object with ownership other than __unsafe_unretained}}
-  [invok setReturnValue:&unsafe_id];
-  [invok setReturnValue:&arg];
-
-  [invok getArgument:&strong_id atIndex:0]; // expected-error {{NSInvocation's getArgument is not safe to be used with an object with ownership other than __unsafe_unretained}}
-  [invok getArgument:&weak_id atIndex:0]; // expected-error {{NSInvocation's getArgument is not safe to be used with an object with ownership other than __unsafe_unretained}}
-  [invok getArgument:&unsafe_id atIndex:0];
-  [invok getArgument:&arg atIndex:0];
-
-  [invok setArgument:&strong_id atIndex:0]; // expected-error {{NSInvocation's setArgument is not safe to be used with an object with ownership other than __unsafe_unretained}}
-  [invok setArgument:&weak_id atIndex:0]; // expected-error {{NSInvocation's setArgument is not safe to be used with an object with ownership other than __unsafe_unretained}}
-  [invok setArgument:&unsafe_id atIndex:0];
-  [invok setArgument:&arg atIndex:0];
-}
-@end
diff --git a/clang/test/ARCMT/check-with-pch.m b/clang/test/ARCMT/check-with-pch.m
deleted file mode 100644
index c2fda3b52cbc9..0000000000000
--- a/clang/test/ARCMT/check-with-pch.m
+++ /dev/null
@@ -1,15 +0,0 @@
-// RUN: %clang_cc1 -x objective-c -triple x86_64-apple-darwin10 %S/Common.h -emit-pch -o %t.pch
-// RUN: %clang_cc1 -include-pch %t.pch -arcmt-action=check -verify -triple x86_64-apple-darwin10 -fblocks -Werror %s
-// REQUIRES: x86-registered-target
-
-@interface I9601437 {
-  __unsafe_unretained id x;
-}
--(void)Meth;
-@end
-
-@implementation I9601437
--(void)Meth {
-  self->x = [NSObject new]; // expected-error {{assigning retained object}}
-}
-@end
diff --git a/clang/test/ARCMT/check-with-serialized-diag.m b/clang/test/ARCMT/check-with-serialized-diag.m
deleted file mode 100644
index 6102be037fcc1..0000000000000
--- a/clang/test/ARCMT/check-with-serialized-diag.m
+++ /dev/null
@@ -1,55 +0,0 @@
-
-@protocol NSObject
-- (id)retain;
-- (unsigned)retainCount;
-- (oneway void)release;
-- (id)autorelease;
-@end
-
-@interface NSObject <NSObject> {}
-- (id)init;
-
-+ (id)new;
-+ (id)alloc;
-- (void)dealloc;
-
-- (void)finalize;
-
-- (id)copy;
-- (id)mutableCopy;
-@end
-
-@interface A : NSObject
-@end
-
-struct UnsafeS {
-  A *__unsafe_unretained unsafeObj;
-};
-
-id global_foo;
-
-void test1(A *a, struct UnsafeS *unsafeS) {
-  [unsafeS->unsafeObj retain];
-  id foo = [unsafeS->unsafeObj retain]; // no warning.
-  [global_foo retain];
-  [a retainCount];
-}
-
-// RUN: not %clang_cc1 -arcmt-action=check -triple x86_64-apple-darwin10 %s -serialize-diagnostic-file %t.diag
-// RUN: c-index-test -read-diagnostics %t.diag > %t 2>&1
-// RUN: FileCheck --input-file=%t %s
-
-// CHECK: {{.*}}check-with-serialized-diag.m:32:4: error: [rewriter] it is not safe to remove 'retain' message on an __unsafe_unretained type
-// CHECK-NEXT: Number FIXITs = 0
-// CHECK-NEXT: {{.*}}check-with-serialized-diag.m:34:4: error: [rewriter] it is not safe to remove 'retain' message on a global variable
-// CHECK-NEXT: Number FIXITs = 0
-// CHECK-NEXT: {{.*}}check-with-serialized-diag.m:32:23: error: ARC forbids explicit message send of 'retain'
-// CHECK-NEXT: Range: {{.*}}check-with-serialized-diag.m:32:4 {{.*}}check-with-serialized-diag.m:32:22
-// CHECK-NEXT: Number FIXITs = 0
-// CHECK-NEXT: {{.*}}check-with-serialized-diag.m:34:15: error: ARC forbids explicit message send of 'retain'
-// CHECK-NEXT: Range: {{.*}}check-with-serialized-diag.m:34:4 {{.*}}check-with-serialized-diag.m:34:14
-// CHECK-NEXT: Number FIXITs = 0
-// CHECK-NEXT: {{.*}}check-with-serialized-diag.m:35:6: error: ARC forbids explicit message send of 'retainCount'
-// CHECK-NEXT: Range: {{.*}}check-with-serialized-diag.m:35:4 {{.*}}check-with-serialized-diag.m:35:5
-// CHECK-NEXT: Number FIXITs = 0
-
diff --git a/clang/test/ARCMT/checking-in-arc.m b/clang/test/ARCMT/checking-in-arc.m
deleted file mode 100644
index 1bf6aca18a342..0000000000000
--- a/clang/test/ARCMT/checking-in-arc.m
+++ /dev/null
@@ -1,50 +0,0 @@
-// RUN: %clang_cc1 -arcmt-action=check -fobjc-arc -fobjc-runtime=macosx-10.8.0 -triple x86_64-apple-darwin12 -fblocks -Werror %s
-
-#if __has_feature(objc_arc)
-#define NS_AUTOMATED_REFCOUNT_UNAVAILABLE __attribute__((unavailable("not available in automatic reference counting mode")))
-#else
-#define NS_AUTOMATED_REFCOUNT_UNAVAILABLE
-#endif
-
-typedef const void * CFTypeRef;
-CFTypeRef CFBridgingRetain(id X);
-id CFBridgingRelease(CFTypeRef);
-
-typedef int BOOL;
-typedef unsigned NSUInteger;
-
-@protocol NSObject
-- (id)retain NS_AUTOMATED_REFCOUNT_UNAVAILABLE;
-- (NSUInteger)retainCount NS_AUTOMATED_REFCOUNT_UNAVAILABLE;
-- (oneway void)release NS_AUTOMATED_REFCOUNT_UNAVAILABLE;
-- (id)autorelease NS_AUTOMATED_REFCOUNT_UNAVAILABLE;
-@end
-
-@interface NSObject <NSObject> {}
-- (id)init;
-
-+ (id)new;
-+ (id)alloc;
-- (void)dealloc;
-
-- (void)finalize;
-
-- (id)copy;
-- (id)mutableCopy;
-@end
-
-typedef const struct __CFString * CFStringRef;
-extern const CFStringRef kUTTypePlainText;
-extern const CFStringRef kUTTypeRTF;
-@class NSString;
-
-@interface Test : NSObject
-@property (weak) NSString *weakProperty;
-@end
-
-@implementation Test
-@end
-
-#if ! __has_feature(objc_arc)
-#error This file must be compiled with ARC (set -fobjc_arc flag on file)
-#endif
diff --git a/clang/test/ARCMT/checking.m b/clang/test/ARCMT/checking.m
deleted file mode 100644
index 5bc456c6301c0..0000000000000
--- a/clang/test/ARCMT/checking.m
+++ /dev/null
@@ -1,351 +0,0 @@
-// RUN: %clang_cc1 -arcmt-action=check -verify -triple x86_64-apple-darwin10 -fblocks -Werror %s
-
-#if __has_feature(objc_arc)
-#define NS_AUTOMATED_REFCOUNT_UNAVAILABLE __attribute__((unavailable("not available in automatic reference counting mode")))
-#else
-#define NS_AUTOMATED_REFCOUNT_UNAVAILABLE
-#endif
-
-typedef const void * CFTypeRef;
-CFTypeRef CFBridgingRetain(id X);
-id CFBridgingRelease(CFTypeRef);
-
-typedef int BOOL;
-typedef unsigned NSUInteger;
-
-@protocol NSObject
-- (id)retain NS_AUTOMATED_REFCOUNT_UNAVAILABLE;
-- (NSUInteger)retainCount NS_AUTOMATED_REFCOUNT_UNAVAILABLE;
-- (oneway void)release NS_AUTOMATED_REFCOUNT_UNAVAILABLE;
-- (id)autorelease NS_AUTOMATED_REFCOUNT_UNAVAILABLE;
-@end
-
-@interface NSObject <NSObject> {}
-- (id)init;
-
-+ (id)new;
-+ (id)alloc;
-- (void)dealloc;
-
-- (void)finalize;
-
-- (id)copy;
-- (id)mutableCopy;
-@end
-
-typedef const struct __CFString * CFStringRef;
-extern const CFStringRef kUTTypePlainText;
-extern const CFStringRef kUTTypeRTF;
-@class NSString;
-@class A;
-
-struct UnsafeS {
-  A *__unsafe_unretained unsafeObj;
-};
-
-@interface A : NSObject
-- (id)retain __attribute__((unavailable)); // expected-note {{'retain' has been explicitly marked unavailable here}}
-- (id)retainCount __attribute__((unavailable)); // expected-note {{'retainCount' has been explicitly marked unavailable here}}
-- (id)autorelease __attribute__((unavailable)); // expected-note 2 {{'autorelease' has been explicitly marked unavailable here}}
-- (id)init;
-- (oneway void)release;
-- (void)dealloc;
--(void)test;
--(id)delegate;
-@end
-
-@implementation A
--(void)test {
-  [super dealloc];
-}
--(void)dealloc {
-  [super dealloc];
-}
-
-- (id)retain { return self; } // expected-error {{ARC forbids implementation}}
-- (id)retainCount { return self; } // expected-error {{ARC forbids implementation}}
-- (id)autorelease { return self; } // expected-error {{ARC forbids implementation}}
-- (oneway void)release { } // expected-error {{ARC forbids implementation}}
-
--(id)delegate { return self; }
-@end
-
-id global_foo;
-
-void test1(A *a, BOOL b, struct UnsafeS *unsafeS) {
-  [[a delegate] release]; // expected-error {{it is not safe to remove 'retain' message on the result of a 'delegate' message; the object that was passed to 'setDelegate:' may not be properly retained}} \
-                          // expected-error {{ARC forbids explicit message send}}
-  [a.delegate release]; // expected-error {{it is not safe to remove 'retain' message on the result of a 'delegate' message; the object that was passed to 'setDelegate:' may not be properly retained}} \
-                        // expected-error {{ARC forbids explicit message send}}
-  [unsafeS->unsafeObj retain]; // expected-error {{it is not safe to remove 'retain' message on an __unsafe_unretained type}} \
-                               // expected-error {{ARC forbids explicit message send}} \
-                               // expected-error {{'retain' is unavailable}}
-  id foo = [unsafeS->unsafeObj retain]; // no warning.
-  [global_foo retain]; // expected-error {{it is not safe to remove 'retain' message on a global variable}} \
-                       // expected-error {{ARC forbids explicit message send}}
-  [global_foo release]; // expected-error {{it is not safe to remove 'release' message on a global variable}} \
-                        // expected-error {{ARC forbids explicit message send}}
-  [a dealloc];
-  [a retain];
-  [a retainCount]; // expected-error {{ARC forbids explicit message send of 'retainCount'}} \
-                   // expected-error {{'retainCount' is unavailable}}
-  [a release];
-  [a autorelease]; // expected-error {{it is not safe to remove an unused 'autorelease' message; its receiver may be destroyed immediately}} \
-                   // expected-error {{ARC forbids explicit message send}} \
-                   // expected-error {{'autorelease' is unavailable}}
-  [a autorelease]; // expected-error {{it is not safe to remove an unused 'autorelease' message; its receiver may be destroyed immediately}} \
-                   // expected-error {{ARC forbids explicit message send}} \
-                   // expected-error {{'autorelease' is unavailable}}
-  a = 0;
-
-  CFStringRef cfstr;
-  NSString *str = (NSString *)cfstr; // expected-error {{cast of C pointer type 'CFStringRef' (aka 'const struct __CFString *') to Objective-C pointer type 'NSString *' requires a bridged cast}} \
-  // expected-note {{use __bridge to convert directly (no change in ownership)}} \
-  // expected-note {{use CFBridgingRelease call to transfer ownership of a +1 'CFStringRef' (aka 'const struct __CFString *') into ARC}} \
-  str = (NSString *)kUTTypePlainText;
-  str = b ? kUTTypeRTF : kUTTypePlainText;
-  str = (NSString *)(b ? kUTTypeRTF : kUTTypePlainText);
-  str = (NSString *)a; // no change.
-
-  SEL s = @selector(retain);  // expected-error {{ARC forbids use of 'retain' in a @selector}}
-  s = @selector(release); // expected-error {{ARC forbids use of 'release' in a @selector}}
-  s = @selector(autorelease); // expected-error {{ARC forbids use of 'autorelease' in a @selector}}
-  s = @selector(dealloc); // expected-error {{ARC forbids use of 'dealloc' in a @selector}}
-
-  static id __autoreleasing X1; // expected-error {{global variables cannot have __autoreleasing ownership}}
-}
-
-struct S {
-  A* a;
-};
-
-@interface B
--(id)alloc;
-- (id)initWithInt: (int) i;
-@end
-
-void rdar8861761(void) {
-  B *o1 = [[B alloc] initWithInt:0];
-  B *o2 = [B alloc];
-  [o2 initWithInt:0];
-}
-
-@interface Test13
-- (id) init0;
-- (void) noninit;
-@end
-@implementation Test13
-- (id) init0 {
-  self = 0;
-}
-- (void) noninit {
-  self = 0; // expected-error {{cannot assign to 'self' outside of a method in the init family}}
-
-  for (__strong id x in collection) { // expected-error {{use of undeclared identifier 'collection'}}
-    x = 0;
-  }
-}
-@end
-
-void * cvt(id arg)
-{
-  void* voidp_val;
-  (void)(int*)arg; // expected-error {{disallowed}}
-  (void)(id)arg;
-  (void)(__autoreleasing id*)arg; // expected-error {{disallowed}}
-  (void)(id*)arg; // expected-error {{disallowed}}
-
-  (void)(__autoreleasing id**)voidp_val;
-  (void)(void*)voidp_val;
-  (void)(void**)arg; // expected-error {{disallowed}}
-  cvt((void*)arg); // expected-error 2 {{requires a bridged cast}} \
-                   // expected-note 2 {{use __bridge to}} expected-note {{use CFBridgingRelease call}} expected-note {{use CFBridgingRetain call}}
-  cvt(0);
-  (void)(__strong id**)(0);
-  return arg; // expected-error {{requires a bridged cast}} expected-note {{use __bridge}} expected-note {{use CFBridgingRetain call}}
-}
-
-
-void test12(id collection) {
-  for (id x in collection) {
-    x = 0;
-  }
-
-  for (__strong id x in collection) {
-    x = 0;
-  }
-}
-
-void test6(unsigned cond) {
-  switch (cond) {
-  case 0:
-    ;
-    id x; // expected-note {{jump bypasses initialization of __strong variable}}
-
-  case 1: // expected-error {{cannot jump}}
-    x = 0;
-    break;
-  }
-}
-
-@class Test8_incomplete;
-@interface Test8_complete @end;
-@interface Test8_super @end;
-@interface Test8 : Test8_super
-- (id) init00;
-- (id) init01; // expected-note {{declaration in interface}}
-- (id) init02;
-- (id) init03; // covariance
-- (id) init04; // covariance
-- (id) init05;
-
-- (void) init10; // expected-note {{declaration in interface is not in the 'init' family because its result type is not an object pointer}}
-- (void) init11;
-- (void) init12;
-- (void) init13; // expected-note {{declaration in interface is not in the 'init' family because its result type is not an object pointer}}
-- (void) init14; // expected-note {{declaration in interface is not in the 'init' family because its result type is not an object pointer}}
-- (void) init15;
-
-// These should be invalid to actually call.
-- (Test8_incomplete*) init20;
-- (Test8_incomplete*) init21; // expected-note {{declaration in interface}}
-- (Test8_incomplete*) init22;
-- (Test8_incomplete*) init23;
-- (Test8_incomplete*) init24;
-- (Test8_incomplete*) init25;
-
-- (Test8_super*) init30; // id exception to covariance
-- (Test8_super*) init31; // expected-note {{declaration in interface}}
-- (Test8_super*) init32;
-- (Test8_super*) init33;
-- (Test8_super*) init34; // covariance
-- (Test8_super*) init35;
-
-- (Test8*) init40; // id exception to covariance
-- (Test8*) init41; // expected-note {{declaration in interface}}
-- (Test8*) init42;
-- (Test8*) init43; // this should be a warning, but that's a general language thing, not an ARC thing
-- (Test8*) init44;
-- (Test8*) init45;
-
-- (Test8_complete*) init50; // expected-error {{init methods must return a type related to the receiver type}}
-- (Test8_complete*) init51; // expected-error {{init methods must return a type related to the receiver type}}
-- (Test8_complete*) init52; // expected-error {{init methods must return a type related to the receiver type}}
-- (Test8_complete*) init53; // expected-error {{init methods must return a type related to the receiver type}}
-- (Test8_complete*) init54; // expected-error {{init methods must return a type related to the receiver type}}
-- (Test8_complete*) init55; // expected-error {{init methods must return a type related to the receiver type}}
-@end
-@implementation Test8
-- (id) init00 { return 0; }
-- (id) init10 { return 0; } // expected-error {{method implementation does not match its declaration}}
-- (id) init20 { return 0; }
-- (id) init30 { return 0; }
-- (id) init40 { return 0; }
-- (id) init50 { return 0; }
-
-- (void) init01 {} // expected-error {{method was declared as an 'init' method, but its implementation doesn't match because its result type is not an object pointer}}
-- (void) init11 {}
-- (void) init21 {} // expected-error {{method was declared as an 'init' method, but its implementation doesn't match because its result type is not an object pointer}}
-- (void) init31 {} // expected-error {{method was declared as an 'init' method, but its implementation doesn't match because its result type is not an object pointer}}
-- (void) init41 {} // expected-error {{method was declared as an 'init' method, but its implementation doesn't match because its result type is not an object pointer}}
-- (void) init51 {}
-
-- (Test8_incomplete*) init02 { return 0; } // expected-error {{init methods must return a type related to the receiver type}}
-- (Test8_incomplete*) init12 { return 0; } // expected-error {{init methods must return a type related to the receiver type}}
-- (Test8_incomplete*) init22 { return 0; } // expected-error {{init methods must return a type related to the receiver type}}
-- (Test8_incomplete*) init32 { return 0; } // expected-error {{init methods must return a type related to the receiver type}}
-- (Test8_incomplete*) init42 { return 0; } // expected-error {{init methods must return a type related to the receiver type}}
-- (Test8_incomplete*) init52 { return 0; } // expected-error {{init methods must return a type related to the receiver type}}
-
-- (Test8_super*) init03 { return 0; }
-- (Test8_super*) init13 { return 0; } // expected-error {{method implementation does not match its declaration}}
-- (Test8_super*) init23 { return 0; }
-- (Test8_super*) init33 { return 0; }
-- (Test8_super*) init43 { return 0; }
-- (Test8_super*) init53 { return 0; }
-
-- (Test8*) init04 { return 0; }
-- (Test8*) init14 { return 0; } // expected-error {{method implementation does not match its declaration}}
-- (Test8*) init24 { return 0; }
-- (Test8*) init34 { return 0; }
-- (Test8*) init44 { return 0; }
-- (Test8*) init54 { return 0; }
-
-- (Test8_complete*) init05 { return 0; } // expected-error {{init methods must return a type related to the receiver type}}
-- (Test8_complete*) init15 { return 0; } // expected-error {{init methods must return a type related to the receiver type}}
-- (Test8_complete*) init25 { return 0; } // expected-error {{init methods must return a type related to the receiver type}}
-- (Test8_complete*) init35 { return 0; } // expected-error {{init methods must return a type related to the receiver type}}
-- (Test8_complete*) init45 { return 0; } // expected-error {{init methods must return a type related to the receiver type}}
-- (Test8_complete*) init55 { return 0; } // expected-error {{init methods must return a type related to the receiver type}}
-@end
-
-@class Test9_incomplete;
-@interface Test9
-- (Test9_incomplete*) init1; // expected-error {{init methods must return a type related to the receiver type}}
-- (Test9_incomplete*) init2;
-@end
-id test9(Test9 *v) {
-  return [v init1];
-}
-
-void rdar9491791(int p) {
-  switch (p) {
-  case 3:;
-    NSObject *o = [[NSObject alloc] init];
-    [o release];
-    break;
-  default:
-    break;
-  }
-}
-
-#define RELEASE_MACRO(x) do { [x release]; } while(1)
-
-void rdar9504750(id p) {
-  RELEASE_MACRO(p); // expected-error {{ARC forbids explicit message send of 'release'}} 
-}
-
-@interface TestReadonlyProperty : NSObject
-@property(assign,readonly) NSObject *value;
-@end
-
-@implementation TestReadonlyProperty
-@synthesize value;
-- (void)viewDidLoad {
-  value = [NSObject new]; // expected-error {{assigning retained object}}
-}
-@end
-
-@interface I9601437 {
-  __unsafe_unretained id x;
-}
--(void)Meth;
-@end
-
-@implementation I9601437
--(void)Meth {
-  self->x = [NSObject new]; // expected-error {{assigning retained object}}
-}
-@end
-
-@interface Test10 : NSObject {
-  CFStringRef cfstr;
-}
-@property (retain) id prop;
--(void)foo;
-@end
-
-void test(Test10 *x) {
-  x.prop = ^{ [x foo]; }; // expected-warning {{likely to lead to a retain cycle}} \
-                          // expected-note {{retained by the captured object}}
-}
-
-@implementation Test10
--(void)foo {
-  ^{
-    NSString *str = (NSString *)cfstr; // expected-error {{cast of C pointer type 'CFStringRef' (aka 'const struct __CFString *') to Objective-C pointer type 'NSString *' requires a bridged cast}} \
-    // expected-note {{use __bridge to convert directly (no change in ownership)}} \
-    // expected-note {{use CFBridgingRelease call to transfer ownership of a +1 'CFStringRef' (aka 'const struct __CFString *') into ARC}}
-  };
-}
-@end
diff --git a/clang/test/ARCMT/cxx-checking.mm b/clang/test/ARCMT/cxx-checking.mm
deleted file mode 100644
index f8836d9d7cd8d..0000000000000
--- a/clang/test/ARCMT/cxx-checking.mm
+++ /dev/null
@@ -1,100 +0,0 @@
-// RUN: %clang_cc1 -arcmt-action=check -verify -triple x86_64-apple-darwin10 -fsyntax-only -fblocks %s
-
-// Classes that have an Objective-C object pointer.
-struct HasObjectMember0 {
-  id x;
-};
-
-struct HasObjectMember1 {
-  id x[3];
-};
-
-struct HasObjectMember2 {
-  id x[3][2];
-};
-
-// Don't complain if the type has non-external linkage
-namespace {
-  struct HasObjectMember3 {
-    id x[3][2];
-  };
-}
-
-// Don't complain if the Objective-C pointer type was explicitly given
-// no lifetime.
-struct HasObjectMember3 { 
-  __unsafe_unretained id x[3][2];
-};
-
-struct HasBlockPointerMember0 {
-  int (^bp)(int);
-};
-
-struct HasBlockPointerMember1 {
-  int (^bp[2][3])(int);
-};
-
-struct NonPOD {
-  NonPOD(const NonPOD&);
-};
-
-struct HasObjectMemberAndNonPOD0 {
-  id x;
-  NonPOD np;
-};
-
-struct HasObjectMemberAndNonPOD1 {
-  NonPOD np;
-  id x[3];
-};
-
-struct HasObjectMemberAndNonPOD2 {
-  NonPOD np;
-  id x[3][2];
-};
-
-struct HasObjectMemberAndNonPOD3 {
-  HasObjectMemberAndNonPOD3 &operator=(const HasObjectMemberAndNonPOD3&);
-  ~HasObjectMemberAndNonPOD3();
-  NonPOD np;
-  id x[3][2];
-};
-
-struct HasBlockPointerMemberAndNonPOD0 {
-  NonPOD np;
-  int (^bp)(int);
-};
-
-struct HasBlockPointerMemberAndNonPOD1 {
-  NonPOD np;
-  int (^bp[2][3])(int);
-};
-
-int check_non_pod_objc_pointer0[__is_pod(id)? 1 : -1];
-int check_non_pod_objc_pointer1[__is_pod(__strong id)? -1 : 1];
-int check_non_pod_objc_pointer2[__is_pod(__unsafe_unretained id)? 1 : -1];
-int check_non_pod_objc_pointer3[__is_pod(id[2][3])? 1 : -1];
-int check_non_pod_objc_pointer4[__is_pod(__unsafe_unretained id[2][3])? 1 : -1];
-int check_non_pod_block0[__is_pod(int (^)(int))? 1 : -1];
-int check_non_pod_block1[__is_pod(int (^ __unsafe_unretained)(int))? 1 : -1];
-
-struct FlexibleArrayMember0 {
-  int length;
-  id array[]; // expected-error{{flexible array member 'array' of type '__strong id[]' with non-trivial destruction}}
-};
-
-struct FlexibleArrayMember1 {
-  int length;
-  __unsafe_unretained id array[];
-};
-
-// It's okay to pass a retainable type through an ellipsis.
-void variadic(...);
-void test_variadic() {
-  variadic(1, 17, @"Foo");
-}
-
-// It's okay to create a VLA of retainable types.
-void vla(int n) {
-  id vla[n];
-}
diff --git a/clang/test/ARCMT/cxx-rewrite.mm b/clang/test/ARCMT/cxx-rewrite.mm
deleted file mode 100644
index 4a9c50c92426e..0000000000000
--- a/clang/test/ARCMT/cxx-rewrite.mm
+++ /dev/null
@@ -1,33 +0,0 @@
-// RUN: %clang_cc1 -triple x86_64-apple-darwin10 -fsyntax-only -fobjc-arc -x objective-c++ %s.result
-// RUN: arcmt-test --args -triple x86_64-apple-darwin10 -fsyntax-only -x objective-c++ %s > %t
-// RUN: diff %t %s.result
-
-#include "Common.h"
-
-@interface NSString : NSObject
-+(id)string;
-@end
-
-struct foo {
-    NSString *s;
-    foo(NSString *s): s([s retain]){
-        NSAutoreleasePool *pool = [NSAutoreleasePool new];
-        [[[NSString string] retain] release];
-        [pool drain];
-        if (s)
-          [s release];
-    }
-    ~foo(){ [s release]; }
-private:
-    foo(foo const &);
-    foo &operator=(foo const &);
-};
-
-int main(){
-    NSAutoreleasePool *pool = [NSAutoreleasePool new];
-
-    foo f([[NSString string] autorelease]);
-
-    [pool drain];
-    return 0;
-}
diff --git a/clang/test/ARCMT/cxx-rewrite.mm.result b/clang/test/ARCMT/cxx-rewrite.mm.result
deleted file mode 100644
index a96d254bf4633..0000000000000
--- a/clang/test/ARCMT/cxx-rewrite.mm.result
+++ /dev/null
@@ -1,31 +0,0 @@
-// RUN: %clang_cc1 -triple x86_64-apple-darwin10 -fsyntax-only -fobjc-arc -x objective-c++ %s.result
-// RUN: arcmt-test --args -triple x86_64-apple-darwin10 -fsyntax-only -x objective-c++ %s > %t
-// RUN: diff %t %s.result
-
-#include "Common.h"
-
-@interface NSString : NSObject
-+(id)string;
-@end
-
-struct foo {
-    NSString *s;
-    foo(NSString *s): s(s){
-        @autoreleasepool {
-            [NSString string];
-        }
-    }
-    ~foo(){  }
-private:
-    foo(foo const &);
-    foo &operator=(foo const &);
-};
-
-int main(){
-    @autoreleasepool {
-
-        foo f([NSString string]);
-
-    }
-    return 0;
-}
diff --git a/clang/test/ARCMT/dealloc.m b/clang/test/ARCMT/dealloc.m
deleted file mode 100644
index d7a72af4f726f..0000000000000
--- a/clang/test/ARCMT/dealloc.m
+++ /dev/null
@@ -1,24 +0,0 @@
-// RUN: %clang_cc1 -triple x86_64-apple-darwin10 -fsyntax-only -fobjc-arc -x objective-c %s.result
-// RUN: arcmt-test --args -triple x86_64-apple-darwin10 -fsyntax-only -x objective-c %s > %t
-// RUN: diff %t %s.result
-
-@interface A
-- (id)retain;
-- (id)autorelease;
-- (oneway void)release;
-- (void)dealloc;
-@end
-
-void test1(A *a) {
-  [a dealloc];
-}
-
-@interface Test2 : A
-- (void) dealloc;
-@end
-
-@implementation Test2
-- (void) dealloc {
-  [super dealloc];
-}
-@end
diff --git a/clang/test/ARCMT/dealloc.m.result b/clang/test/ARCMT/dealloc.m.result
deleted file mode 100644
index fbd9e445d2751..0000000000000
--- a/clang/test/ARCMT/dealloc.m.result
+++ /dev/null
@@ -1,20 +0,0 @@
-// RUN: %clang_cc1 -triple x86_64-apple-darwin10 -fsyntax-only -fobjc-arc -x objective-c %s.result
-// RUN: arcmt-test --args -triple x86_64-apple-darwin10 -fsyntax-only -x objective-c %s > %t
-// RUN: diff %t %s.result
-
-@interface A
-- (id)retain;
-- (id)autorelease;
-- (oneway void)release;
-- (void)dealloc;
-@end
-
-void test1(A *a) {
-}
-
-@interface Test2 : A
-- (void) dealloc;
-@end
-
-@implementation Test2
-@end
diff --git a/clang/test/ARCMT/designated-init-in-header/designated-init-in-header.m b/clang/test/ARCMT/designated-init-in-header/designated-init-in-header.m
deleted file mode 100644
index 8286583b3c898..0000000000000
--- a/clang/test/ARCMT/designated-init-in-header/designated-init-in-header.m
+++ /dev/null
@@ -1,3 +0,0 @@
-// RUN: %clang_cc1 -objcmt-migrate-designated-init -objcmt-migrate-readwrite-property -objcmt-migrate-instancetype -x objective-c %S/file1.m.in -triple x86_64-apple-darwin11 -fobjc-arc -migrate -o %t1.remap
-// RUN: %clang_cc1 -objcmt-migrate-designated-init -objcmt-migrate-readwrite-property -objcmt-migrate-instancetype -x objective-c %S/file2.m.in -triple x86_64-apple-darwin11 -fobjc-arc -migrate -o %t2.remap
-// RUN: c-arcmt-test %t1.remap %t2.remap | arcmt-test -verify-transformed-files %S/header1.h.result %S/file2.m.in.result
diff --git a/clang/test/ARCMT/designated-init-in-header/file1.m.in b/clang/test/ARCMT/designated-init-in-header/file1.m.in
deleted file mode 100644
index 0201b32abd323..0000000000000
--- a/clang/test/ARCMT/designated-init-in-header/file1.m.in
+++ /dev/null
@@ -1,2 +0,0 @@
-#include "header1.h"
-
diff --git a/clang/test/ARCMT/designated-init-in-header/file2.m.in b/clang/test/ARCMT/designated-init-in-header/file2.m.in
deleted file mode 100644
index 258159735a77f..0000000000000
--- a/clang/test/ARCMT/designated-init-in-header/file2.m.in
+++ /dev/null
@@ -1,14 +0,0 @@
-#include "header1.h"
-
-@implementation S1
--(int)prop { return 0; }
--(void)setProp:(int)p {}
-+(id)s1 { return 0; }
--(id)initWithFoo:(NSString*)foo 
-{
-  self = [super init];
-  if (self) {
-  }
-  return self;
-}
-@end
diff --git a/clang/test/ARCMT/designated-init-in-header/file2.m.in.result b/clang/test/ARCMT/designated-init-in-header/file2.m.in.result
deleted file mode 100644
index 7465ed576f5ff..0000000000000
--- a/clang/test/ARCMT/designated-init-in-header/file2.m.in.result
+++ /dev/null
@@ -1,14 +0,0 @@
-#include "header1.h"
-
-@implementation S1
--(int)prop { return 0; }
--(void)setProp:(int)p {}
-+(instancetype)s1 { return 0; }
--(instancetype)initWithFoo:(NSString*)foo 
-{
-  self = [super init];
-  if (self) {
-  }
-  return self;
-}
-@end
diff --git a/clang/test/ARCMT/designated-init-in-header/header1.h b/clang/test/ARCMT/designated-init-in-header/header1.h
deleted file mode 100644
index c5668cc460869..0000000000000
--- a/clang/test/ARCMT/designated-init-in-header/header1.h
+++ /dev/null
@@ -1,14 +0,0 @@
-#define NS_DESIGNATED_INITIALIZER __attribute__((objc_designated_initializer))
-
-@class NSString;
-
-@interface B1
--(id)init;
-@end
-
-@interface S1 : B1
--(int)prop;
--(void)setProp:(int)p;
-+(id)s1;
--(id)initWithFoo:(NSString*)foo;
-@end
diff --git a/clang/test/ARCMT/designated-init-in-header/header1.h.result b/clang/test/ARCMT/designated-init-in-header/header1.h.result
deleted file mode 100644
index 974175b1c3e63..0000000000000
--- a/clang/test/ARCMT/designated-init-in-header/header1.h.result
+++ /dev/null
@@ -1,13 +0,0 @@
-#define NS_DESIGNATED_INITIALIZER __attribute__((objc_designated_initializer))
-
-@class NSString;
-
-@interface B1
--(instancetype)init;
-@end
-
-@interface S1 : B1
-@property (nonatomic) int prop;
-+(instancetype)s1;
--(instancetype)initWithFoo:(NSString*)foo NS_DESIGNATED_INITIALIZER;
-@end
diff --git a/clang/test/ARCMT/dispatch.m b/clang/test/ARCMT/dispatch.m
deleted file mode 100644
index 58c7769638cbe..0000000000000
--- a/clang/test/ARCMT/dispatch.m
+++ /dev/null
@@ -1,18 +0,0 @@
-// RUN: %clang_cc1 -fblocks -fsyntax-only -fobjc-arc -x objective-c %s.result
-// RUN: arcmt-test --args -triple x86_64-apple-darwin10 -fblocks -fsyntax-only -x objective-c %s > %t
-// RUN: diff %t %s.result
-
-#include "Common.h"
-
-dispatch_object_t getme(void);
-
-void func(dispatch_object_t o) {
-  dispatch_retain(o);
-  dispatch_release(o);
-  dispatch_retain(getme());
-}
-
-void func2(xpc_object_t o) {
-  xpc_retain(o);
-  xpc_release(o);
-}
diff --git a/clang/test/ARCMT/dispatch.m.result b/clang/test/ARCMT/dispatch.m.result
deleted file mode 100644
index 55b65585e4f6c..0000000000000
--- a/clang/test/ARCMT/dispatch.m.result
+++ /dev/null
@@ -1,14 +0,0 @@
-// RUN: %clang_cc1 -fblocks -fsyntax-only -fobjc-arc -x objective-c %s.result
-// RUN: arcmt-test --args -triple x86_64-apple-darwin10 -fblocks -fsyntax-only -x objective-c %s > %t
-// RUN: diff %t %s.result
-
-#include "Common.h"
-
-dispatch_object_t getme(void);
-
-void func(dispatch_object_t o) {
-  getme();
-}
-
-void func2(xpc_object_t o) {
-}
diff --git a/clang/test/ARCMT/driver-migrate.m b/clang/test/ARCMT/driver-migrate.m
deleted file mode 100644
index ec3f4cc8c4a31..0000000000000
--- a/clang/test/ARCMT/driver-migrate.m
+++ /dev/null
@@ -1,15 +0,0 @@
-// RUN: %clang -### -ccc-arcmt-migrate /foo/bar -fsyntax-only %s 2>&1 | FileCheck %s
-
-// CHECK: "-arcmt-action=migrate" "-mt-migrate-directory" "{{[^"]*}}/foo/bar"
-
-// RUN: touch %t.o
-// RUN: %clang -ccc-arcmt-check -target i386-apple-darwin9 -### %t.o 2> %t.log
-// RUN: FileCheck -check-prefix=LINK %s < %t.log
-// RUN: %clang -ccc-arcmt-migrate /foo/bar -target i386-apple-darwin9 -### %t.o 2> %t.log
-// RUN: FileCheck -check-prefix=LINK %s < %t.log
-
-// LINK-NOT: {{ld(.exe)?"}}
-// LINK: {{touch(.exe)?"}}
-
-// RUN: %clang -### -ccc-arcmt-migrate /foo/bar -fsyntax-only -fno-objc-arc %s 2>&1 | FileCheck -check-prefix=CHECK-NOARC %s
-// CHECK-NOARC-NOT: argument unused during compilation
diff --git a/clang/test/ARCMT/init.m b/clang/test/ARCMT/init.m
deleted file mode 100644
index b1f127e54fd1b..0000000000000
--- a/clang/test/ARCMT/init.m
+++ /dev/null
@@ -1,39 +0,0 @@
-// RUN: %clang_cc1 -triple x86_64-apple-darwin10 -fsyntax-only -fobjc-arc -x objective-c %s.result
-// RUN: arcmt-test --args -triple x86_64-apple-darwin10 -fsyntax-only -x objective-c %s > %t
-// RUN: diff %t %s.result
-
-#define nil (void *)0
-
-@interface NSObject
--init;
-@end
-
-@interface A : NSObject
--init;
--init2;
--foo;
-+alloc;
-@end
-
-@implementation A
--(id) init {
-  [self init];
-  id a;
-  [a init];
-  a = [[A alloc] init];
-
-  return self;
-}
-
--(id) init2 {
-  [super init];
-  return self;
-}
-
--(id) foo {
-  [self init];
-  [super init];
-
-  return self;
-}
-@end
diff --git a/clang/test/ARCMT/init.m.result b/clang/test/ARCMT/init.m.result
deleted file mode 100644
index d550dedb1dc19..0000000000000
--- a/clang/test/ARCMT/init.m.result
+++ /dev/null
@@ -1,39 +0,0 @@
-// RUN: %clang_cc1 -triple x86_64-apple-darwin10 -fsyntax-only -fobjc-arc -x objective-c %s.result
-// RUN: arcmt-test --args -triple x86_64-apple-darwin10 -fsyntax-only -x objective-c %s > %t
-// RUN: diff %t %s.result
-
-#define nil (void *)0
-
-@interface NSObject
--init;
-@end
-
-@interface A : NSObject
--init;
--init2;
--foo;
-+alloc;
-@end
-
-@implementation A
--(id) init {
-  if (!(self = [self init])) return nil;
-  id a;
-  [a init];
-  a = [[A alloc] init];
-
-  return self;
-}
-
--(id) init2 {
-  if (!(self = [super init])) return nil;
-  return self;
-}
-
--(id) foo {
-  [self init];
-  [super init];
-
-  return self;
-}
-@end
diff --git a/clang/test/ARCMT/lit.local.cfg b/clang/test/ARCMT/lit.local.cfg
deleted file mode 100644
index e9b04164d69df..0000000000000
--- a/clang/test/ARCMT/lit.local.cfg
+++ /dev/null
@@ -1,2 +0,0 @@
-if not config.root.clang_arcmt:
-    config.unsupported = True
diff --git a/clang/test/ARCMT/migrate-emit-errors.m b/clang/test/ARCMT/migrate-emit-errors.m
deleted file mode 100644
index 3e33acf75a591..0000000000000
--- a/clang/test/ARCMT/migrate-emit-errors.m
+++ /dev/null
@@ -1,12 +0,0 @@
-// RUN: %clang_cc1 -arcmt-action=migrate -mt-migrate-directory %t -arcmt-migrate-emit-errors %s 2>&1 | FileCheck %s
-// RUN: rm -rf %t
-
-@protocol NSObject
-- (oneway void)release;
-@end
-
-void test(id p) {
-  [p release];
-}
-
-// CHECK: error: ARC forbids explicit message send of 'release'
diff --git a/clang/test/ARCMT/migrate-on-pch-and-module.m b/clang/test/ARCMT/migrate-on-pch-and-module.m
deleted file mode 100644
index 42e01ea91a9c7..0000000000000
--- a/clang/test/ARCMT/migrate-on-pch-and-module.m
+++ /dev/null
@@ -1,12 +0,0 @@
-// RUN: rm -rf %t-mcp
-// RUN: %clang_cc1 -objcmt-migrate-subscripting -emit-pch -o %t.pch %s -isysroot %S/Inputs/System -triple x86_64-apple-darwin10 -F %S/Inputs -fmodules -fimplicit-module-maps -fmodules-cache-path=%t-mcp -w
-// RUN: %clang_cc1 -objcmt-migrate-subscripting -include-pch %t.pch %s -migrate -o %t.remap -isysroot %S/Inputs/System -triple x86_64-apple-darwin10 -F %S/Inputs -fmodules -fimplicit-module-maps -fmodules-cache-path=%t-mcp
-// REQUIRES: x86-registered-target
-#ifndef HEADER
-#define HEADER
-
-@import Module;
-
-#else
-
-#endif
diff --git a/clang/test/ARCMT/migrate-plist-output.m b/clang/test/ARCMT/migrate-plist-output.m
deleted file mode 100644
index e5710d818eccc..0000000000000
--- a/clang/test/ARCMT/migrate-plist-output.m
+++ /dev/null
@@ -1,51 +0,0 @@
-// RUN: %clang_cc1 -arcmt-action=migrate -mt-migrate-directory %t.dir -arcmt-migrate-report-output %t.plist %s
-// RUN: FileCheck %s -input-file=%t.plist
-// RUN: rm -rf %t.dir
-
-@protocol NSObject
-- (oneway void)release;
-@end
-
-void test(id p) {
-  [p release];
-}
-
-// CHECK: <?xml version="1.0" encoding="UTF-8"?>
-// CHECK: <!DOCTYPE plist PUBLIC "-//Apple Computer//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
-// CHECK: <plist version="1.0">
-// CHECK: <dict>
-// CHECK:  <key>files</key>
-// CHECK:  <array>
-// CHECK:  </array>
-// CHECK:  <key>diagnostics</key>
-// CHECK:  <array>
-// CHECK:   <dict>
-// CHECK:    <key>description</key><string>ARC forbids explicit message send of &apos;release&apos;</string>
-// CHECK:    <key>category</key><string>ARC Restrictions</string>
-// CHECK:    <key>type</key><string>error</string>
-// CHECK:   <key>location</key>
-// CHECK:   <dict>
-// CHECK:    <key>line</key><integer>10</integer>
-// CHECK:    <key>col</key><integer>6</integer>
-// CHECK:    <key>file</key><integer>0</integer>
-// CHECK:   </dict>
-// CHECK:    <key>ranges</key>
-// CHECK:    <array>
-// CHECK:     <array>
-// CHECK:      <dict>
-// CHECK:       <key>line</key><integer>10</integer>
-// CHECK:       <key>col</key><integer>4</integer>
-// CHECK:       <key>file</key><integer>0</integer>
-// CHECK:      </dict>
-// CHECK:      <dict>
-// CHECK:       <key>line</key><integer>10</integer>
-// CHECK:       <key>col</key><integer>4</integer>
-// CHECK:       <key>file</key><integer>0</integer>
-// CHECK:      </dict>
-// CHECK:     </array>
-// CHECK:    </array>
-// CHECK:   </dict>
-// CHECK:  </array>
-// CHECK: </dict>
-// CHECK: </plist>
-
diff --git a/clang/test/ARCMT/migrate-space-in-path.m b/clang/test/ARCMT/migrate-space-in-path.m
deleted file mode 100644
index 14a464d438f40..0000000000000
--- a/clang/test/ARCMT/migrate-space-in-path.m
+++ /dev/null
@@ -1,5 +0,0 @@
-// RUN: rm -rf %t.migrate
-// RUN: %clang_cc1 -arcmt-action=migrate -mt-migrate-directory %t.migrate %S/Inputs/"with space"/test1.m.in -x objective-c
-// RUN: %clang_cc1 -arcmt-action=migrate -mt-migrate-directory %t.migrate %S/Inputs/"with space"/test2.m.in -x objective-c
-// RUN: c-arcmt-test -mt-migrate-directory %t.migrate | arcmt-test -verify-transformed-files %S/Inputs/"with space"/test1.m.in.result %S/Inputs/"with space"/test2.m.in.result %S/Inputs/"with space"/test.h.result
-// RUN: rm -rf %t.migrate
diff --git a/clang/test/ARCMT/migrate-with-pch.m b/clang/test/ARCMT/migrate-with-pch.m
deleted file mode 100644
index d8e261be13d85..0000000000000
--- a/clang/test/ARCMT/migrate-with-pch.m
+++ /dev/null
@@ -1,6 +0,0 @@
-// RUN: rm -rf %t
-// RUN: %clang_cc1 -x objective-c %S/Common.h -emit-pch -o %t.pch
-// RUN: %clang_cc1 -arcmt-action=migrate -mt-migrate-directory %t %S/Inputs/test1.m.in -x objective-c -include-pch %t.pch
-// RUN: %clang_cc1 -arcmt-action=migrate -mt-migrate-directory %t %S/Inputs/test2.m.in -x objective-c -include-pch %t.pch
-// RUN: c-arcmt-test -mt-migrate-directory %t | arcmt-test -verify-transformed-files %S/Inputs/test1.m.in.result %S/Inputs/test2.m.in.result %S/Inputs/test.h.result
-// RUN: rm -rf %t
diff --git a/clang/test/ARCMT/migrate.m b/clang/test/ARCMT/migrate.m
deleted file mode 100644
index bc819df1c8a11..0000000000000
--- a/clang/test/ARCMT/migrate.m
+++ /dev/null
@@ -1,5 +0,0 @@
-// RUN: rm -rf %t
-// RUN: %clang_cc1 -arcmt-action=migrate -mt-migrate-directory %t %S/Inputs/test1.m.in -x objective-c
-// RUN: %clang_cc1 -arcmt-action=migrate -mt-migrate-directory %t %S/Inputs/test2.m.in -x objective-c
-// RUN: c-arcmt-test -mt-migrate-directory %t | arcmt-test -verify-transformed-files %S/Inputs/test1.m.in.result %S/Inputs/test2.m.in.result %S/Inputs/test.h.result
-// RUN: rm -rf %t
diff --git a/clang/test/ARCMT/no-canceling-bridge-to-bridge-cast.m b/clang/test/ARCMT/no-canceling-bridge-to-bridge-cast.m
deleted file mode 100644
index be49394951523..0000000000000
--- a/clang/test/ARCMT/no-canceling-bridge-to-bridge-cast.m
+++ /dev/null
@@ -1,41 +0,0 @@
-// RUN: %clang_cc1 -arcmt-action=check -triple x86_64-apple-darwin10 -fsyntax-only -x objective-c -verify %s
-typedef const void * CFTypeRef;
-CFTypeRef CFBridgingRetain(id X);
-id CFBridgingRelease(CFTypeRef);
-
-extern 
-CFTypeRef CFRetain(CFTypeRef cf);
-
-@interface INTF
-{
-  void *cf_format;
-  id objc_format;
-}
-@end
-
-@interface NSString
-+ (id)stringWithFormat:(NSString *)format;
-@end
-
-@implementation INTF
-- (void) Meth {
-  NSString *result;
-
-  result = (id) CFRetain([NSString stringWithFormat:@"PBXLoopMode"]); // expected-error {{cast of C pointer type 'CFTypeRef' (aka 'const void *') to Objective-C pointer type 'id' requires a bridged cast}} \
-								      // expected-note {{use __bridge to convert directly (no change in ownership)}} \
-								      // expected-note {{use CFBridgingRelease call to transfer ownership of a +1 'CFTypeRef' (aka 'const void *') into ARC}}
-
-  result = (id) CFRetain((id)((objc_format))); // expected-error {{cast of C pointer type 'CFTypeRef' (aka 'const void *') to Objective-C pointer type 'id' requires a bridged cast}} \
-					       // expected-note {{use __bridge to convert directly (no change in ownership)}} \
-					       // expected-note {{use CFBridgingRelease call to transfer ownership of a +1 'CFTypeRef' (aka 'const void *') into ARC}}
-
-  result = (id) CFRetain((id)((cf_format))); // expected-error {{cast of C pointer type 'CFTypeRef' (aka 'const void *') to Objective-C pointer type 'id' requires a bridged cast}} \
-					     // expected-note {{use __bridge to convert directly (no change in ownership)}} \
-                                             // expected-note {{use CFBridgingRelease call to transfer ownership of a +1 'CFTypeRef' (aka 'const void *') into ARC}}
-
-  result = (id) CFRetain((CFTypeRef)((objc_format)));
-
-  result = (id) CFRetain(cf_format); // OK
-}
-@end
-
diff --git a/clang/test/ARCMT/nonobjc-to-objc-cast-2.m b/clang/test/ARCMT/nonobjc-to-objc-cast-2.m
deleted file mode 100644
index 391c636906d53..0000000000000
--- a/clang/test/ARCMT/nonobjc-to-objc-cast-2.m
+++ /dev/null
@@ -1,63 +0,0 @@
-// RUN: %clang_cc1 -arcmt-action=check -verify -triple x86_64-apple-darwin10 %s
-
-#include "Common.h"
-
-typedef const struct __CFString * CFStringRef;
-typedef const void * CFTypeRef;
-CFTypeRef CFBridgingRetain(id X);
-id CFBridgingRelease(CFTypeRef);
-
-struct StrS {
-  CFStringRef sref_member;
-};
-
-@interface NSString : NSObject {
-  CFStringRef sref;
-  struct StrS *strS;
-}
--(id)string;
--(id)newString;
-@end
-
-@implementation NSString
--(id)string {
-  if (0)
-    return sref;
-  else
-    return strS->sref_member;
-}
--(id)newString {
-  return sref; // expected-error {{implicit conversion of C pointer type 'CFStringRef' (aka 'const struct __CFString *') to Objective-C pointer type 'id' requires a bridged cast}} \
-    // expected-note{{use __bridge to convert directly (no change in ownership)}} \
-    // expected-note{{use CFBridgingRelease call to transfer ownership of a +1 'CFStringRef' (aka 'const struct __CFString *') into ARC}}
-}
-@end
-
-void f(BOOL b) {
-  CFStringRef cfstr;
-  NSString *str = (NSString *)cfstr; // expected-error {{cast of C pointer type 'CFStringRef' (aka 'const struct __CFString *') to Objective-C pointer type 'NSString *' requires a bridged cast}} \
-    // expected-note{{use __bridge to convert directly (no change in ownership)}} \
-    // expected-note{{use CFBridgingRelease call to transfer ownership of a +1 'CFStringRef' (aka 'const struct __CFString *') into ARC}}
-  void *vp = str;  // expected-error {{requires a bridged cast}} expected-note {{use CFBridgingRetain call}} expected-note {{use __bridge}}
-}
-
-void f2(NSString *s) {
-  CFStringRef ref;
-  ref = [(CFStringRef)[s string] retain]; // expected-error {{cast of Objective-C pointer type 'id' to C pointer type 'CFStringRef' (aka 'const struct __CFString *') requires a bridged cast}} \
-    // expected-error {{bad receiver type 'CFStringRef' (aka 'const struct __CFString *')}} \
-    // expected-note{{use __bridge to convert directly (no change in ownership)}} \
-    // expected-note{{use CFBridgingRetain call to make an ARC object available as a +1 'CFStringRef' (aka 'const struct __CFString *')}}
-}
-
-CFStringRef f3(void) {
-  return (CFStringRef)[[[NSString alloc] init] autorelease]; // expected-error {{it is not safe to cast to 'CFStringRef' the result of 'autorelease' message; a __bridge cast may result in a pointer to a destroyed object and a __bridge_retained may leak the object}} \
-    // expected-note {{remove the cast and change return type of function to 'NSString *' to have the object automatically autoreleased}}
-}
-
-extern void NSLog(NSString *format, ...);
-
-void f4(NSString *s) {
-  NSLog(@"%@", (CFStringRef)s); // expected-error {{cast of Objective-C pointer type 'NSString *' to C pointer type 'CFStringRef' (aka 'const struct __CFString *') requires a bridged cast}} \
-    // expected-note{{use __bridge to convert directly (no change in ownership)}} \
-    // expected-note{{use CFBridgingRetain call to make an ARC object available as a +1 'CFStringRef' (aka 'const struct __CFString *')}}
-}
diff --git a/clang/test/ARCMT/nonobjc-to-objc-cast.m b/clang/test/ARCMT/nonobjc-to-objc-cast.m
deleted file mode 100644
index 7913661787e50..0000000000000
--- a/clang/test/ARCMT/nonobjc-to-objc-cast.m
+++ /dev/null
@@ -1,83 +0,0 @@
-// RUN: %clang_cc1 -triple x86_64-apple-darwin10 -fsyntax-only -fobjc-arc -x objective-c %s.result
-// RUN: arcmt-test --args -triple x86_64-apple-darwin10 -fsyntax-only -x objective-c %s > %t
-// RUN: diff %t %s.result
-
-#include "Common.h"
-
-typedef const struct __CFString * CFStringRef;
-extern const CFStringRef kUTTypePlainText;
-extern const CFStringRef kUTTypeRTF;
-extern CFStringRef kNonConst;
-
-typedef const struct __CFAllocator * CFAllocatorRef;
-typedef const struct __CFUUID * CFUUIDRef;
-
-extern const CFAllocatorRef kCFAllocatorDefault;
-
-extern CFStringRef CFUUIDCreateString(CFAllocatorRef alloc, CFUUIDRef uuid);
-
-struct StrS {
-  CFStringRef sref_member;
-};
-
-@interface NSString : NSObject {
-  CFStringRef sref;
-  struct StrS *strS;
-}
--(id)string;
--(id)newString;
-@end
-
-void f(BOOL b, id p) {
-  NSString *str = (NSString *)kUTTypePlainText; // no change
-  str = b ? kUTTypeRTF : kUTTypePlainText; // no change
-  str = (NSString *)(b ? kUTTypeRTF : kUTTypePlainText); // no change
-  str = (NSString *)p; // no change.
-
-  str = (NSString *)kNonConst;
-  str = b ? kUTTypeRTF : kNonConst;
-  str = (NSString *)(b ? kUTTypeRTF : kNonConst);
-
-  CFUUIDRef   _uuid;
-  NSString *_uuidString = (NSString *)CFUUIDCreateString(kCFAllocatorDefault, _uuid);
-  _uuidString = [(NSString *)CFUUIDCreateString(kCFAllocatorDefault, _uuid) autorelease];
-  _uuidString = CFRetain(_uuid);
-}
-
-@implementation NSString (StrExt)
-- (NSString *)stringEscapedAsURI {
-  CFStringRef str = (CFStringRef)self;
-  CFStringRef str2 = self;
-  return self;
-}
-@end
-
-@implementation NSString
--(id)string {
-  if (0)
-    return sref;
-  else
-    return strS->sref_member;
-}
--(id)newString { return 0; }
-@end
-
-extern void consumeParam(CFStringRef CF_CONSUMED p);
-
-void f2(NSString *s) {
-  CFStringRef ref = [s string];
-  ref = (CFStringRef)[s string];
-  ref = s.string;
-  ref = [NSString new];
-  ref = [s newString];
-  ref = (CFStringRef)[NSString new];
-  ref = [[NSString alloc] init];
-  ref = [[s string] retain];
-  ref = CFRetain((CFStringRef)[s string]);
-  ref = CFRetain([s string]);
-  ref = CFRetain(s);
-  ref = [s retain];
-
-  consumeParam((CFStringRef)s);
-  consumeParam(s);
-}
diff --git a/clang/test/ARCMT/nonobjc-to-objc-cast.m.result b/clang/test/ARCMT/nonobjc-to-objc-cast.m.result
deleted file mode 100644
index 8f3092f8786d8..0000000000000
--- a/clang/test/ARCMT/nonobjc-to-objc-cast.m.result
+++ /dev/null
@@ -1,83 +0,0 @@
-// RUN: %clang_cc1 -triple x86_64-apple-darwin10 -fsyntax-only -fobjc-arc -x objective-c %s.result
-// RUN: arcmt-test --args -triple x86_64-apple-darwin10 -fsyntax-only -x objective-c %s > %t
-// RUN: diff %t %s.result
-
-#include "Common.h"
-
-typedef const struct __CFString * CFStringRef;
-extern const CFStringRef kUTTypePlainText;
-extern const CFStringRef kUTTypeRTF;
-extern CFStringRef kNonConst;
-
-typedef const struct __CFAllocator * CFAllocatorRef;
-typedef const struct __CFUUID * CFUUIDRef;
-
-extern const CFAllocatorRef kCFAllocatorDefault;
-
-extern CFStringRef CFUUIDCreateString(CFAllocatorRef alloc, CFUUIDRef uuid);
-
-struct StrS {
-  CFStringRef sref_member;
-};
-
-@interface NSString : NSObject {
-  CFStringRef sref;
-  struct StrS *strS;
-}
--(id)string;
--(id)newString;
-@end
-
-void f(BOOL b, id p) {
-  NSString *str = (NSString *)kUTTypePlainText; // no change
-  str = b ? kUTTypeRTF : kUTTypePlainText; // no change
-  str = (NSString *)(b ? kUTTypeRTF : kUTTypePlainText); // no change
-  str = (NSString *)p; // no change.
-
-  str = (__bridge NSString *)kNonConst;
-  str = (__bridge NSString *)(b ? kUTTypeRTF : kNonConst);
-  str = (__bridge NSString *)(b ? kUTTypeRTF : kNonConst);
-
-  CFUUIDRef   _uuid;
-  NSString *_uuidString = (NSString *)CFBridgingRelease(CFUUIDCreateString(kCFAllocatorDefault, _uuid));
-  _uuidString = (NSString *)CFBridgingRelease(CFUUIDCreateString(kCFAllocatorDefault, _uuid));
-  _uuidString = CFBridgingRelease(CFRetain(_uuid));
-}
-
-@implementation NSString (StrExt)
-- (NSString *)stringEscapedAsURI {
-  CFStringRef str = (__bridge CFStringRef)self;
-  CFStringRef str2 = (__bridge CFStringRef)(self);
-  return self;
-}
-@end
-
-@implementation NSString
--(id)string {
-  if (0)
-    return (__bridge id)(sref);
-  else
-    return (__bridge id)(strS->sref_member);
-}
--(id)newString { return 0; }
-@end
-
-extern void consumeParam(CFStringRef CF_CONSUMED p);
-
-void f2(NSString *s) {
-  CFStringRef ref = (__bridge CFStringRef)([s string]);
-  ref = (__bridge CFStringRef)[s string];
-  ref = (__bridge CFStringRef)(s.string);
-  ref = CFBridgingRetain([NSString new]);
-  ref = CFBridgingRetain([s newString]);
-  ref = (CFStringRef)CFBridgingRetain([NSString new]);
-  ref = CFBridgingRetain([[NSString alloc] init]);
-  ref = CFBridgingRetain([s string]);
-  ref = (CFStringRef)CFBridgingRetain([s string]);
-  ref = CFBridgingRetain([s string]);
-  ref = CFBridgingRetain(s);
-  ref = CFBridgingRetain(s);
-
-  consumeParam((CFStringRef)CFBridgingRetain(s));
-  consumeParam(CFBridgingRetain(s));
-}
diff --git a/clang/test/ARCMT/objcmt-arc-cf-annotations.m b/clang/test/ARCMT/objcmt-arc-cf-annotations.m
deleted file mode 100644
index 47c83ac9e3dd5..0000000000000
--- a/clang/test/ARCMT/objcmt-arc-cf-annotations.m
+++ /dev/null
@@ -1,2017 +0,0 @@
-// RUN: rm -rf %t
-// RUN: %clang_cc1 -fblocks -objcmt-migrate-annotation -objcmt-migrate-instancetype -objcmt-migrate-readwrite-property -mt-migrate-directory %t %s -x objective-c -triple x86_64-apple-darwin11
-// RUN: c-arcmt-test -mt-migrate-directory %t | arcmt-test -verify-transformed-files %s.result
-// RUN: %clang_cc1 -fblocks -triple x86_64-apple-darwin10 -fsyntax-only -x objective-c %s.result
-
-#ifndef CF_IMPLICIT_BRIDGING_ENABLED
-#if __has_feature(arc_cf_code_audited)
-#define CF_IMPLICIT_BRIDGING_ENABLED _Pragma("clang arc_cf_code_audited begin")
-#else
-#define CF_IMPLICIT_BRIDGING_ENABLED
-#endif
-#endif
-
-#ifndef CF_IMPLICIT_BRIDGING_DISABLED
-#if __has_feature(arc_cf_code_audited)
-#define CF_IMPLICIT_BRIDGING_DISABLED _Pragma("clang arc_cf_code_audited end")
-#else
-#define CF_IMPLICIT_BRIDGING_DISABLED
-#endif
-#endif
-
-#if __has_feature(attribute_ns_returns_retained)
-#define NS_RETURNS_RETAINED __attribute__((ns_returns_retained))
-#endif
-#if __has_feature(attribute_cf_returns_retained)
-#define CF_RETURNS_RETAINED __attribute__((cf_returns_retained))
-#endif
-#if __has_feature(attribute_ns_returns_not_retained)
-#define NS_RETURNS_NOT_RETAINED __attribute__((ns_returns_not_retained))
-#endif
-#if __has_feature(attribute_cf_returns_not_retained)
-#define CF_RETURNS_NOT_RETAINED __attribute__((cf_returns_not_retained))
-#endif
-#if __has_feature(attribute_ns_consumes_self)
-#define NS_CONSUMES_SELF __attribute__((ns_consumes_self))
-#endif
-#if __has_feature(attribute_ns_consumed)
-#define NS_CONSUMED __attribute__((ns_consumed))
-#endif
-#if __has_feature(attribute_cf_consumed)
-#define CF_CONSUMED __attribute__((cf_consumed))
-#endif
-#if __has_attribute(ns_returns_autoreleased)
-#define NS_RETURNS_AUTORELEASED __attribute__((ns_returns_autoreleased))
-#endif
-
-//===----------------------------------------------------------------------===//
-// The following code is reduced using delta-debugging from Mac OS X headers:
-//
-// #include <Cocoa/Cocoa.h>
-// #include <CoreFoundation/CoreFoundation.h>
-// #include <DiskArbitration/DiskArbitration.h>
-// #include <QuartzCore/QuartzCore.h>
-// #include <Quartz/Quartz.h>
-// #include <IOKit/IOKitLib.h>
-//
-// It includes the basic definitions for the test cases below.
-//===----------------------------------------------------------------------===//
-
-typedef unsigned int __darwin_natural_t;
-typedef unsigned long uintptr_t;
-typedef unsigned int uint32_t;
-typedef unsigned long long uint64_t;
-typedef unsigned int UInt32;
-typedef signed long CFIndex;
-typedef CFIndex CFByteOrder;
-typedef struct {
-    CFIndex location;
-    CFIndex length;
-} CFRange;
-static __inline__ __attribute__((always_inline)) CFRange CFRangeMake(CFIndex loc, CFIndex len) {
-    CFRange range;
-    range.location = loc;
-    range.length = len;
-    return range;
-}
-typedef const void * CFTypeRef;
-typedef const struct __CFString * CFStringRef;
-typedef const struct __CFAllocator * CFAllocatorRef;
-extern const CFAllocatorRef kCFAllocatorDefault;
-extern CFTypeRef CFRetain(CFTypeRef cf);
-extern void CFRelease(CFTypeRef cf);
-extern CFTypeRef CFAutorelease(CFTypeRef cf);
-extern CFTypeRef CFMakeCollectable(CFTypeRef cf);
-typedef struct {
-}
-CFArrayCallBacks;
-extern const CFArrayCallBacks kCFTypeArrayCallBacks;
-typedef const struct __CFArray * CFArrayRef;
-typedef struct __CFArray * CFMutableArrayRef;
-extern CFMutableArrayRef CFArrayCreateMutable(CFAllocatorRef allocator, CFIndex capacity, const CFArrayCallBacks *callBacks);
-extern const void *CFArrayGetValueAtIndex(CFArrayRef theArray, CFIndex idx);
-extern void CFArrayAppendValue(CFMutableArrayRef theArray, const void *value);
-typedef struct {
-}
-CFDictionaryKeyCallBacks;
-extern const CFDictionaryKeyCallBacks kCFTypeDictionaryKeyCallBacks;
-typedef struct {
-}
-CFDictionaryValueCallBacks;
-extern const CFDictionaryValueCallBacks kCFTypeDictionaryValueCallBacks;
-typedef const struct __CFDictionary * CFDictionaryRef;
-typedef struct __CFDictionary * CFMutableDictionaryRef;
-extern CFMutableDictionaryRef CFDictionaryCreateMutable(CFAllocatorRef allocator, CFIndex capacity, const CFDictionaryKeyCallBacks *keyCallBacks, const CFDictionaryValueCallBacks *valueCallBacks);
-typedef UInt32 CFStringEncoding;
-enum {
-kCFStringEncodingMacRoman = 0,     kCFStringEncodingWindowsLatin1 = 0x0500,     kCFStringEncodingISOLatin1 = 0x0201,     kCFStringEncodingNextStepLatin = 0x0B01,     kCFStringEncodingASCII = 0x0600,     kCFStringEncodingUnicode = 0x0100,     kCFStringEncodingUTF8 = 0x08000100,     kCFStringEncodingNonLossyASCII = 0x0BFF      ,     kCFStringEncodingUTF16 = 0x0100,     kCFStringEncodingUTF16BE = 0x10000100,     kCFStringEncodingUTF16LE = 0x14000100,      kCFStringEncodingUTF32 = 0x0c000100,     kCFStringEncodingUTF32BE = 0x18000100,     kCFStringEncodingUTF32LE = 0x1c000100  };
-extern CFStringRef CFStringCreateWithCString(CFAllocatorRef alloc, const char *cStr, CFStringEncoding encoding);
-typedef double CFTimeInterval;
-typedef CFTimeInterval CFAbsoluteTime;
-extern CFAbsoluteTime CFAbsoluteTimeGetCurrent(void);
-typedef const struct __CFDate * CFDateRef;
-extern CFDateRef CFDateCreate(CFAllocatorRef allocator, CFAbsoluteTime at);
-extern CFAbsoluteTime CFDateGetAbsoluteTime(CFDateRef theDate);
-typedef __darwin_natural_t natural_t;
-typedef natural_t mach_port_name_t;
-typedef mach_port_name_t mach_port_t;
-typedef int kern_return_t;
-typedef kern_return_t mach_error_t;
-enum {
-kCFNumberSInt8Type = 1,     kCFNumberSInt16Type = 2,     kCFNumberSInt32Type = 3,     kCFNumberSInt64Type = 4,     kCFNumberFloat32Type = 5,     kCFNumberFloat64Type = 6,      kCFNumberCharType = 7,     kCFNumberShortType = 8,     kCFNumberIntType = 9,     kCFNumberLongType = 10,     kCFNumberLongLongType = 11,     kCFNumberFloatType = 12,     kCFNumberDoubleType = 13,      kCFNumberCFIndexType = 14,      kCFNumberNSIntegerType = 15,     kCFNumberCGFloatType = 16,     kCFNumberMaxType = 16    };
-typedef CFIndex CFNumberType;
-typedef const struct __CFNumber * CFNumberRef;
-extern CFNumberRef CFNumberCreate(CFAllocatorRef allocator, CFNumberType theType, const void *valuePtr);
-typedef const struct __CFAttributedString *CFAttributedStringRef;
-typedef struct __CFAttributedString *CFMutableAttributedStringRef;
-extern CFAttributedStringRef CFAttributedStringCreate(CFAllocatorRef alloc, CFStringRef str, CFDictionaryRef attributes) ;
-extern CFMutableAttributedStringRef CFAttributedStringCreateMutableCopy(CFAllocatorRef alloc, CFIndex maxLength, CFAttributedStringRef aStr) ;
-extern void CFAttributedStringSetAttribute(CFMutableAttributedStringRef aStr, CFRange range, CFStringRef attrName, CFTypeRef value) ;
-typedef signed char BOOL;
-typedef unsigned long NSUInteger;
-@class NSString, Protocol;
-extern void NSLog(NSString *format, ...) __attribute__((format(__NSString__, 1, 2)));
-typedef struct _NSZone NSZone;
-@class NSInvocation, NSMethodSignature, NSCoder, NSString, NSEnumerator;
-@protocol NSObject
-- (BOOL)isEqual:(id)object;
-- (id)retain;
-- (oneway void)release;
-- (id)autorelease;
-- (NSString *)description;
-- (id)init;
-@end
-@protocol NSCopying 
-- (id)copyWithZone:(NSZone *)zone;
-@end
-@protocol NSMutableCopying  - (id)mutableCopyWithZone:(NSZone *)zone;
-@end
-@protocol NSCoding  - (void)encodeWithCoder:(NSCoder *)aCoder;
-@end
-@interface NSObject <NSObject> {}
-+ (id)allocWithZone:(NSZone *)zone;
-+ (id)alloc;
-+ (id)new;
-- (void)dealloc;
-@end
-@interface NSObject (NSCoderMethods)
-- (id)awakeAfterUsingCoder:(NSCoder *)aDecoder;
-@end
-extern id NSAllocateObject(Class aClass, NSUInteger extraBytes, NSZone *zone);
-typedef struct {
-}
-NSFastEnumerationState;
-@protocol NSFastEnumeration 
-- (NSUInteger)countByEnumeratingWithState:(NSFastEnumerationState *)state objects:(id *)stackbuf count:(NSUInteger)len;
-@end
-@class NSString, NSDictionary;
-@interface NSValue : NSObject <NSCopying, NSCoding>  - (void)getValue:(void *)value;
-@end
-@interface NSNumber : NSValue
-- (char)charValue;
-- (id)initWithInt:(int)value;
-+ (NSNumber *)numberWithInt:(int)value;
-@end
-@class NSString;
-@interface NSArray : NSObject <NSCopying, NSMutableCopying, NSCoding, NSFastEnumeration>
-- (NSUInteger)count;
-- (id)initWithObjects:(const id [])objects count:(NSUInteger)cnt;
-+ (id)arrayWithObject:(id)anObject;
-+ (id)arrayWithObjects:(const id [])objects count:(NSUInteger)cnt;
-+ (id)arrayWithObjects:(id)firstObj, ... __attribute__((sentinel(0,1)));
-- (id)initWithObjects:(id)firstObj, ... __attribute__((sentinel(0,1)));
-- (id)initWithArray:(NSArray *)array;
-@end  @interface NSArray (NSArrayCreation)  + (id)array;
-@end       @interface NSAutoreleasePool : NSObject {
-}
-- (void)drain;
-@end extern NSString * const NSBundleDidLoadNotification;
-typedef double NSTimeInterval;
-@interface NSDate : NSObject <NSCopying, NSCoding>  - (NSTimeInterval)timeIntervalSinceReferenceDate;
-@end            typedef unsigned short unichar;
-@interface NSString : NSObject <NSCopying, NSMutableCopying, NSCoding>
-- (NSUInteger)length;
-- (NSString *)stringByAppendingString:(NSString *)aString;
-- ( const char *)UTF8String;
-- (id)initWithUTF8String:(const char *)nullTerminatedCString;
-+ (id)stringWithUTF8String:(const char *)nullTerminatedCString;
-@end        @class NSString, NSURL, NSError;
-@interface NSData : NSObject <NSCopying, NSMutableCopying, NSCoding>  - (NSUInteger)length;
-+ (id)dataWithBytesNoCopy:(void *)bytes length:(NSUInteger)length;
-+ (id)dataWithBytesNoCopy:(void *)bytes length:(NSUInteger)length freeWhenDone:(BOOL)b;
-@end   @class NSLocale, NSDate, NSCalendar, NSTimeZone, NSError, NSArray, NSMutableDictionary;
-@interface NSDictionary : NSObject <NSCopying, NSMutableCopying, NSCoding, NSFastEnumeration>
-- (NSUInteger)count;
-+ (id)dictionaryWithObjects:(NSArray *)objects forKeys:(NSArray *)keys;
-+ (id)dictionaryWithObjects:(const id [])objects forKeys:(const id <NSCopying> [])keys count:(NSUInteger)cnt;
-@end
-@interface NSMutableDictionary : NSDictionary  - (void)removeObjectForKey:(id)aKey;
-- (void)setObject:(id)anObject forKey:(id)aKey;
-@end  @interface NSMutableDictionary (NSMutableDictionaryCreation)  + (id)dictionaryWithCapacity:(NSUInteger)numItems;
-@end  typedef double CGFloat;
-struct CGSize {
-};
-typedef struct CGSize CGSize;
-struct CGRect {
-};
-typedef struct CGRect CGRect;
-typedef mach_port_t io_object_t;
-typedef char io_name_t[128];
-typedef io_object_t io_iterator_t;
-typedef io_object_t io_service_t;
-typedef struct IONotificationPort * IONotificationPortRef;
-typedef void (*IOServiceMatchingCallback)(  void * refcon,  io_iterator_t iterator );
-io_service_t IOServiceGetMatchingService(  mach_port_t mainPort,  CFDictionaryRef matching );
-kern_return_t IOServiceGetMatchingServices(  mach_port_t mainPort,  CFDictionaryRef matching,  io_iterator_t * existing );
-kern_return_t IOServiceAddNotification(  mach_port_t mainPort,  const io_name_t notificationType,  CFDictionaryRef matching,  mach_port_t wakePort,  uintptr_t reference,  io_iterator_t * notification ) __attribute__((deprecated)); // expected-note {{'IOServiceAddNotification' declared here}}
-kern_return_t IOServiceAddMatchingNotification(  IONotificationPortRef notifyPort,  const io_name_t notificationType,  CFDictionaryRef matching,         IOServiceMatchingCallback callback,         void * refCon,  io_iterator_t * notification );
-CFMutableDictionaryRef IOServiceMatching(  const char * name );
-CFMutableDictionaryRef IOServiceNameMatching(  const char * name );
-CFMutableDictionaryRef IOBSDNameMatching(  mach_port_t mainPort,  uint32_t options,  const char * bsdName );
-CFMutableDictionaryRef IOOpenFirmwarePathMatching(  mach_port_t mainPort,  uint32_t options,  const char * path );
-CFMutableDictionaryRef IORegistryEntryIDMatching(  uint64_t entryID );
-typedef struct __DASession * DASessionRef;
-extern DASessionRef DASessionCreate( CFAllocatorRef allocator );
-typedef struct __DADisk * DADiskRef;
-extern DADiskRef DADiskCreateFromBSDName( CFAllocatorRef allocator, DASessionRef session, const char * name );
-extern DADiskRef DADiskCreateFromIOMedia( CFAllocatorRef allocator, DASessionRef session, io_service_t media );
-extern CFDictionaryRef DADiskCopyDescription( DADiskRef disk );
-extern DADiskRef DADiskCopyWholeDisk( DADiskRef disk );
-@interface NSTask : NSObject - (id)init;
-@end                    typedef struct CGColorSpace *CGColorSpaceRef;
-typedef struct CGImage *CGImageRef;
-typedef struct CGLayer *CGLayerRef;
-@interface NSResponder : NSObject <NSCoding> {
-}
-@end    @protocol NSAnimatablePropertyContainer      - (id)animator;
-@end  extern NSString *NSAnimationTriggerOrderIn ;
-@interface NSView : NSResponder  <NSAnimatablePropertyContainer>  {
-}
-@end @protocol NSValidatedUserInterfaceItem - (SEL)action;
-@end   @protocol NSUserInterfaceValidations - (BOOL)validateUserInterfaceItem:(id <NSValidatedUserInterfaceItem>)anItem;
-@end  @class NSDate, NSDictionary, NSError, NSException, NSNotification;
-@class NSTextField, NSPanel, NSArray, NSWindow, NSImage, NSButton, NSError;
-@interface NSApplication : NSResponder <NSUserInterfaceValidations> {
-}
-- (void)beginSheet:(NSWindow *)sheet modalForWindow:(NSWindow *)docWindow modalDelegate:(id)modalDelegate didEndSelector:(SEL)didEndSelector contextInfo:(void *)contextInfo;
-@end   enum {
-NSTerminateCancel = 0,         NSTerminateNow = 1,         NSTerminateLater = 2 };
-typedef NSUInteger NSApplicationTerminateReply;
-@protocol NSApplicationDelegate <NSObject> @optional        - (NSApplicationTerminateReply)applicationShouldTerminate:(NSApplication *)sender;
-@end  @class NSAttributedString, NSEvent, NSFont, NSFormatter, NSImage, NSMenu, NSText, NSView, NSTextView;
-@interface NSCell : NSObject <NSCopying, NSCoding> {
-}
-@end 
-typedef struct {
-}
-CVTimeStamp;
-@interface CIImage : NSObject <NSCoding, NSCopying> {
-}
-typedef int CIFormat;
-@end  enum {
-kDAReturnSuccess = 0,     kDAReturnError = (((0x3eU)&0x3f)<<26) | (((0x368)&0xfff)<<14) | 0x01,     kDAReturnBusy = (((0x3eU)&0x3f)<<26) | (((0x368)&0xfff)<<14) | 0x02,     kDAReturnBadArgument = (((0x3eU)&0x3f)<<26) | (((0x368)&0xfff)<<14) | 0x03,     kDAReturnExclusiveAccess = (((0x3eU)&0x3f)<<26) | (((0x368)&0xfff)<<14) | 0x04,     kDAReturnNoResources = (((0x3eU)&0x3f)<<26) | (((0x368)&0xfff)<<14) | 0x05,     kDAReturnNotFound = (((0x3eU)&0x3f)<<26) | (((0x368)&0xfff)<<14) | 0x06,     kDAReturnNotMounted = (((0x3eU)&0x3f)<<26) | (((0x368)&0xfff)<<14) | 0x07,     kDAReturnNotPermitted = (((0x3eU)&0x3f)<<26) | (((0x368)&0xfff)<<14) | 0x08,     kDAReturnNotPrivileged = (((0x3eU)&0x3f)<<26) | (((0x368)&0xfff)<<14) | 0x09,     kDAReturnNotReady = (((0x3eU)&0x3f)<<26) | (((0x368)&0xfff)<<14) | 0x0A,     kDAReturnNotWritable = (((0x3eU)&0x3f)<<26) | (((0x368)&0xfff)<<14) | 0x0B,     kDAReturnUnsupported = (((0x3eU)&0x3f)<<26) | (((0x368)&0xfff)<<14) | 0x0C };
-typedef mach_error_t DAReturn;
-typedef const struct __DADissenter * DADissenterRef;
-extern DADissenterRef DADissenterCreate( CFAllocatorRef allocator, DAReturn status, CFStringRef string );
-@interface CIContext: NSObject {
-}
-- (CGImageRef)createCGImage:(CIImage *)im fromRect:(CGRect)r;
-- (CGImageRef)createCGImage:(CIImage *)im fromRect:(CGRect)r     format:(CIFormat)f colorSpace:(CGColorSpaceRef)cs;
-- (CGLayerRef)createCGLayerWithSize:(CGSize)size info:(CFDictionaryRef)d;
-@end extern NSString* const QCRendererEventKey;
-@protocol QCCompositionRenderer - (NSDictionary*) attributes;
-@end   @interface QCRenderer : NSObject <QCCompositionRenderer> {
-}
-- (id) createSnapshotImageOfType:(NSString*)type;
-@end  extern NSString* const QCViewDidStartRenderingNotification;
-@interface QCView : NSView <QCCompositionRenderer> {
-}
-- (id) createSnapshotImageOfType:(NSString*)type;
-@end    enum {
-ICEXIFOrientation1 = 1,     ICEXIFOrientation2 = 2,     ICEXIFOrientation3 = 3,     ICEXIFOrientation4 = 4,     ICEXIFOrientation5 = 5,     ICEXIFOrientation6 = 6,     ICEXIFOrientation7 = 7,     ICEXIFOrientation8 = 8, };
-@class ICDevice;
-@protocol ICDeviceDelegate <NSObject>  @required      - (void)didRemoveDevice:(ICDevice*)device;
-@end extern NSString *const ICScannerStatusWarmingUp;
-@class ICScannerDevice;
-@protocol ICScannerDeviceDelegate <ICDeviceDelegate>  @optional       - (void)scannerDeviceDidBecomeAvailable:(ICScannerDevice*)scanner;
-@end
-
-typedef long unsigned int __darwin_size_t;
-typedef __darwin_size_t size_t;
-typedef unsigned long CFTypeID;
-struct CGPoint {
-  CGFloat x;
-  CGFloat y;
-};
-typedef struct CGPoint CGPoint;
-typedef struct CGGradient *CGGradientRef;
-typedef uint32_t CGGradientDrawingOptions;
-extern CFTypeID CGGradientGetTypeID(void);
-extern CGGradientRef CGGradientCreateWithColorComponents(CGColorSpaceRef
-  space, const CGFloat components[], const CGFloat locations[], size_t count);
-extern CGGradientRef CGGradientCreateWithColors(CGColorSpaceRef space,
-  CFArrayRef colors, const CGFloat locations[]);
-extern CGGradientRef CGGradientRetain(CGGradientRef gradient);
-extern void CGGradientRelease(CGGradientRef gradient);
-typedef struct CGContext *CGContextRef;
-extern void CGContextDrawLinearGradient(CGContextRef context,
-    CGGradientRef gradient, CGPoint startPoint, CGPoint endPoint,
-    CGGradientDrawingOptions options);
-extern CGColorSpaceRef CGColorSpaceCreateDeviceRGB(void);
-
-@interface NSMutableArray : NSObject
-- (void)addObject:(id)object;
-+ (id)array;
-@end
-
-// This is how NSMakeCollectable is declared in the OS X 10.8 headers.
-id NSMakeCollectable(CFTypeRef __attribute__((cf_consumed))) __attribute__((ns_returns_retained));
-
-typedef const struct __CFUUID * CFUUIDRef;
-
-extern
-void *CFPlugInInstanceCreate(CFAllocatorRef allocator, CFUUIDRef factoryUUID, CFUUIDRef typeUUID);
-
-//===----------------------------------------------------------------------===//
-// Test cases.
-//===----------------------------------------------------------------------===//
-
-CFAbsoluteTime f1(void) {
-  CFAbsoluteTime t = CFAbsoluteTimeGetCurrent();
-  CFDateRef date = CFDateCreate(0, t);
-  CFRetain(date);
-  CFRelease(date);
-  CFDateGetAbsoluteTime(date); // no-warning
-  CFRelease(date);
-  t = CFDateGetAbsoluteTime(date);   // expected-warning{{Reference-counted object is used after it is released}}
-  return t;
-}
-
-CFAbsoluteTime f2(void) {
-  CFAbsoluteTime t = CFAbsoluteTimeGetCurrent();
-  CFDateRef date = CFDateCreate(0, t);  
-  [((NSDate*) date) retain];
-  CFRelease(date);
-  CFDateGetAbsoluteTime(date); // no-warning
-  [((NSDate*) date) release];
-  t = CFDateGetAbsoluteTime(date);   // expected-warning{{Reference-counted object is used after it is released}}
-  return t;
-}
-
-
-NSDate* global_x;
-
-// Test to see if we suppress an error when we store the pointer
-// to a global.
-
-CFAbsoluteTime f3(void) {
-  CFAbsoluteTime t = CFAbsoluteTimeGetCurrent();
-  CFDateRef date = CFDateCreate(0, t);  
-  [((NSDate*) date) retain];
-  CFRelease(date);
-  CFDateGetAbsoluteTime(date); // no-warning
-  global_x = (NSDate*) date;  
-  [((NSDate*) date) release];
-  t = CFDateGetAbsoluteTime(date);   // no-warning
-  return t;
-}
-
-//---------------------------------------------------------------------------
-// Test case 'f4' differs for region store and basic store.  See
-// retain-release-region-store.m and retain-release-basic-store.m.
-//---------------------------------------------------------------------------
-
-// Test a leak.
-
-CFAbsoluteTime f5(int x) {  
-  CFAbsoluteTime t = CFAbsoluteTimeGetCurrent();
-  CFDateRef date = CFDateCreate(0, t); // expected-warning{{leak}}
-  
-  if (x)
-    CFRelease(date);
-  
-  return t;
-}
-
-// Test a leak involving the return.
-
-CFDateRef f6(int x) {  
-  CFDateRef date = CFDateCreate(0, CFAbsoluteTimeGetCurrent());  // expected-warning{{leak}}
-  CFRetain(date);
-  return date;
-}
-
-// Test a leak involving an overwrite.
-
-CFDateRef f7(void) {
-  CFDateRef date = CFDateCreate(0, CFAbsoluteTimeGetCurrent());  //expected-warning{{leak}}
-  CFRetain(date);
-  date = CFDateCreate(0, CFAbsoluteTimeGetCurrent()); // expected-warning {{leak}}
-  return date;
-}
-
-// Generalization of Create rule.  MyDateCreate returns a CFXXXTypeRef, and
-// has the word create.
-CFDateRef MyDateCreate(void);
-
-CFDateRef f8(void) {
-  CFDateRef date = MyDateCreate(); // expected-warning{{leak}}
-  CFRetain(date);  
-  return date;
-}
-
-__attribute__((cf_returns_retained)) CFDateRef f9(void) {
-  CFDateRef date = CFDateCreate(0, CFAbsoluteTimeGetCurrent()); // no-warning
-  int *p = 0;
-  // When allocations fail, CFDateCreate can return null.
-  if (!date) *p = 1; // expected-warning{{null}}
-  return date;
-}
-
-// Handle DiskArbitration API:
-//
-// http://developer.apple.com/DOCUMENTATION/DARWIN/Reference/DiscArbitrationFramework/
-//
-void f10(io_service_t media, DADiskRef d, CFStringRef s) {
-  DADiskRef disk = DADiskCreateFromBSDName(kCFAllocatorDefault, 0, "hello"); // expected-warning{{leak}}
-  if (disk) NSLog(@"ok");
-  
-  disk = DADiskCreateFromIOMedia(kCFAllocatorDefault, 0, media); // expected-warning{{leak}}
-  if (disk) NSLog(@"ok");
-
-  CFDictionaryRef dict = DADiskCopyDescription(d);  // expected-warning{{leak}}
-  if (dict) NSLog(@"ok"); 
-  
-  disk = DADiskCopyWholeDisk(d); // expected-warning{{leak}}
-  if (disk) NSLog(@"ok");
-    
-  DADissenterRef dissenter = DADissenterCreate(kCFAllocatorDefault,   // expected-warning{{leak}}
-                                                kDAReturnSuccess, s);
-  if (dissenter) NSLog(@"ok");
-  
-  DASessionRef session = DASessionCreate(kCFAllocatorDefault);  // expected-warning{{leak}}
-  if (session) NSLog(@"ok");
-}
-
-// Test retain/release checker with CFString and CFMutableArray.
-void f11(void) {
-  // Create the array.
-  CFMutableArrayRef A = CFArrayCreateMutable(0, 10, &kCFTypeArrayCallBacks);
-
-  // Create a string.
-  CFStringRef s1 = CFStringCreateWithCString(0, "hello world",
-                                             kCFStringEncodingUTF8);
-
-  // Add the string to the array.
-  CFArrayAppendValue(A, s1);
-  
-  // Decrement the reference count.
-  CFRelease(s1); // no-warning
-  
-  // Get the string.  We don't own it.
-  s1 = (CFStringRef) CFArrayGetValueAtIndex(A, 0);
-  
-  // Release the array.
-  CFRelease(A); // no-warning
-  
-  // Release the string.  This is a bug.
-  CFRelease(s1); // expected-warning{{Incorrect decrement of the reference count}}
-}
-
-// PR 3337: Handle functions declared using typedefs.
-typedef CFTypeRef CREATEFUN(void);
-CFTypeRef MyCreateFun(void);
-
-void f12(void) {
-  CFTypeRef o = MyCreateFun(); // expected-warning {{leak}}
-}
-
-void f13_autorelease(void) {
-  CFMutableArrayRef A = CFArrayCreateMutable(0, 10, &kCFTypeArrayCallBacks); // no-warning
-  [(id) A autorelease]; // no-warning
-}
-
-void f13_autorelease_b(void) {
-  CFMutableArrayRef A = CFArrayCreateMutable(0, 10, &kCFTypeArrayCallBacks);
-  [(id) A autorelease];
-  [(id) A autorelease];
-} // expected-warning{{Object autoreleased too many times}}
-
-CFMutableArrayRef f13_autorelease_c(void) {
-  CFMutableArrayRef A = CFArrayCreateMutable(0, 10, &kCFTypeArrayCallBacks);
-  [(id) A autorelease];
-  [(id) A autorelease]; 
-  return A; // expected-warning{{Object autoreleased too many times}}
-}
-
-CFMutableArrayRef f13_autorelease_d(void) {
-  CFMutableArrayRef A = CFArrayCreateMutable(0, 10, &kCFTypeArrayCallBacks);
-  [(id) A autorelease];
-  [(id) A autorelease]; 
-  CFMutableArrayRef B = CFArrayCreateMutable(0, 10, &kCFTypeArrayCallBacks); // expected-warning{{Object autoreleased too many times}}
-  CFRelease(B); // no-warning
-  while (1) {}
-}
-
-
-// This case exercises the logic where the leak site is the same as the allocation site.
-void f14_leakimmediately(void) {
-  CFArrayCreateMutable(0, 10, &kCFTypeArrayCallBacks); // expected-warning{{leak}}
-}
-
-// Test that we track an allocated object beyond the point where the *name*
-// of the variable storing the reference is no longer live.
-void f15(void) {
-  // Create the array.
-  CFMutableArrayRef A = CFArrayCreateMutable(0, 10, &kCFTypeArrayCallBacks);
-  CFMutableArrayRef *B = &A;
-  // At this point, the name 'A' is no longer live.
-  CFRelease(*B);  // no-warning
-}
-
-// Test when we pass NULL to CFRetain/CFRelease/CFMakeCollectable/CFAutorelease.
-void f16(int x, CFTypeRef p) {
-  if (p)
-    return;
-
-  switch (x) {
-  case 0:
-    CFRelease(p);
-    break;
-  case 1:
-    CFRetain(p);
-    break;
-  case 2:
-    CFMakeCollectable(p);
-    break;
-  case 3:
-    CFAutorelease(p);
-    break;
-  default:
-    break;
-  }
-}
-
-// Test that an object is non-null after CFRetain/CFRelease/CFMakeCollectable/CFAutorelease.
-void f17(int x, CFTypeRef p) {
-  switch (x) {
-  case 0:
-    CFRelease(p);
-    if (!p)
-      CFRelease(0); // no-warning
-    break;
-  case 1:
-    CFRetain(p);
-    if (!p)
-      CFRetain(0); // no-warning
-    break;
-  case 2:
-    CFMakeCollectable(p);
-    if (!p)
-      CFMakeCollectable(0); // no-warning
-    break;
-  case 3:
-    CFAutorelease(p);
-    if (!p)
-      CFAutorelease(0); // no-warning
-    break;
-  default:
-    break;
-  }
-}
-
-// Test basic tracking of ivars associated with 'self'.  For the retain/release
-// checker we currently do not want to flag leaks associated with stores
-// of tracked objects to ivars.
-@interface SelfIvarTest : NSObject {
-  id myObj;
-}
-- (void)test_self_tracking;
-@end
-
-@implementation SelfIvarTest
-- (void)test_self_tracking {
-  myObj = (id) CFArrayCreateMutable(0, 10, &kCFTypeArrayCallBacks); // no-warning
-}
-@end
-
-// Test return of non-owned objects in contexts where an owned object
-// is expected.
-@interface TestReturnNotOwnedWhenExpectedOwned
-- (NSString*)newString;
-@end
-
-@implementation TestReturnNotOwnedWhenExpectedOwned
-- (NSString*)newString {
-  NSString *s = [NSString stringWithUTF8String:"hello"];
-  return s; // expected-warning{{Object with a +0 retain count returned to caller where a +1 (owning) retain count is expected}}
-}
-@end
-
-int isFoo(char c);
-
-static void rdar_6659160(char *inkind, char *inname)
-{
-  // We currently expect that [NSObject alloc] cannot fail.  This
-  // will be a toggled flag in the future.  It can indeed return null, but
-  // Cocoa programmers generally aren't expected to reason about out-of-memory
-  // conditions.
-  NSString *kind = [[NSString alloc] initWithUTF8String:inkind];  // expected-warning{{leak}}
-  
-  // We do allow stringWithUTF8String to fail.  This isn't really correct, as
-  // far as returning 0.  In most error conditions it will throw an exception.
-  // If allocation fails it could return 0, but again this
-  // isn't expected.
-  NSString *name = [NSString stringWithUTF8String:inname];
-  if(!name)
-    return;
-
-  const char *kindC = 0;
-  const char *nameC = 0;
-  
-  // In both cases, we cannot reach a point down below where we
-  // dereference kindC or nameC with either being null.  This is because
-  // we assume that [NSObject alloc] doesn't fail and that we have the guard
-  // up above.
-  
-  if(kind)
-    kindC = [kind UTF8String];
-  if(name)
-    nameC = [name UTF8String];
-  if(!isFoo(kindC[0])) // expected-warning{{null}}
-    return;
-  if(!isFoo(nameC[0])) // no-warning
-    return;
-
-  [kind release];
-  [name release]; // expected-warning{{Incorrect decrement of the reference count}}
-}
-
-// PR 3677 - 'allocWithZone' should be treated as following the Cocoa naming
-//  conventions with respect to 'return'ing ownership.
-@interface PR3677: NSObject @end
-@implementation PR3677
-+ (id)allocWithZone:(NSZone *)inZone {
-  return [super allocWithZone:inZone];  // no-warning
-}
-@end
-
-// PR 3820 - Reason about calls to -dealloc
-void pr3820_DeallocInsteadOfRelease(void)
-{
-  id foo = [[NSString alloc] init]; // no-warning
-  [foo dealloc];
-  // foo is not leaked, since it has been deallocated.
-}
-
-void pr3820_ReleaseAfterDealloc(void)
-{
-  id foo = [[NSString alloc] init];
-  [foo dealloc];
-  [foo release];  // expected-warning{{used after it is release}}
-  // NSInternalInconsistencyException: message sent to deallocated object
-}
-
-void pr3820_DeallocAfterRelease(void)
-{
-  NSLog(@"\n\n[%s]", __FUNCTION__);
-  id foo = [[NSString alloc] init];
-  [foo release];
-  [foo dealloc]; // expected-warning{{used after it is released}}
-  // message sent to released object
-}
-
-// The problem here is that 'length' binds to '($0 - 1)' after '--length', but
-// SimpleConstraintManager doesn't know how to reason about
-// '($0 - 1) > constant'.  As a temporary hack, we drop the value of '($0 - 1)'
-// and conjure a new symbol.
-void rdar6704930(unsigned char *s, unsigned int length) {
-  NSString* name = 0;
-  if (s != 0) {
-    if (length > 0) {
-      while (length > 0) {
-        if (*s == ':') {
-          ++s;
-          --length;
-          name = [[NSString alloc] init]; // no-warning
-          break;
-        }
-        ++s;
-        --length;
-      }
-      if ((length == 0) && (name != 0)) {
-        [name release];
-        name = 0;
-      }
-      if (length == 0) { // no ':' found -> use it all as name
-        name = [[NSString alloc] init]; // no-warning
-      }
-    }
-  }
-
-  if (name != 0) {
-    [name release];
-  }
-}
-
-//===----------------------------------------------------------------------===//
-// One build of the analyzer accidentally stopped tracking the allocated
-// object after the 'retain'.
-//===----------------------------------------------------------------------===//
-
-@interface rdar_6833332 : NSObject <NSApplicationDelegate> {
-    NSWindow *window;
-}
-@property (nonatomic, retain) NSWindow *window;
-@end
-
-@implementation rdar_6833332
-@synthesize window;
-- (void)applicationDidFinishLaunching:(NSNotification *)aNotification {
- NSMutableDictionary *dict = [[NSMutableDictionary dictionaryWithCapacity:4] retain]; // expected-warning{{leak}}
-
- [dict setObject:@"foo" forKey:@"bar"];
-
- NSLog(@"%@", dict);
-}
-- (void)dealloc {
-    [window release];
-    [super dealloc];
-}
-
-- (void)radar10102244 {
- NSMutableDictionary *dict = [[NSMutableDictionary dictionaryWithCapacity:4] retain]; // expected-warning{{leak}} 
- if (window) 
-   NSLog(@"%@", window);    
-}
-@end
-
-//===----------------------------------------------------------------------===//
-// clang checker fails to catch use-after-release
-//===----------------------------------------------------------------------===//
-int rdar_6257780_Case1(void) {
-  NSAutoreleasePool * pool = [[NSAutoreleasePool alloc] init];
-  NSArray *array = [NSArray array];
-  [array release]; // expected-warning{{Incorrect decrement of the reference count of an object that is not owned at this point by the caller}}
-  [pool drain];
-  return 0;
-}
-
-//===----------------------------------------------------------------------===//
-// Analyzer is confused about NSAutoreleasePool -allocWithZone:.
-//===----------------------------------------------------------------------===//
-void rdar_10640253_autorelease_allocWithZone(void) {
-    NSAutoreleasePool *pool = [[NSAutoreleasePool allocWithZone:(NSZone*)0] init];
-    (void) pool;
-}
-
-//===----------------------------------------------------------------------===//
-// Checker should understand new/setObject:/release constructs
-//===----------------------------------------------------------------------===//
-void rdar_6866843(void) {
- NSAutoreleasePool * pool = [[NSAutoreleasePool alloc] init];
- NSMutableDictionary* dictionary = [[NSMutableDictionary alloc] init];
- NSArray* array = [[NSArray alloc] init];
- [dictionary setObject:array forKey:@"key"];
- [array release];
- // Using 'array' here should be fine
- NSLog(@"array = %@\n", array); // no-warning
- // Now the array is released
- [dictionary release];
- [pool drain];
-}
-
-
-//===----------------------------------------------------------------------===//
-// Classes typedef-ed to CF objects should get the same treatment as CF objects
-//===----------------------------------------------------------------------===//
-typedef CFTypeRef OtherRef;
-
-@interface RDar6877235 : NSObject {}
-- (CFTypeRef)_copyCFTypeRef;
-- (OtherRef)_copyOtherRef;
-@end
-
-@implementation RDar6877235
-- (CFTypeRef)_copyCFTypeRef {
-  return [[NSString alloc] init]; // no-warning
-}
-- (OtherRef)_copyOtherRef {
-  return [[NSString alloc] init]; // no-warning
-}
-@end
-
-//===----------------------------------------------------------------------===//
-// false positive - init method returns an object owned by caller
-//===----------------------------------------------------------------------===//
-@interface RDar6320065 : NSObject {
-  NSString *_foo;
-}
-- (id)initReturningNewClass;
-- (id)_initReturningNewClassBad;
-- (id)initReturningNewClassBad2;
-@end
-
-@interface RDar6320065Subclass : RDar6320065
-@end
-
-@implementation RDar6320065
-- (id)initReturningNewClass {
-  [self release];
-  self = [[RDar6320065Subclass alloc] init]; // no-warning
-  return self;
-}
-- (id)_initReturningNewClassBad {
-  [self release];
-  [[RDar6320065Subclass alloc] init]; // expected-warning {{leak}}
-  return self;
-}
-- (id)initReturningNewClassBad2 {
-  [self release];
-  self = [[RDar6320065Subclass alloc] init];
-  return [self autorelease]; // expected-warning{{Object with a +0 retain count returned to caller where a +1 (owning) retain count is expected}}
-}
-
-@end
-
-@implementation RDar6320065Subclass
-@end
-
-int RDar6320065_test(void) {
-  RDar6320065 *test = [[RDar6320065 alloc] init]; // no-warning
-  [test release];
-  return 0;
-}
-
-//===----------------------------------------------------------------------===//
-// -awakeAfterUsingCoder: returns an owned object and claims the receiver
-//===----------------------------------------------------------------------===//
-@interface RDar7129086 : NSObject {} @end
-@implementation RDar7129086
-- (id)awakeAfterUsingCoder:(NSCoder *)aDecoder {
-  [self release]; // no-warning
-  return [NSString alloc];  // no-warning
-}
-@end
-
-//===----------------------------------------------------------------------===//
-// [NSData dataWithBytesNoCopy] does not return a retained object
-//===----------------------------------------------------------------------===//
-@interface RDar6859457 : NSObject {}
-- (NSString*) NoCopyString;
-- (NSString*) noCopyString;
-@end
-
-@implementation RDar6859457 
-- (NSString*) NoCopyString { return [[NSString alloc] init]; } // expected-warning{{leak}}
-- (NSString*) noCopyString { return [[NSString alloc] init]; } // expected-warning{{leak}}
-@end
-
-void test_RDar6859457(RDar6859457 *x, void *bytes, NSUInteger dataLength) {
-  [x NoCopyString]; // expected-warning{{leak}}
-  [x noCopyString]; // expected-warning{{leak}}
-  [NSData dataWithBytesNoCopy:bytes length:dataLength];  // no-warning
-  [NSData dataWithBytesNoCopy:bytes length:dataLength freeWhenDone:1]; // no-warning
-}
-
-//===----------------------------------------------------------------------===//
-// PR 4230 - an autorelease pool is not necessarily leaked during a premature
-//  return
-//===----------------------------------------------------------------------===//
-
-static void PR4230(void)
-{
-  NSAutoreleasePool *pool = [[NSAutoreleasePool alloc] init]; // no-warning
-  NSString *object = [[[NSString alloc] init] autorelease]; // no-warning
-  return;
-}
-
-static void PR4230_new(void)
-{
-  NSAutoreleasePool *pool = [NSAutoreleasePool new]; // no-warning
-  NSString *object = [[[NSString alloc] init] autorelease]; // no-warning
-  return;
-}
-
-//===----------------------------------------------------------------------===//
-// Method name that has a null IdentifierInfo* for its first selector slot.
-// This test just makes sure that we handle it.
-//===----------------------------------------------------------------------===//
-@interface TestNullIdentifier
-@end
-
-@implementation TestNullIdentifier
-+ (id):(int)x, ... {
-  return [[NSString alloc] init]; // expected-warning{{leak}}
-}
-@end
-
-//===----------------------------------------------------------------------===//
-// don't flag leaks for return types that cannot be determined to be CF types
-//===----------------------------------------------------------------------===//
-
-// We don't know if 'struct s6893565' represents a Core Foundation type, so
-// we shouldn't emit an error here.
-typedef struct s6893565* TD6893565;
-
-@interface RDar6893565 {}
--(TD6893565)newThing;
-@end
-
-@implementation RDar6893565
--(TD6893565)newThing {  
-  return (TD6893565) [[NSString alloc] init]; // no-warning
-}
-@end
-
-//===----------------------------------------------------------------------===//
-// clang: false positives w/QC and CoreImage methods
-//===----------------------------------------------------------------------===//
-void rdar6902710(QCView *view, QCRenderer *renderer, CIContext *context,
-                 NSString *str, CIImage *img, CGRect rect,
-                 CIFormat form, CGColorSpaceRef cs) {
-  [view createSnapshotImageOfType:str]; // expected-warning{{leak}}
-  [renderer createSnapshotImageOfType:str]; // expected-warning{{leak}}
-  [context createCGImage:img fromRect:rect]; // expected-warning{{leak}}
-  [context createCGImage:img fromRect:rect format:form colorSpace:cs]; // expected-warning{{leak}}
-}
-
-//===----------------------------------------------------------------------===//
-// -[CIContext createCGLayerWithSize:info:] misinterpreted by clang scan-build
-//===----------------------------------------------------------------------===//
-void rdar6945561(CIContext *context, CGSize size, CFDictionaryRef d) {
-  [context createCGLayerWithSize:size info:d]; // expected-warning{{leak}}
-}
-
-//===----------------------------------------------------------------------===//
-// Add knowledge of IOKit functions to retain/release checker.
-//===----------------------------------------------------------------------===//
-void IOBSDNameMatching_wrapper(mach_port_t mainPort, uint32_t options,  const char * bsdName) {  
-  IOBSDNameMatching(mainPort, options, bsdName); // expected-warning{{leak}}
-}
-
-void IOServiceMatching_wrapper(const char * name) {
-  IOServiceMatching(name); // expected-warning{{leak}}
-}
-
-void IOServiceNameMatching_wrapper(const char * name) {
-  IOServiceNameMatching(name); // expected-warning{{leak}}
-}
-
-CF_RETURNS_RETAINED CFDictionaryRef CreateDict(void);
-
-void IOServiceAddNotification_wrapper(mach_port_t mainPort, const io_name_t notificationType,
-  mach_port_t wakePort, uintptr_t reference, io_iterator_t * notification ) {
-
-  CFDictionaryRef matching = CreateDict();
-  CFRelease(matching);
-  IOServiceAddNotification(mainPort, notificationType, matching, // expected-warning{{used after it is released}} expected-warning{{deprecated}}
-                           wakePort, reference, notification);
-}
-
-void IORegistryEntryIDMatching_wrapper(uint64_t entryID ) {
-  IORegistryEntryIDMatching(entryID); // expected-warning{{leak}}
-}
-
-void IOOpenFirmwarePathMatching_wrapper(mach_port_t mainPort, uint32_t options,
-                                        const char * path) {
-  IOOpenFirmwarePathMatching(mainPort, options, path); // expected-warning{{leak}}
-}
-
-void IOServiceGetMatchingService_wrapper(mach_port_t mainPort) {
-  CFDictionaryRef matching = CreateDict();
-  IOServiceGetMatchingService(mainPort, matching);
-  CFRelease(matching); // expected-warning{{used after it is released}}
-}
-
-void IOServiceGetMatchingServices_wrapper(mach_port_t mainPort, io_iterator_t *existing) {
-  CFDictionaryRef matching = CreateDict();
-  IOServiceGetMatchingServices(mainPort, matching, existing);
-  CFRelease(matching); // expected-warning{{used after it is released}}
-}
-
-void IOServiceAddMatchingNotification_wrapper(IONotificationPortRef notifyPort, const io_name_t notificationType, 
-  IOServiceMatchingCallback callback, void * refCon, io_iterator_t * notification) {
-    
-  CFDictionaryRef matching = CreateDict();
-  IOServiceAddMatchingNotification(notifyPort, notificationType, matching, callback, refCon, notification);
-  CFRelease(matching); // expected-warning{{used after it is released}}
-}
-
-//===----------------------------------------------------------------------===//
-// Test of handling objects whose references "escape" to containers.
-//===----------------------------------------------------------------------===//
-void CFDictionaryAddValue(CFMutableDictionaryRef, void *, void *);
-
-void rdar_6539791(CFMutableDictionaryRef y, void* key, void* val_key) {
-  CFMutableDictionaryRef x = CFDictionaryCreateMutable(kCFAllocatorDefault, 1, &kCFTypeDictionaryKeyCallBacks, &kCFTypeDictionaryValueCallBacks);
-  CFDictionaryAddValue(y, key, x);
-  CFRelease(x); // the dictionary keeps a reference, so the object isn't deallocated yet
-  signed z = 1;
-  CFNumberRef value = CFNumberCreate(kCFAllocatorDefault, kCFNumberSInt32Type, &z);
-  if (value) {
-    CFDictionaryAddValue(x, val_key, (void*)value); // no-warning
-    CFRelease(value);
-    CFDictionaryAddValue(y, val_key, (void*)value); // no-warning
-  }
-}
-
-// Same issue, except with "AppendValue" functions.
-void rdar_6560661(CFMutableArrayRef x) {
-  signed z = 1;
-  CFNumberRef value = CFNumberCreate(kCFAllocatorDefault, kCFNumberSInt32Type, &z);
-  // CFArrayAppendValue keeps a reference to value.
-  CFArrayAppendValue(x, value);
-  CFRelease(value);
-  CFRetain(value);
-  CFRelease(value); // no-warning
-}
-
-// Same issue, excwept with "CFAttributeStringSetAttribute".
-void rdar_7152619(CFStringRef str) {
-  CFAttributedStringRef string = CFAttributedStringCreate(kCFAllocatorDefault, str, 0);
-  CFMutableAttributedStringRef attrString = CFAttributedStringCreateMutableCopy(kCFAllocatorDefault, 100, string);
-  CFRelease(string);
-  NSNumber *number = [[NSNumber alloc] initWithInt:5]; // expected-warning{{leak}}
-  CFAttributedStringSetAttribute(attrString, CFRangeMake(0, 1), str, number);
-  [number release];
-  [number retain];
-  CFRelease(attrString);  
-}
-
-//===----------------------------------------------------------------------===//
-// Test of handling CGGradientXXX functions.
-//===----------------------------------------------------------------------===//
-
-void rdar_7184450(CGContextRef myContext, CGFloat x, CGPoint myStartPoint,
-                  CGPoint myEndPoint) {
-  size_t num_locations = 6;
-  CGFloat locations[6] = { 0.0, 0.265, 0.28, 0.31, 0.36, 1.0 };
-  CGFloat components[28] = { 239.0/256.0, 167.0/256.0, 170.0/256.0,
-     x,  // Start color
-    207.0/255.0, 39.0/255.0, 39.0/255.0, x,
-    147.0/255.0, 21.0/255.0, 22.0/255.0, x,
-    175.0/255.0, 175.0/255.0, 175.0/255.0, x,
-    255.0/255.0,255.0/255.0, 255.0/255.0, x,
-    255.0/255.0,255.0/255.0, 255.0/255.0, x
-  }; // End color
-  
-  CGGradientRef myGradient =
-    CGGradientCreateWithColorComponents(CGColorSpaceCreateDeviceRGB(), // expected-warning{{leak}}
-      components, locations, num_locations);
-
-  CGContextDrawLinearGradient(myContext, myGradient, myStartPoint, myEndPoint,
-                              0);
-  CGGradientRelease(myGradient);
-}
-
-void rdar_7184450_pos(CGContextRef myContext, CGFloat x, CGPoint myStartPoint,
-                  CGPoint myEndPoint) {
-  size_t num_locations = 6;
-  CGFloat locations[6] = { 0.0, 0.265, 0.28, 0.31, 0.36, 1.0 };
-  CGFloat components[28] = { 239.0/256.0, 167.0/256.0, 170.0/256.0,
-     x,  // Start color
-    207.0/255.0, 39.0/255.0, 39.0/255.0, x,
-    147.0/255.0, 21.0/255.0, 22.0/255.0, x,
-    175.0/255.0, 175.0/255.0, 175.0/255.0, x,
-    255.0/255.0,255.0/255.0, 255.0/255.0, x,
-    255.0/255.0,255.0/255.0, 255.0/255.0, x
-  }; // End color
-  
-  CGGradientRef myGradient =
-   CGGradientCreateWithColorComponents(CGColorSpaceCreateDeviceRGB(), components, locations, num_locations); // expected-warning 2 {{leak}}
-
-  CGContextDrawLinearGradient(myContext, myGradient, myStartPoint, myEndPoint,
-                              0);
-}
-
-//===----------------------------------------------------------------------===//
-// clang false positive: retained instance passed to thread in pthread_create
-// marked as leak
-//
-// Until we have full IPA, the analyzer should stop tracking the reference
-// count of objects passed to pthread_create.
-//
-//===----------------------------------------------------------------------===//
-struct _opaque_pthread_t {};
-struct _opaque_pthread_attr_t {};
-typedef struct _opaque_pthread_t *__darwin_pthread_t;
-typedef struct _opaque_pthread_attr_t __darwin_pthread_attr_t;
-typedef __darwin_pthread_t pthread_t;
-typedef __darwin_pthread_attr_t pthread_attr_t;
-typedef unsigned long __darwin_pthread_key_t;
-typedef __darwin_pthread_key_t pthread_key_t;
-
-int pthread_create(pthread_t *, const pthread_attr_t *,
-                   void *(*)(void *), void *);
-
-int pthread_setspecific(pthread_key_t key, const void *value);
-
-void *rdar_7299394_start_routine(void *p) {
-  [((id) p) release];
-  return 0;
-}
-void rdar_7299394(pthread_attr_t *attr, pthread_t *thread, void *args) {
-  NSNumber *number = [[NSNumber alloc] initWithInt:5]; // no-warning
-  pthread_create(thread, attr, rdar_7299394_start_routine, number);
-}
-void rdar_7299394_positive(pthread_attr_t *attr, pthread_t *thread) {
-  NSNumber *number = [[NSNumber alloc] initWithInt:5]; // expected-warning{{leak}}
-}
-
-//===----------------------------------------------------------------------===//
-// false positive with not understanding thread local storage
-//===----------------------------------------------------------------------===//
-void rdar11282706(pthread_key_t key) {
-  NSNumber *number = [[NSNumber alloc] initWithInt:5]; // no-warning
-  pthread_setspecific(key, (void*) number);
-}
-
-//===----------------------------------------------------------------------===//
-// False leak associated with call to CVPixelBufferCreateWithBytes ()
-//
-// According to the Core Video Reference (ADC), CVPixelBufferCreateWithBytes and
-// CVPixelBufferCreateWithPlanarBytes can release (via a callback) the
-// pixel buffer object.  These test cases show how the analyzer stops tracking
-// the reference count for the objects passed for this argument.  This
-// could be made smarter.
-//===----------------------------------------------------------------------===//
-typedef int int32_t;
-typedef UInt32 FourCharCode;
-typedef FourCharCode OSType;
-typedef uint64_t CVOptionFlags;
-typedef int32_t CVReturn;
-typedef struct __CVBuffer *CVBufferRef;
-typedef CVBufferRef CVImageBufferRef;
-typedef CVImageBufferRef CVPixelBufferRef;
-typedef void (*CVPixelBufferReleaseBytesCallback)( void *releaseRefCon, const void *baseAddress );
-
-extern CVReturn CVPixelBufferCreateWithBytes(CFAllocatorRef allocator,
-            size_t width,
-            size_t height,
-            OSType pixelFormatType,
-            void *baseAddress,
-            size_t bytesPerRow,
-            CVPixelBufferReleaseBytesCallback releaseCallback,
-            void *releaseRefCon,
-            CFDictionaryRef pixelBufferAttributes,
-                   CVPixelBufferRef *pixelBufferOut) ;
-
-typedef void (*CVPixelBufferReleasePlanarBytesCallback)( void *releaseRefCon, const void *dataPtr, size_t dataSize, size_t numberOfPlanes, const void *planeAddresses[] );
-
-extern CVReturn CVPixelBufferCreateWithPlanarBytes(CFAllocatorRef allocator,
-        size_t width,
-        size_t height,
-        OSType pixelFormatType,
-        void *dataPtr,
-        size_t dataSize,
-        size_t numberOfPlanes,
-        void *planeBaseAddress[],
-        size_t planeWidth[],
-        size_t planeHeight[],
-        size_t planeBytesPerRow[],
-        CVPixelBufferReleasePlanarBytesCallback releaseCallback,
-        void *releaseRefCon,
-        CFDictionaryRef pixelBufferAttributes,
-        CVPixelBufferRef *pixelBufferOut) ;
-
-extern CVReturn CVPixelBufferCreateWithBytes(CFAllocatorRef allocator,
-            size_t width,
-            size_t height,
-            OSType pixelFormatType,
-            void *baseAddress,
-            size_t bytesPerRow,
-            CVPixelBufferReleaseBytesCallback releaseCallback,
-            void *releaseRefCon,
-            CFDictionaryRef pixelBufferAttributes,
-                   CVPixelBufferRef *pixelBufferOut) ;
-
-CVReturn rdar_7283567(CFAllocatorRef allocator, size_t width, size_t height,
-                      OSType pixelFormatType, void *baseAddress,
-                      size_t bytesPerRow,
-                      CVPixelBufferReleaseBytesCallback releaseCallback,
-                      CFDictionaryRef pixelBufferAttributes,
-                      CVPixelBufferRef *pixelBufferOut) {
-
-  // For the allocated object, it doesn't really matter what type it is
-  // for the purpose of this test.  All we want to show is that
-  // this is freed later by the callback.
-  NSNumber *number = [[NSNumber alloc] initWithInt:5]; // no-warning
-  
-  return CVPixelBufferCreateWithBytes(allocator, width, height, pixelFormatType,
-                                baseAddress, bytesPerRow, releaseCallback,
-                                number, // potentially released by callback
-                                pixelBufferAttributes, pixelBufferOut) ;
-}
-
-CVReturn rdar_7283567_2(CFAllocatorRef allocator, size_t width, size_t height,
-        OSType pixelFormatType, void *dataPtr, size_t dataSize,
-        size_t numberOfPlanes, void *planeBaseAddress[],
-        size_t planeWidth[], size_t planeHeight[], size_t planeBytesPerRow[],
-        CVPixelBufferReleasePlanarBytesCallback releaseCallback,
-        CFDictionaryRef pixelBufferAttributes,
-        CVPixelBufferRef *pixelBufferOut) {
-    
-    // For the allocated object, it doesn't really matter what type it is
-    // for the purpose of this test.  All we want to show is that
-    // this is freed later by the callback.
-    NSNumber *number = [[NSNumber alloc] initWithInt:5]; // no-warning
-
-    return CVPixelBufferCreateWithPlanarBytes(allocator,
-              width, height, pixelFormatType, dataPtr, dataSize,
-              numberOfPlanes, planeBaseAddress, planeWidth,
-              planeHeight, planeBytesPerRow, releaseCallback,
-              number, // potentially released by callback
-              pixelBufferAttributes, pixelBufferOut) ;
-}
-
-//===----------------------------------------------------------------------===//
-// False leak associated with CGBitmapContextCreateWithData
-//===----------------------------------------------------------------------===//
-typedef uint32_t CGBitmapInfo;
-typedef void (*CGBitmapContextReleaseDataCallback)(void *releaseInfo, void *data);
-    
-CGContextRef CGBitmapContextCreateWithData(void *data,
-    size_t width, size_t height, size_t bitsPerComponent,
-    size_t bytesPerRow, CGColorSpaceRef space, CGBitmapInfo bitmapInfo,
-    CGBitmapContextReleaseDataCallback releaseCallback, void *releaseInfo);
-
-void rdar_7358899(void *data,
-      size_t width, size_t height, size_t bitsPerComponent,
-      size_t bytesPerRow, CGColorSpaceRef space, CGBitmapInfo bitmapInfo,
-      CGBitmapContextReleaseDataCallback releaseCallback) {
-
-    // For the allocated object, it doesn't really matter what type it is
-    // for the purpose of this test.  All we want to show is that
-    // this is freed later by the callback.
-    NSNumber *number = [[NSNumber alloc] initWithInt:5]; // no-warning
-
-  CGBitmapContextCreateWithData(data, width, height, bitsPerComponent, // expected-warning{{leak}}
-    bytesPerRow, space, bitmapInfo, releaseCallback, number);
-}
-
-//===----------------------------------------------------------------------===//
-// Allow 'new', 'copy', 'alloc', 'init' prefix to start before '_' when
-// determining Cocoa fundamental rule.
-//
-// Previously the retain/release checker just skipped prefixes before the
-// first '_' entirely.  Now the checker honors the prefix if it results in a
-// recognizable naming convention (e.g., 'new', 'init').
-//===----------------------------------------------------------------------===//
-@interface RDar7265711 {}
-- (id) new_stuff;
-@end
-
-void rdar7265711_a(RDar7265711 *x) {
-  id y = [x new_stuff]; // expected-warning{{leak}}
-}
-
-void rdar7265711_b(RDar7265711 *x) {
-  id y = [x new_stuff]; // no-warning
-  [y release];
-}
-
-//===----------------------------------------------------------------------===//
-// clang thinks [NSCursor dragCopyCursor] returns a retained reference
-//===----------------------------------------------------------------------===//
-@interface NSCursor : NSObject
-+ (NSCursor *)dragCopyCursor;
-@end
-
-void rdar7306898(void) {
-  // 'dragCopyCursor' does not follow Cocoa's fundamental rule.  It is a noun, not an sentence
-  // implying a 'copy' of something.
-  NSCursor *c =  [NSCursor dragCopyCursor]; // no-warning
-  NSNumber *number = [[NSNumber alloc] initWithInt:5]; // expected-warning{{leak}}
-}
-
-//===----------------------------------------------------------------------===//
-// Sending 'release', 'retain', etc. to a Class directly is not likely what the
-// user intended.
-//===----------------------------------------------------------------------===//
-@interface RDar7252064 : NSObject @end
-void rdar7252064(void) {
-  [RDar7252064 release]; // expected-warning{{The 'release' message should be sent to instances of class 'RDar7252064' and not the class directly}}
-  [RDar7252064 retain]; // expected-warning{{The 'retain' message should be sent to instances of class 'RDar7252064' and not the class directly}}
-  [RDar7252064 autorelease]; // expected-warning{{The 'autorelease' message should be sent to instances of class 'RDar7252064' and not the class directly}}
-  [NSAutoreleasePool drain]; // expected-warning{{method '+drain' not found}} expected-warning{{The 'drain' message should be sent to instances of class 'NSAutoreleasePool' and not the class directly}}
-}
-
-//===----------------------------------------------------------------------===//
-// Tests of ownership attributes.
-//===----------------------------------------------------------------------===//
-
-typedef NSString* MyStringTy;
-
-@protocol FooP;
-
-@interface TestOwnershipAttr : NSObject
-- (NSString*) returnsAnOwnedString  NS_RETURNS_RETAINED; // no-warning
-- (NSString*) returnsAnOwnedCFString  CF_RETURNS_RETAINED; // no-warning
-- (MyStringTy) returnsAnOwnedTypedString NS_RETURNS_RETAINED; // no-warning
-- (NSString*) newString NS_RETURNS_NOT_RETAINED; // no-warning
-- (NSString*) newString_auto NS_RETURNS_AUTORELEASED; // no-warning
-- (NSString*) newStringNoAttr;
-- (int) returnsAnOwnedInt NS_RETURNS_RETAINED; // expected-warning{{'ns_returns_retained' attribute only applies to methods that return an Objective-C object}}
-- (id) pseudoInit NS_CONSUMES_SELF NS_RETURNS_RETAINED;
-+ (void) consume:(id) NS_CONSUMED x;
-+ (void) consume2:(id) CF_CONSUMED x;
-@end
-
-static int ownership_attribute_doesnt_go_here NS_RETURNS_RETAINED; // expected-warning{{'ns_returns_retained' attribute only applies to functions and methods}}
-
-void test_attr_1(TestOwnershipAttr *X) {
-  NSString *str = [X returnsAnOwnedString]; // expected-warning{{leak}}
-}
-
-void test_attr_1b(TestOwnershipAttr *X) {
-  NSString *str = [X returnsAnOwnedCFString]; // expected-warning{{leak}}
-}
-
-void test_attr1c(TestOwnershipAttr *X) {
-  NSString *str = [X newString]; // no-warning
-  NSString *str2 = [X newStringNoAttr]; // expected-warning{{leak}}
-  NSString *str3 = [X newString_auto]; // no-warning
-  NSString *str4 = [[X newString_auto] retain]; // expected-warning {{leak}}
-}
-
-void testattr2_a(void) {
-  TestOwnershipAttr *x = [TestOwnershipAttr alloc]; // expected-warning{{leak}}
-}
-
-void testattr2_b(void) {
-  TestOwnershipAttr *x = [[TestOwnershipAttr alloc] pseudoInit];  // expected-warning{{leak}}
-}
-
-void testattr2_b_11358224_self_assign_looses_the_leak(void) {
-  TestOwnershipAttr *x = [[TestOwnershipAttr alloc] pseudoInit];// expected-warning{{leak}}
-  x = x;
-}
-
-void testattr2_c(void) {
-  TestOwnershipAttr *x = [[TestOwnershipAttr alloc] pseudoInit]; // no-warning
-  [x release];
-}
-
-void testattr3(void) {
-  TestOwnershipAttr *x = [TestOwnershipAttr alloc]; // no-warning
-  [TestOwnershipAttr consume:x];
-  TestOwnershipAttr *y = [TestOwnershipAttr alloc]; // no-warning
-  [TestOwnershipAttr consume2:y];
-}
-
-void consume_ns(id NS_CONSUMED x);
-void consume_cf(id CF_CONSUMED x);
-
-void testattr4(void) {
-  TestOwnershipAttr *x = [TestOwnershipAttr alloc]; // no-warning
-  consume_ns(x);
-  TestOwnershipAttr *y = [TestOwnershipAttr alloc]; // no-warning
-  consume_cf(y);
-}
-
-@interface TestOwnershipAttr2 : NSObject
-- (NSString*) newString NS_RETURNS_NOT_RETAINED; // no-warning
-@end
-
-@implementation TestOwnershipAttr2
-- (NSString*) newString {
-  return [NSString alloc]; // expected-warning {{Potential leak of an object}}
-}
-@end
-
-@interface MyClassTestCFAttr : NSObject {}
-- (NSDate*) returnsCFRetained CF_RETURNS_RETAINED;
-- (CFDateRef) returnsCFRetainedAsCF CF_RETURNS_RETAINED;
-- (CFDateRef) newCFRetainedAsCF CF_RETURNS_NOT_RETAINED;
-- (CFDateRef) newCFRetainedAsCFNoAttr;
-- (NSDate*) alsoReturnsRetained;
-- (CFDateRef) alsoReturnsRetainedAsCF;
-- (NSDate*) returnsNSRetained NS_RETURNS_RETAINED;
-@end
-
-CF_RETURNS_RETAINED
-CFDateRef returnsRetainedCFDate(void)  {
-  return CFDateCreate(0, CFAbsoluteTimeGetCurrent());
-}
-
-@implementation MyClassTestCFAttr
-- (NSDate*) returnsCFRetained {
-  return (NSDate*) returnsRetainedCFDate(); // No leak.
-}
-
-- (CFDateRef) returnsCFRetainedAsCF {
-  return returnsRetainedCFDate(); // No leak.
-}
-
-- (CFDateRef) newCFRetainedAsCF {
-  return (CFDateRef)[(id)[self returnsCFRetainedAsCF] autorelease];
-}
-
-- (CFDateRef) newCFRetainedAsCFNoAttr {
-  return (CFDateRef)[(id)[self returnsCFRetainedAsCF] autorelease]; // expected-warning{{Object with a +0 retain count returned to caller where a +1 (owning) retain count is expected}}
-}
-
-- (NSDate*) alsoReturnsRetained {
-  return (NSDate*) returnsRetainedCFDate(); // expected-warning{{leak}}
-}
-
-- (CFDateRef) alsoReturnsRetainedAsCF {
-  return returnsRetainedCFDate(); // expected-warning{{leak}}
-}
-
-
-- (NSDate*) returnsNSRetained {
-  return (NSDate*) returnsRetainedCFDate(); // no-warning
-}
-@end
-
-//===----------------------------------------------------------------------===//
-// Test that leaks post-dominated by "panic" functions are not reported.
-//
-// Do not report a leak when post-dominated by a call to a noreturn or panic
-// function.
-//===----------------------------------------------------------------------===//
-void panic(void) __attribute__((noreturn));
-void panic_not_in_hardcoded_list(void) __attribute__((noreturn));
-
-void test_panic_negative(void) {
-  signed z = 1;
-  CFNumberRef value = CFNumberCreate(kCFAllocatorDefault, kCFNumberSInt32Type, &z);  // expected-warning{{leak}}
-}
-
-void test_panic_positive(void) {
-  signed z = 1;
-  CFNumberRef value = CFNumberCreate(kCFAllocatorDefault, kCFNumberSInt32Type, &z); // no-warning
-  panic();
-}
-
-void test_panic_neg_2(int x) {
-  signed z = 1;
-  CFNumberRef value = CFNumberCreate(kCFAllocatorDefault, kCFNumberSInt32Type, &z); // expected-warning{{leak}}
-  if (x)
-    panic();
-}
-
-void test_panic_pos_2(int x) {
-  signed z = 1;
-  CFNumberRef value = CFNumberCreate(kCFAllocatorDefault, kCFNumberSInt32Type, &z); // no-warning
-  if (x)
-    panic();
-  if (!x) {
-    // This showed up previously where we silently missed checking the function
-    // type for noreturn.  "panic()" is a hard-coded known panic function that
-    // isn't always noreturn.
-    panic_not_in_hardcoded_list();
-  }
-}
-
-//===----------------------------------------------------------------------===//
-// Test uses of blocks (closures)
-//===----------------------------------------------------------------------===//
-
-void test_blocks_1_pos(void) {
-  NSNumber *number = [[NSNumber alloc] initWithInt:5]; // expected-warning{{leak}}
-  ^{}();
-}
-
-void test_blocks_1_indirect_release(void) {
-  NSNumber *number = [[NSNumber alloc] initWithInt:5]; // no-warning
-  ^{ [number release]; }();
-}
-
-void test_blocks_1_indirect_retain(void) {
-  // Eventually this should be reported as a leak.
-  NSNumber *number = [[NSNumber alloc] initWithInt:5]; // no-warning
-  ^{ [number retain]; }();
-}
-
-void test_blocks_1_indirect_release_via_call(void) {
-  NSNumber *number = [[NSNumber alloc] initWithInt:5]; // no-warning
-  ^(NSObject *o){ [o release]; }(number);
-}
-
-void test_blocks_1_indirect_retain_via_call(void) {
-  NSNumber *number = [[NSNumber alloc] initWithInt:5]; // expected-warning {{leak}}
-  ^(NSObject *o){ [o retain]; }(number);
-}
-
-//===--------------------------------------------------------------------===//
-// Test sending message to super that returns an object alias.  Previously
-// this caused a crash in the analyzer.
-//===--------------------------------------------------------------------===//
-
-@interface Rdar8015556 : NSObject {} @end
-@implementation Rdar8015556
-- (id)retain {
-  return [super retain];
-}
-@end
-
-// Correcly handle Class<...> in Cocoa Conventions detector.
-@protocol Prot_R8272168 @end
-Class <Prot_R8272168> GetAClassThatImplementsProt_R8272168(void);
-void r8272168(void) {
-  GetAClassThatImplementsProt_R8272168();
-}
-
-// Test case which in the past triggered a false positive.
-@interface RDar8356342
-- (NSDate*) rdar8356342:(NSDate *)inValue;
-@end
-
-@implementation RDar8356342
-- (NSDate*) rdar8356342:(NSDate*)inValue {
-  NSDate *outValue = inValue;
-  if (outValue == 0)
-    outValue = [[NSDate alloc] init]; // no-warning
-
-  if (outValue != inValue)
-    [outValue autorelease];
-
-  return outValue;
-}
-@end
-
-// This test case previously crashed because of a bug in BugReporter.
-extern const void *CFDictionaryGetValue(CFDictionaryRef theDict, const void *key);
-typedef struct __CFError * CFErrorRef;
-extern const CFStringRef kCFErrorUnderlyingErrorKey;
-extern CFDictionaryRef CFErrorCopyUserInfo(CFErrorRef err);
-static void rdar_8724287(CFErrorRef error)
-{
-    CFErrorRef error_to_dump;
-
-    error_to_dump = error;
-    while (error_to_dump != ((void*)0)) {
-        CFDictionaryRef info;
-
-        info = CFErrorCopyUserInfo(error_to_dump); // expected-warning{{Potential leak of an object}}
-
-        if (info != ((void*)0)) {
-        }
-
-        error_to_dump = (CFErrorRef) CFDictionaryGetValue(info, kCFErrorUnderlyingErrorKey);
-    }
-}
-
-// Make sure the model applies cf_consumed correctly in argument positions
-// besides the first.
-extern void *CFStringCreate(void);
-extern void rdar_9234108_helper(void *key, void * CF_CONSUMED value);
-void rdar_9234108(void) {
-  rdar_9234108_helper(0, CFStringCreate());
-}
-
-// Make sure that objc_method_family works to override naming conventions.
-struct TwoDoubles {
-  double one;
-  double two;
-};
-typedef struct TwoDoubles TwoDoubles;
-
-@interface NSValue (Mine)
-- (id)_prefix_initWithTwoDoubles:(TwoDoubles)twoDoubles __attribute__((objc_method_family(init)));
-@end
-
-@implementation NSValue (Mine)
-- (id)_prefix_initWithTwoDoubles:(TwoDoubles)twoDoubles
-{
-  return [self init];
-}
-@end
-
-void rdar9726279(void) {
-  TwoDoubles twoDoubles = { 0.0, 0.0 };
-  NSValue *value = [[NSValue alloc] _prefix_initWithTwoDoubles:twoDoubles];
-  [value release];
-}
-
-// Test camelcase support for CF conventions.  While Core Foundation APIs
-// don't use camel casing, other code is allowed to use it.
-CFArrayRef camelcase_create_1(void) {
-  return CFArrayCreateMutable(0, 10, &kCFTypeArrayCallBacks); // no-warning
-}
-
-CFArrayRef camelcase_createno(void) {
-  return CFArrayCreateMutable(0, 10, &kCFTypeArrayCallBacks); // expected-warning {{leak}}
-}
-
-CFArrayRef camelcase_copy(void) {
-  return CFArrayCreateMutable(0, 10, &kCFTypeArrayCallBacks); // no-warning
-}
-
-CFArrayRef camelcase_copying(void) {
-  return CFArrayCreateMutable(0, 10, &kCFTypeArrayCallBacks); // expected-warning {{leak}}
-}
-
-CFArrayRef copyCamelCase(void) {
-  return CFArrayCreateMutable(0, 10, &kCFTypeArrayCallBacks); // no-warning
-}
-
-CFArrayRef __copyCamelCase(void) {
-  return CFArrayCreateMutable(0, 10, &kCFTypeArrayCallBacks); // no-warning
-}
-
-CFArrayRef __createCamelCase(void) {
-  return CFArrayCreateMutable(0, 10, &kCFTypeArrayCallBacks); // no-warning
-}
-
-CFArrayRef camel_create(void) {
-  return CFArrayCreateMutable(0, 10, &kCFTypeArrayCallBacks); // no-warning
-}
-
-
-CFArrayRef camel_creat(void) {
-  return CFArrayCreateMutable(0, 10, &kCFTypeArrayCallBacks); // expected-warning {{leak}}
-}
-
-CFArrayRef camel_copy(void) {
-  return CFArrayCreateMutable(0, 10, &kCFTypeArrayCallBacks); // no-warning
-}
-
-CFArrayRef camel_copyMachine(void) {
-  return CFArrayCreateMutable(0, 10, &kCFTypeArrayCallBacks); // no-warning
-}
-
-CFArrayRef camel_copymachine(void) {
-  return CFArrayCreateMutable(0, 10, &kCFTypeArrayCallBacks); // expected-warning {{leak}}
-}
-
-@protocol F18P
-- (id) clone;
-@end
-@interface F18 : NSObject<F18P> @end
-@interface F18(Cat)
-- (id) clone NS_RETURNS_RETAINED;
-@end
-
-@implementation F18
-- (id) clone {
-  return [F18 alloc];
-}
-@end
-
-void rdar6582778(void) {
-  CFAbsoluteTime t = CFAbsoluteTimeGetCurrent();
-  CFTypeRef vals[] = { CFDateCreate(0, t) }; // expected-warning {{leak}}
-}
-
-CFTypeRef global;
-
-void rdar6582778_2(void) {
-  CFAbsoluteTime t = CFAbsoluteTimeGetCurrent();
-  global = CFDateCreate(0, t); // no-warning
-}
-
-// Test that objects passed to containers are marked "escaped".
-void rdar10232019(void) {
-  NSMutableArray *array = [NSMutableArray array];
-
-  NSString *string = [[NSString alloc] initWithUTF8String:"foo"];
-  [array addObject:string];
-  [string release];
-
-  NSString *otherString = [string stringByAppendingString:@"bar"]; // no-warning
-  NSLog(@"%@", otherString);
-}
-
-void rdar10232019_positive(void) {
-  NSMutableArray *array = [NSMutableArray array];
-
-  NSString *string = [[NSString alloc] initWithUTF8String:"foo"];
-  [string release];
-
-  NSString *otherString = [string stringByAppendingString:@"bar"]; // expected-warning {{Reference-counted object is used after it is release}}
-  NSLog(@"%@", otherString);
-}
-
-// RetainCountChecker support for XPC.
-typedef void * xpc_object_t;
-xpc_object_t _CFXPCCreateXPCObjectFromCFObject(CFTypeRef cf);
-void xpc_release(xpc_object_t object);
-
-void rdar9658496(void) {
-  CFStringRef cf;
-  xpc_object_t xpc;
-  cf = CFStringCreateWithCString( ((CFAllocatorRef)0), "test", kCFStringEncodingUTF8 ); // no-warning
-  xpc = _CFXPCCreateXPCObjectFromCFObject( cf );
-  CFRelease(cf);
-  xpc_release(xpc);
-}
-
-// Support annotations with method families.
-@interface RDar10824732 : NSObject
-- (id)initWithObj:(id CF_CONSUMED)obj;
-@end
-
-@implementation RDar10824732
-- (id)initWithObj:(id)obj {
-  [obj release];
-  return [super init];
-}
-@end
-
-void rdar_10824732(void) {
-  @autoreleasepool {
-    NSString *obj = @"test";
-    RDar10824732 *foo = [[RDar10824732 alloc] initWithObj:obj]; // no-warning
-    [foo release];
-  }
-}
-
-// Stop tracking objects passed to functions, which take callbacks as parameters.
-typedef int (*CloseCallback) (void *);
-void ReaderForIO(CloseCallback ioclose, void *ioctx);
-int IOClose(void *context);
-
-@protocol SInS <NSObject>
-@end
-
-@interface radar10973977 : NSObject
-- (id<SInS>)inputS;
-- (void)reader;
-@end
-
-@implementation radar10973977
-- (void)reader
-{
-    id<SInS> inputS = [[self inputS] retain];
-    ReaderForIO(IOClose, inputS);
-}
-- (id<SInS>)inputS
-{
-    return 0;
-}
-@end
-
-// Object escapes through a selector callback
-extern id NSApp;
-@interface MySheetController
-- (id<SInS>)inputS;
-- (void)showDoSomethingSheetAction:(id)action;
-- (void)sheetDidEnd:(NSWindow *)sheet returnCode:(int)returnCode contextInfo:(void *)contextInfo;
-@end
-
-@implementation MySheetController
-- (id<SInS>)inputS {
-    return 0;
-}
-- (void)showDoSomethingSheetAction:(id)action {
-  id<SInS> inputS = [[self inputS] retain]; 
-  [NSApp beginSheet:0
-         modalForWindow:0
-         modalDelegate:0
-         didEndSelector:@selector(sheetDidEnd:returnCode:contextInfo:)
-         contextInfo:(void *)inputS]; // no - warning
-}
-- (void)sheetDidEnd:(NSWindow *)sheet returnCode:(int)returnCode contextInfo:(void *)contextInfo {
-   
-      id contextObject = (id)contextInfo;
-      [contextObject release];
-}
-
-- (id)copyAutoreleaseRadar13081402 {
-  id x = [[[NSString alloc] initWithUTF8String:"foo"] autorelease];
-  [x retain];
-  return x; // no warning
-}
-
-@end
-//===----------------------------------------------------------------------===//
-// Test returning allocated memory in a struct.
-//
-// We currently don't have a general way to track pointers that "escape".
-// Here we test that RetainCountChecker doesn't get excited about returning
-// allocated CF objects in struct fields.
-//===----------------------------------------------------------------------===//
-void *malloc(size_t);
-struct rdar11104566 { CFStringRef myStr; };
-struct rdar11104566 test_rdar11104566(void) {
-  CFStringRef cf = CFStringCreateWithCString( ((CFAllocatorRef)0), "test", kCFStringEncodingUTF8 ); // no-warning
-  struct rdar11104566 V;
-  V.myStr = cf;
-  return V; // no-warning
-}
-
-struct rdar11104566 *test_2_rdar11104566(void) {
-  CFStringRef cf = CFStringCreateWithCString( ((CFAllocatorRef)0), "test", kCFStringEncodingUTF8 ); // no-warning
-  struct rdar11104566 *V = (struct rdar11104566 *) malloc(sizeof(*V));
-  V->myStr = cf;
-  return V; // no-warning
-}
-
-//===----------------------------------------------------------------------===//
-// ObjC literals support.
-//===----------------------------------------------------------------------===//
-
-void test_objc_arrays(void) {
-    { // CASE ONE -- OBJECT IN ARRAY CREATED DIRECTLY
-        NSObject *o = [[NSObject alloc] init];
-        NSArray *a = [[NSArray alloc] initWithObjects:o, (void*)0]; // expected-warning {{leak}}
-        [o release];
-        [a description];
-        [o description];
-    }
-
-    { // CASE TWO -- OBJECT IN ARRAY CREATED BY DUPING AUTORELEASED ARRAY
-        NSObject *o = [[NSObject alloc] init];
-        NSArray *a1 = [NSArray arrayWithObjects:o, (void*)0];
-        NSArray *a2 = [[NSArray alloc] initWithArray:a1]; // expected-warning {{leak}}
-        [o release];        
-        [a2 description];
-        [o description];
-    }
-
-    { // CASE THREE -- OBJECT IN RETAINED @[]
-        NSObject *o = [[NSObject alloc] init];
-        NSArray *a3 = [@[o] retain]; // expected-warning {{leak}}
-        [o release];        
-        [a3 description];
-        [o description];
-    }
-    
-    { // CASE FOUR -- OBJECT IN ARRAY CREATED BY DUPING @[]
-        NSObject *o = [[NSObject alloc] init];
-        NSArray *a = [[NSArray alloc] initWithArray:@[o]]; // expected-warning {{leak}}
-        [o release];
-        
-        [a description];
-        [o description];
-    }
-    
-    { // CASE FIVE -- OBJECT IN RETAINED @{}
-        NSValue *o = [[NSValue alloc] init];
-        NSDictionary *a = [@{o : o} retain]; // expected-warning {{leak}}
-        [o release];
-        
-        [a description];
-        [o description];
-    }
-}
-
-void test_objc_integer_literals(void) {
-  id value = [@1 retain]; // expected-warning {{leak}}
-  [value description];
-}
-
-void test_objc_boxed_expressions(int x, const char *y) {
-  id value = [@(x) retain]; // expected-warning {{leak}}
-  [value description];
-
-  value = [@(y) retain]; // expected-warning {{leak}}
-  [value description];
-}
-
-// Test NSLog doesn't escape tracked objects.
-void rdar11400885(int y)
-{
-  @autoreleasepool {
-    NSString *printString;
-    if(y > 2)
-      printString = [[NSString alloc] init];
-    else
-      printString = [[NSString alloc] init];
-    NSLog(@"Once %@", printString);
-    [printString release];
-    NSLog(@"Again: %@", printString); // expected-warning {{Reference-counted object is used after it is released}}
-  }
-}
-
-id makeCollectableNonLeak(void) {
-  extern CFTypeRef CFCreateSomething(void);
-
-  CFTypeRef object = CFCreateSomething(); // +1
-  CFRetain(object); // +2
-  id objCObject = NSMakeCollectable(object); // +2
-  [objCObject release]; // +1
-  return [objCObject autorelease]; // +0
-}
-
-
-void consumeAndStopTracking(id NS_CONSUMED obj, void (^callback)(void));
-void CFConsumeAndStopTracking(CFTypeRef CF_CONSUMED obj, void (^callback)(void));
-
-void testConsumeAndStopTracking(void) {
-  id retained = [@[] retain]; // +1
-  consumeAndStopTracking(retained, ^{}); // no-warning
-
-  id doubleRetained = [[@[] retain] retain]; // +2
-  consumeAndStopTracking(doubleRetained, ^{
-    [doubleRetained release];
-  }); // no-warning
-
-  id unretained = @[]; // +0
-  consumeAndStopTracking(unretained, ^{}); // expected-warning {{Incorrect decrement of the reference count of an object that is not owned at this point by the caller}}
-}
-
-void testCFConsumeAndStopTracking(void) {
-  id retained = [@[] retain]; // +1
-  CFConsumeAndStopTracking((CFTypeRef)retained, ^{}); // no-warning
-
-  id doubleRetained = [[@[] retain] retain]; // +2
-  CFConsumeAndStopTracking((CFTypeRef)doubleRetained, ^{
-    [doubleRetained release];
-  }); // no-warning
-
-  id unretained = @[]; // +0
-  CFConsumeAndStopTracking((CFTypeRef)unretained, ^{}); // expected-warning {{Incorrect decrement of the reference count of an object that is not owned at this point by the caller}}
-}
-//===----------------------------------------------------------------------===//
-// Test 'pragma clang arc_cf_code_audited' support.
-//===----------------------------------------------------------------------===//
-
-typedef void *MyCFType;
-#pragma clang arc_cf_code_audited begin
-MyCFType CreateMyCFType(void);
-#pragma clang arc_cf_code_audited end 
-    
-void test_custom_cf(void) {
-  MyCFType x = CreateMyCFType(); // expected-warning {{leak of an object stored into 'x'}}
-}
-
-//===----------------------------------------------------------------------===//
-// Test calling CFPlugInInstanceCreate, which appears in CF but doesn't
-// return a CF object.
-//===----------------------------------------------------------------------===//
-
-void test_CFPlugInInstanceCreate(CFUUIDRef factoryUUID, CFUUIDRef typeUUID) {
-  CFPlugInInstanceCreate(kCFAllocatorDefault, factoryUUID, typeUUID); // no-warning
-}
-
-//===----------------------------------------------------------------------===//
-// PR14927: -drain only has retain-count semantics on NSAutoreleasePool.
-//===----------------------------------------------------------------------===//
-
-@interface PR14927 : NSObject
-- (void)drain;
-@end
-
-void test_drain(void) {
-  PR14927 *obj = [[PR14927 alloc] init];
-  [obj drain];
-  [obj release]; // no-warning
-}
-
-//===----------------------------------------------------------------------===//
-// Allow cf_returns_retained and cf_returns_not_retained to mark a return
-// value as tracked, even if the object isn't a known CF type.
-//===----------------------------------------------------------------------===//
-
-MyCFType getCustom(void) __attribute__((cf_returns_not_retained));
-MyCFType makeCustom(void) __attribute__((cf_returns_retained));
-
-void testCustomReturnsRetained(void) {
-  MyCFType obj = makeCustom(); // expected-warning {{leak of an object stored into 'obj'}}
-}
-
-void testCustomReturnsNotRetained(void) {
-  CFRelease(getCustom()); // expected-warning {{Incorrect decrement of the reference count of an object that is not owned at this point by the caller}}
-}
-
-//===----------------------------------------------------------------------===//
-// Don't print variables which are out of the current scope.
-//===----------------------------------------------------------------------===//
-@interface MyObj12706177 : NSObject
--(id)initX;
-+(void)test12706177;
-@end
-static int Cond;
-@implementation MyObj12706177
--(id)initX {
-  if (Cond)
-    return 0;
-  self = [super init];
-  return self;
-}
-+(void)test12706177 {
-  id x = [[MyObj12706177 alloc] initX]; //expected-warning {{Potential leak of an object}}
-  [x release]; 
-}
-@end
-
-//===----------------------------------------------------------------------===//
-// xpc_connection_set_finalizer_f
-//===----------------------------------------------------------------------===//
-typedef xpc_object_t xpc_connection_t;
-typedef void (*xpc_finalizer_t)(void *value);
-void xpc_connection_set_context(xpc_connection_t connection, void *ctx);
-void xpc_connection_set_finalizer_f(xpc_connection_t connection,
-                                    xpc_finalizer_t finalizer);
-void releaseAfterXPC(void *context) {
-  [(NSArray *)context release];
-}
-
-void rdar13783514(xpc_connection_t connection) {
-  xpc_connection_set_context(connection, [[NSMutableArray alloc] init]);
-  xpc_connection_set_finalizer_f(connection, releaseAfterXPC);
-} // no-warning
-
-CFAttributedStringRef CFAttributedCreate(void *CFObj CF_CONSUMED) CF_RETURNS_RETAINED;
-
-@interface Action
-- (SEL)action;
-- (void)setAction:(SEL)aSelector;
-- (id) target;
-- (void)setTarget:(id)aTarget;
-@end
diff --git a/clang/test/ARCMT/objcmt-arc-cf-annotations.m.result b/clang/test/ARCMT/objcmt-arc-cf-annotations.m.result
deleted file mode 100644
index 1e94129784880..0000000000000
--- a/clang/test/ARCMT/objcmt-arc-cf-annotations.m.result
+++ /dev/null
@@ -1,2063 +0,0 @@
-// RUN: rm -rf %t
-// RUN: %clang_cc1 -fblocks -objcmt-migrate-annotation -objcmt-migrate-instancetype -objcmt-migrate-readwrite-property -mt-migrate-directory %t %s -x objective-c -triple x86_64-apple-darwin11
-// RUN: c-arcmt-test -mt-migrate-directory %t | arcmt-test -verify-transformed-files %s.result
-// RUN: %clang_cc1 -fblocks -triple x86_64-apple-darwin10 -fsyntax-only -x objective-c %s.result
-
-#ifndef CF_IMPLICIT_BRIDGING_ENABLED
-#if __has_feature(arc_cf_code_audited)
-#define CF_IMPLICIT_BRIDGING_ENABLED _Pragma("clang arc_cf_code_audited begin")
-#else
-#define CF_IMPLICIT_BRIDGING_ENABLED
-#endif
-#endif
-
-#ifndef CF_IMPLICIT_BRIDGING_DISABLED
-#if __has_feature(arc_cf_code_audited)
-#define CF_IMPLICIT_BRIDGING_DISABLED _Pragma("clang arc_cf_code_audited end")
-#else
-#define CF_IMPLICIT_BRIDGING_DISABLED
-#endif
-#endif
-
-#if __has_feature(attribute_ns_returns_retained)
-#define NS_RETURNS_RETAINED __attribute__((ns_returns_retained))
-#endif
-#if __has_feature(attribute_cf_returns_retained)
-#define CF_RETURNS_RETAINED __attribute__((cf_returns_retained))
-#endif
-#if __has_feature(attribute_ns_returns_not_retained)
-#define NS_RETURNS_NOT_RETAINED __attribute__((ns_returns_not_retained))
-#endif
-#if __has_feature(attribute_cf_returns_not_retained)
-#define CF_RETURNS_NOT_RETAINED __attribute__((cf_returns_not_retained))
-#endif
-#if __has_feature(attribute_ns_consumes_self)
-#define NS_CONSUMES_SELF __attribute__((ns_consumes_self))
-#endif
-#if __has_feature(attribute_ns_consumed)
-#define NS_CONSUMED __attribute__((ns_consumed))
-#endif
-#if __has_feature(attribute_cf_consumed)
-#define CF_CONSUMED __attribute__((cf_consumed))
-#endif
-#if __has_attribute(ns_returns_autoreleased)
-#define NS_RETURNS_AUTORELEASED __attribute__((ns_returns_autoreleased))
-#endif
-
-//===----------------------------------------------------------------------===//
-// The following code is reduced using delta-debugging from Mac OS X headers:
-//
-// #include <Cocoa/Cocoa.h>
-// #include <CoreFoundation/CoreFoundation.h>
-// #include <DiskArbitration/DiskArbitration.h>
-// #include <QuartzCore/QuartzCore.h>
-// #include <Quartz/Quartz.h>
-// #include <IOKit/IOKitLib.h>
-//
-// It includes the basic definitions for the test cases below.
-//===----------------------------------------------------------------------===//
-
-typedef unsigned int __darwin_natural_t;
-typedef unsigned long uintptr_t;
-typedef unsigned int uint32_t;
-typedef unsigned long long uint64_t;
-typedef unsigned int UInt32;
-typedef signed long CFIndex;
-typedef CFIndex CFByteOrder;
-typedef struct {
-    CFIndex location;
-    CFIndex length;
-} CFRange;
-static __inline__ __attribute__((always_inline)) CFRange CFRangeMake(CFIndex loc, CFIndex len) {
-    CFRange range;
-    range.location = loc;
-    range.length = len;
-    return range;
-}
-typedef const void * CFTypeRef;
-typedef const struct __CFString * CFStringRef;
-typedef const struct __CFAllocator * CFAllocatorRef;
-extern const CFAllocatorRef kCFAllocatorDefault;
-extern CFTypeRef CFRetain(CFTypeRef cf);
-
-CF_IMPLICIT_BRIDGING_ENABLED
-
-extern void CFRelease(CFTypeRef cf);
-
-CF_IMPLICIT_BRIDGING_DISABLED
-
-extern CFTypeRef CFAutorelease(CFTypeRef cf);
-extern CFTypeRef CFMakeCollectable(CFTypeRef cf);
-typedef struct {
-}
-CFArrayCallBacks;
-extern const CFArrayCallBacks kCFTypeArrayCallBacks;
-typedef const struct __CFArray * CFArrayRef;
-typedef struct __CFArray * CFMutableArrayRef;
-extern CFMutableArrayRef CFArrayCreateMutable(CFAllocatorRef allocator, CFIndex capacity, const CFArrayCallBacks *callBacks) CF_RETURNS_RETAINED;
-extern const void *CFArrayGetValueAtIndex(CFArrayRef theArray, CFIndex idx) CF_RETURNS_NOT_RETAINED;
-extern void CFArrayAppendValue(CFMutableArrayRef theArray, const void *value);
-typedef struct {
-}
-CFDictionaryKeyCallBacks;
-extern const CFDictionaryKeyCallBacks kCFTypeDictionaryKeyCallBacks;
-typedef struct {
-}
-CFDictionaryValueCallBacks;
-extern const CFDictionaryValueCallBacks kCFTypeDictionaryValueCallBacks;
-typedef const struct __CFDictionary * CFDictionaryRef;
-typedef struct __CFDictionary * CFMutableDictionaryRef;
-extern CFMutableDictionaryRef CFDictionaryCreateMutable(CFAllocatorRef allocator, CFIndex capacity, const CFDictionaryKeyCallBacks *keyCallBacks, const CFDictionaryValueCallBacks *valueCallBacks) CF_RETURNS_RETAINED;
-typedef UInt32 CFStringEncoding;
-enum {
-kCFStringEncodingMacRoman = 0,     kCFStringEncodingWindowsLatin1 = 0x0500,     kCFStringEncodingISOLatin1 = 0x0201,     kCFStringEncodingNextStepLatin = 0x0B01,     kCFStringEncodingASCII = 0x0600,     kCFStringEncodingUnicode = 0x0100,     kCFStringEncodingUTF8 = 0x08000100,     kCFStringEncodingNonLossyASCII = 0x0BFF      ,     kCFStringEncodingUTF16 = 0x0100,     kCFStringEncodingUTF16BE = 0x10000100,     kCFStringEncodingUTF16LE = 0x14000100,      kCFStringEncodingUTF32 = 0x0c000100,     kCFStringEncodingUTF32BE = 0x18000100,     kCFStringEncodingUTF32LE = 0x1c000100  };
-extern CFStringRef CFStringCreateWithCString(CFAllocatorRef alloc, const char *cStr, CFStringEncoding encoding) CF_RETURNS_RETAINED;
-typedef double CFTimeInterval;
-typedef CFTimeInterval CFAbsoluteTime;
-extern CFAbsoluteTime CFAbsoluteTimeGetCurrent(void);
-typedef const struct __CFDate * CFDateRef;
-extern CFDateRef CFDateCreate(CFAllocatorRef allocator, CFAbsoluteTime at) CF_RETURNS_RETAINED;
-extern CFAbsoluteTime CFDateGetAbsoluteTime(CFDateRef theDate);
-typedef __darwin_natural_t natural_t;
-typedef natural_t mach_port_name_t;
-typedef mach_port_name_t mach_port_t;
-typedef int kern_return_t;
-typedef kern_return_t mach_error_t;
-enum {
-kCFNumberSInt8Type = 1,     kCFNumberSInt16Type = 2,     kCFNumberSInt32Type = 3,     kCFNumberSInt64Type = 4,     kCFNumberFloat32Type = 5,     kCFNumberFloat64Type = 6,      kCFNumberCharType = 7,     kCFNumberShortType = 8,     kCFNumberIntType = 9,     kCFNumberLongType = 10,     kCFNumberLongLongType = 11,     kCFNumberFloatType = 12,     kCFNumberDoubleType = 13,      kCFNumberCFIndexType = 14,      kCFNumberNSIntegerType = 15,     kCFNumberCGFloatType = 16,     kCFNumberMaxType = 16    };
-typedef CFIndex CFNumberType;
-typedef const struct __CFNumber * CFNumberRef;
-extern CFNumberRef CFNumberCreate(CFAllocatorRef allocator, CFNumberType theType, const void *valuePtr) CF_RETURNS_RETAINED;
-typedef const struct __CFAttributedString *CFAttributedStringRef;
-typedef struct __CFAttributedString *CFMutableAttributedStringRef;
-extern CFAttributedStringRef CFAttributedStringCreate(CFAllocatorRef alloc, CFStringRef str, CFDictionaryRef attributes) CF_RETURNS_RETAINED ;
-extern CFMutableAttributedStringRef CFAttributedStringCreateMutableCopy(CFAllocatorRef alloc, CFIndex maxLength, CFAttributedStringRef aStr) CF_RETURNS_RETAINED ;
-extern void CFAttributedStringSetAttribute(CFMutableAttributedStringRef aStr, CFRange range, CFStringRef attrName, CFTypeRef value) ;
-typedef signed char BOOL;
-typedef unsigned long NSUInteger;
-@class NSString, Protocol;
-extern void NSLog(NSString *format, ...) __attribute__((format(__NSString__, 1, 2)));
-typedef struct _NSZone NSZone;
-@class NSInvocation, NSMethodSignature, NSCoder, NSString, NSEnumerator;
-@protocol NSObject
-- (BOOL)isEqual:(id)object;
-- (id)retain;
-- (oneway void)release;
-- (id)autorelease;
-- (NSString *)description;
-- (instancetype)init;
-@end
-@protocol NSCopying 
-- (id)copyWithZone:(NSZone *)zone;
-@end
-@protocol NSMutableCopying  - (id)mutableCopyWithZone:(NSZone *)zone;
-@end
-@protocol NSCoding  - (void)encodeWithCoder:(NSCoder *)aCoder;
-@end
-@interface NSObject <NSObject> {}
-+ (id)allocWithZone:(NSZone *)zone;
-+ (id)alloc;
-+ (id)new;
-- (void)dealloc;
-@end
-@interface NSObject (NSCoderMethods)
-- (id)awakeAfterUsingCoder:(NSCoder *)aDecoder;
-@end
-extern id NSAllocateObject(Class aClass, NSUInteger extraBytes, NSZone *zone);
-typedef struct {
-}
-NSFastEnumerationState;
-@protocol NSFastEnumeration 
-- (NSUInteger)countByEnumeratingWithState:(NSFastEnumerationState *)state objects:(id *)stackbuf count:(NSUInteger)len;
-@end
-@class NSString, NSDictionary;
-@interface NSValue : NSObject <NSCopying, NSCoding>  - (void)getValue:(void *)value;
-@end
-@interface NSNumber : NSValue
-- (char)charValue;
-- (instancetype)initWithInt:(int)value;
-+ (NSNumber *)numberWithInt:(int)value;
-@end
-@class NSString;
-@interface NSArray : NSObject <NSCopying, NSMutableCopying, NSCoding, NSFastEnumeration>
-- (NSUInteger)count;
-- (instancetype)initWithObjects:(const id [])objects count:(NSUInteger)cnt;
-+ (instancetype)arrayWithObject:(id)anObject;
-+ (instancetype)arrayWithObjects:(const id [])objects count:(NSUInteger)cnt;
-+ (instancetype)arrayWithObjects:(id)firstObj, ... __attribute__((sentinel(0,1)));
-- (instancetype)initWithObjects:(id)firstObj, ... __attribute__((sentinel(0,1)));
-- (instancetype)initWithArray:(NSArray *)array;
-@end  @interface NSArray (NSArrayCreation)  + (instancetype)array;
-@end       @interface NSAutoreleasePool : NSObject {
-}
-- (void)drain;
-@end extern NSString * const NSBundleDidLoadNotification;
-typedef double NSTimeInterval;
-@interface NSDate : NSObject <NSCopying, NSCoding>  - (NSTimeInterval)timeIntervalSinceReferenceDate;
-@end            typedef unsigned short unichar;
-@interface NSString : NSObject <NSCopying, NSMutableCopying, NSCoding>
-- (NSUInteger)length;
-- (NSString *)stringByAppendingString:(NSString *)aString;
-- ( const char *)UTF8String;
-- (instancetype)initWithUTF8String:(const char *)nullTerminatedCString;
-+ (instancetype)stringWithUTF8String:(const char *)nullTerminatedCString;
-@end        @class NSString, NSURL, NSError;
-@interface NSData : NSObject <NSCopying, NSMutableCopying, NSCoding>  - (NSUInteger)length;
-+ (instancetype)dataWithBytesNoCopy:(void *)bytes length:(NSUInteger)length;
-+ (instancetype)dataWithBytesNoCopy:(void *)bytes length:(NSUInteger)length freeWhenDone:(BOOL)b;
-@end   @class NSLocale, NSDate, NSCalendar, NSTimeZone, NSError, NSArray, NSMutableDictionary;
-@interface NSDictionary : NSObject <NSCopying, NSMutableCopying, NSCoding, NSFastEnumeration>
-- (NSUInteger)count;
-+ (instancetype)dictionaryWithObjects:(NSArray *)objects forKeys:(NSArray *)keys;
-+ (instancetype)dictionaryWithObjects:(const id [])objects forKeys:(const id <NSCopying> [])keys count:(NSUInteger)cnt;
-@end
-@interface NSMutableDictionary : NSDictionary  - (void)removeObjectForKey:(id)aKey;
-- (void)setObject:(id)anObject forKey:(id)aKey;
-@end  @interface NSMutableDictionary (NSMutableDictionaryCreation)  + (instancetype)dictionaryWithCapacity:(NSUInteger)numItems;
-@end  typedef double CGFloat;
-struct CGSize {
-};
-typedef struct CGSize CGSize;
-struct CGRect {
-};
-typedef struct CGRect CGRect;
-typedef mach_port_t io_object_t;
-typedef char io_name_t[128];
-typedef io_object_t io_iterator_t;
-typedef io_object_t io_service_t;
-typedef struct IONotificationPort * IONotificationPortRef;
-typedef void (*IOServiceMatchingCallback)(  void * refcon,  io_iterator_t iterator );
-
-CF_IMPLICIT_BRIDGING_ENABLED
-
-io_service_t IOServiceGetMatchingService(  mach_port_t mainPort,  CFDictionaryRef matching );
-kern_return_t IOServiceGetMatchingServices(  mach_port_t mainPort,  CFDictionaryRef matching,  io_iterator_t * existing );
-
-CF_IMPLICIT_BRIDGING_DISABLED
-
-kern_return_t IOServiceAddNotification(  mach_port_t mainPort,  const io_name_t notificationType,  CFDictionaryRef matching,  mach_port_t wakePort,  uintptr_t reference,  io_iterator_t * notification ) __attribute__((deprecated)); // expected-note {{'IOServiceAddNotification' declared here}}
-kern_return_t IOServiceAddMatchingNotification(  IONotificationPortRef notifyPort,  const io_name_t notificationType,  CFDictionaryRef CF_CONSUMED matching,         IOServiceMatchingCallback callback,         void * refCon,  io_iterator_t * notification );
-
-CF_IMPLICIT_BRIDGING_ENABLED
-
-CFMutableDictionaryRef IOServiceMatching(  const char * name );
-CFMutableDictionaryRef IOServiceNameMatching(  const char * name );
-CFMutableDictionaryRef IOBSDNameMatching(  mach_port_t mainPort,  uint32_t options,  const char * bsdName );
-CFMutableDictionaryRef IOOpenFirmwarePathMatching(  mach_port_t mainPort,  uint32_t options,  const char * path );
-CFMutableDictionaryRef IORegistryEntryIDMatching(  uint64_t entryID );
-
-CF_IMPLICIT_BRIDGING_DISABLED
-
-typedef struct __DASession * DASessionRef;
-extern DASessionRef DASessionCreate( CFAllocatorRef allocator ) CF_RETURNS_RETAINED;
-typedef struct __DADisk * DADiskRef;
-extern DADiskRef DADiskCreateFromBSDName( CFAllocatorRef allocator, DASessionRef session, const char * name ) CF_RETURNS_RETAINED;
-extern DADiskRef DADiskCreateFromIOMedia( CFAllocatorRef allocator, DASessionRef session, io_service_t media ) CF_RETURNS_RETAINED;
-extern CFDictionaryRef DADiskCopyDescription( DADiskRef disk ) CF_RETURNS_RETAINED;
-extern DADiskRef DADiskCopyWholeDisk( DADiskRef disk ) CF_RETURNS_RETAINED;
-@interface NSTask : NSObject - (instancetype)init;
-@end                    typedef struct CGColorSpace *CGColorSpaceRef;
-typedef struct CGImage *CGImageRef;
-typedef struct CGLayer *CGLayerRef;
-@interface NSResponder : NSObject <NSCoding> {
-}
-@end    @protocol NSAnimatablePropertyContainer      - (id)animator;
-@end  extern NSString *NSAnimationTriggerOrderIn ;
-@interface NSView : NSResponder  <NSAnimatablePropertyContainer>  {
-}
-@end @protocol NSValidatedUserInterfaceItem - (SEL)action;
-@end   @protocol NSUserInterfaceValidations - (BOOL)validateUserInterfaceItem:(id <NSValidatedUserInterfaceItem>)anItem;
-@end  @class NSDate, NSDictionary, NSError, NSException, NSNotification;
-@class NSTextField, NSPanel, NSArray, NSWindow, NSImage, NSButton, NSError;
-@interface NSApplication : NSResponder <NSUserInterfaceValidations> {
-}
-- (void)beginSheet:(NSWindow *)sheet modalForWindow:(NSWindow *)docWindow modalDelegate:(id)modalDelegate didEndSelector:(SEL)didEndSelector contextInfo:(void *)contextInfo;
-@end   enum {
-NSTerminateCancel = 0,         NSTerminateNow = 1,         NSTerminateLater = 2 };
-typedef NSUInteger NSApplicationTerminateReply;
-@protocol NSApplicationDelegate <NSObject> @optional        - (NSApplicationTerminateReply)applicationShouldTerminate:(NSApplication *)sender;
-@end  @class NSAttributedString, NSEvent, NSFont, NSFormatter, NSImage, NSMenu, NSText, NSView, NSTextView;
-@interface NSCell : NSObject <NSCopying, NSCoding> {
-}
-@end 
-typedef struct {
-}
-CVTimeStamp;
-@interface CIImage : NSObject <NSCoding, NSCopying> {
-}
-typedef int CIFormat;
-@end  enum {
-kDAReturnSuccess = 0,     kDAReturnError = (((0x3eU)&0x3f)<<26) | (((0x368)&0xfff)<<14) | 0x01,     kDAReturnBusy = (((0x3eU)&0x3f)<<26) | (((0x368)&0xfff)<<14) | 0x02,     kDAReturnBadArgument = (((0x3eU)&0x3f)<<26) | (((0x368)&0xfff)<<14) | 0x03,     kDAReturnExclusiveAccess = (((0x3eU)&0x3f)<<26) | (((0x368)&0xfff)<<14) | 0x04,     kDAReturnNoResources = (((0x3eU)&0x3f)<<26) | (((0x368)&0xfff)<<14) | 0x05,     kDAReturnNotFound = (((0x3eU)&0x3f)<<26) | (((0x368)&0xfff)<<14) | 0x06,     kDAReturnNotMounted = (((0x3eU)&0x3f)<<26) | (((0x368)&0xfff)<<14) | 0x07,     kDAReturnNotPermitted = (((0x3eU)&0x3f)<<26) | (((0x368)&0xfff)<<14) | 0x08,     kDAReturnNotPrivileged = (((0x3eU)&0x3f)<<26) | (((0x368)&0xfff)<<14) | 0x09,     kDAReturnNotReady = (((0x3eU)&0x3f)<<26) | (((0x368)&0xfff)<<14) | 0x0A,     kDAReturnNotWritable = (((0x3eU)&0x3f)<<26) | (((0x368)&0xfff)<<14) | 0x0B,     kDAReturnUnsupported = (((0x3eU)&0x3f)<<26) | (((0x368)&0xfff)<<14) | 0x0C };
-typedef mach_error_t DAReturn;
-typedef const struct __DADissenter * DADissenterRef;
-extern DADissenterRef DADissenterCreate( CFAllocatorRef allocator, DAReturn status, CFStringRef string ) CF_RETURNS_RETAINED;
-@interface CIContext: NSObject {
-}
-- (CGImageRef)createCGImage:(CIImage *)im fromRect:(CGRect)r CF_RETURNS_RETAINED;
-- (CGImageRef)createCGImage:(CIImage *)im fromRect:(CGRect)r     format:(CIFormat)f colorSpace:(CGColorSpaceRef)cs CF_RETURNS_RETAINED;
-- (CGLayerRef)createCGLayerWithSize:(CGSize)size info:(CFDictionaryRef)d CF_RETURNS_RETAINED;
-@end extern NSString* const QCRendererEventKey;
-@protocol QCCompositionRenderer - (NSDictionary*) attributes;
-@end   @interface QCRenderer : NSObject <QCCompositionRenderer> {
-}
-- (id) createSnapshotImageOfType:(NSString*)type NS_RETURNS_RETAINED;
-@end  extern NSString* const QCViewDidStartRenderingNotification;
-@interface QCView : NSView <QCCompositionRenderer> {
-}
-- (id) createSnapshotImageOfType:(NSString*)type NS_RETURNS_RETAINED;
-@end    enum {
-ICEXIFOrientation1 = 1,     ICEXIFOrientation2 = 2,     ICEXIFOrientation3 = 3,     ICEXIFOrientation4 = 4,     ICEXIFOrientation5 = 5,     ICEXIFOrientation6 = 6,     ICEXIFOrientation7 = 7,     ICEXIFOrientation8 = 8, };
-@class ICDevice;
-@protocol ICDeviceDelegate <NSObject>  @required      - (void)didRemoveDevice:(ICDevice*)device;
-@end extern NSString *const ICScannerStatusWarmingUp;
-@class ICScannerDevice;
-@protocol ICScannerDeviceDelegate <ICDeviceDelegate>  @optional       - (void)scannerDeviceDidBecomeAvailable:(ICScannerDevice*)scanner;
-@end
-
-typedef long unsigned int __darwin_size_t;
-typedef __darwin_size_t size_t;
-typedef unsigned long CFTypeID;
-struct CGPoint {
-  CGFloat x;
-  CGFloat y;
-};
-typedef struct CGPoint CGPoint;
-typedef struct CGGradient *CGGradientRef;
-typedef uint32_t CGGradientDrawingOptions;
-extern CFTypeID CGGradientGetTypeID(void);
-extern CGGradientRef CGGradientCreateWithColorComponents(CGColorSpaceRef
-  space, const CGFloat components[], const CGFloat locations[], size_t count) CF_RETURNS_RETAINED;
-extern CGGradientRef CGGradientCreateWithColors(CGColorSpaceRef space,
-  CFArrayRef colors, const CGFloat locations[]) CF_RETURNS_RETAINED;
-extern CGGradientRef CGGradientRetain(CGGradientRef gradient);
-
-CF_IMPLICIT_BRIDGING_ENABLED
-
-extern void CGGradientRelease(CGGradientRef gradient);
-
-CF_IMPLICIT_BRIDGING_DISABLED
-
-typedef struct CGContext *CGContextRef;
-extern void CGContextDrawLinearGradient(CGContextRef context,
-    CGGradientRef gradient, CGPoint startPoint, CGPoint endPoint,
-    CGGradientDrawingOptions options);
-
-CF_IMPLICIT_BRIDGING_ENABLED
-
-extern CGColorSpaceRef CGColorSpaceCreateDeviceRGB(void);
-
-CF_IMPLICIT_BRIDGING_DISABLED
-
-
-@interface NSMutableArray : NSObject
-- (void)addObject:(id)object;
-+ (instancetype)array;
-@end
-
-// This is how NSMakeCollectable is declared in the OS X 10.8 headers.
-id NSMakeCollectable(CFTypeRef __attribute__((cf_consumed))) __attribute__((ns_returns_retained));
-
-typedef const struct __CFUUID * CFUUIDRef;
-
-extern
-void *CFPlugInInstanceCreate(CFAllocatorRef allocator, CFUUIDRef factoryUUID, CFUUIDRef typeUUID);
-
-//===----------------------------------------------------------------------===//
-// Test cases.
-//===----------------------------------------------------------------------===//
-
-CFAbsoluteTime f1(void) {
-  CFAbsoluteTime t = CFAbsoluteTimeGetCurrent();
-  CFDateRef date = CFDateCreate(0, t);
-  CFRetain(date);
-  CFRelease(date);
-  CFDateGetAbsoluteTime(date); // no-warning
-  CFRelease(date);
-  t = CFDateGetAbsoluteTime(date);   // expected-warning{{Reference-counted object is used after it is released}}
-  return t;
-}
-
-CFAbsoluteTime f2(void) {
-  CFAbsoluteTime t = CFAbsoluteTimeGetCurrent();
-  CFDateRef date = CFDateCreate(0, t);  
-  [((NSDate*) date) retain];
-  CFRelease(date);
-  CFDateGetAbsoluteTime(date); // no-warning
-  [((NSDate*) date) release];
-  t = CFDateGetAbsoluteTime(date);   // expected-warning{{Reference-counted object is used after it is released}}
-  return t;
-}
-
-
-NSDate* global_x;
-
-// Test to see if we suppress an error when we store the pointer
-// to a global.
-
-CFAbsoluteTime f3(void) {
-  CFAbsoluteTime t = CFAbsoluteTimeGetCurrent();
-  CFDateRef date = CFDateCreate(0, t);  
-  [((NSDate*) date) retain];
-  CFRelease(date);
-  CFDateGetAbsoluteTime(date); // no-warning
-  global_x = (NSDate*) date;  
-  [((NSDate*) date) release];
-  t = CFDateGetAbsoluteTime(date);   // no-warning
-  return t;
-}
-
-//---------------------------------------------------------------------------
-// Test case 'f4' differs for region store and basic store.  See
-// retain-release-region-store.m and retain-release-basic-store.m.
-//---------------------------------------------------------------------------
-
-// Test a leak.
-
-CFAbsoluteTime f5(int x) {  
-  CFAbsoluteTime t = CFAbsoluteTimeGetCurrent();
-  CFDateRef date = CFDateCreate(0, t); // expected-warning{{leak}}
-  
-  if (x)
-    CFRelease(date);
-  
-  return t;
-}
-
-// Test a leak involving the return.
-
-CFDateRef f6(int x) {  
-  CFDateRef date = CFDateCreate(0, CFAbsoluteTimeGetCurrent());  // expected-warning{{leak}}
-  CFRetain(date);
-  return date;
-}
-
-// Test a leak involving an overwrite.
-
-CFDateRef f7(void) {
-  CFDateRef date = CFDateCreate(0, CFAbsoluteTimeGetCurrent());  //expected-warning{{leak}}
-  CFRetain(date);
-  date = CFDateCreate(0, CFAbsoluteTimeGetCurrent()); // expected-warning {{leak}}
-  return date;
-}
-
-// Generalization of Create rule.  MyDateCreate returns a CFXXXTypeRef, and
-// has the word create.
-
-CF_IMPLICIT_BRIDGING_ENABLED
-
-CFDateRef MyDateCreate(void);
-
-CF_IMPLICIT_BRIDGING_DISABLED
-
-
-CFDateRef f8(void) {
-  CFDateRef date = MyDateCreate(); // expected-warning{{leak}}
-  CFRetain(date);  
-  return date;
-}
-
-__attribute__((cf_returns_retained)) CFDateRef f9(void) {
-  CFDateRef date = CFDateCreate(0, CFAbsoluteTimeGetCurrent()); // no-warning
-  int *p = 0;
-  // When allocations fail, CFDateCreate can return null.
-  if (!date) *p = 1; // expected-warning{{null}}
-  return date;
-}
-
-// Handle DiskArbitration API:
-//
-// http://developer.apple.com/DOCUMENTATION/DARWIN/Reference/DiscArbitrationFramework/
-//
-void f10(io_service_t media, DADiskRef d, CFStringRef s) {
-  DADiskRef disk = DADiskCreateFromBSDName(kCFAllocatorDefault, 0, "hello"); // expected-warning{{leak}}
-  if (disk) NSLog(@"ok");
-  
-  disk = DADiskCreateFromIOMedia(kCFAllocatorDefault, 0, media); // expected-warning{{leak}}
-  if (disk) NSLog(@"ok");
-
-  CFDictionaryRef dict = DADiskCopyDescription(d);  // expected-warning{{leak}}
-  if (dict) NSLog(@"ok"); 
-  
-  disk = DADiskCopyWholeDisk(d); // expected-warning{{leak}}
-  if (disk) NSLog(@"ok");
-    
-  DADissenterRef dissenter = DADissenterCreate(kCFAllocatorDefault,   // expected-warning{{leak}}
-                                                kDAReturnSuccess, s);
-  if (dissenter) NSLog(@"ok");
-  
-  DASessionRef session = DASessionCreate(kCFAllocatorDefault);  // expected-warning{{leak}}
-  if (session) NSLog(@"ok");
-}
-
-// Test retain/release checker with CFString and CFMutableArray.
-void f11(void) {
-  // Create the array.
-  CFMutableArrayRef A = CFArrayCreateMutable(0, 10, &kCFTypeArrayCallBacks);
-
-  // Create a string.
-  CFStringRef s1 = CFStringCreateWithCString(0, "hello world",
-                                             kCFStringEncodingUTF8);
-
-  // Add the string to the array.
-  CFArrayAppendValue(A, s1);
-  
-  // Decrement the reference count.
-  CFRelease(s1); // no-warning
-  
-  // Get the string.  We don't own it.
-  s1 = (CFStringRef) CFArrayGetValueAtIndex(A, 0);
-  
-  // Release the array.
-  CFRelease(A); // no-warning
-  
-  // Release the string.  This is a bug.
-  CFRelease(s1); // expected-warning{{Incorrect decrement of the reference count}}
-}
-
-// PR 3337: Handle functions declared using typedefs.
-typedef CFTypeRef CREATEFUN(void);
-
-CF_IMPLICIT_BRIDGING_ENABLED
-
-CFTypeRef MyCreateFun(void);
-
-CF_IMPLICIT_BRIDGING_DISABLED
-
-
-void f12(void) {
-  CFTypeRef o = MyCreateFun(); // expected-warning {{leak}}
-}
-
-void f13_autorelease(void) {
-  CFMutableArrayRef A = CFArrayCreateMutable(0, 10, &kCFTypeArrayCallBacks); // no-warning
-  [(id) A autorelease]; // no-warning
-}
-
-void f13_autorelease_b(void) {
-  CFMutableArrayRef A = CFArrayCreateMutable(0, 10, &kCFTypeArrayCallBacks);
-  [(id) A autorelease];
-  [(id) A autorelease];
-} // expected-warning{{Object autoreleased too many times}}
-
-CFMutableArrayRef f13_autorelease_c(void) {
-  CFMutableArrayRef A = CFArrayCreateMutable(0, 10, &kCFTypeArrayCallBacks);
-  [(id) A autorelease];
-  [(id) A autorelease]; 
-  return A; // expected-warning{{Object autoreleased too many times}}
-}
-
-CFMutableArrayRef f13_autorelease_d(void) {
-  CFMutableArrayRef A = CFArrayCreateMutable(0, 10, &kCFTypeArrayCallBacks);
-  [(id) A autorelease];
-  [(id) A autorelease]; 
-  CFMutableArrayRef B = CFArrayCreateMutable(0, 10, &kCFTypeArrayCallBacks); // expected-warning{{Object autoreleased too many times}}
-  CFRelease(B); // no-warning
-  while (1) {}
-}
-
-
-// This case exercises the logic where the leak site is the same as the allocation site.
-void f14_leakimmediately(void) {
-  CFArrayCreateMutable(0, 10, &kCFTypeArrayCallBacks); // expected-warning{{leak}}
-}
-
-// Test that we track an allocated object beyond the point where the *name*
-// of the variable storing the reference is no longer live.
-void f15(void) {
-  // Create the array.
-  CFMutableArrayRef A = CFArrayCreateMutable(0, 10, &kCFTypeArrayCallBacks);
-  CFMutableArrayRef *B = &A;
-  // At this point, the name 'A' is no longer live.
-  CFRelease(*B);  // no-warning
-}
-
-// Test when we pass NULL to CFRetain/CFRelease/CFMakeCollectable/CFAutorelease.
-void f16(int x, CFTypeRef p) {
-  if (p)
-    return;
-
-  switch (x) {
-  case 0:
-    CFRelease(p);
-    break;
-  case 1:
-    CFRetain(p);
-    break;
-  case 2:
-    CFMakeCollectable(p);
-    break;
-  case 3:
-    CFAutorelease(p);
-    break;
-  default:
-    break;
-  }
-}
-
-// Test that an object is non-null after CFRetain/CFRelease/CFMakeCollectable/CFAutorelease.
-void f17(int x, CFTypeRef p) {
-  switch (x) {
-  case 0:
-    CFRelease(p);
-    if (!p)
-      CFRelease(0); // no-warning
-    break;
-  case 1:
-    CFRetain(p);
-    if (!p)
-      CFRetain(0); // no-warning
-    break;
-  case 2:
-    CFMakeCollectable(p);
-    if (!p)
-      CFMakeCollectable(0); // no-warning
-    break;
-  case 3:
-    CFAutorelease(p);
-    if (!p)
-      CFAutorelease(0); // no-warning
-    break;
-  default:
-    break;
-  }
-}
-
-// Test basic tracking of ivars associated with 'self'.  For the retain/release
-// checker we currently do not want to flag leaks associated with stores
-// of tracked objects to ivars.
-@interface SelfIvarTest : NSObject {
-  id myObj;
-}
-- (void)test_self_tracking;
-@end
-
-@implementation SelfIvarTest
-- (void)test_self_tracking {
-  myObj = (id) CFArrayCreateMutable(0, 10, &kCFTypeArrayCallBacks); // no-warning
-}
-@end
-
-// Test return of non-owned objects in contexts where an owned object
-// is expected.
-@interface TestReturnNotOwnedWhenExpectedOwned
-- (NSString*)newString;
-@end
-
-@implementation TestReturnNotOwnedWhenExpectedOwned
-- (NSString*)newString {
-  NSString *s = [NSString stringWithUTF8String:"hello"];
-  return s; // expected-warning{{Object with a +0 retain count returned to caller where a +1 (owning) retain count is expected}}
-}
-@end
-
-int isFoo(char c);
-
-static void rdar_6659160(char *inkind, char *inname)
-{
-  // We currently expect that [NSObject alloc] cannot fail.  This
-  // will be a toggled flag in the future.  It can indeed return null, but
-  // Cocoa programmers generally aren't expected to reason about out-of-memory
-  // conditions.
-  NSString *kind = [[NSString alloc] initWithUTF8String:inkind];  // expected-warning{{leak}}
-  
-  // We do allow stringWithUTF8String to fail.  This isn't really correct, as
-  // far as returning 0.  In most error conditions it will throw an exception.
-  // If allocation fails it could return 0, but again this
-  // isn't expected.
-  NSString *name = [NSString stringWithUTF8String:inname];
-  if(!name)
-    return;
-
-  const char *kindC = 0;
-  const char *nameC = 0;
-  
-  // In both cases, we cannot reach a point down below where we
-  // dereference kindC or nameC with either being null.  This is because
-  // we assume that [NSObject alloc] doesn't fail and that we have the guard
-  // up above.
-  
-  if(kind)
-    kindC = [kind UTF8String];
-  if(name)
-    nameC = [name UTF8String];
-  if(!isFoo(kindC[0])) // expected-warning{{null}}
-    return;
-  if(!isFoo(nameC[0])) // no-warning
-    return;
-
-  [kind release];
-  [name release]; // expected-warning{{Incorrect decrement of the reference count}}
-}
-
-// PR 3677 - 'allocWithZone' should be treated as following the Cocoa naming
-//  conventions with respect to 'return'ing ownership.
-@interface PR3677: NSObject @end
-@implementation PR3677
-+ (id)allocWithZone:(NSZone *)inZone {
-  return [super allocWithZone:inZone];  // no-warning
-}
-@end
-
-// PR 3820 - Reason about calls to -dealloc
-void pr3820_DeallocInsteadOfRelease(void)
-{
-  id foo = [[NSString alloc] init]; // no-warning
-  [foo dealloc];
-  // foo is not leaked, since it has been deallocated.
-}
-
-void pr3820_ReleaseAfterDealloc(void)
-{
-  id foo = [[NSString alloc] init];
-  [foo dealloc];
-  [foo release];  // expected-warning{{used after it is release}}
-  // NSInternalInconsistencyException: message sent to deallocated object
-}
-
-void pr3820_DeallocAfterRelease(void)
-{
-  NSLog(@"\n\n[%s]", __FUNCTION__);
-  id foo = [[NSString alloc] init];
-  [foo release];
-  [foo dealloc]; // expected-warning{{used after it is released}}
-  // message sent to released object
-}
-
-// The problem here is that 'length' binds to '($0 - 1)' after '--length', but
-// SimpleConstraintManager doesn't know how to reason about
-// '($0 - 1) > constant'.  As a temporary hack, we drop the value of '($0 - 1)'
-// and conjure a new symbol.
-void rdar6704930(unsigned char *s, unsigned int length) {
-  NSString* name = 0;
-  if (s != 0) {
-    if (length > 0) {
-      while (length > 0) {
-        if (*s == ':') {
-          ++s;
-          --length;
-          name = [[NSString alloc] init]; // no-warning
-          break;
-        }
-        ++s;
-        --length;
-      }
-      if ((length == 0) && (name != 0)) {
-        [name release];
-        name = 0;
-      }
-      if (length == 0) { // no ':' found -> use it all as name
-        name = [[NSString alloc] init]; // no-warning
-      }
-    }
-  }
-
-  if (name != 0) {
-    [name release];
-  }
-}
-
-//===----------------------------------------------------------------------===//
-// One build of the analyzer accidentally stopped tracking the allocated
-// object after the 'retain'.
-//===----------------------------------------------------------------------===//
-
-@interface rdar_6833332 : NSObject <NSApplicationDelegate> {
-    NSWindow *window;
-}
-@property (nonatomic, retain) NSWindow *window;
-@end
-
-@implementation rdar_6833332
-@synthesize window;
-- (void)applicationDidFinishLaunching:(NSNotification *)aNotification {
- NSMutableDictionary *dict = [[NSMutableDictionary dictionaryWithCapacity:4] retain]; // expected-warning{{leak}}
-
- [dict setObject:@"foo" forKey:@"bar"];
-
- NSLog(@"%@", dict);
-}
-- (void)dealloc {
-    [window release];
-    [super dealloc];
-}
-
-- (void)radar10102244 {
- NSMutableDictionary *dict = [[NSMutableDictionary dictionaryWithCapacity:4] retain]; // expected-warning{{leak}} 
- if (window) 
-   NSLog(@"%@", window);    
-}
-@end
-
-//===----------------------------------------------------------------------===//
-// clang checker fails to catch use-after-release
-//===----------------------------------------------------------------------===//
-int rdar_6257780_Case1(void) {
-  NSAutoreleasePool * pool = [[NSAutoreleasePool alloc] init];
-  NSArray *array = [NSArray array];
-  [array release]; // expected-warning{{Incorrect decrement of the reference count of an object that is not owned at this point by the caller}}
-  [pool drain];
-  return 0;
-}
-
-//===----------------------------------------------------------------------===//
-// Analyzer is confused about NSAutoreleasePool -allocWithZone:.
-//===----------------------------------------------------------------------===//
-void rdar_10640253_autorelease_allocWithZone(void) {
-    NSAutoreleasePool *pool = [[NSAutoreleasePool allocWithZone:(NSZone*)0] init];
-    (void) pool;
-}
-
-//===----------------------------------------------------------------------===//
-// Checker should understand new/setObject:/release constructs
-//===----------------------------------------------------------------------===//
-void rdar_6866843(void) {
- NSAutoreleasePool * pool = [[NSAutoreleasePool alloc] init];
- NSMutableDictionary* dictionary = [[NSMutableDictionary alloc] init];
- NSArray* array = [[NSArray alloc] init];
- [dictionary setObject:array forKey:@"key"];
- [array release];
- // Using 'array' here should be fine
- NSLog(@"array = %@\n", array); // no-warning
- // Now the array is released
- [dictionary release];
- [pool drain];
-}
-
-
-//===----------------------------------------------------------------------===//
-// Classes typedef-ed to CF objects should get the same treatment as CF objects
-//===----------------------------------------------------------------------===//
-typedef CFTypeRef OtherRef;
-
-@interface RDar6877235 : NSObject {}
-- (CFTypeRef)_copyCFTypeRef CF_RETURNS_RETAINED;
-- (OtherRef)_copyOtherRef CF_RETURNS_RETAINED;
-@end
-
-@implementation RDar6877235
-- (CFTypeRef)_copyCFTypeRef {
-  return [[NSString alloc] init]; // no-warning
-}
-- (OtherRef)_copyOtherRef {
-  return [[NSString alloc] init]; // no-warning
-}
-@end
-
-//===----------------------------------------------------------------------===//
-// false positive - init method returns an object owned by caller
-//===----------------------------------------------------------------------===//
-@interface RDar6320065 : NSObject {
-  NSString *_foo;
-}
-- (instancetype)initReturningNewClass;
-- (id)_initReturningNewClassBad;
-- (instancetype)initReturningNewClassBad2;
-@end
-
-@interface RDar6320065Subclass : RDar6320065
-@end
-
-@implementation RDar6320065
-- (instancetype)initReturningNewClass {
-  [self release];
-  self = [[RDar6320065Subclass alloc] init]; // no-warning
-  return self;
-}
-- (id)_initReturningNewClassBad {
-  [self release];
-  [[RDar6320065Subclass alloc] init]; // expected-warning {{leak}}
-  return self;
-}
-- (instancetype)initReturningNewClassBad2 {
-  [self release];
-  self = [[RDar6320065Subclass alloc] init];
-  return [self autorelease]; // expected-warning{{Object with a +0 retain count returned to caller where a +1 (owning) retain count is expected}}
-}
-
-@end
-
-@implementation RDar6320065Subclass
-@end
-
-int RDar6320065_test(void) {
-  RDar6320065 *test = [[RDar6320065 alloc] init]; // no-warning
-  [test release];
-  return 0;
-}
-
-//===----------------------------------------------------------------------===//
-// -awakeAfterUsingCoder: returns an owned object and claims the receiver
-//===----------------------------------------------------------------------===//
-@interface RDar7129086 : NSObject {} @end
-@implementation RDar7129086
-- (id)awakeAfterUsingCoder:(NSCoder *)aDecoder {
-  [self release]; // no-warning
-  return [NSString alloc];  // no-warning
-}
-@end
-
-//===----------------------------------------------------------------------===//
-// [NSData dataWithBytesNoCopy] does not return a retained object
-//===----------------------------------------------------------------------===//
-@interface RDar6859457 : NSObject {}
-- (NSString*) NoCopyString;
-- (NSString*) noCopyString;
-@end
-
-@implementation RDar6859457 
-- (NSString*) NoCopyString { return [[NSString alloc] init]; } // expected-warning{{leak}}
-- (NSString*) noCopyString { return [[NSString alloc] init]; } // expected-warning{{leak}}
-@end
-
-void test_RDar6859457(RDar6859457 *x, void *bytes, NSUInteger dataLength) {
-  [x NoCopyString]; // expected-warning{{leak}}
-  [x noCopyString]; // expected-warning{{leak}}
-  [NSData dataWithBytesNoCopy:bytes length:dataLength];  // no-warning
-  [NSData dataWithBytesNoCopy:bytes length:dataLength freeWhenDone:1]; // no-warning
-}
-
-//===----------------------------------------------------------------------===//
-// PR 4230 - an autorelease pool is not necessarily leaked during a premature
-//  return
-//===----------------------------------------------------------------------===//
-
-static void PR4230(void)
-{
-  NSAutoreleasePool *pool = [[NSAutoreleasePool alloc] init]; // no-warning
-  NSString *object = [[[NSString alloc] init] autorelease]; // no-warning
-  return;
-}
-
-static void PR4230_new(void)
-{
-  NSAutoreleasePool *pool = [NSAutoreleasePool new]; // no-warning
-  NSString *object = [[[NSString alloc] init] autorelease]; // no-warning
-  return;
-}
-
-//===----------------------------------------------------------------------===//
-// Method name that has a null IdentifierInfo* for its first selector slot.
-// This test just makes sure that we handle it.
-//===----------------------------------------------------------------------===//
-@interface TestNullIdentifier
-@end
-
-@implementation TestNullIdentifier
-+ (id):(int)x, ... {
-  return [[NSString alloc] init]; // expected-warning{{leak}}
-}
-@end
-
-//===----------------------------------------------------------------------===//
-// don't flag leaks for return types that cannot be determined to be CF types
-//===----------------------------------------------------------------------===//
-
-// We don't know if 'struct s6893565' represents a Core Foundation type, so
-// we shouldn't emit an error here.
-typedef struct s6893565* TD6893565;
-
-@interface RDar6893565 {}
--(TD6893565)newThing;
-@end
-
-@implementation RDar6893565
--(TD6893565)newThing {  
-  return (TD6893565) [[NSString alloc] init]; // no-warning
-}
-@end
-
-//===----------------------------------------------------------------------===//
-// clang: false positives w/QC and CoreImage methods
-//===----------------------------------------------------------------------===//
-void rdar6902710(QCView *view, QCRenderer *renderer, CIContext *context,
-                 NSString *str, CIImage *img, CGRect rect,
-                 CIFormat form, CGColorSpaceRef cs) {
-  [view createSnapshotImageOfType:str]; // expected-warning{{leak}}
-  [renderer createSnapshotImageOfType:str]; // expected-warning{{leak}}
-  [context createCGImage:img fromRect:rect]; // expected-warning{{leak}}
-  [context createCGImage:img fromRect:rect format:form colorSpace:cs]; // expected-warning{{leak}}
-}
-
-//===----------------------------------------------------------------------===//
-// -[CIContext createCGLayerWithSize:info:] misinterpreted by clang scan-build
-//===----------------------------------------------------------------------===//
-void rdar6945561(CIContext *context, CGSize size, CFDictionaryRef d) {
-  [context createCGLayerWithSize:size info:d]; // expected-warning{{leak}}
-}
-
-//===----------------------------------------------------------------------===//
-// Add knowledge of IOKit functions to retain/release checker.
-//===----------------------------------------------------------------------===//
-void IOBSDNameMatching_wrapper(mach_port_t mainPort, uint32_t options,  const char * bsdName) {  
-  IOBSDNameMatching(mainPort, options, bsdName); // expected-warning{{leak}}
-}
-
-void IOServiceMatching_wrapper(const char * name) {
-  IOServiceMatching(name); // expected-warning{{leak}}
-}
-
-void IOServiceNameMatching_wrapper(const char * name) {
-  IOServiceNameMatching(name); // expected-warning{{leak}}
-}
-
-CF_RETURNS_RETAINED CFDictionaryRef CreateDict(void);
-
-void IOServiceAddNotification_wrapper(mach_port_t mainPort, const io_name_t notificationType,
-  mach_port_t wakePort, uintptr_t reference, io_iterator_t * notification ) {
-
-  CFDictionaryRef matching = CreateDict();
-  CFRelease(matching);
-  IOServiceAddNotification(mainPort, notificationType, matching, // expected-warning{{used after it is released}} expected-warning{{deprecated}}
-                           wakePort, reference, notification);
-}
-
-void IORegistryEntryIDMatching_wrapper(uint64_t entryID ) {
-  IORegistryEntryIDMatching(entryID); // expected-warning{{leak}}
-}
-
-void IOOpenFirmwarePathMatching_wrapper(mach_port_t mainPort, uint32_t options,
-                                        const char * path) {
-  IOOpenFirmwarePathMatching(mainPort, options, path); // expected-warning{{leak}}
-}
-
-void IOServiceGetMatchingService_wrapper(mach_port_t mainPort) {
-  CFDictionaryRef matching = CreateDict();
-  IOServiceGetMatchingService(mainPort, matching);
-  CFRelease(matching); // expected-warning{{used after it is released}}
-}
-
-void IOServiceGetMatchingServices_wrapper(mach_port_t mainPort, io_iterator_t *existing) {
-  CFDictionaryRef matching = CreateDict();
-  IOServiceGetMatchingServices(mainPort, matching, existing);
-  CFRelease(matching); // expected-warning{{used after it is released}}
-}
-
-void IOServiceAddMatchingNotification_wrapper(IONotificationPortRef notifyPort, const io_name_t notificationType, 
-  IOServiceMatchingCallback callback, void * refCon, io_iterator_t * notification) {
-    
-  CFDictionaryRef matching = CreateDict();
-  IOServiceAddMatchingNotification(notifyPort, notificationType, matching, callback, refCon, notification);
-  CFRelease(matching); // expected-warning{{used after it is released}}
-}
-
-//===----------------------------------------------------------------------===//
-// Test of handling objects whose references "escape" to containers.
-//===----------------------------------------------------------------------===//
-void CFDictionaryAddValue(CFMutableDictionaryRef, void *, void *);
-
-void rdar_6539791(CFMutableDictionaryRef y, void* key, void* val_key) {
-  CFMutableDictionaryRef x = CFDictionaryCreateMutable(kCFAllocatorDefault, 1, &kCFTypeDictionaryKeyCallBacks, &kCFTypeDictionaryValueCallBacks);
-  CFDictionaryAddValue(y, key, x);
-  CFRelease(x); // the dictionary keeps a reference, so the object isn't deallocated yet
-  signed z = 1;
-  CFNumberRef value = CFNumberCreate(kCFAllocatorDefault, kCFNumberSInt32Type, &z);
-  if (value) {
-    CFDictionaryAddValue(x, val_key, (void*)value); // no-warning
-    CFRelease(value);
-    CFDictionaryAddValue(y, val_key, (void*)value); // no-warning
-  }
-}
-
-// Same issue, except with "AppendValue" functions.
-void rdar_6560661(CFMutableArrayRef x) {
-  signed z = 1;
-  CFNumberRef value = CFNumberCreate(kCFAllocatorDefault, kCFNumberSInt32Type, &z);
-  // CFArrayAppendValue keeps a reference to value.
-  CFArrayAppendValue(x, value);
-  CFRelease(value);
-  CFRetain(value);
-  CFRelease(value); // no-warning
-}
-
-// Same issue, excwept with "CFAttributeStringSetAttribute".
-void rdar_7152619(CFStringRef str) {
-  CFAttributedStringRef string = CFAttributedStringCreate(kCFAllocatorDefault, str, 0);
-  CFMutableAttributedStringRef attrString = CFAttributedStringCreateMutableCopy(kCFAllocatorDefault, 100, string);
-  CFRelease(string);
-  NSNumber *number = [[NSNumber alloc] initWithInt:5]; // expected-warning{{leak}}
-  CFAttributedStringSetAttribute(attrString, CFRangeMake(0, 1), str, number);
-  [number release];
-  [number retain];
-  CFRelease(attrString);  
-}
-
-//===----------------------------------------------------------------------===//
-// Test of handling CGGradientXXX functions.
-//===----------------------------------------------------------------------===//
-
-void rdar_7184450(CGContextRef myContext, CGFloat x, CGPoint myStartPoint,
-                  CGPoint myEndPoint) {
-  size_t num_locations = 6;
-  CGFloat locations[6] = { 0.0, 0.265, 0.28, 0.31, 0.36, 1.0 };
-  CGFloat components[28] = { 239.0/256.0, 167.0/256.0, 170.0/256.0,
-     x,  // Start color
-    207.0/255.0, 39.0/255.0, 39.0/255.0, x,
-    147.0/255.0, 21.0/255.0, 22.0/255.0, x,
-    175.0/255.0, 175.0/255.0, 175.0/255.0, x,
-    255.0/255.0,255.0/255.0, 255.0/255.0, x,
-    255.0/255.0,255.0/255.0, 255.0/255.0, x
-  }; // End color
-  
-  CGGradientRef myGradient =
-    CGGradientCreateWithColorComponents(CGColorSpaceCreateDeviceRGB(), // expected-warning{{leak}}
-      components, locations, num_locations);
-
-  CGContextDrawLinearGradient(myContext, myGradient, myStartPoint, myEndPoint,
-                              0);
-  CGGradientRelease(myGradient);
-}
-
-void rdar_7184450_pos(CGContextRef myContext, CGFloat x, CGPoint myStartPoint,
-                  CGPoint myEndPoint) {
-  size_t num_locations = 6;
-  CGFloat locations[6] = { 0.0, 0.265, 0.28, 0.31, 0.36, 1.0 };
-  CGFloat components[28] = { 239.0/256.0, 167.0/256.0, 170.0/256.0,
-     x,  // Start color
-    207.0/255.0, 39.0/255.0, 39.0/255.0, x,
-    147.0/255.0, 21.0/255.0, 22.0/255.0, x,
-    175.0/255.0, 175.0/255.0, 175.0/255.0, x,
-    255.0/255.0,255.0/255.0, 255.0/255.0, x,
-    255.0/255.0,255.0/255.0, 255.0/255.0, x
-  }; // End color
-  
-  CGGradientRef myGradient =
-   CGGradientCreateWithColorComponents(CGColorSpaceCreateDeviceRGB(), components, locations, num_locations); // expected-warning 2 {{leak}}
-
-  CGContextDrawLinearGradient(myContext, myGradient, myStartPoint, myEndPoint,
-                              0);
-}
-
-//===----------------------------------------------------------------------===//
-// clang false positive: retained instance passed to thread in pthread_create
-// marked as leak
-//
-// Until we have full IPA, the analyzer should stop tracking the reference
-// count of objects passed to pthread_create.
-//
-//===----------------------------------------------------------------------===//
-struct _opaque_pthread_t {};
-struct _opaque_pthread_attr_t {};
-typedef struct _opaque_pthread_t *__darwin_pthread_t;
-typedef struct _opaque_pthread_attr_t __darwin_pthread_attr_t;
-typedef __darwin_pthread_t pthread_t;
-typedef __darwin_pthread_attr_t pthread_attr_t;
-typedef unsigned long __darwin_pthread_key_t;
-typedef __darwin_pthread_key_t pthread_key_t;
-
-int pthread_create(pthread_t *, const pthread_attr_t *,
-                   void *(*)(void *), void *);
-
-int pthread_setspecific(pthread_key_t key, const void *value);
-
-void *rdar_7299394_start_routine(void *p) {
-  [((id) p) release];
-  return 0;
-}
-void rdar_7299394(pthread_attr_t *attr, pthread_t *thread, void *args) {
-  NSNumber *number = [[NSNumber alloc] initWithInt:5]; // no-warning
-  pthread_create(thread, attr, rdar_7299394_start_routine, number);
-}
-void rdar_7299394_positive(pthread_attr_t *attr, pthread_t *thread) {
-  NSNumber *number = [[NSNumber alloc] initWithInt:5]; // expected-warning{{leak}}
-}
-
-//===----------------------------------------------------------------------===//
-// false positive with not understanding thread local storage
-//===----------------------------------------------------------------------===//
-void rdar11282706(pthread_key_t key) {
-  NSNumber *number = [[NSNumber alloc] initWithInt:5]; // no-warning
-  pthread_setspecific(key, (void*) number);
-}
-
-//===----------------------------------------------------------------------===//
-// False leak associated with call to CVPixelBufferCreateWithBytes ()
-//
-// According to the Core Video Reference (ADC), CVPixelBufferCreateWithBytes and
-// CVPixelBufferCreateWithPlanarBytes can release (via a callback) the
-// pixel buffer object.  These test cases show how the analyzer stops tracking
-// the reference count for the objects passed for this argument.  This
-// could be made smarter.
-//===----------------------------------------------------------------------===//
-typedef int int32_t;
-typedef UInt32 FourCharCode;
-typedef FourCharCode OSType;
-typedef uint64_t CVOptionFlags;
-typedef int32_t CVReturn;
-typedef struct __CVBuffer *CVBufferRef;
-typedef CVBufferRef CVImageBufferRef;
-typedef CVImageBufferRef CVPixelBufferRef;
-typedef void (*CVPixelBufferReleaseBytesCallback)( void *releaseRefCon, const void *baseAddress );
-
-extern CVReturn CVPixelBufferCreateWithBytes(CFAllocatorRef allocator,
-            size_t width,
-            size_t height,
-            OSType pixelFormatType,
-            void *baseAddress,
-            size_t bytesPerRow,
-            CVPixelBufferReleaseBytesCallback releaseCallback,
-            void *releaseRefCon,
-            CFDictionaryRef pixelBufferAttributes,
-                   CVPixelBufferRef *pixelBufferOut) ;
-
-typedef void (*CVPixelBufferReleasePlanarBytesCallback)( void *releaseRefCon, const void *dataPtr, size_t dataSize, size_t numberOfPlanes, const void *planeAddresses[] );
-
-extern CVReturn CVPixelBufferCreateWithPlanarBytes(CFAllocatorRef allocator,
-        size_t width,
-        size_t height,
-        OSType pixelFormatType,
-        void *dataPtr,
-        size_t dataSize,
-        size_t numberOfPlanes,
-        void *planeBaseAddress[],
-        size_t planeWidth[],
-        size_t planeHeight[],
-        size_t planeBytesPerRow[],
-        CVPixelBufferReleasePlanarBytesCallback releaseCallback,
-        void *releaseRefCon,
-        CFDictionaryRef pixelBufferAttributes,
-        CVPixelBufferRef *pixelBufferOut) ;
-
-extern CVReturn CVPixelBufferCreateWithBytes(CFAllocatorRef allocator,
-            size_t width,
-            size_t height,
-            OSType pixelFormatType,
-            void *baseAddress,
-            size_t bytesPerRow,
-            CVPixelBufferReleaseBytesCallback releaseCallback,
-            void *releaseRefCon,
-            CFDictionaryRef pixelBufferAttributes,
-                   CVPixelBufferRef *pixelBufferOut) ;
-
-CVReturn rdar_7283567(CFAllocatorRef allocator, size_t width, size_t height,
-                      OSType pixelFormatType, void *baseAddress,
-                      size_t bytesPerRow,
-                      CVPixelBufferReleaseBytesCallback releaseCallback,
-                      CFDictionaryRef pixelBufferAttributes,
-                      CVPixelBufferRef *pixelBufferOut) {
-
-  // For the allocated object, it doesn't really matter what type it is
-  // for the purpose of this test.  All we want to show is that
-  // this is freed later by the callback.
-  NSNumber *number = [[NSNumber alloc] initWithInt:5]; // no-warning
-  
-  return CVPixelBufferCreateWithBytes(allocator, width, height, pixelFormatType,
-                                baseAddress, bytesPerRow, releaseCallback,
-                                number, // potentially released by callback
-                                pixelBufferAttributes, pixelBufferOut) ;
-}
-
-CVReturn rdar_7283567_2(CFAllocatorRef allocator, size_t width, size_t height,
-        OSType pixelFormatType, void *dataPtr, size_t dataSize,
-        size_t numberOfPlanes, void *planeBaseAddress[],
-        size_t planeWidth[], size_t planeHeight[], size_t planeBytesPerRow[],
-        CVPixelBufferReleasePlanarBytesCallback releaseCallback,
-        CFDictionaryRef pixelBufferAttributes,
-        CVPixelBufferRef *pixelBufferOut) {
-    
-    // For the allocated object, it doesn't really matter what type it is
-    // for the purpose of this test.  All we want to show is that
-    // this is freed later by the callback.
-    NSNumber *number = [[NSNumber alloc] initWithInt:5]; // no-warning
-
-    return CVPixelBufferCreateWithPlanarBytes(allocator,
-              width, height, pixelFormatType, dataPtr, dataSize,
-              numberOfPlanes, planeBaseAddress, planeWidth,
-              planeHeight, planeBytesPerRow, releaseCallback,
-              number, // potentially released by callback
-              pixelBufferAttributes, pixelBufferOut) ;
-}
-
-//===----------------------------------------------------------------------===//
-// False leak associated with CGBitmapContextCreateWithData
-//===----------------------------------------------------------------------===//
-typedef uint32_t CGBitmapInfo;
-typedef void (*CGBitmapContextReleaseDataCallback)(void *releaseInfo, void *data);
-    
-CGContextRef CGBitmapContextCreateWithData(void *data,
-    size_t width, size_t height, size_t bitsPerComponent,
-    size_t bytesPerRow, CGColorSpaceRef space, CGBitmapInfo bitmapInfo,
-    CGBitmapContextReleaseDataCallback releaseCallback, void *releaseInfo) CF_RETURNS_RETAINED;
-
-void rdar_7358899(void *data,
-      size_t width, size_t height, size_t bitsPerComponent,
-      size_t bytesPerRow, CGColorSpaceRef space, CGBitmapInfo bitmapInfo,
-      CGBitmapContextReleaseDataCallback releaseCallback) {
-
-    // For the allocated object, it doesn't really matter what type it is
-    // for the purpose of this test.  All we want to show is that
-    // this is freed later by the callback.
-    NSNumber *number = [[NSNumber alloc] initWithInt:5]; // no-warning
-
-  CGBitmapContextCreateWithData(data, width, height, bitsPerComponent, // expected-warning{{leak}}
-    bytesPerRow, space, bitmapInfo, releaseCallback, number);
-}
-
-//===----------------------------------------------------------------------===//
-// Allow 'new', 'copy', 'alloc', 'init' prefix to start before '_' when
-// determining Cocoa fundamental rule.
-//
-// Previously the retain/release checker just skipped prefixes before the
-// first '_' entirely.  Now the checker honors the prefix if it results in a
-// recognizable naming convention (e.g., 'new', 'init').
-//===----------------------------------------------------------------------===//
-@interface RDar7265711 {}
-- (id) new_stuff;
-@end
-
-void rdar7265711_a(RDar7265711 *x) {
-  id y = [x new_stuff]; // expected-warning{{leak}}
-}
-
-void rdar7265711_b(RDar7265711 *x) {
-  id y = [x new_stuff]; // no-warning
-  [y release];
-}
-
-//===----------------------------------------------------------------------===//
-// clang thinks [NSCursor dragCopyCursor] returns a retained reference
-//===----------------------------------------------------------------------===//
-@interface NSCursor : NSObject
-+ (NSCursor *)dragCopyCursor;
-@end
-
-void rdar7306898(void) {
-  // 'dragCopyCursor' does not follow Cocoa's fundamental rule.  It is a noun, not an sentence
-  // implying a 'copy' of something.
-  NSCursor *c =  [NSCursor dragCopyCursor]; // no-warning
-  NSNumber *number = [[NSNumber alloc] initWithInt:5]; // expected-warning{{leak}}
-}
-
-//===----------------------------------------------------------------------===//
-// Sending 'release', 'retain', etc. to a Class directly is not likely what the
-// user intended.
-//===----------------------------------------------------------------------===//
-@interface RDar7252064 : NSObject @end
-void rdar7252064(void) {
-  [RDar7252064 release]; // expected-warning{{The 'release' message should be sent to instances of class 'RDar7252064' and not the class directly}}
-  [RDar7252064 retain]; // expected-warning{{The 'retain' message should be sent to instances of class 'RDar7252064' and not the class directly}}
-  [RDar7252064 autorelease]; // expected-warning{{The 'autorelease' message should be sent to instances of class 'RDar7252064' and not the class directly}}
-  [NSAutoreleasePool drain]; // expected-warning{{method '+drain' not found}} expected-warning{{The 'drain' message should be sent to instances of class 'NSAutoreleasePool' and not the class directly}}
-}
-
-//===----------------------------------------------------------------------===//
-// Tests of ownership attributes.
-//===----------------------------------------------------------------------===//
-
-typedef NSString* MyStringTy;
-
-@protocol FooP;
-
-@interface TestOwnershipAttr : NSObject
-- (NSString*) returnsAnOwnedString  NS_RETURNS_RETAINED; // no-warning
-- (NSString*) returnsAnOwnedCFString  CF_RETURNS_RETAINED; // no-warning
-- (MyStringTy) returnsAnOwnedTypedString NS_RETURNS_RETAINED; // no-warning
-- (NSString*) newString NS_RETURNS_NOT_RETAINED; // no-warning
-- (NSString*) newString_auto NS_RETURNS_AUTORELEASED; // no-warning
-- (NSString*) newStringNoAttr;
-- (int) returnsAnOwnedInt NS_RETURNS_RETAINED; // expected-warning{{'ns_returns_retained' attribute only applies to methods that return an Objective-C object}}
-- (id) pseudoInit NS_CONSUMES_SELF NS_RETURNS_RETAINED;
-+ (void) consume:(id) NS_CONSUMED x;
-+ (void) consume2:(id) CF_CONSUMED x;
-@end
-
-static int ownership_attribute_doesnt_go_here NS_RETURNS_RETAINED; // expected-warning{{'ns_returns_retained' attribute only applies to functions and methods}}
-
-void test_attr_1(TestOwnershipAttr *X) {
-  NSString *str = [X returnsAnOwnedString]; // expected-warning{{leak}}
-}
-
-void test_attr_1b(TestOwnershipAttr *X) {
-  NSString *str = [X returnsAnOwnedCFString]; // expected-warning{{leak}}
-}
-
-void test_attr1c(TestOwnershipAttr *X) {
-  NSString *str = [X newString]; // no-warning
-  NSString *str2 = [X newStringNoAttr]; // expected-warning{{leak}}
-  NSString *str3 = [X newString_auto]; // no-warning
-  NSString *str4 = [[X newString_auto] retain]; // expected-warning {{leak}}
-}
-
-void testattr2_a(void) {
-  TestOwnershipAttr *x = [TestOwnershipAttr alloc]; // expected-warning{{leak}}
-}
-
-void testattr2_b(void) {
-  TestOwnershipAttr *x = [[TestOwnershipAttr alloc] pseudoInit];  // expected-warning{{leak}}
-}
-
-void testattr2_b_11358224_self_assign_looses_the_leak(void) {
-  TestOwnershipAttr *x = [[TestOwnershipAttr alloc] pseudoInit];// expected-warning{{leak}}
-  x = x;
-}
-
-void testattr2_c(void) {
-  TestOwnershipAttr *x = [[TestOwnershipAttr alloc] pseudoInit]; // no-warning
-  [x release];
-}
-
-void testattr3(void) {
-  TestOwnershipAttr *x = [TestOwnershipAttr alloc]; // no-warning
-  [TestOwnershipAttr consume:x];
-  TestOwnershipAttr *y = [TestOwnershipAttr alloc]; // no-warning
-  [TestOwnershipAttr consume2:y];
-}
-
-void consume_ns(id NS_CONSUMED x);
-void consume_cf(id CF_CONSUMED x);
-
-void testattr4(void) {
-  TestOwnershipAttr *x = [TestOwnershipAttr alloc]; // no-warning
-  consume_ns(x);
-  TestOwnershipAttr *y = [TestOwnershipAttr alloc]; // no-warning
-  consume_cf(y);
-}
-
-@interface TestOwnershipAttr2 : NSObject
-- (NSString*) newString NS_RETURNS_NOT_RETAINED; // no-warning
-@end
-
-@implementation TestOwnershipAttr2
-- (NSString*) newString {
-  return [NSString alloc]; // expected-warning {{Potential leak of an object}}
-}
-@end
-
-@interface MyClassTestCFAttr : NSObject {}
-- (NSDate*) returnsCFRetained CF_RETURNS_RETAINED;
-- (CFDateRef) returnsCFRetainedAsCF CF_RETURNS_RETAINED;
-- (CFDateRef) newCFRetainedAsCF CF_RETURNS_NOT_RETAINED;
-- (CFDateRef) newCFRetainedAsCFNoAttr CF_RETURNS_RETAINED;
-- (NSDate*) alsoReturnsRetained;
-- (CFDateRef) alsoReturnsRetainedAsCF CF_RETURNS_NOT_RETAINED;
-- (NSDate*) returnsNSRetained NS_RETURNS_RETAINED;
-@end
-
-CF_RETURNS_RETAINED
-CFDateRef returnsRetainedCFDate(void)  {
-  return CFDateCreate(0, CFAbsoluteTimeGetCurrent());
-}
-
-@implementation MyClassTestCFAttr
-- (NSDate*) returnsCFRetained {
-  return (NSDate*) returnsRetainedCFDate(); // No leak.
-}
-
-- (CFDateRef) returnsCFRetainedAsCF {
-  return returnsRetainedCFDate(); // No leak.
-}
-
-- (CFDateRef) newCFRetainedAsCF {
-  return (CFDateRef)[(id)[self returnsCFRetainedAsCF] autorelease];
-}
-
-- (CFDateRef) newCFRetainedAsCFNoAttr {
-  return (CFDateRef)[(id)[self returnsCFRetainedAsCF] autorelease]; // expected-warning{{Object with a +0 retain count returned to caller where a +1 (owning) retain count is expected}}
-}
-
-- (NSDate*) alsoReturnsRetained {
-  return (NSDate*) returnsRetainedCFDate(); // expected-warning{{leak}}
-}
-
-- (CFDateRef) alsoReturnsRetainedAsCF {
-  return returnsRetainedCFDate(); // expected-warning{{leak}}
-}
-
-
-- (NSDate*) returnsNSRetained {
-  return (NSDate*) returnsRetainedCFDate(); // no-warning
-}
-@end
-
-//===----------------------------------------------------------------------===//
-// Test that leaks post-dominated by "panic" functions are not reported.
-//
-// Do not report a leak when post-dominated by a call to a noreturn or panic
-// function.
-//===----------------------------------------------------------------------===//
-void panic(void) __attribute__((noreturn));
-void panic_not_in_hardcoded_list(void) __attribute__((noreturn));
-
-void test_panic_negative(void) {
-  signed z = 1;
-  CFNumberRef value = CFNumberCreate(kCFAllocatorDefault, kCFNumberSInt32Type, &z);  // expected-warning{{leak}}
-}
-
-void test_panic_positive(void) {
-  signed z = 1;
-  CFNumberRef value = CFNumberCreate(kCFAllocatorDefault, kCFNumberSInt32Type, &z); // no-warning
-  panic();
-}
-
-void test_panic_neg_2(int x) {
-  signed z = 1;
-  CFNumberRef value = CFNumberCreate(kCFAllocatorDefault, kCFNumberSInt32Type, &z); // expected-warning{{leak}}
-  if (x)
-    panic();
-}
-
-void test_panic_pos_2(int x) {
-  signed z = 1;
-  CFNumberRef value = CFNumberCreate(kCFAllocatorDefault, kCFNumberSInt32Type, &z); // no-warning
-  if (x)
-    panic();
-  if (!x) {
-    // This showed up previously where we silently missed checking the function
-    // type for noreturn.  "panic()" is a hard-coded known panic function that
-    // isn't always noreturn.
-    panic_not_in_hardcoded_list();
-  }
-}
-
-//===----------------------------------------------------------------------===//
-// Test uses of blocks (closures)
-//===----------------------------------------------------------------------===//
-
-void test_blocks_1_pos(void) {
-  NSNumber *number = [[NSNumber alloc] initWithInt:5]; // expected-warning{{leak}}
-  ^{}();
-}
-
-void test_blocks_1_indirect_release(void) {
-  NSNumber *number = [[NSNumber alloc] initWithInt:5]; // no-warning
-  ^{ [number release]; }();
-}
-
-void test_blocks_1_indirect_retain(void) {
-  // Eventually this should be reported as a leak.
-  NSNumber *number = [[NSNumber alloc] initWithInt:5]; // no-warning
-  ^{ [number retain]; }();
-}
-
-void test_blocks_1_indirect_release_via_call(void) {
-  NSNumber *number = [[NSNumber alloc] initWithInt:5]; // no-warning
-  ^(NSObject *o){ [o release]; }(number);
-}
-
-void test_blocks_1_indirect_retain_via_call(void) {
-  NSNumber *number = [[NSNumber alloc] initWithInt:5]; // expected-warning {{leak}}
-  ^(NSObject *o){ [o retain]; }(number);
-}
-
-//===--------------------------------------------------------------------===//
-// Test sending message to super that returns an object alias.  Previously
-// this caused a crash in the analyzer.
-//===--------------------------------------------------------------------===//
-
-@interface Rdar8015556 : NSObject {} @end
-@implementation Rdar8015556
-- (id)retain {
-  return [super retain];
-}
-@end
-
-// Correcly handle Class<...> in Cocoa Conventions detector.
-@protocol Prot_R8272168 @end
-Class <Prot_R8272168> GetAClassThatImplementsProt_R8272168(void);
-void r8272168(void) {
-  GetAClassThatImplementsProt_R8272168();
-}
-
-// Test case which in the past triggered a false positive.
-@interface RDar8356342
-- (NSDate*) rdar8356342:(NSDate *)inValue;
-@end
-
-@implementation RDar8356342
-- (NSDate*) rdar8356342:(NSDate*)inValue {
-  NSDate *outValue = inValue;
-  if (outValue == 0)
-    outValue = [[NSDate alloc] init]; // no-warning
-
-  if (outValue != inValue)
-    [outValue autorelease];
-
-  return outValue;
-}
-@end
-
-// This test case previously crashed because of a bug in BugReporter.
-extern const void *CFDictionaryGetValue(CFDictionaryRef theDict, const void *key) CF_RETURNS_NOT_RETAINED;
-typedef struct __CFError * CFErrorRef;
-extern const CFStringRef kCFErrorUnderlyingErrorKey;
-extern CFDictionaryRef CFErrorCopyUserInfo(CFErrorRef err) CF_RETURNS_RETAINED;
-static void rdar_8724287(CFErrorRef error)
-{
-    CFErrorRef error_to_dump;
-
-    error_to_dump = error;
-    while (error_to_dump != ((void*)0)) {
-        CFDictionaryRef info;
-
-        info = CFErrorCopyUserInfo(error_to_dump); // expected-warning{{Potential leak of an object}}
-
-        if (info != ((void*)0)) {
-        }
-
-        error_to_dump = (CFErrorRef) CFDictionaryGetValue(info, kCFErrorUnderlyingErrorKey);
-    }
-}
-
-// Make sure the model applies cf_consumed correctly in argument positions
-// besides the first.
-
-CF_IMPLICIT_BRIDGING_ENABLED
-
-extern void *CFStringCreate(void);
-
-CF_IMPLICIT_BRIDGING_DISABLED
-
-extern void rdar_9234108_helper(void *key, void * CF_CONSUMED value);
-void rdar_9234108(void) {
-  rdar_9234108_helper(0, CFStringCreate());
-}
-
-// Make sure that objc_method_family works to override naming conventions.
-struct TwoDoubles {
-  double one;
-  double two;
-};
-typedef struct TwoDoubles TwoDoubles;
-
-@interface NSValue (Mine)
-- (id)_prefix_initWithTwoDoubles:(TwoDoubles)twoDoubles __attribute__((objc_method_family(init)));
-@end
-
-@implementation NSValue (Mine)
-- (id)_prefix_initWithTwoDoubles:(TwoDoubles)twoDoubles
-{
-  return [self init];
-}
-@end
-
-void rdar9726279(void) {
-  TwoDoubles twoDoubles = { 0.0, 0.0 };
-  NSValue *value = [[NSValue alloc] _prefix_initWithTwoDoubles:twoDoubles];
-  [value release];
-}
-
-// Test camelcase support for CF conventions.  While Core Foundation APIs
-// don't use camel casing, other code is allowed to use it.
-CFArrayRef camelcase_create_1(void) {
-  return CFArrayCreateMutable(0, 10, &kCFTypeArrayCallBacks); // no-warning
-}
-
-CFArrayRef camelcase_createno(void) {
-  return CFArrayCreateMutable(0, 10, &kCFTypeArrayCallBacks); // expected-warning {{leak}}
-}
-
-CFArrayRef camelcase_copy(void) {
-  return CFArrayCreateMutable(0, 10, &kCFTypeArrayCallBacks); // no-warning
-}
-
-CFArrayRef camelcase_copying(void) {
-  return CFArrayCreateMutable(0, 10, &kCFTypeArrayCallBacks); // expected-warning {{leak}}
-}
-
-CFArrayRef copyCamelCase(void) {
-  return CFArrayCreateMutable(0, 10, &kCFTypeArrayCallBacks); // no-warning
-}
-
-CFArrayRef __copyCamelCase(void) {
-  return CFArrayCreateMutable(0, 10, &kCFTypeArrayCallBacks); // no-warning
-}
-
-CFArrayRef __createCamelCase(void) {
-  return CFArrayCreateMutable(0, 10, &kCFTypeArrayCallBacks); // no-warning
-}
-
-CFArrayRef camel_create(void) {
-  return CFArrayCreateMutable(0, 10, &kCFTypeArrayCallBacks); // no-warning
-}
-
-
-CFArrayRef camel_creat(void) {
-  return CFArrayCreateMutable(0, 10, &kCFTypeArrayCallBacks); // expected-warning {{leak}}
-}
-
-CFArrayRef camel_copy(void) {
-  return CFArrayCreateMutable(0, 10, &kCFTypeArrayCallBacks); // no-warning
-}
-
-CFArrayRef camel_copyMachine(void) {
-  return CFArrayCreateMutable(0, 10, &kCFTypeArrayCallBacks); // no-warning
-}
-
-CFArrayRef camel_copymachine(void) {
-  return CFArrayCreateMutable(0, 10, &kCFTypeArrayCallBacks); // expected-warning {{leak}}
-}
-
-@protocol F18P
-- (id) clone;
-@end
-@interface F18 : NSObject<F18P> @end
-@interface F18(Cat)
-- (id) clone NS_RETURNS_RETAINED;
-@end
-
-@implementation F18
-- (id) clone {
-  return [F18 alloc];
-}
-@end
-
-void rdar6582778(void) {
-  CFAbsoluteTime t = CFAbsoluteTimeGetCurrent();
-  CFTypeRef vals[] = { CFDateCreate(0, t) }; // expected-warning {{leak}}
-}
-
-CFTypeRef global;
-
-void rdar6582778_2(void) {
-  CFAbsoluteTime t = CFAbsoluteTimeGetCurrent();
-  global = CFDateCreate(0, t); // no-warning
-}
-
-// Test that objects passed to containers are marked "escaped".
-void rdar10232019(void) {
-  NSMutableArray *array = [NSMutableArray array];
-
-  NSString *string = [[NSString alloc] initWithUTF8String:"foo"];
-  [array addObject:string];
-  [string release];
-
-  NSString *otherString = [string stringByAppendingString:@"bar"]; // no-warning
-  NSLog(@"%@", otherString);
-}
-
-void rdar10232019_positive(void) {
-  NSMutableArray *array = [NSMutableArray array];
-
-  NSString *string = [[NSString alloc] initWithUTF8String:"foo"];
-  [string release];
-
-  NSString *otherString = [string stringByAppendingString:@"bar"]; // expected-warning {{Reference-counted object is used after it is release}}
-  NSLog(@"%@", otherString);
-}
-
-// RetainCountChecker support for XPC.
-typedef void * xpc_object_t;
-xpc_object_t _CFXPCCreateXPCObjectFromCFObject(CFTypeRef cf);
-void xpc_release(xpc_object_t object);
-
-void rdar9658496(void) {
-  CFStringRef cf;
-  xpc_object_t xpc;
-  cf = CFStringCreateWithCString( ((CFAllocatorRef)0), "test", kCFStringEncodingUTF8 ); // no-warning
-  xpc = _CFXPCCreateXPCObjectFromCFObject( cf );
-  CFRelease(cf);
-  xpc_release(xpc);
-}
-
-// Support annotations with method families.
-@interface RDar10824732 : NSObject
-- (instancetype)initWithObj:(id CF_CONSUMED)obj;
-@end
-
-@implementation RDar10824732
-- (instancetype)initWithObj:(id)obj {
-  [obj release];
-  return [super init];
-}
-@end
-
-void rdar_10824732(void) {
-  @autoreleasepool {
-    NSString *obj = @"test";
-    RDar10824732 *foo = [[RDar10824732 alloc] initWithObj:obj]; // no-warning
-    [foo release];
-  }
-}
-
-// Stop tracking objects passed to functions, which take callbacks as parameters.
-typedef int (*CloseCallback) (void *);
-void ReaderForIO(CloseCallback ioclose, void *ioctx);
-int IOClose(void *context);
-
-@protocol SInS <NSObject>
-@end
-
-@interface radar10973977 : NSObject
-- (id<SInS>)inputS;
-- (void)reader;
-@end
-
-@implementation radar10973977
-- (void)reader
-{
-    id<SInS> inputS = [[self inputS] retain];
-    ReaderForIO(IOClose, inputS);
-}
-- (id<SInS>)inputS
-{
-    return 0;
-}
-@end
-
-// Object escapes through a selector callback
-extern id NSApp;
-@interface MySheetController
-- (id<SInS>)inputS;
-- (void)showDoSomethingSheetAction:(id)action;
-- (void)sheetDidEnd:(NSWindow *)sheet returnCode:(int)returnCode contextInfo:(void *)contextInfo;
-@end
-
-@implementation MySheetController
-- (id<SInS>)inputS {
-    return 0;
-}
-- (void)showDoSomethingSheetAction:(id)action {
-  id<SInS> inputS = [[self inputS] retain]; 
-  [NSApp beginSheet:0
-         modalForWindow:0
-         modalDelegate:0
-         didEndSelector:@selector(sheetDidEnd:returnCode:contextInfo:)
-         contextInfo:(void *)inputS]; // no - warning
-}
-- (void)sheetDidEnd:(NSWindow *)sheet returnCode:(int)returnCode contextInfo:(void *)contextInfo {
-   
-      id contextObject = (id)contextInfo;
-      [contextObject release];
-}
-
-- (id)copyAutoreleaseRadar13081402 {
-  id x = [[[NSString alloc] initWithUTF8String:"foo"] autorelease];
-  [x retain];
-  return x; // no warning
-}
-
-@end
-//===----------------------------------------------------------------------===//
-// Test returning allocated memory in a struct.
-//
-// We currently don't have a general way to track pointers that "escape".
-// Here we test that RetainCountChecker doesn't get excited about returning
-// allocated CF objects in struct fields.
-//===----------------------------------------------------------------------===//
-void *malloc(size_t);
-struct rdar11104566 { CFStringRef myStr; };
-struct rdar11104566 test_rdar11104566(void) {
-  CFStringRef cf = CFStringCreateWithCString( ((CFAllocatorRef)0), "test", kCFStringEncodingUTF8 ); // no-warning
-  struct rdar11104566 V;
-  V.myStr = cf;
-  return V; // no-warning
-}
-
-struct rdar11104566 *test_2_rdar11104566(void) {
-  CFStringRef cf = CFStringCreateWithCString( ((CFAllocatorRef)0), "test", kCFStringEncodingUTF8 ); // no-warning
-  struct rdar11104566 *V = (struct rdar11104566 *) malloc(sizeof(*V));
-  V->myStr = cf;
-  return V; // no-warning
-}
-
-//===----------------------------------------------------------------------===//
-// ObjC literals support.
-//===----------------------------------------------------------------------===//
-
-void test_objc_arrays(void) {
-    { // CASE ONE -- OBJECT IN ARRAY CREATED DIRECTLY
-        NSObject *o = [[NSObject alloc] init];
-        NSArray *a = [[NSArray alloc] initWithObjects:o, (void*)0]; // expected-warning {{leak}}
-        [o release];
-        [a description];
-        [o description];
-    }
-
-    { // CASE TWO -- OBJECT IN ARRAY CREATED BY DUPING AUTORELEASED ARRAY
-        NSObject *o = [[NSObject alloc] init];
-        NSArray *a1 = [NSArray arrayWithObjects:o, (void*)0];
-        NSArray *a2 = [[NSArray alloc] initWithArray:a1]; // expected-warning {{leak}}
-        [o release];        
-        [a2 description];
-        [o description];
-    }
-
-    { // CASE THREE -- OBJECT IN RETAINED @[]
-        NSObject *o = [[NSObject alloc] init];
-        NSArray *a3 = [@[o] retain]; // expected-warning {{leak}}
-        [o release];        
-        [a3 description];
-        [o description];
-    }
-    
-    { // CASE FOUR -- OBJECT IN ARRAY CREATED BY DUPING @[]
-        NSObject *o = [[NSObject alloc] init];
-        NSArray *a = [[NSArray alloc] initWithArray:@[o]]; // expected-warning {{leak}}
-        [o release];
-        
-        [a description];
-        [o description];
-    }
-    
-    { // CASE FIVE -- OBJECT IN RETAINED @{}
-        NSValue *o = [[NSValue alloc] init];
-        NSDictionary *a = [@{o : o} retain]; // expected-warning {{leak}}
-        [o release];
-        
-        [a description];
-        [o description];
-    }
-}
-
-void test_objc_integer_literals(void) {
-  id value = [@1 retain]; // expected-warning {{leak}}
-  [value description];
-}
-
-void test_objc_boxed_expressions(int x, const char *y) {
-  id value = [@(x) retain]; // expected-warning {{leak}}
-  [value description];
-
-  value = [@(y) retain]; // expected-warning {{leak}}
-  [value description];
-}
-
-// Test NSLog doesn't escape tracked objects.
-void rdar11400885(int y)
-{
-  @autoreleasepool {
-    NSString *printString;
-    if(y > 2)
-      printString = [[NSString alloc] init];
-    else
-      printString = [[NSString alloc] init];
-    NSLog(@"Once %@", printString);
-    [printString release];
-    NSLog(@"Again: %@", printString); // expected-warning {{Reference-counted object is used after it is released}}
-  }
-}
-
-id makeCollectableNonLeak(void) {
-  extern CFTypeRef CFCreateSomething(void);
-
-  CFTypeRef object = CFCreateSomething(); // +1
-  CFRetain(object); // +2
-  id objCObject = NSMakeCollectable(object); // +2
-  [objCObject release]; // +1
-  return [objCObject autorelease]; // +0
-}
-
-
-void consumeAndStopTracking(id NS_CONSUMED obj, void (^callback)(void));
-void CFConsumeAndStopTracking(CFTypeRef CF_CONSUMED obj, void (^callback)(void));
-
-void testConsumeAndStopTracking(void) {
-  id retained = [@[] retain]; // +1
-  consumeAndStopTracking(retained, ^{}); // no-warning
-
-  id doubleRetained = [[@[] retain] retain]; // +2
-  consumeAndStopTracking(doubleRetained, ^{
-    [doubleRetained release];
-  }); // no-warning
-
-  id unretained = @[]; // +0
-  consumeAndStopTracking(unretained, ^{}); // expected-warning {{Incorrect decrement of the reference count of an object that is not owned at this point by the caller}}
-}
-
-void testCFConsumeAndStopTracking(void) {
-  id retained = [@[] retain]; // +1
-  CFConsumeAndStopTracking((CFTypeRef)retained, ^{}); // no-warning
-
-  id doubleRetained = [[@[] retain] retain]; // +2
-  CFConsumeAndStopTracking((CFTypeRef)doubleRetained, ^{
-    [doubleRetained release];
-  }); // no-warning
-
-  id unretained = @[]; // +0
-  CFConsumeAndStopTracking((CFTypeRef)unretained, ^{}); // expected-warning {{Incorrect decrement of the reference count of an object that is not owned at this point by the caller}}
-}
-//===----------------------------------------------------------------------===//
-// Test 'pragma clang arc_cf_code_audited' support.
-//===----------------------------------------------------------------------===//
-
-typedef void *MyCFType;
-#pragma clang arc_cf_code_audited begin
-MyCFType CreateMyCFType(void);
-#pragma clang arc_cf_code_audited end 
-    
-void test_custom_cf(void) {
-  MyCFType x = CreateMyCFType(); // expected-warning {{leak of an object stored into 'x'}}
-}
-
-//===----------------------------------------------------------------------===//
-// Test calling CFPlugInInstanceCreate, which appears in CF but doesn't
-// return a CF object.
-//===----------------------------------------------------------------------===//
-
-void test_CFPlugInInstanceCreate(CFUUIDRef factoryUUID, CFUUIDRef typeUUID) {
-  CFPlugInInstanceCreate(kCFAllocatorDefault, factoryUUID, typeUUID); // no-warning
-}
-
-//===----------------------------------------------------------------------===//
-// PR14927: -drain only has retain-count semantics on NSAutoreleasePool.
-//===----------------------------------------------------------------------===//
-
-@interface PR14927 : NSObject
-- (void)drain;
-@end
-
-void test_drain(void) {
-  PR14927 *obj = [[PR14927 alloc] init];
-  [obj drain];
-  [obj release]; // no-warning
-}
-
-//===----------------------------------------------------------------------===//
-// Allow cf_returns_retained and cf_returns_not_retained to mark a return
-// value as tracked, even if the object isn't a known CF type.
-//===----------------------------------------------------------------------===//
-
-MyCFType getCustom(void) __attribute__((cf_returns_not_retained));
-MyCFType makeCustom(void) __attribute__((cf_returns_retained));
-
-void testCustomReturnsRetained(void) {
-  MyCFType obj = makeCustom(); // expected-warning {{leak of an object stored into 'obj'}}
-}
-
-void testCustomReturnsNotRetained(void) {
-  CFRelease(getCustom()); // expected-warning {{Incorrect decrement of the reference count of an object that is not owned at this point by the caller}}
-}
-
-//===----------------------------------------------------------------------===//
-// Don't print variables which are out of the current scope.
-//===----------------------------------------------------------------------===//
-@interface MyObj12706177 : NSObject
--(instancetype)initX;
-+(void)test12706177;
-@end
-static int Cond;
-@implementation MyObj12706177
--(instancetype)initX {
-  if (Cond)
-    return 0;
-  self = [super init];
-  return self;
-}
-+(void)test12706177 {
-  id x = [[MyObj12706177 alloc] initX]; //expected-warning {{Potential leak of an object}}
-  [x release]; 
-}
-@end
-
-//===----------------------------------------------------------------------===//
-// xpc_connection_set_finalizer_f
-//===----------------------------------------------------------------------===//
-typedef xpc_object_t xpc_connection_t;
-typedef void (*xpc_finalizer_t)(void *value);
-void xpc_connection_set_context(xpc_connection_t connection, void *ctx);
-void xpc_connection_set_finalizer_f(xpc_connection_t connection,
-                                    xpc_finalizer_t finalizer);
-void releaseAfterXPC(void *context) {
-  [(NSArray *)context release];
-}
-
-void rdar13783514(xpc_connection_t connection) {
-  xpc_connection_set_context(connection, [[NSMutableArray alloc] init]);
-  xpc_connection_set_finalizer_f(connection, releaseAfterXPC);
-} // no-warning
-
-CFAttributedStringRef CFAttributedCreate(void *CFObj CF_CONSUMED) CF_RETURNS_RETAINED;
-
-@interface Action
-@property (nonatomic) SEL action;
-@property (nonatomic, assign) id target;
-@end
diff --git a/clang/test/ARCMT/objcmt-atomic-property.m b/clang/test/ARCMT/objcmt-atomic-property.m
deleted file mode 100644
index 00b5e09c8e084..0000000000000
--- a/clang/test/ARCMT/objcmt-atomic-property.m
+++ /dev/null
@@ -1,227 +0,0 @@
-// RUN: rm -rf %t
-// RUN: %clang_cc1 -fblocks -objcmt-migrate-readwrite-property -objcmt-migrate-readonly-property -objcmt-atomic-property -mt-migrate-directory %t %s -x objective-c -fobjc-runtime-has-weak -fobjc-arc -triple x86_64-apple-darwin11
-// RUN: c-arcmt-test -mt-migrate-directory %t | arcmt-test -verify-transformed-files %s.result
-// RUN: %clang_cc1 -fblocks -triple x86_64-apple-darwin10 -fsyntax-only -x objective-c -fobjc-runtime-has-weak -fobjc-arc %s.result
-
-#define WEBKIT_OBJC_METHOD_ANNOTATION(ANNOTATION) ANNOTATION
-#define WEAK_IMPORT_ATTRIBUTE __attribute__((objc_arc_weak_reference_unavailable))
-#define AVAILABLE_WEBKIT_VERSION_3_0_AND_LATER
-#define DEPRECATED  __attribute__((deprecated)) 
-
-typedef char BOOL;
-@class NSString;
-@protocol NSCopying @end
-
-@interface NSObject <NSCopying>
-@end
-
-@interface NSDictionary : NSObject
-@end
-
-@interface I : NSObject {
-  int ivarVal;
-}
-- (void) setWeakProp : (NSString *__weak)Val;
-- (NSString *__weak) WeakProp;
-
-- (NSString *) StrongProp;
-- (void) setStrongProp : (NSString *)Val;
-
-- (NSString *) UnavailProp  __attribute__((unavailable));
-- (void) setUnavailProp  : (NSString *)Val;
-
-- (NSString *) UnavailProp1  __attribute__((unavailable));
-- (void) setUnavailProp1  : (NSString *)Val  __attribute__((unavailable));
-
-- (NSString *) UnavailProp2;
-- (void) setUnavailProp2  : (NSString *)Val  __attribute__((unavailable));
-
-- (NSDictionary*) undoAction;
-- (void) setUndoAction: (NSDictionary*)Arg;
-@end
-
-@implementation I
-@end
-
-@class NSArray;
-
-@interface MyClass2  {
-@private
-    NSArray *_names1;
-    NSArray *_names2;
-    NSArray *_names3;
-    NSArray *_names4;
-}
-- (void)setNames1:(NSArray *)names;
-- (void)setNames4:(__strong NSArray *)names;
-- (void)setNames3:(__strong NSArray *)names;
-- (void)setNames2:(NSArray *)names;
-- (NSArray *) names2;
-- (NSArray *)names3;
-- (__strong NSArray *)names4;
-- (NSArray *) names1;
-@end
-
-// Properties that contain the name "delegate" or "dataSource",
-// or have exact name "target" have unsafe_unretained attribute.
-@interface NSInvocation 
-- (id)target;
-- (void)setTarget:(id)target;
-
-- (id) dataSource;
-
-- (id)xxxdelegateYYY;
-- (void)setXxxdelegateYYY:(id)delegate;
-
-- (void)setDataSource:(id)source;
-
-- (id)MYtarget;
-- (void)setMYtarget: (id)target;
-
-- (id)targetX;
-- (void)setTargetX: (id)t;
- 
-- (int)value;
-- (void)setValue: (int)val;
-
--(BOOL) isContinuous;
--(void) setContinuous:(BOOL)value;
-
-- (id) isAnObject;
-- (void)setAnObject : (id) object;
-
-- (BOOL) isinValid;
-- (void) setInValid : (BOOL) arg;
-
-- (void) Nothing;
-- (int) Length;
-- (id) object;
-+ (double) D;
-- (void *)JSObject WEBKIT_OBJC_METHOD_ANNOTATION(AVAILABLE_WEBKIT_VERSION_3_0_AND_LATER);
-- (BOOL)isIgnoringInteractionEvents;
-
-- (NSString *)getStringValue;
-- (BOOL)getCounterValue;
-- (void)setStringValue:(NSString *)stringValue AVAILABLE_WEBKIT_VERSION_3_0_AND_LATER;
-- (NSDictionary *)getns_dixtionary;
-
-- (BOOL)is3bar; // watch out
-- (NSString *)get3foo; // watch out
-
-- (BOOL) getM;
-- (BOOL) getMA;
-- (BOOL) getALL;
-- (BOOL) getMANY;
-- (BOOL) getSome;
-@end
-
-
-@interface NSInvocation(CAT)
-- (id)target;
-- (void)setTarget:(id)target;
-
-- (id) dataSource;
-
-- (id)xxxdelegateYYY;
-- (void)setXxxdelegateYYY:(id)delegate;
-
-- (void)setDataSource:(id)source;
-
-- (id)MYtarget;
-- (void)setMYtarget: (id)target;
-
-- (id)targetX;
-- (void)setTargetX: (id)t;
-
-- (int)value;
-- (void)setValue: (int)val;
-
--(BOOL) isContinuous;
--(void) setContinuous:(BOOL)value;
-
-- (id) isAnObject;
-- (void)setAnObject : (id) object;
-
-- (BOOL) isinValid;
-- (void) setInValid : (BOOL) arg;
-
-- (void) Nothing;
-- (int) Length;
-- (id) object;
-+ (double) D;
-
-- (BOOL)is3bar; // watch out
-- (NSString *)get3foo; // watch out
-
-- (BOOL) getM;
-- (BOOL) getMA;
-- (BOOL) getALL;
-- (BOOL) getMANY;
-- (BOOL) getSome;
-@end
-
-DEPRECATED
-@interface I_DEP
-- (BOOL) isinValid;
-- (void) setInValid : (BOOL) arg;
-@end
-
-@interface AnotherOne
-- (BOOL) isinValid DEPRECATED;
-- (void) setInValid : (BOOL) arg;
-- (id)MYtarget;
-- (void)setMYtarget: (id)target DEPRECATED;
-- (BOOL) getM DEPRECATED;
-
-- (id)xxxdelegateYYY DEPRECATED;
-- (void)setXxxdelegateYYY:(id)delegate DEPRECATED;
-@end
-
-#define NS_AVAILABLE __attribute__((availability(macosx,introduced=10.0)))
-#define NORETURN __attribute__((noreturn))
-#define ALIGNED __attribute__((aligned(16)))
-
-@interface NSURL
-// Do not infer a property.
-- (NSURL *)appStoreReceiptURL NS_AVAILABLE;
-- (void) setAppStoreReceiptURL : (NSURL *)object;
-
-- (NSURL *)appStoreReceiptURLX NS_AVAILABLE;
-- (void) setAppStoreReceiptURLX : (NSURL *)object NS_AVAILABLE;
-
-// Do not infer a property.
-- (NSURL *)appStoreReceiptURLY ;
-- (void) setAppStoreReceiptURLY : (NSURL *)object NS_AVAILABLE;
-
-- (id)OkToInfer NS_AVAILABLE;
-
-// Do not infer a property.
-- (NSURL *)appStoreReceiptURLZ ;
-- (void) setAppStoreReceiptURLZ : (NSURL *)object NS_AVAILABLE;
-
-// Do not infer a property.
-- (id) t1 NORETURN NS_AVAILABLE;
-- (void) setT1 : (id) arg NS_AVAILABLE;
-
-- (id)method1 ALIGNED NS_AVAILABLE;
-- (void) setMethod1 : (id) object NS_AVAILABLE ALIGNED;
-
-- (NSURL *)init;  // No Change
-+ (id)alloc;      // No Change
-
-- (BOOL)is1stClass; // Not a valid property
-- (BOOL)isClass;    // This is a valid property 'class' is not a keyword in ObjC
-- (BOOL)isDouble; // Not a valid property
-
-@end
-
-@class NSMutableDictionary;
-
-@interface NSArray
-- (id (^)(id, NSArray *, NSMutableDictionary *)) expressionBlock;
-- (id (^)(id, NSArray *, NSMutableDictionary *)) MyBlock;
-- (void) setMyBlock : (id (^)(id, NSArray *, NSMutableDictionary *)) bl;
-- (id (*)(id, NSArray *, NSMutableDictionary *)) expressionFuncptr;
-- (id (*)(id, NSArray *, NSMutableDictionary *)) MyFuncptr;
-- (void) setMyFuncptr : (id (*)(id, NSArray *, NSMutableDictionary *)) bl;
-@end
diff --git a/clang/test/ARCMT/objcmt-atomic-property.m.result b/clang/test/ARCMT/objcmt-atomic-property.m.result
deleted file mode 100644
index c829a7aacb50d..0000000000000
--- a/clang/test/ARCMT/objcmt-atomic-property.m.result
+++ /dev/null
@@ -1,200 +0,0 @@
-// RUN: rm -rf %t
-// RUN: %clang_cc1 -fblocks -objcmt-migrate-readwrite-property -objcmt-migrate-readonly-property -objcmt-atomic-property -mt-migrate-directory %t %s -x objective-c -fobjc-runtime-has-weak -fobjc-arc -triple x86_64-apple-darwin11
-// RUN: c-arcmt-test -mt-migrate-directory %t | arcmt-test -verify-transformed-files %s.result
-// RUN: %clang_cc1 -fblocks -triple x86_64-apple-darwin10 -fsyntax-only -x objective-c -fobjc-runtime-has-weak -fobjc-arc %s.result
-
-#define WEBKIT_OBJC_METHOD_ANNOTATION(ANNOTATION) ANNOTATION
-#define WEAK_IMPORT_ATTRIBUTE __attribute__((objc_arc_weak_reference_unavailable))
-#define AVAILABLE_WEBKIT_VERSION_3_0_AND_LATER
-#define DEPRECATED  __attribute__((deprecated)) 
-
-typedef char BOOL;
-@class NSString;
-@protocol NSCopying @end
-
-@interface NSObject <NSCopying>
-@end
-
-@interface NSDictionary : NSObject
-@end
-
-@interface I : NSObject {
-  int ivarVal;
-}
-@property (weak) NSString *WeakProp;
-
-@property (strong) NSString *StrongProp;
-
-@property (strong) NSString *UnavailProp  __attribute__((unavailable));
-- (void) setUnavailProp  : (NSString *)Val;
-
-@property (strong) NSString *UnavailProp1  __attribute__((unavailable));
-
-@property (strong) NSString *UnavailProp2;
-- (void) setUnavailProp2  : (NSString *)Val  __attribute__((unavailable));
-
-@property (copy) NSDictionary *undoAction;
-@end
-
-@implementation I
-@end
-
-@class NSArray;
-
-@interface MyClass2  {
-@private
-    NSArray *_names1;
-    NSArray *_names2;
-    NSArray *_names3;
-    NSArray *_names4;
-}
-@property (strong) NSArray *names2;
-@property (strong) NSArray *names3;
-@property (strong) NSArray *names4;
-@property (strong) NSArray *names1;
-@end
-
-// Properties that contain the name "delegate" or "dataSource",
-// or have exact name "target" have unsafe_unretained attribute.
-@interface NSInvocation 
-@property (assign) id target;
-
-@property (assign) id dataSource;
-
-@property (assign) id xxxdelegateYYY;
-
-
-@property (strong) id MYtarget;
-
-@property (strong) id targetX;
- 
-@property  int value;
-
-@property (getter=isContinuous) BOOL continuous;
-
-- (id) isAnObject;
-- (void)setAnObject : (id) object;
-
-@property (getter=isinValid, readonly) BOOL inValid;
-- (void) setInValid : (BOOL) arg;
-
-- (void) Nothing;
-@property (readonly) int Length;
-@property (readonly, strong) id object;
-+ (double) D;
-@property (readonly) void *JSObject WEBKIT_OBJC_METHOD_ANNOTATION(AVAILABLE_WEBKIT_VERSION_3_0_AND_LATER);
-@property (getter=isIgnoringInteractionEvents, readonly) BOOL ignoringInteractionEvents;
-
-@property (getter=getStringValue, strong) NSString *stringValue;
-@property (getter=getCounterValue, readonly) BOOL counterValue;
-@property (getter=getns_dixtionary, readonly, copy) NSDictionary *ns_dixtionary;
-
-- (BOOL)is3bar; // watch out
-- (NSString *)get3foo; // watch out
-
-@property (getter=getM, readonly) BOOL m;
-@property (getter=getMA, readonly) BOOL MA;
-@property (getter=getALL, readonly) BOOL ALL;
-@property (getter=getMANY, readonly) BOOL MANY;
-@property (getter=getSome, readonly) BOOL some;
-@end
-
-
-@interface NSInvocation(CAT)
-@property (assign) id target;
-
-@property (assign) id dataSource;
-
-@property (assign) id xxxdelegateYYY;
-
-
-@property (strong) id MYtarget;
-
-@property (strong) id targetX;
-
-@property  int value;
-
-@property (getter=isContinuous) BOOL continuous;
-
-- (id) isAnObject;
-- (void)setAnObject : (id) object;
-
-@property (getter=isinValid, readonly) BOOL inValid;
-- (void) setInValid : (BOOL) arg;
-
-- (void) Nothing;
-@property (readonly) int Length;
-@property (readonly, strong) id object;
-+ (double) D;
-
-- (BOOL)is3bar; // watch out
-- (NSString *)get3foo; // watch out
-
-@property (getter=getM, readonly) BOOL m;
-@property (getter=getMA, readonly) BOOL MA;
-@property (getter=getALL, readonly) BOOL ALL;
-@property (getter=getMANY, readonly) BOOL MANY;
-@property (getter=getSome, readonly) BOOL some;
-@end
-
-DEPRECATED
-@interface I_DEP
-- (BOOL) isinValid;
-- (void) setInValid : (BOOL) arg;
-@end
-
-@interface AnotherOne
-- (BOOL) isinValid DEPRECATED;
-- (void) setInValid : (BOOL) arg;
-- (id)MYtarget;
-- (void)setMYtarget: (id)target DEPRECATED;
-- (BOOL) getM DEPRECATED;
-
-- (id)xxxdelegateYYY DEPRECATED;
-- (void)setXxxdelegateYYY:(id)delegate DEPRECATED;
-@end
-
-#define NS_AVAILABLE __attribute__((availability(macosx,introduced=10.0)))
-#define NORETURN __attribute__((noreturn))
-#define ALIGNED __attribute__((aligned(16)))
-
-@interface NSURL
-// Do not infer a property.
-@property (strong) NSURL *appStoreReceiptURL NS_AVAILABLE;
-- (void) setAppStoreReceiptURL : (NSURL *)object;
-
-@property (strong) NSURL *appStoreReceiptURLX NS_AVAILABLE;
-
-// Do not infer a property.
-@property (strong) NSURL *appStoreReceiptURLY ;
-- (void) setAppStoreReceiptURLY : (NSURL *)object NS_AVAILABLE;
-
-@property (readonly, strong) id OkToInfer NS_AVAILABLE;
-
-// Do not infer a property.
-@property (strong) NSURL *appStoreReceiptURLZ ;
-- (void) setAppStoreReceiptURLZ : (NSURL *)object NS_AVAILABLE;
-
-// Do not infer a property.
-- (id) t1 NORETURN NS_AVAILABLE;
-- (void) setT1 : (id) arg NS_AVAILABLE;
-
-@property (strong) id method1 ALIGNED NS_AVAILABLE;
-
-- (NSURL *)init;  // No Change
-+ (id)alloc;      // No Change
-
-- (BOOL)is1stClass; // Not a valid property
-@property (getter=isClass, readonly) BOOL class;    // This is a valid property 'class' is not a keyword in ObjC
-- (BOOL)isDouble; // Not a valid property
-
-@end
-
-@class NSMutableDictionary;
-
-@interface NSArray
-@property (readonly, copy) id (^expressionBlock)(id, NSArray *, NSMutableDictionary *);
-@property (copy) id (^MyBlock)(id, NSArray *, NSMutableDictionary *);
-@property (readonly) id (*expressionFuncptr)(id, NSArray *, NSMutableDictionary *);
-@property  id (*MyFuncptr)(id, NSArray *, NSMutableDictionary *);
-@end
diff --git a/clang/test/ARCMT/objcmt-boxing.m b/clang/test/ARCMT/objcmt-boxing.m
deleted file mode 100644
index 07ee68dda33bd..0000000000000
--- a/clang/test/ARCMT/objcmt-boxing.m
+++ /dev/null
@@ -1,106 +0,0 @@
-// RUN: rm -rf %t
-// RUN: %clang_cc1 -fobjc-arc -objcmt-migrate-literals -objcmt-migrate-subscripting -mt-migrate-directory %t %s -x objective-c++ -verify
-// RUN: c-arcmt-test -mt-migrate-directory %t | arcmt-test -verify-transformed-files %s.result
-// RUN: %clang_cc1 -triple x86_64-apple-darwin10 -fsyntax-only -x objective-c++ %s.result
-
-#define YES __objc_yes
-#define NO __objc_no
-
-typedef long NSInteger;
-typedef unsigned long NSUInteger;
-typedef signed char BOOL;
-#define nil ((void*) 0)
-
-#define INT_MIN   (-__INT_MAX__  -1)
-
-@interface NSObject
-+ (id)alloc;
-@end
-
-@interface NSNumber : NSObject
-@end
-
-@interface NSNumber (NSNumberCreation)
-- (id)initWithChar:(char)value;
-- (id)initWithUnsignedChar:(unsigned char)value;
-- (id)initWithShort:(short)value;
-- (id)initWithUnsignedShort:(unsigned short)value;
-- (id)initWithInt:(int)value;
-- (id)initWithUnsignedInt:(unsigned int)value;
-- (id)initWithLong:(long)value;
-- (id)initWithUnsignedLong:(unsigned long)value;
-- (id)initWithLongLong:(long long)value;
-- (id)initWithUnsignedLongLong:(unsigned long long)value;
-- (id)initWithFloat:(float)value;
-- (id)initWithDouble:(double)value;
-- (id)initWithBool:(BOOL)value;
-- (id)initWithInteger:(NSInteger)value;
-- (id)initWithUnsignedInteger:(NSUInteger)value;
-
-+ (NSNumber *)numberWithChar:(char)value;
-+ (NSNumber *)numberWithUnsignedChar:(unsigned char)value;
-+ (NSNumber *)numberWithShort:(short)value;
-+ (NSNumber *)numberWithUnsignedShort:(unsigned short)value;
-+ (NSNumber *)numberWithInt:(int)value;
-+ (NSNumber *)numberWithUnsignedInt:(unsigned int)value;
-+ (NSNumber *)numberWithLong:(long)value;
-+ (NSNumber *)numberWithUnsignedLong:(unsigned long)value;
-+ (NSNumber *)numberWithLongLong:(long long)value;
-+ (NSNumber *)numberWithUnsignedLongLong:(unsigned long long)value;
-+ (NSNumber *)numberWithFloat:(float)value;
-+ (NSNumber *)numberWithDouble:(double)value;
-+ (NSNumber *)numberWithBool:(BOOL)value;
-+ (NSNumber *)numberWithInteger:(NSInteger)value;
-+ (NSNumber *)numberWithUnsignedInteger:(NSUInteger)value;
-@end
-
-enum {
-    NSASCIIStringEncoding = 1,
-    NSUTF8StringEncoding = 4,
-    NSUnicodeStringEncoding = 10
-};
-typedef NSUInteger NSStringEncoding;
-
-@interface NSString : NSObject
-@end
-
-@interface NSString (NSStringExtensionMethods)
-+ (id)stringWithUTF8String:(const char *)nullTerminatedCString;
-+ (id)stringWithCString:(const char *)cString encoding:(NSStringEncoding)enc;
-+ (id)stringWithCString:(const char *)bytes;
-- (instancetype)initWithUTF8String:(const char *)nullTerminatedCString;
-@end
-
-enum MyEnm {
-  ME_foo
-};
-
-void foo() {
-  [NSNumber numberWithInt:INT_MIN];
-  bool cppb;
-  [NSNumber numberWithBool:cppb];
-  MyEnm myenum; 
-  [NSNumber numberWithInteger:myenum];
-  [NSNumber numberWithInteger:ME_foo];
-  [NSNumber numberWithDouble:cppb]; // expected-warning {{converting to boxing syntax requires casting 'bool' to 'double'}}
-}
-
-void boxString() {
-  NSString *s = [NSString stringWithUTF8String:"box"];
-  const char *cstr1;
-  char *cstr2;
-  s = [NSString stringWithUTF8String:cstr1];
-  s = [NSString stringWithUTF8String:cstr2];
-  s = [NSString stringWithCString:cstr1 encoding:NSASCIIStringEncoding];
-  s = [NSString stringWithCString:cstr1 encoding:NSUTF8StringEncoding];
-  s = [NSString stringWithCString:cstr1 encoding: NSUnicodeStringEncoding];
-  NSStringEncoding encode;
-  s = [NSString stringWithCString:cstr1 encoding:encode];
-  s = [NSString stringWithCString:cstr1];
-
-  static const char strarr[] = "coolbox";
-  s = [NSString stringWithUTF8String:strarr];
-  const char *utf8Bytes = "blah";
-  NSString *string1 = [NSString stringWithUTF8String:utf8Bytes];
-  NSString *string2 = [[NSString alloc] initWithUTF8String:utf8Bytes];
-}
diff --git a/clang/test/ARCMT/objcmt-boxing.m.result b/clang/test/ARCMT/objcmt-boxing.m.result
deleted file mode 100644
index a60b35900bf38..0000000000000
--- a/clang/test/ARCMT/objcmt-boxing.m.result
+++ /dev/null
@@ -1,106 +0,0 @@
-// RUN: rm -rf %t
-// RUN: %clang_cc1 -fobjc-arc -objcmt-migrate-literals -objcmt-migrate-subscripting -mt-migrate-directory %t %s -x objective-c++ -verify
-// RUN: c-arcmt-test -mt-migrate-directory %t | arcmt-test -verify-transformed-files %s.result
-// RUN: %clang_cc1 -triple x86_64-apple-darwin10 -fsyntax-only -x objective-c++ %s.result
-
-#define YES __objc_yes
-#define NO __objc_no
-
-typedef long NSInteger;
-typedef unsigned long NSUInteger;
-typedef signed char BOOL;
-#define nil ((void*) 0)
-
-#define INT_MIN   (-__INT_MAX__  -1)
-
-@interface NSObject
-+ (id)alloc;
-@end
-
-@interface NSNumber : NSObject
-@end
-
-@interface NSNumber (NSNumberCreation)
-- (id)initWithChar:(char)value;
-- (id)initWithUnsignedChar:(unsigned char)value;
-- (id)initWithShort:(short)value;
-- (id)initWithUnsignedShort:(unsigned short)value;
-- (id)initWithInt:(int)value;
-- (id)initWithUnsignedInt:(unsigned int)value;
-- (id)initWithLong:(long)value;
-- (id)initWithUnsignedLong:(unsigned long)value;
-- (id)initWithLongLong:(long long)value;
-- (id)initWithUnsignedLongLong:(unsigned long long)value;
-- (id)initWithFloat:(float)value;
-- (id)initWithDouble:(double)value;
-- (id)initWithBool:(BOOL)value;
-- (id)initWithInteger:(NSInteger)value;
-- (id)initWithUnsignedInteger:(NSUInteger)value;
-
-+ (NSNumber *)numberWithChar:(char)value;
-+ (NSNumber *)numberWithUnsignedChar:(unsigned char)value;
-+ (NSNumber *)numberWithShort:(short)value;
-+ (NSNumber *)numberWithUnsignedShort:(unsigned short)value;
-+ (NSNumber *)numberWithInt:(int)value;
-+ (NSNumber *)numberWithUnsignedInt:(unsigned int)value;
-+ (NSNumber *)numberWithLong:(long)value;
-+ (NSNumber *)numberWithUnsignedLong:(unsigned long)value;
-+ (NSNumber *)numberWithLongLong:(long long)value;
-+ (NSNumber *)numberWithUnsignedLongLong:(unsigned long long)value;
-+ (NSNumber *)numberWithFloat:(float)value;
-+ (NSNumber *)numberWithDouble:(double)value;
-+ (NSNumber *)numberWithBool:(BOOL)value;
-+ (NSNumber *)numberWithInteger:(NSInteger)value;
-+ (NSNumber *)numberWithUnsignedInteger:(NSUInteger)value;
-@end
-
-enum {
-    NSASCIIStringEncoding = 1,
-    NSUTF8StringEncoding = 4,
-    NSUnicodeStringEncoding = 10
-};
-typedef NSUInteger NSStringEncoding;
-
-@interface NSString : NSObject
-@end
-
-@interface NSString (NSStringExtensionMethods)
-+ (id)stringWithUTF8String:(const char *)nullTerminatedCString;
-+ (id)stringWithCString:(const char *)cString encoding:(NSStringEncoding)enc;
-+ (id)stringWithCString:(const char *)bytes;
-- (instancetype)initWithUTF8String:(const char *)nullTerminatedCString;
-@end
-
-enum MyEnm {
-  ME_foo
-};
-
-void foo() {
-  @INT_MIN;
-  bool cppb;
-  @(cppb);
-  MyEnm myenum; 
-  @(myenum);
-  @(ME_foo);
-  [NSNumber numberWithDouble:cppb]; // expected-warning {{converting to boxing syntax requires casting 'bool' to 'double'}}
-}
-
-void boxString() {
-  NSString *s = @"box";
-  const char *cstr1;
-  char *cstr2;
-  s = @(cstr1);
-  s = @(cstr2);
-  s = @(cstr1);
-  s = @(cstr1);
-  s = [NSString stringWithCString:cstr1 encoding: NSUnicodeStringEncoding];
-  NSStringEncoding encode;
-  s = [NSString stringWithCString:cstr1 encoding:encode];
-  s = @(cstr1);
-
-  static const char strarr[] = "coolbox";
-  s = @(strarr);
-  const char *utf8Bytes = "blah";
-  NSString *string1 = @(utf8Bytes);
-  NSString *string2 = @(utf8Bytes);
-}
diff --git a/clang/test/ARCMT/objcmt-designated-initializer.m b/clang/test/ARCMT/objcmt-designated-initializer.m
deleted file mode 100644
index 279d4f35d8eb1..0000000000000
--- a/clang/test/ARCMT/objcmt-designated-initializer.m
+++ /dev/null
@@ -1,44 +0,0 @@
-// RUN: rm -rf %t
-// RUN: %clang_cc1 -objcmt-migrate-designated-init %s -triple x86_64-apple-darwin11 -fobjc-arc -migrate -o %t.remap
-// RUN: c-arcmt-test %t.remap | arcmt-test -verify-transformed-files %s.result
-// RUN: %clang_cc1 -triple x86_64-apple-darwin11 -fsyntax-only -x objective-c -fobjc-arc %s.result
-
-#define NS_DESIGNATED_INITIALIZER __attribute__((objc_designated_initializer))
-
-@class NSString;
-
-@interface B1
--(id)init;
-@end
-
-@interface S1 : B1
--(id)initWithFoo:(NSString*)foo;
-@end
-
-@implementation S1
--(id)initWithFoo:(NSString*)foo
-{
-  self = [super init];
-  if (self) {
-  }
-  return self;
-}
-@end
-
-@interface B2
--(id)init NS_DESIGNATED_INITIALIZER;
-@end
-
-@interface S2 : B2
--(id)init;
-@end
-
-@implementation S2
--(id)init
-{
-  self = [super init];
-  if (self) {
-  }
-  return self;
-}
-@end
diff --git a/clang/test/ARCMT/objcmt-designated-initializer.m.result b/clang/test/ARCMT/objcmt-designated-initializer.m.result
deleted file mode 100644
index 4c59b0cc58261..0000000000000
--- a/clang/test/ARCMT/objcmt-designated-initializer.m.result
+++ /dev/null
@@ -1,44 +0,0 @@
-// RUN: rm -rf %t
-// RUN: %clang_cc1 -objcmt-migrate-designated-init %s -triple x86_64-apple-darwin11 -fobjc-arc -migrate -o %t.remap
-// RUN: c-arcmt-test %t.remap | arcmt-test -verify-transformed-files %s.result
-// RUN: %clang_cc1 -triple x86_64-apple-darwin11 -fsyntax-only -x objective-c -fobjc-arc %s.result
-
-#define NS_DESIGNATED_INITIALIZER __attribute__((objc_designated_initializer))
-
-@class NSString;
-
-@interface B1
--(id)init;
-@end
-
-@interface S1 : B1
--(id)initWithFoo:(NSString*)foo NS_DESIGNATED_INITIALIZER;
-@end
-
-@implementation S1
--(id)initWithFoo:(NSString*)foo
-{
-  self = [super init];
-  if (self) {
-  }
-  return self;
-}
-@end
-
-@interface B2
--(id)init NS_DESIGNATED_INITIALIZER;
-@end
-
-@interface S2 : B2
--(id)init;
-@end
-
-@implementation S2
--(id)init
-{
-  self = [super init];
-  if (self) {
-  }
-  return self;
-}
-@end
diff --git a/clang/test/ARCMT/objcmt-instancetype-2.m b/clang/test/ARCMT/objcmt-instancetype-2.m
deleted file mode 100644
index fb59265c4be32..0000000000000
--- a/clang/test/ARCMT/objcmt-instancetype-2.m
+++ /dev/null
@@ -1,103 +0,0 @@
-// RUN: rm -rf %t
-// RUN: %clang_cc1 -objcmt-migrate-instancetype -mt-migrate-directory %t %s -x objective-c -fobjc-runtime-has-weak -fobjc-arc -triple x86_64-apple-darwin11
-// RUN: c-arcmt-test -mt-migrate-directory %t | arcmt-test -verify-transformed-files %s.result
-// RUN: %clang_cc1 -triple x86_64-apple-darwin11 -fsyntax-only -x objective-c -fobjc-runtime-has-weak -fobjc-arc %s.result
-
-typedef unsigned int NSUInteger;
-typedef int NSInteger;
-typedef char BOOL;
-@class NSData, NSError, NSProtocolChecker, NSObject;
-@class NSPortNameServer, NSTimeZone;
-
-@interface NSMutableString
-@end
-
-@interface NSString @end
-
-@class NSString, NSURL;
-@interface NSString (NSStringDeprecated)
-+ (id)stringWithContentsOfFile:(NSString *)path __attribute__((availability(macosx,introduced=10.0 ,message="" )));
-+ (id)stringWithContentsOfURL:(NSURL *)url __attribute__((availability(macosx,introduced=10.0 ,message="" )));
-+ (id)stringWithCString:(const char *)bytes length:(NSUInteger)length __attribute__((availability(macosx,introduced=10.0 ,message="" )));
-+ (id)stringWithCString:(const char *)bytes __attribute__((availability(macosx,introduced=10.0 ,message="" )));
-@end
-
-
-typedef enum NSURLBookmarkResolutionOptions {
-                Bookmark
-} NSURLBookmarkResolutionOptions;
-
-@interface NSURL
-+ (id)URLWithString:(NSString *)URLString;
-+ (id)URLWithString:(NSString *)URLString relativeToURL:(NSURL *)baseURL;
-+ (id)URLByResolvingBookmarkData:(NSData *)bookmarkData options:(NSURLBookmarkResolutionOptions)options relativeToURL:(NSURL *)relativeURL bookmarkDataIsStale:(BOOL *)isStale error:(NSError **)error __attribute__((availability(macosx,introduced=10.6)));
-@end
-
-@class NSDictionary;
-@interface NSError
-+ (id)errorWithDomain:(NSString *)domain code:(NSInteger)code userInfo:(NSDictionary *)dict;
-@end
-
-
-@interface NSMutableString (NSMutableStringExtensionMethods)
-+ (id)stringWithCapacity:(NSUInteger)capacity;
-@end
-
-@interface NSMutableData
-+ (id)dataWithCapacity:(NSUInteger)aNumItems;
-+ (id)dataWithLength:(NSUInteger)length;
-@end
-
-@interface NSMutableDictionary @end
-
-@interface NSMutableDictionary (NSSharedKeySetDictionary)
-+ (id )dictionaryWithSharedKeySet:(id)keyset __attribute__((availability(macosx,introduced=10.8)));
-@end
-
-@interface NSProtocolChecker
-+ (id)protocolCheckerWithTarget:(NSObject *)anObject protocol:(Protocol *)aProtocol;
-@end
-
-@interface NSConnection
-+ (id)connectionWithRegisteredName:(NSString *)name host:(NSString *)hostName;
-+ (id)connectionWithRegisteredName:(NSString *)name host:(NSString *)hostName usingNameServer:(NSPortNameServer *)server;
-@end
-
-@interface NSDate
-+ (id)dateWithString:(NSString *)aString __attribute__((availability(macosx,introduced=10.4)));
-@end
-
-@interface NSCalendarDate : NSDate
-+ (id)calendarDate __attribute__((availability(macosx,introduced=10.4)));
-+ (id)dateWithString:(NSString *)description calendarFormat:(NSString *)format locale:(id)locale __attribute__((availability(macosx,introduced=10.4)));
-+ (id)dateWithString:(NSString *)description calendarFormat:(NSString *)format __attribute__((availability(macosx,introduced=10.4)));
-+ (id)dateWithYear:(NSInteger)year month:(NSUInteger)month day:(NSUInteger)day hour:(NSUInteger)hour minute:(NSUInteger)minute second:(NSUInteger)second timeZone:(NSTimeZone *)aTimeZone __attribute__((availability(macosx,introduced=10.4)));
-@end
-
-@interface NSUserDefaults
-+ (id) standardUserDefaults;
-@end
-
-@interface NSNotificationCenter
-+ (id) defaultCenter;
-+  sharedCenter;
-@end
-
-@interface UIApplication
-+ (id)sharedApplication;
-+ defaultApplication;
-@end
-
-//===----------------------------------------------------------------------===//
-// Method name that has a null IdentifierInfo* for its first selector slot.
-// This test just makes sure that we handle it.
-//===----------------------------------------------------------------------===//
-@interface TestNullIdentifier
-@end
-
-@implementation TestNullIdentifier
-+ (id):(int)x, ... {
-  return 0;
-}
-@end
-
diff --git a/clang/test/ARCMT/objcmt-instancetype-2.m.result b/clang/test/ARCMT/objcmt-instancetype-2.m.result
deleted file mode 100644
index 7a32894f57060..0000000000000
--- a/clang/test/ARCMT/objcmt-instancetype-2.m.result
+++ /dev/null
@@ -1,103 +0,0 @@
-// RUN: rm -rf %t
-// RUN: %clang_cc1 -objcmt-migrate-instancetype -mt-migrate-directory %t %s -x objective-c -fobjc-runtime-has-weak -fobjc-arc -triple x86_64-apple-darwin11
-// RUN: c-arcmt-test -mt-migrate-directory %t | arcmt-test -verify-transformed-files %s.result
-// RUN: %clang_cc1 -triple x86_64-apple-darwin11 -fsyntax-only -x objective-c -fobjc-runtime-has-weak -fobjc-arc %s.result
-
-typedef unsigned int NSUInteger;
-typedef int NSInteger;
-typedef char BOOL;
-@class NSData, NSError, NSProtocolChecker, NSObject;
-@class NSPortNameServer, NSTimeZone;
-
-@interface NSMutableString
-@end
-
-@interface NSString @end
-
-@class NSString, NSURL;
-@interface NSString (NSStringDeprecated)
-+ (id)stringWithContentsOfFile:(NSString *)path __attribute__((availability(macosx,introduced=10.0 ,message="" )));
-+ (id)stringWithContentsOfURL:(NSURL *)url __attribute__((availability(macosx,introduced=10.0 ,message="" )));
-+ (id)stringWithCString:(const char *)bytes length:(NSUInteger)length __attribute__((availability(macosx,introduced=10.0 ,message="" )));
-+ (id)stringWithCString:(const char *)bytes __attribute__((availability(macosx,introduced=10.0 ,message="" )));
-@end
-
-
-typedef enum NSURLBookmarkResolutionOptions {
-                Bookmark
-} NSURLBookmarkResolutionOptions;
-
-@interface NSURL
-+ (instancetype)URLWithString:(NSString *)URLString;
-+ (instancetype)URLWithString:(NSString *)URLString relativeToURL:(NSURL *)baseURL;
-+ (instancetype)URLByResolvingBookmarkData:(NSData *)bookmarkData options:(NSURLBookmarkResolutionOptions)options relativeToURL:(NSURL *)relativeURL bookmarkDataIsStale:(BOOL *)isStale error:(NSError **)error __attribute__((availability(macosx,introduced=10.6)));
-@end
-
-@class NSDictionary;
-@interface NSError
-+ (instancetype)errorWithDomain:(NSString *)domain code:(NSInteger)code userInfo:(NSDictionary *)dict;
-@end
-
-
-@interface NSMutableString (NSMutableStringExtensionMethods)
-+ (instancetype)stringWithCapacity:(NSUInteger)capacity;
-@end
-
-@interface NSMutableData
-+ (instancetype)dataWithCapacity:(NSUInteger)aNumItems;
-+ (instancetype)dataWithLength:(NSUInteger)length;
-@end
-
-@interface NSMutableDictionary @end
-
-@interface NSMutableDictionary (NSSharedKeySetDictionary)
-+ (instancetype )dictionaryWithSharedKeySet:(id)keyset __attribute__((availability(macosx,introduced=10.8)));
-@end
-
-@interface NSProtocolChecker
-+ (instancetype)protocolCheckerWithTarget:(NSObject *)anObject protocol:(Protocol *)aProtocol;
-@end
-
-@interface NSConnection
-+ (instancetype)connectionWithRegisteredName:(NSString *)name host:(NSString *)hostName;
-+ (instancetype)connectionWithRegisteredName:(NSString *)name host:(NSString *)hostName usingNameServer:(NSPortNameServer *)server;
-@end
-
-@interface NSDate
-+ (instancetype)dateWithString:(NSString *)aString __attribute__((availability(macosx,introduced=10.4)));
-@end
-
-@interface NSCalendarDate : NSDate
-+ (instancetype)calendarDate __attribute__((availability(macosx,introduced=10.4)));
-+ (instancetype)dateWithString:(NSString *)description calendarFormat:(NSString *)format locale:(id)locale __attribute__((availability(macosx,introduced=10.4)));
-+ (instancetype)dateWithString:(NSString *)description calendarFormat:(NSString *)format __attribute__((availability(macosx,introduced=10.4)));
-+ (instancetype)dateWithYear:(NSInteger)year month:(NSUInteger)month day:(NSUInteger)day hour:(NSUInteger)hour minute:(NSUInteger)minute second:(NSUInteger)second timeZone:(NSTimeZone *)aTimeZone __attribute__((availability(macosx,introduced=10.4)));
-@end
-
-@interface NSUserDefaults
-+ (instancetype) standardUserDefaults;
-@end
-
-@interface NSNotificationCenter
-+ (NSNotificationCenter*) defaultCenter;
-+ (NSNotificationCenter*)  sharedCenter;
-@end
-
-@interface UIApplication
-+ (UIApplication*)sharedApplication;
-+ (UIApplication*) defaultApplication;
-@end
-
-//===----------------------------------------------------------------------===//
-// Method name that has a null IdentifierInfo* for its first selector slot.
-// This test just makes sure that we handle it.
-//===----------------------------------------------------------------------===//
-@interface TestNullIdentifier
-@end
-
-@implementation TestNullIdentifier
-+ (id):(int)x, ... {
-  return 0;
-}
-@end
-
diff --git a/clang/test/ARCMT/objcmt-instancetype-unnecessary-diff.m b/clang/test/ARCMT/objcmt-instancetype-unnecessary-diff.m
deleted file mode 100644
index e250bb0956c05..0000000000000
--- a/clang/test/ARCMT/objcmt-instancetype-unnecessary-diff.m
+++ /dev/null
@@ -1,10 +0,0 @@
-// RUN: %clang_cc1 -objcmt-migrate-instancetype %s -triple x86_64-apple-darwin11 -fobjc-arc -migrate -o %t.remap
-// RUN: FileCheck %s -input-file=%t.remap
-
-// Make sure we don't create an edit unnecessarily.
-// CHECK-NOT: instancetype
-
-@class NSString;
-@interface NSDictionary
-+(instancetype) dictionaryWithURLEncodedString:(NSString *)urlEncodedString;
-@end
diff --git a/clang/test/ARCMT/objcmt-instancetype.m b/clang/test/ARCMT/objcmt-instancetype.m
deleted file mode 100644
index 17dd5a46b1e78..0000000000000
--- a/clang/test/ARCMT/objcmt-instancetype.m
+++ /dev/null
@@ -1,111 +0,0 @@
-// RUN: rm -rf %t
-// RUN: %clang_cc1 -objcmt-migrate-instancetype -mt-migrate-directory %t %s -x objective-c -fobjc-runtime-has-weak -fobjc-arc -triple x86_64-apple-darwin11
-// RUN: c-arcmt-test -mt-migrate-directory %t | arcmt-test -verify-transformed-files %s.result
-// RUN: %clang_cc1 -triple x86_64-apple-darwin11 -fsyntax-only -x objective-c -fobjc-runtime-has-weak -fobjc-arc %s.result
-
-typedef signed char BOOL;
-#define nil ((void*) 0)
-
-@interface NSObject
-+ (id)alloc;
-@end
-
-@interface NSString : NSObject
-+ (id)stringWithString:(NSString *)string;
-- (id)initWithString:(NSString *)aString;
-@end
-
-@implementation NSString : NSObject
-+ (id)stringWithString:(NSString *)string { return 0; };
-- (instancetype)initWithString:(NSString *)aString { return 0; };
-@end
-
-@interface NSArray : NSObject
-- (id)objectAtIndex:(unsigned long)index;
-- (id)objectAtIndexedSubscript:(int)index;
-@end
-
-@interface NSArray (NSArrayCreation)
-+ (id)array;
-+ (id)arrayWithObject:(id)anObject;
-+ (id)arrayWithObjects:(const id [])objects count:(unsigned long)cnt;
-+ (id)arrayWithObjects:(id)firstObj, ...;
-+ arrayWithArray:(NSArray *)array;
-
-- (id)initWithObjects:(const id [])objects count:(unsigned long)cnt;
-- (id)initWithObjects:(id)firstObj, ...;
-- (id)initWithArray:(NSArray *)array;
-
-- (id)objectAtIndex:(unsigned long)index;
-@end
-
-@implementation NSArray (NSArrayCreation)
-+ (id)array { return 0; }
-+ (id)arrayWithObject:(id)anObject {
-  return anObject;
-}
-+ (id)arrayWithObjects:(const id [])objects count:(unsigned long)cnt { return 0; }
-+ (id)arrayWithObjects:(id)firstObj, ... {
-  return 0; }
-+ arrayWithArray:(NSArray *)array {
-  return 0;
-}
-
-- (id)initWithObjects:(const id [])objects count:(unsigned long)cnt { return 0; }
-- (id)initWithObjects:(id)firstObj, ... { return 0; }
-- (id)initWithArray:(NSArray *)array { return 0; }
-
-- (id)objectAtIndex:(unsigned long)index { return 0; }
-@end
-
-@interface NSMutableArray : NSArray
-- (void)replaceObjectAtIndex:(unsigned long)index withObject:(id)anObject;
-- (void)setObject:(id)object atIndexedSubscript:(int)index;
-@end
-
-@interface NSDictionary : NSObject
-- (id)objectForKeyedSubscript:(id)key;
-@end
-
-@interface NSDictionary (NSDictionaryCreation)
-+ (id)dictionary;
-+ (id)dictionaryWithObject:(id)object forKey:(id)key;
-+ (id)dictionaryWithObjects:(const id [])objects forKeys:(const id [])keys count:(unsigned long)cnt;
-+ dictionaryWithObjectsAndKeys:(id)firstObject, ...;
-+ (id)dictionaryWithDictionary:(NSDictionary *)dict;
-+ (id)dictionaryWithObjects:(NSArray *)objects forKeys:(NSArray *)keys;
-
-- (id)initWithObjects:(const id [])objects forKeys:(const id [])keys count:(unsigned long)cnt;
-- (id)initWithObjectsAndKeys:(id)firstObject, ...;
-- (id)initWithDictionary:(NSDictionary *)otherDictionary;
-- (id)initWithObjects:(NSArray *)objects forKeys:(NSArray *)keys;
-
-- (id)objectForKey:(id)aKey;
-@end
-
-@interface NSMutableDictionary : NSDictionary
-- (void)setObject:(id)anObject forKey:(id)aKey;
-- (void)setObject:(id)object forKeyedSubscript:(id)key;
-@end
-
-@interface NSNumber : NSObject
-@end
-
-@interface NSNumber (NSNumberCreation)
-+ (NSNumber *)numberWithInt:(int)value;
-@end
-
-@implementation NSNumber (NSNumberCreation)
-+ (NSNumber *)numberWithInt:(int)value { return 0; }
-@end
-
-#define M(x) (x)
-#define PAIR(x) @#x, [NSNumber numberWithInt:(x)]
-#define TWO(x) ((x), (x))
-
-void foo(void) {
-  NSString *str = M([NSString stringWithString:@"foo"]); // expected-warning {{redundant}}
-  str = [[NSString alloc] initWithString:@"foo"]; // expected-warning {{redundant}}
-  NSArray *arr = [NSArray arrayWithArray:@[str]]; // expected-warning {{redundant}}
-  NSDictionary *dict = [NSDictionary dictionaryWithDictionary:@{str: arr}]; // expected-warning {{redundant}}
-}
diff --git a/clang/test/ARCMT/objcmt-instancetype.m.result b/clang/test/ARCMT/objcmt-instancetype.m.result
deleted file mode 100644
index 5203368aad644..0000000000000
--- a/clang/test/ARCMT/objcmt-instancetype.m.result
+++ /dev/null
@@ -1,111 +0,0 @@
-// RUN: rm -rf %t
-// RUN: %clang_cc1 -objcmt-migrate-instancetype -mt-migrate-directory %t %s -x objective-c -fobjc-runtime-has-weak -fobjc-arc -triple x86_64-apple-darwin11
-// RUN: c-arcmt-test -mt-migrate-directory %t | arcmt-test -verify-transformed-files %s.result
-// RUN: %clang_cc1 -triple x86_64-apple-darwin11 -fsyntax-only -x objective-c -fobjc-runtime-has-weak -fobjc-arc %s.result
-
-typedef signed char BOOL;
-#define nil ((void*) 0)
-
-@interface NSObject
-+ (id)alloc;
-@end
-
-@interface NSString : NSObject
-+ (instancetype)stringWithString:(NSString *)string;
-- (instancetype)initWithString:(NSString *)aString;
-@end
-
-@implementation NSString : NSObject
-+ (instancetype)stringWithString:(NSString *)string { return 0; };
-- (instancetype)initWithString:(NSString *)aString { return 0; };
-@end
-
-@interface NSArray : NSObject
-- (id)objectAtIndex:(unsigned long)index;
-- (id)objectAtIndexedSubscript:(int)index;
-@end
-
-@interface NSArray (NSArrayCreation)
-+ (instancetype)array;
-+ (instancetype)arrayWithObject:(id)anObject;
-+ (instancetype)arrayWithObjects:(const id [])objects count:(unsigned long)cnt;
-+ (instancetype)arrayWithObjects:(id)firstObj, ...;
-+ (instancetype) arrayWithArray:(NSArray *)array;
-
-- (instancetype)initWithObjects:(const id [])objects count:(unsigned long)cnt;
-- (instancetype)initWithObjects:(id)firstObj, ...;
-- (instancetype)initWithArray:(NSArray *)array;
-
-- (id)objectAtIndex:(unsigned long)index;
-@end
-
-@implementation NSArray (NSArrayCreation)
-+ (instancetype)array { return 0; }
-+ (instancetype)arrayWithObject:(id)anObject {
-  return anObject;
-}
-+ (instancetype)arrayWithObjects:(const id [])objects count:(unsigned long)cnt { return 0; }
-+ (instancetype)arrayWithObjects:(id)firstObj, ... {
-  return 0; }
-+ (instancetype) arrayWithArray:(NSArray *)array {
-  return 0;
-}
-
-- (instancetype)initWithObjects:(const id [])objects count:(unsigned long)cnt { return 0; }
-- (instancetype)initWithObjects:(id)firstObj, ... { return 0; }
-- (instancetype)initWithArray:(NSArray *)array { return 0; }
-
-- (id)objectAtIndex:(unsigned long)index { return 0; }
-@end
-
-@interface NSMutableArray : NSArray
-- (void)replaceObjectAtIndex:(unsigned long)index withObject:(id)anObject;
-- (void)setObject:(id)object atIndexedSubscript:(int)index;
-@end
-
-@interface NSDictionary : NSObject
-- (id)objectForKeyedSubscript:(id)key;
-@end
-
-@interface NSDictionary (NSDictionaryCreation)
-+ (instancetype)dictionary;
-+ (instancetype)dictionaryWithObject:(id)object forKey:(id)key;
-+ (instancetype)dictionaryWithObjects:(const id [])objects forKeys:(const id [])keys count:(unsigned long)cnt;
-+ (instancetype) dictionaryWithObjectsAndKeys:(id)firstObject, ...;
-+ (instancetype)dictionaryWithDictionary:(NSDictionary *)dict;
-+ (instancetype)dictionaryWithObjects:(NSArray *)objects forKeys:(NSArray *)keys;
-
-- (instancetype)initWithObjects:(const id [])objects forKeys:(const id [])keys count:(unsigned long)cnt;
-- (instancetype)initWithObjectsAndKeys:(id)firstObject, ...;
-- (instancetype)initWithDictionary:(NSDictionary *)otherDictionary;
-- (instancetype)initWithObjects:(NSArray *)objects forKeys:(NSArray *)keys;
-
-- (id)objectForKey:(id)aKey;
-@end
-
-@interface NSMutableDictionary : NSDictionary
-- (void)setObject:(id)anObject forKey:(id)aKey;
-- (void)setObject:(id)object forKeyedSubscript:(id)key;
-@end
-
-@interface NSNumber : NSObject
-@end
-
-@interface NSNumber (NSNumberCreation)
-+ (NSNumber *)numberWithInt:(int)value;
-@end
-
-@implementation NSNumber (NSNumberCreation)
-+ (NSNumber *)numberWithInt:(int)value { return 0; }
-@end
-
-#define M(x) (x)
-#define PAIR(x) @#x, [NSNumber numberWithInt:(x)]
-#define TWO(x) ((x), (x))
-
-void foo(void) {
-  NSString *str = M([NSString stringWithString:@"foo"]); // expected-warning {{redundant}}
-  str = [[NSString alloc] initWithString:@"foo"]; // expected-warning {{redundant}}
-  NSArray *arr = [NSArray arrayWithArray:@[str]]; // expected-warning {{redundant}}
-  NSDictionary *dict = [NSDictionary dictionaryWithDictionary:@{str: arr}]; // expected-warning {{redundant}}
-}
diff --git a/clang/test/ARCMT/objcmt-invalid-code.mm b/clang/test/ARCMT/objcmt-invalid-code.mm
deleted file mode 100644
index f780d3de57ed8..0000000000000
--- a/clang/test/ARCMT/objcmt-invalid-code.mm
+++ /dev/null
@@ -1,19 +0,0 @@
-// REQUIRES: x86-registered-target
-// RUN: rm -rf %t
-// RUN: %clang_cc1 -triple x86_64-apple-darwin10 -fsyntax-only %s -verify
-// RUN: not %clang_cc1 -triple x86_64-apple-darwin10 -objcmt-migrate-literals -objcmt-migrate-subscripting -mt-migrate-directory %t %s -x objective-c
-// RUN: c-arcmt-test -mt-migrate-directory %t | arcmt-test -verify-transformed-files %s.result
-
-// Make sure there is no crash.
-
-@interface NSObject @end
-@interface NSNumber : NSObject
-@end
-
-@interface NSNumber (NSNumberCreation)
-+ (NSNumber *)numberWithInt:(int)value;
-@end
-
-void foo(int x = undeclared) { // expected-error {{undeclared}}
-  NSNumber *n = [NSNumber numberWithInt:1];
-}
diff --git a/clang/test/ARCMT/objcmt-invalid-code.mm.result b/clang/test/ARCMT/objcmt-invalid-code.mm.result
deleted file mode 100644
index 52b2db00f6109..0000000000000
--- a/clang/test/ARCMT/objcmt-invalid-code.mm.result
+++ /dev/null
@@ -1,19 +0,0 @@
-// REQUIRES: x86-registered-target
-// RUN: rm -rf %t
-// RUN: %clang_cc1 -triple x86_64-apple-darwin10 -fsyntax-only %s -verify
-// RUN: not %clang_cc1 -triple x86_64-apple-darwin10 -objcmt-migrate-literals -objcmt-migrate-subscripting -mt-migrate-directory %t %s -x objective-c
-// RUN: c-arcmt-test -mt-migrate-directory %t | arcmt-test -verify-transformed-files %s.result
-
-// Make sure there is no crash.
-
-@interface NSObject @end
-@interface NSNumber : NSObject
-@end
-
-@interface NSNumber (NSNumberCreation)
-+ (NSNumber *)numberWithInt:(int)value;
-@end
-
-void foo(int x = undeclared) { // expected-error {{undeclared}}
-  NSNumber *n = @1;
-}
diff --git a/clang/test/ARCMT/objcmt-migrate-all.m b/clang/test/ARCMT/objcmt-migrate-all.m
deleted file mode 100644
index 0aa66756b761c..0000000000000
--- a/clang/test/ARCMT/objcmt-migrate-all.m
+++ /dev/null
@@ -1,133 +0,0 @@
-// RUN: rm -rf %t
-// RUN: %clang_cc1 -objcmt-migrate-all -mt-migrate-directory %t %s -x objective-c -fobjc-runtime-has-weak -fobjc-arc -triple x86_64-apple-darwin11
-// RUN: c-arcmt-test -mt-migrate-directory %t | arcmt-test -verify-transformed-files %s.result
-// RUN: %clang_cc1 -triple x86_64-apple-darwin10 -fsyntax-only -x objective-c -fobjc-runtime-has-weak -fobjc-arc %s.result
-
-#ifndef NS_RETURNS_INNER_POINTER // defined in iOS 6 for sure
-#define NS_RETURNS_INNER_POINTER __attribute__((objc_returns_inner_pointer))
-#endif
-
-#define CF_IMPLICIT_BRIDGING_ENABLED _Pragma("clang arc_cf_code_audited begin")
-
-#define CF_IMPLICIT_BRIDGING_DISABLED _Pragma("clang arc_cf_code_audited end")
-
-#if __has_feature(attribute_ns_returns_retained)
-#define NS_RETURNS_RETAINED __attribute__((ns_returns_retained))
-#endif
-#if __has_feature(attribute_cf_returns_retained)
-#define CF_RETURNS_RETAINED __attribute__((cf_returns_retained))
-#endif
-#if __has_feature(attribute_ns_returns_not_retained)
-#define NS_RETURNS_NOT_RETAINED __attribute__((ns_returns_not_retained))
-#endif
-#if __has_feature(attribute_cf_returns_not_retained)
-#define CF_RETURNS_NOT_RETAINED __attribute__((cf_returns_not_retained))
-#endif
-#if __has_feature(attribute_ns_consumes_self)
-#define NS_CONSUMES_SELF __attribute__((ns_consumes_self))
-#endif
-#if __has_feature(attribute_ns_consumed)
-#define NS_CONSUMED __attribute__((ns_consumed))
-#endif
-#if __has_feature(attribute_cf_consumed)
-#define CF_CONSUMED __attribute__((cf_consumed))
-#endif
-#if __has_attribute(ns_returns_autoreleased)
-#define NS_RETURNS_AUTORELEASED __attribute__((ns_returns_autoreleased))
-#endif
-
-#define NS_AVAILABLE __attribute__((availability(macosx,introduced=10.0)))
-
-CF_IMPLICIT_BRIDGING_ENABLED
-
-typedef unsigned long CFTypeID;
-typedef unsigned long CFOptionFlags;
-typedef unsigned long CFHashCode;
-
-typedef signed long CFIndex; /*AnyObj*/
-typedef const struct __CFArray * CFArrayRef;
-typedef struct {
-    CFIndex location;
-    CFIndex length;
-} CFRange;
-
-typedef void (*CFArrayApplierFunction)(const void *value, void *context);
-
-typedef enum CFComparisonResult : CFIndex CFComparisonResult; enum CFComparisonResult : CFIndex {
-    kCFCompareLessThan = -1L,
-    kCFCompareEqualTo = 0,
-    kCFCompareGreaterThan = 1
-};
-
-
-typedef CFComparisonResult (*CFComparatorFunction)(const void *val1, const void *val2, void *context);
-
-typedef struct __CFArray * CFMutableArrayRef;
-
-typedef const struct __CFAttributedString *CFAttributedStringRef;
-typedef struct __CFAttributedString *CFMutableAttributedStringRef;
-
-typedef const struct __CFAllocator * CFAllocatorRef;
-
-typedef const struct __CFString * CFStringRef;
-typedef struct __CFString * CFMutableStringRef;
-
-typedef const struct __CFDictionary * CFDictionaryRef;
-typedef struct __CFDictionary * CFMutableDictionaryRef;
-
-typedef struct CGImage *CGImageRef;
-
-typedef struct OpaqueJSValue* JSObjectRef;
-
-typedef JSObjectRef TTJSObjectRef;
-typedef unsigned int NSUInteger;
-
-CF_IMPLICIT_BRIDGING_DISABLED
-
-@interface I
-- (void*) ReturnsInnerPointer;
-- (int*)  AlreadyReturnsInnerPointer NS_RETURNS_INNER_POINTER;
-@end
-
-@interface UIImage
-- (CGImageRef)CGImage;
-@end
-
-@interface NSData
-- (void *)bytes;
-- (void **) ptr_bytes __attribute__((availability(macosx,unavailable)));
-@end
-
-@interface NSMutableData
-- (void *)mutableBytes  __attribute__((deprecated)) __attribute__((unavailable));
-@end
-
-@interface JS
-- (JSObjectRef)JSObject; 
-- (TTJSObjectRef)JSObject1;
-- (JSObjectRef*)JSObject2;
-@end
-
-typedef void *SecTrustRef;
-
-@interface NSURLProtectionSpace
-@property (readonly) SecTrustRef serverTrust NS_AVAILABLE;
-- (void *) FOO NS_AVAILABLE;
-@property (readonly) void * mitTrust NS_AVAILABLE;
-
-@property (readonly) void * mittiTrust;
-
-@property (readonly) SecTrustRef XserverTrust;
-
-- (SecTrustRef) FOO1 NS_AVAILABLE;
-
-+ (const NSURLProtectionSpace *)ProtectionSpace;
-
-// pointer personality functions
-@property NSUInteger (*hashFunction)(const void *item, NSUInteger (*size)(const void *item));
-@end
-
-@interface MustNotMigrateToInnerPointer
-- (void*) nono;
-- (void) setNono : (void*) val;
-@end
diff --git a/clang/test/ARCMT/objcmt-migrate-all.m.result b/clang/test/ARCMT/objcmt-migrate-all.m.result
deleted file mode 100644
index e0972875e11b8..0000000000000
--- a/clang/test/ARCMT/objcmt-migrate-all.m.result
+++ /dev/null
@@ -1,132 +0,0 @@
-// RUN: rm -rf %t
-// RUN: %clang_cc1 -objcmt-migrate-all -mt-migrate-directory %t %s -x objective-c -fobjc-runtime-has-weak -fobjc-arc -triple x86_64-apple-darwin11
-// RUN: c-arcmt-test -mt-migrate-directory %t | arcmt-test -verify-transformed-files %s.result
-// RUN: %clang_cc1 -triple x86_64-apple-darwin10 -fsyntax-only -x objective-c -fobjc-runtime-has-weak -fobjc-arc %s.result
-
-#ifndef NS_RETURNS_INNER_POINTER // defined in iOS 6 for sure
-#define NS_RETURNS_INNER_POINTER __attribute__((objc_returns_inner_pointer))
-#endif
-
-#define CF_IMPLICIT_BRIDGING_ENABLED _Pragma("clang arc_cf_code_audited begin")
-
-#define CF_IMPLICIT_BRIDGING_DISABLED _Pragma("clang arc_cf_code_audited end")
-
-#if __has_feature(attribute_ns_returns_retained)
-#define NS_RETURNS_RETAINED __attribute__((ns_returns_retained))
-#endif
-#if __has_feature(attribute_cf_returns_retained)
-#define CF_RETURNS_RETAINED __attribute__((cf_returns_retained))
-#endif
-#if __has_feature(attribute_ns_returns_not_retained)
-#define NS_RETURNS_NOT_RETAINED __attribute__((ns_returns_not_retained))
-#endif
-#if __has_feature(attribute_cf_returns_not_retained)
-#define CF_RETURNS_NOT_RETAINED __attribute__((cf_returns_not_retained))
-#endif
-#if __has_feature(attribute_ns_consumes_self)
-#define NS_CONSUMES_SELF __attribute__((ns_consumes_self))
-#endif
-#if __has_feature(attribute_ns_consumed)
-#define NS_CONSUMED __attribute__((ns_consumed))
-#endif
-#if __has_feature(attribute_cf_consumed)
-#define CF_CONSUMED __attribute__((cf_consumed))
-#endif
-#if __has_attribute(ns_returns_autoreleased)
-#define NS_RETURNS_AUTORELEASED __attribute__((ns_returns_autoreleased))
-#endif
-
-#define NS_AVAILABLE __attribute__((availability(macosx,introduced=10.0)))
-
-CF_IMPLICIT_BRIDGING_ENABLED
-
-typedef unsigned long CFTypeID;
-typedef unsigned long CFOptionFlags;
-typedef unsigned long CFHashCode;
-
-typedef signed long CFIndex; /*AnyObj*/
-typedef const struct __CFArray * CFArrayRef;
-typedef struct {
-    CFIndex location;
-    CFIndex length;
-} CFRange;
-
-typedef void (*CFArrayApplierFunction)(const void *value, void *context);
-
-typedef enum CFComparisonResult : CFIndex CFComparisonResult; enum CFComparisonResult : CFIndex {
-    kCFCompareLessThan = -1L,
-    kCFCompareEqualTo = 0,
-    kCFCompareGreaterThan = 1
-};
-
-
-typedef CFComparisonResult (*CFComparatorFunction)(const void *val1, const void *val2, void *context);
-
-typedef struct __CFArray * CFMutableArrayRef;
-
-typedef const struct __CFAttributedString *CFAttributedStringRef;
-typedef struct __CFAttributedString *CFMutableAttributedStringRef;
-
-typedef const struct __CFAllocator * CFAllocatorRef;
-
-typedef const struct __CFString * CFStringRef;
-typedef struct __CFString * CFMutableStringRef;
-
-typedef const struct __CFDictionary * CFDictionaryRef;
-typedef struct __CFDictionary * CFMutableDictionaryRef;
-
-typedef struct CGImage *CGImageRef;
-
-typedef struct OpaqueJSValue* JSObjectRef;
-
-typedef JSObjectRef TTJSObjectRef;
-typedef unsigned int NSUInteger;
-
-CF_IMPLICIT_BRIDGING_DISABLED
-
-@interface I
-@property (nonatomic, readonly) void *ReturnsInnerPointer;
-@property (nonatomic, readonly) int *AlreadyReturnsInnerPointer NS_RETURNS_INNER_POINTER;
-@end
-
-@interface UIImage
-@property (nonatomic, readonly) CGImageRef CGImage CF_RETURNS_NOT_RETAINED;
-@end
-
-@interface NSData
-@property (nonatomic, readonly) void *bytes;
-@property (nonatomic, readonly) void **ptr_bytes __attribute__((availability(macosx,unavailable)));
-@end
-
-@interface NSMutableData
-@property (nonatomic, readonly) void *mutableBytes  __attribute__((deprecated)) __attribute__((unavailable));
-@end
-
-@interface JS
-@property (nonatomic, readonly) JSObjectRef JSObject; 
-@property (nonatomic, readonly) TTJSObjectRef JSObject1;
-@property (nonatomic, readonly) JSObjectRef *JSObject2;
-@end
-
-typedef void *SecTrustRef;
-
-@interface NSURLProtectionSpace
-@property (readonly) SecTrustRef serverTrust NS_AVAILABLE;
-@property (nonatomic, readonly) void *FOO NS_AVAILABLE;
-@property (readonly) void * mitTrust NS_AVAILABLE;
-
-@property (readonly) void * mittiTrust;
-
-@property (readonly) SecTrustRef XserverTrust;
-
-@property (nonatomic, readonly) SecTrustRef FOO1 NS_AVAILABLE;
-
-+ (const NSURLProtectionSpace *)ProtectionSpace;
-
-// pointer personality functions
-@property NSUInteger (*hashFunction)(const void *item, NSUInteger (*size)(const void *item));
-@end
-
-@interface MustNotMigrateToInnerPointer
-@property (nonatomic) void *nono;
-@end
diff --git a/clang/test/ARCMT/objcmt-ns-enum-crash.m b/clang/test/ARCMT/objcmt-ns-enum-crash.m
deleted file mode 100644
index 7c9c708efaefb..0000000000000
--- a/clang/test/ARCMT/objcmt-ns-enum-crash.m
+++ /dev/null
@@ -1,14 +0,0 @@
-// RUN: rm -rf %t
-// RUN: %clang_cc1 -objcmt-migrate-ns-macros -mt-migrate-directory %t %s -x objective-c -fobjc-runtime-has-weak -fobjc-arc -triple x86_64-apple-darwin11
-// RUN: c-arcmt-test -mt-migrate-directory %t | arcmt-test -verify-transformed-files %s.result
-
-#define NS_ENUM(_type, _name) enum _name : _type _name; enum _name : _type
-#define NS_OPTIONS(_type, _name) enum _name : _type _name; enum _name : _type
-typedef long NSInteger;
-
-typedef enum : NSInteger {five} ApplicableEnum;
-
-typedef unsigned long mytd;
-
-#define MY_ENUM(name, type, ...) typedef enum : type { __VA_ARGS__ } name##_t
-MY_ENUM(MyEnum, unsigned int, One);
diff --git a/clang/test/ARCMT/objcmt-ns-enum-crash.m.result b/clang/test/ARCMT/objcmt-ns-enum-crash.m.result
deleted file mode 100644
index 0a76e66ea2111..0000000000000
--- a/clang/test/ARCMT/objcmt-ns-enum-crash.m.result
+++ /dev/null
@@ -1,14 +0,0 @@
-// RUN: rm -rf %t
-// RUN: %clang_cc1 -objcmt-migrate-ns-macros -mt-migrate-directory %t %s -x objective-c -fobjc-runtime-has-weak -fobjc-arc -triple x86_64-apple-darwin11
-// RUN: c-arcmt-test -mt-migrate-directory %t | arcmt-test -verify-transformed-files %s.result
-
-#define NS_ENUM(_type, _name) enum _name : _type _name; enum _name : _type
-#define NS_OPTIONS(_type, _name) enum _name : _type _name; enum _name : _type
-typedef long NSInteger;
-
-typedef NS_ENUM(NSInteger, ApplicableEnum) {five};
-
-typedef unsigned long mytd;
-
-#define MY_ENUM(name, type, ...) typedef enum : type { __VA_ARGS__ } name##_t
-MY_ENUM(MyEnum, unsigned int, One);
diff --git a/clang/test/ARCMT/objcmt-ns-macros.m b/clang/test/ARCMT/objcmt-ns-macros.m
deleted file mode 100644
index 902e765bead3a..0000000000000
--- a/clang/test/ARCMT/objcmt-ns-macros.m
+++ /dev/null
@@ -1,379 +0,0 @@
-// RUN: rm -rf %t
-// RUN: %clang_cc1 -objcmt-migrate-ns-macros -mt-migrate-directory %t %s -x objective-c -fobjc-runtime-has-weak -fobjc-arc -triple x86_64-apple-darwin11
-// RUN: c-arcmt-test -mt-migrate-directory %t | arcmt-test -verify-transformed-files %s.result
-// RUN: %clang_cc1 -triple x86_64-apple-darwin10 -fsyntax-only -x objective-c -fobjc-runtime-has-weak -fobjc-arc %s.result
-
-typedef signed char int8_t;
-typedef short int16_t;
-typedef int int32_t;
-typedef long NSInteger;
-typedef long long int64_t;
-
-typedef unsigned char uint8_t;
-typedef unsigned short uint16_t;
-typedef unsigned int uint32_t;
-typedef unsigned long NSUInteger;
-typedef unsigned long long uint64_t;
-
-#define NS_ENUM(_type, _name) enum _name : _type _name; enum _name : _type
-#define NS_OPTIONS(_type, _name) enum _name : _type _name; enum _name : _type
-#define DEPRECATED  __attribute__((deprecated))
-
-enum {
-  blah,
-  blarg
-};
-typedef NSInteger wibble;
-
-enum {
-    UIViewAutoresizingNone                 = 0,
-    UIViewAutoresizingFlexibleLeftMargin,
-    UIViewAutoresizingFlexibleWidth,
-    UIViewAutoresizingFlexibleRightMargin,
-    UIViewAutoresizingFlexibleTopMargin,
-    UIViewAutoresizingFlexibleHeight,
-    UIViewAutoresizingFlexibleBottomMargin
-};
-typedef NSUInteger UITableViewCellStyle;
-
-typedef enum {
-    UIViewAnimationTransitionNone,
-    UIViewAnimationTransitionFlipFromLeft,
-    UIViewAnimationTransitionFlipFromRight,
-    UIViewAnimationTransitionCurlUp,
-    UIViewAnimationTransitionCurlDown,
-} UIViewAnimationTransition;
-
-typedef enum {
-    UIViewOne   = 0,
-    UIViewTwo   = 1 << 0,
-    UIViewThree = 1 << 1,
-    UIViewFour  = 1 << 2,
-    UIViewFive  = 1 << 3,
-    UIViewSix   = 1 << 4,
-    UIViewSeven = 1 << 5
-} UITableView;
-
-enum {
-  UIOne = 0,
-  UITwo = 0x1,
-  UIthree = 0x8,
-  UIFour = 0x100
-};
-typedef NSInteger UI;
-
-typedef enum {
-  UIP2One = 0,
-  UIP2Two = 0x1,
-  UIP2three = 0x8,
-  UIP2Four = 0x100
-} UIPOWER2;
-
-enum {
-  UNOne,
-  UNTwo
-};
-
-// Should use NS_ENUM even though it is all power of 2.
-enum {
-  UIKOne = 1,
-  UIKTwo = 2,
-};
-typedef NSInteger UIK;
-
-typedef enum  {
-    NSTickMarkBelow = 0,
-    NSTickMarkAbove = 1,
-    NSTickMarkLeft = NSTickMarkAbove,
-    NSTickMarkRight = NSTickMarkBelow
-} NSTickMarkPosition;
-
-enum {
-    UIViewNone         = 0x0,
-    UIViewMargin       = 0x1,
-    UIViewWidth        = 0x2,
-    UIViewRightMargin  = 0x3,
-    UIViewBottomMargin = 0xbadbeef
-};
-typedef NSInteger UITableStyle;
-
-enum {
-    UIView0         = 0,
-    UIView1 = 0XBADBEEF
-};
-typedef NSInteger UIStyle;
-
-enum {
-    NSTIFFFileType,
-    NSBMPFileType,
-    NSGIFFileType,
-    NSJPEGFileType,
-    NSPNGFileType,
-    NSJPEG2000FileType
-};
-typedef NSUInteger NSBitmapImageFileType;
-
-enum {
-    NSWarningAlertStyle = 0,
-    NSInformationalAlertStyle = 1,
-    NSCriticalAlertStyle = 2
-};
-typedef NSUInteger NSAlertStyle;
-
-enum {
-    D_NSTIFFFileType,
-    D_NSBMPFileType,
-    D_NSGIFFileType,
-    D_NSJPEGFileType,
-    D_NSPNGFileType,
-    D_NSJPEG2000FileType
-};
-typedef NSUInteger D_NSBitmapImageFileType DEPRECATED;
-
-typedef enum  {
-    D_NSTickMarkBelow = 0,
-    D_NSTickMarkAbove = 1
-} D_NSTickMarkPosition DEPRECATED;
-
-
-#define NS_ENUM_AVAILABLE(X,Y)
-
-enum {
-    NSFStrongMemory NS_ENUM_AVAILABLE(10_5, 6_0) = (0UL << 0),       
-    NSFOpaqueMemory NS_ENUM_AVAILABLE(10_5, 6_0) = (2UL << 0),
-    NSFMallocMemory NS_ENUM_AVAILABLE(10_5, 6_0) = (3UL << 0),       
-    NSFMachVirtualMemory NS_ENUM_AVAILABLE(10_5, 6_0) = (4UL << 0),
-    NSFWeakMemory NS_ENUM_AVAILABLE(10_8, 6_0) = (5UL << 0),         
-
-    NSFObjectPersonality NS_ENUM_AVAILABLE(10_5, 6_0) = (0UL << 8),         
-    NSFOpaquePersonality NS_ENUM_AVAILABLE(10_5, 6_0) = (1UL << 8),         
-    NSFObjectPointerPersonality NS_ENUM_AVAILABLE(10_5, 6_0) = (2UL << 8),  
-    NSFCStringPersonality NS_ENUM_AVAILABLE(10_5, 6_0) = (3UL << 8),        
-    NSFStructPersonality NS_ENUM_AVAILABLE(10_5, 6_0) = (4UL << 8),         
-    NSFIntegerPersonality NS_ENUM_AVAILABLE(10_5, 6_0) = (5UL << 8),        
-    NSFCopyIn NS_ENUM_AVAILABLE(10_5, 6_0) = (1UL << 16),      
-};
-
-typedef NSUInteger NSFOptions;
-
-typedef enum {
-  UIP0One = 0,
-  UIP0Two = 1,
-  UIP0Three = 2,
-  UIP0Four = 10,
-  UIP0Last = 0x100
-} UIP;
-
-typedef enum {
-  UIPZero = 0x0,
-  UIPOne = 0x1,
-  UIPTwo = 0x2,
-  UIP10 = 0x10,
-  UIPHundred = 0x100
-} UIP_3;
-
-typedef enum {
-  UIP4Zero = 0x0,
-  UIP4One = 0x1,
-  UIP4Two = 0x2,
-  UIP410 = 0x10,
-  UIP4Hundred = 100
-} UIP4_3;
-
-typedef enum {
-  UIP5Zero = 0x0,
-  UIP5Two = 0x2,
-  UIP510 = 0x3,
-  UIP5Hundred = 0x4
-} UIP5_3;
-
-typedef enum {
-  UIP6Zero = 0x0,
-  UIP6One = 0x1,
-  UIP6Two = 0x2,
-  UIP610 = 10,
-  UIP6Hundred = 0x100
-} UIP6_3;
-
-typedef enum {
-  UIP7Zero = 0x0,
-  UIP7One = 1,
-  UIP7Two = 0x2,
-  UIP710 = 10,
-  UIP7Hundred = 100
-} UIP7_3;
-
-
-typedef enum {
-  Random = 0,
-  Random1 = 2,
-  Random2 = 4,
-  Random3 = 0x12345,
-  Random4 = 0x3444444,
-  Random5 = 0xbadbeef,
-  Random6
-} UIP8_3;
-
-#define NS_AVAILABLE_MAC(X)  __attribute__((availability(macosx,introduced=X)))
-#define NS_ENUM_AVAILABLE_MAC(X) __attribute__((availability(macosx,introduced=X)))
-
-enum {
-    NSModalResponseStop                 = (-1000), // Also used as the default response for sheets
-    NSModalResponseAbort                = (-1001),
-    NSModalResponseContinue             = (-1002), 
-} NS_ENUM_AVAILABLE_MAC(10.9);
-typedef NSInteger NSModalResponse NS_AVAILABLE_MAC(10.9);
-
-typedef NSUInteger FarFarAwayOptions;
-
-typedef NSUInteger FarAwayOptions;
-enum {
-     NSWorkspaceLaunchAndPrint =                 0x00000002,
-     NSWorkspaceLaunchWithErrorPresentation    = 0x00000040,
-     NSWorkspaceLaunchInhibitingBackgroundOnly = 0x00000080,
-     NSWorkspaceLaunchWithoutAddingToRecents   = 0x00000100,
-     NSWorkspaceLaunchWithoutActivation        = 0x00000200,
-     NSWorkspaceLaunchAsync                    = 0x00010000,
-     NSWorkspaceLaunchAllowingClassicStartup   = 0x00020000,
-     NSWorkspaceLaunchPreferringClassic        = 0x00040000,
-     NSWorkspaceLaunchNewInstance              = 0x00080000,
-     NSWorkspaceLaunchAndHide                  = 0x00100000,
-     NSWorkspaceLaunchAndHideOthers            = 0x00200000,
-     NSWorkspaceLaunchDefault = NSWorkspaceLaunchAsync | 
-     NSWorkspaceLaunchAllowingClassicStartup
-};
-typedef NSUInteger NSWorkspaceLaunchOptions;
-
-enum {
-    NSExcludeQuickDrawElementsIconCreationOption    = 1 << 1,
-    NSExclude10_4ElementsIconCreationOption         = 1 << 2
-};
-typedef NSUInteger NSExcludeOptions;
-
-enum {
-    NSExcludeQuickDrawElementsCreationOption    = 1 << 1,
-    NSExclude10_4ElementsCreationOption         = 1 << 2
-};
-typedef NSUInteger NSExcludeCreationOption;
-
-enum {
-  FarAway1    = 1 << 1,
-  FarAway2    = 1 << 2
-};
-
-enum {
-    NSExcludeQuickDrawElementsIconOption    = 1 << 1,
-    NSExclude10_4ElementsIconOption         = 1 << 2
-};
-typedef NSUInteger NSExcludeIconOptions;
-
-@interface INTF {
-  NSExcludeIconOptions I1;
-  NSExcludeIconOptions I2;
-}
-@end
-
-enum {
-  FarFarAway1    = 1 << 1,
-  FarFarAway2    = 1 << 2
-};
-
-typedef NS_OPTIONS(NSUInteger, NSWindowOcclusionState) {
-    NSWindowOcclusionStateVisible = 1UL << 1,
-};
-
-typedef NSUInteger NSWindowNumberListOptions;
-
-enum {
-    NSDirectSelection = 0,
-    NSSelectingNext,
-    NSSelectingPrevious
-};
-typedef NSUInteger NSSelectionDirection;
-
-// standard window buttons
-enum {
-    NSWindowCloseButton,
-    NSWindowMiniaturizeButton,
-    NSWindowZoomButton,
-    NSWindowToolbarButton,
-    NSWindowDocumentIconButton
-};
-
-typedef enum : NSUInteger {
-   ThingOne,
-   ThingTwo,
-   ThingThree,
-} Thing;
-
-typedef enum {
-    one = 1
-} NumericEnum;
-
-typedef enum {
-    Two = 2
-}NumericEnum2;
-
-typedef enum {
-    Three = 3
-}
-NumericEnum3;
-
-typedef enum {
-    Four = 4
-}
-
-  NumericEnum4;
-
-enum
-{
-  UI8one = 1
-};
-typedef int8_t MyEnumeratedType;
-
-
-enum {
-  UI16One = 0,
-  UI16Two = 0x1,
-  UI16three = 0x8,
-  UI16Four = 0x100
-};
-typedef int16_t UI16;
-
-enum {
-    UI32ViewAutoresizingNone                 = 0,
-    UI32ViewAutoresizingFlexibleLeftMargin,
-    UI32ViewAutoresizingFlexibleWidth,
-    UI32ViewAutoresizingFlexibleRightMargin,
-    UI32ViewAutoresizingFlexibleTopMargin,
-    UI32ViewAutoresizingFlexibleHeight,
-    UI32ViewAutoresizingFlexibleBottomMargin
-};
-typedef uint32_t UI32TableViewCellStyle;
-
-enum
-{
-        UIU8one = 1
-};
-typedef uint8_t UI8Type;
-
-typedef enum : NSInteger {zero} MyEnum;
-
-typedef enum : NSUInteger {two} MyEnumNSUInteger;
-
-typedef enum : int {three, four} MyEnumint;
-
-typedef enum : unsigned long {five} MyEnumlonglong;
-
-typedef enum : unsigned long long {
-  ll1,
-  ll2= 0xff,
-  ll3,
-  ll4
-} MyEnumunsignedlonglong;
-
-typedef enum : int8_t {int8_one} MyOneEnum;
-
-typedef enum : int16_t {
-          int16_t_one,
-          int16_t_two } Myint16_tEnum;
diff --git a/clang/test/ARCMT/objcmt-ns-macros.m.result b/clang/test/ARCMT/objcmt-ns-macros.m.result
deleted file mode 100644
index d4c0870e8cdc3..0000000000000
--- a/clang/test/ARCMT/objcmt-ns-macros.m.result
+++ /dev/null
@@ -1,355 +0,0 @@
-// RUN: rm -rf %t
-// RUN: %clang_cc1 -objcmt-migrate-ns-macros -mt-migrate-directory %t %s -x objective-c -fobjc-runtime-has-weak -fobjc-arc -triple x86_64-apple-darwin11
-// RUN: c-arcmt-test -mt-migrate-directory %t | arcmt-test -verify-transformed-files %s.result
-// RUN: %clang_cc1 -triple x86_64-apple-darwin10 -fsyntax-only -x objective-c -fobjc-runtime-has-weak -fobjc-arc %s.result
-
-typedef signed char int8_t;
-typedef short int16_t;
-typedef int int32_t;
-typedef long NSInteger;
-typedef long long int64_t;
-
-typedef unsigned char uint8_t;
-typedef unsigned short uint16_t;
-typedef unsigned int uint32_t;
-typedef unsigned long NSUInteger;
-typedef unsigned long long uint64_t;
-
-#define NS_ENUM(_type, _name) enum _name : _type _name; enum _name : _type
-#define NS_OPTIONS(_type, _name) enum _name : _type _name; enum _name : _type
-#define DEPRECATED  __attribute__((deprecated))
-
-typedef NS_ENUM(NSInteger, wibble) {
-  blah,
-  blarg
-};
-
-typedef NS_ENUM(NSUInteger, UITableViewCellStyle) {
-    UIViewAutoresizingNone                 = 0,
-    UIViewAutoresizingFlexibleLeftMargin,
-    UIViewAutoresizingFlexibleWidth,
-    UIViewAutoresizingFlexibleRightMargin,
-    UIViewAutoresizingFlexibleTopMargin,
-    UIViewAutoresizingFlexibleHeight,
-    UIViewAutoresizingFlexibleBottomMargin
-};
-
-typedef NS_ENUM(unsigned int, UIViewAnimationTransition) {
-    UIViewAnimationTransitionNone,
-    UIViewAnimationTransitionFlipFromLeft,
-    UIViewAnimationTransitionFlipFromRight,
-    UIViewAnimationTransitionCurlUp,
-    UIViewAnimationTransitionCurlDown,
-};
-
-typedef NS_OPTIONS(unsigned int, UITableView) {
-    UIViewOne   = 0,
-    UIViewTwo   = 1 << 0,
-    UIViewThree = 1 << 1,
-    UIViewFour  = 1 << 2,
-    UIViewFive  = 1 << 3,
-    UIViewSix   = 1 << 4,
-    UIViewSeven = 1 << 5
-};
-
-typedef NS_OPTIONS(NSUInteger, UI) {
-  UIOne = 0,
-  UITwo = 0x1,
-  UIthree = 0x8,
-  UIFour = 0x100
-};
-
-typedef NS_OPTIONS(unsigned int, UIPOWER2) {
-  UIP2One = 0,
-  UIP2Two = 0x1,
-  UIP2three = 0x8,
-  UIP2Four = 0x100
-};
-
-enum {
-  UNOne,
-  UNTwo
-};
-
-// Should use NS_ENUM even though it is all power of 2.
-typedef NS_ENUM(NSInteger, UIK) {
-  UIKOne = 1,
-  UIKTwo = 2,
-};
-
-typedef NS_ENUM(unsigned int, NSTickMarkPosition) {
-    NSTickMarkBelow = 0,
-    NSTickMarkAbove = 1,
-    NSTickMarkLeft = NSTickMarkAbove,
-    NSTickMarkRight = NSTickMarkBelow
-};
-
-typedef NS_OPTIONS(NSUInteger, UITableStyle) {
-    UIViewNone         = 0x0,
-    UIViewMargin       = 0x1,
-    UIViewWidth        = 0x2,
-    UIViewRightMargin  = 0x3,
-    UIViewBottomMargin = 0xbadbeef
-};
-
-typedef NS_OPTIONS(NSUInteger, UIStyle) {
-    UIView0         = 0,
-    UIView1 = 0XBADBEEF
-};
-
-typedef NS_ENUM(NSUInteger, NSBitmapImageFileType) {
-    NSTIFFFileType,
-    NSBMPFileType,
-    NSGIFFileType,
-    NSJPEGFileType,
-    NSPNGFileType,
-    NSJPEG2000FileType
-};
-
-typedef NS_ENUM(NSUInteger, NSAlertStyle) {
-    NSWarningAlertStyle = 0,
-    NSInformationalAlertStyle = 1,
-    NSCriticalAlertStyle = 2
-};
-
-enum {
-    D_NSTIFFFileType,
-    D_NSBMPFileType,
-    D_NSGIFFileType,
-    D_NSJPEGFileType,
-    D_NSPNGFileType,
-    D_NSJPEG2000FileType
-};
-typedef NSUInteger D_NSBitmapImageFileType DEPRECATED;
-
-typedef enum  {
-    D_NSTickMarkBelow = 0,
-    D_NSTickMarkAbove = 1
-} D_NSTickMarkPosition DEPRECATED;
-
-
-#define NS_ENUM_AVAILABLE(X,Y)
-
-
-typedef NS_OPTIONS(NSUInteger, NSFOptions) {
-    NSFStrongMemory NS_ENUM_AVAILABLE(10_5, 6_0) = (0UL << 0),       
-    NSFOpaqueMemory NS_ENUM_AVAILABLE(10_5, 6_0) = (2UL << 0),
-    NSFMallocMemory NS_ENUM_AVAILABLE(10_5, 6_0) = (3UL << 0),       
-    NSFMachVirtualMemory NS_ENUM_AVAILABLE(10_5, 6_0) = (4UL << 0),
-    NSFWeakMemory NS_ENUM_AVAILABLE(10_8, 6_0) = (5UL << 0),         
-
-    NSFObjectPersonality NS_ENUM_AVAILABLE(10_5, 6_0) = (0UL << 8),         
-    NSFOpaquePersonality NS_ENUM_AVAILABLE(10_5, 6_0) = (1UL << 8),         
-    NSFObjectPointerPersonality NS_ENUM_AVAILABLE(10_5, 6_0) = (2UL << 8),  
-    NSFCStringPersonality NS_ENUM_AVAILABLE(10_5, 6_0) = (3UL << 8),        
-    NSFStructPersonality NS_ENUM_AVAILABLE(10_5, 6_0) = (4UL << 8),         
-    NSFIntegerPersonality NS_ENUM_AVAILABLE(10_5, 6_0) = (5UL << 8),        
-    NSFCopyIn NS_ENUM_AVAILABLE(10_5, 6_0) = (1UL << 16),      
-};
-
-typedef NS_ENUM(unsigned int, UIP) {
-  UIP0One = 0,
-  UIP0Two = 1,
-  UIP0Three = 2,
-  UIP0Four = 10,
-  UIP0Last = 0x100
-};
-
-typedef NS_OPTIONS(unsigned int, UIP_3) {
-  UIPZero = 0x0,
-  UIPOne = 0x1,
-  UIPTwo = 0x2,
-  UIP10 = 0x10,
-  UIPHundred = 0x100
-};
-
-typedef NS_ENUM(unsigned int, UIP4_3) {
-  UIP4Zero = 0x0,
-  UIP4One = 0x1,
-  UIP4Two = 0x2,
-  UIP410 = 0x10,
-  UIP4Hundred = 100
-};
-
-typedef NS_OPTIONS(unsigned int, UIP5_3) {
-  UIP5Zero = 0x0,
-  UIP5Two = 0x2,
-  UIP510 = 0x3,
-  UIP5Hundred = 0x4
-};
-
-typedef NS_ENUM(unsigned int, UIP6_3) {
-  UIP6Zero = 0x0,
-  UIP6One = 0x1,
-  UIP6Two = 0x2,
-  UIP610 = 10,
-  UIP6Hundred = 0x100
-};
-
-typedef NS_ENUM(unsigned int, UIP7_3) {
-  UIP7Zero = 0x0,
-  UIP7One = 1,
-  UIP7Two = 0x2,
-  UIP710 = 10,
-  UIP7Hundred = 100
-};
-
-
-typedef NS_ENUM(unsigned int, UIP8_3) {
-  Random = 0,
-  Random1 = 2,
-  Random2 = 4,
-  Random3 = 0x12345,
-  Random4 = 0x3444444,
-  Random5 = 0xbadbeef,
-  Random6
-};
-
-#define NS_AVAILABLE_MAC(X)  __attribute__((availability(macosx,introduced=X)))
-#define NS_ENUM_AVAILABLE_MAC(X) __attribute__((availability(macosx,introduced=X)))
-
-typedef NS_ENUM(NSInteger, NSModalResponse) {
-    NSModalResponseStop                 = (-1000), // Also used as the default response for sheets
-    NSModalResponseAbort                = (-1001),
-    NSModalResponseContinue             = (-1002), 
-} NS_ENUM_AVAILABLE_MAC(10.9);
-
-typedef NSUInteger FarFarAwayOptions;
-
-typedef NS_OPTIONS(NSUInteger, FarAwayOptions) {
-  FarAway1    = 1 << 1,
-  FarAway2    = 1 << 2
-};
-typedef NS_OPTIONS(NSUInteger, NSWorkspaceLaunchOptions) {
-     NSWorkspaceLaunchAndPrint =                 0x00000002,
-     NSWorkspaceLaunchWithErrorPresentation    = 0x00000040,
-     NSWorkspaceLaunchInhibitingBackgroundOnly = 0x00000080,
-     NSWorkspaceLaunchWithoutAddingToRecents   = 0x00000100,
-     NSWorkspaceLaunchWithoutActivation        = 0x00000200,
-     NSWorkspaceLaunchAsync                    = 0x00010000,
-     NSWorkspaceLaunchAllowingClassicStartup   = 0x00020000,
-     NSWorkspaceLaunchPreferringClassic        = 0x00040000,
-     NSWorkspaceLaunchNewInstance              = 0x00080000,
-     NSWorkspaceLaunchAndHide                  = 0x00100000,
-     NSWorkspaceLaunchAndHideOthers            = 0x00200000,
-     NSWorkspaceLaunchDefault = NSWorkspaceLaunchAsync | 
-     NSWorkspaceLaunchAllowingClassicStartup
-};
-
-typedef NS_OPTIONS(NSUInteger, NSExcludeOptions) {
-    NSExcludeQuickDrawElementsIconCreationOption    = 1 << 1,
-    NSExclude10_4ElementsIconCreationOption         = 1 << 2
-};
-
-typedef NS_OPTIONS(NSUInteger, NSExcludeCreationOption) {
-    NSExcludeQuickDrawElementsCreationOption    = 1 << 1,
-    NSExclude10_4ElementsCreationOption         = 1 << 2
-};
-
-
-typedef NS_OPTIONS(NSUInteger, NSExcludeIconOptions) {
-    NSExcludeQuickDrawElementsIconOption    = 1 << 1,
-    NSExclude10_4ElementsIconOption         = 1 << 2
-};
-
-@interface INTF {
-  NSExcludeIconOptions I1;
-  NSExcludeIconOptions I2;
-}
-@end
-
-enum {
-  FarFarAway1    = 1 << 1,
-  FarFarAway2    = 1 << 2
-};
-
-typedef NS_OPTIONS(NSUInteger, NSWindowOcclusionState) {
-    NSWindowOcclusionStateVisible = 1UL << 1,
-};
-
-typedef NS_ENUM(NSUInteger, NSWindowNumberListOptions) {
-    NSWindowCloseButton,
-    NSWindowMiniaturizeButton,
-    NSWindowZoomButton,
-    NSWindowToolbarButton,
-    NSWindowDocumentIconButton
-};
-
-typedef NS_ENUM(NSUInteger, NSSelectionDirection) {
-    NSDirectSelection = 0,
-    NSSelectingNext,
-    NSSelectingPrevious
-};
-
-// standard window buttons
-
-typedef NS_ENUM(NSUInteger, Thing) {
-   ThingOne,
-   ThingTwo,
-   ThingThree,
-};
-
-typedef NS_ENUM(unsigned int, NumericEnum) {
-    one = 1
-};
-
-typedef NS_ENUM(unsigned int, NumericEnum2) {
-    Two = 2
-};
-
-typedef NS_ENUM(unsigned int, NumericEnum3) {
-    Three = 3
-};
-
-typedef NS_OPTIONS(unsigned int, NumericEnum4) {
-    Four = 4
-};
-
-typedef NS_ENUM(int8_t, MyEnumeratedType)
-{
-  UI8one = 1
-};
-
-
-typedef NS_OPTIONS(uint16_t, UI16) {
-  UI16One = 0,
-  UI16Two = 0x1,
-  UI16three = 0x8,
-  UI16Four = 0x100
-};
-
-typedef NS_ENUM(uint32_t, UI32TableViewCellStyle) {
-    UI32ViewAutoresizingNone                 = 0,
-    UI32ViewAutoresizingFlexibleLeftMargin,
-    UI32ViewAutoresizingFlexibleWidth,
-    UI32ViewAutoresizingFlexibleRightMargin,
-    UI32ViewAutoresizingFlexibleTopMargin,
-    UI32ViewAutoresizingFlexibleHeight,
-    UI32ViewAutoresizingFlexibleBottomMargin
-};
-
-typedef NS_ENUM(uint8_t, UI8Type)
-{
-        UIU8one = 1
-};
-
-typedef NS_ENUM(NSInteger, MyEnum) {zero};
-
-typedef NS_ENUM(NSUInteger, MyEnumNSUInteger) {two};
-
-typedef NS_ENUM(int, MyEnumint) {three, four};
-
-typedef NS_ENUM(unsigned long, MyEnumlonglong) {five};
-
-typedef NS_ENUM(unsigned long long, MyEnumunsignedlonglong) {
-  ll1,
-  ll2= 0xff,
-  ll3,
-  ll4
-};
-
-typedef NS_ENUM(int8_t, MyOneEnum) {int8_one};
-
-typedef NS_ENUM(int16_t, Myint16_tEnum) {
-          int16_t_one,
-          int16_t_two };
diff --git a/clang/test/ARCMT/objcmt-ns-nonatomic-iosonly.m b/clang/test/ARCMT/objcmt-ns-nonatomic-iosonly.m
deleted file mode 100644
index 55a116c8ca5da..0000000000000
--- a/clang/test/ARCMT/objcmt-ns-nonatomic-iosonly.m
+++ /dev/null
@@ -1,233 +0,0 @@
-// RUN: rm -rf %t
-// RUN: %clang_cc1 -fblocks -objcmt-migrate-readwrite-property -objcmt-ns-nonatomic-iosonly -objcmt-migrate-readonly-property -objcmt-atomic-property -mt-migrate-directory %t %s -x objective-c -fobjc-runtime-has-weak -fobjc-arc -triple x86_64-apple-darwin11
-// RUN: c-arcmt-test -mt-migrate-directory %t | arcmt-test -verify-transformed-files %s.result
-// RUN: %clang_cc1 -fblocks -triple x86_64-apple-darwin10 -fsyntax-only -x objective-c -fobjc-runtime-has-weak -fobjc-arc %s.result
-
-#define WEBKIT_OBJC_METHOD_ANNOTATION(ANNOTATION) ANNOTATION
-#define WEAK_IMPORT_ATTRIBUTE __attribute__((objc_arc_weak_reference_unavailable))
-#define AVAILABLE_WEBKIT_VERSION_3_0_AND_LATER
-#define DEPRECATED  __attribute__((deprecated)) 
-
-#if TARGET_OS_IPHONE
-  #define NS_NONATOMIC_IOSONLY nonatomic
-#else
-  #define NS_NONATOMIC_IOSONLY atomic
-#endif
-
-typedef char BOOL;
-@class NSString;
-@protocol NSCopying @end
-
-@interface NSObject <NSCopying>
-@end
-
-@interface NSDictionary : NSObject
-@end
-
-@interface I : NSObject {
-  int ivarVal;
-}
-- (void) setWeakProp : (NSString *__weak)Val;
-- (NSString *__weak) WeakProp;
-
-- (NSString *) StrongProp;
-- (void) setStrongProp : (NSString *)Val;
-
-- (NSString *) UnavailProp  __attribute__((unavailable));
-- (void) setUnavailProp  : (NSString *)Val;
-
-- (NSString *) UnavailProp1  __attribute__((unavailable));
-- (void) setUnavailProp1  : (NSString *)Val  __attribute__((unavailable));
-
-- (NSString *) UnavailProp2;
-- (void) setUnavailProp2  : (NSString *)Val  __attribute__((unavailable));
-
-- (NSDictionary*) undoAction;
-- (void) setUndoAction: (NSDictionary*)Arg;
-@end
-
-@implementation I
-@end
-
-@class NSArray;
-
-@interface MyClass2  {
-@private
-    NSArray *_names1;
-    NSArray *_names2;
-    NSArray *_names3;
-    NSArray *_names4;
-}
-- (void)setNames1:(NSArray *)names;
-- (void)setNames4:(__strong NSArray *)names;
-- (void)setNames3:(__strong NSArray *)names;
-- (void)setNames2:(NSArray *)names;
-- (NSArray *) names2;
-- (NSArray *)names3;
-- (__strong NSArray *)names4;
-- (NSArray *) names1;
-@end
-
-// Properties that contain the name "delegate" or "dataSource",
-// or have exact name "target" have unsafe_unretained attribute.
-@interface NSInvocation 
-- (id)target;
-- (void)setTarget:(id)target;
-
-- (id) dataSource;
-
-- (id)xxxdelegateYYY;
-- (void)setXxxdelegateYYY:(id)delegate;
-
-- (void)setDataSource:(id)source;
-
-- (id)MYtarget;
-- (void)setMYtarget: (id)target;
-
-- (id)targetX;
-- (void)setTargetX: (id)t;
- 
-- (int)value;
-- (void)setValue: (int)val;
-
--(BOOL) isContinuous;
--(void) setContinuous:(BOOL)value;
-
-- (id) isAnObject;
-- (void)setAnObject : (id) object;
-
-- (BOOL) isinValid;
-- (void) setInValid : (BOOL) arg;
-
-- (void) Nothing;
-- (int) Length;
-- (id) object;
-+ (double) D;
-- (void *)JSObject WEBKIT_OBJC_METHOD_ANNOTATION(AVAILABLE_WEBKIT_VERSION_3_0_AND_LATER);
-- (BOOL)isIgnoringInteractionEvents;
-
-- (NSString *)getStringValue;
-- (BOOL)getCounterValue;
-- (void)setStringValue:(NSString *)stringValue AVAILABLE_WEBKIT_VERSION_3_0_AND_LATER;
-- (NSDictionary *)getns_dixtionary;
-
-- (BOOL)is3bar; // watch out
-- (NSString *)get3foo; // watch out
-
-- (BOOL) getM;
-- (BOOL) getMA;
-- (BOOL) getALL;
-- (BOOL) getMANY;
-- (BOOL) getSome;
-@end
-
-
-@interface NSInvocation(CAT)
-- (id)target;
-- (void)setTarget:(id)target;
-
-- (id) dataSource;
-
-- (id)xxxdelegateYYY;
-- (void)setXxxdelegateYYY:(id)delegate;
-
-- (void)setDataSource:(id)source;
-
-- (id)MYtarget;
-- (void)setMYtarget: (id)target;
-
-- (id)targetX;
-- (void)setTargetX: (id)t;
-
-- (int)value;
-- (void)setValue: (int)val;
-
--(BOOL) isContinuous;
--(void) setContinuous:(BOOL)value;
-
-- (id) isAnObject;
-- (void)setAnObject : (id) object;
-
-- (BOOL) isinValid;
-- (void) setInValid : (BOOL) arg;
-
-- (void) Nothing;
-- (int) Length;
-- (id) object;
-+ (double) D;
-
-- (BOOL)is3bar; // watch out
-- (NSString *)get3foo; // watch out
-
-- (BOOL) getM;
-- (BOOL) getMA;
-- (BOOL) getALL;
-- (BOOL) getMANY;
-- (BOOL) getSome;
-@end
-
-DEPRECATED
-@interface I_DEP
-- (BOOL) isinValid;
-- (void) setInValid : (BOOL) arg;
-@end
-
-@interface AnotherOne
-- (BOOL) isinValid DEPRECATED;
-- (void) setInValid : (BOOL) arg;
-- (id)MYtarget;
-- (void)setMYtarget: (id)target DEPRECATED;
-- (BOOL) getM DEPRECATED;
-
-- (id)xxxdelegateYYY DEPRECATED;
-- (void)setXxxdelegateYYY:(id)delegate DEPRECATED;
-@end
-
-#define NS_AVAILABLE __attribute__((availability(macosx,introduced=10.0)))
-#define NORETURN __attribute__((noreturn))
-#define ALIGNED __attribute__((aligned(16)))
-
-@interface NSURL
-// Do not infer a property.
-- (NSURL *)appStoreReceiptURL NS_AVAILABLE;
-- (void) setAppStoreReceiptURL : (NSURL *)object;
-
-- (NSURL *)appStoreReceiptURLX NS_AVAILABLE;
-- (void) setAppStoreReceiptURLX : (NSURL *)object NS_AVAILABLE;
-
-// Do not infer a property.
-- (NSURL *)appStoreReceiptURLY ;
-- (void) setAppStoreReceiptURLY : (NSURL *)object NS_AVAILABLE;
-
-- (id)OkToInfer NS_AVAILABLE;
-
-// Do not infer a property.
-- (NSURL *)appStoreReceiptURLZ ;
-- (void) setAppStoreReceiptURLZ : (NSURL *)object NS_AVAILABLE;
-
-// Do not infer a property.
-- (id) t1 NORETURN NS_AVAILABLE;
-- (void) setT1 : (id) arg NS_AVAILABLE;
-
-- (id)method1 ALIGNED NS_AVAILABLE;
-- (void) setMethod1 : (id) object NS_AVAILABLE ALIGNED;
-
-- (NSURL *)init;  // No Change
-+ (id)alloc;      // No Change
-
-- (BOOL)is1stClass; // Not a valid property
-- (BOOL)isClass;    // This is a valid property 'class' is not a keyword in ObjC
-- (BOOL)isDouble; // Not a valid property
-
-@end
-
-@class NSMutableDictionary;
-
-@interface NSArray
-- (id (^)(id, NSArray *, NSMutableDictionary *)) expressionBlock;
-- (id (^)(id, NSArray *, NSMutableDictionary *)) MyBlock;
-- (void) setMyBlock : (id (^)(id, NSArray *, NSMutableDictionary *)) bl;
-- (id (*)(id, NSArray *, NSMutableDictionary *)) expressionFuncptr;
-- (id (*)(id, NSArray *, NSMutableDictionary *)) MyFuncptr;
-- (void) setMyFuncptr : (id (*)(id, NSArray *, NSMutableDictionary *)) bl;
-@end
diff --git a/clang/test/ARCMT/objcmt-ns-nonatomic-iosonly.m.result b/clang/test/ARCMT/objcmt-ns-nonatomic-iosonly.m.result
deleted file mode 100644
index 512deb12583a9..0000000000000
--- a/clang/test/ARCMT/objcmt-ns-nonatomic-iosonly.m.result
+++ /dev/null
@@ -1,206 +0,0 @@
-// RUN: rm -rf %t
-// RUN: %clang_cc1 -fblocks -objcmt-migrate-readwrite-property -objcmt-ns-nonatomic-iosonly -objcmt-migrate-readonly-property -objcmt-atomic-property -mt-migrate-directory %t %s -x objective-c -fobjc-runtime-has-weak -fobjc-arc -triple x86_64-apple-darwin11
-// RUN: c-arcmt-test -mt-migrate-directory %t | arcmt-test -verify-transformed-files %s.result
-// RUN: %clang_cc1 -fblocks -triple x86_64-apple-darwin10 -fsyntax-only -x objective-c -fobjc-runtime-has-weak -fobjc-arc %s.result
-
-#define WEBKIT_OBJC_METHOD_ANNOTATION(ANNOTATION) ANNOTATION
-#define WEAK_IMPORT_ATTRIBUTE __attribute__((objc_arc_weak_reference_unavailable))
-#define AVAILABLE_WEBKIT_VERSION_3_0_AND_LATER
-#define DEPRECATED  __attribute__((deprecated)) 
-
-#if TARGET_OS_IPHONE
-  #define NS_NONATOMIC_IOSONLY nonatomic
-#else
-  #define NS_NONATOMIC_IOSONLY atomic
-#endif
-
-typedef char BOOL;
-@class NSString;
-@protocol NSCopying @end
-
-@interface NSObject <NSCopying>
-@end
-
-@interface NSDictionary : NSObject
-@end
-
-@interface I : NSObject {
-  int ivarVal;
-}
-@property (NS_NONATOMIC_IOSONLY, weak) NSString *WeakProp;
-
-@property (NS_NONATOMIC_IOSONLY, strong) NSString *StrongProp;
-
-@property (NS_NONATOMIC_IOSONLY, strong) NSString *UnavailProp  __attribute__((unavailable));
-- (void) setUnavailProp  : (NSString *)Val;
-
-@property (NS_NONATOMIC_IOSONLY, strong) NSString *UnavailProp1  __attribute__((unavailable));
-
-@property (NS_NONATOMIC_IOSONLY, strong) NSString *UnavailProp2;
-- (void) setUnavailProp2  : (NSString *)Val  __attribute__((unavailable));
-
-@property (NS_NONATOMIC_IOSONLY, copy) NSDictionary *undoAction;
-@end
-
-@implementation I
-@end
-
-@class NSArray;
-
-@interface MyClass2  {
-@private
-    NSArray *_names1;
-    NSArray *_names2;
-    NSArray *_names3;
-    NSArray *_names4;
-}
-@property (NS_NONATOMIC_IOSONLY, strong) NSArray *names2;
-@property (NS_NONATOMIC_IOSONLY, strong) NSArray *names3;
-@property (NS_NONATOMIC_IOSONLY, strong) NSArray *names4;
-@property (NS_NONATOMIC_IOSONLY, strong) NSArray *names1;
-@end
-
-// Properties that contain the name "delegate" or "dataSource",
-// or have exact name "target" have unsafe_unretained attribute.
-@interface NSInvocation 
-@property (NS_NONATOMIC_IOSONLY, assign) id target;
-
-@property (NS_NONATOMIC_IOSONLY, assign) id dataSource;
-
-@property (NS_NONATOMIC_IOSONLY, assign) id xxxdelegateYYY;
-
-
-@property (NS_NONATOMIC_IOSONLY, strong) id MYtarget;
-
-@property (NS_NONATOMIC_IOSONLY, strong) id targetX;
- 
-@property (NS_NONATOMIC_IOSONLY) int value;
-
-@property (NS_NONATOMIC_IOSONLY, getter=isContinuous) BOOL continuous;
-
-- (id) isAnObject;
-- (void)setAnObject : (id) object;
-
-@property (NS_NONATOMIC_IOSONLY, getter=isinValid, readonly) BOOL inValid;
-- (void) setInValid : (BOOL) arg;
-
-- (void) Nothing;
-@property (NS_NONATOMIC_IOSONLY, readonly) int Length;
-@property (NS_NONATOMIC_IOSONLY, readonly, strong) id object;
-+ (double) D;
-@property (NS_NONATOMIC_IOSONLY, readonly) void *JSObject WEBKIT_OBJC_METHOD_ANNOTATION(AVAILABLE_WEBKIT_VERSION_3_0_AND_LATER);
-@property (NS_NONATOMIC_IOSONLY, getter=isIgnoringInteractionEvents, readonly) BOOL ignoringInteractionEvents;
-
-@property (NS_NONATOMIC_IOSONLY, getter=getStringValue, strong) NSString *stringValue;
-@property (NS_NONATOMIC_IOSONLY, getter=getCounterValue, readonly) BOOL counterValue;
-@property (NS_NONATOMIC_IOSONLY, getter=getns_dixtionary, readonly, copy) NSDictionary *ns_dixtionary;
-
-- (BOOL)is3bar; // watch out
-- (NSString *)get3foo; // watch out
-
-@property (NS_NONATOMIC_IOSONLY, getter=getM, readonly) BOOL m;
-@property (NS_NONATOMIC_IOSONLY, getter=getMA, readonly) BOOL MA;
-@property (NS_NONATOMIC_IOSONLY, getter=getALL, readonly) BOOL ALL;
-@property (NS_NONATOMIC_IOSONLY, getter=getMANY, readonly) BOOL MANY;
-@property (NS_NONATOMIC_IOSONLY, getter=getSome, readonly) BOOL some;
-@end
-
-
-@interface NSInvocation(CAT)
-@property (NS_NONATOMIC_IOSONLY, assign) id target;
-
-@property (NS_NONATOMIC_IOSONLY, assign) id dataSource;
-
-@property (NS_NONATOMIC_IOSONLY, assign) id xxxdelegateYYY;
-
-
-@property (NS_NONATOMIC_IOSONLY, strong) id MYtarget;
-
-@property (NS_NONATOMIC_IOSONLY, strong) id targetX;
-
-@property (NS_NONATOMIC_IOSONLY) int value;
-
-@property (NS_NONATOMIC_IOSONLY, getter=isContinuous) BOOL continuous;
-
-- (id) isAnObject;
-- (void)setAnObject : (id) object;
-
-@property (NS_NONATOMIC_IOSONLY, getter=isinValid, readonly) BOOL inValid;
-- (void) setInValid : (BOOL) arg;
-
-- (void) Nothing;
-@property (NS_NONATOMIC_IOSONLY, readonly) int Length;
-@property (NS_NONATOMIC_IOSONLY, readonly, strong) id object;
-+ (double) D;
-
-- (BOOL)is3bar; // watch out
-- (NSString *)get3foo; // watch out
-
-@property (NS_NONATOMIC_IOSONLY, getter=getM, readonly) BOOL m;
-@property (NS_NONATOMIC_IOSONLY, getter=getMA, readonly) BOOL MA;
-@property (NS_NONATOMIC_IOSONLY, getter=getALL, readonly) BOOL ALL;
-@property (NS_NONATOMIC_IOSONLY, getter=getMANY, readonly) BOOL MANY;
-@property (NS_NONATOMIC_IOSONLY, getter=getSome, readonly) BOOL some;
-@end
-
-DEPRECATED
-@interface I_DEP
-- (BOOL) isinValid;
-- (void) setInValid : (BOOL) arg;
-@end
-
-@interface AnotherOne
-- (BOOL) isinValid DEPRECATED;
-- (void) setInValid : (BOOL) arg;
-- (id)MYtarget;
-- (void)setMYtarget: (id)target DEPRECATED;
-- (BOOL) getM DEPRECATED;
-
-- (id)xxxdelegateYYY DEPRECATED;
-- (void)setXxxdelegateYYY:(id)delegate DEPRECATED;
-@end
-
-#define NS_AVAILABLE __attribute__((availability(macosx,introduced=10.0)))
-#define NORETURN __attribute__((noreturn))
-#define ALIGNED __attribute__((aligned(16)))
-
-@interface NSURL
-// Do not infer a property.
-@property (NS_NONATOMIC_IOSONLY, strong) NSURL *appStoreReceiptURL NS_AVAILABLE;
-- (void) setAppStoreReceiptURL : (NSURL *)object;
-
-@property (NS_NONATOMIC_IOSONLY, strong) NSURL *appStoreReceiptURLX NS_AVAILABLE;
-
-// Do not infer a property.
-@property (NS_NONATOMIC_IOSONLY, strong) NSURL *appStoreReceiptURLY ;
-- (void) setAppStoreReceiptURLY : (NSURL *)object NS_AVAILABLE;
-
-@property (NS_NONATOMIC_IOSONLY, readonly, strong) id OkToInfer NS_AVAILABLE;
-
-// Do not infer a property.
-@property (NS_NONATOMIC_IOSONLY, strong) NSURL *appStoreReceiptURLZ ;
-- (void) setAppStoreReceiptURLZ : (NSURL *)object NS_AVAILABLE;
-
-// Do not infer a property.
-- (id) t1 NORETURN NS_AVAILABLE;
-- (void) setT1 : (id) arg NS_AVAILABLE;
-
-@property (NS_NONATOMIC_IOSONLY, strong) id method1 ALIGNED NS_AVAILABLE;
-
-- (NSURL *)init;  // No Change
-+ (id)alloc;      // No Change
-
-- (BOOL)is1stClass; // Not a valid property
-@property (NS_NONATOMIC_IOSONLY, getter=isClass, readonly) BOOL class;    // This is a valid property 'class' is not a keyword in ObjC
-- (BOOL)isDouble; // Not a valid property
-
-@end
-
-@class NSMutableDictionary;
-
-@interface NSArray
-@property (NS_NONATOMIC_IOSONLY, readonly, copy) id (^expressionBlock)(id, NSArray *, NSMutableDictionary *);
-@property (NS_NONATOMIC_IOSONLY, copy) id (^MyBlock)(id, NSArray *, NSMutableDictionary *);
-@property (NS_NONATOMIC_IOSONLY, readonly) id (*expressionFuncptr)(id, NSArray *, NSMutableDictionary *);
-@property (NS_NONATOMIC_IOSONLY) id (*MyFuncptr)(id, NSArray *, NSMutableDictionary *);
-@end
diff --git a/clang/test/ARCMT/objcmt-ns-returns-inner-pointer.m b/clang/test/ARCMT/objcmt-ns-returns-inner-pointer.m
deleted file mode 100644
index 853d16dc78942..0000000000000
--- a/clang/test/ARCMT/objcmt-ns-returns-inner-pointer.m
+++ /dev/null
@@ -1,128 +0,0 @@
-// RUN: rm -rf %t
-// RUN: %clang_cc1 -objcmt-returns-innerpointer-property -objcmt-migrate-annotation -objcmt-migrate-readwrite-property -mt-migrate-directory %t %s -x objective-c -fobjc-runtime-has-weak -fobjc-arc -triple x86_64-apple-darwin11
-// RUN: c-arcmt-test -mt-migrate-directory %t | arcmt-test -verify-transformed-files %s.result
-// RUN: %clang_cc1 -triple x86_64-apple-darwin10 -fsyntax-only -x objective-c -fobjc-runtime-has-weak -fobjc-arc %s.result
-
-#ifndef NS_RETURNS_INNER_POINTER // defined in iOS 6 for sure
-#define NS_RETURNS_INNER_POINTER __attribute__((objc_returns_inner_pointer))
-#endif
-
-#define CF_IMPLICIT_BRIDGING_ENABLED _Pragma("clang arc_cf_code_audited begin")
-
-#define CF_IMPLICIT_BRIDGING_DISABLED _Pragma("clang arc_cf_code_audited end")
-
-#if __has_feature(attribute_ns_returns_retained)
-#define NS_RETURNS_RETAINED __attribute__((ns_returns_retained))
-#endif
-#if __has_feature(attribute_cf_returns_retained)
-#define CF_RETURNS_RETAINED __attribute__((cf_returns_retained))
-#endif
-#if __has_feature(attribute_ns_returns_not_retained)
-#define NS_RETURNS_NOT_RETAINED __attribute__((ns_returns_not_retained))
-#endif
-#if __has_feature(attribute_cf_returns_not_retained)
-#define CF_RETURNS_NOT_RETAINED __attribute__((cf_returns_not_retained))
-#endif
-#if __has_feature(attribute_ns_consumes_self)
-#define NS_CONSUMES_SELF __attribute__((ns_consumes_self))
-#endif
-#if __has_feature(attribute_ns_consumed)
-#define NS_CONSUMED __attribute__((ns_consumed))
-#endif
-#if __has_feature(attribute_cf_consumed)
-#define CF_CONSUMED __attribute__((cf_consumed))
-#endif
-#if __has_attribute(ns_returns_autoreleased)
-#define NS_RETURNS_AUTORELEASED __attribute__((ns_returns_autoreleased))
-#endif
-
-#define NS_AVAILABLE __attribute__((availability(macosx,introduced=10.0)))
-
-CF_IMPLICIT_BRIDGING_ENABLED
-
-typedef unsigned long CFTypeID;
-typedef unsigned long CFOptionFlags;
-typedef unsigned long CFHashCode;
-
-typedef signed long CFIndex; /*AnyObj*/
-typedef const struct __CFArray * CFArrayRef;
-typedef struct {
-    CFIndex location;
-    CFIndex length;
-} CFRange;
-
-typedef void (*CFArrayApplierFunction)(const void *value, void *context);
-
-typedef enum CFComparisonResult : CFIndex CFComparisonResult; enum CFComparisonResult : CFIndex {
-    kCFCompareLessThan = -1L,
-    kCFCompareEqualTo = 0,
-    kCFCompareGreaterThan = 1
-};
-
-
-typedef CFComparisonResult (*CFComparatorFunction)(const void *val1, const void *val2, void *context);
-
-typedef struct __CFArray * CFMutableArrayRef;
-
-typedef const struct __CFAttributedString *CFAttributedStringRef;
-typedef struct __CFAttributedString *CFMutableAttributedStringRef;
-
-typedef const struct __CFAllocator * CFAllocatorRef;
-
-typedef const struct __CFString * CFStringRef;
-typedef struct __CFString * CFMutableStringRef;
-
-typedef const struct __CFDictionary * CFDictionaryRef;
-typedef struct __CFDictionary * CFMutableDictionaryRef;
-
-typedef struct CGImage *CGImageRef;
-
-typedef struct OpaqueJSValue* JSObjectRef;
-
-typedef JSObjectRef TTJSObjectRef;
-typedef unsigned int NSUInteger;
-
-CF_IMPLICIT_BRIDGING_DISABLED
-
-@interface I
-- (void*) ReturnsInnerPointer;
-- (int*)  AlreadyReturnsInnerPointer NS_RETURNS_INNER_POINTER;
-@end
-
-@interface UIImage
-- (CGImageRef)CGImage;
-@end
-
-@interface NSData
-- (void *)bytes;
-- (void **) ptr_bytes __attribute__((availability(macosx,unavailable)));
-@end
-
-@interface NSMutableData
-- (void *)mutableBytes  __attribute__((deprecated)) __attribute__((unavailable));
-@end
-
-@interface JS
-- (JSObjectRef)JSObject; 
-- (TTJSObjectRef)JSObject1;
-- (JSObjectRef*)JSObject2;
-@end
-
-typedef void *SecTrustRef;
-
-@interface NSURLProtectionSpace
-@property (readonly) SecTrustRef serverTrust NS_AVAILABLE;
-- (void *) FOO NS_AVAILABLE;
-@property (readonly) void * mitTrust NS_AVAILABLE;
-
-@property (readonly) void * mittiTrust;
-
-@property (readonly) SecTrustRef XserverTrust;
-
-- (SecTrustRef) FOO1 NS_AVAILABLE;
-
-+ (const NSURLProtectionSpace *)ProtectionSpace;
-
-// pointer personality functions
-@property NSUInteger (*hashFunction)(const void *item, NSUInteger (*size)(const void *item));
-@end
diff --git a/clang/test/ARCMT/objcmt-ns-returns-inner-pointer.m.result b/clang/test/ARCMT/objcmt-ns-returns-inner-pointer.m.result
deleted file mode 100644
index c89c91eedce35..0000000000000
--- a/clang/test/ARCMT/objcmt-ns-returns-inner-pointer.m.result
+++ /dev/null
@@ -1,128 +0,0 @@
-// RUN: rm -rf %t
-// RUN: %clang_cc1 -objcmt-returns-innerpointer-property -objcmt-migrate-annotation -objcmt-migrate-readwrite-property -mt-migrate-directory %t %s -x objective-c -fobjc-runtime-has-weak -fobjc-arc -triple x86_64-apple-darwin11
-// RUN: c-arcmt-test -mt-migrate-directory %t | arcmt-test -verify-transformed-files %s.result
-// RUN: %clang_cc1 -triple x86_64-apple-darwin10 -fsyntax-only -x objective-c -fobjc-runtime-has-weak -fobjc-arc %s.result
-
-#ifndef NS_RETURNS_INNER_POINTER // defined in iOS 6 for sure
-#define NS_RETURNS_INNER_POINTER __attribute__((objc_returns_inner_pointer))
-#endif
-
-#define CF_IMPLICIT_BRIDGING_ENABLED _Pragma("clang arc_cf_code_audited begin")
-
-#define CF_IMPLICIT_BRIDGING_DISABLED _Pragma("clang arc_cf_code_audited end")
-
-#if __has_feature(attribute_ns_returns_retained)
-#define NS_RETURNS_RETAINED __attribute__((ns_returns_retained))
-#endif
-#if __has_feature(attribute_cf_returns_retained)
-#define CF_RETURNS_RETAINED __attribute__((cf_returns_retained))
-#endif
-#if __has_feature(attribute_ns_returns_not_retained)
-#define NS_RETURNS_NOT_RETAINED __attribute__((ns_returns_not_retained))
-#endif
-#if __has_feature(attribute_cf_returns_not_retained)
-#define CF_RETURNS_NOT_RETAINED __attribute__((cf_returns_not_retained))
-#endif
-#if __has_feature(attribute_ns_consumes_self)
-#define NS_CONSUMES_SELF __attribute__((ns_consumes_self))
-#endif
-#if __has_feature(attribute_ns_consumed)
-#define NS_CONSUMED __attribute__((ns_consumed))
-#endif
-#if __has_feature(attribute_cf_consumed)
-#define CF_CONSUMED __attribute__((cf_consumed))
-#endif
-#if __has_attribute(ns_returns_autoreleased)
-#define NS_RETURNS_AUTORELEASED __attribute__((ns_returns_autoreleased))
-#endif
-
-#define NS_AVAILABLE __attribute__((availability(macosx,introduced=10.0)))
-
-CF_IMPLICIT_BRIDGING_ENABLED
-
-typedef unsigned long CFTypeID;
-typedef unsigned long CFOptionFlags;
-typedef unsigned long CFHashCode;
-
-typedef signed long CFIndex; /*AnyObj*/
-typedef const struct __CFArray * CFArrayRef;
-typedef struct {
-    CFIndex location;
-    CFIndex length;
-} CFRange;
-
-typedef void (*CFArrayApplierFunction)(const void *value, void *context);
-
-typedef enum CFComparisonResult : CFIndex CFComparisonResult; enum CFComparisonResult : CFIndex {
-    kCFCompareLessThan = -1L,
-    kCFCompareEqualTo = 0,
-    kCFCompareGreaterThan = 1
-};
-
-
-typedef CFComparisonResult (*CFComparatorFunction)(const void *val1, const void *val2, void *context);
-
-typedef struct __CFArray * CFMutableArrayRef;
-
-typedef const struct __CFAttributedString *CFAttributedStringRef;
-typedef struct __CFAttributedString *CFMutableAttributedStringRef;
-
-typedef const struct __CFAllocator * CFAllocatorRef;
-
-typedef const struct __CFString * CFStringRef;
-typedef struct __CFString * CFMutableStringRef;
-
-typedef const struct __CFDictionary * CFDictionaryRef;
-typedef struct __CFDictionary * CFMutableDictionaryRef;
-
-typedef struct CGImage *CGImageRef;
-
-typedef struct OpaqueJSValue* JSObjectRef;
-
-typedef JSObjectRef TTJSObjectRef;
-typedef unsigned int NSUInteger;
-
-CF_IMPLICIT_BRIDGING_DISABLED
-
-@interface I
-- (void*) ReturnsInnerPointer NS_RETURNS_INNER_POINTER;
-- (int*)  AlreadyReturnsInnerPointer NS_RETURNS_INNER_POINTER;
-@end
-
-@interface UIImage
-- (CGImageRef)CGImage CF_RETURNS_NOT_RETAINED;
-@end
-
-@interface NSData
-- (void *)bytes NS_RETURNS_INNER_POINTER;
-- (void **) ptr_bytes __attribute__((availability(macosx,unavailable))) NS_RETURNS_INNER_POINTER;
-@end
-
-@interface NSMutableData
-- (void *)mutableBytes  __attribute__((deprecated)) __attribute__((unavailable)) NS_RETURNS_INNER_POINTER;
-@end
-
-@interface JS
-- (JSObjectRef)JSObject; 
-- (TTJSObjectRef)JSObject1;
-- (JSObjectRef*)JSObject2 NS_RETURNS_INNER_POINTER;
-@end
-
-typedef void *SecTrustRef;
-
-@interface NSURLProtectionSpace
-@property (readonly) SecTrustRef  NS_RETURNS_INNER_POINTER serverTrust NS_AVAILABLE;
-- (void *) FOO NS_AVAILABLE NS_RETURNS_INNER_POINTER;
-@property (readonly) void *  NS_RETURNS_INNER_POINTER mitTrust NS_AVAILABLE;
-
-@property (readonly) void *  NS_RETURNS_INNER_POINTER mittiTrust;
-
-@property (readonly) SecTrustRef  NS_RETURNS_INNER_POINTER XserverTrust;
-
-- (SecTrustRef) FOO1 NS_AVAILABLE NS_RETURNS_INNER_POINTER;
-
-+ (const NSURLProtectionSpace *)ProtectionSpace;
-
-// pointer personality functions
-@property NSUInteger (*hashFunction)(const void *item, NSUInteger (*size)(const void *item));
-@end
diff --git a/clang/test/ARCMT/objcmt-numeric-literals.m b/clang/test/ARCMT/objcmt-numeric-literals.m
deleted file mode 100644
index 7f33dc27997e1..0000000000000
--- a/clang/test/ARCMT/objcmt-numeric-literals.m
+++ /dev/null
@@ -1,502 +0,0 @@
-// RUN: rm -rf %t
-// RUN: %clang_cc1 -objcmt-migrate-literals -objcmt-migrate-subscripting -mt-migrate-directory %t %s -x objective-c++ 
-// RUN: c-arcmt-test -mt-migrate-directory %t | arcmt-test -verify-transformed-files %s.result
-// RUN: %clang_cc1 -triple x86_64-apple-darwin10 -fsyntax-only -x objective-c++ %s.result
-
-#define YES __objc_yes
-#define NO __objc_no
-
-typedef long NSInteger;
-typedef unsigned long NSUInteger;
-typedef signed char BOOL;
-#define nil ((void*) 0)
-
-@interface NSObject
-+ (id)alloc;
-@end
-
-@interface NSNumber : NSObject
-@end
-
-@interface NSNumber (NSNumberCreation)
-- (id)initWithChar:(char)value;
-- (id)initWithUnsignedChar:(unsigned char)value;
-- (id)initWithShort:(short)value;
-- (id)initWithUnsignedShort:(unsigned short)value;
-- (id)initWithInt:(int)value;
-- (id)initWithUnsignedInt:(unsigned int)value;
-- (id)initWithLong:(long)value;
-- (id)initWithUnsignedLong:(unsigned long)value;
-- (id)initWithLongLong:(long long)value;
-- (id)initWithUnsignedLongLong:(unsigned long long)value;
-- (id)initWithFloat:(float)value;
-- (id)initWithDouble:(double)value;
-- (id)initWithBool:(BOOL)value;
-- (id)initWithInteger:(NSInteger)value;
-- (id)initWithUnsignedInteger:(NSUInteger)value;
-
-+ (NSNumber *)numberWithChar:(char)value;
-+ (NSNumber *)numberWithUnsignedChar:(unsigned char)value;
-+ (NSNumber *)numberWithShort:(short)value;
-+ (NSNumber *)numberWithUnsignedShort:(unsigned short)value;
-+ (NSNumber *)numberWithInt:(int)value;
-+ (NSNumber *)numberWithUnsignedInt:(unsigned int)value;
-+ (NSNumber *)numberWithLong:(long)value;
-+ (NSNumber *)numberWithUnsignedLong:(unsigned long)value;
-+ (NSNumber *)numberWithLongLong:(long long)value;
-+ (NSNumber *)numberWithUnsignedLongLong:(unsigned long long)value;
-+ (NSNumber *)numberWithFloat:(float)value;
-+ (NSNumber *)numberWithDouble:(double)value;
-+ (NSNumber *)numberWithBool:(BOOL)value;
-+ (NSNumber *)numberWithInteger:(NSInteger)value;
-+ (NSNumber *)numberWithUnsignedInteger:(NSUInteger)value;
-@end
-
-#define VAL_INT 2
-#define VAL_UINT 2U
-#define VAL_CHAR 'a'
-
-void foo() {
-  [NSNumber numberWithChar:'a'];
-  [NSNumber numberWithChar:L'a'];
-  [NSNumber numberWithChar:2];
-  [NSNumber numberWithChar:2U];
-  [NSNumber numberWithChar:2u];
-  [NSNumber numberWithChar:2L];
-  [NSNumber numberWithChar:2l];
-  [NSNumber numberWithChar:2LL];
-  [NSNumber numberWithChar:2ll];
-  [NSNumber numberWithChar:2ul];
-  [NSNumber numberWithChar:2lu];
-  [NSNumber numberWithChar:2ull];
-  [NSNumber numberWithChar:2llu];
-  [NSNumber numberWithChar:2.0];
-  [NSNumber numberWithChar:2.0f];
-  [NSNumber numberWithChar:2.0F];
-  [NSNumber numberWithChar:2.0l];
-  [NSNumber numberWithChar:2.0L];
-  [NSNumber numberWithChar:0x2f];
-  [NSNumber numberWithChar:04];
-  [NSNumber numberWithChar:0];
-  [NSNumber numberWithChar:0.0];
-  [NSNumber numberWithChar:YES];
-  [NSNumber numberWithChar:NO];
-  [NSNumber numberWithChar:true];
-  [NSNumber numberWithChar:false];
-  [NSNumber numberWithChar:VAL_INT];
-  [NSNumber numberWithChar:VAL_UINT];
-  [NSNumber numberWithChar:VAL_CHAR];
-
-  [NSNumber numberWithUnsignedChar:'a'];
-  [NSNumber numberWithUnsignedChar:L'a'];
-  [NSNumber numberWithUnsignedChar:2];
-  [NSNumber numberWithUnsignedChar:2U];
-  [NSNumber numberWithUnsignedChar:2u];
-  [NSNumber numberWithUnsignedChar:2L];
-  [NSNumber numberWithUnsignedChar:2l];
-  [NSNumber numberWithUnsignedChar:2LL];
-  [NSNumber numberWithUnsignedChar:2ll];
-  [NSNumber numberWithUnsignedChar:2ul];
-  [NSNumber numberWithUnsignedChar:2lu];
-  [NSNumber numberWithUnsignedChar:2ull];
-  [NSNumber numberWithUnsignedChar:2llu];
-  [NSNumber numberWithUnsignedChar:2.0];
-  [NSNumber numberWithUnsignedChar:2.0f];
-  [NSNumber numberWithUnsignedChar:2.0F];
-  [NSNumber numberWithUnsignedChar:2.0l];
-  [NSNumber numberWithUnsignedChar:2.0L];
-  [NSNumber numberWithUnsignedChar:0x2f];
-  [NSNumber numberWithUnsignedChar:04];
-  [NSNumber numberWithUnsignedChar:0];
-  [NSNumber numberWithUnsignedChar:0.0];
-  [NSNumber numberWithUnsignedChar:YES];
-  [NSNumber numberWithUnsignedChar:NO];
-  [NSNumber numberWithUnsignedChar:true];
-  [NSNumber numberWithUnsignedChar:false];
-  [NSNumber numberWithUnsignedChar:VAL_INT];
-  [NSNumber numberWithUnsignedChar:VAL_UINT];
-  [NSNumber numberWithUnsignedChar:VAL_CHAR];
-
-  [NSNumber numberWithShort:'a'];
-  [NSNumber numberWithShort:L'a'];
-  [NSNumber numberWithShort:2];
-  [NSNumber numberWithShort:2U];
-  [NSNumber numberWithShort:2u];
-  [NSNumber numberWithShort:2L];
-  [NSNumber numberWithShort:2l];
-  [NSNumber numberWithShort:2LL];
-  [NSNumber numberWithShort:2ll];
-  [NSNumber numberWithShort:2ul];
-  [NSNumber numberWithShort:2lu];
-  [NSNumber numberWithShort:2ull];
-  [NSNumber numberWithShort:2llu];
-  [NSNumber numberWithShort:2.0];
-  [NSNumber numberWithShort:2.0f];
-  [NSNumber numberWithShort:2.0F];
-  [NSNumber numberWithShort:2.0l];
-  [NSNumber numberWithShort:2.0L];
-  [NSNumber numberWithShort:0x2f];
-  [NSNumber numberWithShort:04];
-  [NSNumber numberWithShort:0];
-  [NSNumber numberWithShort:0.0];
-  [NSNumber numberWithShort:YES];
-  [NSNumber numberWithShort:NO];
-  [NSNumber numberWithShort:true];
-  [NSNumber numberWithShort:false];
-  [NSNumber numberWithShort:VAL_INT];
-  [NSNumber numberWithShort:VAL_UINT];
-
-  [NSNumber numberWithUnsignedShort:'a'];
-  [NSNumber numberWithUnsignedShort:L'a'];
-  [NSNumber numberWithUnsignedShort:2];
-  [NSNumber numberWithUnsignedShort:2U];
-  [NSNumber numberWithUnsignedShort:2u];
-  [NSNumber numberWithUnsignedShort:2L];
-  [NSNumber numberWithUnsignedShort:2l];
-  [NSNumber numberWithUnsignedShort:2LL];
-  [NSNumber numberWithUnsignedShort:2ll];
-  [NSNumber numberWithUnsignedShort:2ul];
-  [NSNumber numberWithUnsignedShort:2lu];
-  [NSNumber numberWithUnsignedShort:2ull];
-  [NSNumber numberWithUnsignedShort:2llu];
-  [NSNumber numberWithUnsignedShort:2.0];
-  [NSNumber numberWithUnsignedShort:2.0f];
-  [NSNumber numberWithUnsignedShort:2.0F];
-  [NSNumber numberWithUnsignedShort:2.0l];
-  [NSNumber numberWithUnsignedShort:2.0L];
-  [NSNumber numberWithUnsignedShort:0x2f];
-  [NSNumber numberWithUnsignedShort:04];
-  [NSNumber numberWithUnsignedShort:0];
-  [NSNumber numberWithUnsignedShort:0.0];
-  [NSNumber numberWithUnsignedShort:YES];
-  [NSNumber numberWithUnsignedShort:NO];
-  [NSNumber numberWithUnsignedShort:true];
-  [NSNumber numberWithUnsignedShort:false];
-  [NSNumber numberWithUnsignedShort:VAL_INT];
-  [NSNumber numberWithUnsignedShort:VAL_UINT];
-
-  [NSNumber numberWithInt:'a'];
-  [NSNumber numberWithInt:L'a'];
-  [NSNumber numberWithInt:2];
-  [NSNumber numberWithInt:2U];
-  [NSNumber numberWithInt:2u];
-  [NSNumber numberWithInt:2L];
-  [NSNumber numberWithInt:2l];
-  [NSNumber numberWithInt:2LL];
-  [NSNumber numberWithInt:2ll];
-  [NSNumber numberWithInt:2ul];
-  [NSNumber numberWithInt:2lu];
-  [NSNumber numberWithInt:2ull];
-  [NSNumber numberWithInt:2llu];
-  [NSNumber numberWithInt:2.0];
-  [NSNumber numberWithInt:2.0f];
-  [NSNumber numberWithInt:2.0F];
-  [NSNumber numberWithInt:2.0l];
-  [NSNumber numberWithInt:2.0L];
-  [NSNumber numberWithInt:0x2f];
-  [NSNumber numberWithInt:04];
-  [NSNumber numberWithInt:0];
-  [NSNumber numberWithInt:0.0];
-  [NSNumber numberWithInt:YES];
-  [NSNumber numberWithInt:NO];
-  [NSNumber numberWithInt:true];
-  [NSNumber numberWithInt:false];
-  [NSNumber numberWithInt:VAL_INT];
-  [NSNumber numberWithInt:VAL_UINT];
-
-  (void)[[NSNumber alloc] initWithInt:2];
-  (void)[[NSNumber alloc] initWithInt:2U];
-
-  [NSNumber numberWithInt:+2];
-  [NSNumber numberWithInt:-2];
-
-  [NSNumber numberWithUnsignedInt:'a'];
-  [NSNumber numberWithUnsignedInt:L'a'];
-  [NSNumber numberWithUnsignedInt:2];
-  [NSNumber numberWithUnsignedInt:2U];
-  [NSNumber numberWithUnsignedInt:2u];
-  [NSNumber numberWithUnsignedInt:2L];
-  [NSNumber numberWithUnsignedInt:2l];
-  [NSNumber numberWithUnsignedInt:2LL];
-  [NSNumber numberWithUnsignedInt:2ll];
-  [NSNumber numberWithUnsignedInt:2ul];
-  [NSNumber numberWithUnsignedInt:2lu];
-  [NSNumber numberWithUnsignedInt:2ull];
-  [NSNumber numberWithUnsignedInt:2llu];
-  [NSNumber numberWithUnsignedInt:2.0];
-  [NSNumber numberWithUnsignedInt:2.0f];
-  [NSNumber numberWithUnsignedInt:2.0F];
-  [NSNumber numberWithUnsignedInt:2.0l];
-  [NSNumber numberWithUnsignedInt:2.0L];
-  [NSNumber numberWithUnsignedInt:0x2f];
-  [NSNumber numberWithUnsignedInt:04];
-  [NSNumber numberWithUnsignedInt:0];
-  [NSNumber numberWithUnsignedInt:0.0];
-  [NSNumber numberWithUnsignedInt:YES];
-  [NSNumber numberWithUnsignedInt:NO];
-  [NSNumber numberWithUnsignedInt:true];
-  [NSNumber numberWithUnsignedInt:false];
-  [NSNumber numberWithUnsignedInt:VAL_INT];
-  [NSNumber numberWithUnsignedInt:VAL_UINT];
-
-  [NSNumber numberWithLong:'a'];
-  [NSNumber numberWithLong:L'a'];
-  [NSNumber numberWithLong:2];
-  [NSNumber numberWithLong:2U];
-  [NSNumber numberWithLong:2u];
-  [NSNumber numberWithLong:2L];
-  [NSNumber numberWithLong:2l];
-  [NSNumber numberWithLong:2LL];
-  [NSNumber numberWithLong:2ll];
-  [NSNumber numberWithLong:2ul];
-  [NSNumber numberWithLong:2lu];
-  [NSNumber numberWithLong:2ull];
-  [NSNumber numberWithLong:2llu];
-  [NSNumber numberWithLong:2.0];
-  [NSNumber numberWithLong:2.0f];
-  [NSNumber numberWithLong:2.0F];
-  [NSNumber numberWithLong:2.0l];
-  [NSNumber numberWithLong:2.0L];
-  [NSNumber numberWithLong:0x2f];
-  [NSNumber numberWithLong:04];
-  [NSNumber numberWithLong:0];
-  [NSNumber numberWithLong:0.0];
-  [NSNumber numberWithLong:YES];
-  [NSNumber numberWithLong:NO];
-  [NSNumber numberWithLong:true];
-  [NSNumber numberWithLong:false];
-  [NSNumber numberWithLong:VAL_INT];
-  [NSNumber numberWithLong:VAL_UINT];
-
-  [NSNumber numberWithUnsignedLong:'a'];
-  [NSNumber numberWithUnsignedLong:L'a'];
-  [NSNumber numberWithUnsignedLong:2];
-  [NSNumber numberWithUnsignedLong:2U];
-  [NSNumber numberWithUnsignedLong:2u];
-  [NSNumber numberWithUnsignedLong:2L];
-  [NSNumber numberWithUnsignedLong:2l];
-  [NSNumber numberWithUnsignedLong:2LL];
-  [NSNumber numberWithUnsignedLong:2ll];
-  [NSNumber numberWithUnsignedLong:2ul];
-  [NSNumber numberWithUnsignedLong:2lu];
-  [NSNumber numberWithUnsignedLong:2ull];
-  [NSNumber numberWithUnsignedLong:2llu];
-  [NSNumber numberWithUnsignedLong:2.0];
-  [NSNumber numberWithUnsignedLong:2.0f];
-  [NSNumber numberWithUnsignedLong:2.0F];
-  [NSNumber numberWithUnsignedLong:2.0l];
-  [NSNumber numberWithUnsignedLong:2.0L];
-  [NSNumber numberWithUnsignedLong:0x2f];
-  [NSNumber numberWithUnsignedLong:04];
-  [NSNumber numberWithUnsignedLong:0];
-  [NSNumber numberWithUnsignedLong:0.0];
-  [NSNumber numberWithUnsignedLong:YES];
-  [NSNumber numberWithUnsignedLong:NO];
-  [NSNumber numberWithUnsignedLong:true];
-  [NSNumber numberWithUnsignedLong:false];
-  [NSNumber numberWithUnsignedLong:VAL_INT];
-  [NSNumber numberWithUnsignedLong:VAL_UINT];
-
-  [NSNumber numberWithLongLong:'a'];
-  [NSNumber numberWithLongLong:L'a'];
-  [NSNumber numberWithLongLong:2];
-  [NSNumber numberWithLongLong:2U];
-  [NSNumber numberWithLongLong:2u];
-  [NSNumber numberWithLongLong:2L];
-  [NSNumber numberWithLongLong:2l];
-  [NSNumber numberWithLongLong:2LL];
-  [NSNumber numberWithLongLong:2ll];
-  [NSNumber numberWithLongLong:2ul];
-  [NSNumber numberWithLongLong:2lu];
-  [NSNumber numberWithLongLong:2ull];
-  [NSNumber numberWithLongLong:2llu];
-  [NSNumber numberWithLongLong:2.0];
-  [NSNumber numberWithLongLong:2.0f];
-  [NSNumber numberWithLongLong:2.0F];
-  [NSNumber numberWithLongLong:2.0l];
-  [NSNumber numberWithLongLong:2.0L];
-  [NSNumber numberWithLongLong:0x2f];
-  [NSNumber numberWithLongLong:04];
-  [NSNumber numberWithLongLong:0];
-  [NSNumber numberWithLongLong:0.0];
-  [NSNumber numberWithLongLong:YES];
-  [NSNumber numberWithLongLong:NO];
-  [NSNumber numberWithLongLong:true];
-  [NSNumber numberWithLongLong:false];
-  [NSNumber numberWithLongLong:VAL_INT];
-  [NSNumber numberWithLongLong:VAL_UINT];
-
-  [NSNumber numberWithUnsignedLongLong:'a'];
-  [NSNumber numberWithUnsignedLongLong:L'a'];
-  [NSNumber numberWithUnsignedLongLong:2];
-  [NSNumber numberWithUnsignedLongLong:2U];
-  [NSNumber numberWithUnsignedLongLong:2u];
-  [NSNumber numberWithUnsignedLongLong:2L];
-  [NSNumber numberWithUnsignedLongLong:2l];
-  [NSNumber numberWithUnsignedLongLong:2LL];
-  [NSNumber numberWithUnsignedLongLong:2ll];
-  [NSNumber numberWithUnsignedLongLong:2ul];
-  [NSNumber numberWithUnsignedLongLong:2lu];
-  [NSNumber numberWithUnsignedLongLong:2ull];
-  [NSNumber numberWithUnsignedLongLong:2llu];
-  [NSNumber numberWithUnsignedLongLong:2.0];
-  [NSNumber numberWithUnsignedLongLong:2.0f];
-  [NSNumber numberWithUnsignedLongLong:2.0F];
-  [NSNumber numberWithUnsignedLongLong:2.0l];
-  [NSNumber numberWithUnsignedLongLong:2.0L];
-  [NSNumber numberWithUnsignedLongLong:0x2f];
-  [NSNumber numberWithUnsignedLongLong:04];
-  [NSNumber numberWithUnsignedLongLong:0];
-  [NSNumber numberWithUnsignedLongLong:0.0];
-  [NSNumber numberWithUnsignedLongLong:YES];
-  [NSNumber numberWithUnsignedLongLong:NO];
-  [NSNumber numberWithUnsignedLongLong:true];
-  [NSNumber numberWithUnsignedLongLong:false];
-  [NSNumber numberWithUnsignedLongLong:VAL_INT];
-  [NSNumber numberWithUnsignedLongLong:VAL_UINT];
-
-  [NSNumber numberWithFloat:'a'];
-  [NSNumber numberWithFloat:L'a'];
-  [NSNumber numberWithFloat:2];
-  [NSNumber numberWithFloat:2U];
-  [NSNumber numberWithFloat:2u];
-  [NSNumber numberWithFloat:2L];
-  [NSNumber numberWithFloat:2l];
-  [NSNumber numberWithFloat:2LL];
-  [NSNumber numberWithFloat:2ll];
-  [NSNumber numberWithFloat:2ul];
-  [NSNumber numberWithFloat:2lu];
-  [NSNumber numberWithFloat:2ull];
-  [NSNumber numberWithFloat:2llu];
-  [NSNumber numberWithFloat:2.0];
-  [NSNumber numberWithFloat:2.0f];
-  [NSNumber numberWithFloat:2.0F];
-  [NSNumber numberWithFloat:2.0l];
-  [NSNumber numberWithFloat:2.0L];
-  [NSNumber numberWithFloat:0x2f];
-  [NSNumber numberWithFloat:04];
-  [NSNumber numberWithFloat:0];
-  [NSNumber numberWithFloat:0.0];
-  [NSNumber numberWithFloat:YES];
-  [NSNumber numberWithFloat:NO];
-  [NSNumber numberWithFloat:true];
-  [NSNumber numberWithFloat:false];
-  [NSNumber numberWithFloat:VAL_INT];
-  [NSNumber numberWithFloat:VAL_UINT];
-
-  [NSNumber numberWithDouble:'a'];
-  [NSNumber numberWithDouble:L'a'];
-  [NSNumber numberWithDouble:2];
-  [NSNumber numberWithDouble:2U];
-  [NSNumber numberWithDouble:2u];
-  [NSNumber numberWithDouble:2L];
-  [NSNumber numberWithDouble:2l];
-  [NSNumber numberWithDouble:2LL];
-  [NSNumber numberWithDouble:2ll];
-  [NSNumber numberWithDouble:2ul];
-  [NSNumber numberWithDouble:2lu];
-  [NSNumber numberWithDouble:2ull];
-  [NSNumber numberWithDouble:2llu];
-  [NSNumber numberWithDouble:2.0];
-  [NSNumber numberWithDouble:2.0f];
-  [NSNumber numberWithDouble:2.0F];
-  [NSNumber numberWithDouble:2.0l];
-  [NSNumber numberWithDouble:2.0L];
-  [NSNumber numberWithDouble:0x2f];
-  [NSNumber numberWithDouble:04];
-  [NSNumber numberWithDouble:0];
-  [NSNumber numberWithDouble:0.0];
-  [NSNumber numberWithDouble:YES];
-  [NSNumber numberWithDouble:NO];
-  [NSNumber numberWithDouble:true];
-  [NSNumber numberWithDouble:false];
-  [NSNumber numberWithDouble:VAL_INT];
-  [NSNumber numberWithDouble:VAL_UINT];
-
-  [NSNumber numberWithBool:'a'];
-  [NSNumber numberWithBool:L'a'];
-  [NSNumber numberWithBool:2];
-  [NSNumber numberWithBool:2U];
-  [NSNumber numberWithBool:2u];
-  [NSNumber numberWithBool:2L];
-  [NSNumber numberWithBool:2l];
-  [NSNumber numberWithBool:2LL];
-  [NSNumber numberWithBool:2ll];
-  [NSNumber numberWithBool:2ul];
-  [NSNumber numberWithBool:2lu];
-  [NSNumber numberWithBool:2ull];
-  [NSNumber numberWithBool:2llu];
-  [NSNumber numberWithBool:2.0];
-  [NSNumber numberWithBool:2.0f];
-  [NSNumber numberWithBool:2.0F];
-  [NSNumber numberWithBool:2.0l];
-  [NSNumber numberWithBool:2.0L];
-  [NSNumber numberWithBool:0x2f];
-  [NSNumber numberWithBool:04];
-  [NSNumber numberWithBool:0];
-  [NSNumber numberWithBool:0.0];
-  [NSNumber numberWithBool:YES];
-  [NSNumber numberWithBool:NO];
-  [NSNumber numberWithBool:true];
-  [NSNumber numberWithBool:false];
-  [NSNumber numberWithBool:VAL_INT];
-  [NSNumber numberWithBool:VAL_UINT];
-
-  [NSNumber numberWithInteger:'a'];
-  [NSNumber numberWithInteger:L'a'];
-  [NSNumber numberWithInteger:2];
-  [NSNumber numberWithInteger:2U];
-  [NSNumber numberWithInteger:2u];
-  [NSNumber numberWithInteger:2L];
-  [NSNumber numberWithInteger:2l];
-  [NSNumber numberWithInteger:2LL];
-  [NSNumber numberWithInteger:2ll];
-  [NSNumber numberWithInteger:2ul];
-  [NSNumber numberWithInteger:2lu];
-  [NSNumber numberWithInteger:2ull];
-  [NSNumber numberWithInteger:2llu];
-  [NSNumber numberWithInteger:2.0];
-  [NSNumber numberWithInteger:2.0f];
-  [NSNumber numberWithInteger:2.0F];
-  [NSNumber numberWithInteger:2.0l];
-  [NSNumber numberWithInteger:2.0L];
-  [NSNumber numberWithInteger:0x2f];
-  [NSNumber numberWithInteger:04];
-  [NSNumber numberWithInteger:0];
-  [NSNumber numberWithInteger:0.0];
-  [NSNumber numberWithInteger:YES];
-  [NSNumber numberWithInteger:NO];
-  [NSNumber numberWithInteger:true];
-  [NSNumber numberWithInteger:false];
-  [NSNumber numberWithInteger:VAL_INT];
-  [NSNumber numberWithInteger:VAL_UINT];
-
-  [NSNumber numberWithUnsignedInteger:'a'];
-  [NSNumber numberWithUnsignedInteger:L'a'];
-  [NSNumber numberWithUnsignedInteger:2];
-  [NSNumber numberWithUnsignedInteger:2U];
-  [NSNumber numberWithUnsignedInteger:2u];
-  [NSNumber numberWithUnsignedInteger:2L];
-  [NSNumber numberWithUnsignedInteger:2l];
-  [NSNumber numberWithUnsignedInteger:2LL];
-  [NSNumber numberWithUnsignedInteger:2ll];
-  [NSNumber numberWithUnsignedInteger:2ul];
-  [NSNumber numberWithUnsignedInteger:2lu];
-  [NSNumber numberWithUnsignedInteger:2ull];
-  [NSNumber numberWithUnsignedInteger:2llu];
-  [NSNumber numberWithUnsignedInteger:2.0];
-  [NSNumber numberWithUnsignedInteger:2.0f];
-  [NSNumber numberWithUnsignedInteger:2.0F];
-  [NSNumber numberWithUnsignedInteger:2.0l];
-  [NSNumber numberWithUnsignedInteger:2.0L];
-  [NSNumber numberWithUnsignedInteger:0x2f];
-  [NSNumber numberWithUnsignedInteger:04];
-  [NSNumber numberWithUnsignedInteger:0];
-  [NSNumber numberWithUnsignedInteger:0.0];
-  [NSNumber numberWithUnsignedInteger:YES];
-  [NSNumber numberWithUnsignedInteger:NO];
-  [NSNumber numberWithUnsignedInteger:true];
-  [NSNumber numberWithUnsignedInteger:false];
-  [NSNumber numberWithUnsignedInteger:VAL_INT];
-  [NSNumber numberWithUnsignedInteger:VAL_UINT];
-}
diff --git a/clang/test/ARCMT/objcmt-numeric-literals.m.result b/clang/test/ARCMT/objcmt-numeric-literals.m.result
deleted file mode 100644
index bb7b515566d09..0000000000000
--- a/clang/test/ARCMT/objcmt-numeric-literals.m.result
+++ /dev/null
@@ -1,502 +0,0 @@
-// RUN: rm -rf %t
-// RUN: %clang_cc1 -objcmt-migrate-literals -objcmt-migrate-subscripting -mt-migrate-directory %t %s -x objective-c++ 
-// RUN: c-arcmt-test -mt-migrate-directory %t | arcmt-test -verify-transformed-files %s.result
-// RUN: %clang_cc1 -triple x86_64-apple-darwin10 -fsyntax-only -x objective-c++ %s.result
-
-#define YES __objc_yes
-#define NO __objc_no
-
-typedef long NSInteger;
-typedef unsigned long NSUInteger;
-typedef signed char BOOL;
-#define nil ((void*) 0)
-
-@interface NSObject
-+ (id)alloc;
-@end
-
-@interface NSNumber : NSObject
-@end
-
-@interface NSNumber (NSNumberCreation)
-- (id)initWithChar:(char)value;
-- (id)initWithUnsignedChar:(unsigned char)value;
-- (id)initWithShort:(short)value;
-- (id)initWithUnsignedShort:(unsigned short)value;
-- (id)initWithInt:(int)value;
-- (id)initWithUnsignedInt:(unsigned int)value;
-- (id)initWithLong:(long)value;
-- (id)initWithUnsignedLong:(unsigned long)value;
-- (id)initWithLongLong:(long long)value;
-- (id)initWithUnsignedLongLong:(unsigned long long)value;
-- (id)initWithFloat:(float)value;
-- (id)initWithDouble:(double)value;
-- (id)initWithBool:(BOOL)value;
-- (id)initWithInteger:(NSInteger)value;
-- (id)initWithUnsignedInteger:(NSUInteger)value;
-
-+ (NSNumber *)numberWithChar:(char)value;
-+ (NSNumber *)numberWithUnsignedChar:(unsigned char)value;
-+ (NSNumber *)numberWithShort:(short)value;
-+ (NSNumber *)numberWithUnsignedShort:(unsigned short)value;
-+ (NSNumber *)numberWithInt:(int)value;
-+ (NSNumber *)numberWithUnsignedInt:(unsigned int)value;
-+ (NSNumber *)numberWithLong:(long)value;
-+ (NSNumber *)numberWithUnsignedLong:(unsigned long)value;
-+ (NSNumber *)numberWithLongLong:(long long)value;
-+ (NSNumber *)numberWithUnsignedLongLong:(unsigned long long)value;
-+ (NSNumber *)numberWithFloat:(float)value;
-+ (NSNumber *)numberWithDouble:(double)value;
-+ (NSNumber *)numberWithBool:(BOOL)value;
-+ (NSNumber *)numberWithInteger:(NSInteger)value;
-+ (NSNumber *)numberWithUnsignedInteger:(NSUInteger)value;
-@end
-
-#define VAL_INT 2
-#define VAL_UINT 2U
-#define VAL_CHAR 'a'
-
-void foo() {
-  @'a';
-  [NSNumber numberWithChar:L'a'];
-  [NSNumber numberWithChar:2];
-  [NSNumber numberWithChar:2U];
-  [NSNumber numberWithChar:2u];
-  [NSNumber numberWithChar:2L];
-  [NSNumber numberWithChar:2l];
-  [NSNumber numberWithChar:2LL];
-  [NSNumber numberWithChar:2ll];
-  [NSNumber numberWithChar:2ul];
-  [NSNumber numberWithChar:2lu];
-  [NSNumber numberWithChar:2ull];
-  [NSNumber numberWithChar:2llu];
-  [NSNumber numberWithChar:2.0];
-  [NSNumber numberWithChar:2.0f];
-  [NSNumber numberWithChar:2.0F];
-  [NSNumber numberWithChar:2.0l];
-  [NSNumber numberWithChar:2.0L];
-  [NSNumber numberWithChar:0x2f];
-  [NSNumber numberWithChar:04];
-  [NSNumber numberWithChar:0];
-  [NSNumber numberWithChar:0.0];
-  [NSNumber numberWithChar:YES];
-  [NSNumber numberWithChar:NO];
-  [NSNumber numberWithChar:true];
-  [NSNumber numberWithChar:false];
-  [NSNumber numberWithChar:VAL_INT];
-  [NSNumber numberWithChar:VAL_UINT];
-  @VAL_CHAR;
-
-  [NSNumber numberWithUnsignedChar:'a'];
-  [NSNumber numberWithUnsignedChar:L'a'];
-  [NSNumber numberWithUnsignedChar:2];
-  [NSNumber numberWithUnsignedChar:2U];
-  [NSNumber numberWithUnsignedChar:2u];
-  [NSNumber numberWithUnsignedChar:2L];
-  [NSNumber numberWithUnsignedChar:2l];
-  [NSNumber numberWithUnsignedChar:2LL];
-  [NSNumber numberWithUnsignedChar:2ll];
-  [NSNumber numberWithUnsignedChar:2ul];
-  [NSNumber numberWithUnsignedChar:2lu];
-  [NSNumber numberWithUnsignedChar:2ull];
-  [NSNumber numberWithUnsignedChar:2llu];
-  [NSNumber numberWithUnsignedChar:2.0];
-  [NSNumber numberWithUnsignedChar:2.0f];
-  [NSNumber numberWithUnsignedChar:2.0F];
-  [NSNumber numberWithUnsignedChar:2.0l];
-  [NSNumber numberWithUnsignedChar:2.0L];
-  [NSNumber numberWithUnsignedChar:0x2f];
-  [NSNumber numberWithUnsignedChar:04];
-  [NSNumber numberWithUnsignedChar:0];
-  [NSNumber numberWithUnsignedChar:0.0];
-  [NSNumber numberWithUnsignedChar:YES];
-  [NSNumber numberWithUnsignedChar:NO];
-  [NSNumber numberWithUnsignedChar:true];
-  [NSNumber numberWithUnsignedChar:false];
-  [NSNumber numberWithUnsignedChar:VAL_INT];
-  [NSNumber numberWithUnsignedChar:VAL_UINT];
-  [NSNumber numberWithUnsignedChar:VAL_CHAR];
-
-  [NSNumber numberWithShort:'a'];
-  [NSNumber numberWithShort:L'a'];
-  [NSNumber numberWithShort:2];
-  [NSNumber numberWithShort:2U];
-  [NSNumber numberWithShort:2u];
-  [NSNumber numberWithShort:2L];
-  [NSNumber numberWithShort:2l];
-  [NSNumber numberWithShort:2LL];
-  [NSNumber numberWithShort:2ll];
-  [NSNumber numberWithShort:2ul];
-  [NSNumber numberWithShort:2lu];
-  [NSNumber numberWithShort:2ull];
-  [NSNumber numberWithShort:2llu];
-  [NSNumber numberWithShort:2.0];
-  [NSNumber numberWithShort:2.0f];
-  [NSNumber numberWithShort:2.0F];
-  [NSNumber numberWithShort:2.0l];
-  [NSNumber numberWithShort:2.0L];
-  [NSNumber numberWithShort:0x2f];
-  [NSNumber numberWithShort:04];
-  [NSNumber numberWithShort:0];
-  [NSNumber numberWithShort:0.0];
-  [NSNumber numberWithShort:YES];
-  [NSNumber numberWithShort:NO];
-  [NSNumber numberWithShort:true];
-  [NSNumber numberWithShort:false];
-  [NSNumber numberWithShort:VAL_INT];
-  [NSNumber numberWithShort:VAL_UINT];
-
-  [NSNumber numberWithUnsignedShort:'a'];
-  [NSNumber numberWithUnsignedShort:L'a'];
-  [NSNumber numberWithUnsignedShort:2];
-  [NSNumber numberWithUnsignedShort:2U];
-  [NSNumber numberWithUnsignedShort:2u];
-  [NSNumber numberWithUnsignedShort:2L];
-  [NSNumber numberWithUnsignedShort:2l];
-  [NSNumber numberWithUnsignedShort:2LL];
-  [NSNumber numberWithUnsignedShort:2ll];
-  [NSNumber numberWithUnsignedShort:2ul];
-  [NSNumber numberWithUnsignedShort:2lu];
-  [NSNumber numberWithUnsignedShort:2ull];
-  [NSNumber numberWithUnsignedShort:2llu];
-  [NSNumber numberWithUnsignedShort:2.0];
-  [NSNumber numberWithUnsignedShort:2.0f];
-  [NSNumber numberWithUnsignedShort:2.0F];
-  [NSNumber numberWithUnsignedShort:2.0l];
-  [NSNumber numberWithUnsignedShort:2.0L];
-  [NSNumber numberWithUnsignedShort:0x2f];
-  [NSNumber numberWithUnsignedShort:04];
-  [NSNumber numberWithUnsignedShort:0];
-  [NSNumber numberWithUnsignedShort:0.0];
-  [NSNumber numberWithUnsignedShort:YES];
-  [NSNumber numberWithUnsignedShort:NO];
-  [NSNumber numberWithUnsignedShort:true];
-  [NSNumber numberWithUnsignedShort:false];
-  [NSNumber numberWithUnsignedShort:VAL_INT];
-  [NSNumber numberWithUnsignedShort:VAL_UINT];
-
-  [NSNumber numberWithInt:'a'];
-  [NSNumber numberWithInt:L'a'];
-  @2;
-  @2;
-  @2;
-  @2;
-  @2;
-  @2;
-  @2;
-  @2;
-  @2;
-  @2;
-  @2;
-  [NSNumber numberWithInt:2.0];
-  [NSNumber numberWithInt:2.0f];
-  [NSNumber numberWithInt:2.0F];
-  [NSNumber numberWithInt:2.0l];
-  [NSNumber numberWithInt:2.0L];
-  @0x2f;
-  @04;
-  @0;
-  [NSNumber numberWithInt:0.0];
-  [NSNumber numberWithInt:YES];
-  [NSNumber numberWithInt:NO];
-  [NSNumber numberWithInt:true];
-  [NSNumber numberWithInt:false];
-  @VAL_INT;
-  [NSNumber numberWithInt:VAL_UINT];
-
-  (void)[[NSNumber alloc] initWithInt:2];
-  (void)[[NSNumber alloc] initWithInt:2U];
-
-  @+2;
-  @-2;
-
-  [NSNumber numberWithUnsignedInt:'a'];
-  [NSNumber numberWithUnsignedInt:L'a'];
-  @2U;
-  @2U;
-  @2u;
-  @2U;
-  @2u;
-  @2U;
-  @2u;
-  @2u;
-  @2u;
-  @2u;
-  @2u;
-  [NSNumber numberWithUnsignedInt:2.0];
-  [NSNumber numberWithUnsignedInt:2.0f];
-  [NSNumber numberWithUnsignedInt:2.0F];
-  [NSNumber numberWithUnsignedInt:2.0l];
-  [NSNumber numberWithUnsignedInt:2.0L];
-  @0x2fU;
-  @04U;
-  @0U;
-  [NSNumber numberWithUnsignedInt:0.0];
-  [NSNumber numberWithUnsignedInt:YES];
-  [NSNumber numberWithUnsignedInt:NO];
-  [NSNumber numberWithUnsignedInt:true];
-  [NSNumber numberWithUnsignedInt:false];
-  [NSNumber numberWithUnsignedInt:VAL_INT];
-  @VAL_UINT;
-
-  [NSNumber numberWithLong:'a'];
-  [NSNumber numberWithLong:L'a'];
-  @2L;
-  @2L;
-  @2l;
-  @2L;
-  @2l;
-  @2L;
-  @2l;
-  @2l;
-  @2l;
-  @2l;
-  @2l;
-  [NSNumber numberWithLong:2.0];
-  [NSNumber numberWithLong:2.0f];
-  [NSNumber numberWithLong:2.0F];
-  [NSNumber numberWithLong:2.0l];
-  [NSNumber numberWithLong:2.0L];
-  @0x2fL;
-  @04L;
-  @0L;
-  [NSNumber numberWithLong:0.0];
-  [NSNumber numberWithLong:YES];
-  [NSNumber numberWithLong:NO];
-  [NSNumber numberWithLong:true];
-  [NSNumber numberWithLong:false];
-  [NSNumber numberWithLong:VAL_INT];
-  [NSNumber numberWithLong:VAL_UINT];
-
-  [NSNumber numberWithUnsignedLong:'a'];
-  [NSNumber numberWithUnsignedLong:L'a'];
-  @2UL;
-  @2UL;
-  @2ul;
-  @2UL;
-  @2ul;
-  @2UL;
-  @2ul;
-  @2ul;
-  @2lu;
-  @2ul;
-  @2ul;
-  [NSNumber numberWithUnsignedLong:2.0];
-  [NSNumber numberWithUnsignedLong:2.0f];
-  [NSNumber numberWithUnsignedLong:2.0F];
-  [NSNumber numberWithUnsignedLong:2.0l];
-  [NSNumber numberWithUnsignedLong:2.0L];
-  @0x2fUL;
-  @04UL;
-  @0UL;
-  [NSNumber numberWithUnsignedLong:0.0];
-  [NSNumber numberWithUnsignedLong:YES];
-  [NSNumber numberWithUnsignedLong:NO];
-  [NSNumber numberWithUnsignedLong:true];
-  [NSNumber numberWithUnsignedLong:false];
-  [NSNumber numberWithUnsignedLong:VAL_INT];
-  [NSNumber numberWithUnsignedLong:VAL_UINT];
-
-  [NSNumber numberWithLongLong:'a'];
-  [NSNumber numberWithLongLong:L'a'];
-  @2LL;
-  @2LL;
-  @2ll;
-  @2LL;
-  @2ll;
-  @2LL;
-  @2ll;
-  @2ll;
-  @2ll;
-  @2ll;
-  @2ll;
-  [NSNumber numberWithLongLong:2.0];
-  [NSNumber numberWithLongLong:2.0f];
-  [NSNumber numberWithLongLong:2.0F];
-  [NSNumber numberWithLongLong:2.0l];
-  [NSNumber numberWithLongLong:2.0L];
-  @0x2fLL;
-  @04LL;
-  @0LL;
-  [NSNumber numberWithLongLong:0.0];
-  [NSNumber numberWithLongLong:YES];
-  [NSNumber numberWithLongLong:NO];
-  [NSNumber numberWithLongLong:true];
-  [NSNumber numberWithLongLong:false];
-  [NSNumber numberWithLongLong:VAL_INT];
-  [NSNumber numberWithLongLong:VAL_UINT];
-
-  [NSNumber numberWithUnsignedLongLong:'a'];
-  [NSNumber numberWithUnsignedLongLong:L'a'];
-  @2ULL;
-  @2ULL;
-  @2ull;
-  @2ULL;
-  @2ull;
-  @2ULL;
-  @2ull;
-  @2ull;
-  @2ull;
-  @2ull;
-  @2llu;
-  [NSNumber numberWithUnsignedLongLong:2.0];
-  [NSNumber numberWithUnsignedLongLong:2.0f];
-  [NSNumber numberWithUnsignedLongLong:2.0F];
-  [NSNumber numberWithUnsignedLongLong:2.0l];
-  [NSNumber numberWithUnsignedLongLong:2.0L];
-  @0x2fULL;
-  @04ULL;
-  @0ULL;
-  [NSNumber numberWithUnsignedLongLong:0.0];
-  [NSNumber numberWithUnsignedLongLong:YES];
-  [NSNumber numberWithUnsignedLongLong:NO];
-  [NSNumber numberWithUnsignedLongLong:true];
-  [NSNumber numberWithUnsignedLongLong:false];
-  [NSNumber numberWithUnsignedLongLong:VAL_INT];
-  [NSNumber numberWithUnsignedLongLong:VAL_UINT];
-
-  [NSNumber numberWithFloat:'a'];
-  [NSNumber numberWithFloat:L'a'];
-  @2.0f;
-  @2.0f;
-  @2.0f;
-  @2.0f;
-  @2.0f;
-  @2.0f;
-  @2.0f;
-  @2.0f;
-  @2.0f;
-  @2.0f;
-  @2.0f;
-  @2.0f;
-  @2.0f;
-  @2.0F;
-  @2.0f;
-  @2.0f;
-  [NSNumber numberWithFloat:0x2f];
-  [NSNumber numberWithFloat:04];
-  @0.0f;
-  @0.0f;
-  [NSNumber numberWithFloat:YES];
-  [NSNumber numberWithFloat:NO];
-  [NSNumber numberWithFloat:true];
-  [NSNumber numberWithFloat:false];
-  [NSNumber numberWithFloat:VAL_INT];
-  [NSNumber numberWithFloat:VAL_UINT];
-
-  [NSNumber numberWithDouble:'a'];
-  [NSNumber numberWithDouble:L'a'];
-  @2.0;
-  @2.0;
-  @2.0;
-  @2.0;
-  @2.0;
-  @2.0;
-  @2.0;
-  @2.0;
-  @2.0;
-  @2.0;
-  @2.0;
-  @2.0;
-  @2.0;
-  @2.0;
-  @2.0;
-  @2.0;
-  [NSNumber numberWithDouble:0x2f];
-  [NSNumber numberWithDouble:04];
-  @0.0;
-  @0.0;
-  [NSNumber numberWithDouble:YES];
-  [NSNumber numberWithDouble:NO];
-  [NSNumber numberWithDouble:true];
-  [NSNumber numberWithDouble:false];
-  [NSNumber numberWithDouble:VAL_INT];
-  [NSNumber numberWithDouble:VAL_UINT];
-
-  [NSNumber numberWithBool:'a'];
-  [NSNumber numberWithBool:L'a'];
-  [NSNumber numberWithBool:2];
-  [NSNumber numberWithBool:2U];
-  [NSNumber numberWithBool:2u];
-  [NSNumber numberWithBool:2L];
-  [NSNumber numberWithBool:2l];
-  [NSNumber numberWithBool:2LL];
-  [NSNumber numberWithBool:2ll];
-  [NSNumber numberWithBool:2ul];
-  [NSNumber numberWithBool:2lu];
-  [NSNumber numberWithBool:2ull];
-  [NSNumber numberWithBool:2llu];
-  [NSNumber numberWithBool:2.0];
-  [NSNumber numberWithBool:2.0f];
-  [NSNumber numberWithBool:2.0F];
-  [NSNumber numberWithBool:2.0l];
-  [NSNumber numberWithBool:2.0L];
-  [NSNumber numberWithBool:0x2f];
-  [NSNumber numberWithBool:04];
-  [NSNumber numberWithBool:0];
-  [NSNumber numberWithBool:0.0];
-  @YES;
-  @NO;
-  @true;
-  @false;
-  [NSNumber numberWithBool:VAL_INT];
-  [NSNumber numberWithBool:VAL_UINT];
-
-  [NSNumber numberWithInteger:'a'];
-  [NSNumber numberWithInteger:L'a'];
-  @2;
-  @2;
-  @2;
-  @2L;
-  @2l;
-  @2;
-  @2;
-  @2;
-  @2;
-  @2;
-  @2;
-  [NSNumber numberWithInteger:2.0];
-  [NSNumber numberWithInteger:2.0f];
-  [NSNumber numberWithInteger:2.0F];
-  [NSNumber numberWithInteger:2.0l];
-  [NSNumber numberWithInteger:2.0L];
-  @0x2f;
-  @04;
-  @0;
-  [NSNumber numberWithInteger:0.0];
-  [NSNumber numberWithInteger:YES];
-  [NSNumber numberWithInteger:NO];
-  [NSNumber numberWithInteger:true];
-  [NSNumber numberWithInteger:false];
-  @VAL_INT;
-  [NSNumber numberWithInteger:VAL_UINT];
-
-  [NSNumber numberWithUnsignedInteger:'a'];
-  [NSNumber numberWithUnsignedInteger:L'a'];
-  @2U;
-  @2U;
-  @2u;
-  @2U;
-  @2u;
-  @2U;
-  @2u;
-  @2ul;
-  @2lu;
-  @2u;
-  @2u;
-  [NSNumber numberWithUnsignedInteger:2.0];
-  [NSNumber numberWithUnsignedInteger:2.0f];
-  [NSNumber numberWithUnsignedInteger:2.0F];
-  [NSNumber numberWithUnsignedInteger:2.0l];
-  [NSNumber numberWithUnsignedInteger:2.0L];
-  @0x2fU;
-  @04U;
-  @0U;
-  [NSNumber numberWithUnsignedInteger:0.0];
-  [NSNumber numberWithUnsignedInteger:YES];
-  [NSNumber numberWithUnsignedInteger:NO];
-  [NSNumber numberWithUnsignedInteger:true];
-  [NSNumber numberWithUnsignedInteger:false];
-  [NSNumber numberWithUnsignedInteger:VAL_INT];
-  @VAL_UINT;
-}
diff --git a/clang/test/ARCMT/objcmt-property-availability.m b/clang/test/ARCMT/objcmt-property-availability.m
deleted file mode 100644
index 37ba74f3346fb..0000000000000
--- a/clang/test/ARCMT/objcmt-property-availability.m
+++ /dev/null
@@ -1,45 +0,0 @@
-// RUN: rm -rf %t
-// RUN: %clang_cc1  -objcmt-migrate-readwrite-property -objcmt-migrate-readonly-property -mt-migrate-directory %t %s -x objective-c -fobjc-runtime-has-weak -fobjc-arc -triple x86_64-apple-darwin11
-// RUN: c-arcmt-test -mt-migrate-directory %t | arcmt-test -verify-transformed-files %s.result
-// RUN: %clang_cc1 -triple x86_64-apple-darwin10 -fsyntax-only -x objective-c -fobjc-runtime-has-weak -fobjc-arc %s.result
-
-
-#define __NSi_7_0 introduced=7.0
-#define __NSi_6_0 introduced=6.0
-
-#define CF_AVAILABLE(_mac, _ios) __attribute__((availability(ios,__NSi_##_ios)))
-#define CF_AVAILABLE_MAC(_mac) __attribute__((availability(macosx,__NSi_##_mac)))
-#define CF_AVAILABLE_IOS(_ios) __attribute__((availability(macosx,unavailable)))
-
-#define NS_AVAILABLE(_mac, _ios) CF_AVAILABLE(_mac, _ios)
-#define NS_AVAILABLE_MAC(_mac) CF_AVAILABLE_MAC(_mac)
-#define NS_AVAILABLE_IOS(_ios) CF_AVAILABLE_IOS(_ios)
-
-#define UNAVAILABLE __attribute__((unavailable("not available in automatic reference counting mode")))
-
-@interface MKMapItem
-- (MKMapItem *)source NS_AVAILABLE(10_9, 6_0);
-- (void)setSource:(MKMapItem *)source NS_AVAILABLE(10_9, 7_0);
-
-- (void)setDest:(MKMapItem *)source NS_AVAILABLE(10_9, 6_0);
-- (MKMapItem *)dest NS_AVAILABLE(10_9, 6_0);
-
-- (MKMapItem *)final;
-- (void)setFinal:(MKMapItem *)source;
-
-- (MKMapItem *)total NS_AVAILABLE(10_9, 6_0);
-- (void)setTotal:(MKMapItem *)source;
-
-- (MKMapItem *)comp NS_AVAILABLE(10_9, 6_0);
-- (void)setComp:(MKMapItem *)source UNAVAILABLE;
-
-- (MKMapItem *)tally  UNAVAILABLE NS_AVAILABLE(10_9, 6_0);
-- (void)setTally:(MKMapItem *)source UNAVAILABLE NS_AVAILABLE(10_9, 6_0);
-
-- (MKMapItem *)itally  NS_AVAILABLE(10_9, 6_0);
-- (void)setItally:(MKMapItem *)source UNAVAILABLE NS_AVAILABLE(10_9, 6_0);
-
-- (MKMapItem *)normal  UNAVAILABLE;
-- (void)setNormal:(MKMapItem *)source UNAVAILABLE NS_AVAILABLE(10_9, 6_0);
-@end
-
diff --git a/clang/test/ARCMT/objcmt-property-availability.m.result b/clang/test/ARCMT/objcmt-property-availability.m.result
deleted file mode 100644
index 3a212ac894c4a..0000000000000
--- a/clang/test/ARCMT/objcmt-property-availability.m.result
+++ /dev/null
@@ -1,42 +0,0 @@
-// RUN: rm -rf %t
-// RUN: %clang_cc1  -objcmt-migrate-readwrite-property -objcmt-migrate-readonly-property -mt-migrate-directory %t %s -x objective-c -fobjc-runtime-has-weak -fobjc-arc -triple x86_64-apple-darwin11
-// RUN: c-arcmt-test -mt-migrate-directory %t | arcmt-test -verify-transformed-files %s.result
-// RUN: %clang_cc1 -triple x86_64-apple-darwin10 -fsyntax-only -x objective-c -fobjc-runtime-has-weak -fobjc-arc %s.result
-
-
-#define __NSi_7_0 introduced=7.0
-#define __NSi_6_0 introduced=6.0
-
-#define CF_AVAILABLE(_mac, _ios) __attribute__((availability(ios,__NSi_##_ios)))
-#define CF_AVAILABLE_MAC(_mac) __attribute__((availability(macosx,__NSi_##_mac)))
-#define CF_AVAILABLE_IOS(_ios) __attribute__((availability(macosx,unavailable)))
-
-#define NS_AVAILABLE(_mac, _ios) CF_AVAILABLE(_mac, _ios)
-#define NS_AVAILABLE_MAC(_mac) CF_AVAILABLE_MAC(_mac)
-#define NS_AVAILABLE_IOS(_ios) CF_AVAILABLE_IOS(_ios)
-
-#define UNAVAILABLE __attribute__((unavailable("not available in automatic reference counting mode")))
-
-@interface MKMapItem
-@property (nonatomic, strong) MKMapItem *source NS_AVAILABLE(10_9, 6_0);
-- (void)setSource:(MKMapItem *)source NS_AVAILABLE(10_9, 7_0);
-
-@property (nonatomic, strong) MKMapItem *dest NS_AVAILABLE(10_9, 6_0);
-
-@property (nonatomic, strong) MKMapItem *final;
-
-@property (nonatomic, strong) MKMapItem *total NS_AVAILABLE(10_9, 6_0);
-- (void)setTotal:(MKMapItem *)source;
-
-- (MKMapItem *)comp NS_AVAILABLE(10_9, 6_0);
-- (void)setComp:(MKMapItem *)source UNAVAILABLE;
-
-@property (nonatomic, strong) MKMapItem *tally  UNAVAILABLE NS_AVAILABLE(10_9, 6_0);
-
-- (MKMapItem *)itally  NS_AVAILABLE(10_9, 6_0);
-- (void)setItally:(MKMapItem *)source UNAVAILABLE NS_AVAILABLE(10_9, 6_0);
-
-- (MKMapItem *)normal  UNAVAILABLE;
-- (void)setNormal:(MKMapItem *)source UNAVAILABLE NS_AVAILABLE(10_9, 6_0);
-@end
-
diff --git a/clang/test/ARCMT/objcmt-property-dot-syntax.m b/clang/test/ARCMT/objcmt-property-dot-syntax.m
deleted file mode 100644
index ec75b5140e18d..0000000000000
--- a/clang/test/ARCMT/objcmt-property-dot-syntax.m
+++ /dev/null
@@ -1,117 +0,0 @@
-// RUN: rm -rf %t
-// RUN: %clang_cc1 -objcmt-migrate-property-dot-syntax -mt-migrate-directory %t %s -x objective-c -fobjc-runtime-has-weak -fobjc-arc -triple x86_64-apple-darwin11
-// RUN: c-arcmt-test -mt-migrate-directory %t | arcmt-test -verify-transformed-files %s.result
-// RUN: %clang_cc1 -fblocks -triple x86_64-apple-darwin10 -fsyntax-only -x objective-c -fobjc-runtime-has-weak -fobjc-arc %s.result
-
-@class NSString;
-
-@protocol NSObject
-@property (readonly, copy) NSString *description;
-@end
-
-@interface NSObject <NSObject> @end
-
-@interface P : NSObject
-{
-  P* obj;
-  int i1, i2, i3;
-}
-@property int count;
-@property (copy) P* PropertyReturnsPObj;
-- (P*) MethodReturnsPObj;
-@end
-
-P* fun(void);
-
-@implementation P
-- (int) Meth : (P*)array {
-  [obj setCount : 100];
-
-  [(P*)0 setCount : [array count]];
-
-  [[obj PropertyReturnsPObj] setCount : [array count]];
-
-  [obj setCount : (i1+i2*i3 - 100)];
-
-  return [obj count] -
-         [(P*)0 count] + [array count] +
-         [fun() count] - 
-         [[obj PropertyReturnsPObj] count] +
-         [self->obj count];
-}
-
-- (P*) MethodReturnsPObj { return 0; }
-
-- (NSString *)description { return [super description]; }
-@end
-
-@interface Sub : P
-@end
-
-@implementation Sub
-- (int) Meth : (P*)array {
-  [super setCount : 100];
-
-  [super setCount : [array count]];
-
-  [[super PropertyReturnsPObj] setCount : [array count]];
-
-  [super setCount : (i1+i2*i3 - 100)];
-
-  return [super count] -
-         [(P*)0 count] + [array count] +
-         [fun() count] -
-         [[super PropertyReturnsPObj] count] +
-         [self->obj count];
-}
-@end
-
-
-@interface Rdar19038838
-@property id newItem; // should be marked objc_method_family(none), but isn't.
-@end
-
-id testRdar19038838(Rdar19038838 *obj) {
-  return [obj newItem];
-}
-
-@interface rdar19381786 : NSObject
-{
-  rdar19381786* obj;
-}
-@property int count;
-@end
-
-@protocol PR 
-@property int count;
-@end
-
-@implementation rdar19381786
--(void)test:(id)some : (id<PR>)qsome : (SEL)selsome
-{
-  [obj setCount : 100];
-  [some setCount : [some count]];
-  [qsome setCount : [qsome count]];
-}
-@end
-
-int NSOnState;
-int ArrNSOnState[4];
-@interface rdar19140114 : NSObject
-{
-  rdar19140114* menuItem;
-}
-@property int state;
-@end
-
-@implementation rdar19140114
-- (void) Meth {
-  [menuItem setState:NSOnState];
-  [menuItem setState :NSOnState];
-  [menuItem setState     :ArrNSOnState[NSOnState]];
-  [menuItem setState : NSOnState];
-  [menuItem setState:    NSOnState];
-  [menuItem setState: NSOnState];
-  [menuItem setState     :    NSOnState];
-}
-@end
diff --git a/clang/test/ARCMT/objcmt-property-dot-syntax.m.result b/clang/test/ARCMT/objcmt-property-dot-syntax.m.result
deleted file mode 100644
index 5153b0e658f6a..0000000000000
--- a/clang/test/ARCMT/objcmt-property-dot-syntax.m.result
+++ /dev/null
@@ -1,117 +0,0 @@
-// RUN: rm -rf %t
-// RUN: %clang_cc1 -objcmt-migrate-property-dot-syntax -mt-migrate-directory %t %s -x objective-c -fobjc-runtime-has-weak -fobjc-arc -triple x86_64-apple-darwin11
-// RUN: c-arcmt-test -mt-migrate-directory %t | arcmt-test -verify-transformed-files %s.result
-// RUN: %clang_cc1 -fblocks -triple x86_64-apple-darwin10 -fsyntax-only -x objective-c -fobjc-runtime-has-weak -fobjc-arc %s.result
-
-@class NSString;
-
-@protocol NSObject
-@property (readonly, copy) NSString *description;
-@end
-
-@interface NSObject <NSObject> @end
-
-@interface P : NSObject
-{
-  P* obj;
-  int i1, i2, i3;
-}
-@property int count;
-@property (copy) P* PropertyReturnsPObj;
-- (P*) MethodReturnsPObj;
-@end
-
-P* fun(void);
-
-@implementation P
-- (int) Meth : (P*)array {
-  obj.count = 100;
-
-  ((P*)0).count = array.count;
-
-  obj.PropertyReturnsPObj.count = array.count;
-
-  obj.count = (i1+i2*i3 - 100);
-
-  return obj.count -
-         ((P*)0).count + array.count +
-         fun().count - 
-         obj.PropertyReturnsPObj.count +
-         self->obj.count;
-}
-
-- (P*) MethodReturnsPObj { return 0; }
-
-- (NSString *)description { return super.description; }
-@end
-
-@interface Sub : P
-@end
-
-@implementation Sub
-- (int) Meth : (P*)array {
-  super.count = 100;
-
-  super.count = array.count;
-
-  super.PropertyReturnsPObj.count = array.count;
-
-  super.count = (i1+i2*i3 - 100);
-
-  return super.count -
-         ((P*)0).count + array.count +
-         fun().count -
-         super.PropertyReturnsPObj.count +
-         self->obj.count;
-}
-@end
-
-
-@interface Rdar19038838
-@property id newItem; // should be marked objc_method_family(none), but isn't.
-@end
-
-id testRdar19038838(Rdar19038838 *obj) {
-  return obj.newItem;
-}
-
-@interface rdar19381786 : NSObject
-{
-  rdar19381786* obj;
-}
-@property int count;
-@end
-
-@protocol PR 
-@property int count;
-@end
-
-@implementation rdar19381786
--(void)test:(id)some : (id<PR>)qsome : (SEL)selsome
-{
-  obj.count = 100;
-  [some setCount : [some count]];
-  qsome.count = qsome.count;
-}
-@end
-
-int NSOnState;
-int ArrNSOnState[4];
-@interface rdar19140114 : NSObject
-{
-  rdar19140114* menuItem;
-}
-@property int state;
-@end
-
-@implementation rdar19140114
-- (void) Meth {
-  menuItem.state = NSOnState;
-  menuItem.state = NSOnState;
-  menuItem.state = ArrNSOnState[NSOnState];
-  menuItem.state = NSOnState;
-  menuItem.state = NSOnState;
-  menuItem.state = NSOnState;
-  menuItem.state = NSOnState;
-}
-@end
diff --git a/clang/test/ARCMT/objcmt-property.m b/clang/test/ARCMT/objcmt-property.m
deleted file mode 100644
index f2b722e66d969..0000000000000
--- a/clang/test/ARCMT/objcmt-property.m
+++ /dev/null
@@ -1,243 +0,0 @@
-// RUN: rm -rf %t
-// RUN: %clang_cc1 -fblocks -objcmt-migrate-readwrite-property -objcmt-migrate-readonly-property -mt-migrate-directory %t %s -x objective-c -fobjc-runtime-has-weak -fobjc-arc -triple x86_64-apple-darwin11
-// RUN: c-arcmt-test -mt-migrate-directory %t | arcmt-test -verify-transformed-files %s.result
-// RUN: %clang_cc1 -fblocks -triple x86_64-apple-darwin10 -fsyntax-only -x objective-c -fobjc-runtime-has-weak -fobjc-arc %s.result
-
-#define WEBKIT_OBJC_METHOD_ANNOTATION(ANNOTATION) ANNOTATION
-#define WEAK_IMPORT_ATTRIBUTE __attribute__((objc_arc_weak_reference_unavailable))
-#define AVAILABLE_WEBKIT_VERSION_3_0_AND_LATER
-#define DEPRECATED  __attribute__((deprecated)) 
-
-typedef char BOOL;
-@class NSString;
-@protocol NSCopying @end
-
-@interface NSObject <NSCopying>
-@end
-
-@interface NSDictionary : NSObject
-@end
-
-@interface I : NSObject {
-  int ivarVal;
-}
-- (void) setWeakProp : (NSString *__weak)Val;
-- (NSString *__weak) WeakProp;
-
-- (NSString *) StrongProp;
-- (void) setStrongProp : (NSString *)Val;
-
-- (NSString *) UnavailProp  __attribute__((unavailable));
-- (void) setUnavailProp  : (NSString *)Val;
-
-- (NSString *) UnavailProp1  __attribute__((unavailable));
-- (void) setUnavailProp1  : (NSString *)Val  __attribute__((unavailable));
-
-- (NSString *) UnavailProp2;
-- (void) setUnavailProp2  : (NSString *)Val  __attribute__((unavailable));
-
-- (NSDictionary*) undoAction;
-- (void) setUndoAction: (NSDictionary*)Arg;
-@end
-
-@implementation I
-@end
-
-@class NSArray;
-
-@interface MyClass2  {
-@private
-    NSArray *_names1;
-    NSArray *_names2;
-    NSArray *_names3;
-    NSArray *_names4;
-}
-- (void)setNames1:(NSArray *)names;
-- (void)setNames4:(__strong NSArray *)names;
-- (void)setNames3:(__strong NSArray *)names;
-- (void)setNames2:(NSArray *)names;
-- (NSArray *) names2;
-- (NSArray *)names3;
-- (__strong NSArray *)names4;
-- (NSArray *) names1;
-@end
-
-// Properties that contain the name "delegate" or "dataSource",
-// or have exact name "target" have unsafe_unretained attribute.
-@interface NSInvocation 
-- (id)target;
-- (void)setTarget:(id)target;
-
-- (id) dataSource;
-
-- (id)delegate;
-
-- (id)xxxdelegateYYY;
-- (void)setXxxdelegateYYY:(id)delegate;
-
-- (void)setDataSource:(id)source;
-
-- (id)MYtarget;
-- (void)setMYtarget: (id)target;
-
-- (id)targetX;
-- (void)setTargetX: (id)t;
- 
-- (int)value;
-- (void)setValue: (int)val;
-
--(BOOL) isContinuous;
--(void) setContinuous:(BOOL)value;
-
-- (id) isAnObject;
-- (void)setAnObject : (id) object;
-
-- (BOOL) isinValid;
-- (void) setInValid : (BOOL) arg;
-
-- (void) Nothing;
-- (int) Length;
-- (id) object;
-+ (double) D;
-- (void *)JSObject WEBKIT_OBJC_METHOD_ANNOTATION(AVAILABLE_WEBKIT_VERSION_3_0_AND_LATER);
-- (BOOL)isIgnoringInteractionEvents;
-
-- (NSString *)getStringValue;
-- (BOOL)getCounterValue;
-- (void)setStringValue:(NSString *)stringValue AVAILABLE_WEBKIT_VERSION_3_0_AND_LATER;
-- (NSDictionary *)getns_dixtionary;
-
-- (BOOL)is3bar; // watch out
-- (NSString *)get3foo; // watch out
-
-- (BOOL) getM;
-- (BOOL) getMA;
-- (BOOL) getALL;
-- (BOOL) getMANY;
-- (BOOL) getSome;
-@end
-
-
-@interface NSInvocation(CAT)
-- (id)target;
-- (void)setTarget:(id)target;
-
-- (id) dataSource;
-
-- (id)xxxdelegateYYY;
-- (void)setXxxdelegateYYY:(id)delegate;
-
-- (void)setDataSource:(id)source;
-
-- (id)MYtarget;
-- (void)setMYtarget: (id)target;
-
-- (id)targetX;
-- (void)setTargetX: (id)t;
-
-- (int)value;
-- (void)setValue: (int)val;
-
--(BOOL) isContinuous;
--(void) setContinuous:(BOOL)value;
-
-- (id) isAnObject;
-- (void)setAnObject : (id) object;
-
-- (BOOL) isinValid;
-- (void) setInValid : (BOOL) arg;
-
-- (void) Nothing;
-- (int) Length;
-- (id) object;
-+ (double) D;
-
-- (BOOL)is3bar; // watch out
-- (NSString *)get3foo; // watch out
-
-- (BOOL) getM;
-- (BOOL) getMA;
-- (BOOL) getALL;
-- (BOOL) getMANY;
-- (BOOL) getSome;
-@end
-
-DEPRECATED
-@interface I_DEP
-- (BOOL) isinValid;
-- (void) setInValid : (BOOL) arg;
-@end
-
-@interface AnotherOne
-- (BOOL) isinValid DEPRECATED;
-- (void) setInValid : (BOOL) arg;
-- (id)MYtarget;
-- (void)setMYtarget: (id)target DEPRECATED;
-- (BOOL) getM DEPRECATED;
-
-- (id)xxxdelegateYYY DEPRECATED;
-- (void)setXxxdelegateYYY:(id)delegate DEPRECATED;
-@end
-
-#define NS_AVAILABLE __attribute__((availability(macosx,introduced=10.0)))
-#define NORETURN __attribute__((noreturn))
-#define ALIGNED __attribute__((aligned(16)))
-
-@interface NSURL
-// Do not infer a property.
-- (NSURL *)appStoreReceiptURL NS_AVAILABLE;
-- (void) setAppStoreReceiptURL : (NSURL *)object;
-
-- (NSURL *)appStoreReceiptURLX NS_AVAILABLE;
-- (void) setAppStoreReceiptURLX : (NSURL *)object NS_AVAILABLE;
-
-// Do not infer a property.
-- (NSURL *)appStoreReceiptURLY ;
-- (void) setAppStoreReceiptURLY : (NSURL *)object NS_AVAILABLE;
-
-- (id)OkToInfer NS_AVAILABLE;
-
-// Do not infer a property.
-- (NSURL *)appStoreReceiptURLZ ;
-- (void) setAppStoreReceiptURLZ : (NSURL *)object NS_AVAILABLE;
-
-// Do not infer a property.
-- (id) t1 NORETURN NS_AVAILABLE;
-- (void) setT1 : (id) arg NS_AVAILABLE;
-
-- (id)method1 ALIGNED NS_AVAILABLE;
-- (void) setMethod1 : (id) object NS_AVAILABLE ALIGNED;
-
-- (NSURL *)init;  // No Change
-+ (id)alloc;      // No Change
-
-- (BOOL)is1stClass; // Not a valid property
-- (BOOL)isClass;    // This is a valid property 'class' is not a keyword in ObjC
-- (BOOL)isDouble; // Not a valid property
-
-@end
-
-@class NSMutableDictionary;
-
-@interface NSArray
-- (id (^)(id, NSArray *, NSMutableDictionary *)) expressionBlock;
-- (id (^)(id, NSArray *, NSMutableDictionary *)) MyBlock;
-- (void) setMyBlock : (id (^)(id, NSArray *, NSMutableDictionary *)) bl;
-- (id (*)(id, NSArray *, NSMutableDictionary *)) expressionFuncptr;
-- (id (*)(id, NSArray *, NSMutableDictionary *)) MyFuncptr;
-- (void) setMyFuncptr : (id (*)(id, NSArray *, NSMutableDictionary *)) bl;
-@end
-
-@interface rdar15231241
-@property (nonatomic, readonly) double Ddelegate;
-@property (nonatomic, readonly) float Fdelegate;
-@property (nonatomic, readonly) int Idelegate;
-@property (nonatomic, readonly) BOOL Bdelegate;
-@end
-
-@protocol NSObject @end
-@protocol MyProtocol <NSObject>
-- (id)readonlyProperty;
-- (id)readWriteProperty;
-- (void)setReadWriteProperty:(id)readWriteProperty;
-@end
diff --git a/clang/test/ARCMT/objcmt-property.m.result b/clang/test/ARCMT/objcmt-property.m.result
deleted file mode 100644
index 610f027cee5c1..0000000000000
--- a/clang/test/ARCMT/objcmt-property.m.result
+++ /dev/null
@@ -1,215 +0,0 @@
-// RUN: rm -rf %t
-// RUN: %clang_cc1 -fblocks -objcmt-migrate-readwrite-property -objcmt-migrate-readonly-property -mt-migrate-directory %t %s -x objective-c -fobjc-runtime-has-weak -fobjc-arc -triple x86_64-apple-darwin11
-// RUN: c-arcmt-test -mt-migrate-directory %t | arcmt-test -verify-transformed-files %s.result
-// RUN: %clang_cc1 -fblocks -triple x86_64-apple-darwin10 -fsyntax-only -x objective-c -fobjc-runtime-has-weak -fobjc-arc %s.result
-
-#define WEBKIT_OBJC_METHOD_ANNOTATION(ANNOTATION) ANNOTATION
-#define WEAK_IMPORT_ATTRIBUTE __attribute__((objc_arc_weak_reference_unavailable))
-#define AVAILABLE_WEBKIT_VERSION_3_0_AND_LATER
-#define DEPRECATED  __attribute__((deprecated)) 
-
-typedef char BOOL;
-@class NSString;
-@protocol NSCopying @end
-
-@interface NSObject <NSCopying>
-@end
-
-@interface NSDictionary : NSObject
-@end
-
-@interface I : NSObject {
-  int ivarVal;
-}
-@property (nonatomic, weak) NSString *WeakProp;
-
-@property (nonatomic, strong) NSString *StrongProp;
-
-@property (nonatomic, strong) NSString *UnavailProp  __attribute__((unavailable));
-- (void) setUnavailProp  : (NSString *)Val;
-
-@property (nonatomic, strong) NSString *UnavailProp1  __attribute__((unavailable));
-
-@property (nonatomic, strong) NSString *UnavailProp2;
-- (void) setUnavailProp2  : (NSString *)Val  __attribute__((unavailable));
-
-@property (nonatomic, copy) NSDictionary *undoAction;
-@end
-
-@implementation I
-@end
-
-@class NSArray;
-
-@interface MyClass2  {
-@private
-    NSArray *_names1;
-    NSArray *_names2;
-    NSArray *_names3;
-    NSArray *_names4;
-}
-@property (nonatomic, strong) NSArray *names2;
-@property (nonatomic, strong) NSArray *names3;
-@property (nonatomic, strong) NSArray *names4;
-@property (nonatomic, strong) NSArray *names1;
-@end
-
-// Properties that contain the name "delegate" or "dataSource",
-// or have exact name "target" have unsafe_unretained attribute.
-@interface NSInvocation 
-@property (nonatomic, assign) id target;
-
-@property (nonatomic, assign) id dataSource;
-
-@property (nonatomic, readonly, assign) id delegate;
-
-@property (nonatomic, assign) id xxxdelegateYYY;
-
-
-@property (nonatomic, strong) id MYtarget;
-
-@property (nonatomic, strong) id targetX;
- 
-@property (nonatomic) int value;
-
-@property (nonatomic, getter=isContinuous) BOOL continuous;
-
-- (id) isAnObject;
-- (void)setAnObject : (id) object;
-
-@property (nonatomic, getter=isinValid, readonly) BOOL inValid;
-- (void) setInValid : (BOOL) arg;
-
-- (void) Nothing;
-@property (nonatomic, readonly) int Length;
-@property (nonatomic, readonly, strong) id object;
-+ (double) D;
-@property (nonatomic, readonly) void *JSObject WEBKIT_OBJC_METHOD_ANNOTATION(AVAILABLE_WEBKIT_VERSION_3_0_AND_LATER);
-@property (nonatomic, getter=isIgnoringInteractionEvents, readonly) BOOL ignoringInteractionEvents;
-
-@property (nonatomic, getter=getStringValue, strong) NSString *stringValue;
-@property (nonatomic, getter=getCounterValue, readonly) BOOL counterValue;
-@property (nonatomic, getter=getns_dixtionary, readonly, copy) NSDictionary *ns_dixtionary;
-
-- (BOOL)is3bar; // watch out
-- (NSString *)get3foo; // watch out
-
-@property (nonatomic, getter=getM, readonly) BOOL m;
-@property (nonatomic, getter=getMA, readonly) BOOL MA;
-@property (nonatomic, getter=getALL, readonly) BOOL ALL;
-@property (nonatomic, getter=getMANY, readonly) BOOL MANY;
-@property (nonatomic, getter=getSome, readonly) BOOL some;
-@end
-
-
-@interface NSInvocation(CAT)
-@property (nonatomic, assign) id target;
-
-@property (nonatomic, assign) id dataSource;
-
-@property (nonatomic, assign) id xxxdelegateYYY;
-
-
-@property (nonatomic, strong) id MYtarget;
-
-@property (nonatomic, strong) id targetX;
-
-@property (nonatomic) int value;
-
-@property (nonatomic, getter=isContinuous) BOOL continuous;
-
-- (id) isAnObject;
-- (void)setAnObject : (id) object;
-
-@property (nonatomic, getter=isinValid, readonly) BOOL inValid;
-- (void) setInValid : (BOOL) arg;
-
-- (void) Nothing;
-@property (nonatomic, readonly) int Length;
-@property (nonatomic, readonly, strong) id object;
-+ (double) D;
-
-- (BOOL)is3bar; // watch out
-- (NSString *)get3foo; // watch out
-
-@property (nonatomic, getter=getM, readonly) BOOL m;
-@property (nonatomic, getter=getMA, readonly) BOOL MA;
-@property (nonatomic, getter=getALL, readonly) BOOL ALL;
-@property (nonatomic, getter=getMANY, readonly) BOOL MANY;
-@property (nonatomic, getter=getSome, readonly) BOOL some;
-@end
-
-DEPRECATED
-@interface I_DEP
-- (BOOL) isinValid;
-- (void) setInValid : (BOOL) arg;
-@end
-
-@interface AnotherOne
-- (BOOL) isinValid DEPRECATED;
-- (void) setInValid : (BOOL) arg;
-- (id)MYtarget;
-- (void)setMYtarget: (id)target DEPRECATED;
-- (BOOL) getM DEPRECATED;
-
-- (id)xxxdelegateYYY DEPRECATED;
-- (void)setXxxdelegateYYY:(id)delegate DEPRECATED;
-@end
-
-#define NS_AVAILABLE __attribute__((availability(macosx,introduced=10.0)))
-#define NORETURN __attribute__((noreturn))
-#define ALIGNED __attribute__((aligned(16)))
-
-@interface NSURL
-// Do not infer a property.
-@property (nonatomic, strong) NSURL *appStoreReceiptURL NS_AVAILABLE;
-- (void) setAppStoreReceiptURL : (NSURL *)object;
-
-@property (nonatomic, strong) NSURL *appStoreReceiptURLX NS_AVAILABLE;
-
-// Do not infer a property.
-@property (nonatomic, strong) NSURL *appStoreReceiptURLY ;
-- (void) setAppStoreReceiptURLY : (NSURL *)object NS_AVAILABLE;
-
-@property (nonatomic, readonly, strong) id OkToInfer NS_AVAILABLE;
-
-// Do not infer a property.
-@property (nonatomic, strong) NSURL *appStoreReceiptURLZ ;
-- (void) setAppStoreReceiptURLZ : (NSURL *)object NS_AVAILABLE;
-
-// Do not infer a property.
-- (id) t1 NORETURN NS_AVAILABLE;
-- (void) setT1 : (id) arg NS_AVAILABLE;
-
-@property (nonatomic, strong) id method1 ALIGNED NS_AVAILABLE;
-
-- (NSURL *)init;  // No Change
-+ (id)alloc;      // No Change
-
-- (BOOL)is1stClass; // Not a valid property
-@property (nonatomic, getter=isClass, readonly) BOOL class;    // This is a valid property 'class' is not a keyword in ObjC
-- (BOOL)isDouble; // Not a valid property
-
-@end
-
-@class NSMutableDictionary;
-
-@interface NSArray
-@property (nonatomic, readonly, copy) id (^expressionBlock)(id, NSArray *, NSMutableDictionary *);
-@property (nonatomic, copy) id (^MyBlock)(id, NSArray *, NSMutableDictionary *);
-@property (nonatomic, readonly) id (*expressionFuncptr)(id, NSArray *, NSMutableDictionary *);
-@property (nonatomic) id (*MyFuncptr)(id, NSArray *, NSMutableDictionary *);
-@end
-
-@interface rdar15231241
-@property (nonatomic, readonly) double Ddelegate;
-@property (nonatomic, readonly) float Fdelegate;
-@property (nonatomic, readonly) int Idelegate;
-@property (nonatomic, readonly) BOOL Bdelegate;
-@end
-
-@protocol NSObject @end
-@protocol MyProtocol <NSObject>
-@property (nonatomic, readonly, strong) id readonlyProperty;
-@property (nonatomic, strong) id readWriteProperty;
-@end
diff --git a/clang/test/ARCMT/objcmt-protocol-conformance.m b/clang/test/ARCMT/objcmt-protocol-conformance.m
deleted file mode 100644
index e9bb4ac12e6d4..0000000000000
--- a/clang/test/ARCMT/objcmt-protocol-conformance.m
+++ /dev/null
@@ -1,129 +0,0 @@
-// RUN: rm -rf %t
-// RUN: %clang_cc1 -objcmt-migrate-protocol-conformance -mt-migrate-directory %t %s -x objective-c -fobjc-runtime-has-weak -fobjc-arc -triple x86_64-apple-darwin11
-// RUN: c-arcmt-test -mt-migrate-directory %t | arcmt-test -verify-transformed-files %s.result
-// RUN: %clang_cc1 -triple x86_64-apple-darwin10 -fsyntax-only -x objective-c -fobjc-runtime-has-weak -fobjc-arc %s.result
-
-@interface NSObject @end
-
-@protocol P
-- (id) Meth1: (double) arg;
-@end
-
-@interface Test1  // Test for no super class and no protocol list
-@end
-
-@implementation Test1
-- (id) Meth1: (double) arg { return 0; }
-@end
-
-@protocol P1 @end
-@protocol P2 @end
-
-@interface Test2 <P1, P2>  // Test for no super class and with protocol list
-{
-  id IVAR1;
-  id IVAR2;
-}
-@end
-
-@implementation Test2
-- (id) Meth1: (double) arg { return 0; }
-@end
-
-@interface Test3 : NSObject  { // Test for Super class and no  protocol list
-  id IV1;
-}
-@end
-
-@implementation Test3
-- (id) Meth1: (double) arg { return 0; }
-@end
-
-@interface Test4 : NSObject <P1, P2> // Test for Super class and protocol list
-@end
-
-@implementation Test4
-- (id) Meth1: (double) arg { return 0; }
-@end
-
-// Test5 - conforms to P3 because it implement's P3's property.
-@protocol P3
-@property (copy) id Prop;
-@end
-
-@protocol P4
-@property (copy) id Prop;
-@end
-
-@interface Test5 : NSObject<P3>
-@end
-
-@implementation Test5
-@synthesize Prop=_XXX;
-@end
-
-@protocol P5 <P3, P4>
-@property (copy) id Prop;
-@end
-
-@protocol P6 <P3, P4, P5>
-@property (copy) id Prop;
-@end
-
-@interface Test6 : NSObject // Test for minimal listing of conforming protocols
-@property (copy) id Prop;
-@end
-
-@implementation Test6 
-@end
-
-@class UIDynamicAnimator, UIWindow;
-@interface UIResponder : NSObject
-@end
-
-@protocol EmptyProtocol
-@end
-
-@protocol OptionalMethodsOnly
-@optional
-- (void)dynamicAnimatorWillResume:(UIDynamicAnimator*)animator;
-- (void)dynamicAnimatorDidPause:(UIDynamicAnimator*)animator;
-@end
-
-@protocol OptionalPropertiesOnly
-@optional
-@property (strong, nonatomic) id OptionalProperty;
-@end
-
-@protocol OptionalEvrything
-@optional
-- (void)dynamicAnimatorWillResume:(UIDynamicAnimator*)animator;
-@property (strong, nonatomic) id OptionalProperty;
-- (void)dynamicAnimatorDidPause:(UIDynamicAnimator*)animator;
-@end
-
-@protocol UIApplicationDelegate
-@end
-
-@interface Test7 : UIResponder <UIApplicationDelegate>
-@property (strong, nonatomic) UIWindow *window;
-@end
-
-@implementation Test7
-@end
-
-@interface BTLEBrowser
-@end
-
-@protocol CBCentralManagerDelegate; 
-
-@protocol CBCentralManagerDelegate 
-- (id) Meth1: (double) arg;
-@end
-
-@interface BTLEBrowser() <CBCentralManagerDelegate> 
-@end
-
-@implementation BTLEBrowser
-- (id) Meth15515206: (double) arg { return 0; }
-@end
diff --git a/clang/test/ARCMT/objcmt-protocol-conformance.m.result b/clang/test/ARCMT/objcmt-protocol-conformance.m.result
deleted file mode 100644
index 987532544bc64..0000000000000
--- a/clang/test/ARCMT/objcmt-protocol-conformance.m.result
+++ /dev/null
@@ -1,129 +0,0 @@
-// RUN: rm -rf %t
-// RUN: %clang_cc1 -objcmt-migrate-protocol-conformance -mt-migrate-directory %t %s -x objective-c -fobjc-runtime-has-weak -fobjc-arc -triple x86_64-apple-darwin11
-// RUN: c-arcmt-test -mt-migrate-directory %t | arcmt-test -verify-transformed-files %s.result
-// RUN: %clang_cc1 -triple x86_64-apple-darwin10 -fsyntax-only -x objective-c -fobjc-runtime-has-weak -fobjc-arc %s.result
-
-@interface NSObject @end
-
-@protocol P
-- (id) Meth1: (double) arg;
-@end
-
-@interface Test1<P>   // Test for no super class and no protocol list
-@end
-
-@implementation Test1
-- (id) Meth1: (double) arg { return 0; }
-@end
-
-@protocol P1 @end
-@protocol P2 @end
-
-@interface Test2 <P1, P2, P>  // Test for no super class and with protocol list
-{
-  id IVAR1;
-  id IVAR2;
-}
-@end
-
-@implementation Test2
-- (id) Meth1: (double) arg { return 0; }
-@end
-
-@interface Test3 : NSObject<P>   { // Test for Super class and no  protocol list
-  id IV1;
-}
-@end
-
-@implementation Test3
-- (id) Meth1: (double) arg { return 0; }
-@end
-
-@interface Test4 : NSObject <P1, P2, P> // Test for Super class and protocol list
-@end
-
-@implementation Test4
-- (id) Meth1: (double) arg { return 0; }
-@end
-
-// Test5 - conforms to P3 because it implement's P3's property.
-@protocol P3
-@property (copy) id Prop;
-@end
-
-@protocol P4
-@property (copy) id Prop;
-@end
-
-@interface Test5 : NSObject<P3, P4>
-@end
-
-@implementation Test5
-@synthesize Prop=_XXX;
-@end
-
-@protocol P5 <P3, P4>
-@property (copy) id Prop;
-@end
-
-@protocol P6 <P3, P4, P5>
-@property (copy) id Prop;
-@end
-
-@interface Test6 : NSObject<P6>  // Test for minimal listing of conforming protocols
-@property (copy) id Prop;
-@end
-
-@implementation Test6 
-@end
-
-@class UIDynamicAnimator, UIWindow;
-@interface UIResponder : NSObject
-@end
-
-@protocol EmptyProtocol
-@end
-
-@protocol OptionalMethodsOnly
-@optional
-- (void)dynamicAnimatorWillResume:(UIDynamicAnimator*)animator;
-- (void)dynamicAnimatorDidPause:(UIDynamicAnimator*)animator;
-@end
-
-@protocol OptionalPropertiesOnly
-@optional
-@property (strong, nonatomic) id OptionalProperty;
-@end
-
-@protocol OptionalEvrything
-@optional
-- (void)dynamicAnimatorWillResume:(UIDynamicAnimator*)animator;
-@property (strong, nonatomic) id OptionalProperty;
-- (void)dynamicAnimatorDidPause:(UIDynamicAnimator*)animator;
-@end
-
-@protocol UIApplicationDelegate
-@end
-
-@interface Test7 : UIResponder <UIApplicationDelegate>
-@property (strong, nonatomic) UIWindow *window;
-@end
-
-@implementation Test7
-@end
-
-@interface BTLEBrowser
-@end
-
-@protocol CBCentralManagerDelegate; 
-
-@protocol CBCentralManagerDelegate 
-- (id) Meth1: (double) arg;
-@end
-
-@interface BTLEBrowser() <CBCentralManagerDelegate> 
-@end
-
-@implementation BTLEBrowser
-- (id) Meth15515206: (double) arg { return 0; }
-@end
diff --git a/clang/test/ARCMT/objcmt-subscripting-literals-in-arc.m b/clang/test/ARCMT/objcmt-subscripting-literals-in-arc.m
deleted file mode 100644
index 1f56f4a2cf510..0000000000000
--- a/clang/test/ARCMT/objcmt-subscripting-literals-in-arc.m
+++ /dev/null
@@ -1,108 +0,0 @@
-// RUN: rm -rf %t
-// RUN: %clang_cc1 -fobjc-arc -objcmt-migrate-literals -objcmt-migrate-subscripting -mt-migrate-directory %t %s -x objective-c -triple x86_64-apple-darwin11 
-// RUN: c-arcmt-test -mt-migrate-directory %t | arcmt-test -verify-transformed-files %s.result
-// RUN: %clang_cc1 -fobjc-arc -triple x86_64-apple-darwin10 -fsyntax-only -x objective-c %s.result
-
-typedef signed char BOOL;
-#define nil ((void*) 0)
-
-typedef const struct __CFString * CFStringRef;
-
-@interface NSObject
-+ (id)alloc;
-@end
-
-@protocol NSCopying
-@end
-
-@interface NSString : NSObject
-+ (id)stringWithString:(NSString *)string;
-- (id)initWithString:(NSString *)aString;
-@end
-
-@interface NSArray : NSObject
-- (id)objectAtIndex:(unsigned long)index;
-@end
-
-@interface NSArray (NSExtendedArray)
-- (id)objectAtIndexedSubscript:(unsigned)idx;
-@end
-
-@interface NSArray (NSArrayCreation)
-+ (id)array;
-+ (id)arrayWithObject:(id)anObject;
-+ (id)arrayWithObjects:(const id [])objects count:(unsigned long)cnt;
-+ (id)arrayWithObjects:(id)firstObj, ...;
-+ (id)arrayWithArray:(NSArray *)array;
-
-- (id)initWithObjects:(const id [])objects count:(unsigned long)cnt;
-- (id)initWithObjects:(id)firstObj, ...;
-- (id)initWithArray:(NSArray *)array;
-@end
-
-@interface NSMutableArray : NSArray
-- (void)replaceObjectAtIndex:(unsigned long)index withObject:(id)anObject;
-@end
-
-@interface NSMutableArray (NSExtendedMutableArray)
-- (void)setObject:(id)obj atIndexedSubscript:(unsigned)idx;
-@end
-
-@interface NSDictionary : NSObject
-- (id)objectForKey:(id)aKey;
-@end
-
-@interface NSDictionary (NSExtendedDictionary)
-- (id)objectForKeyedSubscript:(id)key;
-@end
-
-@interface NSDictionary (NSDictionaryCreation)
-+ (id)dictionary;
-+ (id)dictionaryWithObject:(id)object forKey:(id)key;
-+ (id)dictionaryWithObjects:(const id [])objects forKeys:(const id [])keys count:(unsigned long)cnt;
-+ (id)dictionaryWithObjectsAndKeys:(id)firstObject, ...;
-+ (id)dictionaryWithDictionary:(NSDictionary *)dict;
-+ (id)dictionaryWithObjects:(NSArray *)objects forKeys:(NSArray *)keys;
-
-- (id)initWithObjects:(const id [])objects forKeys:(const id [])keys count:(unsigned long)cnt;
-- (id)initWithObjectsAndKeys:(id)firstObject, ...;
-- (id)initWithDictionary:(NSDictionary *)otherDictionary;
-- (id)initWithObjects:(NSArray *)objects forKeys:(NSArray *)keys;
-@end
-
-@interface NSMutableDictionary : NSDictionary
-- (void)setObject:(id)anObject forKey:(id)aKey;
-@end
-
-@interface NSMutableDictionary (NSExtendedMutableDictionary)
-- (void)setObject:(id)obj forKeyedSubscript:(id <NSCopying>)key;
-@end
-
-@interface NSNumber : NSObject
-@end
-
-@interface NSNumber (NSNumberCreation)
-+ (NSNumber *)numberWithInt:(int)value;
-- (id)initWithInt:(int)value;
-@end
-
-@interface I {
-  NSArray *ivarArr;
-}
-@end
-@implementation I
--(void) foo {
-  NSString *str;
-  NSArray *arr;
-  NSDictionary *dict;
-
-  arr = [NSArray arrayWithObjects:str, str, nil];
-  arr = [[NSArray alloc] initWithObjects:str, str, nil];
-  dict = [NSDictionary dictionaryWithObjectsAndKeys: @"value1", @"key1", @"value2", @"key2", nil];
-  dict = [[NSDictionary alloc] initWithObjectsAndKeys: @"value1", @"key1", @"value2", @"key2", nil];
-
-  dict = [[NSDictionary alloc] initWithObjects:[[NSArray alloc] initWithObjects:@"1", @"2", nil] forKeys:[NSArray arrayWithObjects:@"A", @"B", nil]];
-
-  NSNumber *n = [[NSNumber alloc] initWithInt:2];
-}
-@end
diff --git a/clang/test/ARCMT/objcmt-subscripting-literals-in-arc.m.result b/clang/test/ARCMT/objcmt-subscripting-literals-in-arc.m.result
deleted file mode 100644
index d974a2564d430..0000000000000
--- a/clang/test/ARCMT/objcmt-subscripting-literals-in-arc.m.result
+++ /dev/null
@@ -1,108 +0,0 @@
-// RUN: rm -rf %t
-// RUN: %clang_cc1 -fobjc-arc -objcmt-migrate-literals -objcmt-migrate-subscripting -mt-migrate-directory %t %s -x objective-c -triple x86_64-apple-darwin11 
-// RUN: c-arcmt-test -mt-migrate-directory %t | arcmt-test -verify-transformed-files %s.result
-// RUN: %clang_cc1 -fobjc-arc -triple x86_64-apple-darwin10 -fsyntax-only -x objective-c %s.result
-
-typedef signed char BOOL;
-#define nil ((void*) 0)
-
-typedef const struct __CFString * CFStringRef;
-
-@interface NSObject
-+ (id)alloc;
-@end
-
-@protocol NSCopying
-@end
-
-@interface NSString : NSObject
-+ (id)stringWithString:(NSString *)string;
-- (id)initWithString:(NSString *)aString;
-@end
-
-@interface NSArray : NSObject
-- (id)objectAtIndex:(unsigned long)index;
-@end
-
-@interface NSArray (NSExtendedArray)
-- (id)objectAtIndexedSubscript:(unsigned)idx;
-@end
-
-@interface NSArray (NSArrayCreation)
-+ (id)array;
-+ (id)arrayWithObject:(id)anObject;
-+ (id)arrayWithObjects:(const id [])objects count:(unsigned long)cnt;
-+ (id)arrayWithObjects:(id)firstObj, ...;
-+ (id)arrayWithArray:(NSArray *)array;
-
-- (id)initWithObjects:(const id [])objects count:(unsigned long)cnt;
-- (id)initWithObjects:(id)firstObj, ...;
-- (id)initWithArray:(NSArray *)array;
-@end
-
-@interface NSMutableArray : NSArray
-- (void)replaceObjectAtIndex:(unsigned long)index withObject:(id)anObject;
-@end
-
-@interface NSMutableArray (NSExtendedMutableArray)
-- (void)setObject:(id)obj atIndexedSubscript:(unsigned)idx;
-@end
-
-@interface NSDictionary : NSObject
-- (id)objectForKey:(id)aKey;
-@end
-
-@interface NSDictionary (NSExtendedDictionary)
-- (id)objectForKeyedSubscript:(id)key;
-@end
-
-@interface NSDictionary (NSDictionaryCreation)
-+ (id)dictionary;
-+ (id)dictionaryWithObject:(id)object forKey:(id)key;
-+ (id)dictionaryWithObjects:(const id [])objects forKeys:(const id [])keys count:(unsigned long)cnt;
-+ (id)dictionaryWithObjectsAndKeys:(id)firstObject, ...;
-+ (id)dictionaryWithDictionary:(NSDictionary *)dict;
-+ (id)dictionaryWithObjects:(NSArray *)objects forKeys:(NSArray *)keys;
-
-- (id)initWithObjects:(const id [])objects forKeys:(const id [])keys count:(unsigned long)cnt;
-- (id)initWithObjectsAndKeys:(id)firstObject, ...;
-- (id)initWithDictionary:(NSDictionary *)otherDictionary;
-- (id)initWithObjects:(NSArray *)objects forKeys:(NSArray *)keys;
-@end
-
-@interface NSMutableDictionary : NSDictionary
-- (void)setObject:(id)anObject forKey:(id)aKey;
-@end
-
-@interface NSMutableDictionary (NSExtendedMutableDictionary)
-- (void)setObject:(id)obj forKeyedSubscript:(id <NSCopying>)key;
-@end
-
-@interface NSNumber : NSObject
-@end
-
-@interface NSNumber (NSNumberCreation)
-+ (NSNumber *)numberWithInt:(int)value;
-- (id)initWithInt:(int)value;
-@end
-
-@interface I {
-  NSArray *ivarArr;
-}
-@end
-@implementation I
--(void) foo {
-  NSString *str;
-  NSArray *arr;
-  NSDictionary *dict;
-
-  arr = @[str, str];
-  arr = @[str, str];
-  dict = @{@"key1": @"value1", @"key2": @"value2"};
-  dict = @{@"key1": @"value1", @"key2": @"value2"};
-
-  dict = @{@"A": @"1", @"B": @"2"};
-
-  NSNumber *n = @2;
-}
-@end
diff --git a/clang/test/ARCMT/objcmt-subscripting-literals.m b/clang/test/ARCMT/objcmt-subscripting-literals.m
deleted file mode 100644
index e2b03e2d7b587..0000000000000
--- a/clang/test/ARCMT/objcmt-subscripting-literals.m
+++ /dev/null
@@ -1,230 +0,0 @@
-// RUN: rm -rf %t
-// RUN: %clang_cc1 -objcmt-migrate-literals -objcmt-migrate-subscripting -mt-migrate-directory %t %s -x objective-c -triple x86_64-apple-darwin11 
-// RUN: c-arcmt-test -mt-migrate-directory %t | arcmt-test -verify-transformed-files %s.result
-// RUN: %clang_cc1 -triple x86_64-apple-darwin10 -fsyntax-only -x objective-c %s.result
-
-typedef signed char BOOL;
-#define nil ((void*) 0)
-
-typedef const struct __CFString * CFStringRef;
-
-@interface NSObject
-+ (id)alloc;
-@end
-
-@protocol NSCopying
-@end
-
-@interface NSString : NSObject
-+ (id)stringWithString:(NSString *)string;
-- (id)initWithString:(NSString *)aString;
-@end
-
-@interface NSArray : NSObject
-- (id)objectAtIndex:(unsigned long)index;
-@end
-
-@interface NSArray (NSExtendedArray)
-- (id)objectAtIndexedSubscript:(unsigned)idx;
-@end
-
-@interface NSArray (NSArrayCreation)
-+ (id)array;
-+ (id)arrayWithObject:(id)anObject;
-+ (id)arrayWithObjects:(const id [])objects count:(unsigned long)cnt;
-+ (id)arrayWithObjects:(id)firstObj, ...;
-+ (id)arrayWithArray:(NSArray *)array;
-
-- (id)initWithObjects:(const id [])objects count:(unsigned long)cnt;
-- (id)initWithObjects:(id)firstObj, ...;
-- (id)initWithArray:(NSArray *)array;
-@end
-
-@interface NSMutableArray : NSArray
-- (void)replaceObjectAtIndex:(unsigned long)index withObject:(id)anObject;
-@end
-
-@interface NSMutableArray (NSExtendedMutableArray)
-- (void)setObject:(id)obj atIndexedSubscript:(unsigned)idx;
-@end
-
-@interface NSDictionary : NSObject
-- (id)objectForKey:(id)aKey;
-@end
-
-@interface NSDictionary (NSExtendedDictionary)
-- (id)objectForKeyedSubscript:(id)key;
-@end
-
-@interface NSDictionary (NSDictionaryCreation)
-+ (id)dictionary;
-+ (id)dictionaryWithObject:(id)object forKey:(id)key;
-+ (id)dictionaryWithObjects:(const id [])objects forKeys:(const id [])keys count:(unsigned long)cnt;
-+ (id)dictionaryWithObjectsAndKeys:(id)firstObject, ...;
-+ (id)dictionaryWithDictionary:(NSDictionary *)dict;
-+ (id)dictionaryWithObjects:(NSArray *)objects forKeys:(NSArray *)keys;
-
-- (id)initWithObjects:(const id [])objects forKeys:(const id [])keys count:(unsigned long)cnt;
-- (id)initWithObjectsAndKeys:(id)firstObject, ...;
-- (id)initWithDictionary:(NSDictionary *)otherDictionary;
-- (id)initWithObjects:(NSArray *)objects forKeys:(NSArray *)keys;
-@end
-
-@interface NSMutableDictionary : NSDictionary
-- (void)setObject:(id)anObject forKey:(id)aKey;
-@end
-
-@interface NSMutableDictionary (NSExtendedMutableDictionary)
-- (void)setObject:(id)obj forKeyedSubscript:(id <NSCopying>)key;
-@end
-
-@interface NSNumber : NSObject
-@end
-
-@interface NSNumber (NSNumberCreation)
-+ (NSNumber *)numberWithInt:(int)value;
-@end
-
-#define M(x) (x)
-#define PAIR(x) @#x, [NSNumber numberWithInt:(x)]
-#define TWO(x) ((x), (x))
-#define TWO_SEP(x,y) ((x), (y))
-
-@interface I {
-  NSArray *ivarArr;
-}
-@end
-@implementation I
--(void) foo {
-  NSString *str;
-  NSArray *arr;
-  NSDictionary *dict;
-
-  arr = [NSArray array];
-  arr = [NSArray arrayWithObject:str];
-  arr = [NSArray arrayWithObjects:str, str, nil];
-  dict = [NSDictionary dictionary];
-  dict = [NSDictionary dictionaryWithObject:arr forKey:str];
-  dict = [NSDictionary dictionaryWithObjectsAndKeys: @"value1", @"key1", @"value2", @"key2", nil];
-  dict = [NSDictionary dictionaryWithObjectsAndKeys: PAIR(1), PAIR(2), nil];
-  dict = [NSDictionary dictionaryWithObjectsAndKeys:
-                                               @"value1", @"key1",
-#ifdef BLAH
-                                               @"value2", @"key2",
-#else
-                                               @"value3", @"key3",
-#endif
-                                               nil ];
-
-  id o = [arr objectAtIndex:2];
-  o = [dict objectForKey:@"key"];
-  o = TWO([dict objectForKey:@"key"]);
-  o = TWO_SEP([dict objectForKey:@"key"], [arr objectAtIndex:2]);
-  o = [NSDictionary dictionaryWithObject:[NSDictionary dictionary] forKey:@"key"];
-  NSMutableArray *marr = 0;
-  NSMutableDictionary *mdict = 0;
-  [marr replaceObjectAtIndex:2 withObject:@"val"];
-  [mdict setObject:@"value" forKey:@"key"];
-  [marr replaceObjectAtIndex:2 withObject:[arr objectAtIndex:4]];
-  [mdict setObject:[dict objectForKey:@"key2"] forKey:@"key"];
-  [mdict setObject:[dict objectForKey:@"key2"] forKey:
-#if 1
-                     @"key1"
-#else
-                     @"key2"
-#endif
-                    ];
-  [mdict setObject:[dict objectForKey:
-#if 2
-                     @"key3"
-#else
-                     @"key4"
-#endif
-                   ] forKey:@"key"];
-  [mdict setObject:@"value" forKey:[dict objectForKey:
-#if 3
-                     @"key5"
-#else
-                     @"key6"
-#endif
-                   ] ];
-  [mdict setObject:@"val" forKey:[dict objectForKey:@"key2"]];
-  [mdict setObject:[dict objectForKey:@"key1"] forKey:[dict objectForKey:[NSArray arrayWithObject:@"arrkey"]]];
-  __strong NSArray **parr = 0;
-  o = [*parr objectAtIndex:2];
-  void *hd;
-  o = [(NSArray*)hd objectAtIndex:2];
-  o = [ivarArr objectAtIndex:2];
-
-  dict = [NSDictionary dictionaryWithObjects:[NSArray arrayWithObjects:@"1", [NSArray array], nil] forKeys:[NSArray arrayWithObjects:@"A", [arr objectAtIndex:2], nil]];
-  dict = [NSDictionary dictionaryWithObjects:[NSArray arrayWithObjects:@"1", @"2", nil] forKeys:arr];
-  dict = [NSDictionary dictionaryWithObjects:[NSArray arrayWithObjects:@"1", @"2", nil] forKeys:@[@"A", @"B"]];
-  dict = [NSDictionary dictionaryWithObjectsAndKeys:[NSArray array], @"A", [NSArray array], @"B", nil];
-}
-@end
-
-extern const CFStringRef globStr;
-
-void test1(NSString *str) {
-  NSDictionary *dict = [NSDictionary dictionaryWithObjectsAndKeys: str, globStr, nil];
-  dict = [NSDictionary dictionaryWithObjectsAndKeys: globStr, str, nil];
-  dict = [NSDictionary dictionaryWithObject:str forKey:globStr];
-  dict = [NSDictionary dictionaryWithObject:globStr forKey:str];
-
-  NSArray *arr = [NSArray arrayWithObjects: globStr, globStr, nil];
-  arr = [NSArray arrayWithObjects: str, globStr, nil];
-  arr = [NSArray arrayWithObjects: globStr, str, nil];
-  arr = [NSArray arrayWithObject:globStr];
-}
-
-@interface Custom : NSObject
-- (id)objectAtIndex:(unsigned long)index;
-@end
-
-@interface Custom (Extended)
-- (id)objectAtIndexedSubscript:(unsigned)idx;
-@end
-
-@interface MutableCustom : Custom
-- (void)replaceObjectAtIndex:(unsigned long)index withObject:(id)anObject;
-@end
-
-@interface MutableCustom (Extended)
-- (void)setObject:(id)obj atIndexedSubscript:(unsigned)idx;
-@end
-
-@interface CustomUnavail : NSObject
-- (id)objectAtIndex:(unsigned long)index;
-@end
-
-@interface CustomUnavail (Extended)
-- (id)objectAtIndexedSubscript:(unsigned)idx __attribute__((unavailable));
-@end
-
-@interface MutableCustomUnavail : CustomUnavail
-- (void)replaceObjectAtIndex:(unsigned long)index withObject:(id)anObject;
-@end
-
-@interface MutableCustomUnavail (Extended)
-- (void)setObject:(id)obj atIndexedSubscript:(unsigned)idx __attribute__((unavailable));
-@end
-
-void test2(void) {
-  MutableCustom *mutc;
-  id o = [mutc objectAtIndex:4];
-  [mutc replaceObjectAtIndex:2 withObject:@"val"];
-
-  MutableCustomUnavail *mutcunaval;
-  o = [mutcunaval objectAtIndex:4];
-  [mutcunaval replaceObjectAtIndex:2 withObject:@"val"];
-}
-
-@interface NSLocale : NSObject
-+ (id)systemLocale;
-+ (id)currentLocale;
-- (id)objectForKey:(id)key;
-@end
-
-void test3(id key) {
-  id o = [[NSLocale currentLocale] objectForKey:key];
-}
diff --git a/clang/test/ARCMT/objcmt-subscripting-literals.m.result b/clang/test/ARCMT/objcmt-subscripting-literals.m.result
deleted file mode 100644
index e0b385741f01e..0000000000000
--- a/clang/test/ARCMT/objcmt-subscripting-literals.m.result
+++ /dev/null
@@ -1,230 +0,0 @@
-// RUN: rm -rf %t
-// RUN: %clang_cc1 -objcmt-migrate-literals -objcmt-migrate-subscripting -mt-migrate-directory %t %s -x objective-c -triple x86_64-apple-darwin11 
-// RUN: c-arcmt-test -mt-migrate-directory %t | arcmt-test -verify-transformed-files %s.result
-// RUN: %clang_cc1 -triple x86_64-apple-darwin10 -fsyntax-only -x objective-c %s.result
-
-typedef signed char BOOL;
-#define nil ((void*) 0)
-
-typedef const struct __CFString * CFStringRef;
-
-@interface NSObject
-+ (id)alloc;
-@end
-
-@protocol NSCopying
-@end
-
-@interface NSString : NSObject
-+ (id)stringWithString:(NSString *)string;
-- (id)initWithString:(NSString *)aString;
-@end
-
-@interface NSArray : NSObject
-- (id)objectAtIndex:(unsigned long)index;
-@end
-
-@interface NSArray (NSExtendedArray)
-- (id)objectAtIndexedSubscript:(unsigned)idx;
-@end
-
-@interface NSArray (NSArrayCreation)
-+ (id)array;
-+ (id)arrayWithObject:(id)anObject;
-+ (id)arrayWithObjects:(const id [])objects count:(unsigned long)cnt;
-+ (id)arrayWithObjects:(id)firstObj, ...;
-+ (id)arrayWithArray:(NSArray *)array;
-
-- (id)initWithObjects:(const id [])objects count:(unsigned long)cnt;
-- (id)initWithObjects:(id)firstObj, ...;
-- (id)initWithArray:(NSArray *)array;
-@end
-
-@interface NSMutableArray : NSArray
-- (void)replaceObjectAtIndex:(unsigned long)index withObject:(id)anObject;
-@end
-
-@interface NSMutableArray (NSExtendedMutableArray)
-- (void)setObject:(id)obj atIndexedSubscript:(unsigned)idx;
-@end
-
-@interface NSDictionary : NSObject
-- (id)objectForKey:(id)aKey;
-@end
-
-@interface NSDictionary (NSExtendedDictionary)
-- (id)objectForKeyedSubscript:(id)key;
-@end
-
-@interface NSDictionary (NSDictionaryCreation)
-+ (id)dictionary;
-+ (id)dictionaryWithObject:(id)object forKey:(id)key;
-+ (id)dictionaryWithObjects:(const id [])objects forKeys:(const id [])keys count:(unsigned long)cnt;
-+ (id)dictionaryWithObjectsAndKeys:(id)firstObject, ...;
-+ (id)dictionaryWithDictionary:(NSDictionary *)dict;
-+ (id)dictionaryWithObjects:(NSArray *)objects forKeys:(NSArray *)keys;
-
-- (id)initWithObjects:(const id [])objects forKeys:(const id [])keys count:(unsigned long)cnt;
-- (id)initWithObjectsAndKeys:(id)firstObject, ...;
-- (id)initWithDictionary:(NSDictionary *)otherDictionary;
-- (id)initWithObjects:(NSArray *)objects forKeys:(NSArray *)keys;
-@end
-
-@interface NSMutableDictionary : NSDictionary
-- (void)setObject:(id)anObject forKey:(id)aKey;
-@end
-
-@interface NSMutableDictionary (NSExtendedMutableDictionary)
-- (void)setObject:(id)obj forKeyedSubscript:(id <NSCopying>)key;
-@end
-
-@interface NSNumber : NSObject
-@end
-
-@interface NSNumber (NSNumberCreation)
-+ (NSNumber *)numberWithInt:(int)value;
-@end
-
-#define M(x) (x)
-#define PAIR(x) @#x, [NSNumber numberWithInt:(x)]
-#define TWO(x) ((x), (x))
-#define TWO_SEP(x,y) ((x), (y))
-
-@interface I {
-  NSArray *ivarArr;
-}
-@end
-@implementation I
--(void) foo {
-  NSString *str;
-  NSArray *arr;
-  NSDictionary *dict;
-
-  arr = @[];
-  arr = @[str];
-  arr = @[str, str];
-  dict = @{};
-  dict = @{str: arr};
-  dict = @{@"key1": @"value1", @"key2": @"value2"};
-  dict = [NSDictionary dictionaryWithObjectsAndKeys: PAIR(1), PAIR(2), nil];
-  dict = [NSDictionary dictionaryWithObjectsAndKeys:
-                                               @"value1", @"key1",
-#ifdef BLAH
-                                               @"value2", @"key2",
-#else
-                                               @"value3", @"key3",
-#endif
-                                               nil ];
-
-  id o = arr[2];
-  o = dict[@"key"];
-  o = TWO(dict[@"key"]);
-  o = TWO_SEP(dict[@"key"], arr[2]);
-  o = @{@"key": @{}};
-  NSMutableArray *marr = 0;
-  NSMutableDictionary *mdict = 0;
-  marr[2] = @"val";
-  mdict[@"key"] = @"value";
-  marr[2] = arr[4];
-  mdict[@"key"] = dict[@"key2"];
-  [mdict setObject:dict[@"key2"] forKey:
-#if 1
-                     @"key1"
-#else
-                     @"key2"
-#endif
-                    ];
-  mdict[@"key"] = [dict objectForKey:
-#if 2
-                     @"key3"
-#else
-                     @"key4"
-#endif
-                   ];
-  mdict[[dict objectForKey:
-#if 3
-                     @"key5"
-#else
-                     @"key6"
-#endif
-                   ]] = @"value";
-  mdict[dict[@"key2"]] = @"val";
-  mdict[dict[@[@"arrkey"]]] = dict[@"key1"];
-  __strong NSArray **parr = 0;
-  o = (*parr)[2];
-  void *hd;
-  o = ((NSArray*)hd)[2];
-  o = ivarArr[2];
-
-  dict = @{@"A": @"1", arr[2]: @[]};
-  dict = [NSDictionary dictionaryWithObjects:@[@"1", @"2"] forKeys:arr];
-  dict = @{@"A": @"1", @"B": @"2"};
-  dict = @{@"A": @[], @"B": @[]};
-}
-@end
-
-extern const CFStringRef globStr;
-
-void test1(NSString *str) {
-  NSDictionary *dict = @{(id)globStr: str};
-  dict = @{str: (id)globStr};
-  dict = @{(id)globStr: str};
-  dict = @{str: (id)globStr};
-
-  NSArray *arr = @[(id)globStr, (id)globStr];
-  arr = @[str, (id)globStr];
-  arr = @[(id)globStr, str];
-  arr = @[(id)globStr];
-}
-
-@interface Custom : NSObject
-- (id)objectAtIndex:(unsigned long)index;
-@end
-
-@interface Custom (Extended)
-- (id)objectAtIndexedSubscript:(unsigned)idx;
-@end
-
-@interface MutableCustom : Custom
-- (void)replaceObjectAtIndex:(unsigned long)index withObject:(id)anObject;
-@end
-
-@interface MutableCustom (Extended)
-- (void)setObject:(id)obj atIndexedSubscript:(unsigned)idx;
-@end
-
-@interface CustomUnavail : NSObject
-- (id)objectAtIndex:(unsigned long)index;
-@end
-
-@interface CustomUnavail (Extended)
-- (id)objectAtIndexedSubscript:(unsigned)idx __attribute__((unavailable));
-@end
-
-@interface MutableCustomUnavail : CustomUnavail
-- (void)replaceObjectAtIndex:(unsigned long)index withObject:(id)anObject;
-@end
-
-@interface MutableCustomUnavail (Extended)
-- (void)setObject:(id)obj atIndexedSubscript:(unsigned)idx __attribute__((unavailable));
-@end
-
-void test2(void) {
-  MutableCustom *mutc;
-  id o = mutc[4];
-  mutc[2] = @"val";
-
-  MutableCustomUnavail *mutcunaval;
-  o = [mutcunaval objectAtIndex:4];
-  [mutcunaval replaceObjectAtIndex:2 withObject:@"val"];
-}
-
-@interface NSLocale : NSObject
-+ (id)systemLocale;
-+ (id)currentLocale;
-- (id)objectForKey:(id)key;
-@end
-
-void test3(id key) {
-  id o = [[NSLocale currentLocale] objectForKey:key];
-}
diff --git a/clang/test/ARCMT/objcmt-subscripting-unavailable.m b/clang/test/ARCMT/objcmt-subscripting-unavailable.m
deleted file mode 100644
index d72c362e30f4f..0000000000000
--- a/clang/test/ARCMT/objcmt-subscripting-unavailable.m
+++ /dev/null
@@ -1,79 +0,0 @@
-// RUN: rm -rf %t
-// RUN: %clang_cc1 -objcmt-migrate-literals -objcmt-migrate-subscripting -mt-migrate-directory %t %s -x objective-c -triple x86_64-apple-darwin11 
-// RUN: c-arcmt-test -mt-migrate-directory %t | arcmt-test -verify-transformed-files %s.result
-// RUN: %clang_cc1 -triple x86_64-apple-darwin10 -fsyntax-only -x objective-c %s.result
-
-typedef signed char BOOL;
-#define nil ((void*) 0)
-
-@interface NSObject
-+ (id)alloc;
-@end
-
-@interface NSArray : NSObject
-- (id)objectAtIndex:(unsigned long)index;
-@end
-
-@interface NSArray (NSArrayCreation)
-+ (id)array;
-+ (id)arrayWithObject:(id)anObject;
-+ (id)arrayWithObjects:(const id [])objects count:(unsigned long)cnt;
-+ (id)arrayWithObjects:(id)firstObj, ...;
-+ (id)arrayWithArray:(NSArray *)array;
-
-- (id)initWithObjects:(const id [])objects count:(unsigned long)cnt;
-- (id)initWithObjects:(id)firstObj, ...;
-- (id)initWithArray:(NSArray *)array;
-@end
-
-@interface NSMutableArray : NSArray
-- (void)replaceObjectAtIndex:(unsigned long)index withObject:(id)anObject;
-@end
-
-@interface NSDictionary : NSObject
-@end
-
-@interface NSDictionary (NSDictionaryCreation)
-+ (id)dictionary;
-+ (id)dictionaryWithObject:(id)object forKey:(id)key;
-+ (id)dictionaryWithObjects:(const id [])objects forKeys:(const id [])keys count:(unsigned long)cnt;
-+ (id)dictionaryWithObjectsAndKeys:(id)firstObject, ...;
-+ (id)dictionaryWithDictionary:(NSDictionary *)dict;
-+ (id)dictionaryWithObjects:(NSArray *)objects forKeys:(NSArray *)keys;
-
-- (id)initWithObjects:(const id [])objects forKeys:(const id [])keys count:(unsigned long)cnt;
-- (id)initWithObjectsAndKeys:(id)firstObject, ...;
-- (id)initWithDictionary:(NSDictionary *)otherDictionary;
-- (id)initWithObjects:(NSArray *)objects forKeys:(NSArray *)keys;
-
-- (id)objectForKey:(id)aKey;
-@end
-
-@interface NSMutableDictionary : NSDictionary
-- (void)setObject:(id)anObject forKey:(id)aKey;
-@end
-
-@interface I
-@end
-@implementation I
--(void) foo {
-  id str;
-  NSArray *arr;
-  NSDictionary *dict;
-
-  arr = [NSArray array];
-  arr = [NSArray arrayWithObject:str];
-  arr = [NSArray arrayWithObjects:str, str, nil];
-  dict = [NSDictionary dictionary];
-  dict = [NSDictionary dictionaryWithObject:arr forKey:str];
-
-  id o = [arr objectAtIndex:2];
-  o = [dict objectForKey:@"key"];
-  NSMutableArray *marr = 0;
-  NSMutableDictionary *mdict = 0;
-  [marr replaceObjectAtIndex:2 withObject:@"val"];
-  [mdict setObject:@"value" forKey:@"key"];
-  [marr replaceObjectAtIndex:2 withObject:[arr objectAtIndex:4]];
-  [mdict setObject:[dict objectForKey:@"key2"] forKey:@"key"];
-}
-@end
diff --git a/clang/test/ARCMT/objcmt-subscripting-unavailable.m.result b/clang/test/ARCMT/objcmt-subscripting-unavailable.m.result
deleted file mode 100644
index bd74d55838608..0000000000000
--- a/clang/test/ARCMT/objcmt-subscripting-unavailable.m.result
+++ /dev/null
@@ -1,79 +0,0 @@
-// RUN: rm -rf %t
-// RUN: %clang_cc1 -objcmt-migrate-literals -objcmt-migrate-subscripting -mt-migrate-directory %t %s -x objective-c -triple x86_64-apple-darwin11 
-// RUN: c-arcmt-test -mt-migrate-directory %t | arcmt-test -verify-transformed-files %s.result
-// RUN: %clang_cc1 -triple x86_64-apple-darwin10 -fsyntax-only -x objective-c %s.result
-
-typedef signed char BOOL;
-#define nil ((void*) 0)
-
-@interface NSObject
-+ (id)alloc;
-@end
-
-@interface NSArray : NSObject
-- (id)objectAtIndex:(unsigned long)index;
-@end
-
-@interface NSArray (NSArrayCreation)
-+ (id)array;
-+ (id)arrayWithObject:(id)anObject;
-+ (id)arrayWithObjects:(const id [])objects count:(unsigned long)cnt;
-+ (id)arrayWithObjects:(id)firstObj, ...;
-+ (id)arrayWithArray:(NSArray *)array;
-
-- (id)initWithObjects:(const id [])objects count:(unsigned long)cnt;
-- (id)initWithObjects:(id)firstObj, ...;
-- (id)initWithArray:(NSArray *)array;
-@end
-
-@interface NSMutableArray : NSArray
-- (void)replaceObjectAtIndex:(unsigned long)index withObject:(id)anObject;
-@end
-
-@interface NSDictionary : NSObject
-@end
-
-@interface NSDictionary (NSDictionaryCreation)
-+ (id)dictionary;
-+ (id)dictionaryWithObject:(id)object forKey:(id)key;
-+ (id)dictionaryWithObjects:(const id [])objects forKeys:(const id [])keys count:(unsigned long)cnt;
-+ (id)dictionaryWithObjectsAndKeys:(id)firstObject, ...;
-+ (id)dictionaryWithDictionary:(NSDictionary *)dict;
-+ (id)dictionaryWithObjects:(NSArray *)objects forKeys:(NSArray *)keys;
-
-- (id)initWithObjects:(const id [])objects forKeys:(const id [])keys count:(unsigned long)cnt;
-- (id)initWithObjectsAndKeys:(id)firstObject, ...;
-- (id)initWithDictionary:(NSDictionary *)otherDictionary;
-- (id)initWithObjects:(NSArray *)objects forKeys:(NSArray *)keys;
-
-- (id)objectForKey:(id)aKey;
-@end
-
-@interface NSMutableDictionary : NSDictionary
-- (void)setObject:(id)anObject forKey:(id)aKey;
-@end
-
-@interface I
-@end
-@implementation I
--(void) foo {
-  id str;
-  NSArray *arr;
-  NSDictionary *dict;
-
-  arr = @[];
-  arr = @[str];
-  arr = @[str, str];
-  dict = @{};
-  dict = @{str: arr};
-
-  id o = [arr objectAtIndex:2];
-  o = [dict objectForKey:@"key"];
-  NSMutableArray *marr = 0;
-  NSMutableDictionary *mdict = 0;
-  [marr replaceObjectAtIndex:2 withObject:@"val"];
-  [mdict setObject:@"value" forKey:@"key"];
-  [marr replaceObjectAtIndex:2 withObject:[arr objectAtIndex:4]];
-  [mdict setObject:[dict objectForKey:@"key2"] forKey:@"key"];
-}
-@end
diff --git a/clang/test/ARCMT/objcmt-undefined-ns-macros.m b/clang/test/ARCMT/objcmt-undefined-ns-macros.m
deleted file mode 100644
index 473b49589222d..0000000000000
--- a/clang/test/ARCMT/objcmt-undefined-ns-macros.m
+++ /dev/null
@@ -1,22 +0,0 @@
-// RUN: rm -rf %t
-// RUN: %clang_cc1 -fmodules -objcmt-migrate-ns-macros -mt-migrate-directory %t %s -x objective-c -fobjc-runtime-has-weak -fobjc-arc -triple x86_64-apple-darwin11
-// RUN: c-arcmt-test -mt-migrate-directory %t | arcmt-test -verify-transformed-files %s.result
-
-typedef long NSInteger;
-enum {
-    UIViewNone         = 0x0,
-    UIViewMargin       = 0x1,
-    UIViewWidth        = 0x2,
-    UIViewRightMargin  = 0x3,
-    UIViewBottomMargin = 0xbadbeef
-};
-typedef NSInteger UITableStyle;
-
-
-typedef
-  enum { two = 1 } NumericEnum2;
-
-typedef enum { three = 1 } NumericEnum3;
-
-typedef enum { four = 1 } NumericEnum4;
-
diff --git a/clang/test/ARCMT/objcmt-undefined-ns-macros.m.result b/clang/test/ARCMT/objcmt-undefined-ns-macros.m.result
deleted file mode 100644
index a6942e20795fd..0000000000000
--- a/clang/test/ARCMT/objcmt-undefined-ns-macros.m.result
+++ /dev/null
@@ -1,24 +0,0 @@
-// RUN: rm -rf %t
-// RUN: %clang_cc1 -fmodules -objcmt-migrate-ns-macros -mt-migrate-directory %t %s -x objective-c -fobjc-runtime-has-weak -fobjc-arc -triple x86_64-apple-darwin11
-// RUN: c-arcmt-test -mt-migrate-directory %t | arcmt-test -verify-transformed-files %s.result
-
-typedef long NSInteger;
-#ifndef NS_ENUM
-@import Foundation;
-#endif
-typedef NS_OPTIONS(NSUInteger, UITableStyle) {
-    UIViewNone         = 0x0,
-    UIViewMargin       = 0x1,
-    UIViewWidth        = 0x2,
-    UIViewRightMargin  = 0x3,
-    UIViewBottomMargin = 0xbadbeef
-};
-
-
-typedef
-  NS_ENUM(unsigned int, NumericEnum2) { two = 1 };
-
-typedef NS_ENUM(unsigned int, NumericEnum3) { three = 1 };
-
-typedef NS_ENUM(unsigned int, NumericEnum4) { four = 1 };
-
diff --git a/clang/test/ARCMT/objcmt-with-pch.m b/clang/test/ARCMT/objcmt-with-pch.m
deleted file mode 100644
index 0925442d45eb9..0000000000000
--- a/clang/test/ARCMT/objcmt-with-pch.m
+++ /dev/null
@@ -1,17 +0,0 @@
-// REQUIRES: x86-registered-target
-// RUN: rm -rf %t
-// RUN: %clang_cc1 -triple x86_64-apple-darwin10 -x objective-c %S/Common.h -emit-pch -o %t.pch
-// RUN: %clang_cc1 -triple x86_64-apple-darwin10 -objcmt-migrate-literals -objcmt-migrate-subscripting -mt-migrate-directory %t %s -x objective-c -include-pch %t.pch
-// RUN: c-arcmt-test -mt-migrate-directory %t | arcmt-test -verify-transformed-files %s.result
-// RUN: %clang_cc1 -triple x86_64-apple-darwin10 -fsyntax-only -x objective-c %s.result -include-pch %t.pch
-
-@interface NSNumber : NSObject
-@end
-
-@interface NSNumber (NSNumberCreation)
-+ (NSNumber *)numberWithInt:(int)value;
-@end
-
-void foo(void) {
-  NSNumber *n = [NSNumber numberWithInt:1];
-}
diff --git a/clang/test/ARCMT/objcmt-with-pch.m.result b/clang/test/ARCMT/objcmt-with-pch.m.result
deleted file mode 100644
index 6d37d11fe480d..0000000000000
--- a/clang/test/ARCMT/objcmt-with-pch.m.result
+++ /dev/null
@@ -1,17 +0,0 @@
-// REQUIRES: x86-registered-target
-// RUN: rm -rf %t
-// RUN: %clang_cc1 -triple x86_64-apple-darwin10 -x objective-c %S/Common.h -emit-pch -o %t.pch
-// RUN: %clang_cc1 -triple x86_64-apple-darwin10 -objcmt-migrate-literals -objcmt-migrate-subscripting -mt-migrate-directory %t %s -x objective-c -include-pch %t.pch
-// RUN: c-arcmt-test -mt-migrate-directory %t | arcmt-test -verify-transformed-files %s.result
-// RUN: %clang_cc1 -triple x86_64-apple-darwin10 -fsyntax-only -x objective-c %s.result -include-pch %t.pch
-
-@interface NSNumber : NSObject
-@end
-
-@interface NSNumber (NSNumberCreation)
-+ (NSNumber *)numberWithInt:(int)value;
-@end
-
-void foo(void) {
-  NSNumber *n = @1;
-}
diff --git a/clang/test/ARCMT/protected-scope.m b/clang/test/ARCMT/protected-scope.m
deleted file mode 100644
index b522f54cdf7c1..0000000000000
--- a/clang/test/ARCMT/protected-scope.m
+++ /dev/null
@@ -1,36 +0,0 @@
-// RUN: %clang_cc1 -triple x86_64-apple-darwin10 -fsyntax-only -fobjc-arc -x objective-c %s.result
-// RUN: arcmt-test --args -triple x86_64-apple-darwin10 -fsyntax-only -x objective-c %s > %t
-// RUN: diff %t %s.result
-
-#include "Common.h"
-
-void test(id p, int x) {
-  int v;
-  switch(x) {
-  case 0:
-    v++;
-    id w1 = p;
-    id w2 = p;
-    break;
-  case 1:
-    v++;
-    id w3 = p;
-    break;
-  case 2:
-  case 3:
-    break;
-  default:
-    break;
-  }
-}
-
-void test2(int p) {
-  switch (p) {
-  case 3:;
-    NSObject *o = [[NSObject alloc] init];
-    [o release];
-    break;
-  default:
-    break;
-  }
-}
diff --git a/clang/test/ARCMT/protected-scope.m.result b/clang/test/ARCMT/protected-scope.m.result
deleted file mode 100644
index 55070f2732271..0000000000000
--- a/clang/test/ARCMT/protected-scope.m.result
+++ /dev/null
@@ -1,38 +0,0 @@
-// RUN: %clang_cc1 -triple x86_64-apple-darwin10 -fsyntax-only -fobjc-arc -x objective-c %s.result
-// RUN: arcmt-test --args -triple x86_64-apple-darwin10 -fsyntax-only -x objective-c %s > %t
-// RUN: diff %t %s.result
-
-#include "Common.h"
-
-void test(id p, int x) {
-  int v;
-  switch(x) {
-  case 0: {
-    v++;
-    id w1 = p;
-    id w2 = p;
-    break;
-  }
-  case 1: {
-    v++;
-    id w3 = p;
-    break;
-  }
-  case 2:
-  case 3:
-    break;
-  default:
-    break;
-  }
-}
-
-void test2(int p) {
-  switch (p) {
-  case 3: {;
-    NSObject *o = [[NSObject alloc] init];
-    break;
-  }
-  default:
-    break;
-  }
-}
diff --git a/clang/test/ARCMT/releases-driver.m b/clang/test/ARCMT/releases-driver.m
deleted file mode 100644
index 96f96c1d3b482..0000000000000
--- a/clang/test/ARCMT/releases-driver.m
+++ /dev/null
@@ -1,67 +0,0 @@
-// RUN: %clang_cc1 -fblocks -fsyntax-only -fobjc-arc -x objective-c %s.result
-// RUN: cat %s > %t
-// RUN: %clang_cc1 -arcmt-action=modify -triple x86_64-apple-macosx10.6 -x objective-c %t
-// RUN: diff %t %s.result
-// RUN: rm %t
-
-typedef int BOOL;
-
-id IhaveSideEffect(void);
-
-@protocol NSObject
-- (BOOL)isEqual:(id)object;
-- (id)retain;
-- (oneway void)release;
-@end
-
-@interface NSObject <NSObject> {}
-@end
-
-@interface Foo : NSObject {
-  id bar;
-}
-@property (retain) id bar;
--(void)test:(id)obj;
-@end
-
-@implementation Foo
-
-@synthesize bar;
-
--(void)test:(id)obj {
-  id x = self.bar;
-  [x retain];
-  self.bar = obj;
-  // do stuff with x;
-  [x release];
-
-  [IhaveSideEffect() release];
-
-  [x release], x = 0;
-}
-  
-@end
-
-void func(Foo *p) {
-  [p release];
-  (([p release]));
-}
-
-@interface Baz {
-	id <NSObject> _foo;
-}
-@end
-
-@implementation Baz
-- (void) dealloc {
-  [_foo release];
-}
-@end
-
-#define RELEASE_MACRO(x) [x release]
-#define RELEASE_MACRO2(x) RELEASE_MACRO(x)
-
-void test2(id p) {
-  RELEASE_MACRO(p);
-  RELEASE_MACRO2(p);
-}
diff --git a/clang/test/ARCMT/releases-driver.m.result b/clang/test/ARCMT/releases-driver.m.result
deleted file mode 100644
index e7da9a04fc62a..0000000000000
--- a/clang/test/ARCMT/releases-driver.m.result
+++ /dev/null
@@ -1,58 +0,0 @@
-// RUN: %clang_cc1 -fblocks -fsyntax-only -fobjc-arc -x objective-c %s.result
-// RUN: cat %s > %t
-// RUN: %clang_cc1 -arcmt-action=modify -triple x86_64-apple-macosx10.6 -x objective-c %t
-// RUN: diff %t %s.result
-// RUN: rm %t
-
-typedef int BOOL;
-
-id IhaveSideEffect(void);
-
-@protocol NSObject
-- (BOOL)isEqual:(id)object;
-- (id)retain;
-- (oneway void)release;
-@end
-
-@interface NSObject <NSObject> {}
-@end
-
-@interface Foo : NSObject {
-  id bar;
-}
-@property (strong) id bar;
--(void)test:(id)obj;
-@end
-
-@implementation Foo
-
-@synthesize bar;
-
--(void)test:(id)obj {
-  id x = self.bar;
-  self.bar = obj;
-  // do stuff with x;
-
-  IhaveSideEffect();
-
-  x = 0;
-}
-  
-@end
-
-void func(Foo *p) {
-}
-
-@interface Baz {
-	id <NSObject> _foo;
-}
-@end
-
-@implementation Baz
-@end
-
-#define RELEASE_MACRO(x) [x release]
-#define RELEASE_MACRO2(x) RELEASE_MACRO(x)
-
-void test2(id p) {
-}
diff --git a/clang/test/ARCMT/releases.m b/clang/test/ARCMT/releases.m
deleted file mode 100644
index 8636a8a5acea8..0000000000000
--- a/clang/test/ARCMT/releases.m
+++ /dev/null
@@ -1,98 +0,0 @@
-// RUN: %clang_cc1 -fobjc-exceptions -fblocks -fsyntax-only -fobjc-arc -x objective-c %s.result
-// RUN: arcmt-test --args -triple x86_64-apple-darwin10 -fobjc-exceptions -fblocks -fsyntax-only -x objective-c %s > %t
-// RUN: diff %t %s.result
-
-#define nil 0
-
-typedef int BOOL;
-
-id IhaveSideEffect(void);
-
-@protocol NSObject
-- (BOOL)isEqual:(id)object;
-- (id)retain;
-- (oneway void)release;
-@end
-
-@interface NSObject <NSObject> {}
-@end
-
-@interface Foo : NSObject {
-  id bar;
-}
-@property (retain) id bar;
--(void)test:(id)obj;
-@end
-
-@implementation Foo
-
-@synthesize bar;
-
--(void)test:(id)obj {
-  id x = self.bar;
-  [x retain];
-  self.bar = obj;
-  // do stuff with x;
-  [x release];
-
-  [IhaveSideEffect() release];
-
-  [x release], x = 0;
-
-  @try {
-  } @finally {
-    [x release];
-  }
-}
-  
-@end
-
-void func(Foo *p) {
-  [p release];
-  (([p release]));
-}
-
-@interface Baz {
-	id <NSObject> _foo;
-}
-@end
-
-@implementation Baz
-- (void) dealloc {
-  [_foo release];
-}
-@end
-
-void block_test(Foo *p) {
-  id (^B)(void) = ^(void) {
-    if (p) {
-      id (^IB)(void) = ^(void) {
-        id bar = [p retain];
-	      [p release];
-        return bar;
-      };
-      IB();
-    }
-    return [p retain];
-  };
-}
-
-#define RELEASE_MACRO(x) [x release]
-#define RELEASE_MACRO2(x) RELEASE_MACRO(x)
-
-void test2(id p) {
-  RELEASE_MACRO(p);
-  RELEASE_MACRO2(p);
-}
-
-@implementation Foo2
-
-static id internal_var = 0;
-
-+ (void)setIt:(id)newone {
-    if (internal_var != newone) {
-        [internal_var release];
-        internal_var = [newone retain];
-    }
-}
-@end
diff --git a/clang/test/ARCMT/releases.m.result b/clang/test/ARCMT/releases.m.result
deleted file mode 100644
index 261175362b9bd..0000000000000
--- a/clang/test/ARCMT/releases.m.result
+++ /dev/null
@@ -1,87 +0,0 @@
-// RUN: %clang_cc1 -fobjc-exceptions -fblocks -fsyntax-only -fobjc-arc -x objective-c %s.result
-// RUN: arcmt-test --args -triple x86_64-apple-darwin10 -fobjc-exceptions -fblocks -fsyntax-only -x objective-c %s > %t
-// RUN: diff %t %s.result
-
-#define nil 0
-
-typedef int BOOL;
-
-id IhaveSideEffect(void);
-
-@protocol NSObject
-- (BOOL)isEqual:(id)object;
-- (id)retain;
-- (oneway void)release;
-@end
-
-@interface NSObject <NSObject> {}
-@end
-
-@interface Foo : NSObject {
-  id bar;
-}
-@property (strong) id bar;
--(void)test:(id)obj;
-@end
-
-@implementation Foo
-
-@synthesize bar;
-
--(void)test:(id)obj {
-  id x = self.bar;
-  self.bar = obj;
-  // do stuff with x;
-
-  IhaveSideEffect();
-
-  x = 0;
-
-  @try {
-  } @finally {
-    x = nil;
-  }
-}
-  
-@end
-
-void func(Foo *p) {
-}
-
-@interface Baz {
-	id <NSObject> _foo;
-}
-@end
-
-@implementation Baz
-@end
-
-void block_test(Foo *p) {
-  id (^B)(void) = ^(void) {
-    if (p) {
-      id (^IB)(void) = ^(void) {
-        id bar = p;
-        return bar;
-      };
-      IB();
-    }
-    return p;
-  };
-}
-
-#define RELEASE_MACRO(x) [x release]
-#define RELEASE_MACRO2(x) RELEASE_MACRO(x)
-
-void test2(id p) {
-}
-
-@implementation Foo2
-
-static id internal_var = 0;
-
-+ (void)setIt:(id)newone {
-    if (internal_var != newone) {
-        internal_var = newone;
-    }
-}
-@end
diff --git a/clang/test/ARCMT/remap-applying.c b/clang/test/ARCMT/remap-applying.c
deleted file mode 100644
index dee2e391d5bd5..0000000000000
--- a/clang/test/ARCMT/remap-applying.c
+++ /dev/null
@@ -1,4 +0,0 @@
-a bc
-
-// RUN: echo "[{\"file\": \"%/s\", \"offset\": 1, \"remove\": 2, }]" > %t.remap
-// RUN: c-arcmt-test %t.remap | arcmt-test -verify-transformed-files %s.result
diff --git a/clang/test/ARCMT/remap-applying.c.result b/clang/test/ARCMT/remap-applying.c.result
deleted file mode 100644
index 514e9c264915d..0000000000000
--- a/clang/test/ARCMT/remap-applying.c.result
+++ /dev/null
@@ -1,4 +0,0 @@
-ac
-
-// RUN: echo "[{\"file\": \"%/s\", \"offset\": 1, \"remove\": 2, }]" > %t.remap
-// RUN: c-arcmt-test %t.remap | arcmt-test -verify-transformed-files %s.result
diff --git a/clang/test/ARCMT/remove-dealloc-method.m b/clang/test/ARCMT/remove-dealloc-method.m
deleted file mode 100644
index 8e39fc874c950..0000000000000
--- a/clang/test/ARCMT/remove-dealloc-method.m
+++ /dev/null
@@ -1,26 +0,0 @@
-// RUN: %clang_cc1 -triple x86_64-apple-darwin10 -fsyntax-only -fobjc-arc -x objective-c %s.result
-// RUN: arcmt-test --args -triple x86_64-apple-darwin10 -fsyntax-only -x objective-c %s > %t
-// RUN: diff %t %s.result
-
-#define nil ((void*) 0)
-
-@interface Foo 
-@property (retain) id x;
-@property (retain) id y;
-@property (retain) id w;
-@property (retain) id z;
-@end
-
-@implementation Foo 
-@synthesize x;
-@synthesize y;
-@synthesize w;
-@synthesize z;
-
-- (void) dealloc {
-  self.x = 0;
-  [self setY:nil];
-  w = nil;
-  self.z = nil;
-}
-@end
diff --git a/clang/test/ARCMT/remove-dealloc-method.m.result b/clang/test/ARCMT/remove-dealloc-method.m.result
deleted file mode 100644
index 47e31f9d249ae..0000000000000
--- a/clang/test/ARCMT/remove-dealloc-method.m.result
+++ /dev/null
@@ -1,20 +0,0 @@
-// RUN: %clang_cc1 -triple x86_64-apple-darwin10 -fsyntax-only -fobjc-arc -x objective-c %s.result
-// RUN: arcmt-test --args -triple x86_64-apple-darwin10 -fsyntax-only -x objective-c %s > %t
-// RUN: diff %t %s.result
-
-#define nil ((void*) 0)
-
-@interface Foo 
-@property (strong) id x;
-@property (strong) id y;
-@property (strong) id w;
-@property (strong) id z;
-@end
-
-@implementation Foo 
-@synthesize x;
-@synthesize y;
-@synthesize w;
-@synthesize z;
-
-@end
diff --git a/clang/test/ARCMT/remove-dealloc-zerouts.m b/clang/test/ARCMT/remove-dealloc-zerouts.m
deleted file mode 100644
index 4176ec580c3e5..0000000000000
--- a/clang/test/ARCMT/remove-dealloc-zerouts.m
+++ /dev/null
@@ -1,44 +0,0 @@
-// RUN: %clang_cc1 -triple x86_64-apple-darwin10 -fsyntax-only -fobjc-arc -x objective-c %s.result
-// RUN: arcmt-test --args -triple x86_64-apple-darwin10 -fsyntax-only -x objective-c %s > %t
-// RUN: diff %t %s.result
-
-@interface Foo 
-@property (retain) id x;
-@property (retain) id y;
-@property (retain) id w;
-@property (retain) id z;
-@property (strong) id q;
-@end
-
-@implementation Foo 
-@synthesize x;
-@synthesize y;
-@synthesize w;
-@synthesize q;
-@dynamic z;
-
-- (void) dealloc {
-  self.x = self.y = self.w = 0;
-  self.x = 0, w = 0, y = 0;
-  [self setY:0];
-  w = 0;
-  q = 0;
-  self.z = 0;
-}
-@end
-
-@interface Bar
-@property (retain) Foo *a;
-- (void) setA:(Foo*) val;
-- (id) a;
-@end
-
-@implementation Bar
-- (void) dealloc {
-  [self setA:0];  // This is user-defined setter overriding synthesize, don't touch it.
-  self.a.x = 0;  // every dealloc must zero out its own ivar. This patter is not recognized.
-}
-@synthesize a;
-- (void) setA:(Foo*) val { }
-- (id) a {return 0;}
-@end
diff --git a/clang/test/ARCMT/remove-dealloc-zerouts.m.result b/clang/test/ARCMT/remove-dealloc-zerouts.m.result
deleted file mode 100644
index 9ae831abacf25..0000000000000
--- a/clang/test/ARCMT/remove-dealloc-zerouts.m.result
+++ /dev/null
@@ -1,39 +0,0 @@
-// RUN: %clang_cc1 -triple x86_64-apple-darwin10 -fsyntax-only -fobjc-arc -x objective-c %s.result
-// RUN: arcmt-test --args -triple x86_64-apple-darwin10 -fsyntax-only -x objective-c %s > %t
-// RUN: diff %t %s.result
-
-@interface Foo 
-@property (strong) id x;
-@property (strong) id y;
-@property (strong) id w;
-@property (strong) id z;
-@property (strong) id q;
-@end
-
-@implementation Foo 
-@synthesize x;
-@synthesize y;
-@synthesize w;
-@synthesize q;
-@dynamic z;
-
-- (void) dealloc {
-  self.z = 0;
-}
-@end
-
-@interface Bar
-@property (strong) Foo *a;
-- (void) setA:(Foo*) val;
-- (id) a;
-@end
-
-@implementation Bar
-- (void) dealloc {
-  [self setA:0];  // This is user-defined setter overriding synthesize, don't touch it.
-  self.a.x = 0;  // every dealloc must zero out its own ivar. This patter is not recognized.
-}
-@synthesize a;
-- (void) setA:(Foo*) val { }
-- (id) a {return 0;}
-@end
diff --git a/clang/test/ARCMT/remove-statements.m b/clang/test/ARCMT/remove-statements.m
deleted file mode 100644
index 286a8e715e0d3..0000000000000
--- a/clang/test/ARCMT/remove-statements.m
+++ /dev/null
@@ -1,45 +0,0 @@
-// RUN: %clang_cc1 -triple x86_64-apple-darwin10 -fsyntax-only -fobjc-arc -x objective-c %s.result
-// RUN: arcmt-test --args -triple x86_64-apple-darwin10 -fsyntax-only -x objective-c %s > %t
-// RUN: diff %t %s.result
-
-#include "Common.h"
-
-@interface myController : NSObject
--(id)test:(id)x;
-@end
-
-#define MY_MACRO1(x)
-#define MY_MACRO2(x) (void)x
-
-@implementation myController
--(id) test:(id) x {
-  [[x retain] release];
-  return [[x retain] autorelease];
-}
-
--(void)dealloc
-{
-  id array, array_already_empty;
-  for (id element in array_already_empty) {
-  }
-
-  [array release];
-  ;
-
-  int b, b_array_already_empty;
-  if (b)
-    [array release];
-  if (b_array_already_empty) ;
-
-  if (b) {
-    [array release];
-  }
-  if (b_array_already_empty) {
-  }
-
-  if (b)
-    MY_MACRO1(array);
-  if (b)
-    MY_MACRO2(array);
-}
-@end
diff --git a/clang/test/ARCMT/remove-statements.m.result b/clang/test/ARCMT/remove-statements.m.result
deleted file mode 100644
index 6a4ea08b8c95b..0000000000000
--- a/clang/test/ARCMT/remove-statements.m.result
+++ /dev/null
@@ -1,38 +0,0 @@
-// RUN: %clang_cc1 -triple x86_64-apple-darwin10 -fsyntax-only -fobjc-arc -x objective-c %s.result
-// RUN: arcmt-test --args -triple x86_64-apple-darwin10 -fsyntax-only -x objective-c %s > %t
-// RUN: diff %t %s.result
-
-#include "Common.h"
-
-@interface myController : NSObject
--(id)test:(id)x;
-@end
-
-#define MY_MACRO1(x)
-#define MY_MACRO2(x) (void)x
-
-@implementation myController
--(id) test:(id) x {
-  return x;
-}
-
--(void)dealloc
-{
-  id array, array_already_empty;
-  for (id element in array_already_empty) {
-  }
-
-  ;
-
-  int b, b_array_already_empty;
-  if (b_array_already_empty) ;
-
-  if (b_array_already_empty) {
-  }
-
-  if (b)
-    MY_MACRO1(array);
-  if (b)
-    MY_MACRO2(array);
-}
-@end
diff --git a/clang/test/ARCMT/retains.m b/clang/test/ARCMT/retains.m
deleted file mode 100644
index a38efe1596403..0000000000000
--- a/clang/test/ARCMT/retains.m
+++ /dev/null
@@ -1,71 +0,0 @@
-// RUN: %clang_cc1 -fblocks -fsyntax-only -fobjc-arc -x objective-c %s.result
-// RUN: arcmt-test --args -triple x86_64-apple-darwin10 -fblocks -fsyntax-only -x objective-c %s > %t
-// RUN: diff %t %s.result
-
-#include "Common.h"
-
-id IhaveSideEffect(void);
-
-@interface Foo : NSObject {
-  id bar;
-}
-@property (retain) id bar;
--(id)test:(id)obj;
--(id)something;
-@end
-
-#define Something_Macro(key, comment) \
- [[Foo new] something]
-
-@implementation Foo
-
-@synthesize bar;
-
--(id)something { return (id)0; }
-
--(id)test:(id)obj {
-  id x = self.bar;
-  [x retain];
-  self.bar = obj;
-  if (obj)
-    [obj retain];
-
-  [Something_Macro(@"foo", "@bar") retain];
-
-  [IhaveSideEffect() retain];
-
-  [[self something] retain];
-
-  [[self retain] something];
-
-  [[IhaveSideEffect() retain] release];
-  [[x retain] release];
-  // do stuff with x;
-  [x release];
-  return [self retain];
-}
-  
-- (id)test1 {
-  id x=0;
-  ([x retain]);
-  return ((([x retain])));
-}
-@end
-
-id foo (Foo *p) {
-    p = [p retain];
-    return ([p retain]);
-}
-
-void block_tests(Foo *p) {
-  id (^B)(void) = ^(void) {
-    if (p) {
-      id (^IB)(void) = ^(void) {
-        id bar = [p retain];
-        return bar;
-      };
-      IB();
-    }
-    return [p retain];
-  };
-}
diff --git a/clang/test/ARCMT/retains.m.result b/clang/test/ARCMT/retains.m.result
deleted file mode 100644
index cd3bb3848fcec..0000000000000
--- a/clang/test/ARCMT/retains.m.result
+++ /dev/null
@@ -1,65 +0,0 @@
-// RUN: %clang_cc1 -fblocks -fsyntax-only -fobjc-arc -x objective-c %s.result
-// RUN: arcmt-test --args -triple x86_64-apple-darwin10 -fblocks -fsyntax-only -x objective-c %s > %t
-// RUN: diff %t %s.result
-
-#include "Common.h"
-
-id IhaveSideEffect(void);
-
-@interface Foo : NSObject {
-  id bar;
-}
-@property (strong) id bar;
--(id)test:(id)obj;
--(id)something;
-@end
-
-#define Something_Macro(key, comment) \
- [[Foo new] something]
-
-@implementation Foo
-
-@synthesize bar;
-
--(id)something { return (id)0; }
-
--(id)test:(id)obj {
-  id x = self.bar;
-  self.bar = obj;
-
-  Something_Macro(@"foo", "@bar");
-
-  IhaveSideEffect();
-
-  [self something];
-
-  [self something];
-
-  IhaveSideEffect();
-  // do stuff with x;
-  return self;
-}
-  
-- (id)test1 {
-  id x=0;
-  return (((x)));
-}
-@end
-
-id foo (Foo *p) {
-    p = p;
-    return (p);
-}
-
-void block_tests(Foo *p) {
-  id (^B)(void) = ^(void) {
-    if (p) {
-      id (^IB)(void) = ^(void) {
-        id bar = p;
-        return bar;
-      };
-      IB();
-    }
-    return p;
-  };
-}
diff --git a/clang/test/ARCMT/rewrite-block-var.m b/clang/test/ARCMT/rewrite-block-var.m
deleted file mode 100644
index eb3c5b6535971..0000000000000
--- a/clang/test/ARCMT/rewrite-block-var.m
+++ /dev/null
@@ -1,45 +0,0 @@
-// RUN: %clang_cc1 -triple x86_64-apple-darwin10 -fblocks -fsyntax-only -fobjc-arc -x objective-c -fobjc-runtime-has-weak %s.result
-// RUN: arcmt-test --args -triple x86_64-apple-macosx10.7 -fblocks -fsyntax-only %s > %t
-// RUN: diff %t %s.result
-
-#include "Common.h"
-
-@interface Foo : NSObject
--(Foo *)something;
-@end
-
-void bar(void (^block)(void));
-
-void test1(Foo *p) {
-  __block Foo *x = p; // __block used just to break cycle.
-  bar(^{
-    [x something];
-  });
-}
-
-void test2(Foo *p) {
-  __block Foo *x; // __block used as output variable.
-  bar(^{
-    x = [p something];
-  });
-}
-
-void test3(Foo *p) {
-  __block Foo *x; // __block used as output variable.
-  bar(^{
-    [x something];
-  });
-  bar(^{
-    x = 0;
-  });
-}
-
-void test4(Foo *p) {
-  __block Foo *x = p; // __block used just to break cycle.
-  bar(^{
-    [x something];
-  });
-  bar(^{
-    [x something];
-  });
-}
diff --git a/clang/test/ARCMT/rewrite-block-var.m.result b/clang/test/ARCMT/rewrite-block-var.m.result
deleted file mode 100644
index cf5718fbd7f5d..0000000000000
--- a/clang/test/ARCMT/rewrite-block-var.m.result
+++ /dev/null
@@ -1,45 +0,0 @@
-// RUN: %clang_cc1 -triple x86_64-apple-darwin10 -fblocks -fsyntax-only -fobjc-arc -x objective-c -fobjc-runtime-has-weak %s.result
-// RUN: arcmt-test --args -triple x86_64-apple-macosx10.7 -fblocks -fsyntax-only %s > %t
-// RUN: diff %t %s.result
-
-#include "Common.h"
-
-@interface Foo : NSObject
--(Foo *)something;
-@end
-
-void bar(void (^block)(void));
-
-void test1(Foo *p) {
-  __weak Foo *x = p; // __block used just to break cycle.
-  bar(^{
-    [x something];
-  });
-}
-
-void test2(Foo *p) {
-  __block Foo *x; // __block used as output variable.
-  bar(^{
-    x = [p something];
-  });
-}
-
-void test3(Foo *p) {
-  __block Foo *x; // __block used as output variable.
-  bar(^{
-    [x something];
-  });
-  bar(^{
-    x = 0;
-  });
-}
-
-void test4(Foo *p) {
-  __weak Foo *x = p; // __block used just to break cycle.
-  bar(^{
-    [x something];
-  });
-  bar(^{
-    [x something];
-  });
-}
diff --git a/clang/test/ARCMT/safe-arc-assign.m b/clang/test/ARCMT/safe-arc-assign.m
deleted file mode 100644
index 4a0a575794e16..0000000000000
--- a/clang/test/ARCMT/safe-arc-assign.m
+++ /dev/null
@@ -1,14 +0,0 @@
-// RUN: %clang_cc1 -triple x86_64-apple-darwin10 -fsyntax-only -fobjc-arc -x objective-c %s.result
-// RUN: arcmt-test --args -triple x86_64-apple-darwin10 -fsyntax-only -x objective-c %s > %t
-// RUN: diff %t %s.result
-
-void test12(id collection) {
-  for (id x in collection) {
-    x = 0;
-    x = 0;
-  }
-
-  for (__strong id x in collection) {
-    x = 0;
-  }
-}
diff --git a/clang/test/ARCMT/safe-arc-assign.m.result b/clang/test/ARCMT/safe-arc-assign.m.result
deleted file mode 100644
index c25955ea7d1c9..0000000000000
--- a/clang/test/ARCMT/safe-arc-assign.m.result
+++ /dev/null
@@ -1,14 +0,0 @@
-// RUN: %clang_cc1 -triple x86_64-apple-darwin10 -fsyntax-only -fobjc-arc -x objective-c %s.result
-// RUN: arcmt-test --args -triple x86_64-apple-darwin10 -fsyntax-only -x objective-c %s > %t
-// RUN: diff %t %s.result
-
-void test12(id collection) {
-  for (__strong id x in collection) {
-    x = 0;
-    x = 0;
-  }
-
-  for (__strong id x in collection) {
-    x = 0;
-  }
-}
diff --git a/clang/test/ARCMT/verify.m b/clang/test/ARCMT/verify.m
deleted file mode 100644
index 7d245fe80575e..0000000000000
--- a/clang/test/ARCMT/verify.m
+++ /dev/null
@@ -1,17 +0,0 @@
-// RUN: %clang_cc1 -arcmt-action=check -verify %s
-// RUN: not %clang_cc1 -arcmt-action=check -verify %t.invalid 2>&1 | FileCheck %s
-
-#if 0
-// expected-error {{should be ignored}}
-#endif
-
-#error should not be ignored
-// expected-error@-1 {{should not be ignored}}
-
-#error
-// expected-error@-1 {{}}
-
-//      CHECK: error: no expected directives found: consider use of 'expected-no-diagnostics'
-// CHECK-NEXT: error: 'expected-error' diagnostics seen but not expected:
-// CHECK-NEXT:   (frontend): error reading '{{.*}}verify.m.tmp.invalid'
-// CHECK-NEXT: 2 errors generated.
diff --git a/clang/test/ARCMT/with-arc-mode-modify.m b/clang/test/ARCMT/with-arc-mode-modify.m
deleted file mode 100644
index beff6784bbbb0..0000000000000
--- a/clang/test/ARCMT/with-arc-mode-modify.m
+++ /dev/null
@@ -1,13 +0,0 @@
-// RUN: %clang_cc1 -fsyntax-only -fobjc-arc -x objective-c %s.result
-// RUN: cat %s > %t
-// RUN: %clang_cc1 -arcmt-action=modify -fsyntax-only -fobjc-arc -x objective-c %t
-// RUN: diff %t %s.result
-// RUN: rm %t
-
-@protocol NSObject
-- (oneway void)release;
-@end
-
-void test1(id p) {
-  [p release];
-}
diff --git a/clang/test/ARCMT/with-arc-mode-modify.m.result b/clang/test/ARCMT/with-arc-mode-modify.m.result
deleted file mode 100644
index 685226db86541..0000000000000
--- a/clang/test/ARCMT/with-arc-mode-modify.m.result
+++ /dev/null
@@ -1,12 +0,0 @@
-// RUN: %clang_cc1 -fsyntax-only -fobjc-arc -x objective-c %s.result
-// RUN: cat %s > %t
-// RUN: %clang_cc1 -arcmt-action=modify -fsyntax-only -fobjc-arc -x objective-c %t
-// RUN: diff %t %s.result
-// RUN: rm %t
-
-@protocol NSObject
-- (oneway void)release;
-@end
-
-void test1(id p) {
-}
diff --git a/clang/test/AST/ByteCode/builtin-functions.cpp b/clang/test/AST/ByteCode/builtin-functions.cpp
index 7034e8c56bd62..d51b039d40043 100644
--- a/clang/test/AST/ByteCode/builtin-functions.cpp
+++ b/clang/test/AST/ByteCode/builtin-functions.cpp
@@ -1272,6 +1272,22 @@ namespace BuiltinMemcpy {
     return arr[0] * 1000 + arr[1] * 100 + arr[2] * 10 + arr[3];
   }
   static_assert(test_incomplete_array_type() == 1234); // both-error {{constant}} both-note {{in call}}
+
+
+  /// FIXME: memmove needs to support overlapping memory regions.
+  constexpr bool memmoveOverlapping() {
+    char s1[] {1, 2, 3};
+    __builtin_memmove(s1, s1 + 1, 2 * sizeof(char));
+    // Now: 2, 3, 3
+    bool Result1 = (s1[0] == 2 && s1[1] == 3 && s1[2]== 3);
+
+    __builtin_memmove(s1 + 1, s1, 2 * sizeof(char));
+    // Now: 2, 2, 3
+    bool Result2 = (s1[0] == 2 && s1[1] == 2 && s1[2]== 3);
+
+    return Result1 && Result2;
+  }
+  static_assert(memmoveOverlapping()); // expected-error {{failed}}
 }
 
 namespace Memcmp {
diff --git a/clang/test/AST/ByteCode/cxx20.cpp b/clang/test/AST/ByteCode/cxx20.cpp
index 268226a7c143e..6f65fa5c7cfd3 100644
--- a/clang/test/AST/ByteCode/cxx20.cpp
+++ b/clang/test/AST/ByteCode/cxx20.cpp
@@ -1,4 +1,4 @@
-// RUN: %clang_cc1 -fcxx-exceptions -fexperimental-new-constant-interpreter -std=c++20 -verify=both,expected -fcxx-exceptions %s
+// RUN: %clang_cc1 -fcxx-exceptions -fexperimental-new-constant-interpreter -std=c++20 -verify=both,expected -fcxx-exceptions %s -DNEW_INTERP
 // RUN: %clang_cc1 -fcxx-exceptions -std=c++20 -verify=both,ref -fcxx-exceptions %s
 
 void test_alignas_operand() {
@@ -908,3 +908,57 @@ namespace TemporaryInNTTP {
                // expected-note {{created here}}
   B<2> j2; /// Ok.
 }
+
+namespace LocalDestroy {
+  /// This is reduced from a libc++ test case.
+  /// The local f.TI.copied points to the local variable Copied, and we used to
+  /// destroy Copied before f, causing problems later on when a DeadBlock had a
+  /// pointer pointing to it that was already destroyed.
+  struct TrackInitialization {
+    bool *copied_;
+  };
+  struct TrackingPred : TrackInitialization {
+    constexpr TrackingPred(bool *copied) : TrackInitialization(copied) {}
+  };
+  struct F {
+    const TrackingPred &TI;
+  };
+  constexpr int f() {
+    bool Copied = false;
+    TrackingPred TI(&Copied);
+    F f{TI};
+    return 1;
+  }
+  static_assert(f() == 1);
+}
+
+namespace PseudoDtor {
+  constexpr int f1() {
+   using T = int;
+   int a = 0;
+   a.~T();
+   return a; // both-note {{read of object outside its lifetime}}
+  }
+  static_assert(f1() == 0); // both-error {{not an integral constant expression}} \
+                            // both-note {{in call to}}
+
+  constexpr int f2() {
+   using T = int;
+   int a = 0;
+   a.~T();
+   a = 0; // both-note {{assignment to object outside its lifetime}}
+   return a;
+  }
+  static_assert(f2() == 0); // both-error {{not an integral constant expression}} \
+                            // both-note {{in call to}}
+
+#ifdef NEW_INTERP
+  /// FIXME: Currently crashes with the current interpreter, see https://github.com/llvm/llvm-project/issues/53741
+  constexpr int f3() {
+   using T = int;
+   0 .~T();
+   return 0;
+  }
+  static_assert(f3() == 0);
+#endif
+}
diff --git a/clang/test/AST/ByteCode/cxx26.cpp b/clang/test/AST/ByteCode/cxx26.cpp
index 0b0e2b21e8201..cd6b533065010 100644
--- a/clang/test/AST/ByteCode/cxx26.cpp
+++ b/clang/test/AST/ByteCode/cxx26.cpp
@@ -1,10 +1,33 @@
 // RUN: %clang_cc1 -std=c++26 -fsyntax-only -fcxx-exceptions -verify=ref,both %s
 // RUN: %clang_cc1 -std=c++26 -fsyntax-only -fcxx-exceptions -verify=expected,both %s -fexperimental-new-constant-interpreter
 
-// both-no-diagnostics
+namespace std {
+  using size_t = decltype(sizeof(0));
+}
 
 namespace VoidCast {
   constexpr void* p = nullptr;
   constexpr int* q = static_cast<int*>(p);
   static_assert(q == nullptr);
 }
+
+namespace ReplaceableAlloc {
+  struct F {
+    static void* operator new(std::size_t n) {
+      return nullptr; // both-warning {{should not return a null pointer}}
+    }
+  };
+
+  constexpr F *createF() {
+    return new F(); // both-note {{call to class-specific 'operator new'}}
+  }
+
+  constexpr bool foo() {
+    F *f = createF(); // both-note {{in call to}}
+
+    delete f;
+    return true;
+  }
+  static_assert(foo()); // both-error {{not an integral constant expression}} \
+                        // both-note {{in call to}}
+}
diff --git a/clang/test/AST/ByteCode/cxx2a.cpp b/clang/test/AST/ByteCode/cxx2a.cpp
index e478a0ddc4c14..72ef58ca0b1d1 100644
--- a/clang/test/AST/ByteCode/cxx2a.cpp
+++ b/clang/test/AST/ByteCode/cxx2a.cpp
@@ -139,9 +139,7 @@ namespace TypeId {
   static_assert(&B2().ti1 == &typeid(B));
   static_assert(&B2().ti2 == &typeid(B2));
   extern B2 extern_b2;
-  static_assert(&typeid(extern_b2) == &typeid(B2)); // expected-error {{constant expression}} \
-                                                    // expected-note{{typeid applied to object 'extern_b2' whose dynamic type is not constant}}
-
+  static_assert(&typeid(extern_b2) == &typeid(B2));
 
   constexpr B2 b2;
   constexpr const B &b1 = b2;
@@ -170,3 +168,12 @@ namespace TypeId {
   }
   static_assert(side_effects());
 }
+
+consteval int f(int i);
+constexpr bool test(auto i) {
+    return f(0) == 0;
+}
+consteval int f(int i) {
+    return 2 * i;
+}
+static_assert(test(42));
diff --git a/clang/test/AST/ByteCode/literals.cpp b/clang/test/AST/ByteCode/literals.cpp
index b75ca2b19a969..a80ee7ad84fc7 100644
--- a/clang/test/AST/ByteCode/literals.cpp
+++ b/clang/test/AST/ByteCode/literals.cpp
@@ -914,12 +914,18 @@ namespace TypeTraits {
 }
 
 #if __cplusplus >= 201402L
+namespace SomeNS {
+  using MyInt = int;
+}
+
 constexpr int ignoredDecls() {
   static_assert(true, "");
   struct F { int a; };
   enum E { b };
   using A = int;
   typedef int Z;
+  namespace NewNS = SomeNS;
+  using NewNS::MyInt;
 
   return F{12}.a;
 }
diff --git a/clang/test/AST/ByteCode/neon.c b/clang/test/AST/ByteCode/neon.c
new file mode 100644
index 0000000000000..5905d78086039
--- /dev/null
+++ b/clang/test/AST/ByteCode/neon.c
@@ -0,0 +1,19 @@
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +neon -disable-O0-optnone -emit-llvm -o - %s -fexperimental-new-constant-interpreter
+
+// REQUIRES: aarch64-registered-target
+
+/// This just tests that we're not crashing with a non-primitive vector element type.
+
+typedef __mfp8 mfloat8_t;
+typedef __bf16 bfloat16_t;
+
+typedef __attribute__((neon_vector_type(8))) mfloat8_t mfloat8x8_t;
+typedef __attribute__((neon_vector_type(8))) bfloat16_t bfloat16x8_t;
+
+typedef __UINT64_TYPE__ fpm_t;
+#define __ai static __inline__ __attribute__((__always_inline__, __nodebug__))
+__ai __attribute__((target("fp8,neon"))) bfloat16x8_t vcvt1_bf16_mf8_fpm(mfloat8x8_t __p0, fpm_t __p1) {
+  bfloat16x8_t __ret;
+  __ret = (bfloat16x8_t) __builtin_neon_vcvt1_bf16_mf8_fpm(__p0, __p1);
+  return __ret;
+}
diff --git a/clang/test/AST/ByteCode/new-delete.cpp b/clang/test/AST/ByteCode/new-delete.cpp
index a8f073aa03fc1..e60ff894c9715 100644
--- a/clang/test/AST/ByteCode/new-delete.cpp
+++ b/clang/test/AST/ByteCode/new-delete.cpp
@@ -268,11 +268,10 @@ namespace NowThrowNew {
     delete[] p;
     return result;
   }
-  /// This needs support for CXXConstrucExprs with non-constant array sizes.
-  static_assert(erroneous_array_bound_nothrow2(3)); // expected-error {{not an integral constant expression}}
-  static_assert(erroneous_array_bound_nothrow2(0));// expected-error {{not an integral constant expression}}
-  static_assert(erroneous_array_bound_nothrow2(-1) == 0);// expected-error {{not an integral constant expression}}
-  static_assert(!erroneous_array_bound_nothrow2(1LL << 62));// expected-error {{not an integral constant expression}}
+  static_assert(erroneous_array_bound_nothrow2(3));
+  static_assert(erroneous_array_bound_nothrow2(0));
+  static_assert(erroneous_array_bound_nothrow2(-1) == 0);
+  static_assert(!erroneous_array_bound_nothrow2(1LL << 62));
 
   constexpr bool erroneous_array_bound(long long n) {
     delete[] new int[n]; // both-note {{array bound -1 is negative}} both-note {{array bound 4611686018427387904 is too large}}
@@ -857,6 +856,54 @@ struct SS {
 };
 constexpr unsigned short ssmall = SS<unsigned short>(100)[42];
 
+
+
+namespace IncompleteArray {
+  struct A {
+    int b = 10;
+  };
+  constexpr int test1() {
+    int n = 5;
+    int* a = new int[n];
+    int c = a[0]; // both-note {{read of uninitialized object}}
+    delete[] a;
+    return c;
+  }
+  static_assert(test1() == 10); // both-error {{not an integral constant expression}} \
+                                // both-note {{in call to}}
+
+  constexpr int test2() {
+    int n = 0;
+    int* a = new int[n];
+    delete[] a;
+    return 10;
+  }
+  static_assert(test2() == 10);
+
+  /// In this case, the type of the initializer is A[2], while the full size of the
+  /// allocated array is of course 5. The remaining 3 elements need to be initialized
+  /// using A's constructor.
+  constexpr int test3() {
+    int n = 3;
+    A* a = new A[n]{5, 1};
+    int c = a[0].b + a[1].b + a[2].b;
+    delete[] a;
+    return c;
+  }
+  static_assert(test3() == (5 + 1 + 10));
+
+  constexpr int test4() {
+    auto n = 3;
+    int *a = new int[n]{12};
+    int c =  a[0] + a[1];
+    delete[] a;
+    return c;
+  }
+  static_assert(test4() == 12);
+
+
+}
+
 #else
 /// Make sure we reject this prior to C++20
 constexpr int a() { // both-error {{never produces a constant expression}}
diff --git a/clang/test/AST/ByteCode/unions.cpp b/clang/test/AST/ByteCode/unions.cpp
index e90b123c90de0..c6b5e34810f05 100644
--- a/clang/test/AST/ByteCode/unions.cpp
+++ b/clang/test/AST/ByteCode/unions.cpp
@@ -402,7 +402,6 @@ namespace UnionInBase {
   static_assert(return_uninit().a.x == 2);
 }
 
-/// FIXME: Our diagnostic here is a little off.
 namespace One {
   struct A { long x; };
 
@@ -421,4 +420,69 @@ namespace One {
                       // both-note {{constinit}}
 }
 
+namespace CopyAssign {
+  union A {
+    int a;
+    int b;
+  };
+
+  constexpr int f() {
+    A a{12};
+    A b{13};
+
+    b.b = 32;
+    b = a ;
+    return b.a;
+  }
+  static_assert(f()== 12);
+
+
+  constexpr int f2() {
+    A a{12};
+    A b{13};
+
+    b.b = 32;
+    b = a ;
+    return b.b; // both-note {{read of member 'b' of union with active member 'a'}}
+  }
+  static_assert(f2() == 12); // both-error {{not an integral constant expression}} \
+                             // both-note {{in call to}}
+}
+
+namespace MoveAssign {
+  union A {
+    int a;
+    int b;
+  };
+
+  constexpr int f() {
+    A b{13};
+
+    b = A{12} ;
+    return b.a;
+  }
+  static_assert(f()== 12);
+}
+
+namespace IFD {
+  template <class T>
+  struct Optional {
+    struct {
+      union {
+        char null_state;
+        T val;
+      };
+    };
+    constexpr Optional() : null_state(){}
+  };
+
+  constexpr bool test()
+  {
+    Optional<int> opt{};
+    Optional<int> opt2{};
+    opt = opt2;
+    return true;
+  }
+  static_assert(test());
+}
 #endif
diff --git a/clang/test/AST/HLSL/StructuredBuffers-AST.hlsl b/clang/test/AST/HLSL/StructuredBuffers-AST.hlsl
index db3f2d405e686..11be67d45a14c 100644
--- a/clang/test/AST/HLSL/StructuredBuffers-AST.hlsl
+++ b/clang/test/AST/HLSL/StructuredBuffers-AST.hlsl
@@ -1,222 +1,222 @@
-// RUN: %clang_cc1 -triple dxil-pc-shadermodel6.0-library -x hlsl -ast-dump -DEMPTY \
-// RUN:  -DRESOURCE=StructuredBuffer %s | FileCheck -DRESOURCE=StructuredBuffer \
-// RUN:  -check-prefix=EMPTY %s
-//
-// RUN: %clang_cc1 -triple dxil-pc-shadermodel6.0-library -x hlsl -ast-dump \
-// RUN:   -DRESOURCE=StructuredBuffer %s | FileCheck -DRESOURCE=StructuredBuffer \
-// RUN:   -check-prefixes=CHECK,CHECK-SRV,CHECK-SUBSCRIPT,CHECK-LOAD %s
-//
-// RUN: %clang_cc1 -triple dxil-pc-shadermodel6.0-library -x hlsl -ast-dump -DEMPTY \
-// RUN:  -DRESOURCE=RWStructuredBuffer %s | FileCheck -DRESOURCE=RWStructuredBuffer \
-// RUN:  -check-prefix=EMPTY %s
-//
-// RUN: %clang_cc1 -triple dxil-pc-shadermodel6.0-library -x hlsl -ast-dump \
-// RUN:   -DRESOURCE=RWStructuredBuffer %s | FileCheck -DRESOURCE=RWStructuredBuffer \
-// RUN:   -check-prefixes=CHECK,CHECK-UAV,CHECK-SUBSCRIPT,CHECK-COUNTER,CHECK-LOAD %s
-//
-// RUN: %clang_cc1 -triple dxil-pc-shadermodel6.0-library -x hlsl -ast-dump -DEMPTY \
-// RUN:  -DRESOURCE=AppendStructuredBuffer %s | FileCheck -DRESOURCE=AppendStructuredBuffer \
-// RUN:  -check-prefix=EMPTY %s
-//
-// RUN: %clang_cc1 -triple dxil-pc-shadermodel6.0-library -x hlsl -ast-dump \
-// RUN:   -DRESOURCE=AppendStructuredBuffer %s | FileCheck -DRESOURCE=AppendStructuredBuffer \
-// RUN:   -check-prefixes=CHECK,CHECK-UAV,CHECK-NOSUBSCRIPT,CHECK-APPEND %s
-//
-// RUN: %clang_cc1 -triple dxil-pc-shadermodel6.0-library -x hlsl -ast-dump -DEMPTY \
-// RUN:  -DRESOURCE=ConsumeStructuredBuffer %s | FileCheck -DRESOURCE=ConsumeStructuredBuffer \
-// RUN:  -check-prefix=EMPTY %s
-//
-// RUN: %clang_cc1 -triple dxil-pc-shadermodel6.0-library -x hlsl -ast-dump \
-// RUN:   -DRESOURCE=ConsumeStructuredBuffer %s | FileCheck -DRESOURCE=ConsumeStructuredBuffer \
-// RUN:   -check-prefixes=CHECK,CHECK-UAV,CHECK-NOSUBSCRIPT,CHECK-CONSUME %s
-//
-// RUN: %clang_cc1 -triple dxil-pc-shadermodel6.0-library -x hlsl -ast-dump -DEMPTY \
-// RUN:  -DRESOURCE=RasterizerOrderedStructuredBuffer %s | FileCheck -DRESOURCE=RasterizerOrderedStructuredBuffer \
-// RUN:  -check-prefix=EMPTY %s
-//
-// RUN: %clang_cc1 -triple dxil-pc-shadermodel6.0-library -x hlsl -ast-dump \
-// RUN:   -DRESOURCE=RasterizerOrderedStructuredBuffer %s | FileCheck -DRESOURCE=RasterizerOrderedStructuredBuffer \
-// RUN:   -check-prefixes=CHECK,CHECK-UAV,CHECK-ROV,CHECK-SUBSCRIPT,CHECK-LOAD %s
-
-// This test tests two different AST generations for each structured buffer.
-// The "EMPTY" test mode verifies the AST generated by forward declaration
-// of the HLSL types which happens on initializing the HLSL external AST with
-// an AST Context.
-
-// The non-empty mode has a use that requires the resource type be complete,
-// which results in the AST being populated by the external AST source. That
-// case covers the full implementation of the template declaration and the
-// instantiated specialization.
-
-// EMPTY: ClassTemplateDecl 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> <invalid sloc> implicit [[RESOURCE]]
-// EMPTY-NEXT: TemplateTypeParmDecl 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> <invalid sloc> typename depth 0 index 0 element_type
-// EMPTY-NEXT: ConceptSpecializationExpr 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> 'bool' Concept 0x{{[0-9A-Fa-f]+}} '__is_structured_resource_element_compatible'
-// EMPTY-NEXT: ImplicitConceptSpecializationDecl 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> <invalid sloc>
-// EMPTY-NEXT: TemplateArgument type 'type-parameter-0-0'
-// EMPTY-NEXT: TemplateTypeParmType 0x{{[0-9A-Fa-f]+}} 'type-parameter-0-0' dependent depth 0 index 0
-// EMPTY-NEXT: TemplateTypeParm 0x{{[0-9A-Fa-f]+}} ''
-// EMPTY-NEXT: TemplateArgument type 'element_type':'type-parameter-0-0'
-// EMPTY-NEXT: TemplateTypeParmType 0x{{[0-9A-Fa-f]+}} 'element_type' dependent depth 0 index 0
-// EMPTY-NEXT: TemplateTypeParm 0x{{[0-9A-Fa-f]+}} 'element_type'
-// EMPTY-NEXT: CXXRecordDecl 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> <invalid sloc> implicit <undeserialized declarations> class [[RESOURCE]]
-// EMPTY-NEXT: FinalAttr 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> Implicit final
-
-// There should be no more occurrences of [[RESOURCE]]
-// EMPTY-NOT: {{[^[:alnum:]]}}[[RESOURCE]]
-
-#ifndef EMPTY
-
-RESOURCE<float> Buffer;
-
-#endif
-
-// CHECK: ClassTemplateDecl 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> <invalid sloc> implicit [[RESOURCE]]
-// CHECK-NEXT: TemplateTypeParmDecl 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> <invalid sloc> typename depth 0 index 0 element_type
-// CHECK-NEXT: ConceptSpecializationExpr 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> 'bool' Concept 0x{{[0-9A-Fa-f]+}} '__is_structured_resource_element_compatible'
-// CHECK-NEXT: ImplicitConceptSpecializationDecl 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> <invalid sloc>
-// CHECK-NEXT: TemplateArgument type 'type-parameter-0-0'
-// CHECK-NEXT: TemplateTypeParmType 0x{{[0-9A-Fa-f]+}} 'type-parameter-0-0' dependent depth 0 index 0
-// CHECK-NEXT: TemplateTypeParm 0x{{[0-9A-Fa-f]+}} ''
-// CHECK-NEXT: TemplateArgument type 'element_type':'type-parameter-0-0'
-// CHECK-NEXT: TemplateTypeParmType 0x{{[0-9A-Fa-f]+}} 'element_type' dependent depth 0 index 0
-// CHECK-NEXT: TemplateTypeParm 0x{{[0-9A-Fa-f]+}} 'element_type'
-// CHECK-NEXT: CXXRecordDecl 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> <invalid sloc> implicit class [[RESOURCE]] definition
-
-// CHECK: FinalAttr 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> Implicit final
-// CHECK-NEXT: FieldDecl 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> <invalid sloc> implicit __handle '__hlsl_resource_t
-// CHECK-SRV-SAME{LITERAL}: [[hlsl::resource_class(SRV)]]
-// CHECK-UAV-SAME{LITERAL}: [[hlsl::resource_class(UAV)]]
-// CHECK-SAME{LITERAL}: [[hlsl::raw_buffer]]
-// CHECK-SAME{LITERAL}: [[hlsl::contained_type(element_type)]]
-// CHECK-NEXT: HLSLResourceAttr 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> Implicit RawBuffer
-
-// CHECK-SUBSCRIPT: CXXMethodDecl 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> <invalid sloc> operator[] 'const element_type &(unsigned int) const'
-// CHECK-SUBSCRIPT-NEXT: ParmVarDecl 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> <invalid sloc> Index 'unsigned int'
-// CHECK-SUBSCRIPT-NEXT: CompoundStmt 0x{{[0-9A-Fa-f]+}} <<invalid sloc>>
-// CHECK-SUBSCRIPT-NEXT: ReturnStmt 0x{{[0-9A-Fa-f]+}} <<invalid sloc>>
-// CHECK-SUBSCRIPT-NEXT: UnaryOperator 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> 'element_type' prefix '*' cannot overflow
-// CHECK-SUBSCRIPT-NEXT: CallExpr 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> 'element_type *'
-// CHECK-SUBSCRIPT-NEXT: DeclRefExpr 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> '<builtin fn type>' Function 0x{{[0-9A-Fa-f]+}} '__builtin_hlsl_resource_getpointer' 'void (...) noexcept'
-// CHECK-SUBSCRIPT-NEXT: MemberExpr 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> '__hlsl_resource_t
-// CHECK-SUBSCRIPT-SAME{LITERAL}: [[hlsl::resource_class(
-// CHECK-SUBSCRIPT-SAME{LITERAL}: [[hlsl::raw_buffer]]
-// CHECK-SUBSCRIPT-SAME{LITERAL}: [[hlsl::contained_type(element_type)]]
-// CHECK-SUBSCRIPT-SAME: ' lvalue .__handle 0x{{[0-9A-Fa-f]+}}
-// CHECK-SUBSCRIPT-NEXT: CXXThisExpr 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> 'const [[RESOURCE]]<element_type>' lvalue implicit this
-// CHECK-SUBSCRIPT-NEXT: DeclRefExpr 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> 'unsigned int' ParmVar 0x{{[0-9A-Fa-f]+}} 'Index' 'unsigned int'
-// CHECK-SUBSCRIPT-NEXT: AlwaysInlineAttr 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> Implicit always_inline
-
-// CHECK-SUBSCRIPT-NEXT: CXXMethodDecl 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> <invalid sloc> operator[] 'element_type &(unsigned int)'
-// CHECK-SUBSCRIPT-NEXT: ParmVarDecl 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> <invalid sloc> Index 'unsigned int'
-// CHECK-SUBSCRIPT-NEXT: CompoundStmt 0x{{[0-9A-Fa-f]+}} <<invalid sloc>>
-// CHECK-SUBSCRIPT-NEXT: ReturnStmt 0x{{[0-9A-Fa-f]+}} <<invalid sloc>>
-// CHECK-SUBSCRIPT-NEXT: UnaryOperator 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> 'element_type' prefix '*' cannot overflow
-// CHECK-SUBSCRIPT-NEXT: CallExpr 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> 'element_type *'
-// CHECK-SUBSCRIPT-NEXT: DeclRefExpr 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> '<builtin fn type>' Function 0x{{[0-9A-Fa-f]+}} '__builtin_hlsl_resource_getpointer' 'void (...) noexcept'
-// CHECK-SUBSCRIPT-NEXT: MemberExpr 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> '__hlsl_resource_t
-// CHECK-SUBSCRIPT-SAME{LITERAL}: [[hlsl::resource_class(
-// CHECK-SUBSCRIPT-SAME{LITERAL}: [[hlsl::raw_buffer]]
-// CHECK-SUBSCRIPT-SAME{LITERAL}: [[hlsl::contained_type(element_type)]]
-// CHECK-SUBSCRIPT-SAME: ' lvalue .__handle 0x{{[0-9A-Fa-f]+}}
-// CHECK-SUBSCRIPT-NEXT: CXXThisExpr 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> '[[RESOURCE]]<element_type>' lvalue implicit this
-// CHECK-SUBSCRIPT-NEXT: DeclRefExpr 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> 'unsigned int' ParmVar 0x{{[0-9A-Fa-f]+}} 'Index' 'unsigned int'
-// CHECK-SUBSCRIPT-NEXT: AlwaysInlineAttr 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> Implicit always_inline
-
-// CHECK-NOSUBSCRIPT-NOT: CXXMethodDecl 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> <invalid sloc> operator[] 'const element_type &(unsigned int) const'
-// CHECK-NOSUBSCRIPT-NOT: CXXMethodDecl 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> <invalid sloc> operator[] 'element_type &(unsigned int)'
-
-// CHECK-LOAD: CXXMethodDecl 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> <invalid sloc> Load 'element_type (unsigned int)'
-// CHECK-LOAD-NEXT: ParmVarDecl 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> <invalid sloc> Index 'unsigned int'
-// CHECK-LOAD-NEXT: CompoundStmt 0x{{[0-9A-Fa-f]+}} <<invalid sloc>>
-// CHECK-LOAD-NEXT: ReturnStmt 0x{{[0-9A-Fa-f]+}} <<invalid sloc>>
-// CHECK-LOAD-NEXT: UnaryOperator 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> 'element_type' prefix '*' cannot overflow
-// CHECK-LOAD-NEXT: CallExpr 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> 'element_type *'
-// CHECK-LOAD-NEXT: DeclRefExpr 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> '<builtin fn type>' Function 0x{{[0-9A-Fa-f]+}} '__builtin_hlsl_resource_getpointer' 'void (...) noexcept'
-// CHECK-LOAD-NEXT: MemberExpr 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> '__hlsl_resource_t
-// CHECK-LOAD-SAME{LITERAL}: [[hlsl::resource_class(
-// CHECK-LOAD-SAME{LITERAL}: [[hlsl::contained_type(element_type)]]
-// CHECK-LOAD-SAME: ' lvalue .__handle 0x{{[0-9A-Fa-f]+}}
-// CHECK-LOAD-NEXT: CXXThisExpr 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> '[[RESOURCE]]<element_type>' lvalue implicit this
-// CHECK-LOAD-NEXT: DeclRefExpr 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> 'unsigned int' ParmVar 0x{{[0-9A-Fa-f]+}} 'Index' 'unsigned int'
-// CHECK-LOAD-NEXT: AlwaysInlineAttr 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> Implicit always_inline
-
-// CHECK-COUNTER: CXXMethodDecl 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> <invalid sloc> IncrementCounter 'unsigned int ()'
-// CHECK-COUNTER-NEXT: CompoundStmt 0x{{[0-9A-Fa-f]+}} <<invalid sloc>>
-// CHECK-COUNTER-NEXT: ReturnStmt 0x{{[0-9A-Fa-f]+}} <<invalid sloc>>
-// CHECK-COUNTER-NEXT: CallExpr 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> 'unsigned int'
-// CHECK-COUNTER-NEXT: DeclRefExpr 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> '<builtin fn type>' Function 0x{{[0-9A-Fa-f]+}} '__builtin_hlsl_buffer_update_counter' 'unsigned int (...) noexcept'
-// CHECK-COUNTER-NEXT: MemberExpr 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> '__hlsl_resource_t
-// CHECK-COUNTER-SAME{LITERAL}: [[hlsl::resource_class(UAV)]]
-// CHECK-COUNTER-SAME{LITERAL}: [[hlsl::raw_buffer]]
-// CHECK-COUNTER-SAME{LITERAL}: [[hlsl::contained_type(element_type)]]' lvalue .__handle
-// CHECK-COUNTER-NEXT: CXXThisExpr 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> 'RWStructuredBuffer<element_type>' lvalue implicit this
-// CHECK-COUNTER-NEXT: IntegerLiteral 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> 'int' 1
-// CHECK-COUNTER-NEXT: AlwaysInlineAttr 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> Implicit always_inline
-
-// CHECK-COUNTER-NEXT: CXXMethodDecl 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> <invalid sloc> DecrementCounter 'unsigned int ()'
-// CHECK-COUNTER-NEXT: CompoundStmt 0x{{[0-9A-Fa-f]+}} <<invalid sloc>>
-// CHECK-COUNTER-NEXT: ReturnStmt 0x{{[0-9A-Fa-f]+}} <<invalid sloc>>
-// CHECK-COUNTER-NEXT: CallExpr 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> 'unsigned int'
-// CHECK-COUNTER-NEXT: DeclRefExpr 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> '<builtin fn type>' Function 0x{{[0-9A-Fa-f]+}} '__builtin_hlsl_buffer_update_counter' 'unsigned int (...) noexcept'
-// CHECK-COUNTER-NEXT: MemberExpr 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> '__hlsl_resource_t
-// CHECK-COUNTER-SAME{LITERAL}: [[hlsl::resource_class(UAV)]]
-// CHECK-COUNTER-SAME{LITERAL}: [[hlsl::raw_buffer]]
-// CHECK-COUNTER-SAME{LITERAL}: [[hlsl::contained_type(element_type)]]' lvalue .__handle
-// CHECK-COUNTER-NEXT: CXXThisExpr 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> 'RWStructuredBuffer<element_type>' lvalue implicit this
-// CHECK-COUNTER-NEXT: IntegerLiteral 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> 'int' -1
-// CHECK-COUNTER-NEXT: AlwaysInlineAttr 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> Implicit always_inline
-
-// CHECK-APPEND: CXXMethodDecl 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> <invalid sloc> Append 'void (element_type)'
-// CHECK-APPEND-NEXT: ParmVarDecl 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> <invalid sloc> value 'element_type'
-// CHECK-APPEND-NEXT: CompoundStmt 0x{{[0-9A-Fa-f]+}} <<invalid sloc>>
-// CHECK-APPEND-NEXT: BinaryOperator 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> 'element_type' '='
-// CHECK-APPEND-NEXT: UnaryOperator 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> 'element_type' prefix '*' cannot overflow
-// CHECK-APPEND-NEXT: CallExpr 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> 'element_type *'
-// CHECK-APPEND-NEXT: DeclRefExpr 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> '<builtin fn type>' Function 0x{{[0-9A-Fa-f]+}} '__builtin_hlsl_resource_getpointer' 'void (...) noexcept'
-// CHECK-APPEND-NEXT: MemberExpr 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> '__hlsl_resource_t
-// CHECK-APPEND-SAME{LITERAL}: [[hlsl::resource_class(UAV)]]
-// CHECK-APPEND-SAME{LITERAL}: [[hlsl::raw_buffer]]
-// CHECK-APPEND-SAME{LITERAL}: [[hlsl::contained_type(element_type)]]' lvalue .__handle
-// CHECK-APPEND-NEXT: CXXThisExpr 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> '[[RESOURCE]]<element_type>' lvalue implicit this
-// CHECK-APPEND-NEXT: CallExpr 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> 'unsigned int'
-// CHECK-APPEND-NEXT: DeclRefExpr 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> '<builtin fn type>' Function 0x{{[0-9A-Fa-f]+}} '__builtin_hlsl_buffer_update_counter' 'unsigned int (...) noexcept'
-// CHECK-APPEND-NEXT: MemberExpr 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> '__hlsl_resource_t
-// CHECK-APPEND-SAME{LITERAL}: [[hlsl::resource_class(UAV)]]
-// CHECK-APPEND-SAME{LITERAL}: [[hlsl::raw_buffer]]
-// CHECK-APPEND-SAME{LITERAL}: [[hlsl::contained_type(element_type)]]' lvalue .__handle
-// CHECK-APPEND-NEXT: CXXThisExpr 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> '[[RESOURCE]]<element_type>' lvalue implicit this
-// CHECK-APPEND-NEXT: IntegerLiteral 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> 'int' 1
-// CHECK-APPEND-NEXT: DeclRefExpr 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> 'element_type' ParmVar 0x{{[0-9A-Fa-f]+}} 'value' 'element_type'
-
-// CHECK-CONSUME: CXXMethodDecl 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> <invalid sloc> Consume 'element_type ()'
-// CHECK-CONSUME-NEXT: CompoundStmt 0x{{[0-9A-Fa-f]+}} <<invalid sloc>>
-// CHECK-CONSUME-NEXT: ReturnStmt 0x{{[0-9A-Fa-f]+}} <<invalid sloc>>
-// CHECK-CONSUME-NEXT: UnaryOperator 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> 'element_type' prefix '*' cannot overflow
-// CHECK-CONSUME-NEXT: CallExpr 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> 'element_type *'
-// CHECK-CONSUME-NEXT: DeclRefExpr 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> '<builtin fn type>' Function 0x{{[0-9A-Fa-f]+}} '__builtin_hlsl_resource_getpointer' 'void (...) noexcept'
-// CHECK-CONSUME-NEXT: MemberExpr 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> '__hlsl_resource_t
-// CHECK-CONSUME-SAME{LITERAL}: [[hlsl::resource_class(UAV)]]
-// CHECK-CONSUME-SAME{LITERAL}: [[hlsl::raw_buffer]]
-// CHECK-CONSUME-SAME{LITERAL}: [[hlsl::contained_type(element_type)]]' lvalue .__handle
-// CHECK-CONSUME-NEXT: CXXThisExpr 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> '[[RESOURCE]]<element_type>' lvalue implicit this
-// CHECK-CONSUME-NEXT: CallExpr 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> 'unsigned int'
-// CHECK-CONSUME-NEXT: DeclRefExpr 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> '<builtin fn type>' Function 0x{{[0-9A-Fa-f]+}} '__builtin_hlsl_buffer_update_counter' 'unsigned int (...) noexcept'
-// CHECK-CONSUME-NEXT: MemberExpr 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> '__hlsl_resource_t
-// CHECK-CONSUME-SAME{LITERAL}: [[hlsl::resource_class(UAV)]]
-// CHECK-CONSUME-SAME{LITERAL}: [[hlsl::raw_buffer]]
-// CHECK-CONSUME-SAME{LITERAL}: [[hlsl::contained_type(element_type)]]' lvalue .__handle
-// CHECK-CONSUME-NEXT: CXXThisExpr 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> '[[RESOURCE]]<element_type>' lvalue implicit this
-// CHECK-CONSUME-NEXT: IntegerLiteral 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> 'int' -1
-
-// CHECK: ClassTemplateSpecializationDecl 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> <invalid sloc> class [[RESOURCE]] definition
-
-// CHECK: TemplateArgument type 'float'
-// CHECK-NEXT: BuiltinType 0x{{[0-9A-Fa-f]+}} 'float'
-// CHECK-NEXT: FinalAttr 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> Implicit final
-// CHECK-NEXT: FieldDecl 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> <invalid sloc> implicit __handle '__hlsl_resource_t
-// CHECK-SRV-SAME{LITERAL}: [[hlsl::resource_class(SRV)]]
-// CHECK-UAV-SAME{LITERAL}: [[hlsl::resource_class(UAV)]]
-// CHECK-ROV-SAME{LITERAL}: [[hlsl::is_rov]]
-// CHECK-SAME{LITERAL}: [[hlsl::raw_buffer]]
-// CHECK-SAME{LITERAL}: [[hlsl::contained_type(float)]]
-// CHECK-NEXT: HLSLResourceAttr 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> Implicit RawBuffer
+// RUN: %clang_cc1 -triple dxil-pc-shadermodel6.0-library -x hlsl -ast-dump -DEMPTY \
+// RUN:  -DRESOURCE=StructuredBuffer %s | FileCheck -DRESOURCE=StructuredBuffer \
+// RUN:  -check-prefix=EMPTY %s
+//
+// RUN: %clang_cc1 -triple dxil-pc-shadermodel6.0-library -x hlsl -ast-dump \
+// RUN:   -DRESOURCE=StructuredBuffer %s | FileCheck -DRESOURCE=StructuredBuffer \
+// RUN:   -check-prefixes=CHECK,CHECK-SRV,CHECK-SUBSCRIPT,CHECK-LOAD %s
+//
+// RUN: %clang_cc1 -triple dxil-pc-shadermodel6.0-library -x hlsl -ast-dump -DEMPTY \
+// RUN:  -DRESOURCE=RWStructuredBuffer %s | FileCheck -DRESOURCE=RWStructuredBuffer \
+// RUN:  -check-prefix=EMPTY %s
+//
+// RUN: %clang_cc1 -triple dxil-pc-shadermodel6.0-library -x hlsl -ast-dump \
+// RUN:   -DRESOURCE=RWStructuredBuffer %s | FileCheck -DRESOURCE=RWStructuredBuffer \
+// RUN:   -check-prefixes=CHECK,CHECK-UAV,CHECK-SUBSCRIPT,CHECK-COUNTER,CHECK-LOAD %s
+//
+// RUN: %clang_cc1 -triple dxil-pc-shadermodel6.0-library -x hlsl -ast-dump -DEMPTY \
+// RUN:  -DRESOURCE=AppendStructuredBuffer %s | FileCheck -DRESOURCE=AppendStructuredBuffer \
+// RUN:  -check-prefix=EMPTY %s
+//
+// RUN: %clang_cc1 -triple dxil-pc-shadermodel6.0-library -x hlsl -ast-dump \
+// RUN:   -DRESOURCE=AppendStructuredBuffer %s | FileCheck -DRESOURCE=AppendStructuredBuffer \
+// RUN:   -check-prefixes=CHECK,CHECK-UAV,CHECK-NOSUBSCRIPT,CHECK-APPEND %s
+//
+// RUN: %clang_cc1 -triple dxil-pc-shadermodel6.0-library -x hlsl -ast-dump -DEMPTY \
+// RUN:  -DRESOURCE=ConsumeStructuredBuffer %s | FileCheck -DRESOURCE=ConsumeStructuredBuffer \
+// RUN:  -check-prefix=EMPTY %s
+//
+// RUN: %clang_cc1 -triple dxil-pc-shadermodel6.0-library -x hlsl -ast-dump \
+// RUN:   -DRESOURCE=ConsumeStructuredBuffer %s | FileCheck -DRESOURCE=ConsumeStructuredBuffer \
+// RUN:   -check-prefixes=CHECK,CHECK-UAV,CHECK-NOSUBSCRIPT,CHECK-CONSUME %s
+//
+// RUN: %clang_cc1 -triple dxil-pc-shadermodel6.0-library -x hlsl -ast-dump -DEMPTY \
+// RUN:  -DRESOURCE=RasterizerOrderedStructuredBuffer %s | FileCheck -DRESOURCE=RasterizerOrderedStructuredBuffer \
+// RUN:  -check-prefix=EMPTY %s
+//
+// RUN: %clang_cc1 -triple dxil-pc-shadermodel6.0-library -x hlsl -ast-dump \
+// RUN:   -DRESOURCE=RasterizerOrderedStructuredBuffer %s | FileCheck -DRESOURCE=RasterizerOrderedStructuredBuffer \
+// RUN:   -check-prefixes=CHECK,CHECK-UAV,CHECK-ROV,CHECK-SUBSCRIPT,CHECK-LOAD %s
+
+// This test tests two different AST generations for each structured buffer.
+// The "EMPTY" test mode verifies the AST generated by forward declaration
+// of the HLSL types which happens on initializing the HLSL external AST with
+// an AST Context.
+
+// The non-empty mode has a use that requires the resource type be complete,
+// which results in the AST being populated by the external AST source. That
+// case covers the full implementation of the template declaration and the
+// instantiated specialization.
+
+// EMPTY: ClassTemplateDecl 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> <invalid sloc> implicit [[RESOURCE]]
+// EMPTY-NEXT: TemplateTypeParmDecl 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> <invalid sloc> typename depth 0 index 0 element_type
+// EMPTY-NEXT: ConceptSpecializationExpr 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> 'bool' Concept 0x{{[0-9A-Fa-f]+}} '__is_structured_resource_element_compatible'
+// EMPTY-NEXT: ImplicitConceptSpecializationDecl 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> <invalid sloc>
+// EMPTY-NEXT: TemplateArgument type 'type-parameter-0-0'
+// EMPTY-NEXT: TemplateTypeParmType 0x{{[0-9A-Fa-f]+}} 'type-parameter-0-0' dependent depth 0 index 0
+// EMPTY-NEXT: TemplateTypeParm 0x{{[0-9A-Fa-f]+}} depth 0 index 0
+// EMPTY-NEXT: TemplateArgument type 'element_type':'type-parameter-0-0'
+// EMPTY-NEXT: TemplateTypeParmType 0x{{[0-9A-Fa-f]+}} 'element_type' dependent depth 0 index 0
+// EMPTY-NEXT: TemplateTypeParm 0x{{[0-9A-Fa-f]+}} 'element_type'
+// EMPTY-NEXT: CXXRecordDecl 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> <invalid sloc> implicit <undeserialized declarations> class [[RESOURCE]]
+// EMPTY-NEXT: FinalAttr 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> Implicit final
+
+// There should be no more occurrences of [[RESOURCE]]
+// EMPTY-NOT: {{[^[:alnum:]]}}[[RESOURCE]]
+
+#ifndef EMPTY
+
+RESOURCE<float> Buffer;
+
+#endif
+
+// CHECK: ClassTemplateDecl 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> <invalid sloc> implicit [[RESOURCE]]
+// CHECK-NEXT: TemplateTypeParmDecl 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> <invalid sloc> typename depth 0 index 0 element_type
+// CHECK-NEXT: ConceptSpecializationExpr 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> 'bool' Concept 0x{{[0-9A-Fa-f]+}} '__is_structured_resource_element_compatible'
+// CHECK-NEXT: ImplicitConceptSpecializationDecl 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> <invalid sloc>
+// CHECK-NEXT: TemplateArgument type 'type-parameter-0-0'
+// CHECK-NEXT: TemplateTypeParmType 0x{{[0-9A-Fa-f]+}} 'type-parameter-0-0' dependent depth 0 index 0
+// CHECK-NEXT: TemplateTypeParm 0x{{[0-9A-Fa-f]+}} depth 0 index 0
+// CHECK-NEXT: TemplateArgument type 'element_type':'type-parameter-0-0'
+// CHECK-NEXT: TemplateTypeParmType 0x{{[0-9A-Fa-f]+}} 'element_type' dependent depth 0 index 0
+// CHECK-NEXT: TemplateTypeParm 0x{{[0-9A-Fa-f]+}} 'element_type'
+// CHECK-NEXT: CXXRecordDecl 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> <invalid sloc> implicit class [[RESOURCE]] definition
+
+// CHECK: FinalAttr 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> Implicit final
+// CHECK-NEXT: FieldDecl 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> <invalid sloc> implicit __handle '__hlsl_resource_t
+// CHECK-SRV-SAME{LITERAL}: [[hlsl::resource_class(SRV)]]
+// CHECK-UAV-SAME{LITERAL}: [[hlsl::resource_class(UAV)]]
+// CHECK-SAME{LITERAL}: [[hlsl::raw_buffer]]
+// CHECK-SAME{LITERAL}: [[hlsl::contained_type(element_type)]]
+// CHECK-NEXT: HLSLResourceAttr 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> Implicit RawBuffer
+
+// CHECK-SUBSCRIPT: CXXMethodDecl 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> <invalid sloc> operator[] 'const element_type &(unsigned int) const'
+// CHECK-SUBSCRIPT-NEXT: ParmVarDecl 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> <invalid sloc> Index 'unsigned int'
+// CHECK-SUBSCRIPT-NEXT: CompoundStmt 0x{{[0-9A-Fa-f]+}} <<invalid sloc>>
+// CHECK-SUBSCRIPT-NEXT: ReturnStmt 0x{{[0-9A-Fa-f]+}} <<invalid sloc>>
+// CHECK-SUBSCRIPT-NEXT: UnaryOperator 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> 'element_type' prefix '*' cannot overflow
+// CHECK-SUBSCRIPT-NEXT: CallExpr 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> 'element_type *'
+// CHECK-SUBSCRIPT-NEXT: DeclRefExpr 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> '<builtin fn type>' Function 0x{{[0-9A-Fa-f]+}} '__builtin_hlsl_resource_getpointer' 'void (...) noexcept'
+// CHECK-SUBSCRIPT-NEXT: MemberExpr 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> '__hlsl_resource_t
+// CHECK-SUBSCRIPT-SAME{LITERAL}: [[hlsl::resource_class(
+// CHECK-SUBSCRIPT-SAME{LITERAL}: [[hlsl::raw_buffer]]
+// CHECK-SUBSCRIPT-SAME{LITERAL}: [[hlsl::contained_type(element_type)]]
+// CHECK-SUBSCRIPT-SAME: ' lvalue .__handle 0x{{[0-9A-Fa-f]+}}
+// CHECK-SUBSCRIPT-NEXT: CXXThisExpr 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> 'const [[RESOURCE]]<element_type>' lvalue implicit this
+// CHECK-SUBSCRIPT-NEXT: DeclRefExpr 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> 'unsigned int' ParmVar 0x{{[0-9A-Fa-f]+}} 'Index' 'unsigned int'
+// CHECK-SUBSCRIPT-NEXT: AlwaysInlineAttr 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> Implicit always_inline
+
+// CHECK-SUBSCRIPT-NEXT: CXXMethodDecl 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> <invalid sloc> operator[] 'element_type &(unsigned int)'
+// CHECK-SUBSCRIPT-NEXT: ParmVarDecl 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> <invalid sloc> Index 'unsigned int'
+// CHECK-SUBSCRIPT-NEXT: CompoundStmt 0x{{[0-9A-Fa-f]+}} <<invalid sloc>>
+// CHECK-SUBSCRIPT-NEXT: ReturnStmt 0x{{[0-9A-Fa-f]+}} <<invalid sloc>>
+// CHECK-SUBSCRIPT-NEXT: UnaryOperator 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> 'element_type' prefix '*' cannot overflow
+// CHECK-SUBSCRIPT-NEXT: CallExpr 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> 'element_type *'
+// CHECK-SUBSCRIPT-NEXT: DeclRefExpr 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> '<builtin fn type>' Function 0x{{[0-9A-Fa-f]+}} '__builtin_hlsl_resource_getpointer' 'void (...) noexcept'
+// CHECK-SUBSCRIPT-NEXT: MemberExpr 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> '__hlsl_resource_t
+// CHECK-SUBSCRIPT-SAME{LITERAL}: [[hlsl::resource_class(
+// CHECK-SUBSCRIPT-SAME{LITERAL}: [[hlsl::raw_buffer]]
+// CHECK-SUBSCRIPT-SAME{LITERAL}: [[hlsl::contained_type(element_type)]]
+// CHECK-SUBSCRIPT-SAME: ' lvalue .__handle 0x{{[0-9A-Fa-f]+}}
+// CHECK-SUBSCRIPT-NEXT: CXXThisExpr 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> '[[RESOURCE]]<element_type>' lvalue implicit this
+// CHECK-SUBSCRIPT-NEXT: DeclRefExpr 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> 'unsigned int' ParmVar 0x{{[0-9A-Fa-f]+}} 'Index' 'unsigned int'
+// CHECK-SUBSCRIPT-NEXT: AlwaysInlineAttr 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> Implicit always_inline
+
+// CHECK-NOSUBSCRIPT-NOT: CXXMethodDecl 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> <invalid sloc> operator[] 'const element_type &(unsigned int) const'
+// CHECK-NOSUBSCRIPT-NOT: CXXMethodDecl 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> <invalid sloc> operator[] 'element_type &(unsigned int)'
+
+// CHECK-LOAD: CXXMethodDecl 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> <invalid sloc> Load 'element_type (unsigned int)'
+// CHECK-LOAD-NEXT: ParmVarDecl 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> <invalid sloc> Index 'unsigned int'
+// CHECK-LOAD-NEXT: CompoundStmt 0x{{[0-9A-Fa-f]+}} <<invalid sloc>>
+// CHECK-LOAD-NEXT: ReturnStmt 0x{{[0-9A-Fa-f]+}} <<invalid sloc>>
+// CHECK-LOAD-NEXT: UnaryOperator 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> 'element_type' prefix '*' cannot overflow
+// CHECK-LOAD-NEXT: CallExpr 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> 'element_type *'
+// CHECK-LOAD-NEXT: DeclRefExpr 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> '<builtin fn type>' Function 0x{{[0-9A-Fa-f]+}} '__builtin_hlsl_resource_getpointer' 'void (...) noexcept'
+// CHECK-LOAD-NEXT: MemberExpr 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> '__hlsl_resource_t
+// CHECK-LOAD-SAME{LITERAL}: [[hlsl::resource_class(
+// CHECK-LOAD-SAME{LITERAL}: [[hlsl::contained_type(element_type)]]
+// CHECK-LOAD-SAME: ' lvalue .__handle 0x{{[0-9A-Fa-f]+}}
+// CHECK-LOAD-NEXT: CXXThisExpr 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> '[[RESOURCE]]<element_type>' lvalue implicit this
+// CHECK-LOAD-NEXT: DeclRefExpr 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> 'unsigned int' ParmVar 0x{{[0-9A-Fa-f]+}} 'Index' 'unsigned int'
+// CHECK-LOAD-NEXT: AlwaysInlineAttr 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> Implicit always_inline
+
+// CHECK-COUNTER: CXXMethodDecl 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> <invalid sloc> IncrementCounter 'unsigned int ()'
+// CHECK-COUNTER-NEXT: CompoundStmt 0x{{[0-9A-Fa-f]+}} <<invalid sloc>>
+// CHECK-COUNTER-NEXT: ReturnStmt 0x{{[0-9A-Fa-f]+}} <<invalid sloc>>
+// CHECK-COUNTER-NEXT: CallExpr 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> 'unsigned int'
+// CHECK-COUNTER-NEXT: DeclRefExpr 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> '<builtin fn type>' Function 0x{{[0-9A-Fa-f]+}} '__builtin_hlsl_buffer_update_counter' 'unsigned int (...) noexcept'
+// CHECK-COUNTER-NEXT: MemberExpr 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> '__hlsl_resource_t
+// CHECK-COUNTER-SAME{LITERAL}: [[hlsl::resource_class(UAV)]]
+// CHECK-COUNTER-SAME{LITERAL}: [[hlsl::raw_buffer]]
+// CHECK-COUNTER-SAME{LITERAL}: [[hlsl::contained_type(element_type)]]' lvalue .__handle
+// CHECK-COUNTER-NEXT: CXXThisExpr 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> 'RWStructuredBuffer<element_type>' lvalue implicit this
+// CHECK-COUNTER-NEXT: IntegerLiteral 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> 'int' 1
+// CHECK-COUNTER-NEXT: AlwaysInlineAttr 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> Implicit always_inline
+
+// CHECK-COUNTER-NEXT: CXXMethodDecl 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> <invalid sloc> DecrementCounter 'unsigned int ()'
+// CHECK-COUNTER-NEXT: CompoundStmt 0x{{[0-9A-Fa-f]+}} <<invalid sloc>>
+// CHECK-COUNTER-NEXT: ReturnStmt 0x{{[0-9A-Fa-f]+}} <<invalid sloc>>
+// CHECK-COUNTER-NEXT: CallExpr 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> 'unsigned int'
+// CHECK-COUNTER-NEXT: DeclRefExpr 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> '<builtin fn type>' Function 0x{{[0-9A-Fa-f]+}} '__builtin_hlsl_buffer_update_counter' 'unsigned int (...) noexcept'
+// CHECK-COUNTER-NEXT: MemberExpr 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> '__hlsl_resource_t
+// CHECK-COUNTER-SAME{LITERAL}: [[hlsl::resource_class(UAV)]]
+// CHECK-COUNTER-SAME{LITERAL}: [[hlsl::raw_buffer]]
+// CHECK-COUNTER-SAME{LITERAL}: [[hlsl::contained_type(element_type)]]' lvalue .__handle
+// CHECK-COUNTER-NEXT: CXXThisExpr 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> 'RWStructuredBuffer<element_type>' lvalue implicit this
+// CHECK-COUNTER-NEXT: IntegerLiteral 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> 'int' -1
+// CHECK-COUNTER-NEXT: AlwaysInlineAttr 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> Implicit always_inline
+
+// CHECK-APPEND: CXXMethodDecl 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> <invalid sloc> Append 'void (element_type)'
+// CHECK-APPEND-NEXT: ParmVarDecl 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> <invalid sloc> value 'element_type'
+// CHECK-APPEND-NEXT: CompoundStmt 0x{{[0-9A-Fa-f]+}} <<invalid sloc>>
+// CHECK-APPEND-NEXT: BinaryOperator 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> 'element_type' '='
+// CHECK-APPEND-NEXT: UnaryOperator 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> 'element_type' prefix '*' cannot overflow
+// CHECK-APPEND-NEXT: CallExpr 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> 'element_type *'
+// CHECK-APPEND-NEXT: DeclRefExpr 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> '<builtin fn type>' Function 0x{{[0-9A-Fa-f]+}} '__builtin_hlsl_resource_getpointer' 'void (...) noexcept'
+// CHECK-APPEND-NEXT: MemberExpr 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> '__hlsl_resource_t
+// CHECK-APPEND-SAME{LITERAL}: [[hlsl::resource_class(UAV)]]
+// CHECK-APPEND-SAME{LITERAL}: [[hlsl::raw_buffer]]
+// CHECK-APPEND-SAME{LITERAL}: [[hlsl::contained_type(element_type)]]' lvalue .__handle
+// CHECK-APPEND-NEXT: CXXThisExpr 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> '[[RESOURCE]]<element_type>' lvalue implicit this
+// CHECK-APPEND-NEXT: CallExpr 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> 'unsigned int'
+// CHECK-APPEND-NEXT: DeclRefExpr 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> '<builtin fn type>' Function 0x{{[0-9A-Fa-f]+}} '__builtin_hlsl_buffer_update_counter' 'unsigned int (...) noexcept'
+// CHECK-APPEND-NEXT: MemberExpr 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> '__hlsl_resource_t
+// CHECK-APPEND-SAME{LITERAL}: [[hlsl::resource_class(UAV)]]
+// CHECK-APPEND-SAME{LITERAL}: [[hlsl::raw_buffer]]
+// CHECK-APPEND-SAME{LITERAL}: [[hlsl::contained_type(element_type)]]' lvalue .__handle
+// CHECK-APPEND-NEXT: CXXThisExpr 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> '[[RESOURCE]]<element_type>' lvalue implicit this
+// CHECK-APPEND-NEXT: IntegerLiteral 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> 'int' 1
+// CHECK-APPEND-NEXT: DeclRefExpr 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> 'element_type' ParmVar 0x{{[0-9A-Fa-f]+}} 'value' 'element_type'
+
+// CHECK-CONSUME: CXXMethodDecl 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> <invalid sloc> Consume 'element_type ()'
+// CHECK-CONSUME-NEXT: CompoundStmt 0x{{[0-9A-Fa-f]+}} <<invalid sloc>>
+// CHECK-CONSUME-NEXT: ReturnStmt 0x{{[0-9A-Fa-f]+}} <<invalid sloc>>
+// CHECK-CONSUME-NEXT: UnaryOperator 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> 'element_type' prefix '*' cannot overflow
+// CHECK-CONSUME-NEXT: CallExpr 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> 'element_type *'
+// CHECK-CONSUME-NEXT: DeclRefExpr 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> '<builtin fn type>' Function 0x{{[0-9A-Fa-f]+}} '__builtin_hlsl_resource_getpointer' 'void (...) noexcept'
+// CHECK-CONSUME-NEXT: MemberExpr 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> '__hlsl_resource_t
+// CHECK-CONSUME-SAME{LITERAL}: [[hlsl::resource_class(UAV)]]
+// CHECK-CONSUME-SAME{LITERAL}: [[hlsl::raw_buffer]]
+// CHECK-CONSUME-SAME{LITERAL}: [[hlsl::contained_type(element_type)]]' lvalue .__handle
+// CHECK-CONSUME-NEXT: CXXThisExpr 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> '[[RESOURCE]]<element_type>' lvalue implicit this
+// CHECK-CONSUME-NEXT: CallExpr 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> 'unsigned int'
+// CHECK-CONSUME-NEXT: DeclRefExpr 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> '<builtin fn type>' Function 0x{{[0-9A-Fa-f]+}} '__builtin_hlsl_buffer_update_counter' 'unsigned int (...) noexcept'
+// CHECK-CONSUME-NEXT: MemberExpr 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> '__hlsl_resource_t
+// CHECK-CONSUME-SAME{LITERAL}: [[hlsl::resource_class(UAV)]]
+// CHECK-CONSUME-SAME{LITERAL}: [[hlsl::raw_buffer]]
+// CHECK-CONSUME-SAME{LITERAL}: [[hlsl::contained_type(element_type)]]' lvalue .__handle
+// CHECK-CONSUME-NEXT: CXXThisExpr 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> '[[RESOURCE]]<element_type>' lvalue implicit this
+// CHECK-CONSUME-NEXT: IntegerLiteral 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> 'int' -1
+
+// CHECK: ClassTemplateSpecializationDecl 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> <invalid sloc> class [[RESOURCE]] definition
+
+// CHECK: TemplateArgument type 'float'
+// CHECK-NEXT: BuiltinType 0x{{[0-9A-Fa-f]+}} 'float'
+// CHECK-NEXT: FinalAttr 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> Implicit final
+// CHECK-NEXT: FieldDecl 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> <invalid sloc> implicit __handle '__hlsl_resource_t
+// CHECK-SRV-SAME{LITERAL}: [[hlsl::resource_class(SRV)]]
+// CHECK-UAV-SAME{LITERAL}: [[hlsl::resource_class(UAV)]]
+// CHECK-ROV-SAME{LITERAL}: [[hlsl::is_rov]]
+// CHECK-SAME{LITERAL}: [[hlsl::raw_buffer]]
+// CHECK-SAME{LITERAL}: [[hlsl::contained_type(float)]]
+// CHECK-NEXT: HLSLResourceAttr 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> Implicit RawBuffer
diff --git a/clang/test/AST/HLSL/TypdefArrayParam.hlsl b/clang/test/AST/HLSL/TypdefArrayParam.hlsl
new file mode 100644
index 0000000000000..c6ae168f84064
--- /dev/null
+++ b/clang/test/AST/HLSL/TypdefArrayParam.hlsl
@@ -0,0 +1,57 @@
+// RUN: %clang_cc1 -triple dxil-pc-shadermodel6.0-library -finclude-default-header -x hlsl -ast-dump %s | FileCheck %s
+
+typedef uint4 uint32_t4;
+typedef uint32_t4 uint32_t8[2];
+
+// CHECK-LABEL: FunctionDecl {{.*}} used Accumulate 'uint32_t (uint32_t4[2])'
+// CHECK-NEXT: ParmVarDecl {{.*}} used V 'uint32_t4[2]'
+uint32_t Accumulate(uint32_t8 V) {
+  uint32_t4 SumVec = V[0] + V[1];
+  return SumVec.x + SumVec.y + SumVec.z + SumVec.w;
+}
+
+// CHECK-LABEL: FunctionDecl {{.*}} used InOutAccu 'void (inout uint32_t4[2])'
+// CHECK-NEXT: ParmVarDecl {{.*}} used V 'uint32_t4[2]'
+// CHECK-NEXT: HLSLParamModifierAttr {{.*}} inout
+void InOutAccu(inout uint32_t8 V) {
+  uint32_t4 SumVec = V[0] + V[1];
+  V[0] = SumVec;
+}
+
+// CHECK-LABEL: call1
+// CHECK: CallExpr {{.*}} 'void'
+// CHECK-NEXT: ImplicitCastExpr {{.*}} 'void (*)(inout uint32_t4[2])' <FunctionToPointerDecay>
+// CHECK-NEXT: DeclRefExpr {{.*}} 'void (inout uint32_t4[2])' lvalue Function {{.*}} 'InOutAccu' 'void (inout uint32_t4[2])'
+// CHECK-NEXT: HLSLOutArgExpr {{.*}} 'uint32_t4[2]' lvalue inout
+// CHECK-NEXT: OpaqueValueExpr {{.*}} 'uint32_t8':'uint32_t4[2]' lvalue
+// CHECK-NEXT: DeclRefExpr {{.*}} 'uint32_t8':'uint32_t4[2]' lvalue Var {{.*}} 'B' 'uint32_t8':'uint32_t4[2]'
+// CHECK-NEXT: OpaqueValueExpr {{.*}} 'uint32_t4[2]' lvalue
+// CHECK-NEXT: ImplicitCastExpr {{.*}} 'uint32_t4[2]' <HLSLArrayRValue>
+// CHECK-NEXT: OpaqueValueExpr {{.*}} 'uint32_t8':'uint32_t4[2]' lvalue
+// CHECK-NEXT: DeclRefExpr {{.*}} 'uint32_t8':'uint32_t4[2]' lvalue Var {{.*}} 'B' 'uint32_t8':'uint32_t4[2]'
+// CHECK-NEXT: BinaryOperator {{.*}} 'uint32_t8':'uint32_t4[2]' lvalue '='
+// CHECK-NEXT: OpaqueValueExpr {{.*}} 'uint32_t8':'uint32_t4[2]' lvalue
+// CHECK-NEXT: DeclRefExpr {{.*}} 'uint32_t8':'uint32_t4[2]' lvalue Var {{.*}} 'B' 'uint32_t8':'uint32_t4[2]'
+// CHECK-NEXT: ImplicitCastExpr {{.*}} 'uint32_t4[2]' <HLSLArrayRValue>
+// CHECK-NEXT: OpaqueValueExpr {{.*}} 'uint32_t4[2]' lvalue
+// CHECK-NEXT: ImplicitCastExpr {{.*}} 'uint32_t4[2]' <HLSLArrayRValue>
+// CHECK-NEXT: OpaqueValueExpr {{.*}} 'uint32_t8':'uint32_t4[2]' lvalue
+// CHECK-NEXT: DeclRefExpr {{.*}} 'uint32_t8':'uint32_t4[2]' lvalue Var {{.*}} 'B' 'uint32_t8':'uint32_t4[2]'
+void call1() {
+  uint32_t4 A = {1,2,3,4};
+  uint32_t8 B = {A,A};
+  InOutAccu(B);
+}
+
+// CHECK-LABEL: call2
+// CHECK: VarDecl {{.*}} D 'uint32_t':'unsigned int' cinit
+// CHECK-NEXT: CallExpr {{.*}} 'uint32_t':'unsigned int'
+// CHECK-NEXT: ImplicitCastExpr {{.*}} 'uint32_t (*)(uint32_t4[2])' <FunctionToPointerDecay>
+// CHECK-NEXT: DeclRefExpr {{.*}} 'uint32_t (uint32_t4[2])' lvalue Function {{.*}} 'Accumulate' 'uint32_t (uint32_t4[2])'
+// CHECK-NEXT: ImplicitCastExpr {{.*}} 'uint4[2]' <HLSLArrayRValue>
+// CHECK-NEXT: DeclRefExpr {{.*}} 'uint4[2]' lvalue Var {{.*}} 'C' 'uint4[2]'
+void call2() {
+  uint4 A = {1,2,3,4};
+  uint4 C[2] = {A,A};
+  uint32_t D = Accumulate(C);
+}
diff --git a/clang/test/AST/HLSL/TypedBuffers-AST.hlsl b/clang/test/AST/HLSL/TypedBuffers-AST.hlsl
index d6dfb0caba5d9..406ff07d0cf62 100644
--- a/clang/test/AST/HLSL/TypedBuffers-AST.hlsl
+++ b/clang/test/AST/HLSL/TypedBuffers-AST.hlsl
@@ -23,7 +23,7 @@
 // EMPTY-NEXT: ImplicitConceptSpecializationDecl 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> <invalid sloc>
 // EMPTY-NEXT: TemplateArgument type 'type-parameter-0-0'
 // EMPTY-NEXT: TemplateTypeParmType 0x{{[0-9A-Fa-f]+}} 'type-parameter-0-0' dependent depth 0 index 0
-// EMPTY-NEXT: TemplateTypeParm 0x{{[0-9A-Fa-f]+}} ''
+// EMPTY-NEXT: TemplateTypeParm 0x{{[0-9A-Fa-f]+}} depth 0 index 0
 // EMPTY-NEXT: TemplateArgument type 'element_type':'type-parameter-0-0'
 // EMPTY-NEXT: TemplateTypeParmType 0x{{[0-9A-Fa-f]+}} 'element_type' dependent depth 0 index 0
 // EMPTY-NEXT: TemplateTypeParm 0x{{[0-9A-Fa-f]+}} 'element_type'
@@ -45,7 +45,7 @@ RESOURCE<float> Buffer;
 // CHECK-NEXT: ImplicitConceptSpecializationDecl 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> <invalid sloc>
 // CHECK-NEXT: TemplateArgument type 'type-parameter-0-0'
 // CHECK-NEXT: TemplateTypeParmType 0x{{[0-9A-Fa-f]+}} 'type-parameter-0-0' dependent depth 0 index 0
-// CHECK-NEXT: TemplateTypeParm 0x{{[0-9A-Fa-f]+}} ''
+// CHECK-NEXT: TemplateTypeParm 0x{{[0-9A-Fa-f]+}} depth 0 index 0
 // CHECK-NEXT: TemplateArgument type 'element_type':'type-parameter-0-0'
 // CHECK-NEXT: TemplateTypeParmType 0x{{[0-9A-Fa-f]+}} 'element_type' dependent depth 0 index 0
 // CHECK-NEXT: TemplateTypeParm 0x{{[0-9A-Fa-f]+}} 'element_type'
diff --git a/clang/test/AST/HLSL/ast-dump-comment-cbuffer-tbuffer.hlsl b/clang/test/AST/HLSL/ast-dump-comment-cbuffer-tbuffer.hlsl
deleted file mode 100644
index 0bff3ae144037..0000000000000
--- a/clang/test/AST/HLSL/ast-dump-comment-cbuffer-tbuffer.hlsl
+++ /dev/null
@@ -1,62 +0,0 @@
-// RUN: %clang_cc1 -Wdocumentation -ast-dump=json -x hlsl -triple dxil-pc-shadermodel6.3-library %s | FileCheck %s --check-prefix=JSON
-// RUN: %clang_cc1 -Wdocumentation -ast-dump -x hlsl -triple dxil-pc-shadermodel6.3-library %s | FileCheck %s --check-prefix=AST
-
-// JSON:"kind": "HLSLBufferDecl",
-// JSON:"name": "A",
-// JSON-NEXT:"bufferKind": "cbuffer",
-// JSON:"kind": "TextComment",
-// JSON:"text": " CBuffer decl."
-
-/// CBuffer decl.
-cbuffer A {
-    // JSON: "kind": "VarDecl",
-    // JSON: "name": "a",
-    // JSON: "qualType": "float"
-    float a;
-    // JSON: "kind": "VarDecl",
-    // JSON: "name": "b",
-    // JSON: "qualType": "int"
-    int b;
-}
-
-// JSON:"kind": "HLSLBufferDecl",
-// JSON:"name": "B",
-// JSON-NEXT:"bufferKind": "tbuffer",
-// JSON:"kind": "TextComment",
-// JSON:"text": " TBuffer decl."
-
-/// TBuffer decl.
-tbuffer B {
-    // JSON: "kind": "VarDecl",
-    // JSON: "name": "c",
-    // JSON: "qualType": "float"
-    float c;
-    // JSON: "kind": "VarDecl",
-    // JSON: "name": "d",
-    // JSON: "qualType": "int"
-    int d;
-}
-
-// AST: HLSLBufferDecl {{.*}} line:11:9 cbuffer A
-// AST-NEXT: HLSLResourceClassAttr {{.*}} Implicit CBuffer
-// AST-NEXT: HLSLResourceAttr {{.*}} Implicit CBuffer
-// AST-NEXT: FullComment
-// AST-NEXT: ParagraphComment
-// AST-NEXT: TextComment {{.*}} Text=" CBuffer decl."
-// AST-NEXT: VarDecl {{.*}} a 'float'
-// AST-NEXT: VarDecl {{.*}} b 'int'
-// AST-NEXT: CXXRecordDecl {{.*}} implicit class __layout_A definition
-// AST: FieldDecl {{.*}} a 'float'
-// AST-NEXT: FieldDecl {{.*}} b 'int'
-
-// AST-NEXT: HLSLBufferDecl {{.*}} line:29:9 tbuffer B
-// AST-NEXT: HLSLResourceClassAttr {{.*}} Implicit SRV
-// AST-NEXT: HLSLResourceAttr {{.*}} Implicit TBuffer
-// AST-NEXT: FullComment
-// AST-NEXT: ParagraphComment
-// AST-NEXT: TextComment {{.*}} Text=" TBuffer decl."
-// AST-NEXT: VarDecl {{.*}} c 'float'
-// AST-NEXT: VarDecl {{.*}} d 'int'
-// AST-NEXT: CXXRecordDecl {{.*}} implicit class __layout_B definition
-// AST: FieldDecl {{.*}} c 'float'
-// AST-NEXT: FieldDecl {{.*}} d 'int'
diff --git a/clang/test/AST/HLSL/ast-dump-comment-cbuffer.hlsl b/clang/test/AST/HLSL/ast-dump-comment-cbuffer.hlsl
new file mode 100644
index 0000000000000..eca2ed0211d1b
--- /dev/null
+++ b/clang/test/AST/HLSL/ast-dump-comment-cbuffer.hlsl
@@ -0,0 +1,29 @@
+// RUN: %clang_cc1 -Wdocumentation -ast-dump=json -x hlsl -triple dxil-pc-shadermodel6.3-library %s | FileCheck %s --check-prefix=JSON
+// RUN: %clang_cc1 -Wdocumentation -ast-dump -x hlsl -triple dxil-pc-shadermodel6.3-library %s | FileCheck %s --check-prefix=AST
+
+// JSON:"kind": "HLSLBufferDecl",
+// JSON:"name": "A",
+// JSON-NEXT:"bufferKind": "cbuffer",
+// JSON:"kind": "TextComment",
+// JSON:"text": " CBuffer decl."
+
+/// CBuffer decl.
+cbuffer A {
+    // JSON: "kind": "VarDecl",
+    // JSON: "name": "a",
+    // JSON: "qualType": "hlsl_constant float"
+    float a;
+    // JSON: "kind": "VarDecl",
+    // JSON: "name": "b",
+    // JSON: "qualType": "hlsl_constant int"
+    int b;
+}
+
+// AST: HLSLBufferDecl {{.*}} line:11:9 cbuffer A
+// AST-NEXT: HLSLResourceClassAttr {{.*}} Implicit CBuffer
+// AST-NEXT: HLSLResourceAttr {{.*}} Implicit CBuffer
+// AST-NEXT: FullComment
+// AST-NEXT: ParagraphComment
+// AST-NEXT: TextComment {{.*}} Text=" CBuffer decl."
+// AST-NEXT: VarDecl {{.*}} a 'hlsl_constant float'
+// AST-NEXT: VarDecl {{.*}} b 'hlsl_constant int'
diff --git a/clang/test/AST/HLSL/cbuffer.hlsl b/clang/test/AST/HLSL/cbuffer.hlsl
index 721abb290f163..9946fda2355b2 100644
--- a/clang/test/AST/HLSL/cbuffer.hlsl
+++ b/clang/test/AST/HLSL/cbuffer.hlsl
@@ -48,94 +48,108 @@ struct TwoFloats {
 // CHECK: HLSLResourceClassAttr {{.*}} Implicit CBuffer
 // CHECK: HLSLResourceAttr {{.*}} Implicit CBuffer
 cbuffer CB {
-  // CHECK: VarDecl {{.*}} col:9 used a1 'float'
+  // CHECK: VarDecl {{.*}} used a1 'hlsl_constant float'
   float a1;
-  // CHECK: CXXRecordDecl {{.*}} implicit referenced class __layout_CB definition
-  // CHECK: FieldDecl {{.*}} a1 'float'
+  // CHECK: CXXRecordDecl {{.*}} implicit referenced struct __cblayout_CB definition
+  // CHECK: PackedAttr
+  // CHECK-NEXT: FieldDecl {{.*}} a1 'float'
 }
-_Static_assert(__builtin_hlsl_is_scalarized_layout_compatible(OneFloat, __layout_CB), "");
+_Static_assert(__builtin_hlsl_is_scalarized_layout_compatible(OneFloat, __cblayout_CB), "");
 
 // Check that buffer layout struct does not include resources or empty types 
-// CHECK: HLSLBufferDecl {{.*}} line:62:9 cbuffer CB
+// CHECK: HLSLBufferDecl {{.*}} line:[[# @LINE + 3]]:9 cbuffer CB
 // CHECK: HLSLResourceClassAttr {{.*}} Implicit CBuffer
 // CHECK: HLSLResourceAttr {{.*}} Implicit CBuffer
 cbuffer CB {
-  // CHECK: VarDecl {{.*}} col:9 used a2 'float'
+  // CHECK: VarDecl {{.*}} used a2 'hlsl_constant float'
   float a2;
-  // CHECK: VarDecl {{.*}} col:19 b2 'RWBuffer<float>':'hlsl::RWBuffer<float>'
+  // CHECK: VarDecl {{.*}} b2 'RWBuffer<float>':'hlsl::RWBuffer<float>'
   RWBuffer<float> b2; 
-  // CHECK: VarDecl {{.*}} col:15 c2 'EmptyStruct'
+  // CHECK: VarDecl {{.*}} c2 'EmptyStruct'
   EmptyStruct c2;
-  // CHECK: VarDecl {{.*}} col:9 d2 'float[0]'
+  // CHECK: VarDecl {{.*}} d2 'float[0]'
   float d2[0];
-  // CHECK: VarDecl {{.*}} col:9 e2 'float'
+  // CHECK: VarDecl {{.*}} f2 'RWBuffer<float>[2]'
+  RWBuffer<float> f2[2];
+  // CHECK: VarDecl {{.*}} g2 'groupshared float'
+  groupshared float g2;
+  // CHECK: VarDecl {{.*}} h2 '__hlsl_resource_t'
+  __hlsl_resource_t h2;
+  // CHECK: VarDecl {{.*}} e2 'hlsl_constant float'
   float e2;
-  // CHECK: CXXRecordDecl {{.*}} implicit referenced class __layout_CB_1 definition
-  // CHECK: FieldDecl {{.*}} a2 'float'
+  // CHECK: CXXRecordDecl {{.*}} implicit referenced struct __cblayout_CB_1 definition
+  // CHECK: PackedAttr
+  // CHECK-NEXT: FieldDecl {{.*}} a2 'float'
   // CHECK-NEXT: FieldDecl {{.*}} e2 'float'
 }
-_Static_assert(__builtin_hlsl_is_scalarized_layout_compatible(TwoFloats, __layout_CB_1), "");
+_Static_assert(__builtin_hlsl_is_scalarized_layout_compatible(TwoFloats, __cblayout_CB_1), "");
 
 // Check that layout struct is created for B and the empty struct C is removed
-// CHECK: HLSLBufferDecl {{.*}} line:83:9 cbuffer CB
+// CHECK: HLSLBufferDecl {{.*}} line:[[# @LINE + 3]]:9 cbuffer CB
 // CHECK: HLSLResourceClassAttr {{.*}} Implicit CBuffer
 // CHECK: HLSLResourceAttr {{.*}} Implicit CBuffer
 cbuffer CB {
-  // CHECK: VarDecl {{.*}} col:5 used s1 'A'
+  // CHECK: VarDecl {{.*}} used s1 'hlsl_constant A'
   A s1;
-  // CHECK: VarDecl {{.*}} col:5 s2 'B'
+  // CHECK: VarDecl {{.*}} s2 'hlsl_constant B'
   B s2;
-  // CHECK: VarDecl {{.*}} col:12 s3 'CTypedef':'C
+  // CHECK: VarDecl {{.*}} s3 'CTypedef':'C'
   CTypedef s3;
-  // CHECK: CXXRecordDecl {{.*}} implicit referenced class __layout_CB_2 definition
-  // CHECK: FieldDecl {{.*}} s1 'A'
-  // CHECK: FieldDecl {{.*}} s2 '__layout_B'
+  // CHECK: CXXRecordDecl {{.*}} implicit referenced struct __cblayout_CB_2 definition
+  // CHECK: PackedAttr
+  // CHECK-NEXT: FieldDecl {{.*}} s1 'A'
+  // CHECK-NEXT: FieldDecl {{.*}} s2 '__cblayout_B'
 }
-// CHECK: CXXRecordDecl {{.*}} implicit referenced class __layout_B definition
-// CHECK: FieldDecl {{.*}} a 'float'
+// CHECK: CXXRecordDecl {{.*}} implicit referenced struct __cblayout_B definition
+// CHECK: PackedAttr
+// CHECK-NEXT: FieldDecl {{.*}} a 'float'
 
-_Static_assert(__builtin_hlsl_is_scalarized_layout_compatible(OneFloat, __layout_B), "");
-_Static_assert(__builtin_hlsl_is_scalarized_layout_compatible(TwoFloats, __layout_CB_2), "");
+_Static_assert(__builtin_hlsl_is_scalarized_layout_compatible(OneFloat, __cblayout_B), "");
+_Static_assert(__builtin_hlsl_is_scalarized_layout_compatible(TwoFloats, __cblayout_CB_2), "");
 
 // check that layout struct is created for D because of its base struct
-// CHECK: HLSLBufferDecl {{.*}} line:104:9 cbuffer CB
+// CHECK: HLSLBufferDecl {{.*}} line:[[# @LINE + 3]]:9 cbuffer CB
 // CHECK: HLSLResourceClassAttr {{.*}} Implicit CBuffer
 // CHECK: HLSLResourceAttr {{.*}} Implicit CBuffer
 cbuffer CB {
-  // CHECK: VarDecl {{.*}} s4 'D'
+  // CHECK: VarDecl {{.*}} s4 'hlsl_constant D'
   D s4;
-  // CHECK: CXXRecordDecl {{.*}} implicit referenced class __layout_CB_3 definition
-  // CHECK: FieldDecl {{.*}} s4 '__layout_D'
+  // CHECK: CXXRecordDecl {{.*}} implicit referenced struct __cblayout_CB_3 definition
+  // CHECK: PackedAttr
+  // CHECK-NEXT: FieldDecl {{.*}} s4 '__cblayout_D'
 }
-  // CHECK: CXXRecordDecl {{.*}} implicit referenced class __layout_D definition
-  // CHECK: public '__layout_B'
-  // CHECK: FieldDecl {{.*}} b 'float'
-_Static_assert(__builtin_hlsl_is_scalarized_layout_compatible(TwoFloats, __layout_D), "");
-_Static_assert(__builtin_hlsl_is_scalarized_layout_compatible(TwoFloats, __layout_CB_3), "");
+  // CHECK: CXXRecordDecl {{.*}} implicit referenced struct __cblayout_D definition
+  // CHECK: public '__cblayout_B'
+  // CHECK: PackedAttr
+  // CHECK-NEXT: FieldDecl {{.*}} b 'float'
+_Static_assert(__builtin_hlsl_is_scalarized_layout_compatible(TwoFloats, __cblayout_D), "");
+_Static_assert(__builtin_hlsl_is_scalarized_layout_compatible(TwoFloats, __cblayout_CB_3), "");
 
 // check that layout struct is created for E because because its base struct
 // is empty and should be eliminated, and BTypedef should reuse the previously
-// defined '__layout_B' 
-// CHECK: HLSLBufferDecl {{.*}} line:122:9 cbuffer CB
+// defined '__cblayout_B' 
+// CHECK: HLSLBufferDecl {{.*}} line:[[# @LINE + 3]]:9 cbuffer CB
 // CHECK: HLSLResourceClassAttr {{.*}} Implicit CBuffer
 // CHECK: HLSLResourceAttr {{.*}} Implicit CBuffer
 cbuffer CB {
-  // CHECK: VarDecl {{.*}}  s5 'E'
+  // CHECK: VarDecl {{.*}}  s5 'hlsl_constant E'
   E s5;
-  // CHECK: VarDecl {{.*}} s6 'BTypedef':'B'
+  // CHECK: VarDecl {{.*}} s6 'hlsl_constant BTypedef':'hlsl_constant B'
   BTypedef s6;
-  // CHECK: CXXRecordDecl {{.*}}  implicit referenced class __layout_CB_4 definition
-  // CHECK: FieldDecl {{.*}} s5 '__layout_E'
-  // CHECK: FieldDecl {{.*}} s6 '__layout_B'
+  // CHECK: CXXRecordDecl {{.*}}  implicit referenced struct __cblayout_CB_4 definition
+  // CHECK: PackedAttr
+  // CHECK-NEXT: FieldDecl {{.*}} s5 '__cblayout_E'
+  // CHECK-NEXT: FieldDecl {{.*}} s6 '__cblayout_B'
 }
-  // CHECK: CXXRecordDecl {{.*}} implicit referenced class __layout_E definition
-  // CHECK: FieldDecl {{.*}} c 'float'
-  // CHECK-NOT: CXXRecordDecl {{.*}} class __layout_B definition
-_Static_assert(__builtin_hlsl_is_scalarized_layout_compatible(OneFloat, __layout_E), "");
-_Static_assert(__builtin_hlsl_is_scalarized_layout_compatible(TwoFloats, __layout_CB_4), "");
+// CHECK: CXXRecordDecl {{.*}} implicit referenced struct __cblayout_E definition
+// CHECK: PackedAttr
+// CHECK-NEXT: FieldDecl {{.*}} c 'float'
+// CHECK-NOT: CXXRecordDecl {{.*}} struct __cblayout_B definition
+_Static_assert(__builtin_hlsl_is_scalarized_layout_compatible(OneFloat, __cblayout_E), "");
+_Static_assert(__builtin_hlsl_is_scalarized_layout_compatible(TwoFloats, __cblayout_CB_4), "");
 
 // check that this produces empty layout struct
-// CHECK: HLSLBufferDecl {{.*}} line:141:9 cbuffer CB
+// CHECK: HLSLBufferDecl {{.*}} line:[[# @LINE + 3]]:9 cbuffer CB
 // CHECK: HLSLResourceClassAttr {{.*}} Implicit CBuffer
 // CHECK: HLSLResourceAttr {{.*}} Implicit CBuffer
 cbuffer CB {
@@ -149,27 +163,30 @@ cbuffer CB {
   RWBuffer<float> Buf;
   // CHECK: VarDecl {{.*}} ea 'EmptyArrayTypedef':'float[10][0]'
   EmptyArrayTypedef ea;
-  // CHECK: CXXRecordDecl {{.*}} implicit class __layout_CB_5 definition
+  // CHECK: CXXRecordDecl {{.*}} implicit struct __cblayout_CB_5 definition
+  // CHECK: PackedAttr
   // CHECK-NOT: FieldDecl
 }
 
 // check host layout struct with compatible base struct
-// CHECK: HLSLBufferDecl {{.*}} line:160:9 cbuffer CB
+// CHECK: HLSLBufferDecl {{.*}} line:[[# @LINE + 3]]:9 cbuffer CB
 // CHECK: HLSLResourceClassAttr {{.*}} Implicit CBuffer
 // CHECK: HLSLResourceAttr {{.*}} Implicit CBuffer
 cbuffer CB {
-  // CHECK: VarDecl {{.*}} s8 'F'
+  // CHECK: VarDecl {{.*}} s8 'hlsl_constant F'
   F s8;
-  // CHECK: CXXRecordDecl {{.*}} implicit referenced class __layout_CB_6 definition
-  // CHECK: FieldDecl {{.*}} s8 '__layout_F'
+  // CHECK: CXXRecordDecl {{.*}} implicit referenced struct __cblayout_CB_6 definition
+  // CHECK: PackedAttr
+  // CHECK-NEXT: FieldDecl {{.*}} s8 '__cblayout_F'
 }
-  // CHECK: CXXRecordDecl {{.*}} implicit referenced class __layout_F definition
-  // CHECK: public 'A'
-_Static_assert(__builtin_hlsl_is_scalarized_layout_compatible(OneFloat, __layout_F), "");
-_Static_assert(__builtin_hlsl_is_scalarized_layout_compatible(OneFloat, __layout_CB_6), "");
+// CHECK: CXXRecordDecl {{.*}} implicit referenced struct __cblayout_F definition
+// CHECK: public 'A'
+// CHECK: PackedAttr
+_Static_assert(__builtin_hlsl_is_scalarized_layout_compatible(OneFloat, __cblayout_F), "");
+_Static_assert(__builtin_hlsl_is_scalarized_layout_compatible(OneFloat, __cblayout_CB_6), "");
 
 // anonymous structs
-// CHECK: HLSLBufferDecl {{.*}} line:175:9 cbuffer CB
+// CHECK: HLSLBufferDecl {{.*}} line:[[# @LINE + 3]]:9 cbuffer CB
 // CHECK: HLSLResourceClassAttr {{.*}} Implicit CBuffer
 // CHECK: HLSLResourceAttr {{.*}} Implicit CBuffer
 cbuffer CB {
@@ -182,7 +199,7 @@ cbuffer CB {
     // CHECK: FieldDecl {{.*}} f 'RWBuffer<float>':'hlsl::RWBuffer<float>'
     RWBuffer<float> f;
   } s9;
-  // CHECK: VarDecl {{.*}} s9 'struct (unnamed struct at {{.*}}cbuffer.hlsl:177:3
+  // CHECK: VarDecl {{.*}} s9 'hlsl_constant struct (unnamed struct at {{.*}}cbuffer.hlsl:[[# @LINE - 8]]:3
   // CHECK: CXXRecordDecl {{.*}} struct definition
   struct {
     // CHECK: FieldDecl {{.*}} g 'float'
@@ -190,18 +207,21 @@ cbuffer CB {
     // CHECK: FieldDecl {{.*}} f 'RWBuffer<float>':'hlsl::RWBuffer<float>'
     RWBuffer<float> f;
   } s10;
-  // CHECK: VarDecl {{.*}} s10 'struct (unnamed struct at {{.*}}cbuffer.hlsl:187:3
-  // CHECK: CXXRecordDecl {{.*}} implicit referenced class __layout_anon definition
-  // CHECK: FieldDecl {{.*}} e 'float'
-  // CHECK: CXXRecordDecl {{.*}} implicit referenced class __layout_anon_1 definition
-  // CHECK: FieldDecl {{.*}} g 'float'
-  // CHECK: CXXRecordDecl {{.*}} implicit referenced class __layout_CB_7 definition
-  // CHECK: FieldDecl {{.*}} s9 '__layout_anon'
-  // CHECK: FieldDecl {{.*}} s10 '__layout_anon_1'
+  // CHECK: VarDecl {{.*}} s10 'hlsl_constant struct (unnamed struct at {{.*}}cbuffer.hlsl:[[# @LINE - 6]]:3
+  // CHECK: CXXRecordDecl {{.*}} implicit referenced struct __cblayout_anon definition
+  // CHECK: PackedAttr
+  // CHECK-NEXT: FieldDecl {{.*}} e 'float'
+  // CHECK: CXXRecordDecl {{.*}} implicit referenced struct __cblayout_anon_1 definition
+  // CHECK: PackedAttr
+  // CHECK-NEXT: FieldDecl {{.*}} g 'float'
+  // CHECK: CXXRecordDecl {{.*}} implicit referenced struct __cblayout_CB_7 definition
+  // CHECK: PackedAttr
+  // CHECK-NEXT: FieldDecl {{.*}} s9 '__cblayout_anon'
+  // CHECK-NEXT: FieldDecl {{.*}} s10 '__cblayout_anon_1'
 }
-_Static_assert(__builtin_hlsl_is_scalarized_layout_compatible(OneFloat, __layout_anon), "");
-_Static_assert(__builtin_hlsl_is_scalarized_layout_compatible(OneFloat, __layout_anon_1), "");
-_Static_assert(__builtin_hlsl_is_scalarized_layout_compatible(TwoFloats, __layout_CB_7), "");
+_Static_assert(__builtin_hlsl_is_scalarized_layout_compatible(OneFloat, __cblayout_anon), "");
+_Static_assert(__builtin_hlsl_is_scalarized_layout_compatible(OneFloat, __cblayout_anon_1), "");
+_Static_assert(__builtin_hlsl_is_scalarized_layout_compatible(TwoFloats, __cblayout_CB_7), "");
 
 // Add uses for the constant buffer declarations so they are not optimized away
 export float foo() {
diff --git a/clang/test/AST/HLSL/cbuffer_and_namespaces.hlsl b/clang/test/AST/HLSL/cbuffer_and_namespaces.hlsl
index 4b1bbea736f85..09596eda90b6a 100644
--- a/clang/test/AST/HLSL/cbuffer_and_namespaces.hlsl
+++ b/clang/test/AST/HLSL/cbuffer_and_namespaces.hlsl
@@ -19,10 +19,10 @@ namespace NS1 {
       int b;
       EmptyStruct es;
     };
-    // CHECK: CXXRecordDecl {{.*}} implicit class __layout_Foo definition
+    // CHECK: CXXRecordDecl {{.*}} implicit struct __cblayout_Foo definition
     // CHECK: FieldDecl {{.*}} b 'int'
   };
-  // CHECK: CXXRecordDecl {{.*}} implicit class __layout_Foo definition
+  // CHECK: CXXRecordDecl {{.*}} implicit struct __cblayout_Foo definition
   // CHECK: FieldDecl {{.*}} a 'float'
 }
 
@@ -35,18 +35,18 @@ struct Foo {
 // CHECK: HLSLResourceClassAttr {{.*}} Implicit CBuffer
 // CHECK: HLSLResourceAttr {{.*}} Implicit CBuffer
 cbuffer CB1 {
-  // CHECK: VarDecl {{.*}} foo1 'Foo'
+  // CHECK: VarDecl {{.*}} foo1 'hlsl_constant Foo'
   Foo foo1;
-  // CHECK: VarDecl {{.*}} foo2 'NS1::Foo'
+  // CHECK: VarDecl {{.*}} foo2 'hlsl_constant NS1::Foo'
   NS1::Foo foo2;
-  // CHECK: VarDecl {{.*}} foo3 'NS1::Bar::Foo'
+  // CHECK: VarDecl {{.*}} foo3 'hlsl_constant NS1::Bar::Foo'
   NS1::Bar::Foo foo3;
-  // CHECK: CXXRecordDecl {{.*}} implicit referenced class __layout_CB1 definition
-  // CHECK: FieldDecl {{.*}} foo1 '__layout_Foo'
-  // CHECK: FieldDecl {{.*}} foo2 'NS1::__layout_Foo'
-  // CHECK: FieldDecl {{.*}} foo3 'NS1::Bar::__layout_Foo'
+  // CHECK: CXXRecordDecl {{.*}} implicit referenced struct __cblayout_CB1 definition
+  // CHECK: FieldDecl {{.*}} foo1 '__cblayout_Foo'
+  // CHECK: FieldDecl {{.*}} foo2 'NS1::__cblayout_Foo'
+  // CHECK: FieldDecl {{.*}} foo3 'NS1::Bar::__cblayout_Foo'
 }
-// CHECK: CXXRecordDecl {{.*}} implicit class __layout_Foo definition
+// CHECK: CXXRecordDecl {{.*}} implicit struct __cblayout_Foo definition
 // CHECK: FieldDecl {{.*}} c 'double'
 
 struct CB1ExpectedShape {
@@ -54,7 +54,7 @@ struct CB1ExpectedShape {
     float a2;
     int a;
 };
-_Static_assert(__builtin_hlsl_is_scalarized_layout_compatible(CB1ExpectedShape, __layout_CB1), "");
+_Static_assert(__builtin_hlsl_is_scalarized_layout_compatible(CB1ExpectedShape, __cblayout_CB1), "");
 
 namespace NS2 {
   struct Foo { 
@@ -65,21 +65,21 @@ namespace NS2 {
   // CHECK: HLSLResourceClassAttr {{.*}} Implicit CBuffer
   // CHECK: HLSLResourceAttr {{.*}} Implicit CBuffer
   cbuffer CB2 {
-    // CHECK: VarDecl {{.*}} foo0 '::Foo':'Foo'
+    // CHECK: VarDecl {{.*}} foo0 'hlsl_constant ::Foo':'hlsl_constant Foo'
     ::Foo foo0;
-    // CHECK: VarDecl {{.*}} foo1 'Foo':'NS2::Foo'
+    // CHECK: VarDecl {{.*}} foo1 'hlsl_constant Foo':'hlsl_constant NS2::Foo'
     Foo foo1;
-    // CHECK: VarDecl {{.*}} foo2 'NS1::Foo'
+    // CHECK: VarDecl {{.*}} foo2 'hlsl_constant NS1::Foo'
     NS1::Foo foo2;
-    // CHECK: VarDecl {{.*}} foo3 'NS1::Bar::Foo'
+    // CHECK: VarDecl {{.*}} foo3 'hlsl_constant NS1::Bar::Foo'
     NS1::Bar::Foo foo3;
-    // CHECK: CXXRecordDecl {{.*}} implicit referenced class __layout_CB2 definition
-    // CHECK: FieldDecl {{.*}} foo0 '__layout_Foo'
-    // CHECK: FieldDecl {{.*}} foo1 'NS2::__layout_Foo'
-    // CHECK: FieldDecl {{.*}} foo2 'NS1::__layout_Foo'
-    // CHECK: FieldDecl {{.*}} foo3 'NS1::Bar::__layout_Foo'
+    // CHECK: CXXRecordDecl {{.*}} implicit referenced struct __cblayout_CB2 definition
+    // CHECK: FieldDecl {{.*}} foo0 '__cblayout_Foo'
+    // CHECK: FieldDecl {{.*}} foo1 'NS2::__cblayout_Foo'
+    // CHECK: FieldDecl {{.*}} foo2 'NS1::__cblayout_Foo'
+    // CHECK: FieldDecl {{.*}} foo3 'NS1::Bar::__cblayout_Foo'
   }
-  // CHECK: CXXRecordDecl {{.*}} implicit class __layout_Foo definition
+  // CHECK: CXXRecordDecl {{.*}} implicit struct __cblayout_Foo definition
   // CHECK: FieldDecl {{.*}} d 'float[4]'
 }
 
@@ -89,7 +89,7 @@ struct CB2ExpectedShape {
     float a2;
     int a;
 };
-_Static_assert(__builtin_hlsl_is_scalarized_layout_compatible(CB2ExpectedShape, NS2::__layout_CB2), "");
+_Static_assert(__builtin_hlsl_is_scalarized_layout_compatible(CB2ExpectedShape, NS2::__cblayout_CB2), "");
 
 // Add uses for the constant buffer declarations so they are not optimized away
 // CHECK: ExportDecl
diff --git a/clang/test/AST/HLSL/packoffset.hlsl b/clang/test/AST/HLSL/packoffset.hlsl
index 9c928bd6d922e..a9bb90bb386f9 100644
--- a/clang/test/AST/HLSL/packoffset.hlsl
+++ b/clang/test/AST/HLSL/packoffset.hlsl
@@ -6,13 +6,13 @@ cbuffer A
 {
     // CHECK-NEXT:-HLSLResourceClassAttr {{.*}} <<invalid sloc>> Implicit CBuffer
     // CHECK-NEXT:-HLSLResourceAttr {{.*}} <<invalid sloc>> Implicit CBuffer
-    // CHECK-NEXT: VarDecl {{.*}} A1 'float4'
+    // CHECK-NEXT: VarDecl {{.*}} A1 'hlsl_constant float4'
     // CHECK-NEXT: HLSLPackOffsetAttr {{.*}} 0 0
     float4 A1 : packoffset(c);
-    // CHECK-NEXT: VarDecl {{.*}} col:11 A2 'float'
+    // CHECK-NEXT: VarDecl {{.*}} col:11 A2 'hlsl_constant float'
     // CHECK-NEXT: HLSLPackOffsetAttr {{.*}} 1 0
     float A2 : packoffset(c1);
-    // CHECK-NEXT: VarDecl {{.*}} col:11 A3 'float'
+    // CHECK-NEXT: VarDecl {{.*}} col:11 A3 'hlsl_constant float'
     // CHECK-NEXT: HLSLPackOffsetAttr {{.*}} 1 1
     float A3 : packoffset(c1.y);
 }
@@ -20,13 +20,13 @@ cbuffer A
 // CHECK: HLSLBufferDecl {{.*}} cbuffer B
 cbuffer B
 {
-    // CHECK: VarDecl {{.*}} B0 'float'
+    // CHECK: VarDecl {{.*}} B0 'hlsl_constant float'
     // CHECK-NEXT: HLSLPackOffsetAttr {{.*}} 0 1
     float B0 : packoffset(c0.g);
-    // CHECK-NEXT: VarDecl {{.*}} B1 'double'
+    // CHECK-NEXT: VarDecl {{.*}} B1 'hlsl_constant double'
     // CHECK-NEXT: HLSLPackOffsetAttr {{.*}} 0 2
 	double B1 : packoffset(c0.b);
-    // CHECK-NEXT: VarDecl {{.*}} B2 'half'
+    // CHECK-NEXT: VarDecl {{.*}} B2 'hlsl_constant half'
     // CHECK-NEXT: HLSLPackOffsetAttr {{.*}} 0 0
 	half B2 : packoffset(c0.r);
 }
@@ -34,13 +34,13 @@ cbuffer B
 // CHECK: HLSLBufferDecl {{.*}} cbuffer C
 cbuffer C
 {
-    // CHECK: VarDecl {{.*}} C0 'float'
+    // CHECK: VarDecl {{.*}} C0 'hlsl_constant float'
     // CHECK-NEXT: HLSLPackOffsetAttr {{.*}} 1
     float C0 : packoffset(c0.y);
-    // CHECK-NEXT: VarDecl {{.*}} C1 'float2'
+    // CHECK-NEXT: VarDecl {{.*}} C1 'hlsl_constant float2'
     // CHECK-NEXT: HLSLPackOffsetAttr {{.*}} 2
 	float2 C1 : packoffset(c0.z);
-    // CHECK-NEXT: VarDecl {{.*}} C2 'half'
+    // CHECK-NEXT: VarDecl {{.*}} C2 'hlsl_constant half'
     // CHECK-NEXT: HLSLPackOffsetAttr {{.*}} 0
 	half C2 : packoffset(c0.x);
 }
@@ -49,16 +49,16 @@ cbuffer C
 // CHECK: HLSLBufferDecl {{.*}} cbuffer D
 cbuffer D
 {
-    // CHECK: VarDecl {{.*}} D0 'float'
+    // CHECK: VarDecl {{.*}} D0 'hlsl_constant float'
     // CHECK-NEXT: HLSLPackOffsetAttr {{.*}} 0 1
     float D0 : packoffset(c0.y);
-    // CHECK-NEXT: VarDecl {{.*}} D1 'float[2]'
+    // CHECK-NEXT: VarDecl {{.*}} D1 'hlsl_constant float[2]'
     // CHECK-NEXT: HLSLPackOffsetAttr {{.*}} 1 0
 	float D1[2] : packoffset(c1.x);
-    // CHECK-NEXT: VarDecl {{.*}} D2 'half3'
+    // CHECK-NEXT: VarDecl {{.*}} D2 'hlsl_constant half3'
     // CHECK-NEXT: HLSLPackOffsetAttr {{.*}} 2 1
 	half3 D2 : packoffset(c2.y);
-    // CHECK-NEXT: VarDecl {{.*}} D3 'double'
+    // CHECK-NEXT: VarDecl {{.*}} D3 'hlsl_constant double'
     // CHECK-NEXT: HLSLPackOffsetAttr {{.*}} 0 2
 	double D3 : packoffset(c0.z);
 }
@@ -71,13 +71,13 @@ struct ST {
 
 // CHECK: HLSLBufferDecl {{.*}} cbuffer S
 cbuffer S {
-    // CHECK: VarDecl {{.*}} S0 'float'
+    // CHECK: VarDecl {{.*}} S0 'hlsl_constant float'
     // CHECK-NEXT: HLSLPackOffsetAttr {{.*}} 0 1
   float S0 : packoffset(c0.y);
-    // CHECK: VarDecl {{.*}} S1 'ST'
+    // CHECK: VarDecl {{.*}} S1 'hlsl_constant ST'
     // CHECK: HLSLPackOffsetAttr {{.*}} 1 0
   ST S1 : packoffset(c1);
-    // CHECK: VarDecl {{.*}} S2 'double2'
+    // CHECK: VarDecl {{.*}} S2 'hlsl_constant double2'
     // CHECK-NEXT: HLSLPackOffsetAttr {{.*}} 2 0
   double2 S2 : packoffset(c2);
 }
@@ -90,13 +90,13 @@ struct ST2 {
 
 // CHECK: HLSLBufferDecl {{.*}} cbuffer S2
 cbuffer S2 {
-    // CHECK: VarDecl {{.*}} S20 'float'
+    // CHECK: VarDecl {{.*}} S20 'hlsl_constant float'
     // CHECK-NEXT: HLSLPackOffsetAttr {{.*}} 0 3
   float S20 : packoffset(c0.a);
-    // CHECK: VarDecl {{.*}} S21 'ST2'
+    // CHECK: VarDecl {{.*}} S21 'hlsl_constant ST2'
     // CHECK: HLSLPackOffsetAttr {{.*}} 1 0
   ST2 S21 : packoffset(c1);
-    // CHECK: VarDecl {{.*}} S22 'half'
+    // CHECK: VarDecl {{.*}} S22 'hlsl_constant half'
     // CHECK-NEXT: HLSLPackOffsetAttr {{.*}} 3 1
   half S22 : packoffset(c3.y);
 }
diff --git a/clang/test/AST/HLSL/pch_hlsl_buffer.hlsl b/clang/test/AST/HLSL/pch_hlsl_buffer.hlsl
index 3eabbb1f8ae22..754948931ee53 100644
--- a/clang/test/AST/HLSL/pch_hlsl_buffer.hlsl
+++ b/clang/test/AST/HLSL/pch_hlsl_buffer.hlsl
@@ -20,15 +20,15 @@ float foo() {
 // CHECK: HLSLBufferDecl {{.*}} line:7:9 imported <undeserialized declarations> cbuffer A
 // CHECK-NEXT: HLSLResourceClassAttr {{.*}} Implicit CBuffer
 // CHECK-NEXT: HLSLResourceAttr {{.*}} Implicit CBuffer
-// CHECK-NEXT: VarDecl 0x[[A:[0-9a-f]+]] {{.*}} imported used a 'float'
-// CHECK-NEXT: CXXRecordDecl {{.*}} imported implicit <undeserialized declarations> class __layout_A definition
+// CHECK-NEXT: VarDecl 0x[[A:[0-9a-f]+]] {{.*}} imported used a 'hlsl_constant float'
+// CHECK-NEXT: CXXRecordDecl {{.*}} imported implicit <undeserialized declarations> struct __cblayout_A definition
 // CHECK: FieldDecl {{.*}} imported a 'float'
 
 // CHECK: HLSLBufferDecl {{.*}} line:11:9 imported <undeserialized declarations> tbuffer B
 // CHECK-NEXT: HLSLResourceClassAttr {{.*}} Implicit SRV
 // CHECK-NEXT: HLSLResourceAttr {{.*}} Implicit TBuffer
-// CHECK-NEXT: VarDecl 0x[[B:[0-9a-f]+]] {{.*}} imported used b 'float'
-// CHECK-NEXT: CXXRecordDecl 0x{{[0-9a-f]+}} {{.*}} imported implicit <undeserialized declarations> class __layout_B definition
+// CHECK-NEXT: VarDecl 0x[[B:[0-9a-f]+]] {{.*}} imported used b 'hlsl_constant float'
+// CHECK-NEXT: CXXRecordDecl 0x{{[0-9a-f]+}} {{.*}} imported implicit <undeserialized declarations> struct __cblayout_B definition
 // CHECK: FieldDecl 0x{{[0-9a-f]+}} {{.*}} imported b 'float'
 
 // CHECK-NEXT: FunctionDecl {{.*}} line:15:7 imported foo 'float ()'
@@ -36,6 +36,6 @@ float foo() {
 // CHECK-NEXT: ReturnStmt {{.*}}
 // CHECK-NEXT: BinaryOperator {{.*}} 'float' '+'
 // CHECK-NEXT: ImplicitCastExpr {{.*}} 'float' <LValueToRValue>
-// CHECK-NEXT: DeclRefExpr {{.*}} 'float' lvalue Var 0x[[A]] 'a' 'float'
+// CHECK-NEXT: DeclRefExpr {{.*}} 'hlsl_constant float' lvalue Var 0x[[A]] 'a' 'hlsl_constant float'
 // CHECK-NEXT: ImplicitCastExpr {{.*}} 'float' <LValueToRValue>
-// CHECK-NEXT: DeclRefExpr {{.*}} 'float' lvalue Var 0x[[B]] 'b' 'float'
+// CHECK-NEXT: DeclRefExpr {{.*}} 'hlsl_constant float' lvalue Var 0x[[B]] 'b' 'hlsl_constant float'
diff --git a/clang/test/AST/HLSL/resource_binding_attr.hlsl b/clang/test/AST/HLSL/resource_binding_attr.hlsl
index 13957ad3c1fcc..6fac903f75e18 100644
--- a/clang/test/AST/HLSL/resource_binding_attr.hlsl
+++ b/clang/test/AST/HLSL/resource_binding_attr.hlsl
@@ -4,7 +4,7 @@
 // CHECK-NEXT:HLSLResourceClassAttr 0x[[CB:[0-9a-f]+]] {{.*}} Implicit CBuffer
 // CHECK-NEXT:HLSLResourceAttr 0x[[CB:[0-9a-f]+]] {{.*}} Implicit CBuffer
 // CHECK-NEXT:HLSLResourceBindingAttr 0x{{[0-9a-f]+}} <col:14> "b3" "space2"
-// CHECK-NEXT:VarDecl 0x[[A:[0-9a-f]+]] {{.*}} col:9 used a 'float'
+// CHECK-NEXT:VarDecl 0x[[A:[0-9a-f]+]] {{.*}} col:9 used a 'hlsl_constant float'
 cbuffer CB : register(b3, space2) {
   float a;
 }
@@ -13,7 +13,7 @@ cbuffer CB : register(b3, space2) {
 // CHECK-NEXT:HLSLResourceClassAttr 0x[[CB:[0-9a-f]+]] {{.*}} Implicit SRV
 // CHECK-NEXT:HLSLResourceAttr 0x[[CB:[0-9a-f]+]] {{.*}} Implicit TBuffer
 // CHECK-NEXT:HLSLResourceBindingAttr 0x{{[0-9a-f]+}} <col:14> "t2" "space1"
-// CHECK-NEXT:VarDecl 0x[[B:[0-9a-f]+]] {{.*}} col:9 used b 'float'
+// CHECK-NEXT:VarDecl 0x[[B:[0-9a-f]+]] {{.*}} col:9 used b 'hlsl_constant float'
 tbuffer TB : register(t2, space1) {
   float b;
 }
@@ -21,9 +21,9 @@ tbuffer TB : register(t2, space1) {
 float foo() {
 // CHECK: BinaryOperator 0x{{[0-9a-f]+}} <col:10, col:14> 'float' '+'
 // CHECK-NEXT: ImplicitCastExpr 0x{{[0-9a-f]+}} <col:10> 'float' <LValueToRValue>
-// CHECK-NEXT: DeclRefExpr 0x{{[0-9a-f]+}} <col:10> 'float' lvalue Var 0x[[A]] 'a' 'float'
+// CHECK-NEXT: DeclRefExpr 0x{{[0-9a-f]+}} <col:10> 'hlsl_constant float' lvalue Var 0x[[A]] 'a' 'hlsl_constant float'
 // CHECK-NEXT: ImplicitCastExpr 0x{{[0-9a-f]+}} <col:14> 'float' <LValueToRValue>
-// CHECK-NEXT: DeclRefExpr 0x{{[0-9a-f]+}} <col:14> 'float' lvalue Var 0x[[B]] 'b' 'float'
+// CHECK-NEXT: DeclRefExpr 0x{{[0-9a-f]+}} <col:14> 'hlsl_constant float' lvalue Var 0x[[B]] 'b' 'hlsl_constant float'
   return a + b;
 }
 
diff --git a/clang/test/AST/ast-dump-APValue-lvalue.cpp b/clang/test/AST/ast-dump-APValue-lvalue.cpp
new file mode 100644
index 0000000000000..224caddb3eabe
--- /dev/null
+++ b/clang/test/AST/ast-dump-APValue-lvalue.cpp
@@ -0,0 +1,50 @@
+// Test without serialization:
+// RUN: %clang_cc1 -triple x86_64-unknown-unknown -Wno-unused-value -std=gnu++17 \
+// RUN:            -ast-dump %s -ast-dump-filter Test \
+// RUN: | FileCheck --strict-whitespace --match-full-lines %s
+//
+// Test with serialization:
+// RUN: %clang_cc1 -triple x86_64-unknown-unknown -Wno-unused-value -std=gnu++17 -emit-pch -o %t %s
+// RUN: %clang_cc1 -x c++ -triple x86_64-unknown-unknown -Wno-unused-value -std=gnu++17 \
+// RUN:           -include-pch %t -ast-dump-all -ast-dump-filter Test /dev/null \
+// RUN: | sed -e "s/ <undeserialized declarations>//" -e "s/ imported//" \
+// RUN: | FileCheck --strict-whitespace --match-full-lines %s
+
+int i;
+struct S {
+  int i;
+  int ii;
+};
+S s;
+
+struct F {
+  char padding[12];
+  S s;
+};
+F f;
+
+void Test(int (&arr)[10]) {
+  constexpr int *pi = &i;
+  // CHECK:  | `-VarDecl {{.*}} <col:{{.*}}, col:{{.*}}> col:{{.*}} pi 'int *const' constexpr cinit
+  // CHECK-NEXT:  |   |-value: LValue Base=VarDecl {{.*}}, Null=0, Offset=0, HasPath=1, PathLength=0, Path=()
+
+  constexpr int *psi = &s.i;
+  // CHECK:  | `-VarDecl {{.*}} <col:{{.*}}, col:{{.*}}> col:{{.*}} psi 'int *const' constexpr cinit
+  // CHECK-NEXT:  |   |-value: LValue Base=VarDecl {{.*}}, Null=0, Offset=0, HasPath=1, PathLength=1, Path=({{.*}})
+
+  constexpr int *psii = &s.ii;
+  // CHECK:  | `-VarDecl {{.*}} <col:{{.*}}, col:{{.*}}> col:{{.*}} psii 'int *const' constexpr cinit
+  // CHECK-NEXT:  |   |-value: LValue Base=VarDecl {{.*}}, Null=0, Offset=4, HasPath=1, PathLength=1, Path=({{.*}})
+
+  constexpr int *pf = &f.s.ii;
+  // CHECK:  | `-VarDecl {{.*}} <col:{{.*}}, col:{{.*}}> col:{{.*}} pf 'int *const' constexpr cinit
+  // CHECK-NEXT:  |   |-value: LValue Base=VarDecl {{.*}}, Null=0, Offset=16, HasPath=1, PathLength=2, Path=({{.*}}, {{.*}})
+
+  constexpr char *pc = &f.padding[2];
+  // CHECK:  | `-VarDecl {{.*}} <col:{{.*}}, col:{{.*}}> col:{{.*}} pc 'char *const' constexpr cinit
+  // CHECK-NEXT:  |   |-value: LValue Base=VarDecl {{.*}}, Null=0, Offset=2, HasPath=1, PathLength=2, Path=({{.*}}, 2)
+
+  constexpr const int *n = nullptr;
+  // CHECK:    `-VarDecl {{.*}} <col:{{.*}}, col:{{.*}}> col:{{.*}} n 'const int *const' constexpr cinit
+  // CHECK-NEXT:      |-value: LValue Base=null, Null=1, Offset=0, HasPath=1, PathLength=0, Path=()
+}
diff --git a/clang/test/AST/ast-dump-APValue-todo.cpp b/clang/test/AST/ast-dump-APValue-todo.cpp
index 78cc9cf36c73c..acaa82ba53b6f 100644
--- a/clang/test/AST/ast-dump-APValue-todo.cpp
+++ b/clang/test/AST/ast-dump-APValue-todo.cpp
@@ -16,10 +16,6 @@ struct S {
 };
 
 void Test() {
-  constexpr int *pi = &i;
-  // CHECK:  | `-VarDecl {{.*}} <col:{{.*}}, col:{{.*}}> col:{{.*}} pi 'int *const' constexpr cinit
-  // CHECK-NEXT:  |   |-value: LValue <todo>
-
   constexpr int(S::*pmi) = &S::i;
   // CHECK:    `-VarDecl {{.*}} <col:{{.*}}, col:{{.*}}> col:{{.*}} pmi 'int (S::*const)' constexpr cinit
   // CHECK-NEXT:      |-value: MemberPointer <todo>
diff --git a/clang/test/AST/ast-dump-binding-pack.cpp b/clang/test/AST/ast-dump-binding-pack.cpp
new file mode 100644
index 0000000000000..81c75a1268730
--- /dev/null
+++ b/clang/test/AST/ast-dump-binding-pack.cpp
@@ -0,0 +1,74 @@
+// RUN: %clang_cc1 -ast-dump -std=c++26 %s | FileCheck %s
+
+// Test this with PCH.
+// RUN: %clang_cc1 %s -std=c++26 -emit-pch -o %t %s
+// RUN: %clang_cc1 %s -std=c++26 -include-pch %t -ast-dump-all | FileCheck %s
+
+#ifndef PCH_HELPER
+#define PCH_HELPER
+
+template <unsigned N>
+void foo() {
+  int arr[4] = {1, 2, 3, 4};
+  auto [binding_1, ...binding_rest, binding_4] = arr;
+  int arr_2[] = {binding_rest...};
+};
+
+// CHECK-LABEL: FunctionTemplateDecl {{.*}} foo
+// CHECK-LABEL: BindingDecl {{.*}} binding_1
+// CHECK-NEXT: ArraySubscriptExpr {{.*}}
+// CHECK-NEXT: ImplicitCastExpr
+// CHECK-NEXT: DeclRefExpr {{.*}}
+// CHECK-NEXT: IntegerLiteral {{.*}} 'int' 0
+// CHECK-NOT: BindingDecl
+// CHECK-LABEL: BindingDecl {{.*}} binding_rest
+// CHECK-NEXT: ResolvedUnexpandedPackExpr
+// CHECK-NEXT: DeclRefExpr {{.*}} lvalue Binding {{.*}} 'binding_rest'
+// CHECK-NEXT: DeclRefExpr {{.*}} lvalue Binding {{.*}} 'binding_rest'
+// CHECK-NOT: BindingDecl
+// CHECK-LABEL: BindingDecl {{.*}} binding_4
+// CHECK-NEXT: ArraySubscriptExpr
+// CHECK-NEXT: ImplicitCastExpr {{.*}}
+// CHECK-NEXT: DeclRefExpr {{.*}} lvalue Decomposition {{.*}}
+// CHECK-NEXT: IntegerLiteral {{.*}} 'int' 3
+// CHECK-NOT: BindingDecl
+// CHECK-LABEL: VarDecl {{.*}} arr_2
+// CHECK-NEXT: InitListExpr
+// CHECK-NEXT: PackExpansionExpr {{.*}} '<dependent type>' lvalue
+// CHECK-NEXT: DeclRefExpr {{.*}} lvalue Binding {{.*}} 'binding_rest'
+
+struct tag_t { };
+template <unsigned N>
+void bar() {
+  auto [...empty_binding_pack] = tag_t{};
+  static_assert(sizeof...(empty_binding_pack) == 0);
+};
+
+// CHECK-LABEL: FunctionTemplateDecl {{.*}} bar
+// CHECK-NOT: BindingDecl
+// CHECK-LABEL: BindingDecl {{.*}} empty_binding_pack
+// CHECK-NEXT: ResolvedUnexpandedPackExpr
+// CHECK-NOT: DeclRefExpr {{.*}} 'empty_binding_pack'
+// CHECK-NOT: BindingDecl
+// CHECK: DeclStmt
+
+struct int_pair { int x; int y; };
+template <typename T>
+void baz() {
+  auto [binding_1, binding_2, ...empty_binding_pack] = T{};
+  static_assert(sizeof...(empty_binding_pack) == 0);
+};
+
+void(*f)() = baz<int_pair>;
+
+// CHECK-LABEL: FunctionDecl {{.*}} baz {{.*}} implicit_instantiation
+// CHECK-NEXT: TemplateArgument type 'int_pair'
+// CHECK: BindingDecl {{.*}} binding_1
+// CHECK: BindingDecl {{.*}} binding_2
+// CHECK-NOT: BindingDecl
+// CHECK-LABEL: BindingDecl {{.*}} empty_binding_pack
+// CHECK-NEXT: ResolvedUnexpandedPackExpr
+// CHECK-NOT: DeclRefExpr {{.*}} 'empty_binding_pack'
+// CHECK-NOT: BindingDecl
+// CHECK: DeclStmt
+#endif
diff --git a/clang/test/AST/ast-dump-comment.cpp b/clang/test/AST/ast-dump-comment.cpp
index 9798295b420f9..40c3edb62821b 100644
--- a/clang/test/AST/ast-dump-comment.cpp
+++ b/clang/test/AST/ast-dump-comment.cpp
@@ -91,6 +91,19 @@ int Test_HTMLTagComment;
 // CHECK-NEXT:       TextComment{{.*}} Text=" "
 // CHECK-NEXT:       HTMLStartTagComment{{.*}} Name="br" SelfClosing
 
+/// <a
+///     href="foo"
+/// >Aaa</a>b
+int Test_HTMLTagMultilineBCPL;
+// CHECK:      VarDecl{{.*}}Test_HTMLTagMultilineBCPL
+// CHECK-NEXT:   FullComment
+// CHECK-NEXT:     ParagraphComment
+// CHECK-NEXT:       TextComment{{.*}} Text=" "
+// CHECK-NEXT:       HTMLStartTagComment{{.*}} Name="a" Attrs:  "href="foo"
+// CHECK-NEXT:       TextComment{{.*}} Text="Aaa"
+// CHECK-NEXT:       HTMLEndTagComment{{.*}} Name="a"
+// CHECK-NEXT:       TextComment{{.*}} Text="b"
+
 /// \verbatim
 /// Aaa
 /// \endverbatim
diff --git a/clang/test/AST/ast-dump-decl.c b/clang/test/AST/ast-dump-decl.c
index 28b58c8eb648c..683df50f7e91c 100644
--- a/clang/test/AST/ast-dump-decl.c
+++ b/clang/test/AST/ast-dump-decl.c
@@ -121,7 +121,7 @@ struct testIndirectFieldDecl {
   };
 };
 // CHECK:      IndirectFieldDecl{{.*}} TestIndirectFieldDecl 'int'
-// CHECK-NEXT:   Field{{.*}} ''
+// CHECK-NEXT:   Field{{.*}} field_index 0
 // CHECK-NEXT:   Field{{.*}} 'TestIndirectFieldDecl'
 
 // FIXME: It would be nice to dump the enum and its enumerators.
diff --git a/clang/test/AST/ast-dump-records.c b/clang/test/AST/ast-dump-records.c
index c0fbac67bc491..f4a540f3fa872 100644
--- a/clang/test/AST/ast-dump-records.c
+++ b/clang/test/AST/ast-dump-records.c
@@ -58,10 +58,10 @@ struct C {
   };
   // CHECK-NEXT: FieldDecl 0x{{[^ ]*}} <line:[[@LINE-7]]:3> col:3 implicit 'union C::(anonymous at {{.*}}:[[@LINE-7]]:3)'
   // CHECK-NEXT: IndirectFieldDecl 0x{{[^ ]*}} <line:[[@LINE-6]]:9> col:9 implicit c 'int'
-  // CHECK-NEXT: Field 0x{{[^ ]*}} '' 'union C::(anonymous at {{.*}}:[[@LINE-9]]:3)'
+  // CHECK-NEXT: Field 0x{{[^ ]*}} field_index 1 'union C::(anonymous at {{.*}}:[[@LINE-9]]:3)'
   // CHECK-NEXT: Field 0x{{[^ ]*}} 'c' 'int'
   // CHECK-NEXT: IndirectFieldDecl 0x{{[^ ]*}} <line:[[@LINE-7]]:11> col:11 implicit d 'float'
-  // CHECK-NEXT: Field 0x{{[^ ]*}} '' 'union C::(anonymous at {{.*}}:[[@LINE-12]]:3)'
+  // CHECK-NEXT: Field 0x{{[^ ]*}} field_index 1 'union C::(anonymous at {{.*}}:[[@LINE-12]]:3)'
   // CHECK-NEXT: Field 0x{{[^ ]*}} 'd' 'float'
 
   struct {
@@ -72,10 +72,10 @@ struct C {
   };
   // CHECK-NEXT: FieldDecl 0x{{[^ ]*}} <line:[[@LINE-6]]:3> col:3 implicit 'struct C::(anonymous at {{.*}}:[[@LINE-6]]:3)'
   // CHECK-NEXT: IndirectFieldDecl 0x{{[^ ]*}} <line:[[@LINE-5]]:9> col:9 implicit e 'int'
-  // CHECK-NEXT: Field 0x{{[^ ]*}} '' 'struct C::(anonymous at {{.*}}:[[@LINE-8]]:3)'
+  // CHECK-NEXT: Field 0x{{[^ ]*}} field_index 2 'struct C::(anonymous at {{.*}}:[[@LINE-8]]:3)'
   // CHECK-NEXT: Field 0x{{[^ ]*}} 'e' 'int'
   // CHECK-NEXT: IndirectFieldDecl 0x{{[^ ]*}} <col:12> col:12 implicit f 'int'
-  // CHECK-NEXT: Field 0x{{[^ ]*}} '' 'struct C::(anonymous at {{.*}}:[[@LINE-11]]:3)'
+  // CHECK-NEXT: Field 0x{{[^ ]*}} field_index 2 'struct C::(anonymous at {{.*}}:[[@LINE-11]]:3)'
   // CHECK-NEXT: Field 0x{{[^ ]*}} 'f' 'int'
 };
 
@@ -141,10 +141,10 @@ union G {
   };
   // CHECK-NEXT: FieldDecl 0x{{[^ ]*}} <line:[[@LINE-7]]:3> col:3 implicit 'union G::(anonymous at {{.*}}:[[@LINE-7]]:3)'
   // CHECK-NEXT: IndirectFieldDecl 0x{{[^ ]*}} <line:[[@LINE-6]]:9> col:9 implicit c 'int'
-  // CHECK-NEXT: Field 0x{{[^ ]*}} '' 'union G::(anonymous at {{.*}}:[[@LINE-9]]:3)'
+  // CHECK-NEXT: Field 0x{{[^ ]*}} field_index 1 'union G::(anonymous at {{.*}}:[[@LINE-9]]:3)'
   // CHECK-NEXT: Field 0x{{[^ ]*}} 'c' 'int'
   // CHECK-NEXT: IndirectFieldDecl 0x{{[^ ]*}} <line:[[@LINE-7]]:11> col:11 implicit d 'float'
-  // CHECK-NEXT: Field 0x{{[^ ]*}} '' 'union G::(anonymous at {{.*}}:[[@LINE-12]]:3)'
+  // CHECK-NEXT: Field 0x{{[^ ]*}} field_index 1 'union G::(anonymous at {{.*}}:[[@LINE-12]]:3)'
   // CHECK-NEXT: Field 0x{{[^ ]*}} 'd' 'float'
 
   struct {
@@ -155,10 +155,10 @@ union G {
   };
   // CHECK-NEXT: FieldDecl 0x{{[^ ]*}} <line:[[@LINE-6]]:3> col:3 implicit 'struct G::(anonymous at {{.*}}:[[@LINE-6]]:3)'
   // CHECK-NEXT: IndirectFieldDecl 0x{{[^ ]*}} <line:[[@LINE-5]]:9> col:9 implicit e 'int'
-  // CHECK-NEXT: Field 0x{{[^ ]*}} '' 'struct G::(anonymous at {{.*}}:[[@LINE-8]]:3)'
+  // CHECK-NEXT: Field 0x{{[^ ]*}} field_index 2 'struct G::(anonymous at {{.*}}:[[@LINE-8]]:3)'
   // CHECK-NEXT: Field 0x{{[^ ]*}} 'e' 'int'
   // CHECK-NEXT: IndirectFieldDecl 0x{{[^ ]*}} <col:12> col:12 implicit f 'int'
-  // CHECK-NEXT: Field 0x{{[^ ]*}} '' 'struct G::(anonymous at {{.*}}:[[@LINE-11]]:3)'
+  // CHECK-NEXT: Field 0x{{[^ ]*}} field_index 2 'struct G::(anonymous at {{.*}}:[[@LINE-11]]:3)'
   // CHECK-NEXT: Field 0x{{[^ ]*}} 'f' 'int'
 };
 
diff --git a/clang/test/AST/ast-dump-records.cpp b/clang/test/AST/ast-dump-records.cpp
index bfd8892698d4b..e9b37b73002dd 100644
--- a/clang/test/AST/ast-dump-records.cpp
+++ b/clang/test/AST/ast-dump-records.cpp
@@ -90,10 +90,10 @@ struct C {
   };
   // CHECK-NEXT: FieldDecl 0x{{[^ ]*}} <line:[[@LINE-14]]:3> col:3 implicit 'C::(anonymous union at {{.*}}:[[@LINE-14]]:3)'
   // CHECK-NEXT: IndirectFieldDecl 0x{{[^ ]*}} <line:[[@LINE-6]]:9> col:9 implicit c 'int'
-  // CHECK-NEXT: Field 0x{{[^ ]*}} '' 'C::(anonymous union at {{.*}}:[[@LINE-16]]:3)'
+  // CHECK-NEXT: Field 0x{{[^ ]*}} field_index 1 'C::(anonymous union at {{.*}}:[[@LINE-16]]:3)'
   // CHECK-NEXT: Field 0x{{[^ ]*}} 'c' 'int'
   // CHECK-NEXT: IndirectFieldDecl 0x{{[^ ]*}} <line:[[@LINE-7]]:11> col:11 implicit d 'float'
-  // CHECK-NEXT: Field 0x{{[^ ]*}} '' 'C::(anonymous union at {{.*}}:[[@LINE-19]]:3)'
+  // CHECK-NEXT: Field 0x{{[^ ]*}} field_index 1 'C::(anonymous union at {{.*}}:[[@LINE-19]]:3)'
   // CHECK-NEXT: Field 0x{{[^ ]*}} 'd' 'float'
 
   struct {
@@ -111,10 +111,10 @@ struct C {
   };
   // CHECK-NEXT: FieldDecl 0x{{[^ ]*}} <line:[[@LINE-13]]:3> col:3 implicit 'C::(anonymous struct at {{.*}}:[[@LINE-13]]:3)'
   // CHECK-NEXT: IndirectFieldDecl 0x{{[^ ]*}} <line:[[@LINE-5]]:9> col:9 implicit e 'int'
-  // CHECK-NEXT: Field 0x{{[^ ]*}} '' 'C::(anonymous struct at {{.*}}:[[@LINE-15]]:3)'
+  // CHECK-NEXT: Field 0x{{[^ ]*}} field_index 2 'C::(anonymous struct at {{.*}}:[[@LINE-15]]:3)'
   // CHECK-NEXT: Field 0x{{[^ ]*}} 'e' 'int'
   // CHECK-NEXT: IndirectFieldDecl 0x{{[^ ]*}} <col:12> col:12 implicit f 'int'
-  // CHECK-NEXT: Field 0x{{[^ ]*}} '' 'C::(anonymous struct at {{.*}}:[[@LINE-18]]:3)'
+  // CHECK-NEXT: Field 0x{{[^ ]*}} field_index 2 'C::(anonymous struct at {{.*}}:[[@LINE-18]]:3)'
   // CHECK-NEXT: Field 0x{{[^ ]*}} 'f' 'int'
 };
 
@@ -223,10 +223,10 @@ union G {
   };
   // CHECK-NEXT: FieldDecl 0x{{[^ ]*}} <line:[[@LINE-15]]:3> col:3 implicit 'G::(anonymous union at {{.*}}:[[@LINE-15]]:3)'
   // CHECK-NEXT: IndirectFieldDecl 0x{{[^ ]*}} <line:[[@LINE-6]]:9> col:9 implicit c 'int'
-  // CHECK-NEXT: Field 0x{{[^ ]*}} '' 'G::(anonymous union at {{.*}}:[[@LINE-17]]:3)'
+  // CHECK-NEXT: Field 0x{{[^ ]*}} field_index 1 'G::(anonymous union at {{.*}}:[[@LINE-17]]:3)'
   // CHECK-NEXT: Field 0x{{[^ ]*}} 'c' 'int'
   // CHECK-NEXT: IndirectFieldDecl 0x{{[^ ]*}} <line:[[@LINE-7]]:11> col:11 implicit d 'float'
-  // CHECK-NEXT: Field 0x{{[^ ]*}} '' 'G::(anonymous union at {{.*}}:[[@LINE-20]]:3)'
+  // CHECK-NEXT: Field 0x{{[^ ]*}} field_index 1 'G::(anonymous union at {{.*}}:[[@LINE-20]]:3)'
   // CHECK-NEXT: Field 0x{{[^ ]*}} 'd' 'float'
 
   struct {
@@ -245,10 +245,10 @@ union G {
   };
   // CHECK-NEXT: FieldDecl 0x{{[^ ]*}} <line:[[@LINE-14]]:3> col:3 implicit 'G::(anonymous struct at {{.*}}:[[@LINE-14]]:3)'
   // CHECK-NEXT: IndirectFieldDecl 0x{{[^ ]*}} <line:[[@LINE-5]]:9> col:9 implicit e 'int'
-  // CHECK-NEXT: Field 0x{{[^ ]*}} '' 'G::(anonymous struct at {{.*}}:[[@LINE-16]]:3)'
+  // CHECK-NEXT: Field 0x{{[^ ]*}} field_index 2 'G::(anonymous struct at {{.*}}:[[@LINE-16]]:3)'
   // CHECK-NEXT: Field 0x{{[^ ]*}} 'e' 'int'
   // CHECK-NEXT: IndirectFieldDecl 0x{{[^ ]*}} <col:12> col:12 implicit f 'int'
-  // CHECK-NEXT: Field 0x{{[^ ]*}} '' 'G::(anonymous struct at {{.*}}:[[@LINE-19]]:3)'
+  // CHECK-NEXT: Field 0x{{[^ ]*}} field_index 2 'G::(anonymous struct at {{.*}}:[[@LINE-19]]:3)'
   // CHECK-NEXT: Field 0x{{[^ ]*}} 'f' 'int'
 };
 
diff --git a/clang/test/AST/ast-dump-templates.cpp b/clang/test/AST/ast-dump-templates.cpp
index 9fcafbcbcc46b..86af8c50f3174 100644
--- a/clang/test/AST/ast-dump-templates.cpp
+++ b/clang/test/AST/ast-dump-templates.cpp
@@ -1,7 +1,15 @@
-// RUN: %clang_cc1 -std=c++1z -ast-print %s > %t
+// RUN: %clang_cc1 -triple x86_64-unknown-unknown -std=c++17 -ast-dump=json %s | FileCheck --check-prefix=JSON %s
+// RUN: %clang_cc1 -triple x86_64-unknown-unknown -std=c++17 -ast-print %s > %t
 // RUN: FileCheck < %t %s -check-prefix=CHECK1
 // RUN: FileCheck < %t %s -check-prefix=CHECK2
-// RUN: %clang_cc1 -std=c++1z -ast-dump %s | FileCheck --check-prefix=DUMP %s
+// RUN: %clang_cc1 -triple x86_64-unknown-unknown -std=c++17 -ast-dump %s | FileCheck --check-prefix=DUMP %s
+
+// Test with serialization:
+// RUN: %clang_cc1 -triple x86_64-unknown-unknown -std=c++17 -emit-pch -o %t %s
+// RUN: %clang_cc1 -triple x86_64-unknown-unknown -x c++ -std=c++17 -include-pch %t \
+// RUN: -ast-dump-all /dev/null \
+// RUN: | sed -e "s/ <undeserialized declarations>//" -e "s/ imported//" \
+// RUN: | FileCheck --strict-whitespace --check-prefix=DUMP %s
 
 template <int X, typename Y, int Z = 5>
 struct foo {
@@ -118,3 +126,6038 @@ void func() {
 // DUMP-NEXT:     `-TemplateTypeParm {{.*}} 'Key'
 }
 }
+
+namespace test7 {
+  template <template<class> class TT> struct A {};
+  template <class...> class B {};
+  template struct A<B>;
+// DUMP-LABEL: NamespaceDecl {{.*}} test7{{$}}
+// DUMP:       ClassTemplateSpecializationDecl {{.*}} struct A definition explicit_instantiation_definition strict-pack-match{{$}}
+} // namespce test7
+
+// NOTE: CHECK lines have been autogenerated by gen_ast_dump_json_test.py
+
+
+// JSON-NOT: {{^}}Dumping
+// JSON:  "kind": "TranslationUnitDecl",
+// JSON-NEXT:  "loc": {},
+// JSON-NEXT:  "range": {
+// JSON-NEXT:   "begin": {},
+// JSON-NEXT:   "end": {}
+// JSON-NEXT:  },
+// JSON-NEXT:  "inner": [
+// JSON-NEXT:   {
+// JSON-NEXT:    "id": "0x{{.*}}",
+// JSON-NEXT:    "kind": "TypedefDecl",
+// JSON-NEXT:    "loc": {},
+// JSON-NEXT:    "range": {
+// JSON-NEXT:     "begin": {},
+// JSON-NEXT:     "end": {}
+// JSON-NEXT:    },
+// JSON-NEXT:    "isImplicit": true,
+// JSON-NEXT:    "name": "__int128_t",
+// JSON-NEXT:    "type": {
+// JSON-NEXT:     "qualType": "__int128"
+// JSON-NEXT:    },
+// JSON-NEXT:    "inner": [
+// JSON-NEXT:     {
+// JSON-NEXT:      "id": "0x{{.*}}",
+// JSON-NEXT:      "kind": "BuiltinType",
+// JSON-NEXT:      "type": {
+// JSON-NEXT:       "qualType": "__int128"
+// JSON-NEXT:      }
+// JSON-NEXT:     }
+// JSON-NEXT:    ]
+// JSON-NEXT:   },
+// JSON-NEXT:   {
+// JSON-NEXT:    "id": "0x{{.*}}",
+// JSON-NEXT:    "kind": "TypedefDecl",
+// JSON-NEXT:    "loc": {},
+// JSON-NEXT:    "range": {
+// JSON-NEXT:     "begin": {},
+// JSON-NEXT:     "end": {}
+// JSON-NEXT:    },
+// JSON-NEXT:    "isImplicit": true,
+// JSON-NEXT:    "name": "__uint128_t",
+// JSON-NEXT:    "type": {
+// JSON-NEXT:     "qualType": "unsigned __int128"
+// JSON-NEXT:    },
+// JSON-NEXT:    "inner": [
+// JSON-NEXT:     {
+// JSON-NEXT:      "id": "0x{{.*}}",
+// JSON-NEXT:      "kind": "BuiltinType",
+// JSON-NEXT:      "type": {
+// JSON-NEXT:       "qualType": "unsigned __int128"
+// JSON-NEXT:      }
+// JSON-NEXT:     }
+// JSON-NEXT:    ]
+// JSON-NEXT:   },
+// JSON-NEXT:   {
+// JSON-NEXT:    "id": "0x{{.*}}",
+// JSON-NEXT:    "kind": "TypedefDecl",
+// JSON-NEXT:    "loc": {},
+// JSON-NEXT:    "range": {
+// JSON-NEXT:     "begin": {},
+// JSON-NEXT:     "end": {}
+// JSON-NEXT:    },
+// JSON-NEXT:    "isImplicit": true,
+// JSON-NEXT:    "name": "__NSConstantString",
+// JSON-NEXT:    "type": {
+// JSON-NEXT:     "qualType": "__NSConstantString_tag"
+// JSON-NEXT:    },
+// JSON-NEXT:    "inner": [
+// JSON-NEXT:     {
+// JSON-NEXT:      "id": "0x{{.*}}",
+// JSON-NEXT:      "kind": "RecordType",
+// JSON-NEXT:      "type": {
+// JSON-NEXT:       "qualType": "__NSConstantString_tag"
+// JSON-NEXT:      },
+// JSON-NEXT:      "decl": {
+// JSON-NEXT:       "id": "0x{{.*}}",
+// JSON-NEXT:       "kind": "CXXRecordDecl",
+// JSON-NEXT:       "name": "__NSConstantString_tag"
+// JSON-NEXT:      }
+// JSON-NEXT:     }
+// JSON-NEXT:    ]
+// JSON-NEXT:   },
+// JSON-NEXT:   {
+// JSON-NEXT:    "id": "0x{{.*}}",
+// JSON-NEXT:    "kind": "TypedefDecl",
+// JSON-NEXT:    "loc": {},
+// JSON-NEXT:    "range": {
+// JSON-NEXT:     "begin": {},
+// JSON-NEXT:     "end": {}
+// JSON-NEXT:    },
+// JSON-NEXT:    "isImplicit": true,
+// JSON-NEXT:    "name": "__builtin_ms_va_list",
+// JSON-NEXT:    "type": {
+// JSON-NEXT:     "qualType": "char *"
+// JSON-NEXT:    },
+// JSON-NEXT:    "inner": [
+// JSON-NEXT:     {
+// JSON-NEXT:      "id": "0x{{.*}}",
+// JSON-NEXT:      "kind": "PointerType",
+// JSON-NEXT:      "type": {
+// JSON-NEXT:       "qualType": "char *"
+// JSON-NEXT:      },
+// JSON-NEXT:      "inner": [
+// JSON-NEXT:       {
+// JSON-NEXT:        "id": "0x{{.*}}",
+// JSON-NEXT:        "kind": "BuiltinType",
+// JSON-NEXT:        "type": {
+// JSON-NEXT:         "qualType": "char"
+// JSON-NEXT:        }
+// JSON-NEXT:       }
+// JSON-NEXT:      ]
+// JSON-NEXT:     }
+// JSON-NEXT:    ]
+// JSON-NEXT:   },
+// JSON-NEXT:   {
+// JSON-NEXT:    "id": "0x{{.*}}",
+// JSON-NEXT:    "kind": "TypedefDecl",
+// JSON-NEXT:    "loc": {},
+// JSON-NEXT:    "range": {
+// JSON-NEXT:     "begin": {},
+// JSON-NEXT:     "end": {}
+// JSON-NEXT:    },
+// JSON-NEXT:    "isImplicit": true,
+// JSON-NEXT:    "name": "__builtin_va_list",
+// JSON-NEXT:    "type": {
+// JSON-NEXT:     "qualType": "__va_list_tag[1]"
+// JSON-NEXT:    },
+// JSON-NEXT:    "inner": [
+// JSON-NEXT:     {
+// JSON-NEXT:      "id": "0x{{.*}}",
+// JSON-NEXT:      "kind": "ConstantArrayType",
+// JSON-NEXT:      "type": {
+// JSON-NEXT:       "qualType": "__va_list_tag[1]"
+// JSON-NEXT:      },
+// JSON-NEXT:      "size": 1,
+// JSON-NEXT:      "inner": [
+// JSON-NEXT:       {
+// JSON-NEXT:        "id": "0x{{.*}}",
+// JSON-NEXT:        "kind": "RecordType",
+// JSON-NEXT:        "type": {
+// JSON-NEXT:         "qualType": "__va_list_tag"
+// JSON-NEXT:        },
+// JSON-NEXT:        "decl": {
+// JSON-NEXT:         "id": "0x{{.*}}",
+// JSON-NEXT:         "kind": "CXXRecordDecl",
+// JSON-NEXT:         "name": "__va_list_tag"
+// JSON-NEXT:        }
+// JSON-NEXT:       }
+// JSON-NEXT:      ]
+// JSON-NEXT:     }
+// JSON-NEXT:    ]
+// JSON-NEXT:   },
+// JSON-NEXT:   {
+// JSON-NEXT:    "id": "0x{{.*}}",
+// JSON-NEXT:    "kind": "ClassTemplateDecl",
+// JSON-NEXT:    "loc": {
+// JSON-NEXT:     "offset": 812,
+// JSON-NEXT:     "file": "{{.*}}",
+// JSON-NEXT:     "line": 15,
+// JSON-NEXT:     "col": 8,
+// JSON-NEXT:     "tokLen": 3
+// JSON-NEXT:    },
+// JSON-NEXT:    "range": {
+// JSON-NEXT:     "begin": {
+// JSON-NEXT:      "offset": 765,
+// JSON-NEXT:      "line": 14,
+// JSON-NEXT:      "col": 1,
+// JSON-NEXT:      "tokLen": 8
+// JSON-NEXT:     },
+// JSON-NEXT:     "end": {
+// JSON-NEXT:      "offset": 879,
+// JSON-NEXT:      "line": 19,
+// JSON-NEXT:      "col": 1,
+// JSON-NEXT:      "tokLen": 1
+// JSON-NEXT:     }
+// JSON-NEXT:    },
+// JSON-NEXT:    "name": "foo",
+// JSON-NEXT:    "inner": [
+// JSON-NEXT:     {
+// JSON-NEXT:      "id": "0x{{.*}}",
+// JSON-NEXT:      "kind": "NonTypeTemplateParmDecl",
+// JSON-NEXT:      "loc": {
+// JSON-NEXT:       "offset": 779,
+// JSON-NEXT:       "line": 14,
+// JSON-NEXT:       "col": 15,
+// JSON-NEXT:       "tokLen": 1
+// JSON-NEXT:      },
+// JSON-NEXT:      "range": {
+// JSON-NEXT:       "begin": {
+// JSON-NEXT:        "offset": 775,
+// JSON-NEXT:        "col": 11,
+// JSON-NEXT:        "tokLen": 3
+// JSON-NEXT:       },
+// JSON-NEXT:       "end": {
+// JSON-NEXT:        "offset": 779,
+// JSON-NEXT:        "col": 15,
+// JSON-NEXT:        "tokLen": 1
+// JSON-NEXT:       }
+// JSON-NEXT:      },
+// JSON-NEXT:      "isReferenced": true,
+// JSON-NEXT:      "name": "X",
+// JSON-NEXT:      "type": {
+// JSON-NEXT:       "qualType": "int"
+// JSON-NEXT:      },
+// JSON-NEXT:      "depth": 0,
+// JSON-NEXT:      "index": 0
+// JSON-NEXT:     },
+// JSON-NEXT:     {
+// JSON-NEXT:      "id": "0x{{.*}}",
+// JSON-NEXT:      "kind": "TemplateTypeParmDecl",
+// JSON-NEXT:      "loc": {
+// JSON-NEXT:       "offset": 791,
+// JSON-NEXT:       "col": 27,
+// JSON-NEXT:       "tokLen": 1
+// JSON-NEXT:      },
+// JSON-NEXT:      "range": {
+// JSON-NEXT:       "begin": {
+// JSON-NEXT:        "offset": 782,
+// JSON-NEXT:        "col": 18,
+// JSON-NEXT:        "tokLen": 8
+// JSON-NEXT:       },
+// JSON-NEXT:       "end": {
+// JSON-NEXT:        "offset": 791,
+// JSON-NEXT:        "col": 27,
+// JSON-NEXT:        "tokLen": 1
+// JSON-NEXT:       }
+// JSON-NEXT:      },
+// JSON-NEXT:      "isReferenced": true,
+// JSON-NEXT:      "name": "Y",
+// JSON-NEXT:      "tagUsed": "typename",
+// JSON-NEXT:      "depth": 0,
+// JSON-NEXT:      "index": 1
+// JSON-NEXT:     },
+// JSON-NEXT:     {
+// JSON-NEXT:      "id": "0x{{.*}}",
+// JSON-NEXT:      "kind": "NonTypeTemplateParmDecl",
+// JSON-NEXT:      "loc": {
+// JSON-NEXT:       "offset": 798,
+// JSON-NEXT:       "col": 34,
+// JSON-NEXT:       "tokLen": 1
+// JSON-NEXT:      },
+// JSON-NEXT:      "range": {
+// JSON-NEXT:       "begin": {
+// JSON-NEXT:        "offset": 794,
+// JSON-NEXT:        "col": 30,
+// JSON-NEXT:        "tokLen": 3
+// JSON-NEXT:       },
+// JSON-NEXT:       "end": {
+// JSON-NEXT:        "offset": 802,
+// JSON-NEXT:        "col": 38,
+// JSON-NEXT:        "tokLen": 1
+// JSON-NEXT:       }
+// JSON-NEXT:      },
+// JSON-NEXT:      "isReferenced": true,
+// JSON-NEXT:      "name": "Z",
+// JSON-NEXT:      "type": {
+// JSON-NEXT:       "qualType": "int"
+// JSON-NEXT:      },
+// JSON-NEXT:      "depth": 0,
+// JSON-NEXT:      "index": 2,
+// JSON-NEXT:      "defaultArg": {
+// JSON-NEXT:       "kind": "TemplateArgument",
+// JSON-NEXT:       "isExpr": true
+// JSON-NEXT:      },
+// JSON-NEXT:      "inner": [
+// JSON-NEXT:       {
+// JSON-NEXT:        "kind": "TemplateArgument",
+// JSON-NEXT:        "range": {
+// JSON-NEXT:         "begin": {
+// JSON-NEXT:          "offset": 802,
+// JSON-NEXT:          "col": 38,
+// JSON-NEXT:          "tokLen": 1
+// JSON-NEXT:         },
+// JSON-NEXT:         "end": {
+// JSON-NEXT:          "offset": 802,
+// JSON-NEXT:          "col": 38,
+// JSON-NEXT:          "tokLen": 1
+// JSON-NEXT:         }
+// JSON-NEXT:        },
+// JSON-NEXT:        "isExpr": true,
+// JSON-NEXT:        "inner": [
+// JSON-NEXT:         {
+// JSON-NEXT:          "id": "0x{{.*}}",
+// JSON-NEXT:          "kind": "IntegerLiteral",
+// JSON-NEXT:          "range": {
+// JSON-NEXT:           "begin": {
+// JSON-NEXT:            "offset": 802,
+// JSON-NEXT:            "col": 38,
+// JSON-NEXT:            "tokLen": 1
+// JSON-NEXT:           },
+// JSON-NEXT:           "end": {
+// JSON-NEXT:            "offset": 802,
+// JSON-NEXT:            "col": 38,
+// JSON-NEXT:            "tokLen": 1
+// JSON-NEXT:           }
+// JSON-NEXT:          },
+// JSON-NEXT:          "type": {
+// JSON-NEXT:           "qualType": "int"
+// JSON-NEXT:          },
+// JSON-NEXT:          "valueCategory": "prvalue",
+// JSON-NEXT:          "value": "5"
+// JSON-NEXT:         }
+// JSON-NEXT:        ]
+// JSON-NEXT:       }
+// JSON-NEXT:      ]
+// JSON-NEXT:     },
+// JSON-NEXT:     {
+// JSON-NEXT:      "id": "0x{{.*}}",
+// JSON-NEXT:      "kind": "CXXRecordDecl",
+// JSON-NEXT:      "loc": {
+// JSON-NEXT:       "offset": 812,
+// JSON-NEXT:       "line": 15,
+// JSON-NEXT:       "col": 8,
+// JSON-NEXT:       "tokLen": 3
+// JSON-NEXT:      },
+// JSON-NEXT:      "range": {
+// JSON-NEXT:       "begin": {
+// JSON-NEXT:        "offset": 805,
+// JSON-NEXT:        "col": 1,
+// JSON-NEXT:        "tokLen": 6
+// JSON-NEXT:       },
+// JSON-NEXT:       "end": {
+// JSON-NEXT:        "offset": 879,
+// JSON-NEXT:        "line": 19,
+// JSON-NEXT:        "col": 1,
+// JSON-NEXT:        "tokLen": 1
+// JSON-NEXT:       }
+// JSON-NEXT:      },
+// JSON-NEXT:      "name": "foo",
+// JSON-NEXT:      "tagUsed": "struct",
+// JSON-NEXT:      "completeDefinition": true,
+// JSON-NEXT:      "definitionData": {
+// JSON-NEXT:       "canConstDefaultInit": true,
+// JSON-NEXT:       "copyAssign": {
+// JSON-NEXT:        "hasConstParam": true,
+// JSON-NEXT:        "implicitHasConstParam": true,
+// JSON-NEXT:        "needsImplicit": true,
+// JSON-NEXT:        "simple": true,
+// JSON-NEXT:        "trivial": true
+// JSON-NEXT:       },
+// JSON-NEXT:       "copyCtor": {
+// JSON-NEXT:        "hasConstParam": true,
+// JSON-NEXT:        "implicitHasConstParam": true,
+// JSON-NEXT:        "needsImplicit": true,
+// JSON-NEXT:        "simple": true,
+// JSON-NEXT:        "trivial": true
+// JSON-NEXT:       },
+// JSON-NEXT:       "defaultCtor": {
+// JSON-NEXT:        "exists": true,
+// JSON-NEXT:        "nonTrivial": true,
+// JSON-NEXT:        "userProvided": true
+// JSON-NEXT:       },
+// JSON-NEXT:       "dtor": {
+// JSON-NEXT:        "irrelevant": true,
+// JSON-NEXT:        "needsImplicit": true,
+// JSON-NEXT:        "simple": true,
+// JSON-NEXT:        "trivial": true
+// JSON-NEXT:       },
+// JSON-NEXT:       "hasUserDeclaredConstructor": true,
+// JSON-NEXT:       "isStandardLayout": true,
+// JSON-NEXT:       "isTriviallyCopyable": true,
+// JSON-NEXT:       "moveAssign": {
+// JSON-NEXT:        "exists": true,
+// JSON-NEXT:        "needsImplicit": true,
+// JSON-NEXT:        "simple": true,
+// JSON-NEXT:        "trivial": true
+// JSON-NEXT:       },
+// JSON-NEXT:       "moveCtor": {
+// JSON-NEXT:        "exists": true,
+// JSON-NEXT:        "needsImplicit": true,
+// JSON-NEXT:        "simple": true,
+// JSON-NEXT:        "trivial": true
+// JSON-NEXT:       }
+// JSON-NEXT:      },
+// JSON-NEXT:      "inner": [
+// JSON-NEXT:       {
+// JSON-NEXT:        "id": "0x{{.*}}",
+// JSON-NEXT:        "kind": "CXXRecordDecl",
+// JSON-NEXT:        "loc": {
+// JSON-NEXT:         "offset": 812,
+// JSON-NEXT:         "line": 15,
+// JSON-NEXT:         "col": 8,
+// JSON-NEXT:         "tokLen": 3
+// JSON-NEXT:        },
+// JSON-NEXT:        "range": {
+// JSON-NEXT:         "begin": {
+// JSON-NEXT:          "offset": 805,
+// JSON-NEXT:          "col": 1,
+// JSON-NEXT:          "tokLen": 6
+// JSON-NEXT:         },
+// JSON-NEXT:         "end": {
+// JSON-NEXT:          "offset": 812,
+// JSON-NEXT:          "col": 8,
+// JSON-NEXT:          "tokLen": 3
+// JSON-NEXT:         }
+// JSON-NEXT:        },
+// JSON-NEXT:        "isImplicit": true,
+// JSON-NEXT:        "isReferenced": true,
+// JSON-NEXT:        "name": "foo",
+// JSON-NEXT:        "tagUsed": "struct"
+// JSON-NEXT:       },
+// JSON-NEXT:       {
+// JSON-NEXT:        "id": "0x{{.*}}",
+// JSON-NEXT:        "kind": "FieldDecl",
+// JSON-NEXT:        "loc": {
+// JSON-NEXT:         "offset": 824,
+// JSON-NEXT:         "line": 16,
+// JSON-NEXT:         "col": 7,
+// JSON-NEXT:         "tokLen": 8
+// JSON-NEXT:        },
+// JSON-NEXT:        "range": {
+// JSON-NEXT:         "begin": {
+// JSON-NEXT:          "offset": 820,
+// JSON-NEXT:          "col": 3,
+// JSON-NEXT:          "tokLen": 3
+// JSON-NEXT:         },
+// JSON-NEXT:         "end": {
+// JSON-NEXT:          "offset": 824,
+// JSON-NEXT:          "col": 7,
+// JSON-NEXT:          "tokLen": 8
+// JSON-NEXT:         }
+// JSON-NEXT:        },
+// JSON-NEXT:        "name": "constant",
+// JSON-NEXT:        "type": {
+// JSON-NEXT:         "qualType": "int"
+// JSON-NEXT:        }
+// JSON-NEXT:       },
+// JSON-NEXT:       {
+// JSON-NEXT:        "id": "0x{{.*}}",
+// JSON-NEXT:        "kind": "CXXConstructorDecl",
+// JSON-NEXT:        "loc": {
+// JSON-NEXT:         "offset": 836,
+// JSON-NEXT:         "line": 17,
+// JSON-NEXT:         "col": 3,
+// JSON-NEXT:         "tokLen": 3
+// JSON-NEXT:        },
+// JSON-NEXT:        "range": {
+// JSON-NEXT:         "begin": {
+// JSON-NEXT:          "offset": 836,
+// JSON-NEXT:          "col": 3,
+// JSON-NEXT:          "tokLen": 3
+// JSON-NEXT:         },
+// JSON-NEXT:         "end": {
+// JSON-NEXT:          "offset": 843,
+// JSON-NEXT:          "col": 10,
+// JSON-NEXT:          "tokLen": 1
+// JSON-NEXT:         }
+// JSON-NEXT:        },
+// JSON-NEXT:        "name": "foo<X, Y, Z>",
+// JSON-NEXT:        "type": {
+// JSON-NEXT:         "qualType": "void ()"
+// JSON-NEXT:        },
+// JSON-NEXT:        "inner": [
+// JSON-NEXT:         {
+// JSON-NEXT:          "id": "0x{{.*}}",
+// JSON-NEXT:          "kind": "CompoundStmt",
+// JSON-NEXT:          "range": {
+// JSON-NEXT:           "begin": {
+// JSON-NEXT:            "offset": 842,
+// JSON-NEXT:            "col": 9,
+// JSON-NEXT:            "tokLen": 1
+// JSON-NEXT:           },
+// JSON-NEXT:           "end": {
+// JSON-NEXT:            "offset": 843,
+// JSON-NEXT:            "col": 10,
+// JSON-NEXT:            "tokLen": 1
+// JSON-NEXT:           }
+// JSON-NEXT:          }
+// JSON-NEXT:         }
+// JSON-NEXT:        ]
+// JSON-NEXT:       },
+// JSON-NEXT:       {
+// JSON-NEXT:        "id": "0x{{.*}}",
+// JSON-NEXT:        "kind": "CXXMethodDecl",
+// JSON-NEXT:        "loc": {
+// JSON-NEXT:         "offset": 849,
+// JSON-NEXT:         "line": 18,
+// JSON-NEXT:         "col": 5,
+// JSON-NEXT:         "tokLen": 6
+// JSON-NEXT:        },
+// JSON-NEXT:        "range": {
+// JSON-NEXT:         "begin": {
+// JSON-NEXT:          "offset": 847,
+// JSON-NEXT:          "col": 3,
+// JSON-NEXT:          "tokLen": 1
+// JSON-NEXT:         },
+// JSON-NEXT:         "end": {
+// JSON-NEXT:          "offset": 877,
+// JSON-NEXT:          "col": 33,
+// JSON-NEXT:          "tokLen": 1
+// JSON-NEXT:         }
+// JSON-NEXT:        },
+// JSON-NEXT:        "name": "getSum",
+// JSON-NEXT:        "type": {
+// JSON-NEXT:         "qualType": "Y ()"
+// JSON-NEXT:        },
+// JSON-NEXT:        "inner": [
+// JSON-NEXT:         {
+// JSON-NEXT:          "id": "0x{{.*}}",
+// JSON-NEXT:          "kind": "CompoundStmt",
+// JSON-NEXT:          "range": {
+// JSON-NEXT:           "begin": {
+// JSON-NEXT:            "offset": 858,
+// JSON-NEXT:            "col": 14,
+// JSON-NEXT:            "tokLen": 1
+// JSON-NEXT:           },
+// JSON-NEXT:           "end": {
+// JSON-NEXT:            "offset": 877,
+// JSON-NEXT:            "col": 33,
+// JSON-NEXT:            "tokLen": 1
+// JSON-NEXT:           }
+// JSON-NEXT:          },
+// JSON-NEXT:          "inner": [
+// JSON-NEXT:           {
+// JSON-NEXT:            "id": "0x{{.*}}",
+// JSON-NEXT:            "kind": "ReturnStmt",
+// JSON-NEXT:            "range": {
+// JSON-NEXT:             "begin": {
+// JSON-NEXT:              "offset": 860,
+// JSON-NEXT:              "col": 16,
+// JSON-NEXT:              "tokLen": 6
+// JSON-NEXT:             },
+// JSON-NEXT:             "end": {
+// JSON-NEXT:              "offset": 874,
+// JSON-NEXT:              "col": 30,
+// JSON-NEXT:              "tokLen": 1
+// JSON-NEXT:             }
+// JSON-NEXT:            },
+// JSON-NEXT:            "inner": [
+// JSON-NEXT:             {
+// JSON-NEXT:              "id": "0x{{.*}}",
+// JSON-NEXT:              "kind": "CXXUnresolvedConstructExpr",
+// JSON-NEXT:              "range": {
+// JSON-NEXT:               "begin": {
+// JSON-NEXT:                "offset": 867,
+// JSON-NEXT:                "col": 23,
+// JSON-NEXT:                "tokLen": 1
+// JSON-NEXT:               },
+// JSON-NEXT:               "end": {
+// JSON-NEXT:                "offset": 874,
+// JSON-NEXT:                "col": 30,
+// JSON-NEXT:                "tokLen": 1
+// JSON-NEXT:               }
+// JSON-NEXT:              },
+// JSON-NEXT:              "type": {
+// JSON-NEXT:               "qualType": "Y"
+// JSON-NEXT:              },
+// JSON-NEXT:              "valueCategory": "prvalue",
+// JSON-NEXT:              "inner": [
+// JSON-NEXT:               {
+// JSON-NEXT:                "id": "0x{{.*}}",
+// JSON-NEXT:                "kind": "BinaryOperator",
+// JSON-NEXT:                "range": {
+// JSON-NEXT:                 "begin": {
+// JSON-NEXT:                  "offset": 869,
+// JSON-NEXT:                  "col": 25,
+// JSON-NEXT:                  "tokLen": 1
+// JSON-NEXT:                 },
+// JSON-NEXT:                 "end": {
+// JSON-NEXT:                  "offset": 873,
+// JSON-NEXT:                  "col": 29,
+// JSON-NEXT:                  "tokLen": 1
+// JSON-NEXT:                 }
+// JSON-NEXT:                },
+// JSON-NEXT:                "type": {
+// JSON-NEXT:                 "qualType": "int"
+// JSON-NEXT:                },
+// JSON-NEXT:                "valueCategory": "prvalue",
+// JSON-NEXT:                "opcode": "+",
+// JSON-NEXT:                "inner": [
+// JSON-NEXT:                 {
+// JSON-NEXT:                  "id": "0x{{.*}}",
+// JSON-NEXT:                  "kind": "DeclRefExpr",
+// JSON-NEXT:                  "range": {
+// JSON-NEXT:                   "begin": {
+// JSON-NEXT:                    "offset": 869,
+// JSON-NEXT:                    "col": 25,
+// JSON-NEXT:                    "tokLen": 1
+// JSON-NEXT:                   },
+// JSON-NEXT:                   "end": {
+// JSON-NEXT:                    "offset": 869,
+// JSON-NEXT:                    "col": 25,
+// JSON-NEXT:                    "tokLen": 1
+// JSON-NEXT:                   }
+// JSON-NEXT:                  },
+// JSON-NEXT:                  "type": {
+// JSON-NEXT:                   "qualType": "int"
+// JSON-NEXT:                  },
+// JSON-NEXT:                  "valueCategory": "prvalue",
+// JSON-NEXT:                  "referencedDecl": {
+// JSON-NEXT:                   "id": "0x{{.*}}",
+// JSON-NEXT:                   "kind": "NonTypeTemplateParmDecl",
+// JSON-NEXT:                   "name": "X",
+// JSON-NEXT:                   "type": {
+// JSON-NEXT:                    "qualType": "int"
+// JSON-NEXT:                   }
+// JSON-NEXT:                  }
+// JSON-NEXT:                 },
+// JSON-NEXT:                 {
+// JSON-NEXT:                  "id": "0x{{.*}}",
+// JSON-NEXT:                  "kind": "DeclRefExpr",
+// JSON-NEXT:                  "range": {
+// JSON-NEXT:                   "begin": {
+// JSON-NEXT:                    "offset": 873,
+// JSON-NEXT:                    "col": 29,
+// JSON-NEXT:                    "tokLen": 1
+// JSON-NEXT:                   },
+// JSON-NEXT:                   "end": {
+// JSON-NEXT:                    "offset": 873,
+// JSON-NEXT:                    "col": 29,
+// JSON-NEXT:                    "tokLen": 1
+// JSON-NEXT:                   }
+// JSON-NEXT:                  },
+// JSON-NEXT:                  "type": {
+// JSON-NEXT:                   "qualType": "int"
+// JSON-NEXT:                  },
+// JSON-NEXT:                  "valueCategory": "prvalue",
+// JSON-NEXT:                  "referencedDecl": {
+// JSON-NEXT:                   "id": "0x{{.*}}",
+// JSON-NEXT:                   "kind": "NonTypeTemplateParmDecl",
+// JSON-NEXT:                   "name": "Z",
+// JSON-NEXT:                   "type": {
+// JSON-NEXT:                    "qualType": "int"
+// JSON-NEXT:                   }
+// JSON-NEXT:                  }
+// JSON-NEXT:                 }
+// JSON-NEXT:                ]
+// JSON-NEXT:               }
+// JSON-NEXT:              ]
+// JSON-NEXT:             }
+// JSON-NEXT:            ]
+// JSON-NEXT:           }
+// JSON-NEXT:          ]
+// JSON-NEXT:         }
+// JSON-NEXT:        ]
+// JSON-NEXT:       }
+// JSON-NEXT:      ]
+// JSON-NEXT:     },
+// JSON-NEXT:     {
+// JSON-NEXT:      "id": "0x{{.*}}",
+// JSON-NEXT:      "kind": "ClassTemplateSpecializationDecl",
+// JSON-NEXT:      "loc": {
+// JSON-NEXT:       "offset": 812,
+// JSON-NEXT:       "line": 15,
+// JSON-NEXT:       "col": 8,
+// JSON-NEXT:       "tokLen": 3
+// JSON-NEXT:      },
+// JSON-NEXT:      "range": {
+// JSON-NEXT:       "begin": {
+// JSON-NEXT:        "offset": 765,
+// JSON-NEXT:        "line": 14,
+// JSON-NEXT:        "col": 1,
+// JSON-NEXT:        "tokLen": 8
+// JSON-NEXT:       },
+// JSON-NEXT:       "end": {
+// JSON-NEXT:        "offset": 879,
+// JSON-NEXT:        "line": 19,
+// JSON-NEXT:        "col": 1,
+// JSON-NEXT:        "tokLen": 1
+// JSON-NEXT:       }
+// JSON-NEXT:      },
+// JSON-NEXT:      "name": "foo",
+// JSON-NEXT:      "tagUsed": "struct",
+// JSON-NEXT:      "completeDefinition": true,
+// JSON-NEXT:      "definitionData": {
+// JSON-NEXT:       "canConstDefaultInit": true,
+// JSON-NEXT:       "canPassInRegisters": true,
+// JSON-NEXT:       "copyAssign": {
+// JSON-NEXT:        "hasConstParam": true,
+// JSON-NEXT:        "implicitHasConstParam": true,
+// JSON-NEXT:        "needsImplicit": true,
+// JSON-NEXT:        "simple": true,
+// JSON-NEXT:        "trivial": true
+// JSON-NEXT:       },
+// JSON-NEXT:       "copyCtor": {
+// JSON-NEXT:        "hasConstParam": true,
+// JSON-NEXT:        "implicitHasConstParam": true,
+// JSON-NEXT:        "simple": true,
+// JSON-NEXT:        "trivial": true
+// JSON-NEXT:       },
+// JSON-NEXT:       "defaultCtor": {
+// JSON-NEXT:        "exists": true,
+// JSON-NEXT:        "nonTrivial": true,
+// JSON-NEXT:        "userProvided": true
+// JSON-NEXT:       },
+// JSON-NEXT:       "dtor": {
+// JSON-NEXT:        "irrelevant": true,
+// JSON-NEXT:        "simple": true,
+// JSON-NEXT:        "trivial": true
+// JSON-NEXT:       },
+// JSON-NEXT:       "hasUserDeclaredConstructor": true,
+// JSON-NEXT:       "isStandardLayout": true,
+// JSON-NEXT:       "isTriviallyCopyable": true,
+// JSON-NEXT:       "moveAssign": {
+// JSON-NEXT:        "exists": true,
+// JSON-NEXT:        "needsImplicit": true,
+// JSON-NEXT:        "simple": true,
+// JSON-NEXT:        "trivial": true
+// JSON-NEXT:       },
+// JSON-NEXT:       "moveCtor": {
+// JSON-NEXT:        "exists": true,
+// JSON-NEXT:        "simple": true,
+// JSON-NEXT:        "trivial": true
+// JSON-NEXT:       }
+// JSON-NEXT:      },
+// JSON-NEXT:      "inner": [
+// JSON-NEXT:       {
+// JSON-NEXT:        "kind": "TemplateArgument",
+// JSON-NEXT:        "value": 5
+// JSON-NEXT:       },
+// JSON-NEXT:       {
+// JSON-NEXT:        "kind": "TemplateArgument",
+// JSON-NEXT:        "type": {
+// JSON-NEXT:         "qualType": "int"
+// JSON-NEXT:        },
+// JSON-NEXT:        "inner": [
+// JSON-NEXT:         {
+// JSON-NEXT:          "id": "0x{{.*}}",
+// JSON-NEXT:          "kind": "BuiltinType",
+// JSON-NEXT:          "type": {
+// JSON-NEXT:           "qualType": "int"
+// JSON-NEXT:          }
+// JSON-NEXT:         }
+// JSON-NEXT:        ]
+// JSON-NEXT:       },
+// JSON-NEXT:       {
+// JSON-NEXT:        "kind": "TemplateArgument",
+// JSON-NEXT:        "value": 5
+// JSON-NEXT:       },
+// JSON-NEXT:       {
+// JSON-NEXT:        "id": "0x{{.*}}",
+// JSON-NEXT:        "kind": "CXXRecordDecl",
+// JSON-NEXT:        "loc": {
+// JSON-NEXT:         "offset": 812,
+// JSON-NEXT:         "line": 15,
+// JSON-NEXT:         "col": 8,
+// JSON-NEXT:         "tokLen": 3
+// JSON-NEXT:        },
+// JSON-NEXT:        "range": {
+// JSON-NEXT:         "begin": {
+// JSON-NEXT:          "offset": 805,
+// JSON-NEXT:          "col": 1,
+// JSON-NEXT:          "tokLen": 6
+// JSON-NEXT:         },
+// JSON-NEXT:         "end": {
+// JSON-NEXT:          "offset": 812,
+// JSON-NEXT:          "col": 8,
+// JSON-NEXT:          "tokLen": 3
+// JSON-NEXT:         }
+// JSON-NEXT:        },
+// JSON-NEXT:        "isImplicit": true,
+// JSON-NEXT:        "name": "foo",
+// JSON-NEXT:        "tagUsed": "struct"
+// JSON-NEXT:       },
+// JSON-NEXT:       {
+// JSON-NEXT:        "id": "0x{{.*}}",
+// JSON-NEXT:        "kind": "FieldDecl",
+// JSON-NEXT:        "loc": {
+// JSON-NEXT:         "offset": 824,
+// JSON-NEXT:         "line": 16,
+// JSON-NEXT:         "col": 7,
+// JSON-NEXT:         "tokLen": 8
+// JSON-NEXT:        },
+// JSON-NEXT:        "range": {
+// JSON-NEXT:         "begin": {
+// JSON-NEXT:          "offset": 820,
+// JSON-NEXT:          "col": 3,
+// JSON-NEXT:          "tokLen": 3
+// JSON-NEXT:         },
+// JSON-NEXT:         "end": {
+// JSON-NEXT:          "offset": 824,
+// JSON-NEXT:          "col": 7,
+// JSON-NEXT:          "tokLen": 8
+// JSON-NEXT:         }
+// JSON-NEXT:        },
+// JSON-NEXT:        "name": "constant",
+// JSON-NEXT:        "type": {
+// JSON-NEXT:         "qualType": "int"
+// JSON-NEXT:        }
+// JSON-NEXT:       },
+// JSON-NEXT:       {
+// JSON-NEXT:        "id": "0x{{.*}}",
+// JSON-NEXT:        "kind": "CXXConstructorDecl",
+// JSON-NEXT:        "loc": {
+// JSON-NEXT:         "offset": 836,
+// JSON-NEXT:         "line": 17,
+// JSON-NEXT:         "col": 3,
+// JSON-NEXT:         "tokLen": 3
+// JSON-NEXT:        },
+// JSON-NEXT:        "range": {
+// JSON-NEXT:         "begin": {
+// JSON-NEXT:          "offset": 836,
+// JSON-NEXT:          "col": 3,
+// JSON-NEXT:          "tokLen": 3
+// JSON-NEXT:         },
+// JSON-NEXT:         "end": {
+// JSON-NEXT:          "offset": 843,
+// JSON-NEXT:          "col": 10,
+// JSON-NEXT:          "tokLen": 1
+// JSON-NEXT:         }
+// JSON-NEXT:        },
+// JSON-NEXT:        "isUsed": true,
+// JSON-NEXT:        "name": "foo",
+// JSON-NEXT:        "mangledName": "_ZN3fooILi5EiLi5EEC1Ev",
+// JSON-NEXT:        "type": {
+// JSON-NEXT:         "qualType": "void ()"
+// JSON-NEXT:        },
+// JSON-NEXT:        "inner": [
+// JSON-NEXT:         {
+// JSON-NEXT:          "id": "0x{{.*}}",
+// JSON-NEXT:          "kind": "CompoundStmt",
+// JSON-NEXT:          "range": {
+// JSON-NEXT:           "begin": {
+// JSON-NEXT:            "offset": 842,
+// JSON-NEXT:            "col": 9,
+// JSON-NEXT:            "tokLen": 1
+// JSON-NEXT:           },
+// JSON-NEXT:           "end": {
+// JSON-NEXT:            "offset": 843,
+// JSON-NEXT:            "col": 10,
+// JSON-NEXT:            "tokLen": 1
+// JSON-NEXT:           }
+// JSON-NEXT:          }
+// JSON-NEXT:         }
+// JSON-NEXT:        ]
+// JSON-NEXT:       },
+// JSON-NEXT:       {
+// JSON-NEXT:        "id": "0x{{.*}}",
+// JSON-NEXT:        "kind": "CXXMethodDecl",
+// JSON-NEXT:        "loc": {
+// JSON-NEXT:         "offset": 849,
+// JSON-NEXT:         "line": 18,
+// JSON-NEXT:         "col": 5,
+// JSON-NEXT:         "tokLen": 6
+// JSON-NEXT:        },
+// JSON-NEXT:        "range": {
+// JSON-NEXT:         "begin": {
+// JSON-NEXT:          "offset": 847,
+// JSON-NEXT:          "col": 3,
+// JSON-NEXT:          "tokLen": 1
+// JSON-NEXT:         },
+// JSON-NEXT:         "end": {
+// JSON-NEXT:          "offset": 877,
+// JSON-NEXT:          "col": 33,
+// JSON-NEXT:          "tokLen": 1
+// JSON-NEXT:         }
+// JSON-NEXT:        },
+// JSON-NEXT:        "isUsed": true,
+// JSON-NEXT:        "name": "getSum",
+// JSON-NEXT:        "mangledName": "_ZN3fooILi5EiLi5EE6getSumEv",
+// JSON-NEXT:        "type": {
+// JSON-NEXT:         "qualType": "int ()"
+// JSON-NEXT:        },
+// JSON-NEXT:        "inner": [
+// JSON-NEXT:         {
+// JSON-NEXT:          "id": "0x{{.*}}",
+// JSON-NEXT:          "kind": "CompoundStmt",
+// JSON-NEXT:          "range": {
+// JSON-NEXT:           "begin": {
+// JSON-NEXT:            "offset": 858,
+// JSON-NEXT:            "col": 14,
+// JSON-NEXT:            "tokLen": 1
+// JSON-NEXT:           },
+// JSON-NEXT:           "end": {
+// JSON-NEXT:            "offset": 877,
+// JSON-NEXT:            "col": 33,
+// JSON-NEXT:            "tokLen": 1
+// JSON-NEXT:           }
+// JSON-NEXT:          },
+// JSON-NEXT:          "inner": [
+// JSON-NEXT:           {
+// JSON-NEXT:            "id": "0x{{.*}}",
+// JSON-NEXT:            "kind": "ReturnStmt",
+// JSON-NEXT:            "range": {
+// JSON-NEXT:             "begin": {
+// JSON-NEXT:              "offset": 860,
+// JSON-NEXT:              "col": 16,
+// JSON-NEXT:              "tokLen": 6
+// JSON-NEXT:             },
+// JSON-NEXT:             "end": {
+// JSON-NEXT:              "offset": 874,
+// JSON-NEXT:              "col": 30,
+// JSON-NEXT:              "tokLen": 1
+// JSON-NEXT:             }
+// JSON-NEXT:            },
+// JSON-NEXT:            "inner": [
+// JSON-NEXT:             {
+// JSON-NEXT:              "id": "0x{{.*}}",
+// JSON-NEXT:              "kind": "CXXFunctionalCastExpr",
+// JSON-NEXT:              "range": {
+// JSON-NEXT:               "begin": {
+// JSON-NEXT:                "offset": 867,
+// JSON-NEXT:                "col": 23,
+// JSON-NEXT:                "tokLen": 1
+// JSON-NEXT:               },
+// JSON-NEXT:               "end": {
+// JSON-NEXT:                "offset": 874,
+// JSON-NEXT:                "col": 30,
+// JSON-NEXT:                "tokLen": 1
+// JSON-NEXT:               }
+// JSON-NEXT:              },
+// JSON-NEXT:              "type": {
+// JSON-NEXT:               "qualType": "int"
+// JSON-NEXT:              },
+// JSON-NEXT:              "valueCategory": "prvalue",
+// JSON-NEXT:              "castKind": "NoOp",
+// JSON-NEXT:              "inner": [
+// JSON-NEXT:               {
+// JSON-NEXT:                "id": "0x{{.*}}",
+// JSON-NEXT:                "kind": "BinaryOperator",
+// JSON-NEXT:                "range": {
+// JSON-NEXT:                 "begin": {
+// JSON-NEXT:                  "offset": 869,
+// JSON-NEXT:                  "col": 25,
+// JSON-NEXT:                  "tokLen": 1
+// JSON-NEXT:                 },
+// JSON-NEXT:                 "end": {
+// JSON-NEXT:                  "offset": 873,
+// JSON-NEXT:                  "col": 29,
+// JSON-NEXT:                  "tokLen": 1
+// JSON-NEXT:                 }
+// JSON-NEXT:                },
+// JSON-NEXT:                "type": {
+// JSON-NEXT:                 "qualType": "int"
+// JSON-NEXT:                },
+// JSON-NEXT:                "valueCategory": "prvalue",
+// JSON-NEXT:                "opcode": "+",
+// JSON-NEXT:                "inner": [
+// JSON-NEXT:                 {
+// JSON-NEXT:                  "id": "0x{{.*}}",
+// JSON-NEXT:                  "kind": "SubstNonTypeTemplateParmExpr",
+// JSON-NEXT:                  "range": {
+// JSON-NEXT:                   "begin": {
+// JSON-NEXT:                    "offset": 869,
+// JSON-NEXT:                    "col": 25,
+// JSON-NEXT:                    "tokLen": 1
+// JSON-NEXT:                   },
+// JSON-NEXT:                   "end": {
+// JSON-NEXT:                    "offset": 869,
+// JSON-NEXT:                    "col": 25,
+// JSON-NEXT:                    "tokLen": 1
+// JSON-NEXT:                   }
+// JSON-NEXT:                  },
+// JSON-NEXT:                  "type": {
+// JSON-NEXT:                   "qualType": "int"
+// JSON-NEXT:                  },
+// JSON-NEXT:                  "valueCategory": "prvalue",
+// JSON-NEXT:                  "inner": [
+// JSON-NEXT:                   {
+// JSON-NEXT:                    "id": "0x{{.*}}",
+// JSON-NEXT:                    "kind": "NonTypeTemplateParmDecl",
+// JSON-NEXT:                    "loc": {
+// JSON-NEXT:                     "offset": 779,
+// JSON-NEXT:                     "line": 14,
+// JSON-NEXT:                     "col": 15,
+// JSON-NEXT:                     "tokLen": 1
+// JSON-NEXT:                    },
+// JSON-NEXT:                    "range": {
+// JSON-NEXT:                     "begin": {
+// JSON-NEXT:                      "offset": 775,
+// JSON-NEXT:                      "col": 11,
+// JSON-NEXT:                      "tokLen": 3
+// JSON-NEXT:                     },
+// JSON-NEXT:                     "end": {
+// JSON-NEXT:                      "offset": 779,
+// JSON-NEXT:                      "col": 15,
+// JSON-NEXT:                      "tokLen": 1
+// JSON-NEXT:                     }
+// JSON-NEXT:                    },
+// JSON-NEXT:                    "isReferenced": true,
+// JSON-NEXT:                    "name": "X",
+// JSON-NEXT:                    "type": {
+// JSON-NEXT:                     "qualType": "int"
+// JSON-NEXT:                    },
+// JSON-NEXT:                    "depth": 0,
+// JSON-NEXT:                    "index": 0
+// JSON-NEXT:                   },
+// JSON-NEXT:                   {
+// JSON-NEXT:                    "id": "0x{{.*}}",
+// JSON-NEXT:                    "kind": "IntegerLiteral",
+// JSON-NEXT:                    "range": {
+// JSON-NEXT:                     "begin": {
+// JSON-NEXT:                      "offset": 869,
+// JSON-NEXT:                      "line": 18,
+// JSON-NEXT:                      "col": 25,
+// JSON-NEXT:                      "tokLen": 1
+// JSON-NEXT:                     },
+// JSON-NEXT:                     "end": {
+// JSON-NEXT:                      "offset": 869,
+// JSON-NEXT:                      "col": 25,
+// JSON-NEXT:                      "tokLen": 1
+// JSON-NEXT:                     }
+// JSON-NEXT:                    },
+// JSON-NEXT:                    "type": {
+// JSON-NEXT:                     "qualType": "int"
+// JSON-NEXT:                    },
+// JSON-NEXT:                    "valueCategory": "prvalue",
+// JSON-NEXT:                    "value": "5"
+// JSON-NEXT:                   }
+// JSON-NEXT:                  ]
+// JSON-NEXT:                 },
+// JSON-NEXT:                 {
+// JSON-NEXT:                  "id": "0x{{.*}}",
+// JSON-NEXT:                  "kind": "SubstNonTypeTemplateParmExpr",
+// JSON-NEXT:                  "range": {
+// JSON-NEXT:                   "begin": {
+// JSON-NEXT:                    "offset": 873,
+// JSON-NEXT:                    "col": 29,
+// JSON-NEXT:                    "tokLen": 1
+// JSON-NEXT:                   },
+// JSON-NEXT:                   "end": {
+// JSON-NEXT:                    "offset": 873,
+// JSON-NEXT:                    "col": 29,
+// JSON-NEXT:                    "tokLen": 1
+// JSON-NEXT:                   }
+// JSON-NEXT:                  },
+// JSON-NEXT:                  "type": {
+// JSON-NEXT:                   "qualType": "int"
+// JSON-NEXT:                  },
+// JSON-NEXT:                  "valueCategory": "prvalue",
+// JSON-NEXT:                  "inner": [
+// JSON-NEXT:                   {
+// JSON-NEXT:                    "id": "0x{{.*}}",
+// JSON-NEXT:                    "kind": "NonTypeTemplateParmDecl",
+// JSON-NEXT:                    "loc": {
+// JSON-NEXT:                     "offset": 798,
+// JSON-NEXT:                     "line": 14,
+// JSON-NEXT:                     "col": 34,
+// JSON-NEXT:                     "tokLen": 1
+// JSON-NEXT:                    },
+// JSON-NEXT:                    "range": {
+// JSON-NEXT:                     "begin": {
+// JSON-NEXT:                      "offset": 794,
+// JSON-NEXT:                      "col": 30,
+// JSON-NEXT:                      "tokLen": 3
+// JSON-NEXT:                     },
+// JSON-NEXT:                     "end": {
+// JSON-NEXT:                      "offset": 802,
+// JSON-NEXT:                      "col": 38,
+// JSON-NEXT:                      "tokLen": 1
+// JSON-NEXT:                     }
+// JSON-NEXT:                    },
+// JSON-NEXT:                    "isReferenced": true,
+// JSON-NEXT:                    "name": "Z",
+// JSON-NEXT:                    "type": {
+// JSON-NEXT:                     "qualType": "int"
+// JSON-NEXT:                    },
+// JSON-NEXT:                    "depth": 0,
+// JSON-NEXT:                    "index": 2,
+// JSON-NEXT:                    "defaultArg": {
+// JSON-NEXT:                     "kind": "TemplateArgument",
+// JSON-NEXT:                     "isExpr": true
+// JSON-NEXT:                    },
+// JSON-NEXT:                    "inner": [
+// JSON-NEXT:                     {
+// JSON-NEXT:                      "kind": "TemplateArgument",
+// JSON-NEXT:                      "range": {
+// JSON-NEXT:                       "begin": {
+// JSON-NEXT:                        "offset": 802,
+// JSON-NEXT:                        "col": 38,
+// JSON-NEXT:                        "tokLen": 1
+// JSON-NEXT:                       },
+// JSON-NEXT:                       "end": {
+// JSON-NEXT:                        "offset": 802,
+// JSON-NEXT:                        "col": 38,
+// JSON-NEXT:                        "tokLen": 1
+// JSON-NEXT:                       }
+// JSON-NEXT:                      },
+// JSON-NEXT:                      "isExpr": true,
+// JSON-NEXT:                      "inner": [
+// JSON-NEXT:                       {
+// JSON-NEXT:                        "id": "0x{{.*}}",
+// JSON-NEXT:                        "kind": "IntegerLiteral",
+// JSON-NEXT:                        "range": {
+// JSON-NEXT:                         "begin": {
+// JSON-NEXT:                          "offset": 802,
+// JSON-NEXT:                          "col": 38,
+// JSON-NEXT:                          "tokLen": 1
+// JSON-NEXT:                         },
+// JSON-NEXT:                         "end": {
+// JSON-NEXT:                          "offset": 802,
+// JSON-NEXT:                          "col": 38,
+// JSON-NEXT:                          "tokLen": 1
+// JSON-NEXT:                         }
+// JSON-NEXT:                        },
+// JSON-NEXT:                        "type": {
+// JSON-NEXT:                         "qualType": "int"
+// JSON-NEXT:                        },
+// JSON-NEXT:                        "valueCategory": "prvalue",
+// JSON-NEXT:                        "value": "5"
+// JSON-NEXT:                       }
+// JSON-NEXT:                      ]
+// JSON-NEXT:                     }
+// JSON-NEXT:                    ]
+// JSON-NEXT:                   },
+// JSON-NEXT:                   {
+// JSON-NEXT:                    "id": "0x{{.*}}",
+// JSON-NEXT:                    "kind": "IntegerLiteral",
+// JSON-NEXT:                    "range": {
+// JSON-NEXT:                     "begin": {
+// JSON-NEXT:                      "offset": 873,
+// JSON-NEXT:                      "line": 18,
+// JSON-NEXT:                      "col": 29,
+// JSON-NEXT:                      "tokLen": 1
+// JSON-NEXT:                     },
+// JSON-NEXT:                     "end": {
+// JSON-NEXT:                      "offset": 873,
+// JSON-NEXT:                      "col": 29,
+// JSON-NEXT:                      "tokLen": 1
+// JSON-NEXT:                     }
+// JSON-NEXT:                    },
+// JSON-NEXT:                    "type": {
+// JSON-NEXT:                     "qualType": "int"
+// JSON-NEXT:                    },
+// JSON-NEXT:                    "valueCategory": "prvalue",
+// JSON-NEXT:                    "value": "5"
+// JSON-NEXT:                   }
+// JSON-NEXT:                  ]
+// JSON-NEXT:                 }
+// JSON-NEXT:                ]
+// JSON-NEXT:               }
+// JSON-NEXT:              ]
+// JSON-NEXT:             }
+// JSON-NEXT:            ]
+// JSON-NEXT:           }
+// JSON-NEXT:          ]
+// JSON-NEXT:         }
+// JSON-NEXT:        ]
+// JSON-NEXT:       },
+// JSON-NEXT:       {
+// JSON-NEXT:        "id": "0x{{.*}}",
+// JSON-NEXT:        "kind": "CXXConstructorDecl",
+// JSON-NEXT:        "loc": {
+// JSON-NEXT:         "offset": 812,
+// JSON-NEXT:         "line": 15,
+// JSON-NEXT:         "col": 8,
+// JSON-NEXT:         "tokLen": 3
+// JSON-NEXT:        },
+// JSON-NEXT:        "range": {
+// JSON-NEXT:         "begin": {
+// JSON-NEXT:          "offset": 812,
+// JSON-NEXT:          "col": 8,
+// JSON-NEXT:          "tokLen": 3
+// JSON-NEXT:         },
+// JSON-NEXT:         "end": {
+// JSON-NEXT:          "offset": 812,
+// JSON-NEXT:          "col": 8,
+// JSON-NEXT:          "tokLen": 3
+// JSON-NEXT:         }
+// JSON-NEXT:        },
+// JSON-NEXT:        "isImplicit": true,
+// JSON-NEXT:        "name": "foo",
+// JSON-NEXT:        "mangledName": "_ZN3fooILi5EiLi5EEC1ERKS0_",
+// JSON-NEXT:        "type": {
+// JSON-NEXT:         "qualType": "void (const foo<5, int> &)"
+// JSON-NEXT:        },
+// JSON-NEXT:        "inline": true,
+// JSON-NEXT:        "constexpr": true,
+// JSON-NEXT:        "explicitlyDefaulted": "default",
+// JSON-NEXT:        "inner": [
+// JSON-NEXT:         {
+// JSON-NEXT:          "id": "0x{{.*}}",
+// JSON-NEXT:          "kind": "ParmVarDecl",
+// JSON-NEXT:          "loc": {
+// JSON-NEXT:           "offset": 812,
+// JSON-NEXT:           "col": 8,
+// JSON-NEXT:           "tokLen": 3
+// JSON-NEXT:          },
+// JSON-NEXT:          "range": {
+// JSON-NEXT:           "begin": {
+// JSON-NEXT:            "offset": 812,
+// JSON-NEXT:            "col": 8,
+// JSON-NEXT:            "tokLen": 3
+// JSON-NEXT:           },
+// JSON-NEXT:           "end": {
+// JSON-NEXT:            "offset": 812,
+// JSON-NEXT:            "col": 8,
+// JSON-NEXT:            "tokLen": 3
+// JSON-NEXT:           }
+// JSON-NEXT:          },
+// JSON-NEXT:          "type": {
+// JSON-NEXT:           "qualType": "const foo<5, int> &"
+// JSON-NEXT:          }
+// JSON-NEXT:         }
+// JSON-NEXT:        ]
+// JSON-NEXT:       },
+// JSON-NEXT:       {
+// JSON-NEXT:        "id": "0x{{.*}}",
+// JSON-NEXT:        "kind": "CXXConstructorDecl",
+// JSON-NEXT:        "loc": {
+// JSON-NEXT:         "offset": 812,
+// JSON-NEXT:         "col": 8,
+// JSON-NEXT:         "tokLen": 3
+// JSON-NEXT:        },
+// JSON-NEXT:        "range": {
+// JSON-NEXT:         "begin": {
+// JSON-NEXT:          "offset": 812,
+// JSON-NEXT:          "col": 8,
+// JSON-NEXT:          "tokLen": 3
+// JSON-NEXT:         },
+// JSON-NEXT:         "end": {
+// JSON-NEXT:          "offset": 812,
+// JSON-NEXT:          "col": 8,
+// JSON-NEXT:          "tokLen": 3
+// JSON-NEXT:         }
+// JSON-NEXT:        },
+// JSON-NEXT:        "isImplicit": true,
+// JSON-NEXT:        "name": "foo",
+// JSON-NEXT:        "mangledName": "_ZN3fooILi5EiLi5EEC1EOS0_",
+// JSON-NEXT:        "type": {
+// JSON-NEXT:         "qualType": "void (foo<5, int> &&)"
+// JSON-NEXT:        },
+// JSON-NEXT:        "inline": true,
+// JSON-NEXT:        "constexpr": true,
+// JSON-NEXT:        "explicitlyDefaulted": "default",
+// JSON-NEXT:        "inner": [
+// JSON-NEXT:         {
+// JSON-NEXT:          "id": "0x{{.*}}",
+// JSON-NEXT:          "kind": "ParmVarDecl",
+// JSON-NEXT:          "loc": {
+// JSON-NEXT:           "offset": 812,
+// JSON-NEXT:           "col": 8,
+// JSON-NEXT:           "tokLen": 3
+// JSON-NEXT:          },
+// JSON-NEXT:          "range": {
+// JSON-NEXT:           "begin": {
+// JSON-NEXT:            "offset": 812,
+// JSON-NEXT:            "col": 8,
+// JSON-NEXT:            "tokLen": 3
+// JSON-NEXT:           },
+// JSON-NEXT:           "end": {
+// JSON-NEXT:            "offset": 812,
+// JSON-NEXT:            "col": 8,
+// JSON-NEXT:            "tokLen": 3
+// JSON-NEXT:           }
+// JSON-NEXT:          },
+// JSON-NEXT:          "type": {
+// JSON-NEXT:           "qualType": "foo<5, int> &&"
+// JSON-NEXT:          }
+// JSON-NEXT:         }
+// JSON-NEXT:        ]
+// JSON-NEXT:       },
+// JSON-NEXT:       {
+// JSON-NEXT:        "id": "0x{{.*}}",
+// JSON-NEXT:        "kind": "CXXDestructorDecl",
+// JSON-NEXT:        "loc": {
+// JSON-NEXT:         "offset": 812,
+// JSON-NEXT:         "col": 8,
+// JSON-NEXT:         "tokLen": 3
+// JSON-NEXT:        },
+// JSON-NEXT:        "range": {
+// JSON-NEXT:         "begin": {
+// JSON-NEXT:          "offset": 812,
+// JSON-NEXT:          "col": 8,
+// JSON-NEXT:          "tokLen": 3
+// JSON-NEXT:         },
+// JSON-NEXT:         "end": {
+// JSON-NEXT:          "offset": 812,
+// JSON-NEXT:          "col": 8,
+// JSON-NEXT:          "tokLen": 3
+// JSON-NEXT:         }
+// JSON-NEXT:        },
+// JSON-NEXT:        "isImplicit": true,
+// JSON-NEXT:        "isReferenced": true,
+// JSON-NEXT:        "name": "~foo",
+// JSON-NEXT:        "mangledName": "_ZN3fooILi5EiLi5EED1Ev",
+// JSON-NEXT:        "type": {
+// JSON-NEXT:         "qualType": "void () noexcept"
+// JSON-NEXT:        },
+// JSON-NEXT:        "inline": true,
+// JSON-NEXT:        "explicitlyDefaulted": "default"
+// JSON-NEXT:       }
+// JSON-NEXT:      ]
+// JSON-NEXT:     },
+// JSON-NEXT:     {
+// JSON-NEXT:      "id": "0x{{.*}}",
+// JSON-NEXT:      "kind": "ClassTemplateSpecializationDecl",
+// JSON-NEXT:      "loc": {
+// JSON-NEXT:       "offset": 812,
+// JSON-NEXT:       "col": 8,
+// JSON-NEXT:       "tokLen": 3
+// JSON-NEXT:      },
+// JSON-NEXT:      "range": {
+// JSON-NEXT:       "begin": {
+// JSON-NEXT:        "offset": 765,
+// JSON-NEXT:        "line": 14,
+// JSON-NEXT:        "col": 1,
+// JSON-NEXT:        "tokLen": 8
+// JSON-NEXT:       },
+// JSON-NEXT:       "end": {
+// JSON-NEXT:        "offset": 879,
+// JSON-NEXT:        "line": 19,
+// JSON-NEXT:        "col": 1,
+// JSON-NEXT:        "tokLen": 1
+// JSON-NEXT:       }
+// JSON-NEXT:      },
+// JSON-NEXT:      "name": "foo",
+// JSON-NEXT:      "tagUsed": "struct",
+// JSON-NEXT:      "completeDefinition": true,
+// JSON-NEXT:      "definitionData": {
+// JSON-NEXT:       "canConstDefaultInit": true,
+// JSON-NEXT:       "canPassInRegisters": true,
+// JSON-NEXT:       "copyAssign": {
+// JSON-NEXT:        "hasConstParam": true,
+// JSON-NEXT:        "implicitHasConstParam": true,
+// JSON-NEXT:        "needsImplicit": true,
+// JSON-NEXT:        "simple": true,
+// JSON-NEXT:        "trivial": true
+// JSON-NEXT:       },
+// JSON-NEXT:       "copyCtor": {
+// JSON-NEXT:        "hasConstParam": true,
+// JSON-NEXT:        "implicitHasConstParam": true,
+// JSON-NEXT:        "simple": true,
+// JSON-NEXT:        "trivial": true
+// JSON-NEXT:       },
+// JSON-NEXT:       "defaultCtor": {
+// JSON-NEXT:        "exists": true,
+// JSON-NEXT:        "nonTrivial": true,
+// JSON-NEXT:        "userProvided": true
+// JSON-NEXT:       },
+// JSON-NEXT:       "dtor": {
+// JSON-NEXT:        "irrelevant": true,
+// JSON-NEXT:        "simple": true,
+// JSON-NEXT:        "trivial": true
+// JSON-NEXT:       },
+// JSON-NEXT:       "hasUserDeclaredConstructor": true,
+// JSON-NEXT:       "isStandardLayout": true,
+// JSON-NEXT:       "isTriviallyCopyable": true,
+// JSON-NEXT:       "moveAssign": {
+// JSON-NEXT:        "exists": true,
+// JSON-NEXT:        "needsImplicit": true,
+// JSON-NEXT:        "simple": true,
+// JSON-NEXT:        "trivial": true
+// JSON-NEXT:       },
+// JSON-NEXT:       "moveCtor": {
+// JSON-NEXT:        "exists": true,
+// JSON-NEXT:        "simple": true,
+// JSON-NEXT:        "trivial": true
+// JSON-NEXT:       }
+// JSON-NEXT:      },
+// JSON-NEXT:      "inner": [
+// JSON-NEXT:       {
+// JSON-NEXT:        "kind": "TemplateArgument",
+// JSON-NEXT:        "value": 2
+// JSON-NEXT:       },
+// JSON-NEXT:       {
+// JSON-NEXT:        "kind": "TemplateArgument",
+// JSON-NEXT:        "type": {
+// JSON-NEXT:         "qualType": "double"
+// JSON-NEXT:        },
+// JSON-NEXT:        "inner": [
+// JSON-NEXT:         {
+// JSON-NEXT:          "id": "0x{{.*}}",
+// JSON-NEXT:          "kind": "BuiltinType",
+// JSON-NEXT:          "type": {
+// JSON-NEXT:           "qualType": "double"
+// JSON-NEXT:          }
+// JSON-NEXT:         }
+// JSON-NEXT:        ]
+// JSON-NEXT:       },
+// JSON-NEXT:       {
+// JSON-NEXT:        "kind": "TemplateArgument",
+// JSON-NEXT:        "value": 3
+// JSON-NEXT:       },
+// JSON-NEXT:       {
+// JSON-NEXT:        "id": "0x{{.*}}",
+// JSON-NEXT:        "kind": "CXXRecordDecl",
+// JSON-NEXT:        "loc": {
+// JSON-NEXT:         "offset": 812,
+// JSON-NEXT:         "line": 15,
+// JSON-NEXT:         "col": 8,
+// JSON-NEXT:         "tokLen": 3
+// JSON-NEXT:        },
+// JSON-NEXT:        "range": {
+// JSON-NEXT:         "begin": {
+// JSON-NEXT:          "offset": 805,
+// JSON-NEXT:          "col": 1,
+// JSON-NEXT:          "tokLen": 6
+// JSON-NEXT:         },
+// JSON-NEXT:         "end": {
+// JSON-NEXT:          "offset": 812,
+// JSON-NEXT:          "col": 8,
+// JSON-NEXT:          "tokLen": 3
+// JSON-NEXT:         }
+// JSON-NEXT:        },
+// JSON-NEXT:        "isImplicit": true,
+// JSON-NEXT:        "name": "foo",
+// JSON-NEXT:        "tagUsed": "struct"
+// JSON-NEXT:       },
+// JSON-NEXT:       {
+// JSON-NEXT:        "id": "0x{{.*}}",
+// JSON-NEXT:        "kind": "FieldDecl",
+// JSON-NEXT:        "loc": {
+// JSON-NEXT:         "offset": 824,
+// JSON-NEXT:         "line": 16,
+// JSON-NEXT:         "col": 7,
+// JSON-NEXT:         "tokLen": 8
+// JSON-NEXT:        },
+// JSON-NEXT:        "range": {
+// JSON-NEXT:         "begin": {
+// JSON-NEXT:          "offset": 820,
+// JSON-NEXT:          "col": 3,
+// JSON-NEXT:          "tokLen": 3
+// JSON-NEXT:         },
+// JSON-NEXT:         "end": {
+// JSON-NEXT:          "offset": 824,
+// JSON-NEXT:          "col": 7,
+// JSON-NEXT:          "tokLen": 8
+// JSON-NEXT:         }
+// JSON-NEXT:        },
+// JSON-NEXT:        "name": "constant",
+// JSON-NEXT:        "type": {
+// JSON-NEXT:         "qualType": "int"
+// JSON-NEXT:        }
+// JSON-NEXT:       },
+// JSON-NEXT:       {
+// JSON-NEXT:        "id": "0x{{.*}}",
+// JSON-NEXT:        "kind": "CXXConstructorDecl",
+// JSON-NEXT:        "loc": {
+// JSON-NEXT:         "offset": 836,
+// JSON-NEXT:         "line": 17,
+// JSON-NEXT:         "col": 3,
+// JSON-NEXT:         "tokLen": 3
+// JSON-NEXT:        },
+// JSON-NEXT:        "range": {
+// JSON-NEXT:         "begin": {
+// JSON-NEXT:          "offset": 836,
+// JSON-NEXT:          "col": 3,
+// JSON-NEXT:          "tokLen": 3
+// JSON-NEXT:         },
+// JSON-NEXT:         "end": {
+// JSON-NEXT:          "offset": 843,
+// JSON-NEXT:          "col": 10,
+// JSON-NEXT:          "tokLen": 1
+// JSON-NEXT:         }
+// JSON-NEXT:        },
+// JSON-NEXT:        "isUsed": true,
+// JSON-NEXT:        "name": "foo",
+// JSON-NEXT:        "mangledName": "_ZN3fooILi2EdLi3EEC1Ev",
+// JSON-NEXT:        "type": {
+// JSON-NEXT:         "qualType": "void ()"
+// JSON-NEXT:        },
+// JSON-NEXT:        "inner": [
+// JSON-NEXT:         {
+// JSON-NEXT:          "id": "0x{{.*}}",
+// JSON-NEXT:          "kind": "CompoundStmt",
+// JSON-NEXT:          "range": {
+// JSON-NEXT:           "begin": {
+// JSON-NEXT:            "offset": 842,
+// JSON-NEXT:            "col": 9,
+// JSON-NEXT:            "tokLen": 1
+// JSON-NEXT:           },
+// JSON-NEXT:           "end": {
+// JSON-NEXT:            "offset": 843,
+// JSON-NEXT:            "col": 10,
+// JSON-NEXT:            "tokLen": 1
+// JSON-NEXT:           }
+// JSON-NEXT:          }
+// JSON-NEXT:         }
+// JSON-NEXT:        ]
+// JSON-NEXT:       },
+// JSON-NEXT:       {
+// JSON-NEXT:        "id": "0x{{.*}}",
+// JSON-NEXT:        "kind": "CXXMethodDecl",
+// JSON-NEXT:        "loc": {
+// JSON-NEXT:         "offset": 849,
+// JSON-NEXT:         "line": 18,
+// JSON-NEXT:         "col": 5,
+// JSON-NEXT:         "tokLen": 6
+// JSON-NEXT:        },
+// JSON-NEXT:        "range": {
+// JSON-NEXT:         "begin": {
+// JSON-NEXT:          "offset": 847,
+// JSON-NEXT:          "col": 3,
+// JSON-NEXT:          "tokLen": 1
+// JSON-NEXT:         },
+// JSON-NEXT:         "end": {
+// JSON-NEXT:          "offset": 877,
+// JSON-NEXT:          "col": 33,
+// JSON-NEXT:          "tokLen": 1
+// JSON-NEXT:         }
+// JSON-NEXT:        },
+// JSON-NEXT:        "isUsed": true,
+// JSON-NEXT:        "name": "getSum",
+// JSON-NEXT:        "mangledName": "_ZN3fooILi2EdLi3EE6getSumEv",
+// JSON-NEXT:        "type": {
+// JSON-NEXT:         "qualType": "double ()"
+// JSON-NEXT:        },
+// JSON-NEXT:        "inner": [
+// JSON-NEXT:         {
+// JSON-NEXT:          "id": "0x{{.*}}",
+// JSON-NEXT:          "kind": "CompoundStmt",
+// JSON-NEXT:          "range": {
+// JSON-NEXT:           "begin": {
+// JSON-NEXT:            "offset": 858,
+// JSON-NEXT:            "col": 14,
+// JSON-NEXT:            "tokLen": 1
+// JSON-NEXT:           },
+// JSON-NEXT:           "end": {
+// JSON-NEXT:            "offset": 877,
+// JSON-NEXT:            "col": 33,
+// JSON-NEXT:            "tokLen": 1
+// JSON-NEXT:           }
+// JSON-NEXT:          },
+// JSON-NEXT:          "inner": [
+// JSON-NEXT:           {
+// JSON-NEXT:            "id": "0x{{.*}}",
+// JSON-NEXT:            "kind": "ReturnStmt",
+// JSON-NEXT:            "range": {
+// JSON-NEXT:             "begin": {
+// JSON-NEXT:              "offset": 860,
+// JSON-NEXT:              "col": 16,
+// JSON-NEXT:              "tokLen": 6
+// JSON-NEXT:             },
+// JSON-NEXT:             "end": {
+// JSON-NEXT:              "offset": 874,
+// JSON-NEXT:              "col": 30,
+// JSON-NEXT:              "tokLen": 1
+// JSON-NEXT:             }
+// JSON-NEXT:            },
+// JSON-NEXT:            "inner": [
+// JSON-NEXT:             {
+// JSON-NEXT:              "id": "0x{{.*}}",
+// JSON-NEXT:              "kind": "CXXFunctionalCastExpr",
+// JSON-NEXT:              "range": {
+// JSON-NEXT:               "begin": {
+// JSON-NEXT:                "offset": 867,
+// JSON-NEXT:                "col": 23,
+// JSON-NEXT:                "tokLen": 1
+// JSON-NEXT:               },
+// JSON-NEXT:               "end": {
+// JSON-NEXT:                "offset": 874,
+// JSON-NEXT:                "col": 30,
+// JSON-NEXT:                "tokLen": 1
+// JSON-NEXT:               }
+// JSON-NEXT:              },
+// JSON-NEXT:              "type": {
+// JSON-NEXT:               "qualType": "double"
+// JSON-NEXT:              },
+// JSON-NEXT:              "valueCategory": "prvalue",
+// JSON-NEXT:              "castKind": "NoOp",
+// JSON-NEXT:              "inner": [
+// JSON-NEXT:               {
+// JSON-NEXT:                "id": "0x{{.*}}",
+// JSON-NEXT:                "kind": "ImplicitCastExpr",
+// JSON-NEXT:                "range": {
+// JSON-NEXT:                 "begin": {
+// JSON-NEXT:                  "offset": 869,
+// JSON-NEXT:                  "col": 25,
+// JSON-NEXT:                  "tokLen": 1
+// JSON-NEXT:                 },
+// JSON-NEXT:                 "end": {
+// JSON-NEXT:                  "offset": 873,
+// JSON-NEXT:                  "col": 29,
+// JSON-NEXT:                  "tokLen": 1
+// JSON-NEXT:                 }
+// JSON-NEXT:                },
+// JSON-NEXT:                "type": {
+// JSON-NEXT:                 "qualType": "double"
+// JSON-NEXT:                },
+// JSON-NEXT:                "valueCategory": "prvalue",
+// JSON-NEXT:                "castKind": "IntegralToFloating",
+// JSON-NEXT:                "isPartOfExplicitCast": true,
+// JSON-NEXT:                "inner": [
+// JSON-NEXT:                 {
+// JSON-NEXT:                  "id": "0x{{.*}}",
+// JSON-NEXT:                  "kind": "BinaryOperator",
+// JSON-NEXT:                  "range": {
+// JSON-NEXT:                   "begin": {
+// JSON-NEXT:                    "offset": 869,
+// JSON-NEXT:                    "col": 25,
+// JSON-NEXT:                    "tokLen": 1
+// JSON-NEXT:                   },
+// JSON-NEXT:                   "end": {
+// JSON-NEXT:                    "offset": 873,
+// JSON-NEXT:                    "col": 29,
+// JSON-NEXT:                    "tokLen": 1
+// JSON-NEXT:                   }
+// JSON-NEXT:                  },
+// JSON-NEXT:                  "type": {
+// JSON-NEXT:                   "qualType": "int"
+// JSON-NEXT:                  },
+// JSON-NEXT:                  "valueCategory": "prvalue",
+// JSON-NEXT:                  "opcode": "+",
+// JSON-NEXT:                  "inner": [
+// JSON-NEXT:                   {
+// JSON-NEXT:                    "id": "0x{{.*}}",
+// JSON-NEXT:                    "kind": "SubstNonTypeTemplateParmExpr",
+// JSON-NEXT:                    "range": {
+// JSON-NEXT:                     "begin": {
+// JSON-NEXT:                      "offset": 869,
+// JSON-NEXT:                      "col": 25,
+// JSON-NEXT:                      "tokLen": 1
+// JSON-NEXT:                     },
+// JSON-NEXT:                     "end": {
+// JSON-NEXT:                      "offset": 869,
+// JSON-NEXT:                      "col": 25,
+// JSON-NEXT:                      "tokLen": 1
+// JSON-NEXT:                     }
+// JSON-NEXT:                    },
+// JSON-NEXT:                    "type": {
+// JSON-NEXT:                     "qualType": "int"
+// JSON-NEXT:                    },
+// JSON-NEXT:                    "valueCategory": "prvalue",
+// JSON-NEXT:                    "inner": [
+// JSON-NEXT:                     {
+// JSON-NEXT:                      "id": "0x{{.*}}",
+// JSON-NEXT:                      "kind": "NonTypeTemplateParmDecl",
+// JSON-NEXT:                      "loc": {
+// JSON-NEXT:                       "offset": 779,
+// JSON-NEXT:                       "line": 14,
+// JSON-NEXT:                       "col": 15,
+// JSON-NEXT:                       "tokLen": 1
+// JSON-NEXT:                      },
+// JSON-NEXT:                      "range": {
+// JSON-NEXT:                       "begin": {
+// JSON-NEXT:                        "offset": 775,
+// JSON-NEXT:                        "col": 11,
+// JSON-NEXT:                        "tokLen": 3
+// JSON-NEXT:                       },
+// JSON-NEXT:                       "end": {
+// JSON-NEXT:                        "offset": 779,
+// JSON-NEXT:                        "col": 15,
+// JSON-NEXT:                        "tokLen": 1
+// JSON-NEXT:                       }
+// JSON-NEXT:                      },
+// JSON-NEXT:                      "isReferenced": true,
+// JSON-NEXT:                      "name": "X",
+// JSON-NEXT:                      "type": {
+// JSON-NEXT:                       "qualType": "int"
+// JSON-NEXT:                      },
+// JSON-NEXT:                      "depth": 0,
+// JSON-NEXT:                      "index": 0
+// JSON-NEXT:                     },
+// JSON-NEXT:                     {
+// JSON-NEXT:                      "id": "0x{{.*}}",
+// JSON-NEXT:                      "kind": "IntegerLiteral",
+// JSON-NEXT:                      "range": {
+// JSON-NEXT:                       "begin": {
+// JSON-NEXT:                        "offset": 869,
+// JSON-NEXT:                        "line": 18,
+// JSON-NEXT:                        "col": 25,
+// JSON-NEXT:                        "tokLen": 1
+// JSON-NEXT:                       },
+// JSON-NEXT:                       "end": {
+// JSON-NEXT:                        "offset": 869,
+// JSON-NEXT:                        "col": 25,
+// JSON-NEXT:                        "tokLen": 1
+// JSON-NEXT:                       }
+// JSON-NEXT:                      },
+// JSON-NEXT:                      "type": {
+// JSON-NEXT:                       "qualType": "int"
+// JSON-NEXT:                      },
+// JSON-NEXT:                      "valueCategory": "prvalue",
+// JSON-NEXT:                      "value": "2"
+// JSON-NEXT:                     }
+// JSON-NEXT:                    ]
+// JSON-NEXT:                   },
+// JSON-NEXT:                   {
+// JSON-NEXT:                    "id": "0x{{.*}}",
+// JSON-NEXT:                    "kind": "SubstNonTypeTemplateParmExpr",
+// JSON-NEXT:                    "range": {
+// JSON-NEXT:                     "begin": {
+// JSON-NEXT:                      "offset": 873,
+// JSON-NEXT:                      "col": 29,
+// JSON-NEXT:                      "tokLen": 1
+// JSON-NEXT:                     },
+// JSON-NEXT:                     "end": {
+// JSON-NEXT:                      "offset": 873,
+// JSON-NEXT:                      "col": 29,
+// JSON-NEXT:                      "tokLen": 1
+// JSON-NEXT:                     }
+// JSON-NEXT:                    },
+// JSON-NEXT:                    "type": {
+// JSON-NEXT:                     "qualType": "int"
+// JSON-NEXT:                    },
+// JSON-NEXT:                    "valueCategory": "prvalue",
+// JSON-NEXT:                    "inner": [
+// JSON-NEXT:                     {
+// JSON-NEXT:                      "id": "0x{{.*}}",
+// JSON-NEXT:                      "kind": "NonTypeTemplateParmDecl",
+// JSON-NEXT:                      "loc": {
+// JSON-NEXT:                       "offset": 798,
+// JSON-NEXT:                       "line": 14,
+// JSON-NEXT:                       "col": 34,
+// JSON-NEXT:                       "tokLen": 1
+// JSON-NEXT:                      },
+// JSON-NEXT:                      "range": {
+// JSON-NEXT:                       "begin": {
+// JSON-NEXT:                        "offset": 794,
+// JSON-NEXT:                        "col": 30,
+// JSON-NEXT:                        "tokLen": 3
+// JSON-NEXT:                       },
+// JSON-NEXT:                       "end": {
+// JSON-NEXT:                        "offset": 802,
+// JSON-NEXT:                        "col": 38,
+// JSON-NEXT:                        "tokLen": 1
+// JSON-NEXT:                       }
+// JSON-NEXT:                      },
+// JSON-NEXT:                      "isReferenced": true,
+// JSON-NEXT:                      "name": "Z",
+// JSON-NEXT:                      "type": {
+// JSON-NEXT:                       "qualType": "int"
+// JSON-NEXT:                      },
+// JSON-NEXT:                      "depth": 0,
+// JSON-NEXT:                      "index": 2,
+// JSON-NEXT:                      "defaultArg": {
+// JSON-NEXT:                       "kind": "TemplateArgument",
+// JSON-NEXT:                       "isExpr": true
+// JSON-NEXT:                      },
+// JSON-NEXT:                      "inner": [
+// JSON-NEXT:                       {
+// JSON-NEXT:                        "kind": "TemplateArgument",
+// JSON-NEXT:                        "range": {
+// JSON-NEXT:                         "begin": {
+// JSON-NEXT:                          "offset": 802,
+// JSON-NEXT:                          "col": 38,
+// JSON-NEXT:                          "tokLen": 1
+// JSON-NEXT:                         },
+// JSON-NEXT:                         "end": {
+// JSON-NEXT:                          "offset": 802,
+// JSON-NEXT:                          "col": 38,
+// JSON-NEXT:                          "tokLen": 1
+// JSON-NEXT:                         }
+// JSON-NEXT:                        },
+// JSON-NEXT:                        "isExpr": true,
+// JSON-NEXT:                        "inner": [
+// JSON-NEXT:                         {
+// JSON-NEXT:                          "id": "0x{{.*}}",
+// JSON-NEXT:                          "kind": "IntegerLiteral",
+// JSON-NEXT:                          "range": {
+// JSON-NEXT:                           "begin": {
+// JSON-NEXT:                            "offset": 802,
+// JSON-NEXT:                            "col": 38,
+// JSON-NEXT:                            "tokLen": 1
+// JSON-NEXT:                           },
+// JSON-NEXT:                           "end": {
+// JSON-NEXT:                            "offset": 802,
+// JSON-NEXT:                            "col": 38,
+// JSON-NEXT:                            "tokLen": 1
+// JSON-NEXT:                           }
+// JSON-NEXT:                          },
+// JSON-NEXT:                          "type": {
+// JSON-NEXT:                           "qualType": "int"
+// JSON-NEXT:                          },
+// JSON-NEXT:                          "valueCategory": "prvalue",
+// JSON-NEXT:                          "value": "5"
+// JSON-NEXT:                         }
+// JSON-NEXT:                        ]
+// JSON-NEXT:                       }
+// JSON-NEXT:                      ]
+// JSON-NEXT:                     },
+// JSON-NEXT:                     {
+// JSON-NEXT:                      "id": "0x{{.*}}",
+// JSON-NEXT:                      "kind": "IntegerLiteral",
+// JSON-NEXT:                      "range": {
+// JSON-NEXT:                       "begin": {
+// JSON-NEXT:                        "offset": 873,
+// JSON-NEXT:                        "line": 18,
+// JSON-NEXT:                        "col": 29,
+// JSON-NEXT:                        "tokLen": 1
+// JSON-NEXT:                       },
+// JSON-NEXT:                       "end": {
+// JSON-NEXT:                        "offset": 873,
+// JSON-NEXT:                        "col": 29,
+// JSON-NEXT:                        "tokLen": 1
+// JSON-NEXT:                       }
+// JSON-NEXT:                      },
+// JSON-NEXT:                      "type": {
+// JSON-NEXT:                       "qualType": "int"
+// JSON-NEXT:                      },
+// JSON-NEXT:                      "valueCategory": "prvalue",
+// JSON-NEXT:                      "value": "3"
+// JSON-NEXT:                     }
+// JSON-NEXT:                    ]
+// JSON-NEXT:                   }
+// JSON-NEXT:                  ]
+// JSON-NEXT:                 }
+// JSON-NEXT:                ]
+// JSON-NEXT:               }
+// JSON-NEXT:              ]
+// JSON-NEXT:             }
+// JSON-NEXT:            ]
+// JSON-NEXT:           }
+// JSON-NEXT:          ]
+// JSON-NEXT:         }
+// JSON-NEXT:        ]
+// JSON-NEXT:       },
+// JSON-NEXT:       {
+// JSON-NEXT:        "id": "0x{{.*}}",
+// JSON-NEXT:        "kind": "CXXConstructorDecl",
+// JSON-NEXT:        "loc": {
+// JSON-NEXT:         "offset": 812,
+// JSON-NEXT:         "line": 15,
+// JSON-NEXT:         "col": 8,
+// JSON-NEXT:         "tokLen": 3
+// JSON-NEXT:        },
+// JSON-NEXT:        "range": {
+// JSON-NEXT:         "begin": {
+// JSON-NEXT:          "offset": 812,
+// JSON-NEXT:          "col": 8,
+// JSON-NEXT:          "tokLen": 3
+// JSON-NEXT:         },
+// JSON-NEXT:         "end": {
+// JSON-NEXT:          "offset": 812,
+// JSON-NEXT:          "col": 8,
+// JSON-NEXT:          "tokLen": 3
+// JSON-NEXT:         }
+// JSON-NEXT:        },
+// JSON-NEXT:        "isImplicit": true,
+// JSON-NEXT:        "name": "foo",
+// JSON-NEXT:        "mangledName": "_ZN3fooILi2EdLi3EEC1ERKS0_",
+// JSON-NEXT:        "type": {
+// JSON-NEXT:         "qualType": "void (const foo<2, double, 3> &)"
+// JSON-NEXT:        },
+// JSON-NEXT:        "inline": true,
+// JSON-NEXT:        "constexpr": true,
+// JSON-NEXT:        "explicitlyDefaulted": "default",
+// JSON-NEXT:        "inner": [
+// JSON-NEXT:         {
+// JSON-NEXT:          "id": "0x{{.*}}",
+// JSON-NEXT:          "kind": "ParmVarDecl",
+// JSON-NEXT:          "loc": {
+// JSON-NEXT:           "offset": 812,
+// JSON-NEXT:           "col": 8,
+// JSON-NEXT:           "tokLen": 3
+// JSON-NEXT:          },
+// JSON-NEXT:          "range": {
+// JSON-NEXT:           "begin": {
+// JSON-NEXT:            "offset": 812,
+// JSON-NEXT:            "col": 8,
+// JSON-NEXT:            "tokLen": 3
+// JSON-NEXT:           },
+// JSON-NEXT:           "end": {
+// JSON-NEXT:            "offset": 812,
+// JSON-NEXT:            "col": 8,
+// JSON-NEXT:            "tokLen": 3
+// JSON-NEXT:           }
+// JSON-NEXT:          },
+// JSON-NEXT:          "type": {
+// JSON-NEXT:           "qualType": "const foo<2, double, 3> &"
+// JSON-NEXT:          }
+// JSON-NEXT:         }
+// JSON-NEXT:        ]
+// JSON-NEXT:       },
+// JSON-NEXT:       {
+// JSON-NEXT:        "id": "0x{{.*}}",
+// JSON-NEXT:        "kind": "CXXConstructorDecl",
+// JSON-NEXT:        "loc": {
+// JSON-NEXT:         "offset": 812,
+// JSON-NEXT:         "col": 8,
+// JSON-NEXT:         "tokLen": 3
+// JSON-NEXT:        },
+// JSON-NEXT:        "range": {
+// JSON-NEXT:         "begin": {
+// JSON-NEXT:          "offset": 812,
+// JSON-NEXT:          "col": 8,
+// JSON-NEXT:          "tokLen": 3
+// JSON-NEXT:         },
+// JSON-NEXT:         "end": {
+// JSON-NEXT:          "offset": 812,
+// JSON-NEXT:          "col": 8,
+// JSON-NEXT:          "tokLen": 3
+// JSON-NEXT:         }
+// JSON-NEXT:        },
+// JSON-NEXT:        "isImplicit": true,
+// JSON-NEXT:        "name": "foo",
+// JSON-NEXT:        "mangledName": "_ZN3fooILi2EdLi3EEC1EOS0_",
+// JSON-NEXT:        "type": {
+// JSON-NEXT:         "qualType": "void (foo<2, double, 3> &&)"
+// JSON-NEXT:        },
+// JSON-NEXT:        "inline": true,
+// JSON-NEXT:        "constexpr": true,
+// JSON-NEXT:        "explicitlyDefaulted": "default",
+// JSON-NEXT:        "inner": [
+// JSON-NEXT:         {
+// JSON-NEXT:          "id": "0x{{.*}}",
+// JSON-NEXT:          "kind": "ParmVarDecl",
+// JSON-NEXT:          "loc": {
+// JSON-NEXT:           "offset": 812,
+// JSON-NEXT:           "col": 8,
+// JSON-NEXT:           "tokLen": 3
+// JSON-NEXT:          },
+// JSON-NEXT:          "range": {
+// JSON-NEXT:           "begin": {
+// JSON-NEXT:            "offset": 812,
+// JSON-NEXT:            "col": 8,
+// JSON-NEXT:            "tokLen": 3
+// JSON-NEXT:           },
+// JSON-NEXT:           "end": {
+// JSON-NEXT:            "offset": 812,
+// JSON-NEXT:            "col": 8,
+// JSON-NEXT:            "tokLen": 3
+// JSON-NEXT:           }
+// JSON-NEXT:          },
+// JSON-NEXT:          "type": {
+// JSON-NEXT:           "qualType": "foo<2, double, 3> &&"
+// JSON-NEXT:          }
+// JSON-NEXT:         }
+// JSON-NEXT:        ]
+// JSON-NEXT:       },
+// JSON-NEXT:       {
+// JSON-NEXT:        "id": "0x{{.*}}",
+// JSON-NEXT:        "kind": "CXXDestructorDecl",
+// JSON-NEXT:        "loc": {
+// JSON-NEXT:         "offset": 812,
+// JSON-NEXT:         "col": 8,
+// JSON-NEXT:         "tokLen": 3
+// JSON-NEXT:        },
+// JSON-NEXT:        "range": {
+// JSON-NEXT:         "begin": {
+// JSON-NEXT:          "offset": 812,
+// JSON-NEXT:          "col": 8,
+// JSON-NEXT:          "tokLen": 3
+// JSON-NEXT:         },
+// JSON-NEXT:         "end": {
+// JSON-NEXT:          "offset": 812,
+// JSON-NEXT:          "col": 8,
+// JSON-NEXT:          "tokLen": 3
+// JSON-NEXT:         }
+// JSON-NEXT:        },
+// JSON-NEXT:        "isImplicit": true,
+// JSON-NEXT:        "isReferenced": true,
+// JSON-NEXT:        "name": "~foo",
+// JSON-NEXT:        "mangledName": "_ZN3fooILi2EdLi3EED1Ev",
+// JSON-NEXT:        "type": {
+// JSON-NEXT:         "qualType": "void () noexcept"
+// JSON-NEXT:        },
+// JSON-NEXT:        "inline": true,
+// JSON-NEXT:        "explicitlyDefaulted": "default"
+// JSON-NEXT:       }
+// JSON-NEXT:      ]
+// JSON-NEXT:     }
+// JSON-NEXT:    ]
+// JSON-NEXT:   },
+// JSON-NEXT:   {
+// JSON-NEXT:    "id": "0x{{.*}}",
+// JSON-NEXT:    "kind": "FunctionTemplateDecl",
+// JSON-NEXT:    "loc": {
+// JSON-NEXT:     "offset": 914,
+// JSON-NEXT:     "line": 22,
+// JSON-NEXT:     "col": 3,
+// JSON-NEXT:     "tokLen": 3
+// JSON-NEXT:    },
+// JSON-NEXT:    "range": {
+// JSON-NEXT:     "begin": {
+// JSON-NEXT:      "offset": 883,
+// JSON-NEXT:      "line": 21,
+// JSON-NEXT:      "col": 1,
+// JSON-NEXT:      "tokLen": 8
+// JSON-NEXT:     },
+// JSON-NEXT:     "end": {
+// JSON-NEXT:      "offset": 937,
+// JSON-NEXT:      "line": 24,
+// JSON-NEXT:      "col": 1,
+// JSON-NEXT:      "tokLen": 1
+// JSON-NEXT:     }
+// JSON-NEXT:    },
+// JSON-NEXT:    "name": "bar",
+// JSON-NEXT:    "inner": [
+// JSON-NEXT:     {
+// JSON-NEXT:      "id": "0x{{.*}}",
+// JSON-NEXT:      "kind": "NonTypeTemplateParmDecl",
+// JSON-NEXT:      "loc": {
+// JSON-NEXT:       "offset": 897,
+// JSON-NEXT:       "line": 21,
+// JSON-NEXT:       "col": 15,
+// JSON-NEXT:       "tokLen": 1
+// JSON-NEXT:      },
+// JSON-NEXT:      "range": {
+// JSON-NEXT:       "begin": {
+// JSON-NEXT:        "offset": 893,
+// JSON-NEXT:        "col": 11,
+// JSON-NEXT:        "tokLen": 3
+// JSON-NEXT:       },
+// JSON-NEXT:       "end": {
+// JSON-NEXT:        "offset": 897,
+// JSON-NEXT:        "col": 15,
+// JSON-NEXT:        "tokLen": 1
+// JSON-NEXT:       }
+// JSON-NEXT:      },
+// JSON-NEXT:      "isReferenced": true,
+// JSON-NEXT:      "name": "A",
+// JSON-NEXT:      "type": {
+// JSON-NEXT:       "qualType": "int"
+// JSON-NEXT:      },
+// JSON-NEXT:      "depth": 0,
+// JSON-NEXT:      "index": 0
+// JSON-NEXT:     },
+// JSON-NEXT:     {
+// JSON-NEXT:      "id": "0x{{.*}}",
+// JSON-NEXT:      "kind": "TemplateTypeParmDecl",
+// JSON-NEXT:      "loc": {
+// JSON-NEXT:       "offset": 909,
+// JSON-NEXT:       "col": 27,
+// JSON-NEXT:       "tokLen": 1
+// JSON-NEXT:      },
+// JSON-NEXT:      "range": {
+// JSON-NEXT:       "begin": {
+// JSON-NEXT:        "offset": 900,
+// JSON-NEXT:        "col": 18,
+// JSON-NEXT:        "tokLen": 8
+// JSON-NEXT:       },
+// JSON-NEXT:       "end": {
+// JSON-NEXT:        "offset": 909,
+// JSON-NEXT:        "col": 27,
+// JSON-NEXT:        "tokLen": 1
+// JSON-NEXT:       }
+// JSON-NEXT:      },
+// JSON-NEXT:      "isReferenced": true,
+// JSON-NEXT:      "name": "B",
+// JSON-NEXT:      "tagUsed": "typename",
+// JSON-NEXT:      "depth": 0,
+// JSON-NEXT:      "index": 1
+// JSON-NEXT:     },
+// JSON-NEXT:     {
+// JSON-NEXT:      "id": "0x{{.*}}",
+// JSON-NEXT:      "kind": "FunctionDecl",
+// JSON-NEXT:      "loc": {
+// JSON-NEXT:       "offset": 914,
+// JSON-NEXT:       "line": 22,
+// JSON-NEXT:       "col": 3,
+// JSON-NEXT:       "tokLen": 3
+// JSON-NEXT:      },
+// JSON-NEXT:      "range": {
+// JSON-NEXT:       "begin": {
+// JSON-NEXT:        "offset": 912,
+// JSON-NEXT:        "col": 1,
+// JSON-NEXT:        "tokLen": 1
+// JSON-NEXT:       },
+// JSON-NEXT:       "end": {
+// JSON-NEXT:        "offset": 937,
+// JSON-NEXT:        "line": 24,
+// JSON-NEXT:        "col": 1,
+// JSON-NEXT:        "tokLen": 1
+// JSON-NEXT:       }
+// JSON-NEXT:      },
+// JSON-NEXT:      "name": "bar",
+// JSON-NEXT:      "type": {
+// JSON-NEXT:       "qualType": "B ()"
+// JSON-NEXT:      },
+// JSON-NEXT:      "inner": [
+// JSON-NEXT:       {
+// JSON-NEXT:        "id": "0x{{.*}}",
+// JSON-NEXT:        "kind": "CompoundStmt",
+// JSON-NEXT:        "range": {
+// JSON-NEXT:         "begin": {
+// JSON-NEXT:          "offset": 920,
+// JSON-NEXT:          "line": 22,
+// JSON-NEXT:          "col": 9,
+// JSON-NEXT:          "tokLen": 1
+// JSON-NEXT:         },
+// JSON-NEXT:         "end": {
+// JSON-NEXT:          "offset": 937,
+// JSON-NEXT:          "line": 24,
+// JSON-NEXT:          "col": 1,
+// JSON-NEXT:          "tokLen": 1
+// JSON-NEXT:         }
+// JSON-NEXT:        },
+// JSON-NEXT:        "inner": [
+// JSON-NEXT:         {
+// JSON-NEXT:          "id": "0x{{.*}}",
+// JSON-NEXT:          "kind": "ReturnStmt",
+// JSON-NEXT:          "range": {
+// JSON-NEXT:           "begin": {
+// JSON-NEXT:            "offset": 924,
+// JSON-NEXT:            "line": 23,
+// JSON-NEXT:            "col": 3,
+// JSON-NEXT:            "tokLen": 6
+// JSON-NEXT:           },
+// JSON-NEXT:           "end": {
+// JSON-NEXT:            "offset": 934,
+// JSON-NEXT:            "col": 13,
+// JSON-NEXT:            "tokLen": 1
+// JSON-NEXT:           }
+// JSON-NEXT:          },
+// JSON-NEXT:          "inner": [
+// JSON-NEXT:           {
+// JSON-NEXT:            "id": "0x{{.*}}",
+// JSON-NEXT:            "kind": "CXXUnresolvedConstructExpr",
+// JSON-NEXT:            "range": {
+// JSON-NEXT:             "begin": {
+// JSON-NEXT:              "offset": 931,
+// JSON-NEXT:              "col": 10,
+// JSON-NEXT:              "tokLen": 1
+// JSON-NEXT:             },
+// JSON-NEXT:             "end": {
+// JSON-NEXT:              "offset": 934,
+// JSON-NEXT:              "col": 13,
+// JSON-NEXT:              "tokLen": 1
+// JSON-NEXT:             }
+// JSON-NEXT:            },
+// JSON-NEXT:            "type": {
+// JSON-NEXT:             "qualType": "B"
+// JSON-NEXT:            },
+// JSON-NEXT:            "valueCategory": "prvalue",
+// JSON-NEXT:            "inner": [
+// JSON-NEXT:             {
+// JSON-NEXT:              "id": "0x{{.*}}",
+// JSON-NEXT:              "kind": "DeclRefExpr",
+// JSON-NEXT:              "range": {
+// JSON-NEXT:               "begin": {
+// JSON-NEXT:                "offset": 933,
+// JSON-NEXT:                "col": 12,
+// JSON-NEXT:                "tokLen": 1
+// JSON-NEXT:               },
+// JSON-NEXT:               "end": {
+// JSON-NEXT:                "offset": 933,
+// JSON-NEXT:                "col": 12,
+// JSON-NEXT:                "tokLen": 1
+// JSON-NEXT:               }
+// JSON-NEXT:              },
+// JSON-NEXT:              "type": {
+// JSON-NEXT:               "qualType": "int"
+// JSON-NEXT:              },
+// JSON-NEXT:              "valueCategory": "prvalue",
+// JSON-NEXT:              "referencedDecl": {
+// JSON-NEXT:               "id": "0x{{.*}}",
+// JSON-NEXT:               "kind": "NonTypeTemplateParmDecl",
+// JSON-NEXT:               "name": "A",
+// JSON-NEXT:               "type": {
+// JSON-NEXT:                "qualType": "int"
+// JSON-NEXT:               }
+// JSON-NEXT:              }
+// JSON-NEXT:             }
+// JSON-NEXT:            ]
+// JSON-NEXT:           }
+// JSON-NEXT:          ]
+// JSON-NEXT:         }
+// JSON-NEXT:        ]
+// JSON-NEXT:       }
+// JSON-NEXT:      ]
+// JSON-NEXT:     },
+// JSON-NEXT:     {
+// JSON-NEXT:      "id": "0x{{.*}}",
+// JSON-NEXT:      "kind": "FunctionDecl",
+// JSON-NEXT:      "loc": {
+// JSON-NEXT:       "offset": 914,
+// JSON-NEXT:       "line": 22,
+// JSON-NEXT:       "col": 3,
+// JSON-NEXT:       "tokLen": 3
+// JSON-NEXT:      },
+// JSON-NEXT:      "range": {
+// JSON-NEXT:       "begin": {
+// JSON-NEXT:        "offset": 912,
+// JSON-NEXT:        "col": 1,
+// JSON-NEXT:        "tokLen": 1
+// JSON-NEXT:       },
+// JSON-NEXT:       "end": {
+// JSON-NEXT:        "offset": 937,
+// JSON-NEXT:        "line": 24,
+// JSON-NEXT:        "col": 1,
+// JSON-NEXT:        "tokLen": 1
+// JSON-NEXT:       }
+// JSON-NEXT:      },
+// JSON-NEXT:      "isUsed": true,
+// JSON-NEXT:      "name": "bar",
+// JSON-NEXT:      "mangledName": "_Z3barILi5EiET0_v",
+// JSON-NEXT:      "type": {
+// JSON-NEXT:       "qualType": "int ()"
+// JSON-NEXT:      },
+// JSON-NEXT:      "inner": [
+// JSON-NEXT:       {
+// JSON-NEXT:        "kind": "TemplateArgument",
+// JSON-NEXT:        "value": 5
+// JSON-NEXT:       },
+// JSON-NEXT:       {
+// JSON-NEXT:        "kind": "TemplateArgument",
+// JSON-NEXT:        "type": {
+// JSON-NEXT:         "qualType": "int"
+// JSON-NEXT:        },
+// JSON-NEXT:        "inner": [
+// JSON-NEXT:         {
+// JSON-NEXT:          "id": "0x{{.*}}",
+// JSON-NEXT:          "kind": "BuiltinType",
+// JSON-NEXT:          "type": {
+// JSON-NEXT:           "qualType": "int"
+// JSON-NEXT:          }
+// JSON-NEXT:         }
+// JSON-NEXT:        ]
+// JSON-NEXT:       },
+// JSON-NEXT:       {
+// JSON-NEXT:        "id": "0x{{.*}}",
+// JSON-NEXT:        "kind": "CompoundStmt",
+// JSON-NEXT:        "range": {
+// JSON-NEXT:         "begin": {
+// JSON-NEXT:          "offset": 920,
+// JSON-NEXT:          "line": 22,
+// JSON-NEXT:          "col": 9,
+// JSON-NEXT:          "tokLen": 1
+// JSON-NEXT:         },
+// JSON-NEXT:         "end": {
+// JSON-NEXT:          "offset": 937,
+// JSON-NEXT:          "line": 24,
+// JSON-NEXT:          "col": 1,
+// JSON-NEXT:          "tokLen": 1
+// JSON-NEXT:         }
+// JSON-NEXT:        },
+// JSON-NEXT:        "inner": [
+// JSON-NEXT:         {
+// JSON-NEXT:          "id": "0x{{.*}}",
+// JSON-NEXT:          "kind": "ReturnStmt",
+// JSON-NEXT:          "range": {
+// JSON-NEXT:           "begin": {
+// JSON-NEXT:            "offset": 924,
+// JSON-NEXT:            "line": 23,
+// JSON-NEXT:            "col": 3,
+// JSON-NEXT:            "tokLen": 6
+// JSON-NEXT:           },
+// JSON-NEXT:           "end": {
+// JSON-NEXT:            "offset": 934,
+// JSON-NEXT:            "col": 13,
+// JSON-NEXT:            "tokLen": 1
+// JSON-NEXT:           }
+// JSON-NEXT:          },
+// JSON-NEXT:          "inner": [
+// JSON-NEXT:           {
+// JSON-NEXT:            "id": "0x{{.*}}",
+// JSON-NEXT:            "kind": "CXXFunctionalCastExpr",
+// JSON-NEXT:            "range": {
+// JSON-NEXT:             "begin": {
+// JSON-NEXT:              "offset": 931,
+// JSON-NEXT:              "col": 10,
+// JSON-NEXT:              "tokLen": 1
+// JSON-NEXT:             },
+// JSON-NEXT:             "end": {
+// JSON-NEXT:              "offset": 934,
+// JSON-NEXT:              "col": 13,
+// JSON-NEXT:              "tokLen": 1
+// JSON-NEXT:             }
+// JSON-NEXT:            },
+// JSON-NEXT:            "type": {
+// JSON-NEXT:             "qualType": "int"
+// JSON-NEXT:            },
+// JSON-NEXT:            "valueCategory": "prvalue",
+// JSON-NEXT:            "castKind": "NoOp",
+// JSON-NEXT:            "inner": [
+// JSON-NEXT:             {
+// JSON-NEXT:              "id": "0x{{.*}}",
+// JSON-NEXT:              "kind": "SubstNonTypeTemplateParmExpr",
+// JSON-NEXT:              "range": {
+// JSON-NEXT:               "begin": {
+// JSON-NEXT:                "offset": 933,
+// JSON-NEXT:                "col": 12,
+// JSON-NEXT:                "tokLen": 1
+// JSON-NEXT:               },
+// JSON-NEXT:               "end": {
+// JSON-NEXT:                "offset": 933,
+// JSON-NEXT:                "col": 12,
+// JSON-NEXT:                "tokLen": 1
+// JSON-NEXT:               }
+// JSON-NEXT:              },
+// JSON-NEXT:              "type": {
+// JSON-NEXT:               "qualType": "int"
+// JSON-NEXT:              },
+// JSON-NEXT:              "valueCategory": "prvalue",
+// JSON-NEXT:              "inner": [
+// JSON-NEXT:               {
+// JSON-NEXT:                "id": "0x{{.*}}",
+// JSON-NEXT:                "kind": "NonTypeTemplateParmDecl",
+// JSON-NEXT:                "loc": {
+// JSON-NEXT:                 "offset": 897,
+// JSON-NEXT:                 "line": 21,
+// JSON-NEXT:                 "col": 15,
+// JSON-NEXT:                 "tokLen": 1
+// JSON-NEXT:                },
+// JSON-NEXT:                "range": {
+// JSON-NEXT:                 "begin": {
+// JSON-NEXT:                  "offset": 893,
+// JSON-NEXT:                  "col": 11,
+// JSON-NEXT:                  "tokLen": 3
+// JSON-NEXT:                 },
+// JSON-NEXT:                 "end": {
+// JSON-NEXT:                  "offset": 897,
+// JSON-NEXT:                  "col": 15,
+// JSON-NEXT:                  "tokLen": 1
+// JSON-NEXT:                 }
+// JSON-NEXT:                },
+// JSON-NEXT:                "isReferenced": true,
+// JSON-NEXT:                "name": "A",
+// JSON-NEXT:                "type": {
+// JSON-NEXT:                 "qualType": "int"
+// JSON-NEXT:                },
+// JSON-NEXT:                "depth": 0,
+// JSON-NEXT:                "index": 0
+// JSON-NEXT:               },
+// JSON-NEXT:               {
+// JSON-NEXT:                "id": "0x{{.*}}",
+// JSON-NEXT:                "kind": "IntegerLiteral",
+// JSON-NEXT:                "range": {
+// JSON-NEXT:                 "begin": {
+// JSON-NEXT:                  "offset": 933,
+// JSON-NEXT:                  "line": 23,
+// JSON-NEXT:                  "col": 12,
+// JSON-NEXT:                  "tokLen": 1
+// JSON-NEXT:                 },
+// JSON-NEXT:                 "end": {
+// JSON-NEXT:                  "offset": 933,
+// JSON-NEXT:                  "col": 12,
+// JSON-NEXT:                  "tokLen": 1
+// JSON-NEXT:                 }
+// JSON-NEXT:                },
+// JSON-NEXT:                "type": {
+// JSON-NEXT:                 "qualType": "int"
+// JSON-NEXT:                },
+// JSON-NEXT:                "valueCategory": "prvalue",
+// JSON-NEXT:                "value": "5"
+// JSON-NEXT:               }
+// JSON-NEXT:              ]
+// JSON-NEXT:             }
+// JSON-NEXT:            ]
+// JSON-NEXT:           }
+// JSON-NEXT:          ]
+// JSON-NEXT:         }
+// JSON-NEXT:        ]
+// JSON-NEXT:       }
+// JSON-NEXT:      ]
+// JSON-NEXT:     }
+// JSON-NEXT:    ]
+// JSON-NEXT:   },
+// JSON-NEXT:   {
+// JSON-NEXT:    "id": "0x{{.*}}",
+// JSON-NEXT:    "kind": "FunctionDecl",
+// JSON-NEXT:    "loc": {
+// JSON-NEXT:     "offset": 945,
+// JSON-NEXT:     "line": 26,
+// JSON-NEXT:     "col": 6,
+// JSON-NEXT:     "tokLen": 3
+// JSON-NEXT:    },
+// JSON-NEXT:    "range": {
+// JSON-NEXT:     "begin": {
+// JSON-NEXT:      "offset": 940,
+// JSON-NEXT:      "col": 1,
+// JSON-NEXT:      "tokLen": 4
+// JSON-NEXT:     },
+// JSON-NEXT:     "end": {
+// JSON-NEXT:      "offset": 1055,
+// JSON-NEXT:      "line": 30,
+// JSON-NEXT:      "col": 1,
+// JSON-NEXT:      "tokLen": 1
+// JSON-NEXT:     }
+// JSON-NEXT:    },
+// JSON-NEXT:    "name": "baz",
+// JSON-NEXT:    "mangledName": "_Z3bazv",
+// JSON-NEXT:    "type": {
+// JSON-NEXT:     "qualType": "void ()"
+// JSON-NEXT:    },
+// JSON-NEXT:    "inner": [
+// JSON-NEXT:     {
+// JSON-NEXT:      "id": "0x{{.*}}",
+// JSON-NEXT:      "kind": "CompoundStmt",
+// JSON-NEXT:      "range": {
+// JSON-NEXT:       "begin": {
+// JSON-NEXT:        "offset": 951,
+// JSON-NEXT:        "line": 26,
+// JSON-NEXT:        "col": 12,
+// JSON-NEXT:        "tokLen": 1
+// JSON-NEXT:       },
+// JSON-NEXT:       "end": {
+// JSON-NEXT:        "offset": 1055,
+// JSON-NEXT:        "line": 30,
+// JSON-NEXT:        "col": 1,
+// JSON-NEXT:        "tokLen": 1
+// JSON-NEXT:       }
+// JSON-NEXT:      },
+// JSON-NEXT:      "inner": [
+// JSON-NEXT:       {
+// JSON-NEXT:        "id": "0x{{.*}}",
+// JSON-NEXT:        "kind": "DeclStmt",
+// JSON-NEXT:        "range": {
+// JSON-NEXT:         "begin": {
+// JSON-NEXT:          "offset": 955,
+// JSON-NEXT:          "line": 27,
+// JSON-NEXT:          "col": 3,
+// JSON-NEXT:          "tokLen": 3
+// JSON-NEXT:         },
+// JSON-NEXT:         "end": {
+// JSON-NEXT:          "offset": 976,
+// JSON-NEXT:          "col": 24,
+// JSON-NEXT:          "tokLen": 1
+// JSON-NEXT:         }
+// JSON-NEXT:        },
+// JSON-NEXT:        "inner": [
+// JSON-NEXT:         {
+// JSON-NEXT:          "id": "0x{{.*}}",
+// JSON-NEXT:          "kind": "VarDecl",
+// JSON-NEXT:          "loc": {
+// JSON-NEXT:           "offset": 959,
+// JSON-NEXT:           "col": 7,
+// JSON-NEXT:           "tokLen": 1
+// JSON-NEXT:          },
+// JSON-NEXT:          "range": {
+// JSON-NEXT:           "begin": {
+// JSON-NEXT:            "offset": 955,
+// JSON-NEXT:            "col": 3,
+// JSON-NEXT:            "tokLen": 3
+// JSON-NEXT:           },
+// JSON-NEXT:           "end": {
+// JSON-NEXT:            "offset": 975,
+// JSON-NEXT:            "col": 23,
+// JSON-NEXT:            "tokLen": 1
+// JSON-NEXT:           }
+// JSON-NEXT:          },
+// JSON-NEXT:          "name": "x",
+// JSON-NEXT:          "type": {
+// JSON-NEXT:           "qualType": "int"
+// JSON-NEXT:          },
+// JSON-NEXT:          "init": "c",
+// JSON-NEXT:          "inner": [
+// JSON-NEXT:           {
+// JSON-NEXT:            "id": "0x{{.*}}",
+// JSON-NEXT:            "kind": "CallExpr",
+// JSON-NEXT:            "range": {
+// JSON-NEXT:             "begin": {
+// JSON-NEXT:              "offset": 963,
+// JSON-NEXT:              "col": 11,
+// JSON-NEXT:              "tokLen": 3
+// JSON-NEXT:             },
+// JSON-NEXT:             "end": {
+// JSON-NEXT:              "offset": 975,
+// JSON-NEXT:              "col": 23,
+// JSON-NEXT:              "tokLen": 1
+// JSON-NEXT:             }
+// JSON-NEXT:            },
+// JSON-NEXT:            "type": {
+// JSON-NEXT:             "qualType": "int"
+// JSON-NEXT:            },
+// JSON-NEXT:            "valueCategory": "prvalue",
+// JSON-NEXT:            "inner": [
+// JSON-NEXT:             {
+// JSON-NEXT:              "id": "0x{{.*}}",
+// JSON-NEXT:              "kind": "ImplicitCastExpr",
+// JSON-NEXT:              "range": {
+// JSON-NEXT:               "begin": {
+// JSON-NEXT:                "offset": 963,
+// JSON-NEXT:                "col": 11,
+// JSON-NEXT:                "tokLen": 3
+// JSON-NEXT:               },
+// JSON-NEXT:               "end": {
+// JSON-NEXT:                "offset": 973,
+// JSON-NEXT:                "col": 21,
+// JSON-NEXT:                "tokLen": 1
+// JSON-NEXT:               }
+// JSON-NEXT:              },
+// JSON-NEXT:              "type": {
+// JSON-NEXT:               "qualType": "int (*)()"
+// JSON-NEXT:              },
+// JSON-NEXT:              "valueCategory": "prvalue",
+// JSON-NEXT:              "castKind": "FunctionToPointerDecay",
+// JSON-NEXT:              "inner": [
+// JSON-NEXT:               {
+// JSON-NEXT:                "id": "0x{{.*}}",
+// JSON-NEXT:                "kind": "DeclRefExpr",
+// JSON-NEXT:                "range": {
+// JSON-NEXT:                 "begin": {
+// JSON-NEXT:                  "offset": 963,
+// JSON-NEXT:                  "col": 11,
+// JSON-NEXT:                  "tokLen": 3
+// JSON-NEXT:                 },
+// JSON-NEXT:                 "end": {
+// JSON-NEXT:                  "offset": 973,
+// JSON-NEXT:                  "col": 21,
+// JSON-NEXT:                  "tokLen": 1
+// JSON-NEXT:                 }
+// JSON-NEXT:                },
+// JSON-NEXT:                "type": {
+// JSON-NEXT:                 "qualType": "int ()"
+// JSON-NEXT:                },
+// JSON-NEXT:                "valueCategory": "lvalue",
+// JSON-NEXT:                "referencedDecl": {
+// JSON-NEXT:                 "id": "0x{{.*}}",
+// JSON-NEXT:                 "kind": "FunctionDecl",
+// JSON-NEXT:                 "name": "bar",
+// JSON-NEXT:                 "type": {
+// JSON-NEXT:                  "qualType": "int ()"
+// JSON-NEXT:                 }
+// JSON-NEXT:                },
+// JSON-NEXT:                "foundReferencedDecl": {
+// JSON-NEXT:                 "id": "0x{{.*}}",
+// JSON-NEXT:                 "kind": "FunctionTemplateDecl",
+// JSON-NEXT:                 "name": "bar"
+// JSON-NEXT:                }
+// JSON-NEXT:               }
+// JSON-NEXT:              ]
+// JSON-NEXT:             }
+// JSON-NEXT:            ]
+// JSON-NEXT:           }
+// JSON-NEXT:          ]
+// JSON-NEXT:         }
+// JSON-NEXT:        ]
+// JSON-NEXT:       },
+// JSON-NEXT:       {
+// JSON-NEXT:        "id": "0x{{.*}}",
+// JSON-NEXT:        "kind": "DeclStmt",
+// JSON-NEXT:        "range": {
+// JSON-NEXT:         "begin": {
+// JSON-NEXT:          "offset": 980,
+// JSON-NEXT:          "line": 28,
+// JSON-NEXT:          "col": 3,
+// JSON-NEXT:          "tokLen": 3
+// JSON-NEXT:         },
+// JSON-NEXT:         "end": {
+// JSON-NEXT:          "offset": 1010,
+// JSON-NEXT:          "col": 33,
+// JSON-NEXT:          "tokLen": 1
+// JSON-NEXT:         }
+// JSON-NEXT:        },
+// JSON-NEXT:        "inner": [
+// JSON-NEXT:         {
+// JSON-NEXT:          "id": "0x{{.*}}",
+// JSON-NEXT:          "kind": "VarDecl",
+// JSON-NEXT:          "loc": {
+// JSON-NEXT:           "offset": 984,
+// JSON-NEXT:           "col": 7,
+// JSON-NEXT:           "tokLen": 1
+// JSON-NEXT:          },
+// JSON-NEXT:          "range": {
+// JSON-NEXT:           "begin": {
+// JSON-NEXT:            "offset": 980,
+// JSON-NEXT:            "col": 3,
+// JSON-NEXT:            "tokLen": 3
+// JSON-NEXT:           },
+// JSON-NEXT:           "end": {
+// JSON-NEXT:            "offset": 1009,
+// JSON-NEXT:            "col": 32,
+// JSON-NEXT:            "tokLen": 1
+// JSON-NEXT:           }
+// JSON-NEXT:          },
+// JSON-NEXT:          "name": "y",
+// JSON-NEXT:          "type": {
+// JSON-NEXT:           "qualType": "int"
+// JSON-NEXT:          },
+// JSON-NEXT:          "init": "c",
+// JSON-NEXT:          "inner": [
+// JSON-NEXT:           {
+// JSON-NEXT:            "id": "0x{{.*}}",
+// JSON-NEXT:            "kind": "ExprWithCleanups",
+// JSON-NEXT:            "range": {
+// JSON-NEXT:             "begin": {
+// JSON-NEXT:              "offset": 988,
+// JSON-NEXT:              "col": 11,
+// JSON-NEXT:              "tokLen": 3
+// JSON-NEXT:             },
+// JSON-NEXT:             "end": {
+// JSON-NEXT:              "offset": 1009,
+// JSON-NEXT:              "col": 32,
+// JSON-NEXT:              "tokLen": 1
+// JSON-NEXT:             }
+// JSON-NEXT:            },
+// JSON-NEXT:            "type": {
+// JSON-NEXT:             "qualType": "int"
+// JSON-NEXT:            },
+// JSON-NEXT:            "valueCategory": "prvalue",
+// JSON-NEXT:            "inner": [
+// JSON-NEXT:             {
+// JSON-NEXT:              "id": "0x{{.*}}",
+// JSON-NEXT:              "kind": "CXXMemberCallExpr",
+// JSON-NEXT:              "range": {
+// JSON-NEXT:               "begin": {
+// JSON-NEXT:                "offset": 988,
+// JSON-NEXT:                "col": 11,
+// JSON-NEXT:                "tokLen": 3
+// JSON-NEXT:               },
+// JSON-NEXT:               "end": {
+// JSON-NEXT:                "offset": 1009,
+// JSON-NEXT:                "col": 32,
+// JSON-NEXT:                "tokLen": 1
+// JSON-NEXT:               }
+// JSON-NEXT:              },
+// JSON-NEXT:              "type": {
+// JSON-NEXT:               "qualType": "int"
+// JSON-NEXT:              },
+// JSON-NEXT:              "valueCategory": "prvalue",
+// JSON-NEXT:              "inner": [
+// JSON-NEXT:               {
+// JSON-NEXT:                "id": "0x{{.*}}",
+// JSON-NEXT:                "kind": "MemberExpr",
+// JSON-NEXT:                "range": {
+// JSON-NEXT:                 "begin": {
+// JSON-NEXT:                  "offset": 988,
+// JSON-NEXT:                  "col": 11,
+// JSON-NEXT:                  "tokLen": 3
+// JSON-NEXT:                 },
+// JSON-NEXT:                 "end": {
+// JSON-NEXT:                  "offset": 1002,
+// JSON-NEXT:                  "col": 25,
+// JSON-NEXT:                  "tokLen": 6
+// JSON-NEXT:                 }
+// JSON-NEXT:                },
+// JSON-NEXT:                "type": {
+// JSON-NEXT:                 "qualType": "<bound member function type>"
+// JSON-NEXT:                },
+// JSON-NEXT:                "valueCategory": "prvalue",
+// JSON-NEXT:                "name": "getSum",
+// JSON-NEXT:                "isArrow": false,
+// JSON-NEXT:                "referencedMemberDecl": "0x{{.*}}",
+// JSON-NEXT:                "inner": [
+// JSON-NEXT:                 {
+// JSON-NEXT:                  "id": "0x{{.*}}",
+// JSON-NEXT:                  "kind": "MaterializeTemporaryExpr",
+// JSON-NEXT:                  "range": {
+// JSON-NEXT:                   "begin": {
+// JSON-NEXT:                    "offset": 988,
+// JSON-NEXT:                    "col": 11,
+// JSON-NEXT:                    "tokLen": 3
+// JSON-NEXT:                   },
+// JSON-NEXT:                   "end": {
+// JSON-NEXT:                    "offset": 1000,
+// JSON-NEXT:                    "col": 23,
+// JSON-NEXT:                    "tokLen": 1
+// JSON-NEXT:                   }
+// JSON-NEXT:                  },
+// JSON-NEXT:                  "type": {
+// JSON-NEXT:                   "qualType": "foo<5, int>"
+// JSON-NEXT:                  },
+// JSON-NEXT:                  "valueCategory": "xvalue",
+// JSON-NEXT:                  "storageDuration": "full expression",
+// JSON-NEXT:                  "inner": [
+// JSON-NEXT:                   {
+// JSON-NEXT:                    "id": "0x{{.*}}",
+// JSON-NEXT:                    "kind": "CXXTemporaryObjectExpr",
+// JSON-NEXT:                    "range": {
+// JSON-NEXT:                     "begin": {
+// JSON-NEXT:                      "offset": 988,
+// JSON-NEXT:                      "col": 11,
+// JSON-NEXT:                      "tokLen": 3
+// JSON-NEXT:                     },
+// JSON-NEXT:                     "end": {
+// JSON-NEXT:                      "offset": 1000,
+// JSON-NEXT:                      "col": 23,
+// JSON-NEXT:                      "tokLen": 1
+// JSON-NEXT:                     }
+// JSON-NEXT:                    },
+// JSON-NEXT:                    "type": {
+// JSON-NEXT:                     "qualType": "foo<5, int>"
+// JSON-NEXT:                    },
+// JSON-NEXT:                    "valueCategory": "prvalue",
+// JSON-NEXT:                    "ctorType": {
+// JSON-NEXT:                     "qualType": "void ()"
+// JSON-NEXT:                    },
+// JSON-NEXT:                    "hadMultipleCandidates": true,
+// JSON-NEXT:                    "constructionKind": "complete"
+// JSON-NEXT:                   }
+// JSON-NEXT:                  ]
+// JSON-NEXT:                 }
+// JSON-NEXT:                ]
+// JSON-NEXT:               }
+// JSON-NEXT:              ]
+// JSON-NEXT:             }
+// JSON-NEXT:            ]
+// JSON-NEXT:           }
+// JSON-NEXT:          ]
+// JSON-NEXT:         }
+// JSON-NEXT:        ]
+// JSON-NEXT:       },
+// JSON-NEXT:       {
+// JSON-NEXT:        "id": "0x{{.*}}",
+// JSON-NEXT:        "kind": "DeclStmt",
+// JSON-NEXT:        "range": {
+// JSON-NEXT:         "begin": {
+// JSON-NEXT:          "offset": 1014,
+// JSON-NEXT:          "line": 29,
+// JSON-NEXT:          "col": 3,
+// JSON-NEXT:          "tokLen": 6
+// JSON-NEXT:         },
+// JSON-NEXT:         "end": {
+// JSON-NEXT:          "offset": 1053,
+// JSON-NEXT:          "col": 42,
+// JSON-NEXT:          "tokLen": 1
+// JSON-NEXT:         }
+// JSON-NEXT:        },
+// JSON-NEXT:        "inner": [
+// JSON-NEXT:         {
+// JSON-NEXT:          "id": "0x{{.*}}",
+// JSON-NEXT:          "kind": "VarDecl",
+// JSON-NEXT:          "loc": {
+// JSON-NEXT:           "offset": 1021,
+// JSON-NEXT:           "col": 10,
+// JSON-NEXT:           "tokLen": 1
+// JSON-NEXT:          },
+// JSON-NEXT:          "range": {
+// JSON-NEXT:           "begin": {
+// JSON-NEXT:            "offset": 1014,
+// JSON-NEXT:            "col": 3,
+// JSON-NEXT:            "tokLen": 6
+// JSON-NEXT:           },
+// JSON-NEXT:           "end": {
+// JSON-NEXT:            "offset": 1052,
+// JSON-NEXT:            "col": 41,
+// JSON-NEXT:            "tokLen": 1
+// JSON-NEXT:           }
+// JSON-NEXT:          },
+// JSON-NEXT:          "name": "z",
+// JSON-NEXT:          "type": {
+// JSON-NEXT:           "qualType": "double"
+// JSON-NEXT:          },
+// JSON-NEXT:          "init": "c",
+// JSON-NEXT:          "inner": [
+// JSON-NEXT:           {
+// JSON-NEXT:            "id": "0x{{.*}}",
+// JSON-NEXT:            "kind": "ExprWithCleanups",
+// JSON-NEXT:            "range": {
+// JSON-NEXT:             "begin": {
+// JSON-NEXT:              "offset": 1025,
+// JSON-NEXT:              "col": 14,
+// JSON-NEXT:              "tokLen": 3
+// JSON-NEXT:             },
+// JSON-NEXT:             "end": {
+// JSON-NEXT:              "offset": 1052,
+// JSON-NEXT:              "col": 41,
+// JSON-NEXT:              "tokLen": 1
+// JSON-NEXT:             }
+// JSON-NEXT:            },
+// JSON-NEXT:            "type": {
+// JSON-NEXT:             "qualType": "double"
+// JSON-NEXT:            },
+// JSON-NEXT:            "valueCategory": "prvalue",
+// JSON-NEXT:            "inner": [
+// JSON-NEXT:             {
+// JSON-NEXT:              "id": "0x{{.*}}",
+// JSON-NEXT:              "kind": "CXXMemberCallExpr",
+// JSON-NEXT:              "range": {
+// JSON-NEXT:               "begin": {
+// JSON-NEXT:                "offset": 1025,
+// JSON-NEXT:                "col": 14,
+// JSON-NEXT:                "tokLen": 3
+// JSON-NEXT:               },
+// JSON-NEXT:               "end": {
+// JSON-NEXT:                "offset": 1052,
+// JSON-NEXT:                "col": 41,
+// JSON-NEXT:                "tokLen": 1
+// JSON-NEXT:               }
+// JSON-NEXT:              },
+// JSON-NEXT:              "type": {
+// JSON-NEXT:               "qualType": "double"
+// JSON-NEXT:              },
+// JSON-NEXT:              "valueCategory": "prvalue",
+// JSON-NEXT:              "inner": [
+// JSON-NEXT:               {
+// JSON-NEXT:                "id": "0x{{.*}}",
+// JSON-NEXT:                "kind": "MemberExpr",
+// JSON-NEXT:                "range": {
+// JSON-NEXT:                 "begin": {
+// JSON-NEXT:                  "offset": 1025,
+// JSON-NEXT:                  "col": 14,
+// JSON-NEXT:                  "tokLen": 3
+// JSON-NEXT:                 },
+// JSON-NEXT:                 "end": {
+// JSON-NEXT:                  "offset": 1045,
+// JSON-NEXT:                  "col": 34,
+// JSON-NEXT:                  "tokLen": 6
+// JSON-NEXT:                 }
+// JSON-NEXT:                },
+// JSON-NEXT:                "type": {
+// JSON-NEXT:                 "qualType": "<bound member function type>"
+// JSON-NEXT:                },
+// JSON-NEXT:                "valueCategory": "prvalue",
+// JSON-NEXT:                "name": "getSum",
+// JSON-NEXT:                "isArrow": false,
+// JSON-NEXT:                "referencedMemberDecl": "0x{{.*}}",
+// JSON-NEXT:                "inner": [
+// JSON-NEXT:                 {
+// JSON-NEXT:                  "id": "0x{{.*}}",
+// JSON-NEXT:                  "kind": "MaterializeTemporaryExpr",
+// JSON-NEXT:                  "range": {
+// JSON-NEXT:                   "begin": {
+// JSON-NEXT:                    "offset": 1025,
+// JSON-NEXT:                    "col": 14,
+// JSON-NEXT:                    "tokLen": 3
+// JSON-NEXT:                   },
+// JSON-NEXT:                   "end": {
+// JSON-NEXT:                    "offset": 1043,
+// JSON-NEXT:                    "col": 32,
+// JSON-NEXT:                    "tokLen": 1
+// JSON-NEXT:                   }
+// JSON-NEXT:                  },
+// JSON-NEXT:                  "type": {
+// JSON-NEXT:                   "qualType": "foo<2, double, 3>"
+// JSON-NEXT:                  },
+// JSON-NEXT:                  "valueCategory": "xvalue",
+// JSON-NEXT:                  "storageDuration": "full expression",
+// JSON-NEXT:                  "inner": [
+// JSON-NEXT:                   {
+// JSON-NEXT:                    "id": "0x{{.*}}",
+// JSON-NEXT:                    "kind": "CXXTemporaryObjectExpr",
+// JSON-NEXT:                    "range": {
+// JSON-NEXT:                     "begin": {
+// JSON-NEXT:                      "offset": 1025,
+// JSON-NEXT:                      "col": 14,
+// JSON-NEXT:                      "tokLen": 3
+// JSON-NEXT:                     },
+// JSON-NEXT:                     "end": {
+// JSON-NEXT:                      "offset": 1043,
+// JSON-NEXT:                      "col": 32,
+// JSON-NEXT:                      "tokLen": 1
+// JSON-NEXT:                     }
+// JSON-NEXT:                    },
+// JSON-NEXT:                    "type": {
+// JSON-NEXT:                     "qualType": "foo<2, double, 3>"
+// JSON-NEXT:                    },
+// JSON-NEXT:                    "valueCategory": "prvalue",
+// JSON-NEXT:                    "ctorType": {
+// JSON-NEXT:                     "qualType": "void ()"
+// JSON-NEXT:                    },
+// JSON-NEXT:                    "hadMultipleCandidates": true,
+// JSON-NEXT:                    "constructionKind": "complete"
+// JSON-NEXT:                   }
+// JSON-NEXT:                  ]
+// JSON-NEXT:                 }
+// JSON-NEXT:                ]
+// JSON-NEXT:               }
+// JSON-NEXT:              ]
+// JSON-NEXT:             }
+// JSON-NEXT:            ]
+// JSON-NEXT:           }
+// JSON-NEXT:          ]
+// JSON-NEXT:         }
+// JSON-NEXT:        ]
+// JSON-NEXT:       }
+// JSON-NEXT:      ]
+// JSON-NEXT:     }
+// JSON-NEXT:    ]
+// JSON-NEXT:   },
+// JSON-NEXT:   {
+// JSON-NEXT:    "id": "0x{{.*}}",
+// JSON-NEXT:    "kind": "ClassTemplateDecl",
+// JSON-NEXT:    "loc": {
+// JSON-NEXT:     "offset": 1856,
+// JSON-NEXT:     "line": 52,
+// JSON-NEXT:     "col": 33,
+// JSON-NEXT:     "tokLen": 1
+// JSON-NEXT:    },
+// JSON-NEXT:    "range": {
+// JSON-NEXT:     "begin": {
+// JSON-NEXT:      "offset": 1824,
+// JSON-NEXT:      "col": 1,
+// JSON-NEXT:      "tokLen": 8
+// JSON-NEXT:     },
+// JSON-NEXT:     "end": {
+// JSON-NEXT:      "offset": 1896,
+// JSON-NEXT:      "line": 54,
+// JSON-NEXT:      "col": 1,
+// JSON-NEXT:      "tokLen": 1
+// JSON-NEXT:     }
+// JSON-NEXT:    },
+// JSON-NEXT:    "name": "A",
+// JSON-NEXT:    "inner": [
+// JSON-NEXT:     {
+// JSON-NEXT:      "id": "0x{{.*}}",
+// JSON-NEXT:      "kind": "TemplateTypeParmDecl",
+// JSON-NEXT:      "loc": {
+// JSON-NEXT:       "offset": 1846,
+// JSON-NEXT:       "line": 52,
+// JSON-NEXT:       "col": 23,
+// JSON-NEXT:       "tokLen": 1
+// JSON-NEXT:      },
+// JSON-NEXT:      "range": {
+// JSON-NEXT:       "begin": {
+// JSON-NEXT:        "offset": 1834,
+// JSON-NEXT:        "col": 11,
+// JSON-NEXT:        "tokLen": 8
+// JSON-NEXT:       },
+// JSON-NEXT:       "end": {
+// JSON-NEXT:        "offset": 1846,
+// JSON-NEXT:        "col": 23,
+// JSON-NEXT:        "tokLen": 1
+// JSON-NEXT:       }
+// JSON-NEXT:      },
+// JSON-NEXT:      "isReferenced": true,
+// JSON-NEXT:      "name": "T",
+// JSON-NEXT:      "tagUsed": "typename",
+// JSON-NEXT:      "depth": 0,
+// JSON-NEXT:      "index": 0,
+// JSON-NEXT:      "isParameterPack": true
+// JSON-NEXT:     },
+// JSON-NEXT:     {
+// JSON-NEXT:      "id": "0x{{.*}}",
+// JSON-NEXT:      "kind": "CXXRecordDecl",
+// JSON-NEXT:      "loc": {
+// JSON-NEXT:       "offset": 1856,
+// JSON-NEXT:       "col": 33,
+// JSON-NEXT:       "tokLen": 1
+// JSON-NEXT:      },
+// JSON-NEXT:      "range": {
+// JSON-NEXT:       "begin": {
+// JSON-NEXT:        "offset": 1849,
+// JSON-NEXT:        "col": 26,
+// JSON-NEXT:        "tokLen": 6
+// JSON-NEXT:       },
+// JSON-NEXT:       "end": {
+// JSON-NEXT:        "offset": 1896,
+// JSON-NEXT:        "line": 54,
+// JSON-NEXT:        "col": 1,
+// JSON-NEXT:        "tokLen": 1
+// JSON-NEXT:       }
+// JSON-NEXT:      },
+// JSON-NEXT:      "name": "A",
+// JSON-NEXT:      "tagUsed": "struct",
+// JSON-NEXT:      "completeDefinition": true,
+// JSON-NEXT:      "definitionData": {
+// JSON-NEXT:       "canConstDefaultInit": true,
+// JSON-NEXT:       "copyAssign": {
+// JSON-NEXT:        "hasConstParam": true,
+// JSON-NEXT:        "implicitHasConstParam": true,
+// JSON-NEXT:        "needsImplicit": true,
+// JSON-NEXT:        "simple": true,
+// JSON-NEXT:        "trivial": true
+// JSON-NEXT:       },
+// JSON-NEXT:       "copyCtor": {
+// JSON-NEXT:        "hasConstParam": true,
+// JSON-NEXT:        "implicitHasConstParam": true,
+// JSON-NEXT:        "needsImplicit": true,
+// JSON-NEXT:        "simple": true,
+// JSON-NEXT:        "trivial": true
+// JSON-NEXT:       },
+// JSON-NEXT:       "defaultCtor": {
+// JSON-NEXT:        "defaultedIsConstexpr": true,
+// JSON-NEXT:        "exists": true,
+// JSON-NEXT:        "isConstexpr": true,
+// JSON-NEXT:        "needsImplicit": true,
+// JSON-NEXT:        "trivial": true
+// JSON-NEXT:       },
+// JSON-NEXT:       "dtor": {
+// JSON-NEXT:        "irrelevant": true,
+// JSON-NEXT:        "needsImplicit": true,
+// JSON-NEXT:        "simple": true,
+// JSON-NEXT:        "trivial": true
+// JSON-NEXT:       },
+// JSON-NEXT:       "hasConstexprNonCopyMoveConstructor": true,
+// JSON-NEXT:       "isAggregate": true,
+// JSON-NEXT:       "isEmpty": true,
+// JSON-NEXT:       "isLiteral": true,
+// JSON-NEXT:       "isPOD": true,
+// JSON-NEXT:       "isStandardLayout": true,
+// JSON-NEXT:       "isTrivial": true,
+// JSON-NEXT:       "isTriviallyCopyable": true,
+// JSON-NEXT:       "moveAssign": {
+// JSON-NEXT:        "exists": true,
+// JSON-NEXT:        "needsImplicit": true,
+// JSON-NEXT:        "simple": true,
+// JSON-NEXT:        "trivial": true
+// JSON-NEXT:       },
+// JSON-NEXT:       "moveCtor": {
+// JSON-NEXT:        "exists": true,
+// JSON-NEXT:        "needsImplicit": true,
+// JSON-NEXT:        "simple": true,
+// JSON-NEXT:        "trivial": true
+// JSON-NEXT:       }
+// JSON-NEXT:      },
+// JSON-NEXT:      "inner": [
+// JSON-NEXT:       {
+// JSON-NEXT:        "id": "0x{{.*}}",
+// JSON-NEXT:        "kind": "CXXRecordDecl",
+// JSON-NEXT:        "loc": {
+// JSON-NEXT:         "offset": 1856,
+// JSON-NEXT:         "line": 52,
+// JSON-NEXT:         "col": 33,
+// JSON-NEXT:         "tokLen": 1
+// JSON-NEXT:        },
+// JSON-NEXT:        "range": {
+// JSON-NEXT:         "begin": {
+// JSON-NEXT:          "offset": 1849,
+// JSON-NEXT:          "col": 26,
+// JSON-NEXT:          "tokLen": 6
+// JSON-NEXT:         },
+// JSON-NEXT:         "end": {
+// JSON-NEXT:          "offset": 1856,
+// JSON-NEXT:          "col": 33,
+// JSON-NEXT:          "tokLen": 1
+// JSON-NEXT:         }
+// JSON-NEXT:        },
+// JSON-NEXT:        "isImplicit": true,
+// JSON-NEXT:        "name": "A",
+// JSON-NEXT:        "tagUsed": "struct"
+// JSON-NEXT:       },
+// JSON-NEXT:       {
+// JSON-NEXT:        "id": "0x{{.*}}",
+// JSON-NEXT:        "kind": "ClassTemplateDecl",
+// JSON-NEXT:        "loc": {
+// JSON-NEXT:         "offset": 1890,
+// JSON-NEXT:         "line": 53,
+// JSON-NEXT:         "col": 31,
+// JSON-NEXT:         "tokLen": 1
+// JSON-NEXT:        },
+// JSON-NEXT:        "range": {
+// JSON-NEXT:         "begin": {
+// JSON-NEXT:          "offset": 1862,
+// JSON-NEXT:          "col": 3,
+// JSON-NEXT:          "tokLen": 8
+// JSON-NEXT:         },
+// JSON-NEXT:         "end": {
+// JSON-NEXT:          "offset": 1893,
+// JSON-NEXT:          "col": 34,
+// JSON-NEXT:          "tokLen": 1
+// JSON-NEXT:         }
+// JSON-NEXT:        },
+// JSON-NEXT:        "name": "B",
+// JSON-NEXT:        "inner": [
+// JSON-NEXT:         {
+// JSON-NEXT:          "id": "0x{{.*}}",
+// JSON-NEXT:          "kind": "NonTypeTemplateParmDecl",
+// JSON-NEXT:          "loc": {
+// JSON-NEXT:           "offset": 1877,
+// JSON-NEXT:           "col": 18,
+// JSON-NEXT:           "tokLen": 1
+// JSON-NEXT:          },
+// JSON-NEXT:          "range": {
+// JSON-NEXT:           "begin": {
+// JSON-NEXT:            "offset": 1872,
+// JSON-NEXT:            "col": 13,
+// JSON-NEXT:            "tokLen": 1
+// JSON-NEXT:           },
+// JSON-NEXT:           "end": {
+// JSON-NEXT:            "offset": 1880,
+// JSON-NEXT:            "col": 21,
+// JSON-NEXT:            "tokLen": 1
+// JSON-NEXT:           }
+// JSON-NEXT:          },
+// JSON-NEXT:          "name": "x",
+// JSON-NEXT:          "type": {
+// JSON-NEXT:           "qualType": "T[3]..."
+// JSON-NEXT:          },
+// JSON-NEXT:          "depth": 1,
+// JSON-NEXT:          "index": 0,
+// JSON-NEXT:          "isParameterPack": true
+// JSON-NEXT:         },
+// JSON-NEXT:         {
+// JSON-NEXT:          "id": "0x{{.*}}",
+// JSON-NEXT:          "kind": "CXXRecordDecl",
+// JSON-NEXT:          "loc": {
+// JSON-NEXT:           "offset": 1890,
+// JSON-NEXT:           "col": 31,
+// JSON-NEXT:           "tokLen": 1
+// JSON-NEXT:          },
+// JSON-NEXT:          "range": {
+// JSON-NEXT:           "begin": {
+// JSON-NEXT:            "offset": 1883,
+// JSON-NEXT:            "col": 24,
+// JSON-NEXT:            "tokLen": 6
+// JSON-NEXT:           },
+// JSON-NEXT:           "end": {
+// JSON-NEXT:            "offset": 1893,
+// JSON-NEXT:            "col": 34,
+// JSON-NEXT:            "tokLen": 1
+// JSON-NEXT:           }
+// JSON-NEXT:          },
+// JSON-NEXT:          "name": "B",
+// JSON-NEXT:          "tagUsed": "struct",
+// JSON-NEXT:          "completeDefinition": true,
+// JSON-NEXT:          "definitionData": {
+// JSON-NEXT:           "canConstDefaultInit": true,
+// JSON-NEXT:           "copyAssign": {
+// JSON-NEXT:            "hasConstParam": true,
+// JSON-NEXT:            "implicitHasConstParam": true,
+// JSON-NEXT:            "needsImplicit": true,
+// JSON-NEXT:            "simple": true,
+// JSON-NEXT:            "trivial": true
+// JSON-NEXT:           },
+// JSON-NEXT:           "copyCtor": {
+// JSON-NEXT:            "hasConstParam": true,
+// JSON-NEXT:            "implicitHasConstParam": true,
+// JSON-NEXT:            "needsImplicit": true,
+// JSON-NEXT:            "simple": true,
+// JSON-NEXT:            "trivial": true
+// JSON-NEXT:           },
+// JSON-NEXT:           "defaultCtor": {
+// JSON-NEXT:            "defaultedIsConstexpr": true,
+// JSON-NEXT:            "exists": true,
+// JSON-NEXT:            "isConstexpr": true,
+// JSON-NEXT:            "needsImplicit": true,
+// JSON-NEXT:            "trivial": true
+// JSON-NEXT:           },
+// JSON-NEXT:           "dtor": {
+// JSON-NEXT:            "irrelevant": true,
+// JSON-NEXT:            "needsImplicit": true,
+// JSON-NEXT:            "simple": true,
+// JSON-NEXT:            "trivial": true
+// JSON-NEXT:           },
+// JSON-NEXT:           "hasConstexprNonCopyMoveConstructor": true,
+// JSON-NEXT:           "isAggregate": true,
+// JSON-NEXT:           "isEmpty": true,
+// JSON-NEXT:           "isLiteral": true,
+// JSON-NEXT:           "isPOD": true,
+// JSON-NEXT:           "isStandardLayout": true,
+// JSON-NEXT:           "isTrivial": true,
+// JSON-NEXT:           "isTriviallyCopyable": true,
+// JSON-NEXT:           "moveAssign": {
+// JSON-NEXT:            "exists": true,
+// JSON-NEXT:            "needsImplicit": true,
+// JSON-NEXT:            "simple": true,
+// JSON-NEXT:            "trivial": true
+// JSON-NEXT:           },
+// JSON-NEXT:           "moveCtor": {
+// JSON-NEXT:            "exists": true,
+// JSON-NEXT:            "needsImplicit": true,
+// JSON-NEXT:            "simple": true,
+// JSON-NEXT:            "trivial": true
+// JSON-NEXT:           }
+// JSON-NEXT:          },
+// JSON-NEXT:          "inner": [
+// JSON-NEXT:           {
+// JSON-NEXT:            "id": "0x{{.*}}",
+// JSON-NEXT:            "kind": "CXXRecordDecl",
+// JSON-NEXT:            "loc": {
+// JSON-NEXT:             "offset": 1890,
+// JSON-NEXT:             "col": 31,
+// JSON-NEXT:             "tokLen": 1
+// JSON-NEXT:            },
+// JSON-NEXT:            "range": {
+// JSON-NEXT:             "begin": {
+// JSON-NEXT:              "offset": 1883,
+// JSON-NEXT:              "col": 24,
+// JSON-NEXT:              "tokLen": 6
+// JSON-NEXT:             },
+// JSON-NEXT:             "end": {
+// JSON-NEXT:              "offset": 1890,
+// JSON-NEXT:              "col": 31,
+// JSON-NEXT:              "tokLen": 1
+// JSON-NEXT:             }
+// JSON-NEXT:            },
+// JSON-NEXT:            "isImplicit": true,
+// JSON-NEXT:            "name": "B",
+// JSON-NEXT:            "tagUsed": "struct"
+// JSON-NEXT:           }
+// JSON-NEXT:          ]
+// JSON-NEXT:         }
+// JSON-NEXT:        ]
+// JSON-NEXT:       }
+// JSON-NEXT:      ]
+// JSON-NEXT:     }
+// JSON-NEXT:    ]
+// JSON-NEXT:   },
+// JSON-NEXT:   {
+// JSON-NEXT:    "id": "0x{{.*}}",
+// JSON-NEXT:    "kind": "FunctionTemplateDecl",
+// JSON-NEXT:    "loc": {
+// JSON-NEXT:     "offset": 2016,
+// JSON-NEXT:     "line": 58,
+// JSON-NEXT:     "col": 31,
+// JSON-NEXT:     "tokLen": 1
+// JSON-NEXT:    },
+// JSON-NEXT:    "range": {
+// JSON-NEXT:     "begin": {
+// JSON-NEXT:      "offset": 1986,
+// JSON-NEXT:      "col": 1,
+// JSON-NEXT:      "tokLen": 8
+// JSON-NEXT:     },
+// JSON-NEXT:     "end": {
+// JSON-NEXT:      "offset": 2038,
+// JSON-NEXT:      "line": 60,
+// JSON-NEXT:      "col": 1,
+// JSON-NEXT:      "tokLen": 1
+// JSON-NEXT:     }
+// JSON-NEXT:    },
+// JSON-NEXT:    "name": "f",
+// JSON-NEXT:    "inner": [
+// JSON-NEXT:     {
+// JSON-NEXT:      "id": "0x{{.*}}",
+// JSON-NEXT:      "kind": "TemplateTypeParmDecl",
+// JSON-NEXT:      "loc": {
+// JSON-NEXT:       "offset": 2008,
+// JSON-NEXT:       "line": 58,
+// JSON-NEXT:       "col": 23,
+// JSON-NEXT:       "tokLen": 1
+// JSON-NEXT:      },
+// JSON-NEXT:      "range": {
+// JSON-NEXT:       "begin": {
+// JSON-NEXT:        "offset": 1996,
+// JSON-NEXT:        "col": 11,
+// JSON-NEXT:        "tokLen": 8
+// JSON-NEXT:       },
+// JSON-NEXT:       "end": {
+// JSON-NEXT:        "offset": 2008,
+// JSON-NEXT:        "col": 23,
+// JSON-NEXT:        "tokLen": 1
+// JSON-NEXT:       }
+// JSON-NEXT:      },
+// JSON-NEXT:      "isReferenced": true,
+// JSON-NEXT:      "name": "T",
+// JSON-NEXT:      "tagUsed": "typename",
+// JSON-NEXT:      "depth": 0,
+// JSON-NEXT:      "index": 0,
+// JSON-NEXT:      "isParameterPack": true
+// JSON-NEXT:     },
+// JSON-NEXT:     {
+// JSON-NEXT:      "id": "0x{{.*}}",
+// JSON-NEXT:      "kind": "FunctionDecl",
+// JSON-NEXT:      "loc": {
+// JSON-NEXT:       "offset": 2016,
+// JSON-NEXT:       "col": 31,
+// JSON-NEXT:       "tokLen": 1
+// JSON-NEXT:      },
+// JSON-NEXT:      "range": {
+// JSON-NEXT:       "begin": {
+// JSON-NEXT:        "offset": 2011,
+// JSON-NEXT:        "col": 26,
+// JSON-NEXT:        "tokLen": 4
+// JSON-NEXT:       },
+// JSON-NEXT:       "end": {
+// JSON-NEXT:        "offset": 2038,
+// JSON-NEXT:        "line": 60,
+// JSON-NEXT:        "col": 1,
+// JSON-NEXT:        "tokLen": 1
+// JSON-NEXT:       }
+// JSON-NEXT:      },
+// JSON-NEXT:      "name": "f",
+// JSON-NEXT:      "type": {
+// JSON-NEXT:       "qualType": "void ()"
+// JSON-NEXT:      },
+// JSON-NEXT:      "inner": [
+// JSON-NEXT:       {
+// JSON-NEXT:        "id": "0x{{.*}}",
+// JSON-NEXT:        "kind": "CompoundStmt",
+// JSON-NEXT:        "range": {
+// JSON-NEXT:         "begin": {
+// JSON-NEXT:          "offset": 2020,
+// JSON-NEXT:          "line": 58,
+// JSON-NEXT:          "col": 35,
+// JSON-NEXT:          "tokLen": 1
+// JSON-NEXT:         },
+// JSON-NEXT:         "end": {
+// JSON-NEXT:          "offset": 2038,
+// JSON-NEXT:          "line": 60,
+// JSON-NEXT:          "col": 1,
+// JSON-NEXT:          "tokLen": 1
+// JSON-NEXT:         }
+// JSON-NEXT:        },
+// JSON-NEXT:        "inner": [
+// JSON-NEXT:         {
+// JSON-NEXT:          "id": "0x{{.*}}",
+// JSON-NEXT:          "kind": "DeclStmt",
+// JSON-NEXT:          "range": {
+// JSON-NEXT:           "begin": {
+// JSON-NEXT:            "offset": 2024,
+// JSON-NEXT:            "line": 59,
+// JSON-NEXT:            "col": 3,
+// JSON-NEXT:            "tokLen": 1
+// JSON-NEXT:           },
+// JSON-NEXT:           "end": {
+// JSON-NEXT:            "offset": 2036,
+// JSON-NEXT:            "col": 15,
+// JSON-NEXT:            "tokLen": 1
+// JSON-NEXT:           }
+// JSON-NEXT:          },
+// JSON-NEXT:          "inner": [
+// JSON-NEXT:           {
+// JSON-NEXT:            "id": "0x{{.*}}",
+// JSON-NEXT:            "kind": "VarDecl",
+// JSON-NEXT:            "loc": {
+// JSON-NEXT:             "offset": 2035,
+// JSON-NEXT:             "col": 14,
+// JSON-NEXT:             "tokLen": 1
+// JSON-NEXT:            },
+// JSON-NEXT:            "range": {
+// JSON-NEXT:             "begin": {
+// JSON-NEXT:              "offset": 2024,
+// JSON-NEXT:              "col": 3,
+// JSON-NEXT:              "tokLen": 1
+// JSON-NEXT:             },
+// JSON-NEXT:             "end": {
+// JSON-NEXT:              "offset": 2035,
+// JSON-NEXT:              "col": 14,
+// JSON-NEXT:              "tokLen": 1
+// JSON-NEXT:             }
+// JSON-NEXT:            },
+// JSON-NEXT:            "name": "a",
+// JSON-NEXT:            "type": {
+// JSON-NEXT:             "qualType": "A<T[3]...>"
+// JSON-NEXT:            }
+// JSON-NEXT:           }
+// JSON-NEXT:          ]
+// JSON-NEXT:         }
+// JSON-NEXT:        ]
+// JSON-NEXT:       }
+// JSON-NEXT:      ]
+// JSON-NEXT:     }
+// JSON-NEXT:    ]
+// JSON-NEXT:   },
+// JSON-NEXT:   {
+// JSON-NEXT:    "id": "0x{{.*}}",
+// JSON-NEXT:    "kind": "NamespaceDecl",
+// JSON-NEXT:    "loc": {
+// JSON-NEXT:     "offset": 2051,
+// JSON-NEXT:     "line": 62,
+// JSON-NEXT:     "col": 11,
+// JSON-NEXT:     "tokLen": 5
+// JSON-NEXT:    },
+// JSON-NEXT:    "range": {
+// JSON-NEXT:     "begin": {
+// JSON-NEXT:      "offset": 2041,
+// JSON-NEXT:      "col": 1,
+// JSON-NEXT:      "tokLen": 9
+// JSON-NEXT:     },
+// JSON-NEXT:     "end": {
+// JSON-NEXT:      "offset": 2240,
+// JSON-NEXT:      "line": 71,
+// JSON-NEXT:      "col": 1,
+// JSON-NEXT:      "tokLen": 1
+// JSON-NEXT:     }
+// JSON-NEXT:    },
+// JSON-NEXT:    "name": "test2",
+// JSON-NEXT:    "inner": [
+// JSON-NEXT:     {
+// JSON-NEXT:      "id": "0x{{.*}}",
+// JSON-NEXT:      "kind": "FunctionDecl",
+// JSON-NEXT:      "loc": {
+// JSON-NEXT:       "offset": 2064,
+// JSON-NEXT:       "line": 63,
+// JSON-NEXT:       "col": 6,
+// JSON-NEXT:       "tokLen": 4
+// JSON-NEXT:      },
+// JSON-NEXT:      "range": {
+// JSON-NEXT:       "begin": {
+// JSON-NEXT:        "offset": 2059,
+// JSON-NEXT:        "col": 1,
+// JSON-NEXT:        "tokLen": 4
+// JSON-NEXT:       },
+// JSON-NEXT:       "end": {
+// JSON-NEXT:        "offset": 2072,
+// JSON-NEXT:        "col": 14,
+// JSON-NEXT:        "tokLen": 1
+// JSON-NEXT:       }
+// JSON-NEXT:      },
+// JSON-NEXT:      "name": "func",
+// JSON-NEXT:      "mangledName": "_ZN5test24funcEi",
+// JSON-NEXT:      "type": {
+// JSON-NEXT:       "qualType": "void (int)"
+// JSON-NEXT:      },
+// JSON-NEXT:      "inner": [
+// JSON-NEXT:       {
+// JSON-NEXT:        "id": "0x{{.*}}",
+// JSON-NEXT:        "kind": "ParmVarDecl",
+// JSON-NEXT:        "loc": {
+// JSON-NEXT:         "offset": 2072,
+// JSON-NEXT:         "col": 14,
+// JSON-NEXT:         "tokLen": 1
+// JSON-NEXT:        },
+// JSON-NEXT:        "range": {
+// JSON-NEXT:         "begin": {
+// JSON-NEXT:          "offset": 2069,
+// JSON-NEXT:          "col": 11,
+// JSON-NEXT:          "tokLen": 3
+// JSON-NEXT:         },
+// JSON-NEXT:         "end": {
+// JSON-NEXT:          "offset": 2069,
+// JSON-NEXT:          "col": 11,
+// JSON-NEXT:          "tokLen": 3
+// JSON-NEXT:         }
+// JSON-NEXT:        },
+// JSON-NEXT:        "type": {
+// JSON-NEXT:         "qualType": "int"
+// JSON-NEXT:        }
+// JSON-NEXT:       }
+// JSON-NEXT:      ]
+// JSON-NEXT:     },
+// JSON-NEXT:     {
+// JSON-NEXT:      "id": "0x{{.*}}",
+// JSON-NEXT:      "kind": "FunctionDecl",
+// JSON-NEXT:      "loc": {
+// JSON-NEXT:       "offset": 2080,
+// JSON-NEXT:       "line": 64,
+// JSON-NEXT:       "col": 6,
+// JSON-NEXT:       "tokLen": 4
+// JSON-NEXT:      },
+// JSON-NEXT:      "range": {
+// JSON-NEXT:       "begin": {
+// JSON-NEXT:        "offset": 2075,
+// JSON-NEXT:        "col": 1,
+// JSON-NEXT:        "tokLen": 4
+// JSON-NEXT:       },
+// JSON-NEXT:       "end": {
+// JSON-NEXT:        "offset": 2090,
+// JSON-NEXT:        "col": 16,
+// JSON-NEXT:        "tokLen": 1
+// JSON-NEXT:       }
+// JSON-NEXT:      },
+// JSON-NEXT:      "name": "func",
+// JSON-NEXT:      "mangledName": "_ZN5test24funcEf",
+// JSON-NEXT:      "type": {
+// JSON-NEXT:       "qualType": "void (float)"
+// JSON-NEXT:      },
+// JSON-NEXT:      "inner": [
+// JSON-NEXT:       {
+// JSON-NEXT:        "id": "0x{{.*}}",
+// JSON-NEXT:        "kind": "ParmVarDecl",
+// JSON-NEXT:        "loc": {
+// JSON-NEXT:         "offset": 2090,
+// JSON-NEXT:         "col": 16,
+// JSON-NEXT:         "tokLen": 1
+// JSON-NEXT:        },
+// JSON-NEXT:        "range": {
+// JSON-NEXT:         "begin": {
+// JSON-NEXT:          "offset": 2085,
+// JSON-NEXT:          "col": 11,
+// JSON-NEXT:          "tokLen": 5
+// JSON-NEXT:         },
+// JSON-NEXT:         "end": {
+// JSON-NEXT:          "offset": 2085,
+// JSON-NEXT:          "col": 11,
+// JSON-NEXT:          "tokLen": 5
+// JSON-NEXT:         }
+// JSON-NEXT:        },
+// JSON-NEXT:        "type": {
+// JSON-NEXT:         "qualType": "float"
+// JSON-NEXT:        }
+// JSON-NEXT:       }
+// JSON-NEXT:      ]
+// JSON-NEXT:     },
+// JSON-NEXT:     {
+// JSON-NEXT:      "id": "0x{{.*}}",
+// JSON-NEXT:      "kind": "FunctionTemplateDecl",
+// JSON-NEXT:      "loc": {
+// JSON-NEXT:       "offset": 2119,
+// JSON-NEXT:       "line": 66,
+// JSON-NEXT:       "col": 6,
+// JSON-NEXT:       "tokLen": 4
+// JSON-NEXT:      },
+// JSON-NEXT:      "range": {
+// JSON-NEXT:       "begin": {
+// JSON-NEXT:        "offset": 2093,
+// JSON-NEXT:        "line": 65,
+// JSON-NEXT:        "col": 1,
+// JSON-NEXT:        "tokLen": 8
+// JSON-NEXT:       },
+// JSON-NEXT:       "end": {
+// JSON-NEXT:        "offset": 2141,
+// JSON-NEXT:        "line": 68,
+// JSON-NEXT:        "col": 1,
+// JSON-NEXT:        "tokLen": 1
+// JSON-NEXT:       }
+// JSON-NEXT:      },
+// JSON-NEXT:      "name": "tmpl",
+// JSON-NEXT:      "inner": [
+// JSON-NEXT:       {
+// JSON-NEXT:        "id": "0x{{.*}}",
+// JSON-NEXT:        "kind": "TemplateTypeParmDecl",
+// JSON-NEXT:        "loc": {
+// JSON-NEXT:         "offset": 2111,
+// JSON-NEXT:         "line": 65,
+// JSON-NEXT:         "col": 19,
+// JSON-NEXT:         "tokLen": 1
+// JSON-NEXT:        },
+// JSON-NEXT:        "range": {
+// JSON-NEXT:         "begin": {
+// JSON-NEXT:          "offset": 2102,
+// JSON-NEXT:          "col": 10,
+// JSON-NEXT:          "tokLen": 8
+// JSON-NEXT:         },
+// JSON-NEXT:         "end": {
+// JSON-NEXT:          "offset": 2111,
+// JSON-NEXT:          "col": 19,
+// JSON-NEXT:          "tokLen": 1
+// JSON-NEXT:         }
+// JSON-NEXT:        },
+// JSON-NEXT:        "isReferenced": true,
+// JSON-NEXT:        "name": "T",
+// JSON-NEXT:        "tagUsed": "typename",
+// JSON-NEXT:        "depth": 0,
+// JSON-NEXT:        "index": 0
+// JSON-NEXT:       },
+// JSON-NEXT:       {
+// JSON-NEXT:        "id": "0x{{.*}}",
+// JSON-NEXT:        "kind": "FunctionDecl",
+// JSON-NEXT:        "loc": {
+// JSON-NEXT:         "offset": 2119,
+// JSON-NEXT:         "line": 66,
+// JSON-NEXT:         "col": 6,
+// JSON-NEXT:         "tokLen": 4
+// JSON-NEXT:        },
+// JSON-NEXT:        "range": {
+// JSON-NEXT:         "begin": {
+// JSON-NEXT:          "offset": 2114,
+// JSON-NEXT:          "col": 1,
+// JSON-NEXT:          "tokLen": 4
+// JSON-NEXT:         },
+// JSON-NEXT:         "end": {
+// JSON-NEXT:          "offset": 2141,
+// JSON-NEXT:          "line": 68,
+// JSON-NEXT:          "col": 1,
+// JSON-NEXT:          "tokLen": 1
+// JSON-NEXT:         }
+// JSON-NEXT:        },
+// JSON-NEXT:        "name": "tmpl",
+// JSON-NEXT:        "type": {
+// JSON-NEXT:         "qualType": "void ()"
+// JSON-NEXT:        },
+// JSON-NEXT:        "inner": [
+// JSON-NEXT:         {
+// JSON-NEXT:          "id": "0x{{.*}}",
+// JSON-NEXT:          "kind": "CompoundStmt",
+// JSON-NEXT:          "range": {
+// JSON-NEXT:           "begin": {
+// JSON-NEXT:            "offset": 2126,
+// JSON-NEXT:            "line": 66,
+// JSON-NEXT:            "col": 13,
+// JSON-NEXT:            "tokLen": 1
+// JSON-NEXT:           },
+// JSON-NEXT:           "end": {
+// JSON-NEXT:            "offset": 2141,
+// JSON-NEXT:            "line": 68,
+// JSON-NEXT:            "col": 1,
+// JSON-NEXT:            "tokLen": 1
+// JSON-NEXT:           }
+// JSON-NEXT:          },
+// JSON-NEXT:          "inner": [
+// JSON-NEXT:           {
+// JSON-NEXT:            "id": "0x{{.*}}",
+// JSON-NEXT:            "kind": "CallExpr",
+// JSON-NEXT:            "range": {
+// JSON-NEXT:             "begin": {
+// JSON-NEXT:              "offset": 2130,
+// JSON-NEXT:              "line": 67,
+// JSON-NEXT:              "col": 3,
+// JSON-NEXT:              "tokLen": 4
+// JSON-NEXT:             },
+// JSON-NEXT:             "end": {
+// JSON-NEXT:              "offset": 2138,
+// JSON-NEXT:              "col": 11,
+// JSON-NEXT:              "tokLen": 1
+// JSON-NEXT:             }
+// JSON-NEXT:            },
+// JSON-NEXT:            "type": {
+// JSON-NEXT:             "qualType": "<dependent type>"
+// JSON-NEXT:            },
+// JSON-NEXT:            "valueCategory": "prvalue",
+// JSON-NEXT:            "inner": [
+// JSON-NEXT:             {
+// JSON-NEXT:              "id": "0x{{.*}}",
+// JSON-NEXT:              "kind": "UnresolvedLookupExpr",
+// JSON-NEXT:              "range": {
+// JSON-NEXT:               "begin": {
+// JSON-NEXT:                "offset": 2130,
+// JSON-NEXT:                "col": 3,
+// JSON-NEXT:                "tokLen": 4
+// JSON-NEXT:               },
+// JSON-NEXT:               "end": {
+// JSON-NEXT:                "offset": 2130,
+// JSON-NEXT:                "col": 3,
+// JSON-NEXT:                "tokLen": 4
+// JSON-NEXT:               }
+// JSON-NEXT:              },
+// JSON-NEXT:              "type": {
+// JSON-NEXT:               "qualType": "<overloaded function type>"
+// JSON-NEXT:              },
+// JSON-NEXT:              "valueCategory": "lvalue",
+// JSON-NEXT:              "usesADL": true,
+// JSON-NEXT:              "name": "func",
+// JSON-NEXT:              "lookups": [
+// JSON-NEXT:               {
+// JSON-NEXT:                "id": "0x{{.*}}",
+// JSON-NEXT:                "kind": "FunctionDecl",
+// JSON-NEXT:                "name": "func",
+// JSON-NEXT:                "type": {
+// JSON-NEXT:                 "qualType": "void (float)"
+// JSON-NEXT:                }
+// JSON-NEXT:               },
+// JSON-NEXT:               {
+// JSON-NEXT:                "id": "0x{{.*}}",
+// JSON-NEXT:                "kind": "FunctionDecl",
+// JSON-NEXT:                "name": "func",
+// JSON-NEXT:                "type": {
+// JSON-NEXT:                 "qualType": "void (int)"
+// JSON-NEXT:                }
+// JSON-NEXT:               }
+// JSON-NEXT:              ]
+// JSON-NEXT:             },
+// JSON-NEXT:             {
+// JSON-NEXT:              "id": "0x{{.*}}",
+// JSON-NEXT:              "kind": "CXXUnresolvedConstructExpr",
+// JSON-NEXT:              "range": {
+// JSON-NEXT:               "begin": {
+// JSON-NEXT:                "offset": 2135,
+// JSON-NEXT:                "col": 8,
+// JSON-NEXT:                "tokLen": 1
+// JSON-NEXT:               },
+// JSON-NEXT:               "end": {
+// JSON-NEXT:                "offset": 2137,
+// JSON-NEXT:                "col": 10,
+// JSON-NEXT:                "tokLen": 1
+// JSON-NEXT:               }
+// JSON-NEXT:              },
+// JSON-NEXT:              "type": {
+// JSON-NEXT:               "qualType": "T"
+// JSON-NEXT:              },
+// JSON-NEXT:              "valueCategory": "prvalue"
+// JSON-NEXT:             }
+// JSON-NEXT:            ]
+// JSON-NEXT:           }
+// JSON-NEXT:          ]
+// JSON-NEXT:         }
+// JSON-NEXT:        ]
+// JSON-NEXT:       }
+// JSON-NEXT:      ]
+// JSON-NEXT:     }
+// JSON-NEXT:    ]
+// JSON-NEXT:   },
+// JSON-NEXT:   {
+// JSON-NEXT:    "id": "0x{{.*}}",
+// JSON-NEXT:    "kind": "NamespaceDecl",
+// JSON-NEXT:    "loc": {
+// JSON-NEXT:     "offset": 2253,
+// JSON-NEXT:     "line": 73,
+// JSON-NEXT:     "col": 11,
+// JSON-NEXT:     "tokLen": 5
+// JSON-NEXT:    },
+// JSON-NEXT:    "range": {
+// JSON-NEXT:     "begin": {
+// JSON-NEXT:      "offset": 2243,
+// JSON-NEXT:      "col": 1,
+// JSON-NEXT:      "tokLen": 9
+// JSON-NEXT:     },
+// JSON-NEXT:     "end": {
+// JSON-NEXT:      "offset": 2387,
+// JSON-NEXT:      "line": 77,
+// JSON-NEXT:      "col": 1,
+// JSON-NEXT:      "tokLen": 1
+// JSON-NEXT:     }
+// JSON-NEXT:    },
+// JSON-NEXT:    "name": "test3",
+// JSON-NEXT:    "inner": [
+// JSON-NEXT:     {
+// JSON-NEXT:      "id": "0x{{.*}}",
+// JSON-NEXT:      "kind": "ClassTemplateDecl",
+// JSON-NEXT:      "loc": {
+// JSON-NEXT:       "offset": 2291,
+// JSON-NEXT:       "line": 74,
+// JSON-NEXT:       "col": 31,
+// JSON-NEXT:       "tokLen": 1
+// JSON-NEXT:      },
+// JSON-NEXT:      "range": {
+// JSON-NEXT:       "begin": {
+// JSON-NEXT:        "offset": 2263,
+// JSON-NEXT:        "col": 3,
+// JSON-NEXT:        "tokLen": 8
+// JSON-NEXT:       },
+// JSON-NEXT:       "end": {
+// JSON-NEXT:        "offset": 2294,
+// JSON-NEXT:        "col": 34,
+// JSON-NEXT:        "tokLen": 1
+// JSON-NEXT:       }
+// JSON-NEXT:      },
+// JSON-NEXT:      "name": "A",
+// JSON-NEXT:      "inner": [
+// JSON-NEXT:       {
+// JSON-NEXT:        "id": "0x{{.*}}",
+// JSON-NEXT:        "kind": "TemplateTypeParmDecl",
+// JSON-NEXT:        "loc": {
+// JSON-NEXT:         "offset": 2281,
+// JSON-NEXT:         "col": 21,
+// JSON-NEXT:         "tokLen": 1
+// JSON-NEXT:        },
+// JSON-NEXT:        "range": {
+// JSON-NEXT:         "begin": {
+// JSON-NEXT:          "offset": 2272,
+// JSON-NEXT:          "col": 12,
+// JSON-NEXT:          "tokLen": 8
+// JSON-NEXT:         },
+// JSON-NEXT:         "end": {
+// JSON-NEXT:          "offset": 2281,
+// JSON-NEXT:          "col": 21,
+// JSON-NEXT:          "tokLen": 1
+// JSON-NEXT:         }
+// JSON-NEXT:        },
+// JSON-NEXT:        "name": "T",
+// JSON-NEXT:        "tagUsed": "typename",
+// JSON-NEXT:        "depth": 0,
+// JSON-NEXT:        "index": 0
+// JSON-NEXT:       },
+// JSON-NEXT:       {
+// JSON-NEXT:        "id": "0x{{.*}}",
+// JSON-NEXT:        "kind": "CXXRecordDecl",
+// JSON-NEXT:        "loc": {
+// JSON-NEXT:         "offset": 2291,
+// JSON-NEXT:         "col": 31,
+// JSON-NEXT:         "tokLen": 1
+// JSON-NEXT:        },
+// JSON-NEXT:        "range": {
+// JSON-NEXT:         "begin": {
+// JSON-NEXT:          "offset": 2284,
+// JSON-NEXT:          "col": 24,
+// JSON-NEXT:          "tokLen": 6
+// JSON-NEXT:         },
+// JSON-NEXT:         "end": {
+// JSON-NEXT:          "offset": 2294,
+// JSON-NEXT:          "col": 34,
+// JSON-NEXT:          "tokLen": 1
+// JSON-NEXT:         }
+// JSON-NEXT:        },
+// JSON-NEXT:        "name": "A",
+// JSON-NEXT:        "tagUsed": "struct",
+// JSON-NEXT:        "completeDefinition": true,
+// JSON-NEXT:        "definitionData": {
+// JSON-NEXT:         "canConstDefaultInit": true,
+// JSON-NEXT:         "copyAssign": {
+// JSON-NEXT:          "hasConstParam": true,
+// JSON-NEXT:          "implicitHasConstParam": true,
+// JSON-NEXT:          "needsImplicit": true,
+// JSON-NEXT:          "simple": true,
+// JSON-NEXT:          "trivial": true
+// JSON-NEXT:         },
+// JSON-NEXT:         "copyCtor": {
+// JSON-NEXT:          "hasConstParam": true,
+// JSON-NEXT:          "implicitHasConstParam": true,
+// JSON-NEXT:          "needsImplicit": true,
+// JSON-NEXT:          "simple": true,
+// JSON-NEXT:          "trivial": true
+// JSON-NEXT:         },
+// JSON-NEXT:         "defaultCtor": {
+// JSON-NEXT:          "defaultedIsConstexpr": true,
+// JSON-NEXT:          "exists": true,
+// JSON-NEXT:          "isConstexpr": true,
+// JSON-NEXT:          "needsImplicit": true,
+// JSON-NEXT:          "trivial": true
+// JSON-NEXT:         },
+// JSON-NEXT:         "dtor": {
+// JSON-NEXT:          "irrelevant": true,
+// JSON-NEXT:          "needsImplicit": true,
+// JSON-NEXT:          "simple": true,
+// JSON-NEXT:          "trivial": true
+// JSON-NEXT:         },
+// JSON-NEXT:         "hasConstexprNonCopyMoveConstructor": true,
+// JSON-NEXT:         "isAggregate": true,
+// JSON-NEXT:         "isEmpty": true,
+// JSON-NEXT:         "isLiteral": true,
+// JSON-NEXT:         "isPOD": true,
+// JSON-NEXT:         "isStandardLayout": true,
+// JSON-NEXT:         "isTrivial": true,
+// JSON-NEXT:         "isTriviallyCopyable": true,
+// JSON-NEXT:         "moveAssign": {
+// JSON-NEXT:          "exists": true,
+// JSON-NEXT:          "needsImplicit": true,
+// JSON-NEXT:          "simple": true,
+// JSON-NEXT:          "trivial": true
+// JSON-NEXT:         },
+// JSON-NEXT:         "moveCtor": {
+// JSON-NEXT:          "exists": true,
+// JSON-NEXT:          "needsImplicit": true,
+// JSON-NEXT:          "simple": true,
+// JSON-NEXT:          "trivial": true
+// JSON-NEXT:         }
+// JSON-NEXT:        },
+// JSON-NEXT:        "inner": [
+// JSON-NEXT:         {
+// JSON-NEXT:          "id": "0x{{.*}}",
+// JSON-NEXT:          "kind": "CXXRecordDecl",
+// JSON-NEXT:          "loc": {
+// JSON-NEXT:           "offset": 2291,
+// JSON-NEXT:           "col": 31,
+// JSON-NEXT:           "tokLen": 1
+// JSON-NEXT:          },
+// JSON-NEXT:          "range": {
+// JSON-NEXT:           "begin": {
+// JSON-NEXT:            "offset": 2284,
+// JSON-NEXT:            "col": 24,
+// JSON-NEXT:            "tokLen": 6
+// JSON-NEXT:           },
+// JSON-NEXT:           "end": {
+// JSON-NEXT:            "offset": 2291,
+// JSON-NEXT:            "col": 31,
+// JSON-NEXT:            "tokLen": 1
+// JSON-NEXT:           }
+// JSON-NEXT:          },
+// JSON-NEXT:          "isImplicit": true,
+// JSON-NEXT:          "name": "A",
+// JSON-NEXT:          "tagUsed": "struct"
+// JSON-NEXT:         }
+// JSON-NEXT:        ]
+// JSON-NEXT:       },
+// JSON-NEXT:       {
+// JSON-NEXT:        "id": "0x{{.*}}",
+// JSON-NEXT:        "kind": "ClassTemplateSpecializationDecl",
+// JSON-NEXT:        "loc": {
+// JSON-NEXT:         "offset": 2291,
+// JSON-NEXT:         "col": 31,
+// JSON-NEXT:         "tokLen": 1
+// JSON-NEXT:        },
+// JSON-NEXT:        "range": {
+// JSON-NEXT:         "begin": {
+// JSON-NEXT:          "offset": 2263,
+// JSON-NEXT:          "col": 3,
+// JSON-NEXT:          "tokLen": 8
+// JSON-NEXT:         },
+// JSON-NEXT:         "end": {
+// JSON-NEXT:          "offset": 2294,
+// JSON-NEXT:          "col": 34,
+// JSON-NEXT:          "tokLen": 1
+// JSON-NEXT:         }
+// JSON-NEXT:        },
+// JSON-NEXT:        "name": "A",
+// JSON-NEXT:        "tagUsed": "struct",
+// JSON-NEXT:        "inner": [
+// JSON-NEXT:         {
+// JSON-NEXT:          "kind": "TemplateArgument",
+// JSON-NEXT:          "type": {
+// JSON-NEXT:           "qualType": "int"
+// JSON-NEXT:          },
+// JSON-NEXT:          "inner": [
+// JSON-NEXT:           {
+// JSON-NEXT:            "id": "0x{{.*}}",
+// JSON-NEXT:            "kind": "BuiltinType",
+// JSON-NEXT:            "type": {
+// JSON-NEXT:             "qualType": "int"
+// JSON-NEXT:            }
+// JSON-NEXT:           }
+// JSON-NEXT:          ]
+// JSON-NEXT:         }
+// JSON-NEXT:        ]
+// JSON-NEXT:       }
+// JSON-NEXT:      ]
+// JSON-NEXT:     },
+// JSON-NEXT:     {
+// JSON-NEXT:      "id": "0x{{.*}}",
+// JSON-NEXT:      "kind": "FunctionTemplateDecl",
+// JSON-NEXT:      "loc": {
+// JSON-NEXT:       "offset": 2291,
+// JSON-NEXT:       "col": 31,
+// JSON-NEXT:       "tokLen": 1
+// JSON-NEXT:      },
+// JSON-NEXT:      "range": {
+// JSON-NEXT:       "begin": {
+// JSON-NEXT:        "offset": 2263,
+// JSON-NEXT:        "col": 3,
+// JSON-NEXT:        "tokLen": 8
+// JSON-NEXT:       },
+// JSON-NEXT:       "end": {
+// JSON-NEXT:        "offset": 2291,
+// JSON-NEXT:        "col": 31,
+// JSON-NEXT:        "tokLen": 1
+// JSON-NEXT:       }
+// JSON-NEXT:      },
+// JSON-NEXT:      "isImplicit": true,
+// JSON-NEXT:      "name": "<deduction guide for A>",
+// JSON-NEXT:      "inner": [
+// JSON-NEXT:       {
+// JSON-NEXT:        "id": "0x{{.*}}",
+// JSON-NEXT:        "kind": "TemplateTypeParmDecl",
+// JSON-NEXT:        "loc": {
+// JSON-NEXT:         "offset": 2281,
+// JSON-NEXT:         "col": 21,
+// JSON-NEXT:         "tokLen": 1
+// JSON-NEXT:        },
+// JSON-NEXT:        "range": {
+// JSON-NEXT:         "begin": {
+// JSON-NEXT:          "offset": 2272,
+// JSON-NEXT:          "col": 12,
+// JSON-NEXT:          "tokLen": 8
+// JSON-NEXT:         },
+// JSON-NEXT:         "end": {
+// JSON-NEXT:          "offset": 2281,
+// JSON-NEXT:          "col": 21,
+// JSON-NEXT:          "tokLen": 1
+// JSON-NEXT:         }
+// JSON-NEXT:        },
+// JSON-NEXT:        "name": "T",
+// JSON-NEXT:        "tagUsed": "typename",
+// JSON-NEXT:        "depth": 0,
+// JSON-NEXT:        "index": 0
+// JSON-NEXT:       },
+// JSON-NEXT:       {
+// JSON-NEXT:        "id": "0x{{.*}}",
+// JSON-NEXT:        "kind": "CXXDeductionGuideDecl",
+// JSON-NEXT:        "loc": {
+// JSON-NEXT:         "offset": 2291,
+// JSON-NEXT:         "col": 31,
+// JSON-NEXT:         "tokLen": 1
+// JSON-NEXT:        },
+// JSON-NEXT:        "range": {
+// JSON-NEXT:         "begin": {
+// JSON-NEXT:          "offset": 2291,
+// JSON-NEXT:          "col": 31,
+// JSON-NEXT:          "tokLen": 1
+// JSON-NEXT:         },
+// JSON-NEXT:         "end": {
+// JSON-NEXT:          "offset": 2291,
+// JSON-NEXT:          "col": 31,
+// JSON-NEXT:          "tokLen": 1
+// JSON-NEXT:         }
+// JSON-NEXT:        },
+// JSON-NEXT:        "isImplicit": true,
+// JSON-NEXT:        "name": "<deduction guide for A>",
+// JSON-NEXT:        "type": {
+// JSON-NEXT:         "qualType": "auto () -> A<T>"
+// JSON-NEXT:        }
+// JSON-NEXT:       }
+// JSON-NEXT:      ]
+// JSON-NEXT:     },
+// JSON-NEXT:     {
+// JSON-NEXT:      "id": "0x{{.*}}",
+// JSON-NEXT:      "kind": "FunctionTemplateDecl",
+// JSON-NEXT:      "loc": {
+// JSON-NEXT:       "offset": 2291,
+// JSON-NEXT:       "col": 31,
+// JSON-NEXT:       "tokLen": 1
+// JSON-NEXT:      },
+// JSON-NEXT:      "range": {
+// JSON-NEXT:       "begin": {
+// JSON-NEXT:        "offset": 2263,
+// JSON-NEXT:        "col": 3,
+// JSON-NEXT:        "tokLen": 8
+// JSON-NEXT:       },
+// JSON-NEXT:       "end": {
+// JSON-NEXT:        "offset": 2291,
+// JSON-NEXT:        "col": 31,
+// JSON-NEXT:        "tokLen": 1
+// JSON-NEXT:       }
+// JSON-NEXT:      },
+// JSON-NEXT:      "isImplicit": true,
+// JSON-NEXT:      "name": "<deduction guide for A>",
+// JSON-NEXT:      "inner": [
+// JSON-NEXT:       {
+// JSON-NEXT:        "id": "0x{{.*}}",
+// JSON-NEXT:        "kind": "TemplateTypeParmDecl",
+// JSON-NEXT:        "loc": {
+// JSON-NEXT:         "offset": 2281,
+// JSON-NEXT:         "col": 21,
+// JSON-NEXT:         "tokLen": 1
+// JSON-NEXT:        },
+// JSON-NEXT:        "range": {
+// JSON-NEXT:         "begin": {
+// JSON-NEXT:          "offset": 2272,
+// JSON-NEXT:          "col": 12,
+// JSON-NEXT:          "tokLen": 8
+// JSON-NEXT:         },
+// JSON-NEXT:         "end": {
+// JSON-NEXT:          "offset": 2281,
+// JSON-NEXT:          "col": 21,
+// JSON-NEXT:          "tokLen": 1
+// JSON-NEXT:         }
+// JSON-NEXT:        },
+// JSON-NEXT:        "name": "T",
+// JSON-NEXT:        "tagUsed": "typename",
+// JSON-NEXT:        "depth": 0,
+// JSON-NEXT:        "index": 0
+// JSON-NEXT:       },
+// JSON-NEXT:       {
+// JSON-NEXT:        "id": "0x{{.*}}",
+// JSON-NEXT:        "kind": "CXXDeductionGuideDecl",
+// JSON-NEXT:        "loc": {
+// JSON-NEXT:         "offset": 2291,
+// JSON-NEXT:         "col": 31,
+// JSON-NEXT:         "tokLen": 1
+// JSON-NEXT:        },
+// JSON-NEXT:        "range": {
+// JSON-NEXT:         "begin": {
+// JSON-NEXT:          "offset": 2291,
+// JSON-NEXT:          "col": 31,
+// JSON-NEXT:          "tokLen": 1
+// JSON-NEXT:         },
+// JSON-NEXT:         "end": {
+// JSON-NEXT:          "offset": 2291,
+// JSON-NEXT:          "col": 31,
+// JSON-NEXT:          "tokLen": 1
+// JSON-NEXT:         }
+// JSON-NEXT:        },
+// JSON-NEXT:        "isImplicit": true,
+// JSON-NEXT:        "name": "<deduction guide for A>",
+// JSON-NEXT:        "type": {
+// JSON-NEXT:         "qualType": "auto (A<T>) -> A<T>"
+// JSON-NEXT:        },
+// JSON-NEXT:        "inner": [
+// JSON-NEXT:         {
+// JSON-NEXT:          "id": "0x{{.*}}",
+// JSON-NEXT:          "kind": "ParmVarDecl",
+// JSON-NEXT:          "loc": {
+// JSON-NEXT:           "offset": 2291,
+// JSON-NEXT:           "col": 31,
+// JSON-NEXT:           "tokLen": 1
+// JSON-NEXT:          },
+// JSON-NEXT:          "range": {
+// JSON-NEXT:           "begin": {
+// JSON-NEXT:            "offset": 2291,
+// JSON-NEXT:            "col": 31,
+// JSON-NEXT:            "tokLen": 1
+// JSON-NEXT:           },
+// JSON-NEXT:           "end": {
+// JSON-NEXT:            "offset": 2291,
+// JSON-NEXT:            "col": 31,
+// JSON-NEXT:            "tokLen": 1
+// JSON-NEXT:           }
+// JSON-NEXT:          },
+// JSON-NEXT:          "type": {
+// JSON-NEXT:           "qualType": "A<T>"
+// JSON-NEXT:          }
+// JSON-NEXT:         }
+// JSON-NEXT:        ]
+// JSON-NEXT:       }
+// JSON-NEXT:      ]
+// JSON-NEXT:     },
+// JSON-NEXT:     {
+// JSON-NEXT:      "id": "0x{{.*}}",
+// JSON-NEXT:      "kind": "FunctionTemplateDecl",
+// JSON-NEXT:      "loc": {
+// JSON-NEXT:       "offset": 2320,
+// JSON-NEXT:       "line": 75,
+// JSON-NEXT:       "col": 24,
+// JSON-NEXT:       "tokLen": 1
+// JSON-NEXT:      },
+// JSON-NEXT:      "range": {
+// JSON-NEXT:       "begin": {
+// JSON-NEXT:        "offset": 2299,
+// JSON-NEXT:        "col": 3,
+// JSON-NEXT:        "tokLen": 8
+// JSON-NEXT:       },
+// JSON-NEXT:       "end": {
+// JSON-NEXT:        "offset": 2333,
+// JSON-NEXT:        "col": 37,
+// JSON-NEXT:        "tokLen": 1
+// JSON-NEXT:       }
+// JSON-NEXT:      },
+// JSON-NEXT:      "name": "<deduction guide for A>",
+// JSON-NEXT:      "inner": [
+// JSON-NEXT:       {
+// JSON-NEXT:        "id": "0x{{.*}}",
+// JSON-NEXT:        "kind": "TemplateTypeParmDecl",
+// JSON-NEXT:        "loc": {
+// JSON-NEXT:         "offset": 2317,
+// JSON-NEXT:         "col": 21,
+// JSON-NEXT:         "tokLen": 1
+// JSON-NEXT:        },
+// JSON-NEXT:        "range": {
+// JSON-NEXT:         "begin": {
+// JSON-NEXT:          "offset": 2308,
+// JSON-NEXT:          "col": 12,
+// JSON-NEXT:          "tokLen": 8
+// JSON-NEXT:         },
+// JSON-NEXT:         "end": {
+// JSON-NEXT:          "offset": 2317,
+// JSON-NEXT:          "col": 21,
+// JSON-NEXT:          "tokLen": 1
+// JSON-NEXT:         }
+// JSON-NEXT:        },
+// JSON-NEXT:        "isReferenced": true,
+// JSON-NEXT:        "name": "T",
+// JSON-NEXT:        "tagUsed": "typename",
+// JSON-NEXT:        "depth": 0,
+// JSON-NEXT:        "index": 0
+// JSON-NEXT:       },
+// JSON-NEXT:       {
+// JSON-NEXT:        "id": "0x{{.*}}",
+// JSON-NEXT:        "kind": "CXXDeductionGuideDecl",
+// JSON-NEXT:        "loc": {
+// JSON-NEXT:         "offset": 2320,
+// JSON-NEXT:         "col": 24,
+// JSON-NEXT:         "tokLen": 1
+// JSON-NEXT:        },
+// JSON-NEXT:        "range": {
+// JSON-NEXT:         "begin": {
+// JSON-NEXT:          "offset": 2320,
+// JSON-NEXT:          "col": 24,
+// JSON-NEXT:          "tokLen": 1
+// JSON-NEXT:         },
+// JSON-NEXT:         "end": {
+// JSON-NEXT:          "offset": 2333,
+// JSON-NEXT:          "col": 37,
+// JSON-NEXT:          "tokLen": 1
+// JSON-NEXT:         }
+// JSON-NEXT:        },
+// JSON-NEXT:        "name": "<deduction guide for A>",
+// JSON-NEXT:        "type": {
+// JSON-NEXT:         "qualType": "auto (T) -> A<int>"
+// JSON-NEXT:        },
+// JSON-NEXT:        "inner": [
+// JSON-NEXT:         {
+// JSON-NEXT:          "id": "0x{{.*}}",
+// JSON-NEXT:          "kind": "ParmVarDecl",
+// JSON-NEXT:          "loc": {
+// JSON-NEXT:           "offset": 2323,
+// JSON-NEXT:           "col": 27,
+// JSON-NEXT:           "tokLen": 1
+// JSON-NEXT:          },
+// JSON-NEXT:          "range": {
+// JSON-NEXT:           "begin": {
+// JSON-NEXT:            "offset": 2322,
+// JSON-NEXT:            "col": 26,
+// JSON-NEXT:            "tokLen": 1
+// JSON-NEXT:           },
+// JSON-NEXT:           "end": {
+// JSON-NEXT:            "offset": 2322,
+// JSON-NEXT:            "col": 26,
+// JSON-NEXT:            "tokLen": 1
+// JSON-NEXT:           }
+// JSON-NEXT:          },
+// JSON-NEXT:          "type": {
+// JSON-NEXT:           "qualType": "T"
+// JSON-NEXT:          }
+// JSON-NEXT:         }
+// JSON-NEXT:        ]
+// JSON-NEXT:       }
+// JSON-NEXT:      ]
+// JSON-NEXT:     }
+// JSON-NEXT:    ]
+// JSON-NEXT:   },
+// JSON-NEXT:   {
+// JSON-NEXT:    "id": "0x{{.*}}",
+// JSON-NEXT:    "kind": "NamespaceDecl",
+// JSON-NEXT:    "loc": {
+// JSON-NEXT:     "offset": 2400,
+// JSON-NEXT:     "line": 79,
+// JSON-NEXT:     "col": 11,
+// JSON-NEXT:     "tokLen": 5
+// JSON-NEXT:    },
+// JSON-NEXT:    "range": {
+// JSON-NEXT:     "begin": {
+// JSON-NEXT:      "offset": 2390,
+// JSON-NEXT:      "col": 1,
+// JSON-NEXT:      "tokLen": 9
+// JSON-NEXT:     },
+// JSON-NEXT:     "end": {
+// JSON-NEXT:      "offset": 3297,
+// JSON-NEXT:      "line": 103,
+// JSON-NEXT:      "col": 1,
+// JSON-NEXT:      "tokLen": 1
+// JSON-NEXT:     }
+// JSON-NEXT:    },
+// JSON-NEXT:    "name": "test4",
+// JSON-NEXT:    "inner": [
+// JSON-NEXT:     {
+// JSON-NEXT:      "id": "0x{{.*}}",
+// JSON-NEXT:      "kind": "ClassTemplateDecl",
+// JSON-NEXT:      "loc": {
+// JSON-NEXT:       "offset": 2445,
+// JSON-NEXT:       "line": 81,
+// JSON-NEXT:       "col": 8,
+// JSON-NEXT:       "tokLen": 3
+// JSON-NEXT:      },
+// JSON-NEXT:      "range": {
+// JSON-NEXT:       "begin": {
+// JSON-NEXT:        "offset": 2408,
+// JSON-NEXT:        "line": 80,
+// JSON-NEXT:        "col": 1,
+// JSON-NEXT:        "tokLen": 8
+// JSON-NEXT:       },
+// JSON-NEXT:       "end": {
+// JSON-NEXT:        "offset": 2471,
+// JSON-NEXT:        "line": 83,
+// JSON-NEXT:        "col": 1,
+// JSON-NEXT:        "tokLen": 1
+// JSON-NEXT:       }
+// JSON-NEXT:      },
+// JSON-NEXT:      "name": "foo",
+// JSON-NEXT:      "inner": [
+// JSON-NEXT:       {
+// JSON-NEXT:        "id": "0x{{.*}}",
+// JSON-NEXT:        "kind": "NonTypeTemplateParmDecl",
+// JSON-NEXT:        "loc": {
+// JSON-NEXT:         "offset": 2427,
+// JSON-NEXT:         "line": 80,
+// JSON-NEXT:         "col": 20,
+// JSON-NEXT:         "tokLen": 1
+// JSON-NEXT:        },
+// JSON-NEXT:        "range": {
+// JSON-NEXT:         "begin": {
+// JSON-NEXT:          "offset": 2418,
+// JSON-NEXT:          "col": 11,
+// JSON-NEXT:          "tokLen": 8
+// JSON-NEXT:         },
+// JSON-NEXT:         "end": {
+// JSON-NEXT:          "offset": 2427,
+// JSON-NEXT:          "col": 20,
+// JSON-NEXT:          "tokLen": 1
+// JSON-NEXT:         }
+// JSON-NEXT:        },
+// JSON-NEXT:        "name": "X",
+// JSON-NEXT:        "type": {
+// JSON-NEXT:         "qualType": "unsigned int"
+// JSON-NEXT:        },
+// JSON-NEXT:        "depth": 0,
+// JSON-NEXT:        "index": 0
+// JSON-NEXT:       },
+// JSON-NEXT:       {
+// JSON-NEXT:        "id": "0x{{.*}}",
+// JSON-NEXT:        "kind": "NonTypeTemplateParmDecl",
+// JSON-NEXT:        "loc": {
+// JSON-NEXT:         "offset": 2435,
+// JSON-NEXT:         "col": 28,
+// JSON-NEXT:         "tokLen": 1
+// JSON-NEXT:        },
+// JSON-NEXT:        "range": {
+// JSON-NEXT:         "begin": {
+// JSON-NEXT:          "offset": 2430,
+// JSON-NEXT:          "col": 23,
+// JSON-NEXT:          "tokLen": 4
+// JSON-NEXT:         },
+// JSON-NEXT:         "end": {
+// JSON-NEXT:          "offset": 2435,
+// JSON-NEXT:          "col": 28,
+// JSON-NEXT:          "tokLen": 1
+// JSON-NEXT:         }
+// JSON-NEXT:        },
+// JSON-NEXT:        "name": "A",
+// JSON-NEXT:        "type": {
+// JSON-NEXT:         "qualType": "auto"
+// JSON-NEXT:        },
+// JSON-NEXT:        "depth": 0,
+// JSON-NEXT:        "index": 1
+// JSON-NEXT:       },
+// JSON-NEXT:       {
+// JSON-NEXT:        "id": "0x{{.*}}",
+// JSON-NEXT:        "kind": "CXXRecordDecl",
+// JSON-NEXT:        "loc": {
+// JSON-NEXT:         "offset": 2445,
+// JSON-NEXT:         "line": 81,
+// JSON-NEXT:         "col": 8,
+// JSON-NEXT:         "tokLen": 3
+// JSON-NEXT:        },
+// JSON-NEXT:        "range": {
+// JSON-NEXT:         "begin": {
+// JSON-NEXT:          "offset": 2438,
+// JSON-NEXT:          "col": 1,
+// JSON-NEXT:          "tokLen": 6
+// JSON-NEXT:         },
+// JSON-NEXT:         "end": {
+// JSON-NEXT:          "offset": 2471,
+// JSON-NEXT:          "line": 83,
+// JSON-NEXT:          "col": 1,
+// JSON-NEXT:          "tokLen": 1
+// JSON-NEXT:         }
+// JSON-NEXT:        },
+// JSON-NEXT:        "name": "foo",
+// JSON-NEXT:        "tagUsed": "struct",
+// JSON-NEXT:        "completeDefinition": true,
+// JSON-NEXT:        "definitionData": {
+// JSON-NEXT:         "canConstDefaultInit": true,
+// JSON-NEXT:         "copyAssign": {
+// JSON-NEXT:          "hasConstParam": true,
+// JSON-NEXT:          "implicitHasConstParam": true,
+// JSON-NEXT:          "needsImplicit": true,
+// JSON-NEXT:          "simple": true,
+// JSON-NEXT:          "trivial": true
+// JSON-NEXT:         },
+// JSON-NEXT:         "copyCtor": {
+// JSON-NEXT:          "hasConstParam": true,
+// JSON-NEXT:          "implicitHasConstParam": true,
+// JSON-NEXT:          "needsImplicit": true,
+// JSON-NEXT:          "simple": true,
+// JSON-NEXT:          "trivial": true
+// JSON-NEXT:         },
+// JSON-NEXT:         "defaultCtor": {
+// JSON-NEXT:          "defaultedIsConstexpr": true,
+// JSON-NEXT:          "exists": true,
+// JSON-NEXT:          "isConstexpr": true,
+// JSON-NEXT:          "needsImplicit": true,
+// JSON-NEXT:          "trivial": true
+// JSON-NEXT:         },
+// JSON-NEXT:         "dtor": {
+// JSON-NEXT:          "irrelevant": true,
+// JSON-NEXT:          "needsImplicit": true,
+// JSON-NEXT:          "simple": true,
+// JSON-NEXT:          "trivial": true
+// JSON-NEXT:         },
+// JSON-NEXT:         "hasConstexprNonCopyMoveConstructor": true,
+// JSON-NEXT:         "isAggregate": true,
+// JSON-NEXT:         "isEmpty": true,
+// JSON-NEXT:         "isLiteral": true,
+// JSON-NEXT:         "isPOD": true,
+// JSON-NEXT:         "isStandardLayout": true,
+// JSON-NEXT:         "isTrivial": true,
+// JSON-NEXT:         "isTriviallyCopyable": true,
+// JSON-NEXT:         "moveAssign": {
+// JSON-NEXT:          "exists": true,
+// JSON-NEXT:          "needsImplicit": true,
+// JSON-NEXT:          "simple": true,
+// JSON-NEXT:          "trivial": true
+// JSON-NEXT:         },
+// JSON-NEXT:         "moveCtor": {
+// JSON-NEXT:          "exists": true,
+// JSON-NEXT:          "needsImplicit": true,
+// JSON-NEXT:          "simple": true,
+// JSON-NEXT:          "trivial": true
+// JSON-NEXT:         }
+// JSON-NEXT:        },
+// JSON-NEXT:        "inner": [
+// JSON-NEXT:         {
+// JSON-NEXT:          "id": "0x{{.*}}",
+// JSON-NEXT:          "kind": "CXXRecordDecl",
+// JSON-NEXT:          "loc": {
+// JSON-NEXT:           "offset": 2445,
+// JSON-NEXT:           "line": 81,
+// JSON-NEXT:           "col": 8,
+// JSON-NEXT:           "tokLen": 3
+// JSON-NEXT:          },
+// JSON-NEXT:          "range": {
+// JSON-NEXT:           "begin": {
+// JSON-NEXT:            "offset": 2438,
+// JSON-NEXT:            "col": 1,
+// JSON-NEXT:            "tokLen": 6
+// JSON-NEXT:           },
+// JSON-NEXT:           "end": {
+// JSON-NEXT:            "offset": 2445,
+// JSON-NEXT:            "col": 8,
+// JSON-NEXT:            "tokLen": 3
+// JSON-NEXT:           }
+// JSON-NEXT:          },
+// JSON-NEXT:          "isImplicit": true,
+// JSON-NEXT:          "name": "foo",
+// JSON-NEXT:          "tagUsed": "struct"
+// JSON-NEXT:         },
+// JSON-NEXT:         {
+// JSON-NEXT:          "id": "0x{{.*}}",
+// JSON-NEXT:          "kind": "CXXMethodDecl",
+// JSON-NEXT:          "loc": {
+// JSON-NEXT:           "offset": 2465,
+// JSON-NEXT:           "line": 82,
+// JSON-NEXT:           "col": 15,
+// JSON-NEXT:           "tokLen": 2
+// JSON-NEXT:          },
+// JSON-NEXT:          "range": {
+// JSON-NEXT:           "begin": {
+// JSON-NEXT:            "offset": 2453,
+// JSON-NEXT:            "col": 3,
+// JSON-NEXT:            "tokLen": 6
+// JSON-NEXT:           },
+// JSON-NEXT:           "end": {
+// JSON-NEXT:            "offset": 2468,
+// JSON-NEXT:            "col": 18,
+// JSON-NEXT:            "tokLen": 1
+// JSON-NEXT:           }
+// JSON-NEXT:          },
+// JSON-NEXT:          "name": "fn",
+// JSON-NEXT:          "type": {
+// JSON-NEXT:           "qualType": "void ()"
+// JSON-NEXT:          },
+// JSON-NEXT:          "storageClass": "static"
+// JSON-NEXT:         }
+// JSON-NEXT:        ]
+// JSON-NEXT:       },
+// JSON-NEXT:       {
+// JSON-NEXT:        "id": "0x{{.*}}",
+// JSON-NEXT:        "kind": "ClassTemplateSpecializationDecl",
+// JSON-NEXT:        "loc": {
+// JSON-NEXT:         "offset": 2445,
+// JSON-NEXT:         "line": 81,
+// JSON-NEXT:         "col": 8,
+// JSON-NEXT:         "tokLen": 3
+// JSON-NEXT:        },
+// JSON-NEXT:        "range": {
+// JSON-NEXT:         "begin": {
+// JSON-NEXT:          "offset": 2408,
+// JSON-NEXT:          "line": 80,
+// JSON-NEXT:          "col": 1,
+// JSON-NEXT:          "tokLen": 8
+// JSON-NEXT:         },
+// JSON-NEXT:         "end": {
+// JSON-NEXT:          "offset": 2471,
+// JSON-NEXT:          "line": 83,
+// JSON-NEXT:          "col": 1,
+// JSON-NEXT:          "tokLen": 1
+// JSON-NEXT:         }
+// JSON-NEXT:        },
+// JSON-NEXT:        "name": "foo",
+// JSON-NEXT:        "tagUsed": "struct",
+// JSON-NEXT:        "completeDefinition": true,
+// JSON-NEXT:        "definitionData": {
+// JSON-NEXT:         "canConstDefaultInit": true,
+// JSON-NEXT:         "canPassInRegisters": true,
+// JSON-NEXT:         "copyAssign": {
+// JSON-NEXT:          "hasConstParam": true,
+// JSON-NEXT:          "implicitHasConstParam": true,
+// JSON-NEXT:          "needsImplicit": true,
+// JSON-NEXT:          "simple": true,
+// JSON-NEXT:          "trivial": true
+// JSON-NEXT:         },
+// JSON-NEXT:         "copyCtor": {
+// JSON-NEXT:          "hasConstParam": true,
+// JSON-NEXT:          "implicitHasConstParam": true,
+// JSON-NEXT:          "needsImplicit": true,
+// JSON-NEXT:          "simple": true,
+// JSON-NEXT:          "trivial": true
+// JSON-NEXT:         },
+// JSON-NEXT:         "defaultCtor": {
+// JSON-NEXT:          "defaultedIsConstexpr": true,
+// JSON-NEXT:          "exists": true,
+// JSON-NEXT:          "isConstexpr": true,
+// JSON-NEXT:          "needsImplicit": true,
+// JSON-NEXT:          "trivial": true
+// JSON-NEXT:         },
+// JSON-NEXT:         "dtor": {
+// JSON-NEXT:          "irrelevant": true,
+// JSON-NEXT:          "needsImplicit": true,
+// JSON-NEXT:          "simple": true,
+// JSON-NEXT:          "trivial": true
+// JSON-NEXT:         },
+// JSON-NEXT:         "hasConstexprNonCopyMoveConstructor": true,
+// JSON-NEXT:         "isAggregate": true,
+// JSON-NEXT:         "isEmpty": true,
+// JSON-NEXT:         "isLiteral": true,
+// JSON-NEXT:         "isPOD": true,
+// JSON-NEXT:         "isStandardLayout": true,
+// JSON-NEXT:         "isTrivial": true,
+// JSON-NEXT:         "isTriviallyCopyable": true,
+// JSON-NEXT:         "moveAssign": {
+// JSON-NEXT:          "exists": true,
+// JSON-NEXT:          "needsImplicit": true,
+// JSON-NEXT:          "simple": true,
+// JSON-NEXT:          "trivial": true
+// JSON-NEXT:         },
+// JSON-NEXT:         "moveCtor": {
+// JSON-NEXT:          "exists": true,
+// JSON-NEXT:          "needsImplicit": true,
+// JSON-NEXT:          "simple": true,
+// JSON-NEXT:          "trivial": true
+// JSON-NEXT:         }
+// JSON-NEXT:        },
+// JSON-NEXT:        "inner": [
+// JSON-NEXT:         {
+// JSON-NEXT:          "kind": "TemplateArgument",
+// JSON-NEXT:          "value": 0
+// JSON-NEXT:         },
+// JSON-NEXT:         {
+// JSON-NEXT:          "kind": "TemplateArgument",
+// JSON-NEXT:          "value": 0
+// JSON-NEXT:         },
+// JSON-NEXT:         {
+// JSON-NEXT:          "id": "0x{{.*}}",
+// JSON-NEXT:          "kind": "CXXRecordDecl",
+// JSON-NEXT:          "loc": {
+// JSON-NEXT:           "offset": 2445,
+// JSON-NEXT:           "line": 81,
+// JSON-NEXT:           "col": 8,
+// JSON-NEXT:           "tokLen": 3
+// JSON-NEXT:          },
+// JSON-NEXT:          "range": {
+// JSON-NEXT:           "begin": {
+// JSON-NEXT:            "offset": 2438,
+// JSON-NEXT:            "col": 1,
+// JSON-NEXT:            "tokLen": 6
+// JSON-NEXT:           },
+// JSON-NEXT:           "end": {
+// JSON-NEXT:            "offset": 2445,
+// JSON-NEXT:            "col": 8,
+// JSON-NEXT:            "tokLen": 3
+// JSON-NEXT:           }
+// JSON-NEXT:          },
+// JSON-NEXT:          "isImplicit": true,
+// JSON-NEXT:          "name": "foo",
+// JSON-NEXT:          "tagUsed": "struct"
+// JSON-NEXT:         },
+// JSON-NEXT:         {
+// JSON-NEXT:          "id": "0x{{.*}}",
+// JSON-NEXT:          "kind": "CXXMethodDecl",
+// JSON-NEXT:          "loc": {
+// JSON-NEXT:           "offset": 2465,
+// JSON-NEXT:           "line": 82,
+// JSON-NEXT:           "col": 15,
+// JSON-NEXT:           "tokLen": 2
+// JSON-NEXT:          },
+// JSON-NEXT:          "range": {
+// JSON-NEXT:           "begin": {
+// JSON-NEXT:            "offset": 2453,
+// JSON-NEXT:            "col": 3,
+// JSON-NEXT:            "tokLen": 6
+// JSON-NEXT:           },
+// JSON-NEXT:           "end": {
+// JSON-NEXT:            "offset": 2468,
+// JSON-NEXT:            "col": 18,
+// JSON-NEXT:            "tokLen": 1
+// JSON-NEXT:           }
+// JSON-NEXT:          },
+// JSON-NEXT:          "isUsed": true,
+// JSON-NEXT:          "name": "fn",
+// JSON-NEXT:          "mangledName": "_ZN5test43fooILj0ELl0EE2fnEv",
+// JSON-NEXT:          "type": {
+// JSON-NEXT:           "qualType": "void ()"
+// JSON-NEXT:          },
+// JSON-NEXT:          "storageClass": "static"
+// JSON-NEXT:         }
+// JSON-NEXT:        ]
+// JSON-NEXT:       },
+// JSON-NEXT:       {
+// JSON-NEXT:        "id": "0x{{.*}}",
+// JSON-NEXT:        "kind": "ClassTemplateSpecializationDecl",
+// JSON-NEXT:        "name": "foo"
+// JSON-NEXT:       }
+// JSON-NEXT:      ]
+// JSON-NEXT:     },
+// JSON-NEXT:     {
+// JSON-NEXT:      "id": "0x{{.*}}",
+// JSON-NEXT:      "kind": "FunctionDecl",
+// JSON-NEXT:      "loc": {
+// JSON-NEXT:       "offset": 2846,
+// JSON-NEXT:       "line": 92,
+// JSON-NEXT:       "col": 6,
+// JSON-NEXT:       "tokLen": 4
+// JSON-NEXT:      },
+// JSON-NEXT:      "range": {
+// JSON-NEXT:       "begin": {
+// JSON-NEXT:        "offset": 2841,
+// JSON-NEXT:        "col": 1,
+// JSON-NEXT:        "tokLen": 4
+// JSON-NEXT:       },
+// JSON-NEXT:       "end": {
+// JSON-NEXT:        "offset": 2879,
+// JSON-NEXT:        "line": 94,
+// JSON-NEXT:        "col": 1,
+// JSON-NEXT:        "tokLen": 1
+// JSON-NEXT:       }
+// JSON-NEXT:      },
+// JSON-NEXT:      "name": "test",
+// JSON-NEXT:      "mangledName": "_ZN5test44testEv",
+// JSON-NEXT:      "type": {
+// JSON-NEXT:       "qualType": "void ()"
+// JSON-NEXT:      },
+// JSON-NEXT:      "inner": [
+// JSON-NEXT:       {
+// JSON-NEXT:        "id": "0x{{.*}}",
+// JSON-NEXT:        "kind": "CompoundStmt",
+// JSON-NEXT:        "range": {
+// JSON-NEXT:         "begin": {
+// JSON-NEXT:          "offset": 2853,
+// JSON-NEXT:          "line": 92,
+// JSON-NEXT:          "col": 13,
+// JSON-NEXT:          "tokLen": 1
+// JSON-NEXT:         },
+// JSON-NEXT:         "end": {
+// JSON-NEXT:          "offset": 2879,
+// JSON-NEXT:          "line": 94,
+// JSON-NEXT:          "col": 1,
+// JSON-NEXT:          "tokLen": 1
+// JSON-NEXT:         }
+// JSON-NEXT:        },
+// JSON-NEXT:        "inner": [
+// JSON-NEXT:         {
+// JSON-NEXT:          "id": "0x{{.*}}",
+// JSON-NEXT:          "kind": "CallExpr",
+// JSON-NEXT:          "range": {
+// JSON-NEXT:           "begin": {
+// JSON-NEXT:            "offset": 2857,
+// JSON-NEXT:            "line": 93,
+// JSON-NEXT:            "col": 3,
+// JSON-NEXT:            "tokLen": 3
+// JSON-NEXT:           },
+// JSON-NEXT:           "end": {
+// JSON-NEXT:            "offset": 2876,
+// JSON-NEXT:            "col": 22,
+// JSON-NEXT:            "tokLen": 1
+// JSON-NEXT:           }
+// JSON-NEXT:          },
+// JSON-NEXT:          "type": {
+// JSON-NEXT:           "qualType": "void"
+// JSON-NEXT:          },
+// JSON-NEXT:          "valueCategory": "prvalue",
+// JSON-NEXT:          "inner": [
+// JSON-NEXT:           {
+// JSON-NEXT:            "id": "0x{{.*}}",
+// JSON-NEXT:            "kind": "ImplicitCastExpr",
+// JSON-NEXT:            "range": {
+// JSON-NEXT:             "begin": {
+// JSON-NEXT:              "offset": 2857,
+// JSON-NEXT:              "col": 3,
+// JSON-NEXT:              "tokLen": 3
+// JSON-NEXT:             },
+// JSON-NEXT:             "end": {
+// JSON-NEXT:              "offset": 2873,
+// JSON-NEXT:              "col": 19,
+// JSON-NEXT:              "tokLen": 2
+// JSON-NEXT:             }
+// JSON-NEXT:            },
+// JSON-NEXT:            "type": {
+// JSON-NEXT:             "qualType": "void (*)()"
+// JSON-NEXT:            },
+// JSON-NEXT:            "valueCategory": "prvalue",
+// JSON-NEXT:            "castKind": "FunctionToPointerDecay",
+// JSON-NEXT:            "inner": [
+// JSON-NEXT:             {
+// JSON-NEXT:              "id": "0x{{.*}}",
+// JSON-NEXT:              "kind": "DeclRefExpr",
+// JSON-NEXT:              "range": {
+// JSON-NEXT:               "begin": {
+// JSON-NEXT:                "offset": 2857,
+// JSON-NEXT:                "col": 3,
+// JSON-NEXT:                "tokLen": 3
+// JSON-NEXT:               },
+// JSON-NEXT:               "end": {
+// JSON-NEXT:                "offset": 2873,
+// JSON-NEXT:                "col": 19,
+// JSON-NEXT:                "tokLen": 2
+// JSON-NEXT:               }
+// JSON-NEXT:              },
+// JSON-NEXT:              "type": {
+// JSON-NEXT:               "qualType": "void ()"
+// JSON-NEXT:              },
+// JSON-NEXT:              "valueCategory": "lvalue",
+// JSON-NEXT:              "referencedDecl": {
+// JSON-NEXT:               "id": "0x{{.*}}",
+// JSON-NEXT:               "kind": "CXXMethodDecl",
+// JSON-NEXT:               "name": "fn",
+// JSON-NEXT:               "type": {
+// JSON-NEXT:                "qualType": "void ()"
+// JSON-NEXT:               }
+// JSON-NEXT:              }
+// JSON-NEXT:             }
+// JSON-NEXT:            ]
+// JSON-NEXT:           }
+// JSON-NEXT:          ]
+// JSON-NEXT:         }
+// JSON-NEXT:        ]
+// JSON-NEXT:       }
+// JSON-NEXT:      ]
+// JSON-NEXT:     },
+// JSON-NEXT:     {
+// JSON-NEXT:      "id": "0x{{.*}}",
+// JSON-NEXT:      "kind": "ClassTemplateSpecializationDecl",
+// JSON-NEXT:      "loc": {
+// JSON-NEXT:       "offset": 3281,
+// JSON-NEXT:       "line": 102,
+// JSON-NEXT:       "col": 17,
+// JSON-NEXT:       "tokLen": 3
+// JSON-NEXT:      },
+// JSON-NEXT:      "range": {
+// JSON-NEXT:       "begin": {
+// JSON-NEXT:        "offset": 3265,
+// JSON-NEXT:        "col": 1,
+// JSON-NEXT:        "tokLen": 8
+// JSON-NEXT:       },
+// JSON-NEXT:       "end": {
+// JSON-NEXT:        "offset": 3294,
+// JSON-NEXT:        "col": 30,
+// JSON-NEXT:        "tokLen": 1
+// JSON-NEXT:       }
+// JSON-NEXT:      },
+// JSON-NEXT:      "name": "foo",
+// JSON-NEXT:      "tagUsed": "struct",
+// JSON-NEXT:      "completeDefinition": true,
+// JSON-NEXT:      "definitionData": {
+// JSON-NEXT:       "canConstDefaultInit": true,
+// JSON-NEXT:       "canPassInRegisters": true,
+// JSON-NEXT:       "copyAssign": {
+// JSON-NEXT:        "hasConstParam": true,
+// JSON-NEXT:        "implicitHasConstParam": true,
+// JSON-NEXT:        "needsImplicit": true,
+// JSON-NEXT:        "simple": true,
+// JSON-NEXT:        "trivial": true
+// JSON-NEXT:       },
+// JSON-NEXT:       "copyCtor": {
+// JSON-NEXT:        "hasConstParam": true,
+// JSON-NEXT:        "implicitHasConstParam": true,
+// JSON-NEXT:        "needsImplicit": true,
+// JSON-NEXT:        "simple": true,
+// JSON-NEXT:        "trivial": true
+// JSON-NEXT:       },
+// JSON-NEXT:       "defaultCtor": {
+// JSON-NEXT:        "defaultedIsConstexpr": true,
+// JSON-NEXT:        "exists": true,
+// JSON-NEXT:        "isConstexpr": true,
+// JSON-NEXT:        "needsImplicit": true,
+// JSON-NEXT:        "trivial": true
+// JSON-NEXT:       },
+// JSON-NEXT:       "dtor": {
+// JSON-NEXT:        "irrelevant": true,
+// JSON-NEXT:        "needsImplicit": true,
+// JSON-NEXT:        "simple": true,
+// JSON-NEXT:        "trivial": true
+// JSON-NEXT:       },
+// JSON-NEXT:       "hasConstexprNonCopyMoveConstructor": true,
+// JSON-NEXT:       "isAggregate": true,
+// JSON-NEXT:       "isEmpty": true,
+// JSON-NEXT:       "isLiteral": true,
+// JSON-NEXT:       "isPOD": true,
+// JSON-NEXT:       "isStandardLayout": true,
+// JSON-NEXT:       "isTrivial": true,
+// JSON-NEXT:       "isTriviallyCopyable": true,
+// JSON-NEXT:       "moveAssign": {
+// JSON-NEXT:        "exists": true,
+// JSON-NEXT:        "needsImplicit": true,
+// JSON-NEXT:        "simple": true,
+// JSON-NEXT:        "trivial": true
+// JSON-NEXT:       },
+// JSON-NEXT:       "moveCtor": {
+// JSON-NEXT:        "exists": true,
+// JSON-NEXT:        "needsImplicit": true,
+// JSON-NEXT:        "simple": true,
+// JSON-NEXT:        "trivial": true
+// JSON-NEXT:       }
+// JSON-NEXT:      },
+// JSON-NEXT:      "inner": [
+// JSON-NEXT:       {
+// JSON-NEXT:        "kind": "TemplateArgument",
+// JSON-NEXT:        "value": 1
+// JSON-NEXT:       },
+// JSON-NEXT:       {
+// JSON-NEXT:        "kind": "TemplateArgument",
+// JSON-NEXT:        "value": 0
+// JSON-NEXT:       },
+// JSON-NEXT:       {
+// JSON-NEXT:        "id": "0x{{.*}}",
+// JSON-NEXT:        "kind": "CXXRecordDecl",
+// JSON-NEXT:        "loc": {
+// JSON-NEXT:         "offset": 2445,
+// JSON-NEXT:         "line": 81,
+// JSON-NEXT:         "col": 8,
+// JSON-NEXT:         "tokLen": 3
+// JSON-NEXT:        },
+// JSON-NEXT:        "range": {
+// JSON-NEXT:         "begin": {
+// JSON-NEXT:          "offset": 2438,
+// JSON-NEXT:          "col": 1,
+// JSON-NEXT:          "tokLen": 6
+// JSON-NEXT:         },
+// JSON-NEXT:         "end": {
+// JSON-NEXT:          "offset": 2445,
+// JSON-NEXT:          "col": 8,
+// JSON-NEXT:          "tokLen": 3
+// JSON-NEXT:         }
+// JSON-NEXT:        },
+// JSON-NEXT:        "isImplicit": true,
+// JSON-NEXT:        "name": "foo",
+// JSON-NEXT:        "tagUsed": "struct"
+// JSON-NEXT:       },
+// JSON-NEXT:       {
+// JSON-NEXT:        "id": "0x{{.*}}",
+// JSON-NEXT:        "kind": "CXXMethodDecl",
+// JSON-NEXT:        "loc": {
+// JSON-NEXT:         "offset": 2465,
+// JSON-NEXT:         "line": 82,
+// JSON-NEXT:         "col": 15,
+// JSON-NEXT:         "tokLen": 2
+// JSON-NEXT:        },
+// JSON-NEXT:        "range": {
+// JSON-NEXT:         "begin": {
+// JSON-NEXT:          "offset": 2453,
+// JSON-NEXT:          "col": 3,
+// JSON-NEXT:          "tokLen": 6
+// JSON-NEXT:         },
+// JSON-NEXT:         "end": {
+// JSON-NEXT:          "offset": 2468,
+// JSON-NEXT:          "col": 18,
+// JSON-NEXT:          "tokLen": 1
+// JSON-NEXT:         }
+// JSON-NEXT:        },
+// JSON-NEXT:        "name": "fn",
+// JSON-NEXT:        "mangledName": "_ZN5test43fooILj1ELl0EE2fnEv",
+// JSON-NEXT:        "type": {
+// JSON-NEXT:         "qualType": "void ()"
+// JSON-NEXT:        },
+// JSON-NEXT:        "storageClass": "static"
+// JSON-NEXT:       }
+// JSON-NEXT:      ]
+// JSON-NEXT:     }
+// JSON-NEXT:    ]
+// JSON-NEXT:   },
+// JSON-NEXT:   {
+// JSON-NEXT:    "id": "0x{{.*}}",
+// JSON-NEXT:    "kind": "NamespaceDecl",
+// JSON-NEXT:    "loc": {
+// JSON-NEXT:     "offset": 3310,
+// JSON-NEXT:     "line": 105,
+// JSON-NEXT:     "col": 11,
+// JSON-NEXT:     "tokLen": 5
+// JSON-NEXT:    },
+// JSON-NEXT:    "range": {
+// JSON-NEXT:     "begin": {
+// JSON-NEXT:      "offset": 3300,
+// JSON-NEXT:      "col": 1,
+// JSON-NEXT:      "tokLen": 9
+// JSON-NEXT:     },
+// JSON-NEXT:     "end": {
+// JSON-NEXT:      "offset": 3632,
+// JSON-NEXT:      "line": 114,
+// JSON-NEXT:      "col": 1,
+// JSON-NEXT:      "tokLen": 1
+// JSON-NEXT:     }
+// JSON-NEXT:    },
+// JSON-NEXT:    "name": "test5",
+// JSON-NEXT:    "inner": [
+// JSON-NEXT:     {
+// JSON-NEXT:      "id": "0x{{.*}}",
+// JSON-NEXT:      "kind": "FunctionTemplateDecl",
+// JSON-NEXT:      "loc": {
+// JSON-NEXT:       "offset": 3338,
+// JSON-NEXT:       "line": 106,
+// JSON-NEXT:       "col": 21,
+// JSON-NEXT:       "tokLen": 1
+// JSON-NEXT:      },
+// JSON-NEXT:      "range": {
+// JSON-NEXT:       "begin": {
+// JSON-NEXT:        "offset": 3318,
+// JSON-NEXT:        "col": 1,
+// JSON-NEXT:        "tokLen": 8
+// JSON-NEXT:       },
+// JSON-NEXT:       "end": {
+// JSON-NEXT:        "offset": 3343,
+// JSON-NEXT:        "col": 26,
+// JSON-NEXT:        "tokLen": 1
+// JSON-NEXT:       }
+// JSON-NEXT:      },
+// JSON-NEXT:      "name": "f",
+// JSON-NEXT:      "inner": [
+// JSON-NEXT:       {
+// JSON-NEXT:        "id": "0x{{.*}}",
+// JSON-NEXT:        "kind": "NonTypeTemplateParmDecl",
+// JSON-NEXT:        "loc": {
+// JSON-NEXT:         "offset": 3331,
+// JSON-NEXT:         "col": 14,
+// JSON-NEXT:         "tokLen": 1
+// JSON-NEXT:        },
+// JSON-NEXT:        "range": {
+// JSON-NEXT:         "begin": {
+// JSON-NEXT:          "offset": 3327,
+// JSON-NEXT:          "col": 10,
+// JSON-NEXT:          "tokLen": 4
+// JSON-NEXT:         },
+// JSON-NEXT:         "end": {
+// JSON-NEXT:          "offset": 3327,
+// JSON-NEXT:          "col": 10,
+// JSON-NEXT:          "tokLen": 4
+// JSON-NEXT:         }
+// JSON-NEXT:        },
+// JSON-NEXT:        "type": {
+// JSON-NEXT:         "qualType": "long"
+// JSON-NEXT:        },
+// JSON-NEXT:        "depth": 0,
+// JSON-NEXT:        "index": 0
+// JSON-NEXT:       },
+// JSON-NEXT:       {
+// JSON-NEXT:        "id": "0x{{.*}}",
+// JSON-NEXT:        "kind": "FunctionDecl",
+// JSON-NEXT:        "loc": {
+// JSON-NEXT:         "offset": 3338,
+// JSON-NEXT:         "col": 21,
+// JSON-NEXT:         "tokLen": 1
+// JSON-NEXT:        },
+// JSON-NEXT:        "range": {
+// JSON-NEXT:         "begin": {
+// JSON-NEXT:          "offset": 3333,
+// JSON-NEXT:          "col": 16,
+// JSON-NEXT:          "tokLen": 4
+// JSON-NEXT:         },
+// JSON-NEXT:         "end": {
+// JSON-NEXT:          "offset": 3343,
+// JSON-NEXT:          "col": 26,
+// JSON-NEXT:          "tokLen": 1
+// JSON-NEXT:         }
+// JSON-NEXT:        },
+// JSON-NEXT:        "name": "f",
+// JSON-NEXT:        "type": {
+// JSON-NEXT:         "qualType": "void ()"
+// JSON-NEXT:        },
+// JSON-NEXT:        "inner": [
+// JSON-NEXT:         {
+// JSON-NEXT:          "id": "0x{{.*}}",
+// JSON-NEXT:          "kind": "CompoundStmt",
+// JSON-NEXT:          "range": {
+// JSON-NEXT:           "begin": {
+// JSON-NEXT:            "offset": 3342,
+// JSON-NEXT:            "col": 25,
+// JSON-NEXT:            "tokLen": 1
+// JSON-NEXT:           },
+// JSON-NEXT:           "end": {
+// JSON-NEXT:            "offset": 3343,
+// JSON-NEXT:            "col": 26,
+// JSON-NEXT:            "tokLen": 1
+// JSON-NEXT:           }
+// JSON-NEXT:          }
+// JSON-NEXT:         }
+// JSON-NEXT:        ]
+// JSON-NEXT:       },
+// JSON-NEXT:       {
+// JSON-NEXT:        "id": "0x{{.*}}",
+// JSON-NEXT:        "kind": "FunctionDecl",
+// JSON-NEXT:        "loc": {
+// JSON-NEXT:         "offset": 3338,
+// JSON-NEXT:         "col": 21,
+// JSON-NEXT:         "tokLen": 1
+// JSON-NEXT:        },
+// JSON-NEXT:        "range": {
+// JSON-NEXT:         "begin": {
+// JSON-NEXT:          "offset": 3333,
+// JSON-NEXT:          "col": 16,
+// JSON-NEXT:          "tokLen": 4
+// JSON-NEXT:         },
+// JSON-NEXT:         "end": {
+// JSON-NEXT:          "offset": 3343,
+// JSON-NEXT:          "col": 26,
+// JSON-NEXT:          "tokLen": 1
+// JSON-NEXT:         }
+// JSON-NEXT:        },
+// JSON-NEXT:        "isUsed": true,
+// JSON-NEXT:        "name": "f",
+// JSON-NEXT:        "mangledName": "_ZN5test51fILl0EEEvv",
+// JSON-NEXT:        "type": {
+// JSON-NEXT:         "qualType": "void ()"
+// JSON-NEXT:        },
+// JSON-NEXT:        "inner": [
+// JSON-NEXT:         {
+// JSON-NEXT:          "kind": "TemplateArgument",
+// JSON-NEXT:          "value": 0
+// JSON-NEXT:         },
+// JSON-NEXT:         {
+// JSON-NEXT:          "id": "0x{{.*}}",
+// JSON-NEXT:          "kind": "CompoundStmt",
+// JSON-NEXT:          "range": {
+// JSON-NEXT:           "begin": {
+// JSON-NEXT:            "offset": 3342,
+// JSON-NEXT:            "col": 25,
+// JSON-NEXT:            "tokLen": 1
+// JSON-NEXT:           },
+// JSON-NEXT:           "end": {
+// JSON-NEXT:            "offset": 3343,
+// JSON-NEXT:            "col": 26,
+// JSON-NEXT:            "tokLen": 1
+// JSON-NEXT:           }
+// JSON-NEXT:          }
+// JSON-NEXT:         }
+// JSON-NEXT:        ]
+// JSON-NEXT:       }
+// JSON-NEXT:      ]
+// JSON-NEXT:     },
+// JSON-NEXT:     {
+// JSON-NEXT:      "id": "0x{{.*}}",
+// JSON-NEXT:      "kind": "VarDecl",
+// JSON-NEXT:      "loc": {
+// JSON-NEXT:       "offset": 3352,
+// JSON-NEXT:       "line": 107,
+// JSON-NEXT:       "col": 8,
+// JSON-NEXT:       "tokLen": 1
+// JSON-NEXT:      },
+// JSON-NEXT:      "range": {
+// JSON-NEXT:       "begin": {
+// JSON-NEXT:        "offset": 3345,
+// JSON-NEXT:        "col": 1,
+// JSON-NEXT:        "tokLen": 4
+// JSON-NEXT:       },
+// JSON-NEXT:       "end": {
+// JSON-NEXT:        "offset": 3362,
+// JSON-NEXT:        "col": 18,
+// JSON-NEXT:        "tokLen": 1
+// JSON-NEXT:       }
+// JSON-NEXT:      },
+// JSON-NEXT:      "name": "p",
+// JSON-NEXT:      "mangledName": "_ZN5test51pE",
+// JSON-NEXT:      "type": {
+// JSON-NEXT:       "qualType": "void (*)()"
+// JSON-NEXT:      },
+// JSON-NEXT:      "init": "c",
+// JSON-NEXT:      "inner": [
+// JSON-NEXT:       {
+// JSON-NEXT:        "id": "0x{{.*}}",
+// JSON-NEXT:        "kind": "ImplicitCastExpr",
+// JSON-NEXT:        "range": {
+// JSON-NEXT:         "begin": {
+// JSON-NEXT:          "offset": 3359,
+// JSON-NEXT:          "col": 15,
+// JSON-NEXT:          "tokLen": 1
+// JSON-NEXT:         },
+// JSON-NEXT:         "end": {
+// JSON-NEXT:          "offset": 3362,
+// JSON-NEXT:          "col": 18,
+// JSON-NEXT:          "tokLen": 1
+// JSON-NEXT:         }
+// JSON-NEXT:        },
+// JSON-NEXT:        "type": {
+// JSON-NEXT:         "qualType": "void (*)()"
+// JSON-NEXT:        },
+// JSON-NEXT:        "valueCategory": "prvalue",
+// JSON-NEXT:        "castKind": "FunctionToPointerDecay",
+// JSON-NEXT:        "inner": [
+// JSON-NEXT:         {
+// JSON-NEXT:          "id": "0x{{.*}}",
+// JSON-NEXT:          "kind": "DeclRefExpr",
+// JSON-NEXT:          "range": {
+// JSON-NEXT:           "begin": {
+// JSON-NEXT:            "offset": 3359,
+// JSON-NEXT:            "col": 15,
+// JSON-NEXT:            "tokLen": 1
+// JSON-NEXT:           },
+// JSON-NEXT:           "end": {
+// JSON-NEXT:            "offset": 3362,
+// JSON-NEXT:            "col": 18,
+// JSON-NEXT:            "tokLen": 1
+// JSON-NEXT:           }
+// JSON-NEXT:          },
+// JSON-NEXT:          "type": {
+// JSON-NEXT:           "qualType": "void ()"
+// JSON-NEXT:          },
+// JSON-NEXT:          "valueCategory": "lvalue",
+// JSON-NEXT:          "referencedDecl": {
+// JSON-NEXT:           "id": "0x{{.*}}",
+// JSON-NEXT:           "kind": "FunctionDecl",
+// JSON-NEXT:           "name": "f",
+// JSON-NEXT:           "type": {
+// JSON-NEXT:            "qualType": "void ()"
+// JSON-NEXT:           }
+// JSON-NEXT:          },
+// JSON-NEXT:          "foundReferencedDecl": {
+// JSON-NEXT:           "id": "0x{{.*}}",
+// JSON-NEXT:           "kind": "FunctionTemplateDecl",
+// JSON-NEXT:           "name": "f"
+// JSON-NEXT:          }
+// JSON-NEXT:         }
+// JSON-NEXT:        ]
+// JSON-NEXT:       }
+// JSON-NEXT:      ]
+// JSON-NEXT:     },
+// JSON-NEXT:     {
+// JSON-NEXT:      "id": "0x{{.*}}",
+// JSON-NEXT:      "kind": "FunctionTemplateDecl",
+// JSON-NEXT:      "loc": {
+// JSON-NEXT:       "offset": 3393,
+// JSON-NEXT:       "line": 108,
+// JSON-NEXT:       "col": 29,
+// JSON-NEXT:       "tokLen": 1
+// JSON-NEXT:      },
+// JSON-NEXT:      "range": {
+// JSON-NEXT:       "begin": {
+// JSON-NEXT:        "offset": 3365,
+// JSON-NEXT:        "col": 1,
+// JSON-NEXT:        "tokLen": 8
+// JSON-NEXT:       },
+// JSON-NEXT:       "end": {
+// JSON-NEXT:        "offset": 3398,
+// JSON-NEXT:        "col": 34,
+// JSON-NEXT:        "tokLen": 1
+// JSON-NEXT:       }
+// JSON-NEXT:      },
+// JSON-NEXT:      "name": "f",
+// JSON-NEXT:      "inner": [
+// JSON-NEXT:       {
+// JSON-NEXT:        "id": "0x{{.*}}",
+// JSON-NEXT:        "kind": "NonTypeTemplateParmDecl",
+// JSON-NEXT:        "loc": {
+// JSON-NEXT:         "offset": 3383,
+// JSON-NEXT:         "col": 19,
+// JSON-NEXT:         "tokLen": 1
+// JSON-NEXT:        },
+// JSON-NEXT:        "range": {
+// JSON-NEXT:         "begin": {
+// JSON-NEXT:          "offset": 3374,
+// JSON-NEXT:          "col": 10,
+// JSON-NEXT:          "tokLen": 8
+// JSON-NEXT:         },
+// JSON-NEXT:         "end": {
+// JSON-NEXT:          "offset": 3385,
+// JSON-NEXT:          "col": 21,
+// JSON-NEXT:          "tokLen": 1
+// JSON-NEXT:         }
+// JSON-NEXT:        },
+// JSON-NEXT:        "type": {
+// JSON-NEXT:         "qualType": "unsigned int"
+// JSON-NEXT:        },
+// JSON-NEXT:        "depth": 0,
+// JSON-NEXT:        "index": 0,
+// JSON-NEXT:        "defaultArg": {
+// JSON-NEXT:         "kind": "TemplateArgument",
+// JSON-NEXT:         "isExpr": true
+// JSON-NEXT:        },
+// JSON-NEXT:        "inner": [
+// JSON-NEXT:         {
+// JSON-NEXT:          "kind": "TemplateArgument",
+// JSON-NEXT:          "range": {
+// JSON-NEXT:           "begin": {
+// JSON-NEXT:            "offset": 3385,
+// JSON-NEXT:            "col": 21,
+// JSON-NEXT:            "tokLen": 1
+// JSON-NEXT:           },
+// JSON-NEXT:           "end": {
+// JSON-NEXT:            "offset": 3385,
+// JSON-NEXT:            "col": 21,
+// JSON-NEXT:            "tokLen": 1
+// JSON-NEXT:           }
+// JSON-NEXT:          },
+// JSON-NEXT:          "isExpr": true,
+// JSON-NEXT:          "inner": [
+// JSON-NEXT:           {
+// JSON-NEXT:            "id": "0x{{.*}}",
+// JSON-NEXT:            "kind": "IntegerLiteral",
+// JSON-NEXT:            "range": {
+// JSON-NEXT:             "begin": {
+// JSON-NEXT:              "offset": 3385,
+// JSON-NEXT:              "col": 21,
+// JSON-NEXT:              "tokLen": 1
+// JSON-NEXT:             },
+// JSON-NEXT:             "end": {
+// JSON-NEXT:              "offset": 3385,
+// JSON-NEXT:              "col": 21,
+// JSON-NEXT:              "tokLen": 1
+// JSON-NEXT:             }
+// JSON-NEXT:            },
+// JSON-NEXT:            "type": {
+// JSON-NEXT:             "qualType": "int"
+// JSON-NEXT:            },
+// JSON-NEXT:            "valueCategory": "prvalue",
+// JSON-NEXT:            "value": "0"
+// JSON-NEXT:           }
+// JSON-NEXT:          ]
+// JSON-NEXT:         }
+// JSON-NEXT:        ]
+// JSON-NEXT:       },
+// JSON-NEXT:       {
+// JSON-NEXT:        "id": "0x{{.*}}",
+// JSON-NEXT:        "kind": "FunctionDecl",
+// JSON-NEXT:        "loc": {
+// JSON-NEXT:         "offset": 3393,
+// JSON-NEXT:         "col": 29,
+// JSON-NEXT:         "tokLen": 1
+// JSON-NEXT:        },
+// JSON-NEXT:        "range": {
+// JSON-NEXT:         "begin": {
+// JSON-NEXT:          "offset": 3388,
+// JSON-NEXT:          "col": 24,
+// JSON-NEXT:          "tokLen": 4
+// JSON-NEXT:         },
+// JSON-NEXT:         "end": {
+// JSON-NEXT:          "offset": 3398,
+// JSON-NEXT:          "col": 34,
+// JSON-NEXT:          "tokLen": 1
+// JSON-NEXT:         }
+// JSON-NEXT:        },
+// JSON-NEXT:        "name": "f",
+// JSON-NEXT:        "type": {
+// JSON-NEXT:         "qualType": "void ()"
+// JSON-NEXT:        },
+// JSON-NEXT:        "inner": [
+// JSON-NEXT:         {
+// JSON-NEXT:          "id": "0x{{.*}}",
+// JSON-NEXT:          "kind": "CompoundStmt",
+// JSON-NEXT:          "range": {
+// JSON-NEXT:           "begin": {
+// JSON-NEXT:            "offset": 3397,
+// JSON-NEXT:            "col": 33,
+// JSON-NEXT:            "tokLen": 1
+// JSON-NEXT:           },
+// JSON-NEXT:           "end": {
+// JSON-NEXT:            "offset": 3398,
+// JSON-NEXT:            "col": 34,
+// JSON-NEXT:            "tokLen": 1
+// JSON-NEXT:           }
+// JSON-NEXT:          }
+// JSON-NEXT:         }
+// JSON-NEXT:        ]
+// JSON-NEXT:       },
+// JSON-NEXT:       {
+// JSON-NEXT:        "id": "0x{{.*}}",
+// JSON-NEXT:        "kind": "FunctionDecl",
+// JSON-NEXT:        "loc": {
+// JSON-NEXT:         "offset": 3393,
+// JSON-NEXT:         "col": 29,
+// JSON-NEXT:         "tokLen": 1
+// JSON-NEXT:        },
+// JSON-NEXT:        "range": {
+// JSON-NEXT:         "begin": {
+// JSON-NEXT:          "offset": 3388,
+// JSON-NEXT:          "col": 24,
+// JSON-NEXT:          "tokLen": 4
+// JSON-NEXT:         },
+// JSON-NEXT:         "end": {
+// JSON-NEXT:          "offset": 3398,
+// JSON-NEXT:          "col": 34,
+// JSON-NEXT:          "tokLen": 1
+// JSON-NEXT:         }
+// JSON-NEXT:        },
+// JSON-NEXT:        "isUsed": true,
+// JSON-NEXT:        "name": "f",
+// JSON-NEXT:        "mangledName": "_ZN5test51fILj0EEEvv",
+// JSON-NEXT:        "type": {
+// JSON-NEXT:         "qualType": "void ()"
+// JSON-NEXT:        },
+// JSON-NEXT:        "inner": [
+// JSON-NEXT:         {
+// JSON-NEXT:          "kind": "TemplateArgument",
+// JSON-NEXT:          "value": 0
+// JSON-NEXT:         },
+// JSON-NEXT:         {
+// JSON-NEXT:          "id": "0x{{.*}}",
+// JSON-NEXT:          "kind": "CompoundStmt",
+// JSON-NEXT:          "range": {
+// JSON-NEXT:           "begin": {
+// JSON-NEXT:            "offset": 3397,
+// JSON-NEXT:            "col": 33,
+// JSON-NEXT:            "tokLen": 1
+// JSON-NEXT:           },
+// JSON-NEXT:           "end": {
+// JSON-NEXT:            "offset": 3398,
+// JSON-NEXT:            "col": 34,
+// JSON-NEXT:            "tokLen": 1
+// JSON-NEXT:           }
+// JSON-NEXT:          }
+// JSON-NEXT:         }
+// JSON-NEXT:        ]
+// JSON-NEXT:       }
+// JSON-NEXT:      ]
+// JSON-NEXT:     },
+// JSON-NEXT:     {
+// JSON-NEXT:      "id": "0x{{.*}}",
+// JSON-NEXT:      "kind": "VarDecl",
+// JSON-NEXT:      "loc": {
+// JSON-NEXT:       "offset": 3407,
+// JSON-NEXT:       "line": 109,
+// JSON-NEXT:       "col": 8,
+// JSON-NEXT:       "tokLen": 1
+// JSON-NEXT:      },
+// JSON-NEXT:      "range": {
+// JSON-NEXT:       "begin": {
+// JSON-NEXT:        "offset": 3400,
+// JSON-NEXT:        "col": 1,
+// JSON-NEXT:        "tokLen": 4
+// JSON-NEXT:       },
+// JSON-NEXT:       "end": {
+// JSON-NEXT:        "offset": 3416,
+// JSON-NEXT:        "col": 17,
+// JSON-NEXT:        "tokLen": 1
+// JSON-NEXT:       }
+// JSON-NEXT:      },
+// JSON-NEXT:      "name": "q",
+// JSON-NEXT:      "mangledName": "_ZN5test51qE",
+// JSON-NEXT:      "type": {
+// JSON-NEXT:       "qualType": "void (*)()"
+// JSON-NEXT:      },
+// JSON-NEXT:      "init": "c",
+// JSON-NEXT:      "inner": [
+// JSON-NEXT:       {
+// JSON-NEXT:        "id": "0x{{.*}}",
+// JSON-NEXT:        "kind": "ImplicitCastExpr",
+// JSON-NEXT:        "range": {
+// JSON-NEXT:         "begin": {
+// JSON-NEXT:          "offset": 3414,
+// JSON-NEXT:          "col": 15,
+// JSON-NEXT:          "tokLen": 1
+// JSON-NEXT:         },
+// JSON-NEXT:         "end": {
+// JSON-NEXT:          "offset": 3416,
+// JSON-NEXT:          "col": 17,
+// JSON-NEXT:          "tokLen": 1
+// JSON-NEXT:         }
+// JSON-NEXT:        },
+// JSON-NEXT:        "type": {
+// JSON-NEXT:         "qualType": "void (*)()"
+// JSON-NEXT:        },
+// JSON-NEXT:        "valueCategory": "prvalue",
+// JSON-NEXT:        "castKind": "FunctionToPointerDecay",
+// JSON-NEXT:        "inner": [
+// JSON-NEXT:         {
+// JSON-NEXT:          "id": "0x{{.*}}",
+// JSON-NEXT:          "kind": "DeclRefExpr",
+// JSON-NEXT:          "range": {
+// JSON-NEXT:           "begin": {
+// JSON-NEXT:            "offset": 3414,
+// JSON-NEXT:            "col": 15,
+// JSON-NEXT:            "tokLen": 1
+// JSON-NEXT:           },
+// JSON-NEXT:           "end": {
+// JSON-NEXT:            "offset": 3416,
+// JSON-NEXT:            "col": 17,
+// JSON-NEXT:            "tokLen": 1
+// JSON-NEXT:           }
+// JSON-NEXT:          },
+// JSON-NEXT:          "type": {
+// JSON-NEXT:           "qualType": "void ()"
+// JSON-NEXT:          },
+// JSON-NEXT:          "valueCategory": "lvalue",
+// JSON-NEXT:          "referencedDecl": {
+// JSON-NEXT:           "id": "0x{{.*}}",
+// JSON-NEXT:           "kind": "FunctionDecl",
+// JSON-NEXT:           "name": "f",
+// JSON-NEXT:           "type": {
+// JSON-NEXT:            "qualType": "void ()"
+// JSON-NEXT:           }
+// JSON-NEXT:          },
+// JSON-NEXT:          "foundReferencedDecl": {
+// JSON-NEXT:           "id": "0x{{.*}}",
+// JSON-NEXT:           "kind": "FunctionTemplateDecl",
+// JSON-NEXT:           "name": "f"
+// JSON-NEXT:          }
+// JSON-NEXT:         }
+// JSON-NEXT:        ]
+// JSON-NEXT:       }
+// JSON-NEXT:      ]
+// JSON-NEXT:     }
+// JSON-NEXT:    ]
+// JSON-NEXT:   },
+// JSON-NEXT:   {
+// JSON-NEXT:    "id": "0x{{.*}}",
+// JSON-NEXT:    "kind": "NamespaceDecl",
+// JSON-NEXT:    "loc": {
+// JSON-NEXT:     "offset": 3645,
+// JSON-NEXT:     "line": 116,
+// JSON-NEXT:     "col": 11,
+// JSON-NEXT:     "tokLen": 5
+// JSON-NEXT:    },
+// JSON-NEXT:    "range": {
+// JSON-NEXT:     "begin": {
+// JSON-NEXT:      "offset": 3635,
+// JSON-NEXT:      "col": 1,
+// JSON-NEXT:      "tokLen": 9
+// JSON-NEXT:     },
+// JSON-NEXT:     "end": {
+// JSON-NEXT:      "offset": 4000,
+// JSON-NEXT:      "line": 128,
+// JSON-NEXT:      "col": 1,
+// JSON-NEXT:      "tokLen": 1
+// JSON-NEXT:     }
+// JSON-NEXT:    },
+// JSON-NEXT:    "name": "test6",
+// JSON-NEXT:    "inner": [
+// JSON-NEXT:     {
+// JSON-NEXT:      "id": "0x{{.*}}",
+// JSON-NEXT:      "kind": "VarTemplateDecl",
+// JSON-NEXT:      "loc": {
+// JSON-NEXT:       "offset": 3687,
+// JSON-NEXT:       "line": 118,
+// JSON-NEXT:       "col": 16,
+// JSON-NEXT:       "tokLen": 1
+// JSON-NEXT:      },
+// JSON-NEXT:      "range": {
+// JSON-NEXT:       "begin": {
+// JSON-NEXT:        "offset": 3653,
+// JSON-NEXT:        "line": 117,
+// JSON-NEXT:        "col": 1,
+// JSON-NEXT:        "tokLen": 8
+// JSON-NEXT:       },
+// JSON-NEXT:       "end": {
+// JSON-NEXT:        "offset": 3691,
+// JSON-NEXT:        "line": 118,
+// JSON-NEXT:        "col": 20,
+// JSON-NEXT:        "tokLen": 4
+// JSON-NEXT:       }
+// JSON-NEXT:      },
+// JSON-NEXT:      "name": "C",
+// JSON-NEXT:      "inner": [
+// JSON-NEXT:       {
+// JSON-NEXT:        "id": "0x{{.*}}",
+// JSON-NEXT:        "kind": "TemplateTypeParmDecl",
+// JSON-NEXT:        "loc": {
+// JSON-NEXT:         "offset": 3669,
+// JSON-NEXT:         "line": 117,
+// JSON-NEXT:         "col": 17,
+// JSON-NEXT:         "tokLen": 1
+// JSON-NEXT:        },
+// JSON-NEXT:        "range": {
+// JSON-NEXT:         "begin": {
+// JSON-NEXT:          "offset": 3663,
+// JSON-NEXT:          "col": 11,
+// JSON-NEXT:          "tokLen": 5
+// JSON-NEXT:         },
+// JSON-NEXT:         "end": {
+// JSON-NEXT:          "offset": 3669,
+// JSON-NEXT:          "col": 17,
+// JSON-NEXT:          "tokLen": 1
+// JSON-NEXT:         }
+// JSON-NEXT:        },
+// JSON-NEXT:        "name": "D",
+// JSON-NEXT:        "tagUsed": "class",
+// JSON-NEXT:        "depth": 0,
+// JSON-NEXT:        "index": 0
+// JSON-NEXT:       },
+// JSON-NEXT:       {
+// JSON-NEXT:        "id": "0x{{.*}}",
+// JSON-NEXT:        "kind": "VarDecl",
+// JSON-NEXT:        "loc": {
+// JSON-NEXT:         "offset": 3687,
+// JSON-NEXT:         "line": 118,
+// JSON-NEXT:         "col": 16,
+// JSON-NEXT:         "tokLen": 1
+// JSON-NEXT:        },
+// JSON-NEXT:        "range": {
+// JSON-NEXT:         "begin": {
+// JSON-NEXT:          "offset": 3672,
+// JSON-NEXT:          "col": 1,
+// JSON-NEXT:          "tokLen": 9
+// JSON-NEXT:         },
+// JSON-NEXT:         "end": {
+// JSON-NEXT:          "offset": 3691,
+// JSON-NEXT:          "col": 20,
+// JSON-NEXT:          "tokLen": 4
+// JSON-NEXT:         }
+// JSON-NEXT:        },
+// JSON-NEXT:        "name": "C",
+// JSON-NEXT:        "type": {
+// JSON-NEXT:         "qualType": "const bool"
+// JSON-NEXT:        },
+// JSON-NEXT:        "constexpr": true,
+// JSON-NEXT:        "init": "c",
+// JSON-NEXT:        "inner": [
+// JSON-NEXT:         {
+// JSON-NEXT:          "id": "0x{{.*}}",
+// JSON-NEXT:          "kind": "CXXBoolLiteralExpr",
+// JSON-NEXT:          "range": {
+// JSON-NEXT:           "begin": {
+// JSON-NEXT:            "offset": 3691,
+// JSON-NEXT:            "col": 20,
+// JSON-NEXT:            "tokLen": 4
+// JSON-NEXT:           },
+// JSON-NEXT:           "end": {
+// JSON-NEXT:            "offset": 3691,
+// JSON-NEXT:            "col": 20,
+// JSON-NEXT:            "tokLen": 4
+// JSON-NEXT:           }
+// JSON-NEXT:          },
+// JSON-NEXT:          "type": {
+// JSON-NEXT:           "qualType": "bool"
+// JSON-NEXT:          },
+// JSON-NEXT:          "valueCategory": "prvalue",
+// JSON-NEXT:          "value": true
+// JSON-NEXT:         }
+// JSON-NEXT:        ]
+// JSON-NEXT:       }
+// JSON-NEXT:      ]
+// JSON-NEXT:     },
+// JSON-NEXT:     {
+// JSON-NEXT:      "id": "0x{{.*}}",
+// JSON-NEXT:      "kind": "FunctionTemplateDecl",
+// JSON-NEXT:      "loc": {
+// JSON-NEXT:       "offset": 3724,
+// JSON-NEXT:       "line": 121,
+// JSON-NEXT:       "col": 6,
+// JSON-NEXT:       "tokLen": 4
+// JSON-NEXT:      },
+// JSON-NEXT:      "range": {
+// JSON-NEXT:       "begin": {
+// JSON-NEXT:        "offset": 3698,
+// JSON-NEXT:        "line": 120,
+// JSON-NEXT:        "col": 1,
+// JSON-NEXT:        "tokLen": 8
+// JSON-NEXT:       },
+// JSON-NEXT:       "end": {
+// JSON-NEXT:        "offset": 3998,
+// JSON-NEXT:        "line": 127,
+// JSON-NEXT:        "col": 1,
+// JSON-NEXT:        "tokLen": 1
+// JSON-NEXT:       }
+// JSON-NEXT:      },
+// JSON-NEXT:      "name": "func",
+// JSON-NEXT:      "inner": [
+// JSON-NEXT:       {
+// JSON-NEXT:        "id": "0x{{.*}}",
+// JSON-NEXT:        "kind": "TemplateTypeParmDecl",
+// JSON-NEXT:        "loc": {
+// JSON-NEXT:         "offset": 3714,
+// JSON-NEXT:         "line": 120,
+// JSON-NEXT:         "col": 17,
+// JSON-NEXT:         "tokLen": 3
+// JSON-NEXT:        },
+// JSON-NEXT:        "range": {
+// JSON-NEXT:         "begin": {
+// JSON-NEXT:          "offset": 3708,
+// JSON-NEXT:          "col": 11,
+// JSON-NEXT:          "tokLen": 5
+// JSON-NEXT:         },
+// JSON-NEXT:         "end": {
+// JSON-NEXT:          "offset": 3714,
+// JSON-NEXT:          "col": 17,
+// JSON-NEXT:          "tokLen": 3
+// JSON-NEXT:         }
+// JSON-NEXT:        },
+// JSON-NEXT:        "isReferenced": true,
+// JSON-NEXT:        "name": "Key",
+// JSON-NEXT:        "tagUsed": "class",
+// JSON-NEXT:        "depth": 0,
+// JSON-NEXT:        "index": 0
+// JSON-NEXT:       },
+// JSON-NEXT:       {
+// JSON-NEXT:        "id": "0x{{.*}}",
+// JSON-NEXT:        "kind": "FunctionDecl",
+// JSON-NEXT:        "loc": {
+// JSON-NEXT:         "offset": 3724,
+// JSON-NEXT:         "line": 121,
+// JSON-NEXT:         "col": 6,
+// JSON-NEXT:         "tokLen": 4
+// JSON-NEXT:        },
+// JSON-NEXT:        "range": {
+// JSON-NEXT:         "begin": {
+// JSON-NEXT:          "offset": 3719,
+// JSON-NEXT:          "col": 1,
+// JSON-NEXT:          "tokLen": 4
+// JSON-NEXT:         },
+// JSON-NEXT:         "end": {
+// JSON-NEXT:          "offset": 3998,
+// JSON-NEXT:          "line": 127,
+// JSON-NEXT:          "col": 1,
+// JSON-NEXT:          "tokLen": 1
+// JSON-NEXT:         }
+// JSON-NEXT:        },
+// JSON-NEXT:        "name": "func",
+// JSON-NEXT:        "type": {
+// JSON-NEXT:         "qualType": "void ()"
+// JSON-NEXT:        },
+// JSON-NEXT:        "inner": [
+// JSON-NEXT:         {
+// JSON-NEXT:          "id": "0x{{.*}}",
+// JSON-NEXT:          "kind": "CompoundStmt",
+// JSON-NEXT:          "range": {
+// JSON-NEXT:           "begin": {
+// JSON-NEXT:            "offset": 3731,
+// JSON-NEXT:            "line": 121,
+// JSON-NEXT:            "col": 13,
+// JSON-NEXT:            "tokLen": 1
+// JSON-NEXT:           },
+// JSON-NEXT:           "end": {
+// JSON-NEXT:            "offset": 3998,
+// JSON-NEXT:            "line": 127,
+// JSON-NEXT:            "col": 1,
+// JSON-NEXT:            "tokLen": 1
+// JSON-NEXT:           }
+// JSON-NEXT:          },
+// JSON-NEXT:          "inner": [
+// JSON-NEXT:           {
+// JSON-NEXT:            "id": "0x{{.*}}",
+// JSON-NEXT:            "kind": "UnresolvedLookupExpr",
+// JSON-NEXT:            "range": {
+// JSON-NEXT:             "begin": {
+// JSON-NEXT:              "offset": 3735,
+// JSON-NEXT:              "line": 122,
+// JSON-NEXT:              "col": 3,
+// JSON-NEXT:              "tokLen": 1
+// JSON-NEXT:             },
+// JSON-NEXT:             "end": {
+// JSON-NEXT:              "offset": 3740,
+// JSON-NEXT:              "col": 8,
+// JSON-NEXT:              "tokLen": 1
+// JSON-NEXT:             }
+// JSON-NEXT:            },
+// JSON-NEXT:            "type": {
+// JSON-NEXT:             "qualType": "<dependent type>"
+// JSON-NEXT:            },
+// JSON-NEXT:            "valueCategory": "lvalue",
+// JSON-NEXT:            "usesADL": false,
+// JSON-NEXT:            "name": "C",
+// JSON-NEXT:            "lookups": [
+// JSON-NEXT:             {
+// JSON-NEXT:              "id": "0x{{.*}}",
+// JSON-NEXT:              "kind": "VarTemplateDecl",
+// JSON-NEXT:              "name": "C"
+// JSON-NEXT:             }
+// JSON-NEXT:            ],
+// JSON-NEXT:            "inner": [
+// JSON-NEXT:             {
+// JSON-NEXT:              "kind": "TemplateArgument",
+// JSON-NEXT:              "type": {
+// JSON-NEXT:               "qualType": "Key"
+// JSON-NEXT:              },
+// JSON-NEXT:              "inner": [
+// JSON-NEXT:               {
+// JSON-NEXT:                "id": "0x{{.*}}",
+// JSON-NEXT:                "kind": "TemplateTypeParmType",
+// JSON-NEXT:                "type": {
+// JSON-NEXT:                 "qualType": "Key"
+// JSON-NEXT:                },
+// JSON-NEXT:                "isDependent": true,
+// JSON-NEXT:                "isInstantiationDependent": true,
+// JSON-NEXT:                "depth": 0,
+// JSON-NEXT:                "index": 0,
+// JSON-NEXT:                "decl": {
+// JSON-NEXT:                 "id": "0x{{.*}}",
+// JSON-NEXT:                 "kind": "TemplateTypeParmDecl",
+// JSON-NEXT:                 "name": "Key"
+// JSON-NEXT:                }
+// JSON-NEXT:               }
+// JSON-NEXT:              ]
+// JSON-NEXT:             }
+// JSON-NEXT:            ]
+// JSON-NEXT:           }
+// JSON-NEXT:          ]
+// JSON-NEXT:         }
+// JSON-NEXT:        ]
+// JSON-NEXT:       }
+// JSON-NEXT:      ]
+// JSON-NEXT:     }
+// JSON-NEXT:    ]
+// JSON-NEXT:   },
+// JSON-NEXT:   {
+// JSON-NEXT:    "id": "0x{{.*}}",
+// JSON-NEXT:    "kind": "NamespaceDecl",
+// JSON-NEXT:    "loc": {
+// JSON-NEXT:     "offset": 4013,
+// JSON-NEXT:     "line": 130,
+// JSON-NEXT:     "col": 11,
+// JSON-NEXT:     "tokLen": 5
+// JSON-NEXT:    },
+// JSON-NEXT:    "range": {
+// JSON-NEXT:     "begin": {
+// JSON-NEXT:      "offset": 4003,
+// JSON-NEXT:      "col": 1,
+// JSON-NEXT:      "tokLen": 9
+// JSON-NEXT:     },
+// JSON-NEXT:     "end": {
+// JSON-NEXT:      "offset": 4308,
+// JSON-NEXT:      "line": 136,
+// JSON-NEXT:      "col": 1,
+// JSON-NEXT:      "tokLen": 1
+// JSON-NEXT:     }
+// JSON-NEXT:    },
+// JSON-NEXT:    "name": "test7",
+// JSON-NEXT:    "inner": [
+// JSON-NEXT:     {
+// JSON-NEXT:      "id": "0x{{.*}}",
+// JSON-NEXT:      "kind": "ClassTemplateDecl",
+// JSON-NEXT:      "loc": {
+// JSON-NEXT:       "offset": 4066,
+// JSON-NEXT:       "line": 131,
+// JSON-NEXT:       "col": 46,
+// JSON-NEXT:       "tokLen": 1
+// JSON-NEXT:      },
+// JSON-NEXT:      "range": {
+// JSON-NEXT:       "begin": {
+// JSON-NEXT:        "offset": 4023,
+// JSON-NEXT:        "col": 3,
+// JSON-NEXT:        "tokLen": 8
+// JSON-NEXT:       },
+// JSON-NEXT:       "end": {
+// JSON-NEXT:        "offset": 4069,
+// JSON-NEXT:        "col": 49,
+// JSON-NEXT:        "tokLen": 1
+// JSON-NEXT:       }
+// JSON-NEXT:      },
+// JSON-NEXT:      "name": "A",
+// JSON-NEXT:      "inner": [
+// JSON-NEXT:       {
+// JSON-NEXT:        "id": "0x{{.*}}",
+// JSON-NEXT:        "kind": "TemplateTemplateParmDecl",
+// JSON-NEXT:        "loc": {
+// JSON-NEXT:         "offset": 4055,
+// JSON-NEXT:         "col": 35,
+// JSON-NEXT:         "tokLen": 2
+// JSON-NEXT:        },
+// JSON-NEXT:        "range": {
+// JSON-NEXT:         "begin": {
+// JSON-NEXT:          "offset": 4033,
+// JSON-NEXT:          "col": 13,
+// JSON-NEXT:          "tokLen": 8
+// JSON-NEXT:         },
+// JSON-NEXT:         "end": {
+// JSON-NEXT:          "offset": 4055,
+// JSON-NEXT:          "col": 35,
+// JSON-NEXT:          "tokLen": 2
+// JSON-NEXT:         }
+// JSON-NEXT:        },
+// JSON-NEXT:        "name": "TT",
+// JSON-NEXT:        "depth": 0,
+// JSON-NEXT:        "index": 0,
+// JSON-NEXT:        "inner": [
+// JSON-NEXT:         {
+// JSON-NEXT:          "id": "0x{{.*}}",
+// JSON-NEXT:          "kind": "TemplateTypeParmDecl",
+// JSON-NEXT:          "loc": {
+// JSON-NEXT:           "offset": 4047,
+// JSON-NEXT:           "col": 27,
+// JSON-NEXT:           "tokLen": 1
+// JSON-NEXT:          },
+// JSON-NEXT:          "range": {
+// JSON-NEXT:           "begin": {
+// JSON-NEXT:            "offset": 4042,
+// JSON-NEXT:            "col": 22,
+// JSON-NEXT:            "tokLen": 5
+// JSON-NEXT:           },
+// JSON-NEXT:           "end": {
+// JSON-NEXT:            "offset": 4042,
+// JSON-NEXT:            "col": 22,
+// JSON-NEXT:            "tokLen": 5
+// JSON-NEXT:           }
+// JSON-NEXT:          },
+// JSON-NEXT:          "tagUsed": "class",
+// JSON-NEXT:          "depth": 1,
+// JSON-NEXT:          "index": 0
+// JSON-NEXT:         }
+// JSON-NEXT:        ]
+// JSON-NEXT:       },
+// JSON-NEXT:       {
+// JSON-NEXT:        "id": "0x{{.*}}",
+// JSON-NEXT:        "kind": "CXXRecordDecl",
+// JSON-NEXT:        "loc": {
+// JSON-NEXT:         "offset": 4066,
+// JSON-NEXT:         "col": 46,
+// JSON-NEXT:         "tokLen": 1
+// JSON-NEXT:        },
+// JSON-NEXT:        "range": {
+// JSON-NEXT:         "begin": {
+// JSON-NEXT:          "offset": 4059,
+// JSON-NEXT:          "col": 39,
+// JSON-NEXT:          "tokLen": 6
+// JSON-NEXT:         },
+// JSON-NEXT:         "end": {
+// JSON-NEXT:          "offset": 4069,
+// JSON-NEXT:          "col": 49,
+// JSON-NEXT:          "tokLen": 1
+// JSON-NEXT:         }
+// JSON-NEXT:        },
+// JSON-NEXT:        "name": "A",
+// JSON-NEXT:        "tagUsed": "struct",
+// JSON-NEXT:        "completeDefinition": true,
+// JSON-NEXT:        "definitionData": {
+// JSON-NEXT:         "canConstDefaultInit": true,
+// JSON-NEXT:         "copyAssign": {
+// JSON-NEXT:          "hasConstParam": true,
+// JSON-NEXT:          "implicitHasConstParam": true,
+// JSON-NEXT:          "needsImplicit": true,
+// JSON-NEXT:          "simple": true,
+// JSON-NEXT:          "trivial": true
+// JSON-NEXT:         },
+// JSON-NEXT:         "copyCtor": {
+// JSON-NEXT:          "hasConstParam": true,
+// JSON-NEXT:          "implicitHasConstParam": true,
+// JSON-NEXT:          "needsImplicit": true,
+// JSON-NEXT:          "simple": true,
+// JSON-NEXT:          "trivial": true
+// JSON-NEXT:         },
+// JSON-NEXT:         "defaultCtor": {
+// JSON-NEXT:          "defaultedIsConstexpr": true,
+// JSON-NEXT:          "exists": true,
+// JSON-NEXT:          "isConstexpr": true,
+// JSON-NEXT:          "needsImplicit": true,
+// JSON-NEXT:          "trivial": true
+// JSON-NEXT:         },
+// JSON-NEXT:         "dtor": {
+// JSON-NEXT:          "irrelevant": true,
+// JSON-NEXT:          "needsImplicit": true,
+// JSON-NEXT:          "simple": true,
+// JSON-NEXT:          "trivial": true
+// JSON-NEXT:         },
+// JSON-NEXT:         "hasConstexprNonCopyMoveConstructor": true,
+// JSON-NEXT:         "isAggregate": true,
+// JSON-NEXT:         "isEmpty": true,
+// JSON-NEXT:         "isLiteral": true,
+// JSON-NEXT:         "isPOD": true,
+// JSON-NEXT:         "isStandardLayout": true,
+// JSON-NEXT:         "isTrivial": true,
+// JSON-NEXT:         "isTriviallyCopyable": true,
+// JSON-NEXT:         "moveAssign": {
+// JSON-NEXT:          "exists": true,
+// JSON-NEXT:          "needsImplicit": true,
+// JSON-NEXT:          "simple": true,
+// JSON-NEXT:          "trivial": true
+// JSON-NEXT:         },
+// JSON-NEXT:         "moveCtor": {
+// JSON-NEXT:          "exists": true,
+// JSON-NEXT:          "needsImplicit": true,
+// JSON-NEXT:          "simple": true,
+// JSON-NEXT:          "trivial": true
+// JSON-NEXT:         }
+// JSON-NEXT:        },
+// JSON-NEXT:        "inner": [
+// JSON-NEXT:         {
+// JSON-NEXT:          "id": "0x{{.*}}",
+// JSON-NEXT:          "kind": "CXXRecordDecl",
+// JSON-NEXT:          "loc": {
+// JSON-NEXT:           "offset": 4066,
+// JSON-NEXT:           "col": 46,
+// JSON-NEXT:           "tokLen": 1
+// JSON-NEXT:          },
+// JSON-NEXT:          "range": {
+// JSON-NEXT:           "begin": {
+// JSON-NEXT:            "offset": 4059,
+// JSON-NEXT:            "col": 39,
+// JSON-NEXT:            "tokLen": 6
+// JSON-NEXT:           },
+// JSON-NEXT:           "end": {
+// JSON-NEXT:            "offset": 4066,
+// JSON-NEXT:            "col": 46,
+// JSON-NEXT:            "tokLen": 1
+// JSON-NEXT:           }
+// JSON-NEXT:          },
+// JSON-NEXT:          "isImplicit": true,
+// JSON-NEXT:          "name": "A",
+// JSON-NEXT:          "tagUsed": "struct"
+// JSON-NEXT:         }
+// JSON-NEXT:        ]
+// JSON-NEXT:       },
+// JSON-NEXT:       {
+// JSON-NEXT:        "id": "0x{{.*}}",
+// JSON-NEXT:        "kind": "ClassTemplateSpecializationDecl",
+// JSON-NEXT:        "name": "A"
+// JSON-NEXT:       }
+// JSON-NEXT:      ]
+// JSON-NEXT:     },
+// JSON-NEXT:     {
+// JSON-NEXT:      "id": "0x{{.*}}",
+// JSON-NEXT:      "kind": "ClassTemplateDecl",
+// JSON-NEXT:      "loc": {
+// JSON-NEXT:       "offset": 4100,
+// JSON-NEXT:       "line": 132,
+// JSON-NEXT:       "col": 29,
+// JSON-NEXT:       "tokLen": 1
+// JSON-NEXT:      },
+// JSON-NEXT:      "range": {
+// JSON-NEXT:       "begin": {
+// JSON-NEXT:        "offset": 4074,
+// JSON-NEXT:        "col": 3,
+// JSON-NEXT:        "tokLen": 8
+// JSON-NEXT:       },
+// JSON-NEXT:       "end": {
+// JSON-NEXT:        "offset": 4103,
+// JSON-NEXT:        "col": 32,
+// JSON-NEXT:        "tokLen": 1
+// JSON-NEXT:       }
+// JSON-NEXT:      },
+// JSON-NEXT:      "name": "B",
+// JSON-NEXT:      "inner": [
+// JSON-NEXT:       {
+// JSON-NEXT:        "id": "0x{{.*}}",
+// JSON-NEXT:        "kind": "TemplateTypeParmDecl",
+// JSON-NEXT:        "loc": {
+// JSON-NEXT:         "offset": 4092,
+// JSON-NEXT:         "col": 21,
+// JSON-NEXT:         "tokLen": 1
+// JSON-NEXT:        },
+// JSON-NEXT:        "range": {
+// JSON-NEXT:         "begin": {
+// JSON-NEXT:          "offset": 4084,
+// JSON-NEXT:          "col": 13,
+// JSON-NEXT:          "tokLen": 5
+// JSON-NEXT:         },
+// JSON-NEXT:         "end": {
+// JSON-NEXT:          "offset": 4084,
+// JSON-NEXT:          "col": 13,
+// JSON-NEXT:          "tokLen": 5
+// JSON-NEXT:         }
+// JSON-NEXT:        },
+// JSON-NEXT:        "tagUsed": "class",
+// JSON-NEXT:        "depth": 0,
+// JSON-NEXT:        "index": 0,
+// JSON-NEXT:        "isParameterPack": true
+// JSON-NEXT:       },
+// JSON-NEXT:       {
+// JSON-NEXT:        "id": "0x{{.*}}",
+// JSON-NEXT:        "kind": "CXXRecordDecl",
+// JSON-NEXT:        "loc": {
+// JSON-NEXT:         "offset": 4100,
+// JSON-NEXT:         "col": 29,
+// JSON-NEXT:         "tokLen": 1
+// JSON-NEXT:        },
+// JSON-NEXT:        "range": {
+// JSON-NEXT:         "begin": {
+// JSON-NEXT:          "offset": 4094,
+// JSON-NEXT:          "col": 23,
+// JSON-NEXT:          "tokLen": 5
+// JSON-NEXT:         },
+// JSON-NEXT:         "end": {
+// JSON-NEXT:          "offset": 4103,
+// JSON-NEXT:          "col": 32,
+// JSON-NEXT:          "tokLen": 1
+// JSON-NEXT:         }
+// JSON-NEXT:        },
+// JSON-NEXT:        "name": "B",
+// JSON-NEXT:        "tagUsed": "class",
+// JSON-NEXT:        "completeDefinition": true,
+// JSON-NEXT:        "definitionData": {
+// JSON-NEXT:         "canConstDefaultInit": true,
+// JSON-NEXT:         "copyAssign": {
+// JSON-NEXT:          "hasConstParam": true,
+// JSON-NEXT:          "implicitHasConstParam": true,
+// JSON-NEXT:          "needsImplicit": true,
+// JSON-NEXT:          "simple": true,
+// JSON-NEXT:          "trivial": true
+// JSON-NEXT:         },
+// JSON-NEXT:         "copyCtor": {
+// JSON-NEXT:          "hasConstParam": true,
+// JSON-NEXT:          "implicitHasConstParam": true,
+// JSON-NEXT:          "needsImplicit": true,
+// JSON-NEXT:          "simple": true,
+// JSON-NEXT:          "trivial": true
+// JSON-NEXT:         },
+// JSON-NEXT:         "defaultCtor": {
+// JSON-NEXT:          "defaultedIsConstexpr": true,
+// JSON-NEXT:          "exists": true,
+// JSON-NEXT:          "isConstexpr": true,
+// JSON-NEXT:          "needsImplicit": true,
+// JSON-NEXT:          "trivial": true
+// JSON-NEXT:         },
+// JSON-NEXT:         "dtor": {
+// JSON-NEXT:          "irrelevant": true,
+// JSON-NEXT:          "needsImplicit": true,
+// JSON-NEXT:          "simple": true,
+// JSON-NEXT:          "trivial": true
+// JSON-NEXT:         },
+// JSON-NEXT:         "hasConstexprNonCopyMoveConstructor": true,
+// JSON-NEXT:         "isAggregate": true,
+// JSON-NEXT:         "isEmpty": true,
+// JSON-NEXT:         "isLiteral": true,
+// JSON-NEXT:         "isPOD": true,
+// JSON-NEXT:         "isStandardLayout": true,
+// JSON-NEXT:         "isTrivial": true,
+// JSON-NEXT:         "isTriviallyCopyable": true,
+// JSON-NEXT:         "moveAssign": {
+// JSON-NEXT:          "exists": true,
+// JSON-NEXT:          "needsImplicit": true,
+// JSON-NEXT:          "simple": true,
+// JSON-NEXT:          "trivial": true
+// JSON-NEXT:         },
+// JSON-NEXT:         "moveCtor": {
+// JSON-NEXT:          "exists": true,
+// JSON-NEXT:          "needsImplicit": true,
+// JSON-NEXT:          "simple": true,
+// JSON-NEXT:          "trivial": true
+// JSON-NEXT:         }
+// JSON-NEXT:        },
+// JSON-NEXT:        "inner": [
+// JSON-NEXT:         {
+// JSON-NEXT:          "id": "0x{{.*}}",
+// JSON-NEXT:          "kind": "CXXRecordDecl",
+// JSON-NEXT:          "loc": {
+// JSON-NEXT:           "offset": 4100,
+// JSON-NEXT:           "col": 29,
+// JSON-NEXT:           "tokLen": 1
+// JSON-NEXT:          },
+// JSON-NEXT:          "range": {
+// JSON-NEXT:           "begin": {
+// JSON-NEXT:            "offset": 4094,
+// JSON-NEXT:            "col": 23,
+// JSON-NEXT:            "tokLen": 5
+// JSON-NEXT:           },
+// JSON-NEXT:           "end": {
+// JSON-NEXT:            "offset": 4100,
+// JSON-NEXT:            "col": 29,
+// JSON-NEXT:            "tokLen": 1
+// JSON-NEXT:           }
+// JSON-NEXT:          },
+// JSON-NEXT:          "isImplicit": true,
+// JSON-NEXT:          "name": "B",
+// JSON-NEXT:          "tagUsed": "class"
+// JSON-NEXT:         }
+// JSON-NEXT:        ]
+// JSON-NEXT:       }
+// JSON-NEXT:      ]
+// JSON-NEXT:     },
+// JSON-NEXT:     {
+// JSON-NEXT:      "id": "0x{{.*}}",
+// JSON-NEXT:      "kind": "ClassTemplateSpecializationDecl",
+// JSON-NEXT:      "loc": {
+// JSON-NEXT:       "offset": 4124,
+// JSON-NEXT:       "line": 133,
+// JSON-NEXT:       "col": 19,
+// JSON-NEXT:       "tokLen": 1
+// JSON-NEXT:      },
+// JSON-NEXT:      "range": {
+// JSON-NEXT:       "begin": {
+// JSON-NEXT:        "offset": 4108,
+// JSON-NEXT:        "col": 3,
+// JSON-NEXT:        "tokLen": 8
+// JSON-NEXT:       },
+// JSON-NEXT:       "end": {
+// JSON-NEXT:        "offset": 4127,
+// JSON-NEXT:        "col": 22,
+// JSON-NEXT:        "tokLen": 1
+// JSON-NEXT:       }
+// JSON-NEXT:      },
+// JSON-NEXT:      "name": "A",
+// JSON-NEXT:      "tagUsed": "struct",
+// JSON-NEXT:      "completeDefinition": true,
+// JSON-NEXT:      "strict-pack-match": true,
+// JSON-NEXT:      "definitionData": {
+// JSON-NEXT:       "canConstDefaultInit": true,
+// JSON-NEXT:       "canPassInRegisters": true,
+// JSON-NEXT:       "copyAssign": {
+// JSON-NEXT:        "hasConstParam": true,
+// JSON-NEXT:        "implicitHasConstParam": true,
+// JSON-NEXT:        "needsImplicit": true,
+// JSON-NEXT:        "simple": true,
+// JSON-NEXT:        "trivial": true
+// JSON-NEXT:       },
+// JSON-NEXT:       "copyCtor": {
+// JSON-NEXT:        "hasConstParam": true,
+// JSON-NEXT:        "implicitHasConstParam": true,
+// JSON-NEXT:        "needsImplicit": true,
+// JSON-NEXT:        "simple": true,
+// JSON-NEXT:        "trivial": true
+// JSON-NEXT:       },
+// JSON-NEXT:       "defaultCtor": {
+// JSON-NEXT:        "defaultedIsConstexpr": true,
+// JSON-NEXT:        "exists": true,
+// JSON-NEXT:        "isConstexpr": true,
+// JSON-NEXT:        "needsImplicit": true,
+// JSON-NEXT:        "trivial": true
+// JSON-NEXT:       },
+// JSON-NEXT:       "dtor": {
+// JSON-NEXT:        "irrelevant": true,
+// JSON-NEXT:        "needsImplicit": true,
+// JSON-NEXT:        "simple": true,
+// JSON-NEXT:        "trivial": true
+// JSON-NEXT:       },
+// JSON-NEXT:       "hasConstexprNonCopyMoveConstructor": true,
+// JSON-NEXT:       "isAggregate": true,
+// JSON-NEXT:       "isEmpty": true,
+// JSON-NEXT:       "isLiteral": true,
+// JSON-NEXT:       "isPOD": true,
+// JSON-NEXT:       "isStandardLayout": true,
+// JSON-NEXT:       "isTrivial": true,
+// JSON-NEXT:       "isTriviallyCopyable": true,
+// JSON-NEXT:       "moveAssign": {
+// JSON-NEXT:        "exists": true,
+// JSON-NEXT:        "needsImplicit": true,
+// JSON-NEXT:        "simple": true,
+// JSON-NEXT:        "trivial": true
+// JSON-NEXT:       },
+// JSON-NEXT:       "moveCtor": {
+// JSON-NEXT:        "exists": true,
+// JSON-NEXT:        "needsImplicit": true,
+// JSON-NEXT:        "simple": true,
+// JSON-NEXT:        "trivial": true
+// JSON-NEXT:       }
+// JSON-NEXT:      },
+// JSON-NEXT:      "inner": [
+// JSON-NEXT:       {
+// JSON-NEXT:        "kind": "TemplateArgument"
+// JSON-NEXT:       },
+// JSON-NEXT:       {
+// JSON-NEXT:        "id": "0x{{.*}}",
+// JSON-NEXT:        "kind": "CXXRecordDecl",
+// JSON-NEXT:        "loc": {
+// JSON-NEXT:         "offset": 4066,
+// JSON-NEXT:         "line": 131,
+// JSON-NEXT:         "col": 46,
+// JSON-NEXT:         "tokLen": 1
+// JSON-NEXT:        },
+// JSON-NEXT:        "range": {
+// JSON-NEXT:         "begin": {
+// JSON-NEXT:          "offset": 4059,
+// JSON-NEXT:          "col": 39,
+// JSON-NEXT:          "tokLen": 6
+// JSON-NEXT:         },
+// JSON-NEXT:         "end": {
+// JSON-NEXT:          "offset": 4066,
+// JSON-NEXT:          "col": 46,
+// JSON-NEXT:          "tokLen": 1
+// JSON-NEXT:         }
+// JSON-NEXT:        },
+// JSON-NEXT:        "isImplicit": true,
+// JSON-NEXT:        "name": "A",
+// JSON-NEXT:        "tagUsed": "struct"
+// JSON-NEXT:       }
+// JSON-NEXT:      ]
+// JSON-NEXT:     }
+// JSON-NEXT:    ]
+// JSON-NEXT:   }
+// JSON-NEXT:  ]
+// JSON-NEXT: }
diff --git a/clang/test/AST/ast-print-openacc-atomic-construct.cpp b/clang/test/AST/ast-print-openacc-atomic-construct.cpp
new file mode 100644
index 0000000000000..572f2ea4842d8
--- /dev/null
+++ b/clang/test/AST/ast-print-openacc-atomic-construct.cpp
@@ -0,0 +1,33 @@
+// RUN: %clang_cc1 -fopenacc -ast-print %s -o - | FileCheck %s
+
+void foo(int v, int x) {
+// CHECK: #pragma acc atomic read
+// CHECK-NEXT:   v = x;
+#pragma acc atomic read
+  v = x;
+// CHECK-NEXT: pragma acc atomic write
+// CHECK-NEXT:  v = x + 1;
+#pragma acc atomic write
+  v = x + 1;
+// CHECK-NEXT: pragma acc atomic update
+// CHECK-NEXT:  x++;
+#pragma acc atomic update
+  x++;
+// CHECK-NEXT: pragma acc atomic 
+// CHECK-NEXT:  x--;
+#pragma acc atomic
+  x--;
+// CHECK-NEXT: pragma acc atomic capture
+// CHECK-NEXT:  v = x++;
+#pragma acc atomic capture
+  v = x++;
+
+// CHECK-NEXT: #pragma acc atomic capture
+// CHECK-NEXT: { 
+// CHECK-NEXT: x--;
+// CHECK-NEXT: v = x;
+// CHECK-NEXT: }
+#pragma acc atomic capture
+  { x--; v = x; }
+
+}
diff --git a/clang/test/AST/attr-counted-by-late-parsed-struct-ptrs.c b/clang/test/AST/attr-counted-by-late-parsed-struct-ptrs.c
index a585a45eeff03..f9772db8b6554 100644
--- a/clang/test/AST/attr-counted-by-late-parsed-struct-ptrs.c
+++ b/clang/test/AST/attr-counted-by-late-parsed-struct-ptrs.c
@@ -31,7 +31,7 @@ struct on_pointer_anon_count {
 // CHECK-NEXT:  | `-FieldDecl {{.*}} count 'int'
 // CHECK-NEXT:  |-FieldDecl {{.*}} implicit 'struct on_pointer_anon_count::(anonymous at {{.*}})'
 // CHECK-NEXT:  `-IndirectFieldDecl {{.*}} implicit referenced count 'int'
-// CHECK-NEXT:    |-Field {{.*}} '' 'struct on_pointer_anon_count::(anonymous at {{.*}})'
+// CHECK-NEXT:    |-Field {{.*}} field_index 1 'struct on_pointer_anon_count::(anonymous at {{.*}})'
 // CHECK-NEXT:    `-Field {{.*}} 'count' 'int'
 
 //==============================================================================
diff --git a/clang/test/AST/attr-counted-by-or-null-late-parsed-struct-ptrs.c b/clang/test/AST/attr-counted-by-or-null-late-parsed-struct-ptrs.c
index 975c0a0231943..59b866dae720d 100644
--- a/clang/test/AST/attr-counted-by-or-null-late-parsed-struct-ptrs.c
+++ b/clang/test/AST/attr-counted-by-or-null-late-parsed-struct-ptrs.c
@@ -31,7 +31,7 @@ struct on_pointer_anon_count {
 // CHECK-NEXT:  | `-FieldDecl {{.*}} count 'int'
 // CHECK-NEXT:  |-FieldDecl {{.*}} implicit 'struct on_pointer_anon_count::(anonymous at {{.*}})'
 // CHECK-NEXT:  `-IndirectFieldDecl {{.*}} implicit referenced count 'int'
-// CHECK-NEXT:    |-Field {{.*}} '' 'struct on_pointer_anon_count::(anonymous at {{.*}})'
+// CHECK-NEXT:    |-Field {{.*}} field_index 1 'struct on_pointer_anon_count::(anonymous at {{.*}})'
 // CHECK-NEXT:    `-Field {{.*}} 'count' 'int'
 
 //==============================================================================
diff --git a/clang/test/AST/attr-counted-by-or-null-struct-ptrs.c b/clang/test/AST/attr-counted-by-or-null-struct-ptrs.c
index 075f583784fe1..d42547003f0b3 100644
--- a/clang/test/AST/attr-counted-by-or-null-struct-ptrs.c
+++ b/clang/test/AST/attr-counted-by-or-null-struct-ptrs.c
@@ -26,7 +26,7 @@ struct on_member_pointer_complete_ty {
 // CHECK-NEXT:  | `-FieldDecl {{.+}} buf 'struct size_known * __counted_by_or_null(count)':'struct size_known *'
 // CHECK-NEXT:  |-FieldDecl {{.+}} implicit 'struct on_pointer_anon_buf::(anonymous at [[ANON_STRUCT_PATH:.+]])'
 // CHECK-NEXT:  `-IndirectFieldDecl {{.+}} implicit buf 'struct size_known * __counted_by_or_null(count)':'struct size_known *'
-// CHECK-NEXT:    |-Field {{.+}} '' 'struct on_pointer_anon_buf::(anonymous at [[ANON_STRUCT_PATH]])'
+// CHECK-NEXT:    |-Field {{.+}} field_index 1 'struct on_pointer_anon_buf::(anonymous at [[ANON_STRUCT_PATH]])'
 // CHECK-NEXT:    `-Field {{.+}} 'buf' 'struct size_known * __counted_by_or_null(count)':'struct size_known *'
 struct on_pointer_anon_buf {
   int count;
@@ -94,7 +94,7 @@ struct on_nested_pointer_outer {
 // CHECK-NEXT:  | `-FieldDecl {{.+}} buf 'struct size_known * __counted_by_or_null(count)':'struct size_known *'
 // CHECK-NEXT:  |-FieldDecl {{.+}} implicit 'struct on_pointer_anon_buf_ty_pos::(anonymous at [[ANON_STRUCT_PATH2:.+]])'
 // CHECK-NEXT:  `-IndirectFieldDecl {{.+}} implicit buf 'struct size_known * __counted_by_or_null(count)':'struct size_known *'
-// CHECK-NEXT:    |-Field {{.+}} '' 'struct on_pointer_anon_buf_ty_pos::(anonymous at [[ANON_STRUCT_PATH2]])'
+// CHECK-NEXT:    |-Field {{.+}} field_index 1 'struct on_pointer_anon_buf_ty_pos::(anonymous at [[ANON_STRUCT_PATH2]])'
 // CHECK-NEXT:    `-Field {{.+}} 'buf' 'struct size_known * __counted_by_or_null(count)':'struct size_known *'
 struct on_pointer_anon_buf_ty_pos {
   int count;
@@ -108,7 +108,7 @@ struct on_pointer_anon_buf_ty_pos {
 // CHECK-NEXT:  | `-FieldDecl {{.+}} count 'int'
 // CHECK-NEXT:  |-FieldDecl {{.+}} implicit 'struct on_pointer_anon_count_ty_pos::(anonymous at [[ANON_STRUCT_PATH3:.+]])'
 // CHECK-NEXT:  |-IndirectFieldDecl {{.+}} implicit referenced count 'int'
-// CHECK-NEXT:  | |-Field {{.+}} '' 'struct on_pointer_anon_count_ty_pos::(anonymous at [[ANON_STRUCT_PATH3]])'
+// CHECK-NEXT:  | |-Field {{.+}} field_index 0 'struct on_pointer_anon_count_ty_pos::(anonymous at [[ANON_STRUCT_PATH3]])'
 // CHECK-NEXT:  | `-Field {{.+}} 'count' 'int'
 struct on_pointer_anon_count_ty_pos {
   struct {
diff --git a/clang/test/AST/attr-counted-by-struct-ptrs.c b/clang/test/AST/attr-counted-by-struct-ptrs.c
index 0c05258234143..afef9c8c3b95d 100644
--- a/clang/test/AST/attr-counted-by-struct-ptrs.c
+++ b/clang/test/AST/attr-counted-by-struct-ptrs.c
@@ -26,7 +26,7 @@ struct on_member_pointer_complete_ty {
 // CHECK-NEXT:  | `-FieldDecl {{.+}} buf 'struct size_known * __counted_by(count)':'struct size_known *'
 // CHECK-NEXT:  |-FieldDecl {{.+}} implicit 'struct on_pointer_anon_buf::(anonymous at [[ANON_STRUCT_PATH:.+]])'
 // CHECK-NEXT:  `-IndirectFieldDecl {{.+}} implicit buf 'struct size_known * __counted_by(count)':'struct size_known *'
-// CHECK-NEXT:    |-Field {{.+}} '' 'struct on_pointer_anon_buf::(anonymous at [[ANON_STRUCT_PATH]])'
+// CHECK-NEXT:    |-Field {{.+}} field_index 1 'struct on_pointer_anon_buf::(anonymous at [[ANON_STRUCT_PATH]])'
 // CHECK-NEXT:    `-Field {{.+}} 'buf' 'struct size_known * __counted_by(count)':'struct size_known *'
 struct on_pointer_anon_buf {
   int count;
@@ -94,7 +94,7 @@ struct on_nested_pointer_outer {
 // CHECK-NEXT:  | `-FieldDecl {{.+}} buf 'struct size_known * __counted_by(count)':'struct size_known *'
 // CHECK-NEXT:  |-FieldDecl {{.+}} implicit 'struct on_pointer_anon_buf_ty_pos::(anonymous at [[ANON_STRUCT_PATH2:.+]])'
 // CHECK-NEXT:  `-IndirectFieldDecl {{.+}} implicit buf 'struct size_known * __counted_by(count)':'struct size_known *'
-// CHECK-NEXT:    |-Field {{.+}} '' 'struct on_pointer_anon_buf_ty_pos::(anonymous at [[ANON_STRUCT_PATH2]])'
+// CHECK-NEXT:    |-Field {{.+}} field_index 1 'struct on_pointer_anon_buf_ty_pos::(anonymous at [[ANON_STRUCT_PATH2]])'
 // CHECK-NEXT:    `-Field {{.+}} 'buf' 'struct size_known * __counted_by(count)':'struct size_known *'
 struct on_pointer_anon_buf_ty_pos {
   int count;
@@ -108,7 +108,7 @@ struct on_pointer_anon_buf_ty_pos {
 // CHECK-NEXT:  | `-FieldDecl {{.+}} count 'int'
 // CHECK-NEXT:  |-FieldDecl {{.+}} implicit 'struct on_pointer_anon_count_ty_pos::(anonymous at [[ANON_STRUCT_PATH3:.+]])'
 // CHECK-NEXT:  |-IndirectFieldDecl {{.+}} implicit referenced count 'int'
-// CHECK-NEXT:  | |-Field {{.+}} '' 'struct on_pointer_anon_count_ty_pos::(anonymous at [[ANON_STRUCT_PATH3]])'
+// CHECK-NEXT:  | |-Field {{.+}} field_index 0 'struct on_pointer_anon_count_ty_pos::(anonymous at [[ANON_STRUCT_PATH3]])'
 // CHECK-NEXT:  | `-Field {{.+}} 'count' 'int'
 struct on_pointer_anon_count_ty_pos {
   struct {
diff --git a/clang/test/AST/attr-print-emit.cpp b/clang/test/AST/attr-print-emit.cpp
index a9bca6778d0f1..77826f8f9af09 100644
--- a/clang/test/AST/attr-print-emit.cpp
+++ b/clang/test/AST/attr-print-emit.cpp
@@ -91,3 +91,8 @@ ANNOTATE_ATTR NONNULL_ATTR void fn_non_null_annotated_attr(int *) __attribute__(
 
 [[gnu::nonnull(1)]] [[gnu::always_inline]] void cxx11_attr(int*) ANNOTATE_ATTR;
 // CHECK: {{\[\[}}gnu::nonnull(1)]] {{\[\[}}gnu::always_inline]] void cxx11_attr(int *) __attribute__((annotate("Annotated")));
+
+struct Foo;
+
+// CHECK: void as_member_fn_ptr(int *(Foo::*member)(int) __attribute__((alloc_size(1))));
+void as_member_fn_ptr(int* (Foo::*member)(int)  __attribute__((alloc_size(1))));
diff --git a/clang/test/AST/attr-sized-by-late-parsed-struct-ptrs.c b/clang/test/AST/attr-sized-by-late-parsed-struct-ptrs.c
index b58caf608bf97..411b333f37989 100644
--- a/clang/test/AST/attr-sized-by-late-parsed-struct-ptrs.c
+++ b/clang/test/AST/attr-sized-by-late-parsed-struct-ptrs.c
@@ -31,7 +31,7 @@ struct on_pointer_anon_count {
 // CHECK-NEXT:  | `-FieldDecl {{.*}} count 'int'
 // CHECK-NEXT:  |-FieldDecl {{.*}} implicit 'struct on_pointer_anon_count::(anonymous at {{.*}})'
 // CHECK-NEXT:  `-IndirectFieldDecl {{.*}} implicit referenced count 'int'
-// CHECK-NEXT:    |-Field {{.*}} '' 'struct on_pointer_anon_count::(anonymous at {{.*}})'
+// CHECK-NEXT:    |-Field {{.*}} field_index 1 'struct on_pointer_anon_count::(anonymous at {{.*}})'
 // CHECK-NEXT:    `-Field {{.*}} 'count' 'int'
 
 //==============================================================================
diff --git a/clang/test/AST/attr-sized-by-or-null-late-parsed-struct-ptrs.c b/clang/test/AST/attr-sized-by-or-null-late-parsed-struct-ptrs.c
index d55a42ac0fb94..ae89360e17493 100644
--- a/clang/test/AST/attr-sized-by-or-null-late-parsed-struct-ptrs.c
+++ b/clang/test/AST/attr-sized-by-or-null-late-parsed-struct-ptrs.c
@@ -31,7 +31,7 @@ struct on_pointer_anon_count {
 // CHECK-NEXT:  | `-FieldDecl {{.*}} count 'int'
 // CHECK-NEXT:  |-FieldDecl {{.*}} implicit 'struct on_pointer_anon_count::(anonymous at {{.*}})'
 // CHECK-NEXT:  `-IndirectFieldDecl {{.*}} implicit referenced count 'int'
-// CHECK-NEXT:    |-Field {{.*}} '' 'struct on_pointer_anon_count::(anonymous at {{.*}})'
+// CHECK-NEXT:    |-Field {{.*}} field_index 1 'struct on_pointer_anon_count::(anonymous at {{.*}})'
 // CHECK-NEXT:    `-Field {{.*}} 'count' 'int'
 
 //==============================================================================
diff --git a/clang/test/AST/attr-sized-by-or-null-struct-ptrs.c b/clang/test/AST/attr-sized-by-or-null-struct-ptrs.c
index 73b8a71f23503..7273280e4b60c 100644
--- a/clang/test/AST/attr-sized-by-or-null-struct-ptrs.c
+++ b/clang/test/AST/attr-sized-by-or-null-struct-ptrs.c
@@ -26,7 +26,7 @@ struct on_member_pointer_complete_ty {
 // CHECK-NEXT:  | `-FieldDecl {{.+}} buf 'struct size_known * __sized_by_or_null(count)':'struct size_known *'
 // CHECK-NEXT:  |-FieldDecl {{.+}} implicit 'struct on_pointer_anon_buf::(anonymous at [[ANON_STRUCT_PATH:.+]])'
 // CHECK-NEXT:  `-IndirectFieldDecl {{.+}} implicit buf 'struct size_known * __sized_by_or_null(count)':'struct size_known *'
-// CHECK-NEXT:    |-Field {{.+}} '' 'struct on_pointer_anon_buf::(anonymous at [[ANON_STRUCT_PATH]])'
+// CHECK-NEXT:    |-Field {{.+}} field_index 1 'struct on_pointer_anon_buf::(anonymous at [[ANON_STRUCT_PATH]])'
 // CHECK-NEXT:    `-Field {{.+}} 'buf' 'struct size_known * __sized_by_or_null(count)':'struct size_known *'
 struct on_pointer_anon_buf {
   int count;
@@ -94,7 +94,7 @@ struct on_nested_pointer_outer {
 // CHECK-NEXT:  | `-FieldDecl {{.+}} buf 'struct size_known * __sized_by_or_null(count)':'struct size_known *'
 // CHECK-NEXT:  |-FieldDecl {{.+}} implicit 'struct on_pointer_anon_buf_ty_pos::(anonymous at [[ANON_STRUCT_PATH2:.+]])'
 // CHECK-NEXT:  `-IndirectFieldDecl {{.+}} implicit buf 'struct size_known * __sized_by_or_null(count)':'struct size_known *'
-// CHECK-NEXT:    |-Field {{.+}} '' 'struct on_pointer_anon_buf_ty_pos::(anonymous at [[ANON_STRUCT_PATH2]])'
+// CHECK-NEXT:    |-Field {{.+}} field_index 1 'struct on_pointer_anon_buf_ty_pos::(anonymous at [[ANON_STRUCT_PATH2]])'
 // CHECK-NEXT:    `-Field {{.+}} 'buf' 'struct size_known * __sized_by_or_null(count)':'struct size_known *'
 struct on_pointer_anon_buf_ty_pos {
   int count;
@@ -108,7 +108,7 @@ struct on_pointer_anon_buf_ty_pos {
 // CHECK-NEXT:  | `-FieldDecl {{.+}} count 'int'
 // CHECK-NEXT:  |-FieldDecl {{.+}} implicit 'struct on_pointer_anon_count_ty_pos::(anonymous at [[ANON_STRUCT_PATH3:.+]])'
 // CHECK-NEXT:  |-IndirectFieldDecl {{.+}} implicit referenced count 'int'
-// CHECK-NEXT:  | |-Field {{.+}} '' 'struct on_pointer_anon_count_ty_pos::(anonymous at [[ANON_STRUCT_PATH3]])'
+// CHECK-NEXT:  | |-Field {{.+}} field_index 0 'struct on_pointer_anon_count_ty_pos::(anonymous at [[ANON_STRUCT_PATH3]])'
 // CHECK-NEXT:  | `-Field {{.+}} 'count' 'int'
 struct on_pointer_anon_count_ty_pos {
   struct {
diff --git a/clang/test/AST/attr-sized-by-struct-ptrs.c b/clang/test/AST/attr-sized-by-struct-ptrs.c
index 7f7e3dfea2ac7..738eaf8cbf36b 100644
--- a/clang/test/AST/attr-sized-by-struct-ptrs.c
+++ b/clang/test/AST/attr-sized-by-struct-ptrs.c
@@ -26,7 +26,7 @@ struct on_member_pointer_complete_ty {
 // CHECK-NEXT:  | `-FieldDecl {{.+}} buf 'struct size_known * __sized_by(count)':'struct size_known *'
 // CHECK-NEXT:  |-FieldDecl {{.+}} implicit 'struct on_pointer_anon_buf::(anonymous at [[ANON_STRUCT_PATH:.+]])'
 // CHECK-NEXT:  `-IndirectFieldDecl {{.+}} implicit buf 'struct size_known * __sized_by(count)':'struct size_known *'
-// CHECK-NEXT:    |-Field {{.+}} '' 'struct on_pointer_anon_buf::(anonymous at [[ANON_STRUCT_PATH]])'
+// CHECK-NEXT:    |-Field {{.+}} field_index 1 'struct on_pointer_anon_buf::(anonymous at [[ANON_STRUCT_PATH]])'
 // CHECK-NEXT:    `-Field {{.+}} 'buf' 'struct size_known * __sized_by(count)':'struct size_known *'
 struct on_pointer_anon_buf {
   int count;
@@ -94,7 +94,7 @@ struct on_nested_pointer_outer {
 // CHECK-NEXT:  | `-FieldDecl {{.+}} buf 'struct size_known * __sized_by(count)':'struct size_known *'
 // CHECK-NEXT:  |-FieldDecl {{.+}} implicit 'struct on_pointer_anon_buf_ty_pos::(anonymous at [[ANON_STRUCT_PATH2:.+]])'
 // CHECK-NEXT:  `-IndirectFieldDecl {{.+}} implicit buf 'struct size_known * __sized_by(count)':'struct size_known *'
-// CHECK-NEXT:    |-Field {{.+}} '' 'struct on_pointer_anon_buf_ty_pos::(anonymous at [[ANON_STRUCT_PATH2]])'
+// CHECK-NEXT:    |-Field {{.+}} field_index 1 'struct on_pointer_anon_buf_ty_pos::(anonymous at [[ANON_STRUCT_PATH2]])'
 // CHECK-NEXT:    `-Field {{.+}} 'buf' 'struct size_known * __sized_by(count)':'struct size_known *'
 struct on_pointer_anon_buf_ty_pos {
   int count;
@@ -108,7 +108,7 @@ struct on_pointer_anon_buf_ty_pos {
 // CHECK-NEXT:  | `-FieldDecl {{.+}} count 'int'
 // CHECK-NEXT:  |-FieldDecl {{.+}} implicit 'struct on_pointer_anon_count_ty_pos::(anonymous at [[ANON_STRUCT_PATH3:.+]])'
 // CHECK-NEXT:  |-IndirectFieldDecl {{.+}} implicit referenced count 'int'
-// CHECK-NEXT:  | |-Field {{.+}} '' 'struct on_pointer_anon_count_ty_pos::(anonymous at [[ANON_STRUCT_PATH3]])'
+// CHECK-NEXT:  | |-Field {{.+}} field_index 0 'struct on_pointer_anon_count_ty_pos::(anonymous at [[ANON_STRUCT_PATH3]])'
 // CHECK-NEXT:  | `-Field {{.+}} 'count' 'int'
 struct on_pointer_anon_count_ty_pos {
   struct {
diff --git a/clang/test/AST/gen_ast_dump_json_test.py b/clang/test/AST/gen_ast_dump_json_test.py
index 301d60e479dbf..39b8eaadbad32 100644
--- a/clang/test/AST/gen_ast_dump_json_test.py
+++ b/clang/test/AST/gen_ast_dump_json_test.py
@@ -83,6 +83,12 @@ def main():
         action="store",
         default="",
     )
+    parser.add_argument(
+        "--prefix",
+        help="The FileCheck prefix",
+        action="store",
+        default="CHECK",
+    )
     update_or_generate_group = parser.add_mutually_exclusive_group()
     update_or_generate_group.add_argument(
         "--update", help="Update the file in-place", action="store_true"
@@ -113,11 +119,18 @@ def main():
             cmdline_opts=args.opts,
             do_update=args.update,
             force_update=args.update_manual,
+            prefix=args.prefix,
         )
 
 
 def process_file(
-    source_file, clang_binary, cmdline_filters, cmdline_opts, do_update, force_update
+    source_file,
+    clang_binary,
+    cmdline_filters,
+    cmdline_opts,
+    do_update,
+    force_update,
+    prefix,
 ):
     note_firstline = (
         "// NOTE: CHECK lines have been autogenerated by " "gen_ast_dump_json_test.py"
@@ -227,14 +240,14 @@ def process_file(
         for out_ast in out_asts:
             append_str = json.dumps(out_ast, indent=1, ensure_ascii=False)
             out_str = "\n\n"
-            out_str += "// CHECK-NOT: {{^}}Dumping\n"
+            out_str += f"// {prefix}-NOT: {{{{^}}}}Dumping\n"
             index = 0
             for append_line in append_str.splitlines()[2:]:
                 if index == 0:
-                    out_str += "// CHECK: %s\n" % (append_line.rstrip())
+                    out_str += f"// {prefix}: %s\n" % (append_line.rstrip())
                     index += 1
                 else:
-                    out_str += "// CHECK-NEXT: %s\n" % (append_line.rstrip())
+                    out_str += f"// {prefix}-NEXT: %s\n" % (append_line.rstrip())
 
             f.write(out_str)
         f.flush()
diff --git a/clang/test/Analysis/Checkers/WebKit/call-args-counted-const-member.cpp b/clang/test/Analysis/Checkers/WebKit/call-args-counted-const-member.cpp
index 215238a7fcf07..8da415a818a82 100644
--- a/clang/test/Analysis/Checkers/WebKit/call-args-counted-const-member.cpp
+++ b/clang/test/Analysis/Checkers/WebKit/call-args-counted-const-member.cpp
@@ -31,6 +31,7 @@ class Foo {
 public:
   Foo();
   void bar();
+  RefCountable& obj1() const { return m_obj1; }
 
 private:
   const Ref<RefCountable> m_obj1;
@@ -41,6 +42,7 @@ void Foo::bar() {
   m_obj1->method();
   m_obj2->method();
   // expected-warning@-1{{Call argument for 'this' parameter is uncounted and unsafe}}
+  obj1().method();
 }
 
 } // namespace call_args_const_ref_member
@@ -100,6 +102,7 @@ class Foo {
 public:
   Foo();
   void bar();
+  RefCountable& obj1() { return m_obj1; }
 
 private:
   const UniqueRef<RefCountable> m_obj1;
@@ -110,6 +113,7 @@ void Foo::bar() {
   m_obj1->method();
   m_obj2->method();
   // expected-warning@-1{{Call argument for 'this' parameter is uncounted and unsafe}}
+  obj1().method();
 }
 
 } // namespace call_args_const_unique_ref
diff --git a/clang/test/Analysis/Checkers/WebKit/mock-types.h b/clang/test/Analysis/Checkers/WebKit/mock-types.h
index 85397c2d25951..a1f0cc8b046b9 100644
--- a/clang/test/Analysis/Checkers/WebKit/mock-types.h
+++ b/clang/test/Analysis/Checkers/WebKit/mock-types.h
@@ -289,6 +289,7 @@ class UniqueRef {
     u.t = nullptr;
   }
   T &get() const { return *t; }
+  operator T&() const { return *t; }
   T *operator->() const { return t; }
   UniqueRef &operator=(T &) { return *this; }
 };
diff --git a/clang/test/Analysis/Checkers/WebKit/uncounted-lambda-captures.cpp b/clang/test/Analysis/Checkers/WebKit/uncounted-lambda-captures.cpp
index 2173245bc7af3..4f4a960282253 100644
--- a/clang/test/Analysis/Checkers/WebKit/uncounted-lambda-captures.cpp
+++ b/clang/test/Analysis/Checkers/WebKit/uncounted-lambda-captures.cpp
@@ -63,6 +63,19 @@ template<typename Out, typename... In> Function<Out(In...)> adopt(Detail::Callab
     return Function<Out(In...)>(impl, Function<Out(In...)>::Adopt);
 }
 
+template <typename KeyType, typename ValueType>
+class HashMap {
+public:
+  HashMap();
+  HashMap([[clang::noescape]] const Function<ValueType()>&);
+  void ensure(const KeyType&, [[clang::noescape]] const Function<ValueType()>&);
+  bool operator+([[clang::noescape]] const Function<ValueType()>&) const;
+  static void ifAny(HashMap, [[clang::noescape]] const Function<bool(ValueType)>&);
+
+private:
+  ValueType* m_table { nullptr };
+};
+
 } // namespace WTF
 
 struct A {
@@ -252,13 +265,40 @@ struct RefCountableWithLambdaCapturingThis {
     call(lambda);
   }
 
-  void method_captures_this_with_guardian_refPtr() {
+  void method_captures_this_with_guardian_refptr() {
     auto lambda = [this, protectedThis = RefPtr { &*this }]() {
       nonTrivial();
     };
     call(lambda);
   }
 
+  void forEach(const WTF::Function<void(RefCountable&)>&);
+  void method_captures_this_with_lambda_with_no_escape() {
+    auto run = [&]([[clang::noescape]] const WTF::Function<void(RefCountable&)>& func) {
+      forEach(func);
+    };
+    run([&](RefCountable&) {
+      nonTrivial();
+    });
+  }
+
+  static void callLambda([[clang::noescape]] const WTF::Function<RefPtr<RefCountable>()>&);
+  void method_captures_this_in_template_method() {
+    RefCountable* obj = make_obj();
+    WTF::HashMap<int, RefPtr<RefCountable>> nextMap;
+    nextMap.ensure(3, [&] {
+      return obj->next();
+    });
+    nextMap+[&] {
+      return obj->next();
+    };
+    WTF::HashMap<int, RefPtr<RefCountable>>::ifAny(nextMap, [&](auto& item) -> bool {
+      return item->next() && obj->next();
+    });
+    callLambda([&]() -> RefPtr<RefCountable> {
+      return obj->next();
+    });
+  }
 };
 
 struct NonRefCountableWithLambdaCapturingThis {
diff --git a/clang/test/Analysis/Checkers/WebKit/uncounted-obj-arg.cpp b/clang/test/Analysis/Checkers/WebKit/uncounted-obj-arg.cpp
index d654d963a4fae..ffeecbf87c451 100644
--- a/clang/test/Analysis/Checkers/WebKit/uncounted-obj-arg.cpp
+++ b/clang/test/Analysis/Checkers/WebKit/uncounted-obj-arg.cpp
@@ -368,6 +368,11 @@ class RefCounted {
   }
   RefPtr<RefCounted> trivial66() { return children[0]; }
   Ref<RefCounted> trivial67() { return *children[0]; }
+  struct point {
+    double x;
+    double y;
+  };
+  void trivial68() { point pt = { 1.0 }; }
 
   static RefCounted& singleton() {
     static RefCounted s_RefCounted;
@@ -554,6 +559,7 @@ class UnrelatedClass {
     getFieldTrivial().trivial65(); // no-warning
     getFieldTrivial().trivial66()->trivial6(); // no-warning
     getFieldTrivial().trivial67()->trivial6(); // no-warning
+    getFieldTrivial().trivial68(); // no-warning
 
     RefCounted::singleton().trivial18(); // no-warning
     RefCounted::singleton().someFunction(); // no-warning
diff --git a/clang/test/Analysis/Checkers/WebKit/uncounted-obj-arg.mm b/clang/test/Analysis/Checkers/WebKit/uncounted-obj-arg.mm
index 9ad1880e9d118..08319016023e3 100644
--- a/clang/test/Analysis/Checkers/WebKit/uncounted-obj-arg.mm
+++ b/clang/test/Analysis/Checkers/WebKit/uncounted-obj-arg.mm
@@ -1,11 +1,14 @@
 // RUN: %clang_analyze_cc1 -analyzer-checker=alpha.webkit.UncountedCallArgsChecker -verify %s
-// expected-no-diagnostics
 
 #import "mock-types.h"
 #import "mock-system-header.h"
 #import "../../Inputs/system-header-simulator-for-objc-dealloc.h"
 
-@interface Foo : NSObject
+@interface Foo : NSObject {
+  const Ref<RefCountable> _obj1;
+  const RefPtr<RefCountable> _obj2;
+  Ref<RefCountable> _obj3;
+}
 
 @property (nonatomic, readonly) RefPtr<RefCountable> countable;
 
@@ -17,6 +20,11 @@ @implementation Foo
 
 - (void)execute {
   self._protectedRefCountable->method();
+  _obj1->method();
+  _obj1.get().method();
+  (*_obj2).method();
+  _obj3->method();
+  // expected-warning@-1{{Call argument for 'this' parameter is uncounted and unsafe}}
 }
 
 - (RefPtr<RefCountable>)_protectedRefCountable {
@@ -30,6 +38,7 @@ - (void)execute {
   void ref() const;
   void deref() const;
   Ref<RefCountedObject> copy() const;
+  void method();
 };
 
 @interface WrapperObj : NSObject
diff --git a/clang/test/Analysis/Inputs/ctu-test-import-failure-import.cpp b/clang/test/Analysis/Inputs/ctu-test-import-failure-import.cpp
new file mode 100644
index 0000000000000..ffe860870a214
--- /dev/null
+++ b/clang/test/Analysis/Inputs/ctu-test-import-failure-import.cpp
@@ -0,0 +1,52 @@
+namespace std {
+inline namespace __cxx11 {
+template <typename _CharT, typename = int, typename = _CharT>
+class basic_string;
+}
+template <typename, typename> class basic_istream;
+template <typename> struct __get_first_arg;
+struct allocator_traits {
+  using type = __get_first_arg<int>;
+};
+} // namespace std
+namespace std {
+inline namespace __cxx11 {
+template <typename, typename, typename> class basic_string {
+  allocator_traits _M_allocated_capacity;
+  void _M_assign();
+};
+} // namespace __cxx11
+} // namespace std
+namespace std {
+template <typename _CharT, typename _Alloc> void operator!=(_Alloc, _CharT);
+template <typename _CharT, typename _Traits, typename _Alloc>
+basic_istream<_CharT, _Traits> &getline(basic_istream<_CharT, _Traits> &,
+                                        basic_string<_CharT, _Traits, _Alloc> &,
+                                        _CharT);
+} // namespace std
+namespace std {
+template <typename _CharT, typename _Traits, typename _Alloc>
+void basic_string<_CharT, _Traits, _Alloc>::_M_assign() {
+  this != 0;
+}
+template <typename _CharT, typename _Traits, typename _Alloc>
+basic_istream<_CharT, _Traits> &getline(basic_istream<_CharT, _Traits> &,
+                                        basic_string<_CharT, _Traits, _Alloc> &,
+                                        _CharT) {}
+} // namespace std
+struct CommandLineOptionDefinition {
+  void *OutAddress;
+};
+struct CommandLineCommand {
+  CommandLineOptionDefinition Options;
+};
+namespace CommandLine {
+extern const CommandLineCommand RootCommands[];
+extern const int RootExamples[];
+} // namespace CommandLine
+using utf8 = char;
+using u8string = std::basic_string<utf8>;
+u8string _rct2DataPath;
+CommandLineOptionDefinition StandardOptions{&_rct2DataPath};
+const CommandLineCommand CommandLine::RootCommands[]{StandardOptions};
+const int CommandLine::RootExamples[]{};
diff --git a/clang/test/Analysis/Inputs/ctu-test-import-failure-import.cpp.externalDefMap.ast-dump.txt b/clang/test/Analysis/Inputs/ctu-test-import-failure-import.cpp.externalDefMap.ast-dump.txt
new file mode 100644
index 0000000000000..6ffb3795d3e36
--- /dev/null
+++ b/clang/test/Analysis/Inputs/ctu-test-import-failure-import.cpp.externalDefMap.ast-dump.txt
@@ -0,0 +1,5 @@
+47:c:@N@std@S@allocator_traits@F@allocator_traits# ctu-test-import-failure-import.cpp.ast
+29:c:@N@CommandLine@RootCommands ctu-test-import-failure-import.cpp.ast
+55:c:@N@std@N@__cxx11@ST>3#T#T#T@basic_string@F@_M_assign# ctu-test-import-failure-import.cpp.ast
+97:c:@S@CommandLineOptionDefinition@F@CommandLineOptionDefinition#&1$@S@CommandLineOptionDefinition# ctu-test-import-failure-import.cpp.ast
+29:c:@N@CommandLine@RootExamples ctu-test-import-failure-import.cpp.ast
\ No newline at end of file
diff --git a/clang/test/Analysis/NewDelete-checker-test.cpp b/clang/test/Analysis/NewDelete-checker-test.cpp
index 21b4cf817b5df..06754f669b1e6 100644
--- a/clang/test/Analysis/NewDelete-checker-test.cpp
+++ b/clang/test/Analysis/NewDelete-checker-test.cpp
@@ -441,3 +441,31 @@ void testLeakBecauseNTTPIsNotDeallocation() {
   void* p = ::operator new(10);
   deallocate_via_nttp<not_free>(p);
 }  // leak-warning{{Potential leak of memory pointed to by 'p'}}
+
+namespace optional_union {
+  template <typename T>
+  class unique_ptr {
+    T *q;
+  public:
+    unique_ptr() : q(new T) {}
+    ~unique_ptr() {
+      delete q;
+    }
+  };
+
+  union custom_union_t {
+    unique_ptr<int> present;
+    char notpresent;
+    custom_union_t() : present(unique_ptr<int>()) {}
+    ~custom_union_t() {}
+  };
+
+  void testUnionCorrect() {
+    custom_union_t a;
+    a.present.~unique_ptr<int>();
+  }
+
+  void testUnionLeak() {
+    custom_union_t a;
+  } // leak-warning{{Potential leak of memory pointed to by 'a.present.q'}}
+}
diff --git a/clang/test/Analysis/anonymous-decls.cpp b/clang/test/Analysis/anonymous-decls.cpp
new file mode 100644
index 0000000000000..211184523aa51
--- /dev/null
+++ b/clang/test/Analysis/anonymous-decls.cpp
@@ -0,0 +1,89 @@
+// RUN: %clang_analyze_cc1 -analyzer-checker=debug.DumpCFG -triple x86_64-apple-darwin12 -std=c++20 %s 2>&1 | FileCheck %s
+
+struct A {
+  static A a;
+  char b;
+  friend bool operator==(A, A) = default;
+};
+bool _ = A() == A::a;
+
+// FIXME: steps 1 and 5 show anonymous function parameters are
+// not handled correctly.
+
+// CHECK-LABEL: bool operator==(A, A) noexcept = default
+// CHECK-NEXT: [B2 (ENTRY)]
+// CHECK-NEXT:    Succs (1): B1
+// CHECK:      [B1]
+// CHECK-NEXT:    1: function-parameter-0-0
+// CHECK-NEXT:    2: [B1.1].b
+// CHECK-NEXT:    3: [B1.2] (ImplicitCastExpr, LValueToRValue, char)
+// CHECK-NEXT:    4: [B1.3] (ImplicitCastExpr, IntegralCast, int)
+// CHECK-NEXT:    5: function-parameter-0-1
+// CHECK-NEXT:    6: [B1.5].b
+// CHECK-NEXT:    7: [B1.6] (ImplicitCastExpr, LValueToRValue, char)
+// CHECK-NEXT:    8: [B1.7] (ImplicitCastExpr, IntegralCast, int)
+// CHECK-NEXT:    9: [B1.4] == [B1.8]
+// CHECK-NEXT:   10: return [B1.9];
+// CHECK-NEXT:    Preds (1): B2
+// CHECK-NEXT:    Succs (1): B0
+// CHECK:      [B0 (EXIT)]
+// CHECK-NEXT:    Preds (1): B1
+
+namespace std {
+template <class> struct iterator_traits;
+template <class, class> struct pair;
+template <class _Tp> struct iterator_traits<_Tp *> {
+  typedef _Tp &reference;
+};
+template <long, class> struct tuple_element;
+template <class> struct tuple_size;
+template <class _T1, class _T2> struct tuple_size<pair<_T1, _T2>> {
+  static const int value = 2;
+};
+template <class _T1, class _T2> struct tuple_element<0, pair<_T1, _T2>> {
+  using type = _T1;
+};
+template <class _T1, class _T2> struct tuple_element<1, pair<_T1, _T2>> {
+  using type = _T2;
+};
+template <long _Ip, class _T1, class _T2>
+tuple_element<_Ip, pair<_T1, _T2>>::type get(pair<_T1, _T2> &);
+struct __wrap_iter {
+  iterator_traits<pair<int, int> *>::reference operator*();
+  void operator++();
+};
+bool operator!=(__wrap_iter, __wrap_iter);
+struct vector {
+  __wrap_iter begin();
+  __wrap_iter end();
+};
+} // namespace std
+int main() {
+  std::vector v;
+  for (auto &[a, b] : v)
+    ;
+}
+
+// FIXME: On steps 8 and 14, a decomposition is referred by name, which they never have.
+
+// CHECK-LABEL: int main()
+// CHECK:      [B3]
+// CHECK-NEXT:   1: operator*
+// CHECK-NEXT:   2: [B3.1] (ImplicitCastExpr, FunctionToPointerDecay, iterator_traits<pair<int, int> *>::reference (*)(void))
+// CHECK-NEXT:   3: __begin1
+// CHECK-NEXT:   4: * [B3.3] (OperatorCall)
+// CHECK-NEXT:   5: auto &;
+// CHECK-NEXT:   6: get<0UL>
+// CHECK-NEXT:   7: [B3.6] (ImplicitCastExpr, FunctionToPointerDecay, typename tuple_element<0L, pair<int, int> >::type (*)(pair<int, int> &))
+// CHECK-NEXT:   8: decomposition-a-b
+// CHECK-NEXT:   9: [B3.7]([B3.8])
+// CHECK-NEXT:  10: [B3.9]
+// CHECK-NEXT:  11: std::tuple_element<0, std::pair<int, int>>::type a = get<0UL>(decomposition-a-b);
+// CHECK-NEXT:  12: get<1UL>
+// CHECK-NEXT:  13: [B3.12] (ImplicitCastExpr, FunctionToPointerDecay, typename tuple_element<1L, pair<int, int> >::type (*)(pair<int, int> &))
+// CHECK-NEXT:  14: decomposition-a-b
+// CHECK-NEXT:  15: [B3.13]([B3.14])
+// CHECK-NEXT:  16: [B3.15]
+// CHECK-NEXT:  17: std::tuple_element<1, std::pair<int, int>>::type b = get<1UL>(decomposition-a-b);
+// CHECK-NEXT:   Preds (1): B1
+// CHECK-NEXT:   Succs (1): B2
diff --git a/clang/test/Analysis/array-bound-v2-constraint-check.c b/clang/test/Analysis/array-bound-v2-constraint-check.c
deleted file mode 100644
index 91f748e655ce2..0000000000000
--- a/clang/test/Analysis/array-bound-v2-constraint-check.c
+++ /dev/null
@@ -1,112 +0,0 @@
-// RUN: %clang_analyze_cc1 -analyzer-checker=core,unix.Malloc,alpha.security.ArrayBoundV2,debug.ExprInspection \
-// RUN:   -analyzer-config eagerly-assume=false -verify %s
-
-void clang_analyzer_eval(int);
-void clang_analyzer_printState(void);
-
-typedef typeof(sizeof(int)) size_t;
-const char a[] = "abcd"; // extent: 5 bytes
-
-void symbolic_size_t_and_int0(size_t len) {
-  (void)a[len + 1]; // no-warning
-  // We infered that the 'len' must be in a specific range to make the previous indexing valid.
-  // len: [0,3]
-  clang_analyzer_eval(len <= 3); // expected-warning {{TRUE}}
-  clang_analyzer_eval(len <= 2); // expected-warning {{UNKNOWN}}
-}
-
-void symbolic_size_t_and_int1(size_t len) {
-  (void)a[len]; // no-warning
-  // len: [0,4]
-  clang_analyzer_eval(len <= 4); // expected-warning {{TRUE}}
-  clang_analyzer_eval(len <= 3); // expected-warning {{UNKNOWN}}
-}
-
-void symbolic_size_t_and_int2(size_t len) {
-  (void)a[len - 1]; // no-warning
-  // len: [1,5]
-  clang_analyzer_eval(1 <= len && len <= 5); // expected-warning {{TRUE}}
-  clang_analyzer_eval(2 <= len);             // expected-warning {{UNKNOWN}}
-  clang_analyzer_eval(len <= 4);             // expected-warning {{UNKNOWN}}
-}
-
-void symbolic_uint_and_int0(unsigned len) {
-  (void)a[len + 1]; // no-warning
-  // len: [0,3]
-  clang_analyzer_eval(0 <= len && len <= 3); // expected-warning {{TRUE}}
-  clang_analyzer_eval(1 <= len);             // expected-warning {{UNKNOWN}}
-  clang_analyzer_eval(len <= 2);             // expected-warning {{UNKNOWN}}
-}
-
-void symbolic_uint_and_int1(unsigned len) {
-  (void)a[len]; // no-warning
-  // len: [0,4]
-  clang_analyzer_eval(0 <= len && len <= 4); // expected-warning {{TRUE}}
-  clang_analyzer_eval(1 <= len);             // expected-warning {{UNKNOWN}}
-  clang_analyzer_eval(len <= 3);             // expected-warning {{UNKNOWN}}
-}
-void symbolic_uint_and_int2(unsigned len) {
-  (void)a[len - 1]; // no-warning
-  // len: [1,5]
-  clang_analyzer_eval(1 <= len && len <= 5); // expected-warning {{TRUE}}
-  clang_analyzer_eval(2 <= len);             // expected-warning {{UNKNOWN}}
-  clang_analyzer_eval(len <= 4);             // expected-warning {{UNKNOWN}}
-}
-
-void symbolic_int_and_int0(int len) {
-  (void)a[len + 1]; // no-warning
-  // len: [-1,3]
-  clang_analyzer_eval(-1 <= len && len <= 3); // expected-warning {{TRUE}}
-  clang_analyzer_eval(0 <= len);              // expected-warning {{UNKNOWN}}
-  clang_analyzer_eval(len <= 2);              // expected-warning {{UNKNOWN}}
-}
-void symbolic_int_and_int1(int len) {
-  (void)a[len]; // no-warning
-  // len: [0,4]
-  clang_analyzer_eval(0 <= len && len <= 4); // expected-warning {{TRUE}}
-  clang_analyzer_eval(1 <= len);             // expected-warning {{UNKNOWN}}
-  clang_analyzer_eval(len <= 3);             // expected-warning {{UNKNOWN}}
-}
-void symbolic_int_and_int2(int len) {
-  (void)a[len - 1]; // no-warning
-  // len: [1,5]
-  clang_analyzer_eval(1 <= len && len <= 5); // expected-warning {{TRUE}}
-  clang_analyzer_eval(2 <= len);             // expected-warning {{UNKNOWN}}
-  clang_analyzer_eval(len <= 4);             // expected-warning {{UNKNOWN}}
-}
-
-void symbolic_longlong_and_int0(long long len) {
-  (void)a[len + 1]; // no-warning
-  // len: [-1,3]
-  clang_analyzer_eval(-1 <= len && len <= 3); // expected-warning {{TRUE}}
-  clang_analyzer_eval(0 <= len);              // expected-warning {{UNKNOWN}}
-  clang_analyzer_eval(len <= 2);              // expected-warning {{UNKNOWN}}
-}
-
-void *malloc(size_t);
-void free(void *);
-void symbolic_longlong_and_int0_dynamic_extent(long long len) {
-  char *b = malloc(5);
-  (void)b[len + 1]; // no-warning
-  // len: [-1,3]
-  clang_analyzer_eval(-1 <= len && len <= 3); // expected-warning {{TRUE}}
-  clang_analyzer_eval(0 <= len);              // expected-warning {{UNKNOWN}}
-  clang_analyzer_eval(len <= 2);              // expected-warning {{UNKNOWN}}
-  free(b);
-}
-
-void symbolic_longlong_and_int1(long long len) {
-  (void)a[len]; // no-warning
-  // len: [0,4]
-  clang_analyzer_eval(0 <= len && len <= 4); // expected-warning {{TRUE}}
-  clang_analyzer_eval(1 <= len);             // expected-warning {{UNKNOWN}}
-  clang_analyzer_eval(len <= 3);             // expected-warning {{UNKNOWN}}
-}
-
-void symbolic_longlong_and_int2(long long len) {
-  (void)a[len - 1]; // no-warning
-  // len: [1,5]
-  clang_analyzer_eval(1 <= len && len <= 5); // expected-warning {{TRUE}}
-  clang_analyzer_eval(2 <= len);             // expected-warning {{UNKNOWN}}
-  clang_analyzer_eval(len <= 4);             // expected-warning {{UNKNOWN}}
-}
diff --git a/clang/test/Analysis/bugfix-124477.m b/clang/test/Analysis/bugfix-124477.m
new file mode 100644
index 0000000000000..80820f4c93444
--- /dev/null
+++ b/clang/test/Analysis/bugfix-124477.m
@@ -0,0 +1,39 @@
+// RUN: %clang_analyze_cc1 -analyzer-checker=core,apiModeling,nullability.NullableDereferenced,nullability.NullabilityBase -x objective-c %s
+/*
+  This test is reduced from a static analyzer crash. The bug causing
+  the crash is explained in #124477.  It can only be triggered in some
+  rare cases so please do not modify this reproducer.
+*/
+
+#pragma clang assume_nonnull begin
+# 15 "some-sys-header.h" 1 3
+@class NSArray, NSObject;
+
+@interface Base
+@property (readonly, copy) NSArray *array;
+@end
+
+#pragma clang assume_nonnull end
+# 8 "this-file.m" 2
+
+
+@interface Test : Base
+
+@property (readwrite, copy, nullable) NSObject *label;
+@property (readwrite, strong, nullable) Test * field;
+
+- (void)f;
+
+@end
+
+@implementation Test
+- (void)f
+{
+  NSObject * X;
+
+  for (NSObject *ele in self.field.array) {}
+  self.label = X;  
+}
+@end
+
+
diff --git a/clang/test/Analysis/copy-elision.cpp b/clang/test/Analysis/copy-elision.cpp
index cd941854fc7f1..e69f52f351f1b 100644
--- a/clang/test/Analysis/copy-elision.cpp
+++ b/clang/test/Analysis/copy-elision.cpp
@@ -154,20 +154,26 @@ class ClassWithoutDestructor {
   void push() { v.push(this); }
 };
 
+// Two warnings on no-elide: arg v holds the address of the temporary, and we
+// are returning an object which holds v which holds the address of the temporary 
 ClassWithoutDestructor make1(AddressVector<ClassWithoutDestructor> &v) {
-  return ClassWithoutDestructor(v);
+  return ClassWithoutDestructor(v); // no-elide-warning{{Address of stack memory associated with temporary object of type 'ClassWithoutDestructor' returned to caller}}
   // no-elide-warning@-1 {{Address of stack memory associated with temporary \
 object of type 'ClassWithoutDestructor' is still \
 referred to by the caller variable 'v' upon returning to the caller}}
 }
+// Two warnings on no-elide: arg v holds the address of the temporary, and we
+// are returning an object which holds v which holds the address of the temporary 
 ClassWithoutDestructor make2(AddressVector<ClassWithoutDestructor> &v) {
-  return make1(v);
+  return make1(v); // no-elide-warning{{Address of stack memory associated with temporary object of type 'ClassWithoutDestructor' returned to caller}}
   // no-elide-warning@-1 {{Address of stack memory associated with temporary \
 object of type 'ClassWithoutDestructor' is still \
 referred to by the caller variable 'v' upon returning to the caller}}
 }
+// Two warnings on no-elide: arg v holds the address of the temporary, and we
+// are returning an object which holds v which holds the address of the temporary 
 ClassWithoutDestructor make3(AddressVector<ClassWithoutDestructor> &v) {
-  return make2(v);
+  return make2(v); // no-elide-warning{{Address of stack memory associated with temporary object of type 'ClassWithoutDestructor' returned to caller}}
   // no-elide-warning@-1 {{Address of stack memory associated with temporary \
 object of type 'ClassWithoutDestructor' is still \
 referred to by the caller variable 'v' upon returning to the caller}}
@@ -298,21 +304,26 @@ to by the caller variable 'v' upon returning to the caller}}
 #endif
 }
 
-
+// Two warnings on no-elide: arg v holds the address of the temporary, and we
+// are returning an object which holds v which holds the address of the temporary
 ClassWithDestructor make1(AddressVector<ClassWithDestructor> &v) {
-  return ClassWithDestructor(v);
+  return ClassWithDestructor(v); // no-elide-warning{{Address of stack memory associated with temporary object of type 'ClassWithDestructor' returned to caller}}
   // no-elide-warning@-1 {{Address of stack memory associated with temporary \
 object of type 'ClassWithDestructor' is still referred \
 to by the caller variable 'v' upon returning to the caller}}
 }
+// Two warnings on no-elide: arg v holds the address of the temporary, and we
+// are returning an object which holds v which holds the address of the temporary
 ClassWithDestructor make2(AddressVector<ClassWithDestructor> &v) {
-  return make1(v);
+  return make1(v); // no-elide-warning{{Address of stack memory associated with temporary object of type 'ClassWithDestructor' returned to caller}}
   // no-elide-warning@-1 {{Address of stack memory associated with temporary \
 object of type 'ClassWithDestructor' is still referred \
 to by the caller variable 'v' upon returning to the caller}}
 }
+// Two warnings on no-elide: arg v holds the address of the temporary, and we
+// are returning an object which holds v which holds the address of the temporary
 ClassWithDestructor make3(AddressVector<ClassWithDestructor> &v) {
-  return make2(v);
+  return make2(v); // no-elide-warning{{Address of stack memory associated with temporary object of type 'ClassWithDestructor' returned to caller}}
   // no-elide-warning@-1 {{Address of stack memory associated with temporary \
 object of type 'ClassWithDestructor' is still referred \
 to by the caller variable 'v' upon returning to the caller}}
diff --git a/clang/test/Analysis/ctu-test-import-failure.cpp b/clang/test/Analysis/ctu-test-import-failure.cpp
new file mode 100644
index 0000000000000..2295a66538fc9
--- /dev/null
+++ b/clang/test/Analysis/ctu-test-import-failure.cpp
@@ -0,0 +1,34 @@
+// RUN: rm -rf %t && mkdir %t
+// RUN: mkdir -p %t/ctudir
+// RUN: %clang_cc1 -triple x86_64-pc-linux-gnu -std=c++17 \
+// RUN:   -emit-pch -o %t/ctudir/ctu-test-import-failure-import.cpp.ast %S/Inputs/ctu-test-import-failure-import.cpp
+// RUN: cp %S/Inputs/ctu-test-import-failure-import.cpp.externalDefMap.ast-dump.txt %t/ctudir/externalDefMap.txt
+// RUN: %clang_cc1 -triple x86_64-pc-linux-gnu -std=c++17 -analyze \
+// RUN:   -analyzer-checker=core \
+// RUN:   -analyzer-config experimental-enable-naive-ctu-analysis=true \
+// RUN:   -analyzer-config ctu-dir=%t/ctudir \
+// RUN:   -verify %s
+
+// Check that importing this code does not cause crash.
+// Import intentionally fails because mismatch of '__get_first_arg'.
+
+namespace std {
+inline namespace __cxx11 {}
+template <typename _CharT, typename> class basic_istream;
+struct __get_first_arg;
+inline namespace __cxx11 {
+template <typename, typename, typename> class basic_string;
+}
+template <typename _CharT, typename _Traits, typename _Alloc>
+basic_istream<_CharT, _Traits> &getline(basic_istream<_CharT, _Traits> &,
+                                        basic_string<_CharT, _Traits, _Alloc> &,
+                                        _CharT) {}
+} // namespace std
+namespace CommandLine {
+extern const int RootExamples[];
+}
+
+// expected-warning@Inputs/ctu-test-import-failure-import.cpp:14{{incompatible definitions}}
+// expected-warning@Inputs/ctu-test-import-failure-import.cpp:14{{incompatible definitions}}
+// expected-note@Inputs/ctu-test-import-failure-import.cpp:14{{no corresponding field here}}
+// expected-note@Inputs/ctu-test-import-failure-import.cpp:14{{no corresponding field here}}
diff --git a/clang/test/Analysis/dtor-union.cpp b/clang/test/Analysis/dtor-union.cpp
new file mode 100644
index 0000000000000..dac366e6f9df8
--- /dev/null
+++ b/clang/test/Analysis/dtor-union.cpp
@@ -0,0 +1,38 @@
+// RUN: %clang_analyze_cc1 -analyzer-checker=core,debug.ExprInspection -analyzer-config c++-inlining=destructors -verify -std=c++11 %s
+// RUN: %clang_analyze_cc1 -analyzer-checker=core,debug.ExprInspection -analyzer-config c++-inlining=destructors -verify -std=c++17 %s
+
+void clang_analyzer_eval(bool);
+
+struct InlineDtor {
+  static int cnt;
+  static int dtorCalled;
+  ~InlineDtor() {
+    ++dtorCalled;
+  }
+};
+
+int InlineDtor::cnt = 0;
+int InlineDtor::dtorCalled = 0;
+
+void testUnionDtor() {
+  static int unionDtorCalled;
+  InlineDtor::cnt = 0;
+  InlineDtor::dtorCalled = 0;
+  unionDtorCalled = 0;
+  {
+      union UnionDtor {
+          InlineDtor kind1;
+          char kind2;
+          ~UnionDtor() { unionDtorCalled++; }
+      };
+      UnionDtor u1{.kind1{}};
+      UnionDtor u2{.kind2{}};
+      auto u3 = new UnionDtor{.kind1{}};
+      auto u4 = new UnionDtor{.kind2{}};
+      delete u3;
+      delete u4;
+  }
+
+  clang_analyzer_eval(unionDtorCalled == 4); // expected-warning {{TRUE}}
+  clang_analyzer_eval(InlineDtor::dtorCalled == 0); // expected-warning {{TRUE}}
+}
diff --git a/clang/test/Analysis/ftime-trace-bind.cpp b/clang/test/Analysis/ftime-trace-bind.cpp
new file mode 100644
index 0000000000000..de121e4316e03
--- /dev/null
+++ b/clang/test/Analysis/ftime-trace-bind.cpp
@@ -0,0 +1,24 @@
+// RUN: %clang_analyze_cc1 -analyzer-checker=core %s -ftime-trace=%t.raw.json -ftime-trace-granularity=0 -verify
+// RUN: %python -c 'import json, sys; print(json.dumps(json.load(sys.stdin), indent=4))' < %t.raw.json > %t.formatted.json
+// RUN: FileCheck --input-file=%t.formatted.json --check-prefix=CHECK %s
+
+// CHECK:          "name": "RegionStoreManager::bindArray",
+// CHECK-NEXT:     "args": {
+//
+// The below does not necessarily follow immediately,
+// depending on what parts of the array are initialized first.
+//
+// CHECK:              "detail": "'arr[0][1]'"
+// CHECK-NEXT:     }
+//
+// CHECK:              "detail": "'arr[0]'"
+// CHECK-NEXT:     }
+//
+// CHECK:              "detail": "'arr'"
+// CHECK-NEXT:     }
+
+int f() {
+    int arr[2][2][2] = {{{1, 2}, {3, 4}}, {{5, 6}, {7, 8}}};
+    return arr[1][0][1];
+}
+// expected-no-diagnostics
diff --git a/clang/test/Analysis/ftime-trace-removeDead.cpp b/clang/test/Analysis/ftime-trace-removeDead.cpp
new file mode 100644
index 0000000000000..2ffd2ec45908c
--- /dev/null
+++ b/clang/test/Analysis/ftime-trace-removeDead.cpp
@@ -0,0 +1,17 @@
+// RUN: %clang_analyze_cc1 -analyzer-checker=core %s -ftime-trace=%t.raw.json -ftime-trace-granularity=0 -verify
+// RUN: %python -c 'import json, sys; print(json.dumps(json.load(sys.stdin), indent=4))' < %t.raw.json > %t.formatted.json
+// RUN: FileCheck --input-file=%t.formatted.json --check-prefix=CHECK %s
+
+// The trace file is rather large, but it should contain at least one scope for removeDead:
+//
+// CHECK:          "name": "ExprEngine::removeDead"
+
+bool coin();
+int f() {
+    int x = 0;
+    int y = 0;
+    while (coin()) {
+        x = 1;
+    }
+    return x / y; // expected-warning{{Division by zero}}
+}
diff --git a/clang/test/Analysis/ftime-trace.cpp b/clang/test/Analysis/ftime-trace.cpp
new file mode 100644
index 0000000000000..2c369a9bf781e
--- /dev/null
+++ b/clang/test/Analysis/ftime-trace.cpp
@@ -0,0 +1,56 @@
+// RUN: %clang_analyze_cc1 -analyzer-checker=core %s -ftime-trace=%t.raw.json -ftime-trace-granularity=0 -verify
+// RUN: %python -c 'import json, sys; print(json.dumps(json.load(sys.stdin), indent=4))' < %t.raw.json > %t.formatted.json
+// RUN: FileCheck --input-file=%t.formatted.json --check-prefix=CHECK %s
+
+// The trace file is rather large, but it should contain at least the duration of the analysis of 'f':
+//
+// CHECK:          "name": "HandleCode f",
+// CHECK-NEXT:     "args": {
+// CHECK-NEXT:         "detail": "f()",
+// CHECK-NEXT:         "file": "{{.+}}ftime-trace.cpp",
+// CHECK-NEXT:         "line": {{[0-9]+}}
+// CHECK-NEXT:     }
+
+// If any reports are found, "flushing" their equivalence class (EQC) is a separate action:
+//
+// CHECK:          "name": "Flushing EQC Division by zero",
+// CHECK-NEXT:     "args": {
+// CHECK-NEXT:         "detail": "core.DivideZero",
+// CHECK-NEXT:         "file": "{{.+}}ftime-trace.cpp",
+// CHECK-NEXT:         "line": {{[0-9]+}}
+// CHECK-NEXT:     }
+
+// The trace also contains durations of each step, but they are so short that they are not reliably present
+// in each run. However, they are also aggregated into Total *, for example:
+//
+// CHECK:          "name": "Total Loc PostStmt",
+// CHECK-NEXT:     "args": {
+// CHECK-NEXT:         "count": {{[0-9]+}},
+// CHECK-NEXT:         "avg ms": {{[0-9]+}}
+// CHECK-NEXT:     }
+
+// Additionally, the trace lists checker hook points (again, relying on totals here):
+//
+// CHECK:          "name": "Total CheckerManager::runCheckersForStmt (Pre)",
+// CHECK-NEXT:     "args": {
+// CHECK-NEXT:         "count": {{[0-9]+}},
+// CHECK-NEXT:         "avg ms": {{[0-9]+}}
+// CHECK-NEXT:     }
+
+// Finally, each checker call back is also present:
+//
+// CHECK:          "name": "Total Stmt:core.DivideZero",
+// CHECK-NEXT:     "args": {
+// CHECK-NEXT:         "count": {{[0-9]+}},
+// CHECK-NEXT:         "avg ms": {{[0-9]+}}
+// CHECK-NEXT:     }
+
+bool coin();
+int f() {
+    int x = 0;
+    int y = 0;
+    while (coin()) {
+        x = 1;
+    }
+    return x / y; // expected-warning{{Division by zero}}
+}
diff --git a/clang/test/Analysis/index-type.c b/clang/test/Analysis/index-type.c
index 997d45c1e5aba..818806c4aff3b 100644
--- a/clang/test/Analysis/index-type.c
+++ b/clang/test/Analysis/index-type.c
@@ -1,5 +1,5 @@
-// RUN: %clang_analyze_cc1 -triple x86_64-apple-darwin10 -analyzer-checker=core,alpha.security.ArrayBoundV2 -Wno-implicit-function-declaration -verify %s
-// RUN: %clang_analyze_cc1 -triple i386-apple-darwin10 -analyzer-checker=core,alpha.security.ArrayBoundV2 -Wno-implicit-function-declaration -DM32 -verify %s
+// RUN: %clang_analyze_cc1 -triple x86_64-apple-darwin10 -analyzer-checker=core,security.ArrayBound -Wno-implicit-function-declaration -verify %s
+// RUN: %clang_analyze_cc1 -triple i386-apple-darwin10 -analyzer-checker=core,security.ArrayBound -Wno-implicit-function-declaration -DM32 -verify %s
 // expected-no-diagnostics
 
 #define UINT_MAX (~0u)
diff --git a/clang/test/Analysis/live-stmts.cpp b/clang/test/Analysis/live-stmts.cpp
index c60f522588e39..33b8d59305d3d 100644
--- a/clang/test/Analysis/live-stmts.cpp
+++ b/clang/test/Analysis/live-stmts.cpp
@@ -1,3 +1,6 @@
+// Flaky on aarch64: http://llvm.org/PR126619
+// UNSUPPORTED: target=aarch64{{.*}}
+
 // RUN: %clang_analyze_cc1 -w -analyzer-checker=debug.DumpLiveExprs %s 2>&1\
 // RUN:   | FileCheck %s
 
diff --git a/clang/test/Analysis/misc-ps-region-store.m b/clang/test/Analysis/misc-ps-region-store.m
index a882e7eb0dc90..bd8b3350ba6da 100644
--- a/clang/test/Analysis/misc-ps-region-store.m
+++ b/clang/test/Analysis/misc-ps-region-store.m
@@ -1,5 +1,5 @@
-// RUN: %clang_analyze_cc1 -triple i386-apple-darwin9 -analyzer-checker=core,alpha.core.CastToStruct,alpha.security.ReturnPtrRange,alpha.security.ArrayBound -verify -fblocks -Wno-objc-root-class -Wno-strict-prototypes -Wno-error=implicit-function-declaration %s
-// RUN: %clang_analyze_cc1 -triple x86_64-apple-darwin9 -DTEST_64 -analyzer-checker=core,alpha.core.CastToStruct,alpha.security.ReturnPtrRange,alpha.security.ArrayBound -verify -fblocks   -Wno-objc-root-class -Wno-strict-prototypes -Wno-error=implicit-function-declaration %s
+// RUN: %clang_analyze_cc1 -triple i386-apple-darwin9 -analyzer-checker=core,alpha.core.CastToStruct,alpha.security.ReturnPtrRange,security.ArrayBound -verify -fblocks -Wno-objc-root-class -Wno-strict-prototypes -Wno-error=implicit-function-declaration %s
+// RUN: %clang_analyze_cc1 -triple x86_64-apple-darwin9 -DTEST_64 -analyzer-checker=core,alpha.core.CastToStruct,alpha.security.ReturnPtrRange,security.ArrayBound -verify -fblocks   -Wno-objc-root-class -Wno-strict-prototypes -Wno-error=implicit-function-declaration %s
 
 typedef long unsigned int size_t;
 void *memcpy(void *, const void *, size_t);
@@ -928,7 +928,7 @@ void pr6288_pos(int z) {
   int *px[1];
   int i;
   for (i = 0; i < z; i++)
-    px[i] = &x[i]; // expected-warning{{Access out-of-bound array element (buffer overflow)}}
+    px[i] = &x[i]; // expected-warning{{Out of bound access to memory after the end of 'px'}}
   *(px[0]) = 0; // expected-warning{{Dereference of undefined pointer value}}
 }
 
@@ -976,12 +976,17 @@ - (void) rdar7817800_baz {
 }
 @end
 
-// PR 6036 - This test case triggered a crash inside StoreManager::CastRegion because the size
-// of 'unsigned long (*)[0]' is 0.
+// PR 6036 - This test case triggered a crash inside StoreManager::CastRegion
+// because the size of 'unsigned long (*)[0]' is 0.
+// NOTE: This old crash was probably triggered via the old alpha checker
+// `alpha.security.ArrayBound` (which was logic that's different from the
+// current `security.ArrayBound`). Although that code was removed, it's worth
+// to keep this testcase as a generic example of a zero-sized type.
 struct pr6036_a { int pr6036_b; };
 struct pr6036_c;
 void u132monitk (struct pr6036_c *pr6036_d) {
-  (void) ((struct pr6036_a *) (unsigned long (*)[0]) ((char *) pr6036_d - 1))->pr6036_b; // expected-warning{{Casting a non-structure type to a structure type and accessing a field can lead to memory access errors or data corruption}}
+  (void) ((struct pr6036_a *) (unsigned long (*)[0]) ((char *) pr6036_d - 1))->pr6036_b;
+  // expected-warning@-1 {{Casting a non-structure type to a structure type and accessing a field can lead to memory access errors or data corruption}}
 }
 
 // ?-expressions used as a base of a member expression should be treated as an lvalue
diff --git a/clang/test/Analysis/no-outofbounds.c b/clang/test/Analysis/no-outofbounds.c
deleted file mode 100644
index 15155729067e9..0000000000000
--- a/clang/test/Analysis/no-outofbounds.c
+++ /dev/null
@@ -1,32 +0,0 @@
-// RUN: %clang_analyze_cc1 -analyzer-checker=core,alpha.core,alpha.unix,alpha.security.ArrayBound -verify %s
-// expected-no-diagnostics
-
-//===----------------------------------------------------------------------===//
-// This file tests cases where we should not flag out-of-bounds warnings.
-//===----------------------------------------------------------------------===//
-
-void f(void) {
-  long x = 0;
-  char *y = (char*) &x;
-  char c = y[0] + y[1] + y[2]; // no-warning
-  short *z = (short*) &x;
-  short s = z[0] + z[1]; // no-warning
-}
-
-void g(void) {
-  int a[2];
-  char *b = (char*)a;
-  b[3] = 'c'; // no-warning
-}
-
-typedef typeof(sizeof(int)) size_t;
-void *malloc(size_t);
-void free(void *);
-
-void field(void) {
-  struct vec { size_t len; int data[0]; };
-  struct vec *a = malloc(sizeof(struct vec) + 10*sizeof(int));
-  a->len = 10;
-  a->data[1] = 5; // no-warning
-  free(a);
-}
diff --git a/clang/test/Analysis/null-deref-path-notes.cpp b/clang/test/Analysis/null-deref-path-notes.cpp
index c7b0619e297b3..a37bbfe41a2c7 100644
--- a/clang/test/Analysis/null-deref-path-notes.cpp
+++ b/clang/test/Analysis/null-deref-path-notes.cpp
@@ -23,3 +23,38 @@ void c::f(B &g, int &i) {
   f(h, b); // expected-note{{Calling 'c::f'}}
 }
 }
+
+namespace GH124975 {
+void no_crash_in_br_visitors(int *p) {
+  if (p) {}
+  // expected-note@-1 {{Assuming 'p' is null}}
+  // expected-note@-2 {{Taking false branch}}
+
+  extern bool ExternLocalCoin;
+  // expected-note@+2 {{Assuming 'ExternLocalCoin' is false}}
+  // expected-note@+1 {{Taking false branch}}
+  if (ExternLocalCoin)
+    return;
+
+  *p = 4;
+  // expected-warning@-1 {{Dereference of null pointer (loaded from variable 'p')}}
+  // expected-note@-2    {{Dereference of null pointer (loaded from variable 'p')}}
+}
+
+// Thread local variables are implicitly static, so let's test them too.
+void thread_local_alternative(int *p) {
+  if (p) {}
+  // expected-note@-1 {{Assuming 'p' is null}}
+  // expected-note@-2 {{Taking false branch}}
+
+  thread_local bool ThreadLocalCoin;
+  // expected-note@+2 {{'ThreadLocalCoin' is false}}
+  // expected-note@+1 {{Taking false branch}}
+  if (ThreadLocalCoin)
+    return;
+
+  *p = 4;
+  // expected-warning@-1 {{Dereference of null pointer (loaded from variable 'p')}}
+  // expected-note@-2    {{Dereference of null pointer (loaded from variable 'p')}}
+}
+} // namespace GH124975
diff --git a/clang/test/Analysis/out-of-bounds-constraint-check.c b/clang/test/Analysis/out-of-bounds-constraint-check.c
new file mode 100644
index 0000000000000..df48c8c170713
--- /dev/null
+++ b/clang/test/Analysis/out-of-bounds-constraint-check.c
@@ -0,0 +1,112 @@
+// RUN: %clang_analyze_cc1 -analyzer-checker=core,unix.Malloc,security.ArrayBound,debug.ExprInspection \
+// RUN:   -analyzer-config eagerly-assume=false -verify %s
+
+void clang_analyzer_eval(int);
+void clang_analyzer_printState(void);
+
+typedef typeof(sizeof(int)) size_t;
+const char a[] = "abcd"; // extent: 5 bytes
+
+void symbolic_size_t_and_int0(size_t len) {
+  (void)a[len + 1]; // no-warning
+  // We infered that the 'len' must be in a specific range to make the previous indexing valid.
+  // len: [0,3]
+  clang_analyzer_eval(len <= 3); // expected-warning {{TRUE}}
+  clang_analyzer_eval(len <= 2); // expected-warning {{UNKNOWN}}
+}
+
+void symbolic_size_t_and_int1(size_t len) {
+  (void)a[len]; // no-warning
+  // len: [0,4]
+  clang_analyzer_eval(len <= 4); // expected-warning {{TRUE}}
+  clang_analyzer_eval(len <= 3); // expected-warning {{UNKNOWN}}
+}
+
+void symbolic_size_t_and_int2(size_t len) {
+  (void)a[len - 1]; // no-warning
+  // len: [1,5]
+  clang_analyzer_eval(1 <= len && len <= 5); // expected-warning {{TRUE}}
+  clang_analyzer_eval(2 <= len);             // expected-warning {{UNKNOWN}}
+  clang_analyzer_eval(len <= 4);             // expected-warning {{UNKNOWN}}
+}
+
+void symbolic_uint_and_int0(unsigned len) {
+  (void)a[len + 1]; // no-warning
+  // len: [0,3]
+  clang_analyzer_eval(0 <= len && len <= 3); // expected-warning {{TRUE}}
+  clang_analyzer_eval(1 <= len);             // expected-warning {{UNKNOWN}}
+  clang_analyzer_eval(len <= 2);             // expected-warning {{UNKNOWN}}
+}
+
+void symbolic_uint_and_int1(unsigned len) {
+  (void)a[len]; // no-warning
+  // len: [0,4]
+  clang_analyzer_eval(0 <= len && len <= 4); // expected-warning {{TRUE}}
+  clang_analyzer_eval(1 <= len);             // expected-warning {{UNKNOWN}}
+  clang_analyzer_eval(len <= 3);             // expected-warning {{UNKNOWN}}
+}
+void symbolic_uint_and_int2(unsigned len) {
+  (void)a[len - 1]; // no-warning
+  // len: [1,5]
+  clang_analyzer_eval(1 <= len && len <= 5); // expected-warning {{TRUE}}
+  clang_analyzer_eval(2 <= len);             // expected-warning {{UNKNOWN}}
+  clang_analyzer_eval(len <= 4);             // expected-warning {{UNKNOWN}}
+}
+
+void symbolic_int_and_int0(int len) {
+  (void)a[len + 1]; // no-warning
+  // len: [-1,3]
+  clang_analyzer_eval(-1 <= len && len <= 3); // expected-warning {{TRUE}}
+  clang_analyzer_eval(0 <= len);              // expected-warning {{UNKNOWN}}
+  clang_analyzer_eval(len <= 2);              // expected-warning {{UNKNOWN}}
+}
+void symbolic_int_and_int1(int len) {
+  (void)a[len]; // no-warning
+  // len: [0,4]
+  clang_analyzer_eval(0 <= len && len <= 4); // expected-warning {{TRUE}}
+  clang_analyzer_eval(1 <= len);             // expected-warning {{UNKNOWN}}
+  clang_analyzer_eval(len <= 3);             // expected-warning {{UNKNOWN}}
+}
+void symbolic_int_and_int2(int len) {
+  (void)a[len - 1]; // no-warning
+  // len: [1,5]
+  clang_analyzer_eval(1 <= len && len <= 5); // expected-warning {{TRUE}}
+  clang_analyzer_eval(2 <= len);             // expected-warning {{UNKNOWN}}
+  clang_analyzer_eval(len <= 4);             // expected-warning {{UNKNOWN}}
+}
+
+void symbolic_longlong_and_int0(long long len) {
+  (void)a[len + 1]; // no-warning
+  // len: [-1,3]
+  clang_analyzer_eval(-1 <= len && len <= 3); // expected-warning {{TRUE}}
+  clang_analyzer_eval(0 <= len);              // expected-warning {{UNKNOWN}}
+  clang_analyzer_eval(len <= 2);              // expected-warning {{UNKNOWN}}
+}
+
+void *malloc(size_t);
+void free(void *);
+void symbolic_longlong_and_int0_dynamic_extent(long long len) {
+  char *b = malloc(5);
+  (void)b[len + 1]; // no-warning
+  // len: [-1,3]
+  clang_analyzer_eval(-1 <= len && len <= 3); // expected-warning {{TRUE}}
+  clang_analyzer_eval(0 <= len);              // expected-warning {{UNKNOWN}}
+  clang_analyzer_eval(len <= 2);              // expected-warning {{UNKNOWN}}
+  free(b);
+}
+
+void symbolic_longlong_and_int1(long long len) {
+  (void)a[len]; // no-warning
+  // len: [0,4]
+  clang_analyzer_eval(0 <= len && len <= 4); // expected-warning {{TRUE}}
+  clang_analyzer_eval(1 <= len);             // expected-warning {{UNKNOWN}}
+  clang_analyzer_eval(len <= 3);             // expected-warning {{UNKNOWN}}
+}
+
+void symbolic_longlong_and_int2(long long len) {
+  (void)a[len - 1]; // no-warning
+  // len: [1,5]
+  clang_analyzer_eval(1 <= len && len <= 5); // expected-warning {{TRUE}}
+  clang_analyzer_eval(2 <= len);             // expected-warning {{UNKNOWN}}
+  clang_analyzer_eval(len <= 4);             // expected-warning {{UNKNOWN}}
+}
diff --git a/clang/test/Analysis/out-of-bounds-diagnostics.c b/clang/test/Analysis/out-of-bounds-diagnostics.c
index 65bc28f58276f..524fa4e2aaaf7 100644
--- a/clang/test/Analysis/out-of-bounds-diagnostics.c
+++ b/clang/test/Analysis/out-of-bounds-diagnostics.c
@@ -1,5 +1,5 @@
 // RUN: %clang_analyze_cc1 -Wno-array-bounds -analyzer-output=text        \
-// RUN:     -analyzer-checker=core,alpha.security.ArrayBoundV2,unix.Malloc,optin.taint -verify %s
+// RUN:     -analyzer-checker=core,security.ArrayBound,unix.Malloc,optin.taint -verify %s
 
 int TenElements[10];
 
@@ -231,7 +231,16 @@ int arrayOfStructsArrow(void) {
   // expected-note@-2 {{Access of 'itemArray' at index 35, while it holds only 20 'struct item' elements}}
 }
 
+char convertedScalar(long long var) {
+  char *p = ((char*)&var);
+  (void) p[3]; // no-warning
+  return p[13];
+  // expected-warning@-1 {{Out of bound access to memory after the end of 'var'}}
+  // expected-note@-2 {{Access of 'var' at index 13, while it holds only 8 'char' elements}}
+}
+
 short convertedArray(void) {
+  (void) ((short*)TenElements)[17]; // no-warning
   return ((short*)TenElements)[47];
   // expected-warning@-1 {{Out of bound access to memory after the end of 'TenElements'}}
   // expected-note@-2 {{Access of 'TenElements' at index 47, while it holds only 20 'short' elements}}
@@ -268,23 +277,41 @@ int intFromStringDivisible(void) {
 
 typedef __typeof(sizeof(int)) size_t;
 void *malloc(size_t size);
+void free(void *mem);
 
 int *mallocRegion(void) {
   int *mem = (int*)malloc(2*sizeof(int));
 
+  mem[1] = 48; // no-warning
+
   mem[3] = -2;
   // expected-warning@-1 {{Out of bound access to memory after the end of the heap area}}
   // expected-note@-2 {{Access of the heap area at index 3, while it holds only 2 'int' elements}}
   return mem;
 }
 
+typedef struct { size_t len; int data[0]; } vec_t;
+
+void mallocFlexibleArray(void) {
+  vec_t *v = malloc(sizeof(vec_t) + 10 * sizeof(int));
+  v->len = 10;
+  v->data[1] = 5; // no-warning
+  v->data[11] = 99;
+  // TODO: Here ideally we would expect
+  //     {{Out of bound access to memory after the end of the heap area}}
+  //     {{Access of the heap area at index 11, while it holds only 10 'int' elements}}
+  // but the analyzer cannot (yet) deduce the size of the flexible array member
+  // from the size of the whole allocated area.
+  free(v);
+}
+
 int *custom_calloc(size_t a, size_t b) {
   size_t res;
 
   return __builtin_mul_overflow(a, b, &res) ? 0 : malloc(res);
 }
 
-int *mallocRegionOverflow(void) {
+int *mallocMulOverflow(void) {
   int *mem = (int*)custom_calloc(10, sizeof(int));
 
   mem[20] = 10;
diff --git a/clang/test/Analysis/out-of-bounds-new.cpp b/clang/test/Analysis/out-of-bounds-new.cpp
index f541bdf810d79..4e5442422bff4 100644
--- a/clang/test/Analysis/out-of-bounds-new.cpp
+++ b/clang/test/Analysis/out-of-bounds-new.cpp
@@ -1,4 +1,4 @@
-// RUN: %clang_analyze_cc1 -std=c++11 -Wno-array-bounds -analyzer-checker=unix,core,alpha.security.ArrayBoundV2 -verify %s
+// RUN: %clang_analyze_cc1 -std=c++11 -Wno-array-bounds -analyzer-checker=unix,core,security.ArrayBound -verify %s
 
 // Tests doing an out-of-bounds access after the end of an array using:
 // - constant integer index
diff --git a/clang/test/Analysis/out-of-bounds-notes.c b/clang/test/Analysis/out-of-bounds-notes.c
index 391089b6a35d8..7256beb7e893e 100644
--- a/clang/test/Analysis/out-of-bounds-notes.c
+++ b/clang/test/Analysis/out-of-bounds-notes.c
@@ -1,5 +1,5 @@
 // RUN: %clang_analyze_cc1 -Wno-array-bounds -analyzer-output=text        \
-// RUN:     -analyzer-checker=core,alpha.security.ArrayBoundV2,unix.Malloc,optin.taint -verify %s
+// RUN:     -analyzer-checker=core,security.ArrayBound,unix.Malloc,optin.taint -verify %s
 
 int TenElements[10];
 
diff --git a/clang/test/Analysis/out-of-bounds.c b/clang/test/Analysis/out-of-bounds.c
index 1f771c2b3bd13..923797200d0b4 100644
--- a/clang/test/Analysis/out-of-bounds.c
+++ b/clang/test/Analysis/out-of-bounds.c
@@ -1,4 +1,4 @@
-// RUN: %clang_analyze_cc1 -Wno-array-bounds -analyzer-checker=core,alpha.security.ArrayBoundV2,debug.ExprInspection -verify %s
+// RUN: %clang_analyze_cc1 -Wno-array-bounds -analyzer-checker=core,security.ArrayBound,debug.ExprInspection -verify %s
 
 void clang_analyzer_eval(int);
 
@@ -153,7 +153,7 @@ void test_assume_after_access(unsigned long x) {
 int *get_symbolic(void);
 void test_underflow_symbolic(void) {
   int *buf = get_symbolic();
-  buf[-1] = 0; // no-warning;
+  buf[-1] = 0; // no-warning
 }
 
 // But warn if we understand the internal memory layout of a symbolic region.
diff --git a/clang/test/Analysis/outofbound-notwork.c b/clang/test/Analysis/outofbound-notwork.c
index cf2239cee1301..1318c07bbf2a8 100644
--- a/clang/test/Analysis/outofbound-notwork.c
+++ b/clang/test/Analysis/outofbound-notwork.c
@@ -1,4 +1,4 @@
-// RUN: %clang_analyze_cc1 -Wno-array-bounds -analyzer-checker=core,alpha.security.ArrayBound -verify %s
+// RUN: %clang_analyze_cc1 -Wno-array-bounds -analyzer-checker=core,security.ArrayBound -verify %s
 // XFAIL: *
 
 // Once we better handle modeling of sizes of VLAs, we can pull this back
@@ -9,7 +9,7 @@ void sizeof_vla(int a) {
     char x[a];
     int y[sizeof(x)];
     y[4] = 4; // no-warning
-    y[5] = 5; // expected-warning{{out-of-bound}}
+    y[5] = 5; // expected-warning{{Out of bounds access}}
   }
 }
 
@@ -18,7 +18,7 @@ void sizeof_vla_2(int a) {
     char x[a];
     int y[sizeof(x) / sizeof(char)];
     y[4] = 4; // no-warning
-    y[5] = 5; // expected-warning{{out-of-bound}}
+    y[5] = 5; // expected-warning{{Out of bounds access}}
   }
 }
 
@@ -27,6 +27,6 @@ void sizeof_vla_3(int a) {
     char x[a];
     int y[sizeof(*&*&*&x)];
     y[4] = 4; // no-warning
-    y[5] = 5; // expected-warning{{out-of-bound}}
+    y[5] = 5; // expected-warning{{Out of bounds access}}
   }
 }
diff --git a/clang/test/Analysis/outofbound.c b/clang/test/Analysis/outofbound.c
index 009cf33f61309..d3d8ff2b2f0ed 100644
--- a/clang/test/Analysis/outofbound.c
+++ b/clang/test/Analysis/outofbound.c
@@ -1,7 +1,7 @@
 // RUN: %clang_analyze_cc1 -Wno-array-bounds -verify %s \
 // RUN:   -analyzer-checker=core \
 // RUN:   -analyzer-checker=unix \
-// RUN:   -analyzer-checker=alpha.security.ArrayBound \
+// RUN:   -analyzer-checker=security.ArrayBound \
 // RUN:   -analyzer-config unix.DynamicMemoryModeling:Optimistic=true
 
 typedef __typeof(sizeof(int)) size_t;
@@ -11,12 +11,12 @@ void *calloc(size_t, size_t);
 char f1(void) {
   char* s = "abcd";
   char c = s[4]; // no-warning
-  return s[5] + c; // expected-warning{{Access out-of-bound array element (buffer overflow)}}
+  return s[5] + c; // expected-warning{{Out of bound access to memory after}}
 }
 
 void f2(void) {
   int *p = malloc(12);
-  p[3] = 4; // expected-warning{{Access out-of-bound array element (buffer overflow)}}
+  p[3] = 4; // expected-warning{{Out of bound access to memory after}}
 }
 
 struct three_words {
@@ -31,7 +31,7 @@ void f3(void) {
   struct three_words a, *p;
   p = &a;
   p[0] = a; // no-warning
-  p[1] = a; // expected-warning{{Access out-of-bound array element (buffer overflow)}}
+  p[1] = a; // expected-warning{{Out of bound access to memory after}}
 }
 
 void f4(void) {
@@ -39,31 +39,33 @@ void f4(void) {
   struct three_words a, *p = (struct three_words *)&c;
   p[0] = a; // no-warning
   p[1] = a; // no-warning
-  p[2] = a; // expected-warning{{Access out-of-bound array element (buffer overflow)}}
+  p[2] = a; // should warn
+  // FIXME: This is an overflow, but currently security.ArrayBound only checks
+  // that the _beginning_ of the accessed element is within bounds.
 }
 
 void f5(void) {
   char *p = calloc(2,2);
   p[3] = '.'; // no-warning
-  p[4] = '!'; // expected-warning{{out-of-bound}}
+  p[4] = '!'; // expected-warning{{Out of bound access}}
 }
 
 void f6(void) {
   char a[2];
   int *b = (int*)a;
-  b[1] = 3; // expected-warning{{out-of-bound}}
+  b[1] = 3; // expected-warning{{Out of bound access}}
 }
 
 void f7(void) {
   struct three_words a;
-  a.c[3] = 1; // expected-warning{{out-of-bound}}
+  a.c[3] = 1; // expected-warning{{Out of bound access}}
 }
 
 void vla(int a) {
   if (a == 5) {
     int x[a];
     x[4] = 4; // no-warning
-    x[5] = 5; // expected-warning{{out-of-bound}}
+    x[5] = 5; // expected-warning{{Out of bound access}}
   }
 }
 
@@ -71,14 +73,14 @@ void alloca_region(int a) {
   if (a == 5) {
     char *x = __builtin_alloca(a);
     x[4] = 4; // no-warning
-    x[5] = 5; // expected-warning{{out-of-bound}}
+    x[5] = 5; // expected-warning{{Out of bound access}}
   }
 }
 
 int symbolic_index(int a) {
   int x[2] = {1, 2};
   if (a == 2) {
-    return x[a]; // expected-warning{{out-of-bound}}
+    return x[a]; // expected-warning{{Out of bound access}}
   }
   return 0;
 }
@@ -86,7 +88,7 @@ int symbolic_index(int a) {
 int symbolic_index2(int a) {
   int x[2] = {1, 2};
   if (a < 0) {
-    return x[a]; // expected-warning{{out-of-bound}}
+    return x[a]; // expected-warning{{Out of bound access}}
   }
   return 0;
 }
@@ -120,7 +122,7 @@ int overflow_binary_search(double in) {
     } else {
       eee += 1;
     }
-    if (in < ins[eee]) { // expected-warning {{Access out-of-bound array element (buffer overflow)}}
+    if (in < ins[eee]) { // expected-warning {{Out of bound access}}
       eee -= 1;
     }
   }
diff --git a/clang/test/Analysis/rdar-6541136-region.c b/clang/test/Analysis/rdar-6541136-region.c
deleted file mode 100644
index f1a3a48a5fe4a..0000000000000
--- a/clang/test/Analysis/rdar-6541136-region.c
+++ /dev/null
@@ -1,27 +0,0 @@
-// RUN: %clang_analyze_cc1 -verify -analyzer-checker=core,alpha.security.ArrayBound %s
-
-struct tea_cheese { unsigned magic; };
-typedef struct tea_cheese kernel_tea_cheese_t;
-extern kernel_tea_cheese_t _wonky_gesticulate_cheese;
-
-// This test case exercises the ElementRegion::getRValueType() logic.
-
-void test1( void ) {
-  kernel_tea_cheese_t *wonky = &_wonky_gesticulate_cheese;
-  struct load_wine *cmd = (void*) &wonky[1];
-  cmd = cmd;
-  char *p = (void*) &wonky[1];
-  kernel_tea_cheese_t *q = &wonky[1];
-  // This test case tests both the RegionStore logic (doesn't crash) and
-  // the out-of-bounds checking.  We don't expect the warning for now since
-  // out-of-bound checking is temporarily disabled.
-  kernel_tea_cheese_t r = *q; // expected-warning{{Access out-of-bound array element (buffer overflow)}}
-}
-
-void test1_b( void ) {
-  kernel_tea_cheese_t *wonky = &_wonky_gesticulate_cheese;
-  struct load_wine *cmd = (void*) &wonky[1];
-  cmd = cmd;
-  char *p = (void*) &wonky[1];
-  *p = 1;  // expected-warning{{Access out-of-bound array element (buffer overflow)}}
-}
diff --git a/clang/test/Analysis/runtime-regression.c b/clang/test/Analysis/runtime-regression.c
index 55988e9df5ee0..9fa0017d5e470 100644
--- a/clang/test/Analysis/runtime-regression.c
+++ b/clang/test/Analysis/runtime-regression.c
@@ -1,5 +1,5 @@
 // RUN: %clang_analyze_cc1 %s \
-// RUN:   -analyzer-checker=core,alpha.security.ArrayBoundV2 \
+// RUN:   -analyzer-checker=core,security.ArrayBound \
 // RUN:   -analyzer-checker=debug.ExprInspection \
 // RUN:   -triple x86_64-unknown-linux-gnu \
 // RUN:   -verify
diff --git a/clang/test/Analysis/stack-addr-ps.cpp b/clang/test/Analysis/stack-addr-ps.cpp
index 73e9dbeca460f..bf988d0a16959 100644
--- a/clang/test/Analysis/stack-addr-ps.cpp
+++ b/clang/test/Analysis/stack-addr-ps.cpp
@@ -90,6 +90,18 @@ int *mf() {
   return &x; // expected-warning{{Address of stack memory associated with local variable 's1' returned}} expected-warning {{address of stack memory associated with local variable 's1' returned}}
 }
 
+int *return_assign_expr_leak() {
+  int x = 1;
+  int *y;
+  return y = &x; // expected-warning{{Address of stack memory associated with local variable 'x' returned}}
+}
+
+// Additional diagnostic from -Wreturn-stack-address in the simple case?
+int *return_comma_separated_expressions_leak() {
+  int x = 1;
+  return (x=14), &x; // expected-warning{{Address of stack memory associated with local variable 'x' returned to caller}} expected-warning{{address of stack memory associated with local variable 'x' returned}}
+}
+
 void *lf() {
     label:
     void *const &x = &&label; // expected-note {{binding reference variable 'x' here}}
@@ -152,6 +164,18 @@ namespace rdar13296133 {
   }
 } // namespace rdar13296133
 
+void* ret_cpp_static_cast(short x) {
+  return static_cast<void*>(&x); // expected-warning {{Address of stack memory associated with local variable 'x' returned to caller}} expected-warning {{address of stack memory}}
+}
+
+int* ret_cpp_reinterpret_cast(double x) {
+  return reinterpret_cast<int*>(&x); // expected-warning {{Address of stack memory associated with local variable 'x' returned to caller}} expected-warning {{address of stack me}}
+}
+
+int* ret_cpp_const_cast(const int x) {
+  return const_cast<int*>(&x);  // expected-warning {{Address of stack memory associated with local variable 'x' returned to caller}} expected-warning {{address of stack memory}}
+}
+
 void write_stack_address_to(char **q) {
   char local;
   *q = &local;
@@ -178,6 +202,11 @@ void test_copy_elision() {
   C c1 = make1();
 }
 
+C&& return_bind_rvalue_reference_to_temporary() {
+  return C(); // expected-warning{{Address of stack memory associated with temporary object of type 'C' returned to caller}}
+  // expected-warning@-1{{returning reference to local temporary object}} -Wreturn-stack-address
+}
+
 namespace leaking_via_direct_pointer {
 void* returned_direct_pointer_top() {
   int local = 42;
@@ -251,7 +280,7 @@ void* lambda_to_context_direct_pointer_uncalled() {
     int local = 42;
     p = &local; // no-warning: analyzed only as top-level, ignored explicitly by the checker
   };
-  return new MyFunction(&lambda);
+  return new MyFunction(&lambda); // expected-warning{{Address of stack memory associated with local variable 'lambda' returned to caller}}
 }
 
 void lambda_to_context_direct_pointer_lifetime_extended() {
@@ -340,6 +369,13 @@ void param_ptr_to_ptr_to_ptr_callee(void*** ppp) {
   **ppp = &local; // expected-warning{{local variable 'local' is still referred to by the caller variable 'pp'}}
 }
 
+void ***param_ptr_to_ptr_to_ptr_return(void ***ppp) {
+  int local = 0;
+  **ppp = &local;
+  return ppp;
+  // expected-warning@-1 {{Address of stack memory associated with local variable 'local' is still referred to by the caller variable 'ppp' upon returning to the caller.  This will be a dangling reference}}
+}
+
 void param_ptr_to_ptr_to_ptr_caller(void** pp) {
   param_ptr_to_ptr_to_ptr_callee(&pp);
 }
@@ -410,16 +446,16 @@ void** returned_arr_of_ptr_top() {
   int* p = &local;
   void** arr = new void*[2];
   arr[1] = p;
-  return arr;
-} // no-warning False Negative
+  return arr; // expected-warning{{Address of stack memory associated with local variable 'local' returned to caller}}
+}
 
 void** returned_arr_of_ptr_callee() {
   int local = 42;
   int* p = &local;
   void** arr = new void*[2];
   arr[1] = p;
-  return arr;
-} // no-warning False Negative
+  return arr; // expected-warning{{Address of stack memory associated with local variable 'local' returned to caller}}
+}
 
 void returned_arr_of_ptr_caller() {
   void** arr = returned_arr_of_ptr_callee();
@@ -466,16 +502,16 @@ void** returned_arr_of_ptr_top(int idx) {
   int* p = &local;
   void** arr = new void*[2];
   arr[idx] = p;
-  return arr;
-} // no-warning False Negative
+  return arr; // expected-warning{{Address of stack memory associated with local variable 'local' returned to caller}}
+}
 
 void** returned_arr_of_ptr_callee(int idx) {
   int local = 42;
   int* p = &local;
   void** arr = new void*[2];
   arr[idx] = p;
-  return arr;
-} // no-warning False Negative
+  return arr; // expected-warning{{Address of stack memory associated with local variable 'local' returned to caller}}
+}
 
 void returned_arr_of_ptr_caller(int idx) {
   void** arr = returned_arr_of_ptr_callee(idx);
@@ -525,14 +561,25 @@ S returned_struct_with_ptr_top() {
   int local = 42;
   S s;
   s.p = &local;
-  return s;
-} // no-warning False Negative, requires traversing returned LazyCompoundVals
+  return s; // expected-warning{{Address of stack memory associated with local variable 'local' returned to caller}}
+}
 
 S returned_struct_with_ptr_callee() {
   int local = 42;
   S s;
   s.p = &local;
-  return s; // expected-warning{{'local' is still referred to by the caller variable 's'}}
+  return s; // expected-warning {{Address of stack memory associated with local variable 'local' returned to caller}} expected-warning{{Address of stack memory associated with local variable 'local' is still referred to by the caller variable 's' upon returning to the caller.  This will be a dangling reference}}
+}
+
+S leak_through_field_of_returned_object() {
+  int local = 14;
+  S s{&local};
+  return s; // expected-warning {{Address of stack memory associated with local variable 'local' returned to caller}}
+}
+
+S leak_through_compound_literal() {
+  int local = 0;
+  return (S) { &local }; // expected-warning {{Address of stack memory associated with local variable 'local' returned to caller}}
 }
 
 void returned_struct_with_ptr_caller() {
@@ -555,6 +602,30 @@ void static_struct_with_ptr() {
 }
 } // namespace leaking_via_struct_with_ptr
 
+namespace leaking_via_nested_structs_with_ptr {
+struct Inner {
+  int *ptr;
+};
+
+struct Outer {
+  Inner I;
+};
+
+struct Deriving : public Outer {};
+
+Outer leaks_through_nested_objects() {
+  int local = 0;
+  Outer O{&local};
+  return O; // expected-warning {{Address of stack memory associated with local variable 'local' returned to caller}}
+}
+
+Deriving leaks_through_base_objects() {
+  int local = 0;
+  Deriving D{&local};
+  return D; // expected-warning {{Address of stack memory associated with local variable 'local' returned to caller}}
+}
+} // namespace leaking_via_nested_structs_with_ptr
+
 namespace leaking_via_ref_to_struct_with_ptr {
 struct S {
   int* p;
@@ -613,15 +684,15 @@ S* returned_ptr_to_struct_with_ptr_top() {
   int local = 42;
   S* s = new S;
   s->p = &local;
-  return s;
-} // no-warning False Negative
+  return s; // expected-warning {{Address of stack memory associated with local variable 'local' returned to caller}}
+}
 
 S* returned_ptr_to_struct_with_ptr_callee() {
   int local = 42;
   S* s = new S;
   s->p = &local;
-  return s;
-} // no-warning False Negative
+  return s; // expected-warning {{Address of stack memory associated with local variable 'local' returned to caller}}
+}
 
 void returned_ptr_to_struct_with_ptr_caller() {
   S* s = returned_ptr_to_struct_with_ptr_callee();
@@ -676,15 +747,15 @@ S* returned_ptr_to_struct_with_ptr_top() {
   int local = 42;
   S* s = new S[2];
   s[1].p = &local;
-  return s;
-} // no-warning False Negative
+  return s; // expected-warning {{Address of stack memory associated with local variable 'local' returned to caller}}
+}
 
 S* returned_ptr_to_struct_with_ptr_callee() {
   int local = 42;
   S* s = new S[2];
   s[1].p = &local;
-  return s;
-} // no-warning  False Negative
+  return s; // expected-warning {{Address of stack memory associated with local variable 'local' returned to caller}}
+}
 
 void returned_ptr_to_struct_with_ptr_caller() {
   S* s = returned_ptr_to_struct_with_ptr_callee();
@@ -877,3 +948,137 @@ void top_malloc_no_crash_fn() {
   free(pptr);
 }
 } // namespace alloca_region_pointer
+
+// These all warn with -Wreturn-stack-address also
+namespace return_address_of_true_positives {
+int* ret_local_addrOf() {
+  int x = 1;
+  return &*&x; // expected-warning {{Address of stack memory associated with local variable 'x' returned to caller}} expected-warning {{address of stack memory associated with local variable 'x' returned}}
+}
+
+int* ret_local_addrOf_paren() {
+  int x = 1;
+  return (&(*(&x))); // expected-warning {{Address of stack memory associated with local variable 'x' returned to caller}} expected-warning {{address of stack memory associated with local variable 'x' returned}}
+}
+
+int* ret_local_addrOf_ptr_arith() {
+  int x = 1;
+  return &*(&x+1); // expected-warning {{Address of stack memory associated with local variable 'x' returned to caller}} expected-warning {{address of stack memory associated with local variable 'x' returned}}
+}
+
+int* ret_local_addrOf_ptr_arith2() {
+  int x = 1;
+  return &*(&x+1); // expected-warning {{Address of stack memory associated with local variable 'x' returned to caller}} expected-warning {{address of stack memory associated with local variable 'x' returned}}
+}
+
+int* ret_local_field() {
+  struct { int x; } a;
+  return &a.x; // expected-warning {{Address of stack memory associated with local variable 'a' returned to caller}} expected-warning {{address of stack memory associated with local variable 'a' returned}}
+}
+
+int& ret_local_field_ref() {
+  struct { int x; } a;
+  return a.x; // expected-warning {{Address of stack memory associated with local variable 'a' returned to caller}} expected-warning {{reference to stack memory associated with local variable 'a' returned}}
+}
+} //namespace return_address_of_true_positives
+
+namespace true_negatives_return_expressions {
+struct Container { int *x; };
+
+int test2() {
+  int x = 14;
+  return x; // no-warning
+}
+
+void return_void() {
+  return void(); // no-warning
+}
+
+int return_assign_expr_safe() {
+  int y, x = 14;
+  return y = x; // no-warning
+}
+
+int return_comma_separated_expressions_safe() {
+  int x = 1;
+  int *y;
+  return (y=&x), (x=15); // no-warning
+}
+
+int return_comma_separated_expressions_container_safe() {
+  int x = 1;
+  Container Other;
+  return Other = Container{&x}, x = 14; // no-warning
+}
+
+int make_x();
+int return_symbol_safe() {
+  int x = make_x();
+  clang_analyzer_dump(x); // expected-warning-re {{conj_$2{int, {{.+}}}}}
+  return x; // no-warning
+}
+
+int *return_symbolic_region_safe(int **ptr) {
+  return *ptr; // no-warning
+}
+
+int *return_arg_ptr(int *arg) {
+  return arg; // no-warning
+}
+
+// This has a false positive with -Wreturn-stack-address, but CSA should not
+// warn on this
+int *return_conditional_never_false(int *arg) {
+  int x = 14;
+  return true ? arg : &x; // expected-warning {{address of stack memory associated with local variable 'x' returned}}
+}
+
+// This has a false positive with -Wreturn-stack-address, but CSA should not
+// warn on this
+int *return_conditional_never_true(int *arg) {
+  int x = 14;
+  return false ? &x : arg; // expected-warning {{address of stack memory associated with local variable 'x' returned}}
+}
+
+int *return_conditional_path_split(int *arg, bool b) {
+  int x = 14;
+  return b ? arg : &x; // expected-warning {{Address of stack memory associated with local variable 'x' returned to caller}} expected-warning {{address of stack memory associated with local variable 'x' returned}} -Wreturn-stack-address
+}
+
+// compare of two symbolic regions
+// maybe in some implementation, make_ptr returns interior pointers that are comparable
+int *make_ptr();
+bool return_symbolic_exxpression() {
+  int *a = make_ptr();
+  int *b = make_ptr();
+  return a < b; // no-warning
+}
+
+int *return_static() {
+  static int x = 0;
+  return &x; // no-warning
+}
+
+int* return_cpp_reinterpret_cast_no_warning(long x) {
+  return reinterpret_cast<int*>(x); // no-warning
+}
+}
+
+// TODO: The tail call expression tree must not hold a reference to an arg or stack local:
+// "The lifetimes of all local variables and function parameters end immediately before the 
+// [tail] call to the function. This means that it is undefined behaviour to pass a pointer or 
+// reference to a local variable to the called function, which is not the case without the 
+// attribute."
+// These only warn from -Wreturn-stack-address for now
+namespace with_attr_musttail {
+void TakesIntAndPtr(int, int *);
+void PassAddressOfLocal(int a, int *b) {
+  int c;
+  [[clang::musttail]] return TakesIntAndPtr(0, &c); // expected-warning {{address of stack memory associated with local variable 'c' pass\
+ed to musttail function}} False-negative on CSA
+}
+void PassAddressOfParam(int a, int *b) {
+  [[clang::musttail]] return TakesIntAndPtr(0, &a); // expected-warning {{address of stack memory associated with parameter 'a' passed to\
+ musttail function}} False-negative on CSA
+}
+} // namespace with_attr_musttail
diff --git a/clang/test/Analysis/stackaddrleak.c b/clang/test/Analysis/stackaddrleak.c
index 5f508275ba9c8..f8101525401b0 100644
--- a/clang/test/Analysis/stackaddrleak.c
+++ b/clang/test/Analysis/stackaddrleak.c
@@ -56,3 +56,15 @@ void assignAsBool(void) {
   int x;
   b = &x;
 } // no-warning
+
+int *f(int* p __attribute__((lifetimebound)));
+int *g() {
+  int i;
+  return f(&i); // expected-warning {{address of stack memory associated with local variable 'i' returned}}
+}
+
+int *f_no_lifetime_bound(int *p);
+int *g_no_lifetime_bound() {
+  int i = 0;
+  return f_no_lifetime_bound(&i); // no-warning
+}
diff --git a/clang/test/Analysis/stackaddrleak.cpp b/clang/test/Analysis/stackaddrleak.cpp
index 3daffb35a6cd9..df202a2fee776 100644
--- a/clang/test/Analysis/stackaddrleak.cpp
+++ b/clang/test/Analysis/stackaddrleak.cpp
@@ -18,7 +18,7 @@ struct myfunction {
 myfunction create_func() {
   int n;
   auto c = [&n] {};
-  return c; // expected-warning {{Address of stack memory associated with local variable 'n' is still referred to by a temporary object on the stack upon returning to the caller.  This will be a dangling reference}}
+  return c; // expected-warning {{Address of stack memory associated with local variable 'n' returned to caller}} expected-warning{{Address of stack memory associated with local variable 'n' is still referred to by a temporary object on the stack upon returning to the caller.  This will be a dangling reference}}
 }
 void gh_66221() {
   create_func()();
diff --git a/clang/test/Analysis/taint-diagnostic-visitor.c b/clang/test/Analysis/taint-diagnostic-visitor.c
index 223df9951fd6b..afe7117db7150 100644
--- a/clang/test/Analysis/taint-diagnostic-visitor.c
+++ b/clang/test/Analysis/taint-diagnostic-visitor.c
@@ -1,4 +1,4 @@
-// RUN: %clang_cc1 -analyze -analyzer-checker=optin.taint,core,alpha.security.ArrayBoundV2 -analyzer-output=text -verify %s
+// RUN: %clang_cc1 -analyze -analyzer-checker=optin.taint,core,security.ArrayBound -analyzer-output=text -verify %s
 
 // This file is for testing enhanced diagnostics produced by the GenericTaintChecker
 
diff --git a/clang/test/Analysis/taint-generic.c b/clang/test/Analysis/taint-generic.c
index ad5a99fe8b3a3..3c520612c5d9b 100644
--- a/clang/test/Analysis/taint-generic.c
+++ b/clang/test/Analysis/taint-generic.c
@@ -3,7 +3,7 @@
 // RUN:   -analyzer-checker=optin.taint.GenericTaint \
 // RUN:   -analyzer-checker=optin.taint.TaintedDiv \
 // RUN:   -analyzer-checker=core \
-// RUN:   -analyzer-checker=alpha.security.ArrayBoundV2 \
+// RUN:   -analyzer-checker=security.ArrayBound \
 // RUN:   -analyzer-checker=debug.ExprInspection \
 // RUN:   -analyzer-config \
 // RUN:     optin.taint.TaintPropagation:Config=%S/Inputs/taint-generic-config.yaml
@@ -14,7 +14,7 @@
 // RUN:   -analyzer-checker=optin.taint.GenericTaint \
 // RUN:   -analyzer-checker=optin.taint.TaintedDiv \
 // RUN:   -analyzer-checker=core \
-// RUN:   -analyzer-checker=alpha.security.ArrayBoundV2 \
+// RUN:   -analyzer-checker=security.ArrayBound \
 // RUN:   -analyzer-checker=debug.ExprInspection \
 // RUN:   -analyzer-config \
 // RUN:     optin.taint.TaintPropagation:Config=%S/Inputs/taint-generic-config.yaml
diff --git a/clang/test/Analysis/taint-generic.cpp b/clang/test/Analysis/taint-generic.cpp
index 881c5baf889f6..8836e1d3d2d98 100644
--- a/clang/test/Analysis/taint-generic.cpp
+++ b/clang/test/Analysis/taint-generic.cpp
@@ -1,5 +1,5 @@
 // RUN: %clang_analyze_cc1 -std=c++11 -Wno-format-security \
-// RUN:   -analyzer-checker=core,optin.taint,alpha.security.ArrayBoundV2,debug.ExprInspection \
+// RUN:   -analyzer-checker=core,optin.taint,security.ArrayBound,debug.ExprInspection \
 // RUN:   -analyzer-config optin.taint.TaintPropagation:Config=%S/Inputs/taint-generic-config.yaml \
 // RUN:   -verify %s
 
diff --git a/clang/test/CIR/Lowering/global-var-simple.cpp b/clang/test/CIR/Lowering/global-var-simple.cpp
new file mode 100644
index 0000000000000..a5adb4011931a
--- /dev/null
+++ b/clang/test/CIR/Lowering/global-var-simple.cpp
@@ -0,0 +1,102 @@
+// Global variables of intergal types
+// RUN: %clang_cc1 -std=c++20 -triple x86_64-unknown-linux-gnu -fclangir -emit-llvm %s -o -  | FileCheck %s
+
+// Note: Currently unsupported features include default zero-initialization
+//       and alignment. The fact that "external" is only printed for globals
+//       without an initializer is a quirk of the LLVM AsmWriter.
+
+char c;
+// CHECK: @c = external dso_local global i8
+
+signed char sc;
+// CHECK: @sc = external dso_local global i8
+
+unsigned char uc;
+// CHECK: @uc = external dso_local global i8
+
+short ss;
+// CHECK: @ss = external dso_local global i16
+
+unsigned short us = 100;
+// CHECK: @us = dso_local global i16 100
+
+int si = 42;
+// CHECK: @si = dso_local global i32 42
+
+unsigned ui;
+// CHECK: @ui = external dso_local global i32
+
+long sl;
+// CHECK: @sl = external dso_local global i64
+
+unsigned long ul;
+// CHECK: @ul = external dso_local global i64
+
+long long sll;
+// CHECK: @sll = external dso_local global i64
+
+unsigned long long ull = 123456;
+// CHECK: @ull = dso_local global i64 123456
+
+__int128 s128;
+// CHECK: @s128 = external dso_local global i128
+
+unsigned __int128 u128;
+// CHECK: @u128 = external dso_local global i128
+
+wchar_t wc;
+// CHECK: @wc = external dso_local global i32
+
+char8_t c8;
+// CHECK: @c8 = external dso_local global i8
+
+char16_t c16;
+// CHECK: @c16 = external dso_local global i16
+
+char32_t c32;
+// CHECK: @c32 = external dso_local global i32
+
+_BitInt(20) sb20;
+// CHECK: @sb20 = external dso_local global i20
+
+unsigned _BitInt(48) ub48;
+// CHECK: @ub48 = external dso_local global i48
+
+_Float16 f16;
+// CHECK: @f16 = external dso_local global half
+
+__bf16 bf16;
+// CHECK: @bf16 = external dso_local global bfloat
+
+float f;
+// CHECK: @f = external dso_local global float
+
+double d = 1.25;
+// CHECK: @d = dso_local global double 1.250000e+00
+
+long double ld;
+// CHECK: @ld = external dso_local global x86_fp80
+
+__float128 f128;
+// CHECK: @f128 = external dso_local global fp128
+
+void *vp;
+// CHECK: @vp = external dso_local global ptr{{$}}
+
+int *ip = 0;
+// CHECK: @ip = dso_local global ptr null
+
+double *dp;
+// CHECK: @dp = external dso_local global ptr{{$}}
+
+char **cpp;
+// CHECK: @cpp = external dso_local global ptr{{$}}
+
+void (*fp)();
+// CHECK: @fp = external dso_local global ptr{{$}}
+
+int (*fpii)(int) = 0;
+// CHECK: @fpii = dso_local global ptr null
+
+void (*fpvar)(int, ...);
+// CHECK: @fpvar = external dso_local global ptr{{$}}
diff --git a/clang/test/CIR/Lowering/hello.c b/clang/test/CIR/Lowering/hello.c
new file mode 100644
index 0000000000000..ff78b6e6f6a5e
--- /dev/null
+++ b/clang/test/CIR/Lowering/hello.c
@@ -0,0 +1,10 @@
+// Smoke test for ClangIR-to-LLVM IR code generation
+// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -fclangir -emit-llvm %s -o -  | FileCheck %s
+
+int a;
+
+// CHECK: @a = external dso_local global i32
+
+int b = 2;
+
+// CHECK: @b = dso_local global i32 2
diff --git a/clang/test/CMakeLists.txt b/clang/test/CMakeLists.txt
index 5369dc92f69e8..b796a51ef600e 100644
--- a/clang/test/CMakeLists.txt
+++ b/clang/test/CMakeLists.txt
@@ -5,7 +5,6 @@ llvm_canonicalize_cmake_booleans(
   CLANG_BUILD_EXAMPLES
   CLANG_BUILT_STANDALONE
   CLANG_DEFAULT_PIE_ON_LINUX
-  CLANG_ENABLE_ARCMT
   CLANG_ENABLE_STATIC_ANALYZER
   CLANG_PLUGIN_SUPPORT
   CLANG_SPAWN_CC1
@@ -22,6 +21,7 @@ llvm_canonicalize_cmake_booleans(
   LLVM_WITH_Z3
   PPC_LINUX_DEFAULT_IEEELONGDOUBLE
   LLVM_TOOL_LLVM_DRIVER_BUILD
+  LLVM_INCLUDE_SPIRV_TOOLS_TESTS
   )
 
 configure_lit_site_cfg(
@@ -92,13 +92,6 @@ if(CLANG_ENABLE_STATIC_ANALYZER)
     )
 endif()
 
-if (CLANG_ENABLE_ARCMT)
-  list(APPEND CLANG_TEST_DEPS
-    arcmt-test
-    c-arcmt-test
-  )
-endif ()
-
 if(CLANG_BUILD_EXAMPLES AND CLANG_PLUGIN_SUPPORT)
   list(APPEND CLANG_TEST_DEPS
     Attribute
@@ -110,6 +103,15 @@ if(CLANG_BUILD_EXAMPLES AND CLANG_PLUGIN_SUPPORT)
     )
 endif ()
 
+if(LLVM_INCLUDE_SPIRV_TOOLS_TESTS)
+  list(APPEND CLANG_TEST_DEPS
+    spirv-dis
+    spirv-val
+    spirv-as
+    spirv-link
+    )
+endif()    
+
 set(CLANG_TEST_PARAMS
   USE_Z3_SOLVER=0
   )
diff --git a/clang/test/CXX/class/class.init/p1.cpp b/clang/test/CXX/class/class.init/p1.cpp
new file mode 100644
index 0000000000000..717dfba89763a
--- /dev/null
+++ b/clang/test/CXX/class/class.init/p1.cpp
@@ -0,0 +1,14 @@
+// RUN: %clang_cc1 -fsyntax-only -verify %s
+
+namespace test_deleted_ctor_note {
+struct A {
+  int a;
+  A() = delete; // expected-note {{'A' has been explicitly marked deleted here}}
+  A(int a_) : a(a_) { }
+};
+
+struct B {
+  A a1, a2, a3; // expected-note {{default constructed field 'a2' declared here}}
+  B(int a_) : a1(a_), a3(a_) { } // expected-error{{call to deleted constructor of 'A'}}
+};
+}
diff --git a/clang/test/CXX/dcl.decl/dcl.init/p14-0x.cpp b/clang/test/CXX/dcl.decl/dcl.init/p14-0x.cpp
index e7f501352168c..d548f9c8c2fdf 100644
--- a/clang/test/CXX/dcl.decl/dcl.init/p14-0x.cpp
+++ b/clang/test/CXX/dcl.decl/dcl.init/p14-0x.cpp
@@ -27,7 +27,7 @@ class Friend {
 
 
 class S {
-  NoDefault nd1;
+  NoDefault nd1; // expected-note {{default constructed field 'nd1' declared here}}
   NoDefault nd2 = 42;
   Explicit e1; // expected-note {{here}}
   Explicit e2 = 42; // expected-error {{no viable conversion}}
diff --git a/clang/test/CXX/drs/cwg0xx.cpp b/clang/test/CXX/drs/cwg0xx.cpp
index 15f469440c66f..44a0eb520af22 100644
--- a/clang/test/CXX/drs/cwg0xx.cpp
+++ b/clang/test/CXX/drs/cwg0xx.cpp
@@ -521,12 +521,12 @@ namespace example1 {
   namespace A {
     int i;
   }
-  
+
   namespace A1 {
     using A::i;
     using A::i;
   }
-  
+
   void f()
   {
     using A::i;
@@ -1371,7 +1371,7 @@ namespace cwg92 { // cwg92: 4 c++17
   // considered in this context. In C++17, we *do* perform an implicit
   // conversion (which performs initialization), and the exception specification
   // is part of the type of the parameter, so this is invalid.
-  template<void() throw()> struct X {};
+  template<void() throw()> struct X {}; // since-cxx17-note {{template parameter is declared here}}
   X<&f> xp;
   // since-cxx17-error@-1 {{value of type 'void (*)() throw(int, float)' is not implicitly convertible to 'void (*)() throw()'}}
 
diff --git a/clang/test/CXX/drs/cwg12xx.cpp b/clang/test/CXX/drs/cwg12xx.cpp
index 344adb6d72023..e02a7e11b80b2 100644
--- a/clang/test/CXX/drs/cwg12xx.cpp
+++ b/clang/test/CXX/drs/cwg12xx.cpp
@@ -155,6 +155,8 @@ namespace cwg1295 { // cwg1295: 4
   // cxx98-14-error@-1 {{non-type template argument does not refer to any declaration}}
   //   cxx98-14-note@#cwg1295-Y {{template parameter is declared here}}
   // since-cxx17-error@#cwg1295-y {{reference cannot bind to bit-field in converted constant expression}}
+  //   since-cxx17-note@#cwg1295-Y {{template parameter is declared here}}
+
 
 #if __cplusplus >= 201103L
   const unsigned other = 0;
diff --git a/clang/test/CXX/drs/cwg3xx.cpp b/clang/test/CXX/drs/cwg3xx.cpp
index b5e07a66bb4ed..6c420ecd4c91d 100644
--- a/clang/test/CXX/drs/cwg3xx.cpp
+++ b/clang/test/CXX/drs/cwg3xx.cpp
@@ -444,7 +444,7 @@ namespace cwg329 { // cwg329: 3.5
     //   expected-note@#cwg329-b {{in instantiation of template class 'cwg329::A<char>' requested here}}
     //   expected-note@#cwg329-i {{previous definition is here}}
   };
-  A<int> a; 
+  A<int> a;
   A<char> b; // #cwg329-b
 
   void test() {
@@ -688,9 +688,9 @@ namespace cwg341 { // cwg341: sup 1708
   namespace B {
     extern "C" int &cwg341_a = cwg341_a;
     // expected-error@-1 {{redefinition of 'cwg341_a'}}
-    //   expected-note@#cwg341_a {{previous definition is here}} 
+    //   expected-note@#cwg341_a {{previous definition is here}}
   }
-  extern "C" void cwg341_b(); // #cwg341_b 
+  extern "C" void cwg341_b(); // #cwg341_b
 }
 int cwg341_a;
 // expected-error@-1 {{declaration of 'cwg341_a' in global scope conflicts with declaration with C language linkage}}
@@ -708,7 +708,7 @@ namespace cwg341 {
   // expected-error@-1 {{declaration of 'cwg341_d' with C language linkage conflicts with declaration in global scope}}
   //   expected-note@#cwg341_d {{declared in global scope here}}
 
-  namespace A { extern "C" int cwg341_e; } // #cwg341_e 
+  namespace A { extern "C" int cwg341_e; } // #cwg341_e
   namespace B { extern "C" void cwg341_e(); }
   // expected-error@-1 {{redefinition of 'cwg341_e' as different kind of symbol}}
   //   expected-note@#cwg341_e {{previous definition is here}}
@@ -960,6 +960,7 @@ namespace cwg354 { // cwg354: 3.1 c++11
   // cxx11-14-error@#cwg354-p0 {{null non-type template argument must be cast to template parameter type 'int *'}}
   //   cxx11-14-note@#cwg354-ptr {{template parameter is declared here}}
   // since-cxx17-error@#cwg354-p0 {{conversion from 'int' to 'int *' is not allowed in a converted constant expression}}
+  //   since-cxx17-note@#cwg354-ptr {{template parameter is declared here}}
   ptr<(int*)0> p1;
   // cxx98-error@-1 {{non-type template argument does not refer to any declaration}}
   //   cxx98-note@#cwg354-ptr {{template parameter is declared here}}
@@ -969,12 +970,14 @@ namespace cwg354 { // cwg354: 3.1 c++11
   // cxx11-14-error@#cwg354-p2 {{null non-type template argument of type 'float *' does not match template parameter of type 'int *'}}
   //   cxx11-14-note@#cwg354-ptr {{template parameter is declared here}}
   // since-cxx17-error@#cwg354-p2 {{value of type 'float *' is not implicitly convertible to 'int *'}}
+  //   since-cxx17-note@#cwg354-ptr {{template parameter is declared here}}
   ptr<(int S::*)0> p3; // #cwg354-p3
   // cxx98-error@#cwg354-p3 {{non-type template argument does not refer to any declaration}}
   //   cxx98-note@#cwg354-ptr {{template parameter is declared here}}
   // cxx11-14-error@#cwg354-p3 {{null non-type template argument of type 'int S::*' does not match template parameter of type 'int *'}}
   //   cxx11-14-note@#cwg354-ptr {{template parameter is declared here}}
   // since-cxx17-error@#cwg354-p3 {{value of type 'int S::*' is not implicitly convertible to 'int *'}}
+  //   since-cxx17-note@#cwg354-ptr {{template parameter is declared here}}
 
   template<int*> int both(); // #cwg354-both-int-ptr
   template<int> int both(); // #cwg354-both-int
@@ -991,6 +994,7 @@ namespace cwg354 { // cwg354: 3.1 c++11
   // cxx11-14-error@#cwg354-m0 {{null non-type template argument must be cast to template parameter type 'int S::*'}}
   //   cxx11-14-note@#cwg354-ptr_mem {{template parameter is declared here}}
   // since-cxx17-error@#cwg354-m0 {{conversion from 'int' to 'int S::*' is not allowed in a converted constant expression}}
+  //   since-cxx17-note@#cwg354-ptr_mem {{template parameter is declared here}}
   ptr_mem<(int S::*)0> m1;
   // cxx98-error@-1 {{non-type template argument is not a pointer to member constant}}
   ptr_mem<(float S::*)0> m2; // #cwg354-m2
@@ -999,12 +1003,14 @@ namespace cwg354 { // cwg354: 3.1 c++11
   // cxx11-14-error@#cwg354-m2 {{null non-type template argument of type 'float S::*' does not match template parameter of type 'int S::*'}}
   //   cxx11-14-note@#cwg354-ptr_mem {{template parameter is declared here}}
   // since-cxx17-error@#cwg354-m2 {{value of type 'float S::*' is not implicitly convertible to 'int S::*'}}
+  //   since-cxx17-note@#cwg354-ptr_mem {{template parameter is declared here}}
   ptr_mem<(int *)0> m3; // #cwg354-m3
   // cxx98-error@#cwg354-m3 {{non-type template argument of type 'int *' cannot be converted to a value of type 'int S::*'}}
   //   cxx98-note@#cwg354-ptr_mem {{template parameter is declared here}}
   // cxx11-14-error@#cwg354-m3 {{null non-type template argument of type 'int *' does not match template parameter of type 'int S::*'}}
   //   cxx11-14-note@#cwg354-ptr_mem {{template parameter is declared here}}
   // since-cxx17-error@#cwg354-m3 {{value of type 'int *' is not implicitly convertible to 'int S::*'}}
+  //   since-cxx17-note@#cwg354-ptr_mem {{template parameter is declared here}}
 } // namespace cwg354
 
 struct cwg355_S; // cwg355: 2.7
@@ -1116,7 +1122,7 @@ namespace cwg364 { // cwg364: 2.7
 } // namespace cwg364
 
 namespace cwg366 { // cwg366: 2.7
-#if "foo" // expected-error {{invalid token at start of a preprocessor expression}} 
+#if "foo" // expected-error {{invalid token at start of a preprocessor expression}}
 #endif
 } // namespace cwg366
 
diff --git a/clang/test/CXX/expr/expr.const/p3-0x.cpp b/clang/test/CXX/expr/expr.const/p3-0x.cpp
index 5bd70c5250b59..3eedef3cf7712 100644
--- a/clang/test/CXX/expr/expr.const/p3-0x.cpp
+++ b/clang/test/CXX/expr/expr.const/p3-0x.cpp
@@ -1,5 +1,5 @@
-// RUN: %clang_cc1 -fsyntax-only -std=c++11 -verify %s
-// RUN: %clang_cc1 -fsyntax-only -std=c++1z -verify %s
+// RUN: %clang_cc1 -fsyntax-only -std=c++11 -verify=expected,cxx11 %s
+// RUN: %clang_cc1 -fsyntax-only -std=c++17 -verify=expected,cxx17 %s
 
 // A converted constant expression of type T is a core constant expression,
 int nonconst = 8; // expected-note 3 {{here}}
@@ -66,7 +66,8 @@ enum class EEE : unsigned short {
   e = 123456, // expected-error {{enumerator value evaluates to 123456, which cannot be narrowed to type 'unsigned short'}}
   f = -3 // expected-error {{enumerator value evaluates to -3, which cannot be narrowed to type 'unsigned short'}}
 };
-template<unsigned char> using A = int;
+template<unsigned char> using A = int; // cxx17-note 2{{template parameter is declared here}}
+
 using Int = A<E6>;
 using Int = A<EE::EE32>; // expected-error {{not implicitly convertible}}
 using Int = A<(int)EE::EE32>;
@@ -78,6 +79,7 @@ using Int = A<-3>; // expected-error {{template argument evaluates to -3, which
 // integral conversions as well as boolean conversions.
 // FIXME: Per core issue 1407, this is not correct.
 template<typename T, T v> struct Val { static constexpr T value = v; };
+// cxx17-note@-1 2{{template parameter is declared here}}
 static_assert(Val<bool, E1>::value == 1, ""); // ok
 static_assert(Val<bool, '\0'>::value == 0, ""); // ok
 static_assert(Val<bool, U'\1'>::value == 1, ""); // ok
diff --git a/clang/test/CXX/stmt.stmt/stmt.select/stmt.switch/p4.cpp b/clang/test/CXX/stmt.stmt/stmt.select/stmt.switch/p4.cpp
new file mode 100644
index 0000000000000..e816da1803694
--- /dev/null
+++ b/clang/test/CXX/stmt.stmt/stmt.select/stmt.switch/p4.cpp
@@ -0,0 +1,11 @@
+// RUN: %clang -fsyntax-only -std=c++20 -Xclang -verify %s
+
+void Func(int x) {
+    switch (x) {
+        [[likely]] case 0:
+        case 1: 
+            int i = 3; // expected-note {{jump bypasses variable initialization}}
+        case 2: // expected-error {{cannot jump from switch statement to this case label}}
+            break;
+    }
+}
diff --git a/clang/test/CXX/temp/temp.arg/temp.arg.nontype/p1.cpp b/clang/test/CXX/temp/temp.arg/temp.arg.nontype/p1.cpp
index 7fce615516924..629000d88acc3 100644
--- a/clang/test/CXX/temp/temp.arg/temp.arg.nontype/p1.cpp
+++ b/clang/test/CXX/temp/temp.arg/temp.arg.nontype/p1.cpp
@@ -8,7 +8,7 @@
 //   be one of:
 //   -- an integral constant expression; or
 //   -- the name of a non-type template-parameter ; or
-#ifndef CPP11ONLY 
+#ifndef CPP11ONLY
 
 namespace non_type_tmpl_param {
   template <int N> struct X0 { X0(); };
@@ -31,15 +31,18 @@ namespace non_type_tmpl_param {
 //      omitted if the name refers to a function or array and shall be omitted
 //      if the corresopnding template-parameter is a reference; or
 namespace addr_of_obj_or_func {
-  template <int* p> struct X0 { }; // precxx17-note 5{{here}}
+  template <int* p> struct X0 { }; // expected-note 5{{here}}
 #if __cplusplus >= 201103L
   // precxx17-note@-2 2{{template parameter is declared here}}
 #endif
 
-  template <int (*fp)(int)> struct X1 { };
-  template <int &p> struct X2 { }; // precxx17-note 4{{here}}
-  template <const int &p> struct X2k { }; // precxx17-note {{here}}
-  template <int (&fp)(int)> struct X3 { }; // precxx17-note 4{{here}}
+  template <int (*fp)(int)> struct X1 { }; // cxx17-note {{here}}
+#if __cplusplus <= 199711L
+  // precxx17-note@-2 {{here}}
+#endif
+  template <int &p> struct X2 { }; // expected-note 4{{here}}
+  template <const int &p> struct X2k { }; // expected-note {{here}}
+  template <int (&fp)(int)> struct X3 { }; // expected-note 4{{here}}
 
   int i = 42;
 #if __cplusplus >= 201103L
diff --git a/clang/test/CXX/temp/temp.arg/temp.arg.template/p3-0x.cpp b/clang/test/CXX/temp/temp.arg/temp.arg.template/p3-0x.cpp
index 54fcfccad6f52..3caed045c6688 100644
--- a/clang/test/CXX/temp/temp.arg/temp.arg.template/p3-0x.cpp
+++ b/clang/test/CXX/temp/temp.arg/temp.arg.template/p3-0x.cpp
@@ -18,8 +18,9 @@ eval<D<int, 17>> eD; // expected-error{{implicit instantiation of undefined temp
 eval<E<int, float>> eE; // expected-error{{implicit instantiation of undefined template 'eval<E<int, float>>}}
 
 template<
-  template <int ...N> // expected-error{{deduced non-type template argument does not have the same type as the corresponding template parameter ('int' vs 'void *')}}
-  class TT // expected-note {{previous template template parameter is here}}
+  template <int ...N> // expected-error {{cannot be narrowed from type 'int' to 'short'}}
+                      // expected-error@-1 {{conversion from 'int' to 'void *' is not allowed in a converted constant expression}}
+  class TT // expected-note 2{{previous template template parameter is here}}
 > struct X0 { };
 
 template<int I, int J, int ...Rest> struct X0a;
@@ -31,12 +32,13 @@ template<int I, void *J> struct X0e; // expected-note{{template parameter is dec
 X0<X0a> inst_x0a;
 X0<X0b> inst_x0b;
 X0<X0c> inst_x0c;
-X0<X0d> inst_x0d;
+X0<X0d> inst_x0d; // expected-note {{has different template parameters}}
 X0<X0e> inst_x0e; // expected-note{{template template argument has different template parameters than its corresponding template template parameter}}
 
 template<typename T,
-         template <T ...N> // expected-error{{deduced non-type template argument does not have the same type as the corresponding template parameter ('short' vs 'void *')}}
-         class TT // expected-note {{previous template template parameter is here}}
+         template <T ...N> // expected-error {{conversion from 'short' to 'void *' is not allowed in a converted constant expression}}
+                           // expected-error@-1 {{cannot be narrowed from type 'int' to 'short'}}
+         class TT // expected-note 2{{previous template template parameter is here}}
 > struct X1 { };
 
 template<int I, int J, int ...Rest> struct X1a;
@@ -49,8 +51,8 @@ X1<int, X1a> inst_x1a;
 X1<long, X1b> inst_x1b;
 X1<short, X1c> inst_x1c;
 X1<short, X1d> inst_sx1d;
-X1<int, X1d> inst_ix1d;
-X1<short, X1e> inst_x1e; // expected-note{{template template argument has different template parameters than its corresponding template template parameter}}
+X1<int, X1d> inst_ix1d;  // expected-note {{has different template parameters}}
+X1<short, X1e> inst_x1e; // expected-note {{has different template parameters}}
 
 template <int> class X2; // expected-note{{template is declared here}} \
                          // expected-note{{template is declared here}}
diff --git a/clang/test/CXX/temp/temp.decls/temp.variadic/fixed-expansion.cpp b/clang/test/CXX/temp/temp.decls/temp.variadic/fixed-expansion.cpp
index a990c82564aa4..ab4c663d24c7d 100644
--- a/clang/test/CXX/temp/temp.decls/temp.variadic/fixed-expansion.cpp
+++ b/clang/test/CXX/temp/temp.decls/temp.variadic/fixed-expansion.cpp
@@ -121,8 +121,8 @@ namespace PartialSpecialization {
 
 namespace FixedAliasTemplate {
   template<typename,typename,typename> struct S {};
-  template<typename T, typename U> using U = S<T, int, U>; // expected-note 2{{template parameter is declared here}}
-  template<typename...Ts> U<Ts...> &f(U<Ts...>, Ts...); // expected-error 2{{pack expansion used as argument for non-pack parameter of alias template}}
+  template<typename T, typename U> using Z = S<T, int, U>; // expected-note 2{{template parameter is declared here}}
+  template<typename...Ts> Z<Ts...> &f(Z<Ts...>, Ts...); // expected-error 2{{pack expansion used as argument for non-pack parameter of alias template}}
   S<int, int, double> &s1 = f({}, 0, 0.0); // expected-error {{no matching function}}
 }
 
diff --git a/clang/test/ClangScanDeps/modules-context-hash-cwd.c b/clang/test/ClangScanDeps/modules-context-hash-cwd.c
new file mode 100644
index 0000000000000..459d2c90debe6
--- /dev/null
+++ b/clang/test/ClangScanDeps/modules-context-hash-cwd.c
@@ -0,0 +1,188 @@
+// Test current directory pruning when computing the context hash.
+
+// REQUIRES: shell
+
+// RUN: rm -rf %t
+// RUN: split-file %s %t
+// RUN: sed -e "s|DIR|%/t|g" %t/cdb0.json.in > %t/cdb0.json
+// RUN: sed -e "s|DIR|%/t|g" %t/cdb1.json.in > %t/cdb1.json
+// RUN: sed -e "s|DIR|%/t|g" %t/cdb3.json.in > %t/cdb3.json
+// RUN: sed -e "s|DIR|%/t|g" %t/cdb4.json.in > %t/cdb4.json
+// RUN: sed -e "s|DIR|%/t|g" %t/cdb5.json.in > %t/cdb5.json
+// RUN: clang-scan-deps -compilation-database %t/cdb0.json -format experimental-full > %t/result0.json
+// RUN: clang-scan-deps -compilation-database %t/cdb1.json -format experimental-full > %t/result1.json
+// It is not a typo to use cdb1.json for result2. We intend to use the same
+// compilation database, but different clang-scan-deps optimize-args options.
+// RUN: clang-scan-deps -compilation-database %t/cdb1.json -format experimental-full -optimize-args=header-search,system-warnings,vfs,canonicalize-macros > %t/result2.json
+// RUN: clang-scan-deps -compilation-database %t/cdb3.json -format experimental-full > %t/result3.json
+// RUN: clang-scan-deps -compilation-database %t/cdb4.json -format experimental-full > %t/result4.json
+// RUN: clang-scan-deps -compilation-database %t/cdb5.json -format experimental-full > %t/result5.json
+// RUN: cat %t/result0.json %t/result1.json | FileCheck %s
+// RUN: cat %t/result0.json %t/result2.json | FileCheck %s -check-prefix=SKIPOPT
+// RUN: cat %t/result3.json %t/result4.json | FileCheck %s -check-prefix=RELPATH
+// RUN: cat %t/result0.json %t/result5.json | FileCheck %s
+
+//--- cdb0.json.in
+[{
+  "directory": "DIR",
+  "command": "clang -c DIR/tu.c -fmodules -fmodules-cache-path=DIR/cache -IDIR/include/ -o DIR/tu.o",
+  "file": "DIR/tu.c"
+}]
+
+//--- cdb1.json.in
+[{
+  "directory": "DIR/a",
+  "command": "clang -c DIR/tu.c -fmodules -fmodules-cache-path=DIR/cache -IDIR/include/ -o DIR/tu.o",
+  "file": "DIR/tu.c"
+}]
+
+// cdb2 is skipped because we reuse cdb1.
+
+//--- cdb3.json.in
+[{
+  "directory": "DIR",
+  "command": "clang -c DIR/tu.c -fmodules -fmodules-cache-path=DIR/cache -fprebuilt-module-path=.././module -IDIR/include/ -o DIR/tu.o ",
+  "file": "DIR/tu.c"
+}]
+
+//--- cdb4.json.in
+[{
+  "directory": "DIR/a/",
+  "command": "clang -c DIR/tu.c -fmodules -fmodules-cache-path=DIR/cache -fprebuilt-module-path=.././module -IDIR/include/ -o DIR/tu.o ",
+  "file": "DIR/tu.c"
+}]
+
+//--- cdb5.json.in
+[{
+  "directory": "DIR",
+  "command": "clang -c DIR/tu.c -fmodules -fmodules-cache-path=DIR/cache -IDIR/include/ -Xclang -working-directory=DIR/a/ -o DIR/tu.o",
+  "file": "DIR/tu.c"
+}]
+
+//--- include/module.modulemap
+module mod {
+  header "mod.h"
+}
+
+//--- include/mod.h
+
+//--- tu.c
+#include "mod.h"
+
+// Check that result0 and result1/result5 compute the same hash with
+// optimization on. The only difference between result0 and result1/result5 is
+// the compiler's working directory.
+// CHECK:     {
+// CHECK-NEXT:  "modules": [
+// CHECK-NEXT:   {
+// CHECK-NEXT:     "clang-module-deps": [],
+// CHECK:          "context-hash": "[[HASH:.*]]",
+// CHECK:        }
+// CHECK:       "translation-units": [
+// CHECK:        {
+// CHECK:          "commands": [
+// CHECK:          {
+// CHECK-NEXT:        "clang-context-hash": "{{.*}}",
+// CHECK-NEXT:        "clang-module-deps": [
+// CHECK-NEXT:          {
+// CHECK-NEXT:            "context-hash": "[[HASH]]",
+// CHECK-NEXT:            "module-name": "mod"
+// CHECK:               }
+// CHECK:             ],
+// CHECK:     {
+// CHECK-NEXT:   "modules": [
+// CHECK-NEXT:    {
+// CHECK-NEXT:      "clang-module-deps": [],
+// CHECK:           "context-hash": "[[HASH]]",
+// CHECK:         }
+// CHECK:        "translation-units": [
+// CHECK:         {
+// CHECK:           "commands": [
+// CHECK:           {
+// CHECK-NEXT:         "clang-context-hash": "{{.*}}",
+// CHECK-NEXT:         "clang-module-deps": [
+// CHECK-NEXT:           {
+// CHECK-NEXT:             "context-hash": "[[HASH]]",
+// CHECK-NEXT:             "module-name": "mod"
+// CHECK:               }
+// CHECK:              ],
+
+// Check that result0 and result2 compute different hashes because
+// the working directory optmization is turned off for result2.
+// SKIPOPT:      {
+// SKIPOPT-NEXT:   "modules": [
+// SKIPOPT-NEXT:    {
+// SKIPOPT-NEXT:      "clang-module-deps": [],
+// SKIPOPT:           "context-hash": "[[HASH0:.*]]",
+// SKIPOPT:         }
+// SKIPOPT:        "translation-units": [
+// SKIPOPT:         {
+// SKIPOPT:            "commands": [
+// SKIPOPT:             {
+// SKIPOPT-NEXT:          "clang-context-hash": "{{.*}}",
+// SKIPOPT-NEXT:          "clang-module-deps": [
+// SKIPOPT-NEXT:            {
+// SKIPOPT-NEXT:              "context-hash": "[[HASH0]]",
+// SKIPOPT-NEXT:              "module-name": "mod"
+// SKIPOPT:            }
+// SKIPOPT:          ],
+// SKIPOPT:      {
+// SKIPOPT-NEXT:   "modules": [
+// SKIPOPT-NEXT:     {
+// SKIPOPT-NEXT:       "clang-module-deps": [],
+// SKIPOPT-NOT:        "context-hash": "[[HASH0]]",
+// SKIPOPT:            "context-hash": "[[HASH2:.*]]",
+// SKIPOPT:          }
+// SKIPOPT:       "translation-units": [
+// SKIPOPT:         {
+// SKIPOPT:           "commands": [
+// SKIPOPT:             {
+// SKIPOPT-NEXT:          "clang-context-hash": "{{.*}}",
+// SKIPOPT-NEXT:          "clang-module-deps": [
+// SKIPOPT-NEXT:            {
+// SKIPOPT-NOT:              "context-hash": "[[HASH0]]",
+// SKIPOPT-NEXT:             "context-hash": "[[HASH2]]"
+// SKIPOPT-NEXT:              "module-name": "mod"
+// SKIPOPT:            }
+// SKIPOPT:          ],
+
+// Check that result3 and result4 contain different hashes because
+// both have a same relative path as a command line input, and
+// they are produced using different compiler working directories.
+// RELPATH:      {
+// RELPATH-NEXT:   "modules": [
+// RELPATH-NEXT:    {
+// RELPATH-NEXT:      "clang-module-deps": [],
+// RELPATH:           "context-hash": "[[HASH3:.*]]",
+// RELPATH:         }
+// RELPATH:        "translation-units": [
+// RELPATH:         {
+// RELPATH:            "commands": [
+// RELPATH:             {
+// RELPATH-NEXT:          "clang-context-hash": "{{.*}}",
+// RELPATH-NEXT:          "clang-module-deps": [
+// RELPATH-NEXT:            {
+// RELPATH-NEXT:              "context-hash": "[[HASH3]]",
+// RELPATH-NEXT:              "module-name": "mod"
+// RELPATH:            }
+// RELPATH:          ],
+// RELPATH:      {
+// RELPATH-NEXT:   "modules": [
+// RELPATH-NEXT:     {
+// RELPATH-NEXT:       "clang-module-deps": [],
+// RELPATH-NOT:        "context-hash": "[[HASH3]]",
+// RELPATH:            "context-hash": "[[HASH4:.*]]",
+// RELPATH:          }
+// RELPATH:       "translation-units": [
+// RELPATH:         {
+// RELPATH:           "commands": [
+// RELPATH:             {
+// RELPATH-NEXT:          "clang-context-hash": "{{.*}}",
+// RELPATH-NEXT:          "clang-module-deps": [
+// RELPATH-NEXT:            {
+// RELPATH-NOT:              "context-hash": "[[HASH3]]",
+// RELPATH-NEXT:             "context-hash": "[[HASH4]]"
+// RELPATH-NEXT:              "module-name": "mod"
+// RELPATH:            }
+// RELPATH:          ],
+
diff --git a/clang/test/ClangScanDeps/strip-input-args.m b/clang/test/ClangScanDeps/strip-input-args.m
index e1954ecc28914..31f634dd80131 100644
--- a/clang/test/ClangScanDeps/strip-input-args.m
+++ b/clang/test/ClangScanDeps/strip-input-args.m
@@ -11,11 +11,6 @@
 
 // CHECK:        "modules": [
 // CHECK-NEXT:     {
-// CHECK:            "command-line": [
-// CHECK-NOT:          "-arcmt-action=check"
-// CHECK-NOT:          "-objcmt-migrate-literals"
-// CHECK-NOT:          "-mt-migrate-directory"
-// CHECK:            ]
 // CHECK:            "name": "A"
 // CHECK:          }
 // CHECK-NOT:        "name": "A"
@@ -25,12 +20,12 @@
 [
   {
     "directory": "DIR",
-    "command": "clang -Imodules/A -fmodules -fmodules-cache-path=DIR/module-cache -fimplicit-modules -fimplicit-module-maps -ccc-arcmt-check -fsyntax-only DIR/t1.m",
+    "command": "clang -Imodules/A -fmodules -fmodules-cache-path=DIR/module-cache -fimplicit-modules -fimplicit-module-maps -fsyntax-only DIR/t1.m",
     "file": "DIR/t1.m"
   },
   {
     "directory": "DIR",
-    "command": "clang -Imodules/A -fmodules -fmodules-cache-path=DIR/module-cache -fimplicit-modules -fimplicit-module-maps -ccc-objcmt-migrate bob -fsyntax-only DIR/t2.m",
+    "command": "clang -Imodules/A -fmodules -fmodules-cache-path=DIR/module-cache -fimplicit-modules -fimplicit-module-maps -fsyntax-only DIR/t2.m",
     "file": "DIR/t2.m"
   }
 ]
diff --git a/clang/test/ClangScanDeps/tu-buffer.c b/clang/test/ClangScanDeps/tu-buffer.c
new file mode 100644
index 0000000000000..b450b13ff434b
--- /dev/null
+++ b/clang/test/ClangScanDeps/tu-buffer.c
@@ -0,0 +1,111 @@
+// UNSUPPORTED: target=powerpc64-ibm-aix{{.*}}
+
+// RUN: rm -rf %t
+// RUN: split-file %s %t
+
+//--- module.modulemap
+module root { header "root.h" }
+module direct { header "direct.h" }
+module transitive { header "transitive.h" }
+module addition { header "addition.h" }
+//--- root.h
+#include "direct.h"
+#include "root/textual.h"
+//--- direct.h
+#include "transitive.h"
+//--- transitive.h
+// empty
+
+//--- addition.h
+// empty
+
+//--- tu.c
+#include "root.h"
+
+//--- root/textual.h
+// This is here to verify that the "root" directory doesn't clash with name of
+// the "root" module.
+
+//--- cdb.json.template
+[{
+  "file": "",
+  "directory": "DIR",
+  "command": "clang -fmodules -fmodules-cache-path=DIR/cache -I DIR -x c -c"
+}]
+
+// RUN: sed "s|DIR|%/t|g" %t/cdb.json.template > %t/cdb.json
+// RUN: clang-scan-deps -compilation-database %t/cdb.json -format experimental-full -tu-buffer-path %t/tu.c > %t/result.json
+// RUN: cat %t/result.json | sed 's:\\\\\?:/:g' | FileCheck -DPREFIX=%/t %s --check-prefix=CHECK
+
+// CHECK:      {
+// CHECK-NEXT:   "modules": [
+// CHECK-NEXT:     {
+// CHECK-NEXT:       "clang-module-deps": [
+// CHECK-NEXT:         {
+// CHECK-NEXT:           "context-hash": "{{.*}}",
+// CHECK-NEXT:           "module-name": "transitive"
+// CHECK-NEXT:         }
+// CHECK-NEXT:       ],
+// CHECK-NEXT:       "clang-modulemap-file": "[[PREFIX]]/module.modulemap",
+// CHECK-NEXT:       "command-line": [
+// CHECK:            ],
+// CHECK-NEXT:       "context-hash": "{{.*}}",
+// CHECK-NEXT:       "file-deps": [
+// CHECK-NEXT:         "[[PREFIX]]/module.modulemap"
+// CHECK-NEXT:         "[[PREFIX]]/direct.h"
+// CHECK-NEXT:       ],
+// CHECK:            "name": "direct"
+// CHECK-NEXT:     },
+// CHECK-NEXT:     {
+// CHECK-NEXT:       "clang-module-deps": [
+// CHECK-NEXT:         {
+// CHECK-NEXT:           "context-hash": "{{.*}}",
+// CHECK-NEXT:           "module-name": "direct"
+// CHECK-NEXT:         }
+// CHECK-NEXT:       ],
+// CHECK-NEXT:       "clang-modulemap-file": "[[PREFIX]]/module.modulemap",
+// CHECK-NEXT:       "command-line": [
+// CHECK:            ],
+// CHECK-NEXT:       "context-hash": "{{.*}}",
+// CHECK-NEXT:       "file-deps": [
+// CHECK-NEXT:         "[[PREFIX]]/module.modulemap"
+// CHECK-NEXT:         "[[PREFIX]]/root.h"
+// CHECK-NEXT:         "[[PREFIX]]/root/textual.h"
+// CHECK-NEXT:       ],
+// CHECK:            "name": "root"
+// CHECK-NEXT:     },
+// CHECK-NEXT:     {
+// CHECK-NEXT:       "clang-module-deps": [],
+// CHECK-NEXT:       "clang-modulemap-file": "[[PREFIX]]/module.modulemap",
+// CHECK-NEXT:       "command-line": [
+// CHECK:            ],
+// CHECK-NEXT:       "context-hash": "{{.*}}",
+// CHECK-NEXT:       "file-deps": [
+// CHECK-NEXT:         "[[PREFIX]]/module.modulemap"
+// CHECK-NEXT:         "[[PREFIX]]/transitive.h"
+// CHECK-NEXT:       ],
+// CHECK:            "name": "transitive"
+// CHECK-NEXT:     }
+// CHECK-NEXT:   ],
+// CHECK-NEXT:   "translation-units": [
+// CHECK-NEXT:     {
+// CHECK-NEXT:       "commands": [
+// CHECK-NEXT:         {
+// CHECK-NEXT:           "clang-context-hash": "{{.*}}",
+// CHECK-NEXT:           "clang-module-deps": [
+// CHECK-NEXT:             {
+// CHECK-NEXT:               "context-hash": "{{.*}}",
+// CHECK-NEXT:               "module-name": "root"
+// CHECK-NEXT:             }
+// CHECK-NEXT:           ],
+// CHECK-NEXT:           "command-line": [
+// CHECK:                ],
+// CHECK:                "file-deps": [
+// CHECK-NEXT:             [[PREFIX]]/tu.c
+// CHECK-NEXT:           ],
+// CHECK-NEXT:           "input-file": "[[PREFIX]]/tu.c"
+// CHECK-NEXT:         }
+// CHECK-NEXT:       ]
+// CHECK-NEXT:     }
+// CHECK-NEXT:   ]
+// CHECK-NEXT: }
diff --git a/clang/test/ClangScanDeps/working-dir.m b/clang/test/ClangScanDeps/working-dir.m
index a04f8c2486b98..c6b7b1988d3cf 100644
--- a/clang/test/ClangScanDeps/working-dir.m
+++ b/clang/test/ClangScanDeps/working-dir.m
@@ -2,7 +2,7 @@
 // RUN: split-file %s %t
 // RUN: sed -e "s|DIR|%/t|g" %t/build/compile-commands.json.in > %t/build/compile-commands.json
 // RUN: clang-scan-deps -compilation-database %t/build/compile-commands.json \
-// RUN:   -j 1 -format experimental-full --optimize-args=all > %t/deps.db
+// RUN:   -j 1 -format experimental-full --optimize-args=header-search,system-warnings,vfs,canonicalize-macros > %t/deps.db
 // RUN: cat %t/deps.db | sed 's:\\\\\?:/:g' | FileCheck %s -DPREFIX=%/t
 
 // Check that there are two separate modules hashes. One for each working dir.
diff --git a/clang/test/CodeCompletion/member-access.cpp b/clang/test/CodeCompletion/member-access.cpp
index bf35f7ad021f7..b181466cdb628 100644
--- a/clang/test/CodeCompletion/member-access.cpp
+++ b/clang/test/CodeCompletion/member-access.cpp
@@ -417,3 +417,21 @@ void f() {
   // CHECK-DEPENDENT-NESTEDCLASS: [#int#]field
 }
 }
+
+namespace template_alias {
+struct A {
+  int b;
+};
+template <typename T>
+struct S {
+  A a;
+};
+template <typename T>
+using Alias = S<T>;
+template <typename T>
+void f(Alias<T> s) {
+  s.a.b;
+  // RUN: %clang_cc1 -fsyntax-only -code-completion-at=%s:433:7 %s -o - | FileCheck -check-prefix=CHECK-TEMPLATE-ALIAS %s
+  // CHECK-TEMPLATE-ALIAS: [#int#]b
+}
+}
diff --git a/clang/test/CodeGen/AArch64/args.cpp b/clang/test/CodeGen/AArch64/args.cpp
index fe1298cc683a4..6f1f3d5e2a062 100644
--- a/clang/test/CodeGen/AArch64/args.cpp
+++ b/clang/test/CodeGen/AArch64/args.cpp
@@ -1,6 +1,6 @@
-// RUN: %clang_cc1 -triple arm64-apple-ios7.0 -target-abi darwinpcs -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -triple aarch64-linux-gnu -emit-llvm -o - -x c %s | FileCheck %s --check-prefix=CHECK-GNU-C
-// RUN: %clang_cc1 -triple aarch64-linux-gnu -emit-llvm -o - %s | FileCheck %s --check-prefix=CHECK-GNU-CXX
+// RUN: %clang_cc1 -triple arm64-apple-ios7.0 -target-abi darwinpcs -emit-llvm -o - %s | FileCheck %s --check-prefixes=CHECK,DARWIN
+// RUN: %clang_cc1 -triple aarch64-linux-gnu -emit-llvm -o - -x c %s | FileCheck %s --check-prefixes=CHECK,C
+// RUN: %clang_cc1 -triple aarch64-linux-gnu -emit-llvm -o - %s | FileCheck %s --check-prefixes=CHECK,CXX
 
 // Empty structs are ignored for PCS purposes on Darwin and in C mode elsewhere.
 // In C++ mode on ELF they consume a register slot though. Functions are
@@ -15,16 +15,16 @@
 
 struct Empty {};
 
-// CHECK: define{{.*}} i32 @empty_arg(i32 noundef %a)
-// CHECK-GNU-C: define{{.*}} i32 @empty_arg(i32 noundef %a)
-// CHECK-GNU-CXX: define{{.*}} i32 @empty_arg(i8 %e.coerce, i32 noundef %a)
+// DARWIN: define{{.*}} i32 @empty_arg(i32 noundef %a)
+// C: define{{.*}} i32 @empty_arg(i32 noundef %a)
+// CXX: define{{.*}} i32 @empty_arg(i8 %e.coerce, i32 noundef %a)
 EXTERNC int empty_arg(struct Empty e, int a) {
   return a;
 }
 
-// CHECK: define{{.*}} void @empty_ret()
-// CHECK-GNU-C: define{{.*}} void @empty_ret()
-// CHECK-GNU-CXX: define{{.*}} void @empty_ret()
+// DARWIN: define{{.*}} void @empty_ret()
+// C: define{{.*}} void @empty_ret()
+// CXX: define{{.*}} void @empty_ret()
 EXTERNC struct Empty empty_ret(void) {
   struct Empty e;
   return e;
@@ -38,30 +38,75 @@ struct SuperEmpty {
   int arr[0];
 };
 
-// CHECK: define{{.*}} i32 @super_empty_arg(i32 noundef %a)
-// CHECK-GNU-C: define{{.*}} i32 @super_empty_arg(i32 noundef %a)
-// CHECK-GNU-CXX: define{{.*}} i32 @super_empty_arg(i32 noundef %a)
+// DARWIN: define{{.*}} i32 @super_empty_arg(i32 noundef %a)
+// C: define{{.*}} i32 @super_empty_arg(i32 noundef %a)
+// CXX: define{{.*}} i32 @super_empty_arg(i32 noundef %a)
 EXTERNC int super_empty_arg(struct SuperEmpty e, int a) {
   return a;
 }
 
-// This is not empty. It has 0 size but consumes a register slot for GCC.
+// This is also not empty, and non-standard. We previously considered it to
+// consume a register slot, but GCC does not, so we match that.
 
 struct SortOfEmpty {
   struct SuperEmpty e;
 };
 
-// CHECK: define{{.*}} i32 @sort_of_empty_arg(i32 noundef %a)
-// CHECK-GNU-C: define{{.*}} i32 @sort_of_empty_arg(i32 noundef %a)
-// CHECK-GNU-CXX: define{{.*}} i32 @sort_of_empty_arg(i8 %e.coerce, i32 noundef %a)
-EXTERNC int sort_of_empty_arg(struct Empty e, int a) {
+// DARWIN: define{{.*}} i32 @sort_of_empty_arg(i32 noundef %a)
+// C: define{{.*}} i32 @sort_of_empty_arg(i32 noundef %a)
+// CXX: define{{.*}} i32 @sort_of_empty_arg(i32 noundef %a)
+EXTERNC int sort_of_empty_arg(struct SortOfEmpty e, int a) {
   return a;
 }
 
-// CHECK: define{{.*}} void @sort_of_empty_ret()
-// CHECK-GNU-C: define{{.*}} void @sort_of_empty_ret()
-// CHECK-GNU-CXX: define{{.*}} void @sort_of_empty_ret()
+// DARWIN: define{{.*}} void @sort_of_empty_ret()
+// C: define{{.*}} void @sort_of_empty_ret()
+// CXX: define{{.*}} void @sort_of_empty_ret()
 EXTERNC struct SortOfEmpty sort_of_empty_ret(void) {
   struct SortOfEmpty e;
   return e;
 }
+
+#include <stdarg.h>
+
+// va_arg matches the above rules, consuming an incoming argument in cases
+// where one would be passed, and not doing so when the argument should be
+// ignored.
+
+EXTERNC struct Empty empty_arg_variadic(int a, ...) {
+// CHECK-LABEL: @empty_arg_variadic(
+// DARWIN-NOT: {{ getelementptr }}
+// C-NOT: {{ getelementptr }}
+// CXX: %new_reg_offs = add i32 %gr_offs, 8
+// CXX: %new_stack = getelementptr inbounds i8, ptr %stack, i64 8
+  va_list vl;
+  va_start(vl, a);
+  struct Empty b = va_arg(vl, struct Empty);
+  va_end(vl);
+  return b;
+}
+
+EXTERNC struct SuperEmpty super_empty_arg_variadic(int a, ...) {
+// CHECK-LABEL: @super_empty_arg_variadic(
+// DARWIN-NOT: {{ getelementptr }}
+// C-NOT: {{ getelementptr }}
+// CXX-NOT: {{ getelementptr }}
+  va_list vl;
+  va_start(vl, a);
+  struct SuperEmpty b = va_arg(vl, struct SuperEmpty);
+  va_end(vl);
+  return b;
+}
+
+EXTERNC struct SortOfEmpty sort_of_empty_arg_variadic(int a, ...) {
+// CHECK-LABEL: @sort_of_empty_arg_variadic(
+// DARWIN: %argp.next = getelementptr inbounds i8, ptr %argp.cur, i64 0
+// C-NOT: {{ getelementptr }}
+// CXX-NOT: {{ getelementptr }}
+  va_list vl;
+  va_start(vl, a);
+  struct SortOfEmpty b = va_arg(vl, struct SortOfEmpty);
+  va_end(vl);
+  return b;
+}
+
diff --git a/clang/test/CodeGen/AArch64/builtin-shufflevector-fp8.c b/clang/test/CodeGen/AArch64/builtin-shufflevector-fp8.c
new file mode 100644
index 0000000000000..147ca1d1becc1
--- /dev/null
+++ b/clang/test/CodeGen/AArch64/builtin-shufflevector-fp8.c
@@ -0,0 +1,123 @@
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --version 5
+// RUN: %clang_cc1 -triple aarch64-linux -target-feature +neon -disable-O0-optnone -emit-llvm -o - %s | opt -S -passes=mem2reg | FileCheck %s
+
+// REQUIRES: aarch64-registered-target
+
+typedef __attribute__((neon_vector_type(8))) signed char int8x8_t;
+typedef __attribute__((neon_vector_type(16))) signed char int8x16_t;
+
+typedef __attribute__((neon_vector_type(8))) __mfp8 mfloat8x8_t;
+typedef __attribute__((neon_vector_type(16))) __mfp8 mfloat8x16_t;
+
+// CHECK-LABEL: define dso_local <8 x i8> @test_8x8(
+// CHECK-SAME: <8 x i8> [[X:%.*]]) #[[ATTR0:[0-9]+]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[SHUFFLE:%.*]] = shufflevector <8 x i8> [[X]], <8 x i8> [[X]], <8 x i32> <i32 3, i32 2, i32 1, i32 0, i32 3, i32 2, i32 1, i32 0>
+// CHECK-NEXT:    ret <8 x i8> [[SHUFFLE]]
+//
+mfloat8x8_t test_8x8(mfloat8x8_t x) {
+  return __builtin_shufflevector(x, x, 3, 2, 1, 0, 3, 2, 1, 0);
+}
+
+// CHECK-LABEL: define dso_local <8 x i8> @test_8x8_v(
+// CHECK-SAME: <8 x i8> [[X:%.*]], <8 x i8> noundef [[P:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[MASK:%.*]] = and <8 x i8> [[P]], splat (i8 7)
+// CHECK-NEXT:    [[SHUF_IDX:%.*]] = extractelement <8 x i8> [[MASK]], i64 0
+// CHECK-NEXT:    [[SHUF_ELT:%.*]] = extractelement <8 x i8> [[X]], i8 [[SHUF_IDX]]
+// CHECK-NEXT:    [[SHUF_INS:%.*]] = insertelement <8 x i8> poison, i8 [[SHUF_ELT]], i64 0
+// CHECK-NEXT:    [[SHUF_IDX1:%.*]] = extractelement <8 x i8> [[MASK]], i64 1
+// CHECK-NEXT:    [[SHUF_ELT2:%.*]] = extractelement <8 x i8> [[X]], i8 [[SHUF_IDX1]]
+// CHECK-NEXT:    [[SHUF_INS3:%.*]] = insertelement <8 x i8> [[SHUF_INS]], i8 [[SHUF_ELT2]], i64 1
+// CHECK-NEXT:    [[SHUF_IDX4:%.*]] = extractelement <8 x i8> [[MASK]], i64 2
+// CHECK-NEXT:    [[SHUF_ELT5:%.*]] = extractelement <8 x i8> [[X]], i8 [[SHUF_IDX4]]
+// CHECK-NEXT:    [[SHUF_INS6:%.*]] = insertelement <8 x i8> [[SHUF_INS3]], i8 [[SHUF_ELT5]], i64 2
+// CHECK-NEXT:    [[SHUF_IDX7:%.*]] = extractelement <8 x i8> [[MASK]], i64 3
+// CHECK-NEXT:    [[SHUF_ELT8:%.*]] = extractelement <8 x i8> [[X]], i8 [[SHUF_IDX7]]
+// CHECK-NEXT:    [[SHUF_INS9:%.*]] = insertelement <8 x i8> [[SHUF_INS6]], i8 [[SHUF_ELT8]], i64 3
+// CHECK-NEXT:    [[SHUF_IDX10:%.*]] = extractelement <8 x i8> [[MASK]], i64 4
+// CHECK-NEXT:    [[SHUF_ELT11:%.*]] = extractelement <8 x i8> [[X]], i8 [[SHUF_IDX10]]
+// CHECK-NEXT:    [[SHUF_INS12:%.*]] = insertelement <8 x i8> [[SHUF_INS9]], i8 [[SHUF_ELT11]], i64 4
+// CHECK-NEXT:    [[SHUF_IDX13:%.*]] = extractelement <8 x i8> [[MASK]], i64 5
+// CHECK-NEXT:    [[SHUF_ELT14:%.*]] = extractelement <8 x i8> [[X]], i8 [[SHUF_IDX13]]
+// CHECK-NEXT:    [[SHUF_INS15:%.*]] = insertelement <8 x i8> [[SHUF_INS12]], i8 [[SHUF_ELT14]], i64 5
+// CHECK-NEXT:    [[SHUF_IDX16:%.*]] = extractelement <8 x i8> [[MASK]], i64 6
+// CHECK-NEXT:    [[SHUF_ELT17:%.*]] = extractelement <8 x i8> [[X]], i8 [[SHUF_IDX16]]
+// CHECK-NEXT:    [[SHUF_INS18:%.*]] = insertelement <8 x i8> [[SHUF_INS15]], i8 [[SHUF_ELT17]], i64 6
+// CHECK-NEXT:    [[SHUF_IDX19:%.*]] = extractelement <8 x i8> [[MASK]], i64 7
+// CHECK-NEXT:    [[SHUF_ELT20:%.*]] = extractelement <8 x i8> [[X]], i8 [[SHUF_IDX19]]
+// CHECK-NEXT:    [[SHUF_INS21:%.*]] = insertelement <8 x i8> [[SHUF_INS18]], i8 [[SHUF_ELT20]], i64 7
+// CHECK-NEXT:    ret <8 x i8> [[SHUF_INS21]]
+//
+mfloat8x8_t test_8x8_v(mfloat8x8_t x, int8x8_t p) {
+  return __builtin_shufflevector(x, p);
+}
+
+// CHECK-LABEL: define dso_local <16 x i8> @test_8x16(
+// CHECK-SAME: <16 x i8> [[X:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[SHUFFLE:%.*]] = shufflevector <16 x i8> [[X]], <16 x i8> [[X]], <16 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+// CHECK-NEXT:    ret <16 x i8> [[SHUFFLE]]
+//
+mfloat8x16_t test_8x16(mfloat8x16_t x) {
+  return __builtin_shufflevector(x, x, 7, 6, 5, 4, 3, 2, 1, 0, 7, 6, 5, 4, 3, 2,
+                                 1, 0);
+}
+
+// CHECK-LABEL: define dso_local <16 x i8> @test_8x16_v(
+// CHECK-SAME: <16 x i8> [[X:%.*]], <16 x i8> noundef [[P:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[MASK:%.*]] = and <16 x i8> [[P]], splat (i8 15)
+// CHECK-NEXT:    [[SHUF_IDX:%.*]] = extractelement <16 x i8> [[MASK]], i64 0
+// CHECK-NEXT:    [[SHUF_ELT:%.*]] = extractelement <16 x i8> [[X]], i8 [[SHUF_IDX]]
+// CHECK-NEXT:    [[SHUF_INS:%.*]] = insertelement <16 x i8> poison, i8 [[SHUF_ELT]], i64 0
+// CHECK-NEXT:    [[SHUF_IDX1:%.*]] = extractelement <16 x i8> [[MASK]], i64 1
+// CHECK-NEXT:    [[SHUF_ELT2:%.*]] = extractelement <16 x i8> [[X]], i8 [[SHUF_IDX1]]
+// CHECK-NEXT:    [[SHUF_INS3:%.*]] = insertelement <16 x i8> [[SHUF_INS]], i8 [[SHUF_ELT2]], i64 1
+// CHECK-NEXT:    [[SHUF_IDX4:%.*]] = extractelement <16 x i8> [[MASK]], i64 2
+// CHECK-NEXT:    [[SHUF_ELT5:%.*]] = extractelement <16 x i8> [[X]], i8 [[SHUF_IDX4]]
+// CHECK-NEXT:    [[SHUF_INS6:%.*]] = insertelement <16 x i8> [[SHUF_INS3]], i8 [[SHUF_ELT5]], i64 2
+// CHECK-NEXT:    [[SHUF_IDX7:%.*]] = extractelement <16 x i8> [[MASK]], i64 3
+// CHECK-NEXT:    [[SHUF_ELT8:%.*]] = extractelement <16 x i8> [[X]], i8 [[SHUF_IDX7]]
+// CHECK-NEXT:    [[SHUF_INS9:%.*]] = insertelement <16 x i8> [[SHUF_INS6]], i8 [[SHUF_ELT8]], i64 3
+// CHECK-NEXT:    [[SHUF_IDX10:%.*]] = extractelement <16 x i8> [[MASK]], i64 4
+// CHECK-NEXT:    [[SHUF_ELT11:%.*]] = extractelement <16 x i8> [[X]], i8 [[SHUF_IDX10]]
+// CHECK-NEXT:    [[SHUF_INS12:%.*]] = insertelement <16 x i8> [[SHUF_INS9]], i8 [[SHUF_ELT11]], i64 4
+// CHECK-NEXT:    [[SHUF_IDX13:%.*]] = extractelement <16 x i8> [[MASK]], i64 5
+// CHECK-NEXT:    [[SHUF_ELT14:%.*]] = extractelement <16 x i8> [[X]], i8 [[SHUF_IDX13]]
+// CHECK-NEXT:    [[SHUF_INS15:%.*]] = insertelement <16 x i8> [[SHUF_INS12]], i8 [[SHUF_ELT14]], i64 5
+// CHECK-NEXT:    [[SHUF_IDX16:%.*]] = extractelement <16 x i8> [[MASK]], i64 6
+// CHECK-NEXT:    [[SHUF_ELT17:%.*]] = extractelement <16 x i8> [[X]], i8 [[SHUF_IDX16]]
+// CHECK-NEXT:    [[SHUF_INS18:%.*]] = insertelement <16 x i8> [[SHUF_INS15]], i8 [[SHUF_ELT17]], i64 6
+// CHECK-NEXT:    [[SHUF_IDX19:%.*]] = extractelement <16 x i8> [[MASK]], i64 7
+// CHECK-NEXT:    [[SHUF_ELT20:%.*]] = extractelement <16 x i8> [[X]], i8 [[SHUF_IDX19]]
+// CHECK-NEXT:    [[SHUF_INS21:%.*]] = insertelement <16 x i8> [[SHUF_INS18]], i8 [[SHUF_ELT20]], i64 7
+// CHECK-NEXT:    [[SHUF_IDX22:%.*]] = extractelement <16 x i8> [[MASK]], i64 8
+// CHECK-NEXT:    [[SHUF_ELT23:%.*]] = extractelement <16 x i8> [[X]], i8 [[SHUF_IDX22]]
+// CHECK-NEXT:    [[SHUF_INS24:%.*]] = insertelement <16 x i8> [[SHUF_INS21]], i8 [[SHUF_ELT23]], i64 8
+// CHECK-NEXT:    [[SHUF_IDX25:%.*]] = extractelement <16 x i8> [[MASK]], i64 9
+// CHECK-NEXT:    [[SHUF_ELT26:%.*]] = extractelement <16 x i8> [[X]], i8 [[SHUF_IDX25]]
+// CHECK-NEXT:    [[SHUF_INS27:%.*]] = insertelement <16 x i8> [[SHUF_INS24]], i8 [[SHUF_ELT26]], i64 9
+// CHECK-NEXT:    [[SHUF_IDX28:%.*]] = extractelement <16 x i8> [[MASK]], i64 10
+// CHECK-NEXT:    [[SHUF_ELT29:%.*]] = extractelement <16 x i8> [[X]], i8 [[SHUF_IDX28]]
+// CHECK-NEXT:    [[SHUF_INS30:%.*]] = insertelement <16 x i8> [[SHUF_INS27]], i8 [[SHUF_ELT29]], i64 10
+// CHECK-NEXT:    [[SHUF_IDX31:%.*]] = extractelement <16 x i8> [[MASK]], i64 11
+// CHECK-NEXT:    [[SHUF_ELT32:%.*]] = extractelement <16 x i8> [[X]], i8 [[SHUF_IDX31]]
+// CHECK-NEXT:    [[SHUF_INS33:%.*]] = insertelement <16 x i8> [[SHUF_INS30]], i8 [[SHUF_ELT32]], i64 11
+// CHECK-NEXT:    [[SHUF_IDX34:%.*]] = extractelement <16 x i8> [[MASK]], i64 12
+// CHECK-NEXT:    [[SHUF_ELT35:%.*]] = extractelement <16 x i8> [[X]], i8 [[SHUF_IDX34]]
+// CHECK-NEXT:    [[SHUF_INS36:%.*]] = insertelement <16 x i8> [[SHUF_INS33]], i8 [[SHUF_ELT35]], i64 12
+// CHECK-NEXT:    [[SHUF_IDX37:%.*]] = extractelement <16 x i8> [[MASK]], i64 13
+// CHECK-NEXT:    [[SHUF_ELT38:%.*]] = extractelement <16 x i8> [[X]], i8 [[SHUF_IDX37]]
+// CHECK-NEXT:    [[SHUF_INS39:%.*]] = insertelement <16 x i8> [[SHUF_INS36]], i8 [[SHUF_ELT38]], i64 13
+// CHECK-NEXT:    [[SHUF_IDX40:%.*]] = extractelement <16 x i8> [[MASK]], i64 14
+// CHECK-NEXT:    [[SHUF_ELT41:%.*]] = extractelement <16 x i8> [[X]], i8 [[SHUF_IDX40]]
+// CHECK-NEXT:    [[SHUF_INS42:%.*]] = insertelement <16 x i8> [[SHUF_INS39]], i8 [[SHUF_ELT41]], i64 14
+// CHECK-NEXT:    [[SHUF_IDX43:%.*]] = extractelement <16 x i8> [[MASK]], i64 15
+// CHECK-NEXT:    [[SHUF_ELT44:%.*]] = extractelement <16 x i8> [[X]], i8 [[SHUF_IDX43]]
+// CHECK-NEXT:    [[SHUF_INS45:%.*]] = insertelement <16 x i8> [[SHUF_INS42]], i8 [[SHUF_ELT44]], i64 15
+// CHECK-NEXT:    ret <16 x i8> [[SHUF_INS45]]
+//
+mfloat8x16_t test_8x16_v(mfloat8x16_t x, int8x16_t p) {
+  return __builtin_shufflevector(x, p);
+}
diff --git a/clang/test/CodeGen/AArch64/fp8-cast.c b/clang/test/CodeGen/AArch64/fp8-cast.c
new file mode 100644
index 0000000000000..a9ce31b9e6bea
--- /dev/null
+++ b/clang/test/CodeGen/AArch64/fp8-cast.c
@@ -0,0 +1,193 @@
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --version 5
+// RUN: %clang_cc1        -triple aarch64-none-linux-gnu -target-feature +neon -disable-O0-optnone -Werror -Wall -emit-llvm -o - %s | opt -S -p mem2reg | FileCheck %s
+// RUN: %clang_cc1 -x c++ -triple aarch64-none-linux-gnu -target-feature +neon -disable-O0-optnone -Werror -Wall -emit-llvm -o - %s | opt -S -p mem2reg | FileCheck %s -check-prefix CHECK-CXX
+
+// RUN: %clang_cc1        -triple aarch64-none-linux-gnu -target-feature +neon -disable-O0-optnone -Werror -Wall -S -o /dev/null %s
+
+// REQUIRES: aarch64-registered-target
+
+#include <arm_neon.h>
+
+// Bitcast between FP8 Neon vectors
+// CHECK-LABEL: define dso_local <8 x i8> @test_f8_f8(
+// CHECK-SAME: <8 x i8> [[X:%.*]]) #[[ATTR0:[0-9]+]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    ret <8 x i8> [[X]]
+//
+// CHECK-CXX-LABEL: define dso_local <8 x i8> @_Z10test_f8_f813__Mfloat8x8_t(
+// CHECK-CXX-SAME: <8 x i8> [[X:%.*]]) #[[ATTR0:[0-9]+]] {
+// CHECK-CXX-NEXT:  [[ENTRY:.*:]]
+// CHECK-CXX-NEXT:    ret <8 x i8> [[X]]
+//
+mfloat8x8_t test_f8_f8(mfloat8x8_t x) {
+    return (mfloat8x8_t) x;
+}
+
+// CHECK-LABEL: define dso_local <16 x i8> @testq_f8_f8(
+// CHECK-SAME: <16 x i8> [[X:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    ret <16 x i8> [[X]]
+//
+// CHECK-CXX-LABEL: define dso_local <16 x i8> @_Z11testq_f8_f814__Mfloat8x16_t(
+// CHECK-CXX-SAME: <16 x i8> [[X:%.*]]) #[[ATTR0]] {
+// CHECK-CXX-NEXT:  [[ENTRY:.*:]]
+// CHECK-CXX-NEXT:    ret <16 x i8> [[X]]
+//
+mfloat8x16_t testq_f8_f8(mfloat8x16_t x) {
+    return (mfloat8x16_t) x;
+}
+
+// Bitcast between FP8 and int8 Neon vectors
+// CHECK-LABEL: define dso_local <8 x i8> @test_f8_s8(
+// CHECK-SAME: <8 x i8> noundef [[X:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    ret <8 x i8> [[X]]
+//
+// CHECK-CXX-LABEL: define dso_local <8 x i8> @_Z10test_f8_s810__Int8x8_t(
+// CHECK-CXX-SAME: <8 x i8> noundef [[X:%.*]]) #[[ATTR0]] {
+// CHECK-CXX-NEXT:  [[ENTRY:.*:]]
+// CHECK-CXX-NEXT:    ret <8 x i8> [[X]]
+//
+mfloat8x8_t test_f8_s8(int8x8_t x) {
+    return (mfloat8x8_t) x;
+}
+
+// CHECK-LABEL: define dso_local <8 x i8> @test_s8_f8(
+// CHECK-SAME: <8 x i8> [[X:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    ret <8 x i8> [[X]]
+//
+// CHECK-CXX-LABEL: define dso_local noundef <8 x i8> @_Z10test_s8_f813__Mfloat8x8_t(
+// CHECK-CXX-SAME: <8 x i8> [[X:%.*]]) #[[ATTR0]] {
+// CHECK-CXX-NEXT:  [[ENTRY:.*:]]
+// CHECK-CXX-NEXT:    ret <8 x i8> [[X]]
+//
+int8x8_t test_s8_f8(mfloat8x8_t x) {
+    return (int8x8_t) x;
+}
+
+// CHECK-LABEL: define dso_local <16 x i8> @testq_f8_s8(
+// CHECK-SAME: <16 x i8> noundef [[X:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    ret <16 x i8> [[X]]
+//
+// CHECK-CXX-LABEL: define dso_local <16 x i8> @_Z11testq_f8_s811__Int8x16_t(
+// CHECK-CXX-SAME: <16 x i8> noundef [[X:%.*]]) #[[ATTR0]] {
+// CHECK-CXX-NEXT:  [[ENTRY:.*:]]
+// CHECK-CXX-NEXT:    ret <16 x i8> [[X]]
+//
+mfloat8x16_t testq_f8_s8(int8x16_t x) {
+    return (mfloat8x16_t) x;
+}
+
+// CHECK-LABEL: define dso_local <16 x i8> @testq_s8_f8(
+// CHECK-SAME: <16 x i8> [[X:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    ret <16 x i8> [[X]]
+//
+// CHECK-CXX-LABEL: define dso_local noundef <16 x i8> @_Z11testq_s8_f814__Mfloat8x16_t(
+// CHECK-CXX-SAME: <16 x i8> [[X:%.*]]) #[[ATTR0]] {
+// CHECK-CXX-NEXT:  [[ENTRY:.*:]]
+// CHECK-CXX-NEXT:    ret <16 x i8> [[X]]
+//
+int8x16_t testq_s8_f8(mfloat8x16_t x) {
+    return (int8x16_t) x;
+}
+
+// Bitcast between FP8 and float32 Neon vectors
+// CHECK-LABEL: define dso_local <8 x i8> @test_f8_f32(
+// CHECK-SAME: <2 x float> noundef [[X:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[TMP0:%.*]] = bitcast <2 x float> [[X]] to <8 x i8>
+// CHECK-NEXT:    ret <8 x i8> [[TMP0]]
+//
+// CHECK-CXX-LABEL: define dso_local <8 x i8> @_Z11test_f8_f3213__Float32x2_t(
+// CHECK-CXX-SAME: <2 x float> noundef [[X:%.*]]) #[[ATTR0]] {
+// CHECK-CXX-NEXT:  [[ENTRY:.*:]]
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = bitcast <2 x float> [[X]] to <8 x i8>
+// CHECK-CXX-NEXT:    ret <8 x i8> [[TMP0]]
+//
+mfloat8x8_t test_f8_f32(float32x2_t x) {
+    return (mfloat8x8_t) x;
+}
+
+// CHECK-LABEL: define dso_local <2 x float> @test_f32_f8(
+// CHECK-SAME: <8 x i8> [[X:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[TMP0:%.*]] = bitcast <8 x i8> [[X]] to <2 x float>
+// CHECK-NEXT:    ret <2 x float> [[TMP0]]
+//
+// CHECK-CXX-LABEL: define dso_local noundef <2 x float> @_Z11test_f32_f813__Mfloat8x8_t(
+// CHECK-CXX-SAME: <8 x i8> [[X:%.*]]) #[[ATTR0]] {
+// CHECK-CXX-NEXT:  [[ENTRY:.*:]]
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = bitcast <8 x i8> [[X]] to <2 x float>
+// CHECK-CXX-NEXT:    ret <2 x float> [[TMP0]]
+//
+float32x2_t test_f32_f8(mfloat8x8_t x) {
+    return (float32x2_t) x;
+}
+
+// CHECK-LABEL: define dso_local <16 x i8> @testq_f8_f32(
+// CHECK-SAME: <4 x float> noundef [[X:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[TMP0:%.*]] = bitcast <4 x float> [[X]] to <16 x i8>
+// CHECK-NEXT:    ret <16 x i8> [[TMP0]]
+//
+// CHECK-CXX-LABEL: define dso_local <16 x i8> @_Z12testq_f8_f3213__Float32x4_t(
+// CHECK-CXX-SAME: <4 x float> noundef [[X:%.*]]) #[[ATTR0]] {
+// CHECK-CXX-NEXT:  [[ENTRY:.*:]]
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = bitcast <4 x float> [[X]] to <16 x i8>
+// CHECK-CXX-NEXT:    ret <16 x i8> [[TMP0]]
+//
+mfloat8x16_t testq_f8_f32(float32x4_t x) {
+    return (mfloat8x16_t) x;
+}
+
+// CHECK-LABEL: define dso_local <4 x float> @testq_f32_f8(
+// CHECK-SAME: <16 x i8> [[X:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[TMP0:%.*]] = bitcast <16 x i8> [[X]] to <4 x float>
+// CHECK-NEXT:    ret <4 x float> [[TMP0]]
+//
+// CHECK-CXX-LABEL: define dso_local noundef <4 x float> @_Z12testq_f32_f814__Mfloat8x16_t(
+// CHECK-CXX-SAME: <16 x i8> [[X:%.*]]) #[[ATTR0]] {
+// CHECK-CXX-NEXT:  [[ENTRY:.*:]]
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = bitcast <16 x i8> [[X]] to <4 x float>
+// CHECK-CXX-NEXT:    ret <4 x float> [[TMP0]]
+//
+float32x4_t testq_f32_f8(mfloat8x16_t x) {
+    return (float32x4_t) x;
+}
+
+// Bitcast between FP8 and poly128_t (which is integral)
+// CHECK-LABEL: define dso_local <16 x i8> @testq_f8_p128(
+// CHECK-SAME: i128 noundef [[X:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[TMP0:%.*]] = bitcast i128 [[X]] to <16 x i8>
+// CHECK-NEXT:    ret <16 x i8> [[TMP0]]
+//
+// CHECK-CXX-LABEL: define dso_local <16 x i8> @_Z13testq_f8_p128o(
+// CHECK-CXX-SAME: i128 noundef [[X:%.*]]) #[[ATTR0]] {
+// CHECK-CXX-NEXT:  [[ENTRY:.*:]]
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = bitcast i128 [[X]] to <16 x i8>
+// CHECK-CXX-NEXT:    ret <16 x i8> [[TMP0]]
+//
+mfloat8x16_t testq_f8_p128(poly128_t x) {
+    return (mfloat8x16_t) x;
+}
+
+// CHECK-LABEL: define dso_local i128 @testq_p128_f8(
+// CHECK-SAME: <16 x i8> [[X:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[TMP0:%.*]] = bitcast <16 x i8> [[X]] to i128
+// CHECK-NEXT:    ret i128 [[TMP0]]
+//
+// CHECK-CXX-LABEL: define dso_local noundef i128 @_Z13testq_p128_f814__Mfloat8x16_t(
+// CHECK-CXX-SAME: <16 x i8> [[X:%.*]]) #[[ATTR0]] {
+// CHECK-CXX-NEXT:  [[ENTRY:.*:]]
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = bitcast <16 x i8> [[X]] to i128
+// CHECK-CXX-NEXT:    ret i128 [[TMP0]]
+//
+poly128_t testq_p128_f8(mfloat8x16_t x) {
+    return (poly128_t) x;
+}
diff --git a/clang/test/CodeGen/AArch64/fp8-intrinsics/acle_neon_fp8_cvt.c b/clang/test/CodeGen/AArch64/fp8-intrinsics/acle_neon_fp8_cvt.c
new file mode 100644
index 0000000000000..4305b840f2a05
--- /dev/null
+++ b/clang/test/CodeGen/AArch64/fp8-intrinsics/acle_neon_fp8_cvt.c
@@ -0,0 +1,316 @@
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --version 5
+// RUN: %clang_cc1        -triple aarch64-none-linux-gnu -target-feature +neon -target-feature +bf16 -target-feature +fp8 -disable-O0-optnone -Werror -Wall -emit-llvm -o - %s | opt -S -p mem2reg | FileCheck %s
+// RUN: %clang_cc1 -x c++ -triple aarch64-none-linux-gnu -target-feature +neon -target-feature +bf16 -target-feature +fp8 -disable-O0-optnone -Werror -Wall -emit-llvm -o - %s | opt -S -p mem2reg | FileCheck %s -check-prefix CHECK-CXX
+
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +neon -target-feature +bf16 -target-feature +fp8 -S -O3 -o /dev/null %s
+
+// REQUIRES: aarch64-registered-target
+
+#include <arm_neon.h>
+
+// CHECK-LABEL: define dso_local <8 x bfloat> @test_vcvt1_bf16_mf8_fpm(
+// CHECK-SAME: <8 x i8> [[OP:%.*]], i64 noundef [[FPM:%.*]]) #[[ATTR0:[0-9]+]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    call void @llvm.aarch64.set.fpmr(i64 [[FPM]])
+// CHECK-NEXT:    [[VBFCVT1_I:%.*]] = call <8 x bfloat> @llvm.aarch64.neon.fp8.cvtl1.v8bf16.v8i8(<8 x i8> [[OP]])
+// CHECK-NEXT:    ret <8 x bfloat> [[VBFCVT1_I]]
+//
+// CHECK-CXX-LABEL: define dso_local noundef <8 x bfloat> @_Z23test_vcvt1_bf16_mf8_fpm13__Mfloat8x8_tm(
+// CHECK-CXX-SAME: <8 x i8> [[OP:%.*]], i64 noundef [[FPM:%.*]]) #[[ATTR0:[0-9]+]] {
+// CHECK-CXX-NEXT:  [[ENTRY:.*:]]
+// CHECK-CXX-NEXT:    call void @llvm.aarch64.set.fpmr(i64 [[FPM]])
+// CHECK-CXX-NEXT:    [[VBFCVT1_I:%.*]] = call <8 x bfloat> @llvm.aarch64.neon.fp8.cvtl1.v8bf16.v8i8(<8 x i8> [[OP]])
+// CHECK-CXX-NEXT:    ret <8 x bfloat> [[VBFCVT1_I]]
+//
+bfloat16x8_t test_vcvt1_bf16_mf8_fpm(mfloat8x8_t op, fpm_t fpm) {
+  return vcvt1_bf16_mf8_fpm(op, fpm);
+}
+
+// CHECK-LABEL: define dso_local <8 x bfloat> @test_vcvt1_low_bf16_mf8_fpm(
+// CHECK-SAME: <16 x i8> [[OP:%.*]], i64 noundef [[FPM:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[TMP0:%.*]] = call <8 x i8> @llvm.vector.extract.v8i8.v16i8(<16 x i8> [[OP]], i64 0)
+// CHECK-NEXT:    call void @llvm.aarch64.set.fpmr(i64 [[FPM]])
+// CHECK-NEXT:    [[VBFCVT1_I:%.*]] = call <8 x bfloat> @llvm.aarch64.neon.fp8.cvtl1.v8bf16.v8i8(<8 x i8> [[TMP0]])
+// CHECK-NEXT:    ret <8 x bfloat> [[VBFCVT1_I]]
+//
+// CHECK-CXX-LABEL: define dso_local noundef <8 x bfloat> @_Z27test_vcvt1_low_bf16_mf8_fpm14__Mfloat8x16_tm(
+// CHECK-CXX-SAME: <16 x i8> [[OP:%.*]], i64 noundef [[FPM:%.*]]) #[[ATTR0]] {
+// CHECK-CXX-NEXT:  [[ENTRY:.*:]]
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = call <8 x i8> @llvm.vector.extract.v8i8.v16i8(<16 x i8> [[OP]], i64 0)
+// CHECK-CXX-NEXT:    call void @llvm.aarch64.set.fpmr(i64 [[FPM]])
+// CHECK-CXX-NEXT:    [[VBFCVT1_I:%.*]] = call <8 x bfloat> @llvm.aarch64.neon.fp8.cvtl1.v8bf16.v8i8(<8 x i8> [[TMP0]])
+// CHECK-CXX-NEXT:    ret <8 x bfloat> [[VBFCVT1_I]]
+//
+bfloat16x8_t test_vcvt1_low_bf16_mf8_fpm(mfloat8x16_t op, fpm_t fpm) {
+  return vcvt1_low_bf16_mf8_fpm(op, fpm);
+}
+
+// CHECK-LABEL: define dso_local <8 x bfloat> @test_vcvt2_bf16_mf8_fpm(
+// CHECK-SAME: <8 x i8> [[OP:%.*]], i64 noundef [[FPM:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    call void @llvm.aarch64.set.fpmr(i64 [[FPM]])
+// CHECK-NEXT:    [[VBFCVT2_I:%.*]] = call <8 x bfloat> @llvm.aarch64.neon.fp8.cvtl2.v8bf16.v8i8(<8 x i8> [[OP]])
+// CHECK-NEXT:    ret <8 x bfloat> [[VBFCVT2_I]]
+//
+// CHECK-CXX-LABEL: define dso_local noundef <8 x bfloat> @_Z23test_vcvt2_bf16_mf8_fpm13__Mfloat8x8_tm(
+// CHECK-CXX-SAME: <8 x i8> [[OP:%.*]], i64 noundef [[FPM:%.*]]) #[[ATTR0]] {
+// CHECK-CXX-NEXT:  [[ENTRY:.*:]]
+// CHECK-CXX-NEXT:    call void @llvm.aarch64.set.fpmr(i64 [[FPM]])
+// CHECK-CXX-NEXT:    [[VBFCVT2_I:%.*]] = call <8 x bfloat> @llvm.aarch64.neon.fp8.cvtl2.v8bf16.v8i8(<8 x i8> [[OP]])
+// CHECK-CXX-NEXT:    ret <8 x bfloat> [[VBFCVT2_I]]
+//
+bfloat16x8_t test_vcvt2_bf16_mf8_fpm(mfloat8x8_t op, fpm_t fpm) {
+  return vcvt2_bf16_mf8_fpm(op, fpm);
+}
+
+// CHECK-LABEL: define dso_local <8 x bfloat> @test_vcvt2_low_bf16_mf8_fpm(
+// CHECK-SAME: <16 x i8> [[OP:%.*]], i64 noundef [[FPM:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[TMP0:%.*]] = call <8 x i8> @llvm.vector.extract.v8i8.v16i8(<16 x i8> [[OP]], i64 0)
+// CHECK-NEXT:    call void @llvm.aarch64.set.fpmr(i64 [[FPM]])
+// CHECK-NEXT:    [[VBFCVT2_I:%.*]] = call <8 x bfloat> @llvm.aarch64.neon.fp8.cvtl2.v8bf16.v8i8(<8 x i8> [[TMP0]])
+// CHECK-NEXT:    ret <8 x bfloat> [[VBFCVT2_I]]
+//
+// CHECK-CXX-LABEL: define dso_local noundef <8 x bfloat> @_Z27test_vcvt2_low_bf16_mf8_fpm14__Mfloat8x16_tm(
+// CHECK-CXX-SAME: <16 x i8> [[OP:%.*]], i64 noundef [[FPM:%.*]]) #[[ATTR0]] {
+// CHECK-CXX-NEXT:  [[ENTRY:.*:]]
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = call <8 x i8> @llvm.vector.extract.v8i8.v16i8(<16 x i8> [[OP]], i64 0)
+// CHECK-CXX-NEXT:    call void @llvm.aarch64.set.fpmr(i64 [[FPM]])
+// CHECK-CXX-NEXT:    [[VBFCVT2_I:%.*]] = call <8 x bfloat> @llvm.aarch64.neon.fp8.cvtl2.v8bf16.v8i8(<8 x i8> [[TMP0]])
+// CHECK-CXX-NEXT:    ret <8 x bfloat> [[VBFCVT2_I]]
+//
+bfloat16x8_t test_vcvt2_low_bf16_mf8_fpm(mfloat8x16_t op, fpm_t fpm) {
+  return vcvt2_low_bf16_mf8_fpm(op, fpm);
+}
+
+// CHECK-LABEL: define dso_local <8 x bfloat> @test_vcvt1_high_bf16_mf8_fpm(
+// CHECK-SAME: <16 x i8> [[OP:%.*]], i64 noundef [[FPM:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    call void @llvm.aarch64.set.fpmr(i64 [[FPM]])
+// CHECK-NEXT:    [[VBFCVT1_I:%.*]] = call <8 x bfloat> @llvm.aarch64.neon.fp8.cvtl1.v8bf16.v16i8(<16 x i8> [[OP]])
+// CHECK-NEXT:    ret <8 x bfloat> [[VBFCVT1_I]]
+//
+// CHECK-CXX-LABEL: define dso_local noundef <8 x bfloat> @_Z28test_vcvt1_high_bf16_mf8_fpm14__Mfloat8x16_tm(
+// CHECK-CXX-SAME: <16 x i8> [[OP:%.*]], i64 noundef [[FPM:%.*]]) #[[ATTR0]] {
+// CHECK-CXX-NEXT:  [[ENTRY:.*:]]
+// CHECK-CXX-NEXT:    call void @llvm.aarch64.set.fpmr(i64 [[FPM]])
+// CHECK-CXX-NEXT:    [[VBFCVT1_I:%.*]] = call <8 x bfloat> @llvm.aarch64.neon.fp8.cvtl1.v8bf16.v16i8(<16 x i8> [[OP]])
+// CHECK-CXX-NEXT:    ret <8 x bfloat> [[VBFCVT1_I]]
+//
+bfloat16x8_t test_vcvt1_high_bf16_mf8_fpm(mfloat8x16_t op, fpm_t fpm) {
+  return vcvt1_high_bf16_mf8_fpm(op, fpm);
+}
+
+// CHECK-LABEL: define dso_local <8 x bfloat> @test_vcvt2_high_bf16_mf8_fpm(
+// CHECK-SAME: <16 x i8> [[OP:%.*]], i64 noundef [[FPM:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    call void @llvm.aarch64.set.fpmr(i64 [[FPM]])
+// CHECK-NEXT:    [[VBFCVT2_I:%.*]] = call <8 x bfloat> @llvm.aarch64.neon.fp8.cvtl2.v8bf16.v16i8(<16 x i8> [[OP]])
+// CHECK-NEXT:    ret <8 x bfloat> [[VBFCVT2_I]]
+//
+// CHECK-CXX-LABEL: define dso_local noundef <8 x bfloat> @_Z28test_vcvt2_high_bf16_mf8_fpm14__Mfloat8x16_tm(
+// CHECK-CXX-SAME: <16 x i8> [[OP:%.*]], i64 noundef [[FPM:%.*]]) #[[ATTR0]] {
+// CHECK-CXX-NEXT:  [[ENTRY:.*:]]
+// CHECK-CXX-NEXT:    call void @llvm.aarch64.set.fpmr(i64 [[FPM]])
+// CHECK-CXX-NEXT:    [[VBFCVT2_I:%.*]] = call <8 x bfloat> @llvm.aarch64.neon.fp8.cvtl2.v8bf16.v16i8(<16 x i8> [[OP]])
+// CHECK-CXX-NEXT:    ret <8 x bfloat> [[VBFCVT2_I]]
+//
+bfloat16x8_t test_vcvt2_high_bf16_mf8_fpm(mfloat8x16_t op, fpm_t fpm) {
+  return vcvt2_high_bf16_mf8_fpm(op, fpm);
+}
+
+// CHECK-LABEL: define dso_local <8 x half> @test_vcvt1_f16_mf8_fpm(
+// CHECK-SAME: <8 x i8> [[OP:%.*]], i64 noundef [[FPM:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    call void @llvm.aarch64.set.fpmr(i64 [[FPM]])
+// CHECK-NEXT:    [[VBFCVT1_I:%.*]] = call <8 x half> @llvm.aarch64.neon.fp8.cvtl1.v8f16.v8i8(<8 x i8> [[OP]])
+// CHECK-NEXT:    ret <8 x half> [[VBFCVT1_I]]
+//
+// CHECK-CXX-LABEL: define dso_local noundef <8 x half> @_Z22test_vcvt1_f16_mf8_fpm13__Mfloat8x8_tm(
+// CHECK-CXX-SAME: <8 x i8> [[OP:%.*]], i64 noundef [[FPM:%.*]]) #[[ATTR0]] {
+// CHECK-CXX-NEXT:  [[ENTRY:.*:]]
+// CHECK-CXX-NEXT:    call void @llvm.aarch64.set.fpmr(i64 [[FPM]])
+// CHECK-CXX-NEXT:    [[VBFCVT1_I:%.*]] = call <8 x half> @llvm.aarch64.neon.fp8.cvtl1.v8f16.v8i8(<8 x i8> [[OP]])
+// CHECK-CXX-NEXT:    ret <8 x half> [[VBFCVT1_I]]
+//
+float16x8_t test_vcvt1_f16_mf8_fpm(mfloat8x8_t op, fpm_t fpm) {
+  return vcvt1_f16_mf8_fpm(op, fpm);
+}
+
+// CHECK-LABEL: define dso_local <8 x half> @test_vcvt1_low_f16_mf8_fpm(
+// CHECK-SAME: <16 x i8> [[OP:%.*]], i64 noundef [[FPM:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[TMP0:%.*]] = call <8 x i8> @llvm.vector.extract.v8i8.v16i8(<16 x i8> [[OP]], i64 0)
+// CHECK-NEXT:    call void @llvm.aarch64.set.fpmr(i64 [[FPM]])
+// CHECK-NEXT:    [[VBFCVT1_I:%.*]] = call <8 x half> @llvm.aarch64.neon.fp8.cvtl1.v8f16.v8i8(<8 x i8> [[TMP0]])
+// CHECK-NEXT:    ret <8 x half> [[VBFCVT1_I]]
+//
+// CHECK-CXX-LABEL: define dso_local noundef <8 x half> @_Z26test_vcvt1_low_f16_mf8_fpm14__Mfloat8x16_tm(
+// CHECK-CXX-SAME: <16 x i8> [[OP:%.*]], i64 noundef [[FPM:%.*]]) #[[ATTR0]] {
+// CHECK-CXX-NEXT:  [[ENTRY:.*:]]
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = call <8 x i8> @llvm.vector.extract.v8i8.v16i8(<16 x i8> [[OP]], i64 0)
+// CHECK-CXX-NEXT:    call void @llvm.aarch64.set.fpmr(i64 [[FPM]])
+// CHECK-CXX-NEXT:    [[VBFCVT1_I:%.*]] = call <8 x half> @llvm.aarch64.neon.fp8.cvtl1.v8f16.v8i8(<8 x i8> [[TMP0]])
+// CHECK-CXX-NEXT:    ret <8 x half> [[VBFCVT1_I]]
+//
+float16x8_t test_vcvt1_low_f16_mf8_fpm(mfloat8x16_t op, fpm_t fpm) {
+  return vcvt1_low_f16_mf8_fpm(op, fpm);
+}
+
+// CHECK-LABEL: define dso_local <8 x half> @test_vcvt2_f16_mf8_fpm(
+// CHECK-SAME: <8 x i8> [[OP:%.*]], i64 noundef [[FPM:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    call void @llvm.aarch64.set.fpmr(i64 [[FPM]])
+// CHECK-NEXT:    [[VBFCVT2_I:%.*]] = call <8 x half> @llvm.aarch64.neon.fp8.cvtl2.v8f16.v8i8(<8 x i8> [[OP]])
+// CHECK-NEXT:    ret <8 x half> [[VBFCVT2_I]]
+//
+// CHECK-CXX-LABEL: define dso_local noundef <8 x half> @_Z22test_vcvt2_f16_mf8_fpm13__Mfloat8x8_tm(
+// CHECK-CXX-SAME: <8 x i8> [[OP:%.*]], i64 noundef [[FPM:%.*]]) #[[ATTR0]] {
+// CHECK-CXX-NEXT:  [[ENTRY:.*:]]
+// CHECK-CXX-NEXT:    call void @llvm.aarch64.set.fpmr(i64 [[FPM]])
+// CHECK-CXX-NEXT:    [[VBFCVT2_I:%.*]] = call <8 x half> @llvm.aarch64.neon.fp8.cvtl2.v8f16.v8i8(<8 x i8> [[OP]])
+// CHECK-CXX-NEXT:    ret <8 x half> [[VBFCVT2_I]]
+//
+float16x8_t test_vcvt2_f16_mf8_fpm(mfloat8x8_t op, fpm_t fpm) {
+  return vcvt2_f16_mf8_fpm(op, fpm);
+}
+
+// CHECK-LABEL: define dso_local <8 x half> @test_vcvt2_low_f16_mf8_fpm(
+// CHECK-SAME: <16 x i8> [[OP:%.*]], i64 noundef [[FPM:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[TMP0:%.*]] = call <8 x i8> @llvm.vector.extract.v8i8.v16i8(<16 x i8> [[OP]], i64 0)
+// CHECK-NEXT:    call void @llvm.aarch64.set.fpmr(i64 [[FPM]])
+// CHECK-NEXT:    [[VBFCVT2_I:%.*]] = call <8 x half> @llvm.aarch64.neon.fp8.cvtl2.v8f16.v8i8(<8 x i8> [[TMP0]])
+// CHECK-NEXT:    ret <8 x half> [[VBFCVT2_I]]
+//
+// CHECK-CXX-LABEL: define dso_local noundef <8 x half> @_Z26test_vcvt2_low_f16_mf8_fpm14__Mfloat8x16_tm(
+// CHECK-CXX-SAME: <16 x i8> [[OP:%.*]], i64 noundef [[FPM:%.*]]) #[[ATTR0]] {
+// CHECK-CXX-NEXT:  [[ENTRY:.*:]]
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = call <8 x i8> @llvm.vector.extract.v8i8.v16i8(<16 x i8> [[OP]], i64 0)
+// CHECK-CXX-NEXT:    call void @llvm.aarch64.set.fpmr(i64 [[FPM]])
+// CHECK-CXX-NEXT:    [[VBFCVT2_I:%.*]] = call <8 x half> @llvm.aarch64.neon.fp8.cvtl2.v8f16.v8i8(<8 x i8> [[TMP0]])
+// CHECK-CXX-NEXT:    ret <8 x half> [[VBFCVT2_I]]
+//
+float16x8_t test_vcvt2_low_f16_mf8_fpm(mfloat8x16_t op, fpm_t fpm) {
+  return vcvt2_low_f16_mf8_fpm(op, fpm);
+}
+
+// CHECK-LABEL: define dso_local <8 x half> @test_vcvt1_high_f16_mf8_fpm(
+// CHECK-SAME: <16 x i8> [[OP:%.*]], i64 noundef [[FPM:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    call void @llvm.aarch64.set.fpmr(i64 [[FPM]])
+// CHECK-NEXT:    [[VBFCVT1_I:%.*]] = call <8 x half> @llvm.aarch64.neon.fp8.cvtl1.v8f16.v16i8(<16 x i8> [[OP]])
+// CHECK-NEXT:    ret <8 x half> [[VBFCVT1_I]]
+//
+// CHECK-CXX-LABEL: define dso_local noundef <8 x half> @_Z27test_vcvt1_high_f16_mf8_fpm14__Mfloat8x16_tm(
+// CHECK-CXX-SAME: <16 x i8> [[OP:%.*]], i64 noundef [[FPM:%.*]]) #[[ATTR0]] {
+// CHECK-CXX-NEXT:  [[ENTRY:.*:]]
+// CHECK-CXX-NEXT:    call void @llvm.aarch64.set.fpmr(i64 [[FPM]])
+// CHECK-CXX-NEXT:    [[VBFCVT1_I:%.*]] = call <8 x half> @llvm.aarch64.neon.fp8.cvtl1.v8f16.v16i8(<16 x i8> [[OP]])
+// CHECK-CXX-NEXT:    ret <8 x half> [[VBFCVT1_I]]
+//
+float16x8_t test_vcvt1_high_f16_mf8_fpm(mfloat8x16_t op, fpm_t fpm) {
+  return vcvt1_high_f16_mf8_fpm(op, fpm);
+}
+
+// CHECK-LABEL: define dso_local <8 x half> @test_vcvt2_high_f16_mf8_fpm(
+// CHECK-SAME: <16 x i8> [[OP:%.*]], i64 noundef [[FPM:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    call void @llvm.aarch64.set.fpmr(i64 [[FPM]])
+// CHECK-NEXT:    [[VBFCVT2_I:%.*]] = call <8 x half> @llvm.aarch64.neon.fp8.cvtl2.v8f16.v16i8(<16 x i8> [[OP]])
+// CHECK-NEXT:    ret <8 x half> [[VBFCVT2_I]]
+//
+// CHECK-CXX-LABEL: define dso_local noundef <8 x half> @_Z27test_vcvt2_high_f16_mf8_fpm14__Mfloat8x16_tm(
+// CHECK-CXX-SAME: <16 x i8> [[OP:%.*]], i64 noundef [[FPM:%.*]]) #[[ATTR0]] {
+// CHECK-CXX-NEXT:  [[ENTRY:.*:]]
+// CHECK-CXX-NEXT:    call void @llvm.aarch64.set.fpmr(i64 [[FPM]])
+// CHECK-CXX-NEXT:    [[VBFCVT2_I:%.*]] = call <8 x half> @llvm.aarch64.neon.fp8.cvtl2.v8f16.v16i8(<16 x i8> [[OP]])
+// CHECK-CXX-NEXT:    ret <8 x half> [[VBFCVT2_I]]
+//
+float16x8_t test_vcvt2_high_f16_mf8_fpm(mfloat8x16_t op, fpm_t fpm) {
+  return vcvt2_high_f16_mf8_fpm(op, fpm);
+}
+
+// CHECK-LABEL: define dso_local <8 x i8> @test_vcvt_mf8_f32_fpm(
+// CHECK-SAME: <4 x float> noundef [[VN:%.*]], <4 x float> noundef [[VM:%.*]], i64 noundef [[FPM:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    call void @llvm.aarch64.set.fpmr(i64 [[FPM]])
+// CHECK-NEXT:    [[VFCVTN_I:%.*]] = call <8 x i8> @llvm.aarch64.neon.fp8.fcvtn.v8i8.v4f32(<4 x float> [[VN]], <4 x float> [[VM]])
+// CHECK-NEXT:    ret <8 x i8> [[VFCVTN_I]]
+//
+// CHECK-CXX-LABEL: define dso_local <8 x i8> @_Z21test_vcvt_mf8_f32_fpm13__Float32x4_tS_m(
+// CHECK-CXX-SAME: <4 x float> noundef [[VN:%.*]], <4 x float> noundef [[VM:%.*]], i64 noundef [[FPM:%.*]]) #[[ATTR0]] {
+// CHECK-CXX-NEXT:  [[ENTRY:.*:]]
+// CHECK-CXX-NEXT:    call void @llvm.aarch64.set.fpmr(i64 [[FPM]])
+// CHECK-CXX-NEXT:    [[VFCVTN_I:%.*]] = call <8 x i8> @llvm.aarch64.neon.fp8.fcvtn.v8i8.v4f32(<4 x float> [[VN]], <4 x float> [[VM]])
+// CHECK-CXX-NEXT:    ret <8 x i8> [[VFCVTN_I]]
+//
+mfloat8x8_t test_vcvt_mf8_f32_fpm(float32x4_t vn, float32x4_t vm, fpm_t fpm) {
+  return vcvt_mf8_f32_fpm(vn, vm, fpm);
+}
+
+// CHECK-LABEL: define dso_local <16 x i8> @test_vcvt_high_mf8_f32_fpm(
+// CHECK-SAME: <8 x i8> [[VD:%.*]], <4 x float> noundef [[VN:%.*]], <4 x float> noundef [[VM:%.*]], i64 noundef [[FPM:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[TMP0:%.*]] = call <16 x i8> @llvm.vector.insert.v16i8.v8i8(<16 x i8> poison, <8 x i8> [[VD]], i64 0)
+// CHECK-NEXT:    call void @llvm.aarch64.set.fpmr(i64 [[FPM]])
+// CHECK-NEXT:    [[VFCVTN2_I:%.*]] = call <16 x i8> @llvm.aarch64.neon.fp8.fcvtn2.v16i8.v4f32(<16 x i8> [[TMP0]], <4 x float> [[VN]], <4 x float> [[VM]])
+// CHECK-NEXT:    ret <16 x i8> [[VFCVTN2_I]]
+//
+// CHECK-CXX-LABEL: define dso_local <16 x i8> @_Z26test_vcvt_high_mf8_f32_fpm13__Mfloat8x8_t13__Float32x4_tS0_m(
+// CHECK-CXX-SAME: <8 x i8> [[VD:%.*]], <4 x float> noundef [[VN:%.*]], <4 x float> noundef [[VM:%.*]], i64 noundef [[FPM:%.*]]) #[[ATTR0]] {
+// CHECK-CXX-NEXT:  [[ENTRY:.*:]]
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = call <16 x i8> @llvm.vector.insert.v16i8.v8i8(<16 x i8> poison, <8 x i8> [[VD]], i64 0)
+// CHECK-CXX-NEXT:    call void @llvm.aarch64.set.fpmr(i64 [[FPM]])
+// CHECK-CXX-NEXT:    [[VFCVTN2_I:%.*]] = call <16 x i8> @llvm.aarch64.neon.fp8.fcvtn2.v16i8.v4f32(<16 x i8> [[TMP0]], <4 x float> [[VN]], <4 x float> [[VM]])
+// CHECK-CXX-NEXT:    ret <16 x i8> [[VFCVTN2_I]]
+//
+mfloat8x16_t test_vcvt_high_mf8_f32_fpm(mfloat8x8_t vd, float32x4_t vn,
+                                    float32x4_t vm, fpm_t fpm) {
+  return vcvt_high_mf8_f32_fpm(vd, vn, vm, fpm);
+}
+
+// CHECK-LABEL: define dso_local <8 x i8> @test_vcvt_mf8_f16_fpm(
+// CHECK-SAME: <4 x half> noundef [[VN:%.*]], <4 x half> noundef [[VM:%.*]], i64 noundef [[FPM:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[TMP0:%.*]] = bitcast <4 x half> [[VN]] to <8 x i8>
+// CHECK-NEXT:    [[TMP1:%.*]] = bitcast <4 x half> [[VM]] to <8 x i8>
+// CHECK-NEXT:    call void @llvm.aarch64.set.fpmr(i64 [[FPM]])
+// CHECK-NEXT:    [[VFCVTN2_I:%.*]] = call <8 x i8> @llvm.aarch64.neon.fp8.fcvtn.v8i8.v4f16(<4 x half> [[VN]], <4 x half> [[VM]])
+// CHECK-NEXT:    ret <8 x i8> [[VFCVTN2_I]]
+//
+// CHECK-CXX-LABEL: define dso_local <8 x i8> @_Z21test_vcvt_mf8_f16_fpm13__Float16x4_tS_m(
+// CHECK-CXX-SAME: <4 x half> noundef [[VN:%.*]], <4 x half> noundef [[VM:%.*]], i64 noundef [[FPM:%.*]]) #[[ATTR0]] {
+// CHECK-CXX-NEXT:  [[ENTRY:.*:]]
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = bitcast <4 x half> [[VN]] to <8 x i8>
+// CHECK-CXX-NEXT:    [[TMP1:%.*]] = bitcast <4 x half> [[VM]] to <8 x i8>
+// CHECK-CXX-NEXT:    call void @llvm.aarch64.set.fpmr(i64 [[FPM]])
+// CHECK-CXX-NEXT:    [[VFCVTN2_I:%.*]] = call <8 x i8> @llvm.aarch64.neon.fp8.fcvtn.v8i8.v4f16(<4 x half> [[VN]], <4 x half> [[VM]])
+// CHECK-CXX-NEXT:    ret <8 x i8> [[VFCVTN2_I]]
+//
+mfloat8x8_t test_vcvt_mf8_f16_fpm(float16x4_t vn, float16x4_t vm, fpm_t fpm) {
+  return vcvt_mf8_f16_fpm(vn, vm, fpm);
+}
+
+// CHECK-LABEL: define dso_local <16 x i8> @test_vcvtq_mf8_f16_fpm(
+// CHECK-SAME: <8 x half> noundef [[VN:%.*]], <8 x half> noundef [[VM:%.*]], i64 noundef [[FPM:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[TMP0:%.*]] = bitcast <8 x half> [[VN]] to <16 x i8>
+// CHECK-NEXT:    [[TMP1:%.*]] = bitcast <8 x half> [[VM]] to <16 x i8>
+// CHECK-NEXT:    call void @llvm.aarch64.set.fpmr(i64 [[FPM]])
+// CHECK-NEXT:    [[VFCVTN2_I:%.*]] = call <16 x i8> @llvm.aarch64.neon.fp8.fcvtn.v16i8.v8f16(<8 x half> [[VN]], <8 x half> [[VM]])
+// CHECK-NEXT:    ret <16 x i8> [[VFCVTN2_I]]
+//
+// CHECK-CXX-LABEL: define dso_local <16 x i8> @_Z22test_vcvtq_mf8_f16_fpm13__Float16x8_tS_m(
+// CHECK-CXX-SAME: <8 x half> noundef [[VN:%.*]], <8 x half> noundef [[VM:%.*]], i64 noundef [[FPM:%.*]]) #[[ATTR0]] {
+// CHECK-CXX-NEXT:  [[ENTRY:.*:]]
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = bitcast <8 x half> [[VN]] to <16 x i8>
+// CHECK-CXX-NEXT:    [[TMP1:%.*]] = bitcast <8 x half> [[VM]] to <16 x i8>
+// CHECK-CXX-NEXT:    call void @llvm.aarch64.set.fpmr(i64 [[FPM]])
+// CHECK-CXX-NEXT:    [[VFCVTN2_I:%.*]] = call <16 x i8> @llvm.aarch64.neon.fp8.fcvtn.v16i8.v8f16(<8 x half> [[VN]], <8 x half> [[VM]])
+// CHECK-CXX-NEXT:    ret <16 x i8> [[VFCVTN2_I]]
+//
+mfloat8x16_t test_vcvtq_mf8_f16_fpm(float16x8_t vn, float16x8_t vm, fpm_t fpm) {
+  return vcvtq_mf8_f16_fpm(vn, vm, fpm);
+}
diff --git a/clang/test/CodeGen/AArch64/fp8-intrinsics/acle_neon_fp8_fdot.c b/clang/test/CodeGen/AArch64/fp8-intrinsics/acle_neon_fp8_fdot.c
new file mode 100644
index 0000000000000..4d2f5d550c4dc
--- /dev/null
+++ b/clang/test/CodeGen/AArch64/fp8-intrinsics/acle_neon_fp8_fdot.c
@@ -0,0 +1,254 @@
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --version 5
+
+// RUN: %clang_cc1        -triple aarch64-none-linux-gnu -target-feature +neon -target-feature +bf16 -target-feature +fp8 -target-feature +fp8dot2 -target-feature +fp8dot4 -disable-O0-optnone -Werror -Wall -emit-llvm -o - %s | opt -S -p mem2reg | FileCheck %s
+// RUN: %clang_cc1 -x c++ -triple aarch64-none-linux-gnu -target-feature +neon -target-feature +bf16 -target-feature +fp8 -target-feature +fp8dot2 -target-feature +fp8dot4 -disable-O0-optnone -Werror -Wall -emit-llvm -o - %s | opt -S -p mem2reg | FileCheck %s -check-prefix CHECK-CXX
+
+// RUN: %clang_cc1        -triple aarch64-none-linux-gnu -target-feature +neon -target-feature +bf16 -target-feature +fp8 -target-feature +fp8dot2 -target-feature +fp8dot4 -O3 -Werror -Wall -S -o /dev/null %s
+
+// REQUIRES: aarch64-registered-target
+
+#include <arm_neon.h>
+
+// CHECK-LABEL: define dso_local <4 x half> @test_vdot_f16(
+// CHECK-SAME: <4 x half> noundef [[VD:%.*]], <8 x i8> [[VN:%.*]], <8 x i8> [[VM:%.*]], i64 noundef [[FPMR:%.*]]) #[[ATTR0:[0-9]+]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[TMP0:%.*]] = bitcast <4 x half> [[VD]] to <8 x i8>
+// CHECK-NEXT:    call void @llvm.aarch64.set.fpmr(i64 [[FPMR]])
+// CHECK-NEXT:    [[FDOT21_I:%.*]] = call <4 x half> @llvm.aarch64.neon.fp8.fdot2.v4f16.v8i8(<4 x half> [[VD]], <8 x i8> [[VN]], <8 x i8> [[VM]])
+// CHECK-NEXT:    ret <4 x half> [[FDOT21_I]]
+//
+// CHECK-CXX-LABEL: define dso_local noundef <4 x half> @_Z13test_vdot_f1613__Float16x4_t13__Mfloat8x8_tS0_m(
+// CHECK-CXX-SAME: <4 x half> noundef [[VD:%.*]], <8 x i8> [[VN:%.*]], <8 x i8> [[VM:%.*]], i64 noundef [[FPMR:%.*]]) #[[ATTR0:[0-9]+]] {
+// CHECK-CXX-NEXT:  [[ENTRY:.*:]]
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = bitcast <4 x half> [[VD]] to <8 x i8>
+// CHECK-CXX-NEXT:    call void @llvm.aarch64.set.fpmr(i64 [[FPMR]])
+// CHECK-CXX-NEXT:    [[FDOT21_I:%.*]] = call <4 x half> @llvm.aarch64.neon.fp8.fdot2.v4f16.v8i8(<4 x half> [[VD]], <8 x i8> [[VN]], <8 x i8> [[VM]])
+// CHECK-CXX-NEXT:    ret <4 x half> [[FDOT21_I]]
+//
+float16x4_t test_vdot_f16(float16x4_t vd, mfloat8x8_t vn, mfloat8x8_t vm, fpm_t fpmr) {
+  return vdot_f16_mf8_fpm(vd, vn, vm, fpmr);
+}
+
+// CHECK-LABEL: define dso_local <8 x half> @test_vdotq_f16(
+// CHECK-SAME: <8 x half> noundef [[VD:%.*]], <16 x i8> [[VN:%.*]], <16 x i8> [[VM:%.*]], i64 noundef [[FPMR:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[TMP0:%.*]] = bitcast <8 x half> [[VD]] to <16 x i8>
+// CHECK-NEXT:    call void @llvm.aarch64.set.fpmr(i64 [[FPMR]])
+// CHECK-NEXT:    [[FDOT21_I:%.*]] = call <8 x half> @llvm.aarch64.neon.fp8.fdot2.v8f16.v16i8(<8 x half> [[VD]], <16 x i8> [[VN]], <16 x i8> [[VM]])
+// CHECK-NEXT:    ret <8 x half> [[FDOT21_I]]
+//
+// CHECK-CXX-LABEL: define dso_local noundef <8 x half> @_Z14test_vdotq_f1613__Float16x8_t14__Mfloat8x16_tS0_m(
+// CHECK-CXX-SAME: <8 x half> noundef [[VD:%.*]], <16 x i8> [[VN:%.*]], <16 x i8> [[VM:%.*]], i64 noundef [[FPMR:%.*]]) #[[ATTR0]] {
+// CHECK-CXX-NEXT:  [[ENTRY:.*:]]
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = bitcast <8 x half> [[VD]] to <16 x i8>
+// CHECK-CXX-NEXT:    call void @llvm.aarch64.set.fpmr(i64 [[FPMR]])
+// CHECK-CXX-NEXT:    [[FDOT21_I:%.*]] = call <8 x half> @llvm.aarch64.neon.fp8.fdot2.v8f16.v16i8(<8 x half> [[VD]], <16 x i8> [[VN]], <16 x i8> [[VM]])
+// CHECK-CXX-NEXT:    ret <8 x half> [[FDOT21_I]]
+//
+float16x8_t test_vdotq_f16(float16x8_t vd, mfloat8x16_t vn, mfloat8x16_t vm, fpm_t fpmr) {
+  return vdotq_f16_mf8_fpm(vd, vn, vm, fpmr);
+}
+
+// CHECK-LABEL: define dso_local <4 x half> @test_vdot_lane_f16(
+// CHECK-SAME: <4 x half> noundef [[VD:%.*]], <8 x i8> [[VN:%.*]], <8 x i8> [[VM:%.*]], i64 noundef [[FPMR:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[TMP0:%.*]] = bitcast <4 x half> [[VD]] to <8 x i8>
+// CHECK-NEXT:    [[TMP1:%.*]] = call <16 x i8> @llvm.vector.insert.v16i8.v8i8(<16 x i8> poison, <8 x i8> [[VM]], i64 0)
+// CHECK-NEXT:    call void @llvm.aarch64.set.fpmr(i64 [[FPMR]])
+// CHECK-NEXT:    [[FDOT2_LANE:%.*]] = bitcast <8 x i8> [[TMP0]] to <4 x half>
+// CHECK-NEXT:    [[FDOT2_LANE1:%.*]] = call <4 x half> @llvm.aarch64.neon.fp8.fdot2.lane.v4f16.v8i8(<4 x half> [[FDOT2_LANE]], <8 x i8> [[VN]], <16 x i8> [[TMP1]], i32 3)
+// CHECK-NEXT:    ret <4 x half> [[FDOT2_LANE1]]
+//
+// CHECK-CXX-LABEL: define dso_local noundef <4 x half> @_Z18test_vdot_lane_f1613__Float16x4_t13__Mfloat8x8_tS0_m(
+// CHECK-CXX-SAME: <4 x half> noundef [[VD:%.*]], <8 x i8> [[VN:%.*]], <8 x i8> [[VM:%.*]], i64 noundef [[FPMR:%.*]]) #[[ATTR0]] {
+// CHECK-CXX-NEXT:  [[ENTRY:.*:]]
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = bitcast <4 x half> [[VD]] to <8 x i8>
+// CHECK-CXX-NEXT:    [[TMP1:%.*]] = call <16 x i8> @llvm.vector.insert.v16i8.v8i8(<16 x i8> poison, <8 x i8> [[VM]], i64 0)
+// CHECK-CXX-NEXT:    call void @llvm.aarch64.set.fpmr(i64 [[FPMR]])
+// CHECK-CXX-NEXT:    [[FDOT2_LANE:%.*]] = bitcast <8 x i8> [[TMP0]] to <4 x half>
+// CHECK-CXX-NEXT:    [[FDOT2_LANE1:%.*]] = call <4 x half> @llvm.aarch64.neon.fp8.fdot2.lane.v4f16.v8i8(<4 x half> [[FDOT2_LANE]], <8 x i8> [[VN]], <16 x i8> [[TMP1]], i32 3)
+// CHECK-CXX-NEXT:    ret <4 x half> [[FDOT2_LANE1]]
+//
+float16x4_t test_vdot_lane_f16(float16x4_t vd, mfloat8x8_t vn, mfloat8x8_t vm, fpm_t fpmr) {
+  return vdot_lane_f16_mf8_fpm(vd, vn, vm, 3, fpmr);
+}
+
+// CHECK-LABEL: define dso_local <4 x half> @test_vdot_laneq_f16(
+// CHECK-SAME: <4 x half> noundef [[VD:%.*]], <8 x i8> [[VN:%.*]], <16 x i8> [[VM:%.*]], i64 noundef [[FPMR:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[TMP0:%.*]] = bitcast <4 x half> [[VD]] to <8 x i8>
+// CHECK-NEXT:    call void @llvm.aarch64.set.fpmr(i64 [[FPMR]])
+// CHECK-NEXT:    [[FDOT2_LANE:%.*]] = bitcast <8 x i8> [[TMP0]] to <4 x half>
+// CHECK-NEXT:    [[FDOT2_LANE1:%.*]] = call <4 x half> @llvm.aarch64.neon.fp8.fdot2.lane.v4f16.v8i8(<4 x half> [[FDOT2_LANE]], <8 x i8> [[VN]], <16 x i8> [[VM]], i32 7)
+// CHECK-NEXT:    ret <4 x half> [[FDOT2_LANE1]]
+//
+// CHECK-CXX-LABEL: define dso_local noundef <4 x half> @_Z19test_vdot_laneq_f1613__Float16x4_t13__Mfloat8x8_t14__Mfloat8x16_tm(
+// CHECK-CXX-SAME: <4 x half> noundef [[VD:%.*]], <8 x i8> [[VN:%.*]], <16 x i8> [[VM:%.*]], i64 noundef [[FPMR:%.*]]) #[[ATTR0]] {
+// CHECK-CXX-NEXT:  [[ENTRY:.*:]]
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = bitcast <4 x half> [[VD]] to <8 x i8>
+// CHECK-CXX-NEXT:    call void @llvm.aarch64.set.fpmr(i64 [[FPMR]])
+// CHECK-CXX-NEXT:    [[FDOT2_LANE:%.*]] = bitcast <8 x i8> [[TMP0]] to <4 x half>
+// CHECK-CXX-NEXT:    [[FDOT2_LANE1:%.*]] = call <4 x half> @llvm.aarch64.neon.fp8.fdot2.lane.v4f16.v8i8(<4 x half> [[FDOT2_LANE]], <8 x i8> [[VN]], <16 x i8> [[VM]], i32 7)
+// CHECK-CXX-NEXT:    ret <4 x half> [[FDOT2_LANE1]]
+//
+float16x4_t test_vdot_laneq_f16(float16x4_t vd, mfloat8x8_t vn, mfloat8x16_t vm, fpm_t fpmr) {
+  return vdot_laneq_f16_mf8_fpm(vd, vn, vm, 7, fpmr);
+}
+
+// CHECK-LABEL: define dso_local <8 x half> @test_vdotq_lane_f16(
+// CHECK-SAME: <8 x half> noundef [[VD:%.*]], <16 x i8> [[VN:%.*]], <8 x i8> [[VM:%.*]], i64 noundef [[FPMR:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[TMP0:%.*]] = bitcast <8 x half> [[VD]] to <16 x i8>
+// CHECK-NEXT:    [[TMP1:%.*]] = call <16 x i8> @llvm.vector.insert.v16i8.v8i8(<16 x i8> poison, <8 x i8> [[VM]], i64 0)
+// CHECK-NEXT:    call void @llvm.aarch64.set.fpmr(i64 [[FPMR]])
+// CHECK-NEXT:    [[FDOT2_LANE:%.*]] = bitcast <16 x i8> [[TMP0]] to <8 x half>
+// CHECK-NEXT:    [[FDOT2_LANE1:%.*]] = call <8 x half> @llvm.aarch64.neon.fp8.fdot2.lane.v8f16.v16i8(<8 x half> [[FDOT2_LANE]], <16 x i8> [[VN]], <16 x i8> [[TMP1]], i32 3)
+// CHECK-NEXT:    ret <8 x half> [[FDOT2_LANE1]]
+//
+// CHECK-CXX-LABEL: define dso_local noundef <8 x half> @_Z19test_vdotq_lane_f1613__Float16x8_t14__Mfloat8x16_t13__Mfloat8x8_tm(
+// CHECK-CXX-SAME: <8 x half> noundef [[VD:%.*]], <16 x i8> [[VN:%.*]], <8 x i8> [[VM:%.*]], i64 noundef [[FPMR:%.*]]) #[[ATTR0]] {
+// CHECK-CXX-NEXT:  [[ENTRY:.*:]]
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = bitcast <8 x half> [[VD]] to <16 x i8>
+// CHECK-CXX-NEXT:    [[TMP1:%.*]] = call <16 x i8> @llvm.vector.insert.v16i8.v8i8(<16 x i8> poison, <8 x i8> [[VM]], i64 0)
+// CHECK-CXX-NEXT:    call void @llvm.aarch64.set.fpmr(i64 [[FPMR]])
+// CHECK-CXX-NEXT:    [[FDOT2_LANE:%.*]] = bitcast <16 x i8> [[TMP0]] to <8 x half>
+// CHECK-CXX-NEXT:    [[FDOT2_LANE1:%.*]] = call <8 x half> @llvm.aarch64.neon.fp8.fdot2.lane.v8f16.v16i8(<8 x half> [[FDOT2_LANE]], <16 x i8> [[VN]], <16 x i8> [[TMP1]], i32 3)
+// CHECK-CXX-NEXT:    ret <8 x half> [[FDOT2_LANE1]]
+//
+float16x8_t test_vdotq_lane_f16(float16x8_t vd, mfloat8x16_t vn, mfloat8x8_t vm, fpm_t fpmr) {
+  return vdotq_lane_f16_mf8_fpm(vd, vn, vm, 3, fpmr);
+}
+
+// CHECK-LABEL: define dso_local <8 x half> @test_vdotq_laneq_f16(
+// CHECK-SAME: <8 x half> noundef [[VD:%.*]], <16 x i8> [[VN:%.*]], <16 x i8> [[VM:%.*]], i64 noundef [[FPMR:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[TMP0:%.*]] = bitcast <8 x half> [[VD]] to <16 x i8>
+// CHECK-NEXT:    call void @llvm.aarch64.set.fpmr(i64 [[FPMR]])
+// CHECK-NEXT:    [[FDOT2_LANE:%.*]] = bitcast <16 x i8> [[TMP0]] to <8 x half>
+// CHECK-NEXT:    [[FDOT2_LANE1:%.*]] = call <8 x half> @llvm.aarch64.neon.fp8.fdot2.lane.v8f16.v16i8(<8 x half> [[FDOT2_LANE]], <16 x i8> [[VN]], <16 x i8> [[VM]], i32 7)
+// CHECK-NEXT:    ret <8 x half> [[FDOT2_LANE1]]
+//
+// CHECK-CXX-LABEL: define dso_local noundef <8 x half> @_Z20test_vdotq_laneq_f1613__Float16x8_t14__Mfloat8x16_tS0_m(
+// CHECK-CXX-SAME: <8 x half> noundef [[VD:%.*]], <16 x i8> [[VN:%.*]], <16 x i8> [[VM:%.*]], i64 noundef [[FPMR:%.*]]) #[[ATTR0]] {
+// CHECK-CXX-NEXT:  [[ENTRY:.*:]]
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = bitcast <8 x half> [[VD]] to <16 x i8>
+// CHECK-CXX-NEXT:    call void @llvm.aarch64.set.fpmr(i64 [[FPMR]])
+// CHECK-CXX-NEXT:    [[FDOT2_LANE:%.*]] = bitcast <16 x i8> [[TMP0]] to <8 x half>
+// CHECK-CXX-NEXT:    [[FDOT2_LANE1:%.*]] = call <8 x half> @llvm.aarch64.neon.fp8.fdot2.lane.v8f16.v16i8(<8 x half> [[FDOT2_LANE]], <16 x i8> [[VN]], <16 x i8> [[VM]], i32 7)
+// CHECK-CXX-NEXT:    ret <8 x half> [[FDOT2_LANE1]]
+//
+float16x8_t test_vdotq_laneq_f16(float16x8_t vd, mfloat8x16_t vn, mfloat8x16_t vm, fpm_t fpmr) {
+  return vdotq_laneq_f16_mf8_fpm(vd, vn, vm, 7, fpmr);
+}
+
+// CHECK-LABEL: define dso_local <2 x float> @test_vdot_f32(
+// CHECK-SAME: <2 x float> noundef [[VD:%.*]], <8 x i8> [[VN:%.*]], <8 x i8> [[VM:%.*]], i64 noundef [[FPMR:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    call void @llvm.aarch64.set.fpmr(i64 [[FPMR]])
+// CHECK-NEXT:    [[FDOT4_I:%.*]] = call <2 x float> @llvm.aarch64.neon.fp8.fdot4.v2f32.v8i8(<2 x float> [[VD]], <8 x i8> [[VN]], <8 x i8> [[VM]])
+// CHECK-NEXT:    ret <2 x float> [[FDOT4_I]]
+//
+// CHECK-CXX-LABEL: define dso_local noundef <2 x float> @_Z13test_vdot_f3213__Float32x2_t13__Mfloat8x8_tS0_m(
+// CHECK-CXX-SAME: <2 x float> noundef [[VD:%.*]], <8 x i8> [[VN:%.*]], <8 x i8> [[VM:%.*]], i64 noundef [[FPMR:%.*]]) #[[ATTR0]] {
+// CHECK-CXX-NEXT:  [[ENTRY:.*:]]
+// CHECK-CXX-NEXT:    call void @llvm.aarch64.set.fpmr(i64 [[FPMR]])
+// CHECK-CXX-NEXT:    [[FDOT4_I:%.*]] = call <2 x float> @llvm.aarch64.neon.fp8.fdot4.v2f32.v8i8(<2 x float> [[VD]], <8 x i8> [[VN]], <8 x i8> [[VM]])
+// CHECK-CXX-NEXT:    ret <2 x float> [[FDOT4_I]]
+//
+float32x2_t test_vdot_f32(float32x2_t vd, mfloat8x8_t vn, mfloat8x8_t vm, fpm_t fpmr) {
+  return vdot_f32_mf8_fpm(vd, vn, vm, fpmr);
+}
+
+// CHECK-LABEL: define dso_local <4 x float> @test_vdotq_f32(
+// CHECK-SAME: <4 x float> noundef [[VD:%.*]], <16 x i8> [[VN:%.*]], <16 x i8> [[VM:%.*]], i64 noundef [[FPMR:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    call void @llvm.aarch64.set.fpmr(i64 [[FPMR]])
+// CHECK-NEXT:    [[FDOT4_I:%.*]] = call <4 x float> @llvm.aarch64.neon.fp8.fdot4.v4f32.v16i8(<4 x float> [[VD]], <16 x i8> [[VN]], <16 x i8> [[VM]])
+// CHECK-NEXT:    ret <4 x float> [[FDOT4_I]]
+//
+// CHECK-CXX-LABEL: define dso_local noundef <4 x float> @_Z14test_vdotq_f3213__Float32x4_t14__Mfloat8x16_tS0_m(
+// CHECK-CXX-SAME: <4 x float> noundef [[VD:%.*]], <16 x i8> [[VN:%.*]], <16 x i8> [[VM:%.*]], i64 noundef [[FPMR:%.*]]) #[[ATTR0]] {
+// CHECK-CXX-NEXT:  [[ENTRY:.*:]]
+// CHECK-CXX-NEXT:    call void @llvm.aarch64.set.fpmr(i64 [[FPMR]])
+// CHECK-CXX-NEXT:    [[FDOT4_I:%.*]] = call <4 x float> @llvm.aarch64.neon.fp8.fdot4.v4f32.v16i8(<4 x float> [[VD]], <16 x i8> [[VN]], <16 x i8> [[VM]])
+// CHECK-CXX-NEXT:    ret <4 x float> [[FDOT4_I]]
+//
+float32x4_t test_vdotq_f32(float32x4_t vd, mfloat8x16_t vn, mfloat8x16_t vm, fpm_t fpmr) {
+  return vdotq_f32_mf8_fpm(vd, vn, vm, fpmr);
+}
+
+// CHECK-LABEL: define dso_local <2 x float> @test_vdot_lane_f32(
+// CHECK-SAME: <2 x float> noundef [[VD:%.*]], <8 x i8> [[VN:%.*]], <8 x i8> [[VM:%.*]], i64 noundef [[FPMR:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[TMP0:%.*]] = call <16 x i8> @llvm.vector.insert.v16i8.v8i8(<16 x i8> poison, <8 x i8> [[VM]], i64 0)
+// CHECK-NEXT:    call void @llvm.aarch64.set.fpmr(i64 [[FPMR]])
+// CHECK-NEXT:    [[FDOT4_LANE:%.*]] = call <2 x float> @llvm.aarch64.neon.fp8.fdot4.lane.v2f32.v8i8(<2 x float> [[VD]], <8 x i8> [[VN]], <16 x i8> [[TMP0]], i32 1)
+// CHECK-NEXT:    ret <2 x float> [[FDOT4_LANE]]
+//
+// CHECK-CXX-LABEL: define dso_local noundef <2 x float> @_Z18test_vdot_lane_f3213__Float32x2_t13__Mfloat8x8_tS0_m(
+// CHECK-CXX-SAME: <2 x float> noundef [[VD:%.*]], <8 x i8> [[VN:%.*]], <8 x i8> [[VM:%.*]], i64 noundef [[FPMR:%.*]]) #[[ATTR0]] {
+// CHECK-CXX-NEXT:  [[ENTRY:.*:]]
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = call <16 x i8> @llvm.vector.insert.v16i8.v8i8(<16 x i8> poison, <8 x i8> [[VM]], i64 0)
+// CHECK-CXX-NEXT:    call void @llvm.aarch64.set.fpmr(i64 [[FPMR]])
+// CHECK-CXX-NEXT:    [[FDOT4_LANE:%.*]] = call <2 x float> @llvm.aarch64.neon.fp8.fdot4.lane.v2f32.v8i8(<2 x float> [[VD]], <8 x i8> [[VN]], <16 x i8> [[TMP0]], i32 1)
+// CHECK-CXX-NEXT:    ret <2 x float> [[FDOT4_LANE]]
+//
+float32x2_t test_vdot_lane_f32(float32x2_t vd, mfloat8x8_t vn, mfloat8x8_t vm, fpm_t fpmr) {
+  return vdot_lane_f32_mf8_fpm(vd, vn, vm, 1, fpmr);
+}
+
+// CHECK-LABEL: define dso_local <2 x float> @test_vdot_laneq_f32(
+// CHECK-SAME: <2 x float> noundef [[VD:%.*]], <8 x i8> [[VN:%.*]], <16 x i8> [[VM:%.*]], i64 noundef [[FPMR:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    call void @llvm.aarch64.set.fpmr(i64 [[FPMR]])
+// CHECK-NEXT:    [[FDOT4_LANE:%.*]] = call <2 x float> @llvm.aarch64.neon.fp8.fdot4.lane.v2f32.v8i8(<2 x float> [[VD]], <8 x i8> [[VN]], <16 x i8> [[VM]], i32 3)
+// CHECK-NEXT:    ret <2 x float> [[FDOT4_LANE]]
+//
+// CHECK-CXX-LABEL: define dso_local noundef <2 x float> @_Z19test_vdot_laneq_f3213__Float32x2_t13__Mfloat8x8_t14__Mfloat8x16_tm(
+// CHECK-CXX-SAME: <2 x float> noundef [[VD:%.*]], <8 x i8> [[VN:%.*]], <16 x i8> [[VM:%.*]], i64 noundef [[FPMR:%.*]]) #[[ATTR0]] {
+// CHECK-CXX-NEXT:  [[ENTRY:.*:]]
+// CHECK-CXX-NEXT:    call void @llvm.aarch64.set.fpmr(i64 [[FPMR]])
+// CHECK-CXX-NEXT:    [[FDOT4_LANE:%.*]] = call <2 x float> @llvm.aarch64.neon.fp8.fdot4.lane.v2f32.v8i8(<2 x float> [[VD]], <8 x i8> [[VN]], <16 x i8> [[VM]], i32 3)
+// CHECK-CXX-NEXT:    ret <2 x float> [[FDOT4_LANE]]
+//
+float32x2_t test_vdot_laneq_f32(float32x2_t vd, mfloat8x8_t vn, mfloat8x16_t vm, fpm_t fpmr) {
+  return vdot_laneq_f32_mf8_fpm(vd, vn, vm, 3, fpmr);
+}
+
+// CHECK-LABEL: define dso_local <4 x float> @test_vdotq_lane_f32(
+// CHECK-SAME: <4 x float> noundef [[VD:%.*]], <16 x i8> [[VN:%.*]], <8 x i8> [[VM:%.*]], i64 noundef [[FPMR:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[TMP0:%.*]] = call <16 x i8> @llvm.vector.insert.v16i8.v8i8(<16 x i8> poison, <8 x i8> [[VM]], i64 0)
+// CHECK-NEXT:    call void @llvm.aarch64.set.fpmr(i64 [[FPMR]])
+// CHECK-NEXT:    [[FDOT4_LANE:%.*]] = call <4 x float> @llvm.aarch64.neon.fp8.fdot4.lane.v4f32.v16i8(<4 x float> [[VD]], <16 x i8> [[VN]], <16 x i8> [[TMP0]], i32 1)
+// CHECK-NEXT:    ret <4 x float> [[FDOT4_LANE]]
+//
+// CHECK-CXX-LABEL: define dso_local noundef <4 x float> @_Z19test_vdotq_lane_f3213__Float32x4_t14__Mfloat8x16_t13__Mfloat8x8_tm(
+// CHECK-CXX-SAME: <4 x float> noundef [[VD:%.*]], <16 x i8> [[VN:%.*]], <8 x i8> [[VM:%.*]], i64 noundef [[FPMR:%.*]]) #[[ATTR0]] {
+// CHECK-CXX-NEXT:  [[ENTRY:.*:]]
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = call <16 x i8> @llvm.vector.insert.v16i8.v8i8(<16 x i8> poison, <8 x i8> [[VM]], i64 0)
+// CHECK-CXX-NEXT:    call void @llvm.aarch64.set.fpmr(i64 [[FPMR]])
+// CHECK-CXX-NEXT:    [[FDOT4_LANE:%.*]] = call <4 x float> @llvm.aarch64.neon.fp8.fdot4.lane.v4f32.v16i8(<4 x float> [[VD]], <16 x i8> [[VN]], <16 x i8> [[TMP0]], i32 1)
+// CHECK-CXX-NEXT:    ret <4 x float> [[FDOT4_LANE]]
+//
+float32x4_t test_vdotq_lane_f32(float32x4_t vd, mfloat8x16_t vn, mfloat8x8_t vm, fpm_t fpmr) {
+  return vdotq_lane_f32_mf8_fpm(vd, vn, vm, 1, fpmr);
+}
+
+// CHECK-LABEL: define dso_local <4 x float> @test_vdotq_laneq_f32(
+// CHECK-SAME: <4 x float> noundef [[VD:%.*]], <16 x i8> [[VN:%.*]], <16 x i8> [[VM:%.*]], i64 noundef [[FPMR:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    call void @llvm.aarch64.set.fpmr(i64 [[FPMR]])
+// CHECK-NEXT:    [[FDOT4_LANE:%.*]] = call <4 x float> @llvm.aarch64.neon.fp8.fdot4.lane.v4f32.v16i8(<4 x float> [[VD]], <16 x i8> [[VN]], <16 x i8> [[VM]], i32 3)
+// CHECK-NEXT:    ret <4 x float> [[FDOT4_LANE]]
+//
+// CHECK-CXX-LABEL: define dso_local noundef <4 x float> @_Z20test_vdotq_laneq_f3213__Float32x4_t14__Mfloat8x16_tS0_m(
+// CHECK-CXX-SAME: <4 x float> noundef [[VD:%.*]], <16 x i8> [[VN:%.*]], <16 x i8> [[VM:%.*]], i64 noundef [[FPMR:%.*]]) #[[ATTR0]] {
+// CHECK-CXX-NEXT:  [[ENTRY:.*:]]
+// CHECK-CXX-NEXT:    call void @llvm.aarch64.set.fpmr(i64 [[FPMR]])
+// CHECK-CXX-NEXT:    [[FDOT4_LANE:%.*]] = call <4 x float> @llvm.aarch64.neon.fp8.fdot4.lane.v4f32.v16i8(<4 x float> [[VD]], <16 x i8> [[VN]], <16 x i8> [[VM]], i32 3)
+// CHECK-CXX-NEXT:    ret <4 x float> [[FDOT4_LANE]]
+//
+float32x4_t test_vdotq_laneq_f32(float32x4_t vd, mfloat8x16_t vn, mfloat8x16_t vm, fpm_t fpmr) {
+  return vdotq_laneq_f32_mf8_fpm(vd, vn, vm, 3, fpmr);
+}
diff --git a/clang/test/CodeGen/AArch64/fp8-intrinsics/acle_neon_fp8_fmla.c b/clang/test/CodeGen/AArch64/fp8-intrinsics/acle_neon_fp8_fmla.c
new file mode 100644
index 0000000000000..736538073cb39
--- /dev/null
+++ b/clang/test/CodeGen/AArch64/fp8-intrinsics/acle_neon_fp8_fmla.c
@@ -0,0 +1,365 @@
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --version 5
+// RUN: %clang_cc1        -triple aarch64-none-linux-gnu -target-feature +neon -target-feature +bf16 -target-feature +fp8 -target-feature +fp8fma -disable-O0-optnone -Werror -Wall -emit-llvm -o - %s | opt -S -p mem2reg | FileCheck %s
+// RUN: %clang_cc1 -x c++ -triple aarch64-none-linux-gnu -target-feature +neon -target-feature +bf16 -target-feature +fp8 -target-feature +fp8fma -disable-O0-optnone -Werror -Wall -emit-llvm -o - %s | opt -S -p mem2reg | FileCheck %s -check-prefix CHECK-CXX
+
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +neon -target-feature +bf16 -target-feature +fp8 -target-feature +fp8fma -disable-O0-optnone -Werror -Wall -S -o /dev/null %s
+
+// REQUIRES: aarch64-registered-target
+
+#include <arm_neon.h>
+
+// CHECK-LABEL: define dso_local <8 x half> @test_vmlalb(
+// CHECK-SAME: <8 x half> noundef [[VD:%.*]], <16 x i8> [[VN:%.*]], <16 x i8> [[VM:%.*]], i64 noundef [[FPM:%.*]]) #[[ATTR0:[0-9]+]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[TMP0:%.*]] = bitcast <8 x half> [[VD]] to <16 x i8>
+// CHECK-NEXT:    call void @llvm.aarch64.set.fpmr(i64 [[FPM]])
+// CHECK-NEXT:    [[VMLAL1_I:%.*]] = call <8 x half> @llvm.aarch64.neon.fp8.fmlalb.v8f16(<8 x half> [[VD]], <16 x i8> [[VN]], <16 x i8> [[VM]])
+// CHECK-NEXT:    ret <8 x half> [[VMLAL1_I]]
+//
+// CHECK-CXX-LABEL: define dso_local noundef <8 x half> @_Z11test_vmlalb13__Float16x8_t14__Mfloat8x16_tS0_m(
+// CHECK-CXX-SAME: <8 x half> noundef [[VD:%.*]], <16 x i8> [[VN:%.*]], <16 x i8> [[VM:%.*]], i64 noundef [[FPM:%.*]]) #[[ATTR0:[0-9]+]] {
+// CHECK-CXX-NEXT:  [[ENTRY:.*:]]
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = bitcast <8 x half> [[VD]] to <16 x i8>
+// CHECK-CXX-NEXT:    call void @llvm.aarch64.set.fpmr(i64 [[FPM]])
+// CHECK-CXX-NEXT:    [[VMLAL1_I:%.*]] = call <8 x half> @llvm.aarch64.neon.fp8.fmlalb.v8f16(<8 x half> [[VD]], <16 x i8> [[VN]], <16 x i8> [[VM]])
+// CHECK-CXX-NEXT:    ret <8 x half> [[VMLAL1_I]]
+//
+float16x8_t test_vmlalb(float16x8_t vd, mfloat8x16_t vn, mfloat8x16_t vm, fpm_t fpm) {
+  return vmlalbq_f16_mf8_fpm(vd, vn, vm, fpm);
+}
+
+// CHECK-LABEL: define dso_local <8 x half> @test_vmlalt(
+// CHECK-SAME: <8 x half> noundef [[VD:%.*]], <16 x i8> [[VN:%.*]], <16 x i8> [[VM:%.*]], i64 noundef [[FPM:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[TMP0:%.*]] = bitcast <8 x half> [[VD]] to <16 x i8>
+// CHECK-NEXT:    call void @llvm.aarch64.set.fpmr(i64 [[FPM]])
+// CHECK-NEXT:    [[VMLAL1_I:%.*]] = call <8 x half> @llvm.aarch64.neon.fp8.fmlalt.v8f16(<8 x half> [[VD]], <16 x i8> [[VN]], <16 x i8> [[VM]])
+// CHECK-NEXT:    ret <8 x half> [[VMLAL1_I]]
+//
+// CHECK-CXX-LABEL: define dso_local noundef <8 x half> @_Z11test_vmlalt13__Float16x8_t14__Mfloat8x16_tS0_m(
+// CHECK-CXX-SAME: <8 x half> noundef [[VD:%.*]], <16 x i8> [[VN:%.*]], <16 x i8> [[VM:%.*]], i64 noundef [[FPM:%.*]]) #[[ATTR0]] {
+// CHECK-CXX-NEXT:  [[ENTRY:.*:]]
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = bitcast <8 x half> [[VD]] to <16 x i8>
+// CHECK-CXX-NEXT:    call void @llvm.aarch64.set.fpmr(i64 [[FPM]])
+// CHECK-CXX-NEXT:    [[VMLAL1_I:%.*]] = call <8 x half> @llvm.aarch64.neon.fp8.fmlalt.v8f16(<8 x half> [[VD]], <16 x i8> [[VN]], <16 x i8> [[VM]])
+// CHECK-CXX-NEXT:    ret <8 x half> [[VMLAL1_I]]
+//
+float16x8_t test_vmlalt(float16x8_t vd, mfloat8x16_t vn, mfloat8x16_t vm, fpm_t fpm) {
+  return vmlaltq_f16_mf8_fpm(vd, vn, vm, fpm);
+}
+
+// CHECK-LABEL: define dso_local <4 x float> @test_vmlallbb(
+// CHECK-SAME: <4 x float> noundef [[VD:%.*]], <16 x i8> [[VN:%.*]], <16 x i8> [[VM:%.*]], i64 noundef [[FPM:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    call void @llvm.aarch64.set.fpmr(i64 [[FPM]])
+// CHECK-NEXT:    [[VMLALL_I:%.*]] = call <4 x float> @llvm.aarch64.neon.fp8.fmlallbb.v4f32(<4 x float> [[VD]], <16 x i8> [[VN]], <16 x i8> [[VM]])
+// CHECK-NEXT:    ret <4 x float> [[VMLALL_I]]
+//
+// CHECK-CXX-LABEL: define dso_local noundef <4 x float> @_Z13test_vmlallbb13__Float32x4_t14__Mfloat8x16_tS0_m(
+// CHECK-CXX-SAME: <4 x float> noundef [[VD:%.*]], <16 x i8> [[VN:%.*]], <16 x i8> [[VM:%.*]], i64 noundef [[FPM:%.*]]) #[[ATTR0]] {
+// CHECK-CXX-NEXT:  [[ENTRY:.*:]]
+// CHECK-CXX-NEXT:    call void @llvm.aarch64.set.fpmr(i64 [[FPM]])
+// CHECK-CXX-NEXT:    [[VMLALL_I:%.*]] = call <4 x float> @llvm.aarch64.neon.fp8.fmlallbb.v4f32(<4 x float> [[VD]], <16 x i8> [[VN]], <16 x i8> [[VM]])
+// CHECK-CXX-NEXT:    ret <4 x float> [[VMLALL_I]]
+//
+float32x4_t test_vmlallbb(float32x4_t vd, mfloat8x16_t vn, mfloat8x16_t vm, fpm_t fpm) {
+  return vmlallbbq_f32_mf8_fpm(vd, vn, vm, fpm);
+}
+
+// CHECK-LABEL: define dso_local <4 x float> @test_vmlallbt(
+// CHECK-SAME: <4 x float> noundef [[VD:%.*]], <16 x i8> [[VN:%.*]], <16 x i8> [[VM:%.*]], i64 noundef [[FPM:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    call void @llvm.aarch64.set.fpmr(i64 [[FPM]])
+// CHECK-NEXT:    [[VMLALL_I:%.*]] = call <4 x float> @llvm.aarch64.neon.fp8.fmlallbt.v4f32(<4 x float> [[VD]], <16 x i8> [[VN]], <16 x i8> [[VM]])
+// CHECK-NEXT:    ret <4 x float> [[VMLALL_I]]
+//
+// CHECK-CXX-LABEL: define dso_local noundef <4 x float> @_Z13test_vmlallbt13__Float32x4_t14__Mfloat8x16_tS0_m(
+// CHECK-CXX-SAME: <4 x float> noundef [[VD:%.*]], <16 x i8> [[VN:%.*]], <16 x i8> [[VM:%.*]], i64 noundef [[FPM:%.*]]) #[[ATTR0]] {
+// CHECK-CXX-NEXT:  [[ENTRY:.*:]]
+// CHECK-CXX-NEXT:    call void @llvm.aarch64.set.fpmr(i64 [[FPM]])
+// CHECK-CXX-NEXT:    [[VMLALL_I:%.*]] = call <4 x float> @llvm.aarch64.neon.fp8.fmlallbt.v4f32(<4 x float> [[VD]], <16 x i8> [[VN]], <16 x i8> [[VM]])
+// CHECK-CXX-NEXT:    ret <4 x float> [[VMLALL_I]]
+//
+float32x4_t test_vmlallbt(float32x4_t vd, mfloat8x16_t vn, mfloat8x16_t vm, fpm_t fpm) {
+  return vmlallbtq_f32_mf8_fpm(vd, vn, vm, fpm);
+}
+
+// CHECK-LABEL: define dso_local <4 x float> @test_vmlalltb(
+// CHECK-SAME: <4 x float> noundef [[VD:%.*]], <16 x i8> [[VN:%.*]], <16 x i8> [[VM:%.*]], i64 noundef [[FPM:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    call void @llvm.aarch64.set.fpmr(i64 [[FPM]])
+// CHECK-NEXT:    [[VMLALL_I:%.*]] = call <4 x float> @llvm.aarch64.neon.fp8.fmlalltb.v4f32(<4 x float> [[VD]], <16 x i8> [[VN]], <16 x i8> [[VM]])
+// CHECK-NEXT:    ret <4 x float> [[VMLALL_I]]
+//
+// CHECK-CXX-LABEL: define dso_local noundef <4 x float> @_Z13test_vmlalltb13__Float32x4_t14__Mfloat8x16_tS0_m(
+// CHECK-CXX-SAME: <4 x float> noundef [[VD:%.*]], <16 x i8> [[VN:%.*]], <16 x i8> [[VM:%.*]], i64 noundef [[FPM:%.*]]) #[[ATTR0]] {
+// CHECK-CXX-NEXT:  [[ENTRY:.*:]]
+// CHECK-CXX-NEXT:    call void @llvm.aarch64.set.fpmr(i64 [[FPM]])
+// CHECK-CXX-NEXT:    [[VMLALL_I:%.*]] = call <4 x float> @llvm.aarch64.neon.fp8.fmlalltb.v4f32(<4 x float> [[VD]], <16 x i8> [[VN]], <16 x i8> [[VM]])
+// CHECK-CXX-NEXT:    ret <4 x float> [[VMLALL_I]]
+//
+float32x4_t test_vmlalltb(float32x4_t vd, mfloat8x16_t vn, mfloat8x16_t vm, fpm_t fpm) {
+  return vmlalltbq_f32_mf8_fpm(vd, vn, vm, fpm);
+}
+
+// CHECK-LABEL: define dso_local <4 x float> @test_vmlalltt(
+// CHECK-SAME: <4 x float> noundef [[VD:%.*]], <16 x i8> [[VN:%.*]], <16 x i8> [[VM:%.*]], i64 noundef [[FPM:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    call void @llvm.aarch64.set.fpmr(i64 [[FPM]])
+// CHECK-NEXT:    [[VMLALL_I:%.*]] = call <4 x float> @llvm.aarch64.neon.fp8.fmlalltt.v4f32(<4 x float> [[VD]], <16 x i8> [[VN]], <16 x i8> [[VM]])
+// CHECK-NEXT:    ret <4 x float> [[VMLALL_I]]
+//
+// CHECK-CXX-LABEL: define dso_local noundef <4 x float> @_Z13test_vmlalltt13__Float32x4_t14__Mfloat8x16_tS0_m(
+// CHECK-CXX-SAME: <4 x float> noundef [[VD:%.*]], <16 x i8> [[VN:%.*]], <16 x i8> [[VM:%.*]], i64 noundef [[FPM:%.*]]) #[[ATTR0]] {
+// CHECK-CXX-NEXT:  [[ENTRY:.*:]]
+// CHECK-CXX-NEXT:    call void @llvm.aarch64.set.fpmr(i64 [[FPM]])
+// CHECK-CXX-NEXT:    [[VMLALL_I:%.*]] = call <4 x float> @llvm.aarch64.neon.fp8.fmlalltt.v4f32(<4 x float> [[VD]], <16 x i8> [[VN]], <16 x i8> [[VM]])
+// CHECK-CXX-NEXT:    ret <4 x float> [[VMLALL_I]]
+//
+float32x4_t test_vmlalltt(float32x4_t vd, mfloat8x16_t vn, mfloat8x16_t vm, fpm_t fpm) {
+  return vmlallttq_f32_mf8_fpm(vd, vn, vm, fpm);
+}
+
+// CHECK-LABEL: define dso_local <8 x half> @test_vmlalb_lane(
+// CHECK-SAME: <8 x half> noundef [[VD:%.*]], <16 x i8> [[VN:%.*]], <8 x i8> [[VM:%.*]], i64 noundef [[FPM:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[TMP0:%.*]] = bitcast <8 x half> [[VD]] to <16 x i8>
+// CHECK-NEXT:    [[TMP1:%.*]] = call <16 x i8> @llvm.vector.insert.v16i8.v8i8(<16 x i8> poison, <8 x i8> [[VM]], i64 0)
+// CHECK-NEXT:    call void @llvm.aarch64.set.fpmr(i64 [[FPM]])
+// CHECK-NEXT:    [[VMLAL_LANE:%.*]] = bitcast <16 x i8> [[TMP0]] to <8 x half>
+// CHECK-NEXT:    [[VMLAL_LANE1:%.*]] = call <8 x half> @llvm.aarch64.neon.fp8.fmlalb.lane.v8f16(<8 x half> [[VMLAL_LANE]], <16 x i8> [[VN]], <16 x i8> [[TMP1]], i32 0)
+// CHECK-NEXT:    ret <8 x half> [[VMLAL_LANE1]]
+//
+// CHECK-CXX-LABEL: define dso_local noundef <8 x half> @_Z16test_vmlalb_lane13__Float16x8_t14__Mfloat8x16_t13__Mfloat8x8_tm(
+// CHECK-CXX-SAME: <8 x half> noundef [[VD:%.*]], <16 x i8> [[VN:%.*]], <8 x i8> [[VM:%.*]], i64 noundef [[FPM:%.*]]) #[[ATTR0]] {
+// CHECK-CXX-NEXT:  [[ENTRY:.*:]]
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = bitcast <8 x half> [[VD]] to <16 x i8>
+// CHECK-CXX-NEXT:    [[TMP1:%.*]] = call <16 x i8> @llvm.vector.insert.v16i8.v8i8(<16 x i8> poison, <8 x i8> [[VM]], i64 0)
+// CHECK-CXX-NEXT:    call void @llvm.aarch64.set.fpmr(i64 [[FPM]])
+// CHECK-CXX-NEXT:    [[VMLAL_LANE:%.*]] = bitcast <16 x i8> [[TMP0]] to <8 x half>
+// CHECK-CXX-NEXT:    [[VMLAL_LANE1:%.*]] = call <8 x half> @llvm.aarch64.neon.fp8.fmlalb.lane.v8f16(<8 x half> [[VMLAL_LANE]], <16 x i8> [[VN]], <16 x i8> [[TMP1]], i32 0)
+// CHECK-CXX-NEXT:    ret <8 x half> [[VMLAL_LANE1]]
+//
+float16x8_t test_vmlalb_lane(float16x8_t vd, mfloat8x16_t vn, mfloat8x8_t vm, fpm_t fpm) {
+  return vmlalbq_lane_f16_mf8_fpm(vd, vn, vm, 0, fpm);
+}
+
+// CHECK-LABEL: define dso_local <8 x half> @test_vmlalb_laneq(
+// CHECK-SAME: <8 x half> noundef [[VD:%.*]], <16 x i8> [[VN:%.*]], <16 x i8> [[VM:%.*]], i64 noundef [[FPM:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[TMP0:%.*]] = bitcast <8 x half> [[VD]] to <16 x i8>
+// CHECK-NEXT:    call void @llvm.aarch64.set.fpmr(i64 [[FPM]])
+// CHECK-NEXT:    [[VMLAL_LANE:%.*]] = bitcast <16 x i8> [[TMP0]] to <8 x half>
+// CHECK-NEXT:    [[VMLAL_LANE1:%.*]] = call <8 x half> @llvm.aarch64.neon.fp8.fmlalb.lane.v8f16(<8 x half> [[VMLAL_LANE]], <16 x i8> [[VN]], <16 x i8> [[VM]], i32 0)
+// CHECK-NEXT:    ret <8 x half> [[VMLAL_LANE1]]
+//
+// CHECK-CXX-LABEL: define dso_local noundef <8 x half> @_Z17test_vmlalb_laneq13__Float16x8_t14__Mfloat8x16_tS0_m(
+// CHECK-CXX-SAME: <8 x half> noundef [[VD:%.*]], <16 x i8> [[VN:%.*]], <16 x i8> [[VM:%.*]], i64 noundef [[FPM:%.*]]) #[[ATTR0]] {
+// CHECK-CXX-NEXT:  [[ENTRY:.*:]]
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = bitcast <8 x half> [[VD]] to <16 x i8>
+// CHECK-CXX-NEXT:    call void @llvm.aarch64.set.fpmr(i64 [[FPM]])
+// CHECK-CXX-NEXT:    [[VMLAL_LANE:%.*]] = bitcast <16 x i8> [[TMP0]] to <8 x half>
+// CHECK-CXX-NEXT:    [[VMLAL_LANE1:%.*]] = call <8 x half> @llvm.aarch64.neon.fp8.fmlalb.lane.v8f16(<8 x half> [[VMLAL_LANE]], <16 x i8> [[VN]], <16 x i8> [[VM]], i32 0)
+// CHECK-CXX-NEXT:    ret <8 x half> [[VMLAL_LANE1]]
+//
+float16x8_t test_vmlalb_laneq(float16x8_t vd, mfloat8x16_t vn, mfloat8x16_t vm, fpm_t fpm) {
+  return vmlalbq_laneq_f16_mf8_fpm(vd, vn, vm, 0, fpm);
+}
+
+// CHECK-LABEL: define dso_local <8 x half> @test_vmlalt_lane(
+// CHECK-SAME: <8 x half> noundef [[VD:%.*]], <16 x i8> [[VN:%.*]], <8 x i8> [[VM:%.*]], i64 noundef [[FPM:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[TMP0:%.*]] = bitcast <8 x half> [[VD]] to <16 x i8>
+// CHECK-NEXT:    [[TMP1:%.*]] = call <16 x i8> @llvm.vector.insert.v16i8.v8i8(<16 x i8> poison, <8 x i8> [[VM]], i64 0)
+// CHECK-NEXT:    call void @llvm.aarch64.set.fpmr(i64 [[FPM]])
+// CHECK-NEXT:    [[VMLAL_LANE:%.*]] = bitcast <16 x i8> [[TMP0]] to <8 x half>
+// CHECK-NEXT:    [[VMLAL_LANE1:%.*]] = call <8 x half> @llvm.aarch64.neon.fp8.fmlalt.lane.v8f16(<8 x half> [[VMLAL_LANE]], <16 x i8> [[VN]], <16 x i8> [[TMP1]], i32 7)
+// CHECK-NEXT:    ret <8 x half> [[VMLAL_LANE1]]
+//
+// CHECK-CXX-LABEL: define dso_local noundef <8 x half> @_Z16test_vmlalt_lane13__Float16x8_t14__Mfloat8x16_t13__Mfloat8x8_tm(
+// CHECK-CXX-SAME: <8 x half> noundef [[VD:%.*]], <16 x i8> [[VN:%.*]], <8 x i8> [[VM:%.*]], i64 noundef [[FPM:%.*]]) #[[ATTR0]] {
+// CHECK-CXX-NEXT:  [[ENTRY:.*:]]
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = bitcast <8 x half> [[VD]] to <16 x i8>
+// CHECK-CXX-NEXT:    [[TMP1:%.*]] = call <16 x i8> @llvm.vector.insert.v16i8.v8i8(<16 x i8> poison, <8 x i8> [[VM]], i64 0)
+// CHECK-CXX-NEXT:    call void @llvm.aarch64.set.fpmr(i64 [[FPM]])
+// CHECK-CXX-NEXT:    [[VMLAL_LANE:%.*]] = bitcast <16 x i8> [[TMP0]] to <8 x half>
+// CHECK-CXX-NEXT:    [[VMLAL_LANE1:%.*]] = call <8 x half> @llvm.aarch64.neon.fp8.fmlalt.lane.v8f16(<8 x half> [[VMLAL_LANE]], <16 x i8> [[VN]], <16 x i8> [[TMP1]], i32 7)
+// CHECK-CXX-NEXT:    ret <8 x half> [[VMLAL_LANE1]]
+//
+float16x8_t test_vmlalt_lane(float16x8_t vd, mfloat8x16_t vn, mfloat8x8_t vm, fpm_t fpm) {
+  return vmlaltq_lane_f16_mf8_fpm(vd, vn, vm, 7, fpm);
+}
+
+// CHECK-LABEL: define dso_local <8 x half> @test_vmlalt_laneq(
+// CHECK-SAME: <8 x half> noundef [[VD:%.*]], <16 x i8> [[VN:%.*]], <16 x i8> [[VM:%.*]], i64 noundef [[FPM:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[TMP0:%.*]] = bitcast <8 x half> [[VD]] to <16 x i8>
+// CHECK-NEXT:    call void @llvm.aarch64.set.fpmr(i64 [[FPM]])
+// CHECK-NEXT:    [[VMLAL_LANE:%.*]] = bitcast <16 x i8> [[TMP0]] to <8 x half>
+// CHECK-NEXT:    [[VMLAL_LANE1:%.*]] = call <8 x half> @llvm.aarch64.neon.fp8.fmlalt.lane.v8f16(<8 x half> [[VMLAL_LANE]], <16 x i8> [[VN]], <16 x i8> [[VM]], i32 15)
+// CHECK-NEXT:    ret <8 x half> [[VMLAL_LANE1]]
+//
+// CHECK-CXX-LABEL: define dso_local noundef <8 x half> @_Z17test_vmlalt_laneq13__Float16x8_t14__Mfloat8x16_tS0_m(
+// CHECK-CXX-SAME: <8 x half> noundef [[VD:%.*]], <16 x i8> [[VN:%.*]], <16 x i8> [[VM:%.*]], i64 noundef [[FPM:%.*]]) #[[ATTR0]] {
+// CHECK-CXX-NEXT:  [[ENTRY:.*:]]
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = bitcast <8 x half> [[VD]] to <16 x i8>
+// CHECK-CXX-NEXT:    call void @llvm.aarch64.set.fpmr(i64 [[FPM]])
+// CHECK-CXX-NEXT:    [[VMLAL_LANE:%.*]] = bitcast <16 x i8> [[TMP0]] to <8 x half>
+// CHECK-CXX-NEXT:    [[VMLAL_LANE1:%.*]] = call <8 x half> @llvm.aarch64.neon.fp8.fmlalt.lane.v8f16(<8 x half> [[VMLAL_LANE]], <16 x i8> [[VN]], <16 x i8> [[VM]], i32 15)
+// CHECK-CXX-NEXT:    ret <8 x half> [[VMLAL_LANE1]]
+//
+float16x8_t test_vmlalt_laneq(float16x8_t vd, mfloat8x16_t vn, mfloat8x16_t vm, fpm_t fpm) {
+  return vmlaltq_laneq_f16_mf8_fpm(vd, vn, vm, 15, fpm);
+}
+
+// CHECK-LABEL: define dso_local <4 x float> @test_vmlallbb_lane(
+// CHECK-SAME: <4 x float> noundef [[VD:%.*]], <16 x i8> [[VN:%.*]], <8 x i8> [[VM:%.*]], i64 noundef [[FPM:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[TMP0:%.*]] = call <16 x i8> @llvm.vector.insert.v16i8.v8i8(<16 x i8> poison, <8 x i8> [[VM]], i64 0)
+// CHECK-NEXT:    call void @llvm.aarch64.set.fpmr(i64 [[FPM]])
+// CHECK-NEXT:    [[VMLALL_LANE:%.*]] = call <4 x float> @llvm.aarch64.neon.fp8.fmlallbb.lane.v4f32(<4 x float> [[VD]], <16 x i8> [[VN]], <16 x i8> [[TMP0]], i32 0)
+// CHECK-NEXT:    ret <4 x float> [[VMLALL_LANE]]
+//
+// CHECK-CXX-LABEL: define dso_local noundef <4 x float> @_Z18test_vmlallbb_lane13__Float32x4_t14__Mfloat8x16_t13__Mfloat8x8_tm(
+// CHECK-CXX-SAME: <4 x float> noundef [[VD:%.*]], <16 x i8> [[VN:%.*]], <8 x i8> [[VM:%.*]], i64 noundef [[FPM:%.*]]) #[[ATTR0]] {
+// CHECK-CXX-NEXT:  [[ENTRY:.*:]]
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = call <16 x i8> @llvm.vector.insert.v16i8.v8i8(<16 x i8> poison, <8 x i8> [[VM]], i64 0)
+// CHECK-CXX-NEXT:    call void @llvm.aarch64.set.fpmr(i64 [[FPM]])
+// CHECK-CXX-NEXT:    [[VMLALL_LANE:%.*]] = call <4 x float> @llvm.aarch64.neon.fp8.fmlallbb.lane.v4f32(<4 x float> [[VD]], <16 x i8> [[VN]], <16 x i8> [[TMP0]], i32 0)
+// CHECK-CXX-NEXT:    ret <4 x float> [[VMLALL_LANE]]
+//
+float32x4_t test_vmlallbb_lane(float32x4_t vd, mfloat8x16_t vn, mfloat8x8_t vm, fpm_t fpm) {
+  return vmlallbbq_lane_f32_mf8_fpm(vd, vn, vm, 0, fpm);
+}
+
+// CHECK-LABEL: define dso_local <4 x float> @test_vmlallbb_laneq(
+// CHECK-SAME: <4 x float> noundef [[VD:%.*]], <16 x i8> [[VN:%.*]], <16 x i8> [[VM:%.*]], i64 noundef [[FPM:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    call void @llvm.aarch64.set.fpmr(i64 [[FPM]])
+// CHECK-NEXT:    [[VMLALL_LANE:%.*]] = call <4 x float> @llvm.aarch64.neon.fp8.fmlallbb.lane.v4f32(<4 x float> [[VD]], <16 x i8> [[VN]], <16 x i8> [[VM]], i32 0)
+// CHECK-NEXT:    ret <4 x float> [[VMLALL_LANE]]
+//
+// CHECK-CXX-LABEL: define dso_local noundef <4 x float> @_Z19test_vmlallbb_laneq13__Float32x4_t14__Mfloat8x16_tS0_m(
+// CHECK-CXX-SAME: <4 x float> noundef [[VD:%.*]], <16 x i8> [[VN:%.*]], <16 x i8> [[VM:%.*]], i64 noundef [[FPM:%.*]]) #[[ATTR0]] {
+// CHECK-CXX-NEXT:  [[ENTRY:.*:]]
+// CHECK-CXX-NEXT:    call void @llvm.aarch64.set.fpmr(i64 [[FPM]])
+// CHECK-CXX-NEXT:    [[VMLALL_LANE:%.*]] = call <4 x float> @llvm.aarch64.neon.fp8.fmlallbb.lane.v4f32(<4 x float> [[VD]], <16 x i8> [[VN]], <16 x i8> [[VM]], i32 0)
+// CHECK-CXX-NEXT:    ret <4 x float> [[VMLALL_LANE]]
+//
+float32x4_t test_vmlallbb_laneq(float32x4_t vd, mfloat8x16_t vn, mfloat8x16_t vm, fpm_t fpm) {
+  return vmlallbbq_laneq_f32_mf8_fpm(vd, vn, vm, 0, fpm);
+}
+
+// CHECK-LABEL: define dso_local <4 x float> @test_vmlallbt_lane(
+// CHECK-SAME: <4 x float> noundef [[VD:%.*]], <16 x i8> [[VN:%.*]], <8 x i8> [[VM:%.*]], i64 noundef [[FPM:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[TMP0:%.*]] = call <16 x i8> @llvm.vector.insert.v16i8.v8i8(<16 x i8> poison, <8 x i8> [[VM]], i64 0)
+// CHECK-NEXT:    call void @llvm.aarch64.set.fpmr(i64 [[FPM]])
+// CHECK-NEXT:    [[VMLALL_LANE:%.*]] = call <4 x float> @llvm.aarch64.neon.fp8.fmlallbt.lane.v4f32(<4 x float> [[VD]], <16 x i8> [[VN]], <16 x i8> [[TMP0]], i32 3)
+// CHECK-NEXT:    ret <4 x float> [[VMLALL_LANE]]
+//
+// CHECK-CXX-LABEL: define dso_local noundef <4 x float> @_Z18test_vmlallbt_lane13__Float32x4_t14__Mfloat8x16_t13__Mfloat8x8_tm(
+// CHECK-CXX-SAME: <4 x float> noundef [[VD:%.*]], <16 x i8> [[VN:%.*]], <8 x i8> [[VM:%.*]], i64 noundef [[FPM:%.*]]) #[[ATTR0]] {
+// CHECK-CXX-NEXT:  [[ENTRY:.*:]]
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = call <16 x i8> @llvm.vector.insert.v16i8.v8i8(<16 x i8> poison, <8 x i8> [[VM]], i64 0)
+// CHECK-CXX-NEXT:    call void @llvm.aarch64.set.fpmr(i64 [[FPM]])
+// CHECK-CXX-NEXT:    [[VMLALL_LANE:%.*]] = call <4 x float> @llvm.aarch64.neon.fp8.fmlallbt.lane.v4f32(<4 x float> [[VD]], <16 x i8> [[VN]], <16 x i8> [[TMP0]], i32 3)
+// CHECK-CXX-NEXT:    ret <4 x float> [[VMLALL_LANE]]
+//
+float32x4_t test_vmlallbt_lane(float32x4_t vd, mfloat8x16_t vn, mfloat8x8_t vm, fpm_t fpm) {
+  return vmlallbtq_lane_f32_mf8_fpm(vd, vn, vm, 3, fpm);
+}
+
+// CHECK-LABEL: define dso_local <4 x float> @test_vmlallbt_laneq(
+// CHECK-SAME: <4 x float> noundef [[VD:%.*]], <16 x i8> [[VN:%.*]], <16 x i8> [[VM:%.*]], i64 noundef [[FPM:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    call void @llvm.aarch64.set.fpmr(i64 [[FPM]])
+// CHECK-NEXT:    [[VMLALL_LANE:%.*]] = call <4 x float> @llvm.aarch64.neon.fp8.fmlallbt.lane.v4f32(<4 x float> [[VD]], <16 x i8> [[VN]], <16 x i8> [[VM]], i32 3)
+// CHECK-NEXT:    ret <4 x float> [[VMLALL_LANE]]
+//
+// CHECK-CXX-LABEL: define dso_local noundef <4 x float> @_Z19test_vmlallbt_laneq13__Float32x4_t14__Mfloat8x16_tS0_m(
+// CHECK-CXX-SAME: <4 x float> noundef [[VD:%.*]], <16 x i8> [[VN:%.*]], <16 x i8> [[VM:%.*]], i64 noundef [[FPM:%.*]]) #[[ATTR0]] {
+// CHECK-CXX-NEXT:  [[ENTRY:.*:]]
+// CHECK-CXX-NEXT:    call void @llvm.aarch64.set.fpmr(i64 [[FPM]])
+// CHECK-CXX-NEXT:    [[VMLALL_LANE:%.*]] = call <4 x float> @llvm.aarch64.neon.fp8.fmlallbt.lane.v4f32(<4 x float> [[VD]], <16 x i8> [[VN]], <16 x i8> [[VM]], i32 3)
+// CHECK-CXX-NEXT:    ret <4 x float> [[VMLALL_LANE]]
+//
+float32x4_t test_vmlallbt_laneq(float32x4_t vd, mfloat8x16_t vn, mfloat8x16_t vm, fpm_t fpm) {
+  return vmlallbtq_laneq_f32_mf8_fpm(vd, vn, vm, 3, fpm);
+}
+
+// CHECK-LABEL: define dso_local <4 x float> @test_vmlalltb_lane(
+// CHECK-SAME: <4 x float> noundef [[VD:%.*]], <16 x i8> [[VN:%.*]], <8 x i8> [[VM:%.*]], i64 noundef [[FPM:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[TMP0:%.*]] = call <16 x i8> @llvm.vector.insert.v16i8.v8i8(<16 x i8> poison, <8 x i8> [[VM]], i64 0)
+// CHECK-NEXT:    call void @llvm.aarch64.set.fpmr(i64 [[FPM]])
+// CHECK-NEXT:    [[VMLALL_LANE:%.*]] = call <4 x float> @llvm.aarch64.neon.fp8.fmlalltb.lane.v4f32(<4 x float> [[VD]], <16 x i8> [[VN]], <16 x i8> [[TMP0]], i32 7)
+// CHECK-NEXT:    ret <4 x float> [[VMLALL_LANE]]
+//
+// CHECK-CXX-LABEL: define dso_local noundef <4 x float> @_Z18test_vmlalltb_lane13__Float32x4_t14__Mfloat8x16_t13__Mfloat8x8_tm(
+// CHECK-CXX-SAME: <4 x float> noundef [[VD:%.*]], <16 x i8> [[VN:%.*]], <8 x i8> [[VM:%.*]], i64 noundef [[FPM:%.*]]) #[[ATTR0]] {
+// CHECK-CXX-NEXT:  [[ENTRY:.*:]]
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = call <16 x i8> @llvm.vector.insert.v16i8.v8i8(<16 x i8> poison, <8 x i8> [[VM]], i64 0)
+// CHECK-CXX-NEXT:    call void @llvm.aarch64.set.fpmr(i64 [[FPM]])
+// CHECK-CXX-NEXT:    [[VMLALL_LANE:%.*]] = call <4 x float> @llvm.aarch64.neon.fp8.fmlalltb.lane.v4f32(<4 x float> [[VD]], <16 x i8> [[VN]], <16 x i8> [[TMP0]], i32 7)
+// CHECK-CXX-NEXT:    ret <4 x float> [[VMLALL_LANE]]
+//
+float32x4_t test_vmlalltb_lane(float32x4_t vd, mfloat8x16_t vn, mfloat8x8_t vm, fpm_t fpm) {
+  return vmlalltbq_lane_f32_mf8_fpm(vd, vn, vm, 7, fpm);
+}
+
+// CHECK-LABEL: define dso_local <4 x float> @test_vmlalltb_laneq(
+// CHECK-SAME: <4 x float> noundef [[VD:%.*]], <16 x i8> [[VN:%.*]], <16 x i8> [[VM:%.*]], i64 noundef [[FPM:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    call void @llvm.aarch64.set.fpmr(i64 [[FPM]])
+// CHECK-NEXT:    [[VMLALL_LANE:%.*]] = call <4 x float> @llvm.aarch64.neon.fp8.fmlalltb.lane.v4f32(<4 x float> [[VD]], <16 x i8> [[VN]], <16 x i8> [[VM]], i32 7)
+// CHECK-NEXT:    ret <4 x float> [[VMLALL_LANE]]
+//
+// CHECK-CXX-LABEL: define dso_local noundef <4 x float> @_Z19test_vmlalltb_laneq13__Float32x4_t14__Mfloat8x16_tS0_m(
+// CHECK-CXX-SAME: <4 x float> noundef [[VD:%.*]], <16 x i8> [[VN:%.*]], <16 x i8> [[VM:%.*]], i64 noundef [[FPM:%.*]]) #[[ATTR0]] {
+// CHECK-CXX-NEXT:  [[ENTRY:.*:]]
+// CHECK-CXX-NEXT:    call void @llvm.aarch64.set.fpmr(i64 [[FPM]])
+// CHECK-CXX-NEXT:    [[VMLALL_LANE:%.*]] = call <4 x float> @llvm.aarch64.neon.fp8.fmlalltb.lane.v4f32(<4 x float> [[VD]], <16 x i8> [[VN]], <16 x i8> [[VM]], i32 7)
+// CHECK-CXX-NEXT:    ret <4 x float> [[VMLALL_LANE]]
+//
+float32x4_t test_vmlalltb_laneq(float32x4_t vd, mfloat8x16_t vn, mfloat8x16_t vm, fpm_t fpm) {
+  return vmlalltbq_laneq_f32_mf8_fpm(vd, vn, vm, 7, fpm);
+}
+
+// CHECK-LABEL: define dso_local <4 x float> @test_vmlalltt_lane(
+// CHECK-SAME: <4 x float> noundef [[VD:%.*]], <16 x i8> [[VN:%.*]], <8 x i8> [[VM:%.*]], i64 noundef [[FPM:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[TMP0:%.*]] = call <16 x i8> @llvm.vector.insert.v16i8.v8i8(<16 x i8> poison, <8 x i8> [[VM]], i64 0)
+// CHECK-NEXT:    call void @llvm.aarch64.set.fpmr(i64 [[FPM]])
+// CHECK-NEXT:    [[VMLALL_LANE:%.*]] = call <4 x float> @llvm.aarch64.neon.fp8.fmlalltt.lane.v4f32(<4 x float> [[VD]], <16 x i8> [[VN]], <16 x i8> [[TMP0]], i32 7)
+// CHECK-NEXT:    ret <4 x float> [[VMLALL_LANE]]
+//
+// CHECK-CXX-LABEL: define dso_local noundef <4 x float> @_Z18test_vmlalltt_lane13__Float32x4_t14__Mfloat8x16_t13__Mfloat8x8_tm(
+// CHECK-CXX-SAME: <4 x float> noundef [[VD:%.*]], <16 x i8> [[VN:%.*]], <8 x i8> [[VM:%.*]], i64 noundef [[FPM:%.*]]) #[[ATTR0]] {
+// CHECK-CXX-NEXT:  [[ENTRY:.*:]]
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = call <16 x i8> @llvm.vector.insert.v16i8.v8i8(<16 x i8> poison, <8 x i8> [[VM]], i64 0)
+// CHECK-CXX-NEXT:    call void @llvm.aarch64.set.fpmr(i64 [[FPM]])
+// CHECK-CXX-NEXT:    [[VMLALL_LANE:%.*]] = call <4 x float> @llvm.aarch64.neon.fp8.fmlalltt.lane.v4f32(<4 x float> [[VD]], <16 x i8> [[VN]], <16 x i8> [[TMP0]], i32 7)
+// CHECK-CXX-NEXT:    ret <4 x float> [[VMLALL_LANE]]
+//
+float32x4_t test_vmlalltt_lane(float32x4_t vd, mfloat8x16_t vn, mfloat8x8_t vm, fpm_t fpm) {
+  return vmlallttq_lane_f32_mf8_fpm(vd, vn, vm, 7, fpm);
+}
+
+// CHECK-LABEL: define dso_local <4 x float> @test_vmlalltt_laneq(
+// CHECK-SAME: <4 x float> noundef [[VD:%.*]], <16 x i8> [[VN:%.*]], <16 x i8> [[VM:%.*]], i64 noundef [[FPM:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    call void @llvm.aarch64.set.fpmr(i64 [[FPM]])
+// CHECK-NEXT:    [[VMLALL_LANE:%.*]] = call <4 x float> @llvm.aarch64.neon.fp8.fmlalltt.lane.v4f32(<4 x float> [[VD]], <16 x i8> [[VN]], <16 x i8> [[VM]], i32 15)
+// CHECK-NEXT:    ret <4 x float> [[VMLALL_LANE]]
+//
+// CHECK-CXX-LABEL: define dso_local noundef <4 x float> @_Z19test_vmlalltt_laneq13__Float32x4_t14__Mfloat8x16_tS0_m(
+// CHECK-CXX-SAME: <4 x float> noundef [[VD:%.*]], <16 x i8> [[VN:%.*]], <16 x i8> [[VM:%.*]], i64 noundef [[FPM:%.*]]) #[[ATTR0]] {
+// CHECK-CXX-NEXT:  [[ENTRY:.*:]]
+// CHECK-CXX-NEXT:    call void @llvm.aarch64.set.fpmr(i64 [[FPM]])
+// CHECK-CXX-NEXT:    [[VMLALL_LANE:%.*]] = call <4 x float> @llvm.aarch64.neon.fp8.fmlalltt.lane.v4f32(<4 x float> [[VD]], <16 x i8> [[VN]], <16 x i8> [[VM]], i32 15)
+// CHECK-CXX-NEXT:    ret <4 x float> [[VMLALL_LANE]]
+//
+float32x4_t test_vmlalltt_laneq(float32x4_t vd, mfloat8x16_t vn, mfloat8x16_t vm, fpm_t fpm) {
+  return vmlallttq_laneq_f32_mf8_fpm(vd, vn, vm, 15, fpm);
+}
diff --git a/clang/test/CodeGen/AArch64/fp8-intrinsics/acle_neon_fp8_reinterpret.c b/clang/test/CodeGen/AArch64/fp8-intrinsics/acle_neon_fp8_reinterpret.c
new file mode 100644
index 0000000000000..201d4dbbe34ad
--- /dev/null
+++ b/clang/test/CodeGen/AArch64/fp8-intrinsics/acle_neon_fp8_reinterpret.c
@@ -0,0 +1,855 @@
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --version 5
+#include <arm_neon.h>
+// RUN: %clang_cc1        -triple aarch64-none-linux-gnu -target-feature +neon -target-feature +bf16 -target-feature +fp8 -target-feature +fp8fma -disable-O0-optnone -Werror -Wall -emit-llvm -o - %s | opt -S -p mem2reg | FileCheck %s
+// RUN: %clang_cc1 -x c++ -triple aarch64-none-linux-gnu -target-feature +neon -target-feature +bf16 -target-feature +fp8 -target-feature +fp8fma -disable-O0-optnone -Werror -Wall -emit-llvm -o - %s | opt -S -p mem2reg | FileCheck %s -check-prefix CHECK-CXX
+
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +neon -target-feature +bf16 -target-feature +fp8 -target-feature +fp8fma -disable-O0-optnone -Werror -Wall -S -o /dev/null %s
+
+// REQUIRES: aarch64-registered-target
+
+// CHECK-LABEL: define dso_local <8 x i8> @test_vreinterpret_p8_mf8(
+// CHECK-SAME: <8 x i8> [[V:%.*]]) #[[ATTR0:[0-9]+]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    ret <8 x i8> [[V]]
+//
+// CHECK-CXX-LABEL: define dso_local noundef <8 x i8> @_Z24test_vreinterpret_p8_mf813__Mfloat8x8_t(
+// CHECK-CXX-SAME: <8 x i8> [[V:%.*]]) #[[ATTR0:[0-9]+]] {
+// CHECK-CXX-NEXT:  [[ENTRY:.*:]]
+// CHECK-CXX-NEXT:    ret <8 x i8> [[V]]
+//
+poly8x8_t test_vreinterpret_p8_mf8(mfloat8x8_t v) {
+  return vreinterpret_p8_mf8(v);
+}
+// CHECK-LABEL: define dso_local <1 x i64> @test_vreinterpret_p64_mf8(
+// CHECK-SAME: <8 x i8> [[V:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[TMP0:%.*]] = bitcast <8 x i8> [[V]] to <1 x i64>
+// CHECK-NEXT:    ret <1 x i64> [[TMP0]]
+//
+// CHECK-CXX-LABEL: define dso_local noundef <1 x i64> @_Z25test_vreinterpret_p64_mf813__Mfloat8x8_t(
+// CHECK-CXX-SAME: <8 x i8> [[V:%.*]]) #[[ATTR0]] {
+// CHECK-CXX-NEXT:  [[ENTRY:.*:]]
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = bitcast <8 x i8> [[V]] to <1 x i64>
+// CHECK-CXX-NEXT:    ret <1 x i64> [[TMP0]]
+//
+poly64x1_t test_vreinterpret_p64_mf8(mfloat8x8_t v) {
+  return vreinterpret_p64_mf8(v);
+}
+// CHECK-LABEL: define dso_local <4 x i16> @test_vreinterpret_p16_mf8(
+// CHECK-SAME: <8 x i8> [[V:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[TMP0:%.*]] = bitcast <8 x i8> [[V]] to <4 x i16>
+// CHECK-NEXT:    ret <4 x i16> [[TMP0]]
+//
+// CHECK-CXX-LABEL: define dso_local noundef <4 x i16> @_Z25test_vreinterpret_p16_mf813__Mfloat8x8_t(
+// CHECK-CXX-SAME: <8 x i8> [[V:%.*]]) #[[ATTR0]] {
+// CHECK-CXX-NEXT:  [[ENTRY:.*:]]
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = bitcast <8 x i8> [[V]] to <4 x i16>
+// CHECK-CXX-NEXT:    ret <4 x i16> [[TMP0]]
+//
+poly16x4_t test_vreinterpret_p16_mf8(mfloat8x8_t v) {
+  return vreinterpret_p16_mf8(v);
+}
+// CHECK-LABEL: define dso_local <16 x i8> @test_vreinterpretq_p8_mf8(
+// CHECK-SAME: <16 x i8> [[V:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    ret <16 x i8> [[V]]
+//
+// CHECK-CXX-LABEL: define dso_local noundef <16 x i8> @_Z25test_vreinterpretq_p8_mf814__Mfloat8x16_t(
+// CHECK-CXX-SAME: <16 x i8> [[V:%.*]]) #[[ATTR0]] {
+// CHECK-CXX-NEXT:  [[ENTRY:.*:]]
+// CHECK-CXX-NEXT:    ret <16 x i8> [[V]]
+//
+poly8x16_t test_vreinterpretq_p8_mf8(mfloat8x16_t v) {
+  return vreinterpretq_p8_mf8(v);
+}
+// CHECK-LABEL: define dso_local i128 @test_vreinterpretq_p128_mf8(
+// CHECK-SAME: <16 x i8> [[V:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[TMP0:%.*]] = bitcast <16 x i8> [[V]] to i128
+// CHECK-NEXT:    ret i128 [[TMP0]]
+//
+// CHECK-CXX-LABEL: define dso_local noundef i128 @_Z27test_vreinterpretq_p128_mf814__Mfloat8x16_t(
+// CHECK-CXX-SAME: <16 x i8> [[V:%.*]]) #[[ATTR0]] {
+// CHECK-CXX-NEXT:  [[ENTRY:.*:]]
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = bitcast <16 x i8> [[V]] to i128
+// CHECK-CXX-NEXT:    ret i128 [[TMP0]]
+//
+poly128_t test_vreinterpretq_p128_mf8(mfloat8x16_t v) {
+  return vreinterpretq_p128_mf8(v);
+}
+// CHECK-LABEL: define dso_local <2 x i64> @test_vreinterpretq_p64_mf8(
+// CHECK-SAME: <16 x i8> [[V:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[TMP0:%.*]] = bitcast <16 x i8> [[V]] to <2 x i64>
+// CHECK-NEXT:    ret <2 x i64> [[TMP0]]
+//
+// CHECK-CXX-LABEL: define dso_local noundef <2 x i64> @_Z26test_vreinterpretq_p64_mf814__Mfloat8x16_t(
+// CHECK-CXX-SAME: <16 x i8> [[V:%.*]]) #[[ATTR0]] {
+// CHECK-CXX-NEXT:  [[ENTRY:.*:]]
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = bitcast <16 x i8> [[V]] to <2 x i64>
+// CHECK-CXX-NEXT:    ret <2 x i64> [[TMP0]]
+//
+poly64x2_t test_vreinterpretq_p64_mf8(mfloat8x16_t v) {
+  return vreinterpretq_p64_mf8(v);
+}
+// CHECK-LABEL: define dso_local <8 x i16> @test_vreinterpretq_p16_mf8(
+// CHECK-SAME: <16 x i8> [[V:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[TMP0:%.*]] = bitcast <16 x i8> [[V]] to <8 x i16>
+// CHECK-NEXT:    ret <8 x i16> [[TMP0]]
+//
+// CHECK-CXX-LABEL: define dso_local noundef <8 x i16> @_Z26test_vreinterpretq_p16_mf814__Mfloat8x16_t(
+// CHECK-CXX-SAME: <16 x i8> [[V:%.*]]) #[[ATTR0]] {
+// CHECK-CXX-NEXT:  [[ENTRY:.*:]]
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = bitcast <16 x i8> [[V]] to <8 x i16>
+// CHECK-CXX-NEXT:    ret <8 x i16> [[TMP0]]
+//
+poly16x8_t test_vreinterpretq_p16_mf8(mfloat8x16_t v) {
+  return vreinterpretq_p16_mf8(v);
+}
+// CHECK-LABEL: define dso_local <16 x i8> @test_vreinterpretq_u8_mf8(
+// CHECK-SAME: <16 x i8> [[V:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    ret <16 x i8> [[V]]
+//
+// CHECK-CXX-LABEL: define dso_local noundef <16 x i8> @_Z25test_vreinterpretq_u8_mf814__Mfloat8x16_t(
+// CHECK-CXX-SAME: <16 x i8> [[V:%.*]]) #[[ATTR0]] {
+// CHECK-CXX-NEXT:  [[ENTRY:.*:]]
+// CHECK-CXX-NEXT:    ret <16 x i8> [[V]]
+//
+uint8x16_t test_vreinterpretq_u8_mf8(mfloat8x16_t v) {
+  return vreinterpretq_u8_mf8(v);
+}
+// CHECK-LABEL: define dso_local <4 x i32> @test_vreinterpretq_u32_mf8(
+// CHECK-SAME: <16 x i8> [[V:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[TMP0:%.*]] = bitcast <16 x i8> [[V]] to <4 x i32>
+// CHECK-NEXT:    ret <4 x i32> [[TMP0]]
+//
+// CHECK-CXX-LABEL: define dso_local noundef <4 x i32> @_Z26test_vreinterpretq_u32_mf814__Mfloat8x16_t(
+// CHECK-CXX-SAME: <16 x i8> [[V:%.*]]) #[[ATTR0]] {
+// CHECK-CXX-NEXT:  [[ENTRY:.*:]]
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = bitcast <16 x i8> [[V]] to <4 x i32>
+// CHECK-CXX-NEXT:    ret <4 x i32> [[TMP0]]
+//
+uint32x4_t test_vreinterpretq_u32_mf8(mfloat8x16_t v) {
+  return vreinterpretq_u32_mf8(v);
+}
+// CHECK-LABEL: define dso_local <2 x i64> @test_vreinterpretq_u64_mf8(
+// CHECK-SAME: <16 x i8> [[V:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[TMP0:%.*]] = bitcast <16 x i8> [[V]] to <2 x i64>
+// CHECK-NEXT:    ret <2 x i64> [[TMP0]]
+//
+// CHECK-CXX-LABEL: define dso_local noundef <2 x i64> @_Z26test_vreinterpretq_u64_mf814__Mfloat8x16_t(
+// CHECK-CXX-SAME: <16 x i8> [[V:%.*]]) #[[ATTR0]] {
+// CHECK-CXX-NEXT:  [[ENTRY:.*:]]
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = bitcast <16 x i8> [[V]] to <2 x i64>
+// CHECK-CXX-NEXT:    ret <2 x i64> [[TMP0]]
+//
+uint64x2_t test_vreinterpretq_u64_mf8(mfloat8x16_t v) {
+  return vreinterpretq_u64_mf8(v);
+}
+// CHECK-LABEL: define dso_local <8 x i16> @test_vreinterpretq_u16_mf8(
+// CHECK-SAME: <16 x i8> [[V:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[TMP0:%.*]] = bitcast <16 x i8> [[V]] to <8 x i16>
+// CHECK-NEXT:    ret <8 x i16> [[TMP0]]
+//
+// CHECK-CXX-LABEL: define dso_local noundef <8 x i16> @_Z26test_vreinterpretq_u16_mf814__Mfloat8x16_t(
+// CHECK-CXX-SAME: <16 x i8> [[V:%.*]]) #[[ATTR0]] {
+// CHECK-CXX-NEXT:  [[ENTRY:.*:]]
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = bitcast <16 x i8> [[V]] to <8 x i16>
+// CHECK-CXX-NEXT:    ret <8 x i16> [[TMP0]]
+//
+uint16x8_t test_vreinterpretq_u16_mf8(mfloat8x16_t v) {
+  return vreinterpretq_u16_mf8(v);
+}
+// CHECK-LABEL: define dso_local <16 x i8> @test_vreinterpretq_s8_mf8(
+// CHECK-SAME: <16 x i8> [[V:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    ret <16 x i8> [[V]]
+//
+// CHECK-CXX-LABEL: define dso_local noundef <16 x i8> @_Z25test_vreinterpretq_s8_mf814__Mfloat8x16_t(
+// CHECK-CXX-SAME: <16 x i8> [[V:%.*]]) #[[ATTR0]] {
+// CHECK-CXX-NEXT:  [[ENTRY:.*:]]
+// CHECK-CXX-NEXT:    ret <16 x i8> [[V]]
+//
+int8x16_t test_vreinterpretq_s8_mf8(mfloat8x16_t v) {
+  return vreinterpretq_s8_mf8(v);
+}
+// CHECK-LABEL: define dso_local <2 x double> @test_vreinterpretq_f64_mf8(
+// CHECK-SAME: <16 x i8> [[V:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[TMP0:%.*]] = bitcast <16 x i8> [[V]] to <2 x double>
+// CHECK-NEXT:    ret <2 x double> [[TMP0]]
+//
+// CHECK-CXX-LABEL: define dso_local noundef <2 x double> @_Z26test_vreinterpretq_f64_mf814__Mfloat8x16_t(
+// CHECK-CXX-SAME: <16 x i8> [[V:%.*]]) #[[ATTR0]] {
+// CHECK-CXX-NEXT:  [[ENTRY:.*:]]
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = bitcast <16 x i8> [[V]] to <2 x double>
+// CHECK-CXX-NEXT:    ret <2 x double> [[TMP0]]
+//
+float64x2_t test_vreinterpretq_f64_mf8(mfloat8x16_t v) {
+  return vreinterpretq_f64_mf8(v);
+}
+// CHECK-LABEL: define dso_local <4 x float> @test_vreinterpretq_f32_mf8(
+// CHECK-SAME: <16 x i8> [[V:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[TMP0:%.*]] = bitcast <16 x i8> [[V]] to <4 x float>
+// CHECK-NEXT:    ret <4 x float> [[TMP0]]
+//
+// CHECK-CXX-LABEL: define dso_local noundef <4 x float> @_Z26test_vreinterpretq_f32_mf814__Mfloat8x16_t(
+// CHECK-CXX-SAME: <16 x i8> [[V:%.*]]) #[[ATTR0]] {
+// CHECK-CXX-NEXT:  [[ENTRY:.*:]]
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = bitcast <16 x i8> [[V]] to <4 x float>
+// CHECK-CXX-NEXT:    ret <4 x float> [[TMP0]]
+//
+float32x4_t test_vreinterpretq_f32_mf8(mfloat8x16_t v) {
+  return vreinterpretq_f32_mf8(v);
+}
+// CHECK-LABEL: define dso_local <8 x half> @test_vreinterpretq_f16_mf8(
+// CHECK-SAME: <16 x i8> [[V:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[TMP0:%.*]] = bitcast <16 x i8> [[V]] to <8 x half>
+// CHECK-NEXT:    ret <8 x half> [[TMP0]]
+//
+// CHECK-CXX-LABEL: define dso_local noundef <8 x half> @_Z26test_vreinterpretq_f16_mf814__Mfloat8x16_t(
+// CHECK-CXX-SAME: <16 x i8> [[V:%.*]]) #[[ATTR0]] {
+// CHECK-CXX-NEXT:  [[ENTRY:.*:]]
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = bitcast <16 x i8> [[V]] to <8 x half>
+// CHECK-CXX-NEXT:    ret <8 x half> [[TMP0]]
+//
+float16x8_t test_vreinterpretq_f16_mf8(mfloat8x16_t v) {
+  return vreinterpretq_f16_mf8(v);
+}
+// CHECK-LABEL: define dso_local <4 x i32> @test_vreinterpretq_s32_mf8(
+// CHECK-SAME: <16 x i8> [[V:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[TMP0:%.*]] = bitcast <16 x i8> [[V]] to <4 x i32>
+// CHECK-NEXT:    ret <4 x i32> [[TMP0]]
+//
+// CHECK-CXX-LABEL: define dso_local noundef <4 x i32> @_Z26test_vreinterpretq_s32_mf814__Mfloat8x16_t(
+// CHECK-CXX-SAME: <16 x i8> [[V:%.*]]) #[[ATTR0]] {
+// CHECK-CXX-NEXT:  [[ENTRY:.*:]]
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = bitcast <16 x i8> [[V]] to <4 x i32>
+// CHECK-CXX-NEXT:    ret <4 x i32> [[TMP0]]
+//
+int32x4_t test_vreinterpretq_s32_mf8(mfloat8x16_t v) {
+  return vreinterpretq_s32_mf8(v);
+}
+// CHECK-LABEL: define dso_local <2 x i64> @test_vreinterpretq_s64_mf8(
+// CHECK-SAME: <16 x i8> [[V:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[TMP0:%.*]] = bitcast <16 x i8> [[V]] to <2 x i64>
+// CHECK-NEXT:    ret <2 x i64> [[TMP0]]
+//
+// CHECK-CXX-LABEL: define dso_local noundef <2 x i64> @_Z26test_vreinterpretq_s64_mf814__Mfloat8x16_t(
+// CHECK-CXX-SAME: <16 x i8> [[V:%.*]]) #[[ATTR0]] {
+// CHECK-CXX-NEXT:  [[ENTRY:.*:]]
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = bitcast <16 x i8> [[V]] to <2 x i64>
+// CHECK-CXX-NEXT:    ret <2 x i64> [[TMP0]]
+//
+int64x2_t test_vreinterpretq_s64_mf8(mfloat8x16_t v) {
+  return vreinterpretq_s64_mf8(v);
+}
+// CHECK-LABEL: define dso_local <16 x i8> @test_vreinterpretq_mf8_p8(
+// CHECK-SAME: <16 x i8> noundef [[V:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    ret <16 x i8> [[V]]
+//
+// CHECK-CXX-LABEL: define dso_local <16 x i8> @_Z25test_vreinterpretq_mf8_p812__Poly8x16_t(
+// CHECK-CXX-SAME: <16 x i8> noundef [[V:%.*]]) #[[ATTR0]] {
+// CHECK-CXX-NEXT:  [[ENTRY:.*:]]
+// CHECK-CXX-NEXT:    ret <16 x i8> [[V]]
+//
+mfloat8x16_t test_vreinterpretq_mf8_p8(poly8x16_t v) {
+  return vreinterpretq_mf8_p8(v);
+}
+// CHECK-LABEL: define dso_local <16 x i8> @test_vreinterpretq_mf8_p128(
+// CHECK-SAME: i128 noundef [[V:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[TMP0:%.*]] = bitcast i128 [[V]] to <16 x i8>
+// CHECK-NEXT:    ret <16 x i8> [[TMP0]]
+//
+// CHECK-CXX-LABEL: define dso_local <16 x i8> @_Z27test_vreinterpretq_mf8_p128o(
+// CHECK-CXX-SAME: i128 noundef [[V:%.*]]) #[[ATTR0]] {
+// CHECK-CXX-NEXT:  [[ENTRY:.*:]]
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = bitcast i128 [[V]] to <16 x i8>
+// CHECK-CXX-NEXT:    ret <16 x i8> [[TMP0]]
+//
+mfloat8x16_t test_vreinterpretq_mf8_p128(poly128_t v) {
+  return vreinterpretq_mf8_p128(v);
+}
+// CHECK-LABEL: define dso_local <16 x i8> @test_vreinterpretq_mf8_p64(
+// CHECK-SAME: <2 x i64> noundef [[V:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[TMP0:%.*]] = bitcast <2 x i64> [[V]] to <16 x i8>
+// CHECK-NEXT:    ret <16 x i8> [[TMP0]]
+//
+// CHECK-CXX-LABEL: define dso_local <16 x i8> @_Z26test_vreinterpretq_mf8_p6412__Poly64x2_t(
+// CHECK-CXX-SAME: <2 x i64> noundef [[V:%.*]]) #[[ATTR0]] {
+// CHECK-CXX-NEXT:  [[ENTRY:.*:]]
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = bitcast <2 x i64> [[V]] to <16 x i8>
+// CHECK-CXX-NEXT:    ret <16 x i8> [[TMP0]]
+//
+mfloat8x16_t test_vreinterpretq_mf8_p64(poly64x2_t v) {
+  return vreinterpretq_mf8_p64(v);
+}
+// CHECK-LABEL: define dso_local <16 x i8> @test_vreinterpretq_mf8_p16(
+// CHECK-SAME: <8 x i16> noundef [[V:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[TMP0:%.*]] = bitcast <8 x i16> [[V]] to <16 x i8>
+// CHECK-NEXT:    ret <16 x i8> [[TMP0]]
+//
+// CHECK-CXX-LABEL: define dso_local <16 x i8> @_Z26test_vreinterpretq_mf8_p1612__Poly16x8_t(
+// CHECK-CXX-SAME: <8 x i16> noundef [[V:%.*]]) #[[ATTR0]] {
+// CHECK-CXX-NEXT:  [[ENTRY:.*:]]
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = bitcast <8 x i16> [[V]] to <16 x i8>
+// CHECK-CXX-NEXT:    ret <16 x i8> [[TMP0]]
+//
+mfloat8x16_t test_vreinterpretq_mf8_p16(poly16x8_t v) {
+  return vreinterpretq_mf8_p16(v);
+}
+// CHECK-LABEL: define dso_local <16 x i8> @test_vreinterpretq_mf8_u8(
+// CHECK-SAME: <16 x i8> noundef [[V:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    ret <16 x i8> [[V]]
+//
+// CHECK-CXX-LABEL: define dso_local <16 x i8> @_Z25test_vreinterpretq_mf8_u812__Uint8x16_t(
+// CHECK-CXX-SAME: <16 x i8> noundef [[V:%.*]]) #[[ATTR0]] {
+// CHECK-CXX-NEXT:  [[ENTRY:.*:]]
+// CHECK-CXX-NEXT:    ret <16 x i8> [[V]]
+//
+mfloat8x16_t test_vreinterpretq_mf8_u8(uint8x16_t v) {
+  return vreinterpretq_mf8_u8(v);
+}
+// CHECK-LABEL: define dso_local <16 x i8> @test_vreinterpretq_mf8_u32(
+// CHECK-SAME: <4 x i32> noundef [[V:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[TMP0:%.*]] = bitcast <4 x i32> [[V]] to <16 x i8>
+// CHECK-NEXT:    ret <16 x i8> [[TMP0]]
+//
+// CHECK-CXX-LABEL: define dso_local <16 x i8> @_Z26test_vreinterpretq_mf8_u3212__Uint32x4_t(
+// CHECK-CXX-SAME: <4 x i32> noundef [[V:%.*]]) #[[ATTR0]] {
+// CHECK-CXX-NEXT:  [[ENTRY:.*:]]
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = bitcast <4 x i32> [[V]] to <16 x i8>
+// CHECK-CXX-NEXT:    ret <16 x i8> [[TMP0]]
+//
+mfloat8x16_t test_vreinterpretq_mf8_u32(uint32x4_t v) {
+  return vreinterpretq_mf8_u32(v);
+}
+// CHECK-LABEL: define dso_local <16 x i8> @test_vreinterpretq_mf8_u64(
+// CHECK-SAME: <2 x i64> noundef [[V:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[TMP0:%.*]] = bitcast <2 x i64> [[V]] to <16 x i8>
+// CHECK-NEXT:    ret <16 x i8> [[TMP0]]
+//
+// CHECK-CXX-LABEL: define dso_local <16 x i8> @_Z26test_vreinterpretq_mf8_u6412__Uint64x2_t(
+// CHECK-CXX-SAME: <2 x i64> noundef [[V:%.*]]) #[[ATTR0]] {
+// CHECK-CXX-NEXT:  [[ENTRY:.*:]]
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = bitcast <2 x i64> [[V]] to <16 x i8>
+// CHECK-CXX-NEXT:    ret <16 x i8> [[TMP0]]
+//
+mfloat8x16_t test_vreinterpretq_mf8_u64(uint64x2_t v) {
+  return vreinterpretq_mf8_u64(v);
+}
+// CHECK-LABEL: define dso_local <16 x i8> @test_vreinterpretq_mf8_u16(
+// CHECK-SAME: <8 x i16> noundef [[V:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[TMP0:%.*]] = bitcast <8 x i16> [[V]] to <16 x i8>
+// CHECK-NEXT:    ret <16 x i8> [[TMP0]]
+//
+// CHECK-CXX-LABEL: define dso_local <16 x i8> @_Z26test_vreinterpretq_mf8_u1612__Uint16x8_t(
+// CHECK-CXX-SAME: <8 x i16> noundef [[V:%.*]]) #[[ATTR0]] {
+// CHECK-CXX-NEXT:  [[ENTRY:.*:]]
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = bitcast <8 x i16> [[V]] to <16 x i8>
+// CHECK-CXX-NEXT:    ret <16 x i8> [[TMP0]]
+//
+mfloat8x16_t test_vreinterpretq_mf8_u16(uint16x8_t v) {
+  return vreinterpretq_mf8_u16(v);
+}
+// CHECK-LABEL: define dso_local <16 x i8> @test_vreinterpretq_mf8_s8(
+// CHECK-SAME: <16 x i8> noundef [[V:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    ret <16 x i8> [[V]]
+//
+// CHECK-CXX-LABEL: define dso_local <16 x i8> @_Z25test_vreinterpretq_mf8_s811__Int8x16_t(
+// CHECK-CXX-SAME: <16 x i8> noundef [[V:%.*]]) #[[ATTR0]] {
+// CHECK-CXX-NEXT:  [[ENTRY:.*:]]
+// CHECK-CXX-NEXT:    ret <16 x i8> [[V]]
+//
+mfloat8x16_t test_vreinterpretq_mf8_s8(int8x16_t v) {
+  return vreinterpretq_mf8_s8(v);
+}
+// CHECK-LABEL: define dso_local <16 x i8> @test_vreinterpretq_mf8_f64(
+// CHECK-SAME: <2 x double> noundef [[V:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[TMP0:%.*]] = bitcast <2 x double> [[V]] to <16 x i8>
+// CHECK-NEXT:    ret <16 x i8> [[TMP0]]
+//
+// CHECK-CXX-LABEL: define dso_local <16 x i8> @_Z26test_vreinterpretq_mf8_f6413__Float64x2_t(
+// CHECK-CXX-SAME: <2 x double> noundef [[V:%.*]]) #[[ATTR0]] {
+// CHECK-CXX-NEXT:  [[ENTRY:.*:]]
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = bitcast <2 x double> [[V]] to <16 x i8>
+// CHECK-CXX-NEXT:    ret <16 x i8> [[TMP0]]
+//
+mfloat8x16_t test_vreinterpretq_mf8_f64(float64x2_t v) {
+  return vreinterpretq_mf8_f64(v);
+}
+// CHECK-LABEL: define dso_local <16 x i8> @test_vreinterpretq_mf8_f32(
+// CHECK-SAME: <4 x float> noundef [[V:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[TMP0:%.*]] = bitcast <4 x float> [[V]] to <16 x i8>
+// CHECK-NEXT:    ret <16 x i8> [[TMP0]]
+//
+// CHECK-CXX-LABEL: define dso_local <16 x i8> @_Z26test_vreinterpretq_mf8_f3213__Float32x4_t(
+// CHECK-CXX-SAME: <4 x float> noundef [[V:%.*]]) #[[ATTR0]] {
+// CHECK-CXX-NEXT:  [[ENTRY:.*:]]
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = bitcast <4 x float> [[V]] to <16 x i8>
+// CHECK-CXX-NEXT:    ret <16 x i8> [[TMP0]]
+//
+mfloat8x16_t test_vreinterpretq_mf8_f32(float32x4_t v) {
+  return vreinterpretq_mf8_f32(v);
+}
+// CHECK-LABEL: define dso_local <16 x i8> @test_vreinterpretq_mf8_f16(
+// CHECK-SAME: <8 x half> noundef [[V:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[TMP0:%.*]] = bitcast <8 x half> [[V]] to <16 x i8>
+// CHECK-NEXT:    ret <16 x i8> [[TMP0]]
+//
+// CHECK-CXX-LABEL: define dso_local <16 x i8> @_Z26test_vreinterpretq_mf8_f1613__Float16x8_t(
+// CHECK-CXX-SAME: <8 x half> noundef [[V:%.*]]) #[[ATTR0]] {
+// CHECK-CXX-NEXT:  [[ENTRY:.*:]]
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = bitcast <8 x half> [[V]] to <16 x i8>
+// CHECK-CXX-NEXT:    ret <16 x i8> [[TMP0]]
+//
+mfloat8x16_t test_vreinterpretq_mf8_f16(float16x8_t v) {
+  return vreinterpretq_mf8_f16(v);
+}
+// CHECK-LABEL: define dso_local <16 x i8> @test_vreinterpretq_mf8_s32(
+// CHECK-SAME: <4 x i32> noundef [[V:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[TMP0:%.*]] = bitcast <4 x i32> [[V]] to <16 x i8>
+// CHECK-NEXT:    ret <16 x i8> [[TMP0]]
+//
+// CHECK-CXX-LABEL: define dso_local <16 x i8> @_Z26test_vreinterpretq_mf8_s3211__Int32x4_t(
+// CHECK-CXX-SAME: <4 x i32> noundef [[V:%.*]]) #[[ATTR0]] {
+// CHECK-CXX-NEXT:  [[ENTRY:.*:]]
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = bitcast <4 x i32> [[V]] to <16 x i8>
+// CHECK-CXX-NEXT:    ret <16 x i8> [[TMP0]]
+//
+mfloat8x16_t test_vreinterpretq_mf8_s32(int32x4_t v) {
+  return vreinterpretq_mf8_s32(v);
+}
+// CHECK-LABEL: define dso_local <16 x i8> @test_vreinterpretq_mf8_s64(
+// CHECK-SAME: <2 x i64> noundef [[V:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[TMP0:%.*]] = bitcast <2 x i64> [[V]] to <16 x i8>
+// CHECK-NEXT:    ret <16 x i8> [[TMP0]]
+//
+// CHECK-CXX-LABEL: define dso_local <16 x i8> @_Z26test_vreinterpretq_mf8_s6411__Int64x2_t(
+// CHECK-CXX-SAME: <2 x i64> noundef [[V:%.*]]) #[[ATTR0]] {
+// CHECK-CXX-NEXT:  [[ENTRY:.*:]]
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = bitcast <2 x i64> [[V]] to <16 x i8>
+// CHECK-CXX-NEXT:    ret <16 x i8> [[TMP0]]
+//
+mfloat8x16_t test_vreinterpretq_mf8_s64(int64x2_t v) {
+  return vreinterpretq_mf8_s64(v);
+}
+// CHECK-LABEL: define dso_local <16 x i8> @test_vreinterpretq_mf8_s16(
+// CHECK-SAME: <8 x i16> noundef [[V:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[TMP0:%.*]] = bitcast <8 x i16> [[V]] to <16 x i8>
+// CHECK-NEXT:    ret <16 x i8> [[TMP0]]
+//
+// CHECK-CXX-LABEL: define dso_local <16 x i8> @_Z26test_vreinterpretq_mf8_s1611__Int16x8_t(
+// CHECK-CXX-SAME: <8 x i16> noundef [[V:%.*]]) #[[ATTR0]] {
+// CHECK-CXX-NEXT:  [[ENTRY:.*:]]
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = bitcast <8 x i16> [[V]] to <16 x i8>
+// CHECK-CXX-NEXT:    ret <16 x i8> [[TMP0]]
+//
+mfloat8x16_t test_vreinterpretq_mf8_s16(int16x8_t v) {
+  return vreinterpretq_mf8_s16(v);
+}
+// CHECK-LABEL: define dso_local <8 x i16> @test_vreinterpretq_s16_mf8(
+// CHECK-SAME: <16 x i8> [[V:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[TMP0:%.*]] = bitcast <16 x i8> [[V]] to <8 x i16>
+// CHECK-NEXT:    ret <8 x i16> [[TMP0]]
+//
+// CHECK-CXX-LABEL: define dso_local noundef <8 x i16> @_Z26test_vreinterpretq_s16_mf814__Mfloat8x16_t(
+// CHECK-CXX-SAME: <16 x i8> [[V:%.*]]) #[[ATTR0]] {
+// CHECK-CXX-NEXT:  [[ENTRY:.*:]]
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = bitcast <16 x i8> [[V]] to <8 x i16>
+// CHECK-CXX-NEXT:    ret <8 x i16> [[TMP0]]
+//
+int16x8_t test_vreinterpretq_s16_mf8(mfloat8x16_t v) {
+  return vreinterpretq_s16_mf8(v);
+}
+// CHECK-LABEL: define dso_local <8 x i8> @test_vreinterpret_u8_mf8(
+// CHECK-SAME: <8 x i8> [[V:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    ret <8 x i8> [[V]]
+//
+// CHECK-CXX-LABEL: define dso_local noundef <8 x i8> @_Z24test_vreinterpret_u8_mf813__Mfloat8x8_t(
+// CHECK-CXX-SAME: <8 x i8> [[V:%.*]]) #[[ATTR0]] {
+// CHECK-CXX-NEXT:  [[ENTRY:.*:]]
+// CHECK-CXX-NEXT:    ret <8 x i8> [[V]]
+//
+uint8x8_t test_vreinterpret_u8_mf8(mfloat8x8_t v) {
+  return vreinterpret_u8_mf8(v);
+}
+// CHECK-LABEL: define dso_local <2 x i32> @test_vreinterpret_u32_mf8(
+// CHECK-SAME: <8 x i8> [[V:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[TMP0:%.*]] = bitcast <8 x i8> [[V]] to <2 x i32>
+// CHECK-NEXT:    ret <2 x i32> [[TMP0]]
+//
+// CHECK-CXX-LABEL: define dso_local noundef <2 x i32> @_Z25test_vreinterpret_u32_mf813__Mfloat8x8_t(
+// CHECK-CXX-SAME: <8 x i8> [[V:%.*]]) #[[ATTR0]] {
+// CHECK-CXX-NEXT:  [[ENTRY:.*:]]
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = bitcast <8 x i8> [[V]] to <2 x i32>
+// CHECK-CXX-NEXT:    ret <2 x i32> [[TMP0]]
+//
+uint32x2_t test_vreinterpret_u32_mf8(mfloat8x8_t v) {
+  return vreinterpret_u32_mf8(v);
+}
+// CHECK-LABEL: define dso_local <1 x i64> @test_vreinterpret_u64_mf8(
+// CHECK-SAME: <8 x i8> [[V:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[TMP0:%.*]] = bitcast <8 x i8> [[V]] to <1 x i64>
+// CHECK-NEXT:    ret <1 x i64> [[TMP0]]
+//
+// CHECK-CXX-LABEL: define dso_local noundef <1 x i64> @_Z25test_vreinterpret_u64_mf813__Mfloat8x8_t(
+// CHECK-CXX-SAME: <8 x i8> [[V:%.*]]) #[[ATTR0]] {
+// CHECK-CXX-NEXT:  [[ENTRY:.*:]]
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = bitcast <8 x i8> [[V]] to <1 x i64>
+// CHECK-CXX-NEXT:    ret <1 x i64> [[TMP0]]
+//
+uint64x1_t test_vreinterpret_u64_mf8(mfloat8x8_t v) {
+  return vreinterpret_u64_mf8(v);
+}
+// CHECK-LABEL: define dso_local <4 x i16> @test_vreinterpret_u16_mf8(
+// CHECK-SAME: <8 x i8> [[V:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[TMP0:%.*]] = bitcast <8 x i8> [[V]] to <4 x i16>
+// CHECK-NEXT:    ret <4 x i16> [[TMP0]]
+//
+// CHECK-CXX-LABEL: define dso_local noundef <4 x i16> @_Z25test_vreinterpret_u16_mf813__Mfloat8x8_t(
+// CHECK-CXX-SAME: <8 x i8> [[V:%.*]]) #[[ATTR0]] {
+// CHECK-CXX-NEXT:  [[ENTRY:.*:]]
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = bitcast <8 x i8> [[V]] to <4 x i16>
+// CHECK-CXX-NEXT:    ret <4 x i16> [[TMP0]]
+//
+uint16x4_t test_vreinterpret_u16_mf8(mfloat8x8_t v) {
+  return vreinterpret_u16_mf8(v);
+}
+// CHECK-LABEL: define dso_local <8 x i8> @test_vreinterpret_s8_mf8(
+// CHECK-SAME: <8 x i8> [[V:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    ret <8 x i8> [[V]]
+//
+// CHECK-CXX-LABEL: define dso_local noundef <8 x i8> @_Z24test_vreinterpret_s8_mf813__Mfloat8x8_t(
+// CHECK-CXX-SAME: <8 x i8> [[V:%.*]]) #[[ATTR0]] {
+// CHECK-CXX-NEXT:  [[ENTRY:.*:]]
+// CHECK-CXX-NEXT:    ret <8 x i8> [[V]]
+//
+int8x8_t test_vreinterpret_s8_mf8(mfloat8x8_t v) {
+  return vreinterpret_s8_mf8(v);
+}
+// CHECK-LABEL: define dso_local <1 x double> @test_vreinterpret_f64_mf8(
+// CHECK-SAME: <8 x i8> [[V:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[TMP0:%.*]] = bitcast <8 x i8> [[V]] to <1 x double>
+// CHECK-NEXT:    ret <1 x double> [[TMP0]]
+//
+// CHECK-CXX-LABEL: define dso_local noundef <1 x double> @_Z25test_vreinterpret_f64_mf813__Mfloat8x8_t(
+// CHECK-CXX-SAME: <8 x i8> [[V:%.*]]) #[[ATTR0]] {
+// CHECK-CXX-NEXT:  [[ENTRY:.*:]]
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = bitcast <8 x i8> [[V]] to <1 x double>
+// CHECK-CXX-NEXT:    ret <1 x double> [[TMP0]]
+//
+float64x1_t test_vreinterpret_f64_mf8(mfloat8x8_t v) {
+  return vreinterpret_f64_mf8(v);
+}
+// CHECK-LABEL: define dso_local <2 x float> @test_vreinterpret_f32_mf8(
+// CHECK-SAME: <8 x i8> [[V:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[TMP0:%.*]] = bitcast <8 x i8> [[V]] to <2 x float>
+// CHECK-NEXT:    ret <2 x float> [[TMP0]]
+//
+// CHECK-CXX-LABEL: define dso_local noundef <2 x float> @_Z25test_vreinterpret_f32_mf813__Mfloat8x8_t(
+// CHECK-CXX-SAME: <8 x i8> [[V:%.*]]) #[[ATTR0]] {
+// CHECK-CXX-NEXT:  [[ENTRY:.*:]]
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = bitcast <8 x i8> [[V]] to <2 x float>
+// CHECK-CXX-NEXT:    ret <2 x float> [[TMP0]]
+//
+float32x2_t test_vreinterpret_f32_mf8(mfloat8x8_t v) {
+  return vreinterpret_f32_mf8(v);
+}
+// CHECK-LABEL: define dso_local <4 x half> @test_vreinterpret_f16_mf8(
+// CHECK-SAME: <8 x i8> [[V:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[TMP0:%.*]] = bitcast <8 x i8> [[V]] to <4 x half>
+// CHECK-NEXT:    ret <4 x half> [[TMP0]]
+//
+// CHECK-CXX-LABEL: define dso_local noundef <4 x half> @_Z25test_vreinterpret_f16_mf813__Mfloat8x8_t(
+// CHECK-CXX-SAME: <8 x i8> [[V:%.*]]) #[[ATTR0]] {
+// CHECK-CXX-NEXT:  [[ENTRY:.*:]]
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = bitcast <8 x i8> [[V]] to <4 x half>
+// CHECK-CXX-NEXT:    ret <4 x half> [[TMP0]]
+//
+float16x4_t test_vreinterpret_f16_mf8(mfloat8x8_t v) {
+  return vreinterpret_f16_mf8(v);
+}
+// CHECK-LABEL: define dso_local <2 x i32> @test_vreinterpret_s32_mf8(
+// CHECK-SAME: <8 x i8> [[V:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[TMP0:%.*]] = bitcast <8 x i8> [[V]] to <2 x i32>
+// CHECK-NEXT:    ret <2 x i32> [[TMP0]]
+//
+// CHECK-CXX-LABEL: define dso_local noundef <2 x i32> @_Z25test_vreinterpret_s32_mf813__Mfloat8x8_t(
+// CHECK-CXX-SAME: <8 x i8> [[V:%.*]]) #[[ATTR0]] {
+// CHECK-CXX-NEXT:  [[ENTRY:.*:]]
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = bitcast <8 x i8> [[V]] to <2 x i32>
+// CHECK-CXX-NEXT:    ret <2 x i32> [[TMP0]]
+//
+int32x2_t test_vreinterpret_s32_mf8(mfloat8x8_t v) {
+  return vreinterpret_s32_mf8(v);
+}
+// CHECK-LABEL: define dso_local <1 x i64> @test_vreinterpret_s64_mf8(
+// CHECK-SAME: <8 x i8> [[V:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[TMP0:%.*]] = bitcast <8 x i8> [[V]] to <1 x i64>
+// CHECK-NEXT:    ret <1 x i64> [[TMP0]]
+//
+// CHECK-CXX-LABEL: define dso_local noundef <1 x i64> @_Z25test_vreinterpret_s64_mf813__Mfloat8x8_t(
+// CHECK-CXX-SAME: <8 x i8> [[V:%.*]]) #[[ATTR0]] {
+// CHECK-CXX-NEXT:  [[ENTRY:.*:]]
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = bitcast <8 x i8> [[V]] to <1 x i64>
+// CHECK-CXX-NEXT:    ret <1 x i64> [[TMP0]]
+//
+int64x1_t test_vreinterpret_s64_mf8(mfloat8x8_t v) {
+  return vreinterpret_s64_mf8(v);
+}
+// CHECK-LABEL: define dso_local <8 x i8> @test_vreinterpret_mf8_p8(
+// CHECK-SAME: <8 x i8> noundef [[V:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    ret <8 x i8> [[V]]
+//
+// CHECK-CXX-LABEL: define dso_local <8 x i8> @_Z24test_vreinterpret_mf8_p811__Poly8x8_t(
+// CHECK-CXX-SAME: <8 x i8> noundef [[V:%.*]]) #[[ATTR0]] {
+// CHECK-CXX-NEXT:  [[ENTRY:.*:]]
+// CHECK-CXX-NEXT:    ret <8 x i8> [[V]]
+//
+mfloat8x8_t test_vreinterpret_mf8_p8(poly8x8_t v) {
+  return vreinterpret_mf8_p8(v);
+}
+// CHECK-LABEL: define dso_local <8 x i8> @test_vreinterpret_mf8_p64(
+// CHECK-SAME: <1 x i64> noundef [[V:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[TMP0:%.*]] = bitcast <1 x i64> [[V]] to <8 x i8>
+// CHECK-NEXT:    ret <8 x i8> [[TMP0]]
+//
+// CHECK-CXX-LABEL: define dso_local <8 x i8> @_Z25test_vreinterpret_mf8_p6412__Poly64x1_t(
+// CHECK-CXX-SAME: <1 x i64> noundef [[V:%.*]]) #[[ATTR0]] {
+// CHECK-CXX-NEXT:  [[ENTRY:.*:]]
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = bitcast <1 x i64> [[V]] to <8 x i8>
+// CHECK-CXX-NEXT:    ret <8 x i8> [[TMP0]]
+//
+mfloat8x8_t test_vreinterpret_mf8_p64(poly64x1_t v) {
+  return vreinterpret_mf8_p64(v);
+}
+// CHECK-LABEL: define dso_local <8 x i8> @test_vreinterpret_mf8_p16(
+// CHECK-SAME: <4 x i16> noundef [[V:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[TMP0:%.*]] = bitcast <4 x i16> [[V]] to <8 x i8>
+// CHECK-NEXT:    ret <8 x i8> [[TMP0]]
+//
+// CHECK-CXX-LABEL: define dso_local <8 x i8> @_Z25test_vreinterpret_mf8_p1612__Poly16x4_t(
+// CHECK-CXX-SAME: <4 x i16> noundef [[V:%.*]]) #[[ATTR0]] {
+// CHECK-CXX-NEXT:  [[ENTRY:.*:]]
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = bitcast <4 x i16> [[V]] to <8 x i8>
+// CHECK-CXX-NEXT:    ret <8 x i8> [[TMP0]]
+//
+mfloat8x8_t test_vreinterpret_mf8_p16(poly16x4_t v) {
+  return vreinterpret_mf8_p16(v);
+}
+// CHECK-LABEL: define dso_local <8 x i8> @test_vreinterpret_mf8_u8(
+// CHECK-SAME: <8 x i8> noundef [[V:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    ret <8 x i8> [[V]]
+//
+// CHECK-CXX-LABEL: define dso_local <8 x i8> @_Z24test_vreinterpret_mf8_u811__Uint8x8_t(
+// CHECK-CXX-SAME: <8 x i8> noundef [[V:%.*]]) #[[ATTR0]] {
+// CHECK-CXX-NEXT:  [[ENTRY:.*:]]
+// CHECK-CXX-NEXT:    ret <8 x i8> [[V]]
+//
+mfloat8x8_t test_vreinterpret_mf8_u8(uint8x8_t v) {
+  return vreinterpret_mf8_u8(v);
+}
+// CHECK-LABEL: define dso_local <8 x i8> @test_vreinterpret_mf8_u32(
+// CHECK-SAME: <2 x i32> noundef [[V:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[TMP0:%.*]] = bitcast <2 x i32> [[V]] to <8 x i8>
+// CHECK-NEXT:    ret <8 x i8> [[TMP0]]
+//
+// CHECK-CXX-LABEL: define dso_local <8 x i8> @_Z25test_vreinterpret_mf8_u3212__Uint32x2_t(
+// CHECK-CXX-SAME: <2 x i32> noundef [[V:%.*]]) #[[ATTR0]] {
+// CHECK-CXX-NEXT:  [[ENTRY:.*:]]
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = bitcast <2 x i32> [[V]] to <8 x i8>
+// CHECK-CXX-NEXT:    ret <8 x i8> [[TMP0]]
+//
+mfloat8x8_t test_vreinterpret_mf8_u32(uint32x2_t v) {
+  return vreinterpret_mf8_u32(v);
+}
+// CHECK-LABEL: define dso_local <8 x i8> @test_vreinterpret_mf8_u64(
+// CHECK-SAME: <1 x i64> noundef [[V:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[TMP0:%.*]] = bitcast <1 x i64> [[V]] to <8 x i8>
+// CHECK-NEXT:    ret <8 x i8> [[TMP0]]
+//
+// CHECK-CXX-LABEL: define dso_local <8 x i8> @_Z25test_vreinterpret_mf8_u6412__Uint64x1_t(
+// CHECK-CXX-SAME: <1 x i64> noundef [[V:%.*]]) #[[ATTR0]] {
+// CHECK-CXX-NEXT:  [[ENTRY:.*:]]
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = bitcast <1 x i64> [[V]] to <8 x i8>
+// CHECK-CXX-NEXT:    ret <8 x i8> [[TMP0]]
+//
+mfloat8x8_t test_vreinterpret_mf8_u64(uint64x1_t v) {
+  return vreinterpret_mf8_u64(v);
+}
+// CHECK-LABEL: define dso_local <8 x i8> @test_vreinterpret_mf8_u16(
+// CHECK-SAME: <4 x i16> noundef [[V:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[TMP0:%.*]] = bitcast <4 x i16> [[V]] to <8 x i8>
+// CHECK-NEXT:    ret <8 x i8> [[TMP0]]
+//
+// CHECK-CXX-LABEL: define dso_local <8 x i8> @_Z25test_vreinterpret_mf8_u1612__Uint16x4_t(
+// CHECK-CXX-SAME: <4 x i16> noundef [[V:%.*]]) #[[ATTR0]] {
+// CHECK-CXX-NEXT:  [[ENTRY:.*:]]
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = bitcast <4 x i16> [[V]] to <8 x i8>
+// CHECK-CXX-NEXT:    ret <8 x i8> [[TMP0]]
+//
+mfloat8x8_t test_vreinterpret_mf8_u16(uint16x4_t v) {
+  return vreinterpret_mf8_u16(v);
+}
+// CHECK-LABEL: define dso_local <8 x i8> @test_vreinterpret_mf8_s8(
+// CHECK-SAME: <8 x i8> noundef [[V:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    ret <8 x i8> [[V]]
+//
+// CHECK-CXX-LABEL: define dso_local <8 x i8> @_Z24test_vreinterpret_mf8_s810__Int8x8_t(
+// CHECK-CXX-SAME: <8 x i8> noundef [[V:%.*]]) #[[ATTR0]] {
+// CHECK-CXX-NEXT:  [[ENTRY:.*:]]
+// CHECK-CXX-NEXT:    ret <8 x i8> [[V]]
+//
+mfloat8x8_t test_vreinterpret_mf8_s8(int8x8_t v) {
+  return vreinterpret_mf8_s8(v);
+}
+// CHECK-LABEL: define dso_local <8 x i8> @test_vreinterpret_mf8_f64(
+// CHECK-SAME: <1 x double> noundef [[V:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[TMP0:%.*]] = bitcast <1 x double> [[V]] to <8 x i8>
+// CHECK-NEXT:    ret <8 x i8> [[TMP0]]
+//
+// CHECK-CXX-LABEL: define dso_local <8 x i8> @_Z25test_vreinterpret_mf8_f6413__Float64x1_t(
+// CHECK-CXX-SAME: <1 x double> noundef [[V:%.*]]) #[[ATTR0]] {
+// CHECK-CXX-NEXT:  [[ENTRY:.*:]]
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = bitcast <1 x double> [[V]] to <8 x i8>
+// CHECK-CXX-NEXT:    ret <8 x i8> [[TMP0]]
+//
+mfloat8x8_t test_vreinterpret_mf8_f64(float64x1_t v) {
+  return vreinterpret_mf8_f64(v);
+}
+// CHECK-LABEL: define dso_local <8 x i8> @test_vreinterpret_mf8_f32(
+// CHECK-SAME: <2 x float> noundef [[V:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[TMP0:%.*]] = bitcast <2 x float> [[V]] to <8 x i8>
+// CHECK-NEXT:    ret <8 x i8> [[TMP0]]
+//
+// CHECK-CXX-LABEL: define dso_local <8 x i8> @_Z25test_vreinterpret_mf8_f3213__Float32x2_t(
+// CHECK-CXX-SAME: <2 x float> noundef [[V:%.*]]) #[[ATTR0]] {
+// CHECK-CXX-NEXT:  [[ENTRY:.*:]]
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = bitcast <2 x float> [[V]] to <8 x i8>
+// CHECK-CXX-NEXT:    ret <8 x i8> [[TMP0]]
+//
+mfloat8x8_t test_vreinterpret_mf8_f32(float32x2_t v) {
+  return vreinterpret_mf8_f32(v);
+}
+// CHECK-LABEL: define dso_local <8 x i8> @test_vreinterpret_mf8_f16(
+// CHECK-SAME: <4 x half> noundef [[V:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[TMP0:%.*]] = bitcast <4 x half> [[V]] to <8 x i8>
+// CHECK-NEXT:    ret <8 x i8> [[TMP0]]
+//
+// CHECK-CXX-LABEL: define dso_local <8 x i8> @_Z25test_vreinterpret_mf8_f1613__Float16x4_t(
+// CHECK-CXX-SAME: <4 x half> noundef [[V:%.*]]) #[[ATTR0]] {
+// CHECK-CXX-NEXT:  [[ENTRY:.*:]]
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = bitcast <4 x half> [[V]] to <8 x i8>
+// CHECK-CXX-NEXT:    ret <8 x i8> [[TMP0]]
+//
+mfloat8x8_t test_vreinterpret_mf8_f16(float16x4_t v) {
+  return vreinterpret_mf8_f16(v);
+}
+// CHECK-LABEL: define dso_local <8 x i8> @test_vreinterpret_mf8_s32(
+// CHECK-SAME: <2 x i32> noundef [[V:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[TMP0:%.*]] = bitcast <2 x i32> [[V]] to <8 x i8>
+// CHECK-NEXT:    ret <8 x i8> [[TMP0]]
+//
+// CHECK-CXX-LABEL: define dso_local <8 x i8> @_Z25test_vreinterpret_mf8_s3211__Int32x2_t(
+// CHECK-CXX-SAME: <2 x i32> noundef [[V:%.*]]) #[[ATTR0]] {
+// CHECK-CXX-NEXT:  [[ENTRY:.*:]]
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = bitcast <2 x i32> [[V]] to <8 x i8>
+// CHECK-CXX-NEXT:    ret <8 x i8> [[TMP0]]
+//
+mfloat8x8_t test_vreinterpret_mf8_s32(int32x2_t v) {
+  return vreinterpret_mf8_s32(v);
+}
+// CHECK-LABEL: define dso_local <8 x i8> @test_vreinterpret_mf8_s64(
+// CHECK-SAME: <1 x i64> noundef [[V:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[TMP0:%.*]] = bitcast <1 x i64> [[V]] to <8 x i8>
+// CHECK-NEXT:    ret <8 x i8> [[TMP0]]
+//
+// CHECK-CXX-LABEL: define dso_local <8 x i8> @_Z25test_vreinterpret_mf8_s6411__Int64x1_t(
+// CHECK-CXX-SAME: <1 x i64> noundef [[V:%.*]]) #[[ATTR0]] {
+// CHECK-CXX-NEXT:  [[ENTRY:.*:]]
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = bitcast <1 x i64> [[V]] to <8 x i8>
+// CHECK-CXX-NEXT:    ret <8 x i8> [[TMP0]]
+//
+mfloat8x8_t test_vreinterpret_mf8_s64(int64x1_t v) {
+  return vreinterpret_mf8_s64(v);
+}
+// CHECK-LABEL: define dso_local <8 x i8> @test_vreinterpret_mf8_s16(
+// CHECK-SAME: <4 x i16> noundef [[V:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[TMP0:%.*]] = bitcast <4 x i16> [[V]] to <8 x i8>
+// CHECK-NEXT:    ret <8 x i8> [[TMP0]]
+//
+// CHECK-CXX-LABEL: define dso_local <8 x i8> @_Z25test_vreinterpret_mf8_s1611__Int16x4_t(
+// CHECK-CXX-SAME: <4 x i16> noundef [[V:%.*]]) #[[ATTR0]] {
+// CHECK-CXX-NEXT:  [[ENTRY:.*:]]
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = bitcast <4 x i16> [[V]] to <8 x i8>
+// CHECK-CXX-NEXT:    ret <8 x i8> [[TMP0]]
+//
+mfloat8x8_t test_vreinterpret_mf8_s16(int16x4_t v) {
+  return vreinterpret_mf8_s16(v);
+}
+// CHECK-LABEL: define dso_local <4 x i16> @test_vreinterpret_s16_mf8(
+// CHECK-SAME: <8 x i8> [[V:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[TMP0:%.*]] = bitcast <8 x i8> [[V]] to <4 x i16>
+// CHECK-NEXT:    ret <4 x i16> [[TMP0]]
+//
+// CHECK-CXX-LABEL: define dso_local noundef <4 x i16> @_Z25test_vreinterpret_s16_mf813__Mfloat8x8_t(
+// CHECK-CXX-SAME: <8 x i8> [[V:%.*]]) #[[ATTR0]] {
+// CHECK-CXX-NEXT:  [[ENTRY:.*:]]
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = bitcast <8 x i8> [[V]] to <4 x i16>
+// CHECK-CXX-NEXT:    ret <4 x i16> [[TMP0]]
+//
+int16x4_t test_vreinterpret_s16_mf8(mfloat8x8_t v) {
+  return vreinterpret_s16_mf8(v);
+}
diff --git a/clang/test/CodeGen/AArch64/pure-scalable-args.c b/clang/test/CodeGen/AArch64/pure-scalable-args.c
index b03011e70b6a6..fecd370d09be3 100644
--- a/clang/test/CodeGen/AArch64/pure-scalable-args.c
+++ b/clang/test/CodeGen/AArch64/pure-scalable-args.c
@@ -67,7 +67,7 @@ void test_argpass_simple(PST *p) {
     void argpass_simple_callee(PST);
     argpass_simple_callee(*p);
 }
-// CHECK-AAPCS:      define dso_local void @test_argpass_simple(ptr nocapture noundef readonly %p)
+// CHECK-AAPCS:      define dso_local void @test_argpass_simple(ptr noundef readonly captures(none) %p)
 // CHECK-AAPCS-NEXT: entry:
 // CHECK-AAPCS-NEXT: %0 = load <2 x i8>, ptr %p, align 16
 // CHECK-AAPCS-NEXT: %cast.scalable = tail call <vscale x 2 x i8> @llvm.vector.insert.nxv2i8.v2i8(<vscale x 2 x i8> poison, <2 x i8> %0, i64 0)
@@ -292,7 +292,7 @@ PST test_return(PST *p) {
     return *p;
 }
 // CHECK-AAPCS:  define dso_local <{ <vscale x 16 x i1>, <vscale x 2 x double>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 16 x i8>, <vscale x 16 x i1> }> @test_return(ptr
-// CHECK-DARWIN: define void @test_return(ptr dead_on_unwind noalias nocapture writable writeonly sret(%struct.PST) align 16 initializes((0, 96)) %agg.result, ptr nocapture noundef readonly %p)
+// CHECK-DARWIN: define void @test_return(ptr dead_on_unwind noalias writable writeonly sret(%struct.PST) align 16 captures(none) initializes((0, 96)) %agg.result, ptr noundef readonly captures(none) %p)
 
 // Corner case of 1-element aggregate
 //   p->x -> q0
@@ -300,7 +300,7 @@ SmallPST test_return_small_pst(SmallPST *p) {
     return *p;
 }
 // CHECK-AAPCS:  define dso_local <vscale x 4 x float> @test_return_small_pst(ptr
-// CHECK-DARWIN: define i128 @test_return_small_pst(ptr nocapture noundef readonly %p)
+// CHECK-DARWIN: define i128 @test_return_small_pst(ptr noundef readonly captures(none) %p)
 
 
 // Big PST, returned indirectly
@@ -308,8 +308,8 @@ SmallPST test_return_small_pst(SmallPST *p) {
 BigPST test_return_big_pst(BigPST *p) {
     return *p;
 }
-// CHECK-AAPCS:  define dso_local void @test_return_big_pst(ptr dead_on_unwind noalias nocapture writable writeonly sret(%struct.BigPST) align 16 initializes((0, 176)) %agg.result, ptr nocapture noundef readonly %p)
-// CHECK-DARWIN: define void @test_return_big_pst(ptr dead_on_unwind noalias nocapture writable writeonly sret(%struct.BigPST) align 16 initializes((0, 176)) %agg.result, ptr nocapture noundef readonly %p)
+// CHECK-AAPCS:  define dso_local void @test_return_big_pst(ptr dead_on_unwind noalias writable writeonly sret(%struct.BigPST) align 16 captures(none) initializes((0, 176)) %agg.result, ptr noundef readonly captures(none) %p)
+// CHECK-DARWIN: define void @test_return_big_pst(ptr dead_on_unwind noalias writable writeonly sret(%struct.BigPST) align 16 captures(none) initializes((0, 176)) %agg.result, ptr noundef readonly captures(none) %p)
 
 // Variadic arguments are unnamed, PST passed indirectly.
 // (Passing SVE types to a variadic function currently unsupported by
diff --git a/clang/test/CodeGen/AArch64/sme-attributes-member-function-pointer.cpp b/clang/test/CodeGen/AArch64/sme-attributes-member-function-pointer.cpp
new file mode 100644
index 0000000000000..ee784c816a060
--- /dev/null
+++ b/clang/test/CodeGen/AArch64/sme-attributes-member-function-pointer.cpp
@@ -0,0 +1,37 @@
+// RUN: %clang_cc1 -triple aarch64 -target-feature +sme -target-feature +sme2 -x c++ -std=c++20  -emit-llvm -o - %s | FileCheck %s --check-prefixes=CHECK
+
+struct TestStruct;
+
+__arm_new("za", "zt0") void test(TestStruct& TS,
+  void (TestStruct::*streaming_member_ptr)() __arm_streaming,
+  void (TestStruct::*streaming_compat_member)() __arm_streaming_compatible,
+  void (TestStruct::*arm_in_member)() __arm_in("za", "zt0"),
+  void (TestStruct::*arm_inout_member)() __arm_inout("za", "zt0"),
+  void (TestStruct::*arm_preserves_member)() __arm_preserves("za", "zt0"),
+  void (TestStruct::*arm_agnostic_member)() __arm_agnostic("sme_za_state")) {
+
+  // CHECK: call void %{{.*}} [[STREAMING_MEMBER_CALL_ATTRS:#.+]]
+  (TS.*streaming_member_ptr)();
+
+  // CHECK: call void %{{.*}} [[STREAMING_COMPAT_MEMBER_CALL_ATTRS:#.+]]
+  (TS.*streaming_compat_member)();
+
+  // CHECK: call void %{{.*}} [[ARM_IN_MEMBER_CALL_ATTRS:#.+]]
+  (TS.*arm_in_member)();
+
+  // CHECK: call void %{{.*}} [[ARM_INOUT_MEMBER_CALL_ATTRS:#.+]]
+  (TS.*arm_inout_member)();
+
+  // CHECK: call void %{{.*}} [[ARM_PRESERVES_MEMBER_CALL_ATTRS:#.+]]
+  (TS.*arm_preserves_member)();
+
+  // CHECK: call void %{{.*}} [[ARM_AGNOSTIC_MEMBER_CALL_ATTRS:#.+]]
+  (TS.*arm_agnostic_member)();
+}
+
+// CHECK: attributes [[STREAMING_MEMBER_CALL_ATTRS]] = { "aarch64_pstate_sm_enabled" }
+// CHECK: attributes [[STREAMING_COMPAT_MEMBER_CALL_ATTRS]] = { "aarch64_pstate_sm_compatible" }
+// CHECK: attributes [[ARM_IN_MEMBER_CALL_ATTRS]] = { "aarch64_in_za" "aarch64_in_zt0" }
+// CHECK: attributes [[ARM_INOUT_MEMBER_CALL_ATTRS]] = { "aarch64_inout_za" "aarch64_inout_zt0" }
+// CHECK: attributes [[ARM_PRESERVES_MEMBER_CALL_ATTRS]] = { "aarch64_preserves_za" "aarch64_preserves_zt0" }
+// CHECK: attributes [[ARM_AGNOSTIC_MEMBER_CALL_ATTRS]] = { "aarch64_za_state_agnostic" }
diff --git a/clang/test/CodeGen/AArch64/sme-intrinsics/aarch64-sme-attrs.cpp b/clang/test/CodeGen/AArch64/sme-intrinsics/aarch64-sme-attrs.cpp
index 54762c8b41412..c734c6953e5d1 100644
--- a/clang/test/CodeGen/AArch64/sme-intrinsics/aarch64-sme-attrs.cpp
+++ b/clang/test/CodeGen/AArch64/sme-intrinsics/aarch64-sme-attrs.cpp
@@ -300,12 +300,12 @@ int test_variadic_template() __arm_inout("za") {
               preserves_za_decl);
 }
 
-// CHECK: attributes #[[SM_ENABLED]] = { mustprogress noinline nounwind "aarch64_pstate_sm_enabled" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+bf16,+sme" }
+// CHECK: attributes #[[SM_ENABLED]] = { mustprogress noinline nounwind vscale_range(1,16) "aarch64_pstate_sm_enabled" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+bf16,+sme" }
 // CHECK: attributes #[[NORMAL_DECL]] = { "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+bf16,+sme" }
 // CHECK: attributes #[[SM_ENABLED_DECL]] = { "aarch64_pstate_sm_enabled" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+bf16,+sme" }
 // CHECK: attributes #[[SM_COMPATIBLE]] = { mustprogress noinline nounwind "aarch64_pstate_sm_compatible" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+bf16,+sme" }
 // CHECK: attributes #[[SM_COMPATIBLE_DECL]] = { "aarch64_pstate_sm_compatible" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+bf16,+sme" }
-// CHECK: attributes #[[SM_BODY]] = { mustprogress noinline nounwind "aarch64_pstate_sm_body" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+bf16,+sme" }
+// CHECK: attributes #[[SM_BODY]] = { mustprogress noinline nounwind vscale_range(1,16) "aarch64_pstate_sm_body" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+bf16,+sme" }
 // CHECK: attributes #[[ZA_SHARED]] = { mustprogress noinline nounwind "aarch64_inout_za" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+bf16,+sme" }
 // CHECK: attributes #[[ZA_SHARED_DECL]] = { "aarch64_inout_za" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+bf16,+sme" }
 // CHECK: attributes #[[ZA_PRESERVED]] = { mustprogress noinline nounwind "aarch64_preserves_za" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+bf16,+sme" }
diff --git a/clang/test/CodeGen/AArch64/sme-intrinsics/acle_sme_read.c b/clang/test/CodeGen/AArch64/sme-intrinsics/acle_sme_read.c
index 508fad09ea715..0605f9eef036f 100644
--- a/clang/test/CodeGen/AArch64/sme-intrinsics/acle_sme_read.c
+++ b/clang/test/CodeGen/AArch64/sme-intrinsics/acle_sme_read.c
@@ -318,6 +318,41 @@ svuint64_t test_svread_hor_za64_u64_1(svuint64_t zd, svbool_t pg, uint32_t slice
     return SME_ACLE_FUNC(svread_hor_za64, _u64, _m)(zd, pg, 7, slice);
 }
 
+// CHECK-C-LABEL: define dso_local <vscale x 16 x i8> @test_svread_hor_za8_mf8(
+// CHECK-C-SAME: <vscale x 16 x i8> [[ZD:%.*]], <vscale x 16 x i1> [[PG:%.*]], i32 noundef [[SLICE_BASE:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.aarch64.sme.read.horiz.nxv16i8(<vscale x 16 x i8> [[ZD]], <vscale x 16 x i1> [[PG]], i32 0, i32 [[SLICE_BASE]])
+// CHECK-C-NEXT:    ret <vscale x 16 x i8> [[TMP0]]
+//
+// CHECK-CXX-LABEL: define dso_local <vscale x 16 x i8> @_Z23test_svread_hor_za8_mf8u13__SVMfloat8_tu10__SVBool_tj(
+// CHECK-CXX-SAME: <vscale x 16 x i8> [[ZD:%.*]], <vscale x 16 x i1> [[PG:%.*]], i32 noundef [[SLICE_BASE:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.aarch64.sme.read.horiz.nxv16i8(<vscale x 16 x i8> [[ZD]], <vscale x 16 x i1> [[PG]], i32 0, i32 [[SLICE_BASE]])
+// CHECK-CXX-NEXT:    ret <vscale x 16 x i8> [[TMP0]]
+//
+svmfloat8_t test_svread_hor_za8_mf8(svmfloat8_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming __arm_in("za") {
+    return SME_ACLE_FUNC(svread_hor_za8, _mf8, _m)(zd, pg, 0, slice_base);
+}
+
+// CHECK-C-LABEL: define dso_local <vscale x 16 x i8> @test_svread_hor_za8_mf8_1(
+// CHECK-C-SAME: <vscale x 16 x i8> [[ZD:%.*]], <vscale x 16 x i1> [[PG:%.*]], i32 noundef [[SLICE_BASE:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], 15
+// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.aarch64.sme.read.horiz.nxv16i8(<vscale x 16 x i8> [[ZD]], <vscale x 16 x i1> [[PG]], i32 0, i32 [[ADD]])
+// CHECK-C-NEXT:    ret <vscale x 16 x i8> [[TMP0]]
+//
+// CHECK-CXX-LABEL: define dso_local <vscale x 16 x i8> @_Z25test_svread_hor_za8_mf8_1u13__SVMfloat8_tu10__SVBool_tj(
+// CHECK-CXX-SAME: <vscale x 16 x i8> [[ZD:%.*]], <vscale x 16 x i1> [[PG:%.*]], i32 noundef [[SLICE_BASE:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], 15
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.aarch64.sme.read.horiz.nxv16i8(<vscale x 16 x i8> [[ZD]], <vscale x 16 x i1> [[PG]], i32 0, i32 [[ADD]])
+// CHECK-CXX-NEXT:    ret <vscale x 16 x i8> [[TMP0]]
+//
+svmfloat8_t test_svread_hor_za8_mf8_1(svmfloat8_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming __arm_in("za") {
+    uint32_t slice = slice_base + 15;
+    return SME_ACLE_FUNC(svread_hor_za8, _mf8, _m)(zd, pg, 0, slice);
+}
+
 // CHECK-C-LABEL: define dso_local <vscale x 8 x half> @test_svread_hor_za16_f16(
 // CHECK-C-SAME: <vscale x 8 x half> [[ZD:%.*]], <vscale x 16 x i1> [[PG:%.*]], i32 noundef [[SLICE_BASE:%.*]]) local_unnamed_addr #[[ATTR0]] {
 // CHECK-C-NEXT:  entry:
@@ -754,6 +789,38 @@ svuint64_t test_svread_hor_za128_u64_1(svuint64_t zd, svbool_t pg, uint32_t slic
     return SME_ACLE_FUNC(svread_hor_za128, _u64, _m)(zd, pg, 15, slice_base);
 }
 
+// CHECK-C-LABEL: define dso_local <vscale x 16 x i8> @test_svread_hor_za128_mf8(
+// CHECK-C-SAME: <vscale x 16 x i8> [[ZD:%.*]], <vscale x 16 x i1> [[PG:%.*]], i32 noundef [[SLICE_BASE:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.aarch64.sme.readq.horiz.nxv16i8(<vscale x 16 x i8> [[ZD]], <vscale x 16 x i1> [[PG]], i32 0, i32 [[SLICE_BASE]])
+// CHECK-C-NEXT:    ret <vscale x 16 x i8> [[TMP0]]
+//
+// CHECK-CXX-LABEL: define dso_local <vscale x 16 x i8> @_Z25test_svread_hor_za128_mf8u13__SVMfloat8_tu10__SVBool_tj(
+// CHECK-CXX-SAME: <vscale x 16 x i8> [[ZD:%.*]], <vscale x 16 x i1> [[PG:%.*]], i32 noundef [[SLICE_BASE:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.aarch64.sme.readq.horiz.nxv16i8(<vscale x 16 x i8> [[ZD]], <vscale x 16 x i1> [[PG]], i32 0, i32 [[SLICE_BASE]])
+// CHECK-CXX-NEXT:    ret <vscale x 16 x i8> [[TMP0]]
+//
+svmfloat8_t test_svread_hor_za128_mf8(svmfloat8_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming __arm_in("za") {
+    return SME_ACLE_FUNC(svread_hor_za128, _mf8, _m)(zd, pg, 0, slice_base);
+}
+
+// CHECK-C-LABEL: define dso_local <vscale x 16 x i8> @test_svread_hor_za128_mf8_1(
+// CHECK-C-SAME: <vscale x 16 x i8> [[ZD:%.*]], <vscale x 16 x i1> [[PG:%.*]], i32 noundef [[SLICE_BASE:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.aarch64.sme.readq.horiz.nxv16i8(<vscale x 16 x i8> [[ZD]], <vscale x 16 x i1> [[PG]], i32 15, i32 [[SLICE_BASE]])
+// CHECK-C-NEXT:    ret <vscale x 16 x i8> [[TMP0]]
+//
+// CHECK-CXX-LABEL: define dso_local <vscale x 16 x i8> @_Z27test_svread_hor_za128_mf8_1u13__SVMfloat8_tu10__SVBool_tj(
+// CHECK-CXX-SAME: <vscale x 16 x i8> [[ZD:%.*]], <vscale x 16 x i1> [[PG:%.*]], i32 noundef [[SLICE_BASE:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.aarch64.sme.readq.horiz.nxv16i8(<vscale x 16 x i8> [[ZD]], <vscale x 16 x i1> [[PG]], i32 15, i32 [[SLICE_BASE]])
+// CHECK-CXX-NEXT:    ret <vscale x 16 x i8> [[TMP0]]
+//
+svmfloat8_t test_svread_hor_za128_mf8_1(svmfloat8_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming __arm_in("za") {
+    return SME_ACLE_FUNC(svread_hor_za128, _mf8, _m)(zd, pg, 15, slice_base);
+}
+
 // CHECK-C-LABEL: define dso_local <vscale x 8 x half> @test_svread_hor_za128_f16(
 // CHECK-C-SAME: <vscale x 8 x half> [[ZD:%.*]], <vscale x 16 x i1> [[PG:%.*]], i32 noundef [[SLICE_BASE:%.*]]) local_unnamed_addr #[[ATTR0]] {
 // CHECK-C-NEXT:  entry:
@@ -1202,6 +1269,41 @@ svuint64_t test_svread_ver_za64_u64_1(svuint64_t zd, svbool_t pg, uint32_t slice
     return SME_ACLE_FUNC(svread_ver_za64, _u64, _m)(zd, pg, 7, slice);
 }
 
+// CHECK-C-LABEL: define dso_local <vscale x 16 x i8> @test_svread_ver_za8_mf8(
+// CHECK-C-SAME: <vscale x 16 x i8> [[ZD:%.*]], <vscale x 16 x i1> [[PG:%.*]], i32 noundef [[SLICE_BASE:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.aarch64.sme.read.vert.nxv16i8(<vscale x 16 x i8> [[ZD]], <vscale x 16 x i1> [[PG]], i32 0, i32 [[SLICE_BASE]])
+// CHECK-C-NEXT:    ret <vscale x 16 x i8> [[TMP0]]
+//
+// CHECK-CXX-LABEL: define dso_local <vscale x 16 x i8> @_Z23test_svread_ver_za8_mf8u13__SVMfloat8_tu10__SVBool_tj(
+// CHECK-CXX-SAME: <vscale x 16 x i8> [[ZD:%.*]], <vscale x 16 x i1> [[PG:%.*]], i32 noundef [[SLICE_BASE:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.aarch64.sme.read.vert.nxv16i8(<vscale x 16 x i8> [[ZD]], <vscale x 16 x i1> [[PG]], i32 0, i32 [[SLICE_BASE]])
+// CHECK-CXX-NEXT:    ret <vscale x 16 x i8> [[TMP0]]
+//
+svmfloat8_t test_svread_ver_za8_mf8(svmfloat8_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming __arm_in("za") {
+    return SME_ACLE_FUNC(svread_ver_za8, _mf8, _m)(zd, pg, 0, slice_base);
+}
+
+// CHECK-C-LABEL: define dso_local <vscale x 16 x i8> @test_svread_ver_za8_mf8_1(
+// CHECK-C-SAME: <vscale x 16 x i8> [[ZD:%.*]], <vscale x 16 x i1> [[PG:%.*]], i32 noundef [[SLICE_BASE:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], 15
+// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.aarch64.sme.read.vert.nxv16i8(<vscale x 16 x i8> [[ZD]], <vscale x 16 x i1> [[PG]], i32 0, i32 [[ADD]])
+// CHECK-C-NEXT:    ret <vscale x 16 x i8> [[TMP0]]
+//
+// CHECK-CXX-LABEL: define dso_local <vscale x 16 x i8> @_Z25test_svread_ver_za8_mf8_1u13__SVMfloat8_tu10__SVBool_tj(
+// CHECK-CXX-SAME: <vscale x 16 x i8> [[ZD:%.*]], <vscale x 16 x i1> [[PG:%.*]], i32 noundef [[SLICE_BASE:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], 15
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.aarch64.sme.read.vert.nxv16i8(<vscale x 16 x i8> [[ZD]], <vscale x 16 x i1> [[PG]], i32 0, i32 [[ADD]])
+// CHECK-CXX-NEXT:    ret <vscale x 16 x i8> [[TMP0]]
+//
+svmfloat8_t test_svread_ver_za8_mf8_1(svmfloat8_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming __arm_in("za") {
+    uint32_t slice = slice_base + 15;
+    return SME_ACLE_FUNC(svread_ver_za8, _mf8, _m)(zd, pg, 0, slice);
+}
+
 // CHECK-C-LABEL: define dso_local <vscale x 8 x half> @test_svread_ver_za16_f16(
 // CHECK-C-SAME: <vscale x 8 x half> [[ZD:%.*]], <vscale x 16 x i1> [[PG:%.*]], i32 noundef [[SLICE_BASE:%.*]]) local_unnamed_addr #[[ATTR0]] {
 // CHECK-C-NEXT:  entry:
@@ -1638,6 +1740,38 @@ svuint64_t test_svread_ver_za128_u64_1(svuint64_t zd, svbool_t pg, uint32_t slic
     return SME_ACLE_FUNC(svread_ver_za128, _u64, _m)(zd, pg, 15, slice_base);
 }
 
+// CHECK-C-LABEL: define dso_local <vscale x 16 x i8> @test_svread_ver_za128_mf8(
+// CHECK-C-SAME: <vscale x 16 x i8> [[ZD:%.*]], <vscale x 16 x i1> [[PG:%.*]], i32 noundef [[SLICE_BASE:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.aarch64.sme.readq.vert.nxv16i8(<vscale x 16 x i8> [[ZD]], <vscale x 16 x i1> [[PG]], i32 0, i32 [[SLICE_BASE]])
+// CHECK-C-NEXT:    ret <vscale x 16 x i8> [[TMP0]]
+//
+// CHECK-CXX-LABEL: define dso_local <vscale x 16 x i8> @_Z25test_svread_ver_za128_mf8u13__SVMfloat8_tu10__SVBool_tj(
+// CHECK-CXX-SAME: <vscale x 16 x i8> [[ZD:%.*]], <vscale x 16 x i1> [[PG:%.*]], i32 noundef [[SLICE_BASE:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.aarch64.sme.readq.vert.nxv16i8(<vscale x 16 x i8> [[ZD]], <vscale x 16 x i1> [[PG]], i32 0, i32 [[SLICE_BASE]])
+// CHECK-CXX-NEXT:    ret <vscale x 16 x i8> [[TMP0]]
+//
+svmfloat8_t test_svread_ver_za128_mf8(svmfloat8_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming __arm_in("za") {
+    return SME_ACLE_FUNC(svread_ver_za128, _mf8, _m)(zd, pg, 0, slice_base);
+}
+
+// CHECK-C-LABEL: define dso_local <vscale x 16 x i8> @test_svread_ver_za128_mf8_1(
+// CHECK-C-SAME: <vscale x 16 x i8> [[ZD:%.*]], <vscale x 16 x i1> [[PG:%.*]], i32 noundef [[SLICE_BASE:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.aarch64.sme.readq.vert.nxv16i8(<vscale x 16 x i8> [[ZD]], <vscale x 16 x i1> [[PG]], i32 15, i32 [[SLICE_BASE]])
+// CHECK-C-NEXT:    ret <vscale x 16 x i8> [[TMP0]]
+//
+// CHECK-CXX-LABEL: define dso_local <vscale x 16 x i8> @_Z27test_svread_ver_za128_mf8_1u13__SVMfloat8_tu10__SVBool_tj(
+// CHECK-CXX-SAME: <vscale x 16 x i8> [[ZD:%.*]], <vscale x 16 x i1> [[PG:%.*]], i32 noundef [[SLICE_BASE:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.aarch64.sme.readq.vert.nxv16i8(<vscale x 16 x i8> [[ZD]], <vscale x 16 x i1> [[PG]], i32 15, i32 [[SLICE_BASE]])
+// CHECK-CXX-NEXT:    ret <vscale x 16 x i8> [[TMP0]]
+//
+svmfloat8_t test_svread_ver_za128_mf8_1(svmfloat8_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming __arm_in("za") {
+    return SME_ACLE_FUNC(svread_ver_za128, _mf8, _m)(zd, pg, 15, slice_base);
+}
+
 // CHECK-C-LABEL: define dso_local <vscale x 8 x half> @test_svread_ver_za128_f16(
 // CHECK-C-SAME: <vscale x 8 x half> [[ZD:%.*]], <vscale x 16 x i1> [[PG:%.*]], i32 noundef [[SLICE_BASE:%.*]]) local_unnamed_addr #[[ATTR0]] {
 // CHECK-C-NEXT:  entry:
diff --git a/clang/test/CodeGen/AArch64/sme-intrinsics/acle_sme_write.c b/clang/test/CodeGen/AArch64/sme-intrinsics/acle_sme_write.c
index 86a691f14623f..99b96ee87553c 100644
--- a/clang/test/CodeGen/AArch64/sme-intrinsics/acle_sme_write.c
+++ b/clang/test/CodeGen/AArch64/sme-intrinsics/acle_sme_write.c
@@ -318,6 +318,41 @@ void test_svwrite_hor_za64_u64_1(uint32_t slice_base, svbool_t pg, svuint64_t zn
   SME_ACLE_FUNC(svwrite_hor_za64, _u64, _m)(7, slice, pg, zn);
 }
 
+// CHECK-C-LABEL: define dso_local void @test_svwrite_hor_za8_mf8(
+// CHECK-C-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], <vscale x 16 x i8> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.write.horiz.nxv16i8(i32 0, i32 [[SLICE_BASE]], <vscale x 16 x i1> [[PG]], <vscale x 16 x i8> [[ZN]])
+// CHECK-C-NEXT:    ret void
+//
+// CHECK-CXX-LABEL: define dso_local void @_Z24test_svwrite_hor_za8_mf8ju10__SVBool_tu13__SVMfloat8_t(
+// CHECK-CXX-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], <vscale x 16 x i8> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.write.horiz.nxv16i8(i32 0, i32 [[SLICE_BASE]], <vscale x 16 x i1> [[PG]], <vscale x 16 x i8> [[ZN]])
+// CHECK-CXX-NEXT:    ret void
+//
+void test_svwrite_hor_za8_mf8(uint32_t slice_base, svbool_t pg, svmfloat8_t zn) __arm_streaming __arm_inout("za") {
+  SME_ACLE_FUNC(svwrite_hor_za8, _mf8, _m)(0, slice_base, pg, zn);
+}
+
+// CHECK-C-LABEL: define dso_local void @test_svwrite_hor_za8_mf8_1(
+// CHECK-C-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], <vscale x 16 x i8> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], 15
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.write.horiz.nxv16i8(i32 0, i32 [[ADD]], <vscale x 16 x i1> [[PG]], <vscale x 16 x i8> [[ZN]])
+// CHECK-C-NEXT:    ret void
+//
+// CHECK-CXX-LABEL: define dso_local void @_Z26test_svwrite_hor_za8_mf8_1ju10__SVBool_tu13__SVMfloat8_t(
+// CHECK-CXX-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], <vscale x 16 x i8> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], 15
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.write.horiz.nxv16i8(i32 0, i32 [[ADD]], <vscale x 16 x i1> [[PG]], <vscale x 16 x i8> [[ZN]])
+// CHECK-CXX-NEXT:    ret void
+//
+void test_svwrite_hor_za8_mf8_1(uint32_t slice_base, svbool_t pg, svmfloat8_t zn) __arm_streaming __arm_inout("za") {
+   uint32_t slice = slice_base + 15;
+  SME_ACLE_FUNC(svwrite_hor_za8, _mf8, _m)(0, slice, pg, zn);
+}
+
 // CHECK-C-LABEL: define dso_local void @test_svwrite_hor_za16_f16(
 // CHECK-C-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], <vscale x 8 x half> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
 // CHECK-C-NEXT:  entry:
@@ -754,6 +789,39 @@ void test_svwrite_hor_za128_u64_1(uint32_t slice_base, svbool_t pg, svuint64_t z
   SME_ACLE_FUNC(svwrite_hor_za128, _u64, _m)(15, slice_base, pg, zn);
 }
 
+
+// CHECK-C-LABEL: define dso_local void @test_svwrite_hor_za128_mf8(
+// CHECK-C-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], <vscale x 16 x i8> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.writeq.horiz.nxv16i8(i32 0, i32 [[SLICE_BASE]], <vscale x 16 x i1> [[PG]], <vscale x 16 x i8> [[ZN]])
+// CHECK-C-NEXT:    ret void
+//
+// CHECK-CXX-LABEL: define dso_local void @_Z26test_svwrite_hor_za128_mf8ju10__SVBool_tu13__SVMfloat8_t(
+// CHECK-CXX-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], <vscale x 16 x i8> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.writeq.horiz.nxv16i8(i32 0, i32 [[SLICE_BASE]], <vscale x 16 x i1> [[PG]], <vscale x 16 x i8> [[ZN]])
+// CHECK-CXX-NEXT:    ret void
+//
+void test_svwrite_hor_za128_mf8(uint32_t slice_base, svbool_t pg, svmfloat8_t zn) __arm_streaming __arm_inout("za") {
+  SME_ACLE_FUNC(svwrite_hor_za128, _mf8, _m)(0, slice_base, pg, zn);
+}
+
+// CHECK-C-LABEL: define dso_local void @test_svwrite_hor_za128_mf8_1(
+// CHECK-C-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], <vscale x 16 x i8> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.writeq.horiz.nxv16i8(i32 15, i32 [[SLICE_BASE]], <vscale x 16 x i1> [[PG]], <vscale x 16 x i8> [[ZN]])
+// CHECK-C-NEXT:    ret void
+//
+// CHECK-CXX-LABEL: define dso_local void @_Z28test_svwrite_hor_za128_mf8_1ju10__SVBool_tu13__SVMfloat8_t(
+// CHECK-CXX-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], <vscale x 16 x i8> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.writeq.horiz.nxv16i8(i32 15, i32 [[SLICE_BASE]], <vscale x 16 x i1> [[PG]], <vscale x 16 x i8> [[ZN]])
+// CHECK-CXX-NEXT:    ret void
+//
+void test_svwrite_hor_za128_mf8_1(uint32_t slice_base, svbool_t pg, svmfloat8_t zn) __arm_streaming __arm_inout("za") {
+  SME_ACLE_FUNC(svwrite_hor_za128, _mf8, _m)(15, slice_base, pg, zn);
+}
+
 // CHECK-C-LABEL: define dso_local void @test_svwrite_hor_za128_f16(
 // CHECK-C-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], <vscale x 8 x half> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
 // CHECK-C-NEXT:  entry:
@@ -1202,6 +1270,41 @@ void test_svwrite_ver_za64_u64_1(uint32_t slice_base, svbool_t pg, svuint64_t zn
   SME_ACLE_FUNC(svwrite_ver_za64, _u64, _m)(7, slice, pg, zn);
 }
 
+// CHECK-C-LABEL: define dso_local void @test_svwrite_ver_za8_mf8(
+// CHECK-C-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], <vscale x 16 x i8> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.write.vert.nxv16i8(i32 0, i32 [[SLICE_BASE]], <vscale x 16 x i1> [[PG]], <vscale x 16 x i8> [[ZN]])
+// CHECK-C-NEXT:    ret void
+//
+// CHECK-CXX-LABEL: define dso_local void @_Z24test_svwrite_ver_za8_mf8ju10__SVBool_tu13__SVMfloat8_t(
+// CHECK-CXX-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], <vscale x 16 x i8> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.write.vert.nxv16i8(i32 0, i32 [[SLICE_BASE]], <vscale x 16 x i1> [[PG]], <vscale x 16 x i8> [[ZN]])
+// CHECK-CXX-NEXT:    ret void
+//
+void test_svwrite_ver_za8_mf8(uint32_t slice_base, svbool_t pg, svmfloat8_t zn) __arm_streaming __arm_inout("za") {
+  SME_ACLE_FUNC(svwrite_ver_za8, _mf8, _m)(0, slice_base, pg, zn);
+}
+
+// CHECK-C-LABEL: define dso_local void @test_svwrite_ver_za8_mf8_1(
+// CHECK-C-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], <vscale x 16 x i8> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], 15
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.write.vert.nxv16i8(i32 0, i32 [[ADD]], <vscale x 16 x i1> [[PG]], <vscale x 16 x i8> [[ZN]])
+// CHECK-C-NEXT:    ret void
+//
+// CHECK-CXX-LABEL: define dso_local void @_Z26test_svwrite_ver_za8_mf8_1ju10__SVBool_tu13__SVMfloat8_t(
+// CHECK-CXX-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], <vscale x 16 x i8> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], 15
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.write.vert.nxv16i8(i32 0, i32 [[ADD]], <vscale x 16 x i1> [[PG]], <vscale x 16 x i8> [[ZN]])
+// CHECK-CXX-NEXT:    ret void
+//
+void test_svwrite_ver_za8_mf8_1(uint32_t slice_base, svbool_t pg, svmfloat8_t zn) __arm_streaming __arm_inout("za") {
+  uint32_t slice = slice_base + 15;
+  SME_ACLE_FUNC(svwrite_ver_za8, _mf8, _m)(0, slice, pg, zn);
+}
+
 // CHECK-C-LABEL: define dso_local void @test_svwrite_ver_za16_f16(
 // CHECK-C-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], <vscale x 8 x half> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
 // CHECK-C-NEXT:  entry:
@@ -1656,6 +1759,38 @@ void test_svwrite_ver_za128_f16(uint32_t slice_base, svbool_t pg, svfloat16_t zn
   SME_ACLE_FUNC(svwrite_ver_za128, _f16, _m)(0, slice_base, pg, zn);
 }
 
+// CHECK-C-LABEL: define dso_local void @test_svwrite_ver_za128_mf8(
+// CHECK-C-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], <vscale x 16 x i8> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.writeq.vert.nxv16i8(i32 0, i32 [[SLICE_BASE]], <vscale x 16 x i1> [[PG]], <vscale x 16 x i8> [[ZN]])
+// CHECK-C-NEXT:    ret void
+//
+// CHECK-CXX-LABEL: define dso_local void @_Z26test_svwrite_ver_za128_mf8ju10__SVBool_tu13__SVMfloat8_t(
+// CHECK-CXX-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], <vscale x 16 x i8> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.writeq.vert.nxv16i8(i32 0, i32 [[SLICE_BASE]], <vscale x 16 x i1> [[PG]], <vscale x 16 x i8> [[ZN]])
+// CHECK-CXX-NEXT:    ret void
+//
+void test_svwrite_ver_za128_mf8(uint32_t slice_base, svbool_t pg, svmfloat8_t zn) __arm_streaming __arm_inout("za") {
+  SME_ACLE_FUNC(svwrite_ver_za128, _mf8, _m)(0, slice_base, pg, zn);
+}
+
+// CHECK-C-LABEL: define dso_local void @test_svwrite_ver_za128_mf8_1(
+// CHECK-C-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], <vscale x 16 x i8> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.writeq.vert.nxv16i8(i32 15, i32 [[SLICE_BASE]], <vscale x 16 x i1> [[PG]], <vscale x 16 x i8> [[ZN]])
+// CHECK-C-NEXT:    ret void
+//
+// CHECK-CXX-LABEL: define dso_local void @_Z28test_svwrite_ver_za128_mf8_1ju10__SVBool_tu13__SVMfloat8_t(
+// CHECK-CXX-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], <vscale x 16 x i8> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.writeq.vert.nxv16i8(i32 15, i32 [[SLICE_BASE]], <vscale x 16 x i1> [[PG]], <vscale x 16 x i8> [[ZN]])
+// CHECK-CXX-NEXT:    ret void
+//
+void test_svwrite_ver_za128_mf8_1(uint32_t slice_base, svbool_t pg, svmfloat8_t zn) __arm_streaming __arm_inout("za") {
+  SME_ACLE_FUNC(svwrite_ver_za128, _mf8, _m)(15, slice_base, pg, zn);
+}
+
 // CHECK-C-LABEL: define dso_local void @test_svwrite_ver_za128_f16_1(
 // CHECK-C-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], <vscale x 8 x half> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
 // CHECK-C-NEXT:  entry:
diff --git a/clang/test/CodeGen/AArch64/sme2-intrinsics/acle_sme2_luti2_lane_zt.c b/clang/test/CodeGen/AArch64/sme2-intrinsics/acle_sme2_luti2_lane_zt.c
index 8de82abe05324..1ab02afbe0904 100644
--- a/clang/test/CodeGen/AArch64/sme2-intrinsics/acle_sme2_luti2_lane_zt.c
+++ b/clang/test/CodeGen/AArch64/sme2-intrinsics/acle_sme2_luti2_lane_zt.c
@@ -52,6 +52,19 @@ svuint16_t test_svluti2_lane_zt_u16(svuint8_t zn) __arm_streaming __arm_in("zt0"
   return svluti2_lane_zt_u16(0, zn, 15);
 }
 
+// CHECK-LABEL: @test_svluti2_lane_zt_mf8(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.aarch64.sme.luti2.lane.zt.nxv16i8(i32 0, <vscale x 16 x i8> [[ZN:%.*]], i32 15)
+// CHECK-NEXT:    ret <vscale x 16 x i8> [[TMP0]]
+//
+// CPP-CHECK-LABEL: @_Z24test_svluti2_lane_zt_mf8u11__SVUint8_t(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.aarch64.sme.luti2.lane.zt.nxv16i8(i32 0, <vscale x 16 x i8> [[ZN:%.*]], i32 15)
+// CPP-CHECK-NEXT:    ret <vscale x 16 x i8> [[TMP0]]
+//
+svmfloat8_t test_svluti2_lane_zt_mf8(svuint8_t zn) __arm_streaming __arm_in("zt0") {
+  return svluti2_lane_zt_mf8(0, zn, 15);
+}
 
 // CHECK-LABEL: @test_svluti2_lane_zt_s16(
 // CHECK-NEXT:  entry:
diff --git a/clang/test/CodeGen/AArch64/sme2-intrinsics/acle_sme2_luti2_lane_zt_x2.c b/clang/test/CodeGen/AArch64/sme2-intrinsics/acle_sme2_luti2_lane_zt_x2.c
index 3b17c6d9edb19..e97075703b185 100644
--- a/clang/test/CodeGen/AArch64/sme2-intrinsics/acle_sme2_luti2_lane_zt_x2.c
+++ b/clang/test/CodeGen/AArch64/sme2-intrinsics/acle_sme2_luti2_lane_zt_x2.c
@@ -37,6 +37,20 @@ svint8x2_t test_svluti2_lane_zt_s8(svuint8_t zn) __arm_streaming __arm_in("zt0")
   return svluti2_lane_zt_s8_x2(0, zn, 7);
 }
 
+// CHECK-LABEL: @test_svluti2_lane_zt_mf8(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call { <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sme.luti2.lane.zt.x2.nxv16i8(i32 0, <vscale x 16 x i8> [[ZN:%.*]], i32 7)
+// CHECK-NEXT:    ret { <vscale x 16 x i8>, <vscale x 16 x i8> } [[TMP0]]
+//
+// CPP-CHECK-LABEL: @_Z24test_svluti2_lane_zt_mf8u11__SVUint8_t(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call { <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sme.luti2.lane.zt.x2.nxv16i8(i32 0, <vscale x 16 x i8> [[ZN:%.*]], i32 7)
+// CPP-CHECK-NEXT:    ret { <vscale x 16 x i8>, <vscale x 16 x i8> } [[TMP0]]
+//
+svmfloat8x2_t test_svluti2_lane_zt_mf8(svuint8_t zn) __arm_streaming __arm_in("zt0") {
+  return svluti2_lane_zt_mf8_x2(0, zn, 7);
+}
+
 // CHECK-LABEL: @test_svluti2_lane_zt_u16(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call { <vscale x 8 x i16>, <vscale x 8 x i16> } @llvm.aarch64.sme.luti2.lane.zt.x2.nxv8i16(i32 0, <vscale x 16 x i8> [[ZN:%.*]], i32 7)
diff --git a/clang/test/CodeGen/AArch64/sme2-intrinsics/acle_sme2_luti2_lane_zt_x4.c b/clang/test/CodeGen/AArch64/sme2-intrinsics/acle_sme2_luti2_lane_zt_x4.c
index 38059019737f8..0730812b1f06f 100644
--- a/clang/test/CodeGen/AArch64/sme2-intrinsics/acle_sme2_luti2_lane_zt_x4.c
+++ b/clang/test/CodeGen/AArch64/sme2-intrinsics/acle_sme2_luti2_lane_zt_x4.c
@@ -37,6 +37,20 @@ svint8x4_t test_svluti2_lane_zt_s8(svuint8_t zn) __arm_streaming __arm_in("zt0")
   return svluti2_lane_zt_s8_x4(0, zn, 3);
 }
 
+// CHECK-LABEL: @test_svluti2_lane_zt_mf8(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sme.luti2.lane.zt.x4.nxv16i8(i32 0, <vscale x 16 x i8> [[ZN:%.*]], i32 3)
+// CHECK-NEXT:    ret { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } [[TMP0]]
+//
+// CPP-CHECK-LABEL: @_Z24test_svluti2_lane_zt_mf8u11__SVUint8_t(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sme.luti2.lane.zt.x4.nxv16i8(i32 0, <vscale x 16 x i8> [[ZN:%.*]], i32 3)
+// CPP-CHECK-NEXT:    ret { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } [[TMP0]]
+//
+svmfloat8x4_t test_svluti2_lane_zt_mf8(svuint8_t zn) __arm_streaming __arm_in("zt0") {
+  return svluti2_lane_zt_mf8_x4(0, zn, 3);
+}
+
 // CHECK-LABEL: @test_svluti2_lane_zt_u16(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } @llvm.aarch64.sme.luti2.lane.zt.x4.nxv8i16(i32 0, <vscale x 16 x i8> [[ZN:%.*]], i32 3)
diff --git a/clang/test/CodeGen/AArch64/sme2-intrinsics/acle_sme2_luti4_lane_zt.c b/clang/test/CodeGen/AArch64/sme2-intrinsics/acle_sme2_luti4_lane_zt.c
index 16e66d1c21222..b687b580b15a6 100644
--- a/clang/test/CodeGen/AArch64/sme2-intrinsics/acle_sme2_luti4_lane_zt.c
+++ b/clang/test/CodeGen/AArch64/sme2-intrinsics/acle_sme2_luti4_lane_zt.c
@@ -38,6 +38,20 @@ svint8_t test_svluti4_lane_zt_s8(svuint8_t zn) __arm_streaming __arm_in("zt0") {
   return svluti4_lane_zt_s8(0, zn, 7);
 }
 
+// CHECK-LABEL: @test_svluti4_lane_zt_mf8(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.aarch64.sme.luti4.lane.zt.nxv16i8(i32 0, <vscale x 16 x i8> [[ZN:%.*]], i32 7)
+// CHECK-NEXT:    ret <vscale x 16 x i8> [[TMP0]]
+//
+// CPP-CHECK-LABEL: @_Z24test_svluti4_lane_zt_mf8u11__SVUint8_t(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.aarch64.sme.luti4.lane.zt.nxv16i8(i32 0, <vscale x 16 x i8> [[ZN:%.*]], i32 7)
+// CPP-CHECK-NEXT:    ret <vscale x 16 x i8> [[TMP0]]
+//
+svmfloat8_t test_svluti4_lane_zt_mf8(svuint8_t zn) __arm_streaming __arm_in("zt0") {
+  return svluti4_lane_zt_mf8(0, zn, 7);
+}
+
 // CHECK-LABEL: @test_svluti4_lane_zt_u16(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i16> @llvm.aarch64.sme.luti4.lane.zt.nxv8i16(i32 0, <vscale x 16 x i8> [[ZN:%.*]], i32 7)
diff --git a/clang/test/CodeGen/AArch64/sme2-intrinsics/acle_sme2_luti4_lane_zt_x2.c b/clang/test/CodeGen/AArch64/sme2-intrinsics/acle_sme2_luti4_lane_zt_x2.c
index db615b3cd1c24..1a9e9d84c6359 100644
--- a/clang/test/CodeGen/AArch64/sme2-intrinsics/acle_sme2_luti4_lane_zt_x2.c
+++ b/clang/test/CodeGen/AArch64/sme2-intrinsics/acle_sme2_luti4_lane_zt_x2.c
@@ -37,6 +37,20 @@ svint8x2_t test_svluti4_lane_zt_s8(svuint8_t zn) __arm_streaming __arm_in("zt0")
   return svluti4_lane_zt_s8_x2(0, zn, 3);
 }
 
+// CHECK-LABEL: @test_svluti4_lane_zt_mf8(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call { <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sme.luti4.lane.zt.x2.nxv16i8(i32 0, <vscale x 16 x i8> [[ZN:%.*]], i32 3)
+// CHECK-NEXT:    ret { <vscale x 16 x i8>, <vscale x 16 x i8> } [[TMP0]]
+//
+// CPP-CHECK-LABEL: @_Z24test_svluti4_lane_zt_mf8u11__SVUint8_t(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call { <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sme.luti4.lane.zt.x2.nxv16i8(i32 0, <vscale x 16 x i8> [[ZN:%.*]], i32 3)
+// CPP-CHECK-NEXT:    ret { <vscale x 16 x i8>, <vscale x 16 x i8> } [[TMP0]]
+//
+svmfloat8x2_t test_svluti4_lane_zt_mf8(svuint8_t zn) __arm_streaming __arm_in("zt0") {
+  return svluti4_lane_zt_mf8_x2(0, zn, 3);
+}
+
 // CHECK-LABEL: @test_svluti4_lane_zt_u16(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call { <vscale x 8 x i16>, <vscale x 8 x i16> } @llvm.aarch64.sme.luti4.lane.zt.x2.nxv8i16(i32 0, <vscale x 16 x i8> [[ZN:%.*]], i32 3)
diff --git a/clang/test/CodeGen/AArch64/sme2-intrinsics/acle_sme2_read.c b/clang/test/CodeGen/AArch64/sme2-intrinsics/acle_sme2_read.c
index b8cd1e1653ea9..c9d532d5fce45 100644
--- a/clang/test/CodeGen/AArch64/sme2-intrinsics/acle_sme2_read.c
+++ b/clang/test/CodeGen/AArch64/sme2-intrinsics/acle_sme2_read.c
@@ -35,6 +35,20 @@ svint8x2_t test_svread_ver_za8_s8_vg2(uint32_t base) __arm_streaming __arm_in("z
   return svread_ver_za8_s8_vg2(0, base);
 }
 
+// CHECK-LABEL: @test_svread_ver_za8_mf8_vg2(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call { <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sme.read.ver.vg2.nxv16i8(i32 0, i32 [[BASE:%.*]])
+// CHECK-NEXT:    ret { <vscale x 16 x i8>, <vscale x 16 x i8> } [[TMP0]]
+//
+// CPP-CHECK-LABEL: @_Z27test_svread_ver_za8_mf8_vg2j(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call { <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sme.read.ver.vg2.nxv16i8(i32 0, i32 [[BASE:%.*]])
+// CPP-CHECK-NEXT:    ret { <vscale x 16 x i8>, <vscale x 16 x i8> } [[TMP0]]
+//
+svmfloat8x2_t test_svread_ver_za8_mf8_vg2(uint32_t base) __arm_streaming __arm_in("za") {
+  return svread_ver_za8_mf8_vg2(0, base);
+}
+
 // CHECK-LABEL: @test_svread_hor_za8_u8_vg2(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call { <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sme.read.hor.vg2.nxv16i8(i32 0, i32 [[BASE:%.*]])
@@ -63,6 +77,20 @@ svint8x2_t test_svread_hor_za8_s8_vg2(uint32_t base) __arm_streaming __arm_in("z
   return svread_hor_za8_s8_vg2(0, base);
 }
 
+// CHECK-LABEL: @test_svread_hor_za8_mf8_vg2(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call { <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sme.read.hor.vg2.nxv16i8(i32 0, i32 [[BASE:%.*]])
+// CHECK-NEXT:    ret { <vscale x 16 x i8>, <vscale x 16 x i8> } [[TMP0]]
+//
+// CPP-CHECK-LABEL: @_Z27test_svread_hor_za8_mf8_vg2j(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call { <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sme.read.hor.vg2.nxv16i8(i32 0, i32 [[BASE:%.*]])
+// CPP-CHECK-NEXT:    ret { <vscale x 16 x i8>, <vscale x 16 x i8> } [[TMP0]]
+//
+svmfloat8x2_t test_svread_hor_za8_mf8_vg2(uint32_t base) __arm_streaming __arm_in("za") {
+  return svread_hor_za8_mf8_vg2(0, base);
+}
+
 // CHECK-LABEL: @test_svread_hor_za8_u8_vg4(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sme.read.hor.vg4.nxv16i8(i32 0, i32 [[BASE:%.*]])
@@ -91,6 +119,20 @@ svint8x4_t test_svread_hor_za8_s8_vg4(uint32_t base) __arm_streaming __arm_in("z
   return svread_hor_za8_s8_vg4(0, base);
 }
 
+// CHECK-LABEL: @test_svread_hor_za8_mf8_vg4(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sme.read.hor.vg4.nxv16i8(i32 0, i32 [[BASE:%.*]])
+// CHECK-NEXT:    ret { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } [[TMP0]]
+//
+// CPP-CHECK-LABEL: @_Z27test_svread_hor_za8_mf8_vg4j(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sme.read.hor.vg4.nxv16i8(i32 0, i32 [[BASE:%.*]])
+// CPP-CHECK-NEXT:    ret { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } [[TMP0]]
+//
+svmfloat8x4_t test_svread_hor_za8_mf8_vg4(uint32_t base) __arm_streaming __arm_in("za") {
+  return svread_hor_za8_mf8_vg4(0, base);
+}
+
 // CHECK-LABEL: @test_svread_ver_za8_u8_vg4(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sme.read.ver.vg4.nxv16i8(i32 0, i32 [[BASE:%.*]])
@@ -119,6 +161,20 @@ svint8x4_t test_svread_ver_za8_s8_vg4(uint32_t base) __arm_streaming __arm_in("z
   return svread_ver_za8_s8_vg4(0, base);
 }
 
+// CHECK-LABEL: @test_svread_ver_za8_mf8_vg4(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sme.read.ver.vg4.nxv16i8(i32 0, i32 [[BASE:%.*]])
+// CHECK-NEXT:    ret { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } [[TMP0]]
+//
+// CPP-CHECK-LABEL: @_Z27test_svread_ver_za8_mf8_vg4j(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sme.read.ver.vg4.nxv16i8(i32 0, i32 [[BASE:%.*]])
+// CPP-CHECK-NEXT:    ret { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } [[TMP0]]
+//
+svmfloat8x4_t test_svread_ver_za8_mf8_vg4(uint32_t base) __arm_streaming __arm_in("za") {
+  return svread_ver_za8_mf8_vg4(0, base);
+}
+
 // CHECK-LABEL: @test_svread_hor_za16_u16_vg2(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call { <vscale x 8 x i16>, <vscale x 8 x i16> } @llvm.aarch64.sme.read.hor.vg2.nxv8i16(i32 1, i32 [[BASE:%.*]])
@@ -581,7 +637,6 @@ svfloat64x2_t test_svread_ver_za64_f64_vg2(uint32_t base) __arm_streaming __arm_
   return svread_ver_za64_f64_vg2(7, base);
 }
 
-//
 // CHECK-LABEL: @test_svread_ver_za64_s64_vg2(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call { <vscale x 2 x i64>, <vscale x 2 x i64> } @llvm.aarch64.sme.read.ver.vg2.nxv2i64(i32 7, i32 [[BASE:%.*]])
@@ -708,6 +763,19 @@ svuint8x2_t test_svread_za8_u8_vg1x2(uint32_t base) __arm_streaming __arm_in("za
   return svread_za8_u8_vg1x2(base);
 }
 
+// CHECK-LABEL: @test_svread_za8_mf8_vg1x2(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call { <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sme.read.vg1x2.nxv16i8(i32 [[BASE:%.*]])
+// CHECK-NEXT:    ret { <vscale x 16 x i8>, <vscale x 16 x i8> } [[TMP0]]
+//
+// CPP-CHECK-LABEL: @_Z25test_svread_za8_mf8_vg1x2j(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call { <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sme.read.vg1x2.nxv16i8(i32 [[BASE:%.*]])
+// CPP-CHECK-NEXT:    ret { <vscale x 16 x i8>, <vscale x 16 x i8> } [[TMP0]]
+//
+svmfloat8x2_t test_svread_za8_mf8_vg1x2(uint32_t base) __arm_streaming __arm_in("za") {
+  return svread_za8_mf8_vg1x2(base);
+}
 // CHECK-LABEL: @test_svread_za16_s16_vg1x2(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call { <vscale x 8 x i16>, <vscale x 8 x i16> } @llvm.aarch64.sme.read.vg1x2.nxv8i16(i32 [[BASE:%.*]])
@@ -876,6 +944,20 @@ svuint8x4_t test_svread_za8_u8_vg1x4(uint32_t base) __arm_streaming __arm_in("za
   return svread_za8_u8_vg1x4(base);
 }
 
+// CHECK-LABEL: @test_svread_za8_mf8_vg1x4(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sme.read.vg1x4.nxv16i8(i32 [[BASE:%.*]])
+// CHECK-NEXT:    ret { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } [[TMP0]]
+//
+// CPP-CHECK-LABEL: @_Z25test_svread_za8_mf8_vg1x4j(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sme.read.vg1x4.nxv16i8(i32 [[BASE:%.*]])
+// CPP-CHECK-NEXT:    ret { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } [[TMP0]]
+//
+svmfloat8x4_t test_svread_za8_mf8_vg1x4(uint32_t base) __arm_streaming __arm_in("za") {
+  return svread_za8_mf8_vg1x4(base);
+}
+
 // CHECK-LABEL: @test_svread_za16_s16_vg1x4(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } @llvm.aarch64.sme.read.vg1x4.nxv8i16(i32 [[BASE:%.*]])
diff --git a/clang/test/CodeGen/AArch64/sme2-intrinsics/acle_sme2_vector_selx2.c b/clang/test/CodeGen/AArch64/sme2-intrinsics/acle_sme2_vector_selx2.c
index 4047b2fbd1965..893cc7519a1d0 100644
--- a/clang/test/CodeGen/AArch64/sme2-intrinsics/acle_sme2_vector_selx2.c
+++ b/clang/test/CodeGen/AArch64/sme2-intrinsics/acle_sme2_vector_selx2.c
@@ -45,6 +45,19 @@ svuint8x2_t test_svsel_u8_x2(svcount_t pn, svuint8x2_t zn, svuint8x2_t zm) __arm
   return SVE_ACLE_FUNC(svsel,_u8_x2)(pn, zn, zm);
 }
 
+// CHECK-LABEL: @test_svsel_mf8_x2(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call { <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.sel.x2.nxv16i8(target("aarch64.svcount") [[PN:%.*]], <vscale x 16 x i8> [[ZN_COERCE0:%.*]], <vscale x 16 x i8> [[ZN_COERCE1:%.*]], <vscale x 16 x i8> [[ZM_COERCE0:%.*]], <vscale x 16 x i8> [[ZM_COERCE1:%.*]])
+// CHECK-NEXT:    ret { <vscale x 16 x i8>, <vscale x 16 x i8> } [[TMP0]]
+//
+// CPP-CHECK-LABEL: @_Z17test_svsel_mf8_x2u11__SVCount_t13svmfloat8x2_tS0_(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call { <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.sel.x2.nxv16i8(target("aarch64.svcount") [[PN:%.*]], <vscale x 16 x i8> [[ZN_COERCE0:%.*]], <vscale x 16 x i8> [[ZN_COERCE1:%.*]], <vscale x 16 x i8> [[ZM_COERCE0:%.*]], <vscale x 16 x i8> [[ZM_COERCE1:%.*]])
+// CPP-CHECK-NEXT:    ret { <vscale x 16 x i8>, <vscale x 16 x i8> } [[TMP0]]
+//
+svmfloat8x2_t test_svsel_mf8_x2(svcount_t pn, svmfloat8x2_t zn, svmfloat8x2_t zm) __arm_streaming {
+  return SVE_ACLE_FUNC(svsel,_mf8_x2)(pn, zn, zm);
+}
 // 16-bit SELs
 
 // CHECK-LABEL: @test_svsel_s16_x2(
diff --git a/clang/test/CodeGen/AArch64/sme2-intrinsics/acle_sme2_vector_selx4.c b/clang/test/CodeGen/AArch64/sme2-intrinsics/acle_sme2_vector_selx4.c
index 871d70943c9df..d4e77d998e3c2 100644
--- a/clang/test/CodeGen/AArch64/sme2-intrinsics/acle_sme2_vector_selx4.c
+++ b/clang/test/CodeGen/AArch64/sme2-intrinsics/acle_sme2_vector_selx4.c
@@ -33,29 +33,43 @@ svint8x4_t test_svsel_s8_x4(svcount_t pn, svint8x4_t zn1, svint8x4_t zn2) __arm_
 
 // CHECK-LABEL: @test_svsel_u8_x4(
 // CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP2:%.*]] = tail call { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.sel.x4.nxv16i8(target("aarch64.svcount") [[PN:%.*]], <vscale x 16 x i8> [[ZN1_COERCE0:%.*]], <vscale x 16 x i8> [[ZN1_COERCE1:%.*]], <vscale x 16 x i8> [[ZN1_COERCE2:%.*]], <vscale x 16 x i8> [[ZN1_COERCE3:%.*]], <vscale x 16 x i8> [[ZN2_COERCE0:%.*]], <vscale x 16 x i8> [[ZN2_COERCE1:%.*]], <vscale x 16 x i8> [[ZN2_COERCE2:%.*]], <vscale x 16 x i8> [[ZN2_COERCE3:%.*]])
+// CHECK-NEXT:    ret { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } [[TMP2]]
+//
+// CPP-CHECK-LABEL: @_Z16test_svsel_u8_x4u11__SVCount_t11svuint8x4_tS0_S0_S0_(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP2:%.*]] = tail call { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.sel.x4.nxv16i8(target("aarch64.svcount") [[PN:%.*]], <vscale x 16 x i8> [[ZN1_COERCE0:%.*]], <vscale x 16 x i8> [[ZN1_COERCE1:%.*]], <vscale x 16 x i8> [[ZN1_COERCE2:%.*]], <vscale x 16 x i8> [[ZN1_COERCE3:%.*]], <vscale x 16 x i8> [[ZN2_COERCE0:%.*]], <vscale x 16 x i8> [[ZN2_COERCE1:%.*]], <vscale x 16 x i8> [[ZN2_COERCE2:%.*]], <vscale x 16 x i8> [[ZN2_COERCE3:%.*]])
+// CPP-CHECK-NEXT:    ret { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } [[TMP2]]
+//
+svuint8x4_t test_svsel_u8_x4(svcount_t pn, svuint8x4_t zn1, svuint8x4_t zn2, svuint8x4_t zn3, svuint8x4_t zn4) __arm_streaming {
+  return SVE_ACLE_FUNC(svsel,_u8_x4)(pn, zn1, zn2);
+}
+
+// CHECK-LABEL: @test_svsel_mf8_x4(
+// CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.sel.x4.nxv16i8(target("aarch64.svcount") [[PN:%.*]], <vscale x 16 x i8> [[ZN1_COERCE0:%.*]], <vscale x 16 x i8> [[ZN1_COERCE1:%.*]], <vscale x 16 x i8> [[ZN1_COERCE2:%.*]], <vscale x 16 x i8> [[ZN1_COERCE3:%.*]], <vscale x 16 x i8> [[ZN2_COERCE0:%.*]], <vscale x 16 x i8> [[ZN2_COERCE1:%.*]], <vscale x 16 x i8> [[ZN2_COERCE2:%.*]], <vscale x 16 x i8> [[ZN2_COERCE3:%.*]])
 // CHECK-NEXT:    ret { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } [[TMP0]]
 //
-// CPP-CHECK-LABEL: @_Z16test_svsel_u8_x4u11__SVCount_t11svuint8x4_tS0_S0_S0_(
+// CPP-CHECK-LABEL: @_Z17test_svsel_mf8_x4u11__SVCount_t13svmfloat8x4_tS0_(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.sel.x4.nxv16i8(target("aarch64.svcount") [[PN:%.*]], <vscale x 16 x i8> [[ZN1_COERCE0:%.*]], <vscale x 16 x i8> [[ZN1_COERCE1:%.*]], <vscale x 16 x i8> [[ZN1_COERCE2:%.*]], <vscale x 16 x i8> [[ZN1_COERCE3:%.*]], <vscale x 16 x i8> [[ZN2_COERCE0:%.*]], <vscale x 16 x i8> [[ZN2_COERCE1:%.*]], <vscale x 16 x i8> [[ZN2_COERCE2:%.*]], <vscale x 16 x i8> [[ZN2_COERCE3:%.*]])
 // CPP-CHECK-NEXT:    ret { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } [[TMP0]]
 //
-svuint8x4_t test_svsel_u8_x4(svcount_t pn, svuint8x4_t zn1, svuint8x4_t zn2, svuint8x4_t zn3, svuint8x4_t zn4) __arm_streaming {
-  return SVE_ACLE_FUNC(svsel,_u8_x4)(pn, zn1, zn2);
+svmfloat8x4_t test_svsel_mf8_x4(svcount_t pn, svmfloat8x4_t zn1, svmfloat8x4_t zn2) __arm_streaming {
+  return SVE_ACLE_FUNC(svsel,_mf8_x4)(pn, zn1, zn2);
 }
 
 // 16-bit SELs
 
 // CHECK-LABEL: @test_svsel_s16_x4(
 // CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TMP0:%.*]] = tail call { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } @llvm.aarch64.sve.sel.x4.nxv8i16(target("aarch64.svcount") [[PN:%.*]], <vscale x 8 x i16> [[ZN1_COERCE0:%.*]], <vscale x 8 x i16> [[ZN1_COERCE1:%.*]], <vscale x 8 x i16> [[ZN1_COERCE2:%.*]], <vscale x 8 x i16> [[ZN1_COERCE3:%.*]], <vscale x 8 x i16> [[ZN2_COERCE0:%.*]], <vscale x 8 x i16> [[ZN2_COERCE1:%.*]], <vscale x 8 x i16> [[ZN2_COERCE2:%.*]], <vscale x 8 x i16> [[ZN2_COERCE3:%.*]])
-// CHECK-NEXT:    ret { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } [[TMP0]]
+// CHECK-NEXT:    [[TMP2:%.*]] = tail call { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } @llvm.aarch64.sve.sel.x4.nxv8i16(target("aarch64.svcount") [[PN:%.*]], <vscale x 8 x i16> [[ZN1_COERCE0:%.*]], <vscale x 8 x i16> [[ZN1_COERCE1:%.*]], <vscale x 8 x i16> [[ZN1_COERCE2:%.*]], <vscale x 8 x i16> [[ZN1_COERCE3:%.*]], <vscale x 8 x i16> [[ZN2_COERCE0:%.*]], <vscale x 8 x i16> [[ZN2_COERCE1:%.*]], <vscale x 8 x i16> [[ZN2_COERCE2:%.*]], <vscale x 8 x i16> [[ZN2_COERCE3:%.*]])
+// CHECK-NEXT:    ret { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } [[TMP2]]
 //
 // CPP-CHECK-LABEL: @_Z17test_svsel_s16_x4u11__SVCount_t11svint16x4_tS0_S0_S0_(
 // CPP-CHECK-NEXT:  entry:
-// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } @llvm.aarch64.sve.sel.x4.nxv8i16(target("aarch64.svcount") [[PN:%.*]], <vscale x 8 x i16> [[ZN1_COERCE0:%.*]], <vscale x 8 x i16> [[ZN1_COERCE1:%.*]], <vscale x 8 x i16> [[ZN1_COERCE2:%.*]], <vscale x 8 x i16> [[ZN1_COERCE3:%.*]], <vscale x 8 x i16> [[ZN2_COERCE0:%.*]], <vscale x 8 x i16> [[ZN2_COERCE1:%.*]], <vscale x 8 x i16> [[ZN2_COERCE2:%.*]], <vscale x 8 x i16> [[ZN2_COERCE3:%.*]])
-// CPP-CHECK-NEXT:    ret { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } [[TMP0]]
+// CPP-CHECK-NEXT:    [[TMP2:%.*]] = tail call { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } @llvm.aarch64.sve.sel.x4.nxv8i16(target("aarch64.svcount") [[PN:%.*]], <vscale x 8 x i16> [[ZN1_COERCE0:%.*]], <vscale x 8 x i16> [[ZN1_COERCE1:%.*]], <vscale x 8 x i16> [[ZN1_COERCE2:%.*]], <vscale x 8 x i16> [[ZN1_COERCE3:%.*]], <vscale x 8 x i16> [[ZN2_COERCE0:%.*]], <vscale x 8 x i16> [[ZN2_COERCE1:%.*]], <vscale x 8 x i16> [[ZN2_COERCE2:%.*]], <vscale x 8 x i16> [[ZN2_COERCE3:%.*]])
+// CPP-CHECK-NEXT:    ret { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } [[TMP2]]
 //
 svint16x4_t test_svsel_s16_x4(svcount_t pn, svint16x4_t zn1, svint16x4_t zn2, svint16x4_t zn3, svint16x4_t zn4) __arm_streaming {
   return SVE_ACLE_FUNC(svsel,_s16_x4)(pn, zn1, zn2);
@@ -63,13 +77,13 @@ svint16x4_t test_svsel_s16_x4(svcount_t pn, svint16x4_t zn1, svint16x4_t zn2, sv
 
 // CHECK-LABEL: @test_svsel_u16_x4(
 // CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TMP0:%.*]] = tail call { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } @llvm.aarch64.sve.sel.x4.nxv8i16(target("aarch64.svcount") [[PN:%.*]], <vscale x 8 x i16> [[ZN1_COERCE0:%.*]], <vscale x 8 x i16> [[ZN1_COERCE1:%.*]], <vscale x 8 x i16> [[ZN1_COERCE2:%.*]], <vscale x 8 x i16> [[ZN1_COERCE3:%.*]], <vscale x 8 x i16> [[ZN2_COERCE0:%.*]], <vscale x 8 x i16> [[ZN2_COERCE1:%.*]], <vscale x 8 x i16> [[ZN2_COERCE2:%.*]], <vscale x 8 x i16> [[ZN2_COERCE3:%.*]])
-// CHECK-NEXT:    ret { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } [[TMP0]]
+// CHECK-NEXT:    [[TMP2:%.*]] = tail call { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } @llvm.aarch64.sve.sel.x4.nxv8i16(target("aarch64.svcount") [[PN:%.*]], <vscale x 8 x i16> [[ZN1_COERCE0:%.*]], <vscale x 8 x i16> [[ZN1_COERCE1:%.*]], <vscale x 8 x i16> [[ZN1_COERCE2:%.*]], <vscale x 8 x i16> [[ZN1_COERCE3:%.*]], <vscale x 8 x i16> [[ZN2_COERCE0:%.*]], <vscale x 8 x i16> [[ZN2_COERCE1:%.*]], <vscale x 8 x i16> [[ZN2_COERCE2:%.*]], <vscale x 8 x i16> [[ZN2_COERCE3:%.*]])
+// CHECK-NEXT:    ret { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } [[TMP2]]
 //
 // CPP-CHECK-LABEL: @_Z17test_svsel_u16_x4u11__SVCount_t12svuint16x4_tS0_S0_S0_(
 // CPP-CHECK-NEXT:  entry:
-// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } @llvm.aarch64.sve.sel.x4.nxv8i16(target("aarch64.svcount") [[PN:%.*]], <vscale x 8 x i16> [[ZN1_COERCE0:%.*]], <vscale x 8 x i16> [[ZN1_COERCE1:%.*]], <vscale x 8 x i16> [[ZN1_COERCE2:%.*]], <vscale x 8 x i16> [[ZN1_COERCE3:%.*]], <vscale x 8 x i16> [[ZN2_COERCE0:%.*]], <vscale x 8 x i16> [[ZN2_COERCE1:%.*]], <vscale x 8 x i16> [[ZN2_COERCE2:%.*]], <vscale x 8 x i16> [[ZN2_COERCE3:%.*]])
-// CPP-CHECK-NEXT:    ret { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } [[TMP0]]
+// CPP-CHECK-NEXT:    [[TMP2:%.*]] = tail call { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } @llvm.aarch64.sve.sel.x4.nxv8i16(target("aarch64.svcount") [[PN:%.*]], <vscale x 8 x i16> [[ZN1_COERCE0:%.*]], <vscale x 8 x i16> [[ZN1_COERCE1:%.*]], <vscale x 8 x i16> [[ZN1_COERCE2:%.*]], <vscale x 8 x i16> [[ZN1_COERCE3:%.*]], <vscale x 8 x i16> [[ZN2_COERCE0:%.*]], <vscale x 8 x i16> [[ZN2_COERCE1:%.*]], <vscale x 8 x i16> [[ZN2_COERCE2:%.*]], <vscale x 8 x i16> [[ZN2_COERCE3:%.*]])
+// CPP-CHECK-NEXT:    ret { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } [[TMP2]]
 //
 svuint16x4_t test_svsel_u16_x4(svcount_t pn, svuint16x4_t zn1, svuint16x4_t zn2, svuint16x4_t zn3, svuint16x4_t zn4) __arm_streaming {
   return SVE_ACLE_FUNC(svsel,_u16_x4)(pn, zn1, zn2);
@@ -77,13 +91,13 @@ svuint16x4_t test_svsel_u16_x4(svcount_t pn, svuint16x4_t zn1, svuint16x4_t zn2,
 
 // CHECK-LABEL: @test_svsel_f16_x4(
 // CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TMP0:%.*]] = tail call { <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half> } @llvm.aarch64.sve.sel.x4.nxv8f16(target("aarch64.svcount") [[PN:%.*]], <vscale x 8 x half> [[ZN1_COERCE0:%.*]], <vscale x 8 x half> [[ZN1_COERCE1:%.*]], <vscale x 8 x half> [[ZN1_COERCE2:%.*]], <vscale x 8 x half> [[ZN1_COERCE3:%.*]], <vscale x 8 x half> [[ZN2_COERCE0:%.*]], <vscale x 8 x half> [[ZN2_COERCE1:%.*]], <vscale x 8 x half> [[ZN2_COERCE2:%.*]], <vscale x 8 x half> [[ZN2_COERCE3:%.*]])
-// CHECK-NEXT:    ret { <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half> } [[TMP0]]
+// CHECK-NEXT:    [[TMP2:%.*]] = tail call { <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half> } @llvm.aarch64.sve.sel.x4.nxv8f16(target("aarch64.svcount") [[PN:%.*]], <vscale x 8 x half> [[ZN1_COERCE0:%.*]], <vscale x 8 x half> [[ZN1_COERCE1:%.*]], <vscale x 8 x half> [[ZN1_COERCE2:%.*]], <vscale x 8 x half> [[ZN1_COERCE3:%.*]], <vscale x 8 x half> [[ZN2_COERCE0:%.*]], <vscale x 8 x half> [[ZN2_COERCE1:%.*]], <vscale x 8 x half> [[ZN2_COERCE2:%.*]], <vscale x 8 x half> [[ZN2_COERCE3:%.*]])
+// CHECK-NEXT:    ret { <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half> } [[TMP2]]
 //
 // CPP-CHECK-LABEL: @_Z17test_svsel_f16_x4u11__SVCount_t13svfloat16x4_tS0_S0_S0_(
 // CPP-CHECK-NEXT:  entry:
-// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call { <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half> } @llvm.aarch64.sve.sel.x4.nxv8f16(target("aarch64.svcount") [[PN:%.*]], <vscale x 8 x half> [[ZN1_COERCE0:%.*]], <vscale x 8 x half> [[ZN1_COERCE1:%.*]], <vscale x 8 x half> [[ZN1_COERCE2:%.*]], <vscale x 8 x half> [[ZN1_COERCE3:%.*]], <vscale x 8 x half> [[ZN2_COERCE0:%.*]], <vscale x 8 x half> [[ZN2_COERCE1:%.*]], <vscale x 8 x half> [[ZN2_COERCE2:%.*]], <vscale x 8 x half> [[ZN2_COERCE3:%.*]])
-// CPP-CHECK-NEXT:    ret { <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half> } [[TMP0]]
+// CPP-CHECK-NEXT:    [[TMP2:%.*]] = tail call { <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half> } @llvm.aarch64.sve.sel.x4.nxv8f16(target("aarch64.svcount") [[PN:%.*]], <vscale x 8 x half> [[ZN1_COERCE0:%.*]], <vscale x 8 x half> [[ZN1_COERCE1:%.*]], <vscale x 8 x half> [[ZN1_COERCE2:%.*]], <vscale x 8 x half> [[ZN1_COERCE3:%.*]], <vscale x 8 x half> [[ZN2_COERCE0:%.*]], <vscale x 8 x half> [[ZN2_COERCE1:%.*]], <vscale x 8 x half> [[ZN2_COERCE2:%.*]], <vscale x 8 x half> [[ZN2_COERCE3:%.*]])
+// CPP-CHECK-NEXT:    ret { <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half> } [[TMP2]]
 //
 svfloat16x4_t test_svsel_f16_x4(svcount_t pn, svfloat16x4_t zn1, svfloat16x4_t zn2, svfloat16x4_t zn3, svfloat16x4_t zn4) __arm_streaming {
   return SVE_ACLE_FUNC(svsel,_f16_x4)(pn, zn1, zn2);
@@ -91,13 +105,13 @@ svfloat16x4_t test_svsel_f16_x4(svcount_t pn, svfloat16x4_t zn1, svfloat16x4_t z
 
 // CHECK-LABEL: @test_svsel_bf16_x4(
 // CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TMP0:%.*]] = tail call { <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } @llvm.aarch64.sve.sel.x4.nxv8bf16(target("aarch64.svcount") [[PN:%.*]], <vscale x 8 x bfloat> [[ZN1_COERCE0:%.*]], <vscale x 8 x bfloat> [[ZN1_COERCE1:%.*]], <vscale x 8 x bfloat> [[ZN1_COERCE2:%.*]], <vscale x 8 x bfloat> [[ZN1_COERCE3:%.*]], <vscale x 8 x bfloat> [[ZN2_COERCE0:%.*]], <vscale x 8 x bfloat> [[ZN2_COERCE1:%.*]], <vscale x 8 x bfloat> [[ZN2_COERCE2:%.*]], <vscale x 8 x bfloat> [[ZN2_COERCE3:%.*]])
-// CHECK-NEXT:    ret { <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } [[TMP0]]
+// CHECK-NEXT:    [[TMP2:%.*]] = tail call { <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } @llvm.aarch64.sve.sel.x4.nxv8bf16(target("aarch64.svcount") [[PN:%.*]], <vscale x 8 x bfloat> [[ZN1_COERCE0:%.*]], <vscale x 8 x bfloat> [[ZN1_COERCE1:%.*]], <vscale x 8 x bfloat> [[ZN1_COERCE2:%.*]], <vscale x 8 x bfloat> [[ZN1_COERCE3:%.*]], <vscale x 8 x bfloat> [[ZN2_COERCE0:%.*]], <vscale x 8 x bfloat> [[ZN2_COERCE1:%.*]], <vscale x 8 x bfloat> [[ZN2_COERCE2:%.*]], <vscale x 8 x bfloat> [[ZN2_COERCE3:%.*]])
+// CHECK-NEXT:    ret { <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } [[TMP2]]
 //
 // CPP-CHECK-LABEL: @_Z18test_svsel_bf16_x4u11__SVCount_t14svbfloat16x4_tS0_S0_S0_(
 // CPP-CHECK-NEXT:  entry:
-// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call { <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } @llvm.aarch64.sve.sel.x4.nxv8bf16(target("aarch64.svcount") [[PN:%.*]], <vscale x 8 x bfloat> [[ZN1_COERCE0:%.*]], <vscale x 8 x bfloat> [[ZN1_COERCE1:%.*]], <vscale x 8 x bfloat> [[ZN1_COERCE2:%.*]], <vscale x 8 x bfloat> [[ZN1_COERCE3:%.*]], <vscale x 8 x bfloat> [[ZN2_COERCE0:%.*]], <vscale x 8 x bfloat> [[ZN2_COERCE1:%.*]], <vscale x 8 x bfloat> [[ZN2_COERCE2:%.*]], <vscale x 8 x bfloat> [[ZN2_COERCE3:%.*]])
-// CPP-CHECK-NEXT:    ret { <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } [[TMP0]]
+// CPP-CHECK-NEXT:    [[TMP2:%.*]] = tail call { <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } @llvm.aarch64.sve.sel.x4.nxv8bf16(target("aarch64.svcount") [[PN:%.*]], <vscale x 8 x bfloat> [[ZN1_COERCE0:%.*]], <vscale x 8 x bfloat> [[ZN1_COERCE1:%.*]], <vscale x 8 x bfloat> [[ZN1_COERCE2:%.*]], <vscale x 8 x bfloat> [[ZN1_COERCE3:%.*]], <vscale x 8 x bfloat> [[ZN2_COERCE0:%.*]], <vscale x 8 x bfloat> [[ZN2_COERCE1:%.*]], <vscale x 8 x bfloat> [[ZN2_COERCE2:%.*]], <vscale x 8 x bfloat> [[ZN2_COERCE3:%.*]])
+// CPP-CHECK-NEXT:    ret { <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } [[TMP2]]
 //
 svbfloat16x4_t test_svsel_bf16_x4(svcount_t pn, svbfloat16x4_t zn1, svbfloat16x4_t zn2, svbfloat16x4_t zn3, svbfloat16x4_t zn4) __arm_streaming {
   return SVE_ACLE_FUNC(svsel,_bf16_x4)(pn, zn1, zn2);
@@ -107,13 +121,13 @@ svbfloat16x4_t test_svsel_bf16_x4(svcount_t pn, svbfloat16x4_t zn1, svbfloat16x4
 
 // CHECK-LABEL: @test_svsel_s32_x4(
 // CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TMP0:%.*]] = tail call { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.aarch64.sve.sel.x4.nxv4i32(target("aarch64.svcount") [[PN:%.*]], <vscale x 4 x i32> [[ZN1_COERCE0:%.*]], <vscale x 4 x i32> [[ZN1_COERCE1:%.*]], <vscale x 4 x i32> [[ZN1_COERCE2:%.*]], <vscale x 4 x i32> [[ZN1_COERCE3:%.*]], <vscale x 4 x i32> [[ZN2_COERCE0:%.*]], <vscale x 4 x i32> [[ZN2_COERCE1:%.*]], <vscale x 4 x i32> [[ZN2_COERCE2:%.*]], <vscale x 4 x i32> [[ZN2_COERCE3:%.*]])
-// CHECK-NEXT:    ret { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[TMP0]]
+// CHECK-NEXT:    [[TMP2:%.*]] = tail call { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.aarch64.sve.sel.x4.nxv4i32(target("aarch64.svcount") [[PN:%.*]], <vscale x 4 x i32> [[ZN1_COERCE0:%.*]], <vscale x 4 x i32> [[ZN1_COERCE1:%.*]], <vscale x 4 x i32> [[ZN1_COERCE2:%.*]], <vscale x 4 x i32> [[ZN1_COERCE3:%.*]], <vscale x 4 x i32> [[ZN2_COERCE0:%.*]], <vscale x 4 x i32> [[ZN2_COERCE1:%.*]], <vscale x 4 x i32> [[ZN2_COERCE2:%.*]], <vscale x 4 x i32> [[ZN2_COERCE3:%.*]])
+// CHECK-NEXT:    ret { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[TMP2]]
 //
 // CPP-CHECK-LABEL: @_Z17test_svsel_s32_x4u11__SVCount_t11svint32x4_tS0_S0_S0_(
 // CPP-CHECK-NEXT:  entry:
-// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.aarch64.sve.sel.x4.nxv4i32(target("aarch64.svcount") [[PN:%.*]], <vscale x 4 x i32> [[ZN1_COERCE0:%.*]], <vscale x 4 x i32> [[ZN1_COERCE1:%.*]], <vscale x 4 x i32> [[ZN1_COERCE2:%.*]], <vscale x 4 x i32> [[ZN1_COERCE3:%.*]], <vscale x 4 x i32> [[ZN2_COERCE0:%.*]], <vscale x 4 x i32> [[ZN2_COERCE1:%.*]], <vscale x 4 x i32> [[ZN2_COERCE2:%.*]], <vscale x 4 x i32> [[ZN2_COERCE3:%.*]])
-// CPP-CHECK-NEXT:    ret { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[TMP0]]
+// CPP-CHECK-NEXT:    [[TMP2:%.*]] = tail call { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.aarch64.sve.sel.x4.nxv4i32(target("aarch64.svcount") [[PN:%.*]], <vscale x 4 x i32> [[ZN1_COERCE0:%.*]], <vscale x 4 x i32> [[ZN1_COERCE1:%.*]], <vscale x 4 x i32> [[ZN1_COERCE2:%.*]], <vscale x 4 x i32> [[ZN1_COERCE3:%.*]], <vscale x 4 x i32> [[ZN2_COERCE0:%.*]], <vscale x 4 x i32> [[ZN2_COERCE1:%.*]], <vscale x 4 x i32> [[ZN2_COERCE2:%.*]], <vscale x 4 x i32> [[ZN2_COERCE3:%.*]])
+// CPP-CHECK-NEXT:    ret { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[TMP2]]
 //
 svint32x4_t test_svsel_s32_x4(svcount_t pn, svint32x4_t zn1, svint32x4_t zn2, svint32x4_t zn3, svint32x4_t zn4) __arm_streaming {
   return SVE_ACLE_FUNC(svsel,_s32_x4)(pn, zn1, zn2);
@@ -121,13 +135,13 @@ svint32x4_t test_svsel_s32_x4(svcount_t pn, svint32x4_t zn1, svint32x4_t zn2, sv
 
 // CHECK-LABEL: @test_svsel_u32_x4(
 // CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TMP0:%.*]] = tail call { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.aarch64.sve.sel.x4.nxv4i32(target("aarch64.svcount") [[PN:%.*]], <vscale x 4 x i32> [[ZN1_COERCE0:%.*]], <vscale x 4 x i32> [[ZN1_COERCE1:%.*]], <vscale x 4 x i32> [[ZN1_COERCE2:%.*]], <vscale x 4 x i32> [[ZN1_COERCE3:%.*]], <vscale x 4 x i32> [[ZN2_COERCE0:%.*]], <vscale x 4 x i32> [[ZN2_COERCE1:%.*]], <vscale x 4 x i32> [[ZN2_COERCE2:%.*]], <vscale x 4 x i32> [[ZN2_COERCE3:%.*]])
-// CHECK-NEXT:    ret { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[TMP0]]
+// CHECK-NEXT:    [[TMP2:%.*]] = tail call { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.aarch64.sve.sel.x4.nxv4i32(target("aarch64.svcount") [[PN:%.*]], <vscale x 4 x i32> [[ZN1_COERCE0:%.*]], <vscale x 4 x i32> [[ZN1_COERCE1:%.*]], <vscale x 4 x i32> [[ZN1_COERCE2:%.*]], <vscale x 4 x i32> [[ZN1_COERCE3:%.*]], <vscale x 4 x i32> [[ZN2_COERCE0:%.*]], <vscale x 4 x i32> [[ZN2_COERCE1:%.*]], <vscale x 4 x i32> [[ZN2_COERCE2:%.*]], <vscale x 4 x i32> [[ZN2_COERCE3:%.*]])
+// CHECK-NEXT:    ret { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[TMP2]]
 //
 // CPP-CHECK-LABEL: @_Z17test_svsel_u32_x4u11__SVCount_t12svuint32x4_tS0_S0_S0_(
 // CPP-CHECK-NEXT:  entry:
-// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.aarch64.sve.sel.x4.nxv4i32(target("aarch64.svcount") [[PN:%.*]], <vscale x 4 x i32> [[ZN1_COERCE0:%.*]], <vscale x 4 x i32> [[ZN1_COERCE1:%.*]], <vscale x 4 x i32> [[ZN1_COERCE2:%.*]], <vscale x 4 x i32> [[ZN1_COERCE3:%.*]], <vscale x 4 x i32> [[ZN2_COERCE0:%.*]], <vscale x 4 x i32> [[ZN2_COERCE1:%.*]], <vscale x 4 x i32> [[ZN2_COERCE2:%.*]], <vscale x 4 x i32> [[ZN2_COERCE3:%.*]])
-// CPP-CHECK-NEXT:    ret { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[TMP0]]
+// CPP-CHECK-NEXT:    [[TMP2:%.*]] = tail call { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.aarch64.sve.sel.x4.nxv4i32(target("aarch64.svcount") [[PN:%.*]], <vscale x 4 x i32> [[ZN1_COERCE0:%.*]], <vscale x 4 x i32> [[ZN1_COERCE1:%.*]], <vscale x 4 x i32> [[ZN1_COERCE2:%.*]], <vscale x 4 x i32> [[ZN1_COERCE3:%.*]], <vscale x 4 x i32> [[ZN2_COERCE0:%.*]], <vscale x 4 x i32> [[ZN2_COERCE1:%.*]], <vscale x 4 x i32> [[ZN2_COERCE2:%.*]], <vscale x 4 x i32> [[ZN2_COERCE3:%.*]])
+// CPP-CHECK-NEXT:    ret { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[TMP2]]
 //
 svuint32x4_t test_svsel_u32_x4(svcount_t pn, svuint32x4_t zn1, svuint32x4_t zn2, svuint32x4_t zn3, svuint32x4_t zn4) __arm_streaming {
   return SVE_ACLE_FUNC(svsel,_u32_x4)(pn, zn1, zn2);
@@ -135,13 +149,13 @@ svuint32x4_t test_svsel_u32_x4(svcount_t pn, svuint32x4_t zn1, svuint32x4_t zn2,
 
 // CHECK-LABEL: @test_svsel_f32_x4(
 // CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TMP0:%.*]] = tail call { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } @llvm.aarch64.sve.sel.x4.nxv4f32(target("aarch64.svcount") [[PN:%.*]], <vscale x 4 x float> [[ZN1_COERCE0:%.*]], <vscale x 4 x float> [[ZN1_COERCE1:%.*]], <vscale x 4 x float> [[ZN1_COERCE2:%.*]], <vscale x 4 x float> [[ZN1_COERCE3:%.*]], <vscale x 4 x float> [[ZN2_COERCE0:%.*]], <vscale x 4 x float> [[ZN2_COERCE1:%.*]], <vscale x 4 x float> [[ZN2_COERCE2:%.*]], <vscale x 4 x float> [[ZN2_COERCE3:%.*]])
-// CHECK-NEXT:    ret { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } [[TMP0]]
+// CHECK-NEXT:    [[TMP2:%.*]] = tail call { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } @llvm.aarch64.sve.sel.x4.nxv4f32(target("aarch64.svcount") [[PN:%.*]], <vscale x 4 x float> [[ZN1_COERCE0:%.*]], <vscale x 4 x float> [[ZN1_COERCE1:%.*]], <vscale x 4 x float> [[ZN1_COERCE2:%.*]], <vscale x 4 x float> [[ZN1_COERCE3:%.*]], <vscale x 4 x float> [[ZN2_COERCE0:%.*]], <vscale x 4 x float> [[ZN2_COERCE1:%.*]], <vscale x 4 x float> [[ZN2_COERCE2:%.*]], <vscale x 4 x float> [[ZN2_COERCE3:%.*]])
+// CHECK-NEXT:    ret { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } [[TMP2]]
 //
 // CPP-CHECK-LABEL: @_Z17test_svsel_f32_x4u11__SVCount_t13svfloat32x4_tS0_S0_S0_(
 // CPP-CHECK-NEXT:  entry:
-// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } @llvm.aarch64.sve.sel.x4.nxv4f32(target("aarch64.svcount") [[PN:%.*]], <vscale x 4 x float> [[ZN1_COERCE0:%.*]], <vscale x 4 x float> [[ZN1_COERCE1:%.*]], <vscale x 4 x float> [[ZN1_COERCE2:%.*]], <vscale x 4 x float> [[ZN1_COERCE3:%.*]], <vscale x 4 x float> [[ZN2_COERCE0:%.*]], <vscale x 4 x float> [[ZN2_COERCE1:%.*]], <vscale x 4 x float> [[ZN2_COERCE2:%.*]], <vscale x 4 x float> [[ZN2_COERCE3:%.*]])
-// CPP-CHECK-NEXT:    ret { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } [[TMP0]]
+// CPP-CHECK-NEXT:    [[TMP2:%.*]] = tail call { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } @llvm.aarch64.sve.sel.x4.nxv4f32(target("aarch64.svcount") [[PN:%.*]], <vscale x 4 x float> [[ZN1_COERCE0:%.*]], <vscale x 4 x float> [[ZN1_COERCE1:%.*]], <vscale x 4 x float> [[ZN1_COERCE2:%.*]], <vscale x 4 x float> [[ZN1_COERCE3:%.*]], <vscale x 4 x float> [[ZN2_COERCE0:%.*]], <vscale x 4 x float> [[ZN2_COERCE1:%.*]], <vscale x 4 x float> [[ZN2_COERCE2:%.*]], <vscale x 4 x float> [[ZN2_COERCE3:%.*]])
+// CPP-CHECK-NEXT:    ret { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } [[TMP2]]
 //
 svfloat32x4_t test_svsel_f32_x4(svcount_t pn, svfloat32x4_t zn1, svfloat32x4_t zn2, svfloat32x4_t zn3, svfloat32x4_t zn4) __arm_streaming {
   return SVE_ACLE_FUNC(svsel,_f32_x4)(pn, zn1, zn2);
@@ -151,13 +165,13 @@ svfloat32x4_t test_svsel_f32_x4(svcount_t pn, svfloat32x4_t zn1, svfloat32x4_t z
 
 // CHECK-LABEL: @test_svsel_s64_x4(
 // CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TMP0:%.*]] = tail call { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } @llvm.aarch64.sve.sel.x4.nxv2i64(target("aarch64.svcount") [[PN:%.*]], <vscale x 2 x i64> [[ZN1_COERCE0:%.*]], <vscale x 2 x i64> [[ZN1_COERCE1:%.*]], <vscale x 2 x i64> [[ZN1_COERCE2:%.*]], <vscale x 2 x i64> [[ZN1_COERCE3:%.*]], <vscale x 2 x i64> [[ZN2_COERCE0:%.*]], <vscale x 2 x i64> [[ZN2_COERCE1:%.*]], <vscale x 2 x i64> [[ZN2_COERCE2:%.*]], <vscale x 2 x i64> [[ZN2_COERCE3:%.*]])
-// CHECK-NEXT:    ret { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } [[TMP0]]
+// CHECK-NEXT:    [[TMP2:%.*]] = tail call { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } @llvm.aarch64.sve.sel.x4.nxv2i64(target("aarch64.svcount") [[PN:%.*]], <vscale x 2 x i64> [[ZN1_COERCE0:%.*]], <vscale x 2 x i64> [[ZN1_COERCE1:%.*]], <vscale x 2 x i64> [[ZN1_COERCE2:%.*]], <vscale x 2 x i64> [[ZN1_COERCE3:%.*]], <vscale x 2 x i64> [[ZN2_COERCE0:%.*]], <vscale x 2 x i64> [[ZN2_COERCE1:%.*]], <vscale x 2 x i64> [[ZN2_COERCE2:%.*]], <vscale x 2 x i64> [[ZN2_COERCE3:%.*]])
+// CHECK-NEXT:    ret { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } [[TMP2]]
 //
 // CPP-CHECK-LABEL: @_Z17test_svsel_s64_x4u11__SVCount_t11svint64x4_tS0_S0_S0_(
 // CPP-CHECK-NEXT:  entry:
-// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } @llvm.aarch64.sve.sel.x4.nxv2i64(target("aarch64.svcount") [[PN:%.*]], <vscale x 2 x i64> [[ZN1_COERCE0:%.*]], <vscale x 2 x i64> [[ZN1_COERCE1:%.*]], <vscale x 2 x i64> [[ZN1_COERCE2:%.*]], <vscale x 2 x i64> [[ZN1_COERCE3:%.*]], <vscale x 2 x i64> [[ZN2_COERCE0:%.*]], <vscale x 2 x i64> [[ZN2_COERCE1:%.*]], <vscale x 2 x i64> [[ZN2_COERCE2:%.*]], <vscale x 2 x i64> [[ZN2_COERCE3:%.*]])
-// CPP-CHECK-NEXT:    ret { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } [[TMP0]]
+// CPP-CHECK-NEXT:    [[TMP2:%.*]] = tail call { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } @llvm.aarch64.sve.sel.x4.nxv2i64(target("aarch64.svcount") [[PN:%.*]], <vscale x 2 x i64> [[ZN1_COERCE0:%.*]], <vscale x 2 x i64> [[ZN1_COERCE1:%.*]], <vscale x 2 x i64> [[ZN1_COERCE2:%.*]], <vscale x 2 x i64> [[ZN1_COERCE3:%.*]], <vscale x 2 x i64> [[ZN2_COERCE0:%.*]], <vscale x 2 x i64> [[ZN2_COERCE1:%.*]], <vscale x 2 x i64> [[ZN2_COERCE2:%.*]], <vscale x 2 x i64> [[ZN2_COERCE3:%.*]])
+// CPP-CHECK-NEXT:    ret { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } [[TMP2]]
 //
 svint64x4_t test_svsel_s64_x4(svcount_t pn, svint64x4_t zn1, svint64x4_t zn2, svint64x4_t zn3, svint64x4_t zn4) __arm_streaming {
   return SVE_ACLE_FUNC(svsel,_s64_x4)(pn, zn1, zn2);
@@ -165,13 +179,13 @@ svint64x4_t test_svsel_s64_x4(svcount_t pn, svint64x4_t zn1, svint64x4_t zn2, sv
 
 // CHECK-LABEL: @test_svsel_u64_x4(
 // CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TMP0:%.*]] = tail call { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } @llvm.aarch64.sve.sel.x4.nxv2i64(target("aarch64.svcount") [[PN:%.*]], <vscale x 2 x i64> [[ZN1_COERCE0:%.*]], <vscale x 2 x i64> [[ZN1_COERCE1:%.*]], <vscale x 2 x i64> [[ZN1_COERCE2:%.*]], <vscale x 2 x i64> [[ZN1_COERCE3:%.*]], <vscale x 2 x i64> [[ZN2_COERCE0:%.*]], <vscale x 2 x i64> [[ZN2_COERCE1:%.*]], <vscale x 2 x i64> [[ZN2_COERCE2:%.*]], <vscale x 2 x i64> [[ZN2_COERCE3:%.*]])
-// CHECK-NEXT:    ret { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } [[TMP0]]
+// CHECK-NEXT:    [[TMP2:%.*]] = tail call { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } @llvm.aarch64.sve.sel.x4.nxv2i64(target("aarch64.svcount") [[PN:%.*]], <vscale x 2 x i64> [[ZN1_COERCE0:%.*]], <vscale x 2 x i64> [[ZN1_COERCE1:%.*]], <vscale x 2 x i64> [[ZN1_COERCE2:%.*]], <vscale x 2 x i64> [[ZN1_COERCE3:%.*]], <vscale x 2 x i64> [[ZN2_COERCE0:%.*]], <vscale x 2 x i64> [[ZN2_COERCE1:%.*]], <vscale x 2 x i64> [[ZN2_COERCE2:%.*]], <vscale x 2 x i64> [[ZN2_COERCE3:%.*]])
+// CHECK-NEXT:    ret { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } [[TMP2]]
 //
 // CPP-CHECK-LABEL: @_Z17test_svsel_u64_x4u11__SVCount_t12svuint64x4_tS0_S0_S0_(
 // CPP-CHECK-NEXT:  entry:
-// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } @llvm.aarch64.sve.sel.x4.nxv2i64(target("aarch64.svcount") [[PN:%.*]], <vscale x 2 x i64> [[ZN1_COERCE0:%.*]], <vscale x 2 x i64> [[ZN1_COERCE1:%.*]], <vscale x 2 x i64> [[ZN1_COERCE2:%.*]], <vscale x 2 x i64> [[ZN1_COERCE3:%.*]], <vscale x 2 x i64> [[ZN2_COERCE0:%.*]], <vscale x 2 x i64> [[ZN2_COERCE1:%.*]], <vscale x 2 x i64> [[ZN2_COERCE2:%.*]], <vscale x 2 x i64> [[ZN2_COERCE3:%.*]])
-// CPP-CHECK-NEXT:    ret { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } [[TMP0]]
+// CPP-CHECK-NEXT:    [[TMP2:%.*]] = tail call { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } @llvm.aarch64.sve.sel.x4.nxv2i64(target("aarch64.svcount") [[PN:%.*]], <vscale x 2 x i64> [[ZN1_COERCE0:%.*]], <vscale x 2 x i64> [[ZN1_COERCE1:%.*]], <vscale x 2 x i64> [[ZN1_COERCE2:%.*]], <vscale x 2 x i64> [[ZN1_COERCE3:%.*]], <vscale x 2 x i64> [[ZN2_COERCE0:%.*]], <vscale x 2 x i64> [[ZN2_COERCE1:%.*]], <vscale x 2 x i64> [[ZN2_COERCE2:%.*]], <vscale x 2 x i64> [[ZN2_COERCE3:%.*]])
+// CPP-CHECK-NEXT:    ret { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } [[TMP2]]
 //
 svuint64x4_t test_svsel_u64_x4(svcount_t pn, svuint64x4_t zn1, svuint64x4_t zn2, svuint64x4_t zn3, svuint64x4_t zn4) __arm_streaming {
   return SVE_ACLE_FUNC(svsel,_u64_x4)(pn, zn1, zn2);
@@ -179,13 +193,13 @@ svuint64x4_t test_svsel_u64_x4(svcount_t pn, svuint64x4_t zn1, svuint64x4_t zn2,
 
 // CHECK-LABEL: @test_svsel_f64_x4(
 // CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TMP0:%.*]] = tail call { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } @llvm.aarch64.sve.sel.x4.nxv2f64(target("aarch64.svcount") [[PN:%.*]], <vscale x 2 x double> [[ZN1_COERCE0:%.*]], <vscale x 2 x double> [[ZN1_COERCE1:%.*]], <vscale x 2 x double> [[ZN1_COERCE2:%.*]], <vscale x 2 x double> [[ZN1_COERCE3:%.*]], <vscale x 2 x double> [[ZN2_COERCE0:%.*]], <vscale x 2 x double> [[ZN2_COERCE1:%.*]], <vscale x 2 x double> [[ZN2_COERCE2:%.*]], <vscale x 2 x double> [[ZN2_COERCE3:%.*]])
-// CHECK-NEXT:    ret { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } [[TMP0]]
+// CHECK-NEXT:    [[TMP2:%.*]] = tail call { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } @llvm.aarch64.sve.sel.x4.nxv2f64(target("aarch64.svcount") [[PN:%.*]], <vscale x 2 x double> [[ZN1_COERCE0:%.*]], <vscale x 2 x double> [[ZN1_COERCE1:%.*]], <vscale x 2 x double> [[ZN1_COERCE2:%.*]], <vscale x 2 x double> [[ZN1_COERCE3:%.*]], <vscale x 2 x double> [[ZN2_COERCE0:%.*]], <vscale x 2 x double> [[ZN2_COERCE1:%.*]], <vscale x 2 x double> [[ZN2_COERCE2:%.*]], <vscale x 2 x double> [[ZN2_COERCE3:%.*]])
+// CHECK-NEXT:    ret { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } [[TMP2]]
 //
 // CPP-CHECK-LABEL: @_Z17test_svsel_f64_x4u11__SVCount_t13svfloat64x4_tS0_S0_S0_(
 // CPP-CHECK-NEXT:  entry:
-// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } @llvm.aarch64.sve.sel.x4.nxv2f64(target("aarch64.svcount") [[PN:%.*]], <vscale x 2 x double> [[ZN1_COERCE0:%.*]], <vscale x 2 x double> [[ZN1_COERCE1:%.*]], <vscale x 2 x double> [[ZN1_COERCE2:%.*]], <vscale x 2 x double> [[ZN1_COERCE3:%.*]], <vscale x 2 x double> [[ZN2_COERCE0:%.*]], <vscale x 2 x double> [[ZN2_COERCE1:%.*]], <vscale x 2 x double> [[ZN2_COERCE2:%.*]], <vscale x 2 x double> [[ZN2_COERCE3:%.*]])
-// CPP-CHECK-NEXT:    ret { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } [[TMP0]]
+// CPP-CHECK-NEXT:    [[TMP2:%.*]] = tail call { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } @llvm.aarch64.sve.sel.x4.nxv2f64(target("aarch64.svcount") [[PN:%.*]], <vscale x 2 x double> [[ZN1_COERCE0:%.*]], <vscale x 2 x double> [[ZN1_COERCE1:%.*]], <vscale x 2 x double> [[ZN1_COERCE2:%.*]], <vscale x 2 x double> [[ZN1_COERCE3:%.*]], <vscale x 2 x double> [[ZN2_COERCE0:%.*]], <vscale x 2 x double> [[ZN2_COERCE1:%.*]], <vscale x 2 x double> [[ZN2_COERCE2:%.*]], <vscale x 2 x double> [[ZN2_COERCE3:%.*]])
+// CPP-CHECK-NEXT:    ret { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } [[TMP2]]
 //
 svfloat64x4_t test_svsel_f64_x4(svcount_t pn, svfloat64x4_t zn1, svfloat64x4_t zn2, svfloat64x4_t zn3, svfloat64x4_t zn4) __arm_streaming {
   return SVE_ACLE_FUNC(svsel,_f64_x4)(pn, zn1, zn2);
diff --git a/clang/test/CodeGen/AArch64/sme2-intrinsics/acle_sme2_vector_uzpx2.c b/clang/test/CodeGen/AArch64/sme2-intrinsics/acle_sme2_vector_uzpx2.c
index 9a66ee5262082..ec3a2952b2ac6 100644
--- a/clang/test/CodeGen/AArch64/sme2-intrinsics/acle_sme2_vector_uzpx2.c
+++ b/clang/test/CodeGen/AArch64/sme2-intrinsics/acle_sme2_vector_uzpx2.c
@@ -46,6 +46,20 @@ svuint8x2_t test_svuzp_u8_x2(svuint8x2_t zn) __arm_streaming {
   return SVE_ACLE_FUNC(svuzp,_u8_x2)(zn);
 }
 
+// CHECK-LABEL: @test_svuzp_mf8_x2(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call { <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.uzp.x2.nxv16i8(<vscale x 16 x i8> [[ZN_COERCE0:%.*]], <vscale x 16 x i8> [[ZN_COERCE1:%.*]])
+// CHECK-NEXT:    ret { <vscale x 16 x i8>, <vscale x 16 x i8> } [[TMP0]]
+//
+// CPP-CHECK-LABEL: @_Z17test_svuzp_mf8_x213svmfloat8x2_t(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call { <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.uzp.x2.nxv16i8(<vscale x 16 x i8> [[ZN_COERCE0:%.*]], <vscale x 16 x i8> [[ZN_COERCE1:%.*]])
+// CPP-CHECK-NEXT:    ret { <vscale x 16 x i8>, <vscale x 16 x i8> } [[TMP0]]
+//
+svmfloat8x2_t test_svuzp_mf8_x2(svmfloat8x2_t zn) __arm_streaming {
+  return SVE_ACLE_FUNC(svuzp,_mf8_x2)(zn);
+}
+
 // 16-bit UZPs
 
 // CHECK-LABEL: @test_svuzp_s16_x2(
@@ -222,6 +236,20 @@ svuint8x2_t test_svuzpq_u8_x2(svuint8x2_t zn) __arm_streaming {
   return SVE_ACLE_FUNC(svuzpq,_u8_x2)(zn);
 }
 
+// CHECK-LABEL: @test_svuzpq_mf8_x2(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call { <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.uzpq.x2.nxv16i8(<vscale x 16 x i8> [[ZN_COERCE0:%.*]], <vscale x 16 x i8> [[ZN_COERCE1:%.*]])
+// CHECK-NEXT:    ret { <vscale x 16 x i8>, <vscale x 16 x i8> } [[TMP0]]
+//
+// CPP-CHECK-LABEL: @_Z18test_svuzpq_mf8_x213svmfloat8x2_t(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call { <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.uzpq.x2.nxv16i8(<vscale x 16 x i8> [[ZN_COERCE0:%.*]], <vscale x 16 x i8> [[ZN_COERCE1:%.*]])
+// CPP-CHECK-NEXT:    ret { <vscale x 16 x i8>, <vscale x 16 x i8> } [[TMP0]]
+//
+svmfloat8x2_t test_svuzpq_mf8_x2(svmfloat8x2_t zn) __arm_streaming {
+  return SVE_ACLE_FUNC(svuzpq,_mf8_x2)(zn);
+}
+
 // CHECK-LABEL: @test_svuzpq_s16_x2(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call { <vscale x 8 x i16>, <vscale x 8 x i16> } @llvm.aarch64.sve.uzpq.x2.nxv8i16(<vscale x 8 x i16> [[ZN_COERCE0:%.*]], <vscale x 8 x i16> [[ZN_COERCE1:%.*]])
diff --git a/clang/test/CodeGen/AArch64/sme2-intrinsics/acle_sme2_vector_uzpx4.c b/clang/test/CodeGen/AArch64/sme2-intrinsics/acle_sme2_vector_uzpx4.c
index 131928615edcd..aeac2ae78f6e6 100644
--- a/clang/test/CodeGen/AArch64/sme2-intrinsics/acle_sme2_vector_uzpx4.c
+++ b/clang/test/CodeGen/AArch64/sme2-intrinsics/acle_sme2_vector_uzpx4.c
@@ -46,6 +46,20 @@ svuint8x4_t test_svuzp_u8_x4(svuint8x4_t zn) __arm_streaming {
   return SVE_ACLE_FUNC(svuzp,_u8_x4)(zn);
 }
 
+// CHECK-LABEL: @test_svuzp_mf8_x4(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.uzp.x4.nxv16i8(<vscale x 16 x i8> [[ZN_COERCE0:%.*]], <vscale x 16 x i8> [[ZN_COERCE1:%.*]], <vscale x 16 x i8> [[ZN_COERCE2:%.*]], <vscale x 16 x i8> [[ZN_COERCE3:%.*]])
+// CHECK-NEXT:    ret { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } [[TMP0]]
+//
+// CPP-CHECK-LABEL: @_Z17test_svuzp_mf8_x413svmfloat8x4_t(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.uzp.x4.nxv16i8(<vscale x 16 x i8> [[ZN_COERCE0:%.*]], <vscale x 16 x i8> [[ZN_COERCE1:%.*]], <vscale x 16 x i8> [[ZN_COERCE2:%.*]], <vscale x 16 x i8> [[ZN_COERCE3:%.*]])
+// CPP-CHECK-NEXT:    ret { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } [[TMP0]]
+//
+svmfloat8x4_t test_svuzp_mf8_x4(svmfloat8x4_t zn) __arm_streaming {
+  return SVE_ACLE_FUNC(svuzp,_mf8_x4)(zn);
+}
+
 // 16-bit UZPs
 
 // CHECK-LABEL: @test_svuzp_s16_x4(
@@ -222,6 +236,20 @@ svuint8x4_t test_svuzpq_u8_x4(svuint8x4_t zn) __arm_streaming {
   return SVE_ACLE_FUNC(svuzpq,_u8_x4)(zn);
 }
 
+// CHECK-LABEL: @test_svuzpq_mf8_x4(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.uzpq.x4.nxv16i8(<vscale x 16 x i8> [[ZN_COERCE0:%.*]], <vscale x 16 x i8> [[ZN_COERCE1:%.*]], <vscale x 16 x i8> [[ZN_COERCE2:%.*]], <vscale x 16 x i8> [[ZN_COERCE3:%.*]])
+// CHECK-NEXT:    ret { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } [[TMP0]]
+//
+// CPP-CHECK-LABEL: @_Z18test_svuzpq_mf8_x413svmfloat8x4_t(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.uzpq.x4.nxv16i8(<vscale x 16 x i8> [[ZN_COERCE0:%.*]], <vscale x 16 x i8> [[ZN_COERCE1:%.*]], <vscale x 16 x i8> [[ZN_COERCE2:%.*]], <vscale x 16 x i8> [[ZN_COERCE3:%.*]])
+// CPP-CHECK-NEXT:    ret { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } [[TMP0]]
+//
+svmfloat8x4_t test_svuzpq_mf8_x4(svmfloat8x4_t zn) __arm_streaming {
+  return SVE_ACLE_FUNC(svuzpq,_mf8_x4)(zn);
+}
+
 // CHECK-LABEL: @test_svuzpq_s16_x4(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } @llvm.aarch64.sve.uzpq.x4.nxv8i16(<vscale x 8 x i16> [[ZN_COERCE0:%.*]], <vscale x 8 x i16> [[ZN_COERCE1:%.*]], <vscale x 8 x i16> [[ZN_COERCE2:%.*]], <vscale x 8 x i16> [[ZN_COERCE3:%.*]])
diff --git a/clang/test/CodeGen/AArch64/sme2-intrinsics/acle_sme2_vector_zipx2.c b/clang/test/CodeGen/AArch64/sme2-intrinsics/acle_sme2_vector_zipx2.c
index 787b7d0b3ea1a..735b3697f150b 100644
--- a/clang/test/CodeGen/AArch64/sme2-intrinsics/acle_sme2_vector_zipx2.c
+++ b/clang/test/CodeGen/AArch64/sme2-intrinsics/acle_sme2_vector_zipx2.c
@@ -45,6 +45,20 @@ svuint8x2_t test_svzip_u8_x2(svuint8x2_t zn) __arm_streaming {
   return SVE_ACLE_FUNC(svzip,_u8_x2)(zn);
 }
 
+// CHECK-LABEL: @test_svzip_mf8_x2(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call { <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.zip.x2.nxv16i8(<vscale x 16 x i8> [[ZN_COERCE0:%.*]], <vscale x 16 x i8> [[ZN_COERCE1:%.*]])
+// CHECK-NEXT:    ret { <vscale x 16 x i8>, <vscale x 16 x i8> } [[TMP0]]
+//
+// CPP-CHECK-LABEL: @_Z17test_svzip_mf8_x213svmfloat8x2_t(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call { <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.zip.x2.nxv16i8(<vscale x 16 x i8> [[ZN_COERCE0:%.*]], <vscale x 16 x i8> [[ZN_COERCE1:%.*]])
+// CPP-CHECK-NEXT:    ret { <vscale x 16 x i8>, <vscale x 16 x i8> } [[TMP0]]
+//
+svmfloat8x2_t test_svzip_mf8_x2(svmfloat8x2_t zn) __arm_streaming {
+  return SVE_ACLE_FUNC(svzip,_mf8_x2)(zn);
+}
+
 // 16-bit ZIPs
 
 // CHECK-LABEL: @test_svzip_s16_x2(
@@ -221,6 +235,20 @@ svuint8x2_t test_svzipq_u8_x2(svuint8x2_t zn) __arm_streaming {
   return SVE_ACLE_FUNC(svzipq,_u8_x2)(zn);
 }
 
+// CHECK-LABEL: @test_svzipq_mf8_x2(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call { <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.zipq.x2.nxv16i8(<vscale x 16 x i8> [[ZN_COERCE0:%.*]], <vscale x 16 x i8> [[ZN_COERCE1:%.*]])
+// CHECK-NEXT:    ret { <vscale x 16 x i8>, <vscale x 16 x i8> } [[TMP0]]
+//
+// CPP-CHECK-LABEL: @_Z18test_svzipq_mf8_x213svmfloat8x2_t(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call { <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.zipq.x2.nxv16i8(<vscale x 16 x i8> [[ZN_COERCE0:%.*]], <vscale x 16 x i8> [[ZN_COERCE1:%.*]])
+// CPP-CHECK-NEXT:    ret { <vscale x 16 x i8>, <vscale x 16 x i8> } [[TMP0]]
+//
+svmfloat8x2_t test_svzipq_mf8_x2(svmfloat8x2_t zn) __arm_streaming {
+  return SVE_ACLE_FUNC(svzipq,_mf8_x2)(zn);
+}
+
 // CHECK-LABEL: @test_svzipq_s16_x2(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call { <vscale x 8 x i16>, <vscale x 8 x i16> } @llvm.aarch64.sve.zipq.x2.nxv8i16(<vscale x 8 x i16> [[ZN_COERCE0:%.*]], <vscale x 8 x i16> [[ZN_COERCE1:%.*]])
diff --git a/clang/test/CodeGen/AArch64/sme2-intrinsics/acle_sme2_vector_zipx4.c b/clang/test/CodeGen/AArch64/sme2-intrinsics/acle_sme2_vector_zipx4.c
index 9bea471bc9837..341ae290e9b0e 100644
--- a/clang/test/CodeGen/AArch64/sme2-intrinsics/acle_sme2_vector_zipx4.c
+++ b/clang/test/CodeGen/AArch64/sme2-intrinsics/acle_sme2_vector_zipx4.c
@@ -45,6 +45,20 @@ svuint8x4_t test_svzip_u8_x4(svuint8x4_t zn) __arm_streaming {
   return SVE_ACLE_FUNC(svzip,_u8_x4)(zn);
 }
 
+// CHECK-LABEL: @test_svzip_mf8_x4(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.zip.x4.nxv16i8(<vscale x 16 x i8> [[ZN_COERCE0:%.*]], <vscale x 16 x i8> [[ZN_COERCE1:%.*]], <vscale x 16 x i8> [[ZN_COERCE2:%.*]], <vscale x 16 x i8> [[ZN_COERCE3:%.*]])
+// CHECK-NEXT:    ret { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } [[TMP0]]
+//
+// CPP-CHECK-LABEL: @_Z17test_svzip_mf8_x413svmfloat8x4_t(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.zip.x4.nxv16i8(<vscale x 16 x i8> [[ZN_COERCE0:%.*]], <vscale x 16 x i8> [[ZN_COERCE1:%.*]], <vscale x 16 x i8> [[ZN_COERCE2:%.*]], <vscale x 16 x i8> [[ZN_COERCE3:%.*]])
+// CPP-CHECK-NEXT:    ret { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } [[TMP0]]
+//
+svmfloat8x4_t test_svzip_mf8_x4(svmfloat8x4_t zn) __arm_streaming {
+  return SVE_ACLE_FUNC(svzip,_mf8_x4)(zn);
+}
+
 // 16-bit ZIPs
 
 // CHECK-LABEL: @test_svzip_s16_x4(
@@ -221,6 +235,20 @@ svuint8x4_t test_svzipq_u8_x4(svuint8x4_t zn) __arm_streaming {
   return SVE_ACLE_FUNC(svzipq,_u8_x4)(zn);
 }
 
+// CHECK-LABEL: @test_svzipq_mf8_x4(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.zipq.x4.nxv16i8(<vscale x 16 x i8> [[ZN_COERCE0:%.*]], <vscale x 16 x i8> [[ZN_COERCE1:%.*]], <vscale x 16 x i8> [[ZN_COERCE2:%.*]], <vscale x 16 x i8> [[ZN_COERCE3:%.*]])
+// CHECK-NEXT:    ret { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } [[TMP0]]
+//
+// CPP-CHECK-LABEL: @_Z18test_svzipq_mf8_x413svmfloat8x4_t(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.zipq.x4.nxv16i8(<vscale x 16 x i8> [[ZN_COERCE0:%.*]], <vscale x 16 x i8> [[ZN_COERCE1:%.*]], <vscale x 16 x i8> [[ZN_COERCE2:%.*]], <vscale x 16 x i8> [[ZN_COERCE3:%.*]])
+// CPP-CHECK-NEXT:    ret { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } [[TMP0]]
+//
+svmfloat8x4_t test_svzipq_mf8_x4(svmfloat8x4_t zn) __arm_streaming {
+  return SVE_ACLE_FUNC(svzipq,_mf8_x4)(zn);
+}
+
 // CHECK-LABEL: @test_svzipq_s16_x4(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } @llvm.aarch64.sve.zipq.x4.nxv8i16(<vscale x 8 x i16> [[ZN_COERCE0:%.*]], <vscale x 8 x i16> [[ZN_COERCE1:%.*]], <vscale x 8 x i16> [[ZN_COERCE2:%.*]], <vscale x 8 x i16> [[ZN_COERCE3:%.*]])
diff --git a/clang/test/CodeGen/AArch64/sme2-intrinsics/acle_sme2_write.c b/clang/test/CodeGen/AArch64/sme2-intrinsics/acle_sme2_write.c
index 1bdca4a12bcbd..7d0fbc9479a87 100644
--- a/clang/test/CodeGen/AArch64/sme2-intrinsics/acle_sme2_write.c
+++ b/clang/test/CodeGen/AArch64/sme2-intrinsics/acle_sme2_write.c
@@ -44,6 +44,20 @@ void test_svwrite_ver_za8_s8_vg2(uint32_t base, svint8x2_t val) __arm_streaming
   SVE_ACLE_FUNC(svwrite_ver_za8,_s8,_vg2,)(0, base, val);
 }
 
+// CHECK-LABEL: @test_svwrite_ver_za8_mf8_vg2(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    tail call void @llvm.aarch64.sme.write.ver.vg2.nxv16i8(i32 0, i32 [[BASE:%.*]], <vscale x 16 x i8> [[VAL_COERCE0:%.*]], <vscale x 16 x i8> [[VAL_COERCE1:%.*]])
+// CHECK-NEXT:    ret void
+//
+// CPP-CHECK-LABEL: @_Z28test_svwrite_ver_za8_mf8_vg2j13svmfloat8x2_t(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    tail call void @llvm.aarch64.sme.write.ver.vg2.nxv16i8(i32 0, i32 [[BASE:%.*]], <vscale x 16 x i8> [[VAL_COERCE0:%.*]], <vscale x 16 x i8> [[VAL_COERCE1:%.*]])
+// CPP-CHECK-NEXT:    ret void
+//
+void test_svwrite_ver_za8_mf8_vg2(uint32_t base, svmfloat8x2_t val) __arm_streaming __arm_inout("za") {
+  SVE_ACLE_FUNC(svwrite_ver_za8,_mf8,_vg2,)(0, base, val);
+}
+
 // CHECK-LABEL: @test_svwrite_hor_za8_u8_vg2(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    tail call void @llvm.aarch64.sme.write.hor.vg2.nxv16i8(i32 0, i32 [[BASE:%.*]], <vscale x 16 x i8> [[VAL_COERCE0:%.*]], <vscale x 16 x i8> [[VAL_COERCE1:%.*]])
@@ -72,6 +86,20 @@ void test_svwrite_hor_za8_s8_vg2(uint32_t base, svint8x2_t val) __arm_streaming
   SVE_ACLE_FUNC(svwrite_hor_za8,_s8,_vg2,)(0, base, val);
 }
 
+// CHECK-LABEL: @test_svwrite_hor_za8_mf8_vg2(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    tail call void @llvm.aarch64.sme.write.hor.vg2.nxv16i8(i32 0, i32 [[BASE:%.*]], <vscale x 16 x i8> [[VAL_COERCE0:%.*]], <vscale x 16 x i8> [[VAL_COERCE1:%.*]])
+// CHECK-NEXT:    ret void
+//
+// CPP-CHECK-LABEL: @_Z28test_svwrite_hor_za8_mf8_vg2j13svmfloat8x2_t(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    tail call void @llvm.aarch64.sme.write.hor.vg2.nxv16i8(i32 0, i32 [[BASE:%.*]], <vscale x 16 x i8> [[VAL_COERCE0:%.*]], <vscale x 16 x i8> [[VAL_COERCE1:%.*]])
+// CPP-CHECK-NEXT:    ret void
+//
+void test_svwrite_hor_za8_mf8_vg2(uint32_t base, svmfloat8x2_t val) __arm_streaming __arm_inout("za") {
+  SVE_ACLE_FUNC(svwrite_hor_za8,_mf8,_vg2,)(0, base, val);
+}
+
 // CHECK-LABEL: @test_svwrite_hor_za8_u8_vg4(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    tail call void @llvm.aarch64.sme.write.hor.vg4.nxv16i8(i32 0, i32 [[BASE:%.*]], <vscale x 16 x i8> [[VAL_COERCE0:%.*]], <vscale x 16 x i8> [[VAL_COERCE1:%.*]], <vscale x 16 x i8> [[VAL_COERCE2:%.*]], <vscale x 16 x i8> [[VAL_COERCE3:%.*]])
@@ -100,6 +128,20 @@ void test_svwrite_hor_za8_s8_vg4(uint32_t base, svint8x4_t val) __arm_streaming
   SVE_ACLE_FUNC(svwrite_hor_za8,_s8,_vg4,)(0, base, val);
 }
 
+// CHECK-LABEL: @test_svwrite_hor_za8_mf8_vg4(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    tail call void @llvm.aarch64.sme.write.hor.vg4.nxv16i8(i32 0, i32 [[BASE:%.*]], <vscale x 16 x i8> [[VAL_COERCE0:%.*]], <vscale x 16 x i8> [[VAL_COERCE1:%.*]], <vscale x 16 x i8> [[VAL_COERCE2:%.*]], <vscale x 16 x i8> [[VAL_COERCE3:%.*]])
+// CHECK-NEXT:    ret void
+//
+// CPP-CHECK-LABEL: @_Z28test_svwrite_hor_za8_mf8_vg4j13svmfloat8x4_t(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    tail call void @llvm.aarch64.sme.write.hor.vg4.nxv16i8(i32 0, i32 [[BASE:%.*]], <vscale x 16 x i8> [[VAL_COERCE0:%.*]], <vscale x 16 x i8> [[VAL_COERCE1:%.*]], <vscale x 16 x i8> [[VAL_COERCE2:%.*]], <vscale x 16 x i8> [[VAL_COERCE3:%.*]])
+// CPP-CHECK-NEXT:    ret void
+//
+void test_svwrite_hor_za8_mf8_vg4(uint32_t base, svmfloat8x4_t val) __arm_streaming __arm_inout("za") {
+  SVE_ACLE_FUNC(svwrite_hor_za8,_mf8,_vg4,)(0, base, val);
+}
+
 // CHECK-LABEL: @test_svwrite_ver_za8_u8_vg4(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    tail call void @llvm.aarch64.sme.write.ver.vg4.nxv16i8(i32 0, i32 [[BASE:%.*]], <vscale x 16 x i8> [[VAL_COERCE0:%.*]], <vscale x 16 x i8> [[VAL_COERCE1:%.*]], <vscale x 16 x i8> [[VAL_COERCE2:%.*]], <vscale x 16 x i8> [[VAL_COERCE3:%.*]])
@@ -128,6 +170,20 @@ void test_svwrite_ver_za8_s8_vg4(uint32_t base, svint8x4_t val) __arm_streaming
   SVE_ACLE_FUNC(svwrite_ver_za8,_s8,_vg4,)(0, base, val);
 }
 
+// CHECK-LABEL: @test_svwrite_ver_za8_mf8_vg4(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    tail call void @llvm.aarch64.sme.write.ver.vg4.nxv16i8(i32 0, i32 [[BASE:%.*]], <vscale x 16 x i8> [[VAL_COERCE0:%.*]], <vscale x 16 x i8> [[VAL_COERCE1:%.*]], <vscale x 16 x i8> [[VAL_COERCE2:%.*]], <vscale x 16 x i8> [[VAL_COERCE3:%.*]])
+// CHECK-NEXT:    ret void
+//
+// CPP-CHECK-LABEL: @_Z28test_svwrite_ver_za8_mf8_vg4j13svmfloat8x4_t(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    tail call void @llvm.aarch64.sme.write.ver.vg4.nxv16i8(i32 0, i32 [[BASE:%.*]], <vscale x 16 x i8> [[VAL_COERCE0:%.*]], <vscale x 16 x i8> [[VAL_COERCE1:%.*]], <vscale x 16 x i8> [[VAL_COERCE2:%.*]], <vscale x 16 x i8> [[VAL_COERCE3:%.*]])
+// CPP-CHECK-NEXT:    ret void
+//
+void test_svwrite_ver_za8_mf8_vg4(uint32_t base, svmfloat8x4_t val) __arm_streaming __arm_inout("za") {
+  SVE_ACLE_FUNC(svwrite_ver_za8,_mf8,_vg4,)(0, base, val);
+}
+
 // CHECK-LABEL: @test_svwrite_hor_za16_u16_vg2(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    tail call void @llvm.aarch64.sme.write.hor.vg2.nxv8i16(i32 1, i32 [[BASE:%.*]], <vscale x 8 x i16> [[VAL_COERCE0:%.*]], <vscale x 8 x i16> [[VAL_COERCE1:%.*]])
@@ -716,6 +772,20 @@ void test_svwrite_za8_u8_vg1x2(uint32_t base, svuint8x2_t val) __arm_streaming _
   SVE_ACLE_FUNC(svwrite_za8,_u8,_vg1x2,)(base, val);
 }
 
+// CHECK-LABEL: @test_svwrite_za8_mf8_vg1x2(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    tail call void @llvm.aarch64.sme.write.vg1x2.nxv16i8(i32 [[BASE:%.*]], <vscale x 16 x i8> [[VAL_COERCE0:%.*]], <vscale x 16 x i8> [[VAL_COERCE1:%.*]])
+// CHECK-NEXT:    ret void
+//
+// CPP-CHECK-LABEL: @_Z26test_svwrite_za8_mf8_vg1x2j13svmfloat8x2_t(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    tail call void @llvm.aarch64.sme.write.vg1x2.nxv16i8(i32 [[BASE:%.*]], <vscale x 16 x i8> [[VAL_COERCE0:%.*]], <vscale x 16 x i8> [[VAL_COERCE1:%.*]])
+// CPP-CHECK-NEXT:    ret void
+//
+void test_svwrite_za8_mf8_vg1x2(uint32_t base, svmfloat8x2_t val) __arm_streaming __arm_inout("za") {
+  SVE_ACLE_FUNC(svwrite_za8,_mf8,_vg1x2,)(base, val);
+}
+
 // CHECK-LABEL: @test_svwrite_za16_s16_vg1x2(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    tail call void @llvm.aarch64.sme.write.vg1x2.nxv8i16(i32 [[BASE:%.*]], <vscale x 8 x i16> [[VAL_COERCE0:%.*]], <vscale x 8 x i16> [[VAL_COERCE1:%.*]])
@@ -856,6 +926,20 @@ void test_svwrite_za64_s64_vg1x2(uint32_t base, svint64x2_t val) __arm_streaming
   SVE_ACLE_FUNC(svwrite_za64,_s64,_vg1x2,)(base, val);
 }
 
+// CHECK-LABEL: @test_svwrite_za8_mf8_vg1x4(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    tail call void @llvm.aarch64.sme.write.vg1x4.nxv16i8(i32 [[BASE:%.*]], <vscale x 16 x i8> [[VAL_COERCE0:%.*]], <vscale x 16 x i8> [[VAL_COERCE1:%.*]], <vscale x 16 x i8> [[VAL_COERCE2:%.*]], <vscale x 16 x i8> [[VAL_COERCE3:%.*]])
+// CHECK-NEXT:    ret void
+//
+// CPP-CHECK-LABEL: @_Z26test_svwrite_za8_mf8_vg1x4j13svmfloat8x4_t(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    tail call void @llvm.aarch64.sme.write.vg1x4.nxv16i8(i32 [[BASE:%.*]], <vscale x 16 x i8> [[VAL_COERCE0:%.*]], <vscale x 16 x i8> [[VAL_COERCE1:%.*]], <vscale x 16 x i8> [[VAL_COERCE2:%.*]], <vscale x 16 x i8> [[VAL_COERCE3:%.*]])
+// CPP-CHECK-NEXT:    ret void
+//
+void test_svwrite_za8_mf8_vg1x4(uint32_t base, svmfloat8x4_t val) __arm_streaming __arm_inout("za") {
+  SVE_ACLE_FUNC(svwrite_za8,_mf8,_vg1x4,)(base, val);
+}
+
 // CHECK-LABEL: @test_svwrite_za8_s8_vg1x4(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    tail call void @llvm.aarch64.sme.write.vg1x4.nxv16i8(i32 [[BASE:%.*]], <vscale x 16 x i8> [[VAL_COERCE0:%.*]], <vscale x 16 x i8> [[VAL_COERCE1:%.*]], <vscale x 16 x i8> [[VAL_COERCE2:%.*]], <vscale x 16 x i8> [[VAL_COERCE3:%.*]])
diff --git a/clang/test/CodeGen/AArch64/sme2p1-intrinsics/acle_sme2p1_movaz.c b/clang/test/CodeGen/AArch64/sme2p1-intrinsics/acle_sme2p1_movaz.c
index 7fa2249827c4e..98324e78b16bc 100644
--- a/clang/test/CodeGen/AArch64/sme2p1-intrinsics/acle_sme2p1_movaz.c
+++ b/clang/test/CodeGen/AArch64/sme2p1-intrinsics/acle_sme2p1_movaz.c
@@ -42,6 +42,23 @@ svuint8x2_t test_svreadz_hor_za8_u8_x2(uint32_t slice) __arm_streaming __arm_ino
    return svreadz_hor_za8_u8_vg2(0, slice);
 }
 
+// CHECK-LABEL: define dso_local { <vscale x 16 x i8>, <vscale x 16 x i8> } @test_svreadz_hor_za8_mf8_x2(
+// CHECK-SAME: i32 noundef [[SLICE:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call { <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sme.readz.horiz.x2.nxv16i8(i32 0, i32 [[SLICE]])
+// CHECK-NEXT:    ret { <vscale x 16 x i8>, <vscale x 16 x i8> } [[TMP0]]
+//
+// CPP-CHECK-LABEL: define dso_local { <vscale x 16 x i8>, <vscale x 16 x i8> } @_Z27test_svreadz_hor_za8_mf8_x2j(
+// CPP-CHECK-SAME: i32 noundef [[SLICE:%.*]]) #[[ATTR0]] {
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call { <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sme.readz.horiz.x2.nxv16i8(i32 0, i32 [[SLICE]])
+// CPP-CHECK-NEXT:    ret { <vscale x 16 x i8>, <vscale x 16 x i8> } [[TMP0]]
+//
+svmfloat8x2_t test_svreadz_hor_za8_mf8_x2(uint32_t slice) __arm_streaming __arm_inout("za")
+{
+   return svreadz_hor_za8_mf8_vg2(0, slice);
+}
+
 // CHECK-LABEL: define dso_local { <vscale x 8 x i16>, <vscale x 8 x i16> } @test_svreadz_hor_za16_s16_x2(
 // CHECK-SAME: i32 noundef [[SLICE:%.*]]) #[[ATTR0]] {
 // CHECK-NEXT:  entry:
@@ -252,6 +269,23 @@ svuint8x2_t test_svreadz_ver_za8_u8_x2(uint32_t slice) __arm_streaming __arm_ino
    return svreadz_ver_za8_u8_vg2(0, slice);
 }
 
+// CHECK-LABEL: define dso_local { <vscale x 16 x i8>, <vscale x 16 x i8> } @test_svreadz_ver_za8_mf8_x2(
+// CHECK-SAME: i32 noundef [[SLICE:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call { <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sme.readz.vert.x2.nxv16i8(i32 0, i32 [[SLICE]])
+// CHECK-NEXT:    ret { <vscale x 16 x i8>, <vscale x 16 x i8> } [[TMP0]]
+//
+// CPP-CHECK-LABEL: define dso_local { <vscale x 16 x i8>, <vscale x 16 x i8> } @_Z27test_svreadz_ver_za8_mf8_x2j(
+// CPP-CHECK-SAME: i32 noundef [[SLICE:%.*]]) #[[ATTR0]] {
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call { <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sme.readz.vert.x2.nxv16i8(i32 0, i32 [[SLICE]])
+// CPP-CHECK-NEXT:    ret { <vscale x 16 x i8>, <vscale x 16 x i8> } [[TMP0]]
+//
+svmfloat8x2_t test_svreadz_ver_za8_mf8_x2(uint32_t slice) __arm_streaming __arm_inout("za")
+{
+   return svreadz_ver_za8_mf8_vg2(0, slice);
+}
+
 // CHECK-LABEL: define dso_local { <vscale x 8 x i16>, <vscale x 8 x i16> } @test_svreadz_ver_za16_s16_x2(
 // CHECK-SAME: i32 noundef [[SLICE:%.*]]) #[[ATTR0]] {
 // CHECK-NEXT:  entry:
@@ -460,6 +494,23 @@ svuint8x4_t test_svreadz_hor_za8_u8_x4(uint32_t slice) __arm_streaming __arm_ino
    return svreadz_hor_za8_u8_vg4(0, slice);
 }
 
+// CHECK-LABEL: define dso_local { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @test_svreadz_hor_za8_mf8_x4(
+// CHECK-SAME: i32 noundef [[SLICE:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sme.readz.horiz.x4.nxv16i8(i32 0, i32 [[SLICE]])
+// CHECK-NEXT:    ret { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } [[TMP0]]
+//
+// CPP-CHECK-LABEL: define dso_local { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @_Z27test_svreadz_hor_za8_mf8_x4j(
+// CPP-CHECK-SAME: i32 noundef [[SLICE:%.*]]) #[[ATTR0]] {
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sme.readz.horiz.x4.nxv16i8(i32 0, i32 [[SLICE]])
+// CPP-CHECK-NEXT:    ret { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } [[TMP0]]
+//
+svmfloat8x4_t test_svreadz_hor_za8_mf8_x4(uint32_t slice) __arm_streaming __arm_inout("za")
+{
+   return svreadz_hor_za8_mf8_vg4(0, slice);
+}
+
 // CHECK-LABEL: define dso_local { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } @test_svreadz_hor_za16_s16_x4(
 // CHECK-SAME: i32 noundef [[SLICE:%.*]]) #[[ATTR0]] {
 // CHECK-NEXT:  entry:
@@ -667,6 +718,23 @@ svuint8x4_t test_svreadz_ver_za8_u8_x4(uint32_t slice) __arm_streaming __arm_ino
    return svreadz_ver_za8_u8_vg4(0, slice);
 }
 
+// CHECK-LABEL: define dso_local { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @test_svreadz_ver_za8_mf8_x4(
+// CHECK-SAME: i32 noundef [[SLICE:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sme.readz.vert.x4.nxv16i8(i32 0, i32 [[SLICE]])
+// CHECK-NEXT:    ret { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } [[TMP0]]
+//
+// CPP-CHECK-LABEL: define dso_local { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @_Z27test_svreadz_ver_za8_mf8_x4j(
+// CPP-CHECK-SAME: i32 noundef [[SLICE:%.*]]) #[[ATTR0]] {
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sme.readz.vert.x4.nxv16i8(i32 0, i32 [[SLICE]])
+// CPP-CHECK-NEXT:    ret { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } [[TMP0]]
+//
+svmfloat8x4_t test_svreadz_ver_za8_mf8_x4(uint32_t slice) __arm_streaming __arm_inout("za")
+{
+   return svreadz_ver_za8_mf8_vg4(0, slice);
+}
+
 // CHECK-LABEL: define dso_local { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } @test_svreadz_ver_za16_s16_x4(
 // CHECK-SAME: i32 noundef [[SLICE:%.*]]) #[[ATTR0]] {
 // CHECK-NEXT:  entry:
@@ -872,6 +940,23 @@ svuint8_t test_svreadz_hor_za8_u8(uint32_t slice) __arm_streaming __arm_inout("z
    return svreadz_hor_za8_u8(0, slice);
 }
 
+// CHECK-LABEL: define dso_local <vscale x 16 x i8> @test_svreadz_hor_za8_mf8(
+// CHECK-SAME: i32 noundef [[SLICE:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.aarch64.sme.readz.horiz.nxv16i8(i32 0, i32 [[SLICE]])
+// CHECK-NEXT:    ret <vscale x 16 x i8> [[TMP0]]
+//
+// CPP-CHECK-LABEL: define dso_local <vscale x 16 x i8> @_Z24test_svreadz_hor_za8_mf8j(
+// CPP-CHECK-SAME: i32 noundef [[SLICE:%.*]]) #[[ATTR0]] {
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.aarch64.sme.readz.horiz.nxv16i8(i32 0, i32 [[SLICE]])
+// CPP-CHECK-NEXT:    ret <vscale x 16 x i8> [[TMP0]]
+//
+svmfloat8_t test_svreadz_hor_za8_mf8(uint32_t slice) __arm_streaming __arm_inout("za")
+{
+   return svreadz_hor_za8_mf8(0, slice);
+}
+
 // CHECK-LABEL: define dso_local <vscale x 8 x i16> @test_svreadz_hor_za16_s16(
 // CHECK-SAME: i32 noundef [[SLICE:%.*]]) #[[ATTR0]] {
 // CHECK-NEXT:  entry:
@@ -1078,6 +1163,23 @@ svuint8_t test_svreadz_hor_za128_u8(uint32_t slice) __arm_streaming __arm_inout(
    return svreadz_hor_za128_u8(1, slice);
 }
 
+// CHECK-LABEL: define dso_local <vscale x 16 x i8> @test_svreadz_hor_za128_mf8(
+// CHECK-SAME: i32 noundef [[SLICE:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.aarch64.sme.readz.q.horiz.nxv16i8(i32 0, i32 [[SLICE]])
+// CHECK-NEXT:    ret <vscale x 16 x i8> [[TMP0]]
+//
+// CPP-CHECK-LABEL: define dso_local <vscale x 16 x i8> @_Z26test_svreadz_hor_za128_mf8j(
+// CPP-CHECK-SAME: i32 noundef [[SLICE:%.*]]) #[[ATTR0]] {
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.aarch64.sme.readz.q.horiz.nxv16i8(i32 0, i32 [[SLICE]])
+// CPP-CHECK-NEXT:    ret <vscale x 16 x i8> [[TMP0]]
+//
+svmfloat8_t test_svreadz_hor_za128_mf8(uint32_t slice) __arm_streaming __arm_inout("za")
+{
+   return svreadz_hor_za128_mf8(0, slice);
+}
+
 // CHECK-LABEL: define dso_local <vscale x 8 x i16> @test_svreadz_hor_za128_s16(
 // CHECK-SAME: i32 noundef [[SLICE:%.*]]) #[[ATTR0]] {
 // CHECK-NEXT:  entry:
@@ -1287,6 +1389,23 @@ svuint8x2_t test_svreadz_za8_u8_x2(uint32_t slice) __arm_streaming __arm_inout("
    return svreadz_za8_u8_vg1x2(slice);
 }
 
+// CHECK-LABEL: define dso_local { <vscale x 16 x i8>, <vscale x 16 x i8> } @test_svreadz_za8_mf8_x2(
+// CHECK-SAME: i32 noundef [[SLICE:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call { <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sme.readz.x2.nxv16i8(i32 [[SLICE]])
+// CHECK-NEXT:    ret { <vscale x 16 x i8>, <vscale x 16 x i8> } [[TMP0]]
+//
+// CPP-CHECK-LABEL: define dso_local { <vscale x 16 x i8>, <vscale x 16 x i8> } @_Z23test_svreadz_za8_mf8_x2j(
+// CPP-CHECK-SAME: i32 noundef [[SLICE:%.*]]) #[[ATTR0]] {
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call { <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sme.readz.x2.nxv16i8(i32 [[SLICE]])
+// CPP-CHECK-NEXT:    ret { <vscale x 16 x i8>, <vscale x 16 x i8> } [[TMP0]]
+//
+svmfloat8x2_t test_svreadz_za8_mf8_x2(uint32_t slice) __arm_streaming __arm_inout("za")
+{
+   return svreadz_za8_mf8_vg1x2(slice);
+}
+
 // CHECK-LABEL: define dso_local { <vscale x 8 x i16>, <vscale x 8 x i16> } @test_svreadz_za16_s16_x2(
 // CHECK-SAME: i32 noundef [[SLICE:%.*]]) #[[ATTR0]] {
 // CHECK-NEXT:  entry:
@@ -1495,6 +1614,23 @@ svuint8x4_t test_svreadz_za8_u8_x4(uint32_t slice) __arm_streaming __arm_inout("
    return svreadz_za8_u8_vg1x4(slice);
 }
 
+// CHECK-LABEL: define dso_local { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @test_svreadz_za8_mf8_x4(
+// CHECK-SAME: i32 noundef [[SLICE:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sme.readz.x4.nxv16i8(i32 [[SLICE]])
+// CHECK-NEXT:    ret { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } [[TMP0]]
+//
+// CPP-CHECK-LABEL: define dso_local { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @_Z23test_svreadz_za8_mf8_x4j(
+// CPP-CHECK-SAME: i32 noundef [[SLICE:%.*]]) #[[ATTR0]] {
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sme.readz.x4.nxv16i8(i32 [[SLICE]])
+// CPP-CHECK-NEXT:    ret { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } [[TMP0]]
+//
+svmfloat8x4_t test_svreadz_za8_mf8_x4(uint32_t slice) __arm_streaming __arm_inout("za")
+{
+   return svreadz_za8_mf8_vg1x4(slice);
+}
+
 // CHECK-LABEL: define dso_local { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } @test_svreadz_za16_s16_x4(
 // CHECK-SAME: i32 noundef [[SLICE:%.*]]) #[[ATTR0]] {
 // CHECK-NEXT:  entry:
diff --git a/clang/test/CodeGen/AArch64/sve-acle-__ARM_FEATURE_SVE_VECTOR_OPERATORS.c b/clang/test/CodeGen/AArch64/sve-acle-__ARM_FEATURE_SVE_VECTOR_OPERATORS.c
index 28d69d52c9ae7..bbed683ac1fd7 100644
--- a/clang/test/CodeGen/AArch64/sve-acle-__ARM_FEATURE_SVE_VECTOR_OPERATORS.c
+++ b/clang/test/CodeGen/AArch64/sve-acle-__ARM_FEATURE_SVE_VECTOR_OPERATORS.c
@@ -59,7 +59,7 @@ typedef int8_t vec_int8 __attribute__((vector_size(N / 8)));
 // CHECK128-NEXT:    ret <16 x i8> [[CASTFIXEDSVE]]
 
 // CHECK-LABEL: define{{.*}} void @f2(
-// CHECK-SAME:   ptr dead_on_unwind noalias nocapture writable writeonly sret(<[[#div(VBITS,8)]] x i8>) align 16 initializes((0, [[#div(VBITS,8)]])) %agg.result, ptr nocapture noundef readonly %0)
+// CHECK-SAME:   ptr dead_on_unwind noalias writable writeonly sret(<[[#div(VBITS,8)]] x i8>) align 16 captures(none) initializes((0, [[#div(VBITS,8)]])) %agg.result, ptr noundef readonly captures(none) %0)
 // CHECK-NEXT: entry:
 // CHECK-NEXT:   [[X:%.*]] = load <[[#div(VBITS,8)]] x i8>, ptr [[TMP0:%.*]], align 16, [[TBAA6:!tbaa !.*]]
 // CHECK-NEXT:   [[TMP1:%.*]] = tail call <vscale x 16 x i1> @llvm.aarch64.sve.ptrue.nxv16i1(i32 31)
diff --git a/clang/test/CodeGen/AArch64/sve2-intrinsics/acle_sve2_revd.c b/clang/test/CodeGen/AArch64/sve2-intrinsics/acle_sve2_revd.c
index b1121eb46e2a2..d947cf78660ff 100644
--- a/clang/test/CodeGen/AArch64/sve2-intrinsics/acle_sve2_revd.c
+++ b/clang/test/CodeGen/AArch64/sve2-intrinsics/acle_sve2_revd.c
@@ -397,6 +397,20 @@ svuint64_t test_svrevd_u64_x(svbool_t pg, svuint64_t op) MODE_ATTR {
 }
 
 
+// CHECK-LABEL: @test_svrevd_mf8_z(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.aarch64.sve.revd.nxv16i8(<vscale x 16 x i8> zeroinitializer, <vscale x 16 x i1> [[PG:%.*]], <vscale x 16 x i8> [[OP:%.*]])
+// CHECK-NEXT:    ret <vscale x 16 x i8> [[TMP0]]
+//
+// CPP-CHECK-LABEL: @_Z17test_svrevd_mf8_zu10__SVBool_tu13__SVMfloat8_t(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.aarch64.sve.revd.nxv16i8(<vscale x 16 x i8> zeroinitializer, <vscale x 16 x i1> [[PG:%.*]], <vscale x 16 x i8> [[OP:%.*]])
+// CPP-CHECK-NEXT:    ret <vscale x 16 x i8> [[TMP0]]
+//
+svmfloat8_t test_svrevd_mf8_z(svbool_t pg, svmfloat8_t op) MODE_ATTR {
+  return SVE_ACLE_FUNC(svrevd, _mf8, _z, )(pg, op);
+}
+
 // CHECK-LABEL: @test_svrevd_bf16_z(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
@@ -461,6 +475,20 @@ svfloat64_t test_svrevd_f64_z(svbool_t pg, svfloat64_t op) MODE_ATTR {
   return SVE_ACLE_FUNC(svrevd, _f64, _z, )(pg, op);
 }
 
+// CHECK-LABEL: @test_svrevd_mf8_m(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.aarch64.sve.revd.nxv16i8(<vscale x 16 x i8> [[INACTIVE:%.*]], <vscale x 16 x i1> [[PG:%.*]], <vscale x 16 x i8> [[OP:%.*]])
+// CHECK-NEXT:    ret <vscale x 16 x i8> [[TMP0]]
+//
+// CPP-CHECK-LABEL: @_Z17test_svrevd_mf8_mu13__SVMfloat8_tu10__SVBool_tS_(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.aarch64.sve.revd.nxv16i8(<vscale x 16 x i8> [[INACTIVE:%.*]], <vscale x 16 x i1> [[PG:%.*]], <vscale x 16 x i8> [[OP:%.*]])
+// CPP-CHECK-NEXT:    ret <vscale x 16 x i8> [[TMP0]]
+//
+svmfloat8_t test_svrevd_mf8_m(svmfloat8_t inactive, svbool_t pg, svmfloat8_t op) MODE_ATTR {
+  return SVE_ACLE_FUNC(svrevd, _mf8, _m, )(inactive, pg, op);
+}
+
 // CHECK-LABEL: @test_svrevd_bf16_m(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
@@ -525,6 +553,20 @@ svfloat64_t test_svrevd_f64_m(svfloat64_t inactive, svbool_t pg, svfloat64_t op)
   return SVE_ACLE_FUNC(svrevd, _f64, _m, )(inactive, pg, op);
 }
 
+// CHECK-LABEL: @test_svrevd_mf8_x(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.aarch64.sve.revd.nxv16i8(<vscale x 16 x i8> undef, <vscale x 16 x i1> [[PG:%.*]], <vscale x 16 x i8> [[OP:%.*]])
+// CHECK-NEXT:    ret <vscale x 16 x i8> [[TMP0]]
+//
+// CPP-CHECK-LABEL: @_Z17test_svrevd_mf8_xu10__SVBool_tu13__SVMfloat8_t(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.aarch64.sve.revd.nxv16i8(<vscale x 16 x i8> undef, <vscale x 16 x i1> [[PG:%.*]], <vscale x 16 x i8> [[OP:%.*]])
+// CPP-CHECK-NEXT:    ret <vscale x 16 x i8> [[TMP0]]
+//
+svmfloat8_t test_svrevd_mf8_x(svbool_t pg, svmfloat8_t op) MODE_ATTR {
+  return SVE_ACLE_FUNC(svrevd, _mf8, _x, )(pg, op);
+}
+
 // CHECK-LABEL: @test_svrevd_bf16_x(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
diff --git a/clang/test/CodeGen/PowerPC/aix-vaargs.c b/clang/test/CodeGen/PowerPC/aix-vaargs.c
index 724ba6560cdb9..8716a9e4698c4 100644
--- a/clang/test/CodeGen/PowerPC/aix-vaargs.c
+++ b/clang/test/CodeGen/PowerPC/aix-vaargs.c
@@ -68,8 +68,8 @@ void testva (int n, ...) {
 
 // CHECK: declare void @llvm.va_start.p0(ptr)
 
-// AIX32: declare void @llvm.memcpy.p0.p0.i32(ptr noalias nocapture writeonly, ptr noalias nocapture readonly, i32, i1 immarg)
-// AIX64: declare void @llvm.memcpy.p0.p0.i64(ptr noalias nocapture writeonly, ptr noalias nocapture readonly, i64, i1 immarg)
+// AIX32: declare void @llvm.memcpy.p0.p0.i32(ptr noalias writeonly captures(none), ptr noalias readonly captures(none), i32, i1 immarg)
+// AIX64: declare void @llvm.memcpy.p0.p0.i64(ptr noalias writeonly captures(none), ptr noalias readonly captures(none), i64, i1 immarg)
 
 // CHECK: declare void @llvm.va_copy.p0(ptr, ptr)
 // CHECK: declare void @llvm.va_end.p0(ptr)
diff --git a/clang/test/CodeGen/SystemZ/systemz-inline-asm.c b/clang/test/CodeGen/SystemZ/systemz-inline-asm.c
index 2a9d6a5f87454..9e62b8e107900 100644
--- a/clang/test/CodeGen/SystemZ/systemz-inline-asm.c
+++ b/clang/test/CodeGen/SystemZ/systemz-inline-asm.c
@@ -123,7 +123,7 @@ double test_f64(double f, double g) {
 long double test_f128(long double f, long double g) {
   asm("axbr %0, %2" : "=f" (f) : "0" (f), "f" (g));
   return f;
-// CHECK: define{{.*}} void @test_f128(ptr dead_on_unwind noalias nocapture writable writeonly sret(fp128) align 8 initializes((0, 16)) [[DEST:%.*]], ptr nocapture noundef readonly %0, ptr nocapture noundef readonly %1)
+// CHECK: define{{.*}} void @test_f128(ptr dead_on_unwind noalias writable writeonly sret(fp128) align 8 captures(none) initializes((0, 16)) [[DEST:%.*]], ptr noundef readonly captures(none) %0, ptr noundef readonly captures(none) %1)
 // CHECK: %f = load fp128, ptr %0
 // CHECK: %g = load fp128, ptr %1
 // CHECK: [[RESULT:%.*]] = tail call fp128 asm "axbr $0, $2", "=f,0,f"(fp128 %f, fp128 %g)
diff --git a/clang/test/CodeGen/X86/avx-cxx-record.cpp b/clang/test/CodeGen/X86/avx-cxx-record.cpp
new file mode 100644
index 0000000000000..bcd9c361fda90
--- /dev/null
+++ b/clang/test/CodeGen/X86/avx-cxx-record.cpp
@@ -0,0 +1,23 @@
+// RUN: %clang_cc1 %s -triple x86_64-unknown-linux-gnu -emit-llvm -O2 -target-cpu x86-64-v3 -o - | FileCheck %s
+
+using UInt64x2 = unsigned long long __attribute__((__vector_size__(16), may_alias));
+
+template<int id>
+struct XMM1 {
+    UInt64x2 x;
+};
+
+struct XMM2 : XMM1<0>, XMM1<1> {
+};
+
+// CHECK: define{{.*}} @_Z3foov({{.*}} [[ARG:%.*]]){{.*}}
+// CHECK: entry:
+// CHECK-NEXT: store {{.*}}, ptr [[ARG]]{{.*}}
+// CHECK-NEXT: [[TMP1:%.*]] = getelementptr {{.*}}, ptr [[ARG]]{{.*}}
+// CHECK-NEXT: store {{.*}}, ptr [[TMP1]]{{.*}}
+XMM2 foo() {
+  XMM2 result;
+  ((XMM1<0>*)&result)->x = UInt64x2{1, 2};
+  ((XMM1<1>*)&result)->x = UInt64x2{3, 4};
+  return result;
+}
diff --git a/clang/test/CodeGen/X86/avx10_2_512convert-builtins.c b/clang/test/CodeGen/X86/avx10_2_512convert-builtins.c
index 22503c640a727..dcf7bbc005a7c 100644
--- a/clang/test/CodeGen/X86/avx10_2_512convert-builtins.c
+++ b/clang/test/CodeGen/X86/avx10_2_512convert-builtins.c
@@ -201,22 +201,22 @@ __m512i test_mm512_maskz_cvts2ph_hf8(__mmask64 __U, __m512h __A, __m512h __B) {
   return _mm512_maskz_cvts2ph_hf8(__U, __A, __B);
 }
 
-__m512h test_mm512_cvthf8(__m256i __A) {
-  // CHECK-LABEL: @test_mm512_cvthf8(
+__m512h test_mm512_cvthf8_ph(__m256i __A) {
+  // CHECK-LABEL: @test_mm512_cvthf8_ph(
   // CHECK: call <32 x half> @llvm.x86.avx10.mask.vcvthf82ph512(
-  return _mm512_cvthf8(__A);
+  return _mm512_cvthf8_ph(__A);
 }
 
-__m512h test_mm512_mask_cvthf8(__m512h __A, __mmask32 __B, __m256i __C) {
-  // CHECK-LABEL: @test_mm512_mask_cvthf8(
+__m512h test_mm512_mask_cvthf8_ph(__m512h __A, __mmask32 __B, __m256i __C) {
+  // CHECK-LABEL: @test_mm512_mask_cvthf8_ph(
   // CHECK: call <32 x half> @llvm.x86.avx10.mask.vcvthf82ph512(
-  return _mm512_mask_cvthf8(__A, __B, __C);
+  return _mm512_mask_cvthf8_ph(__A, __B, __C);
 }
 
-__m512h test_mm512_maskz_cvthf8(__mmask32 __A, __m256i __B) {
-  // CHECK-LABEL: @test_mm512_maskz_cvthf8(
+__m512h test_mm512_maskz_cvthf8_ph(__mmask32 __A, __m256i __B) {
+  // CHECK-LABEL: @test_mm512_maskz_cvthf8_ph(
   // CHECK: call <32 x half> @llvm.x86.avx10.mask.vcvthf82ph512(
-  return _mm512_maskz_cvthf8(__A, __B);
+  return _mm512_maskz_cvthf8_ph(__A, __B);
 }
 
 __m256i test_mm512_cvtph_bf8(__m512h __A) {
diff --git a/clang/test/CodeGen/X86/avx10_2convert-builtins.c b/clang/test/CodeGen/X86/avx10_2convert-builtins.c
index efd9a31c40875..87fc6ffd7bc17 100644
--- a/clang/test/CodeGen/X86/avx10_2convert-builtins.c
+++ b/clang/test/CodeGen/X86/avx10_2convert-builtins.c
@@ -231,7 +231,7 @@ __m256i test_mm256_cvt2ph_bf8(__m256h __A, __m256h __B) {
   return _mm256_cvt2ph_bf8(__A, __B);
 }
 
-__m256i test_mm256_mask_cvt2ph_bf8(__m256i __W, __mmask16 __U, __m256h __A, __m256h __B) {
+__m256i test_mm256_mask_cvt2ph_bf8(__m256i __W, __mmask32 __U, __m256h __A, __m256h __B) {
   // CHECK-LABEL: @test_mm256_mask_cvt2ph_bf8(
   // CHECK: call <32 x i8> @llvm.x86.avx10.vcvt2ph2bf8256(
   // CHECK: select <32 x i1> %{{.*}}, <32 x i8> %{{.*}}, <32 x i8> %{{.*}}
@@ -239,7 +239,7 @@ __m256i test_mm256_mask_cvt2ph_bf8(__m256i __W, __mmask16 __U, __m256h __A, __m2
   return _mm256_mask_cvt2ph_bf8(__W, __U, __A, __B);
 }
 
-__m256i test_mm256_maskz_cvt2ph_bf8(__mmask16 __U, __m256h __A, __m256h __B) {
+__m256i test_mm256_maskz_cvt2ph_bf8(__mmask32 __U, __m256h __A, __m256h __B) {
   // CHECK-LABEL: @test_mm256_maskz_cvt2ph_bf8(
   // CHECK: call <32 x i8> @llvm.x86.avx10.vcvt2ph2bf8256(
   // CHECK: zeroinitializer
@@ -275,7 +275,7 @@ __m256i test_mm256_cvts2ph_bf8(__m256h __A, __m256h __B) {
   return _mm256_cvts2ph_bf8(__A, __B);
 }
 
-__m256i test_mm256_mask_cvts2ph_bf8(__m256i __W, __mmask16 __U, __m256h __A, __m256h __B) {
+__m256i test_mm256_mask_cvts2ph_bf8(__m256i __W, __mmask32 __U, __m256h __A, __m256h __B) {
   // CHECK-LABEL: @test_mm256_mask_cvts2ph_bf8(
   // CHECK: call <32 x i8> @llvm.x86.avx10.vcvt2ph2bf8s256(
   // CHECK: select <32 x i1> %{{.*}}, <32 x i8> %{{.*}}, <32 x i8> %{{.*}}
@@ -283,7 +283,7 @@ __m256i test_mm256_mask_cvts2ph_bf8(__m256i __W, __mmask16 __U, __m256h __A, __m
   return _mm256_mask_cvts2ph_bf8(__W, __U, __A, __B);
 }
 
-__m256i test_mm256_maskz_cvts2ph_bf8(__mmask16 __U, __m256h __A, __m256h __B) {
+__m256i test_mm256_maskz_cvts2ph_bf8(__mmask32 __U, __m256h __A, __m256h __B) {
   // CHECK-LABEL: @test_mm256_maskz_cvts2ph_bf8(
   // CHECK: call <32 x i8> @llvm.x86.avx10.vcvt2ph2bf8s256(
   // CHECK: zeroinitializer
@@ -319,7 +319,7 @@ __m256i test_mm256_cvt2ph_hf8(__m256h __A, __m256h __B) {
   return _mm256_cvt2ph_hf8(__A, __B);
 }
 
-__m256i test_mm256_mask_cvt2ph_hf8(__m256i __W, __mmask16 __U, __m256h __A, __m256h __B) {
+__m256i test_mm256_mask_cvt2ph_hf8(__m256i __W, __mmask32 __U, __m256h __A, __m256h __B) {
   // CHECK-LABEL: @test_mm256_mask_cvt2ph_hf8(
   // CHECK: call <32 x i8> @llvm.x86.avx10.vcvt2ph2hf8256(
   // CHECK: select <32 x i1> %{{.*}}, <32 x i8> %{{.*}}, <32 x i8> %{{.*}}
@@ -327,7 +327,7 @@ __m256i test_mm256_mask_cvt2ph_hf8(__m256i __W, __mmask16 __U, __m256h __A, __m2
   return _mm256_mask_cvt2ph_hf8(__W, __U, __A, __B);
 }
 
-__m256i test_mm256_maskz_cvt2ph_hf8(__mmask16 __U, __m256h __A, __m256h __B) {
+__m256i test_mm256_maskz_cvt2ph_hf8(__mmask32 __U, __m256h __A, __m256h __B) {
   // CHECK-LABEL: @test_mm256_maskz_cvt2ph_hf8(
   // CHECK: call <32 x i8> @llvm.x86.avx10.vcvt2ph2hf8256(
   // CHECK: zeroinitializer
@@ -363,7 +363,7 @@ __m256i test_mm256_cvts2ph_hf8(__m256h __A, __m256h __B) {
   return _mm256_cvts2ph_hf8(__A, __B);
 }
 
-__m256i test_mm256_mask_cvts2ph_hf8(__m256i __W, __mmask16 __U, __m256h __A, __m256h __B) {
+__m256i test_mm256_mask_cvts2ph_hf8(__m256i __W, __mmask32 __U, __m256h __A, __m256h __B) {
   // CHECK-LABEL: @test_mm256_mask_cvts2ph_hf8(
   // CHECK: call <32 x i8> @llvm.x86.avx10.vcvt2ph2hf8s256(
   // CHECK: select <32 x i1> %{{.*}}, <32 x i8> %{{.*}}, <32 x i8> %{{.*}}
@@ -371,7 +371,7 @@ __m256i test_mm256_mask_cvts2ph_hf8(__m256i __W, __mmask16 __U, __m256h __A, __m
   return _mm256_mask_cvts2ph_hf8(__W, __U, __A, __B);
 }
 
-__m256i test_mm256_maskz_cvts2ph_hf8(__mmask16 __U, __m256h __A, __m256h __B) {
+__m256i test_mm256_maskz_cvts2ph_hf8(__mmask32 __U, __m256h __A, __m256h __B) {
   // CHECK-LABEL: @test_mm256_maskz_cvts2ph_hf8(
   // CHECK: call <32 x i8> @llvm.x86.avx10.vcvt2ph2hf8s256(
   // CHECK: zeroinitializer
@@ -379,40 +379,40 @@ __m256i test_mm256_maskz_cvts2ph_hf8(__mmask16 __U, __m256h __A, __m256h __B) {
   return _mm256_maskz_cvts2ph_hf8(__U, __A, __B);
 }
 
-__m128h test_mm_cvthf8(__m128i __A) {
-  // CHECK-LABEL: @test_mm_cvthf8(
+__m128h test_mm_cvthf8_ph(__m128i __A) {
+  // CHECK-LABEL: @test_mm_cvthf8_ph(
   // CHECK: call <8 x half> @llvm.x86.avx10.mask.vcvthf82ph128(
-  return _mm_cvthf8(__A);
+  return _mm_cvthf8_ph(__A);
 }
 
-__m128h test_mm_mask_cvthf8(__m128h __A, __mmask8 __B, __m128i __C) {
-  // CHECK-LABEL: @test_mm_mask_cvthf8(
+__m128h test_mm_mask_cvthf8_ph(__m128h __A, __mmask8 __B, __m128i __C) {
+  // CHECK-LABEL: @test_mm_mask_cvthf8_ph(
   // CHECK: call <8 x half> @llvm.x86.avx10.mask.vcvthf82ph128(
-  return _mm_mask_cvthf8(__A, __B, __C);
+  return _mm_mask_cvthf8_ph(__A, __B, __C);
 }
 
-__m128h test_mm_maskz_cvthf8(__mmask8 __A, __m128i __B) {
-  // CHECK-LABEL: @test_mm_maskz_cvthf8(
+__m128h test_mm_maskz_cvthf8_ph(__mmask8 __A, __m128i __B) {
+  // CHECK-LABEL: @test_mm_maskz_cvthf8_ph(
   // CHECK: call <8 x half> @llvm.x86.avx10.mask.vcvthf82ph128(
-  return _mm_maskz_cvthf8(__A, __B);
+  return _mm_maskz_cvthf8_ph(__A, __B);
 }
 
-__m256h test_mm256_cvthf8(__m128i __A) {
-  // CHECK-LABEL: @test_mm256_cvthf8(
+__m256h test_mm256_cvthf8_ph(__m128i __A) {
+  // CHECK-LABEL: @test_mm256_cvthf8_ph(
   // CHECK: call <16 x half> @llvm.x86.avx10.mask.vcvthf82ph256(
-  return _mm256_cvthf8(__A);
+  return _mm256_cvthf8_ph(__A);
 }
 
-__m256h test_mm256_mask_cvthf8(__m256h __A, __mmask16 __B, __m128i __C) {
-  // CHECK-LABEL: @test_mm256_mask_cvthf8(
+__m256h test_mm256_mask_cvthf8_ph(__m256h __A, __mmask16 __B, __m128i __C) {
+  // CHECK-LABEL: @test_mm256_mask_cvthf8_ph(
   // CHECK: call <16 x half> @llvm.x86.avx10.mask.vcvthf82ph256(
-  return _mm256_mask_cvthf8(__A, __B, __C);
+  return _mm256_mask_cvthf8_ph(__A, __B, __C);
 }
 
-__m256h test_mm256_maskz_cvthf8(__mmask16 __A, __m128i __B) {
-  // CHECK-LABEL: @test_mm256_maskz_cvthf8(
+__m256h test_mm256_maskz_cvthf8_ph(__mmask16 __A, __m128i __B) {
+  // CHECK-LABEL: @test_mm256_maskz_cvthf8_ph(
   // CHECK: call <16 x half> @llvm.x86.avx10.mask.vcvthf82ph256(
-  return _mm256_maskz_cvthf8(__A, __B);
+  return _mm256_maskz_cvthf8_ph(__A, __B);
 }
 
 __m128i test_mm_cvtph_bf8(__m128h __A) {
diff --git a/clang/test/CodeGen/allow-ubsan-check-inline.c b/clang/test/CodeGen/allow-ubsan-check-inline.c
index 1de24ab90dac0..eed48cf15ecca 100644
--- a/clang/test/CodeGen/allow-ubsan-check-inline.c
+++ b/clang/test/CodeGen/allow-ubsan-check-inline.c
@@ -1,3 +1,8 @@
+// RUN: %clang_cc1 -triple x86_64-pc-linux-gnu -emit-llvm -o - %s -fsanitize=signed-integer-overflow -fsanitize-skip-hot-cutoff=signed-integer-overflow=0.000001 -O3 -mllvm -lower-allow-check-random-rate=1 -Rpass=lower-allow-check -Rpass-missed=lower-allow-check -fno-inline 2>&1 | FileCheck %s --check-prefixes=NOINL --implicit-check-not="remark:"
+// RUN: %clang_cc1 -triple x86_64-pc-linux-gnu -emit-llvm -o - %s -fsanitize=signed-integer-overflow -fsanitize-skip-hot-cutoff=signed-integer-overflow=0.000001 -O3 -mllvm -lower-allow-check-random-rate=1 -Rpass=lower-allow-check -Rpass-missed=lower-allow-check 2>&1 | FileCheck %s --check-prefixes=INLINE --implicit-check-not="remark:"
+//
+// -ubsan-guard-checks is deprecated and will be removed in the future;
+// use -fsanitize-skip-hot-cutoff, as shown above.
 // RUN: %clang_cc1 -triple x86_64-pc-linux-gnu -emit-llvm -o - %s -fsanitize=signed-integer-overflow -mllvm -ubsan-guard-checks -O3 -mllvm -lower-allow-check-random-rate=1 -Rpass=lower-allow-check -Rpass-missed=lower-allow-check -fno-inline 2>&1 | FileCheck %s --check-prefixes=NOINL --implicit-check-not="remark:"
 // RUN: %clang_cc1 -triple x86_64-pc-linux-gnu -emit-llvm -o - %s -fsanitize=signed-integer-overflow -mllvm -ubsan-guard-checks -O3 -mllvm -lower-allow-check-random-rate=1 -Rpass=lower-allow-check -Rpass-missed=lower-allow-check 2>&1 | FileCheck %s --check-prefixes=INLINE --implicit-check-not="remark:"
 
diff --git a/clang/test/CodeGen/allow-ubsan-check.c b/clang/test/CodeGen/allow-ubsan-check.c
index 38b4848c1edc1..0cd81a77f5cc5 100644
--- a/clang/test/CodeGen/allow-ubsan-check.c
+++ b/clang/test/CodeGen/allow-ubsan-check.c
@@ -1,4 +1,7 @@
 // NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --version 5
+//
+// We can't use -fsanitize-skip-hot-cutoff because that includes both -ubsan-guard-checks and
+//-lower-allow-check-percentile-cutoff.
 // RUN: %clang_cc1 -triple x86_64-pc-linux-gnu -emit-llvm -O1 -o - %s -fsanitize=signed-integer-overflow,integer-divide-by-zero,null,local-bounds -mllvm -ubsan-guard-checks | FileCheck %s
 // RUN: %clang_cc1 -triple x86_64-pc-linux-gnu -emit-llvm -O1 -o - %s -fsanitize=signed-integer-overflow,integer-divide-by-zero,null,local-bounds -mllvm -ubsan-guard-checks -fsanitize-trap=signed-integer-overflow,integer-divide-by-zero,null,local-bounds | FileCheck %s --check-prefixes=TR
 // RUN: %clang_cc1 -triple x86_64-pc-linux-gnu -emit-llvm -O1 -o - %s -fsanitize=signed-integer-overflow,integer-divide-by-zero,null,local-bounds -mllvm -ubsan-guard-checks -fsanitize-recover=signed-integer-overflow,integer-divide-by-zero,null,local-bounds | FileCheck %s --check-prefixes=REC
@@ -7,18 +10,26 @@
 // CHECK-LABEL: define dso_local noundef i32 @div(
 // CHECK-SAME: i32 noundef [[X:%.*]], i32 noundef [[Y:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
 // CHECK-NEXT:  [[ENTRY:.*:]]
-// CHECK-NEXT:    [[TMP0:%.*]] = icmp eq i32 [[Y]], 0, !nosanitize [[META2:![0-9]+]]
-// CHECK-NEXT:    [[TMP1:%.*]] = icmp eq i32 [[X]], -2147483648, !nosanitize [[META2]]
-// CHECK-NEXT:    [[TMP2:%.*]] = icmp eq i32 [[Y]], -1, !nosanitize [[META2]]
-// CHECK-NEXT:    [[OR_NOT5:%.*]] = and i1 [[TMP1]], [[TMP2]]
-// CHECK-NEXT:    [[DOTNOT3:%.*]] = or i1 [[TMP0]], [[OR_NOT5]]
-// CHECK-NEXT:    [[TMP3:%.*]] = tail call i1 @llvm.allow.ubsan.check(i8 3), !nosanitize [[META2]]
-// CHECK-NEXT:    [[DOTNOT1:%.*]] = and i1 [[DOTNOT3]], [[TMP3]]
-// CHECK-NEXT:    br i1 [[DOTNOT1]], label %[[HANDLER_DIVREM_OVERFLOW:.*]], label %[[CONT:.*]], !prof [[PROF3:![0-9]+]], !nosanitize [[META2]]
+// CHECK-NEXT:    [[TMP0:%.*]] = icmp ne i32 [[Y]], 0, !nosanitize [[META2:![0-9]+]]
+// CHECK-NEXT:    [[TMP1:%.*]] = icmp ne i32 [[X]], -2147483648, !nosanitize [[META2]]
+// CHECK-NEXT:    [[TMP2:%.*]] = icmp ne i32 [[Y]], -1, !nosanitize [[META2]]
+// CHECK-NEXT:    [[OR:%.*]] = or i1 [[TMP1]], [[TMP2]], !nosanitize [[META2]]
+//
+//                                                                       27 == SO_IntegerDivideByZero
+// CHECK-NEXT:    [[TMP3:%.*]] = tail call i1 @llvm.allow.ubsan.check(i8 27), !nosanitize [[META2]]
+// CHECK-NEXT:    [[TMP4:%.*]] = xor i1 [[TMP3]], true, !nosanitize [[META2]]
+// CHECK-NEXT:    [[TMP5:%.*]] = or i1 [[TMP0]], [[TMP4]], !nosanitize [[META2]]
+//
+//                                                                       41 == SO_SignedIntegerOverflow
+// CHECK-NEXT:    [[TMP6:%.*]] = tail call i1 @llvm.allow.ubsan.check(i8 41), !nosanitize [[META2]]
+// CHECK-NEXT:    [[TMP7:%.*]] = xor i1 [[TMP6]], true, !nosanitize [[META2]]
+// CHECK-NEXT:    [[TMP8:%.*]] = or i1 [[OR]], [[TMP7]], !nosanitize [[META2]]
+// CHECK-NEXT:    [[TMP9:%.*]] = and i1 [[TMP5]], [[TMP8]], !nosanitize [[META2]]
+// CHECK-NEXT:    br i1 [[TMP9]], label %[[CONT:.*]], label %[[HANDLER_DIVREM_OVERFLOW:.*]], !prof [[PROF3:![0-9]+]], !nosanitize [[META2]]
 // CHECK:       [[HANDLER_DIVREM_OVERFLOW]]:
-// CHECK-NEXT:    [[TMP4:%.*]] = zext i32 [[X]] to i64, !nosanitize [[META2]]
-// CHECK-NEXT:    [[TMP5:%.*]] = zext i32 [[Y]] to i64, !nosanitize [[META2]]
-// CHECK-NEXT:    tail call void @__ubsan_handle_divrem_overflow_abort(ptr nonnull @[[GLOB1:[0-9]+]], i64 [[TMP4]], i64 [[TMP5]]) #[[ATTR6:[0-9]+]], !nosanitize [[META2]]
+// CHECK-NEXT:    [[TMP10:%.*]] = zext i32 [[X]] to i64, !nosanitize [[META2]]
+// CHECK-NEXT:    [[TMP11:%.*]] = zext i32 [[Y]] to i64, !nosanitize [[META2]]
+// CHECK-NEXT:    tail call void @__ubsan_handle_divrem_overflow_abort(ptr nonnull @[[GLOB1:[0-9]+]], i64 [[TMP10]], i64 [[TMP11]]) #[[ATTR6:[0-9]+]], !nosanitize [[META2]]
 // CHECK-NEXT:    unreachable, !nosanitize [[META2]]
 // CHECK:       [[CONT]]:
 // CHECK-NEXT:    [[DIV:%.*]] = sdiv i32 [[X]], [[Y]]
@@ -27,14 +38,18 @@
 // TR-LABEL: define dso_local noundef i32 @div(
 // TR-SAME: i32 noundef [[X:%.*]], i32 noundef [[Y:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
 // TR-NEXT:  [[ENTRY:.*:]]
-// TR-NEXT:    [[TMP0:%.*]] = icmp eq i32 [[Y]], 0, !nosanitize [[META2:![0-9]+]]
-// TR-NEXT:    [[TMP1:%.*]] = icmp eq i32 [[X]], -2147483648, !nosanitize [[META2]]
-// TR-NEXT:    [[TMP2:%.*]] = icmp eq i32 [[Y]], -1, !nosanitize [[META2]]
-// TR-NEXT:    [[OR_NOT5:%.*]] = and i1 [[TMP1]], [[TMP2]]
-// TR-NEXT:    [[DOTNOT3:%.*]] = or i1 [[TMP0]], [[OR_NOT5]]
-// TR-NEXT:    [[TMP3:%.*]] = tail call i1 @llvm.allow.ubsan.check(i8 3), !nosanitize [[META2]]
-// TR-NEXT:    [[DOTNOT1:%.*]] = and i1 [[DOTNOT3]], [[TMP3]]
-// TR-NEXT:    br i1 [[DOTNOT1]], label %[[TRAP:.*]], label %[[CONT:.*]], !nosanitize [[META2]]
+// TR-NEXT:    [[TMP0:%.*]] = icmp ne i32 [[Y]], 0, !nosanitize [[META2:![0-9]+]]
+// TR-NEXT:    [[TMP1:%.*]] = icmp ne i32 [[X]], -2147483648, !nosanitize [[META2]]
+// TR-NEXT:    [[TMP2:%.*]] = icmp ne i32 [[Y]], -1, !nosanitize [[META2]]
+// TR-NEXT:    [[OR:%.*]] = or i1 [[TMP1]], [[TMP2]], !nosanitize [[META2]]
+// TR-NEXT:    [[TMP3:%.*]] = tail call i1 @llvm.allow.ubsan.check(i8 27), !nosanitize [[META2]]
+// TR-NEXT:    [[TMP4:%.*]] = xor i1 [[TMP3]], true, !nosanitize [[META2]]
+// TR-NEXT:    [[TMP5:%.*]] = or i1 [[TMP0]], [[TMP4]], !nosanitize [[META2]]
+// TR-NEXT:    [[TMP6:%.*]] = tail call i1 @llvm.allow.ubsan.check(i8 41), !nosanitize [[META2]]
+// TR-NEXT:    [[TMP7:%.*]] = xor i1 [[TMP6]], true, !nosanitize [[META2]]
+// TR-NEXT:    [[TMP8:%.*]] = or i1 [[OR]], [[TMP7]], !nosanitize [[META2]]
+// TR-NEXT:    [[TMP9:%.*]] = and i1 [[TMP5]], [[TMP8]], !nosanitize [[META2]]
+// TR-NEXT:    br i1 [[TMP9]], label %[[CONT:.*]], label %[[TRAP:.*]], !nosanitize [[META2]]
 // TR:       [[TRAP]]:
 // TR-NEXT:    tail call void @llvm.ubsantrap(i8 3) #[[ATTR5:[0-9]+]], !nosanitize [[META2]]
 // TR-NEXT:    unreachable, !nosanitize [[META2]]
@@ -45,18 +60,22 @@
 // REC-LABEL: define dso_local noundef i32 @div(
 // REC-SAME: i32 noundef [[X:%.*]], i32 noundef [[Y:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
 // REC-NEXT:  [[ENTRY:.*:]]
-// REC-NEXT:    [[TMP0:%.*]] = icmp eq i32 [[Y]], 0, !nosanitize [[META2:![0-9]+]]
-// REC-NEXT:    [[TMP1:%.*]] = icmp eq i32 [[X]], -2147483648, !nosanitize [[META2]]
-// REC-NEXT:    [[TMP2:%.*]] = icmp eq i32 [[Y]], -1, !nosanitize [[META2]]
-// REC-NEXT:    [[OR_NOT5:%.*]] = and i1 [[TMP1]], [[TMP2]]
-// REC-NEXT:    [[DOTNOT3:%.*]] = or i1 [[TMP0]], [[OR_NOT5]]
-// REC-NEXT:    [[TMP3:%.*]] = tail call i1 @llvm.allow.ubsan.check(i8 3), !nosanitize [[META2]]
-// REC-NEXT:    [[DOTNOT1:%.*]] = and i1 [[DOTNOT3]], [[TMP3]]
-// REC-NEXT:    br i1 [[DOTNOT1]], label %[[HANDLER_DIVREM_OVERFLOW:.*]], label %[[CONT:.*]], !prof [[PROF3:![0-9]+]], !nosanitize [[META2]]
+// REC-NEXT:    [[TMP0:%.*]] = icmp ne i32 [[Y]], 0, !nosanitize [[META2:![0-9]+]]
+// REC-NEXT:    [[TMP1:%.*]] = icmp ne i32 [[X]], -2147483648, !nosanitize [[META2]]
+// REC-NEXT:    [[TMP2:%.*]] = icmp ne i32 [[Y]], -1, !nosanitize [[META2]]
+// REC-NEXT:    [[OR:%.*]] = or i1 [[TMP1]], [[TMP2]], !nosanitize [[META2]]
+// REC-NEXT:    [[TMP3:%.*]] = tail call i1 @llvm.allow.ubsan.check(i8 27), !nosanitize [[META2]]
+// REC-NEXT:    [[TMP4:%.*]] = xor i1 [[TMP3]], true, !nosanitize [[META2]]
+// REC-NEXT:    [[TMP5:%.*]] = or i1 [[TMP0]], [[TMP4]], !nosanitize [[META2]]
+// REC-NEXT:    [[TMP6:%.*]] = tail call i1 @llvm.allow.ubsan.check(i8 41), !nosanitize [[META2]]
+// REC-NEXT:    [[TMP7:%.*]] = xor i1 [[TMP6]], true, !nosanitize [[META2]]
+// REC-NEXT:    [[TMP8:%.*]] = or i1 [[OR]], [[TMP7]], !nosanitize [[META2]]
+// REC-NEXT:    [[TMP9:%.*]] = and i1 [[TMP5]], [[TMP8]], !nosanitize [[META2]]
+// REC-NEXT:    br i1 [[TMP9]], label %[[CONT:.*]], label %[[HANDLER_DIVREM_OVERFLOW:.*]], !prof [[PROF3:![0-9]+]], !nosanitize [[META2]]
 // REC:       [[HANDLER_DIVREM_OVERFLOW]]:
-// REC-NEXT:    [[TMP4:%.*]] = zext i32 [[X]] to i64, !nosanitize [[META2]]
-// REC-NEXT:    [[TMP5:%.*]] = zext i32 [[Y]] to i64, !nosanitize [[META2]]
-// REC-NEXT:    tail call void @__ubsan_handle_divrem_overflow(ptr nonnull @[[GLOB1:[0-9]+]], i64 [[TMP4]], i64 [[TMP5]]) #[[ATTR6:[0-9]+]], !nosanitize [[META2]]
+// REC-NEXT:    [[TMP10:%.*]] = zext i32 [[X]] to i64, !nosanitize [[META2]]
+// REC-NEXT:    [[TMP11:%.*]] = zext i32 [[Y]] to i64, !nosanitize [[META2]]
+// REC-NEXT:    tail call void @__ubsan_handle_divrem_overflow(ptr nonnull @[[GLOB1:[0-9]+]], i64 [[TMP10]], i64 [[TMP11]]) #[[ATTR6:[0-9]+]], !nosanitize [[META2]]
 // REC-NEXT:    br label %[[CONT]], !nosanitize [[META2]]
 // REC:       [[CONT]]:
 // REC-NEXT:    [[DIV:%.*]] = sdiv i32 [[X]], [[Y]]
@@ -70,21 +89,23 @@ int div(int x, int y) {
 // CHECK-SAME: ptr noundef readonly [[X:%.*]]) local_unnamed_addr #[[ATTR0]] {
 // CHECK-NEXT:  [[ENTRY:.*:]]
 // CHECK-NEXT:    [[TMP0:%.*]] = icmp eq ptr [[X]], null, !nosanitize [[META2]]
-// CHECK-NEXT:    [[TMP1:%.*]] = tail call i1 @llvm.allow.ubsan.check(i8 22), !nosanitize [[META2]]
+//
+//                                                                       29 == SO_Null
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call i1 @llvm.allow.ubsan.check(i8 29), !nosanitize [[META2]]
 // CHECK-NEXT:    [[DOTNOT1:%.*]] = and i1 [[TMP0]], [[TMP1]]
-// CHECK-NEXT:    br i1 [[DOTNOT1]], label %[[HANDLER_TYPE_MISMATCH:.*]], label %[[CONT:.*]], !prof [[PROF3]], !nosanitize [[META2]]
+// CHECK-NEXT:    br i1 [[DOTNOT1]], label %[[HANDLER_TYPE_MISMATCH:.*]], label %[[CONT:.*]], !prof [[PROF4:![0-9]+]], !nosanitize [[META2]]
 // CHECK:       [[HANDLER_TYPE_MISMATCH]]:
 // CHECK-NEXT:    tail call void @__ubsan_handle_type_mismatch_v1_abort(ptr nonnull @[[GLOB2:[0-9]+]], i64 0) #[[ATTR6]], !nosanitize [[META2]]
 // CHECK-NEXT:    unreachable, !nosanitize [[META2]]
 // CHECK:       [[CONT]]:
-// CHECK-NEXT:    [[TMP2:%.*]] = load i32, ptr [[X]], align 4, !tbaa [[TBAA4:![0-9]+]]
+// CHECK-NEXT:    [[TMP2:%.*]] = load i32, ptr [[X]], align 4, !tbaa [[TBAA5:![0-9]+]]
 // CHECK-NEXT:    ret i32 [[TMP2]]
 //
 // TR-LABEL: define dso_local i32 @null(
 // TR-SAME: ptr noundef readonly [[X:%.*]]) local_unnamed_addr #[[ATTR0]] {
 // TR-NEXT:  [[ENTRY:.*:]]
 // TR-NEXT:    [[TMP0:%.*]] = icmp eq ptr [[X]], null, !nosanitize [[META2]]
-// TR-NEXT:    [[TMP1:%.*]] = tail call i1 @llvm.allow.ubsan.check(i8 22), !nosanitize [[META2]]
+// TR-NEXT:    [[TMP1:%.*]] = tail call i1 @llvm.allow.ubsan.check(i8 29), !nosanitize [[META2]]
 // TR-NEXT:    [[DOTNOT1:%.*]] = and i1 [[TMP0]], [[TMP1]]
 // TR-NEXT:    br i1 [[DOTNOT1]], label %[[TRAP:.*]], label %[[CONT:.*]], !nosanitize [[META2]]
 // TR:       [[TRAP]]:
@@ -98,14 +119,14 @@ int div(int x, int y) {
 // REC-SAME: ptr noundef readonly [[X:%.*]]) local_unnamed_addr #[[ATTR0]] {
 // REC-NEXT:  [[ENTRY:.*:]]
 // REC-NEXT:    [[TMP0:%.*]] = icmp eq ptr [[X]], null, !nosanitize [[META2]]
-// REC-NEXT:    [[TMP1:%.*]] = tail call i1 @llvm.allow.ubsan.check(i8 22), !nosanitize [[META2]]
+// REC-NEXT:    [[TMP1:%.*]] = tail call i1 @llvm.allow.ubsan.check(i8 29), !nosanitize [[META2]]
 // REC-NEXT:    [[DOTNOT1:%.*]] = and i1 [[TMP0]], [[TMP1]]
-// REC-NEXT:    br i1 [[DOTNOT1]], label %[[HANDLER_TYPE_MISMATCH:.*]], label %[[CONT:.*]], !prof [[PROF3]], !nosanitize [[META2]]
+// REC-NEXT:    br i1 [[DOTNOT1]], label %[[HANDLER_TYPE_MISMATCH:.*]], label %[[CONT:.*]], !prof [[PROF4:![0-9]+]], !nosanitize [[META2]]
 // REC:       [[HANDLER_TYPE_MISMATCH]]:
 // REC-NEXT:    tail call void @__ubsan_handle_type_mismatch_v1(ptr nonnull @[[GLOB2:[0-9]+]], i64 0) #[[ATTR6]], !nosanitize [[META2]]
 // REC-NEXT:    br label %[[CONT]], !nosanitize [[META2]]
 // REC:       [[CONT]]:
-// REC-NEXT:    [[TMP2:%.*]] = load i32, ptr [[X]], align 4, !tbaa [[TBAA4:![0-9]+]]
+// REC-NEXT:    [[TMP2:%.*]] = load i32, ptr [[X]], align 4, !tbaa [[TBAA5:![0-9]+]]
 // REC-NEXT:    ret i32 [[TMP2]]
 //
 int null(int* x) {
@@ -117,9 +138,11 @@ int null(int* x) {
 // CHECK-NEXT:  [[ENTRY:.*:]]
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call { i32, i1 } @llvm.sadd.with.overflow.i32(i32 [[X]], i32 [[Y]]), !nosanitize [[META2]]
 // CHECK-NEXT:    [[TMP1:%.*]] = extractvalue { i32, i1 } [[TMP0]], 1, !nosanitize [[META2]]
-// CHECK-NEXT:    [[TMP2:%.*]] = tail call i1 @llvm.allow.ubsan.check(i8 0), !nosanitize [[META2]]
+//
+//                                                                       41 == SO_SignedIntegerOverflow
+// CHECK-NEXT:    [[TMP2:%.*]] = tail call i1 @llvm.allow.ubsan.check(i8 41), !nosanitize [[META2]]
 // CHECK-NEXT:    [[DOTDEMORGAN:%.*]] = and i1 [[TMP1]], [[TMP2]]
-// CHECK-NEXT:    br i1 [[DOTDEMORGAN]], label %[[HANDLER_ADD_OVERFLOW:.*]], label %[[CONT:.*]], !prof [[PROF3]], !nosanitize [[META2]]
+// CHECK-NEXT:    br i1 [[DOTDEMORGAN]], label %[[HANDLER_ADD_OVERFLOW:.*]], label %[[CONT:.*]], !prof [[PROF4]], !nosanitize [[META2]]
 // CHECK:       [[HANDLER_ADD_OVERFLOW]]:
 // CHECK-NEXT:    [[TMP3:%.*]] = zext i32 [[X]] to i64, !nosanitize [[META2]]
 // CHECK-NEXT:    [[TMP4:%.*]] = zext i32 [[Y]] to i64, !nosanitize [[META2]]
@@ -134,7 +157,7 @@ int null(int* x) {
 // TR-NEXT:  [[ENTRY:.*:]]
 // TR-NEXT:    [[TMP0:%.*]] = tail call { i32, i1 } @llvm.sadd.with.overflow.i32(i32 [[X]], i32 [[Y]]), !nosanitize [[META2]]
 // TR-NEXT:    [[TMP1:%.*]] = extractvalue { i32, i1 } [[TMP0]], 1, !nosanitize [[META2]]
-// TR-NEXT:    [[TMP2:%.*]] = tail call i1 @llvm.allow.ubsan.check(i8 0), !nosanitize [[META2]]
+// TR-NEXT:    [[TMP2:%.*]] = tail call i1 @llvm.allow.ubsan.check(i8 41), !nosanitize [[META2]]
 // TR-NEXT:    [[DOTDEMORGAN:%.*]] = and i1 [[TMP1]], [[TMP2]]
 // TR-NEXT:    br i1 [[DOTDEMORGAN]], label %[[TRAP:.*]], label %[[CONT:.*]], !nosanitize [[META2]]
 // TR:       [[TRAP]]:
@@ -149,9 +172,9 @@ int null(int* x) {
 // REC-NEXT:  [[ENTRY:.*:]]
 // REC-NEXT:    [[TMP0:%.*]] = tail call { i32, i1 } @llvm.sadd.with.overflow.i32(i32 [[X]], i32 [[Y]]), !nosanitize [[META2]]
 // REC-NEXT:    [[TMP1:%.*]] = extractvalue { i32, i1 } [[TMP0]], 1, !nosanitize [[META2]]
-// REC-NEXT:    [[TMP2:%.*]] = tail call i1 @llvm.allow.ubsan.check(i8 0), !nosanitize [[META2]]
+// REC-NEXT:    [[TMP2:%.*]] = tail call i1 @llvm.allow.ubsan.check(i8 41), !nosanitize [[META2]]
 // REC-NEXT:    [[DOTDEMORGAN:%.*]] = and i1 [[TMP1]], [[TMP2]]
-// REC-NEXT:    br i1 [[DOTDEMORGAN]], label %[[HANDLER_ADD_OVERFLOW:.*]], label %[[CONT:.*]], !prof [[PROF3]], !nosanitize [[META2]]
+// REC-NEXT:    br i1 [[DOTDEMORGAN]], label %[[HANDLER_ADD_OVERFLOW:.*]], label %[[CONT:.*]], !prof [[PROF4]], !nosanitize [[META2]]
 // REC:       [[HANDLER_ADD_OVERFLOW]]:
 // REC-NEXT:    [[TMP3:%.*]] = zext i32 [[X]] to i64, !nosanitize [[META2]]
 // REC-NEXT:    [[TMP4:%.*]] = zext i32 [[Y]] to i64, !nosanitize [[META2]]
@@ -175,12 +198,14 @@ void use(double*);
 // CHECK-NEXT:    call void @use(ptr noundef nonnull [[VLA]]) #[[ATTR7:[0-9]+]]
 // CHECK-NEXT:    [[IDXPROM:%.*]] = sext i32 [[I]] to i64
 // CHECK-NEXT:    [[TMP1:%.*]] = icmp ule i64 [[TMP0]], [[IDXPROM]]
+//
+//                                                                  71 == SO_LocalBounds
 // CHECK-NEXT:    [[TMP2:%.*]] = call i1 @llvm.allow.ubsan.check(i8 71), !nosanitize [[META2]]
 // CHECK-NEXT:    [[TMP3:%.*]] = and i1 [[TMP1]], [[TMP2]], !nosanitize [[META2]]
 // CHECK-NEXT:    br i1 [[TMP3]], label %[[TRAP:.*]], label %[[BB4:.*]]
 // CHECK:       [[BB4]]:
 // CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds double, ptr [[VLA]], i64 [[IDXPROM]]
-// CHECK-NEXT:    [[TMP5:%.*]] = load double, ptr [[ARRAYIDX]], align 8, !tbaa [[TBAA8:![0-9]+]]
+// CHECK-NEXT:    [[TMP5:%.*]] = load double, ptr [[ARRAYIDX]], align 8, !tbaa [[TBAA9:![0-9]+]]
 // CHECK-NEXT:    ret double [[TMP5]]
 // CHECK:       [[TRAP]]:
 // CHECK-NEXT:    call void @__ubsan_handle_local_out_of_bounds_abort() #[[ATTR6]], !nosanitize [[META2]]
@@ -202,7 +227,7 @@ void use(double*);
 // TR-NEXT:    [[TMP5:%.*]] = load double, ptr [[ARRAYIDX]], align 8, !tbaa [[TBAA7:![0-9]+]]
 // TR-NEXT:    ret double [[TMP5]]
 // TR:       [[TRAP]]:
-// TR-NEXT:    call void @llvm.ubsantrap(i8 3) #[[ATTR5]], !nosanitize [[META2]]
+// TR-NEXT:    call void @llvm.ubsantrap(i8 71) #[[ATTR5]], !nosanitize [[META2]]
 // TR-NEXT:    unreachable, !nosanitize [[META2]]
 //
 // REC-LABEL: define dso_local double @lbounds(
@@ -218,7 +243,7 @@ void use(double*);
 // REC-NEXT:    br i1 [[TMP3]], label %[[TRAP:.*]], label %[[BB4:.*]]
 // REC:       [[BB4]]:
 // REC-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds double, ptr [[VLA]], i64 [[IDXPROM]]
-// REC-NEXT:    [[TMP5:%.*]] = load double, ptr [[ARRAYIDX]], align 8, !tbaa [[TBAA8:![0-9]+]]
+// REC-NEXT:    [[TMP5:%.*]] = load double, ptr [[ARRAYIDX]], align 8, !tbaa [[TBAA9:![0-9]+]]
 // REC-NEXT:    ret double [[TMP5]]
 // REC:       [[TRAP]]:
 // REC-NEXT:    call void @__ubsan_handle_local_out_of_bounds() #[[ATTR6]], !nosanitize [[META2]]
@@ -232,13 +257,14 @@ double lbounds(int b, int i) {
 
 //.
 // CHECK: [[META2]] = !{}
-// CHECK: [[PROF3]] = !{!"branch_weights", i32 1, i32 1048575}
-// CHECK: [[TBAA4]] = !{[[META5:![0-9]+]], [[META5]], i64 0}
-// CHECK: [[META5]] = !{!"int", [[META6:![0-9]+]], i64 0}
-// CHECK: [[META6]] = !{!"omnipotent char", [[META7:![0-9]+]], i64 0}
-// CHECK: [[META7]] = !{!"Simple C/C++ TBAA"}
-// CHECK: [[TBAA8]] = !{[[META9:![0-9]+]], [[META9]], i64 0}
-// CHECK: [[META9]] = !{!"double", [[META6]], i64 0}
+// CHECK: [[PROF3]] = !{!"branch_weights", i32 1048575, i32 1}
+// CHECK: [[PROF4]] = !{!"branch_weights", i32 1, i32 1048575}
+// CHECK: [[TBAA5]] = !{[[META6:![0-9]+]], [[META6]], i64 0}
+// CHECK: [[META6]] = !{!"int", [[META7:![0-9]+]], i64 0}
+// CHECK: [[META7]] = !{!"omnipotent char", [[META8:![0-9]+]], i64 0}
+// CHECK: [[META8]] = !{!"Simple C/C++ TBAA"}
+// CHECK: [[TBAA9]] = !{[[META10:![0-9]+]], [[META10]], i64 0}
+// CHECK: [[META10]] = !{!"double", [[META7]], i64 0}
 //.
 // TR: [[META2]] = !{}
 // TR: [[TBAA3]] = !{[[META4:![0-9]+]], [[META4]], i64 0}
@@ -249,11 +275,12 @@ double lbounds(int b, int i) {
 // TR: [[META8]] = !{!"double", [[META5]], i64 0}
 //.
 // REC: [[META2]] = !{}
-// REC: [[PROF3]] = !{!"branch_weights", i32 1, i32 1048575}
-// REC: [[TBAA4]] = !{[[META5:![0-9]+]], [[META5]], i64 0}
-// REC: [[META5]] = !{!"int", [[META6:![0-9]+]], i64 0}
-// REC: [[META6]] = !{!"omnipotent char", [[META7:![0-9]+]], i64 0}
-// REC: [[META7]] = !{!"Simple C/C++ TBAA"}
-// REC: [[TBAA8]] = !{[[META9:![0-9]+]], [[META9]], i64 0}
-// REC: [[META9]] = !{!"double", [[META6]], i64 0}
+// REC: [[PROF3]] = !{!"branch_weights", i32 1048575, i32 1}
+// REC: [[PROF4]] = !{!"branch_weights", i32 1, i32 1048575}
+// REC: [[TBAA5]] = !{[[META6:![0-9]+]], [[META6]], i64 0}
+// REC: [[META6]] = !{!"int", [[META7:![0-9]+]], i64 0}
+// REC: [[META7]] = !{!"omnipotent char", [[META8:![0-9]+]], i64 0}
+// REC: [[META8]] = !{!"Simple C/C++ TBAA"}
+// REC: [[TBAA9]] = !{[[META10:![0-9]+]], [[META10]], i64 0}
+// REC: [[META10]] = !{!"double", [[META7]], i64 0}
 //.
diff --git a/clang/test/CodeGen/arm-cmse-attr.c b/clang/test/CodeGen/arm-cmse-attr.c
index 6322a82dfa462..b01124a03df7b 100644
--- a/clang/test/CodeGen/arm-cmse-attr.c
+++ b/clang/test/CodeGen/arm-cmse-attr.c
@@ -29,9 +29,9 @@ void f4(void) __attribute__((cmse_nonsecure_entry))
 {
 }
 
-// CHECK: define{{.*}} void @f1(ptr nocapture noundef readonly %fptr) {{[^#]*}}#0 {
+// CHECK: define{{.*}} void @f1(ptr noundef readonly captures(none) %fptr) {{[^#]*}}#0 {
 // CHECK: call void %fptr() #2
-// CHECK: define{{.*}} void @f2(ptr nocapture noundef readonly %fptr) {{[^#]*}}#0 {
+// CHECK: define{{.*}} void @f2(ptr noundef readonly captures(none) %fptr) {{[^#]*}}#0 {
 // CHECK: call void %fptr() #2
 // CHECK: define{{.*}} void @f3() {{[^#]*}}#1 {
 // CHECK: define{{.*}} void @f4() {{[^#]*}}#1 {
diff --git a/clang/test/CodeGen/arm-empty-args.cpp b/clang/test/CodeGen/arm-empty-args.cpp
new file mode 100644
index 0000000000000..4e61c78b73ab9
--- /dev/null
+++ b/clang/test/CodeGen/arm-empty-args.cpp
@@ -0,0 +1,131 @@
+// RUN: %clang_cc1 -triple armv7a-linux-gnueabi -emit-llvm -o - -x c %s | FileCheck %s --check-prefixes=CHECK,C
+// RUN: %clang_cc1 -triple armv7a-linux-gnueabi -emit-llvm -o - %s | FileCheck %s --check-prefixes=CHECK,CXX
+// RUN: %clang_cc1 -triple armv7a-linux-gnueabi -emit-llvm -o - %s -fclang-abi-compat=19 | FileCheck %s --check-prefixes=CHECK,CXXCLANG19
+// RUN: %clang_cc1 -triple thumbv7k-apple-watchos2.0 -target-abi aapcs16 -emit-llvm -o - %s | FileCheck %s --check-prefixes=CHECK,WATCHOS
+
+// Empty structs are ignored for PCS purposes on WatchOS and in C mode
+// elsewhere.  In C++ mode they consume a register slot though. Functions are
+// slightly bigger than minimal to make confirmation against actual GCC
+// behaviour easier.
+
+#if __cplusplus
+#define EXTERNC extern "C"
+#else
+#define EXTERNC
+#endif
+
+struct Empty {};
+
+// C: define{{.*}} i32 @empty_arg(i32 noundef %a)
+// CXX: define{{.*}} i32 @empty_arg(i8 %e.coerce, i32 noundef %a)
+// CXXCLANG19: define{{.*}} i32 @empty_arg(i32 noundef %a)
+// WATCHOS: define{{.*}} i32 @empty_arg(i32 noundef %a)
+EXTERNC int empty_arg(struct Empty e, int a) {
+  return a;
+}
+
+// C: define{{.*}} void @empty_ret()
+// CXX: define{{.*}} void @empty_ret()
+// CXXCLANG19: define{{.*}} void @empty_ret()
+// WATCHOS: define{{.*}} void @empty_ret()
+EXTERNC struct Empty empty_ret(void) {
+  struct Empty e;
+  return e;
+}
+
+// However, what counts as "empty" is a baroque mess. This is super-empty, it's
+// ignored even in C++ mode. It also has sizeof == 0, violating C++, but that's
+// legacy for you:
+
+struct SuperEmpty {
+  int arr[0];
+};
+
+// C: define{{.*}} i32 @super_empty_arg(i32 noundef %a)
+// CXX: define{{.*}} i32 @super_empty_arg(i32 noundef %a)
+// CXXCLANG19: define{{.*}} i32 @super_empty_arg(i32 noundef %a)
+// WATCHOS: define{{.*}} i32 @super_empty_arg(i32 noundef %a)
+EXTERNC int super_empty_arg(struct SuperEmpty e, int a) {
+  return a;
+}
+
+struct SortOfEmpty {
+  struct SuperEmpty e;
+};
+
+// C: define{{.*}} i32 @sort_of_empty_arg(i32 noundef %a)
+// CXX: define{{.*}} i32 @sort_of_empty_arg(i8 %e.coerce, i32 noundef %a)
+// CXXCLANG19: define{{.*}} i32 @sort_of_empty_arg(i32 noundef %a)
+// WATCHOS: define{{.*}} i32 @sort_of_empty_arg(i32 noundef %a)
+EXTERNC int sort_of_empty_arg(struct Empty e, int a) {
+  return a;
+}
+
+// C: define{{.*}} void @sort_of_empty_ret()
+// CXX: define{{.*}} void @sort_of_empty_ret()
+// CXXCLANG19: define{{.*}} void @sort_of_empty_ret()
+// WATCHOS: define{{.*}} void @sort_of_empty_ret()
+EXTERNC struct SortOfEmpty sort_of_empty_ret(void) {
+  struct SortOfEmpty e;
+  return e;
+}
+
+#include <stdarg.h>
+
+// va_arg matches the above rules, consuming an incoming argument in cases
+// where one would be passed, and not doing so when the argument should be
+// ignored.
+
+EXTERNC int empty_arg_variadic(int a, ...) {
+// CHECK-LABEL: @empty_arg_variadic(
+// C: %argp.next = getelementptr inbounds i8, ptr %argp.cur, i32 4
+// C-NOT: {{ getelementptr }}
+// CXX: %argp.next = getelementptr inbounds i8, ptr %argp.cur, i32 4
+// CXX: %argp.next2 = getelementptr inbounds i8, ptr %argp.cur1, i32 4
+// CXXCLANG19: %argp.next = getelementptr inbounds i8, ptr %argp.cur, i32 4
+// CXXCLANG19-NOT: {{ getelementptr }}
+// WATCHOS: %argp.next = getelementptr inbounds i8, ptr %argp.cur, i32 4
+// WATCHOS-NOT: {{ getelementptr }}
+  va_list vl;
+  va_start(vl, a);
+  struct Empty b = va_arg(vl, struct Empty);
+  int c = va_arg(vl, int);
+  va_end(vl);
+  return c;
+}
+
+EXTERNC int super_empty_arg_variadic(int a, ...) {
+// CHECK-LABEL: @super_empty_arg_variadic(
+// C: %argp.next = getelementptr inbounds i8, ptr %argp.cur, i32 4
+// C-NOT: {{ getelementptr }}
+// CXX: %argp.next = getelementptr inbounds i8, ptr %argp.cur, i32 4
+// CXX-NOT: {{ getelementptr }}
+// CXXCLANG19: %argp.next = getelementptr inbounds i8, ptr %argp.cur, i32 4
+// CXXCLANG19-NOT: {{ getelementptr }}
+// WATCHOS: %argp.next = getelementptr inbounds i8, ptr %argp.cur, i32 4
+// WATCHOS-NOT: {{ getelementptr }}
+  va_list vl;
+  va_start(vl, a);
+  struct SuperEmpty b = va_arg(vl, struct SuperEmpty);
+  int c = va_arg(vl, int);
+  va_end(vl);
+  return c;
+}
+
+EXTERNC int sort_of_empty_arg_variadic(int a, ...) {
+// CHECK-LABEL: @sort_of_empty_arg_variadic(
+// C: %argp.next = getelementptr inbounds i8, ptr %argp.cur, i32 4
+// C-NOT: {{ getelementptr }}
+// CXX: %argp.next = getelementptr inbounds i8, ptr %argp.cur, i32 4
+// CXX-NOT: {{ getelementptr }}
+// CXXCLANG19: %argp.next = getelementptr inbounds i8, ptr %argp.cur, i32 4
+// CXXCLANG19-NOT: {{ getelementptr }}
+// WATCHOS: %argp.next = getelementptr inbounds i8, ptr %argp.cur, i32 4
+// WATCHOS-NOT: {{ getelementptr }}
+  va_list vl;
+  va_start(vl, a);
+  struct SortOfEmpty b = va_arg(vl, struct SortOfEmpty);
+  int c = va_arg(vl, int);
+  va_end(vl);
+  return c;
+}
diff --git a/clang/test/CodeGen/arm-mfp8.c b/clang/test/CodeGen/arm-mfp8.c
index bf91066335a25..9385b537f18b3 100644
--- a/clang/test/CodeGen/arm-mfp8.c
+++ b/clang/test/CodeGen/arm-mfp8.c
@@ -1,6 +1,6 @@
 // NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --version 5
-// RUN: %clang_cc1 -emit-llvm -triple aarch64-arm-none-eabi -target-feature -fp8 -target-feature +neon -o - %s | FileCheck %s --check-prefixes=CHECK,CHECK-C
-// RUN: %clang_cc1 -emit-llvm -triple aarch64-arm-none-eabi -target-feature -fp8 -target-feature +neon -o -  -x c++ %s | FileCheck %s --check-prefixes=CHECK,CHECK-CXX
+// RUN: %clang_cc1 -emit-llvm -triple aarch64-arm-none-eabi -target-feature -fp8 -target-feature +neon -disable-O0-optnone -o -        %s | opt -S --passes=mem2reg | FileCheck %s --check-prefixes=CHECK-C
+// RUN: %clang_cc1 -emit-llvm -triple aarch64-arm-none-eabi -target-feature -fp8 -target-feature +neon -disable-O0-optnone -o - -x c++ %s | opt -S --passes=mem2reg | FileCheck %s --check-prefixes=CHECK-CXX
 
 // REQUIRES: aarch64-registered-target
 
@@ -10,18 +10,12 @@
 // CHECK-C-LABEL: define dso_local <16 x i8> @test_ret_mfloat8x16_t(
 // CHECK-C-SAME: <16 x i8> [[V:%.*]]) #[[ATTR0:[0-9]+]] {
 // CHECK-C-NEXT:  [[ENTRY:.*:]]
-// CHECK-C-NEXT:    [[V_ADDR:%.*]] = alloca <16 x i8>, align 16
-// CHECK-C-NEXT:    store <16 x i8> [[V]], ptr [[V_ADDR]], align 16
-// CHECK-C-NEXT:    [[TMP0:%.*]] = load <16 x i8>, ptr [[V_ADDR]], align 16
-// CHECK-C-NEXT:    ret <16 x i8> [[TMP0]]
+// CHECK-C-NEXT:    ret <16 x i8> [[V]]
 //
-// CHECK-CXX-LABEL: define dso_local <16 x i8> @_Z21test_ret_mfloat8x16_tu14__MFloat8x16_t(
+// CHECK-CXX-LABEL: define dso_local <16 x i8> @_Z21test_ret_mfloat8x16_t14__Mfloat8x16_t(
 // CHECK-CXX-SAME: <16 x i8> [[V:%.*]]) #[[ATTR0:[0-9]+]] {
 // CHECK-CXX-NEXT:  [[ENTRY:.*:]]
-// CHECK-CXX-NEXT:    [[V_ADDR:%.*]] = alloca <16 x i8>, align 16
-// CHECK-CXX-NEXT:    store <16 x i8> [[V]], ptr [[V_ADDR]], align 16
-// CHECK-CXX-NEXT:    [[TMP0:%.*]] = load <16 x i8>, ptr [[V_ADDR]], align 16
-// CHECK-CXX-NEXT:    ret <16 x i8> [[TMP0]]
+// CHECK-CXX-NEXT:    ret <16 x i8> [[V]]
 //
 mfloat8x16_t test_ret_mfloat8x16_t(mfloat8x16_t v) {
   return v;
@@ -30,18 +24,12 @@ mfloat8x16_t test_ret_mfloat8x16_t(mfloat8x16_t v) {
 // CHECK-C-LABEL: define dso_local <8 x i8> @test_ret_mfloat8x8_t(
 // CHECK-C-SAME: <8 x i8> [[V:%.*]]) #[[ATTR0]] {
 // CHECK-C-NEXT:  [[ENTRY:.*:]]
-// CHECK-C-NEXT:    [[V_ADDR:%.*]] = alloca <8 x i8>, align 8
-// CHECK-C-NEXT:    store <8 x i8> [[V]], ptr [[V_ADDR]], align 8
-// CHECK-C-NEXT:    [[TMP0:%.*]] = load <8 x i8>, ptr [[V_ADDR]], align 8
-// CHECK-C-NEXT:    ret <8 x i8> [[TMP0]]
+// CHECK-C-NEXT:    ret <8 x i8> [[V]]
 //
-// CHECK-CXX-LABEL: define dso_local <8 x i8> @_Z20test_ret_mfloat8x8_tu13__MFloat8x8_t(
+// CHECK-CXX-LABEL: define dso_local <8 x i8> @_Z20test_ret_mfloat8x8_t13__Mfloat8x8_t(
 // CHECK-CXX-SAME: <8 x i8> [[V:%.*]]) #[[ATTR0]] {
 // CHECK-CXX-NEXT:  [[ENTRY:.*:]]
-// CHECK-CXX-NEXT:    [[V_ADDR:%.*]] = alloca <8 x i8>, align 8
-// CHECK-CXX-NEXT:    store <8 x i8> [[V]], ptr [[V_ADDR]], align 8
-// CHECK-CXX-NEXT:    [[TMP0:%.*]] = load <8 x i8>, ptr [[V_ADDR]], align 8
-// CHECK-CXX-NEXT:    ret <8 x i8> [[TMP0]]
+// CHECK-CXX-NEXT:    ret <8 x i8> [[V]]
 //
 mfloat8x8_t test_ret_mfloat8x8_t(mfloat8x8_t v) {
   return v;
@@ -50,28 +38,22 @@ mfloat8x8_t test_ret_mfloat8x8_t(mfloat8x8_t v) {
 // CHECK-C-LABEL: define dso_local <1 x i8> @func1n(
 // CHECK-C-SAME: <1 x i8> [[MFP8:%.*]]) #[[ATTR0]] {
 // CHECK-C-NEXT:  [[ENTRY:.*:]]
-// CHECK-C-NEXT:    [[MFP8_ADDR:%.*]] = alloca <1 x i8>, align 1
 // CHECK-C-NEXT:    [[F1N:%.*]] = alloca [10 x <1 x i8>], align 1
-// CHECK-C-NEXT:    store <1 x i8> [[MFP8]], ptr [[MFP8_ADDR]], align 1
-// CHECK-C-NEXT:    [[TMP0:%.*]] = load <1 x i8>, ptr [[MFP8_ADDR]], align 1
 // CHECK-C-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [10 x <1 x i8>], ptr [[F1N]], i64 0, i64 2
-// CHECK-C-NEXT:    store <1 x i8> [[TMP0]], ptr [[ARRAYIDX]], align 1
+// CHECK-C-NEXT:    store <1 x i8> [[MFP8]], ptr [[ARRAYIDX]], align 1
 // CHECK-C-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds [10 x <1 x i8>], ptr [[F1N]], i64 0, i64 2
-// CHECK-C-NEXT:    [[TMP1:%.*]] = load <1 x i8>, ptr [[ARRAYIDX1]], align 1
-// CHECK-C-NEXT:    ret <1 x i8> [[TMP1]]
+// CHECK-C-NEXT:    [[TMP0:%.*]] = load <1 x i8>, ptr [[ARRAYIDX1]], align 1
+// CHECK-C-NEXT:    ret <1 x i8> [[TMP0]]
 //
 // CHECK-CXX-LABEL: define dso_local <1 x i8> @_Z6func1nu6__mfp8(
 // CHECK-CXX-SAME: <1 x i8> [[MFP8:%.*]]) #[[ATTR0]] {
 // CHECK-CXX-NEXT:  [[ENTRY:.*:]]
-// CHECK-CXX-NEXT:    [[MFP8_ADDR:%.*]] = alloca <1 x i8>, align 1
 // CHECK-CXX-NEXT:    [[F1N:%.*]] = alloca [10 x <1 x i8>], align 1
-// CHECK-CXX-NEXT:    store <1 x i8> [[MFP8]], ptr [[MFP8_ADDR]], align 1
-// CHECK-CXX-NEXT:    [[TMP0:%.*]] = load <1 x i8>, ptr [[MFP8_ADDR]], align 1
 // CHECK-CXX-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [10 x <1 x i8>], ptr [[F1N]], i64 0, i64 2
-// CHECK-CXX-NEXT:    store <1 x i8> [[TMP0]], ptr [[ARRAYIDX]], align 1
+// CHECK-CXX-NEXT:    store <1 x i8> [[MFP8]], ptr [[ARRAYIDX]], align 1
 // CHECK-CXX-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds [10 x <1 x i8>], ptr [[F1N]], i64 0, i64 2
-// CHECK-CXX-NEXT:    [[TMP1:%.*]] = load <1 x i8>, ptr [[ARRAYIDX1]], align 1
-// CHECK-CXX-NEXT:    ret <1 x i8> [[TMP1]]
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = load <1 x i8>, ptr [[ARRAYIDX1]], align 1
+// CHECK-CXX-NEXT:    ret <1 x i8> [[TMP0]]
 //
 __mfp8 func1n(__mfp8 mfp8) {
   __mfp8 f1n[10];
@@ -79,7 +61,43 @@ __mfp8 func1n(__mfp8 mfp8) {
   return f1n[2];
 }
 
+// CHECK-C-LABEL: define dso_local <1 x i8> @test_extract_element(
+// CHECK-C-SAME: <16 x i8> [[X:%.*]], i32 noundef [[I:%.*]]) #[[ATTR0]] {
+// CHECK-C-NEXT:  [[ENTRY:.*:]]
+// CHECK-C-NEXT:    [[RETVAL:%.*]] = alloca <1 x i8>, align 1
+// CHECK-C-NEXT:    [[VECEXT:%.*]] = extractelement <16 x i8> [[X]], i32 [[I]]
+// CHECK-C-NEXT:    store i8 [[VECEXT]], ptr [[RETVAL]], align 1
+// CHECK-C-NEXT:    [[TMP0:%.*]] = load <1 x i8>, ptr [[RETVAL]], align 1
+// CHECK-C-NEXT:    ret <1 x i8> [[TMP0]]
+//
+// CHECK-CXX-LABEL: define dso_local <1 x i8> @_Z20test_extract_element14__Mfloat8x16_ti(
+// CHECK-CXX-SAME: <16 x i8> [[X:%.*]], i32 noundef [[I:%.*]]) #[[ATTR0]] {
+// CHECK-CXX-NEXT:  [[ENTRY:.*:]]
+// CHECK-CXX-NEXT:    [[RETVAL:%.*]] = alloca <1 x i8>, align 1
+// CHECK-CXX-NEXT:    [[VECEXT:%.*]] = extractelement <16 x i8> [[X]], i32 [[I]]
+// CHECK-CXX-NEXT:    store i8 [[VECEXT]], ptr [[RETVAL]], align 1
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = load <1 x i8>, ptr [[RETVAL]], align 1
+// CHECK-CXX-NEXT:    ret <1 x i8> [[TMP0]]
+//
+mfloat8_t test_extract_element(mfloat8x16_t x, int i) {
+  return x[i];
+}
 
-
-//// NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
-// CHECK: {{.*}}
+// CHECK-C-LABEL: define dso_local <16 x i8> @test_insert_element(
+// CHECK-C-SAME: <16 x i8> [[X:%.*]], i32 noundef [[I:%.*]], <1 x i8> [[V:%.*]]) #[[ATTR0]] {
+// CHECK-C-NEXT:  [[ENTRY:.*:]]
+// CHECK-C-NEXT:    [[TMP0:%.*]] = bitcast <1 x i8> [[V]] to i8
+// CHECK-C-NEXT:    [[VECINS:%.*]] = insertelement <16 x i8> [[X]], i8 [[TMP0]], i32 [[I]]
+// CHECK-C-NEXT:    ret <16 x i8> [[VECINS]]
+//
+// CHECK-CXX-LABEL: define dso_local <16 x i8> @_Z19test_insert_element14__Mfloat8x16_tiu6__mfp8(
+// CHECK-CXX-SAME: <16 x i8> [[X:%.*]], i32 noundef [[I:%.*]], <1 x i8> [[V:%.*]]) #[[ATTR0]] {
+// CHECK-CXX-NEXT:  [[ENTRY:.*:]]
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = bitcast <1 x i8> [[V]] to i8
+// CHECK-CXX-NEXT:    [[VECINS:%.*]] = insertelement <16 x i8> [[X]], i8 [[TMP0]], i32 [[I]]
+// CHECK-CXX-NEXT:    ret <16 x i8> [[VECINS]]
+//
+mfloat8x16_t test_insert_element(mfloat8x16_t x, int i, mfloat8_t v) {
+  x[i] = v;
+  return x;
+}
diff --git a/clang/test/CodeGen/arm-vfp16-arguments.c b/clang/test/CodeGen/arm-vfp16-arguments.c
index 3c6691df4747a..c0bbb3df72bf3 100644
--- a/clang/test/CodeGen/arm-vfp16-arguments.c
+++ b/clang/test/CodeGen/arm-vfp16-arguments.c
@@ -71,6 +71,6 @@ void test_hfa(hfa_t a) {}
 
 hfa_t ghfa;
 hfa_t test_ret_hfa(void) { return ghfa; }
-// CHECK-SOFT: define{{.*}} void @test_ret_hfa(ptr dead_on_unwind noalias nocapture writable writeonly sret(%struct.hfa_t) align 8 initializes((0, 16)) %agg.result)
+// CHECK-SOFT: define{{.*}} void @test_ret_hfa(ptr dead_on_unwind noalias writable writeonly sret(%struct.hfa_t) align 8 captures(none) initializes((0, 16)) %agg.result)
 // CHECK-HARD: define{{.*}} arm_aapcs_vfpcc [2 x <2 x i32>] @test_ret_hfa()
 // CHECK-FULL: define{{.*}} arm_aapcs_vfpcc %struct.hfa_t @test_ret_hfa()
diff --git a/clang/test/CodeGen/arm-vfp16-arguments2.cpp b/clang/test/CodeGen/arm-vfp16-arguments2.cpp
index b810cfd0a6648..6e9a24e70c141 100644
--- a/clang/test/CodeGen/arm-vfp16-arguments2.cpp
+++ b/clang/test/CodeGen/arm-vfp16-arguments2.cpp
@@ -37,27 +37,27 @@ struct S5 : B1 {
   B1 M[1];
 };
 
-// CHECK-SOFT: define{{.*}} void @_Z2f12S1(ptr dead_on_unwind noalias nocapture writable writeonly sret(%struct.S1) align 8 initializes((0, 16)) %agg.result, [2 x i64] %s1.coerce)
+// CHECK-SOFT: define{{.*}} void @_Z2f12S1(ptr dead_on_unwind noalias writable writeonly sret(%struct.S1) align 8 captures(none) initializes((0, 16)) %agg.result, [2 x i64] %s1.coerce)
 // CHECK-HARD: define{{.*}} arm_aapcs_vfpcc [2 x <2 x i32>] @_Z2f12S1([2 x <2 x i32>] returned %s1.coerce)
 // CHECK-FULL: define{{.*}} arm_aapcs_vfpcc %struct.S1 @_Z2f12S1(%struct.S1 returned %s1.coerce)
 struct S1 f1(struct S1 s1) { return s1; }
 
-// CHECK-SOFT: define{{.*}} void @_Z2f22S2(ptr dead_on_unwind noalias nocapture writable writeonly sret(%struct.S2) align 8 initializes((0, 16)) %agg.result, [4 x i32] %s2.coerce)
+// CHECK-SOFT: define{{.*}} void @_Z2f22S2(ptr dead_on_unwind noalias writable writeonly sret(%struct.S2) align 8 captures(none) initializes((0, 16)) %agg.result, [4 x i32] %s2.coerce)
 // CHECK-HARD: define{{.*}} arm_aapcs_vfpcc [2 x <2 x i32>] @_Z2f22S2([2 x <2 x i32>] returned %s2.coerce)
 // CHECK-FULL: define{{.*}} arm_aapcs_vfpcc %struct.S2 @_Z2f22S2(%struct.S2 %s2.coerce)
 struct S2 f2(struct S2 s2) { return s2; }
 
-// CHECK-SOFT: define{{.*}} void @_Z2f32S3(ptr dead_on_unwind noalias nocapture writable writeonly sret(%struct.S3) align 8 initializes((0, 16)) %agg.result, [2 x i64] %s3.coerce)
+// CHECK-SOFT: define{{.*}} void @_Z2f32S3(ptr dead_on_unwind noalias writable writeonly sret(%struct.S3) align 8 captures(none) initializes((0, 16)) %agg.result, [2 x i64] %s3.coerce)
 // CHECK-HARD: define{{.*}} arm_aapcs_vfpcc [2 x <2 x i32>] @_Z2f32S3([2 x <2 x i32>] returned %s3.coerce)
 // CHECK-FULL: define{{.*}} arm_aapcs_vfpcc %struct.S3 @_Z2f32S3(%struct.S3 %s3.coerce)
 struct S3 f3(struct S3 s3) { return s3; }
 
-// CHECK-SOFT: define{{.*}} void @_Z2f42S4(ptr dead_on_unwind noalias nocapture writable writeonly sret(%struct.S4) align 8 initializes((0, 16)) %agg.result, [2 x i64] %s4.coerce)
+// CHECK-SOFT: define{{.*}} void @_Z2f42S4(ptr dead_on_unwind noalias writable writeonly sret(%struct.S4) align 8 captures(none) initializes((0, 16)) %agg.result, [2 x i64] %s4.coerce)
 // CHECK-HARD: define{{.*}} arm_aapcs_vfpcc [2 x <2 x i32>] @_Z2f42S4([2 x <2 x i32>] returned %s4.coerce)
 // CHECK-FULL: define{{.*}} arm_aapcs_vfpcc %struct.S4 @_Z2f42S4(%struct.S4 %s4.coerce)
 struct S4 f4(struct S4 s4) { return s4; }
 
-// CHECK-SOFT: define{{.*}} void @_Z2f52S5(ptr dead_on_unwind noalias nocapture writable writeonly sret(%struct.S5) align 8 initializes((0, 16)) %agg.result, [2 x i64] %s5.coerce)
+// CHECK-SOFT: define{{.*}} void @_Z2f52S5(ptr dead_on_unwind noalias writable writeonly sret(%struct.S5) align 8 captures(none) initializes((0, 16)) %agg.result, [2 x i64] %s5.coerce)
 // CHECK-HARD: define{{.*}} arm_aapcs_vfpcc %struct.S5 @_Z2f52S5(%struct.S5 %s5.coerce)
 // CHECK-FULL: define{{.*}} arm_aapcs_vfpcc %struct.S5 @_Z2f52S5(%struct.S5 %s5.coerce)
 struct S5 f5(struct S5 s5) { return s5; }
diff --git a/clang/test/CodeGen/attr-counted-by-pr110385.c b/clang/test/CodeGen/attr-counted-by-pr110385.c
index f211610c3173b..412c12cb687c4 100644
--- a/clang/test/CodeGen/attr-counted-by-pr110385.c
+++ b/clang/test/CodeGen/attr-counted-by-pr110385.c
@@ -26,7 +26,7 @@ struct bucket2 {
 void init(void * __attribute__((pass_dynamic_object_size(0))));
 
 // CHECK-LABEL: define dso_local void @test1(
-// CHECK-SAME: ptr nocapture noundef readonly [[FOO:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
+// CHECK-SAME: ptr noundef readonly captures(none) [[FOO:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[GROWABLE:%.*]] = getelementptr inbounds nuw i8, ptr [[FOO]], i64 8
 // CHECK-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[GROWABLE]], align 8, !tbaa [[TBAA2:![0-9]+]]
diff --git a/clang/test/CodeGen/attr-counted-by.c b/clang/test/CodeGen/attr-counted-by.c
index 1066e2e74160a..e85f3db1121af 100644
--- a/clang/test/CodeGen/attr-counted-by.c
+++ b/clang/test/CodeGen/attr-counted-by.c
@@ -9,11 +9,13 @@
 #endif
 
 #ifdef COUNTED_BY
-#define __counted_by(member)    __attribute__((__counted_by__(member)))
+#define __counted_by(member)   __attribute__((__counted_by__(member)))
 #else
 #define __counted_by(member)
 #endif
 
+#define __bdos(P)              __builtin_dynamic_object_size(P, 0)
+
 #define DECLARE_FLEX_ARRAY(TYPE, NAME)                              \
   struct {                                                          \
     struct { } __empty_ ## NAME;                                    \
@@ -75,7 +77,7 @@ struct anon_struct {
 // SANITIZE-WITH-ATTR-NEXT:    ret void
 //
 // NO-SANITIZE-WITH-ATTR-LABEL: define dso_local void @test1(
-// NO-SANITIZE-WITH-ATTR-SAME: ptr nocapture noundef writeonly [[P:%.*]], i32 noundef [[INDEX:%.*]], i32 noundef [[VAL:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
+// NO-SANITIZE-WITH-ATTR-SAME: ptr noundef writeonly captures(none) [[P:%.*]], i32 noundef [[INDEX:%.*]], i32 noundef [[VAL:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
 // NO-SANITIZE-WITH-ATTR-NEXT:  entry:
 // NO-SANITIZE-WITH-ATTR-NEXT:    [[ARRAY:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i64 12
 // NO-SANITIZE-WITH-ATTR-NEXT:    [[IDXPROM:%.*]] = sext i32 [[INDEX]] to i64
@@ -93,7 +95,7 @@ struct anon_struct {
 // SANITIZE-WITHOUT-ATTR-NEXT:    ret void
 //
 // NO-SANITIZE-WITHOUT-ATTR-LABEL: define dso_local void @test1(
-// NO-SANITIZE-WITHOUT-ATTR-SAME: ptr nocapture noundef writeonly [[P:%.*]], i32 noundef [[INDEX:%.*]], i32 noundef [[VAL:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
+// NO-SANITIZE-WITHOUT-ATTR-SAME: ptr noundef writeonly captures(none) [[P:%.*]], i32 noundef [[INDEX:%.*]], i32 noundef [[VAL:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
 // NO-SANITIZE-WITHOUT-ATTR-NEXT:  entry:
 // NO-SANITIZE-WITHOUT-ATTR-NEXT:    [[ARRAY:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i64 12
 // NO-SANITIZE-WITHOUT-ATTR-NEXT:    [[IDXPROM:%.*]] = sext i32 [[INDEX]] to i64
@@ -112,11 +114,11 @@ void test1(struct annotated *p, int index, int val) {
 // SANITIZE-WITH-ATTR-NEXT:    [[COUNTED_BY_LOAD:%.*]] = load i32, ptr [[COUNTED_BY_GEP]], align 4
 // SANITIZE-WITH-ATTR-NEXT:    [[TMP0:%.*]] = zext i32 [[COUNTED_BY_LOAD]] to i64, !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR-NEXT:    [[TMP1:%.*]] = icmp ult i64 [[INDEX]], [[TMP0]], !nosanitize [[META2]]
-// SANITIZE-WITH-ATTR-NEXT:    br i1 [[TMP1]], label [[CONT3:%.*]], label [[HANDLER_OUT_OF_BOUNDS:%.*]], !prof [[PROF3]], !nosanitize [[META2]]
+// SANITIZE-WITH-ATTR-NEXT:    br i1 [[TMP1]], label [[CONT6:%.*]], label [[HANDLER_OUT_OF_BOUNDS:%.*]], !prof [[PROF3]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR:       handler.out_of_bounds:
 // SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB3:[0-9]+]], i64 [[INDEX]]) #[[ATTR8]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR-NEXT:    unreachable, !nosanitize [[META2]]
-// SANITIZE-WITH-ATTR:       cont3:
+// SANITIZE-WITH-ATTR:       cont6:
 // SANITIZE-WITH-ATTR-NEXT:    [[ARRAY:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i64 12
 // SANITIZE-WITH-ATTR-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds nuw [0 x i32], ptr [[ARRAY]], i64 0, i64 [[INDEX]]
 // SANITIZE-WITH-ATTR-NEXT:    [[TMP2:%.*]] = tail call i32 @llvm.smax.i32(i32 [[COUNTED_BY_LOAD]], i32 0)
@@ -125,13 +127,13 @@ void test1(struct annotated *p, int index, int val) {
 // SANITIZE-WITH-ATTR-NEXT:    ret void
 //
 // NO-SANITIZE-WITH-ATTR-LABEL: define dso_local void @test2(
-// NO-SANITIZE-WITH-ATTR-SAME: ptr nocapture noundef [[P:%.*]], i64 noundef [[INDEX:%.*]]) local_unnamed_addr #[[ATTR1:[0-9]+]] {
+// NO-SANITIZE-WITH-ATTR-SAME: ptr noundef captures(none) [[P:%.*]], i64 noundef [[INDEX:%.*]]) local_unnamed_addr #[[ATTR1:[0-9]+]] {
 // NO-SANITIZE-WITH-ATTR-NEXT:  entry:
+// NO-SANITIZE-WITH-ATTR-NEXT:    [[ARRAY:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i64 12
 // NO-SANITIZE-WITH-ATTR-NEXT:    [[COUNTED_BY_GEP:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i64 8
 // NO-SANITIZE-WITH-ATTR-NEXT:    [[COUNTED_BY_LOAD:%.*]] = load i32, ptr [[COUNTED_BY_GEP]], align 4
 // NO-SANITIZE-WITH-ATTR-NEXT:    [[TMP0:%.*]] = tail call i32 @llvm.smax.i32(i32 [[COUNTED_BY_LOAD]], i32 0)
 // NO-SANITIZE-WITH-ATTR-NEXT:    [[CONV:%.*]] = shl i32 [[TMP0]], 2
-// NO-SANITIZE-WITH-ATTR-NEXT:    [[ARRAY:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i64 12
 // NO-SANITIZE-WITH-ATTR-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds nuw [0 x i32], ptr [[ARRAY]], i64 0, i64 [[INDEX]]
 // NO-SANITIZE-WITH-ATTR-NEXT:    store i32 [[CONV]], ptr [[ARRAYIDX]], align 4, !tbaa [[TBAA2]]
 // NO-SANITIZE-WITH-ATTR-NEXT:    ret void
@@ -153,30 +155,30 @@ void test1(struct annotated *p, int index, int val) {
 // NO-SANITIZE-WITHOUT-ATTR-NEXT:    ret void
 //
 void test2(struct annotated *p, size_t index) {
-  p->array[index] = __builtin_dynamic_object_size(p->array, 1);
+  p->array[index] = __bdos(p->array);
 }
 
 // SANITIZE-WITH-ATTR-LABEL: define dso_local range(i64 -8589934592, 8589934589) i64 @test2_bdos(
-// SANITIZE-WITH-ATTR-SAME: ptr nocapture noundef readonly [[P:%.*]]) local_unnamed_addr #[[ATTR2:[0-9]+]] {
+// SANITIZE-WITH-ATTR-SAME: ptr noundef [[P:%.*]]) local_unnamed_addr #[[ATTR0]] {
 // SANITIZE-WITH-ATTR-NEXT:  entry:
 // SANITIZE-WITH-ATTR-NEXT:    [[COUNTED_BY_GEP:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i64 8
 // SANITIZE-WITH-ATTR-NEXT:    [[COUNTED_BY_LOAD:%.*]] = load i32, ptr [[COUNTED_BY_GEP]], align 4
-// SANITIZE-WITH-ATTR-NEXT:    [[TMP0:%.*]] = sext i32 [[COUNTED_BY_LOAD]] to i64
-// SANITIZE-WITH-ATTR-NEXT:    [[TMP1:%.*]] = shl nsw i64 [[TMP0]], 2
-// SANITIZE-WITH-ATTR-NEXT:    [[TMP2:%.*]] = icmp sgt i32 [[COUNTED_BY_LOAD]], -1
-// SANITIZE-WITH-ATTR-NEXT:    [[TMP3:%.*]] = select i1 [[TMP2]], i64 [[TMP1]], i64 0
-// SANITIZE-WITH-ATTR-NEXT:    ret i64 [[TMP3]]
+// SANITIZE-WITH-ATTR-NEXT:    [[COUNT:%.*]] = sext i32 [[COUNTED_BY_LOAD]] to i64
+// SANITIZE-WITH-ATTR-NEXT:    [[FLEXIBLE_ARRAY_MEMBER_SIZE:%.*]] = shl nsw i64 [[COUNT]], 2
+// SANITIZE-WITH-ATTR-NEXT:    [[TMP0:%.*]] = icmp sgt i32 [[COUNTED_BY_LOAD]], -1
+// SANITIZE-WITH-ATTR-NEXT:    [[TMP1:%.*]] = select i1 [[TMP0]], i64 [[FLEXIBLE_ARRAY_MEMBER_SIZE]], i64 0
+// SANITIZE-WITH-ATTR-NEXT:    ret i64 [[TMP1]]
 //
 // NO-SANITIZE-WITH-ATTR-LABEL: define dso_local range(i64 -8589934592, 8589934589) i64 @test2_bdos(
-// NO-SANITIZE-WITH-ATTR-SAME: ptr nocapture noundef readonly [[P:%.*]]) local_unnamed_addr #[[ATTR2:[0-9]+]] {
+// NO-SANITIZE-WITH-ATTR-SAME: ptr noundef readonly captures(none) [[P:%.*]]) local_unnamed_addr #[[ATTR2:[0-9]+]] {
 // NO-SANITIZE-WITH-ATTR-NEXT:  entry:
 // NO-SANITIZE-WITH-ATTR-NEXT:    [[COUNTED_BY_GEP:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i64 8
 // NO-SANITIZE-WITH-ATTR-NEXT:    [[COUNTED_BY_LOAD:%.*]] = load i32, ptr [[COUNTED_BY_GEP]], align 4
-// NO-SANITIZE-WITH-ATTR-NEXT:    [[TMP0:%.*]] = sext i32 [[COUNTED_BY_LOAD]] to i64
-// NO-SANITIZE-WITH-ATTR-NEXT:    [[TMP1:%.*]] = shl nsw i64 [[TMP0]], 2
-// NO-SANITIZE-WITH-ATTR-NEXT:    [[TMP2:%.*]] = icmp sgt i32 [[COUNTED_BY_LOAD]], -1
-// NO-SANITIZE-WITH-ATTR-NEXT:    [[TMP3:%.*]] = select i1 [[TMP2]], i64 [[TMP1]], i64 0
-// NO-SANITIZE-WITH-ATTR-NEXT:    ret i64 [[TMP3]]
+// NO-SANITIZE-WITH-ATTR-NEXT:    [[COUNT:%.*]] = sext i32 [[COUNTED_BY_LOAD]] to i64
+// NO-SANITIZE-WITH-ATTR-NEXT:    [[FLEXIBLE_ARRAY_MEMBER_SIZE:%.*]] = shl nsw i64 [[COUNT]], 2
+// NO-SANITIZE-WITH-ATTR-NEXT:    [[TMP0:%.*]] = icmp sgt i32 [[COUNTED_BY_LOAD]], -1
+// NO-SANITIZE-WITH-ATTR-NEXT:    [[TMP1:%.*]] = select i1 [[TMP0]], i64 [[FLEXIBLE_ARRAY_MEMBER_SIZE]], i64 0
+// NO-SANITIZE-WITH-ATTR-NEXT:    ret i64 [[TMP1]]
 //
 // SANITIZE-WITHOUT-ATTR-LABEL: define dso_local i64 @test2_bdos(
 // SANITIZE-WITHOUT-ATTR-SAME: ptr noundef [[P:%.*]]) local_unnamed_addr #[[ATTR0]] {
@@ -189,7 +191,7 @@ void test2(struct annotated *p, size_t index) {
 // NO-SANITIZE-WITHOUT-ATTR-NEXT:    ret i64 -1
 //
 size_t test2_bdos(struct annotated *p) {
-  return __builtin_dynamic_object_size(p->array, 1);
+  return __bdos(p->array);
 }
 
 // SANITIZE-WITH-ATTR-LABEL: define dso_local void @test3(
@@ -236,11 +238,11 @@ size_t test2_bdos(struct annotated *p) {
 void test3(struct annotated *p, size_t index) {
   // This test differs from 'test2' by checking bdos on the whole array and not
   // just the FAM.
-  p->array[index] = __builtin_dynamic_object_size(p, 0);
+  p->array[index] = __bdos(p);
 }
 
 // SANITIZE-WITH-ATTR-LABEL: define dso_local i64 @test3_bdos(
-// SANITIZE-WITH-ATTR-SAME: ptr noundef readnone [[P:%.*]]) local_unnamed_addr #[[ATTR3:[0-9]+]] {
+// SANITIZE-WITH-ATTR-SAME: ptr noundef readnone [[P:%.*]]) local_unnamed_addr #[[ATTR2:[0-9]+]] {
 // SANITIZE-WITH-ATTR-NEXT:  entry:
 // SANITIZE-WITH-ATTR-NEXT:    ret i64 -1
 //
@@ -260,107 +262,122 @@ void test3(struct annotated *p, size_t index) {
 // NO-SANITIZE-WITHOUT-ATTR-NEXT:    ret i64 -1
 //
 size_t test3_bdos(struct annotated *p) {
-  return __builtin_dynamic_object_size(p, 0);
+  return __bdos(p);
 }
 
 // SANITIZE-WITH-ATTR-LABEL: define dso_local void @test4(
 // SANITIZE-WITH-ATTR-SAME: ptr noundef [[P:%.*]], i32 noundef [[INDEX:%.*]], i32 noundef [[FAM_IDX:%.*]]) local_unnamed_addr #[[ATTR0]] {
 // SANITIZE-WITH-ATTR-NEXT:  entry:
-// SANITIZE-WITH-ATTR-NEXT:    [[COUNTED_BY_GEP:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i64 8
-// SANITIZE-WITH-ATTR-NEXT:    [[COUNTED_BY_LOAD:%.*]] = load i32, ptr [[COUNTED_BY_GEP]], align 4
 // SANITIZE-WITH-ATTR-NEXT:    [[ARRAY:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i64 12
-// SANITIZE-WITH-ATTR-NEXT:    [[IDXPROM:%.*]] = sext i32 [[INDEX]] to i64
-// SANITIZE-WITH-ATTR-NEXT:    [[TMP0:%.*]] = zext i32 [[COUNTED_BY_LOAD]] to i64, !nosanitize [[META2]]
-// SANITIZE-WITH-ATTR-NEXT:    [[TMP1:%.*]] = icmp ult i64 [[IDXPROM]], [[TMP0]], !nosanitize [[META2]]
-// SANITIZE-WITH-ATTR-NEXT:    br i1 [[TMP1]], label [[CONT4:%.*]], label [[HANDLER_OUT_OF_BOUNDS:%.*]], !prof [[PROF3]], !nosanitize [[META2]]
+// SANITIZE-WITH-ATTR-NEXT:    [[DOTCOUNTED_BY_GEP:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i64 8
+// SANITIZE-WITH-ATTR-NEXT:    [[DOTCOUNTED_BY_LOAD:%.*]] = load i32, ptr [[DOTCOUNTED_BY_GEP]], align 4
+// SANITIZE-WITH-ATTR-NEXT:    [[TMP0:%.*]] = zext i32 [[DOTCOUNTED_BY_LOAD]] to i64, !nosanitize [[META2]]
+// SANITIZE-WITH-ATTR-NEXT:    [[TMP1:%.*]] = icmp ugt i32 [[DOTCOUNTED_BY_LOAD]], 2
+// SANITIZE-WITH-ATTR-NEXT:    br i1 [[TMP1]], label [[CONT1:%.*]], label [[HANDLER_OUT_OF_BOUNDS:%.*]], !prof [[PROF3]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR:       handler.out_of_bounds:
-// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB5:[0-9]+]], i64 [[IDXPROM]]) #[[ATTR8]], !nosanitize [[META2]]
+// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB5:[0-9]+]], i64 3) #[[ATTR8]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR-NEXT:    unreachable, !nosanitize [[META2]]
-// SANITIZE-WITH-ATTR:       cont4:
-// SANITIZE-WITH-ATTR-NEXT:    [[TMP2:%.*]] = icmp sgt i32 [[COUNTED_BY_LOAD]], 2
-// SANITIZE-WITH-ATTR-NEXT:    [[TMP3:%.*]] = shl i32 [[COUNTED_BY_LOAD]], 2
-// SANITIZE-WITH-ATTR-NEXT:    [[TMP4:%.*]] = add i32 [[TMP3]], 244
-// SANITIZE-WITH-ATTR-NEXT:    [[TMP5:%.*]] = and i32 [[TMP4]], 252
-// SANITIZE-WITH-ATTR-NEXT:    [[CONV1:%.*]] = select i1 [[TMP2]], i32 [[TMP5]], i32 0
-// SANITIZE-WITH-ATTR-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds nuw [0 x i32], ptr [[ARRAY]], i64 0, i64 [[IDXPROM]]
-// SANITIZE-WITH-ATTR-NEXT:    store i32 [[CONV1]], ptr [[ARRAYIDX]], align 4, !tbaa [[TBAA4]]
-// SANITIZE-WITH-ATTR-NEXT:    [[ADD:%.*]] = add nsw i32 [[INDEX]], 1
-// SANITIZE-WITH-ATTR-NEXT:    [[IDXPROM12:%.*]] = sext i32 [[ADD]] to i64
-// SANITIZE-WITH-ATTR-NEXT:    [[TMP6:%.*]] = icmp ult i64 [[IDXPROM12]], [[TMP0]], !nosanitize [[META2]]
-// SANITIZE-WITH-ATTR-NEXT:    br i1 [[TMP6]], label [[CONT19:%.*]], label [[HANDLER_OUT_OF_BOUNDS15:%.*]], !prof [[PROF3]], !nosanitize [[META2]]
-// SANITIZE-WITH-ATTR:       handler.out_of_bounds15:
-// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB6:[0-9]+]], i64 [[IDXPROM12]]) #[[ATTR8]], !nosanitize [[META2]]
+// SANITIZE-WITH-ATTR:       cont1:
+// SANITIZE-WITH-ATTR-NEXT:    [[FLEXIBLE_ARRAY_MEMBER_SIZE:%.*]] = shl i32 [[DOTCOUNTED_BY_LOAD]], 2
+// SANITIZE-WITH-ATTR-NEXT:    [[IDXPROM:%.*]] = sext i32 [[INDEX]] to i64
+// SANITIZE-WITH-ATTR-NEXT:    [[TMP2:%.*]] = icmp ult i64 [[IDXPROM]], [[TMP0]], !nosanitize [[META2]]
+// SANITIZE-WITH-ATTR-NEXT:    br i1 [[TMP2]], label [[CONT12:%.*]], label [[HANDLER_OUT_OF_BOUNDS8:%.*]], !prof [[PROF3]], !nosanitize [[META2]]
+// SANITIZE-WITH-ATTR:       handler.out_of_bounds8:
+// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB6:[0-9]+]], i64 [[IDXPROM]]) #[[ATTR8]], !nosanitize [[META2]]
+// SANITIZE-WITH-ATTR-NEXT:    unreachable, !nosanitize [[META2]]
+// SANITIZE-WITH-ATTR:       cont12:
+// SANITIZE-WITH-ATTR-NEXT:    [[TMP3:%.*]] = icmp sgt i32 [[DOTCOUNTED_BY_LOAD]], 2
+// SANITIZE-WITH-ATTR-NEXT:    [[RESULT:%.*]] = add i32 [[FLEXIBLE_ARRAY_MEMBER_SIZE]], 244
+// SANITIZE-WITH-ATTR-NEXT:    [[TMP4:%.*]] = and i32 [[RESULT]], 252
+// SANITIZE-WITH-ATTR-NEXT:    [[CONV2:%.*]] = select i1 [[TMP3]], i32 [[TMP4]], i32 0
+// SANITIZE-WITH-ATTR-NEXT:    [[ARRAYIDX10:%.*]] = getelementptr inbounds nuw [0 x i32], ptr [[ARRAY]], i64 0, i64 [[IDXPROM]]
+// SANITIZE-WITH-ATTR-NEXT:    store i32 [[CONV2]], ptr [[ARRAYIDX10]], align 4, !tbaa [[TBAA4]]
+// SANITIZE-WITH-ATTR-NEXT:    [[DOTNOT81:%.*]] = icmp eq i32 [[DOTCOUNTED_BY_LOAD]], 3
+// SANITIZE-WITH-ATTR-NEXT:    br i1 [[DOTNOT81]], label [[HANDLER_OUT_OF_BOUNDS18:%.*]], label [[CONT19:%.*]], !prof [[PROF8:![0-9]+]], !nosanitize [[META2]]
+// SANITIZE-WITH-ATTR:       handler.out_of_bounds18:
+// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB7:[0-9]+]], i64 4) #[[ATTR8]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR-NEXT:    unreachable, !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR:       cont19:
-// SANITIZE-WITH-ATTR-NEXT:    [[TMP7:%.*]] = icmp sgt i32 [[COUNTED_BY_LOAD]], 3
-// SANITIZE-WITH-ATTR-NEXT:    [[TMP8:%.*]] = add i32 [[TMP3]], 240
-// SANITIZE-WITH-ATTR-NEXT:    [[TMP9:%.*]] = and i32 [[TMP8]], 252
-// SANITIZE-WITH-ATTR-NEXT:    [[CONV8:%.*]] = select i1 [[TMP7]], i32 [[TMP9]], i32 0
-// SANITIZE-WITH-ATTR-NEXT:    [[ARRAYIDX17:%.*]] = getelementptr inbounds nuw [0 x i32], ptr [[ARRAY]], i64 0, i64 [[IDXPROM12]]
-// SANITIZE-WITH-ATTR-NEXT:    store i32 [[CONV8]], ptr [[ARRAYIDX17]], align 4, !tbaa [[TBAA4]]
-// SANITIZE-WITH-ATTR-NEXT:    [[COUNTED_BY_LOAD21:%.*]] = load i32, ptr [[COUNTED_BY_GEP]], align 4
-// SANITIZE-WITH-ATTR-NEXT:    [[ADD27:%.*]] = add nsw i32 [[INDEX]], 2
-// SANITIZE-WITH-ATTR-NEXT:    [[IDXPROM28:%.*]] = sext i32 [[ADD27]] to i64
-// SANITIZE-WITH-ATTR-NEXT:    [[TMP10:%.*]] = zext i32 [[COUNTED_BY_LOAD21]] to i64, !nosanitize [[META2]]
-// SANITIZE-WITH-ATTR-NEXT:    [[TMP11:%.*]] = icmp ult i64 [[IDXPROM28]], [[TMP10]], !nosanitize [[META2]]
-// SANITIZE-WITH-ATTR-NEXT:    br i1 [[TMP11]], label [[CONT35:%.*]], label [[HANDLER_OUT_OF_BOUNDS31:%.*]], !prof [[PROF3]], !nosanitize [[META2]]
-// SANITIZE-WITH-ATTR:       handler.out_of_bounds31:
-// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB7:[0-9]+]], i64 [[IDXPROM28]]) #[[ATTR8]], !nosanitize [[META2]]
+// SANITIZE-WITH-ATTR-NEXT:    [[ADD:%.*]] = add nsw i32 [[INDEX]], 1
+// SANITIZE-WITH-ATTR-NEXT:    [[IDXPROM31:%.*]] = sext i32 [[ADD]] to i64
+// SANITIZE-WITH-ATTR-NEXT:    [[TMP5:%.*]] = icmp ult i64 [[IDXPROM31]], [[TMP0]], !nosanitize [[META2]]
+// SANITIZE-WITH-ATTR-NEXT:    br i1 [[TMP5]], label [[CONT38:%.*]], label [[HANDLER_OUT_OF_BOUNDS34:%.*]], !prof [[PROF3]], !nosanitize [[META2]]
+// SANITIZE-WITH-ATTR:       handler.out_of_bounds34:
+// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB8:[0-9]+]], i64 [[IDXPROM31]]) #[[ATTR8]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR-NEXT:    unreachable, !nosanitize [[META2]]
-// SANITIZE-WITH-ATTR:       cont35:
-// SANITIZE-WITH-ATTR-NEXT:    [[ARRAYIDX33:%.*]] = getelementptr inbounds nuw [0 x i32], ptr [[ARRAY]], i64 0, i64 [[IDXPROM28]]
-// SANITIZE-WITH-ATTR-NEXT:    [[TMP12:%.*]] = icmp sgt i32 [[FAM_IDX]], -1
-// SANITIZE-WITH-ATTR-NEXT:    [[TMP13:%.*]] = sext i32 [[COUNTED_BY_LOAD21]] to i64
-// SANITIZE-WITH-ATTR-NEXT:    [[TMP14:%.*]] = sext i32 [[FAM_IDX]] to i64
-// SANITIZE-WITH-ATTR-NEXT:    [[TMP15:%.*]] = sub nsw i64 [[TMP13]], [[TMP14]]
-// SANITIZE-WITH-ATTR-NEXT:    [[TMP16:%.*]] = icmp sgt i64 [[TMP15]], -1
-// SANITIZE-WITH-ATTR-NEXT:    [[TMP17:%.*]] = and i1 [[TMP12]], [[TMP16]]
-// SANITIZE-WITH-ATTR-NEXT:    [[DOTTR:%.*]] = trunc i64 [[TMP15]] to i32
-// SANITIZE-WITH-ATTR-NEXT:    [[TMP18:%.*]] = shl i32 [[DOTTR]], 2
-// SANITIZE-WITH-ATTR-NEXT:    [[TMP19:%.*]] = and i32 [[TMP18]], 252
-// SANITIZE-WITH-ATTR-NEXT:    [[CONV23:%.*]] = select i1 [[TMP17]], i32 [[TMP19]], i32 0
-// SANITIZE-WITH-ATTR-NEXT:    store i32 [[CONV23]], ptr [[ARRAYIDX33]], align 4, !tbaa [[TBAA4]]
+// SANITIZE-WITH-ATTR:       cont38:
+// SANITIZE-WITH-ATTR-NEXT:    [[TMP6:%.*]] = icmp sgt i32 [[DOTCOUNTED_BY_LOAD]], 3
+// SANITIZE-WITH-ATTR-NEXT:    [[RESULT25:%.*]] = add i32 [[FLEXIBLE_ARRAY_MEMBER_SIZE]], 240
+// SANITIZE-WITH-ATTR-NEXT:    [[TMP7:%.*]] = and i32 [[RESULT25]], 252
+// SANITIZE-WITH-ATTR-NEXT:    [[CONV27:%.*]] = select i1 [[TMP6]], i32 [[TMP7]], i32 0
+// SANITIZE-WITH-ATTR-NEXT:    [[ARRAYIDX36:%.*]] = getelementptr inbounds nuw [0 x i32], ptr [[ARRAY]], i64 0, i64 [[IDXPROM31]]
+// SANITIZE-WITH-ATTR-NEXT:    store i32 [[CONV27]], ptr [[ARRAYIDX36]], align 4, !tbaa [[TBAA4]]
+// SANITIZE-WITH-ATTR-NEXT:    [[IDXPROM42:%.*]] = sext i32 [[FAM_IDX]] to i64
+// SANITIZE-WITH-ATTR-NEXT:    [[DOTCOUNTED_BY_LOAD44:%.*]] = load i32, ptr [[DOTCOUNTED_BY_GEP]], align 4
+// SANITIZE-WITH-ATTR-NEXT:    [[TMP8:%.*]] = zext i32 [[DOTCOUNTED_BY_LOAD44]] to i64, !nosanitize [[META2]]
+// SANITIZE-WITH-ATTR-NEXT:    [[DOTNOT:%.*]] = icmp ugt i64 [[IDXPROM42]], [[TMP8]], !nosanitize [[META2]]
+// SANITIZE-WITH-ATTR-NEXT:    br i1 [[DOTNOT]], label [[HANDLER_OUT_OF_BOUNDS45:%.*]], label [[CONT46:%.*]], !prof [[PROF8]], !nosanitize [[META2]]
+// SANITIZE-WITH-ATTR:       handler.out_of_bounds45:
+// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB9:[0-9]+]], i64 [[IDXPROM42]]) #[[ATTR8]], !nosanitize [[META2]]
+// SANITIZE-WITH-ATTR-NEXT:    unreachable, !nosanitize [[META2]]
+// SANITIZE-WITH-ATTR:       cont46:
+// SANITIZE-WITH-ATTR-NEXT:    [[ADD59:%.*]] = add nsw i32 [[INDEX]], 2
+// SANITIZE-WITH-ATTR-NEXT:    [[IDXPROM60:%.*]] = sext i32 [[ADD59]] to i64
+// SANITIZE-WITH-ATTR-NEXT:    [[TMP9:%.*]] = icmp ult i64 [[IDXPROM60]], [[TMP8]], !nosanitize [[META2]]
+// SANITIZE-WITH-ATTR-NEXT:    br i1 [[TMP9]], label [[CONT67:%.*]], label [[HANDLER_OUT_OF_BOUNDS63:%.*]], !prof [[PROF3]], !nosanitize [[META2]]
+// SANITIZE-WITH-ATTR:       handler.out_of_bounds63:
+// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB10:[0-9]+]], i64 [[IDXPROM60]]) #[[ATTR8]], !nosanitize [[META2]]
+// SANITIZE-WITH-ATTR-NEXT:    unreachable, !nosanitize [[META2]]
+// SANITIZE-WITH-ATTR:       cont67:
+// SANITIZE-WITH-ATTR-NEXT:    [[ARRAYIDX65:%.*]] = getelementptr inbounds nuw [0 x i32], ptr [[ARRAY]], i64 0, i64 [[IDXPROM60]]
+// SANITIZE-WITH-ATTR-NEXT:    [[COUNT50:%.*]] = sext i32 [[DOTCOUNTED_BY_LOAD44]] to i64
+// SANITIZE-WITH-ATTR-NEXT:    [[TMP10:%.*]] = sub nsw i64 [[COUNT50]], [[IDXPROM42]]
+// SANITIZE-WITH-ATTR-NEXT:    [[TMP11:%.*]] = tail call i64 @llvm.smax.i64(i64 [[TMP10]], i64 0)
+// SANITIZE-WITH-ATTR-NEXT:    [[DOTTR:%.*]] = trunc nuw i64 [[TMP11]] to i32
+// SANITIZE-WITH-ATTR-NEXT:    [[CONV54:%.*]] = shl i32 [[DOTTR]], 2
+// SANITIZE-WITH-ATTR-NEXT:    [[CONV55:%.*]] = and i32 [[CONV54]], 252
+// SANITIZE-WITH-ATTR-NEXT:    store i32 [[CONV55]], ptr [[ARRAYIDX65]], align 4, !tbaa [[TBAA4]]
 // SANITIZE-WITH-ATTR-NEXT:    ret void
 //
 // NO-SANITIZE-WITH-ATTR-LABEL: define dso_local void @test4(
-// NO-SANITIZE-WITH-ATTR-SAME: ptr nocapture noundef [[P:%.*]], i32 noundef [[INDEX:%.*]], i32 noundef [[FAM_IDX:%.*]]) local_unnamed_addr #[[ATTR1]] {
+// NO-SANITIZE-WITH-ATTR-SAME: ptr noundef captures(none) [[P:%.*]], i32 noundef [[INDEX:%.*]], i32 noundef [[FAM_IDX:%.*]]) local_unnamed_addr #[[ATTR1]] {
 // NO-SANITIZE-WITH-ATTR-NEXT:  entry:
+// NO-SANITIZE-WITH-ATTR-NEXT:    [[ARRAY:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i64 12
 // NO-SANITIZE-WITH-ATTR-NEXT:    [[COUNTED_BY_GEP:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i64 8
 // NO-SANITIZE-WITH-ATTR-NEXT:    [[COUNTED_BY_LOAD:%.*]] = load i32, ptr [[COUNTED_BY_GEP]], align 4
-// NO-SANITIZE-WITH-ATTR-NEXT:    [[TMP0:%.*]] = shl i32 [[COUNTED_BY_LOAD]], 2
-// NO-SANITIZE-WITH-ATTR-NEXT:    [[TMP1:%.*]] = add i32 [[TMP0]], 244
-// NO-SANITIZE-WITH-ATTR-NEXT:    [[TMP2:%.*]] = icmp sgt i32 [[COUNTED_BY_LOAD]], 2
-// NO-SANITIZE-WITH-ATTR-NEXT:    [[TMP3:%.*]] = and i32 [[TMP1]], 252
-// NO-SANITIZE-WITH-ATTR-NEXT:    [[CONV1:%.*]] = select i1 [[TMP2]], i32 [[TMP3]], i32 0
-// NO-SANITIZE-WITH-ATTR-NEXT:    [[ARRAY:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i64 12
+// NO-SANITIZE-WITH-ATTR-NEXT:    [[FLEXIBLE_ARRAY_MEMBER_SIZE:%.*]] = shl i32 [[COUNTED_BY_LOAD]], 2
+// NO-SANITIZE-WITH-ATTR-NEXT:    [[RESULT:%.*]] = add i32 [[FLEXIBLE_ARRAY_MEMBER_SIZE]], 244
+// NO-SANITIZE-WITH-ATTR-NEXT:    [[TMP0:%.*]] = icmp sgt i32 [[COUNTED_BY_LOAD]], 2
+// NO-SANITIZE-WITH-ATTR-NEXT:    [[TMP1:%.*]] = and i32 [[RESULT]], 252
+// NO-SANITIZE-WITH-ATTR-NEXT:    [[CONV1:%.*]] = select i1 [[TMP0]], i32 [[TMP1]], i32 0
 // NO-SANITIZE-WITH-ATTR-NEXT:    [[IDXPROM:%.*]] = sext i32 [[INDEX]] to i64
-// NO-SANITIZE-WITH-ATTR-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [0 x i32], ptr [[ARRAY]], i64 0, i64 [[IDXPROM]]
-// NO-SANITIZE-WITH-ATTR-NEXT:    store i32 [[CONV1]], ptr [[ARRAYIDX]], align 4, !tbaa [[TBAA2]]
-// NO-SANITIZE-WITH-ATTR-NEXT:    [[COUNTED_BY_LOAD3:%.*]] = load i32, ptr [[COUNTED_BY_GEP]], align 4
-// NO-SANITIZE-WITH-ATTR-NEXT:    [[TMP4:%.*]] = shl i32 [[COUNTED_BY_LOAD3]], 2
-// NO-SANITIZE-WITH-ATTR-NEXT:    [[TMP5:%.*]] = add i32 [[TMP4]], 240
-// NO-SANITIZE-WITH-ATTR-NEXT:    [[TMP6:%.*]] = icmp sgt i32 [[COUNTED_BY_LOAD3]], 3
-// NO-SANITIZE-WITH-ATTR-NEXT:    [[TMP7:%.*]] = and i32 [[TMP5]], 252
-// NO-SANITIZE-WITH-ATTR-NEXT:    [[CONV5:%.*]] = select i1 [[TMP6]], i32 [[TMP7]], i32 0
+// NO-SANITIZE-WITH-ATTR-NEXT:    [[ARRAYIDX3:%.*]] = getelementptr inbounds [0 x i32], ptr [[ARRAY]], i64 0, i64 [[IDXPROM]]
+// NO-SANITIZE-WITH-ATTR-NEXT:    store i32 [[CONV1]], ptr [[ARRAYIDX3]], align 4, !tbaa [[TBAA2]]
+// NO-SANITIZE-WITH-ATTR-NEXT:    [[COUNTED_BY_LOAD7:%.*]] = load i32, ptr [[COUNTED_BY_GEP]], align 4
+// NO-SANITIZE-WITH-ATTR-NEXT:    [[FLEXIBLE_ARRAY_MEMBER_SIZE9:%.*]] = shl i32 [[COUNTED_BY_LOAD7]], 2
+// NO-SANITIZE-WITH-ATTR-NEXT:    [[RESULT10:%.*]] = add i32 [[FLEXIBLE_ARRAY_MEMBER_SIZE9]], 240
+// NO-SANITIZE-WITH-ATTR-NEXT:    [[TMP2:%.*]] = icmp sgt i32 [[COUNTED_BY_LOAD7]], 3
+// NO-SANITIZE-WITH-ATTR-NEXT:    [[TMP3:%.*]] = and i32 [[RESULT10]], 252
+// NO-SANITIZE-WITH-ATTR-NEXT:    [[CONV12:%.*]] = select i1 [[TMP2]], i32 [[TMP3]], i32 0
 // NO-SANITIZE-WITH-ATTR-NEXT:    [[ADD:%.*]] = add nsw i32 [[INDEX]], 1
-// NO-SANITIZE-WITH-ATTR-NEXT:    [[IDXPROM7:%.*]] = sext i32 [[ADD]] to i64
-// NO-SANITIZE-WITH-ATTR-NEXT:    [[ARRAYIDX8:%.*]] = getelementptr inbounds [0 x i32], ptr [[ARRAY]], i64 0, i64 [[IDXPROM7]]
-// NO-SANITIZE-WITH-ATTR-NEXT:    store i32 [[CONV5]], ptr [[ARRAYIDX8]], align 4, !tbaa [[TBAA2]]
-// NO-SANITIZE-WITH-ATTR-NEXT:    [[COUNTED_BY_LOAD10:%.*]] = load i32, ptr [[COUNTED_BY_GEP]], align 4
-// NO-SANITIZE-WITH-ATTR-NEXT:    [[TMP8:%.*]] = sext i32 [[COUNTED_BY_LOAD10]] to i64
-// NO-SANITIZE-WITH-ATTR-NEXT:    [[TMP9:%.*]] = sext i32 [[FAM_IDX]] to i64
-// NO-SANITIZE-WITH-ATTR-NEXT:    [[TMP10:%.*]] = sub nsw i64 [[TMP8]], [[TMP9]]
-// NO-SANITIZE-WITH-ATTR-NEXT:    [[TMP11:%.*]] = icmp sgt i64 [[TMP10]], -1
-// NO-SANITIZE-WITH-ATTR-NEXT:    [[TMP12:%.*]] = icmp sgt i32 [[FAM_IDX]], -1
-// NO-SANITIZE-WITH-ATTR-NEXT:    [[TMP13:%.*]] = and i1 [[TMP12]], [[TMP11]]
-// NO-SANITIZE-WITH-ATTR-NEXT:    [[DOTTR:%.*]] = trunc i64 [[TMP10]] to i32
-// NO-SANITIZE-WITH-ATTR-NEXT:    [[TMP14:%.*]] = shl i32 [[DOTTR]], 2
-// NO-SANITIZE-WITH-ATTR-NEXT:    [[TMP15:%.*]] = and i32 [[TMP14]], 252
-// NO-SANITIZE-WITH-ATTR-NEXT:    [[CONV12:%.*]] = select i1 [[TMP13]], i32 [[TMP15]], i32 0
-// NO-SANITIZE-WITH-ATTR-NEXT:    [[ADD14:%.*]] = add nsw i32 [[INDEX]], 2
-// NO-SANITIZE-WITH-ATTR-NEXT:    [[IDXPROM15:%.*]] = sext i32 [[ADD14]] to i64
-// NO-SANITIZE-WITH-ATTR-NEXT:    [[ARRAYIDX16:%.*]] = getelementptr inbounds [0 x i32], ptr [[ARRAY]], i64 0, i64 [[IDXPROM15]]
-// NO-SANITIZE-WITH-ATTR-NEXT:    store i32 [[CONV12]], ptr [[ARRAYIDX16]], align 4, !tbaa [[TBAA2]]
+// NO-SANITIZE-WITH-ATTR-NEXT:    [[IDXPROM14:%.*]] = sext i32 [[ADD]] to i64
+// NO-SANITIZE-WITH-ATTR-NEXT:    [[ARRAYIDX15:%.*]] = getelementptr inbounds [0 x i32], ptr [[ARRAY]], i64 0, i64 [[IDXPROM14]]
+// NO-SANITIZE-WITH-ATTR-NEXT:    store i32 [[CONV12]], ptr [[ARRAYIDX15]], align 4, !tbaa [[TBAA2]]
+// NO-SANITIZE-WITH-ATTR-NEXT:    [[IDXPROM17:%.*]] = sext i32 [[FAM_IDX]] to i64
+// NO-SANITIZE-WITH-ATTR-NEXT:    [[COUNTED_BY_LOAD20:%.*]] = load i32, ptr [[COUNTED_BY_GEP]], align 4
+// NO-SANITIZE-WITH-ATTR-NEXT:    [[COUNT21:%.*]] = sext i32 [[COUNTED_BY_LOAD20]] to i64
+// NO-SANITIZE-WITH-ATTR-NEXT:    [[TMP4:%.*]] = sub nsw i64 [[COUNT21]], [[IDXPROM17]]
+// NO-SANITIZE-WITH-ATTR-NEXT:    [[TMP5:%.*]] = icmp sgt i64 [[TMP4]], -1
+// NO-SANITIZE-WITH-ATTR-NEXT:    [[TMP6:%.*]] = icmp sgt i32 [[FAM_IDX]], -1
+// NO-SANITIZE-WITH-ATTR-NEXT:    [[TMP7:%.*]] = and i1 [[TMP6]], [[TMP5]]
+// NO-SANITIZE-WITH-ATTR-NEXT:    [[DOTTR:%.*]] = trunc i64 [[TMP4]] to i32
+// NO-SANITIZE-WITH-ATTR-NEXT:    [[TMP8:%.*]] = shl i32 [[DOTTR]], 2
+// NO-SANITIZE-WITH-ATTR-NEXT:    [[TMP9:%.*]] = and i32 [[TMP8]], 252
+// NO-SANITIZE-WITH-ATTR-NEXT:    [[CONV26:%.*]] = select i1 [[TMP7]], i32 [[TMP9]], i32 0
+// NO-SANITIZE-WITH-ATTR-NEXT:    [[ADD28:%.*]] = add nsw i32 [[INDEX]], 2
+// NO-SANITIZE-WITH-ATTR-NEXT:    [[IDXPROM29:%.*]] = sext i32 [[ADD28]] to i64
+// NO-SANITIZE-WITH-ATTR-NEXT:    [[ARRAYIDX30:%.*]] = getelementptr inbounds [0 x i32], ptr [[ARRAY]], i64 0, i64 [[IDXPROM29]]
+// NO-SANITIZE-WITH-ATTR-NEXT:    store i32 [[CONV26]], ptr [[ARRAYIDX30]], align 4, !tbaa [[TBAA2]]
 // NO-SANITIZE-WITH-ATTR-NEXT:    ret void
 //
 // SANITIZE-WITHOUT-ATTR-LABEL: define dso_local void @test4(
@@ -399,40 +416,44 @@ size_t test3_bdos(struct annotated *p) {
 //
 void test4(struct annotated *p, int index, int fam_idx) {
   // This tests calculating the size from a pointer inside the FAM.
-  p->array[index] = (unsigned char)__builtin_dynamic_object_size(&p->array[3], 1);
-  p->array[index + 1] = (unsigned char)__builtin_dynamic_object_size(&(p->array[4]), 1);
-  p->array[index + 2] = (unsigned char)__builtin_dynamic_object_size(&(p->array[fam_idx]), 1);
+  p->array[index] = (unsigned char)__bdos(&p->array[3]);
+  p->array[index + 1] = (unsigned char)__bdos(&(p->array[4]));
+  p->array[index + 2] = (unsigned char)__bdos(&(p->array[fam_idx]));
 }
 
-// SANITIZE-WITH-ATTR-LABEL: define dso_local range(i64 -17179869180, 17179869181) i64 @test4_bdos(
-// SANITIZE-WITH-ATTR-SAME: ptr nocapture noundef readonly [[P:%.*]], i32 noundef [[INDEX:%.*]]) local_unnamed_addr #[[ATTR2]] {
+// SANITIZE-WITH-ATTR-LABEL: define dso_local range(i64 0, 8589934589) i64 @test4_bdos(
+// SANITIZE-WITH-ATTR-SAME: ptr noundef [[P:%.*]], i32 noundef [[INDEX:%.*]]) local_unnamed_addr #[[ATTR0]] {
 // SANITIZE-WITH-ATTR-NEXT:  entry:
-// SANITIZE-WITH-ATTR-NEXT:    [[COUNTED_BY_GEP:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i64 8
-// SANITIZE-WITH-ATTR-NEXT:    [[COUNTED_BY_LOAD:%.*]] = load i32, ptr [[COUNTED_BY_GEP]], align 4
-// SANITIZE-WITH-ATTR-NEXT:    [[TMP0:%.*]] = sext i32 [[COUNTED_BY_LOAD]] to i64
-// SANITIZE-WITH-ATTR-NEXT:    [[TMP1:%.*]] = sext i32 [[INDEX]] to i64
-// SANITIZE-WITH-ATTR-NEXT:    [[TMP2:%.*]] = sub nsw i64 [[TMP0]], [[TMP1]]
-// SANITIZE-WITH-ATTR-NEXT:    [[TMP3:%.*]] = shl nsw i64 [[TMP2]], 2
-// SANITIZE-WITH-ATTR-NEXT:    [[TMP4:%.*]] = icmp sgt i64 [[TMP2]], -1
-// SANITIZE-WITH-ATTR-NEXT:    [[TMP5:%.*]] = icmp sgt i32 [[INDEX]], -1
-// SANITIZE-WITH-ATTR-NEXT:    [[TMP6:%.*]] = and i1 [[TMP5]], [[TMP4]]
-// SANITIZE-WITH-ATTR-NEXT:    [[TMP7:%.*]] = select i1 [[TMP6]], i64 [[TMP3]], i64 0
-// SANITIZE-WITH-ATTR-NEXT:    ret i64 [[TMP7]]
+// SANITIZE-WITH-ATTR-NEXT:    [[IDXPROM:%.*]] = sext i32 [[INDEX]] to i64
+// SANITIZE-WITH-ATTR-NEXT:    [[DOTCOUNTED_BY_GEP:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i64 8
+// SANITIZE-WITH-ATTR-NEXT:    [[DOTCOUNTED_BY_LOAD:%.*]] = load i32, ptr [[DOTCOUNTED_BY_GEP]], align 4
+// SANITIZE-WITH-ATTR-NEXT:    [[TMP0:%.*]] = zext i32 [[DOTCOUNTED_BY_LOAD]] to i64, !nosanitize [[META2]]
+// SANITIZE-WITH-ATTR-NEXT:    [[DOTNOT:%.*]] = icmp ugt i64 [[IDXPROM]], [[TMP0]], !nosanitize [[META2]]
+// SANITIZE-WITH-ATTR-NEXT:    br i1 [[DOTNOT]], label [[HANDLER_OUT_OF_BOUNDS:%.*]], label [[CONT1:%.*]], !prof [[PROF8]], !nosanitize [[META2]]
+// SANITIZE-WITH-ATTR:       handler.out_of_bounds:
+// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB11:[0-9]+]], i64 [[IDXPROM]]) #[[ATTR8]], !nosanitize [[META2]]
+// SANITIZE-WITH-ATTR-NEXT:    unreachable, !nosanitize [[META2]]
+// SANITIZE-WITH-ATTR:       cont1:
+// SANITIZE-WITH-ATTR-NEXT:    [[COUNT:%.*]] = sext i32 [[DOTCOUNTED_BY_LOAD]] to i64
+// SANITIZE-WITH-ATTR-NEXT:    [[TMP1:%.*]] = sub nsw i64 [[COUNT]], [[IDXPROM]]
+// SANITIZE-WITH-ATTR-NEXT:    [[TMP2:%.*]] = tail call i64 @llvm.smax.i64(i64 [[TMP1]], i64 0)
+// SANITIZE-WITH-ATTR-NEXT:    [[TMP3:%.*]] = shl nuw nsw i64 [[TMP2]], 2
+// SANITIZE-WITH-ATTR-NEXT:    ret i64 [[TMP3]]
 //
 // NO-SANITIZE-WITH-ATTR-LABEL: define dso_local range(i64 -17179869180, 17179869181) i64 @test4_bdos(
-// NO-SANITIZE-WITH-ATTR-SAME: ptr nocapture noundef readonly [[P:%.*]], i32 noundef [[INDEX:%.*]]) local_unnamed_addr #[[ATTR2]] {
+// NO-SANITIZE-WITH-ATTR-SAME: ptr noundef readonly captures(none) [[P:%.*]], i32 noundef [[INDEX:%.*]]) local_unnamed_addr #[[ATTR2]] {
 // NO-SANITIZE-WITH-ATTR-NEXT:  entry:
+// NO-SANITIZE-WITH-ATTR-NEXT:    [[IDXPROM:%.*]] = sext i32 [[INDEX]] to i64
 // NO-SANITIZE-WITH-ATTR-NEXT:    [[COUNTED_BY_GEP:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i64 8
 // NO-SANITIZE-WITH-ATTR-NEXT:    [[COUNTED_BY_LOAD:%.*]] = load i32, ptr [[COUNTED_BY_GEP]], align 4
-// NO-SANITIZE-WITH-ATTR-NEXT:    [[TMP0:%.*]] = sext i32 [[COUNTED_BY_LOAD]] to i64
-// NO-SANITIZE-WITH-ATTR-NEXT:    [[TMP1:%.*]] = sext i32 [[INDEX]] to i64
-// NO-SANITIZE-WITH-ATTR-NEXT:    [[TMP2:%.*]] = sub nsw i64 [[TMP0]], [[TMP1]]
-// NO-SANITIZE-WITH-ATTR-NEXT:    [[TMP3:%.*]] = shl nsw i64 [[TMP2]], 2
-// NO-SANITIZE-WITH-ATTR-NEXT:    [[TMP4:%.*]] = icmp sgt i64 [[TMP2]], -1
-// NO-SANITIZE-WITH-ATTR-NEXT:    [[TMP5:%.*]] = icmp sgt i32 [[INDEX]], -1
-// NO-SANITIZE-WITH-ATTR-NEXT:    [[TMP6:%.*]] = and i1 [[TMP5]], [[TMP4]]
-// NO-SANITIZE-WITH-ATTR-NEXT:    [[TMP7:%.*]] = select i1 [[TMP6]], i64 [[TMP3]], i64 0
-// NO-SANITIZE-WITH-ATTR-NEXT:    ret i64 [[TMP7]]
+// NO-SANITIZE-WITH-ATTR-NEXT:    [[COUNT:%.*]] = sext i32 [[COUNTED_BY_LOAD]] to i64
+// NO-SANITIZE-WITH-ATTR-NEXT:    [[TMP0:%.*]] = sub nsw i64 [[COUNT]], [[IDXPROM]]
+// NO-SANITIZE-WITH-ATTR-NEXT:    [[RESULT:%.*]] = shl nsw i64 [[TMP0]], 2
+// NO-SANITIZE-WITH-ATTR-NEXT:    [[TMP1:%.*]] = icmp sgt i64 [[TMP0]], -1
+// NO-SANITIZE-WITH-ATTR-NEXT:    [[TMP2:%.*]] = icmp sgt i32 [[INDEX]], -1
+// NO-SANITIZE-WITH-ATTR-NEXT:    [[TMP3:%.*]] = and i1 [[TMP2]], [[TMP1]]
+// NO-SANITIZE-WITH-ATTR-NEXT:    [[TMP4:%.*]] = select i1 [[TMP3]], i64 [[RESULT]], i64 0
+// NO-SANITIZE-WITH-ATTR-NEXT:    ret i64 [[TMP4]]
 //
 // SANITIZE-WITHOUT-ATTR-LABEL: define dso_local i64 @test4_bdos(
 // SANITIZE-WITHOUT-ATTR-SAME: ptr noundef [[P:%.*]], i32 noundef [[INDEX:%.*]]) local_unnamed_addr #[[ATTR0]] {
@@ -445,7 +466,7 @@ void test4(struct annotated *p, int index, int fam_idx) {
 // NO-SANITIZE-WITHOUT-ATTR-NEXT:    ret i64 -1
 //
 size_t test4_bdos(struct annotated *p, int index) {
-  return __builtin_dynamic_object_size(&p->array[index], 1);
+  return __bdos(&p->array[index]);
 }
 
 // SANITIZE-WITH-ATTR-LABEL: define dso_local void @test5(
@@ -457,7 +478,7 @@ size_t test4_bdos(struct annotated *p, int index) {
 // SANITIZE-WITH-ATTR-NEXT:    [[TMP0:%.*]] = icmp ugt i64 [[DOTCOUNTED_BY_LOAD]], [[IDXPROM]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR-NEXT:    br i1 [[TMP0]], label [[CONT3:%.*]], label [[HANDLER_OUT_OF_BOUNDS:%.*]], !prof [[PROF3]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR:       handler.out_of_bounds:
-// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB8:[0-9]+]], i64 [[IDXPROM]]) #[[ATTR8]], !nosanitize [[META2]]
+// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB12:[0-9]+]], i64 [[IDXPROM]]) #[[ATTR8]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR-NEXT:    unreachable, !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR:       cont3:
 // SANITIZE-WITH-ATTR-NEXT:    [[TMP1:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i64 16
@@ -493,11 +514,11 @@ size_t test4_bdos(struct annotated *p, int index) {
 // NO-SANITIZE-WITHOUT-ATTR-NEXT:    ret void
 //
 void test5(struct anon_struct *p, int index) {
-  p->array[index] = __builtin_dynamic_object_size(p, 1);
+  p->array[index] = __bdos(p);
 }
 
 // SANITIZE-WITH-ATTR-LABEL: define dso_local i64 @test5_bdos(
-// SANITIZE-WITH-ATTR-SAME: ptr noundef readnone [[P:%.*]]) local_unnamed_addr #[[ATTR3]] {
+// SANITIZE-WITH-ATTR-SAME: ptr noundef readnone [[P:%.*]]) local_unnamed_addr #[[ATTR2]] {
 // SANITIZE-WITH-ATTR-NEXT:  entry:
 // SANITIZE-WITH-ATTR-NEXT:    ret i64 -1
 //
@@ -517,7 +538,7 @@ void test5(struct anon_struct *p, int index) {
 // NO-SANITIZE-WITHOUT-ATTR-NEXT:    ret i64 -1
 //
 size_t test5_bdos(struct anon_struct *p) {
-  return __builtin_dynamic_object_size(p, 1);
+  return __bdos(p);
 }
 
 // SANITIZE-WITH-ATTR-LABEL: define dso_local void @test6(
@@ -527,30 +548,30 @@ size_t test5_bdos(struct anon_struct *p) {
 // SANITIZE-WITH-ATTR-NEXT:    [[COUNTED_BY_LOAD:%.*]] = load i64, ptr [[COUNTED_BY_GEP]], align 4
 // SANITIZE-WITH-ATTR-NEXT:    [[IDXPROM:%.*]] = sext i32 [[INDEX]] to i64
 // SANITIZE-WITH-ATTR-NEXT:    [[TMP0:%.*]] = icmp ugt i64 [[COUNTED_BY_LOAD]], [[IDXPROM]], !nosanitize [[META2]]
-// SANITIZE-WITH-ATTR-NEXT:    br i1 [[TMP0]], label [[CONT3:%.*]], label [[HANDLER_OUT_OF_BOUNDS:%.*]], !prof [[PROF3]], !nosanitize [[META2]]
+// SANITIZE-WITH-ATTR-NEXT:    br i1 [[TMP0]], label [[CONT6:%.*]], label [[HANDLER_OUT_OF_BOUNDS:%.*]], !prof [[PROF3]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR:       handler.out_of_bounds:
-// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB9:[0-9]+]], i64 [[IDXPROM]]) #[[ATTR8]], !nosanitize [[META2]]
+// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB13:[0-9]+]], i64 [[IDXPROM]]) #[[ATTR8]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR-NEXT:    unreachable, !nosanitize [[META2]]
-// SANITIZE-WITH-ATTR:       cont3:
+// SANITIZE-WITH-ATTR:       cont6:
 // SANITIZE-WITH-ATTR-NEXT:    [[TMP1:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i64 16
 // SANITIZE-WITH-ATTR-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [0 x i32], ptr [[TMP1]], i64 0, i64 [[IDXPROM]]
-// SANITIZE-WITH-ATTR-NEXT:    [[TMP2:%.*]] = tail call i64 @llvm.smax.i64(i64 [[COUNTED_BY_LOAD]], i64 0)
-// SANITIZE-WITH-ATTR-NEXT:    [[DOTTR:%.*]] = trunc i64 [[TMP2]] to i32
-// SANITIZE-WITH-ATTR-NEXT:    [[CONV:%.*]] = shl i32 [[DOTTR]], 2
+// SANITIZE-WITH-ATTR-NEXT:    [[FLEXIBLE_ARRAY_MEMBER_SIZE:%.*]] = shl nuw i64 [[COUNTED_BY_LOAD]], 2
+// SANITIZE-WITH-ATTR-NEXT:    [[TMP2:%.*]] = tail call i64 @llvm.smax.i64(i64 [[FLEXIBLE_ARRAY_MEMBER_SIZE]], i64 0)
+// SANITIZE-WITH-ATTR-NEXT:    [[CONV:%.*]] = trunc i64 [[TMP2]] to i32
 // SANITIZE-WITH-ATTR-NEXT:    store i32 [[CONV]], ptr [[ARRAYIDX]], align 4, !tbaa [[TBAA4]]
 // SANITIZE-WITH-ATTR-NEXT:    ret void
 //
 // NO-SANITIZE-WITH-ATTR-LABEL: define dso_local void @test6(
-// NO-SANITIZE-WITH-ATTR-SAME: ptr nocapture noundef [[P:%.*]], i32 noundef [[INDEX:%.*]]) local_unnamed_addr #[[ATTR1]] {
+// NO-SANITIZE-WITH-ATTR-SAME: ptr noundef captures(none) [[P:%.*]], i32 noundef [[INDEX:%.*]]) local_unnamed_addr #[[ATTR1]] {
 // NO-SANITIZE-WITH-ATTR-NEXT:  entry:
+// NO-SANITIZE-WITH-ATTR-NEXT:    [[TMP0:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i64 16
 // NO-SANITIZE-WITH-ATTR-NEXT:    [[COUNTED_BY_GEP:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i64 8
 // NO-SANITIZE-WITH-ATTR-NEXT:    [[COUNTED_BY_LOAD:%.*]] = load i64, ptr [[COUNTED_BY_GEP]], align 4
-// NO-SANITIZE-WITH-ATTR-NEXT:    [[TMP0:%.*]] = tail call i64 @llvm.smax.i64(i64 [[COUNTED_BY_LOAD]], i64 0)
-// NO-SANITIZE-WITH-ATTR-NEXT:    [[DOTTR:%.*]] = trunc i64 [[TMP0]] to i32
-// NO-SANITIZE-WITH-ATTR-NEXT:    [[CONV:%.*]] = shl i32 [[DOTTR]], 2
-// NO-SANITIZE-WITH-ATTR-NEXT:    [[TMP1:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i64 16
+// NO-SANITIZE-WITH-ATTR-NEXT:    [[FLEXIBLE_ARRAY_MEMBER_SIZE:%.*]] = shl nuw i64 [[COUNTED_BY_LOAD]], 2
+// NO-SANITIZE-WITH-ATTR-NEXT:    [[TMP1:%.*]] = tail call i64 @llvm.smax.i64(i64 [[FLEXIBLE_ARRAY_MEMBER_SIZE]], i64 0)
+// NO-SANITIZE-WITH-ATTR-NEXT:    [[CONV:%.*]] = trunc i64 [[TMP1]] to i32
 // NO-SANITIZE-WITH-ATTR-NEXT:    [[IDXPROM:%.*]] = sext i32 [[INDEX]] to i64
-// NO-SANITIZE-WITH-ATTR-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [0 x i32], ptr [[TMP1]], i64 0, i64 [[IDXPROM]]
+// NO-SANITIZE-WITH-ATTR-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [0 x i32], ptr [[TMP0]], i64 0, i64 [[IDXPROM]]
 // NO-SANITIZE-WITH-ATTR-NEXT:    store i32 [[CONV]], ptr [[ARRAYIDX]], align 4, !tbaa [[TBAA2]]
 // NO-SANITIZE-WITH-ATTR-NEXT:    ret void
 //
@@ -573,26 +594,26 @@ size_t test5_bdos(struct anon_struct *p) {
 // NO-SANITIZE-WITHOUT-ATTR-NEXT:    ret void
 //
 void test6(struct anon_struct *p, int index) {
-  p->array[index] = __builtin_dynamic_object_size(p->array, 1);
+  p->array[index] = __bdos(p->array);
 }
 
-// SANITIZE-WITH-ATTR-LABEL: define dso_local range(i64 0, -3) i64 @test6_bdos(
-// SANITIZE-WITH-ATTR-SAME: ptr nocapture noundef readonly [[P:%.*]]) local_unnamed_addr #[[ATTR2]] {
+// SANITIZE-WITH-ATTR-LABEL: define dso_local range(i64 0, -9223372036854775808) i64 @test6_bdos(
+// SANITIZE-WITH-ATTR-SAME: ptr noundef [[P:%.*]]) local_unnamed_addr #[[ATTR0]] {
 // SANITIZE-WITH-ATTR-NEXT:  entry:
 // SANITIZE-WITH-ATTR-NEXT:    [[COUNTED_BY_GEP:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i64 8
 // SANITIZE-WITH-ATTR-NEXT:    [[COUNTED_BY_LOAD:%.*]] = load i64, ptr [[COUNTED_BY_GEP]], align 4
-// SANITIZE-WITH-ATTR-NEXT:    [[TMP0:%.*]] = tail call i64 @llvm.smax.i64(i64 [[COUNTED_BY_LOAD]], i64 0)
-// SANITIZE-WITH-ATTR-NEXT:    [[TMP1:%.*]] = shl i64 [[TMP0]], 2
-// SANITIZE-WITH-ATTR-NEXT:    ret i64 [[TMP1]]
+// SANITIZE-WITH-ATTR-NEXT:    [[FLEXIBLE_ARRAY_MEMBER_SIZE:%.*]] = shl nuw i64 [[COUNTED_BY_LOAD]], 2
+// SANITIZE-WITH-ATTR-NEXT:    [[TMP0:%.*]] = tail call i64 @llvm.smax.i64(i64 [[FLEXIBLE_ARRAY_MEMBER_SIZE]], i64 0)
+// SANITIZE-WITH-ATTR-NEXT:    ret i64 [[TMP0]]
 //
-// NO-SANITIZE-WITH-ATTR-LABEL: define dso_local range(i64 0, -3) i64 @test6_bdos(
-// NO-SANITIZE-WITH-ATTR-SAME: ptr nocapture noundef readonly [[P:%.*]]) local_unnamed_addr #[[ATTR2]] {
+// NO-SANITIZE-WITH-ATTR-LABEL: define dso_local range(i64 0, -9223372036854775808) i64 @test6_bdos(
+// NO-SANITIZE-WITH-ATTR-SAME: ptr noundef readonly captures(none) [[P:%.*]]) local_unnamed_addr #[[ATTR2]] {
 // NO-SANITIZE-WITH-ATTR-NEXT:  entry:
 // NO-SANITIZE-WITH-ATTR-NEXT:    [[COUNTED_BY_GEP:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i64 8
 // NO-SANITIZE-WITH-ATTR-NEXT:    [[COUNTED_BY_LOAD:%.*]] = load i64, ptr [[COUNTED_BY_GEP]], align 4
-// NO-SANITIZE-WITH-ATTR-NEXT:    [[TMP0:%.*]] = tail call i64 @llvm.smax.i64(i64 [[COUNTED_BY_LOAD]], i64 0)
-// NO-SANITIZE-WITH-ATTR-NEXT:    [[TMP1:%.*]] = shl i64 [[TMP0]], 2
-// NO-SANITIZE-WITH-ATTR-NEXT:    ret i64 [[TMP1]]
+// NO-SANITIZE-WITH-ATTR-NEXT:    [[FLEXIBLE_ARRAY_MEMBER_SIZE:%.*]] = shl nuw i64 [[COUNTED_BY_LOAD]], 2
+// NO-SANITIZE-WITH-ATTR-NEXT:    [[TMP0:%.*]] = tail call i64 @llvm.smax.i64(i64 [[FLEXIBLE_ARRAY_MEMBER_SIZE]], i64 0)
+// NO-SANITIZE-WITH-ATTR-NEXT:    ret i64 [[TMP0]]
 //
 // SANITIZE-WITHOUT-ATTR-LABEL: define dso_local i64 @test6_bdos(
 // SANITIZE-WITHOUT-ATTR-SAME: ptr noundef [[P:%.*]]) local_unnamed_addr #[[ATTR0]] {
@@ -605,7 +626,7 @@ void test6(struct anon_struct *p, int index) {
 // NO-SANITIZE-WITHOUT-ATTR-NEXT:    ret i64 -1
 //
 size_t test6_bdos(struct anon_struct *p) {
-  return __builtin_dynamic_object_size(p->array, 1);
+  return __bdos(p->array);
 }
 
 // SANITIZE-WITH-ATTR-LABEL: define dso_local void @test7(
@@ -618,12 +639,12 @@ size_t test6_bdos(struct anon_struct *p) {
 // SANITIZE-WITH-ATTR-NEXT:    [[TMP2:%.*]] = icmp ult i64 [[IDXPROM]], [[TMP1]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR-NEXT:    br i1 [[TMP2]], label [[CONT7:%.*]], label [[HANDLER_OUT_OF_BOUNDS:%.*]], !prof [[PROF3]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR:       handler.out_of_bounds:
-// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB11:[0-9]+]], i64 [[IDXPROM]]) #[[ATTR8]], !nosanitize [[META2]]
+// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB15:[0-9]+]], i64 [[IDXPROM]]) #[[ATTR8]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR-NEXT:    unreachable, !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR:       cont7:
 // SANITIZE-WITH-ATTR-NEXT:    [[INTS:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i64 9
 // SANITIZE-WITH-ATTR-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds nuw [0 x i8], ptr [[INTS]], i64 0, i64 [[IDXPROM]]
-// SANITIZE-WITH-ATTR-NEXT:    store i8 -1, ptr [[ARRAYIDX]], align 1, !tbaa [[TBAA8:![0-9]+]]
+// SANITIZE-WITH-ATTR-NEXT:    store i8 -1, ptr [[ARRAYIDX]], align 1, !tbaa [[TBAA9:![0-9]+]]
 // SANITIZE-WITH-ATTR-NEXT:    ret void
 //
 // NO-SANITIZE-WITH-ATTR-LABEL: define dso_local void @test7(
@@ -654,11 +675,11 @@ size_t test6_bdos(struct anon_struct *p) {
 // NO-SANITIZE-WITHOUT-ATTR-NEXT:    ret void
 //
 void test7(struct union_of_fams *p, int index) {
-  p->ints[index] = __builtin_dynamic_object_size(p, 1);
+  p->ints[index] = __bdos(p);
 }
 
 // SANITIZE-WITH-ATTR-LABEL: define dso_local i64 @test7_bdos(
-// SANITIZE-WITH-ATTR-SAME: ptr noundef readnone [[P:%.*]]) local_unnamed_addr #[[ATTR3]] {
+// SANITIZE-WITH-ATTR-SAME: ptr noundef readnone [[P:%.*]]) local_unnamed_addr #[[ATTR2]] {
 // SANITIZE-WITH-ATTR-NEXT:  entry:
 // SANITIZE-WITH-ATTR-NEXT:    ret i64 -1
 //
@@ -678,33 +699,33 @@ void test7(struct union_of_fams *p, int index) {
 // NO-SANITIZE-WITHOUT-ATTR-NEXT:    ret i64 -1
 //
 size_t test7_bdos(struct union_of_fams *p) {
-  return __builtin_dynamic_object_size(p, 1);
+  return __bdos(p);
 }
 
 // SANITIZE-WITH-ATTR-LABEL: define dso_local void @test8(
 // SANITIZE-WITH-ATTR-SAME: ptr noundef [[P:%.*]], i32 noundef [[INDEX:%.*]]) local_unnamed_addr #[[ATTR0]] {
 // SANITIZE-WITH-ATTR-NEXT:  entry:
-// SANITIZE-WITH-ATTR-NEXT:    [[COUNTED_BY_GEP:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i64 8
-// SANITIZE-WITH-ATTR-NEXT:    [[COUNTED_BY_LOAD:%.*]] = load i8, ptr [[COUNTED_BY_GEP]], align 4
+// SANITIZE-WITH-ATTR-NEXT:    [[TMP0:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i64 8
+// SANITIZE-WITH-ATTR-NEXT:    [[COUNTED_BY_LOAD:%.*]] = load i8, ptr [[TMP0]], align 4
 // SANITIZE-WITH-ATTR-NEXT:    [[IDXPROM:%.*]] = sext i32 [[INDEX]] to i64
-// SANITIZE-WITH-ATTR-NEXT:    [[TMP0:%.*]] = zext i8 [[COUNTED_BY_LOAD]] to i64, !nosanitize [[META2]]
-// SANITIZE-WITH-ATTR-NEXT:    [[TMP1:%.*]] = icmp ult i64 [[IDXPROM]], [[TMP0]], !nosanitize [[META2]]
-// SANITIZE-WITH-ATTR-NEXT:    br i1 [[TMP1]], label [[CONT7:%.*]], label [[HANDLER_OUT_OF_BOUNDS:%.*]], !prof [[PROF3]], !nosanitize [[META2]]
+// SANITIZE-WITH-ATTR-NEXT:    [[TMP1:%.*]] = zext i8 [[COUNTED_BY_LOAD]] to i64, !nosanitize [[META2]]
+// SANITIZE-WITH-ATTR-NEXT:    [[TMP2:%.*]] = icmp ult i64 [[IDXPROM]], [[TMP1]], !nosanitize [[META2]]
+// SANITIZE-WITH-ATTR-NEXT:    br i1 [[TMP2]], label [[CONT14:%.*]], label [[HANDLER_OUT_OF_BOUNDS:%.*]], !prof [[PROF3]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR:       handler.out_of_bounds:
-// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB12:[0-9]+]], i64 [[IDXPROM]]) #[[ATTR8]], !nosanitize [[META2]]
+// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB16:[0-9]+]], i64 [[IDXPROM]]) #[[ATTR8]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR-NEXT:    unreachable, !nosanitize [[META2]]
-// SANITIZE-WITH-ATTR:       cont7:
+// SANITIZE-WITH-ATTR:       cont14:
 // SANITIZE-WITH-ATTR-NEXT:    [[INTS:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i64 9
 // SANITIZE-WITH-ATTR-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds nuw [0 x i8], ptr [[INTS]], i64 0, i64 [[IDXPROM]]
-// SANITIZE-WITH-ATTR-NEXT:    store i8 [[COUNTED_BY_LOAD]], ptr [[ARRAYIDX]], align 1, !tbaa [[TBAA8]]
+// SANITIZE-WITH-ATTR-NEXT:    store i8 [[COUNTED_BY_LOAD]], ptr [[ARRAYIDX]], align 1, !tbaa [[TBAA9]]
 // SANITIZE-WITH-ATTR-NEXT:    ret void
 //
 // NO-SANITIZE-WITH-ATTR-LABEL: define dso_local void @test8(
-// NO-SANITIZE-WITH-ATTR-SAME: ptr nocapture noundef [[P:%.*]], i32 noundef [[INDEX:%.*]]) local_unnamed_addr #[[ATTR1]] {
+// NO-SANITIZE-WITH-ATTR-SAME: ptr noundef captures(none) [[P:%.*]], i32 noundef [[INDEX:%.*]]) local_unnamed_addr #[[ATTR1]] {
 // NO-SANITIZE-WITH-ATTR-NEXT:  entry:
-// NO-SANITIZE-WITH-ATTR-NEXT:    [[COUNTED_BY_GEP:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i64 8
-// NO-SANITIZE-WITH-ATTR-NEXT:    [[COUNTED_BY_LOAD:%.*]] = load i8, ptr [[COUNTED_BY_GEP]], align 4
+// NO-SANITIZE-WITH-ATTR-NEXT:    [[TMP0:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i64 8
 // NO-SANITIZE-WITH-ATTR-NEXT:    [[INTS:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i64 9
+// NO-SANITIZE-WITH-ATTR-NEXT:    [[COUNTED_BY_LOAD:%.*]] = load i8, ptr [[TMP0]], align 4
 // NO-SANITIZE-WITH-ATTR-NEXT:    [[IDXPROM:%.*]] = sext i32 [[INDEX]] to i64
 // NO-SANITIZE-WITH-ATTR-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [0 x i8], ptr [[INTS]], i64 0, i64 [[IDXPROM]]
 // NO-SANITIZE-WITH-ATTR-NEXT:    store i8 [[COUNTED_BY_LOAD]], ptr [[ARRAYIDX]], align 1, !tbaa [[TBAA6]]
@@ -729,24 +750,24 @@ size_t test7_bdos(struct union_of_fams *p) {
 // NO-SANITIZE-WITHOUT-ATTR-NEXT:    ret void
 //
 void test8(struct union_of_fams *p, int index) {
-  p->ints[index] = __builtin_dynamic_object_size(p->ints, 1);
+  p->ints[index] = __bdos(p->ints);
 }
 
 // SANITIZE-WITH-ATTR-LABEL: define dso_local range(i64 0, 256) i64 @test8_bdos(
-// SANITIZE-WITH-ATTR-SAME: ptr nocapture noundef readonly [[P:%.*]]) local_unnamed_addr #[[ATTR2]] {
+// SANITIZE-WITH-ATTR-SAME: ptr noundef [[P:%.*]]) local_unnamed_addr #[[ATTR0]] {
 // SANITIZE-WITH-ATTR-NEXT:  entry:
-// SANITIZE-WITH-ATTR-NEXT:    [[COUNTED_BY_GEP:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i64 8
-// SANITIZE-WITH-ATTR-NEXT:    [[COUNTED_BY_LOAD:%.*]] = load i8, ptr [[COUNTED_BY_GEP]], align 4
-// SANITIZE-WITH-ATTR-NEXT:    [[TMP0:%.*]] = zext i8 [[COUNTED_BY_LOAD]] to i64
-// SANITIZE-WITH-ATTR-NEXT:    ret i64 [[TMP0]]
+// SANITIZE-WITH-ATTR-NEXT:    [[TMP0:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i64 8
+// SANITIZE-WITH-ATTR-NEXT:    [[COUNTED_BY_LOAD:%.*]] = load i8, ptr [[TMP0]], align 4
+// SANITIZE-WITH-ATTR-NEXT:    [[COUNT:%.*]] = zext i8 [[COUNTED_BY_LOAD]] to i64
+// SANITIZE-WITH-ATTR-NEXT:    ret i64 [[COUNT]]
 //
 // NO-SANITIZE-WITH-ATTR-LABEL: define dso_local range(i64 0, 256) i64 @test8_bdos(
-// NO-SANITIZE-WITH-ATTR-SAME: ptr nocapture noundef readonly [[P:%.*]]) local_unnamed_addr #[[ATTR2]] {
+// NO-SANITIZE-WITH-ATTR-SAME: ptr noundef readonly captures(none) [[P:%.*]]) local_unnamed_addr #[[ATTR2]] {
 // NO-SANITIZE-WITH-ATTR-NEXT:  entry:
-// NO-SANITIZE-WITH-ATTR-NEXT:    [[COUNTED_BY_GEP:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i64 8
-// NO-SANITIZE-WITH-ATTR-NEXT:    [[COUNTED_BY_LOAD:%.*]] = load i8, ptr [[COUNTED_BY_GEP]], align 4
-// NO-SANITIZE-WITH-ATTR-NEXT:    [[TMP0:%.*]] = zext i8 [[COUNTED_BY_LOAD]] to i64
-// NO-SANITIZE-WITH-ATTR-NEXT:    ret i64 [[TMP0]]
+// NO-SANITIZE-WITH-ATTR-NEXT:    [[TMP0:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i64 8
+// NO-SANITIZE-WITH-ATTR-NEXT:    [[COUNTED_BY_LOAD:%.*]] = load i8, ptr [[TMP0]], align 4
+// NO-SANITIZE-WITH-ATTR-NEXT:    [[COUNT:%.*]] = zext i8 [[COUNTED_BY_LOAD]] to i64
+// NO-SANITIZE-WITH-ATTR-NEXT:    ret i64 [[COUNT]]
 //
 // SANITIZE-WITHOUT-ATTR-LABEL: define dso_local i64 @test8_bdos(
 // SANITIZE-WITHOUT-ATTR-SAME: ptr noundef [[P:%.*]]) local_unnamed_addr #[[ATTR0]] {
@@ -759,7 +780,7 @@ void test8(struct union_of_fams *p, int index) {
 // NO-SANITIZE-WITHOUT-ATTR-NEXT:    ret i64 -1
 //
 size_t test8_bdos(struct union_of_fams *p) {
-  return __builtin_dynamic_object_size(p->ints, 1);
+  return __bdos(p->ints);
 }
 
 // SANITIZE-WITH-ATTR-LABEL: define dso_local void @test9(
@@ -772,12 +793,12 @@ size_t test8_bdos(struct union_of_fams *p) {
 // SANITIZE-WITH-ATTR-NEXT:    [[TMP2:%.*]] = icmp ult i64 [[IDXPROM]], [[TMP1]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR-NEXT:    br i1 [[TMP2]], label [[CONT7:%.*]], label [[HANDLER_OUT_OF_BOUNDS:%.*]], !prof [[PROF3]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR:       handler.out_of_bounds:
-// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB14:[0-9]+]], i64 [[IDXPROM]]) #[[ATTR8]], !nosanitize [[META2]]
+// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB18:[0-9]+]], i64 [[IDXPROM]]) #[[ATTR8]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR-NEXT:    unreachable, !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR:       cont7:
 // SANITIZE-WITH-ATTR-NEXT:    [[BYTES:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i64 12
 // SANITIZE-WITH-ATTR-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds nuw [0 x i8], ptr [[BYTES]], i64 0, i64 [[IDXPROM]]
-// SANITIZE-WITH-ATTR-NEXT:    store i8 -1, ptr [[ARRAYIDX]], align 1, !tbaa [[TBAA8]]
+// SANITIZE-WITH-ATTR-NEXT:    store i8 -1, ptr [[ARRAYIDX]], align 1, !tbaa [[TBAA9]]
 // SANITIZE-WITH-ATTR-NEXT:    ret void
 //
 // NO-SANITIZE-WITH-ATTR-LABEL: define dso_local void @test9(
@@ -808,11 +829,11 @@ size_t test8_bdos(struct union_of_fams *p) {
 // NO-SANITIZE-WITHOUT-ATTR-NEXT:    ret void
 //
 void test9(struct union_of_fams *p, int index) {
-  p->bytes[index] = (unsigned char)__builtin_dynamic_object_size(p, 1);
+  p->bytes[index] = (unsigned char)__bdos(p);
 }
 
 // SANITIZE-WITH-ATTR-LABEL: define dso_local i64 @test9_bdos(
-// SANITIZE-WITH-ATTR-SAME: ptr noundef readnone [[P:%.*]]) local_unnamed_addr #[[ATTR3]] {
+// SANITIZE-WITH-ATTR-SAME: ptr noundef readnone [[P:%.*]]) local_unnamed_addr #[[ATTR2]] {
 // SANITIZE-WITH-ATTR-NEXT:  entry:
 // SANITIZE-WITH-ATTR-NEXT:    ret i64 -1
 //
@@ -832,37 +853,37 @@ void test9(struct union_of_fams *p, int index) {
 // NO-SANITIZE-WITHOUT-ATTR-NEXT:    ret i64 -1
 //
 size_t test9_bdos(struct union_of_fams *p) {
-  return __builtin_dynamic_object_size(p, 1);
+  return __bdos(p);
 }
 
 // SANITIZE-WITH-ATTR-LABEL: define dso_local void @test10(
 // SANITIZE-WITH-ATTR-SAME: ptr noundef [[P:%.*]], i32 noundef [[INDEX:%.*]]) local_unnamed_addr #[[ATTR0]] {
 // SANITIZE-WITH-ATTR-NEXT:  entry:
-// SANITIZE-WITH-ATTR-NEXT:    [[COUNTED_BY_GEP:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i64 8
-// SANITIZE-WITH-ATTR-NEXT:    [[COUNTED_BY_LOAD:%.*]] = load i32, ptr [[COUNTED_BY_GEP]], align 4
+// SANITIZE-WITH-ATTR-NEXT:    [[TMP0:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i64 8
+// SANITIZE-WITH-ATTR-NEXT:    [[COUNTED_BY_LOAD:%.*]] = load i32, ptr [[TMP0]], align 4
 // SANITIZE-WITH-ATTR-NEXT:    [[IDXPROM:%.*]] = sext i32 [[INDEX]] to i64
-// SANITIZE-WITH-ATTR-NEXT:    [[TMP0:%.*]] = zext i32 [[COUNTED_BY_LOAD]] to i64, !nosanitize [[META2]]
-// SANITIZE-WITH-ATTR-NEXT:    [[TMP1:%.*]] = icmp ult i64 [[IDXPROM]], [[TMP0]], !nosanitize [[META2]]
-// SANITIZE-WITH-ATTR-NEXT:    br i1 [[TMP1]], label [[CONT7:%.*]], label [[HANDLER_OUT_OF_BOUNDS:%.*]], !prof [[PROF3]], !nosanitize [[META2]]
+// SANITIZE-WITH-ATTR-NEXT:    [[TMP1:%.*]] = zext i32 [[COUNTED_BY_LOAD]] to i64, !nosanitize [[META2]]
+// SANITIZE-WITH-ATTR-NEXT:    [[TMP2:%.*]] = icmp ult i64 [[IDXPROM]], [[TMP1]], !nosanitize [[META2]]
+// SANITIZE-WITH-ATTR-NEXT:    br i1 [[TMP2]], label [[CONT14:%.*]], label [[HANDLER_OUT_OF_BOUNDS:%.*]], !prof [[PROF3]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR:       handler.out_of_bounds:
-// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB15:[0-9]+]], i64 [[IDXPROM]]) #[[ATTR8]], !nosanitize [[META2]]
+// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB19:[0-9]+]], i64 [[IDXPROM]]) #[[ATTR8]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR-NEXT:    unreachable, !nosanitize [[META2]]
-// SANITIZE-WITH-ATTR:       cont7:
+// SANITIZE-WITH-ATTR:       cont14:
 // SANITIZE-WITH-ATTR-NEXT:    [[BYTES:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i64 12
 // SANITIZE-WITH-ATTR-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds nuw [0 x i8], ptr [[BYTES]], i64 0, i64 [[IDXPROM]]
 // SANITIZE-WITH-ATTR-NEXT:    [[NARROW:%.*]] = tail call i32 @llvm.smax.i32(i32 [[COUNTED_BY_LOAD]], i32 0)
 // SANITIZE-WITH-ATTR-NEXT:    [[CONV:%.*]] = trunc i32 [[NARROW]] to i8
-// SANITIZE-WITH-ATTR-NEXT:    store i8 [[CONV]], ptr [[ARRAYIDX]], align 1, !tbaa [[TBAA8]]
+// SANITIZE-WITH-ATTR-NEXT:    store i8 [[CONV]], ptr [[ARRAYIDX]], align 1, !tbaa [[TBAA9]]
 // SANITIZE-WITH-ATTR-NEXT:    ret void
 //
 // NO-SANITIZE-WITH-ATTR-LABEL: define dso_local void @test10(
-// NO-SANITIZE-WITH-ATTR-SAME: ptr nocapture noundef [[P:%.*]], i32 noundef [[INDEX:%.*]]) local_unnamed_addr #[[ATTR1]] {
+// NO-SANITIZE-WITH-ATTR-SAME: ptr noundef captures(none) [[P:%.*]], i32 noundef [[INDEX:%.*]]) local_unnamed_addr #[[ATTR1]] {
 // NO-SANITIZE-WITH-ATTR-NEXT:  entry:
-// NO-SANITIZE-WITH-ATTR-NEXT:    [[COUNTED_BY_GEP:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i64 8
-// NO-SANITIZE-WITH-ATTR-NEXT:    [[COUNTED_BY_LOAD:%.*]] = load i32, ptr [[COUNTED_BY_GEP]], align 4
+// NO-SANITIZE-WITH-ATTR-NEXT:    [[TMP0:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i64 8
+// NO-SANITIZE-WITH-ATTR-NEXT:    [[BYTES:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i64 12
+// NO-SANITIZE-WITH-ATTR-NEXT:    [[COUNTED_BY_LOAD:%.*]] = load i32, ptr [[TMP0]], align 4
 // NO-SANITIZE-WITH-ATTR-NEXT:    [[NARROW:%.*]] = tail call i32 @llvm.smax.i32(i32 [[COUNTED_BY_LOAD]], i32 0)
 // NO-SANITIZE-WITH-ATTR-NEXT:    [[CONV:%.*]] = trunc i32 [[NARROW]] to i8
-// NO-SANITIZE-WITH-ATTR-NEXT:    [[BYTES:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i64 12
 // NO-SANITIZE-WITH-ATTR-NEXT:    [[IDXPROM:%.*]] = sext i32 [[INDEX]] to i64
 // NO-SANITIZE-WITH-ATTR-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [0 x i8], ptr [[BYTES]], i64 0, i64 [[IDXPROM]]
 // NO-SANITIZE-WITH-ATTR-NEXT:    store i8 [[CONV]], ptr [[ARRAYIDX]], align 1, !tbaa [[TBAA6]]
@@ -887,26 +908,26 @@ size_t test9_bdos(struct union_of_fams *p) {
 // NO-SANITIZE-WITHOUT-ATTR-NEXT:    ret void
 //
 void test10(struct union_of_fams *p, int index) {
-  p->bytes[index] = (unsigned char)__builtin_dynamic_object_size(p->bytes, 1);
+  p->bytes[index] = (unsigned char)__bdos(p->bytes);
 }
 
 // SANITIZE-WITH-ATTR-LABEL: define dso_local range(i64 0, 2147483648) i64 @test10_bdos(
-// SANITIZE-WITH-ATTR-SAME: ptr nocapture noundef readonly [[P:%.*]]) local_unnamed_addr #[[ATTR2]] {
+// SANITIZE-WITH-ATTR-SAME: ptr noundef [[P:%.*]]) local_unnamed_addr #[[ATTR0]] {
 // SANITIZE-WITH-ATTR-NEXT:  entry:
-// SANITIZE-WITH-ATTR-NEXT:    [[COUNTED_BY_GEP:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i64 8
-// SANITIZE-WITH-ATTR-NEXT:    [[COUNTED_BY_LOAD:%.*]] = load i32, ptr [[COUNTED_BY_GEP]], align 4
+// SANITIZE-WITH-ATTR-NEXT:    [[TMP0:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i64 8
+// SANITIZE-WITH-ATTR-NEXT:    [[COUNTED_BY_LOAD:%.*]] = load i32, ptr [[TMP0]], align 4
 // SANITIZE-WITH-ATTR-NEXT:    [[NARROW:%.*]] = tail call i32 @llvm.smax.i32(i32 [[COUNTED_BY_LOAD]], i32 0)
-// SANITIZE-WITH-ATTR-NEXT:    [[TMP0:%.*]] = zext nneg i32 [[NARROW]] to i64
-// SANITIZE-WITH-ATTR-NEXT:    ret i64 [[TMP0]]
+// SANITIZE-WITH-ATTR-NEXT:    [[TMP1:%.*]] = zext nneg i32 [[NARROW]] to i64
+// SANITIZE-WITH-ATTR-NEXT:    ret i64 [[TMP1]]
 //
 // NO-SANITIZE-WITH-ATTR-LABEL: define dso_local range(i64 0, 2147483648) i64 @test10_bdos(
-// NO-SANITIZE-WITH-ATTR-SAME: ptr nocapture noundef readonly [[P:%.*]]) local_unnamed_addr #[[ATTR2]] {
+// NO-SANITIZE-WITH-ATTR-SAME: ptr noundef readonly captures(none) [[P:%.*]]) local_unnamed_addr #[[ATTR2]] {
 // NO-SANITIZE-WITH-ATTR-NEXT:  entry:
-// NO-SANITIZE-WITH-ATTR-NEXT:    [[COUNTED_BY_GEP:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i64 8
-// NO-SANITIZE-WITH-ATTR-NEXT:    [[COUNTED_BY_LOAD:%.*]] = load i32, ptr [[COUNTED_BY_GEP]], align 4
+// NO-SANITIZE-WITH-ATTR-NEXT:    [[TMP0:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i64 8
+// NO-SANITIZE-WITH-ATTR-NEXT:    [[COUNTED_BY_LOAD:%.*]] = load i32, ptr [[TMP0]], align 4
 // NO-SANITIZE-WITH-ATTR-NEXT:    [[NARROW:%.*]] = tail call i32 @llvm.smax.i32(i32 [[COUNTED_BY_LOAD]], i32 0)
-// NO-SANITIZE-WITH-ATTR-NEXT:    [[TMP0:%.*]] = zext nneg i32 [[NARROW]] to i64
-// NO-SANITIZE-WITH-ATTR-NEXT:    ret i64 [[TMP0]]
+// NO-SANITIZE-WITH-ATTR-NEXT:    [[TMP1:%.*]] = zext nneg i32 [[NARROW]] to i64
+// NO-SANITIZE-WITH-ATTR-NEXT:    ret i64 [[TMP1]]
 //
 // SANITIZE-WITHOUT-ATTR-LABEL: define dso_local i64 @test10_bdos(
 // SANITIZE-WITHOUT-ATTR-SAME: ptr noundef [[P:%.*]]) local_unnamed_addr #[[ATTR0]] {
@@ -919,34 +940,44 @@ void test10(struct union_of_fams *p, int index) {
 // NO-SANITIZE-WITHOUT-ATTR-NEXT:    ret i64 -1
 //
 size_t test10_bdos(struct union_of_fams *p) {
-  return __builtin_dynamic_object_size(p->bytes, 1);
+  return __bdos(p->bytes);
 }
 
 // SANITIZE-WITH-ATTR-LABEL: define dso_local void @test11(
 // SANITIZE-WITH-ATTR-SAME: ptr noundef [[P:%.*]], i32 noundef [[INDEX:%.*]]) local_unnamed_addr #[[ATTR0]] {
 // SANITIZE-WITH-ATTR-NEXT:  entry:
+// SANITIZE-WITH-ATTR-NEXT:    [[COUNTED_BY_GEP:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i64 8
+// SANITIZE-WITH-ATTR-NEXT:    [[COUNTED_BY_LOAD:%.*]] = load i32, ptr [[COUNTED_BY_GEP]], align 4
 // SANITIZE-WITH-ATTR-NEXT:    [[IDXPROM:%.*]] = sext i32 [[INDEX]] to i64
-// SANITIZE-WITH-ATTR-NEXT:    [[DOTCOUNTED_BY_GEP:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i64 8
-// SANITIZE-WITH-ATTR-NEXT:    [[DOTCOUNTED_BY_LOAD:%.*]] = load i32, ptr [[DOTCOUNTED_BY_GEP]], align 4
-// SANITIZE-WITH-ATTR-NEXT:    [[TMP0:%.*]] = zext i32 [[DOTCOUNTED_BY_LOAD]] to i64, !nosanitize [[META2]]
+// SANITIZE-WITH-ATTR-NEXT:    [[TMP0:%.*]] = zext i32 [[COUNTED_BY_LOAD]] to i64, !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR-NEXT:    [[TMP1:%.*]] = icmp ult i64 [[IDXPROM]], [[TMP0]], !nosanitize [[META2]]
-// SANITIZE-WITH-ATTR-NEXT:    br i1 [[TMP1]], label [[CONT3:%.*]], label [[HANDLER_OUT_OF_BOUNDS:%.*]], !prof [[PROF3]], !nosanitize [[META2]]
+// SANITIZE-WITH-ATTR-NEXT:    br i1 [[TMP1]], label [[CONT6:%.*]], label [[HANDLER_OUT_OF_BOUNDS:%.*]], !prof [[PROF3]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR:       handler.out_of_bounds:
-// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB16:[0-9]+]], i64 [[IDXPROM]]) #[[ATTR8]], !nosanitize [[META2]]
+// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB20:[0-9]+]], i64 [[IDXPROM]]) #[[ATTR8]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR-NEXT:    unreachable, !nosanitize [[META2]]
-// SANITIZE-WITH-ATTR:       cont3:
+// SANITIZE-WITH-ATTR:       cont6:
 // SANITIZE-WITH-ATTR-NEXT:    [[ARRAY:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i64 12
 // SANITIZE-WITH-ATTR-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds nuw [0 x i32], ptr [[ARRAY]], i64 0, i64 [[IDXPROM]]
-// SANITIZE-WITH-ATTR-NEXT:    store i32 4, ptr [[ARRAYIDX]], align 4, !tbaa [[TBAA4]]
+// SANITIZE-WITH-ATTR-NEXT:    [[TMP2:%.*]] = icmp sgt i32 [[COUNTED_BY_LOAD]], -3
+// SANITIZE-WITH-ATTR-NEXT:    [[FLEXIBLE_ARRAY_MEMBER_SIZE:%.*]] = shl i32 [[COUNTED_BY_LOAD]], 2
+// SANITIZE-WITH-ATTR-NEXT:    [[RESULT:%.*]] = add i32 [[FLEXIBLE_ARRAY_MEMBER_SIZE]], 8
+// SANITIZE-WITH-ATTR-NEXT:    [[CONV:%.*]] = select i1 [[TMP2]], i32 [[RESULT]], i32 0
+// SANITIZE-WITH-ATTR-NEXT:    store i32 [[CONV]], ptr [[ARRAYIDX]], align 4, !tbaa [[TBAA4]]
 // SANITIZE-WITH-ATTR-NEXT:    ret void
 //
 // NO-SANITIZE-WITH-ATTR-LABEL: define dso_local void @test11(
-// NO-SANITIZE-WITH-ATTR-SAME: ptr nocapture noundef writeonly [[P:%.*]], i32 noundef [[INDEX:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// NO-SANITIZE-WITH-ATTR-SAME: ptr noundef captures(none) [[P:%.*]], i32 noundef [[INDEX:%.*]]) local_unnamed_addr #[[ATTR1]] {
 // NO-SANITIZE-WITH-ATTR-NEXT:  entry:
+// NO-SANITIZE-WITH-ATTR-NEXT:    [[COUNTED_BY_GEP:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i64 8
+// NO-SANITIZE-WITH-ATTR-NEXT:    [[COUNTED_BY_LOAD:%.*]] = load i32, ptr [[COUNTED_BY_GEP]], align 4
+// NO-SANITIZE-WITH-ATTR-NEXT:    [[FLEXIBLE_ARRAY_MEMBER_SIZE:%.*]] = shl i32 [[COUNTED_BY_LOAD]], 2
+// NO-SANITIZE-WITH-ATTR-NEXT:    [[RESULT:%.*]] = add i32 [[FLEXIBLE_ARRAY_MEMBER_SIZE]], 8
+// NO-SANITIZE-WITH-ATTR-NEXT:    [[TMP0:%.*]] = icmp sgt i32 [[COUNTED_BY_LOAD]], -3
+// NO-SANITIZE-WITH-ATTR-NEXT:    [[CONV:%.*]] = select i1 [[TMP0]], i32 [[RESULT]], i32 0
 // NO-SANITIZE-WITH-ATTR-NEXT:    [[ARRAY:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i64 12
 // NO-SANITIZE-WITH-ATTR-NEXT:    [[IDXPROM:%.*]] = sext i32 [[INDEX]] to i64
 // NO-SANITIZE-WITH-ATTR-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [0 x i32], ptr [[ARRAY]], i64 0, i64 [[IDXPROM]]
-// NO-SANITIZE-WITH-ATTR-NEXT:    store i32 4, ptr [[ARRAYIDX]], align 4, !tbaa [[TBAA2]]
+// NO-SANITIZE-WITH-ATTR-NEXT:    store i32 [[CONV]], ptr [[ARRAYIDX]], align 4, !tbaa [[TBAA2]]
 // NO-SANITIZE-WITH-ATTR-NEXT:    ret void
 //
 // SANITIZE-WITHOUT-ATTR-LABEL: define dso_local void @test11(
@@ -955,44 +986,58 @@ size_t test10_bdos(struct union_of_fams *p) {
 // SANITIZE-WITHOUT-ATTR-NEXT:    [[ARRAY:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i64 12
 // SANITIZE-WITHOUT-ATTR-NEXT:    [[IDXPROM:%.*]] = sext i32 [[INDEX]] to i64
 // SANITIZE-WITHOUT-ATTR-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [0 x i32], ptr [[ARRAY]], i64 0, i64 [[IDXPROM]]
-// SANITIZE-WITHOUT-ATTR-NEXT:    store i32 4, ptr [[ARRAYIDX]], align 4, !tbaa [[TBAA2]]
+// SANITIZE-WITHOUT-ATTR-NEXT:    store i32 -1, ptr [[ARRAYIDX]], align 4, !tbaa [[TBAA2]]
 // SANITIZE-WITHOUT-ATTR-NEXT:    ret void
 //
 // NO-SANITIZE-WITHOUT-ATTR-LABEL: define dso_local void @test11(
-// NO-SANITIZE-WITHOUT-ATTR-SAME: ptr nocapture noundef writeonly [[P:%.*]], i32 noundef [[INDEX:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// NO-SANITIZE-WITHOUT-ATTR-SAME: ptr noundef [[P:%.*]], i32 noundef [[INDEX:%.*]]) local_unnamed_addr #[[ATTR0]] {
 // NO-SANITIZE-WITHOUT-ATTR-NEXT:  entry:
 // NO-SANITIZE-WITHOUT-ATTR-NEXT:    [[ARRAY:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i64 12
 // NO-SANITIZE-WITHOUT-ATTR-NEXT:    [[IDXPROM:%.*]] = sext i32 [[INDEX]] to i64
 // NO-SANITIZE-WITHOUT-ATTR-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [0 x i32], ptr [[ARRAY]], i64 0, i64 [[IDXPROM]]
-// NO-SANITIZE-WITHOUT-ATTR-NEXT:    store i32 4, ptr [[ARRAYIDX]], align 4, !tbaa [[TBAA2]]
+// NO-SANITIZE-WITHOUT-ATTR-NEXT:    store i32 -1, ptr [[ARRAYIDX]], align 4, !tbaa [[TBAA2]]
 // NO-SANITIZE-WITHOUT-ATTR-NEXT:    ret void
 //
 void test11(struct annotated *p, int index) {
-  p->array[index] = __builtin_dynamic_object_size(&p->count, 1);
+  p->array[index] = __bdos(&p->count);
 }
 
-// SANITIZE-WITH-ATTR-LABEL: define dso_local noundef i64 @test11_bdos(
-// SANITIZE-WITH-ATTR-SAME: ptr nocapture noundef readnone [[P:%.*]]) local_unnamed_addr #[[ATTR3]] {
+// SANITIZE-WITH-ATTR-LABEL: define dso_local range(i64 -8589934584, 8589934597) i64 @test11_bdos(
+// SANITIZE-WITH-ATTR-SAME: ptr noundef [[P:%.*]]) local_unnamed_addr #[[ATTR0]] {
 // SANITIZE-WITH-ATTR-NEXT:  entry:
-// SANITIZE-WITH-ATTR-NEXT:    ret i64 4
+// SANITIZE-WITH-ATTR-NEXT:    [[COUNTED_BY_GEP:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i64 8
+// SANITIZE-WITH-ATTR-NEXT:    [[COUNTED_BY_LOAD:%.*]] = load i32, ptr [[COUNTED_BY_GEP]], align 4
+// SANITIZE-WITH-ATTR-NEXT:    [[COUNT1:%.*]] = sext i32 [[COUNTED_BY_LOAD]] to i64
+// SANITIZE-WITH-ATTR-NEXT:    [[FLEXIBLE_ARRAY_MEMBER_SIZE:%.*]] = shl nsw i64 [[COUNT1]], 2
+// SANITIZE-WITH-ATTR-NEXT:    [[RESULT:%.*]] = add nsw i64 [[FLEXIBLE_ARRAY_MEMBER_SIZE]], 8
+// SANITIZE-WITH-ATTR-NEXT:    [[TMP0:%.*]] = icmp sgt i32 [[COUNTED_BY_LOAD]], -3
+// SANITIZE-WITH-ATTR-NEXT:    [[TMP1:%.*]] = select i1 [[TMP0]], i64 [[RESULT]], i64 0
+// SANITIZE-WITH-ATTR-NEXT:    ret i64 [[TMP1]]
 //
-// NO-SANITIZE-WITH-ATTR-LABEL: define dso_local noundef i64 @test11_bdos(
-// NO-SANITIZE-WITH-ATTR-SAME: ptr nocapture noundef readnone [[P:%.*]]) local_unnamed_addr #[[ATTR3]] {
+// NO-SANITIZE-WITH-ATTR-LABEL: define dso_local range(i64 -8589934584, 8589934597) i64 @test11_bdos(
+// NO-SANITIZE-WITH-ATTR-SAME: ptr noundef readonly captures(none) [[P:%.*]]) local_unnamed_addr #[[ATTR2]] {
 // NO-SANITIZE-WITH-ATTR-NEXT:  entry:
-// NO-SANITIZE-WITH-ATTR-NEXT:    ret i64 4
+// NO-SANITIZE-WITH-ATTR-NEXT:    [[COUNTED_BY_GEP:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i64 8
+// NO-SANITIZE-WITH-ATTR-NEXT:    [[COUNTED_BY_LOAD:%.*]] = load i32, ptr [[COUNTED_BY_GEP]], align 4
+// NO-SANITIZE-WITH-ATTR-NEXT:    [[COUNT1:%.*]] = sext i32 [[COUNTED_BY_LOAD]] to i64
+// NO-SANITIZE-WITH-ATTR-NEXT:    [[FLEXIBLE_ARRAY_MEMBER_SIZE:%.*]] = shl nsw i64 [[COUNT1]], 2
+// NO-SANITIZE-WITH-ATTR-NEXT:    [[RESULT:%.*]] = add nsw i64 [[FLEXIBLE_ARRAY_MEMBER_SIZE]], 8
+// NO-SANITIZE-WITH-ATTR-NEXT:    [[TMP0:%.*]] = icmp sgt i32 [[COUNTED_BY_LOAD]], -3
+// NO-SANITIZE-WITH-ATTR-NEXT:    [[TMP1:%.*]] = select i1 [[TMP0]], i64 [[RESULT]], i64 0
+// NO-SANITIZE-WITH-ATTR-NEXT:    ret i64 [[TMP1]]
 //
-// SANITIZE-WITHOUT-ATTR-LABEL: define dso_local noundef i64 @test11_bdos(
-// SANITIZE-WITHOUT-ATTR-SAME: ptr nocapture noundef readnone [[P:%.*]]) local_unnamed_addr #[[ATTR2]] {
+// SANITIZE-WITHOUT-ATTR-LABEL: define dso_local i64 @test11_bdos(
+// SANITIZE-WITHOUT-ATTR-SAME: ptr noundef [[P:%.*]]) local_unnamed_addr #[[ATTR0]] {
 // SANITIZE-WITHOUT-ATTR-NEXT:  entry:
-// SANITIZE-WITHOUT-ATTR-NEXT:    ret i64 4
+// SANITIZE-WITHOUT-ATTR-NEXT:    ret i64 -1
 //
-// NO-SANITIZE-WITHOUT-ATTR-LABEL: define dso_local noundef i64 @test11_bdos(
-// NO-SANITIZE-WITHOUT-ATTR-SAME: ptr nocapture noundef readnone [[P:%.*]]) local_unnamed_addr #[[ATTR1]] {
+// NO-SANITIZE-WITHOUT-ATTR-LABEL: define dso_local i64 @test11_bdos(
+// NO-SANITIZE-WITHOUT-ATTR-SAME: ptr noundef readnone [[P:%.*]]) local_unnamed_addr #[[ATTR1]] {
 // NO-SANITIZE-WITHOUT-ATTR-NEXT:  entry:
-// NO-SANITIZE-WITHOUT-ATTR-NEXT:    ret i64 4
+// NO-SANITIZE-WITHOUT-ATTR-NEXT:    ret i64 -1
 //
 size_t test11_bdos(struct annotated *p) {
-  return __builtin_dynamic_object_size(&p->count, 1);
+  return __bdos(&p->count);
 }
 
 struct {
@@ -1011,16 +1056,16 @@ struct hang {
 int test12_a, test12_b;
 
 // SANITIZE-WITH-ATTR-LABEL: define dso_local noundef i32 @test12(
-// SANITIZE-WITH-ATTR-SAME: i32 noundef [[INDEX:%.*]]) local_unnamed_addr #[[ATTR4:[0-9]+]] {
+// SANITIZE-WITH-ATTR-SAME: i32 noundef [[INDEX:%.*]]) local_unnamed_addr #[[ATTR3:[0-9]+]] {
 // SANITIZE-WITH-ATTR-NEXT:  entry:
 // SANITIZE-WITH-ATTR-NEXT:    [[BAZ:%.*]] = alloca [[STRUCT_HANG:%.*]], align 4
 // SANITIZE-WITH-ATTR-NEXT:    call void @llvm.lifetime.start.p0(i64 24, ptr nonnull [[BAZ]]) #[[ATTR9:[0-9]+]]
-// SANITIZE-WITH-ATTR-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr noundef nonnull align 4 dereferenceable(24) [[BAZ]], ptr noundef nonnull align 4 dereferenceable(24) @test12_bar, i64 24, i1 false), !tbaa.struct [[TBAA_STRUCT9:![0-9]+]]
+// SANITIZE-WITH-ATTR-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr noundef nonnull align 4 dereferenceable(24) [[BAZ]], ptr noundef nonnull align 4 dereferenceable(24) @test12_bar, i64 24, i1 false), !tbaa.struct [[TBAA_STRUCT10:![0-9]+]]
 // SANITIZE-WITH-ATTR-NEXT:    [[TMP0:%.*]] = icmp ult i32 [[INDEX]], 6
 // SANITIZE-WITH-ATTR-NEXT:    [[TMP1:%.*]] = zext i32 [[INDEX]] to i64
 // SANITIZE-WITH-ATTR-NEXT:    br i1 [[TMP0]], label [[CONT:%.*]], label [[HANDLER_OUT_OF_BOUNDS:%.*]], !prof [[PROF3]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR:       handler.out_of_bounds:
-// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB18:[0-9]+]], i64 [[TMP1]]) #[[ATTR8]], !nosanitize [[META2]]
+// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB22:[0-9]+]], i64 [[TMP1]]) #[[ATTR8]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR-NEXT:    unreachable, !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR:       cont:
 // SANITIZE-WITH-ATTR-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds nuw [6 x i32], ptr [[BAZ]], i64 0, i64 [[TMP1]]
@@ -1028,19 +1073,19 @@ int test12_a, test12_b;
 // SANITIZE-WITH-ATTR-NEXT:    store i32 [[TMP2]], ptr @test12_b, align 4, !tbaa [[TBAA4]]
 // SANITIZE-WITH-ATTR-NEXT:    [[DOTCOUNTED_BY_LOAD:%.*]] = load i32, ptr @test12_foo, align 4
 // SANITIZE-WITH-ATTR-NEXT:    [[DOTNOT:%.*]] = icmp eq i32 [[DOTCOUNTED_BY_LOAD]], 0
-// SANITIZE-WITH-ATTR-NEXT:    br i1 [[DOTNOT]], label [[HANDLER_OUT_OF_BOUNDS4:%.*]], label [[HANDLER_TYPE_MISMATCH6:%.*]], !prof [[PROF10:![0-9]+]], !nosanitize [[META2]]
+// SANITIZE-WITH-ATTR-NEXT:    br i1 [[DOTNOT]], label [[HANDLER_OUT_OF_BOUNDS4:%.*]], label [[HANDLER_TYPE_MISMATCH6:%.*]], !prof [[PROF8]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR:       handler.out_of_bounds4:
-// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB20:[0-9]+]], i64 0) #[[ATTR8]], !nosanitize [[META2]]
+// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB24:[0-9]+]], i64 0) #[[ATTR8]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR-NEXT:    unreachable, !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR:       handler.type_mismatch6:
-// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_type_mismatch_v1_abort(ptr nonnull @[[GLOB21:[0-9]+]], i64 ptrtoint (ptr getelementptr inbounds nuw (i8, ptr @test12_foo, i64 4) to i64)) #[[ATTR8]], !nosanitize [[META2]]
+// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_type_mismatch_v1_abort(ptr nonnull @[[GLOB25:[0-9]+]], i64 ptrtoint (ptr getelementptr inbounds nuw (i8, ptr @test12_foo, i64 4) to i64)) #[[ATTR8]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR-NEXT:    unreachable, !nosanitize [[META2]]
 //
 // NO-SANITIZE-WITH-ATTR-LABEL: define dso_local noundef i32 @test12(
 // NO-SANITIZE-WITH-ATTR-SAME: i32 noundef [[INDEX:%.*]]) local_unnamed_addr #[[ATTR4:[0-9]+]] {
 // NO-SANITIZE-WITH-ATTR-NEXT:  entry:
 // NO-SANITIZE-WITH-ATTR-NEXT:    [[BAZ:%.*]] = alloca [[STRUCT_HANG:%.*]], align 4
-// NO-SANITIZE-WITH-ATTR-NEXT:    call void @llvm.lifetime.start.p0(i64 24, ptr nonnull [[BAZ]]) #[[ATTR11:[0-9]+]]
+// NO-SANITIZE-WITH-ATTR-NEXT:    call void @llvm.lifetime.start.p0(i64 24, ptr nonnull [[BAZ]]) #[[ATTR12:[0-9]+]]
 // NO-SANITIZE-WITH-ATTR-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr noundef nonnull align 4 dereferenceable(24) [[BAZ]], ptr noundef nonnull align 4 dereferenceable(24) @test12_bar, i64 24, i1 false), !tbaa.struct [[TBAA_STRUCT7:![0-9]+]]
 // NO-SANITIZE-WITH-ATTR-NEXT:    [[IDXPROM:%.*]] = sext i32 [[INDEX]] to i64
 // NO-SANITIZE-WITH-ATTR-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [6 x i32], ptr [[BAZ]], i64 0, i64 [[IDXPROM]]
@@ -1056,13 +1101,13 @@ int test12_a, test12_b;
 // SANITIZE-WITHOUT-ATTR-SAME: i32 noundef [[INDEX:%.*]]) local_unnamed_addr #[[ATTR3:[0-9]+]] {
 // SANITIZE-WITHOUT-ATTR-NEXT:  entry:
 // SANITIZE-WITHOUT-ATTR-NEXT:    [[BAZ:%.*]] = alloca [[STRUCT_HANG:%.*]], align 4
-// SANITIZE-WITHOUT-ATTR-NEXT:    call void @llvm.lifetime.start.p0(i64 24, ptr nonnull [[BAZ]]) #[[ATTR6:[0-9]+]]
+// SANITIZE-WITHOUT-ATTR-NEXT:    call void @llvm.lifetime.start.p0(i64 24, ptr nonnull [[BAZ]]) #[[ATTR7:[0-9]+]]
 // SANITIZE-WITHOUT-ATTR-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr noundef nonnull align 4 dereferenceable(24) [[BAZ]], ptr noundef nonnull align 4 dereferenceable(24) @test12_bar, i64 24, i1 false), !tbaa.struct [[TBAA_STRUCT7:![0-9]+]]
 // SANITIZE-WITHOUT-ATTR-NEXT:    [[TMP0:%.*]] = icmp ult i32 [[INDEX]], 6
 // SANITIZE-WITHOUT-ATTR-NEXT:    [[TMP1:%.*]] = zext i32 [[INDEX]] to i64
 // SANITIZE-WITHOUT-ATTR-NEXT:    br i1 [[TMP0]], label [[CONT:%.*]], label [[HANDLER_OUT_OF_BOUNDS:%.*]], !prof [[PROF8:![0-9]+]], !nosanitize [[META9:![0-9]+]]
 // SANITIZE-WITHOUT-ATTR:       handler.out_of_bounds:
-// SANITIZE-WITHOUT-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB2:[0-9]+]], i64 [[TMP1]]) #[[ATTR7:[0-9]+]], !nosanitize [[META9]]
+// SANITIZE-WITHOUT-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB2:[0-9]+]], i64 [[TMP1]]) #[[ATTR8:[0-9]+]], !nosanitize [[META9]]
 // SANITIZE-WITHOUT-ATTR-NEXT:    unreachable, !nosanitize [[META9]]
 // SANITIZE-WITHOUT-ATTR:       cont:
 // SANITIZE-WITHOUT-ATTR-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds nuw [6 x i32], ptr [[BAZ]], i64 0, i64 [[TMP1]]
@@ -1072,17 +1117,17 @@ int test12_a, test12_b;
 // SANITIZE-WITHOUT-ATTR-NEXT:    [[DOTNOT:%.*]] = icmp eq i32 [[DOTCOUNTED_BY_LOAD]], 0
 // SANITIZE-WITHOUT-ATTR-NEXT:    br i1 [[DOTNOT]], label [[HANDLER_OUT_OF_BOUNDS4:%.*]], label [[HANDLER_TYPE_MISMATCH6:%.*]], !prof [[PROF10:![0-9]+]], !nosanitize [[META9]]
 // SANITIZE-WITHOUT-ATTR:       handler.out_of_bounds4:
-// SANITIZE-WITHOUT-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB4:[0-9]+]], i64 0) #[[ATTR7]], !nosanitize [[META9]]
+// SANITIZE-WITHOUT-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB4:[0-9]+]], i64 0) #[[ATTR8]], !nosanitize [[META9]]
 // SANITIZE-WITHOUT-ATTR-NEXT:    unreachable, !nosanitize [[META9]]
 // SANITIZE-WITHOUT-ATTR:       handler.type_mismatch6:
-// SANITIZE-WITHOUT-ATTR-NEXT:    tail call void @__ubsan_handle_type_mismatch_v1_abort(ptr nonnull @[[GLOB5:[0-9]+]], i64 ptrtoint (ptr getelementptr inbounds nuw (i8, ptr @test12_foo, i64 4) to i64)) #[[ATTR7]], !nosanitize [[META9]]
+// SANITIZE-WITHOUT-ATTR-NEXT:    tail call void @__ubsan_handle_type_mismatch_v1_abort(ptr nonnull @[[GLOB5:[0-9]+]], i64 ptrtoint (ptr getelementptr inbounds nuw (i8, ptr @test12_foo, i64 4) to i64)) #[[ATTR8]], !nosanitize [[META9]]
 // SANITIZE-WITHOUT-ATTR-NEXT:    unreachable, !nosanitize [[META9]]
 //
 // NO-SANITIZE-WITHOUT-ATTR-LABEL: define dso_local noundef i32 @test12(
 // NO-SANITIZE-WITHOUT-ATTR-SAME: i32 noundef [[INDEX:%.*]]) local_unnamed_addr #[[ATTR2:[0-9]+]] {
 // NO-SANITIZE-WITHOUT-ATTR-NEXT:  entry:
 // NO-SANITIZE-WITHOUT-ATTR-NEXT:    [[BAZ:%.*]] = alloca [[STRUCT_HANG:%.*]], align 4
-// NO-SANITIZE-WITHOUT-ATTR-NEXT:    call void @llvm.lifetime.start.p0(i64 24, ptr nonnull [[BAZ]]) #[[ATTR9:[0-9]+]]
+// NO-SANITIZE-WITHOUT-ATTR-NEXT:    call void @llvm.lifetime.start.p0(i64 24, ptr nonnull [[BAZ]]) #[[ATTR10:[0-9]+]]
 // NO-SANITIZE-WITHOUT-ATTR-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr noundef nonnull align 4 dereferenceable(24) [[BAZ]], ptr noundef nonnull align 4 dereferenceable(24) @test12_bar, i64 24, i1 false), !tbaa.struct [[TBAA_STRUCT7:![0-9]+]]
 // NO-SANITIZE-WITHOUT-ATTR-NEXT:    [[IDXPROM:%.*]] = sext i32 [[INDEX]] to i64
 // NO-SANITIZE-WITHOUT-ATTR-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [6 x i32], ptr [[BAZ]], i64 0, i64 [[IDXPROM]]
@@ -1123,7 +1168,7 @@ struct test13_bar {
 // SANITIZE-WITH-ATTR-NEXT:    [[TMP2:%.*]] = icmp ult i64 [[INDEX]], [[TMP1]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR-NEXT:    br i1 [[TMP2]], label [[CONT5:%.*]], label [[HANDLER_OUT_OF_BOUNDS:%.*]], !prof [[PROF3]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR:       handler.out_of_bounds:
-// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB24:[0-9]+]], i64 [[INDEX]]) #[[ATTR8]], !nosanitize [[META2]]
+// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB28:[0-9]+]], i64 [[INDEX]]) #[[ATTR8]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR-NEXT:    unreachable, !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR:       cont5:
 // SANITIZE-WITH-ATTR-NEXT:    [[REVMAP:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP0]], i64 16
@@ -1150,7 +1195,7 @@ struct test13_bar {
 // SANITIZE-WITHOUT-ATTR-NEXT:    [[TMP2:%.*]] = icmp ult i64 [[INDEX]], [[TMP1]], !nosanitize [[META9]]
 // SANITIZE-WITHOUT-ATTR-NEXT:    br i1 [[TMP2]], label [[CONT5:%.*]], label [[HANDLER_OUT_OF_BOUNDS:%.*]], !prof [[PROF8]], !nosanitize [[META9]]
 // SANITIZE-WITHOUT-ATTR:       handler.out_of_bounds:
-// SANITIZE-WITHOUT-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB8:[0-9]+]], i64 [[INDEX]]) #[[ATTR7]], !nosanitize [[META9]]
+// SANITIZE-WITHOUT-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB8:[0-9]+]], i64 [[INDEX]]) #[[ATTR8]], !nosanitize [[META9]]
 // SANITIZE-WITHOUT-ATTR-NEXT:    unreachable, !nosanitize [[META9]]
 // SANITIZE-WITHOUT-ATTR:       cont5:
 // SANITIZE-WITHOUT-ATTR-NEXT:    [[REVMAP:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP0]], i64 16
@@ -1184,7 +1229,7 @@ struct test14_foo {
 // SANITIZE-WITH-ATTR-NEXT:    br i1 [[TMP0]], label [[CONT3:%.*]], label [[HANDLER_OUT_OF_BOUNDS:%.*]], !prof [[PROF3]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR:       handler.out_of_bounds:
 // SANITIZE-WITH-ATTR-NEXT:    [[IDXPROM:%.*]] = sext i32 [[IDX]] to i64
-// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB25:[0-9]+]], i64 [[IDXPROM]]) #[[ATTR8]], !nosanitize [[META2]]
+// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB29:[0-9]+]], i64 [[IDXPROM]]) #[[ATTR8]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR-NEXT:    unreachable, !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR:       cont3:
 // SANITIZE-WITH-ATTR-NEXT:    ret i32 undef
@@ -1209,7 +1254,7 @@ struct test14_foo {
 // SANITIZE-WITHOUT-ATTR-NEXT:    br i1 [[TMP0]], label [[CONT3:%.*]], label [[HANDLER_OUT_OF_BOUNDS:%.*]], !prof [[PROF8]], !nosanitize [[META9]]
 // SANITIZE-WITHOUT-ATTR:       handler.out_of_bounds:
 // SANITIZE-WITHOUT-ATTR-NEXT:    [[IDXPROM:%.*]] = sext i32 [[IDX]] to i64
-// SANITIZE-WITHOUT-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB9:[0-9]+]], i64 [[IDXPROM]]) #[[ATTR7]], !nosanitize [[META9]]
+// SANITIZE-WITHOUT-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB9:[0-9]+]], i64 [[IDXPROM]]) #[[ATTR8]], !nosanitize [[META9]]
 // SANITIZE-WITHOUT-ATTR-NEXT:    unreachable, !nosanitize [[META9]]
 // SANITIZE-WITHOUT-ATTR:       cont3:
 // SANITIZE-WITHOUT-ATTR-NEXT:    ret i32 undef
@@ -1238,7 +1283,7 @@ int test14(int idx) {
 // SANITIZE-WITH-ATTR-NEXT:    br i1 [[TMP0]], label [[CONT1:%.*]], label [[HANDLER_OUT_OF_BOUNDS:%.*]], !prof [[PROF3]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR:       handler.out_of_bounds:
 // SANITIZE-WITH-ATTR-NEXT:    [[IDXPROM:%.*]] = sext i32 [[IDX]] to i64
-// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB27:[0-9]+]], i64 [[IDXPROM]]) #[[ATTR8]], !nosanitize [[META2]]
+// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB31:[0-9]+]], i64 [[IDXPROM]]) #[[ATTR8]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR-NEXT:    unreachable, !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR:       cont1:
 // SANITIZE-WITH-ATTR-NEXT:    ret i32 undef
@@ -1258,7 +1303,7 @@ int test14(int idx) {
 // SANITIZE-WITHOUT-ATTR-NEXT:    br i1 [[TMP0]], label [[CONT1:%.*]], label [[HANDLER_OUT_OF_BOUNDS:%.*]], !prof [[PROF8]], !nosanitize [[META9]]
 // SANITIZE-WITHOUT-ATTR:       handler.out_of_bounds:
 // SANITIZE-WITHOUT-ATTR-NEXT:    [[IDXPROM:%.*]] = sext i32 [[IDX]] to i64
-// SANITIZE-WITHOUT-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB11:[0-9]+]], i64 [[IDXPROM]]) #[[ATTR7]], !nosanitize [[META9]]
+// SANITIZE-WITHOUT-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB11:[0-9]+]], i64 [[IDXPROM]]) #[[ATTR8]], !nosanitize [[META9]]
 // SANITIZE-WITHOUT-ATTR-NEXT:    unreachable, !nosanitize [[META9]]
 // SANITIZE-WITHOUT-ATTR:       cont1:
 // SANITIZE-WITHOUT-ATTR-NEXT:    ret i32 undef
@@ -1280,13 +1325,21 @@ int test15(int idx) {
   return foo.blah[idx];
 }
 
-// SANITIZE-WITH-ATTR-LABEL: define dso_local noundef i64 @test19(
-// SANITIZE-WITH-ATTR-SAME: ptr nocapture noundef readnone [[P:%.*]]) local_unnamed_addr #[[ATTR3]] {
+// SANITIZE-WITH-ATTR-LABEL: define dso_local i64 @test19(
+// SANITIZE-WITH-ATTR-SAME: ptr noundef [[P:%.*]]) local_unnamed_addr #[[ATTR0]] {
 // SANITIZE-WITH-ATTR-NEXT:  entry:
+// SANITIZE-WITH-ATTR-NEXT:    [[DOTCOUNTED_BY_GEP:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i64 680
+// SANITIZE-WITH-ATTR-NEXT:    [[DOTCOUNTED_BY_LOAD:%.*]] = load i32, ptr [[DOTCOUNTED_BY_GEP]], align 4
+// SANITIZE-WITH-ATTR-NEXT:    [[TMP0:%.*]] = icmp ugt i32 [[DOTCOUNTED_BY_LOAD]], 1
+// SANITIZE-WITH-ATTR-NEXT:    br i1 [[TMP0]], label [[CONT1:%.*]], label [[HANDLER_OUT_OF_BOUNDS:%.*]], !prof [[PROF3]], !nosanitize [[META2]]
+// SANITIZE-WITH-ATTR:       handler.out_of_bounds:
+// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB32:[0-9]+]], i64 2) #[[ATTR8]], !nosanitize [[META2]]
+// SANITIZE-WITH-ATTR-NEXT:    unreachable, !nosanitize [[META2]]
+// SANITIZE-WITH-ATTR:       cont1:
 // SANITIZE-WITH-ATTR-NEXT:    ret i64 -1
 //
-// NO-SANITIZE-WITH-ATTR-LABEL: define dso_local noundef i64 @test19(
-// NO-SANITIZE-WITH-ATTR-SAME: ptr nocapture noundef readnone [[P:%.*]]) local_unnamed_addr #[[ATTR3]] {
+// NO-SANITIZE-WITH-ATTR-LABEL: define dso_local i64 @test19(
+// NO-SANITIZE-WITH-ATTR-SAME: ptr noundef readnone [[P:%.*]]) local_unnamed_addr #[[ATTR3]] {
 // NO-SANITIZE-WITH-ATTR-NEXT:  entry:
 // NO-SANITIZE-WITH-ATTR-NEXT:    ret i64 -1
 //
@@ -1302,107 +1355,107 @@ int test15(int idx) {
 //
 size_t test19(struct annotated *p) {
   // Avoid pointer arithmetic. It could lead to security issues.
-  return __builtin_dynamic_object_size(&(p + 42)->array[2], 1);
+  return __bdos(&(p + 42)->array[2]);
 }
 
 // SANITIZE-WITH-ATTR-LABEL: define dso_local noundef i64 @test20(
-// SANITIZE-WITH-ATTR-SAME: ptr nocapture noundef readnone [[P:%.*]]) local_unnamed_addr #[[ATTR3]] {
+// SANITIZE-WITH-ATTR-SAME: ptr noundef readnone captures(none) [[P:%.*]]) local_unnamed_addr #[[ATTR2]] {
 // SANITIZE-WITH-ATTR-NEXT:  entry:
 // SANITIZE-WITH-ATTR-NEXT:    ret i64 -1
 //
 // NO-SANITIZE-WITH-ATTR-LABEL: define dso_local noundef i64 @test20(
-// NO-SANITIZE-WITH-ATTR-SAME: ptr nocapture noundef readnone [[P:%.*]]) local_unnamed_addr #[[ATTR3]] {
+// NO-SANITIZE-WITH-ATTR-SAME: ptr noundef readnone captures(none) [[P:%.*]]) local_unnamed_addr #[[ATTR3]] {
 // NO-SANITIZE-WITH-ATTR-NEXT:  entry:
 // NO-SANITIZE-WITH-ATTR-NEXT:    ret i64 -1
 //
 // SANITIZE-WITHOUT-ATTR-LABEL: define dso_local noundef i64 @test20(
-// SANITIZE-WITHOUT-ATTR-SAME: ptr nocapture noundef readnone [[P:%.*]]) local_unnamed_addr #[[ATTR2]] {
+// SANITIZE-WITHOUT-ATTR-SAME: ptr noundef readnone captures(none) [[P:%.*]]) local_unnamed_addr #[[ATTR2]] {
 // SANITIZE-WITHOUT-ATTR-NEXT:  entry:
 // SANITIZE-WITHOUT-ATTR-NEXT:    ret i64 -1
 //
 // NO-SANITIZE-WITHOUT-ATTR-LABEL: define dso_local noundef i64 @test20(
-// NO-SANITIZE-WITHOUT-ATTR-SAME: ptr nocapture noundef readnone [[P:%.*]]) local_unnamed_addr #[[ATTR1]] {
+// NO-SANITIZE-WITHOUT-ATTR-SAME: ptr noundef readnone captures(none) [[P:%.*]]) local_unnamed_addr #[[ATTR1]] {
 // NO-SANITIZE-WITHOUT-ATTR-NEXT:  entry:
 // NO-SANITIZE-WITHOUT-ATTR-NEXT:    ret i64 -1
 //
 size_t test20(struct annotated *p) {
   // Avoid side-effects.
-  return __builtin_dynamic_object_size(&(++p)->array[2], 1);
+  return __bdos(&(++p)->array[2]);
 }
 
 // SANITIZE-WITH-ATTR-LABEL: define dso_local noundef i64 @test21(
-// SANITIZE-WITH-ATTR-SAME: ptr nocapture noundef readnone [[P:%.*]]) local_unnamed_addr #[[ATTR3]] {
+// SANITIZE-WITH-ATTR-SAME: ptr noundef readnone captures(none) [[P:%.*]]) local_unnamed_addr #[[ATTR2]] {
 // SANITIZE-WITH-ATTR-NEXT:  entry:
 // SANITIZE-WITH-ATTR-NEXT:    ret i64 -1
 //
 // NO-SANITIZE-WITH-ATTR-LABEL: define dso_local noundef i64 @test21(
-// NO-SANITIZE-WITH-ATTR-SAME: ptr nocapture noundef readnone [[P:%.*]]) local_unnamed_addr #[[ATTR3]] {
+// NO-SANITIZE-WITH-ATTR-SAME: ptr noundef readnone captures(none) [[P:%.*]]) local_unnamed_addr #[[ATTR3]] {
 // NO-SANITIZE-WITH-ATTR-NEXT:  entry:
 // NO-SANITIZE-WITH-ATTR-NEXT:    ret i64 -1
 //
 // SANITIZE-WITHOUT-ATTR-LABEL: define dso_local noundef i64 @test21(
-// SANITIZE-WITHOUT-ATTR-SAME: ptr nocapture noundef readnone [[P:%.*]]) local_unnamed_addr #[[ATTR2]] {
+// SANITIZE-WITHOUT-ATTR-SAME: ptr noundef readnone captures(none) [[P:%.*]]) local_unnamed_addr #[[ATTR2]] {
 // SANITIZE-WITHOUT-ATTR-NEXT:  entry:
 // SANITIZE-WITHOUT-ATTR-NEXT:    ret i64 -1
 //
 // NO-SANITIZE-WITHOUT-ATTR-LABEL: define dso_local noundef i64 @test21(
-// NO-SANITIZE-WITHOUT-ATTR-SAME: ptr nocapture noundef readnone [[P:%.*]]) local_unnamed_addr #[[ATTR1]] {
+// NO-SANITIZE-WITHOUT-ATTR-SAME: ptr noundef readnone captures(none) [[P:%.*]]) local_unnamed_addr #[[ATTR1]] {
 // NO-SANITIZE-WITHOUT-ATTR-NEXT:  entry:
 // NO-SANITIZE-WITHOUT-ATTR-NEXT:    ret i64 -1
 //
 size_t test21(struct annotated *p) {
   // Avoid side-effects.
-  return __builtin_dynamic_object_size(&(p++)->array[2], 1);
+  return __bdos(&(p++)->array[2]);
 }
 
 // SANITIZE-WITH-ATTR-LABEL: define dso_local noundef i64 @test22(
-// SANITIZE-WITH-ATTR-SAME: ptr nocapture noundef readnone [[P:%.*]]) local_unnamed_addr #[[ATTR3]] {
+// SANITIZE-WITH-ATTR-SAME: ptr noundef readnone captures(none) [[P:%.*]]) local_unnamed_addr #[[ATTR2]] {
 // SANITIZE-WITH-ATTR-NEXT:  entry:
 // SANITIZE-WITH-ATTR-NEXT:    ret i64 -1
 //
 // NO-SANITIZE-WITH-ATTR-LABEL: define dso_local noundef i64 @test22(
-// NO-SANITIZE-WITH-ATTR-SAME: ptr nocapture noundef readnone [[P:%.*]]) local_unnamed_addr #[[ATTR3]] {
+// NO-SANITIZE-WITH-ATTR-SAME: ptr noundef readnone captures(none) [[P:%.*]]) local_unnamed_addr #[[ATTR3]] {
 // NO-SANITIZE-WITH-ATTR-NEXT:  entry:
 // NO-SANITIZE-WITH-ATTR-NEXT:    ret i64 -1
 //
 // SANITIZE-WITHOUT-ATTR-LABEL: define dso_local noundef i64 @test22(
-// SANITIZE-WITHOUT-ATTR-SAME: ptr nocapture noundef readnone [[P:%.*]]) local_unnamed_addr #[[ATTR2]] {
+// SANITIZE-WITHOUT-ATTR-SAME: ptr noundef readnone captures(none) [[P:%.*]]) local_unnamed_addr #[[ATTR2]] {
 // SANITIZE-WITHOUT-ATTR-NEXT:  entry:
 // SANITIZE-WITHOUT-ATTR-NEXT:    ret i64 -1
 //
 // NO-SANITIZE-WITHOUT-ATTR-LABEL: define dso_local noundef i64 @test22(
-// NO-SANITIZE-WITHOUT-ATTR-SAME: ptr nocapture noundef readnone [[P:%.*]]) local_unnamed_addr #[[ATTR1]] {
+// NO-SANITIZE-WITHOUT-ATTR-SAME: ptr noundef readnone captures(none) [[P:%.*]]) local_unnamed_addr #[[ATTR1]] {
 // NO-SANITIZE-WITHOUT-ATTR-NEXT:  entry:
 // NO-SANITIZE-WITHOUT-ATTR-NEXT:    ret i64 -1
 //
 size_t test22(struct annotated *p) {
   // Avoid side-effects.
-  return __builtin_dynamic_object_size(&(--p)->array[2], 1);
+  return __bdos(&(--p)->array[2]);
 }
 
 // SANITIZE-WITH-ATTR-LABEL: define dso_local noundef i64 @test23(
-// SANITIZE-WITH-ATTR-SAME: ptr nocapture noundef readnone [[P:%.*]]) local_unnamed_addr #[[ATTR3]] {
+// SANITIZE-WITH-ATTR-SAME: ptr noundef readnone captures(none) [[P:%.*]]) local_unnamed_addr #[[ATTR2]] {
 // SANITIZE-WITH-ATTR-NEXT:  entry:
 // SANITIZE-WITH-ATTR-NEXT:    ret i64 -1
 //
 // NO-SANITIZE-WITH-ATTR-LABEL: define dso_local noundef i64 @test23(
-// NO-SANITIZE-WITH-ATTR-SAME: ptr nocapture noundef readnone [[P:%.*]]) local_unnamed_addr #[[ATTR3]] {
+// NO-SANITIZE-WITH-ATTR-SAME: ptr noundef readnone captures(none) [[P:%.*]]) local_unnamed_addr #[[ATTR3]] {
 // NO-SANITIZE-WITH-ATTR-NEXT:  entry:
 // NO-SANITIZE-WITH-ATTR-NEXT:    ret i64 -1
 //
 // SANITIZE-WITHOUT-ATTR-LABEL: define dso_local noundef i64 @test23(
-// SANITIZE-WITHOUT-ATTR-SAME: ptr nocapture noundef readnone [[P:%.*]]) local_unnamed_addr #[[ATTR2]] {
+// SANITIZE-WITHOUT-ATTR-SAME: ptr noundef readnone captures(none) [[P:%.*]]) local_unnamed_addr #[[ATTR2]] {
 // SANITIZE-WITHOUT-ATTR-NEXT:  entry:
 // SANITIZE-WITHOUT-ATTR-NEXT:    ret i64 -1
 //
 // NO-SANITIZE-WITHOUT-ATTR-LABEL: define dso_local noundef i64 @test23(
-// NO-SANITIZE-WITHOUT-ATTR-SAME: ptr nocapture noundef readnone [[P:%.*]]) local_unnamed_addr #[[ATTR1]] {
+// NO-SANITIZE-WITHOUT-ATTR-SAME: ptr noundef readnone captures(none) [[P:%.*]]) local_unnamed_addr #[[ATTR1]] {
 // NO-SANITIZE-WITHOUT-ATTR-NEXT:  entry:
 // NO-SANITIZE-WITHOUT-ATTR-NEXT:    ret i64 -1
 //
 size_t test23(struct annotated *p) {
   // Avoid side-effects.
-  return __builtin_dynamic_object_size(&(p--)->array[2], 1);
+  return __bdos(&(p--)->array[2]);
 }
 
 struct tests_foo {
@@ -1418,7 +1471,7 @@ struct tests_foo {
 // SANITIZE-WITH-ATTR-NEXT:    [[TMP0:%.*]] = icmp ugt i32 [[DOTCOUNTED_BY_LOAD]], 10
 // SANITIZE-WITH-ATTR-NEXT:    br i1 [[TMP0]], label [[CONT4:%.*]], label [[HANDLER_OUT_OF_BOUNDS:%.*]], !prof [[PROF3]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR:       handler.out_of_bounds:
-// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB28:[0-9]+]], i64 10) #[[ATTR8]], !nosanitize [[META2]]
+// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB33:[0-9]+]], i64 10) #[[ATTR8]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR-NEXT:    unreachable, !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR:       cont4:
 // SANITIZE-WITH-ATTR-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds nuw i8, ptr [[VAR]], i64 84
@@ -1426,7 +1479,7 @@ struct tests_foo {
 // SANITIZE-WITH-ATTR-NEXT:    ret i32 [[TMP1]]
 //
 // NO-SANITIZE-WITH-ATTR-LABEL: define dso_local i32 @test24(
-// NO-SANITIZE-WITH-ATTR-SAME: i32 noundef [[C:%.*]], ptr nocapture noundef readonly [[VAR:%.*]]) local_unnamed_addr #[[ATTR2]] {
+// NO-SANITIZE-WITH-ATTR-SAME: i32 noundef [[C:%.*]], ptr noundef readonly captures(none) [[VAR:%.*]]) local_unnamed_addr #[[ATTR2]] {
 // NO-SANITIZE-WITH-ATTR-NEXT:  entry:
 // NO-SANITIZE-WITH-ATTR-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds nuw i8, ptr [[VAR]], i64 84
 // NO-SANITIZE-WITH-ATTR-NEXT:    [[TMP0:%.*]] = load i32, ptr [[ARRAYIDX1]], align 4, !tbaa [[TBAA2]]
@@ -1440,7 +1493,7 @@ struct tests_foo {
 // SANITIZE-WITHOUT-ATTR-NEXT:    ret i32 [[TMP0]]
 //
 // NO-SANITIZE-WITHOUT-ATTR-LABEL: define dso_local i32 @test24(
-// NO-SANITIZE-WITHOUT-ATTR-SAME: i32 noundef [[C:%.*]], ptr nocapture noundef readonly [[VAR:%.*]]) local_unnamed_addr #[[ATTR6:[0-9]+]] {
+// NO-SANITIZE-WITHOUT-ATTR-SAME: i32 noundef [[C:%.*]], ptr noundef readonly captures(none) [[VAR:%.*]]) local_unnamed_addr #[[ATTR6:[0-9]+]] {
 // NO-SANITIZE-WITHOUT-ATTR-NEXT:  entry:
 // NO-SANITIZE-WITHOUT-ATTR-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds nuw i8, ptr [[VAR]], i64 84
 // NO-SANITIZE-WITHOUT-ATTR-NEXT:    [[TMP0:%.*]] = load i32, ptr [[ARRAYIDX1]], align 4, !tbaa [[TBAA2]]
@@ -1459,7 +1512,7 @@ int test24(int c, struct tests_foo *var) {
 // SANITIZE-WITH-ATTR-NEXT:    [[TMP1:%.*]] = icmp ugt i32 [[DOTCOUNTED_BY_LOAD]], 10
 // SANITIZE-WITH-ATTR-NEXT:    br i1 [[TMP1]], label [[CONT5:%.*]], label [[HANDLER_OUT_OF_BOUNDS:%.*]], !prof [[PROF3]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR:       handler.out_of_bounds:
-// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB29:[0-9]+]], i64 10) #[[ATTR8]], !nosanitize [[META2]]
+// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB34:[0-9]+]], i64 10) #[[ATTR8]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR-NEXT:    unreachable, !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR:       cont5:
 // SANITIZE-WITH-ATTR-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP0]], i64 44
@@ -1467,7 +1520,7 @@ int test24(int c, struct tests_foo *var) {
 // SANITIZE-WITH-ATTR-NEXT:    ret i32 [[TMP2]]
 //
 // NO-SANITIZE-WITH-ATTR-LABEL: define dso_local i32 @test25(
-// NO-SANITIZE-WITH-ATTR-SAME: i32 noundef [[C:%.*]], ptr nocapture noundef readonly [[VAR:%.*]]) local_unnamed_addr #[[ATTR8:[0-9]+]] {
+// NO-SANITIZE-WITH-ATTR-SAME: i32 noundef [[C:%.*]], ptr noundef readonly captures(none) [[VAR:%.*]]) local_unnamed_addr #[[ATTR8:[0-9]+]] {
 // NO-SANITIZE-WITH-ATTR-NEXT:  entry:
 // NO-SANITIZE-WITH-ATTR-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VAR]], align 8, !tbaa [[TBAA14:![0-9]+]]
 // NO-SANITIZE-WITH-ATTR-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP0]], i64 44
@@ -1483,7 +1536,7 @@ int test24(int c, struct tests_foo *var) {
 // SANITIZE-WITHOUT-ATTR-NEXT:    ret i32 [[TMP1]]
 //
 // NO-SANITIZE-WITHOUT-ATTR-LABEL: define dso_local i32 @test25(
-// NO-SANITIZE-WITHOUT-ATTR-SAME: i32 noundef [[C:%.*]], ptr nocapture noundef readonly [[VAR:%.*]]) local_unnamed_addr #[[ATTR7:[0-9]+]] {
+// NO-SANITIZE-WITHOUT-ATTR-SAME: i32 noundef [[C:%.*]], ptr noundef readonly captures(none) [[VAR:%.*]]) local_unnamed_addr #[[ATTR7:[0-9]+]] {
 // NO-SANITIZE-WITHOUT-ATTR-NEXT:  entry:
 // NO-SANITIZE-WITHOUT-ATTR-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VAR]], align 8, !tbaa [[TBAA14:![0-9]+]]
 // NO-SANITIZE-WITHOUT-ATTR-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP0]], i64 44
@@ -1511,7 +1564,7 @@ struct test26_foo {
 // SANITIZE-WITH-ATTR-NEXT:    [[TMP1:%.*]] = icmp ult i64 [[IDXPROM]], [[TMP0]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR-NEXT:    br i1 [[TMP1]], label [[CONT5:%.*]], label [[HANDLER_OUT_OF_BOUNDS:%.*]], !prof [[PROF3]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR:       handler.out_of_bounds:
-// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB30:[0-9]+]], i64 [[IDXPROM]]) #[[ATTR8]], !nosanitize [[META2]]
+// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB35:[0-9]+]], i64 [[IDXPROM]]) #[[ATTR8]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR-NEXT:    unreachable, !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR:       cont5:
 // SANITIZE-WITH-ATTR-NEXT:    [[ARR:%.*]] = getelementptr inbounds nuw i8, ptr [[FOO]], i64 8
@@ -1520,7 +1573,7 @@ struct test26_foo {
 // SANITIZE-WITH-ATTR-NEXT:    ret i32 [[TMP2]]
 //
 // NO-SANITIZE-WITH-ATTR-LABEL: define dso_local i32 @test26(
-// NO-SANITIZE-WITH-ATTR-SAME: i32 noundef [[C:%.*]], ptr nocapture noundef readonly [[FOO:%.*]]) local_unnamed_addr #[[ATTR2]] {
+// NO-SANITIZE-WITH-ATTR-SAME: i32 noundef [[C:%.*]], ptr noundef readonly captures(none) [[FOO:%.*]]) local_unnamed_addr #[[ATTR2]] {
 // NO-SANITIZE-WITH-ATTR-NEXT:  entry:
 // NO-SANITIZE-WITH-ATTR-NEXT:    [[ARR:%.*]] = getelementptr inbounds nuw i8, ptr [[FOO]], i64 8
 // NO-SANITIZE-WITH-ATTR-NEXT:    [[IDXPROM:%.*]] = sext i32 [[C]] to i64
@@ -1538,7 +1591,7 @@ struct test26_foo {
 // SANITIZE-WITHOUT-ATTR-NEXT:    ret i32 [[TMP0]]
 //
 // NO-SANITIZE-WITHOUT-ATTR-LABEL: define dso_local i32 @test26(
-// NO-SANITIZE-WITHOUT-ATTR-SAME: i32 noundef [[C:%.*]], ptr nocapture noundef readonly [[FOO:%.*]]) local_unnamed_addr #[[ATTR6]] {
+// NO-SANITIZE-WITHOUT-ATTR-SAME: i32 noundef [[C:%.*]], ptr noundef readonly captures(none) [[FOO:%.*]]) local_unnamed_addr #[[ATTR6]] {
 // NO-SANITIZE-WITHOUT-ATTR-NEXT:  entry:
 // NO-SANITIZE-WITHOUT-ATTR-NEXT:    [[ARR:%.*]] = getelementptr inbounds nuw i8, ptr [[FOO]], i64 8
 // NO-SANITIZE-WITHOUT-ATTR-NEXT:    [[IDXPROM:%.*]] = sext i32 [[C]] to i64
@@ -1582,7 +1635,7 @@ struct test27_foo {
 // SANITIZE-WITH-ATTR-NEXT:    [[TMP1:%.*]] = icmp ult i64 [[IDXPROM]], [[TMP0]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR-NEXT:    br i1 [[TMP1]], label [[CONT3:%.*]], label [[HANDLER_OUT_OF_BOUNDS:%.*]], !prof [[PROF3]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR:       handler.out_of_bounds:
-// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB32:[0-9]+]], i64 [[IDXPROM]]) #[[ATTR8]], !nosanitize [[META2]]
+// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB37:[0-9]+]], i64 [[IDXPROM]]) #[[ATTR8]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR-NEXT:    unreachable, !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR:       cont3:
 // SANITIZE-WITH-ATTR-NEXT:    [[ENTRIES:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i64 24
@@ -1593,7 +1646,7 @@ struct test27_foo {
 // SANITIZE-WITH-ATTR-NEXT:    ret ptr [[ARRAYIDX5]]
 //
 // NO-SANITIZE-WITH-ATTR-LABEL: define dso_local ptr @test27(
-// NO-SANITIZE-WITH-ATTR-SAME: ptr nocapture noundef readonly [[P:%.*]], i32 noundef [[I:%.*]], i32 noundef [[J:%.*]]) local_unnamed_addr #[[ATTR2]] {
+// NO-SANITIZE-WITH-ATTR-SAME: ptr noundef readonly captures(none) [[P:%.*]], i32 noundef [[I:%.*]], i32 noundef [[J:%.*]]) local_unnamed_addr #[[ATTR2]] {
 // NO-SANITIZE-WITH-ATTR-NEXT:  entry:
 // NO-SANITIZE-WITH-ATTR-NEXT:    [[ENTRIES:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i64 24
 // NO-SANITIZE-WITH-ATTR-NEXT:    [[IDXPROM:%.*]] = sext i32 [[I]] to i64
@@ -1615,7 +1668,7 @@ struct test27_foo {
 // SANITIZE-WITHOUT-ATTR-NEXT:    ret ptr [[ARRAYIDX4]]
 //
 // NO-SANITIZE-WITHOUT-ATTR-LABEL: define dso_local ptr @test27(
-// NO-SANITIZE-WITHOUT-ATTR-SAME: ptr nocapture noundef readonly [[P:%.*]], i32 noundef [[I:%.*]], i32 noundef [[J:%.*]]) local_unnamed_addr #[[ATTR6]] {
+// NO-SANITIZE-WITHOUT-ATTR-SAME: ptr noundef readonly captures(none) [[P:%.*]], i32 noundef [[I:%.*]], i32 noundef [[J:%.*]]) local_unnamed_addr #[[ATTR6]] {
 // NO-SANITIZE-WITHOUT-ATTR-NEXT:  entry:
 // NO-SANITIZE-WITHOUT-ATTR-NEXT:    [[ENTRIES:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i64 24
 // NO-SANITIZE-WITHOUT-ATTR-NEXT:    [[IDXPROM:%.*]] = sext i32 [[I]] to i64
@@ -1648,7 +1701,7 @@ struct test28_foo {
 // SANITIZE-WITH-ATTR-NEXT:    [[TMP4:%.*]] = icmp ult i64 [[IDXPROM]], [[TMP3]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR-NEXT:    br i1 [[TMP4]], label [[CONT17:%.*]], label [[HANDLER_OUT_OF_BOUNDS:%.*]], !prof [[PROF3]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR:       handler.out_of_bounds:
-// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB34:[0-9]+]], i64 [[IDXPROM]]) #[[ATTR8]], !nosanitize [[META2]]
+// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB39:[0-9]+]], i64 [[IDXPROM]]) #[[ATTR8]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR-NEXT:    unreachable, !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR:       cont17:
 // SANITIZE-WITH-ATTR-NEXT:    [[ARR:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP2]], i64 12
@@ -1657,7 +1710,7 @@ struct test28_foo {
 // SANITIZE-WITH-ATTR-NEXT:    ret i32 [[TMP5]]
 //
 // NO-SANITIZE-WITH-ATTR-LABEL: define dso_local i32 @test28(
-// NO-SANITIZE-WITH-ATTR-SAME: ptr nocapture noundef readonly [[P:%.*]], i32 noundef [[I:%.*]]) local_unnamed_addr #[[ATTR8]] {
+// NO-SANITIZE-WITH-ATTR-SAME: ptr noundef readonly captures(none) [[P:%.*]], i32 noundef [[I:%.*]]) local_unnamed_addr #[[ATTR8]] {
 // NO-SANITIZE-WITH-ATTR-NEXT:  entry:
 // NO-SANITIZE-WITH-ATTR-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[P]], align 8, !tbaa [[TBAA18:![0-9]+]]
 // NO-SANITIZE-WITH-ATTR-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[TMP0]], align 8, !tbaa [[TBAA18]]
@@ -1681,7 +1734,7 @@ struct test28_foo {
 // SANITIZE-WITHOUT-ATTR-NEXT:    ret i32 [[TMP3]]
 //
 // NO-SANITIZE-WITHOUT-ATTR-LABEL: define dso_local i32 @test28(
-// NO-SANITIZE-WITHOUT-ATTR-SAME: ptr nocapture noundef readonly [[P:%.*]], i32 noundef [[I:%.*]]) local_unnamed_addr #[[ATTR7]] {
+// NO-SANITIZE-WITHOUT-ATTR-SAME: ptr noundef readonly captures(none) [[P:%.*]], i32 noundef [[I:%.*]]) local_unnamed_addr #[[ATTR7]] {
 // NO-SANITIZE-WITHOUT-ATTR-NEXT:  entry:
 // NO-SANITIZE-WITHOUT-ATTR-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[P]], align 8, !tbaa [[TBAA18:![0-9]+]]
 // NO-SANITIZE-WITHOUT-ATTR-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[TMP0]], align 8, !tbaa [[TBAA18]]
@@ -1710,42 +1763,42 @@ struct annotated_struct_array {
 // SANITIZE-WITH-ATTR-NEXT:    [[TMP1:%.*]] = zext i32 [[IDX1]] to i64
 // SANITIZE-WITH-ATTR-NEXT:    br i1 [[TMP0]], label [[CONT3:%.*]], label [[HANDLER_OUT_OF_BOUNDS:%.*]], !prof [[PROF3]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR:       handler.out_of_bounds:
-// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB36:[0-9]+]], i64 [[TMP1]]) #[[ATTR8]], !nosanitize [[META2]]
+// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB41:[0-9]+]], i64 [[TMP1]]) #[[ATTR8]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR-NEXT:    unreachable, !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR:       cont3:
 // SANITIZE-WITH-ATTR-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds nuw [10 x ptr], ptr [[ANN]], i64 0, i64 [[TMP1]]
 // SANITIZE-WITH-ATTR-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[ARRAYIDX]], align 8, !tbaa [[TBAA23:![0-9]+]]
 // SANITIZE-WITH-ATTR-NEXT:    [[COUNTED_BY_GEP:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP2]], i64 8
 // SANITIZE-WITH-ATTR-NEXT:    [[COUNTED_BY_LOAD:%.*]] = load i32, ptr [[COUNTED_BY_GEP]], align 4
-// SANITIZE-WITH-ATTR-NEXT:    [[IDXPROM15:%.*]] = sext i32 [[IDX2]] to i64
+// SANITIZE-WITH-ATTR-NEXT:    [[IDXPROM27:%.*]] = sext i32 [[IDX2]] to i64
 // SANITIZE-WITH-ATTR-NEXT:    [[TMP3:%.*]] = zext i32 [[COUNTED_BY_LOAD]] to i64, !nosanitize [[META2]]
-// SANITIZE-WITH-ATTR-NEXT:    [[TMP4:%.*]] = icmp ult i64 [[IDXPROM15]], [[TMP3]], !nosanitize [[META2]]
-// SANITIZE-WITH-ATTR-NEXT:    br i1 [[TMP4]], label [[CONT20:%.*]], label [[HANDLER_OUT_OF_BOUNDS16:%.*]], !prof [[PROF3]], !nosanitize [[META2]]
-// SANITIZE-WITH-ATTR:       handler.out_of_bounds16:
-// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB37:[0-9]+]], i64 [[IDXPROM15]]) #[[ATTR8]], !nosanitize [[META2]]
+// SANITIZE-WITH-ATTR-NEXT:    [[TMP4:%.*]] = icmp ult i64 [[IDXPROM27]], [[TMP3]], !nosanitize [[META2]]
+// SANITIZE-WITH-ATTR-NEXT:    br i1 [[TMP4]], label [[CONT32:%.*]], label [[HANDLER_OUT_OF_BOUNDS28:%.*]], !prof [[PROF3]], !nosanitize [[META2]]
+// SANITIZE-WITH-ATTR:       handler.out_of_bounds28:
+// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB42:[0-9]+]], i64 [[IDXPROM27]]) #[[ATTR8]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR-NEXT:    unreachable, !nosanitize [[META2]]
-// SANITIZE-WITH-ATTR:       cont20:
+// SANITIZE-WITH-ATTR:       cont32:
 // SANITIZE-WITH-ATTR-NEXT:    [[ARRAY:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP2]], i64 12
-// SANITIZE-WITH-ATTR-NEXT:    [[ARRAYIDX18:%.*]] = getelementptr inbounds nuw [0 x i32], ptr [[ARRAY]], i64 0, i64 [[IDXPROM15]]
+// SANITIZE-WITH-ATTR-NEXT:    [[ARRAYIDX30:%.*]] = getelementptr inbounds nuw [0 x i32], ptr [[ARRAY]], i64 0, i64 [[IDXPROM27]]
 // SANITIZE-WITH-ATTR-NEXT:    [[TMP5:%.*]] = tail call i32 @llvm.smax.i32(i32 [[COUNTED_BY_LOAD]], i32 0)
 // SANITIZE-WITH-ATTR-NEXT:    [[CONV:%.*]] = shl i32 [[TMP5]], 2
-// SANITIZE-WITH-ATTR-NEXT:    store i32 [[CONV]], ptr [[ARRAYIDX18]], align 4, !tbaa [[TBAA4]]
+// SANITIZE-WITH-ATTR-NEXT:    store i32 [[CONV]], ptr [[ARRAYIDX30]], align 4, !tbaa [[TBAA4]]
 // SANITIZE-WITH-ATTR-NEXT:    ret void
 //
 // NO-SANITIZE-WITH-ATTR-LABEL: define dso_local void @test29(
-// NO-SANITIZE-WITH-ATTR-SAME: ptr nocapture noundef readonly [[ANN:%.*]], i32 noundef [[IDX1:%.*]], i32 noundef [[IDX2:%.*]]) local_unnamed_addr #[[ATTR9:[0-9]+]] {
+// NO-SANITIZE-WITH-ATTR-SAME: ptr noundef readonly captures(none) [[ANN:%.*]], i32 noundef [[IDX1:%.*]], i32 noundef [[IDX2:%.*]]) local_unnamed_addr #[[ATTR9:[0-9]+]] {
 // NO-SANITIZE-WITH-ATTR-NEXT:  entry:
 // NO-SANITIZE-WITH-ATTR-NEXT:    [[IDXPROM:%.*]] = sext i32 [[IDX1]] to i64
 // NO-SANITIZE-WITH-ATTR-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [10 x ptr], ptr [[ANN]], i64 0, i64 [[IDXPROM]]
 // NO-SANITIZE-WITH-ATTR-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[ARRAYIDX]], align 8, !tbaa [[TBAA20:![0-9]+]]
+// NO-SANITIZE-WITH-ATTR-NEXT:    [[ARRAY:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP0]], i64 12
 // NO-SANITIZE-WITH-ATTR-NEXT:    [[COUNTED_BY_GEP:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP0]], i64 8
 // NO-SANITIZE-WITH-ATTR-NEXT:    [[COUNTED_BY_LOAD:%.*]] = load i32, ptr [[COUNTED_BY_GEP]], align 4
 // NO-SANITIZE-WITH-ATTR-NEXT:    [[TMP1:%.*]] = tail call i32 @llvm.smax.i32(i32 [[COUNTED_BY_LOAD]], i32 0)
 // NO-SANITIZE-WITH-ATTR-NEXT:    [[CONV:%.*]] = shl i32 [[TMP1]], 2
-// NO-SANITIZE-WITH-ATTR-NEXT:    [[ARRAY:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP0]], i64 12
-// NO-SANITIZE-WITH-ATTR-NEXT:    [[IDXPROM4:%.*]] = sext i32 [[IDX2]] to i64
-// NO-SANITIZE-WITH-ATTR-NEXT:    [[ARRAYIDX5:%.*]] = getelementptr inbounds [0 x i32], ptr [[ARRAY]], i64 0, i64 [[IDXPROM4]]
-// NO-SANITIZE-WITH-ATTR-NEXT:    store i32 [[CONV]], ptr [[ARRAYIDX5]], align 4, !tbaa [[TBAA2]]
+// NO-SANITIZE-WITH-ATTR-NEXT:    [[IDXPROM8:%.*]] = sext i32 [[IDX2]] to i64
+// NO-SANITIZE-WITH-ATTR-NEXT:    [[ARRAYIDX9:%.*]] = getelementptr inbounds [0 x i32], ptr [[ARRAY]], i64 0, i64 [[IDXPROM8]]
+// NO-SANITIZE-WITH-ATTR-NEXT:    store i32 [[CONV]], ptr [[ARRAYIDX9]], align 4, !tbaa [[TBAA2]]
 // NO-SANITIZE-WITH-ATTR-NEXT:    ret void
 //
 // SANITIZE-WITHOUT-ATTR-LABEL: define dso_local void @test29(
@@ -1755,7 +1808,7 @@ struct annotated_struct_array {
 // SANITIZE-WITHOUT-ATTR-NEXT:    [[TMP1:%.*]] = zext i32 [[IDX1]] to i64
 // SANITIZE-WITHOUT-ATTR-NEXT:    br i1 [[TMP0]], label [[CONT21:%.*]], label [[HANDLER_OUT_OF_BOUNDS:%.*]], !prof [[PROF8]], !nosanitize [[META9]]
 // SANITIZE-WITHOUT-ATTR:       handler.out_of_bounds:
-// SANITIZE-WITHOUT-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB13:[0-9]+]], i64 [[TMP1]]) #[[ATTR7]], !nosanitize [[META9]]
+// SANITIZE-WITHOUT-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB13:[0-9]+]], i64 [[TMP1]]) #[[ATTR8]], !nosanitize [[META9]]
 // SANITIZE-WITHOUT-ATTR-NEXT:    unreachable, !nosanitize [[META9]]
 // SANITIZE-WITHOUT-ATTR:       cont21:
 // SANITIZE-WITHOUT-ATTR-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds nuw [10 x ptr], ptr [[ANN]], i64 0, i64 [[TMP1]]
@@ -1767,7 +1820,7 @@ struct annotated_struct_array {
 // SANITIZE-WITHOUT-ATTR-NEXT:    ret void
 //
 // NO-SANITIZE-WITHOUT-ATTR-LABEL: define dso_local void @test29(
-// NO-SANITIZE-WITHOUT-ATTR-SAME: ptr nocapture noundef readonly [[ANN:%.*]], i32 noundef [[IDX1:%.*]], i32 noundef [[IDX2:%.*]]) local_unnamed_addr #[[ATTR8:[0-9]+]] {
+// NO-SANITIZE-WITHOUT-ATTR-SAME: ptr noundef readonly captures(none) [[ANN:%.*]], i32 noundef [[IDX1:%.*]], i32 noundef [[IDX2:%.*]]) local_unnamed_addr #[[ATTR8:[0-9]+]] {
 // NO-SANITIZE-WITHOUT-ATTR-NEXT:  entry:
 // NO-SANITIZE-WITHOUT-ATTR-NEXT:    [[IDXPROM:%.*]] = sext i32 [[IDX1]] to i64
 // NO-SANITIZE-WITHOUT-ATTR-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [10 x ptr], ptr [[ANN]], i64 0, i64 [[IDXPROM]]
@@ -1779,7 +1832,7 @@ struct annotated_struct_array {
 // NO-SANITIZE-WITHOUT-ATTR-NEXT:    ret void
 //
 void test29(struct annotated_struct_array *ann, int idx1, int idx2) {
-  ann->ann_array[idx1]->array[idx2] = __builtin_dynamic_object_size(ann->ann_array[idx1]->array, 1);
+  ann->ann_array[idx1]->array[idx2] = __bdos(ann->ann_array[idx1]->array);
 }
 
 typedef struct {
@@ -1794,10 +1847,10 @@ struct test30_struct {
 };
 
 // SANITIZE-WITH-ATTR-LABEL: define dso_local void @test30(
-// SANITIZE-WITH-ATTR-SAME: ptr noundef [[PTR:%.*]], i32 noundef [[IDX:%.*]]) local_unnamed_addr #[[ATTR4]] {
+// SANITIZE-WITH-ATTR-SAME: ptr noundef [[PTR:%.*]], i32 noundef [[IDX:%.*]]) local_unnamed_addr #[[ATTR3]] {
 // SANITIZE-WITH-ATTR-NEXT:  entry:
 // SANITIZE-WITH-ATTR-NEXT:    [[TMP0:%.*]] = zext i32 [[IDX]] to i64, !nosanitize [[META2]]
-// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB39:[0-9]+]], i64 [[TMP0]]) #[[ATTR8]], !nosanitize [[META2]]
+// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB44:[0-9]+]], i64 [[TMP0]]) #[[ATTR8]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR-NEXT:    unreachable, !nosanitize [[META2]]
 //
 // NO-SANITIZE-WITH-ATTR-LABEL: define dso_local void @test30(
@@ -1813,7 +1866,7 @@ struct test30_struct {
 // SANITIZE-WITHOUT-ATTR-SAME: ptr noundef [[PTR:%.*]], i32 noundef [[IDX:%.*]]) local_unnamed_addr #[[ATTR3]] {
 // SANITIZE-WITHOUT-ATTR-NEXT:  entry:
 // SANITIZE-WITHOUT-ATTR-NEXT:    [[TMP0:%.*]] = zext i32 [[IDX]] to i64, !nosanitize [[META9]]
-// SANITIZE-WITHOUT-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB15:[0-9]+]], i64 [[TMP0]]) #[[ATTR7]], !nosanitize [[META9]]
+// SANITIZE-WITHOUT-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB15:[0-9]+]], i64 [[TMP0]]) #[[ATTR8]], !nosanitize [[META9]]
 // SANITIZE-WITHOUT-ATTR-NEXT:    unreachable, !nosanitize [[META9]]
 //
 // NO-SANITIZE-WITHOUT-ATTR-LABEL: define dso_local void @test30(
@@ -1826,7 +1879,7 @@ struct test30_struct {
 // NO-SANITIZE-WITHOUT-ATTR-NEXT:    ret void
 //
 void test30(struct test30_struct *ptr, int idx) {
-  ptr->pcpu_refcnt.__padding[idx] = __builtin_dynamic_object_size(ptr, 1);
+  ptr->pcpu_refcnt.__padding[idx] = __bdos(ptr);
 }
 
 struct test31_empty {};
@@ -1838,7 +1891,7 @@ struct test31_struct {
 };
 
 // SANITIZE-WITH-ATTR-LABEL: define dso_local i32 @test31(
-// SANITIZE-WITH-ATTR-SAME: ptr noundef [[PTR:%.*]], i32 noundef [[IDX:%.*]]) local_unnamed_addr #[[ATTR3]] {
+// SANITIZE-WITH-ATTR-SAME: ptr noundef [[PTR:%.*]], i32 noundef [[IDX:%.*]]) local_unnamed_addr #[[ATTR2]] {
 // SANITIZE-WITH-ATTR-NEXT:  entry:
 // SANITIZE-WITH-ATTR-NEXT:    ret i32 -1
 //
@@ -1858,5 +1911,339 @@ struct test31_struct {
 // NO-SANITIZE-WITHOUT-ATTR-NEXT:    ret i32 -1
 //
 int test31(struct test31_struct *ptr, int idx) {
-  return __builtin_dynamic_object_size(ptr, 0);
+  return __bdos(ptr);
+}
+
+struct annotated_with_array {
+  unsigned long flags[42];
+  int count;
+  size_t array[] __counted_by(count);
+};
+
+// SANITIZE-WITH-ATTR-LABEL: define dso_local void @test32(
+// SANITIZE-WITH-ATTR-SAME: ptr noundef [[PTR:%.*]], i32 noundef [[IDX1:%.*]], i32 noundef [[IDX2:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// SANITIZE-WITH-ATTR-NEXT:  entry:
+// SANITIZE-WITH-ATTR-NEXT:    [[TMP0:%.*]] = icmp ult i32 [[IDX2]], 43
+// SANITIZE-WITH-ATTR-NEXT:    br i1 [[TMP0]], label [[CONT1:%.*]], label [[HANDLER_OUT_OF_BOUNDS:%.*]], !prof [[PROF3]], !nosanitize [[META2]]
+// SANITIZE-WITH-ATTR:       handler.out_of_bounds:
+// SANITIZE-WITH-ATTR-NEXT:    [[TMP1:%.*]] = zext i32 [[IDX2]] to i64, !nosanitize [[META2]]
+// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB46:[0-9]+]], i64 [[TMP1]]) #[[ATTR8]], !nosanitize [[META2]]
+// SANITIZE-WITH-ATTR-NEXT:    unreachable, !nosanitize [[META2]]
+// SANITIZE-WITH-ATTR:       cont1:
+// SANITIZE-WITH-ATTR-NEXT:    [[COUNTED_BY_GEP:%.*]] = getelementptr inbounds nuw i8, ptr [[PTR]], i64 336
+// SANITIZE-WITH-ATTR-NEXT:    [[COUNTED_BY_LOAD:%.*]] = load i32, ptr [[COUNTED_BY_GEP]], align 4
+// SANITIZE-WITH-ATTR-NEXT:    [[IDXPROM4:%.*]] = sext i32 [[IDX1]] to i64
+// SANITIZE-WITH-ATTR-NEXT:    [[TMP2:%.*]] = zext i32 [[COUNTED_BY_LOAD]] to i64, !nosanitize [[META2]]
+// SANITIZE-WITH-ATTR-NEXT:    [[TMP3:%.*]] = icmp ult i64 [[IDXPROM4]], [[TMP2]], !nosanitize [[META2]]
+// SANITIZE-WITH-ATTR-NEXT:    br i1 [[TMP3]], label [[CONT9:%.*]], label [[HANDLER_OUT_OF_BOUNDS5:%.*]], !prof [[PROF3]], !nosanitize [[META2]]
+// SANITIZE-WITH-ATTR:       handler.out_of_bounds5:
+// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB48:[0-9]+]], i64 [[IDXPROM4]]) #[[ATTR8]], !nosanitize [[META2]]
+// SANITIZE-WITH-ATTR-NEXT:    unreachable, !nosanitize [[META2]]
+// SANITIZE-WITH-ATTR:       cont9:
+// SANITIZE-WITH-ATTR-NEXT:    [[ARRAY:%.*]] = getelementptr inbounds nuw i8, ptr [[PTR]], i64 344
+// SANITIZE-WITH-ATTR-NEXT:    [[ARRAYIDX7:%.*]] = getelementptr inbounds nuw [0 x i64], ptr [[ARRAY]], i64 0, i64 [[IDXPROM4]]
+// SANITIZE-WITH-ATTR-NEXT:    [[COUNT:%.*]] = sext i32 [[COUNTED_BY_LOAD]] to i64
+// SANITIZE-WITH-ATTR-NEXT:    [[FLEXIBLE_ARRAY_MEMBER_SIZE:%.*]] = shl nsw i64 [[COUNT]], 3
+// SANITIZE-WITH-ATTR-NEXT:    [[TMP4:%.*]] = shl nuw nsw i32 [[IDX2]], 3
+// SANITIZE-WITH-ATTR-NEXT:    [[FIELD_OFFSET:%.*]] = zext nneg i32 [[TMP4]] to i64
+// SANITIZE-WITH-ATTR-NEXT:    [[REASS_SUB:%.*]] = sub nsw i64 [[FLEXIBLE_ARRAY_MEMBER_SIZE]], [[FIELD_OFFSET]]
+// SANITIZE-WITH-ATTR-NEXT:    [[TMP5:%.*]] = tail call i64 @llvm.smax.i64(i64 [[REASS_SUB]], i64 -344)
+// SANITIZE-WITH-ATTR-NEXT:    [[TMP6:%.*]] = add nsw i64 [[TMP5]], 344
+// SANITIZE-WITH-ATTR-NEXT:    store i64 [[TMP6]], ptr [[ARRAYIDX7]], align 8, !tbaa [[TBAA25:![0-9]+]]
+// SANITIZE-WITH-ATTR-NEXT:    ret void
+//
+// NO-SANITIZE-WITH-ATTR-LABEL: define dso_local void @test32(
+// NO-SANITIZE-WITH-ATTR-SAME: ptr noundef captures(none) [[PTR:%.*]], i32 noundef [[IDX1:%.*]], i32 noundef [[IDX2:%.*]]) local_unnamed_addr #[[ATTR1]] {
+// NO-SANITIZE-WITH-ATTR-NEXT:  entry:
+// NO-SANITIZE-WITH-ATTR-NEXT:    [[IDXPROM:%.*]] = sext i32 [[IDX2]] to i64
+// NO-SANITIZE-WITH-ATTR-NEXT:    [[COUNTED_BY_GEP:%.*]] = getelementptr inbounds nuw i8, ptr [[PTR]], i64 336
+// NO-SANITIZE-WITH-ATTR-NEXT:    [[COUNTED_BY_LOAD:%.*]] = load i32, ptr [[COUNTED_BY_GEP]], align 4
+// NO-SANITIZE-WITH-ATTR-NEXT:    [[COUNT:%.*]] = sext i32 [[COUNTED_BY_LOAD]] to i64
+// NO-SANITIZE-WITH-ATTR-NEXT:    [[TMP0:%.*]] = sub nsw i64 [[COUNT]], [[IDXPROM]]
+// NO-SANITIZE-WITH-ATTR-NEXT:    [[REASS_SUB:%.*]] = shl nsw i64 [[TMP0]], 3
+// NO-SANITIZE-WITH-ATTR-NEXT:    [[RESULT:%.*]] = add nsw i64 [[REASS_SUB]], 344
+// NO-SANITIZE-WITH-ATTR-NEXT:    [[TMP1:%.*]] = icmp sgt i64 [[TMP0]], -44
+// NO-SANITIZE-WITH-ATTR-NEXT:    [[TMP2:%.*]] = icmp sgt i32 [[IDX2]], -1
+// NO-SANITIZE-WITH-ATTR-NEXT:    [[TMP3:%.*]] = and i1 [[TMP2]], [[TMP1]]
+// NO-SANITIZE-WITH-ATTR-NEXT:    [[TMP4:%.*]] = select i1 [[TMP3]], i64 [[RESULT]], i64 0
+// NO-SANITIZE-WITH-ATTR-NEXT:    [[ARRAY:%.*]] = getelementptr inbounds nuw i8, ptr [[PTR]], i64 344
+// NO-SANITIZE-WITH-ATTR-NEXT:    [[IDXPROM1:%.*]] = sext i32 [[IDX1]] to i64
+// NO-SANITIZE-WITH-ATTR-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds [0 x i64], ptr [[ARRAY]], i64 0, i64 [[IDXPROM1]]
+// NO-SANITIZE-WITH-ATTR-NEXT:    store i64 [[TMP4]], ptr [[ARRAYIDX2]], align 8, !tbaa [[TBAA22:![0-9]+]]
+// NO-SANITIZE-WITH-ATTR-NEXT:    ret void
+//
+// SANITIZE-WITHOUT-ATTR-LABEL: define dso_local void @test32(
+// SANITIZE-WITHOUT-ATTR-SAME: ptr noundef [[PTR:%.*]], i32 noundef [[IDX1:%.*]], i32 noundef [[IDX2:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// SANITIZE-WITHOUT-ATTR-NEXT:  entry:
+// SANITIZE-WITHOUT-ATTR-NEXT:    [[TMP0:%.*]] = icmp ult i32 [[IDX2]], 43
+// SANITIZE-WITHOUT-ATTR-NEXT:    br i1 [[TMP0]], label [[CONT7:%.*]], label [[HANDLER_OUT_OF_BOUNDS:%.*]], !prof [[PROF8]], !nosanitize [[META9]]
+// SANITIZE-WITHOUT-ATTR:       handler.out_of_bounds:
+// SANITIZE-WITHOUT-ATTR-NEXT:    [[TMP1:%.*]] = zext i32 [[IDX2]] to i64, !nosanitize [[META9]]
+// SANITIZE-WITHOUT-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB17:[0-9]+]], i64 [[TMP1]]) #[[ATTR8]], !nosanitize [[META9]]
+// SANITIZE-WITHOUT-ATTR-NEXT:    unreachable, !nosanitize [[META9]]
+// SANITIZE-WITHOUT-ATTR:       cont7:
+// SANITIZE-WITHOUT-ATTR-NEXT:    [[ARRAY:%.*]] = getelementptr inbounds nuw i8, ptr [[PTR]], i64 344
+// SANITIZE-WITHOUT-ATTR-NEXT:    [[IDXPROM4:%.*]] = sext i32 [[IDX1]] to i64
+// SANITIZE-WITHOUT-ATTR-NEXT:    [[ARRAYIDX5:%.*]] = getelementptr inbounds [0 x i64], ptr [[ARRAY]], i64 0, i64 [[IDXPROM4]]
+// SANITIZE-WITHOUT-ATTR-NEXT:    store i64 -1, ptr [[ARRAYIDX5]], align 8, !tbaa [[TBAA25:![0-9]+]]
+// SANITIZE-WITHOUT-ATTR-NEXT:    ret void
+//
+// NO-SANITIZE-WITHOUT-ATTR-LABEL: define dso_local void @test32(
+// NO-SANITIZE-WITHOUT-ATTR-SAME: ptr noundef [[PTR:%.*]], i32 noundef [[IDX1:%.*]], i32 noundef [[IDX2:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// NO-SANITIZE-WITHOUT-ATTR-NEXT:  entry:
+// NO-SANITIZE-WITHOUT-ATTR-NEXT:    [[ARRAY:%.*]] = getelementptr inbounds nuw i8, ptr [[PTR]], i64 344
+// NO-SANITIZE-WITHOUT-ATTR-NEXT:    [[IDXPROM1:%.*]] = sext i32 [[IDX1]] to i64
+// NO-SANITIZE-WITHOUT-ATTR-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds [0 x i64], ptr [[ARRAY]], i64 0, i64 [[IDXPROM1]]
+// NO-SANITIZE-WITHOUT-ATTR-NEXT:    store i64 -1, ptr [[ARRAYIDX2]], align 8, !tbaa [[TBAA22:![0-9]+]]
+// NO-SANITIZE-WITHOUT-ATTR-NEXT:    ret void
+//
+void test32(struct annotated_with_array *ptr, int idx1, int idx2) {
+  ptr->array[idx1] =  __bdos(&ptr->flags[idx2]);
+}
+
+// SANITIZE-WITH-ATTR-LABEL: define dso_local range(i64 0, 17179869521) i64 @test32_bdos(
+// SANITIZE-WITH-ATTR-SAME: ptr noundef [[PTR:%.*]], i32 noundef [[INDEX:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// SANITIZE-WITH-ATTR-NEXT:  entry:
+// SANITIZE-WITH-ATTR-NEXT:    [[TMP0:%.*]] = icmp ult i32 [[INDEX]], 43
+// SANITIZE-WITH-ATTR-NEXT:    br i1 [[TMP0]], label [[CONT1:%.*]], label [[HANDLER_OUT_OF_BOUNDS:%.*]], !prof [[PROF3]], !nosanitize [[META2]]
+// SANITIZE-WITH-ATTR:       handler.out_of_bounds:
+// SANITIZE-WITH-ATTR-NEXT:    [[TMP1:%.*]] = zext i32 [[INDEX]] to i64, !nosanitize [[META2]]
+// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB49:[0-9]+]], i64 [[TMP1]]) #[[ATTR8]], !nosanitize [[META2]]
+// SANITIZE-WITH-ATTR-NEXT:    unreachable, !nosanitize [[META2]]
+// SANITIZE-WITH-ATTR:       cont1:
+// SANITIZE-WITH-ATTR-NEXT:    [[COUNTED_BY_GEP:%.*]] = getelementptr inbounds nuw i8, ptr [[PTR]], i64 336
+// SANITIZE-WITH-ATTR-NEXT:    [[COUNTED_BY_LOAD:%.*]] = load i32, ptr [[COUNTED_BY_GEP]], align 4
+// SANITIZE-WITH-ATTR-NEXT:    [[COUNT:%.*]] = sext i32 [[COUNTED_BY_LOAD]] to i64
+// SANITIZE-WITH-ATTR-NEXT:    [[FLEXIBLE_ARRAY_MEMBER_SIZE:%.*]] = shl nsw i64 [[COUNT]], 3
+// SANITIZE-WITH-ATTR-NEXT:    [[TMP2:%.*]] = shl nuw nsw i32 [[INDEX]], 3
+// SANITIZE-WITH-ATTR-NEXT:    [[FIELD_OFFSET:%.*]] = zext nneg i32 [[TMP2]] to i64
+// SANITIZE-WITH-ATTR-NEXT:    [[REASS_SUB:%.*]] = sub nsw i64 [[FLEXIBLE_ARRAY_MEMBER_SIZE]], [[FIELD_OFFSET]]
+// SANITIZE-WITH-ATTR-NEXT:    [[TMP3:%.*]] = tail call i64 @llvm.smax.i64(i64 [[REASS_SUB]], i64 -344)
+// SANITIZE-WITH-ATTR-NEXT:    [[TMP4:%.*]] = add nsw i64 [[TMP3]], 344
+// SANITIZE-WITH-ATTR-NEXT:    ret i64 [[TMP4]]
+//
+// NO-SANITIZE-WITH-ATTR-LABEL: define dso_local range(i64 -34359738016, 34359738705) i64 @test32_bdos(
+// NO-SANITIZE-WITH-ATTR-SAME: ptr noundef readonly captures(none) [[PTR:%.*]], i32 noundef [[INDEX:%.*]]) local_unnamed_addr #[[ATTR2]] {
+// NO-SANITIZE-WITH-ATTR-NEXT:  entry:
+// NO-SANITIZE-WITH-ATTR-NEXT:    [[IDXPROM:%.*]] = sext i32 [[INDEX]] to i64
+// NO-SANITIZE-WITH-ATTR-NEXT:    [[COUNTED_BY_GEP:%.*]] = getelementptr inbounds nuw i8, ptr [[PTR]], i64 336
+// NO-SANITIZE-WITH-ATTR-NEXT:    [[COUNTED_BY_LOAD:%.*]] = load i32, ptr [[COUNTED_BY_GEP]], align 4
+// NO-SANITIZE-WITH-ATTR-NEXT:    [[COUNT:%.*]] = sext i32 [[COUNTED_BY_LOAD]] to i64
+// NO-SANITIZE-WITH-ATTR-NEXT:    [[TMP0:%.*]] = sub nsw i64 [[COUNT]], [[IDXPROM]]
+// NO-SANITIZE-WITH-ATTR-NEXT:    [[REASS_SUB:%.*]] = shl nsw i64 [[TMP0]], 3
+// NO-SANITIZE-WITH-ATTR-NEXT:    [[RESULT:%.*]] = add nsw i64 [[REASS_SUB]], 344
+// NO-SANITIZE-WITH-ATTR-NEXT:    [[TMP1:%.*]] = icmp sgt i64 [[TMP0]], -44
+// NO-SANITIZE-WITH-ATTR-NEXT:    [[TMP2:%.*]] = icmp sgt i32 [[INDEX]], -1
+// NO-SANITIZE-WITH-ATTR-NEXT:    [[TMP3:%.*]] = and i1 [[TMP2]], [[TMP1]]
+// NO-SANITIZE-WITH-ATTR-NEXT:    [[TMP4:%.*]] = select i1 [[TMP3]], i64 [[RESULT]], i64 0
+// NO-SANITIZE-WITH-ATTR-NEXT:    ret i64 [[TMP4]]
+//
+// SANITIZE-WITHOUT-ATTR-LABEL: define dso_local i64 @test32_bdos(
+// SANITIZE-WITHOUT-ATTR-SAME: ptr noundef [[PTR:%.*]], i32 noundef [[INDEX:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// SANITIZE-WITHOUT-ATTR-NEXT:  entry:
+// SANITIZE-WITHOUT-ATTR-NEXT:    [[TMP0:%.*]] = icmp ult i32 [[INDEX]], 43
+// SANITIZE-WITHOUT-ATTR-NEXT:    br i1 [[TMP0]], label [[CONT1:%.*]], label [[HANDLER_OUT_OF_BOUNDS:%.*]], !prof [[PROF8]], !nosanitize [[META9]]
+// SANITIZE-WITHOUT-ATTR:       handler.out_of_bounds:
+// SANITIZE-WITHOUT-ATTR-NEXT:    [[TMP1:%.*]] = zext i32 [[INDEX]] to i64
+// SANITIZE-WITHOUT-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB18:[0-9]+]], i64 [[TMP1]]) #[[ATTR8]], !nosanitize [[META9]]
+// SANITIZE-WITHOUT-ATTR-NEXT:    unreachable, !nosanitize [[META9]]
+// SANITIZE-WITHOUT-ATTR:       cont1:
+// SANITIZE-WITHOUT-ATTR-NEXT:    ret i64 -1
+//
+// NO-SANITIZE-WITHOUT-ATTR-LABEL: define dso_local i64 @test32_bdos(
+// NO-SANITIZE-WITHOUT-ATTR-SAME: ptr noundef readnone [[PTR:%.*]], i32 noundef [[INDEX:%.*]]) local_unnamed_addr #[[ATTR1]] {
+// NO-SANITIZE-WITHOUT-ATTR-NEXT:  entry:
+// NO-SANITIZE-WITHOUT-ATTR-NEXT:    ret i64 -1
+//
+size_t test32_bdos(struct annotated_with_array *ptr, int index) {
+  return __bdos(&ptr->flags[index]);
+}
+
+// SANITIZE-WITH-ATTR-LABEL: define dso_local i64 @test33(
+// SANITIZE-WITH-ATTR-SAME: ptr noundef [[PTR:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// SANITIZE-WITH-ATTR-NEXT:  entry:
+// SANITIZE-WITH-ATTR-NEXT:    ret i64 -1
+//
+// NO-SANITIZE-WITH-ATTR-LABEL: define dso_local i64 @test33(
+// NO-SANITIZE-WITH-ATTR-SAME: ptr noundef readnone [[PTR:%.*]]) local_unnamed_addr #[[ATTR3]] {
+// NO-SANITIZE-WITH-ATTR-NEXT:  entry:
+// NO-SANITIZE-WITH-ATTR-NEXT:    ret i64 -1
+//
+// SANITIZE-WITHOUT-ATTR-LABEL: define dso_local i64 @test33(
+// SANITIZE-WITHOUT-ATTR-SAME: ptr noundef [[PTR:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// SANITIZE-WITHOUT-ATTR-NEXT:  entry:
+// SANITIZE-WITHOUT-ATTR-NEXT:    ret i64 -1
+//
+// NO-SANITIZE-WITHOUT-ATTR-LABEL: define dso_local i64 @test33(
+// NO-SANITIZE-WITHOUT-ATTR-SAME: ptr noundef readnone [[PTR:%.*]]) local_unnamed_addr #[[ATTR1]] {
+// NO-SANITIZE-WITHOUT-ATTR-NEXT:  entry:
+// NO-SANITIZE-WITHOUT-ATTR-NEXT:    ret i64 -1
+//
+size_t test33(struct annotated *ptr) {
+  // Don't handle '&ptr->array' like normal.
+  return __bdos(&*&*&*&ptr->array);
+}
+
+struct multi_subscripts {
+  unsigned long flags[42][42];
+  int count;
+  int array[] __counted_by(count);
+};
+
+// SANITIZE-WITH-ATTR-LABEL: define dso_local i64 @test34(
+// SANITIZE-WITH-ATTR-SAME: ptr noundef [[PTR:%.*]], i32 noundef [[IDX1:%.*]], i32 noundef [[IDX2:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// SANITIZE-WITH-ATTR-NEXT:  entry:
+// SANITIZE-WITH-ATTR-NEXT:    [[TMP0:%.*]] = icmp ult i32 [[IDX1]], 42
+// SANITIZE-WITH-ATTR-NEXT:    br i1 [[TMP0]], label [[CONT1:%.*]], label [[HANDLER_OUT_OF_BOUNDS:%.*]], !prof [[PROF3]], !nosanitize [[META2]]
+// SANITIZE-WITH-ATTR:       handler.out_of_bounds:
+// SANITIZE-WITH-ATTR-NEXT:    [[TMP1:%.*]] = zext i32 [[IDX1]] to i64, !nosanitize [[META2]]
+// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB51:[0-9]+]], i64 [[TMP1]]) #[[ATTR8]], !nosanitize [[META2]]
+// SANITIZE-WITH-ATTR-NEXT:    unreachable, !nosanitize [[META2]]
+// SANITIZE-WITH-ATTR:       cont1:
+// SANITIZE-WITH-ATTR-NEXT:    [[TMP2:%.*]] = icmp ult i32 [[IDX2]], 43
+// SANITIZE-WITH-ATTR-NEXT:    br i1 [[TMP2]], label [[CONT3:%.*]], label [[HANDLER_OUT_OF_BOUNDS2:%.*]], !prof [[PROF3]], !nosanitize [[META2]]
+// SANITIZE-WITH-ATTR:       handler.out_of_bounds2:
+// SANITIZE-WITH-ATTR-NEXT:    [[TMP3:%.*]] = zext i32 [[IDX2]] to i64
+// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB52:[0-9]+]], i64 [[TMP3]]) #[[ATTR8]], !nosanitize [[META2]]
+// SANITIZE-WITH-ATTR-NEXT:    unreachable, !nosanitize [[META2]]
+// SANITIZE-WITH-ATTR:       cont3:
+// SANITIZE-WITH-ATTR-NEXT:    ret i64 -1
+//
+// NO-SANITIZE-WITH-ATTR-LABEL: define dso_local i64 @test34(
+// NO-SANITIZE-WITH-ATTR-SAME: ptr noundef readnone [[PTR:%.*]], i32 noundef [[IDX1:%.*]], i32 noundef [[IDX2:%.*]]) local_unnamed_addr #[[ATTR3]] {
+// NO-SANITIZE-WITH-ATTR-NEXT:  entry:
+// NO-SANITIZE-WITH-ATTR-NEXT:    ret i64 -1
+//
+// SANITIZE-WITHOUT-ATTR-LABEL: define dso_local i64 @test34(
+// SANITIZE-WITHOUT-ATTR-SAME: ptr noundef [[PTR:%.*]], i32 noundef [[IDX1:%.*]], i32 noundef [[IDX2:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// SANITIZE-WITHOUT-ATTR-NEXT:  entry:
+// SANITIZE-WITHOUT-ATTR-NEXT:    [[TMP0:%.*]] = icmp ult i32 [[IDX1]], 42
+// SANITIZE-WITHOUT-ATTR-NEXT:    br i1 [[TMP0]], label [[CONT1:%.*]], label [[HANDLER_OUT_OF_BOUNDS:%.*]], !prof [[PROF8]], !nosanitize [[META9]]
+// SANITIZE-WITHOUT-ATTR:       handler.out_of_bounds:
+// SANITIZE-WITHOUT-ATTR-NEXT:    [[TMP1:%.*]] = zext i32 [[IDX1]] to i64, !nosanitize [[META9]]
+// SANITIZE-WITHOUT-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB20:[0-9]+]], i64 [[TMP1]]) #[[ATTR8]], !nosanitize [[META9]]
+// SANITIZE-WITHOUT-ATTR-NEXT:    unreachable, !nosanitize [[META9]]
+// SANITIZE-WITHOUT-ATTR:       cont1:
+// SANITIZE-WITHOUT-ATTR-NEXT:    [[TMP2:%.*]] = icmp ult i32 [[IDX2]], 43
+// SANITIZE-WITHOUT-ATTR-NEXT:    br i1 [[TMP2]], label [[CONT3:%.*]], label [[HANDLER_OUT_OF_BOUNDS2:%.*]], !prof [[PROF8]], !nosanitize [[META9]]
+// SANITIZE-WITHOUT-ATTR:       handler.out_of_bounds2:
+// SANITIZE-WITHOUT-ATTR-NEXT:    [[TMP3:%.*]] = zext i32 [[IDX2]] to i64
+// SANITIZE-WITHOUT-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB21:[0-9]+]], i64 [[TMP3]]) #[[ATTR8]], !nosanitize [[META9]]
+// SANITIZE-WITHOUT-ATTR-NEXT:    unreachable, !nosanitize [[META9]]
+// SANITIZE-WITHOUT-ATTR:       cont3:
+// SANITIZE-WITHOUT-ATTR-NEXT:    ret i64 -1
+//
+// NO-SANITIZE-WITHOUT-ATTR-LABEL: define dso_local i64 @test34(
+// NO-SANITIZE-WITHOUT-ATTR-SAME: ptr noundef readnone [[PTR:%.*]], i32 noundef [[IDX1:%.*]], i32 noundef [[IDX2:%.*]]) local_unnamed_addr #[[ATTR1]] {
+// NO-SANITIZE-WITHOUT-ATTR-NEXT:  entry:
+// NO-SANITIZE-WITHOUT-ATTR-NEXT:    ret i64 -1
+//
+size_t test34(struct multi_subscripts *ptr, int idx1, int idx2) {
+  return __bdos(&ptr->flags[idx1][idx2]);
+}
+
+// SANITIZE-WITH-ATTR-LABEL: define dso_local void @test35(
+// SANITIZE-WITH-ATTR-SAME: ptr noundef [[P:%.*]], i64 noundef [[INDEX:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// SANITIZE-WITH-ATTR-NEXT:  entry:
+// SANITIZE-WITH-ATTR-NEXT:    [[DOTCOUNTED_BY_GEP:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i64 8
+// SANITIZE-WITH-ATTR-NEXT:    [[DOTCOUNTED_BY_LOAD:%.*]] = load i32, ptr [[DOTCOUNTED_BY_GEP]], align 4
+// SANITIZE-WITH-ATTR-NEXT:    [[TMP0:%.*]] = zext i32 [[DOTCOUNTED_BY_LOAD]] to i64, !nosanitize [[META2]]
+// SANITIZE-WITH-ATTR-NEXT:    [[TMP1:%.*]] = icmp ult i64 [[INDEX]], [[TMP0]], !nosanitize [[META2]]
+// SANITIZE-WITH-ATTR-NEXT:    br i1 [[TMP1]], label [[CONT3:%.*]], label [[HANDLER_OUT_OF_BOUNDS:%.*]], !prof [[PROF3]], !nosanitize [[META2]]
+// SANITIZE-WITH-ATTR:       handler.out_of_bounds:
+// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB53:[0-9]+]], i64 [[INDEX]]) #[[ATTR8]], !nosanitize [[META2]]
+// SANITIZE-WITH-ATTR-NEXT:    unreachable, !nosanitize [[META2]]
+// SANITIZE-WITH-ATTR:       cont3:
+// SANITIZE-WITH-ATTR-NEXT:    [[ARRAY:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i64 12
+// SANITIZE-WITH-ATTR-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds nuw [0 x i32], ptr [[ARRAY]], i64 0, i64 [[INDEX]]
+// SANITIZE-WITH-ATTR-NEXT:    store i32 0, ptr [[ARRAYIDX]], align 4, !tbaa [[TBAA4]]
+// SANITIZE-WITH-ATTR-NEXT:    ret void
+//
+// NO-SANITIZE-WITH-ATTR-LABEL: define dso_local void @test35(
+// NO-SANITIZE-WITH-ATTR-SAME: ptr noundef writeonly captures(none) [[P:%.*]], i64 noundef [[INDEX:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// NO-SANITIZE-WITH-ATTR-NEXT:  entry:
+// NO-SANITIZE-WITH-ATTR-NEXT:    [[ARRAY:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i64 12
+// NO-SANITIZE-WITH-ATTR-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds nuw [0 x i32], ptr [[ARRAY]], i64 0, i64 [[INDEX]]
+// NO-SANITIZE-WITH-ATTR-NEXT:    store i32 0, ptr [[ARRAYIDX]], align 4, !tbaa [[TBAA2]]
+// NO-SANITIZE-WITH-ATTR-NEXT:    ret void
+//
+// SANITIZE-WITHOUT-ATTR-LABEL: define dso_local void @test35(
+// SANITIZE-WITHOUT-ATTR-SAME: ptr noundef [[P:%.*]], i64 noundef [[INDEX:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// SANITIZE-WITHOUT-ATTR-NEXT:  entry:
+// SANITIZE-WITHOUT-ATTR-NEXT:    [[ARRAY:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i64 12
+// SANITIZE-WITHOUT-ATTR-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds nuw [0 x i32], ptr [[ARRAY]], i64 0, i64 [[INDEX]]
+// SANITIZE-WITHOUT-ATTR-NEXT:    store i32 0, ptr [[ARRAYIDX]], align 4, !tbaa [[TBAA2]]
+// SANITIZE-WITHOUT-ATTR-NEXT:    ret void
+//
+// NO-SANITIZE-WITHOUT-ATTR-LABEL: define dso_local void @test35(
+// NO-SANITIZE-WITHOUT-ATTR-SAME: ptr noundef writeonly captures(none) [[P:%.*]], i64 noundef [[INDEX:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// NO-SANITIZE-WITHOUT-ATTR-NEXT:  entry:
+// NO-SANITIZE-WITHOUT-ATTR-NEXT:    [[ARRAY:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i64 12
+// NO-SANITIZE-WITHOUT-ATTR-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds nuw [0 x i32], ptr [[ARRAY]], i64 0, i64 [[INDEX]]
+// NO-SANITIZE-WITHOUT-ATTR-NEXT:    store i32 0, ptr [[ARRAYIDX]], align 4, !tbaa [[TBAA2]]
+// NO-SANITIZE-WITHOUT-ATTR-NEXT:    ret void
+//
+void test35(struct annotated *p, size_t index) {
+  p->array[index] = __bdos(&p->array[-42]);
+}
+
+// SANITIZE-WITH-ATTR-LABEL: define dso_local noundef i64 @test35_bdos(
+// SANITIZE-WITH-ATTR-SAME: ptr noundef readnone captures(none) [[P:%.*]]) local_unnamed_addr #[[ATTR2]] {
+// SANITIZE-WITH-ATTR-NEXT:  entry:
+// SANITIZE-WITH-ATTR-NEXT:    ret i64 0
+//
+// NO-SANITIZE-WITH-ATTR-LABEL: define dso_local noundef i64 @test35_bdos(
+// NO-SANITIZE-WITH-ATTR-SAME: ptr noundef readnone captures(none) [[P:%.*]]) local_unnamed_addr #[[ATTR3]] {
+// NO-SANITIZE-WITH-ATTR-NEXT:  entry:
+// NO-SANITIZE-WITH-ATTR-NEXT:    ret i64 0
+//
+// SANITIZE-WITHOUT-ATTR-LABEL: define dso_local noundef i64 @test35_bdos(
+// SANITIZE-WITHOUT-ATTR-SAME: ptr noundef readnone captures(none) [[P:%.*]]) local_unnamed_addr #[[ATTR2]] {
+// SANITIZE-WITHOUT-ATTR-NEXT:  entry:
+// SANITIZE-WITHOUT-ATTR-NEXT:    ret i64 0
+//
+// NO-SANITIZE-WITHOUT-ATTR-LABEL: define dso_local noundef i64 @test35_bdos(
+// NO-SANITIZE-WITHOUT-ATTR-SAME: ptr noundef readnone captures(none) [[P:%.*]]) local_unnamed_addr #[[ATTR1]] {
+// NO-SANITIZE-WITHOUT-ATTR-NEXT:  entry:
+// NO-SANITIZE-WITHOUT-ATTR-NEXT:    ret i64 0
+//
+size_t test35_bdos(struct annotated *p) {
+  return __bdos(&p->array[-42]);
+}
+
+// See https://github.com/llvm/llvm-project/pull/122198#issuecomment-2627868702
+
+typedef struct {
+  char __padding[0];
+} spinlock_t;
+
+struct {
+  int priv_len;
+  spinlock_t addr_list_lock;
+  char *dev_addr;
+  char priv[] __attribute__((__counted_by__(priv_len)));
+} x;
+
+// SANITIZE-WITH-ATTR-LABEL: define dso_local i64 @test36(
+// SANITIZE-WITH-ATTR-SAME: ) local_unnamed_addr #[[ATTR6:[0-9]+]] {
+// SANITIZE-WITH-ATTR-NEXT:  entry:
+// SANITIZE-WITH-ATTR-NEXT:    ret i64 -1
+//
+// NO-SANITIZE-WITH-ATTR-LABEL: define dso_local i64 @test36(
+// NO-SANITIZE-WITH-ATTR-SAME: ) local_unnamed_addr #[[ATTR10:[0-9]+]] {
+// NO-SANITIZE-WITH-ATTR-NEXT:  entry:
+// NO-SANITIZE-WITH-ATTR-NEXT:    ret i64 -1
+//
+// SANITIZE-WITHOUT-ATTR-LABEL: define dso_local i64 @test36(
+// SANITIZE-WITHOUT-ATTR-SAME: ) local_unnamed_addr #[[ATTR6:[0-9]+]] {
+// SANITIZE-WITHOUT-ATTR-NEXT:  entry:
+// SANITIZE-WITHOUT-ATTR-NEXT:    ret i64 -1
+//
+// NO-SANITIZE-WITHOUT-ATTR-LABEL: define dso_local i64 @test36(
+// NO-SANITIZE-WITHOUT-ATTR-SAME: ) local_unnamed_addr #[[ATTR9:[0-9]+]] {
+// NO-SANITIZE-WITHOUT-ATTR-NEXT:  entry:
+// NO-SANITIZE-WITHOUT-ATTR-NEXT:    ret i64 -1
+//
+size_t test36() {
+  return __builtin_dynamic_object_size(&x.dev_addr[4], 1);
 }
diff --git a/clang/test/CodeGen/blocks-windows.c b/clang/test/CodeGen/blocks-windows.c
index 4379cd2e6b63a..4deccfa931720 100644
--- a/clang/test/CodeGen/blocks-windows.c
+++ b/clang/test/CodeGen/blocks-windows.c
@@ -1,44 +1,50 @@
 // RUN: %clang_cc1 -triple thumbv7-windows -fblocks -fdeclspec -DBLOCKS_IN_BLOCKS_DECL -emit-llvm %s -o - | FileCheck %s -check-prefix CHECK-BLOCKS-IN-BLOCKS-DECL
 // RUN: %clang_cc1 -triple thumbv7-windows -fblocks -fdeclspec -DBLOCKS_IN_BLOCKS_DEFN -emit-llvm %s -o - | FileCheck %s -check-prefix CHECK-BLOCKS-IN-BLOCKS-DEFN
 // RUN: %clang_cc1 -triple thumbv7-windows -fblocks -fdeclspec -emit-llvm %s -o - | FileCheck %s -check-prefix CHECK-BLOCKS-NOT-IN-BLOCKS
+// RUN: %clang_cc1 -triple thumbv7-windows -fblocks -fdeclspec -static-libclosure -emit-llvm %s -o - | FileCheck %s -check-prefix CHECK-BLOCKS-NOT-IN-STATIC-BLOCKS
 // RUN: %clang_cc1 -triple thumbv7-windows -fblocks -fdeclspec -DBLOCKS_NOT_IN_BLOCKS_EXTERN -emit-llvm %s -o - | FileCheck %s -check-prefix CHECK-BLOCKS-NOT-IN-BLOCKS-EXTERN
-// RUN: %clang_cc1 -triple thumbv7-windows -fblocks -fdeclspec -DBLCOKS_NOT_IN_BLOCKS_EXTERN_DLLIMPORT -emit-llvm %s -o - | FileCheck %s -check-prefix CHECK-BLOCKS-NOT-IN-BLOCKS-EXTERN-DLLIMPORT
-// RUN: %clang_cc1 -triple thumbv7-windows -fblocks -fdeclspec -DBLCOKS_NOT_IN_BLOCKS_DLLIMPORT -emit-llvm %s -o - | FileCheck %s -check-prefix CHECK-BLOCKS-NOT-IN-BLOCKS-DLLIMPORT
+// RUN: %clang_cc1 -triple thumbv7-windows -fblocks -fdeclspec -DBLOCKS_NOT_IN_BLOCKS_EXTERN_DLLIMPORT -emit-llvm %s -o - | FileCheck %s -check-prefix CHECK-BLOCKS-NOT-IN-BLOCKS-EXTERN-DLLIMPORT
+// RUN: %clang_cc1 -triple thumbv7-windows -fblocks -fdeclspec -DBLOCKS_NOT_IN_BLOCKS_DLLIMPORT -emit-llvm %s -o - | FileCheck %s -check-prefix CHECK-BLOCKS-NOT-IN-BLOCKS-DLLIMPORT
 
 // RUN: %clang_cc1 -triple thumbv7-windows -fblocks -fdeclspec -DBLOCKS_IN_BLOCKS_DECL -Os -emit-llvm %s -o - | FileCheck %s -check-prefix CHECK-BLOCKS-IN-BLOCKS-DECL
 // RUN: %clang_cc1 -triple thumbv7-windows -fblocks -fdeclspec -DBLOCKS_IN_BLOCKS_DEFN -Os -emit-llvm %s -o - | FileCheck %s -check-prefix CHECK-BLOCKS-IN-BLOCKS-DEFN
 // RUN: %clang_cc1 -triple thumbv7-windows -fblocks -fdeclspec -Os -emit-llvm %s -o - | FileCheck %s -check-prefix CHECK-BLOCKS-NOT-IN-BLOCKS
+// RUN: %clang_cc1 -triple thumbv7-windows -fblocks -fdeclspec -Os -static-libclosure -emit-llvm %s -o - | FileCheck %s -check-prefix CHECK-BLOCKS-NOT-IN-STATIC-BLOCKS
 // RUN: %clang_cc1 -triple thumbv7-windows -fblocks -fdeclspec -DBLOCKS_NOT_IN_BLOCKS_EXTERN -Os -emit-llvm %s -o - | FileCheck %s -check-prefix CHECK-BLOCKS-NOT-IN-BLOCKS-EXTERN
-// RUN: %clang_cc1 -triple thumbv7-windows -fblocks -fdeclspec -DBLCOKS_NOT_IN_BLOCKS_EXTERN_DLLIMPORT -Os -emit-llvm %s -o - | FileCheck %s -check-prefix CHECK-BLOCKS-NOT-IN-BLOCKS-EXTERN-DLLIMPORT
-// RUN: %clang_cc1 -triple thumbv7-windows -fblocks -fdeclspec -DBLCOKS_NOT_IN_BLOCKS_DLLIMPORT -Os -emit-llvm %s -o - | FileCheck %s -check-prefix CHECK-BLOCKS-NOT-IN-BLOCKS-DLLIMPORT
+// RUN: %clang_cc1 -triple thumbv7-windows -fblocks -fdeclspec -DBLOCKS_NOT_IN_BLOCKS_EXTERN_DLLIMPORT -Os -emit-llvm %s -o - | FileCheck %s -check-prefix CHECK-BLOCKS-NOT-IN-BLOCKS-EXTERN-DLLIMPORT
+// RUN: %clang_cc1 -triple thumbv7-windows -fblocks -fdeclspec -DBLOCKS_NOT_IN_BLOCKS_DLLIMPORT -Os -emit-llvm %s -o - | FileCheck %s -check-prefix CHECK-BLOCKS-NOT-IN-BLOCKS-DLLIMPORT
 
 // RUN: %clang_cc1 -triple i686-windows -fblocks -fdeclspec -DBLOCKS_IN_BLOCKS_DECL -emit-llvm %s -o - | FileCheck %s -check-prefix CHECK-BLOCKS-IN-BLOCKS-DECL
 // RUN: %clang_cc1 -triple i686-windows -fblocks -fdeclspec -DBLOCKS_IN_BLOCKS_DEFN -emit-llvm %s -o - | FileCheck %s -check-prefix CHECK-BLOCKS-IN-BLOCKS-DEFN
 // RUN: %clang_cc1 -triple i686-windows -fblocks -fdeclspec -emit-llvm %s -o - | FileCheck %s -check-prefix CHECK-BLOCKS-NOT-IN-BLOCKS
+// RUN: %clang_cc1 -triple i686-windows -fblocks -fdeclspec -static-libclosure -emit-llvm %s -o - | FileCheck %s -check-prefix CHECK-BLOCKS-NOT-IN-STATIC-BLOCKS
 // RUN: %clang_cc1 -triple i686-windows -fblocks -fdeclspec -DBLOCKS_NOT_IN_BLOCKS_EXTERN -emit-llvm %s -o - | FileCheck %s -check-prefix CHECK-BLOCKS-NOT-IN-BLOCKS-EXTERN
-// RUN: %clang_cc1 -triple i686-windows -fblocks -fdeclspec -DBLCOKS_NOT_IN_BLOCKS_EXTERN_DLLIMPORT -emit-llvm %s -o - | FileCheck %s -check-prefix CHECK-BLOCKS-NOT-IN-BLOCKS-EXTERN-DLLIMPORT
-// RUN: %clang_cc1 -triple i686-windows -fblocks -fdeclspec -DBLCOKS_NOT_IN_BLOCKS_DLLIMPORT -emit-llvm %s -o - | FileCheck %s -check-prefix CHECK-BLOCKS-NOT-IN-BLOCKS-DLLIMPORT
+// RUN: %clang_cc1 -triple i686-windows -fblocks -fdeclspec -DBLOCKS_NOT_IN_BLOCKS_EXTERN_DLLIMPORT -emit-llvm %s -o - | FileCheck %s -check-prefix CHECK-BLOCKS-NOT-IN-BLOCKS-EXTERN-DLLIMPORT
+// RUN: %clang_cc1 -triple i686-windows -fblocks -fdeclspec -DBLOCKS_NOT_IN_BLOCKS_DLLIMPORT -emit-llvm %s -o - | FileCheck %s -check-prefix CHECK-BLOCKS-NOT-IN-BLOCKS-DLLIMPORT
 
 // RUN: %clang_cc1 -triple i686-windows -fblocks -fdeclspec -DBLOCKS_IN_BLOCKS_DECL -Os -emit-llvm %s -o - | FileCheck %s -check-prefix CHECK-BLOCKS-IN-BLOCKS-DECL
 // RUN: %clang_cc1 -triple i686-windows -fblocks -fdeclspec -DBLOCKS_IN_BLOCKS_DEFN -Os -emit-llvm %s -o - | FileCheck %s -check-prefix CHECK-BLOCKS-IN-BLOCKS-DEFN
 // RUN: %clang_cc1 -triple i686-windows -fblocks -fdeclspec -Os -emit-llvm %s -o - | FileCheck %s -check-prefix CHECK-BLOCKS-NOT-IN-BLOCKS
+// RUN: %clang_cc1 -triple i686-windows -fblocks -fdeclspec -Os -static-libclosure -emit-llvm %s -o - | FileCheck %s -check-prefix CHECK-BLOCKS-NOT-IN-STATIC-BLOCKS
 // RUN: %clang_cc1 -triple i686-windows -fblocks -fdeclspec -DBLOCKS_NOT_IN_BLOCKS_EXTERN -Os -emit-llvm %s -o - | FileCheck %s -check-prefix CHECK-BLOCKS-NOT-IN-BLOCKS-EXTERN
-// RUN: %clang_cc1 -triple i686-windows -fblocks -fdeclspec -DBLCOKS_NOT_IN_BLOCKS_EXTERN_DLLIMPORT -Os -emit-llvm %s -o - | FileCheck %s -check-prefix CHECK-BLOCKS-NOT-IN-BLOCKS-EXTERN-DLLIMPORT
-// RUN: %clang_cc1 -triple i686-windows -fblocks -fdeclspec -DBLCOKS_NOT_IN_BLOCKS_DLLIMPORT -Os -emit-llvm %s -o - | FileCheck %s -check-prefix CHECK-BLOCKS-NOT-IN-BLOCKS-DLLIMPORT
+// RUN: %clang_cc1 -triple i686-windows -fblocks -fdeclspec -DBLOCKS_NOT_IN_BLOCKS_EXTERN_DLLIMPORT -Os -emit-llvm %s -o - | FileCheck %s -check-prefix CHECK-BLOCKS-NOT-IN-BLOCKS-EXTERN-DLLIMPORT
+// RUN: %clang_cc1 -triple i686-windows -fblocks -fdeclspec -DBLOCKS_NOT_IN_BLOCKS_DLLIMPORT -Os -emit-llvm %s -o - | FileCheck %s -check-prefix CHECK-BLOCKS-NOT-IN-BLOCKS-DLLIMPORT
 
 // RUN: %clang_cc1 -triple x86_64-windows -fblocks -fdeclspec -DBLOCKS_IN_BLOCKS_DECL -emit-llvm %s -o - | FileCheck %s -check-prefix CHECK-BLOCKS-IN-BLOCKS-DECL
 // RUN: %clang_cc1 -triple x86_64-windows -fblocks -fdeclspec -DBLOCKS_IN_BLOCKS_DEFN -emit-llvm %s -o - | FileCheck %s -check-prefix CHECK-BLOCKS-IN-BLOCKS-DEFN
 // RUN: %clang_cc1 -triple x86_64-windows -fblocks -fdeclspec -emit-llvm %s -o - | FileCheck %s -check-prefix CHECK-BLOCKS-NOT-IN-BLOCKS
+// RUN: %clang_cc1 -triple x86_64-windows -fblocks -fdeclspec -static-libclosure -emit-llvm %s -o - | FileCheck %s -check-prefix CHECK-BLOCKS-NOT-IN-STATIC-BLOCKS
 // RUN: %clang_cc1 -triple x86_64-windows -fblocks -fdeclspec -DBLOCKS_NOT_IN_BLOCKS_EXTERN -emit-llvm %s -o - | FileCheck %s -check-prefix CHECK-BLOCKS-NOT-IN-BLOCKS-EXTERN
-// RUN: %clang_cc1 -triple x86_64-windows -fblocks -fdeclspec -DBLCOKS_NOT_IN_BLOCKS_EXTERN_DLLIMPORT -emit-llvm %s -o - | FileCheck %s -check-prefix CHECK-BLOCKS-NOT-IN-BLOCKS-EXTERN-DLLIMPORT
-// RUN: %clang_cc1 -triple x86_64-windows -fblocks -fdeclspec -DBLCOKS_NOT_IN_BLOCKS_DLLIMPORT -emit-llvm %s -o - | FileCheck %s -check-prefix CHECK-BLOCKS-NOT-IN-BLOCKS-DLLIMPORT
+// RUN: %clang_cc1 -triple x86_64-windows -fblocks -fdeclspec -DBLOCKS_NOT_IN_BLOCKS_EXTERN_DLLIMPORT -emit-llvm %s -o - | FileCheck %s -check-prefix CHECK-BLOCKS-NOT-IN-BLOCKS-EXTERN-DLLIMPORT
+// RUN: %clang_cc1 -triple x86_64-windows -fblocks -fdeclspec -DBLOCKS_NOT_IN_BLOCKS_DLLIMPORT -emit-llvm %s -o - | FileCheck %s -check-prefix CHECK-BLOCKS-NOT-IN-BLOCKS-DLLIMPORT
 
 // RUN: %clang_cc1 -triple x86_64-windows -fblocks -fdeclspec -DBLOCKS_IN_BLOCKS_DECL -Os -emit-llvm %s -o - | FileCheck %s -check-prefix CHECK-BLOCKS-IN-BLOCKS-DECL
 // RUN: %clang_cc1 -triple x86_64-windows -fblocks -fdeclspec -DBLOCKS_IN_BLOCKS_DEFN -Os -emit-llvm %s -o - | FileCheck %s -check-prefix CHECK-BLOCKS-IN-BLOCKS-DEFN
 // RUN: %clang_cc1 -triple x86_64-windows -fblocks -fdeclspec -Os -emit-llvm %s -o - | FileCheck %s -check-prefix CHECK-BLOCKS-NOT-IN-BLOCKS
+// RUN: %clang_cc1 -triple x86_64-windows -fblocks -fdeclspec -Os -static-libclosure -emit-llvm %s -o - | FileCheck %s -check-prefix CHECK-BLOCKS-NOT-IN-STATIC-BLOCKS
 // RUN: %clang_cc1 -triple x86_64-windows -fblocks -fdeclspec -DBLOCKS_NOT_IN_BLOCKS_EXTERN -Os -emit-llvm %s -o - | FileCheck %s -check-prefix CHECK-BLOCKS-NOT-IN-BLOCKS-EXTERN
-// RUN: %clang_cc1 -triple x86_64-windows -fblocks -fdeclspec -DBLCOKS_NOT_IN_BLOCKS_EXTERN_DLLIMPORT -Os -emit-llvm %s -o - | FileCheck %s -check-prefix CHECK-BLOCKS-NOT-IN-BLOCKS-EXTERN-DLLIMPORT
-// RUN: %clang_cc1 -triple x86_64-windows -fblocks -fdeclspec -DBLCOKS_NOT_IN_BLOCKS_DLLIMPORT -Os -emit-llvm %s -o - | FileCheck %s -check-prefix CHECK-BLOCKS-NOT-IN-BLOCKS-DLLIMPORT
+// RUN: %clang_cc1 -triple x86_64-windows -fblocks -fdeclspec -DBLOCKS_NOT_IN_BLOCKS_EXTERN_DLLIMPORT -Os -emit-llvm %s -o - | FileCheck %s -check-prefix CHECK-BLOCKS-NOT-IN-BLOCKS-EXTERN-DLLIMPORT
+// RUN: %clang_cc1 -triple x86_64-windows -fblocks -fdeclspec -DBLOCKS_NOT_IN_BLOCKS_DLLIMPORT -Os -emit-llvm %s -o - | FileCheck %s -check-prefix CHECK-BLOCKS-NOT-IN-BLOCKS-DLLIMPORT
 
 void *_Block_copy(void *);
 
@@ -70,6 +76,7 @@ int (*g(void))(void) {
 // CHECK-BLOCKS-IN-BLOCKS-DECL: @_NSConcreteStackBlock = external dso_local dllexport global ptr
 // CHECK-BLOCKS-IN-BLOCKS-DEFN: @_NSConcreteStackBlock = dso_local dllexport global [5 x i32]
 // CHECK-BLOCKS-NOT-IN-BLOCKS: @_NSConcreteStackBlock = external dllimport global ptr
+// CHECK-BLOCKS-NOT-IN-STATIC-BLOCKS: @_NSConcreteStackBlock = external dso_local global ptr
 // CHECK-BLOCKS-NOT-IN-BLOCKS-EXTERN: @_NSConcreteStackBlock = external dllimport global ptr
 // CHECK-BLOCKS-NOT-IN-BLOCKS-EXTERN-DLLIMPORT: @_NSConcreteStackBlock = external dllimport global ptr
 // CHECK-BLOCKS-NOT-IN-BLOCKS-DLLIMPORT: @_NSConcreteStackBlock = external dllimport global ptr
diff --git a/clang/test/CodeGen/builtins-elementwise-math.c b/clang/test/CodeGen/builtins-elementwise-math.c
index 7f6b5f26eb930..5661bbc6008a0 100644
--- a/clang/test/CodeGen/builtins-elementwise-math.c
+++ b/clang/test/CodeGen/builtins-elementwise-math.c
@@ -14,10 +14,18 @@ __attribute__((address_space(1))) int int_as_one;
 typedef int bar;
 bar b;
 
+struct StructWithBitfield {
+  int i : 5;
+  short s : 3;
+  char c: 2;
+  long long int lli : 3;
+};
+
 void test_builtin_elementwise_abs(float f1, float f2, double d1, double d2,
                                   float4 vf1, float4 vf2, si8 vi1, si8 vi2,
                                   long long int i1, long long int i2, short si,
-                                  _BitInt(31) bi1, _BitInt(31) bi2) {
+                                  _BitInt(31) bi1, _BitInt(31) bi2, int i,
+                                  char ci) {
   // CHECK-LABEL: define void @test_builtin_elementwise_abs(
   // CHECK:      [[F1:%.+]] = load float, ptr %f1.addr, align 4
   // CHECK-NEXT:  call float @llvm.fabs.f32(float [[F1]])
@@ -35,6 +43,11 @@ void test_builtin_elementwise_abs(float f1, float f2, double d1, double d2,
   // CHECK-NEXT: call i64 @llvm.abs.i64(i64 [[I1]], i1 false)
   i2 = __builtin_elementwise_abs(i1);
 
+  // CHECK:      [[I1:%.+]] = load i64, ptr %i1.addr, align 8
+  // CHECK:      [[S1:%.+]] = trunc i64 [[I1]] to i16
+  // CHECK-NEXT: call i16 @llvm.abs.i16(i16 [[S1]], i1 false)
+  i1 = __builtin_elementwise_abs((short)i1);
+
   // CHECK:      [[VI1:%.+]] = load <8 x i16>, ptr %vi1.addr, align 16
   // CHECK-NEXT: call <8 x i16> @llvm.abs.v8i16(<8 x i16> [[VI1]], i1 false)
   vi2 = __builtin_elementwise_abs(vi1);
@@ -57,10 +70,37 @@ void test_builtin_elementwise_abs(float f1, float f2, double d1, double d2,
   b = __builtin_elementwise_abs(-10);
 
   // CHECK:      [[SI:%.+]] = load i16, ptr %si.addr, align 2
-  // CHECK-NEXT: [[SI_EXT:%.+]] = sext i16 [[SI]] to i32
-  // CHECK-NEXT: [[RES:%.+]] = call i32 @llvm.abs.i32(i32 [[SI_EXT]], i1 false)
-  // CHECK-NEXT: = trunc i32 [[RES]] to i16
+  // CHECK-NEXT: [[RES:%.+]] = call i16 @llvm.abs.i16(i16 [[SI]], i1 false)
   si = __builtin_elementwise_abs(si);
+
+  struct StructWithBitfield t;
+
+  // CHECK:      [[BFLOAD:%.+]] = load i16, ptr %t, align 8
+  // CHECK-NEXT: [[BFSHL:%.+]] = shl i16 [[BFLOAD]], 11
+  // CHECK-NEXT: [[BFASHR:%.+]] = ashr i16 [[BFSHL]], 11
+  // CHECK-NEXT: [[BFCAST:%.+]] = sext i16 [[BFASHR]] to i32
+  // CHECK-NEXT: [[RES:%.+]] = call i32 @llvm.abs.i32(i32 [[BFCAST]], i1 false)
+  i = __builtin_elementwise_abs(t.i);
+
+  // CHECK:      [[BFLOAD:%.+]] = load i16, ptr %t, align 8
+  // CHECK-NEXT: [[BFSHL:%.+]] = shl i16 [[BFLOAD]], 8
+  // CHECK-NEXT: [[BFASHR:%.+]] = ashr i16 [[BFSHL]], 13
+  // CHECK-NEXT: [[RES:%.+]] = call i16 @llvm.abs.i16(i16 [[BFASHR]], i1 false)
+  si = __builtin_elementwise_abs(t.s);
+
+  // CHECK:      [[BFLOAD:%.+]] = load i16, ptr %t, align 8
+  // CHECK-NEXT: [[BFSHL:%.+]] = shl i16 [[BFLOAD]], 6
+  // CHECK-NEXT: [[BFASHR:%.+]] = ashr i16 [[BFSHL]], 14
+  // CHECK-NEXT: [[BFCAST:%.+]] = trunc i16 [[BFASHR]] to i8
+  // CHECK-NEXT: [[RES:%.+]] = call i8 @llvm.abs.i8(i8 [[BFCAST]], i1 false)
+  ci = __builtin_elementwise_abs(t.c);
+
+  // CHECK:      [[BFLOAD:%.+]] = load i16, ptr %t, align 8
+  // CHECK-NEXT: [[BFSHL:%.+]] = shl i16 [[BFLOAD]], 3
+  // CHECK-NEXT: [[BFASHR:%.+]] = ashr i16 [[BFSHL]], 13
+  // CHECK-NEXT: [[BFCAST:%.+]] = sext i16 [[BFASHR]] to i64
+  // CHECK-NEXT: [[RES:%.+]] = call i64 @llvm.abs.i64(i64 [[BFCAST]], i1 false)
+  i1 = __builtin_elementwise_abs(t.lli);
 }
 
 void test_builtin_elementwise_add_sat(float f1, float f2, double d1, double d2,
@@ -68,7 +108,10 @@ void test_builtin_elementwise_add_sat(float f1, float f2, double d1, double d2,
                                       long long int i2, si8 vi1, si8 vi2,
                                       unsigned u1, unsigned u2, u4 vu1, u4 vu2,
                                       _BitInt(31) bi1, _BitInt(31) bi2,
-                                      unsigned _BitInt(55) bu1, unsigned _BitInt(55) bu2) {
+                                      unsigned _BitInt(55) bu1, unsigned _BitInt(55) bu2,
+                                      char c1, char c2, unsigned char uc1,
+                                      unsigned char uc2, short s1, short s2,
+                                      unsigned short us1, unsigned short us2) {
   // CHECK:      [[I1:%.+]] = load i64, ptr %i1.addr, align 8
   // CHECK-NEXT: [[I2:%.+]] = load i64, ptr %i2.addr, align 8
   // CHECK-NEXT: call i64 @llvm.sadd.sat.i64(i64 [[I1]], i64 [[I2]])
@@ -76,7 +119,7 @@ void test_builtin_elementwise_add_sat(float f1, float f2, double d1, double d2,
 
   // CHECK:      [[I1:%.+]] = load i64, ptr %i1.addr, align 8
   // CHECK-NEXT: call i64 @llvm.sadd.sat.i64(i64 [[I1]], i64 10)
-  i1 = __builtin_elementwise_add_sat(i1, 10);
+  i1 = __builtin_elementwise_add_sat(i1, 10ll);
 
   // CHECK:      [[VI1:%.+]] = load <8 x i16>, ptr %vi1.addr, align 16
   // CHECK-NEXT: [[VI2:%.+]] = load <8 x i16>, ptr %vi2.addr, align 16
@@ -114,6 +157,33 @@ void test_builtin_elementwise_add_sat(float f1, float f2, double d1, double d2,
 
   // CHECK: store i64 98, ptr %i1.addr, align 8
   i1 = __builtin_elementwise_add_sat(1, 'a');
+
+  // CHECK:      [[C1:%.+]] = load i8, ptr %c1.addr, align 1
+  // CHECK-NEXT: [[C2:%.+]] = load i8, ptr %c2.addr, align 1
+  // CHECK-NEXT: call i8 @llvm.sadd.sat.i8(i8 [[C1]], i8 [[C2]])
+  c1 = __builtin_elementwise_add_sat(c1, c2);
+
+  // CHECK:      [[UC1:%.+]] = load i8, ptr %uc1.addr, align 1
+  // CHECK-NEXT: [[UC2:%.+]] = load i8, ptr %uc2.addr, align 1
+  // CHECK-NEXT: call i8 @llvm.uadd.sat.i8(i8 [[UC1]], i8 [[UC2]])
+  uc1 = __builtin_elementwise_add_sat(uc1, uc2);
+
+  // CHECK:      [[S1:%.+]] = load i16, ptr %s1.addr, align 2
+  // CHECK-NEXT: [[S2:%.+]] = load i16, ptr %s2.addr, align 2
+  // CHECK-NEXT: call i16 @llvm.sadd.sat.i16(i16 [[S1]], i16 [[S2]])
+  s1 = __builtin_elementwise_add_sat(s1, s2);
+
+  // CHECK:      [[S1:%.+]] = load i16, ptr %s1.addr, align 2
+  // CHECK:      [[I1:%.+]] = sext i16 [[S1]] to i32
+  // CHECK-NEXT: [[S2:%.+]] = load i16, ptr %s2.addr, align 2
+  // CHECK:      [[I2:%.+]] = sext i16 [[S2]] to i32
+  // CHECK-NEXT: call i32 @llvm.sadd.sat.i32(i32 [[I1]], i32 [[I2]])
+  s1 = __builtin_elementwise_add_sat((int)s1, (int)s2);
+
+  // CHECK:      [[US1:%.+]] = load i16, ptr %us1.addr, align 2
+  // CHECK-NEXT: [[US2:%.+]] = load i16, ptr %us2.addr, align 2
+  // CHECK-NEXT: call i16 @llvm.uadd.sat.i16(i16 [[US1]], i16 [[US2]])
+  us1 = __builtin_elementwise_add_sat(us1, us2);
 }
 
 void test_builtin_elementwise_sub_sat(float f1, float f2, double d1, double d2,
@@ -121,7 +191,10 @@ void test_builtin_elementwise_sub_sat(float f1, float f2, double d1, double d2,
                                       long long int i2, si8 vi1, si8 vi2,
                                       unsigned u1, unsigned u2, u4 vu1, u4 vu2,
                                       _BitInt(31) bi1, _BitInt(31) bi2,
-                                      unsigned _BitInt(55) bu1, unsigned _BitInt(55) bu2) {
+                                      unsigned _BitInt(55) bu1, unsigned _BitInt(55) bu2,
+                                      char c1, char c2, unsigned char uc1,
+                                      unsigned char uc2, short s1, short s2,
+                                      unsigned short us1, unsigned short us2) {
   // CHECK:      [[I1:%.+]] = load i64, ptr %i1.addr, align 8
   // CHECK-NEXT: [[I2:%.+]] = load i64, ptr %i2.addr, align 8
   // CHECK-NEXT: call i64 @llvm.ssub.sat.i64(i64 [[I1]], i64 [[I2]])
@@ -129,7 +202,7 @@ void test_builtin_elementwise_sub_sat(float f1, float f2, double d1, double d2,
 
   // CHECK:      [[I1:%.+]] = load i64, ptr %i1.addr, align 8
   // CHECK-NEXT: call i64 @llvm.ssub.sat.i64(i64 [[I1]], i64 10)
-  i1 = __builtin_elementwise_sub_sat(i1, 10);
+  i1 = __builtin_elementwise_sub_sat(i1, 10ll);
 
   // CHECK:      [[VI1:%.+]] = load <8 x i16>, ptr %vi1.addr, align 16
   // CHECK-NEXT: [[VI2:%.+]] = load <8 x i16>, ptr %vi2.addr, align 16
@@ -167,6 +240,26 @@ void test_builtin_elementwise_sub_sat(float f1, float f2, double d1, double d2,
 
   // CHECK: store i64 -96, ptr %i1.addr, align 8
   i1 = __builtin_elementwise_sub_sat(1, 'a');
+
+  // CHECK:      [[C1:%.+]] = load i8, ptr %c1.addr, align 1
+  // CHECK-NEXT: [[C2:%.+]] = load i8, ptr %c2.addr, align 1
+  // CHECK-NEXT: call i8 @llvm.ssub.sat.i8(i8 [[C1]], i8 [[C2]])
+  c1 = __builtin_elementwise_sub_sat(c1, c2);
+
+  // CHECK:      [[UC1:%.+]] = load i8, ptr %uc1.addr, align 1
+  // CHECK-NEXT: [[UC2:%.+]] = load i8, ptr %uc2.addr, align 1
+  // CHECK-NEXT: call i8 @llvm.usub.sat.i8(i8 [[UC1]], i8 [[UC2]])
+  uc1 = __builtin_elementwise_sub_sat(uc1, uc2);
+
+  // CHECK:      [[S1:%.+]] = load i16, ptr %s1.addr, align 2
+  // CHECK-NEXT: [[S2:%.+]] = load i16, ptr %s2.addr, align 2
+  // CHECK-NEXT: call i16 @llvm.ssub.sat.i16(i16 [[S1]], i16 [[S2]])
+  s1 = __builtin_elementwise_sub_sat(s1, s2);
+
+  // CHECK:      [[US1:%.+]] = load i16, ptr %us1.addr, align 2
+  // CHECK-NEXT: [[US2:%.+]] = load i16, ptr %us2.addr, align 2
+  // CHECK-NEXT: call i16 @llvm.usub.sat.i16(i16 [[US1]], i16 [[US2]])
+  us1 = __builtin_elementwise_sub_sat(us1, us2);
 }
 
 void test_builtin_elementwise_maximum(float f1, float f2, double d1, double d2,
@@ -278,7 +371,7 @@ void test_builtin_elementwise_max(float f1, float f2, double d1, double d2,
 
   // CHECK:      [[I1:%.+]] = load i64, ptr %i1.addr, align 8
   // CHECK-NEXT: call i64 @llvm.smax.i64(i64 [[I1]], i64 10)
-  i1 = __builtin_elementwise_max(i1, 10);
+  i1 = __builtin_elementwise_max(i1, 10ll);
 
   // CHECK:      [[VI1:%.+]] = load <8 x i16>, ptr %vi1.addr, align 16
   // CHECK-NEXT: [[VI2:%.+]] = load <8 x i16>, ptr %vi2.addr, align 16
@@ -362,7 +455,14 @@ void test_builtin_elementwise_min(float f1, float f2, double d1, double d2,
 
   // CHECK:      [[I2:%.+]] = load i64, ptr %i2.addr, align 8
   // CHECK-NEXT: call i64 @llvm.smin.i64(i64 -11, i64 [[I2]])
-  i1 = __builtin_elementwise_min(-11, i2);
+  i1 = __builtin_elementwise_min(-11ll, i2);
+
+  // CHECK:      [[I1:%.+]] = load i64, ptr %i1.addr, align 8
+  // CHECK:      [[S1:%.+]] = trunc i64 [[I1]] to i16
+  // CHECK-NEXT: [[I2:%.+]] = load i64, ptr %i2.addr, align 8
+  // CHECK:      [[S2:%.+]] = trunc i64 [[I2]] to i16
+  // CHECK-NEXT: call i16 @llvm.smin.i16(i16 [[S1]], i16 [[S2]])
+  i1 = __builtin_elementwise_min((short)i1, (short)i2);
 
   // CHECK:      [[VI1:%.+]] = load <8 x i16>, ptr %vi1.addr, align 16
   // CHECK-NEXT: [[VI2:%.+]] = load <8 x i16>, ptr %vi2.addr, align 16
@@ -374,12 +474,6 @@ void test_builtin_elementwise_min(float f1, float f2, double d1, double d2,
   // CHECK-NEXT: call i32 @llvm.umin.i32(i32 [[U1]], i32 [[U2]])
   u1 = __builtin_elementwise_min(u1, u2);
 
-  // CHECK:      [[U1:%.+]] = load i32, ptr %u1.addr, align 4
-  // CHECK-NEXT: [[ZEXT_U1:%.+]] = zext i32 [[U1]] to i64
-  // CHECK-NEXT: [[I2:%.+]] = load i64, ptr %i2.addr, align 8
-  // CHECK-NEXT: call i64 @llvm.smin.i64(i64 [[ZEXT_U1]], i64 [[I2]])
-  u1 = __builtin_elementwise_min(u1, i2);
-
   // CHECK:      [[VU1:%.+]] = load <4 x i32>, ptr %vu1.addr, align 16
   // CHECK-NEXT: [[VU2:%.+]] = load <4 x i32>, ptr %vu2.addr, align 16
   // CHECK-NEXT: call <4 x i32> @llvm.umin.v4i32(<4 x i32> [[VU1]], <4 x i32> [[VU2]])
@@ -418,7 +512,8 @@ void test_builtin_elementwise_min(float f1, float f2, double d1, double d2,
 
 void test_builtin_elementwise_bitreverse(si8 vi1, si8 vi2,
                                   long long int i1, long long int i2, short si,
-                                  _BitInt(31) bi1, _BitInt(31) bi2) {
+                                  _BitInt(31) bi1, _BitInt(31) bi2,
+                                  char ci) {
   
 
   // CHECK:      [[I1:%.+]] = load i64, ptr %i1.addr, align 8
@@ -447,10 +542,24 @@ void test_builtin_elementwise_bitreverse(si8 vi1, si8 vi2,
   b = __builtin_elementwise_bitreverse(-10);
 
   // CHECK:      [[SI:%.+]] = load i16, ptr %si.addr, align 2
-  // CHECK-NEXT: [[SI_EXT:%.+]] = sext i16 [[SI]] to i32
-  // CHECK-NEXT: [[RES:%.+]] = call i32 @llvm.bitreverse.i32(i32 [[SI_EXT]])
-  // CHECK-NEXT: = trunc i32 [[RES]] to i16
+  // CHECK-NEXT: [[RES:%.+]] = call i16 @llvm.bitreverse.i16(i16 [[SI]])
   si = __builtin_elementwise_bitreverse(si);
+
+  // CHECK:      store i16 28671, ptr %si.addr, align 2
+  si = __builtin_elementwise_bitreverse((short)-10);
+
+  // CHECK:      store i16 28671, ptr %si.addr, align 2
+  si = __builtin_elementwise_bitreverse((unsigned short)-10);
+
+  // CHECK:      [[CI:%.+]] = load i8, ptr %ci.addr, align 1
+  // CHECK-NEXT: [[RES:%.+]] = call i8 @llvm.bitreverse.i8(i8 [[CI]])
+  ci = __builtin_elementwise_bitreverse(ci);
+
+  // CHECK:      store i8 111, ptr %ci.addr, align 1
+  ci = __builtin_elementwise_bitreverse((unsigned char)-10);
+
+  // CHECK:      store i8 111, ptr %ci.addr, align 1
+  ci = __builtin_elementwise_bitreverse((char)-10);
 }
 
 void test_builtin_elementwise_ceil(float f1, float f2, double d1, double d2,
@@ -668,7 +777,8 @@ void test_builtin_elementwise_log2(float f1, float f2, double d1, double d2,
 
 void test_builtin_elementwise_popcount(si8 vi1, si8 vi2, long long int i1,
                                        long long int i2, short si,
-                                       _BitInt(31) bi1, _BitInt(31) bi2) {
+                                       _BitInt(31) bi1, _BitInt(31) bi2,
+                                       char ci) {
   // CHECK:      [[I1:%.+]] = load i64, ptr %i1.addr, align 8
   // CHECK-NEXT: call i64 @llvm.ctpop.i64(i64 [[I1]])
   i2 = __builtin_elementwise_popcount(i1);
@@ -695,10 +805,24 @@ void test_builtin_elementwise_popcount(si8 vi1, si8 vi2, long long int i1,
   b = __builtin_elementwise_popcount(-10);
 
   // CHECK:      [[SI:%.+]] = load i16, ptr %si.addr, align 2
-  // CHECK-NEXT: [[SI_EXT:%.+]] = sext i16 [[SI]] to i32
-  // CHECK-NEXT: [[RES:%.+]] = call i32 @llvm.ctpop.i32(i32 [[SI_EXT]])
-  // CHECK-NEXT: = trunc i32 [[RES]] to i16
+  // CHECK-NEXT: [[RES:%.+]] = call i16 @llvm.ctpop.i16(i16 [[SI]])
   si = __builtin_elementwise_popcount(si);
+
+  // CHECK:      store i16 3, ptr %si.addr, align 2
+  si = __builtin_elementwise_popcount((unsigned short)32771);
+
+  // CHECK:      store i16 3, ptr %si.addr, align 2
+  si = __builtin_elementwise_popcount((short)32771);
+
+  // CHECK:      [[CI:%.+]] = load i8, ptr %ci.addr, align 1
+  // CHECK-NEXT: [[RES:%.+]] = call i8 @llvm.ctpop.i8(i8 [[CI]])
+  ci = __builtin_elementwise_popcount(ci);
+
+  // CHECK:      store i8 2, ptr %ci.addr, align 1
+  ci = __builtin_elementwise_popcount((unsigned char)192);
+
+  // CHECK:      store i8 2, ptr %ci.addr, align 1
+  ci = __builtin_elementwise_popcount((char)192);
 }
 
 void test_builtin_elementwise_fmod(float f1, float f2, double d1, double d2,
diff --git a/clang/test/CodeGen/builtins-nvptx.c b/clang/test/CodeGen/builtins-nvptx.c
index 163aee4799ff0..ffa41c85c2734 100644
--- a/clang/test/CodeGen/builtins-nvptx.c
+++ b/clang/test/CodeGen/builtins-nvptx.c
@@ -28,6 +28,11 @@
 // RUN: %clang_cc1 -ffp-contract=off -triple nvptx64-unknown-unknown -target-cpu sm_89 -target-feature +ptx81 \
 // RUN:            -fcuda-is-device -emit-llvm -o - -x cuda %s \
 // RUN:   | FileCheck -check-prefix=CHECK -check-prefix=CHECK_PTX81_SM89 %s
+// ###  The last run to check with the highest SM and PTX version available
+// ###  to make sure target builtins are still accepted.
+// RUN: %clang_cc1 -ffp-contract=off -triple nvptx64-unknown-unknown -target-cpu sm_100a -target-feature +ptx87 \
+// RUN:            -fcuda-is-device -emit-llvm -o - -x cuda %s \
+// RUN:   | FileCheck -check-prefix=CHECK -check-prefix=CHECK_PTX81_SM89 %s
 
 #define __device__ __attribute__((device))
 #define __global__ __attribute__((global))
@@ -202,7 +207,7 @@ __device__ void exit() {
 // NVVM intrinsics
 
 // The idea is not to test all intrinsics, just that Clang is recognizing the
-// builtins defined in BuiltinsNVPTX.def
+// builtins defined in BuiltinsNVPTX.td
 __device__ void nvvm_math(float f1, float f2, double d1, double d2) {
 // CHECK: call float @llvm.nvvm.fmax.f
   float t1 = __nvvm_fmax_f(f1, f2);
diff --git a/clang/test/CodeGen/debug-info-enum-extensibility.c b/clang/test/CodeGen/debug-info-enum-extensibility.c
new file mode 100644
index 0000000000000..4f8a42bff3f01
--- /dev/null
+++ b/clang/test/CodeGen/debug-info-enum-extensibility.c
@@ -0,0 +1,49 @@
+// RUN: %clang_cc1 -emit-llvm -debug-info-kind=limited %s -o - | FileCheck %s
+
+// CHECK-NOT:  enumKind
+// CHECK:      !DICompositeType(tag: DW_TAG_enumeration_type, name: "ClosedEnum"
+// CHECK-SAME:                  enumKind: DW_APPLE_ENUM_KIND_Closed)
+// CHECK:      !DICompositeType(tag: DW_TAG_enumeration_type, name: "OpenEnum"
+// CHECK-SAME:                  enumKind: DW_APPLE_ENUM_KIND_Open)
+// CHECK:      !DICompositeType(tag: DW_TAG_enumeration_type, name: "ClosedFlagEnum"
+// CHECK-SAME:                  enumKind: DW_APPLE_ENUM_KIND_Closed)
+// CHECK:      !DICompositeType(tag: DW_TAG_enumeration_type, name: "OpenFlagEnum"
+// CHECK-SAME:                  enumKind: DW_APPLE_ENUM_KIND_Open)
+// CHECK:      !DICompositeType(tag: DW_TAG_enumeration_type, name: "MixedEnum"
+// CHECK-SAME:                  enumKind: DW_APPLE_ENUM_KIND_Open)
+
+enum Enum {
+  E0, E1
+};
+
+enum FlagEnum {
+  FE0 = 1 << 0, FE1 = 1 << 1
+};
+
+enum __attribute__((enum_extensibility(closed))) ClosedEnum {
+  A0, A1
+};
+
+enum __attribute__((enum_extensibility(open))) OpenEnum {
+  B0, B1
+};
+
+enum __attribute__((enum_extensibility(closed),flag_enum)) ClosedFlagEnum {
+  C0 = 1 << 0, C1 = 1 << 1
+};
+
+enum __attribute__((enum_extensibility(open),flag_enum)) OpenFlagEnum {
+  D0 = 1 << 0, D1 = 1 << 1
+};
+
+enum __attribute__((enum_extensibility(open), enum_extensibility(closed))) MixedEnum {
+  M0, M1
+};
+
+enum Enum e;
+enum FlagEnum fe;
+enum ClosedEnum ce;
+enum OpenEnum oe;
+enum ClosedFlagEnum cfe;
+enum OpenFlagEnum ofe;
+enum MixedEnum me;
diff --git a/clang/test/CodeGen/extend-variable-liveness-except.cpp b/clang/test/CodeGen/extend-variable-liveness-except.cpp
new file mode 100644
index 0000000000000..c3a29f57bdae5
--- /dev/null
+++ b/clang/test/CodeGen/extend-variable-liveness-except.cpp
@@ -0,0 +1,34 @@
+// RUN: %clang_cc1 %s -triple %itanium_abi_triple -emit-llvm -fextend-variable-liveness -fcxx-exceptions -fexceptions -o - | FileCheck %s
+// This test checks that the fake uses can be generated in exception handling
+// blocks and that we can emit fake uses for the __int128 data type.
+
+extern int bar();
+
+/// Try block: fake use ends at try-block scope.
+// [[BAR_VAL::%[a-zA-Z0-9\.]+]] = invoke{{.*}} i32 @_Z3barv()
+// store i32 %[[BAR_VAL]], ptr [[K_ALLOC_VAL:%[a-zA-Z0-9\.]+]], align 4
+// [[K_FAKE_USE:%[a-zA-Z0-9\.]+]] = load i32, ptr [[K_ALLOC_VAL]], align 4
+// call void (...) @llvm.fake.use(i32 [[K_FAKE_USE]]) #2
+// br label
+
+/// Catch block: fetching the caught value...
+// CHECK: [[CATCH_PTR:%[a-zA-Z0-9\.]+]] = call ptr @__cxa_begin_catch(
+// CHECK: [[L_VAL:%[a-zA-Z0-9\.]+]] = load i32, ptr [[CATCH_PTR]], align 4
+
+/// Storing to allocas...
+// CHECK-DAG: store i32 8, ptr [[M_ALLOC_VAL:%[a-zA-Z0-9\.]+]]
+// CHECK-DAG: store i32 [[L_VAL]], ptr [[L_ALLOC_VAL:%[a-zA-Z0-9\.]+]], align 4
+
+/// Load into fake uses - expect M to precede L.
+// CHECK: [[M_FAKE_VAL:%[a-zA-Z0-9\.]+]] = load i32, ptr [[M_ALLOC_VAL]]
+// CHECK: call void (...) @llvm.fake.use(i32 [[M_FAKE_VAL]])
+// CHECK: [[L_FAKE_VAL:%[a-zA-Z0-9\.]+]] = load i32, ptr [[L_ALLOC_VAL]]
+// CHECK: call void (...) @llvm.fake.use(i32 [[L_FAKE_VAL]])
+void foo() {
+  try {
+    int k = bar();
+  } catch (int l) {
+    /// The catch block contains a fake use for the local within its scope.
+    int m = 8;
+  }
+}
diff --git a/clang/test/CodeGen/extend-variable-liveness-wide-scalar.cpp b/clang/test/CodeGen/extend-variable-liveness-wide-scalar.cpp
new file mode 100644
index 0000000000000..42b893cc3cdf7
--- /dev/null
+++ b/clang/test/CodeGen/extend-variable-liveness-wide-scalar.cpp
@@ -0,0 +1,11 @@
+// RUN: %clang_cc1 %s -emit-llvm -fextend-variable-liveness -triple x86_64-unknown-linux -o - | FileCheck %s
+// REQUIRES: x86-registered-target
+// This test checks that the fake uses can be generated in exception handling
+// blocks and that we can emit fake uses for the __int128 data type.
+
+void bar();
+
+// CHECK: call void (...) @llvm.fake.use(i128 %
+void foo(__int128 wide_int) {
+  bar();
+}
diff --git a/clang/test/CodeGen/extend-variable-liveness.c b/clang/test/CodeGen/extend-variable-liveness.c
new file mode 100644
index 0000000000000..c6bca48e6469c
--- /dev/null
+++ b/clang/test/CodeGen/extend-variable-liveness.c
@@ -0,0 +1,29 @@
+// RUN: %clang_cc1 %s -triple x86_64-unknown-linux-gnu -emit-llvm -fextend-variable-liveness -o - | FileCheck %s --implicit-check-not=llvm.fake.use
+// Check that fake use calls are emitted at the correct locations, i.e.
+// at the end of lexical blocks and at the end of the function.
+
+int glob_i;
+char glob_c;
+float glob_f;
+
+int foo(int i) {
+  // CHECK-LABEL: define{{.*}}foo
+  if (i < 4) {
+    char j = i * 3;
+    if (glob_i > 3) {
+      float f = glob_f;
+      j = f;
+      glob_c = j;
+      // CHECK: call void (...) @llvm.fake.use(float %
+      // CHECK-NEXT: br label %
+    }
+    glob_i = j;
+    // CHECK: call void (...) @llvm.fake.use(i8 %
+    // CHECK-NEXT: br label %
+  }
+  // CHECK: call void (...) @llvm.fake.use(i32 %
+  // CHECK-NEXT: ret
+  return 4;
+}
+
+// CHECK: declare void @llvm.fake.use(...)
diff --git a/clang/test/CodeGen/fake-use-determinism.c b/clang/test/CodeGen/fake-use-determinism.c
new file mode 100644
index 0000000000000..039d8de692431
--- /dev/null
+++ b/clang/test/CodeGen/fake-use-determinism.c
@@ -0,0 +1,20 @@
+// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -emit-llvm -fextend-variable-liveness %s -o - | FileCheck %s
+//
+// We are checking that the fake.use calls for i, j and k appear
+// in a particular order. It is not the order itself that is important
+// but that it remains the same between different test runs.
+
+// CHECK:      [[K_FAKE_USE:%[a-zA-Z0-9\.]+]] = load i32, ptr %k.addr
+// CHECK-NEXT: call void (...) @llvm.fake.use(i32 [[K_FAKE_USE]]) #2
+// CHECK-NEXT: [[J_FAKE_USE:%[a-zA-Z0-9\.]+]] = load i32, ptr %j.addr
+// CHECK-NEXT: call void (...) @llvm.fake.use(i32 [[J_FAKE_USE]]) #2
+// CHECK-NEXT: [[I_FAKE_USE:%[a-zA-Z0-9\.]+]] = load i32, ptr %i.addr
+// CHECK-NEXT: call void (...) @llvm.fake.use(i32 [[I_FAKE_USE]]) #2
+
+void bar();
+void foo(int i, int j, int k)
+{
+   for (int l = 0; l < i; l++) {
+      bar();
+   }
+}
diff --git a/clang/test/CodeGen/fake-use-lambda.cpp b/clang/test/CodeGen/fake-use-lambda.cpp
new file mode 100644
index 0000000000000..aaf25caab6a39
--- /dev/null
+++ b/clang/test/CodeGen/fake-use-lambda.cpp
@@ -0,0 +1,43 @@
+// RUN: %clang_cc1 %s -triple=%itanium_abi_triple -O1 -emit-llvm -fextend-variable-liveness -o - | FileCheck %s
+// Make sure we don't crash compiling a lambda that is not nested in a function.
+// We also check that fake uses are properly issued in lambdas.
+
+int glob;
+
+extern int foo();
+
+struct S {
+  static const int a;
+};
+
+const int S::a = [](int b) __attribute__((noinline)) {
+  return b * foo();
+}
+(glob);
+
+int func(int param) {
+  return ([=](int lambdaparm) __attribute__((noinline))->int {
+    int lambdalocal = lambdaparm * 2;
+    return lambdalocal;
+  }(glob));
+}
+
+// We are looking for the first lambda's call operator, which should contain
+// 2 fake uses, one for 'b' and one for its 'this' pointer (in that order).
+// The mangled function name contains a $_0, followed by 'cl'.
+// This lambda is an orphaned lambda, i.e. one without lexical parent.
+//
+// CHECK-LABEL: define internal {{.+\"_Z.+\$_0.*cl.*\"}}
+// CHECK-NOT:   ret
+// CHECK:       fake.use(i32
+// CHECK-NOT:   ret
+// CHECK:       fake.use(ptr
+
+// The second lambda. We are looking for 3 fake uses.
+// CHECK-LABEL: define internal {{.+\"_Z.+\$_0.*cl.*\"}}
+// CHECK-NOT:   ret
+// CHECK:       fake.use(i32
+// CHECK-NOT:   ret
+// CHECK:       fake.use(i32
+// CHECK-NOT:   ret
+// CHECK:       fake.use(ptr
diff --git a/clang/test/CodeGen/fake-use-landingpad.c b/clang/test/CodeGen/fake-use-landingpad.c
new file mode 100644
index 0000000000000..ffaf3975ef33f
--- /dev/null
+++ b/clang/test/CodeGen/fake-use-landingpad.c
@@ -0,0 +1,14 @@
+// RUN: %clang_cc1 %s -emit-llvm -fextend-variable-liveness -fexceptions -o - | FileCheck %s --implicit-check-not="landingpad {"
+
+// Check that fake uses do not mistakenly cause a landing pad to be generated when
+// exceptions are enabled.
+
+extern void bar(int);
+void foo(int p) {
+  int a = 17;
+  bar(a);
+}
+
+// CHECK:      define {{.*}} @foo
+// CHECK-NOT:  personality
+// CHECK:      call void (...) @llvm.fake.use
diff --git a/clang/test/CodeGen/fake-use-noreturn.cpp b/clang/test/CodeGen/fake-use-noreturn.cpp
new file mode 100644
index 0000000000000..db7f27735b1d3
--- /dev/null
+++ b/clang/test/CodeGen/fake-use-noreturn.cpp
@@ -0,0 +1,28 @@
+// RUN: %clang_cc1 %s -triple x86_64-unknown-linux-gnu -emit-llvm -fextend-variable-liveness -o - | FileCheck %s
+//
+// Check we can correctly produce fake uses for function-level variables even
+// when we have a return in a nested conditional and there is no code at the end
+// of the function.
+
+// CHECK-LABEL: define{{.*}}@_Z3fooi
+// CHECK:         [[I_FAKE_USE:%[a-zA-Z0-9\.]+]] = load i32, ptr %i.addr
+// CHECK:         call void (...) @llvm.fake.use(i32 [[I_FAKE_USE]])
+// CHECK-LABEL: define{{.*}}@_ZN1C3barEi
+// CHECK:         [[J_FAKE_USE:%[a-zA-Z0-9\.]+]] = load i32, ptr %j.addr
+// CHECK:         call void (...) @llvm.fake.use(i32 [[J_FAKE_USE]])
+
+void foo(int i) {
+   while (0)
+     if (1)
+       return;
+}
+
+class C {
+  void bar(int j);
+};
+
+void C::bar(int j) {
+  while (0)
+    if (1)
+      return;
+}
diff --git a/clang/test/CodeGen/fake-use-return-line.c b/clang/test/CodeGen/fake-use-return-line.c
new file mode 100644
index 0000000000000..50d5885c28348
--- /dev/null
+++ b/clang/test/CodeGen/fake-use-return-line.c
@@ -0,0 +1,16 @@
+// RUN: %clang_cc1 -emit-llvm -debug-info-kind=limited -fextend-variable-liveness -o - %s | FileCheck %s
+
+// Clang adjusts the line numbers of returns based on the line numbers of
+// dominating stores to %retval; we test that fake use intrinsics do not affect
+// this, and the return is given the correct line.
+
+// CHECK: define{{.*}}@main
+// CHECK: call void (...) @llvm.fake.use(i32
+// CHECK-NEXT: ret i32{{.*}}!dbg ![[MDINDEX:[0-9]*]]
+// CHECK: ![[MDINDEX]] = !DILocation(line: [[# @LINE + 5]]
+int main()
+{
+  volatile int a = 1;
+  int b = a + 2;
+  return b;
+}
diff --git a/clang/test/CodeGen/fake-use-sanitizer.cpp b/clang/test/CodeGen/fake-use-sanitizer.cpp
new file mode 100644
index 0000000000000..e808c645b9c60
--- /dev/null
+++ b/clang/test/CodeGen/fake-use-sanitizer.cpp
@@ -0,0 +1,61 @@
+// RUN: %clang_cc1 %s -triple x86_64-unknown-linux-gnu -emit-llvm -fextend-variable-liveness -fsanitize=null -fsanitize-trap=null -o - | FileCheck --check-prefixes=CHECK,NULL --implicit-check-not=ubsantrap %s
+// RUN: %clang_cc1 %s -triple x86_64-unknown-linux-gnu -emit-llvm -fextend-variable-liveness -o - | FileCheck %s
+
+// With -fextend-variable-liveness, the compiler previously generated a fake.use of any
+// reference variable at the end of the scope in which its alloca exists. This
+// caused two issues, where we would get fake uses for uninitialized variables
+// if that variable was declared after an early-return, and UBSan's null checks
+// would complain about this.
+// This test verifies that UBSan does not produce null-checks for arguments to
+// llvm.fake.use, and that fake uses are not emitted for a variable on paths
+// it has not been declared.
+
+struct A { short s1, s2; };
+extern long& getA();
+
+void foo()
+{
+  auto& va = getA();
+  if (va < 5)
+    return;
+
+  auto& vb = getA();
+}
+
+// CHECK-LABEL:  define{{.*}}foo
+// CHECK:  [[VA_CALL:%.+]] = call{{.*}} ptr @_Z4getAv()
+
+/// We check here for the first UBSan check for "va".
+// NULL:   [[VA_ISNULL:%.+]] = icmp ne ptr [[VA_CALL]], null
+// NULL:   br i1 [[VA_ISNULL]], label %{{[^,]+}}, label %[[VA_TRAP:[^,]+]]
+// NULL: [[VA_TRAP]]:
+// NULL:   call void @llvm.ubsantrap(
+
+// CHECK:       [[VA_PTR:%.+]] = load ptr, ptr %va
+// CHECK-NEXT:  [[VA_CMP:%.+]] = load i64, ptr [[VA_PTR]]
+// CHECK-NEXT:  [[VA_CMP_RES:%.+]] = icmp slt i64 [[VA_CMP]], 5
+// CHECK-NEXT:  br i1 [[VA_CMP_RES]], label %[[EARLY_EXIT:[^,]+]], label %[[NOT_EARLY_EXIT:[^,]+]]
+
+// CHECK: [[EARLY_EXIT]]:
+// CHECK:   br label %cleanup
+
+/// The fake use for "vb" only appears on the path where its declaration is
+/// reached.
+// CHECK:     [[NOT_EARLY_EXIT]]:
+// CHECK:  [[VB_CALL:%.+]] = call{{.*}} ptr @_Z4getAv()
+
+/// We check here for the second UBSan check for "vb".
+// NULL:   [[VB_ISNULL:%.+]] = icmp ne ptr [[VB_CALL]], null
+// NULL:   br i1 [[VB_ISNULL]], label %{{[^,]+}}, label %[[VB_TRAP:[^,]+]]
+// NULL: [[VB_TRAP]]:
+// NULL:   call void @llvm.ubsantrap(
+
+// CHECK:       [[VB_FAKE_USE:%.+]] = load ptr, ptr %vb
+// CHECK-NEXT:  call void (...) @llvm.fake.use(ptr [[VB_FAKE_USE]])
+// CHECK:       br label %cleanup
+
+// CHECK:     cleanup:
+// CHECK:       [[VA_FAKE_USE:%.+]] = load ptr, ptr %va
+// CHECK-NEXT:  call void (...) @llvm.fake.use(ptr [[VA_FAKE_USE]])
+
+// NULL: declare void @llvm.ubsantrap
diff --git a/clang/test/CodeGen/fake-use-scalar.c b/clang/test/CodeGen/fake-use-scalar.c
new file mode 100644
index 0000000000000..8514d57958920
--- /dev/null
+++ b/clang/test/CodeGen/fake-use-scalar.c
@@ -0,0 +1,41 @@
+// RUN: %clang_cc1 %s -emit-llvm -fextend-variable-liveness -o - | FileCheck %s --implicit-check-not=fake.use
+// Make sure we don't generate fake.use for non-scalar variables, unless they
+// are small enough that they may be represented as a scalar in LLVM IR.
+// Make sure we don't generate fake.use for volatile variables
+// and parameters even when they are scalar.
+
+struct BigAggr {
+  unsigned long t;
+  char c[1024];
+  unsigned char r[32];
+};
+
+struct SmallAggr {
+  int i;
+  int j;
+};
+
+int foo(volatile int vol_param, int param)
+{
+  struct BigAggr big;
+  struct SmallAggr small;
+  volatile int vol_local;
+  int local;
+  unsigned long_arr[5];
+  unsigned short_arr[4];
+  return 0;
+}
+
+// CHECK: [[SMALL_ARR_FAKE_USE:%.+]] = load [4 x i[[#UINT_SIZE:]]], ptr %short_arr
+// CHECK: call void (...) @llvm.fake.use([4 x i[[#UINT_SIZE]]] [[SMALL_ARR_FAKE_USE]])
+
+// CHECK: [[LOCAL_FAKE_USE:%.+]] = load i32, ptr %local
+// CHECK: call void (...) @llvm.fake.use(i32 [[LOCAL_FAKE_USE]])
+
+// CHECK: [[SMALL_FAKE_USE:%.+]] = load %struct.SmallAggr, ptr %small
+// CHECK: call void (...) @llvm.fake.use(%struct.SmallAggr [[SMALL_FAKE_USE]])
+
+// CHECK: [[PARAM_FAKE_USE:%.+]] = load i32, ptr %param.addr
+// CHECK: call void (...) @llvm.fake.use(i32 [[PARAM_FAKE_USE]])
+
+// CHECK: declare void @llvm.fake.use
diff --git a/clang/test/CodeGen/fake-use-this.cpp b/clang/test/CodeGen/fake-use-this.cpp
new file mode 100644
index 0000000000000..a0ed93d222d50
--- /dev/null
+++ b/clang/test/CodeGen/fake-use-this.cpp
@@ -0,0 +1,34 @@
+// RUN: %clang_cc1 %s -triple x86_64-unknown-linux-gnu -emit-llvm -fextend-variable-liveness=this -o - | FileCheck %s --implicit-check-not=fake.use
+// Check that we generate a fake_use call with the 'this' pointer as argument,
+// and no other fake uses.
+// The call should appear after the call to bar().
+
+void bar();
+
+class C
+{
+public:
+    bool test(int p);
+    C(int v): v(v) {}
+
+private:
+    int v;
+};
+
+bool C::test(int p)
+{
+// CHECK-LABEL: define{{.*}}_ZN1C4testEi(ptr{{[^,]*}} %this, i32{{.*}} %p)
+// CHECK:   %this.addr = alloca ptr
+// CHECK:   store ptr %this, ptr %this.addr
+    int res = p - v;
+
+    bar();
+// CHECK: call{{.*}}bar
+
+    return res != 0;
+// CHECK:      [[FAKE_USE:%.+]] = load ptr, ptr %this.addr
+// CHECK-NEXT: call void (...) @llvm.fake.use(ptr{{.*}} [[FAKE_USE]])
+// CHECK-NEXT: ret
+}
+
+// CHECK: declare void @llvm.fake.use
diff --git a/clang/test/CodeGen/fake-use-while.c b/clang/test/CodeGen/fake-use-while.c
new file mode 100644
index 0000000000000..a74887d979528
--- /dev/null
+++ b/clang/test/CodeGen/fake-use-while.c
@@ -0,0 +1,18 @@
+// RUN: %clang_cc1 %s -emit-llvm -fextend-variable-liveness -o - | FileCheck %s
+//
+// Check we don't assert when there is no more code after a while statement
+// and the body of the while statement ends in a return, i.e. no insertion point
+// is available.
+
+// CHECK: define{{.*}}foo
+// CHECK: call{{.*}}llvm.fake.use
+
+void foo() {
+  {
+    while (1) {
+      int ret;
+      if (1)
+        return;
+    }
+  }
+}
diff --git a/clang/test/CodeGen/fat-lto-objects-cfi.cpp b/clang/test/CodeGen/fat-lto-objects-cfi.cpp
index 628951847053a..4055c5bee9b8a 100644
--- a/clang/test/CodeGen/fat-lto-objects-cfi.cpp
+++ b/clang/test/CodeGen/fat-lto-objects-cfi.cpp
@@ -2,20 +2,20 @@
 
 // RUN: %clang_cc1 -triple x86_64-unknown-fuchsia -O2 -flto -ffat-lto-objects \
 // RUN:          -fsanitize=cfi-icall -fsanitize-trap=cfi-icall -fvisibility=hidden  -emit-llvm -o - %s \
-// RUN:   | FileCheck %s
+// RUN:   | FileCheck %s --check-prefix=TYPE_TEST
 
-// CHECK: llvm.embedded.object
-// CHECK-SAME: section ".llvm.lto"
+// TYPE_TEST: llvm.embedded.object
+// TYPE_TEST-SAME: section ".llvm.lto"
 
-// CHECK-LABEL: define hidden void @foo
-//      CHECK: entry:
-// CHECK-NEXT:   %cmp14.not = icmp eq i64 %len, 0
-// CHECK-NEXT:   br i1 %cmp14.not, label %for.end7, label %for.cond1.preheader.preheader
-//      CHECK: for.cond1.preheader.preheader:                    ; preds = %entry
-// CHECK-NEXT:   %arrayidx.1 = getelementptr inbounds nuw i8, ptr %ptr, i64 4
-// CHECK-NEXT:   br label %for.cond1.preheader
+// TYPE_TEST-LABEL: define hidden void @foo
+//      TYPE_TEST: entry:
+// TYPE_TEST-NEXT:   %cmp14.not = icmp eq i64 %len, 0
+// TYPE_TEST-NEXT:   br i1 %cmp14.not, label %for.end7, label %for.cond1.preheader.preheader
+//      TYPE_TEST: for.cond1.preheader.preheader:                    ; preds = %entry
+// TYPE_TEST-NEXT:   %arrayidx.1 = getelementptr inbounds nuw i8, ptr %ptr, i64 4
+// TYPE_TEST-NEXT:   br label %for.cond1.preheader
 
-// CHECK-NOT: @llvm.type.test
+// TYPE_TEST-NOT: @llvm.type.test
 
 // The code below is a reduced case from https://github.com/llvm/llvm-project/issues/112053
 #define __PRINTFLIKE(__fmt, __varargs) __attribute__((__format__(__printf__, __fmt, __varargs)))
diff --git a/clang/test/CodeGen/import-call-optimization.c b/clang/test/CodeGen/import-call-optimization.c
new file mode 100644
index 0000000000000..cc4e37fda7bb1
--- /dev/null
+++ b/clang/test/CodeGen/import-call-optimization.c
@@ -0,0 +1,5 @@
+// RUN: %clang_cc1 -import-call-optimization -emit-llvm %s -o - | FileCheck %s
+
+void f(void) {}
+
+// CHECK: !"import-call-optimization", i32 1}
diff --git a/clang/test/CodeGen/integer-overflow.c b/clang/test/CodeGen/integer-overflow.c
index 9e8cde8b33b16..a3a66e6137bed 100644
--- a/clang/test/CodeGen/integer-overflow.c
+++ b/clang/test/CodeGen/integer-overflow.c
@@ -1,7 +1,7 @@
 // RUN: %clang_cc1 -triple x86_64-apple-darwin %s -emit-llvm -o - | FileCheck %s --check-prefix=DEFAULT
 // RUN: %clang_cc1 -triple x86_64-apple-darwin %s -emit-llvm -o - -fwrapv | FileCheck %s --check-prefix=WRAPV
 // RUN: %clang_cc1 -triple x86_64-apple-darwin %s -emit-llvm -o - -ftrapv | FileCheck %s --check-prefix=TRAPV
-// RUN: %clang_cc1 -triple x86_64-apple-darwin %s -emit-llvm -o - -fsanitize=signed-integer-overflow | FileCheck %s --check-prefixes=CATCH_UB,CATCH_UB_POINTER
+// RUN: %clang_cc1 -triple x86_64-apple-darwin %s -emit-llvm -o - -fsanitize=signed-integer-overflow | FileCheck %s --check-prefixes=CATCH_UB,NOCATCH_UB_POINTER
 // RUN: %clang_cc1 -triple x86_64-apple-darwin %s -emit-llvm -o - -fsanitize=signed-integer-overflow -fwrapv | FileCheck %s --check-prefixes=CATCH_UB,NOCATCH_UB_POINTER
 // RUN: %clang_cc1 -triple x86_64-apple-darwin %s -emit-llvm -o - -ftrapv -ftrapv-handler foo | FileCheck %s --check-prefix=TRAPV_HANDLER
 
@@ -57,14 +57,14 @@ void test1(void) {
   // TRAPV_HANDLER: foo(
   --a;
   
-  // -fwrapv should turn off inbounds for GEP's, PR9256
+  // -fwrapv does not affect inbounds for GEP's.
+  // This is controlled by -fwrapv-pointer instead.
   extern int* P;
   ++P;
   // DEFAULT: getelementptr inbounds nuw i32, ptr
-  // WRAPV: getelementptr i32, ptr
+  // WRAPV: getelementptr inbounds nuw i32, ptr
   // TRAPV: getelementptr inbounds nuw i32, ptr
-  // CATCH_UB_POINTER: getelementptr inbounds nuw i32, ptr
-  // NOCATCH_UB_POINTER: getelementptr i32, ptr
+  // NOCATCH_UB_POINTER: getelementptr inbounds nuw i32, ptr
 
   // PR9350: char pre-increment never overflows.
   extern volatile signed char PR9350_char_inc;
diff --git a/clang/test/CodeGen/isfpclass.c b/clang/test/CodeGen/isfpclass.c
index 1bf60b8fbca17..26dd846a2bf20 100644
--- a/clang/test/CodeGen/isfpclass.c
+++ b/clang/test/CodeGen/isfpclass.c
@@ -160,7 +160,7 @@ int4 check_isfpclass_nan_strict_v4f32(float4 x) {
 }
 
 // CHECK-LABEL: define dso_local void @check_isfpclass_nan_v4f64
-// CHECK-SAME: (ptr dead_on_unwind noalias nocapture writable writeonly sret(<4 x i64>) align 16 initializes((0, 32)) [[AGG_RESULT:%.*]], ptr nocapture noundef readonly [[TMP0:%.*]]) local_unnamed_addr #[[ATTR3:[0-9]+]] {
+// CHECK-SAME: (ptr dead_on_unwind noalias writable writeonly sret(<4 x i64>) align 16 captures(none) initializes((0, 32)) [[AGG_RESULT:%.*]], ptr noundef readonly captures(none) [[TMP0:%.*]]) local_unnamed_addr #[[ATTR3:[0-9]+]] {
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[X:%.*]] = load <4 x double>, ptr [[TMP0]], align 16, !tbaa [[TBAA2:![0-9]+]]
 // CHECK-NEXT:    [[TMP1:%.*]] = fcmp uno <4 x double> [[X]], zeroinitializer
diff --git a/clang/test/CodeGen/kcfi-arity.c b/clang/test/CodeGen/kcfi-arity.c
new file mode 100644
index 0000000000000..220d444eca30e
--- /dev/null
+++ b/clang/test/CodeGen/kcfi-arity.c
@@ -0,0 +1,7 @@
+// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -emit-llvm -fsanitize=kcfi -fsanitize-kcfi-arity -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -emit-llvm -fsanitize=kcfi -fsanitize-kcfi-arity -x c++ -o - %s | FileCheck %s
+#if !__has_feature(kcfi_arity)
+#error Missing kcfi_arity?
+#endif
+
+// CHECK: ![[#]] = !{i32 4, !"kcfi-arity", i32 1}
diff --git a/clang/test/CodeGen/math-libcalls-tbaa-indirect-args.c b/clang/test/CodeGen/math-libcalls-tbaa-indirect-args.c
index 440db83fef5da..eb706154300a2 100644
--- a/clang/test/CodeGen/math-libcalls-tbaa-indirect-args.c
+++ b/clang/test/CodeGen/math-libcalls-tbaa-indirect-args.c
@@ -53,7 +53,7 @@ long double powl(long double a, long double b);
 // CHECK-SPIR:    [[CALL:%.*]] = tail call spir_func double @powl(double noundef [[A]], double noundef [[B]]) #[[ATTR3:[0-9]+]], !tbaa [[TBAA2:![0-9]+]]
 //
 // CHECK-MINGW32-LABEL: define dso_local void @test_powl(
-// CHECK-MINGW32-SAME: ptr dead_on_unwind noalias nocapture writable writeonly sret(x86_fp80) align 16 initializes((0, 10)) [[AGG_RESULT:%.*]], ptr nocapture noundef readonly [[TMP0:%.*]], ptr nocapture noundef readonly [[TMP1:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
+// CHECK-MINGW32-SAME: ptr dead_on_unwind noalias writable writeonly sret(x86_fp80) align 16 captures(none) initializes((0, 10)) [[AGG_RESULT:%.*]], ptr noundef readonly captures(none) [[TMP0:%.*]], ptr noundef readonly captures(none) [[TMP1:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
 // CHECK-MINGW32:    [[A:%.*]] = load x86_fp80, ptr [[TMP0]], align 16, !tbaa [[TBAA3:![0-9]+]]
 // CHECK-MINGW32:    [[B:%.*]] = load x86_fp80, ptr [[TMP1]], align 16, !tbaa [[TBAA3]]
 // CHECK-MINGW32:    store x86_fp80 [[A]], ptr [[BYVAL_TEMP:%.*]], align 16, !tbaa [[TBAA3]]
@@ -67,7 +67,7 @@ long double test_powl(long double a, long double b) {
 }
 
 // CHECK-LABEL: define dso_local { x86_fp80, x86_fp80 } @test_cargl(
-// CHECK-SAME: ptr nocapture noundef readonly byval({ x86_fp80, x86_fp80 }) align 16 [[CLD:%.*]]) local_unnamed_addr #[[ATTR2:[0-9]+]] {
+// CHECK-SAME: ptr noundef readonly byval({ x86_fp80, x86_fp80 }) align 16 captures(none) [[CLD:%.*]]) local_unnamed_addr #[[ATTR2:[0-9]+]] {
 // CHECK:    [[CLD_REAL:%.*]] = load x86_fp80, ptr [[CLD]], align 16
 // CHECK:    [[CLD_IMAG:%.*]] = load x86_fp80, ptr [[CLD_IMAGP:%.*]], align 16
 // CHECK:    store x86_fp80 [[CLD_REAL]], ptr [[BYVAL_TEMP:%.*]], align 16
@@ -75,7 +75,7 @@ long double test_powl(long double a, long double b) {
 // CHECK:    [[CALL:%.*]] = tail call x86_fp80 @cargl(ptr noundef nonnull byval({ x86_fp80, x86_fp80 }) align 16 [[BYVAL_TEMP]]) #[[ATTR5]]
 //
 // CHECK-WIN64-LABEL: define dso_local { x86_fp80, x86_fp80 } @test_cargl(
-// CHECK-WIN64-SAME: ptr nocapture noundef readonly byval({ x86_fp80, x86_fp80 }) align 16 [[CLD:%.*]]) local_unnamed_addr #[[ATTR2:[0-9]+]] {
+// CHECK-WIN64-SAME: ptr noundef readonly byval({ x86_fp80, x86_fp80 }) align 16 captures(none) [[CLD:%.*]]) local_unnamed_addr #[[ATTR2:[0-9]+]] {
 // CHECK-WIN64:    [[CLD_REAL:%.*]] = load x86_fp80, ptr [[CLD]], align 16
 // CHECK-WIN64:    [[CLD_IMAG:%.*]] = load x86_fp80, ptr [[CLD_IMAGP:%.*]], align 16
 // CHECK-WIN64:    store x86_fp80 [[CLD_REAL]], ptr [[BYVAL_TEMP:%.*]], align 16
@@ -83,7 +83,7 @@ long double test_powl(long double a, long double b) {
 // CHECK-WIN64:    [[CALL:%.*]] = tail call x86_fp80 @cargl(ptr noundef nonnull byval({ x86_fp80, x86_fp80 }) align 16 [[BYVAL_TEMP]]) #[[ATTR5]]
 //
 // CHECK-I686-LABEL: define dso_local void @test_cargl(
-// CHECK-I686-SAME: ptr dead_on_unwind noalias nocapture writable writeonly sret({ x86_fp80, x86_fp80 }) align 4 initializes((0, 10), (12, 22)) [[AGG_RESULT:%.*]], ptr nocapture noundef readonly byval({ x86_fp80, x86_fp80 }) align 4 [[CLD:%.*]]) local_unnamed_addr #[[ATTR2:[0-9]+]] {
+// CHECK-I686-SAME: ptr dead_on_unwind noalias writable writeonly sret({ x86_fp80, x86_fp80 }) align 4 captures(none) initializes((0, 10), (12, 22)) [[AGG_RESULT:%.*]], ptr noundef readonly byval({ x86_fp80, x86_fp80 }) align 4 captures(none) [[CLD:%.*]]) local_unnamed_addr #[[ATTR2:[0-9]+]] {
 // CHECK-I686:    [[CLD_REAL:%.*]] = load x86_fp80, ptr [[CLD]], align 4
 // CHECK-I686:    [[CLD_IMAG:%.*]] = load x86_fp80, ptr [[CLD_IMAGP:%.*]], align 4
 // CHECK-I686:    store x86_fp80 [[CLD_REAL]], ptr [[BYVAL_TEMP:%.*]], align 4
@@ -93,7 +93,7 @@ long double test_powl(long double a, long double b) {
 // CHECK-I686:    store x86_fp80 [[MUL_IR:%.*]], ptr [[AGG_RESULT_IMAGP:%.*]], align 4
 //
 // CHECK-PPC-LABEL: define dso_local void @test_cargl(
-// CHECK-PPC-SAME: ptr dead_on_unwind noalias nocapture writable writeonly sret({ ppc_fp128, ppc_fp128 }) align 16 initializes((0, 32)) [[AGG_RESULT:%.*]], ptr nocapture noundef readonly byval({ ppc_fp128, ppc_fp128 }) align 16 [[CLD:%.*]]) local_unnamed_addr #[[ATTR1:[0-9]+]] {
+// CHECK-PPC-SAME: ptr dead_on_unwind noalias writable writeonly sret({ ppc_fp128, ppc_fp128 }) align 16 captures(none) initializes((0, 32)) [[AGG_RESULT:%.*]], ptr noundef readonly byval({ ppc_fp128, ppc_fp128 }) align 16 captures(none) [[CLD:%.*]]) local_unnamed_addr #[[ATTR1:[0-9]+]] {
 // CHECK-PPC:    [[CLD_REAL:%.*]] = load ppc_fp128, ptr [[CLD]], align 16
 // CHECK-PPC:    [[CLD_IMAG:%.*]] = load ppc_fp128, ptr [[CLD_IMAGP:%.*]], align 16
 // CHECK-PPC:    store ppc_fp128 [[CLD_REAL]], ptr [[BYVAL_TEMP:%.*]], align 16
@@ -103,7 +103,7 @@ long double test_powl(long double a, long double b) {
 // CHECK-PPC:    store ppc_fp128 [[MUL_IR:%.*]], ptr [[AGG_RESULT_IMAGP:%.*]], align 16
 //
 // CHECK-ARM-LABEL: define dso_local void @test_cargl(
-// CHECK-ARM-SAME: ptr dead_on_unwind noalias nocapture writable writeonly sret({ double, double }) align 8 initializes((0, 16)) [[AGG_RESULT:%.*]], [2 x i64] noundef [[CLD_COERCE:%.*]]) local_unnamed_addr #[[ATTR1:[0-9]+]] {
+// CHECK-ARM-SAME: ptr dead_on_unwind noalias writable writeonly sret({ double, double }) align 8 captures(none) initializes((0, 16)) [[AGG_RESULT:%.*]], [2 x i64] noundef [[CLD_COERCE:%.*]]) local_unnamed_addr #[[ATTR1:[0-9]+]] {
 // CHECK-ARM:    [[CALL:%.*]] = tail call double @cargl([2 x i64] noundef [[CLD_COERCE]]) #[[ATTR2]], !tbaa [[TBAA3]]
 // CHECK-ARM:    store double [[MUL_RL:%.*]], ptr [[AGG_RESULT]], align 8
 // CHECK-ARM:    store double [[MUL_IR:%.*]], ptr [[AGG_RESULT_IMAGP:%.*]], align 8
@@ -121,7 +121,7 @@ long double test_powl(long double a, long double b) {
 // CHECK-AARCH:    [[CALL:%.*]] = tail call fp128 @cargl([2 x fp128] noundef alignstack(16) [[CLD_COERCE]]) #[[ATTR2]], !tbaa [[TBAA2]]
 //
 // CHECK-SPIR-LABEL: define dso_local spir_func void @test_cargl(
-// CHECK-SPIR-SAME: ptr dead_on_unwind noalias nocapture writable writeonly sret({ double, double }) align 8 initializes((0, 16)) [[AGG_RESULT:%.*]], ptr nocapture noundef readonly byval({ double, double }) align 8 [[CLD:%.*]]) local_unnamed_addr #[[ATTR1:[0-9]+]] {
+// CHECK-SPIR-SAME: ptr dead_on_unwind noalias writable writeonly sret({ double, double }) align 8 captures(none) initializes((0, 16)) [[AGG_RESULT:%.*]], ptr noundef readonly byval({ double, double }) align 8 captures(none) [[CLD:%.*]]) local_unnamed_addr #[[ATTR1:[0-9]+]] {
 // CHECK-SPIR:    [[CLD_REAL:%.*]] = load double, ptr [[CLD]], align 8
 // CHECK-SPIR:    [[CLD_IMAG:%.*]] = load double, ptr [[CLD_IMAGP:%.*]], align 8
 // CHECK-SPIR:    store double [[CLD_REAL]], ptr [[BYVAL_TEMP:%.*]], align 8
@@ -131,7 +131,7 @@ long double test_powl(long double a, long double b) {
 // CHECK-SPIR:    store double [[MUL_IR:%.*]], ptr [[AGG_RESULT_IMAGP:%.*]], align 8
 //
 // CHECK-MINGW32-LABEL: define dso_local void @test_cargl(
-// CHECK-MINGW32-SAME: ptr dead_on_unwind noalias nocapture writable writeonly sret({ x86_fp80, x86_fp80 }) align 16 initializes((0, 10), (16, 26)) [[AGG_RESULT:%.*]], ptr nocapture noundef readonly [[CLD:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-MINGW32-SAME: ptr dead_on_unwind noalias writable writeonly sret({ x86_fp80, x86_fp80 }) align 16 captures(none) initializes((0, 10), (16, 26)) [[AGG_RESULT:%.*]], ptr noundef readonly captures(none) [[CLD:%.*]]) local_unnamed_addr #[[ATTR0]] {
 // CHECK-MINGW32:    [[CLD_REAL:%.*]] = load x86_fp80, ptr [[CLD]], align 16
 // CHECK-MINGW32:    [[CLD_IMAG:%.*]] = load x86_fp80, ptr [[CLD_IMAGP:%.*]], align 16
 // CHECK-MINGW32:    store x86_fp80 [[CLD_REAL]], ptr [[BYVAL_TEMP:%.*]], align 16
@@ -189,7 +189,7 @@ int ilogbl(long double a);
 // CHECK-SPIR:    [[CALL:%.*]] = tail call spir_func i32 @ilogbl(double noundef [[A]]) #[[ATTR3]], !tbaa [[TBAA2]]
 //
 // CHECK-MINGW32-LABEL: define dso_local i32 @test_ilogb(
-// CHECK-MINGW32-SAME: ptr nocapture noundef readonly [[TMP0:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-MINGW32-SAME: ptr noundef readonly captures(none) [[TMP0:%.*]]) local_unnamed_addr #[[ATTR0]] {
 // CHECK-MINGW32:    [[A:%.*]] = load x86_fp80, ptr [[TMP0]], align 16, !tbaa [[TBAA3]]
 // CHECK-MINGW32:    store x86_fp80 [[A]], ptr [[BYVAL_TEMP:%.*]], align 16, !tbaa [[TBAA3]]
 // CHECK-MINGW32:    [[CALL:%.*]] = call i32 @ilogbl(ptr noundef nonnull [[BYVAL_TEMP]]) #[[ATTR3]]
diff --git a/clang/test/CodeGen/math-libcalls-tbaa.c b/clang/test/CodeGen/math-libcalls-tbaa.c
index 17a93ed2aed2d..f4e81ea6dff17 100644
--- a/clang/test/CodeGen/math-libcalls-tbaa.c
+++ b/clang/test/CodeGen/math-libcalls-tbaa.c
@@ -14,7 +14,7 @@ float crealf(float _Complex);
 // Emit int TBAA metadata on FP math libcalls, which is useful for alias analysis
 
 // CHECK-LABEL: define dso_local float @test_expf(
-// CHECK-SAME: ptr nocapture noundef readonly [[NUM:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
+// CHECK-SAME: ptr noundef readonly captures(none) [[NUM:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
 // CHECK-NEXT:  [[ENTRY:.*:]]
 // CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i8, ptr [[NUM]], i64 40
 // CHECK-NEXT:    [[TMP0:%.*]] = load float, ptr [[ARRAYIDX]], align 4, !tbaa [[TBAA2:![0-9]+]]
@@ -29,7 +29,7 @@ float test_expf (float num[]) {
 }
 
 // CHECK-LABEL: define dso_local float @test_builtin_expf(
-// CHECK-SAME: ptr nocapture noundef readonly [[NUM:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-SAME: ptr noundef readonly captures(none) [[NUM:%.*]]) local_unnamed_addr #[[ATTR0]] {
 // CHECK-NEXT:  [[ENTRY:.*:]]
 // CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i8, ptr [[NUM]], i64 40
 // CHECK-NEXT:    [[TMP0:%.*]] = load float, ptr [[ARRAYIDX]], align 4, !tbaa [[TBAA2]]
@@ -46,7 +46,7 @@ float test_builtin_expf (float num[]) {
 //
 // Negative test: fabs cannot set errno
 // CHECK-LABEL: define dso_local double @test_fabs(
-// CHECK-SAME: ptr nocapture noundef readonly [[NUM:%.*]]) local_unnamed_addr #[[ATTR3:[0-9]+]] {
+// CHECK-SAME: ptr noundef readonly captures(none) [[NUM:%.*]]) local_unnamed_addr #[[ATTR3:[0-9]+]] {
 // CHECK-NEXT:  [[ENTRY:.*:]]
 // CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i8, ptr [[NUM]], i64 80
 // CHECK-NEXT:    [[TMP0:%.*]] = load double, ptr [[ARRAYIDX]], align 8, !tbaa [[TBAA8:![0-9]+]]
@@ -61,7 +61,7 @@ double test_fabs (double num[]) {
 }
 
 // CHECK-LABEL: define dso_local double @test_remainder(
-// CHECK-SAME: ptr nocapture noundef readonly [[NUM:%.*]], double noundef [[A:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-SAME: ptr noundef readonly captures(none) [[NUM:%.*]], double noundef [[A:%.*]]) local_unnamed_addr #[[ATTR0]] {
 // CHECK-NEXT:  [[ENTRY:.*:]]
 // CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i8, ptr [[NUM]], i64 80
 // CHECK-NEXT:    [[TMP0:%.*]] = load double, ptr [[ARRAYIDX]], align 8, !tbaa [[TBAA8]]
@@ -79,7 +79,7 @@ double test_remainder (double num[], double a) {
 // TODO: frexp is not subject to any errors, but also writes to
 // its int pointer out argument, so it could emit int TBAA metadata.
 // CHECK-LABEL: define dso_local double @test_frexp(
-// CHECK-SAME: ptr nocapture noundef readonly [[NUM:%.*]]) local_unnamed_addr #[[ATTR5:[0-9]+]] {
+// CHECK-SAME: ptr noundef readonly captures(none) [[NUM:%.*]]) local_unnamed_addr #[[ATTR5:[0-9]+]] {
 // CHECK-NEXT:  [[ENTRY:.*:]]
 // CHECK-NEXT:    [[E:%.*]] = alloca i32, align 4
 // CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr nonnull [[E]]) #[[ATTR9]]
@@ -101,7 +101,7 @@ double test_frexp (double num[]) {
 // Negative test: sincos is a library function, but is not a builtin function
 // checked in CodeGenFunction::EmitCallExpr.
 // CHECK-LABEL: define dso_local float @test_sincos(
-// CHECK-SAME: ptr nocapture noundef readonly [[NUM:%.*]]) local_unnamed_addr #[[ATTR7:[0-9]+]] {
+// CHECK-SAME: ptr noundef readonly captures(none) [[NUM:%.*]]) local_unnamed_addr #[[ATTR7:[0-9]+]] {
 // CHECK-NEXT:  [[ENTRY:.*:]]
 // CHECK-NEXT:    [[SIN:%.*]] = alloca float, align 4
 // CHECK-NEXT:    [[COS:%.*]] = alloca float, align 4
@@ -128,7 +128,7 @@ float test_sincos (float num[]) {
 
 // TODO: The builtin return a complex type
 // CHECK-LABEL: define dso_local float @test_cacoshf(
-// CHECK-SAME: ptr nocapture noundef readonly [[NUM:%.*]]) local_unnamed_addr #[[ATTR7]] {
+// CHECK-SAME: ptr noundef readonly captures(none) [[NUM:%.*]]) local_unnamed_addr #[[ATTR7]] {
 // CHECK-NEXT:  [[ENTRY:.*:]]
 // CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i8, ptr [[NUM]], i64 8
 // CHECK-NEXT:    [[TMP0:%.*]] = load float, ptr [[ARRAYIDX]], align 4, !tbaa [[TBAA2]]
diff --git a/clang/test/CodeGen/nofpclass.c b/clang/test/CodeGen/nofpclass.c
index 75aa0318421de..8e61ae22ef87f 100644
--- a/clang/test/CodeGen/nofpclass.c
+++ b/clang/test/CodeGen/nofpclass.c
@@ -920,7 +920,7 @@ _Complex _Float16 defined_complex_func_f16_ret(_Complex _Float16 c) {
 //
 // CLFINITEONLY: Function Attrs: convergent norecurse nounwind
 // CLFINITEONLY-LABEL: define dso_local nofpclass(nan inf) float @call_variadic
-// CLFINITEONLY-SAME: (float noundef nofpclass(nan inf) [[F32:%.*]], double noundef nofpclass(nan inf) [[F64:%.*]], half noundef nofpclass(nan inf) [[F16:%.*]], double noundef nofpclass(nan inf) [[V2F32_COERCE:%.*]], <2 x double> noundef nofpclass(nan inf) [[V2F64:%.*]], i32 noundef [[V2F16_COERCE:%.*]], <2 x float> noundef nofpclass(nan inf) [[CF32_COERCE:%.*]], double noundef nofpclass(nan inf) [[CF64_COERCE0:%.*]], double noundef nofpclass(nan inf) [[CF64_COERCE1:%.*]], ptr nocapture noundef readonly byval({ half, half }) align 8 [[CF16:%.*]]) local_unnamed_addr #[[ATTR5]] {
+// CLFINITEONLY-SAME: (float noundef nofpclass(nan inf) [[F32:%.*]], double noundef nofpclass(nan inf) [[F64:%.*]], half noundef nofpclass(nan inf) [[F16:%.*]], double noundef nofpclass(nan inf) [[V2F32_COERCE:%.*]], <2 x double> noundef nofpclass(nan inf) [[V2F64:%.*]], i32 noundef [[V2F16_COERCE:%.*]], <2 x float> noundef nofpclass(nan inf) [[CF32_COERCE:%.*]], double noundef nofpclass(nan inf) [[CF64_COERCE0:%.*]], double noundef nofpclass(nan inf) [[CF64_COERCE1:%.*]], ptr noundef readonly byval({ half, half }) align 8 captures(none) [[CF16:%.*]]) local_unnamed_addr #[[ATTR5]] {
 // CLFINITEONLY-NEXT:  entry:
 // CLFINITEONLY-NEXT:    [[BYVAL_TEMP:%.*]] = alloca { double, double }, align 8
 // CLFINITEONLY-NEXT:    [[CONV:%.*]] = fpext nnan ninf float [[F32]] to double
@@ -1173,7 +1173,7 @@ float call_variadic(float f32, double f64, _Float16 f16,
 //
 // CLFINITEONLY: Function Attrs: convergent norecurse nounwind
 // CLFINITEONLY-LABEL: define dso_local nofpclass(nan inf) float @call_variadic_indirect
-// CLFINITEONLY-SAME: (ptr nocapture noundef readonly [[FPTR:%.*]], float noundef nofpclass(nan inf) [[F32:%.*]], double noundef nofpclass(nan inf) [[F64:%.*]], half noundef nofpclass(nan inf) [[F16:%.*]], double noundef nofpclass(nan inf) [[V2F32_COERCE:%.*]], <2 x double> noundef nofpclass(nan inf) [[V2F64:%.*]], i32 noundef [[V2F16_COERCE:%.*]], <2 x float> noundef nofpclass(nan inf) [[CF32_COERCE:%.*]], double noundef nofpclass(nan inf) [[CF64_COERCE0:%.*]], double noundef nofpclass(nan inf) [[CF64_COERCE1:%.*]], ptr nocapture noundef readonly byval({ half, half }) align 8 [[CF16:%.*]]) local_unnamed_addr #[[ATTR5]] {
+// CLFINITEONLY-SAME: (ptr noundef readonly captures(none) [[FPTR:%.*]], float noundef nofpclass(nan inf) [[F32:%.*]], double noundef nofpclass(nan inf) [[F64:%.*]], half noundef nofpclass(nan inf) [[F16:%.*]], double noundef nofpclass(nan inf) [[V2F32_COERCE:%.*]], <2 x double> noundef nofpclass(nan inf) [[V2F64:%.*]], i32 noundef [[V2F16_COERCE:%.*]], <2 x float> noundef nofpclass(nan inf) [[CF32_COERCE:%.*]], double noundef nofpclass(nan inf) [[CF64_COERCE0:%.*]], double noundef nofpclass(nan inf) [[CF64_COERCE1:%.*]], ptr noundef readonly byval({ half, half }) align 8 captures(none) [[CF16:%.*]]) local_unnamed_addr #[[ATTR5]] {
 // CLFINITEONLY-NEXT:  entry:
 // CLFINITEONLY-NEXT:    [[BYVAL_TEMP:%.*]] = alloca { double, double }, align 8
 // CLFINITEONLY-NEXT:    [[CONV:%.*]] = fpext nnan ninf float [[F32]] to double
diff --git a/clang/test/CodeGen/pointer-overflow.c b/clang/test/CodeGen/pointer-overflow.c
new file mode 100644
index 0000000000000..9c7821b841980
--- /dev/null
+++ b/clang/test/CodeGen/pointer-overflow.c
@@ -0,0 +1,12 @@
+// RUN: %clang_cc1 -triple x86_64-apple-darwin %s -emit-llvm -o - | FileCheck %s --check-prefix=DEFAULT
+// RUN: %clang_cc1 -triple x86_64-apple-darwin %s -emit-llvm -o - -fwrapv | FileCheck %s --check-prefix=DEFAULT
+// RUN: %clang_cc1 -triple x86_64-apple-darwin %s -emit-llvm -o - -ftrapv | FileCheck %s --check-prefix=DEFAULT
+// RUN: %clang_cc1 -triple x86_64-apple-darwin %s -emit-llvm -o - -fwrapv-pointer | FileCheck %s --check-prefix=FWRAPV-POINTER
+
+void test(void) {
+  // -fwrapv-pointer should turn off inbounds for GEP's
+  extern int* P;
+  ++P;
+  // DEFAULT: getelementptr inbounds nuw i32, ptr
+  // FWRAPV-POINTER: getelementptr i32, ptr
+}
diff --git a/clang/test/CodeGen/profile-continuous.c b/clang/test/CodeGen/profile-continuous.c
new file mode 100644
index 0000000000000..86fa1d149b971
--- /dev/null
+++ b/clang/test/CodeGen/profile-continuous.c
@@ -0,0 +1,11 @@
+// RUN: %clang_cc1 -emit-llvm -fprofile-instrument=llvm -fprofile-continuous %s -o - | FileCheck %s --check-prefix=IRPGO
+// RUN: %clang_cc1 -emit-llvm -fprofile-instrument=llvm -fprofile-continuous -fprofile-instrument-path=mydir/default_%m.profraw -mllvm -runtime-counter-relocation %s -o - \
+// RUN:  | FileCheck %s --check-prefix=IRPGO_EQ
+// RUN: %clang_cc1 -emit-llvm -O2 -fprofile-instrument=csllvm -fprofile-continuous %s -o - | FileCheck %s --check-prefix=CSIRPGO
+// RUN: %clang_cc1 -emit-llvm -fprofile-instrument=clang -fprofile-continuous -fprofile-instrument-path=default.profraw %s -o - | FileCheck %s --check-prefix=CLANG_PGO
+
+// IRPGO: @__llvm_profile_filename = {{.*}} c"%cdefault_%m.profraw\00"
+// IRPGO_EQ: @__llvm_profile_filename = {{.*}} c"%cmydir/default_%m.profraw\00"
+// CSIRPGO: @__llvm_profile_filename = {{.*}} c"%cdefault_%m.profraw\00"
+// CLANG_PGO: @__llvm_profile_filename = {{.*}} c"%cdefault.profraw\00"
+void foo(){}
diff --git a/clang/test/CodeGen/sanitize-metadata-nosanitize.c b/clang/test/CodeGen/sanitize-metadata-nosanitize.c
index fd2fdce31b52f..eabcbd1409fe2 100644
--- a/clang/test/CodeGen/sanitize-metadata-nosanitize.c
+++ b/clang/test/CodeGen/sanitize-metadata-nosanitize.c
@@ -23,7 +23,7 @@ __attribute__((noinline, not_tail_called)) void escape(const volatile void *p) {
 
 // CHECK: Function Attrs: mustprogress nofree norecurse nounwind willreturn memory(write, argmem: readwrite, inaccessiblemem: none)
 // CHECK-LABEL: define dso_local i32 @normal_function
-// CHECK-SAME: (ptr noundef [[X:%.*]], ptr nocapture noundef readonly [[Y:%.*]]) local_unnamed_addr #[[ATTR1:[0-9]+]] !pcsections [[META4:![0-9]+]] {
+// CHECK-SAME: (ptr noundef [[X:%.*]], ptr noundef readonly captures(none) [[Y:%.*]]) local_unnamed_addr #[[ATTR1:[0-9]+]] !pcsections [[META4:![0-9]+]] {
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[X_ADDR:%.*]] = alloca ptr, align 8
 // CHECK-NEXT:    store ptr [[X]], ptr [[X_ADDR]], align 8, !tbaa [[TBAA6:![0-9]+]]
@@ -40,7 +40,7 @@ int normal_function(int *x, int *y) {
 
 // CHECK: Function Attrs: disable_sanitizer_instrumentation mustprogress nofree norecurse nounwind willreturn memory(write, argmem: readwrite, inaccessiblemem: none)
 // CHECK-LABEL: define dso_local i32 @test_disable_sanitize_instrumentation
-// CHECK-SAME: (ptr noundef [[X:%.*]], ptr nocapture noundef readonly [[Y:%.*]]) local_unnamed_addr #[[ATTR2:[0-9]+]] {
+// CHECK-SAME: (ptr noundef [[X:%.*]], ptr noundef readonly captures(none) [[Y:%.*]]) local_unnamed_addr #[[ATTR2:[0-9]+]] {
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[X_ADDR:%.*]] = alloca ptr, align 8
 // CHECK-NEXT:    store ptr [[X]], ptr [[X_ADDR]], align 8, !tbaa [[TBAA6]]
@@ -57,7 +57,7 @@ __attribute__((disable_sanitizer_instrumentation)) int test_disable_sanitize_ins
 
 // CHECK: Function Attrs: mustprogress nofree norecurse nounwind willreturn memory(write, argmem: readwrite, inaccessiblemem: none)
 // CHECK-LABEL: define dso_local i32 @test_no_sanitize_thread
-// CHECK-SAME: (ptr noundef [[X:%.*]], ptr nocapture noundef readonly [[Y:%.*]]) local_unnamed_addr #[[ATTR3:[0-9]+]] !pcsections [[META14:![0-9]+]] {
+// CHECK-SAME: (ptr noundef [[X:%.*]], ptr noundef readonly captures(none) [[Y:%.*]]) local_unnamed_addr #[[ATTR3:[0-9]+]] !pcsections [[META14:![0-9]+]] {
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[X_ADDR:%.*]] = alloca ptr, align 8
 // CHECK-NEXT:    store ptr [[X]], ptr [[X_ADDR]], align 8, !tbaa [[TBAA6]]
@@ -74,7 +74,7 @@ __attribute__((no_sanitize("thread"))) int test_no_sanitize_thread(int *x, int *
 
 // CHECK: Function Attrs: mustprogress nofree norecurse nounwind willreturn memory(write, argmem: readwrite, inaccessiblemem: none)
 // CHECK-LABEL: define dso_local i32 @test_no_sanitize_all
-// CHECK-SAME: (ptr noundef [[X:%.*]], ptr nocapture noundef readonly [[Y:%.*]]) local_unnamed_addr #[[ATTR3]] !pcsections [[META14]] {
+// CHECK-SAME: (ptr noundef [[X:%.*]], ptr noundef readonly captures(none) [[Y:%.*]]) local_unnamed_addr #[[ATTR3]] !pcsections [[META14]] {
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[X_ADDR:%.*]] = alloca ptr, align 8
 // CHECK-NEXT:    store ptr [[X]], ptr [[X_ADDR]], align 8, !tbaa [[TBAA6]]
diff --git a/clang/test/CodeGen/struct-copy.c b/clang/test/CodeGen/struct-copy.c
index bb4166c9464ff..299f98833a309 100644
--- a/clang/test/CodeGen/struct-copy.c
+++ b/clang/test/CodeGen/struct-copy.c
@@ -8,7 +8,7 @@ void foo(struct x *P, struct x *Q) {
   *P = *Q;
 }
 
-// CHECK: declare void @llvm.memcpy.p0.p0{{.*}}(ptr noalias nocapture writeonly, ptr noalias nocapture readonly
+// CHECK: declare void @llvm.memcpy.p0.p0{{.*}}(ptr noalias writeonly captures(none), ptr noalias readonly
 
 void bar(struct x *P, struct x *Q) {
 // CHECK-LABEL: @bar(
diff --git a/clang/test/CodeGen/target-data.c b/clang/test/CodeGen/target-data.c
index 71eb849433ed4..fe29aadb1dd53 100644
--- a/clang/test/CodeGen/target-data.c
+++ b/clang/test/CodeGen/target-data.c
@@ -160,11 +160,11 @@
 
 // RUN: %clang_cc1 -triple nvptx-unknown -o - -emit-llvm %s | \
 // RUN: FileCheck %s -check-prefix=NVPTX
-// NVPTX: target datalayout = "e-p:32:32-i64:64-i128:128-v16:16-v32:32-n16:32:64"
+// NVPTX: target datalayout = "e-p:32:32-p6:32:32-i64:64-i128:128-v16:16-v32:32-n16:32:64"
 
 // RUN: %clang_cc1 -triple nvptx64-unknown -o - -emit-llvm %s | \
 // RUN: FileCheck %s -check-prefix=NVPTX64
-// NVPTX64: target datalayout = "e-i64:64-i128:128-v16:16-v32:32-n16:32:64"
+// NVPTX64: target datalayout = "e-p6:32:32-i64:64-i128:128-v16:16-v32:32-n16:32:64"
 
 // RUN: %clang_cc1 -triple r600-unknown -o - -emit-llvm %s | \
 // RUN: FileCheck %s -check-prefix=R600
diff --git a/clang/test/CodeGen/tbaa-pointers.c b/clang/test/CodeGen/tbaa-pointers.c
index 4aae2552f107a..48adac503357f 100644
--- a/clang/test/CodeGen/tbaa-pointers.c
+++ b/clang/test/CodeGen/tbaa-pointers.c
@@ -208,12 +208,9 @@ int void_ptrs(void **ptr) {
 // COMMON-LABEL: define i32 @void_ptrs(
 // COMMON-SAME: ptr noundef [[PTRA:%.+]])
 // COMMON:        [[PTR_ADDR:%.+]]  = alloca ptr, align 8
-// DISABLE-NEXT:  store ptr [[PTRA]], ptr [[PTR_ADDR]], align 8, !tbaa [[ANYPTR]]
-// DISABLE-NEXT:  [[L0:%.+]] = load ptr, ptr  [[PTR_ADDR]], align 8, !tbaa  [[ANYPTR]]
-// DISABLE-NEXT:  [[L1:%.+]] = load ptr, ptr [[L0]], align 8, !tbaa  [[ANYPTR]]
-// DEFAULT-NEXT:  store ptr [[PTRA]], ptr [[PTR_ADDR]], align 8, !tbaa [[P2VOID:!.+]]
-// DEFAULT-NEXT:  [[L0:%.+]] = load ptr, ptr  [[PTR_ADDR]], align 8, !tbaa  [[P2VOID]]
-// DEFAULT-NEXT:  [[L1:%.+]] = load ptr, ptr [[L0]], align 8, !tbaa  [[P1VOID:!.+]]
+// COMMON-NEXT:   store ptr [[PTRA]], ptr [[PTR_ADDR]], align 8, !tbaa [[ANYPTR]]
+// COMMON-NEXT:   [[L0:%.+]] = load ptr, ptr  [[PTR_ADDR]], align 8, !tbaa  [[ANYPTR]]
+// COMMON-NEXT:   [[L1:%.+]] = load ptr, ptr [[L0]], align 8, !tbaa  [[ANYPTR]]
 // COMMON-NEXT:   [[BOOL:%.+]] = icmp ne ptr [[L1]], null
 // COMMON-NEXT:   [[BOOL_EXT:%.+]] = zext i1 [[BOOL]] to i64
 // COMMON-NEXT:   [[COND:%.+]] = select i1 [[BOOL]], i32 0, i32 1
@@ -254,7 +251,3 @@ int void_ptrs(void **ptr) {
 // COMMON:  [[INT_TAG]] = !{[[INT_TY:!.+]], [[INT_TY]], i64 0}
 // COMMON:  [[INT_TY]] = !{!"int", [[CHAR]], i64 0}
 // DEFAULT: [[ANYPTR]] = !{[[ANY_POINTER]],  [[ANY_POINTER]], i64 0}
-// DEFAULT: [[P2VOID]] = !{[[P2VOID_TY:!.+]], [[P2VOID_TY]], i64 0}
-// DEFAULT: [[P2VOID_TY]] = !{!"p2 void", [[ANY_POINTER]], i64 0}
-// DEFAULT: [[P1VOID]] = !{[[P1VOID_TY:!.+]], [[P1VOID_TY]], i64 0}
-// DEFAULT: [[P1VOID_TY]] = !{!"p1 void", [[ANY_POINTER]], i64 0}
diff --git a/clang/test/CodeGen/tbaa-struct-bitfield-endianness.cpp b/clang/test/CodeGen/tbaa-struct-bitfield-endianness.cpp
index 7173b6e8fbe2a..1177691ca511e 100644
--- a/clang/test/CodeGen/tbaa-struct-bitfield-endianness.cpp
+++ b/clang/test/CodeGen/tbaa-struct-bitfield-endianness.cpp
@@ -16,7 +16,7 @@ struct NamedBitfields {
 };
 
 // CHECK-LABEL: _Z4copyP14NamedBitfieldsS0_
-// CHECK-SAME: ptr nocapture noundef writeonly initializes((0, 16)) [[A1:%.*]], ptr nocapture noundef readonly [[A2:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
+// CHECK-SAME: ptr noundef writeonly captures(none) initializes((0, 16)) [[A1:%.*]], ptr noundef readonly captures(none) [[A2:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    tail call void @llvm.memcpy.p0.p0.i64(ptr noundef nonnull align 8 dereferenceable(16) [[A1]], ptr noundef nonnull align 8 dereferenceable(16) [[A2]], i64 16, i1 false), !tbaa.struct [[TBAA_STRUCT2:![0-9]+]]
 // CHECK-NEXT:    ret void
diff --git a/clang/test/CodeGen/transparent-union-type.c b/clang/test/CodeGen/transparent-union-type.c
index f7fac25dc0984..5c81b8a02477f 100644
--- a/clang/test/CodeGen/transparent-union-type.c
+++ b/clang/test/CodeGen/transparent-union-type.c
@@ -53,7 +53,7 @@ void ftest0(tu_c_t uc) { }
 void ftest1(tu_s_t uc) { }
 
 // CHECK-LABEL: define{{.*}} void @ftest1b(
-// CHECK-SAME: ptr nocapture noundef readnone [[UC:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-SAME: ptr noundef readnone captures(none) [[UC:%.*]]) local_unnamed_addr #[[ATTR0]] {
 // CHECK-NEXT:  [[ENTRY:.*:]]
 // CHECK-NEXT:    ret void
 //
@@ -94,14 +94,14 @@ typedef union tu_ptr {
 } tu_ptr_t __attribute__((transparent_union));
 
 // CHECK-LABEL: define{{.*}} void @ftest5(
-// CHECK-SAME: ptr nocapture readnone [[UC_COERCE:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
+// CHECK-SAME: ptr readnone captures(none) [[UC_COERCE:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
 // CHECK-NEXT:  [[ENTRY:.*:]]
 // CHECK-NEXT:    ret void
 //
 void ftest5(tu_ptr_t uc) { }
 
 // CHECK-LABEL: define{{.*}} void @ftest6(
-// CHECK-SAME: ptr nocapture noundef readnone [[UC:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-SAME: ptr noundef readnone captures(none) [[UC:%.*]]) local_unnamed_addr #[[ATTR0]] {
 // CHECK-NEXT:  [[ENTRY:.*:]]
 // CHECK-NEXT:    ret void
 //
diff --git a/clang/test/CodeGen/union-tbaa1.c b/clang/test/CodeGen/union-tbaa1.c
index 0f7a67cb7eccd..1e2f384c29a5b 100644
--- a/clang/test/CodeGen/union-tbaa1.c
+++ b/clang/test/CodeGen/union-tbaa1.c
@@ -8,7 +8,7 @@ typedef union __attribute__((aligned(4))) {
 void bar(vect32 p[][2]);
 
 // CHECK-LABEL: define dso_local void @fred
-// CHECK-SAME: (i32 noundef [[NUM:%.*]], ptr nocapture noundef writeonly initializes((0, 8)) [[VEC:%.*]], ptr nocapture noundef readonly [[INDEX:%.*]], ptr nocapture noundef readonly [[ARR:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
+// CHECK-SAME: (i32 noundef [[NUM:%.*]], ptr noundef writeonly captures(none) initializes((0, 8)) [[VEC:%.*]], ptr noundef readonly captures(none) [[INDEX:%.*]], ptr noundef readonly captures(none) [[ARR:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP:%.*]] = alloca [4 x [2 x %union.vect32]], align 8
 // CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 32, ptr nonnull [[TMP]]) #[[ATTR3:[0-9]+]]
diff --git a/clang/test/CodeGen/xfail-alloc-align-fn-pointers.cpp b/clang/test/CodeGen/xfail-alloc-align-fn-pointers.cpp
new file mode 100644
index 0000000000000..80067500284b1
--- /dev/null
+++ b/clang/test/CodeGen/xfail-alloc-align-fn-pointers.cpp
@@ -0,0 +1,10 @@
+
+// RUN: %clang_cc1 %s
+
+// FIXME: These should not crash!
+// XFAIL: *
+
+void aa_fn_ptr(char* (*member)(char*)  __attribute__((alloc_align(1))));
+
+struct Test;
+void aa_member_fn_ptr(char* (Test::*member)(char*)  __attribute__((alloc_align(1))));
diff --git a/clang/test/CodeGenCUDA/amdgpu-kernel-arg-pointer-type.cu b/clang/test/CodeGenCUDA/amdgpu-kernel-arg-pointer-type.cu
index 19730e3925515..a48affaec3c8a 100644
--- a/clang/test/CodeGenCUDA/amdgpu-kernel-arg-pointer-type.cu
+++ b/clang/test/CodeGenCUDA/amdgpu-kernel-arg-pointer-type.cu
@@ -50,7 +50,7 @@
 // CHECK-SPIRV-NEXT:    ret void
 //
 // OPT-LABEL: define dso_local amdgpu_kernel void @_Z7kernel1Pi(
-// OPT-SAME: ptr addrspace(1) nocapture noundef [[X_COERCE:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
+// OPT-SAME: ptr addrspace(1) noundef captures(none) [[X_COERCE:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
 // OPT-NEXT:  [[ENTRY:.*:]]
 // OPT-NEXT:    [[TMP0:%.*]] = load i32, ptr addrspace(1) [[X_COERCE]], align 4
 // OPT-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP0]], 1
@@ -118,7 +118,7 @@ __global__ void kernel1(int *x) {
 // CHECK-SPIRV-NEXT:    ret void
 //
 // OPT-LABEL: define dso_local amdgpu_kernel void @_Z7kernel2Ri(
-// OPT-SAME: ptr addrspace(1) nocapture noundef nonnull align 4 dereferenceable(4) [[X_COERCE:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// OPT-SAME: ptr addrspace(1) noundef nonnull align 4 captures(none) dereferenceable(4) [[X_COERCE:%.*]]) local_unnamed_addr #[[ATTR0]] {
 // OPT-NEXT:  [[ENTRY:.*:]]
 // OPT-NEXT:    [[TMP0:%.*]] = load i32, ptr addrspace(1) [[X_COERCE]], align 4
 // OPT-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP0]], 1
@@ -188,14 +188,14 @@ __global__ void kernel2(int &x) {
 // CHECK-SPIRV-NEXT:    ret void
 //
 // OPT-LABEL: define dso_local amdgpu_kernel void @_Z7kernel3PU3AS2iPU3AS1i(
-// OPT-SAME: ptr addrspace(2) nocapture noundef readonly [[X:%.*]], ptr addrspace(1) nocapture noundef writeonly initializes((0, 4)) [[Y:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// OPT-SAME: ptr addrspace(2) noundef readonly captures(none) [[X:%.*]], ptr addrspace(1) noundef writeonly captures(none) initializes((0, 4)) [[Y:%.*]]) local_unnamed_addr #[[ATTR0]] {
 // OPT-NEXT:  [[ENTRY:.*:]]
 // OPT-NEXT:    [[TMP0:%.*]] = load i32, ptr addrspace(2) [[X]], align 4
 // OPT-NEXT:    store i32 [[TMP0]], ptr addrspace(1) [[Y]], align 4
 // OPT-NEXT:    ret void
 //
 // OPT-SPIRV-LABEL: define spir_kernel void @_Z7kernel3PU3AS2iPU3AS1i(
-// OPT-SPIRV-SAME: ptr addrspace(2) nocapture noundef readonly [[X:%.*]], ptr addrspace(1) nocapture noundef writeonly initializes((0, 4)) [[Y:%.*]]) local_unnamed_addr addrspace(4) #[[ATTR1:[0-9]+]] !max_work_group_size [[META5]] {
+// OPT-SPIRV-SAME: ptr addrspace(2) noundef readonly captures(none) [[X:%.*]], ptr addrspace(1) noundef writeonly captures(none) initializes((0, 4)) [[Y:%.*]]) local_unnamed_addr addrspace(4) #[[ATTR1:[0-9]+]] !max_work_group_size [[META5]] {
 // OPT-SPIRV-NEXT:  [[ENTRY:.*:]]
 // OPT-SPIRV-NEXT:    [[TMP0:%.*]] = load i32, ptr addrspace(2) [[X]], align 4
 // OPT-SPIRV-NEXT:    store i32 [[TMP0]], ptr addrspace(1) [[Y]], align 4
@@ -253,7 +253,7 @@ __global__ void kernel3(__attribute__((address_space(2))) int *x,
 // CHECK-SPIRV-NEXT:    ret void
 //
 // OPT-LABEL: define dso_local void @_Z4funcPi(
-// OPT-SAME: ptr nocapture noundef [[X:%.*]]) local_unnamed_addr #[[ATTR1:[0-9]+]] {
+// OPT-SAME: ptr noundef captures(none) [[X:%.*]]) local_unnamed_addr #[[ATTR1:[0-9]+]] {
 // OPT-NEXT:  [[ENTRY:.*:]]
 // OPT-NEXT:    [[TMP0:%.*]] = load i32, ptr [[X]], align 4
 // OPT-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP0]], 1
@@ -261,7 +261,7 @@ __global__ void kernel3(__attribute__((address_space(2))) int *x,
 // OPT-NEXT:    ret void
 //
 // OPT-SPIRV-LABEL: define spir_func void @_Z4funcPi(
-// OPT-SPIRV-SAME: ptr addrspace(4) nocapture noundef [[X:%.*]]) local_unnamed_addr addrspace(4) #[[ATTR2:[0-9]+]] {
+// OPT-SPIRV-SAME: ptr addrspace(4) noundef captures(none) [[X:%.*]]) local_unnamed_addr addrspace(4) #[[ATTR2:[0-9]+]] {
 // OPT-SPIRV-NEXT:  [[ENTRY:.*:]]
 // OPT-SPIRV-NEXT:    [[TMP0:%.*]] = load i32, ptr addrspace(4) [[X]], align 4
 // OPT-SPIRV-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP0]], 1
@@ -327,7 +327,7 @@ struct S {
 // CHECK-SPIRV-NEXT:    ret void
 //
 // OPT-LABEL: define dso_local amdgpu_kernel void @_Z7kernel41S(
-// OPT-SAME: ptr addrspace(4) nocapture noundef readonly byref([[STRUCT_S:%.*]]) align 8 [[TMP0:%.*]]) local_unnamed_addr #[[ATTR2:[0-9]+]] {
+// OPT-SAME: ptr addrspace(4) noundef readonly byref([[STRUCT_S:%.*]]) align 8 captures(none) [[TMP0:%.*]]) local_unnamed_addr #[[ATTR2:[0-9]+]] {
 // OPT-NEXT:  [[ENTRY:.*:]]
 // OPT-NEXT:    [[COERCE_SROA_0_0_COPYLOAD:%.*]] = load ptr, ptr addrspace(4) [[TMP0]], align 8, !amdgpu.noclobber [[META4:![0-9]+]]
 // OPT-NEXT:    [[TMP1:%.*]] = addrspacecast ptr [[COERCE_SROA_0_0_COPYLOAD]] to ptr addrspace(1)
@@ -432,7 +432,7 @@ __global__ void kernel4(struct S s) {
 // CHECK-SPIRV-NEXT:    ret void
 //
 // OPT-LABEL: define dso_local amdgpu_kernel void @_Z7kernel5P1S(
-// OPT-SAME: ptr addrspace(1) nocapture noundef readonly [[S_COERCE:%.*]]) local_unnamed_addr #[[ATTR2]] {
+// OPT-SAME: ptr addrspace(1) noundef readonly captures(none) [[S_COERCE:%.*]]) local_unnamed_addr #[[ATTR2]] {
 // OPT-NEXT:  [[ENTRY:.*:]]
 // OPT-NEXT:    [[TMP0:%.*]] = load ptr, ptr addrspace(1) [[S_COERCE]], align 8
 // OPT-NEXT:    [[TMP1:%.*]] = load i32, ptr [[TMP0]], align 4
@@ -535,7 +535,7 @@ struct T {
 // CHECK-SPIRV-NEXT:    ret void
 //
 // OPT-LABEL: define dso_local amdgpu_kernel void @_Z7kernel61T(
-// OPT-SAME: ptr addrspace(4) nocapture noundef readonly byref([[STRUCT_T:%.*]]) align 8 [[TMP0:%.*]]) local_unnamed_addr #[[ATTR2]] {
+// OPT-SAME: ptr addrspace(4) noundef readonly byref([[STRUCT_T:%.*]]) align 8 captures(none) [[TMP0:%.*]]) local_unnamed_addr #[[ATTR2]] {
 // OPT-NEXT:  [[ENTRY:.*:]]
 // OPT-NEXT:    [[COERCE_SROA_0_0_COPYLOAD:%.*]] = load ptr, ptr addrspace(4) [[TMP0]], align 8, !amdgpu.noclobber [[META4]]
 // OPT-NEXT:    [[TMP1:%.*]] = addrspacecast ptr [[COERCE_SROA_0_0_COPYLOAD]] to ptr addrspace(1)
@@ -623,7 +623,7 @@ __global__ void kernel6(struct T t) {
 // CHECK-SPIRV-NEXT:    ret void
 //
 // OPT-LABEL: define dso_local amdgpu_kernel void @_Z7kernel7Pi(
-// OPT-SAME: ptr addrspace(1) noalias nocapture noundef [[X_COERCE:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// OPT-SAME: ptr addrspace(1) noalias noundef captures(none) [[X_COERCE:%.*]]) local_unnamed_addr #[[ATTR0]] {
 // OPT-NEXT:  [[ENTRY:.*:]]
 // OPT-NEXT:    [[TMP0:%.*]] = load i32, ptr addrspace(1) [[X_COERCE]], align 4
 // OPT-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP0]], 1
@@ -692,7 +692,7 @@ struct SS {
 // CHECK-SPIRV-NEXT:    ret void
 //
 // OPT-LABEL: define dso_local amdgpu_kernel void @_Z7kernel82SS(
-// OPT-SAME: ptr addrspace(1) nocapture [[A_COERCE:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// OPT-SAME: ptr addrspace(1) captures(none) [[A_COERCE:%.*]]) local_unnamed_addr #[[ATTR0]] {
 // OPT-NEXT:  [[ENTRY:.*:]]
 // OPT-NEXT:    [[TMP0:%.*]] = load float, ptr addrspace(1) [[A_COERCE]], align 4
 // OPT-NEXT:    [[ADD:%.*]] = fadd contract float [[TMP0]], 3.000000e+00
diff --git a/clang/test/CodeGenCUDA/offloading-entries.cu b/clang/test/CodeGenCUDA/offloading-entries.cu
index d46a25969e3ec..c053cf586f8f5 100644
--- a/clang/test/CodeGenCUDA/offloading-entries.cu
+++ b/clang/test/CodeGenCUDA/offloading-entries.cu
@@ -18,64 +18,62 @@
 
 //.
 // CUDA: @managed = global i32 undef, align 4
-// CUDA: @.offloading.entry_name = internal unnamed_addr constant [8 x i8] c"_Z3foov\00", section ".llvm.rodata.offloading", align 1
-// CUDA: @.offloading.entry._Z3foov = weak constant %struct.__tgt_offload_entry { ptr @_Z18__device_stub__foov, ptr @.offloading.entry_name, i64 0, i32 0, i32 0 }, section "cuda_offloading_entries", align 1
-// CUDA: @.offloading.entry_name.1 = internal unnamed_addr constant [11 x i8] c"_Z6kernelv\00", section ".llvm.rodata.offloading", align 1
-// CUDA: @.offloading.entry._Z6kernelv = weak constant %struct.__tgt_offload_entry { ptr @_Z21__device_stub__kernelv, ptr @.offloading.entry_name.1, i64 0, i32 0, i32 0 }, section "cuda_offloading_entries", align 1
-// CUDA: @.offloading.entry_name.2 = internal unnamed_addr constant [4 x i8] c"var\00", section ".llvm.rodata.offloading", align 1
-// CUDA: @.offloading.entry.var = weak constant %struct.__tgt_offload_entry { ptr @var, ptr @.offloading.entry_name.2, i64 4, i32 0, i32 0 }, section "cuda_offloading_entries", align 1
-// CUDA: @.offloading.entry_name.3 = internal unnamed_addr constant [8 x i8] c"managed\00", section ".llvm.rodata.offloading", align 1
-// CUDA: @.offloading.entry.managed = weak constant %struct.__tgt_offload_entry { ptr @managed, ptr @.offloading.entry_name.3, i64 4, i32 0, i32 0 }, section "cuda_offloading_entries", align 1
-// CUDA: @.offloading.entry_name.4 = internal unnamed_addr constant [5 x i8] c"surf\00", section ".llvm.rodata.offloading", align 1
-// CUDA: @.offloading.entry.surf = weak constant %struct.__tgt_offload_entry { ptr @surf, ptr @.offloading.entry_name.4, i64 4, i32 2, i32 1 }, section "cuda_offloading_entries", align 1
-// CUDA: @.offloading.entry_name.5 = internal unnamed_addr constant [4 x i8] c"tex\00", section ".llvm.rodata.offloading", align 1
-// CUDA: @.offloading.entry.tex = weak constant %struct.__tgt_offload_entry { ptr @tex, ptr @.offloading.entry_name.5, i64 4, i32 3, i32 1 }, section "cuda_offloading_entries", align 1
+// CUDA: @.offloading.entry_name = internal unnamed_addr constant [8 x i8] c"_Z3foov\00", section ".llvm.rodata.offloading"
+// CUDA: @.offloading.entry._Z3foov = weak constant %struct.__tgt_offload_entry { i64 0, i16 1, i16 2, i32 0, ptr @_Z18__device_stub__foov, ptr @.offloading.entry_name, i64 0, i64 0, ptr null }, section "llvm_offload_entries"
+// CUDA: @.offloading.entry_name.1 = internal unnamed_addr constant [11 x i8] c"_Z6kernelv\00", section ".llvm.rodata.offloading"
+// CUDA: @.offloading.entry._Z6kernelv = weak constant %struct.__tgt_offload_entry { i64 0, i16 1, i16 2, i32 0, ptr @_Z21__device_stub__kernelv, ptr @.offloading.entry_name.1, i64 0, i64 0, ptr null }, section "llvm_offload_entries"
+// CUDA: @.offloading.entry_name.2 = internal unnamed_addr constant [4 x i8] c"var\00", section ".llvm.rodata.offloading"
+// CUDA: @.offloading.entry.var = weak constant %struct.__tgt_offload_entry { i64 0, i16 1, i16 2, i32 0, ptr @var, ptr @.offloading.entry_name.2, i64 4, i64 0, ptr null }, section "llvm_offload_entries"
+// CUDA: @.offloading.entry_name.3 = internal unnamed_addr constant [8 x i8] c"managed\00", section ".llvm.rodata.offloading"
+// CUDA: @.offloading.entry.managed = weak constant %struct.__tgt_offload_entry { i64 0, i16 1, i16 2, i32 0, ptr @managed, ptr @.offloading.entry_name.3, i64 4, i64 0, ptr null }, section "llvm_offload_entries"
+// CUDA: @.offloading.entry_name.4 = internal unnamed_addr constant [5 x i8] c"surf\00", section ".llvm.rodata.offloading"
+// CUDA: @.offloading.entry.surf = weak constant %struct.__tgt_offload_entry { i64 0, i16 1, i16 2, i32 2, ptr @surf, ptr @.offloading.entry_name.4, i64 4, i64 1, ptr null }, section "llvm_offload_entries"
+// CUDA: @.offloading.entry_name.5 = internal unnamed_addr constant [4 x i8] c"tex\00", section ".llvm.rodata.offloading"
+// CUDA: @.offloading.entry.tex = weak constant %struct.__tgt_offload_entry { i64 0, i16 1, i16 2, i32 3, ptr @tex, ptr @.offloading.entry_name.5, i64 4, i64 1, ptr null }, section "llvm_offload_entries"
 //.
 // HIP: @managed.managed = global i32 0, align 4
 // HIP: @managed = externally_initialized global ptr null
-// HIP: @.offloading.entry_name = internal unnamed_addr constant [8 x i8] c"_Z3foov\00", section ".llvm.rodata.offloading", align 1
-// HIP: @.offloading.entry._Z3foov = weak constant %struct.__tgt_offload_entry { ptr @_Z3foov, ptr @.offloading.entry_name, i64 0, i32 0, i32 0 }, section "hip_offloading_entries", align 1
-// HIP: @.offloading.entry_name.1 = internal unnamed_addr constant [11 x i8] c"_Z6kernelv\00", section ".llvm.rodata.offloading", align 1
-// HIP: @.offloading.entry._Z6kernelv = weak constant %struct.__tgt_offload_entry { ptr @_Z6kernelv, ptr @.offloading.entry_name.1, i64 0, i32 0, i32 0 }, section "hip_offloading_entries", align 1
-// HIP: @.offloading.entry_name.2 = internal unnamed_addr constant [4 x i8] c"var\00", section ".llvm.rodata.offloading", align 1
-// HIP: @.offloading.entry.var = weak constant %struct.__tgt_offload_entry { ptr @var, ptr @.offloading.entry_name.2, i64 4, i32 0, i32 0 }, section "hip_offloading_entries", align 1
-// HIP: @managed.managed.3 = private constant %struct.__managed_var { ptr @managed, ptr @managed.managed }
-// HIP: @.offloading.entry_name.4 = internal unnamed_addr constant [8 x i8] c"managed\00", section ".llvm.rodata.offloading", align 1
-// HIP: @.offloading.entry.managed = weak constant %struct.__tgt_offload_entry { ptr @managed.managed.3, ptr @.offloading.entry_name.4, i64 4, i32 1, i32 4 }, section "hip_offloading_entries", align 1
-// HIP: @.offloading.entry_name.5 = internal unnamed_addr constant [5 x i8] c"surf\00", section ".llvm.rodata.offloading", align 1
-// HIP: @.offloading.entry.surf = weak constant %struct.__tgt_offload_entry { ptr @surf, ptr @.offloading.entry_name.5, i64 4, i32 2, i32 1 }, section "hip_offloading_entries", align 1
-// HIP: @.offloading.entry_name.6 = internal unnamed_addr constant [4 x i8] c"tex\00", section ".llvm.rodata.offloading", align 1
-// HIP: @.offloading.entry.tex = weak constant %struct.__tgt_offload_entry { ptr @tex, ptr @.offloading.entry_name.6, i64 4, i32 3, i32 1 }, section "hip_offloading_entries", align 1
+// HIP: @.offloading.entry_name = internal unnamed_addr constant [8 x i8] c"_Z3foov\00", section ".llvm.rodata.offloading"
+// HIP: @.offloading.entry._Z3foov = weak constant %struct.__tgt_offload_entry { i64 0, i16 1, i16 3, i32 0, ptr @_Z3foov, ptr @.offloading.entry_name, i64 0, i64 0, ptr null }, section "llvm_offload_entries"
+// HIP: @.offloading.entry_name.1 = internal unnamed_addr constant [11 x i8] c"_Z6kernelv\00", section ".llvm.rodata.offloading"
+// HIP: @.offloading.entry._Z6kernelv = weak constant %struct.__tgt_offload_entry { i64 0, i16 1, i16 3, i32 0, ptr @_Z6kernelv, ptr @.offloading.entry_name.1, i64 0, i64 0, ptr null }, section "llvm_offload_entries"
+// HIP: @.offloading.entry_name.2 = internal unnamed_addr constant [4 x i8] c"var\00", section ".llvm.rodata.offloading"
+// HIP: @.offloading.entry.var = weak constant %struct.__tgt_offload_entry { i64 0, i16 1, i16 3, i32 0, ptr @var, ptr @.offloading.entry_name.2, i64 4, i64 0, ptr null }, section "llvm_offload_entries"
+// HIP: @.offloading.entry_name.3 = internal unnamed_addr constant [8 x i8] c"managed\00", section ".llvm.rodata.offloading"
+// HIP: @.offloading.entry.managed = weak constant %struct.__tgt_offload_entry { i64 0, i16 1, i16 3, i32 1, ptr @managed.managed, ptr @.offloading.entry_name.3, i64 4, i64 4, ptr @managed }, section "llvm_offload_entries"
+// HIP: @.offloading.entry_name.4 = internal unnamed_addr constant [5 x i8] c"surf\00", section ".llvm.rodata.offloading"
+// HIP: @.offloading.entry.surf = weak constant %struct.__tgt_offload_entry { i64 0, i16 1, i16 3, i32 2, ptr @surf, ptr @.offloading.entry_name.4, i64 4, i64 1, ptr null }, section "llvm_offload_entries"
+// HIP: @.offloading.entry_name.5 = internal unnamed_addr constant [4 x i8] c"tex\00", section ".llvm.rodata.offloading"
+// HIP: @.offloading.entry.tex = weak constant %struct.__tgt_offload_entry { i64 0, i16 1, i16 3, i32 3, ptr @tex, ptr @.offloading.entry_name.5, i64 4, i64 1, ptr null }, section "llvm_offload_entries"
 //.
 // CUDA-COFF: @managed = dso_local global i32 undef, align 4
-// CUDA-COFF: @.offloading.entry_name = internal unnamed_addr constant [8 x i8] c"_Z3foov\00", section ".llvm.rodata.offloading", align 1
-// CUDA-COFF: @.offloading.entry._Z3foov = weak constant %struct.__tgt_offload_entry { ptr @_Z18__device_stub__foov, ptr @.offloading.entry_name, i64 0, i32 0, i32 0 }, section "cuda_offloading_entries$OE", align 1
-// CUDA-COFF: @.offloading.entry_name.1 = internal unnamed_addr constant [11 x i8] c"_Z6kernelv\00", section ".llvm.rodata.offloading", align 1
-// CUDA-COFF: @.offloading.entry._Z6kernelv = weak constant %struct.__tgt_offload_entry { ptr @_Z21__device_stub__kernelv, ptr @.offloading.entry_name.1, i64 0, i32 0, i32 0 }, section "cuda_offloading_entries$OE", align 1
-// CUDA-COFF: @.offloading.entry_name.2 = internal unnamed_addr constant [4 x i8] c"var\00", section ".llvm.rodata.offloading", align 1
-// CUDA-COFF: @.offloading.entry.var = weak constant %struct.__tgt_offload_entry { ptr @var, ptr @.offloading.entry_name.2, i64 4, i32 0, i32 0 }, section "cuda_offloading_entries$OE", align 1
-// CUDA-COFF: @.offloading.entry_name.3 = internal unnamed_addr constant [8 x i8] c"managed\00", section ".llvm.rodata.offloading", align 1
-// CUDA-COFF: @.offloading.entry.managed = weak constant %struct.__tgt_offload_entry { ptr @managed, ptr @.offloading.entry_name.3, i64 4, i32 0, i32 0 }, section "cuda_offloading_entries$OE", align 1
-// CUDA-COFF: @.offloading.entry_name.4 = internal unnamed_addr constant [5 x i8] c"surf\00", section ".llvm.rodata.offloading", align 1
-// CUDA-COFF: @.offloading.entry.surf = weak constant %struct.__tgt_offload_entry { ptr @surf, ptr @.offloading.entry_name.4, i64 4, i32 2, i32 1 }, section "cuda_offloading_entries$OE", align 1
-// CUDA-COFF: @.offloading.entry_name.5 = internal unnamed_addr constant [4 x i8] c"tex\00", section ".llvm.rodata.offloading", align 1
-// CUDA-COFF: @.offloading.entry.tex = weak constant %struct.__tgt_offload_entry { ptr @tex, ptr @.offloading.entry_name.5, i64 4, i32 3, i32 1 }, section "cuda_offloading_entries$OE", align 1
+// CUDA-COFF: @.offloading.entry_name = internal unnamed_addr constant [8 x i8] c"_Z3foov\00", section ".llvm.rodata.offloading"
+// CUDA-COFF: @.offloading.entry._Z3foov = weak constant %struct.__tgt_offload_entry { i64 0, i16 1, i16 2, i32 0, ptr @_Z18__device_stub__foov, ptr @.offloading.entry_name, i64 0, i64 0, ptr null }, section "llvm_offload_entries$OE"
+// CUDA-COFF: @.offloading.entry_name.1 = internal unnamed_addr constant [11 x i8] c"_Z6kernelv\00", section ".llvm.rodata.offloading"
+// CUDA-COFF: @.offloading.entry._Z6kernelv = weak constant %struct.__tgt_offload_entry { i64 0, i16 1, i16 2, i32 0, ptr @_Z21__device_stub__kernelv, ptr @.offloading.entry_name.1, i64 0, i64 0, ptr null }, section "llvm_offload_entries$OE"
+// CUDA-COFF: @.offloading.entry_name.2 = internal unnamed_addr constant [4 x i8] c"var\00", section ".llvm.rodata.offloading"
+// CUDA-COFF: @.offloading.entry.var = weak constant %struct.__tgt_offload_entry { i64 0, i16 1, i16 2, i32 0, ptr @var, ptr @.offloading.entry_name.2, i64 4, i64 0, ptr null }, section "llvm_offload_entries$OE"
+// CUDA-COFF: @.offloading.entry_name.3 = internal unnamed_addr constant [8 x i8] c"managed\00", section ".llvm.rodata.offloading"
+// CUDA-COFF: @.offloading.entry.managed = weak constant %struct.__tgt_offload_entry { i64 0, i16 1, i16 2, i32 0, ptr @managed, ptr @.offloading.entry_name.3, i64 4, i64 0, ptr null }, section "llvm_offload_entries$OE"
+// CUDA-COFF: @.offloading.entry_name.4 = internal unnamed_addr constant [5 x i8] c"surf\00", section ".llvm.rodata.offloading"
+// CUDA-COFF: @.offloading.entry.surf = weak constant %struct.__tgt_offload_entry { i64 0, i16 1, i16 2, i32 2, ptr @surf, ptr @.offloading.entry_name.4, i64 4, i64 1, ptr null }, section "llvm_offload_entries$OE"
+// CUDA-COFF: @.offloading.entry_name.5 = internal unnamed_addr constant [4 x i8] c"tex\00", section ".llvm.rodata.offloading"
+// CUDA-COFF: @.offloading.entry.tex = weak constant %struct.__tgt_offload_entry { i64 0, i16 1, i16 2, i32 3, ptr @tex, ptr @.offloading.entry_name.5, i64 4, i64 1, ptr null }, section "llvm_offload_entries$OE"
 //.
 // HIP-COFF: @managed.managed = dso_local global i32 0, align 4
 // HIP-COFF: @managed = dso_local externally_initialized global ptr null
-// HIP-COFF: @.offloading.entry_name = internal unnamed_addr constant [8 x i8] c"_Z3foov\00", section ".llvm.rodata.offloading", align 1
-// HIP-COFF: @.offloading.entry._Z3foov = weak constant %struct.__tgt_offload_entry { ptr @_Z3foov, ptr @.offloading.entry_name, i64 0, i32 0, i32 0 }, section "hip_offloading_entries$OE", align 1
-// HIP-COFF: @.offloading.entry_name.1 = internal unnamed_addr constant [11 x i8] c"_Z6kernelv\00", section ".llvm.rodata.offloading", align 1
-// HIP-COFF: @.offloading.entry._Z6kernelv = weak constant %struct.__tgt_offload_entry { ptr @_Z6kernelv, ptr @.offloading.entry_name.1, i64 0, i32 0, i32 0 }, section "hip_offloading_entries$OE", align 1
-// HIP-COFF: @.offloading.entry_name.2 = internal unnamed_addr constant [4 x i8] c"var\00", section ".llvm.rodata.offloading", align 1
-// HIP-COFF: @.offloading.entry.var = weak constant %struct.__tgt_offload_entry { ptr @var, ptr @.offloading.entry_name.2, i64 4, i32 0, i32 0 }, section "hip_offloading_entries$OE", align 1
-// HIP-COFF: @managed.managed.3 = private constant %struct.__managed_var { ptr @managed, ptr @managed.managed }
-// HIP-COFF: @.offloading.entry_name.4 = internal unnamed_addr constant [8 x i8] c"managed\00", section ".llvm.rodata.offloading", align 1
-// HIP-COFF: @.offloading.entry.managed = weak constant %struct.__tgt_offload_entry { ptr @managed.managed.3, ptr @.offloading.entry_name.4, i64 4, i32 1, i32 4 }, section "hip_offloading_entries$OE", align 1
-// HIP-COFF: @.offloading.entry_name.5 = internal unnamed_addr constant [5 x i8] c"surf\00", section ".llvm.rodata.offloading", align 1
-// HIP-COFF: @.offloading.entry.surf = weak constant %struct.__tgt_offload_entry { ptr @surf, ptr @.offloading.entry_name.5, i64 4, i32 2, i32 1 }, section "hip_offloading_entries$OE", align 1
-// HIP-COFF: @.offloading.entry_name.6 = internal unnamed_addr constant [4 x i8] c"tex\00", section ".llvm.rodata.offloading", align 1
-// HIP-COFF: @.offloading.entry.tex = weak constant %struct.__tgt_offload_entry { ptr @tex, ptr @.offloading.entry_name.6, i64 4, i32 3, i32 1 }, section "hip_offloading_entries$OE", align 1
+// HIP-COFF: @.offloading.entry_name = internal unnamed_addr constant [8 x i8] c"_Z3foov\00", section ".llvm.rodata.offloading"
+// HIP-COFF: @.offloading.entry._Z3foov = weak constant %struct.__tgt_offload_entry { i64 0, i16 1, i16 3, i32 0, ptr @_Z3foov, ptr @.offloading.entry_name, i64 0, i64 0, ptr null }, section "llvm_offload_entries$OE"
+// HIP-COFF: @.offloading.entry_name.1 = internal unnamed_addr constant [11 x i8] c"_Z6kernelv\00", section ".llvm.rodata.offloading"
+// HIP-COFF: @.offloading.entry._Z6kernelv = weak constant %struct.__tgt_offload_entry { i64 0, i16 1, i16 3, i32 0, ptr @_Z6kernelv, ptr @.offloading.entry_name.1, i64 0, i64 0, ptr null }, section "llvm_offload_entries$OE"
+// HIP-COFF: @.offloading.entry_name.2 = internal unnamed_addr constant [4 x i8] c"var\00", section ".llvm.rodata.offloading"
+// HIP-COFF: @.offloading.entry.var = weak constant %struct.__tgt_offload_entry { i64 0, i16 1, i16 3, i32 0, ptr @var, ptr @.offloading.entry_name.2, i64 4, i64 0, ptr null }, section "llvm_offload_entries$OE"
+// HIP-COFF: @.offloading.entry_name.3 = internal unnamed_addr constant [8 x i8] c"managed\00", section ".llvm.rodata.offloading"
+// HIP-COFF: @.offloading.entry.managed = weak constant %struct.__tgt_offload_entry { i64 0, i16 1, i16 3, i32 1, ptr @managed.managed, ptr @.offloading.entry_name.3, i64 4, i64 4, ptr @managed }, section "llvm_offload_entries$OE"
+// HIP-COFF: @.offloading.entry_name.4 = internal unnamed_addr constant [5 x i8] c"surf\00", section ".llvm.rodata.offloading"
+// HIP-COFF: @.offloading.entry.surf = weak constant %struct.__tgt_offload_entry { i64 0, i16 1, i16 3, i32 2, ptr @surf, ptr @.offloading.entry_name.4, i64 4, i64 1, ptr null }, section "llvm_offload_entries$OE"
+// HIP-COFF: @.offloading.entry_name.5 = internal unnamed_addr constant [4 x i8] c"tex\00", section ".llvm.rodata.offloading"
+// HIP-COFF: @.offloading.entry.tex = weak constant %struct.__tgt_offload_entry { i64 0, i16 1, i16 3, i32 3, ptr @tex, ptr @.offloading.entry_name.5, i64 4, i64 1, ptr null }, section "llvm_offload_entries$OE"
 //.
 // CUDA-LABEL: @_Z18__device_stub__foov(
 // CUDA-NEXT:  entry:
diff --git a/clang/test/CodeGenCXX/aarch64-mangle-neon-vectors.cpp b/clang/test/CodeGenCXX/aarch64-mangle-neon-vectors.cpp
index 3b4a309327fe6..9b855698f57fd 100644
--- a/clang/test/CodeGenCXX/aarch64-mangle-neon-vectors.cpp
+++ b/clang/test/CodeGenCXX/aarch64-mangle-neon-vectors.cpp
@@ -11,6 +11,7 @@ typedef unsigned short poly16_t;
 typedef __fp16 float16_t;
 typedef float float32_t;
 typedef double float64_t;
+typedef __mfp8 mfloat8_t;
 
 typedef __attribute__((neon_vector_type(8))) int8_t int8x8_t;
 typedef __attribute__((neon_vector_type(16))) int8_t int8x16_t;
@@ -26,6 +27,8 @@ typedef __attribute__((neon_vector_type(8))) uint16_t uint16x8_t;
 typedef __attribute__((neon_vector_type(2))) unsigned int uint32x2_t;
 typedef __attribute__((neon_vector_type(4))) unsigned int uint32x4_t;
 typedef __attribute__((neon_vector_type(2))) uint64_t uint64x2_t;
+typedef __attribute__((neon_vector_type(8))) mfloat8_t mfloat8x8_t;
+typedef __attribute__((neon_vector_type(16))) mfloat8_t mfloat8x16_t;
 typedef __attribute__((neon_vector_type(4))) float16_t float16x4_t;
 typedef __attribute__((neon_vector_type(8))) float16_t float16x8_t;
 typedef __attribute__((neon_vector_type(2))) float32_t float32x2_t;
@@ -82,3 +85,7 @@ void f21(int64x2_t) {}
 void f22(uint64x2_t) {}
 // CHECK: 13__Float64x2_t
 void f23(float64x2_t) {}
+// CHECK: 13__Mfloat8x8_t
+void f24(mfloat8x8_t) {}
+// CHECK: 14__Mfloat8x16_t
+void f25(mfloat8x16_t) {}
diff --git a/clang/test/CodeGenCXX/aarch64-ms-mangle-mfp8.cpp b/clang/test/CodeGenCXX/aarch64-ms-mangle-mfp8.cpp
new file mode 100644
index 0000000000000..b5fd9171ad81a
--- /dev/null
+++ b/clang/test/CodeGenCXX/aarch64-ms-mangle-mfp8.cpp
@@ -0,0 +1,14 @@
+// RUN: %clang_cc1 -triple aarch64-windows-msvc -emit-llvm -o - %s | FileCheck %s
+
+typedef __mfp8 mf8;
+typedef __attribute__((neon_vector_type(8))) __mfp8 mf8x8_t;
+typedef __attribute__((neon_vector_type(16))) __mfp8 mf8x16_t;
+
+// CHECK: "?f@@YAXU__mfp8@__clang@@@Z"
+void f(mf8 v) {}
+
+// CHECK: "?f@@YAXT?$__vector@U__mfp8@__clang@@$07@__clang@@@Z"
+void f(mf8x8_t v) {}
+
+// CHECK: "?f@@YAXT?$__vector@U__mfp8@__clang@@$0BA@@__clang@@@Z"
+void f(mf8x16_t v) {}
diff --git a/clang/test/CodeGenCXX/attr-annotate2.cpp b/clang/test/CodeGenCXX/attr-annotate2.cpp
index 1c9c39c9efb83..0b57af1b42a05 100644
--- a/clang/test/CodeGenCXX/attr-annotate2.cpp
+++ b/clang/test/CodeGenCXX/attr-annotate2.cpp
@@ -4,18 +4,28 @@
 // CHECK: @[[STR:.*]] = private unnamed_addr constant [45 x i8] c"_Generic selection expression should be fine\00", section "llvm.metadata"
 // CHECK-NEXT: @[[FILENAME:.*]] = private unnamed_addr constant {{.*}}, section "llvm.metadata"
 // CHECK-NEXT: @[[ARGS:.*]] = private unnamed_addr constant { i32 } zeroinitializer, section "llvm.metadata"
+// CHECK-NEXT: @[[STR2:.*]] = private unnamed_addr constant [14 x i8] c"void is undef\00", section "llvm.metadata"
+// CHECK-NEXT: @[[ARGS3:.*]] = private unnamed_addr constant { i8, i8, i32 } { i8 undef, i8 undef, i32 7 }, section "llvm.metadata"
+
+
 
 // CHECK-LABEL: @_Z1fv(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[N:%.*]] = alloca i32, align 4
 // CHECK-NEXT:    [[J:%.*]] = alloca i32, align 4
+// CHECK-NEXT:    [[K:%.*]] = alloca i32, align 4
 // CHECK-NEXT:    store i32 10, ptr [[N]], align 4
 // CHECK-NEXT:    call void @llvm.var.annotation.p0.p0(ptr [[J]], ptr @[[STR]], ptr @[[FILENAME]], i32 {{.*}}, ptr @[[ARGS]])
 // CHECK-NEXT:    store i32 0, ptr [[J]], align 4
+// CHECK-NEXT:    call void @llvm.var.annotation.p0.p0(ptr [[K]], ptr @[[STR2]], ptr @[[FILENAME]], i32 {{.*}}, ptr @[[ARGS3]])
+// CHECK-NEXT:    store i32 0, ptr [[K]], align 4
 // CHECK-NEXT:    ret void
 //
 void f() {
   int n = 10;
   [[clang::annotate("_Generic selection expression should be fine", _Generic(n, int : 0, default : 1))]]
   int j = 0; // second arg should resolve to 0 fine
+
+  [[clang::annotate("void is undef", (void)2, (void)4, 7)]]
+  int k = 0;
 }
diff --git a/clang/test/CodeGenCXX/bitfield-ir.cpp b/clang/test/CodeGenCXX/bitfield-ir.cpp
index d91b089d09621..8fc4375a4f414 100644
--- a/clang/test/CodeGenCXX/bitfield-ir.cpp
+++ b/clang/test/CodeGenCXX/bitfield-ir.cpp
@@ -21,7 +21,7 @@ struct Int {
 
 
 // CHECK-LABEL: define dso_local void @_Z1AP4Tail
-// CHECK-SAME: (ptr nocapture noundef [[P:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
+// CHECK-SAME: (ptr noundef captures(none) [[P:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[BF_LOAD:%.*]] = load i16, ptr [[P]], align 4
 // CHECK-NEXT:    [[INC:%.*]] = add i16 [[BF_LOAD]], 1
@@ -33,7 +33,7 @@ void A (Tail *p) {
 }
 
 // CHECK-LABEL: define dso_local void @_Z1BP4Tail
-// CHECK-SAME: (ptr nocapture noundef [[P:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-SAME: (ptr noundef captures(none) [[P:%.*]]) local_unnamed_addr #[[ATTR0]] {
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[B:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i64 2
 // CHECK-NEXT:    [[BF_LOAD:%.*]] = load i8, ptr [[B]], align 2
@@ -46,7 +46,7 @@ void B (Tail *p) {
 }
 
 // CHECK-LABEL: define dso_local void @_Z1AP4Char
-// CHECK-SAME: (ptr nocapture noundef [[P:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-SAME: (ptr noundef captures(none) [[P:%.*]]) local_unnamed_addr #[[ATTR0]] {
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[BF_LOAD:%.*]] = load i16, ptr [[P]], align 4
 // CHECK-NEXT:    [[INC:%.*]] = add i16 [[BF_LOAD]], 1
@@ -58,7 +58,7 @@ void A (Char *p) {
 }
 
 // CHECK-LABEL: define dso_local void @_Z1BP4Char
-// CHECK-SAME: (ptr nocapture noundef [[P:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-SAME: (ptr noundef captures(none) [[P:%.*]]) local_unnamed_addr #[[ATTR0]] {
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[B:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i64 2
 // CHECK-NEXT:    [[BF_LOAD:%.*]] = load i8, ptr [[B]], align 2
@@ -71,7 +71,7 @@ void B (Char *p) {
 }
 
 // CHECK-LABEL: define dso_local void @_Z1AP3Int
-// CHECK-SAME: (ptr nocapture noundef [[P:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-SAME: (ptr noundef captures(none) [[P:%.*]]) local_unnamed_addr #[[ATTR0]] {
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[BF_LOAD:%.*]] = load i32, ptr [[P]], align 4
 // CHECK-NEXT:    [[INC:%.*]] = add i32 [[BF_LOAD]], 1
@@ -86,7 +86,7 @@ void A (Int *p) {
 }
 
 // CHECK-LABEL: define dso_local void @_Z1BP3Int
-// CHECK-SAME: (ptr nocapture noundef [[P:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-SAME: (ptr noundef captures(none) [[P:%.*]]) local_unnamed_addr #[[ATTR0]] {
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[BF_LOAD:%.*]] = load i32, ptr [[P]], align 4
 // CHECK-NEXT:    [[BF_VALUE:%.*]] = add i32 [[BF_LOAD]], 65536
diff --git a/clang/test/CodeGenCXX/cxx2c-decomposition.cpp b/clang/test/CodeGenCXX/cxx2c-decomposition.cpp
new file mode 100644
index 0000000000000..27562524fd4b0
--- /dev/null
+++ b/clang/test/CodeGenCXX/cxx2c-decomposition.cpp
@@ -0,0 +1,56 @@
+// RUN: %clang_cc1 -std=c++26 -triple x86_64-linux-gnu -emit-llvm -o - %s | FileCheck %s
+
+namespace std {
+  using size_t = decltype(sizeof(0));
+  template<typename> struct tuple_size;
+  template<size_t, typename> struct tuple_element;
+}
+
+struct Y { int n; };
+struct X { X(); X(Y); X(const X&); ~X(); };
+
+struct A { int a : 13; bool b; };
+
+struct B {};
+template<> struct std::tuple_size<B> { enum { value = 2 }; };
+template<> struct std::tuple_element<0,B> { using type = X; };
+template<> struct std::tuple_element<1,B> { using type = const int&; };
+template<int N> auto get(B) {
+  if constexpr (N == 0)
+    return Y();
+  else
+    return 0.0;
+}
+
+using C = int[2];
+
+template<typename T> T &make();
+
+// CHECK-LABEL: define {{.*}} @_Z8big_testIiEiv()
+template <typename T>
+int big_test() {
+  A& a = make<A>();
+A:
+  auto &[...an] = a;
+  an...[0] = 5;
+  // CHECK: %[[a1:.*]].load = load i16, ptr %[[BITFIELD:.*]],
+  // CHECK: %[[a1]].clear = and i16 %[[a1]].load, -8192
+  // CHECK: %[[a1]].set = or i16 %[[a1]].clear, 5
+  // CHECK: store i16 %[[a1]].set, ptr %[[BITFIELD]],
+B:
+  auto [b1, ...bn] = make<B>();
+  // CHECK: @_Z4makeI1BERT_v()
+  //   CHECK: call i32 @_Z3getILi0EEDa1B()
+  //   CHECK: call void @_ZN1XC1E1Y(ptr {{[^,]*}} %[[b1:.*]], i32
+  //
+  //   CHECK: call noundef double @_Z3getILi1EEDa1B()
+  //   CHECK: %[[cvt:.*]] = fptosi double %{{.*}} to i32
+  //   CHECK: store i32 %[[cvt]], ptr %[[b2:.*]],
+  //   CHECK: store ptr %[[b2]], ptr %[[b2ref:.*]],
+  int bn2 = bn...[0];
+  // CHECK load ptr, ptr %[[b2ref]]
+
+  return 0;
+}
+
+int g = big_test<int>();
diff --git a/clang/test/CodeGenCXX/debug-info-ms-novtable.cpp b/clang/test/CodeGenCXX/debug-info-ms-novtable.cpp
new file mode 100644
index 0000000000000..d8800608f8b87
--- /dev/null
+++ b/clang/test/CodeGenCXX/debug-info-ms-novtable.cpp
@@ -0,0 +1,16 @@
+// RUN: %clang -g -S -emit-llvm -fms-extensions -fms-compatibility -target x86_64-pc-windows-msvc -o - %s | FileCheck %s
+
+// CHECK-DAG: ![[FOO:[0-9]+]] = distinct !DICompositeType(tag: DW_TAG_class_type, name: "Foo", file: !{{[0-9]+}}, line: {{[0-9]+}}, size: 128, flags: DIFlagTypePassByReference | DIFlagNonTrivial, elements: ![[FOO_ELEMENTS:[0-9]+]], vtableHolder: ![[FOO]], identifier: ".?AVFoo@@")
+// CHECK-DAG: ![[FOO_ELEMENTS]] = !{![[FOO_VTBL_TY:[0-9]+]], ![[FOO_VTBL_MEMBER:[0-9]+]], ![[FOO_MEMBER:[0-9]+]], ![[FOO_DUMMY:[0-9]+]]}
+// CHECK-DAG: ![[FOO_VTBL_TY]] = !DIDerivedType(tag: DW_TAG_pointer_type, name: "__vtbl_ptr_type", baseType: null, size: 64)
+// CHECK-DAG: ![[FOO_VTBL_MEMBER]] = !DIDerivedType(tag: DW_TAG_member, name: "_vptr$Foo", scope: !10, file: !{{[0-9]+}}, baseType: ![[FOO_VTBL_PTR_TY:[0-9]+]], size: 64, flags: DIFlagArtificial)
+// CHECK-DAG: ![[FOO_VTBL_PTR_TY]] = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: ![[FOO_VTBL_TY]], size: 64)
+// CHECK-DAG: ![[FOO_MEMBER]] = !DIDerivedType(tag: DW_TAG_member, name: "member", scope: ![[FOO]], file: !{{[0-9]+}}, line: {{[0-9]+}}, baseType: !{{[0-9]+}}, size: 32, offset: 64)
+// CHECK-DAG: ![[FOO_DUMMY]] = !DISubprogram(name: "dummy", linkageName: "?dummy@Foo@@EEAAXXZ", scope: ![[FOO]], file: !{{[0-9]+}}, line: {{[0-9]+}}, type: !{{[0-9]+}}, scopeLine: {{[0-9]+}}, containingType: ![[FOO]], virtualIndex: 0, flags: DIFlagPrototyped | DIFlagIntroducedVirtual, spFlags: DISPFlagVirtual)
+class __declspec(novtable) Foo {
+    virtual void dummy() noexcept {};
+
+    int member = 1;
+};
+
+void foo(Foo) {}
diff --git a/clang/test/CodeGenCXX/gh119046.cpp b/clang/test/CodeGenCXX/gh119046.cpp
new file mode 100644
index 0000000000000..cad76879f0862
--- /dev/null
+++ b/clang/test/CodeGenCXX/gh119046.cpp
@@ -0,0 +1,32 @@
+// RUN: %clang_cc1 -std=c++2a -triple x86_64-elf-gnu %s -emit-llvm -o - | FileCheck %s
+
+struct S {
+    consteval void operator()() {}
+};
+
+template <class Fn>
+constexpr void dispatch(Fn fn) {
+    fn();
+}
+
+template <class Visitor>
+struct value_visitor {
+    constexpr void operator()() { visitor(); }
+    Visitor&& visitor;
+};
+
+template <class Visitor>
+constexpr auto make_dispatch() {
+    return dispatch<value_visitor<S>>;
+}
+
+template <class Visitor>
+constexpr void visit(Visitor&&) {
+    make_dispatch<Visitor>();
+}
+
+void f() { visit(S{}); }
+
+// CHECK: define {{.*}} @_Z1fv
+// CHECK-NOT: define {{.*}} @_Z5visitI1SEvOT_
+// CHECK-NOT: define {{.*}} @_Z13make_dispatchI1SEDav
diff --git a/clang/test/CodeGenCXX/inline-then-fold-variadics.cpp b/clang/test/CodeGenCXX/inline-then-fold-variadics.cpp
index 855787731c8b0..5f83545f78127 100644
--- a/clang/test/CodeGenCXX/inline-then-fold-variadics.cpp
+++ b/clang/test/CodeGenCXX/inline-then-fold-variadics.cpp
@@ -101,14 +101,14 @@ extern "C" {
 typedef uint64_t ulong2 __attribute__((__vector_size__(16), __aligned__(16)));
 
 // CHECK-LABEL: define {{[^@]+}}@first_i32_ulong2
-// CHECK-SAME: (i32 noundef returned [[X:%.*]], ptr nocapture noundef readonly [[Y:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-SAME: (i32 noundef returned [[X:%.*]], ptr noundef readonly captures(none) [[Y:%.*]]) local_unnamed_addr #[[ATTR0]] {
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    ret i32 [[X]]
 //
 int first_i32_ulong2(int x, ulong2 *y) { return first<int, ulong2>(x, *y); }
 
 // CHECK-LABEL: define {{[^@]+}}@second_i32_ulong2
-// CHECK-SAME: (i32 noundef [[X:%.*]], ptr nocapture noundef readonly [[Y:%.*]], ptr nocapture noundef writeonly initializes((0, 16)) [[R:%.*]]) local_unnamed_addr #[[ATTR1:[0-9]+]] {
+// CHECK-SAME: (i32 noundef [[X:%.*]], ptr noundef readonly captures(none) [[Y:%.*]], ptr noundef writeonly captures(none) initializes((0, 16)) [[R:%.*]]) local_unnamed_addr #[[ATTR1:[0-9]+]] {
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = load <2 x i64>, ptr [[Y]], align 16, !tbaa [[TBAA2:![0-9]+]]
 // CHECK-NEXT:    store <2 x i64> [[TMP0]], ptr [[R]], align 16, !tbaa [[TBAA2]]
@@ -119,7 +119,7 @@ void second_i32_ulong2(int x, ulong2 *y, ulong2 *r) {
 }
 
 // CHECK-LABEL: define {{[^@]+}}@first_ulong2_i32
-// CHECK-SAME: (ptr nocapture noundef readonly [[X:%.*]], i32 noundef [[Y:%.*]], ptr nocapture noundef writeonly initializes((0, 16)) [[R:%.*]]) local_unnamed_addr #[[ATTR1]] {
+// CHECK-SAME: (ptr noundef readonly captures(none) [[X:%.*]], i32 noundef [[Y:%.*]], ptr noundef writeonly captures(none) initializes((0, 16)) [[R:%.*]]) local_unnamed_addr #[[ATTR1]] {
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = load <2 x i64>, ptr [[X]], align 16, !tbaa [[TBAA2]]
 // CHECK-NEXT:    store <2 x i64> [[TMP0]], ptr [[R]], align 16, !tbaa [[TBAA2]]
@@ -130,7 +130,7 @@ void first_ulong2_i32(ulong2 *x, int y, ulong2 *r) {
 }
 
 // CHECK-LABEL: define {{[^@]+}}@second_ulong2_i32
-// CHECK-SAME: (ptr nocapture noundef readonly [[X:%.*]], i32 noundef returned [[Y:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-SAME: (ptr noundef readonly captures(none) [[X:%.*]], i32 noundef returned [[Y:%.*]]) local_unnamed_addr #[[ATTR0]] {
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    ret i32 [[Y]]
 //
@@ -150,14 +150,14 @@ typedef struct {
 extern "C" {
 
 // CHECK-LABEL: define {{[^@]+}}@first_i32_asc
-// CHECK-SAME: (i32 noundef returned [[X:%.*]], ptr nocapture noundef readonly [[Y:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-SAME: (i32 noundef returned [[X:%.*]], ptr noundef readonly captures(none) [[Y:%.*]]) local_unnamed_addr #[[ATTR0]] {
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    ret i32 [[X]]
 //
 int first_i32_asc(int x, asc *y) { return first<int, asc>(x, *y); }
 
 // CHECK-LABEL: define {{[^@]+}}@second_i32_asc
-// CHECK-SAME: (i32 noundef [[X:%.*]], ptr nocapture noundef readonly [[Y:%.*]], ptr nocapture noundef writeonly initializes((0, 24)) [[R:%.*]]) local_unnamed_addr #[[ATTR1]] {
+// CHECK-SAME: (i32 noundef [[X:%.*]], ptr noundef readonly captures(none) [[Y:%.*]], ptr noundef writeonly captures(none) initializes((0, 24)) [[R:%.*]]) local_unnamed_addr #[[ATTR1]] {
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    tail call void @llvm.memmove.p0.p0.i32(ptr noundef nonnull align 8 dereferenceable(24) [[R]], ptr noundef nonnull align 1 dereferenceable(24) [[Y]], i32 24, i1 false)
 // CHECK-NEXT:    ret void
@@ -165,7 +165,7 @@ int first_i32_asc(int x, asc *y) { return first<int, asc>(x, *y); }
 void second_i32_asc(int x, asc *y, asc *r) { *r = second<int, asc>(x, *y); }
 
 // CHECK-LABEL: define {{[^@]+}}@first_asc_i32
-// CHECK-SAME: (ptr nocapture noundef readonly [[X:%.*]], i32 noundef [[Y:%.*]], ptr nocapture noundef writeonly initializes((0, 24)) [[R:%.*]]) local_unnamed_addr #[[ATTR1]] {
+// CHECK-SAME: (ptr noundef readonly captures(none) [[X:%.*]], i32 noundef [[Y:%.*]], ptr noundef writeonly captures(none) initializes((0, 24)) [[R:%.*]]) local_unnamed_addr #[[ATTR1]] {
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    tail call void @llvm.memmove.p0.p0.i32(ptr noundef nonnull align 8 dereferenceable(24) [[R]], ptr noundef nonnull align 1 dereferenceable(24) [[X]], i32 24, i1 false)
 // CHECK-NEXT:    ret void
@@ -173,7 +173,7 @@ void second_i32_asc(int x, asc *y, asc *r) { *r = second<int, asc>(x, *y); }
 void first_asc_i32(asc *x, int y, asc *r) { *r = first<asc, int>(*x, y); }
 
 // CHECK-LABEL: define {{[^@]+}}@second_asc_i32
-// CHECK-SAME: (ptr nocapture noundef readonly [[X:%.*]], i32 noundef returned [[Y:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-SAME: (ptr noundef readonly captures(none) [[X:%.*]], i32 noundef returned [[Y:%.*]]) local_unnamed_addr #[[ATTR0]] {
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    ret i32 [[Y]]
 //
diff --git a/clang/test/CodeGenCXX/mangle-neon-vectors.cpp b/clang/test/CodeGenCXX/mangle-neon-vectors.cpp
index cb5e40be6a6df..2139a8ae98caf 100644
--- a/clang/test/CodeGenCXX/mangle-neon-vectors.cpp
+++ b/clang/test/CodeGenCXX/mangle-neon-vectors.cpp
@@ -9,6 +9,7 @@ typedef __fp16 float16_t;
 #if defined(__aarch64__)
 typedef unsigned char poly8_t;
 typedef unsigned short poly16_t;
+typedef __mfp8 mfloat8_t;
 #else
 typedef signed char poly8_t;
 typedef short poly16_t;
@@ -29,6 +30,8 @@ typedef __attribute__((neon_vector_type(4))) float16_t float16x4_t;
 typedef __attribute__((neon_vector_type(8))) float16_t float16x8_t;
 #ifdef __aarch64__
 typedef __attribute__((neon_vector_type(2))) float64_t float64x2_t;
+typedef __attribute__((neon_vector_type(8))) mfloat8_t mfloat8x8_t;
+typedef __attribute__((neon_vector_type(16))) mfloat8_t mfloat8x16_t;
 #endif
 typedef __attribute__((neon_polyvector_type(16))) poly8_t  poly8x16_t;
 typedef __attribute__((neon_polyvector_type(8)))  poly16_t poly16x8_t;
@@ -86,3 +89,11 @@ void f11(float64x2_t v) { }
 // CHECK-AARCH64-BF16: 14__Bfloat16x4_t
 void f12(bfloat16x4_t v) {}
 #endif
+
+
+#ifdef __aarch64__
+// CHECK-AARCH64: 13__Mfloat8x8_t
+void f13(mfloat8x8_t v) { }
+// CHECK-AARCH64: 14__Mfloat8x16_t
+void f14(mfloat8x16_t v) { }
+#endif
diff --git a/clang/test/CodeGenCXX/noescape.cpp b/clang/test/CodeGenCXX/noescape.cpp
index e6d7a727ebb36..c3fc90e2ea54d 100644
--- a/clang/test/CodeGenCXX/noescape.cpp
+++ b/clang/test/CodeGenCXX/noescape.cpp
@@ -8,26 +8,26 @@ struct S {
   virtual void vm1(int *, int * __attribute__((noescape)));
 };
 
-// CHECK: define{{.*}} void @_ZN1SC2EPiS0_(ptr {{.*}}, {{.*}}, {{.*}} nocapture noundef {{%.*}})
-// CHECK: define{{.*}} void @_ZN1SC1EPiS0_(ptr {{.*}}, {{.*}}, {{.*}} nocapture noundef {{%.*}}) {{.*}} {
-// CHECK: call void @_ZN1SC2EPiS0_(ptr {{.*}}, {{.*}}, {{.*}} nocapture {{.*}})
+// CHECK: define{{.*}} void @_ZN1SC2EPiS0_(ptr {{.*}}, {{.*}}, {{.*}} noundef captures(none) {{%.*}})
+// CHECK: define{{.*}} void @_ZN1SC1EPiS0_(ptr {{.*}}, {{.*}}, {{.*}} noundef captures(none) {{%.*}}) {{.*}} {
+// CHECK: call void @_ZN1SC2EPiS0_(ptr {{.*}}, {{.*}}, {{.*}} captures(none) {{.*}})
 
 S::S(int *, int * __attribute__((noescape))) {}
 
-// CHECK: define {{.*}} ptr @_ZN1SaSEPi(ptr {{.*}}, {{.*}} nocapture noundef {{%.*}})
+// CHECK: define {{.*}} ptr @_ZN1SaSEPi(ptr {{.*}}, {{.*}} noundef captures(none) {{%.*}})
 S &S::operator=(int * __attribute__((noescape))) { return *this; }
 
-// CHECK: define{{.*}} void @_ZN1S2m0EPiS0_(ptr {{.*}}, {{.*}} nocapture noundef {{%.*}})
+// CHECK: define{{.*}} void @_ZN1S2m0EPiS0_(ptr {{.*}}, {{.*}} noundef captures(none) {{%.*}})
 void S::m0(int *, int * __attribute__((noescape))) {}
 
-// CHECK: define{{.*}} void @_ZN1S3vm1EPiS0_(ptr {{.*}}, {{.*}} nocapture noundef {{%.*}})
+// CHECK: define{{.*}} void @_ZN1S3vm1EPiS0_(ptr {{.*}}, {{.*}} noundef captures(none) {{%.*}})
 void S::vm1(int *, int * __attribute__((noescape))) {}
 
 // CHECK-LABEL: define{{.*}} void @_Z5test0P1SPiS1_(
-// CHECK: call void @_ZN1SC1EPiS0_(ptr {{.*}}, {{.*}}, {{.*}} nocapture noundef {{.*}})
-// CHECK: call {{.*}} ptr @_ZN1SaSEPi(ptr {{.*}}, {{.*}} nocapture noundef {{.*}})
-// CHECK: call void @_ZN1S2m0EPiS0_(ptr {{.*}}, {{.*}}, {{.*}} nocapture noundef {{.*}})
-// CHECK: call void {{.*}}(ptr {{.*}}, {{.*}}, {{.*}} nocapture noundef {{.*}})
+// CHECK: call void @_ZN1SC1EPiS0_(ptr {{.*}}, {{.*}}, {{.*}} noundef captures(none) {{.*}})
+// CHECK: call {{.*}} ptr @_ZN1SaSEPi(ptr {{.*}}, {{.*}} noundef captures(none) {{.*}})
+// CHECK: call void @_ZN1S2m0EPiS0_(ptr {{.*}}, {{.*}}, {{.*}} noundef captures(none) {{.*}})
+// CHECK: call void {{.*}}(ptr {{.*}}, {{.*}}, {{.*}} noundef captures(none) {{.*}})
 void test0(S *s, int *p0, int *p1) {
   S t(p0, p1);
   t = p1;
@@ -39,27 +39,27 @@ namespace std {
   typedef decltype(sizeof(0)) size_t;
 }
 
-// CHECK: define {{.*}} @_ZnwmPv({{.*}}, {{.*}} nocapture {{.*}})
+// CHECK: define {{.*}} @_ZnwmPv({{.*}}, {{.*}} captures(none) {{.*}})
 void *operator new(std::size_t, void * __attribute__((noescape)) p) {
   return p;
 }
 
 // CHECK-LABEL: define{{.*}} ptr @_Z5test1Pv(
-// CHECK: %call = call {{.*}} @_ZnwmPv({{.*}}, {{.*}} nocapture {{.*}})
+// CHECK: %call = call {{.*}} @_ZnwmPv({{.*}}, {{.*}} captures(none) {{.*}})
 void *test1(void *p0) {
   return ::operator new(16, p0);
 }
 
 // CHECK-LABEL: define{{.*}} void @_Z5test2PiS_(
-// CHECK: call void @"_ZZ5test2PiS_ENK3$_0clES_S_"({{.*}}, {{.*}}, {{.*}} nocapture {{.*}})
-// CHECK: define internal void @"_ZZ5test2PiS_ENK3$_0clES_S_"({{.*}}, {{.*}}, {{.*}} nocapture noundef {{%.*}})
+// CHECK: call void @"_ZZ5test2PiS_ENK3$_0clES_S_"({{.*}}, {{.*}}, {{.*}} captures(none) {{.*}})
+// CHECK: define internal void @"_ZZ5test2PiS_ENK3$_0clES_S_"({{.*}}, {{.*}}, {{.*}} noundef captures(none) {{%.*}})
 void test2(int *p0, int *p1) {
   auto t = [](int *, int * __attribute__((noescape))){};
   t(p0, p1);
 }
 
 // CHECK-LABEL: define{{.*}} void @_Z5test3PFvU8noescapePiES_(
-// CHECK: call void {{.*}}(ptr nocapture noundef {{.*}})
+// CHECK: call void {{.*}}(ptr noundef captures(none) {{.*}})
 typedef void (*NoEscapeFunc)(__attribute__((noescape)) int *);
 
 void test3(NoEscapeFunc f, int *p) {
diff --git a/clang/test/CodeGenCXX/template-param-objects.cpp b/clang/test/CodeGenCXX/template-param-objects.cpp
index 11ebd21521e83..ff6acc438d137 100644
--- a/clang/test/CodeGenCXX/template-param-objects.cpp
+++ b/clang/test/CodeGenCXX/template-param-objects.cpp
@@ -5,6 +5,9 @@ struct S { char buf[32]; };
 template<S s> constexpr const char *begin() { return s.buf; }
 template<S s> constexpr const char *end() { return s.buf + __builtin_strlen(s.buf); }
 
+namespace { struct T { char buf[32]; }; }
+template<T t> constexpr const char* begin_anon() { return t.buf; }
+
 // ITANIUM: [[HELLO:@_ZTAXtl1StlA32_cLc104ELc101ELc108ELc108ELc111ELc32ELc119ELc111ELc114ELc108ELc100EEEE]]
 // MSABI: [[HELLO:@"[?][?]__N2US@@3D0GI@@0GF@@0GM@@0GM@@0GP@@0CA@@0HH@@0GP@@0HC@@0GM@@0GE@@0A@@0A@@0A@@0A@@0A@@0A@@0A@@0A@@0A@@0A@@0A@@0A@@0A@@0A@@0A@@0A@@0A@@0A@@0A@@0A@@0A@@@@@"]]
 // ITANIUM-SAME: = linkonce_odr constant { <{ [11 x i8], [21 x i8] }> } { <{ [11 x i8], [21 x i8] }> <{ [11 x i8] c"hello world", [21 x i8] zeroinitializer }> }, comdat
@@ -19,3 +22,10 @@ const char *p = begin<S{"hello world"}>();
 // MSABI: @"?q@@3PEBDEB"
 // CHECK-SAME: global ptr getelementptr (i8, ptr [[HELLO]], i64 11)
 const char *q = end<S{"hello world"}>();
+
+
+// CHECK: internal constant { <{ [10 x i8], [22 x i8] }> } { <{ [10 x i8], [22 x i8] }> <{ [10 x i8] c"hello anon", [22 x i8] zeroinitializer }> }
+// CHECK-NOT: comdat
+// ITANIUM: @r
+// MSABI: @"?r@@3PEBDEB"
+const char *r = begin_anon<T{"hello anon"}>();
diff --git a/clang/test/CodeGenCXX/wasm-args-returns.cpp b/clang/test/CodeGenCXX/wasm-args-returns.cpp
index b57896b0e0ffe..fbb152ac1bb3d 100644
--- a/clang/test/CodeGenCXX/wasm-args-returns.cpp
+++ b/clang/test/CodeGenCXX/wasm-args-returns.cpp
@@ -30,7 +30,7 @@ struct two_fields {
   double d, e;
 };
 test(two_fields);
-// CHECK: define void @_Z7forward10two_fields(ptr dead_on_unwind noalias nocapture writable writeonly sret(%struct.two_fields) align 8 initializes((0, 16)) %{{.*}}, ptr nocapture readonly byval(%struct.two_fields) align 8 %{{.*}})
+// CHECK: define void @_Z7forward10two_fields(ptr dead_on_unwind noalias writable writeonly sret(%struct.two_fields) align 8 captures(none) initializes((0, 16)) %{{.*}}, ptr readonly byval(%struct.two_fields) align 8 captures(none) %{{.*}})
 //
 // CHECK: define void @_Z15test_two_fieldsv()
 // CHECK: %[[tmp:.*]] = alloca %struct.two_fields, align 8
diff --git a/clang/test/CodeGenHLSL/BasicFeatures/ArrayElementwiseCast.hlsl b/clang/test/CodeGenHLSL/BasicFeatures/ArrayElementwiseCast.hlsl
new file mode 100644
index 0000000000000..18f82bff3b308
--- /dev/null
+++ b/clang/test/CodeGenHLSL/BasicFeatures/ArrayElementwiseCast.hlsl
@@ -0,0 +1,144 @@
+// RUN: %clang_cc1 -triple dxil-pc-shadermodel6.0-library -disable-llvm-passes -emit-llvm -finclude-default-header -o - %s | FileCheck %s
+
+// array truncation to a scalar
+// CHECK-LABEL: define void {{.*}}call0
+// CHECK: [[A:%.*]] = alloca [2 x i32], align 4
+// CHECK-NEXT: [[B:%.*]] = alloca float, align 4
+// CHECK-NEXT: [[Tmp:%.*]] = alloca [2 x i32], align 4
+// CHECK-NEXT: call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[A]], ptr align 4 {{.*}}, i32 8, i1 false)
+// CHECK-NEXT: call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[Tmp]], ptr align 4 [[A]], i32 8, i1 false)
+// CHECK-NEXT: [[G1:%.*]] = getelementptr inbounds [2 x i32], ptr [[Tmp]], i32 0, i32 0
+// CHECK-NEXT: [[G2:%.*]] = getelementptr inbounds [2 x i32], ptr [[Tmp]], i32 0, i32 1
+// CHECK-NEXT: [[L:%.*]] = load i32, ptr [[G1]], align 4
+// CHECK-NEXT: store i32 [[L]], ptr [[B]], align 4
+export void call0() {
+  int A[2] = {0,1};
+  float B = (float)A;
+}
+
+// array truncation
+// CHECK-LABEL: define void {{.*}}call1
+// CHECK: [[A:%.*]] = alloca [2 x i32], align 4
+// CHECK-NEXT: [[B:%.*]] = alloca [1 x i32], align 4
+// CHECK-NEXT: [[Tmp:%.*]] = alloca [2 x i32], align 4
+// CHECK-NEXT: call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[A]], ptr align 4 {{.*}}, i32 8, i1 false)
+// CHECK-NEXT: call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[B]], ptr align 4 {{.*}}, i32 4, i1 false)
+// CHECK-NEXT: call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[Tmp]], ptr align 4 [[A]], i32 8, i1 false)
+// CHECK-NEXT: [[G1:%.*]] = getelementptr inbounds [1 x i32], ptr [[B]], i32 0, i32 0
+// CHECK-NEXT: [[G2:%.*]] = getelementptr inbounds [2 x i32], ptr [[Tmp]], i32 0, i32 0
+// CHECK-NEXT: [[G3:%.*]] = getelementptr inbounds [2 x i32], ptr [[Tmp]], i32 0, i32 1
+// CHECK-NEXT: [[L:%.*]] = load i32, ptr [[G2]], align 4
+// CHECK-NEXT: store i32 [[L]], ptr [[G1]], align 4
+export void call1() {
+  int A[2] = {0,1};
+  int B[1] = {4};
+  B = (int[1])A;
+}
+
+// just a cast
+// CHECK-LABEL: define void {{.*}}call2
+// CHECK: [[A:%.*]] = alloca [1 x i32], align 4
+// CHECK-NEXT: [[B:%.*]] = alloca [1 x float], align 4
+// CHECK-NEXT: [[Tmp:%.*]] = alloca [1 x i32], align 4
+// CHECK-NEXT: call void @llvm.memset.p0.i32(ptr align 4 [[A]], i8 0, i32 4, i1 false)
+// CHECK-NEXT: call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[B]], ptr align 4 {{.*}}, i32 4, i1 false)
+// CHECK-NEXT: call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[Tmp]], ptr align 4 [[A]], i32 4, i1 false)
+// CHECK-NEXT: [[G1:%.*]] = getelementptr inbounds [1 x float], ptr [[B]], i32 0, i32 0
+// CHECK-NEXT: [[G2:%.*]] = getelementptr inbounds [1 x i32], ptr [[Tmp]], i32 0, i32 0
+// CHECK-NEXT: [[L:%.*]] = load i32, ptr [[G2]], align 4
+// CHECK-NEXT: [[C:%.*]] = sitofp i32 [[L]] to float
+// CHECK-NEXT: store float [[C]], ptr [[G1]], align 4
+export void call2() {
+  int A[1] = {0};
+  float B[1] = {1.0};
+  B = (float[1])A;
+}
+
+// vector to array
+// CHECK-LABEL: define void {{.*}}call3
+// CHECK: [[A:%.*]] = alloca <1 x float>, align 4
+// CHECK-NEXT: [[B:%.*]] = alloca [1 x i32], align 4
+// CHECK-NEXT: store <1 x float> splat (float 0x3FF3333340000000), ptr [[A]], align 4
+// CHECK-NEXT: call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[B]], ptr align 4 {{.*}}, i32 4, i1 false)
+// CHECK-NEXT: [[C:%.*]] = load <1 x float>, ptr [[A]], align 4
+// CHECK-NEXT: [[G1:%.*]] = getelementptr inbounds [1 x i32], ptr [[B]], i32 0, i32 0
+// CHECK-NEXT: [[V:%.*]] = extractelement <1 x float> [[C]], i64 0
+// CHECK-NEXT: [[C:%.*]] = fptosi float [[V]] to i32
+// CHECK-NEXT: store i32 [[C]], ptr [[G1]], align 4
+export void call3() {
+  float1 A = {1.2};
+  int B[1] = {1};
+  B = (int[1])A;
+}
+
+// flatten array of vector to array with cast
+// CHECK-LABEL: define void {{.*}}call5
+// CHECK: [[A:%.*]] = alloca [1 x <2 x float>], align 8
+// CHECK-NEXT: [[B:%.*]] = alloca [2 x i32], align 4
+// CHECK-NEXT: [[Tmp:%.*]] = alloca [1 x <2 x float>], align 8
+// CHECK-NEXT: call void @llvm.memcpy.p0.p0.i32(ptr align 8 [[A]], ptr align 8 {{.*}}, i32 8, i1 false)
+// CHECK-NEXT: call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[B]], ptr align 4 {{.*}}, i32 8, i1 false)
+// CHECK-NEXT: call void @llvm.memcpy.p0.p0.i32(ptr align 8 [[Tmp]], ptr align 8 [[A]], i32 8, i1 false)
+// CHECK-NEXT: [[G1:%.*]] = getelementptr inbounds [2 x i32], ptr [[B]], i32 0, i32 0
+// CHECK-NEXT: [[G2:%.*]] = getelementptr inbounds [2 x i32], ptr [[B]], i32 0, i32 1
+// CHECK-NEXT: [[VG:%.*]] = getelementptr inbounds [1 x <2 x float>], ptr [[Tmp]], i32 0, i32 0
+// CHECK-NEXT: [[L:%.*]] = load <2 x float>, ptr [[VG]], align 8
+// CHECK-NEXT: [[VL:%.*]] = extractelement <2 x float> [[L]], i32 0
+// CHECK-NEXT: [[C:%.*]] = fptosi float [[VL]] to i32
+// CHECK-NEXT: store i32 [[C]], ptr [[G1]], align 4
+// CHECK-NEXT: [[L4:%.*]] = load <2 x float>, ptr [[VG]], align 8
+// CHECK-NEXT: [[VL5:%.*]] = extractelement <2 x float> [[L4]], i32 1
+// CHECK-NEXT: [[C6:%.*]] = fptosi float [[VL5]] to i32
+// CHECK-NEXT: store i32 [[C6]], ptr [[G2]], align 4
+export void call5() {
+  float2 A[1] = {{1.2,3.4}};
+  int B[2] = {1,2};
+  B = (int[2])A;
+}
+
+// flatten 2d array
+// CHECK-LABEL: define void {{.*}}call6
+// CHECK: [[A:%.*]] = alloca [2 x [1 x i32]], align 4
+// CHECK-NEXT: [[B:%.*]] = alloca [2 x i32], align 4
+// CHECK-NEXT: [[Tmp:%.*]] = alloca [2 x [1 x i32]], align 4
+// CHECK-NEXT: call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[A]], ptr align 4 {{.*}}, i32 8, i1 false)
+// CHECK-NEXT: call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[B]], ptr align 4 {{.*}}, i32 8, i1 false)
+// CHECK-NEXT: call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[Tmp]], ptr align 4 [[A]], i32 8, i1 false)
+// CHECK-NEXT: [[G1:%.*]] = getelementptr inbounds [2 x i32], ptr [[B]], i32 0, i32 0
+// CHECK-NEXT: [[G2:%.*]] = getelementptr inbounds [2 x i32], ptr [[B]], i32 0, i32 1
+// CHECK-NEXT: [[G3:%.*]] = getelementptr inbounds [2 x [1 x i32]], ptr [[Tmp]], i32 0, i32 0, i32 0
+// CHECK-NEXT: [[G4:%.*]] = getelementptr inbounds [2 x [1 x i32]], ptr [[Tmp]], i32 0, i32 1, i32 0
+// CHECK-NEXT: [[L:%.*]] = load i32, ptr [[G3]], align 4
+// CHECK-NEXT: store i32 [[L]], ptr [[G1]], align 4
+// CHECK-NEXT: [[L4:%.*]] = load i32, ptr [[G4]], align 4
+// CHECK-NEXT: store i32 [[L4]], ptr [[G2]], align 4
+export void call6() {
+  int A[2][1] = {{1},{3}};
+  int B[2] = {1,2};
+  B = (int[2])A;
+}
+
+struct S {
+  int X;
+  float Y;
+};
+
+// flatten and truncate from a struct
+// CHECK-LABEL: define void {{.*}}call7
+// CHECK: [[s:%.*]] = alloca %struct.S, align 4
+// CHECK-NEXT: [[A:%.*]] = alloca [1 x i32], align 4
+// CHECK-NEXT: [[Tmp:%.*]] = alloca %struct.S, align 4
+// CHECK-NEXT: call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[s]], ptr align 4 {{.*}}, i32 8, i1 false)
+// CHECK-NEXT: call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[A]], ptr align 4 {{.*}}, i32 4, i1 false)
+// CHECK-NEXT: call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[Tmp]], ptr align 4 [[s]], i32 8, i1 false)
+// CHECK-NEXT: [[G1:%.*]] = getelementptr inbounds [1 x i32], ptr [[A]], i32 0, i32 0
+// CHECK-NEXT: [[G2:%.*]] = getelementptr inbounds %struct.S, ptr [[Tmp]], i32 0, i32 0
+// CHECK-NEXT: [[G3:%.*]] = getelementptr inbounds %struct.S, ptr [[Tmp]], i32 0, i32 1
+// CHECK-NEXT: [[L:%.*]] = load i32, ptr [[G2]], align 4
+// CHECK-NEXT: store i32 [[L]], ptr [[G1]], align 4
+export void call7() {
+  S s = {1, 2.9};
+  int A[1] = {1};
+  A = (int[1])s;
+}
+
diff --git a/clang/test/CodeGenHLSL/BasicFeatures/StructElementwiseCast.hlsl b/clang/test/CodeGenHLSL/BasicFeatures/StructElementwiseCast.hlsl
new file mode 100644
index 0000000000000..26fde37c901dd
--- /dev/null
+++ b/clang/test/CodeGenHLSL/BasicFeatures/StructElementwiseCast.hlsl
@@ -0,0 +1,140 @@
+// RUN: %clang_cc1 -finclude-default-header -triple dxil-pc-shadermodel6.3-library -x hlsl -emit-llvm -disable-llvm-passes -o - %s | FileCheck %s
+
+struct S {
+  int X;
+  float Y;
+};
+
+// struct truncation to a scalar
+// CHECK-LABEL: define void {{.*}}call0
+// CHECK: [[s:%.*]] = alloca %struct.S, align 4
+// CHECK-NEXT: [[A:%.*]] = alloca i32, align 4
+// CHECK-NEXT: [[Tmp:%.*]] = alloca %struct.S, align 4
+// CHECK-NEXT: call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[s]], ptr align 4 {{.*}}, i32 8, i1 false)
+// CHECK-NEXT: call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[Tmp]], ptr align 4 [[s]], i32 8, i1 false)
+// CHECK-NEXT: [[G1:%.*]] = getelementptr inbounds %struct.S, ptr [[Tmp]], i32 0, i32 0
+// CHECK-NEXT: [[G2:%.*]] = getelementptr inbounds %struct.S, ptr [[Tmp]], i32 0, i32 1
+// CHECK-NEXT: [[L:%.*]] = load i32, ptr [[G1]], align 4
+// CHECK-NEXT: store i32 [[L]], ptr [[A]], align 4
+export void call0() {
+  S s = {1,2};
+  int A = (int)s;
+}
+
+// struct from vector
+// CHECK-LABEL: define void {{.*}}call1
+// CHECK: [[A:%.*]] = alloca <2 x i32>, align 8
+// CHECK-NEXT: [[s:%.*]] = alloca %struct.S, align 4
+// CHECK-NEXT: store <2 x i32> <i32 1, i32 2>, ptr [[A]], align 8
+// CHECK-NEXT: [[L:%.*]] = load <2 x i32>, ptr [[A]], align 8
+// CHECK-NEXT: [[G1:%.*]] = getelementptr inbounds %struct.S, ptr [[s]], i32 0, i32 0
+// CHECK-NEXT: [[G2:%.*]] = getelementptr inbounds %struct.S, ptr [[s]], i32 0, i32 1
+// CHECK-NEXT: [[VL:%.*]] = extractelement <2 x i32> [[L]], i64 0
+// CHECK-NEXT: store i32 [[VL]], ptr [[G1]], align 4
+// CHECK-NEXT: [[VL2:%.*]] = extractelement <2 x i32> [[L]], i64 1
+// CHECK-NEXT: [[C:%.*]] = sitofp i32 [[VL2]] to float
+// CHECK-NEXT: store float [[C]], ptr [[G2]], align 4
+export void call1() {
+  int2 A = {1,2};
+  S s = (S)A;
+}
+
+
+// struct from array
+// CHECK-LABEL: define void {{.*}}call2
+// CHECK: [[A:%.*]] = alloca [2 x i32], align 4
+// CHECK-NEXT: [[s:%.*]] = alloca %struct.S, align 4
+// CHECK-NEXT: [[Tmp:%.*]] = alloca [2 x i32], align 4
+// CHECK-NEXT: call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[A]], ptr align 4 {{.*}}, i32 8, i1 false)
+// CHECK-NEXT: call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[Tmp]], ptr align 4 [[A]], i32 8, i1 false)
+// CHECK-NEXT: [[G1:%.*]] = getelementptr inbounds %struct.S, ptr [[s]], i32 0, i32 0
+// CHECK-NEXT: [[G2:%.*]] = getelementptr inbounds %struct.S, ptr [[s]], i32 0, i32 1
+// CHECK-NEXT: [[G3:%.*]] = getelementptr inbounds [2 x i32], ptr [[Tmp]], i32 0, i32 0
+// CHECK-NEXT: [[G4:%.*]] = getelementptr inbounds [2 x i32], ptr [[Tmp]], i32 0, i32 1
+// CHECK-NEXT: [[L:%.*]] = load i32, ptr [[G3]], align 4
+// CHECK-NEXT: store i32 [[L]], ptr [[G1]], align 4
+// CHECK-NEXT: [[L4:%.*]] = load i32, ptr [[G4]], align 4
+// CHECK-NEXT: [[C:%.*]] = sitofp i32 [[L4]] to float
+// CHECK-NEXT: store float [[C]], ptr [[G2]], align 4
+export void call2() {
+  int A[2] = {1,2};
+  S s = (S)A;
+}
+
+struct Q {
+  int Z;
+};
+
+struct R {
+  Q q;
+  float F;
+};
+
+// struct from nested struct?
+// CHECK-LABEL: define void {{.*}}call6
+// CHECK: [[r:%.*]] = alloca %struct.R, align 4
+// CHECK-NEXT: [[s:%.*]] = alloca %struct.S, align 4
+// CHECK-NEXT: [[Tmp:%.*]] = alloca %struct.R, align 4
+// CHECK-NEXT: call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[r]], ptr align 4 {{.*}}, i32 8, i1 false)
+// CHECK-NEXT: call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[Tmp]], ptr align 4 [[r]], i32 8, i1 false)
+// CHECK-NEXT: [[G1:%.*]] = getelementptr inbounds %struct.S, ptr [[s]], i32 0, i32 0
+// CHECK-NEXT: [[G2:%.*]] = getelementptr inbounds %struct.S, ptr [[s]], i32 0, i32 1
+// CHECK-NEXT: [[G3:%.*]] = getelementptr inbounds %struct.R, ptr [[Tmp]], i32 0, i32 0
+// CHECK-NEXT: [[G4:%.*]] = getelementptr inbounds %struct.R, ptr [[Tmp]], i32 0, i32 1
+// CHECK-NEXT: [[L:%.*]] = load i32, ptr [[G3]], align 4
+// CHECK-NEXT: store i32 [[L]], ptr [[G1]], align 4
+// CHECK-NEXT: [[L4:%.*]] = load float, ptr [[G4]], align 4
+// CHECK-NEXT: store float [[L4]], ptr [[G2]], align 4
+export void call6() {
+  R r = {{1}, 2.0};
+  S s = (S)r;
+}
+
+// nested struct from array?
+// CHECK-LABEL: define void {{.*}}call7
+// CHECK: [[A:%.*]] = alloca [2 x i32], align 4
+// CHECK-NEXT: [[r:%.*]] = alloca %struct.R, align 4
+// CHECK-NEXT: [[Tmp:%.*]] = alloca [2 x i32], align 4
+// CHECK-NEXT: call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[A]], ptr align 4 {{.*}}, i32 8, i1 false)
+// CHECK-NEXT: call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[Tmp]], ptr align 4 [[A]], i32 8, i1 false)
+// CHECK-NEXT: [[G1:%.*]] = getelementptr inbounds %struct.R, ptr [[r]], i32 0, i32 0
+// CHECK-NEXT: [[G2:%.*]] = getelementptr inbounds %struct.R, ptr [[r]], i32 0, i32 1
+// CHECK-NEXT: [[G3:%.*]] = getelementptr inbounds [2 x i32], ptr [[Tmp]], i32 0, i32 0
+// CHECK-NEXT: [[G4:%.*]] = getelementptr inbounds [2 x i32], ptr [[Tmp]], i32 0, i32 1
+// CHECK-NEXT: [[L:%.*]] = load i32, ptr [[G3]], align 4
+// CHECK-NEXT: store i32 [[L]], ptr [[G1]], align 4
+// CHECK-NEXT: [[L4:%.*]] = load i32, ptr [[G4]], align 4
+// CHECK-NEXT: [[C:%.*]] = sitofp i32 [[L4]] to float
+// CHECK-NEXT: store float [[C]], ptr [[G2]], align 4
+export void call7() {
+  int A[2] = {1,2};
+  R r = (R)A;
+}
+
+struct T {
+  int A;
+  int B;
+  int C;
+};
+
+// struct truncation
+// CHECK-LABEL: define void {{.*}}call8
+// CHECK: [[t:%.*]] = alloca %struct.T, align 4
+// CHECK-NEXT: [[s:%.*]] = alloca %struct.S, align 4
+// CHECK-NEXT: [[Tmp:%.*]] = alloca %struct.T, align 4
+// CHECK-NEXT: call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[t]], ptr align 4 {{.*}}, i32 12, i1 false)
+// CHECK-NEXT: call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[Tmp]], ptr align 4 [[t]], i32 12, i1 false)
+// CHECK-NEXT: [[G1:%.*]] = getelementptr inbounds %struct.S, ptr [[s]], i32 0, i32 0
+// CHECK-NEXT: [[G2:%.*]] = getelementptr inbounds %struct.S, ptr [[s]], i32 0, i32 1
+// CHECK-NEXT: [[G3:%.*]] = getelementptr inbounds %struct.T, ptr [[Tmp]], i32 0, i32 0
+// CHECK-NEXT: %gep3 = getelementptr inbounds %struct.T, ptr %agg-temp, i32 0, i32 1
+// CHECK-NEXT: %gep4 = getelementptr inbounds %struct.T, ptr %agg-temp, i32 0, i32 2
+// CHECK-NEXT: %load = load i32, ptr %gep2, align 4
+// CHECK-NEXT: store i32 %load, ptr %gep, align 4
+// CHECK-NEXT: %load5 = load i32, ptr %gep3, align 4
+// CHECK-NEXT: %conv = sitofp i32 %load5 to float
+// CHECK-NEXT: store float %conv, ptr %gep1, align 4
+export void call8() {
+  T t = {1,2,3};
+  S s = (S)t;
+}
diff --git a/clang/test/CodeGenHLSL/BasicFeatures/VectorElementwiseCast.hlsl b/clang/test/CodeGenHLSL/BasicFeatures/VectorElementwiseCast.hlsl
new file mode 100644
index 0000000000000..f579dfb377de5
--- /dev/null
+++ b/clang/test/CodeGenHLSL/BasicFeatures/VectorElementwiseCast.hlsl
@@ -0,0 +1,81 @@
+// RUN: %clang_cc1 -finclude-default-header -triple dxil-pc-shadermodel6.3-library -x hlsl -emit-llvm -disable-llvm-passes -o - %s | FileCheck %s
+
+// vector flat cast from array
+// CHECK-LABEL: define void {{.*}}call2
+// CHECK: [[A:%.*]] = alloca [2 x [1 x i32]], align 4
+// CHECK-NEXT: [[B:%.*]] = alloca <2 x i32>, align 8
+// CHECK-NEXT: [[Tmp:%.*]] = alloca [2 x [1 x i32]], align 4
+// CHECK-NEXT: [[Tmp2:%.*]] = alloca <2 x i32>, align 8
+// CHECK-NEXT: call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[A]], ptr align 4 {{.*}}, i32 8, i1 false)
+// CHECK-NEXT: call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[Tmp]], ptr align 4 [[A]], i32 8, i1 false)
+// CHECK-NEXT: [[G1:%.*]] = getelementptr inbounds [2 x [1 x i32]], ptr [[Tmp]], i32 0, i32 0, i32 0
+// CHECK-NEXT: [[G2:%.*]] = getelementptr inbounds [2 x [1 x i32]], ptr [[Tmp]], i32 0, i32 1, i32 0
+// CHECK-NEXT: [[C:%.*]] = load <2 x i32>, ptr [[Tmp2]], align 8
+// CHECK-NEXT: [[L:%.*]] = load i32, ptr [[G1]], align 4
+// CHECK-NEXT: [[D:%.*]] = insertelement <2 x i32> [[C]], i32 [[L]], i64 0
+// CHECK-NEXT: [[L2:%.*]] = load i32, ptr [[G2]], align 4
+// CHECK-NEXT: [[E:%.*]] = insertelement <2 x i32> [[D]], i32 [[L2]], i64 1
+// CHECK-NEXT: store <2 x i32> [[E]], ptr [[B]], align 8
+export void call2() {
+  int A[2][1] = {{1},{2}};
+  int2 B = (int2)A;
+}
+
+struct S {
+  int X;
+  float Y;
+};
+
+// vector flat cast from struct
+// CHECK-LABEL: define void {{.*}}call3
+// CHECK: [[s:%.*]] = alloca %struct.S, align 4
+// CHECK-NEXT: [[A:%.*]] = alloca <2 x i32>, align 8
+// CHECK-NEXT: [[Tmp:%.*]] = alloca %struct.S, align 4
+// CHECK-NEXT: [[Tmp2:%.*]] = alloca <2 x i32>, align 8
+// CHECK-NEXT: call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[s]], ptr align 4 {{.*}}, i32 8, i1 false)
+// CHECK-NEXT: call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[Tmp]], ptr align 4 [[s]], i32 8, i1 false)
+// CHECK-NEXT: [[G1:%.*]] = getelementptr inbounds %struct.S, ptr [[Tmp]], i32 0, i32 0
+// CHECK-NEXT: [[G2:%.*]] = getelementptr inbounds %struct.S, ptr [[Tmp]], i32 0, i32 1
+// CHECK-NEXT: [[B:%.*]] = load <2 x i32>, ptr [[Tmp2]], align 8
+// CHECK-NEXT: [[L:%.*]] = load i32, ptr [[G1]], align 4
+// CHECK-NEXT: [[C:%.*]] = insertelement <2 x i32> [[B]], i32 [[L]], i64 0
+// CHECK-NEXT: [[L2:%.*]] = load float, ptr [[G2]], align 4
+// CHECK-NEXT: [[D:%.*]] = fptosi float [[L2]] to i32
+// CHECK-NEXT: [[E:%.*]] = insertelement <2 x i32> [[C]], i32 [[D]], i64 1
+// CHECK-NEXT: store <2 x i32> [[E]], ptr [[A]], align 8
+export void call3() {
+  S s = {1, 2.0};
+  int2 A = (int2)s;
+}
+
+// truncate array to scalar
+// CHECK-LABEL: define void {{.*}}call4
+// CHECK: [[A:%.*]] = alloca [2 x i32], align 4
+// CHECK-NEXT: [[B:%.*]] = alloca i32, align 4
+// CHECK-NEXT: [[Tmp:%.*]] = alloca [2 x i32], align 4
+// CHECK-NEXT: call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[A]], ptr align 4 {{.*}}, i32 8, i1 false)
+// CHECK-NEXT: call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[Tmp]], ptr align 4 [[A]], i32 8, i1 false)
+// CHECK-NEXT: [[G1:%.*]] = getelementptr inbounds [2 x i32], ptr [[Tmp]], i32 0, i32 0
+// CHECK-NEXT: [[G2:%.*]] = getelementptr inbounds [2 x i32], ptr [[Tmp]], i32 0, i32 1
+// CHECK-NEXT: [[L:%.*]] = load i32, ptr [[G1]], align 4
+// CHECK-NEXT: store i32 [[L]], ptr [[B]], align 4
+export void call4() {
+ int A[2] = {1,2};
+ int B = (int)A;
+}
+
+// truncate struct to scalar
+// CHECK-LABEL: define void {{.*}}call5
+// CHECK: [[s:%.*]] = alloca %struct.S, align 4
+// CHECK-NEXT: [[A:%.*]] = alloca i32, align 4
+// CHECK-NEXT: [[Tmp:%.*]] = alloca %struct.S, align 4
+// CHECK-NEXT: call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[s]], ptr align 4 {{.*}}, i32 8, i1 false)
+// CHECK-NEXT: call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[Tmp]], ptr align 4 [[s]], i32 8, i1 false)
+// CHECK-NEXT: [[G1:%.*]] = getelementptr inbounds %struct.S, ptr [[Tmp]], i32 0, i32 0
+// CHECK-NEXT: [[G2:%.*]] = getelementptr inbounds %struct.S, ptr [[Tmp]], i32 0, i32 1
+// CHECK-NEXT: [[L:%.*]] = load i32, ptr [[G1]], align 4
+// CHECK-NEXT: store i32 [[L]], ptr [[A]], align 4
+export void call5() {
+ S s = {1, 2.0};
+ int A = (int)s;
+}
diff --git a/clang/test/CodeGenHLSL/builtins/AddUint64.hlsl b/clang/test/CodeGenHLSL/builtins/AddUint64.hlsl
new file mode 100644
index 0000000000000..e1832bdbbf33f
--- /dev/null
+++ b/clang/test/CodeGenHLSL/builtins/AddUint64.hlsl
@@ -0,0 +1,58 @@
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --version 5
+// RUN: %clang_cc1 -finclude-default-header -triple dxil-pc-shadermodel6.3-library %s \
+// RUN:  -emit-llvm -disable-llvm-passes -o - | \
+// RUN:  FileCheck %s --check-prefixes=CHECK
+
+
+// CHECK-LABEL: define noundef <2 x i32> @_Z20test_AddUint64_uint2Dv2_jS_(
+// CHECK-SAME: <2 x i32> noundef [[A:%.*]], <2 x i32> noundef [[B:%.*]]) #[[ATTR0:[0-9]+]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[A_ADDR:%.*]] = alloca <2 x i32>, align 8
+// CHECK-NEXT:    [[B_ADDR:%.*]] = alloca <2 x i32>, align 8
+// CHECK-NEXT:    store <2 x i32> [[A]], ptr [[A_ADDR]], align 8
+// CHECK-NEXT:    store <2 x i32> [[B]], ptr [[B_ADDR]], align 8
+// CHECK-NEXT:    [[TMP0:%.*]] = load <2 x i32>, ptr [[A_ADDR]], align 8
+// CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i32>, ptr [[B_ADDR]], align 8
+// CHECK-NEXT:    [[LOWA:%.*]] = extractelement <2 x i32> [[TMP0]], i64 0
+// CHECK-NEXT:    [[HIGHA:%.*]] = extractelement <2 x i32> [[TMP0]], i64 1
+// CHECK-NEXT:    [[LOWB:%.*]] = extractelement <2 x i32> [[TMP1]], i64 0
+// CHECK-NEXT:    [[HIGHB:%.*]] = extractelement <2 x i32> [[TMP1]], i64 1
+// CHECK-NEXT:    [[TMP2:%.*]] = call { i32, i1 } @llvm.uadd.with.overflow.i32(i32 [[LOWA]], i32 [[LOWB]])
+// CHECK-NEXT:    [[TMP3:%.*]] = extractvalue { i32, i1 } [[TMP2]], 1
+// CHECK-NEXT:    [[TMP4:%.*]] = extractvalue { i32, i1 } [[TMP2]], 0
+// CHECK-NEXT:    [[CARRYZEXT:%.*]] = zext i1 [[TMP3]] to i32
+// CHECK-NEXT:    [[HIGHSUM:%.*]] = add i32 [[HIGHA]], [[HIGHB]]
+// CHECK-NEXT:    [[HIGHSUMPLUSCARRY:%.*]] = add i32 [[HIGHSUM]], [[CARRYZEXT]]
+// CHECK-NEXT:    [[HLSL_ADDUINT64_UPTO0:%.*]] = insertelement <2 x i32> poison, i32 [[TMP4]], i64 0
+// CHECK-NEXT:    [[HLSL_ADDUINT64:%.*]] = insertelement <2 x i32> [[HLSL_ADDUINT64_UPTO0]], i32 [[HIGHSUMPLUSCARRY]], i64 1
+// CHECK-NEXT:    ret <2 x i32> [[HLSL_ADDUINT64]]
+//
+uint2 test_AddUint64_uint2(uint2 a, uint2 b) {
+  return AddUint64(a, b);
+}
+
+// CHECK-LABEL: define noundef <4 x i32> @_Z20test_AddUint64_uint4Dv4_jS_(
+// CHECK-SAME: <4 x i32> noundef [[A:%.*]], <4 x i32> noundef [[B:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[A_ADDR:%.*]] = alloca <4 x i32>, align 16
+// CHECK-NEXT:    [[B_ADDR:%.*]] = alloca <4 x i32>, align 16
+// CHECK-NEXT:    store <4 x i32> [[A]], ptr [[A_ADDR]], align 16
+// CHECK-NEXT:    store <4 x i32> [[B]], ptr [[B_ADDR]], align 16
+// CHECK-NEXT:    [[TMP0:%.*]] = load <4 x i32>, ptr [[A_ADDR]], align 16
+// CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr [[B_ADDR]], align 16
+// CHECK-NEXT:    [[LOWA:%.*]] = shufflevector <4 x i32> [[TMP0]], <4 x i32> poison, <2 x i32> <i32 0, i32 2>
+// CHECK-NEXT:    [[HIGHA:%.*]] = shufflevector <4 x i32> [[TMP0]], <4 x i32> poison, <2 x i32> <i32 1, i32 3>
+// CHECK-NEXT:    [[LOWB:%.*]] = shufflevector <4 x i32> [[TMP1]], <4 x i32> poison, <2 x i32> <i32 0, i32 2>
+// CHECK-NEXT:    [[HIGHB:%.*]] = shufflevector <4 x i32> [[TMP1]], <4 x i32> poison, <2 x i32> <i32 1, i32 3>
+// CHECK-NEXT:    [[TMP2:%.*]] = call { <2 x i32>, <2 x i1> } @llvm.uadd.with.overflow.v2i32(<2 x i32> [[LOWA]], <2 x i32> [[LOWB]])
+// CHECK-NEXT:    [[TMP3:%.*]] = extractvalue { <2 x i32>, <2 x i1> } [[TMP2]], 1
+// CHECK-NEXT:    [[TMP4:%.*]] = extractvalue { <2 x i32>, <2 x i1> } [[TMP2]], 0
+// CHECK-NEXT:    [[CARRYZEXT:%.*]] = zext <2 x i1> [[TMP3]] to <2 x i32>
+// CHECK-NEXT:    [[HIGHSUM:%.*]] = add <2 x i32> [[HIGHA]], [[HIGHB]]
+// CHECK-NEXT:    [[HIGHSUMPLUSCARRY:%.*]] = add <2 x i32> [[HIGHSUM]], [[CARRYZEXT]]
+// CHECK-NEXT:    [[HLSL_ADDUINT64:%.*]] = shufflevector <2 x i32> [[TMP4]], <2 x i32> [[HIGHSUMPLUSCARRY]], <4 x i32> <i32 0, i32 2, i32 1, i32 3>
+// CHECK-NEXT:    ret <4 x i32> [[HLSL_ADDUINT64]]
+//
+uint4 test_AddUint64_uint4(uint4 a, uint4 b) {
+  return AddUint64(a, b);
+}
diff --git a/clang/test/CodeGenHLSL/builtins/WaveActiveMax.hlsl b/clang/test/CodeGenHLSL/builtins/WaveActiveMax.hlsl
new file mode 100644
index 0000000000000..7891cfc1989af
--- /dev/null
+++ b/clang/test/CodeGenHLSL/builtins/WaveActiveMax.hlsl
@@ -0,0 +1,46 @@
+// RUN: %clang_cc1 -std=hlsl2021 -finclude-default-header -triple \
+// RUN:   dxil-pc-shadermodel6.3-compute %s -emit-llvm -disable-llvm-passes -o - | \
+// RUN:   FileCheck %s --check-prefixes=CHECK,CHECK-DXIL
+// RUN: %clang_cc1 -std=hlsl2021 -finclude-default-header -triple \
+// RUN:   spirv-pc-vulkan-compute %s -emit-llvm -disable-llvm-passes -o - | \
+// RUN:   FileCheck %s --check-prefixes=CHECK,CHECK-SPIRV
+
+// Test basic lowering to runtime function call.
+
+// CHECK-LABEL: test_int
+int test_int(int expr) {
+  // CHECK-SPIRV:  %[[RET:.*]] = call spir_func [[TY:.*]] @llvm.spv.wave.reduce.max.i32([[TY]] %[[#]])
+  // CHECK-DXIL:  %[[RET:.*]] = call [[TY:.*]] @llvm.dx.wave.reduce.max.i32([[TY]] %[[#]])
+  // CHECK:  ret [[TY]] %[[RET]]
+  return WaveActiveMax(expr);
+}
+
+// CHECK-DXIL: declare [[TY]] @llvm.dx.wave.reduce.max.i32([[TY]]) #[[#attr:]]
+// CHECK-SPIRV: declare spir_func [[TY]] @llvm.spv.wave.reduce.max.i32([[TY]]) #[[#attr:]]
+
+// CHECK-LABEL: test_uint64_t
+uint64_t test_uint64_t(uint64_t expr) {
+  // CHECK-SPIRV:  %[[RET:.*]] = call spir_func [[TY:.*]] @llvm.spv.wave.reduce.umax.i64([[TY]] %[[#]])
+  // CHECK-DXIL:  %[[RET:.*]] = call [[TY:.*]] @llvm.dx.wave.reduce.umax.i64([[TY]] %[[#]])
+  // CHECK:  ret [[TY]] %[[RET]]
+  return WaveActiveMax(expr);
+}
+
+// CHECK-DXIL: declare [[TY]] @llvm.dx.wave.reduce.umax.i64([[TY]]) #[[#attr:]]
+// CHECK-SPIRV: declare spir_func [[TY]] @llvm.spv.wave.reduce.umax.i64([[TY]]) #[[#attr:]]
+
+// Test basic lowering to runtime function call with array and float value.
+
+// CHECK-LABEL: test_floatv4
+float4 test_floatv4(float4 expr) {
+  // CHECK-SPIRV:  %[[RET1:.*]] = call reassoc nnan ninf nsz arcp afn spir_func [[TY1:.*]] @llvm.spv.wave.reduce.max.v4f32([[TY1]] %[[#]]
+  // CHECK-DXIL:  %[[RET1:.*]] = call reassoc nnan ninf nsz arcp afn [[TY1:.*]] @llvm.dx.wave.reduce.max.v4f32([[TY1]] %[[#]])
+  // CHECK:  ret [[TY1]] %[[RET1]]
+  return WaveActiveMax(expr);
+}
+
+// CHECK-DXIL: declare [[TY1]] @llvm.dx.wave.reduce.max.v4f32([[TY1]]) #[[#attr]]
+// CHECK-SPIRV: declare spir_func [[TY1]] @llvm.spv.wave.reduce.max.v4f32([[TY1]]) #[[#attr]]
+
+// CHECK: attributes #[[#attr]] = {{{.*}} convergent {{.*}}}
+
diff --git a/clang/test/CodeGenHLSL/builtins/dot-builtin.hlsl b/clang/test/CodeGenHLSL/builtins/dot-builtin.hlsl
index 4045169d6516c..36c73f875e944 100644
--- a/clang/test/CodeGenHLSL/builtins/dot-builtin.hlsl
+++ b/clang/test/CodeGenHLSL/builtins/dot-builtin.hlsl
@@ -6,7 +6,7 @@
 // CHECK: %conv2 = fptrunc reassoc nnan ninf nsz arcp afn double %hlsl.dot to float
 // CHECK: ret float %conv2
 float builtin_bool_to_float_type_promotion ( float p0, bool p1 ) {
-  return __builtin_hlsl_dot ( p0, p1 );
+  return __builtin_hlsl_dot ( (double)p0, (double)p1 );
 }
 
 // CHECK-LABEL: builtin_bool_to_float_arg1_type_promotion
@@ -16,7 +16,7 @@ float builtin_bool_to_float_type_promotion ( float p0, bool p1 ) {
 // CHECK: %conv2 = fptrunc reassoc nnan ninf nsz arcp afn double %hlsl.dot to float
 // CHECK: ret float %conv2
 float builtin_bool_to_float_arg1_type_promotion ( bool p0, float p1 ) {
-  return __builtin_hlsl_dot ( p0, p1 );
+  return __builtin_hlsl_dot ( (double)p0, (double)p1 );
 }
 
 // CHECK-LABEL: builtin_dot_int_to_float_promotion
@@ -26,5 +26,5 @@ float builtin_bool_to_float_arg1_type_promotion ( bool p0, float p1 ) {
 // CHECK: %conv2 = fptrunc reassoc nnan ninf nsz arcp afn double %hlsl.dot to float
 // CHECK: ret float %conv2
 float builtin_dot_int_to_float_promotion ( float p0, int p1 ) {
-  return __builtin_hlsl_dot ( p0, p1 );
+  return __builtin_hlsl_dot ( (double)p0, (double)p1 );
 }
diff --git a/clang/test/CodeGenHLSL/cbuf.hlsl b/clang/test/CodeGenHLSL/cbuf.hlsl
index 3f9d4514967dd..825e7b8161a60 100644
--- a/clang/test/CodeGenHLSL/cbuf.hlsl
+++ b/clang/test/CodeGenHLSL/cbuf.hlsl
@@ -1,7 +1,14 @@
-// RUN: %clang_cc1 -finclude-default-header -x hlsl -triple \
-// RUN:   dxil-pc-shadermodel6.3-library %s \
+// RUN: %clang_cc1 -finclude-default-header -triple dxil-pc-shadermodel6.3-library %s \
 // RUN:   -emit-llvm -disable-llvm-passes -o - | FileCheck %s
 
+// RUN: %clang_cc1 -finclude-default-header -triple spirv-pc-vulkan-library %s \
+// RUN:   -emit-llvm -disable-llvm-passes -o - | FileCheck %s
+
+// CHECK: @a = external addrspace(2) externally_initialized global float, align 4
+// CHECK: @b = external addrspace(2) externally_initialized global double, align 8
+// CHECK: @c = external addrspace(2) externally_initialized global float, align 4
+// CHECK: @d = external addrspace(2) externally_initialized global double, align 8
+
 // CHECK: @[[CB:.+]] = external constant { float, double }
 cbuffer A : register(b0, space2) {
   float a;
@@ -15,10 +22,10 @@ tbuffer A : register(t2, space1) {
 }
 
 float foo() {
-// CHECK: load float, ptr @[[CB]], align 4
-// CHECK: load double, ptr getelementptr ({ float, double }, ptr @[[CB]], i32 0, i32 1), align 8
-// CHECK: load float, ptr @[[TB]], align 4
-// CHECK: load double, ptr getelementptr ({ float, double }, ptr @[[TB]], i32 0, i32 1), align 8
+// CHECK: load float, ptr addrspace(2) @a, align 4
+// CHECK: load double, ptr addrspace(2) @b, align 8
+// CHECK: load float, ptr addrspace(2) @c, align 4
+// CHECK: load double, ptr addrspace(2) @d, align 8
   return a + b + c*d;
 }
 
diff --git a/clang/test/CodeGenHLSL/cbuf_in_namespace.hlsl b/clang/test/CodeGenHLSL/cbuf_in_namespace.hlsl
index 73dc376942dfb..13c401d428331 100644
--- a/clang/test/CodeGenHLSL/cbuf_in_namespace.hlsl
+++ b/clang/test/CodeGenHLSL/cbuf_in_namespace.hlsl
@@ -1,8 +1,14 @@
-// RUN: %clang_cc1 -finclude-default-header -x hlsl -triple \
-// RUN:   dxil-pc-shadermodel6.3-library %s \
+// RUN: %clang_cc1 -finclude-default-header -triple dxil-pc-shadermodel6.3-library %s \
+// RUN:   -emit-llvm -disable-llvm-passes -o - | FileCheck %s
+
+// RUN: %clang_cc1 -finclude-default-header -triple spirv-pc-vulkan-library %s \
 // RUN:   -emit-llvm -disable-llvm-passes -o - | FileCheck %s
 
 // Make sure cbuffer inside namespace works.
+
+// CHECK: @_ZN2n02n11aE = external addrspace(2) externally_initialized global float, align 4
+// CHECK: @_ZN2n01bE = external addrspace(2) externally_initialized global float, align 4
+
 // CHECK: @[[CB:.+]] = external constant { float }
 // CHECK: @[[TB:.+]] = external constant { float }
 namespace n0 {
@@ -17,7 +23,7 @@ namespace n1 {
 }
 
 float foo() {
-// CHECK: load float, ptr @[[CB]], align 4
-// CHECK: load float, ptr @[[TB]], align 4
+// CHECK: load float, ptr addrspace(2) @_ZN2n02n11aE, align 4
+// CHECK: load float, ptr addrspace(2) @_ZN2n01bE, align 4
   return n0::n1::a + n0::b;
 }
diff --git a/clang/test/CodeGenHLSL/implicit-norecurse-attrib.hlsl b/clang/test/CodeGenHLSL/implicit-norecurse-attrib.hlsl
index ca01960678175..a8ab6ce98ae7e 100644
--- a/clang/test/CodeGenHLSL/implicit-norecurse-attrib.hlsl
+++ b/clang/test/CodeGenHLSL/implicit-norecurse-attrib.hlsl
@@ -37,7 +37,7 @@ uint Find(Node SortedTree[MAX], uint key) {
 // Imagine the inout works
 export
 bool InitTree(/*inout*/ Node tree[MAX], RWBuffer<uint4> encodedTree, uint maxDepth) {
-  uint size = pow(2.f, maxDepth) - 1;
+  uint size = pow(2.f, (float)maxDepth) - 1;
   if (size > MAX) return false;
   for (uint i = 1; i < size; i++) {
     tree[i].value = encodedTree[i].x;
diff --git a/clang/test/CodeGenHLSL/inline-functions.hlsl b/clang/test/CodeGenHLSL/inline-functions.hlsl
index fa9c88db26dfc..4748eeee7475f 100644
--- a/clang/test/CodeGenHLSL/inline-functions.hlsl
+++ b/clang/test/CodeGenHLSL/inline-functions.hlsl
@@ -1,9 +1,9 @@
-// RUN: %clang_cc1 -triple dxil-pc-shadermodel6.3-library %s -emit-llvm -disable-llvm-passes -o - | FileCheck %s --check-prefixes=CHECK,NOINLINE
-// RUN: %clang_cc1 -triple dxil-pc-shadermodel6.3-library %s -emit-llvm -O0 -o - | FileCheck %s --check-prefixes=CHECK,INLINE
-// RUN: %clang_cc1 -triple dxil-pc-shadermodel6.3-library %s -emit-llvm -O1 -o - | FileCheck %s --check-prefixes=CHECK,INLINE
+// RUN: %clang_cc1 -triple dxil-pc-shadermodel6.3-library %s -emit-llvm -disable-llvm-passes -o - | FileCheck %s --check-prefixes=CHECK,NOINLINE,OPT_ATTR
+// RUN: %clang_cc1 -triple dxil-pc-shadermodel6.3-library %s -emit-llvm -O0 -o - | FileCheck %s --check-prefixes=CHECK,INLINE,OPT_ATTR
+// RUN: %clang_cc1 -triple dxil-pc-shadermodel6.3-library %s -emit-llvm -O1 -o - | FileCheck %s --check-prefixes=CHECK,INLINE,NOOPT_ATTR
 // RUN: %clang_cc1 -triple dxil-pc-shadermodel6.0-compute %s -emit-llvm -disable-llvm-passes -o - | FileCheck %s --check-prefixes=CHECK,NOINLINE
-// RUN: %clang_cc1 -triple dxil-pc-shadermodel6.0-compute %s -emit-llvm -O0 -o - | FileCheck %s --check-prefixes=CHECK,INLINE
-// RUN: %clang_cc1 -triple dxil-pc-shadermodel6.0-compute %s -emit-llvm -O1 -o - | FileCheck %s --check-prefixes=CHECK,INLINE
+// RUN: %clang_cc1 -triple dxil-pc-shadermodel6.0-compute %s -emit-llvm -O0 -o - | FileCheck %s --check-prefixes=CHECK,INLINE,OPT_ATTR
+// RUN: %clang_cc1 -triple dxil-pc-shadermodel6.0-compute %s -emit-llvm -O1 -o - | FileCheck %s --check-prefixes=CHECK,INLINE,NOOPT_ATTR
 
 // Tests that user functions will always be inlined.
 // This includes exported functions and mangled entry point implementation functions.
@@ -43,7 +43,7 @@ void BubbleSort(unsigned Buf[MAX], unsigned size) {
 
 // Note ExtAttr is the inlined export set of attribs
 // CHECK: Function Attrs: alwaysinline
-// CHECK: define noundef i32 @_Z11RemoveDupesA100_jj(ptr {{[a-z_ ]*}}noundef byval([100 x i32]) align 4 %Buf, i32 noundef %size) {{[a-z_ ]*}}[[ExtAttr:\#[0-9]+]]
+// CHECK: define noundef i32 @_Z11RemoveDupesA100_jj(ptr {{[a-z_ ]*}}noundef byval([100 x i32]) align 4 {{.*}}%Buf, i32 noundef %size) {{[a-z_ ]*}}[[ExtAttr:\#[0-9]+]]
 // CHECK: ret i32
 // Sort Buf and remove any duplicate values
 // returns the number of values left
@@ -71,7 +71,8 @@ RWBuffer<unsigned> Indices;
 // NOINLINE: ret void
 
 // The unmangled version is not inlined, EntryAttr reflects that
-// CHECK: Function Attrs: {{.*}}noinline
+// OPT_ATTR: Function Attrs: {{.*}}optnone
+// NOOPT_ATTR-NOT: Function Attrs: {{.*}}optnone
 // CHECK: define void @main() {{[a-z_ ]*}}[[EntryAttr:\#[0-9]+]]
 // Make sure function calls are inlined when AlwaysInline is run
 // This only leaves calls to llvm. intrinsics
@@ -98,7 +99,8 @@ void main(unsigned int GI : SV_GroupIndex) {
 // NOINLINE: ret void
 
 // The unmangled version is not inlined, EntryAttr reflects that
-// CHECK: Function Attrs: {{.*}}noinline
+// OPT_ATTR: Function Attrs: {{.*}}optnone
+// NOOPT_ATTR-NOT: Function Attrs: {{.*}}optnone
 // CHECK: define void @main10() {{[a-z_ ]*}}[[EntryAttr]]
 // Make sure function calls are inlined when AlwaysInline is run
 // This only leaves calls to llvm. intrinsics
diff --git a/clang/test/CodeGenHLSL/resource-bindings.hlsl b/clang/test/CodeGenHLSL/resource-bindings.hlsl
index bfa7896bd9811..57e8cc29572b1 100644
--- a/clang/test/CodeGenHLSL/resource-bindings.hlsl
+++ b/clang/test/CodeGenHLSL/resource-bindings.hlsl
@@ -2,14 +2,17 @@
 
 // CHECK: define internal void @_init_resource_U0S0()
 // CHECK: %U0S0_h = call target("dx.TypedBuffer", <4 x float>, 1, 0, 0) @llvm.dx.resource.handlefrombinding.tdx.TypedBuffer_v4f32_1_0_0t(i32 0, i32 0, i32 1, i32 0, i1 false)
+// CHECK: store target("dx.TypedBuffer", <4 x float>, 1, 0, 0) %U0S0_h, ptr @U0S0, align 4
 RWBuffer<float4> U0S0 : register(u0);
 
 // CHECK: define internal void @_init_resource_U5S3()
 // CHECK: %U5S3_h = call target("dx.TypedBuffer", float, 1, 0, 0) @llvm.dx.resource.handlefrombinding.tdx.TypedBuffer_f32_1_0_0t(i32 3, i32 5, i32 1, i32 0, i1 false)
+// CHECK: store target("dx.TypedBuffer", float, 1, 0, 0) %U5S3_h, ptr @U5S3, align 4
 RWBuffer<float> U5S3 : register(u5, space3);
 
 // CHECK: define internal void @_init_resource_T2S2()
 // CHECK: %T2S2_h = call target("dx.RawBuffer", i32, 0, 0) @llvm.dx.resource.handlefrombinding.tdx.RawBuffer_i32_0_0t(i32 2, i32 2, i32 1, i32 0, i1 false)
+// CHECK: store target("dx.RawBuffer", i32, 0, 0) %T2S2_h, ptr @T2S2, align 4
 StructuredBuffer<int> T2S2 : register(t2, space2);
 struct S {
   float4 f;
@@ -18,6 +21,7 @@ struct S {
 
 // CHECK: define internal void @_init_resource_T3S0()
 // CHECK: %T3S0_h = call target("dx.RawBuffer", %struct.S, 0, 0) @llvm.dx.resource.handlefrombinding.tdx.RawBuffer_s_struct.Ss_0_0t(i32 0, i32 3, i32 1, i32 0, i1 false)
+// CHECK: store target("dx.RawBuffer", %struct.S, 0, 0) %T3S0_h, ptr @T3S0, align 4
 StructuredBuffer<S> T3S0 : register(t3);
 
 // CHECK: define void @main()
diff --git a/clang/test/CodeGenHLSL/static_global_and_function_in_cb.hlsl b/clang/test/CodeGenHLSL/static_global_and_function_in_cb.hlsl
index f85bab2113170..25f51cce2017d 100644
--- a/clang/test/CodeGenHLSL/static_global_and_function_in_cb.hlsl
+++ b/clang/test/CodeGenHLSL/static_global_and_function_in_cb.hlsl
@@ -1,16 +1,21 @@
 // RUN: %clang_cc1 -finclude-default-header -triple dxil-pc-shadermodel6.3-library %s \
 // RUN:   -emit-llvm -disable-llvm-passes -o - | FileCheck %s
 
-// CHECK-DAG: @[[CB:.+]] = external constant { float }
+// RUN: %clang_cc1 -finclude-default-header -triple spirv-pc-vulkan-library %s \
+// RUN:   -emit-llvm -disable-llvm-passes -o - | FileCheck %s
 
 cbuffer A {
-    float a;
-  // CHECK-DAG:@_ZL1b = internal global float 3.000000e+00, align 4
+  // CHECK: @a = external addrspace(2) externally_initialized global float, align 4
+  float a;
+  // CHECK: @_ZL1b = internal global float 3.000000e+00, align 4
   static float b = 3;
-  // CHECK:load float, ptr @[[CB]], align 4
-  // CHECK:load float, ptr @_ZL1b, align 4
   float foo() { return a + b; }
 }
+// CHECK: @[[CB:.+]] = external constant { float }
+
+// CHECK:define {{.*}} float @_Z3foov()
+// CHECK:load float, ptr addrspace(2) @a, align 4
+// CHECK:load float, ptr @_ZL1b, align 4
 
 float bar() {
   return foo();
diff --git a/clang/test/CodeGenObjC/noescape.m b/clang/test/CodeGenObjC/noescape.m
index 395f110251f3c..e1dbc0eb92e54 100644
--- a/clang/test/CodeGenObjC/noescape.m
+++ b/clang/test/CodeGenObjC/noescape.m
@@ -26,37 +26,37 @@
 // CHECK: @[[BLOCK_DESCIPTOR_TMP_2:.*ls32l8"]] = linkonce_odr hidden unnamed_addr constant { i64, i64, ptr, i64 } { i64 0, i64 40, ptr @{{.*}}, i64 256 }, align 8
 
 // CHECK-LABEL: define{{.*}} void @test0(
-// CHECK: call void @noescapeFunc0({{.*}}, {{.*}} nocapture {{.*}})
-// CHECK: declare void @noescapeFunc0(ptr noundef, {{.*}} nocapture noundef)
+// CHECK: call void @noescapeFunc0({{.*}}, {{.*}} captures(none) {{.*}})
+// CHECK: declare void @noescapeFunc0(ptr noundef, {{.*}} noundef captures(none))
 void test0(BlockTy b) {
   noescapeFunc0(0, b);
 }
 
 // CHECK-LABEL: define{{.*}} void @test1(
-// CHECK: call void @noescapeFunc1({{.*}} nocapture {{.*}})
-// CHECK: declare void @noescapeFunc1({{.*}} nocapture noundef)
+// CHECK: call void @noescapeFunc1({{.*}} captures(none) {{.*}})
+// CHECK: declare void @noescapeFunc1({{.*}} noundef captures(none))
 void test1(int *i) {
   noescapeFunc1(i);
 }
 
 // CHECK-LABEL: define{{.*}} void @test2(
-// CHECK: call void @noescapeFunc2({{.*}} nocapture {{.*}})
-// CHECK: declare void @noescapeFunc2({{.*}} nocapture noundef)
+// CHECK: call void @noescapeFunc2({{.*}} captures(none) {{.*}})
+// CHECK: declare void @noescapeFunc2({{.*}} noundef captures(none))
 void test2(id i) {
   noescapeFunc2(i);
 }
 
 // CHECK-LABEL: define{{.*}} void @test3(
-// CHECK: call void @noescapeFunc3({{.*}} nocapture {{.*}})
-// CHECK: declare void @noescapeFunc3({{.*}} nocapture)
+// CHECK: call void @noescapeFunc3({{.*}} captures(none) {{.*}})
+// CHECK: declare void @noescapeFunc3({{.*}} captures(none))
 void test3(union U u) {
   noescapeFunc3(u);
 }
 
-// CHECK: define internal void @"\01-[C0 m0:]"({{.*}}, {{.*}}, {{.*}} nocapture {{.*}})
+// CHECK: define internal void @"\01-[C0 m0:]"({{.*}}, {{.*}}, {{.*}} captures(none) {{.*}})
 
 // CHECK-LABEL: define{{.*}} void @test4(
-// CHECK: call void @objc_msgSend(ptr {{.*}}, ptr {{.*}}, ptr nocapture {{.*}})
+// CHECK: call void @objc_msgSend(ptr {{.*}}, ptr {{.*}}, ptr noundef captures(none) {{.*}})
 
 @interface C0
 -(void) m0:(int*)__attribute__((noescape)) p0;
@@ -72,9 +72,9 @@ void test4(C0 *c0, int *p) {
 }
 
 // CHECK-LABEL: define{{.*}} void @test5(
-// CHECK: call void {{.*}}(ptr noundef @{{.*}}, ptr nocapture {{.*}})
-// CHECK: call void {{.*}}(ptr {{.*}}, ptr nocapture {{.*}})
-// CHECK: define internal void @{{.*}}(ptr {{.*}}, ptr nocapture {{.*}})
+// CHECK: call void {{.*}}(ptr noundef @{{.*}}, ptr noundef captures(none) {{.*}})
+// CHECK: call void {{.*}}(ptr {{.*}}, ptr noundef captures(none) {{.*}})
+// CHECK: define internal void @{{.*}}(ptr {{.*}}, ptr noundef captures(none) {{.*}})
 
 typedef void (^BlockTy2)(__attribute__((noescape)) int *);
 
diff --git a/clang/test/CodeGenOpenCL/amdgpu-abi-struct-coerce.cl b/clang/test/CodeGenOpenCL/amdgpu-abi-struct-coerce.cl
index 35a08a90d8cf9..6776e2227847e 100644
--- a/clang/test/CodeGenOpenCL/amdgpu-abi-struct-coerce.cl
+++ b/clang/test/CodeGenOpenCL/amdgpu-abi-struct-coerce.cl
@@ -307,7 +307,7 @@ void func_single_struct_element_struct_arg(single_struct_element_struct_arg_t ar
 // CHECK: void @func_different_size_type_pair_arg(i64 %arg1.coerce0, i32 %arg1.coerce1)
 void func_different_size_type_pair_arg(different_size_type_pair arg1) { }
 
-// CHECK: void @func_flexible_array_arg(ptr addrspace(5) nocapture noundef readnone byval(%struct.flexible_array) align 4 %arg)
+// CHECK: void @func_flexible_array_arg(ptr addrspace(5) noundef readnone byval(%struct.flexible_array) align 4 captures(none) %arg)
 void func_flexible_array_arg(flexible_array arg) { }
 
 // CHECK: define{{.*}} float @func_f32_ret()
@@ -402,14 +402,14 @@ struct_arr16 func_ret_struct_arr16()
   return s;
 }
 
-// CHECK: define{{.*}} void @func_ret_struct_arr32(ptr addrspace(5) dead_on_unwind noalias nocapture writable writeonly sret(%struct.struct_arr32) align 4 initializes((0, 128)) %agg.result)
+// CHECK: define{{.*}} void @func_ret_struct_arr32(ptr addrspace(5) dead_on_unwind noalias writable writeonly sret(%struct.struct_arr32) align 4 captures(none) initializes((0, 128)) %agg.result)
 struct_arr32 func_ret_struct_arr32()
 {
   struct_arr32 s = { 0 };
   return s;
 }
 
-// CHECK: define{{.*}} void @func_ret_struct_arr33(ptr addrspace(5) dead_on_unwind noalias nocapture writable writeonly sret(%struct.struct_arr33) align 4 initializes((0, 132)) %agg.result)
+// CHECK: define{{.*}} void @func_ret_struct_arr33(ptr addrspace(5) dead_on_unwind noalias writable writeonly sret(%struct.struct_arr33) align 4 captures(none) initializes((0, 132)) %agg.result)
 struct_arr33 func_ret_struct_arr33()
 {
   struct_arr33 s = { 0 };
@@ -438,7 +438,7 @@ different_size_type_pair func_different_size_type_pair_ret()
   return s;
 }
 
-// CHECK: define{{.*}} void @func_flexible_array_ret(ptr addrspace(5) dead_on_unwind noalias nocapture writable writeonly sret(%struct.flexible_array) align 4 initializes((0, 4)) %agg.result)
+// CHECK: define{{.*}} void @func_flexible_array_ret(ptr addrspace(5) dead_on_unwind noalias writable writeonly sret(%struct.flexible_array) align 4 captures(none) initializes((0, 4)) %agg.result)
 flexible_array func_flexible_array_ret()
 {
   flexible_array s = { 0 };
@@ -448,11 +448,11 @@ flexible_array func_flexible_array_ret()
 // CHECK: define{{.*}} void @func_reg_state_lo(<4 x i32> noundef %arg0, <4 x i32> noundef %arg1, <4 x i32> noundef %arg2, i32 noundef %arg3, i32 %s.coerce0, float %s.coerce1, i32 %s.coerce2)
 void func_reg_state_lo(int4 arg0, int4 arg1, int4 arg2, int arg3, struct_arg_t s) { }
 
-// CHECK: define{{.*}} void @func_reg_state_hi(<4 x i32> noundef %arg0, <4 x i32> noundef %arg1, <4 x i32> noundef %arg2, i32 noundef %arg3, i32 noundef %arg4, ptr addrspace(5) nocapture noundef readnone byref(%struct.struct_arg) align 4 %{{.*}})
+// CHECK: define{{.*}} void @func_reg_state_hi(<4 x i32> noundef %arg0, <4 x i32> noundef %arg1, <4 x i32> noundef %arg2, i32 noundef %arg3, i32 noundef %arg4, ptr addrspace(5) noundef readnone byref(%struct.struct_arg) align 4 captures(none) %{{.*}})
 void func_reg_state_hi(int4 arg0, int4 arg1, int4 arg2, int arg3, int arg4, struct_arg_t s) { }
 
 // XXX - Why don't the inner structs flatten?
-// CHECK: define{{.*}} void @func_reg_state_num_regs_nested_struct(<4 x i32> noundef %arg0, i32 noundef %arg1, i32 %arg2.coerce0, %struct.nested %arg2.coerce1, i32 %arg3.coerce0, %struct.nested %arg3.coerce1, ptr addrspace(5) nocapture noundef readnone byref(%struct.num_regs_nested_struct) align 8 %{{.*}})
+// CHECK: define{{.*}} void @func_reg_state_num_regs_nested_struct(<4 x i32> noundef %arg0, i32 noundef %arg1, i32 %arg2.coerce0, %struct.nested %arg2.coerce1, i32 %arg3.coerce0, %struct.nested %arg3.coerce1, ptr addrspace(5) noundef readnone byref(%struct.num_regs_nested_struct) align 8 captures(none) %{{.*}})
 void func_reg_state_num_regs_nested_struct(int4 arg0, int arg1, num_regs_nested_struct arg2, num_regs_nested_struct arg3, num_regs_nested_struct arg4) { }
 
 // CHECK: define{{.*}} void @func_double_nested_struct_arg(<4 x i32> noundef %arg0, i32 noundef %arg1, i32 %arg2.coerce0, %struct.double_nested %arg2.coerce1, i16 %arg2.coerce2)
@@ -467,7 +467,7 @@ double_nested_struct func_double_nested_struct_ret(int4 arg0, int arg1) {
 // CHECK: define{{.*}} void @func_large_struct_padding_arg_direct(i8 %arg.coerce0, i32 %arg.coerce1, i8 %arg.coerce2, i32 %arg.coerce3, i8 %arg.coerce4, i8 %arg.coerce5, i16 %arg.coerce6, i16 %arg.coerce7, [3 x i8] %arg.coerce8, i64 %arg.coerce9, i32 %arg.coerce10, i8 %arg.coerce11, i32 %arg.coerce12, i16 %arg.coerce13, i8 %arg.coerce14)
 void func_large_struct_padding_arg_direct(large_struct_padding arg) { }
 
-// CHECK: define{{.*}} void @func_large_struct_padding_arg_store(ptr addrspace(1) nocapture noundef writeonly initializes((0, 56)) %out, ptr addrspace(5) nocapture noundef readonly byref(%struct.large_struct_padding) align 8 %{{.*}})
+// CHECK: define{{.*}} void @func_large_struct_padding_arg_store(ptr addrspace(1) noundef writeonly captures(none) initializes((0, 56)) %out, ptr addrspace(5) noundef readonly byref(%struct.large_struct_padding) align 8 captures(none) %{{.*}})
 void func_large_struct_padding_arg_store(global large_struct_padding* out, large_struct_padding arg) {
   *out = arg;
 }
@@ -477,7 +477,7 @@ void v3i32_reg_count(int3 arg1, int3 arg2, int3 arg3, int3 arg4, struct_arg_t ar
 
 // Function signature from blender, nothing should be passed byval. The v3i32
 // should not count as 4 passed registers.
-// CHECK: define{{.*}} void @v3i32_pair_reg_count(ptr addrspace(5) nocapture noundef readnone %arg0, <3 x i32> %arg1.coerce0, <3 x i32> %arg1.coerce1, <3 x i32> noundef %arg2, <3 x i32> %arg3.coerce0, <3 x i32> %arg3.coerce1, <3 x i32> noundef %arg4, float noundef %arg5)
+// CHECK: define{{.*}} void @v3i32_pair_reg_count(ptr addrspace(5) noundef readnone captures(none) %arg0, <3 x i32> %arg1.coerce0, <3 x i32> %arg1.coerce1, <3 x i32> noundef %arg2, <3 x i32> %arg3.coerce0, <3 x i32> %arg3.coerce1, <3 x i32> noundef %arg4, float noundef %arg5)
 void v3i32_pair_reg_count(int3_pair *arg0, int3_pair arg1, int3 arg2, int3_pair arg3, int3 arg4, float arg5) { }
 
 // Each short4 should fit pack into 2 registers.
@@ -485,7 +485,7 @@ void v3i32_pair_reg_count(int3_pair *arg0, int3_pair arg1, int3 arg2, int3_pair
 void v4i16_reg_count(short4 arg0, short4 arg1, short4 arg2, short4 arg3,
                      short4 arg4, short4 arg5, struct_4regs arg6) { }
 
-// CHECK: define{{.*}} void @v4i16_pair_reg_count_over(<4 x i16> noundef %arg0, <4 x i16> noundef %arg1, <4 x i16> noundef %arg2, <4 x i16> noundef %arg3, <4 x i16> noundef %arg4, <4 x i16> noundef %arg5, <4 x i16> noundef %arg6, ptr addrspace(5) nocapture noundef readnone byref(%struct.struct_4regs) align 4 %{{.*}})
+// CHECK: define{{.*}} void @v4i16_pair_reg_count_over(<4 x i16> noundef %arg0, <4 x i16> noundef %arg1, <4 x i16> noundef %arg2, <4 x i16> noundef %arg3, <4 x i16> noundef %arg4, <4 x i16> noundef %arg5, <4 x i16> noundef %arg6, ptr addrspace(5) noundef readnone byref(%struct.struct_4regs) align 4 captures(none) %{{.*}})
 void v4i16_pair_reg_count_over(short4 arg0, short4 arg1, short4 arg2, short4 arg3,
                                short4 arg4, short4 arg5, short4 arg6, struct_4regs arg7) { }
 
@@ -493,7 +493,7 @@ void v4i16_pair_reg_count_over(short4 arg0, short4 arg1, short4 arg2, short4 arg
 void v3i16_reg_count(short3 arg0, short3 arg1, short3 arg2, short3 arg3,
                      short3 arg4, short3 arg5, struct_4regs arg6) { }
 
-// CHECK: define{{.*}} void @v3i16_reg_count_over(<3 x i16> noundef %arg0, <3 x i16> noundef %arg1, <3 x i16> noundef %arg2, <3 x i16> noundef %arg3, <3 x i16> noundef %arg4, <3 x i16> noundef %arg5, <3 x i16> noundef %arg6, ptr addrspace(5) nocapture noundef readnone byref(%struct.struct_4regs) align 4 %{{.*}})
+// CHECK: define{{.*}} void @v3i16_reg_count_over(<3 x i16> noundef %arg0, <3 x i16> noundef %arg1, <3 x i16> noundef %arg2, <3 x i16> noundef %arg3, <3 x i16> noundef %arg4, <3 x i16> noundef %arg5, <3 x i16> noundef %arg6, ptr addrspace(5) noundef readnone byref(%struct.struct_4regs) align 4 captures(none) %{{.*}})
 void v3i16_reg_count_over(short3 arg0, short3 arg1, short3 arg2, short3 arg3,
                           short3 arg4, short3 arg5, short3 arg6, struct_4regs arg7) { }
 
@@ -503,7 +503,7 @@ void v2i16_reg_count(short2 arg0, short2 arg1, short2 arg2, short2 arg3,
                      short2 arg8, short2 arg9, short2 arg10, short2 arg11,
                      struct_4regs arg13) { }
 
-// CHECK: define{{.*}} void @v2i16_reg_count_over(<2 x i16> noundef %arg0, <2 x i16> noundef %arg1, <2 x i16> noundef %arg2, <2 x i16> noundef %arg3, <2 x i16> noundef %arg4, <2 x i16> noundef %arg5, <2 x i16> noundef %arg6, <2 x i16> noundef %arg7, <2 x i16> noundef %arg8, <2 x i16> noundef %arg9, <2 x i16> noundef %arg10, <2 x i16> noundef %arg11, <2 x i16> noundef %arg12, ptr addrspace(5) nocapture noundef readnone byref(%struct.struct_4regs) align 4 %{{.*}})
+// CHECK: define{{.*}} void @v2i16_reg_count_over(<2 x i16> noundef %arg0, <2 x i16> noundef %arg1, <2 x i16> noundef %arg2, <2 x i16> noundef %arg3, <2 x i16> noundef %arg4, <2 x i16> noundef %arg5, <2 x i16> noundef %arg6, <2 x i16> noundef %arg7, <2 x i16> noundef %arg8, <2 x i16> noundef %arg9, <2 x i16> noundef %arg10, <2 x i16> noundef %arg11, <2 x i16> noundef %arg12, ptr addrspace(5) noundef readnone byref(%struct.struct_4regs) align 4 captures(none) %{{.*}})
 void v2i16_reg_count_over(short2 arg0, short2 arg1, short2 arg2, short2 arg3,
                           short2 arg4, short2 arg5, short2 arg6, short2 arg7,
                           short2 arg8, short2 arg9, short2 arg10, short2 arg11,
@@ -513,7 +513,7 @@ void v2i16_reg_count_over(short2 arg0, short2 arg1, short2 arg2, short2 arg3,
 void v2i8_reg_count(char2 arg0, char2 arg1, char2 arg2, char2 arg3,
                     char2 arg4, char2 arg5, struct_4regs arg6) { }
 
-// CHECK: define{{.*}} void @v2i8_reg_count_over(<2 x i8> noundef %arg0, <2 x i8> noundef %arg1, <2 x i8> noundef %arg2, <2 x i8> noundef %arg3, <2 x i8> noundef %arg4, <2 x i8> noundef %arg5, i32 noundef %arg6, ptr addrspace(5) nocapture noundef readnone byref(%struct.struct_4regs) align 4 %{{.*}})
+// CHECK: define{{.*}} void @v2i8_reg_count_over(<2 x i8> noundef %arg0, <2 x i8> noundef %arg1, <2 x i8> noundef %arg2, <2 x i8> noundef %arg3, <2 x i8> noundef %arg4, <2 x i8> noundef %arg5, i32 noundef %arg6, ptr addrspace(5) noundef readnone byref(%struct.struct_4regs) align 4 captures(none) %{{.*}})
 void v2i8_reg_count_over(char2 arg0, char2 arg1, char2 arg2, char2 arg3,
                          char2 arg4, char2 arg5, int arg6, struct_4regs arg7) { }
 
diff --git a/clang/test/CodeGenOpenCL/amdgpu-call-kernel.cl b/clang/test/CodeGenOpenCL/amdgpu-call-kernel.cl
index dfca09d034cdb..70a2185320538 100755
--- a/clang/test/CodeGenOpenCL/amdgpu-call-kernel.cl
+++ b/clang/test/CodeGenOpenCL/amdgpu-call-kernel.cl
@@ -1,6 +1,6 @@
 // REQUIRES: amdgpu-registered-target
 // RUN: %clang_cc1 -triple amdgcn-unknown-unknown -emit-llvm -o - %s | FileCheck %s
-// CHECK: define{{.*}} amdgpu_kernel void @test_call_kernel(ptr addrspace(1) nocapture noundef writeonly align 4 initializes((0, 4)) %out)
+// CHECK: define{{.*}} amdgpu_kernel void @test_call_kernel(ptr addrspace(1) noundef writeonly align 4 captures(none) initializes((0, 4)) %out)
 // CHECK: store i32 4, ptr addrspace(1) %out, align 4
 
 kernel void test_kernel(global int *out)
diff --git a/clang/test/CodeGenOpenCL/amdgpu-enqueue-kernel.cl b/clang/test/CodeGenOpenCL/amdgpu-enqueue-kernel.cl
index 5599f4dd50f04..ace34dd0ca6dc 100644
--- a/clang/test/CodeGenOpenCL/amdgpu-enqueue-kernel.cl
+++ b/clang/test/CodeGenOpenCL/amdgpu-enqueue-kernel.cl
@@ -651,7 +651,7 @@ kernel void test_target_features_kernel(global int *i) {
 //
 // GFX900: Function Attrs: convergent nounwind
 // GFX900-LABEL: define {{[^@]+}}@__test_block_invoke_3_kernel
-// GFX900-SAME: (<{ i32, i32, ptr, ptr addrspace(1), ptr addrspace(1), i64, i8 }> [[TMP0:%.*]], ptr addrspace(3) [[TMP1:%.*]]) #[[ATTR6]] !kernel_arg_addr_space [[META28:![0-9]+]] !kernel_arg_access_qual [[META29:![0-9]+]] !kernel_arg_type [[META30:![0-9]+]] !kernel_arg_base_type [[META30]] !kernel_arg_type_qual [[META31:![0-9]+]] {
+// GFX900-SAME: (<{ i32, i32, ptr, ptr addrspace(1), ptr addrspace(1), i64, i8 }> [[TMP0:%.*]], ptr addrspace(3) [[TMP1:%.*]]) #[[ATTR6]] !kernel_arg_addr_space [[META27:![0-9]+]] !kernel_arg_access_qual [[META28:![0-9]+]] !kernel_arg_type [[META29:![0-9]+]] !kernel_arg_base_type [[META29]] !kernel_arg_type_qual [[META30:![0-9]+]] {
 // GFX900-NEXT:  entry:
 // GFX900-NEXT:    [[TMP2:%.*]] = alloca <{ i32, i32, ptr, ptr addrspace(1), ptr addrspace(1), i64, i8 }>, align 8, addrspace(5)
 // GFX900-NEXT:    store <{ i32, i32, ptr, ptr addrspace(1), ptr addrspace(1), i64, i8 }> [[TMP0]], ptr addrspace(5) [[TMP2]], align 8
@@ -688,7 +688,7 @@ kernel void test_target_features_kernel(global int *i) {
 //
 // GFX900: Function Attrs: convergent norecurse nounwind
 // GFX900-LABEL: define {{[^@]+}}@test_target_features_kernel
-// GFX900-SAME: (ptr addrspace(1) noundef align 4 [[I:%.*]]) #[[ATTR2]] !kernel_arg_addr_space [[META32:![0-9]+]] !kernel_arg_access_qual [[META23]] !kernel_arg_type [[META33:![0-9]+]] !kernel_arg_base_type [[META33]] !kernel_arg_type_qual [[META25]] {
+// GFX900-SAME: (ptr addrspace(1) noundef align 4 [[I:%.*]]) #[[ATTR2]] !kernel_arg_addr_space [[META31:![0-9]+]] !kernel_arg_access_qual [[META23]] !kernel_arg_type [[META32:![0-9]+]] !kernel_arg_base_type [[META32]] !kernel_arg_type_qual [[META25]] {
 // GFX900-NEXT:  entry:
 // GFX900-NEXT:    [[I_ADDR:%.*]] = alloca ptr addrspace(1), align 8, addrspace(5)
 // GFX900-NEXT:    [[DEFAULT_QUEUE:%.*]] = alloca ptr addrspace(1), align 8, addrspace(5)
@@ -700,7 +700,7 @@ kernel void test_target_features_kernel(global int *i) {
 // GFX900-NEXT:    [[FLAGS_ASCAST:%.*]] = addrspacecast ptr addrspace(5) [[FLAGS]] to ptr
 // GFX900-NEXT:    [[NDRANGE_ASCAST:%.*]] = addrspacecast ptr addrspace(5) [[NDRANGE]] to ptr
 // GFX900-NEXT:    [[TMP_ASCAST:%.*]] = addrspacecast ptr addrspace(5) [[TMP]] to ptr
-// GFX900-NEXT:    store ptr addrspace(1) [[I]], ptr [[I_ADDR_ASCAST]], align 8, !tbaa [[TBAA34:![0-9]+]]
+// GFX900-NEXT:    store ptr addrspace(1) [[I]], ptr [[I_ADDR_ASCAST]], align 8, !tbaa [[TBAA33:![0-9]+]]
 // GFX900-NEXT:    call void @llvm.lifetime.start.p5(i64 8, ptr addrspace(5) [[DEFAULT_QUEUE]]) #[[ATTR8]]
 // GFX900-NEXT:    call void @llvm.lifetime.start.p5(i64 4, ptr addrspace(5) [[FLAGS]]) #[[ATTR8]]
 // GFX900-NEXT:    store i32 0, ptr [[FLAGS_ASCAST]], align 4, !tbaa [[TBAA17]]
@@ -803,16 +803,15 @@ kernel void test_target_features_kernel(global int *i) {
 // GFX900: [[META23]] = !{!"none"}
 // GFX900: [[META24]] = !{!"__block_literal"}
 // GFX900: [[META25]] = !{!""}
-// GFX900: [[TBAA26]] = !{[[META27:![0-9]+]], [[META27]], i64 0}
-// GFX900: [[META27]] = !{!"p1 void", [[META9]], i64 0}
-// GFX900: [[META28]] = !{i32 0, i32 3}
-// GFX900: [[META29]] = !{!"none", !"none"}
-// GFX900: [[META30]] = !{!"__block_literal", !"void*"}
-// GFX900: [[META31]] = !{!"", !""}
-// GFX900: [[META32]] = !{i32 1}
-// GFX900: [[META33]] = !{!"int*"}
-// GFX900: [[TBAA34]] = !{[[META35:![0-9]+]], [[META35]], i64 0}
-// GFX900: [[META35]] = !{!"p1 int", [[META9]], i64 0}
+// GFX900: [[TBAA26]] = !{[[META9]], [[META9]], i64 0}
+// GFX900: [[META27]] = !{i32 0, i32 3}
+// GFX900: [[META28]] = !{!"none", !"none"}
+// GFX900: [[META29]] = !{!"__block_literal", !"void*"}
+// GFX900: [[META30]] = !{!"", !""}
+// GFX900: [[META31]] = !{i32 1}
+// GFX900: [[META32]] = !{!"int*"}
+// GFX900: [[TBAA33]] = !{[[META34:![0-9]+]], [[META34]], i64 0}
+// GFX900: [[META34]] = !{!"p1 int", [[META9]], i64 0}
 //.
 //// NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
 // CHECK: {{.*}}
diff --git a/clang/test/CodeGenOpenCL/atomic-builtins-default-to-device-scope.cl b/clang/test/CodeGenOpenCL/atomic-builtins-default-to-device-scope.cl
index 5af2d807b4189..0cf961fc572b9 100644
--- a/clang/test/CodeGenOpenCL/atomic-builtins-default-to-device-scope.cl
+++ b/clang/test/CodeGenOpenCL/atomic-builtins-default-to-device-scope.cl
@@ -5,202 +5,202 @@
 // RUN:   | FileCheck %s --check-prefix=SPIRV
 
 // AMDGCN-LABEL: define dso_local i32 @load(
-// AMDGCN-SAME: ptr nocapture noundef readonly [[P:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
+// AMDGCN-SAME: ptr noundef readonly captures(none) [[P:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
 // AMDGCN-NEXT:  [[ENTRY:.*:]]
 // AMDGCN-NEXT:    [[TMP0:%.*]] = load atomic i32, ptr [[P]] syncscope("agent") seq_cst, align 4
 // AMDGCN-NEXT:    ret i32 [[TMP0]]
 //
 // SPIRV-LABEL: define spir_func i32 @load(
-// SPIRV-SAME: ptr addrspace(4) nocapture noundef readonly [[P:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
+// SPIRV-SAME: ptr addrspace(4) noundef readonly captures(none) [[P:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
 // SPIRV-NEXT:  [[ENTRY:.*:]]
 // SPIRV-NEXT:    [[TMP0:%.*]] = load atomic i32, ptr addrspace(4) [[P]] syncscope("device") seq_cst, align 4
 // SPIRV-NEXT:    ret i32 [[TMP0]]
 //
 int load(int *p) { return __atomic_load_n(p, __ATOMIC_SEQ_CST); }
 // AMDGCN-LABEL: define dso_local void @store(
-// AMDGCN-SAME: ptr nocapture noundef writeonly [[P:%.*]], i32 noundef [[X:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// AMDGCN-SAME: ptr noundef writeonly captures(none) [[P:%.*]], i32 noundef [[X:%.*]]) local_unnamed_addr #[[ATTR0]] {
 // AMDGCN-NEXT:  [[ENTRY:.*:]]
 // AMDGCN-NEXT:    store atomic i32 [[X]], ptr [[P]] syncscope("agent") seq_cst, align 4
 // AMDGCN-NEXT:    ret void
 //
 // SPIRV-LABEL: define spir_func void @store(
-// SPIRV-SAME: ptr addrspace(4) nocapture noundef writeonly [[P:%.*]], i32 noundef [[X:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// SPIRV-SAME: ptr addrspace(4) noundef writeonly captures(none) [[P:%.*]], i32 noundef [[X:%.*]]) local_unnamed_addr #[[ATTR0]] {
 // SPIRV-NEXT:  [[ENTRY:.*:]]
 // SPIRV-NEXT:    store atomic i32 [[X]], ptr addrspace(4) [[P]] syncscope("device") seq_cst, align 4
 // SPIRV-NEXT:    ret void
 //
 void store(int *p, int x) { return __atomic_store_n(p, x, __ATOMIC_SEQ_CST); }
 // AMDGCN-LABEL: define dso_local i32 @add(
-// AMDGCN-SAME: ptr nocapture noundef [[P:%.*]], i32 noundef [[X:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// AMDGCN-SAME: ptr noundef captures(none) [[P:%.*]], i32 noundef [[X:%.*]]) local_unnamed_addr #[[ATTR0]] {
 // AMDGCN-NEXT:  [[ENTRY:.*:]]
 // AMDGCN-NEXT:    [[TMP0:%.*]] = atomicrmw add ptr [[P]], i32 [[X]] syncscope("agent") seq_cst, align 4
 // AMDGCN-NEXT:    ret i32 [[TMP0]]
 //
 // SPIRV-LABEL: define spir_func i32 @add(
-// SPIRV-SAME: ptr addrspace(4) nocapture noundef [[P:%.*]], i32 noundef [[X:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// SPIRV-SAME: ptr addrspace(4) noundef captures(none) [[P:%.*]], i32 noundef [[X:%.*]]) local_unnamed_addr #[[ATTR0]] {
 // SPIRV-NEXT:  [[ENTRY:.*:]]
 // SPIRV-NEXT:    [[TMP0:%.*]] = atomicrmw add ptr addrspace(4) [[P]], i32 [[X]] syncscope("device") seq_cst, align 4
 // SPIRV-NEXT:    ret i32 [[TMP0]]
 //
 int add(int *p, int x) { return __atomic_fetch_add(p, x, __ATOMIC_SEQ_CST); }
 // AMDGCN-LABEL: define dso_local float @fadd(
-// AMDGCN-SAME: ptr nocapture noundef [[P:%.*]], float noundef [[X:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// AMDGCN-SAME: ptr noundef captures(none) [[P:%.*]], float noundef [[X:%.*]]) local_unnamed_addr #[[ATTR0]] {
 // AMDGCN-NEXT:  [[ENTRY:.*:]]
 // AMDGCN-NEXT:    [[TMP0:%.*]] = atomicrmw fadd ptr [[P]], float [[X]] syncscope("agent") seq_cst, align 4
 // AMDGCN-NEXT:    ret float [[TMP0]]
 //
 // SPIRV-LABEL: define spir_func float @fadd(
-// SPIRV-SAME: ptr addrspace(4) nocapture noundef [[P:%.*]], float noundef [[X:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// SPIRV-SAME: ptr addrspace(4) noundef captures(none) [[P:%.*]], float noundef [[X:%.*]]) local_unnamed_addr #[[ATTR0]] {
 // SPIRV-NEXT:  [[ENTRY:.*:]]
 // SPIRV-NEXT:    [[TMP0:%.*]] = atomicrmw fadd ptr addrspace(4) [[P]], float [[X]] syncscope("device") seq_cst, align 4
 // SPIRV-NEXT:    ret float [[TMP0]]
 //
 float fadd(float *p, float x) { return __atomic_fetch_add(p, x, __ATOMIC_SEQ_CST); }
 // AMDGCN-LABEL: define dso_local i32 @sub(
-// AMDGCN-SAME: ptr nocapture noundef [[P:%.*]], i32 noundef [[X:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// AMDGCN-SAME: ptr noundef captures(none) [[P:%.*]], i32 noundef [[X:%.*]]) local_unnamed_addr #[[ATTR0]] {
 // AMDGCN-NEXT:  [[ENTRY:.*:]]
 // AMDGCN-NEXT:    [[TMP0:%.*]] = atomicrmw sub ptr [[P]], i32 [[X]] syncscope("agent") seq_cst, align 4
 // AMDGCN-NEXT:    ret i32 [[TMP0]]
 //
 // SPIRV-LABEL: define spir_func i32 @sub(
-// SPIRV-SAME: ptr addrspace(4) nocapture noundef [[P:%.*]], i32 noundef [[X:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// SPIRV-SAME: ptr addrspace(4) noundef captures(none) [[P:%.*]], i32 noundef [[X:%.*]]) local_unnamed_addr #[[ATTR0]] {
 // SPIRV-NEXT:  [[ENTRY:.*:]]
 // SPIRV-NEXT:    [[TMP0:%.*]] = atomicrmw sub ptr addrspace(4) [[P]], i32 [[X]] syncscope("device") seq_cst, align 4
 // SPIRV-NEXT:    ret i32 [[TMP0]]
 //
 int sub(int *p, int x) { return __atomic_fetch_sub(p, x, __ATOMIC_SEQ_CST); }
 // AMDGCN-LABEL: define dso_local float @fsub(
-// AMDGCN-SAME: ptr nocapture noundef [[P:%.*]], float noundef [[X:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// AMDGCN-SAME: ptr noundef captures(none) [[P:%.*]], float noundef [[X:%.*]]) local_unnamed_addr #[[ATTR0]] {
 // AMDGCN-NEXT:  [[ENTRY:.*:]]
 // AMDGCN-NEXT:    [[TMP0:%.*]] = atomicrmw fsub ptr [[P]], float [[X]] syncscope("agent") seq_cst, align 4
 // AMDGCN-NEXT:    ret float [[TMP0]]
 //
 // SPIRV-LABEL: define spir_func float @fsub(
-// SPIRV-SAME: ptr addrspace(4) nocapture noundef [[P:%.*]], float noundef [[X:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// SPIRV-SAME: ptr addrspace(4) noundef captures(none) [[P:%.*]], float noundef [[X:%.*]]) local_unnamed_addr #[[ATTR0]] {
 // SPIRV-NEXT:  [[ENTRY:.*:]]
 // SPIRV-NEXT:    [[TMP0:%.*]] = atomicrmw fsub ptr addrspace(4) [[P]], float [[X]] syncscope("device") seq_cst, align 4
 // SPIRV-NEXT:    ret float [[TMP0]]
 //
 float fsub(float *p, float x) { return __atomic_fetch_sub(p, x, __ATOMIC_SEQ_CST); }
 // AMDGCN-LABEL: define dso_local i32 @and(
-// AMDGCN-SAME: ptr nocapture noundef [[P:%.*]], i32 noundef [[X:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// AMDGCN-SAME: ptr noundef captures(none) [[P:%.*]], i32 noundef [[X:%.*]]) local_unnamed_addr #[[ATTR0]] {
 // AMDGCN-NEXT:  [[ENTRY:.*:]]
 // AMDGCN-NEXT:    [[TMP0:%.*]] = atomicrmw and ptr [[P]], i32 [[X]] syncscope("agent") seq_cst, align 4
 // AMDGCN-NEXT:    ret i32 [[TMP0]]
 //
 // SPIRV-LABEL: define spir_func i32 @and(
-// SPIRV-SAME: ptr addrspace(4) nocapture noundef [[P:%.*]], i32 noundef [[X:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// SPIRV-SAME: ptr addrspace(4) noundef captures(none) [[P:%.*]], i32 noundef [[X:%.*]]) local_unnamed_addr #[[ATTR0]] {
 // SPIRV-NEXT:  [[ENTRY:.*:]]
 // SPIRV-NEXT:    [[TMP0:%.*]] = atomicrmw and ptr addrspace(4) [[P]], i32 [[X]] syncscope("device") seq_cst, align 4
 // SPIRV-NEXT:    ret i32 [[TMP0]]
 //
 int and(int *p, int x) { return __atomic_fetch_and(p, x, __ATOMIC_SEQ_CST); }
 // AMDGCN-LABEL: define dso_local i32 @nand(
-// AMDGCN-SAME: ptr nocapture noundef [[P:%.*]], i32 noundef [[X:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// AMDGCN-SAME: ptr noundef captures(none) [[P:%.*]], i32 noundef [[X:%.*]]) local_unnamed_addr #[[ATTR0]] {
 // AMDGCN-NEXT:  [[ENTRY:.*:]]
 // AMDGCN-NEXT:    [[TMP0:%.*]] = atomicrmw nand ptr [[P]], i32 [[X]] syncscope("agent") seq_cst, align 4
 // AMDGCN-NEXT:    ret i32 [[TMP0]]
 //
 // SPIRV-LABEL: define spir_func i32 @nand(
-// SPIRV-SAME: ptr addrspace(4) nocapture noundef [[P:%.*]], i32 noundef [[X:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// SPIRV-SAME: ptr addrspace(4) noundef captures(none) [[P:%.*]], i32 noundef [[X:%.*]]) local_unnamed_addr #[[ATTR0]] {
 // SPIRV-NEXT:  [[ENTRY:.*:]]
 // SPIRV-NEXT:    [[TMP0:%.*]] = atomicrmw nand ptr addrspace(4) [[P]], i32 [[X]] syncscope("device") seq_cst, align 4
 // SPIRV-NEXT:    ret i32 [[TMP0]]
 //
 int nand(int *p, int x) { return __atomic_fetch_nand(p, x, __ATOMIC_SEQ_CST); }
 // AMDGCN-LABEL: define dso_local i32 @or(
-// AMDGCN-SAME: ptr nocapture noundef [[P:%.*]], i32 noundef [[X:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// AMDGCN-SAME: ptr noundef captures(none) [[P:%.*]], i32 noundef [[X:%.*]]) local_unnamed_addr #[[ATTR0]] {
 // AMDGCN-NEXT:  [[ENTRY:.*:]]
 // AMDGCN-NEXT:    [[TMP0:%.*]] = atomicrmw or ptr [[P]], i32 [[X]] syncscope("agent") seq_cst, align 4
 // AMDGCN-NEXT:    ret i32 [[TMP0]]
 //
 // SPIRV-LABEL: define spir_func i32 @or(
-// SPIRV-SAME: ptr addrspace(4) nocapture noundef [[P:%.*]], i32 noundef [[X:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// SPIRV-SAME: ptr addrspace(4) noundef captures(none) [[P:%.*]], i32 noundef [[X:%.*]]) local_unnamed_addr #[[ATTR0]] {
 // SPIRV-NEXT:  [[ENTRY:.*:]]
 // SPIRV-NEXT:    [[TMP0:%.*]] = atomicrmw or ptr addrspace(4) [[P]], i32 [[X]] syncscope("device") seq_cst, align 4
 // SPIRV-NEXT:    ret i32 [[TMP0]]
 //
 int or(int *p, int x) { return __atomic_fetch_or(p, x, __ATOMIC_SEQ_CST); }
 // AMDGCN-LABEL: define dso_local i32 @xor(
-// AMDGCN-SAME: ptr nocapture noundef [[P:%.*]], i32 noundef [[X:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// AMDGCN-SAME: ptr noundef captures(none) [[P:%.*]], i32 noundef [[X:%.*]]) local_unnamed_addr #[[ATTR0]] {
 // AMDGCN-NEXT:  [[ENTRY:.*:]]
 // AMDGCN-NEXT:    [[TMP0:%.*]] = atomicrmw xor ptr [[P]], i32 [[X]] syncscope("agent") seq_cst, align 4
 // AMDGCN-NEXT:    ret i32 [[TMP0]]
 //
 // SPIRV-LABEL: define spir_func i32 @xor(
-// SPIRV-SAME: ptr addrspace(4) nocapture noundef [[P:%.*]], i32 noundef [[X:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// SPIRV-SAME: ptr addrspace(4) noundef captures(none) [[P:%.*]], i32 noundef [[X:%.*]]) local_unnamed_addr #[[ATTR0]] {
 // SPIRV-NEXT:  [[ENTRY:.*:]]
 // SPIRV-NEXT:    [[TMP0:%.*]] = atomicrmw xor ptr addrspace(4) [[P]], i32 [[X]] syncscope("device") seq_cst, align 4
 // SPIRV-NEXT:    ret i32 [[TMP0]]
 //
 int xor(int *p, int x) { return __atomic_fetch_xor(p, x, __ATOMIC_SEQ_CST); }
 // AMDGCN-LABEL: define dso_local i32 @min(
-// AMDGCN-SAME: ptr nocapture noundef [[P:%.*]], i32 noundef [[X:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// AMDGCN-SAME: ptr noundef captures(none) [[P:%.*]], i32 noundef [[X:%.*]]) local_unnamed_addr #[[ATTR0]] {
 // AMDGCN-NEXT:  [[ENTRY:.*:]]
 // AMDGCN-NEXT:    [[TMP0:%.*]] = atomicrmw min ptr [[P]], i32 [[X]] syncscope("agent") seq_cst, align 4
 // AMDGCN-NEXT:    ret i32 [[TMP0]]
 //
 // SPIRV-LABEL: define spir_func i32 @min(
-// SPIRV-SAME: ptr addrspace(4) nocapture noundef [[P:%.*]], i32 noundef [[X:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// SPIRV-SAME: ptr addrspace(4) noundef captures(none) [[P:%.*]], i32 noundef [[X:%.*]]) local_unnamed_addr #[[ATTR0]] {
 // SPIRV-NEXT:  [[ENTRY:.*:]]
 // SPIRV-NEXT:    [[TMP0:%.*]] = atomicrmw min ptr addrspace(4) [[P]], i32 [[X]] syncscope("device") seq_cst, align 4
 // SPIRV-NEXT:    ret i32 [[TMP0]]
 //
 int min(int *p, int x) { return __atomic_fetch_min(p, x, __ATOMIC_SEQ_CST); }
 // AMDGCN-LABEL: define dso_local float @fmin(
-// AMDGCN-SAME: ptr nocapture noundef [[P:%.*]], float noundef [[X:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// AMDGCN-SAME: ptr noundef captures(none) [[P:%.*]], float noundef [[X:%.*]]) local_unnamed_addr #[[ATTR0]] {
 // AMDGCN-NEXT:  [[ENTRY:.*:]]
 // AMDGCN-NEXT:    [[TMP0:%.*]] = atomicrmw fmin ptr [[P]], float [[X]] syncscope("agent") seq_cst, align 4
 // AMDGCN-NEXT:    ret float [[TMP0]]
 //
 // SPIRV-LABEL: define spir_func float @fmin(
-// SPIRV-SAME: ptr addrspace(4) nocapture noundef [[P:%.*]], float noundef [[X:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// SPIRV-SAME: ptr addrspace(4) noundef captures(none) [[P:%.*]], float noundef [[X:%.*]]) local_unnamed_addr #[[ATTR0]] {
 // SPIRV-NEXT:  [[ENTRY:.*:]]
 // SPIRV-NEXT:    [[TMP0:%.*]] = atomicrmw fmin ptr addrspace(4) [[P]], float [[X]] syncscope("device") seq_cst, align 4
 // SPIRV-NEXT:    ret float [[TMP0]]
 //
 float fmin(float *p, float x) { return __atomic_fetch_min(p, x, __ATOMIC_SEQ_CST); }
 // AMDGCN-LABEL: define dso_local i32 @max(
-// AMDGCN-SAME: ptr nocapture noundef [[P:%.*]], i32 noundef [[X:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// AMDGCN-SAME: ptr noundef captures(none) [[P:%.*]], i32 noundef [[X:%.*]]) local_unnamed_addr #[[ATTR0]] {
 // AMDGCN-NEXT:  [[ENTRY:.*:]]
 // AMDGCN-NEXT:    [[TMP0:%.*]] = atomicrmw max ptr [[P]], i32 [[X]] syncscope("agent") seq_cst, align 4
 // AMDGCN-NEXT:    ret i32 [[TMP0]]
 //
 // SPIRV-LABEL: define spir_func i32 @max(
-// SPIRV-SAME: ptr addrspace(4) nocapture noundef [[P:%.*]], i32 noundef [[X:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// SPIRV-SAME: ptr addrspace(4) noundef captures(none) [[P:%.*]], i32 noundef [[X:%.*]]) local_unnamed_addr #[[ATTR0]] {
 // SPIRV-NEXT:  [[ENTRY:.*:]]
 // SPIRV-NEXT:    [[TMP0:%.*]] = atomicrmw max ptr addrspace(4) [[P]], i32 [[X]] syncscope("device") seq_cst, align 4
 // SPIRV-NEXT:    ret i32 [[TMP0]]
 //
 int max(int *p, int x) { return __atomic_fetch_max(p, x, __ATOMIC_SEQ_CST); }
 // AMDGCN-LABEL: define dso_local float @fmax(
-// AMDGCN-SAME: ptr nocapture noundef [[P:%.*]], float noundef [[X:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// AMDGCN-SAME: ptr noundef captures(none) [[P:%.*]], float noundef [[X:%.*]]) local_unnamed_addr #[[ATTR0]] {
 // AMDGCN-NEXT:  [[ENTRY:.*:]]
 // AMDGCN-NEXT:    [[TMP0:%.*]] = atomicrmw fmax ptr [[P]], float [[X]] syncscope("agent") seq_cst, align 4
 // AMDGCN-NEXT:    ret float [[TMP0]]
 //
 // SPIRV-LABEL: define spir_func float @fmax(
-// SPIRV-SAME: ptr addrspace(4) nocapture noundef [[P:%.*]], float noundef [[X:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// SPIRV-SAME: ptr addrspace(4) noundef captures(none) [[P:%.*]], float noundef [[X:%.*]]) local_unnamed_addr #[[ATTR0]] {
 // SPIRV-NEXT:  [[ENTRY:.*:]]
 // SPIRV-NEXT:    [[TMP0:%.*]] = atomicrmw fmax ptr addrspace(4) [[P]], float [[X]] syncscope("device") seq_cst, align 4
 // SPIRV-NEXT:    ret float [[TMP0]]
 //
 float fmax(float *p, float x) { return __atomic_fetch_max(p, x, __ATOMIC_SEQ_CST); }
 // AMDGCN-LABEL: define dso_local i32 @xchg(
-// AMDGCN-SAME: ptr nocapture noundef [[P:%.*]], i32 noundef [[X:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// AMDGCN-SAME: ptr noundef captures(none) [[P:%.*]], i32 noundef [[X:%.*]]) local_unnamed_addr #[[ATTR0]] {
 // AMDGCN-NEXT:  [[ENTRY:.*:]]
 // AMDGCN-NEXT:    [[TMP0:%.*]] = atomicrmw xchg ptr [[P]], i32 [[X]] syncscope("agent") seq_cst, align 4
 // AMDGCN-NEXT:    ret i32 [[TMP0]]
 //
 // SPIRV-LABEL: define spir_func i32 @xchg(
-// SPIRV-SAME: ptr addrspace(4) nocapture noundef [[P:%.*]], i32 noundef [[X:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// SPIRV-SAME: ptr addrspace(4) noundef captures(none) [[P:%.*]], i32 noundef [[X:%.*]]) local_unnamed_addr #[[ATTR0]] {
 // SPIRV-NEXT:  [[ENTRY:.*:]]
 // SPIRV-NEXT:    [[TMP0:%.*]] = atomicrmw xchg ptr addrspace(4) [[P]], i32 [[X]] syncscope("device") seq_cst, align 4
 // SPIRV-NEXT:    ret i32 [[TMP0]]
 //
 int xchg(int *p, int x) { return __atomic_exchange_n(p, x, __ATOMIC_SEQ_CST); }
 // AMDGCN-LABEL: define dso_local range(i32 0, 2) i32 @cmpxchg(
-// AMDGCN-SAME: ptr nocapture noundef [[P:%.*]], i32 noundef [[X:%.*]], i32 noundef [[Y:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// AMDGCN-SAME: ptr noundef captures(none) [[P:%.*]], i32 noundef [[X:%.*]], i32 noundef [[Y:%.*]]) local_unnamed_addr #[[ATTR0]] {
 // AMDGCN-NEXT:  [[ENTRY:.*:]]
 // AMDGCN-NEXT:    [[TMP0:%.*]] = cmpxchg ptr [[P]], i32 [[X]], i32 [[Y]] syncscope("agent") seq_cst seq_cst, align 4
 // AMDGCN-NEXT:    [[TMP1:%.*]] = extractvalue { i32, i1 } [[TMP0]], 1
@@ -208,7 +208,7 @@ int xchg(int *p, int x) { return __atomic_exchange_n(p, x, __ATOMIC_SEQ_CST); }
 // AMDGCN-NEXT:    ret i32 [[CONV]]
 //
 // SPIRV-LABEL: define spir_func range(i32 0, 2) i32 @cmpxchg(
-// SPIRV-SAME: ptr addrspace(4) nocapture noundef [[P:%.*]], i32 noundef [[X:%.*]], i32 noundef [[Y:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// SPIRV-SAME: ptr addrspace(4) noundef captures(none) [[P:%.*]], i32 noundef [[X:%.*]], i32 noundef [[Y:%.*]]) local_unnamed_addr #[[ATTR0]] {
 // SPIRV-NEXT:  [[ENTRY:.*:]]
 // SPIRV-NEXT:    [[TMP0:%.*]] = cmpxchg ptr addrspace(4) [[P]], i32 [[X]], i32 [[Y]] syncscope("device") seq_cst seq_cst, align 4
 // SPIRV-NEXT:    [[TMP1:%.*]] = extractvalue { i32, i1 } [[TMP0]], 1
@@ -217,7 +217,7 @@ int xchg(int *p, int x) { return __atomic_exchange_n(p, x, __ATOMIC_SEQ_CST); }
 //
 int cmpxchg(int *p, int x, int y) { return __atomic_compare_exchange(p, &x, &y, 0, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST); }
 // AMDGCN-LABEL: define dso_local range(i32 0, 2) i32 @cmpxchg_weak(
-// AMDGCN-SAME: ptr nocapture noundef [[P:%.*]], i32 noundef [[X:%.*]], i32 noundef [[Y:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// AMDGCN-SAME: ptr noundef captures(none) [[P:%.*]], i32 noundef [[X:%.*]], i32 noundef [[Y:%.*]]) local_unnamed_addr #[[ATTR0]] {
 // AMDGCN-NEXT:  [[ENTRY:.*:]]
 // AMDGCN-NEXT:    [[TMP0:%.*]] = cmpxchg weak ptr [[P]], i32 [[X]], i32 [[Y]] syncscope("agent") seq_cst seq_cst, align 4
 // AMDGCN-NEXT:    [[TMP1:%.*]] = extractvalue { i32, i1 } [[TMP0]], 1
@@ -225,7 +225,7 @@ int cmpxchg(int *p, int x, int y) { return __atomic_compare_exchange(p, &x, &y,
 // AMDGCN-NEXT:    ret i32 [[CONV]]
 //
 // SPIRV-LABEL: define spir_func range(i32 0, 2) i32 @cmpxchg_weak(
-// SPIRV-SAME: ptr addrspace(4) nocapture noundef [[P:%.*]], i32 noundef [[X:%.*]], i32 noundef [[Y:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// SPIRV-SAME: ptr addrspace(4) noundef captures(none) [[P:%.*]], i32 noundef [[X:%.*]], i32 noundef [[Y:%.*]]) local_unnamed_addr #[[ATTR0]] {
 // SPIRV-NEXT:  [[ENTRY:.*:]]
 // SPIRV-NEXT:    [[TMP0:%.*]] = cmpxchg weak ptr addrspace(4) [[P]], i32 [[X]], i32 [[Y]] syncscope("device") seq_cst seq_cst, align 4
 // SPIRV-NEXT:    [[TMP1:%.*]] = extractvalue { i32, i1 } [[TMP0]], 1
diff --git a/clang/test/CodeGenOpenCL/builtins-amdgcn-gfx11-err.cl b/clang/test/CodeGenOpenCL/builtins-amdgcn-gfx11-err.cl
index 08f70a25276f1..d518fe3a11a81 100644
--- a/clang/test/CodeGenOpenCL/builtins-amdgcn-gfx11-err.cl
+++ b/clang/test/CodeGenOpenCL/builtins-amdgcn-gfx11-err.cl
@@ -2,7 +2,7 @@
 
 // RUN: %clang_cc1 -triple amdgcn-unknown-unknown -target-cpu gfx1100 -verify -emit-llvm -o - %s
 
-void test_s_sleep_var(int d)
-{
-  __builtin_amdgcn_s_sleep_var(d); // expected-error {{'__builtin_amdgcn_s_sleep_var' needs target feature gfx12-insts}}
+void builtin_test_unsupported(int a, int b) {
+  __builtin_amdgcn_s_sleep_var(a); // expected-error {{'__builtin_amdgcn_s_sleep_var' needs target feature gfx12-insts}}
+  b = __builtin_amdgcn_ds_bpermute_fi_b32(a, b); // expected-error {{'__builtin_amdgcn_ds_bpermute_fi_b32' needs target feature gfx12-insts}}
 }
diff --git a/clang/test/CodeGenOpenCL/builtins-amdgcn-gfx12.cl b/clang/test/CodeGenOpenCL/builtins-amdgcn-gfx12.cl
index 5b5ae419f0a4a..234ad4fd8cde6 100644
--- a/clang/test/CodeGenOpenCL/builtins-amdgcn-gfx12.cl
+++ b/clang/test/CodeGenOpenCL/builtins-amdgcn-gfx12.cl
@@ -296,3 +296,26 @@ void test_s_buffer_prefetch_data(__amdgpu_buffer_rsrc_t rsrc, unsigned int len)
   __builtin_amdgcn_s_buffer_prefetch_data(rsrc, 128, len);
   __builtin_amdgcn_s_buffer_prefetch_data(rsrc, 0, 31);
 }
+
+// CHECK-LABEL: @test_ds_bpermute_fi_b32(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[OUT_ADDR:%.*]] = alloca ptr addrspace(1), align 8, addrspace(5)
+// CHECK-NEXT:    [[A_ADDR:%.*]] = alloca i32, align 4, addrspace(5)
+// CHECK-NEXT:    [[B_ADDR:%.*]] = alloca i32, align 4, addrspace(5)
+// CHECK-NEXT:    [[OUT_ADDR_ASCAST:%.*]] = addrspacecast ptr addrspace(5) [[OUT_ADDR]] to ptr
+// CHECK-NEXT:    [[A_ADDR_ASCAST:%.*]] = addrspacecast ptr addrspace(5) [[A_ADDR]] to ptr
+// CHECK-NEXT:    [[B_ADDR_ASCAST:%.*]] = addrspacecast ptr addrspace(5) [[B_ADDR]] to ptr
+// CHECK-NEXT:    store ptr addrspace(1) [[OUT:%.*]], ptr [[OUT_ADDR_ASCAST]], align 8
+// CHECK-NEXT:    store i32 [[A:%.*]], ptr [[A_ADDR_ASCAST]], align 4
+// CHECK-NEXT:    store i32 [[B:%.*]], ptr [[B_ADDR_ASCAST]], align 4
+// CHECK-NEXT:    [[TMP0:%.*]] = load i32, ptr [[A_ADDR_ASCAST]], align 4
+// CHECK-NEXT:    [[TMP1:%.*]] = load i32, ptr [[B_ADDR_ASCAST]], align 4
+// CHECK-NEXT:    [[TMP2:%.*]] = call i32 @llvm.amdgcn.ds.bpermute.fi.b32(i32 [[TMP0]], i32 [[TMP1]])
+// CHECK-NEXT:    [[TMP3:%.*]] = load ptr addrspace(1), ptr [[OUT_ADDR_ASCAST]], align 8
+// CHECK-NEXT:    store i32 [[TMP2]], ptr addrspace(1) [[TMP3]], align 4
+// CHECK-NEXT:    ret void
+//
+void test_ds_bpermute_fi_b32(global int* out, int a, int b)
+{
+  *out = __builtin_amdgcn_ds_bpermute_fi_b32(a, b);
+}
diff --git a/clang/test/CodeGenOpenCL/builtins-amdgcn-gfx950-read-tr.cl b/clang/test/CodeGenOpenCL/builtins-amdgcn-gfx950-read-tr.cl
index 91e04430d4973..af1f434403767 100644
--- a/clang/test/CodeGenOpenCL/builtins-amdgcn-gfx950-read-tr.cl
+++ b/clang/test/CodeGenOpenCL/builtins-amdgcn-gfx950-read-tr.cl
@@ -8,7 +8,7 @@ typedef half   v4h   __attribute__((ext_vector_type(4)));
 typedef __bf16 v4y   __attribute__((ext_vector_type(4)));
 
 // GFX950-LABEL: define dso_local <2 x i32> @test_amdgcn_ds_read_b64_tr_b4_v2i32(
-// GFX950-SAME: ptr addrspace(3) nocapture noundef readonly [[INPTR:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
+// GFX950-SAME: ptr addrspace(3) noundef readonly captures(none) [[INPTR:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
 // GFX950-NEXT:  entry:
 // GFX950-NEXT:    [[TMP0:%.*]] = tail call <2 x i32> @llvm.amdgcn.ds.read.tr4.b64.v2i32(ptr addrspace(3) [[INPTR]])
 // GFX950-NEXT:    ret <2 x i32> [[TMP0]]
@@ -19,7 +19,7 @@ v2i test_amdgcn_ds_read_b64_tr_b4_v2i32(local v2i* inptr)
 }
 
 // GFX950-LABEL: define dso_local <3 x i32> @test_amdgcn_ds_read_b96_tr_b6_v3i32(
-// GFX950-SAME: ptr addrspace(3) nocapture noundef readonly [[INPTR:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
+// GFX950-SAME: ptr addrspace(3) noundef readonly captures(none) [[INPTR:%.*]]) local_unnamed_addr #[[ATTR0]] {
 // GFX950-NEXT:  entry:
 // GFX950-NEXT:    [[TMP0:%.*]] = tail call <3 x i32> @llvm.amdgcn.ds.read.tr6.b96.v3i32(ptr addrspace(3) [[INPTR]])
 // GFX950-NEXT:    ret <3 x i32> [[TMP0]]
@@ -30,7 +30,7 @@ v3i test_amdgcn_ds_read_b96_tr_b6_v3i32(local v3i* inptr)
 }
 
 // GFX950-LABEL: define dso_local <2 x i32> @test_amdgcn_ds_read_b64_tr_b8_v2i32(
-// GFX950-SAME: ptr addrspace(3) nocapture noundef readonly [[INPTR:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
+// GFX950-SAME: ptr addrspace(3) noundef readonly captures(none) [[INPTR:%.*]]) local_unnamed_addr #[[ATTR0]] {
 // GFX950-NEXT:  entry:
 // GFX950-NEXT:    [[TMP0:%.*]] = tail call <2 x i32> @llvm.amdgcn.ds.read.tr8.b64.v2i32(ptr addrspace(3) [[INPTR]])
 // GFX950-NEXT:    ret <2 x i32> [[TMP0]]
@@ -41,7 +41,7 @@ v2i test_amdgcn_ds_read_b64_tr_b8_v2i32(local v2i* inptr)
 }
 
 // GFX950-LABEL: define dso_local <4 x i16> @test_amdgcn_ds_read_b64_tr_b16_v2i16(
-// GFX950-SAME: ptr addrspace(3) nocapture noundef readonly [[INPTR:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
+// GFX950-SAME: ptr addrspace(3) noundef readonly captures(none) [[INPTR:%.*]]) local_unnamed_addr #[[ATTR0]] {
 // GFX950-NEXT:  entry:
 // GFX950-NEXT:    [[TMP0:%.*]] = tail call <4 x i16> @llvm.amdgcn.ds.read.tr16.b64.v4i16(ptr addrspace(3) [[INPTR]])
 // GFX950-NEXT:    ret <4 x i16> [[TMP0]]
@@ -52,7 +52,7 @@ v4s test_amdgcn_ds_read_b64_tr_b16_v2i16(local v4s* inptr)
 }
 
 // GFX950-LABEL: define dso_local <4 x half> @test_amdgcn_ds_read_b64_tr_b16_v2f16(
-// GFX950-SAME: ptr addrspace(3) nocapture noundef readonly [[INPTR:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
+// GFX950-SAME: ptr addrspace(3) noundef readonly captures(none) [[INPTR:%.*]]) local_unnamed_addr #[[ATTR0]] {
 // GFX950-NEXT:  entry:
 // GFX950-NEXT:    [[TMP0:%.*]] = tail call <4 x half> @llvm.amdgcn.ds.read.tr16.b64.v4f16(ptr addrspace(3) [[INPTR]])
 // GFX950-NEXT:    ret <4 x half> [[TMP0]]
@@ -63,10 +63,11 @@ v4h test_amdgcn_ds_read_b64_tr_b16_v2f16(local v4h* inptr)
 }
 
 // GFX950-LABEL: define dso_local <4 x bfloat> @test_amdgcn_ds_read_b64_tr_b16_v2bf16(
-// GFX950-SAME: ptr addrspace(3) nocapture noundef readonly [[INPTR:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
+// GFX950-SAME: ptr addrspace(3) noundef readonly captures(none) [[INPTR:%.*]]) local_unnamed_addr #[[ATTR0]] {
 // GFX950-NEXT:  entry:
 // GFX950-NEXT:    [[TMP0:%.*]] = tail call <4 x bfloat> @llvm.amdgcn.ds.read.tr16.b64.v4bf16(ptr addrspace(3) [[INPTR]])
 // GFX950-NEXT:    ret <4 x bfloat> [[TMP0]]
+//
 v4y test_amdgcn_ds_read_b64_tr_b16_v2bf16(local v4y* inptr)
 {
   return __builtin_amdgcn_ds_read_tr16_b64_v4bf16(inptr);
diff --git a/clang/test/CodeGenOpenCL/kernel-param-alignment.cl b/clang/test/CodeGenOpenCL/kernel-param-alignment.cl
index 1bce77043981e..38922dba1ca37 100644
--- a/clang/test/CodeGenOpenCL/kernel-param-alignment.cl
+++ b/clang/test/CodeGenOpenCL/kernel-param-alignment.cl
@@ -17,10 +17,10 @@ kernel void test(
     global void *v,
     global struct packed *p) {
 // CHECK-LABEL: spir_kernel void @test(
-// CHECK-SAME: ptr nocapture noundef readnone align 4 %i32,
-// CHECK-SAME: ptr nocapture noundef readnone align 8 %i64,
-// CHECK-SAME: ptr nocapture noundef readnone align 16 %v4i32,
-// CHECK-SAME: ptr nocapture noundef readnone align 8 %v2f32,
-// CHECK-SAME: ptr nocapture noundef readnone %v,
-// CHECK-SAME: ptr nocapture noundef readnone align 1 %p)
+// CHECK-SAME: ptr noundef readnone align 4 captures(none) %i32,
+// CHECK-SAME: ptr noundef readnone align 8 captures(none) %i64,
+// CHECK-SAME: ptr noundef readnone align 16 captures(none) %v4i32,
+// CHECK-SAME: ptr noundef readnone align 8 captures(none) %v2f32,
+// CHECK-SAME: ptr noundef readnone captures(none) %v,
+// CHECK-SAME: ptr noundef readnone align 1 captures(none) %p)
 }
diff --git a/clang/test/CodeGenOpenCL/kernels-have-spir-cc-by-default.cl b/clang/test/CodeGenOpenCL/kernels-have-spir-cc-by-default.cl
index 2aeeb637795a9..d8b6e13633141 100644
--- a/clang/test/CodeGenOpenCL/kernels-have-spir-cc-by-default.cl
+++ b/clang/test/CodeGenOpenCL/kernels-have-spir-cc-by-default.cl
@@ -27,16 +27,16 @@ typedef struct test_struct {
 kernel void test_single(int_single input, global int* output) {
 // CHECK: spir_kernel
 // AMDGCN: define{{.*}} amdgpu_kernel void @test_single
-// CHECK: ptr nocapture {{.*}} byval(%struct.int_single)
-// CHECK: ptr nocapture noundef writeonly align 4 initializes((0, 4)) %output
+// CHECK: ptr {{.*}} byval(%struct.int_single) align 4 captures(none)
+// CHECK: ptr noundef writeonly align 4 captures(none) initializes((0, 4)) %output
  output[0] = input.a;
 }
 
 kernel void test_pair(int_pair input, global int* output) {
 // CHECK: spir_kernel
 // AMDGCN: define{{.*}} amdgpu_kernel void @test_pair
-// CHECK: ptr nocapture {{.*}} byval(%struct.int_pair)
-// CHECK: ptr nocapture noundef writeonly align 4 initializes((0, 8)) %output
+// CHECK: ptr {{.*}} byval(%struct.int_pair) align 8 captures(none)
+// CHECK: ptr noundef writeonly align 4 captures(none) initializes((0, 8)) %output
  output[0] = (int)input.a;
  output[1] = (int)input.b;
 }
@@ -44,8 +44,8 @@ kernel void test_pair(int_pair input, global int* output) {
 kernel void test_kernel(test_struct input, global int* output) {
 // CHECK: spir_kernel
 // AMDGCN: define{{.*}} amdgpu_kernel void @test_kernel
-// CHECK: ptr nocapture {{.*}} byval(%struct.test_struct)
-// CHECK: ptr nocapture noundef writeonly align 4 initializes((0, 32)) %output
+// CHECK: ptr {{.*}} byval(%struct.test_struct) align 8 captures(none)
+// CHECK: ptr noundef writeonly align 4 captures(none) initializes((0, 32)) %output
  output[0] = input.elementA;
  output[1] = input.elementB;
  output[2] = (int)input.elementC;
@@ -59,7 +59,7 @@ kernel void test_kernel(test_struct input, global int* output) {
 void test_function(int_pair input, global int* output) {
 // CHECK-NOT: spir_kernel
 // AMDGCN-NOT: define{{.*}} amdgpu_kernel void @test_function
-// CHECK: i64 %input.coerce0, i64 %input.coerce1, ptr nocapture noundef writeonly initializes((0, 8)) %output
+// CHECK: i64 %input.coerce0, i64 %input.coerce1, ptr noundef writeonly captures(none) initializes((0, 8)) %output
  output[0] = (int)input.a;
  output[1] = (int)input.b;
 }
diff --git a/clang/test/CodeGenOpenCL/preserve_vec3.cl b/clang/test/CodeGenOpenCL/preserve_vec3.cl
index 747cc301feff6..49ebae6fc7013 100644
--- a/clang/test/CodeGenOpenCL/preserve_vec3.cl
+++ b/clang/test/CodeGenOpenCL/preserve_vec3.cl
@@ -9,7 +9,7 @@ typedef float float3 __attribute__((ext_vector_type(3)));
 typedef float float4 __attribute__((ext_vector_type(4)));
 
 // CHECK-LABEL: define dso_local spir_kernel void @foo(
-// CHECK-SAME: ptr addrspace(1) nocapture noundef readonly align 16 [[A:%.*]], ptr addrspace(1) nocapture noundef writeonly align 16 initializes((0, 16)) [[B:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] !kernel_arg_addr_space [[META3:![0-9]+]] !kernel_arg_access_qual [[META4:![0-9]+]] !kernel_arg_type [[META5:![0-9]+]] !kernel_arg_base_type [[META6:![0-9]+]] !kernel_arg_type_qual [[META7:![0-9]+]] {
+// CHECK-SAME: ptr addrspace(1) noundef readonly align 16 captures(none) [[A:%.*]], ptr addrspace(1) noundef writeonly align 16 captures(none) initializes((0, 16)) [[B:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] !kernel_arg_addr_space [[META3:![0-9]+]] !kernel_arg_access_qual [[META4:![0-9]+]] !kernel_arg_type [[META5:![0-9]+]] !kernel_arg_base_type [[META6:![0-9]+]] !kernel_arg_type_qual [[META7:![0-9]+]] {
 // CHECK-NEXT:  [[ENTRY:.*:]]
 // CHECK-NEXT:    [[LOADVECN:%.*]] = load <4 x float>, ptr addrspace(1) [[A]], align 16
 // CHECK-NEXT:    [[EXTRACTVEC1:%.*]] = shufflevector <4 x float> [[LOADVECN]], <4 x float> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 poison>
@@ -21,7 +21,7 @@ void kernel foo(global float3 *a, global float3 *b) {
 }
 
 // CHECK-LABEL: define dso_local spir_kernel void @float4_to_float3(
-// CHECK-SAME: ptr addrspace(1) nocapture noundef writeonly align 16 initializes((0, 16)) [[A:%.*]], ptr addrspace(1) nocapture noundef readonly align 16 [[B:%.*]]) local_unnamed_addr #[[ATTR0]] !kernel_arg_addr_space [[META3]] !kernel_arg_access_qual [[META4]] !kernel_arg_type [[META11:![0-9]+]] !kernel_arg_base_type [[META12:![0-9]+]] !kernel_arg_type_qual [[META7]] {
+// CHECK-SAME: ptr addrspace(1) noundef writeonly align 16 captures(none) initializes((0, 16)) [[A:%.*]], ptr addrspace(1) noundef readonly align 16 captures(none) [[B:%.*]]) local_unnamed_addr #[[ATTR0]] !kernel_arg_addr_space [[META3]] !kernel_arg_access_qual [[META4]] !kernel_arg_type [[META11:![0-9]+]] !kernel_arg_base_type [[META12:![0-9]+]] !kernel_arg_type_qual [[META7]] {
 // CHECK-NEXT:  [[ENTRY:.*:]]
 // CHECK-NEXT:    [[TMP0:%.*]] = load <4 x float>, ptr addrspace(1) [[B]], align 16, !tbaa [[TBAA8]]
 // CHECK-NEXT:    [[EXTRACTVEC:%.*]] = shufflevector <4 x float> [[TMP0]], <4 x float> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 poison>
@@ -33,7 +33,7 @@ void kernel float4_to_float3(global float3 *a, global float4 *b) {
 }
 
 // CHECK-LABEL: define dso_local spir_kernel void @float3_to_float4(
-// CHECK-SAME: ptr addrspace(1) nocapture noundef readonly align 16 [[A:%.*]], ptr addrspace(1) nocapture noundef writeonly align 16 initializes((0, 16)) [[B:%.*]]) local_unnamed_addr #[[ATTR0]] !kernel_arg_addr_space [[META3]] !kernel_arg_access_qual [[META4]] !kernel_arg_type [[META11]] !kernel_arg_base_type [[META12]] !kernel_arg_type_qual [[META7]] {
+// CHECK-SAME: ptr addrspace(1) noundef readonly align 16 captures(none) [[A:%.*]], ptr addrspace(1) noundef writeonly align 16 captures(none) initializes((0, 16)) [[B:%.*]]) local_unnamed_addr #[[ATTR0]] !kernel_arg_addr_space [[META3]] !kernel_arg_access_qual [[META4]] !kernel_arg_type [[META11]] !kernel_arg_base_type [[META12]] !kernel_arg_type_qual [[META7]] {
 // CHECK-NEXT:  [[ENTRY:.*:]]
 // CHECK-NEXT:    [[LOADVECN:%.*]] = load <4 x float>, ptr addrspace(1) [[A]], align 16
 // CHECK-NEXT:    [[ASTYPE:%.*]] = shufflevector <4 x float> [[LOADVECN]], <4 x float> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 poison>
@@ -45,7 +45,7 @@ void kernel float3_to_float4(global float3 *a, global float4 *b) {
 }
 
 // CHECK-LABEL: define dso_local spir_kernel void @float3_to_double2(
-// CHECK-SAME: ptr addrspace(1) nocapture noundef readonly align 16 [[A:%.*]], ptr addrspace(1) nocapture noundef writeonly align 16 initializes((0, 16)) [[B:%.*]]) local_unnamed_addr #[[ATTR0]] !kernel_arg_addr_space [[META3]] !kernel_arg_access_qual [[META4]] !kernel_arg_type [[META13:![0-9]+]] !kernel_arg_base_type [[META14:![0-9]+]] !kernel_arg_type_qual [[META7]] {
+// CHECK-SAME: ptr addrspace(1) noundef readonly align 16 captures(none) [[A:%.*]], ptr addrspace(1) noundef writeonly align 16 captures(none) initializes((0, 16)) [[B:%.*]]) local_unnamed_addr #[[ATTR0]] !kernel_arg_addr_space [[META3]] !kernel_arg_access_qual [[META4]] !kernel_arg_type [[META13:![0-9]+]] !kernel_arg_base_type [[META14:![0-9]+]] !kernel_arg_type_qual [[META7]] {
 // CHECK-NEXT:  [[ENTRY:.*:]]
 // CHECK-NEXT:    [[LOADVECN:%.*]] = load <4 x float>, ptr addrspace(1) [[A]], align 16
 // CHECK-NEXT:    [[TMP0:%.*]] = shufflevector <4 x float> [[LOADVECN]], <4 x float> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 poison>
@@ -57,7 +57,7 @@ void kernel float3_to_double2(global float3 *a, global double2 *b) {
 }
 
 // CHECK-LABEL: define dso_local spir_kernel void @char8_to_short3(
-// CHECK-SAME: ptr addrspace(1) nocapture noundef writeonly align 8 initializes((0, 8)) [[A:%.*]], ptr addrspace(1) nocapture noundef readonly align 8 [[B:%.*]]) local_unnamed_addr #[[ATTR0]] !kernel_arg_addr_space [[META3]] !kernel_arg_access_qual [[META4]] !kernel_arg_type [[META15:![0-9]+]] !kernel_arg_base_type [[META16:![0-9]+]] !kernel_arg_type_qual [[META7]] {
+// CHECK-SAME: ptr addrspace(1) noundef writeonly align 8 captures(none) initializes((0, 8)) [[A:%.*]], ptr addrspace(1) noundef readonly align 8 captures(none) [[B:%.*]]) local_unnamed_addr #[[ATTR0]] !kernel_arg_addr_space [[META3]] !kernel_arg_access_qual [[META4]] !kernel_arg_type [[META15:![0-9]+]] !kernel_arg_base_type [[META16:![0-9]+]] !kernel_arg_type_qual [[META7]] {
 // CHECK-NEXT:  [[ENTRY:.*:]]
 // CHECK-NEXT:    [[TMP0:%.*]] = load <4 x i16>, ptr addrspace(1) [[B]], align 8, !tbaa [[TBAA8]]
 // CHECK-NEXT:    [[EXTRACTVEC:%.*]] = shufflevector <4 x i16> [[TMP0]], <4 x i16> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 poison>
@@ -69,7 +69,7 @@ void kernel char8_to_short3(global short3 *a, global char8 *b) {
 }
 
 // CHECK-LABEL: define dso_local spir_func void @from_char3(
-// CHECK-SAME: <3 x i8> noundef [[A:%.*]], ptr addrspace(1) nocapture noundef writeonly initializes((0, 4)) [[OUT:%.*]]) local_unnamed_addr #[[ATTR1:[0-9]+]] {
+// CHECK-SAME: <3 x i8> noundef [[A:%.*]], ptr addrspace(1) noundef writeonly captures(none) initializes((0, 4)) [[OUT:%.*]]) local_unnamed_addr #[[ATTR1:[0-9]+]] {
 // CHECK-NEXT:  [[ENTRY:.*:]]
 // CHECK-NEXT:    [[EXTRACTVEC:%.*]] = shufflevector <3 x i8> [[A]], <3 x i8> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 poison>
 // CHECK-NEXT:    store <4 x i8> [[EXTRACTVEC]], ptr addrspace(1) [[OUT]], align 4, !tbaa [[TBAA17:![0-9]+]]
@@ -80,7 +80,7 @@ void from_char3(char3 a, global int *out) {
 }
 
 // CHECK-LABEL: define dso_local spir_func void @from_short3(
-// CHECK-SAME: <3 x i16> noundef [[A:%.*]], ptr addrspace(1) nocapture noundef writeonly initializes((0, 8)) [[OUT:%.*]]) local_unnamed_addr #[[ATTR1]] {
+// CHECK-SAME: <3 x i16> noundef [[A:%.*]], ptr addrspace(1) noundef writeonly captures(none) initializes((0, 8)) [[OUT:%.*]]) local_unnamed_addr #[[ATTR1]] {
 // CHECK-NEXT:  [[ENTRY:.*:]]
 // CHECK-NEXT:    [[EXTRACTVEC:%.*]] = shufflevector <3 x i16> [[A]], <3 x i16> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 poison>
 // CHECK-NEXT:    store <4 x i16> [[EXTRACTVEC]], ptr addrspace(1) [[OUT]], align 8, !tbaa [[TBAA19:![0-9]+]]
@@ -91,7 +91,7 @@ void from_short3(short3 a, global long *out) {
 }
 
 // CHECK-LABEL: define dso_local spir_func void @scalar_to_char3(
-// CHECK-SAME: i32 noundef [[A:%.*]], ptr addrspace(1) nocapture noundef writeonly initializes((0, 4)) [[OUT:%.*]]) local_unnamed_addr #[[ATTR1]] {
+// CHECK-SAME: i32 noundef [[A:%.*]], ptr addrspace(1) noundef writeonly captures(none) initializes((0, 4)) [[OUT:%.*]]) local_unnamed_addr #[[ATTR1]] {
 // CHECK-NEXT:  [[ENTRY:.*:]]
 // CHECK-NEXT:    [[TMP0:%.*]] = bitcast i32 [[A]] to <4 x i8>
 // CHECK-NEXT:    [[EXTRACTVEC:%.*]] = shufflevector <4 x i8> [[TMP0]], <4 x i8> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 poison>
@@ -103,7 +103,7 @@ void scalar_to_char3(int a, global char3 *out) {
 }
 
 // CHECK-LABEL: define dso_local spir_func void @scalar_to_short3(
-// CHECK-SAME: i64 noundef [[A:%.*]], ptr addrspace(1) nocapture noundef writeonly initializes((0, 8)) [[OUT:%.*]]) local_unnamed_addr #[[ATTR1]] {
+// CHECK-SAME: i64 noundef [[A:%.*]], ptr addrspace(1) noundef writeonly captures(none) initializes((0, 8)) [[OUT:%.*]]) local_unnamed_addr #[[ATTR1]] {
 // CHECK-NEXT:  [[ENTRY:.*:]]
 // CHECK-NEXT:    [[TMP0:%.*]] = bitcast i64 [[A]] to <4 x i16>
 // CHECK-NEXT:    [[EXTRACTVEC:%.*]] = shufflevector <4 x i16> [[TMP0]], <4 x i16> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 poison>
diff --git a/clang/test/CodeGenOpenCLCXX/array-type-infinite-loop.clcpp b/clang/test/CodeGenOpenCLCXX/array-type-infinite-loop.clcpp
index 697e50e199a04..8d8f0b0b5d699 100644
--- a/clang/test/CodeGenOpenCLCXX/array-type-infinite-loop.clcpp
+++ b/clang/test/CodeGenOpenCLCXX/array-type-infinite-loop.clcpp
@@ -2,7 +2,7 @@
 //RUN: %clang_cc1 %s -triple spir -emit-llvm -O1 -o - | FileCheck %s
 
 // CHECK-LABEL: define dso_local spir_kernel void @test(
-// CHECK-SAME: ptr addrspace(1) nocapture noundef readonly align 8 [[IN:%.*]], ptr addrspace(1) nocapture noundef writeonly align 8 initializes((0, 8)) [[OUT:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] !kernel_arg_addr_space [[META4:![0-9]+]] !kernel_arg_access_qual [[META5:![0-9]+]] !kernel_arg_type [[META6:![0-9]+]] !kernel_arg_base_type [[META6]] !kernel_arg_type_qual [[META7:![0-9]+]] {
+// CHECK-SAME: ptr addrspace(1) noundef readonly align 8 captures(none) [[IN:%.*]], ptr addrspace(1) noundef writeonly align 8 captures(none) initializes((0, 8)) [[OUT:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] !kernel_arg_addr_space [[META4:![0-9]+]] !kernel_arg_access_qual [[META5:![0-9]+]] !kernel_arg_type [[META6:![0-9]+]] !kernel_arg_base_type [[META6]] !kernel_arg_type_qual [[META7:![0-9]+]] {
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds nuw i8, ptr addrspace(1) [[IN]], i32 8
 // CHECK-NEXT:    [[TMP0:%.*]] = load i64, ptr addrspace(1) [[ARRAYIDX1]], align 8, !tbaa [[TBAA8:![0-9]+]]
diff --git a/clang/test/Driver/Xarch.c b/clang/test/Driver/Xarch.c
index f7693fb689d58..95bd0c502ddb3 100644
--- a/clang/test/Driver/Xarch.c
+++ b/clang/test/Driver/Xarch.c
@@ -1,8 +1,13 @@
 // RUN: %clang -target i386-apple-darwin11 -m32 -Xarch_i386 -O3 %s -S -### 2>&1 | FileCheck -check-prefix=O3ONCE %s
+// RUN: %clang -target x86_64-unknown-linux-gnu -Xarch_x86_64 -O3 %s -S -### 2>&1 | FileCheck -check-prefix=O3ONCE %s
+// RUN: %clang -target x86_64-unknown-windows-msvc -Xarch_x86_64 -O3 %s -S -### 2>&1 | FileCheck -check-prefix=O3ONCE %s
+// RUN: %clang -target aarch64-unknown-linux-gnu -Xarch_aarch64 -O3 %s -S -### 2>&1 | FileCheck -check-prefix=O3ONCE %s
+// RUN: %clang -target powerpc64le-unknown-linux-gnu -Xarch_powerpc64le -O3 %s -S -### 2>&1 | FileCheck -check-prefix=O3ONCE %s
 // O3ONCE: "-O3"
 // O3ONCE-NOT: "-O3"
 
 // RUN: %clang -target i386-apple-darwin11 -m64 -Xarch_i386 -O3 %s -S -### 2>&1 | FileCheck -check-prefix=O3NONE %s
+// RUN: %clang -target x86_64-unknown-linux-gnu -m64 -Xarch_i386 -O3 %s -S -### 2>&1 | FileCheck -check-prefix=O3NONE %s
 // O3NONE-NOT: "-O3"
 // O3NONE: argument unused during compilation: '-Xarch_i386 -O3'
 
@@ -10,3 +15,10 @@
 // INVALID: error: invalid Xarch argument: '-Xarch_i386 -o'
 // INVALID: error: invalid Xarch argument: '-Xarch_i386 -S'
 // INVALID: error: invalid Xarch argument: '-Xarch_i386 -o'
+
+// RUN: %clang -target x86_64-unknown-linux-gnu -Xarch_x86_64 -Wl,foo %s -### 2>&1 | FileCheck -check-prefix=LINKER %s
+// LINKER: "foo"
+
+// RUN: %clang -target x86_64-unknown-linux-gnu -Xarch_x86_64 -O1 %s -S -### 2>&1 | FileCheck -check-prefix=O1ONCE %s
+// O1ONCE: "-O1"
+// O1ONCE-NOT: "-O1"
diff --git a/clang/test/Driver/aarch64-ptrauth.c b/clang/test/Driver/aarch64-ptrauth.c
index d036189e61498..1d2993f4c60c4 100644
--- a/clang/test/Driver/aarch64-ptrauth.c
+++ b/clang/test/Driver/aarch64-ptrauth.c
@@ -64,22 +64,31 @@
 
 //// The only branch protection option compatible with PAuthABI is BTI.
 // RUN: not %clang -### -c --target=aarch64-linux -mabi=pauthtest -mbranch-protection=pac-ret %s 2>&1 | \
-// RUN:   FileCheck %s --check-prefix=ERR4
+// RUN:   FileCheck %s --check-prefix=ERR4_1
 // RUN: not %clang -### -c --target=aarch64-linux-pauthtest       -mbranch-protection=pac-ret %s 2>&1 | \
-// RUN:   FileCheck %s --check-prefix=ERR4
-// ERR4: error: unsupported option '-mbranch-protection=pac-ret' for target 'aarch64-unknown-linux-pauthtest'
+// RUN:   FileCheck %s --check-prefix=ERR4_1
+// RUN: not %clang -### -c --target=aarch64 -fptrauth-returns     -mbranch-protection=pac-ret %s 2>&1 | \
+// RUN:   FileCheck %s --check-prefix=ERR4_2
+// ERR4_1: error: unsupported option '-mbranch-protection=pac-ret' for target 'aarch64-unknown-linux-pauthtest'
+// ERR4_2: error: the combination of '-mbranch-protection=pac-ret' and '-fptrauth-returns' is incompatible
 
 // RUN: not %clang -### -c --target=aarch64-linux -mabi=pauthtest -mbranch-protection=gcs %s 2>&1 | \
-// RUN:   FileCheck %s --check-prefix=ERR5
+// RUN:   FileCheck %s --check-prefix=ERR5_1
 // RUN: not %clang -### -c --target=aarch64-linux-pauthtest       -mbranch-protection=gcs %s 2>&1 | \
-// RUN:   FileCheck %s --check-prefix=ERR5
-// ERR5: error: unsupported option '-mbranch-protection=gcs' for target 'aarch64-unknown-linux-pauthtest'
+// RUN:   FileCheck %s --check-prefix=ERR5_1
+// RUN: not %clang -### -c --target=aarch64 -fptrauth-returns     -mbranch-protection=gcs %s 2>&1 | \
+// RUN:   FileCheck %s --check-prefix=ERR5_2
+// ERR5_1: error: unsupported option '-mbranch-protection=gcs' for target 'aarch64-unknown-linux-pauthtest'
+// ERR5_2: error: the combination of '-mbranch-protection=gcs' and '-fptrauth-returns' is incompatible
 
 // RUN: not %clang -### -c --target=aarch64-linux -mabi=pauthtest -mbranch-protection=standard %s 2>&1 | \
-// RUN:   FileCheck %s --check-prefix=ERR6
+// RUN:   FileCheck %s --check-prefix=ERR6_1
 // RUN: not %clang -### -c --target=aarch64-linux-pauthtest       -mbranch-protection=standard %s 2>&1 | \
-// RUN:   FileCheck %s --check-prefix=ERR6
-// ERR6: error: unsupported option '-mbranch-protection=standard' for target 'aarch64-unknown-linux-pauthtest'
+// RUN:   FileCheck %s --check-prefix=ERR6_1
+// RUN: not %clang -### -c --target=aarch64 -fptrauth-returns     -mbranch-protection=standard %s 2>&1 | \
+// RUN:   FileCheck %s --check-prefix=ERR6_2
+// ERR6_1: error: unsupported option '-mbranch-protection=standard' for target 'aarch64-unknown-linux-pauthtest'
+// ERR6_2: error: the combination of '-mbranch-protection=standard' and '-fptrauth-returns' is incompatible
 
 // RUN: not %clang -### -c --target=aarch64-linux -mabi=pauthtest -msign-return-address=all %s 2>&1 | \
 // RUN:   FileCheck %s --check-prefix=ERR7
diff --git a/clang/test/Driver/amdgpu-openmp-sanitize-options.c b/clang/test/Driver/amdgpu-openmp-sanitize-options.c
new file mode 100644
index 0000000000000..c28a758bfc0c5
--- /dev/null
+++ b/clang/test/Driver/amdgpu-openmp-sanitize-options.c
@@ -0,0 +1,65 @@
+// REQUIRES: x86-registered-target, amdgpu-registered-target
+
+// Fail on invalid ROCm Path.
+// RUN:   not %clang -no-canonical-prefixes -### --target=x86_64-unknown-linux-gnu -fopenmp=libomp --offload-arch=gfx908:xnack+ -fsanitize=address -fgpu-sanitize -nogpuinc --rocm-path=%S/Inputs/rocm-invalid  %s 2>&1 \
+// RUN:   | FileCheck --check-prefix=FAIL %s
+
+// Enable multiple sanitizer's apart from ASan with invalid rocm-path.
+// RUN:   not %clang -no-canonical-prefixes -### --target=x86_64-unknown-linux-gnu -fopenmp=libomp --offload-arch=gfx908:xnack+ -fsanitize=address -fsanitize=leak -fgpu-sanitize --rocm-path=%S/Inputs/rocm-invalid -nogpuinc  %s 2>&1 \
+// RUN:   | FileCheck --check-prefixes=NOTSUPPORTED,FAIL %s
+
+// Memory, Leak, UndefinedBehaviour and Thread Sanitizer are not supported on AMDGPU.
+// RUN:   %clang -no-canonical-prefixes -### --target=x86_64-unknown-linux-gnu -fopenmp=libomp --offload-arch=gfx908:xnack+ -fsanitize=address -fsanitize=leak -fgpu-sanitize --rocm-path=%S/Inputs/rocm -nogpuinc  %s 2>&1 \
+// RUN:   | FileCheck --check-prefix=NOTSUPPORTED %s
+
+// GPU ASan Enabled Test Cases
+// ASan enabled for amdgpu-arch [gfx908]
+// RUN:   %clang -no-canonical-prefixes -### --target=x86_64-unknown-linux-gnu -fopenmp=libomp --offload-arch=gfx908 -fsanitize=address -fgpu-sanitize --rocm-path=%S/Inputs/rocm %s 2>&1 \
+// RUN:   | FileCheck -check-prefixes=NOXNACK,GPUSAN %s
+
+// GPU ASan enabled for amdgpu-arch [gfx908:xnack-]
+// RUN:   %clang -no-canonical-prefixes -### --target=x86_64-unknown-linux-gnu -fopenmp=libomp --offload-arch=gfx908:xnack- -fsanitize=address -fgpu-sanitize --rocm-path=%S/Inputs/rocm %s 2>&1 \
+// RUN:   | FileCheck -check-prefixes=XNACKNEG,GPUSAN %s
+
+// GPU ASan enabled for amdgpu-arch [gfx908:xnack+]
+// RUN:   %clang -no-canonical-prefixes -### --target=x86_64-unknown-linux-gnu -fopenmp=libomp --offload-arch=gfx908:xnack+ -fsanitize=address -fgpu-sanitize --rocm-path=%S/Inputs/rocm %s 2>&1 \
+// RUN:   | FileCheck -check-prefixes=GPUSAN %s
+
+// ASan enabled for multiple amdgpu-arch [gfx908:xnack+,gfx900:xnack+]
+// RUN:   %clang -no-canonical-prefixes -### --target=x86_64-unknown-linux-gnu -fopenmp=libomp --offload-arch=gfx908:xnack+ --offload-arch=gfx900:xnack+ -fsanitize=address -fgpu-sanitize --rocm-path=%S/Inputs/rocm %s 2>&1 \
+// RUN:   | FileCheck -check-prefixes=GPUSAN %s
+
+// GPU ASan Disabled Test Cases
+// ASan disabled for amdgpu-arch [gfx908]
+// RUN:   %clang -no-canonical-prefixes -### --target=x86_64-unknown-linux-gnu -fopenmp=libomp --offload-arch=gfx908 -fsanitize=address -fno-gpu-sanitize --rocm-path=%S/Inputs/rocm %s 2>&1 \
+// RUN:   | FileCheck -check-prefixes=NOGPUSAN %s
+
+// GPU ASan disabled for amdgpu-arch [gfx908:xnack-]
+// RUN:   %clang -no-canonical-prefixes -### --target=x86_64-unknown-linux-gnu -fopenmp=libomp --offload-arch=gfx908:xnack- -fsanitize=address -fno-gpu-sanitize --rocm-path=%S/Inputs/rocm %s 2>&1 \
+// RUN:   | FileCheck -check-prefixes=NOGPUSAN %s
+
+// GPU ASan disabled for amdgpu-arch [gfx908:xnack+]
+// RUN:   %clang -no-canonical-prefixes -### --target=x86_64-unknown-linux-gnu -fopenmp=libomp --offload-arch=gfx908:xnack+ -fsanitize=address -fno-gpu-sanitize --rocm-path=%S/Inputs/rocm %s 2>&1 \
+// RUN:   | FileCheck -check-prefixes=NOGPUSAN %s
+
+// ASan disabled for amdgpu-arch [gfx908:xnack+,gfx900:xnack+]
+// RUN:   %clang -no-canonical-prefixes -### --target=x86_64-unknown-linux-gnu -fopenmp=libomp --offload-arch=gfx908:xnack+ --offload-arch=gfx900:xnack+ -fsanitize=address -fno-gpu-sanitize --rocm-path=%S/Inputs/rocm %s 2>&1 \
+// RUN:   | FileCheck -check-prefixes=NOGPUSAN %s
+
+// FAIL-DAG: error: cannot find ROCm device library for ABI version 5; provide its path via '--rocm-path' or '--rocm-device-lib-path', or pass '-nogpulib' to build without ROCm device library
+// NOTSUPPORTED-DAG: warning: ignoring '-fsanitize=leak' option as it is not currently supported for target 'amdgcn-amd-amdhsa'
+
+// NOXNACK: warning: ignoring '-fsanitize=address' option for offload arch 'gfx908' as it is not currently supported there. Use it with an offload arch containing 'xnack+' instead
+// XNACKNEG: warning: ignoring '-fsanitize=address' option for offload arch 'gfx908:xnack-' as it is not currently supported there. Use it with an offload arch containing 'xnack+' instead
+
+// GPUSAN: {{"[^"]*clang[^"]*" "-cc1" "-triple" "x86_64-unknown-linux-gnu".* "-fopenmp".* "-fsanitize=address".* "-fopenmp-targets=amdgcn-amd-amdhsa".* "-x" "c".*}}
+// GPUSAN: {{"[^"]*clang[^"]*" "-cc1" "-triple" "amdgcn-amd-amdhsa" "-aux-triple" "x86_64-unknown-linux-gnu".* "-emit-llvm-bc".* "-target-cpu" "(gfx908|gfx900)".* "-fopenmp".* "-fsanitize=address".* "-x" "c".*}}
+// GPUSAN: {{"[^"]*clang-offload-packager[^"]*" "-o".* "--image=file=.*.bc,triple=amdgcn-amd-amdhsa,arch=gfx908(:xnack\-|:xnack\+)?,kind=openmp(,feature=(\-xnack|\+xnack))?"}}
+// GPUSAN: {{"[^"]*clang[^"]*" "-cc1" "-triple" "x86_64-unknown-linux-gnu".* "-fopenmp".* "-fsanitize=address".* "-fopenmp-targets=amdgcn-amd-amdhsa".* "-x" "ir".*}}
+// GPUSAN: {{"[^"]*clang-linker-wrapper[^"]*".* "--host-triple=x86_64-unknown-linux-gnu".* "--linker-path=[^"]*".* "--whole-archive" "[^"]*(libclang_rt.asan_static.a|libclang_rt.asan_static-x86_64.a)".* "--whole-archive" "[^"]*(libclang_rt.asan.a|libclang_rt.asan-x86_64.a)".*}}
+
+// NOGPUSAN: {{"[^"]*clang[^"]*" "-cc1" "-triple" "x86_64-unknown-linux-gnu".* "-fopenmp".* "-fsanitize=address".* "-fopenmp-targets=amdgcn-amd-amdhsa".* "-x" "c".*}}
+// NOGPUSAN: {{"[^"]*clang[^"]*" "-cc1" "-triple" "amdgcn-amd-amdhsa" "-aux-triple" "x86_64-unknown-linux-gnu".* "-emit-llvm-bc".* "-target-cpu" "(gfx908|gfx900)".* "-fopenmp".* "-x" "c".*}}
+// NOGPUSAN: {{"[^"]*clang-offload-packager[^"]*" "-o".* "--image=file=.*.bc,triple=amdgcn-amd-amdhsa,arch=gfx908(:xnack\-|:xnack\+)?,kind=openmp(,feature=(\-xnack|\+xnack))?"}}
+// NOGPUSAN: {{"[^"]*clang[^"]*" "-cc1" "-triple" "x86_64-unknown-linux-gnu".* "-fopenmp".* "-fsanitize=address".* "-fopenmp-targets=amdgcn-amd-amdhsa".* "-x" "ir".*}}
+// NOGPUSAN: {{"[^"]*clang-linker-wrapper[^"]*".* "--host-triple=x86_64-unknown-linux-gnu".* "--linker-path=[^"]*".* "--whole-archive" "[^"]*(libclang_rt.asan_static.a|libclang_rt.asan_static-x86_64.a)".* "--whole-archive" "[^"]*(libclang_rt.asan.a|libclang_rt.asan-x86_64.a)".*}}
diff --git a/clang/test/Driver/amdgpu-openmp-system-arch-fail.c b/clang/test/Driver/amdgpu-openmp-system-arch-fail.c
index b7e1d0b2c5665..eb037183b4c3c 100644
--- a/clang/test/Driver/amdgpu-openmp-system-arch-fail.c
+++ b/clang/test/Driver/amdgpu-openmp-system-arch-fail.c
@@ -12,9 +12,9 @@
 // case when amdgpu_arch returns nothing or fails
 // RUN:   not %clang -### --target=x86_64-unknown-linux-gnu -fopenmp=libomp -fopenmp-targets=amdgcn-amd-amdhsa -nogpulib --amdgpu-arch-tool=%t/amdgpu_arch_fail %s 2>&1 \
 // RUN:   | FileCheck %s --check-prefix=NO-OUTPUT-ERROR
-// NO-OUTPUT-ERROR: error: cannot determine amdgcn architecture{{.*}}; consider passing it via '-march'
+// NO-OUTPUT-ERROR: error: cannot determine amdgcn architecture{{.*}}; consider passing it via '--offload-arch'
 
 // case when amdgpu_arch does not return anything with successful execution
 // RUN:   not %clang -### --target=x86_64-unknown-linux-gnu -fopenmp=libomp -fopenmp-targets=amdgcn-amd-amdhsa -nogpulib --amdgpu-arch-tool=%t/amdgpu_arch_empty %s 2>&1 \
 // RUN:   | FileCheck %s --check-prefix=EMPTY-OUTPUT
-// EMPTY-OUTPUT: error: cannot determine amdgcn architecture: No AMD GPU detected in the system; consider passing it via '-march'
+// EMPTY-OUTPUT: error: cannot determine amdgcn architecture: No AMD GPU detected in the system; consider passing it via '--offload-arch'
diff --git a/clang/test/Driver/amdgpu-openmp-toolchain.c b/clang/test/Driver/amdgpu-openmp-toolchain.c
index 1c2ee26173139..b3784537cb836 100644
--- a/clang/test/Driver/amdgpu-openmp-toolchain.c
+++ b/clang/test/Driver/amdgpu-openmp-toolchain.c
@@ -16,12 +16,12 @@
 // CHECK-PHASES: 0: input, "[[INPUT:.+]]", c, (host-openmp)
 // CHECK-PHASES: 1: preprocessor, {0}, cpp-output, (host-openmp)
 // CHECK-PHASES: 2: compiler, {1}, ir, (host-openmp)
-// CHECK-PHASES: 3: input, "[[INPUT]]", c, (device-openmp)
-// CHECK-PHASES: 4: preprocessor, {3}, cpp-output, (device-openmp)
-// CHECK-PHASES: 5: compiler, {4}, ir, (device-openmp)
-// CHECK-PHASES: 6: offload, "host-openmp (x86_64-unknown-linux-gnu)" {2}, "device-openmp (amdgcn-amd-amdhsa)" {5}, ir
-// CHECK-PHASES: 7: backend, {6}, ir, (device-openmp)
-// CHECK-PHASES: 8: offload, "device-openmp (amdgcn-amd-amdhsa)" {7}, ir
+// CHECK-PHASES: 3: input, "[[INPUT]]", c, (device-openmp, gfx906)
+// CHECK-PHASES: 4: preprocessor, {3}, cpp-output, (device-openmp, gfx906)
+// CHECK-PHASES: 5: compiler, {4}, ir, (device-openmp, gfx906)
+// CHECK-PHASES: 6: offload, "host-openmp (x86_64-unknown-linux-gnu)" {2}, "device-openmp (amdgcn-amd-amdhsa:gfx906)" {5}, ir
+// CHECK-PHASES: 7: backend, {6}, ir, (device-openmp, gfx906)
+// CHECK-PHASES: 8: offload, "device-openmp (amdgcn-amd-amdhsa:gfx906)" {7}, ir
 // CHECK-PHASES: 9: clang-offload-packager, {8}, image, (device-openmp)
 // CHECK-PHASES: 10: offload, "host-openmp (x86_64-unknown-linux-gnu)" {2}, "device-openmp (x86_64-unknown-linux-gnu)" {9}, ir
 // CHECK-PHASES: 11: backend, {10}, assembler, (host-openmp)
@@ -72,7 +72,7 @@
 
 // RUN: %clang -### -target x86_64-pc-linux-gnu -fopenmp --offload-arch=gfx90a \
 // RUN:   -O3 -nogpulib %s 2>&1 | FileCheck %s --check-prefix=CHECK-OPT
-// CHECK-OPT: clang-linker-wrapper{{.*}}"--opt-level=O3"
+// CHECK-OPT: clang-linker-wrapper{{.*}}"--device-compiler=amdgcn-amd-amdhsa=-O3"
 
 // RUN:   %clang -### --target=x86_64-unknown-linux-gnu -emit-llvm -S -fopenmp -fopenmp-targets=amdgcn-amd-amdhsa -Xopenmp-target=amdgcn-amd-amdhsa -march=gfx803 -nogpulib %s 2>&1 | FileCheck %s --check-prefix=CHECK-WARN-ATOMIC
 // CHECK-WARN-ATOMIC-NOT: "-cc1" "-triple" "x86_64-unknown-linux-gnu"{{.*}}"-Werror=atomic-alignment"
@@ -84,4 +84,4 @@
 
 // RUN: %clang -### -target x86_64-pc-linux-gnu -nogpulib  -fopenmp --offload-arch=gfx90a \
 // RUN:   -ftime-report %s 2>&1 | FileCheck %s --check-prefix=CHECK-TIME-REPORT
-// CHECK-TIME-REPORT: clang-linker-wrapper{{.*}}"--device-compiler=-ftime-report"
+// CHECK-TIME-REPORT: clang-linker-wrapper{{.*}}"--device-compiler=amdgcn-amd-amdhsa=-ftime-report"
diff --git a/clang/test/Driver/cl-options.c b/clang/test/Driver/cl-options.c
index 29a0fcbc17ac6..9f9ca1bf1a8fd 100644
--- a/clang/test/Driver/cl-options.c
+++ b/clang/test/Driver/cl-options.c
@@ -738,9 +738,13 @@
 // RUN:     -fno-modules-search-all \
 // RUN:     -fimplicit-modules \
 // RUN:     -fno-implicit-modules \
+// RUN:     -fstrict-overflow \
+// RUN:     -fno-strict-overflow \
 // RUN:     -ftrivial-auto-var-init=zero \
 // RUN:     -fwrapv \
 // RUN:     -fno-wrapv \
+// RUN:     -fwrapv-pointer \
+// RUN:     -fno-wrapv-pointer \
 // RUN:     --version \
 // RUN:     -Werror /Zs -- %s 2>&1
 
diff --git a/clang/test/Driver/clang_f_opts.c b/clang/test/Driver/clang_f_opts.c
index 38f25898c9556..7454ce3d30f5f 100644
--- a/clang/test/Driver/clang_f_opts.c
+++ b/clang/test/Driver/clang_f_opts.c
@@ -45,6 +45,13 @@
 // CHECK-UNROLL-LOOPS: "-funroll-loops"
 // CHECK-NO-UNROLL-LOOPS: "-fno-unroll-loops"
 
+// RUN: %clang -### -S -floop-interchange %s 2>&1 | FileCheck -check-prefix=CHECK-INTERCHANGE-LOOPS %s
+// RUN: %clang -### -S -fno-loop-interchange %s 2>&1 | FileCheck -check-prefix=CHECK-NO-INTERCHANGE-LOOPS %s
+// RUN: %clang -### -S -fno-loop-interchange -floop-interchange %s 2>&1 | FileCheck -check-prefix=CHECK-INTERCHANGE-LOOPS %s
+// RUN: %clang -### -S -floop-interchange -fno-loop-interchange %s 2>&1 | FileCheck -check-prefix=CHECK-NO-INTERCHANGE-LOOPS %s
+// CHECK-INTERCHANGE-LOOPS: "-floop-interchange"
+// CHECK-NO-INTERCHANGE-LOOPS: "-fno-loop-interchange"
+
 // RUN: %clang -### -S -fprofile-sample-accurate %s 2>&1 | FileCheck -check-prefix=CHECK-PROFILE-SAMPLE-ACCURATE %s
 // CHECK-PROFILE-SAMPLE-ACCURATE: "-fprofile-sample-accurate"
 
diff --git a/clang/test/Driver/clang_wrapv_opts.c b/clang/test/Driver/clang_wrapv_opts.c
index 826468e0678d0..295d8deb0d99d 100644
--- a/clang/test/Driver/clang_wrapv_opts.c
+++ b/clang/test/Driver/clang_wrapv_opts.c
@@ -1,11 +1,20 @@
-// RUN: %clang -### -S -fwrapv -fno-wrapv -fwrapv %s 2>&1 | FileCheck -check-prefix=CHECK1 %s
-// CHECK1: -fwrapv
-//
-// RUN: %clang -### -S -fstrict-overflow -fno-strict-overflow %s 2>&1 | FileCheck -check-prefix=CHECK2 %s
-// CHECK2: -fwrapv
-//
-// RUN: %clang -### -S -fwrapv -fstrict-overflow %s 2>&1 | FileCheck -check-prefix=CHECK3 %s
-// CHECK3: -fwrapv
-//
-// RUN: %clang -### -S -fno-wrapv -fno-strict-overflow %s 2>&1 | FileCheck -check-prefix=CHECK4 %s
-// CHECK4-NOT: -fwrapv
+// RUN: %clang -### -S -fwrapv -fno-wrapv -fwrapv -Werror %s 2>&1 | FileCheck -check-prefix=CHECK1 %s
+// CHECK1: "-fwrapv"
+
+// RUN: %clang -### -S -fwrapv-pointer -fno-wrapv-pointer -fwrapv-pointer -Werror %s 2>&1 | FileCheck -check-prefix=CHECK1-POINTER %s
+// CHECK1-POINTER: "-fwrapv-pointer"
+
+// RUN: %clang -### -S -fstrict-overflow -fno-strict-overflow -Werror %s 2>&1 | FileCheck -check-prefix=CHECK2 %s
+// CHECK2: "-fwrapv"{{.*}}"-fwrapv-pointer"
+
+// RUN: %clang -### -S -fwrapv -fstrict-overflow -Werror -Werror %s 2>&1 | FileCheck -check-prefix=CHECK3 %s --implicit-check-not="-fwrapv-pointer"
+// CHECK3-NOT: "-fwrapv"
+
+// RUN: %clang -### -S -fwrapv-pointer -fstrict-overflow -Werror %s 2>&1 | FileCheck -check-prefix=CHECK3-POINTER %s --implicit-check-not="-fwrapv"
+// CHECK3-POINTER-NOT: "-fwrapv-pointer"
+
+// RUN: %clang -### -S -fno-wrapv -fno-strict-overflow -fno-wrapv-pointer -Werror %s 2>&1 | FileCheck -check-prefix=CHECK4 %s --implicit-check-not="-fwrapv-pointer"
+// CHECK4: "-fwrapv"
+
+// RUN: %clang -### -S -fno-wrapv-pointer -fno-strict-overflow -fno-wrapv -Werror %s 2>&1 | FileCheck -check-prefix=CHECK4-POINTER %s --implicit-check-not="-fwrapv"
+// CHECK4-POINTER: "-fwrapv-pointer"
diff --git a/clang/test/Driver/csky-toolchain.c b/clang/test/Driver/csky-toolchain.c
index 66485464652ac..638ce64ec98cd 100644
--- a/clang/test/Driver/csky-toolchain.c
+++ b/clang/test/Driver/csky-toolchain.c
@@ -3,6 +3,7 @@
 
 // RUN: %clang -### %s --target=csky 2>&1 | FileCheck -check-prefix=CC1 %s
 // CC1: "-cc1" "-triple" "csky"
+// CC1: "-fno-signed-char"
 
 // In the below tests, --rtlib=platform is used so that the driver ignores
 // the configure-time CLANG_DEFAULT_RTLIB option when choosing the runtime lib
diff --git a/clang/test/Driver/cuda-cross-compiling.c b/clang/test/Driver/cuda-cross-compiling.c
index 7817e462c47be..1df231ecb4479 100644
--- a/clang/test/Driver/cuda-cross-compiling.c
+++ b/clang/test/Driver/cuda-cross-compiling.c
@@ -57,19 +57,6 @@
 
 // LINK: clang-nvlink-wrapper{{.*}}"-o" "a.out" "-arch" "sm_61"{{.*}}[[CUBIN:.+]].o
 
-//
-// Test to ensure that we enable handling global constructors in a freestanding
-// Nvidia compilation.
-//
-// RUN: %clang -target nvptx64-nvidia-cuda -march=sm_70 %s -### 2>&1 \
-// RUN:   | FileCheck -check-prefix=LOWERING %s
-// RUN: %clang -target nvptx64-nvidia-cuda -march=sm_70 -flto -c %s -### 2>&1 \
-// RUN:   | FileCheck -check-prefix=LOWERING-LTO %s
-
-// LOWERING: -cc1" "-triple" "nvptx64-nvidia-cuda" {{.*}} "-mllvm" "--nvptx-lower-global-ctor-dtor"
-// LOWERING: clang-nvlink-wrapper{{.*}} "-mllvm" "--nvptx-lower-global-ctor-dtor"
-// LOWERING-LTO-NOT: "--nvptx-lower-global-ctor-dtor"
-
 //
 // Test passing arguments directly to nvlink.
 //
diff --git a/clang/test/Driver/dep-file-flag-with-multiple-offload-archs.hip b/clang/test/Driver/dep-file-flag-with-multiple-offload-archs.hip
new file mode 100644
index 0000000000000..f17e56acfb7f7
--- /dev/null
+++ b/clang/test/Driver/dep-file-flag-with-multiple-offload-archs.hip
@@ -0,0 +1,12 @@
+// RUN: %clang -### -nogpuinc -nogpulib --offload-arch=gfx1030 --offload-arch=gfx1100 --offload-arch=gfx1101 --target=x86_64-linux-gnu -MD -MF tmp.d %s 2>&1 | FileCheck %s
+
+// CHECK-NOT: {{.*}}clang{{.*}}"-target-cpu" "gfx1030"{{.*}}"-dependency-file" "tmp.d"
+// CHECK: {{.*}}lld{{.*}}"-plugin-opt=mcpu=gfx1030"
+// CHECK-NOT: {{.*}}clang{{.*}}"-target-cpu" "gfx1100"{{.*}}"-dependency-file" "tmp.d"
+// CHECK: {{.*}}lld{{.*}}"-plugin-opt=mcpu=gfx1100"
+// CHECK-NOT: {{.*}}clang{{.*}}"-target-cpu" "gfx1101"{{.*}}"-dependency-file" "tmp.d"
+// CHECK: {{.*}}lld{{.*}}"-plugin-opt=mcpu=gfx1101"
+// CHECK: {{.*}}clang-offload-bundler
+// CHECK: {{.*}}clang{{.*}}"-target-cpu"{{.*}}"-dependency-file" "tmp.d"
+
+void main(){}
diff --git a/clang/test/Driver/extend-variable-liveness.c b/clang/test/Driver/extend-variable-liveness.c
new file mode 100644
index 0000000000000..bbfb2ece6f297
--- /dev/null
+++ b/clang/test/Driver/extend-variable-liveness.c
@@ -0,0 +1,14 @@
+// Tests that -fextend-variable-liveness and its aliases are correctly passed
+// by the driver.
+
+// RUN: %clang -### -c %s 2>&1 | FileCheck %s --check-prefixes=CHECK,DEFAULT
+// RUN: %clang -fextend-variable-liveness=none -### -c %s 2>&1 | FileCheck %s --check-prefixes=CHECK,NONE
+// RUN: %clang -fextend-variable-liveness=this -### -c %s 2>&1 | FileCheck %s --check-prefixes=CHECK,THIS
+// RUN: %clang -fextend-variable-liveness=all -### -c %s 2>&1 | FileCheck %s --check-prefixes=CHECK,ALL
+// RUN: %clang -fextend-variable-liveness -### -c %s 2>&1 | FileCheck %s --check-prefixes=CHECK,ALL
+
+// CHECK:       "-cc1"
+// DEFAULT-NOT: -fextend-variable-liveness
+// NONE-SAME:   "-fextend-variable-liveness=none"
+// THIS-SAME:   "-fextend-variable-liveness=this"
+// ALL-SAME:    "-fextend-variable-liveness=all"
diff --git a/clang/test/Driver/fprofile-continuous.c b/clang/test/Driver/fprofile-continuous.c
new file mode 100644
index 0000000000000..81719fb70cb1e
--- /dev/null
+++ b/clang/test/Driver/fprofile-continuous.c
@@ -0,0 +1,21 @@
+// 1) test on platforms that (do or do not) require runtime relocation
+
+// RUN: %clang --target=x86_64-darwin -fprofile-generate -fprofile-continuous -### -c %s 2>&1 | FileCheck %s --check-prefix=NO_RELOC
+// NO_RELOC: "-cc1" {{.*}} "-fprofile-continuous"
+// NO_RELOC-NOT: "-mllvm" "-runtime-counter-relocation"
+
+// RUN: %clang --target=powerpc64-ibm-aix -fprofile-generate -fprofile-continuous -### -c %s 2>&1 | FileCheck %s --check-prefix=RELOC
+// RUN: %clang --target=x86_64-unknown-fuchsia -fprofile-generate -fprofile-continuous -### -c %s 2>&1 | FileCheck %s --check-prefix=RELOC
+// RELOC: "-cc1" {{.*}} "-fprofile-continuous" "-mllvm" "-runtime-counter-relocation"
+
+// 2) test -fprofile-continuous with cs-profile-generate and -fprofile-instr-generate
+
+// RUN: %clang --target=powerpc-ibm-aix -fprofile-instr-generate -fprofile-continuous -### -c %s 2>&1 | FileCheck %s --check-prefix=CLANG_PGO
+// RUN: %clang --target=powerpc64le-unknown-linux -fprofile-instr-generate= -fprofile-continuous -### -c %s 2>&1 | FileCheck %s --check-prefix=CLANG_PGO
+// CLANG_PGO: "-cc1" {{.*}} "-fprofile-continuous" "-mllvm" "-runtime-counter-relocation" "-fprofile-instrument-path=default.profraw"
+
+// RUN: %clang --target=x86_64-unknown-fuchsia -fcs-profile-generate -fprofile-continuous -### -c %s 2>&1 | FileCheck %s --check-prefix=RELOC
+
+// RUN: not %clang -fprofile-continuous -### -c %s 2>&1 | FileCheck %s --check-prefix=ERROR
+// ERROR: error: invalid argument '-fprofile-continuous' only allowed with '-fprofile-generate, -fprofile-instr-generate, or -fcs-profile-generate'
+void foo(){}
diff --git a/clang/test/Driver/frelaxed-template-template-args.cpp b/clang/test/Driver/frelaxed-template-template-args.cpp
deleted file mode 100644
index 7a7fd6f0bbc8f..0000000000000
--- a/clang/test/Driver/frelaxed-template-template-args.cpp
+++ /dev/null
@@ -1,9 +0,0 @@
-// RUN: %clang -fsyntax-only -### %s 2>&1 | FileCheck --check-prefix=CHECK-DEF %s
-// RUN: %clang -fsyntax-only -frelaxed-template-template-args %s 2>&1 | FileCheck --check-prefix=CHECK-ON %s
-// RUN: %clang -fsyntax-only -fno-relaxed-template-template-args %s 2>&1 | FileCheck --check-prefix=CHECK-OFF %s
-// RUN: %clang -fsyntax-only -fno-relaxed-template-template-args -Wno-deprecated-no-relaxed-template-template-args %s 2>&1 | FileCheck --check-prefix=CHECK-DIS --allow-empty %s
-
-// CHECK-DEF-NOT: "-cc1"{{.*}} "-fno-relaxed-template-template-args"
-// CHECK-ON:  warning: argument '-frelaxed-template-template-args' is deprecated [-Wdeprecated]
-// CHECK-OFF: warning: argument '-fno-relaxed-template-template-args' is deprecated [-Wdeprecated-no-relaxed-template-template-args]
-// CHECK-DIS-NOT: warning: argument '-fno-relaxed-template-template-args' is deprecated
diff --git a/clang/test/Driver/hexagon-cpu-default.c b/clang/test/Driver/hexagon-cpu-default.c
new file mode 100644
index 0000000000000..31fb839f21656
--- /dev/null
+++ b/clang/test/Driver/hexagon-cpu-default.c
@@ -0,0 +1,4 @@
+// CHECK: "-target-cpu" "hexagonv68"
+
+// RUN: %clang -c %s -### --target=hexagon-unknown-elf \
+// RUN:  2>&1 | FileCheck  %s
diff --git a/clang/test/Driver/hip-sanitize-options.hip b/clang/test/Driver/hip-sanitize-options.hip
index d94cbdacdaeb3..8a852867f5b3b 100644
--- a/clang/test/Driver/hip-sanitize-options.hip
+++ b/clang/test/Driver/hip-sanitize-options.hip
@@ -1,5 +1,5 @@
 // RUN: %clang -### --target=x86_64-unknown-linux-gnu --offload-arch=gfx900:xnack+ \
-// RUN:   -fsanitize=address \
+// RUN:   -fsanitize=address -fgpu-sanitize \
 // RUN:   -nogpuinc --rocm-path=%S/Inputs/rocm \
 // RUN:   %s 2>&1 | FileCheck -check-prefixes=NORDC %s
 
@@ -18,7 +18,7 @@
 // RUN:   -nogpuinc --rocm-path=%S/Inputs/rocm \
 // RUN:   %s 2>&1 | FileCheck -check-prefixes=RDC %s
 
-// RUN: not %clang -### --target=x86_64-unknown-linux-gnu --offload-arch=gfx900:xnack+ \
+// RUN: not %clang -### --target=x86_64-unknown-linux-gnu -mcode-object-version=5 --offload-arch=gfx900:xnack+ \
 // RUN:   -fsanitize=address -fgpu-sanitize \
 // RUN:   -nogpuinc --rocm-path=%S/Inputs/rocm-invalid \
 // RUN:   %s 2>&1 | FileCheck -check-prefixes=FAIL %s
@@ -52,15 +52,15 @@
 // CHECK-NOT: {{"[^"]*lld(\.exe){0,1}".* ".*hip.bc"}}
 // CHECK: {{"[^"]*clang[^"]*".* "-triple" "x86_64-unknown-linux-gnu".* "-fsanitize=address"}}
 
-// NORDC: {{"[^"]*clang[^"]*".* "-emit-obj".* "-fcuda-is-device".* "-mlink-bitcode-file" ".*asanrtl.bc".* "-mlink-builtin-bitcode" ".*hip.bc".* "-fsanitize=address".*}} "-o" "[[OUT:[^"]*.o]]"
+// NORDC: {{"[^"]*clang[^"]*".* "-emit-obj".* "-fcuda-is-device".* "-mlink-builtin-bitcode" ".*hip.bc".* "-mlink-bitcode-file" ".*asanrtl.bc".* "-fsanitize=address".*}} "-o" "[[OUT:[^"]*.o]]"
 // NORDC-NOT: {{"[^"]*lld(\.exe){0,1}".*}} "[[OUT]]" {{".*asanrtl.bc" ".*hip.bc"}}
 // NORDC: {{"[^"]*clang[^"]*".* "-triple" "x86_64-unknown-linux-gnu".* "-fsanitize=address"}}
 
 // RDC: {{"[^"]*clang[^"]*".* "-triple" "x86_64-unknown-linux-gnu".* "-fsanitize=address"}}
-// RDC: {{"[^"]*clang[^"]*".* "-emit-llvm-bc".* "-fcuda-is-device".* "-mlink-bitcode-file" ".*asanrtl.bc".* "-mlink-builtin-bitcode" ".*hip.bc".* "-fsanitize=address".*}} "-o" "[[OUT:[^"]*.bc]]"
+// RDC: {{"[^"]*clang[^"]*".* "-emit-llvm-bc".* "-fcuda-is-device".* "-mlink-builtin-bitcode" ".*hip.bc".* "-mlink-bitcode-file" ".*asanrtl.bc".* "-fsanitize=address".*}} "-o" "[[OUT:[^"]*.bc]]"
 // RDC-NOT: {{"[^"]*lld(\.exe){0,1}".*}} "[[OUT]]" {{".*asanrtl.bc" ".*hip.bc"}}
 
-// FAIL: AMDGPU address sanitizer runtime library (asanrtl) is not found. Please install ROCm device library which supports address sanitizer
+// FAIL: error: cannot find ROCm device library for ABI version 5; provide its path via '--rocm-path' or '--rocm-device-lib-path', or pass '-nogpulib' to build without ROCm device library
 
 // XNACK-DAG: warning: ignoring '-fsanitize=leak' option as it is not currently supported for target 'amdgcn-amd-amdhsa'
 // XNACK-DAG: warning: ignoring '-fsanitize=address' option for offload arch 'gfx900:xnack-' as it is not currently supported there. Use it with an offload arch containing 'xnack+' instead
diff --git a/clang/test/Driver/linker-wrapper-image.c b/clang/test/Driver/linker-wrapper-image.c
index 7f96f629e9127..775385137c75f 100644
--- a/clang/test/Driver/linker-wrapper-image.c
+++ b/clang/test/Driver/linker-wrapper-image.c
@@ -14,18 +14,18 @@
 // RUN: clang-linker-wrapper --print-wrapped-module --dry-run --host-triple=x86_64-unknown-windows-gnu \
 // RUN:   --linker-path=/usr/bin/ld %t.o -o a.out 2>&1 | FileCheck %s --check-prefixes=OPENMP,OPENMP-COFF
 
-//      OPENMP-ELF: @__start_omp_offloading_entries = external hidden constant [0 x %struct.__tgt_offload_entry]
-// OPENMP-ELF-NEXT: @__stop_omp_offloading_entries = external hidden constant [0 x %struct.__tgt_offload_entry]
-// OPENMP-ELF-NEXT: @__dummy.omp_offloading_entries = internal constant [0 x %struct.__tgt_offload_entry] zeroinitializer, section "omp_offloading_entries"
+//      OPENMP-ELF: @__start_llvm_offload_entries = external hidden constant [0 x %struct.__tgt_offload_entry]
+// OPENMP-ELF-NEXT: @__stop_llvm_offload_entries = external hidden constant [0 x %struct.__tgt_offload_entry]
+// OPENMP-ELF-NEXT: @__dummy.llvm_offload_entries = internal constant [0 x %struct.__tgt_offload_entry] zeroinitializer, section "llvm_offload_entries"
 
-//      OPENMP-COFF: @__start_omp_offloading_entries = weak_odr hidden constant [0 x %struct.__tgt_offload_entry] zeroinitializer, section "omp_offloading_entries$OA"
-// OPENMP-COFF-NEXT: @__stop_omp_offloading_entries = weak_odr hidden constant [0 x %struct.__tgt_offload_entry] zeroinitializer, section "omp_offloading_entries$OZ"
+//      OPENMP-COFF: @__start_llvm_offload_entries = weak_odr hidden constant [0 x %struct.__tgt_offload_entry] zeroinitializer, section "llvm_offload_entries$OA"
+// OPENMP-COFF-NEXT: @__stop_llvm_offload_entries = weak_odr hidden constant [0 x %struct.__tgt_offload_entry] zeroinitializer, section "llvm_offload_entries$OZ"
 
 // OPENMP-REL: @.omp_offloading.device_image = internal unnamed_addr constant [[[SIZE:[0-9]+]] x i8] c"\10\FF\10\AD{{.*}}", section ".llvm.offloading.relocatable", align 8
 
 //      OPENMP: @.omp_offloading.device_image = internal unnamed_addr constant [[[SIZE:[0-9]+]] x i8] c"\10\FF\10\AD{{.*}}", section ".llvm.offloading", align 8
-// OPENMP-NEXT: @.omp_offloading.device_images = internal unnamed_addr constant [1 x %__tgt_device_image] [%__tgt_device_image { ptr getelementptr ([[[BEGIN:[0-9]+]] x i8], ptr @.omp_offloading.device_image, i64 0, i64 144), ptr getelementptr ([[[END:[0-9]+]] x i8], ptr @.omp_offloading.device_image, i64 0, i64 144), ptr @__start_omp_offloading_entries, ptr @__stop_omp_offloading_entries }]
-// OPENMP-NEXT: @.omp_offloading.descriptor = internal constant %__tgt_bin_desc { i32 1, ptr @.omp_offloading.device_images, ptr @__start_omp_offloading_entries, ptr @__stop_omp_offloading_entries }
+// OPENMP-NEXT: @.omp_offloading.device_images = internal unnamed_addr constant [1 x %__tgt_device_image] [%__tgt_device_image { ptr getelementptr ([[[BEGIN:[0-9]+]] x i8], ptr @.omp_offloading.device_image, i64 0, i64 144), ptr getelementptr ([[[END:[0-9]+]] x i8], ptr @.omp_offloading.device_image, i64 0, i64 144), ptr @__start_llvm_offload_entries, ptr @__stop_llvm_offload_entries }]
+// OPENMP-NEXT: @.omp_offloading.descriptor = internal constant %__tgt_bin_desc { i32 1, ptr @.omp_offloading.device_images, ptr @__start_llvm_offload_entries, ptr @__stop_llvm_offload_entries }
 // OPENMP-NEXT: @llvm.global_ctors = appending global [1 x { i32, ptr, ptr }] [{ i32, ptr, ptr } { i32 101, ptr @.omp_offloading.descriptor_reg, ptr null }]
 
 //      OPENMP: define internal void @.omp_offloading.descriptor_reg() section ".text.startup" {
@@ -51,12 +51,12 @@
 // RUN: clang-linker-wrapper --print-wrapped-module --dry-run --host-triple=x86_64-unknown-windows-gnu \
 // RUN:   --linker-path=/usr/bin/ld %t.o -o a.out 2>&1 | FileCheck %s --check-prefixes=CUDA,CUDA-COFF
 
-//      CUDA-ELF: @__start_cuda_offloading_entries = external hidden constant [0 x %struct.__tgt_offload_entry]
-// CUDA-ELF-NEXT: @__stop_cuda_offloading_entries = external hidden constant [0 x %struct.__tgt_offload_entry]
-// CUDA-ELF-NEXT: @__dummy.cuda_offloading_entries = internal constant [0 x %struct.__tgt_offload_entry] zeroinitializer, section "cuda_offloading_entries"
+//      CUDA-ELF: @__start_llvm_offload_entries = external hidden constant [0 x %struct.__tgt_offload_entry]
+// CUDA-ELF-NEXT: @__stop_llvm_offload_entries = external hidden constant [0 x %struct.__tgt_offload_entry]
+// CUDA-ELF-NEXT: @__dummy.llvm_offload_entries = internal constant [0 x %struct.__tgt_offload_entry] zeroinitializer, section "llvm_offload_entries"
 
-//      CUDA-COFF: @__start_cuda_offloading_entries = weak_odr hidden constant [0 x %struct.__tgt_offload_entry] zeroinitializer, section "cuda_offloading_entries$OA"
-// CUDA-COFF-NEXT: @__stop_cuda_offloading_entries = weak_odr hidden constant [0 x %struct.__tgt_offload_entry] zeroinitializer, section "cuda_offloading_entries$OZ"
+//      CUDA-COFF: @__start_llvm_offload_entries = weak_odr hidden constant [0 x %struct.__tgt_offload_entry] zeroinitializer, section "llvm_offload_entries$OA"
+// CUDA-COFF-NEXT: @__stop_llvm_offload_entries = weak_odr hidden constant [0 x %struct.__tgt_offload_entry] zeroinitializer, section "llvm_offload_entries$OZ"
 
 //      CUDA: @.fatbin_image = internal constant [0 x i8] zeroinitializer, section ".nv_fatbin"
 // CUDA-NEXT: @.fatbin_wrapper = internal constant %fatbin_wrapper { i32 1180844977, i32 1, ptr @.fatbin_image, ptr null }, section ".nvFatBinSegment", align 8
@@ -73,45 +73,54 @@
 // CUDA-NEXT:   %1 = call i32 @atexit(ptr @.cuda.fatbin_unreg)
 // CUDA-NEXT:   ret void
 // CUDA-NEXT: }
-
+//
 //      CUDA: define internal void @.cuda.fatbin_unreg() section ".text.startup" {
 // CUDA-NEXT: entry:
 // CUDA-NEXT:   %0 = load ptr, ptr @.cuda.binary_handle, align 8
 // CUDA-NEXT:   call void @__cudaUnregisterFatBinary(ptr %0)
 // CUDA-NEXT:   ret void
 // CUDA-NEXT: }
-
+//
 //      CUDA: define internal void @.cuda.globals_reg(ptr %0) section ".text.startup" {
 // CUDA-NEXT: entry:
-// CUDA-NEXT:   %1 = icmp ne ptr @__start_cuda_offloading_entries, @__stop_cuda_offloading_entries
+// CUDA-NEXT:   %1 = icmp ne ptr @__start_llvm_offload_entries, @__stop_llvm_offload_entries
 // CUDA-NEXT:   br i1 %1, label %while.entry, label %while.end
-
+//
 //      CUDA: while.entry:
-// CUDA-NEXT:   %entry1 = phi ptr [ @__start_cuda_offloading_entries, %entry ], [ %13, %if.end ]
-// CUDA-NEXT:   %2 = getelementptr inbounds %struct.__tgt_offload_entry, ptr %entry1, i64 0, i32 0
+// CUDA-NEXT:   %entry1 = phi ptr [ @__start_llvm_offload_entries, %entry ], [ %16, %if.end ]
+// CUDA-NEXT:   %2 = getelementptr inbounds %struct.__tgt_offload_entry, ptr %entry1, i32 0, i32 4
 // CUDA-NEXT:   %addr = load ptr, ptr %2, align 8
-// CUDA-NEXT:   %3 = getelementptr inbounds %struct.__tgt_offload_entry, ptr %entry1, i64 0, i32 1
-// CUDA-NEXT:   %name = load ptr, ptr %3, align 8
-// CUDA-NEXT:   %4 = getelementptr inbounds %struct.__tgt_offload_entry, ptr %entry1, i64 0, i32 2
-// CUDA-NEXT:   %size = load i64, ptr %4, align 4
-// CUDA-NEXT:   %5 = getelementptr inbounds %struct.__tgt_offload_entry, ptr %entry1, i64 0, i32 3
-// CUDA-NEXT:   %flags = load i32, ptr %5, align 4
-// CUDA-NEXT:   %6 = getelementptr inbounds %struct.__tgt_offload_entry, ptr %entry1, i64 0, i32 4
-// CUDA-NEXT:   %textype = load i32, ptr %6, align 4
+// CUDA-NEXT:   %3 = getelementptr inbounds %struct.__tgt_offload_entry, ptr %entry1, i32 0, i32 8
+// CUDA-NEXT:   %aux_addr = load ptr, ptr %3, align 8
+// CUDA-NEXT:   %4 = getelementptr inbounds %struct.__tgt_offload_entry, ptr %entry1, i32 0, i32 2
+// CUDA-NEXT:   %kind = load i16, ptr %4, align 2
+// CUDA-NEXT:   %5 = getelementptr inbounds %struct.__tgt_offload_entry, ptr %entry1, i32 0, i32 5
+// CUDA-NEXT:   %name = load ptr, ptr %5, align 8
+// CUDA-NEXT:   %6 = getelementptr inbounds %struct.__tgt_offload_entry, ptr %entry1, i32 0, i32 6
+// CUDA-NEXT:   %size = load i64, ptr %6, align 4
+// CUDA-NEXT:   %7 = getelementptr inbounds %struct.__tgt_offload_entry, ptr %entry1, i32 0, i32 3
+// CUDA-NEXT:   %flags = load i32, ptr %7, align 4
+// CUDA-NEXT:   %8 = getelementptr inbounds %struct.__tgt_offload_entry, ptr %entry1, i32 0, i32 7
+// CUDA-NEXT:   %data = load i64, ptr %8, align 4
+// CUDA-NEXT:   %9 = trunc i64 %data to i32
 // CUDA-NEXT:   %type = and i32 %flags, 7
-// CUDA-NEXT:   %7 = and i32 %flags, 8
-// CUDA-NEXT:   %extern = lshr i32 %7, 3
-// CUDA-NEXT:   %8 = and i32 %flags, 16
-// CUDA-NEXT:   %constant = lshr i32 %8, 4
-// CUDA-NEXT:   %9 = and i32 %flags, 32
-// CUDA-NEXT:   %normalized = lshr i32 %9, 5
-// CUDA-NEXT:   %10 = icmp eq i64 %size, 0
-// CUDA-NEXT:   br i1 %10, label %if.then, label %if.else
-
+// CUDA-NEXT:   %10 = and i32 %flags, 8
+// CUDA-NEXT:   %extern = lshr i32 %10, 3
+// CUDA-NEXT:   %11 = and i32 %flags, 16
+// CUDA-NEXT:   %constant = lshr i32 %11, 4
+// CUDA-NEXT:   %12 = and i32 %flags, 32
+// CUDA-NEXT:   %normalized = lshr i32 %12, 5
+// CUDA-NEXT:   %13 = icmp eq i16 %kind, 2
+// CUDA-NEXT:   br i1 %13, label %if.kind, label %if.end
+//
+//      CUDA: if.kind:
+// CUDA-NEXT:   %14 = icmp eq i64 %size, 0
+// CUDA-NEXT:   br i1 %14, label %if.then, label %if.else
+//
 //      CUDA: if.then:
-// CUDA-NEXT:   %11 = call i32 @__cudaRegisterFunction(ptr %0, ptr %addr, ptr %name, ptr %name, i32 -1, ptr null, ptr null, ptr null, ptr null, ptr null)
+// CUDA-NEXT:   %15 = call i32 @__cudaRegisterFunction(ptr %0, ptr %addr, ptr %name, ptr %name, i32 -1, ptr null, ptr null, ptr null, ptr null, ptr null)
 // CUDA-NEXT:   br label %if.end
-
+//
 //      CUDA: if.else:
 // CUDA-NEXT:   switch i32 %type, label %if.end [
 // CUDA-NEXT:     i32 0, label %sw.global
@@ -119,29 +128,26 @@
 // CUDA-NEXT:     i32 2, label %sw.surface
 // CUDA-NEXT:     i32 3, label %sw.texture
 // CUDA-NEXT:   ]
-
+//
 //      CUDA: sw.global:
 // CUDA-NEXT:   call void @__cudaRegisterVar(ptr %0, ptr %addr, ptr %name, ptr %name, i32 %extern, i64 %size, i32 %constant, i32 0)
 // CUDA-NEXT:   br label %if.end
-
+//
 //      CUDA: sw.managed:
-// CUDA-NEXT:  %managed.addr = load ptr, ptr %addr, align 8
-// CUDA-NEXT:  %12 = getelementptr inbounds ptr, ptr %addr, i64 1
-// CUDA-NEXT:  %managed.addr2 = load ptr, ptr %12, align 8
-// CUDA-NEXT:  call void @__cudaRegisterManagedVar(ptr %0, ptr %managed.addr, ptr %managed.addr2, ptr %name, i64 %size, i32 %textype)
-// CUDA-NEXT:  br label %if.end
-
+// CUDA-NEXT:   call void @__cudaRegisterManagedVar(ptr %0, ptr %aux_addr, ptr %addr, ptr %name, i64 %size, i32 %9)
+// CUDA-NEXT:   br label %if.end
+//
 //      CUDA: sw.surface:
 // CUDA-NEXT:   br label %if.end
-
+//
 //      CUDA: sw.texture:
 // CUDA-NEXT:   br label %if.end
-
+//
 //      CUDA: if.end:
-// CUDA-NEXT:   %13 = getelementptr inbounds %struct.__tgt_offload_entry, ptr %entry1, i64 1
-// CUDA-NEXT:   %14 = icmp eq ptr %13, @__stop_cuda_offloading_entries
-// CUDA-NEXT:   br i1 %14, label %while.end, label %while.entry
-
+// CUDA-NEXT:   %16 = getelementptr inbounds %struct.__tgt_offload_entry, ptr %entry1, i64 1
+// CUDA-NEXT:   %17 = icmp eq ptr %16, @__stop_llvm_offload_entries
+// CUDA-NEXT:   br i1 %17, label %while.end, label %while.entry
+//
 //      CUDA: while.end:
 // CUDA-NEXT:   ret void
 // CUDA-NEXT: }
@@ -156,12 +162,12 @@
 // RUN: clang-linker-wrapper --print-wrapped-module --dry-run --host-triple=x86_64-unknown-windows-gnu \
 // RUN:   --linker-path=/usr/bin/ld %t.o -o a.out 2>&1 | FileCheck %s --check-prefixes=HIP,HIP-COFF
 
-//      HIP-ELF: @__start_hip_offloading_entries = external hidden constant [0 x %struct.__tgt_offload_entry]
-// HIP-ELF-NEXT: @__stop_hip_offloading_entries = external hidden constant [0 x %struct.__tgt_offload_entry]
-// HIP-ELF-NEXT: @__dummy.hip_offloading_entries = internal constant [0 x %struct.__tgt_offload_entry] zeroinitializer, section "hip_offloading_entries"
+//      HIP-ELF: @__start_llvm_offload_entries = external hidden constant [0 x %struct.__tgt_offload_entry]
+// HIP-ELF-NEXT: @__stop_llvm_offload_entries = external hidden constant [0 x %struct.__tgt_offload_entry]
+// HIP-ELF-NEXT: @__dummy.llvm_offload_entries = internal constant [0 x %struct.__tgt_offload_entry] zeroinitializer, section "llvm_offload_entries"
 
-//      HIP-COFF: @__start_hip_offloading_entries = weak_odr hidden constant [0 x %struct.__tgt_offload_entry] zeroinitializer, section "hip_offloading_entries$OA"
-// HIP-COFF-NEXT: @__stop_hip_offloading_entries = weak_odr hidden constant [0 x %struct.__tgt_offload_entry] zeroinitializer, section "hip_offloading_entries$OZ"
+//      HIP-COFF: @__start_llvm_offload_entries = weak_odr hidden constant [0 x %struct.__tgt_offload_entry] zeroinitializer, section "llvm_offload_entries$OA"
+// HIP-COFF-NEXT: @__stop_llvm_offload_entries = weak_odr hidden constant [0 x %struct.__tgt_offload_entry] zeroinitializer, section "llvm_offload_entries$OZ"
 
 //      HIP: @.fatbin_image = internal constant [0 x i8] zeroinitializer, section ".hip_fatbin"
 // HIP-NEXT: @.fatbin_wrapper = internal constant %fatbin_wrapper { i32 1212764230, i32 1, ptr @.fatbin_image, ptr null }, section ".hipFatBinSegment", align 8
@@ -177,45 +183,54 @@
 // HIP-NEXT:   %1 = call i32 @atexit(ptr @.hip.fatbin_unreg)
 // HIP-NEXT:   ret void
 // HIP-NEXT: }
-
+//
 //      HIP: define internal void @.hip.fatbin_unreg() section ".text.startup" {
 // HIP-NEXT: entry:
 // HIP-NEXT:   %0 = load ptr, ptr @.hip.binary_handle, align 8
 // HIP-NEXT:   call void @__hipUnregisterFatBinary(ptr %0)
 // HIP-NEXT:   ret void
 // HIP-NEXT: }
-
+//
 //      HIP: define internal void @.hip.globals_reg(ptr %0) section ".text.startup" {
 // HIP-NEXT: entry:
-// HIP-NEXT:   %1 = icmp ne ptr @__start_hip_offloading_entries, @__stop_hip_offloading_entries
+// HIP-NEXT:   %1 = icmp ne ptr @__start_llvm_offload_entries, @__stop_llvm_offload_entries
 // HIP-NEXT:   br i1 %1, label %while.entry, label %while.end
-
+//
 //      HIP: while.entry:
-// HIP-NEXT:   %entry1 = phi ptr [ @__start_hip_offloading_entries, %entry ], [ %13, %if.end ]
-// HIP-NEXT:   %2 = getelementptr inbounds %struct.__tgt_offload_entry, ptr %entry1, i64 0, i32 0
+// HIP-NEXT:   %entry1 = phi ptr [ @__start_llvm_offload_entries, %entry ], [ %16, %if.end ]
+// HIP-NEXT:   %2 = getelementptr inbounds %struct.__tgt_offload_entry, ptr %entry1, i32 0, i32 4
 // HIP-NEXT:   %addr = load ptr, ptr %2, align 8
-// HIP-NEXT:   %3 = getelementptr inbounds %struct.__tgt_offload_entry, ptr %entry1, i64 0, i32 1
-// HIP-NEXT:   %name = load ptr, ptr %3, align 8
-// HIP-NEXT:   %4 = getelementptr inbounds %struct.__tgt_offload_entry, ptr %entry1, i64 0, i32 2
-// HIP-NEXT:   %size = load i64, ptr %4, align 4
-// HIP-NEXT:   %5 = getelementptr inbounds %struct.__tgt_offload_entry, ptr %entry1, i64 0, i32 3
-// HIP-NEXT:   %flags = load i32, ptr %5, align 4
-// HIP-NEXT:   %6 = getelementptr inbounds %struct.__tgt_offload_entry, ptr %entry1, i64 0, i32 4
-// HIP-NEXT:   %textype = load i32, ptr %6, align 4
+// HIP-NEXT:   %3 = getelementptr inbounds %struct.__tgt_offload_entry, ptr %entry1, i32 0, i32 8
+// HIP-NEXT:   %aux_addr = load ptr, ptr %3, align 8
+// HIP-NEXT:   %4 = getelementptr inbounds %struct.__tgt_offload_entry, ptr %entry1, i32 0, i32 2
+// HIP-NEXT:   %kind = load i16, ptr %4, align 2
+// HIP-NEXT:   %5 = getelementptr inbounds %struct.__tgt_offload_entry, ptr %entry1, i32 0, i32 5
+// HIP-NEXT:   %name = load ptr, ptr %5, align 8
+// HIP-NEXT:   %6 = getelementptr inbounds %struct.__tgt_offload_entry, ptr %entry1, i32 0, i32 6
+// HIP-NEXT:   %size = load i64, ptr %6, align 4
+// HIP-NEXT:   %7 = getelementptr inbounds %struct.__tgt_offload_entry, ptr %entry1, i32 0, i32 3
+// HIP-NEXT:   %flags = load i32, ptr %7, align 4
+// HIP-NEXT:   %8 = getelementptr inbounds %struct.__tgt_offload_entry, ptr %entry1, i32 0, i32 7
+// HIP-NEXT:   %data = load i64, ptr %8, align 4
+// HIP-NEXT:   %9 = trunc i64 %data to i32
 // HIP-NEXT:   %type = and i32 %flags, 7
-// HIP-NEXT:   %7 = and i32 %flags, 8
-// HIP-NEXT:   %extern = lshr i32 %7, 3
-// HIP-NEXT:   %8 = and i32 %flags, 16
-// HIP-NEXT:   %constant = lshr i32 %8, 4
-// HIP-NEXT:   %9 = and i32 %flags, 32
-// HIP-NEXT:   %normalized = lshr i32 %9, 5
-// HIP-NEXT:   %10 = icmp eq i64 %size, 0
-// HIP-NEXT:   br i1 %10, label %if.then, label %if.else
-
+// HIP-NEXT:   %10 = and i32 %flags, 8
+// HIP-NEXT:   %extern = lshr i32 %10, 3
+// HIP-NEXT:   %11 = and i32 %flags, 16
+// HIP-NEXT:   %constant = lshr i32 %11, 4
+// HIP-NEXT:   %12 = and i32 %flags, 32
+// HIP-NEXT:   %normalized = lshr i32 %12, 5
+// HIP-NEXT:   %13 = icmp eq i16 %kind, 3
+// HIP-NEXT:   br i1 %13, label %if.kind, label %if.end
+//
+//      HIP: if.kind:
+// HIP-NEXT:   %14 = icmp eq i64 %size, 0
+// HIP-NEXT:   br i1 %14, label %if.then, label %if.else
+//
 //      HIP: if.then:
-// HIP-NEXT:   %11 = call i32 @__hipRegisterFunction(ptr %0, ptr %addr, ptr %name, ptr %name, i32 -1, ptr null, ptr null, ptr null, ptr null, ptr null)
+// HIP-NEXT:   %15 = call i32 @__hipRegisterFunction(ptr %0, ptr %addr, ptr %name, ptr %name, i32 -1, ptr null, ptr null, ptr null, ptr null, ptr null)
 // HIP-NEXT:   br label %if.end
-
+//
 //      HIP: if.else:
 // HIP-NEXT:   switch i32 %type, label %if.end [
 // HIP-NEXT:     i32 0, label %sw.global
@@ -223,31 +238,28 @@
 // HIP-NEXT:     i32 2, label %sw.surface
 // HIP-NEXT:     i32 3, label %sw.texture
 // HIP-NEXT:   ]
-
+//
 //      HIP: sw.global:
 // HIP-NEXT:   call void @__hipRegisterVar(ptr %0, ptr %addr, ptr %name, ptr %name, i32 %extern, i64 %size, i32 %constant, i32 0)
 // HIP-NEXT:   br label %if.end
-
+//
 //      HIP: sw.managed:
-// HIP-NEXT:  %managed.addr = load ptr, ptr %addr, align 8
-// HIP-NEXT:  %12 = getelementptr inbounds ptr, ptr %addr, i64 1
-// HIP-NEXT:  %managed.addr2 = load ptr, ptr %12, align 8
-// HIP-NEXT:  call void @__hipRegisterManagedVar(ptr %0, ptr %managed.addr, ptr %managed.addr2, ptr %name, i64 %size, i32 %textype)
-// HIP-NEXT:  br label %if.end
-
+// HIP-NEXT:   call void @__hipRegisterManagedVar(ptr %0, ptr %aux_addr, ptr %addr, ptr %name, i64 %size, i32 %9)
+// HIP-NEXT:   br label %if.end
+//
 //      HIP: sw.surface:
-// HIP-NEXT:   call void @__hipRegisterSurface(ptr %0, ptr %addr, ptr %name, ptr %name, i32 %textype, i32 %extern)
+// HIP-NEXT:   call void @__hipRegisterSurface(ptr %0, ptr %addr, ptr %name, ptr %name, i32 %9, i32 %extern)
 // HIP-NEXT:   br label %if.end
-
+//
 //      HIP: sw.texture:
-// HIP-NEXT:   call void @__hipRegisterTexture(ptr %0, ptr %addr, ptr %name, ptr %name, i32 %textype, i32 %normalized, i32 %extern)
+// HIP-NEXT:   call void @__hipRegisterTexture(ptr %0, ptr %addr, ptr %name, ptr %name, i32 %9, i32 %normalized, i32 %extern)
 // HIP-NEXT:   br label %if.end
-
+//
 //      HIP: if.end:
-// HIP-NEXT:   %13 = getelementptr inbounds %struct.__tgt_offload_entry, ptr %entry1, i64 1
-// HIP-NEXT:   %14 = icmp eq ptr %13, @__stop_hip_offloading_entries
-// HIP-NEXT:   br i1 %14, label %while.end, label %while.entry
-
+// HIP-NEXT:   %16 = getelementptr inbounds %struct.__tgt_offload_entry, ptr %entry1, i64 1
+// HIP-NEXT:   %17 = icmp eq ptr %16, @__stop_llvm_offload_entries
+// HIP-NEXT:   br i1 %17, label %while.end, label %while.entry
+//
 //      HIP: while.end:
 // HIP-NEXT:   ret void
 // HIP-NEXT: }
diff --git a/clang/test/Driver/linker-wrapper.c b/clang/test/Driver/linker-wrapper.c
index f416ee5f4463b..df0a1d1e9a84d 100644
--- a/clang/test/Driver/linker-wrapper.c
+++ b/clang/test/Driver/linker-wrapper.c
@@ -59,7 +59,7 @@ __attribute__((visibility("protected"), used)) int x;
 // RUN:   --linker-path=/usr/bin/ld.lld --whole-archive %t.a --no-whole-archive \
 // RUN:   %t.o -o a.out 2>&1 | FileCheck %s --check-prefix=CPU-LINK
 
-// CPU-LINK: clang{{.*}} -o {{.*}}.img --target=x86_64-unknown-linux-gnu -march=native -O2 -flto -Wl,--no-undefined {{.*}}.o {{.*}}.o -Wl,-Bsymbolic -shared -Wl,--whole-archive {{.*}}.a -Wl,--no-whole-archive
+// CPU-LINK: clang{{.*}} -o {{.*}}.img --target=x86_64-unknown-linux-gnu -O2 -flto -Wl,--no-undefined {{.*}}.o {{.*}}.o -Wl,-Bsymbolic -shared -Wl,--whole-archive {{.*}}.a -Wl,--no-whole-archive
 
 // RUN: %clang -cc1 %s -triple x86_64-unknown-linux-gnu -emit-obj -o %t.o
 // RUN: clang-linker-wrapper --dry-run --host-triple=x86_64-unknown-linux-gnu -mllvm -openmp-opt-disable \
diff --git a/clang/test/Driver/objc-sdk-migration-options.m b/clang/test/Driver/objc-sdk-migration-options.m
deleted file mode 100644
index 8f7e5c72a3cff..0000000000000
--- a/clang/test/Driver/objc-sdk-migration-options.m
+++ /dev/null
@@ -1,8 +0,0 @@
-// Check miscellaneous Objective-C sdk migration options.
-
-// RUN: %clang  -objcmt-migrate-property-dot-syntax -target x86_64-apple-darwin10 -S -### %s \
-// RUN:   -arch x86_64 2> %t
-// RUN: FileCheck < %t %s
-
-// CHECK: "-cc1"
-// CHECK: -objcmt-migrate-property-dot-syntax
diff --git a/clang/test/Driver/offload-Xarch.c b/clang/test/Driver/offload-Xarch.c
new file mode 100644
index 0000000000000..0f8f40a5cbd74
--- /dev/null
+++ b/clang/test/Driver/offload-Xarch.c
@@ -0,0 +1,45 @@
+// UNSUPPORTED: target={{.*darwin.*}}
+
+// RUN: %clang --target=x86_64-unknown-linux-gnu -x cuda %s -Xarch_nvptx64 -O3 -S -nogpulib -nogpuinc -### 2>&1 | FileCheck -check-prefix=O3ONCE %s
+// RUN: %clang -x cuda %s -Xarch_device -O3 -S -nogpulib -nogpuinc -### 2>&1 | FileCheck -check-prefix=O3ONCE %s
+// RUN: %clang -x hip %s -Xarch_amdgcn -O3 -S -nogpulib -nogpuinc -### 2>&1 | FileCheck -check-prefix=O3ONCE %s
+// RUN: %clang -fopenmp=libomp -fopenmp-targets=amdgcn-amd-amdhsa -nogpulib -nogpuinc \
+// RUN:   -Xarch_amdgcn -march=gfx90a -Xarch_amdgcn -O3 -S -### %s 2>&1 \
+// RUN: | FileCheck -check-prefix=O3ONCE %s
+// RUN: %clang -fopenmp=libomp -fopenmp-targets=nvptx64-nvidia-cuda -nogpulib -nogpuinc \
+// RUN:   -Xarch_nvptx64 -march=sm_52 -Xarch_nvptx64 -O3 -S -### %s 2>&1 \
+// RUN: | FileCheck -check-prefix=O3ONCE %s
+// O3ONCE: "-O3"
+// O3ONCE-NOT: "-O3"
+
+// RUN: %clang -fopenmp=libomp -fopenmp-targets=nvptx64-nvidia-cuda,amdgcn-amd-amdhsa -nogpulib \
+// RUN:   --target=x86_64-unknown-linux-gnu -Xopenmp-target=nvptx64-nvidia-cuda --offload-arch=sm_52,sm_60 -nogpuinc \
+// RUN:   -Xopenmp-target=amdgcn-amd-amdhsa --offload-arch=gfx90a,gfx1030 -ccc-print-bindings -### %s 2>&1 \
+// RUN: | FileCheck -check-prefix=OPENMP %s
+
+// OPENMP: # "x86_64-unknown-linux-gnu" - "clang", inputs: ["[[INPUT:.+]]"], output: "[[HOST_BC:.+]]"
+// OPENMP: # "amdgcn-amd-amdhsa" - "clang", inputs: ["[[INPUT]]", "[[HOST_BC]]"], output: "[[GFX1030_BC:.+]]"
+// OPENMP: # "amdgcn-amd-amdhsa" - "clang", inputs: ["[[INPUT]]", "[[HOST_BC]]"], output: "[[GFX90A_BC:.+]]"
+// OPENMP: # "nvptx64-nvidia-cuda" - "clang", inputs: ["[[INPUT]]", "[[HOST_BC]]"], output: "[[SM52_PTX:.+]]"
+// OPENMP: # "nvptx64-nvidia-cuda" - "NVPTX::Assembler", inputs: ["[[SM52_PTX]]"], output: "[[SM52_CUBIN:.+]]"
+// OPENMP: # "nvptx64-nvidia-cuda" - "clang", inputs: ["[[INPUT]]", "[[HOST_BC]]"], output: "[[SM60_PTX:.+]]"
+// OPENMP: # "nvptx64-nvidia-cuda" - "NVPTX::Assembler", inputs: ["[[SM60_PTX]]"], output: "[[SM60_CUBIN:.+]]"
+// OPENMP: # "x86_64-unknown-linux-gnu" - "Offload::Packager", inputs: ["[[GFX1030_BC]]", "[[GFX90A_BC]]", "[[SM52_CUBIN]]", "[[SM60_CUBIN]]"], output: "[[BINARY:.+]]"
+// OPENMP: # "x86_64-unknown-linux-gnu" - "clang", inputs: ["[[HOST_BC]]", "[[BINARY]]"], output: "[[HOST_OBJ:.+]]"
+// OPENMP: # "x86_64-unknown-linux-gnu" - "Offload::Linker", inputs: ["[[HOST_OBJ]]"], output: "a.out"
+
+// RUN: %clang -x cuda %s --offload-arch=sm_52,sm_60 -Xarch_sm_52 -O3 -Xarch_sm_60 -O0 \
+// RUN:   --target=x86_64-unknown-linux-gnu -Xarch_host -O3 -S -nogpulib -nogpuinc -### 2>&1 \
+// RUN: | FileCheck -check-prefix=CUDA %s
+// CUDA: "-cc1" "-triple" "nvptx64-nvidia-cuda" {{.*}}"-target-cpu" "sm_52" {{.*}}"-O3"
+// CUDA: "-cc1" "-triple" "nvptx64-nvidia-cuda" {{.*}}"-target-cpu" "sm_60" {{.*}}"-O0"
+// CUDA: "-cc1" "-triple" "x86_64-unknown-linux-gnu" {{.*}}"-O3"
+
+// Make sure that `-Xarch_amdgcn` forwards libraries to the device linker.
+// RUN: %clang -fopenmp=libomp --offload-arch=gfx90a -nogpulib -nogpuinc \
+// RUN:   --target=x86_64-unknown-linux-gnu -Xarch_amdgcn -Wl,-lfoo -### %s 2>&1 \
+// RUN: | FileCheck -check-prefix=LIBS %s
+// RUN: %clang -fopenmp=libomp --offload-arch=gfx90a -nogpulib -nogpuinc \
+// RUN:   -Xoffload-linker-amdgcn-amd-amdhsa -lfoo -### %s 2>&1 \
+// RUN: | FileCheck -check-prefix=LIBS %s
+// LIBS: "--device-linker=amdgcn-amd-amdhsa=-lfoo"
diff --git a/clang/test/Driver/openbsd.c b/clang/test/Driver/openbsd.c
index 672cd3adf44a6..6639e9d2d9d67 100644
--- a/clang/test/Driver/openbsd.c
+++ b/clang/test/Driver/openbsd.c
@@ -136,3 +136,13 @@
 // RUN: %clang --target=amd64-unknown-openbsd -flto -### %s 2>&1 \
 // RUN:   | FileCheck -check-prefix=CHECK-LTO-FLAGS %s
 // CHECK-LTO-FLAGS: "-plugin-opt=mcpu=x86-64"
+
+// Check 64-bit ARM for BTI and PAC flags
+// RUN: %clang --target=aarch64-unknown-openbsd -### -c %s 2>&1 \
+// RUN:   | FileCheck -check-prefix=CHECK-AARCH64-BTI-PAC %s
+// CHECK-AARCH64-BTI-PAC: "-msign-return-address=non-leaf" "-msign-return-address-key=a_key" "-mbranch-target-enforce"
+
+// Check 64-bit X86 for IBT flags
+// RUN: %clang --target=amd64-unknown-openbsd -### -c %s 2>&1 \
+// RUN:   | FileCheck -check-prefix=CHECK-AMD64-IBT %s
+// CHECK-AMD64-IBT: "-fcf-protection=branch" "-fno-jump-tables"
diff --git a/clang/test/Driver/openmp-offload-gpu.c b/clang/test/Driver/openmp-offload-gpu.c
index 74bd2a6aeee46..1f7e2996068c4 100644
--- a/clang/test/Driver/openmp-offload-gpu.c
+++ b/clang/test/Driver/openmp-offload-gpu.c
@@ -235,13 +235,13 @@
 // CHECK-PHASES: 0: input, "[[INPUT:.+]]", c, (host-openmp)
 // CHECK-PHASES: 1: preprocessor, {0}, cpp-output, (host-openmp)
 // CHECK-PHASES: 2: compiler, {1}, ir, (host-openmp)
-// CHECK-PHASES: 3: input, "[[INPUT]]", c, (device-openmp)
-// CHECK-PHASES: 4: preprocessor, {3}, cpp-output, (device-openmp)
-// CHECK-PHASES: 5: compiler, {4}, ir, (device-openmp)
-// CHECK-PHASES: 6: offload, "host-openmp (x86_64-unknown-linux-gnu)" {2}, "device-openmp (nvptx64-nvidia-cuda)" {5}, ir
-// CHECK-PHASES: 7: backend, {6}, assembler, (device-openmp)
-// CHECK-PHASES: 8: assembler, {7}, object, (device-openmp)
-// CHECK-PHASES: 9: offload, "device-openmp (nvptx64-nvidia-cuda)" {8}, object
+// CHECK-PHASES: 3: input, "[[INPUT]]", c, (device-openmp, sm_52)
+// CHECK-PHASES: 4: preprocessor, {3}, cpp-output, (device-openmp, sm_52)
+// CHECK-PHASES: 5: compiler, {4}, ir, (device-openmp, sm_52)
+// CHECK-PHASES: 6: offload, "host-openmp (x86_64-unknown-linux-gnu)" {2}, "device-openmp (nvptx64-nvidia-cuda:sm_52)" {5}, ir
+// CHECK-PHASES: 7: backend, {6}, assembler, (device-openmp, sm_52)
+// CHECK-PHASES: 8: assembler, {7}, object, (device-openmp, sm_52)
+// CHECK-PHASES: 9: offload, "device-openmp (nvptx64-nvidia-cuda:sm_52)" {8}, object
 // CHECK-PHASES: 10: clang-offload-packager, {9}, image
 // CHECK-PHASES: 11: offload, "host-openmp (x86_64-unknown-linux-gnu)" {2}, "device-openmp (x86_64-unknown-linux-gnu)" {10}, ir
 // CHECK-PHASES: 12: backend, {11}, assembler, (host-openmp)
@@ -315,7 +315,7 @@
 // RUN:     -Xopenmp-target=nvptx64-nvidia-cuda -march=sm_52 --offload-device-only -nogpulib %s 2>&1 | FileCheck %s --check-prefix=CHECK-DEVICE-ONLY
 // CHECK-DEVICE-ONLY: "x86_64-unknown-linux-gnu" - "clang", inputs: ["[[INPUT:.*]]"], output: "[[HOST_BC:.*]]"
 // CHECK-DEVICE-ONLY: "nvptx64-nvidia-cuda" - "clang", inputs: ["[[INPUT]]", "[[HOST_BC]]"], output: "[[DEVICE_ASM:.*]]"
-// CHECK-DEVICE-ONLY: "nvptx64-nvidia-cuda" - "NVPTX::Assembler", inputs: ["[[DEVICE_ASM]]"], output: "{{.*}}-openmp-nvptx64-nvidia-cuda.o"
+// CHECK-DEVICE-ONLY: "nvptx64-nvidia-cuda" - "NVPTX::Assembler", inputs: ["[[DEVICE_ASM]]"], output: "{{.*}}-openmp-nvptx64-nvidia-cuda-sm_52.o"
 
 // RUN:   %clang -### --target=x86_64-unknown-linux-gnu -ccc-print-bindings -fopenmp=libomp -fopenmp-targets=nvptx64-nvidia-cuda \
 // RUN:     -Xopenmp-target=nvptx64-nvidia-cuda -march=sm_52 --offload-device-only -E -nogpulib %s 2>&1 | FileCheck %s --check-prefix=CHECK-DEVICE-ONLY-PP
diff --git a/clang/test/Driver/openmp-offload-jit.c b/clang/test/Driver/openmp-offload-jit.c
index 57f265ac37eac..b3566f06bbee1 100644
--- a/clang/test/Driver/openmp-offload-jit.c
+++ b/clang/test/Driver/openmp-offload-jit.c
@@ -1,29 +1,29 @@
 // Check that we enable LTO-mode properly with '-fopenmp-target-jit' and that it
 // still enabled LTO-mode if `-fno-offload-lto` is on.
 // RUN: %clang -### --target=x86_64-unknown-linux-gnu -ccc-print-phases -fopenmp=libomp \
-// RUN:   -fopenmp-targets=nvptx64-nvidia-cuda -fopenmp-target-jit %s 2>&1 \
+// RUN:  --offload-arch=sm_52 -fopenmp-target-jit %s 2>&1 \
 // RUN: | FileCheck -check-prefix=PHASES-JIT %s
 // RUN: %clang -### --target=x86_64-unknown-linux-gnu -ccc-print-phases -fopenmp=libomp \
-// RUN:   -fopenmp-targets=nvptx64-nvidia-cuda -foffload-lto -fopenmp-target-jit %s 2>&1 \
+// RUN:   --offload-arch=sm_52 -foffload-lto -fopenmp-target-jit %s 2>&1 \
 // RUN: | FileCheck -check-prefix=PHASES-JIT %s
 // RUN: %clang -### --target=x86_64-unknown-linux-gnu -ccc-print-phases -fopenmp=libomp \
-// RUN:   -fopenmp-targets=amdgcn-amd-amdhsa -fopenmp-target-jit %s 2>&1 \
+// RUN:   --offload-arch=gfx90a -fopenmp-target-jit %s 2>&1 \
 // RUN: | FileCheck -check-prefix=PHASES-JIT %s
 // RUN: %clang -### --target=x86_64-unknown-linux-gnu -ccc-print-phases -fopenmp=libomp \
-// RUN:   -fopenmp-targets=amdgcn-amd-amdhsa -foffload-lto -fopenmp-target-jit %s 2>&1 \
+// RUN:   --offload-arch=gfx90a -foffload-lto -fopenmp-target-jit %s 2>&1 \
 // RUN: | FileCheck -check-prefix=PHASES-JIT %s
 // RUN: not %clang -### --target=x86_64-unknown-linux-gnu -ccc-print-phases -fopenmp=libomp \
-// RUN:   -fopenmp-targets=amdgcn-amd-amdhsa -fno-offload-lto -fopenmp-target-jit %s 2>&1 \
+// RUN:   --offload-arch=gfx90a -fno-offload-lto -fopenmp-target-jit %s 2>&1 \
 // RUN: | FileCheck -check-prefix=PHASES-JIT %s
 //
 //      PHASES-JIT: 0: input, "[[INPUT:.+]]", c, (host-openmp)
 // PHASES-JIT-NEXT: 1: preprocessor, {0}, cpp-output, (host-openmp)
 // PHASES-JIT-NEXT: 2: compiler, {1}, ir, (host-openmp)
-// PHASES-JIT-NEXT: 3: input, "[[INPUT]]", c, (device-openmp)
-// PHASES-JIT-NEXT: 4: preprocessor, {3}, cpp-output, (device-openmp)
-// PHASES-JIT-NEXT: 5: compiler, {4}, ir, (device-openmp)
+// PHASES-JIT-NEXT: 3: input, "[[INPUT]]", c, (device-openmp, {{.*}})
+// PHASES-JIT-NEXT: 4: preprocessor, {3}, cpp-output, (device-openmp, {{.*}})
+// PHASES-JIT-NEXT: 5: compiler, {4}, ir, (device-openmp, {{.*}})
 // PHASES-JIT-NEXT: 6: offload, "host-openmp (x86_64-unknown-linux-gnu)" {2}, "device-openmp ([[TARGET:.+]])" {5}, ir
-// PHASES-JIT-NEXT: 7: backend, {6}, lto-bc, (device-openmp)
+// PHASES-JIT-NEXT: 7: backend, {6}, lto-bc, (device-openmp, {{.*}})
 // PHASES-JIT-NEXT: 8: offload, "device-openmp ([[TARGET]])" {7}, lto-bc
 // PHASES-JIT-NEXT: 9: clang-offload-packager, {8}, image, (device-openmp)
 // PHASES-JIT-NEXT: 10: offload, "host-openmp (x86_64-unknown-linux-gnu)" {2}, "device-openmp (x86_64-unknown-linux-gnu)" {9}, ir
@@ -41,11 +41,11 @@
 // Check for incompatible combinations
 
 // RUN: not %clang -### --target=x86_64-unknown-linux-gnu -fopenmp=libomp -fno-offload-lto \
-// RUN:   -fopenmp-targets=nvptx64-nvidia-cuda -fopenmp-target-jit %s 2>&1 \
+// RUN:   --offload-arch=sm_52 -fopenmp-target-jit %s 2>&1 \
 // RUN: | FileCheck -check-prefix=NO-LTO %s
 // NO-LTO: error: the combination of '-fno-offload-lto' and '-fopenmp-target-jit' is incompatible
 
 // RUN: not %clang -### --target=x86_64-unknown-linux-gnu -fopenmp=libomp -foffload-lto=thin \
-// RUN:   -fopenmp-targets=nvptx64-nvidia-cuda -fopenmp-target-jit %s 2>&1 \
+// RUN:   --offload-arch=sm_52 -fopenmp-targets=nvptx64-nvidia-cuda -fopenmp-target-jit %s 2>&1 \
 // RUN: | FileCheck -check-prefix=THIN-LTO %s
 // THIN-LTO: error: the combination of '-foffload-lto=' and '-fopenmp-target-jit' is incompatible
diff --git a/clang/test/Driver/openmp-offload.c b/clang/test/Driver/openmp-offload.c
index caedc223a5c76..6f56ae00ba065 100644
--- a/clang/test/Driver/openmp-offload.c
+++ b/clang/test/Driver/openmp-offload.c
@@ -184,13 +184,13 @@
 // RUN:   %clang -### --target=powerpc64le-linux -fopenmp=libomp -fopenmp-targets=powerpc64le-ibm-linux-gnu -g %s 2>&1 \
 // RUN:   | FileCheck -check-prefix=CHK-NEW-DRIVER-DEBUG %s
 
-// CHK-NEW-DRIVER-DEBUG: clang-linker-wrapper{{.*}} "--device-debug"
+// CHK-NEW-DRIVER-DEBUG: clang-linker-wrapper{{.*}} "--device-compiler=powerpc64le-ibm-linux-gnu=-g"
 
 /// Check arguments to the linker wrapper
 // RUN:   %clang -### --target=powerpc64le-linux -fopenmp=libomp -fopenmp-targets=powerpc64le-ibm-linux-gnu \
 // RUN:     -mllvm -abc %s 2>&1 | FileCheck -check-prefix=CHK-NEW-DRIVER-MLLVM %s
 
-// CHK-NEW-DRIVER-MLLVM: clang-linker-wrapper{{.*}} "-abc"
+// CHK-NEW-DRIVER-MLLVM: clang-linker-wrapper{{.*}} "--device-linker=powerpc64le-ibm-linux-gnu=-mllvm" "--device-linker=powerpc64le-ibm-linux-gnu=-abc"
 
 //
 // Ensure that we generate the correct bindings for '-fsyntax-only' for OpenMP.
diff --git a/clang/test/Driver/openmp-system-arch.c b/clang/test/Driver/openmp-system-arch.c
index d097c6bc06548..51021e352f4d3 100644
--- a/clang/test/Driver/openmp-system-arch.c
+++ b/clang/test/Driver/openmp-system-arch.c
@@ -68,13 +68,13 @@
 // RUN:   not %clang -### --target=x86_64-unknown-linux-gnu -nogpulib -fopenmp=libomp \
 // RUN:     -fopenmp-targets=nvptx64-nvidia-cuda --nvptx-arch-tool=%t/nvptx_arch_empty %s 2>&1 \
 // RUN:   | FileCheck %s --check-prefix=NVPTX
-// NVPTX: error: cannot determine nvptx64 architecture: No NVIDIA GPU detected in the system; consider passing it via '-march'
+// NVPTX: error: cannot determine nvptx64 architecture: No NVIDIA GPU detected in the system; consider passing it via '--offload-arch'
 
 // case when 'amdgpu-arch' returns nothing using `-fopenmp-targets=`.
 // RUN:   not %clang -### --target=x86_64-unknown-linux-gnu -nogpulib -fopenmp=libomp \
 // RUN:     -fopenmp-targets=amdgcn-amd-amdhsa --amdgpu-arch-tool=%t/amdgpu_arch_empty %s 2>&1 \
 // RUN:   | FileCheck %s --check-prefix=AMDGPU
-// AMDGPU: error: cannot determine amdgcn architecture: No AMD GPU detected in the system; consider passing it via '-march'
+// AMDGPU: error: cannot determine amdgcn architecture: No AMD GPU detected in the system; consider passing it via '--offload-arch'
 
 // case when CLANG_TOOLCHAIN_PROGRAM_TIMEOUT is malformed for nvptx-arch.
 // RUN: env CLANG_TOOLCHAIN_PROGRAM_TIMEOUT=foo \
@@ -82,7 +82,7 @@
 // RUN:     -fopenmp-targets=nvptx64-nvidia-cuda -nogpulib \
 // RUN:     --nvptx-arch-tool=%t/nvptx_arch_sm_70 %s 2>&1 | \
 // RUN:   FileCheck %s --check-prefix=BAD-TIMEOUT-NVPTX
-// BAD-TIMEOUT-NVPTX: clang: error: cannot determine nvptx64 architecture: CLANG_TOOLCHAIN_PROGRAM_TIMEOUT expected an integer, got 'foo'; consider passing it via '-march'; environment variable CLANG_TOOLCHAIN_PROGRAM_TIMEOUT specifies the tool timeout (integer secs, <=0 is infinite)
+// BAD-TIMEOUT-NVPTX: clang: error: cannot determine nvptx64 architecture: CLANG_TOOLCHAIN_PROGRAM_TIMEOUT expected an integer, got 'foo'; consider passing it via '--offload-arch'; environment variable CLANG_TOOLCHAIN_PROGRAM_TIMEOUT specifies the tool timeout (integer secs, <=0 is infinite)
 
 // case when CLANG_TOOLCHAIN_PROGRAM_TIMEOUT is malformed for amdgpu-arch.
 // RUN: env CLANG_TOOLCHAIN_PROGRAM_TIMEOUT= \
@@ -90,4 +90,4 @@
 // RUN:     -fopenmp-targets=amdgcn-amd-amdhsa -nogpulib \
 // RUN:     --amdgpu-arch-tool=%t/amdgpu_arch_gfx906 %s 2>&1 | \
 // RUN:   FileCheck %s --check-prefix=BAD-TIMEOUT-AMDGPU
-// BAD-TIMEOUT-AMDGPU: clang: error: cannot determine amdgcn architecture: CLANG_TOOLCHAIN_PROGRAM_TIMEOUT expected an integer, got ''; consider passing it via '-march'; environment variable CLANG_TOOLCHAIN_PROGRAM_TIMEOUT specifies the tool timeout (integer secs, <=0 is infinite)
+// BAD-TIMEOUT-AMDGPU: clang: error: cannot determine amdgcn architecture: CLANG_TOOLCHAIN_PROGRAM_TIMEOUT expected an integer, got ''; consider passing it via '--offload-arch'; environment variable CLANG_TOOLCHAIN_PROGRAM_TIMEOUT specifies the tool timeout (integer secs, <=0 is infinite)
diff --git a/clang/test/Driver/print-supported-extensions-riscv.c b/clang/test/Driver/print-supported-extensions-riscv.c
index ae3a1c29df397..3443ff0b69de9 100644
--- a/clang/test/Driver/print-supported-extensions-riscv.c
+++ b/clang/test/Driver/print-supported-extensions-riscv.c
@@ -157,6 +157,8 @@
 // CHECK-NEXT:     xcvmac               1.0       'XCVmac' (CORE-V Multiply-Accumulate)
 // CHECK-NEXT:     xcvmem               1.0       'XCVmem' (CORE-V Post-incrementing Load & Store)
 // CHECK-NEXT:     xcvsimd              1.0       'XCVsimd' (CORE-V SIMD ALU)
+// CHECK-NEXT:     xmipscmove           1.0       'XMIPSCMove' (MIPS conditional move instruction(s) (ccmov))
+// CHECK-NEXT:     xmipslsp             1.0       'XMIPSLSP' (MIPS optimization for hardware load-store bonding)
 // CHECK-NEXT:     xsfcease             1.0       'XSfcease' (SiFive sf.cease Instruction)
 // CHECK-NEXT:     xsfvcp               1.0       'XSfvcp' (SiFive Custom Vector Coprocessor Interface Instructions)
 // CHECK-NEXT:     xsfvfnrclipxfqf      1.0       'XSfvfnrclipxfqf' (SiFive FP32-to-int8 Ranged Clip Instructions)
@@ -191,7 +193,7 @@
 // CHECK-NEXT:     ssctr                1.0       'Ssctr' (Control Transfer Records Supervisor Level)
 // CHECK-NEXT:     svukte               0.3       'Svukte' (Address-Independent Latency of User-Mode Faults to Supervisor Addresses)
 // CHECK-NEXT:     xqcia                0.2       'Xqcia' (Qualcomm uC Arithmetic Extension)
-// CHECK-NEXT:     xqciac               0.2       'Xqciac' (Qualcomm uC Load-Store Address Calculation Extension)
+// CHECK-NEXT:     xqciac               0.3       'Xqciac' (Qualcomm uC Load-Store Address Calculation Extension)
 // CHECK-NEXT:     xqcicli              0.2       'Xqcicli' (Qualcomm uC Conditional Load Immediate Extension)
 // CHECK-NEXT:     xqcicm               0.2       'Xqcicm' (Qualcomm uC Conditional Move Extension)
 // CHECK-NEXT:     xqcics               0.2       'Xqcics' (Qualcomm uC Conditional Select Extension)
diff --git a/clang/test/Driver/rocm-device-libs.cl b/clang/test/Driver/rocm-device-libs.cl
index 6837e219dc35d..f9766e6fa4d99 100644
--- a/clang/test/Driver/rocm-device-libs.cl
+++ b/clang/test/Driver/rocm-device-libs.cl
@@ -145,8 +145,8 @@
 // RUN: 2>&1 | FileCheck  --check-prefixes=NOASAN %s
 
 // COMMON: "-triple" "amdgcn-amd-amdhsa"
-// ASAN-SAME: "-mlink-bitcode-file" "{{.*}}/amdgcn/bitcode/asanrtl.bc"
 // COMMON-SAME: "-mlink-builtin-bitcode" "{{.*}}/amdgcn/bitcode/opencl.bc"
+// ASAN-SAME: "-mlink-bitcode-file" "{{.*}}/amdgcn/bitcode/asanrtl.bc"
 // COMMON-SAME: "-mlink-builtin-bitcode" "{{.*}}/amdgcn/bitcode/ocml.bc"
 // COMMON-SAME: "-mlink-builtin-bitcode" "{{.*}}/amdgcn/bitcode/ockl.bc"
 
diff --git a/clang/test/Driver/sycl-offload-jit.cpp b/clang/test/Driver/sycl-offload-jit.cpp
index eb192e08a3bc0..e040f4ded18e9 100644
--- a/clang/test/Driver/sycl-offload-jit.cpp
+++ b/clang/test/Driver/sycl-offload-jit.cpp
@@ -27,7 +27,7 @@
 // CHK-DEVICE-TRIPLE-SAME: "-aux-triple" "x86_64-unknown-linux-gnu"
 // CHK-DEVICE-TRIPLE-SAME: "-fsycl-is-device"
 // CHK-DEVICE-TRIPLE-SAME: "-O2"
-// CHK-DEVICE-TRIPLE: clang-offload-packager{{.*}} "--image=file={{.*}}.bc,triple=spirv64-unknown-unknown,arch=,kind=sycl"
+// CHK-DEVICE-TRIPLE: clang-offload-packager{{.*}} "--image=file={{.*}}.bc,triple=spirv64-unknown-unknown,arch=generic,kind=sycl"
 
 /// Check -fsycl-is-device is passed when compiling for the device.
 /// Check -fsycl-is-host is passed when compiling for host.
diff --git a/clang/test/Driver/tls-dialect.c b/clang/test/Driver/tls-dialect.c
index 3471b55b0ebae..9ab79e87353d8 100644
--- a/clang/test/Driver/tls-dialect.c
+++ b/clang/test/Driver/tls-dialect.c
@@ -10,6 +10,11 @@
 /// TLSDESC is not on by default in Linux, even on RISC-V, and is covered above
 // RUN: %clang -### --target=riscv64-android %s 2>&1 | FileCheck --check-prefix=DESC %s
 
+/// Fuchsia supports TLSDESC by default for all architectures.
+// RUN: %clang -### --target=riscv64-unknown-fuchsia %s 2>&1 | FileCheck --check-prefix=DESC %s
+// RUN: %clang -### --target=aarch64-unknown-fuchsia %s 2>&1 | FileCheck --check-prefix=DESC %s
+// RUN: %clang -### --target=x86_64-unknown-fuchsia %s 2>&1 | FileCheck --check-prefix=DESC %s
+
 /// LTO
 // RUN: %clang -### --target=loongarch64-linux -flto -mtls-dialect=desc %s 2>&1 | FileCheck --check-prefix=LTO-DESC %s
 // RUN: %clang -### --target=loongarch64-linux -flto %s 2>&1 | FileCheck --check-prefix=LTO-NODESC %s
diff --git a/clang/test/Driver/uefi-constructed-args.c b/clang/test/Driver/uefi-constructed-args.c
index 3cc5abe697453..44d2271e0c8ba 100644
--- a/clang/test/Driver/uefi-constructed-args.c
+++ b/clang/test/Driver/uefi-constructed-args.c
@@ -1,3 +1,5 @@
+// RUN: %clang -### --target=x86_64-unknown-uefi -g -- %s 2>&1 \
+// RUN:     | FileCheck -check-prefixes=CHECK %s
 // RUN: %clang_cl -### --target=x86_64-unknown-uefi -g -- %s 2>&1 \
 // RUN:     | FileCheck -check-prefixes=CHECK %s
 // CHECK: "-cc1"
diff --git a/clang/test/Driver/unsupported-target-arch.c b/clang/test/Driver/unsupported-target-arch.c
index 8df0ee9fe7d06..c0ec235439bca 100644
--- a/clang/test/Driver/unsupported-target-arch.c
+++ b/clang/test/Driver/unsupported-target-arch.c
@@ -63,3 +63,21 @@
 // RUN: not %clang --target=powerpc-apple-darwin -o /dev/null %s 2> %t.err
 // RUN: FileCheck --input-file=%t.err --check-prefix=CHECK-PPCMAC %s
 // CHECK-PPCMAC: error: unknown target triple 'unknown-apple-macosx{{.*}}'
+
+// RUN: not %clang --target=aarch64-unknown-uefi -o %t.o %s 2> %t.err
+// RUN: FileCheck --input-file=%t.err --check-prefix=CHECK-AARCH64 %s
+// RUN: not %clang_cl --target=aarch64-unknown-uefi -o %t.o %s 2> %t.err
+// RUN: FileCheck --input-file=%t.err --check-prefix=CHECK-AARCH64 %s
+// CHECK-AARCH64: error: unknown target triple 'aarch64-unknown-uefi'{{$}}
+
+// RUN: not %clang --target=arm-unknown-uefi -o %t.o %s 2> %t.err
+// RUN: FileCheck --input-file=%t.err -check-prefixes=CHECK-ARM %s
+// RUN: not %clang_cl --target=arm-unknown-uefi -o %t.o %s 2> %t.err
+// RUN: FileCheck --input-file=%t.err -check-prefixes=CHECK-ARM %s
+// CHECK-ARM: error: unknown target triple 'arm-unknown-uefi'{{$}}
+
+// RUN: not %clang --target=x86-unknown-uefi -o %t.o %s 2> %t.err
+// RUN: FileCheck --input-file=%t.err -check-prefixes=CHECK-x86 %s
+// RUN: not %clang_cl --target=x86-unknown-uefi -o %t.o %s 2> %t.err
+// RUN: FileCheck --input-file=%t.err -check-prefixes=CHECK-x86 %s
+// CHECK-x86: error: unknown target triple 'x86-unknown-uefi'{{$}}
diff --git a/clang/test/Driver/warning-suppression-mappings-not-parsed.cpp b/clang/test/Driver/warning-suppression-mappings-not-parsed.cpp
new file mode 100644
index 0000000000000..8f52fb1c6cc7d
--- /dev/null
+++ b/clang/test/Driver/warning-suppression-mappings-not-parsed.cpp
@@ -0,0 +1,5 @@
+// RUN: rm -rf %t
+// RUN: mkdir -p %t
+// RUN: echo '[unknown-warning]' > %t/foo.txt
+// RUN: %clang -fdriver-only --warning-suppression-mappings=%t/foo.txt %s | FileCheck -allow-empty %s
+// CHECK-NOT: unknown warning option 'unknown-warning'
diff --git a/clang/test/Driver/wasm-toolchain.c b/clang/test/Driver/wasm-toolchain.c
index 2d14052082776..f516a4e457da7 100644
--- a/clang/test/Driver/wasm-toolchain.c
+++ b/clang/test/Driver/wasm-toolchain.c
@@ -224,6 +224,14 @@
 // RUN:   | FileCheck -check-prefix=WASM_LEGACY_EH_NO_EH %s
 // WASM_LEGACY_EH_NO_EH: invalid argument '-wasm-use-legacy-eh' not allowed with '-mno-exception-handling'
 
+// When invoking clang with multiple files in a single command line, target
+// feature flags should be equally added to the multiple clang-cc1 command lines
+// RUN: %clang -### --target=wasm32-unknown-unknown \
+// RUN:    --sysroot=/foo %s %s -mllvm -wasm-enable-sjlj 2>&1 \
+// RUN:  | FileCheck -check-prefix=WASM_SJLJ_MULTI_FILES %s
+// WASM_SJLJ_MULTI_FILES: "-cc1" {{.*}} "-target-feature" "+exception-handling" "-target-feature" "+multivalue" "-target-feature" "+reference-types" "-exception-model=wasm"
+// WASM_SJLJ_MULTI_FILES: "-cc1" {{.*}} "-target-feature" "+exception-handling" "-target-feature" "+multivalue" "-target-feature" "+reference-types" "-exception-model=wasm"
+
 // RUN: %clang -### %s -fsanitize=address --target=wasm32-unknown-emscripten 2>&1 | FileCheck -check-prefix=CHECK-ASAN-EMSCRIPTEN %s
 // CHECK-ASAN-EMSCRIPTEN: "-fsanitize=address"
 // CHECK-ASAN-EMSCRIPTEN: "-fsanitize-address-globals-dead-stripping"
diff --git a/clang/test/Driver/x86-target-features.c b/clang/test/Driver/x86-target-features.c
index 339f593dc760a..18361251dcebc 100644
--- a/clang/test/Driver/x86-target-features.c
+++ b/clang/test/Driver/x86-target-features.c
@@ -395,7 +395,8 @@
 // EVEX512: "-target-feature" "+evex512"
 // NO-EVEX512: "-target-feature" "-evex512"
 
-// RUN: %clang --target=i386 -mavx10.1 %s -### -o %t.o 2>&1 | FileCheck -check-prefix=AVX10_1_256 %s
+// RUN: not %clang --target=i386 -march=i386 -mavx10.1 %s -### -o %t.o 2>&1 | FileCheck -check-prefix=UNSUPPORT-AVX10 %s
+// RUN: not %clang --target=i386 -march=i386 -mno-avx10.1 %s -### -o %t.o 2>&1 | FileCheck -check-prefix=UNSUPPORT-AVX10 %s
 // RUN: %clang --target=i386 -mavx10.1-256 %s -### -o %t.o 2>&1 | FileCheck -check-prefix=AVX10_1_256 %s
 // RUN: %clang --target=i386 -mavx10.1-512 %s -### -o %t.o 2>&1 | FileCheck -check-prefix=AVX10_1_512 %s
 // RUN: %clang --target=i386 -mavx10.1-256 -mavx10.1-512 %s -### -o %t.o 2>&1 | FileCheck -check-prefix=AVX10_1_512 %s
@@ -403,15 +404,18 @@
 // RUN: not %clang --target=i386 -march=i386 -mavx10.1-128 %s -### -o %t.o 2>&1 | FileCheck -check-prefix=BAD-AVX10 %s
 // RUN: not %clang --target=i386 -march=i386 -mavx10.a-256 %s -### -o %t.o 2>&1 | FileCheck -check-prefix=BAD-AVX10 %s
 // RUN: not %clang --target=i386 -march=i386 -mavx10.1024-512 %s -### -o %t.o 2>&1 | FileCheck -check-prefix=BAD-AVX10 %s
-// RUN: %clang --target=i386 -march=i386 -mavx10.1 -mavx512f %s -### -o %t.o 2>&1 | FileCheck -check-prefix=AVX10-AVX512 %s
-// RUN: %clang --target=i386 -march=i386 -mavx10.1 -mno-avx512f %s -### -o %t.o 2>&1 | FileCheck -check-prefix=AVX10-AVX512 %s
-// RUN: %clang --target=i386 -march=i386 -mavx10.1 -mevex512 %s -### -o %t.o 2>&1 | FileCheck -check-prefix=AVX10-EVEX512 %s
-// RUN: %clang --target=i386 -march=i386 -mavx10.1 -mno-evex512 %s -### -o %t.o 2>&1 | FileCheck -check-prefix=AVX10-EVEX512 %s
-// RUN: %clang --target=i386 -mavx10.2 %s -### -o %t.o 2>&1 | FileCheck -check-prefix=AVX10_2_256 %s
+// RUN: %clang --target=i386 -march=i386 -mavx10.1-256 -mavx512f %s -### -o %t.o 2>&1 | FileCheck -check-prefix=AVX10-AVX512 %s
+// RUN: %clang --target=i386 -march=i386 -mavx10.1-256 -mno-avx512f %s -### -o %t.o 2>&1 | FileCheck -check-prefix=AVX10-AVX512 %s
+// RUN: %clang --target=i386 -march=i386 -mavx10.1-256 -mevex512 %s -### -o %t.o 2>&1 | FileCheck -check-prefix=AVX10-EVEX512 %s
+// RUN: %clang --target=i386 -march=i386 -mavx10.1-256 -mno-evex512 %s -### -o %t.o 2>&1 | FileCheck -check-prefix=AVX10-EVEX512 %s
+// RUN: %clang --target=i386 -mavx10.2 %s -### -o %t.o 2>&1 | FileCheck -check-prefix=AVX10_2_512 %s
+// RUN: %clang --target=i386 -mno-avx10.2 %s -### -o %t.o 2>&1 | FileCheck -check-prefix=NO-AVX10_2 %s
 // RUN: %clang --target=i386 -mavx10.2-256 %s -### -o %t.o 2>&1 | FileCheck -check-prefix=AVX10_2_256 %s
 // RUN: %clang --target=i386 -mavx10.2-512 %s -### -o %t.o 2>&1 | FileCheck -check-prefix=AVX10_2_512 %s
 // RUN: %clang --target=i386 -mavx10.2-256 -mavx10.1-512 %s -### -o %t.o 2>&1 | FileCheck -check-prefixes=AVX10_2_256,AVX10_1_512 %s
 // RUN: %clang --target=i386 -mavx10.2-512 -mavx10.1-256 %s -### -o %t.o 2>&1 | FileCheck -check-prefixes=AVX10_2_512,AVX10_1_256 %s
+// UNSUPPORT-AVX10: error: unsupported option '-m{{.*}}avx10.1' for target 'i386'
+// NO-AVX10_2: "-target-feature" "-avx10.2-256"
 // AVX10_2_256: "-target-feature" "+avx10.2-256"
 // AVX10_2_512: "-target-feature" "+avx10.2-512"
 // AVX10_1_256: "-target-feature" "+avx10.1-256"
diff --git a/clang/test/ExtractAPI/anonymous_record_no_typedef.c b/clang/test/ExtractAPI/anonymous_record_no_typedef.c
index c0c76ef1f06b5..d278cbb8ec348 100644
--- a/clang/test/ExtractAPI/anonymous_record_no_typedef.c
+++ b/clang/test/ExtractAPI/anonymous_record_no_typedef.c
@@ -191,4 +191,44 @@ union Vector {
 // VEC-DAG: "!testRelLabel": "memberOf $ c:@U@Vector@Sa@FI@X $ c:@U@Vector"
 // VEC-DAG: "!testRelLabel": "memberOf $ c:@U@Vector@Sa@FI@Y $ c:@U@Vector"
 
+// RUN: FileCheck %s --input-file %t/output-c.symbols.json --check-prefix MYSTRUCT
+// RUN: FileCheck %s --input-file %t/output-cxx.symbols.json --check-prefix MYSTRUCT
+// RUN: FileCheck %s --input-file %t/output-c.symbols.json --check-prefix COUNTS
+// RUN: FileCheck %s --input-file %t/output-cxx.symbols.json --check-prefix COUNTS
+struct MyStruct {
+    struct {
+        int count;
+    } counts[1];
+};
+// MYSTRUCT-NOT: "spelling": ""
+// MYSTRUCT-NOT: "title": ""
+
+// COUNTS-LABEL: "!testLabel": "c:@S@MyStruct@FI@counts"
+// COUNTS:      "declarationFragments": [
+// COUNTS-NEXT:   {
+// COUNTS-NEXT:     "kind": "keyword",
+// COUNTS-NEXT:     "spelling": "struct"
+// COUNTS-NEXT:   },
+// COUNTS-NEXT:   {
+// COUNTS-NEXT:     "kind": "text",
+// COUNTS-NEXT:     "spelling": " { ... } "
+// COUNTS-NEXT:   },
+// COUNTS-NEXT:   {
+// COUNTS-NEXT:     "kind": "identifier",
+// COUNTS-NEXT:     "spelling": "counts"
+// COUNTS-NEXT:   },
+// COUNTS-NEXT:   {
+// COUNTS-NEXT:     "kind": "text",
+// COUNTS-NEXT:     "spelling": "["
+// COUNTS-NEXT:   },
+// COUNTS-NEXT:   {
+// COUNTS-NEXT:     "kind": "number",
+// COUNTS-NEXT:     "spelling": "1"
+// COUNTS-NEXT:   },
+// COUNTS-NEXT:   {
+// COUNTS-NEXT:     "kind": "text",
+// COUNTS-NEXT:     "spelling": "];"
+// COUNTS-NEXT:   }
+// COUNTS-NEXT: ],
+
 // expected-no-diagnostics
diff --git a/clang/test/ExtractAPI/typedef_underscore.c b/clang/test/ExtractAPI/typedef_underscore.c
new file mode 100644
index 0000000000000..a42046907b46d
--- /dev/null
+++ b/clang/test/ExtractAPI/typedef_underscore.c
@@ -0,0 +1,69 @@
+// RUN: rm -rf %t
+// RUN: %clang_cc1 -extract-api --pretty-sgf --emit-sgf-symbol-labels-for-testing \
+// RUN:   --product-name=TypedefChain -triple arm64-apple-macosx -x c-header %s -o %t/typedefchain-c.symbols.json -verify
+// RUN: %clang_cc1 -extract-api --pretty-sgf --emit-sgf-symbol-labels-for-testing \
+// RUN:   --product-name=TypedefChain -triple arm64-apple-macosx -x c++-header %s -o %t/typedefchain-cxx.symbols.json -verify
+
+// RUN: FileCheck %s --input-file %t/typedefchain-c.symbols.json --check-prefix MYSTRUCT
+// RUN: FileCheck %s --input-file %t/typedefchain-cxx.symbols.json --check-prefix MYSTRUCT
+typedef struct _MyStruct { } MyStruct;
+
+// MYSTRUCT-LABEL: "!testLabel": "c:@S@_MyStruct"
+// MYSTRUCT:      "accessLevel": "public",
+// MYSTRUCT:      "declarationFragments": [
+// MYSTRUCT-NEXT:   {
+// MYSTRUCT-NEXT:     "kind": "keyword",
+// MYSTRUCT-NEXT:     "spelling": "typedef"
+// MYSTRUCT-NEXT:   },
+// MYSTRUCT-NEXT:   {
+// MYSTRUCT-NEXT:     "kind": "text",
+// MYSTRUCT-NEXT:     "spelling": " "
+// MYSTRUCT-NEXT:   },
+// MYSTRUCT-NEXT:   {
+// MYSTRUCT-NEXT:     "kind": "keyword",
+// MYSTRUCT-NEXT:     "spelling": "struct"
+// MYSTRUCT-NEXT:   },
+// MYSTRUCT-NEXT:   {
+// MYSTRUCT-NEXT:     "kind": "text",
+// MYSTRUCT-NEXT:     "spelling": " "
+// MYSTRUCT-NEXT:   },
+// MYSTRUCT-NEXT:   {
+// MYSTRUCT-NEXT:     "kind": "identifier",
+// MYSTRUCT-NEXT:     "spelling": "_MyStruct"
+// MYSTRUCT-NEXT:   },
+// MYSTRUCT-NEXT:   {
+// MYSTRUCT-NEXT:     "kind": "text",
+// MYSTRUCT-NEXT:     "spelling": " { ... } "
+// MYSTRUCT-NEXT:   },
+// MYSTRUCT-NEXT:   {
+// MYSTRUCT-NEXT:     "kind": "identifier",
+// MYSTRUCT-NEXT:     "spelling": "MyStruct"
+// MYSTRUCT-NEXT:   },
+// MYSTRUCT-NEXT:   {
+// MYSTRUCT-NEXT:     "kind": "text",
+// MYSTRUCT-NEXT:     "spelling": ";"
+// MYSTRUCT-NEXT:   }
+// MYSTRUCT-NEXT: ],
+// MYSTRUCT:      "kind": {
+// MYSTRUCT-NEXT:   "displayName": "Structure",
+// MYSTRUCT-NEXT:   "identifier": "c{{(\+\+)?}}.struct"
+// MYSTRUCT:           "names": {
+// MYSTRUCT-NEXT:        "navigator": [
+// MYSTRUCT-NEXT:          {
+// MYSTRUCT-NEXT:            "kind": "identifier",
+// MYSTRUCT-NEXT:            "spelling": "MyStruct"
+// MYSTRUCT-NEXT:          }
+// MYSTRUCT-NEXT:        ],
+// MYSTRUCT-NEXT:        "subHeading": [
+// MYSTRUCT-NEXT:          {
+// MYSTRUCT-NEXT:            "kind": "identifier",
+// MYSTRUCT-NEXT:            "spelling": "MyStruct"
+// MYSTRUCT-NEXT:          }
+// MYSTRUCT-NEXT:        ],
+// MYSTRUCT-NEXT:        "title": "MyStruct"
+// MYSTRUCT-NEXT:      },
+// MYSTRUCT:      "pathComponents": [
+// MYSTRUCT-NEXT:    "MyStruct"
+// MYSTRUCT-NEXT:  ]
+
+// expected-no-diagnostics
diff --git a/clang/test/Frontend/aarch64-ignore-branch-protection-attribute.c b/clang/test/Frontend/aarch64-ignore-branch-protection-attribute.c
new file mode 100644
index 0000000000000..32cc98dd4e037
--- /dev/null
+++ b/clang/test/Frontend/aarch64-ignore-branch-protection-attribute.c
@@ -0,0 +1,31 @@
+// REQUIRES: aarch64-registered-target
+
+// RUN: %clang -target aarch64-linux-pauthtest   %s -S -emit-llvm -o - 2>&1 | FileCheck --implicit-check-not=warning: %s
+// RUN: %clang -target aarch64 -fptrauth-returns %s -S -emit-llvm -o - 2>&1 | FileCheck --implicit-check-not=warning: %s
+
+/// Unsupported with pauthtest, warning emitted
+__attribute__((target("branch-protection=pac-ret"))) void f1() {}
+// CHECK:      warning: unsupported 'branch-protection' in the 'target' attribute string; 'target' attribute ignored [-Wignored-attributes]
+// CHECK-NEXT: __attribute__((target("branch-protection=pac-ret"))) void f1() {}
+__attribute__((target("branch-protection=gcs"))) void f2() {}
+// CHECK:      warning: unsupported 'branch-protection' in the 'target' attribute string; 'target' attribute ignored [-Wignored-attributes]
+// CHECK-NEXT: __attribute__((target("branch-protection=gcs"))) void f2() {}
+__attribute__((target("branch-protection=standard"))) void f3() {}
+// CHECK:      warning: unsupported 'branch-protection' in the 'target' attribute string; 'target' attribute ignored [-Wignored-attributes]
+// CHECK-NEXT: __attribute__((target("branch-protection=standard"))) void f3() {}
+
+/// Supported with pauthtest, no warning emitted
+__attribute__((target("branch-protection=bti"))) void f4() {}
+
+/// Supported with pauthtest, no warning emitted
+__attribute__((target("branch-protection=none"))) void f5() {}
+
+/// Check there are no branch protection function attributes which are unsupported with pauthtest
+
+// CHECK-NOT:  attributes {{.*}} "sign-return-address"
+// CHECK-NOT:  attributes {{.*}} "sign-return-address-key"
+// CHECK-NOT:  attributes {{.*}} "branch-protection-pauth-lr"
+// CHECK-NOT:  attributes {{.*}} "guarded-control-stack"
+
+/// Check function attributes which are supported with pauthtest
+// CHECK:      attributes {{.*}} "branch-target-enforcement"
diff --git a/clang/test/Frontend/custom-diag-werror-interaction.c b/clang/test/Frontend/custom-diag-werror-interaction.c
new file mode 100644
index 0000000000000..997c8c11ff0e0
--- /dev/null
+++ b/clang/test/Frontend/custom-diag-werror-interaction.c
@@ -0,0 +1,9 @@
+// RUN: %clang_cc1 -emit-llvm-only -fprofile-instrument=clang -fcoverage-mcdc -Werror -Wno-unused-value %s -verify
+
+int foo(int x);
+
+int main(void) {
+    int a, b, c;
+    a && foo( b && c ); // expected-warning{{unsupported MC/DC boolean expression; contains an operation with a nested boolean expression. Expression will not be covered}}
+    return 0;
+}
diff --git a/clang/test/Headers/__cpuidex_conflict.c b/clang/test/Headers/__cpuidex_conflict.c
index 0f5e6e5e0a0ff..8687a6aa2f897 100644
--- a/clang/test/Headers/__cpuidex_conflict.c
+++ b/clang/test/Headers/__cpuidex_conflict.c
@@ -3,9 +3,7 @@
 // RUN: %clang_cc1 %s -ffreestanding -fms-extensions -fms-compatibility \
 // RUN:  -fms-compatibility-version=19.00 -triple x86_64-pc-windows-msvc -emit-llvm -o -
 // %clang_cc1 %s -ffreestanding -triple x86_64-w64-windows-gnu -fms-extensions -emit-llvm -o -
-//
-// FIXME: See https://github.com/llvm/llvm-project/pull/121839
-// RUN: not %clang_cc1 %s -ffreestanding -fopenmp -fopenmp-is-target-device -aux-triple x86_64-unknown-linux-gnu
+// RUN: %clang_cc1 %s -ffreestanding -fopenmp -fopenmp-is-target-device -aux-triple x86_64-unknown-linux-gnu
 
 typedef __SIZE_TYPE__ size_t;
 
diff --git a/clang/test/Headers/gpuintrin.c b/clang/test/Headers/gpuintrin.c
index 281339716c3ed..89efe12ee8def 100644
--- a/clang/test/Headers/gpuintrin.c
+++ b/clang/test/Headers/gpuintrin.c
@@ -38,7 +38,7 @@
 // AMDGPU-NEXT:    [[CALL20:%.*]] = call i64 @__gpu_ballot(i64 noundef -1, i1 noundef zeroext true) #[[ATTR7]]
 // AMDGPU-NEXT:    call void @__gpu_sync_threads() #[[ATTR7]]
 // AMDGPU-NEXT:    call void @__gpu_sync_lane(i64 noundef -1) #[[ATTR7]]
-// AMDGPU-NEXT:    [[CALL21:%.*]] = call i32 @__gpu_shuffle_idx_u32(i64 noundef -1, i32 noundef -1, i32 noundef -1) #[[ATTR7]]
+// AMDGPU-NEXT:    [[CALL21:%.*]] = call i32 @__gpu_shuffle_idx_u32(i64 noundef -1, i32 noundef -1, i32 noundef -1, i32 noundef 0) #[[ATTR7]]
 // AMDGPU-NEXT:    [[CALL22:%.*]] = call i64 @__gpu_first_lane_id(i64 noundef -1) #[[ATTR7]]
 // AMDGPU-NEXT:    [[CALL23:%.*]] = call zeroext i1 @__gpu_is_first_in_lane(i64 noundef -1) #[[ATTR7]]
 // AMDGPU-NEXT:    call void @__gpu_exit() #[[ATTR8:[0-9]+]]
@@ -70,7 +70,7 @@
 // NVPTX-NEXT:    [[CALL20:%.*]] = call i64 @__gpu_ballot(i64 noundef -1, i1 noundef zeroext true) #[[ATTR6]]
 // NVPTX-NEXT:    call void @__gpu_sync_threads() #[[ATTR6]]
 // NVPTX-NEXT:    call void @__gpu_sync_lane(i64 noundef -1) #[[ATTR6]]
-// NVPTX-NEXT:    [[CALL21:%.*]] = call i32 @__gpu_shuffle_idx_u32(i64 noundef -1, i32 noundef -1, i32 noundef -1) #[[ATTR6]]
+// NVPTX-NEXT:    [[CALL21:%.*]] = call i32 @__gpu_shuffle_idx_u32(i64 noundef -1, i32 noundef -1, i32 noundef -1, i32 noundef 0) #[[ATTR6]]
 // NVPTX-NEXT:    [[CALL22:%.*]] = call i64 @__gpu_first_lane_id(i64 noundef -1) #[[ATTR6]]
 // NVPTX-NEXT:    [[CALL23:%.*]] = call zeroext i1 @__gpu_is_first_in_lane(i64 noundef -1) #[[ATTR6]]
 // NVPTX-NEXT:    call void @__gpu_exit() #[[ATTR7:[0-9]+]]
@@ -90,6 +90,68 @@ __gpu_kernel void foo() {
   __gpu_num_threads_z();
   __gpu_num_threads(0);
   __gpu_thread_id_x();
+// AMDGPU-LABEL: define internal i32 @__gpu_thread_id(
+// AMDGPU-SAME: i32 noundef [[__DIM:%.*]]) #[[ATTR0]] {
+// AMDGPU-NEXT:  [[ENTRY:.*:]]
+// AMDGPU-NEXT:    [[RETVAL:%.*]] = alloca i32, align 4, addrspace(5)
+// AMDGPU-NEXT:    [[__DIM_ADDR:%.*]] = alloca i32, align 4, addrspace(5)
+// AMDGPU-NEXT:    [[RETVAL_ASCAST:%.*]] = addrspacecast ptr addrspace(5) [[RETVAL]] to ptr
+// AMDGPU-NEXT:    [[__DIM_ADDR_ASCAST:%.*]] = addrspacecast ptr addrspace(5) [[__DIM_ADDR]] to ptr
+// AMDGPU-NEXT:    store i32 [[__DIM]], ptr [[__DIM_ADDR_ASCAST]], align 4
+// AMDGPU-NEXT:    [[TMP0:%.*]] = load i32, ptr [[__DIM_ADDR_ASCAST]], align 4
+// AMDGPU-NEXT:    switch i32 [[TMP0]], label %[[SW_DEFAULT:.*]] [
+// AMDGPU-NEXT:      i32 0, label %[[SW_BB:.*]]
+// AMDGPU-NEXT:      i32 1, label %[[SW_BB1:.*]]
+// AMDGPU-NEXT:      i32 2, label %[[SW_BB3:.*]]
+// AMDGPU-NEXT:    ]
+// AMDGPU:       [[SW_BB]]:
+// AMDGPU-NEXT:    [[CALL:%.*]] = call i32 @__gpu_thread_id_x() #[[ATTR7]]
+// AMDGPU-NEXT:    store i32 [[CALL]], ptr [[RETVAL_ASCAST]], align 4
+// AMDGPU-NEXT:    br label %[[RETURN:.*]]
+// AMDGPU:       [[SW_BB1]]:
+// AMDGPU-NEXT:    [[CALL2:%.*]] = call i32 @__gpu_thread_id_y() #[[ATTR7]]
+// AMDGPU-NEXT:    store i32 [[CALL2]], ptr [[RETVAL_ASCAST]], align 4
+// AMDGPU-NEXT:    br label %[[RETURN]]
+// AMDGPU:       [[SW_BB3]]:
+// AMDGPU-NEXT:    [[CALL4:%.*]] = call i32 @__gpu_thread_id_z() #[[ATTR7]]
+// AMDGPU-NEXT:    store i32 [[CALL4]], ptr [[RETVAL_ASCAST]], align 4
+// AMDGPU-NEXT:    br label %[[RETURN]]
+// AMDGPU:       [[SW_DEFAULT]]:
+// AMDGPU-NEXT:    unreachable
+// AMDGPU:       [[RETURN]]:
+// AMDGPU-NEXT:    [[TMP1:%.*]] = load i32, ptr [[RETVAL_ASCAST]], align 4
+// AMDGPU-NEXT:    ret i32 [[TMP1]]
+//
+// NVPTX-LABEL: define internal i32 @__gpu_thread_id(
+// NVPTX-SAME: i32 noundef [[__DIM:%.*]]) #[[ATTR0]] {
+// NVPTX-NEXT:  [[ENTRY:.*:]]
+// NVPTX-NEXT:    [[RETVAL:%.*]] = alloca i32, align 4
+// NVPTX-NEXT:    [[__DIM_ADDR:%.*]] = alloca i32, align 4
+// NVPTX-NEXT:    store i32 [[__DIM]], ptr [[__DIM_ADDR]], align 4
+// NVPTX-NEXT:    [[TMP0:%.*]] = load i32, ptr [[__DIM_ADDR]], align 4
+// NVPTX-NEXT:    switch i32 [[TMP0]], label %[[SW_DEFAULT:.*]] [
+// NVPTX-NEXT:      i32 0, label %[[SW_BB:.*]]
+// NVPTX-NEXT:      i32 1, label %[[SW_BB1:.*]]
+// NVPTX-NEXT:      i32 2, label %[[SW_BB3:.*]]
+// NVPTX-NEXT:    ]
+// NVPTX:       [[SW_BB]]:
+// NVPTX-NEXT:    [[CALL:%.*]] = call i32 @__gpu_thread_id_x() #[[ATTR6]]
+// NVPTX-NEXT:    store i32 [[CALL]], ptr [[RETVAL]], align 4
+// NVPTX-NEXT:    br label %[[RETURN:.*]]
+// NVPTX:       [[SW_BB1]]:
+// NVPTX-NEXT:    [[CALL2:%.*]] = call i32 @__gpu_thread_id_y() #[[ATTR6]]
+// NVPTX-NEXT:    store i32 [[CALL2]], ptr [[RETVAL]], align 4
+// NVPTX-NEXT:    br label %[[RETURN]]
+// NVPTX:       [[SW_BB3]]:
+// NVPTX-NEXT:    [[CALL4:%.*]] = call i32 @__gpu_thread_id_z() #[[ATTR6]]
+// NVPTX-NEXT:    store i32 [[CALL4]], ptr [[RETVAL]], align 4
+// NVPTX-NEXT:    br label %[[RETURN]]
+// NVPTX:       [[SW_DEFAULT]]:
+// NVPTX-NEXT:    unreachable
+// NVPTX:       [[RETURN]]:
+// NVPTX-NEXT:    [[TMP1:%.*]] = load i32, ptr [[RETVAL]], align 4
+// NVPTX-NEXT:    ret i32 [[TMP1]]
+//
   __gpu_thread_id_y();
   __gpu_thread_id_z();
   __gpu_thread_id(0);
@@ -100,7 +162,7 @@ __gpu_kernel void foo() {
   __gpu_ballot(-1, 1);
   __gpu_sync_threads();
   __gpu_sync_lane(-1);
-  __gpu_shuffle_idx_u32(-1, -1, -1);
+  __gpu_shuffle_idx_u32(-1, -1, -1, 0);
   __gpu_first_lane_id(-1);
   __gpu_is_first_in_lane(-1);
   __gpu_exit();
diff --git a/clang/test/Import/cxx-anon-namespace/test.cpp b/clang/test/Import/cxx-anon-namespace/test.cpp
index e7668cc36f0e8..9a2df03d35bbc 100644
--- a/clang/test/Import/cxx-anon-namespace/test.cpp
+++ b/clang/test/Import/cxx-anon-namespace/test.cpp
@@ -14,12 +14,12 @@
 // CHECK-NEXT: CompoundStmt
 // This is for the nested anonymous namespace.
 // CHECK-NEXT: UsingDirectiveDecl
-// CHECK-SAME: ''
+// CHECK-SAME: <col:1, line:24:1>
 // CHECK: FunctionDecl
 // CHECK-SAME: func1
 // CHECK-NEXT: CompoundStmt
 // CHECK-NEXT: UsingDirectiveDecl
-// CHECK-SAME: ''
+// CHECK-SAME: <col:1, line:4:1>
 
 // CHECK: NamespaceDecl
 // CHECK-SAME: test_namespace1
@@ -28,7 +28,7 @@
 // CHECK-SAME: func2
 // CHECK-NEXT: CompoundStmt
 // CHECK-NEXT: UsingDirectiveDecl
-// CHECK-SAME: ''
+// CHECK-SAME: <col:1, line:9:1>
 
 // CHECK-NEXT: NamespaceDecl
 // CHECK-SAME: test_namespace2
@@ -39,7 +39,7 @@
 // CHECK-SAME: func3
 // CHECK-NEXT: CompoundStmt
 // CHECK-NEXT: UsingDirectiveDecl
-// CHECK-SAME: ''
+// CHECK-SAME: <col:1, line:17:1>
 
 void expr() {
   func1();
diff --git a/clang/test/Index/index-deduction-guide.cpp b/clang/test/Index/index-deduction-guide.cpp
new file mode 100644
index 0000000000000..a29162e8588e8
--- /dev/null
+++ b/clang/test/Index/index-deduction-guide.cpp
@@ -0,0 +1,10 @@
+// RUN: c-index-test core -print-source-symbols -- %s -std=gnu++17 | FileCheck %s
+
+template<typename T>
+typename T::type declval() {}
+template <typename T> struct Test;
+template <typename C, typename T = decltype(declval<C>().d())> Test(C &) -> Test<T>;
+// CHECK: [[@LINE-1]]:45 | function/C | declval
+// CHECK-NOT: RelCall
+// CHECK: [[@LINE-3]]:77 | struct(Gen)/C++ | Test
+// CHECK: [[@LINE-4]]:64 | struct(Gen)/C++ | Test
diff --git a/clang/test/Interpreter/simple-exception.cpp b/clang/test/Interpreter/simple-exception.cpp
index 6749acd6e6bd2..651e8d9402f89 100644
--- a/clang/test/Interpreter/simple-exception.cpp
+++ b/clang/test/Interpreter/simple-exception.cpp
@@ -1,7 +1,7 @@
 // clang-format off
 // UNSUPPORTED: system-aix
-// XFAIL for arm and arm64, or running on Windows.
-// XFAIL: target=arm{{.*}}, system-windows
+// XFAIL for arm, or running on Windows.
+// XFAIL: target=arm-{{.*}}, target=armv{{.*}}, system-windows
 // RUN: cat %s | clang-repl | FileCheck %s
 
 // Incompatible with msan. It passes with -O3 but fail -Oz. Interpreter
diff --git a/clang/test/Lexer/cxx-features.cpp b/clang/test/Lexer/cxx-features.cpp
index 3a60318d3f231..b423e94b26aca 100644
--- a/clang/test/Lexer/cxx-features.cpp
+++ b/clang/test/Lexer/cxx-features.cpp
@@ -7,7 +7,6 @@
 // RUN: %clang_cc1 -std=c++2c -fcxx-exceptions -verify %s
 
 //
-// RUN: %clang_cc1 -std=c++17 -fcxx-exceptions -fno-relaxed-template-template-args -DNO_RELAXED_TEMPLATE_TEMPLATE_ARGS=1 -verify %s
 // RUN: %clang_cc1 -std=c++17 -fcxx-exceptions -DCONCEPTS_TS=1 -verify %s
 // RUN: %clang_cc1 -std=c++14 -fno-rtti -fno-threadsafe-statics -verify %s -DNO_EXCEPTIONS -DNO_RTTI -DNO_THREADSAFE_STATICS
 // RUN: %clang_cc1 -std=c++14 -fchar8_t -DNO_EXCEPTIONS -DCHAR8_T -verify %s
@@ -239,9 +238,7 @@
 #error "wrong value for __cpp_nontype_template_args"
 #endif
 
-#if !defined(NO_RELAXED_TEMPLATE_TEMPLATE_ARGS) \
-    ? check(template_template_args, 201611, 201611, 201611, 201611, 201611, 201611, 201611) \
-    : check(template_template_args, 0, 0, 0, 0, 0, 0, 0)
+#if check(template_template_args, 201611, 201611, 201611, 201611, 201611, 201611, 201611)
 #error "wrong value for __cpp_template_template_args"
 #endif
 
diff --git a/clang/test/Misc/warning-flags.c b/clang/test/Misc/warning-flags.c
index 1fd0244083335..46b4e5a064c36 100644
--- a/clang/test/Misc/warning-flags.c
+++ b/clang/test/Misc/warning-flags.c
@@ -18,7 +18,7 @@ This test serves two purposes:
 
 The list of warnings below should NEVER grow.  It should gradually shrink to 0.
 
-CHECK: Warnings without flags (61):
+CHECK: Warnings without flags (59):
 
 CHECK-NEXT:   ext_expected_semi_decl_list
 CHECK-NEXT:   ext_missing_whitespace_after_macro_name
@@ -32,7 +32,6 @@ CHECK-NEXT:   pp_invalid_string_literal
 CHECK-NEXT:   pp_out_of_date_dependency
 CHECK-NEXT:   pp_poisoning_existing_macro
 CHECK-NEXT:   warn_accessor_property_type_mismatch
-CHECK-NEXT:   warn_arcmt_nsalloc_realloc
 CHECK-NEXT:   warn_asm_label_on_auto_decl
 CHECK-NEXT:   warn_c_kext
 CHECK-NEXT:   warn_call_wrong_number_of_arguments
@@ -60,7 +59,6 @@ CHECK-NEXT:   warn_method_param_redefinition
 CHECK-NEXT:   warn_missing_case_for_condition
 CHECK-NEXT:   warn_missing_dependent_template_keyword
 CHECK-NEXT:   warn_missing_whitespace_after_macro_name
-CHECK-NEXT:   warn_mt_message
 CHECK-NEXT:   warn_no_constructor_for_refconst
 CHECK-NEXT:   warn_not_compound_assign
 CHECK-NEXT:   warn_objc_property_copy_missing_on_block
diff --git a/clang/test/Modules/cxx-templates.cpp b/clang/test/Modules/cxx-templates.cpp
index b197f319e0d15..f587af4beb7ce 100644
--- a/clang/test/Modules/cxx-templates.cpp
+++ b/clang/test/Modules/cxx-templates.cpp
@@ -40,7 +40,10 @@ void g() {
 
   template_param_kinds_1<0>(); // ok, from cxx-templates-a.h
   template_param_kinds_1<int>(); // ok, from cxx-templates-b.h
-  template_param_kinds_2<Tmpl_T_C>(); // ok, from cxx-templates-b.h
+
+  template_param_kinds_2<Tmpl_T_C>(); // expected-error {{no matching function for call}}
+  // expected-note@Inputs/cxx-templates-a.h:11 {{candidate}}
+  // expected-note@Inputs/cxx-templates-b.h:11 {{candidate}}
 
   template_param_kinds_2<Tmpl_T_I_I>(); // expected-error {{ambiguous}}
   // expected-note@Inputs/cxx-templates-a.h:11 {{candidate}}
diff --git a/clang/test/Modules/odr_hash.cpp b/clang/test/Modules/odr_hash.cpp
index 4de0e50dbc0eb..b0f5b904f2b39 100644
--- a/clang/test/Modules/odr_hash.cpp
+++ b/clang/test/Modules/odr_hash.cpp
@@ -4020,7 +4020,7 @@ struct Valid {
 };
 #else
 Invalid::L2<1>::L3<1> invalid;
-// expected-error@second.h:* {{'Types::InjectedClassName::Invalid::L2::L3::x' from module 'SecondModule' is not present in definition of 'L3<>' in module 'FirstModule'}}
+// expected-error@second.h:* {{'Types::InjectedClassName::Invalid::L2::L3::x' from module 'SecondModule' is not present in definition of 'L3<value-parameter-1-0>' in module 'FirstModule'}}
 // expected-note@first.h:* {{declaration of 'x' does not match}}
 Valid::L2<1>::L3<1> valid;
 #endif
diff --git a/clang/test/Modules/pr120277-2.cpp b/clang/test/Modules/pr120277-2.cpp
new file mode 100644
index 0000000000000..f3a7e47431848
--- /dev/null
+++ b/clang/test/Modules/pr120277-2.cpp
@@ -0,0 +1,66 @@
+// RUN: rm -rf %t
+// RUN: mkdir -p %t
+// RUN: split-file %s %t
+
+// RUN: %clang_cc1 -std=c++20 -emit-header-unit -xc++-user-header %t/hu-01.h \
+// RUN:  -o %t/hu-01.pcm
+
+// RUN: %clang_cc1 -std=c++20 -emit-header-unit -xc++-user-header %t/hu-02.h \
+// RUN:  -Wno-experimental-header-units -fmodule-file=%t/hu-01.pcm -o %t/hu-02.pcm
+
+// RUN: %clang_cc1 -std=c++20 -emit-header-unit -xc++-user-header %t/hu-03.h \
+// RUN:  -Wno-experimental-header-units \
+// RUN:  -fmodule-file=%t/hu-01.pcm -o %t/hu-03.pcm
+
+// RUN: %clang_cc1 -std=c++20 -emit-header-unit -xc++-user-header %t/hu-04.h \
+// RUN:  -Wno-experimental-header-units -fmodule-file=%t/hu-02.pcm \
+// RUN:  -fmodule-file=%t/hu-03.pcm -o %t/hu-04.pcm
+
+// RUN: %clang_cc1 -std=c++20 -emit-obj %t/main.cpp \
+// RUN:  -Wno-experimental-header-units -fmodule-file=%t/hu-04.pcm
+//--- hu-01.h
+template <typename T>
+struct A {
+  ~A() { f(); }
+  auto f() const { return 0; }
+};
+
+template <typename T>
+struct B {
+  int g() const { return a.f(); }
+  A<T> a;
+};
+
+//--- hu-02.h
+import "hu-01.h";
+
+template <typename = void>
+struct C {
+  void h() {
+    B<int>().g();
+  }
+};
+
+template struct A<double>;
+
+//--- hu-03.h
+import "hu-01.h";
+
+inline B<int> b() {
+  return {};
+}
+
+//--- hu-04.h
+import "hu-02.h";
+import "hu-03.h";
+
+inline void f4() {
+  C{}.h();
+}
+
+//--- main.cpp
+import "hu-04.h";
+
+int main() {
+  f4();
+}
diff --git a/clang/test/Modules/pr121245.cpp b/clang/test/Modules/pr121245.cpp
new file mode 100644
index 0000000000000..0e276ad0e435d
--- /dev/null
+++ b/clang/test/Modules/pr121245.cpp
@@ -0,0 +1,93 @@
+// If this test fails, it should be investigated under Debug builds.
+// Before the PR, this test was encountering an `llvm_unreachable()`.
+
+// RUN: rm -rf %t
+// RUN: mkdir -p %t
+// RUN: split-file %s %t
+// RUN: cd %t
+
+// RUN: %clang_cc1 -std=c++20 -emit-header-unit -xc++-user-header %t/hu-01.h \
+// RUN:  -fcxx-exceptions -o %t/hu-01.pcm
+
+// RUN: %clang_cc1 -std=c++20 -emit-header-unit -xc++-user-header %t/hu-02.h \
+// RUN:  -Wno-experimental-header-units -fcxx-exceptions \
+// RUN:  -fmodule-file=%t/hu-01.pcm -o %t/hu-02.pcm
+
+// RUN: %clang_cc1 -std=c++20 -emit-header-unit -xc++-user-header %t/hu-03.h \
+// RUN:  -Wno-experimental-header-units -fcxx-exceptions \
+// RUN:  -fmodule-file=%t/hu-01.pcm -o %t/hu-03.pcm
+
+// RUN: %clang_cc1 -std=c++20 -emit-header-unit -xc++-user-header %t/hu-04.h \
+// RUN:  -Wno-experimental-header-units -fcxx-exceptions \
+// RUN:  -fmodule-file=%t/hu-01.pcm -o %t/hu-04.pcm
+
+// RUN: %clang_cc1 -std=c++20 -emit-header-unit -xc++-user-header %t/hu-05.h \
+// RUN:  -Wno-experimental-header-units -fcxx-exceptions \
+// RUN:  -fmodule-file=%t/hu-03.pcm -fmodule-file=%t/hu-04.pcm \
+// RUN:  -fmodule-file=%t/hu-01.pcm -o %t/hu-05.pcm
+
+// RUN: %clang_cc1 -std=c++20 -emit-obj %t/main.cpp \
+// RUN:  -Wno-experimental-header-units -fcxx-exceptions \
+// RUN:  -fmodule-file=%t/hu-02.pcm -fmodule-file=%t/hu-05.pcm \
+// RUN:  -fmodule-file=%t/hu-04.pcm -fmodule-file=%t/hu-03.pcm \
+// RUN:  -fmodule-file=%t/hu-01.pcm
+
+//--- hu-01.h
+template <typename T>
+struct A {
+  A() {}
+  ~A() {}
+};
+
+template <typename T>
+struct EBO : T {
+  EBO() = default;
+};
+
+template <typename T>
+struct HT : EBO<A<T>> {};
+
+//--- hu-02.h
+import "hu-01.h";
+
+inline void f() {
+  HT<int>();
+}
+
+//--- hu-03.h
+import "hu-01.h";
+
+struct C {
+  C();
+
+  HT<long> _;
+};
+
+//--- hu-04.h
+import "hu-01.h";
+
+void g(HT<long> = {});
+
+//--- hu-05.h
+import "hu-03.h";
+import "hu-04.h";
+import "hu-01.h";
+
+struct B {
+  virtual ~B() = default;
+
+  virtual void f() {
+    HT<long>();
+  }
+};
+
+//--- main.cpp
+import "hu-02.h";
+import "hu-05.h";
+import "hu-03.h";
+
+int main() {
+  f();
+  C();
+  B();
+}
diff --git a/clang/test/Modules/pr125521.cppm b/clang/test/Modules/pr125521.cppm
new file mode 100644
index 0000000000000..d064cdfe3eb73
--- /dev/null
+++ b/clang/test/Modules/pr125521.cppm
@@ -0,0 +1,57 @@
+// RUN: rm -rf %t
+// RUN: mkdir -p %t
+// RUN: split-file %s %t
+//
+// RUN: %clang_cc1 -std=c++20 %t/mod2.cppm -emit-module-interface -o %t/mod2.pcm
+// RUN: %clang_cc1 -std=c++20 %t/mod1.cppm -emit-module-interface -o %t/mod1.pcm \
+// RUN:     -fmodule-file=Mod2=%t/mod2.pcm
+// RUN: %clang_cc1 -std=c++20 %t/test.cc -fmodule-file=Mod2=%t/mod2.pcm -fmodule-file=Mod=%t/mod1.pcm \
+// RUN:     -fsyntax-only -verify
+
+// RUN: %clang_cc1 -std=c++20 %t/mod2.cppm -emit-module-interface -o %t/mod2.pcm
+// RUN: %clang_cc1 -std=c++20 %t/mod1.cppm -emit-module-interface -o %t/mod1.pcm \
+// RUN:     -fmodule-file=Mod2=%t/mod2.pcm
+// RUN: %clang_cc1 -std=c++20 %t/mod1.pcm  -fmodule-file=Mod2=%t/mod2.pcm -emit-llvm -o - \
+// RUN:     | FileCheck %t/mod1.cppm
+
+//--- hello.h
+template <typename V> int get() noexcept {return 0;};
+
+template <typename T>
+class List
+{
+    template <typename V> friend int get() noexcept;
+};
+
+//--- mod2.cppm
+module;
+#include "hello.h"
+export module Mod2;
+export const char *modFn2() {
+    List<int> a;
+    return "hello";
+}
+
+//--- mod1.cppm
+module;
+#include "hello.h"
+export module Mod;
+import Mod2;
+export extern "C" const char *modFn() {
+    List<int> a;
+    List<double> b;
+    return modFn2();
+}
+
+// Fine enough to check it won't crash.
+// CHECK: define {{.*}}@modFn
+
+//--- test.cc
+// expected-no-diagnostics
+import Mod;
+import Mod2;
+
+void test() {
+    modFn();
+    modFn2();
+}
diff --git a/clang/test/Modules/pr126373.cppm b/clang/test/Modules/pr126373.cppm
new file mode 100644
index 0000000000000..f176a587b51ce
--- /dev/null
+++ b/clang/test/Modules/pr126373.cppm
@@ -0,0 +1,34 @@
+// RUN: rm -rf %t
+// RUN: mkdir -p %t
+// RUN: split-file %s %t
+//
+// RUN: %clang_cc1 -std=c++20 %t/module1.cppm -emit-module-interface -o %t/module1.pcm
+// RUN: %clang_cc1 -std=c++20 -fmodule-file=module1=%t/module1.pcm  %t/module2.cppm \
+// RUN:     -emit-module-interface -o %t/module2.pcm
+// RUN: %clang_cc1 -std=c++20 %t/module2.pcm -fmodule-file=module1=%t/module1.pcm \
+// RUN:     -emit-llvm -o - | FileCheck %t/module2.cppm
+
+//--- test.h
+template<typename T>
+struct Test {
+  template<typename U>
+  friend class Test;
+};
+
+//--- module1.cppm
+module;
+#include "test.h"
+export module module1;
+export void f1(Test<int>) {}
+
+//--- module2.cppm
+module;
+#include "test.h"
+export module module2;
+import module1;
+export void f2(Test<float>) {}
+
+extern "C" void func() {}
+
+// Fine enough to check the IR is emitted correctly.
+// CHECK: define{{.*}}@func
diff --git a/clang/test/OpenMP/assume_lambda.cpp b/clang/test/OpenMP/assume_lambda.cpp
index 4000dc1600b23..fa0d9ac2c327c 100644
--- a/clang/test/OpenMP/assume_lambda.cpp
+++ b/clang/test/OpenMP/assume_lambda.cpp
@@ -1,4 +1,4 @@
-// RUN: %clang_cc1 -fopenmp -x c++ -std=c++11 -ast-print %s | FileCheck %s
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=60 -x c++ -std=c++11 -ast-print %s | FileCheck %s
 // expected-no-diagnostics
 
 extern int bar(int);
@@ -11,6 +11,12 @@ int foo(int arg)
 // CHECK: auto fn = [](int x) {
     return fn(5);
   }
+  #pragma omp assume no_openmp_constructs
+  {
+    auto fn = [](int x) { return bar(x); };
+// CHECK: auto fn = [](int x) {
+    return fn(6);
+  }
 }
 
 class C {
diff --git a/clang/test/OpenMP/assume_nesting_tmpl.cpp b/clang/test/OpenMP/assume_nesting_tmpl.cpp
index 59ef603f5874d..b0f50b0ddddf7 100644
--- a/clang/test/OpenMP/assume_nesting_tmpl.cpp
+++ b/clang/test/OpenMP/assume_nesting_tmpl.cpp
@@ -1,4 +1,4 @@
-// RUN: %clang_cc1 -fopenmp -x c++ -std=c++11 -ast-print %s | FileCheck %s
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=60 -x c++ -std=c++11 -ast-print %s | FileCheck %s
 // expected-no-diagnostics
 
 extern void bar();
@@ -14,6 +14,14 @@ void foo()
     {}
   }
 
+  #pragma omp assume no_openmp_constructs
+  // CHECK: omp assume no_openmp_constructs
+  {
+    #pragma omp assume no_parallelism
+    // CHECK: omp assume no_parallelism
+    {}
+  }
+
   #pragma omp target
   // CHECK: omp target
   {
diff --git a/clang/test/OpenMP/assume_template.cpp b/clang/test/OpenMP/assume_template.cpp
index 20d1c5d437145..2bf7567f675e9 100644
--- a/clang/test/OpenMP/assume_template.cpp
+++ b/clang/test/OpenMP/assume_template.cpp
@@ -1,6 +1,6 @@
-// RUN: %clang_cc1 -verify -fopenmp -ast-print %s | FileCheck %s
-// RUN: %clang_cc1 -fopenmp -x c++ -std=c++11 -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp -std=c++11 -include-pch %t -verify %s -ast-print | FileCheck %s
+// RUN: %clang_cc1 -verify -fopenmp -fopenmp-version=60 -ast-print %s | FileCheck %s
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=60 -x c++ -std=c++11 -emit-pch -o %t %s
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=60 -std=c++11 -include-pch %t -verify %s -ast-print | FileCheck %s
 // expected-no-diagnostics
 
 #ifndef HEADER
@@ -17,6 +17,12 @@ int foo(T arg)
 // CHECK: auto fn = [](int x) {
     return fn(5);
   }
+  #pragma omp assume no_openmp_constructs
+  {
+    auto fn = [](int x) { return qux(x); };
+// CHECK: auto fn = [](int x) {
+    return fn(6);
+  }
 }
 
 template<typename T>
diff --git a/clang/test/OpenMP/assumes_codegen.cpp b/clang/test/OpenMP/assumes_codegen.cpp
index 4206e5a9caab7..a52ea956af245 100644
--- a/clang/test/OpenMP/assumes_codegen.cpp
+++ b/clang/test/OpenMP/assumes_codegen.cpp
@@ -1,11 +1,11 @@
-// RUN: %clang_cc1 -verify -fopenmp -x c++ -emit-llvm %s -fexceptions -fcxx-exceptions -triple x86_64-unknown-unknown -o - | FileCheck %s
-// RUN: %clang_cc1 -verify -fopenmp -ast-print %s | FileCheck %s --check-prefix=AST
-// RUN: %clang_cc1 -fopenmp -x c++ -std=c++11 -triple x86_64-unknown-unknown -fexceptions -fcxx-exceptions -emit-pch -verify -o %t %s
-// RUN: %clang_cc1 -fopenmp -x c++ -triple x86_64-unknown-unknown -fexceptions -fcxx-exceptions -std=c++11 -include-pch %t -verify=pch %s -emit-llvm -o - | FileCheck %s
-// RUN: %clang_cc1 -verify -triple x86_64-apple-darwin10 -fopenmp -fexceptions -fcxx-exceptions -debug-info-kind=line-tables-only -x c++ -emit-llvm %s -o - | FileCheck %s
-// RUN: %clang_cc1 -verify -fopenmp -fopenmp-enable-irbuilder -x c++ -emit-llvm %s -fexceptions -fcxx-exceptions -triple x86_64-unknown-unknown -o - | FileCheck %s
-// RUN: %clang_cc1 -fopenmp -fopenmp-enable-irbuilder -x c++ -std=c++11 -triple x86_64-unknown-unknown -fexceptions -fcxx-exceptions -emit-pch -verify -o %t %s
-// RUN: %clang_cc1 -fopenmp -fopenmp-enable-irbuilder -x c++ -triple x86_64-unknown-unknown -fexceptions -fcxx-exceptions -std=c++11 -include-pch %t -verify=pch %s -emit-llvm -o - | FileCheck %s
+// RUN: %clang_cc1 -verify -fopenmp -fopenmp-version=60 -x c++ -emit-llvm %s -fexceptions -fcxx-exceptions -triple x86_64-unknown-unknown -o - | FileCheck %s
+// RUN: %clang_cc1 -verify -fopenmp -fopenmp-version=60 -ast-print %s | FileCheck %s --check-prefix=AST
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=60 -x c++ -std=c++11 -triple x86_64-unknown-unknown -fexceptions -fcxx-exceptions -emit-pch -verify -o %t %s
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=60 -x c++ -triple x86_64-unknown-unknown -fexceptions -fcxx-exceptions -std=c++11 -include-pch %t -verify=pch %s -emit-llvm -o - | FileCheck %s
+// RUN: %clang_cc1 -verify -triple x86_64-apple-darwin10 -fopenmp -fopenmp-version=60 -fexceptions -fcxx-exceptions -debug-info-kind=line-tables-only -x c++ -emit-llvm %s -o - | FileCheck %s
+// RUN: %clang_cc1 -verify -fopenmp -fopenmp-version=60 -fopenmp-enable-irbuilder -x c++ -emit-llvm %s -fexceptions -fcxx-exceptions -triple x86_64-unknown-unknown -o - | FileCheck %s
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=60 -fopenmp-enable-irbuilder -x c++ -std=c++11 -triple x86_64-unknown-unknown -fexceptions -fcxx-exceptions -emit-pch -verify -o %t %s
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=60 -fopenmp-enable-irbuilder -x c++ -triple x86_64-unknown-unknown -fexceptions -fcxx-exceptions -std=c++11 -include-pch %t -verify=pch %s -emit-llvm -o - | FileCheck %s
 
 // pch-no-diagnostics
 
@@ -15,7 +15,7 @@
 void foo() {
 }
 
-#pragma omp assumes no_openmp_routines warning ext_another_warning(1) ext_after_invalid_clauses // expected-warning {{valid assumes clauses start with 'ext_', 'absent', 'contains', 'holds', 'no_openmp', 'no_openmp_routines', 'no_parallelism'; token will be ignored}} expected-warning {{'ext_another_warning' clause should not be followed by arguments; tokens will be ignored}} expected-note {{the ignored tokens spans until here}}
+#pragma omp assumes no_openmp_routines warning ext_another_warning(1) ext_after_invalid_clauses // expected-warning {{valid assumes clauses start with 'ext_', 'absent', 'contains', 'holds', 'no_openmp', 'no_openmp_routines', 'no_openmp_constructs', 'no_parallelism'; token will be ignored}} expected-warning {{'ext_another_warning' clause should not be followed by arguments; tokens will be ignored}} expected-note {{the ignored tokens spans until here}}
 
 #pragma omp assumes no_openmp
 
diff --git a/clang/test/OpenMP/assumes_include_nvptx.cpp b/clang/test/OpenMP/assumes_include_nvptx.cpp
index 4577ea4c9c2b5..c5040989a0e40 100644
--- a/clang/test/OpenMP/assumes_include_nvptx.cpp
+++ b/clang/test/OpenMP/assumes_include_nvptx.cpp
@@ -11,11 +11,11 @@
 
 // TODO: Think about teaching the OMPIRBuilder about default attributes as well so the __kmpc* declarations are annotated.
 
-// CHECK: define weak_odr protected void @__omp_offloading_{{.*}}__Z17complex_reductionIfEvv_{{.*}}({{.*}}) [[attr0:#[0-9]]]
+// CHECK: define weak_odr protected ptx_kernel void @__omp_offloading_{{.*}}__Z17complex_reductionIfEvv_{{.*}}({{.*}}) [[attr0:#[0-9]]]
 // CHECK: call i32 @__kmpc_target_init(
 // CHECK: declare noundef float @_Z3sinf(float noundef) [[attr1:#[0-9]*]]
 // CHECK: declare void @__kmpc_target_deinit(
-// CHECK: define weak_odr protected void @__omp_offloading_{{.*}}__Z17complex_reductionIdEvv_{{.*}}({{.*}}) [[attr0]]
+// CHECK: define weak_odr protected ptx_kernel void @__omp_offloading_{{.*}}__Z17complex_reductionIdEvv_{{.*}}({{.*}}) [[attr0]]
 // CHECK: %call = call noundef double @_Z3sind(double noundef 0.000000e+00) [[attr2:#[0-9]]]
 // CHECK: declare noundef double @_Z3sind(double noundef) [[attr1]]
 
diff --git a/clang/test/OpenMP/assumes_messages.c b/clang/test/OpenMP/assumes_messages.c
index 99bbe93dc1fc2..9bbedf59b77fc 100644
--- a/clang/test/OpenMP/assumes_messages.c
+++ b/clang/test/OpenMP/assumes_messages.c
@@ -1,46 +1,46 @@
-// RUN: %clang_cc1 -triple=x86_64-pc-win32 -verify -fopenmp -x c -std=c99 -fms-extensions -Wno-pragma-pack %s
+// RUN: %clang_cc1 -triple=x86_64-pc-win32 -verify -fopenmp -fopenmp-version=60 -x c -std=c99 -fms-extensions -Wno-pragma-pack %s
 
-// RUN: %clang_cc1 -triple=x86_64-pc-win32 -verify -fopenmp-simd -x c -std=c99 -fms-extensions -Wno-pragma-pack %s
+// RUN: %clang_cc1 -triple=x86_64-pc-win32 -verify -fopenmp-simd -fopenmp-version=60 -x c -std=c99 -fms-extensions -Wno-pragma-pack %s
 
-#pragma omp assumes // expected-error {{expected at least one 'ext_', 'absent', 'contains', 'holds', 'no_openmp', 'no_openmp_routines', 'no_parallelism' clause for '#pragma omp assumes'}}
+#pragma omp assumes // expected-error {{expected at least one 'ext_', 'absent', 'contains', 'holds', 'no_openmp', 'no_openmp_routines', 'no_openmp_constructs', 'no_parallelism' clause for '#pragma omp assumes'}}
 #pragma omp begin // expected-error {{expected an OpenMP directive}}
-#pragma omp begin assumes // expected-error {{expected at least one 'ext_', 'absent', 'contains', 'holds', 'no_openmp', 'no_openmp_routines', 'no_parallelism' clause for '#pragma omp begin assumes'}}
+#pragma omp begin assumes // expected-error {{expected at least one 'ext_', 'absent', 'contains', 'holds', 'no_openmp', 'no_openmp_routines', 'no_openmp_constructs', 'no_parallelism' clause for '#pragma omp begin assumes'}}
 #pragma omp end assumes
 
-#pragma omp assumes foobar // expected-warning {{valid assumes clauses start with 'ext_', 'absent', 'contains', 'holds', 'no_openmp', 'no_openmp_routines', 'no_parallelism'; token will be ignored}}
-#pragma omp begin assumes foobar // expected-warning {{valid begin assumes clauses start with 'ext_', 'absent', 'contains', 'holds', 'no_openmp', 'no_openmp_routines', 'no_parallelism'; token will be ignored}}
+#pragma omp assumes foobar // expected-warning {{valid assumes clauses start with 'ext_', 'absent', 'contains', 'holds', 'no_openmp', 'no_openmp_routines', 'no_openmp_constructs', 'no_parallelism'; token will be ignored}}
+#pragma omp begin assumes foobar // expected-warning {{valid begin assumes clauses start with 'ext_', 'absent', 'contains', 'holds', 'no_openmp', 'no_openmp_routines', 'no_openmp_constructs', 'no_parallelism'; token will be ignored}}
 #pragma omp end assumes
 
-#pragma omp begin assumes foobar(foo 2 no_openmp // expected-error {{expected ')'}} expected-warning {{valid begin assumes clauses start with 'ext_', 'absent', 'contains', 'holds', 'no_openmp', 'no_openmp_routines', 'no_parallelism'; tokens will be ignored}} expected-note {{to match this '('}}
-#pragma omp assumes foobar(foo 2 no_openmp // expected-error {{expected ')'}} expected-warning {{valid assumes clauses start with 'ext_', 'absent', 'contains', 'holds', 'no_openmp', 'no_openmp_routines', 'no_parallelism'; tokens will be ignored}} expected-note {{to match this '('}}
+#pragma omp begin assumes foobar(foo 2 no_openmp // expected-error {{expected ')'}} expected-warning {{valid begin assumes clauses start with 'ext_', 'absent', 'contains', 'holds', 'no_openmp', 'no_openmp_routines', 'no_openmp_constructs', 'no_parallelism'; tokens will be ignored}} expected-note {{to match this '('}}
+#pragma omp assumes foobar(foo 2 no_openmp // expected-error {{expected ')'}} expected-warning {{valid assumes clauses start with 'ext_', 'absent', 'contains', 'holds', 'no_openmp', 'no_openmp_routines', 'no_openmp_constructs', 'no_parallelism'; tokens will be ignored}} expected-note {{to match this '('}}
 #pragma omp end assumes
 
-#pragma omp begin assumes foobar(foo 2 baz) // expected-warning {{valid begin assumes clauses start with 'ext_', 'absent', 'contains', 'holds', 'no_openmp', 'no_openmp_routines', 'no_parallelism'; tokens will be ignored}} expected-note {{the ignored tokens spans until here}}
-#pragma omp assumes foobar(foo 2 baz) // expected-warning {{valid assumes clauses start with 'ext_', 'absent', 'contains', 'holds', 'no_openmp', 'no_openmp_routines', 'no_parallelism'; tokens will be ignored}} expected-note {{the ignored tokens spans until here}}
+#pragma omp begin assumes foobar(foo 2 baz) // expected-warning {{valid begin assumes clauses start with 'ext_', 'absent', 'contains', 'holds', 'no_openmp', 'no_openmp_routines', 'no_openmp_constructs', 'no_parallelism'; tokens will be ignored}} expected-note {{the ignored tokens spans until here}}
+#pragma omp assumes foobar(foo 2 baz) // expected-warning {{valid assumes clauses start with 'ext_', 'absent', 'contains', 'holds', 'no_openmp', 'no_openmp_routines', 'no_openmp_constructs', 'no_parallelism'; tokens will be ignored}} expected-note {{the ignored tokens spans until here}}
 #pragma omp end assumes
 
-#pragma omp begin assumes foobar foo 2 baz) bar // expected-warning {{valid begin assumes clauses start with 'ext_', 'absent', 'contains', 'holds', 'no_openmp', 'no_openmp_routines', 'no_parallelism'; token will be ignored}} expected-warning {{valid begin assumes clauses start with 'ext_', 'absent', 'contains', 'holds', 'no_openmp', 'no_openmp_routines', 'no_parallelism'; token will be ignored}} expected-warning {{valid begin assumes clauses start with 'ext_', 'absent', 'contains', 'holds', 'no_openmp', 'no_openmp_routines', 'no_parallelism'; token will be ignored}} expected-warning {{valid begin assumes clauses start with 'ext_', 'absent', 'contains', 'holds', 'no_openmp', 'no_openmp_routines', 'no_parallelism'; token will be ignored}} expected-warning {{valid begin assumes clauses start with 'ext_', 'absent', 'contains', 'holds', 'no_openmp', 'no_openmp_routines', 'no_parallelism'; token will be ignored}} expected-warning {{valid begin assumes clauses start with 'ext_', 'absent', 'contains', 'holds', 'no_openmp', 'no_openmp_routines', 'no_parallelism'; token will be ignored}}
-#pragma omp assumes foobar foo 2 baz) bar // expected-warning {{valid assumes clauses start with 'ext_', 'absent', 'contains', 'holds', 'no_openmp', 'no_openmp_routines', 'no_parallelism'; token will be ignored}} expected-warning {{valid assumes clauses start with 'ext_', 'absent', 'contains', 'holds', 'no_openmp', 'no_openmp_routines', 'no_parallelism'; token will be ignored}} expected-warning {{valid assumes clauses start with 'ext_', 'absent', 'contains', 'holds', 'no_openmp', 'no_openmp_routines', 'no_parallelism'; token will be ignored}} expected-warning {{valid assumes clauses start with 'ext_', 'absent', 'contains', 'holds', 'no_openmp', 'no_openmp_routines', 'no_parallelism'; token will be ignored}} expected-warning {{valid assumes clauses start with 'ext_', 'absent', 'contains', 'holds', 'no_openmp', 'no_openmp_routines', 'no_parallelism'; token will be ignored}} expected-warning {{valid assumes clauses start with 'ext_', 'absent', 'contains', 'holds', 'no_openmp', 'no_openmp_routines', 'no_parallelism'; token will be ignored}}
+#pragma omp begin assumes foobar foo 2 baz) bar // expected-warning {{valid begin assumes clauses start with 'ext_', 'absent', 'contains', 'holds', 'no_openmp', 'no_openmp_routines', 'no_openmp_constructs', 'no_parallelism'; token will be ignored}} expected-warning {{valid begin assumes clauses start with 'ext_', 'absent', 'contains', 'holds', 'no_openmp', 'no_openmp_routines', 'no_openmp_constructs', 'no_parallelism'; token will be ignored}} expected-warning {{valid begin assumes clauses start with 'ext_', 'absent', 'contains', 'holds', 'no_openmp', 'no_openmp_routines', 'no_openmp_constructs', 'no_parallelism'; token will be ignored}} expected-warning {{valid begin assumes clauses start with 'ext_', 'absent', 'contains', 'holds', 'no_openmp', 'no_openmp_routines', 'no_openmp_constructs', 'no_parallelism'; token will be ignored}} expected-warning {{valid begin assumes clauses start with 'ext_', 'absent', 'contains', 'holds', 'no_openmp', 'no_openmp_routines', 'no_openmp_constructs', 'no_parallelism'; token will be ignored}} expected-warning {{valid begin assumes clauses start with 'ext_', 'absent', 'contains', 'holds', 'no_openmp', 'no_openmp_routines', 'no_openmp_constructs', 'no_parallelism'; token will be ignored}}
+#pragma omp assumes foobar foo 2 baz) bar // expected-warning {{valid assumes clauses start with 'ext_', 'absent', 'contains', 'holds', 'no_openmp', 'no_openmp_routines', 'no_openmp_constructs', 'no_parallelism'; token will be ignored}} expected-warning {{valid assumes clauses start with 'ext_', 'absent', 'contains', 'holds', 'no_openmp', 'no_openmp_routines', 'no_openmp_constructs', 'no_parallelism'; token will be ignored}} expected-warning {{valid assumes clauses start with 'ext_', 'absent', 'contains', 'holds', 'no_openmp', 'no_openmp_routines', 'no_openmp_constructs', 'no_parallelism'; token will be ignored}} expected-warning {{valid assumes clauses start with 'ext_', 'absent', 'contains', 'holds', 'no_openmp', 'no_openmp_routines', 'no_openmp_constructs', 'no_parallelism'; token will be ignored}} expected-warning {{valid assumes clauses start with 'ext_', 'absent', 'contains', 'holds', 'no_openmp', 'no_openmp_routines', 'no_openmp_constructs', 'no_parallelism'; token will be ignored}} expected-warning {{valid assumes clauses start with 'ext_', 'absent', 'contains', 'holds', 'no_openmp', 'no_openmp_routines', 'no_openmp_constructs', 'no_parallelism'; token will be ignored}}
 #pragma omp end assumes
 
 #pragma omp assumes no_openmp(1) // expected-warning {{'no_openmp' clause should not be followed by arguments; tokens will be ignored}} expected-note {{the ignored tokens spans until here}}
 #pragma omp begin assumes no_openmp(1 2 3) // expected-warning {{'no_openmp' clause should not be followed by arguments; tokens will be ignored}} expected-note {{the ignored tokens spans until here}}
 #pragma omp end assumes no_openmp(1)
 
-#pragma omp assumes foobar no_openmp bazbaz // expected-warning {{valid assumes clauses start with 'ext_', 'absent', 'contains', 'holds', 'no_openmp', 'no_openmp_routines', 'no_parallelism'; token will be ignored}} expected-warning {{valid assumes clauses start with 'ext_', 'absent', 'contains', 'holds', 'no_openmp', 'no_openmp_routines', 'no_parallelism'; token will be ignored}}
-#pragma omp begin assumes foobar no_openmp bazbaz // expected-warning {{valid begin assumes clauses start with 'ext_', 'absent', 'contains', 'holds', 'no_openmp', 'no_openmp_routines', 'no_parallelism'; token will be ignored}} expected-warning {{valid begin assumes clauses start with 'ext_', 'absent', 'contains', 'holds', 'no_openmp', 'no_openmp_routines', 'no_parallelism'; token will be ignored}}
+#pragma omp assumes foobar no_openmp bazbaz // expected-warning {{valid assumes clauses start with 'ext_', 'absent', 'contains', 'holds', 'no_openmp', 'no_openmp_routines', 'no_openmp_constructs', 'no_parallelism'; token will be ignored}} expected-warning {{valid assumes clauses start with 'ext_', 'absent', 'contains', 'holds', 'no_openmp', 'no_openmp_routines', 'no_openmp_constructs', 'no_parallelism'; token will be ignored}}
+#pragma omp begin assumes foobar no_openmp bazbaz // expected-warning {{valid begin assumes clauses start with 'ext_', 'absent', 'contains', 'holds', 'no_openmp', 'no_openmp_routines', 'no_openmp_constructs', 'no_parallelism'; token will be ignored}} expected-warning {{valid begin assumes clauses start with 'ext_', 'absent', 'contains', 'holds', 'no_openmp', 'no_openmp_routines', 'no_openmp_constructs', 'no_parallelism'; token will be ignored}}
 #pragma omp end assumes
 
-#pragma omp begin assumes foobar(foo 2 baz) no_openmp bazbaz(foo 2 baz) // expected-warning {{valid begin assumes clauses start with 'ext_', 'absent', 'contains', 'holds', 'no_openmp', 'no_openmp_routines', 'no_parallelism'; tokens will be ignored}} expected-warning {{valid begin assumes clauses start with 'ext_', 'absent', 'contains', 'holds', 'no_openmp', 'no_openmp_routines', 'no_parallelism'; tokens will be ignored}} expected-note {{the ignored tokens spans until here}} expected-note {{the ignored tokens spans until here}}
-#pragma omp assumes foobar(foo 2 baz) no_openmp bazbaz(foo 2 baz) // expected-warning {{valid assumes clauses start with 'ext_', 'absent', 'contains', 'holds', 'no_openmp', 'no_openmp_routines', 'no_parallelism'; tokens will be ignored}} expected-warning {{valid assumes clauses start with 'ext_', 'absent', 'contains', 'holds', 'no_openmp', 'no_openmp_routines', 'no_parallelism'; tokens will be ignored}} expected-note {{the ignored tokens spans until here}} expected-note {{the ignored tokens spans until here}}
+#pragma omp begin assumes foobar(foo 2 baz) no_openmp bazbaz(foo 2 baz) // expected-warning {{valid begin assumes clauses start with 'ext_', 'absent', 'contains', 'holds', 'no_openmp', 'no_openmp_routines', 'no_openmp_constructs', 'no_parallelism'; tokens will be ignored}} expected-warning {{valid begin assumes clauses start with 'ext_', 'absent', 'contains', 'holds', 'no_openmp', 'no_openmp_routines', 'no_openmp_constructs', 'no_parallelism'; tokens will be ignored}} expected-note {{the ignored tokens spans until here}} expected-note {{the ignored tokens spans until here}}
+#pragma omp assumes foobar(foo 2 baz) no_openmp bazbaz(foo 2 baz) // expected-warning {{valid assumes clauses start with 'ext_', 'absent', 'contains', 'holds', 'no_openmp', 'no_openmp_routines', 'no_openmp_constructs', 'no_parallelism'; tokens will be ignored}} expected-warning {{valid assumes clauses start with 'ext_', 'absent', 'contains', 'holds', 'no_openmp', 'no_openmp_routines', 'no_openmp_constructs', 'no_parallelism'; tokens will be ignored}} expected-note {{the ignored tokens spans until here}} expected-note {{the ignored tokens spans until here}}
 #pragma omp end assumes
 
-#pragma omp begin assumes foobar(foo (2) baz) no_openmp bazbaz(foo (2)) baz) // expected-warning {{valid begin assumes clauses start with 'ext_', 'absent', 'contains', 'holds', 'no_openmp', 'no_openmp_routines', 'no_parallelism'; tokens will be ignored}} expected-warning {{valid begin assumes clauses start with 'ext_', 'absent', 'contains', 'holds', 'no_openmp', 'no_openmp_routines', 'no_parallelism'; tokens will be ignored}} expected-warning {{valid begin assumes clauses start with 'ext_', 'absent', 'contains', 'holds', 'no_openmp', 'no_openmp_routines', 'no_parallelism'; token will be ignored}} expected-warning {{valid begin assumes clauses start with 'ext_', 'absent', 'contains', 'holds', 'no_openmp', 'no_openmp_routines', 'no_parallelism'; token will be ignored}} expected-note {{the ignored tokens spans until here}} expected-note {{the ignored tokens spans until here}}
-#pragma omp assumes foobar(foo () baz) no_openmp bazbaz(foo ((2) baz) // expected-error {{expected ')'}} expected-warning {{valid assumes clauses start with 'ext_', 'absent', 'contains', 'holds', 'no_openmp', 'no_openmp_routines', 'no_parallelism'; tokens will be ignored}} expected-warning {{valid assumes clauses start with 'ext_', 'absent', 'contains', 'holds', 'no_openmp', 'no_openmp_routines', 'no_parallelism'; tokens will be ignored}} expected-note {{the ignored tokens spans until here}} expected-note {{to match this '('}}
+#pragma omp begin assumes foobar(foo (2) baz) no_openmp bazbaz(foo (2)) baz) // expected-warning {{valid begin assumes clauses start with 'ext_', 'absent', 'contains', 'holds', 'no_openmp', 'no_openmp_routines', 'no_openmp_constructs', 'no_parallelism'; tokens will be ignored}} expected-warning {{valid begin assumes clauses start with 'ext_', 'absent', 'contains', 'holds', 'no_openmp', 'no_openmp_routines', 'no_openmp_constructs', 'no_parallelism'; tokens will be ignored}} expected-warning {{valid begin assumes clauses start with 'ext_', 'absent', 'contains', 'holds', 'no_openmp', 'no_openmp_routines', 'no_openmp_constructs', 'no_parallelism'; token will be ignored}} expected-warning {{valid begin assumes clauses start with 'ext_', 'absent', 'contains', 'holds', 'no_openmp', 'no_openmp_routines', 'no_openmp_constructs', 'no_parallelism'; token will be ignored}} expected-note {{the ignored tokens spans until here}} expected-note {{the ignored tokens spans until here}}
+#pragma omp assumes foobar(foo () baz) no_openmp bazbaz(foo ((2) baz) // expected-error {{expected ')'}} expected-warning {{valid assumes clauses start with 'ext_', 'absent', 'contains', 'holds', 'no_openmp', 'no_openmp_routines', 'no_openmp_constructs', 'no_parallelism'; tokens will be ignored}} expected-warning {{valid assumes clauses start with 'ext_', 'absent', 'contains', 'holds', 'no_openmp', 'no_openmp_routines', 'no_openmp_constructs', 'no_parallelism'; tokens will be ignored}} expected-note {{the ignored tokens spans until here}} expected-note {{to match this '('}}
 #pragma omp end assumes
 
-#pragma omp assumes no_openmp foobar no_openmp // expected-warning {{valid assumes clauses start with 'ext_', 'absent', 'contains', 'holds', 'no_openmp', 'no_openmp_routines', 'no_parallelism'; token will be ignored}}
-#pragma omp begin assumes no_openmp foobar no_openmp // expected-warning {{valid begin assumes clauses start with 'ext_', 'absent', 'contains', 'holds', 'no_openmp', 'no_openmp_routines', 'no_parallelism'; token will be ignored}}
+#pragma omp assumes no_openmp foobar no_openmp // expected-warning {{valid assumes clauses start with 'ext_', 'absent', 'contains', 'holds', 'no_openmp', 'no_openmp_routines', 'no_openmp_constructs', 'no_parallelism'; token will be ignored}}
+#pragma omp begin assumes no_openmp foobar no_openmp // expected-warning {{valid begin assumes clauses start with 'ext_', 'absent', 'contains', 'holds', 'no_openmp', 'no_openmp_routines', 'no_openmp_constructs', 'no_parallelism'; token will be ignored}}
 #pragma omp end assumes
 
 #pragma omp assumes holds(1, 2 3)
@@ -55,8 +55,8 @@
 #pragma omp begin assumes contains(1, 2 3)
 #pragma omp end assumes
 
-#pragma omp assumes ext // expected-warning {{valid assumes clauses start with 'ext_', 'absent', 'contains', 'holds', 'no_openmp', 'no_openmp_routines', 'no_parallelism'; token will be ignored}}
-#pragma omp begin assumes ext // expected-warning {{valid begin assumes clauses start with 'ext_', 'absent', 'contains', 'holds', 'no_openmp', 'no_openmp_routines', 'no_parallelism'; token will be ignored}}
+#pragma omp assumes ext // expected-warning {{valid assumes clauses start with 'ext_', 'absent', 'contains', 'holds', 'no_openmp', 'no_openmp_routines', 'no_openmp_constructs', 'no_parallelism'; token will be ignored}}
+#pragma omp begin assumes ext // expected-warning {{valid begin assumes clauses start with 'ext_', 'absent', 'contains', 'holds', 'no_openmp', 'no_openmp_routines', 'no_openmp_constructs', 'no_parallelism'; token will be ignored}}
 #pragma omp end assumes
 
 #pragma omp assumes ext_123(not allowed) // expected-warning {{'ext_123' clause should not be followed by arguments; tokens will be ignored}} expected-note {{the ignored tokens spans until here}}
diff --git a/clang/test/OpenMP/assumes_messages_attr.c b/clang/test/OpenMP/assumes_messages_attr.c
index c9fc68ce65d35..d68767493a512 100644
--- a/clang/test/OpenMP/assumes_messages_attr.c
+++ b/clang/test/OpenMP/assumes_messages_attr.c
@@ -1,33 +1,33 @@
 // RUN: %clang_cc1 -triple=x86_64-pc-win32 -verify -fopenmp -std=c99 -fms-extensions -Wno-pragma-pack %s
 // RUN: %clang_cc1 -triple=x86_64-pc-win32 -verify -fopenmp-simd -std=c99 -fms-extensions -Wno-pragma-pack %s
 
-[[omp::directive(assumes)]]; // expected-error {{expected at least one 'ext_', 'absent', 'contains', 'holds', 'no_openmp', 'no_openmp_routines', 'no_parallelism' clause for '#pragma omp assumes'}}
+[[omp::directive(assumes)]]; // expected-error {{expected at least one 'ext_', 'absent', 'contains', 'holds', 'no_openmp', 'no_openmp_routines', 'no_openmp_constructs', 'no_parallelism' clause for '#pragma omp assumes'}}
 [[omp::directive(begin)]]; // expected-error {{expected an OpenMP directive}}
-[[omp::directive(begin assumes)]]; // expected-error {{expected at least one 'ext_', 'absent', 'contains', 'holds', 'no_openmp', 'no_openmp_routines', 'no_parallelism' clause for '#pragma omp begin assumes'}}
+[[omp::directive(begin assumes)]]; // expected-error {{expected at least one 'ext_', 'absent', 'contains', 'holds', 'no_openmp', 'no_openmp_routines', 'no_openmp_constructs', 'no_parallelism' clause for '#pragma omp begin assumes'}}
 [[omp::directive(end assumes)]];
 
-[[omp::directive(assumes foobar)]]; // expected-warning {{valid assumes clauses start with 'ext_', 'absent', 'contains', 'holds', 'no_openmp', 'no_openmp_routines', 'no_parallelism'; token will be ignored}}
-[[omp::directive(begin assumes foobar)]]; // expected-warning {{valid begin assumes clauses start with 'ext_', 'absent', 'contains', 'holds', 'no_openmp', 'no_openmp_routines', 'no_parallelism'; token will be ignored}}
+[[omp::directive(assumes foobar)]]; // expected-warning {{valid assumes clauses start with 'ext_', 'absent', 'contains', 'holds', 'no_openmp', 'no_openmp_routines', 'no_openmp_constructs', 'no_parallelism'; token will be ignored}}
+[[omp::directive(begin assumes foobar)]]; // expected-warning {{valid begin assumes clauses start with 'ext_', 'absent', 'contains', 'holds', 'no_openmp', 'no_openmp_routines', 'no_openmp_constructs', 'no_parallelism'; token will be ignored}}
 [[omp::directive(end assumes)]];
 
-[[omp::directive(begin assumes foobar(foo 2 baz))]]; // expected-warning {{valid begin assumes clauses start with 'ext_', 'absent', 'contains', 'holds', 'no_openmp', 'no_openmp_routines', 'no_parallelism'; tokens will be ignored}} expected-note {{the ignored tokens spans until here}}
-[[omp::directive(assumes foobar(foo 2 baz))]]; // expected-warning {{valid assumes clauses start with 'ext_', 'absent', 'contains', 'holds', 'no_openmp', 'no_openmp_routines', 'no_parallelism'; tokens will be ignored}} expected-note {{the ignored tokens spans until here}}
+[[omp::directive(begin assumes foobar(foo 2 baz))]]; // expected-warning {{valid begin assumes clauses start with 'ext_', 'absent', 'contains', 'holds', 'no_openmp', 'no_openmp_routines', 'no_openmp_constructs', 'no_parallelism'; tokens will be ignored}} expected-note {{the ignored tokens spans until here}}
+[[omp::directive(assumes foobar(foo 2 baz))]]; // expected-warning {{valid assumes clauses start with 'ext_', 'absent', 'contains', 'holds', 'no_openmp', 'no_openmp_routines', 'no_openmp_constructs', 'no_parallelism'; tokens will be ignored}} expected-note {{the ignored tokens spans until here}}
 [[omp::directive(end assumes)]];
 
 [[omp::directive(assumes no_openmp(1))]]; // expected-warning {{'no_openmp' clause should not be followed by arguments; tokens will be ignored}} expected-note {{the ignored tokens spans until here}}
 [[omp::directive(begin assumes no_openmp(1 2 3))]]; // expected-warning {{'no_openmp' clause should not be followed by arguments; tokens will be ignored}} expected-note {{the ignored tokens spans until here}}
 [[omp::directive(end assumes no_openmp(1))]];
 
-[[omp::directive(assumes foobar no_openmp bazbaz)]]; // expected-warning {{valid assumes clauses start with 'ext_', 'absent', 'contains', 'holds', 'no_openmp', 'no_openmp_routines', 'no_parallelism'; token will be ignored}} expected-warning {{valid assumes clauses start with 'ext_', 'absent', 'contains', 'holds', 'no_openmp', 'no_openmp_routines', 'no_parallelism'; token will be ignored}}
-[[omp::directive(begin assumes foobar no_openmp bazbaz)]]; // expected-warning {{valid begin assumes clauses start with 'ext_', 'absent', 'contains', 'holds', 'no_openmp', 'no_openmp_routines', 'no_parallelism'; token will be ignored}} expected-warning {{valid begin assumes clauses start with 'ext_', 'absent', 'contains', 'holds', 'no_openmp', 'no_openmp_routines', 'no_parallelism'; token will be ignored}}
+[[omp::directive(assumes foobar no_openmp bazbaz)]]; // expected-warning {{valid assumes clauses start with 'ext_', 'absent', 'contains', 'holds', 'no_openmp', 'no_openmp_routines', 'no_openmp_constructs', 'no_parallelism'; token will be ignored}} expected-warning {{valid assumes clauses start with 'ext_', 'absent', 'contains', 'holds', 'no_openmp', 'no_openmp_routines', 'no_openmp_constructs', 'no_parallelism'; token will be ignored}}
+[[omp::directive(begin assumes foobar no_openmp bazbaz)]]; // expected-warning {{valid begin assumes clauses start with 'ext_', 'absent', 'contains', 'holds', 'no_openmp', 'no_openmp_routines', 'no_openmp_constructs', 'no_parallelism'; token will be ignored}} expected-warning {{valid begin assumes clauses start with 'ext_', 'absent', 'contains', 'holds', 'no_openmp', 'no_openmp_routines', 'no_openmp_constructs', 'no_parallelism'; token will be ignored}}
 [[omp::directive(end assumes)]];
 
-[[omp::directive(begin assumes foobar(foo 2 baz) no_openmp bazbaz(foo 2 baz))]]; // expected-warning {{valid begin assumes clauses start with 'ext_', 'absent', 'contains', 'holds', 'no_openmp', 'no_openmp_routines', 'no_parallelism'; tokens will be ignored}} expected-warning {{valid begin assumes clauses start with 'ext_', 'absent', 'contains', 'holds', 'no_openmp', 'no_openmp_routines', 'no_parallelism'; tokens will be ignored}} expected-note {{the ignored tokens spans until here}} expected-note {{the ignored tokens spans until here}}
-[[omp::directive(assumes foobar(foo 2 baz) no_openmp bazbaz(foo 2 baz))]]; // expected-warning {{valid assumes clauses start with 'ext_', 'absent', 'contains', 'holds', 'no_openmp', 'no_openmp_routines', 'no_parallelism'; tokens will be ignored}} expected-warning {{valid assumes clauses start with 'ext_', 'absent', 'contains', 'holds', 'no_openmp', 'no_openmp_routines', 'no_parallelism'; tokens will be ignored}} expected-note {{the ignored tokens spans until here}} expected-note {{the ignored tokens spans until here}}
+[[omp::directive(begin assumes foobar(foo 2 baz) no_openmp bazbaz(foo 2 baz))]]; // expected-warning {{valid begin assumes clauses start with 'ext_', 'absent', 'contains', 'holds', 'no_openmp', 'no_openmp_routines', 'no_openmp_constructs', 'no_parallelism'; tokens will be ignored}} expected-warning {{valid begin assumes clauses start with 'ext_', 'absent', 'contains', 'holds', 'no_openmp', 'no_openmp_routines', 'no_openmp_constructs', 'no_parallelism'; tokens will be ignored}} expected-note {{the ignored tokens spans until here}} expected-note {{the ignored tokens spans until here}}
+[[omp::directive(assumes foobar(foo 2 baz) no_openmp bazbaz(foo 2 baz))]]; // expected-warning {{valid assumes clauses start with 'ext_', 'absent', 'contains', 'holds', 'no_openmp', 'no_openmp_routines', 'no_openmp_constructs', 'no_parallelism'; tokens will be ignored}} expected-warning {{valid assumes clauses start with 'ext_', 'absent', 'contains', 'holds', 'no_openmp', 'no_openmp_routines', 'no_openmp_constructs', 'no_parallelism'; tokens will be ignored}} expected-note {{the ignored tokens spans until here}} expected-note {{the ignored tokens spans until here}}
 [[omp::directive(end assumes)]];
 
-[[omp::directive(assumes no_openmp foobar no_openmp)]]; // expected-warning {{valid assumes clauses start with 'ext_', 'absent', 'contains', 'holds', 'no_openmp', 'no_openmp_routines', 'no_parallelism'; token will be ignored}}
-[[omp::directive(begin assumes no_openmp foobar no_openmp)]]; // expected-warning {{valid begin assumes clauses start with 'ext_', 'absent', 'contains', 'holds', 'no_openmp', 'no_openmp_routines', 'no_parallelism'; token will be ignored}}
+[[omp::directive(assumes no_openmp foobar no_openmp)]]; // expected-warning {{valid assumes clauses start with 'ext_', 'absent', 'contains', 'holds', 'no_openmp', 'no_openmp_routines', 'no_openmp_constructs', 'no_parallelism'; token will be ignored}}
+[[omp::directive(begin assumes no_openmp foobar no_openmp)]]; // expected-warning {{valid begin assumes clauses start with 'ext_', 'absent', 'contains', 'holds', 'no_openmp', 'no_openmp_routines', 'no_openmp_constructs', 'no_parallelism'; token will be ignored}}
 [[omp::directive(end assumes)]];
 
 [[omp::directive(assumes holds(1, 2 3))]];
@@ -42,8 +42,8 @@
 [[omp::directive(begin assumes contains(1, 2 3))]];
 [[omp::directive(end assumes)]];
 
-[[omp::directive(assumes ext)]]; // expected-warning {{valid assumes clauses start with 'ext_', 'absent', 'contains', 'holds', 'no_openmp', 'no_openmp_routines', 'no_parallelism'; token will be ignored}}
-[[omp::directive(begin assumes ext)]]; // expected-warning {{valid begin assumes clauses start with 'ext_', 'absent', 'contains', 'holds', 'no_openmp', 'no_openmp_routines', 'no_parallelism'; token will be ignored}}
+[[omp::directive(assumes ext)]]; // expected-warning {{valid assumes clauses start with 'ext_', 'absent', 'contains', 'holds', 'no_openmp', 'no_openmp_routines', 'no_openmp_constructs', 'no_parallelism'; token will be ignored}}
+[[omp::directive(begin assumes ext)]]; // expected-warning {{valid begin assumes clauses start with 'ext_', 'absent', 'contains', 'holds', 'no_openmp', 'no_openmp_routines', 'no_openmp_constructs', 'no_parallelism'; token will be ignored}}
 [[omp::directive(end assumes)]];
 
 [[omp::directive(assumes ext_123(not allowed))]]; // expected-warning {{'ext_123' clause should not be followed by arguments; tokens will be ignored}} expected-note {{the ignored tokens spans until here}}
diff --git a/clang/test/OpenMP/assumes_print.cpp b/clang/test/OpenMP/assumes_print.cpp
index 9254c29ab8335..25796ae8afcf5 100644
--- a/clang/test/OpenMP/assumes_print.cpp
+++ b/clang/test/OpenMP/assumes_print.cpp
@@ -1,10 +1,10 @@
-// RUN: %clang_cc1 -verify -fopenmp -ast-print %s | FileCheck %s
-// RUN: %clang_cc1 -fopenmp -x c++ -std=c++11 -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp -std=c++11 -include-pch %t -verify %s -ast-print | FileCheck %s
+// RUN: %clang_cc1 -verify -fopenmp -fopenmp-version=60 -ast-print %s | FileCheck %s
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=60 -x c++ -std=c++11 -emit-pch -o %t %s
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=60 -std=c++11 -include-pch %t -verify %s -ast-print | FileCheck %s
 
-// RUN: %clang_cc1 -verify -fopenmp-simd -ast-print %s | FileCheck %s
-// RUN: %clang_cc1 -fopenmp-simd -x c++ -std=c++11 -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp-simd -std=c++11 -include-pch %t -verify %s -ast-print | FileCheck %s
+// RUN: %clang_cc1 -verify -fopenmp-simd -fopenmp-version=60 -ast-print %s | FileCheck %s
+// RUN: %clang_cc1 -fopenmp-simd -fopenmp-version=60 -x c++ -std=c++11 -emit-pch -o %t %s
+// RUN: %clang_cc1 -fopenmp-simd -fopenmp-version=60 -std=c++11 -include-pch %t -verify %s -ast-print | FileCheck %s
 // expected-no-diagnostics
 
 #ifndef HEADER
@@ -14,6 +14,7 @@ void foo() {
 }
 
 #pragma omp assumes no_openmp_routines
+#pragma omp assumes no_openmp_constructs
 
 namespace inner {
   #pragma omp assumes no_openmp
@@ -37,8 +38,8 @@ void baz() {
 }
 #pragma omp end assumes
 
-// CHECK{LITERAL}: void foo() [[omp::assume("omp_no_openmp_routines")]] [[omp::assume("omp_no_openmp")]]
-// CHECK{LITERAL}: [[omp::assume("ompx_range_bar_only")]] [[omp::assume("ompx_range_bar_only_2")]] [[omp::assume("omp_no_openmp_routines")]] [[omp::assume("omp_no_openmp")]] void bar()
-// CHECK{LITERAL}: [[omp::assume("ompx_1234")]] [[omp::assume("omp_no_openmp_routines")]] [[omp::assume("omp_no_openmp")]] void baz()
+// CHECK{LITERAL}: void foo() [[omp::assume("omp_no_openmp_routines")]] [[omp::assume("omp_no_openmp_constructs")]] [[omp::assume("omp_no_openmp")]]
+// CHECK{LITERAL}: [[omp::assume("ompx_range_bar_only")]] [[omp::assume("ompx_range_bar_only_2")]] [[omp::assume("omp_no_openmp_routines")]] [[omp::assume("omp_no_openmp_constructs")]] [[omp::assume("omp_no_openmp")]] void bar()
+// CHECK{LITERAL}: [[omp::assume("ompx_1234")]] [[omp::assume("omp_no_openmp_routines")]] [[omp::assume("omp_no_openmp_constructs")]] [[omp::assume("omp_no_openmp")]] void baz()
 
 #endif
diff --git a/clang/test/OpenMP/attr-assume.cpp b/clang/test/OpenMP/attr-assume.cpp
index 09c22f98d1e29..2d807d9a00b0d 100644
--- a/clang/test/OpenMP/attr-assume.cpp
+++ b/clang/test/OpenMP/attr-assume.cpp
@@ -1,4 +1,4 @@
-// RUN: %clang_cc1 -fsyntax-only -fopenmp -verify %s
+// RUN: %clang_cc1 -fsyntax-only -fopenmp -fopenmp-version=60 -verify %s
 [[omp::assume(3)]] void f1(); // expected-error {{expected string literal as argument of 'assume' attribute}}
 [[omp::assume(int)]] void f2(); // expected-error {{expected string literal as argument of 'assume' attribute}}
 [[omp::assume(for)]] void f3(); // expected-error {{expected string literal as argument of 'assume' attribute}}
@@ -8,6 +8,7 @@
 [[omp::assume("omp_no_openmp_routine")]] void f7(); // expected-warning {{unknown assumption string 'omp_no_openmp_routine' may be misspelled; attribute is potentially ignored, did you mean 'omp_no_openmp_routines'?}}
 [[omp::assume("omp_no_openmp1")]] void f8(); // expected-warning {{unknown assumption string 'omp_no_openmp1' may be misspelled; attribute is potentially ignored, did you mean 'omp_no_openmp'?}}
 [[omp::assume("omp_no_openmp", "omp_no_openmp")]] void f9(); // expected-error {{'assume' attribute takes one argument}}
+[[omp::assume("omp_no_openmp_construct")]] void f10(); // expected-warning {{unknown assumption string 'omp_no_openmp_construct' may be misspelled; attribute is potentially ignored, did you mean 'omp_no_openmp_constructs'?}}
 
 [[omp::assume(3)]] int g1; // expected-error {{expected string literal as argument of 'assume' attribute}}
 [[omp::assume("omp_no_openmp")]] int g2; // expected-warning {{'assume' attribute only applies to functions and Objective-C methods}}
diff --git a/clang/test/OpenMP/barrier_codegen.cpp b/clang/test/OpenMP/barrier_codegen.cpp
index 79e4076846526..e1b50793169d0 100644
--- a/clang/test/OpenMP/barrier_codegen.cpp
+++ b/clang/test/OpenMP/barrier_codegen.cpp
@@ -46,7 +46,7 @@ int main(int argc, char **argv) {
 // IRBUILDER:          ; Function Attrs: nounwind
 // IRBUILDER-NEXT:     declare i32 @__kmpc_global_thread_num(ptr) #
 // IRBUILDER_OPT:      ; Function Attrs: nofree nosync nounwind willreturn memory(argmem: read, inaccessiblemem: read)
-// IRBUILDER_OPT-NEXT: declare i32 @__kmpc_global_thread_num(ptr nocapture nofree readonly) #
+// IRBUILDER_OPT-NEXT: declare i32 @__kmpc_global_thread_num(ptr nofree readonly captures(none)) #
 
 // CHECK: define {{.+}} [[TMAIN_INT]](
 // CHECK: [[GTID:%.+]] = call i32 @__kmpc_global_thread_num(ptr [[LOC]])
diff --git a/clang/test/OpenMP/begin_declare_variant_messages.c b/clang/test/OpenMP/begin_declare_variant_messages.c
index ffa56002f1953..f87714a47dce0 100644
--- a/clang/test/OpenMP/begin_declare_variant_messages.c
+++ b/clang/test/OpenMP/begin_declare_variant_messages.c
@@ -32,27 +32,27 @@ const int var;
 #pragma omp end declare variant
 #pragma omp begin declare variant match // expected-error {{expected '(' after 'match'}}
 #pragma omp end declare variant
-#pragma omp begin declare variant match( // expected-error {{expected ')'}} expected-warning {{expected identifier or string literal describing a context set; set skipped}} expected-note {{context set options are: 'construct' 'device' 'implementation' 'user'}} expected-note {{the ignored set spans until here}} expected-note {{to match this '('}}
+#pragma omp begin declare variant match( // expected-error {{expected ')'}} expected-warning {{expected identifier or string literal describing a context set; set skipped}} expected-note {{context set options are: 'construct' 'device' 'target_device' 'implementation' 'user'}} expected-note {{the ignored set spans until here}} expected-note {{to match this '('}}
 #pragma omp end declare variant
-#pragma omp begin declare variant match() // expected-warning {{expected identifier or string literal describing a context set; set skipped}} expected-note {{context set options are: 'construct' 'device' 'implementation' 'user'}} expected-note {{the ignored set spans until here}}
+#pragma omp begin declare variant match() // expected-warning {{expected identifier or string literal describing a context set; set skipped}} expected-note {{context set options are: 'construct' 'device' 'target_device' 'implementation' 'user'}} expected-note {{the ignored set spans until here}}
 #pragma omp end declare variant
-#pragma omp begin declare variant match(xxx) // expected-warning {{'xxx' is not a valid context set in a `declare variant`; set ignored}} expected-note {{context set options are: 'construct' 'device' 'implementation' 'user'}} expected-note {{the ignored set spans until here}}
+#pragma omp begin declare variant match(xxx) // expected-warning {{'xxx' is not a valid context set in a `declare variant`; set ignored}} expected-note {{context set options are: 'construct' 'device' 'target_device' 'implementation' 'user'}} expected-note {{the ignored set spans until here}}
 #pragma omp end declare variant
-#pragma omp begin declare variant match(xxx=) // expected-warning {{'xxx' is not a valid context set in a `declare variant`; set ignored}} expected-note {{context set options are: 'construct' 'device' 'implementation' 'user'}} expected-note {{the ignored set spans until here}}
+#pragma omp begin declare variant match(xxx=) // expected-warning {{'xxx' is not a valid context set in a `declare variant`; set ignored}} expected-note {{context set options are: 'construct' 'device' 'target_device' 'implementation' 'user'}} expected-note {{the ignored set spans until here}}
 #pragma omp end declare variant
-#pragma omp begin declare variant match(xxx=yyy) // expected-warning {{'xxx' is not a valid context set in a `declare variant`; set ignored}} expected-note {{context set options are: 'construct' 'device' 'implementation' 'user'}} expected-note {{the ignored set spans until here}}
+#pragma omp begin declare variant match(xxx=yyy) // expected-warning {{'xxx' is not a valid context set in a `declare variant`; set ignored}} expected-note {{context set options are: 'construct' 'device' 'target_device' 'implementation' 'user'}} expected-note {{the ignored set spans until here}}
 #pragma omp end declare variant
-#pragma omp begin declare variant match(xxx=yyy}) // expected-error {{expected ')'}} expected-warning {{'xxx' is not a valid context set in a `declare variant`; set ignored}} expected-warning {{extra tokens at the end of '#pragma omp begin declare variant' are ignored}} expected-note {{context set options are: 'construct' 'device' 'implementation' 'user'}} expected-note {{the ignored set spans until here}} expected-note {{to match this '('}}
+#pragma omp begin declare variant match(xxx=yyy}) // expected-error {{expected ')'}} expected-warning {{'xxx' is not a valid context set in a `declare variant`; set ignored}} expected-warning {{extra tokens at the end of '#pragma omp begin declare variant' are ignored}} expected-note {{context set options are: 'construct' 'device' 'target_device' 'implementation' 'user'}} expected-note {{the ignored set spans until here}} expected-note {{to match this '('}}
 #pragma omp end declare variant
-#pragma omp begin declare variant match(xxx={) // expected-error {{expected ')'}} expected-warning {{'xxx' is not a valid context set in a `declare variant`; set ignored}} expected-note {{context set options are: 'construct' 'device' 'implementation' 'user'}} expected-note {{the ignored set spans until here}} expected-note {{to match this '('}}
+#pragma omp begin declare variant match(xxx={) // expected-error {{expected ')'}} expected-warning {{'xxx' is not a valid context set in a `declare variant`; set ignored}} expected-note {{context set options are: 'construct' 'device' 'target_device' 'implementation' 'user'}} expected-note {{the ignored set spans until here}} expected-note {{to match this '('}}
 #pragma omp end declare variant
-#pragma omp begin declare variant match(xxx={}) // expected-warning {{'xxx' is not a valid context set in a `declare variant`; set ignored}} expected-note {{context set options are: 'construct' 'device' 'implementation' 'user'}} expected-note {{the ignored set spans until here}}
+#pragma omp begin declare variant match(xxx={}) // expected-warning {{'xxx' is not a valid context set in a `declare variant`; set ignored}} expected-note {{context set options are: 'construct' 'device' 'target_device' 'implementation' 'user'}} expected-note {{the ignored set spans until here}}
 #pragma omp end declare variant
-#pragma omp begin declare variant match(xxx={vvv, vvv}) // expected-warning {{'xxx' is not a valid context set in a `declare variant`; set ignored}} expected-note {{context set options are: 'construct' 'device' 'implementation' 'user'}} expected-note {{the ignored set spans until here}}
+#pragma omp begin declare variant match(xxx={vvv, vvv}) // expected-warning {{'xxx' is not a valid context set in a `declare variant`; set ignored}} expected-note {{context set options are: 'construct' 'device' 'target_device' 'implementation' 'user'}} expected-note {{the ignored set spans until here}}
 #pragma omp end declare variant
-#pragma omp begin declare variant match(xxx={vvv} xxx) // expected-warning {{'xxx' is not a valid context set in a `declare variant`; set ignored}} expected-note {{context set options are: 'construct' 'device' 'implementation' 'user'}} expected-note {{the ignored set spans until here}}
+#pragma omp begin declare variant match(xxx={vvv} xxx) // expected-warning {{'xxx' is not a valid context set in a `declare variant`; set ignored}} expected-note {{context set options are: 'construct' 'device' 'target_device' 'implementation' 'user'}} expected-note {{the ignored set spans until here}}
 #pragma omp end declare variant
-#pragma omp begin declare variant match(xxx={vvv}) xxx // expected-warning {{'xxx' is not a valid context set in a `declare variant`; set ignored}} expected-warning {{extra tokens at the end of '#pragma omp begin declare variant' are ignored}} expected-note {{context set options are: 'construct' 'device' 'implementation' 'user'}} expected-note {{the ignored set spans until here}}
+#pragma omp begin declare variant match(xxx={vvv}) xxx // expected-warning {{'xxx' is not a valid context set in a `declare variant`; set ignored}} expected-warning {{extra tokens at the end of '#pragma omp begin declare variant' are ignored}} expected-note {{context set options are: 'construct' 'device' 'target_device' 'implementation' 'user'}} expected-note {{the ignored set spans until here}}
 #pragma omp end declare variant
 #pragma omp begin declare variant match(implementation={xxx}) // expected-warning {{'xxx' is not a valid context selector for the context set 'implementation'; selector ignored}} expected-note {{context selector options are: 'vendor' 'extension' 'unified_address' 'unified_shared_memory' 'reverse_offload' 'dynamic_allocators' 'atomic_default_mem_order'}} expected-note {{the ignored selector spans until here}}
 #pragma omp end declare variant
diff --git a/clang/test/OpenMP/bug54082.c b/clang/test/OpenMP/bug54082.c
index 337c120983e0a..da32be2867411 100644
--- a/clang/test/OpenMP/bug54082.c
+++ b/clang/test/OpenMP/bug54082.c
@@ -80,28 +80,28 @@ void foo() {
 //
 //
 // CHECK-LABEL: define {{[^@]+}}@foo.omp_outlined
-// CHECK-SAME: (ptr noalias nocapture noundef readonly [[DOTGLOBAL_TID_:%.*]], ptr noalias nocapture readnone [[DOTBOUND_TID_:%.*]], ptr nocapture noundef nonnull readonly align 8 dereferenceable(8) [[X_ALLOC:%.*]]) #[[ATTR4:[0-9]+]] {
+// CHECK-SAME: (ptr noalias noundef readonly captures(none) [[DOTGLOBAL_TID_:%.*]], ptr noalias readnone captures(none) [[DOTBOUND_TID_:%.*]], ptr noundef nonnull readonly align 8 captures(none) dereferenceable(8) [[X_ALLOC:%.*]]) #[[ATTR4:[0-9]+]] {
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[DOTOMP_LB:%.*]] = alloca i32, align 4
 // CHECK-NEXT:    [[DOTOMP_UB:%.*]] = alloca i32, align 4
 // CHECK-NEXT:    [[DOTOMP_STRIDE:%.*]] = alloca i32, align 4
 // CHECK-NEXT:    [[DOTOMP_IS_LAST:%.*]] = alloca i32, align 4
 // CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr nonnull [[DOTOMP_LB]]) #[[ATTR5]]
-// CHECK-NEXT:    store i32 0, ptr [[DOTOMP_LB]], align 4, !tbaa [[TBAA6:![0-9]+]]
+// CHECK-NEXT:    store i32 0, ptr [[DOTOMP_LB]], align 4, !tbaa [[TBAA7:![0-9]+]]
 // CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr nonnull [[DOTOMP_UB]]) #[[ATTR5]]
-// CHECK-NEXT:    store i32 1023, ptr [[DOTOMP_UB]], align 4, !tbaa [[TBAA6]]
+// CHECK-NEXT:    store i32 1023, ptr [[DOTOMP_UB]], align 4, !tbaa [[TBAA7]]
 // CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr nonnull [[DOTOMP_STRIDE]]) #[[ATTR5]]
-// CHECK-NEXT:    store i32 1, ptr [[DOTOMP_STRIDE]], align 4, !tbaa [[TBAA6]]
+// CHECK-NEXT:    store i32 1, ptr [[DOTOMP_STRIDE]], align 4, !tbaa [[TBAA7]]
 // CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr nonnull [[DOTOMP_IS_LAST]]) #[[ATTR5]]
-// CHECK-NEXT:    store i32 0, ptr [[DOTOMP_IS_LAST]], align 4, !tbaa [[TBAA6]]
-// CHECK-NEXT:    [[TMP0:%.*]] = load i32, ptr [[DOTGLOBAL_TID_]], align 4, !tbaa [[TBAA6]]
+// CHECK-NEXT:    store i32 0, ptr [[DOTOMP_IS_LAST]], align 4, !tbaa [[TBAA7]]
+// CHECK-NEXT:    [[TMP0:%.*]] = load i32, ptr [[DOTGLOBAL_TID_]], align 4, !tbaa [[TBAA7]]
 // CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr [[X_ALLOC]], align 8, !tbaa [[TBAA3]]
 // CHECK-NEXT:    [[CONV:%.*]] = inttoptr i64 [[TMP1]] to ptr
 // CHECK-NEXT:    [[DOTX__VOID_ADDR:%.*]] = tail call ptr @__kmpc_alloc(i32 [[TMP0]], i64 8, ptr [[CONV]])
 // CHECK-NEXT:    call void @__kmpc_for_static_init_4(ptr nonnull @[[GLOB1:[0-9]+]], i32 [[TMP0]], i32 34, ptr nonnull [[DOTOMP_IS_LAST]], ptr nonnull [[DOTOMP_LB]], ptr nonnull [[DOTOMP_UB]], ptr nonnull [[DOTOMP_STRIDE]], i32 1, i32 1)
-// CHECK-NEXT:    [[TMP2:%.*]] = load i32, ptr [[DOTOMP_UB]], align 4
+// CHECK-NEXT:    [[TMP2:%.*]] = load i32, ptr [[DOTOMP_UB]], align 4, !tbaa [[TBAA7]]
 // CHECK-NEXT:    [[COND:%.*]] = call i32 @llvm.smin.i32(i32 [[TMP2]], i32 1023)
-// CHECK-NEXT:    store i32 [[COND]], ptr [[DOTOMP_UB]], align 4, !tbaa [[TBAA6]]
+// CHECK-NEXT:    store i32 [[COND]], ptr [[DOTOMP_UB]], align 4, !tbaa [[TBAA7]]
 // CHECK-NEXT:    call void @__kmpc_for_static_fini(ptr nonnull @[[GLOB1]], i32 [[TMP0]])
 // CHECK-NEXT:    [[TMP3:%.*]] = load i64, ptr [[X_ALLOC]], align 8, !tbaa [[TBAA3]]
 // CHECK-NEXT:    [[CONV5:%.*]] = inttoptr i64 [[TMP3]] to ptr
diff --git a/clang/test/OpenMP/declare_mapper_codegen.cpp b/clang/test/OpenMP/declare_mapper_codegen.cpp
index f9da3d97766d9..81453223b2a27 100644
--- a/clang/test/OpenMP/declare_mapper_codegen.cpp
+++ b/clang/test/OpenMP/declare_mapper_codegen.cpp
@@ -23,7 +23,7 @@
 // Mapper function code generation and runtime interface.
 
 // CK0: [[IDENT_T:%.+]] = type { i32, i32, i32, i32, ptr }
-// CK0: [[ENTRY:%.+]] = type { ptr, ptr, i[[SZ:32|64]], i32, i32 }
+// CK0: [[ENTRY:%.+]] = type { i64, i16, i16, i32, ptr, ptr, i64, i64, ptr }
 // CK0: [[ANON_T:%.+]] = type { ptr }
 // CK0: [[ANON_T_0:%.+]] = type { ptr }
 // CK0: [[KMP_TASK_T_WITH_PRIVATES:%.+]] = type { [[KMP_TASK_T:%[^,]+]], [[KMP_PRIVATES_T:%.+]] }
@@ -235,7 +235,7 @@ void foo(int a){
 // CK0-DAG: [[PGEP]] = getelementptr inbounds {{.+}}[[PS:%[^,]+]], i32 0, i32 0
 // CK0-DAG: [[BP1:%.+]] = getelementptr inbounds {{.+}}[[BPS]], i32 0, i32 0
 // CK0-DAG: [[P1:%.+]] = getelementptr inbounds {{.+}}[[PS]], i32 0, i32 0
-// CK0-DAG: [[MPR1:%.+]] = getelementptr inbounds {{.+}}[[MPR]], i[[sz]] 0, i[[sz]] 0
+// CK0-DAG: [[MPR1:%.+]] = getelementptr inbounds {{.+}}[[MPR]], i[[sz:32|64]] 0, i[[sz]] 0
 // CK0-DAG: store ptr [[VAL:%[^,]+]], ptr [[BP1]]
 // CK0-DAG: store ptr [[VAL]], ptr [[P1]]
 // CK0-DAG: store ptr [[MPRFUNC]], ptr [[MPR1]]
@@ -250,7 +250,7 @@ void foo(int a){
   // CK0: store ptr [[CADDR:%[^,]+]], ptr [[BP2GEP]], align
   // CK0: [[P2GEP:%.+]] = getelementptr inbounds [1 x ptr], ptr [[OFFLOAD_P2:%[^,]+]], i32 0, i32 0
   // CK0: store ptr [[CADDR]], ptr [[P2GEP]], align
-  // CK0: [[MAPPER2GEP:%.+]] = getelementptr inbounds [1 x ptr], ptr [[OFFLOAD_MAPPER2:%[^,]+]], i[[SZ]] 0, i[[SZ]] 0
+  // CK0: [[MAPPER2GEP:%.+]] = getelementptr inbounds [1 x ptr], ptr [[OFFLOAD_MAPPER2:%[^,]+]], i[[SZ:32|64]] 0, i[[SZ]] 0
   // CK0: store ptr [[MPRFUNC]], ptr [[MAPPER2GEP]], align
   // CK0: [[BP2:%.+]] = getelementptr inbounds [1 x ptr], ptr [[OFFLOAD_BP2]], i32 0, i32 0
   // CK0: [[P2:%.+]] = getelementptr inbounds [1 x ptr], ptr [[OFFLOAD_P2]], i32 0, i32 0
diff --git a/clang/test/OpenMP/declare_mapper_messages.c b/clang/test/OpenMP/declare_mapper_messages.c
index 288caca097648..2238689227311 100644
--- a/clang/test/OpenMP/declare_mapper_messages.c
+++ b/clang/test/OpenMP/declare_mapper_messages.c
@@ -1,10 +1,12 @@
 // RUN: %clang_cc1 -verify=omp50,expected -fopenmp -fopenmp-version=50 -ferror-limit 100 -DOMP50 %s
 // RUN: %clang_cc1 -verify=omp51,expected -fopenmp -ferror-limit 100 %s
 // RUN: %clang_cc1 -verify=expected,omp52 -fopenmp -fopenmp-version=52 -ferror-limit 100 -DOMP52 %s
+// RUN: %clang_cc1 -verify=expected,omp60 -fopenmp -fopenmp-version=60 -ferror-limit 100 -DOMP60 %s
 
 // RUN: %clang_cc1 -verify=omp50,expected -fopenmp-simd -fopenmp-version=50 -ferror-limit 100 -DOMP50 %s
 // RUN: %clang_cc1 -verify=omp51-simd,expected -fopenmp-simd -ferror-limit 100 %s
 // RUN: %clang_cc1 -verify=expected,omp52 -fopenmp-simd -fopenmp-version=52 -ferror-limit 100 -DOMP52 %s
+// RUN: %clang_cc1 -verify=expected,omp60-simd -fopenmp-simd -fopenmp-version=60 -ferror-limit 100 -DOMP60 %s
 
 int temp; // expected-note {{'temp' declared here}}
 
@@ -32,11 +34,11 @@ struct vec {                                                            // expec
 #pragma omp declare mapper(struct vec v) map(v.len)                     // expected-error {{redefinition of user-defined mapper for type 'struct vec' with name 'default'}}
 #pragma omp declare mapper(int v) map(v)                                // expected-error {{mapper type must be of struct, union or class type}}
 
-#ifndef OMP52
-// omp51-simd-error@+6 {{incorrect map type modifier, expected one of: 'always', 'close', 'mapper', 'present', 'ompx_hold'}} 
-// omp50-error@+5 {{incorrect map type modifier, expected one of: 'always', 'close', 'mapper', 'ompx_hold'}} 
-// omp51-error@+4 {{incorrect map type modifier, expected one of: 'always', 'close', 'mapper', 'present', 'ompx_hold'}} 
-// expected-error@+3 {{only variable 'vvec' is allowed in map clauses of this 'omp declare mapper' directive}} 
+#if !defined(OMP52) && !defined(OMP60)
+// omp51-simd-error@+6 {{incorrect map type modifier, expected one of: 'always', 'close', 'mapper', 'present', 'ompx_hold'}}
+// omp50-error@+5 {{incorrect map type modifier, expected one of: 'always', 'close', 'mapper', 'ompx_hold'}}
+// omp51-error@+4 {{incorrect map type modifier, expected one of: 'always', 'close', 'mapper', 'present', 'ompx_hold'}}
+// expected-error@+3 {{only variable 'vvec' is allowed in map clauses of this 'omp declare mapper' directive}}
 // expected-error@+2 {{expected at least one clause on '#pragma omp declare mapper' directive}}
 // expected-note@+1 {{'it' declared here}}
 #pragma omp declare mapper(id2: struct vec vvec) map(iterator(it=0:vvec.len:2), tofrom:vvec.data[it]) 
@@ -68,15 +70,28 @@ int fun(int arg) {
       {}
 #pragma omp target map(mapper(aa :vv)                                   // expected-error {{use of undeclared identifier 'aa'}} expected-error {{expected ')'}} expected-error {{call to undeclared function 'mapper'}} expected-note {{to match this '('}}
       {}
+#ifndef OMP60
 #pragma omp target map(mapper(ab) :vv)                                  // expected-error {{missing map type}} expected-error {{cannot find a valid user-defined mapper for type 'struct vec' with name 'ab'}}
+#endif
       {}
+#ifndef OMP60
 #pragma omp target map(mapper(ab) :arr[0:2])                            // expected-error {{missing map type}} expected-error {{cannot find a valid user-defined mapper for type 'struct vec' with name 'ab'}}
+#endif
       {}
-#pragma omp target map(mapper(aa) :vv)                                  // expected-error {{missing map type}}
+#ifndef OMP60
+#pragma omp target map(mapper(aa) :vv)                                  // omp50-error {{missing map type}} omp51-error {{missing map type}} omp52-error {{missing map type}} omp51-simd-error {{missing map type}}
       {}
-#pragma omp target map(mapper(aa) to:d)                                 // expected-error {{mapper type must be of struct, union or class type}} omp52-error{{missing ',' after map type modifier}}
+#endif
+// expected-error@+4 {{mapper type must be of struct, union or class type}}
+// omp52-error@+3 {{missing ',' after map type modifier}}
+// omp60-error@+2 {{missing ',' after map type modifier}}
+// omp60-simd-error@+1 {{missing ',' after map type modifier}}
+#pragma omp target map(mapper(aa) to:d)
       {}
-#pragma omp target map(mapper(aa) to:vv) map(close mapper(aa) from:v1) map(mapper(aa) to:arr[0]) // omp52-error 4 {{missing ',' after map type modifier}}
+// omp52-error@+3 4 {{missing ',' after map type modifier}}
+// omp60-error@+2 4 {{missing ',' after map type modifier}}
+// omp60-simd-error@+1 4 {{missing ',' after map type modifier}}
+#pragma omp target map(mapper(aa) to:vv) map(close mapper(aa) from:v1) map(mapper(aa) to:arr[0])
       {}
 
 #pragma omp target update to(mapper)                                    // expected-error {{expected '(' after 'mapper'}} expected-error {{expected expression}} expected-error {{expected at least one 'to' clause or 'from' clause specified to '#pragma omp target update'}}
diff --git a/clang/test/OpenMP/declare_target_ast_print.cpp b/clang/test/OpenMP/declare_target_ast_print.cpp
index 3334b7491fab8..27d7a9fe21e52 100644
--- a/clang/test/OpenMP/declare_target_ast_print.cpp
+++ b/clang/test/OpenMP/declare_target_ast_print.cpp
@@ -4,6 +4,8 @@
 // RUN: %clang_cc1 -verify -fopenmp -fopenmp-version=50 -I %S/Inputs -ast-print %s | FileCheck %s --check-prefix=CHECK --check-prefix=OMP50
 // RUN: %clang_cc1 -verify -fopenmp -I %S/Inputs -ast-print %s | FileCheck %s --check-prefix=CHECK --check-prefix=OMP51
 // RUN: %clang_cc1 -verify -fopenmp -fopenmp-version=52 -I %S/Inputs -ast-print %s | FileCheck %s --check-prefix=CHECK --check-prefix=OMP52
+// RUN: %clang_cc1 -verify -fopenmp -fopenmp-version=60 -I %S/Inputs -ast-print %s | FileCheck %s --check-prefix=CHECK
+
 // RUN: %clang_cc1 -fopenmp -fopenmp-version=50 -x c++ -std=c++11 -I %S/Inputs -emit-pch -o %t %s
 // RUN: %clang_cc1 -fopenmp -fopenmp-version=50 -std=c++11 -include-pch %t -I %S/Inputs -verify %s -ast-print | FileCheck %s --check-prefix=CHECK --check-prefix=OMP50
 // RUN: %clang_cc1 -fopenmp -x c++ -std=c++11 -I %S/Inputs -emit-pch -o %t %s
@@ -229,7 +231,7 @@ void f1() {
 int b1, b2, b3;
 void f2() {
 }
-#if _OPENMP == 202111
+#if _OPENMP >= 202111
 #pragma omp declare target enter(b1) enter(b2), enter(b3, f2)
 #else
 #pragma omp declare target to(b1) to(b2), to(b3, f2)
@@ -336,7 +338,7 @@ int baz() { return 1; }
 
 #pragma omp declare target
 int abc1() { return 1; }
-#if _OPENMP == 202111
+#if _OPENMP >= 202111
 #pragma omp declare target enter(abc1) device_type(nohost)
 #else
 #pragma omp declare target to(abc1) device_type(nohost)
@@ -379,7 +381,7 @@ int main (int argc, char **argv) {
   baz<float>();
   baz<int>();
 
-#if _OPENMP == 202111
+#if _OPENMP >= 202111
 #pragma omp declare target enter(foo2)
 #else
 #pragma omp declare target to (foo2)
diff --git a/clang/test/OpenMP/declare_target_codegen.cpp b/clang/test/OpenMP/declare_target_codegen.cpp
index ba93772ede3e8..d2fd29dcfbc50 100644
--- a/clang/test/OpenMP/declare_target_codegen.cpp
+++ b/clang/test/OpenMP/declare_target_codegen.cpp
@@ -30,7 +30,7 @@
 // CHECK-DAG: @dx = {{protected | }}global i32 0,
 // CHECK-DAG: @dy = {{protected | }}global i32 0,
 // CHECK-DAG: @bbb = {{protected | }}global i32 0,
-// CHECK-DAG: weak constant %struct.__tgt_offload_entry { ptr @bbb,
+// CHECK-DAG: weak constant %struct.__tgt_offload_entry {
 // CHECK-DAG: @ccc = external global i32,
 // CHECK-DAG: @ddd = {{protected | }}global i32 0,
 // CHECK-DAG: @hhh_decl_tgt_ref_ptr = weak global ptr null
diff --git a/clang/test/OpenMP/declare_target_link_codegen.cpp b/clang/test/OpenMP/declare_target_link_codegen.cpp
index 189c9ac59c153..61e7bd8274fed 100644
--- a/clang/test/OpenMP/declare_target_link_codegen.cpp
+++ b/clang/test/OpenMP/declare_target_link_codegen.cpp
@@ -27,11 +27,11 @@
 // HOST: [[SIZES:@.+]] = private unnamed_addr constant [3 x i64] [i64 4, i64 4, i64 4]
 // HOST: [[MAPTYPES:@.+]] = private unnamed_addr constant [3 x i64] [i64 35, i64 531, i64 531]
 // HOST: @.offloading.entry_name{{.*}} = internal unnamed_addr constant [{{[0-9]+}} x i8] c"c_decl_tgt_ref_ptr\00"
-// HOST: @.offloading.entry.c_decl_tgt_ref_ptr = weak constant %struct.__tgt_offload_entry { ptr @c_decl_tgt_ref_ptr, ptr @.offloading.entry_name, i64 8, i32 1, i32 0 }, section "omp_offloading_entries", align 1
-// HOST-COFF: @.offloading.entry.{{.*}} = weak constant %struct.__tgt_offload_entry { ptr @.{{.*}}, ptr @.{{.*}}, i64 0, i32 0, i32 0 }, section "omp_offloading_entries$OE", align 1 
+// HOST: @.offloading.entry.c_decl_tgt_ref_ptr = weak constant %struct.__tgt_offload_entry { i64 0, i16 1, i16 1, i32 1, ptr @c_decl_tgt_ref_ptr, ptr @.offloading.entry_name, i64 8, i64 0, ptr null }, section "llvm_offload_entries"
+// HOST-COFF: @.offloading.entry.{{.*}} = weak constant %struct.__tgt_offload_entry { {{.*}} }, section "llvm_offload_entries$OE"
 // DEVICE-NOT: internal unnamed_addr constant [{{[0-9]+}} x i8] c"c_{{.*}}_decl_tgt_ref_ptr\00"
 // HOST: @.offloading.entry_name{{.*}} = internal unnamed_addr constant [{{[0-9]+}} x i8] c"_{{.*}}d_{{.*}}_decl_tgt_ref_ptr\00"
-// HOST: @.offloading.entry.[[D_PTR]] = weak constant %struct.__tgt_offload_entry { ptr @[[D_PTR]], ptr @.offloading.entry_name{{.*}}
+// HOST: @.offloading.entry.[[D_PTR]] = weak constant %struct.__tgt_offload_entry { i64 0, i16 1, i16 1, i32 1, ptr @[[D_PTR]], ptr @.offloading.entry_name.3, i64 8, i64 0, ptr null }, section "llvm_offload_entries"
 
 extern int c;
 #pragma omp declare target link(c)
diff --git a/clang/test/OpenMP/declare_target_messages.cpp b/clang/test/OpenMP/declare_target_messages.cpp
index de831f8575ee5..3c0e766cf72ca 100644
--- a/clang/test/OpenMP/declare_target_messages.cpp
+++ b/clang/test/OpenMP/declare_target_messages.cpp
@@ -1,46 +1,97 @@
-// RUN: %clang_cc1 -triple x86_64-apple-macos10.7.0 -verify=expected,omp45 -fopenmp -fopenmp-version=45 -fnoopenmp-use-tls -ferror-limit 100 -o - %s
-// RUN: %clang_cc1 -triple x86_64-apple-macos10.7.0 -verify=expected,omp5,host5 -fopenmp -fopenmp-version=50 -fopenmp-targets=x86_64-apple-macos10.7.0 -fnoopenmp-use-tls -ferror-limit 100 -o - %s
-// RUN: %clang_cc1 -triple x86_64-apple-macos10.7.0 -verify=expected,omp5,dev5 -fopenmp -fopenmp-version=50 -fopenmp-is-target-device -fopenmp-targets=x86_64-apple-macos10.7.0 -aux-triple x86_64-apple-macos10.7.0 -fnoopenmp-use-tls -ferror-limit 100 -o - %s
+// DEFINE: %{common_opts_mac} = -triple x86_64-apple-macos10.7.0
+// DEFINE: %{limit} = -fnoopenmp-use-tls -ferror-limit 100 
+// DEFINE: %{target_mac} = -fopenmp-targets=x86_64-apple-macos10.7.0
+// DEFINE: %{aux_triple} = -aux-triple x86_64-apple-macos10.7.0
+// DEFINE: %{openmp45} = -fopenmp -fopenmp-version=45
+// DEFINE: %{openmp50} = -fopenmp -fopenmp-version=50
+// DEFINE: %{openmp50_simd} = -fopenmp-simd -fopenmp-version=50
+// DEFINE: %{openmp52} = -fopenmp -fopenmp-version=52
+// DEFINE: %{openmp60} = -fopenmp -fopenmp-version=60
+// DEFINE: %{openmp60_simd} = -fopenmp-simd -fopenmp-version=60
+
+// RUN: %clang_cc1 %{common_opts_mac} -verify=expected,omp45,omp45-to-51,omp45-to-51-var,omp45-to-51-clause,omp45-to-51-clause  %{openmp45} %{limit} -o - %s
+// RUN: %clang_cc1 %{common_opts_mac} -verify=expected,omp5,ompvar,omp45-to-51,omp5-and-51,omp5-or-later,omp5-or-later-var,omp45-to-51-var,omp45-to-51-clause,host5,host-5-and-51,no-host5-and-51  %{openmp50} %{target_mac} %{limit} -o - %s
+// RUN: %clang_cc1 %{common_opts_mac} -verify=expected,omp52,ompvar,omp5-or-later,omp5-or-later-var  %{openmp60} %{target_mac} %{limit} -o - %s
+// RUN: %clang_cc1 %{common_opts_mac} -verify=expected,omp5,ompvar,omp45-to-51,omp5-and-51,omp5-or-later,omp5-or-later-var,omp45-to-51-var,omp45-to-51-clause,host-5-and-51,no-host5-and-51,dev5  %{openmp50} -fopenmp-is-target-device %{target_mac} %{aux_triple} %{limit} -o - %s
+// RUN: %clang_cc1 %{common_opts_mac} -verify=expected,omp52,ompvar,omp5-or-later,omp5-or-later-var %{openmp60} -fopenmp-is-target-device %{target_mac} %{aux_triple} %{limit} -o - %s
+
+// RUN: %clang_cc1 %{common_opts_mac} -verify=expected,omp5,ompvar,omp45-to-51,omp5-and-51,omp5-or-later,omp5-or-later-var,omp45-to-51-var,omp45-to-51-clause,host5,host-5-and-51,no-host5-and-51 %{openmp50_simd} %{target_mac} %{limit} -o - %s
+// RUN: %clang_cc1 %{common_opts_mac} -verify=expected,omp52,ompvar,omp5-or-later,omp5-or-later-var %{openmp60_simd} %{target_mac} %{limit} -o - %s
+// RUN: %clang_cc1 %{common_opts_mac} -verify=expected,omp5,ompvar,omp45-to-51,omp5-and-51,omp5-or-later,omp5-or-later-var,omp45-to-51-var,omp45-to-51-clause,host5,host-5-and-51,no-host5-and-51 %{openmp50_simd} -fopenmp-is-target-device %{target_mac} %{limit} -o - %s
+// RUN: %clang_cc1 %{common_opts_mac} -verify=expected,omp52,ompvar,omp5-or-later,omp5-or-later-var %{openmp60_simd} -fopenmp-is-target-device %{target_mac} %{limit} -o - %s
+
+// RUN: %clang_cc1 %{common_opts_mac} -verify=expected,omp45,omp45-to-51,omp45-to-51-var,omp45-to-51-clause -fopenmp-version=45 -fopenmp-simd %{limit} -o - %s
+// RUN: %clang_cc1 %{common_opts_mac} -verify=expected,omp51,ompvar,omp45-to-51,omp5-and-51,omp5-or-later,omp5-or-later-var,omp45-to-51-var,omp45-to-51-clause,host-5-and-51,no-host5-and-51 -fopenmp %{limit} -o - %s
+// RUN: %clang_cc1 %{common_opts_mac} -verify=expected,omp51,ompvar,omp45-to-51,omp5-and-51,omp5-or-later,omp5-or-later-var,omp45-to-51-var,omp45-to-51-clause,host-5-and-51,no-host5-and-51 -fopenmp %{limit} -DTESTEND=1 -o - %s
+// RUN: %clang_cc1 %{common_opts_mac} -verify=expected,omp51,ompvar,omp45-to-51,omp5-and-51,omp5-or-later,omp5-or-later-var,omp45-to-51-var,omp45-to-51-clause,host-5-and-51,no-host5-and-51 -fopenmp %{limit} -I%S/Inputs -DTESTENDINC=1 -o - %s
+// RUN: %clang_cc1 %{common_opts_mac} -verify=expected,omp51,ompvar,omp45-to-51,omp5-and-51,omp5-or-later,omp5-or-later-var,omp45-to-51-var,omp45-to-51-clause,host-5-and-51,no-host5-and-51 -fopenmp-simd %{limit} -o - %s
+
+// RUN: %clang_cc1 %{common_opts_mac} -verify=expected,omp52,ompvar,omp5-or-later,omp5-or-later-var %{openmp52} -DVERBOSE_MODE=1 %{limit} -o - %s
+// RUN: %clang_cc1 %{common_opts_mac} -verify=expected,omp52,ompvar,omp5-or-later,omp5-or-later-var %{openmp60} -DVERBOSE_MODE=1 %{limit} -o - %s
+
+// RUN: %clang_cc1 %{common_opts_mac} -verify=expected,omp5,ompvar,omp45-to-51,omp5-and-51,omp5-or-later,omp5-or-later-var,omp45-to-51-var,omp45-to-51-clause,host-5-and-51,no-host5-and-51 %{openmp50} %{limit} -o - %s
+// RUN: %clang_cc1 %{common_opts_mac} -verify=expected,omp51,ompvar,omp45-to-51,omp5-and-51,omp5-or-later,omp5-or-later-var,omp45-to-51-var,omp45-to-51-clause,host-5-and-51,no-host5-and-51 -fopenmp %{limit} -o - %s
+// RUN: %clang_cc1 %{common_opts_mac} -verify=expected,omp52,ompvar,omp5-or-later,omp5-or-later-var %{openmp60} %{limit} -o - %s
 
-// RUN: %clang_cc1 -triple x86_64-apple-macos10.7.0 -verify=expected,omp5,host5 -fopenmp-simd -fopenmp-version=50 -fopenmp-targets=x86_64-apple-macos10.7.0 -fnoopenmp-use-tls -ferror-limit 100 -o - %s
-// RUN: %clang_cc1 -triple x86_64-apple-macos10.7.0 -verify=expected,omp5,host5 -fopenmp-simd -fopenmp-version=50 -fopenmp-is-target-device -fopenmp-targets=x86_64-apple-macos10.7.0 -fnoopenmp-use-tls -ferror-limit 100 -o - %s
-// RUN: %clang_cc1 -triple x86_64-apple-macos10.7.0 -verify=expected,omp45 -fopenmp-version=45 -fopenmp-simd -fnoopenmp-use-tls -ferror-limit 100 -o - %s
-// RUN: %clang_cc1 -triple x86_64-apple-macos10.7.0 -verify=expected,omp51 -fopenmp -fnoopenmp-use-tls -ferror-limit 100 -o - %s
-// RUN: %clang_cc1 -triple x86_64-apple-macos10.7.0 -verify=expected,omp51 -fopenmp -fnoopenmp-use-tls -ferror-limit 100 -DTESTEND=1 -o - %s
-// RUN: %clang_cc1 -triple x86_64-apple-macos10.7.0 -verify=expected,omp51 -fopenmp -fnoopenmp-use-tls -ferror-limit 100 -I%S/Inputs -DTESTENDINC=1 -o - %s
-// RUN: %clang_cc1 -triple x86_64-apple-macos10.7.0 -verify=expected,omp51 -fopenmp-simd -fnoopenmp-use-tls -ferror-limit 100 -o - %s
-// RUN: %clang_cc1 -triple x86_64-apple-macos10.7.0 -verify=expected,omp51 -fopenmp-simd -fnoopenmp-use-tls -ferror-limit 100 -o - %s
+#pragma omp begin declare target
+static int gg;
+// expected-warning@+1 {{variable 'recursive' is uninitialized when used within its own initialization}}
+int recursive = recursive ^ 3 + gg;
+#pragma omp end declare target
 
-// RUN: %clang_cc1 -triple x86_64-apple-macos10.7.0 -verify=expected,omp52 -fopenmp -fopenmp-version=52 -DVERBOSE_MODE=1 -fnoopenmp-use-tls -ferror-limit 100 -o - %s
-// RUN: %clang_cc1 -triple x86_64-apple-macos10.7.0 -verify=expected,omp5 -fopenmp -fopenmp-version=50 -fnoopenmp-use-tls -ferror-limit 100 -o - %s
-// RUN: %clang_cc1 -triple x86_64-apple-macos10.7.0 -verify=expected,omp51 -fopenmp -fnoopenmp-use-tls -ferror-limit 100 -o - %s
-#pragma omp end declare target // expected-error {{unexpected OpenMP directive '#pragma omp end declare target'}}
+// expected-error@+1 {{unexpected OpenMP directive '#pragma omp end declare target'}}
+#pragma omp end declare target 
 
-int a, b, z; // omp5-error {{variable captured in declare target region must appear in a to clause}} // omp51-error {{variable captured in declare target region must appear in a to clause}} omp52-error {{variable captured in declare target region must appear in a to clause}}
-__thread int t; // expected-note {{defined as threadprivate or thread local}}
+// ompvar-error@+1 {{variable captured in declare target region must appear in a to clause}}
+int a, b, z;
+// expected-note@+1 {{defined as threadprivate or thread local}}
+__thread int t;
 
-#pragma omp declare target . // expected-error {{expected '(' after 'declare target'}}
+// expected-error@+1 {{expected '(' after 'declare target'}}
+#pragma omp declare target . 
 
 #pragma omp declare target
 void f();
-#pragma omp end declare target shared(a) // expected-warning {{extra tokens at the end of '#pragma omp end declare target' are ignored}}
-
-#pragma omp declare target map(a) // omp45-error {{expected at least one 'to' or 'link' clause}} omp5-error {{expected at least one 'to' or 'link' clause}} omp51-error {{expected at least one 'to', 'link' or 'indirect' clause}} omp45-error {{unexpected 'map' clause, only 'to' or 'link' clauses expected}} omp5-error {{unexpected 'map' clause, only 'to', 'link' or 'device_type' clauses expected}} omp51-error {{unexpected 'map' clause, only 'to', 'link', 'device_type' or 'indirect' clauses expected}} omp52-error {{unexpected 'map' clause, only 'enter', 'link', 'device_type' or 'indirect' clauses expected}} omp52-error {{expected at least one 'enter', 'link' or 'indirect' clause}}
-
-#pragma omp declare target to(foo1) // omp45-error {{use of undeclared identifier 'foo1'}} omp5-error {{use of undeclared identifier 'foo1'}} omp51-error {{use of undeclared identifier 'foo1'}} omp52-error {{unexpected 'to' clause, use 'enter' instead}} omp52-error {{expected at least one 'enter', 'link' or 'indirect' clause}}
-
-#pragma omp declare target link(foo2) // expected-error {{use of undeclared identifier 'foo2'}}
-
-#pragma omp declare target to(f) device_type(host) // omp45-error {{unexpected 'device_type' clause, only 'to' or 'link' clauses expected}} dev5-note {{marked as 'device_type(host)' here}} omp52-error {{unexpected 'to' clause, use 'enter' instead}} omp52-error {{expected at least one 'enter', 'link' or 'indirect' clause}}
+// expected-warning@+1 {{extra tokens at the end of '#pragma omp end declare target' are ignored}}
+#pragma omp end declare target shared(a) 
+
+// omp52-error@+8 {{unexpected 'map' clause, only 'enter', 'link', 'device_type' or 'indirect' clauses expected}}
+// omp52-error@+7 {{expected at least one 'enter', 'link' or 'indirect' clause}}
+// omp51-error@+6 {{unexpected 'map' clause, only 'to', 'link', 'device_type' or 'indirect' clauses expected}} 
+// omp51-error@+5 {{expected at least one 'to', 'link' or 'indirect' clause}}
+// omp5-error@+4 {{unexpected 'map' clause, only 'to', 'link' or 'device_type' clauses expected}}
+// omp5-error@+3 {{expected at least one 'to' or 'link' clause}}
+// omp45-error@+2 {{unexpected 'map' clause, only 'to' or 'link' clauses expected}}
+// omp45-error@+1 {{expected at least one 'to' or 'link' clause}} 
+#pragma omp declare target map(a)
+
+// omp52-error@+3 {{unexpected 'to' clause, use 'enter' instead}}
+// omp52-error@+2 {{expected at least one 'enter', 'link' or 'indirect' clause}}
+// omp45-to-51-error@+1 {{use of undeclared identifier 'foo1'}}
+#pragma omp declare target to(foo1) 
+
+// expected-error@+1 {{use of undeclared identifier 'foo2'}}
+#pragma omp declare target link(foo2) 
+
+// omp52-error@+4 {{unexpected 'to' clause, use 'enter' instead}}
+// omp52-error@+3 {{expected at least one 'enter', 'link' or 'indirect' clause}}
+// dev5-note@+2 {{marked as 'device_type(host)' here}}
+// omp45-error@+1 {{unexpected 'device_type' clause, only 'to' or 'link' clauses expected}}
+#pragma omp declare target to(f) device_type(host)
 
 void q();
-#pragma omp declare target to(q) device_type(any) device_type(any) device_type(host) // omp45-error {{unexpected 'device_type' clause, only 'to' or 'link' clauses expected}} omp5-warning {{more than one 'device_type' clause is specified}} // omp51-warning {{more than one 'device_type' clause is specified}} omp52-error {{unexpected 'to' clause, use 'enter' instead}} omp52-error {{expected at least one 'enter', 'link' or 'indirect' clause}}
+// omp52-error@+4 {{unexpected 'to' clause, use 'enter' instead}}
+// omp52-error@+3 {{expected at least one 'enter', 'link' or 'indirect' clause}}
+// omp5-and-51-warning@+2 {{more than one 'device_type' clause is specified}}
+// omp45-error@+1 {{unexpected 'device_type' clause, only 'to' or 'link' clauses expected}}
+#pragma omp declare target to(q) device_type(any) device_type(any) device_type(host) 
 
 #if _OPENMP == 202011
 // omp51-error@+1 {{directive '#pragma omp declare target' cannot contain more than one 'indirect' clause}}
 #pragma omp declare target to(q) indirect(true) indirect(false)
 
-int xxx; //expected-note {{declared here}}
+// expected-note@+1 {{declared here}}
+int xxx;
 // omp51-error@+2 {{expression is not an integral constant expression}}
 // omp51-note@+1 {{read of non-const variable 'xxx' is not allowed in a constant expression}}
 #pragma omp declare target to(q) indirect(xxx)
@@ -67,13 +118,20 @@ void bar();
 
 void c();
 
-void func() {} // expected-note {{'func' defined here}}
+// expected-note@+1 {{'func' defined here}}
+void func() {} 
 
-#pragma omp declare target link(func) allocate(a) // expected-error {{function name is not allowed in 'link' clause}} omp45-error {{unexpected 'allocate' clause, only 'to' or 'link' clauses expected}} omp5-error {{unexpected 'allocate' clause, only 'to', 'link' or 'device_type' clauses expected}} omp51-error {{unexpected 'allocate' clause, only 'to', 'link', 'device_type' or 'indirect' clauses expected}} omp52-error {{unexpected 'allocate' clause, only 'enter', 'link', 'device_type' or 'indirect' clauses expected}}
+// omp52-error@+5 {{unexpected 'allocate' clause, only 'enter', 'link', 'device_type' or 'indirect' clauses expected}}
+// omp51-error@+4 {{unexpected 'allocate' clause, only 'to', 'link', 'device_type' or 'indirect' clauses expected}}
+// omp5-error@+3 {{unexpected 'allocate' clause, only 'to', 'link' or 'device_type' clauses expected}}
+// expected-error@+2 {{function name is not allowed in 'link' clause}}
+// omp45-error@+1 {{unexpected 'allocate' clause, only 'to' or 'link' clauses expected}}
+#pragma omp declare target link(func) allocate(a)
 
 void bar();
 void baz() {bar();}
-#pragma omp declare target(bar) // omp5-warning {{declaration marked as declare target after first use, it may lead to incorrect results}} // omp51-warning {{declaration marked as declare target after first use, it may lead to incorrect results}} omp52-warning {{declaration marked as declare target after first use, it may lead to incorrect results}}
+// omp5-or-later-warning@+1 {{declaration marked as declare target after first use, it may lead to incorrect results}}
+#pragma omp declare target(bar)
 
 extern int b;
 
@@ -113,7 +171,8 @@ void t2() {
   void abc();
 #pragma omp end declare target
 void cba();
-#pragma omp end declare target // expected-error {{unexpected OpenMP directive '#pragma omp end declare target'}}
+// expected-error@+1 {{unexpected OpenMP directive '#pragma omp end declare target'}}
+#pragma omp end declare target 
 
 #pragma omp declare target
 #pragma omp declare target
@@ -122,7 +181,8 @@ void def();
 void fed();
 
 #pragma omp declare target
-#pragma omp threadprivate(a) // expected-note {{defined as threadprivate or thread local}}
+// expected-note@+1 {{defined as threadprivate or thread local}}
+#pragma omp threadprivate(a) 
 extern int b;
 int g;
 
@@ -150,27 +210,32 @@ int C::method1() {
 }
 
 void foo(int p) {
-  a = 0; // expected-error {{threadprivate variables cannot be used in target constructs}}
+// expected-error@+1 {{threadprivate variables cannot be used in target constructs}}
+  a = 0; 
   b = 0;
-  t = 1; // expected-error {{threadprivate variables cannot be used in target constructs}}
+// expected-error@+1 {{threadprivate variables cannot be used in target constructs}}
+  t = 1; 
   C object;
   VC object1;
   g = object.method();
   g += object.method1();
   g += object1.method() + p;
-  f(); // dev5-error {{function with 'device_type(host)' is not available on device}}
+  // dev5-error@+1 {{function with 'device_type(host)' is not available on device}}
+  f(); 
   q();
   c();
 }
 #pragma omp declare target
 void foo1() {
-  [&](){ (void)(b+z);}(); // omp5-note {{variable 'z' is captured here}} //omp51-note {{variable 'z' is captured here}} omp52-note {{variable 'z' is captured here}}
+  // omp5-or-later-var-note@+1 {{variable 'z' is captured here}}
+  [&](){ (void)(b+z);}(); 
 }
 #pragma omp end declare target
 
 #pragma omp end declare target
 #pragma omp end declare target
-#pragma omp end declare target // expected-error {{unexpected OpenMP directive '#pragma omp end declare target'}}
+// expected-error@+1 {{unexpected OpenMP directive '#pragma omp end declare target'}}
+#pragma omp end declare target 
 
 int C::method() {
   return 0;
@@ -190,12 +255,20 @@ int *y;
 int **w = &y;
 int main (int argc, char **argv) {
   int a = 2;
-#pragma omp declare target // expected-error {{unexpected OpenMP directive '#pragma omp declare target'}}
+// expected-error@+1 {{unexpected OpenMP directive '#pragma omp declare target'}}
+#pragma omp declare target 
   int v;
-#pragma omp end declare target // expected-error {{unexpected OpenMP directive '#pragma omp end declare target'}}
+// expected-error@+1 {{unexpected OpenMP directive '#pragma omp end declare target'}}
+#pragma omp end declare target 
   foo(v);
-#pragma omp declare target to(foo3) link(w) // omp52-error {{unexpected 'to' clause, use 'enter' instead}} omp52-error {{expected at least one 'enter', 'link' or 'indirect' clause}}
-#pragma omp declare target to(a) //omp45-error {{local variable 'a' should not be used in 'declare target' directive}} omp5-error {{local variable 'a' should not be used in 'declare target' directive}} omp51-error {{local variable 'a' should not be used in 'declare target' directive}} omp52-error {{unexpected 'to' clause, use 'enter' instead}} omp52-error {{expected at least one 'enter', 'link' or 'indirect' clause}}
+
+  // omp52-error@+2 {{expected at least one 'enter', 'link' or 'indirect' clause}}
+  // omp52-error@+1 {{unexpected 'to' clause, use 'enter' instead}}
+#pragma omp declare target to(foo3) link(w)
+  // omp52-error@+3 {{unexpected 'to' clause, use 'enter' instead}}
+  // omp52-error@+2 {{expected at least one 'enter', 'link' or 'indirect' clause}}
+  // omp45-to-51-var-error@+1 {{local variable 'a' should not be used in 'declare target' directive}}
+#pragma omp declare target to(a) 
   return (0);
 }
 
@@ -205,77 +278,134 @@ namespace {
 }
 #pragma omp end declare target
 
-#pragma omp declare target link(S) // expected-error {{'S' used in declare target directive is not a variable or a function name}}
+// expected-error@+1 {{'S' used in declare target directive is not a variable or a function name}}
+#pragma omp declare target link(S) 
 
-#pragma omp declare target (x, x) // expected-error {{'x' appears multiple times in clauses on the same declare target directive}}
-#pragma omp declare target to(x) to(x) // omp45-error {{'x' appears multiple times in clauses on the same declare target directive}} omp5-error {{'x' appears multiple times in clauses on the same declare target directive}} omp51-error {{'x' appears multiple times in clauses on the same declare target directive}} omp52-error {{unexpected 'to' clause, use 'enter' instead}} omp52-error {{expected at least one 'enter', 'link' or 'indirect' clause}}
-#pragma omp declare target link(x) // expected-error {{'x' must not appear in both clauses 'to' and 'link'}}
+// expected-error@+1 {{'x' appears multiple times in clauses on the same declare target directive}}
+#pragma omp declare target (x, x) 
+// omp52-error@+3 {{unexpected 'to' clause, use 'enter' instead}}
+// omp52-error@+2 {{expected at least one 'enter', 'link' or 'indirect' clause}}
+// omp45-to-51-clause-error@+1 {{'x' appears multiple times in clauses on the same declare target directive}}
+#pragma omp declare target to(x) to(x)
+// expected-error@+1 {{'x' must not appear in both clauses 'to' and 'link'}}
+#pragma omp declare target link(x) 
 
 void bazz() {}
-#pragma omp declare target to(bazz) device_type(nohost) // omp45-error {{unexpected 'device_type' clause, only 'to' or 'link' clauses expected}} host5-note 3{{marked as 'device_type(nohost)' here}} omp52-error {{unexpected 'to' clause, use 'enter' instead}} omp52-error {{expected at least one 'enter', 'link' or 'indirect' clause}}
+// omp52-error@+4 {{unexpected 'to' clause, use 'enter' instead}}
+// omp52-error@+3 {{expected at least one 'enter', 'link' or 'indirect' clause}}
+// host5-note@+2 3 {{marked as 'device_type(nohost)' here}}
+// omp45-error@+1 {{unexpected 'device_type' clause, only 'to' or 'link' clauses expected}} 
+#pragma omp declare target to(bazz) device_type(nohost)
 void bazzz() {bazz();}
-#pragma omp declare target to(bazzz) device_type(nohost) // omp45-error {{unexpected 'device_type' clause, only 'to' or 'link' clauses expected}} omp52-error {{unexpected 'to' clause, use 'enter' instead}} omp52-error {{expected at least one 'enter', 'link' or 'indirect' clause}}
-void any() {bazz();} // host5-error {{function with 'device_type(nohost)' is not available on host}}
-void host1() {bazz();} // host5-error {{function with 'device_type(nohost)' is not available on host}}
-#pragma omp declare target to(host1) device_type(host) // omp45-error {{unexpected 'device_type' clause, only 'to' or 'link' clauses expected}} dev5-note 3 {{marked as 'device_type(host)' here}} omp52-error {{unexpected 'to' clause, use 'enter' instead}} omp52-error {{expected at least one 'enter', 'link' or 'indirect' clause}}
-void host2() {bazz();} //host5-error {{function with 'device_type(nohost)' is not available on host}}
-#pragma omp declare target to(host2) // omp52-error {{unexpected 'to' clause, use 'enter' instead}} omp52-error {{expected at least one 'enter', 'link' or 'indirect' clause}}
-void device() {host1();} // dev5-error {{function with 'device_type(host)' is not available on device}}
-#pragma omp declare target to(device) device_type(nohost) // omp45-error {{unexpected 'device_type' clause, only 'to' or 'link' clauses expected}} host5-note 2 {{marked as 'device_type(nohost)' here}} omp52-error {{unexpected 'to' clause, use 'enter' instead}} omp52-error {{expected at least one 'enter', 'link' or 'indirect' clause}}
+// omp52-error@+3 {{unexpected 'to' clause, use 'enter' instead}}
+// omp52-error@+2 {{expected at least one 'enter', 'link' or 'indirect' clause}}
+// omp45-error@+1 {{unexpected 'device_type' clause, only 'to' or 'link' clauses expected}}
+#pragma omp declare target to(bazzz) device_type(nohost) 
+// host5-error@+1 {{function with 'device_type(nohost)' is not available on host}}
+void any() {bazz();} 
+// host5-error@+1 {{function with 'device_type(nohost)' is not available on host}}
+void host1() {bazz();}
+// omp52-error@+4 {{unexpected 'to' clause, use 'enter' instead}}
+// omp52-error@+3 {{expected at least one 'enter', 'link' or 'indirect' clause}}
+// dev5-note@+2 3 {{marked as 'device_type(host)' here}}
+// omp45-error@+1 {{unexpected 'device_type' clause, only 'to' or 'link' clauses expected}}
+#pragma omp declare target to(host1) device_type(host)
+//host5-error@+1 {{function with 'device_type(nohost)' is not available on host}}
+void host2() {bazz();}
+// omp52-error@+2 {{unexpected 'to' clause, use 'enter' instead}}
+// omp52-error@+1 {{expected at least one 'enter', 'link' or 'indirect' clause}}
+#pragma omp declare target to(host2) 
+// dev5-error@+1 {{function with 'device_type(host)' is not available on device}}
+void device() {host1();}
+// omp52-error@+4 {{unexpected 'to' clause, use 'enter' instead}}
+// omp52-error@+3 {{expected at least one 'enter', 'link' or 'indirect' clause}}
+// host5-note@+2 2 {{marked as 'device_type(nohost)' here}} 
+// omp45-error@+1 {{unexpected 'device_type' clause, only 'to' or 'link' clauses expected}}
+#pragma omp declare target to(device) device_type(nohost)
 void host3() {host1();} // dev5-error {{function with 'device_type(host)' is not available on device}}
-#pragma omp declare target to(host3) // omp52-error {{unexpected 'to' clause, use 'enter' instead}} omp52-error {{expected at least one 'enter', 'link' or 'indirect' clause}}
+// omp52-error@+2 {{unexpected 'to' clause, use 'enter' instead}}
+// omp52-error@+1 {{expected at least one 'enter', 'link' or 'indirect' clause}}
+#pragma omp declare target to(host3)
 
 #pragma omp declare target
 void any1() {any();}
-void any2() {host1();} // dev5-error {{function with 'device_type(host)' is not available on device}}
-void any3() {device();} // host5-error {{function with 'device_type(nohost)' is not available on host}}
+// dev5-error@+1 {{function with 'device_type(host)' is not available on device}}
+void any2() {host1();} 
+// host5-error@+1 {{function with 'device_type(nohost)' is not available on host}}
+void any3() {device();}
 void any4() {any2();}
 #pragma omp end declare target
 
 void any5() {any();}
 void any6() {host1();}
-void any7() {device();} // host5-error {{function with 'device_type(nohost)' is not available on host}}
+// host5-error@+1 {{function with 'device_type(nohost)' is not available on host}}
+void any7() {device();}
 void any8() {any2();}
 
 int MultiDevTy;
-#pragma omp declare target to(MultiDevTy) device_type(any)    // omp45-error {{unexpected 'device_type' clause, only 'to' or 'link' clauses expected}} omp52-error {{unexpected 'to' clause, use 'enter' instead}} omp52-error {{expected at least one 'enter', 'link' or 'indirect' clause}}
-#pragma omp declare target to(MultiDevTy) device_type(host)   // omp45-error {{unexpected 'device_type' clause, only 'to' or 'link' clauses expected}} omp5-error {{'device_type(host)' does not match previously specified 'device_type(any)' for the same declaration}} omp51-error {{'device_type(host)' does not match previously specified 'device_type(any)' for the same declaration}} omp52-error {{unexpected 'to' clause, use 'enter' instead}} omp52-error {{expected at least one 'enter', 'link' or 'indirect' clause}}
-#pragma omp declare target to(MultiDevTy) device_type(nohost) // omp45-error {{unexpected 'device_type' clause, only 'to' or 'link' clauses expected}} omp5-error {{'device_type(nohost)' does not match previously specified 'device_type(any)' for the same declaration}} // omp51-error {{'device_type(nohost)' does not match previously specified 'device_type(any)' for the same declaration}} omp52-error {{unexpected 'to' clause, use 'enter' instead}} omp52-error {{expected at least one 'enter', 'link' or 'indirect' clause}}
-
-static int variable = 100;  //expected-warning {{declaration is not declared in any declare target region}}
+// omp52-error@+3 {{unexpected 'to' clause, use 'enter' instead}}
+// omp52-error@+2 {{expected at least one 'enter', 'link' or 'indirect' clause}}
+// omp45-error@+1 {{unexpected 'device_type' clause, only 'to' or 'link' clauses expected}}
+#pragma omp declare target to(MultiDevTy) device_type(any)
+// omp52-error@+4 {{unexpected 'to' clause, use 'enter' instead}}
+// omp52-error@+3 {{expected at least one 'enter', 'link' or 'indirect' clause}}
+// host-5-and-51-error@+2 {{'device_type(host)' does not match previously specified 'device_type(any)' for the same declaration}}
+// omp45-error@+1 {{unexpected 'device_type' clause, only 'to' or 'link' clauses expected}}
+#pragma omp declare target to(MultiDevTy) device_type(host)
+// omp52-error@+4 {{unexpected 'to' clause, use 'enter' instead}}
+// omp52-error@+3 {{expected at least one 'enter', 'link' or 'indirect' clause}}
+// no-host5-and-51-error@+2 {{'device_type(nohost)' does not match previously specified 'device_type(any)' for the same declaration}}
+// omp45-error@+1 {{unexpected 'device_type' clause, only 'to' or 'link' clauses expected}}
+#pragma omp declare target to(MultiDevTy) device_type(nohost)
+
+// expected-warning@+1 {{declaration is not declared in any declare target region}}
+static int variable = 100; 
 static float variable1 = 200;
-static float variable2 = variable1;  //expected-warning {{declaration is not declared in any declare target region}}
+// expected-warning@+1 {{declaration is not declared in any declare target region}}
+static float variable2 = variable1;  
 
-static int var = 1;  //expected-warning {{declaration is not declared in any declare target region}}
+// expected-warning@+1 {{declaration is not declared in any declare target region}}
+static int var = 1;  
 
 static int var1 = 10;
 static int *var2 = &var1;
-static int **ptr1 = &var2;  //expected-warning {{declaration is not declared in any declare target region}}
+// expected-warning@+1 {{declaration is not declared in any declare target region}}
+static int **ptr1 = &var2;  
 
 int arr[2] = {1,2};
-int (*arrptr)[2] = &arr;  //expected-warning {{declaration is not declared in any declare target region}}
+// expected-warning@+1 {{declaration is not declared in any declare target region}}
+int (*arrptr)[2] = &arr;  
 
 class declare{
   public: int x;
           void print();
 };
 declare obj1;
-declare *obj2 = &obj1;  //expected-warning {{declaration is not declared in any declare target region}}
+// expected-warning@+1 {{declaration is not declared in any declare target region}}
+declare *obj2 = &obj1;  
 
 struct target{
   int x;
   void print();
 };
-static target S;  //expected-warning {{declaration is not declared in any declare target region}}
+// expected-warning@+1 {{declaration is not declared in any declare target region}}
+static target S;  
 
 #pragma omp declare target
-int target_var = variable;  //expected-note {{used here}}
-float target_var1 = variable2;  //expected-note {{used here}}
-int *ptr = &var;  //expected-note {{used here}}
-int ***ptr2 = &ptr1;  //expected-note {{used here}}
-int (**ptr3)[2] = &arrptr;  //expected-note {{used here}}
-declare **obj3 = &obj2;  //expected-note {{used here}}
-target *S1 = &S; //expected-note {{used here}}
+// expected-note@+1 {{used here}}
+int target_var = variable;  
+// expected-note@+1 {{used here}}
+float target_var1 = variable2;  
+// expected-note@+1 {{used here}}
+int *ptr = &var;  
+// expected-note@+1 {{used here}}
+int ***ptr2 = &ptr1; 
+// expected-note@+1 {{used here}}
+int (**ptr3)[2] = &arrptr;
+// expected-note@+1 {{used here}}
+declare **obj3 = &obj2;
+// expected-note@+1 {{used here}}
+target *S1 = &S;
 #pragma omp end declare target
 
 #if TESTENDINC
diff --git a/clang/test/OpenMP/declare_variant_ast_print.c b/clang/test/OpenMP/declare_variant_ast_print.c
index 0df10263cde5e..9bd0b6d0d6162 100644
--- a/clang/test/OpenMP/declare_variant_ast_print.c
+++ b/clang/test/OpenMP/declare_variant_ast_print.c
@@ -20,6 +20,8 @@ int foo(void);
 #pragma omp declare variant(foo) match(implementation={vendor(score(5): ibm, xxx, ibm)}, device={kind(cpu, nohost)})
 #pragma omp declare variant(foo) match(device={kind(host)})
 #pragma omp declare variant(foo) match(device={kind(nohost), xxx})
+#pragma omp declare variant(foo) match(target_device={kind(host)})
+#pragma omp declare variant(foo) match(target_device={kind(nohost), xxx})
 #pragma omp declare variant(foo) match(implementation={extension(match_all)})
 #pragma omp declare variant(foo) match(implementation={extension(match_any)})
 #pragma omp declare variant(foo) match(implementation={extension(match_none)})
@@ -29,6 +31,8 @@ int bar(void);
 // CHECK-NEXT: #pragma omp declare variant(foo) match(implementation={extension(match_none)})
 // CHECK-NEXT: #pragma omp declare variant(foo) match(implementation={extension(match_any)})
 // CHECK-NEXT: #pragma omp declare variant(foo) match(implementation={extension(match_all)})
+// CHECK-NEXT: #pragma omp declare variant(foo) match(target_device={kind(nohost)})
+// CHECK-NEXT: #pragma omp declare variant(foo) match(target_device={kind(host)})
 // CHECK-NEXT: #pragma omp declare variant(foo) match(device={kind(nohost)})
 // CHECK-NEXT: #pragma omp declare variant(foo) match(device={kind(host)})
 // CHECK-NEXT: #pragma omp declare variant(foo) match(implementation={vendor(score(5): ibm)}, device={kind(cpu, nohost)})
diff --git a/clang/test/OpenMP/declare_variant_bind_to_decl.cpp b/clang/test/OpenMP/declare_variant_bind_to_decl.cpp
index dca18abb36c89..34191417a3934 100644
--- a/clang/test/OpenMP/declare_variant_bind_to_decl.cpp
+++ b/clang/test/OpenMP/declare_variant_bind_to_decl.cpp
@@ -29,6 +29,6 @@ int main() {
 // CHECK-LABEL: define {{[^@]+}}@main
 // CHECK-SAME: () #[[ATTR1:[0-9]+]] {
 // CHECK-NEXT:  entry:
-// CHECK-NEXT:    call void @"_Z74foo$ompvariant$S2$s7$Pppc64le$Pppc64$S3$s9$Pmatch_any$Pbind_to_declarationv"()
+// CHECK-NEXT:    call void @{{"_Z[0-9]+foo\$ompvariant\$.*"}}()
 // CHECK-NEXT:    ret i32 0
 //
diff --git a/clang/test/OpenMP/declare_variant_messages.c b/clang/test/OpenMP/declare_variant_messages.c
index 0ae276effd1be..14637d8200671 100644
--- a/clang/test/OpenMP/declare_variant_messages.c
+++ b/clang/test/OpenMP/declare_variant_messages.c
@@ -16,17 +16,17 @@ int foo(void);
 #pragma omp declare variant(foo) // omp50-error {{expected 'match' clause on 'omp declare variant' directive}} omp51-error {{expected 'match', 'adjust_args', or 'append_args' clause on 'omp declare variant' directive}}
 #pragma omp declare variant(foo) xxx // omp50-error {{expected 'match' clause on 'omp declare variant' directive}} omp51-error {{expected 'match', 'adjust_args', or 'append_args' clause on 'omp declare variant' directive}}
 #pragma omp declare variant(foo) match // expected-error {{expected '(' after 'match'}}
-#pragma omp declare variant(foo) match( // expected-error {{expected ')'}} expected-warning {{expected identifier or string literal describing a context set; set skipped}} expected-note {{context set options are: 'construct' 'device' 'implementation' 'user'}} expected-note {{the ignored set spans until here}} expected-note {{to match this '('}}
-#pragma omp declare variant(foo) match() // expected-warning {{expected identifier or string literal describing a context set; set skipped}} expected-note {{context set options are: 'construct' 'device' 'implementation' 'user'}} expected-note {{the ignored set spans until here}}
-#pragma omp declare variant(foo) match(xxx) // expected-warning {{'xxx' is not a valid context set in a `declare variant`; set ignored}} expected-note {{context set options are: 'construct' 'device' 'implementation' 'user'}} expected-note {{the ignored set spans until here}}
-#pragma omp declare variant(foo) match(xxx=) // expected-warning {{'xxx' is not a valid context set in a `declare variant`; set ignored}} expected-note {{context set options are: 'construct' 'device' 'implementation' 'user'}} expected-note {{the ignored set spans until here}}
-#pragma omp declare variant(foo) match(xxx=yyy) // expected-warning {{'xxx' is not a valid context set in a `declare variant`; set ignored}} expected-note {{context set options are: 'construct' 'device' 'implementation' 'user'}} expected-note {{the ignored set spans until here}}
-#pragma omp declare variant(foo) match(xxx=yyy}) // expected-error {{expected ')'}} expected-warning {{'xxx' is not a valid context set in a `declare variant`; set ignored}} expected-note {{context set options are: 'construct' 'device' 'implementation' 'user'}} expected-note {{the ignored set spans until here}} expected-note {{to match this '('}} omp50-error {{expected 'match' clause on 'omp declare variant' directive}} omp51-error {{expected 'match', 'adjust_args', or 'append_args' clause on 'omp declare variant' directive}}
-#pragma omp declare variant(foo) match(xxx={) // expected-error {{expected ')'}} expected-warning {{'xxx' is not a valid context set in a `declare variant`; set ignored}} expected-note {{context set options are: 'construct' 'device' 'implementation' 'user'}} expected-note {{the ignored set spans until here}} expected-note {{to match this '('}}
-#pragma omp declare variant(foo) match(xxx={}) // expected-warning {{'xxx' is not a valid context set in a `declare variant`; set ignored}} expected-note {{context set options are: 'construct' 'device' 'implementation' 'user'}} expected-note {{the ignored set spans until here}}
-#pragma omp declare variant(foo) match(xxx={vvv, vvv}) // expected-warning {{'xxx' is not a valid context set in a `declare variant`; set ignored}} expected-note {{context set options are: 'construct' 'device' 'implementation' 'user'}} expected-note {{the ignored set spans until here}}
-#pragma omp declare variant(foo) match(xxx={vvv} xxx) // expected-warning {{'xxx' is not a valid context set in a `declare variant`; set ignored}} expected-note {{context set options are: 'construct' 'device' 'implementation' 'user'}} expected-note {{the ignored set spans until here}}
-#pragma omp declare variant(foo) match(xxx={vvv}) xxx // expected-warning {{'xxx' is not a valid context set in a `declare variant`; set ignored}} expected-note {{context set options are: 'construct' 'device' 'implementation' 'user'}} expected-note {{the ignored set spans until here}} omp50-error {{expected 'match' clause on 'omp declare variant' directive}} omp51-error {{expected 'match', 'adjust_args', or 'append_args' clause on 'omp declare variant' directive}}
+#pragma omp declare variant(foo) match( // expected-error {{expected ')'}} expected-warning {{expected identifier or string literal describing a context set; set skipped}} expected-note {{context set options are: 'construct' 'device' 'target_device' 'implementation' 'user'}} expected-note {{the ignored set spans until here}} expected-note {{to match this '('}}
+#pragma omp declare variant(foo) match() // expected-warning {{expected identifier or string literal describing a context set; set skipped}} expected-note {{context set options are: 'construct' 'device' 'target_device' 'implementation' 'user'}} expected-note {{the ignored set spans until here}}
+#pragma omp declare variant(foo) match(xxx) // expected-warning {{'xxx' is not a valid context set in a `declare variant`; set ignored}} expected-note {{context set options are: 'construct' 'device' 'target_device' 'implementation' 'user'}} expected-note {{the ignored set spans until here}}
+#pragma omp declare variant(foo) match(xxx=) // expected-warning {{'xxx' is not a valid context set in a `declare variant`; set ignored}} expected-note {{context set options are: 'construct' 'device' 'target_device' 'implementation' 'user'}} expected-note {{the ignored set spans until here}}
+#pragma omp declare variant(foo) match(xxx=yyy) // expected-warning {{'xxx' is not a valid context set in a `declare variant`; set ignored}} expected-note {{context set options are: 'construct' 'device' 'target_device' 'implementation' 'user'}} expected-note {{the ignored set spans until here}}
+#pragma omp declare variant(foo) match(xxx=yyy}) // expected-error {{expected ')'}} expected-warning {{'xxx' is not a valid context set in a `declare variant`; set ignored}} expected-note {{context set options are: 'construct' 'device' 'target_device' 'implementation' 'user'}} expected-note {{the ignored set spans until here}} expected-note {{to match this '('}} omp50-error {{expected 'match' clause on 'omp declare variant' directive}} omp51-error {{expected 'match', 'adjust_args', or 'append_args' clause on 'omp declare variant' directive}}
+#pragma omp declare variant(foo) match(xxx={) // expected-error {{expected ')'}} expected-warning {{'xxx' is not a valid context set in a `declare variant`; set ignored}} expected-note {{context set options are: 'construct' 'device' 'target_device' 'implementation' 'user'}} expected-note {{the ignored set spans until here}} expected-note {{to match this '('}}
+#pragma omp declare variant(foo) match(xxx={}) // expected-warning {{'xxx' is not a valid context set in a `declare variant`; set ignored}} expected-note {{context set options are: 'construct' 'device' 'target_device' 'implementation' 'user'}} expected-note {{the ignored set spans until here}}
+#pragma omp declare variant(foo) match(xxx={vvv, vvv}) // expected-warning {{'xxx' is not a valid context set in a `declare variant`; set ignored}} expected-note {{context set options are: 'construct' 'device' 'target_device' 'implementation' 'user'}} expected-note {{the ignored set spans until here}}
+#pragma omp declare variant(foo) match(xxx={vvv} xxx) // expected-warning {{'xxx' is not a valid context set in a `declare variant`; set ignored}} expected-note {{context set options are: 'construct' 'device' 'target_device' 'implementation' 'user'}} expected-note {{the ignored set spans until here}}
+#pragma omp declare variant(foo) match(xxx={vvv}) xxx // expected-warning {{'xxx' is not a valid context set in a `declare variant`; set ignored}} expected-note {{context set options are: 'construct' 'device' 'target_device' 'implementation' 'user'}} expected-note {{the ignored set spans until here}} omp50-error {{expected 'match' clause on 'omp declare variant' directive}} omp51-error {{expected 'match', 'adjust_args', or 'append_args' clause on 'omp declare variant' directive}}
 #pragma omp declare variant(foo) match(implementation={xxx}) // expected-warning {{'xxx' is not a valid context selector for the context set 'implementation'; selector ignored}} expected-note {{context selector options are: 'vendor' 'extension' 'unified_address' 'unified_shared_memory' 'reverse_offload' 'dynamic_allocators' 'atomic_default_mem_order'}} expected-note {{the ignored selector spans until here}}
 #pragma omp declare variant(foo) match(implementation={vendor}) // expected-warning {{the context selector 'vendor' in context set 'implementation' requires a context property defined in parentheses; selector ignored}} expected-note {{the ignored selector spans until here}}
 #pragma omp declare variant(foo) match(implementation={vendor(}) // expected-error {{expected ')'}} expected-warning {{expected identifier or string literal describing a context property; property skipped}} expected-note {{context property options are: 'amd' 'arm' 'bsc' 'cray' 'fujitsu' 'gnu' 'ibm' 'intel' 'llvm' 'nec' 'nvidia' 'pgi' 'ti' 'unknown'}} expected-note {{to match this '('}}
@@ -48,8 +48,18 @@ int foo(void);
 #pragma omp declare variant(foo) match(device={kind(score(5): host), kind(llvm)}) // expected-warning {{the context selector 'kind' in the context set 'device' cannot have a score ('5'); score ignored}} expected-warning {{the context selector 'kind' was used already in the same 'omp declare variant' directive; selector ignored}} expected-note {{the previous context selector 'kind' used here}} expected-note {{the ignored selector spans until here}}
 #pragma omp declare variant(foo) match(device={kind(score(5): nohost), vendor(llvm)}) // expected-warning {{the context selector 'kind' in the context set 'device' cannot have a score ('5'); score ignored}} expected-warning {{the context selector 'vendor' is not valid for the context set 'device'; selector ignored}} expected-note {{the context selector 'vendor' can be nested in the context set 'implementation'; try 'match(implementation={vendor(property)})'}} expected-note {{the ignored selector spans until here}}
 #pragma omp declare variant(foo) match(implementation={extension("aaa")}) // expected-warning {{'aaa' is not a valid context property for the context selector 'extension' and the context set 'implementation'; property ignored}} expected-note {{context property options are: 'match_all' 'match_any' 'match_none'}} expected-note {{the ignored property spans until here}}
+#pragma omp declare variant(foo) match(target_device={}) // expected-warning {{expected identifier or string literal describing a context selector; selector skipped}} expected-note {{context selector options are: 'kind' 'device_num' 'arch' 'isa'}} expected-note {{the ignored selector spans until here}}
+#pragma omp declare variant(foo) match(target_device={xxx}) // expected-warning {{'xxx' is not a valid context selector for the context set 'target_device'; selector ignored}} expected-note {{context selector options are: 'kind' 'device_num' 'arch' 'isa'}} expected-note {{the ignored selector spans until here}}
+#pragma omp declare variant(foo) match(target_device={kind}) // expected-warning {{the context selector 'kind' in context set 'target_device' requires a context property defined in parentheses; selector ignored}} expected-note {{the ignored selector spans until here}}
+#pragma omp declare variant(foo) match(target_device={kind(}) // expected-error {{expected ')'}} expected-warning {{expected identifier or string literal describing a context property; property skipped}} expected-note {{context property options are: 'host' 'nohost' 'cpu' 'gpu' 'fpga' 'any'}} expected-note {{to match this '('}}
+#pragma omp declare variant(foo) match(target_device={kind()}) // expected-warning {{expected identifier or string literal describing a context property; property skipped}} expected-note {{context property options are: 'host' 'nohost' 'cpu' 'gpu' 'fpga' 'any'}}
+#pragma omp declare variant(foo) match(target_device={device_num}) // expected-warning {{the context selector 'device_num' in context set 'target_device' requires a context property defined in parentheses; selector ignored}} expected-note {{the ignored selector spans until here}} 
+#pragma omp declare variant(foo) match(target_device={device_num()}) // expected-error {{expected expression}}
+#pragma omp declare variant(foo) match(target_device={device_num(-1)}) // expected-error {{argument to 'device_num' clause must be a non-negative integer value}}
+#pragma omp declare variant(foo) match(target_device={device_num(abc)}) // expected-error {{expected expression}} expected-error {{use of undeclared identifier 'abc'}}
 int bar(void);
 
+
 #pragma omp declare variant(foo) match(implementation = {vendor(score(foo) :llvm)}) // expected-warning {{score expressions in the OpenMP context selector need to be constant; foo is not and will be ignored}}
 #pragma omp declare variant(foo) match(implementation = {vendor(score(foo()) :llvm)}) // expected-warning {{score expressions in the OpenMP context selector need to be constant; foo() is not and will be ignored}}
 #pragma omp declare variant(foo) match(implementation = {vendor(score(<expr>) :llvm)}) // expected-error {{expected expression}} expected-error {{use of undeclared identifier 'expr'}} expected-error {{expected expression}}
@@ -64,10 +74,10 @@ int score_and_cond_non_const(void);
 #pragma omp declare variant(foo) match(construct={for simd}) // expected-error {{expected ')'}} expected-warning {{expected '}' after the context selectors for the context set "construct"; '}' assumed}} expected-note {{to match this '('}} omp50-error {{expected 'match' clause on 'omp declare variant' directive}} omp51-error {{expected 'match', 'adjust_args', or 'append_args' clause on 'omp declare variant' directive}}
 int construct(void);
 
-#pragma omp declare variant(foo) match(xxx={}) // expected-warning {{'xxx' is not a valid context set in a `declare variant`; set ignored}} expected-note {{context set options are: 'construct' 'device' 'implementation' 'user'}} expected-note {{the ignored set spans until here}}
+#pragma omp declare variant(foo) match(xxx={}) // expected-warning {{'xxx' is not a valid context set in a `declare variant`; set ignored}} expected-note {{context set options are: 'construct' 'device' 'target_device' 'implementation' 'user'}} expected-note {{the ignored set spans until here}}
 int a; // expected-error {{'#pragma omp declare variant' can only be applied to functions}}
 
-#pragma omp declare variant(foo) match(xxx={}) // expected-warning {{'xxx' is not a valid context set in a `declare variant`; set ignored}} expected-note {{context set options are: 'construct' 'device' 'implementation' 'user'}} expected-note {{the ignored set spans until here}}
+#pragma omp declare variant(foo) match(xxx={}) // expected-warning {{'xxx' is not a valid context set in a `declare variant`; set ignored}} expected-note {{context set options are: 'construct' 'device' 'target_device' 'implementation' 'user'}} expected-note {{the ignored set spans until here}}
 #pragma omp threadprivate(a) // expected-error {{'#pragma omp declare variant' can only be applied to functions}}
 int var;
 #pragma omp threadprivate(var)
@@ -91,21 +101,21 @@ int main(void);
 int main(void);
 
 
-#pragma omp declare variant(foo) match(xxx={}) // expected-error {{single declaration is expected after 'declare variant' directive}} expected-warning {{'xxx' is not a valid context set in a `declare variant`; set ignored}} expected-note {{context set options are: 'construct' 'device' 'implementation' 'user'}} expected-note {{the ignored set spans until here}}
+#pragma omp declare variant(foo) match(xxx={}) // expected-error {{single declaration is expected after 'declare variant' directive}} expected-warning {{'xxx' is not a valid context set in a `declare variant`; set ignored}} expected-note {{context set options are: 'construct' 'device' 'target_device' 'implementation' 'user'}} expected-note {{the ignored set spans until here}}
 int b, c;
 
 int no_proto();
-#pragma omp declare variant(no_proto) match(xxx={}) // expected-warning {{'xxx' is not a valid context set in a `declare variant`; set ignored}} expected-note {{context set options are: 'construct' 'device' 'implementation' 'user'}} expected-note {{the ignored set spans until here}}
+#pragma omp declare variant(no_proto) match(xxx={}) // expected-warning {{'xxx' is not a valid context set in a `declare variant`; set ignored}} expected-note {{context set options are: 'construct' 'device' 'target_device' 'implementation' 'user'}} expected-note {{the ignored set spans until here}}
 int no_proto_too();
 
 int proto1(int);
 
-#pragma omp declare variant(proto1) match(xxx={}) // expected-warning {{'xxx' is not a valid context set in a `declare variant`; set ignored}} expected-note {{context set options are: 'construct' 'device' 'implementation' 'user'}} expected-note {{the ignored set spans until here}}
+#pragma omp declare variant(proto1) match(xxx={}) // expected-warning {{'xxx' is not a valid context set in a `declare variant`; set ignored}} expected-note {{context set options are: 'construct' 'device' 'target_device' 'implementation' 'user'}} expected-note {{the ignored set spans until here}}
 int diff_proto(); // expected-note {{previous declaration is here}}
 
 int diff_proto(double); // expected-error {{conflicting types for 'diff_proto'}}
 
-#pragma omp declare variant(no_proto) match(xxx={}) // expected-warning {{'xxx' is not a valid context set in a `declare variant`; set ignored}} expected-note {{context set options are: 'construct' 'device' 'implementation' 'user'}} expected-note {{the ignored set spans until here}}
+#pragma omp declare variant(no_proto) match(xxx={}) // expected-warning {{'xxx' is not a valid context set in a `declare variant`; set ignored}} expected-note {{context set options are: 'construct' 'device' 'target_device' 'implementation' 'user'}} expected-note {{the ignored set spans until here}}
 int diff_proto1(double);
 
 int after_use_variant(void);
@@ -124,24 +134,24 @@ void self_test(int n, int d_no) {
   self(n);
 }
 
-#pragma omp declare variant(after_use_variant) match(xxx={}) // expected-warning {{'xxx' is not a valid context set in a `declare variant`; set ignored}} expected-warning {{'#pragma omp declare variant' cannot be applied for function after first usage; the original function might be used}} expected-note {{context set options are: 'construct' 'device' 'implementation' 'user'}} expected-note {{the ignored set spans until here}}
+#pragma omp declare variant(after_use_variant) match(xxx={}) // expected-warning {{'xxx' is not a valid context set in a `declare variant`; set ignored}} expected-warning {{'#pragma omp declare variant' cannot be applied for function after first usage; the original function might be used}} expected-note {{context set options are: 'construct' 'device' 'target_device' 'implementation' 'user'}} expected-note {{the ignored set spans until here}}
 int after_use(void);
-#pragma omp declare variant(after_use_variant) match(xxx={}) // expected-warning {{'xxx' is not a valid context set in a `declare variant`; set ignored}} expected-note {{context set options are: 'construct' 'device' 'implementation' 'user'}} expected-note {{the ignored set spans until here}}
+#pragma omp declare variant(after_use_variant) match(xxx={}) // expected-warning {{'xxx' is not a valid context set in a `declare variant`; set ignored}} expected-note {{context set options are: 'construct' 'device' 'target_device' 'implementation' 'user'}} expected-note {{the ignored set spans until here}}
 int defined(void) { return 0; }
 int defined1(void) { return 0; }
 
-#pragma omp declare variant(after_use_variant) match(xxx={}) // expected-warning {{'xxx' is not a valid context set in a `declare variant`; set ignored}} expected-warning {{'#pragma omp declare variant' cannot be applied to the function that was defined already; the original function might be used}} expected-note {{context set options are: 'construct' 'device' 'implementation' 'user'}} expected-note {{the ignored set spans until here}}
+#pragma omp declare variant(after_use_variant) match(xxx={}) // expected-warning {{'xxx' is not a valid context set in a `declare variant`; set ignored}} expected-warning {{'#pragma omp declare variant' cannot be applied to the function that was defined already; the original function might be used}} expected-note {{context set options are: 'construct' 'device' 'target_device' 'implementation' 'user'}} expected-note {{the ignored set spans until here}}
 int defined1(void);
 
 
 int diff_cc_variant(void);
 
-#pragma omp declare variant(diff_cc_variant) match(xxx={}) // expected-error {{variant in '#pragma omp declare variant' with type 'int (void)' is incompatible with type 'int (void) __attribute__((vectorcall))'}} expected-warning {{'xxx' is not a valid context set in a `declare variant`; set ignored}} expected-note {{context set options are: 'construct' 'device' 'implementation' 'user'}} expected-note {{the ignored set spans until here}}
+#pragma omp declare variant(diff_cc_variant) match(xxx={}) // expected-error {{variant in '#pragma omp declare variant' with type 'int (void)' is incompatible with type 'int (void) __attribute__((vectorcall))'}} expected-warning {{'xxx' is not a valid context set in a `declare variant`; set ignored}} expected-note {{context set options are: 'construct' 'device' 'target_device' 'implementation' 'user'}} expected-note {{the ignored set spans until here}}
 __vectorcall int diff_cc(void);
 
 int diff_ret_variant(void);
 
-#pragma omp declare variant(diff_ret_variant) match(xxx={}) // expected-error {{variant in '#pragma omp declare variant' with type 'int (void)' is incompatible with type 'void (void)'}} expected-warning {{'xxx' is not a valid context set in a `declare variant`; set ignored}} expected-note {{context set options are: 'construct' 'device' 'implementation' 'user'}} expected-note {{the ignored set spans until here}}
+#pragma omp declare variant(diff_ret_variant) match(xxx={}) // expected-error {{variant in '#pragma omp declare variant' with type 'int (void)' is incompatible with type 'void (void)'}} expected-warning {{'xxx' is not a valid context set in a `declare variant`; set ignored}} expected-note {{context set options are: 'construct' 'device' 'target_device' 'implementation' 'user'}} expected-note {{the ignored set spans until here}}
 void diff_ret(void);
 
 void incompat_attr_variant(void);
@@ -173,7 +183,7 @@ void not_marked(void);
 #pragma omp declare variant(not_marked) match(implementation={vendor(unknown)}, device={kind(cpu)}) // expected-note {{marked as 'declare variant' here}}
 void marked_variant(void);
 
-#pragma omp declare variant(marked_variant) match(xxx={}) // expected-warning {{'xxx' is not a valid context set in a `declare variant`; set ignored}} expected-warning {{variant function in '#pragma omp declare variant' is itself marked as '#pragma omp declare variant'}} expected-note {{context set options are: 'construct' 'device' 'implementation' 'user'}} expected-note {{the ignored set spans until here}}
+#pragma omp declare variant(marked_variant) match(xxx={}) // expected-warning {{'xxx' is not a valid context set in a `declare variant`; set ignored}} expected-warning {{variant function in '#pragma omp declare variant' is itself marked as '#pragma omp declare variant'}} expected-note {{context set options are: 'construct' 'device' 'target_device' 'implementation' 'user'}} expected-note {{the ignored set spans until here}}
 void marked(void);
 
 #pragma omp declare variant(foo) match(device = {isa("foo")})
@@ -214,3 +224,4 @@ int conflicting_nested_score(void);
 #pragma omp declare variant(foo) match(user = {condition(1)}) // expected-error {{nested user conditions in OpenMP context selector not supported (yet)}}
 int conflicting_nested_condition(void);
 #pragma omp end declare variant
+
diff --git a/clang/test/OpenMP/declare_variant_messages.cpp b/clang/test/OpenMP/declare_variant_messages.cpp
index b8a806e7ef75b..2d6e5bfc1f2f2 100644
--- a/clang/test/OpenMP/declare_variant_messages.cpp
+++ b/clang/test/OpenMP/declare_variant_messages.cpp
@@ -21,8 +21,8 @@ T foofoo();
 #pragma omp declare variant(foofoo <int>) // omp50-error {{expected 'match' clause on 'omp declare variant' directive}} omp51-error {{expected 'match', 'adjust_args', or 'append_args' clause on 'omp declare variant' directive}}
 #pragma omp declare variant(foofoo <int>) xxx // omp50-error {{expected 'match' clause on 'omp declare variant' directive}} omp51-error {{expected 'match', 'adjust_args', or 'append_args' clause on 'omp declare variant' directive}}
 #pragma omp declare variant(foofoo <int>) match // expected-error {{expected '(' after 'match'}}
-#pragma omp declare variant(foofoo <int>) match( // expected-error {{expected ')'}} expected-warning {{expected identifier or string literal describing a context set; set skipped}} expected-note {{context set options are: 'construct' 'device' 'implementation' 'user'}} expected-note {{the ignored set spans until here}} expected-note {{to match this '('}}
-#pragma omp declare variant(foofoo <int>) match() // expected-warning {{expected identifier or string literal describing a context set; set skipped}} expected-note {{context set options are: 'construct' 'device' 'implementation' 'user'}} expected-note {{the ignored set spans until here}}
+#pragma omp declare variant(foofoo <int>) match( // expected-error {{expected ')'}} expected-warning {{expected identifier or string literal describing a context set; set skipped}} expected-note {{context set options are: 'construct' 'device' 'target_device' 'implementation' 'user'}} expected-note {{the ignored set spans until here}} expected-note {{to match this '('}}
+#pragma omp declare variant(foofoo <int>) match() // expected-warning {{expected identifier or string literal describing a context set; set skipped}} expected-note {{context set options are: 'construct' 'device' 'target_device' 'implementation' 'user'}} expected-note {{the ignored set spans until here}}
 #pragma omp declare variant(foofoo <int>) match(implementation) // expected-warning {{expected '=' after the context set name "implementation"; '=' assumed}} expected-warning {{expected '{' after the '=' that follows the context set name "implementation"; '{' assumed}} expected-warning {{expected identifier or string literal describing a context selector; selector skipped}} expected-warning {{expected '}' after the context selectors for the context set "implementation"; '}' assumed}} expected-note {{context selector options are: 'vendor' 'extension' 'unified_address' 'unified_shared_memory' 'reverse_offload' 'dynamic_allocators' 'atomic_default_mem_order'}} expected-note {{the ignored selector spans until here}}
 #pragma omp declare variant(foofoo <int>) match(implementation =) // expected-warning {{expected '{' after the '=' that follows the context set name "implementation"; '{' assumed}} expected-warning {{expected identifier or string literal describing a context selector; selector skipped}} expected-warning {{expected '}' after the context selectors for the context set "implementation"; '}' assumed}} expected-note {{context selector options are: 'vendor' 'extension' 'unified_address' 'unified_shared_memory' 'reverse_offload' 'dynamic_allocators' 'atomic_default_mem_order'}} expected-note {{the ignored selector spans until here}}
 #pragma omp declare variant(foofoo <int>) match(implementation = yyy) // expected-warning {{expected '{' after the '=' that follows the context set name "implementation"; '{' assumed}} expected-warning {{'yyy' is not a valid context selector for the context set 'implementation'; selector ignored}} expected-warning {{expected '}' after the context selectors for the context set "implementation"; '}' assumed}} expected-note {{context selector options are: 'vendor' 'extension' 'unified_address' 'unified_shared_memory' 'reverse_offload' 'dynamic_allocators' 'atomic_default_mem_order'}} expected-note {{the ignored selector spans until here}}
@@ -63,8 +63,8 @@ int bar();
 #pragma omp declare variant(foofoo <T>) // omp50-error {{expected 'match' clause on 'omp declare variant' directive}} omp51-error {{expected 'match', 'adjust_args', or 'append_args' clause on 'omp declare variant' directive}}
 #pragma omp declare variant(foofoo <T>) xxx // omp50-error {{expected 'match' clause on 'omp declare variant' directive}} omp51-error {{expected 'match', 'adjust_args', or 'append_args' clause on 'omp declare variant' directive}}
 #pragma omp declare variant(foofoo <T>) match // expected-error {{expected '(' after 'match'}}
-#pragma omp declare variant(foofoo <T>) match( // expected-error {{expected ')'}} expected-warning {{expected identifier or string literal describing a context set; set skipped}} expected-note {{context set options are: 'construct' 'device' 'implementation' 'user'}} expected-note {{the ignored set spans until here}} expected-note {{to match this '('}}
-#pragma omp declare variant(foofoo <T>) match() // expected-warning {{expected identifier or string literal describing a context set; set skipped}} expected-note {{context set options are: 'construct' 'device' 'implementation' 'user'}} expected-note {{the ignored set spans until here}}
+#pragma omp declare variant(foofoo <T>) match( // expected-error {{expected ')'}} expected-warning {{expected identifier or string literal describing a context set; set skipped}} expected-note {{context set options are: 'construct' 'device' 'target_device' 'implementation' 'user'}} expected-note {{the ignored set spans until here}} expected-note {{to match this '('}}
+#pragma omp declare variant(foofoo <T>) match() // expected-warning {{expected identifier or string literal describing a context set; set skipped}} expected-note {{context set options are: 'construct' 'device' 'target_device' 'implementation' 'user'}} expected-note {{the ignored set spans until here}}
 #pragma omp declare variant(foofoo <T>) match(implementation) // expected-warning {{expected '=' after the context set name "implementation"; '=' assumed}} expected-warning {{expected '{' after the '=' that follows the context set name "implementation"; '{' assumed}} expected-warning {{expected identifier or string literal describing a context selector; selector skipped}} expected-warning {{expected '}' after the context selectors for the context set "implementation"; '}' assumed}} expected-note {{context selector options are: 'vendor' 'extension' 'unified_address' 'unified_shared_memory' 'reverse_offload' 'dynamic_allocators' 'atomic_default_mem_order'}} expected-note {{the ignored selector spans until here}}
 #pragma omp declare variant(foofoo <T>) match(implementation =) // expected-warning {{expected '{' after the '=' that follows the context set name "implementation"; '{' assumed}} expected-warning {{expected identifier or string literal describing a context selector; selector skipped}} expected-warning {{expected '}' after the context selectors for the context set "implementation"; '}' assumed}} expected-note {{context selector options are: 'vendor' 'extension' 'unified_address' 'unified_shared_memory' 'reverse_offload' 'dynamic_allocators' 'atomic_default_mem_order'}} expected-note {{the ignored selector spans until here}}
 #pragma omp declare variant(foofoo <T>) match(implementation = {) // expected-warning {{expected identifier or string literal describing a context selector; selector skipped}} expected-warning {{expected '}' after the context selectors for the context set "implementation"; '}' assumed}} expected-note {{context selector options are: 'vendor' 'extension' 'unified_address' 'unified_shared_memory' 'reverse_offload' 'dynamic_allocators' 'atomic_default_mem_order'}} expected-note {{the ignored selector spans until here}}
@@ -200,7 +200,7 @@ int after_use(void);
 
 int fn();
 int fn(int);
-#pragma omp declare variant(fn) match(xxx = {}) // expected-warning {{'xxx' is not a valid context set in a `declare variant`; set ignored}} expected-note {{context set options are: 'construct' 'device' 'implementation' 'user'}} expected-note {{the ignored set spans until here}}
+#pragma omp declare variant(fn) match(xxx = {}) // expected-warning {{'xxx' is not a valid context set in a `declare variant`; set ignored}} expected-note {{context set options are: 'construct' 'device' 'target_device' 'implementation' 'user'}} expected-note {{the ignored set spans until here}}
 int overload(void);
 
 int fn1();
@@ -211,7 +211,7 @@ int overload1(float);
 
 int fn_constexpr_variant();
 
-#pragma omp declare variant(fn_constexpr_variant) match(xxx = {}) // expected-warning {{'xxx' is not a valid context set in a `declare variant`; set ignored}} expected-note {{context set options are: 'construct' 'device' 'implementation' 'user'}} expected-note {{the ignored set spans until here}}
+#pragma omp declare variant(fn_constexpr_variant) match(xxx = {}) // expected-warning {{'xxx' is not a valid context set in a `declare variant`; set ignored}} expected-note {{context set options are: 'construct' 'device' 'target_device' 'implementation' 'user'}} expected-note {{the ignored set spans until here}}
 constexpr int fn_constexpr(); // expected-error {{'#pragma omp declare variant' does not support constexpr functions}}
 
 constexpr int fn_constexpr_variant1();
@@ -221,7 +221,7 @@ int fn_constexpr1();
 
 int fn_sc_variant();
 
-#pragma omp declare variant(fn_sc_variant) match(xxx = {}) // expected-error {{function with '#pragma omp declare variant' has a different storage class}} expected-warning {{'xxx' is not a valid context set in a `declare variant`; set ignored}} expected-note {{context set options are: 'construct' 'device' 'implementation' 'user'}} expected-note {{the ignored set spans until here}}
+#pragma omp declare variant(fn_sc_variant) match(xxx = {}) // expected-error {{function with '#pragma omp declare variant' has a different storage class}} expected-warning {{'xxx' is not a valid context set in a `declare variant`; set ignored}} expected-note {{context set options are: 'construct' 'device' 'target_device' 'implementation' 'user'}} expected-note {{the ignored set spans until here}}
 static int fn_sc();
 
 static int fn_sc_variant1();
@@ -231,7 +231,7 @@ int fn_sc1();
 
 int fn_inline_variant();
 
-#pragma omp declare variant(fn_inline_variant) match(xxx = {}) // expected-error {{function with '#pragma omp declare variant' has a different inline specification}} expected-warning {{'xxx' is not a valid context set in a `declare variant`; set ignored}} expected-note {{context set options are: 'construct' 'device' 'implementation' 'user'}} expected-note {{the ignored set spans until here}}
+#pragma omp declare variant(fn_inline_variant) match(xxx = {}) // expected-error {{function with '#pragma omp declare variant' has a different inline specification}} expected-warning {{'xxx' is not a valid context set in a `declare variant`; set ignored}} expected-note {{context set options are: 'construct' 'device' 'target_device' 'implementation' 'user'}} expected-note {{the ignored set spans until here}}
 inline int fn_inline();
 
 inline int fn_inline_variant1();
@@ -240,7 +240,7 @@ inline int fn_inline_variant1();
 int fn_inline1();
 
 auto fn_deduced_variant() { return 0; }
-#pragma omp declare variant(fn_deduced_variant) match(xxx = {}) // expected-warning {{'xxx' is not a valid context set in a `declare variant`; set ignored}} expected-note {{context set options are: 'construct' 'device' 'implementation' 'user'}} expected-note {{the ignored set spans until here}}
+#pragma omp declare variant(fn_deduced_variant) match(xxx = {}) // expected-warning {{'xxx' is not a valid context set in a `declare variant`; set ignored}} expected-note {{context set options are: 'construct' 'device' 'target_device' 'implementation' 'user'}} expected-note {{the ignored set spans until here}}
 int fn_deduced();
 
 int fn_deduced_variant1();
@@ -254,7 +254,7 @@ auto fn_deduced3();
 
 auto fn_deduced_variant2() { return 0; }
 
-#pragma omp declare variant(fn_deduced_variant2) match(xxx = {}) // expected-error {{variant in '#pragma omp declare variant' with type 'int ()' is incompatible with type 'float ()'}} expected-warning {{'xxx' is not a valid context set in a `declare variant`; set ignored}} expected-note {{context set options are: 'construct' 'device' 'implementation' 'user'}} expected-note {{the ignored set spans until here}}
+#pragma omp declare variant(fn_deduced_variant2) match(xxx = {}) // expected-error {{variant in '#pragma omp declare variant' with type 'int ()' is incompatible with type 'float ()'}} expected-warning {{'xxx' is not a valid context set in a `declare variant`; set ignored}} expected-note {{context set options are: 'construct' 'device' 'target_device' 'implementation' 'user'}} expected-note {{the ignored set spans until here}}
 float fn_deduced2();
 
 
@@ -266,7 +266,7 @@ int fn_except() noexcept(false); // expected-note {{previous declaration is here
 
 int fn_except_variant1() noexcept(false); // expected-error {{exception specification in declaration does not match previous declaration}}
 
-#pragma omp declare variant(fn_except_variant1) match(xxx = {}) // expected-warning {{'xxx' is not a valid context set in a `declare variant`; set ignored}} expected-note {{context set options are: 'construct' 'device' 'implementation' 'user'}} expected-note {{the ignored set spans until here}}
+#pragma omp declare variant(fn_except_variant1) match(xxx = {}) // expected-warning {{'xxx' is not a valid context set in a `declare variant`; set ignored}} expected-note {{context set options are: 'construct' 'device' 'target_device' 'implementation' 'user'}} expected-note {{the ignored set spans until here}}
 int fn_except1() noexcept(true); // expected-note {{previous declaration is here}}
 
 struct SpecialFuncs {
@@ -275,7 +275,7 @@ struct SpecialFuncs {
 #pragma omp declare variant(SpecialFuncs::vd) match(implementation = {}) // expected-warning {{expected identifier or string literal describing a context selector; selector skipped}} expected-note {{context selector options are: 'vendor' 'extension' 'unified_address' 'unified_shared_memory' 'reverse_offload' 'dynamic_allocators' 'atomic_default_mem_order'}} expected-note {{the ignored selector spans until here}}
   SpecialFuncs(); // expected-error {{'#pragma omp declare variant' does not support constructors}}
 
-#pragma omp declare variant(SpecialFuncs::vd) match(xxx = {}) // expected-warning {{'xxx' is not a valid context set in a `declare variant`; set ignored}} expected-note {{context set options are: 'construct' 'device' 'implementation' 'user'}} expected-note {{the ignored set spans until here}}
+#pragma omp declare variant(SpecialFuncs::vd) match(xxx = {}) // expected-warning {{'xxx' is not a valid context set in a `declare variant`; set ignored}} expected-note {{context set options are: 'construct' 'device' 'target_device' 'implementation' 'user'}} expected-note {{the ignored set spans until here}}
   ~SpecialFuncs(); // expected-error {{'#pragma omp declare variant' does not support destructors}}
 
   void baz();
@@ -288,14 +288,14 @@ struct SpecialFuncs {
 #pragma omp declare variant(SpecialFuncs::dar) match(construct={dispatch}) // expected-error {{'#pragma omp declare variant' does not support virtual functions}}
 
 #pragma omp declare variant(SpecialFuncs::baz) match(implementation = {}) // expected-warning {{expected identifier or string literal describing a context selector; selector skipped}} expected-note {{context selector options are: 'vendor' 'extension' 'unified_address' 'unified_shared_memory' 'reverse_offload' 'dynamic_allocators' 'atomic_default_mem_order'}} expected-note {{the ignored selector spans until here}}
-#pragma omp declare variant(SpecialFuncs::bar) match(xxx = {}) // expected-warning {{'xxx' is not a valid context set in a `declare variant`; set ignored}} expected-note {{context set options are: 'construct' 'device' 'implementation' 'user'}} expected-note {{the ignored set spans until here}}
+#pragma omp declare variant(SpecialFuncs::bar) match(xxx = {}) // expected-warning {{'xxx' is not a valid context set in a `declare variant`; set ignored}} expected-note {{context set options are: 'construct' 'device' 'target_device' 'implementation' 'user'}} expected-note {{the ignored set spans until here}}
 
 #pragma omp declare variant(fn_sc_variant1) match(implementation = {}) // expected-error {{variant in '#pragma omp declare variant' with type 'int (*)()' is incompatible with type 'void (SpecialFuncs::*)()'}} expected-warning {{expected identifier or string literal describing a context selector; selector skipped}} expected-note {{context selector options are: 'vendor' 'extension' 'unified_address' 'unified_shared_memory' 'reverse_offload' 'dynamic_allocators' 'atomic_default_mem_order'}} expected-note {{the ignored selector spans until here}}
   void foo1();
   SpecialFuncs& foo(const SpecialFuncs&);
   SpecialFuncs& bar(SpecialFuncs&&);
 
-#pragma omp declare variant(SpecialFuncs::foo) match(xxx = {}) // expected-warning {{'xxx' is not a valid context set in a `declare variant`; set ignored}} expected-note {{context set options are: 'construct' 'device' 'implementation' 'user'}} expected-note {{the ignored set spans until here}}
+#pragma omp declare variant(SpecialFuncs::foo) match(xxx = {}) // expected-warning {{'xxx' is not a valid context set in a `declare variant`; set ignored}} expected-note {{context set options are: 'construct' 'device' 'target_device' 'implementation' 'user'}} expected-note {{the ignored set spans until here}}
   SpecialFuncs& operator=(const SpecialFuncs&) = default; // expected-error {{'#pragma omp declare variant' does not support defaulted functions}}
 
 #pragma omp declare variant(SpecialFuncs::bar) match(implementation = {}) // expected-warning {{expected identifier or string literal describing a context selector; selector skipped}} expected-note {{context selector options are: 'vendor' 'extension' 'unified_address' 'unified_shared_memory' 'reverse_offload' 'dynamic_allocators' 'atomic_default_mem_order'}} expected-note {{the ignored selector spans until here}}
diff --git a/clang/test/OpenMP/depobj_messages.cpp b/clang/test/OpenMP/depobj_messages.cpp
index 9d750f651d81f..a57ae0200e086 100644
--- a/clang/test/OpenMP/depobj_messages.cpp
+++ b/clang/test/OpenMP/depobj_messages.cpp
@@ -4,6 +4,8 @@
 // RUN: -ferror-limit 100 %s -Wuninitialized
 // RUN: %clang_cc1 -verify=expected,omp52 -fopenmp -fopenmp-version=52 \
 // RUN: -ferror-limit 100 %s -Wuninitialized
+// RUN: %clang_cc1 -verify=expected,omp52 -fopenmp -fopenmp-version=60 \
+// RUN: -ferror-limit 100 %s -Wuninitialized
 
 // RUN: %clang_cc1 -verify=expected,omp50 -fopenmp-simd -fopenmp-version=50 \
 // RUN: -ferror-limit 100 %s -Wuninitialized
@@ -11,6 +13,8 @@
 // RUN: -ferror-limit 100 %s -Wuninitialized
 // RUN: %clang_cc1 -verify=expected,omp52 -fopenmp-simd -fopenmp-version=52 \
 // RUN: -ferror-limit 100 %s -Wuninitialized
+// RUN: %clang_cc1 -verify=expected,omp52 -fopenmp-simd -fopenmp-version=60 \
+// RUN: -ferror-limit 100 %s -Wuninitialized
 
 struct S1 { // expected-note 2 {{declared here}}
   int a;
diff --git a/clang/test/OpenMP/distribute_parallel_for_ast_print.cpp b/clang/test/OpenMP/distribute_parallel_for_ast_print.cpp
index 7af4e5f6b2b44..f91afd7ae1141 100644
--- a/clang/test/OpenMP/distribute_parallel_for_ast_print.cpp
+++ b/clang/test/OpenMP/distribute_parallel_for_ast_print.cpp
@@ -7,6 +7,9 @@
 // RUN: %clang_cc1 -verify -std=c++11 -fopenmp -fopenmp-version=51 -ast-print %s -Wno-openmp-mapping -DOMP51 | FileCheck %s --check-prefix=CHECK --check-prefix=OMP51
 // RUN: %clang_cc1 -fopenmp -fopenmp-version=51 -x c++ -std=c++11 -emit-pch -o %t %s -DOMP51
 // RUN: %clang_cc1 -fopenmp -fopenmp-version=51 -std=c++11 -include-pch %t -verify %s -ast-print -Wno-openmp-mapping -DOMP51 | FileCheck %s --check-prefix=CHECK --check-prefix=OMP51
+// RUN: %clang_cc1 -verify -std=c++11 -fopenmp -fopenmp-version=60 -ast-print %s -Wno-openmp-mapping -DOMP51 | FileCheck %s --check-prefix=CHECK --check-prefix=OMP51
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=60 -x c++ -std=c++11 -emit-pch -o %t %s -DOMP51
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=60 -std=c++11 -include-pch %t -verify %s -ast-print -Wno-openmp-mapping -DOMP51 | FileCheck %s --check-prefix=CHECK --check-prefix=OMP51
 
 // RUN: %clang_cc1 -verify -std=c++11 -fopenmp-simd -fopenmp-version=45 -ast-print %s -Wno-openmp-mapping | FileCheck %s --check-prefix=CHECK --check-prefix=OMP45
 // RUN: %clang_cc1 -fopenmp-simd -fopenmp-version=45 -x c++ -std=c++11 -emit-pch -o %t %s
@@ -17,6 +20,9 @@
 // RUN: %clang_cc1 -verify -std=c++11 -fopenmp-simd -fopenmp-version=51 -ast-print %s -Wno-openmp-mapping -DOMP51 | FileCheck %s --check-prefix=CHECK --check-prefix=OMP51
 // RUN: %clang_cc1 -fopenmp-simd -fopenmp-version=51 -x c++ -std=c++11 -emit-pch -o %t %s -DOMP51
 // RUN: %clang_cc1 -fopenmp-simd -fopenmp-version=51 -std=c++11 -include-pch %t -verify %s -ast-print -Wno-openmp-mapping -DOMP51 | FileCheck %s --check-prefix=CHECK --check-prefix=OMP51
+// RUN: %clang_cc1 -verify -std=c++11 -fopenmp-simd -fopenmp-version=60 -ast-print %s -Wno-openmp-mapping -DOMP51 | FileCheck %s --check-prefix=CHECK --check-prefix=OMP51
+// RUN: %clang_cc1 -fopenmp-simd -fopenmp-version=60 -x c++ -std=c++11 -emit-pch -o %t %s -DOMP51
+// RUN: %clang_cc1 -fopenmp-simd -fopenmp-version=60 -std=c++11 -include-pch %t -verify %s -ast-print -Wno-openmp-mapping -DOMP51 | FileCheck %s --check-prefix=CHECK --check-prefix=OMP51
 // expected-no-diagnostics
 
 #ifndef HEADER
diff --git a/clang/test/OpenMP/distribute_parallel_for_simd_ast_print.cpp b/clang/test/OpenMP/distribute_parallel_for_simd_ast_print.cpp
index ae3caa88d4547..0495bff45b72a 100644
--- a/clang/test/OpenMP/distribute_parallel_for_simd_ast_print.cpp
+++ b/clang/test/OpenMP/distribute_parallel_for_simd_ast_print.cpp
@@ -7,6 +7,9 @@
 // RUN: %clang_cc1 -verify -fopenmp -fopenmp-version=51 -ast-print %s -Wno-openmp-mapping -DOMP51 | FileCheck %s --check-prefix CHECK --check-prefix OMP51
 // RUN: %clang_cc1 -fopenmp -fopenmp-version=51 -x c++ -std=c++11 -emit-pch -o %t %s -DOMP51
 // RUN: %clang_cc1 -fopenmp -fopenmp-version=51 -std=c++11 -include-pch %t -verify %s -ast-print -Wno-openmp-mapping -DOMP51 | FileCheck %s --check-prefix CHECK --check-prefix OMP51
+// RUN: %clang_cc1 -verify -fopenmp -fopenmp-version=60 -ast-print %s -Wno-openmp-mapping -DOMP51 | FileCheck %s --check-prefix CHECK --check-prefix OMP51
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=60 -x c++ -std=c++11 -emit-pch -o %t %s -DOMP51
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=60 -std=c++11 -include-pch %t -verify %s -ast-print -Wno-openmp-mapping -DOMP51 | FileCheck %s --check-prefix CHECK --check-prefix OMP51
 
 // RUN: %clang_cc1 -verify -fopenmp-simd -fopenmp-version=45 -ast-print %s -Wno-openmp-mapping | FileCheck %s --check-prefix CHECK --check-prefix OMP45
 // RUN: %clang_cc1 -fopenmp-simd -fopenmp-version=45 -x c++ -std=c++11 -emit-pch -o %t %s
@@ -17,6 +20,9 @@
 // RUN: %clang_cc1 -verify -fopenmp-simd -fopenmp-version=51 -ast-print %s -Wno-openmp-mapping -DOMP51 | FileCheck %s --check-prefix CHECK --check-prefix OMP51
 // RUN: %clang_cc1 -fopenmp-simd -fopenmp-version=51 -x c++ -std=c++11 -emit-pch -o %t %s -DOMP51
 // RUN: %clang_cc1 -fopenmp-simd -fopenmp-version=51 -std=c++11 -include-pch %t -verify %s -ast-print -Wno-openmp-mapping -DOMP51 | FileCheck %s --check-prefix CHECK --check-prefix OMP51
+// RUN: %clang_cc1 -verify -fopenmp-simd -fopenmp-version=60 -ast-print %s -Wno-openmp-mapping -DOMP51 | FileCheck %s --check-prefix CHECK --check-prefix OMP51
+// RUN: %clang_cc1 -fopenmp-simd -fopenmp-version=60 -x c++ -std=c++11 -emit-pch -o %t %s -DOMP51
+// RUN: %clang_cc1 -fopenmp-simd -fopenmp-version=60 -std=c++11 -include-pch %t -verify %s -ast-print -Wno-openmp-mapping -DOMP51 | FileCheck %s --check-prefix CHECK --check-prefix OMP51
 // expected-no-diagnostics
 
 #ifndef HEADER
diff --git a/clang/test/OpenMP/distribute_simd_ast_print.cpp b/clang/test/OpenMP/distribute_simd_ast_print.cpp
index 9130203008311..0fbc0e1342a57 100644
--- a/clang/test/OpenMP/distribute_simd_ast_print.cpp
+++ b/clang/test/OpenMP/distribute_simd_ast_print.cpp
@@ -4,6 +4,11 @@
 // RUN: %clang_cc1 -verify -fopenmp -ast-print %s -Wno-openmp-mapping -DOMP5 | FileCheck %s --check-prefix=CHECK --check-prefix=OMP50
 // RUN: %clang_cc1 -fopenmp -x c++ -std=c++11 -emit-pch -o %t %s -DOMP5
 // RUN: %clang_cc1 -fopenmp -std=c++11 -include-pch %t -verify %s -ast-print -Wno-openmp-mapping -DOMP5 | FileCheck %s --check-prefix=CHECK --check-prefix=OMP50
+
+// RUN: %clang_cc1 -verify -fopenmp -fopenmp-version=60 -ast-print %s -Wno-openmp-mapping -DOMP51 | FileCheck %s --check-prefix=CHECK --check-prefix=OMP51
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=60 -x c++ -std=c++11 -emit-pch -o %t %s -DOMP51
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=60 -std=c++11 -include-pch %t -verify %s -ast-print -Wno-openmp-mapping -DOMP51 | FileCheck %s --check-prefix=CHECK --check-prefix=OMP51
+
 // RUN: %clang_cc1 -verify -fopenmp -fopenmp-version=51 -ast-print %s -Wno-openmp-mapping -DOMP51 | FileCheck %s --check-prefix=CHECK --check-prefix=OMP51
 // RUN: %clang_cc1 -fopenmp -fopenmp-version=51 -x c++ -std=c++11 -emit-pch -o %t %s -DOMP51
 // RUN: %clang_cc1 -fopenmp -fopenmp-version=51 -std=c++11 -include-pch %t -verify %s -ast-print -Wno-openmp-mapping -DOMP51 | FileCheck %s --check-prefix=CHECK --check-prefix=OMP51
@@ -20,6 +25,9 @@
 // RUN: %clang_cc1 -verify -fopenmp-simd -fopenmp-version=51 -ast-print %s -Wno-openmp-mapping -DOMP51 | FileCheck %s --check-prefix=CHECK --check-prefix=OMP51
 // RUN: %clang_cc1 -fopenmp-simd -fopenmp-version=51 -x c++ -std=c++11 -emit-pch -o %t %s -DOMP51
 // RUN: %clang_cc1 -fopenmp-simd -fopenmp-version=51 -std=c++11 -include-pch %t -verify %s -ast-print -Wno-openmp-mapping -DOMP51 | FileCheck %s --check-prefix=CHECK --check-prefix=OMP51
+// RUN: %clang_cc1 -verify -fopenmp-simd -fopenmp-version=60 -ast-print %s -Wno-openmp-mapping -DOMP51 | FileCheck %s --check-prefix=CHECK --check-prefix=OMP51
+// RUN: %clang_cc1 -fopenmp-simd -fopenmp-version=60 -x c++ -std=c++11 -emit-pch -o %t %s -DOMP51
+// RUN: %clang_cc1 -fopenmp-simd -fopenmp-version=60 -std=c++11 -include-pch %t -verify %s -ast-print -Wno-openmp-mapping -DOMP51 | FileCheck %s --check-prefix=CHECK --check-prefix=OMP51
 // expected-no-diagnostics
 // RUN: %clang_cc1 -verify -fopenmp-simd -fopenmp-version=52 -ast-print %s -Wno-openmp-mapping -DOMP52 | FileCheck %s --check-prefix=CHECK --check-prefix=OMP52
 // RUN: %clang_cc1 -fopenmp-simd -fopenmp-version=52 -x c++ -std=c++11 -emit-pch -o %t %s -DOMP52
diff --git a/clang/test/OpenMP/distribute_simd_misc_messages.c b/clang/test/OpenMP/distribute_simd_misc_messages.c
index f78b0ea8c56ce..8cbf96cd7a014 100644
--- a/clang/test/OpenMP/distribute_simd_misc_messages.c
+++ b/clang/test/OpenMP/distribute_simd_misc_messages.c
@@ -1,16 +1,20 @@
 // RUN: %clang_cc1 -fsyntax-only -fopenmp -fopenmp-version=45 -verify=expected,omp45 %s -Wuninitialized
-// RUN: %clang_cc1 -fsyntax-only -fopenmp -fopenmp-version=50 -verify=expected,omp50 %s -Wuninitialized
-// RUN: %clang_cc1 -fsyntax-only -fopenmp -verify=expected,omp51 %s -Wuninitialized
+// RUN: %clang_cc1 -fsyntax-only -fopenmp -fopenmp-version=50 -verify=expected,omp50,omp-50-and-later,omp-50-and-later-temporal,omp50-temporal,omp50-and-later-var %s -Wuninitialized
+// RUN: %clang_cc1 -fsyntax-only -fopenmp -fopenmp-version=60 -verify=expected,omp60,omp-clause,omp-50-and-later,omp-50-and-later-temporal,omp50-temporal,omp50-and-later-var %s -Wuninitialized
+// RUN: %clang_cc1 -fsyntax-only -fopenmp -verify=expected,omp51,omp-clause,omp-50-and-later,omp-50-and-later-temporal,omp50-temporal,omp50-and-later-var %s -Wuninitialized
 
 // RUN: %clang_cc1 -fsyntax-only -fopenmp-simd -fopenmp-version=45 -verify=expected,omp45 %s -Wuninitialized
-// RUN: %clang_cc1 -fsyntax-only -fopenmp-simd -fopenmp-version=50 -verify=expected,omp50 %s -Wuninitialized
-// RUN: %clang_cc1 -fsyntax-only -fopenmp-simd -verify=expected,omp51 %s -Wuninitialized
+// RUN: %clang_cc1 -fsyntax-only -fopenmp-simd -fopenmp-version=50 -verify=expected,omp50,omp-50-and-later,omp-50-and-later-temporal,omp50-temporal,omp50-and-later-var %s -Wuninitialized
+// RUN: %clang_cc1 -fsyntax-only -fopenmp-simd -fopenmp-version=60 -verify=expected,omp60,omp-clause,omp-50-and-later,omp-50-and-later-temporal,omp50-temporal,omp50-and-later-var %s -Wuninitialized
+// RUN: %clang_cc1 -fsyntax-only -fopenmp-simd -verify=expected,omp51,omp-clause,omp-50-and-later,omp-50-and-later-temporal,omp50-temporal,omp50-and-later-var %s -Wuninitialized
 
 void xxx(int argc) {
-  int x; // expected-note {{initialize the variable 'x' to silence this warning}}
+  // expected-note@+1 {{initialize the variable 'x' to silence this warning}}
+  int x;
 #pragma omp distribute simd
   for (int i = 0; i < 10; ++i)
-    argc = x; // expected-warning {{variable 'x' is uninitialized when used here}}
+    // expected-warning@+1 {{variable 'x' is uninitialized when used here}}
+    argc = x;
 }
 
 // expected-error@+1 {{unexpected OpenMP directive '#pragma omp distribute simd'}}
@@ -49,9 +53,11 @@ void test_branch_protected_scope(void) {
 #pragma omp distribute simd
   for (i = 0; i < 16; ++i) {
     if (i == 5)
-      goto L1; // expected-error {{use of undeclared label 'L1'}}
+      // expected-error@+1 {{use of undeclared label 'L1'}}
+      goto L1;
     else if (i == 6)
-      return; // expected-error {{cannot return from OpenMP region}}
+      // expected-error@+1 {{cannot return from OpenMP region}}
+      return;
     else if (i == 7)
       goto L2;
     else if (i == 8) {
@@ -61,7 +67,8 @@ void test_branch_protected_scope(void) {
   }
 
   if (x[0] == 0)
-    goto L2; // expected-error {{use of undeclared label 'L2'}}
+    // expected-error@+1 {{use of undeclared label 'L2'}}
+    goto L2;
   else if (x[1] == 1)
     goto L1;
 }
@@ -411,41 +418,49 @@ void test_collapse(void) {
 // expected-note@+1 {{to match this '('}} expected-note@+1 {{as specified in 'collapse' clause}}
 #pragma omp distribute simd collapse(4
   for (i = 0; i < 16; ++i)
-    ; // expected-error {{expected 4 for loops after '#pragma omp distribute simd', but found only 1}}
+// expected-error@+1 {{expected 4 for loops after '#pragma omp distribute simd', but found only 1}}
+    ;
 #pragma omp target
 #pragma omp teams
 // expected-error@+2 {{expected ')'}}
 // expected-note@+1 {{to match this '('}} expected-note@+1 {{as specified in 'collapse' clause}}
 #pragma omp distribute simd collapse(4,
   for (i = 0; i < 16; ++i)
-    ; // expected-error {{expected 4 for loops after '#pragma omp distribute simd', but found only 1}}
+// expected-error@+1 {{expected 4 for loops after '#pragma omp distribute simd', but found only 1}}
+    ;
 #pragma omp target
 #pragma omp teams
 // expected-error@+2 {{expected ')'}}
 // expected-note@+1 {{to match this '('}} expected-note@+1 {{as specified in 'collapse' clause}}
 #pragma omp distribute simd collapse(4, )
   for (i = 0; i < 16; ++i)
-    ; // expected-error {{expected 4 for loops after '#pragma omp distribute simd', but found only 1}}
+// expected-error@+1 {{expected 4 for loops after '#pragma omp distribute simd', but found only 1}}
+    ;
 #pragma omp target
 #pragma omp teams
-// xxpected-error@+1 {{expected expression}} expected-note@+1 {{as specified in 'collapse' clause}}
+// expected-note@+1 {{as specified in 'collapse' clause}}
 #pragma omp distribute simd collapse(4)
   for (i = 0; i < 16; ++i)
-    ; // expected-error {{expected 4 for loops after '#pragma omp distribute simd', but found only 1}}
+// expected-error@+1 {{expected 4 for loops after '#pragma omp distribute simd', but found only 1}}
+    ;
 #pragma omp target
 #pragma omp teams
-// expected-error@+2 {{expected ')'}}
-// expected-note@+1 {{to match this '('}} expected-note@+1 {{as specified in 'collapse' clause}}
+// expected-error@+3 {{expected ')'}}
+// expected-note@+2 {{to match this '('}}
+// expected-note@+1 {{as specified in 'collapse' clause}}
 #pragma omp distribute simd collapse(4 4)
   for (i = 0; i < 16; ++i)
-    ; // expected-error {{expected 4 for loops after '#pragma omp distribute simd', but found only 1}}
+// expected-error@+1 {{expected 4 for loops after '#pragma omp distribute simd', but found only 1}}
+    ;
 #pragma omp target
 #pragma omp teams
-// expected-error@+2 {{expected ')'}}
-// expected-note@+1 {{to match this '('}} expected-note@+1 {{as specified in 'collapse' clause}}
+// expected-error@+3 {{expected ')'}}
+// expected-note@+2 {{to match this '('}}
+// expected-note@+1 {{as specified in 'collapse' clause}}
 #pragma omp distribute simd collapse(4, , 4)
   for (i = 0; i < 16; ++i)
-    ; // expected-error {{expected 4 for loops after '#pragma omp distribute simd', but found only 1}}
+// expected-error@+1 {{expected 4 for loops after '#pragma omp distribute simd', but found only 1}}
+    ;
 #pragma omp target
 #pragma omp teams
 #pragma omp distribute simd collapse(4)
@@ -456,11 +471,13 @@ void test_collapse(void) {
           foo();
 #pragma omp target
 #pragma omp teams
-// expected-error@+2 {{expected ')'}}
-// expected-note@+1 {{to match this '('}} expected-note@+1 {{as specified in 'collapse' clause}}
+// expected-error@+3 {{expected ')'}}
+// expected-note@+2 {{to match this '('}}
+// expected-note@+1 {{as specified in 'collapse' clause}}
 #pragma omp distribute simd collapse(4, 8)
   for (i = 0; i < 16; ++i)
-    ; // expected-error {{expected 4 for loops after '#pragma omp distribute simd', but found only 1}}
+// expected-error@+1 {{expected 4 for loops after '#pragma omp distribute simd', but found only 1}}
+    ;
 #pragma omp target
 #pragma omp teams
 // expected-error@+1 {{integer constant expression}}
@@ -719,14 +736,16 @@ void test_private(void) {
   int i;
 #pragma omp target
 #pragma omp teams
-// expected-error@+2 {{expected expression}}
-// expected-error@+1 {{expected ')'}} expected-note@+1 {{to match this '('}}
+// expected-error@+3 {{expected expression}}
+// expected-error@+2 {{expected ')'}}
+// expected-note@+1 {{to match this '('}}
 #pragma omp distribute simd private(
   for (i = 0; i < 16; ++i)
     ;
 #pragma omp target
 #pragma omp teams
-// expected-error@+2 {{expected ')'}} expected-note@+2 {{to match this '('}}
+// expected-error@+3 {{expected ')'}}
+//  expected-note@+2 {{to match this '('}}
 // expected-error@+1 2 {{expected expression}}
 #pragma omp distribute simd private(,
   for (i = 0; i < 16; ++i)
@@ -779,7 +798,8 @@ void test_firstprivate(void) {
   int i;
 #pragma omp target
 #pragma omp teams
-// expected-error@+2 {{expected ')'}} expected-note@+2 {{to match this '('}}
+// expected-error@+3 {{expected ')'}}
+//  expected-note@+2 {{to match this '('}}
 // expected-error@+1 {{expected expression}}
 #pragma omp distribute simd firstprivate(
   for (i = 0; i < 16; ++i)
@@ -790,7 +810,8 @@ void test_lastprivate(void) {
   int i;
 #pragma omp target
 #pragma omp teams
-// expected-error@+2 {{expected ')'}} expected-note@+2 {{to match this '('}}
+// expected-error@+3 {{expected ')'}}
+// expected-note@+2 {{to match this '('}}
 // expected-error@+1 {{expected expression}}
 #pragma omp distribute simd lastprivate(
   for (i = 0; i < 16; ++i)
@@ -798,7 +819,8 @@ void test_lastprivate(void) {
 
 #pragma omp target
 #pragma omp teams
-// expected-error@+2 {{expected ')'}} expected-note@+2 {{to match this '('}}
+// expected-error@+3 {{expected ')'}}
+// expected-note@+2 {{to match this '('}}
 // expected-error@+1 2 {{expected expression}}
 #pragma omp distribute simd lastprivate(,
   for (i = 0; i < 16; ++i)
@@ -850,7 +872,8 @@ void test_reduction(void) {
   int i, x, y;
 #pragma omp target
 #pragma omp teams
-// expected-error@+3 {{expected ')'}} expected-note@+3 {{to match this '('}}
+// expected-error@+4 {{expected ')'}}
+// expected-note@+3 {{to match this '('}}
 // expected-error@+2 {{expected identifier}}
 // expected-warning@+1 {{missing ':' after reduction identifier - ignoring}}
 #pragma omp distribute simd reduction(
@@ -878,7 +901,8 @@ void test_reduction(void) {
     ;
 #pragma omp target
 #pragma omp teams
-// expected-error@+3 {{expected ')'}} expected-note@+3 {{to match this '('}}
+// expected-error@+4 {{expected ')'}}
+// expected-note@+3 {{to match this '('}}
 // expected-error@+2 {{expected identifier}}
 // expected-warning@+1 {{missing ':' after reduction identifier - ignoring}}
 #pragma omp distribute simd reduction(,
@@ -886,7 +910,8 @@ void test_reduction(void) {
     ;
 #pragma omp target
 #pragma omp teams
-// expected-error@+3 {{expected ')'}} expected-note@+3 {{to match this '('}}
+// expected-error@+4 {{expected ')'}}
+// expected-note@+3 {{to match this '('}}
 // expected-error@+2 {{expected expression}}
 // expected-warning@+1 {{missing ':' after reduction identifier - ignoring}}
 #pragma omp distribute simd reduction(+
@@ -895,8 +920,8 @@ void test_reduction(void) {
 
 #pragma omp target
 #pragma omp teams
-// expected-error@+3 {{expected ')'}} expected-note@+3 {{to match this '('}}
-//
+// expected-error@+3 {{expected ')'}}
+//  expected-note@+2 {{to match this '('}}
 // expected-error@+1 {{expected expression}}
 #pragma omp distribute simd reduction(+:
   for (i = 0; i < 16; ++i)
@@ -938,6 +963,7 @@ void test_reduction(void) {
     ;
 #pragma omp target
 #pragma omp teams
+  // omp60-error@+1 {{incorrect reduction identifier, expected one of '+', '*', '&', '|', '^', '&&', '||', 'min' or 'max' or declare reduction for type 'int'}}
 #pragma omp distribute simd reduction(- : x)
   for (i = 0; i < 16; ++i)
     ;
@@ -1020,77 +1046,103 @@ void linear_modifiers(int argc) {
   for (k = 0; k < argc; ++k) ++k;
 #pragma omp target
 #pragma omp teams
+  // omp60-error@+1 {{old syntax 'linear-modifier(list)' on 'linear' clause was deprecated, use new syntax 'linear(list: [linear-modifier,] step(step-size))'}}
 #pragma omp distribute simd linear(val(k))
   for (k = 0; k < argc; ++k) ++k;
 #pragma omp target
 #pragma omp teams
-#pragma omp distribute simd linear(uval(k)) // expected-error {{expected 'val' modifier}}
+  // omp60-error@+2 {{old syntax 'linear-modifier(list)' on 'linear' clause was deprecated, use new syntax 'linear(list: [linear-modifier,] step(step-size))'}}
+  // expected-error@+1 {{expected 'val' modifier}}
+#pragma omp distribute simd linear(uval(k))
   for (k = 0; k < argc; ++k) ++k;
 #pragma omp target
 #pragma omp teams
-#pragma omp distribute simd linear(ref(k)) // expected-error {{expected 'val' modifier}}
+// omp60-error@+2 {{old syntax 'linear-modifier(list)' on 'linear' clause was deprecated, use new syntax 'linear(list: [linear-modifier,] step(step-size))'}}
+// expected-error@+1 {{expected 'val' modifier}}
+#pragma omp distribute simd linear(ref(k))
   for (k = 0; k < argc; ++k) ++k;
 #pragma omp target
 #pragma omp teams
-#pragma omp distribute simd linear(foo(k)) // expected-error {{expected 'val' modifier}}
+// omp60-error@+2 {{old syntax 'linear-modifier(list)' on 'linear' clause was deprecated, use new syntax 'linear(list: [linear-modifier,] step(step-size))'}}
+// expected-error@+1 {{expected 'val' modifier}}
+#pragma omp distribute simd linear(foo(k))
   for (k = 0; k < argc; ++k) ++k;
 }
 
 void test_nontemporal(void) {
   int i;
-// omp45-error@+1 {{unexpected OpenMP clause 'nontemporal' in directive '#pragma omp distribute simd'}} expected-error@+1 {{expected expression}} expected-error@+1 {{expected ')'}} expected-note@+1 {{to match this '('}}
+// omp45-error@+4 {{unexpected OpenMP clause 'nontemporal' in directive '#pragma omp distribute simd'}}
+// expected-error@+3 {{expected expression}}
+// expected-error@+2 {{expected ')'}}
+// expected-note@+1 {{to match this '('}}
 #pragma omp distribute simd nontemporal(
   for (i = 0; i < 16; ++i)
     ;
-// omp45-error@+1 {{unexpected OpenMP clause 'nontemporal' in directive '#pragma omp distribute simd'}} expected-error@+1 2 {{expected expression}} expected-error@+1 {{expected ')'}} expected-note@+1 {{to match this '('}}
+// omp45-error@+4 {{unexpected OpenMP clause 'nontemporal' in directive '#pragma omp distribute simd'}}
+// expected-error@+3 2 {{expected expression}}
+// expected-error@+2 {{expected ')'}}
+// expected-note@+1 {{to match this '('}}
 #pragma omp distribute simd nontemporal(,
   for (i = 0; i < 16; ++i)
     ;
-// omp45-error@+1 {{unexpected OpenMP clause 'nontemporal' in directive '#pragma omp distribute simd'}} expected-error@+1 2 {{expected expression}}
+// omp45-error@+2 {{unexpected OpenMP clause 'nontemporal' in directive '#pragma omp distribute simd'}}
+// expected-error@+1 2 {{expected expression}}
 #pragma omp distribute simd nontemporal(, )
   for (i = 0; i < 16; ++i)
     ;
-// omp45-error@+1 {{unexpected OpenMP clause 'nontemporal' in directive '#pragma omp distribute simd'}} expected-error@+1 {{expected expression}}
+// omp45-error@+2 {{unexpected OpenMP clause 'nontemporal' in directive '#pragma omp distribute simd'}}
+// expected-error@+1 {{expected expression}}
 #pragma omp distribute simd nontemporal()
   for (i = 0; i < 16; ++i)
     ;
-// omp45-error@+1 {{unexpected OpenMP clause 'nontemporal' in directive '#pragma omp distribute simd'}} expected-error@+1 {{expected expression}}
+// omp45-error@+2 {{unexpected OpenMP clause 'nontemporal' in directive '#pragma omp distribute simd'}}
+// expected-error@+1 {{expected expression}}
 #pragma omp distribute simd nontemporal(int)
   for (i = 0; i < 16; ++i)
     ;
-// omp45-error@+1 {{unexpected OpenMP clause 'nontemporal' in directive '#pragma omp distribute simd'}} omp50-error@+1 {{expected variable name}} omp51-error@+1 {{expected variable name}}
+// omp45-error@+2 {{unexpected OpenMP clause 'nontemporal' in directive '#pragma omp distribute simd'}}
+// omp50-and-later-var-error@+1 {{expected variable name}}
 #pragma omp distribute simd nontemporal(0)
   for (i = 0; i < 16; ++i)
     ;
-// omp45-error@+1 {{unexpected OpenMP clause 'nontemporal' in directive '#pragma omp distribute simd'}} expected-error@+1 {{use of undeclared identifier 'x'}}
+// omp45-error@+2 {{unexpected OpenMP clause 'nontemporal' in directive '#pragma omp distribute simd'}}
+// expected-error@+1 {{use of undeclared identifier 'x'}}
 #pragma omp distribute simd nontemporal(x)
   for (i = 0; i < 16; ++i)
     ;
-// expected-error@+2 {{use of undeclared identifier 'x'}}
-// omp45-error@+1 {{unexpected OpenMP clause 'nontemporal' in directive '#pragma omp distribute simd'}} expected-error@+1 {{use of undeclared identifier 'y'}}
+// expected-error@+3 {{use of undeclared identifier 'x'}}
+// omp45-error@+2 {{unexpected OpenMP clause 'nontemporal' in directive '#pragma omp distribute simd'}}
+// expected-error@+1 {{use of undeclared identifier 'y'}}
 #pragma omp distribute simd nontemporal(x, y)
   for (i = 0; i < 16; ++i)
     ;
-// expected-error@+3 {{use of undeclared identifier 'x'}}
-// expected-error@+2 {{use of undeclared identifier 'y'}}
-// omp45-error@+1 {{unexpected OpenMP clause 'nontemporal' in directive '#pragma omp distribute simd'}} expected-error@+1 {{use of undeclared identifier 'z'}}
+// expected-error@+4 {{use of undeclared identifier 'x'}}
+// expected-error@+3 {{use of undeclared identifier 'y'}}
+// omp45-error@+2 {{unexpected OpenMP clause 'nontemporal' in directive '#pragma omp distribute simd'}}
+// expected-error@+1 {{use of undeclared identifier 'z'}}
 #pragma omp distribute simd nontemporal(x, y, z)
   for (i = 0; i < 16; ++i)
     ;
 
   int x, y;
-// omp45-error@+1 {{unexpected OpenMP clause 'nontemporal' in directive '#pragma omp distribute simd'}} expected-error@+1 {{expected ',' or ')' in 'nontemporal' clause}} expected-error@+1 {{expected ')'}} expected-note@+1 {{to match this '('}}
+// omp45-error@+4 {{unexpected OpenMP clause 'nontemporal' in directive '#pragma omp distribute simd'}}
+// expected-error@+3 {{expected ',' or ')' in 'nontemporal' clause}}
+// expected-error@+2 {{expected ')'}}
+// expected-note@+1 {{to match this '('}}
 #pragma omp distribute simd nontemporal(x :)
   for (i = 0; i < 16; ++i)
     ;
-// omp45-error@+1 {{unexpected OpenMP clause 'nontemporal' in directive '#pragma omp distribute simd'}} expected-error@+1 {{expected ')'}} expected-note@+1 {{to match this '('}} expected-error@+1 {{expected ',' or ')' in 'nontemporal' clause}}
+// omp45-error@+4 {{unexpected OpenMP clause 'nontemporal' in directive '#pragma omp distribute simd'}}
+// expected-error@+3 {{expected ')'}}
+// expected-note@+2 {{to match this '('}}
+// expected-error@+1 {{expected ',' or ')' in 'nontemporal' clause}}
 #pragma omp distribute simd nontemporal(x :, )
   for (i = 0; i < 16; ++i)
     ;
 
-// omp51-note@+3 {{defined as nontemporal}}
-// omp50-note@+2 {{defined as nontemporal}}
-// omp45-error@+1 2 {{unexpected OpenMP clause 'nontemporal' in directive '#pragma omp distribute simd'}} omp50-error@+1 {{a variable cannot appear in more than one nontemporal clause}} omp51-error@+1 {{a variable cannot appear in more than one nontemporal clause}}
+// omp50-temporal-note@+3 {{defined as nontemporal}}
+// omp-50-and-later-temporal-error@+2 {{a variable cannot appear in more than one nontemporal clause}}
+// omp45-error@+1 2 {{unexpected OpenMP clause 'nontemporal' in directive '#pragma omp distribute simd'}}
 #pragma omp distribute simd nontemporal(x) nontemporal(x)
   for (i = 0; i < 16; ++i)
     ;
@@ -1105,7 +1157,9 @@ void test_nontemporal(void) {
   for (i = 0; i < 16; ++i)
     ;
 
-// omp45-error@+1 {{unexpected OpenMP clause 'nontemporal' in directive '#pragma omp distribute simd'}} expected-note@+1 {{to match this '('}} expected-error@+1 {{expected ',' or ')' in 'nontemporal' clause}} expected-error@+1 {{expected ')'}}
+// omp45-error@+3 {{unexpected OpenMP clause 'nontemporal' in directive '#pragma omp distribute simd'}}
+// expected-note@+2 {{to match this '('}}
+// expected-error@+1 {{expected ',' or ')' in 'nontemporal' clause}} expected-error@+1 {{expected ')'}}
 #pragma omp distribute simd nontemporal(x, y : 0)
   for (i = 0; i < 16; ++i)
     ;
@@ -1119,35 +1173,60 @@ void test_nontemporal(void) {
 #pragma omp distribute simd lastprivate(x) nontemporal(x)
   for (i = 0; i < 16; ++i)
     ;
-#pragma omp distribute simd order // omp45-error {{unexpected OpenMP clause 'order' in directive '#pragma omp distribute simd'}} expected-error {{expected '(' after 'order'}}
+// omp45-error@+2 {{unexpected OpenMP clause 'order' in directive '#pragma omp distribute simd'}}
+// expected-error@+1 {{expected '(' after 'order'}}
+#pragma omp distribute simd order
   for (int i = 0; i < 10; ++i)
     ;
-#pragma omp distribute simd order( // omp45-error {{unexpected OpenMP clause 'order' in directive '#pragma omp distribute simd'}} expected-error {{expected ')'}} expected-note {{to match this '('}} omp50-error {{expected 'concurrent' in OpenMP clause 'order'}} omp51-error {{expected 'concurrent' in OpenMP clause 'order'}}
+// omp-50-and-later-error@+4 {{expected 'concurrent' in OpenMP clause 'order'}}
+// expected-note@+3 {{to match this '('}}
+// expected-error@+2 {{expected ')'}}
+// omp45-error@+1 {{unexpected OpenMP clause 'order' in directive '#pragma omp distribute simd'}}
+#pragma omp distribute simd order(
   for (int i = 0; i < 10; ++i)
     ;
-#pragma omp distribute simd order(none // omp45-error {{unexpected OpenMP clause 'order' in directive '#pragma omp distribute simd'}} expected-error {{expected ')'}} expected-note {{to match this '('}} omp50-error {{expected 'concurrent' in OpenMP clause 'order'}} omp51-error {{expected 'concurrent' in OpenMP clause 'order'}}
+// omp-50-and-later-error@+4 {{expected 'concurrent' in OpenMP clause 'order'}}
+// expected-note@+3 {{to match this '('}}
+// expected-error@+2 {{expected ')'}} 
+// omp45-error@+1 {{unexpected OpenMP clause 'order' in directive '#pragma omp distribute simd'}} 
+#pragma omp distribute simd order(none
   for (int i = 0; i < 10; ++i)
     ;
-#pragma omp distribute simd order(concurrent // omp45-error {{unexpected OpenMP clause 'order' in directive '#pragma omp distribute simd'}} expected-error {{expected ')'}} expected-note {{to match this '('}}
+// expected-note@+3 {{to match this '('}}
+// expected-error@+2 {{expected ')'}}
+// omp45-error@+1 {{unexpected OpenMP clause 'order' in directive '#pragma omp distribute simd'}}
+#pragma omp distribute simd order(concurrent
   for (int i = 0; i < 10; ++i)
     ;
-#pragma omp distribute simd order(concurrent) // omp45-error {{unexpected OpenMP clause 'order' in directive '#pragma omp distribute simd'}}
+// omp45-error@+1 {{unexpected OpenMP clause 'order' in directive '#pragma omp distribute simd'}}
+#pragma omp distribute simd order(concurrent)
   for (int i = 0; i < 10; ++i)
     ;
-#pragma omp distribute simd order(unconstrained:) // omp45-error {{unexpected OpenMP clause 'order' in directive '#pragma omp distribute simd'}} omp50-error {{expected 'concurrent' in OpenMP clause 'order'}} omp51-error {{expected 'concurrent' in OpenMP clause 'order'}}
+  // omp-50-and-later-error@+2 {{expected 'concurrent' in OpenMP clause 'order'}}
+  // omp45-error@+1 {{unexpected OpenMP clause 'order' in directive '#pragma omp distribute simd'}}
+#pragma omp distribute simd order(unconstrained:)
   for (int i = 0; i < 10; ++i)
     ;
-#pragma omp distribute simd order(reproducible:concurrent // omp45-error {{unexpected OpenMP clause 'order' in directive '#pragma omp distribute simd'}} expected-error {{expected ')'}} expected-note {{to match this '('}} omp50-error {{expected 'concurrent' in OpenMP clause 'order'}}
+  // omp50-error@+4 {{expected 'concurrent' in OpenMP clause 'order'}}
+  // expected-note@+3 {{to match this '('}}
+  // expected-error@+2 {{expected ')'}}
+  // omp45-error@+1 {{unexpected OpenMP clause 'order' in directive '#pragma omp distribute simd'}}
+#pragma omp distribute simd order(reproducible:concurrent
   for (int i = 0; i < 10; ++i)
     ;
-#pragma omp distribute simd order(reproducible:concurrent) // omp45-error {{unexpected OpenMP clause 'order' in directive '#pragma omp distribute simd'}} omp50-error {{expected 'concurrent' in OpenMP clause 'order'}}
+  // omp50-error@+2 {{expected 'concurrent' in OpenMP clause 'order'}}
+  // omp45-error@+1 {{unexpected OpenMP clause 'order' in directive '#pragma omp distribute simd'}}
+#pragma omp distribute simd order(reproducible:concurrent)
   for (int i = 0; i < 10; ++i)
     ;
-#pragma omp distribute simd order(unconstrained:concurrent) // omp45-error {{unexpected OpenMP clause 'order' in directive '#pragma omp distribute simd'}} omp50-error {{expected 'concurrent' in OpenMP clause 'order'}}
+  // omp50-error@+2 {{expected 'concurrent' in OpenMP clause 'order'}}
+  // omp45-error@+1 {{unexpected OpenMP clause 'order' in directive '#pragma omp distribute simd'}}
+#pragma omp distribute simd order(unconstrained:concurrent)
   for (int i = 0; i < 10; ++i)
     ;
-#pragma omp distribute simd order(concurrent) order(concurrent) // omp45-error {{unexpected OpenMP clause 'order' in directive '#pragma omp distribute simd'}} omp45-error {{unexpected OpenMP clause 'order' in directive '#pragma omp distribute simd'}} omp51-error {{directive '#pragma omp distribute simd' cannot contain more than one 'order' clause}}
+  // omp-clause-error@+2 {{directive '#pragma omp distribute simd' cannot contain more than one 'order' clause}}
+  // omp45-error@+1 2 {{unexpected OpenMP clause 'order' in directive '#pragma omp distribute simd'}}
+#pragma omp distribute simd order(concurrent) order(concurrent)
   for (int i = 0; i < 10; ++i)
     ;
 }
-
diff --git a/clang/test/OpenMP/driver.c b/clang/test/OpenMP/driver.c
index 0e9b2f71aae40..9ed791ba99da7 100644
--- a/clang/test/OpenMP/driver.c
+++ b/clang/test/OpenMP/driver.c
@@ -15,9 +15,11 @@
 // RUN: %clang %s -c -E -dM -fopenmp=libomp -fopenmp-version=1 | FileCheck --check-prefix=CHECK-DEFAULT-VERSION %s
 // RUN: %clang %s -c -E -dM -fopenmp=libomp -fopenmp-version=0 | FileCheck --check-prefix=CHECK-DEFAULT-VERSION %s
 // RUN: %clang %s -c -E -dM -fopenmp=libomp -fopenmp-version=100 | FileCheck --check-prefix=CHECK-DEFAULT-VERSION %s
+// RUN: %clang %s -c -E -dM -fopenmp=libomp -fopenmp-version=60 | FileCheck --check-prefix=CHECK-60-VERSION %s
 // RUN: %clang %s -c -E -dM -fopenmp=libomp | FileCheck --check-prefix=CHECK-DEFAULT-VERSION %s
 
 // CHECK-DEFAULT-VERSION: #define _OPENMP 202011
+// CHECK-60-VERSION: #define _OPENMP 202411
 
 // RUN: %clang %s -c -E -dM -fopenmp=libomp -fopenmp-version=31 | FileCheck --check-prefix=CHECK-31-VERSION %s
 // CHECK-31-VERSION: #define _OPENMP 201107
@@ -33,6 +35,8 @@
 // RUN: %clang %s -c -E -dM -fopenmp=libomp -fopenmp-version=50 | FileCheck --check-prefix=CHECK-50-VERSION %s
 // CHECK-50-VERSION: #define _OPENMP 201811
 
+// RUN: %clang %s -c -E -dM -fopenmp=libomp -fopenmp-version=60 | FileCheck --check-prefix=CHECK-60-VERSION %s
+
 // RUN: %clang %s -c -E -dM -fopenmp=libomp | FileCheck --check-prefix=CHECK-51-VERSION %s
 // CHECK-51-VERSION: #define _OPENMP 202011
 
@@ -52,6 +56,7 @@
 // RUN: %clang %s -c -E -dM -fopenmp-simd -fopenmp-version=31 | FileCheck --check-prefix=CHECK-VERSION %s
 // RUN: %clang %s -c -E -dM -fopenmp-simd -fopenmp-version=40 | FileCheck --check-prefix=CHECK-VERSION %s
 // RUN: %clang %s -c -E -dM -fopenmp-simd -fopenmp-version=45 | FileCheck --check-prefix=CHECK-VERSION %s
+// RUN: %clang %s -c -E -dM -fopenmp-simd -fopenmp-version=60 | FileCheck --check-prefix=CHECK-VERSION %s
 // RUN: %clang %s -c -E -dM -fopenmp-simd | FileCheck --check-prefix=CHECK-VERSION %s
 
 // CHECK-VERSION-NOT: #define _OPENMP
diff --git a/clang/test/OpenMP/error_ast_print.cpp b/clang/test/OpenMP/error_ast_print.cpp
index 8e40f7da71324..43c6d8553dfde 100644
--- a/clang/test/OpenMP/error_ast_print.cpp
+++ b/clang/test/OpenMP/error_ast_print.cpp
@@ -2,9 +2,17 @@
 // RUN: %clang_cc1 -fopenmp -x c++ -std=c++11 -fopenmp-version=51 -emit-pch -o %t %s
 // RUN: %clang_cc1 -fopenmp -std=c++11 -fopenmp-version=51 -include-pch %t -verify %s -ast-print | FileCheck %s
 
+// RUN: %clang_cc1 -verify -fopenmp -fopenmp-version=60 -ast-print %s | FileCheck %s
+// RUN: %clang_cc1 -fopenmp -x c++ -std=c++11 -fopenmp-version=60 -emit-pch -o %t %s
+// RUN: %clang_cc1 -fopenmp -std=c++11 -fopenmp-version=60 -include-pch %t -verify %s -ast-print | FileCheck %s
+
 // RUN: %clang_cc1 -verify -fopenmp-simd -fopenmp-version=51 -ast-print %s | FileCheck %s
 // RUN: %clang_cc1 -fopenmp-simd -x c++ -std=c++11 -fopenmp-version=51 -emit-pch -o %t %s
 // RUN: %clang_cc1 -fopenmp-simd -std=c++11 -fopenmp-version=51 -include-pch %t -verify %s -ast-print | FileCheck %s
+
+// RUN: %clang_cc1 -verify -fopenmp-simd -fopenmp-version=60 -ast-print %s | FileCheck %s
+// RUN: %clang_cc1 -fopenmp-simd -x c++ -std=c++11 -fopenmp-version=60 -emit-pch -o %t %s
+// RUN: %clang_cc1 -fopenmp-simd -std=c++11 -fopenmp-version=60 -include-pch %t -verify %s -ast-print | FileCheck %s
 // expected-no-diagnostics
 
 #ifndef HEADER
diff --git a/clang/test/OpenMP/error_codegen.cpp b/clang/test/OpenMP/error_codegen.cpp
index 0f4d1863e69fc..70d493e01a709 100644
--- a/clang/test/OpenMP/error_codegen.cpp
+++ b/clang/test/OpenMP/error_codegen.cpp
@@ -1,23 +1,30 @@
 // RUN: %clang_cc1 -std=c++11 -fopenmp -fopenmp-version=51 -triple x86_64 \
 // RUN:   -emit-llvm -o - %s | FileCheck %s
 
+// RUN: %clang_cc1 -std=c++11 -fopenmp -fopenmp-version=60 -triple x86_64 \
+// RUN:   -emit-llvm -o - %s | FileCheck %s
+
 // RUN: %clang_cc1 -std=c++11 -fopenmp-simd -fopenmp-version=51 \
 // RUN:  -debug-info-kind=limited -triple x86_64 -emit-llvm -o - %s |  \
 // RUN:  FileCheck  --check-prefix SIMD %s
 
+// RUN: %clang_cc1 -std=c++11 -fopenmp-simd -fopenmp-version=60 \
+// RUN:  -debug-info-kind=limited -triple x86_64 -emit-llvm -o - %s |  \
+// RUN:  FileCheck  --check-prefix SIMD %s
+
 //CHECK: @.str = private unnamed_addr constant [23 x i8] c"GPU compiler required.\00", align 1
-//CHECK: @0 = private unnamed_addr constant {{.*}}error_codegen.cpp;main;52;1;;\00", align 1
+//CHECK: @0 = private unnamed_addr constant {{.*}}error_codegen.cpp;main;59;1;;\00", align 1
 //CHECK: @1 = private unnamed_addr constant %struct.ident_t { i32 0, i32 2, i32 0, i32 {{.*}}, ptr @0 }, align 8
 //CHECK: @.str.1 = private unnamed_addr constant [27 x i8] c"Note this is functioncall.\00", align 1
-//CHECK: @2 = private unnamed_addr constant {{.*}}error_codegen.cpp;main;54;1;;\00", align 1
+//CHECK: @2 = private unnamed_addr constant {{.*}}error_codegen.cpp;main;61;1;;\00", align 1
 //CHECK: @3 = private unnamed_addr constant %struct.ident_t { i32 0, i32 2, i32 0, i32 {{.*}}, ptr @2 }, align 8
 //CHECK: @.str.2 = private unnamed_addr constant [23 x i8] c"GNU compiler required.\00", align 1
-//CHECK: @4 = private unnamed_addr constant {{.*}}error_codegen.cpp;tmain;29;1;;\00", align 1
+//CHECK: @4 = private unnamed_addr constant {{.*}}error_codegen.cpp;tmain;36;1;;\00", align 1
 //CHECK: @5 = private unnamed_addr constant %struct.ident_t { i32 0, i32 2, i32 0, i32 {{.*}}, ptr @4 }, align 8
 //CHECK: @.str.3 = private unnamed_addr constant [22 x i8] c"Notice: add for loop.\00", align 1
-//CHECK: @6 = private unnamed_addr constant {{.*}}error_codegen.cpp;tmain;32;1;;\00", align 1
+//CHECK: @6 = private unnamed_addr constant {{.*}}error_codegen.cpp;tmain;39;1;;\00", align 1
 //CHECK: @7 = private unnamed_addr constant %struct.ident_t { i32 0, i32 2, i32 0, i32 {{.*}}, ptr @6 }, align 8
-//CHECK: @8 = private unnamed_addr constant {{.*}}error_codegen.cpp;tmain;38;1;;\00", align 1
+//CHECK: @8 = private unnamed_addr constant {{.*}}error_codegen.cpp;tmain;45;1;;\00", align 1
 //CHECK: @9 = private unnamed_addr constant %struct.ident_t { i32 0, i32 2, i32 0, i32 {{.*}}, ptr @8 }, align 8
 
 void foo() {}
diff --git a/clang/test/OpenMP/error_message.cpp b/clang/test/OpenMP/error_message.cpp
index 227cbf699777f..aed2df99ea8f6 100644
--- a/clang/test/OpenMP/error_message.cpp
+++ b/clang/test/OpenMP/error_message.cpp
@@ -1,6 +1,8 @@
 // RUN: %clang_cc1 -verify -fopenmp -fopenmp-version=51 -ferror-limit 100 %s -Wuninitialized
+// RUN: %clang_cc1 -verify -fopenmp -fopenmp-version=60 -ferror-limit 100 %s -Wuninitialized
 
 // RUN: %clang_cc1 -verify -fopenmp-simd -fopenmp-version=51 -ferror-limit 100 %s -Wuninitialized
+// RUN: %clang_cc1 -verify -fopenmp-simd -fopenmp-version=60 -ferror-limit 100 %s -Wuninitialized
 
 template <class T>
 T tmain(T argc) {
diff --git a/clang/test/OpenMP/flush_ast_print.cpp b/clang/test/OpenMP/flush_ast_print.cpp
index 768282422032f..484705e200eb6 100644
--- a/clang/test/OpenMP/flush_ast_print.cpp
+++ b/clang/test/OpenMP/flush_ast_print.cpp
@@ -2,9 +2,17 @@
 // RUN: %clang_cc1 -fopenmp -fopenmp-version=51 -x c++ -std=c++11 -emit-pch -o %t %s
 // RUN: %clang_cc1 -fopenmp -fopenmp-version=51 -std=c++11 -include-pch %t  -verify %s -ast-print | FileCheck %s
 
+// RUN: %clang_cc1 -verify -fopenmp -fopenmp-version=60 -ast-print %s | FileCheck %s
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=60 -x c++ -std=c++11 -emit-pch -o %t %s
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=60 -std=c++11 -include-pch %t  -verify %s -ast-print | FileCheck %s
+
 // RUN: %clang_cc1 -verify -fopenmp-simd -fopenmp-version=51 -ast-print %s | FileCheck %s
 // RUN: %clang_cc1 -fopenmp-simd -fopenmp-version=51 -x c++ -std=c++11 -emit-pch -o %t %s
 // RUN: %clang_cc1 -fopenmp-simd -fopenmp-version=51 -std=c++11 -include-pch %t -verify %s -ast-print | FileCheck %s
+
+// RUN: %clang_cc1 -verify -fopenmp-simd -fopenmp-version=60 -ast-print %s | FileCheck %s
+// RUN: %clang_cc1 -fopenmp-simd -fopenmp-version=60 -x c++ -std=c++11 -emit-pch -o %t %s
+// RUN: %clang_cc1 -fopenmp-simd -fopenmp-version=60 -std=c++11 -include-pch %t -verify %s -ast-print | FileCheck %s
 // expected-no-diagnostics
 
 #ifndef HEADER
diff --git a/clang/test/OpenMP/flush_codegen.cpp b/clang/test/OpenMP/flush_codegen.cpp
index fa2586d9fe258..022bf856d82c9 100644
--- a/clang/test/OpenMP/flush_codegen.cpp
+++ b/clang/test/OpenMP/flush_codegen.cpp
@@ -5,10 +5,22 @@
 // RUN: %clang_cc1 -fopenmp -fopenmp-version=51 -fopenmp-enable-irbuilder -x c++ -std=c++11 -triple x86_64-unknown-unknown -fexceptions -fcxx-exceptions -emit-pch -o %t %s
 // RUN: %clang_cc1 -fopenmp -fopenmp-version=51 -fopenmp-enable-irbuilder -x c++ -triple x86_64-unknown-unknown -fexceptions -fcxx-exceptions -debug-info-kind=limited -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s
 
+// RUN: %clang_cc1 -verify -fopenmp -fopenmp-version=60 -x c++ -emit-llvm %s -fexceptions -fcxx-exceptions -o - | FileCheck %s
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=60 -x c++ -std=c++11 -triple x86_64-unknown-unknown -fexceptions -fcxx-exceptions -emit-pch -o %t %s
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=60 -x c++ -triple x86_64-unknown-unknown -fexceptions -fcxx-exceptions -debug-info-kind=limited -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s
+// RUN: %clang_cc1 -verify -fopenmp -fopenmp-version=60 -fopenmp-enable-irbuilder -x c++ -emit-llvm %s -fexceptions -fcxx-exceptions -o - | FileCheck %s
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=60 -fopenmp-enable-irbuilder -x c++ -std=c++11 -triple x86_64-unknown-unknown -fexceptions -fcxx-exceptions -emit-pch -o %t %s
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=60 -fopenmp-enable-irbuilder -x c++ -triple x86_64-unknown-unknown -fexceptions -fcxx-exceptions -debug-info-kind=limited -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s
+
 // RUN: %clang_cc1 -verify -fopenmp-simd -fopenmp-version=51 -x c++ -emit-llvm %s -fexceptions -fcxx-exceptions -o - | FileCheck --check-prefix SIMD-ONLY0 %s
 // RUN: %clang_cc1 -fopenmp-simd -fopenmp-version=51 -x c++ -std=c++11 -triple x86_64-unknown-unknown -fexceptions -fcxx-exceptions -emit-pch -o %t %s
 // RUN: %clang_cc1 -fopenmp-simd -fopenmp-version=51 -x c++ -triple x86_64-unknown-unknown -fexceptions -fcxx-exceptions -debug-info-kind=limited -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY0 %s
 // SIMD-ONLY0-NOT: {{__kmpc|__tgt}}
+
+// RUN: %clang_cc1 -verify -fopenmp-simd -fopenmp-version=60 -x c++ -emit-llvm %s -fexceptions -fcxx-exceptions -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -fopenmp-simd -fopenmp-version=60 -x c++ -std=c++11 -triple x86_64-unknown-unknown -fexceptions -fcxx-exceptions -emit-pch -o %t %s
+// RUN: %clang_cc1 -fopenmp-simd -fopenmp-version=60 -x c++ -triple x86_64-unknown-unknown -fexceptions -fcxx-exceptions -debug-info-kind=limited -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// SIMD-ONLY0-NOT: {{__kmpc|__tgt}}
 // expected-no-diagnostics
 #ifndef HEADER
 #define HEADER
diff --git a/clang/test/OpenMP/for_linear_messages.cpp b/clang/test/OpenMP/for_linear_messages.cpp
index d8d3391c0c271..6253a3032c0e3 100644
--- a/clang/test/OpenMP/for_linear_messages.cpp
+++ b/clang/test/OpenMP/for_linear_messages.cpp
@@ -1,8 +1,10 @@
 // RUN: %clang_cc1 -verify -fopenmp %s -Wuninitialized
 // RUN: %clang_cc1 -verify=expected,omp52 -fopenmp -fopenmp-version=52 -DOMP52 %s -Wuninitialized
+// RUN: %clang_cc1 -verify=expected,omp52 -fopenmp -fopenmp-version=60 -DOMP52 %s -Wuninitialized
 
 // RUN: %clang_cc1 -verify -fopenmp-simd %s -Wuninitialized
 // RUN: %clang_cc1 -verify=expected,omp52 -fopenmp-simd -fopenmp-version=52 -DOMP52 %s -Wuninitialized
+// RUN: %clang_cc1 -verify=expected,omp52 -fopenmp-simd -fopenmp-version=60 -DOMP52 %s -Wuninitialized
 
 extern int omp_default_mem_alloc;
 
diff --git a/clang/test/OpenMP/for_order_messages.cpp b/clang/test/OpenMP/for_order_messages.cpp
index d38f48d2004f0..530c051849201 100644
--- a/clang/test/OpenMP/for_order_messages.cpp
+++ b/clang/test/OpenMP/for_order_messages.cpp
@@ -1,8 +1,12 @@
 // RUN: %clang_cc1 -fsyntax-only -fopenmp -fopenmp-version=50 -triple x86_64-unknown-unknown -verify=expected,omp50 %s -Wuninitialized
-// RUN: %clang_cc1 -fsyntax-only -fopenmp -triple x86_64-unknown-unknown -verify=expected,omp51 %s -Wuninitialized
+// RUN: %clang_cc1 -fsyntax-only -fopenmp -fopenmp-version=51 -triple x86_64-unknown-unknown -verify=expected,omp51 %s -Wuninitialized
+// RUN: %clang_cc1 -fsyntax-only -fopenmp -fopenmp-version=52 -triple x86_64-unknown-unknown -verify=expected,omp51 %s -Wuninitialized
+// RUN: %clang_cc1 -fsyntax-only -fopenmp -fopenmp-version=60 -triple x86_64-unknown-unknown -verify=expected,omp60 %s -Wuninitialized
 
 // RUN: %clang_cc1 -fsyntax-only -fopenmp-simd -fopenmp-version=50 -triple x86_64-unknown-unknown -verify=expected,omp50 %s -Wuninitialized
-// RUN: %clang_cc1 -fsyntax-only -fopenmp-simd -triple x86_64-unknown-unknown -verify=expected,omp51 %s -Wuninitialized
+// RUN: %clang_cc1 -fsyntax-only -fopenmp-simd -fopenmp-version=51 -triple x86_64-unknown-unknown -verify=expected,omp51 %s -Wuninitialized
+// RUN: %clang_cc1 -fsyntax-only -fopenmp-simd -fopenmp-version=52 -triple x86_64-unknown-unknown -verify=expected,omp51 %s -Wuninitialized
+// RUN: %clang_cc1 -fsyntax-only -fopenmp-simd -fopenmp-version=60 -triple x86_64-unknown-unknown -verify=expected,omp60 %s -Wuninitialized
 
 extern int omp_get_num_threads  (void);
 
@@ -35,13 +39,61 @@ int main(int argc, char **argv) {
 
 #pragma omp parallel for order(reproducible: concurrent) // omp50-error {{expected 'concurrent' in OpenMP clause 'order'}}
   for (int i = 0; i < 10; ++i) {
-#pragma omp target //omp51-error {{construct 'target' not allowed in a region associated with a directive with 'order' clause}}
+#pragma omp target //omp51-error {{construct 'target' not allowed in a region associated with a directive with 'order' clause}} omp60-error {{construct 'target' not allowed in a region associated with a directive with 'order' clause}}
       A++;
   }
 
 #pragma omp parallel for order(unconstrained: concurrent) // omp50-error {{expected 'concurrent' in OpenMP clause 'order'}}
   for (int i = 0; i < 10; ++i) {
-#pragma omp target //omp51-error {{construct 'target' not allowed in a region associated with a directive with 'order' clause}}
+#pragma omp target //omp51-error {{construct 'target' not allowed in a region associated with a directive with 'order' clause}} omp60-error {{construct 'target' not allowed in a region associated with a directive with 'order' clause}}
       A++;
   }
+
+#pragma omp loop bind(parallel) order(concurrent)
+  for (int i = 0; i < 10; ++i) {
+#pragma omp parallel for //omp60-error {{construct 'parallel for' not allowed in a region associated with a directive with 'order' clause}}
+    for (int j = 0; j < 10; ++j) {
+      A += j;
+    }
+  }
+
+#pragma omp distribute order(concurrent)
+  for (int i = 0; i < 10; ++i) {
+#pragma omp parallel for simd //omp60-error {{construct 'parallel for simd' not allowed in a region associated with a directive with 'order' clause}}
+    for (int j = 0; j < 10; ++j) {
+      A += j;
+    }
+  }
+
+#pragma omp for order(concurrent)
+  for (int i = 0; i < 10; ++i) {
+#pragma omp parallel master //omp60-error {{construct 'parallel master' not allowed in a region associated with a directive with 'order' clause}}
+    for (int j = 0; j < 10; ++j) {
+      A += j;
+    }
+  }
+
+#pragma omp for order(concurrent)
+  for (int i = 0; i < 10; ++i) {
+#pragma omp parallel master taskloop //omp60-error {{construct 'parallel master taskloop' not allowed in a region associated with a directive with 'order' clause}}
+    for (int j = 0; j < 10; ++j) {
+      A += j;
+    }
+  }
+
+#pragma omp for order(concurrent)
+  for (int i = 0; i < 10; ++i) {
+#pragma omp parallel master taskloop simd //omp60-error {{construct 'parallel master taskloop simd' not allowed in a region associated with a directive with 'order' clause}}
+    for (int j = 0; j < 10; ++j) {
+      A += j;
+    }
+  }
+
+#pragma omp for order(concurrent)
+  for (int i = 0; i < 10; ++i) {
+    #pragma omp parallel sections //omp60-error {{construct 'parallel sections' not allowed in a region associated with a directive with 'order' clause}}
+    {
+      A++;
+    }
+  }
 }
diff --git a/clang/test/OpenMP/for_simd_ast_print.cpp b/clang/test/OpenMP/for_simd_ast_print.cpp
index 7f77107b8ed7c..5ee0e3d7393b3 100644
--- a/clang/test/OpenMP/for_simd_ast_print.cpp
+++ b/clang/test/OpenMP/for_simd_ast_print.cpp
@@ -7,6 +7,9 @@
 // RUN: %clang_cc1 -verify -fopenmp -fopenmp-version=51 -ast-print %s -DOMP51 | FileCheck %s --check-prefix=CHECK --check-prefix=OMP51
 // RUN: %clang_cc1 -fopenmp -fopenmp-version=51 -x c++ -std=c++11 -emit-pch -o %t %s -DOMP51
 // RUN: %clang_cc1 -fopenmp -fopenmp-version=51 -std=c++11 -include-pch %t -verify %s -ast-print -DOMP51 | FileCheck %s --check-prefix=CHECK --check-prefix=OMP52
+// RUN: %clang_cc1 -verify -fopenmp -fopenmp-version=60 -ast-print %s -DOMP51 | FileCheck %s --check-prefix=CHECK --check-prefix=OMP51
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=60 -x c++ -std=c++11 -emit-pch -o %t %s -DOMP51
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=60 -std=c++11 -include-pch %t -verify %s -ast-print -DOMP51 | FileCheck %s --check-prefix=CHECK --check-prefix=OMP52
 // RUN: %clang_cc1 -verify -fopenmp -fopenmp-version=52 -ast-print %s -DOMP52 | FileCheck %s --check-prefix=CHECK --check-prefix=OMP52
 // RUN: %clang_cc1 -fopenmp -fopenmp-version=52 -x c++ -std=c++11 -emit-pch -o %t %s -DOMP52
 // RUN: %clang_cc1 -fopenmp -fopenmp-version=52 -std=c++11 -include-pch %t -verify %s -ast-print -DOMP52 | FileCheck %s --check-prefix=CHECK --check-prefix=OMP52
@@ -20,6 +23,9 @@
 // RUN: %clang_cc1 -verify -fopenmp-simd -fopenmp-version=51 -ast-print %s -DOMP51 | FileCheck %s --check-prefix=CHECK --check-prefix=OMP51
 // RUN: %clang_cc1 -fopenmp-simd -fopenmp-version=51 -x c++ -std=c++11 -emit-pch -o %t %s -DOMP51
 // RUN: %clang_cc1 -fopenmp-simd -fopenmp-version=51 -std=c++11 -include-pch %t -verify %s -ast-print -DOMP51 | FileCheck %s --check-prefix=CHECK --check-prefix=OMP51
+// RUN: %clang_cc1 -verify -fopenmp-simd -fopenmp-version=60 -ast-print %s -DOMP51 | FileCheck %s --check-prefix=CHECK --check-prefix=OMP51
+// RUN: %clang_cc1 -fopenmp-simd -fopenmp-version=60 -x c++ -std=c++11 -emit-pch -o %t %s -DOMP51
+// RUN: %clang_cc1 -fopenmp-simd -fopenmp-version=60 -std=c++11 -include-pch %t -verify %s -ast-print -DOMP51 | FileCheck %s --check-prefix=CHECK --check-prefix=OMP51
 // expected-no-diagnostics
 // RUN: %clang_cc1 -verify -fopenmp-simd -fopenmp-version=52 -ast-print %s -DOMP52 | FileCheck %s --check-prefix=CHECK --check-prefix=OMP52
 // RUN: %clang_cc1 -fopenmp-simd -fopenmp-version=52 -x c++ -std=c++11 -emit-pch -o %t %s -DOMP52
diff --git a/clang/test/OpenMP/metadirective_messages.cpp b/clang/test/OpenMP/metadirective_messages.cpp
index b342a094a7870..7fce9fa446058 100644
--- a/clang/test/OpenMP/metadirective_messages.cpp
+++ b/clang/test/OpenMP/metadirective_messages.cpp
@@ -5,7 +5,7 @@
 void foo() {
 #pragma omp metadirective // expected-error {{expected expression}}
   ;
-#pragma omp metadirective when() // expected-error {{expected valid context selector in when clause}} expected-error {{expected expression}} expected-warning {{expected identifier or string literal describing a context set; set skipped}} expected-note {{context set options are: 'construct' 'device' 'implementation' 'user'}} expected-note {{the ignored set spans until here}}
+#pragma omp metadirective when() // expected-error {{expected valid context selector in when clause}} expected-error {{expected expression}} expected-warning {{expected identifier or string literal describing a context set; set skipped}} expected-note {{context set options are: 'construct' 'device' 'target_device' 'implementation' 'user'}} expected-note {{the ignored set spans until here}}
   ;
 #pragma omp metadirective when(device{}) // expected-warning {{expected '=' after the context set name "device"; '=' assumed}} expected-warning {{expected identifier or string literal describing a context selector; selector skipped}} expected-note {{context selector options are: 'kind' 'arch' 'isa'}} expected-note {{the ignored selector spans until here}} expected-error {{expected valid context selector in when clause}} expected-error {{expected expression}}
   ;
diff --git a/clang/test/OpenMP/nvptx_declare_variant_name_mangling.cpp b/clang/test/OpenMP/nvptx_declare_variant_name_mangling.cpp
index 0a5690e77a7b5..e128370d821e8 100644
--- a/clang/test/OpenMP/nvptx_declare_variant_name_mangling.cpp
+++ b/clang/test/OpenMP/nvptx_declare_variant_name_mangling.cpp
@@ -6,10 +6,10 @@
 
 // CHECK-DAG: @_Z3barv
 // CHECK-DAG: @_Z3bazv
-// CHECK-DAG: define{{.*}} @"_Z53bar$ompvariant$S2$s7$Pnvptx$Pnvptx64$S3$s9$Pmatch_anyv"
-// CHECK-DAG: define{{.*}} @"_Z53baz$ompvariant$S2$s7$Pnvptx$Pnvptx64$S3$s9$Pmatch_anyv"
-// CHECK-DAG: call noundef i32 @"_Z53bar$ompvariant$S2$s7$Pnvptx$Pnvptx64$S3$s9$Pmatch_anyv"()
-// CHECK-DAG: call noundef i32 @"_Z53baz$ompvariant$S2$s7$Pnvptx$Pnvptx64$S3$s9$Pmatch_anyv"()
+// CHECK-DAG: define{{.*}} @{{"_Z[0-9]+bar\$ompvariant\$.*"}}
+// CHECK-DAG: define{{.*}} @{{"_Z[0-9]+baz\$ompvariant\$.*"}}
+// CHECK-DAG: call noundef i32 @{{"_Z[0-9]+bar\$ompvariant\$.*"}}()
+// CHECK-DAG: call noundef i32 @{{"_Z[0-9]+baz\$ompvariant\$.*"}}()
 
 #ifndef HEADER
 #define HEADER
diff --git a/clang/test/OpenMP/nvptx_target_firstprivate_codegen.cpp b/clang/test/OpenMP/nvptx_target_firstprivate_codegen.cpp
index d573f1cd193d6..94ace20826db4 100644
--- a/clang/test/OpenMP/nvptx_target_firstprivate_codegen.cpp
+++ b/clang/test/OpenMP/nvptx_target_firstprivate_codegen.cpp
@@ -90,7 +90,7 @@ int foo(int n, double *ptr) {
     ptr[0]++;
   }
 
-  // TCHECK:  define weak_odr protected void @__omp_offloading_{{.+}}(ptr {{[^,]+}}, ptr noundef [[PTR_IN:%.+]])
+  // TCHECK:  define weak_odr protected ptx_kernel void @__omp_offloading_{{.+}}(ptr {{[^,]+}}, ptr noundef [[PTR_IN:%.+]])
   // TCHECK:  [[DYN_PTR_ADDR:%.+]] = alloca ptr,
   // TCHECK:  [[PTR_ADDR:%.+]] = alloca ptr,
   // TCHECK-NOT: alloca ptr,
diff --git a/clang/test/OpenMP/nvptx_target_requires_unified_shared_memory.cpp b/clang/test/OpenMP/nvptx_target_requires_unified_shared_memory.cpp
index c1bfe36507d14..2f2b8383bcceb 100644
--- a/clang/test/OpenMP/nvptx_target_requires_unified_shared_memory.cpp
+++ b/clang/test/OpenMP/nvptx_target_requires_unified_shared_memory.cpp
@@ -47,10 +47,10 @@ int bar(int n){
 // CHECK-HOST: [[OFFLOAD_MAPTYPES:@.+]] = private unnamed_addr constant [2 x i64] [i64 800, i64 800]
 
 // CHECK-HOST: [[OMP_OFFLOAD_ENTRY_LINK_VAR_PTR_NAME:@.+]] = internal unnamed_addr constant [21 x i8]
-// CHECK-HOST: [[OMP_OFFLOAD_ENTRY_LINK_VAR_PTR:@.+]] = weak{{.*}} constant %struct.__tgt_offload_entry { ptr [[VAR_DECL_TGT_LINK_PTR]], ptr [[OMP_OFFLOAD_ENTRY_LINK_VAR_PTR_NAME]], i64 8, i32 1, i32 0 }, section "omp_offloading_entries"
+// CHECK-HOST: [[OMP_OFFLOAD_ENTRY_LINK_VAR_PTR:@.+]] = weak constant %struct.__tgt_offload_entry { i64 0, i16 1, i16 1, i32 1, ptr [[VAR_DECL_TGT_LINK_PTR]], ptr [[OMP_OFFLOAD_ENTRY_LINK_VAR_PTR_NAME]], i64 8, i64 0, ptr null }, section "llvm_offload_entries"
 
 // CHECK-HOST: [[OMP_OFFLOAD_ENTRY_TO_VAR_PTR_NAME:@.+]] = internal unnamed_addr constant [24 x i8]
-// CHECK-HOST: [[OMP_OFFLOAD_ENTRY_TO_VAR_PTR:@.+]] = weak{{.*}} constant %struct.__tgt_offload_entry { ptr [[VAR_DECL_TGT_TO_PTR]], ptr [[OMP_OFFLOAD_ENTRY_TO_VAR_PTR_NAME]], i64 8, i32 0, i32 0 }, section "omp_offloading_entries"
+// CHECK-HOST: [[OMP_OFFLOAD_ENTRY_TO_VAR_PTR:@.+]] = weak constant %struct.__tgt_offload_entry { i64 0, i16 1, i16 1, i32 0, ptr [[VAR_DECL_TGT_TO_PTR]], ptr @.offloading.entry_name.1, i64 8, i64 0, ptr null }, section "llvm_offload_entries"
 
 // CHECK-HOST: [[N_CASTED:%.+]] = alloca i64
 // CHECK-HOST: [[SUM_CASTED:%.+]] = alloca i64
diff --git a/clang/test/OpenMP/openmp_offload_registration.cpp b/clang/test/OpenMP/openmp_offload_registration.cpp
index aff8d431650dc..be6d308fbda94 100644
--- a/clang/test/OpenMP/openmp_offload_registration.cpp
+++ b/clang/test/OpenMP/openmp_offload_registration.cpp
@@ -8,7 +8,7 @@ void foo(void) {
   {}
 }
 
-// CHECK-DAG: [[ENTTY:%.+]] = type { ptr, ptr, i[[SZ:32|64]], i32, i32 }
+// CHECK-DAG: [[ENTTY:%.+]] = type { i64, i16, i16, i32, ptr, ptr, i64, i64, ptr }
 
 // Check presence of foo() and the outlined target region
 // CHECK: define{{.*}} void [[FOO:@.+]]()
diff --git a/clang/test/OpenMP/spirv_target_codegen_basic.cpp b/clang/test/OpenMP/spirv_target_codegen_basic.cpp
new file mode 100644
index 0000000000000..fb2810e88c063
--- /dev/null
+++ b/clang/test/OpenMP/spirv_target_codegen_basic.cpp
@@ -0,0 +1,17 @@
+// RUN: %clang_cc1 -verify -fopenmp -x c++ -triple x86_64-unknown-linux -fopenmp-targets=spirv64-intel -emit-llvm-bc %s -o %t-host.bc
+// RUN: %clang_cc1 -verify -fopenmp -x c++ -triple spirv64-intel -fopenmp-targets=spirv64-intel -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-host.bc -o - | FileCheck %s
+
+// expected-no-diagnostics
+
+// CHECK: @__omp_offloading_{{.*}}_dynamic_environment = weak_odr protected addrspace(1) global %struct.DynamicEnvironmentTy zeroinitializer
+// CHECK: @__omp_offloading_{{.*}}_kernel_environment = weak_odr protected addrspace(1) constant %struct.KernelEnvironmentTy
+
+// CHECK: define weak_odr protected spir_kernel void @__omp_offloading_{{.*}}
+
+int main() {
+  int ret = 0;
+  #pragma omp target
+  for(int i = 0; i < 5; i++)
+    ret++;
+  return ret;
+}
diff --git a/clang/test/OpenMP/target_codegen.cpp b/clang/test/OpenMP/target_codegen.cpp
index 1e38e9c3fe082..ff126fbe4d02c 100644
--- a/clang/test/OpenMP/target_codegen.cpp
+++ b/clang/test/OpenMP/target_codegen.cpp
@@ -67,12 +67,12 @@
 // CHECK-DAG: [[TT:%.+]] = type { i64, i8 }
 // CHECK-DAG: [[S1:%.+]] = type { double }
 // CHECK-DAG: [[S2:%.+]] = type { i32, i32, i32 }
-// CHECK-DAG: [[ENTTY:%.+]] = type { ptr, ptr, i[[SZ:32|64]], i32, i32 }
+// CHECK-DAG: [[ENTTY:%.+]] = type { i64, i16, i16, i32, ptr, ptr, i64, i64, ptr }
 // CHECK-DAG: [[ANON_T:%.+]] = type { ptr, i32, i32 }
 // CHECK-32-DAG: [[KMP_PRIVATES_T]] = type { [2 x i64], ptr, i32, [2 x ptr], [2 x ptr] }
 // CHECK-64-DAG: [[KMP_PRIVATES_T]] = type { ptr, [2 x ptr], [2 x ptr], [2 x i64], i32 }
 
-// TCHECK: [[ENTTY:%.+]] = type { ptr, ptr, i{{32|64}}, i32, i32 }
+// TCHECK: [[ENTTY:%.+]] = type { i64, i16, i16, i32, ptr, ptr, i64, i64, ptr }
 
 // We have 9 target regions, but only 8 that actually will generate offloading
 // code and have mapped arguments, and only 6 have all-constant map sizes.
@@ -180,7 +180,7 @@ int foo(int n) {
     local1 = global;
   }
 
-  // CHECK:       call void [[HVT1:@.+]](i[[SZ]] {{[^,]+}})
+  // CHECK:       call void [[HVT1:@.+]](i[[SZ:32|64]] {{[^,]+}})
   #pragma omp target if(0) firstprivate(global)
   {
     global += 1;
diff --git a/clang/test/OpenMP/target_codegen_registration.cpp b/clang/test/OpenMP/target_codegen_registration.cpp
index 4927147d080f2..97994f0e3182a 100644
--- a/clang/test/OpenMP/target_codegen_registration.cpp
+++ b/clang/test/OpenMP/target_codegen_registration.cpp
@@ -51,9 +51,9 @@
 // CHECK-DAG: [[SE:%.+]] = type { [64 x i32] }
 // CHECK-DAG: [[ST1:%.+]] = type { [228 x i32] }
 // CHECK-DAG: [[ST2:%.+]] = type { [1128 x i32] }
-// CHECK-DAG: [[ENTTY:%.+]] = type { ptr, ptr, i[[SZ:32|64]], i32, i32 }
+// CHECK-DAG: [[ENTTY:%.+]] = type { i64, i16, i16, i32, ptr, ptr, i64, i64, ptr }
 
-// TCHECK:    [[ENTTY:%.+]] = type { ptr, ptr, i[[SZ:32|64]], i32, i32 }
+// TCHECK: [[ENTTY:%.+]] = type { i64, i16, i16, i32, ptr, ptr, i64, i64, ptr }
 
 // CHECK-DAG: [[A1:@.+]] = internal global [[SA]]
 // CHECK-DAG: [[A2:@.+]] ={{.*}} global [[SA]]
@@ -119,54 +119,30 @@
 // CHECK-NTARGET-NOT: private unnamed_addr constant [1 x i
 
 // CHECK-DAG: [[NAMEPTR1:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME1:__omp_offloading_[0-9a-f]+_[0-9a-f]+__Z.+_l[0-9]+]]\00"
-// CHECK-DAG: @.offloading.entry.[[NAME1]] = weak{{.*}} constant [[ENTTY]] { ptr @{{.*}}, ptr [[NAMEPTR1]], i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // CHECK-DAG: [[NAMEPTR2:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME2:.+]]\00"
-// CHECK-DAG: @.offloading.entry.[[NAME2]] = weak{{.*}} constant [[ENTTY]] { ptr @{{.*}}, ptr [[NAMEPTR2]], i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // CHECK-DAG: [[NAMEPTR3:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME3:.+]]\00"
-// CHECK-DAG: @.offloading.entry.[[NAME3]] = weak{{.*}} constant [[ENTTY]] { ptr @{{.*}}, ptr [[NAMEPTR3]], i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // CHECK-DAG: [[NAMEPTR4:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME4:.+]]\00"
-// CHECK-DAG: @.offloading.entry.[[NAME4]] = weak{{.*}} constant [[ENTTY]] { ptr @{{.*}}, ptr [[NAMEPTR4]], i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // CHECK-DAG: [[NAMEPTR5:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME5:.+]]\00"
-// CHECK-DAG: @.offloading.entry.[[NAME5]] = weak{{.*}} constant [[ENTTY]] { ptr @{{.*}}, ptr [[NAMEPTR5]], i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // CHECK-DAG: [[NAMEPTR6:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME6:.+]]\00"
-// CHECK-DAG: @.offloading.entry.[[NAME6]] = weak{{.*}} constant [[ENTTY]] { ptr @{{.*}}, ptr [[NAMEPTR6]], i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // CHECK-DAG: [[NAMEPTR7:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME7:.+]]\00"
-// CHECK-DAG: @.offloading.entry.[[NAME7]] = weak{{.*}} constant [[ENTTY]] { ptr @{{.*}}, ptr [[NAMEPTR7]], i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // CHECK-DAG: [[NAMEPTR8:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME8:.+]]\00"
-// CHECK-DAG: @.offloading.entry.[[NAME8]] = weak{{.*}} constant [[ENTTY]] { ptr @{{.*}}, ptr [[NAMEPTR8]], i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // CHECK-DAG: [[NAMEPTR9:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME9:.+]]\00"
-// CHECK-DAG: @.offloading.entry.[[NAME9]] = weak{{.*}} constant [[ENTTY]] { ptr @{{.*}}, ptr [[NAMEPTR9]], i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // CHECK-DAG: [[NAMEPTR10:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME10:.+]]\00"
-// CHECK-DAG: @.offloading.entry.[[NAME10]] = weak{{.*}} constant [[ENTTY]] { ptr @{{.*}}, ptr [[NAMEPTR10]], i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // CHECK-DAG: [[NAMEPTR11:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME11:.+]]\00"
-// CHECK-DAG: @.offloading.entry.[[NAME11]] = weak{{.*}} constant [[ENTTY]] { ptr @{{.*}}, ptr [[NAMEPTR11]], i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // CHECK-DAG: [[NAMEPTR12:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME12:.+]]\00"
-// CHECK-DAG: @.offloading.entry.[[NAME12]] = weak{{.*}} constant [[ENTTY]] { ptr @{{.*}}, ptr [[NAMEPTR12]], i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 
 // TCHECK-DAG: [[NAMEPTR1:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME1:__omp_offloading_[0-9a-f]+_[0-9a-f]+__Z.+_l[0-9]+]]\00"
-// TCHECK-DAG: @.offloading.entry.[[NAME1]] = weak{{.*}} constant [[ENTTY]] { ptr @{{.*}}, ptr [[NAMEPTR1]], i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // TCHECK-DAG: [[NAMEPTR2:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME2:.+]]\00"
-// TCHECK-DAG: @.offloading.entry.[[NAME2]] = weak{{.*}} constant [[ENTTY]] { ptr @{{.*}}, ptr [[NAMEPTR2]], i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // TCHECK-DAG: [[NAMEPTR3:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME3:.+]]\00"
-// TCHECK-DAG: @.offloading.entry.[[NAME3]] = weak{{.*}} constant [[ENTTY]] { ptr @{{.*}}, ptr [[NAMEPTR3]], i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // TCHECK-DAG: [[NAMEPTR4:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME4:.+]]\00"
-// TCHECK-DAG: @.offloading.entry.[[NAME4]] = weak{{.*}} constant [[ENTTY]] { ptr @{{.*}}, ptr [[NAMEPTR4]], i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // TCHECK-DAG: [[NAMEPTR5:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME5:.+]]\00"
-// TCHECK-DAG: @.offloading.entry.[[NAME5]] = weak{{.*}} constant [[ENTTY]] { ptr @{{.*}}, ptr [[NAMEPTR5]], i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // TCHECK-DAG: [[NAMEPTR6:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME6:.+]]\00"
-// TCHECK-DAG: @.offloading.entry.[[NAME6]] = weak{{.*}} constant [[ENTTY]] { ptr @{{.*}}, ptr [[NAMEPTR6]], i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // TCHECK-DAG: [[NAMEPTR7:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME7:.+]]\00"
-// TCHECK-DAG: @.offloading.entry.[[NAME7]] = weak{{.*}} constant [[ENTTY]] { ptr @{{.*}}, ptr [[NAMEPTR7]], i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // TCHECK-DAG: [[NAMEPTR8:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME8:.+]]\00"
-// TCHECK-DAG: @.offloading.entry.[[NAME8]] = weak{{.*}} constant [[ENTTY]] { ptr @{{.*}}, ptr [[NAMEPTR8]], i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // TCHECK-DAG: [[NAMEPTR9:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME9:.+]]\00"
-// TCHECK-DAG: @.offloading.entry.[[NAME9]] = weak{{.*}} constant [[ENTTY]] { ptr @{{.*}}, ptr [[NAMEPTR9]], i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // TCHECK-DAG: [[NAMEPTR10:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME10:.+]]\00"
-// TCHECK-DAG: @.offloading.entry.[[NAME10]] = weak{{.*}} constant [[ENTTY]] { ptr @{{.*}}, ptr [[NAMEPTR10]], i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // TCHECK-DAG: [[NAMEPTR11:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME11:.+]]\00"
-// TCHECK-DAG: @.offloading.entry.[[NAME11]] = weak{{.*}} constant [[ENTTY]] { ptr @{{.*}}, ptr [[NAMEPTR11]], i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // TCHECK-DAG: [[NAMEPTR12:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME12:.+]]\00"
-// TCHECK-DAG: @.offloading.entry.[[NAME12]] = weak{{.*}} constant [[ENTTY]] { ptr @{{.*}}, ptr [[NAMEPTR12]], i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 
 // We have 4 initializers, one for the 500 priority, another one for 501, or more for the default priority, and the last one for the offloading registration function.
 // CHECK: @llvm.global_ctors = appending global [3 x { i32, ptr, ptr }] [
@@ -401,31 +377,31 @@ int bar(int a){
 
 // Check metadata is properly generated:
 // CHECK:     !omp_offload.info = !{!{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}}
-// CHECK-DAG: = !{i32 0, i32 [[DEVID:-?[0-9]+]], i32 [[FILEID:-?[0-9]+]], !"_ZN2SB3fooEv", i32 204, i32 0, i32 {{[0-9]+}}}
-// CHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2SDD1Ev", i32 254, i32 0, i32 {{[0-9]+}}}
-// CHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2SEC1Ev", i32 270, i32 0, i32 {{[0-9]+}}}
-// CHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2SED1Ev", i32 276, i32 0, i32 {{[0-9]+}}}
-// CHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2STILi1000EE3fooEv", i32 287, i32 0, i32 {{[0-9]+}}}
-// CHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2STILi100EEC1Ev", i32 293, i32 0, i32 {{[0-9]+}}}
-// CHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_Z3bari", i32 396, i32 0, i32 {{[0-9]+}}}
-// CHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2STILi100EED1Ev", i32 299, i32 0, i32 {{[0-9]+}}}
-// CHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2STILi1000EEC1Ev", i32 293, i32 0, i32 {{[0-9]+}}}
-// CHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2STILi1000EED1Ev", i32 299, i32 0, i32 {{[0-9]+}}}
-// CHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2STILi100EE3fooEv", i32 287, i32 0, i32 {{[0-9]+}}}
-// CHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2SCC1Ev", i32 229, i32 0, i32 {{[0-9]+}}}
+// CHECK-DAG: = !{i32 0, i32 [[DEVID:-?[0-9]+]], i32 [[FILEID:-?[0-9]+]], !"_ZN2SB3fooEv", i32 {{[0-9]+}}, i32 0, i32 {{[0-9]+}}}
+// CHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2SDD1Ev", i32 {{[0-9]+}}, i32 0, i32 {{[0-9]+}}}
+// CHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2SEC1Ev", i32 {{[0-9]+}}, i32 0, i32 {{[0-9]+}}}
+// CHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2SED1Ev", i32 {{[0-9]+}}, i32 0, i32 {{[0-9]+}}}
+// CHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2STILi1000EE3fooEv", i32 {{[0-9]+}}, i32 0, i32 {{[0-9]+}}}
+// CHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2STILi100EEC1Ev", i32 {{[0-9]+}}, i32 0, i32 {{[0-9]+}}}
+// CHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_Z3bari", i32 {{[0-9]+}}, i32 0, i32 {{[0-9]+}}}
+// CHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2STILi100EED1Ev", i32 {{[0-9]+}}, i32 0, i32 {{[0-9]+}}}
+// CHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2STILi1000EEC1Ev", i32 {{[0-9]+}}, i32 0, i32 {{[0-9]+}}}
+// CHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2STILi1000EED1Ev", i32 {{[0-9]+}}, i32 0, i32 {{[0-9]+}}}
+// CHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2STILi100EE3fooEv", i32 {{[0-9]+}}, i32 0, i32 {{[0-9]+}}}
+// CHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2SCC1Ev", i32 {{[0-9]+}}, i32 0, i32 {{[0-9]+}}}
 
 // TCHECK:     !omp_offload.info = !{!{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}}
-// TCHECK-DAG: = !{i32 0, i32 [[DEVID:-?[0-9]+]], i32 [[FILEID:-?[0-9]+]], !"_ZN2SB3fooEv", i32 204, i32 0, i32 {{[0-9]+}}}
-// TCHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2SDD1Ev", i32 254, i32 0, i32 {{[0-9]+}}}
-// TCHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2SEC1Ev", i32 270, i32 0, i32 {{[0-9]+}}}
-// TCHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2SED1Ev", i32 276, i32 0, i32 {{[0-9]+}}}
-// TCHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2STILi1000EE3fooEv", i32 287, i32 0, i32 {{[0-9]+}}}
-// TCHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2STILi100EEC1Ev", i32 293, i32 0, i32 {{[0-9]+}}}
-// TCHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_Z3bari", i32 396, i32 0, i32 {{[0-9]+}}}
-// TCHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2STILi100EED1Ev", i32 299, i32 0, i32 {{[0-9]+}}}
-// TCHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2STILi1000EEC1Ev", i32 293, i32 0, i32 {{[0-9]+}}}
-// TCHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2STILi1000EED1Ev", i32 299, i32 0, i32 {{[0-9]+}}}
-// TCHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2STILi100EE3fooEv", i32 287, i32 0, i32 {{[0-9]+}}}
-// TCHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2SCC1Ev", i32 229, i32 0, i32 {{[0-9]+}}}
+// TCHECK-DAG: = !{i32 0, i32 [[DEVID:-?[0-9]+]], i32 [[FILEID:-?[0-9]+]], !"_ZN2SB3fooEv", i32 {{[0-9]+}}, i32 0, i32 {{[0-9]+}}}
+// TCHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2SDD1Ev", i32 {{[0-9]+}}, i32 0, i32 {{[0-9]+}}}
+// TCHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2SEC1Ev", i32 {{[0-9]+}}, i32 0, i32 {{[0-9]+}}}
+// TCHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2SED1Ev", i32 {{[0-9]+}}, i32 0, i32 {{[0-9]+}}}
+// TCHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2STILi1000EE3fooEv", i32 {{[0-9]+}}, i32 0, i32 {{[0-9]+}}}
+// TCHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2STILi100EEC1Ev", i32 {{[0-9]+}}, i32 0, i32 {{[0-9]+}}}
+// TCHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_Z3bari", i32 {{[0-9]+}}, i32 0, i32 {{[0-9]+}}}
+// TCHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2STILi100EED1Ev", i32 {{[0-9]+}}, i32 0, i32 {{[0-9]+}}}
+// TCHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2STILi1000EEC1Ev", i32 {{[0-9]+}}, i32 0, i32 {{[0-9]+}}}
+// TCHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2STILi1000EED1Ev", i32 {{[0-9]+}}, i32 0, i32 {{[0-9]+}}}
+// TCHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2STILi100EE3fooEv", i32 {{[0-9]+}}, i32 0, i32 {{[0-9]+}}}
+// TCHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2SCC1Ev", i32 {{[0-9]+}}, i32 0, i32 {{[0-9]+}}}
 
 #endif
diff --git a/clang/test/OpenMP/target_depend_codegen.cpp b/clang/test/OpenMP/target_depend_codegen.cpp
index babd2843309f7..73ffa120452c1 100644
--- a/clang/test/OpenMP/target_depend_codegen.cpp
+++ b/clang/test/OpenMP/target_depend_codegen.cpp
@@ -39,9 +39,9 @@
 #define HEADER
 
 // CHECK-DAG: [[TT:%.+]] = type { i64, i8 }
-// CHECK-DAG: [[ENTTY:%.+]] = type { ptr, ptr, i[[SZ:32|64]], i32, i32 }
+// CHECK-DAG: [[ENTTY:%.+]] = type { i64, i16, i16, i32, ptr, ptr, i64, i64, ptr }
 
-// TCHECK: [[ENTTY:%.+]] = type { ptr, ptr, i{{32|64}}, i32, i32 }
+// TCHECK: [[ENTTY:%.+]] = type { i64, i16, i16, i32, ptr, ptr, i64, i64, ptr }
 
 // CHECK-DAG: [[SIZET:@.+]] = private unnamed_addr constant [3 x i64] [i64 0, i64 4, i64 {{16|12}}]
 // CHECK-DAG: [[MAPT:@.+]] = private unnamed_addr constant [3 x i64] [i64 544, i64 800, i64 3]
@@ -79,7 +79,7 @@ int foo(int n) {
 // CHECK:       [[GEP:%.+]] = getelementptr inbounds nuw %{{.+}}, ptr %{{.+}}, i32 0, i32 0
 // CHECK:       [[DEV:%.+]] = load i32, ptr [[DEVICE_CAP]],
 // CHECK:       store i32 [[DEV]], ptr [[GEP]],
-// CHECK:       [[TASK:%.+]] = call ptr @__kmpc_omp_task_alloc(ptr @1, i32 [[GTID:%.+]], i32 1, i[[SZ]] {{20|40}}, i[[SZ]] 4, ptr [[TASK_ENTRY0:@.+]])
+// CHECK:       [[TASK:%.+]] = call ptr @__kmpc_omp_task_alloc(ptr @1, i32 [[GTID:%.+]], i32 1, i[[SZ:32|64]] {{20|40}}, i[[SZ]] 4, ptr [[TASK_ENTRY0:@.+]])
 // CHECK:       getelementptr %struct.kmp_depend_info, ptr [[DEP_START:%.+]], i[[SZ]] 1
 // CHECK:       getelementptr %struct.kmp_depend_info, ptr [[DEP_START]], i[[SZ]] 2
 // CHECK:       getelementptr %struct.kmp_depend_info, ptr [[DEP_START]], i[[SZ]] 3
diff --git a/clang/test/OpenMP/target_indirect_codegen.cpp b/clang/test/OpenMP/target_indirect_codegen.cpp
index 974f8b20c0bfc..20a36c2935516 100644
--- a/clang/test/OpenMP/target_indirect_codegen.cpp
+++ b/clang/test/OpenMP/target_indirect_codegen.cpp
@@ -11,13 +11,13 @@
 //.
 // HOST: @[[VAR:.+]] = global i8 0, align 1
 // HOST: @[[FOO_ENTRY_NAME:.+]] = internal unnamed_addr constant [{{[0-9]+}} x i8] c"[[FOO_NAME:__omp_offloading_[0-9a-z]+_[0-9a-z]+_foo_l[0-9]+]]\00"
-// HOST: @.offloading.entry.[[FOO_NAME]] = weak constant %struct.__tgt_offload_entry { ptr @_Z3foov, ptr @[[FOO_ENTRY_NAME]], i64 8, i32 8, i32 0 }, section "omp_offloading_entries", align 1
+// HOST: @.offloading.entry.[[FOO_NAME]] = weak constant %struct.__tgt_offload_entry { i64 0, i16 1, i16 1, i32 8, ptr @_Z3foov, ptr @[[FOO_ENTRY_NAME]], i64 8, i64 0, ptr null }
 // HOST: @[[BAZ_ENTRY_NAME:.+]] = internal unnamed_addr constant [{{[0-9]+}} x i8] c"[[BAZ_NAME:__omp_offloading_[0-9a-z]+_[0-9a-z]+_baz_l[0-9]+]]\00"
-// HOST: @.offloading.entry.[[BAZ_NAME]] = weak constant %struct.__tgt_offload_entry { ptr @_Z3bazv, ptr @[[BAZ_ENTRY_NAME]], i64 8, i32 8, i32 0 }, section "omp_offloading_entries", align 1
+// HOST: @.offloading.entry.[[BAZ_NAME]] = weak constant %struct.__tgt_offload_entry { i64 0, i16 1, i16 1, i32 8, ptr @_Z3bazv, ptr @[[BAZ_ENTRY_NAME]], i64 8, i64 0, ptr null }
 // HOST: @[[VAR_ENTRY_NAME:.+]] = internal unnamed_addr constant [4 x i8] c"var\00"
-// HOST: @.offloading.entry.var = weak constant %struct.__tgt_offload_entry { ptr @[[VAR]], ptr @[[VAR_ENTRY_NAME]], i64 1, i32 0, i32 0 }, section "omp_offloading_entries", align 1
+// HOST: @.offloading.entry.var = weak constant %struct.__tgt_offload_entry { i64 0, i16 1, i16 1, i32 0, ptr @[[VAR]], ptr @[[VAR_ENTRY_NAME]], i64 1, i64 0, ptr null }
 // HOST: @[[BAR_ENTRY_NAME:.+]] = internal unnamed_addr constant [{{[0-9]+}} x i8] c"[[BAR_NAME:__omp_offloading_[0-9a-z]+_[0-9a-z]+_bar_l[0-9]+]]\00"
-// HOST: @.offloading.entry.[[BAR_NAME]] = weak constant %struct.__tgt_offload_entry { ptr @_ZL3barv, ptr @[[BAR_ENTRY_NAME]], i64 8, i32 8, i32 0 }, section "omp_offloading_entries", align 1
+// HOST: @.offloading.entry.[[BAR_NAME]] = weak constant %struct.__tgt_offload_entry { i64 0, i16 1, i16 1, i32 8, ptr @_ZL3barv, ptr @[[BAR_ENTRY_NAME]], i64 8, i64 0, ptr null }
 //.
 // DEVICE: @[[FOO_NAME:__omp_offloading_[0-9a-z]+_[0-9a-z]+_foo_l[0-9]+]] = protected addrspace(1) constant ptr @_Z3foov
 // DEVICE: @[[BAZ_NAME:__omp_offloading_[0-9a-z]+_[0-9a-z]+_baz_l[0-9]+]] = protected addrspace(1) constant ptr @_Z3bazv
diff --git a/clang/test/OpenMP/target_parallel_codegen_registration.cpp b/clang/test/OpenMP/target_parallel_codegen_registration.cpp
index dd3ec98bcfef7..0997a6f006b52 100644
--- a/clang/test/OpenMP/target_parallel_codegen_registration.cpp
+++ b/clang/test/OpenMP/target_parallel_codegen_registration.cpp
@@ -92,9 +92,9 @@
 // CHECK-DAG: [[SE:%.+]] = type { [64 x i32] }
 // CHECK-DAG: [[ST1:%.+]] = type { [228 x i32] }
 // CHECK-DAG: [[ST2:%.+]] = type { [1128 x i32] }
-// CHECK-DAG: [[ENTTY:%.+]] = type { ptr, ptr, i[[SZ:32|64]], i32, i32 }
+// CHECK-DAG: [[ENTTY:%.+]] = type { i64, i16, i16, i32, ptr, ptr, i64, i64, ptr }
 
-// TCHECK:    [[ENTTY:%.+]] = type { ptr, ptr, i[[SZ:32|64]], i32, i32 }
+// TCHECK: [[ENTTY:%.+]] = type { i64, i16, i16, i32, ptr, ptr, i64, i64, ptr }
 
 // CHECK-DAG: [[A1:@.+]] = internal global [[SA]]
 // CHECK-DAG: [[A2:@.+]] ={{.*}} global [[SA]]
@@ -160,54 +160,30 @@
 // CHECK-NTARGET-NOT: private unnamed_addr constant [1 x i
 
 // CHECK-DAG: [[NAMEPTR1:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME1:__omp_offloading_[0-9a-f]+_[0-9a-f]+__Z.+_l[0-9]+]]\00"
-// CHECK-DAG: [[ENTRY1:@.+]] = weak{{.*}} constant [[ENTTY]] { ptr @{{.*}}, ptr [[NAMEPTR1]], i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // CHECK-DAG: [[NAMEPTR2:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME2:.+]]\00"
-// CHECK-DAG: [[ENTRY2:@.+]] = weak{{.*}} constant [[ENTTY]] { ptr @{{.*}}, ptr [[NAMEPTR2]], i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // CHECK-DAG: [[NAMEPTR3:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME3:.+]]\00"
-// CHECK-DAG: [[ENTRY3:@.+]] = weak{{.*}} constant [[ENTTY]] { ptr @{{.*}}, ptr [[NAMEPTR3]], i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // CHECK-DAG: [[NAMEPTR4:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME4:.+]]\00"
-// CHECK-DAG: [[ENTRY4:@.+]] = weak{{.*}} constant [[ENTTY]] { ptr @{{.*}}, ptr [[NAMEPTR4]], i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // CHECK-DAG: [[NAMEPTR5:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME5:.+]]\00"
-// CHECK-DAG: [[ENTRY5:@.+]] = weak{{.*}} constant [[ENTTY]] { ptr @{{.*}}, ptr [[NAMEPTR5]], i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // CHECK-DAG: [[NAMEPTR6:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME6:.+]]\00"
-// CHECK-DAG: [[ENTRY6:@.+]] = weak{{.*}} constant [[ENTTY]] { ptr @{{.*}}, ptr [[NAMEPTR6]], i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // CHECK-DAG: [[NAMEPTR7:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME7:.+]]\00"
-// CHECK-DAG: [[ENTRY7:@.+]] = weak{{.*}} constant [[ENTTY]] { ptr @{{.*}}, ptr [[NAMEPTR7]], i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // CHECK-DAG: [[NAMEPTR8:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME8:.+]]\00"
-// CHECK-DAG: [[ENTRY8:@.+]] = weak{{.*}} constant [[ENTTY]] { ptr @{{.*}}, ptr [[NAMEPTR8]], i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // CHECK-DAG: [[NAMEPTR9:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME9:.+]]\00"
-// CHECK-DAG: [[ENTRY9:@.+]] = weak{{.*}} constant [[ENTTY]] { ptr @{{.*}}, ptr [[NAMEPTR9]], i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // CHECK-DAG: [[NAMEPTR10:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME10:.+]]\00"
-// CHECK-DAG: [[ENTRY10:@.+]] = weak{{.*}} constant [[ENTTY]] { ptr @{{.*}}, ptr [[NAMEPTR10]], i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // CHECK-DAG: [[NAMEPTR11:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME11:.+]]\00"
-// CHECK-DAG: [[ENTRY11:@.+]] = weak{{.*}} constant [[ENTTY]] { ptr @{{.*}}, ptr [[NAMEPTR11]], i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // CHECK-DAG: [[NAMEPTR12:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME12:.+]]\00"
-// CHECK-DAG: [[ENTRY12:@.+]] = weak{{.*}} constant [[ENTTY]] { ptr @{{.*}}, ptr [[NAMEPTR12]], i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 
 // TCHECK-DAG: [[NAMEPTR1:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME1:__omp_offloading_[0-9a-f]+_[0-9a-f]+__Z.+_l[0-9]+]]\00"
-// TCHECK-DAG: [[ENTRY1:@.+]] = weak{{.*}} constant [[ENTTY]] { ptr @{{.*}}, ptr [[NAMEPTR1]], i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // TCHECK-DAG: [[NAMEPTR2:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME2:.+]]\00"
-// TCHECK-DAG: [[ENTRY2:@.+]] = weak{{.*}} constant [[ENTTY]] { ptr @{{.*}}, ptr [[NAMEPTR2]], i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // TCHECK-DAG: [[NAMEPTR3:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME3:.+]]\00"
-// TCHECK-DAG: [[ENTRY3:@.+]] = weak{{.*}} constant [[ENTTY]] { ptr @{{.*}}, ptr [[NAMEPTR3]], i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // TCHECK-DAG: [[NAMEPTR4:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME4:.+]]\00"
-// TCHECK-DAG: [[ENTRY4:@.+]] = weak{{.*}} constant [[ENTTY]] { ptr @{{.*}}, ptr [[NAMEPTR4]], i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // TCHECK-DAG: [[NAMEPTR5:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME5:.+]]\00"
-// TCHECK-DAG: [[ENTRY5:@.+]] = weak{{.*}} constant [[ENTTY]] { ptr @{{.*}}, ptr [[NAMEPTR5]], i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // TCHECK-DAG: [[NAMEPTR6:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME6:.+]]\00"
-// TCHECK-DAG: [[ENTRY6:@.+]] = weak{{.*}} constant [[ENTTY]] { ptr @{{.*}}, ptr [[NAMEPTR6]], i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // TCHECK-DAG: [[NAMEPTR7:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME7:.+]]\00"
-// TCHECK-DAG: [[ENTRY7:@.+]] = weak{{.*}} constant [[ENTTY]] { ptr @{{.*}}, ptr [[NAMEPTR7]], i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // TCHECK-DAG: [[NAMEPTR8:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME8:.+]]\00"
-// TCHECK-DAG: [[ENTRY8:@.+]] = weak{{.*}} constant [[ENTTY]] { ptr @{{.*}}, ptr [[NAMEPTR8]], i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // TCHECK-DAG: [[NAMEPTR9:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME9:.+]]\00"
-// TCHECK-DAG: [[ENTRY9:@.+]] = weak{{.*}} constant [[ENTTY]] { ptr @{{.*}}, ptr [[NAMEPTR9]], i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // TCHECK-DAG: [[NAMEPTR10:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME10:.+]]\00"
-// TCHECK-DAG: [[ENTRY10:@.+]] = weak{{.*}} constant [[ENTTY]] { ptr @{{.*}}, ptr [[NAMEPTR10]], i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // TCHECK-DAG: [[NAMEPTR11:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME11:.+]]\00"
-// TCHECK-DAG: [[ENTRY11:@.+]] = weak{{.*}} constant [[ENTTY]] { ptr @{{.*}}, ptr [[NAMEPTR11]], i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // TCHECK-DAG: [[NAMEPTR12:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME12:.+]]\00"
-// TCHECK-DAG: [[ENTRY12:@.+]] = weak{{.*}} constant [[ENTTY]] { ptr @{{.*}}, ptr [[NAMEPTR12]], i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 
 // We have 4 initializers, one for the 500 priority, another one for 501, or more for the default priority, and the last one for the offloading registration function.
 // CHECK: @llvm.global_ctors = appending global [3 x { i32, ptr, ptr }] [
@@ -442,31 +418,31 @@ int bar(int a){
 
 // Check metadata is properly generated:
 // CHECK:     !omp_offload.info = !{!{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}}
-// CHECK-DAG: = !{i32 0, i32 [[DEVID:-?[0-9]+]], i32 [[FILEID:-?[0-9]+]], !"_ZN2SB3fooEv", i32 245, i32 0, i32 {{[0-9]+}}}
-// CHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2SDD1Ev", i32 295, i32 0, i32 {{[0-9]+}}}
-// CHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2SEC1Ev", i32 311, i32 0, i32 {{[0-9]+}}}
-// CHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2SED1Ev", i32 317, i32 0, i32 {{[0-9]+}}}
-// CHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2STILi1000EE3fooEv", i32 328, i32 0, i32 {{[0-9]+}}}
-// CHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2STILi100EEC1Ev", i32 334, i32 0, i32 {{[0-9]+}}}
-// CHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_Z3bari", i32 437, i32 0, i32 {{[0-9]+}}}
-// CHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2STILi100EED1Ev", i32 340, i32 0, i32 {{[0-9]+}}}
-// CHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2STILi1000EEC1Ev", i32 334, i32 0, i32 {{[0-9]+}}}
-// CHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2STILi1000EED1Ev", i32 340, i32 0, i32 {{[0-9]+}}}
-// CHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2STILi100EE3fooEv", i32 328, i32 0, i32 {{[0-9]+}}}
-// CHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2SCC1Ev", i32 270, i32 0, i32 {{[0-9]+}}}
+// CHECK-DAG: = !{i32 0, i32 [[DEVID:-?[0-9]+]], i32 [[FILEID:-?[0-9]+]], !"_ZN2SB3fooEv", i32 {{[0-9]+}}, i32 0, i32 {{[0-9]+}}}
+// CHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2SDD1Ev", i32 {{[0-9]+}}, i32 0, i32 {{[0-9]+}}}
+// CHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2SEC1Ev", i32 {{[0-9]+}}, i32 0, i32 {{[0-9]+}}}
+// CHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2SED1Ev", i32 {{[0-9]+}}, i32 0, i32 {{[0-9]+}}}
+// CHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2STILi1000EE3fooEv", i32 {{[0-9]+}}, i32 0, i32 {{[0-9]+}}}
+// CHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2STILi100EEC1Ev", i32 {{[0-9]+}}, i32 0, i32 {{[0-9]+}}}
+// CHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_Z3bari", i32 {{[0-9]+}}, i32 0, i32 {{[0-9]+}}}
+// CHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2STILi100EED1Ev", i32 {{[0-9]+}}, i32 0, i32 {{[0-9]+}}}
+// CHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2STILi1000EEC1Ev", i32 {{[0-9]+}}, i32 0, i32 {{[0-9]+}}}
+// CHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2STILi1000EED1Ev", i32 {{[0-9]+}}, i32 0, i32 {{[0-9]+}}}
+// CHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2STILi100EE3fooEv", i32 {{[0-9]+}}, i32 0, i32 {{[0-9]+}}}
+// CHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2SCC1Ev", i32 {{[0-9]+}}, i32 0, i32 {{[0-9]+}}}
 
 // TCHECK:     !omp_offload.info = !{!{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}}
-// TCHECK-DAG: = !{i32 0, i32 [[DEVID:-?[0-9]+]], i32 [[FILEID:-?[0-9]+]], !"_ZN2SB3fooEv", i32 245, i32 0, i32 {{[0-9]+}}}
-// TCHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2SDD1Ev", i32 295, i32 0, i32 {{[0-9]+}}}
-// TCHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2SEC1Ev", i32 311, i32 0, i32 {{[0-9]+}}}
-// TCHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2SED1Ev", i32 317, i32 0, i32 {{[0-9]+}}}
-// TCHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2STILi1000EE3fooEv", i32 328, i32 0, i32 {{[0-9]+}}}
-// TCHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2STILi100EEC1Ev", i32 334, i32 0, i32 {{[0-9]+}}}
-// TCHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_Z3bari", i32 437, i32 0, i32 {{[0-9]+}}}
-// TCHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2STILi100EED1Ev", i32 340, i32 0, i32 {{[0-9]+}}}
-// TCHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2STILi1000EEC1Ev", i32 334, i32 0, i32 {{[0-9]+}}}
-// TCHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2STILi1000EED1Ev", i32 340, i32 0, i32 {{[0-9]+}}}
-// TCHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2STILi100EE3fooEv", i32 328, i32 0, i32 {{[0-9]+}}}
-// TCHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2SCC1Ev", i32 270, i32 0, i32 {{[0-9]+}}}
+// TCHECK-DAG: = !{i32 0, i32 [[DEVID:-?[0-9]+]], i32 [[FILEID:-?[0-9]+]], !"_ZN2SB3fooEv", i32 {{[0-9]+}}, i32 0, i32 {{[0-9]+}}}
+// TCHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2SDD1Ev", i32 {{[0-9]+}}, i32 0, i32 {{[0-9]+}}}
+// TCHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2SEC1Ev", i32 {{[0-9]+}}, i32 0, i32 {{[0-9]+}}}
+// TCHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2SED1Ev", i32 {{[0-9]+}}, i32 0, i32 {{[0-9]+}}}
+// TCHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2STILi1000EE3fooEv", i32 {{[0-9]+}}, i32 0, i32 {{[0-9]+}}}
+// TCHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2STILi100EEC1Ev", i32 {{[0-9]+}}, i32 0, i32 {{[0-9]+}}}
+// TCHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_Z3bari", i32 {{[0-9]+}}, i32 0, i32 {{[0-9]+}}}
+// TCHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2STILi100EED1Ev", i32 {{[0-9]+}}, i32 0, i32 {{[0-9]+}}}
+// TCHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2STILi1000EEC1Ev", i32 {{[0-9]+}}, i32 0, i32 {{[0-9]+}}}
+// TCHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2STILi1000EED1Ev", i32 {{[0-9]+}}, i32 0, i32 {{[0-9]+}}}
+// TCHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2STILi100EE3fooEv", i32 {{[0-9]+}}, i32 0, i32 {{[0-9]+}}}
+// TCHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2SCC1Ev", i32 {{[0-9]+}}, i32 0, i32 {{[0-9]+}}}
 
 #endif
diff --git a/clang/test/OpenMP/target_parallel_depend_codegen.cpp b/clang/test/OpenMP/target_parallel_depend_codegen.cpp
index 86c26523c2e14..52264ee79e123 100644
--- a/clang/test/OpenMP/target_parallel_depend_codegen.cpp
+++ b/clang/test/OpenMP/target_parallel_depend_codegen.cpp
@@ -39,9 +39,9 @@
 #define HEADER
 
 // CHECK-DAG: [[TT:%.+]] = type { i64, i8 }
-// CHECK-DAG: [[ENTTY:%.+]] = type { ptr, ptr, i[[SZ:32|64]], i32, i32 }
+// CHECK-DAG: [[ENTTY:%.+]] = type { i64, i16, i16, i32, ptr, ptr, i64, i64, ptr }
 
-// TCHECK: [[ENTTY:%.+]] = type { ptr, ptr, i{{32|64}}, i32, i32 }
+// TCHECK: [[ENTTY:%.+]] = type { i64, i16, i16, i32, ptr, ptr, i64, i64, ptr }
 
 // CHECK-DAG: [[SIZET:@.+]] = private unnamed_addr constant [2 x i64] [i64 0, i64 4]
 // CHECK-DAG: [[MAPT:@.+]] = private unnamed_addr constant [2 x i64] [i64 544, i64 800]
@@ -76,7 +76,7 @@ int foo(int n) {
 // CHECK:       [[GEP:%.+]] = getelementptr inbounds nuw %{{.+}}, ptr %{{.+}}, i32 0, i32 0
 // CHECK:       [[DEV:%.+]] = load i32, ptr [[DEVICE_CAP]],
 // CHECK:       store i32 [[DEV]], ptr [[GEP]],
-// CHECK:       [[TASK:%.+]] = call ptr @__kmpc_omp_task_alloc(ptr @1, i32 [[GTID:%.+]], i32 1, i[[SZ]] {{20|40}}, i[[SZ]] 4, ptr [[TASK_ENTRY0:@.+]])
+// CHECK:       [[TASK:%.+]] = call ptr @__kmpc_omp_task_alloc(ptr @1, i32 [[GTID:%.+]], i32 1, i[[SZ:32|64]] {{20|40}}, i[[SZ]] 4, ptr [[TASK_ENTRY0:@.+]])
 // CHECK:       getelementptr %struct.kmp_depend_info, ptr %{{.+}}, i[[SZ]] 0
 // CHECK:       getelementptr %struct.kmp_depend_info, ptr %{{.+}}, i[[SZ]] 1
 // CHECK:       getelementptr %struct.kmp_depend_info, ptr %{{.+}}, i[[SZ]] 2
diff --git a/clang/test/OpenMP/target_parallel_for_codegen_registration.cpp b/clang/test/OpenMP/target_parallel_for_codegen_registration.cpp
index b65241109b0f8..653f5996d0142 100644
--- a/clang/test/OpenMP/target_parallel_for_codegen_registration.cpp
+++ b/clang/test/OpenMP/target_parallel_for_codegen_registration.cpp
@@ -92,9 +92,9 @@
 // CHECK-DAG: [[SE:%.+]] = type { [64 x i32] }
 // CHECK-DAG: [[ST1:%.+]] = type { [228 x i32] }
 // CHECK-DAG: [[ST2:%.+]] = type { [1128 x i32] }
-// CHECK-DAG: [[ENTTY:%.+]] = type { ptr, ptr, i[[SZ:32|64]], i32, i32 }
+// CHECK-DAG: [[ENTTY:%.+]] = type { i64, i16, i16, i32, ptr, ptr, i64, i64, ptr }
 
-// TCHECK:    [[ENTTY:%.+]] = type { ptr, ptr, i[[SZ:32|64]], i32, i32 }
+// TCHECK: [[ENTTY:%.+]] = type { i64, i16, i16, i32, ptr, ptr, i64, i64, ptr }
 
 // CHECK-DAG: [[A1:@.+]] = internal global [[SA]]
 // CHECK-DAG: [[A2:@.+]] ={{.*}} global [[SA]]
@@ -160,54 +160,30 @@
 // CHECK-NTARGET-NOT: private unnamed_addr constant [1 x i
 
 // CHECK-DAG: [[NAMEPTR1:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME1:__omp_offloading_[0-9a-f]+_[0-9a-f]+__Z.+_l[0-9]+]]\00"
-// CHECK-DAG: [[ENTRY1:@.+]] = weak{{.*}} constant [[ENTTY]] { ptr @{{.*}}, ptr [[NAMEPTR1]], i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // CHECK-DAG: [[NAMEPTR2:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME2:.+]]\00"
-// CHECK-DAG: [[ENTRY2:@.+]] = weak{{.*}} constant [[ENTTY]] { ptr @{{.*}}, ptr [[NAMEPTR2]], i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // CHECK-DAG: [[NAMEPTR3:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME3:.+]]\00"
-// CHECK-DAG: [[ENTRY3:@.+]] = weak{{.*}} constant [[ENTTY]] { ptr @{{.*}}, ptr [[NAMEPTR3]], i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // CHECK-DAG: [[NAMEPTR4:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME4:.+]]\00"
-// CHECK-DAG: [[ENTRY4:@.+]] = weak{{.*}} constant [[ENTTY]] { ptr @{{.*}}, ptr [[NAMEPTR4]], i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // CHECK-DAG: [[NAMEPTR5:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME5:.+]]\00"
-// CHECK-DAG: [[ENTRY5:@.+]] = weak{{.*}} constant [[ENTTY]] { ptr @{{.*}}, ptr [[NAMEPTR5]], i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // CHECK-DAG: [[NAMEPTR6:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME6:.+]]\00"
-// CHECK-DAG: [[ENTRY6:@.+]] = weak{{.*}} constant [[ENTTY]] { ptr @{{.*}}, ptr [[NAMEPTR6]], i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // CHECK-DAG: [[NAMEPTR7:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME7:.+]]\00"
-// CHECK-DAG: [[ENTRY7:@.+]] = weak{{.*}} constant [[ENTTY]] { ptr @{{.*}}, ptr [[NAMEPTR7]], i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // CHECK-DAG: [[NAMEPTR8:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME8:.+]]\00"
-// CHECK-DAG: [[ENTRY8:@.+]] = weak{{.*}} constant [[ENTTY]] { ptr @{{.*}}, ptr [[NAMEPTR8]], i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // CHECK-DAG: [[NAMEPTR9:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME9:.+]]\00"
-// CHECK-DAG: [[ENTRY9:@.+]] = weak{{.*}} constant [[ENTTY]] { ptr @{{.*}}, ptr [[NAMEPTR9]], i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // CHECK-DAG: [[NAMEPTR10:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME10:.+]]\00"
-// CHECK-DAG: [[ENTRY10:@.+]] = weak{{.*}} constant [[ENTTY]] { ptr @{{.*}}, ptr [[NAMEPTR10]], i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // CHECK-DAG: [[NAMEPTR11:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME11:.+]]\00"
-// CHECK-DAG: [[ENTRY11:@.+]] = weak{{.*}} constant [[ENTTY]] { ptr @{{.*}}, ptr [[NAMEPTR11]], i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // CHECK-DAG: [[NAMEPTR12:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME12:.+]]\00"
-// CHECK-DAG: [[ENTRY12:@.+]] = weak{{.*}} constant [[ENTTY]] { ptr @{{.*}}, ptr [[NAMEPTR12]], i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 
 // TCHECK-DAG: [[NAMEPTR1:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME1:__omp_offloading_[0-9a-f]+_[0-9a-f]+__Z.+_l[0-9]+]]\00"
-// TCHECK-DAG: [[ENTRY1:@.+]] = weak{{.*}} constant [[ENTTY]] { ptr @{{.*}}, ptr [[NAMEPTR1]], i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // TCHECK-DAG: [[NAMEPTR2:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME2:.+]]\00"
-// TCHECK-DAG: [[ENTRY2:@.+]] = weak{{.*}} constant [[ENTTY]] { ptr @{{.*}}, ptr [[NAMEPTR2]], i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // TCHECK-DAG: [[NAMEPTR3:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME3:.+]]\00"
-// TCHECK-DAG: [[ENTRY3:@.+]] = weak{{.*}} constant [[ENTTY]] { ptr @{{.*}}, ptr [[NAMEPTR3]], i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // TCHECK-DAG: [[NAMEPTR4:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME4:.+]]\00"
-// TCHECK-DAG: [[ENTRY4:@.+]] = weak{{.*}} constant [[ENTTY]] { ptr @{{.*}}, ptr [[NAMEPTR4]], i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // TCHECK-DAG: [[NAMEPTR5:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME5:.+]]\00"
-// TCHECK-DAG: [[ENTRY5:@.+]] = weak{{.*}} constant [[ENTTY]] { ptr @{{.*}}, ptr [[NAMEPTR5]], i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // TCHECK-DAG: [[NAMEPTR6:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME6:.+]]\00"
-// TCHECK-DAG: [[ENTRY6:@.+]] = weak{{.*}} constant [[ENTTY]] { ptr @{{.*}}, ptr [[NAMEPTR6]], i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // TCHECK-DAG: [[NAMEPTR7:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME7:.+]]\00"
-// TCHECK-DAG: [[ENTRY7:@.+]] = weak{{.*}} constant [[ENTTY]] { ptr @{{.*}}, ptr [[NAMEPTR7]], i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // TCHECK-DAG: [[NAMEPTR8:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME8:.+]]\00"
-// TCHECK-DAG: [[ENTRY8:@.+]] = weak{{.*}} constant [[ENTTY]] { ptr @{{.*}}, ptr [[NAMEPTR8]], i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // TCHECK-DAG: [[NAMEPTR9:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME9:.+]]\00"
-// TCHECK-DAG: [[ENTRY9:@.+]] = weak{{.*}} constant [[ENTTY]] { ptr @{{.*}}, ptr [[NAMEPTR9]], i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // TCHECK-DAG: [[NAMEPTR10:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME10:.+]]\00"
-// TCHECK-DAG: [[ENTRY10:@.+]] = weak{{.*}} constant [[ENTTY]] { ptr @{{.*}}, ptr [[NAMEPTR10]], i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // TCHECK-DAG: [[NAMEPTR11:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME11:.+]]\00"
-// TCHECK-DAG: [[ENTRY11:@.+]] = weak{{.*}} constant [[ENTTY]] { ptr @{{.*}}, ptr [[NAMEPTR11]], i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // TCHECK-DAG: [[NAMEPTR12:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME12:.+]]\00"
-// TCHECK-DAG: [[ENTRY12:@.+]] = weak{{.*}} constant [[ENTTY]] { ptr @{{.*}}, ptr [[NAMEPTR12]], i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 
 // We have 4 initializers, one for the 500 priority, another one for 501, or more for the default priority, and the last one for the offloading registration function.
 // CHECK: @llvm.global_ctors = appending global [3 x { i32, ptr, ptr }] [
@@ -452,31 +428,31 @@ int bar(int a){
 
 // Check metadata is properly generated:
 // CHECK:     !omp_offload.info = !{!{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}}
-// CHECK-DAG: = !{i32 0, i32 [[DEVID:-?[0-9]+]], i32 [[FILEID:-?[0-9]+]], !"_ZN2SB3fooEv", i32 245, i32 0, i32 {{[0-9]+}}}
-// CHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2SDD1Ev", i32 297, i32 0, i32 {{[0-9]+}}}
-// CHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2SEC1Ev", i32 315, i32 0, i32 {{[0-9]+}}}
-// CHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2SED1Ev", i32 322, i32 0, i32 {{[0-9]+}}}
-// CHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2STILi1000EE3fooEv", i32 334, i32 0, i32 {{[0-9]+}}}
-// CHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2STILi100EEC1Ev", i32 341, i32 0, i32 {{[0-9]+}}}
-// CHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_Z3bari", i32 446, i32 0, i32 {{[0-9]+}}}
-// CHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2STILi100EED1Ev", i32 348, i32 0, i32 {{[0-9]+}}}
-// CHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2STILi1000EEC1Ev", i32 341, i32 0, i32 {{[0-9]+}}}
-// CHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2STILi1000EED1Ev", i32 348, i32 0, i32 {{[0-9]+}}}
-// CHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2STILi100EE3fooEv", i32 334, i32 0, i32 {{[0-9]+}}}
-// CHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2SCC1Ev", i32 271, i32 0, i32 {{[0-9]+}}}
+// CHECK-DAG: = !{i32 0, i32 [[DEVID:-?[0-9]+]], i32 [[FILEID:-?[0-9]+]], !"_ZN2SB3fooEv", i32 {{[0-9]+}}, i32 0, i32 {{[0-9]+}}}
+// CHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2SDD1Ev", i32 {{[0-9]+}}, i32 0, i32 {{[0-9]+}}}
+// CHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2SEC1Ev", i32 {{[0-9]+}}, i32 0, i32 {{[0-9]+}}}
+// CHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2SED1Ev", i32 {{[0-9]+}}, i32 0, i32 {{[0-9]+}}}
+// CHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2STILi1000EE3fooEv", i32 {{[0-9]+}}, i32 0, i32 {{[0-9]+}}}
+// CHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2STILi100EEC1Ev", i32 {{[0-9]+}}, i32 0, i32 {{[0-9]+}}}
+// CHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_Z3bari", i32 {{[0-9]+}}, i32 0, i32 {{[0-9]+}}}
+// CHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2STILi100EED1Ev", i32 {{[0-9]+}}, i32 0, i32 {{[0-9]+}}}
+// CHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2STILi1000EEC1Ev", i32 {{[0-9]+}}, i32 0, i32 {{[0-9]+}}}
+// CHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2STILi1000EED1Ev", i32 {{[0-9]+}}, i32 0, i32 {{[0-9]+}}}
+// CHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2STILi100EE3fooEv", i32 {{[0-9]+}}, i32 0, i32 {{[0-9]+}}}
+// CHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2SCC1Ev", i32 {{[0-9]+}}, i32 0, i32 {{[0-9]+}}}
 
 // TCHECK:     !omp_offload.info = !{!{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}}
-// TCHECK-DAG: = !{i32 0, i32 [[DEVID:-?[0-9]+]], i32 [[FILEID:-?[0-9]+]], !"_ZN2SB3fooEv", i32 245, i32 0, i32 {{[0-9]+}}}
-// TCHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2SDD1Ev", i32 297, i32 0, i32 {{[0-9]+}}}
-// TCHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2SEC1Ev", i32 315, i32 0, i32 {{[0-9]+}}}
-// TCHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2SED1Ev", i32 322, i32 0, i32 {{[0-9]+}}}
-// TCHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2STILi1000EE3fooEv", i32 334, i32 0, i32 {{[0-9]+}}}
-// TCHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2STILi100EEC1Ev", i32 341, i32 0, i32 {{[0-9]+}}}
-// TCHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_Z3bari", i32 446, i32 0, i32 {{[0-9]+}}}
-// TCHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2STILi100EED1Ev", i32 348, i32 0, i32 {{[0-9]+}}}
-// TCHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2STILi1000EEC1Ev", i32 341, i32 0, i32 {{[0-9]+}}}
-// TCHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2STILi1000EED1Ev", i32 348, i32 0, i32 {{[0-9]+}}}
-// TCHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2STILi100EE3fooEv", i32 334, i32 0, i32 {{[0-9]+}}}
-// TCHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2SCC1Ev", i32 271, i32 0, i32 {{[0-9]+}}}
+// TCHECK-DAG: = !{i32 0, i32 [[DEVID:-?[0-9]+]], i32 [[FILEID:-?[0-9]+]], !"_ZN2SB3fooEv", i32 {{[0-9]+}}, i32 0, i32 {{[0-9]+}}}
+// TCHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2SDD1Ev", i32 {{[0-9]+}}, i32 0, i32 {{[0-9]+}}}
+// TCHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2SEC1Ev", i32 {{[0-9]+}}, i32 0, i32 {{[0-9]+}}}
+// TCHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2SED1Ev", i32 {{[0-9]+}}, i32 0, i32 {{[0-9]+}}}
+// TCHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2STILi1000EE3fooEv", i32 {{[0-9]+}}, i32 0, i32 {{[0-9]+}}}
+// TCHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2STILi100EEC1Ev", i32 {{[0-9]+}}, i32 0, i32 {{[0-9]+}}}
+// TCHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_Z3bari", i32 {{[0-9]+}}, i32 0, i32 {{[0-9]+}}}
+// TCHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2STILi100EED1Ev", i32 {{[0-9]+}}, i32 0, i32 {{[0-9]+}}}
+// TCHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2STILi1000EEC1Ev", i32 {{[0-9]+}}, i32 0, i32 {{[0-9]+}}}
+// TCHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2STILi1000EED1Ev", i32 {{[0-9]+}}, i32 0, i32 {{[0-9]+}}}
+// TCHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2STILi100EE3fooEv", i32 {{[0-9]+}}, i32 0, i32 {{[0-9]+}}}
+// TCHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2SCC1Ev", i32 {{[0-9]+}}, i32 0, i32 {{[0-9]+}}}
 
 #endif
diff --git a/clang/test/OpenMP/target_parallel_for_depend_codegen.cpp b/clang/test/OpenMP/target_parallel_for_depend_codegen.cpp
index 928ea0397571a..aec4feda15cf0 100644
--- a/clang/test/OpenMP/target_parallel_for_depend_codegen.cpp
+++ b/clang/test/OpenMP/target_parallel_for_depend_codegen.cpp
@@ -39,9 +39,9 @@
 #define HEADER
 
 // CHECK-DAG: [[TT:%.+]] = type { i64, i8 }
-// CHECK-DAG: [[ENTTY:%.+]] = type { ptr, ptr, i[[SZ:32|64]], i32, i32 }
+// CHECK-DAG: [[ENTTY:%.+]] = type { i64, i16, i16, i32, ptr, ptr, i64, i64, ptr }
 
-// TCHECK: [[ENTTY:%.+]] = type { ptr, ptr, i{{32|64}}, i32, i32 }
+// TCHECK: [[ENTTY:%.+]] = type { i64, i16, i16, i32, ptr, ptr, i64, i64, ptr }
 
 // CHECK-DAG: [[SIZET:@.+]] = private unnamed_addr constant [2 x i64] [i64 0, i64 4]
 // CHECK-DAG: [[MAPT:@.+]] = private unnamed_addr constant [2 x i64] [i64 544, i64 800]
@@ -76,7 +76,7 @@ int foo(int n) {
   // CHECK:       [[GEP:%.+]] = getelementptr inbounds nuw %{{.+}}, ptr %{{.+}}, i32 0, i32 0
   // CHECK:       [[DEV:%.+]] = load i32, ptr [[DEVICE_CAP]],
   // CHECK:       store i32 [[DEV]], ptr [[GEP]],
-  // CHECK:       [[TASK:%.+]] = call ptr @__kmpc_omp_task_alloc(ptr [[IN:@.+]], i32 [[GTID:%.+]], i32 1, i[[SZ]] {{20|40}}, i[[SZ]] 4, ptr [[TASK_ENTRY0:@.+]])
+  // CHECK:       [[TASK:%.+]] = call ptr @__kmpc_omp_task_alloc(ptr [[IN:@.+]], i32 [[GTID:%.+]], i32 1, i[[SZ:32|64]] {{20|40}}, i[[SZ]] 4, ptr [[TASK_ENTRY0:@.+]])
   // CHECK:       getelementptr %struct.kmp_depend_info, ptr %{{.+}}, i[[SZ]] 0
   // CHECK:       getelementptr %struct.kmp_depend_info, ptr %{{.+}}, i[[SZ]] 1
   // CHECK:       getelementptr %struct.kmp_depend_info, ptr %{{.+}}, i[[SZ]] 2
diff --git a/clang/test/OpenMP/target_parallel_for_simd_codegen_registration.cpp b/clang/test/OpenMP/target_parallel_for_simd_codegen_registration.cpp
index 4c996c88de530..312ad5615ca1c 100644
--- a/clang/test/OpenMP/target_parallel_for_simd_codegen_registration.cpp
+++ b/clang/test/OpenMP/target_parallel_for_simd_codegen_registration.cpp
@@ -92,9 +92,9 @@
 // CHECK-DAG: [[SE:%.+]] = type { [64 x i32] }
 // CHECK-DAG: [[ST1:%.+]] = type { [228 x i32] }
 // CHECK-DAG: [[ST2:%.+]] = type { [1128 x i32] }
-// CHECK-DAG: [[ENTTY:%.+]] = type { ptr, ptr, i[[SZ:32|64]], i32, i32 }
+// CHECK-DAG: [[ENTTY:%.+]] = type { i64, i16, i16, i32, ptr, ptr, i64, i64, ptr }
 
-// TCHECK:    [[ENTTY:%.+]] = type { ptr, ptr, i[[SZ:32|64]], i32, i32 }
+// TCHECK: [[ENTTY:%.+]] = type { i64, i16, i16, i32, ptr, ptr, i64, i64, ptr }
 
 // CHECK-DAG: [[A1:@.+]] = internal global [[SA]]
 // CHECK-DAG: [[A2:@.+]] ={{.*}} global [[SA]]
@@ -160,54 +160,30 @@
 // CHECK-NTARGET-NOT: private unnamed_addr constant [1 x i
 
 // CHECK-DAG: [[NAMEPTR1:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME1:__omp_offloading_[0-9a-f]+_[0-9a-f]+__Z.+_l[0-9]+]]\00"
-// CHECK-DAG: [[ENTRY1:@.+]] = weak{{.*}} constant [[ENTTY]] { ptr @{{.*}}, ptr [[NAMEPTR1]], i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // CHECK-DAG: [[NAMEPTR2:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME2:.+]]\00"
-// CHECK-DAG: [[ENTRY2:@.+]] = weak{{.*}} constant [[ENTTY]] { ptr @{{.*}}, ptr [[NAMEPTR2]], i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // CHECK-DAG: [[NAMEPTR3:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME3:.+]]\00"
-// CHECK-DAG: [[ENTRY3:@.+]] = weak{{.*}} constant [[ENTTY]] { ptr @{{.*}}, ptr [[NAMEPTR3]], i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // CHECK-DAG: [[NAMEPTR4:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME4:.+]]\00"
-// CHECK-DAG: [[ENTRY4:@.+]] = weak{{.*}} constant [[ENTTY]] { ptr @{{.*}}, ptr [[NAMEPTR4]], i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // CHECK-DAG: [[NAMEPTR5:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME5:.+]]\00"
-// CHECK-DAG: [[ENTRY5:@.+]] = weak{{.*}} constant [[ENTTY]] { ptr @{{.*}}, ptr [[NAMEPTR5]], i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // CHECK-DAG: [[NAMEPTR6:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME6:.+]]\00"
-// CHECK-DAG: [[ENTRY6:@.+]] = weak{{.*}} constant [[ENTTY]] { ptr @{{.*}}, ptr [[NAMEPTR6]], i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // CHECK-DAG: [[NAMEPTR7:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME7:.+]]\00"
-// CHECK-DAG: [[ENTRY7:@.+]] = weak{{.*}} constant [[ENTTY]] { ptr @{{.*}}, ptr [[NAMEPTR7]], i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // CHECK-DAG: [[NAMEPTR8:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME8:.+]]\00"
-// CHECK-DAG: [[ENTRY8:@.+]] = weak{{.*}} constant [[ENTTY]] { ptr @{{.*}}, ptr [[NAMEPTR8]], i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // CHECK-DAG: [[NAMEPTR9:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME9:.+]]\00"
-// CHECK-DAG: [[ENTRY9:@.+]] = weak{{.*}} constant [[ENTTY]] { ptr @{{.*}}, ptr [[NAMEPTR9]], i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // CHECK-DAG: [[NAMEPTR10:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME10:.+]]\00"
-// CHECK-DAG: [[ENTRY10:@.+]] = weak{{.*}} constant [[ENTTY]] { ptr @{{.*}}, ptr [[NAMEPTR10]], i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // CHECK-DAG: [[NAMEPTR11:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME11:.+]]\00"
-// CHECK-DAG: [[ENTRY11:@.+]] = weak{{.*}} constant [[ENTTY]] { ptr @{{.*}}, ptr [[NAMEPTR11]], i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // CHECK-DAG: [[NAMEPTR12:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME12:.+]]\00"
-// CHECK-DAG: [[ENTRY12:@.+]] = weak{{.*}} constant [[ENTTY]] { ptr @{{.*}}, ptr [[NAMEPTR12]], i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 
 // TCHECK-DAG: [[NAMEPTR1:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME1:__omp_offloading_[0-9a-f]+_[0-9a-f]+__Z.+_l[0-9]+]]\00"
-// TCHECK-DAG: [[ENTRY1:@.+]] = weak{{.*}} constant [[ENTTY]] { ptr @{{.*}}, ptr [[NAMEPTR1]], i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // TCHECK-DAG: [[NAMEPTR2:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME2:.+]]\00"
-// TCHECK-DAG: [[ENTRY2:@.+]] = weak{{.*}} constant [[ENTTY]] { ptr @{{.*}}, ptr [[NAMEPTR2]], i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // TCHECK-DAG: [[NAMEPTR3:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME3:.+]]\00"
-// TCHECK-DAG: [[ENTRY3:@.+]] = weak{{.*}} constant [[ENTTY]] { ptr @{{.*}}, ptr [[NAMEPTR3]], i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // TCHECK-DAG: [[NAMEPTR4:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME4:.+]]\00"
-// TCHECK-DAG: [[ENTRY4:@.+]] = weak{{.*}} constant [[ENTTY]] { ptr @{{.*}}, ptr [[NAMEPTR4]], i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // TCHECK-DAG: [[NAMEPTR5:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME5:.+]]\00"
-// TCHECK-DAG: [[ENTRY5:@.+]] = weak{{.*}} constant [[ENTTY]] { ptr @{{.*}}, ptr [[NAMEPTR5]], i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // TCHECK-DAG: [[NAMEPTR6:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME6:.+]]\00"
-// TCHECK-DAG: [[ENTRY6:@.+]] = weak{{.*}} constant [[ENTTY]] { ptr @{{.*}}, ptr [[NAMEPTR6]], i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // TCHECK-DAG: [[NAMEPTR7:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME7:.+]]\00"
-// TCHECK-DAG: [[ENTRY7:@.+]] = weak{{.*}} constant [[ENTTY]] { ptr @{{.*}}, ptr [[NAMEPTR7]], i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // TCHECK-DAG: [[NAMEPTR8:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME8:.+]]\00"
-// TCHECK-DAG: [[ENTRY8:@.+]] = weak{{.*}} constant [[ENTTY]] { ptr @{{.*}}, ptr [[NAMEPTR8]], i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // TCHECK-DAG: [[NAMEPTR9:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME9:.+]]\00"
-// TCHECK-DAG: [[ENTRY9:@.+]] = weak{{.*}} constant [[ENTTY]] { ptr @{{.*}}, ptr [[NAMEPTR9]], i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // TCHECK-DAG: [[NAMEPTR10:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME10:.+]]\00"
-// TCHECK-DAG: [[ENTRY10:@.+]] = weak{{.*}} constant [[ENTTY]] { ptr @{{.*}}, ptr [[NAMEPTR10]], i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // TCHECK-DAG: [[NAMEPTR11:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME11:.+]]\00"
-// TCHECK-DAG: [[ENTRY11:@.+]] = weak{{.*}} constant [[ENTTY]] { ptr @{{.*}}, ptr [[NAMEPTR11]], i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // TCHECK-DAG: [[NAMEPTR12:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME12:.+]]\00"
-// TCHECK-DAG: [[ENTRY12:@.+]] = weak{{.*}} constant [[ENTTY]] { ptr @{{.*}}, ptr [[NAMEPTR12]], i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 
 // We have 4 initializers, one for the 500 priority, another one for 501, or more for the default priority, and the last one for the offloading registration function.
 // CHECK: @llvm.global_ctors = appending global [3 x { i32, ptr, ptr }] [
@@ -452,31 +428,31 @@ int bar(int a){
 
 // Check metadata is properly generated:
 // CHECK:     !omp_offload.info = !{!{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}}
-// CHECK-DAG: = !{i32 0, i32 [[DEVID:-?[0-9]+]], i32 [[FILEID:-?[0-9]+]], !"_ZN2SB3fooEv", i32 245, i32 0, i32 {{[0-9]+}}}
-// CHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2SDD1Ev", i32 297, i32 0, i32 {{[0-9]+}}}
-// CHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2SEC1Ev", i32 315, i32 0, i32 {{[0-9]+}}}
-// CHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2SED1Ev", i32 322, i32 0, i32 {{[0-9]+}}}
-// CHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2STILi1000EE3fooEv", i32 334, i32 0, i32 {{[0-9]+}}}
-// CHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2STILi100EEC1Ev", i32 341, i32 0, i32 {{[0-9]+}}}
-// CHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_Z3bari", i32 446, i32 0, i32 {{[0-9]+}}}
-// CHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2STILi100EED1Ev", i32 348, i32 0, i32 {{[0-9]+}}}
-// CHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2STILi1000EEC1Ev", i32 341, i32 0, i32 {{[0-9]+}}}
-// CHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2STILi1000EED1Ev", i32 348, i32 0, i32 {{[0-9]+}}}
-// CHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2STILi100EE3fooEv", i32 334, i32 0, i32 {{[0-9]+}}}
-// CHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2SCC1Ev", i32 271, i32 0, i32 {{[0-9]+}}}
+// CHECK-DAG: = !{i32 0, i32 [[DEVID:-?[0-9]+]], i32 [[FILEID:-?[0-9]+]], !"_ZN2SB3fooEv", i32 {{[0-9]+}}, i32 0, i32 {{[0-9]+}}}
+// CHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2SDD1Ev", i32 {{[0-9]+}}, i32 0, i32 {{[0-9]+}}}
+// CHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2SEC1Ev", i32 {{[0-9]+}}, i32 0, i32 {{[0-9]+}}}
+// CHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2SED1Ev", i32 {{[0-9]+}}, i32 0, i32 {{[0-9]+}}}
+// CHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2STILi1000EE3fooEv", i32 {{[0-9]+}}, i32 0, i32 {{[0-9]+}}}
+// CHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2STILi100EEC1Ev", i32 {{[0-9]+}}, i32 0, i32 {{[0-9]+}}}
+// CHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_Z3bari", i32 {{[0-9]+}}, i32 0, i32 {{[0-9]+}}}
+// CHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2STILi100EED1Ev", i32 {{[0-9]+}}, i32 0, i32 {{[0-9]+}}}
+// CHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2STILi1000EEC1Ev", i32 {{[0-9]+}}, i32 0, i32 {{[0-9]+}}}
+// CHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2STILi1000EED1Ev", i32 {{[0-9]+}}, i32 0, i32 {{[0-9]+}}}
+// CHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2STILi100EE3fooEv", i32 {{[0-9]+}}, i32 0, i32 {{[0-9]+}}}
+// CHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2SCC1Ev", i32 {{[0-9]+}}, i32 0, i32 {{[0-9]+}}}
 
 // TCHECK:     !omp_offload.info = !{!{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}}
-// TCHECK-DAG: = !{i32 0, i32 [[DEVID:-?[0-9]+]], i32 [[FILEID:-?[0-9]+]], !"_ZN2SB3fooEv", i32 245, i32 0, i32 {{[0-9]+}}}
-// TCHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2SDD1Ev", i32 297, i32 0, i32 {{[0-9]+}}}
-// TCHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2SEC1Ev", i32 315, i32 0, i32 {{[0-9]+}}}
-// TCHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2SED1Ev", i32 322, i32 0, i32 {{[0-9]+}}}
-// TCHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2STILi1000EE3fooEv", i32 334, i32 0, i32 {{[0-9]+}}}
-// TCHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2STILi100EEC1Ev", i32 341, i32 0, i32 {{[0-9]+}}}
-// TCHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_Z3bari", i32 446, i32 0, i32 {{[0-9]+}}}
-// TCHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2STILi100EED1Ev", i32 348, i32 0, i32 {{[0-9]+}}}
-// TCHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2STILi1000EEC1Ev", i32 341, i32 0, i32 {{[0-9]+}}}
-// TCHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2STILi1000EED1Ev", i32 348, i32 0, i32 {{[0-9]+}}}
-// TCHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2STILi100EE3fooEv", i32 334, i32 0, i32 {{[0-9]+}}}
-// TCHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2SCC1Ev", i32 271, i32 0, i32 {{[0-9]+}}}
+// TCHECK-DAG: = !{i32 0, i32 [[DEVID:-?[0-9]+]], i32 [[FILEID:-?[0-9]+]], !"_ZN2SB3fooEv", i32 {{[0-9]+}}, i32 0, i32 {{[0-9]+}}}
+// TCHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2SDD1Ev", i32 {{[0-9]+}}, i32 0, i32 {{[0-9]+}}}
+// TCHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2SEC1Ev", i32 {{[0-9]+}}, i32 0, i32 {{[0-9]+}}}
+// TCHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2SED1Ev", i32 {{[0-9]+}}, i32 0, i32 {{[0-9]+}}}
+// TCHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2STILi1000EE3fooEv", i32 {{[0-9]+}}, i32 0, i32 {{[0-9]+}}}
+// TCHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2STILi100EEC1Ev", i32 {{[0-9]+}}, i32 0, i32 {{[0-9]+}}}
+// TCHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_Z3bari", i32 {{[0-9]+}}, i32 0, i32 {{[0-9]+}}}
+// TCHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2STILi100EED1Ev", i32 {{[0-9]+}}, i32 0, i32 {{[0-9]+}}}
+// TCHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2STILi1000EEC1Ev", i32 {{[0-9]+}}, i32 0, i32 {{[0-9]+}}}
+// TCHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2STILi1000EED1Ev", i32 {{[0-9]+}}, i32 0, i32 {{[0-9]+}}}
+// TCHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2STILi100EE3fooEv", i32 {{[0-9]+}}, i32 0, i32 {{[0-9]+}}}
+// TCHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2SCC1Ev", i32 {{[0-9]+}}, i32 0, i32 {{[0-9]+}}}
 
 #endif
diff --git a/clang/test/OpenMP/target_parallel_for_simd_depend_codegen.cpp b/clang/test/OpenMP/target_parallel_for_simd_depend_codegen.cpp
index bb16edbb8a090..62f772ea79156 100644
--- a/clang/test/OpenMP/target_parallel_for_simd_depend_codegen.cpp
+++ b/clang/test/OpenMP/target_parallel_for_simd_depend_codegen.cpp
@@ -39,9 +39,9 @@
 #define HEADER
 
 // CHECK-DAG: [[TT:%.+]] = type { i64, i8 }
-// CHECK-DAG: [[ENTTY:%.+]] = type { ptr, ptr, i[[SZ:32|64]], i32, i32 }
+// CHECK-DAG: [[ENTTY:%.+]] = type { i64, i16, i16, i32, ptr, ptr, i64, i64, ptr }
 
-// TCHECK: [[ENTTY:%.+]] = type { ptr, ptr, i{{32|64}}, i32, i32 }
+// TCHECK: [[ENTTY:%.+]] = type { i64, i16, i16, i32, ptr, ptr, i64, i64, ptr }
 
 // CHECK-DAG: [[SIZET:@.+]] = private unnamed_addr constant [2 x i64] [i64 0, i64 4]
 // CHECK-DAG: [[MAPT:@.+]] = private unnamed_addr constant [2 x i64] [i64 544, i64 800]
@@ -76,7 +76,7 @@ int foo(int n) {
   // CHECK:       [[GEP:%.+]] = getelementptr inbounds nuw %{{.+}}, ptr %{{.+}}, i32 0, i32 0
   // CHECK:       [[DEV:%.+]] = load i32, ptr [[DEVICE_CAP]],
   // CHECK:       store i32 [[DEV]], ptr [[GEP]],
-  // CHECK:       [[TASK:%.+]] = call ptr @__kmpc_omp_task_alloc(ptr [[IN:@.+]], i32 [[GTID:%.+]], i32 1, i[[SZ]] {{20|40}}, i[[SZ]] 4, ptr [[TASK_ENTRY0:@.+]])
+  // CHECK:       [[TASK:%.+]] = call ptr @__kmpc_omp_task_alloc(ptr [[IN:@.+]], i32 [[GTID:%.+]], i32 1, i[[SZ:32|64]] {{20|40}}, i[[SZ]] 4, ptr [[TASK_ENTRY0:@.+]])
   // CHECK:       getelementptr %struct.kmp_depend_info, ptr %{{.+}}, i[[SZ]] 0
   // CHECK:       getelementptr %struct.kmp_depend_info, ptr %{{.+}}, i[[SZ]] 1
   // CHECK:       getelementptr %struct.kmp_depend_info, ptr %{{.+}}, i[[SZ]] 2
diff --git a/clang/test/OpenMP/target_simd_codegen.cpp b/clang/test/OpenMP/target_simd_codegen.cpp
index e2ff3d5f53e44..c8ecaa814bafe 100644
--- a/clang/test/OpenMP/target_simd_codegen.cpp
+++ b/clang/test/OpenMP/target_simd_codegen.cpp
@@ -71,9 +71,9 @@
 // CHECK-DAG: [[KMP_TASK_T]] = type { ptr, ptr, i32, %{{[^,]+}}, %{{[^,]+}} }
 // CHECK-DAG: [[TT:%.+]] = type { i64, i8 }
 // CHECK-DAG: [[S1:%.+]] = type { double }
-// CHECK-DAG: [[ENTTY:%.+]] = type { ptr, ptr, i[[SZ:32|64]], i32, i32 }
+// CHECK-DAG: [[ENTTY:%.+]] = type { i64, i16, i16, i32, ptr, ptr, i64, i64, ptr }
 
-// TCHECK: [[ENTTY:%.+]] = type { ptr, ptr, i{{32|64}}, i32, i32 }
+// TCHECK: [[ENTTY:%.+]] = type { i64, i16, i16, i32, ptr, ptr, i64, i64, ptr }
 
 // We have 8 target regions, but only 7 that actually will generate offloading
 // code, only 6 will have mapped arguments, and only 4 have all-constant map
@@ -136,7 +136,7 @@ int foo(int n) {
   for (int i = 3; i < 32; i += 5) {
   }
 
-  // CHECK:       call void [[HVT1:@.+]](i[[SZ]] {{[^,]+}}, {{[^)]+}})
+  // CHECK:       call void [[HVT1:@.+]](i[[SZ:32|64]] {{[^,]+}}, {{[^)]+}})
   long long k = get_val();
   #pragma omp target simd if(target: 0) linear(k : 3)
   for (int i = 10; i > 1; i--) {
diff --git a/clang/test/OpenMP/target_simd_codegen_registration.cpp b/clang/test/OpenMP/target_simd_codegen_registration.cpp
index 4c0ca8947b3fc..69cfe5c8e200f 100644
--- a/clang/test/OpenMP/target_simd_codegen_registration.cpp
+++ b/clang/test/OpenMP/target_simd_codegen_registration.cpp
@@ -92,9 +92,9 @@
 // CHECK-DAG: [[SE:%.+]] = type { [64 x i32] }
 // CHECK-DAG: [[ST1:%.+]] = type { [228 x i32] }
 // CHECK-DAG: [[ST2:%.+]] = type { [1128 x i32] }
-// CHECK-DAG: [[ENTTY:%.+]] = type { ptr, ptr, i[[SZ:32|64]], i32, i32 }
+// CHECK-DAG: [[ENTTY:%.+]] = type { i64, i16, i16, i32, ptr, ptr, i64, i64, ptr }
 
-// TCHECK:    [[ENTTY:%.+]] = type { ptr, ptr, i[[SZ:32|64]], i32, i32 }
+// TCHECK: [[ENTTY:%.+]] = type { i64, i16, i16, i32, ptr, ptr, i64, i64, ptr }
 
 // CHECK-DAG: [[A1:@.+]] = internal global [[SA]]
 // CHECK-DAG: [[A2:@.+]] ={{.*}} global [[SA]]
@@ -160,54 +160,30 @@
 // CHECK-NTARGET-NOT: private unnamed_addr constant [1 x i
 
 // CHECK-DAG: [[NAMEPTR1:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME1:__omp_offloading_[0-9a-f]+_[0-9a-f]+__Z.+_l[0-9]+]]\00"
-// CHECK-DAG: [[ENTRY1:@.+]] = weak{{.*}} constant [[ENTTY]] { ptr @{{.*}}, ptr [[NAMEPTR1]], i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // CHECK-DAG: [[NAMEPTR2:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME2:.+]]\00"
-// CHECK-DAG: [[ENTRY2:@.+]] = weak{{.*}} constant [[ENTTY]] { ptr @{{.*}}, ptr [[NAMEPTR2]], i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // CHECK-DAG: [[NAMEPTR3:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME3:.+]]\00"
-// CHECK-DAG: [[ENTRY3:@.+]] = weak{{.*}} constant [[ENTTY]] { ptr @{{.*}}, ptr [[NAMEPTR3]], i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // CHECK-DAG: [[NAMEPTR4:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME4:.+]]\00"
-// CHECK-DAG: [[ENTRY4:@.+]] = weak{{.*}} constant [[ENTTY]] { ptr @{{.*}}, ptr [[NAMEPTR4]], i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // CHECK-DAG: [[NAMEPTR5:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME5:.+]]\00"
-// CHECK-DAG: [[ENTRY5:@.+]] = weak{{.*}} constant [[ENTTY]] { ptr @{{.*}}, ptr [[NAMEPTR5]], i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // CHECK-DAG: [[NAMEPTR6:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME6:.+]]\00"
-// CHECK-DAG: [[ENTRY6:@.+]] = weak{{.*}} constant [[ENTTY]] { ptr @{{.*}}, ptr [[NAMEPTR6]], i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // CHECK-DAG: [[NAMEPTR7:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME7:.+]]\00"
-// CHECK-DAG: [[ENTRY7:@.+]] = weak{{.*}} constant [[ENTTY]] { ptr @{{.*}}, ptr [[NAMEPTR7]], i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // CHECK-DAG: [[NAMEPTR8:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME8:.+]]\00"
-// CHECK-DAG: [[ENTRY8:@.+]] = weak{{.*}} constant [[ENTTY]] { ptr @{{.*}}, ptr [[NAMEPTR8]], i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // CHECK-DAG: [[NAMEPTR9:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME9:.+]]\00"
-// CHECK-DAG: [[ENTRY9:@.+]] = weak{{.*}} constant [[ENTTY]] { ptr @{{.*}}, ptr [[NAMEPTR9]], i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // CHECK-DAG: [[NAMEPTR10:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME10:.+]]\00"
-// CHECK-DAG: [[ENTRY10:@.+]] = weak{{.*}} constant [[ENTTY]] { ptr @{{.*}}, ptr [[NAMEPTR10]], i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // CHECK-DAG: [[NAMEPTR11:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME11:.+]]\00"
-// CHECK-DAG: [[ENTRY11:@.+]] = weak{{.*}} constant [[ENTTY]] { ptr @{{.*}}, ptr [[NAMEPTR11]], i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // CHECK-DAG: [[NAMEPTR12:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME12:.+]]\00"
-// CHECK-DAG: [[ENTRY12:@.+]] = weak{{.*}} constant [[ENTTY]] { ptr @{{.*}}, ptr [[NAMEPTR12]], i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 
 // TCHECK-DAG: [[NAMEPTR1:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME1:__omp_offloading_[0-9a-f]+_[0-9a-f]+__Z.+_l[0-9]+]]\00"
-// TCHECK-DAG: [[ENTRY1:@.+]] = weak{{.*}} constant [[ENTTY]] { ptr @{{.*}}, ptr [[NAMEPTR1]], i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // TCHECK-DAG: [[NAMEPTR2:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME2:.+]]\00"
-// TCHECK-DAG: [[ENTRY2:@.+]] = weak{{.*}} constant [[ENTTY]] { ptr @{{.*}}, ptr [[NAMEPTR2]], i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // TCHECK-DAG: [[NAMEPTR3:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME3:.+]]\00"
-// TCHECK-DAG: [[ENTRY3:@.+]] = weak{{.*}} constant [[ENTTY]] { ptr @{{.*}}, ptr [[NAMEPTR3]], i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // TCHECK-DAG: [[NAMEPTR4:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME4:.+]]\00"
-// TCHECK-DAG: [[ENTRY4:@.+]] = weak{{.*}} constant [[ENTTY]] { ptr @{{.*}}, ptr [[NAMEPTR4]], i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // TCHECK-DAG: [[NAMEPTR5:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME5:.+]]\00"
-// TCHECK-DAG: [[ENTRY5:@.+]] = weak{{.*}} constant [[ENTTY]] { ptr @{{.*}}, ptr [[NAMEPTR5]], i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // TCHECK-DAG: [[NAMEPTR6:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME6:.+]]\00"
-// TCHECK-DAG: [[ENTRY6:@.+]] = weak{{.*}} constant [[ENTTY]] { ptr @{{.*}}, ptr [[NAMEPTR6]], i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // TCHECK-DAG: [[NAMEPTR7:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME7:.+]]\00"
-// TCHECK-DAG: [[ENTRY7:@.+]] = weak{{.*}} constant [[ENTTY]] { ptr @{{.*}}, ptr [[NAMEPTR7]], i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // TCHECK-DAG: [[NAMEPTR8:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME8:.+]]\00"
-// TCHECK-DAG: [[ENTRY8:@.+]] = weak{{.*}} constant [[ENTTY]] { ptr @{{.*}}, ptr [[NAMEPTR8]], i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // TCHECK-DAG: [[NAMEPTR9:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME9:.+]]\00"
-// TCHECK-DAG: [[ENTRY9:@.+]] = weak{{.*}} constant [[ENTTY]] { ptr @{{.*}}, ptr [[NAMEPTR9]], i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // TCHECK-DAG: [[NAMEPTR10:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME10:.+]]\00"
-// TCHECK-DAG: [[ENTRY10:@.+]] = weak{{.*}} constant [[ENTTY]] { ptr @{{.*}}, ptr [[NAMEPTR10]], i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // TCHECK-DAG: [[NAMEPTR11:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME11:.+]]\00"
-// TCHECK-DAG: [[ENTRY11:@.+]] = weak{{.*}} constant [[ENTTY]] { ptr @{{.*}}, ptr [[NAMEPTR11]], i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // TCHECK-DAG: [[NAMEPTR12:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME12:.+]]\00"
-// TCHECK-DAG: [[ENTRY12:@.+]] = weak{{.*}} constant [[ENTTY]] { ptr @{{.*}}, ptr [[NAMEPTR12]], i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 
 // We have 4 initializers, one for the 500 priority, another one for 501, or more for the default priority, and the last one for the offloading registration function.
 // CHECK: @llvm.global_ctors = appending global [3 x { i32, ptr, ptr }] [
@@ -452,31 +428,31 @@ int bar(int a){
 
 // Check metadata is properly generated:
 // CHECK:     !omp_offload.info = !{!{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}}
-// CHECK-DAG: = !{i32 0, i32 [[DEVID:-?[0-9]+]], i32 [[FILEID:-?[0-9]+]], !"_ZN2SB3fooEv", i32 245, i32 0, i32 {{[0-9]+}}}
-// CHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2SDD1Ev", i32 297, i32 0, i32 {{[0-9]+}}}
-// CHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2SEC1Ev", i32 315, i32 0, i32 {{[0-9]+}}}
-// CHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2SED1Ev", i32 322, i32 0, i32 {{[0-9]+}}}
-// CHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2STILi1000EE3fooEv", i32 334, i32 0, i32 {{[0-9]+}}}
-// CHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2STILi100EEC1Ev", i32 341, i32 0, i32 {{[0-9]+}}}
-// CHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_Z3bari", i32 446, i32 0, i32 {{[0-9]+}}}
-// CHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2STILi100EED1Ev", i32 348, i32 0, i32 {{[0-9]+}}}
-// CHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2STILi1000EEC1Ev", i32 341, i32 0, i32 {{[0-9]+}}}
-// CHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2STILi1000EED1Ev", i32 348, i32 0, i32 {{[0-9]+}}}
-// CHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2STILi100EE3fooEv", i32 334, i32 0, i32 {{[0-9]+}}}
-// CHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2SCC1Ev", i32 271, i32 0, i32 {{[0-9]+}}}
+// CHECK-DAG: = !{i32 0, i32 [[DEVID:-?[0-9]+]], i32 [[FILEID:-?[0-9]+]], !"_ZN2SB3fooEv", i32 {{[0-9]+}}, i32 0, i32 {{[0-9]+}}}
+// CHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2SDD1Ev", i32 {{[0-9]+}}, i32 0, i32 {{[0-9]+}}}
+// CHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2SEC1Ev", i32 {{[0-9]+}}, i32 0, i32 {{[0-9]+}}}
+// CHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2SED1Ev", i32 {{[0-9]+}}, i32 0, i32 {{[0-9]+}}}
+// CHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2STILi1000EE3fooEv", i32 {{[0-9]+}}, i32 0, i32 {{[0-9]+}}}
+// CHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2STILi100EEC1Ev", i32 {{[0-9]+}}, i32 0, i32 {{[0-9]+}}}
+// CHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_Z3bari", i32 {{[0-9]+}}, i32 0, i32 {{[0-9]+}}}
+// CHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2STILi100EED1Ev", i32 {{[0-9]+}}, i32 0, i32 {{[0-9]+}}}
+// CHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2STILi1000EEC1Ev", i32 {{[0-9]+}}, i32 0, i32 {{[0-9]+}}}
+// CHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2STILi1000EED1Ev", i32 {{[0-9]+}}, i32 0, i32 {{[0-9]+}}}
+// CHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2STILi100EE3fooEv", i32 {{[0-9]+}}, i32 0, i32 {{[0-9]+}}}
+// CHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2SCC1Ev", i32 {{[0-9]+}}, i32 0, i32 {{[0-9]+}}}
 
 // TCHECK:     !omp_offload.info = !{!{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}}
-// TCHECK-DAG: = !{i32 0, i32 [[DEVID:-?[0-9]+]], i32 [[FILEID:-?[0-9]+]], !"_ZN2SB3fooEv", i32 245, i32 0, i32 {{[0-9]+}}}
-// TCHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2SDD1Ev", i32 297, i32 0, i32 {{[0-9]+}}}
-// TCHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2SEC1Ev", i32 315, i32 0, i32 {{[0-9]+}}}
-// TCHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2SED1Ev", i32 322, i32 0, i32 {{[0-9]+}}}
-// TCHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2STILi1000EE3fooEv", i32 334, i32 0, i32 {{[0-9]+}}}
-// TCHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2STILi100EEC1Ev", i32 341, i32 0, i32 {{[0-9]+}}}
-// TCHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_Z3bari", i32 446, i32 0, i32 {{[0-9]+}}}
-// TCHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2STILi100EED1Ev", i32 348, i32 0, i32 {{[0-9]+}}}
-// TCHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2STILi1000EEC1Ev", i32 341, i32 0, i32 {{[0-9]+}}}
-// TCHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2STILi1000EED1Ev", i32 348, i32 0, i32 {{[0-9]+}}}
-// TCHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2STILi100EE3fooEv", i32 334, i32 0, i32 {{[0-9]+}}}
-// TCHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2SCC1Ev", i32 271, i32 0, i32 {{[0-9]+}}}
+// TCHECK-DAG: = !{i32 0, i32 [[DEVID:-?[0-9]+]], i32 [[FILEID:-?[0-9]+]], !"_ZN2SB3fooEv", i32 {{[0-9]+}}, i32 0, i32 {{[0-9]+}}}
+// TCHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2SDD1Ev", i32 {{[0-9]+}}, i32 0, i32 {{[0-9]+}}}
+// TCHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2SEC1Ev", i32 {{[0-9]+}}, i32 0, i32 {{[0-9]+}}}
+// TCHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2SED1Ev", i32 {{[0-9]+}}, i32 0, i32 {{[0-9]+}}}
+// TCHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2STILi1000EE3fooEv", i32 {{[0-9]+}}, i32 0, i32 {{[0-9]+}}}
+// TCHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2STILi100EEC1Ev", i32 {{[0-9]+}}, i32 0, i32 {{[0-9]+}}}
+// TCHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_Z3bari", i32 {{[0-9]+}}, i32 0, i32 {{[0-9]+}}}
+// TCHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2STILi100EED1Ev", i32 {{[0-9]+}}, i32 0, i32 {{[0-9]+}}}
+// TCHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2STILi1000EEC1Ev", i32 {{[0-9]+}}, i32 0, i32 {{[0-9]+}}}
+// TCHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2STILi1000EED1Ev", i32 {{[0-9]+}}, i32 0, i32 {{[0-9]+}}}
+// TCHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2STILi100EE3fooEv", i32 {{[0-9]+}}, i32 0, i32 {{[0-9]+}}}
+// TCHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2SCC1Ev", i32 {{[0-9]+}}, i32 0, i32 {{[0-9]+}}}
 
 #endif
diff --git a/clang/test/OpenMP/target_simd_depend_codegen.cpp b/clang/test/OpenMP/target_simd_depend_codegen.cpp
index d127078af5f4e..d813173dffbef 100644
--- a/clang/test/OpenMP/target_simd_depend_codegen.cpp
+++ b/clang/test/OpenMP/target_simd_depend_codegen.cpp
@@ -39,9 +39,9 @@
 #define HEADER
 
 // CHECK-DAG: [[TT:%.+]] = type { i64, i8 }
-// CHECK-DAG: [[ENTTY:%.+]] = type { ptr, ptr, i[[SZ:32|64]], i32, i32 }
+// CHECK-DAG: [[ENTTY:%.+]] = type { i64, i16, i16, i32, ptr, ptr, i64, i64, ptr }
 
-// TCHECK: [[ENTTY:%.+]] = type { ptr, ptr, i{{32|64}}, i32, i32 }
+// TCHECK: [[ENTTY:%.+]] = type { i64, i16, i16, i32, ptr, ptr, i64, i64, ptr }
 
 // CHECK-DAG: [[SIZET:@.+]] = private unnamed_addr constant [2 x i64] [i64 0, i64 4]
 // CHECK-DAG: [[MAPT:@.+]] = private unnamed_addr constant [2 x i64] [i64 544, i64 800]
@@ -76,7 +76,7 @@ int foo(int n) {
 // CHECK:       [[GEP:%.+]] = getelementptr inbounds nuw %{{.+}}, ptr %{{.+}}, i32 0, i32 0
 // CHECK:       [[DEV:%.+]] = load i32, ptr [[DEVICE_CAP]],
 // CHECK:       store i32 [[DEV]], ptr [[GEP]],
-// CHECK:       [[TASK:%.+]] = call ptr @__kmpc_omp_task_alloc(ptr @1, i32 [[GTID:%.+]], i32 1, i[[SZ]] {{20|40}}, i[[SZ]] 4, ptr [[TASK_ENTRY0:@.+]])
+// CHECK:       [[TASK:%.+]] = call ptr @__kmpc_omp_task_alloc(ptr @1, i32 [[GTID:%.+]], i32 1, i[[SZ:32|64]] {{20|40}}, i[[SZ]] 4, ptr [[TASK_ENTRY0:@.+]])
 // CHECK:       getelementptr %struct.kmp_depend_info, ptr %{{.+}}, i[[SZ]] 0
 // CHECK:       getelementptr %struct.kmp_depend_info, ptr %{{.+}}, i[[SZ]] 1
 // CHECK:       getelementptr %struct.kmp_depend_info, ptr %{{.+}}, i[[SZ]] 2
diff --git a/clang/test/OpenMP/target_teams_codegen_registration.cpp b/clang/test/OpenMP/target_teams_codegen_registration.cpp
index 60c596853a64f..0627f7a822770 100644
--- a/clang/test/OpenMP/target_teams_codegen_registration.cpp
+++ b/clang/test/OpenMP/target_teams_codegen_registration.cpp
@@ -92,9 +92,9 @@
 // CHECK-DAG: [[SE:%.+]] = type { [64 x i32] }
 // CHECK-DAG: [[ST1:%.+]] = type { [228 x i32] }
 // CHECK-DAG: [[ST2:%.+]] = type { [1128 x i32] }
-// CHECK-DAG: [[ENTTY:%.+]] = type { ptr, ptr, i[[SZ:32|64]], i32, i32 }
+// CHECK-DAG: [[ENTTY:%.+]] = type { i64, i16, i16, i32, ptr, ptr, i64, i64, ptr }
 
-// TCHECK:    [[ENTTY:%.+]] = type { ptr, ptr, i[[SZ:32|64]], i32, i32 }
+// TCHECK: [[ENTTY:%.+]] = type { i64, i16, i16, i32, ptr, ptr, i64, i64, ptr }
 
 // CHECK-DAG: [[A1:@.+]] = internal global [[SA]]
 // CHECK-DAG: [[A2:@.+]] ={{.*}} global [[SA]]
@@ -160,54 +160,30 @@
 // CHECK-NTARGET-NOT: private unnamed_addr constant [1 x i
 
 // CHECK-DAG: [[NAMEPTR1:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME1:__omp_offloading_[0-9a-f]+_[0-9a-f]+__Z.+_l[0-9]+]]\00"
-// CHECK-DAG: [[ENTRY1:@.+]] = weak{{.*}} constant [[ENTTY]] { ptr @{{.*}}, ptr [[NAMEPTR1]], i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // CHECK-DAG: [[NAMEPTR2:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME2:.+]]\00"
-// CHECK-DAG: [[ENTRY2:@.+]] = weak{{.*}} constant [[ENTTY]] { ptr @{{.*}}, ptr [[NAMEPTR2]], i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // CHECK-DAG: [[NAMEPTR3:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME3:.+]]\00"
-// CHECK-DAG: [[ENTRY3:@.+]] = weak{{.*}} constant [[ENTTY]] { ptr @{{.*}}, ptr [[NAMEPTR3]], i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // CHECK-DAG: [[NAMEPTR4:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME4:.+]]\00"
-// CHECK-DAG: [[ENTRY4:@.+]] = weak{{.*}} constant [[ENTTY]] { ptr @{{.*}}, ptr [[NAMEPTR4]], i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // CHECK-DAG: [[NAMEPTR5:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME5:.+]]\00"
-// CHECK-DAG: [[ENTRY5:@.+]] = weak{{.*}} constant [[ENTTY]] { ptr @{{.*}}, ptr [[NAMEPTR5]], i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // CHECK-DAG: [[NAMEPTR6:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME6:.+]]\00"
-// CHECK-DAG: [[ENTRY6:@.+]] = weak{{.*}} constant [[ENTTY]] { ptr @{{.*}}, ptr [[NAMEPTR6]], i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // CHECK-DAG: [[NAMEPTR7:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME7:.+]]\00"
-// CHECK-DAG: [[ENTRY7:@.+]] = weak{{.*}} constant [[ENTTY]] { ptr @{{.*}}, ptr [[NAMEPTR7]], i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // CHECK-DAG: [[NAMEPTR8:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME8:.+]]\00"
-// CHECK-DAG: [[ENTRY8:@.+]] = weak{{.*}} constant [[ENTTY]] { ptr @{{.*}}, ptr [[NAMEPTR8]], i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // CHECK-DAG: [[NAMEPTR9:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME9:.+]]\00"
-// CHECK-DAG: [[ENTRY9:@.+]] = weak{{.*}} constant [[ENTTY]] { ptr @{{.*}}, ptr [[NAMEPTR9]], i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // CHECK-DAG: [[NAMEPTR10:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME10:.+]]\00"
-// CHECK-DAG: [[ENTRY10:@.+]] = weak{{.*}} constant [[ENTTY]] { ptr @{{.*}}, ptr [[NAMEPTR10]], i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // CHECK-DAG: [[NAMEPTR11:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME11:.+]]\00"
-// CHECK-DAG: [[ENTRY11:@.+]] = weak{{.*}} constant [[ENTTY]] { ptr @{{.*}}, ptr [[NAMEPTR11]], i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // CHECK-DAG: [[NAMEPTR12:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME12:.+]]\00"
-// CHECK-DAG: [[ENTRY12:@.+]] = weak{{.*}} constant [[ENTTY]] { ptr @{{.*}}, ptr [[NAMEPTR12]], i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 
 // TCHECK-DAG: [[NAMEPTR1:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME1:__omp_offloading_[0-9a-f]+_[0-9a-f]+__Z.+_l[0-9]+]]\00"
-// TCHECK-DAG: [[ENTRY1:@.+]] = weak{{.*}} constant [[ENTTY]] { ptr @{{.*}}, ptr [[NAMEPTR1]], i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // TCHECK-DAG: [[NAMEPTR2:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME2:.+]]\00"
-// TCHECK-DAG: [[ENTRY2:@.+]] = weak{{.*}} constant [[ENTTY]] { ptr @{{.*}}, ptr [[NAMEPTR2]], i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // TCHECK-DAG: [[NAMEPTR3:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME3:.+]]\00"
-// TCHECK-DAG: [[ENTRY3:@.+]] = weak{{.*}} constant [[ENTTY]] { ptr @{{.*}}, ptr [[NAMEPTR3]], i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // TCHECK-DAG: [[NAMEPTR4:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME4:.+]]\00"
-// TCHECK-DAG: [[ENTRY4:@.+]] = weak{{.*}} constant [[ENTTY]] { ptr @{{.*}}, ptr [[NAMEPTR4]], i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // TCHECK-DAG: [[NAMEPTR5:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME5:.+]]\00"
-// TCHECK-DAG: [[ENTRY5:@.+]] = weak{{.*}} constant [[ENTTY]] { ptr @{{.*}}, ptr [[NAMEPTR5]], i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // TCHECK-DAG: [[NAMEPTR6:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME6:.+]]\00"
-// TCHECK-DAG: [[ENTRY6:@.+]] = weak{{.*}} constant [[ENTTY]] { ptr @{{.*}}, ptr [[NAMEPTR6]], i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // TCHECK-DAG: [[NAMEPTR7:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME7:.+]]\00"
-// TCHECK-DAG: [[ENTRY7:@.+]] = weak{{.*}} constant [[ENTTY]] { ptr @{{.*}}, ptr [[NAMEPTR7]], i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // TCHECK-DAG: [[NAMEPTR8:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME8:.+]]\00"
-// TCHECK-DAG: [[ENTRY8:@.+]] = weak{{.*}} constant [[ENTTY]] { ptr @{{.*}}, ptr [[NAMEPTR8]], i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // TCHECK-DAG: [[NAMEPTR9:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME9:.+]]\00"
-// TCHECK-DAG: [[ENTRY9:@.+]] = weak{{.*}} constant [[ENTTY]] { ptr @{{.*}}, ptr [[NAMEPTR9]], i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // TCHECK-DAG: [[NAMEPTR10:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME10:.+]]\00"
-// TCHECK-DAG: [[ENTRY10:@.+]] = weak{{.*}} constant [[ENTTY]] { ptr @{{.*}}, ptr [[NAMEPTR10]], i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // TCHECK-DAG: [[NAMEPTR11:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME11:.+]]\00"
-// TCHECK-DAG: [[ENTRY11:@.+]] = weak{{.*}} constant [[ENTTY]] { ptr @{{.*}}, ptr [[NAMEPTR11]], i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // TCHECK-DAG: [[NAMEPTR12:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME12:.+]]\00"
-// TCHECK-DAG: [[ENTRY12:@.+]] = weak{{.*}} constant [[ENTTY]] { ptr @{{.*}}, ptr [[NAMEPTR12]], i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 
 // We have 4 initializers, one for the 500 priority, another one for 501, or more for the default priority, and the last one for the offloading registration function.
 // CHECK: @llvm.global_ctors = appending global [3 x { i32, ptr, ptr }] [
@@ -442,31 +418,31 @@ int bar(int a){
 
 // Check metadata is properly generated:
 // CHECK:     !omp_offload.info = !{!{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}}
-// CHECK-DAG: = !{i32 0, i32 [[DEVID:-?[0-9]+]], i32 [[FILEID:-?[0-9]+]], !"_ZN2SB3fooEv", i32 245, i32 0, i32 {{[0-9]+}}}
-// CHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2SDD1Ev", i32 295, i32 0, i32 {{[0-9]+}}}
-// CHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2SEC1Ev", i32 311, i32 0, i32 {{[0-9]+}}}
-// CHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2SED1Ev", i32 317, i32 0, i32 {{[0-9]+}}}
-// CHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2STILi1000EE3fooEv", i32 328, i32 0, i32 {{[0-9]+}}}
-// CHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2STILi100EEC1Ev", i32 334, i32 0, i32 {{[0-9]+}}}
-// CHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_Z3bari", i32 437, i32 0, i32 {{[0-9]+}}}
-// CHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2STILi100EED1Ev", i32 340, i32 0, i32 {{[0-9]+}}}
-// CHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2STILi1000EEC1Ev", i32 334, i32 0, i32 {{[0-9]+}}}
-// CHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2STILi1000EED1Ev", i32 340, i32 0, i32 {{[0-9]+}}}
-// CHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2STILi100EE3fooEv", i32 328, i32 0, i32 {{[0-9]+}}}
-// CHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2SCC1Ev", i32 270, i32 0, i32 {{[0-9]+}}}
+// CHECK-DAG: = !{i32 0, i32 [[DEVID:-?[0-9]+]], i32 [[FILEID:-?[0-9]+]], !"_ZN2SB3fooEv", i32 {{[0-9]+}}, i32 0, i32 {{[0-9]+}}}
+// CHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2SDD1Ev", i32 {{[0-9]+}}, i32 0, i32 {{[0-9]+}}}
+// CHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2SEC1Ev", i32 {{[0-9]+}}, i32 0, i32 {{[0-9]+}}}
+// CHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2SED1Ev", i32 {{[0-9]+}}, i32 0, i32 {{[0-9]+}}}
+// CHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2STILi1000EE3fooEv", i32 {{[0-9]+}}, i32 0, i32 {{[0-9]+}}}
+// CHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2STILi100EEC1Ev", i32 {{[0-9]+}}, i32 0, i32 {{[0-9]+}}}
+// CHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_Z3bari", i32 {{[0-9]+}}, i32 0, i32 {{[0-9]+}}}
+// CHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2STILi100EED1Ev", i32 {{[0-9]+}}, i32 0, i32 {{[0-9]+}}}
+// CHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2STILi1000EEC1Ev", i32 {{[0-9]+}}, i32 0, i32 {{[0-9]+}}}
+// CHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2STILi1000EED1Ev", i32 {{[0-9]+}}, i32 0, i32 {{[0-9]+}}}
+// CHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2STILi100EE3fooEv", i32 {{[0-9]+}}, i32 0, i32 {{[0-9]+}}}
+// CHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2SCC1Ev", i32 {{[0-9]+}}, i32 0, i32 {{[0-9]+}}}
 
 // TCHECK:     !omp_offload.info = !{!{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}}
-// TCHECK-DAG: = !{i32 0, i32 [[DEVID:-?[0-9]+]], i32 [[FILEID:-?[0-9]+]], !"_ZN2SB3fooEv", i32 245, i32 0, i32 {{[0-9]+}}}
-// TCHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2SDD1Ev", i32 295, i32 0, i32 {{[0-9]+}}}
-// TCHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2SEC1Ev", i32 311, i32 0, i32 {{[0-9]+}}}
-// TCHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2SED1Ev", i32 317, i32 0, i32 {{[0-9]+}}}
-// TCHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2STILi1000EE3fooEv", i32 328, i32 0, i32 {{[0-9]+}}}
-// TCHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2STILi100EEC1Ev", i32 334, i32 0, i32 {{[0-9]+}}}
-// TCHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_Z3bari", i32 437, i32 0, i32 {{[0-9]+}}}
-// TCHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2STILi100EED1Ev", i32 340, i32 0, i32 {{[0-9]+}}}
-// TCHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2STILi1000EEC1Ev", i32 334, i32 0, i32 {{[0-9]+}}}
-// TCHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2STILi1000EED1Ev", i32 340, i32 0, i32 {{[0-9]+}}}
-// TCHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2STILi100EE3fooEv", i32 328, i32 0, i32 {{[0-9]+}}}
-// TCHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2SCC1Ev", i32 270, i32 0, i32 {{[0-9]+}}}
+// TCHECK-DAG: = !{i32 0, i32 [[DEVID:-?[0-9]+]], i32 [[FILEID:-?[0-9]+]], !"_ZN2SB3fooEv", i32 {{[0-9]+}}, i32 0, i32 {{[0-9]+}}}
+// TCHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2SDD1Ev", i32 {{[0-9]+}}, i32 0, i32 {{[0-9]+}}}
+// TCHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2SEC1Ev", i32 {{[0-9]+}}, i32 0, i32 {{[0-9]+}}}
+// TCHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2SED1Ev", i32 {{[0-9]+}}, i32 0, i32 {{[0-9]+}}}
+// TCHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2STILi1000EE3fooEv", i32 {{[0-9]+}}, i32 0, i32 {{[0-9]+}}}
+// TCHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2STILi100EEC1Ev", i32 {{[0-9]+}}, i32 0, i32 {{[0-9]+}}}
+// TCHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_Z3bari", i32 {{[0-9]+}}, i32 0, i32 {{[0-9]+}}}
+// TCHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2STILi100EED1Ev", i32 {{[0-9]+}}, i32 0, i32 {{[0-9]+}}}
+// TCHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2STILi1000EEC1Ev", i32 {{[0-9]+}}, i32 0, i32 {{[0-9]+}}}
+// TCHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2STILi1000EED1Ev", i32 {{[0-9]+}}, i32 0, i32 {{[0-9]+}}}
+// TCHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2STILi100EE3fooEv", i32 {{[0-9]+}}, i32 0, i32 {{[0-9]+}}}
+// TCHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2SCC1Ev", i32 {{[0-9]+}}, i32 0, i32 {{[0-9]+}}}
 
 #endif
diff --git a/clang/test/OpenMP/target_teams_depend_codegen.cpp b/clang/test/OpenMP/target_teams_depend_codegen.cpp
index 960c5980cbe68..b2280b80995fb 100644
--- a/clang/test/OpenMP/target_teams_depend_codegen.cpp
+++ b/clang/test/OpenMP/target_teams_depend_codegen.cpp
@@ -39,9 +39,9 @@
 #define HEADER
 
 // CHECK-DAG: [[TT:%.+]] = type { i64, i8 }
-// CHECK-DAG: [[ENTTY:%.+]] = type { ptr, ptr, i[[SZ:32|64]], i32, i32 }
+// CHECK-DAG: [[ENTTY:%.+]] = type { i64, i16, i16, i32, ptr, ptr, i64, i64, ptr }
 
-// TCHECK: [[ENTTY:%.+]] = type { ptr, ptr, i{{32|64}}, i32, i32 }
+// TCHECK: [[ENTTY:%.+]] = type { i64, i16, i16, i32, ptr, ptr, i64, i64, ptr }
 
 // CHECK-DAG: [[SIZET:@.+]] = private unnamed_addr constant [2 x i64] [i64 0, i64 4]
 // CHECK-DAG: [[MAPT:@.+]] = private unnamed_addr constant [2 x i64] [i64 544, i64 800]
@@ -76,7 +76,7 @@ int foo(int n) {
 // CHECK:       [[GEP:%.+]] = getelementptr inbounds nuw %{{.+}}, ptr %{{.+}}, i32 0, i32 0
 // CHECK:       [[DEV:%.+]] = load i32, ptr [[DEVICE_CAP]],
 // CHECK:       store i32 [[DEV]], ptr [[GEP]],
-// CHECK:       [[TASK:%.+]] = call ptr @__kmpc_omp_task_alloc(ptr @1, i32 [[GTID:%.+]], i32 1, i[[SZ]] {{20|40}}, i[[SZ]] 4, ptr [[TASK_ENTRY0:@.+]])
+// CHECK:       [[TASK:%.+]] = call ptr @__kmpc_omp_task_alloc(ptr @1, i32 [[GTID:%.+]], i32 1, i[[SZ:32|64]] {{20|40}}, i[[SZ]] 4, ptr [[TASK_ENTRY0:@.+]])
 // CHECK:       getelementptr %struct.kmp_depend_info, ptr %{{.+}}, i[[SZ]] 0
 // CHECK:       getelementptr %struct.kmp_depend_info, ptr %{{.+}}, i[[SZ]] 1
 // CHECK:       getelementptr %struct.kmp_depend_info, ptr %{{.+}}, i[[SZ]] 2
diff --git a/clang/test/OpenMP/target_teams_distribute_codegen_registration.cpp b/clang/test/OpenMP/target_teams_distribute_codegen_registration.cpp
index d619927d5bb80..21099be1dfc86 100644
--- a/clang/test/OpenMP/target_teams_distribute_codegen_registration.cpp
+++ b/clang/test/OpenMP/target_teams_distribute_codegen_registration.cpp
@@ -92,9 +92,9 @@
 // CHECK-DAG: [[SE:%.+]] = type { [64 x i32] }
 // CHECK-DAG: [[ST1:%.+]] = type { [228 x i32] }
 // CHECK-DAG: [[ST2:%.+]] = type { [1128 x i32] }
-// CHECK-DAG: [[ENTTY:%.+]] = type { ptr, ptr, i[[SZ:32|64]], i32, i32 }
+// CHECK-DAG: [[ENTTY:%.+]] = type { i64, i16, i16, i32, ptr, ptr, i64, i64, ptr }
 
-// TCHECK:    [[ENTTY:%.+]] = type { ptr, ptr, i[[SZ:32|64]], i32, i32 }
+// TCHECK: [[ENTTY:%.+]] = type { i64, i16, i16, i32, ptr, ptr, i64, i64, ptr }
 
 // CHECK-DAG: [[A1:@.+]] = internal global [[SA]]
 // CHECK-DAG: [[A2:@.+]] ={{.*}} global [[SA]]
@@ -160,54 +160,30 @@
 // CHECK-NTARGET-NOT: private unnamed_addr constant [1 x i
 
 // CHECK-DAG: [[NAMEPTR1:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME1:__omp_offloading_[0-9a-f]+_[0-9a-f]+__Z.+_l[0-9]+]]\00"
-// CHECK-DAG: [[ENTRY1:@.+]] = weak{{.*}} constant [[ENTTY]] { ptr @{{.*}}, ptr [[NAMEPTR1]], i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // CHECK-DAG: [[NAMEPTR2:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME2:.+]]\00"
-// CHECK-DAG: [[ENTRY2:@.+]] = weak{{.*}} constant [[ENTTY]] { ptr @{{.*}}, ptr [[NAMEPTR2]], i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // CHECK-DAG: [[NAMEPTR3:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME3:.+]]\00"
-// CHECK-DAG: [[ENTRY3:@.+]] = weak{{.*}} constant [[ENTTY]] { ptr @{{.*}}, ptr [[NAMEPTR3]], i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // CHECK-DAG: [[NAMEPTR4:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME4:.+]]\00"
-// CHECK-DAG: [[ENTRY4:@.+]] = weak{{.*}} constant [[ENTTY]] { ptr @{{.*}}, ptr [[NAMEPTR4]], i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // CHECK-DAG: [[NAMEPTR5:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME5:.+]]\00"
-// CHECK-DAG: [[ENTRY5:@.+]] = weak{{.*}} constant [[ENTTY]] { ptr @{{.*}}, ptr [[NAMEPTR5]], i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // CHECK-DAG: [[NAMEPTR6:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME6:.+]]\00"
-// CHECK-DAG: [[ENTRY6:@.+]] = weak{{.*}} constant [[ENTTY]] { ptr @{{.*}}, ptr [[NAMEPTR6]], i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // CHECK-DAG: [[NAMEPTR7:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME7:.+]]\00"
-// CHECK-DAG: [[ENTRY7:@.+]] = weak{{.*}} constant [[ENTTY]] { ptr @{{.*}}, ptr [[NAMEPTR7]], i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // CHECK-DAG: [[NAMEPTR8:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME8:.+]]\00"
-// CHECK-DAG: [[ENTRY8:@.+]] = weak{{.*}} constant [[ENTTY]] { ptr @{{.*}}, ptr [[NAMEPTR8]], i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // CHECK-DAG: [[NAMEPTR9:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME9:.+]]\00"
-// CHECK-DAG: [[ENTRY9:@.+]] = weak{{.*}} constant [[ENTTY]] { ptr @{{.*}}, ptr [[NAMEPTR9]], i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // CHECK-DAG: [[NAMEPTR10:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME10:.+]]\00"
-// CHECK-DAG: [[ENTRY10:@.+]] = weak{{.*}} constant [[ENTTY]] { ptr @{{.*}}, ptr [[NAMEPTR10]], i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // CHECK-DAG: [[NAMEPTR11:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME11:.+]]\00"
-// CHECK-DAG: [[ENTRY11:@.+]] = weak{{.*}} constant [[ENTTY]] { ptr @{{.*}}, ptr [[NAMEPTR11]], i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // CHECK-DAG: [[NAMEPTR12:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME12:.+]]\00"
-// CHECK-DAG: [[ENTRY12:@.+]] = weak{{.*}} constant [[ENTTY]] { ptr @{{.*}}, ptr [[NAMEPTR12]], i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 
 // TCHECK-DAG: [[NAMEPTR1:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME1:__omp_offloading_[0-9a-f]+_[0-9a-f]+__Z.+_l[0-9]+]]\00"
-// TCHECK-DAG: [[ENTRY1:@.+]] = weak{{.*}} constant [[ENTTY]] { ptr @{{.*}}, ptr [[NAMEPTR1]], i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // TCHECK-DAG: [[NAMEPTR2:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME2:.+]]\00"
-// TCHECK-DAG: [[ENTRY2:@.+]] = weak{{.*}} constant [[ENTTY]] { ptr @{{.*}}, ptr [[NAMEPTR2]], i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // TCHECK-DAG: [[NAMEPTR3:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME3:.+]]\00"
-// TCHECK-DAG: [[ENTRY3:@.+]] = weak{{.*}} constant [[ENTTY]] { ptr @{{.*}}, ptr [[NAMEPTR3]], i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // TCHECK-DAG: [[NAMEPTR4:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME4:.+]]\00"
-// TCHECK-DAG: [[ENTRY4:@.+]] = weak{{.*}} constant [[ENTTY]] { ptr @{{.*}}, ptr [[NAMEPTR4]], i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // TCHECK-DAG: [[NAMEPTR5:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME5:.+]]\00"
-// TCHECK-DAG: [[ENTRY5:@.+]] = weak{{.*}} constant [[ENTTY]] { ptr @{{.*}}, ptr [[NAMEPTR5]], i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // TCHECK-DAG: [[NAMEPTR6:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME6:.+]]\00"
-// TCHECK-DAG: [[ENTRY6:@.+]] = weak{{.*}} constant [[ENTTY]] { ptr @{{.*}}, ptr [[NAMEPTR6]], i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // TCHECK-DAG: [[NAMEPTR7:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME7:.+]]\00"
-// TCHECK-DAG: [[ENTRY7:@.+]] = weak{{.*}} constant [[ENTTY]] { ptr @{{.*}}, ptr [[NAMEPTR7]], i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // TCHECK-DAG: [[NAMEPTR8:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME8:.+]]\00"
-// TCHECK-DAG: [[ENTRY8:@.+]] = weak{{.*}} constant [[ENTTY]] { ptr @{{.*}}, ptr [[NAMEPTR8]], i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // TCHECK-DAG: [[NAMEPTR9:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME9:.+]]\00"
-// TCHECK-DAG: [[ENTRY9:@.+]] = weak{{.*}} constant [[ENTTY]] { ptr @{{.*}}, ptr [[NAMEPTR9]], i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // TCHECK-DAG: [[NAMEPTR10:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME10:.+]]\00"
-// TCHECK-DAG: [[ENTRY10:@.+]] = weak{{.*}} constant [[ENTTY]] { ptr @{{.*}}, ptr [[NAMEPTR10]], i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // TCHECK-DAG: [[NAMEPTR11:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME11:.+]]\00"
-// TCHECK-DAG: [[ENTRY11:@.+]] = weak{{.*}} constant [[ENTTY]] { ptr @{{.*}}, ptr [[NAMEPTR11]], i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // TCHECK-DAG: [[NAMEPTR12:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME12:.+]]\00"
-// TCHECK-DAG: [[ENTRY12:@.+]] = weak{{.*}} constant [[ENTTY]] { ptr @{{.*}}, ptr [[NAMEPTR12]], i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 
 // We have 4 initializers, one for the 500 priority, another one for 501, or more for the default priority, and the last one for the offloading registration function.
 // CHECK: @llvm.global_ctors = appending global [3 x { i32, ptr, ptr }] [
@@ -452,31 +428,31 @@ int bar(int a){
 
 // Check metadata is properly generated:
 // CHECK:     !omp_offload.info = !{!{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}}
-// CHECK-DAG: = !{i32 0, i32 [[DEVID:-?[0-9]+]], i32 [[FILEID:-?[0-9]+]], !"_ZN2SB3fooEv", i32 245, i32 0, i32 {{[0-9]+}}}
-// CHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2SDD1Ev", i32 297, i32 0, i32 {{[0-9]+}}}
-// CHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2SEC1Ev", i32 315, i32 0, i32 {{[0-9]+}}}
-// CHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2SED1Ev", i32 322, i32 0, i32 {{[0-9]+}}}
-// CHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2STILi1000EE3fooEv", i32 334, i32 0, i32 {{[0-9]+}}}
-// CHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2STILi100EEC1Ev", i32 341, i32 0, i32 {{[0-9]+}}}
-// CHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_Z3bari", i32 446, i32 0, i32 {{[0-9]+}}}
-// CHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2STILi100EED1Ev", i32 348, i32 0, i32 {{[0-9]+}}}
-// CHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2STILi1000EEC1Ev", i32 341, i32 0, i32 {{[0-9]+}}}
-// CHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2STILi1000EED1Ev", i32 348, i32 0, i32 {{[0-9]+}}}
-// CHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2STILi100EE3fooEv", i32 334, i32 0, i32 {{[0-9]+}}}
-// CHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2SCC1Ev", i32 271, i32 0, i32 {{[0-9]+}}}
+// CHECK-DAG: = !{i32 0, i32 [[DEVID:-?[0-9]+]], i32 [[FILEID:-?[0-9]+]], !"_ZN2SB3fooEv", i32 {{[0-9]+}}, i32 0, i32 {{[0-9]+}}}
+// CHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2SDD1Ev", i32 {{[0-9]+}}, i32 0, i32 {{[0-9]+}}}
+// CHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2SEC1Ev", i32 {{[0-9]+}}, i32 0, i32 {{[0-9]+}}}
+// CHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2SED1Ev", i32 {{[0-9]+}}, i32 0, i32 {{[0-9]+}}}
+// CHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2STILi1000EE3fooEv", i32 {{[0-9]+}}, i32 0, i32 {{[0-9]+}}}
+// CHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2STILi100EEC1Ev", i32 {{[0-9]+}}, i32 0, i32 {{[0-9]+}}}
+// CHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_Z3bari", i32 {{[0-9]+}}, i32 0, i32 {{[0-9]+}}}
+// CHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2STILi100EED1Ev", i32 {{[0-9]+}}, i32 0, i32 {{[0-9]+}}}
+// CHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2STILi1000EEC1Ev", i32 {{[0-9]+}}, i32 0, i32 {{[0-9]+}}}
+// CHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2STILi1000EED1Ev", i32 {{[0-9]+}}, i32 0, i32 {{[0-9]+}}}
+// CHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2STILi100EE3fooEv", i32 {{[0-9]+}}, i32 0, i32 {{[0-9]+}}}
+// CHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2SCC1Ev", i32 {{[0-9]+}}, i32 0, i32 {{[0-9]+}}}
 
 // TCHECK:     !omp_offload.info = !{!{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}}
-// TCHECK-DAG: = !{i32 0, i32 [[DEVID:-?[0-9]+]], i32 [[FILEID:-?[0-9]+]], !"_ZN2SB3fooEv", i32 245, i32 0, i32 {{[0-9]+}}}
-// TCHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2SDD1Ev", i32 297, i32 0, i32 {{[0-9]+}}}
-// TCHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2SEC1Ev", i32 315, i32 0, i32 {{[0-9]+}}}
-// TCHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2SED1Ev", i32 322, i32 0, i32 {{[0-9]+}}}
-// TCHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2STILi1000EE3fooEv", i32 334, i32 0, i32 {{[0-9]+}}}
-// TCHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2STILi100EEC1Ev", i32 341, i32 0, i32 {{[0-9]+}}}
-// TCHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_Z3bari", i32 446, i32 0, i32 {{[0-9]+}}}
-// TCHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2STILi100EED1Ev", i32 348, i32 0, i32 {{[0-9]+}}}
-// TCHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2STILi1000EEC1Ev", i32 341, i32 0, i32 {{[0-9]+}}}
-// TCHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2STILi1000EED1Ev", i32 348, i32 0, i32 {{[0-9]+}}}
-// TCHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2STILi100EE3fooEv", i32 334, i32 0, i32 {{[0-9]+}}}
-// TCHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2SCC1Ev", i32 271, i32 0, i32 {{[0-9]+}}}
+// TCHECK-DAG: = !{i32 0, i32 [[DEVID:-?[0-9]+]], i32 [[FILEID:-?[0-9]+]], !"_ZN2SB3fooEv", i32 {{[0-9]+}}, i32 0, i32 {{[0-9]+}}}
+// TCHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2SDD1Ev", i32 {{[0-9]+}}, i32 0, i32 {{[0-9]+}}}
+// TCHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2SEC1Ev", i32 {{[0-9]+}}, i32 0, i32 {{[0-9]+}}}
+// TCHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2SED1Ev", i32 {{[0-9]+}}, i32 0, i32 {{[0-9]+}}}
+// TCHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2STILi1000EE3fooEv", i32 {{[0-9]+}}, i32 0, i32 {{[0-9]+}}}
+// TCHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2STILi100EEC1Ev", i32 {{[0-9]+}}, i32 0, i32 {{[0-9]+}}}
+// TCHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_Z3bari", i32 {{[0-9]+}}, i32 0, i32 {{[0-9]+}}}
+// TCHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2STILi100EED1Ev", i32 {{[0-9]+}}, i32 0, i32 {{[0-9]+}}}
+// TCHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2STILi1000EEC1Ev", i32 {{[0-9]+}}, i32 0, i32 {{[0-9]+}}}
+// TCHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2STILi1000EED1Ev", i32 {{[0-9]+}}, i32 0, i32 {{[0-9]+}}}
+// TCHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2STILi100EE3fooEv", i32 {{[0-9]+}}, i32 0, i32 {{[0-9]+}}}
+// TCHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2SCC1Ev", i32 {{[0-9]+}}, i32 0, i32 {{[0-9]+}}}
 
 #endif
diff --git a/clang/test/OpenMP/target_teams_distribute_depend_codegen.cpp b/clang/test/OpenMP/target_teams_distribute_depend_codegen.cpp
index c0db03d29711f..f9306d3091ee8 100644
--- a/clang/test/OpenMP/target_teams_distribute_depend_codegen.cpp
+++ b/clang/test/OpenMP/target_teams_distribute_depend_codegen.cpp
@@ -39,9 +39,9 @@
 #define HEADER
 
 // CHECK-DAG: [[TT:%.+]] = type { i64, i8 }
-// CHECK-DAG: [[ENTTY:%.+]] = type { ptr, ptr, i[[SZ:32|64]], i32, i32 }
+// CHECK-DAG: [[ENTTY:%.+]] = type { i64, i16, i16, i32, ptr, ptr, i64, i64, ptr }
 
-// TCHECK: [[ENTTY:%.+]] = type { ptr, ptr, i{{32|64}}, i32, i32 }
+// TCHECK: [[ENTTY:%.+]] = type { i64, i16, i16, i32, ptr, ptr, i64, i64, ptr }
 
 // CHECK-DAG: [[SIZET:@.+]] = private unnamed_addr constant [2 x i64] [i64 0, i64 4]
 // CHECK-DAG: [[MAPT:@.+]] = private unnamed_addr constant [2 x i64] [i64 544, i64 800]
@@ -76,7 +76,7 @@ int foo(int n) {
   // CHECK:       [[GEP:%.+]] = getelementptr inbounds nuw %{{.+}}, ptr %{{.+}}, i32 0, i32 0
   // CHECK:       [[DEV:%.+]] = load i32, ptr [[DEVICE_CAP]],
   // CHECK:       store i32 [[DEV]], ptr [[GEP]],
-  // CHECK:       [[TASK:%.+]] = call ptr @__kmpc_omp_task_alloc(ptr [[ID:@.+]], i32 [[GTID:%.+]], i32 1, i[[SZ]] {{20|40}}, i[[SZ]] 4, ptr [[TASK_ENTRY0:@.+]])
+  // CHECK:       [[TASK:%.+]] = call ptr @__kmpc_omp_task_alloc(ptr [[ID:@.+]], i32 [[GTID:%.+]], i32 1, i[[SZ:32|64]] {{20|40}}, i[[SZ]] 4, ptr [[TASK_ENTRY0:@.+]])
   // CHECK:       getelementptr %struct.kmp_depend_info, ptr %{{.+}}, i[[SZ]] 0
   // CHECK:       getelementptr %struct.kmp_depend_info, ptr %{{.+}}, i[[SZ]] 1
   // CHECK:       getelementptr %struct.kmp_depend_info, ptr %{{.+}}, i[[SZ]] 2
diff --git a/clang/test/OpenMP/target_teams_distribute_parallel_for_depend_codegen.cpp b/clang/test/OpenMP/target_teams_distribute_parallel_for_depend_codegen.cpp
index 57e19df273825..4d93515c738ed 100644
--- a/clang/test/OpenMP/target_teams_distribute_parallel_for_depend_codegen.cpp
+++ b/clang/test/OpenMP/target_teams_distribute_parallel_for_depend_codegen.cpp
@@ -39,9 +39,9 @@
 #define HEADER
 
 // CHECK-DAG: [[TT:%.+]] = type { i64, i8 }
-// CHECK-DAG: [[ENTTY:%.+]] = type { ptr, ptr, i[[SZ:32|64]], i32, i32 }
+// CHECK-DAG: [[ENTTY:%.+]] = type { i64, i16, i16, i32, ptr, ptr, i64, i64, ptr }
 
-// TCHECK: [[ENTTY:%.+]] = type { ptr, ptr, i{{32|64}}, i32, i32 }
+// TCHECK: [[ENTTY:%.+]] = type { i64, i16, i16, i32, ptr, ptr, i64, i64, ptr }
 
 // CHECK-DAG: [[SIZET:@.+]] = private unnamed_addr constant [2 x i64] [i64 0, i64 4]
 // CHECK-DAG: [[MAPT:@.+]] = private unnamed_addr constant [2 x i64] [i64 544, i64 800]
@@ -76,7 +76,7 @@ int foo(int n) {
   // CHECK:       [[GEP:%.+]] = getelementptr inbounds nuw %{{.+}}, ptr %{{.+}}, i32 0, i32 0
   // CHECK:       [[DEV:%.+]] = load i32, ptr [[DEVICE_CAP]],
   // CHECK:       store i32 [[DEV]], ptr [[GEP]],
-  // CHECK:       [[TASK:%.+]] = call ptr @__kmpc_omp_task_alloc(ptr [[ID:@.+]], i32 [[GTID:%.+]], i32 1, i[[SZ]] {{20|40}}, i[[SZ]] 4, ptr [[TASK_ENTRY0:@.+]])
+  // CHECK:       [[TASK:%.+]] = call ptr @__kmpc_omp_task_alloc(ptr [[ID:@.+]], i32 [[GTID:%.+]], i32 1, i[[SZ:32|64]] {{20|40}}, i[[SZ]] 4, ptr [[TASK_ENTRY0:@.+]])
   // CHECK:       getelementptr %struct.kmp_depend_info, ptr %{{.+}}, i[[SZ]] 0
   // CHECK:       getelementptr %struct.kmp_depend_info, ptr %{{.+}}, i[[SZ]] 1
   // CHECK:       getelementptr %struct.kmp_depend_info, ptr %{{.+}}, i[[SZ]] 2
diff --git a/clang/test/OpenMP/target_teams_distribute_parallel_for_simd_codegen_registration.cpp b/clang/test/OpenMP/target_teams_distribute_parallel_for_simd_codegen_registration.cpp
index f93ac400a2660..5a12a8de3d8fc 100644
--- a/clang/test/OpenMP/target_teams_distribute_parallel_for_simd_codegen_registration.cpp
+++ b/clang/test/OpenMP/target_teams_distribute_parallel_for_simd_codegen_registration.cpp
@@ -92,9 +92,9 @@
 // CHECK-DAG: [[SE:%.+]] = type { [64 x i32] }
 // CHECK-DAG: [[ST1:%.+]] = type { [228 x i32] }
 // CHECK-DAG: [[ST2:%.+]] = type { [1128 x i32] }
-// CHECK-DAG: [[ENTTY:%.+]] = type { ptr, ptr, i[[SZ:32|64]], i32, i32 }
+// CHECK-DAG: [[ENTTY:%.+]] = type { i64, i16, i16, i32, ptr, ptr, i64, i64, ptr }
 
-// TCHECK:    [[ENTTY:%.+]] = type { ptr, ptr, i[[SZ:32|64]], i32, i32 }
+// TCHECK: [[ENTTY:%.+]] = type { i64, i16, i16, i32, ptr, ptr, i64, i64, ptr }
 
 // CHECK-DAG: [[A1:@.+]] = internal global [[SA]]
 // CHECK-DAG: [[A2:@.+]] ={{.*}} global [[SA]]
@@ -160,54 +160,30 @@
 // CHECK-NTARGET-NOT: private unnamed_addr constant [1 x i
 
 // CHECK-DAG: [[NAMEPTR1:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME1:__omp_offloading_[0-9a-f]+_[0-9a-f]+__Z.+_l[0-9]+]]\00"
-// CHECK-DAG: [[ENTRY1:@.+]] = weak{{.*}} constant [[ENTTY]] { ptr @{{.*}}, ptr [[NAMEPTR1]], i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // CHECK-DAG: [[NAMEPTR2:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME2:.+]]\00"
-// CHECK-DAG: [[ENTRY2:@.+]] = weak{{.*}} constant [[ENTTY]] { ptr @{{.*}}, ptr [[NAMEPTR2]], i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // CHECK-DAG: [[NAMEPTR3:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME3:.+]]\00"
-// CHECK-DAG: [[ENTRY3:@.+]] = weak{{.*}} constant [[ENTTY]] { ptr @{{.*}}, ptr [[NAMEPTR3]], i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // CHECK-DAG: [[NAMEPTR4:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME4:.+]]\00"
-// CHECK-DAG: [[ENTRY4:@.+]] = weak{{.*}} constant [[ENTTY]] { ptr @{{.*}}, ptr [[NAMEPTR4]], i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // CHECK-DAG: [[NAMEPTR5:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME5:.+]]\00"
-// CHECK-DAG: [[ENTRY5:@.+]] = weak{{.*}} constant [[ENTTY]] { ptr @{{.*}}, ptr [[NAMEPTR5]], i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // CHECK-DAG: [[NAMEPTR6:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME6:.+]]\00"
-// CHECK-DAG: [[ENTRY6:@.+]] = weak{{.*}} constant [[ENTTY]] { ptr @{{.*}}, ptr [[NAMEPTR6]], i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // CHECK-DAG: [[NAMEPTR7:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME7:.+]]\00"
-// CHECK-DAG: [[ENTRY7:@.+]] = weak{{.*}} constant [[ENTTY]] { ptr @{{.*}}, ptr [[NAMEPTR7]], i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // CHECK-DAG: [[NAMEPTR8:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME8:.+]]\00"
-// CHECK-DAG: [[ENTRY8:@.+]] = weak{{.*}} constant [[ENTTY]] { ptr @{{.*}}, ptr [[NAMEPTR8]], i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // CHECK-DAG: [[NAMEPTR9:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME9:.+]]\00"
-// CHECK-DAG: [[ENTRY9:@.+]] = weak{{.*}} constant [[ENTTY]] { ptr @{{.*}}, ptr [[NAMEPTR9]], i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // CHECK-DAG: [[NAMEPTR10:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME10:.+]]\00"
-// CHECK-DAG: [[ENTRY10:@.+]] = weak{{.*}} constant [[ENTTY]] { ptr @{{.*}}, ptr [[NAMEPTR10]], i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // CHECK-DAG: [[NAMEPTR11:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME11:.+]]\00"
-// CHECK-DAG: [[ENTRY11:@.+]] = weak{{.*}} constant [[ENTTY]] { ptr @{{.*}}, ptr [[NAMEPTR11]], i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // CHECK-DAG: [[NAMEPTR12:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME12:.+]]\00"
-// CHECK-DAG: [[ENTRY12:@.+]] = weak{{.*}} constant [[ENTTY]] { ptr @{{.*}}, ptr [[NAMEPTR12]], i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 
 // TCHECK-DAG: [[NAMEPTR1:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME1:__omp_offloading_[0-9a-f]+_[0-9a-f]+__Z.+_l[0-9]+]]\00"
-// TCHECK-DAG: [[ENTRY1:@.+]] = weak{{.*}} constant [[ENTTY]] { ptr @{{.*}}, ptr [[NAMEPTR1]], i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // TCHECK-DAG: [[NAMEPTR2:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME2:.+]]\00"
-// TCHECK-DAG: [[ENTRY2:@.+]] = weak{{.*}} constant [[ENTTY]] { ptr @{{.*}}, ptr [[NAMEPTR2]], i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // TCHECK-DAG: [[NAMEPTR3:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME3:.+]]\00"
-// TCHECK-DAG: [[ENTRY3:@.+]] = weak{{.*}} constant [[ENTTY]] { ptr @{{.*}}, ptr [[NAMEPTR3]], i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // TCHECK-DAG: [[NAMEPTR4:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME4:.+]]\00"
-// TCHECK-DAG: [[ENTRY4:@.+]] = weak{{.*}} constant [[ENTTY]] { ptr @{{.*}}, ptr [[NAMEPTR4]], i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // TCHECK-DAG: [[NAMEPTR5:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME5:.+]]\00"
-// TCHECK-DAG: [[ENTRY5:@.+]] = weak{{.*}} constant [[ENTTY]] { ptr @{{.*}}, ptr [[NAMEPTR5]], i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // TCHECK-DAG: [[NAMEPTR6:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME6:.+]]\00"
-// TCHECK-DAG: [[ENTRY6:@.+]] = weak{{.*}} constant [[ENTTY]] { ptr @{{.*}}, ptr [[NAMEPTR6]], i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // TCHECK-DAG: [[NAMEPTR7:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME7:.+]]\00"
-// TCHECK-DAG: [[ENTRY7:@.+]] = weak{{.*}} constant [[ENTTY]] { ptr @{{.*}}, ptr [[NAMEPTR7]], i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // TCHECK-DAG: [[NAMEPTR8:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME8:.+]]\00"
-// TCHECK-DAG: [[ENTRY8:@.+]] = weak{{.*}} constant [[ENTTY]] { ptr @{{.*}}, ptr [[NAMEPTR8]], i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // TCHECK-DAG: [[NAMEPTR9:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME9:.+]]\00"
-// TCHECK-DAG: [[ENTRY9:@.+]] = weak{{.*}} constant [[ENTTY]] { ptr @{{.*}}, ptr [[NAMEPTR9]], i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // TCHECK-DAG: [[NAMEPTR10:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME10:.+]]\00"
-// TCHECK-DAG: [[ENTRY10:@.+]] = weak{{.*}} constant [[ENTTY]] { ptr @{{.*}}, ptr [[NAMEPTR10]], i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // TCHECK-DAG: [[NAMEPTR11:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME11:.+]]\00"
-// TCHECK-DAG: [[ENTRY11:@.+]] = weak{{.*}} constant [[ENTTY]] { ptr @{{.*}}, ptr [[NAMEPTR11]], i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // TCHECK-DAG: [[NAMEPTR12:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME12:.+]]\00"
-// TCHECK-DAG: [[ENTRY12:@.+]] = weak{{.*}} constant [[ENTTY]] { ptr @{{.*}}, ptr [[NAMEPTR12]], i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 
 // We have 4 initializers, one for the 500 priority, another one for 501, or more for the default priority, and the last one for the offloading registration function.
 // CHECK: @llvm.global_ctors = appending global [3 x { i32, ptr, ptr }] [
@@ -452,31 +428,31 @@ int bar(int a){
 
 // Check metadata is properly generated:
 // CHECK:     !omp_offload.info = !{!{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}}
-// CHECK-DAG: = !{i32 0, i32 [[DEVID:-?[0-9]+]], i32 [[FILEID:-?[0-9]+]], !"_ZN2SB3fooEv", i32 245, i32 0, i32 {{[0-9]+}}}
-// CHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2SDD1Ev", i32 297, i32 0, i32 {{[0-9]+}}}
-// CHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2SEC1Ev", i32 315, i32 0, i32 {{[0-9]+}}}
-// CHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2SED1Ev", i32 322, i32 0, i32 {{[0-9]+}}}
-// CHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2STILi1000EE3fooEv", i32 334, i32 0, i32 {{[0-9]+}}}
-// CHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2STILi100EEC1Ev", i32 341, i32 0, i32 {{[0-9]+}}}
-// CHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_Z3bari", i32 446, i32 0, i32 {{[0-9]+}}}
-// CHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2STILi100EED1Ev", i32 348, i32 0, i32 {{[0-9]+}}}
-// CHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2STILi1000EEC1Ev", i32 341, i32 0, i32 {{[0-9]+}}}
-// CHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2STILi1000EED1Ev", i32 348, i32 0, i32 {{[0-9]+}}}
-// CHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2STILi100EE3fooEv", i32 334, i32 0, i32 {{[0-9]+}}}
-// CHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2SCC1Ev", i32 271, i32 0, i32 {{[0-9]+}}}
+// CHECK-DAG: = !{i32 0, i32 [[DEVID:-?[0-9]+]], i32 [[FILEID:-?[0-9]+]], !"_ZN2SB3fooEv", i32 {{[0-9]+}}, i32 0, i32 {{[0-9]+}}}
+// CHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2SDD1Ev", i32 {{[0-9]+}}, i32 0, i32 {{[0-9]+}}}
+// CHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2SEC1Ev", i32 {{[0-9]+}}, i32 0, i32 {{[0-9]+}}}
+// CHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2SED1Ev", i32 {{[0-9]+}}, i32 0, i32 {{[0-9]+}}}
+// CHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2STILi1000EE3fooEv", i32 {{[0-9]+}}, i32 0, i32 {{[0-9]+}}}
+// CHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2STILi100EEC1Ev", i32 {{[0-9]+}}, i32 0, i32 {{[0-9]+}}}
+// CHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_Z3bari", i32 {{[0-9]+}}, i32 0, i32 {{[0-9]+}}}
+// CHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2STILi100EED1Ev", i32 {{[0-9]+}}, i32 0, i32 {{[0-9]+}}}
+// CHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2STILi1000EEC1Ev", i32 {{[0-9]+}}, i32 0, i32 {{[0-9]+}}}
+// CHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2STILi1000EED1Ev", i32 {{[0-9]+}}, i32 0, i32 {{[0-9]+}}}
+// CHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2STILi100EE3fooEv", i32 {{[0-9]+}}, i32 0, i32 {{[0-9]+}}}
+// CHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2SCC1Ev", i32 {{[0-9]+}}, i32 0, i32 {{[0-9]+}}}
 
 // TCHECK:     !omp_offload.info = !{!{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}}
-// TCHECK-DAG: = !{i32 0, i32 [[DEVID:-?[0-9]+]], i32 [[FILEID:-?[0-9]+]], !"_ZN2SB3fooEv", i32 245, i32 0, i32 {{[0-9]+}}}
-// TCHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2SDD1Ev", i32 297, i32 0, i32 {{[0-9]+}}}
-// TCHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2SEC1Ev", i32 315, i32 0, i32 {{[0-9]+}}}
-// TCHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2SED1Ev", i32 322, i32 0, i32 {{[0-9]+}}}
-// TCHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2STILi1000EE3fooEv", i32 334, i32 0, i32 {{[0-9]+}}}
-// TCHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2STILi100EEC1Ev", i32 341, i32 0, i32 {{[0-9]+}}}
-// TCHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_Z3bari", i32 446, i32 0, i32 {{[0-9]+}}}
-// TCHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2STILi100EED1Ev", i32 348, i32 0, i32 {{[0-9]+}}}
-// TCHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2STILi1000EEC1Ev", i32 341, i32 0, i32 {{[0-9]+}}}
-// TCHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2STILi1000EED1Ev", i32 348, i32 0, i32 {{[0-9]+}}}
-// TCHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2STILi100EE3fooEv", i32 334, i32 0, i32 {{[0-9]+}}}
-// TCHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2SCC1Ev", i32 271, i32 0, i32 {{[0-9]+}}}
+// TCHECK-DAG: = !{i32 0, i32 [[DEVID:-?[0-9]+]], i32 [[FILEID:-?[0-9]+]], !"_ZN2SB3fooEv", i32 {{[0-9]+}}, i32 0, i32 {{[0-9]+}}}
+// TCHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2SDD1Ev", i32 {{[0-9]+}}, i32 0, i32 {{[0-9]+}}}
+// TCHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2SEC1Ev", i32 {{[0-9]+}}, i32 0, i32 {{[0-9]+}}}
+// TCHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2SED1Ev", i32 {{[0-9]+}}, i32 0, i32 {{[0-9]+}}}
+// TCHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2STILi1000EE3fooEv", i32 {{[0-9]+}}, i32 0, i32 {{[0-9]+}}}
+// TCHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2STILi100EEC1Ev", i32 {{[0-9]+}}, i32 0, i32 {{[0-9]+}}}
+// TCHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_Z3bari", i32 {{[0-9]+}}, i32 0, i32 {{[0-9]+}}}
+// TCHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2STILi100EED1Ev", i32 {{[0-9]+}}, i32 0, i32 {{[0-9]+}}}
+// TCHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2STILi1000EEC1Ev", i32 {{[0-9]+}}, i32 0, i32 {{[0-9]+}}}
+// TCHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2STILi1000EED1Ev", i32 {{[0-9]+}}, i32 0, i32 {{[0-9]+}}}
+// TCHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2STILi100EE3fooEv", i32 {{[0-9]+}}, i32 0, i32 {{[0-9]+}}}
+// TCHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2SCC1Ev", i32 {{[0-9]+}}, i32 0, i32 {{[0-9]+}}}
 
 #endif
diff --git a/clang/test/OpenMP/target_teams_distribute_parallel_for_simd_depend_codegen.cpp b/clang/test/OpenMP/target_teams_distribute_parallel_for_simd_depend_codegen.cpp
index 15c116bad4b68..ed760d7e2e000 100644
--- a/clang/test/OpenMP/target_teams_distribute_parallel_for_simd_depend_codegen.cpp
+++ b/clang/test/OpenMP/target_teams_distribute_parallel_for_simd_depend_codegen.cpp
@@ -39,9 +39,9 @@
 #define HEADER
 
 // CHECK-DAG: [[TT:%.+]] = type { i64, i8 }
-// CHECK-DAG: [[ENTTY:%.+]] = type { ptr, ptr, i[[SZ:32|64]], i32, i32 }
+// CHECK-DAG: [[ENTTY:%.+]] = type { i64, i16, i16, i32, ptr, ptr, i64, i64, ptr }
 
-// TCHECK: [[ENTTY:%.+]] = type { ptr, ptr, i{{32|64}}, i32, i32 }
+// TCHECK: [[ENTTY:%.+]] = type { i64, i16, i16, i32, ptr, ptr, i64, i64, ptr }
 
 // CHECK-DAG: [[SIZET:@.+]] = private unnamed_addr constant [2 x i64] [i64 0, i64 4]
 // CHECK-DAG: [[MAPT:@.+]] = private unnamed_addr constant [2 x i64] [i64 544, i64 800]
@@ -76,7 +76,7 @@ int foo(int n) {
   // CHECK:       [[GEP:%.+]] = getelementptr inbounds nuw %{{.+}}, ptr %{{.+}}, i32 0, i32 0
   // CHECK:       [[DEV:%.+]] = load i32, ptr [[DEVICE_CAP]],
   // CHECK:       store i32 [[DEV]], ptr [[GEP]],
-  // CHECK:       [[TASK:%.+]] = call ptr @__kmpc_omp_task_alloc(ptr [[ID:@.+]], i32 [[GTID:%.+]], i32 1, i[[SZ]] {{20|40}}, i[[SZ]] 4, ptr [[TASK_ENTRY0:@.+]])
+  // CHECK:       [[TASK:%.+]] = call ptr @__kmpc_omp_task_alloc(ptr [[ID:@.+]], i32 [[GTID:%.+]], i32 1, i[[SZ:32|64]] {{20|40}}, i[[SZ]] 4, ptr [[TASK_ENTRY0:@.+]])
   // CHECK:       getelementptr %struct.kmp_depend_info, ptr %{{.+}}, i[[SZ]] 0
   // CHECK:       getelementptr %struct.kmp_depend_info, ptr %{{.+}}, i[[SZ]] 1
   // CHECK:       getelementptr %struct.kmp_depend_info, ptr %{{.+}}, i[[SZ]] 2
diff --git a/clang/test/OpenMP/target_teams_distribute_simd_codegen_registration.cpp b/clang/test/OpenMP/target_teams_distribute_simd_codegen_registration.cpp
index 9814448d5a6b0..18acfc4f3b690 100644
--- a/clang/test/OpenMP/target_teams_distribute_simd_codegen_registration.cpp
+++ b/clang/test/OpenMP/target_teams_distribute_simd_codegen_registration.cpp
@@ -92,9 +92,9 @@
 // CHECK-DAG: [[SE:%.+]] = type { [64 x i32] }
 // CHECK-DAG: [[ST1:%.+]] = type { [228 x i32] }
 // CHECK-DAG: [[ST2:%.+]] = type { [1128 x i32] }
-// CHECK-DAG: [[ENTTY:%.+]] = type { ptr, ptr, i[[SZ:32|64]], i32, i32 }
+// CHECK-DAG: [[ENTTY:%.+]] = type { i64, i16, i16, i32, ptr, ptr, i64, i64, ptr }
 
-// TCHECK:    [[ENTTY:%.+]] = type { ptr, ptr, i[[SZ:32|64]], i32, i32 }
+// TCHECK: [[ENTTY:%.+]] = type { i64, i16, i16, i32, ptr, ptr, i64, i64, ptr }
 
 // CHECK-DAG: [[A1:@.+]] = internal global [[SA]]
 // CHECK-DAG: [[A2:@.+]] ={{.*}} global [[SA]]
@@ -160,54 +160,30 @@
 // CHECK-NTARGET-NOT: private unnamed_addr constant [1 x i
 
 // CHECK-DAG: [[NAMEPTR1:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME1:__omp_offloading_[0-9a-f]+_[0-9a-f]+__Z.+_l[0-9]+]]\00"
-// CHECK-DAG: [[ENTRY1:@.+]] = weak{{.*}} constant [[ENTTY]] { ptr @{{.*}}, ptr [[NAMEPTR1]], i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // CHECK-DAG: [[NAMEPTR2:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME2:.+]]\00"
-// CHECK-DAG: [[ENTRY2:@.+]] = weak{{.*}} constant [[ENTTY]] { ptr @{{.*}}, ptr [[NAMEPTR2]], i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // CHECK-DAG: [[NAMEPTR3:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME3:.+]]\00"
-// CHECK-DAG: [[ENTRY3:@.+]] = weak{{.*}} constant [[ENTTY]] { ptr @{{.*}}, ptr [[NAMEPTR3]], i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // CHECK-DAG: [[NAMEPTR4:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME4:.+]]\00"
-// CHECK-DAG: [[ENTRY4:@.+]] = weak{{.*}} constant [[ENTTY]] { ptr @{{.*}}, ptr [[NAMEPTR4]], i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // CHECK-DAG: [[NAMEPTR5:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME5:.+]]\00"
-// CHECK-DAG: [[ENTRY5:@.+]] = weak{{.*}} constant [[ENTTY]] { ptr @{{.*}}, ptr [[NAMEPTR5]], i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // CHECK-DAG: [[NAMEPTR6:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME6:.+]]\00"
-// CHECK-DAG: [[ENTRY6:@.+]] = weak{{.*}} constant [[ENTTY]] { ptr @{{.*}}, ptr [[NAMEPTR6]], i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // CHECK-DAG: [[NAMEPTR7:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME7:.+]]\00"
-// CHECK-DAG: [[ENTRY7:@.+]] = weak{{.*}} constant [[ENTTY]] { ptr @{{.*}}, ptr [[NAMEPTR7]], i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // CHECK-DAG: [[NAMEPTR8:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME8:.+]]\00"
-// CHECK-DAG: [[ENTRY8:@.+]] = weak{{.*}} constant [[ENTTY]] { ptr @{{.*}}, ptr [[NAMEPTR8]], i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // CHECK-DAG: [[NAMEPTR9:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME9:.+]]\00"
-// CHECK-DAG: [[ENTRY9:@.+]] = weak{{.*}} constant [[ENTTY]] { ptr @{{.*}}, ptr [[NAMEPTR9]], i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // CHECK-DAG: [[NAMEPTR10:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME10:.+]]\00"
-// CHECK-DAG: [[ENTRY10:@.+]] = weak{{.*}} constant [[ENTTY]] { ptr @{{.*}}, ptr [[NAMEPTR10]], i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // CHECK-DAG: [[NAMEPTR11:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME11:.+]]\00"
-// CHECK-DAG: [[ENTRY11:@.+]] = weak{{.*}} constant [[ENTTY]] { ptr @{{.*}}, ptr [[NAMEPTR11]], i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // CHECK-DAG: [[NAMEPTR12:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME12:.+]]\00"
-// CHECK-DAG: [[ENTRY12:@.+]] = weak{{.*}} constant [[ENTTY]] { ptr @{{.*}}, ptr [[NAMEPTR12]], i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 
 // TCHECK-DAG: [[NAMEPTR1:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME1:__omp_offloading_[0-9a-f]+_[0-9a-f]+__Z.+_l[0-9]+]]\00"
-// TCHECK-DAG: [[ENTRY1:@.+]] = weak{{.*}} constant [[ENTTY]] { ptr @{{.*}}, ptr [[NAMEPTR1]], i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // TCHECK-DAG: [[NAMEPTR2:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME2:.+]]\00"
-// TCHECK-DAG: [[ENTRY2:@.+]] = weak{{.*}} constant [[ENTTY]] { ptr @{{.*}}, ptr [[NAMEPTR2]], i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // TCHECK-DAG: [[NAMEPTR3:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME3:.+]]\00"
-// TCHECK-DAG: [[ENTRY3:@.+]] = weak{{.*}} constant [[ENTTY]] { ptr @{{.*}}, ptr [[NAMEPTR3]], i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // TCHECK-DAG: [[NAMEPTR4:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME4:.+]]\00"
-// TCHECK-DAG: [[ENTRY4:@.+]] = weak{{.*}} constant [[ENTTY]] { ptr @{{.*}}, ptr [[NAMEPTR4]], i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // TCHECK-DAG: [[NAMEPTR5:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME5:.+]]\00"
-// TCHECK-DAG: [[ENTRY5:@.+]] = weak{{.*}} constant [[ENTTY]] { ptr @{{.*}}, ptr [[NAMEPTR5]], i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // TCHECK-DAG: [[NAMEPTR6:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME6:.+]]\00"
-// TCHECK-DAG: [[ENTRY6:@.+]] = weak{{.*}} constant [[ENTTY]] { ptr @{{.*}}, ptr [[NAMEPTR6]], i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // TCHECK-DAG: [[NAMEPTR7:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME7:.+]]\00"
-// TCHECK-DAG: [[ENTRY7:@.+]] = weak{{.*}} constant [[ENTTY]] { ptr @{{.*}}, ptr [[NAMEPTR7]], i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // TCHECK-DAG: [[NAMEPTR8:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME8:.+]]\00"
-// TCHECK-DAG: [[ENTRY8:@.+]] = weak{{.*}} constant [[ENTTY]] { ptr @{{.*}}, ptr [[NAMEPTR8]], i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // TCHECK-DAG: [[NAMEPTR9:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME9:.+]]\00"
-// TCHECK-DAG: [[ENTRY9:@.+]] = weak{{.*}} constant [[ENTTY]] { ptr @{{.*}}, ptr [[NAMEPTR9]], i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // TCHECK-DAG: [[NAMEPTR10:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME10:.+]]\00"
-// TCHECK-DAG: [[ENTRY10:@.+]] = weak{{.*}} constant [[ENTTY]] { ptr @{{.*}}, ptr [[NAMEPTR10]], i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // TCHECK-DAG: [[NAMEPTR11:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME11:.+]]\00"
-// TCHECK-DAG: [[ENTRY11:@.+]] = weak{{.*}} constant [[ENTTY]] { ptr @{{.*}}, ptr [[NAMEPTR11]], i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // TCHECK-DAG: [[NAMEPTR12:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME12:.+]]\00"
-// TCHECK-DAG: [[ENTRY12:@.+]] = weak{{.*}} constant [[ENTTY]] { ptr @{{.*}}, ptr [[NAMEPTR12]], i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 
 // We have 4 initializers, one for the 500 priority, another one for 501, or more for the default priority, and the last one for the offloading registration function.
 // CHECK: @llvm.global_ctors = appending global [3 x { i32, ptr, ptr }] [
@@ -452,32 +428,32 @@ int bar(int a){
 
 // Check metadata is properly generated:
 // CHECK:     !omp_offload.info = !{!{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}}
-// CHECK-DAG: = !{i32 0, i32 [[DEVID:-?[0-9]+]], i32 [[FILEID:-?[0-9]+]], !"_ZN2SB3fooEv", i32 245, i32 0, i32 {{[0-9]+}}}
-// CHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2SDD1Ev", i32 297, i32 0, i32 {{[0-9]+}}}
-// CHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2SEC1Ev", i32 315, i32 0, i32 {{[0-9]+}}}
-// CHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2SED1Ev", i32 322, i32 0, i32 {{[0-9]+}}}
-// CHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2STILi1000EE3fooEv", i32 334, i32 0, i32 {{[0-9]+}}}
-// CHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2STILi100EEC1Ev", i32 341, i32 0, i32 {{[0-9]+}}}
-// CHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_Z3bari", i32 446, i32 0, i32 {{[0-9]+}}}
-// CHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2STILi100EED1Ev", i32 348, i32 0, i32 {{[0-9]+}}}
-// CHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2STILi1000EEC1Ev", i32 341, i32 0, i32 {{[0-9]+}}}
-// CHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2STILi1000EED1Ev", i32 348, i32 0, i32 {{[0-9]+}}}
-// CHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2STILi100EE3fooEv", i32 334, i32 0, i32 {{[0-9]+}}}
-// CHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2SCC1Ev", i32 271, i32 0, i32 {{[0-9]+}}}
+// CHECK-DAG: = !{i32 0, i32 [[DEVID:-?[0-9]+]], i32 [[FILEID:-?[0-9]+]], !"_ZN2SB3fooEv", i32 {{[0-9]+}}, i32 0, i32 {{[0-9]+}}}
+// CHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2SDD1Ev", i32 {{[0-9]+}}, i32 0, i32 {{[0-9]+}}}
+// CHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2SEC1Ev", i32 {{[0-9]+}}, i32 0, i32 {{[0-9]+}}}
+// CHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2SED1Ev", i32 {{[0-9]+}}, i32 0, i32 {{[0-9]+}}}
+// CHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2STILi1000EE3fooEv", i32 {{[0-9]+}}, i32 0, i32 {{[0-9]+}}}
+// CHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2STILi100EEC1Ev", i32 {{[0-9]+}}, i32 0, i32 {{[0-9]+}}}
+// CHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_Z3bari", i32 {{[0-9]+}}, i32 0, i32 {{[0-9]+}}}
+// CHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2STILi100EED1Ev", i32 {{[0-9]+}}, i32 0, i32 {{[0-9]+}}}
+// CHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2STILi1000EEC1Ev", i32 {{[0-9]+}}, i32 0, i32 {{[0-9]+}}}
+// CHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2STILi1000EED1Ev", i32 {{[0-9]+}}, i32 0, i32 {{[0-9]+}}}
+// CHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2STILi100EE3fooEv", i32 {{[0-9]+}}, i32 0, i32 {{[0-9]+}}}
+// CHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2SCC1Ev", i32 {{[0-9]+}}, i32 0, i32 {{[0-9]+}}}
 
 // TCHECK:     !omp_offload.info = !{!{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}}
-// TCHECK-DAG: = !{i32 0, i32 [[DEVID:-?[0-9]+]], i32 [[FILEID:-?[0-9]+]], !"_ZN2SB3fooEv", i32 245, i32 0, i32 {{[0-9]+}}}
-// TCHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2SDD1Ev", i32 297, i32 0, i32 {{[0-9]+}}}
-// TCHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2SEC1Ev", i32 315, i32 0, i32 {{[0-9]+}}}
-// TCHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2SED1Ev", i32 322, i32 0, i32 {{[0-9]+}}}
-// TCHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2STILi1000EE3fooEv", i32 334, i32 0, i32 {{[0-9]+}}}
-// TCHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2STILi100EEC1Ev", i32 341, i32 0, i32 {{[0-9]+}}}
-// TCHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_Z3bari", i32 446, i32 0, i32 {{[0-9]+}}}
-// TCHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2STILi100EED1Ev", i32 348, i32 0, i32 {{[0-9]+}}}
-// TCHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2STILi1000EEC1Ev", i32 341, i32 0, i32 {{[0-9]+}}}
-// TCHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2STILi1000EED1Ev", i32 348, i32 0, i32 {{[0-9]+}}}
-// TCHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2STILi100EE3fooEv", i32 334, i32 0, i32 {{[0-9]+}}}
-// TCHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2SCC1Ev", i32 271, i32 0, i32 {{[0-9]+}}}
+// TCHECK-DAG: = !{i32 0, i32 [[DEVID:-?[0-9]+]], i32 [[FILEID:-?[0-9]+]], !"_ZN2SB3fooEv", i32 {{[0-9]+}}, i32 0, i32 {{[0-9]+}}}
+// TCHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2SDD1Ev", i32 {{[0-9]+}}, i32 0, i32 {{[0-9]+}}}
+// TCHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2SEC1Ev", i32 {{[0-9]+}}, i32 0, i32 {{[0-9]+}}}
+// TCHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2SED1Ev", i32 {{[0-9]+}}, i32 0, i32 {{[0-9]+}}}
+// TCHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2STILi1000EE3fooEv", i32 {{[0-9]+}}, i32 0, i32 {{[0-9]+}}}
+// TCHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2STILi100EEC1Ev", i32 {{[0-9]+}}, i32 0, i32 {{[0-9]+}}}
+// TCHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_Z3bari", i32 {{[0-9]+}}, i32 0, i32 {{[0-9]+}}}
+// TCHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2STILi100EED1Ev", i32 {{[0-9]+}}, i32 0, i32 {{[0-9]+}}}
+// TCHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2STILi1000EEC1Ev", i32 {{[0-9]+}}, i32 0, i32 {{[0-9]+}}}
+// TCHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2STILi1000EED1Ev", i32 {{[0-9]+}}, i32 0, i32 {{[0-9]+}}}
+// TCHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2STILi100EE3fooEv", i32 {{[0-9]+}}, i32 0, i32 {{[0-9]+}}}
+// TCHECK-DAG: = !{i32 0, i32 [[DEVID]], i32 [[FILEID]], !"_ZN2SCC1Ev", i32 {{[0-9]+}}, i32 0, i32 {{[0-9]+}}}
 
 // TCHECK-DAG: !{!"llvm.loop.vectorize.enable", i1 true}
 // CHECK-DAG: !{!"llvm.loop.vectorize.enable", i1 true}
diff --git a/clang/test/OpenMP/target_teams_distribute_simd_depend_codegen.cpp b/clang/test/OpenMP/target_teams_distribute_simd_depend_codegen.cpp
index f3dfd1f10316f..9335c65bb2296 100644
--- a/clang/test/OpenMP/target_teams_distribute_simd_depend_codegen.cpp
+++ b/clang/test/OpenMP/target_teams_distribute_simd_depend_codegen.cpp
@@ -48,9 +48,9 @@
 #define HEADER
 
 // CHECK-DAG: [[TT:%.+]] = type { i64, i8 }
-// CHECK-DAG: [[ENTTY:%.+]] = type { ptr, ptr, i[[SZ:32|64]], i32, i32 }
+// CHECK-DAG: [[ENTTY:%.+]] = type { i64, i16, i16, i32, ptr, ptr, i64, i64, ptr }
 
-// TCHECK: [[ENTTY:%.+]] = type { ptr, ptr, i{{32|64}}, i32, i32 }
+// TCHECK: [[ENTTY:%.+]] = type { i64, i16, i16, i32, ptr, ptr, i64, i64, ptr }
 
 // OMP45-DAG: [[SIZET:@.+]] = private unnamed_addr constant [2 x i64] [i64 0, i64 4]
 // OMP45-DAG: [[MAPT:@.+]] = private unnamed_addr constant [2 x i64] [i64 544, i64 800]
@@ -90,7 +90,7 @@ int foo(int n) {
   // CHECK:       [[GEP:%.+]] = getelementptr inbounds nuw %{{.+}}, ptr %{{.+}}, i32 0, i32 0
   // CHECK:       [[DEV:%.+]] = load i32, ptr [[DEVICE_CAP]],
   // CHECK:       store i32 [[DEV]], ptr [[GEP]],
-  // CHECK:       [[TASK:%.+]] = call ptr @__kmpc_omp_task_alloc(ptr [[ID:@.+]], i32 [[GTID:%.+]], i32 1, i[[SZ]] {{20|40}}, i[[SZ]] 4, ptr [[TASK_ENTRY0:@.+]])
+  // CHECK:       [[TASK:%.+]] = call ptr @__kmpc_omp_task_alloc(ptr [[ID:@.+]], i32 [[GTID:%.+]], i32 1, i[[SZ:32|64]] {{20|40}}, i[[SZ]] 4, ptr [[TASK_ENTRY0:@.+]])
   // CHECK:       getelementptr %struct.kmp_depend_info, ptr %{{.+}}, i[[SZ]] 0
   // CHECK:       getelementptr %struct.kmp_depend_info, ptr %{{.+}}, i[[SZ]] 1
   // CHECK:       getelementptr %struct.kmp_depend_info, ptr %{{.+}}, i[[SZ]] 2
diff --git a/clang/test/PCH/cuda-kernel-call.cu b/clang/test/PCH/cuda-kernel-call.cu
index ffb0c1444fe69..da9d81c531c41 100644
--- a/clang/test/PCH/cuda-kernel-call.cu
+++ b/clang/test/PCH/cuda-kernel-call.cu
@@ -1,5 +1,7 @@
 // RUN: %clang_cc1 -emit-pch -o %t %s
 // RUN: %clang_cc1 -include-pch %t -fsyntax-only %s 
+// RUN: %clang_cc1 -emit-pch -fcuda-is-device -o %t-device %s
+// RUN: %clang_cc1 -fcuda-is-device -include-pch %t-device -fsyntax-only %s
 
 #ifndef HEADER
 #define HEADER
@@ -14,12 +16,21 @@ void kcall(void (*kp)()) {
 __global__ void kern() {
 }
 
+// Make sure that target overloaded functions remain
+// available as overloads after PCH deserialization.
+__host__ int overloaded_func();
+__device__ int overloaded_func();
+
 #else
 // Using the header.
 
 void test() {
   kcall(kern);
   kern<<<1, 1>>>();
+  overloaded_func();
 }
 
+__device__ void test () {
+  overloaded_func();
+}
 #endif
diff --git a/clang/test/Parser/cxx2c-binding-pack.cpp b/clang/test/Parser/cxx2c-binding-pack.cpp
new file mode 100644
index 0000000000000..0daaad3a459ed
--- /dev/null
+++ b/clang/test/Parser/cxx2c-binding-pack.cpp
@@ -0,0 +1,25 @@
+// RUN: %clang_cc1 -std=c++2c -verify -fsyntax-only %s
+
+template <unsigned N>
+void decompose_array() {
+  int arr[4] = {1, 2, 3, 5};
+  auto [x, ... // #1
+    rest, ...more_rest] = arr; // expected-error{{multiple packs in structured binding declaration}}
+                               // expected-note@#1{{previous binding pack specified here}}
+
+  auto [y...] = arr; // expected-error{{'...' must immediately precede declared identifier}}
+
+  auto [...] = arr; // #2
+                    // expected-error@#2{{expected identifier}}
+                    // expected-error@#2{{{no names were provided}}}
+                    // expected-warning@#2{{{does not allow a decomposition group to be empty}}}
+  auto [a, ..., b] = arr; // #3
+                          // expected-error@#3{{expected identifier}}
+                          // expected-error@#3{{{only 1 name was provided}}}
+  auto [a1, ...] = arr; // #4
+                        // expected-error@#4{{expected identifier}}
+                        // expected-error@#4{{{only 1 name was provided}}}
+  auto [..., b] = arr; // #5
+                       // expected-error@#5{{expected identifier}}
+                       // expected-error@#5{{{no names were provided}}}
+}
diff --git a/clang/test/ParserOpenACC/parse-constructs.c b/clang/test/ParserOpenACC/parse-constructs.c
index 886a912713c58..f0698495a3cc2 100644
--- a/clang/test/ParserOpenACC/parse-constructs.c
+++ b/clang/test/ParserOpenACC/parse-constructs.c
@@ -109,30 +109,23 @@ void func() {
   for(int i = 0; i < 6;++i){}
 
   int i = 0, j = 0, k = 0;
-  // expected-warning@+1{{OpenACC construct 'atomic' not yet implemented, pragma ignored}}
 #pragma acc atomic
-  i = j;
-  // expected-error@+2{{invalid OpenACC clause 'garbage'}}
-  // expected-warning@+1{{OpenACC construct 'atomic' not yet implemented, pragma ignored}}
+  i = i + 1;
+  // expected-error@+1{{invalid OpenACC clause 'garbage'}}
 #pragma acc atomic garbage
-  i = j;
-  // expected-error@+2{{invalid OpenACC clause 'garbage'}}
-  // expected-warning@+1{{OpenACC construct 'atomic' not yet implemented, pragma ignored}}
+  i = i + 1;
+  // expected-error@+1{{invalid OpenACC clause 'garbage'}}
 #pragma acc atomic garbage clause list
-  i = j;
-  // expected-warning@+1{{OpenACC construct 'atomic' not yet implemented, pragma ignored}}
+  i = i + 1;
 #pragma acc atomic read
   i = j;
-  // expected-error@+2{{invalid OpenACC clause 'clause'}}
-  // expected-warning@+1{{OpenACC construct 'atomic' not yet implemented, pragma ignored}}
+  // expected-error@+1{{invalid OpenACC clause 'clause'}}
 #pragma acc atomic write clause list
   i = i + j;
-  // expected-error@+2{{invalid OpenACC clause 'clause'}}
-  // expected-warning@+1{{OpenACC construct 'atomic' not yet implemented, pragma ignored}}
+  // expected-error@+1{{invalid OpenACC clause 'clause'}}
 #pragma acc atomic update clause list
   i++;
-  // expected-error@+2{{invalid OpenACC clause 'clause'}}
-  // expected-warning@+1{{OpenACC construct 'atomic' not yet implemented, pragma ignored}}
+  // expected-error@+1{{invalid OpenACC clause 'clause'}}
 #pragma acc atomic capture clause list
   i = j++;
 
diff --git a/clang/test/Preprocessor/arm-target-features.c b/clang/test/Preprocessor/arm-target-features.c
index ecf9d7eb5c19c..94dcfc2424bb1 100644
--- a/clang/test/Preprocessor/arm-target-features.c
+++ b/clang/test/Preprocessor/arm-target-features.c
@@ -1013,3 +1013,17 @@
 // CHECK-MVE1_2: #define __ARM_FEATURE_MVE 1
 // RUN: %clang -target arm-arm-none-eabi -march=armv8.1-m.main+mve.fp -x c -E -dM %s -o - | FileCheck -check-prefix=CHECK-MVE3 %s
 // CHECK-MVE3: #define __ARM_FEATURE_MVE 3
+
+// Cortex-R52 and Cortex-R52Plus correctly enable the `fpv5-sp-d16` FPU when compiling for the SP only version of the CPU.
+// RUN: %clang -target arm-none-eabi -mcpu=cortex-r52+nosimd+nofp.dp -mfloat-abi=hard -x c -E -dM -o - %s | FileCheck -check-prefix=CHECK-R52 %s
+// RUN: %clang -target arm-none-eabi -mcpu=cortex-r52plus+nosimd+nofp.dp -mfloat-abi=hard -x c -E -dM -o - %s | FileCheck -check-prefix=CHECK-R52 %s
+// RUN: %clang -target arm-none-eabi -mcpu=cortex-r52+nofp.dp -mfloat-abi=hard -x c -E -dM -o - %s | FileCheck -check-prefix=CHECK-R52 %s
+// RUN: %clang -target arm-none-eabi -mcpu=cortex-r52plus+nofp.dp -mfloat-abi=hard -x c -E -dM -o - %s | FileCheck -check-prefix=CHECK-R52 %s
+// CHECK-R52: #define __ARM_FEATURE_FMA 1
+// CHECK-R52: #define __ARM_FP 0x6
+// CHECK-R52: #define __ARM_FPV5__ 1
+// CHECK-R52: #define __ARM_VFPV2__ 1
+// CHECK-R52-NEXT: #define __ARM_VFPV3__ 1
+// CHECK-R52-NEXT: #define __ARM_VFPV4__ 1
+// CHECK-R52-NOT: #define __ARM_NEON 1
+// CHECK-R52-NOT: #define __ARM_NEON__
diff --git a/clang/test/Preprocessor/builtin_aux_info.cpp b/clang/test/Preprocessor/builtin_aux_info.cpp
deleted file mode 100644
index 60c8c6c492479..0000000000000
--- a/clang/test/Preprocessor/builtin_aux_info.cpp
+++ /dev/null
@@ -1,18 +0,0 @@
-// RUN: %clang_cc1 -fopenmp -triple=spirv64 -fopenmp-is-target-device \
-// RUN: -aux-triple x86_64-linux-unknown -E %s | FileCheck -implicit-check-not=BAD %s
-
-// RUN: %clang_cc1 -fopenmp -triple=nvptx64 -fopenmp-is-target-device \
-// RUN: -aux-triple x86_64-linux-unknown -E %s | FileCheck -implicit-check-not=BAD %s
-
-// RUN: %clang_cc1 -fopenmp -triple=amdgcn-amd-amdhsa -fopenmp-is-target-device \
-// RUN: -aux-triple x86_64-linux-unknown -E %s | FileCheck -implicit-check-not=BAD %s
-
-// RUN: %clang_cc1 -fopenmp -triple=aarch64 -fopenmp-is-target-device \
-// RUN: -aux-triple x86_64-linux-unknown -E %s | FileCheck -implicit-check-not=BAD %s
-
-// CHECK: GOOD
-#if __has_builtin(__builtin_ia32_pause)
-  BAD
-#else
-  GOOD
-#endif
diff --git a/clang/test/Preprocessor/deprecate-threads-macro-definition-msvc1939.c b/clang/test/Preprocessor/deprecate-threads-macro-definition-msvc1939.c
new file mode 100644
index 0000000000000..e197d8d403a3f
--- /dev/null
+++ b/clang/test/Preprocessor/deprecate-threads-macro-definition-msvc1939.c
@@ -0,0 +1,15 @@
+// RUN: %clang_cc1 -E -dM -triple=arm64ec-windows-msvc -std=c89 -fms-compatibility-version=19.33 -ffreestanding < /dev/null | FileCheck -check-prefix=C89_MSVC33 %s
+// RUN: %clang_cc1 -E -dM -triple=arm64ec-windows-msvc -std=c99 -fms-compatibility-version=19.33 -ffreestanding < /dev/null | FileCheck -check-prefix=C99_MSVC33 %s
+// RUN: %clang_cc1 -E -dM -triple=arm64ec-windows-msvc -std=c11 -fms-compatibility-version=19.33 -ffreestanding < /dev/null | FileCheck -check-prefix=C11_MSVC33 %s
+// RUN: %clang_cc1 -E -dM -triple=arm64ec-windows-msvc -std=c89 -fms-compatibility-version=19.39 -ffreestanding < /dev/null | FileCheck -check-prefix=C89_MSVC39 %s
+// RUN: %clang_cc1 -E -dM -triple=arm64ec-windows-msvc -std=c99 -fms-compatibility-version=19.39 -ffreestanding < /dev/null | FileCheck -check-prefix=C99_MSVC39 %s
+// RUN: %clang_cc1 -E -dM -triple=arm64ec-windows-msvc -std=c11 -fms-compatibility-version=19.39 -ffreestanding < /dev/null | FileCheck -check-prefix=C11_MSVC39 %s
+// RUN: %clang_cc1 -E -dM -triple=arm64ec-windows-msvc -std=c11 -fms-compatibility-version=19.40 -ffreestanding < /dev/null | FileCheck -check-prefix=C11_MSVC40 %s
+
+// C89_MSVC33: #define __STDC_NO_THREADS__ 1
+// C99_MSVC33: #define __STDC_NO_THREADS__ 1
+// C11_MSVC33: #define __STDC_NO_THREADS__ 1
+// C89_MSVC39: #define __STDC_NO_THREADS__ 1
+// C99_MSVC39: #define __STDC_NO_THREADS__ 1
+// C11_MSVC39-NOT: #define __STDC_NO_THREADS__
+// C11_MSVC40-NOT: #define __STDC_NO_THREADS__
diff --git a/clang/test/Preprocessor/init-aarch64.c b/clang/test/Preprocessor/init-aarch64.c
index 8578993dbfaeb..3036b496db25d 100644
--- a/clang/test/Preprocessor/init-aarch64.c
+++ b/clang/test/Preprocessor/init-aarch64.c
@@ -125,8 +125,8 @@
 // AARCH64-NEXT: #define __FP_FAST_FMAF 1
 // AARCH64-NEXT: #define __FUNCTION_MULTI_VERSIONING_SUPPORT_LEVEL 202430
 // AARCH64-NEXT: #define __GCC_ASM_FLAG_OUTPUTS__ 1
-// AARCH64-NEXT: #define __GCC_CONSTRUCTIVE_SIZE {{.+}}
-// AARCH64-NEXT: #define __GCC_DESTRUCTIVE_SIZE {{.+}}
+// AARCH64-NEXT: #define __GCC_CONSTRUCTIVE_SIZE 64
+// AARCH64-NEXT: #define __GCC_DESTRUCTIVE_SIZE 256
 // AARCH64-NEXT: #define __GCC_HAVE_SYNC_COMPARE_AND_SWAP_1 1
 // AARCH64-NEXT: #define __GCC_HAVE_SYNC_COMPARE_AND_SWAP_16 1
 // AARCH64-NEXT: #define __GCC_HAVE_SYNC_COMPARE_AND_SWAP_2 1
@@ -135,26 +135,31 @@
 // AARCH64_CXX-NEXT: #define __GLIBCXX_BITSIZE_INT_N_0 128
 // AARCH64_CXX-NEXT: #define __GLIBCXX_TYPE_INT_N_0 __int128
 // AARCH64-NEXT: #define __HAVE_FUNCTION_MULTI_VERSIONING 1
+// AARCH64-NEXT: #define __INT16_C(c) c
 // AARCH64-NEXT: #define __INT16_C_SUFFIX__
 // AARCH64-NEXT: #define __INT16_FMTd__ "hd"
 // AARCH64-NEXT: #define __INT16_FMTi__ "hi"
 // AARCH64-NEXT: #define __INT16_MAX__ 32767
 // AARCH64-NEXT: #define __INT16_TYPE__ short
+// AARCH64-NEXT: #define __INT32_C(c) c
 // AARCH64-NEXT: #define __INT32_C_SUFFIX__
 // AARCH64-NEXT: #define __INT32_FMTd__ "d"
 // AARCH64-NEXT: #define __INT32_FMTi__ "i"
 // AARCH64-NEXT: #define __INT32_MAX__ 2147483647
 // AARCH64-NEXT: #define __INT32_TYPE__ int
+// AARCH64-NEXT: #define __INT64_C(c) c##L
 // AARCH64-NEXT: #define __INT64_C_SUFFIX__ L
 // AARCH64-NEXT: #define __INT64_FMTd__ "ld"
 // AARCH64-NEXT: #define __INT64_FMTi__ "li"
 // AARCH64-NEXT: #define __INT64_MAX__ 9223372036854775807L
 // AARCH64-NEXT: #define __INT64_TYPE__ long int
+// AARCH64-NEXT: #define __INT8_C(c) c
 // AARCH64-NEXT: #define __INT8_C_SUFFIX__
 // AARCH64-NEXT: #define __INT8_FMTd__ "hhd"
 // AARCH64-NEXT: #define __INT8_FMTi__ "hhi"
 // AARCH64-NEXT: #define __INT8_MAX__ 127
 // AARCH64-NEXT: #define __INT8_TYPE__ signed char
+// AARCH64-NEXT: #define __INTMAX_C(c) c##L
 // AARCH64-NEXT: #define __INTMAX_C_SUFFIX__ L
 // AARCH64-NEXT: #define __INTMAX_FMTd__ "ld"
 // AARCH64-NEXT: #define __INTMAX_FMTi__ "li"
@@ -287,6 +292,7 @@
 // AARCH64-NEXT: #define __STDC_UTF_32__ 1
 // AARCH64_C: #define __STDC_VERSION__ 201710L
 // AARCH64-NEXT: #define __STDC__ 1
+// AARCH64-NEXT: #define __UINT16_C(c) c
 // AARCH64-NEXT: #define __UINT16_C_SUFFIX__
 // AARCH64-NEXT: #define __UINT16_FMTX__ "hX"
 // AARCH64-NEXT: #define __UINT16_FMTo__ "ho"
@@ -294,6 +300,7 @@
 // AARCH64-NEXT: #define __UINT16_FMTx__ "hx"
 // AARCH64-NEXT: #define __UINT16_MAX__ 65535
 // AARCH64-NEXT: #define __UINT16_TYPE__ unsigned short
+// AARCH64-NEXT: #define __UINT32_C(c) c##U
 // AARCH64-NEXT: #define __UINT32_C_SUFFIX__ U
 // AARCH64-NEXT: #define __UINT32_FMTX__ "X"
 // AARCH64-NEXT: #define __UINT32_FMTo__ "o"
@@ -301,6 +308,7 @@
 // AARCH64-NEXT: #define __UINT32_FMTx__ "x"
 // AARCH64-NEXT: #define __UINT32_MAX__ 4294967295U
 // AARCH64-NEXT: #define __UINT32_TYPE__ unsigned int
+// AARCH64-NEXT: #define __UINT64_C(c) c##UL
 // AARCH64-NEXT: #define __UINT64_C_SUFFIX__ UL
 // AARCH64-NEXT: #define __UINT64_FMTX__ "lX"
 // AARCH64-NEXT: #define __UINT64_FMTo__ "lo"
@@ -308,6 +316,7 @@
 // AARCH64-NEXT: #define __UINT64_FMTx__ "lx"
 // AARCH64-NEXT: #define __UINT64_MAX__ 18446744073709551615UL
 // AARCH64-NEXT: #define __UINT64_TYPE__ long unsigned int
+// AARCH64-NEXT: #define __UINT8_C(c) c
 // AARCH64-NEXT: #define __UINT8_C_SUFFIX__
 // AARCH64-NEXT: #define __UINT8_FMTX__ "hhX"
 // AARCH64-NEXT: #define __UINT8_FMTo__ "hho"
@@ -315,6 +324,7 @@
 // AARCH64-NEXT: #define __UINT8_FMTx__ "hhx"
 // AARCH64-NEXT: #define __UINT8_MAX__ 255
 // AARCH64-NEXT: #define __UINT8_TYPE__ unsigned char
+// AARCH64-NEXT: #define __UINTMAX_C(c) c##UL
 // AARCH64-NEXT: #define __UINTMAX_C_SUFFIX__ UL
 // AARCH64-NEXT: #define __UINTMAX_FMTX__ "lX"
 // AARCH64-NEXT: #define __UINTMAX_FMTo__ "lo"
@@ -435,26 +445,31 @@
 // AARCH64-DARWIN: #define __FLT_MIN__ 1.17549435e-38F
 // AARCH64-DARWIN: #define __FLT_RADIX__ 2
 // AARCH64-DARWIN: #define __FUNCTION_MULTI_VERSIONING_SUPPORT_LEVEL 202430
+// AARCH64-DARWIN: #define __INT16_C(c) c
 // AARCH64-DARWIN: #define __INT16_C_SUFFIX__
 // AARCH64-DARWIN: #define __INT16_FMTd__ "hd"
 // AARCH64-DARWIN: #define __INT16_FMTi__ "hi"
 // AARCH64-DARWIN: #define __INT16_MAX__ 32767
 // AARCH64-DARWIN: #define __INT16_TYPE__ short
+// AARCH64-DARWIN: #define __INT32_C(c) c
 // AARCH64-DARWIN: #define __INT32_C_SUFFIX__
 // AARCH64-DARWIN: #define __INT32_FMTd__ "d"
 // AARCH64-DARWIN: #define __INT32_FMTi__ "i"
 // AARCH64-DARWIN: #define __INT32_MAX__ 2147483647
 // AARCH64-DARWIN: #define __INT32_TYPE__ int
+// AARCH64-DARWIN: #define __INT64_C(c) c##LL
 // AARCH64-DARWIN: #define __INT64_C_SUFFIX__ LL
 // AARCH64-DARWIN: #define __INT64_FMTd__ "lld"
 // AARCH64-DARWIN: #define __INT64_FMTi__ "lli"
 // AARCH64-DARWIN: #define __INT64_MAX__ 9223372036854775807LL
 // AARCH64-DARWIN: #define __INT64_TYPE__ long long int
+// AARCH64-DARWIN: #define __INT8_C(c) c
 // AARCH64-DARWIN: #define __INT8_C_SUFFIX__
 // AARCH64-DARWIN: #define __INT8_FMTd__ "hhd"
 // AARCH64-DARWIN: #define __INT8_FMTi__ "hhi"
 // AARCH64-DARWIN: #define __INT8_MAX__ 127
 // AARCH64-DARWIN: #define __INT8_TYPE__ signed char
+// AARCH64-DARWIN: #define __INTMAX_C(c) c##L
 // AARCH64-DARWIN: #define __INTMAX_C_SUFFIX__ L
 // AARCH64-DARWIN: #define __INTMAX_FMTd__ "ld"
 // AARCH64-DARWIN: #define __INTMAX_FMTi__ "li"
@@ -538,18 +553,23 @@
 // AARCH64-DARWIN: #define __SIZE_MAX__ 18446744073709551615UL
 // AARCH64-DARWIN: #define __SIZE_TYPE__ long unsigned int
 // AARCH64-DARWIN: #define __SIZE_WIDTH__ 64
+// AARCH64-DARWIN: #define __UINT16_C(c) c
 // AARCH64-DARWIN: #define __UINT16_C_SUFFIX__
 // AARCH64-DARWIN: #define __UINT16_MAX__ 65535
 // AARCH64-DARWIN: #define __UINT16_TYPE__ unsigned short
+// AARCH64-DARWIN: #define __UINT32_C(c) c##U
 // AARCH64-DARWIN: #define __UINT32_C_SUFFIX__ U
 // AARCH64-DARWIN: #define __UINT32_MAX__ 4294967295U
 // AARCH64-DARWIN: #define __UINT32_TYPE__ unsigned int
+// AARCH64-DARWIN: #define __UINT64_C(c) c##ULL
 // AARCH64-DARWIN: #define __UINT64_C_SUFFIX__ ULL
 // AARCH64-DARWIN: #define __UINT64_MAX__ 18446744073709551615ULL
 // AARCH64-DARWIN: #define __UINT64_TYPE__ long long unsigned int
+// AARCH64-DARWIN: #define __UINT8_C(c) c
 // AARCH64-DARWIN: #define __UINT8_C_SUFFIX__
 // AARCH64-DARWIN: #define __UINT8_MAX__ 255
 // AARCH64-DARWIN: #define __UINT8_TYPE__ unsigned char
+// AARCH64-DARWIN: #define __UINTMAX_C(c) c##UL
 // AARCH64-DARWIN: #define __UINTMAX_C_SUFFIX__ UL
 // AARCH64-DARWIN: #define __UINTMAX_MAX__ 18446744073709551615UL
 // AARCH64-DARWIN: #define __UINTMAX_TYPE__ long unsigned int
@@ -703,18 +723,23 @@
 // AARCH64-MSVC: #define __STDC_UTF_32__ 1
 // AARCH64-MSVC: #define __STDC_VERSION__ 201710L
 // AARCH64-MSVC: #define __STDC__ 1
+// AARCH64-MSVC: #define __UINT16_C(c) c
 // AARCH64-MSVC: #define __UINT16_C_SUFFIX__
 // AARCH64-MSVC: #define __UINT16_MAX__ 65535
 // AARCH64-MSVC: #define __UINT16_TYPE__ unsigned short
+// AARCH64-MSVC: #define __UINT32_C(c) c##U
 // AARCH64-MSVC: #define __UINT32_C_SUFFIX__ U
 // AARCH64-MSVC: #define __UINT32_MAX__ 4294967295U
 // AARCH64-MSVC: #define __UINT32_TYPE__ unsigned int
+// AARCH64-MSVC: #define __UINT64_C(c) c##ULL
 // AARCH64-MSVC: #define __UINT64_C_SUFFIX__ ULL
 // AARCH64-MSVC: #define __UINT64_MAX__ 18446744073709551615ULL
 // AARCH64-MSVC: #define __UINT64_TYPE__ long long unsigned int
+// AARCH64-MSVC: #define __UINT8_C(c) c
 // AARCH64-MSVC: #define __UINT8_C_SUFFIX__
 // AARCH64-MSVC: #define __UINT8_MAX__ 255
 // AARCH64-MSVC: #define __UINT8_TYPE__ unsigned char
+// AARCH64-MSVC: #define __UINTMAX_C(c) c##ULL
 // AARCH64-MSVC: #define __UINTMAX_C_SUFFIX__ ULL
 // AARCH64-MSVC: #define __UINTMAX_MAX__ 18446744073709551615ULL
 // AARCH64-MSVC: #define __UINTMAX_TYPE__ long long unsigned int
@@ -747,7 +772,7 @@
 // AARCH64-MSVC: #define __WINT_WIDTH__ 16
 // AARCH64-MSVC: #define __aarch64__ 1
 
-// RUN: %clang_cc1 -E -dM -ffreestanding -triple=arm64ec-windows-msvc < /dev/null | FileCheck -match-full-lines -check-prefix ARM64EC-MSVC %s
+// RUN: %clang_cc1 -E -dM -fms-compatibility-version=19.33 -ffreestanding -triple=arm64ec-windows-msvc < /dev/null | FileCheck -match-full-lines -check-prefix ARM64EC-MSVC %s
 
 // ARM64EC-MSVC: #define _INTEGRAL_MAX_BITS 64
 // ARM64EC-MSVC: #define _M_AMD64 100
@@ -867,26 +892,31 @@
 // ARM64EC-MSVC: #define __GCC_HAVE_SYNC_COMPARE_AND_SWAP_4 1
 // ARM64EC-MSVC: #define __GCC_HAVE_SYNC_COMPARE_AND_SWAP_8 1
 // ARM64EC-MSVC: #define __HAVE_FUNCTION_MULTI_VERSIONING 1
+// ARM64EC-MSVC: #define __INT16_C(c) c
 // ARM64EC-MSVC: #define __INT16_C_SUFFIX__
 // ARM64EC-MSVC: #define __INT16_FMTd__ "hd"
 // ARM64EC-MSVC: #define __INT16_FMTi__ "hi"
 // ARM64EC-MSVC: #define __INT16_MAX__ 32767
 // ARM64EC-MSVC: #define __INT16_TYPE__ short
+// ARM64EC-MSVC: #define __INT32_C(c) c
 // ARM64EC-MSVC: #define __INT32_C_SUFFIX__
 // ARM64EC-MSVC: #define __INT32_FMTd__ "d"
 // ARM64EC-MSVC: #define __INT32_FMTi__ "i"
 // ARM64EC-MSVC: #define __INT32_MAX__ 2147483647
 // ARM64EC-MSVC: #define __INT32_TYPE__ int
+// ARM64EC-MSVC: #define __INT64_C(c) c##LL
 // ARM64EC-MSVC: #define __INT64_C_SUFFIX__ LL
 // ARM64EC-MSVC: #define __INT64_FMTd__ "lld"
 // ARM64EC-MSVC: #define __INT64_FMTi__ "lli"
 // ARM64EC-MSVC: #define __INT64_MAX__ 9223372036854775807LL
 // ARM64EC-MSVC: #define __INT64_TYPE__ long long int
+// ARM64EC-MSVC: #define __INT8_C(c) c
 // ARM64EC-MSVC: #define __INT8_C_SUFFIX__
 // ARM64EC-MSVC: #define __INT8_FMTd__ "hhd"
 // ARM64EC-MSVC: #define __INT8_FMTi__ "hhi"
 // ARM64EC-MSVC: #define __INT8_MAX__ 127
 // ARM64EC-MSVC: #define __INT8_TYPE__ signed char
+// ARM64EC-MSVC: #define __INTMAX_C(c) c##LL
 // ARM64EC-MSVC: #define __INTMAX_C_SUFFIX__ LL
 // ARM64EC-MSVC: #define __INTMAX_FMTd__ "lld"
 // ARM64EC-MSVC: #define __INTMAX_FMTi__ "lli"
@@ -1013,6 +1043,7 @@
 // ARM64EC-MSVC: #define __STDC_UTF_32__ 1
 // ARM64EC-MSVC: #define __STDC_VERSION__ 201710L
 // ARM64EC-MSVC: #define __STDC__ 1
+// ARM64EC-MSVC: #define __UINT16_C(c) c
 // ARM64EC-MSVC: #define __UINT16_C_SUFFIX__
 // ARM64EC-MSVC: #define __UINT16_FMTX__ "hX"
 // ARM64EC-MSVC: #define __UINT16_FMTo__ "ho"
@@ -1020,6 +1051,7 @@
 // ARM64EC-MSVC: #define __UINT16_FMTx__ "hx"
 // ARM64EC-MSVC: #define __UINT16_MAX__ 65535
 // ARM64EC-MSVC: #define __UINT16_TYPE__ unsigned short
+// ARM64EC-MSVC: #define __UINT32_C(c) c##U
 // ARM64EC-MSVC: #define __UINT32_C_SUFFIX__ U
 // ARM64EC-MSVC: #define __UINT32_FMTX__ "X"
 // ARM64EC-MSVC: #define __UINT32_FMTo__ "o"
@@ -1027,6 +1059,7 @@
 // ARM64EC-MSVC: #define __UINT32_FMTx__ "x"
 // ARM64EC-MSVC: #define __UINT32_MAX__ 4294967295U
 // ARM64EC-MSVC: #define __UINT32_TYPE__ unsigned int
+// ARM64EC-MSVC: #define __UINT64_C(c) c##ULL
 // ARM64EC-MSVC: #define __UINT64_C_SUFFIX__ ULL
 // ARM64EC-MSVC: #define __UINT64_FMTX__ "llX"
 // ARM64EC-MSVC: #define __UINT64_FMTo__ "llo"
@@ -1034,6 +1067,7 @@
 // ARM64EC-MSVC: #define __UINT64_FMTx__ "llx"
 // ARM64EC-MSVC: #define __UINT64_MAX__ 18446744073709551615ULL
 // ARM64EC-MSVC: #define __UINT64_TYPE__ long long unsigned int
+// ARM64EC-MSVC: #define __UINT8_C(c) c
 // ARM64EC-MSVC: #define __UINT8_C_SUFFIX__
 // ARM64EC-MSVC: #define __UINT8_FMTX__ "hhX"
 // ARM64EC-MSVC: #define __UINT8_FMTo__ "hho"
@@ -1041,6 +1075,7 @@
 // ARM64EC-MSVC: #define __UINT8_FMTx__ "hhx"
 // ARM64EC-MSVC: #define __UINT8_MAX__ 255
 // ARM64EC-MSVC: #define __UINT8_TYPE__ unsigned char
+// ARM64EC-MSVC: #define __UINTMAX_C(c) c##ULL
 // ARM64EC-MSVC: #define __UINTMAX_C_SUFFIX__ ULL
 // ARM64EC-MSVC: #define __UINTMAX_FMTX__ "llX"
 // ARM64EC-MSVC: #define __UINTMAX_FMTo__ "llo"
diff --git a/clang/test/Preprocessor/init-arm.c b/clang/test/Preprocessor/init-arm.c
index 6e3acacc5c3a5..d2fcfe94bcd3d 100644
--- a/clang/test/Preprocessor/init-arm.c
+++ b/clang/test/Preprocessor/init-arm.c
@@ -46,26 +46,31 @@
 // ARM:#define __FLT_MIN_EXP__ (-125)
 // ARM:#define __FLT_MIN__ 1.17549435e-38F
 // ARM:#define __FLT_RADIX__ 2
+// ARM:#define __INT16_C(c) c
 // ARM:#define __INT16_C_SUFFIX__
 // ARM:#define __INT16_FMTd__ "hd"
 // ARM:#define __INT16_FMTi__ "hi"
 // ARM:#define __INT16_MAX__ 32767
 // ARM:#define __INT16_TYPE__ short
+// ARM:#define __INT32_C(c) c
 // ARM:#define __INT32_C_SUFFIX__
 // ARM:#define __INT32_FMTd__ "d"
 // ARM:#define __INT32_FMTi__ "i"
 // ARM:#define __INT32_MAX__ 2147483647
 // ARM:#define __INT32_TYPE__ int
+// ARM:#define __INT64_C(c) c##LL
 // ARM:#define __INT64_C_SUFFIX__ LL
 // ARM:#define __INT64_FMTd__ "lld"
 // ARM:#define __INT64_FMTi__ "lli"
 // ARM:#define __INT64_MAX__ 9223372036854775807LL
 // ARM:#define __INT64_TYPE__ long long int
+// ARM:#define __INT8_C(c) c
 // ARM:#define __INT8_C_SUFFIX__
 // ARM:#define __INT8_FMTd__ "hhd"
 // ARM:#define __INT8_FMTi__ "hhi"
 // ARM:#define __INT8_MAX__ 127
 // ARM:#define __INT8_TYPE__ signed char
+// ARM:#define __INTMAX_C(c) c##LL
 // ARM:#define __INTMAX_C_SUFFIX__ LL
 // ARM:#define __INTMAX_FMTd__ "lld"
 // ARM:#define __INTMAX_FMTi__ "lli"
@@ -151,18 +156,23 @@
 // ARM:#define __SIZE_TYPE__ unsigned int
 // ARM:#define __SIZE_WIDTH__ 32
 // ARM-CXX:#define __STDCPP_DEFAULT_NEW_ALIGNMENT__ 8U
+// ARM:#define __UINT16_C(c) c
 // ARM:#define __UINT16_C_SUFFIX__
 // ARM:#define __UINT16_MAX__ 65535
 // ARM:#define __UINT16_TYPE__ unsigned short
+// ARM:#define __UINT32_C(c) c##U
 // ARM:#define __UINT32_C_SUFFIX__ U
 // ARM:#define __UINT32_MAX__ 4294967295U
 // ARM:#define __UINT32_TYPE__ unsigned int
+// ARM:#define __UINT64_C(c) c##ULL
 // ARM:#define __UINT64_C_SUFFIX__ ULL
 // ARM:#define __UINT64_MAX__ 18446744073709551615ULL
 // ARM:#define __UINT64_TYPE__ long long unsigned int
+// ARM:#define __UINT8_C(c) c
 // ARM:#define __UINT8_C_SUFFIX__
 // ARM:#define __UINT8_MAX__ 255
 // ARM:#define __UINT8_TYPE__ unsigned char
+// ARM:#define __UINTMAX_C(c) c##ULL
 // ARM:#define __UINTMAX_C_SUFFIX__ ULL
 // ARM:#define __UINTMAX_MAX__ 18446744073709551615ULL
 // ARM:#define __UINTMAX_TYPE__ long long unsigned int
@@ -248,26 +258,31 @@
 // ARM-BE:#define __FLT_MIN_EXP__ (-125)
 // ARM-BE:#define __FLT_MIN__ 1.17549435e-38F
 // ARM-BE:#define __FLT_RADIX__ 2
+// ARM-BE:#define __INT16_C(c) c
 // ARM-BE:#define __INT16_C_SUFFIX__
 // ARM-BE:#define __INT16_FMTd__ "hd"
 // ARM-BE:#define __INT16_FMTi__ "hi"
 // ARM-BE:#define __INT16_MAX__ 32767
 // ARM-BE:#define __INT16_TYPE__ short
+// ARM-BE:#define __INT32_C(c) c
 // ARM-BE:#define __INT32_C_SUFFIX__
 // ARM-BE:#define __INT32_FMTd__ "d"
 // ARM-BE:#define __INT32_FMTi__ "i"
 // ARM-BE:#define __INT32_MAX__ 2147483647
 // ARM-BE:#define __INT32_TYPE__ int
+// ARM-BE:#define __INT64_C(c) c##LL
 // ARM-BE:#define __INT64_C_SUFFIX__ LL
 // ARM-BE:#define __INT64_FMTd__ "lld"
 // ARM-BE:#define __INT64_FMTi__ "lli"
 // ARM-BE:#define __INT64_MAX__ 9223372036854775807LL
 // ARM-BE:#define __INT64_TYPE__ long long int
+// ARM-BE:#define __INT8_C(c) c
 // ARM-BE:#define __INT8_C_SUFFIX__
 // ARM-BE:#define __INT8_FMTd__ "hhd"
 // ARM-BE:#define __INT8_FMTi__ "hhi"
 // ARM-BE:#define __INT8_MAX__ 127
 // ARM-BE:#define __INT8_TYPE__ signed char
+// ARM-BE:#define __INTMAX_C(c) c##LL
 // ARM-BE:#define __INTMAX_C_SUFFIX__ LL
 // ARM-BE:#define __INTMAX_FMTd__ "lld"
 // ARM-BE:#define __INTMAX_FMTi__ "lli"
@@ -351,18 +366,23 @@
 // ARM-BE:#define __SIZE_MAX__ 4294967295U
 // ARM-BE:#define __SIZE_TYPE__ unsigned int
 // ARM-BE:#define __SIZE_WIDTH__ 32
+// ARM-BE:#define __UINT16_C(c) c
 // ARM-BE:#define __UINT16_C_SUFFIX__
 // ARM-BE:#define __UINT16_MAX__ 65535
 // ARM-BE:#define __UINT16_TYPE__ unsigned short
+// ARM-BE:#define __UINT32_C(c) c##U
 // ARM-BE:#define __UINT32_C_SUFFIX__ U
 // ARM-BE:#define __UINT32_MAX__ 4294967295U
 // ARM-BE:#define __UINT32_TYPE__ unsigned int
+// ARM-BE:#define __UINT64_C(c) c##ULL
 // ARM-BE:#define __UINT64_C_SUFFIX__ ULL
 // ARM-BE:#define __UINT64_MAX__ 18446744073709551615ULL
 // ARM-BE:#define __UINT64_TYPE__ long long unsigned int
+// ARM-BE:#define __UINT8_C(c) c
 // ARM-BE:#define __UINT8_C_SUFFIX__
 // ARM-BE:#define __UINT8_MAX__ 255
 // ARM-BE:#define __UINT8_TYPE__ unsigned char
+// ARM-BE:#define __UINTMAX_C(c) c##ULL
 // ARM-BE:#define __UINTMAX_C_SUFFIX__ ULL
 // ARM-BE:#define __UINTMAX_MAX__ 18446744073709551615ULL
 // ARM-BE:#define __UINTMAX_TYPE__ long long unsigned int
@@ -440,26 +460,31 @@
 // ARMEABISOFT:#define __FLT_MIN_EXP__ (-125)
 // ARMEABISOFT:#define __FLT_MIN__ 1.17549435e-38F
 // ARMEABISOFT:#define __FLT_RADIX__ 2
+// ARMEABISOFT:#define __INT16_C(c) c
 // ARMEABISOFT:#define __INT16_C_SUFFIX__
 // ARMEABISOFT:#define __INT16_FMTd__ "hd"
 // ARMEABISOFT:#define __INT16_FMTi__ "hi"
 // ARMEABISOFT:#define __INT16_MAX__ 32767
 // ARMEABISOFT:#define __INT16_TYPE__ short
+// ARMEABISOFT:#define __INT32_C(c) c
 // ARMEABISOFT:#define __INT32_C_SUFFIX__
 // ARMEABISOFT:#define __INT32_FMTd__ "d"
 // ARMEABISOFT:#define __INT32_FMTi__ "i"
 // ARMEABISOFT:#define __INT32_MAX__ 2147483647
 // ARMEABISOFT:#define __INT32_TYPE__ int
+// ARMEABISOFT:#define __INT64_C(c) c##LL
 // ARMEABISOFT:#define __INT64_C_SUFFIX__ LL
 // ARMEABISOFT:#define __INT64_FMTd__ "lld"
 // ARMEABISOFT:#define __INT64_FMTi__ "lli"
 // ARMEABISOFT:#define __INT64_MAX__ 9223372036854775807LL
 // ARMEABISOFT:#define __INT64_TYPE__ long long int
+// ARMEABISOFT:#define __INT8_C(c) c
 // ARMEABISOFT:#define __INT8_C_SUFFIX__
 // ARMEABISOFT:#define __INT8_FMTd__ "hhd"
 // ARMEABISOFT:#define __INT8_FMTi__ "hhi"
 // ARMEABISOFT:#define __INT8_MAX__ 127
 // ARMEABISOFT:#define __INT8_TYPE__ signed char
+// ARMEABISOFT:#define __INTMAX_C(c) c##LL
 // ARMEABISOFT:#define __INTMAX_C_SUFFIX__ LL
 // ARMEABISOFT:#define __INTMAX_FMTd__ "lld"
 // ARMEABISOFT:#define __INTMAX_FMTi__ "lli"
@@ -545,18 +570,23 @@
 // ARMEABISOFT:#define __SIZE_TYPE__ unsigned int
 // ARMEABISOFT:#define __SIZE_WIDTH__ 32
 // ARMEABISOFT:#define __SOFTFP__ 1
+// ARMEABISOFT:#define __UINT16_C(c) c
 // ARMEABISOFT:#define __UINT16_C_SUFFIX__
 // ARMEABISOFT:#define __UINT16_MAX__ 65535
 // ARMEABISOFT:#define __UINT16_TYPE__ unsigned short
+// ARMEABISOFT:#define __UINT32_C(c) c##U
 // ARMEABISOFT:#define __UINT32_C_SUFFIX__ U
 // ARMEABISOFT:#define __UINT32_MAX__ 4294967295U
 // ARMEABISOFT:#define __UINT32_TYPE__ unsigned int
+// ARMEABISOFT:#define __UINT64_C(c) c##ULL
 // ARMEABISOFT:#define __UINT64_C_SUFFIX__ ULL
 // ARMEABISOFT:#define __UINT64_MAX__ 18446744073709551615ULL
 // ARMEABISOFT:#define __UINT64_TYPE__ long long unsigned int
+// ARMEABISOFT:#define __UINT8_C(c) c
 // ARMEABISOFT:#define __UINT8_C_SUFFIX__
 // ARMEABISOFT:#define __UINT8_MAX__ 255
 // ARMEABISOFT:#define __UINT8_TYPE__ unsigned char
+// ARMEABISOFT:#define __UINTMAX_C(c) c##ULL
 // ARMEABISOFT:#define __UINTMAX_C_SUFFIX__ ULL
 // ARMEABISOFT:#define __UINTMAX_MAX__ 18446744073709551615ULL
 // ARMEABISOFT:#define __UINTMAX_TYPE__ long long unsigned int
@@ -640,26 +670,31 @@
 // ARMEABISOFTFP_NOFP:#define __FLT_MIN_EXP__ (-125)
 // ARMEABISOFTFP_NOFP:#define __FLT_MIN__ 1.17549435e-38F
 // ARMEABISOFTFP_NOFP:#define __FLT_RADIX__ 2
+// ARMEABISOFTFP_NOFP:#define __INT16_C(c) c
 // ARMEABISOFTFP_NOFP:#define __INT16_C_SUFFIX__
 // ARMEABISOFTFP_NOFP:#define __INT16_FMTd__ "hd"
 // ARMEABISOFTFP_NOFP:#define __INT16_FMTi__ "hi"
 // ARMEABISOFTFP_NOFP:#define __INT16_MAX__ 32767
 // ARMEABISOFTFP_NOFP:#define __INT16_TYPE__ short
+// ARMEABISOFTFP_NOFP:#define __INT32_C(c) c
 // ARMEABISOFTFP_NOFP:#define __INT32_C_SUFFIX__
 // ARMEABISOFTFP_NOFP:#define __INT32_FMTd__ "d"
 // ARMEABISOFTFP_NOFP:#define __INT32_FMTi__ "i"
 // ARMEABISOFTFP_NOFP:#define __INT32_MAX__ 2147483647
 // ARMEABISOFTFP_NOFP:#define __INT32_TYPE__ int
+// ARMEABISOFTFP_NOFP:#define __INT64_C(c) c##LL
 // ARMEABISOFTFP_NOFP:#define __INT64_C_SUFFIX__ LL
 // ARMEABISOFTFP_NOFP:#define __INT64_FMTd__ "lld"
 // ARMEABISOFTFP_NOFP:#define __INT64_FMTi__ "lli"
 // ARMEABISOFTFP_NOFP:#define __INT64_MAX__ 9223372036854775807LL
 // ARMEABISOFTFP_NOFP:#define __INT64_TYPE__ long long int
+// ARMEABISOFTFP_NOFP:#define __INT8_C(c) c
 // ARMEABISOFTFP_NOFP:#define __INT8_C_SUFFIX__
 // ARMEABISOFTFP_NOFP:#define __INT8_FMTd__ "hhd"
 // ARMEABISOFTFP_NOFP:#define __INT8_FMTi__ "hhi"
 // ARMEABISOFTFP_NOFP:#define __INT8_MAX__ 127
 // ARMEABISOFTFP_NOFP:#define __INT8_TYPE__ signed char
+// ARMEABISOFTFP_NOFP:#define __INTMAX_C(c) c##LL
 // ARMEABISOFTFP_NOFP:#define __INTMAX_C_SUFFIX__ LL
 // ARMEABISOFTFP_NOFP:#define __INTMAX_FMTd__ "lld"
 // ARMEABISOFTFP_NOFP:#define __INTMAX_FMTi__ "lli"
@@ -745,18 +780,23 @@
 // ARMEABISOFTFP_NOFP:#define __SIZE_TYPE__ unsigned int
 // ARMEABISOFTFP_NOFP:#define __SIZE_WIDTH__ 32
 // ARMEABISOFTFP_NOFP:#define __SOFTFP__ 1
+// ARMEABISOFTFP_NOFP:#define __UINT16_C(c) c
 // ARMEABISOFTFP_NOFP:#define __UINT16_C_SUFFIX__
 // ARMEABISOFTFP_NOFP:#define __UINT16_MAX__ 65535
 // ARMEABISOFTFP_NOFP:#define __UINT16_TYPE__ unsigned short
+// ARMEABISOFTFP_NOFP:#define __UINT32_C(c) c##U
 // ARMEABISOFTFP_NOFP:#define __UINT32_C_SUFFIX__ U
 // ARMEABISOFTFP_NOFP:#define __UINT32_MAX__ 4294967295U
 // ARMEABISOFTFP_NOFP:#define __UINT32_TYPE__ unsigned int
+// ARMEABISOFTFP_NOFP:#define __UINT64_C(c) c##ULL
 // ARMEABISOFTFP_NOFP:#define __UINT64_C_SUFFIX__ ULL
 // ARMEABISOFTFP_NOFP:#define __UINT64_MAX__ 18446744073709551615ULL
 // ARMEABISOFTFP_NOFP:#define __UINT64_TYPE__ long long unsigned int
+// ARMEABISOFTFP_NOFP:#define __UINT8_C(c) c
 // ARMEABISOFTFP_NOFP:#define __UINT8_C_SUFFIX__
 // ARMEABISOFTFP_NOFP:#define __UINT8_MAX__ 255
 // ARMEABISOFTFP_NOFP:#define __UINT8_TYPE__ unsigned char
+// ARMEABISOFTFP_NOFP:#define __UINTMAX_C(c) c##ULL
 // ARMEABISOFTFP_NOFP:#define __UINTMAX_C_SUFFIX__ ULL
 // ARMEABISOFTFP_NOFP:#define __UINTMAX_MAX__ 18446744073709551615ULL
 // ARMEABISOFTFP_NOFP:#define __UINTMAX_TYPE__ long long unsigned int
@@ -834,26 +874,31 @@
 // ARMEABISOFTFP_FP:#define __FLT_MIN_EXP__ (-125)
 // ARMEABISOFTFP_FP:#define __FLT_MIN__ 1.17549435e-38F
 // ARMEABISOFTFP_FP:#define __FLT_RADIX__ 2
+// ARMEABISOFTFP_FP:#define __INT16_C(c) c
 // ARMEABISOFTFP_FP:#define __INT16_C_SUFFIX__
 // ARMEABISOFTFP_FP:#define __INT16_FMTd__ "hd"
 // ARMEABISOFTFP_FP:#define __INT16_FMTi__ "hi"
 // ARMEABISOFTFP_FP:#define __INT16_MAX__ 32767
 // ARMEABISOFTFP_FP:#define __INT16_TYPE__ short
+// ARMEABISOFTFP_FP:#define __INT32_C(c) c
 // ARMEABISOFTFP_FP:#define __INT32_C_SUFFIX__
 // ARMEABISOFTFP_FP:#define __INT32_FMTd__ "d"
 // ARMEABISOFTFP_FP:#define __INT32_FMTi__ "i"
 // ARMEABISOFTFP_FP:#define __INT32_MAX__ 2147483647
 // ARMEABISOFTFP_FP:#define __INT32_TYPE__ int
+// ARMEABISOFTFP_FP:#define __INT64_C(c) c##LL
 // ARMEABISOFTFP_FP:#define __INT64_C_SUFFIX__ LL
 // ARMEABISOFTFP_FP:#define __INT64_FMTd__ "lld"
 // ARMEABISOFTFP_FP:#define __INT64_FMTi__ "lli"
 // ARMEABISOFTFP_FP:#define __INT64_MAX__ 9223372036854775807LL
 // ARMEABISOFTFP_FP:#define __INT64_TYPE__ long long int
+// ARMEABISOFTFP_FP:#define __INT8_C(c) c
 // ARMEABISOFTFP_FP:#define __INT8_C_SUFFIX__
 // ARMEABISOFTFP_FP:#define __INT8_FMTd__ "hhd"
 // ARMEABISOFTFP_FP:#define __INT8_FMTi__ "hhi"
 // ARMEABISOFTFP_FP:#define __INT8_MAX__ 127
 // ARMEABISOFTFP_FP:#define __INT8_TYPE__ signed char
+// ARMEABISOFTFP_FP:#define __INTMAX_C(c) c##LL
 // ARMEABISOFTFP_FP:#define __INTMAX_C_SUFFIX__ LL
 // ARMEABISOFTFP_FP:#define __INTMAX_FMTd__ "lld"
 // ARMEABISOFTFP_FP:#define __INTMAX_FMTi__ "lli"
@@ -939,18 +984,23 @@
 // ARMEABISOFTFP_FP:#define __SIZE_TYPE__ unsigned int
 // ARMEABISOFTFP_FP:#define __SIZE_WIDTH__ 32
 // ARMEABISOFTFP_FP-NOT:#define __SOFTFP__ 1
+// ARMEABISOFTFP_FP:#define __UINT16_C(c) c
 // ARMEABISOFTFP_FP:#define __UINT16_C_SUFFIX__
 // ARMEABISOFTFP_FP:#define __UINT16_MAX__ 65535
 // ARMEABISOFTFP_FP:#define __UINT16_TYPE__ unsigned short
+// ARMEABISOFTFP_FP:#define __UINT32_C(c) c##U
 // ARMEABISOFTFP_FP:#define __UINT32_C_SUFFIX__ U
 // ARMEABISOFTFP_FP:#define __UINT32_MAX__ 4294967295U
 // ARMEABISOFTFP_FP:#define __UINT32_TYPE__ unsigned int
+// ARMEABISOFTFP_FP:#define __UINT64_C(c) c##ULL
 // ARMEABISOFTFP_FP:#define __UINT64_C_SUFFIX__ ULL
 // ARMEABISOFTFP_FP:#define __UINT64_MAX__ 18446744073709551615ULL
 // ARMEABISOFTFP_FP:#define __UINT64_TYPE__ long long unsigned int
+// ARMEABISOFTFP_FP:#define __UINT8_C(c) c
 // ARMEABISOFTFP_FP:#define __UINT8_C_SUFFIX__
 // ARMEABISOFTFP_FP:#define __UINT8_MAX__ 255
 // ARMEABISOFTFP_FP:#define __UINT8_TYPE__ unsigned char
+// ARMEABISOFTFP_FP:#define __UINTMAX_C(c) c##ULL
 // ARMEABISOFTFP_FP:#define __UINTMAX_C_SUFFIX__ ULL
 // ARMEABISOFTFP_FP:#define __UINTMAX_MAX__ 18446744073709551615ULL
 // ARMEABISOFTFP_FP:#define __UINTMAX_TYPE__ long long unsigned int
@@ -1028,26 +1078,31 @@
 // ARMEABIHARDFP:#define __FLT_MIN_EXP__ (-125)
 // ARMEABIHARDFP:#define __FLT_MIN__ 1.17549435e-38F
 // ARMEABIHARDFP:#define __FLT_RADIX__ 2
+// ARMEABIHARDFP:#define __INT16_C(c) c
 // ARMEABIHARDFP:#define __INT16_C_SUFFIX__
 // ARMEABIHARDFP:#define __INT16_FMTd__ "hd"
 // ARMEABIHARDFP:#define __INT16_FMTi__ "hi"
 // ARMEABIHARDFP:#define __INT16_MAX__ 32767
 // ARMEABIHARDFP:#define __INT16_TYPE__ short
+// ARMEABIHARDFP:#define __INT32_C(c) c
 // ARMEABIHARDFP:#define __INT32_C_SUFFIX__
 // ARMEABIHARDFP:#define __INT32_FMTd__ "d"
 // ARMEABIHARDFP:#define __INT32_FMTi__ "i"
 // ARMEABIHARDFP:#define __INT32_MAX__ 2147483647
 // ARMEABIHARDFP:#define __INT32_TYPE__ int
+// ARMEABIHARDFP:#define __INT64_C(c) c##LL
 // ARMEABIHARDFP:#define __INT64_C_SUFFIX__ LL
 // ARMEABIHARDFP:#define __INT64_FMTd__ "lld"
 // ARMEABIHARDFP:#define __INT64_FMTi__ "lli"
 // ARMEABIHARDFP:#define __INT64_MAX__ 9223372036854775807LL
 // ARMEABIHARDFP:#define __INT64_TYPE__ long long int
+// ARMEABIHARDFP:#define __INT8_C(c) c
 // ARMEABIHARDFP:#define __INT8_C_SUFFIX__
 // ARMEABIHARDFP:#define __INT8_FMTd__ "hhd"
 // ARMEABIHARDFP:#define __INT8_FMTi__ "hhi"
 // ARMEABIHARDFP:#define __INT8_MAX__ 127
 // ARMEABIHARDFP:#define __INT8_TYPE__ signed char
+// ARMEABIHARDFP:#define __INTMAX_C(c) c##LL
 // ARMEABIHARDFP:#define __INTMAX_C_SUFFIX__ LL
 // ARMEABIHARDFP:#define __INTMAX_FMTd__ "lld"
 // ARMEABIHARDFP:#define __INTMAX_FMTi__ "lli"
@@ -1133,18 +1188,23 @@
 // ARMEABIHARDFP:#define __SIZE_TYPE__ unsigned int
 // ARMEABIHARDFP:#define __SIZE_WIDTH__ 32
 // ARMEABIHARDFP-NOT:#define __SOFTFP__ 1
+// ARMEABIHARDFP:#define __UINT16_C(c) c
 // ARMEABIHARDFP:#define __UINT16_C_SUFFIX__
 // ARMEABIHARDFP:#define __UINT16_MAX__ 65535
 // ARMEABIHARDFP:#define __UINT16_TYPE__ unsigned short
+// ARMEABIHARDFP:#define __UINT32_C(c) c##U
 // ARMEABIHARDFP:#define __UINT32_C_SUFFIX__ U
 // ARMEABIHARDFP:#define __UINT32_MAX__ 4294967295U
 // ARMEABIHARDFP:#define __UINT32_TYPE__ unsigned int
+// ARMEABIHARDFP:#define __UINT64_C(c) c##ULL
 // ARMEABIHARDFP:#define __UINT64_C_SUFFIX__ ULL
 // ARMEABIHARDFP:#define __UINT64_MAX__ 18446744073709551615ULL
 // ARMEABIHARDFP:#define __UINT64_TYPE__ long long unsigned int
+// ARMEABIHARDFP:#define __UINT8_C(c) c
 // ARMEABIHARDFP:#define __UINT8_C_SUFFIX__
 // ARMEABIHARDFP:#define __UINT8_MAX__ 255
 // ARMEABIHARDFP:#define __UINT8_TYPE__ unsigned char
+// ARMEABIHARDFP:#define __UINTMAX_C(c) c##ULL
 // ARMEABIHARDFP:#define __UINTMAX_C_SUFFIX__ ULL
 // ARMEABIHARDFP:#define __UINTMAX_MAX__ 18446744073709551615ULL
 // ARMEABIHARDFP:#define __UINTMAX_TYPE__ long long unsigned int
@@ -1220,26 +1280,31 @@
 // ARM-NETBSD:#define __FLT_MIN_EXP__ (-125)
 // ARM-NETBSD:#define __FLT_MIN__ 1.17549435e-38F
 // ARM-NETBSD:#define __FLT_RADIX__ 2
+// ARM-NETBSD:#define __INT16_C(c) c
 // ARM-NETBSD:#define __INT16_C_SUFFIX__
 // ARM-NETBSD:#define __INT16_FMTd__ "hd"
 // ARM-NETBSD:#define __INT16_FMTi__ "hi"
 // ARM-NETBSD:#define __INT16_MAX__ 32767
 // ARM-NETBSD:#define __INT16_TYPE__ short
+// ARM-NETBSD:#define __INT32_C(c) c
 // ARM-NETBSD:#define __INT32_C_SUFFIX__
 // ARM-NETBSD:#define __INT32_FMTd__ "d"
 // ARM-NETBSD:#define __INT32_FMTi__ "i"
 // ARM-NETBSD:#define __INT32_MAX__ 2147483647
 // ARM-NETBSD:#define __INT32_TYPE__ int
+// ARM-NETBSD:#define __INT64_C(c) c##LL
 // ARM-NETBSD:#define __INT64_C_SUFFIX__ LL
 // ARM-NETBSD:#define __INT64_FMTd__ "lld"
 // ARM-NETBSD:#define __INT64_FMTi__ "lli"
 // ARM-NETBSD:#define __INT64_MAX__ 9223372036854775807LL
 // ARM-NETBSD:#define __INT64_TYPE__ long long int
+// ARM-NETBSD:#define __INT8_C(c) c
 // ARM-NETBSD:#define __INT8_C_SUFFIX__
 // ARM-NETBSD:#define __INT8_FMTd__ "hhd"
 // ARM-NETBSD:#define __INT8_FMTi__ "hhi"
 // ARM-NETBSD:#define __INT8_MAX__ 127
 // ARM-NETBSD:#define __INT8_TYPE__ signed char
+// ARM-NETBSD:#define __INTMAX_C(c) c##LL
 // ARM-NETBSD:#define __INTMAX_C_SUFFIX__ LL
 // ARM-NETBSD:#define __INTMAX_FMTd__ "lld"
 // ARM-NETBSD:#define __INTMAX_FMTi__ "lli"
@@ -1325,18 +1390,23 @@
 // ARM-NETBSD:#define __SIZE_TYPE__ long unsigned int
 // ARM-NETBSD:#define __SIZE_WIDTH__ 32
 // ARM-NETBSD:#define __SOFTFP__ 1
+// ARM-NETBSD:#define __UINT16_C(c) c
 // ARM-NETBSD:#define __UINT16_C_SUFFIX__
 // ARM-NETBSD:#define __UINT16_MAX__ 65535
 // ARM-NETBSD:#define __UINT16_TYPE__ unsigned short
+// ARM-NETBSD:#define __UINT32_C(c) c##U
 // ARM-NETBSD:#define __UINT32_C_SUFFIX__ U
 // ARM-NETBSD:#define __UINT32_MAX__ 4294967295U
 // ARM-NETBSD:#define __UINT32_TYPE__ unsigned int
+// ARM-NETBSD:#define __UINT64_C(c) c##ULL
 // ARM-NETBSD:#define __UINT64_C_SUFFIX__ ULL
 // ARM-NETBSD:#define __UINT64_MAX__ 18446744073709551615ULL
 // ARM-NETBSD:#define __UINT64_TYPE__ long long unsigned int
+// ARM-NETBSD:#define __UINT8_C(c) c
 // ARM-NETBSD:#define __UINT8_C_SUFFIX__
 // ARM-NETBSD:#define __UINT8_MAX__ 255
 // ARM-NETBSD:#define __UINT8_TYPE__ unsigned char
+// ARM-NETBSD:#define __UINTMAX_C(c) c##ULL
 // ARM-NETBSD:#define __UINTMAX_C_SUFFIX__ ULL
 // ARM-NETBSD:#define __UINTMAX_MAX__ 18446744073709551615ULL
 // ARM-NETBSD:#define __UINTMAX_TYPE__ long long unsigned int
@@ -1372,6 +1442,7 @@
 // RUN: %clang -E -dM -ffreestanding -target arm-netbsd-eabihf %s -o - | FileCheck -match-full-lines -check-prefix ARMHF-NETBSD %s
 // ARMHF-NETBSD:#define __SIZE_WIDTH__ 32
 // ARMHF-NETBSD-NOT:#define __SOFTFP__ 1
+// ARMHF-NETBSD:#define __UINT16_C(c) c
 // ARMHF-NETBSD:#define __UINT16_C_SUFFIX__
 
 // RUN: %clang_cc1 -E -dM -ffreestanding -triple=arm-none-eabi < /dev/null | FileCheck -match-full-lines -check-prefix ARM-NONE-EABI %s
diff --git a/clang/test/Preprocessor/init-csky.c b/clang/test/Preprocessor/init-csky.c
index f7868e02644aa..99c5ad1010edb 100644
--- a/clang/test/Preprocessor/init-csky.c
+++ b/clang/test/Preprocessor/init-csky.c
@@ -66,18 +66,23 @@
 // CSKY: #define __GNUC__ {{.*}}
 // CSKY: #define __GXX_ABI_VERSION {{.*}}
 // CSKY: #define __ILP32__ 1
+// CSKY: #define __INT16_C(c) c
 // CSKY: #define __INT16_C_SUFFIX__
 // CSKY: #define __INT16_MAX__ 32767
 // CSKY: #define __INT16_TYPE__ short
+// CSKY: #define __INT32_C(c) c
 // CSKY: #define __INT32_C_SUFFIX__
 // CSKY: #define __INT32_MAX__ 2147483647
 // CSKY: #define __INT32_TYPE__ int
+// CSKY: #define __INT64_C(c) c##LL
 // CSKY: #define __INT64_C_SUFFIX__ LL
 // CSKY: #define __INT64_MAX__ 9223372036854775807LL
 // CSKY: #define __INT64_TYPE__ long long int
+// CSKY: #define __INT8_C(c) c
 // CSKY: #define __INT8_C_SUFFIX__
 // CSKY: #define __INT8_MAX__ 127
 // CSKY: #define __INT8_TYPE__ signed char
+// CSKY: #define __INTMAX_C(c) c##LL
 // CSKY: #define __INTMAX_C_SUFFIX__ LL
 // CSKY: #define __INTMAX_MAX__ 9223372036854775807LL
 // CSKY: #define __INTMAX_TYPE__ long long int
@@ -152,18 +157,23 @@
 // CSKY: #define __STDC_UTF_32__ 1
 // CSKY: #define __STDC_VERSION__ 201710L
 // CSKY: #define __STDC__ 1
+// CSKY: #define __UINT16_C(c) c
 // CSKY: #define __UINT16_C_SUFFIX__
 // CSKY: #define __UINT16_MAX__ 65535
 // CSKY: #define __UINT16_TYPE__ unsigned short
+// CSKY: #define __UINT32_C(c) c##U
 // CSKY: #define __UINT32_C_SUFFIX__ U
 // CSKY: #define __UINT32_MAX__ 4294967295U
 // CSKY: #define __UINT32_TYPE__ unsigned int
+// CSKY: #define __UINT64_C(c) c##ULL
 // CSKY: #define __UINT64_C_SUFFIX__ ULL
 // CSKY: #define __UINT64_MAX__ 18446744073709551615ULL
 // CSKY: #define __UINT64_TYPE__ long long unsigned int
+// CSKY: #define __UINT8_C(c) c
 // CSKY: #define __UINT8_C_SUFFIX__
 // CSKY: #define __UINT8_MAX__ 255
 // CSKY: #define __UINT8_TYPE__ unsigned char
+// CSKY: #define __UINTMAX_C(c) c##ULL
 // CSKY: #define __UINTMAX_C_SUFFIX__ ULL
 // CSKY: #define __UINTMAX_MAX__ 18446744073709551615ULL
 // CSKY: #define __UINTMAX_TYPE__ long long unsigned int
diff --git a/clang/test/Preprocessor/init-loongarch.c b/clang/test/Preprocessor/init-loongarch.c
index f6fd603dc39c0..ac461b371162f 100644
--- a/clang/test/Preprocessor/init-loongarch.c
+++ b/clang/test/Preprocessor/init-loongarch.c
@@ -85,26 +85,31 @@
 // LA32: #define __GCC_HAVE_SYNC_COMPARE_AND_SWAP_2 1
 // LA32: #define __GCC_HAVE_SYNC_COMPARE_AND_SWAP_4 1
 // LA32: #define __ILP32__ 1
+// LA32: #define __INT16_C(c) c
 // LA32: #define __INT16_C_SUFFIX__
 // LA32: #define __INT16_FMTd__ "hd"
 // LA32: #define __INT16_FMTi__ "hi"
 // LA32: #define __INT16_MAX__ 32767
 // LA32: #define __INT16_TYPE__ short
+// LA32: #define __INT32_C(c) c
 // LA32: #define __INT32_C_SUFFIX__
 // LA32: #define __INT32_FMTd__ "d"
 // LA32: #define __INT32_FMTi__ "i"
 // LA32: #define __INT32_MAX__ 2147483647
 // LA32: #define __INT32_TYPE__ int
+// LA32: #define __INT64_C(c) c##LL
 // LA32: #define __INT64_C_SUFFIX__ LL
 // LA32: #define __INT64_FMTd__ "lld"
 // LA32: #define __INT64_FMTi__ "lli"
 // LA32: #define __INT64_MAX__ 9223372036854775807LL
 // LA32: #define __INT64_TYPE__ long long int
+// LA32: #define __INT8_C(c) c
 // LA32: #define __INT8_C_SUFFIX__
 // LA32: #define __INT8_FMTd__ "hhd"
 // LA32: #define __INT8_FMTi__ "hhi"
 // LA32: #define __INT8_MAX__ 127
 // LA32: #define __INT8_TYPE__ signed char
+// LA32: #define __INTMAX_C(c) c##LL
 // LA32: #define __INTMAX_C_SUFFIX__ LL
 // LA32: #define __INTMAX_FMTd__ "lld"
 // LA32: #define __INTMAX_FMTi__ "lli"
@@ -227,6 +232,7 @@
 // LA32: #define __STDC_UTF_32__ 1
 // LA32: #define __STDC_VERSION__ 201710L
 // LA32: #define __STDC__ 1
+// LA32: #define __UINT16_C(c) c
 // LA32: #define __UINT16_C_SUFFIX__
 // LA32: #define __UINT16_FMTX__ "hX"
 // LA32: #define __UINT16_FMTo__ "ho"
@@ -234,6 +240,7 @@
 // LA32: #define __UINT16_FMTx__ "hx"
 // LA32: #define __UINT16_MAX__ 65535
 // LA32: #define __UINT16_TYPE__ unsigned short
+// LA32: #define __UINT32_C(c) c##U
 // LA32: #define __UINT32_C_SUFFIX__ U
 // LA32: #define __UINT32_FMTX__ "X"
 // LA32: #define __UINT32_FMTo__ "o"
@@ -241,6 +248,7 @@
 // LA32: #define __UINT32_FMTx__ "x"
 // LA32: #define __UINT32_MAX__ 4294967295U
 // LA32: #define __UINT32_TYPE__ unsigned int
+// LA32: #define __UINT64_C(c) c##ULL
 // LA32: #define __UINT64_C_SUFFIX__ ULL
 // LA32: #define __UINT64_FMTX__ "llX"
 // LA32: #define __UINT64_FMTo__ "llo"
@@ -248,6 +256,7 @@
 // LA32: #define __UINT64_FMTx__ "llx"
 // LA32: #define __UINT64_MAX__ 18446744073709551615ULL
 // LA32: #define __UINT64_TYPE__ long long unsigned int
+// LA32: #define __UINT8_C(c) c
 // LA32: #define __UINT8_C_SUFFIX__
 // LA32: #define __UINT8_FMTX__ "hhX"
 // LA32: #define __UINT8_FMTo__ "hho"
@@ -255,6 +264,7 @@
 // LA32: #define __UINT8_FMTx__ "hhx"
 // LA32: #define __UINT8_MAX__ 255
 // LA32: #define __UINT8_TYPE__ unsigned char
+// LA32: #define __UINTMAX_C(c) c##ULL
 // LA32: #define __UINTMAX_C_SUFFIX__ ULL
 // LA32: #define __UINTMAX_FMTX__ "llX"
 // LA32: #define __UINTMAX_FMTo__ "llo"
@@ -406,26 +416,31 @@
 // LA64: #define __GCC_HAVE_SYNC_COMPARE_AND_SWAP_2 1
 // LA64: #define __GCC_HAVE_SYNC_COMPARE_AND_SWAP_4 1
 // LA64: #define __GCC_HAVE_SYNC_COMPARE_AND_SWAP_8 1
+// LA64: #define __INT16_C(c) c
 // LA64: #define __INT16_C_SUFFIX__
 // LA64: #define __INT16_FMTd__ "hd"
 // LA64: #define __INT16_FMTi__ "hi"
 // LA64: #define __INT16_MAX__ 32767
 // LA64: #define __INT16_TYPE__ short
+// LA64: #define __INT32_C(c) c
 // LA64: #define __INT32_C_SUFFIX__
 // LA64: #define __INT32_FMTd__ "d"
 // LA64: #define __INT32_FMTi__ "i"
 // LA64: #define __INT32_MAX__ 2147483647
 // LA64: #define __INT32_TYPE__ int
+// LA64: #define __INT64_C(c) c##L
 // LA64: #define __INT64_C_SUFFIX__ L
 // LA64: #define __INT64_FMTd__ "ld"
 // LA64: #define __INT64_FMTi__ "li"
 // LA64: #define __INT64_MAX__ 9223372036854775807L
 // LA64: #define __INT64_TYPE__ long int
+// LA64: #define __INT8_C(c) c
 // LA64: #define __INT8_C_SUFFIX__
 // LA64: #define __INT8_FMTd__ "hhd"
 // LA64: #define __INT8_FMTi__ "hhi"
 // LA64: #define __INT8_MAX__ 127
 // LA64: #define __INT8_TYPE__ signed char
+// LA64: #define __INTMAX_C(c) c##L
 // LA64: #define __INTMAX_C_SUFFIX__ L
 // LA64: #define __INTMAX_FMTd__ "ld"
 // LA64: #define __INTMAX_FMTi__ "li"
@@ -549,6 +564,7 @@
 // LA64: #define __STDC_UTF_32__ 1
 // LA64: #define __STDC_VERSION__ 201710L
 // LA64: #define __STDC__ 1
+// LA64: #define __UINT16_C(c) c
 // LA64: #define __UINT16_C_SUFFIX__
 // LA64: #define __UINT16_FMTX__ "hX"
 // LA64: #define __UINT16_FMTo__ "ho"
@@ -556,6 +572,7 @@
 // LA64: #define __UINT16_FMTx__ "hx"
 // LA64: #define __UINT16_MAX__ 65535
 // LA64: #define __UINT16_TYPE__ unsigned short
+// LA64: #define __UINT32_C(c) c##U
 // LA64: #define __UINT32_C_SUFFIX__ U
 // LA64: #define __UINT32_FMTX__ "X"
 // LA64: #define __UINT32_FMTo__ "o"
@@ -563,6 +580,7 @@
 // LA64: #define __UINT32_FMTx__ "x"
 // LA64: #define __UINT32_MAX__ 4294967295U
 // LA64: #define __UINT32_TYPE__ unsigned int
+// LA64: #define __UINT64_C(c) c##UL
 // LA64: #define __UINT64_C_SUFFIX__ UL
 // LA64: #define __UINT64_FMTX__ "lX"
 // LA64: #define __UINT64_FMTo__ "lo"
@@ -570,6 +588,7 @@
 // LA64: #define __UINT64_FMTx__ "lx"
 // LA64: #define __UINT64_MAX__ 18446744073709551615UL
 // LA64: #define __UINT64_TYPE__ long unsigned int
+// LA64: #define __UINT8_C(c) c
 // LA64: #define __UINT8_C_SUFFIX__
 // LA64: #define __UINT8_FMTX__ "hhX"
 // LA64: #define __UINT8_FMTo__ "hho"
@@ -577,6 +596,7 @@
 // LA64: #define __UINT8_FMTx__ "hhx"
 // LA64: #define __UINT8_MAX__ 255
 // LA64: #define __UINT8_TYPE__ unsigned char
+// LA64: #define __UINTMAX_C(c) c##UL
 // LA64: #define __UINTMAX_C_SUFFIX__ UL
 // LA64: #define __UINTMAX_FMTX__ "lX"
 // LA64: #define __UINTMAX_FMTo__ "lo"
diff --git a/clang/test/Preprocessor/init-mips.c b/clang/test/Preprocessor/init-mips.c
index 34091ea3690da..4fead33bd826e 100644
--- a/clang/test/Preprocessor/init-mips.c
+++ b/clang/test/Preprocessor/init-mips.c
@@ -49,26 +49,31 @@
 // MIPS32BE:#define __FLT_MIN_EXP__ (-125)
 // MIPS32BE:#define __FLT_MIN__ 1.17549435e-38F
 // MIPS32BE:#define __FLT_RADIX__ 2
+// MIPS32BE:#define __INT16_C(c) c
 // MIPS32BE:#define __INT16_C_SUFFIX__
 // MIPS32BE:#define __INT16_FMTd__ "hd"
 // MIPS32BE:#define __INT16_FMTi__ "hi"
 // MIPS32BE:#define __INT16_MAX__ 32767
 // MIPS32BE:#define __INT16_TYPE__ short
+// MIPS32BE:#define __INT32_C(c) c
 // MIPS32BE:#define __INT32_C_SUFFIX__
 // MIPS32BE:#define __INT32_FMTd__ "d"
 // MIPS32BE:#define __INT32_FMTi__ "i"
 // MIPS32BE:#define __INT32_MAX__ 2147483647
 // MIPS32BE:#define __INT32_TYPE__ int
+// MIPS32BE:#define __INT64_C(c) c##LL
 // MIPS32BE:#define __INT64_C_SUFFIX__ LL
 // MIPS32BE:#define __INT64_FMTd__ "lld"
 // MIPS32BE:#define __INT64_FMTi__ "lli"
 // MIPS32BE:#define __INT64_MAX__ 9223372036854775807LL
 // MIPS32BE:#define __INT64_TYPE__ long long int
+// MIPS32BE:#define __INT8_C(c) c
 // MIPS32BE:#define __INT8_C_SUFFIX__
 // MIPS32BE:#define __INT8_FMTd__ "hhd"
 // MIPS32BE:#define __INT8_FMTi__ "hhi"
 // MIPS32BE:#define __INT8_MAX__ 127
 // MIPS32BE:#define __INT8_TYPE__ signed char
+// MIPS32BE:#define __INTMAX_C(c) c##LL
 // MIPS32BE:#define __INTMAX_C_SUFFIX__ LL
 // MIPS32BE:#define __INTMAX_FMTd__ "lld"
 // MIPS32BE:#define __INTMAX_FMTi__ "lli"
@@ -159,18 +164,23 @@
 // MIPS32BE:#define __STDC_HOSTED__ 0
 // MIPS32BE-C:#define __STDC_VERSION__ 201710L
 // MIPS32BE:#define __STDC__ 1
+// MIPS32BE:#define __UINT16_C(c) c
 // MIPS32BE:#define __UINT16_C_SUFFIX__
 // MIPS32BE:#define __UINT16_MAX__ 65535
 // MIPS32BE:#define __UINT16_TYPE__ unsigned short
+// MIPS32BE:#define __UINT32_C(c) c##U
 // MIPS32BE:#define __UINT32_C_SUFFIX__ U
 // MIPS32BE:#define __UINT32_MAX__ 4294967295U
 // MIPS32BE:#define __UINT32_TYPE__ unsigned int
+// MIPS32BE:#define __UINT64_C(c) c##ULL
 // MIPS32BE:#define __UINT64_C_SUFFIX__ ULL
 // MIPS32BE:#define __UINT64_MAX__ 18446744073709551615ULL
 // MIPS32BE:#define __UINT64_TYPE__ long long unsigned int
+// MIPS32BE:#define __UINT8_C(c) c
 // MIPS32BE:#define __UINT8_C_SUFFIX__
 // MIPS32BE:#define __UINT8_MAX__ 255
 // MIPS32BE:#define __UINT8_TYPE__ unsigned char
+// MIPS32BE:#define __UINTMAX_C(c) c##ULL
 // MIPS32BE:#define __UINTMAX_C_SUFFIX__ ULL
 // MIPS32BE:#define __UINTMAX_MAX__ 18446744073709551615ULL
 // MIPS32BE:#define __UINTMAX_TYPE__ long long unsigned int
@@ -259,26 +269,31 @@
 // MIPS32EL:#define __FLT_MIN_EXP__ (-125)
 // MIPS32EL:#define __FLT_MIN__ 1.17549435e-38F
 // MIPS32EL:#define __FLT_RADIX__ 2
+// MIPS32EL:#define __INT16_C(c) c
 // MIPS32EL:#define __INT16_C_SUFFIX__
 // MIPS32EL:#define __INT16_FMTd__ "hd"
 // MIPS32EL:#define __INT16_FMTi__ "hi"
 // MIPS32EL:#define __INT16_MAX__ 32767
 // MIPS32EL:#define __INT16_TYPE__ short
+// MIPS32EL:#define __INT32_C(c) c
 // MIPS32EL:#define __INT32_C_SUFFIX__
 // MIPS32EL:#define __INT32_FMTd__ "d"
 // MIPS32EL:#define __INT32_FMTi__ "i"
 // MIPS32EL:#define __INT32_MAX__ 2147483647
 // MIPS32EL:#define __INT32_TYPE__ int
+// MIPS32EL:#define __INT64_C(c) c##LL
 // MIPS32EL:#define __INT64_C_SUFFIX__ LL
 // MIPS32EL:#define __INT64_FMTd__ "lld"
 // MIPS32EL:#define __INT64_FMTi__ "lli"
 // MIPS32EL:#define __INT64_MAX__ 9223372036854775807LL
 // MIPS32EL:#define __INT64_TYPE__ long long int
+// MIPS32EL:#define __INT8_C(c) c
 // MIPS32EL:#define __INT8_C_SUFFIX__
 // MIPS32EL:#define __INT8_FMTd__ "hhd"
 // MIPS32EL:#define __INT8_FMTi__ "hhi"
 // MIPS32EL:#define __INT8_MAX__ 127
 // MIPS32EL:#define __INT8_TYPE__ signed char
+// MIPS32EL:#define __INTMAX_C(c) c##LL
 // MIPS32EL:#define __INTMAX_C_SUFFIX__ LL
 // MIPS32EL:#define __INTMAX_FMTd__ "lld"
 // MIPS32EL:#define __INTMAX_FMTi__ "lli"
@@ -366,18 +381,23 @@
 // MIPS32EL:#define __SIZE_MAX__ 4294967295U
 // MIPS32EL:#define __SIZE_TYPE__ unsigned int
 // MIPS32EL:#define __SIZE_WIDTH__ 32
+// MIPS32EL:#define __UINT16_C(c) c
 // MIPS32EL:#define __UINT16_C_SUFFIX__
 // MIPS32EL:#define __UINT16_MAX__ 65535
 // MIPS32EL:#define __UINT16_TYPE__ unsigned short
+// MIPS32EL:#define __UINT32_C(c) c##U
 // MIPS32EL:#define __UINT32_C_SUFFIX__ U
 // MIPS32EL:#define __UINT32_MAX__ 4294967295U
 // MIPS32EL:#define __UINT32_TYPE__ unsigned int
+// MIPS32EL:#define __UINT64_C(c) c##ULL
 // MIPS32EL:#define __UINT64_C_SUFFIX__ ULL
 // MIPS32EL:#define __UINT64_MAX__ 18446744073709551615ULL
 // MIPS32EL:#define __UINT64_TYPE__ long long unsigned int
+// MIPS32EL:#define __UINT8_C(c) c
 // MIPS32EL:#define __UINT8_C_SUFFIX__
 // MIPS32EL:#define __UINT8_MAX__ 255
 // MIPS32EL:#define __UINT8_TYPE__ unsigned char
+// MIPS32EL:#define __UINTMAX_C(c) c##ULL
 // MIPS32EL:#define __UINTMAX_C_SUFFIX__ ULL
 // MIPS32EL:#define __UINTMAX_MAX__ 18446744073709551615ULL
 // MIPS32EL:#define __UINTMAX_TYPE__ long long unsigned int
@@ -496,26 +516,31 @@
 // MIPSN32BE: #define __GNUC__ 4
 // MIPSN32BE: #define __GXX_ABI_VERSION 1002
 // MIPSN32BE: #define __ILP32__ 1
+// MIPSN32BE: #define __INT16_C(c) c
 // MIPSN32BE: #define __INT16_C_SUFFIX__
 // MIPSN32BE: #define __INT16_FMTd__ "hd"
 // MIPSN32BE: #define __INT16_FMTi__ "hi"
 // MIPSN32BE: #define __INT16_MAX__ 32767
 // MIPSN32BE: #define __INT16_TYPE__ short
+// MIPSN32BE: #define __INT32_C(c) c
 // MIPSN32BE: #define __INT32_C_SUFFIX__
 // MIPSN32BE: #define __INT32_FMTd__ "d"
 // MIPSN32BE: #define __INT32_FMTi__ "i"
 // MIPSN32BE: #define __INT32_MAX__ 2147483647
 // MIPSN32BE: #define __INT32_TYPE__ int
+// MIPSN32BE: #define __INT64_C(c) c##LL
 // MIPSN32BE: #define __INT64_C_SUFFIX__ LL
 // MIPSN32BE: #define __INT64_FMTd__ "lld"
 // MIPSN32BE: #define __INT64_FMTi__ "lli"
 // MIPSN32BE: #define __INT64_MAX__ 9223372036854775807LL
 // MIPSN32BE: #define __INT64_TYPE__ long long int
+// MIPSN32BE: #define __INT8_C(c) c
 // MIPSN32BE: #define __INT8_C_SUFFIX__
 // MIPSN32BE: #define __INT8_FMTd__ "hhd"
 // MIPSN32BE: #define __INT8_FMTi__ "hhi"
 // MIPSN32BE: #define __INT8_MAX__ 127
 // MIPSN32BE: #define __INT8_TYPE__ signed char
+// MIPSN32BE: #define __INTMAX_C(c) c##LL
 // MIPSN32BE: #define __INTMAX_C_SUFFIX__ LL
 // MIPSN32BE: #define __INTMAX_FMTd__ "lld"
 // MIPSN32BE: #define __INTMAX_FMTi__ "lli"
@@ -618,6 +643,7 @@
 // MIPSN32BE: #define __STDC_UTF_32__ 1
 // MIPSN32BE-C: #define __STDC_VERSION__ 201710L
 // MIPSN32BE: #define __STDC__ 1
+// MIPSN32BE: #define __UINT16_C(c) c
 // MIPSN32BE: #define __UINT16_C_SUFFIX__
 // MIPSN32BE: #define __UINT16_FMTX__ "hX"
 // MIPSN32BE: #define __UINT16_FMTo__ "ho"
@@ -625,6 +651,7 @@
 // MIPSN32BE: #define __UINT16_FMTx__ "hx"
 // MIPSN32BE: #define __UINT16_MAX__ 65535
 // MIPSN32BE: #define __UINT16_TYPE__ unsigned short
+// MIPSN32BE: #define __UINT32_C(c) c##U
 // MIPSN32BE: #define __UINT32_C_SUFFIX__ U
 // MIPSN32BE: #define __UINT32_FMTX__ "X"
 // MIPSN32BE: #define __UINT32_FMTo__ "o"
@@ -632,6 +659,7 @@
 // MIPSN32BE: #define __UINT32_FMTx__ "x"
 // MIPSN32BE: #define __UINT32_MAX__ 4294967295U
 // MIPSN32BE: #define __UINT32_TYPE__ unsigned int
+// MIPSN32BE: #define __UINT64_C(c) c##ULL
 // MIPSN32BE: #define __UINT64_C_SUFFIX__ ULL
 // MIPSN32BE: #define __UINT64_FMTX__ "llX"
 // MIPSN32BE: #define __UINT64_FMTo__ "llo"
@@ -639,6 +667,7 @@
 // MIPSN32BE: #define __UINT64_FMTx__ "llx"
 // MIPSN32BE: #define __UINT64_MAX__ 18446744073709551615ULL
 // MIPSN32BE: #define __UINT64_TYPE__ long long unsigned int
+// MIPSN32BE: #define __UINT8_C(c) c
 // MIPSN32BE: #define __UINT8_C_SUFFIX__
 // MIPSN32BE: #define __UINT8_FMTX__ "hhX"
 // MIPSN32BE: #define __UINT8_FMTo__ "hho"
@@ -646,6 +675,7 @@
 // MIPSN32BE: #define __UINT8_FMTx__ "hhx"
 // MIPSN32BE: #define __UINT8_MAX__ 255
 // MIPSN32BE: #define __UINT8_TYPE__ unsigned char
+// MIPSN32BE: #define __UINTMAX_C(c) c##ULL
 // MIPSN32BE: #define __UINTMAX_C_SUFFIX__ ULL
 // MIPSN32BE: #define __UINTMAX_FMTX__ "llX"
 // MIPSN32BE: #define __UINTMAX_FMTo__ "llo"
@@ -803,26 +833,31 @@
 // MIPSN32EL: #define __GNUC__ 4
 // MIPSN32EL: #define __GXX_ABI_VERSION 1002
 // MIPSN32EL: #define __ILP32__ 1
+// MIPSN32EL: #define __INT16_C(c) c
 // MIPSN32EL: #define __INT16_C_SUFFIX__
 // MIPSN32EL: #define __INT16_FMTd__ "hd"
 // MIPSN32EL: #define __INT16_FMTi__ "hi"
 // MIPSN32EL: #define __INT16_MAX__ 32767
 // MIPSN32EL: #define __INT16_TYPE__ short
+// MIPSN32EL: #define __INT32_C(c) c
 // MIPSN32EL: #define __INT32_C_SUFFIX__
 // MIPSN32EL: #define __INT32_FMTd__ "d"
 // MIPSN32EL: #define __INT32_FMTi__ "i"
 // MIPSN32EL: #define __INT32_MAX__ 2147483647
 // MIPSN32EL: #define __INT32_TYPE__ int
+// MIPSN32EL: #define __INT64_C(c) c##LL
 // MIPSN32EL: #define __INT64_C_SUFFIX__ LL
 // MIPSN32EL: #define __INT64_FMTd__ "lld"
 // MIPSN32EL: #define __INT64_FMTi__ "lli"
 // MIPSN32EL: #define __INT64_MAX__ 9223372036854775807LL
 // MIPSN32EL: #define __INT64_TYPE__ long long int
+// MIPSN32EL: #define __INT8_C(c) c
 // MIPSN32EL: #define __INT8_C_SUFFIX__
 // MIPSN32EL: #define __INT8_FMTd__ "hhd"
 // MIPSN32EL: #define __INT8_FMTi__ "hhi"
 // MIPSN32EL: #define __INT8_MAX__ 127
 // MIPSN32EL: #define __INT8_TYPE__ signed char
+// MIPSN32EL: #define __INTMAX_C(c) c##LL
 // MIPSN32EL: #define __INTMAX_C_SUFFIX__ LL
 // MIPSN32EL: #define __INTMAX_FMTd__ "lld"
 // MIPSN32EL: #define __INTMAX_FMTi__ "lli"
@@ -925,6 +960,7 @@
 // MIPSN32EL: #define __STDC_UTF_32__ 1
 // MIPSN32EL: #define __STDC_VERSION__ 201710L
 // MIPSN32EL: #define __STDC__ 1
+// MIPSN32EL: #define __UINT16_C(c) c
 // MIPSN32EL: #define __UINT16_C_SUFFIX__
 // MIPSN32EL: #define __UINT16_FMTX__ "hX"
 // MIPSN32EL: #define __UINT16_FMTo__ "ho"
@@ -932,6 +968,7 @@
 // MIPSN32EL: #define __UINT16_FMTx__ "hx"
 // MIPSN32EL: #define __UINT16_MAX__ 65535
 // MIPSN32EL: #define __UINT16_TYPE__ unsigned short
+// MIPSN32EL: #define __UINT32_C(c) c##U
 // MIPSN32EL: #define __UINT32_C_SUFFIX__ U
 // MIPSN32EL: #define __UINT32_FMTX__ "X"
 // MIPSN32EL: #define __UINT32_FMTo__ "o"
@@ -939,6 +976,7 @@
 // MIPSN32EL: #define __UINT32_FMTx__ "x"
 // MIPSN32EL: #define __UINT32_MAX__ 4294967295U
 // MIPSN32EL: #define __UINT32_TYPE__ unsigned int
+// MIPSN32EL: #define __UINT64_C(c) c##ULL
 // MIPSN32EL: #define __UINT64_C_SUFFIX__ ULL
 // MIPSN32EL: #define __UINT64_FMTX__ "llX"
 // MIPSN32EL: #define __UINT64_FMTo__ "llo"
@@ -946,6 +984,7 @@
 // MIPSN32EL: #define __UINT64_FMTx__ "llx"
 // MIPSN32EL: #define __UINT64_MAX__ 18446744073709551615ULL
 // MIPSN32EL: #define __UINT64_TYPE__ long long unsigned int
+// MIPSN32EL: #define __UINT8_C(c) c
 // MIPSN32EL: #define __UINT8_C_SUFFIX__
 // MIPSN32EL: #define __UINT8_FMTX__ "hhX"
 // MIPSN32EL: #define __UINT8_FMTo__ "hho"
@@ -953,6 +992,7 @@
 // MIPSN32EL: #define __UINT8_FMTx__ "hhx"
 // MIPSN32EL: #define __UINT8_MAX__ 255
 // MIPSN32EL: #define __UINT8_TYPE__ unsigned char
+// MIPSN32EL: #define __UINTMAX_C(c) c##ULL
 // MIPSN32EL: #define __UINTMAX_C_SUFFIX__ ULL
 // MIPSN32EL: #define __UINTMAX_FMTX__ "llX"
 // MIPSN32EL: #define __UINTMAX_FMTo__ "llo"
@@ -1086,26 +1126,31 @@
 // MIPS64BE:#define __FLT_MIN_EXP__ (-125)
 // MIPS64BE:#define __FLT_MIN__ 1.17549435e-38F
 // MIPS64BE:#define __FLT_RADIX__ 2
+// MIPS64BE:#define __INT16_C(c) c
 // MIPS64BE:#define __INT16_C_SUFFIX__
 // MIPS64BE:#define __INT16_FMTd__ "hd"
 // MIPS64BE:#define __INT16_FMTi__ "hi"
 // MIPS64BE:#define __INT16_MAX__ 32767
 // MIPS64BE:#define __INT16_TYPE__ short
+// MIPS64BE:#define __INT32_C(c) c
 // MIPS64BE:#define __INT32_C_SUFFIX__
 // MIPS64BE:#define __INT32_FMTd__ "d"
 // MIPS64BE:#define __INT32_FMTi__ "i"
 // MIPS64BE:#define __INT32_MAX__ 2147483647
 // MIPS64BE:#define __INT32_TYPE__ int
+// MIPS64BE:#define __INT64_C(c) c##L
 // MIPS64BE:#define __INT64_C_SUFFIX__ L
 // MIPS64BE:#define __INT64_FMTd__ "ld"
 // MIPS64BE:#define __INT64_FMTi__ "li"
 // MIPS64BE:#define __INT64_MAX__ 9223372036854775807L
 // MIPS64BE:#define __INT64_TYPE__ long int
+// MIPS64BE:#define __INT8_C(c) c
 // MIPS64BE:#define __INT8_C_SUFFIX__
 // MIPS64BE:#define __INT8_FMTd__ "hhd"
 // MIPS64BE:#define __INT8_FMTi__ "hhi"
 // MIPS64BE:#define __INT8_MAX__ 127
 // MIPS64BE:#define __INT8_TYPE__ signed char
+// MIPS64BE:#define __INTMAX_C(c) c##L
 // MIPS64BE:#define __INTMAX_C_SUFFIX__ L
 // MIPS64BE:#define __INTMAX_FMTd__ "ld"
 // MIPS64BE:#define __INTMAX_FMTi__ "li"
@@ -1194,18 +1239,23 @@
 // MIPS64BE:#define __SIZE_TYPE__ long unsigned int
 // MIPS64BE:#define __SIZE_WIDTH__ 64
 // MIPS64BE-CXX:#define __STDCPP_DEFAULT_NEW_ALIGNMENT__ 16UL
+// MIPS64BE:#define __UINT16_C(c) c
 // MIPS64BE:#define __UINT16_C_SUFFIX__
 // MIPS64BE:#define __UINT16_MAX__ 65535
 // MIPS64BE:#define __UINT16_TYPE__ unsigned short
+// MIPS64BE:#define __UINT32_C(c) c##U
 // MIPS64BE:#define __UINT32_C_SUFFIX__ U
 // MIPS64BE:#define __UINT32_MAX__ 4294967295U
 // MIPS64BE:#define __UINT32_TYPE__ unsigned int
+// MIPS64BE:#define __UINT64_C(c) c##UL
 // MIPS64BE:#define __UINT64_C_SUFFIX__ UL
 // MIPS64BE:#define __UINT64_MAX__ 18446744073709551615UL
 // MIPS64BE:#define __UINT64_TYPE__ long unsigned int
+// MIPS64BE:#define __UINT8_C(c) c
 // MIPS64BE:#define __UINT8_C_SUFFIX__
 // MIPS64BE:#define __UINT8_MAX__ 255
 // MIPS64BE:#define __UINT8_TYPE__ unsigned char
+// MIPS64BE:#define __UINTMAX_C(c) c##UL
 // MIPS64BE:#define __UINTMAX_C_SUFFIX__ UL
 // MIPS64BE:#define __UINTMAX_MAX__ 18446744073709551615UL
 // MIPS64BE:#define __UINTMAX_TYPE__ long unsigned int
@@ -1296,26 +1346,31 @@
 // MIPS64EL:#define __FLT_MIN_EXP__ (-125)
 // MIPS64EL:#define __FLT_MIN__ 1.17549435e-38F
 // MIPS64EL:#define __FLT_RADIX__ 2
+// MIPS64EL:#define __INT16_C(c) c
 // MIPS64EL:#define __INT16_C_SUFFIX__
 // MIPS64EL:#define __INT16_FMTd__ "hd"
 // MIPS64EL:#define __INT16_FMTi__ "hi"
 // MIPS64EL:#define __INT16_MAX__ 32767
 // MIPS64EL:#define __INT16_TYPE__ short
+// MIPS64EL:#define __INT32_C(c) c
 // MIPS64EL:#define __INT32_C_SUFFIX__
 // MIPS64EL:#define __INT32_FMTd__ "d"
 // MIPS64EL:#define __INT32_FMTi__ "i"
 // MIPS64EL:#define __INT32_MAX__ 2147483647
 // MIPS64EL:#define __INT32_TYPE__ int
+// MIPS64EL:#define __INT64_C(c) c##L
 // MIPS64EL:#define __INT64_C_SUFFIX__ L
 // MIPS64EL:#define __INT64_FMTd__ "ld"
 // MIPS64EL:#define __INT64_FMTi__ "li"
 // MIPS64EL:#define __INT64_MAX__ 9223372036854775807L
 // MIPS64EL:#define __INT64_TYPE__ long int
+// MIPS64EL:#define __INT8_C(c) c
 // MIPS64EL:#define __INT8_C_SUFFIX__
 // MIPS64EL:#define __INT8_FMTd__ "hhd"
 // MIPS64EL:#define __INT8_FMTi__ "hhi"
 // MIPS64EL:#define __INT8_MAX__ 127
 // MIPS64EL:#define __INT8_TYPE__ signed char
+// MIPS64EL:#define __INTMAX_C(c) c##L
 // MIPS64EL:#define __INTMAX_C_SUFFIX__ L
 // MIPS64EL:#define __INTMAX_FMTd__ "ld"
 // MIPS64EL:#define __INTMAX_FMTi__ "li"
@@ -1404,18 +1459,23 @@
 // MIPS64EL:#define __SIZE_MAX__ 18446744073709551615UL
 // MIPS64EL:#define __SIZE_TYPE__ long unsigned int
 // MIPS64EL:#define __SIZE_WIDTH__ 64
+// MIPS64EL:#define __UINT16_C(c) c
 // MIPS64EL:#define __UINT16_C_SUFFIX__
 // MIPS64EL:#define __UINT16_MAX__ 65535
 // MIPS64EL:#define __UINT16_TYPE__ unsigned short
+// MIPS64EL:#define __UINT32_C(c) c##U
 // MIPS64EL:#define __UINT32_C_SUFFIX__ U
 // MIPS64EL:#define __UINT32_MAX__ 4294967295U
 // MIPS64EL:#define __UINT32_TYPE__ unsigned int
+// MIPS64EL:#define __UINT64_C(c) c##UL
 // MIPS64EL:#define __UINT64_C_SUFFIX__ UL
 // MIPS64EL:#define __UINT64_MAX__ 18446744073709551615UL
 // MIPS64EL:#define __UINT64_TYPE__ long unsigned int
+// MIPS64EL:#define __UINT8_C(c) c
 // MIPS64EL:#define __UINT8_C_SUFFIX__
 // MIPS64EL:#define __UINT8_MAX__ 255
 // MIPS64EL:#define __UINT8_TYPE__ unsigned char
+// MIPS64EL:#define __UINTMAX_C(c) c##UL
 // MIPS64EL:#define __UINTMAX_C_SUFFIX__ UL
 // MIPS64EL:#define __UINTMAX_MAX__ 18446744073709551615UL
 // MIPS64EL:#define __UINTMAX_TYPE__ long unsigned int
diff --git a/clang/test/Preprocessor/init-ppc.c b/clang/test/Preprocessor/init-ppc.c
index 1421b102a3dfd..6b7eceda9b97b 100644
--- a/clang/test/Preprocessor/init-ppc.c
+++ b/clang/test/Preprocessor/init-ppc.c
@@ -41,26 +41,31 @@
 // PPC603E:#define __FLT_MIN_EXP__ (-125)
 // PPC603E:#define __FLT_MIN__ 1.17549435e-38F
 // PPC603E:#define __FLT_RADIX__ 2
+// PPC603E:#define __INT16_C(c) c
 // PPC603E:#define __INT16_C_SUFFIX__
 // PPC603E:#define __INT16_FMTd__ "hd"
 // PPC603E:#define __INT16_FMTi__ "hi"
 // PPC603E:#define __INT16_MAX__ 32767
 // PPC603E:#define __INT16_TYPE__ short
+// PPC603E:#define __INT32_C(c) c
 // PPC603E:#define __INT32_C_SUFFIX__
 // PPC603E:#define __INT32_FMTd__ "d"
 // PPC603E:#define __INT32_FMTi__ "i"
 // PPC603E:#define __INT32_MAX__ 2147483647
 // PPC603E:#define __INT32_TYPE__ int
+// PPC603E:#define __INT64_C(c) c##LL
 // PPC603E:#define __INT64_C_SUFFIX__ LL
 // PPC603E:#define __INT64_FMTd__ "lld"
 // PPC603E:#define __INT64_FMTi__ "lli"
 // PPC603E:#define __INT64_MAX__ 9223372036854775807LL
 // PPC603E:#define __INT64_TYPE__ long long int
+// PPC603E:#define __INT8_C(c) c
 // PPC603E:#define __INT8_C_SUFFIX__
 // PPC603E:#define __INT8_FMTd__ "hhd"
 // PPC603E:#define __INT8_FMTi__ "hhi"
 // PPC603E:#define __INT8_MAX__ 127
 // PPC603E:#define __INT8_TYPE__ signed char
+// PPC603E:#define __INTMAX_C(c) c##LL
 // PPC603E:#define __INTMAX_C_SUFFIX__ LL
 // PPC603E:#define __INTMAX_FMTd__ "lld"
 // PPC603E:#define __INTMAX_FMTi__ "lli"
@@ -150,18 +155,23 @@
 // PPC603E:#define __SIZE_TYPE__ long unsigned int
 // PPC603E:#define __SIZE_WIDTH__ 32
 // PPC603E-CXX:#define __STDCPP_DEFAULT_NEW_ALIGNMENT__ 16UL
+// PPC603E:#define __UINT16_C(c) c
 // PPC603E:#define __UINT16_C_SUFFIX__
 // PPC603E:#define __UINT16_MAX__ 65535
 // PPC603E:#define __UINT16_TYPE__ unsigned short
+// PPC603E:#define __UINT32_C(c) c##U
 // PPC603E:#define __UINT32_C_SUFFIX__ U
 // PPC603E:#define __UINT32_MAX__ 4294967295U
 // PPC603E:#define __UINT32_TYPE__ unsigned int
+// PPC603E:#define __UINT64_C(c) c##ULL
 // PPC603E:#define __UINT64_C_SUFFIX__ ULL
 // PPC603E:#define __UINT64_MAX__ 18446744073709551615ULL
 // PPC603E:#define __UINT64_TYPE__ long long unsigned int
+// PPC603E:#define __UINT8_C(c) c
 // PPC603E:#define __UINT8_C_SUFFIX__
 // PPC603E:#define __UINT8_MAX__ 255
 // PPC603E:#define __UINT8_TYPE__ unsigned char
+// PPC603E:#define __UINTMAX_C(c) c##ULL
 // PPC603E:#define __UINTMAX_C_SUFFIX__ ULL
 // PPC603E:#define __UINTMAX_MAX__ 18446744073709551615ULL
 // PPC603E:#define __UINTMAX_TYPE__ long long unsigned int
@@ -235,26 +245,31 @@
 // PPC:#define __FLT_MIN__ 1.17549435e-38F
 // PPC:#define __FLT_RADIX__ 2
 // PPC:#define __HAVE_BSWAP__ 1
+// PPC:#define __INT16_C(c) c
 // PPC:#define __INT16_C_SUFFIX__
 // PPC:#define __INT16_FMTd__ "hd"
 // PPC:#define __INT16_FMTi__ "hi"
 // PPC:#define __INT16_MAX__ 32767
 // PPC:#define __INT16_TYPE__ short
+// PPC:#define __INT32_C(c) c
 // PPC:#define __INT32_C_SUFFIX__
 // PPC:#define __INT32_FMTd__ "d"
 // PPC:#define __INT32_FMTi__ "i"
 // PPC:#define __INT32_MAX__ 2147483647
 // PPC:#define __INT32_TYPE__ int
+// PPC:#define __INT64_C(c) c##LL
 // PPC:#define __INT64_C_SUFFIX__ LL
 // PPC:#define __INT64_FMTd__ "lld"
 // PPC:#define __INT64_FMTi__ "lli"
 // PPC:#define __INT64_MAX__ 9223372036854775807LL
 // PPC:#define __INT64_TYPE__ long long int
+// PPC:#define __INT8_C(c) c
 // PPC:#define __INT8_C_SUFFIX__
 // PPC:#define __INT8_FMTd__ "hhd"
 // PPC:#define __INT8_FMTi__ "hhi"
 // PPC:#define __INT8_MAX__ 127
 // PPC:#define __INT8_TYPE__ signed char
+// PPC:#define __INTMAX_C(c) c##LL
 // PPC:#define __INTMAX_C_SUFFIX__ LL
 // PPC:#define __INTMAX_FMTd__ "lld"
 // PPC:#define __INTMAX_FMTi__ "lli"
@@ -344,18 +359,23 @@
 // PPC:#define __SIZE_MAX__ 4294967295UL
 // PPC:#define __SIZE_TYPE__ long unsigned int
 // PPC:#define __SIZE_WIDTH__ 32
+// PPC:#define __UINT16_C(c) c
 // PPC:#define __UINT16_C_SUFFIX__
 // PPC:#define __UINT16_MAX__ 65535
 // PPC:#define __UINT16_TYPE__ unsigned short
+// PPC:#define __UINT32_C(c) c##U
 // PPC:#define __UINT32_C_SUFFIX__ U
 // PPC:#define __UINT32_MAX__ 4294967295U
 // PPC:#define __UINT32_TYPE__ unsigned int
+// PPC:#define __UINT64_C(c) c##ULL
 // PPC:#define __UINT64_C_SUFFIX__ ULL
 // PPC:#define __UINT64_MAX__ 18446744073709551615ULL
 // PPC:#define __UINT64_TYPE__ long long unsigned int
+// PPC:#define __UINT8_C(c) c
 // PPC:#define __UINT8_C_SUFFIX__
 // PPC:#define __UINT8_MAX__ 255
 // PPC:#define __UINT8_TYPE__ unsigned char
+// PPC:#define __UINTMAX_C(c) c##ULL
 // PPC:#define __UINTMAX_C_SUFFIX__ ULL
 // PPC:#define __UINTMAX_MAX__ 18446744073709551615ULL
 // PPC:#define __UINTMAX_TYPE__ long long unsigned int
@@ -435,26 +455,31 @@
 // PPC-AIX:#define __FLT_MIN__ 1.17549435e-38F
 // PPC-AIX:#define __FLT_RADIX__ 2
 // PPC-AIX:#define __HOS_AIX__ 1
+// PPC-AIX:#define __INT16_C(c) c
 // PPC-AIX:#define __INT16_C_SUFFIX__
 // PPC-AIX:#define __INT16_FMTd__ "hd"
 // PPC-AIX:#define __INT16_FMTi__ "hi"
 // PPC-AIX:#define __INT16_MAX__ 32767
 // PPC-AIX:#define __INT16_TYPE__ short
+// PPC-AIX:#define __INT32_C(c) c
 // PPC-AIX:#define __INT32_C_SUFFIX__
 // PPC-AIX:#define __INT32_FMTd__ "d"
 // PPC-AIX:#define __INT32_FMTi__ "i"
 // PPC-AIX:#define __INT32_MAX__ 2147483647
 // PPC-AIX:#define __INT32_TYPE__ int
+// PPC-AIX:#define __INT64_C(c) c##LL
 // PPC-AIX:#define __INT64_C_SUFFIX__ LL
 // PPC-AIX:#define __INT64_FMTd__ "lld"
 // PPC-AIX:#define __INT64_FMTi__ "lli"
 // PPC-AIX:#define __INT64_MAX__ 9223372036854775807LL
 // PPC-AIX:#define __INT64_TYPE__ long long int
+// PPC-AIX:#define __INT8_C(c) c
 // PPC-AIX:#define __INT8_C_SUFFIX__
 // PPC-AIX:#define __INT8_FMTd__ "hhd"
 // PPC-AIX:#define __INT8_FMTi__ "hhi"
 // PPC-AIX:#define __INT8_MAX__ 127
 // PPC-AIX:#define __INT8_TYPE__ signed char
+// PPC-AIX:#define __INTMAX_C(c) c##LL
 // PPC-AIX:#define __INTMAX_C_SUFFIX__ LL
 // PPC-AIX:#define __INTMAX_FMTd__ "lld"
 // PPC-AIX:#define __INTMAX_FMTi__ "lli"
@@ -546,18 +571,23 @@
 // PPC-AIX:#define __THW_BIG_ENDIAN__ 1
 // PPC-AIX:#define __THW_PPC__ 1
 // PPC-AIX:#define __TOS_AIX__ 1
+// PPC-AIX:#define __UINT16_C(c) c
 // PPC-AIX:#define __UINT16_C_SUFFIX__
 // PPC-AIX:#define __UINT16_MAX__ 65535
 // PPC-AIX:#define __UINT16_TYPE__ unsigned short
+// PPC-AIX:#define __UINT32_C(c) c##U
 // PPC-AIX:#define __UINT32_C_SUFFIX__ U
 // PPC-AIX:#define __UINT32_MAX__ 4294967295U
 // PPC-AIX:#define __UINT32_TYPE__ unsigned int
+// PPC-AIX:#define __UINT64_C(c) c##ULL
 // PPC-AIX:#define __UINT64_C_SUFFIX__ ULL
 // PPC-AIX:#define __UINT64_MAX__ 18446744073709551615ULL
 // PPC-AIX:#define __UINT64_TYPE__ long long unsigned int
+// PPC-AIX:#define __UINT8_C(c) c
 // PPC-AIX:#define __UINT8_C_SUFFIX__
 // PPC-AIX:#define __UINT8_MAX__ 255
 // PPC-AIX:#define __UINT8_TYPE__ unsigned char
+// PPC-AIX:#define __UINTMAX_C(c) c##ULL
 // PPC-AIX:#define __UINTMAX_C_SUFFIX__ ULL
 // PPC-AIX:#define __UINTMAX_MAX__ 18446744073709551615ULL
 // PPC-AIX:#define __UINTMAX_TYPE__ long long unsigned int
@@ -807,26 +837,31 @@
 // PPC-LINUX:#define __FLT_MIN__ 1.17549435e-38F
 // PPC-LINUX:#define __FLT_RADIX__ 2
 // PPC-LINUX:#define __HAVE_BSWAP__ 1
+// PPC-LINUX:#define __INT16_C(c) c
 // PPC-LINUX:#define __INT16_C_SUFFIX__
 // PPC-LINUX:#define __INT16_FMTd__ "hd"
 // PPC-LINUX:#define __INT16_FMTi__ "hi"
 // PPC-LINUX:#define __INT16_MAX__ 32767
 // PPC-LINUX:#define __INT16_TYPE__ short
+// PPC-LINUX:#define __INT32_C(c) c
 // PPC-LINUX:#define __INT32_C_SUFFIX__
 // PPC-LINUX:#define __INT32_FMTd__ "d"
 // PPC-LINUX:#define __INT32_FMTi__ "i"
 // PPC-LINUX:#define __INT32_MAX__ 2147483647
 // PPC-LINUX:#define __INT32_TYPE__ int
+// PPC-LINUX:#define __INT64_C(c) c##LL
 // PPC-LINUX:#define __INT64_C_SUFFIX__ LL
 // PPC-LINUX:#define __INT64_FMTd__ "lld"
 // PPC-LINUX:#define __INT64_FMTi__ "lli"
 // PPC-LINUX:#define __INT64_MAX__ 9223372036854775807LL
 // PPC-LINUX:#define __INT64_TYPE__ long long int
+// PPC-LINUX:#define __INT8_C(c) c
 // PPC-LINUX:#define __INT8_C_SUFFIX__
 // PPC-LINUX:#define __INT8_FMTd__ "hhd"
 // PPC-LINUX:#define __INT8_FMTi__ "hhi"
 // PPC-LINUX:#define __INT8_MAX__ 127
 // PPC-LINUX:#define __INT8_TYPE__ signed char
+// PPC-LINUX:#define __INTMAX_C(c) c##LL
 // PPC-LINUX:#define __INTMAX_C_SUFFIX__ LL
 // PPC-LINUX:#define __INTMAX_FMTd__ "lld"
 // PPC-LINUX:#define __INTMAX_FMTi__ "lli"
@@ -915,18 +950,23 @@
 // PPC-LINUX:#define __SIZE_MAX__ 4294967295U
 // PPC-LINUX:#define __SIZE_TYPE__ unsigned int
 // PPC-LINUX:#define __SIZE_WIDTH__ 32
+// PPC-LINUX:#define __UINT16_C(c) c
 // PPC-LINUX:#define __UINT16_C_SUFFIX__
 // PPC-LINUX:#define __UINT16_MAX__ 65535
 // PPC-LINUX:#define __UINT16_TYPE__ unsigned short
+// PPC-LINUX:#define __UINT32_C(c) c##U
 // PPC-LINUX:#define __UINT32_C_SUFFIX__ U
 // PPC-LINUX:#define __UINT32_MAX__ 4294967295U
 // PPC-LINUX:#define __UINT32_TYPE__ unsigned int
+// PPC-LINUX:#define __UINT64_C(c) c##ULL
 // PPC-LINUX:#define __UINT64_C_SUFFIX__ ULL
 // PPC-LINUX:#define __UINT64_MAX__ 18446744073709551615ULL
 // PPC-LINUX:#define __UINT64_TYPE__ long long unsigned int
+// PPC-LINUX:#define __UINT8_C(c) c
 // PPC-LINUX:#define __UINT8_C_SUFFIX__
 // PPC-LINUX:#define __UINT8_MAX__ 255
 // PPC-LINUX:#define __UINT8_TYPE__ unsigned char
+// PPC-LINUX:#define __UINTMAX_C(c) c##ULL
 // PPC-LINUX:#define __UINTMAX_C_SUFFIX__ ULL
 // PPC-LINUX:#define __UINTMAX_MAX__ 18446744073709551615ULL
 // PPC-LINUX:#define __UINTMAX_TYPE__ long long unsigned int
diff --git a/clang/test/Preprocessor/init-ppc64.c b/clang/test/Preprocessor/init-ppc64.c
index 57e2ca31d5d53..7dffd4627481b 100644
--- a/clang/test/Preprocessor/init-ppc64.c
+++ b/clang/test/Preprocessor/init-ppc64.c
@@ -47,26 +47,31 @@
 // PPC64:#define __FLT_MIN__ 1.17549435e-38F
 // PPC64:#define __FLT_RADIX__ 2
 // PPC64:#define __HAVE_BSWAP__ 1
+// PPC64:#define __INT16_C(c) c
 // PPC64:#define __INT16_C_SUFFIX__
 // PPC64:#define __INT16_FMTd__ "hd"
 // PPC64:#define __INT16_FMTi__ "hi"
 // PPC64:#define __INT16_MAX__ 32767
 // PPC64:#define __INT16_TYPE__ short
+// PPC64:#define __INT32_C(c) c
 // PPC64:#define __INT32_C_SUFFIX__
 // PPC64:#define __INT32_FMTd__ "d"
 // PPC64:#define __INT32_FMTi__ "i"
 // PPC64:#define __INT32_MAX__ 2147483647
 // PPC64:#define __INT32_TYPE__ int
+// PPC64:#define __INT64_C(c) c##L
 // PPC64:#define __INT64_C_SUFFIX__ L
 // PPC64:#define __INT64_FMTd__ "ld"
 // PPC64:#define __INT64_FMTi__ "li"
 // PPC64:#define __INT64_MAX__ 9223372036854775807L
 // PPC64:#define __INT64_TYPE__ long int
+// PPC64:#define __INT8_C(c) c
 // PPC64:#define __INT8_C_SUFFIX__
 // PPC64:#define __INT8_FMTd__ "hhd"
 // PPC64:#define __INT8_FMTi__ "hhi"
 // PPC64:#define __INT8_MAX__ 127
 // PPC64:#define __INT8_TYPE__ signed char
+// PPC64:#define __INTMAX_C(c) c##L
 // PPC64:#define __INTMAX_C_SUFFIX__ L
 // PPC64:#define __INTMAX_FMTd__ "ld"
 // PPC64:#define __INTMAX_FMTi__ "li"
@@ -157,18 +162,23 @@
 // PPC64:#define __SIZE_TYPE__ long unsigned int
 // PPC64:#define __SIZE_WIDTH__ 64
 // PPC64-CXX:#define __STDCPP_DEFAULT_NEW_ALIGNMENT__ 16UL
+// PPC64:#define __UINT16_C(c) c
 // PPC64:#define __UINT16_C_SUFFIX__
 // PPC64:#define __UINT16_MAX__ 65535
 // PPC64:#define __UINT16_TYPE__ unsigned short
+// PPC64:#define __UINT32_C(c) c##U
 // PPC64:#define __UINT32_C_SUFFIX__ U
 // PPC64:#define __UINT32_MAX__ 4294967295U
 // PPC64:#define __UINT32_TYPE__ unsigned int
+// PPC64:#define __UINT64_C(c) c##UL
 // PPC64:#define __UINT64_C_SUFFIX__ UL
 // PPC64:#define __UINT64_MAX__ 18446744073709551615UL
 // PPC64:#define __UINT64_TYPE__ long unsigned int
+// PPC64:#define __UINT8_C(c) c
 // PPC64:#define __UINT8_C_SUFFIX__
 // PPC64:#define __UINT8_MAX__ 255
 // PPC64:#define __UINT8_TYPE__ unsigned char
+// PPC64:#define __UINTMAX_C(c) c##UL
 // PPC64:#define __UINTMAX_C_SUFFIX__ UL
 // PPC64:#define __UINTMAX_MAX__ 18446744073709551615UL
 // PPC64:#define __UINTMAX_TYPE__ long unsigned int
@@ -250,26 +260,31 @@
 // PPC64LE:#define __FLT_MIN__ 1.17549435e-38F
 // PPC64LE:#define __FLT_RADIX__ 2
 // PPC64LE:#define __HAVE_BSWAP__ 1
+// PPC64LE:#define __INT16_C(c) c
 // PPC64LE:#define __INT16_C_SUFFIX__
 // PPC64LE:#define __INT16_FMTd__ "hd"
 // PPC64LE:#define __INT16_FMTi__ "hi"
 // PPC64LE:#define __INT16_MAX__ 32767
 // PPC64LE:#define __INT16_TYPE__ short
+// PPC64LE:#define __INT32_C(c) c
 // PPC64LE:#define __INT32_C_SUFFIX__
 // PPC64LE:#define __INT32_FMTd__ "d"
 // PPC64LE:#define __INT32_FMTi__ "i"
 // PPC64LE:#define __INT32_MAX__ 2147483647
 // PPC64LE:#define __INT32_TYPE__ int
+// PPC64LE:#define __INT64_C(c) c##L
 // PPC64LE:#define __INT64_C_SUFFIX__ L
 // PPC64LE:#define __INT64_FMTd__ "ld"
 // PPC64LE:#define __INT64_FMTi__ "li"
 // PPC64LE:#define __INT64_MAX__ 9223372036854775807L
 // PPC64LE:#define __INT64_TYPE__ long int
+// PPC64LE:#define __INT8_C(c) c
 // PPC64LE:#define __INT8_C_SUFFIX__
 // PPC64LE:#define __INT8_FMTd__ "hhd"
 // PPC64LE:#define __INT8_FMTi__ "hhi"
 // PPC64LE:#define __INT8_MAX__ 127
 // PPC64LE:#define __INT8_TYPE__ signed char
+// PPC64LE:#define __INTMAX_C(c) c##L
 // PPC64LE:#define __INTMAX_C_SUFFIX__ L
 // PPC64LE:#define __INTMAX_FMTd__ "ld"
 // PPC64LE:#define __INTMAX_FMTi__ "li"
@@ -361,18 +376,23 @@
 // PPC64LE:#define __SIZE_TYPE__ long unsigned int
 // PPC64LE:#define __SIZE_WIDTH__ 64
 // PPC64LE:#define __STRUCT_PARM_ALIGN__ 16
+// PPC64LE:#define __UINT16_C(c) c
 // PPC64LE:#define __UINT16_C_SUFFIX__
 // PPC64LE:#define __UINT16_MAX__ 65535
 // PPC64LE:#define __UINT16_TYPE__ unsigned short
+// PPC64LE:#define __UINT32_C(c) c##U
 // PPC64LE:#define __UINT32_C_SUFFIX__ U
 // PPC64LE:#define __UINT32_MAX__ 4294967295U
 // PPC64LE:#define __UINT32_TYPE__ unsigned int
+// PPC64LE:#define __UINT64_C(c) c##UL
 // PPC64LE:#define __UINT64_C_SUFFIX__ UL
 // PPC64LE:#define __UINT64_MAX__ 18446744073709551615UL
 // PPC64LE:#define __UINT64_TYPE__ long unsigned int
+// PPC64LE:#define __UINT8_C(c) c
 // PPC64LE:#define __UINT8_C_SUFFIX__
 // PPC64LE:#define __UINT8_MAX__ 255
 // PPC64LE:#define __UINT8_TYPE__ unsigned char
+// PPC64LE:#define __UINTMAX_C(c) c##UL
 // PPC64LE:#define __UINTMAX_C_SUFFIX__ UL
 // PPC64LE:#define __UINTMAX_MAX__ 18446744073709551615UL
 // PPC64LE:#define __UINTMAX_TYPE__ long unsigned int
@@ -733,26 +753,31 @@
 // PPC64-AIX:#define __FLT_MIN__ 1.17549435e-38F
 // PPC64-AIX:#define __FLT_RADIX__ 2
 // PPC64-AIX-NOT:#define __ILP32__ 1
+// PPC64-AIX:#define __INT16_C(c) c
 // PPC64-AIX:#define __INT16_C_SUFFIX__
 // PPC64-AIX:#define __INT16_FMTd__ "hd"
 // PPC64-AIX:#define __INT16_FMTi__ "hi"
 // PPC64-AIX:#define __INT16_MAX__ 32767
 // PPC64-AIX:#define __INT16_TYPE__ short
+// PPC64-AIX:#define __INT32_C(c) c
 // PPC64-AIX:#define __INT32_C_SUFFIX__
 // PPC64-AIX:#define __INT32_FMTd__ "d"
 // PPC64-AIX:#define __INT32_FMTi__ "i"
 // PPC64-AIX:#define __INT32_MAX__ 2147483647
 // PPC64-AIX:#define __INT32_TYPE__ int
+// PPC64-AIX:#define __INT64_C(c) c##L
 // PPC64-AIX:#define __INT64_C_SUFFIX__ L
 // PPC64-AIX:#define __INT64_FMTd__ "ld"
 // PPC64-AIX:#define __INT64_FMTi__ "li"
 // PPC64-AIX:#define __INT64_MAX__ 9223372036854775807L
 // PPC64-AIX:#define __INT64_TYPE__ long int
+// PPC64-AIX:#define __INT8_C(c) c
 // PPC64-AIX:#define __INT8_C_SUFFIX__
 // PPC64-AIX:#define __INT8_FMTd__ "hhd"
 // PPC64-AIX:#define __INT8_FMTi__ "hhi"
 // PPC64-AIX:#define __INT8_MAX__ 127
 // PPC64-AIX:#define __INT8_TYPE__ signed char
+// PPC64-AIX:#define __INTMAX_C(c) c##L
 // PPC64-AIX:#define __INTMAX_C_SUFFIX__ L
 // PPC64-AIX:#define __INTMAX_FMTd__ "ld"
 // PPC64-AIX:#define __INTMAX_FMTi__ "li"
@@ -842,18 +867,23 @@
 // PPC64-AIX:#define __SIZE_MAX__ 18446744073709551615UL
 // PPC64-AIX:#define __SIZE_TYPE__ long unsigned int
 // PPC64-AIX:#define __SIZE_WIDTH__ 64
+// PPC64-AIX:#define __UINT16_C(c) c
 // PPC64-AIX:#define __UINT16_C_SUFFIX__
 // PPC64-AIX:#define __UINT16_MAX__ 65535
 // PPC64-AIX:#define __UINT16_TYPE__ unsigned short
+// PPC64-AIX:#define __UINT32_C(c) c##U
 // PPC64-AIX:#define __UINT32_C_SUFFIX__ U
 // PPC64-AIX:#define __UINT32_MAX__ 4294967295U
 // PPC64-AIX:#define __UINT32_TYPE__ unsigned int
+// PPC64-AIX:#define __UINT64_C(c) c##UL
 // PPC64-AIX:#define __UINT64_C_SUFFIX__ UL
 // PPC64-AIX:#define __UINT64_MAX__ 18446744073709551615UL
 // PPC64-AIX:#define __UINT64_TYPE__ long unsigned int
+// PPC64-AIX:#define __UINT8_C(c) c
 // PPC64-AIX:#define __UINT8_C_SUFFIX__
 // PPC64-AIX:#define __UINT8_MAX__ 255
 // PPC64-AIX:#define __UINT8_TYPE__ unsigned char
+// PPC64-AIX:#define __UINTMAX_C(c) c##UL
 // PPC64-AIX:#define __UINTMAX_C_SUFFIX__ UL
 // PPC64-AIX:#define __UINTMAX_MAX__ 18446744073709551615UL
 // PPC64-AIX:#define __UINTMAX_TYPE__ long unsigned int
@@ -930,26 +960,31 @@
 // PPC64-LINUX:#define __FLT_MIN__ 1.17549435e-38F
 // PPC64-LINUX:#define __FLT_RADIX__ 2
 // PPC64-LINUX:#define __HAVE_BSWAP__ 1
+// PPC64-LINUX:#define __INT16_C(c) c
 // PPC64-LINUX:#define __INT16_C_SUFFIX__
 // PPC64-LINUX:#define __INT16_FMTd__ "hd"
 // PPC64-LINUX:#define __INT16_FMTi__ "hi"
 // PPC64-LINUX:#define __INT16_MAX__ 32767
 // PPC64-LINUX:#define __INT16_TYPE__ short
+// PPC64-LINUX:#define __INT32_C(c) c
 // PPC64-LINUX:#define __INT32_C_SUFFIX__
 // PPC64-LINUX:#define __INT32_FMTd__ "d"
 // PPC64-LINUX:#define __INT32_FMTi__ "i"
 // PPC64-LINUX:#define __INT32_MAX__ 2147483647
 // PPC64-LINUX:#define __INT32_TYPE__ int
+// PPC64-LINUX:#define __INT64_C(c) c##L
 // PPC64-LINUX:#define __INT64_C_SUFFIX__ L
 // PPC64-LINUX:#define __INT64_FMTd__ "ld"
 // PPC64-LINUX:#define __INT64_FMTi__ "li"
 // PPC64-LINUX:#define __INT64_MAX__ 9223372036854775807L
 // PPC64-LINUX:#define __INT64_TYPE__ long int
+// PPC64-LINUX:#define __INT8_C(c) c
 // PPC64-LINUX:#define __INT8_C_SUFFIX__
 // PPC64-LINUX:#define __INT8_FMTd__ "hhd"
 // PPC64-LINUX:#define __INT8_FMTi__ "hhi"
 // PPC64-LINUX:#define __INT8_MAX__ 127
 // PPC64-LINUX:#define __INT8_TYPE__ signed char
+// PPC64-LINUX:#define __INTMAX_C(c) c##L
 // PPC64-LINUX:#define __INTMAX_C_SUFFIX__ L
 // PPC64-LINUX:#define __INTMAX_FMTd__ "ld"
 // PPC64-LINUX:#define __INTMAX_FMTi__ "li"
@@ -1039,18 +1074,23 @@
 // PPC64-LINUX:#define __SIZE_MAX__ 18446744073709551615UL
 // PPC64-LINUX:#define __SIZE_TYPE__ long unsigned int
 // PPC64-LINUX:#define __SIZE_WIDTH__ 64
+// PPC64-LINUX:#define __UINT16_C(c) c
 // PPC64-LINUX:#define __UINT16_C_SUFFIX__
 // PPC64-LINUX:#define __UINT16_MAX__ 65535
 // PPC64-LINUX:#define __UINT16_TYPE__ unsigned short
+// PPC64-LINUX:#define __UINT32_C(c) c##U
 // PPC64-LINUX:#define __UINT32_C_SUFFIX__ U
 // PPC64-LINUX:#define __UINT32_MAX__ 4294967295U
 // PPC64-LINUX:#define __UINT32_TYPE__ unsigned int
+// PPC64-LINUX:#define __UINT64_C(c) c##UL
 // PPC64-LINUX:#define __UINT64_C_SUFFIX__ UL
 // PPC64-LINUX:#define __UINT64_MAX__ 18446744073709551615UL
 // PPC64-LINUX:#define __UINT64_TYPE__ long unsigned int
+// PPC64-LINUX:#define __UINT8_C(c) c
 // PPC64-LINUX:#define __UINT8_C_SUFFIX__
 // PPC64-LINUX:#define __UINT8_MAX__ 255
 // PPC64-LINUX:#define __UINT8_TYPE__ unsigned char
+// PPC64-LINUX:#define __UINTMAX_C(c) c##UL
 // PPC64-LINUX:#define __UINTMAX_C_SUFFIX__ UL
 // PPC64-LINUX:#define __UINTMAX_MAX__ 18446744073709551615UL
 // PPC64-LINUX:#define __UINTMAX_TYPE__ long unsigned int
diff --git a/clang/test/Preprocessor/init-s390x.c b/clang/test/Preprocessor/init-s390x.c
index 6d08e9bfcb632..a8fbde46cbb75 100644
--- a/clang/test/Preprocessor/init-s390x.c
+++ b/clang/test/Preprocessor/init-s390x.c
@@ -34,26 +34,31 @@
 // S390X:#define __FLT_MIN_EXP__ (-125)
 // S390X:#define __FLT_MIN__ 1.17549435e-38F
 // S390X:#define __FLT_RADIX__ 2
+// S390X:#define __INT16_C(c) c
 // S390X:#define __INT16_C_SUFFIX__
 // S390X:#define __INT16_FMTd__ "hd"
 // S390X:#define __INT16_FMTi__ "hi"
 // S390X:#define __INT16_MAX__ 32767
 // S390X:#define __INT16_TYPE__ short
+// S390X:#define __INT32_C(c) c
 // S390X:#define __INT32_C_SUFFIX__
 // S390X:#define __INT32_FMTd__ "d"
 // S390X:#define __INT32_FMTi__ "i"
 // S390X:#define __INT32_MAX__ 2147483647
 // S390X:#define __INT32_TYPE__ int
+// S390X:#define __INT64_C(c) c##L
 // S390X:#define __INT64_C_SUFFIX__ L
 // S390X:#define __INT64_FMTd__ "ld"
 // S390X:#define __INT64_FMTi__ "li"
 // S390X:#define __INT64_MAX__ 9223372036854775807L
 // S390X:#define __INT64_TYPE__ long int
+// S390X:#define __INT8_C(c) c
 // S390X:#define __INT8_C_SUFFIX__
 // S390X:#define __INT8_FMTd__ "hhd"
 // S390X:#define __INT8_FMTi__ "hhi"
 // S390X:#define __INT8_MAX__ 127
 // S390X:#define __INT8_TYPE__ signed char
+// S390X:#define __INTMAX_C(c) c##L
 // S390X:#define __INTMAX_C_SUFFIX__ L
 // S390X:#define __INTMAX_FMTd__ "ld"
 // S390X:#define __INTMAX_FMTi__ "li"
@@ -136,18 +141,23 @@
 // S390X:#define __SIZE_TYPE__ long unsigned int
 // S390X:#define __SIZE_WIDTH__ 64
 // S390X-CXX:#define __STDCPP_DEFAULT_NEW_ALIGNMENT__ 8UL
+// S390X:#define __UINT16_C(c) c
 // S390X:#define __UINT16_C_SUFFIX__
 // S390X:#define __UINT16_MAX__ 65535
 // S390X:#define __UINT16_TYPE__ unsigned short
+// S390X:#define __UINT32_C(c) c##U
 // S390X:#define __UINT32_C_SUFFIX__ U
 // S390X:#define __UINT32_MAX__ 4294967295U
 // S390X:#define __UINT32_TYPE__ unsigned int
+// S390X:#define __UINT64_C(c) c##UL
 // S390X:#define __UINT64_C_SUFFIX__ UL
 // S390X:#define __UINT64_MAX__ 18446744073709551615UL
 // S390X:#define __UINT64_TYPE__ long unsigned int
+// S390X:#define __UINT8_C(c) c
 // S390X:#define __UINT8_C_SUFFIX__
 // S390X:#define __UINT8_MAX__ 255
 // S390X:#define __UINT8_TYPE__ unsigned char
+// S390X:#define __UINTMAX_C(c) c##UL
 // S390X:#define __UINTMAX_C_SUFFIX__ UL
 // S390X:#define __UINTMAX_MAX__ 18446744073709551615UL
 // S390X:#define __UINTMAX_TYPE__ long unsigned int
diff --git a/clang/test/Preprocessor/init-v7k-compat.c b/clang/test/Preprocessor/init-v7k-compat.c
index ff5d4bbdea53a..a9c6e7a290646 100644
--- a/clang/test/Preprocessor/init-v7k-compat.c
+++ b/clang/test/Preprocessor/init-v7k-compat.c
@@ -39,26 +39,31 @@
 // CHECK: #define __FLT_MIN_EXP__ (-125)
 // CHECK: #define __FLT_MIN__ 1.17549435e-38F
 // CHECK: #define __FLT_RADIX__ 2
+// CHECK: #define __INT16_C(c) c
 // CHECK: #define __INT16_C_SUFFIX__ {{$}}
 // CHECK: #define __INT16_FMTd__ "hd"
 // CHECK: #define __INT16_FMTi__ "hi"
 // CHECK: #define __INT16_MAX__ 32767
 // CHECK: #define __INT16_TYPE__ short
+// CHECK: #define __INT32_C(c) c
 // CHECK: #define __INT32_C_SUFFIX__ {{$}}
 // CHECK: #define __INT32_FMTd__ "d"
 // CHECK: #define __INT32_FMTi__ "i"
 // CHECK: #define __INT32_MAX__ 2147483647
 // CHECK: #define __INT32_TYPE__ int
+// CHECK: #define __INT64_C(c) c##LL
 // CHECK: #define __INT64_C_SUFFIX__ LL
 // CHECK: #define __INT64_FMTd__ "lld"
 // CHECK: #define __INT64_FMTi__ "lli"
 // CHECK: #define __INT64_MAX__ 9223372036854775807LL
 // CHECK: #define __INT64_TYPE__ long long int
+// CHECK: #define __INT8_C(c) c
 // CHECK: #define __INT8_C_SUFFIX__ {{$}}
 // CHECK: #define __INT8_FMTd__ "hhd"
 // CHECK: #define __INT8_FMTi__ "hhi"
 // CHECK: #define __INT8_MAX__ 127
 // CHECK: #define __INT8_TYPE__ signed char
+// CHECK: #define __INTMAX_C(c) c##LL
 // CHECK: #define __INTMAX_C_SUFFIX__ LL
 // CHECK: #define __INTMAX_FMTd__ "lld"
 // CHECK: #define __INTMAX_FMTi__ "lli"
@@ -140,18 +145,23 @@
 // CHECK: #define __SIZE_MAX__ 4294967295UL
 // CHECK: #define __SIZE_TYPE__ long unsigned int
 // CHECK: #define __SIZE_WIDTH__ 32
+// CHECK: #define __UINT16_C(c) c
 // CHECK: #define __UINT16_C_SUFFIX__ {{$}}
 // CHECK: #define __UINT16_MAX__ 65535
 // CHECK: #define __UINT16_TYPE__ unsigned short
+// CHECK: #define __UINT32_C(c) c##U
 // CHECK: #define __UINT32_C_SUFFIX__ U
 // CHECK: #define __UINT32_MAX__ 4294967295U
 // CHECK: #define __UINT32_TYPE__ unsigned int
+// CHECK: #define __UINT64_C(c) c##ULL
 // CHECK: #define __UINT64_C_SUFFIX__ ULL
 // CHECK: #define __UINT64_MAX__ 18446744073709551615ULL
 // CHECK: #define __UINT64_TYPE__ long long unsigned int
+// CHECK: #define __UINT8_C(c) c
 // CHECK: #define __UINT8_C_SUFFIX__ {{$}}
 // CHECK: #define __UINT8_MAX__ 255
 // CHECK: #define __UINT8_TYPE__ unsigned char
+// CHECK: #define __UINTMAX_C(c) c##ULL
 // CHECK: #define __UINTMAX_C_SUFFIX__ ULL
 // CHECK: #define __UINTMAX_MAX__ 18446744073709551615ULL
 // CHECK: #define __UINTMAX_TYPE__ long long unsigned int
diff --git a/clang/test/Preprocessor/init-ve.c b/clang/test/Preprocessor/init-ve.c
index 13bdb12387db4..711c2a04865b3 100644
--- a/clang/test/Preprocessor/init-ve.c
+++ b/clang/test/Preprocessor/init-ve.c
@@ -45,26 +45,31 @@
 // VE:#define __FLT_MIN_EXP__ (-125)
 // VE:#define __FLT_MIN__ 1.17549435e-38F
 // VE:#define __FLT_RADIX__ 2
+// VE:#define __INT16_C(c) c
 // VE:#define __INT16_C_SUFFIX__
 // VE:#define __INT16_FMTd__ "hd"
 // VE:#define __INT16_FMTi__ "hi"
 // VE:#define __INT16_MAX__ 32767
 // VE:#define __INT16_TYPE__ short
+// VE:#define __INT32_C(c) c
 // VE:#define __INT32_C_SUFFIX__
 // VE:#define __INT32_FMTd__ "d"
 // VE:#define __INT32_FMTi__ "i"
 // VE:#define __INT32_MAX__ 2147483647
 // VE:#define __INT32_TYPE__ int
+// VE:#define __INT64_C(c) c##L
 // VE:#define __INT64_C_SUFFIX__ L
 // VE:#define __INT64_FMTd__ "ld"
 // VE:#define __INT64_FMTi__ "li"
 // VE:#define __INT64_MAX__ 9223372036854775807L
 // VE:#define __INT64_TYPE__ long int
+// VE:#define __INT8_C(c) c
 // VE:#define __INT8_C_SUFFIX__
 // VE:#define __INT8_FMTd__ "hhd"
 // VE:#define __INT8_FMTi__ "hhi"
 // VE:#define __INT8_MAX__ 127
 // VE:#define __INT8_TYPE__ signed char
+// VE:#define __INTMAX_C(c) c##L
 // VE:#define __INTMAX_C_SUFFIX__ L
 // VE:#define __INTMAX_FMTd__ "ld"
 // VE:#define __INTMAX_FMTi__ "li"
@@ -164,6 +169,7 @@
 // VE-CXX:#define __STDCPP_DEFAULT_NEW_ALIGNMENT__ 16UL
 // VE-HOSTED:#define __STDC_HOSTED__ 1
 // VE-FREESTANDING:#define __STDC_HOSTED__ 0
+// VE:#define __UINT16_C(c) c
 // VE:#define __UINT16_C_SUFFIX__
 // VE:#define __UINT16_FMTX__ "hX"
 // VE:#define __UINT16_FMTo__ "ho"
@@ -171,6 +177,7 @@
 // VE:#define __UINT16_FMTx__ "hx"
 // VE:#define __UINT16_MAX__ 65535
 // VE:#define __UINT16_TYPE__ unsigned short
+// VE:#define __UINT32_C(c) c##U
 // VE:#define __UINT32_C_SUFFIX__ U
 // VE:#define __UINT32_FMTX__ "X"
 // VE:#define __UINT32_FMTo__ "o"
@@ -178,6 +185,7 @@
 // VE:#define __UINT32_FMTx__ "x"
 // VE:#define __UINT32_MAX__ 4294967295U
 // VE:#define __UINT32_TYPE__ unsigned int
+// VE:#define __UINT64_C(c) c##UL
 // VE:#define __UINT64_C_SUFFIX__ UL
 // VE:#define __UINT64_FMTX__ "lX"
 // VE:#define __UINT64_FMTo__ "lo"
@@ -185,6 +193,7 @@
 // VE:#define __UINT64_FMTx__ "lx"
 // VE:#define __UINT64_MAX__ 18446744073709551615UL
 // VE:#define __UINT64_TYPE__ long unsigned int
+// VE:#define __UINT8_C(c) c
 // VE:#define __UINT8_C_SUFFIX__
 // VE:#define __UINT8_FMTX__ "hhX"
 // VE:#define __UINT8_FMTo__ "hho"
@@ -192,6 +201,7 @@
 // VE:#define __UINT8_FMTx__ "hhx"
 // VE:#define __UINT8_MAX__ 255
 // VE:#define __UINT8_TYPE__ unsigned char
+// VE:#define __UINTMAX_C(c) c##UL
 // VE:#define __UINTMAX_C_SUFFIX__ UL
 // VE:#define __UINTMAX_FMTX__ "lX"
 // VE:#define __UINTMAX_FMTo__ "lo"
diff --git a/clang/test/Preprocessor/init-x86.c b/clang/test/Preprocessor/init-x86.c
index 6f5aa5674e48e..cb77b5583407c 100644
--- a/clang/test/Preprocessor/init-x86.c
+++ b/clang/test/Preprocessor/init-x86.c
@@ -35,26 +35,31 @@
 // I386:#define __FLT_MIN_EXP__ (-125)
 // I386:#define __FLT_MIN__ 1.17549435e-38F
 // I386:#define __FLT_RADIX__ 2
+// I386:#define __INT16_C(c) c
 // I386:#define __INT16_C_SUFFIX__
 // I386:#define __INT16_FMTd__ "hd"
 // I386:#define __INT16_FMTi__ "hi"
 // I386:#define __INT16_MAX__ 32767
 // I386:#define __INT16_TYPE__ short
+// I386:#define __INT32_C(c) c
 // I386:#define __INT32_C_SUFFIX__
 // I386:#define __INT32_FMTd__ "d"
 // I386:#define __INT32_FMTi__ "i"
 // I386:#define __INT32_MAX__ 2147483647
 // I386:#define __INT32_TYPE__ int
+// I386:#define __INT64_C(c) c##LL
 // I386:#define __INT64_C_SUFFIX__ LL
 // I386:#define __INT64_FMTd__ "lld"
 // I386:#define __INT64_FMTi__ "lli"
 // I386:#define __INT64_MAX__ 9223372036854775807LL
 // I386:#define __INT64_TYPE__ long long int
+// I386:#define __INT8_C(c) c
 // I386:#define __INT8_C_SUFFIX__
 // I386:#define __INT8_FMTd__ "hhd"
 // I386:#define __INT8_FMTi__ "hhi"
 // I386:#define __INT8_MAX__ 127
 // I386:#define __INT8_TYPE__ signed char
+// I386:#define __INTMAX_C(c) c##LL
 // I386:#define __INTMAX_C_SUFFIX__ LL
 // I386:#define __INTMAX_FMTd__ "lld"
 // I386:#define __INTMAX_FMTi__ "lli"
@@ -140,18 +145,23 @@
 // I386:#define __SIZE_MAX__ 4294967295U
 // I386:#define __SIZE_TYPE__ unsigned int
 // I386:#define __SIZE_WIDTH__ 32
+// I386:#define __UINT16_C(c) c
 // I386:#define __UINT16_C_SUFFIX__
 // I386:#define __UINT16_MAX__ 65535
 // I386:#define __UINT16_TYPE__ unsigned short
+// I386:#define __UINT32_C(c) c##U
 // I386:#define __UINT32_C_SUFFIX__ U
 // I386:#define __UINT32_MAX__ 4294967295U
 // I386:#define __UINT32_TYPE__ unsigned int
+// I386:#define __UINT64_C(c) c##ULL
 // I386:#define __UINT64_C_SUFFIX__ ULL
 // I386:#define __UINT64_MAX__ 18446744073709551615ULL
 // I386:#define __UINT64_TYPE__ long long unsigned int
+// I386:#define __UINT8_C(c) c
 // I386:#define __UINT8_C_SUFFIX__
 // I386:#define __UINT8_MAX__ 255
 // I386:#define __UINT8_TYPE__ unsigned char
+// I386:#define __UINTMAX_C(c) c##ULL
 // I386:#define __UINTMAX_C_SUFFIX__ ULL
 // I386:#define __UINTMAX_MAX__ 18446744073709551615ULL
 // I386:#define __UINTMAX_TYPE__ long long unsigned int
@@ -235,26 +245,31 @@
 // I386-LINUX:#define __GCC_ATOMIC_SHORT_LOCK_FREE 2
 // I386-LINUX:#define __GCC_ATOMIC_TEST_AND_SET_TRUEVAL 1
 // I386-LINUX:#define __GCC_ATOMIC_WCHAR_T_LOCK_FREE 2
+// I386-LINUX:#define __INT16_C(c) c
 // I386-LINUX:#define __INT16_C_SUFFIX__
 // I386-LINUX:#define __INT16_FMTd__ "hd"
 // I386-LINUX:#define __INT16_FMTi__ "hi"
 // I386-LINUX:#define __INT16_MAX__ 32767
 // I386-LINUX:#define __INT16_TYPE__ short
+// I386-LINUX:#define __INT32_C(c) c
 // I386-LINUX:#define __INT32_C_SUFFIX__
 // I386-LINUX:#define __INT32_FMTd__ "d"
 // I386-LINUX:#define __INT32_FMTi__ "i"
 // I386-LINUX:#define __INT32_MAX__ 2147483647
 // I386-LINUX:#define __INT32_TYPE__ int
+// I386-LINUX:#define __INT64_C(c) c##LL
 // I386-LINUX:#define __INT64_C_SUFFIX__ LL
 // I386-LINUX:#define __INT64_FMTd__ "lld"
 // I386-LINUX:#define __INT64_FMTi__ "lli"
 // I386-LINUX:#define __INT64_MAX__ 9223372036854775807LL
 // I386-LINUX:#define __INT64_TYPE__ long long int
+// I386-LINUX:#define __INT8_C(c) c
 // I386-LINUX:#define __INT8_C_SUFFIX__
 // I386-LINUX:#define __INT8_FMTd__ "hhd"
 // I386-LINUX:#define __INT8_FMTi__ "hhi"
 // I386-LINUX:#define __INT8_MAX__ 127
 // I386-LINUX:#define __INT8_TYPE__ signed char
+// I386-LINUX:#define __INTMAX_C(c) c##LL
 // I386-LINUX:#define __INTMAX_C_SUFFIX__ LL
 // I386-LINUX:#define __INTMAX_FMTd__ "lld"
 // I386-LINUX:#define __INTMAX_FMTi__ "lli"
@@ -341,18 +356,23 @@
 // I386-LINUX:#define __SIZE_TYPE__ unsigned int
 // I386-LINUX:#define __SIZE_WIDTH__ 32
 // I386-LINUX-CXX:#define __STDCPP_DEFAULT_NEW_ALIGNMENT__ 8U
+// I386-LINUX:#define __UINT16_C(c) c
 // I386-LINUX:#define __UINT16_C_SUFFIX__
 // I386-LINUX:#define __UINT16_MAX__ 65535
 // I386-LINUX:#define __UINT16_TYPE__ unsigned short
+// I386-LINUX:#define __UINT32_C(c) c##U
 // I386-LINUX:#define __UINT32_C_SUFFIX__ U
 // I386-LINUX:#define __UINT32_MAX__ 4294967295U
 // I386-LINUX:#define __UINT32_TYPE__ unsigned int
+// I386-LINUX:#define __UINT64_C(c) c##ULL
 // I386-LINUX:#define __UINT64_C_SUFFIX__ ULL
 // I386-LINUX:#define __UINT64_MAX__ 18446744073709551615ULL
 // I386-LINUX:#define __UINT64_TYPE__ long long unsigned int
+// I386-LINUX:#define __UINT8_C(c) c
 // I386-LINUX:#define __UINT8_C_SUFFIX__
 // I386-LINUX:#define __UINT8_MAX__ 255
 // I386-LINUX:#define __UINT8_TYPE__ unsigned char
+// I386-LINUX:#define __UINTMAX_C(c) c##ULL
 // I386-LINUX:#define __UINTMAX_C_SUFFIX__ ULL
 // I386-LINUX:#define __UINTMAX_MAX__ 18446744073709551615ULL
 // I386-LINUX:#define __UINTMAX_TYPE__ long long unsigned int
@@ -436,26 +456,31 @@
 // I386-NETBSD:#define __GCC_ATOMIC_SHORT_LOCK_FREE 2
 // I386-NETBSD:#define __GCC_ATOMIC_TEST_AND_SET_TRUEVAL 1
 // I386-NETBSD:#define __GCC_ATOMIC_WCHAR_T_LOCK_FREE 2
+// I386-NETBSD:#define __INT16_C(c) c
 // I386-NETBSD:#define __INT16_C_SUFFIX__
 // I386-NETBSD:#define __INT16_FMTd__ "hd"
 // I386-NETBSD:#define __INT16_FMTi__ "hi"
 // I386-NETBSD:#define __INT16_MAX__ 32767
 // I386-NETBSD:#define __INT16_TYPE__ short
+// I386-NETBSD:#define __INT32_C(c) c
 // I386-NETBSD:#define __INT32_C_SUFFIX__
 // I386-NETBSD:#define __INT32_FMTd__ "d"
 // I386-NETBSD:#define __INT32_FMTi__ "i"
 // I386-NETBSD:#define __INT32_MAX__ 2147483647
 // I386-NETBSD:#define __INT32_TYPE__ int
+// I386-NETBSD:#define __INT64_C(c) c##LL
 // I386-NETBSD:#define __INT64_C_SUFFIX__ LL
 // I386-NETBSD:#define __INT64_FMTd__ "lld"
 // I386-NETBSD:#define __INT64_FMTi__ "lli"
 // I386-NETBSD:#define __INT64_MAX__ 9223372036854775807LL
 // I386-NETBSD:#define __INT64_TYPE__ long long int
+// I386-NETBSD:#define __INT8_C(c) c
 // I386-NETBSD:#define __INT8_C_SUFFIX__
 // I386-NETBSD:#define __INT8_FMTd__ "hhd"
 // I386-NETBSD:#define __INT8_FMTi__ "hhi"
 // I386-NETBSD:#define __INT8_MAX__ 127
 // I386-NETBSD:#define __INT8_TYPE__ signed char
+// I386-NETBSD:#define __INTMAX_C(c) c##LL
 // I386-NETBSD:#define __INTMAX_C_SUFFIX__ LL
 // I386-NETBSD:#define __INTMAX_FMTd__ "lld"
 // I386-NETBSD:#define __INTMAX_FMTi__ "lli"
@@ -542,18 +567,23 @@
 // I386-NETBSD:#define __SIZE_TYPE__ unsigned int
 // I386-NETBSD:#define __SIZE_WIDTH__ 32
 // I386-NETBSD-CXX:#define __STDCPP_DEFAULT_NEW_ALIGNMENT__ 4U
+// I386-NETBSD:#define __UINT16_C(c) c
 // I386-NETBSD:#define __UINT16_C_SUFFIX__
 // I386-NETBSD:#define __UINT16_MAX__ 65535
 // I386-NETBSD:#define __UINT16_TYPE__ unsigned short
+// I386-NETBSD:#define __UINT32_C(c) c##U
 // I386-NETBSD:#define __UINT32_C_SUFFIX__ U
 // I386-NETBSD:#define __UINT32_MAX__ 4294967295U
 // I386-NETBSD:#define __UINT32_TYPE__ unsigned int
+// I386-NETBSD:#define __UINT64_C(c) c##ULL
 // I386-NETBSD:#define __UINT64_C_SUFFIX__ ULL
 // I386-NETBSD:#define __UINT64_MAX__ 18446744073709551615ULL
 // I386-NETBSD:#define __UINT64_TYPE__ long long unsigned int
+// I386-NETBSD:#define __UINT8_C(c) c
 // I386-NETBSD:#define __UINT8_C_SUFFIX__
 // I386-NETBSD:#define __UINT8_MAX__ 255
 // I386-NETBSD:#define __UINT8_TYPE__ unsigned char
+// I386-NETBSD:#define __UINTMAX_C(c) c##ULL
 // I386-NETBSD:#define __UINTMAX_C_SUFFIX__ ULL
 // I386-NETBSD:#define __UINTMAX_MAX__ 18446744073709551615ULL
 // I386-NETBSD:#define __UINTMAX_TYPE__ long long unsigned int
@@ -636,26 +666,31 @@
 // X86_64:#define __FLT_MIN_EXP__ (-125)
 // X86_64:#define __FLT_MIN__ 1.17549435e-38F
 // X86_64:#define __FLT_RADIX__ 2
+// X86_64:#define __INT16_C(c) c
 // X86_64:#define __INT16_C_SUFFIX__
 // X86_64:#define __INT16_FMTd__ "hd"
 // X86_64:#define __INT16_FMTi__ "hi"
 // X86_64:#define __INT16_MAX__ 32767
 // X86_64:#define __INT16_TYPE__ short
+// X86_64:#define __INT32_C(c) c
 // X86_64:#define __INT32_C_SUFFIX__
 // X86_64:#define __INT32_FMTd__ "d"
 // X86_64:#define __INT32_FMTi__ "i"
 // X86_64:#define __INT32_MAX__ 2147483647
 // X86_64:#define __INT32_TYPE__ int
+// X86_64:#define __INT64_C(c) c##L
 // X86_64:#define __INT64_C_SUFFIX__ L
 // X86_64:#define __INT64_FMTd__ "ld"
 // X86_64:#define __INT64_FMTi__ "li"
 // X86_64:#define __INT64_MAX__ 9223372036854775807L
 // X86_64:#define __INT64_TYPE__ long int
+// X86_64:#define __INT8_C(c) c
 // X86_64:#define __INT8_C_SUFFIX__
 // X86_64:#define __INT8_FMTd__ "hhd"
 // X86_64:#define __INT8_FMTi__ "hhi"
 // X86_64:#define __INT8_MAX__ 127
 // X86_64:#define __INT8_TYPE__ signed char
+// X86_64:#define __INTMAX_C(c) c##L
 // X86_64:#define __INTMAX_C_SUFFIX__ L
 // X86_64:#define __INTMAX_FMTd__ "ld"
 // X86_64:#define __INTMAX_FMTi__ "li"
@@ -748,18 +783,23 @@
 // X86_64:#define __SSE_MATH__ 1
 // X86_64:#define __SSE__ 1
 // X86_64-CXX:#define __STDCPP_DEFAULT_NEW_ALIGNMENT__ 16UL
+// X86_64:#define __UINT16_C(c) c
 // X86_64:#define __UINT16_C_SUFFIX__
 // X86_64:#define __UINT16_MAX__ 65535
 // X86_64:#define __UINT16_TYPE__ unsigned short
+// X86_64:#define __UINT32_C(c) c##U
 // X86_64:#define __UINT32_C_SUFFIX__ U
 // X86_64:#define __UINT32_MAX__ 4294967295U
 // X86_64:#define __UINT32_TYPE__ unsigned int
+// X86_64:#define __UINT64_C(c) c##UL
 // X86_64:#define __UINT64_C_SUFFIX__ UL
 // X86_64:#define __UINT64_MAX__ 18446744073709551615UL
 // X86_64:#define __UINT64_TYPE__ long unsigned int
+// X86_64:#define __UINT8_C(c) c
 // X86_64:#define __UINT8_C_SUFFIX__
 // X86_64:#define __UINT8_MAX__ 255
 // X86_64:#define __UINT8_TYPE__ unsigned char
+// X86_64:#define __UINTMAX_C(c) c##UL
 // X86_64:#define __UINTMAX_C_SUFFIX__ UL
 // X86_64:#define __UINTMAX_MAX__ 18446744073709551615UL
 // X86_64:#define __UINTMAX_TYPE__ long unsigned int
@@ -842,26 +882,31 @@
 // X32:#define __FLT_RADIX__ 2
 // X32:#define __ILP32__ 1
 // X32-NOT:#define __LP64__ 1
+// X32:#define __INT16_C(c) c
 // X32:#define __INT16_C_SUFFIX__
 // X32:#define __INT16_FMTd__ "hd"
 // X32:#define __INT16_FMTi__ "hi"
 // X32:#define __INT16_MAX__ 32767
 // X32:#define __INT16_TYPE__ short
+// X32:#define __INT32_C(c) c
 // X32:#define __INT32_C_SUFFIX__
 // X32:#define __INT32_FMTd__ "d"
 // X32:#define __INT32_FMTi__ "i"
 // X32:#define __INT32_MAX__ 2147483647
 // X32:#define __INT32_TYPE__ int
+// X32:#define __INT64_C(c) c##LL
 // X32:#define __INT64_C_SUFFIX__ LL
 // X32:#define __INT64_FMTd__ "lld"
 // X32:#define __INT64_FMTi__ "lli"
 // X32:#define __INT64_MAX__ 9223372036854775807LL
 // X32:#define __INT64_TYPE__ long long int
+// X32:#define __INT8_C(c) c
 // X32:#define __INT8_C_SUFFIX__
 // X32:#define __INT8_FMTd__ "hhd"
 // X32:#define __INT8_FMTi__ "hhi"
 // X32:#define __INT8_MAX__ 127
 // X32:#define __INT8_TYPE__ signed char
+// X32:#define __INTMAX_C(c) c##LL
 // X32:#define __INTMAX_C_SUFFIX__ LL
 // X32:#define __INTMAX_FMTd__ "lld"
 // X32:#define __INTMAX_FMTi__ "lli"
@@ -952,18 +997,23 @@
 // X32:#define __SSE_MATH__ 1
 // X32:#define __SSE__ 1
 // X32-CXX:#define __STDCPP_DEFAULT_NEW_ALIGNMENT__ 16U
+// X32:#define __UINT16_C(c) c
 // X32:#define __UINT16_C_SUFFIX__
 // X32:#define __UINT16_MAX__ 65535
 // X32:#define __UINT16_TYPE__ unsigned short
+// X32:#define __UINT32_C(c) c##U
 // X32:#define __UINT32_C_SUFFIX__ U
 // X32:#define __UINT32_MAX__ 4294967295U
 // X32:#define __UINT32_TYPE__ unsigned int
+// X32:#define __UINT64_C(c) c##ULL
 // X32:#define __UINT64_C_SUFFIX__ ULL
 // X32:#define __UINT64_MAX__ 18446744073709551615ULL
 // X32:#define __UINT64_TYPE__ long long unsigned int
+// X32:#define __UINT8_C(c) c
 // X32:#define __UINT8_C_SUFFIX__
 // X32:#define __UINT8_MAX__ 255
 // X32:#define __UINT8_TYPE__ unsigned char
+// X32:#define __UINTMAX_C(c) c##ULL
 // X32:#define __UINTMAX_C_SUFFIX__ ULL
 // X32:#define __UINTMAX_MAX__ 18446744073709551615ULL
 // X32:#define __UINTMAX_TYPE__ long long unsigned int
@@ -1046,26 +1096,31 @@
 // X86_64-LINUX:#define __GCC_ATOMIC_SHORT_LOCK_FREE 2
 // X86_64-LINUX:#define __GCC_ATOMIC_TEST_AND_SET_TRUEVAL 1
 // X86_64-LINUX:#define __GCC_ATOMIC_WCHAR_T_LOCK_FREE 2
+// X86_64-LINUX:#define __INT16_C(c) c
 // X86_64-LINUX:#define __INT16_C_SUFFIX__
 // X86_64-LINUX:#define __INT16_FMTd__ "hd"
 // X86_64-LINUX:#define __INT16_FMTi__ "hi"
 // X86_64-LINUX:#define __INT16_MAX__ 32767
 // X86_64-LINUX:#define __INT16_TYPE__ short
+// X86_64-LINUX:#define __INT32_C(c) c
 // X86_64-LINUX:#define __INT32_C_SUFFIX__
 // X86_64-LINUX:#define __INT32_FMTd__ "d"
 // X86_64-LINUX:#define __INT32_FMTi__ "i"
 // X86_64-LINUX:#define __INT32_MAX__ 2147483647
 // X86_64-LINUX:#define __INT32_TYPE__ int
+// X86_64-LINUX:#define __INT64_C(c) c##L
 // X86_64-LINUX:#define __INT64_C_SUFFIX__ L
 // X86_64-LINUX:#define __INT64_FMTd__ "ld"
 // X86_64-LINUX:#define __INT64_FMTi__ "li"
 // X86_64-LINUX:#define __INT64_MAX__ 9223372036854775807L
 // X86_64-LINUX:#define __INT64_TYPE__ long int
+// X86_64-LINUX:#define __INT8_C(c) c
 // X86_64-LINUX:#define __INT8_C_SUFFIX__
 // X86_64-LINUX:#define __INT8_FMTd__ "hhd"
 // X86_64-LINUX:#define __INT8_FMTi__ "hhi"
 // X86_64-LINUX:#define __INT8_MAX__ 127
 // X86_64-LINUX:#define __INT8_TYPE__ signed char
+// X86_64-LINUX:#define __INTMAX_C(c) c##L
 // X86_64-LINUX:#define __INTMAX_C_SUFFIX__ L
 // X86_64-LINUX:#define __INTMAX_FMTd__ "ld"
 // X86_64-LINUX:#define __INTMAX_FMTi__ "li"
@@ -1156,18 +1211,23 @@
 // X86_64-LINUX:#define __SSE2__ 1
 // X86_64-LINUX:#define __SSE_MATH__ 1
 // X86_64-LINUX:#define __SSE__ 1
+// X86_64-LINUX:#define __UINT16_C(c) c
 // X86_64-LINUX:#define __UINT16_C_SUFFIX__
 // X86_64-LINUX:#define __UINT16_MAX__ 65535
 // X86_64-LINUX:#define __UINT16_TYPE__ unsigned short
+// X86_64-LINUX:#define __UINT32_C(c) c##U
 // X86_64-LINUX:#define __UINT32_C_SUFFIX__ U
 // X86_64-LINUX:#define __UINT32_MAX__ 4294967295U
 // X86_64-LINUX:#define __UINT32_TYPE__ unsigned int
+// X86_64-LINUX:#define __UINT64_C(c) c##UL
 // X86_64-LINUX:#define __UINT64_C_SUFFIX__ UL
 // X86_64-LINUX:#define __UINT64_MAX__ 18446744073709551615UL
 // X86_64-LINUX:#define __UINT64_TYPE__ long unsigned int
+// X86_64-LINUX:#define __UINT8_C(c) c
 // X86_64-LINUX:#define __UINT8_C_SUFFIX__
 // X86_64-LINUX:#define __UINT8_MAX__ 255
 // X86_64-LINUX:#define __UINT8_TYPE__ unsigned char
+// X86_64-LINUX:#define __UINTMAX_C(c) c##UL
 // X86_64-LINUX:#define __UINTMAX_C_SUFFIX__ UL
 // X86_64-LINUX:#define __UINTMAX_MAX__ 18446744073709551615UL
 // X86_64-LINUX:#define __UINTMAX_TYPE__ long unsigned int
@@ -1258,26 +1318,31 @@
 // X86_64-NETBSD:#define __GCC_ATOMIC_SHORT_LOCK_FREE 2
 // X86_64-NETBSD:#define __GCC_ATOMIC_TEST_AND_SET_TRUEVAL 1
 // X86_64-NETBSD:#define __GCC_ATOMIC_WCHAR_T_LOCK_FREE 2
+// X86_64-NETBSD:#define __INT16_C(c) c
 // X86_64-NETBSD:#define __INT16_C_SUFFIX__
 // X86_64-NETBSD:#define __INT16_FMTd__ "hd"
 // X86_64-NETBSD:#define __INT16_FMTi__ "hi"
 // X86_64-NETBSD:#define __INT16_MAX__ 32767
 // X86_64-NETBSD:#define __INT16_TYPE__ short
+// X86_64-NETBSD:#define __INT32_C(c) c
 // X86_64-NETBSD:#define __INT32_C_SUFFIX__
 // X86_64-NETBSD:#define __INT32_FMTd__ "d"
 // X86_64-NETBSD:#define __INT32_FMTi__ "i"
 // X86_64-NETBSD:#define __INT32_MAX__ 2147483647
 // X86_64-NETBSD:#define __INT32_TYPE__ int
+// X86_64-NETBSD:#define __INT64_C(c) c##L
 // X86_64-NETBSD:#define __INT64_C_SUFFIX__ L
 // X86_64-NETBSD:#define __INT64_FMTd__ "ld"
 // X86_64-NETBSD:#define __INT64_FMTi__ "li"
 // X86_64-NETBSD:#define __INT64_MAX__ 9223372036854775807L
 // X86_64-NETBSD:#define __INT64_TYPE__ long int
+// X86_64-NETBSD:#define __INT8_C(c) c
 // X86_64-NETBSD:#define __INT8_C_SUFFIX__
 // X86_64-NETBSD:#define __INT8_FMTd__ "hhd"
 // X86_64-NETBSD:#define __INT8_FMTi__ "hhi"
 // X86_64-NETBSD:#define __INT8_MAX__ 127
 // X86_64-NETBSD:#define __INT8_TYPE__ signed char
+// X86_64-NETBSD:#define __INTMAX_C(c) c##L
 // X86_64-NETBSD:#define __INTMAX_C_SUFFIX__ L
 // X86_64-NETBSD:#define __INTMAX_FMTd__ "ld"
 // X86_64-NETBSD:#define __INTMAX_FMTi__ "li"
@@ -1368,18 +1433,23 @@
 // X86_64-NETBSD:#define __SSE2__ 1
 // X86_64-NETBSD:#define __SSE_MATH__ 1
 // X86_64-NETBSD:#define __SSE__ 1
+// X86_64-NETBSD:#define __UINT16_C(c) c
 // X86_64-NETBSD:#define __UINT16_C_SUFFIX__
 // X86_64-NETBSD:#define __UINT16_MAX__ 65535
 // X86_64-NETBSD:#define __UINT16_TYPE__ unsigned short
+// X86_64-NETBSD:#define __UINT32_C(c) c##U
 // X86_64-NETBSD:#define __UINT32_C_SUFFIX__ U
 // X86_64-NETBSD:#define __UINT32_MAX__ 4294967295U
 // X86_64-NETBSD:#define __UINT32_TYPE__ unsigned int
+// X86_64-NETBSD:#define __UINT64_C(c) c##UL
 // X86_64-NETBSD:#define __UINT64_C_SUFFIX__ UL
 // X86_64-NETBSD:#define __UINT64_MAX__ 18446744073709551615UL
 // X86_64-NETBSD:#define __UINT64_TYPE__ long unsigned int
+// X86_64-NETBSD:#define __UINT8_C(c) c
 // X86_64-NETBSD:#define __UINT8_C_SUFFIX__
 // X86_64-NETBSD:#define __UINT8_MAX__ 255
 // X86_64-NETBSD:#define __UINT8_TYPE__ unsigned char
+// X86_64-NETBSD:#define __UINTMAX_C(c) c##UL
 // X86_64-NETBSD:#define __UINTMAX_C_SUFFIX__ UL
 // X86_64-NETBSD:#define __UINTMAX_MAX__ 18446744073709551615UL
 // X86_64-NETBSD:#define __UINTMAX_TYPE__ long unsigned int
diff --git a/clang/test/Preprocessor/init.c b/clang/test/Preprocessor/init.c
index 5999b9c1d1bc3..1ac325d444662 100644
--- a/clang/test/Preprocessor/init.c
+++ b/clang/test/Preprocessor/init.c
@@ -426,26 +426,31 @@
 // MSP430:#define __FLT_MIN_EXP__ (-125)
 // MSP430:#define __FLT_MIN__ 1.17549435e-38F
 // MSP430:#define __FLT_RADIX__ 2
+// MSP430:#define __INT16_C(c) c
 // MSP430:#define __INT16_C_SUFFIX__
 // MSP430:#define __INT16_FMTd__ "hd"
 // MSP430:#define __INT16_FMTi__ "hi"
 // MSP430:#define __INT16_MAX__ 32767
 // MSP430:#define __INT16_TYPE__ short
+// MSP430:#define __INT32_C(c) c##L
 // MSP430:#define __INT32_C_SUFFIX__ L
 // MSP430:#define __INT32_FMTd__ "ld"
 // MSP430:#define __INT32_FMTi__ "li"
 // MSP430:#define __INT32_MAX__ 2147483647L
 // MSP430:#define __INT32_TYPE__ long int
+// MSP430:#define __INT64_C(c) c##LL
 // MSP430:#define __INT64_C_SUFFIX__ LL
 // MSP430:#define __INT64_FMTd__ "lld"
 // MSP430:#define __INT64_FMTi__ "lli"
 // MSP430:#define __INT64_MAX__ 9223372036854775807LL
 // MSP430:#define __INT64_TYPE__ long long int
+// MSP430:#define __INT8_C(c) c
 // MSP430:#define __INT8_C_SUFFIX__
 // MSP430:#define __INT8_FMTd__ "hhd"
 // MSP430:#define __INT8_FMTi__ "hhi"
 // MSP430:#define __INT8_MAX__ 127
 // MSP430:#define __INT8_TYPE__ signed char
+// MSP430:#define __INTMAX_C(c) c##LL
 // MSP430:#define __INTMAX_C_SUFFIX__ LL
 // MSP430:#define __INTMAX_FMTd__ "lld"
 // MSP430:#define __INTMAX_FMTi__ "lli"
@@ -531,18 +536,23 @@
 // MSP430:#define __SIZE_TYPE__ unsigned int
 // MSP430:#define __SIZE_WIDTH__ 16
 // MSP430-CXX:#define __STDCPP_DEFAULT_NEW_ALIGNMENT__ 2U
+// MSP430:#define __UINT16_C(c) c##U
 // MSP430:#define __UINT16_C_SUFFIX__ U
 // MSP430:#define __UINT16_MAX__ 65535U
 // MSP430:#define __UINT16_TYPE__ unsigned short
+// MSP430:#define __UINT32_C(c) c##UL
 // MSP430:#define __UINT32_C_SUFFIX__ UL
 // MSP430:#define __UINT32_MAX__ 4294967295UL
 // MSP430:#define __UINT32_TYPE__ long unsigned int
+// MSP430:#define __UINT64_C(c) c##ULL
 // MSP430:#define __UINT64_C_SUFFIX__ ULL
 // MSP430:#define __UINT64_MAX__ 18446744073709551615ULL
 // MSP430:#define __UINT64_TYPE__ long long unsigned int
+// MSP430:#define __UINT8_C(c) c
 // MSP430:#define __UINT8_C_SUFFIX__
 // MSP430:#define __UINT8_MAX__ 255
 // MSP430:#define __UINT8_TYPE__ unsigned char
+// MSP430:#define __UINTMAX_C(c) c##ULL
 // MSP430:#define __UINTMAX_C_SUFFIX__ ULL
 // MSP430:#define __UINTMAX_MAX__ 18446744073709551615ULL
 // MSP430:#define __UINTMAX_TYPE__ long long unsigned int
@@ -613,26 +623,31 @@
 // NVPTX32:#define __FLT_MIN_EXP__ (-125)
 // NVPTX32:#define __FLT_MIN__ 1.17549435e-38F
 // NVPTX32:#define __FLT_RADIX__ 2
+// NVPTX32:#define __INT16_C(c) c
 // NVPTX32:#define __INT16_C_SUFFIX__
 // NVPTX32:#define __INT16_FMTd__ "hd"
 // NVPTX32:#define __INT16_FMTi__ "hi"
 // NVPTX32:#define __INT16_MAX__ 32767
 // NVPTX32:#define __INT16_TYPE__ short
+// NVPTX32:#define __INT32_C(c) c
 // NVPTX32:#define __INT32_C_SUFFIX__
 // NVPTX32:#define __INT32_FMTd__ "d"
 // NVPTX32:#define __INT32_FMTi__ "i"
 // NVPTX32:#define __INT32_MAX__ 2147483647
 // NVPTX32:#define __INT32_TYPE__ int
+// NVPTX32:#define __INT64_C(c) c##LL
 // NVPTX32:#define __INT64_C_SUFFIX__ LL
 // NVPTX32:#define __INT64_FMTd__ "lld"
 // NVPTX32:#define __INT64_FMTi__ "lli"
 // NVPTX32:#define __INT64_MAX__ 9223372036854775807LL
 // NVPTX32:#define __INT64_TYPE__ long long int
+// NVPTX32:#define __INT8_C(c) c
 // NVPTX32:#define __INT8_C_SUFFIX__
 // NVPTX32:#define __INT8_FMTd__ "hhd"
 // NVPTX32:#define __INT8_FMTi__ "hhi"
 // NVPTX32:#define __INT8_MAX__ 127
 // NVPTX32:#define __INT8_TYPE__ signed char
+// NVPTX32:#define __INTMAX_C(c) c##LL
 // NVPTX32:#define __INTMAX_C_SUFFIX__ LL
 // NVPTX32:#define __INTMAX_FMTd__ "lld"
 // NVPTX32:#define __INTMAX_FMTi__ "lli"
@@ -720,18 +735,23 @@
 // NVPTX32:#define __SIZE_TYPE__ unsigned int
 // NVPTX32:#define __SIZE_WIDTH__ 32
 // NVPTX32-CXX:#define __STDCPP_DEFAULT_NEW_ALIGNMENT__ 8U
+// NVPTX32:#define __UINT16_C(c) c
 // NVPTX32:#define __UINT16_C_SUFFIX__
 // NVPTX32:#define __UINT16_MAX__ 65535
 // NVPTX32:#define __UINT16_TYPE__ unsigned short
+// NVPTX32:#define __UINT32_C(c) c##U
 // NVPTX32:#define __UINT32_C_SUFFIX__ U
 // NVPTX32:#define __UINT32_MAX__ 4294967295U
 // NVPTX32:#define __UINT32_TYPE__ unsigned int
+// NVPTX32:#define __UINT64_C(c) c##ULL
 // NVPTX32:#define __UINT64_C_SUFFIX__ ULL
 // NVPTX32:#define __UINT64_MAX__ 18446744073709551615ULL
 // NVPTX32:#define __UINT64_TYPE__ long long unsigned int
+// NVPTX32:#define __UINT8_C(c) c
 // NVPTX32:#define __UINT8_C_SUFFIX__
 // NVPTX32:#define __UINT8_MAX__ 255
 // NVPTX32:#define __UINT8_TYPE__ unsigned char
+// NVPTX32:#define __UINTMAX_C(c) c##ULL
 // NVPTX32:#define __UINTMAX_C_SUFFIX__ ULL
 // NVPTX32:#define __UINTMAX_MAX__ 18446744073709551615ULL
 // NVPTX32:#define __UINTMAX_TYPE__ long long unsigned int
@@ -801,26 +821,31 @@
 // NVPTX64:#define __FLT_MIN_EXP__ (-125)
 // NVPTX64:#define __FLT_MIN__ 1.17549435e-38F
 // NVPTX64:#define __FLT_RADIX__ 2
+// NVPTX64:#define __INT16_C(c) c
 // NVPTX64:#define __INT16_C_SUFFIX__
 // NVPTX64:#define __INT16_FMTd__ "hd"
 // NVPTX64:#define __INT16_FMTi__ "hi"
 // NVPTX64:#define __INT16_MAX__ 32767
 // NVPTX64:#define __INT16_TYPE__ short
+// NVPTX64:#define __INT32_C(c) c
 // NVPTX64:#define __INT32_C_SUFFIX__
 // NVPTX64:#define __INT32_FMTd__ "d"
 // NVPTX64:#define __INT32_FMTi__ "i"
 // NVPTX64:#define __INT32_MAX__ 2147483647
 // NVPTX64:#define __INT32_TYPE__ int
+// NVPTX64:#define __INT64_C(c) c##LL
 // NVPTX64:#define __INT64_C_SUFFIX__ LL
 // NVPTX64:#define __INT64_FMTd__ "lld"
 // NVPTX64:#define __INT64_FMTi__ "lli"
 // NVPTX64:#define __INT64_MAX__ 9223372036854775807LL
 // NVPTX64:#define __INT64_TYPE__ long long int
+// NVPTX64:#define __INT8_C(c) c
 // NVPTX64:#define __INT8_C_SUFFIX__
 // NVPTX64:#define __INT8_FMTd__ "hhd"
 // NVPTX64:#define __INT8_FMTi__ "hhi"
 // NVPTX64:#define __INT8_MAX__ 127
 // NVPTX64:#define __INT8_TYPE__ signed char
+// NVPTX64:#define __INTMAX_C(c) c##LL
 // NVPTX64:#define __INTMAX_C_SUFFIX__ LL
 // NVPTX64:#define __INTMAX_FMTd__ "lld"
 // NVPTX64:#define __INTMAX_FMTi__ "lli"
@@ -908,18 +933,23 @@
 // NVPTX64:#define __SIZE_TYPE__ long unsigned int
 // NVPTX64:#define __SIZE_WIDTH__ 64
 // NVPTX64-CXX:#define __STDCPP_DEFAULT_NEW_ALIGNMENT__ 8UL
+// NVPTX64:#define __UINT16_C(c) c
 // NVPTX64:#define __UINT16_C_SUFFIX__
 // NVPTX64:#define __UINT16_MAX__ 65535
 // NVPTX64:#define __UINT16_TYPE__ unsigned short
+// NVPTX64:#define __UINT32_C(c) c##U
 // NVPTX64:#define __UINT32_C_SUFFIX__ U
 // NVPTX64:#define __UINT32_MAX__ 4294967295U
 // NVPTX64:#define __UINT32_TYPE__ unsigned int
+// NVPTX64:#define __UINT64_C(c) c##ULL
 // NVPTX64:#define __UINT64_C_SUFFIX__ ULL
 // NVPTX64:#define __UINT64_MAX__ 18446744073709551615ULL
 // NVPTX64:#define __UINT64_TYPE__ long long unsigned int
+// NVPTX64:#define __UINT8_C(c) c
 // NVPTX64:#define __UINT8_C_SUFFIX__
 // NVPTX64:#define __UINT8_MAX__ 255
 // NVPTX64:#define __UINT8_TYPE__ unsigned char
+// NVPTX64:#define __UINTMAX_C(c) c##ULL
 // NVPTX64:#define __UINTMAX_C_SUFFIX__ ULL
 // NVPTX64:#define __UINTMAX_MAX__ 18446744073709551615ULL
 // NVPTX64:#define __UINTMAX_TYPE__ long long unsigned int
@@ -1003,26 +1033,31 @@
 // SPARC:#define __FLT_MIN__ 1.17549435e-38F
 // SPARC:#define __FLT_RADIX__ 2
 // SPARC:#define __GCC_ATOMIC_LLONG_LOCK_FREE 1
+// SPARC:#define __INT16_C(c) c
 // SPARC:#define __INT16_C_SUFFIX__
 // SPARC:#define __INT16_FMTd__ "hd"
 // SPARC:#define __INT16_FMTi__ "hi"
 // SPARC:#define __INT16_MAX__ 32767
 // SPARC:#define __INT16_TYPE__ short
+// SPARC:#define __INT32_C(c) c
 // SPARC:#define __INT32_C_SUFFIX__
 // SPARC:#define __INT32_FMTd__ "d"
 // SPARC:#define __INT32_FMTi__ "i"
 // SPARC:#define __INT32_MAX__ 2147483647
 // SPARC:#define __INT32_TYPE__ int
+// SPARC:#define __INT64_C(c) c##LL
 // SPARC:#define __INT64_C_SUFFIX__ LL
 // SPARC:#define __INT64_FMTd__ "lld"
 // SPARC:#define __INT64_FMTi__ "lli"
 // SPARC:#define __INT64_MAX__ 9223372036854775807LL
 // SPARC:#define __INT64_TYPE__ long long int
+// SPARC:#define __INT8_C(c) c
 // SPARC:#define __INT8_C_SUFFIX__
 // SPARC:#define __INT8_FMTd__ "hhd"
 // SPARC:#define __INT8_FMTi__ "hhi"
 // SPARC:#define __INT8_MAX__ 127
 // SPARC:#define __INT8_TYPE__ signed char
+// SPARC:#define __INTMAX_C(c) c##LL
 // SPARC:#define __INTMAX_C_SUFFIX__ LL
 // SPARC:#define __INTMAX_FMTd__ "lld"
 // SPARC:#define __INTMAX_FMTi__ "lli"
@@ -1114,18 +1149,23 @@
 // SPARC-NETOPENBSD:#define __SIZE_TYPE__ long unsigned int
 // SPARC:#define __SIZE_WIDTH__ 32
 // SPARC-DEFAULT-CXX:#define __STDCPP_DEFAULT_NEW_ALIGNMENT__ 8U
+// SPARC:#define __UINT16_C(c) c
 // SPARC:#define __UINT16_C_SUFFIX__
 // SPARC:#define __UINT16_MAX__ 65535
 // SPARC:#define __UINT16_TYPE__ unsigned short
+// SPARC:#define __UINT32_C(c) c##U
 // SPARC:#define __UINT32_C_SUFFIX__ U
 // SPARC:#define __UINT32_MAX__ 4294967295U
 // SPARC:#define __UINT32_TYPE__ unsigned int
+// SPARC:#define __UINT64_C(c) c##ULL
 // SPARC:#define __UINT64_C_SUFFIX__ ULL
 // SPARC:#define __UINT64_MAX__ 18446744073709551615ULL
 // SPARC:#define __UINT64_TYPE__ long long unsigned int
+// SPARC:#define __UINT8_C(c) c
 // SPARC:#define __UINT8_C_SUFFIX__
 // SPARC:#define __UINT8_MAX__ 255
 // SPARC:#define __UINT8_TYPE__ unsigned char
+// SPARC:#define __UINTMAX_C(c) c##ULL
 // SPARC:#define __UINTMAX_C_SUFFIX__ ULL
 // SPARC:#define __UINTMAX_MAX__ 18446744073709551615ULL
 // SPARC:#define __UINTMAX_TYPE__ long long unsigned int
@@ -1201,21 +1241,25 @@
 // TCE:#define __FLT_MIN_EXP__ (-125)
 // TCE:#define __FLT_MIN__ 1.17549435e-38F
 // TCE:#define __FLT_RADIX__ 2
+// TCE:#define __INT16_C(c) c
 // TCE:#define __INT16_C_SUFFIX__
 // TCE:#define __INT16_FMTd__ "hd"
 // TCE:#define __INT16_FMTi__ "hi"
 // TCE:#define __INT16_MAX__ 32767
 // TCE:#define __INT16_TYPE__ short
+// TCE:#define __INT32_C(c) c
 // TCE:#define __INT32_C_SUFFIX__
 // TCE:#define __INT32_FMTd__ "d"
 // TCE:#define __INT32_FMTi__ "i"
 // TCE:#define __INT32_MAX__ 2147483647
 // TCE:#define __INT32_TYPE__ int
+// TCE:#define __INT8_C(c) c
 // TCE:#define __INT8_C_SUFFIX__
 // TCE:#define __INT8_FMTd__ "hhd"
 // TCE:#define __INT8_FMTi__ "hhi"
 // TCE:#define __INT8_MAX__ 127
 // TCE:#define __INT8_TYPE__ signed char
+// TCE:#define __INTMAX_C(c) c##L
 // TCE:#define __INTMAX_C_SUFFIX__ L
 // TCE:#define __INTMAX_FMTd__ "ld"
 // TCE:#define __INTMAX_FMTi__ "li"
@@ -1293,15 +1337,19 @@
 // TCE-CXX:#define __STDCPP_DEFAULT_NEW_ALIGNMENT__ 4U
 // TCE:#define __TCE_V1__ 1
 // TCE:#define __TCE__ 1
+// TCE:#define __UINT16_C(c) c
 // TCE:#define __UINT16_C_SUFFIX__
 // TCE:#define __UINT16_MAX__ 65535
 // TCE:#define __UINT16_TYPE__ unsigned short
+// TCE:#define __UINT32_C(c) c##U
 // TCE:#define __UINT32_C_SUFFIX__ U
 // TCE:#define __UINT32_MAX__ 4294967295U
 // TCE:#define __UINT32_TYPE__ unsigned int
+// TCE:#define __UINT8_C(c) c
 // TCE:#define __UINT8_C_SUFFIX__
 // TCE:#define __UINT8_MAX__ 255
 // TCE:#define __UINT8_TYPE__ unsigned char
+// TCE:#define __UINTMAX_C(c) c##UL
 // TCE:#define __UINTMAX_C_SUFFIX__ UL
 // TCE:#define __UINTMAX_MAX__ 4294967295UL
 // TCE:#define __UINTMAX_TYPE__ long unsigned int
@@ -1373,6 +1421,7 @@
 // PS4:#define __FreeBSD_cc_version 900001
 // PS4:#define __INT16_TYPE__ short
 // PS4:#define __INT32_TYPE__ int
+// PS4:#define __INT64_C(c) c##L
 // PS4:#define __INT64_C_SUFFIX__ L
 // PS4:#define __INT64_TYPE__ long int
 // PS4:#define __INT8_TYPE__ signed char
@@ -1464,6 +1513,7 @@
 // RUN: %clang_cc1 -E -dM -ffreestanding -triple=sparc64-none-none < /dev/null | FileCheck -match-full-lines -check-prefix SPARCV9 %s
 // SPARCV9:#define __BIGGEST_ALIGNMENT__ 16
 // SPARCV9:#define __INT64_TYPE__ long int
+// SPARCV9:#define __INTMAX_C(c) c##L
 // SPARCV9:#define __INTMAX_C_SUFFIX__ L
 // SPARCV9:#define __INTMAX_TYPE__ long int
 // SPARCV9:#define __INTPTR_TYPE__ long int
@@ -1475,8 +1525,10 @@
 //
 // RUN: %clang_cc1 -E -dM -ffreestanding -triple=sparc64-none-openbsd < /dev/null | FileCheck -match-full-lines -check-prefix SPARC64-OBSD %s
 // SPARC64-OBSD:#define __INT64_TYPE__ long long int
+// SPARC64-OBSD:#define __INTMAX_C(c) c##LL
 // SPARC64-OBSD:#define __INTMAX_C_SUFFIX__ LL
 // SPARC64-OBSD:#define __INTMAX_TYPE__ long long int
+// SPARC64-OBSD:#define __UINTMAX_C(c) c##ULL
 // SPARC64-OBSD:#define __UINTMAX_C_SUFFIX__ ULL
 // SPARC64-OBSD:#define __UINTMAX_TYPE__ long long unsigned int
 //
@@ -1720,26 +1772,31 @@
 // WEBASSEMBLY-NEXT:#define __GXX_ABI_VERSION 1002
 // WEBASSEMBLY32-NEXT:#define __ILP32__ 1
 // WEBASSEMBLY64-NOT:#define __ILP32__
+// WEBASSEMBLY-NEXT:#define __INT16_C(c) c
 // WEBASSEMBLY-NEXT:#define __INT16_C_SUFFIX__
 // WEBASSEMBLY-NEXT:#define __INT16_FMTd__ "hd"
 // WEBASSEMBLY-NEXT:#define __INT16_FMTi__ "hi"
 // WEBASSEMBLY-NEXT:#define __INT16_MAX__ 32767
 // WEBASSEMBLY-NEXT:#define __INT16_TYPE__ short
+// WEBASSEMBLY-NEXT:#define __INT32_C(c) c
 // WEBASSEMBLY-NEXT:#define __INT32_C_SUFFIX__
 // WEBASSEMBLY-NEXT:#define __INT32_FMTd__ "d"
 // WEBASSEMBLY-NEXT:#define __INT32_FMTi__ "i"
 // WEBASSEMBLY-NEXT:#define __INT32_MAX__ 2147483647
 // WEBASSEMBLY-NEXT:#define __INT32_TYPE__ int
+// WEBASSEMBLY-NEXT:#define __INT64_C(c) c##LL
 // WEBASSEMBLY-NEXT:#define __INT64_C_SUFFIX__ LL
 // WEBASSEMBLY-NEXT:#define __INT64_FMTd__ "lld"
 // WEBASSEMBLY-NEXT:#define __INT64_FMTi__ "lli"
 // WEBASSEMBLY-NEXT:#define __INT64_MAX__ 9223372036854775807LL
 // WEBASSEMBLY-NEXT:#define __INT64_TYPE__ long long int
+// WEBASSEMBLY-NEXT:#define __INT8_C(c) c
 // WEBASSEMBLY-NEXT:#define __INT8_C_SUFFIX__
 // WEBASSEMBLY-NEXT:#define __INT8_FMTd__ "hhd"
 // WEBASSEMBLY-NEXT:#define __INT8_FMTi__ "hhi"
 // WEBASSEMBLY-NEXT:#define __INT8_MAX__ 127
 // WEBASSEMBLY-NEXT:#define __INT8_TYPE__ signed char
+// WEBASSEMBLY-NEXT:#define __INTMAX_C(c) c##LL
 // WEBASSEMBLY-NEXT:#define __INTMAX_C_SUFFIX__ LL
 // WEBASSEMBLY-NEXT:#define __INTMAX_FMTd__ "lld"
 // WEBASSEMBLY-NEXT:#define __INTMAX_FMTi__ "lli"
@@ -1892,6 +1949,7 @@
 // WEBASSEMBLY-NEXT:#define __STDC_UTF_32__ 1
 // WEBASSEMBLY-NEXT:#define __STDC_VERSION__ 201710L
 // WEBASSEMBLY-NEXT:#define __STDC__ 1
+// WEBASSEMBLY-NEXT:#define __UINT16_C(c) c
 // WEBASSEMBLY-NEXT:#define __UINT16_C_SUFFIX__
 // WEBASSEMBLY-NEXT:#define __UINT16_FMTX__ "hX"
 // WEBASSEMBLY-NEXT:#define __UINT16_FMTo__ "ho"
@@ -1899,6 +1957,7 @@
 // WEBASSEMBLY-NEXT:#define __UINT16_FMTx__ "hx"
 // WEBASSEMBLY-NEXT:#define __UINT16_MAX__ 65535
 // WEBASSEMBLY-NEXT:#define __UINT16_TYPE__ unsigned short
+// WEBASSEMBLY-NEXT:#define __UINT32_C(c) c##U
 // WEBASSEMBLY-NEXT:#define __UINT32_C_SUFFIX__ U
 // WEBASSEMBLY-NEXT:#define __UINT32_FMTX__ "X"
 // WEBASSEMBLY-NEXT:#define __UINT32_FMTo__ "o"
@@ -1906,6 +1965,7 @@
 // WEBASSEMBLY-NEXT:#define __UINT32_FMTx__ "x"
 // WEBASSEMBLY-NEXT:#define __UINT32_MAX__ 4294967295U
 // WEBASSEMBLY-NEXT:#define __UINT32_TYPE__ unsigned int
+// WEBASSEMBLY-NEXT:#define __UINT64_C(c) c##ULL
 // WEBASSEMBLY-NEXT:#define __UINT64_C_SUFFIX__ ULL
 // WEBASSEMBLY-NEXT:#define __UINT64_FMTX__ "llX"
 // WEBASSEMBLY-NEXT:#define __UINT64_FMTo__ "llo"
@@ -1913,6 +1973,7 @@
 // WEBASSEMBLY-NEXT:#define __UINT64_FMTx__ "llx"
 // WEBASSEMBLY-NEXT:#define __UINT64_MAX__ 18446744073709551615ULL
 // WEBASSEMBLY-NEXT:#define __UINT64_TYPE__ long long unsigned int
+// WEBASSEMBLY-NEXT:#define __UINT8_C(c) c
 // WEBASSEMBLY-NEXT:#define __UINT8_C_SUFFIX__
 // WEBASSEMBLY-NEXT:#define __UINT8_FMTX__ "hhX"
 // WEBASSEMBLY-NEXT:#define __UINT8_FMTo__ "hho"
@@ -1920,6 +1981,7 @@
 // WEBASSEMBLY-NEXT:#define __UINT8_FMTx__ "hhx"
 // WEBASSEMBLY-NEXT:#define __UINT8_MAX__ 255
 // WEBASSEMBLY-NEXT:#define __UINT8_TYPE__ unsigned char
+// WEBASSEMBLY-NEXT:#define __UINTMAX_C(c) c##ULL
 // WEBASSEMBLY-NEXT:#define __UINTMAX_C_SUFFIX__ ULL
 // WEBASSEMBLY-NEXT:#define __UINTMAX_FMTX__ "llX"
 // WEBASSEMBLY-NEXT:#define __UINTMAX_FMTo__ "llo"
@@ -2092,18 +2154,23 @@
 // AVR:#define __GCC_ATOMIC_TEST_AND_SET_TRUEVAL 1
 // AVR:#define __GCC_ATOMIC_WCHAR_T_LOCK_FREE 1
 // AVR:#define __GXX_ABI_VERSION 1002
+// AVR:#define __INT16_C(c) c
 // AVR:#define __INT16_C_SUFFIX__
 // AVR:#define __INT16_MAX__ 32767
 // AVR:#define __INT16_TYPE__ int
+// AVR:#define __INT32_C(c) c##L
 // AVR:#define __INT32_C_SUFFIX__ L
 // AVR:#define __INT32_MAX__ 2147483647L
 // AVR:#define __INT32_TYPE__ long int
+// AVR:#define __INT64_C(c) c##LL
 // AVR:#define __INT64_C_SUFFIX__ LL
 // AVR:#define __INT64_MAX__ 9223372036854775807LL
 // AVR:#define __INT64_TYPE__ long long int
+// AVR:#define __INT8_C(c) c
 // AVR:#define __INT8_C_SUFFIX__
 // AVR:#define __INT8_MAX__ 127
 // AVR:#define __INT8_TYPE__ signed char
+// AVR:#define __INTMAX_C(c) c##LL
 // AVR:#define __INTMAX_C_SUFFIX__ LL
 // AVR:#define __INTMAX_MAX__ 9223372036854775807LL
 // AVR:#define __INTMAX_TYPE__ long long int
@@ -2175,15 +2242,19 @@
 // AVR:#define __STDC__ 1
 // AVR:#define __UINT16_MAX__ 65535U
 // AVR:#define __UINT16_TYPE__ unsigned int
+// AVR:#define __UINT32_C(c) c##UL
 // AVR:#define __UINT32_C_SUFFIX__ UL
 // AVR:#define __UINT32_MAX__ 4294967295UL
 // AVR:#define __UINT32_TYPE__ long unsigned int
+// AVR:#define __UINT64_C(c) c##ULL
 // AVR:#define __UINT64_C_SUFFIX__ ULL
 // AVR:#define __UINT64_MAX__ 18446744073709551615ULL
 // AVR:#define __UINT64_TYPE__ long long unsigned int
+// AVR:#define __UINT8_C(c) c
 // AVR:#define __UINT8_C_SUFFIX__
 // AVR:#define __UINT8_MAX__ 255
 // AVR:#define __UINT8_TYPE__ unsigned char
+// AVR:#define __UINTMAX_C(c) c##ULL
 // AVR:#define __UINTMAX_C_SUFFIX__ ULL
 // AVR:#define __UINTMAX_MAX__ 18446744073709551615ULL
 // AVR:#define __UINTMAX_TYPE__ long long unsigned int
@@ -2383,18 +2454,23 @@
 // RISCV32: #define __GNUC__ {{.*}}
 // RISCV32: #define __GXX_ABI_VERSION {{.*}}
 // RISCV32: #define __ILP32__ 1
+// RISCV32: #define __INT16_C(c) c
 // RISCV32: #define __INT16_C_SUFFIX__
 // RISCV32: #define __INT16_MAX__ 32767
 // RISCV32: #define __INT16_TYPE__ short
+// RISCV32: #define __INT32_C(c) c
 // RISCV32: #define __INT32_C_SUFFIX__
 // RISCV32: #define __INT32_MAX__ 2147483647
 // RISCV32: #define __INT32_TYPE__ int
+// RISCV32: #define __INT64_C(c) c##LL
 // RISCV32: #define __INT64_C_SUFFIX__ LL
 // RISCV32: #define __INT64_MAX__ 9223372036854775807LL
 // RISCV32: #define __INT64_TYPE__ long long int
+// RISCV32: #define __INT8_C(c) c
 // RISCV32: #define __INT8_C_SUFFIX__
 // RISCV32: #define __INT8_MAX__ 127
 // RISCV32: #define __INT8_TYPE__ signed char
+// RISCV32: #define __INTMAX_C(c) c##LL
 // RISCV32: #define __INTMAX_C_SUFFIX__ LL
 // RISCV32: #define __INTMAX_MAX__ 9223372036854775807LL
 // RISCV32: #define __INTMAX_TYPE__ long long int
@@ -2474,18 +2550,23 @@
 // RISCV32: #define __STDC_UTF_32__ 1
 // RISCV32: #define __STDC_VERSION__ 201710L
 // RISCV32: #define __STDC__ 1
+// RISCV32: #define __UINT16_C(c) c
 // RISCV32: #define __UINT16_C_SUFFIX__
 // RISCV32: #define __UINT16_MAX__ 65535
 // RISCV32: #define __UINT16_TYPE__ unsigned short
+// RISCV32: #define __UINT32_C(c) c##U
 // RISCV32: #define __UINT32_C_SUFFIX__ U
 // RISCV32: #define __UINT32_MAX__ 4294967295U
 // RISCV32: #define __UINT32_TYPE__ unsigned int
+// RISCV32: #define __UINT64_C(c) c##ULL
 // RISCV32: #define __UINT64_C_SUFFIX__ ULL
 // RISCV32: #define __UINT64_MAX__ 18446744073709551615ULL
 // RISCV32: #define __UINT64_TYPE__ long long unsigned int
+// RISCV32: #define __UINT8_C(c) c
 // RISCV32: #define __UINT8_C_SUFFIX__
 // RISCV32: #define __UINT8_MAX__ 255
 // RISCV32: #define __UINT8_TYPE__ unsigned char
+// RISCV32: #define __UINTMAX_C(c) c##ULL
 // RISCV32: #define __UINTMAX_C_SUFFIX__ ULL
 // RISCV32: #define __UINTMAX_MAX__ 18446744073709551615ULL
 // RISCV32: #define __UINTMAX_TYPE__ long long unsigned int
@@ -2596,18 +2677,23 @@
 // RISCV64: #define __GNUC_STDC_INLINE__ 1
 // RISCV64: #define __GNUC__ {{.*}}
 // RISCV64: #define __GXX_ABI_VERSION {{.*}}
+// RISCV64: #define __INT16_C(c) c
 // RISCV64: #define __INT16_C_SUFFIX__
 // RISCV64: #define __INT16_MAX__ 32767
 // RISCV64: #define __INT16_TYPE__ short
+// RISCV64: #define __INT32_C(c) c
 // RISCV64: #define __INT32_C_SUFFIX__
 // RISCV64: #define __INT32_MAX__ 2147483647
 // RISCV64: #define __INT32_TYPE__ int
+// RISCV64: #define __INT64_C(c) c##L
 // RISCV64: #define __INT64_C_SUFFIX__ L
 // RISCV64: #define __INT64_MAX__ 9223372036854775807L
 // RISCV64: #define __INT64_TYPE__ long int
+// RISCV64: #define __INT8_C(c) c
 // RISCV64: #define __INT8_C_SUFFIX__
 // RISCV64: #define __INT8_MAX__ 127
 // RISCV64: #define __INT8_TYPE__ signed char
+// RISCV64: #define __INTMAX_C(c) c##L
 // RISCV64: #define __INTMAX_C_SUFFIX__ L
 // RISCV64: #define __INTMAX_MAX__ 9223372036854775807L
 // RISCV64: #define __INTMAX_TYPE__ long int
@@ -2687,18 +2773,23 @@
 // RISCV64: #define __STDC_UTF_32__ 1
 // RISCV64: #define __STDC_VERSION__ 201710L
 // RISCV64: #define __STDC__ 1
+// RISCV64: #define __UINT16_C(c) c
 // RISCV64: #define __UINT16_C_SUFFIX__
 // RISCV64: #define __UINT16_MAX__ 65535
 // RISCV64: #define __UINT16_TYPE__ unsigned short
+// RISCV64: #define __UINT32_C(c) c##U
 // RISCV64: #define __UINT32_C_SUFFIX__ U
 // RISCV64: #define __UINT32_MAX__ 4294967295U
 // RISCV64: #define __UINT32_TYPE__ unsigned int
+// RISCV64: #define __UINT64_C(c) c##UL
 // RISCV64: #define __UINT64_C_SUFFIX__ UL
 // RISCV64: #define __UINT64_MAX__ 18446744073709551615UL
 // RISCV64: #define __UINT64_TYPE__ long unsigned int
+// RISCV64: #define __UINT8_C(c) c
 // RISCV64: #define __UINT8_C_SUFFIX__
 // RISCV64: #define __UINT8_MAX__ 255
 // RISCV64: #define __UINT8_TYPE__ unsigned char
+// RISCV64: #define __UINTMAX_C(c) c##UL
 // RISCV64: #define __UINTMAX_C_SUFFIX__ UL
 // RISCV64: #define __UINTMAX_MAX__ 18446744073709551615UL
 // RISCV64: #define __UINTMAX_TYPE__ long unsigned int
@@ -2837,18 +2928,23 @@
 // XTENSA: #define __GNUC__ {{.*}}
 // XTENSA: #define __GXX_ABI_VERSION {{.*}}
 // XTENSA: #define __ILP32__ 1
+// XTENSA: #define __INT16_C(c) c
 // XTENSA: #define __INT16_C_SUFFIX__ 
 // XTENSA: #define __INT16_MAX__ 32767
 // XTENSA: #define __INT16_TYPE__ short
+// XTENSA: #define __INT32_C(c) c
 // XTENSA: #define __INT32_C_SUFFIX__ 
 // XTENSA: #define __INT32_MAX__ 2147483647
 // XTENSA: #define __INT32_TYPE__ int
+// XTENSA: #define __INT64_C(c) c##LL
 // XTENSA: #define __INT64_C_SUFFIX__ LL
 // XTENSA: #define __INT64_MAX__ 9223372036854775807LL
 // XTENSA: #define __INT64_TYPE__ long long int
+// XTENSA: #define __INT8_C(c) c
 // XTENSA: #define __INT8_C_SUFFIX__ 
 // XTENSA: #define __INT8_MAX__ 127
 // XTENSA: #define __INT8_TYPE__ signed char
+// XTENSA: #define __INTMAX_C(c) c##LL
 // XTENSA: #define __INTMAX_C_SUFFIX__ LL
 // XTENSA: #define __INTMAX_MAX__ 9223372036854775807LL
 // XTENSA: #define __INTMAX_TYPE__ long long int
@@ -2945,18 +3041,23 @@
 // XTENSA: #define __STDC_UTF_32__ 1
 // XTENSA: #define __STDC_VERSION__ 201710L
 // XTENSA: #define __STDC__ 1
+// XTENSA: #define __UINT16_C(c) c
 // XTENSA: #define __UINT16_C_SUFFIX__ 
 // XTENSA: #define __UINT16_MAX__ 65535
 // XTENSA: #define __UINT16_TYPE__ unsigned short
+// XTENSA: #define __UINT32_C(c) c##U
 // XTENSA: #define __UINT32_C_SUFFIX__ U
 // XTENSA: #define __UINT32_MAX__ 4294967295U
 // XTENSA: #define __UINT32_TYPE__ unsigned int
+// XTENSA: #define __UINT64_C(c) c##ULL
 // XTENSA: #define __UINT64_C_SUFFIX__ ULL
 // XTENSA: #define __UINT64_MAX__ 18446744073709551615ULL
 // XTENSA: #define __UINT64_TYPE__ long long unsigned int
+// XTENSA: #define __UINT8_C(c) c
 // XTENSA: #define __UINT8_C_SUFFIX__ 
 // XTENSA: #define __UINT8_MAX__ 255
 // XTENSA: #define __UINT8_TYPE__ unsigned char
+// XTENSA: #define __UINTMAX_C(c) c##ULL
 // XTENSA: #define __UINTMAX_C_SUFFIX__ ULL
 // XTENSA: #define __UINTMAX_MAX__ 18446744073709551615ULL
 // XTENSA: #define __UINTMAX_TYPE__ long long unsigned int
diff --git a/clang/test/Preprocessor/x86_target_features.c b/clang/test/Preprocessor/x86_target_features.c
index fa3d0038f05a9..63222a882ff53 100644
--- a/clang/test/Preprocessor/x86_target_features.c
+++ b/clang/test/Preprocessor/x86_target_features.c
@@ -742,10 +742,8 @@
 // AVXVNNIINT16NOAVX2-NOT: #define __AVX2__ 1
 // AVXVNNIINT16NOAVX2-NOT: #define __AVXVNNIINT16__ 1
 
-// RUN: %clang -target i686-unknown-linux-gnu -march=atom -mavx10.1 -x c -E -dM -o - %s | FileCheck  -check-prefix=AVX10_1_256 %s
 // RUN: %clang -target i686-unknown-linux-gnu -march=atom -mavx10.1-256 -x c -E -dM -o - %s | FileCheck  -check-prefix=AVX10_1_256 %s
 // RUN: %clang -target i686-unknown-linux-gnu -march=atom -mavx10.1-256 -mno-avx512f -x c -E -dM -o - %s | FileCheck  -check-prefix=AVX10_1_256 %s
-// RUN: %clang -target i686-unknown-linux-gnu -march=atom -mavx10.2 -x c -E -dM -o - %s | FileCheck  -check-prefixes=AVX10_1_256,AVX10_2_256 %s
 // RUN: %clang -target i686-unknown-linux-gnu -march=atom -mavx10.2-256 -x c -E -dM -o - %s | FileCheck  -check-prefixes=AVX10_1_256,AVX10_2_256 %s
 // AVX10_1_256-NOT: __AVX10_1_512__
 // AVX10_1_256: #define __AVX10_1__ 1
@@ -758,6 +756,7 @@
 // RUN: %clang -target i686-unknown-linux-gnu -march=atom -mavx10.1-512 -x c -E -dM -o - %s | FileCheck  -check-prefix=AVX10_1_512 %s
 // RUN: %clang -target i686-unknown-linux-gnu -march=atom -mavx10.1-512 -mno-avx512f -x c -E -dM -o - %s | FileCheck  -check-prefix=AVX10_1_512 %s
 // RUN: %clang -target i686-unknown-linux-gnu -march=atom -mavx10.1-512 -mno-evex512 -x c -E -dM -o - %s | FileCheck  -check-prefix=AVX10_1_512 %s
+// RUN: %clang -target i686-unknown-linux-gnu -march=atom -mavx10.2 -x c -E -dM -o - %s | FileCheck  -check-prefixes=AVX10_1_512,AVX10_2_512 %s
 // RUN: %clang -target i686-unknown-linux-gnu -march=atom -mavx10.2-512 -x c -E -dM -o - %s | FileCheck  -check-prefixes=AVX10_1_512,AVX10_2_512 %s
 // AVX10_1_512: #define __AVX10_1_512__ 1
 // AVX10_1_512: #define __AVX10_1__ 1
diff --git a/clang/test/Rewriter/lit.local.cfg b/clang/test/Rewriter/lit.local.cfg
index f5e1d0349f52d..2fc1173d7ba31 100644
--- a/clang/test/Rewriter/lit.local.cfg
+++ b/clang/test/Rewriter/lit.local.cfg
@@ -1,3 +1,3 @@
 # The Objective-C rewriters are currently grouped with ARCMT.
-if not config.root.clang_arcmt:
+if not config.root.clang_objc_rewriter:
     config.unsupported = True
diff --git a/clang/test/Sema/MicrosoftCompatibility.c b/clang/test/Sema/MicrosoftCompatibility.c
index 9a1f050747f9d..8d402d53e004d 100644
--- a/clang/test/Sema/MicrosoftCompatibility.c
+++ b/clang/test/Sema/MicrosoftCompatibility.c
@@ -1,5 +1,5 @@
-// RUN: %clang_cc1 %s -fsyntax-only -Wno-unused-value -Wmicrosoft -verify -fms-compatibility -DMSVCCOMPAT -triple i686-pc-win32
-// RUN: %clang_cc1 %s -fsyntax-only -Wno-unused-value -Wmicrosoft -verify -fms-extensions -triple i686-pc-win32
+// RUN: %clang_cc1 %s -fsyntax-only -Wno-unused-value -Wmicrosoft -verify=expected,compat -fms-compatibility -DMSVCCOMPAT -triple i686-pc-win32
+// RUN: %clang_cc1 %s -fsyntax-only -Wno-unused-value -Wmicrosoft -verify=expected,ext -fms-extensions -triple i686-pc-win32
 
 #ifdef MSVCCOMPAT
 enum ENUM1; // expected-warning {{forward references to 'enum' types are a Microsoft extension}}
@@ -35,3 +35,15 @@ size_t x;
 #else
 size_t x; // expected-error {{unknown type name 'size_t'}}
 #endif
+
+/* Microsoft allows inline, __inline, and __forceinline to appear on a typedef
+   of a function type; this is used in their system headers such as ufxclient.h
+   See GitHub #124869 for more details.
+ */
+typedef int inline Foo1(int);       // compat-warning {{'inline' can only appear on functions}} \
+                                       ext-error {{'inline' can only appear on functions}}
+typedef int __inline Foo2(int);     // compat-warning {{'inline' can only appear on functions}} \
+                                       ext-error {{'inline' can only appear on functions}}
+typedef int __forceinline Foo(int); // compat-warning {{'inline' can only appear on functions}} \
+                                       ext-error {{'inline' can only appear on functions}} \
+                                       expected-warning {{'__forceinline' attribute only applies to functions and statements}}
diff --git a/clang/test/Sema/MicrosoftCompatibility.cpp b/clang/test/Sema/MicrosoftCompatibility.cpp
index 90a45dfaaf176..391977e2765c5 100644
--- a/clang/test/Sema/MicrosoftCompatibility.cpp
+++ b/clang/test/Sema/MicrosoftCompatibility.cpp
@@ -8,3 +8,10 @@ struct cls {
 };
 
 char * cls::* __uptr wrong2 = &cls::m; // expected-error {{'__uptr' attribute cannot be used with pointers to members}}
+
+// Microsoft allows inline, __inline, and __forceinline to appear on a typedef
+// of a function type, but only in C. See GitHub #124869 for more details.
+typedef int inline Foo1(int);       // expected-error {{'inline' can only appear on functions}}
+typedef int __inline Foo2(int);     // expected-error {{'inline' can only appear on functions}}
+typedef int __forceinline Foo(int); // expected-error {{'inline' can only appear on functions}} \
+                                       expected-warning {{'__forceinline' attribute only applies to functions and statements}}
diff --git a/clang/test/Sema/aarch64-fp8-cast.c b/clang/test/Sema/aarch64-fp8-cast.c
new file mode 100644
index 0000000000000..ad25401919b5a
--- /dev/null
+++ b/clang/test/Sema/aarch64-fp8-cast.c
@@ -0,0 +1,104 @@
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +neon -verify -emit-llvm -o - %s
+
+// REQUIRES: aarch64-registered-target
+
+#include <arm_neon.h>
+
+// Bitcast between FP8 Neon vectors
+mfloat8x8_t err_test_f8_f8(mfloat8x16_t x) {
+    return (mfloat8x8_t) x;
+// expected-error@-1 {{invalid conversion between vector type 'mfloat8x8_t' (vector of 8 'mfloat8_t' values) and 'mfloat8x16_t' (vector of 16 'mfloat8_t' values) of different size}}
+}
+
+mfloat8x16_t err_testq_f8_f8(mfloat8x8_t x) {
+    return (mfloat8x16_t) x;
+// expected-error@-1 {{invalid conversion between vector type 'mfloat8x16_t' (vector of 16 'mfloat8_t' values) and 'mfloat8x8_t' (vector of 8 'mfloat8_t' values) of different size}}
+}
+
+// Bitcast between FP8 and int8 Neon vectors
+mfloat8x8_t err_test_f8_s8(int8x16_t x) {
+    return (mfloat8x8_t) x;
+// expected-error@-1 {{invalid conversion between vector type 'mfloat8x8_t' (vector of 8 'mfloat8_t' values) and 'int8x16_t' (vector of 16 'int8_t' values) of different size}}
+}
+
+int8x8_t err_test_s8_f8(mfloat8x16_t x) {
+    return (int8x8_t) x;
+// expected-error@-1 {{invalid conversion between vector type 'int8x8_t' (vector of 8 'int8_t' values) and 'mfloat8x16_t' (vector of 16 'mfloat8_t' values) of different size}}
+}
+
+mfloat8x16_t err_testq_f8_s8(int8x8_t x) {
+    return (mfloat8x16_t) x;
+// expected-error@-1 {{invalid conversion between vector type 'mfloat8x16_t' (vector of 16 'mfloat8_t' values) and 'int8x8_t' (vector of 8 'int8_t' values) of different size}}
+}
+
+int8x16_t err_testq_s8_f8(mfloat8x8_t x) {
+    return (int8x16_t) x;
+// expected-error@-1 {{invalid conversion between vector type 'int8x16_t' (vector of 16 'int8_t' values) and 'mfloat8x8_t' (vector of 8 'mfloat8_t' values) of different size}}
+}
+
+// Bitcast between FP8 and float32 Neon vectors
+mfloat8x8_t err_test_f8_f32(float32x4_t x) {
+    return (mfloat8x8_t) x;
+// expected-error@-1 {{invalid conversion between vector type 'mfloat8x8_t' (vector of 8 'mfloat8_t' values) and 'float32x4_t' (vector of 4 'float32_t' values) of different size}}
+}
+
+float32x2_t err_test_f32_f8(mfloat8x16_t x) {
+    return (float32x2_t) x;
+// expected-error@-1 {{invalid conversion between vector type 'float32x2_t' (vector of 2 'float32_t' values) and 'mfloat8x16_t' (vector of 16 'mfloat8_t' values) of different size}}
+}
+
+mfloat8x16_t err_testq_f8_f32(float32x2_t x) {
+    return (mfloat8x16_t) x;
+// expected-error@-1 {{invalid conversion between vector type 'mfloat8x16_t' (vector of 16 'mfloat8_t' values) and 'float32x2_t' (vector of 2 'float32_t' values) of different size}}
+}
+
+float32x4_t err_testq_f32_f8(mfloat8x8_t x) {
+    return (float32x4_t) x;
+// expected-error@-1 {{invalid conversion between vector type 'float32x4_t' (vector of 4 'float32_t' values) and 'mfloat8x8_t' (vector of 8 'mfloat8_t' values) of different size}}
+}
+
+// Bitcast between FP8 and poly128_t (which is integral)
+mfloat8x8_t err_testq_f8_p128(poly128_t x) {
+    return (mfloat8x8_t) x;
+// expected-error@-1 {{invalid conversion between vector type 'mfloat8x8_t' (vector of 8 'mfloat8_t' values) and integer type 'poly128_t' (aka 'unsigned __int128') of different size}}
+}
+
+poly128_t err_testq_p128_f8(mfloat8x8_t x) {
+    return (poly128_t) x;
+// expected-error@-1 {{invalid conversion between vector type 'mfloat8x8_t' (vector of 8 'mfloat8_t' values) and integer type 'poly128_t' (aka 'unsigned __int128') of different size}}
+}
+
+// Bitcast between FP8 and a non-integral type
+mfloat8x8_t err_test_f8_ptr(void *p) {
+    return (mfloat8x8_t) p;
+// expected-error@-1 {{invalid conversion between vector type 'mfloat8x8_t' (vector of 8 'mfloat8_t' values) and scalar type 'void *'}}
+}
+
+void *err_test_ptr_f8(mfloat8x8_t v) {
+    return (void *) v;
+// expected-error@-1 {{invalid conversion between vector type 'mfloat8x8_t' (vector of 8 'mfloat8_t' values) and scalar type 'void *'}}
+}
+
+mfloat8x8_t err_test_f8_dbl(double v) {
+    return (mfloat8x8_t) v;
+// expected-error@-1 {{invalid conversion between vector type 'mfloat8x8_t' (vector of 8 'mfloat8_t' values) and scalar type 'double'}}
+}
+
+double err_test_dbl_f8(mfloat8x8_t v) {
+    return (double) v;
+// expected-error@-1 {{invalid conversion between vector type 'mfloat8x8_t' (vector of 8 'mfloat8_t' values) and scalar type 'double'}}
+}
+
+struct S {
+    char ch[16];
+};
+
+mfloat8x16_t err_test_f8_agg(struct S s) {
+    return (mfloat8x16_t) s;
+// expected-error@-1 {{operand of type 'struct S' where arithmetic or pointer type is required}}
+}
+
+struct S err_test_agg_f8(mfloat8x16_t v) {
+    return (struct S) v;
+// expected-error@-1 {{used type 'struct S' where arithmetic or pointer type is required}}
+}
diff --git a/clang/test/Sema/aarch64-fp8-intrinsics/acle_neon_fp8_cvt.c b/clang/test/Sema/aarch64-fp8-intrinsics/acle_neon_fp8_cvt.c
new file mode 100644
index 0000000000000..2c7004c7968a4
--- /dev/null
+++ b/clang/test/Sema/aarch64-fp8-intrinsics/acle_neon_fp8_cvt.c
@@ -0,0 +1,43 @@
+// RUN: %clang_cc1 -triple aarch64-linux-gnu -target-feature +neon -target-feature +bf16 -target-feature +faminmax -emit-llvm -verify %s -o /dev/null
+
+// REQUIRES: aarch64-registered-target
+
+#include <arm_neon.h>
+
+void test_features(float16x4_t vd4, float16x8_t vd8, float32x4_t va4,
+                   mfloat8x8_t v8, mfloat8x16_t v16, fpm_t fpm) {
+  (void) vcvt1_bf16_mf8_fpm(v8, fpm);
+  // expected-error@-1 {{'vcvt1_bf16_mf8_fpm' requires target feature 'fp8'}}
+  (void) vcvt1_low_bf16_mf8_fpm(v16, fpm);
+  // expected-error@-1 {{'vcvt1_low_bf16_mf8_fpm' requires target feature 'fp8'}}
+  (void) vcvt2_bf16_mf8_fpm(v8, fpm);
+  // expected-error@-1 {{'vcvt2_bf16_mf8_fpm' requires target feature 'fp8'}}
+  (void) vcvt2_low_bf16_mf8_fpm(v16, fpm);
+  // expected-error@-1 {{'vcvt2_low_bf16_mf8_fpm' requires target feature 'fp8'}}
+
+  (void) vcvt1_high_bf16_mf8_fpm(v16, fpm);
+  // expected-error@-1 {{'vcvt1_high_bf16_mf8_fpm' requires target feature 'fp8'}}
+  (void) vcvt2_high_bf16_mf8_fpm(v16, fpm);
+  // expected-error@-1 {{'vcvt2_high_bf16_mf8_fpm' requires target feature 'fp8'}}
+
+  (void) vcvt1_f16_mf8_fpm(v8, fpm);
+  // expected-error@-1 {{'vcvt1_f16_mf8_fpm' requires target feature 'fp8'}}
+  (void) vcvt1_low_f16_mf8_fpm(v16, fpm);
+  // expected-error@-1 {{'vcvt1_low_f16_mf8_fpm' requires target feature 'fp8'}}
+  (void) vcvt2_f16_mf8_fpm(v8, fpm);
+  // expected-error@-1 {{'vcvt2_f16_mf8_fpm' requires target feature 'fp8'}}
+  (void) vcvt2_low_f16_mf8_fpm(v16, fpm);
+  // expected-error@-1 {{'vcvt2_low_f16_mf8_fpm' requires target feature 'fp8'}}
+  (void) vcvt1_high_f16_mf8_fpm(v16, fpm);
+  // expected-error@-1 {{'vcvt1_high_f16_mf8_fpm' requires target feature 'fp8'}}
+  (void) vcvt2_high_f16_mf8_fpm(v16, fpm);
+  // expected-error@-1 {{'vcvt2_high_f16_mf8_fpm' requires target feature 'fp8'}}
+  (void) vcvt_mf8_f32_fpm(va4, va4, fpm);
+  // expected-error@-1 {{'vcvt_mf8_f32_fpm' requires target feature 'fp8'}}
+  (void) vcvt_high_mf8_f32_fpm(v8, va4, va4, fpm);
+  // expected-error@-1 {{'vcvt_high_mf8_f32_fpm' requires target feature 'fp8'}}
+  (void) vcvt_mf8_f16_fpm(vd4, vd4, fpm);
+  // expected-error@-1 {{'vcvt_mf8_f16_fpm' requires target feature 'fp8'}}
+  (void) vcvtq_mf8_f16_fpm(vd8, vd8, fpm);
+  // expected-error@-1 {{'vcvtq_mf8_f16_fpm' requires target feature 'fp8'}}
+}
diff --git a/clang/test/Sema/aarch64-fp8-intrinsics/acle_neon_fp8_fdot.c b/clang/test/Sema/aarch64-fp8-intrinsics/acle_neon_fp8_fdot.c
new file mode 100644
index 0000000000000..8bfe3ac26ab2c
--- /dev/null
+++ b/clang/test/Sema/aarch64-fp8-intrinsics/acle_neon_fp8_fdot.c
@@ -0,0 +1,54 @@
+// RUN: %clang_cc1 -triple aarch64-linux-gnu -target-feature +neon -target-feature +bf16 -target-feature +faminmax -target-feature +fp8 -emit-llvm -verify %s -o /dev/null
+
+// REQUIRES: aarch64-registered-target
+
+#include <arm_neon.h>
+
+void test_features(float16x4_t vd4, float16x8_t vd8, float32x4_t va4, float32x2_t va2,
+                   mfloat8x8_t v8, mfloat8x16_t v16, fpm_t fpm) {
+  (void) vdot_f16_mf8_fpm(vd4, v8, v8, fpm);
+// expected-error@-1 {{'vdot_f16_mf8_fpm' requires target feature 'fp8dot2'}}
+  (void) vdotq_f16_mf8_fpm(vd8, v16, v16, fpm);
+// expected-error@-1 {{'vdotq_f16_mf8_fpm' requires target feature 'fp8dot2'}}
+  (void) vdot_lane_f16_mf8_fpm(vd4, v8, v8, 3, fpm);
+// expected-error@-1 {{'__builtin_neon_vdot_lane_f16_mf8_fpm' needs target feature fp8dot2,neon}}
+  (void) vdot_laneq_f16_mf8_fpm(vd4, v8, v16, 7, fpm);
+// expected-error@-1 {{'__builtin_neon_vdot_laneq_f16_mf8_fpm' needs target feature fp8dot2,neon}}
+  (void) vdotq_lane_f16_mf8_fpm(vd8, v16, v8, 3, fpm);
+// expected-error@-1 {{'__builtin_neon_vdotq_lane_f16_mf8_fpm' needs target feature fp8dot2,neon}}
+  (void) vdotq_laneq_f16_mf8_fpm(vd8, v16, v16, 7, fpm);
+// expected-error@-1 {{'__builtin_neon_vdotq_laneq_f16_mf8_fpm' needs target feature fp8dot2,neon}}
+
+  (void) vdot_f32_mf8_fpm(va2, v8, v8, fpm);
+// expected-error@-1 {{'vdot_f32_mf8_fpm' requires target feature 'fp8dot4'}}
+  (void) vdotq_f32_mf8_fpm(va4, v16, v16, fpm);
+// expected-error@-1 {{'vdotq_f32_mf8_fpm' requires target feature 'fp8dot4}}
+  (void) vdot_lane_f32_mf8_fpm(va2, v8, v8, 1, fpm);
+// expected-error@-1 {{'__builtin_neon_vdot_lane_f32_mf8_fpm' needs target feature fp8dot4,neon}}
+  (void) vdot_laneq_f32_mf8_fpm(va2, v8, v16, 3, fpm);
+// expected-error@-1 {{'__builtin_neon_vdot_laneq_f32_mf8_fpm' needs target feature fp8dot4,neon}}
+  (void) vdotq_lane_f32_mf8_fpm(va4, v16, v8, 1, fpm);
+// expected-error@-1 {{'__builtin_neon_vdotq_lane_f32_mf8_fpm' needs target feature fp8dot4,neon}}
+  (void) vdotq_laneq_f32_mf8_fpm(va4, v16, v16, 3, fpm);
+// expected-error@-1 {{'__builtin_neon_vdotq_laneq_f32_mf8_fpm' needs target feature fp8dot4,neon}}
+}
+
+void test_imm(float16x4_t vd4, float16x8_t vd8, float32x2_t va2, float32x4_t va4,
+              mfloat8x8_t v8, mfloat8x16_t v16, fpm_t fpm) {
+  (void) vdot_lane_f16_mf8_fpm(vd4, v8, v8, -1, fpm);
+  // expected-error@-1 {{argument value -1 is outside the valid range [0, 3]}}
+  (void) vdot_laneq_f16_mf8_fpm(vd4, v8, v16, -1, fpm);
+  // expected-error@-1 {{argument value -1 is outside the valid range [0, 7]}}
+  (void) vdotq_lane_f16_mf8_fpm(vd8, v16, v8, -1, fpm);
+  // expected-error@-1 {{argument value -1 is outside the valid range [0, 3]}}
+  (void) vdotq_laneq_f16_mf8_fpm(vd8, v16, v16, -1, fpm);
+  // expected-error@-1 {{argument value -1 is outside the valid range [0, 7]}}
+  (void) vdot_lane_f32_mf8_fpm(va2, v8, v8, -1, fpm);
+  // expected-error@-1 {{argument value -1 is outside the valid range [0, 1]}}
+  (void) vdot_laneq_f32_mf8_fpm(va2, v8, v16, -1, fpm);
+  // expected-error@-1 {{argument value -1 is outside the valid range [0, 3]}}
+  (void) vdotq_lane_f32_mf8_fpm(va4, v16, v8, -1, fpm);
+  // expected-error@-1 {{argument value -1 is outside the valid range [0, 1]}}
+  (void) vdotq_laneq_f32_mf8_fpm(va4, v16, v16, -1, fpm);
+  // expected-error@-1 {{argument value -1 is outside the valid range [0, 3]}}
+}
diff --git a/clang/test/Sema/aarch64-fp8-intrinsics/acle_neon_fp8_fmla.c b/clang/test/Sema/aarch64-fp8-intrinsics/acle_neon_fp8_fmla.c
new file mode 100644
index 0000000000000..4a507b08040ff
--- /dev/null
+++ b/clang/test/Sema/aarch64-fp8-intrinsics/acle_neon_fp8_fmla.c
@@ -0,0 +1,49 @@
+// RUN: %clang_cc1 -triple aarch64-linux-gnu -target-feature +neon -target-feature +bf16 -target-feature +faminmax -target-feature +fp8 -emit-llvm -verify %s -o /dev/null
+
+// REQUIRES: aarch64-registered-target
+
+#include <arm_neon.h>
+
+void test_features(float16x8_t a, float32x4_t b, mfloat8x16_t u, fpm_t fpm) {
+  (void) vmlalbq_f16_mf8_fpm(a, u, u, fpm);
+  // expected-error@-1 {{'vmlalbq_f16_mf8_fpm' requires target feature 'fp8fma'}}
+  (void) vmlaltq_f16_mf8_fpm(a, u, u, fpm);
+  // expected-error@-1 {{'vmlaltq_f16_mf8_fpm' requires target feature 'fp8fma'}}
+  (void) vmlallbbq_f32_mf8_fpm(b, u, u, fpm);
+  // expected-error@-1 {{'vmlallbbq_f32_mf8_fpm' requires target feature 'fp8fma'}}
+  (void) vmlallbtq_f32_mf8_fpm(b, u, u, fpm);
+  // expected-error@-1 {{'vmlallbtq_f32_mf8_fpm' requires target feature 'fp8fma'}}
+  (void) vmlalltbq_f32_mf8_fpm(b, u, u, fpm);
+  // expected-error@-1 {{'vmlalltbq_f32_mf8_fpm' requires target feature 'fp8fma'}}
+  (void) vmlallttq_f32_mf8_fpm(b, u, u, fpm);
+  // expected-error@-1 {{'vmlallttq_f32_mf8_fpm' requires target feature 'fp8fma'}}
+}
+
+void test_imm(float16x8_t d, float32x4_t c, mfloat8x16_t a, mfloat8x8_t b, fpm_t fpm) {
+(void) vmlalbq_lane_f16_mf8_fpm(d, a, b, -1, fpm);
+// expected-error@-1 {{argument value -1 is outside the valid range [0, 7]}}
+(void) vmlalbq_laneq_f16_mf8_fpm(d, a, a, -1, fpm);
+// expected-error@-1 {{argument value -1 is outside the valid range [0, 15]}}
+(void) vmlaltq_lane_f16_mf8_fpm(d, a, b, -1, fpm);
+// expected-error@-1 {{argument value -1 is outside the valid range [0, 7]}}
+(void) vmlaltq_laneq_f16_mf8_fpm(d, a, a, -1, fpm);
+// expected-error@-1 {{argument value -1 is outside the valid range [0, 15]}}
+
+(void) vmlallbbq_lane_f32_mf8_fpm(c, a, b, -1, fpm);
+// expected-error@-1 {{argument value -1 is outside the valid range [0, 7]}}
+(void) vmlallbbq_laneq_f32_mf8_fpm(c, a, a, -1, fpm);
+// expected-error@-1 {{argument value -1 is outside the valid range [0, 15]}}
+(void) vmlallbtq_lane_f32_mf8_fpm(c, a, b, -1, fpm);
+// expected-error@-1 {{argument value -1 is outside the valid range [0, 7]}}
+(void) vmlallbtq_laneq_f32_mf8_fpm(c, a, a, -1, fpm);
+// expected-error@-1 {{argument value -1 is outside the valid range [0, 15]}}
+(void) vmlalltbq_lane_f32_mf8_fpm(c, a, b, -1, fpm);
+// expected-error@-1 {{argument value -1 is outside the valid range [0, 7]}}
+(void) vmlalltbq_laneq_f32_mf8_fpm(c, a, a, -1, fpm);
+// expected-error@-1 {{argument value -1 is outside the valid range [0, 15]}}
+(void) vmlallttq_lane_f32_mf8_fpm(c, a, b, -1, fpm);
+// expected-error@-1 {{argument value -1 is outside the valid range [0, 7]}}
+(void) vmlallttq_laneq_f32_mf8_fpm(c, a, a, -1, fpm);
+// expected-error@-1 {{argument value -1 is outside the valid range [0, 15]}}
+}
+
diff --git a/clang/test/Sema/aarch64-incompat-sm-builtin-calls.c b/clang/test/Sema/aarch64-incompat-sm-builtin-calls.c
deleted file mode 100644
index 27fa8f7c9dccb..0000000000000
--- a/clang/test/Sema/aarch64-incompat-sm-builtin-calls.c
+++ /dev/null
@@ -1,128 +0,0 @@
-// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py
-// RUN: %clang_cc1  -triple aarch64-none-linux-gnu -target-feature +sve \
-// RUN:   -target-feature +bf16 -target-feature +sve -target-feature +sme -target-feature +sme2 -target-feature +sve2 -target-feature +neon -Waarch64-sme-attributes -fsyntax-only -verify %s
-
-// REQUIRES: aarch64-registered-target
-
-#include "arm_neon.h"
-#include "arm_sme.h"
-#include "arm_sve.h"
-
-int16x8_t incompat_neon_sm(int16x8_t splat) __arm_streaming {
-  // expected-error@+1 {{builtin can only be called from a non-streaming function}}
-  return (int16x8_t)__builtin_neon_vqaddq_v((int8x16_t)splat, (int8x16_t)splat, 33);
-}
-
-__arm_locally_streaming int16x8_t incompat_neon_ls(int16x8_t splat) {
-  // expected-error@+1 {{builtin can only be called from a non-streaming function}}
-  return (int16x8_t)__builtin_neon_vqaddq_v((int8x16_t)splat, (int8x16_t)splat, 33);
-}
-
-int16x8_t incompat_neon_smc(int16x8_t splat) __arm_streaming_compatible {
-  // expected-error@+1 {{builtin can only be called from a non-streaming function}}
-  return (int16x8_t)__builtin_neon_vqaddq_v((int8x16_t)splat, (int8x16_t)splat, 33);
-}
-
-void incompat_sme_smc(svbool_t pg, void const *ptr) __arm_streaming_compatible __arm_inout("za") {
-  // expected-error@+1 {{builtin can only be called from a streaming function}}
-  return __builtin_sme_svld1_hor_za128(0, 0, pg, ptr);
-}
-
-float incomp_sve_sm_fadda_sm(void) __arm_streaming {
-  // expected-error@+1 {{builtin can only be called from a non-streaming function}}
-  return svadda(svptrue_b32(), 0, svdup_f32(1));
-}
-
-float incomp_sve_sm_fadda_smc(void) __arm_streaming_compatible {
-  // expected-error@+1 {{builtin can only be called from a non-streaming function}}
-  return svadda(svptrue_b32(), 0, svdup_f32(1));
-}
-
-svuint32_t incompat_sve_sm(svbool_t pg, svuint32_t a, int16_t b) __arm_streaming {
-  // expected-error@+1 {{builtin can only be called from a non-streaming function}}
-  return __builtin_sve_svld1_gather_u32base_index_u32(pg, a, b);
-}
-
-// expected-warning@+2 {{returning a VL-dependent argument from a locally streaming function is undefined behaviour when the streaming and non-streaming vector lengths are different at runtime}}
-// expected-warning@+1 {{passing a VL-dependent argument to a locally streaming function is undefined behaviour when the streaming and non-streaming vector lengths are different at runtime}}
-__arm_locally_streaming svuint32_t incompat_sve_ls(svbool_t pg, svuint32_t a, int64_t b) {
-  // expected-error@+1 {{builtin can only be called from a non-streaming function}}
-  return __builtin_sve_svld1_gather_u32base_index_u32(pg, a, b);
-}
-
-svuint32_t incompat_sve_smc(svbool_t pg, svuint32_t a, int64_t b) __arm_streaming_compatible {
-  // expected-error@+1 {{builtin can only be called from a non-streaming function}}
-  return __builtin_sve_svld1_gather_u32base_index_u32(pg, a, b);
-}
-
-svuint32_t incompat_sve2_sm(svbool_t pg, svuint32_t a, int64_t b) __arm_streaming {
-  // expected-error@+1 {{builtin can only be called from a non-streaming function}}
-  return __builtin_sve_svldnt1_gather_u32base_index_u32(pg, a, b);
-}
-
-// expected-warning@+2 {{returning a VL-dependent argument from a locally streaming function is undefined behaviour when the streaming and non-streaming vector lengths are different at runtime}}
-// expected-warning@+1 {{passing a VL-dependent argument to a locally streaming function is undefined behaviour when the streaming and non-streaming vector lengths are different at runtime}}
-__arm_locally_streaming svuint32_t incompat_sve2_ls(svbool_t pg, svuint32_t a, int64_t b) {
-  // expected-error@+1 {{builtin can only be called from a non-streaming function}}
-  return __builtin_sve_svldnt1_gather_u32base_index_u32(pg, a, b);
-}
-
-svuint32_t incompat_sve2_smc(svbool_t pg, svuint32_t a, int64_t b) __arm_streaming_compatible {
-  // expected-error@+1 {{builtin can only be called from a non-streaming function}}
-  return __builtin_sve_svldnt1_gather_u32base_index_u32(pg, a, b);
-}
-
-void incompat_sme_sm(svbool_t pn, svbool_t pm, svfloat32_t zn, svfloat32_t zm) __arm_inout("za") {
-  // expected-error@+1 {{builtin can only be called from a streaming function}}
-  svmops_za32_f32_m(0, pn, pm, zn, zm);
-}
-
-svfloat64_t streaming_caller_sve(svbool_t pg, svfloat64_t a, float64_t b) __arm_streaming {
-  return svadd_n_f64_m(pg, a, b);
-}
-
-// expected-warning@+2 {{returning a VL-dependent argument from a locally streaming function is undefined behaviour when the streaming and non-streaming vector lengths are different at runtime}}
-// expected-warning@+1 {{passing a VL-dependent argument to a locally streaming function is undefined behaviour when the streaming and non-streaming vector lengths are different at runtime}}
-__arm_locally_streaming svfloat64_t locally_streaming_caller_sve(svbool_t pg, svfloat64_t a, float64_t b) {
-  return svadd_n_f64_m(pg, a, b);
-}
-
-svfloat64_t streaming_compatible_caller_sve(svbool_t pg, svfloat64_t a, float64_t b) __arm_streaming_compatible {
-  return svadd_n_f64_m(pg, a, b);
-}
-
-svint16_t streaming_caller_sve2(svint16_t op1, svint16_t op2) __arm_streaming {
-  return svmul_lane_s16(op1, op2, 0);
-}
-
-// expected-warning@+2 {{returning a VL-dependent argument from a locally streaming function is undefined behaviour when the streaming and non-streaming vector lengths are different at runtime}}
-// expected-warning@+1 {{passing a VL-dependent argument to a locally streaming function is undefined behaviour when the streaming and non-streaming vector lengths are different at runtime}}
-__arm_locally_streaming svint16_t locally_streaming_caller_sve2(svint16_t op1, svint16_t op2) {
-  return svmul_lane_s16(op1, op2, 0);
-}
-
-svint16_t streaming_compatible_caller_sve2(svint16_t op1, svint16_t op2) __arm_streaming_compatible {
-  return svmul_lane_s16(op1, op2, 0);
-}
-
-svbool_t streaming_caller_ptrue(void) __arm_streaming {
-  return svand_z(svptrue_b16(), svptrue_pat_b16(SV_ALL), svptrue_pat_b16(SV_VL4));
-}
-
-svint8_t missing_za(svint8_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming {
-  // expected-warning@+1 {{builtin call is not valid when calling from a function without active ZA state}}
-    return svread_hor_za8_s8_m(zd, pg, 0, slice_base);
-}
-
-__arm_new("za")
-svint8_t new_za(svint8_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming {
-    return svread_hor_za8_s8_m(zd, pg, 0, slice_base);
-}
-
-void missing_zt0(void) __arm_streaming {
-  // expected-warning@+1 {{builtin call is not valid when calling from a function without active ZT0 state}}
-  svzero_zt(0);
-}
-
-__arm_new("zt0")
-void new_zt0(void) __arm_streaming { svzero_zt(0); }
diff --git a/clang/test/Sema/aarch64-incompat-sm-builtin-calls.cpp b/clang/test/Sema/aarch64-incompat-sm-builtin-calls.cpp
new file mode 100644
index 0000000000000..3fbcaf4a13d67
--- /dev/null
+++ b/clang/test/Sema/aarch64-incompat-sm-builtin-calls.cpp
@@ -0,0 +1,194 @@
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py
+// RUN: %clang_cc1  -std=c++23 -triple aarch64-none-linux-gnu -target-feature +sve \
+// RUN:   -target-feature +bf16 -target-feature +sve -target-feature +sme -target-feature +sme2 -target-feature +sve2 -target-feature +neon -Waarch64-sme-attributes -fsyntax-only -verify %s
+
+// REQUIRES: aarch64-registered-target
+
+#include "arm_neon.h"
+#include "arm_sme.h"
+#include "arm_sve.h"
+
+int16x8_t incompat_neon_sm(int16x8_t splat) __arm_streaming {
+  // expected-error@+1 {{builtin can only be called from a non-streaming function}}
+  return (int16x8_t)__builtin_neon_vqaddq_v((int8x16_t)splat, (int8x16_t)splat, 33);
+}
+
+__arm_locally_streaming int16x8_t incompat_neon_ls(int16x8_t splat) {
+  // expected-error@+1 {{builtin can only be called from a non-streaming function}}
+  return (int16x8_t)__builtin_neon_vqaddq_v((int8x16_t)splat, (int8x16_t)splat, 33);
+}
+
+int16x8_t incompat_neon_smc(int16x8_t splat) __arm_streaming_compatible {
+  // expected-error@+1 {{builtin can only be called from a non-streaming function}}
+  return (int16x8_t)__builtin_neon_vqaddq_v((int8x16_t)splat, (int8x16_t)splat, 33);
+}
+
+void incompat_sme_smc(svbool_t pg, void const *ptr) __arm_streaming_compatible __arm_inout("za") {
+  // expected-error@+1 {{builtin can only be called from a streaming function}}
+  return __builtin_sme_svld1_hor_za128(0, 0, pg, ptr);
+}
+
+float incomp_sve_sm_fadda_sm(void) __arm_streaming {
+  // expected-error@+1 {{builtin can only be called from a non-streaming function}}
+  return svadda(svptrue_b32(), 0, svdup_f32(1));
+}
+
+float incomp_sve_sm_fadda_smc(void) __arm_streaming_compatible {
+  // expected-error@+1 {{builtin can only be called from a non-streaming function}}
+  return svadda(svptrue_b32(), 0, svdup_f32(1));
+}
+
+svuint32_t incompat_sve_sm(svbool_t pg, svuint32_t a, int16_t b) __arm_streaming {
+  // expected-error@+1 {{builtin can only be called from a non-streaming function}}
+  return __builtin_sve_svld1_gather_u32base_index_u32(pg, a, b);
+}
+
+// expected-warning@+2 {{returning a VL-dependent argument from a locally streaming function is undefined behaviour when the streaming and non-streaming vector lengths are different at runtime}}
+// expected-warning@+1 {{passing a VL-dependent argument to a locally streaming function is undefined behaviour when the streaming and non-streaming vector lengths are different at runtime}}
+__arm_locally_streaming svuint32_t incompat_sve_ls(svbool_t pg, svuint32_t a, int64_t b) {
+  // expected-error@+1 {{builtin can only be called from a non-streaming function}}
+  return __builtin_sve_svld1_gather_u32base_index_u32(pg, a, b);
+}
+
+svuint32_t incompat_sve_smc(svbool_t pg, svuint32_t a, int64_t b) __arm_streaming_compatible {
+  // expected-error@+1 {{builtin can only be called from a non-streaming function}}
+  return __builtin_sve_svld1_gather_u32base_index_u32(pg, a, b);
+}
+
+svuint32_t incompat_sve2_sm(svbool_t pg, svuint32_t a, int64_t b) __arm_streaming {
+  // expected-error@+1 {{builtin can only be called from a non-streaming function}}
+  return __builtin_sve_svldnt1_gather_u32base_index_u32(pg, a, b);
+}
+
+// expected-warning@+2 {{returning a VL-dependent argument from a locally streaming function is undefined behaviour when the streaming and non-streaming vector lengths are different at runtime}}
+// expected-warning@+1 {{passing a VL-dependent argument to a locally streaming function is undefined behaviour when the streaming and non-streaming vector lengths are different at runtime}}
+__arm_locally_streaming svuint32_t incompat_sve2_ls(svbool_t pg, svuint32_t a, int64_t b) {
+  // expected-error@+1 {{builtin can only be called from a non-streaming function}}
+  return __builtin_sve_svldnt1_gather_u32base_index_u32(pg, a, b);
+}
+
+svuint32_t incompat_sve2_smc(svbool_t pg, svuint32_t a, int64_t b) __arm_streaming_compatible {
+  // expected-error@+1 {{builtin can only be called from a non-streaming function}}
+  return __builtin_sve_svldnt1_gather_u32base_index_u32(pg, a, b);
+}
+
+void incompat_sme_sm(svbool_t pn, svbool_t pm, svfloat32_t zn, svfloat32_t zm) __arm_inout("za") {
+  // expected-error@+1 {{builtin can only be called from a streaming function}}
+  svmops_za32_f32_m(0, pn, pm, zn, zm);
+}
+
+svfloat64_t streaming_caller_sve(svbool_t pg, svfloat64_t a, float64_t b) __arm_streaming {
+  return svadd_n_f64_m(pg, a, b);
+}
+
+// expected-warning@+2 {{returning a VL-dependent argument from a locally streaming function is undefined behaviour when the streaming and non-streaming vector lengths are different at runtime}}
+// expected-warning@+1 {{passing a VL-dependent argument to a locally streaming function is undefined behaviour when the streaming and non-streaming vector lengths are different at runtime}}
+__arm_locally_streaming svfloat64_t locally_streaming_caller_sve(svbool_t pg, svfloat64_t a, float64_t b) {
+  return svadd_n_f64_m(pg, a, b);
+}
+
+svfloat64_t streaming_compatible_caller_sve(svbool_t pg, svfloat64_t a, float64_t b) __arm_streaming_compatible {
+  return svadd_n_f64_m(pg, a, b);
+}
+
+svint16_t streaming_caller_sve2(svint16_t op1, svint16_t op2) __arm_streaming {
+  return svmul_lane_s16(op1, op2, 0);
+}
+
+// expected-warning@+2 {{returning a VL-dependent argument from a locally streaming function is undefined behaviour when the streaming and non-streaming vector lengths are different at runtime}}
+// expected-warning@+1 {{passing a VL-dependent argument to a locally streaming function is undefined behaviour when the streaming and non-streaming vector lengths are different at runtime}}
+__arm_locally_streaming svint16_t locally_streaming_caller_sve2(svint16_t op1, svint16_t op2) {
+  return svmul_lane_s16(op1, op2, 0);
+}
+
+svint16_t streaming_compatible_caller_sve2(svint16_t op1, svint16_t op2) __arm_streaming_compatible {
+  return svmul_lane_s16(op1, op2, 0);
+}
+
+svbool_t streaming_caller_ptrue(void) __arm_streaming {
+  return svand_z(svptrue_b16(), svptrue_pat_b16(SV_ALL), svptrue_pat_b16(SV_VL4));
+}
+
+svint8_t missing_za(svint8_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming {
+  // expected-warning@+1 {{builtin call is not valid when calling from a function without active ZA state}}
+    return svread_hor_za8_s8_m(zd, pg, 0, slice_base);
+}
+
+__arm_new("za")
+svint8_t new_za(svint8_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming {
+    return svread_hor_za8_s8_m(zd, pg, 0, slice_base);
+}
+
+void missing_zt0(void) __arm_streaming {
+  // expected-warning@+1 {{builtin call is not valid when calling from a function without active ZT0 state}}
+  svzero_zt(0);
+}
+
+__arm_new("zt0")
+void new_zt0(void) __arm_streaming { svzero_zt(0); }
+
+/// C++ lambda tests:
+
+void use_streaming_builtin_in_lambda(uint32_t slice_base, svbool_t pg, const void *ptr) __arm_streaming __arm_out("za")
+{
+  [&]{
+    /// The lambda is its own function and does not inherit the SME attributes (so this should error).
+    // expected-error@+1 {{builtin can only be called from a streaming function}}
+    svld1_hor_za64(0, slice_base, pg, ptr);
+  }();
+}
+
+void use_streaming_builtin(uint32_t slice_base, svbool_t pg, const void *ptr) __arm_streaming __arm_out("za")
+{
+  /// Without the lambda the same builtin is okay (as the SME attributes apply).
+  svld1_hor_za64(0, slice_base, pg, ptr);
+}
+
+int16x8_t use_neon_builtin_sm(int16x8_t splat) __arm_streaming_compatible {
+  // expected-error@+1 {{builtin can only be called from a non-streaming function}}
+  return (int16x8_t)__builtin_neon_vqaddq_v((int8x16_t)splat, (int8x16_t)splat, 33);
+}
+
+int16x8_t use_neon_builtin_sm_in_lambda(int16x8_t splat) __arm_streaming_compatible {
+  return [&]{
+    /// This should not error (as we switch out of streaming mode to execute the lambda).
+    /// Note: The result int16x8_t is spilled and reloaded as a q-register.
+    return (int16x8_t)__builtin_neon_vqaddq_v((int8x16_t)splat, (int8x16_t)splat, 33);
+  }();
+}
+
+float use_incomp_sve_builtin_sm() __arm_streaming {
+  // expected-error@+1 {{builtin can only be called from a non-streaming function}}
+  return svadda(svptrue_b32(), 0, svdup_f32(1));
+}
+
+float incomp_sve_sm_fadda_sm_in_lambda(void) __arm_streaming {
+  return [&]{
+    /// This should work like the Neon builtin.
+    return svadda(svptrue_b32(), 0, svdup_f32(1));
+  }();
+}
+
+void use_streaming_builtin_in_streaming_lambda(uint32_t slice_base, const void *ptr)
+{
+  [&]  __arm_new("za") () __arm_streaming {
+    // Here the lambda is streaming with ZA state, so this is okay.
+    svld1_hor_za64(0, slice_base, svptrue_b64(), ptr);
+  }();
+}
+
+int16x8_t use_neon_builtin_in_streaming_lambda(int16x8_t splat) {
+  return [&]() __arm_streaming_compatible {
+    /// This should error as the lambda is streaming-compatible.
+    // expected-error@+1 {{builtin can only be called from a non-streaming function}}
+    return (int16x8_t)__builtin_neon_vqaddq_v((int8x16_t)splat, (int8x16_t)splat, 33);
+  }();
+}
+
+float incomp_sve_fadda_in_streaming_lambda(void) {
+  return [&]() __arm_streaming {
+    // Should error (like the Neon case above).
+    // expected-error@+1 {{builtin can only be called from a non-streaming function}}
+    return svadda(svptrue_b32(), 0, svdup_f32(1));
+  }();
+}
diff --git a/clang/test/Sema/aarch64-sme-attrs-openmp-captured-region.c b/clang/test/Sema/aarch64-sme-attrs-openmp-captured-region.c
new file mode 100644
index 0000000000000..6fb7c60d02cd7
--- /dev/null
+++ b/clang/test/Sema/aarch64-sme-attrs-openmp-captured-region.c
@@ -0,0 +1,81 @@
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sme -target-feature +sme2 -fopenmp -fsyntax-only -verify %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sme -target-feature +sme2 -fopenmp -fsyntax-only -verify=expected-cpp -x c++ %s
+
+int compute(int);
+
+void streaming_openmp_captured_region(int * out) __arm_streaming {
+  // expected-error@+2 {{OpenMP captured regions are not yet supported in streaming functions}}
+  // expected-cpp-error@+1 {{OpenMP captured regions are not yet supported in streaming functions}}
+  #pragma omp parallel for num_threads(32)
+  for (int ci = 0; ci < 8; ci++) {
+    out[ci] = compute(ci);
+  }
+}
+
+__arm_locally_streaming void locally_streaming_openmp_captured_region(int * out) {
+  // expected-error@+2 {{OpenMP captured regions are not yet supported in streaming functions}}
+  // expected-cpp-error@+1 {{OpenMP captured regions are not yet supported in streaming functions}}
+  #pragma omp parallel for num_threads(32)
+  for (int ci = 0; ci < 8; ci++) {
+    out[ci] = compute(ci);
+  }
+}
+
+void za_state_captured_region(int * out) __arm_inout("za") {
+  // expected-error@+2 {{OpenMP captured regions are not yet supported in functions with ZA state}}
+  // expected-cpp-error@+1 {{OpenMP captured regions are not yet supported in functions with ZA state}}
+  #pragma omp parallel for num_threads(32)
+  for (int ci = 0; ci < 8; ci++) {
+    out[ci] = compute(ci);
+  }
+}
+
+__arm_new("za") void new_za_state_captured_region(int * out) {
+  // expected-error@+2 {{OpenMP captured regions are not yet supported in functions with ZA state}}
+  // expected-cpp-error@+1 {{OpenMP captured regions are not yet supported in functions with ZA state}}
+  #pragma omp parallel for num_threads(32)
+  for (int ci = 0; ci < 8; ci++) {
+    out[ci] = compute(ci);
+  }
+}
+
+void zt0_state_openmp_captured_region(int * out) __arm_inout("zt0") {
+  // expected-error@+2 {{OpenMP captured regions are not yet supported in functions with ZT0 state}}
+  // expected-cpp-error@+1 {{OpenMP captured regions are not yet supported in functions with ZT0 state}}
+  #pragma omp parallel for num_threads(32)
+  for (int ci = 0; ci < 8; ci++) {
+    out[ci] = compute(ci);
+  }
+}
+
+__arm_new("zt0") void new_zt0_state_openmp_captured_region(int * out) {
+  // expected-error@+2 {{OpenMP captured regions are not yet supported in functions with ZT0 state}}
+  // expected-cpp-error@+1 {{OpenMP captured regions are not yet supported in functions with ZT0 state}}
+  #pragma omp parallel for num_threads(32)
+  for (int ci = 0; ci < 8; ci++) {
+    out[ci] = compute(ci);
+  }
+}
+
+/// OpenMP directives that don't create a captured region are okay:
+
+void streaming_function_openmp(int * out) __arm_streaming __arm_inout("za", "zt0") {
+  #pragma omp unroll full
+  for (int ci = 0; ci < 8; ci++) {
+    out[ci] = compute(ci);
+  }
+}
+
+__arm_locally_streaming void locally_streaming_openmp(int * out) __arm_inout("za", "zt0") {
+  #pragma omp unroll full
+  for (int ci = 0; ci < 8; ci++) {
+    out[ci] = compute(ci);
+  }
+}
+
+__arm_new("za", "zt0") void arm_new_openmp(int * out) {
+  #pragma omp unroll full
+  for (int ci = 0; ci < 8; ci++) {
+    out[ci] = compute(ci);
+  }
+}
diff --git a/clang/test/Sema/arm-execute-only-tls.c b/clang/test/Sema/arm-execute-only-tls.c
new file mode 100644
index 0000000000000..b17ff450cb74d
--- /dev/null
+++ b/clang/test/Sema/arm-execute-only-tls.c
@@ -0,0 +1,10 @@
+// RUN: %clang_cc1 -triple arm-none-eabi                               -fsyntax-only -verify=default      %s
+// RUN: %clang_cc1 -triple arm-none-eabi -target-feature +execute-only -fsyntax-only -verify=execute-only %s
+
+// default-no-diagnostics
+
+/// Thread-local code generation requires constant pools because most of the
+/// relocations needed for it operate on data, so it cannot be used with
+/// -mexecute-only (or -mpure-code, which is aliased in the driver).
+
+_Thread_local int t; // execute-only-error {{thread-local storage is not supported for the current target}}
diff --git a/clang/test/Sema/arm-mfp8.cpp b/clang/test/Sema/arm-mfp8.cpp
index be5bc9bb71dbd..1b4e6791420ec 100644
--- a/clang/test/Sema/arm-mfp8.cpp
+++ b/clang/test/Sema/arm-mfp8.cpp
@@ -48,17 +48,27 @@ void test_vector_sve(svmfloat8_t a, svuint8_t c) {
 #include <arm_neon.h>
 
 void test_vector(mfloat8x8_t a, mfloat8x16_t b, uint8x8_t c) {
-  a + b;  // neon-error {{invalid operands to binary expression ('mfloat8x8_t' (aka '__MFloat8x8_t') and 'mfloat8x16_t' (aka '__MFloat8x16_t'))}}
-  a - b;  // neon-error {{invalid operands to binary expression ('mfloat8x8_t' (aka '__MFloat8x8_t') and 'mfloat8x16_t' (aka '__MFloat8x16_t'))}}
-  a * b;  // neon-error {{invalid operands to binary expression ('mfloat8x8_t' (aka '__MFloat8x8_t') and 'mfloat8x16_t' (aka '__MFloat8x16_t'))}}
-  a / b;  // neon-error {{invalid operands to binary expression ('mfloat8x8_t' (aka '__MFloat8x8_t') and 'mfloat8x16_t' (aka '__MFloat8x16_t'))}}
+  a + a;  // neon-error {{invalid operands to binary expression ('mfloat8x8_t' (vector of 8 'mfloat8_t' values) and 'mfloat8x8_t')}}
+  a - a;  // neon-error {{invalid operands to binary expression ('mfloat8x8_t' (vector of 8 'mfloat8_t' values) and 'mfloat8x8_t')}}
+  a * a;  // neon-error {{invalid operands to binary expression ('mfloat8x8_t' (vector of 8 'mfloat8_t' values) and 'mfloat8x8_t')}}
+  a / a;  // neon-error {{invalid operands to binary expression ('mfloat8x8_t' (vector of 8 'mfloat8_t' values) and 'mfloat8x8_t')}}
 
-  a + c;  // neon-error {{cannot convert between vector and non-scalar values ('mfloat8x8_t' (aka '__MFloat8x8_t') and 'uint8x8_t' (vector of 8 'uint8_t' values))}}
-  a - c;  // neon-error {{cannot convert between vector and non-scalar values ('mfloat8x8_t' (aka '__MFloat8x8_t') and 'uint8x8_t' (vector of 8 'uint8_t' values))}}
-  a * c;  // neon-error {{cannot convert between vector and non-scalar values ('mfloat8x8_t' (aka '__MFloat8x8_t') and 'uint8x8_t' (vector of 8 'uint8_t' values))}}
-  a / c;  // neon-error {{cannot convert between vector and non-scalar values ('mfloat8x8_t' (aka '__MFloat8x8_t') and 'uint8x8_t' (vector of 8 'uint8_t' values))}}
-  c + b;  // neon-error {{cannot convert between vector and non-scalar values ('uint8x8_t' (vector of 8 'uint8_t' values) and 'mfloat8x16_t' (aka '__MFloat8x16_t'))}}
-  c - b;  // neon-error {{cannot convert between vector and non-scalar values ('uint8x8_t' (vector of 8 'uint8_t' values) and 'mfloat8x16_t' (aka '__MFloat8x16_t'))}}
-  c * b;  // neon-error {{cannot convert between vector and non-scalar values ('uint8x8_t' (vector of 8 'uint8_t' values) and 'mfloat8x16_t' (aka '__MFloat8x16_t'))}}
-  c / b;  // neon-error {{cannot convert between vector and non-scalar values ('uint8x8_t' (vector of 8 'uint8_t' values) and 'mfloat8x16_t' (aka '__MFloat8x16_t'))}}
+  b + b;  // neon-error {{invalid operands to binary expression ('mfloat8x16_t' (vector of 16 'mfloat8_t' values) and 'mfloat8x16_t')}}
+  b - b;  // neon-error {{invalid operands to binary expression ('mfloat8x16_t' (vector of 16 'mfloat8_t' values) and 'mfloat8x16_t')}}
+  b * b;  // neon-error {{invalid operands to binary expression ('mfloat8x16_t' (vector of 16 'mfloat8_t' values) and 'mfloat8x16_t')}}
+  b / b;  // neon-error {{invalid operands to binary expression ('mfloat8x16_t' (vector of 16 'mfloat8_t' values) and 'mfloat8x16_t')}}
+
+  a + b;  // neon-error {{invalid operands to binary expression ('mfloat8x8_t' (vector of 8 'mfloat8_t' values) and 'mfloat8x16_t' (vector of 16 'mfloat8_t' values))}}
+  a - b;  // neon-error {{invalid operands to binary expression ('mfloat8x8_t' (vector of 8 'mfloat8_t' values) and 'mfloat8x16_t' (vector of 16 'mfloat8_t' values))}}
+  a * b;  // neon-error {{invalid operands to binary expression ('mfloat8x8_t' (vector of 8 'mfloat8_t' values) and 'mfloat8x16_t' (vector of 16 'mfloat8_t' values))}}
+  a / b;  // neon-error {{invalid operands to binary expression ('mfloat8x8_t' (vector of 8 'mfloat8_t' values) and 'mfloat8x16_t' (vector of 16 'mfloat8_t' values))}}
+
+  a + c;  // neon-error {{invalid operands to binary expression ('mfloat8x8_t' (vector of 8 'mfloat8_t' values) and 'uint8x8_t' (vector of 8 'uint8_t' values))}}
+  a - c;  // neon-error {{invalid operands to binary expression ('mfloat8x8_t' (vector of 8 'mfloat8_t' values) and 'uint8x8_t' (vector of 8 'uint8_t' values))}}
+  a * c;  // neon-error {{invalid operands to binary expression ('mfloat8x8_t' (vector of 8 'mfloat8_t' values) and 'uint8x8_t' (vector of 8 'uint8_t' values))}}
+  a / c;  // neon-error {{invalid operands to binary expression ('mfloat8x8_t' (vector of 8 'mfloat8_t' values) and 'uint8x8_t' (vector of 8 'uint8_t' values))}}
+  c + b;  // neon-error {{invalid operands to binary expression ('uint8x8_t' (vector of 8 'uint8_t' values) and 'mfloat8x16_t' (vector of 16 'mfloat8_t' values))}}
+  c - b;  // neon-error {{invalid operands to binary expression ('uint8x8_t' (vector of 8 'uint8_t' values) and 'mfloat8x16_t' (vector of 16 'mfloat8_t' values))}}
+  c * b;  // neon-error {{invalid operands to binary expression ('uint8x8_t' (vector of 8 'uint8_t' values) and 'mfloat8x16_t' (vector of 16 'mfloat8_t' values))}}
+  c / b;  // neon-error {{invalid operands to binary expression ('uint8x8_t' (vector of 8 'uint8_t' values) and 'mfloat8x16_t' (vector of 16 'mfloat8_t' values))}}
 }
diff --git a/clang/test/Sema/attr-cpuspecific.c b/clang/test/Sema/attr-cpuspecific.c
index 3cd58f49faa5e..238db0ac0b85d 100644
--- a/clang/test/Sema/attr-cpuspecific.c
+++ b/clang/test/Sema/attr-cpuspecific.c
@@ -44,7 +44,8 @@ int allow_fwd_decl2(void);
 void use_fwd_decl(void) {
   allow_fwd_decl2();
 }
-// expected-error@+1 {{function declaration cannot become a multiversioned function after first usage}}
+// expected-error@+2 {{function declaration cannot become a multiversioned function after first usage}}
+// expected-note@-5 {{previous declaration is here}}
 int __attribute__((cpu_dispatch(atom))) allow_fwd_decl2(void) {}
 
 
diff --git a/clang/test/Sema/attr-target-mv.c b/clang/test/Sema/attr-target-mv.c
index ddb1d82b02f09..dfc3d614dc1e0 100644
--- a/clang/test/Sema/attr-target-mv.c
+++ b/clang/test/Sema/attr-target-mv.c
@@ -66,7 +66,8 @@ int use3(void) {
   return mv_after_use();
 }
 
-// expected-error@+1 {{function declaration cannot become a multiversioned function after first usage}}
+// expected-error@+2 {{function declaration cannot become a multiversioned function after first usage}}
+// expected-note@-6 {{previous declaration is here}}
 int __attribute__((target("arch=sandybridge")))  mv_after_use(void) { return 2; }
 
 int __attribute__((target("sse4.2,arch=sandybridge"))) mangle(void) { return 1; }
diff --git a/clang/test/Sema/attr-target-version.c b/clang/test/Sema/attr-target-version.c
index cfcc1622abe5c..d062212848daf 100644
--- a/clang/test/Sema/attr-target-version.c
+++ b/clang/test/Sema/attr-target-version.c
@@ -88,7 +88,8 @@ int bar() {
   nodef();
   return def();
 }
-// expected-error@+1 {{function declaration cannot become a multiversioned function after first usage}}
+// expected-error@+2 {{function declaration cannot become a multiversioned function after first usage}}
+// expected-note@-13 {{previous declaration is here}}
 int __attribute__((target_version("sha2"))) def(void) { return 1; }
 
 int __attribute__((target_version("sve"))) prot();
diff --git a/clang/test/Sema/avr-interript-signal-attr.c b/clang/test/Sema/avr-interript-signal-attr.c
new file mode 100644
index 0000000000000..7360dbdedb695
--- /dev/null
+++ b/clang/test/Sema/avr-interript-signal-attr.c
@@ -0,0 +1,22 @@
+// RUN: %clang_cc1 %s -triple avr-unknown-unknown -verify -fsyntax-only
+struct a { int b; };
+
+struct a test __attribute__((interrupt)); // expected-warning {{'interrupt' attribute only applies to functions}}
+
+__attribute__((interrupt(12))) void foo(void) { } // expected-error {{'interrupt' attribute takes no arguments}}
+
+__attribute__((interrupt)) int fooa(void) { return 0; } // expected-warning {{'interrupt' attribute only applies to functions that have a 'void' return type}}
+
+__attribute__((interrupt)) void foob(int a) {} // expected-warning {{'interrupt' attribute only applies to functions that have no parameters}}
+
+__attribute__((signal)) int fooc(void) { return 0; } // expected-warning {{'signal' attribute only applies to functions that have a 'void' return type}}
+
+__attribute__((signal)) void food(int a) {} // expected-warning {{'signal' attribute only applies to functions that have no parameters}}
+
+__attribute__((interrupt)) void fooe(void) {}
+
+__attribute__((interrupt)) void foof() {}
+
+__attribute__((signal)) void foog(void) {}
+
+__attribute__((signal)) void fooh() {}
diff --git a/clang/test/Sema/avr-interrupt-attr.c b/clang/test/Sema/avr-interrupt-attr.c
deleted file mode 100644
index 7aee7babcbe16..0000000000000
--- a/clang/test/Sema/avr-interrupt-attr.c
+++ /dev/null
@@ -1,8 +0,0 @@
-// RUN: %clang_cc1 %s -triple avr-unknown-unknown -verify -fsyntax-only
-struct a { int b; };
-
-struct a test __attribute__((interrupt)); // expected-warning {{'interrupt' attribute only applies to functions}}
-
-__attribute__((interrupt(12))) void foo(void) { } // expected-error {{'interrupt' attribute takes no arguments}}
-
-__attribute__((interrupt)) void food(void) {}
diff --git a/clang/test/Sema/avr-signal-attr.c b/clang/test/Sema/avr-signal-attr.c
deleted file mode 100644
index 1ec36c74a25eb..0000000000000
--- a/clang/test/Sema/avr-signal-attr.c
+++ /dev/null
@@ -1,8 +0,0 @@
-// RUN: %clang_cc1 %s -triple avr-unknown-unknown -verify -fsyntax-only
-struct a { int b; };
-
-struct a test __attribute__((signal)); // expected-warning {{'signal' attribute only applies to functions}}
-
-__attribute__((signal(12))) void foo(void) { } // expected-error {{'signal' attribute takes no arguments}}
-
-__attribute__((signal)) void food(void) {}
diff --git a/clang/test/Sema/builtins-elementwise-math.c b/clang/test/Sema/builtins-elementwise-math.c
index 6002e91f8ec6f..d6785f78340ca 100644
--- a/clang/test/Sema/builtins-elementwise-math.c
+++ b/clang/test/Sema/builtins-elementwise-math.c
@@ -69,14 +69,18 @@ void test_builtin_elementwise_add_sat(int i, short s, double d, float4 v, int3 i
   // expected-error@-1 {{1st argument must be a vector of integers (was 'float4' (vector of 4 'float' values))}}
 
   s = __builtin_elementwise_add_sat(i, s);
+  // expected-error@-1 {{arguments are of different types ('int' vs 'short')}}
 
   enum e { one,
            two };
   i = __builtin_elementwise_add_sat(one, two);
 
+  i = __builtin_elementwise_add_sat(one, d);
+  // expected-error@-1 {{arguments are of different types ('int' vs 'double')}}
+
   enum f { three };
   enum f x = __builtin_elementwise_add_sat(one, three);
-  // expected-warning@-1 {{comparison of different enumeration types ('enum e' and 'enum f')}}
+  // expected-error@-1 {{invalid arithmetic between different enumeration types ('enum e' and 'enum f')}}
 
   _BitInt(32) ext; // expected-warning {{'_BitInt' in C17 and earlier is a Clang extension}}
   ext = __builtin_elementwise_add_sat(ext, ext);
@@ -128,14 +132,18 @@ void test_builtin_elementwise_sub_sat(int i, short s, double d, float4 v, int3 i
   // expected-error@-1 {{1st argument must be a vector of integers (was 'float4' (vector of 4 'float' values))}}
 
   s = __builtin_elementwise_sub_sat(i, s);
+  // expected-error@-1 {{arguments are of different types ('int' vs 'short')}}
 
   enum e { one,
            two };
   i = __builtin_elementwise_sub_sat(one, two);
 
+  i = __builtin_elementwise_sub_sat(one, d);
+  // expected-error@-1 {{arguments are of different types ('int' vs 'double')}}
+
   enum f { three };
   enum f x = __builtin_elementwise_sub_sat(one, three);
-  // expected-warning@-1 {{comparison of different enumeration types ('enum e' and 'enum f')}}
+  // expected-error@-1 {{invalid arithmetic between different enumeration types ('enum e' and 'enum f')}}
 
   _BitInt(32) ext; // expected-warning {{'_BitInt' in C17 and earlier is a Clang extension}}
   ext = __builtin_elementwise_sub_sat(ext, ext);
@@ -184,14 +192,18 @@ void test_builtin_elementwise_max(int i, short s, double d, float4 v, int3 iv, u
   // expected-error@-1 {{arguments are of different types ('unsigned3' (vector of 3 'unsigned int' values) vs 'int3' (vector of 3 'int' values))}}
 
   s = __builtin_elementwise_max(i, s);
+  // expected-error@-1 {{arguments are of different types ('int' vs 'short')}}
 
   enum e { one,
            two };
   i = __builtin_elementwise_max(one, two);
 
+  i = __builtin_elementwise_max(one, d);
+  // expected-error@-1 {{arguments are of different types ('int' vs 'double')}}
+
   enum f { three };
   enum f x = __builtin_elementwise_max(one, three);
-  // expected-warning@-1 {{comparison of different enumeration types ('enum e' and 'enum f')}}
+  // expected-error@-1 {{invalid arithmetic between different enumeration types ('enum e' and 'enum f')}}
 
   _BitInt(32) ext; // expected-warning {{'_BitInt' in C17 and earlier is a Clang extension}}
   ext = __builtin_elementwise_max(ext, ext);
@@ -240,14 +252,18 @@ void test_builtin_elementwise_min(int i, short s, double d, float4 v, int3 iv, u
   // expected-error@-1 {{arguments are of different types ('unsigned3' (vector of 3 'unsigned int' values) vs 'int3' (vector of 3 'int' values))}}
 
   s = __builtin_elementwise_min(i, s);
+  // expected-error@-1 {{arguments are of different types ('int' vs 'short')}}
 
   enum e { one,
            two };
   i = __builtin_elementwise_min(one, two);
 
+  i = __builtin_elementwise_min(one, d);
+  // expected-error@-1 {{arguments are of different types ('int' vs 'double')}}
+
   enum f { three };
   enum f x = __builtin_elementwise_min(one, three);
-  // expected-warning@-1 {{comparison of different enumeration types ('enum e' and 'enum f')}}
+  // expected-error@-1 {{invalid arithmetic between different enumeration types ('enum e' and 'enum f')}}
 
   _BitInt(32) ext; // expected-warning {{'_BitInt' in C17 and earlier is a Clang extension}}
   ext = __builtin_elementwise_min(ext, ext);
@@ -273,7 +289,7 @@ void test_builtin_elementwise_min(int i, short s, double d, float4 v, int3 iv, u
   // expected-error@-1 {{1st argument must be a vector, integer or floating point type (was '_Complex float')}}
 }
 
-void test_builtin_elementwise_maximum(int i, short s, float f, double d, float4 v, int3 iv, unsigned3 uv, int *p) {
+void test_builtin_elementwise_maximum(int i, short s, float f, double d, float4 fv, double4 dv, int3 iv, unsigned3 uv, int *p) {
   i = __builtin_elementwise_maximum(p, d);
   // expected-error@-1 {{arguments are of different types ('int *' vs 'double')}}
 
@@ -289,15 +305,19 @@ void test_builtin_elementwise_maximum(int i, short s, float f, double d, float4
   i = __builtin_elementwise_maximum(i, i, i);
   // expected-error@-1 {{too many arguments to function call, expected 2, have 3}}
 
-  i = __builtin_elementwise_maximum(v, iv);
+  i = __builtin_elementwise_maximum(fv, iv);
   // expected-error@-1 {{arguments are of different types ('float4' (vector of 4 'float' values) vs 'int3' (vector of 3 'int' values))}}
 
   i = __builtin_elementwise_maximum(uv, iv);
   // expected-error@-1 {{arguments are of different types ('unsigned3' (vector of 3 'unsigned int' values) vs 'int3' (vector of 3 'int' values))}}
 
+  dv = __builtin_elementwise_maximum(fv, dv);
+  // expected-error@-1 {{arguments are of different types ('float4' (vector of 4 'float' values) vs 'double4' (vector of 4 'double' values))}}
+
   d = __builtin_elementwise_maximum(f, d);
+  // expected-error@-1 {{arguments are of different types ('float' vs 'double')}}
 
-  v = __builtin_elementwise_maximum(v, v);
+  fv = __builtin_elementwise_maximum(fv, fv);
 
   i = __builtin_elementwise_maximum(iv, iv);
   // expected-error@-1 {{1st argument must be a floating point type (was 'int3' (vector of 3 'int' values))}}
@@ -314,7 +334,7 @@ void test_builtin_elementwise_maximum(int i, short s, float f, double d, float4
   // expected-error@-1 {{1st argument must be a floating point type (was '_Complex float')}}
 }
 
-void test_builtin_elementwise_minimum(int i, short s, float f, double d, float4 v, int3 iv, unsigned3 uv, int *p) {
+void test_builtin_elementwise_minimum(int i, short s, float f, double d, float4 fv, double4 dv, int3 iv, unsigned3 uv, int *p) {
   i = __builtin_elementwise_minimum(p, d);
   // expected-error@-1 {{arguments are of different types ('int *' vs 'double')}}
 
@@ -330,15 +350,19 @@ void test_builtin_elementwise_minimum(int i, short s, float f, double d, float4
   i = __builtin_elementwise_minimum(i, i, i);
   // expected-error@-1 {{too many arguments to function call, expected 2, have 3}}
 
-  i = __builtin_elementwise_minimum(v, iv);
+  i = __builtin_elementwise_minimum(fv, iv);
   // expected-error@-1 {{arguments are of different types ('float4' (vector of 4 'float' values) vs 'int3' (vector of 3 'int' values))}}
 
   i = __builtin_elementwise_minimum(uv, iv);
   // expected-error@-1 {{arguments are of different types ('unsigned3' (vector of 3 'unsigned int' values) vs 'int3' (vector of 3 'int' values))}}
 
+  dv = __builtin_elementwise_minimum(fv, dv);
+  // expected-error@-1 {{arguments are of different types ('float4' (vector of 4 'float' values) vs 'double4' (vector of 4 'double' values))}}
+
   d = __builtin_elementwise_minimum(f, d);
+  // expected-error@-1 {{arguments are of different types ('float' vs 'double')}}
 
-  v = __builtin_elementwise_minimum(v, v);
+  fv = __builtin_elementwise_minimum(fv, fv);
 
   i = __builtin_elementwise_minimum(iv, iv);
   // expected-error@-1 {{1st argument must be a floating point type (was 'int3' (vector of 3 'int' values))}}
diff --git a/clang/test/Sema/diagnose_if.c b/clang/test/Sema/diagnose_if.c
index 4df39916c031e..e9b8497d5ca4e 100644
--- a/clang/test/Sema/diagnose_if.c
+++ b/clang/test/Sema/diagnose_if.c
@@ -2,10 +2,10 @@
 
 #define _diagnose_if(...) __attribute__((diagnose_if(__VA_ARGS__)))
 
-void failure1(void) _diagnose_if(); // expected-error{{exactly 3 arguments}}
-void failure2(void) _diagnose_if(0); // expected-error{{exactly 3 arguments}}
-void failure3(void) _diagnose_if(0, ""); // expected-error{{exactly 3 arguments}}
-void failure4(void) _diagnose_if(0, "", "error", 1); // expected-error{{exactly 3 arguments}}
+void failure1(void) _diagnose_if(); // expected-error{{at least 3 arguments}}
+void failure2(void) _diagnose_if(0); // expected-error{{at least 3 arguments}}
+void failure3(void) _diagnose_if(0, ""); // expected-error{{at least 3 arguments}}
+void failure4(void) _diagnose_if(0, "", "error", 1); // expected-error{{expected string literal as argument}}
 void failure5(void) _diagnose_if(0, 0, "error"); // expected-error{{expected string literal as argument of 'diagnose_if' attribute}}
 void failure6(void) _diagnose_if(0, "", "invalid"); // expected-error{{invalid diagnostic type for 'diagnose_if'; use "error" or "warning" instead}}
 void failure7(void) _diagnose_if(0, "", "ERROR"); // expected-error{{invalid diagnostic type}}
diff --git a/clang/test/Rewriter/missing-dllimport.c b/clang/test/Sema/missing-dllimport.c
similarity index 100%
rename from clang/test/Rewriter/missing-dllimport.c
rename to clang/test/Sema/missing-dllimport.c
diff --git a/clang/test/Sema/nullptr-prec2x.c b/clang/test/Sema/nullptr-prec2x.c
index 39479d4343a56..c516c448ca1ab 100644
--- a/clang/test/Sema/nullptr-prec2x.c
+++ b/clang/test/Sema/nullptr-prec2x.c
@@ -6,3 +6,7 @@ int nullptr; // expected-warning {{'nullptr' is a keyword in C23}}
 
 nullptr_t val; // expected-error {{unknown type name 'nullptr_t'}}
 
+void foo(void *);
+void bar() { foo(__nullptr); } // Test that it converts properly to an arbitrary pointer type without warning
+_Static_assert(__nullptr == 0, "value of __nullptr"); // Test that its value matches that of NULL
+_Static_assert(_Generic(__typeof(__nullptr), int : 0, void * : 0, default : 1), "type of __nullptr"); // Test that it's type is not the same as what NULL would generally have.
diff --git a/clang/test/Sema/nullptr.c b/clang/test/Sema/nullptr.c
index d11765a9c881a..b8c371a418e3e 100644
--- a/clang/test/Sema/nullptr.c
+++ b/clang/test/Sema/nullptr.c
@@ -108,3 +108,10 @@ void test_f1() {
   int ir = (f1)(nullptr);
 }
 
+// __nullptr keyword in C
+void foo(void *);
+void bar() { foo(__nullptr); }
+static_assert(nullptr == __nullptr);
+static_assert(__nullptr == 0);  // Test that its value matches that of NULL
+static_assert(_Generic(typeof(__nullptr), nullptr_t: true, default: false));
+static_assert(_Generic(__typeof(__nullptr), int : 0, void * : 0, default : 1)); // Test that it's type is not the same as what NULL would generally have.
diff --git a/clang/test/Sema/tautological-pointer-comparison.c b/clang/test/Sema/tautological-pointer-comparison.c
index 1c5973b01a30d..f2a944b5305e4 100644
--- a/clang/test/Sema/tautological-pointer-comparison.c
+++ b/clang/test/Sema/tautological-pointer-comparison.c
@@ -1,5 +1,5 @@
 // RUN: %clang_cc1 -fsyntax-only -verify %s
-// RUN: %clang_cc1 -fsyntax-only -fwrapv -verify=fwrapv %s
+// RUN: %clang_cc1 -fsyntax-only -fwrapv-pointer -verify=fwrapv %s
 
 // fwrapv-no-diagnostics
 
diff --git a/clang/test/SemaCUDA/device-var-init.cu b/clang/test/SemaCUDA/device-var-init.cu
index 1555d151c2590..a9e3557c20ebf 100644
--- a/clang/test/SemaCUDA/device-var-init.cu
+++ b/clang/test/SemaCUDA/device-var-init.cu
@@ -485,3 +485,12 @@ void instantiate() {
   bar<NontrivialInitializer><<<1, 1>>>();
 // expected-note@-1 {{in instantiation of function template specialization 'bar<NontrivialInitializer>' requested here}}
 }
+
+__device__ void *ptr1 = nullptr;
+__device__ void *ptr2 = ptr1;
+// expected-error@-1 {{dynamic initialization is not supported for __device__, __constant__, __shared__, and __managed__ variables}}
+
+__device__ [[gnu::constructor(101)]] void ctor() {}
+// expected-error@-1 {{CUDA does not support global constructors for __device__ functions}}
+__device__ [[gnu::destructor(101)]] void dtor() {}
+// expected-error@-1 {{CUDA does not support global destructors for __device__ functions}}
diff --git a/clang/test/SemaCUDA/inherited-ctor.cu b/clang/test/SemaCUDA/inherited-ctor.cu
index 8ac59e7b539f3..ef3938555b983 100644
--- a/clang/test/SemaCUDA/inherited-ctor.cu
+++ b/clang/test/SemaCUDA/inherited-ctor.cu
@@ -81,7 +81,7 @@ namespace DefaultCtorInvalid {
   };
 
   struct C {
-    struct B b;
+    struct B b; // expected-note{{default constructed field 'b' declared here}}
     C() {} // expected-error{{call to implicitly-deleted default constructor of 'struct B'}}
            // expected-note@-6{{default constructor of 'B' is implicitly deleted because field 's' has a deleted default constructor}}
            // expected-note@-15{{'S' has been explicitly marked deleted here}}
diff --git a/clang/test/SemaCXX/alias-template.cpp b/clang/test/SemaCXX/alias-template.cpp
index 5189405e23db5..b49d36a6267e6 100644
--- a/clang/test/SemaCXX/alias-template.cpp
+++ b/clang/test/SemaCXX/alias-template.cpp
@@ -54,18 +54,24 @@ namespace LookupFilter {
   template<typename U> using S = S<U>*; // ok
 }
 
-namespace InFunctions {
+namespace UnexpandedPack {
   template<typename...T> struct S0 {
     template<typename Z> using U = T*; // expected-error {{declaration type contains unexpanded parameter pack 'T'}}
     U<char> u;
   };
+}
 
+namespace InvalidType {
   template<typename Z> using T1 = int;
   template<typename Z> using T2 = int[-1]; // expected-error {{array size is negative}}
+}
+
+namespace ShadowTemplateParam {
   template<typename...T> struct S3 { // expected-note {{template parameter is declared here}}
     template<typename Z> using T = int; // expected-error {{declaration of 'T' shadows template parameter}}
   };
-  template<typename Z> using Z = Z;
+  template<typename Z> // expected-note {{template parameter is declared here}}
+  using Z = Z; // expected-error {{declaration of 'Z' shadows template parameter}}
 }
 
 namespace ClassNameRedecl {
diff --git a/clang/test/SemaCXX/array-type-trait-with-template.cpp b/clang/test/SemaCXX/array-type-trait-with-template.cpp
new file mode 100644
index 0000000000000..942714ec5d55a
--- /dev/null
+++ b/clang/test/SemaCXX/array-type-trait-with-template.cpp
@@ -0,0 +1,129 @@
+// RUN: %clang_cc1 -fsyntax-only %s
+// RUN: %clang_cc1 -fsyntax-only -std=c++20 -DWITH_AUTO_FUNCTION_PARAMETER=1 %s
+
+// When __array_rank is used with a template type parameter, this test
+// ensures clang considers the final expression could be used with
+// static_assert/constexpr.
+//
+// Although array_extent was handled well, we add it as a precaution.
+
+template <typename T>
+using remove_reference_t = __remove_reference_t(T);
+
+template <typename T, int N>
+constexpr int array_rank(T (&lhs)[N]) {
+  return __array_rank(T[N]);
+}
+
+template <int I, typename T, int N>
+ constexpr int array_extent(T (&lhs)[N]) {
+  return __array_extent(T[N], I);
+}
+
+template <typename T>
+struct Rank {
+  using ArrayT = remove_reference_t<T>;
+
+  template <int N>
+  static constexpr int call(ArrayT (&lhs)[N]) {
+    return __array_rank(ArrayT[N]);
+  }
+};
+
+template <typename T>
+struct Extent {
+  using ArrayT = remove_reference_t<T>;
+
+  template <int I, int N>
+  static constexpr int call(ArrayT (&lhs)[N]) {
+    return __array_extent(ArrayT[N], I);
+  }
+};
+
+#ifdef WITH_AUTO_FUNCTION_PARAMETER
+template <int N>
+constexpr int array_rank_auto(auto (&lhs)[N]) {
+  return __array_rank(remove_reference_t<decltype(lhs[0])>[N]);
+}
+
+template <int I, int N>
+constexpr int array_extent_auto(auto (&lhs)[N]) {
+  return __array_extent(remove_reference_t<decltype(lhs[0])>[N], I);
+}
+#endif
+
+template <int N>
+constexpr int array_rank_int(const int (&lhs)[N]) {
+  return __array_rank(const int[N]);
+}
+
+template <int I, int N>
+constexpr int array_extent_int(const int (&lhs)[N]) {
+  return __array_extent(const int[N], I);
+}
+
+template <int M, int N>
+constexpr int array_rank_int(const int (&lhs)[M][N]) {
+  return __array_rank(const int[M][N]);
+}
+
+template <int I, int M, int N>
+constexpr int array_extent_int(const int (&lhs)[M][N]) {
+  return __array_extent(const int[M][N], I);
+}
+
+int main() {
+  constexpr int vec[] = {0, 1, 2, 1};
+  constexpr int mat[4][4] = {
+    {1, 0, 0, 0},
+    {0, 1, 0, 0},
+    {0, 0, 1, 0},
+    {0, 0, 0, 1}
+  };
+
+#define ATT_TESTS_WITH_ASSERT(ATT_ASSERT)	\
+  { ATT_ASSERT(RANK(vec) == 1);	}		\
+  { ATT_ASSERT(RANK(mat) == 2);	}		\
+  { ATT_ASSERT(EXTENT(vec, 0) == 4); }		\
+  { ATT_ASSERT(EXTENT(vec, 1) == 0); }		\
+  { ATT_ASSERT(EXTENT(mat, 1) == 4); }
+
+#define ATT_TESTS()				\
+  ATT_TESTS_WITH_ASSERT( constexpr bool cst = )	\
+  ATT_TESTS_WITH_ASSERT( (void) )		\
+  ATT_TESTS_WITH_ASSERT( static_assert )
+
+  {
+#define RANK(lhs) array_rank(lhs)
+#define EXTENT(lhs, i) array_extent<i>(lhs)
+    ATT_TESTS();
+#undef RANK
+#undef EXTENT
+  }
+
+  {
+#define RANK(lhs) Rank<decltype(lhs[0])>::call(lhs)
+#define EXTENT(lhs, i) Extent<decltype(lhs[0])>::call<i>(lhs)
+    ATT_TESTS();
+#undef RANK
+#undef EXTENT
+  }
+
+#ifdef WITH_AUTO_FUNCTION_PARAMETER
+  {
+#define RANK(lhs) array_rank_auto(lhs)
+#define EXTENT(lhs, i) array_extent_auto<i>(lhs)
+    ATT_TESTS();
+#undef RANK
+#undef EXTENT
+  }
+#endif
+
+  {
+#define RANK(lhs) array_rank_int(lhs)
+#define EXTENT(lhs, i) array_extent_int<i>(lhs)
+    ATT_TESTS();
+#undef RANK
+#undef EXTENT
+  }
+}
diff --git a/clang/test/SemaCXX/attr-annotate.cpp b/clang/test/SemaCXX/attr-annotate.cpp
index 846ef4119f1d7..ff538fee9cb3c 100644
--- a/clang/test/SemaCXX/attr-annotate.cpp
+++ b/clang/test/SemaCXX/attr-annotate.cpp
@@ -134,3 +134,7 @@ constexpr int foldable_but_invalid() {
 template <typename T> [[clang::annotate()]] void f2() {}
 // expected-error@-1 {{'annotate' attribute takes at least 1 argument}}
 }
+
+namespace test5 {
+  void bir [[clang::annotate("B", (void)1)]] ();
+}
diff --git a/clang/test/SemaCXX/attr-cxx0x.cpp b/clang/test/SemaCXX/attr-cxx0x.cpp
index 4d64d2b0cd8c6..58180a21ca8e9 100644
--- a/clang/test/SemaCXX/attr-cxx0x.cpp
+++ b/clang/test/SemaCXX/attr-cxx0x.cpp
@@ -53,3 +53,26 @@ alignas(4) auto PR19252 = 0;
 
 // Check the diagnostic message
 class alignas(void) AlignasVoid {}; // expected-error {{invalid application of 'alignas' to an incomplete type 'void'}}
+
+namespace GH108819 {
+void a([[maybe_unused]] void) {}                 // expected-warning {{attribute 'maybe_unused' cannot be applied to a 'void' parameter}}\
+                                                 // expected-warning {{use of the 'maybe_unused' attribute is a C++17 extension}}
+void b([[deprecated, maybe_unused]] void) {}     // expected-warning {{attribute 'deprecated' cannot be applied to a 'void' parameter}} \
+                                                 // expected-warning {{attribute 'maybe_unused' cannot be applied to a 'void' parameter}} \
+                                                 // expected-warning {{use of the 'deprecated' attribute is a C++14 extension}} \
+                                                 // expected-warning {{use of the 'maybe_unused' attribute is a C++17 extension}}
+void c([[clang::lifetimebound]] void) {}         // expected-warning {{attribute 'lifetimebound' cannot be applied to a 'void' parameter}}
+void d([[clang::annotate("a", "b", 1)]] void) {} // expected-warning {{attribute 'annotate' cannot be applied to a 'void' parameter}}
+
+struct S {
+  void e([[maybe_unused]] void) {} // expected-warning {{attribute 'maybe_unused' cannot be applied to a 'void' parameter}} \
+                                   // expected-warning {{use of the 'maybe_unused' attribute is a C++17 extension}}
+};
+
+template <typename T>
+void f([[maybe_unused]] void) {} // expected-warning {{attribute 'maybe_unused' cannot be applied to a 'void' parameter}} \
+                                 // expected-warning {{use of the 'maybe_unused' attribute is a C++17 extension}}
+
+auto g = []([[maybe_unused]] void) { }; // expected-warning {{attribute 'maybe_unused' cannot be applied to a 'void' parameter}} \
+                                        // expected-warning {{use of the 'maybe_unused' attribute is a C++17 extension}}
+}
diff --git a/clang/test/SemaCXX/attr-no-sanitize.cpp b/clang/test/SemaCXX/attr-no-sanitize.cpp
index 8951f616ce0f0..cd60e71963ac3 100644
--- a/clang/test/SemaCXX/attr-no-sanitize.cpp
+++ b/clang/test/SemaCXX/attr-no-sanitize.cpp
@@ -21,8 +21,8 @@ int f3() __attribute__((no_sanitize("address")));
 
 // DUMP-LABEL: FunctionDecl {{.*}} f4
 // DUMP: NoSanitizeAttr {{.*}} hwaddress
-// PRINT: {{\[\[}}clang::no_sanitize("hwaddress")]] int f4()
-[[clang::no_sanitize("hwaddress")]] int f4();
+// PRINT: {{\[\[}}gnu::no_sanitize("hwaddress")]] int f4()
+[[gnu::no_sanitize("hwaddress")]] int f4();
 
 // DUMP-LABEL: FunctionDecl {{.*}} f5
 // DUMP: NoSanitizeAttr {{.*}} address thread hwaddress
diff --git a/clang/test/SemaCXX/constant-expression-cxx11.cpp b/clang/test/SemaCXX/constant-expression-cxx11.cpp
index a1234b67acd6d..76e2f81947051 100644
--- a/clang/test/SemaCXX/constant-expression-cxx11.cpp
+++ b/clang/test/SemaCXX/constant-expression-cxx11.cpp
@@ -1462,7 +1462,7 @@ namespace InstantiateCaseStmt {
 
 namespace ConvertedConstantExpr {
   extern int &m;
-  extern int &n; // expected-note 2{{declared here}}
+  extern int &n; // pre-cxx23-note 2{{declared here}}
 
   constexpr int k = 4;
   int &m = const_cast<int&>(k);
@@ -1471,9 +1471,9 @@ namespace ConvertedConstantExpr {
   // useless note and instead just point to the non-constant subexpression.
   enum class E {
     em = m,
-    en = n, // cxx23-note {{initializer of 'n' is not a constant expression}} expected-error {{enumerator value is not a constant expression}} cxx11_20-note {{initializer of 'n' is unknown}}
-    eo = (m + // expected-error {{not a constant expression}}
-          n // cxx23-note {{initializer of 'n' is not a constant expression}} cxx11_20-note {{initializer of 'n' is unknown}}
+    en = n, // expected-error {{enumerator value is not a constant expression}} cxx11_20-note {{initializer of 'n' is unknown}}
+    eo = (m + // pre-cxx23-error {{not a constant expression}}
+          n // cxx11_20-note {{initializer of 'n' is unknown}} cxx23-error {{not a constant expression}}
           ),
     eq = reinterpret_cast<long>((int*)0) // expected-error {{not a constant expression}} expected-note {{reinterpret_cast}}
   };
diff --git a/clang/test/SemaCXX/constant-expression-p2280r4.cpp b/clang/test/SemaCXX/constant-expression-p2280r4.cpp
index 0f85c60629eed..65e5e6b34b48f 100644
--- a/clang/test/SemaCXX/constant-expression-p2280r4.cpp
+++ b/clang/test/SemaCXX/constant-expression-p2280r4.cpp
@@ -1,4 +1,5 @@
 // RUN: %clang_cc1 -std=c++23 -verify %s
+// RUN: %clang_cc1 -std=c++23 -verify %s -fexperimental-new-constant-interpreter
 
 using size_t = decltype(sizeof(0));
 
@@ -47,11 +48,10 @@ void splash(Swim& swam) {
 }
 
 extern Swim dc;
-extern Swim& trident; // expected-note {{declared here}}
+extern Swim& trident;
 
 constexpr auto& sandeno   = typeid(dc);         // ok: can only be typeid(Swim)
 constexpr auto& gallagher = typeid(trident);    // expected-error {{constexpr variable 'gallagher' must be initialized by a constant expression}}
-                                                // expected-note@-1 {{initializer of 'trident' is not a constant expression}}
 
 namespace explicitThis {
 struct C {
diff --git a/clang/test/SemaCXX/cxx1z-constexpr-lambdas.cpp b/clang/test/SemaCXX/cxx1z-constexpr-lambdas.cpp
index 6a1f48bf7958f..0c20dd9dc58c6 100644
--- a/clang/test/SemaCXX/cxx1z-constexpr-lambdas.cpp
+++ b/clang/test/SemaCXX/cxx1z-constexpr-lambdas.cpp
@@ -349,3 +349,27 @@ static_assert(OtherCaptures(), "");
 } // namespace PR36054
 
 #endif // ndef CPP14_AND_EARLIER
+
+
+#if __cpp_constexpr >= 201907L
+namespace GH114234 {
+template <auto Arg>
+auto g() { return Arg; }
+
+template <typename>
+auto f() {
+    []<typename>() {
+        g<[] { return 123; }()>();
+    }.template operator()<int>();
+}
+
+void test() { f<int>(); }
+}
+
+namespace GH97958 {
+static_assert(
+  []<int I=0>() -> decltype([]{ return true; })
+  { return {}; }()());
+}
+
+#endif
diff --git a/clang/test/SemaCXX/cxx1z-decomposition.cpp b/clang/test/SemaCXX/cxx1z-decomposition.cpp
index a8914fe4e9cd8..95c64bc3b8bff 100644
--- a/clang/test/SemaCXX/cxx1z-decomposition.cpp
+++ b/clang/test/SemaCXX/cxx1z-decomposition.cpp
@@ -200,38 +200,32 @@ namespace lambdas {
 
 namespace by_value_array_copy {
   struct explicit_copy {
-    explicit_copy() = default; // expected-note 2{{candidate constructor not viable: requires 0 arguments, but 1 was provided}}
-    explicit explicit_copy(const explicit_copy&) = default; // expected-note 2{{explicit constructor is not a candidate}}
+    explicit_copy() = default; // expected-note {{candidate constructor not viable: requires 0 arguments, but 1 was provided}}
+    explicit explicit_copy(const explicit_copy&) = default; // expected-note {{explicit constructor is not a candidate}}
   };
 
-  constexpr int direct_initialization_for_elements() {
-    explicit_copy ec_arr[2];
-    auto [a1, b1](ec_arr);
+  constexpr int simple_array_elements() {
+    int arr[2]{1, 2};
 
-    int arr[3]{1, 2, 3};
-    auto [a2, b2, c2](arr);
-    arr[0]--;
-    return a2 + b2 + c2 + arr[0];
-  }
-  static_assert(direct_initialization_for_elements() == 6);
+    auto [a1, a2] = arr;
+    auto [b1, b2](arr);
+    auto [c1, c2]{arr}; // GH31813
 
-  constexpr int copy_initialization_for_elements() {
-    int arr[2]{4, 5};
-    auto [a1, b1] = arr;
-    auto [a2, b2]{arr}; // GH31813
     arr[0] = 0;
-    return a1 + b1 + a2 + b2 + arr[0];
+    return arr[0] + a1 + a2 + b1 + b2 + c1 + c2;
   }
-  static_assert(copy_initialization_for_elements() == 18);
+  static_assert(simple_array_elements() == 9);
+
+  void explicit_copy_ctor_array_elements() {
+    explicit_copy ec_arr[1];
 
-  void copy_initialization_for_elements_with_explicit_copy_ctor() {
-    explicit_copy ec_arr[2];
-    auto [a1, b1] = ec_arr; // expected-error {{no matching constructor for initialization of 'explicit_copy[2]'}}
-    auto [a2, b2]{ec_arr}; // expected-error {{no matching constructor for initialization of 'explicit_copy[2]'}}
+    auto [a] = ec_arr; // expected-error {{no matching constructor for initialization of 'explicit_copy[1]'}}
+    auto [b](ec_arr);
+    auto [c]{ec_arr};
 
     // Test prvalue
-    using T = explicit_copy[2];
-    auto [a3, b3] = T{};
-    auto [a4, b4]{T{}};
+    using T = explicit_copy[1];
+    auto [d] = T{};
   }
+
 } // namespace by_value_array_copy
diff --git a/clang/test/SemaCXX/cxx20-ctad-type-alias.cpp b/clang/test/SemaCXX/cxx20-ctad-type-alias.cpp
index 2d43e46b9e3d7..37dca2215af6b 100644
--- a/clang/test/SemaCXX/cxx20-ctad-type-alias.cpp
+++ b/clang/test/SemaCXX/cxx20-ctad-type-alias.cpp
@@ -234,11 +234,23 @@ int i = 0;
 AFoo s{i};
 static_assert(__is_same(decltype(s.t), int));
 
+template<class T>
+using BFoo = AFoo<T>;
+
+// template explicit deduction guide.
+template<class T>
+Foo(T) -> Foo<float>;
+static_assert(__is_same(decltype(AFoo(i).t), float));
+static_assert(__is_same(decltype(BFoo(i).t), float));
+
 // explicit deduction guide.
 Foo(int) -> Foo<X>;
-AFoo s2{i};
-// FIXME: the type should be X because of the above explicit deduction guide.
-static_assert(__is_same(decltype(s2.t), int));
+static_assert(__is_same(decltype(AFoo(i).t), X));
+static_assert(__is_same(decltype(BFoo(i).t), X));
+
+Foo(double) -> Foo<int>;
+static_assert(__is_same(decltype(AFoo(1.0).t), int));
+static_assert(__is_same(decltype(BFoo(1.0).t), int));
 } // namespace test16
 
 namespace test17 {
diff --git a/clang/test/SemaCXX/cxx2b-consteval-propagate.cpp b/clang/test/SemaCXX/cxx2b-consteval-propagate.cpp
index 3f3123eaee76b..dd5063cb29c5b 100644
--- a/clang/test/SemaCXX/cxx2b-consteval-propagate.cpp
+++ b/clang/test/SemaCXX/cxx2b-consteval-propagate.cpp
@@ -528,3 +528,51 @@ D d(0); // expected-note {{in implicit initialization for inherited constructor
 // expected-error@-1 {{call to immediate function 'GH112677::D::SimpleCtor' is not a constant expression}}
 
 }
+
+namespace GH123405 {
+
+consteval void fn() {}
+
+template <typename>
+constexpr auto tfn(int) {
+    auto p = &fn;  // expected-note {{'tfn<int>' is an immediate function because its body evaluates the address of a consteval function 'fn'}}
+    return p;
+}
+
+void g() {
+   int a; // expected-note {{declared here}}
+   tfn<int>(a); // expected-error {{call to immediate function 'GH123405::tfn<int>' is not a constant expression}}\
+                // expected-note {{read of non-const variable 'a' is not allowed in a constant expression}}
+}
+} // namespace GH123405
+
+namespace GH118000 {
+consteval int baz() { return 0;}
+struct S {
+    int mSize = baz();
+};
+
+consteval void bar() {
+    S s;
+}
+
+void foo() {
+    S s;
+}
+} // namespace GH118000
+
+namespace GH119046 {
+
+template <typename Cls> constexpr auto tfn(int) {
+  return (unsigned long long)(&Cls::sfn);
+  //expected-note@-1 {{'tfn<GH119046::S>' is an immediate function because its body evaluates the address of a consteval function 'sfn'}}
+};
+struct S { static consteval void sfn() {} };
+
+int f() {
+  int a = 0; // expected-note{{declared here}}
+  return tfn<S>(a);
+  //expected-error@-1 {{call to immediate function 'GH119046::tfn<GH119046::S>' is not a constant expression}}
+  //expected-note@-2 {{read of non-const variable 'a' is not allowed in a constant expression}}
+}
+}
diff --git a/clang/test/SemaCXX/cxx2c-binding-pack-nontemplate.cpp b/clang/test/SemaCXX/cxx2c-binding-pack-nontemplate.cpp
new file mode 100644
index 0000000000000..ea94757dc66b6
--- /dev/null
+++ b/clang/test/SemaCXX/cxx2c-binding-pack-nontemplate.cpp
@@ -0,0 +1,11 @@
+// RUN: %clang_cc1 -std=c++26 -fsyntax-only %s -verify=nontemplate
+// RUN: %clang_cc1 -std=c++2c -verify=cxx26,nontemplate -fsyntax-only -Wpre-c++26-compat %s
+// RUN: %clang_cc1 -std=c++23 -verify=cxx23,nontemplate -fsyntax-only -Wc++26-extensions %s
+
+void decompose_array() {
+  int arr[4] = {1, 2, 3, 6};
+  // cxx26-warning@+3 {{structured binding packs are incompatible with C++ standards before C++2c}}
+  // cxx23-warning@+2 {{structured binding packs are a C++2c extension}}
+  // nontemplate-error@+1 {{pack declaration outside of template}}
+  auto [x, ...rest, y] = arr;
+}
diff --git a/clang/test/SemaCXX/cxx2c-binding-pack.cpp b/clang/test/SemaCXX/cxx2c-binding-pack.cpp
new file mode 100644
index 0000000000000..5ca249f52b3d8
--- /dev/null
+++ b/clang/test/SemaCXX/cxx2c-binding-pack.cpp
@@ -0,0 +1,190 @@
+// RUN: %clang_cc1 -fsyntax-only -std=c++26 %s -verify
+
+template <typename T>
+struct type_ { };
+
+template <typename ...T>
+auto sum(T... t) { return (t + ...); }
+
+struct my_struct {
+  int a;
+  int b;
+  int c;
+  int d;
+};
+
+struct fake_tuple {
+  int arr[4] = {1, 2, 3, 6};
+
+  template <unsigned i>
+  int get() {
+    return arr[i];
+  }
+};
+
+namespace std {
+  template <typename T>
+  struct tuple_size;
+  template <unsigned i, typename T>
+  struct tuple_element;
+
+  template <>
+  struct tuple_size<fake_tuple> {
+    static constexpr unsigned value = 4;
+  };
+
+  template <unsigned i>
+  struct tuple_element<i, fake_tuple> {
+    using type = int;
+  };
+}
+
+
+template <typename T>
+void decompose_tuple() {
+  auto tup = T{{1, 2, 3, 6}};
+  auto&& [x, ...rest, y] = tup;
+
+  ((void)type_<int>(type_<decltype(rest)>{}), ...);
+
+  T arrtup[2] = {T{{1, 2, 3, 6}},
+                 T{{7, 9, 10, 11}}};
+  int sum = 0;
+  for (auto [...xs] : arrtup) {
+    sum += (xs + ...);
+  }
+}
+
+template <typename T>
+void decompose_struct() {
+  T obj{1, 2, 3, 6};
+  auto [x, ...rest, y] = obj;
+
+  auto [...empty] = type_<int>{};
+  static_assert(sizeof...(empty) == 0);
+}
+
+template <typename T>
+void decompose_array() {
+  int arr[4] = {1, 2, 3, 6};
+  auto [x, ...rest, y] = arr;
+
+  static_assert(sizeof...(rest) == 2);
+  int size = sizeof...(rest);
+  T arr2[sizeof...(rest)] = {rest...};
+  auto [...pack] = arr2;
+
+  // Array of size 1.
+  int arr1[1] = {1};
+  auto [a, ...b] = arr1;
+  static_assert(sizeof...(b) == 0);
+  auto [...c] = arr1;
+  static_assert(sizeof...(c) == 1);
+  auto [a1, ...b1, c1] = arr1; // expected-error{{decomposes into 1 element, but 3 names were provided}}
+}
+
+// Test case by Younan Zhang.
+template <unsigned... P>
+struct S {
+  template <unsigned... Q>
+  struct N {
+    void foo() {
+      int arr[] = {P..., Q...};
+      auto [x, y, ...rest] = arr;
+      [&]() {
+        static_assert(sizeof...(rest) + 2 == sizeof...(P) + sizeof...(Q));
+      }();
+    }
+  };
+};
+
+struct bit_fields {
+  int a : 4 {1};
+  int b : 4 {2};
+  int c : 4 {3};
+  int d : 4 {4};
+};
+
+template <typename T>
+void decompose_bit_field() {
+  auto [...x] = T{};
+  static_assert(sizeof...(x) == 4);
+  int a = x...[0];
+  int b = x...[1];
+  int c = x...[2];
+  int d = x...[3];
+}
+
+template <typename T>
+void lambda_capture() {
+  auto [...x] = T{};
+  [=] { (void)sum(x...); }();
+  [&] { (void)sum(x...); }();
+  [x...] { (void)sum(x...); }();
+  [&x...] { (void)sum(x...); }();
+}
+
+int main() {
+  decompose_array<int>();
+  decompose_tuple<fake_tuple>();
+  decompose_struct<my_struct>();
+  S<1, 2, 3, 4>::N<5, 6>().foo();
+  decompose_bit_field<bit_fields>();
+  lambda_capture<int[5]>();
+  lambda_capture<fake_tuple>();
+  lambda_capture<my_struct>();
+}
+
+// P1061R10 Stuff
+namespace {
+struct C { int x, y, z; };
+
+template <class T>
+void now_i_know_my() {
+  auto [a, b, c] = C(); // OK, SB0 is a, SB1 is b, and SB2 is c
+  auto [d, ...e] = C(); // OK, SB0 is d, the pack e (v1) contains two structured bindings: SB1 and SB2
+  static_assert(sizeof...(e) == 2);
+  auto [...f, g] = C(); // OK, the pack f (v0) contains two structured bindings: SB0 and SB1, and SB2 is g
+  static_assert(sizeof...(e) == 2);
+  auto [h, i, j, ...k] = C(); // OK, the pack k is empty
+  static_assert(sizeof...(e) == 0);
+  auto [l, m, n, o, ...p] = C(); // expected-error{{{decomposes into 3 elements, but 5 names were provided}}}
+}
+}  // namespace
+
+namespace {
+auto g() -> int(&)[4];
+
+template <unsigned long N>
+void h(int (&arr)[N]) {
+  auto [a, ...b, c] = arr;  // a names the first element of the array,
+                            // b is a pack referring to the second and
+                            // third elements, and c names the fourth element
+  static_assert(sizeof...(b) == 2);
+  auto& [...e] = arr;        // e is a pack referring to the four elements of the array
+  static_assert(sizeof...(e) == 4);
+}
+
+void call_h() {
+ h(g());
+}
+}  // namespace
+
+namespace {
+struct D { };
+
+int g(...) { return 1; }
+
+template <typename T>
+constexpr int f() {
+  D arr[1];
+  auto [...e] = arr;
+  return g(e...);
+}
+
+constexpr int g(D) { return 2; }
+
+void other_main() {
+  static_assert(f<int>() == 2);
+}
+}  // namespace
diff --git a/clang/test/SemaCXX/diagnose_if-warning-group.cpp b/clang/test/SemaCXX/diagnose_if-warning-group.cpp
new file mode 100644
index 0000000000000..a39c0c0c33c9e
--- /dev/null
+++ b/clang/test/SemaCXX/diagnose_if-warning-group.cpp
@@ -0,0 +1,63 @@
+// RUN: %clang_cc1 %s -verify=expected,wall -fno-builtin -Wno-pedantic -Werror=comment -Wno-error=abi -Wfatal-errors=assume -Wno-fatal-errors=assume -Wno-format
+// RUN: %clang_cc1 %s -verify=expected,wno-all,pedantic,format -fno-builtin -Wno-all -Werror=comment -Wno-error=abi -Werror=assume -Wformat
+
+#define diagnose_if(...) __attribute__((diagnose_if(__VA_ARGS__)))
+
+#ifndef EMTY_WARNING_GROUP
+void bougus_warning() diagnose_if(true, "oh no", "warning", "bogus warning") {} // expected-error {{unknown warning group 'bogus warning'}}
+
+void show_in_system_header() diagnose_if(true, "oh no", "warning", "assume", "Banane") {} // expected-error {{'diagnose_if' attribute takes no more than 4 arguments}}
+#endif // EMTY_WARNING_GROUP
+
+template <bool b>
+void diagnose_if_wcomma() diagnose_if(b, "oh no", "warning", "comma") {}
+
+template <bool b>
+void diagnose_if_wcomment() diagnose_if(b, "oh no", "warning", "comment") {}
+
+void empty_warning_group() diagnose_if(true, "oh no", "warning", "") {} // expected-error {{unknown warning group ''}}
+void empty_warning_group_error() diagnose_if(true, "oh no", "error", "") {} // expected-error {{unknown warning group ''}}
+
+void diagnose_if_wabi_default_error() diagnose_if(true, "ABI stuff", "error", "abi") {}
+void diagnose_assume() diagnose_if(true, "Assume diagnostic", "warning", "assume") {}
+
+void Wall() diagnose_if(true, "oh no", "warning", "all") {}
+void Wpedantic() diagnose_if(true, "oh no", "warning", "pedantic") {}
+void Wformat_extra_args() diagnose_if(true, "oh no", "warning", "format-extra-args") {}
+
+void call() {
+  diagnose_if_wcomma<true>(); // expected-warning {{oh no}}
+  diagnose_if_wcomma<false>();
+  diagnose_if_wcomment<true>(); // expected-error {{oh no}}
+  diagnose_if_wcomment<false>();
+
+#pragma clang diagnostic push
+#pragma clang diagnostic ignored "-Wcomma"
+  diagnose_if_wcomma<true>();
+  diagnose_if_wcomment<true>(); // expected-error {{oh no}}
+#pragma clang diagnostic pop
+
+#pragma clang diagnostic push
+#pragma clang diagnostic ignored "-Wcomment"
+  diagnose_if_wcomma<true>(); // expected-warning {{oh no}}
+  diagnose_if_wcomment<true>();
+#pragma clang diagnostic pop
+
+  diagnose_if_wcomma<true>(); // expected-warning {{oh no}}
+  diagnose_if_wcomment<true>(); // expected-error {{oh no}}
+
+  diagnose_if_wabi_default_error(); // expected-warning {{ABI stuff}}
+  diagnose_assume(); // expected-error {{Assume diagnostic}}
+
+  // Make sure that the -Wassume diagnostic isn't fatal
+  diagnose_if_wabi_default_error(); // expected-warning {{ABI stuff}}
+
+  Wall(); // wall-warning {{oh no}}
+  Wpedantic(); // pedantic-warning {{oh no}}
+  Wformat_extra_args(); // format-warning {{oh no}}
+
+#pragma clang diagnostic push
+#pragma clang diagnostic ignored "-Wformat"
+  Wformat_extra_args();
+#pragma clang diagnostic pop
+}
diff --git a/clang/test/SemaCXX/member-enum-declarations.cpp b/clang/test/SemaCXX/member-enum-declarations.cpp
new file mode 100644
index 0000000000000..e08f6e7a3fcd6
--- /dev/null
+++ b/clang/test/SemaCXX/member-enum-declarations.cpp
@@ -0,0 +1,112 @@
+// RUN: %clang_cc1 -std=c++11 -fsyntax-only %s -verify
+// RUN: %clang_cc1 -std=c++14 -fsyntax-only %s -verify
+// RUN: %clang_cc1 -std=c++20 -fsyntax-only %s -verify
+
+
+namespace ScopedEnumerations {
+
+template <typename T>
+struct S1 {
+  enum class E : T;
+};
+
+template <typename T>
+enum class S1<T>::E : T {
+  S1_X = 0x123
+};
+
+static_assert(static_cast<int>(S1<int>::E::S1_X) == 0x123, "");
+
+template <typename T>
+struct S2 {
+  static constexpr T f(int) { return 0; };
+  enum class E : T;
+  static constexpr T f(char) { return 1; };
+  enum class E : T { X = f(T{}) };
+};
+
+static_assert(static_cast<int>(S2<char>::E::X) == 1, "");
+
+template <typename T>
+struct S3 {
+  enum class E : T;
+  enum class E : T { X = 0x7FFFFF00 }; // expected-error {{cannot be narrowed to type 'char'}} expected-warning {{implicit conversion from 'int' to 'char'}}
+};
+template struct S3<char>; // expected-note {{in instantiation}}
+
+template <typename T>
+struct S4 {
+  enum class E : T;
+  enum class E : T { S4_X = 5 };
+};
+
+auto x4 = S4<int>::E::S4_X;
+
+template <typename T>
+T f1() {
+  enum class E : T { X_F1, Y_F1, Z_F1 };
+  return X_F1;  // expected-error {{use of undeclared identifier 'X_F1'}}
+}
+
+const int resf1 = f1<int>();
+
+}
+
+
+namespace UnscopedEnumerations {
+
+template <typename T>
+struct S1 {
+  enum E : T;
+};
+
+template <typename T>
+enum S1<T>::E : T {
+  S1_X = 0x123
+};
+
+static_assert(static_cast<int>(S1<int>::S1_X) == 0x123, "");
+
+template <typename T>
+struct S2 {
+  static constexpr T f(int) { return 0; };
+  enum E : T;
+  static constexpr T f(char) { return 1; };
+  enum E : T { S2_X = f(T{}) };
+};
+
+static_assert(static_cast<int>(S2<char>::E::S2_X) == 1, "");
+
+template <typename T>
+struct S3 {
+  enum E : T;
+  enum E : T { S3_X = 0x7FFFFF00 }; // expected-error {{cannot be narrowed to type 'char'}} expected-warning {{implicit conversion from 'int' to 'char'}}
+};
+template struct S3<char>; // expected-note {{in instantiation of template class}}
+
+template <typename T>
+struct S4 {
+  enum E : T;
+  enum E : T { S4_X = 5 };
+};
+
+auto x4 = S4<int>::S4_X;
+
+template <typename T>
+struct S5 {
+  enum E : T;
+  T S5_X = 5; // expected-note {{previous definition is here}}
+  enum E : T { S5_X = 5 }; // expected-error {{redefinition of 'S5_X'}}
+};
+
+
+template <typename T>
+T f1() {
+  enum E : T { X_F2, Y_F2, Z_F2 };
+  return X_F2;
+}
+
+const int resf1 = f1<int>();
+
+}
+
diff --git a/clang/test/SemaCXX/remove_pointer.mm b/clang/test/SemaCXX/remove_pointer.mm
deleted file mode 100644
index d1cf1fa9f4efc..0000000000000
--- a/clang/test/SemaCXX/remove_pointer.mm
+++ /dev/null
@@ -1,8 +0,0 @@
-// RUN: %clang_cc1 -fsyntax-only -verify %s
-
-// expected-no-diagnostics
-
-@class X;
-
-static_assert(__is_same(__remove_pointer(X *), X), "");
-static_assert(__is_same(__remove_pointer(id), id), "");
diff --git a/clang/test/SemaCXX/type-traits.cpp b/clang/test/SemaCXX/type-traits.cpp
index 1b9e2ba6ff162..b130024503101 100644
--- a/clang/test/SemaCXX/type-traits.cpp
+++ b/clang/test/SemaCXX/type-traits.cpp
@@ -779,36 +779,6 @@ void is_unbounded_array(int n) {
   (void)__is_unbounded_array(decltype(t32)); // expected-error{{variable length arrays are not supported in '__is_unbounded_array'}}
 }
 
-void is_referenceable() {
-  static_assert(__is_referenceable(int));
-  static_assert(__is_referenceable(const int));
-  static_assert(__is_referenceable(volatile int));
-  static_assert(__is_referenceable(const volatile int));
-  static_assert(__is_referenceable(int *));
-  static_assert(__is_referenceable(int &));
-  static_assert(__is_referenceable(int &&));
-  static_assert(__is_referenceable(int (*)()));
-  static_assert(__is_referenceable(int (&)()));
-  static_assert(__is_referenceable(int(&&)()));
-  static_assert(__is_referenceable(IntAr));
-  static_assert(__is_referenceable(IntArNB));
-  static_assert(__is_referenceable(decltype(nullptr)));
-  static_assert(__is_referenceable(Empty));
-  static_assert(__is_referenceable(Union));
-  static_assert(__is_referenceable(Derives));
-  static_assert(__is_referenceable(Enum));
-  static_assert(__is_referenceable(EnumClass));
-  static_assert(__is_referenceable(int Empty::*));
-  static_assert(__is_referenceable(int(Empty::*)()));
-  static_assert(__is_referenceable(AnIncompleteType));
-  static_assert(__is_referenceable(struct AnIncompleteType));
-
-  using function_type = void(int);
-  static_assert(__is_referenceable(function_type));
-
-  static_assert(!__is_referenceable(void));
-}
-
 template <typename T> void tmpl_func(T&) {}
 
 template <typename T> struct type_wrapper {
@@ -4739,8 +4709,6 @@ struct CheckAbominableFunction<M S::*> {
     static_assert(__is_same(remove_cvref_t<M>, M));
     static_assert(__is_same(remove_pointer_t<M>, M));
     static_assert(__is_same(remove_reference_t<M>, M));
-
-    static_assert(!__is_referenceable(M));
   }
 };
 
diff --git a/clang/test/SemaCXX/uninitialized.cpp b/clang/test/SemaCXX/uninitialized.cpp
index 52d9897cf9be6..4af2c998f082e 100644
--- a/clang/test/SemaCXX/uninitialized.cpp
+++ b/clang/test/SemaCXX/uninitialized.cpp
@@ -2,6 +2,8 @@
 // RUN: %clang_cc1 -fsyntax-only -Wall -Wc++20-compat -Wuninitialized -Wno-unused-value -Wno-unused-lambda-capture -Wno-uninitialized-const-reference -std=c++1z -verify %s -fexperimental-new-constant-interpreter
 // RUN: %clang_cc1 -fsyntax-only -Wall -Wc++20-compat -Wuninitialized -Wno-unused-value -Wno-unused-lambda-capture -Wno-uninitialized-const-reference -std=c++20 -verify %s
 
+void* operator new(__SIZE_TYPE__, void*);
+
 // definitions for std::move
 namespace std {
 inline namespace foo {
@@ -1540,6 +1542,80 @@ void aggregate() {
     };
   };
 
+  struct CopyAndMove {
+    CopyAndMove() = default;
+    CopyAndMove(const CopyAndMove &) {}
+    CopyAndMove(CopyAndMove &&) {}
+  };
+  struct Embed {
+    int embed1;  // #FIELD_EMBED1
+    int embed2 [[clang::require_explicit_initialization]];  // #FIELD_EMBED2
+    CopyAndMove force_separate_move_ctor;
+  };
+  struct EmbedDerived : Embed {};
+  struct F {
+    Embed f1;
+    // expected-warning@+1 {{field in 'Embed' requires explicit initialization but is not explicitly initialized}} expected-note@#FIELD_EMBED2 {{'embed2' declared here}}
+    explicit F(const char(&)[1]) : f1() {
+      // expected-warning@+1 {{field in 'Embed' requires explicit initialization but is not explicitly initialized}} expected-note@#FIELD_EMBED2 {{'embed2' declared here}}
+      ::new(static_cast<void*>(&f1)) decltype(f1);
+      // expected-warning@+1 {{field in 'Embed' requires explicit initialization but is not explicitly initialized}} expected-note@#FIELD_EMBED2 {{'embed2' declared here}}
+      ::new(static_cast<void*>(&f1)) decltype(f1)();
+#if __cplusplus >= 202002L
+      // expected-warning@+1 {{field 'embed2' requires explicit initialization but is not explicitly initialized}} expected-note@#FIELD_EMBED2 {{'embed2' declared here}}
+      ::new(static_cast<void*>(&f1)) decltype(f1)(1);
+#endif
+      // expected-warning@+1 {{field 'embed2' requires explicit initialization but is not explicitly initialized}} expected-note@#FIELD_EMBED2 {{'embed2' declared here}}
+      ::new(static_cast<void*>(&f1)) decltype(f1){1};
+    }
+#if __cplusplus >= 202002L
+    // expected-warning@+1 {{field 'embed2' requires explicit initialization but is not explicitly initialized}} expected-note@#FIELD_EMBED2 {{'embed2' declared here}}
+    explicit F(const char(&)[2]) : f1(1) {}
+#else
+    explicit F(const char(&)[2]) : f1{1, 2} { }
+#endif
+    // expected-warning@+1 {{field 'embed2' requires explicit initialization but is not explicitly initialized}} expected-note@#FIELD_EMBED2 {{'embed2' declared here}}
+    explicit F(const char(&)[3]) : f1{} {}
+    // expected-warning@+1 {{field 'embed2' requires explicit initialization but is not explicitly initialized}} expected-note@#FIELD_EMBED2 {{'embed2' declared here}}
+    explicit F(const char(&)[4]) : f1{1} {}
+    // expected-warning@+1 {{field 'embed2' requires explicit initialization but is not explicitly initialized}} expected-note@#FIELD_EMBED2 {{'embed2' declared here}}
+    explicit F(const char(&)[5]) : f1{.embed1 = 1} {}
+  };
+  F ctors[] = {
+      F(""),
+      F("_"),
+      F("__"),
+      F("___"),
+      F("____")
+  };
+
+  struct MoveOrCopy {
+    Embed e;
+    EmbedDerived ed;
+    F f;
+    // no-error
+    MoveOrCopy(const MoveOrCopy &c) : e(c.e), ed(c.ed), f(c.f) {}
+    // no-error
+    MoveOrCopy(MoveOrCopy &&c)
+        : e(std::move(c.e)), ed(std::move(c.ed)), f(std::move(c.f)) {}
+  };
+  F copy1(ctors[0]); // no-error
+  (void)copy1;
+  F move1(std::move(ctors[0])); // no-error
+  (void)move1;
+  F copy2{ctors[0]}; // no-error
+  (void)copy2;
+  F move2{std::move(ctors[0])}; // no-error
+  (void)move2;
+  F copy3 = ctors[0]; // no-error
+  (void)copy3;
+  F move3 = std::move(ctors[0]); // no-error
+  (void)move3;
+  F copy4 = {ctors[0]}; // no-error
+  (void)copy4;
+  F move4 = {std::move(ctors[0])}; // no-error
+  (void)move4;
+
   S::foo(S{1, 2, 3, 4});
   S::foo(S{.s1 = 100, .s4 = 100});
   S::foo(S{.s1 = 100}); // expected-warning {{field 's4' requires explicit initialization but is not explicitly initialized}} expected-note@#FIELD_S4 {{'s4' declared here}}
diff --git a/clang/test/SemaCXX/unique_object_duplication.cpp b/clang/test/SemaCXX/unique_object_duplication.cpp
new file mode 100644
index 0000000000000..4b41bfbfdc2f7
--- /dev/null
+++ b/clang/test/SemaCXX/unique_object_duplication.cpp
@@ -0,0 +1,16 @@
+// RUN: %clang_cc1 -fsyntax-only -verify=hidden -Wunique-object-duplication -fvisibility=hidden -Wno-unused-value %s
+// RUN: %clang_cc1 -fsyntax-only -verify -Wunique-object-duplication -Wno-unused-value %s
+// The check is currently disabled on windows in MSVC-like environments. The test should fail because we're not getting the expected warnings.
+// XFAIL: target={{.*}}-windows-msvc, {{.*}}-ps{{(4|5)(-.+)?}}
+
+#include "unique_object_duplication.h"
+
+// Everything in these namespaces here is defined in the cpp file,
+// so won't get duplicated
+
+namespace GlobalTest {
+  float Test::allowedStaticMember1 = 2.3;
+}
+
+bool disallowed4 = true;
+constexpr inline bool disallowed5 = true;
\ No newline at end of file
diff --git a/clang/test/SemaCXX/unique_object_duplication.h b/clang/test/SemaCXX/unique_object_duplication.h
new file mode 100644
index 0000000000000..5b2002c31be7c
--- /dev/null
+++ b/clang/test/SemaCXX/unique_object_duplication.h
@@ -0,0 +1,157 @@
+/**
+ * This file contains tests for the -Wunique_object_duplication warning.
+ * See the warning's documentation for more information.
+ */
+
+#define HIDDEN __attribute__((visibility("hidden")))
+#define DEFAULT __attribute__((visibility("default")))
+
+// Helper functions
+constexpr int init_constexpr(int x) { return x; };
+extern double init_dynamic(int);
+
+/******************************************************************************
+ * Case one: Static local variables in an externally-visible function
+ ******************************************************************************/
+namespace StaticLocalTest {
+
+inline void has_static_locals_external() {
+  // Mutable
+  static int disallowedStatic1 = 0; // hidden-warning {{'disallowedStatic1' may be duplicated when built into a shared library: it is mutable, has hidden visibility, and external linkage}}
+  // Initialization might run more than once
+  static const double disallowedStatic2 = disallowedStatic1++; // hidden-warning {{initializeation of 'disallowedStatic2' may run twice when built into a shared library: it has hidden visibility and external linkage}}
+  
+  // OK, because immutable and compile-time-initialized
+  static constexpr int allowedStatic1 = 0;
+  static const float allowedStatic2 = 1;
+  static constexpr int allowedStatic3 = init_constexpr(2);
+  static const int allowedStatic4 = init_constexpr(3);
+}
+
+// Don't warn for non-inline functions, since they can't (legally) appear
+// in more than one TU in the first place.
+void has_static_locals_non_inline() {
+  // Mutable
+  static int allowedStatic1 = 0;
+  // Initialization might run more than once
+  static const double allowedStatic2 = allowedStatic1++;
+}
+
+// Everything in this function is OK because the function is TU-local
+static void has_static_locals_internal() {
+  static int allowedStatic1 = 0;
+  static double allowedStatic2 = init_dynamic(2);
+  static char allowedStatic3 = []() { return allowedStatic1++; }();
+  static constexpr int allowedStatic4 = 0;
+}
+
+namespace {
+
+// Everything in this function is OK because the function is also TU-local
+void has_static_locals_anon() {
+  static int allowedStatic1 = 0;
+  static double allowedStatic2 = init_dynamic(2);
+  static char allowedStatic3 = []() { return allowedStatic1++; }();
+  static constexpr int allowedStatic4 = init_constexpr(3);
+} 
+
+} // Anonymous namespace
+
+HIDDEN inline void static_local_always_hidden() {
+    static int disallowedStatic1 = 3; // hidden-warning {{'disallowedStatic1' may be duplicated when built into a shared library: it is mutable, has hidden visibility, and external linkage}}
+                                      // expected-warning@-1 {{'disallowedStatic1' may be duplicated when built into a shared library: it is mutable, has hidden visibility, and external linkage}}
+    {
+      static int disallowedStatic2 = 3; // hidden-warning {{'disallowedStatic2' may be duplicated when built into a shared library: it is mutable, has hidden visibility, and external linkage}}
+                                        // expected-warning@-1 {{'disallowedStatic2' may be duplicated when built into a shared library: it is mutable, has hidden visibility, and external linkage}}
+    }
+
+    auto lmb = []() {
+      static int disallowedStatic3 = 3; // hidden-warning {{'disallowedStatic3' may be duplicated when built into a shared library: it is mutable, has hidden visibility, and external linkage}}
+                                        // expected-warning@-1 {{'disallowedStatic3' may be duplicated when built into a shared library: it is mutable, has hidden visibility, and external linkage}}
+    };
+}
+
+DEFAULT void static_local_never_hidden() {
+    static int allowedStatic1 = 3; 
+
+    {
+      static int allowedStatic2 = 3; 
+    }
+
+    auto lmb = []() {
+      static int allowedStatic3 = 3;
+    };
+}
+
+// Don't warn on this because it's not in a function
+const int setByLambda = ([]() { static int x = 3; return x++; })();
+
+inline void has_extern_local() {
+  extern int allowedAddressExtern; // Not a definition
+}
+
+inline void has_regular_local() {
+  int allowedAddressLocal = 0;
+}
+
+inline void has_thread_local() {
+  // thread_local variables are static by default
+  thread_local int disallowedThreadLocal = 0; // hidden-warning {{'disallowedThreadLocal' may be duplicated when built into a shared library: it is mutable, has hidden visibility, and external linkage}}
+}
+
+} // namespace StaticLocalTest
+
+/******************************************************************************
+ * Case two: Globals with external linkage
+ ******************************************************************************/
+namespace GlobalTest {
+  // Mutable
+  inline float disallowedGlobal1 = 3.14; // hidden-warning {{'disallowedGlobal1' may be duplicated when built into a shared library: it is mutable, has hidden visibility, and external linkage}}
+  
+  // Initialization might run more than once
+  inline const double disallowedGlobal5 = disallowedGlobal1++; // hidden-warning {{initializeation of 'disallowedGlobal5' may run twice when built into a shared library: it has hidden visibility and external linkage}}
+
+  // OK because internal linkage, so duplication is intended
+  static float allowedGlobal1 = 3.14;
+  const double allowedGlobal2 = init_dynamic(2);
+  static const char allowedGlobal3 = []() { return disallowedGlobal1++; }();
+  static inline double allowedGlobal4 = init_dynamic(2);
+
+  // OK, because immutable and compile-time-initialized
+  constexpr int allowedGlobal5 = 0;
+  const float allowedGlobal6 = 1;
+  constexpr int allowedGlobal7 = init_constexpr(2);
+  const int allowedGlobal8 = init_constexpr(3);
+
+  // We don't warn on this because non-inline variables can't (legally) appear
+  // in more than one TU.
+  float allowedGlobal9 = 3.14;
+  
+  // Pointers need to be double-const-qualified
+  inline float& nonConstReference = disallowedGlobal1; // hidden-warning {{'nonConstReference' may be duplicated when built into a shared library: it is mutable, has hidden visibility, and external linkage}}
+  const inline int& constReference = allowedGlobal5;
+
+  inline int* nonConstPointerToNonConst = nullptr; // hidden-warning {{'nonConstPointerToNonConst' may be duplicated when built into a shared library: it is mutable, has hidden visibility, and external linkage}}
+  inline int const* nonConstPointerToConst = nullptr; // hidden-warning {{'nonConstPointerToConst' may be duplicated when built into a shared library: it is mutable, has hidden visibility, and external linkage}}
+  inline int* const constPointerToNonConst = nullptr; // hidden-warning {{'constPointerToNonConst' may be duplicated when built into a shared library: it is mutable, has hidden visibility, and external linkage}}
+  inline int const* const constPointerToConst = nullptr;
+  // Don't warn on new because it tends to generate false positives
+  inline int const* const constPointerToConstNew = new int(7);
+
+  inline int const * const * const * const nestedConstPointer = nullptr;
+  inline int const * const ** const * const nestedNonConstPointer = nullptr; // hidden-warning {{'nestedNonConstPointer' may be duplicated when built into a shared library: it is mutable, has hidden visibility, and external linkage}}
+
+  struct Test {
+    static inline float disallowedStaticMember1; // hidden-warning {{'disallowedStaticMember1' may be duplicated when built into a shared library: it is mutable, has hidden visibility, and external linkage}}       
+    // Defined below, in the header file
+    static float disallowedStaticMember2;                                       
+    // Defined in the cpp file, so won't get duplicated
+    static float allowedStaticMember1;
+
+    // Tests here are sparse because the AddrTest case below will define plenty
+    // more, which aren't problematic to define (because they're immutable), but
+    // may still cause problems if their address is taken.
+  };
+
+  inline float Test::disallowedStaticMember2 = 2.3; // hidden-warning {{'disallowedStaticMember2' may be duplicated when built into a shared library: it is mutable, has hidden visibility, and external linkage}}
+} // namespace GlobalTest
\ No newline at end of file
diff --git a/clang/test/SemaCXX/warn-base-type-qualifiers.cpp b/clang/test/SemaCXX/warn-base-type-qualifiers.cpp
index b9fd577c574b9..7c775a552dd88 100644
--- a/clang/test/SemaCXX/warn-base-type-qualifiers.cpp
+++ b/clang/test/SemaCXX/warn-base-type-qualifiers.cpp
@@ -8,7 +8,7 @@ template <typename T> using add_const_t = typename add_const<T>::type;
 class A { };
 
 typedef const A A_Const;
-class B : public A_Const { }; // expected-warning {{'const' qualifier on base class type 'A_Const' (aka 'const A') have no effect}} \
+class B : public A_Const { }; // expected-warning {{'const' qualifier on base class type 'A_Const' (aka 'const A') has no effect}} \
                               // expected-note {{base class 'A_Const' (aka 'const A') specified here}}
 
 typedef const volatile A A_Const_Volatile;
@@ -19,7 +19,7 @@ struct D {
   D(int);
 };
 
-template <typename T> struct E : T { // expected-warning {{'const' qualifier on base class type 'const D' have no effect}} \
+template <typename T> struct E : T { // expected-warning {{'const' qualifier on base class type 'const D' has no effect}} \
                                      // expected-note {{base class 'const D' specified here}}
   using T::T;
   E(int &) : E(0) {}
@@ -27,7 +27,7 @@ template <typename T> struct E : T { // expected-warning {{'const' qualifier on
 E<const D> e(1); // expected-note {{in instantiation of template class 'E<const D>' requested here}}
 
 template <typename T>
-struct G : add_const<T>::type { // expected-warning {{'const' qualifier on base class type 'add_const<D>::type' (aka 'const D') have no effect}} \
+struct G : add_const<T>::type { // expected-warning {{'const' qualifier on base class type 'add_const<D>::type' (aka 'const D') has no effect}} \
                                 // expected-note {{base class 'add_const<D>::type' (aka 'const D') specified here}}
   using T::T;
   G(int &) : G(0) {}
diff --git a/clang/test/SemaCXX/warn-unsafe-buffer-usage-array.cpp b/clang/test/SemaCXX/warn-unsafe-buffer-usage-array.cpp
index e80b54b7c6967..9bfc31bd07b0e 100644
--- a/clang/test/SemaCXX/warn-unsafe-buffer-usage-array.cpp
+++ b/clang/test/SemaCXX/warn-unsafe-buffer-usage-array.cpp
@@ -124,3 +124,38 @@ void array_indexed_const_expr(unsigned idx) {
   k = arr[get_const(5)]; // expected-note {{used in buffer access here}}
   k = arr[get_const(4)];
 }
+
+template<unsigned length>
+consteval bool isNullTerminated(const char (&literal)[length])
+{
+  return literal[length - 1] == '\0';
+}
+
+template <typename T, unsigned M, unsigned N>
+T access2DArray(const T (&arr)[M][N]) {
+  return arr[M-1][N-1];
+}
+
+template<unsigned idx>
+constexpr int access_elements() {
+  int arr[idx + 20];
+  return arr[idx + 1];
+}
+
+// Test array accesses where const sized arrays are accessed safely with indices
+// that evaluate to a const values and depend on template arguments.
+void test_template_methods()
+{
+  constexpr char arr[] = "Good Morning!"; // = {'a', 'b', 'c', 'd', 'e'};
+  isNullTerminated(arr);
+  isNullTerminated("");
+  auto _ = isNullTerminated("hello world\n");
+  access_elements<5>();
+
+  int arr1[3][4] = {
+        {1, 2, 3, 4},
+        {5, 6, 7, 8},
+        {9, 10, 11, 12}
+    };
+  access2DArray(arr1);
+}
diff --git a/clang/test/SemaCXX/warn-unused-private-field.cpp b/clang/test/SemaCXX/warn-unused-private-field.cpp
index 0cc6f687f1b35..0bcca6b82227f 100644
--- a/clang/test/SemaCXX/warn-unused-private-field.cpp
+++ b/clang/test/SemaCXX/warn-unused-private-field.cpp
@@ -1,6 +1,6 @@
-// RUN: %clang_cc1 -fsyntax-only -Wunused-private-field -Wused-but-marked-unused -Wno-uninitialized -verify -std=c++11 %s
-// RUN: %clang_cc1 -fsyntax-only -Wunused-private-field -Wused-but-marked-unused -Wno-uninitialized -verify -std=c++17 %s
-// RUN: %clang_cc1 -fsyntax-only -Wunused-private-field -Wused-but-marked-unused -Wno-uninitialized -verify -std=c++20 %s
+// RUN: %clang_cc1 -fsyntax-only -Wunused -Wused-but-marked-unused -Wno-uninitialized -verify -std=c++11 %s
+// RUN: %clang_cc1 -fsyntax-only -Wunused -Wused-but-marked-unused -Wno-uninitialized -verify -std=c++17 %s
+// RUN: %clang_cc1 -fsyntax-only -Wunused -Wused-but-marked-unused -Wno-uninitialized -verify -std=c++20 %s
 
 #if __cplusplus >= 202002L
 
@@ -108,7 +108,7 @@ class ClassWithTemplateFriend {
 template <typename T> class TemplateFriend {
 public:
   TemplateFriend(ClassWithTemplateFriend my_friend) {
-    int var = my_friend.used_by_friend_;
+    int var = my_friend.used_by_friend_; // expected-warning {{unused variable 'var'}}
   }
 };
 
@@ -181,10 +181,10 @@ class EverythingUsed {
  public:
   EverythingUsed() : as_array_index_(0), var_(by_initializer_) {
     var_ = sizeof(sizeof_);
-    int *use = &by_reference_;
+    int *use = &by_reference_; // expected-warning {{unused variable 'use'}}
     int test[2];
     test[as_array_index_] = 42;
-    int EverythingUsed::*ptr = &EverythingUsed::by_pointer_to_member_;
+    int EverythingUsed::*ptr = &EverythingUsed::by_pointer_to_member_; // expected-warning {{unused variable 'ptr'}}
   }
 
   template<class T>
@@ -329,3 +329,28 @@ class C {
   MaybeUnusedTypedef t; // no-warning
 };
 }
+
+namespace GH62472 {
+class [[gnu::warn_unused]] S {
+public:
+  S();
+};
+
+struct [[maybe_unused]] T {};
+
+void f() {
+  int i = 0; // expected-warning {{unused variable 'i'}}
+  S s;       // expected-warning {{unused variable 's'}}
+  T t;       // ok
+}
+
+class C {
+private:
+  const int i = 0; // expected-warning {{private field 'i' is not used}}
+  int j = 0;       // expected-warning {{private field 'j' is not used}}
+  const S s1;      // expected-warning {{private field 's1' is not used}}
+  const T t1;      // ok
+  S s2;            // expected-warning {{private field 's2' is not used}}
+  T t2;            // ok
+};
+}
diff --git a/clang/test/SemaHLSL/BuiltIns/AddUint64-errors.hlsl b/clang/test/SemaHLSL/BuiltIns/AddUint64-errors.hlsl
new file mode 100644
index 0000000000000..ec9d026bb6fe7
--- /dev/null
+++ b/clang/test/SemaHLSL/BuiltIns/AddUint64-errors.hlsl
@@ -0,0 +1,41 @@
+// RUN: %clang_cc1 -finclude-default-header -triple dxil-pc-shadermodel6.6-library %s -fnative-half-type -emit-llvm-only -disable-llvm-passes -verify
+
+uint2 test_too_few_arg() {
+  return __builtin_hlsl_adduint64();
+  // expected-error@-1 {{too few arguments to function call, expected 2, have 0}}
+}
+
+uint4 test_too_many_arg(uint4 a) {
+  return __builtin_hlsl_adduint64(a, a, a);
+  // expected-error@-1 {{too many arguments to function call, expected 2, have 3}}
+}
+
+uint2 test_mismatched_arg_types(uint2 a, uint4 b) {
+  return __builtin_hlsl_adduint64(a, b);
+  // expected-error@-1 {{all arguments to '__builtin_hlsl_adduint64' must have the same type}}
+}
+
+uint2 test_bad_num_arg_elements(uint3 a, uint3 b) {
+  return __builtin_hlsl_adduint64(a, b);
+  // expected-error@-1 {{invalid element count of 3 in vector operand (expected an even element count in the range of 2 and 4)}}
+}
+
+uint2 test_scalar_arg_type(uint a) {
+  return __builtin_hlsl_adduint64(a, a);
+  // expected-error@-1 {{all arguments to AddUint64 must be vectors}}
+}
+
+uint2 test_signed_integer_args(int2 a, int2 b) {
+  return __builtin_hlsl_adduint64(a, b);
+// expected-error@-1 {{passing 'int2' (aka 'vector<int, 2>') to parameter of incompatible type '__attribute__((__vector_size__(2 * sizeof(unsigned int)))) unsigned int' (vector of 2 'unsigned int' values)}}
+}
+
+struct S {
+  uint2 a;
+};
+
+uint2 test_incorrect_arg_type(S a) {
+  return __builtin_hlsl_adduint64(a, a);
+  // expected-error@-1 {{passing 'S' to parameter of incompatible type 'unsigned int'}}
+}
+
diff --git a/clang/test/SemaHLSL/BuiltIns/WaveActiveMax-errors.hlsl b/clang/test/SemaHLSL/BuiltIns/WaveActiveMax-errors.hlsl
new file mode 100644
index 0000000000000..e077a40ba5165
--- /dev/null
+++ b/clang/test/SemaHLSL/BuiltIns/WaveActiveMax-errors.hlsl
@@ -0,0 +1,29 @@
+// RUN: %clang_cc1 -finclude-default-header -triple dxil-pc-shadermodel6.6-library %s -emit-llvm-only -disable-llvm-passes -verify
+
+int test_too_few_arg() {
+  return __builtin_hlsl_wave_active_max();
+  // expected-error@-1 {{too few arguments to function call, expected 1, have 0}}
+}
+
+float2 test_too_many_arg(float2 p0) {
+  return __builtin_hlsl_wave_active_max(p0, p0);
+  // expected-error@-1 {{too many arguments to function call, expected 1, have 2}}
+}
+
+bool test_expr_bool_type_check(bool p0) {
+  return __builtin_hlsl_wave_active_max(p0);
+  // expected-error@-1 {{invalid operand of type 'bool'}}
+}
+
+bool2 test_expr_bool_vec_type_check(bool2 p0) {
+  return __builtin_hlsl_wave_active_max(p0);
+  // expected-error@-1 {{invalid operand of type 'bool2' (aka 'vector<bool, 2>')}}
+}
+
+struct S { float f; };
+
+S test_expr_struct_type_check(S p0) {
+  return __builtin_hlsl_wave_active_max(p0);
+  // expected-error@-1 {{invalid operand of type 'S' where a scalar or vector is required}}
+}
+
diff --git a/clang/test/SemaHLSL/BuiltIns/mad-errors.hlsl b/clang/test/SemaHLSL/BuiltIns/mad-errors.hlsl
index ee4605528f410..b96761fb76cc0 100644
--- a/clang/test/SemaHLSL/BuiltIns/mad-errors.hlsl
+++ b/clang/test/SemaHLSL/BuiltIns/mad-errors.hlsl
@@ -84,3 +84,10 @@ float builtin_mad_int_to_float_promotion(float p0, int p1) {
   return __builtin_hlsl_mad(p0, p0, p1);
   // expected-error@-1 {{3rd argument must be a floating point type (was 'int')}}
 }
+
+int builtin_mad_mixed_enums() {
+  enum e { one, two };
+  enum f { three };
+  return __builtin_hlsl_mad(one, two, three);
+  // expected-error@-1 {{invalid arithmetic between different enumeration types ('e' and 'f')}}
+}
diff --git a/clang/test/SemaHLSL/BuiltIns/vector-constructors-erros.hlsl b/clang/test/SemaHLSL/BuiltIns/vector-constructors-erros.hlsl
index 7f6bdc7e67836..b004acdc7c502 100644
--- a/clang/test/SemaHLSL/BuiltIns/vector-constructors-erros.hlsl
+++ b/clang/test/SemaHLSL/BuiltIns/vector-constructors-erros.hlsl
@@ -17,6 +17,4 @@ void entry() {
   // These _should_ work in HLSL but aren't yet supported.
   S s;
   float2 GettingStrange = float2(s, s); // expected-error{{no viable conversion from 'S' to 'float'}} expected-error{{no viable conversion from 'S' to 'float'}}
-  S2 s2;
-  float2 EvenStranger = float2(s2); // expected-error{{cannot convert 'S2' to 'float2' (vector of 2 'float' values) without a conversion operator}}
 }
diff --git a/clang/test/SemaHLSL/Language/ElementwiseCast-errors.hlsl b/clang/test/SemaHLSL/Language/ElementwiseCast-errors.hlsl
new file mode 100644
index 0000000000000..c900c83a063a0
--- /dev/null
+++ b/clang/test/SemaHLSL/Language/ElementwiseCast-errors.hlsl
@@ -0,0 +1,29 @@
+// RUN: %clang_cc1 -finclude-default-header -triple dxil-pc-shadermodel6.6-library %s -verify
+
+export void cantCast() {
+  int A[3] = {1,2,3};
+  int B[4] = {1,2,3,4};
+  B = (int[4])A;
+  // expected-error@-1 {{C-style cast from 'int *' to 'int[4]' is not allowed}}
+}
+
+struct S {
+// expected-note@-1 {{candidate constructor (the implicit copy constructor) not viable: no known conversion from 'int2' (aka 'vector<int, 2>') to 'const S' for 1st argument}}
+// expected-note@-2 {{candidate constructor (the implicit move constructor) not viable: no known conversion from 'int2' (aka 'vector<int, 2>') to 'S' for 1st argument}}
+// expected-note@-3 {{candidate constructor (the implicit default constructor) not viable: requires 0 arguments, but 1 was provided}}
+  int A : 8;
+  int B;
+};
+
+// casting types which contain bitfields is not yet supported.
+export void cantCast2() {
+  S s = {1,2};
+  int2 C = (int2)s;
+  // expected-error@-1 {{cannot convert 'S' to 'int2' (aka 'vector<int, 2>') without a conversion operator}}
+}
+
+export void cantCast3() {
+  int2 C = {1,2};
+  S s = (S)C;
+  // expected-error@-1 {{no matching conversion for C-style cast from 'int2' (aka 'vector<int, 2>') to 'S'}}
+}
diff --git a/clang/test/SemaHLSL/Language/ElementwiseCasts.hlsl b/clang/test/SemaHLSL/Language/ElementwiseCasts.hlsl
new file mode 100644
index 0000000000000..563d3f02a1485
--- /dev/null
+++ b/clang/test/SemaHLSL/Language/ElementwiseCasts.hlsl
@@ -0,0 +1,23 @@
+// RUN: %clang_cc1 -triple dxil-pc-shadermodel6.3-library -finclude-default-header -fnative-half-type %s -ast-dump | FileCheck %s
+
+// truncation
+// CHECK-LABEL: call1
+// CHECK: CStyleCastExpr {{.*}} 'int[1]' <HLSLElementwiseCast>
+// CHECK-NEXT: ImplicitCastExpr {{.*}} 'int[2]' <HLSLArrayRValue> part_of_explicit_cast
+// CHECK-NEXT: DeclRefExpr {{.*}} 'int[2]' lvalue Var {{.*}} 'A' 'int[2]'
+export void call1() {
+  int A[2] = {0,1};
+  int B[1] = {4};
+  B = (int[1])A;
+}
+
+// flat cast of equal size
+// CHECK-LABEL: call2
+// CHECK: CStyleCastExpr {{.*}} 'float[1]' <HLSLElementwiseCast>
+// CHECK-NEXT: ImplicitCastExpr {{.*}} 'int[1]' <HLSLArrayRValue> part_of_explicit_cast
+// CHECK-NEXT: DeclRefExpr {{.*}} 'int[1]' lvalue Var {{.*}} 'A' 'int[1]'
+export void call2() {
+  int A[1] = {0};
+  float B[1] = {1.0};
+  B = (float[1])A;
+}
diff --git a/clang/test/SemaObjCXX/noescape.mm b/clang/test/SemaObjCXX/noescape.mm
index 999a91b87300b..4484e3b955ac0 100644
--- a/clang/test/SemaObjCXX/noescape.mm
+++ b/clang/test/SemaObjCXX/noescape.mm
@@ -77,12 +77,11 @@ -(void) m0:(int*) p {}
 void (*fnptr0)(int *);
 void (*fnptr1)(__attribute__((noescape)) int *);
 template<void (*fn)(int*)> struct S4 {};
-template<void (*fn)(int* __attribute__((noescape)))> struct S5 {};
-
 #if __cplusplus < 201406
-  // expected-note@-4 {{template parameter is declared here}}
-  // expected-note@-4 {{template parameter is declared here}}
+// expected-note@-2 {{template parameter is declared here}}
 #endif
+template<void (*fn)(int* __attribute__((noescape)))> struct S5 {};
+// expected-note@-1 {{template parameter is declared here}}
 
 void test0() {
   fnptr0 = &func0;
diff --git a/clang/test/SemaObjCXX/type-traits.mm b/clang/test/SemaObjCXX/type-traits.mm
new file mode 100644
index 0000000000000..81b9573b52192
--- /dev/null
+++ b/clang/test/SemaObjCXX/type-traits.mm
@@ -0,0 +1,17 @@
+// RUN: %clang_cc1 -triple x86_64-apple-darwin10 -fsyntax-only -verify -std=c++17  %s
+
+// expected-no-diagnostics
+
+@interface I;
+@end
+
+@class C;
+
+static_assert(__is_same(__add_pointer(id), id*));
+static_assert(__is_same(__add_pointer(I), I*));
+
+static_assert(__is_same(__remove_pointer(C*), C));
+static_assert(!__is_same(__remove_pointer(id), id));
+static_assert(__is_same(__remove_pointer(id*), id));
+static_assert(__is_same(__remove_pointer(__add_pointer(id)), id));
+static_assert(__is_same(__add_pointer(__remove_pointer(id)), id));
diff --git a/clang/test/SemaOpenACC/atomic-construct-ast.cpp b/clang/test/SemaOpenACC/atomic-construct-ast.cpp
new file mode 100644
index 0000000000000..6579b87941e5a
--- /dev/null
+++ b/clang/test/SemaOpenACC/atomic-construct-ast.cpp
@@ -0,0 +1,170 @@
+// RUN: %clang_cc1 %s -fopenacc -ast-dump | FileCheck %s
+
+// Test this with PCH.
+// RUN: %clang_cc1 %s -fopenacc -emit-pch -o %t %s
+// RUN: %clang_cc1 %s -fopenacc -include-pch %t -ast-dump-all | FileCheck %s
+
+#ifndef PCH_HELPER
+#define PCH_HELPER
+
+void foo(int v, int x) {
+  // CHECK: FunctionDecl{{.*}} foo 'void (int, int)'
+  // CHECK-NEXT: ParmVarDecl
+  // CHECK-NEXT: ParmVarDecl
+  // CHECK-NEXT: CompoundStmt
+
+// CHECK-NEXT: OpenACCAtomicConstruct{{.*}} atomic read
+// CHECK-NEXT: BinaryOperator{{.*}} 'int' lvalue '='
+// CHECK-NEXT: DeclRefExpr{{.*}}'v' 'int'
+// CHECK-NEXT: ImplicitCastExpr{{.*}}'int' <LValueToRValue>
+// CHECK-NEXT: DeclRefExpr{{.*}}'x' 'int'
+#pragma acc atomic read
+  v = x;
+
+// CHECK-NEXT: OpenACCAtomicConstruct{{.*}} atomic write
+// CHECK-NEXT: BinaryOperator{{.*}} 'int' lvalue '='
+// CHECK-NEXT: DeclRefExpr{{.*}}'v' 'int'
+// CHECK-NEXT: BinaryOperator{{.*}}'int' '+'
+// CHECK-NEXT: ImplicitCastExpr{{.*}}'int' <LValueToRValue>
+// CHECK-NEXT: DeclRefExpr{{.*}}'x' 'int'
+// CHECK-NEXT: IntegerLiteral{{.*}} 'int' 1
+#pragma acc atomic write
+  v = x + 1;
+
+// CHECK-NEXT: OpenACCAtomicConstruct{{.*}} atomic update 
+// CHECK-NEXT: UnaryOperator{{.*}} 'int' postfix '++'
+// CHECK-NEXT: DeclRefExpr{{.*}}'x' 'int'
+#pragma acc atomic update
+  x++;
+// CHECK-NEXT: OpenACCAtomicConstruct{{.*}} atomic <none>
+// CHECK-NEXT: UnaryOperator{{.*}} 'int' postfix '--'
+// CHECK-NEXT: DeclRefExpr{{.*}}'x' 'int'
+#pragma acc atomic
+  x--;
+// CHECK-NEXT: OpenACCAtomicConstruct{{.*}} atomic capture
+// CHECK-NEXT: BinaryOperator{{.*}} 'int' lvalue '='
+// CHECK-NEXT: DeclRefExpr{{.*}}'v' 'int'
+// CHECK-NEXT: UnaryOperator{{.*}} 'int' postfix '++'
+// CHECK-NEXT: DeclRefExpr{{.*}}'x' 'int'
+#pragma acc atomic capture
+  v = x++;
+
+// CHECK-NEXT: OpenACCAtomicConstruct{{.*}} atomic capture
+// CHECK-NEXT: CompoundStmt
+// CHECK-NEXT: UnaryOperator{{.*}} 'int' postfix '--'
+// CHECK-NEXT: DeclRefExpr{{.*}}'x' 'int'
+// CHECK-NEXT: BinaryOperator{{.*}} 'int' lvalue '='
+// CHECK-NEXT: DeclRefExpr{{.*}}'v' 'int'
+// CHECK-NEXT: ImplicitCastExpr{{.*}}'int' <LValueToRValue>
+// CHECK-NEXT: DeclRefExpr{{.*}}'x' 'int'
+#pragma acc atomic capture
+  { x--; v = x; }
+
+}
+
+template<typename T, int I>
+void templ_foo(T v, T x) {
+  // CHECK-NEXT: FunctionTemplateDecl{{.*}}templ_foo
+  // CHECK-NEXT: TemplateTypeParmDecl{{.*}} T
+  // CHECK-NEXT: NonTypeTemplateParmDecl{{.*}} I
+  // CHECK-NEXT: FunctionDecl{{.*}} templ_foo 'void (T, T)'
+  // CHECK-NEXT: ParmVarDecl{{.*}} v 'T'
+  // CHECK-NEXT: ParmVarDecl{{.*}} x 'T'
+  // CHECK-NEXT: CompoundStmt
+
+// CHECK-NEXT: OpenACCAtomicConstruct{{.*}} atomic read
+// CHECK-NEXT: BinaryOperator{{.*}} '<dependent type>' '='
+// CHECK-NEXT: DeclRefExpr{{.*}}'v' 'T'
+// CHECK-NEXT: DeclRefExpr{{.*}}'x' 'T'
+#pragma acc atomic read
+  v = x;
+
+// CHECK-NEXT: OpenACCAtomicConstruct{{.*}} atomic write
+// CHECK-NEXT: BinaryOperator{{.*}} '<dependent type>' '='
+// CHECK-NEXT: DeclRefExpr{{.*}}'v' 'T'
+// CHECK-NEXT: BinaryOperator{{.*}}'<dependent type>' '+'
+// CHECK-NEXT: DeclRefExpr{{.*}}'x' 'T'
+// CHECK-NEXT: DeclRefExpr{{.*}} 'I' 'int'
+#pragma acc atomic write
+  v = x + I;
+
+// CHECK-NEXT: OpenACCAtomicConstruct{{.*}} atomic update 
+// CHECK-NEXT: UnaryOperator{{.*}} '<dependent type>' postfix '++'
+// CHECK-NEXT: DeclRefExpr{{.*}}'x' 'T'
+#pragma acc atomic update
+  x++;
+// CHECK-NEXT: OpenACCAtomicConstruct{{.*}} atomic <none>
+// CHECK-NEXT: UnaryOperator{{.*}} '<dependent type>' postfix '--'
+// CHECK-NEXT: DeclRefExpr{{.*}}'x' 'T'
+#pragma acc atomic
+  x--;
+// CHECK-NEXT: OpenACCAtomicConstruct{{.*}} atomic capture
+// CHECK-NEXT: BinaryOperator{{.*}} '<dependent type>' '='
+// CHECK-NEXT: DeclRefExpr{{.*}}'v' 'T'
+// CHECK-NEXT: UnaryOperator{{.*}} '<dependent type>' postfix '++'
+// CHECK-NEXT: DeclRefExpr{{.*}}'x' 'T'
+#pragma acc atomic capture
+  v = x++;
+
+// CHECK-NEXT: OpenACCAtomicConstruct{{.*}} atomic capture
+// CHECK-NEXT: CompoundStmt
+// CHECK-NEXT: UnaryOperator{{.*}} '<dependent type>' postfix '--'
+// CHECK-NEXT: DeclRefExpr{{.*}}'x' 'T'
+// CHECK-NEXT: BinaryOperator{{.*}} '<dependent type>' '='
+// CHECK-NEXT: DeclRefExpr{{.*}}'v' 'T'
+// CHECK-NEXT: DeclRefExpr{{.*}}'x' 'T'
+#pragma acc atomic capture
+  { x--; v = x; }
+
+  // CHECK-NEXT: FunctionDecl{{.*}} templ_foo 'void (int, int)' implicit_instantiation
+  // CHECK-NEXT: TemplateArgument type 'int'
+  // CHECK-NEXT: BuiltinType{{.*}} 'int'
+  // CHECK-NEXT: TemplateArgument integral '5'
+  // CHECK-NEXT: ParmVarDecl{{.*}} v 'int'
+  // CHECK-NEXT: ParmVarDecl{{.*}} x 'int'
+  // CHECK-NEXT: CompoundStmt
+
+// CHECK-NEXT: OpenACCAtomicConstruct{{.*}} atomic read
+// CHECK-NEXT: BinaryOperator{{.*}} 'int' lvalue '='
+// CHECK-NEXT: DeclRefExpr{{.*}}'v' 'int'
+// CHECK-NEXT: ImplicitCastExpr{{.*}}'int' <LValueToRValue>
+// CHECK-NEXT: DeclRefExpr{{.*}}'x' 'int'
+
+// CHECK-NEXT: OpenACCAtomicConstruct{{.*}} atomic write
+// CHECK-NEXT: BinaryOperator{{.*}} 'int' lvalue '='
+// CHECK-NEXT: DeclRefExpr{{.*}}'v' 'int'
+// CHECK-NEXT: BinaryOperator{{.*}}'int' '+'
+// CHECK-NEXT: ImplicitCastExpr{{.*}}'int' <LValueToRValue>
+// CHECK-NEXT: DeclRefExpr{{.*}}'x' 'int'
+// CHECK-NEXT: SubstNonTypeTemplateParmExpr{{.*}} 'int'
+// CHECK-NEXT: NonTypeTemplateParmDecl{{.*}} 'int'{{.*}}I
+// CHECK-NEXT: IntegerLiteral{{.*}} 'int' 5
+
+// CHECK-NEXT: OpenACCAtomicConstruct{{.*}} atomic update 
+// CHECK-NEXT: UnaryOperator{{.*}} 'int' postfix '++'
+// CHECK-NEXT: DeclRefExpr{{.*}}'x' 'int'
+
+// CHECK-NEXT: OpenACCAtomicConstruct{{.*}} atomic <none>
+// CHECK-NEXT: UnaryOperator{{.*}} 'int' postfix '--'
+// CHECK-NEXT: DeclRefExpr{{.*}}'x' 'int'
+
+// CHECK-NEXT: OpenACCAtomicConstruct{{.*}} atomic capture
+// CHECK-NEXT: BinaryOperator{{.*}} 'int' lvalue '='
+// CHECK-NEXT: DeclRefExpr{{.*}}'v' 'int'
+// CHECK-NEXT: UnaryOperator{{.*}} 'int' postfix '++'
+// CHECK-NEXT: DeclRefExpr{{.*}}'x' 'int'
+
+// CHECK-NEXT: OpenACCAtomicConstruct{{.*}} atomic capture
+// CHECK-NEXT: CompoundStmt
+// CHECK-NEXT: UnaryOperator{{.*}} 'int' postfix '--'
+// CHECK-NEXT: DeclRefExpr{{.*}}'x' 'int'
+// CHECK-NEXT: BinaryOperator{{.*}} 'int' lvalue '='
+// CHECK-NEXT: DeclRefExpr{{.*}}'v' 'int'
+// CHECK-NEXT: ImplicitCastExpr{{.*}}'int' <LValueToRValue>
+// CHECK-NEXT: DeclRefExpr{{.*}}'x' 'int'
+}
+
+void use() {
+  templ_foo<int, 5>(1, 2);
+}
+#endif
diff --git a/clang/test/SemaOpenACC/atomic-construct.cpp b/clang/test/SemaOpenACC/atomic-construct.cpp
new file mode 100644
index 0000000000000..7357d91d704fd
--- /dev/null
+++ b/clang/test/SemaOpenACC/atomic-construct.cpp
@@ -0,0 +1,1846 @@
+// RUN: %clang_cc1 %s -fopenacc -Wno-unused-value -verify 
+
+void NormalFunc(int I) {
+  // No clauses are valid, but we parse them anyway, just mark them as not valid
+  // on this construct.
+ 
+  // expected-error@+1{{OpenACC 'copy' clause is not valid on 'atomic' directive}}
+#pragma acc atomic copy(I)
+  I = I + 1;
+  // expected-error@+1{{OpenACC 'copy' clause is not valid on 'atomic' directive}}
+#pragma acc atomic read copy(I)
+  I = I;
+}
+
+struct Struct{
+  Struct *getPtr();
+  Struct &operator++();
+  Struct &operator--();
+  Struct &operator++(int);
+  Struct &operator--(int);
+
+  Struct &operator+=(int);
+  Struct &operator*=(int);
+  Struct &operator-=(int);
+  Struct &operator/=(int);
+  Struct &operator&=(int);
+  Struct &operator|=(int);
+  Struct &operator<<=(int);
+  Struct &operator>>=(int);
+  Struct &operator^=(int);
+  Struct &operator%=(int);
+  Struct &operator!=(int);
+  Struct &operator+();
+  Struct &operator-();
+
+  operator int();
+  void operator()();
+  Struct &operator*();
+  Struct &operator=(int);
+};
+
+int operator+(Struct&, int);
+int operator+(int, Struct&);
+Struct &operator+(Struct&, Struct&);
+Struct &operator*(Struct&, Struct&);
+Struct &operator-(Struct&, Struct&);
+
+Struct S1, S2;
+
+template<typename T>
+T &getRValue();
+
+template<typename T>
+void AtomicReadTemplate(T LHS, T RHS) {
+#pragma acc atomic read
+  LHS = RHS;
+
+  T *LHSPtr, *RHSPtr;
+
+#pragma acc atomic read
+  LHSPtr = RHSPtr;
+
+  // expected-error@+2{{statement associated with OpenACC 'atomic read' directive is invalid}}
+  // expected-note@+2{{right operand to assignment expression must be an l-value}}
+#pragma acc atomic read
+  LHS = RHS + 1;
+
+#pragma acc atomic read
+  *LHSPtr = RHS;
+
+#pragma acc atomic read
+  LHS = *RHSPtr;
+
+  // expected-error@+2{{statement associated with OpenACC 'atomic read' directive is invalid}}
+  // expected-note@+2{{right operand to assignment expression must be an l-value}}
+#pragma acc atomic read
+  LHS = getRValue<T>();
+}
+
+template<typename T>
+void AtomicReadTemplate2(T LHS, T RHS) {
+  // expected-error@+2{{statement associated with OpenACC 'atomic read' directive is invalid}}
+  // expected-note@+2{{left operand to assignment expression must be of scalar type (was 'Struct')}}
+#pragma acc atomic read
+  LHS = RHS;
+
+  T *LHSPtr, *RHSPtr;
+  // Fine, now a pointer.
+#pragma acc atomic read
+  LHSPtr = RHSPtr;
+
+  // expected-error@+2{{statement associated with OpenACC 'atomic read' directive is invalid}}
+  // expected-note@+2{{right operand to assignment expression must be an l-value}}
+#pragma acc atomic read
+  LHS = *RHS.getPtr();
+
+  // expected-error@+2{{statement associated with OpenACC 'atomic read' directive is invalid}}
+  // expected-note@+2{{left operand to assignment expression must be of scalar type (was 'Struct')}}
+#pragma acc atomic read
+  *LHSPtr = RHS;
+
+  // expected-error@+2{{statement associated with OpenACC 'atomic read' directive is invalid}}
+  // expected-note@+2{{left operand to assignment expression must be of scalar type (was 'Struct')}}
+#pragma acc atomic read
+  LHS = *RHSPtr;
+
+  // expected-error@+2{{statement associated with OpenACC 'atomic read' directive is invalid}}
+  // expected-note@+2{{left operand to assignment expression must be an l-value}}
+#pragma acc atomic read
+  getRValue<T>() = getRValue<T>();
+
+  // expected-error@+2{{statement associated with OpenACC 'atomic read' directive is invalid}}
+  // expected-note@+2{{right operand to assignment expression must be an l-value}}
+#pragma acc atomic read
+  LHS = getRValue<T>();
+}
+
+void AtomicRead(int LHS, int RHS) {
+  AtomicReadTemplate(LHS, RHS);
+  AtomicReadTemplate2(S1, S2); // expected-note{{in instantiation of function template specialization}}
+
+#pragma acc atomic read
+  LHS = RHS;
+
+  int *LHSPtr, *RHSPtr;
+
+#pragma acc atomic read
+  LHSPtr = RHSPtr;
+
+  // expected-error@+2{{statement associated with OpenACC 'atomic read' directive is invalid}}
+  // expected-note@+2{{left operand to assignment expression must be of scalar type (was 'Struct')}}
+#pragma acc atomic read
+  S1 = S2;
+
+  // expected-error@+2{{statement associated with OpenACC 'atomic read' directive is invalid}}
+  // expected-note@+2{{right operand to assignment expression must be an l-value}}
+#pragma acc atomic read
+  LHS = RHS + 1;
+
+#pragma acc atomic read
+  *LHSPtr = RHS;
+
+#pragma acc atomic read
+  LHS = *RHSPtr;
+
+  // There is no way to test that = is an overloaded operator, since there
+  // really isn't a way to create an operator= without a class type on one side
+  // or the other.
+}
+
+template<typename T>
+void AtomicWriteTemplate(T LHS, T RHS) {
+#pragma acc atomic write
+  LHS = RHS;
+
+  T *LHSPtr, *RHSPtr;
+#pragma acc atomic write
+  LHSPtr = RHSPtr;
+
+#pragma acc atomic write
+  *LHSPtr = *RHSPtr;
+
+  // allowed, expr is ok.
+#pragma acc atomic write
+  LHS = *RHSPtr;
+
+#pragma acc atomic write
+  LHS = RHS * 2;
+
+  // expected-error@+2{{statement associated with OpenACC 'atomic write' directive is invalid}}
+  // expected-note@+2{{left operand to assignment expression must be an l-value}}
+#pragma acc atomic write
+  getRValue<T>() = getRValue<T>();
+
+#pragma acc atomic write
+  LHS = getRValue<T>();
+}
+
+template<typename T>
+void AtomicWriteTemplate2(T LHS, T RHS) {
+  // expected-error@+2{{statement associated with OpenACC 'atomic write' directive is invalid}}
+  // expected-note@+2{{left operand to assignment expression must be of scalar type (was 'Struct')}}
+#pragma acc atomic write
+  LHS = RHS;
+
+  T *LHSPtr, *RHSPtr;
+#pragma acc atomic write
+  LHSPtr = RHSPtr;
+
+  // expected-error@+2{{statement associated with OpenACC 'atomic write' directive is invalid}}
+  // expected-note@+2{{left operand to assignment expression must be of scalar type (was 'Struct')}}
+#pragma acc atomic write
+  LHS = *RHSPtr;
+
+#pragma acc atomic write
+  LHSPtr = RHS.getPtr();
+
+  // expected-error@+2{{statement associated with OpenACC 'atomic write' directive is invalid}}
+  // expected-note@+2{{left operand to assignment expression must be an l-value}}
+#pragma acc atomic write
+  getRValue<T>() = getRValue<T>();
+
+  // expected-error@+2{{statement associated with OpenACC 'atomic write' directive is invalid}}
+  // expected-note@+2{{left operand to assignment expression must be of scalar type (was 'Struct')}}
+#pragma acc atomic write
+  LHS = getRValue<T>();
+}
+
+void AtomicWrite(int LHS, int RHS) {
+  AtomicWriteTemplate(LHS, RHS);
+  AtomicWriteTemplate2(S1, S2); // expected-note{{in instantiation of function template specialization}}
+
+#pragma acc atomic write
+  LHS = RHS;
+
+  int *LHSPtr, *RHSPtr;
+#pragma acc atomic write
+  LHSPtr = RHSPtr;
+
+#pragma acc atomic write
+  *LHSPtr = *RHSPtr;
+
+  // allowed, expr is ok.
+#pragma acc atomic write
+  LHS = *RHSPtr;
+
+#pragma acc atomic write
+  LHS = RHS * 2;
+}
+
+template<typename T>
+void AtomicUpdateTemplate(T LHS, T RHS) {
+#pragma acc atomic
+  LHS++;
+
+#pragma acc atomic update
+  LHS--;
+
+#pragma acc atomic
+  ++LHS;
+
+#pragma acc atomic update
+  --LHS;
+
+  // expected-error@+2{{statement associated with OpenACC 'atomic' directive is invalid}}
+  // expected-note@+2{{unary operator not supported, only increment and decrement operations permitted}}
+#pragma acc atomic
+  +LHS;
+
+  // expected-error@+2{{statement associated with OpenACC 'atomic update' directive is invalid}}
+  // expected-note@+2{{unary operator not supported, only increment and decrement operations permitted}}
+#pragma acc atomic update
+  -LHS;
+
+  // expected-error@+2{{statement associated with OpenACC 'atomic update' directive is invalid}}
+  // expected-note@+2{{expected binary operation on right hand side of assignment operator}}
+#pragma acc atomic update
+  LHS = RHS;
+
+  T *LHSPtr, *RHSPtr;
+
+  // expected-error@+2{{statement associated with OpenACC 'atomic' directive is invalid}}
+  // expected-note@+2{{expected binary operation on right hand side of assignment operator}}
+#pragma acc atomic
+  *LHSPtr = *RHSPtr;
+
+  // x binop= expr;
+#pragma acc atomic
+  LHS += 1 + RHS;
+#pragma acc atomic update
+  LHS *= 1 + RHS;
+#pragma acc atomic
+  LHS -= 1 + RHS;
+#pragma acc atomic update
+  LHS /= 1 + RHS;
+#pragma acc atomic
+  LHS &= 1 + RHS;
+#pragma acc atomic update
+  LHS ^= 1 + RHS;
+#pragma acc atomic
+  LHS |= 1 + RHS;
+#pragma acc atomic update
+  LHS <<= 1 + RHS;
+#pragma acc atomic
+  LHS >>= 1 + RHS;
+
+  // expected-error@+2{{statement associated with OpenACC 'atomic update' directive is invalid}}
+  // expected-note@+2{{compound binary operator not supported, only +=, *=, -=, /=, &=, ^=, |=, <<=, or >>= are permitted}}
+#pragma acc atomic update
+  LHS != 1 + RHS;
+
+  // expected-error@+2{{statement associated with OpenACC 'atomic update' directive is invalid}}
+  // expected-note@+2{{compound binary operator not supported, only +=, *=, -=, /=, &=, ^=, |=, <<=, or >>= are permitted}}
+#pragma acc atomic update
+  LHS <= 1 + RHS;
+
+  // expected-error@+2{{statement associated with OpenACC 'atomic update' directive is invalid}}
+  // expected-note@+2{{compound binary operator not supported, only +=, *=, -=, /=, &=, ^=, |=, <<=, or >>= are permitted}}
+#pragma acc atomic update
+  LHS >= 1 + RHS;
+
+  // expected-error@+2{{statement associated with OpenACC 'atomic update' directive is invalid}}
+  // expected-note@+2{{compound binary operator not supported, only +=, *=, -=, /=, &=, ^=, |=, <<=, or >>= are permitted}}
+#pragma acc atomic update
+  LHS %= 1 + RHS;
+
+  // x = x binop expr.
+#pragma acc atomic
+  LHS = LHS + getRValue<T>();
+#pragma acc atomic update
+  LHS = LHS * getRValue<T>();
+#pragma acc atomic update
+  LHS = LHS - getRValue<T>();
+#pragma acc atomic update
+  LHS = LHS / getRValue<T>();
+#pragma acc atomic update
+  LHS = LHS & getRValue<T>();
+#pragma acc atomic update
+  LHS = LHS ^ getRValue<T>();
+#pragma acc atomic update
+  LHS = LHS | getRValue<T>();
+#pragma acc atomic update
+  LHS = LHS << getRValue<T>();
+#pragma acc atomic update
+  LHS = LHS >> getRValue<T>();
+  // expected-error@+2{{statement associated with OpenACC 'atomic update' directive is invalid}}
+  // expected-note@+2{{binary operator not supported, only +, *, -, /, &, ^, |, <<, or >> are permitted}}
+#pragma acc atomic update
+  LHS = LHS < getRValue<T>();
+  // expected-error@+2{{statement associated with OpenACC 'atomic update' directive is invalid}}
+  // expected-note@+2{{binary operator not supported, only +, *, -, /, &, ^, |, <<, or >> are permitted}}
+#pragma acc atomic update
+  LHS = LHS > getRValue<T>();
+  // expected-error@+2{{statement associated with OpenACC 'atomic update' directive is invalid}}
+  // expected-note@+2{{binary operator not supported, only +, *, -, /, &, ^, |, <<, or >> are permitted}}
+#pragma acc atomic update
+  LHS = LHS <= getRValue<T>();
+  // expected-error@+2{{statement associated with OpenACC 'atomic update' directive is invalid}}
+  // expected-note@+2{{binary operator not supported, only +, *, -, /, &, ^, |, <<, or >> are permitted}}
+#pragma acc atomic update
+  LHS = LHS >= getRValue<T>();
+#pragma acc atomic update
+  LHS = LHS ^ getRValue<T>();
+
+
+  // x = expr binop x.
+#pragma acc atomic
+  LHS = getRValue<T>() + LHS;
+#pragma acc atomic update
+  LHS = getRValue<T>() * LHS;
+#pragma acc atomic update
+  LHS = getRValue<T>() - LHS;
+#pragma acc atomic update
+  LHS = getRValue<T>() / LHS;
+#pragma acc atomic update
+  LHS = getRValue<T>() & LHS;
+#pragma acc atomic update
+  LHS = getRValue<T>() ^ LHS;
+#pragma acc atomic update
+  LHS = getRValue<T>() | LHS;
+#pragma acc atomic update
+  LHS = getRValue<T>() << LHS;
+#pragma acc atomic update
+  LHS = getRValue<T>() >> LHS;
+  // expected-error@+2{{statement associated with OpenACC 'atomic update' directive is invalid}}
+  // expected-note@+2{{binary operator not supported, only +, *, -, /, &, ^, |, <<, or >> are permitted}}
+#pragma acc atomic update
+  LHS = getRValue<T>() < LHS;
+  // expected-error@+2{{statement associated with OpenACC 'atomic update' directive is invalid}}
+  // expected-note@+2{{binary operator not supported, only +, *, -, /, &, ^, |, <<, or >> are permitted}}
+#pragma acc atomic update
+  LHS = getRValue<T>() > LHS;
+  // expected-error@+2{{statement associated with OpenACC 'atomic update' directive is invalid}}
+  // expected-note@+2{{binary operator not supported, only +, *, -, /, &, ^, |, <<, or >> are permitted}}
+#pragma acc atomic update
+  LHS = getRValue<T>() <= LHS;
+  // expected-error@+2{{statement associated with OpenACC 'atomic update' directive is invalid}}
+  // expected-note@+2{{binary operator not supported, only +, *, -, /, &, ^, |, <<, or >> are permitted}}
+#pragma acc atomic update
+  LHS = getRValue<T>() >= LHS;
+#pragma acc atomic update
+  LHS = getRValue<T>() ^ LHS;
+
+#pragma acc atomic update
+  LHS = LHS + getRValue<T>();
+  // expected-error@+2{{statement associated with OpenACC 'atomic update' directive is invalid}}
+  // expected-note@+2{{left hand side of assignment operation('LHS') must match one side of the sub-operation on the right hand side('RHS' and 'getRValue<T>()')}}
+#pragma acc atomic update
+  LHS = RHS + getRValue<T>();
+
+#pragma acc atomic update
+  LHS = getRValue<T>() - LHS;
+  // expected-error@+2{{statement associated with OpenACC 'atomic update' directive is invalid}}
+  // expected-note@+2{{left hand side of assignment operation('LHS') must match one side of the sub-operation on the right hand side('getRValue<T>()' and 'RHS')}}
+#pragma acc atomic update
+  LHS = getRValue<T>() + RHS;
+}
+
+template<typename T>
+void AtomicUpdateTemplate2(T LHS, T RHS) {
+  // expected-error@+2{{statement associated with OpenACC 'atomic' directive is invalid}}
+  // expected-note@+2{{operand to increment expression must be of scalar type (was 'Struct')}}
+#pragma acc atomic
+  LHS++;
+
+  // expected-error@+2{{statement associated with OpenACC 'atomic update' directive is invalid}}
+  // expected-note@+2{{operand to decrement expression must be of scalar type (was 'Struct')}}
+#pragma acc atomic update
+  LHS--;
+
+  // expected-error@+2{{statement associated with OpenACC 'atomic' directive is invalid}}
+  // expected-note@+2{{operand to increment expression must be of scalar type (was 'Struct')}}
+#pragma acc atomic
+  ++LHS;
+
+  // expected-error@+2{{statement associated with OpenACC 'atomic update' directive is invalid}}
+  // expected-note@+2{{operand to decrement expression must be of scalar type (was 'Struct')}}
+#pragma acc atomic update
+  --LHS;
+
+  // expected-error@+2{{statement associated with OpenACC 'atomic' directive is invalid}}
+  // expected-note@+2{{unary operator not supported, only increment and decrement operations permitted}}
+#pragma acc atomic
+  +LHS;
+
+  // expected-error@+2{{statement associated with OpenACC 'atomic update' directive is invalid}}
+  // expected-note@+2{{unary operator not supported, only increment and decrement operations permitted}}
+#pragma acc atomic update
+  -LHS;
+
+  // expected-error@+2{{statement associated with OpenACC 'atomic' directive is invalid}}
+  // expected-note@+2{{expected assignment, compound assignment, increment, or decrement expression}}
+#pragma acc atomic
+  LHS();
+
+  // expected-error@+2{{statement associated with OpenACC 'atomic' directive is invalid}}
+  // expected-note@+2{{unary operator not supported, only increment and decrement operations permitted}}
+#pragma acc atomic
+  *LHS;
+
+  // expected-error@+2{{statement associated with OpenACC 'atomic update' directive is invalid}}
+  // expected-note@+2{{expected binary operation on right hand side of assignment operator}}
+#pragma acc atomic update
+  LHS = RHS;
+
+  T *LHSPtr, *RHSPtr;
+
+  // expected-error@+2{{statement associated with OpenACC 'atomic' directive is invalid}}
+  // expected-note@+2{{expected binary operation on right hand side of assignment operator}}
+#pragma acc atomic
+  *LHSPtr = *RHSPtr;
+
+  // x binop= expr;
+  // expected-error@+2{{statement associated with OpenACC 'atomic' directive is invalid}}
+  // expected-note@+2{{left operand to compound assignment expression must be of scalar type (was 'Struct')}}
+#pragma acc atomic
+  LHS += 1 + RHS;
+  // expected-error@+2{{statement associated with OpenACC 'atomic update' directive is invalid}}
+  // expected-note@+2{{left operand to compound assignment expression must be of scalar type (was 'Struct')}}
+#pragma acc atomic update
+  LHS *= 1 + RHS;
+  // expected-error@+2{{statement associated with OpenACC 'atomic' directive is invalid}}
+  // expected-note@+2{{left operand to compound assignment expression must be of scalar type (was 'Struct')}}
+#pragma acc atomic
+  LHS -= 1 + RHS;
+  // expected-error@+2{{statement associated with OpenACC 'atomic update' directive is invalid}}
+  // expected-note@+2{{left operand to compound assignment expression must be of scalar type (was 'Struct')}}
+#pragma acc atomic update
+  LHS /= 1 + RHS;
+  // expected-error@+2{{statement associated with OpenACC 'atomic' directive is invalid}}
+  // expected-note@+2{{left operand to compound assignment expression must be of scalar type (was 'Struct')}}
+#pragma acc atomic
+  LHS &= 1 + RHS;
+  // expected-error@+2{{statement associated with OpenACC 'atomic update' directive is invalid}}
+  // expected-note@+2{{left operand to compound assignment expression must be of scalar type (was 'Struct')}}
+#pragma acc atomic update
+  LHS ^= 1 + RHS;
+  // expected-error@+2{{statement associated with OpenACC 'atomic' directive is invalid}}
+  // expected-note@+2{{left operand to compound assignment expression must be of scalar type (was 'Struct')}}
+#pragma acc atomic
+  LHS |= 1 + RHS;
+  // expected-error@+2{{statement associated with OpenACC 'atomic update' directive is invalid}}
+  // expected-note@+2{{left operand to compound assignment expression must be of scalar type (was 'Struct')}}
+#pragma acc atomic update
+  LHS <<= 1 + RHS;
+  // expected-error@+2{{statement associated with OpenACC 'atomic' directive is invalid}}
+  // expected-note@+2{{left operand to compound assignment expression must be of scalar type (was 'Struct')}}
+#pragma acc atomic
+  LHS >>= 1 + RHS;
+
+  // expected-error@+2{{statement associated with OpenACC 'atomic update' directive is invalid}}
+  // expected-note@+2{{compound binary operator not supported, only +=, *=, -=, /=, &=, ^=, |=, <<=, or >>= are permitted}}
+#pragma acc atomic update
+  LHS != 1 + RHS;
+
+  // expected-error@+2{{statement associated with OpenACC 'atomic update' directive is invalid}}
+  // expected-note@+2{{compound binary operator not supported, only +=, *=, -=, /=, &=, ^=, |=, <<=, or >>= are permitted}}
+#pragma acc atomic update
+  LHS %= 1 + RHS;
+
+  // x = x binop expr.
+  // expected-error@+2{{statement associated with OpenACC 'atomic' directive is invalid}}
+  // expected-note@+2{{left operand to assignment expression must be of scalar type (was 'Struct')}}
+#pragma acc atomic
+  LHS = LHS + getRValue<T>();
+  // expected-error@+2{{statement associated with OpenACC 'atomic update' directive is invalid}}
+  // expected-note@+2{{left operand to assignment expression must be of scalar type (was 'Struct')}}
+#pragma acc atomic update
+  LHS = LHS * getRValue<T>();
+  // expected-error@+2{{statement associated with OpenACC 'atomic update' directive is invalid}}
+  // expected-note@+2{{left operand to assignment expression must be of scalar type (was 'Struct')}}
+#pragma acc atomic update
+  LHS = LHS - getRValue<T>();
+
+  // x = expr binop x.
+  // expected-error@+2{{statement associated with OpenACC 'atomic' directive is invalid}}
+  // expected-note@+2{{left operand to assignment expression must be of scalar type (was 'Struct')}}
+#pragma acc atomic
+  LHS = getRValue<T>() + LHS;
+  // expected-error@+2{{statement associated with OpenACC 'atomic update' directive is invalid}}
+  // expected-note@+2{{left operand to assignment expression must be of scalar type (was 'Struct')}}
+#pragma acc atomic update
+  LHS = getRValue<T>() * LHS;
+  // expected-error@+2{{statement associated with OpenACC 'atomic update' directive is invalid}}
+  // expected-note@+2{{left operand to assignment expression must be of scalar type (was 'Struct')}}
+#pragma acc atomic update
+  LHS = getRValue<T>() - LHS;
+
+  // expected-error@+2{{statement associated with OpenACC 'atomic update' directive is invalid}}
+  // expected-note@+2{{left operand to assignment expression must be of scalar type (was 'Struct')}}
+#pragma acc atomic update
+  LHS = LHS + getRValue<T>();
+  // expected-error@+2{{statement associated with OpenACC 'atomic update' directive is invalid}}
+  // expected-note@+2{{left hand side of assignment operation('LHS') must match one side of the sub-operation on the right hand side('RHS' and 'getRValue<T>()')}}
+#pragma acc atomic update
+  LHS = RHS + getRValue<T>();
+
+  // expected-error@+2{{statement associated with OpenACC 'atomic update' directive is invalid}}
+  // expected-note@+2{{left operand to assignment expression must be of scalar type (was 'Struct')}}
+#pragma acc atomic update
+  LHS = getRValue<T>() - LHS;
+  // expected-error@+2{{statement associated with OpenACC 'atomic update' directive is invalid}}
+  // expected-note@+2{{left hand side of assignment operation('LHS') must match one side of the sub-operation on the right hand side('getRValue<T>()' and 'RHS')}}
+#pragma acc atomic update
+  LHS = getRValue<T>() + RHS;
+}
+
+void AtomicUpdate() {
+  AtomicUpdateTemplate(1, 2);
+  AtomicUpdateTemplate2(S1, S2); //expected-note{{in instantiation of function template specialization}}
+
+  int I, J;
+
+#pragma acc atomic
+  I++;
+#pragma acc atomic update
+  --I;
+  // expected-error@+2{{statement associated with OpenACC 'atomic' directive is invalid}}
+  // expected-note@+2{{operand to increment expression must be of scalar type (was 'Struct')}}
+#pragma acc atomic
+  S1++;
+  // expected-error@+2{{statement associated with OpenACC 'atomic update' directive is invalid}}
+  // expected-note@+2{{operand to decrement expression must be of scalar type (was 'Struct')}}
+#pragma acc atomic update
+  --S2;
+
+  // expected-error@+2{{statement associated with OpenACC 'atomic' directive is invalid}}
+  // expected-note@+2{{unary operator not supported, only increment and decrement operations permitted}}
+#pragma acc atomic
+  +I;
+  // expected-error@+2{{statement associated with OpenACC 'atomic update' directive is invalid}}
+  // expected-note@+2{{unary operator not supported, only increment and decrement operations permitted}}
+#pragma acc atomic update
+  -J;
+
+#pragma acc atomic update
+  I ^= 1 + J;
+
+  // expected-error@+2{{statement associated with OpenACC 'atomic update' directive is invalid}}
+  // expected-note@+2{{compound binary operator not supported, only +=, *=, -=, /=, &=, ^=, |=, <<=, or >>= are permitted}}
+#pragma acc atomic update
+  I%= 1 + J;
+
+  // expected-error@+2{{statement associated with OpenACC 'atomic update' directive is invalid}}
+  // expected-note@+2{{left operand to compound assignment expression must be of scalar type (was 'Struct')}}
+#pragma acc atomic update
+  S1 ^= 1 + J;
+
+  // expected-error@+2{{statement associated with OpenACC 'atomic update' directive is invalid}}
+  // expected-note@+2{{compound binary operator not supported, only +=, *=, -=, /=, &=, ^=, |=, <<=, or >>= are permitted}}
+#pragma acc atomic update
+  S2 %= 1 + J;
+
+#pragma acc atomic update
+  I = I + getRValue<int>();
+  // expected-error@+2{{statement associated with OpenACC 'atomic update' directive is invalid}}
+  // expected-note@+2{{left hand side of assignment operation('I') must match one side of the sub-operation on the right hand side('J' and 'getRValue<int>()')}}
+#pragma acc atomic update
+  I = J + getRValue<int>();
+
+#pragma acc atomic update
+  I = getRValue<int>() - I;
+  // expected-error@+2{{statement associated with OpenACC 'atomic update' directive is invalid}}
+  // expected-note@+2{{left hand side of assignment operation('I') must match one side of the sub-operation on the right hand side('getRValue<int>()' and 'J')}}
+#pragma acc atomic update
+  I = getRValue<int>() + J;
+}
+
+template<typename T>
+void AtomicCaptureTemplateSimple(T LHS, T RHS) {
+  // expected-error@+2{{statement associated with OpenACC 'atomic capture' directive is invalid}}
+  // expected-note@+2{{expected assignment expression}}
+#pragma acc atomic capture
+  LHS++;
+  // expected-error@+2{{statement associated with OpenACC 'atomic capture' directive is invalid}}
+  // expected-note@+2{{expected assignment expression}}
+#pragma acc atomic capture
+--LHS;
+
+  // expected-error@+2{{statement associated with OpenACC 'atomic capture' directive is invalid}}
+  // expected-note@+2{{expected assignment expression}}
+#pragma acc atomic capture
+  LHS += 1 + RHS;
+
+  // expected-error@+2{{statement associated with OpenACC 'atomic capture' directive is invalid}}
+  // expected-note@+2{{expected assignment, compound assignment, increment, or decrement expression}}
+#pragma acc atomic capture
+  LHS = RHS;
+
+#pragma acc atomic capture
+  LHS = RHS++;
+
+#pragma acc atomic capture
+  LHS = RHS--;
+
+#pragma acc atomic capture
+  LHS = ++RHS;
+
+#pragma acc atomic capture
+  LHS = --RHS;
+
+  // expected-error@+2{{statement associated with OpenACC 'atomic capture' directive is invalid}}
+  // expected-note@+2{{unary operator not supported, only increment and decrement operations permitted}}
+#pragma acc atomic capture
+  LHS = +RHS;
+
+#pragma acc atomic capture
+  LHS = RHS += 1 + RHS;
+#pragma acc atomic capture
+  LHS = RHS *= 1 + RHS;
+#pragma acc atomic capture
+  LHS = RHS -= 1 + RHS;
+#pragma acc atomic capture
+  LHS = RHS /= 1 + RHS;
+#pragma acc atomic capture
+  LHS = RHS &= 1 + RHS;
+#pragma acc atomic capture
+  LHS = RHS ^= 1 + RHS;
+#pragma acc atomic capture
+  LHS = RHS >>= 1 + RHS;
+#pragma acc atomic capture
+  LHS = RHS |= 1 + RHS;
+#pragma acc atomic capture
+  LHS = RHS <<= 1 + RHS;
+#pragma acc atomic capture
+  LHS = RHS >>= 1 + RHS;
+
+#pragma acc atomic capture
+  LHS = RHS ^= 1 + RHS;
+  // expected-error@+2{{statement associated with OpenACC 'atomic capture' directive is invalid}}
+  // expected-note@+2{{compound binary operator not supported, only +=, *=, -=, /=, &=, ^=, |=, <<=, or >>= are permitted}}
+#pragma acc atomic capture
+  LHS = RHS <= 1 + RHS;
+  // expected-error@+2{{statement associated with OpenACC 'atomic capture' directive is invalid}}
+  // expected-note@+2{{compound binary operator not supported, only +=, *=, -=, /=, &=, ^=, |=, <<=, or >>= are permitted}}
+#pragma acc atomic capture
+  LHS = RHS >= 1 + RHS;
+
+  // expected-error@+2{{statement associated with OpenACC 'atomic capture' directive is invalid}}
+  // expected-note@+2{{compound binary operator not supported, only +=, *=, -=, /=, &=, ^=, |=, <<=, or >>= are permitted}}
+#pragma acc atomic capture
+  LHS = RHS + 1;
+  // expected-error@+2{{statement associated with OpenACC 'atomic capture' directive is invalid}}
+  // expected-note@+2{{compound binary operator not supported, only +=, *=, -=, /=, &=, ^=, |=, <<=, or >>= are permitted}}
+#pragma acc atomic capture
+  LHS = RHS < 1 + RHS;
+  // expected-error@+2{{statement associated with OpenACC 'atomic capture' directive is invalid}}
+  // expected-note@+2{{compound binary operator not supported, only +=, *=, -=, /=, &=, ^=, |=, <<=, or >>= are permitted}}
+#pragma acc atomic capture
+  LHS = RHS > 1 + RHS;
+  // expected-error@+2{{statement associated with OpenACC 'atomic capture' directive is invalid}}
+  // expected-note@+2{{compound binary operator not supported, only +=, *=, -=, /=, &=, ^=, |=, <<=, or >>= are permitted}}
+#pragma acc atomic capture
+  LHS = RHS ^ 1 + RHS;
+
+#pragma acc atomic capture
+  LHS = RHS = RHS + 1;
+#pragma acc atomic capture
+  LHS = RHS = 1 + RHS;
+#pragma acc atomic capture
+  LHS = RHS = RHS * 1;
+#pragma acc atomic capture
+  LHS = RHS = 1 * RHS;
+#pragma acc atomic capture
+  LHS = RHS = RHS / 1;
+#pragma acc atomic capture
+  LHS = RHS = 1 / RHS;
+#pragma acc atomic capture
+  LHS = RHS = RHS ^ 1;
+  // expected-error@+2{{statement associated with OpenACC 'atomic capture' directive is invalid}}
+  // expected-note@+2{{binary operator not supported, only +, *, -, /, &, ^, |, <<, or >> are permitted}}
+#pragma acc atomic capture
+  LHS = RHS = 1 % RHS;
+
+  // expected-error@+2{{statement associated with OpenACC 'atomic capture' directive is invalid}}
+  // expected-note@+2{{binary operator not supported, only +, *, -, /, &, ^, |, <<, or >> are permitted}}
+#pragma acc atomic capture
+  LHS = RHS = RHS < 1;
+  // expected-error@+2{{statement associated with OpenACC 'atomic capture' directive is invalid}}
+  // expected-note@+2{{binary operator not supported, only +, *, -, /, &, ^, |, <<, or >> are permitted}}
+#pragma acc atomic capture
+  LHS = RHS = 1 > RHS;
+
+  // expected-error@+2{{statement associated with OpenACC 'atomic capture' directive is invalid}}
+  // expected-note@+2{{left hand side of assignment operation('LHS') must match one side of the sub-operation on the right hand side('RHS' and 'getRValue<T>()')}}
+#pragma acc atomic capture
+  LHS = LHS = RHS + getRValue<T>();
+
+  // expected-error@+2{{statement associated with OpenACC 'atomic capture' directive is invalid}}
+  // expected-note@+2{{left hand side of assignment operation('LHS') must match one side of the sub-operation on the right hand side('getRValue<T>()' and 'RHS')}}
+#pragma acc atomic capture
+  LHS = LHS = getRValue<T>() + RHS;
+}
+template<typename T>
+void AtomicCaptureTemplateSimple2(T LHS, T RHS) {
+  // expected-error@+2{{statement associated with OpenACC 'atomic capture' directive is invalid}}
+  // expected-note@+2{{expected assignment expression}}
+#pragma acc atomic capture
+  LHS++;
+  // expected-error@+2{{statement associated with OpenACC 'atomic capture' directive is invalid}}
+  // expected-note@+2{{expected assignment expression}}
+#pragma acc atomic capture
+--LHS;
+
+  // expected-error@+2{{statement associated with OpenACC 'atomic capture' directive is invalid}}
+  // expected-note@+2{{expected assignment expression}}
+#pragma acc atomic capture
+  LHS += 1 + RHS;
+
+  // expected-error@+2{{statement associated with OpenACC 'atomic capture' directive is invalid}}
+  // expected-note@+2{{expected assignment, compound assignment, increment, or decrement expression}}
+#pragma acc atomic capture
+  LHS = RHS;
+
+  // expected-error@+2{{statement associated with OpenACC 'atomic capture' directive is invalid}}
+  // expected-note@+2{{left operand to assignment expression must be of scalar type (was 'Struct')}}
+#pragma acc atomic capture
+  LHS = RHS++;
+
+  // expected-error@+2{{statement associated with OpenACC 'atomic capture' directive is invalid}}
+  // expected-note@+2{{left operand to assignment expression must be of scalar type (was 'Struct')}}
+#pragma acc atomic capture
+  LHS = RHS--;
+
+  // expected-error@+2{{statement associated with OpenACC 'atomic capture' directive is invalid}}
+  // expected-note@+2{{left operand to assignment expression must be of scalar type (was 'Struct')}}
+#pragma acc atomic capture
+  LHS = ++RHS;
+
+  // expected-error@+2{{statement associated with OpenACC 'atomic capture' directive is invalid}}
+  // expected-note@+2{{left operand to assignment expression must be of scalar type (was 'Struct')}}
+#pragma acc atomic capture
+  LHS = --RHS;
+
+  // expected-error@+2{{statement associated with OpenACC 'atomic capture' directive is invalid}}
+  // expected-note@+2{{unary operator not supported, only increment and decrement operations permitted}}
+#pragma acc atomic capture
+  LHS = +RHS;
+
+
+  // expected-error@+2{{statement associated with OpenACC 'atomic capture' directive is invalid}}
+  // expected-note@+2{{left operand to assignment expression must be of scalar type (was 'Struct')}}
+#pragma acc atomic capture
+  LHS = RHS += 1 + RHS;
+  // expected-error@+2{{statement associated with OpenACC 'atomic capture' directive is invalid}}
+  // expected-note@+2{{left operand to assignment expression must be of scalar type (was 'Struct')}}
+#pragma acc atomic capture
+  LHS = RHS *= 1 + RHS;
+  // expected-error@+2{{statement associated with OpenACC 'atomic capture' directive is invalid}}
+  // expected-note@+2{{left operand to assignment expression must be of scalar type (was 'Struct')}}
+#pragma acc atomic capture
+  LHS = RHS -= 1 + RHS;
+  // expected-error@+2{{statement associated with OpenACC 'atomic capture' directive is invalid}}
+  // expected-note@+2{{left operand to assignment expression must be of scalar type (was 'Struct')}}
+#pragma acc atomic capture
+  LHS = RHS /= 1 + RHS;
+  // expected-error@+2{{statement associated with OpenACC 'atomic capture' directive is invalid}}
+  // expected-note@+2{{left operand to assignment expression must be of scalar type (was 'Struct')}}
+#pragma acc atomic capture
+  LHS = RHS &= 1 + RHS;
+  // expected-error@+2{{statement associated with OpenACC 'atomic capture' directive is invalid}}
+  // expected-note@+2{{left operand to assignment expression must be of scalar type (was 'Struct')}}
+#pragma acc atomic capture
+  LHS = RHS ^= 1 + RHS;
+  // expected-error@+2{{statement associated with OpenACC 'atomic capture' directive is invalid}}
+  // expected-note@+2{{left operand to assignment expression must be of scalar type (was 'Struct')}}
+#pragma acc atomic capture
+  LHS = RHS >>= 1 + RHS;
+  // expected-error@+2{{statement associated with OpenACC 'atomic capture' directive is invalid}}
+  // expected-note@+2{{left operand to assignment expression must be of scalar type (was 'Struct')}}
+#pragma acc atomic capture
+  LHS = RHS |= 1 + RHS;
+  // expected-error@+2{{statement associated with OpenACC 'atomic capture' directive is invalid}}
+  // expected-note@+2{{left operand to assignment expression must be of scalar type (was 'Struct')}}
+#pragma acc atomic capture
+  LHS = RHS <<= 1 + RHS;
+  // expected-error@+2{{statement associated with OpenACC 'atomic capture' directive is invalid}}
+  // expected-note@+2{{left operand to assignment expression must be of scalar type (was 'Struct')}}
+#pragma acc atomic capture
+  LHS = RHS >>= 1 + RHS;
+
+  // expected-error@+2{{statement associated with OpenACC 'atomic capture' directive is invalid}}
+  // expected-note@+2{{left operand to assignment expression must be of scalar type (was 'Struct')}}
+#pragma acc atomic capture
+  LHS = RHS ^= 1 + RHS;
+  // expected-error@+2{{statement associated with OpenACC 'atomic capture' directive is invalid}}
+  // expected-note@+2{{compound binary operator not supported, only +=, *=, -=, /=, &=, ^=, |=, <<=, or >>= are permitted}}
+#pragma acc atomic capture
+  LHS = RHS <= 1 + RHS;
+  // expected-error@+2{{statement associated with OpenACC 'atomic capture' directive is invalid}}
+  // expected-note@+2{{compound binary operator not supported, only +=, *=, -=, /=, &=, ^=, |=, <<=, or >>= are permitted}}
+#pragma acc atomic capture
+  LHS = RHS >= 1 + RHS;
+
+  // expected-error@+2{{statement associated with OpenACC 'atomic capture' directive is invalid}}
+  // expected-note@+2{{compound binary operator not supported, only +=, *=, -=, /=, &=, ^=, |=, <<=, or >>= are permitted}}
+#pragma acc atomic capture
+  LHS = RHS + 1;
+  // expected-error@+2{{statement associated with OpenACC 'atomic capture' directive is invalid}}
+  // expected-note@+2{{compound binary operator not supported, only +=, *=, -=, /=, &=, ^=, |=, <<=, or >>= are permitted}}
+#pragma acc atomic capture
+  LHS = RHS < 1 + RHS;
+  // expected-error@+2{{statement associated with OpenACC 'atomic capture' directive is invalid}}
+  // expected-note@+2{{compound binary operator not supported, only +=, *=, -=, /=, &=, ^=, |=, <<=, or >>= are permitted}}
+#pragma acc atomic capture
+  LHS = RHS > 1 + RHS;
+  // expected-error@+2{{statement associated with OpenACC 'atomic capture' directive is invalid}}
+  // expected-note@+2{{compound binary operator not supported, only +=, *=, -=, /=, &=, ^=, |=, <<=, or >>= are permitted}}
+#pragma acc atomic capture
+  LHS = RHS ^ 1 + RHS;
+
+  // expected-error@+2{{statement associated with OpenACC 'atomic capture' directive is invalid}}
+  // expected-note@+2{{left operand to assignment expression must be of scalar type (was 'Struct')}}
+#pragma acc atomic capture
+  LHS = RHS = RHS + 1;
+  // expected-error@+2{{statement associated with OpenACC 'atomic capture' directive is invalid}}
+  // expected-note@+2{{left operand to assignment expression must be of scalar type (was 'Struct')}}
+#pragma acc atomic capture
+  LHS = RHS = 1 + RHS;
+  // expected-error@+2{{statement associated with OpenACC 'atomic capture' directive is invalid}}
+  // expected-note@+2{{left operand to assignment expression must be of scalar type (was 'Struct')}}
+#pragma acc atomic capture
+  LHS = RHS = RHS * 1;
+  // expected-error@+2{{statement associated with OpenACC 'atomic capture' directive is invalid}}
+  // expected-note@+2{{left operand to assignment expression must be of scalar type (was 'Struct')}}
+#pragma acc atomic capture
+  LHS = RHS = 1 * RHS;
+  // expected-error@+2{{statement associated with OpenACC 'atomic capture' directive is invalid}}
+  // expected-note@+2{{left operand to assignment expression must be of scalar type (was 'Struct')}}
+#pragma acc atomic capture
+  LHS = RHS = RHS / 1;
+  // expected-error@+2{{statement associated with OpenACC 'atomic capture' directive is invalid}}
+  // expected-note@+2{{left operand to assignment expression must be of scalar type (was 'Struct')}}
+#pragma acc atomic capture
+  LHS = RHS = 1 / RHS;
+  // expected-error@+2{{statement associated with OpenACC 'atomic capture' directive is invalid}}
+  // expected-note@+2{{left operand to assignment expression must be of scalar type (was 'Struct')}}
+#pragma acc atomic capture
+  LHS = RHS = RHS ^ 1;
+  // expected-error@+2{{statement associated with OpenACC 'atomic capture' directive is invalid}}
+  // expected-note@+2{{binary operator not supported, only +, *, -, /, &, ^, |, <<, or >> are permitted}}
+#pragma acc atomic capture
+  LHS = RHS = 1 % RHS;
+
+  // expected-error@+2{{statement associated with OpenACC 'atomic capture' directive is invalid}}
+  // expected-note@+2{{binary operator not supported, only +, *, -, /, &, ^, |, <<, or >> are permitted}}
+#pragma acc atomic capture
+  LHS = RHS = RHS < 1;
+  // expected-error@+2{{statement associated with OpenACC 'atomic capture' directive is invalid}}
+  // expected-note@+2{{binary operator not supported, only +, *, -, /, &, ^, |, <<, or >> are permitted}}
+#pragma acc atomic capture
+  LHS = RHS = 1 > RHS;
+
+  // expected-error@+2{{statement associated with OpenACC 'atomic capture' directive is invalid}}
+  // expected-note@+2{{left hand side of assignment operation('LHS') must match one side of the sub-operation on the right hand side('RHS' and 'getRValue<T>()')}}
+#pragma acc atomic capture
+  LHS = LHS = RHS + getRValue<T>();
+
+  // expected-error@+2{{statement associated with OpenACC 'atomic capture' directive is invalid}}
+  // expected-note@+2{{left hand side of assignment operation('LHS') must match one side of the sub-operation on the right hand side('getRValue<T>()' and 'RHS')}}
+#pragma acc atomic capture
+  LHS = LHS = getRValue<T>() + RHS;
+}
+
+void AtomicCaptureSimple(int LHS, int RHS) {
+  AtomicCaptureTemplateSimple(1, 2);
+  AtomicCaptureTemplateSimple2(S1, S2); //expected-note{{in instantiation of function template specialization}}
+
+  // expected-error@+2{{statement associated with OpenACC 'atomic capture' directive is invalid}}
+  // expected-note@+2{{expected assignment expression}}
+#pragma acc atomic capture
+  LHS++;
+  // expected-error@+2{{statement associated with OpenACC 'atomic capture' directive is invalid}}
+  // expected-note@+2{{expected assignment expression}}
+#pragma acc atomic capture
+--LHS;
+
+  // expected-error@+2{{statement associated with OpenACC 'atomic capture' directive is invalid}}
+  // expected-note@+2{{expected assignment expression}}
+#pragma acc atomic capture
+  LHS += 1 + RHS;
+
+  // expected-error@+2{{statement associated with OpenACC 'atomic capture' directive is invalid}}
+  // expected-note@+2{{expected assignment, compound assignment, increment, or decrement expression}}
+#pragma acc atomic capture
+  LHS = RHS;
+
+#pragma acc atomic capture
+  LHS = RHS++;
+
+#pragma acc atomic capture
+  LHS = RHS--;
+
+  // expected-error@+2{{statement associated with OpenACC 'atomic capture' directive is invalid}}
+  // expected-note@+2{{left operand to assignment expression must be of scalar type (was 'Struct')}}
+#pragma acc atomic capture
+  S1 = ++S2;
+
+  // expected-error@+2{{statement associated with OpenACC 'atomic capture' directive is invalid}}
+  // expected-note@+2{{left operand to assignment expression must be of scalar type (was 'Struct')}}
+#pragma acc atomic capture
+  S1 = --S2 ;
+
+  // expected-error@+2{{statement associated with OpenACC 'atomic capture' directive is invalid}}
+  // expected-note@+2{{unary operator not supported, only increment and decrement operations permitted}}
+#pragma acc atomic capture
+  LHS = +RHS;
+
+#pragma acc atomic capture
+  LHS = RHS += 1 + RHS;
+#pragma acc atomic capture
+  LHS = RHS *= 1 + RHS;
+  // expected-error@+2{{statement associated with OpenACC 'atomic capture' directive is invalid}}
+  // expected-note@+2{{left operand to assignment expression must be of scalar type (was 'Struct')}}
+#pragma acc atomic capture
+  S1 = RHS -= 1 + RHS;
+  // expected-error@+2{{statement associated with OpenACC 'atomic capture' directive is invalid}}
+  // expected-note@+2{{expected assignment, compound assignment, increment, or decrement expression}}
+#pragma acc atomic capture
+  LHS = S1 /= 1 + RHS;
+#pragma acc atomic capture
+  LHS = RHS &= 1 + S2;
+  // expected-error@+2{{statement associated with OpenACC 'atomic capture' directive is invalid}}
+  // expected-note@+2{{expected assignment, compound assignment, increment, or decrement expression}}
+#pragma acc atomic capture
+  LHS = S1^= 1 + S2;
+
+#pragma acc atomic capture
+  LHS = RHS ^= 1 + RHS;
+  // expected-error@+2{{statement associated with OpenACC 'atomic capture' directive is invalid}}
+  // expected-note@+2{{compound binary operator not supported, only +=, *=, -=, /=, &=, ^=, |=, <<=, or >>= are permitted}}
+#pragma acc atomic capture
+  LHS = RHS <= 1 + RHS;
+  // expected-error@+2{{statement associated with OpenACC 'atomic capture' directive is invalid}}
+  // expected-note@+2{{left operand to assignment expression must be of scalar type (was 'Struct')}}
+#pragma acc atomic capture
+  S1 = RHS ^= 1 + RHS;
+  // expected-error@+2{{statement associated with OpenACC 'atomic capture' directive is invalid}}
+  // expected-note@+2{{compound binary operator not supported, only +=, *=, -=, /=, &=, ^=, |=, <<=, or >>= are permitted}}
+#pragma acc atomic capture
+  LHS = S1 <= 1 + RHS;
+  // expected-error@+2{{statement associated with OpenACC 'atomic capture' directive is invalid}}
+  // expected-note@+2{{compound binary operator not supported, only +=, *=, -=, /=, &=, ^=, |=, <<=, or >>= are permitted}}
+#pragma acc atomic capture
+  LHS = RHS <= 1 + S2;
+
+#pragma acc atomic capture
+  LHS = RHS = RHS + 1;
+#pragma acc atomic capture
+  LHS = RHS = 1 + RHS;
+  // expected-error@+2{{statement associated with OpenACC 'atomic capture' directive is invalid}}
+  // expected-note@+2{{left operand to assignment expression must be of scalar type (was 'Struct')}}
+#pragma acc atomic capture
+  S1 = RHS = RHS * 1;
+  // A little weird, because this contains a 'operator int' call here rather
+  // than a conversion, so the diagnostic could be better.
+  // expected-error@+2{{statement associated with OpenACC 'atomic capture' directive is invalid}}
+  // expected-note@+2{{expected assignment, compound assignment, increment, or decrement expression}}
+#pragma acc atomic capture
+  LHS = S2 = 1 * S2;
+
+  // expected-error@+2{{statement associated with OpenACC 'atomic capture' directive is invalid}}
+  // expected-note@+2{{binary operator not supported, only +, *, -, /, &, ^, |, <<, or >> are permitted}}
+#pragma acc atomic capture
+  LHS = RHS = RHS < 1;
+  // expected-error@+2{{statement associated with OpenACC 'atomic capture' directive is invalid}}
+  // expected-note@+2{{binary operator not supported, only +, *, -, /, &, ^, |, <<, or >> are permitted}}
+#pragma acc atomic capture
+  LHS = RHS = 1 > RHS;
+  // expected-error@+2{{statement associated with OpenACC 'atomic capture' directive is invalid}}
+  // expected-note@+2{{left operand to assignment expression must be of scalar type (was 'Struct')}}
+#pragma acc atomic capture
+  S1 = RHS = RHS < 1;
+
+  // A little weird, because this contains a 'operator int' call here rather
+  // than a conversion, so the diagnostic could be better.
+  // expected-error@+2{{statement associated with OpenACC 'atomic capture' directive is invalid}}
+  // expected-note@+2{{expected assignment, compound assignment, increment, or decrement expression}}
+#pragma acc atomic capture
+  LHS = S1 = 1 > S1;
+
+  // expected-error@+2{{statement associated with OpenACC 'atomic capture' directive is invalid}}
+  // expected-note@+2{{left hand side of assignment operation('LHS') must match one side of the sub-operation on the right hand side('RHS' and 'getRValue<int>()')}}
+#pragma acc atomic capture
+  LHS = LHS = RHS + getRValue<int>();
+
+  // expected-error@+2{{statement associated with OpenACC 'atomic capture' directive is invalid}}
+  // expected-note@+2{{left hand side of assignment operation('LHS') must match one side of the sub-operation on the right hand side('getRValue<int>()' and 'RHS')}}
+#pragma acc atomic capture
+  LHS = LHS = getRValue<int>() + RHS;
+}
+
+template<typename T>
+void AtomicCaptureTemplateCompound(T LHS, T RHS) {
+
+  // expected-error@+2{{statement associated with OpenACC 'atomic capture' directive is invalid}}
+  // expected-note@+2{{expected assignment, compound assignment, increment, or decrement expression}}
+#pragma acc atomic capture
+  {
+  }
+
+  // expected-error@+2{{statement associated with OpenACC 'atomic capture' directive is invalid}}
+  // expected-note@+4{{expected assignment, compound assignment, increment, or decrement expression}}
+#pragma acc atomic capture
+  {
+    LHS = RHS;
+  }
+
+  // expected-error@+2{{statement associated with OpenACC 'atomic capture' directive is invalid}}
+  // expected-note@+3{{'atomic capture' with a compound statement only supports two statements}}
+#pragma acc atomic capture
+  {
+    LHS = RHS; RHS += 1; LHS=RHS;
+  }
+
+
+#pragma acc atomic capture
+  {
+    LHS++;
+    RHS = LHS;
+  }
+
+#pragma acc atomic capture
+  {
+    ++LHS;
+    RHS = LHS;
+  }
+
+#pragma acc atomic capture
+  {
+    --LHS;
+    RHS = LHS;
+  }
+
+
+#pragma acc atomic capture
+  {
+    LHS--;
+    // expected-error@-3{{statement associated with OpenACC 'atomic capture' directive is invalid}}
+    // expected-note@+1{{variable on right hand side of assignment('RHS') must match variable used in unary expression('LHS') from the first statement}}
+    LHS = RHS;
+  }
+
+#pragma acc atomic capture
+  {
+    // expected-error@-2{{statement associated with OpenACC 'atomic capture' directive is invalid}}
+    // expected-note@+1{{unary operator not supported, only increment and decrement operations permitted}}
+    -LHS;
+    RHS = LHS;
+  }
+
+#pragma acc atomic capture
+  {
+    --LHS;
+    // expected-error@-3{{statement associated with OpenACC 'atomic capture' directive is invalid}}
+    // expected-note@+1{{expected assignment expression}}
+    RHS += LHS;
+  }
+
+  // { x binop = expr; v = x; }
+#pragma acc atomic capture
+  {
+    LHS += 1;
+    RHS = LHS;
+  }
+#pragma acc atomic capture
+  {
+    LHS *= 1;
+    // expected-error@-3{{statement associated with OpenACC 'atomic capture' directive is invalid}}
+    // expected-note@+1{{variable on right hand side of assignment('RHS') must match variable used on left hand side of compound assignment('LHS') from the first statement}}
+    LHS = RHS;
+  }
+#pragma acc atomic capture
+  {
+    LHS /= 1;
+  // expected-error@-3{{statement associated with OpenACC 'atomic capture' directive is invalid}}
+  // expected-note@+1{{expected assignment expression}}
+    RHS += LHS;
+  }
+
+  // { x = x binop expr; v = x; }
+#pragma acc atomic capture
+  {
+    LHS = LHS + 1;
+    RHS = LHS;
+  }
+
+#pragma acc atomic capture
+  {
+  // expected-error@-2{{statement associated with OpenACC 'atomic capture' directive is invalid}}
+  // expected-note@+1{{left hand side of assignment operation('LHS') must match one side of the sub-operation on the right hand side('RHS' and '1')}}
+    LHS = RHS - 1;
+    RHS = LHS;
+  }
+#pragma acc atomic capture
+  {
+    LHS = LHS * 1;
+    // expected-error@-3{{statement associated with OpenACC 'atomic capture' directive is invalid}}
+    // expected-note@+1{{variable on right hand side of assignment('RHS') must match variable used on left hand side of assignment('LHS') from the first statement}}
+    RHS = RHS;
+  }
+#pragma acc atomic capture
+  {
+    LHS = LHS / 1;
+  // expected-error@-3{{statement associated with OpenACC 'atomic capture' directive is invalid}}
+  // expected-note@+1{{expected assignment expression}}
+    RHS += LHS;
+  }
+
+  // { x = expr binop x; v = x; }
+#pragma acc atomic capture
+  {
+    LHS = 1 ^ LHS;
+    RHS = LHS;
+  }
+
+#pragma acc atomic capture
+  {
+    // expected-error@-2{{statement associated with OpenACC 'atomic capture' directive is invalid}}
+    // expected-note@+1{{left hand side of assignment operation('LHS') must match one side of the sub-operation on the right hand side('1' and 'RHS')}}
+    LHS = 1 & RHS;
+    RHS = LHS;
+  }
+#pragma acc atomic capture
+  {
+    LHS = LHS | 1;
+    // expected-error@-3{{statement associated with OpenACC 'atomic capture' directive is invalid}}
+    // expected-note@+1{{variable on right hand side of assignment('RHS') must match variable used on left hand side of assignment('LHS') from the first statement}}
+    RHS = RHS;
+  }
+#pragma acc atomic capture
+  {
+    LHS = LHS << 1;
+    // expected-error@-3{{statement associated with OpenACC 'atomic capture' directive is invalid}}
+    // expected-note@+1{{expected assignment expression}}
+    RHS += LHS;
+  }
+
+  // { v = x; x binop = expr; }
+#pragma acc atomic capture
+  {
+    LHS = RHS;
+    RHS += 1;
+  }
+
+  // { v = x; x = x binop expr; }
+#pragma acc atomic capture
+  {
+    LHS = RHS;
+    RHS = RHS / 1;
+  }
+#pragma acc atomic capture
+  {
+    LHS = RHS;
+    // expected-error@-3{{statement associated with OpenACC 'atomic capture' directive is invalid}}
+    // expected-note@+1{{left hand side of assignment operation('RHS') must match one side of the sub-operation on the right hand side('LHS' and '1')}}
+    RHS = LHS ^ 1;
+  }
+#pragma acc atomic capture
+  {
+    LHS = RHS;
+    // expected-error@-3{{statement associated with OpenACC 'atomic capture' directive is invalid}}
+    // expected-note@+1{{left hand side of assignment operation('LHS') must match one side of the sub-operation on the right hand side('RHS' and '1')}}
+    LHS = RHS << 1;
+  }
+  // { v = x; x = expr binop x; }
+#pragma acc atomic capture
+  {
+    LHS = RHS;
+    RHS = 1 / RHS;
+  }
+#pragma acc atomic capture
+  {
+    LHS = RHS;
+    // expected-error@-3{{statement associated with OpenACC 'atomic capture' directive is invalid}}
+    // expected-note@+1{{left hand side of assignment operation('RHS') must match one side of the sub-operation on the right hand side('1' and 'LHS')}}
+    RHS = 1 ^ LHS;
+  }
+#pragma acc atomic capture
+  {
+    LHS = RHS;
+    // expected-error@-3{{statement associated with OpenACC 'atomic capture' directive is invalid}}
+    // expected-note@+1{{left hand side of assignment operation('LHS') must match one side of the sub-operation on the right hand side('1' and 'RHS')}}
+    LHS = 1 << RHS;
+  }
+
+  // { v = x; x = expr; }
+#pragma acc atomic capture
+  {
+    LHS = RHS;
+    RHS = 1;
+  }
+#pragma acc atomic capture
+  {
+    LHS = RHS;
+    // expected-error@-3{{statement associated with OpenACC 'atomic capture' directive is invalid}}
+    // expected-note@+1{{variable on left hand side of assignment('LHS') must match variable used on right hand side of assignment('RHS') from the first statement}}
+    LHS = 1;
+  }
+
+  // { v = x; x++; }
+  // { v = x; ++x; }
+  // { v = x; x--; }
+  // { v = x; --x; }
+#pragma acc atomic capture
+  {
+    LHS = RHS;
+    RHS++;
+  }
+#pragma acc atomic capture
+  {
+    LHS = RHS;
+    RHS--;
+  }
+#pragma acc atomic capture
+  {
+    LHS = RHS;
+    ++RHS;
+  }
+#pragma acc atomic capture
+  {
+    LHS = RHS;
+    --RHS;
+  }
+#pragma acc atomic capture
+  {
+    LHS = RHS;
+    // expected-error@-3{{statement associated with OpenACC 'atomic capture' directive is invalid}}
+    // expected-note@+1{{unary operator not supported, only increment and decrement operations permitted}}
+    -RHS;
+  }
+#pragma acc atomic capture
+  {
+    LHS = RHS;
+    // expected-error@-3{{statement associated with OpenACC 'atomic capture' directive is invalid}}
+    // expected-note@+1{{variable in unary expression('LHS') must match variable used on right hand side of assignment('RHS') from the first statement}}
+    LHS++;
+  }
+}
+
+template<typename T>
+void AtomicCaptureTemplateCompound2(T LHS, T RHS) {
+
+  // expected-error@+2{{statement associated with OpenACC 'atomic capture' directive is invalid}}
+  // expected-note@+2{{expected assignment, compound assignment, increment, or decrement expression}}
+#pragma acc atomic capture
+  {
+  }
+
+  // expected-error@+2{{statement associated with OpenACC 'atomic capture' directive is invalid}}
+  // expected-note@+4{{expected assignment, compound assignment, increment, or decrement expression}}
+#pragma acc atomic capture
+  {
+    LHS = RHS;
+  }
+
+  // expected-error@+2{{statement associated with OpenACC 'atomic capture' directive is invalid}}
+  // expected-note@+3{{'atomic capture' with a compound statement only supports two statements}}
+#pragma acc atomic capture
+  {
+    LHS = RHS; RHS += 1; LHS=RHS;
+  }
+
+
+#pragma acc atomic capture
+  {
+    // expected-error@-2{{statement associated with OpenACC 'atomic capture' directive is invalid}}
+    // expected-note@+1{{operand to increment expression must be of scalar type (was 'Struct')}}
+    LHS++;
+    RHS = LHS;
+  }
+
+#pragma acc atomic capture
+  {
+    // expected-error@-2{{statement associated with OpenACC 'atomic capture' directive is invalid}}
+    // expected-note@+1{{operand to increment expression must be of scalar type (was 'Struct')}}
+    ++LHS;
+    RHS = LHS;
+  }
+
+#pragma acc atomic capture
+  {
+    // expected-error@-2{{statement associated with OpenACC 'atomic capture' directive is invalid}}
+    // expected-note@+1{{operand to decrement expression must be of scalar type (was 'Struct')}}
+    --LHS;
+    RHS = LHS;
+  }
+
+
+#pragma acc atomic capture
+  {
+    LHS--;
+    // expected-error@-3{{statement associated with OpenACC 'atomic capture' directive is invalid}}
+    // expected-note@+1{{variable on right hand side of assignment('RHS') must match variable used in unary expression('LHS') from the first statement}}
+    LHS = RHS;
+  }
+
+#pragma acc atomic capture
+  {
+    // expected-error@-2{{statement associated with OpenACC 'atomic capture' directive is invalid}}
+    // expected-note@+1{{unary operator not supported, only increment and decrement operations permitted}}
+    -LHS;
+    RHS = LHS;
+  }
+
+#pragma acc atomic capture
+  {
+    --LHS;
+    // expected-error@-3{{statement associated with OpenACC 'atomic capture' directive is invalid}}
+    // expected-note@+1{{expected assignment expression}}
+    RHS += LHS;
+  }
+
+  // { x binop = expr; v = x; }
+#pragma acc atomic capture
+  {
+    // expected-error@-2{{statement associated with OpenACC 'atomic capture' directive is invalid}}
+    // expected-note@+1{{left operand to compound assignment expression must be of scalar type (was 'Struct')}}
+    LHS += 1;
+    RHS = LHS;
+  }
+#pragma acc atomic capture
+  {
+    LHS *= 1;
+    // expected-error@-3{{statement associated with OpenACC 'atomic capture' directive is invalid}}
+    // expected-note@+1{{variable on right hand side of assignment('RHS') must match variable used on left hand side of compound assignment('LHS') from the first statement}}
+    LHS = RHS;
+  }
+#pragma acc atomic capture
+  {
+    LHS /= 1;
+    // expected-error@-3{{statement associated with OpenACC 'atomic capture' directive is invalid}}
+    // expected-note@+1{{expected assignment expression}}
+    RHS += LHS;
+  }
+
+  // { x = x binop expr; v = x; }
+#pragma acc atomic capture
+  {
+    // expected-error@-2{{statement associated with OpenACC 'atomic capture' directive is invalid}}
+    // expected-note@+1{{left operand to assignment expression must be of scalar type (was 'Struct')}}
+    LHS = LHS + 1;
+    RHS = LHS;
+  }
+
+#pragma acc atomic capture
+  {
+  // expected-error@-2{{statement associated with OpenACC 'atomic capture' directive is invalid}}
+  // expected-note@+1{{left hand side of assignment operation('LHS') must match one side of the sub-operation on the right hand side('RHS' and '1')}}
+    LHS = RHS - 1;
+    RHS = LHS;
+  }
+#pragma acc atomic capture
+  {
+    LHS = LHS * 1;
+    // expected-error@-3{{statement associated with OpenACC 'atomic capture' directive is invalid}}
+    // expected-note@+1{{variable on right hand side of assignment('RHS') must match variable used on left hand side of assignment('LHS') from the first statement}}
+    RHS = RHS;
+  }
+#pragma acc atomic capture
+  {
+    LHS = LHS / 1;
+    // expected-error@-3{{statement associated with OpenACC 'atomic capture' directive is invalid}}
+    // expected-note@+1{{expected assignment expression}}
+    RHS += LHS;
+  }
+
+  // { x = expr binop x; v = x; }
+#pragma acc atomic capture
+  {
+    // expected-error@-2{{statement associated with OpenACC 'atomic capture' directive is invalid}}
+    // expected-note@+1{{left operand to assignment expression must be of scalar type (was 'Struct')}}
+    LHS = 1 ^ LHS;
+    RHS = LHS;
+  }
+
+#pragma acc atomic capture
+  {
+  // expected-error@-2{{statement associated with OpenACC 'atomic capture' directive is invalid}}
+  // expected-note@+1{{left hand side of assignment operation('LHS') must match one side of the sub-operation on the right hand side('1' and 'RHS')}}
+    LHS = 1 & RHS;
+    RHS = LHS;
+  }
+#pragma acc atomic capture
+  {
+    LHS = LHS | 1;
+    // expected-error@-3{{statement associated with OpenACC 'atomic capture' directive is invalid}}
+    // expected-note@+1{{variable on right hand side of assignment('RHS') must match variable used on left hand side of assignment('LHS') from the first statement}}
+    RHS = RHS;
+  }
+#pragma acc atomic capture
+  {
+    LHS = LHS << 1;
+    // expected-error@-3{{statement associated with OpenACC 'atomic capture' directive is invalid}}
+    // expected-note@+1{{expected assignment expression}}
+    RHS += LHS;
+  }
+
+  // { v = x; x binop = expr; }
+#pragma acc atomic capture
+  {
+    // expected-error@-2{{statement associated with OpenACC 'atomic capture' directive is invalid}}
+    // expected-note@+1{{left operand to assignment expression must be of scalar type (was 'Struct')}}
+    LHS = RHS;
+    RHS += 1;
+  }
+
+  // { v = x; x = x binop expr; }
+#pragma acc atomic capture
+  {
+    // expected-error@-2{{statement associated with OpenACC 'atomic capture' directive is invalid}}
+    // expected-note@+1{{left operand to assignment expression must be of scalar type (was 'Struct')}}
+    LHS = RHS;
+    RHS = RHS / 1;
+  }
+#pragma acc atomic capture
+  {
+    LHS = RHS;
+    // expected-error@-3{{statement associated with OpenACC 'atomic capture' directive is invalid}}
+    // expected-note@+1{{left hand side of assignment operation('RHS') must match one side of the sub-operation on the right hand side('LHS' and '1')}}
+    RHS = LHS ^ 1;
+  }
+#pragma acc atomic capture
+  {
+    LHS = RHS;
+    // expected-error@-3{{statement associated with OpenACC 'atomic capture' directive is invalid}}
+    // expected-note@+1{{left hand side of assignment operation('LHS') must match one side of the sub-operation on the right hand side('RHS' and '1')}}
+    LHS = RHS << 1;
+  }
+  // { v = x; x = expr binop x; }
+#pragma acc atomic capture
+  {
+    // expected-error@-2{{statement associated with OpenACC 'atomic capture' directive is invalid}}
+    // expected-note@+1{{left operand to assignment expression must be of scalar type (was 'Struct')}}
+    LHS = RHS;
+    RHS = 1 / RHS;
+  }
+#pragma acc atomic capture
+  {
+    LHS = RHS;
+    // expected-error@-3{{statement associated with OpenACC 'atomic capture' directive is invalid}}
+    // expected-note@+1{{left hand side of assignment operation('RHS') must match one side of the sub-operation on the right hand side('1' and 'LHS')}}
+    RHS = 1 ^ LHS;
+  }
+#pragma acc atomic capture
+  {
+    LHS = RHS;
+    // expected-error@-3{{statement associated with OpenACC 'atomic capture' directive is invalid}}
+    // expected-note@+1{{left hand side of assignment operation('LHS') must match one side of the sub-operation on the right hand side('1' and 'RHS')}}
+    LHS = 1 << RHS;
+  }
+
+  // { v = x; x = expr; }
+#pragma acc atomic capture
+  {
+    // expected-error@-2{{statement associated with OpenACC 'atomic capture' directive is invalid}}
+    // expected-note@+1{{left operand to assignment expression must be of scalar type (was 'Struct')}}
+    LHS = RHS;
+    RHS = 1;
+  }
+#pragma acc atomic capture
+  {
+    LHS = RHS;
+    // expected-error@-3{{statement associated with OpenACC 'atomic capture' directive is invalid}}
+    // expected-note@+1{{variable on left hand side of assignment('LHS') must match variable used on right hand side of assignment('RHS') from the first statement}}
+    LHS = 1;
+  }
+
+  // { v = x; x++; }
+  // { v = x; ++x; }
+  // { v = x; x--; }
+  // { v = x; --x; }
+#pragma acc atomic capture
+  {
+    // expected-error@-2{{statement associated with OpenACC 'atomic capture' directive is invalid}}
+    // expected-note@+1{{left operand to assignment expression must be of scalar type (was 'Struct')}}
+    LHS = RHS;
+    RHS++;
+  }
+#pragma acc atomic capture
+  {
+    // expected-error@-2{{statement associated with OpenACC 'atomic capture' directive is invalid}}
+    // expected-note@+1{{left operand to assignment expression must be of scalar type (was 'Struct')}}
+    LHS = RHS;
+    RHS--;
+  }
+#pragma acc atomic capture
+  {
+    // expected-error@-2{{statement associated with OpenACC 'atomic capture' directive is invalid}}
+    // expected-note@+1{{left operand to assignment expression must be of scalar type (was 'Struct')}}
+    LHS = RHS;
+    ++RHS;
+  }
+#pragma acc atomic capture
+  {
+    // expected-error@-2{{statement associated with OpenACC 'atomic capture' directive is invalid}}
+    // expected-note@+1{{left operand to assignment expression must be of scalar type (was 'Struct')}}
+    LHS = RHS;
+    --RHS;
+  }
+#pragma acc atomic capture
+  {
+    LHS = RHS;
+    // expected-error@-3{{statement associated with OpenACC 'atomic capture' directive is invalid}}
+    // expected-note@+1{{unary operator not supported, only increment and decrement operations permitted}}
+    -RHS;
+  }
+#pragma acc atomic capture
+  {
+    LHS = RHS;
+    // expected-error@-3{{statement associated with OpenACC 'atomic capture' directive is invalid}}
+    // expected-note@+1{{variable in unary expression('LHS') must match variable used on right hand side of assignment('RHS') from the first statement}}
+    LHS++;
+  }
+}
+void AtomicCaptureCompound(int LHS, int RHS) {
+  AtomicCaptureTemplateCompound(1, 2); 
+  AtomicCaptureTemplateCompound2(S1, S2); //expected-note{{in instantiation of function template specialization}}
+
+  // expected-error@+2{{statement associated with OpenACC 'atomic capture' directive is invalid}}
+  // expected-note@+2{{expected assignment, compound assignment, increment, or decrement expression}}
+#pragma acc atomic capture
+  {
+  }
+
+  // expected-error@+2{{statement associated with OpenACC 'atomic capture' directive is invalid}}
+  // expected-note@+4{{expected assignment, compound assignment, increment, or decrement expression}}
+#pragma acc atomic capture
+  {
+    LHS = RHS;
+  }
+
+  // expected-error@+2{{statement associated with OpenACC 'atomic capture' directive is invalid}}
+  // expected-note@+3{{'atomic capture' with a compound statement only supports two statements}}
+#pragma acc atomic capture
+  {
+    LHS = RHS; RHS += 1; LHS=RHS;
+  }
+
+
+#pragma acc atomic capture
+  {
+    LHS++;
+    RHS = LHS;
+  }
+#pragma acc atomic capture
+  {
+    // expected-error@-2{{statement associated with OpenACC 'atomic capture' directive is invalid}}
+    // expected-note@+1{{operand to increment expression must be of scalar type (was 'Struct')}}
+    S1++;
+    S2= S1;
+  }
+
+#pragma acc atomic capture
+  {
+    ++LHS;
+    RHS = LHS;
+  }
+
+#pragma acc atomic capture
+  {
+    --LHS;
+    RHS = LHS;
+  }
+
+
+#pragma acc atomic capture
+  {
+    LHS--;
+    // expected-error@-3{{statement associated with OpenACC 'atomic capture' directive is invalid}}
+    // expected-note@+1{{variable on right hand side of assignment('RHS') must match variable used in unary expression('LHS') from the first statement}}
+    LHS = RHS;
+  }
+
+#pragma acc atomic capture
+  {
+    // expected-error@-2{{statement associated with OpenACC 'atomic capture' directive is invalid}}
+    // expected-note@+1{{unary operator not supported, only increment and decrement operations permitted}}
+    -LHS;
+    RHS = LHS;
+  }
+
+#pragma acc atomic capture
+  {
+    --LHS;
+  // expected-error@-3{{statement associated with OpenACC 'atomic capture' directive is invalid}}
+  // expected-note@+1{{expected assignment expression}}
+    RHS += LHS;
+  }
+
+  // { x binop = expr; v = x; }
+#pragma acc atomic capture
+  {
+    LHS += 1;
+    RHS = LHS;
+  }
+#pragma acc atomic capture
+  {
+    // expected-error@-2{{statement associated with OpenACC 'atomic capture' directive is invalid}}
+    // expected-note@+1{{left operand to compound assignment expression must be of scalar type (was 'Struct')}}
+    S1 += 1;
+    S2= S1;
+  }
+#pragma acc atomic capture
+  {
+    LHS *= 1;
+    // expected-error@-3{{statement associated with OpenACC 'atomic capture' directive is invalid}}
+    // expected-note@+1{{variable on right hand side of assignment('RHS') must match variable used on left hand side of compound assignment('LHS') from the first statement}}
+    LHS = RHS;
+  }
+#pragma acc atomic capture
+  {
+    LHS /= 1;
+  // expected-error@-3{{statement associated with OpenACC 'atomic capture' directive is invalid}}
+  // expected-note@+1{{expected assignment expression}}
+    RHS += LHS;
+  }
+
+  // { x = x binop expr; v = x; }
+#pragma acc atomic capture
+  {
+    LHS = LHS + 1;
+    RHS = LHS;
+  }
+#pragma acc atomic capture
+  {
+    // expected-error@-2{{statement associated with OpenACC 'atomic capture' directive is invalid}}
+    // expected-note@+1{{left operand to assignment expression must be of scalar type (was 'Struct')}}
+    S1 = S1 + 1;
+    S2= S1;
+  }
+
+#pragma acc atomic capture
+  {
+    // expected-error@-2{{statement associated with OpenACC 'atomic capture' directive is invalid}}
+    // expected-note@+1{{left hand side of assignment operation('LHS') must match one side of the sub-operation on the right hand side('RHS' and '1')}}
+    LHS = RHS - 1;
+    RHS = LHS;
+  }
+#pragma acc atomic capture
+  {
+    LHS = LHS * 1;
+    // expected-error@-3{{statement associated with OpenACC 'atomic capture' directive is invalid}}
+    // expected-note@+1{{variable on right hand side of assignment('RHS') must match variable used on left hand side of assignment('LHS') from the first statement}}
+    RHS = RHS;
+  }
+#pragma acc atomic capture
+  {
+    LHS = LHS / 1;
+  // expected-error@-3{{statement associated with OpenACC 'atomic capture' directive is invalid}}
+  // expected-note@+1{{expected assignment expression}}
+    RHS += LHS;
+  }
+
+  // { x = expr binop x; v = x; }
+#pragma acc atomic capture
+  {
+    LHS = 1 ^ LHS;
+    RHS = LHS;
+  }
+#pragma acc atomic capture
+  {
+    // expected-error@-2{{statement associated with OpenACC 'atomic capture' directive is invalid}}
+    // expected-note@+1{{left operand to assignment expression must be of scalar type (was 'Struct')}}
+    S1 = 1 ^ S1;
+    S2 = S1;
+  }
+
+#pragma acc atomic capture
+  {
+    // expected-error@-2{{statement associated with OpenACC 'atomic capture' directive is invalid}}
+    // expected-note@+1{{left hand side of assignment operation('LHS') must match one side of the sub-operation on the right hand side('1' and 'RHS')}}
+    LHS = 1 & RHS;
+    RHS = LHS;
+  }
+#pragma acc atomic capture
+  {
+    LHS = LHS | 1;
+    // expected-error@-3{{statement associated with OpenACC 'atomic capture' directive is invalid}}
+    // expected-note@+1{{variable on right hand side of assignment('RHS') must match variable used on left hand side of assignment('LHS') from the first statement}}
+    RHS = RHS;
+  }
+#pragma acc atomic capture
+  {
+    LHS = LHS << 1;
+  // expected-error@-3{{statement associated with OpenACC 'atomic capture' directive is invalid}}
+  // expected-note@+1{{expected assignment expression}}
+    RHS += LHS;
+  }
+
+  // { v = x; x binop = expr; }
+#pragma acc atomic capture
+  {
+    LHS = RHS;
+    RHS += 1;
+  }
+
+#pragma acc atomic capture
+  {
+    // expected-error@-2{{statement associated with OpenACC 'atomic capture' directive is invalid}}
+    // expected-note@+1{{left operand to assignment expression must be of scalar type (was 'Struct')}}
+    S1 = S2;
+    S2 += 1;
+  }
+
+  // { v = x; x = x binop expr; }
+#pragma acc atomic capture
+  {
+    LHS = RHS;
+    RHS = RHS / 1;
+  }
+#pragma acc atomic capture
+  {
+    LHS = RHS;
+    // expected-error@-3{{statement associated with OpenACC 'atomic capture' directive is invalid}}
+    // expected-note@+1{{left hand side of assignment operation('RHS') must match one side of the sub-operation on the right hand side('LHS' and '1')}}
+    RHS = LHS ^ 1;
+  }
+#pragma acc atomic capture
+  {
+    LHS = RHS;
+    // expected-error@-3{{statement associated with OpenACC 'atomic capture' directive is invalid}}
+    // expected-note@+1{{left hand side of assignment operation('LHS') must match one side of the sub-operation on the right hand side('RHS' and '1')}}
+    LHS = RHS << 1;
+  }
+  // { v = x; x = expr binop x; }
+#pragma acc atomic capture
+  {
+    LHS = RHS;
+    RHS = 1 / RHS;
+  }
+#pragma acc atomic capture
+  {
+    LHS = RHS;
+    // expected-error@-3{{statement associated with OpenACC 'atomic capture' directive is invalid}}
+    // expected-note@+1{{left hand side of assignment operation('RHS') must match one side of the sub-operation on the right hand side('1' and 'LHS')}}
+    RHS = 1 ^ LHS;
+  }
+#pragma acc atomic capture
+  {
+    LHS = RHS;
+    // expected-error@-3{{statement associated with OpenACC 'atomic capture' directive is invalid}}
+    // expected-note@+1{{left hand side of assignment operation('LHS') must match one side of the sub-operation on the right hand side('1' and 'RHS')}}
+    LHS = 1 << RHS;
+  }
+
+  // { v = x; x = expr; }
+#pragma acc atomic capture
+  {
+    LHS = RHS;
+    RHS = 1;
+  }
+#pragma acc atomic capture
+  {
+    LHS = RHS;
+    // expected-error@-3{{statement associated with OpenACC 'atomic capture' directive is invalid}}
+    // expected-note@+1{{variable on left hand side of assignment('LHS') must match variable used on right hand side of assignment('RHS') from the first statement}}
+    LHS = 1;
+  }
+
+  // { v = x; x++; }
+  // { v = x; ++x; }
+  // { v = x; x--; }
+  // { v = x; --x; }
+#pragma acc atomic capture
+  {
+    LHS = RHS;
+    RHS++;
+  }
+#pragma acc atomic capture
+  {
+    LHS = RHS;
+    RHS--;
+  }
+#pragma acc atomic capture
+  {
+    LHS = RHS;
+    ++RHS;
+  }
+#pragma acc atomic capture
+  {
+    LHS = RHS;
+    --RHS;
+  }
+#pragma acc atomic capture
+  {
+    // expected-error@-2{{statement associated with OpenACC 'atomic capture' directive is invalid}}
+    // expected-note@+1{{left operand to assignment expression must be of scalar type (was 'Struct')}}
+    S1= S2;
+    --S2;
+  }
+#pragma acc atomic capture
+  {
+    LHS = RHS;
+    // expected-error@-3{{statement associated with OpenACC 'atomic capture' directive is invalid}}
+    // expected-note@+1{{unary operator not supported, only increment and decrement operations permitted}}
+    -RHS;
+  }
+#pragma acc atomic capture
+  {
+    LHS = RHS;
+    // expected-error@-3{{statement associated with OpenACC 'atomic capture' directive is invalid}}
+    // expected-note@+1{{variable in unary expression('LHS') must match variable used on right hand side of assignment('RHS') from the first statement}}
+    LHS++;
+  }
+}
diff --git a/clang/test/SemaTemplate/GH55509.cpp b/clang/test/SemaTemplate/GH55509.cpp
new file mode 100644
index 0000000000000..773a84305a0cd
--- /dev/null
+++ b/clang/test/SemaTemplate/GH55509.cpp
@@ -0,0 +1,112 @@
+// RUN: %clang_cc1 -fsyntax-only -verify -std=c++26 %s
+
+namespace t1 {
+  template<int N> struct A {
+    template<class C> friend auto cica(const A<N-1>&, C) {
+      return N;
+    }
+  };
+
+  template<> struct A<0> {
+    template<class C> friend auto cica(const A<0>&, C);
+    // expected-note@-1 {{declared here}}
+  };
+
+  void test() {
+    cica(A<0>{}, 0);
+    // expected-error@-1 {{function 'cica<int>' with deduced return type cannot be used before it is defined}}
+
+    (void)A<1>{};
+    cica(A<0>{}, 0);
+  }
+} // namespace t1
+namespace t2 {
+  template<int N> struct A {
+    template<class C> friend auto cica(const A<N-1>&, C) {
+      return N;
+    }
+  };
+
+  template<> struct A<0> {
+    template<class C> friend auto cica(const A<0>&, C);
+  };
+
+  template <int N, class = decltype(cica(A<N>{}, nullptr))>
+  void MakeCica();
+  // expected-note@-1 {{candidate function}}
+
+  template <int N> void MakeCica(A<N+1> = {});
+  // expected-note@-1 {{candidate function}}
+
+  void test() {
+    MakeCica<0>();
+
+    MakeCica<0>();
+    // expected-error@-1 {{call to 'MakeCica' is ambiguous}}
+  }
+} // namespace t2
+namespace t3 {
+  template<int N> struct A {
+    template<class C> friend auto cica(const A<N-1>&, C) {
+      return N-1;
+    }
+  };
+
+  template<> struct A<0> {
+    template<class C> friend auto cica(const A<0>&, C);
+  };
+
+  template <int N, class AT, class = decltype(cica(AT{}, nullptr))>
+  static constexpr bool MakeCica(int);
+
+  template <int N, class AT>
+  static constexpr bool MakeCica(short, A<N+1> = {});
+
+  template <int N, class AT = A<N>, class Val = decltype(MakeCica<N, AT>(0))>
+  static constexpr bool has_cica = Val{};
+
+  constexpr bool cica2 = has_cica<0> || has_cica<0>;
+} // namespace t3
+namespace t4 {
+  template<int N> struct A {
+    template<class C> friend auto cica(const A<N-1>&, C);
+  };
+
+  template<> struct A<0> {
+    template<class C> friend auto cica(const A<0>&, C) {
+      C a;
+    }
+  };
+
+  template struct A<1>;
+
+  void test() {
+    cica(A<0>{}, 0);
+  }
+} // namespace t4
+namespace regression1 {
+  template <class> class A;
+
+  template <class T> [[gnu::abi_tag("TAG")]] void foo(A<T>);
+
+  template <class> struct A {
+    friend void foo <>(A);
+  };
+
+  template struct A<int>;
+
+  template <class T> [[gnu::abi_tag("TAG")]] void foo(A<T>) {}
+
+  template void foo<int>(A<int>);
+} // namespace regression1
+namespace regression2 {
+  template <class> struct A {
+    template <class T> static void f() {
+      A<int>::f<T>();
+    }
+  };
+  template <> template <class T> void A<int>::f() {
+    static_assert(__is_same(T, long));
+  }
+  template void A<void>::f<long>();
+} // namespace regression2
diff --git a/clang/test/SemaTemplate/address_space-dependent.cpp b/clang/test/SemaTemplate/address_space-dependent.cpp
index 2ca9b8007ab41..eb8dbc69a945e 100644
--- a/clang/test/SemaTemplate/address_space-dependent.cpp
+++ b/clang/test/SemaTemplate/address_space-dependent.cpp
@@ -43,7 +43,7 @@ void neg() {
 
 template <long int I>
 void tooBig() {
-  __attribute__((address_space(I))) int *bounds; // expected-error {{address space is larger than the maximum supported (8388586)}}
+  __attribute__((address_space(I))) int *bounds; // expected-error {{address space is larger than the maximum supported (8388585)}}
 }
 
 template <long int I>
diff --git a/clang/test/SemaTemplate/aggregate-deduction-candidate.cpp b/clang/test/SemaTemplate/aggregate-deduction-candidate.cpp
index 49afb6b860620..0854ac9178b4f 100644
--- a/clang/test/SemaTemplate/aggregate-deduction-candidate.cpp
+++ b/clang/test/SemaTemplate/aggregate-deduction-candidate.cpp
@@ -274,10 +274,10 @@ namespace TrailingPack {
   // CHECK: |-TemplateArgument pack
   // CHECK: | |-TemplateArgument type 'TrailingPack::(lambda at {{.*}})'
   // CHECK: | | `-RecordType {{.*}} 'TrailingPack::(lambda at {{.*}})'
-  // CHECK: | |   `-CXXRecord {{.*}} ''
+  // CHECK: | |   `-CXXRecord {{.*}} <line:262:5>
   // CHECK: | `-TemplateArgument type 'TrailingPack::(lambda at {{.*}})'
   // CHECK: |   `-RecordType {{.*}} 'TrailingPack::(lambda at {{.*}})'
-  // CHECK: |     `-CXXRecord {{.*}} ''
+  // CHECK: |     `-CXXRecord {{.*}} <line:263:5>
   // CHECK: |-ParmVarDecl {{.*}} 'TrailingPack::(lambda at {{.*}})'
   // CHECK: `-ParmVarDecl {{.*}} 'TrailingPack::(lambda at {{.*}})'
   // CHECK: FunctionProtoType {{.*}} 'auto (T...) -> A<T...>' dependent trailing_return cdecl
diff --git a/clang/test/SemaTemplate/concepts-out-of-line-def.cpp b/clang/test/SemaTemplate/concepts-out-of-line-def.cpp
index 6c1a229a9fdda..5af4ec75cae90 100644
--- a/clang/test/SemaTemplate/concepts-out-of-line-def.cpp
+++ b/clang/test/SemaTemplate/concepts-out-of-line-def.cpp
@@ -722,6 +722,34 @@ template struct d<int, int>;
 
 } // namespace GH115098
 
+namespace GH123441 {
+
+struct buf {
+  constexpr buf(auto&&... initList) requires (sizeof...(initList) <= 8);
+};
+
+constexpr buf::buf(auto&&... initList) requires (sizeof...(initList) <= 8) {}
+
+template <class>
+struct buffer {
+  constexpr buffer(auto&&... initList) requires (sizeof...(initList) <= 8);
+};
+
+template <class T>
+constexpr buffer<T>::buffer(auto&&... initList) requires (sizeof...(initList) <= 8) {}
+
+template <class...>
+struct foo { // expected-note {{foo defined here}}
+  constexpr foo(auto&&... initList)
+    requires (sizeof...(initList) <= 8);
+};
+
+template <class... T>
+constexpr foo<T...>::foo(auto&&... initList) // expected-error {{does not match any declaration}}
+  requires (sizeof...(T) <= 8) {}
+
+} // namespace GH123441
+
 namespace GH114685 {
 
 template <typename T> struct ptr {
diff --git a/clang/test/SemaTemplate/cwg2398.cpp b/clang/test/SemaTemplate/cwg2398.cpp
index 21a1b89ce79b4..51d98d4b3b202 100644
--- a/clang/test/SemaTemplate/cwg2398.cpp
+++ b/clang/test/SemaTemplate/cwg2398.cpp
@@ -1,14 +1,12 @@
-// RUN: %clang_cc1 %s -fsyntax-only -std=c++23                                     -verify=expected,new
-// RUN: %clang_cc1 %s -fsyntax-only -std=c++23 -fno-relaxed-template-template-args -verify=expected,old
+// RUN: %clang_cc1 %s -fsyntax-only -std=c++23 -verify
 
 namespace issue1 {
   template<class T, class U = T> class B {};
   template<template<class> class P, class T> void f(P<T>);
-  // new-note@-1 {{deduced type 'B<[...], (default) int>' of 1st parameter does not match adjusted type 'B<[...], float>' of argument [with P = B, T = int]}}
-  // old-note@-2 2{{template template argument has different template parameters}}
+  // expected-note@-1 {{deduced type 'B<[...], (default) int>' of 1st parameter does not match adjusted type 'B<[...], float>' of argument [with P = B, T = int]}}
 
   void g() {
-    f(B<int>()); // old-error {{no matching function for call}}
+    f(B<int>());
     f(B<int,float>()); // expected-error {{no matching function for call}}
   }
 } // namespace issue1
@@ -116,41 +114,32 @@ namespace gcc_issue {
   template<class T1, class T2> struct A;
 
   template<template<class T1> class TT1, class T2> struct A<TT1<T2>, typename TT1<T2>::type>;
-  // new-note@-1 {{partial specialization matches}}
+  // expected-note@-1 {{partial specialization matches}}
 
   template<template<class T3, class T4> class TT2, class T5, class T6>
   struct A<TT2<T5, T6>, typename TT2<T5, T5>::type>;
-  // new-note@-1 {{partial specialization matches}}
-  // old-note@-2 {{template is declared here}}
+  // expected-note@-1 {{partial specialization matches}}
 
   template <class T7, class T8 = T7> struct B { using type = int; };
 
   template struct A<B<int>, int>;
-  // new-error@-1 {{ambiguous partial specializations}}
-  // old-error@-2 {{explicit instantiation of undefined template}}
+  // expected-error@-1 {{ambiguous partial specializations}}
 } // namespace gcc_issue
 
 namespace ttp_defaults {
   template <template <class T1> class TT1> struct A {};
-  // old-note@-1 2{{previous template template parameter}}
 
   template <template <class T2> class TT2> void f(A<TT2>);
-  // new-note@-1 {{explicit instantiation candidate}}
-  // old-note@-2 {{invalid explicitly-specified argument for template parameter 'TT2'}}
+  // expected-note@-1 {{explicit instantiation candidate}}
 
   // FIXME: The default arguments on the TTP are not available during partial ordering.
   template <template <class T3, class T4 = float> class TT3> void f(A<TT3>) {};
-  // new-note@-1 {{explicit instantiation candidate}}
-  // old-error@-2 {{template template argument has different template parameters}}
-  // old-note@-3 {{too many template parameters}}
+  // expected-note@-1 {{explicit instantiation candidate}}
 
   template <class T5, class T6 = int> struct B;
-  // old-note@-1 {{too many template parameters}}
 
   template void f<B>(A<B>);
-  // new-error@-1 {{partial ordering for explicit instantiation of 'f' is ambiguous}}
-  // old-error@-2 {{template template argument has different template parameters}}
-  // old-error@-3 {{explicit instantiation of 'f' does not refer to a function template}}
+  // expected-error@-1 {{partial ordering for explicit instantiation of 'f' is ambiguous}}
 } // namespace ttp_defaults
 
 namespace ttp_only {
@@ -193,16 +182,16 @@ namespace consistency {
     template<template<class, class> class TT1,
              class T1, class T2, class T3, class T4>
     struct A<TT1<T1, T2>, TT1<T3, T4>, typename nondeduced<TT1<T1, T4>>::type> {};
-    // new-note@-1 {{partial specialization matches}}
+    // expected-note@-1 {{partial specialization matches}}
 
     template<template<class> class UU1,
              template<class> class UU2,
              class U1, class U2>
     struct A<UU1<U1>, UU2<U2>, typename nondeduced<UU1<U1>>::type>;
-    // new-note@-1 {{partial specialization matches}}
+    // expected-note@-1 {{partial specialization matches}}
 
     template struct A<B<int>, B<int>, B<int>>;
-    // new-error@-1 {{ambiguous partial specializations}}
+    // expected-error@-1 {{ambiguous partial specializations}}
   } // namespace t2
   namespace t3 {
     template<class T1, class T2, class T3> struct A;
@@ -210,15 +199,15 @@ namespace consistency {
     template<template<class, class> class TT1,
              class T1, class T2, class T3, class T4>
     struct A<TT1<T1, T2>, TT1<T3, T4>, typename nondeduced<TT1<T1, T2>>::type> {};
-    // new-note@-1 {{partial specialization matches}}
+    // expected-note@-1 {{partial specialization matches}}
 
     template<template<class> class UU1,
              class U1, class U2>
     struct A<UU1<U1>, UU1<U2>, typename nondeduced<UU1<U1>>::type>;
-    // new-note@-1 {{partial specialization matches}}
+    // expected-note@-1 {{partial specialization matches}}
 
     template struct A<B<int>, B<int>, B<int>>;
-    // new-error@-1 {{ambiguous partial specializations}}
+    // expected-error@-1 {{ambiguous partial specializations}}
   } // namespace t3
   namespace t4 {
     template<class T1, class T2, class T3> struct A;
@@ -226,15 +215,15 @@ namespace consistency {
     template<template<class, class> class TT1,
              class T1, class T2, class T3, class T4>
     struct A<TT1<T1, T2>, TT1<T3, T4>, typename nondeduced<TT1<T1, T4>>::type> {};
-    // new-note@-1 {{partial specialization matches}}
+    // expected-note@-1 {{partial specialization matches}}
 
     template<template<class> class UU1,
              class U1, class U2>
     struct A<UU1<U1>, UU1<U2>, typename nondeduced<UU1<U1>>::type>;
-    // new-note@-1 {{partial specialization matches}}
+    // expected-note@-1 {{partial specialization matches}}
 
     template struct A<B<int>, B<int>, B<int>>;
-    // new-error@-1 {{ambiguous partial specializations}}
+    // expected-error@-1 {{ambiguous partial specializations}}
   } // namespace t4
   namespace t5 {
     template<class T1, class T2> struct A;
@@ -242,15 +231,15 @@ namespace consistency {
     template<template<class, class> class TT1,
              class T1, class T2, class T3, class T4>
     struct A<TT1<T1, T2>, TT1<T3, T4>> {};
-    // new-note@-1 {{partial specialization matches}}
+    // expected-note@-1 {{partial specialization matches}}
 
     template<template<class> class UU1,
              class U1, class U2>
     struct A<UU1<U1>, UU1<U2>>;
-    // new-note@-1 {{partial specialization matches}}
+    // expected-note@-1 {{partial specialization matches}}
 
     template struct A<B<int>, B<int>>;
-    // new-error@-1 {{ambiguous partial specializations}}
+    // expected-error@-1 {{ambiguous partial specializations}}
   } // namespace t5
   namespace t6 {
     template<class T1, class T2> struct A;
@@ -258,15 +247,15 @@ namespace consistency {
     template<template<class, class> class TT1,
              class T1, class T2, class T3>
     struct A<TT1<T1, T2>, TT1<T1, T3>> {};
-    // new-note@-1 {{partial specialization matches}}
+    // expected-note@-1 {{partial specialization matches}}
 
     template<template<class> class UU1,
              class U1, class U2>
     struct A<UU1<U1>, UU1<U2>>;
-    // new-note@-1 {{partial specialization matches}}
+    // expected-note@-1 {{partial specialization matches}}
 
     template struct A<B<int>, B<int>>;
-    // new-error@-1 {{ambiguous partial specializations}}
+    // expected-error@-1 {{ambiguous partial specializations}}
   } // namespace t6
 } // namespace consistency
 
@@ -275,8 +264,7 @@ namespace classes {
     template<class T, class U> struct A {};
 
     template<template<class> class TT> auto f(TT<int> a) { return a; }
-    // old-note@-1 2{{template template argument has different template parameters}}
-    // new-note@-2 2{{substitution failure: too few template arguments}}
+    // expected-note@-1 2{{substitution failure: too few template arguments}}
 
     A<int, float> v1;
     A<int, double> v2;
@@ -292,8 +280,7 @@ namespace classes {
       static constexpr auto val = E1;
     };
     template <template <class T3> class TT> void f(TT<int> v) {
-      // old-note@-1 {{template template argument has different template parameters}}
-      // new-note@-2 {{substitution failure: too few template arguments}}
+      // expected-note@-1 {{substitution failure: too few template arguments}}
       static_assert(v.val == 3);
     };
     void test() {
@@ -307,8 +294,7 @@ namespace classes {
     };
 
     template <template <class T3> class TT> void f(TT<int> v) {
-      // old-note@-1 {{template template argument has different template parameters}}
-      // new-note@-2 {{deduced type 'A<[...], (no argument), (no argument), (no argument)>' of 1st parameter does not match adjusted type 'A<[...], void, void, void>' of argument [with TT = A]}}
+      // expected-note@-1 {{deduced type 'A<[...], (no argument), (no argument), (no argument)>' of 1st parameter does not match adjusted type 'A<[...], void, void, void>' of argument [with TT = A]}}
       static_assert(v.val == 3);
     };
     void test() {
@@ -327,8 +313,7 @@ namespace classes {
     }
 
     template <template <class T2, int V3> class TT2> auto g(TT2<double, 1>) {
-      // new-note@-1 {{too few template arguments for class template 'A'}}
-      // old-note@-2 {{template template argument has different template parameters}}
+      // expected-note@-1 {{too few template arguments for class template 'A'}}
       return f(TT2<int, 2>());
     }
 
@@ -346,13 +331,11 @@ namespace classes {
     };
 
     template <template <class> class TT> TT<float> f(TT<int>);
-    // new-note@-1  {{deduced type 'A<[...], (default) int *>' of 1st parameter does not match adjusted type 'A<[...], double *>' of argument [with TT = A]}}
-    // old-note@-2 2{{template template argument has different template parameters}}
+    // expected-note@-1  {{deduced type 'A<[...], (default) int *>' of 1st parameter does not match adjusted type 'A<[...], double *>' of argument [with TT = A]}}
 
-    using X = int*; // new-note {{previous definition is here}}
+    using X = int*; // expected-note {{previous definition is here}}
     using X = decltype(f(A<int>()))::type;
-    // new-error@-1 {{different types ('decltype(f(A<int>()))::type' (aka 'float *') vs 'int *')}}
-    // old-error@-2 {{no matching function for call}}
+    // expected-error@-1 {{different types ('decltype(f(A<int>()))::type' (aka 'float *') vs 'int *')}}
 
     using Y = double*;
     using Y = decltype(f(A<int, double*>()))::type;
@@ -362,56 +345,40 @@ namespace classes {
 
 namespace packs {
   namespace t1 {
-    // FIXME: This should be rejected
     template<template<int, int...> class> struct A {};
-    // old-note@-1 {{previous non-type template parameter with type 'int' is here}}
+    // expected-error@-1 {{non-type parameter of template template parameter cannot be narrowed from type 'int' to 'char'}}
+    // expected-note@-2 {{previous template template parameter is here}}
 
     template<char> struct B;
-    // old-note@-1 {{template non-type parameter has a different type 'char' in template argument}}
-
     template struct A<B>;
-    // old-error@-1 {{has different template parameters}}
+    // expected-note@-1 {{has different template parameters}}
   } // namespace t1
   namespace t2 {
     template<template<char, int...> class> struct A {};
-    // old-note@-1 {{previous non-type template parameter with type 'char' is here}}
-
     template<int> struct B;
-    // old-note@-1 {{template non-type parameter has a different type 'int' in template argument}}
-
     template struct A<B>;
-    // old-error@-1 {{has different template parameters}}
   } // namespace t2
   namespace t3 {
-    // FIXME: This should be rejected
     template<template<int...> class> struct A {};
-    // old-note@-1 {{previous non-type template parameter with type 'int' is here}}
+    // expected-error@-1 {{non-type parameter of template template parameter cannot be narrowed from type 'int' to 'char'}}
+    // expected-note@-2 {{previous template template parameter is here}}
 
     template<char> struct B;
-    // old-note@-1 {{template non-type parameter has a different type 'char' in template argument}}
-
     template struct A<B>;
-    // old-error@-1 {{has different template parameters}}
+    // expected-note@-1 {{has different template parameters}}
   } // namespace t3
   namespace t4 {
     template<template<char...> class> struct A {};
-    // old-note@-1 {{previous non-type template parameter with type 'char' is here}}
-
     template<int> struct B;
-    // old-note@-1 {{template non-type parameter has a different type 'int' in template argument}}
-
     template struct A<B>;
-    // old-error@-1 {{has different template parameters}}
   } // namespace t4
 } // namespace packs
 
 namespace fun_tmpl_call {
   namespace match_func {
     template <template <class> class TT> void f(TT<int>) {};
-    // old-note@-1 {{has different template parameters}}
     template <class...> struct A {};
     void test() { f(A<int>()); }
-    // old-error@-1 {{no matching function for call to 'f'}}
   } // namespace match_func
   namespace order_func_nonpack {
     template <template <class> class TT> void f(TT<int>) {}
@@ -423,19 +390,22 @@ namespace fun_tmpl_call {
   namespace order_func_pack {
     template <template <class> class TT> void f(TT<int>) = delete;
     template <template <class...> class TT> void f(TT<int>) {}
-
     template <class...> struct A {};
     void test() { f(A<int>()); }
   } // namespace order_func_pack
+  namespace match_enum {
+    enum A {};
+    template<template<A> class TT1> void f(TT1<{}>) {}
+    template<int> struct B {};
+    template void f<B>(B<{}>);
+  } // namespace match_enum
   namespace match_method {
     struct A {
       template <template <class> class TT> void f(TT<int>) {};
-      // old-note@-1 {{has different template parameters}}
     };
     template <class...> struct B {};
     void test() { A().f(B<int>()); }
-    // old-error@-1 {{no matching member function for call to 'f'}}
-  } // namespace t2
+  } // namespace match_method
   namespace order_method_nonpack {
     struct A {
       template <template <class> class TT> void f(TT<int>) {}
@@ -455,12 +425,9 @@ namespace fun_tmpl_call {
   namespace match_conv {
     struct A {
       template <template <class> class TT> operator TT<int>() { return {}; }
-      // old-note@-1 {{different template parameters}}
     };
     template <class...> struct B {};
-    // old-note@-1 2{{not viable}}
     void test() { B<int> b = A(); }
-    // old-error@-1 {{no viable conversion from 'A' to 'B<int>'}}
   } // namespace match_conv
   namespace order_conv_nonpack {
     struct A {
@@ -524,21 +491,21 @@ namespace regression1 {
 
 namespace constraints {
   template <class T> concept C1 = true;
-  // new-note@-1 {{similar constraint expression here}}
-  // new-note@-2 2{{similar constraint expressions not considered equivalent}}
+  // expected-note@-1 {{similar constraint expression here}}
+  // expected-note@-2 2{{similar constraint expressions not considered equivalent}}
 
   template <class T> concept C2 = C1<T> && true;
-  // new-note@-1 2{{similar constraint expression here}}
+  // expected-note@-1 2{{similar constraint expression here}}
 
   template <class T> concept D1 = true;
-  // new-note@-1 {{similar constraint expressions not considered equivalent}}
+  // expected-note@-1 {{similar constraint expressions not considered equivalent}}
 
   namespace t1 {
-    template<template<C1, class... T1s> class TT1> // new-note {{TT1' declared here}}
+    template<template<C1, class... T1s> class TT1> // expected-note {{TT1' declared here}}
     struct A {};
-    template<D1, class T2> struct B {}; // new-note {{'B' declared here}}
+    template<D1, class T2> struct B {}; // expected-note {{'B' declared here}}
     template struct A<B>;
-    // new-error@-1 {{'B' is more constrained than template template parameter 'TT1'}}
+    // expected-error@-1 {{'B' is more constrained than template template parameter 'TT1'}}
   } // namespace t1
   namespace t2 {
     template<template<C2, class... T1s> class TT1> struct A {};
@@ -546,34 +513,34 @@ namespace constraints {
     template struct A<B>;
   } // namespace t2
   namespace t3 {
-    template<template<C1, class... T1s> class TT1> // new-note {{'TT1' declared here}}
+    template<template<C1, class... T1s> class TT1> // expected-note {{'TT1' declared here}}
     struct A {};
-    template<C2, class T2> struct B {}; // new-note {{'B' declared here}}
+    template<C2, class T2> struct B {}; // expected-note {{'B' declared here}}
     template struct A<B>;
-    // new-error@-1 {{'B' is more constrained than template template parameter 'TT1'}}
+    // expected-error@-1 {{'B' is more constrained than template template parameter 'TT1'}}
   } // namespace t2
   namespace t4 {
     // FIXME: This should be accepted.
-    template<template<C1... T1s> class TT1> // new-note {{'TT1' declared here}}
+    template<template<C1... T1s> class TT1> // expected-note {{'TT1' declared here}}
     struct A {};
-    template<C1 T2> struct B {}; // new-note {{'B' declared here}}
+    template<C1 T2> struct B {}; // expected-note {{'B' declared here}}
     template struct A<B>;
-    // new-error@-1 {{'B' is more constrained than template template parameter 'TT1'}}
+    // expected-error@-1 {{'B' is more constrained than template template parameter 'TT1'}}
   } // namespace t4
   namespace t5 {
     // FIXME: This should be accepted
-    template<template<C2... T1s> class TT1> // new-note {{'TT1' declared here}}
+    template<template<C2... T1s> class TT1> // expected-note {{'TT1' declared here}}
     struct A {};
-    template<C1 T2> struct B {}; // new-note {{'B' declared here}}
+    template<C1 T2> struct B {}; // expected-note {{'B' declared here}}
     template struct A<B>;
-    // new-error@-1 {{'B' is more constrained than template template parameter 'TT1'}}
+    // expected-error@-1 {{'B' is more constrained than template template parameter 'TT1'}}
   } // namespace t5
   namespace t6 {
-    template<template<C1... T1s> class TT1> // new-note {{'TT1' declared here}}
+    template<template<C1... T1s> class TT1> // expected-note {{'TT1' declared here}}
     struct A {};
-    template<C2 T2> struct B {}; // new-note {{'B' declared here}}
+    template<C2 T2> struct B {}; // expected-note {{'B' declared here}}
     template struct A<B>;
-    // new-error@-1 {{'B' is more constrained than template template parameter 'TT1'}}
+    // expected-error@-1 {{'B' is more constrained than template template parameter 'TT1'}}
   } // namespace t6
   namespace t7 {
     template<template<class... T1s> class TT1>
@@ -588,19 +555,19 @@ namespace constraints {
     template struct A<B>;
   } // namespace t8
   namespace t9 {
-    template<template<C1... T1s> class TT1> // new-note {{'TT1' declared here}}
+    template<template<C1... T1s> class TT1> // expected-note {{'TT1' declared here}}
     struct A {};
-    template<D1 T2> struct B {}; // new-note {{'B' declared here}}
+    template<D1 T2> struct B {}; // expected-note {{'B' declared here}}
     template struct A<B>;
-    // new-error@-1 {{'B' is more constrained than template template parameter 'TT1'}}
+    // expected-error@-1 {{'B' is more constrained than template template parameter 'TT1'}}
   } // namespace t9
   namespace t10 {
-    template<template<class...> requires C1<int> class TT1> // new-note {{'TT1' declared here}}
+    template<template<class...> requires C1<int> class TT1> // expected-note {{'TT1' declared here}}
     struct A {};
 
-    template<class> requires C2<int> struct B {}; // new-note {{'B' declared here}}
+    template<class> requires C2<int> struct B {}; // expected-note {{'B' declared here}}
     template struct A<B>;
-    // new-error@-1 {{'B' is more constrained than template template parameter 'TT1'}}
+    // expected-error@-1 {{'B' is more constrained than template template parameter 'TT1'}}
   } // namespace t10
   namespace t11 {
     template<template<class...> requires C2<int> class TT1> struct A {};
@@ -618,6 +585,23 @@ namespace regression2 {
   template <typename, int> struct Matrix;
   template struct D<Matrix<double, 3>>;
 } // namespace regression2
+namespace regression3 {
+  struct None {};
+  template<class T> struct Node { using type = T; };
+
+  template <template<class> class TT, class T>
+  struct A {
+    static_assert(!__is_same(T, None));
+    using type2 = typename A<TT, typename T::type>::type2;
+  };
+
+  template <template<class> class TT> struct A<TT, None> {
+    using type2 = void;
+  };
+
+  template <class...> class B {};
+  template struct A<B, Node<None>>;
+} // namespace regression3
 
 namespace nttp_auto {
   namespace t1 {
@@ -628,23 +612,66 @@ namespace nttp_auto {
   namespace t2 {
     // FIXME: Shouldn't accept parameters after a parameter pack.
     template<template<auto... Va1, auto Va2> class> struct A {};
-    // new-error@-1 {{deduced non-type template argument does not have the same type as the corresponding template parameter ('auto' vs 'int')}}
+    // expected-error@-1 {{deduced non-type template argument does not have the same type as the corresponding template parameter ('auto' vs 'int')}}
     // expected-note@-2 {{previous template template parameter is here}}
     template<int... Vi> struct B;
-    // new-note@-1 {{template parameter is declared here}}
-    // old-note@-2 {{too few template parameters}}
+    // expected-note@-1 {{template parameter is declared here}}
     template struct A<B>;
-    // new-note@-1 {{different template parameters}}
-    // old-error@-2 {{different template parameters}}
+    // expected-note@-1 {{different template parameters}}
   } // namespace t2
   namespace t3 {
     // FIXME: Shouldn't accept parameters after a parameter pack.
     template<template<auto... Va1, auto... Va2> class> struct A {};
-    // new-error@-1 {{deduced non-type template argument does not have the same type as the corresponding template parameter ('auto' vs 'int')}}
-    // new-note@-2 {{previous template template parameter is here}}
+    // expected-error@-1 {{deduced non-type template argument does not have the same type as the corresponding template parameter ('auto' vs 'int')}}
+    // expected-note@-2 {{previous template template parameter is here}}
     template<int... Vi> struct B;
-    // new-note@-1 {{template parameter is declared here}}
+    // expected-note@-1 {{template parameter is declared here}}
     template struct A<B>;
-    // new-note@-1 {{different template parameters}}
+    // expected-note@-1 {{different template parameters}}
   } // namespace t3
 } // namespace nttp_auto
+
+namespace nttp_partial_order {
+  namespace t1 {
+    template<template<short> class TT1> void f(TT1<0>);
+    template<template<int>   class TT2> void f(TT2<0>) {}
+    template<int> struct B {};
+    template void f<B>(B<0>);
+  } // namespace t1
+  namespace t2 {
+    struct A {} a;
+    template<template<A&>       class TT1> void f(TT1<a>);
+    template<template<const A&> class TT2> void f(TT2<a>) {}
+    template<const A&> struct B {};
+    template void f<B>(B<a>);
+  } // namespace t2
+  namespace t3 {
+    enum A {};
+    template<template<A>   class TT1> void f(TT1<{}>);
+    template<template<int> class TT2> void f(TT2<{}>) {}
+    template<int> struct B {};
+    template void f<B>(B<{}>);
+  } // namespace t3
+  namespace t4 {
+    struct A {} a;
+    template<template<A*>       class TT1> void f(TT1<&a>);
+    template<template<const A*> class TT2> void f(TT2<&a>) {}
+    template<const A*> struct B {};
+    template void f<B>(B<&a>);
+  } // namespace t4
+  namespace t5 {
+    struct A { int m; };
+    template<template<int A::*>       class TT1> void f(TT1<&A::m>);
+    template<template<const int A::*> class TT2> void f(TT2<&A::m>) {}
+    template<const int A::*> struct B {};
+    template void f<B>(B<&A::m>);
+  } // namespace t5
+  namespace t6 {
+    struct A {};
+    using nullptr_t = decltype(nullptr);
+    template<template<nullptr_t> class TT2> void f(TT2<nullptr>);
+    template<template<A*>        class TT1> void f(TT1<nullptr>) {}
+    template<A*> struct B {};
+    template void f<B>(B<nullptr>);
+  } // namespace t6
+} // namespace nttp_partial_order
diff --git a/clang/test/SemaTemplate/deduction-crash.cpp b/clang/test/SemaTemplate/deduction-crash.cpp
index e3f97c594aa90..287c61aef00d6 100644
--- a/clang/test/SemaTemplate/deduction-crash.cpp
+++ b/clang/test/SemaTemplate/deduction-crash.cpp
@@ -24,16 +24,16 @@ struct state_machine
   {
     ant(0);
   }
-  
+
   template<class>
   struct region_processing_helper
   {
     template<class,int=0>
     struct In;
-    
+
     template<int my>
     struct In<a::int_<aaa::a>,my>; // expected-error +{{}}
-        
+
     template<class Event>
     int process(Event)
     {
@@ -61,18 +61,18 @@ template <class _Tp> struct remove_reference<_Tp&> ;
 
 template <class > struct __tuple_like;
 
-template <class _Tp, class _Up, int = __tuple_like<typename remove_reference<_Tp>::type>::value> 
+template <class _Tp, class _Up, int = __tuple_like<typename remove_reference<_Tp>::type>::value>
 struct __tuple_convertible;
 
 struct pair
 {
-template<class _Tuple, int = enable_if<__tuple_convertible<_Tuple, pair>::value>::type> 
+template<class _Tuple, int = enable_if<__tuple_convertible<_Tuple, pair>::value>::type>
 pair(_Tuple&& );
 };
 
 template <class> struct basic_ostream;
 
-template <int> 
+template <int>
 void endl( ) ;
 
 extern basic_ostream<char> cout;
@@ -166,9 +166,9 @@ namespace PR51872_part1 {
   template<int> class T1 { template <struct U1> T1(); };
   // expected-error@-1 {{non-type template parameter has incomplete type 'struct U1'}}
   // expected-note@-2  {{forward declaration of 'PR51872_part1::U1'}}
-  // expected-note@-3  {{implicit deduction guide declared as 'template <int> T1(T1<>) -> T1<>'}}
+  // expected-note@-3  {{implicit deduction guide declared as 'template <int> T1(T1<value-parameter-0-0>) -> T1<value-parameter-0-0>'}}
 
   T1 t1 = 0;
   // expected-error@-1 {{no viable constructor or deduction guide for deduction of template arguments of 'T1'}}
-  // expected-note@-7  {{candidate template ignored: could not match 'T1<>' against 'int'}}
+  // expected-note@-7  {{candidate template ignored: could not match 'T1<value-parameter-0-0>' against 'int'}}
 }
diff --git a/clang/test/SemaTemplate/deduction-guide.cpp b/clang/test/SemaTemplate/deduction-guide.cpp
index c0b56f9e6ac7b..a4c523595fca2 100644
--- a/clang/test/SemaTemplate/deduction-guide.cpp
+++ b/clang/test/SemaTemplate/deduction-guide.cpp
@@ -227,20 +227,20 @@ F s(0);
 // CHECK: FunctionTemplateDecl
 // CHECK: |-NonTypeTemplateParmDecl {{.*}} 'char' depth 0 index 0
 // CHECK:   `-TemplateArgument {{.*}} expr
-// CHECK: |   |-inherited from NonTypeTemplateParm {{.*}} '' 'char'
+// CHECK: |   |-inherited from NonTypeTemplateParm {{.*}} depth 0 index 0 'char'
 // CHECK: |   `-CharacterLiteral {{.*}} 'char' 120
 // CHECK: |-TemplateTypeParmDecl {{.*}} typename depth 0 index 1 U
 // CHECK: |-ParenExpr {{.*}} 'bool'
 // CHECK: | `-CXXBoolLiteralExpr {{.*}} 'bool' false
-// CHECK: |-CXXDeductionGuideDecl {{.*}} implicit <deduction guide for F> 'auto (U) -> F<>'
+// CHECK: |-CXXDeductionGuideDecl {{.*}} implicit <deduction guide for F> 'auto (U) -> F<value-parameter-0-0>'
 // CHECK: | `-ParmVarDecl {{.*}} 'U'
 // CHECK: `-CXXDeductionGuideDecl {{.*}} implicit <deduction guide for F> 'auto (int) -> F<>'
 // CHECK:   |-TemplateArgument integral ''x''
 // CHECK:   |-TemplateArgument type 'int'
 // CHECK:   | `-BuiltinType {{.*}} 'int'
 // CHECK:   `-ParmVarDecl {{.*}} 'int'
-// CHECK: FunctionProtoType {{.*}} 'auto (U) -> F<>' dependent trailing_return cdecl
-// CHECK: |-InjectedClassNameType {{.*}} 'F<>' dependent
+// CHECK: FunctionProtoType {{.*}} 'auto (U) -> F<value-parameter-0-0>' dependent trailing_return cdecl
+// CHECK: |-InjectedClassNameType {{.*}} 'F<value-parameter-0-0>' dependent
 // CHECK: | `-CXXRecord {{.*}} 'F'
 // CHECK: `-TemplateTypeParmType {{.*}} 'U' dependent depth 0 index 1
 
diff --git a/clang/test/SemaTemplate/default-arguments.cpp b/clang/test/SemaTemplate/default-arguments.cpp
index 3b1fbda414c12..5ea34c0254ec1 100644
--- a/clang/test/SemaTemplate/default-arguments.cpp
+++ b/clang/test/SemaTemplate/default-arguments.cpp
@@ -52,20 +52,20 @@ struct X2 {
   template<typename U = typename X1<T>::type> // expected-error{{no type named 'type' in 'X1<int>'}} \
                                               // expected-error{{no type named 'type' in 'X1<char>'}}
   struct Inner1 { }; // expected-note{{template is declared here}}
-  
+
   template<T Value = X1<T>::value> // expected-error{{no member named 'value' in 'X1<int>'}} \
                                    // expected-error{{no member named 'value' in 'X1<char>'}}
   struct NonType1 { }; // expected-note{{template is declared here}}
-  
+
   template<T Value>
   struct Inner2 { };
-  
+
   template<typename U>
   struct Inner3 {
     template<typename X = T, typename V = U>
     struct VeryInner { };
-    
-    template<T Value1 = sizeof(T), T Value2 = sizeof(U), 
+
+    template<T Value1 = sizeof(T), T Value2 = sizeof(U),
              T Value3 = Value1 + Value2>
     struct NonType2 { };
   };
@@ -89,11 +89,11 @@ struct is_same { static const bool value = false; };
 template<typename T>
 struct is_same<T, T> { static const bool value = true; };
 
-int array1[is_same<__typeof__(vi), 
+int array1[is_same<__typeof__(vi),
                X2<int>::Inner3<float>::VeryInner<int, float> >::value? 1 : -1];
 
 int array2[is_same<__typeof(x2_deep_nontype),
-                   X2<char>::Inner3<int>::NonType2<sizeof(char), sizeof(int), 
+                   X2<char>::Inner3<int>::NonType2<sizeof(char), sizeof(int),
                                     sizeof(char)+sizeof(int)> >::value? 1 : -1];
 
 // Template template parameter defaults
@@ -109,18 +109,20 @@ struct add_pointer {
 
 template<typename T, template<typename> class X = T::template apply>
   struct X4;
-int array4[is_same<X4<add_pointer>, 
+int array4[is_same<X4<add_pointer>,
                    X4<add_pointer, add_pointer::apply> >::value? 1 : -1];
 
 template<int> struct X5 {};
-template<long> struct X5b {};
-template<typename T, 
-         template<T> class B = X5>
+template<long long> struct X5b {};
+template<typename T,
+         template<T> class B = X5> // expected-error {{cannot be narrowed from type 'long long' to 'int'}}
+                                   // expected-note@-1 {{has different template parameters}}
+                                   // expected-note@-2 {{previous template template parameter is here}}
   struct X6 {};
 
 X6<int> x6a;
-X6<long> x6b;
-X6<long, X5b> x6c;
+X6<long long> x6b; // expected-note {{while checking a default template argument used here}}
+X6<long long, X5b> x6c;
 
 
 template<template<class> class X = B<int> > struct X7; // expected-error{{must be a class template}}
diff --git a/clang/test/SemaTemplate/instantiate-template-template-parm.cpp b/clang/test/SemaTemplate/instantiate-template-template-parm.cpp
index 39aeeb1c1a6a3..dce30aa2af4ee 100644
--- a/clang/test/SemaTemplate/instantiate-template-template-parm.cpp
+++ b/clang/test/SemaTemplate/instantiate-template-template-parm.cpp
@@ -24,11 +24,13 @@ template<int> struct B;
 
 template<typename T,
          template<T Value> class X> // expected-error{{cannot have type 'float'}}
+                                    // expected-error@-1 {{cannot be narrowed from type 'long long' to 'int'}}
+                                    // expected-note@-2 {{previous template template parameter is here}}
 struct X0 { };
 
 X0<int, B> x0b1;
 X0<float, B> x0b2; // expected-note{{while substituting}}
-X0<long, B> x0b3;
+X0<long long, B> x0b3; // expected-note {{has different template parameters}}
 
 template<template<int V> class TT>
 struct X1 { };
diff --git a/clang/test/SemaTemplate/instantiation-default-2.cpp b/clang/test/SemaTemplate/instantiation-default-2.cpp
index 99c25000bcb0b..ffa77cf1ee567 100644
--- a/clang/test/SemaTemplate/instantiation-default-2.cpp
+++ b/clang/test/SemaTemplate/instantiation-default-2.cpp
@@ -2,8 +2,10 @@
 // RUN: %clang_cc1 -fsyntax-only -verify=expected,cxx17,precxx20 -std=c++17 %s
 // RUN: %clang_cc1 -fsyntax-only -verify=expected,cxx20 %std_cxx20- %s
 
-template<typename T, T Value> struct Constant; // precxx17-note{{template parameter is declared here}} \
-// FIXME: bad location precxx20-error{{a non-type template parameter cannot have type 'float'}}
+template<typename T, T Value> struct Constant;
+// FIXME: bad location precxx20-error@-1 {{a non-type template parameter cannot have type 'float'}}
+// expected-note@-2 {{template parameter is declared here}}
+// cxx20-note@-3 {{template parameter is declared here}}
 
 Constant<int, 5> *c1;
 
diff --git a/clang/test/SemaTemplate/nested-template.cpp b/clang/test/SemaTemplate/nested-template.cpp
index a54992a397187..4d7f2d9c5c946 100644
--- a/clang/test/SemaTemplate/nested-template.cpp
+++ b/clang/test/SemaTemplate/nested-template.cpp
@@ -112,7 +112,9 @@ template struct X1<int>::B<bool>;
 // Template template parameters
 template<typename T>
 struct X2 {
-  template<template<class U, T Value> class>  // expected-error{{cannot have type 'float'}}
+  template<template<class U, T Value> class>  // expected-error {{cannot have type 'float'}}
+                                              // expected-error@-1 {{cannot be narrowed from type 'long long' to 'int'}}
+                                              // expected-note@-2 {{previous template template parameter is here}}
     struct Inner { };
 };
 
@@ -121,7 +123,7 @@ template<typename T, int Value>
 
 X2<int>::Inner<X2_arg> x2i1;
 X2<float> x2a; // expected-note{{instantiation}}
-X2<long>::Inner<X2_arg> x2i3;
+X2<long long>::Inner<X2_arg> x2i3; // expected-note {{has different template parameters}}
 
 namespace PR10896 {
   template<typename TN>
diff --git a/clang/test/SemaTemplate/temp_arg_nontype_cxx1z.cpp b/clang/test/SemaTemplate/temp_arg_nontype_cxx1z.cpp
index 8d9356bb3201c..c35743b87adbc 100644
--- a/clang/test/SemaTemplate/temp_arg_nontype_cxx1z.cpp
+++ b/clang/test/SemaTemplate/temp_arg_nontype_cxx1z.cpp
@@ -1,6 +1,6 @@
 // RUN: %clang_cc1 -fsyntax-only -verify -std=c++1z %s
 
-template<typename T, T val> struct A {};
+template<typename T, T val> struct A {}; // expected-note 3{{template parameter is declared here}}
 
 template<typename T, typename U> constexpr bool is_same = false;
 template<typename T> constexpr bool is_same<T, T> = true;
@@ -122,13 +122,13 @@ namespace DeduceDifferentType {
   int a_imp = a(A<3>()); // expected-error {{no matching function}}
   int a_exp = a<3>(A<3>());
 
-  template<decltype(nullptr)> struct B {};
+  template<decltype(nullptr)> struct B {}; // expected-note {{template parameter is declared here}}
   template<int *P> int b(B<P>); // expected-error {{value of type 'int *' is not implicitly convertible to 'decltype(nullptr)'}}
   int b_imp = b(B<nullptr>()); // expected-error {{no matching function}}
   int b_exp = b<nullptr>(B<nullptr>()); // expected-error {{no matching function}}
 
   struct X { constexpr operator int() { return 0; } } x;
-  template<X &> struct C {};
+  template<X &> struct C {}; // expected-note {{template parameter is declared here}}
   template<int N> int c(C<N>); // expected-error {{value of type 'int' is not implicitly convertible to 'X &'}}
   int c_imp = c(C<x>()); // expected-error {{no matching function}}
   int c_exp = c<x>(C<x>()); // expected-error {{no matching function}}
@@ -448,7 +448,7 @@ namespace PR42108 {
   struct R {};
   struct S { constexpr S() {} constexpr S(R) {} };
   struct T { constexpr operator S() { return {}; } };
-  template <const S &> struct A {};
+  template <const S &> struct A {}; // expected-note {{template parameter is declared here}}
   void f() {
     A<R{}>(); // expected-error {{would bind reference to a temporary}}
     A<S{}>(); // expected-error {{reference to temporary object}}
diff --git a/clang/test/SemaTemplate/temp_arg_nontype_cxx20.cpp b/clang/test/SemaTemplate/temp_arg_nontype_cxx20.cpp
index ad73daa8e214c..56ceb7af4ccd9 100644
--- a/clang/test/SemaTemplate/temp_arg_nontype_cxx20.cpp
+++ b/clang/test/SemaTemplate/temp_arg_nontype_cxx20.cpp
@@ -101,7 +101,7 @@ namespace ConvertedConstant {
   struct {
     int i : 2;
   } b;
-  template <const int&> struct Y {};
+  template <const int&> struct Y {}; // expected-note {{template parameter is declared here}}
   void f(Y<b.i>) {} // expected-error {{reference cannot bind to bit-field in converted constant expression}}
 }
 
@@ -238,7 +238,7 @@ namespace UnnamedBitfield {
 }
 
 namespace Temporary {
-  template<const int &> struct A {};
+  template<const int &> struct A {}; // expected-note {{template parameter is declared here}}
   A<0> a0; // expected-error {{conversion from 'int' to 'const int &' in converted constant expression would bind reference to a temporary}}
 
   A<(const int&)1> a1; // expected-error {{reference to temporary object is not allowed in a template argument}}
diff --git a/clang/test/SemaTemplate/temp_arg_template.cpp b/clang/test/SemaTemplate/temp_arg_template.cpp
index 9908af5e78669..aa53dba652050 100644
--- a/clang/test/SemaTemplate/temp_arg_template.cpp
+++ b/clang/test/SemaTemplate/temp_arg_template.cpp
@@ -7,14 +7,13 @@ template<template<typename T> class X> struct A; // #A
 template<template<typename T, int I> class X> struct B; // expected-note{{previous template template parameter is here}}
 
 template<template<int I> class X> struct C;
-// precxx17-error@-1 {{deduced non-type template argument does not have the same type as the corresponding template parameter ('int' vs 'const int &')}}
-// cxx17-error@-2 {{conversion from 'int' to 'const int &' in converted constant expression would bind reference to a temporary}}
-// expected-note@-3 {{previous template template parameter is here}}
+// expected-error@-1 {{conversion from 'int' to 'const int &' in converted constant expression would bind reference to a temporary}}
+// expected-note@-2 {{previous template template parameter is here}}
 
 template<class> struct X; // expected-note {{template is declared here}}
 template<int N> struct Y; // expected-note {{template parameter is declared here}}
 template<long N> struct Ylong;
-template<const int &N> struct Yref; // precxx17-note {{template parameter is declared here}}
+template<const int &N> struct Yref; // expected-note {{template parameter is declared here}}
 
 namespace N {
   template<class> struct Z;
diff --git a/clang/test/SemaTemplate/temp_arg_template_p0522.cpp b/clang/test/SemaTemplate/temp_arg_template_p0522.cpp
index dcfc7b5b27288..2e5a36ae6ed08 100644
--- a/clang/test/SemaTemplate/temp_arg_template_p0522.cpp
+++ b/clang/test/SemaTemplate/temp_arg_template_p0522.cpp
@@ -66,8 +66,9 @@ namespace DependentType {
   using ok = Pt<tT0<int, i>, tT0<int, iDi>>;
   using err1 = tT0<int, ii>; // expected-error {{too few template arguments for class template 'ii'}}
                              // expected-note@-1 {{different template parameters}}
-  using err2 = tT0<short, i>; // FIXME: should this be OK?
-  using err2a = tT0<long long, i>; // FIXME: should this be OK (if long long is larger than int)?
+  using err2 = tT0<short, i>;
+  using err2a = tT0<long long, i>; // expected-error@#tT0 {{cannot be narrowed from type 'long long' to 'int'}}
+                                   // expected-note@-1 {{different template parameters}}
   using err2b = tT0<void*, i>; // expected-error@#tT0 {{value of type 'void *' is not implicitly convertible to 'int'}}
                                // expected-note@-1 {{different template parameters}}
   using err3 = tT0<short, t0>; // expected-error@#tT0 {{template argument for template type parameter must be a type}}
diff --git a/clang/test/TableGen/target-builtins-prototype-parser.td b/clang/test/TableGen/target-builtins-prototype-parser.td
index a753f906a674f..451f1a18b8ad0 100644
--- a/clang/test/TableGen/target-builtins-prototype-parser.td
+++ b/clang/test/TableGen/target-builtins-prototype-parser.td
@@ -10,55 +10,55 @@
 include "clang/Basic/BuiltinsBase.td"
 
 def : Builtin {
-// CHECK: BUILTIN(__builtin_01, "E8idE4b", "")
+// CHECK: Builtin::Info{{.*}} __builtin_01 {{.*}} /* E8idE4b */
   let Prototype = "_ExtVector<8,int>(double, _ExtVector<4,        bool>)";
   let Spellings = ["__builtin_01"];
 }
 
 def : Builtin {
-// CHECK: BUILTIN(__builtin_02, "E8UiE4s", "")
+// CHECK: Builtin::Info{{.*}} __builtin_02 {{.*}} /* E8UiE4s */
   let Prototype = "_ExtVector<8,unsigned int>(_ExtVector<4, short>)";
   let Spellings = ["__builtin_02"];
 }
 
 def : Builtin {
-// CHECK: BUILTIN(__builtin_03, "di", "")
+// CHECK: Builtin::Info{{.*}} __builtin_03 {{.*}} /* di */
   let Prototype = "double(int)";
   let Spellings = ["__builtin_03"];
 }
 
 def : Builtin {
-// CHECK: BUILTIN(__builtin_04, "diIUi", "")
+// CHECK: Builtin::Info{{.*}} __builtin_04 {{.*}} /* diIUi */
  let Prototype = "double(int, _Constant unsigned int)";
   let Spellings = ["__builtin_04"];
 }
 
 def : Builtin {
-// CHECK: BUILTIN(__builtin_05, "v&v&", "")
+// CHECK: Builtin::Info{{.*}} __builtin_05 {{.*}} /* v&v& */
  let Prototype = "void&(void&)";
   let Spellings = ["__builtin_05"];
 }
 
 def : Builtin {
-// CHECK: BUILTIN(__builtin_06, "v*v*cC*.", "")
+// CHECK: Builtin::Info{{.*}} __builtin_06 {{.*}} /* v*v*cC*. */
  let Prototype = "void*(void*, char const*, ...)";
   let Spellings = ["__builtin_06"];
 }
 
 def : Builtin {
-// CHECK: BUILTIN(__builtin_07, "E8iE4dE4b.", "")
+// CHECK: Builtin::Info{{.*}} __builtin_07 {{.*}} /* E8iE4dE4b. */
   let Prototype = "_ExtVector<8, int>(_ExtVector<4,double>, _ExtVector<4, bool>, ...)";
   let Spellings = ["__builtin_07"];
 }
 
 def : Builtin {
-// CHECK: BUILTIN(__builtin_08, "di*R", "")
+// CHECK: Builtin::Info{{.*}} __builtin_08 {{.*}} /* di*R */
   let Prototype = "double(int * restrict)";
   let Spellings = ["__builtin_08"];
 }
 
 def : Builtin {
-// CHECK: BUILTIN(__builtin_09, "V2yy", "")
+// CHECK: Builtin::Info{{.*}} __builtin_09 {{.*}} /* V2yy */
   let Prototype = "_Vector<2, __bf16>(__bf16)";
   let Spellings = ["__builtin_09"];
 }
diff --git a/clang/test/Tooling/clang-linker-wrapper-spirv-elf.cpp b/clang/test/Tooling/clang-linker-wrapper-spirv-elf.cpp
new file mode 100644
index 0000000000000..9b16727d74192
--- /dev/null
+++ b/clang/test/Tooling/clang-linker-wrapper-spirv-elf.cpp
@@ -0,0 +1,19 @@
+// Verify the ELF packaging of OpenMP SPIR-V device images.
+// REQUIRES: system-linux
+// REQUIRES: spirv-tools
+// REQUIRES: llvm-spirv
+// RUN: mkdir -p %t_tmp
+// RUN: cd %t_tmp
+// RUN: %clangxx -fopenmp -fopenmp-targets=spirv64-intel -nogpulib -c -o %t_clang-linker-wrapper-spirv-elf.o %s
+// RUN: not clang-linker-wrapper -o a.out %t_clang-linker-wrapper-spirv-elf.o --save-temps --linker-path=ld
+// RUN: clang-offload-packager --image=triple=spirv64-intel,kind=openmp,file=%t.elf  %t_tmp/a.out.openmp.image.wrapper.o
+// RUN: llvm-readelf -t %t.elf | FileCheck -check-prefix=CHECK-SECTION %s
+// RUN: llvm-readelf -n %t.elf | FileCheck -check-prefix=CHECK-NOTES %s
+
+// CHECK-SECTION: .note.inteloneompoffload
+// CHECK-SECTION: __openmp_offload_spirv_0
+
+// CHECK-NOTES-COUNT-3: INTELONEOMPOFFLOAD
+int main(int argc, char** argv) {
+  return 0;
+}
diff --git a/clang/test/Tooling/lit.local.cfg b/clang/test/Tooling/lit.local.cfg
index 4cd8ba72fa767..61f328c91e4d3 100644
--- a/clang/test/Tooling/lit.local.cfg
+++ b/clang/test/Tooling/lit.local.cfg
@@ -1,2 +1,14 @@
+import lit.util
+
 if not config.root.clang_staticanalyzer:
     config.unsupported = True
+
+if config.spirv_tools_tests:
+    config.available_features.add("spirv-tools")
+    config.substitutions.append(("spirv-dis", os.path.join(config.llvm_tools_dir, "spirv-dis")))
+    config.substitutions.append(("spirv-val", os.path.join(config.llvm_tools_dir, "spirv-val")))
+    config.substitutions.append(("spirv-as", os.path.join(config.llvm_tools_dir, "spirv-as")))
+    config.substitutions.append(("spirv-link", os.path.join(config.llvm_tools_dir, "spirv-link")))
+
+if lit.util.which("llvm-spirv"):
+    config.available_features.add("llvm-spirv")
diff --git a/clang/test/lit.site.cfg.py.in b/clang/test/lit.site.cfg.py.in
index 1cbd876ac5bb9..6890da5327cb9 100644
--- a/clang/test/lit.site.cfg.py.in
+++ b/clang/test/lit.site.cfg.py.in
@@ -22,7 +22,7 @@ config.host_cxx = "@CMAKE_CXX_COMPILER@"
 config.llvm_use_sanitizer = "@LLVM_USE_SANITIZER@"
 config.have_zlib = @LLVM_ENABLE_ZLIB@
 config.have_zstd = @LLVM_ENABLE_ZSTD@
-config.clang_arcmt = @CLANG_ENABLE_ARCMT@
+config.clang_objc_rewriter = "@CLANG_ENABLE_OBJC_REWRITER@" == "ON"
 config.clang_default_pie_on_linux = @CLANG_DEFAULT_PIE_ON_LINUX@
 config.clang_default_cxx_stdlib = "@CLANG_DEFAULT_CXX_STDLIB@"
 config.clang_staticanalyzer = @CLANG_ENABLE_STATIC_ANALYZER@
@@ -43,6 +43,7 @@ config.llvm_external_lit = path(r"@LLVM_EXTERNAL_LIT@")
 config.standalone_build = @CLANG_BUILT_STANDALONE@
 config.ppc_linux_default_ieeelongdouble = @PPC_LINUX_DEFAULT_IEEELONGDOUBLE@
 config.have_llvm_driver = @LLVM_TOOL_LLVM_DRIVER_BUILD@
+config.spirv_tools_tests = @LLVM_INCLUDE_SPIRV_TOOLS_TESTS@
 config.substitutions.append(("%llvm-version-major", "@LLVM_VERSION_MAJOR@"))
 
 import lit.llvm
diff --git a/clang/tools/CMakeLists.txt b/clang/tools/CMakeLists.txt
index 98c018e96848d..de0b47f3751f1 100644
--- a/clang/tools/CMakeLists.txt
+++ b/clang/tools/CMakeLists.txt
@@ -27,11 +27,6 @@ if(UNIX OR (MSVC AND LLVM_BUILD_LLVM_DYLIB_VIS) OR (MINGW AND LLVM_LINK_LLVM_DYL
   add_clang_subdirectory(clang-shlib)
 endif()
 
-if(CLANG_ENABLE_ARCMT)
-  add_clang_subdirectory(arcmt-test)
-  add_clang_subdirectory(c-arcmt-test)
-endif()
-
 if(CLANG_ENABLE_STATIC_ANALYZER)
   add_clang_subdirectory(clang-check)
   add_clang_subdirectory(clang-extdef-mapping)
diff --git a/clang/tools/arcmt-test/CMakeLists.txt b/clang/tools/arcmt-test/CMakeLists.txt
deleted file mode 100644
index d3e6580e16efa..0000000000000
--- a/clang/tools/arcmt-test/CMakeLists.txt
+++ /dev/null
@@ -1,16 +0,0 @@
-set(LLVM_LINK_COMPONENTS
-  support
-  )
-
-add_clang_executable(arcmt-test
-  arcmt-test.cpp
-  )
-
-clang_target_link_libraries(arcmt-test
-  PRIVATE
-  clangARCMigrate
-  clangBasic
-  clangFrontend
-  clangLex
-  clangSerialization
-  )
diff --git a/clang/tools/arcmt-test/arcmt-test.cpp b/clang/tools/arcmt-test/arcmt-test.cpp
deleted file mode 100644
index b61f38e9905db..0000000000000
--- a/clang/tools/arcmt-test/arcmt-test.cpp
+++ /dev/null
@@ -1,376 +0,0 @@
-//===-- arcmt-test.cpp - ARC Migration Tool testbed -----------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "clang/ARCMigrate/ARCMT.h"
-#include "clang/AST/ASTContext.h"
-#include "clang/Frontend/PCHContainerOperations.h"
-#include "clang/Frontend/TextDiagnosticPrinter.h"
-#include "clang/Frontend/Utils.h"
-#include "clang/Frontend/VerifyDiagnosticConsumer.h"
-#include "clang/Lex/Preprocessor.h"
-#include "clang/Lex/PreprocessorOptions.h"
-#include "llvm/Support/FileSystem.h"
-#include "llvm/Support/MemoryBuffer.h"
-#include "llvm/Support/Path.h"
-#include "llvm/Support/Signals.h"
-#include <system_error>
-
-using namespace clang;
-using namespace arcmt;
-
-static llvm::cl::opt<bool>
-CheckOnly("check-only",
-      llvm::cl::desc("Just check for issues that need to be handled manually"));
-
-//static llvm::cl::opt<bool>
-//TestResultForARC("test-result",
-//llvm::cl::desc("Test the result of transformations by parsing it in ARC mode"));
-
-static llvm::cl::opt<bool>
-OutputTransformations("output-transformations",
-                      llvm::cl::desc("Print the source transformations"));
-
-static llvm::cl::opt<bool>
-VerifyDiags("verify",llvm::cl::desc("Verify emitted diagnostics and warnings"));
-
-static llvm::cl::opt<bool>
-VerboseOpt("v", llvm::cl::desc("Enable verbose output"));
-
-static llvm::cl::opt<bool>
-VerifyTransformedFiles("verify-transformed-files",
-llvm::cl::desc("Read pairs of file mappings (typically the output of "
-               "c-arcmt-test) and compare their contents with the filenames "
-               "provided in command-line"));
-
-static llvm::cl::opt<std::string>
-RemappingsFile("remappings-file",
-               llvm::cl::desc("Pairs of file mappings (typically the output of "
-               "c-arcmt-test)"));
-
-static llvm::cl::list<std::string>
-ResultFiles(llvm::cl::Positional, llvm::cl::desc("<filename>..."));
-
-static llvm::cl::extrahelp extraHelp(
-  "\nusage with compiler args: arcmt-test [options] --args [compiler flags]\n");
-
-// This function isn't referenced outside its translation unit, but it
-// can't use the "static" keyword because its address is used for
-// GetMainExecutable (since some platforms don't support taking the
-// address of main, and some platforms can't implement GetMainExecutable
-// without being given the address of a function in the main executable).
-std::string GetExecutablePath(const char *Argv0) {
-  // This just needs to be some symbol in the binary; C++ doesn't
-  // allow taking the address of ::main however.
-  void *MainAddr = (void*) (intptr_t) GetExecutablePath;
-  return llvm::sys::fs::getMainExecutable(Argv0, MainAddr);
-}
-
-static void printSourceLocation(SourceLocation loc, ASTContext &Ctx,
-                                raw_ostream &OS);
-static void printSourceRange(CharSourceRange range, ASTContext &Ctx,
-                             raw_ostream &OS);
-
-namespace {
-
-class PrintTransforms : public MigrationProcess::RewriteListener {
-  ASTContext *Ctx;
-  raw_ostream &OS;
-
-public:
-  PrintTransforms(raw_ostream &OS)
-    : Ctx(nullptr), OS(OS) {}
-
-  void start(ASTContext &ctx) override { Ctx = &ctx; }
-  void finish() override { Ctx = nullptr; }
-
-  void insert(SourceLocation loc, StringRef text) override {
-    assert(Ctx);
-    OS << "Insert: ";
-    printSourceLocation(loc, *Ctx, OS);
-    OS << " \"" << text << "\"\n";
-  }
-
-  void remove(CharSourceRange range) override {
-    assert(Ctx);
-    OS << "Remove: ";
-    printSourceRange(range, *Ctx, OS);
-    OS << '\n';
-  }
-};
-
-} // anonymous namespace
-
-static bool checkForMigration(StringRef resourcesPath,
-                              ArrayRef<const char *> Args) {
-  IntrusiveRefCntPtr<DiagnosticOptions> DiagOpts = new DiagnosticOptions();
-  DiagnosticConsumer *DiagClient =
-    new TextDiagnosticPrinter(llvm::errs(), &*DiagOpts);
-  IntrusiveRefCntPtr<DiagnosticIDs> DiagID(new DiagnosticIDs());
-  IntrusiveRefCntPtr<DiagnosticsEngine> Diags(
-      new DiagnosticsEngine(DiagID, &*DiagOpts, DiagClient));
-  // Chain in -verify checker, if requested.
-  VerifyDiagnosticConsumer *verifyDiag = nullptr;
-  if (VerifyDiags) {
-    verifyDiag = new VerifyDiagnosticConsumer(*Diags);
-    Diags->setClient(verifyDiag);
-  }
-
-  CompilerInvocation CI;
-  if (!CompilerInvocation::CreateFromArgs(CI, Args, *Diags))
-    return true;
-
-  if (CI.getFrontendOpts().Inputs.empty()) {
-    llvm::errs() << "error: no input files\n";
-    return true;
-  }
-
-  if (!CI.getLangOpts().ObjC)
-    return false;
-
-  arcmt::checkForManualIssues(CI, CI.getFrontendOpts().Inputs[0],
-                              std::make_shared<PCHContainerOperations>(),
-                              Diags->getClient());
-  return Diags->getClient()->getNumErrors() > 0;
-}
-
-static void printResult(FileRemapper &remapper, raw_ostream &OS) {
-  remapper.forEachMapping([](StringRef, StringRef) {},
-                          [&](StringRef, const llvm::MemoryBufferRef &Buffer) {
-                            OS << Buffer.getBuffer();
-                          });
-}
-
-static bool performTransformations(StringRef resourcesPath,
-                                   ArrayRef<const char *> Args) {
-  // Check first.
-  if (checkForMigration(resourcesPath, Args))
-    return true;
-
-  IntrusiveRefCntPtr<DiagnosticOptions> DiagOpts = new DiagnosticOptions();
-  DiagnosticConsumer *DiagClient =
-    new TextDiagnosticPrinter(llvm::errs(), &*DiagOpts);
-  IntrusiveRefCntPtr<DiagnosticIDs> DiagID(new DiagnosticIDs());
-  IntrusiveRefCntPtr<DiagnosticsEngine> TopDiags(
-      new DiagnosticsEngine(DiagID, &*DiagOpts, &*DiagClient));
-
-  CompilerInvocation origCI;
-  if (!CompilerInvocation::CreateFromArgs(origCI, Args, *TopDiags))
-    return true;
-
-  if (origCI.getFrontendOpts().Inputs.empty()) {
-    llvm::errs() << "error: no input files\n";
-    return true;
-  }
-
-  if (!origCI.getLangOpts().ObjC)
-    return false;
-
-  MigrationProcess migration(origCI, std::make_shared<PCHContainerOperations>(),
-                             DiagClient);
-
-  std::vector<TransformFn>
-    transforms = arcmt::getAllTransformations(origCI.getLangOpts().getGC(),
-                                 origCI.getMigratorOpts().NoFinalizeRemoval);
-  assert(!transforms.empty());
-
-  std::unique_ptr<PrintTransforms> transformPrinter;
-  if (OutputTransformations)
-    transformPrinter.reset(new PrintTransforms(llvm::outs()));
-
-  for (unsigned i=0, e = transforms.size(); i != e; ++i) {
-    bool err = migration.applyTransform(transforms[i], transformPrinter.get());
-    if (err) return true;
-
-    if (VerboseOpt) {
-      if (i == e-1)
-        llvm::errs() << "\n##### FINAL RESULT #####\n";
-      else
-        llvm::errs() << "\n##### OUTPUT AFTER "<< i+1 <<". TRANSFORMATION #####\n";
-      printResult(migration.getRemapper(), llvm::errs());
-      llvm::errs() << "\n##########################\n\n";
-    }
-  }
-
-  if (!OutputTransformations)
-    printResult(migration.getRemapper(), llvm::outs());
-
-  // FIXME: TestResultForARC
-
-  return false;
-}
-
-static bool filesCompareEqual(StringRef fname1, StringRef fname2) {
-  using namespace llvm;
-
-  ErrorOr<std::unique_ptr<MemoryBuffer>> file1 =
-      MemoryBuffer::getFile(fname1, /*IsText=*/true);
-  if (!file1)
-    return false;
-
-  ErrorOr<std::unique_ptr<MemoryBuffer>> file2 =
-      MemoryBuffer::getFile(fname2, /*IsText=*/true);
-  if (!file2)
-    return false;
-
-  return file1.get()->getBuffer() == file2.get()->getBuffer();
-}
-
-static bool verifyTransformedFiles(ArrayRef<std::string> resultFiles) {
-  using namespace llvm;
-
-  assert(!resultFiles.empty());
-
-  std::map<StringRef, StringRef> resultMap;
-
-  for (ArrayRef<std::string>::iterator
-         I = resultFiles.begin(), E = resultFiles.end(); I != E; ++I) {
-    StringRef fname(*I);
-    if (!fname.ends_with(".result")) {
-      errs() << "error: filename '" << fname
-                   << "' does not have '.result' extension\n";
-      return true;
-    }
-    resultMap[sys::path::stem(fname)] = fname;
-  }
-
-  ErrorOr<std::unique_ptr<MemoryBuffer>> inputBuf = std::error_code();
-  if (RemappingsFile.empty())
-    inputBuf = MemoryBuffer::getSTDIN();
-  else
-    inputBuf = MemoryBuffer::getFile(RemappingsFile, /*IsText=*/true);
-  if (!inputBuf) {
-    errs() << "error: could not read remappings input\n";
-    return true;
-  }
-
-  SmallVector<StringRef, 8> strs;
-  inputBuf.get()->getBuffer().split(strs, "\n", /*MaxSplit=*/-1,
-                                    /*KeepEmpty=*/false);
-
-  if (strs.empty()) {
-    errs() << "error: no files to verify from stdin\n";
-    return true;
-  }
-  if (strs.size() % 2 != 0) {
-    errs() << "error: files to verify are not original/result pairs\n";
-    return true;
-  }
-
-  for (unsigned i = 0, e = strs.size(); i != e; i += 2) {
-    StringRef inputOrigFname = strs[i];
-    StringRef inputResultFname = strs[i+1];
-
-    std::map<StringRef, StringRef>::iterator It;
-    It = resultMap.find(sys::path::filename(inputOrigFname));
-    if (It == resultMap.end()) {
-      errs() << "error: '" << inputOrigFname << "' is not in the list of "
-             << "transformed files to verify\n";
-      return true;
-    }
-
-    if (!sys::fs::exists(It->second)) {
-      errs() << "error: '" << It->second << "' does not exist\n";
-      return true;
-    }
-    if (!sys::fs::exists(inputResultFname)) {
-      errs() << "error: '" << inputResultFname << "' does not exist\n";
-      return true;
-    }
-
-    if (!filesCompareEqual(It->second, inputResultFname)) {
-      errs() << "error: '" << It->second << "' is different than "
-             << "'" << inputResultFname << "'\n";
-      return true;
-    }
-
-    resultMap.erase(It);
-  }
-
-  if (!resultMap.empty()) {
-    for (std::map<StringRef, StringRef>::iterator
-           I = resultMap.begin(), E = resultMap.end(); I != E; ++I)
-      errs() << "error: '" << I->second << "' was not verified!\n";
-    return true;
-  }
-
-  return false; 
-}
-
-//===----------------------------------------------------------------------===//
-// Misc. functions.
-//===----------------------------------------------------------------------===//
-
-static void printSourceLocation(SourceLocation loc, ASTContext &Ctx,
-                                raw_ostream &OS) {
-  SourceManager &SM = Ctx.getSourceManager();
-  PresumedLoc PL = SM.getPresumedLoc(loc);
-
-  OS << llvm::sys::path::filename(PL.getFilename());
-  OS << ":" << PL.getLine() << ":"
-            << PL.getColumn();
-}
-
-static void printSourceRange(CharSourceRange range, ASTContext &Ctx,
-                             raw_ostream &OS) {
-  SourceManager &SM = Ctx.getSourceManager();
-  const LangOptions &langOpts = Ctx.getLangOpts();
-
-  PresumedLoc PL = SM.getPresumedLoc(range.getBegin());
-
-  OS << llvm::sys::path::filename(PL.getFilename());
-  OS << " [" << PL.getLine() << ":"
-             << PL.getColumn();
-  OS << " - ";
-
-  SourceLocation end = range.getEnd();
-  PL = SM.getPresumedLoc(end);
-
-  unsigned endCol = PL.getColumn() - 1;
-  if (!range.isTokenRange())
-    endCol += Lexer::MeasureTokenLength(end, SM, langOpts);
-  OS << PL.getLine() << ":" << endCol << "]";
-}
-
-//===----------------------------------------------------------------------===//
-// Command line processing.
-//===----------------------------------------------------------------------===//
-
-int main(int argc, const char **argv) {
-  void *MainAddr = (void*) (intptr_t) GetExecutablePath;
-  llvm::sys::PrintStackTraceOnErrorSignal(argv[0]);
-
-  std::string
-    resourcesPath = CompilerInvocation::GetResourcesPath(argv[0], MainAddr);
-
-  int optargc = 0;
-  for (; optargc != argc; ++optargc) {
-    if (StringRef(argv[optargc]) == "--args")
-      break;
-  }
-  llvm::cl::ParseCommandLineOptions(optargc, argv, "arcmt-test");
-
-  if (VerifyTransformedFiles) {
-    if (ResultFiles.empty()) {
-      llvm::cl::PrintHelpMessage();
-      return 1;
-    }
-    return verifyTransformedFiles(ResultFiles);
-  }
-
-  if (optargc == argc) {
-    llvm::cl::PrintHelpMessage();
-    return 1;
-  }
-
-  ArrayRef<const char*> Args(argv+optargc+1, argc-optargc-1);
-
-  if (CheckOnly)
-    return checkForMigration(resourcesPath, Args);
-
-  return performTransformations(resourcesPath, Args);
-}
diff --git a/clang/tools/c-arcmt-test/CMakeLists.txt b/clang/tools/c-arcmt-test/CMakeLists.txt
deleted file mode 100644
index 08ac93c176db1..0000000000000
--- a/clang/tools/c-arcmt-test/CMakeLists.txt
+++ /dev/null
@@ -1,19 +0,0 @@
-add_clang_executable(c-arcmt-test
-  c-arcmt-test.c
-  )
-
-if (LLVM_BUILD_STATIC)
-  target_link_libraries(c-arcmt-test
-    PRIVATE
-    libclang_static
-    )
-else()
-  target_link_libraries(c-arcmt-test
-    PRIVATE
-    libclang
-    )
-endif()
-
-set_target_properties(c-arcmt-test
-  PROPERTIES
-  LINKER_LANGUAGE CXX)
diff --git a/clang/tools/c-arcmt-test/c-arcmt-test.c b/clang/tools/c-arcmt-test/c-arcmt-test.c
deleted file mode 100644
index 4d0c418714b95..0000000000000
--- a/clang/tools/c-arcmt-test/c-arcmt-test.c
+++ /dev/null
@@ -1,138 +0,0 @@
-/* c-arcmt-test.c */
-
-#include "clang-c/Index.h"
-#include "llvm/Support/AutoConvert.h"
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-#if defined(_WIN32)
-#include <io.h>
-#include <fcntl.h>
-#endif
-
-static int print_remappings(const char *path) {
-  CXRemapping remap;
-  unsigned i, N;
-  CXString origFname;
-  CXString transFname;
-
-  remap = clang_getRemappings(path);
-  if (!remap)
-    return 1;
-
-  N = clang_remap_getNumFiles(remap);
-  for (i = 0; i != N; ++i) {
-    clang_remap_getFilenames(remap, i, &origFname, &transFname);
-
-    fprintf(stdout, "%s\n", clang_getCString(origFname));
-    fprintf(stdout, "%s\n", clang_getCString(transFname));
-
-    clang_disposeString(origFname);
-    clang_disposeString(transFname);
-  }
-
-  clang_remap_dispose(remap);
-  return 0;
-}
-
-static int print_remappings_filelist(const char **files, unsigned numFiles) {
-  CXRemapping remap;
-  unsigned i, N;
-  CXString origFname;
-  CXString transFname;
-
-  remap = clang_getRemappingsFromFileList(files, numFiles);
-  if (!remap)
-    return 1;
-
-  N = clang_remap_getNumFiles(remap);
-  for (i = 0; i != N; ++i) {
-    clang_remap_getFilenames(remap, i, &origFname, &transFname);
-
-    fprintf(stdout, "%s\n", clang_getCString(origFname));
-    fprintf(stdout, "%s\n", clang_getCString(transFname));
-
-    clang_disposeString(origFname);
-    clang_disposeString(transFname);
-  }
-
-  clang_remap_dispose(remap);
-  return 0;
-}
-
-/******************************************************************************/
-/* Command line processing.                                                   */
-/******************************************************************************/
-
-static void print_usage(void) {
-  fprintf(stderr,
-    "usage: c-arcmt-test -mt-migrate-directory <path>\n"
-    "       c-arcmt-test <remap-file-path1> <remap-file-path2> ...\n\n\n");
-}
-
-/***/
-
-int carcmttest_main(int argc, const char **argv) {
-  clang_enableStackTraces();
-  if (argc == 3 && strncmp(argv[1], "-mt-migrate-directory", 21) == 0)
-    return print_remappings(argv[2]);
-
-  if (argc > 1)
-    return print_remappings_filelist(argv+1, argc-1);
-  
-  print_usage();
-  return 1;
-}
-
-/***/
-
-/* We intentionally run in a separate thread to ensure we at least minimal
- * testing of a multithreaded environment (for example, having a reduced stack
- * size). */
-
-typedef struct thread_info {
-  int argc;
-  const char **argv;
-  int result;
-} thread_info;
-void thread_runner(void *client_data_v) {
-  thread_info *client_data = client_data_v;
-  client_data->result = carcmttest_main(client_data->argc, client_data->argv);
-}
-
-static void flush_atexit(void) {
-  /* stdout, and surprisingly even stderr, are not always flushed on process
-   * and thread exit, particularly when the system is under heavy load. */
-  fflush(stdout);
-  fflush(stderr);
-}
-
-int main(int argc, const char **argv) {
-#ifdef __MVS__
-  if (enablezOSAutoConversion(fileno(stdout)) == -1)
-    fprintf(stderr, "Setting conversion on stdout failed\n");
-
-  if (enablezOSAutoConversion(fileno(stderr)) == -1)
-    fprintf(stderr, "Setting conversion on stderr failed\n");
-#endif
-
-  thread_info client_data;
-
-  atexit(flush_atexit);
-
-#if defined(_WIN32)
-  if (getenv("LIBCLANG_LOGGING") == NULL)
-    putenv("LIBCLANG_LOGGING=1");
-  _setmode( _fileno(stdout), _O_BINARY );
-#else
-  setenv("LIBCLANG_LOGGING", "1", /*overwrite=*/0);
-#endif
-
-  if (getenv("CINDEXTEST_NOTHREADS"))
-    return carcmttest_main(argc, argv);
-
-  client_data.argc = argc;
-  client_data.argv = argv;
-  clang_executeOnThread(thread_runner, &client_data, 0);
-  return client_data.result;
-}
diff --git a/clang/tools/clang-format/clang-format.el b/clang/tools/clang-format/clang-format.el
index fb943b7b722f8..b356e1bfa1436 100644
--- a/clang/tools/clang-format/clang-format.el
+++ b/clang/tools/clang-format/clang-format.el
@@ -32,6 +32,7 @@
 
 (require 'cl-lib)
 (require 'xml)
+(require 'vc-git)
 
 (defgroup clang-format nil
   "Format code using clang-format."
@@ -146,24 +147,133 @@ is a zero-based file offset, assuming ‘utf-8-unix’ coding."
     (lambda (byte &optional _quality _coding-system)
       (byte-to-position (1+ byte)))))
 
-;;;###autoload
-(defun clang-format-region (start end &optional style assume-file-name)
-  "Use clang-format to format the code between START and END according to STYLE.
-If called interactively uses the region or the current statement if there is no
-no active region. If no STYLE is given uses `clang-format-style'. Use
-ASSUME-FILE-NAME to locate a style config file, if no ASSUME-FILE-NAME is given
-uses the function `buffer-file-name'."
-  (interactive
-   (if (use-region-p)
-       (list (region-beginning) (region-end))
-     (list (point) (point))))
+(defmacro clang-format--with-delete-files-guard (bind-files-to-delete &rest body)
+  "Execute BODY which may add temp files to BIND-FILES-TO-DELETE."
+  (declare (indent 1))
+  `(let ((,bind-files-to-delete nil))
+     (unwind-protect
+         (progn
+           ,@body)
+       (while ,bind-files-to-delete
+         (with-demoted-errors "failed to remove file: %S"
+           (delete-file (pop ,bind-files-to-delete)))))))
+
+
+(defun clang-format--vc-diff-get-diff-lines (file-orig file-new)
+  "Return all line regions that contain diffs between FILE-ORIG and
+FILE-NEW.  If there is no diff ‘nil’ is returned. Otherwise the return
+is a ‘list’ of line ranges to format. The list of line ranges can be
+passed to ‘clang-format--region-impl’"
+  ;; Use temporary buffer for output of diff.
+  (with-temp-buffer
+    ;; We could use diff.el:diff-no-select here. The reason we don't
+    ;; is diff-no-select requires extra copies on the buffers which
+    ;; induces noticeable slowdowns, especially on larger files.
+    (let ((status (call-process
+                   diff-command
+                   nil
+                   (current-buffer)
+                   nil
+                   ;; Binary diff has different behaviors that we
+                   ;; aren't interested in.
+                   "-a"
+                   ;; Get minimal diff (copy diff config for git-clang-format).
+                   "-U0"
+                   file-orig
+                   file-new))
+          (stderr (concat (if (zerop (buffer-size)) "" ": ")
+                          (buffer-substring-no-properties
+                           (point-min) (line-end-position))))
+          (diff-lines '()))
+      (cond
+       ((stringp status)
+        (error "clang-format: (diff killed by signal %s%s)" status stderr))
+       ;; Return of 0 indicates no diff.
+       ((= status 0) nil)
+       ;; Return of 1 indicates found diffs and no error.
+       ((= status 1)
+        ;; Find and collect all diff lines.
+        ;; We are matching something like:
+        ;; "@@ -80 +80 @@" or "@@ -80,2 +80,2 @@"
+        (goto-char (point-min))
+        (while (re-search-forward
+                "^@@[[:blank:]]-[[:digit:],]+[[:blank:]]\\+\\([[:digit:]]+\\)\\(,\\([[:digit:]]+\\)\\)?[[:blank:]]@@$"
+                nil
+                t
+                1)
+          (let ((match1 (string-to-number (match-string 1)))
+                (match3 (let ((match3_or_nil (match-string 3)))
+                          (if match3_or_nil
+                              (string-to-number match3_or_nil)
+                            nil))))
+            (push (cons match1 (if match3 (+ match1 match3) match1)) diff-lines)))
+        (nreverse diff-lines))
+       ;; Any return != 0 && != 1 indicates some level of error.
+       (t
+        (error "clang-format: (diff returned unsuccessfully %s%s)" status stderr))))))
+
+(defun clang-format--vc-diff-get-vc-head-file (tmpfile-vc-head)
+  "Stores the contents of ‘buffer-file-name’ at vc revision HEAD into
+‘tmpfile-vc-head’. If the current buffer is either not a file or not
+in a vc repo, this results in an error. Currently git is the only
+supported vc."
+  ;; We need the current buffer to be a file.
+  (unless (buffer-file-name)
+    (error "clang-format: Buffer is not visiting a file"))
+
+  (let ((base-dir (vc-root-dir))
+        (backend (vc-backend (buffer-file-name))))
+    ;; We need to be able to find version control (git) root.
+    (unless base-dir
+      (error "clang-format: File not known to git"))
+    (cond
+     ((string-equal backend "Git")
+      ;; Get the filename relative to git root.
+      (let ((vc-file-name (substring
+                           (expand-file-name (buffer-file-name))
+                           (string-width (expand-file-name base-dir))
+                           nil)))
+        (let ((status (call-process
+                       vc-git-program
+                       nil
+                       `(:file ,tmpfile-vc-head)
+                       nil
+                       "show" (concat "HEAD:" vc-file-name)))
+              (stderr (with-temp-buffer
+                        (unless (zerop (cadr (insert-file-contents tmpfile-vc-head)))
+                          (insert ": "))
+                        (buffer-substring-no-properties
+                         (point-min) (line-end-position)))))
+          (when (stringp status)
+            (error "clang-format: (git show HEAD:%s killed by signal %s%s)"
+                   vc-file-name status stderr))
+          (unless (zerop status)
+            (error "clang-format: (git show HEAD:%s returned unsuccessfully %s%s)"
+                   vc-file-name status stderr)))))
+     (t
+      (error
+       "Version control %s isn't supported, currently supported backends: git"
+       backend)))))
+
 
+(defun clang-format--region-impl (start end &optional style assume-file-name lines)
+  "Common implementation for ‘clang-format-buffer’,
+‘clang-format-region’, and ‘clang-format-vc-diff’. START and END
+refer to the region to be formatter. STYLE and ASSUME-FILE-NAME are
+used for configuring the clang-format. And LINES is used to pass
+specific locations for reformatting (i.e diff locations)."
   (unless style
     (setq style clang-format-style))
 
   (unless assume-file-name
     (setq assume-file-name (buffer-file-name (buffer-base-buffer))))
 
+  ;; Convert list of line ranges to list command for ‘clang-format’ executable.
+  (when lines
+    (setq lines (mapcar (lambda (range)
+                          (format "--lines=%d:%d" (car range) (cdr range)))
+                        lines)))
+
   (let ((file-start (clang-format--bufferpos-to-filepos start 'approximate
                                                         'utf-8-unix))
         (file-end (clang-format--bufferpos-to-filepos end 'approximate
@@ -190,8 +300,12 @@ uses the function `buffer-file-name'."
                                       (list "--assume-filename" assume-file-name))
                                ,@(and style (list "--style" style))
                                "--fallback-style" ,clang-format-fallback-style
-                               "--offset" ,(number-to-string file-start)
-                               "--length" ,(number-to-string (- file-end file-start))
+                               ,@(and lines lines)
+                               ,@(and (not lines)
+                                      (list
+                                       "--offset" (number-to-string file-start)
+                                       "--length" (number-to-string
+                                                   (- file-end file-start))))
                                "--cursor" ,(number-to-string cursor))))
               (stderr (with-temp-buffer
                         (unless (zerop (cadr (insert-file-contents temp-file)))
@@ -216,9 +330,62 @@ uses the function `buffer-file-name'."
             (if incomplete-format
                 (message "(clang-format: incomplete (syntax errors)%s)" stderr)
               (message "(clang-format: success%s)" stderr))))
-      (delete-file temp-file)
+      (with-demoted-errors
+          "clang-format: Failed to delete temporary file: %S"
+        (delete-file temp-file))
       (when (buffer-name temp-buffer) (kill-buffer temp-buffer)))))
 
+
+;;;###autoload
+(defun clang-format-vc-diff (&optional style assume-file-name)
+  "The same as ‘clang-format-buffer’ but only operates on the vc
+diffs from HEAD in the buffer. If no STYLE is given uses
+‘clang-format-style’. Use ASSUME-FILE-NAME to locate a style config
+file. If no ASSUME-FILE-NAME is given uses the function
+‘buffer-file-name’."
+  (interactive)
+  (clang-format--with-delete-files-guard tmp-files
+    (let ((tmpfile-vc-head nil)
+          (tmpfile-curbuf nil))
+      (setq tmpfile-vc-head
+            (make-temp-file "clang-format-vc-tmp-head-content"))
+      (push tmpfile-vc-head tmp-files)
+      (clang-format--vc-diff-get-vc-head-file tmpfile-vc-head)
+      ;; Move the current buffer to a temporary file to take a
+      ;; diff. Even if current-buffer is backed by a file, we
+      ;; want to diff the buffer contents which might not be
+      ;; saved.
+      (setq tmpfile-curbuf (make-temp-file "clang-format-vc-tmp"))
+      (push tmpfile-curbuf tmp-files)
+      (write-region nil nil tmpfile-curbuf nil 'nomessage)
+      ;; Get a list of lines with a diff.
+      (let ((diff-lines
+             (clang-format--vc-diff-get-diff-lines
+              tmpfile-vc-head tmpfile-curbuf)))
+        ;; If we have any diffs, format them.
+        (when diff-lines
+          (clang-format--region-impl
+           (point-min)
+           (point-max)
+           style
+           assume-file-name
+           diff-lines))))))
+
+
+;;;###autoload
+(defun clang-format-region (start end &optional style assume-file-name)
+  "Use clang-format to format the code between START and END according
+to STYLE.  If called interactively uses the region or the current
+statement if there is no no active region. If no STYLE is given uses
+`clang-format-style'. Use ASSUME-FILE-NAME to locate a style config
+file, if no ASSUME-FILE-NAME is given uses the function
+`buffer-file-name'."
+  (interactive
+   (if (use-region-p)
+       (list (region-beginning) (region-end))
+     (list (point) (point))))
+  (clang-format--region-impl start end style assume-file-name))
+
 ;;;###autoload
 (defun clang-format-buffer (&optional style assume-file-name)
   "Use clang-format to format the current buffer according to STYLE.
@@ -226,7 +393,11 @@ If no STYLE is given uses `clang-format-style'. Use ASSUME-FILE-NAME
 to locate a style config file. If no ASSUME-FILE-NAME is given uses
 the function `buffer-file-name'."
   (interactive)
-  (clang-format-region (point-min) (point-max) style assume-file-name))
+  (clang-format--region-impl
+   (point-min)
+   (point-max)
+   style
+   assume-file-name))
 
 ;;;###autoload
 (defalias 'clang-format 'clang-format-region)
diff --git a/clang/tools/clang-installapi/Options.cpp b/clang/tools/clang-installapi/Options.cpp
index 8a2c3463189fa..0dddcfce58ca6 100644
--- a/clang/tools/clang-installapi/Options.cpp
+++ b/clang/tools/clang-installapi/Options.cpp
@@ -263,11 +263,12 @@ bool Options::processInstallAPIXOptions(InputArgList &Args) {
     }
     const StringRef ASpelling = NextA->getSpelling();
     const auto &AValues = NextA->getValues();
+    auto &UniqueArgs = FEOpts.UniqueArgs[Label];
     if (AValues.empty())
-      FEOpts.UniqueArgs[Label].emplace_back(ASpelling.str());
+      UniqueArgs.emplace_back(ASpelling.str());
     else
       for (const StringRef Val : AValues)
-        FEOpts.UniqueArgs[Label].emplace_back((ASpelling + Val).str());
+        UniqueArgs.emplace_back((ASpelling + Val).str());
 
     A->claim();
     NextA->claim();
@@ -608,32 +609,37 @@ Options::processAndFilterOutInstallAPIOptions(ArrayRef<const char *> Args) {
       ParsedArgs.hasArg(OPT_not_for_dyld_shared_cache);
 
   for (const Arg *A : ParsedArgs.filtered(OPT_allowable_client)) {
+    auto It = ArgToArchMap.find(A);
     LinkerOpts.AllowableClients[A->getValue()] =
-        ArgToArchMap.count(A) ? ArgToArchMap[A] : ArchitectureSet();
+        It != ArgToArchMap.end() ? It->second : ArchitectureSet();
     A->claim();
   }
 
   for (const Arg *A : ParsedArgs.filtered(OPT_reexport_l)) {
+    auto It = ArgToArchMap.find(A);
     LinkerOpts.ReexportedLibraries[A->getValue()] =
-        ArgToArchMap.count(A) ? ArgToArchMap[A] : ArchitectureSet();
+        It != ArgToArchMap.end() ? It->second : ArchitectureSet();
     A->claim();
   }
 
   for (const Arg *A : ParsedArgs.filtered(OPT_reexport_library)) {
+    auto It = ArgToArchMap.find(A);
     LinkerOpts.ReexportedLibraryPaths[A->getValue()] =
-        ArgToArchMap.count(A) ? ArgToArchMap[A] : ArchitectureSet();
+        It != ArgToArchMap.end() ? It->second : ArchitectureSet();
     A->claim();
   }
 
   for (const Arg *A : ParsedArgs.filtered(OPT_reexport_framework)) {
+    auto It = ArgToArchMap.find(A);
     LinkerOpts.ReexportedFrameworks[A->getValue()] =
-        ArgToArchMap.count(A) ? ArgToArchMap[A] : ArchitectureSet();
+        It != ArgToArchMap.end() ? It->second : ArchitectureSet();
     A->claim();
   }
 
   for (const Arg *A : ParsedArgs.filtered(OPT_rpath)) {
+    auto It = ArgToArchMap.find(A);
     LinkerOpts.RPaths[A->getValue()] =
-        ArgToArchMap.count(A) ? ArgToArchMap[A] : ArchitectureSet();
+        It != ArgToArchMap.end() ? It->second : ArchitectureSet();
     A->claim();
   }
 
diff --git a/clang/tools/clang-linker-wrapper/ClangLinkerWrapper.cpp b/clang/tools/clang-linker-wrapper/ClangLinkerWrapper.cpp
index c92590581a645..aa43b2f5f2a1b 100644
--- a/clang/tools/clang-linker-wrapper/ClangLinkerWrapper.cpp
+++ b/clang/tools/clang-linker-wrapper/ClangLinkerWrapper.cpp
@@ -474,8 +474,6 @@ Expected<StringRef> clang(ArrayRef<StringRef> InputFiles, const ArgList &Args) {
 
   const llvm::Triple Triple(Args.getLastArgValue(OPT_triple_EQ));
   StringRef Arch = Args.getLastArgValue(OPT_arch_EQ);
-  if (Arch.empty())
-    Arch = "native";
   // Create a new file to write the linked device image to. Assume that the
   // input filename already has the device and architecture.
   auto TempFileOrErr =
@@ -492,11 +490,14 @@ Expected<StringRef> clang(ArrayRef<StringRef> InputFiles, const ArgList &Args) {
       "-o",
       *TempFileOrErr,
       Args.MakeArgString("--target=" + Triple.getTriple()),
-      Triple.isAMDGPU() ? Args.MakeArgString("-mcpu=" + Arch)
-                        : Args.MakeArgString("-march=" + Arch),
-      Args.MakeArgString("-" + OptLevel),
   };
 
+  if (!Arch.empty())
+    Triple.isAMDGPU() ? CmdArgs.push_back(Args.MakeArgString("-mcpu=" + Arch))
+                      : CmdArgs.push_back(Args.MakeArgString("-march=" + Arch));
+
+  CmdArgs.push_back(Args.MakeArgString("-" + OptLevel));
+
   // Forward all of the `--offload-opt` and similar options to the device.
   CmdArgs.push_back("-flto");
   for (auto &Arg : Args.filtered(OPT_offload_opt_eq_minus, OPT_mllvm))
@@ -605,6 +606,15 @@ Expected<StringRef> linkDevice(ArrayRef<StringRef> InputFiles,
   }
 }
 
+Error containerizeRawImage(std::unique_ptr<MemoryBuffer> &Img, OffloadKind Kind,
+                           const ArgList &Args) {
+  llvm::Triple Triple(Args.getLastArgValue(OPT_triple_EQ));
+  if (Kind != OFK_OpenMP || !Triple.isSPIRV() ||
+      Triple.getVendor() != llvm::Triple::Intel)
+    return Error::success();
+  return offloading::intel::containerizeOpenMPSPIRVImage(Img);
+}
+
 Expected<StringRef> writeOffloadFile(const OffloadFile &File) {
   const OffloadBinary &Binary = *File.getBinary();
 
@@ -691,22 +701,19 @@ wrapDeviceImages(ArrayRef<std::unique_ptr<MemoryBuffer>> Buffers,
   switch (Kind) {
   case OFK_OpenMP:
     if (Error Err = offloading::wrapOpenMPBinaries(
-            M, BuffersToWrap,
-            offloading::getOffloadEntryArray(M, "omp_offloading_entries"),
+            M, BuffersToWrap, offloading::getOffloadEntryArray(M),
             /*Suffix=*/"", /*Relocatable=*/Args.hasArg(OPT_relocatable)))
       return std::move(Err);
     break;
   case OFK_Cuda:
     if (Error Err = offloading::wrapCudaBinary(
-            M, BuffersToWrap.front(),
-            offloading::getOffloadEntryArray(M, "cuda_offloading_entries"),
+            M, BuffersToWrap.front(), offloading::getOffloadEntryArray(M),
             /*Suffix=*/"", /*EmitSurfacesAndTextures=*/false))
       return std::move(Err);
     break;
   case OFK_HIP:
     if (Error Err = offloading::wrapHIPBinary(
-            M, BuffersToWrap.front(),
-            offloading::getOffloadEntryArray(M, "hip_offloading_entries")))
+            M, BuffersToWrap.front(), offloading::getOffloadEntryArray(M)))
       return std::move(Err);
     break;
   default:
@@ -820,8 +827,9 @@ DerivedArgList getLinkerArgs(ArrayRef<OffloadFile> Input,
 
   // Set the subarchitecture and target triple for this compilation.
   const OptTable &Tbl = getOptTable();
+  StringRef Arch = Args.MakeArgString(Input.front().getBinary()->getArch());
   DAL.AddJoinedArg(nullptr, Tbl.getOption(OPT_arch_EQ),
-                   Args.MakeArgString(Input.front().getBinary()->getArch()));
+                   Arch == "generic" ? "" : Arch);
   DAL.AddJoinedArg(nullptr, Tbl.getOption(OPT_triple_EQ),
                    Args.MakeArgString(Input.front().getBinary()->getTriple()));
 
@@ -960,6 +968,10 @@ Expected<SmallVector<StringRef>> linkAndWrapDeviceFiles(
           return createFileError(*OutputOrErr, EC);
       }
 
+      // Manually containerize offloading images not in ELF format.
+      if (Error E = containerizeRawImage(*FileOrErr, Kind, LinkerArgs))
+        return E;
+
       std::scoped_lock<decltype(ImageMtx)> Guard(ImageMtx);
       OffloadingImage TheImage{};
       TheImage.TheImageKind =
diff --git a/clang/tools/clang-scan-deps/ClangScanDeps.cpp b/clang/tools/clang-scan-deps/ClangScanDeps.cpp
index 709dc513be281..4b39ae9c35c04 100644
--- a/clang/tools/clang-scan-deps/ClangScanDeps.cpp
+++ b/clang/tools/clang-scan-deps/ClangScanDeps.cpp
@@ -23,6 +23,7 @@
 #include "llvm/Support/Format.h"
 #include "llvm/Support/JSON.h"
 #include "llvm/Support/LLVMDriver.h"
+#include "llvm/Support/MemoryBuffer.h"
 #include "llvm/Support/Program.h"
 #include "llvm/Support/Signals.h"
 #include "llvm/Support/TargetSelect.h"
@@ -31,6 +32,7 @@
 #include "llvm/Support/Timer.h"
 #include "llvm/Support/VirtualFileSystem.h"
 #include "llvm/TargetParser/Host.h"
+#include <memory>
 #include <mutex>
 #include <optional>
 #include <thread>
@@ -85,8 +87,9 @@ static std::string ModuleFilesDir;
 static bool EagerLoadModules;
 static unsigned NumThreads = 0;
 static std::string CompilationDB;
-static std::string ModuleName;
+static std::optional<std::string> ModuleName;
 static std::vector<std::string> ModuleDepTargets;
+static std::string TranslationUnitFile;
 static bool DeprecatedDriverCommand;
 static ResourceDirRecipeKind ResourceDirRecipe;
 static bool Verbose;
@@ -164,6 +167,8 @@ static void ParseArgs(int argc, char **argv) {
             .Case("system-warnings", ScanningOptimizations::SystemWarnings)
             .Case("vfs", ScanningOptimizations::VFS)
             .Case("canonicalize-macros", ScanningOptimizations::Macros)
+            .Case("ignore-current-working-dir",
+                  ScanningOptimizations::IgnoreCWD)
             .Case("all", ScanningOptimizations::All)
             .Default(std::nullopt);
     if (!Optimization) {
@@ -204,6 +209,9 @@ static void ParseArgs(int argc, char **argv) {
   for (const llvm::opt::Arg *A : Args.filtered(OPT_dependency_target_EQ))
     ModuleDepTargets.emplace_back(A->getValue());
 
+  if (const llvm::opt::Arg *A = Args.getLastArg(OPT_tu_buffer_path_EQ))
+    TranslationUnitFile = A->getValue();
+
   DeprecatedDriverCommand = Args.hasArg(OPT_deprecated_driver_command);
 
   if (const llvm::opt::Arg *A = Args.getLastArg(OPT_resource_dir_recipe_EQ)) {
@@ -940,7 +948,7 @@ int clang_scan_deps_main(int argc, char **argv, const llvm::ToolContext &) {
   };
 
   if (Format == ScanningOutputFormat::Full)
-    FD.emplace(ModuleName.empty() ? Inputs.size() : 0);
+    FD.emplace(!ModuleName ? Inputs.size() : 0);
 
   std::atomic<size_t> NumStatusCalls = 0;
   std::atomic<size_t> NumOpenFileForReadCalls = 0;
@@ -959,10 +967,6 @@ int clang_scan_deps_main(int argc, char **argv, const llvm::ToolContext &) {
       std::string Filename = std::move(Input->Filename);
       std::string CWD = std::move(Input->Directory);
 
-      std::optional<StringRef> MaybeModuleName;
-      if (!ModuleName.empty())
-        MaybeModuleName = ModuleName;
-
       std::string OutputDir(ModuleFilesDir);
       if (OutputDir.empty())
         OutputDir = getModuleCachePath(Input->CommandLine);
@@ -1018,16 +1022,31 @@ int clang_scan_deps_main(int argc, char **argv, const llvm::ToolContext &) {
                                              MakeformatOS, Errs))
             HadErrors = true;
         }
-      } else if (MaybeModuleName) {
+      } else if (ModuleName) {
         auto MaybeModuleDepsGraph = WorkerTool.getModuleDependencies(
-            *MaybeModuleName, Input->CommandLine, CWD, AlreadySeenModules,
+            *ModuleName, Input->CommandLine, CWD, AlreadySeenModules,
             LookupOutput);
-        if (handleModuleResult(*MaybeModuleName, MaybeModuleDepsGraph, *FD,
+        if (handleModuleResult(*ModuleName, MaybeModuleDepsGraph, *FD,
                                LocalIndex, DependencyOS, Errs))
           HadErrors = true;
       } else {
+        std::unique_ptr<llvm::MemoryBuffer> TU;
+        std::optional<llvm::MemoryBufferRef> TUBuffer;
+        if (!TranslationUnitFile.empty()) {
+          auto MaybeTU = llvm::MemoryBuffer::getFile(TranslationUnitFile);
+          if (!MaybeTU) {
+            llvm::errs() << "cannot open input translation unit: "
+                         << MaybeTU.getError().message() << "\n";
+            HadErrors = true;
+            continue;
+          }
+          TU = std::move(*MaybeTU);
+          TUBuffer = TU->getMemBufferRef();
+          Filename = TU->getBufferIdentifier();
+        }
         auto MaybeTUDeps = WorkerTool.getTranslationUnitDependencies(
-            Input->CommandLine, CWD, AlreadySeenModules, LookupOutput);
+            Input->CommandLine, CWD, AlreadySeenModules, LookupOutput,
+            TUBuffer);
         if (handleTranslationUnitResult(Filename, MaybeTUDeps, *FD, LocalIndex,
                                         DependencyOS, Errs))
           HadErrors = true;
diff --git a/clang/tools/clang-scan-deps/Opts.td b/clang/tools/clang-scan-deps/Opts.td
index 4837ce6f070d7..9cccbb3aaf0c8 100644
--- a/clang/tools/clang-scan-deps/Opts.td
+++ b/clang/tools/clang-scan-deps/Opts.td
@@ -29,6 +29,8 @@ defm compilation_database : Eq<"compilation-database", "Compilation database">;
 defm module_name : Eq<"module-name", "the module of which the dependencies are to be computed">;
 defm dependency_target : Eq<"dependency-target", "The names of dependency targets for the dependency file">;
 
+defm tu_buffer_path: Eq<"tu-buffer-path", "The path to the translation unit for depscan. Not compatible with -module-name">;
+
 def deprecated_driver_command : F<"deprecated-driver-command", "use a single driver command to build the tu (deprecated)">;
 
 defm resource_dir_recipe : Eq<"resource-dir-recipe", "How to produce missing '-resource-dir' argument">;
diff --git a/clang/tools/diagtool/ListWarnings.cpp b/clang/tools/diagtool/ListWarnings.cpp
index a71f6e3a66c8e..9f9647126dd8a 100644
--- a/clang/tools/diagtool/ListWarnings.cpp
+++ b/clang/tools/diagtool/ListWarnings.cpp
@@ -53,13 +53,13 @@ int ListWarnings::run(unsigned int argc, char **argv, llvm::raw_ostream &out) {
   for (const DiagnosticRecord &DR : getBuiltinDiagnosticsByName()) {
     const unsigned diagID = DR.DiagID;
 
-    if (DiagnosticIDs::isBuiltinNote(diagID))
+    if (DiagnosticIDs{}.isNote(diagID))
       continue;
 
-    if (!DiagnosticIDs::isBuiltinWarningOrExtension(diagID))
+    if (!DiagnosticIDs{}.isWarningOrExtension(diagID))
       continue;
 
-    Entry entry(DR.getName(), DiagnosticIDs::getWarningOptionForDiag(diagID));
+    Entry entry(DR.getName(), DiagnosticIDs{}.getWarningOptionForDiag(diagID));
 
     if (entry.Flag.empty())
       Unflagged.push_back(entry);
@@ -97,4 +97,3 @@ int ListWarnings::run(unsigned int argc, char **argv, llvm::raw_ostream &out) {
 
   return 0;
 }
-
diff --git a/clang/tools/diagtool/ShowEnabledWarnings.cpp b/clang/tools/diagtool/ShowEnabledWarnings.cpp
index 48bed7c828c16..1f32f791de082 100644
--- a/clang/tools/diagtool/ShowEnabledWarnings.cpp
+++ b/clang/tools/diagtool/ShowEnabledWarnings.cpp
@@ -119,10 +119,10 @@ int ShowEnabledWarnings::run(unsigned int argc, char **argv, raw_ostream &Out) {
   for (const DiagnosticRecord &DR : getBuiltinDiagnosticsByName()) {
     unsigned DiagID = DR.DiagID;
 
-    if (DiagnosticIDs::isBuiltinNote(DiagID))
+    if (DiagnosticIDs{}.isNote(DiagID))
       continue;
 
-    if (!DiagnosticIDs::isBuiltinWarningOrExtension(DiagID))
+    if (!DiagnosticIDs{}.isWarningOrExtension(DiagID))
       continue;
 
     DiagnosticsEngine::Level DiagLevel =
@@ -130,7 +130,7 @@ int ShowEnabledWarnings::run(unsigned int argc, char **argv, raw_ostream &Out) {
     if (DiagLevel == DiagnosticsEngine::Ignored)
       continue;
 
-    StringRef WarningOpt = DiagnosticIDs::getWarningOptionForDiag(DiagID);
+    StringRef WarningOpt = DiagnosticIDs{}.getWarningOptionForDiag(DiagID);
     Active.push_back(PrettyDiag(DR.getName(), WarningOpt, DiagLevel));
   }
 
diff --git a/clang/tools/driver/CMakeLists.txt b/clang/tools/driver/CMakeLists.txt
index a4debc2dd2e89..ad336fcc45b60 100644
--- a/clang/tools/driver/CMakeLists.txt
+++ b/clang/tools/driver/CMakeLists.txt
@@ -23,6 +23,18 @@ if(CLANG_PLUGIN_SUPPORT)
   set(support_plugins SUPPORT_PLUGINS)
 endif()
 
+set(CLANG_BOLT OFF CACHE STRING "Apply BOLT optimization to Clang. \
+  May be specified as Instrument or Perf or LBR to use a particular profiling \
+  mechanism.")
+string(TOUPPER "${CLANG_BOLT}" CLANG_BOLT)
+
+if (CLANG_BOLT AND NOT LLVM_BUILD_INSTRUMENTED)
+  set(CLANG_BOLT_DEPS clear-bolt-fdata llvm-bolt llvm-readobj)
+  if (NOT CLANG_BOLT STREQUAL "INSTRUMENT")
+    list(APPEND CLANG_BOLT_DEPS clear-perf-data)
+  endif()
+endif()
+
 add_clang_tool(clang
   driver.cpp
   cc1_main.cpp
@@ -35,6 +47,7 @@ add_clang_tool(clang
   ARMTargetParserTableGen
   AArch64TargetParserTableGen
   ${support_plugins}
+  ${CLANG_BOLT_DEPS}
   GENERATE_DRIVER
   )
 
@@ -134,3 +147,42 @@ if(CLANG_ORDER_FILE AND
     set_target_properties(clang PROPERTIES LINK_DEPENDS ${CLANG_ORDER_FILE})
   endif()
 endif()
+
+if (CLANG_BOLT AND NOT LLVM_BUILD_INSTRUMENTED)
+  # Add a clang-bolt target for backwards compatibility.
+  add_custom_target(clang-bolt DEPENDS clang)
+
+  set(CLANG_BOLT_INSTRUMENTED "clang-bolt.inst" CACHE STRING
+    "Name of BOLT-instrumented Clang binary")
+  set(CLANG_INSTRUMENTED ${LLVM_RUNTIME_OUTPUT_INTDIR}/${CLANG_BOLT_INSTRUMENTED})
+  set(PERF_TRAINING_BINARY_DIR ${CMAKE_CURRENT_BINARY_DIR}/../../utils/perf-training)
+  set(BOLT_FDATA ${PERF_TRAINING_BINARY_DIR}/prof.fdata)
+  get_llvm_lit_path(
+    lit_base_dir
+    lit_file_name
+    ALLOW_EXTERNAL
+  )
+  set(LIT_COMMAND "${lit_base_dir}/${lit_file_name}")
+
+  # This POST_BUILD command is executed unconditionally even if the clang target
+  # is already built.  We need to wrap the whole bolt optimization process in
+  # a single python wrapper, so that we can first check if the binary has
+  # already been optimized and then exit early with a 0 status if it has.
+  add_custom_command(
+    TARGET clang POST_BUILD
+    COMMAND  "${Python3_EXECUTABLE}" ${CMAKE_CURRENT_SOURCE_DIR}/../../utils/perf-training/perf-helper.py
+             bolt-optimize
+	     --method ${CLANG_BOLT}
+	     --input $<TARGET_FILE:clang>
+	     --instrumented-output ${CLANG_INSTRUMENTED}
+	     --fdata ${BOLT_FDATA}
+	     --perf-training-binary-dir ${PERF_TRAINING_BINARY_DIR}
+	     --readelf $<TARGET_FILE:llvm-readobj>
+	     --bolt $<TARGET_FILE:llvm-bolt>
+	     --lit "${LIT_COMMAND}"
+	     --merge-fdata $<TARGET_FILE:merge-fdata>
+    COMMENT "Optimizing Clang with BOLT"
+    USES_TERMINAL
+    VERBATIM
+  )
+endif()
diff --git a/clang/tools/driver/driver.cpp b/clang/tools/driver/driver.cpp
index 74923247b7ee1..00c00cea16f47 100644
--- a/clang/tools/driver/driver.cpp
+++ b/clang/tools/driver/driver.cpp
@@ -318,6 +318,10 @@ int clang_main(int Argc, char **Argv, const llvm::ToolContext &ToolContext) {
 
   IntrusiveRefCntPtr<DiagnosticOptions> DiagOpts =
       CreateAndPopulateDiagOpts(Args);
+  // Driver's diagnostics don't use suppression mappings, so don't bother
+  // parsing them. CC1 still receives full args, so this doesn't impact other
+  // actions.
+  DiagOpts->DiagnosticSuppressionMappingsFile.clear();
 
   TextDiagnosticPrinter *DiagClient
     = new TextDiagnosticPrinter(llvm::errs(), &*DiagOpts);
diff --git a/clang/tools/libclang/ARCMigrate.cpp b/clang/tools/libclang/ARCMigrate.cpp
deleted file mode 100644
index da8a7e4b9130b..0000000000000
--- a/clang/tools/libclang/ARCMigrate.cpp
+++ /dev/null
@@ -1,138 +0,0 @@
-//===- ARCMigrate.cpp - Clang-C ARC Migration Library ---------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-//
-// This file implements the main API hooks in the Clang-C ARC Migration library.
-//
-//===----------------------------------------------------------------------===//
-
-#include "clang-c/Index.h"
-#include "CXString.h"
-#include "clang/ARCMigrate/ARCMT.h"
-#include "clang/Config/config.h"
-#include "clang/Frontend/TextDiagnosticBuffer.h"
-#include "llvm/Support/FileSystem.h"
-
-using namespace clang;
-using namespace arcmt;
-
-namespace {
-
-struct Remap {
-  std::vector<std::pair<std::string, std::string> > Vec;
-};
-
-} // anonymous namespace.
-
-//===----------------------------------------------------------------------===//
-// libClang public APIs.
-//===----------------------------------------------------------------------===//
-
-CXRemapping clang_getRemappings(const char *migrate_dir_path) {
-#if !CLANG_ENABLE_ARCMT
-  llvm::errs() << "error: feature not enabled in this build\n";
-  return nullptr;
-#else
-  bool Logging = ::getenv("LIBCLANG_LOGGING");
-
-  if (!migrate_dir_path) {
-    if (Logging)
-      llvm::errs() << "clang_getRemappings was called with NULL parameter\n";
-    return nullptr;
-  }
-
-  if (!llvm::sys::fs::exists(migrate_dir_path)) {
-    if (Logging) {
-      llvm::errs() << "Error by clang_getRemappings(\"" << migrate_dir_path
-                   << "\")\n";
-      llvm::errs() << "\"" << migrate_dir_path << "\" does not exist\n";
-    }
-    return nullptr;
-  }
-
-  TextDiagnosticBuffer diagBuffer;
-  std::unique_ptr<Remap> remap(new Remap());
-
-  bool err = arcmt::getFileRemappings(remap->Vec, migrate_dir_path,&diagBuffer);
-
-  if (err) {
-    if (Logging) {
-      llvm::errs() << "Error by clang_getRemappings(\"" << migrate_dir_path
-                   << "\")\n";
-      for (TextDiagnosticBuffer::const_iterator
-             I = diagBuffer.err_begin(), E = diagBuffer.err_end(); I != E; ++I)
-        llvm::errs() << I->second << '\n';
-    }
-    return nullptr;
-  }
-
-  return remap.release();
-#endif
-}
-
-CXRemapping clang_getRemappingsFromFileList(const char **filePaths,
-                                            unsigned numFiles) {
-#if !CLANG_ENABLE_ARCMT
-  llvm::errs() << "error: feature not enabled in this build\n";
-  return nullptr;
-#else
-  bool Logging = ::getenv("LIBCLANG_LOGGING");
-
-  std::unique_ptr<Remap> remap(new Remap());
-
-  if (numFiles == 0) {
-    if (Logging)
-      llvm::errs() << "clang_getRemappingsFromFileList was called with "
-                      "numFiles=0\n";
-    return remap.release();
-  }
-
-  if (!filePaths) {
-    if (Logging)
-      llvm::errs() << "clang_getRemappingsFromFileList was called with "
-                      "NULL filePaths\n";
-    return nullptr;
-  }
-
-  TextDiagnosticBuffer diagBuffer;
-  SmallVector<StringRef, 32> Files(filePaths, filePaths + numFiles);
-
-  bool err = arcmt::getFileRemappingsFromFileList(remap->Vec, Files,
-                                                  &diagBuffer);
-
-  if (err) {
-    if (Logging) {
-      llvm::errs() << "Error by clang_getRemappingsFromFileList\n";
-      for (TextDiagnosticBuffer::const_iterator
-             I = diagBuffer.err_begin(), E = diagBuffer.err_end(); I != E; ++I)
-        llvm::errs() << I->second << '\n';
-    }
-    return remap.release();
-  }
-
-  return remap.release();
-#endif
-}
-
-unsigned clang_remap_getNumFiles(CXRemapping map) {
-  return static_cast<Remap *>(map)->Vec.size();
-  
-}
-
-void clang_remap_getFilenames(CXRemapping map, unsigned index,
-                              CXString *original, CXString *transformed) {
-  if (original)
-    *original = cxstring::createDup(
-                    static_cast<Remap *>(map)->Vec[index].first);
-  if (transformed)
-    *transformed = cxstring::createDup(
-                    static_cast<Remap *>(map)->Vec[index].second);
-}
-
-void clang_remap_dispose(CXRemapping map) {
-  delete static_cast<Remap *>(map);
-}
diff --git a/clang/tools/libclang/CIndex.cpp b/clang/tools/libclang/CIndex.cpp
index 42f095fea2db2..285ac31420007 100644
--- a/clang/tools/libclang/CIndex.cpp
+++ b/clang/tools/libclang/CIndex.cpp
@@ -2193,6 +2193,8 @@ class EnqueueVisitor : public ConstStmtVisitor<EnqueueVisitor, void>,
   void VisitOpenACCInitConstruct(const OpenACCInitConstruct *D);
   void VisitOpenACCShutdownConstruct(const OpenACCShutdownConstruct *D);
   void VisitOpenACCSetConstruct(const OpenACCSetConstruct *D);
+  void VisitOpenACCUpdateConstruct(const OpenACCUpdateConstruct *D);
+  void VisitOpenACCAtomicConstruct(const OpenACCAtomicConstruct *D);
   void VisitOMPExecutableDirective(const OMPExecutableDirective *D);
   void VisitOMPLoopBasedDirective(const OMPLoopBasedDirective *D);
   void VisitOMPLoopDirective(const OMPLoopDirective *D);
@@ -2463,6 +2465,9 @@ void OMPClauseEnqueue::VisitOMPNoOpenMPClause(const OMPNoOpenMPClause *) {}
 void OMPClauseEnqueue::VisitOMPNoOpenMPRoutinesClause(
     const OMPNoOpenMPRoutinesClause *) {}
 
+void OMPClauseEnqueue::VisitOMPNoOpenMPConstructsClause(
+    const OMPNoOpenMPConstructsClause *) {}
+
 void OMPClauseEnqueue::VisitOMPNoParallelismClause(
     const OMPNoParallelismClause *) {}
 
@@ -3682,6 +3687,18 @@ void EnqueueVisitor::VisitOpenACCSetConstruct(const OpenACCSetConstruct *C) {
     EnqueueChildren(Clause);
 }
 
+void EnqueueVisitor::VisitOpenACCUpdateConstruct(
+    const OpenACCUpdateConstruct *C) {
+  EnqueueChildren(C);
+  for (auto *Clause : C->clauses())
+    EnqueueChildren(Clause);
+}
+
+void EnqueueVisitor::VisitOpenACCAtomicConstruct(
+    const OpenACCAtomicConstruct *C) {
+  EnqueueChildren(C);
+}
+
 void EnqueueVisitor::VisitAnnotateAttr(const AnnotateAttr *A) {
   EnqueueChildren(A);
 }
@@ -5356,12 +5373,12 @@ CXString clang_getCursorSpelling(CXCursor C) {
 
     case CXCursor_OverloadedDeclRef: {
       OverloadedDeclRefStorage Storage = getCursorOverloadedDeclRef(C).first;
-      if (const Decl *D = Storage.dyn_cast<const Decl *>()) {
+      if (const Decl *D = dyn_cast<const Decl *>(Storage)) {
         if (const NamedDecl *ND = dyn_cast<NamedDecl>(D))
           return cxstring::createDup(ND->getNameAsString());
         return cxstring::createEmpty();
       }
-      if (const OverloadExpr *E = Storage.dyn_cast<const OverloadExpr *>())
+      if (const OverloadExpr *E = dyn_cast<const OverloadExpr *>(Storage))
         return cxstring::createDup(E->getName().getAsString());
       OverloadedTemplateStorage *Ovl =
           cast<OverloadedTemplateStorage *>(Storage);
@@ -6454,6 +6471,8 @@ CXString clang_getCursorKindSpelling(enum CXCursorKind Kind) {
     return cxstring::createRef("OpenACCSetConstruct");
   case CXCursor_OpenACCUpdateConstruct:
     return cxstring::createRef("OpenACCUpdateConstruct");
+  case CXCursor_OpenACCAtomicConstruct:
+    return cxstring::createRef("OpenACCAtomicConstruct");
   }
 
   llvm_unreachable("Unhandled CXCursorKind");
@@ -7415,11 +7434,11 @@ unsigned clang_getNumOverloadedDecls(CXCursor C) {
     return 0;
 
   OverloadedDeclRefStorage Storage = getCursorOverloadedDeclRef(C).first;
-  if (const OverloadExpr *E = Storage.dyn_cast<const OverloadExpr *>())
+  if (const OverloadExpr *E = dyn_cast<const OverloadExpr *>(Storage))
     return E->getNumDecls();
 
   if (OverloadedTemplateStorage *S =
-          Storage.dyn_cast<OverloadedTemplateStorage *>())
+          dyn_cast<OverloadedTemplateStorage *>(Storage))
     return S->size();
 
   const Decl *D = cast<const Decl *>(Storage);
@@ -7438,11 +7457,11 @@ CXCursor clang_getOverloadedDecl(CXCursor cursor, unsigned index) {
 
   CXTranslationUnit TU = getCursorTU(cursor);
   OverloadedDeclRefStorage Storage = getCursorOverloadedDeclRef(cursor).first;
-  if (const OverloadExpr *E = Storage.dyn_cast<const OverloadExpr *>())
+  if (const OverloadExpr *E = dyn_cast<const OverloadExpr *>(Storage))
     return MakeCXCursor(E->decls_begin()[index], TU);
 
   if (OverloadedTemplateStorage *S =
-          Storage.dyn_cast<OverloadedTemplateStorage *>())
+          dyn_cast<OverloadedTemplateStorage *>(Storage))
     return MakeCXCursor(S->begin()[index], TU);
 
   const Decl *D = cast<const Decl *>(Storage);
diff --git a/clang/tools/libclang/CIndexDiagnostic.cpp b/clang/tools/libclang/CIndexDiagnostic.cpp
index 34792d5bdfaaf..92271d9c37f86 100644
--- a/clang/tools/libclang/CIndexDiagnostic.cpp
+++ b/clang/tools/libclang/CIndexDiagnostic.cpp
@@ -92,7 +92,8 @@ class CXDiagnosticRenderer : public DiagnosticNoteRenderer {
   void beginDiagnostic(DiagOrStoredDiag D,
                        DiagnosticsEngine::Level Level) override {
 
-    const StoredDiagnostic *SD = D.dyn_cast<const StoredDiagnostic*>();
+    const StoredDiagnostic *SD =
+        dyn_cast_if_present<const StoredDiagnostic *>(D);
     if (!SD)
       return;
     
diff --git a/clang/tools/libclang/CMakeLists.txt b/clang/tools/libclang/CMakeLists.txt
index 00a1223c0831a..299c14660f3d4 100644
--- a/clang/tools/libclang/CMakeLists.txt
+++ b/clang/tools/libclang/CMakeLists.txt
@@ -20,7 +20,6 @@ endif()
 #       to clean up previous inconsistencies.
 
 set(SOURCES
-  ARCMigrate.cpp
   BuildSystem.cpp
   CIndex.cpp
   CIndexCXX.cpp
@@ -71,10 +70,6 @@ set(LIBS
   clangTooling
 )
 
-if (CLANG_ENABLE_ARCMT)
-  list(APPEND LIBS clangARCMigrate)
-endif ()
-
 if (HAVE_LIBDL)
   list(APPEND LIBS ${CMAKE_DL_LIBS})
 elseif (CLANG_BUILT_STANDALONE)
@@ -208,9 +203,6 @@ if(ENABLE_SHARED)
       target_link_options(libclang PRIVATE "-Wl,--version-script,${CMAKE_CURRENT_SOURCE_DIR}/libclang.map")
     endif()
     # Ensure that libclang.so gets rebuilt when the linker script changes.
-    set_property(SOURCE ARCMigrate.cpp APPEND PROPERTY
-                 OBJECT_DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/libclang.map)
-
     set_target_properties(libclang PROPERTIES
                           VERSION ${LLVM_VERSION_MAJOR}.${LLVM_VERSION_MINOR}.${LLVM_VERSION_PATCH}${LLVM_VERSION_SUFFIX}
                           ${LIBCLANG_SOVERSION_ARG})
diff --git a/clang/tools/libclang/CXCursor.cpp b/clang/tools/libclang/CXCursor.cpp
index b9f0b089e41b0..127f22bc5bdca 100644
--- a/clang/tools/libclang/CXCursor.cpp
+++ b/clang/tools/libclang/CXCursor.cpp
@@ -338,6 +338,7 @@ CXCursor cxcursor::MakeCXCursor(const Stmt *S, const Decl *Parent,
   case Stmt::EmbedExprClass:
   case Stmt::HLSLOutArgExprClass:
   case Stmt::OpenACCAsteriskSizeExprClass:
+  case Stmt::ResolvedUnexpandedPackExprClass:
     K = CXCursor_UnexposedExpr;
     break;
 
@@ -919,6 +920,9 @@ CXCursor cxcursor::MakeCXCursor(const Stmt *S, const Decl *Parent,
   case Stmt::OpenACCUpdateConstructClass:
     K = CXCursor_OpenACCUpdateConstruct;
     break;
+  case Stmt::OpenACCAtomicConstructClass:
+    K = CXCursor_OpenACCAtomicConstruct;
+    break;
   case Stmt::OMPTargetParallelGenericLoopDirectiveClass:
     K = CXCursor_OMPTargetParallelGenericLoopDirective;
     break;
diff --git a/clang/tools/libclang/CXStoredDiagnostic.cpp b/clang/tools/libclang/CXStoredDiagnostic.cpp
index 03018229549bd..6fb3050f5f844 100644
--- a/clang/tools/libclang/CXStoredDiagnostic.cpp
+++ b/clang/tools/libclang/CXStoredDiagnostic.cpp
@@ -51,7 +51,9 @@ CXString CXStoredDiagnostic::getSpelling() const {
 
 CXString CXStoredDiagnostic::getDiagnosticOption(CXString *Disable) const {
   unsigned ID = Diag.getID();
-  StringRef Option = DiagnosticIDs::getWarningOptionForDiag(ID);
+  if (DiagnosticIDs::IsCustomDiag(ID))
+    return cxstring::createEmpty();
+  StringRef Option = DiagnosticIDs{}.getWarningOptionForDiag(ID);
   if (!Option.empty()) {
     if (Disable)
       *Disable = cxstring::createDup((Twine("-Wno-") + Option).str());
diff --git a/clang/tools/libclang/libclang.map b/clang/tools/libclang/libclang.map
index 8ca8a58b76d9e..600f86f00aaea 100644
--- a/clang/tools/libclang/libclang.map
+++ b/clang/tools/libclang/libclang.map
@@ -327,8 +327,6 @@ LLVM_13 {
     clang_getRange;
     clang_getRangeEnd;
     clang_getRangeStart;
-    clang_getRemappings;
-    clang_getRemappingsFromFileList;
     clang_getResultType;
     clang_getSkippedRanges;
     clang_getSpecializedCursorTemplate;
@@ -389,9 +387,6 @@ LLVM_13 {
     clang_parseTranslationUnit;
     clang_parseTranslationUnit2;
     clang_parseTranslationUnit2FullArgv;
-    clang_remap_dispose;
-    clang_remap_getFilenames;
-    clang_remap_getNumFiles;
     clang_reparseTranslationUnit;
     clang_saveTranslationUnit;
     clang_sortCodeCompletionResults;
diff --git a/clang/unittests/AST/ASTImporterTest.cpp b/clang/unittests/AST/ASTImporterTest.cpp
index 791248e7a394f..0bac95eb40b20 100644
--- a/clang/unittests/AST/ASTImporterTest.cpp
+++ b/clang/unittests/AST/ASTImporterTest.cpp
@@ -3441,6 +3441,7 @@ TEST_P(ASTImporterOptionSpecificTestBase, ImportParmVarDecl) {
   ASSERT_TRUE(FromVar);
   ASSERT_TRUE(FromVar->hasUninstantiatedDefaultArg());
   ASSERT_TRUE(FromVar->getUninstantiatedDefaultArg());
+  ASSERT_FALSE(FromVar->isExplicitObjectParameter());
 
   const auto *ToVar = Import(FromVar, Lang_CXX11);
   EXPECT_TRUE(ToVar);
@@ -3448,6 +3449,25 @@ TEST_P(ASTImporterOptionSpecificTestBase, ImportParmVarDecl) {
   EXPECT_TRUE(ToVar->getUninstantiatedDefaultArg());
   EXPECT_NE(FromVar->getUninstantiatedDefaultArg(),
             ToVar->getUninstantiatedDefaultArg());
+  EXPECT_FALSE(ToVar->isExplicitObjectParameter());
+}
+
+TEST_P(ASTImporterOptionSpecificTestBase, ImportParmVarDecl_Explicit) {
+  const auto *Code = R"(
+    struct Wrapper {
+      void func(this Wrapper) {}
+    };
+    )";
+  Decl *FromTU = getTuDecl(Code, Lang_CXX23);
+  auto *FromVar = FirstDeclMatcher<ParmVarDecl>().match(FromTU, parmVarDecl());
+  ASSERT_TRUE(FromVar);
+  ASSERT_TRUE(FromVar->isExplicitObjectParameter());
+
+  const auto *ToVar = Import(FromVar, Lang_CXX23);
+  EXPECT_TRUE(ToVar);
+  EXPECT_TRUE(ToVar->isExplicitObjectParameter());
+  EXPECT_NE(ToVar->getExplicitObjectParamThisLoc(),
+            FromVar->getExplicitObjectParamThisLoc());
 }
 
 TEST_P(ASTImporterOptionSpecificTestBase, ImportOfNonEquivalentField) {
@@ -8105,6 +8125,9 @@ TEST_P(ImportFunctions, CTADImplicit) {
   auto *ToD = Import(FromD, Lang_CXX17);
   ASSERT_TRUE(ToD);
   EXPECT_EQ(ToD->getDeductionCandidateKind(), DeductionCandidate::Copy);
+  EXPECT_EQ(ToD->getSourceDeductionGuide(), nullptr);
+  EXPECT_EQ(ToD->getSourceDeductionGuideKind(),
+            CXXDeductionGuideDecl::SourceDeductionGuideKind::None);
   // Check that the deduced class template is also imported.
   EXPECT_TRUE(findFromTU(FromD)->Importer->GetAlreadyImportedOrNull(
       FromD->getDeducedTemplate()));
@@ -8129,6 +8152,9 @@ TEST_P(ImportFunctions, CTADUserDefinedExplicit) {
   ASSERT_TRUE(ToD);
   EXPECT_FALSE(FromD->isImplicit());
   EXPECT_TRUE(ToD->isExplicit());
+  EXPECT_EQ(ToD->getSourceDeductionGuide(), nullptr);
+  EXPECT_EQ(ToD->getSourceDeductionGuideKind(),
+            CXXDeductionGuideDecl::SourceDeductionGuideKind::None);
 }
 
 TEST_P(ImportFunctions, CTADWithLocalTypedef) {
@@ -8147,6 +8173,49 @@ TEST_P(ImportFunctions, CTADWithLocalTypedef) {
   ASSERT_TRUE(ToD);
 }
 
+TEST_P(ImportFunctions, CTADAliasTemplate) {
+  Decl *TU = getTuDecl(
+      R"(
+      template <typename T> struct A {
+        A(T);
+      };
+      template<typename T>
+      using B = A<T>;
+      B b{(int)0};
+      )",
+      Lang_CXX20, "input.cc");
+  auto *FromD = FirstDeclMatcher<CXXDeductionGuideDecl>().match(
+      TU, cxxDeductionGuideDecl(hasParameter(0, hasType(asString("int")))));
+  auto *ToD = Import(FromD, Lang_CXX20);
+  ASSERT_TRUE(ToD);
+  EXPECT_TRUE(ToD->getSourceDeductionGuideKind() ==
+              CXXDeductionGuideDecl::SourceDeductionGuideKind::Alias);
+  EXPECT_TRUE(ToD->getSourceDeductionGuide());
+}
+
+TEST_P(ImportFunctions, CTADAliasTemplateWithExplicitSourceDeductionGuide) {
+  Decl *TU = getTuDecl(
+      R"(
+      template <typename T> struct A {
+        A(T);
+      };
+      template<typename T>
+      using B = A<T>;
+      A(int) -> A<double>; // explicit
+      B b{(int)0};
+      )",
+      Lang_CXX20, "input.cc");
+  auto *FromD = FirstDeclMatcher<CXXDeductionGuideDecl>().match(
+      TU, cxxDeductionGuideDecl(hasParameter(0, hasType(asString("int"))),
+                                hasName("<deduction guide for B>"),
+                                hasReturnTypeLoc(loc(asString("A<double>")))));
+  auto *ToD = Import(FromD, Lang_CXX20);
+  ASSERT_TRUE(ToD);
+  EXPECT_TRUE(ToD->getSourceDeductionGuideKind() ==
+              CXXDeductionGuideDecl::SourceDeductionGuideKind::Alias);
+  EXPECT_TRUE(ToD->getSourceDeductionGuide());
+}
+
 TEST_P(ImportFunctions, ParmVarDeclDeclContext) {
   constexpr auto FromTUCode = R"(
       void f(int P);
diff --git a/clang/unittests/AST/CommentLexer.cpp b/clang/unittests/AST/CommentLexer.cpp
index 1e7bad89898f4..22866f0eb23ed 100644
--- a/clang/unittests/AST/CommentLexer.cpp
+++ b/clang/unittests/AST/CommentLexer.cpp
@@ -1453,6 +1453,160 @@ TEST_F(CommentLexerTest, HTML19) {
   ASSERT_EQ(tok::newline,      Toks[2].getKind());
 }
 
+TEST_F(CommentLexerTest, HTML20) {
+  const char *Source = "// <a\n"
+                       "// \n"
+                       "// href=\"foo\"\n"
+                       "// \n"
+                       "// bar>text</a>";
+
+  std::vector<Token> Toks;
+
+  lexString(Source, Toks);
+
+  ASSERT_EQ(11U, Toks.size());
+
+  ASSERT_EQ(tok::text, Toks[0].getKind());
+  ASSERT_EQ(StringRef(" "), Toks[0].getText());
+
+  ASSERT_EQ(tok::html_start_tag, Toks[1].getKind());
+  ASSERT_EQ(StringRef("a"), Toks[1].getHTMLTagStartName());
+
+  ASSERT_EQ(tok::html_ident, Toks[2].getKind());
+  ASSERT_EQ(StringRef("href"), Toks[2].getHTMLIdent());
+
+  ASSERT_EQ(tok::html_equals, Toks[3].getKind());
+
+  ASSERT_EQ(tok::html_quoted_string, Toks[4].getKind());
+  ASSERT_EQ(StringRef("foo"), Toks[4].getHTMLQuotedString());
+
+  ASSERT_EQ(tok::html_ident, Toks[5].getKind());
+  ASSERT_EQ(StringRef("bar"), Toks[5].getHTMLIdent());
+
+  ASSERT_EQ(tok::html_greater, Toks[6].getKind());
+
+  ASSERT_EQ(tok::text, Toks[7].getKind());
+  ASSERT_EQ(StringRef("text"), Toks[7].getText());
+
+  ASSERT_EQ(tok::html_end_tag, Toks[8].getKind());
+  ASSERT_EQ(StringRef("a"), Toks[8].getHTMLTagEndName());
+
+  ASSERT_EQ(tok::html_greater, Toks[9].getKind());
+
+  ASSERT_EQ(tok::newline, Toks[10].getKind());
+}
+
+TEST_F(CommentLexerTest, HTML21) {
+  const char *Source = "/**\n"
+                       " * <a\n"
+                       " * \n"
+                       " * href=\"foo\"\n"
+                       " * \n"
+                       " * bar>text</a>\n"
+                       " */";
+
+  std::vector<Token> Toks;
+
+  lexString(Source, Toks);
+
+  ASSERT_EQ(15U, Toks.size());
+
+  ASSERT_EQ(tok::newline, Toks[0].getKind());
+
+  ASSERT_EQ(tok::text, Toks[1].getKind());
+  ASSERT_EQ(StringRef(" "), Toks[1].getText());
+
+  ASSERT_EQ(tok::html_start_tag, Toks[2].getKind());
+  ASSERT_EQ(StringRef("a"), Toks[2].getHTMLTagStartName());
+
+  ASSERT_EQ(tok::html_ident, Toks[3].getKind());
+  ASSERT_EQ(StringRef("href"), Toks[3].getHTMLIdent());
+
+  ASSERT_EQ(tok::html_equals, Toks[4].getKind());
+
+  ASSERT_EQ(tok::html_quoted_string, Toks[5].getKind());
+  ASSERT_EQ(StringRef("foo"), Toks[5].getHTMLQuotedString());
+
+  ASSERT_EQ(tok::html_ident, Toks[6].getKind());
+  ASSERT_EQ(StringRef("bar"), Toks[6].getHTMLIdent());
+
+  ASSERT_EQ(tok::html_greater, Toks[7].getKind());
+
+  ASSERT_EQ(tok::text, Toks[8].getKind());
+  ASSERT_EQ(StringRef("text"), Toks[8].getText());
+
+  ASSERT_EQ(tok::html_end_tag, Toks[9].getKind());
+  ASSERT_EQ(StringRef("a"), Toks[9].getHTMLTagEndName());
+
+  ASSERT_EQ(tok::html_greater, Toks[10].getKind());
+
+  ASSERT_EQ(tok::newline, Toks[11].getKind());
+
+  ASSERT_EQ(tok::text, Toks[12].getKind());
+  ASSERT_EQ(StringRef(" "), Toks[12].getText());
+
+  ASSERT_EQ(tok::newline, Toks[13].getKind());
+
+  ASSERT_EQ(tok::newline, Toks[14].getKind());
+}
+
+TEST_F(CommentLexerTest, HTML22) {
+  const char *Source = "/**\n"
+                       " * <a\n"
+                       " */";
+
+  std::vector<Token> Toks;
+
+  lexString(Source, Toks);
+
+  ASSERT_EQ(6U, Toks.size());
+
+  ASSERT_EQ(tok::newline, Toks[0].getKind());
+
+  ASSERT_EQ(tok::text, Toks[1].getKind());
+  ASSERT_EQ(StringRef(" "), Toks[1].getText());
+
+  ASSERT_EQ(tok::html_start_tag, Toks[2].getKind());
+  ASSERT_EQ(StringRef("a"), Toks[2].getHTMLTagStartName());
+
+  ASSERT_EQ(tok::newline, Toks[3].getKind());
+
+  ASSERT_EQ(tok::newline, Toks[4].getKind());
+
+  ASSERT_EQ(tok::newline, Toks[5].getKind());
+}
+
+TEST_F(CommentLexerTest, HTML23) {
+  // NOTE: "//<" is considered a comment start
+  const char *Source = "// <\n"
+                       "// a\n"
+                       "// >";
+
+  std::vector<Token> Toks;
+
+  lexString(Source, Toks);
+
+  ASSERT_EQ(7U, Toks.size());
+
+  ASSERT_EQ(tok::text, Toks[0].getKind());
+  ASSERT_EQ(StringRef(" "), Toks[0].getText());
+
+  ASSERT_EQ(tok::text, Toks[1].getKind());
+  ASSERT_EQ(StringRef("<"), Toks[1].getText());
+
+  ASSERT_EQ(tok::newline, Toks[2].getKind());
+
+  ASSERT_EQ(tok::text, Toks[3].getKind());
+  ASSERT_EQ(StringRef(" a"), Toks[3].getText());
+
+  ASSERT_EQ(tok::newline, Toks[4].getKind());
+
+  ASSERT_EQ(tok::text, Toks[5].getKind());
+  ASSERT_EQ(StringRef(" >"), Toks[5].getText());
+
+  ASSERT_EQ(tok::newline, Toks[6].getKind());
+}
+
 TEST_F(CommentLexerTest, NotAKnownHTMLTag1) {
   const char *Source = "// <tag>";
 
diff --git a/clang/unittests/AST/CommentParser.cpp b/clang/unittests/AST/CommentParser.cpp
index e0df182d430c3..aa08b6718e506 100644
--- a/clang/unittests/AST/CommentParser.cpp
+++ b/clang/unittests/AST/CommentParser.cpp
@@ -1065,9 +1065,10 @@ TEST_F(CommentParserTest, InlineCommand5) {
 
 TEST_F(CommentParserTest, HTML1) {
   const char *Sources[] = {
-    "// <a",
-    "// <a>",
-    "// <a >"
+      "// <a",
+      "// <a>",
+      "// <a >",
+      "// <a\n// >",
   };
 
   for (size_t i = 0, e = std::size(Sources); i != e; i++) {
@@ -1088,8 +1089,9 @@ TEST_F(CommentParserTest, HTML1) {
 
 TEST_F(CommentParserTest, HTML2) {
   const char *Sources[] = {
-    "// <br/>",
-    "// <br />"
+      "// <br/>",
+      "// <br />",
+      "// <br \n// />",
   };
 
   for (size_t i = 0, e = std::size(Sources); i != e; i++) {
@@ -1110,10 +1112,8 @@ TEST_F(CommentParserTest, HTML2) {
 
 TEST_F(CommentParserTest, HTML3) {
   const char *Sources[] = {
-    "// <a href",
-    "// <a href ",
-    "// <a href>",
-    "// <a href >",
+      "// <a href",   "// <a href ",       "// <a href>",
+      "// <a href >", "// <a \n// href >",
   };
 
   for (size_t i = 0, e = std::size(Sources); i != e; i++) {
@@ -1134,8 +1134,9 @@ TEST_F(CommentParserTest, HTML3) {
 
 TEST_F(CommentParserTest, HTML4) {
   const char *Sources[] = {
-    "// <a href=\"bbb\"",
-    "// <a href=\"bbb\">",
+      "// <a href=\"bbb\"",
+      "// <a href=\"bbb\">",
+      "// <a \n// href=\"bbb\">",
   };
 
   for (size_t i = 0, e = std::size(Sources); i != e; i++) {
diff --git a/clang/unittests/Analysis/FlowSensitive/SmartPointerAccessorCachingTest.cpp b/clang/unittests/Analysis/FlowSensitive/SmartPointerAccessorCachingTest.cpp
index 3f75dff60ee5f..18b9f80e32bbf 100644
--- a/clang/unittests/Analysis/FlowSensitive/SmartPointerAccessorCachingTest.cpp
+++ b/clang/unittests/Analysis/FlowSensitive/SmartPointerAccessorCachingTest.cpp
@@ -190,5 +190,57 @@ TEST(SmartPointerAccessorCachingTest, MatchesWithValueAndNonConstOverloads) {
       isSmartPointerLikeValueMethodCall()));
 }
 
+TEST(SmartPointerAccessorCachingTest, MatchesWithTypeAliases) {
+  llvm::StringRef Decls(R"cc(
+    template <class T>
+    struct HasGetAndValue {
+      using pointer_t = T*;
+      using reference_t = T&;
+
+      const pointer_t operator->() const;
+      pointer_t operator->();
+      const reference_t operator*() const;
+      reference_t operator*();
+      const reference_t value() const;
+      reference_t value();
+      const pointer_t get() const;
+      pointer_t get();
+    };
+
+    struct S { int i; };
+  )cc");
+
+  EXPECT_TRUE(matches(
+      Decls,
+      "int target(HasGetAndValue<S> &NonConst) { return (*NonConst).i; }",
+      isSmartPointerLikeOperatorStar()));
+  EXPECT_TRUE(matches(
+      Decls,
+      "int target(const HasGetAndValue<S> &Const) { return (*Const).i; }",
+      isSmartPointerLikeOperatorStar()));
+  EXPECT_TRUE(matches(
+      Decls, "int target(HasGetAndValue<S> &NonConst) { return NonConst->i; }",
+      isSmartPointerLikeOperatorArrow()));
+  EXPECT_TRUE(matches(
+      Decls, "int target(const HasGetAndValue<S> &Const) { return Const->i; }",
+      isSmartPointerLikeOperatorArrow()));
+  EXPECT_TRUE(matches(
+      Decls,
+      "int target(HasGetAndValue<S> &NonConst) { return NonConst.value().i; }",
+      isSmartPointerLikeValueMethodCall()));
+  EXPECT_TRUE(matches(
+      Decls,
+      "int target(const HasGetAndValue<S> &Const) { return Const.value().i; }",
+      isSmartPointerLikeValueMethodCall()));
+  EXPECT_TRUE(matches(
+      Decls,
+      "int target(HasGetAndValue<S> &NonConst) { return NonConst.get()->i; }",
+      isSmartPointerLikeGetMethodCall()));
+  EXPECT_TRUE(matches(
+      Decls,
+      "int target(const HasGetAndValue<S> &Const) { return Const.get()->i; }",
+      isSmartPointerLikeGetMethodCall()));
+}
+
 } // namespace
 } // namespace clang::dataflow
diff --git a/clang/unittests/Basic/DiagnosticTest.cpp b/clang/unittests/Basic/DiagnosticTest.cpp
index e03d9a464df7f..88fa1800f0ff2 100644
--- a/clang/unittests/Basic/DiagnosticTest.cpp
+++ b/clang/unittests/Basic/DiagnosticTest.cpp
@@ -62,7 +62,7 @@ TEST(DiagnosticTest, suppressAndTrap) {
 
     // Diag that would set FatalErrorOccurred
     // (via non-note following a fatal error).
-    Diags.Report(diag::warn_mt_message) << "warning";
+    Diags.Report(diag::warn_apinotes_message) << "warning";
 
     EXPECT_TRUE(trap.hasErrorOccurred());
     EXPECT_TRUE(trap.hasUnrecoverableErrorOccurred());
@@ -87,7 +87,7 @@ TEST(DiagnosticTest, fatalsAsError) {
 
     // Diag that would set FatalErrorOccurred
     // (via non-note following a fatal error).
-    Diags.Report(diag::warn_mt_message) << "warning";
+    Diags.Report(diag::warn_apinotes_message) << "warning";
 
     EXPECT_TRUE(Diags.hasErrorOccurred());
     EXPECT_EQ(Diags.hasFatalErrorOccurred(), FatalsAsError ? 0u : 1u);
@@ -346,4 +346,13 @@ TEST_F(SuppressionMappingTest, IsIgnored) {
   EXPECT_FALSE(Diags.isIgnored(diag::warn_unused_function,
                                SM.getLocForStartOfFile(ClangID)));
 }
+
+TEST_F(SuppressionMappingTest, ParsingRespectsOtherWarningOpts) {
+  Diags.getDiagnosticOptions().DiagnosticSuppressionMappingsFile = "foo.txt";
+  FS->addFile("foo.txt", /*ModificationTime=*/{},
+              llvm::MemoryBuffer::getMemBuffer("[non-existing-warning]"));
+  Diags.getDiagnosticOptions().Warnings.push_back("no-unknown-warning-option");
+  clang::ProcessWarningOptions(Diags, Diags.getDiagnosticOptions(), *FS);
+  EXPECT_THAT(diags(), IsEmpty());
+}
 } // namespace
diff --git a/clang/unittests/CodeGen/TBAAMetadataTest.cpp b/clang/unittests/CodeGen/TBAAMetadataTest.cpp
index cad8783ea73fb..f05c9787c63ea 100644
--- a/clang/unittests/CodeGen/TBAAMetadataTest.cpp
+++ b/clang/unittests/CodeGen/TBAAMetadataTest.cpp
@@ -117,15 +117,9 @@ TEST(TBAAMetadataTest, BasicTypes) {
   ASSERT_TRUE(I);
 
   I = matchNext(I,
-      MInstruction(Instruction::Store,
-        MValType(PointerType::getUnqual(Compiler.Context)),
-        MMTuple(
-          MMTuple(
-            MMString("p1 void"),
-            AnyPtr,
-            MConstInt(0)),
-          MSameAs(0),
-          MConstInt(0))));
+                MInstruction(Instruction::Store,
+                             MValType(PointerType::getUnqual(Compiler.Context)),
+                             MMTuple(AnyPtr, MSameAs(0), MConstInt(0))));
   ASSERT_TRUE(I);
 
   I = matchNext(I,
diff --git a/clang/unittests/Driver/ToolChainTest.cpp b/clang/unittests/Driver/ToolChainTest.cpp
index 0787e7d2d3339..14c505b5fbd9e 100644
--- a/clang/unittests/Driver/ToolChainTest.cpp
+++ b/clang/unittests/Driver/ToolChainTest.cpp
@@ -21,6 +21,7 @@
 #include "clang/Frontend/CompilerInstance.h"
 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/StringExtras.h"
+#include "llvm/Frontend/Debug/Options.h"
 #include "llvm/MC/TargetRegistry.h"
 #include "llvm/Support/TargetSelect.h"
 #include "llvm/Support/VirtualFileSystem.h"
@@ -595,6 +596,22 @@ TEST(ToolChainTest, UEFICallingConventionTest) {
             TargetInfo::CallingConvKind::CCK_MicrosoftWin64);
 }
 
+TEST(ToolChainTest, UEFIDefaultDebugFormatTest) {
+  IntrusiveRefCntPtr<DiagnosticOptions> DiagOpts = new DiagnosticOptions();
+  IntrusiveRefCntPtr<DiagnosticIDs> DiagID(new DiagnosticIDs());
+  struct TestDiagnosticConsumer : public DiagnosticConsumer {};
+  DiagnosticsEngine Diags(DiagID, &*DiagOpts, new TestDiagnosticConsumer);
+  IntrusiveRefCntPtr<llvm::vfs::InMemoryFileSystem> InMemoryFileSystem(
+      new llvm::vfs::InMemoryFileSystem);
+  Driver CCDriver("/home/test/bin/clang", "x86_64-unknown-uefi", Diags,
+                  "clang LLVM compiler", InMemoryFileSystem);
+  CCDriver.setCheckInputsExist(false);
+  std::unique_ptr<Compilation> CC(
+      CCDriver.BuildCompilation({"/home/test/bin/clang", "foo.cpp"}));
+  EXPECT_EQ(CC->getDefaultToolChain().getDefaultDebugFormat(),
+            llvm::codegenoptions::DIF_CodeView);
+}
+
 TEST(GetDriverMode, PrefersLastDriverMode) {
   static constexpr const char *Args[] = {"clang-cl", "--driver-mode=foo",
                                          "--driver-mode=bar", "foo.cpp"};
diff --git a/clang/unittests/Format/ConfigParseTest.cpp b/clang/unittests/Format/ConfigParseTest.cpp
index 9746aa3547846..9cd262960b724 100644
--- a/clang/unittests/Format/ConfigParseTest.cpp
+++ b/clang/unittests/Format/ConfigParseTest.cpp
@@ -144,6 +144,10 @@ TEST(ConfigParseTest, GetsCorrectBasedOnStyle) {
   EXPECT_EQ(0, parseConfiguration(TEXT, &Style).value());                      \
   EXPECT_EQ(VALUE, Style.FIELD) << "Unexpected value after parsing!"
 
+#define CHECK_PARSE_INT(FIELD) CHECK_PARSE(#FIELD ": -1234", FIELD, -1234)
+
+#define CHECK_PARSE_UNSIGNED(FIELD) CHECK_PARSE(#FIELD ": 1234", FIELD, 1234u)
+
 #define CHECK_PARSE_LIST(FIELD)                                                \
   CHECK_PARSE(#FIELD ": [foo]", FIELD, std::vector<std::string>{"foo"})
 
@@ -164,8 +168,10 @@ TEST(ConfigParseTest, ParsesConfigurationBools) {
   CHECK_PARSE_BOOL(AllowShortLoopsOnASingleLine);
   CHECK_PARSE_BOOL(AllowShortNamespacesOnASingleLine);
   CHECK_PARSE_BOOL(BinPackArguments);
+  CHECK_PARSE_BOOL(BinPackLongBracedList);
   CHECK_PARSE_BOOL(BreakAdjacentStringLiterals);
   CHECK_PARSE_BOOL(BreakAfterJavaFieldAnnotations);
+  CHECK_PARSE_BOOL(BreakBeforeTemplateCloser);
   CHECK_PARSE_BOOL(BreakBeforeTernaryOperators);
   CHECK_PARSE_BOOL(BreakStringLiterals);
   CHECK_PARSE_BOOL(CompactNamespaces);
@@ -254,33 +260,40 @@ TEST(ConfigParseTest, ParsesConfigurationBools) {
 
 #undef CHECK_PARSE_BOOL
 
+TEST(ConfigParseTest, ParsesConfigurationIntegers) {
+  FormatStyle Style = {};
+  Style.Language = FormatStyle::LK_Cpp;
+
+  CHECK_PARSE_INT(AccessModifierOffset);
+  CHECK_PARSE_INT(PPIndentWidth);
+
+  CHECK_PARSE_UNSIGNED(BracedInitializerIndentWidth);
+  CHECK_PARSE_UNSIGNED(ColumnLimit);
+  CHECK_PARSE_UNSIGNED(ConstructorInitializerIndentWidth);
+  CHECK_PARSE_UNSIGNED(ContinuationIndentWidth);
+  CHECK_PARSE_UNSIGNED(IndentWidth);
+  CHECK_PARSE_UNSIGNED(MaxEmptyLinesToKeep);
+  CHECK_PARSE_UNSIGNED(ObjCBlockIndentWidth);
+  CHECK_PARSE_UNSIGNED(PenaltyBreakAssignment);
+  CHECK_PARSE_UNSIGNED(PenaltyBreakBeforeFirstCallParameter);
+  CHECK_PARSE_UNSIGNED(PenaltyBreakBeforeMemberAccess);
+  CHECK_PARSE_UNSIGNED(PenaltyBreakComment);
+  CHECK_PARSE_UNSIGNED(PenaltyBreakFirstLessLess);
+  CHECK_PARSE_UNSIGNED(PenaltyBreakOpenParenthesis);
+  CHECK_PARSE_UNSIGNED(PenaltyBreakScopeResolution);
+  CHECK_PARSE_UNSIGNED(PenaltyBreakString);
+  CHECK_PARSE_UNSIGNED(PenaltyBreakTemplateDeclaration);
+  CHECK_PARSE_UNSIGNED(PenaltyExcessCharacter);
+  CHECK_PARSE_UNSIGNED(PenaltyIndentedWhitespace);
+  CHECK_PARSE_UNSIGNED(PenaltyReturnTypeOnItsOwnLine);
+  CHECK_PARSE_UNSIGNED(ShortNamespaceLines);
+  CHECK_PARSE_UNSIGNED(SpacesBeforeTrailingComments);
+  CHECK_PARSE_UNSIGNED(TabWidth);
+}
+
 TEST(ConfigParseTest, ParsesConfiguration) {
   FormatStyle Style = {};
   Style.Language = FormatStyle::LK_Cpp;
-  CHECK_PARSE("AccessModifierOffset: -1234", AccessModifierOffset, -1234);
-  CHECK_PARSE("ConstructorInitializerIndentWidth: 1234",
-              ConstructorInitializerIndentWidth, 1234u);
-  CHECK_PARSE("ObjCBlockIndentWidth: 1234", ObjCBlockIndentWidth, 1234u);
-  CHECK_PARSE("ColumnLimit: 1234", ColumnLimit, 1234u);
-  CHECK_PARSE("MaxEmptyLinesToKeep: 1234", MaxEmptyLinesToKeep, 1234u);
-  CHECK_PARSE("PenaltyBreakAssignment: 1234", PenaltyBreakAssignment, 1234u);
-  CHECK_PARSE("PenaltyBreakBeforeFirstCallParameter: 1234",
-              PenaltyBreakBeforeFirstCallParameter, 1234u);
-  CHECK_PARSE("PenaltyBreakTemplateDeclaration: 1234",
-              PenaltyBreakTemplateDeclaration, 1234u);
-  CHECK_PARSE("PenaltyBreakOpenParenthesis: 1234", PenaltyBreakOpenParenthesis,
-              1234u);
-  CHECK_PARSE("PenaltyBreakScopeResolution: 1234", PenaltyBreakScopeResolution,
-              1234u);
-  CHECK_PARSE("PenaltyExcessCharacter: 1234", PenaltyExcessCharacter, 1234u);
-  CHECK_PARSE("PenaltyReturnTypeOnItsOwnLine: 1234",
-              PenaltyReturnTypeOnItsOwnLine, 1234u);
-  CHECK_PARSE("SpacesBeforeTrailingComments: 1234",
-              SpacesBeforeTrailingComments, 1234u);
-  CHECK_PARSE("IndentWidth: 32", IndentWidth, 32u);
-  CHECK_PARSE("ContinuationIndentWidth: 11", ContinuationIndentWidth, 11u);
-  CHECK_PARSE("BracedInitializerIndentWidth: 34", BracedInitializerIndentWidth,
-              34);
   CHECK_PARSE("CommentPragmas: '// abc$'", CommentPragmas, "// abc$");
 
   Style.QualifierAlignment = FormatStyle::QAS_Right;
diff --git a/clang/unittests/Format/FormatTest.cpp b/clang/unittests/Format/FormatTest.cpp
index 265461561d201..9b9ce35f83bc5 100644
--- a/clang/unittests/Format/FormatTest.cpp
+++ b/clang/unittests/Format/FormatTest.cpp
@@ -9614,6 +9614,10 @@ TEST_F(FormatTest, AlignsAfterOpenBracket) {
                "        auto aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\n"
                "    ) {};",
                Style);
+  verifyFormat("aaaaaaaaaaaaaaaaaaaaaaaa(\n"
+               "    &bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb\n"
+               ");",
+               Style);
 }
 
 TEST_F(FormatTest, ParenthesesAndOperandAlignment) {
@@ -11220,6 +11224,236 @@ TEST_F(FormatTest, WrapsTemplateDeclarationsWithComments) {
       Style);
 }
 
+TEST_F(FormatTest, BreakBeforeTemplateCloser) {
+  auto Style = getLLVMStyle();
+  // Begin with tests covering the case where there is no constraint on the
+  // column limit.
+  Style.ColumnLimit = 0;
+  Style.BreakBeforeTemplateCloser = true;
+  // BreakBeforeTemplateCloser should NOT force template declarations onto
+  // multiple lines.
+  verifyFormat("template <typename Foo>\n"
+               "void foo() {}",
+               Style);
+  verifyFormat("template <typename Foo, typename Bar>\n"
+               "void foo() {}",
+               Style);
+  // It should add a line break before > if not already present:
+  verifyFormat("template <\n"
+               "    typename Foo\n"
+               ">\n"
+               "void foo() {}",
+               "template <\n"
+               "    typename Foo>\n"
+               "void foo() {}",
+               Style);
+  verifyFormat("template <\n"
+               "    typename Foo,\n"
+               "    typename Bar\n"
+               ">\n"
+               "void foo() {}",
+               "template <\n"
+               "    typename Foo,\n"
+               "    typename Bar>\n"
+               "void foo() {}",
+               Style);
+  // When within an indent scope, the > should be placed accordingly:
+  verifyFormat("struct Baz {\n"
+               "  template <\n"
+               "      typename Foo,\n"
+               "      typename Bar\n"
+               "  >\n"
+               "  void foo() {}\n"
+               "};",
+               "struct Baz {\n"
+               "  template <\n"
+               "      typename Foo,\n"
+               "      typename Bar>\n"
+               "  void foo() {}\n"
+               "};",
+               Style);
+
+  // Test from https://github.com/llvm/llvm-project/issues/80049:
+  verifyFormat(
+      "using type = std::remove_cv_t<\n"
+      "    add_common_cv_reference<\n"
+      "        std::common_type_t<std::decay_t<T0>, std::decay_t<T1>>,\n"
+      "        T0,\n"
+      "        T1\n"
+      "    >\n"
+      ">;",
+      "using type = std::remove_cv_t<\n"
+      "    add_common_cv_reference<\n"
+      "        std::common_type_t<std::decay_t<T0>, std::decay_t<T1>>,\n"
+      "        T0,\n"
+      "        T1>>;",
+      Style);
+
+  // Test lambda goes to next line:
+  verifyFormat("void foo() {\n"
+               "  auto lambda = []<\n"
+               "                    typename T\n"
+               "                >(T t) {\n"
+               "  };\n"
+               "}",
+               "void foo() {\n"
+               "  auto lambda = []<\n"
+               "  typename T>(T t){\n"
+               "  };\n"
+               "}",
+               Style);
+  // With no column limit, two parameters can go on the same line:
+  verifyFormat("void foo() {\n"
+               "  auto lambda = []<\n"
+               "                    typename T, typename Foo\n"
+               "                >(T t) {\n"
+               "  };\n"
+               "}",
+               "void foo() {\n"
+               "  auto lambda = []<\n"
+               "  typename T, typename Foo>(T t){\n"
+               "  };\n"
+               "}",
+               Style);
+  // Or on different lines:
+  verifyFormat("void foo() {\n"
+               "  auto lambda = []<\n"
+               "                    typename T,\n"
+               "                    typename Foo\n"
+               "                >(T t) {\n"
+               "  };\n"
+               "}",
+               "void foo() {\n"
+               "  auto lambda = []<\n"
+               "  typename T,\n"
+               "  typename Foo>(T t){\n"
+               "  };\n"
+               "}",
+               Style);
+
+  // Test template usage goes to next line too:
+  verifyFormat("void foo() {\n"
+               "  myFunc<\n"
+               "      T\n"
+               "  >();\n"
+               "}",
+               "void foo() {\n"
+               "  myFunc<\n"
+               "  T>();\n"
+               "}",
+               Style);
+
+  // Now test that it handles the cases when the column limit forces wrapping.
+  Style.ColumnLimit = 40;
+  // The typename goes on the first line if it fits:
+  verifyFormat("template <typename Fooooooooooooooooooo,\n"
+               "          typename Bar>\n"
+               "void foo() {}",
+               Style);
+  verifyFormat("template <typename Foo,\n"
+               "          typename Barrrrrrrrrrrrrrrrrr>\n"
+               "void foo() {}",
+               Style);
+  // Long names should be split in one step:
+  verifyFormat("template <\n"
+               "    typename Foo,\n"
+               "    typename Barrrrrrrrrrrrrrrrrrr\n"
+               ">\n"
+               "void foo() {}",
+               "template <typename Foo, typename Barrrrrrrrrrrrrrrrrrr>\n"
+               "void foo() {}",
+               Style);
+  verifyFormat("template <\n"
+               "    typename Foooooooooooooooooooo,\n"
+               "    typename Bar\n"
+               ">\n"
+               "void foo() {}",
+               "template <typename Foooooooooooooooooooo, typename Bar>\n"
+               "void foo() {}",
+               Style);
+  // Even when there is only one long name:
+  verifyFormat("template <\n"
+               "    typename Foooooooooooooooooooo\n"
+               ">\n"
+               "void foo() {}",
+               "template <typename Foooooooooooooooooooo>\n"
+               "void foo() {}",
+               Style);
+  // Test lambda goes to next line if the type is looong:
+  verifyFormat("void foo() {\n"
+               "  auto lambda =\n"
+               "      []<\n"
+               "          typename Loooooooooooooooooooooooooooooooooong\n"
+               "      >(T t) {};\n"
+               "  auto lambda =\n"
+               "      [looooooooooooooong]<\n"
+               "          typename Loooooooooooooooooooooooooooooooooong\n"
+               "      >(T t) {};\n"
+               "  auto lambda =\n"
+               "      []<\n"
+               "          typename T,\n"
+               "          typename Loooooooooooooooooooooooooooooooooong\n"
+               "      >(T t) {};\n"
+               // Nested:
+               "  auto lambda =\n"
+               "      []<\n"
+               "          template <typename, typename>\n"
+               "          typename Looooooooooooooooooong\n"
+               "      >(T t) {};\n"
+               // Same idea, the "T" is now short rather than Looong:
+               "  auto lambda =\n"
+               "      []<template <typename, typename>\n"
+               "         typename T>(T t) {};\n"
+               // Nested with long capture forces the style to block indent:
+               "  auto lambda =\n"
+               "      [loooooooooooooooooooong]<\n"
+               "          template <typename, typename>\n"
+               "          typename Looooooooooooooooooong\n"
+               "      >(T t) {};\n"
+               // But *now* it stays block indented even when T is short:
+               "  auto lambda =\n"
+               "      [loooooooooooooooooooong]<\n"
+               "          template <typename, typename>\n"
+               "          typename T\n"
+               "      >(T t) {};\n"
+               // Nested, with long name and long captures:
+               "  auto lambda =\n"
+               "      [loooooooooooooooooooong]<\n"
+               "          template <\n"
+               "              typename Foooooooooooooooo,\n"
+               "              typename\n"
+               "          >\n"
+               "          typename T\n"
+               "      >(T t) {};\n"
+               // Allow the nested template to be on the same line:
+               "  auto lambda =\n"
+               "      [loooooooooooooooooooong]<\n"
+               "          template <typename Fooooooooo,\n"
+               "                    typename>\n"
+               "          typename T\n"
+               "      >(T t) {};\n"
+               "}",
+               Style);
+
+  // Test template usage goes to next line if the type is looong:
+  verifyFormat("void foo() {\n"
+               "  myFunc<\n"
+               "      Looooooooooooooooooooooooong\n"
+               "  >();\n"
+               "}",
+               Style);
+  // Even a single type in the middle is enough to force it to block indent
+  // style:
+  verifyFormat("void foo() {\n"
+               "  myFunc<\n"
+               "      Foo, Foo, Foo,\n"
+               "      Foooooooooooooooooooooooooooooo,\n"
+               "      Foo, Foo, Foo, Foo\n"
+               "  >();\n"
+               "}",
+               Style);
+}
+
 TEST_F(FormatTest, WrapsTemplateParameters) {
   FormatStyle Style = getLLVMStyle();
   Style.AlignAfterOpenBracket = FormatStyle::BAS_DontAlign;
@@ -14186,6 +14420,51 @@ TEST_F(FormatTest, LayoutCxx11BraceInitializers) {
       "};",
       NoBinPacking);
 
+  NoBinPacking.BinPackLongBracedList = false;
+  verifyFormat("const Aaaaaa aaaaa = {aaaaa,\n"
+               "                      bbbbb,\n"
+               "                      ccccc,\n"
+               "                      ddddd,\n"
+               "                      eeeee,\n"
+               "                      ffffff,\n"
+               "                      ggggg,\n"
+               "                      hhhhhh,\n"
+               "                      iiiiii,\n"
+               "                      jjjjjj,\n"
+               "                      kkkkkk,\n"
+               "                      aaaaa,\n"
+               "                      bbbbb,\n"
+               "                      ccccc,\n"
+               "                      ddddd,\n"
+               "                      eeeee,\n"
+               "                      ffffff,\n"
+               "                      ggggg,\n"
+               "                      hhhhhh,\n"
+               "                      iiiiii};",
+               NoBinPacking);
+  verifyFormat("const Aaaaaa aaaaa = {\n"
+               "    aaaaa,\n"
+               "    bbbbb,\n"
+               "    ccccc,\n"
+               "    ddddd,\n"
+               "    eeeee,\n"
+               "    ffffff,\n"
+               "    ggggg,\n"
+               "    hhhhhh,\n"
+               "    iiiiii,\n"
+               "    jjjjjj,\n"
+               "    kkkkkk,\n"
+               "    aaaaa,\n"
+               "    bbbbb,\n"
+               "    ccccc,\n"
+               "    ddddd,\n"
+               "    eeeee,\n"
+               "    ffffff,\n"
+               "    ggggg,\n"
+               "    hhhhhh,\n"
+               "};",
+               NoBinPacking);
+
   NoBinPacking.AlignAfterOpenBracket = FormatStyle::BAS_AlwaysBreak;
   verifyFormat("static uint8 CddDp83848Reg[] = {\n"
                "    CDDDP83848_BMCR_REGISTER,\n"
@@ -22365,6 +22644,24 @@ TEST_F(FormatTest, BreakPenaltyAfterForLoopLParen) {
                Style);
 }
 
+TEST_F(FormatTest, BreakPenaltyBeforeMemberAccess) {
+  auto Style = getLLVMStyle();
+  EXPECT_EQ(Style.PenaltyBreakBeforeMemberAccess, 150u);
+
+  Style.ColumnLimit = 60;
+  Style.PenaltyBreakBeforeMemberAccess = 110;
+  verifyFormat("aaaaaaaa.aaaaaaaa.bbbbbbbb()\n"
+               "    .ccccccccccccccccccccc(dddddddd);\n"
+               "aaaaaaaa.aaaaaaaa\n"
+               "    .bbbbbbbb(cccccccccccccccccccccccccccccccc);",
+               Style);
+
+  Style.ColumnLimit = 13;
+  verifyFormat("foo->bar\n"
+               "    .b(a);",
+               Style);
+}
+
 TEST_F(FormatTest, BreakPenaltyScopeResolution) {
   FormatStyle Style = getLLVMStyle();
   Style.ColumnLimit = 20;
@@ -26460,6 +26757,9 @@ TEST_F(FormatTest, RequiresClauses) {
                "foo();\n"
                "#endif\n"
                "bar(requires);");
+
+  verifyNoCrash("template <class T>\n"
+                "    requires(requires { std::declval<T>()");
 }
 
 TEST_F(FormatTest, RequiresExpressionIndentation) {
diff --git a/clang/unittests/Format/FormatTestTableGen.cpp b/clang/unittests/Format/FormatTestTableGen.cpp
index 7771f6a109a9a..92377c31f2e91 100644
--- a/clang/unittests/Format/FormatTestTableGen.cpp
+++ b/clang/unittests/Format/FormatTestTableGen.cpp
@@ -101,6 +101,9 @@ TEST_F(FormatTestTableGen, BangOperators) {
                "                                  \"zerozero\",\n"
                "                                  true:  // default\n"
                "                                  \"positivepositive\");\n"
+               "  let CondOpe3WithCommentAfterLParen = !cond(\n"
+               "      // comment\n"
+               "      !eq(/* comment */ x, 0): \"zero\");\n"
                "}");
 }
 
diff --git a/clang/unittests/Format/QualifierFixerTest.cpp b/clang/unittests/Format/QualifierFixerTest.cpp
index 129828b0d187a..3eae39f267c3e 100644
--- a/clang/unittests/Format/QualifierFixerTest.cpp
+++ b/clang/unittests/Format/QualifierFixerTest.cpp
@@ -1291,6 +1291,21 @@ TEST_F(QualifierFixerTest, WithCpp11Attribute) {
                "[[maybe_unused]] constexpr static int A", Style);
 }
 
+TEST_F(QualifierFixerTest, WithQualifiedTypeName) {
+  auto Style = getLLVMStyle();
+  Style.QualifierAlignment = FormatStyle::QAS_Custom;
+  Style.QualifierOrder = {"constexpr", "type", "const"};
+
+  verifyFormat("constexpr ::int64_t x{1};", "::int64_t constexpr x{1};", Style);
+  verifyFormat("constexpr std::int64_t x{123};",
+               "std::int64_t constexpr x{123};", Style);
+  verifyFormat("constexpr ::std::int64_t x{123};",
+               "::std::int64_t constexpr x{123};", Style);
+
+  Style.TypeNames.push_back("bar");
+  verifyFormat("constexpr foo::bar x{12};", "foo::bar constexpr x{12};", Style);
+}
+
 TEST_F(QualifierFixerTest, DisableRegions) {
   FormatStyle Style = getLLVMStyle();
   Style.QualifierAlignment = FormatStyle::QAS_Custom;
diff --git a/clang/unittests/Format/TokenAnnotatorTest.cpp b/clang/unittests/Format/TokenAnnotatorTest.cpp
index 9ac60ce73750b..54d8ff0571ca6 100644
--- a/clang/unittests/Format/TokenAnnotatorTest.cpp
+++ b/clang/unittests/Format/TokenAnnotatorTest.cpp
@@ -479,11 +479,13 @@ TEST_F(TokenAnnotatorTest, UnderstandsUsesOfPlusAndMinus) {
 TEST_F(TokenAnnotatorTest, UnderstandsClasses) {
   auto Tokens = annotate("class C {};");
   ASSERT_EQ(Tokens.size(), 6u) << Tokens;
+  EXPECT_TOKEN(Tokens[1], tok::identifier, TT_ClassHeadName);
   EXPECT_TOKEN(Tokens[2], tok::l_brace, TT_ClassLBrace);
   EXPECT_TOKEN(Tokens[3], tok::r_brace, TT_ClassRBrace);
 
   Tokens = annotate("const class C {} c;");
   ASSERT_EQ(Tokens.size(), 8u) << Tokens;
+  EXPECT_TOKEN(Tokens[2], tok::identifier, TT_ClassHeadName);
   EXPECT_TOKEN(Tokens[3], tok::l_brace, TT_ClassLBrace);
   EXPECT_TOKEN(Tokens[4], tok::r_brace, TT_ClassRBrace);
 
@@ -494,6 +496,7 @@ TEST_F(TokenAnnotatorTest, UnderstandsClasses) {
 
   Tokens = annotate("class [[deprecated(\"\")]] C { int i; };");
   ASSERT_EQ(Tokens.size(), 17u) << Tokens;
+  EXPECT_TOKEN(Tokens[9], tok::identifier, TT_ClassHeadName);
   EXPECT_TOKEN(Tokens[10], tok::l_brace, TT_ClassLBrace);
   EXPECT_TOKEN(Tokens[14], tok::r_brace, TT_ClassRBrace);
 }
@@ -501,21 +504,25 @@ TEST_F(TokenAnnotatorTest, UnderstandsClasses) {
 TEST_F(TokenAnnotatorTest, UnderstandsStructs) {
   auto Tokens = annotate("struct S {};");
   ASSERT_EQ(Tokens.size(), 6u) << Tokens;
+  EXPECT_TOKEN(Tokens[1], tok::identifier, TT_ClassHeadName);
   EXPECT_TOKEN(Tokens[2], tok::l_brace, TT_StructLBrace);
   EXPECT_TOKEN(Tokens[3], tok::r_brace, TT_StructRBrace);
 
   Tokens = annotate("struct macro(a) S {};");
   ASSERT_EQ(Tokens.size(), 10u) << Tokens;
+  EXPECT_TOKEN(Tokens[5], tok::identifier, TT_ClassHeadName);
   EXPECT_TOKEN(Tokens[6], tok::l_brace, TT_StructLBrace);
   EXPECT_TOKEN(Tokens[7], tok::r_brace, TT_StructRBrace);
 
   Tokens = annotate("struct EXPORT_MACRO [[nodiscard]] C { int i; };");
   ASSERT_EQ(Tokens.size(), 15u) << Tokens;
+  EXPECT_TOKEN(Tokens[7], tok::identifier, TT_ClassHeadName);
   EXPECT_TOKEN(Tokens[8], tok::l_brace, TT_StructLBrace);
   EXPECT_TOKEN(Tokens[12], tok::r_brace, TT_StructRBrace);
 
   Tokens = annotate("struct [[deprecated]] [[nodiscard]] C { int i; };");
   ASSERT_EQ(Tokens.size(), 19u) << Tokens;
+  EXPECT_TOKEN(Tokens[11], tok::identifier, TT_ClassHeadName);
   EXPECT_TOKEN(Tokens[12], tok::l_brace, TT_StructLBrace);
   EXPECT_TOKEN(Tokens[16], tok::r_brace, TT_StructRBrace);
 
@@ -523,12 +530,14 @@ TEST_F(TokenAnnotatorTest, UnderstandsStructs) {
                     "  void f(T &t);\n"
                     "};");
   ASSERT_EQ(Tokens.size(), 18u) << Tokens;
+  EXPECT_TOKEN(Tokens[5], tok::identifier, TT_ClassHeadName);
   EXPECT_TOKEN(Tokens[6], tok::l_brace, TT_StructLBrace);
   EXPECT_TOKEN(Tokens[11], tok::amp, TT_PointerOrReference);
   EXPECT_TOKEN(Tokens[15], tok::r_brace, TT_StructRBrace);
 
   Tokens = annotate("template <typename T> struct S<const T[N]> {};");
   ASSERT_EQ(Tokens.size(), 18u) << Tokens;
+  EXPECT_TOKEN(Tokens[6], tok::identifier, TT_ClassHeadName);
   EXPECT_TOKEN(Tokens[7], tok::less, TT_TemplateOpener);
   EXPECT_TOKEN(Tokens[10], tok::l_square, TT_ArraySubscriptLSquare);
   EXPECT_TOKEN(Tokens[13], tok::greater, TT_TemplateCloser);
@@ -537,6 +546,7 @@ TEST_F(TokenAnnotatorTest, UnderstandsStructs) {
 
   Tokens = annotate("template <typename T> struct S<T const[N]> {};");
   ASSERT_EQ(Tokens.size(), 18u) << Tokens;
+  EXPECT_TOKEN(Tokens[6], tok::identifier, TT_ClassHeadName);
   EXPECT_TOKEN(Tokens[7], tok::less, TT_TemplateOpener);
   EXPECT_TOKEN(Tokens[10], tok::l_square, TT_ArraySubscriptLSquare);
   EXPECT_TOKEN(Tokens[13], tok::greater, TT_TemplateCloser);
@@ -547,6 +557,7 @@ TEST_F(TokenAnnotatorTest, UnderstandsStructs) {
                     "  void f(T const (&a)[n]);\n"
                     "};");
   ASSERT_EQ(Tokens.size(), 35u) << Tokens;
+  EXPECT_TOKEN(Tokens[9], tok::identifier, TT_ClassHeadName);
   EXPECT_TOKEN(Tokens[10], tok::less, TT_TemplateOpener);
   EXPECT_TOKEN(Tokens[13], tok::l_square, TT_ArraySubscriptLSquare);
   EXPECT_TOKEN(Tokens[16], tok::greater, TT_TemplateCloser);
@@ -558,13 +569,33 @@ TEST_F(TokenAnnotatorTest, UnderstandsStructs) {
 
   Tokens = annotate("template <typename T, enum E e> struct S {};");
   ASSERT_EQ(Tokens.size(), 15u) << Tokens;
+  EXPECT_TOKEN(Tokens[10], tok::identifier, TT_ClassHeadName);
   EXPECT_TOKEN(Tokens[11], tok::l_brace, TT_StructLBrace);
 
+  Tokens = annotate(
+      "template <> struct __declspec(foo) Op<Bar *> : OpImpl<Bar *> {};");
+  ASSERT_EQ(Tokens.size(), 23u) << Tokens;
+  EXPECT_TOKEN(Tokens[5], tok::l_paren, TT_AttributeLParen);
+  EXPECT_TOKEN(Tokens[7], tok::r_paren, TT_AttributeRParen);
+  EXPECT_TOKEN(Tokens[8], tok::identifier, TT_ClassHeadName);
+  EXPECT_TOKEN(Tokens[13], tok::colon, TT_InheritanceColon);
+  EXPECT_TOKEN(Tokens[19], tok::l_brace, TT_StructLBrace);
+  EXPECT_TOKEN(Tokens[20], tok::r_brace, TT_StructRBrace);
+
+  constexpr StringRef Code{"struct EXPORT StructName {};"};
+
+  Tokens = annotate(Code);
+  ASSERT_EQ(Tokens.size(), 7u) << Tokens;
+  EXPECT_TOKEN(Tokens[2], tok::identifier, TT_ClassHeadName);
+  EXPECT_TOKEN(Tokens[3], tok::l_brace, TT_StructLBrace);
+  EXPECT_TOKEN(Tokens[4], tok::r_brace, TT_StructRBrace);
+
   auto Style = getLLVMStyle();
   Style.AttributeMacros.push_back("EXPORT");
-  Tokens = annotate("struct EXPORT StructName {};", Style);
+  Tokens = annotate(Code, Style);
   ASSERT_EQ(Tokens.size(), 7u) << Tokens;
   EXPECT_TOKEN(Tokens[1], tok::identifier, TT_AttributeMacro);
+  EXPECT_TOKEN(Tokens[2], tok::identifier, TT_ClassHeadName);
   EXPECT_TOKEN(Tokens[3], tok::l_brace, TT_StructLBrace);
   EXPECT_TOKEN(Tokens[4], tok::r_brace, TT_StructRBrace);
 }
@@ -572,11 +603,13 @@ TEST_F(TokenAnnotatorTest, UnderstandsStructs) {
 TEST_F(TokenAnnotatorTest, UnderstandsUnions) {
   auto Tokens = annotate("union U {};");
   ASSERT_EQ(Tokens.size(), 6u) << Tokens;
+  EXPECT_TOKEN(Tokens[1], tok::identifier, TT_ClassHeadName);
   EXPECT_TOKEN(Tokens[2], tok::l_brace, TT_UnionLBrace);
   EXPECT_TOKEN(Tokens[3], tok::r_brace, TT_UnionRBrace);
 
   Tokens = annotate("union U { void f() { return; } };");
   ASSERT_EQ(Tokens.size(), 14u) << Tokens;
+  EXPECT_TOKEN(Tokens[1], tok::identifier, TT_ClassHeadName);
   EXPECT_TOKEN(Tokens[2], tok::l_brace, TT_UnionLBrace);
   EXPECT_TOKEN(Tokens[7], tok::l_brace, TT_FunctionLBrace);
   EXPECT_TOKEN(Tokens[11], tok::r_brace, TT_UnionRBrace);
@@ -685,7 +718,7 @@ TEST_F(TokenAnnotatorTest, UnderstandsNonTemplateAngleBrackets) {
   EXPECT_TOKEN(Tokens[4], tok::less, TT_BinaryOperator);
   EXPECT_TOKEN(Tokens[8], tok::greater, TT_BinaryOperator);
 
-  Tokens = annotate("return A < B ^ A > B;");
+  Tokens = annotate("return A < B != A > B;");
   ASSERT_EQ(Tokens.size(), 10u) << Tokens;
   EXPECT_TOKEN(Tokens[2], tok::less, TT_BinaryOperator);
   EXPECT_TOKEN(Tokens[6], tok::greater, TT_BinaryOperator);
@@ -709,8 +742,10 @@ TEST_F(TokenAnnotatorTest, UnderstandsTemplateTemplateParameters) {
   EXPECT_TOKEN(Tokens[11], tok::less, TT_TemplateOpener);
   EXPECT_TOKEN(Tokens[14], tok::greater, TT_TemplateCloser);
   EXPECT_FALSE(Tokens[14]->ClosesTemplateDeclaration);
+  EXPECT_TOKEN(Tokens[16], tok::identifier, TT_Unknown);
   EXPECT_TOKEN(Tokens[21], tok::greater, TT_TemplateCloser);
   EXPECT_TRUE(Tokens[21]->ClosesTemplateDeclaration);
+  EXPECT_TOKEN(Tokens[23], tok::identifier, TT_ClassHeadName);
 }
 
 TEST_F(TokenAnnotatorTest, UnderstandsWhitespaceSensitiveMacros) {
@@ -793,6 +828,7 @@ TEST_F(TokenAnnotatorTest, UnderstandsCasts) {
 
   Tokens = annotate("return (struct foo){};");
   ASSERT_EQ(Tokens.size(), 9u) << Tokens;
+  EXPECT_TOKEN(Tokens[3], tok::identifier, TT_Unknown);
   EXPECT_TOKEN(Tokens[4], tok::r_paren, TT_CastRParen);
 
   Tokens = annotate("#define FOO(bar) foo((uint64_t)&bar)");
@@ -838,6 +874,12 @@ TEST_F(TokenAnnotatorTest, UnderstandsCasts) {
   EXPECT_TOKEN(Tokens[14], tok::r_paren, TT_CastRParen);
   EXPECT_TOKEN(Tokens[15], tok::amp, TT_UnaryOperator);
 
+  Tokens = annotate("return (Foo (Bar::*)())&Bar::foo;");
+  ASSERT_EQ(Tokens.size(), 17u) << Tokens;
+  EXPECT_TOKEN(Tokens[3], tok::l_paren, TT_FunctionTypeLParen);
+  EXPECT_TOKEN(Tokens[10], tok::r_paren, TT_CastRParen);
+  EXPECT_TOKEN(Tokens[11], tok::amp, TT_UnaryOperator);
+
   auto Style = getLLVMStyle();
   Style.TypeNames.push_back("Foo");
   Tokens = annotate("#define FOO(bar) foo((Foo)&bar)", Style);
@@ -1025,6 +1067,12 @@ TEST_F(TokenAnnotatorTest, UnderstandsOverloadedOperators) {
   EXPECT_TOKEN(Tokens[6], tok::l_paren, TT_OverloadedOperatorLParen);
   EXPECT_TOKEN(Tokens[8], tok::amp, TT_PointerOrReference);
   EXPECT_TOKEN(Tokens[12], tok::amp, TT_PointerOrReference);
+
+  Tokens = annotate("SomeLoooooooooooooooooType::Awaitable\n"
+                    "SomeLoooooooooooooooooType::operator co_await();");
+  ASSERT_EQ(Tokens.size(), 11u) << Tokens;
+  EXPECT_TOKEN(Tokens[3], tok::identifier, TT_FunctionDeclarationName);
+  EXPECT_TOKEN(Tokens[7], tok::l_paren, TT_OverloadedOperatorLParen);
 }
 
 TEST_F(TokenAnnotatorTest, OverloadedOperatorInTemplate) {
@@ -1130,9 +1178,13 @@ TEST_F(TokenAnnotatorTest, UnderstandsRequiresClausesAndConcepts) {
   EXPECT_TOKEN(Tokens[16], tok::pipepipe, TT_BinaryOperator);
   EXPECT_TOKEN(Tokens[21], tok::ampamp, TT_BinaryOperator);
   EXPECT_TOKEN(Tokens[27], tok::ampamp, TT_BinaryOperator);
+  // Not TT_TrailingAnnotation.
+  EXPECT_TOKEN(Tokens[28], tok::identifier, TT_Unknown);
   EXPECT_TOKEN(Tokens[31], tok::greater, TT_TemplateCloser);
   EXPECT_EQ(Tokens[31]->FakeRParens, 1u);
   EXPECT_TRUE(Tokens[31]->ClosesRequiresClause);
+  // Not TT_TrailingAnnotation.
+  EXPECT_TOKEN(Tokens[33], tok::identifier, TT_Unknown);
 
   Tokens =
       annotate("template<typename T>\n"
@@ -1150,6 +1202,7 @@ TEST_F(TokenAnnotatorTest, UnderstandsRequiresClausesAndConcepts) {
   EXPECT_EQ(Tokens[32]->FakeRParens, 1u);
   EXPECT_TOKEN(Tokens[33], tok::r_paren, TT_Unknown);
   EXPECT_TRUE(Tokens[33]->ClosesRequiresClause);
+  EXPECT_TOKEN(Tokens[35], tok::identifier, TT_Unknown);
 
   Tokens = annotate("template <typename T>\n"
                     "void foo(T) noexcept requires Bar<T>;");
@@ -1231,6 +1284,8 @@ TEST_F(TokenAnnotatorTest, UnderstandsRequiresClausesAndConcepts) {
                     "return number_zero_v<T>; }\n"
                     "};");
   ASSERT_EQ(Tokens.size(), 44u) << Tokens;
+  EXPECT_TOKEN(Tokens[6], tok::identifier, TT_ClassHeadName);
+  EXPECT_TOKEN(Tokens[11], tok::identifier, TT_Unknown);
   EXPECT_TOKEN(Tokens[13], tok::kw_requires, TT_RequiresClause);
   EXPECT_TOKEN(Tokens[14], tok::kw_requires, TT_RequiresExpression);
   EXPECT_TOKEN(Tokens[15], tok::l_brace, TT_RequiresExpressionLBrace);
@@ -1242,6 +1297,8 @@ TEST_F(TokenAnnotatorTest, UnderstandsRequiresClausesAndConcepts) {
       annotate("template <class A, class B> concept C ="
                "std::same_as<std::iter_value_t<A>, std::iter_value_t<B>>;");
   ASSERT_EQ(Tokens.size(), 31u) << Tokens;
+  EXPECT_TOKEN(Tokens[3], tok::identifier, TT_Unknown);
+  EXPECT_TOKEN(Tokens[6], tok::identifier, TT_Unknown);
   EXPECT_TOKEN(Tokens[8], tok::kw_concept, TT_Unknown);
   EXPECT_TOKEN(Tokens[14], tok::less, TT_TemplateOpener);
   EXPECT_TOKEN(Tokens[18], tok::less, TT_TemplateOpener);
@@ -1858,6 +1915,7 @@ TEST_F(TokenAnnotatorTest, UnderstandsLambdas) {
   ASSERT_EQ(Tokens.size(), 12u) << Tokens;
   EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare);
   EXPECT_TOKEN(Tokens[2], tok::arrow, TT_LambdaArrow);
+  EXPECT_TOKEN(Tokens[4], tok::identifier, TT_Unknown);
   EXPECT_TOKEN(Tokens[5], tok::l_brace, TT_LambdaLBrace);
 
   Tokens = annotate("foo([&](u32 bar) __attribute__((attr)) -> void {});");
@@ -2738,6 +2796,7 @@ TEST_F(TokenAnnotatorTest, UnderstandTableGenTokens) {
 
   // Structured statements.
   Tokens = Annotate("class Foo {}");
+  EXPECT_TOKEN(Tokens[1], tok::identifier, TT_StartOfName);
   EXPECT_TOKEN(Tokens[2], tok::l_brace, TT_FunctionLBrace);
   Tokens = Annotate("def Def: Foo {}");
   EXPECT_TOKEN(Tokens[2], tok::colon, TT_InheritanceColon);
@@ -3203,6 +3262,7 @@ TEST_F(TokenAnnotatorTest, BraceKind) {
 
   Tokens = annotate("class Foo<int> f() {}");
   ASSERT_EQ(Tokens.size(), 11u) << Tokens;
+  EXPECT_TOKEN(Tokens[1], tok::identifier, TT_Unknown);
   EXPECT_TOKEN(Tokens[5], tok::identifier, TT_FunctionDeclarationName);
   EXPECT_TOKEN(Tokens[6], tok::l_paren, TT_FunctionDeclarationLParen);
   EXPECT_TOKEN(Tokens[8], tok::l_brace, TT_FunctionLBrace);
@@ -3211,6 +3271,7 @@ TEST_F(TokenAnnotatorTest, BraceKind) {
 
   Tokens = annotate("template <typename T> class Foo<T> f() {}");
   ASSERT_EQ(Tokens.size(), 16u) << Tokens;
+  EXPECT_TOKEN(Tokens[6], tok::identifier, TT_Unknown);
   EXPECT_TOKEN(Tokens[10], tok::identifier, TT_FunctionDeclarationName);
   EXPECT_TOKEN(Tokens[11], tok::l_paren, TT_FunctionDeclarationLParen);
   EXPECT_TOKEN(Tokens[13], tok::l_brace, TT_FunctionLBrace);
@@ -3321,36 +3382,52 @@ TEST_F(TokenAnnotatorTest, BraceKind) {
 
   Tokens = annotate("struct ::Foo {};");
   ASSERT_EQ(Tokens.size(), 7u) << Tokens;
+  EXPECT_TOKEN(Tokens[2], tok::identifier, TT_ClassHeadName);
   EXPECT_BRACE_KIND(Tokens[3], BK_Block);
   EXPECT_BRACE_KIND(Tokens[4], BK_Block);
 
   Tokens = annotate("struct NS::Foo {};");
   ASSERT_EQ(Tokens.size(), 8u) << Tokens;
+  EXPECT_TOKEN(Tokens[1], tok::identifier, TT_Unknown);
+  EXPECT_TOKEN(Tokens[3], tok::identifier, TT_ClassHeadName);
   EXPECT_BRACE_KIND(Tokens[4], BK_Block);
   EXPECT_BRACE_KIND(Tokens[5], BK_Block);
 
   Tokens = annotate("struct Foo<int> {};");
   ASSERT_EQ(Tokens.size(), 9u) << Tokens;
+  EXPECT_TOKEN(Tokens[1], tok::identifier, TT_ClassHeadName);
   EXPECT_BRACE_KIND(Tokens[5], BK_Block);
   EXPECT_BRACE_KIND(Tokens[6], BK_Block);
 
   Tokens = annotate("struct Foo<int>::Bar {};");
   ASSERT_EQ(Tokens.size(), 11u) << Tokens;
+  EXPECT_TOKEN(Tokens[1], tok::identifier, TT_Unknown);
+  EXPECT_TOKEN(Tokens[6], tok::identifier, TT_ClassHeadName);
   EXPECT_BRACE_KIND(Tokens[7], BK_Block);
   EXPECT_BRACE_KIND(Tokens[8], BK_Block);
 
   Tokens = annotate("struct Foo<int> : Base {};");
   ASSERT_EQ(Tokens.size(), 11u) << Tokens;
+  EXPECT_TOKEN(Tokens[1], tok::identifier, TT_ClassHeadName);
   EXPECT_BRACE_KIND(Tokens[7], BK_Block);
   EXPECT_BRACE_KIND(Tokens[8], BK_Block);
 
+  Tokens = annotate("struct Foo<int> : Base::Bar {};");
+  ASSERT_EQ(Tokens.size(), 13u) << Tokens;
+  EXPECT_TOKEN(Tokens[1], tok::identifier, TT_ClassHeadName);
+  EXPECT_TOKEN(Tokens[8], tok::identifier, TT_Unknown); // Not TT_ClassHeadName.
+  EXPECT_BRACE_KIND(Tokens[9], BK_Block);
+  EXPECT_BRACE_KIND(Tokens[10], BK_Block);
+
   Tokens = annotate("struct Foo final {};");
   ASSERT_EQ(Tokens.size(), 7u) << Tokens;
+  EXPECT_TOKEN(Tokens[1], tok::identifier, TT_ClassHeadName);
   EXPECT_BRACE_KIND(Tokens[3], BK_Block);
   EXPECT_BRACE_KIND(Tokens[4], BK_Block);
 
   Tokens = annotate("struct [[foo]] [[bar]] Foo final : Base1, Base2 {};");
   ASSERT_EQ(Tokens.size(), 21u) << Tokens;
+  EXPECT_TOKEN(Tokens[11], tok::identifier, TT_ClassHeadName);
   EXPECT_BRACE_KIND(Tokens[17], BK_Block);
   EXPECT_BRACE_KIND(Tokens[18], BK_Block);
 
@@ -3386,6 +3463,7 @@ TEST_F(TokenAnnotatorTest, BraceKind) {
                     "#endif\n"
                     "};");
   ASSERT_EQ(Tokens.size(), 29u) << Tokens;
+  EXPECT_TOKEN(Tokens[8], tok::identifier, TT_ClassHeadName);
   EXPECT_BRACE_KIND(Tokens[11], BK_Block);
   EXPECT_BRACE_KIND(Tokens[17], BK_Block);
   EXPECT_BRACE_KIND(Tokens[22], BK_Block);
@@ -3437,6 +3515,7 @@ TEST_F(TokenAnnotatorTest, BraceKind) {
   Tokens = annotate("a = class extends goog.a {};",
                     getGoogleStyle(FormatStyle::LK_JavaScript));
   ASSERT_EQ(Tokens.size(), 11u) << Tokens;
+  EXPECT_TOKEN(Tokens[3], tok::identifier, TT_Unknown);
   EXPECT_TOKEN(Tokens[7], tok::l_brace, TT_ClassLBrace);
   EXPECT_BRACE_KIND(Tokens[7], BK_Block);
   EXPECT_TOKEN(Tokens[8], tok::r_brace, TT_ClassRBrace);
@@ -3445,6 +3524,8 @@ TEST_F(TokenAnnotatorTest, BraceKind) {
   Tokens = annotate("a = class Foo extends goog.a {};",
                     getGoogleStyle(FormatStyle::LK_JavaScript));
   ASSERT_EQ(Tokens.size(), 12u) << Tokens;
+  EXPECT_TOKEN(Tokens[3], tok::identifier, TT_ClassHeadName);
+  EXPECT_TOKEN(Tokens[4], tok::identifier, TT_Unknown); // Not TT_StartOfName
   EXPECT_TOKEN(Tokens[8], tok::l_brace, TT_ClassLBrace);
   EXPECT_BRACE_KIND(Tokens[8], BK_Block);
   EXPECT_TOKEN(Tokens[9], tok::r_brace, TT_ClassRBrace);
@@ -3453,6 +3534,8 @@ TEST_F(TokenAnnotatorTest, BraceKind) {
   Tokens = annotate("#define FOO(X) \\\n"
                     "  struct X##_tag_ {};");
   ASSERT_EQ(Tokens.size(), 14u) << Tokens;
+  EXPECT_TOKEN(Tokens[7], tok::identifier, TT_Unknown);
+  EXPECT_TOKEN(Tokens[9], tok::identifier, TT_ClassHeadName);
   EXPECT_TOKEN(Tokens[10], tok::l_brace, TT_StructLBrace);
   EXPECT_BRACE_KIND(Tokens[10], BK_Block);
   EXPECT_TOKEN(Tokens[11], tok::r_brace, TT_StructRBrace);
@@ -3463,6 +3546,7 @@ TEST_F(TokenAnnotatorTest, BraceKind) {
                     "    void f() { return; } \\\n"
                     "  };");
   ASSERT_EQ(Tokens.size(), 20u) << Tokens;
+  EXPECT_TOKEN(Tokens[4], tok::identifier, TT_ClassHeadName);
   EXPECT_TOKEN(Tokens[8], tok::l_brace, TT_StructLBrace);
   EXPECT_BRACE_KIND(Tokens[8], BK_Block);
   EXPECT_TOKEN(Tokens[10], tok::identifier, TT_FunctionDeclarationName);
@@ -3665,6 +3749,11 @@ TEST_F(TokenAnnotatorTest, TemplateInstantiation) {
   EXPECT_TOKEN(Tokens[2], tok::less, TT_TemplateOpener);
   EXPECT_TOKEN(Tokens[6], tok::greater, TT_TemplateCloser);
 
+  Tokens = annotate("return FixedInt<N | M>(foo);");
+  ASSERT_EQ(Tokens.size(), 12u) << Tokens;
+  EXPECT_TOKEN(Tokens[2], tok::less, TT_TemplateOpener);
+  EXPECT_TOKEN(Tokens[6], tok::greater, TT_TemplateCloser);
+
   Tokens = annotate("return std::conditional_t<T::value == U::value, T, U>{};");
   ASSERT_EQ(Tokens.size(), 21u) << Tokens;
   EXPECT_TOKEN(Tokens[4], tok::less, TT_TemplateOpener);
diff --git a/clang/unittests/Sema/HeuristicResolverTest.cpp b/clang/unittests/Sema/HeuristicResolverTest.cpp
index 2cd5486b3227f..c7cfe7917c532 100644
--- a/clang/unittests/Sema/HeuristicResolverTest.cpp
+++ b/clang/unittests/Sema/HeuristicResolverTest.cpp
@@ -155,6 +155,66 @@ TEST(HeuristicResolver, MemberExpr_SmartPointer_Qualified) {
       cxxMethodDecl(hasName("find"), isConst()).bind("output"));
 }
 
+TEST(HeuristicResolver, MemberExpr_Static_Qualified) {
+  std::string Code = R"cpp(
+    template <typename T>
+    struct Waldo {
+      static void find();
+    };
+    template <typename T>
+    void foo(const Waldo<T>& t) {
+      t.find();
+    }
+  )cpp";
+  // Test resolution of "find" in "t.find()".
+  // The object being `const` should have no bearing on a call to a static
+  // method.
+  expectResolution(
+      Code, &HeuristicResolver::resolveMemberExpr,
+      cxxDependentScopeMemberExpr(hasMemberName("find")).bind("input"),
+      cxxMethodDecl(hasName("find")).bind("output"));
+}
+
+TEST(HeuristicResolver, MemberExpr_AutoTypeDeduction1) {
+  std::string Code = R"cpp(
+    template <typename T>
+    struct A {
+      int waldo;
+    };
+    template <typename T>
+    void foo(A<T> a) {
+      auto copy = a;
+      copy.waldo;
+    }
+  )cpp";
+  expectResolution(
+      Code, &HeuristicResolver::resolveMemberExpr,
+      cxxDependentScopeMemberExpr(hasMemberName("waldo")).bind("input"),
+      fieldDecl(hasName("waldo")).bind("output"));
+}
+
+TEST(HeuristicResolver, MemberExpr_AutoTypeDeduction2) {
+  std::string Code = R"cpp(
+    struct B {
+      int waldo;
+    };
+
+    template <typename T>
+    struct A {
+      B b;
+    };
+    template <typename T>
+    void foo(A<T> a) {
+      auto b = a.b;
+      b.waldo;
+    }
+  )cpp";
+  expectResolution(
+      Code, &HeuristicResolver::resolveMemberExpr,
+      cxxDependentScopeMemberExpr(hasMemberName("waldo")).bind("input"),
+      fieldDecl(hasName("waldo")).bind("output"));
+}
+
 TEST(HeuristicResolver, MemberExpr_Chained) {
   std::string Code = R"cpp(
     struct A { void foo() {} };
@@ -173,6 +233,48 @@ TEST(HeuristicResolver, MemberExpr_Chained) {
       cxxMethodDecl(hasName("foo")).bind("output"));
 }
 
+TEST(HeuristicResolver, MemberExpr_ReferenceType) {
+  std::string Code = R"cpp(
+    struct B {
+      int waldo;
+    };
+    template <typename T>
+    struct A {
+      B &b;
+    };
+    template <typename T>
+    void foo(A<T> &a) {
+      a.b.waldo;
+    }
+  )cpp";
+  // Test resolution of "waldo" in "a.b.waldo".
+  expectResolution(
+      Code, &HeuristicResolver::resolveMemberExpr,
+      cxxDependentScopeMemberExpr(hasMemberName("waldo")).bind("input"),
+      fieldDecl(hasName("waldo")).bind("output"));
+}
+
+TEST(HeuristicResolver, MemberExpr_PointerType) {
+  std::string Code = R"cpp(
+    struct B {
+      int waldo;
+    };
+    template <typename T>
+    struct A {
+      B *b;
+    };
+    template <typename T>
+    void foo(A<T> &a) {
+      a.b->waldo;
+    }
+  )cpp";
+  // Test resolution of "waldo" in "a.b->waldo".
+  expectResolution(
+      Code, &HeuristicResolver::resolveMemberExpr,
+      cxxDependentScopeMemberExpr(hasMemberName("waldo")).bind("input"),
+      fieldDecl(hasName("waldo")).bind("output"));
+}
+
 TEST(HeuristicResolver, MemberExpr_TemplateArgs) {
   std::string Code = R"cpp(
     struct Foo {
@@ -255,6 +357,25 @@ TEST(HeuristicResolver, MemberExpr_Metafunction) {
       cxxMethodDecl(hasName("find")).bind("output"));
 }
 
+TEST(HeuristicResolver, MemberExpr_Metafunction_Enumerator) {
+  std::string Code = R"cpp(
+    enum class State { Hidden };
+    template <typename T>
+    struct Meta {
+      using Type = State;
+    };
+    template <typename T>
+    void foo(typename Meta<T>::Type t) {
+      t.Hidden;
+    }
+  )cpp";
+  // Test resolution of "Hidden" in "t.Hidden".
+  expectResolution(
+      Code, &HeuristicResolver::resolveMemberExpr,
+      cxxDependentScopeMemberExpr(hasMemberName("Hidden")).bind("input"),
+      enumConstantDecl(hasName("Hidden")).bind("output"));
+}
+
 TEST(HeuristicResolver, MemberExpr_DeducedNonTypeTemplateParameter) {
   std::string Code = R"cpp(
     template <int N>
@@ -273,6 +394,22 @@ TEST(HeuristicResolver, MemberExpr_DeducedNonTypeTemplateParameter) {
       fieldDecl(hasName("found")).bind("output"));
 }
 
+TEST(HeuristicResolver, MemberExpr_HangIssue126536) {
+  std::string Code = R"cpp(
+    template <class T>
+    void foo() {
+      T bar;
+      auto baz = (bar, bar);
+      baz.foo();
+    }
+  )cpp";
+  // Test resolution of "foo" in "baz.foo()".
+  // Here, we are testing that we do not get into an infinite loop.
+  expectResolution(
+      Code, &HeuristicResolver::resolveMemberExpr,
+      cxxDependentScopeMemberExpr(hasMemberName("foo")).bind("input"));
+}
+
 TEST(HeuristicResolver, DeclRefExpr_StaticMethod) {
   std::string Code = R"cpp(
     template <typename T>
@@ -345,6 +482,27 @@ TEST(HeuristicResolver, DeclRefExpr_RespectScope) {
       dependentScopeDeclRefExpr(hasDependentName("getPointer")).bind("input"));
 }
 
+TEST(HeuristicResolver, DeclRefExpr_Nested) {
+  std::string Code = R"cpp(
+    struct S {
+      static int Waldo;
+    };
+    template <typename T>
+    struct Meta {
+      using Type = S;
+    };
+    template <typename T>
+    void foo() {
+      Meta<T>::Type::Waldo;
+    }
+  )cpp";
+  // Test resolution of "Waldo" in "Meta<T>::Type::Waldo".
+  expectResolution(
+      Code, &HeuristicResolver::resolveDeclRefExpr,
+      dependentScopeDeclRefExpr(hasDependentName("Waldo")).bind("input"),
+      varDecl(hasName("Waldo")).bind("output"));
+}
+
 TEST(HeuristicResolver, DependentNameType) {
   std::string Code = R"cpp(
     template <typename>
diff --git a/clang/unittests/StaticAnalyzer/CMakeLists.txt b/clang/unittests/StaticAnalyzer/CMakeLists.txt
index f5da86e545603..3b01a4e9e5327 100644
--- a/clang/unittests/StaticAnalyzer/CMakeLists.txt
+++ b/clang/unittests/StaticAnalyzer/CMakeLists.txt
@@ -15,6 +15,7 @@ add_clang_unittest(StaticAnalysisTests
   IsCLibraryFunctionTest.cpp
   MemRegionDescriptiveNameTest.cpp
   NoStateChangeFuncVisitorTest.cpp
+  ObjcBug-124477.cpp
   ParamRegionTest.cpp
   RangeSetTest.cpp
   RegisterCustomCheckersTest.cpp
diff --git a/clang/unittests/StaticAnalyzer/ObjcBug-124477.cpp b/clang/unittests/StaticAnalyzer/ObjcBug-124477.cpp
new file mode 100644
index 0000000000000..51bd33210032c
--- /dev/null
+++ b/clang/unittests/StaticAnalyzer/ObjcBug-124477.cpp
@@ -0,0 +1,63 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "CheckerRegistration.h"
+#include "clang/StaticAnalyzer/Core/Checker.h"
+#include "clang/StaticAnalyzer/Core/PathSensitive/CheckerContext.h"
+#include "clang/StaticAnalyzer/Core/PathSensitive/ProgramStateTrait.h"
+#include "clang/StaticAnalyzer/Frontend/AnalysisConsumer.h"
+#include "clang/StaticAnalyzer/Frontend/CheckerRegistry.h"
+#include "gtest/gtest.h"
+
+using namespace clang;
+using namespace ento;
+
+// Some dummy trait that we can mutate back and forth to force a new State.
+REGISTER_TRAIT_WITH_PROGRAMSTATE(Flag, bool)
+
+namespace {
+class FlipFlagOnCheckLocation : public Checker<check::Location> {
+public:
+  // We make sure we alter the State every time we model a checkLocation event.
+  void checkLocation(SVal l, bool isLoad, const Stmt *S,
+                     CheckerContext &C) const {
+    ProgramStateRef State = C.getState();
+    State = State->set<Flag>(!State->get<Flag>());
+    C.addTransition(State);
+  }
+};
+
+void addFlagFlipperChecker(AnalysisASTConsumer &AnalysisConsumer,
+                           AnalyzerOptions &AnOpts) {
+  AnOpts.CheckersAndPackages = {{"test.FlipFlagOnCheckLocation", true}};
+  AnalysisConsumer.AddCheckerRegistrationFn([](CheckerRegistry &Registry) {
+    Registry.addChecker<FlipFlagOnCheckLocation>("test.FlipFlagOnCheckLocation",
+                                                 "Description", "");
+  });
+}
+
+TEST(ObjCTest, CheckLocationEventsShouldMaterializeInObjCForCollectionStmts) {
+  // Previously, the `ExprEngine::hasMoreIteration` may fired an assertion
+  // because we forgot to handle correctly the resulting nodes of the
+  // check::Location callback for the ObjCForCollectionStmts.
+  // This caused inconsistencies in the graph and triggering the assertion.
+  // See #124477 for more details.
+  std::string Diags;
+  EXPECT_TRUE(runCheckerOnCodeWithArgs<addFlagFlipperChecker>(
+      R"(
+    @class NSArray, NSDictionary, NSString;
+    extern void NSLog(NSString *format, ...) __attribute__((format(__NSString__, 1, 2)));
+    void entrypoint(NSArray *bowl) {
+      for (NSString *fruit in bowl) { // no-crash
+        NSLog(@"Fruit: %@", fruit);
+      }
+    })",
+      {"-x", "objective-c"}, Diags));
+}
+
+} // namespace
diff --git a/clang/utils/TableGen/ClangAttrEmitter.cpp b/clang/utils/TableGen/ClangAttrEmitter.cpp
index de12c7062666a..af7478b7986f9 100644
--- a/clang/utils/TableGen/ClangAttrEmitter.cpp
+++ b/clang/utils/TableGen/ClangAttrEmitter.cpp
@@ -108,6 +108,14 @@ GetFlattenedSpellings(const Record &Attr) {
       Ret.emplace_back("CXX11", Name, "clang", false, *Spelling);
       if (Spelling->getValueAsBit("AllowInC"))
         Ret.emplace_back("C23", Name, "clang", false, *Spelling);
+    } else if (Variety == "ClangGCC") {
+      Ret.emplace_back("GNU", Name, "", false, *Spelling);
+      Ret.emplace_back("CXX11", Name, "clang", false, *Spelling);
+      Ret.emplace_back("CXX11", Name, "gnu", false, *Spelling);
+      if (Spelling->getValueAsBit("AllowInC")) {
+        Ret.emplace_back("C23", Name, "clang", false, *Spelling);
+        Ret.emplace_back("C23", Name, "gnu", false, *Spelling);
+      }
     } else {
       Ret.push_back(FlattenedSpelling(*Spelling));
     }
diff --git a/clang/utils/TableGen/ClangBuiltinsEmitter.cpp b/clang/utils/TableGen/ClangBuiltinsEmitter.cpp
index 6aca4edfdfb88..352765acf5338 100644
--- a/clang/utils/TableGen/ClangBuiltinsEmitter.cpp
+++ b/clang/utils/TableGen/ClangBuiltinsEmitter.cpp
@@ -15,7 +15,9 @@
 #include "llvm/ADT/StringSwitch.h"
 #include "llvm/TableGen/Error.h"
 #include "llvm/TableGen/Record.h"
+#include "llvm/TableGen/StringToOffsetTable.h"
 #include "llvm/TableGen/TableGenBackend.h"
+#include <sstream>
 
 using namespace llvm;
 
@@ -29,6 +31,125 @@ enum class BuiltinType {
   TargetLibBuiltin,
 };
 
+class HeaderNameParser {
+public:
+  HeaderNameParser(const Record *Builtin) {
+    for (char c : Builtin->getValueAsString("Header")) {
+      if (std::islower(c))
+        HeaderName += static_cast<char>(std::toupper(c));
+      else if (c == '.' || c == '_' || c == '/' || c == '-')
+        HeaderName += '_';
+      else
+        PrintFatalError(Builtin->getLoc(), "Unexpected header name");
+    }
+  }
+
+  void Print(raw_ostream &OS) const { OS << HeaderName; }
+
+private:
+  std::string HeaderName;
+};
+
+struct Builtin {
+  BuiltinType BT;
+  std::string Name;
+  std::string Type;
+  std::string Attributes;
+
+  const Record *BuiltinRecord;
+
+  void EmitEnumerator(llvm::raw_ostream &OS) const {
+    OS << "    BI";
+    // If there is a required name prefix, include its spelling in the
+    // enumerator.
+    if (auto *PrefixRecord =
+            BuiltinRecord->getValueAsOptionalDef("RequiredNamePrefix"))
+      OS << PrefixRecord->getValueAsString("Spelling");
+    OS << Name << ",\n";
+  }
+
+  void EmitInfo(llvm::raw_ostream &OS, const StringToOffsetTable &Table) const {
+    OS << "    Builtin::Info{Builtin::Info::StrOffsets{"
+       << Table.GetStringOffset(Name) << " /* " << Name << " */, "
+       << Table.GetStringOffset(Type) << " /* " << Type << " */, "
+       << Table.GetStringOffset(Attributes) << " /* " << Attributes << " */, ";
+    if (BT == BuiltinType::TargetBuiltin) {
+      const auto &Features = BuiltinRecord->getValueAsString("Features");
+      OS << Table.GetStringOffset(Features) << " /* " << Features << " */";
+    } else {
+      OS << "0";
+    }
+    OS << "}, ";
+    if (BT == BuiltinType::LibBuiltin || BT == BuiltinType::TargetLibBuiltin) {
+      OS << "HeaderDesc::";
+      HeaderNameParser{BuiltinRecord}.Print(OS);
+    } else {
+      OS << "HeaderDesc::NO_HEADER";
+    }
+    OS << ", ";
+    if (BT == BuiltinType::LibBuiltin || BT == BuiltinType::LangBuiltin ||
+        BT == BuiltinType::TargetLibBuiltin) {
+      OS << BuiltinRecord->getValueAsString("Languages");
+    } else {
+      OS << "ALL_LANGUAGES";
+    }
+    OS << "},\n";
+  }
+
+  void EmitXMacro(llvm::raw_ostream &OS) const {
+    if (BuiltinRecord->getValueAsBit("RequiresUndef"))
+      OS << "#undef " << Name << '\n';
+    switch (BT) {
+    case BuiltinType::LibBuiltin:
+      OS << "LIBBUILTIN";
+      break;
+    case BuiltinType::LangBuiltin:
+      OS << "LANGBUILTIN";
+      break;
+    case BuiltinType::Builtin:
+      OS << "BUILTIN";
+      break;
+    case BuiltinType::AtomicBuiltin:
+      OS << "ATOMIC_BUILTIN";
+      break;
+    case BuiltinType::TargetBuiltin:
+      OS << "TARGET_BUILTIN";
+      break;
+    case BuiltinType::TargetLibBuiltin:
+      OS << "TARGET_HEADER_BUILTIN";
+      break;
+    }
+
+    OS << "(" << Name << ", \"" << Type << "\", \"" << Attributes << "\"";
+
+    switch (BT) {
+    case BuiltinType::LibBuiltin: {
+      OS << ", ";
+      HeaderNameParser{BuiltinRecord}.Print(OS);
+      [[fallthrough]];
+    }
+    case BuiltinType::LangBuiltin: {
+      OS << ", " << BuiltinRecord->getValueAsString("Languages");
+      break;
+    }
+    case BuiltinType::TargetLibBuiltin: {
+      OS << ", ";
+      HeaderNameParser{BuiltinRecord}.Print(OS);
+      OS << ", " << BuiltinRecord->getValueAsString("Languages");
+      [[fallthrough]];
+    }
+    case BuiltinType::TargetBuiltin: {
+      OS << ", \"" << BuiltinRecord->getValueAsString("Features") << "\"";
+      break;
+    }
+    case BuiltinType::AtomicBuiltin:
+    case BuiltinType::Builtin:
+      break;
+    }
+    OS << ")\n";
+  }
+};
+
 class PrototypeParser {
 public:
   PrototypeParser(StringRef Substitution, const Record *Builtin)
@@ -37,6 +158,8 @@ class PrototypeParser {
     ParsePrototype(Builtin->getValueAsString("Prototype"));
   }
 
+  std::string takeTypeString() && { return std::move(Type); }
+
 private:
   void ParsePrototype(StringRef Prototype) {
     Prototype = Prototype.trim();
@@ -104,9 +227,39 @@ class PrototypeParser {
 
   void ParseType(StringRef T) {
     T = T.trim();
+
+    auto ConsumeAddrSpace = [&]() -> std::optional<unsigned> {
+      T = T.trim();
+      if (!T.consume_back(">"))
+        return std::nullopt;
+
+      auto Open = T.find_last_of('<');
+      if (Open == StringRef::npos)
+        PrintFatalError(Loc, "Mismatched angle-brackets in type");
+
+      StringRef ArgStr = T.substr(Open + 1);
+      T = T.slice(0, Open);
+      if (!T.consume_back("address_space"))
+        PrintFatalError(Loc,
+                        "Only `address_space<N>` supported as a parameterized "
+                        "pointer or reference type qualifier");
+
+      unsigned Number = 0;
+      if (ArgStr.getAsInteger(10, Number))
+        PrintFatalError(
+            Loc, "Expected an integer argument to the address_space qualifier");
+      if (Number == 0)
+        PrintFatalError(Loc, "No need for a qualifier for address space `0`");
+      return Number;
+    };
+
     if (T.consume_back("*")) {
+      // Pointers may have an address space qualifier immediately before them.
+      std::optional<unsigned> AS = ConsumeAddrSpace();
       ParseType(T);
       Type += "*";
+      if (AS)
+        Type += std::to_string(*AS);
     } else if (T.consume_back("const")) {
       ParseType(T);
       Type += "C";
@@ -117,6 +270,13 @@ class PrototypeParser {
       ParseType(T);
       Type += "R";
     } else if (T.consume_back("&")) {
+      // References may have an address space qualifier immediately before them.
+      std::optional<unsigned> AS = ConsumeAddrSpace();
+      ParseType(T);
+      Type += "&";
+      if (AS)
+        Type += std::to_string(*AS);
+    } else if (T.consume_back(")")) {
       ParseType(T);
       Type += "&";
     } else if (EnableOpenCLLong && T.consume_front("long long")) {
@@ -206,37 +366,15 @@ class PrototypeParser {
     }
   }
 
-public:
-  void Print(raw_ostream &OS) const { OS << ", \"" << Type << '\"'; }
-
-private:
   SMLoc Loc;
   StringRef Substitution;
   bool EnableOpenCLLong;
   std::string Type;
 };
 
-class HeaderNameParser {
-public:
-  HeaderNameParser(const Record *Builtin) {
-    for (char c : Builtin->getValueAsString("Header")) {
-      if (std::islower(c))
-        HeaderName += static_cast<char>(std::toupper(c));
-      else if (c == '.' || c == '_' || c == '/' || c == '-')
-        HeaderName += '_';
-      else
-        PrintFatalError(Builtin->getLoc(), "Unexpected header name");
-    }
-  }
-
-  void Print(raw_ostream &OS) const { OS << HeaderName; }
-
-private:
-  std::string HeaderName;
-};
-
-void PrintAttributes(const Record *Builtin, BuiltinType BT, raw_ostream &OS) {
-  OS << '\"';
+std::string renderAttributes(const Record *Builtin, BuiltinType BT) {
+  std::string Attributes;
+  raw_string_ostream OS(Attributes);
   if (Builtin->isSubClassOf("LibBuiltin")) {
     if (BT == BuiltinType::LibBuiltin) {
       OS << 'f';
@@ -265,63 +403,18 @@ void PrintAttributes(const Record *Builtin, BuiltinType BT, raw_ostream &OS) {
       OS << '>';
     }
   }
-  OS << '\"';
+  return Attributes;
 }
 
-void EmitBuiltinDef(raw_ostream &OS, StringRef Substitution,
-                    const Record *Builtin, Twine Spelling, BuiltinType BT) {
-  if (Builtin->getValueAsBit("RequiresUndef"))
-    OS << "#undef " << Spelling << '\n';
-  switch (BT) {
-  case BuiltinType::LibBuiltin:
-    OS << "LIBBUILTIN";
-    break;
-  case BuiltinType::LangBuiltin:
-    OS << "LANGBUILTIN";
-    break;
-  case BuiltinType::Builtin:
-    OS << "BUILTIN";
-    break;
-  case BuiltinType::AtomicBuiltin:
-    OS << "ATOMIC_BUILTIN";
-    break;
-  case BuiltinType::TargetBuiltin:
-    OS << "TARGET_BUILTIN";
-    break;
-  case BuiltinType::TargetLibBuiltin:
-    OS << "TARGET_HEADER_BUILTIN";
-    break;
-  }
-
-  OS << "(" << Spelling;
-  PrototypeParser{Substitution, Builtin}.Print(OS);
-  OS << ", ";
-  PrintAttributes(Builtin, BT, OS);
-
-  switch (BT) {
-  case BuiltinType::LibBuiltin: {
-    OS << ", ";
-    HeaderNameParser{Builtin}.Print(OS);
-    [[fallthrough]];
-  }
-  case BuiltinType::LangBuiltin: {
-    OS << ", " << Builtin->getValueAsString("Languages");
-    break;
-  }
-  case BuiltinType::TargetLibBuiltin: {
-    OS << ", ";
-    HeaderNameParser{Builtin}.Print(OS);
-    OS << ", " << Builtin->getValueAsString("Languages");
-    [[fallthrough]];
-  }
-  case BuiltinType::TargetBuiltin:
-    OS << ", \"" << Builtin->getValueAsString("Features") << "\"";
-    break;
-  case BuiltinType::AtomicBuiltin:
-  case BuiltinType::Builtin:
-    break;
-  }
-  OS << ")\n";
+Builtin buildBuiltin(StringRef Substitution, const Record *BuiltinRecord,
+                     Twine Spelling, BuiltinType BT) {
+  Builtin B;
+  B.BT = BT;
+  B.Name = Spelling.str();
+  B.Type = PrototypeParser(Substitution, BuiltinRecord).takeTypeString();
+  B.Attributes = renderAttributes(BuiltinRecord, BT);
+  B.BuiltinRecord = BuiltinRecord;
+  return B;
 }
 
 struct TemplateInsts {
@@ -347,10 +440,11 @@ TemplateInsts getTemplateInsts(const Record *R) {
   return temp;
 }
 
-void EmitBuiltin(raw_ostream &OS, const Record *Builtin) {
+void collectBuiltins(const Record *BuiltinRecord,
+                     SmallVectorImpl<Builtin> &Builtins) {
   TemplateInsts Templates = {};
-  if (Builtin->isSubClassOf("Template")) {
-    Templates = getTemplateInsts(Builtin);
+  if (BuiltinRecord->isSubClassOf("Template")) {
+    Templates = getTemplateInsts(BuiltinRecord);
   } else {
     Templates.Affix.emplace_back();
     Templates.Substitution.emplace_back();
@@ -358,26 +452,28 @@ void EmitBuiltin(raw_ostream &OS, const Record *Builtin) {
 
   for (auto [Substitution, Affix] :
        zip(Templates.Substitution, Templates.Affix)) {
-    for (StringRef Spelling : Builtin->getValueAsListOfStrings("Spellings")) {
+    for (StringRef Spelling :
+         BuiltinRecord->getValueAsListOfStrings("Spellings")) {
       auto FullSpelling =
           (Templates.IsPrefix ? Affix + Spelling : Spelling + Affix).str();
       BuiltinType BT = BuiltinType::Builtin;
-      if (Builtin->isSubClassOf("AtomicBuiltin")) {
+      if (BuiltinRecord->isSubClassOf("AtomicBuiltin")) {
         BT = BuiltinType::AtomicBuiltin;
-      } else if (Builtin->isSubClassOf("LangBuiltin")) {
+      } else if (BuiltinRecord->isSubClassOf("LangBuiltin")) {
         BT = BuiltinType::LangBuiltin;
-      } else if (Builtin->isSubClassOf("TargetLibBuiltin")) {
+      } else if (BuiltinRecord->isSubClassOf("TargetLibBuiltin")) {
         BT = BuiltinType::TargetLibBuiltin;
-      } else if (Builtin->isSubClassOf("TargetBuiltin")) {
+      } else if (BuiltinRecord->isSubClassOf("TargetBuiltin")) {
         BT = BuiltinType::TargetBuiltin;
-      } else if (Builtin->isSubClassOf("LibBuiltin")) {
+      } else if (BuiltinRecord->isSubClassOf("LibBuiltin")) {
         BT = BuiltinType::LibBuiltin;
-        if (Builtin->getValueAsBit("AddBuiltinPrefixedAlias"))
-          EmitBuiltinDef(OS, Substitution, Builtin,
-                         std::string("__builtin_") + FullSpelling,
-                         BuiltinType::Builtin);
+        if (BuiltinRecord->getValueAsBit("AddBuiltinPrefixedAlias"))
+          Builtins.push_back(buildBuiltin(
+              Substitution, BuiltinRecord,
+              std::string("__builtin_") + FullSpelling, BuiltinType::Builtin));
       }
-      EmitBuiltinDef(OS, Substitution, Builtin, FullSpelling, BT);
+      Builtins.push_back(
+          buildBuiltin(Substitution, BuiltinRecord, FullSpelling, BT));
     }
   }
 }
@@ -386,47 +482,112 @@ void EmitBuiltin(raw_ostream &OS, const Record *Builtin) {
 void clang::EmitClangBuiltins(const RecordKeeper &Records, raw_ostream &OS) {
   emitSourceFileHeader("List of builtins that Clang recognizes", OS);
 
-  OS << R"c++(
-#if defined(BUILTIN) && !defined(LIBBUILTIN)
-#  define LIBBUILTIN(ID, TYPE, ATTRS, HEADER, BUILTIN_LANG) BUILTIN(ID, TYPE, ATTRS)
-#endif
-
-#if defined(BUILTIN) && !defined(LANGBUILTIN)
-#  define LANGBUILTIN(ID, TYPE, ATTRS, BUILTIN_LANG) BUILTIN(ID, TYPE, ATTRS)
-#endif
-
-// Some of our atomics builtins are handled by AtomicExpr rather than
-// as normal builtin CallExprs. This macro is used for such builtins.
-#ifndef ATOMIC_BUILTIN
-#  define ATOMIC_BUILTIN(ID, TYPE, ATTRS) BUILTIN(ID, TYPE, ATTRS)
-#endif
-
-#if defined(BUILTIN) && !defined(TARGET_BUILTIN)
-#  define TARGET_BUILTIN(ID, TYPE, ATTRS, FEATURE) BUILTIN(ID, TYPE, ATTRS)
-#endif
-
-#if defined(BUILTIN) && !defined(TARGET_HEADER_BUILTIN)
-#  define TARGET_HEADER_BUILTIN(ID, TYPE, ATTRS, HEADER, LANG, FEATURE) BUILTIN(ID, TYPE, ATTRS)
-#endif
-)c++";
+  SmallVector<Builtin> Builtins;
+  // AtomicBuiltins are order dependent. Emit them first to make manual checking
+  // easier and so we can build a special atomic builtin X-macro.
+  for (const auto *BuiltinRecord :
+       Records.getAllDerivedDefinitions("AtomicBuiltin"))
+    collectBuiltins(BuiltinRecord, Builtins);
+  unsigned NumAtomicBuiltins = Builtins.size();
+
+  for (const auto *BuiltinRecord :
+       Records.getAllDerivedDefinitions("Builtin")) {
+    if (BuiltinRecord->isSubClassOf("AtomicBuiltin"))
+      continue;
+    // Prefixed builtins are also special and we emit them last so they can have
+    // their own representation that skips the prefix.
+    if (BuiltinRecord->getValueAsOptionalDef("RequiredNamePrefix"))
+      continue;
 
-  // AtomicBuiltins are order dependent
-  // emit them first to make manual checking easier
-  for (const auto *Builtin : Records.getAllDerivedDefinitions("AtomicBuiltin"))
-    EmitBuiltin(OS, Builtin);
+    collectBuiltins(BuiltinRecord, Builtins);
+  }
 
-  for (const auto *Builtin : Records.getAllDerivedDefinitions("Builtin")) {
-    if (Builtin->isSubClassOf("AtomicBuiltin"))
+  // Now collect (and count) the prefixed builtins.
+  unsigned NumPrefixedBuiltins = Builtins.size();
+  const Record *FirstPrefix = nullptr;
+  for (const auto *BuiltinRecord :
+       Records.getAllDerivedDefinitions("Builtin")) {
+    auto *Prefix = BuiltinRecord->getValueAsOptionalDef("RequiredNamePrefix");
+    if (!Prefix)
       continue;
-    EmitBuiltin(OS, Builtin);
+
+    if (!FirstPrefix)
+      FirstPrefix = Prefix;
+    assert(Prefix == FirstPrefix &&
+           "Multiple distinct prefixes which is not currently supported!");
+    assert(!BuiltinRecord->isSubClassOf("AtomicBuiltin") &&
+           "Cannot require a name prefix for an atomic builtin.");
+    collectBuiltins(BuiltinRecord, Builtins);
   }
+  NumPrefixedBuiltins = Builtins.size() - NumPrefixedBuiltins;
+
+  auto AtomicBuiltins = ArrayRef(Builtins).slice(0, NumAtomicBuiltins);
+  auto UnprefixedBuiltins = ArrayRef(Builtins).drop_back(NumPrefixedBuiltins);
+  auto PrefixedBuiltins = ArrayRef(Builtins).take_back(NumPrefixedBuiltins);
+
+  // Collect strings into a table.
+  StringToOffsetTable Table;
+  Table.GetOrAddStringOffset("");
+  for (const auto &B : Builtins) {
+    Table.GetOrAddStringOffset(B.Name);
+    Table.GetOrAddStringOffset(B.Type);
+    Table.GetOrAddStringOffset(B.Attributes);
+    if (B.BT == BuiltinType::TargetBuiltin)
+      Table.GetOrAddStringOffset(B.BuiltinRecord->getValueAsString("Features"));
+  }
+
+  // Emit enumerators.
+  OS << R"c++(
+#ifdef GET_BUILTIN_ENUMERATORS
+)c++";
+  for (const auto &B : Builtins)
+    B.EmitEnumerator(OS);
+  OS << R"c++(
+#endif // GET_BUILTIN_ENUMERATORS
+)c++";
+
+  // Emit a string table that can be referenced for these builtins.
+  OS << R"c++(
+#ifdef GET_BUILTIN_STR_TABLE
+)c++";
+  Table.EmitStringTableDef(OS, "BuiltinStrings");
+  OS << R"c++(
+#endif // GET_BUILTIN_STR_TABLE
+)c++";
+
+  // Emit a direct set of `Builtin::Info` initializers, first for the unprefixed
+  // builtins and then for the prefixed builtins.
+  OS << R"c++(
+#ifdef GET_BUILTIN_INFOS
+)c++";
+  for (const auto &B : UnprefixedBuiltins)
+    B.EmitInfo(OS, Table);
+  OS << R"c++(
+#endif // GET_BUILTIN_INFOS
+)c++";
+
+  OS << R"c++(
+#ifdef GET_BUILTIN_PREFIXED_INFOS
+)c++";
+  for (const auto &B : PrefixedBuiltins)
+    B.EmitInfo(OS, Table);
+  OS << R"c++(
+#endif // GET_BUILTIN_PREFIXED_INFOS
+)c++";
 
+  // Emit X-macros for the atomic builtins to support various custome patterns
+  // used exclusively with those builtins.
+  //
+  // FIXME: We should eventually move this to a separate file so that users
+  // don't need to include the full set of builtins.
+  OS << R"c++(
+#ifdef ATOMIC_BUILTIN
+)c++";
+  for (const auto &Builtin : AtomicBuiltins) {
+    Builtin.EmitXMacro(OS);
+  }
   OS << R"c++(
+#endif // ATOMIC_BUILTIN
 #undef ATOMIC_BUILTIN
-#undef BUILTIN
-#undef LIBBUILTIN
-#undef LANGBUILTIN
-#undef TARGET_BUILTIN
-#undef TARGET_HEADER_BUILTIN
 )c++";
 }
diff --git a/clang/utils/TableGen/ClangDiagnosticsEmitter.cpp b/clang/utils/TableGen/ClangDiagnosticsEmitter.cpp
index 5f03efdb80434..8f846a4744bbf 100644
--- a/clang/utils/TableGen/ClangDiagnosticsEmitter.cpp
+++ b/clang/utils/TableGen/ClangDiagnosticsEmitter.cpp
@@ -359,7 +359,7 @@ void InferPedantic::compute(VecOrSet DiagsInPedantic,
 
     // The diagnostic is not included in a group that is (transitively) in
     // -Wpedantic.  Include it in -Wpedantic directly.
-    if (auto *V = DiagsInPedantic.dyn_cast<RecordVec *>())
+    if (auto *V = dyn_cast<RecordVec *>(DiagsInPedantic))
       V->push_back(R);
     else
       cast<RecordSet *>(DiagsInPedantic)->insert(R);
@@ -386,7 +386,7 @@ void InferPedantic::compute(VecOrSet DiagsInPedantic,
     if (Parents.size() > 0 && AllParentsInPedantic)
       continue;
 
-    if (auto *V = GroupsInPedantic.dyn_cast<RecordVec *>())
+    if (auto *V = dyn_cast<RecordVec *>(GroupsInPedantic))
       V->push_back(Group);
     else
       cast<RecordSet *>(GroupsInPedantic)->insert(Group);
@@ -1785,8 +1785,7 @@ static void emitDiagArrays(DiagsInGroupTy &DiagsInGroup,
 /// This creates an `llvm::StringTable` of all the diagnostic group names.
 static void emitDiagGroupNames(const StringToOffsetTable &GroupNames,
                                raw_ostream &OS) {
-  GroupNames.EmitStringLiteralDef(
-      OS, "static constexpr llvm::StringTable DiagGroupNames");
+  GroupNames.EmitStringTableDef(OS, "DiagGroupNames");
   OS << "\n";
 }
 
@@ -1939,9 +1938,6 @@ void clang::EmitClangDiagGroups(const RecordKeeper &Records, raw_ostream &OS) {
   inferPedantic.compute(&DiagsInPedantic, &GroupsInPedantic);
 
   StringToOffsetTable GroupNames;
-  // Add an empty string to the table first so we can use `llvm::StringTable`.
-  // TODO: Factor this into `StringToOffsetTable`.
-  GroupNames.GetOrAddStringOffset("");
   for (const auto &[Name, Group] : DiagsInGroup) {
     GroupNames.GetOrAddStringOffset(Name);
   }
diff --git a/clang/utils/TableGen/ClangOptionDocEmitter.cpp b/clang/utils/TableGen/ClangOptionDocEmitter.cpp
index e08fb11df3100..b6c1aad90b5cb 100644
--- a/clang/utils/TableGen/ClangOptionDocEmitter.cpp
+++ b/clang/utils/TableGen/ClangOptionDocEmitter.cpp
@@ -109,13 +109,17 @@ Documentation extractDocumentation(const RecordKeeper &Records,
     // Pretend no-X and Xno-Y options are aliases of X and XY.
     std::string Name = std::string(R->getValueAsString("Name"));
     if (Name.size() >= 4) {
-      if (Name.substr(0, 3) == "no-" && OptionsByName[Name.substr(3)]) {
-        Aliases[OptionsByName[Name.substr(3)]].push_back(R);
-        continue;
+      if (Name.substr(0, 3) == "no-") {
+        if (const Record *Opt = OptionsByName[Name.substr(3)]) {
+          Aliases[Opt].push_back(R);
+          continue;
+        }
       }
-      if (Name.substr(1, 3) == "no-" && OptionsByName[Name[0] + Name.substr(4)]) {
-        Aliases[OptionsByName[Name[0] + Name.substr(4)]].push_back(R);
-        continue;
+      if (Name.substr(1, 3) == "no-") {
+        if (const Record *Opt = OptionsByName[Name[0] + Name.substr(4)]) {
+          Aliases[Opt].push_back(R);
+          continue;
+        }
       }
     }
 
diff --git a/clang/utils/TableGen/MveEmitter.cpp b/clang/utils/TableGen/MveEmitter.cpp
index 8ebd0bb800fef..7bee2996382c1 100644
--- a/clang/utils/TableGen/MveEmitter.cpp
+++ b/clang/utils/TableGen/MveEmitter.cpp
@@ -209,9 +209,7 @@ class PointerType : public Type {
       Name = "const " + Name;
     return Name + " *";
   }
-  std::string llvmName() const override {
-    return "llvm::PointerType::getUnqual(" + Pointee->llvmName() + ")";
-  }
+  std::string llvmName() const override { return "Builder.getPtrTy()"; }
   const Type *getPointeeType() const { return Pointee; }
 
   static bool classof(const Type *T) {
@@ -1631,17 +1629,10 @@ void EmitterBase::EmitBuiltinCG(raw_ostream &OS) {
       for (const auto &OI : kv.second)
         key.push_back(OI.ParamValues[i]);
 
-      auto Found = ParamNumberMap.find(key);
-      if (Found != ParamNumberMap.end()) {
-        // Yes, an existing parameter variable can be reused for this.
-        ParamNumbers.push_back(Found->second);
-        continue;
-      }
-
-      // No, we need a new parameter variable.
-      int ExistingIndex = ParamNumberMap.size();
-      ParamNumberMap[key] = ExistingIndex;
-      ParamNumbers.push_back(ExistingIndex);
+      // Obtain a new parameter variable if we don't have one.
+      int ParamNum =
+          ParamNumberMap.try_emplace(key, ParamNumberMap.size()).first->second;
+      ParamNumbers.push_back(ParamNum);
     }
 
     // Now we're ready to do the pass 2 code generation, which will emit the
@@ -1949,27 +1940,53 @@ void MveEmitter::EmitHeader(raw_ostream &OS) {
 }
 
 void MveEmitter::EmitBuiltinDef(raw_ostream &OS) {
-  for (const auto &kv : ACLEIntrinsics) {
-    const ACLEIntrinsic &Int = *kv.second;
-    OS << "BUILTIN(__builtin_arm_mve_" << Int.fullName()
-       << ", \"\", \"n\")\n";
+  llvm::StringToOffsetTable Table;
+  Table.GetOrAddStringOffset("n");
+  Table.GetOrAddStringOffset("nt");
+  Table.GetOrAddStringOffset("ntu");
+  Table.GetOrAddStringOffset("vi.");
+
+  for (const auto &[_, Int] : ACLEIntrinsics)
+    Table.GetOrAddStringOffset(Int->fullName());
+
+  std::map<std::string, ACLEIntrinsic *> ShortNameIntrinsics;
+  for (const auto &[_, Int] : ACLEIntrinsics) {
+    if (!Int->polymorphic())
+      continue;
+
+    StringRef Name = Int->shortName();
+    if (ShortNameIntrinsics.insert({Name.str(), Int.get()}).second)
+      Table.GetOrAddStringOffset(Name);
   }
 
-  std::set<std::string> ShortNamesSeen;
+  OS << "#ifdef GET_MVE_BUILTIN_ENUMERATORS\n";
+  for (const auto &[_, Int] : ACLEIntrinsics) {
+    OS << "  BI__builtin_arm_mve_" << Int->fullName() << ",\n";
+  }
+  for (const auto &[Name, _] : ShortNameIntrinsics) {
+    OS << "  BI__builtin_arm_mve_" << Name << ",\n";
+  }
+  OS << "#endif // GET_MVE_BUILTIN_ENUMERATORS\n\n";
 
-  for (const auto &kv : ACLEIntrinsics) {
-    const ACLEIntrinsic &Int = *kv.second;
-    if (Int.polymorphic()) {
-      StringRef Name = Int.shortName();
-      if (ShortNamesSeen.find(std::string(Name)) == ShortNamesSeen.end()) {
-        OS << "BUILTIN(__builtin_arm_mve_" << Name << ", \"vi.\", \"nt";
-        if (Int.nonEvaluating())
-          OS << "u"; // indicate that this builtin doesn't evaluate its args
-        OS << "\")\n";
-        ShortNamesSeen.insert(std::string(Name));
-      }
-    }
+  OS << "#ifdef GET_MVE_BUILTIN_STR_TABLE\n";
+  Table.EmitStringTableDef(OS, "BuiltinStrings");
+  OS << "#endif // GET_MVE_BUILTIN_STR_TABLE\n\n";
+
+  OS << "#ifdef GET_MVE_BUILTIN_INFOS\n";
+  for (const auto &[_, Int] : ACLEIntrinsics) {
+    OS << "    Builtin::Info{Builtin::Info::StrOffsets{"
+       << Table.GetStringOffset(Int->fullName()) << " /* " << Int->fullName()
+       << " */, " << Table.GetStringOffset("") << ", "
+       << Table.GetStringOffset("n") << " /* n */}},\n";
   }
+  for (const auto &[Name, Int] : ShortNameIntrinsics) {
+    StringRef Attrs = Int->nonEvaluating() ? "ntu" : "nt";
+    OS << "    Builtin::Info{Builtin::Info::StrOffsets{"
+       << Table.GetStringOffset(Name) << " /* " << Name << " */, "
+       << Table.GetStringOffset("vi.") << " /* vi. */, "
+       << Table.GetStringOffset(Attrs) << " /* " << Attrs << " */}},\n";
+  }
+  OS << "#endif // GET_MVE_BUILTIN_INFOS\n\n";
 }
 
 void MveEmitter::EmitBuiltinSema(raw_ostream &OS) {
@@ -2157,13 +2174,31 @@ void CdeEmitter::EmitHeader(raw_ostream &OS) {
 }
 
 void CdeEmitter::EmitBuiltinDef(raw_ostream &OS) {
-  for (const auto &kv : ACLEIntrinsics) {
-    if (kv.second->headerOnly())
-      continue;
-    const ACLEIntrinsic &Int = *kv.second;
-    OS << "BUILTIN(__builtin_arm_cde_" << Int.fullName()
-       << ", \"\", \"ncU\")\n";
-  }
+  llvm::StringToOffsetTable Table;
+  Table.GetOrAddStringOffset("ncU");
+
+  for (const auto &[_, Int] : ACLEIntrinsics)
+    if (!Int->headerOnly())
+      Table.GetOrAddStringOffset(Int->fullName());
+
+  OS << "#ifdef GET_CDE_BUILTIN_ENUMERATORS\n";
+  for (const auto &[_, Int] : ACLEIntrinsics)
+    if (!Int->headerOnly())
+      OS << "  BI__builtin_arm_cde_" << Int->fullName() << ",\n";
+  OS << "#endif // GET_CDE_BUILTIN_ENUMERATORS\n\n";
+
+  OS << "#ifdef GET_CDE_BUILTIN_STR_TABLE\n";
+  Table.EmitStringTableDef(OS, "BuiltinStrings");
+  OS << "#endif // GET_CDE_BUILTIN_STR_TABLE\n\n";
+
+  OS << "#ifdef GET_CDE_BUILTIN_INFOS\n";
+  for (const auto &[_, Int] : ACLEIntrinsics)
+    if (!Int->headerOnly())
+      OS << "    Builtin::Info{Builtin::Info::StrOffsets{"
+         << Table.GetStringOffset(Int->fullName()) << " /* " << Int->fullName()
+         << " */, " << Table.GetStringOffset("") << ", "
+         << Table.GetStringOffset("ncU") << " /* ncU */}},\n";
+  OS << "#endif // GET_CDE_BUILTIN_INFOS\n\n";
 }
 
 void CdeEmitter::EmitBuiltinSema(raw_ostream &OS) {
diff --git a/clang/utils/TableGen/NeonEmitter.cpp b/clang/utils/TableGen/NeonEmitter.cpp
index 49633bb7b7f58..a18f78697af1c 100644
--- a/clang/utils/TableGen/NeonEmitter.cpp
+++ b/clang/utils/TableGen/NeonEmitter.cpp
@@ -37,6 +37,7 @@
 #include "llvm/TableGen/Error.h"
 #include "llvm/TableGen/Record.h"
 #include "llvm/TableGen/SetTheory.h"
+#include "llvm/TableGen/StringToOffsetTable.h"
 #include <algorithm>
 #include <cassert>
 #include <cctype>
@@ -74,6 +75,7 @@ enum ClassKind {
   ClassI,     // generic integer instruction, e.g., "i8" suffix
   ClassS,     // signed/unsigned/poly, e.g., "s8", "u8" or "p8" suffix
   ClassW,     // width-specific instruction, e.g., "8" suffix
+  ClassV,     // void-suffix instruction, no suffix
   ClassB,     // bitcast arguments with enum argument to specify type
   ClassL,     // Logical instructions which are op instructions
               // but we need to not emit any suffix for in our
@@ -102,7 +104,7 @@ enum EltType {
   Float32,
   Float64,
   BFloat16,
-  MFloat8 // Not used by Sema or CodeGen in Clang
+  MFloat8
 };
 
 } // end namespace NeonTypeFlags
@@ -144,7 +146,7 @@ class Type {
 private:
   TypeSpec TS;
 
-  enum TypeKind { Void, Float, SInt, UInt, Poly, BFloat16, MFloat8 };
+  enum TypeKind { Void, Float, SInt, UInt, Poly, BFloat16, MFloat8, FPM };
   TypeKind Kind;
   bool Immediate, Constant, Pointer;
   // ScalarForMangling and NoManglingQ are really not suited to live here as
@@ -198,6 +200,7 @@ class Type {
   bool isVoid() const { return Kind == Void; }
   bool isBFloat16() const { return Kind == BFloat16; }
   bool isMFloat8() const { return Kind == MFloat8; }
+  bool isFPM() const { return Kind == FPM; }
   unsigned getNumElements() const { return Bitwidth / ElementBitwidth; }
   unsigned getSizeInBits() const { return Bitwidth; }
   unsigned getElementSizeInBits() const { return ElementBitwidth; }
@@ -600,6 +603,7 @@ class NeonEmitter {
     const Record *SI = R.getClass("SInst");
     const Record *II = R.getClass("IInst");
     const Record *WI = R.getClass("WInst");
+    const Record *VI = R.getClass("VInst");
     const Record *SOpI = R.getClass("SOpInst");
     const Record *IOpI = R.getClass("IOpInst");
     const Record *WOpI = R.getClass("WOpInst");
@@ -609,6 +613,7 @@ class NeonEmitter {
     ClassMap[SI] = ClassS;
     ClassMap[II] = ClassI;
     ClassMap[WI] = ClassW;
+    ClassMap[VI] = ClassV;
     ClassMap[SOpI] = ClassS;
     ClassMap[IOpI] = ClassI;
     ClassMap[WOpI] = ClassW;
@@ -641,6 +646,9 @@ class NeonEmitter {
 std::string Type::str() const {
   if (isVoid())
     return "void";
+  if (isFPM())
+    return "fpm_t";
+
   std::string S;
 
   if (isInteger() && !isSigned())
@@ -699,6 +707,8 @@ std::string Type::builtin_str() const {
   } else if (isMFloat8()) {
     assert(ElementBitwidth == 8 && "MFloat8 can only be 8 bits");
     S += "m";
+  } else if (isFPM()) {
+    S += "UWi";
   } else
     switch (ElementBitwidth) {
     case 16: S += "h"; break;
@@ -925,6 +935,13 @@ void Type::applyModifiers(StringRef Mods) {
     case 'P':
       Kind = Poly;
       break;
+    case 'V':
+      Kind = FPM;
+      Bitwidth = ElementBitwidth = 64;
+      NumVectors = 0;
+      Immediate = Constant = Pointer = false;
+      ScalarForMangling = NoManglingQ = true;
+      break;
     case '>':
       assert(ElementBitwidth < 128);
       ElementBitwidth *= 2;
@@ -1000,11 +1017,14 @@ std::string Intrinsic::getInstTypeCode(Type T, ClassKind CK) const {
   if (CK == ClassB && TargetGuard == "neon")
     return "";
 
+  if (this->CK == ClassV)
+    return "";
+
   if (T.isBFloat16())
     return "bf16";
 
   if (T.isMFloat8())
-    return "mfp8";
+    return "mf8";
 
   if (T.isPoly())
     typeCode = 'p';
@@ -2042,40 +2062,51 @@ void NeonEmitter::createIntrinsic(const Record *R,
   CurrentRecord = nullptr;
 }
 
-/// genBuiltinsDef: Generate the BuiltinsARM.def and  BuiltinsAArch64.def
-/// declaration of builtins, checking for unique builtin declarations.
+/// genBuiltinsDef: Generate the builtin infos, checking for unique builtin
+/// declarations.
 void NeonEmitter::genBuiltinsDef(raw_ostream &OS,
                                  SmallVectorImpl<Intrinsic *> &Defs) {
-  OS << "#ifdef GET_NEON_BUILTINS\n";
+  // We only want to emit a builtin once, and in order of its name.
+  std::map<std::string, Intrinsic *> Builtins;
 
-  // We only want to emit a builtin once, and we want to emit them in
-  // alphabetical order, so use a std::set.
-  std::set<std::pair<std::string, std::string>> Builtins;
+  llvm::StringToOffsetTable Table;
+  Table.GetOrAddStringOffset("");
+  Table.GetOrAddStringOffset("n");
 
   for (auto *Def : Defs) {
     if (Def->hasBody())
       continue;
 
-    std::string S = "__builtin_neon_" + Def->getMangledName() + ", \"";
-    S += Def->getBuiltinTypeStr();
-    S += "\", \"n\"";
-
-    Builtins.emplace(S, Def->getTargetGuard());
+    if (Builtins.insert({Def->getMangledName(), Def}).second) {
+      Table.GetOrAddStringOffset(Def->getMangledName());
+      Table.GetOrAddStringOffset(Def->getBuiltinTypeStr());
+      Table.GetOrAddStringOffset(Def->getTargetGuard());
+    }
   }
 
-  for (auto &S : Builtins) {
-    if (S.second == "")
-      OS << "BUILTIN(";
-    else
-      OS << "TARGET_BUILTIN(";
-    OS << S.first;
-    if (S.second == "")
-      OS << ")\n";
-    else
-      OS << ", \"" << S.second << "\")\n";
+  OS << "#ifdef GET_NEON_BUILTIN_ENUMERATORS\n";
+  for (const auto &[Name, Def] : Builtins) {
+    OS << "  BI__builtin_neon_" << Name << ",\n";
   }
+  OS << "#endif // GET_NEON_BUILTIN_ENUMERATORS\n\n";
 
-  OS << "#endif\n\n";
+  OS << "#ifdef GET_NEON_BUILTIN_STR_TABLE\n";
+  Table.EmitStringTableDef(OS, "BuiltinStrings");
+  OS << "#endif // GET_NEON_BUILTIN_STR_TABLE\n\n";
+
+  OS << "#ifdef GET_NEON_BUILTIN_INFOS\n";
+  for (const auto &[Name, Def] : Builtins) {
+    OS << "    Builtin::Info{Builtin::Info::StrOffsets{"
+       << Table.GetStringOffset(Def->getMangledName()) << " /* "
+       << Def->getMangledName() << " */, ";
+    OS << Table.GetStringOffset(Def->getBuiltinTypeStr()) << " /* "
+       << Def->getBuiltinTypeStr() << " */, ";
+    OS << Table.GetStringOffset("n") << " /* n */, ";
+    OS << Table.GetStringOffset(Def->getTargetGuard()) << " /* "
+       << Def->getTargetGuard() << " */}, ";
+    OS << "HeaderDesc::NO_HEADER, ALL_LANGUAGES},\n";
+  }
+  OS << "#endif // GET_NEON_BUILTIN_INFOS\n\n";
 }
 
 void NeonEmitter::genStreamingSVECompatibleList(
@@ -2281,9 +2312,7 @@ static void emitNeonTypeDefs(const std::string& types, raw_ostream &OS) {
       InIfdef = true;
     }
 
-    if (T.isMFloat8())
-      OS << "typedef __MFloat8x";
-    else if (T.isPoly())
+    if (T.isPoly())
       OS << "typedef __attribute__((neon_polyvector_type(";
     else
       OS << "typedef __attribute__((neon_vector_type(";
@@ -2291,10 +2320,7 @@ static void emitNeonTypeDefs(const std::string& types, raw_ostream &OS) {
     Type T2 = T;
     T2.makeScalar();
     OS << T.getNumElements();
-    if (T.isMFloat8())
-      OS << "_t ";
-    else
-      OS << "))) " << T2.str();
+    OS << "))) " << T2.str();
     OS << " " << T.str() << ";\n";
   }
   if (InIfdef)
diff --git a/clang/utils/TableGen/RISCVVEmitter.cpp b/clang/utils/TableGen/RISCVVEmitter.cpp
index acba1a3191281..0cdde20060b63 100644
--- a/clang/utils/TableGen/RISCVVEmitter.cpp
+++ b/clang/utils/TableGen/RISCVVEmitter.cpp
@@ -18,10 +18,12 @@
 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/StringExtras.h"
 #include "llvm/ADT/StringMap.h"
+#include "llvm/ADT/StringRef.h"
 #include "llvm/ADT/StringSwitch.h"
 #include "llvm/ADT/Twine.h"
 #include "llvm/TableGen/Error.h"
 #include "llvm/TableGen/Record.h"
+#include "llvm/TableGen/StringToOffsetTable.h"
 #include <optional>
 
 using namespace llvm;
@@ -498,31 +500,68 @@ void RVVEmitter::createBuiltins(raw_ostream &OS) {
   std::vector<std::unique_ptr<RVVIntrinsic>> Defs;
   createRVVIntrinsics(Defs);
 
-  // Map to keep track of which builtin names have already been emitted.
-  StringMap<RVVIntrinsic *> BuiltinMap;
+  llvm::StringToOffsetTable Table;
+  // Ensure offset zero is the empty string.
+  Table.GetOrAddStringOffset("");
+  // Hard coded strings used in the builtin structures.
+  Table.GetOrAddStringOffset("n");
+  Table.GetOrAddStringOffset("zve32x");
 
-  OS << "#if defined(TARGET_BUILTIN) && !defined(RISCVV_BUILTIN)\n";
-  OS << "#define RISCVV_BUILTIN(ID, TYPE, ATTRS) TARGET_BUILTIN(ID, TYPE, "
-        "ATTRS, \"zve32x\")\n";
-  OS << "#endif\n";
+  // Map to unique the builtin names.
+  StringMap<RVVIntrinsic *> BuiltinMap;
+  std::vector<RVVIntrinsic *> UniqueDefs;
   for (auto &Def : Defs) {
-    auto P =
-        BuiltinMap.insert(std::make_pair(Def->getBuiltinName(), Def.get()));
-    if (!P.second) {
-      // Verf that this would have produced the same builtin definition.
-      if (P.first->second->hasBuiltinAlias() != Def->hasBuiltinAlias())
-        PrintFatalError("Builtin with same name has different hasAutoDef");
-      else if (!Def->hasBuiltinAlias() &&
-               P.first->second->getBuiltinTypeStr() != Def->getBuiltinTypeStr())
-        PrintFatalError("Builtin with same name has different type string");
+    auto P = BuiltinMap.insert({Def->getBuiltinName(), Def.get()});
+    if (P.second) {
+      Table.GetOrAddStringOffset(Def->getBuiltinName());
+      if (!Def->hasBuiltinAlias())
+        Table.GetOrAddStringOffset(Def->getBuiltinTypeStr());
+      UniqueDefs.push_back(Def.get());
       continue;
     }
-    OS << "RISCVV_BUILTIN(__builtin_rvv_" << Def->getBuiltinName() << ",\"";
-    if (!Def->hasBuiltinAlias())
-      OS << Def->getBuiltinTypeStr();
-    OS << "\", \"n\")\n";
+
+    // Verf that this would have produced the same builtin definition.
+    if (P.first->second->hasBuiltinAlias() != Def->hasBuiltinAlias())
+      PrintFatalError("Builtin with same name has different hasAutoDef");
+    else if (!Def->hasBuiltinAlias() &&
+             P.first->second->getBuiltinTypeStr() != Def->getBuiltinTypeStr())
+      PrintFatalError("Builtin with same name has different type string");
+  }
+
+  // Emit the enumerators of RVV builtins. Note that these are emitted without
+  // any outer context to enable concatenating them.
+  OS << "// RISCV Vector builtin enumerators\n";
+  OS << "#ifdef GET_RISCVV_BUILTIN_ENUMERATORS\n";
+  for (RVVIntrinsic *Def : UniqueDefs)
+    OS << "  BI__builtin_rvv_" << Def->getBuiltinName() << ",\n";
+  OS << "#endif // GET_RISCVV_BUILTIN_ENUMERATORS\n\n";
+
+  // Emit the string table for the RVV builtins.
+  OS << "// RISCV Vector builtin enumerators\n";
+  OS << "#ifdef GET_RISCVV_BUILTIN_STR_TABLE\n";
+  Table.EmitStringTableDef(OS, "BuiltinStrings");
+  OS << "#endif // GET_RISCVV_BUILTIN_STR_TABLE\n\n";
+
+  // Emit the info structs of RVV builtins. Note that these are emitted without
+  // any outer context to enable concatenating them.
+  OS << "// RISCV Vector builtin infos\n";
+  OS << "#ifdef GET_RISCVV_BUILTIN_INFOS\n";
+  for (RVVIntrinsic *Def : UniqueDefs) {
+    OS << "    Builtin::Info{Builtin::Info::StrOffsets{"
+       << Table.GetStringOffset(Def->getBuiltinName()) << " /* "
+       << Def->getBuiltinName() << " */, ";
+    if (Def->hasBuiltinAlias()) {
+      OS << "0, ";
+    } else {
+      OS << Table.GetStringOffset(Def->getBuiltinTypeStr()) << " /* "
+         << Def->getBuiltinTypeStr() << " */, ";
+    }
+    OS << Table.GetStringOffset("n") << " /* n */, ";
+    OS << Table.GetStringOffset("zve32x") << " /* zve32x */}, ";
+
+    OS << "HeaderDesc::NO_HEADER, ALL_LANGUAGES},\n";
   }
-  OS << "#undef RISCVV_BUILTIN\n";
+  OS << "#endif // GET_RISCVV_BUILTIN_INFOS\n\n";
 }
 
 void RVVEmitter::createCodeGen(raw_ostream &OS) {
diff --git a/clang/utils/TableGen/SveEmitter.cpp b/clang/utils/TableGen/SveEmitter.cpp
index 0ecbf7cede1da..e226987b4844b 100644
--- a/clang/utils/TableGen/SveEmitter.cpp
+++ b/clang/utils/TableGen/SveEmitter.cpp
@@ -27,9 +27,11 @@
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/StringExtras.h"
 #include "llvm/ADT/StringMap.h"
+#include "llvm/Support/raw_ostream.h"
 #include "llvm/TableGen/AArch64ImmCheck.h"
 #include "llvm/TableGen/Error.h"
 #include "llvm/TableGen/Record.h"
+#include "llvm/TableGen/StringToOffsetTable.h"
 #include <array>
 #include <cctype>
 #include <set>
@@ -181,6 +183,8 @@ class Intrinsic {
 
   SmallVector<ImmCheck, 2> ImmChecks;
 
+  bool SetsFPMR;
+
 public:
   Intrinsic(StringRef Name, StringRef Proto, uint64_t MergeTy,
             StringRef MergeSuffix, uint64_t MemoryElementTy, StringRef LLVMName,
@@ -198,7 +202,9 @@ class Intrinsic {
 
   StringRef getSVEGuard() const { return SVEGuard; }
   StringRef getSMEGuard() const { return SMEGuard; }
-  void printGuard(raw_ostream &OS) const {
+  std::string getGuard() const {
+    std::string Guard;
+    llvm::raw_string_ostream OS(Guard);
     if (!SVEGuard.empty() && SMEGuard.empty())
       OS << SVEGuard;
     else if (SVEGuard.empty() && !SMEGuard.empty())
@@ -216,6 +222,7 @@ class Intrinsic {
       else
         OS << SMEGuard;
     }
+    return Guard;
   }
   ClassKind getClassKind() const { return Class; }
 
@@ -278,6 +285,7 @@ class Intrinsic {
 
 private:
   std::string getMergeSuffix() const { return MergeSuffix; }
+  StringRef getFPMSuffix() const { return SetsFPMR ? "_fpm" : ""; }
   std::string mangleName(ClassKind LocalCK) const;
   std::string mangleLLVMName() const;
   std::string replaceTemplatedArgs(std::string Name, TypeSpec TS,
@@ -449,7 +457,7 @@ std::string SVEType::builtinBaseType() const {
   case TypeKind::PredicatePattern:
     return "i";
   case TypeKind::Fpm:
-    return "Wi";
+    return "UWi";
   case TypeKind::Predicate:
     return "b";
   case TypeKind::BFloat16:
@@ -457,7 +465,7 @@ std::string SVEType::builtinBaseType() const {
     return "y";
   case TypeKind::MFloat8:
     assert(ElementBitwidth == 8 && "Invalid MFloat8!");
-    return "c";
+    return "m";
   case TypeKind::Float:
     switch (ElementBitwidth) {
     case 16:
@@ -983,6 +991,7 @@ Intrinsic::Intrinsic(StringRef Name, StringRef Proto, uint64_t MergeTy,
     std::tie(Mod, NumVectors) = getProtoModifier(Proto, I);
     SVEType T(BaseTypeSpec, Mod, NumVectors);
     Types.push_back(T);
+    SetsFPMR = T.isFpm();
 
     // Add range checks for immediates
     if (I > 0) {
@@ -1001,6 +1010,8 @@ Intrinsic::Intrinsic(StringRef Name, StringRef Proto, uint64_t MergeTy,
   this->Flags |= Emitter.encodeMergeType(MergeTy);
   if (hasSplat())
     this->Flags |= Emitter.encodeSplatOperand(getSplatIdx());
+  if (SetsFPMR)
+    this->Flags |= Emitter.getEnumValueForFlag("SetsFPMR");
 }
 
 std::string Intrinsic::getBuiltinTypeStr() {
@@ -1089,8 +1100,9 @@ std::string Intrinsic::mangleName(ClassKind LocalCK) const {
   }
 
   // Replace all {d} like expressions with e.g. 'u32'
-  return replaceTemplatedArgs(S, getBaseTypeSpec(), getProto()) +
-         getMergeSuffix();
+  return replaceTemplatedArgs(S, getBaseTypeSpec(), getProto())
+      .append(getMergeSuffix())
+      .append(getFPMSuffix());
 }
 
 void Intrinsic::emitIntrinsic(raw_ostream &OS, SVEEmitter &Emitter,
@@ -1472,19 +1484,19 @@ void SVEEmitter::createBuiltins(raw_ostream &OS) {
     return A->getMangledName() < B->getMangledName();
   });
 
-  OS << "#ifdef GET_SVE_BUILTINS\n";
-  for (auto &Def : Defs) {
-    // Only create BUILTINs for non-overloaded intrinsics, as overloaded
-    // declarations only live in the header file.
+  llvm::StringToOffsetTable Table;
+  Table.GetOrAddStringOffset("");
+  Table.GetOrAddStringOffset("n");
+
+  for (const auto &Def : Defs)
     if (Def->getClassKind() != ClassG) {
-      OS << "TARGET_BUILTIN(__builtin_sve_" << Def->getMangledName() << ", \""
-         << Def->getBuiltinTypeStr() << "\", \"n\", \"";
-      Def->printGuard(OS);
-      OS << "\")\n";
+      Table.GetOrAddStringOffset(Def->getMangledName());
+      Table.GetOrAddStringOffset(Def->getBuiltinTypeStr());
+      Table.GetOrAddStringOffset(Def->getGuard());
     }
-  }
 
-  // Add reinterpret functions.
+  Table.GetOrAddStringOffset("sme|sve");
+  SmallVector<std::pair<std::string, std::string>> ReinterpretBuiltins;
   for (auto [N, Suffix] :
        std::initializer_list<std::pair<unsigned, const char *>>{
            {1, ""}, {2, "_x2"}, {3, "_x3"}, {4, "_x4"}}) {
@@ -1492,14 +1504,54 @@ void SVEEmitter::createBuiltins(raw_ostream &OS) {
       SVEType ToV(To.BaseType, N);
       for (const ReinterpretTypeInfo &From : Reinterprets) {
         SVEType FromV(From.BaseType, N);
-        OS << "TARGET_BUILTIN(__builtin_sve_reinterpret_" << To.Suffix << "_"
-           << From.Suffix << Suffix << +", \"" << ToV.builtin_str()
-           << FromV.builtin_str() << "\", \"n\", \"sme|sve\")\n";
+        std::string Name =
+            (Twine("reinterpret_") + To.Suffix + "_" + From.Suffix + Suffix)
+                .str();
+        std::string Type = ToV.builtin_str() + FromV.builtin_str();
+        Table.GetOrAddStringOffset(Name);
+        Table.GetOrAddStringOffset(Type);
+        ReinterpretBuiltins.push_back({Name, Type});
       }
     }
   }
 
-  OS << "#endif\n\n";
+  OS << "#ifdef GET_SVE_BUILTIN_ENUMERATORS\n";
+  for (const auto &Def : Defs)
+    if (Def->getClassKind() != ClassG)
+      OS << "  BI__builtin_sve_" << Def->getMangledName() << ",\n";
+  for (const auto &[Name, _] : ReinterpretBuiltins)
+    OS << "  BI__builtin_sve_" << Name << ",\n";
+  OS << "#endif // GET_SVE_BUILTIN_ENUMERATORS\n\n";
+
+  OS << "#ifdef GET_SVE_BUILTIN_STR_TABLE\n";
+  Table.EmitStringTableDef(OS, "BuiltinStrings");
+  OS << "#endif // GET_SVE_BUILTIN_STR_TABLE\n\n";
+
+  OS << "#ifdef GET_SVE_BUILTIN_INFOS\n";
+  for (const auto &Def : Defs) {
+    // Only create BUILTINs for non-overloaded intrinsics, as overloaded
+    // declarations only live in the header file.
+    if (Def->getClassKind() != ClassG) {
+      OS << "    Builtin::Info{Builtin::Info::StrOffsets{"
+         << Table.GetStringOffset(Def->getMangledName()) << " /* "
+         << Def->getMangledName() << " */, ";
+      OS << Table.GetStringOffset(Def->getBuiltinTypeStr()) << " /* "
+         << Def->getBuiltinTypeStr() << " */, ";
+      OS << Table.GetStringOffset("n") << " /* n */, ";
+      OS << Table.GetStringOffset(Def->getGuard()) << " /* " << Def->getGuard()
+         << " */}, ";
+      OS << "HeaderDesc::NO_HEADER, ALL_LANGUAGES},\n";
+    }
+  }
+  for (const auto &[Name, Type] : ReinterpretBuiltins) {
+    OS << "    Builtin::Info{Builtin::Info::StrOffsets{"
+       << Table.GetStringOffset(Name) << " /* " << Name << " */, ";
+    OS << Table.GetStringOffset(Type) << " /* " << Type << " */, ";
+    OS << Table.GetStringOffset("n") << " /* n */, ";
+    OS << Table.GetStringOffset("sme|sve") << " /* sme|sve */}, ";
+    OS << "HeaderDesc::NO_HEADER, ALL_LANGUAGES},\n";
+  }
+  OS << "#endif // GET_SVE_BUILTIN_INFOS\n\n";
 }
 
 void SVEEmitter::createCodeGenMap(raw_ostream &OS) {
@@ -1672,19 +1724,44 @@ void SVEEmitter::createSMEBuiltins(raw_ostream &OS) {
     return A->getMangledName() < B->getMangledName();
   });
 
-  OS << "#ifdef GET_SME_BUILTINS\n";
-  for (auto &Def : Defs) {
+  llvm::StringToOffsetTable Table;
+  Table.GetOrAddStringOffset("");
+  Table.GetOrAddStringOffset("n");
+
+  for (const auto &Def : Defs)
+    if (Def->getClassKind() != ClassG) {
+      Table.GetOrAddStringOffset(Def->getMangledName());
+      Table.GetOrAddStringOffset(Def->getBuiltinTypeStr());
+      Table.GetOrAddStringOffset(Def->getGuard());
+    }
+
+  OS << "#ifdef GET_SME_BUILTIN_ENUMERATORS\n";
+  for (const auto &Def : Defs)
+    if (Def->getClassKind() != ClassG)
+      OS << "  BI__builtin_sme_" << Def->getMangledName() << ",\n";
+  OS << "#endif // GET_SME_BUILTIN_ENUMERATORS\n\n";
+
+  OS << "#ifdef GET_SME_BUILTIN_STR_TABLE\n";
+  Table.EmitStringTableDef(OS, "BuiltinStrings");
+  OS << "#endif // GET_SME_BUILTIN_STR_TABLE\n\n";
+
+  OS << "#ifdef GET_SME_BUILTIN_INFOS\n";
+  for (const auto &Def : Defs) {
     // Only create BUILTINs for non-overloaded intrinsics, as overloaded
     // declarations only live in the header file.
     if (Def->getClassKind() != ClassG) {
-      OS << "TARGET_BUILTIN(__builtin_sme_" << Def->getMangledName() << ", \""
-         << Def->getBuiltinTypeStr() << "\", \"n\", \"";
-      Def->printGuard(OS);
-      OS << "\")\n";
+      OS << "    Builtin::Info{Builtin::Info::StrOffsets{"
+         << Table.GetStringOffset(Def->getMangledName()) << " /* "
+         << Def->getMangledName() << " */, ";
+      OS << Table.GetStringOffset(Def->getBuiltinTypeStr()) << " /* "
+         << Def->getBuiltinTypeStr() << " */, ";
+      OS << Table.GetStringOffset("n") << " /* n */, ";
+      OS << Table.GetStringOffset(Def->getGuard()) << " /* " << Def->getGuard()
+         << " */}, ";
+      OS << "HeaderDesc::NO_HEADER, ALL_LANGUAGES},\n";
     }
   }
-
-  OS << "#endif\n\n";
+  OS << "#endif // GET_SME_BUILTIN_INFOS\n\n";
 }
 
 void SVEEmitter::createSMECodeGenMap(raw_ostream &OS) {
diff --git a/clang/utils/analyzer/entrypoint.py b/clang/utils/analyzer/entrypoint.py
index 4deb42db0a0b1..c8dfc1a9f2ed8 100644
--- a/clang/utils/analyzer/entrypoint.py
+++ b/clang/utils/analyzer/entrypoint.py
@@ -54,7 +54,6 @@ def is_cmake_needed():
     "cmake -G Ninja -DCMAKE_BUILD_TYPE=Release "
     "-DCMAKE_INSTALL_PREFIX=/analyzer -DLLVM_TARGETS_TO_BUILD=X86 "
     '-DLLVM_ENABLE_PROJECTS="clang;openmp" -DLLVM_BUILD_RUNTIME=OFF '
-    "-DCLANG_ENABLE_ARCMT=OFF "
     "-DCLANG_ENABLE_STATIC_ANALYZER=ON"
 )
 
diff --git a/clang/utils/perf-training/CMakeLists.txt b/clang/utils/perf-training/CMakeLists.txt
index 49673790ff6e8..4aed086563ee9 100644
--- a/clang/utils/perf-training/CMakeLists.txt
+++ b/clang/utils/perf-training/CMakeLists.txt
@@ -83,8 +83,6 @@ if(APPLE AND DTRACE AND NOT LLVM_TOOL_LLVM_DRIVER_BUILD)
 endif()
 
 if(CLANG_BOLT AND NOT LLVM_BUILD_INSTRUMENTED)
-  set(CLANG_BOLT_INSTRUMENTED "clang-bolt.inst" CACHE STRING
-    "Name of BOLT-instrumented Clang binary")
   configure_lit_site_cfg(
     ${CMAKE_CURRENT_SOURCE_DIR}/bolt.lit.site.cfg.in
     ${CMAKE_CURRENT_BINARY_DIR}/bolt-fdata/lit.site.cfg
@@ -93,7 +91,7 @@ if(CLANG_BOLT AND NOT LLVM_BUILD_INSTRUMENTED)
   add_lit_testsuite(generate-bolt-fdata "Generating BOLT profile for Clang"
     ${CMAKE_CURRENT_BINARY_DIR}/bolt-fdata/
     EXCLUDE_FROM_CHECK_ALL
-    DEPENDS clang-bolt-training-deps clear-bolt-fdata clear-perf-data
+    DEPENDS clear-bolt-fdata clear-perf-data
     )
 
   add_custom_target(clear-bolt-fdata
@@ -104,26 +102,4 @@ if(CLANG_BOLT AND NOT LLVM_BUILD_INSTRUMENTED)
     COMMAND "${Python3_EXECUTABLE}" ${CMAKE_CURRENT_SOURCE_DIR}/perf-helper.py clean ${CMAKE_CURRENT_BINARY_DIR} perf.data
     COMMENT "Clearing old perf data")
 
-  string(TOUPPER "${CLANG_BOLT}" CLANG_BOLT)
-  if (CLANG_BOLT STREQUAL "LBR")
-    set(BOLT_LBR "--lbr")
-  endif()
-
-  add_custom_target(merge-fdata-deps)
-  if (CLANG_BOLT STREQUAL "INSTRUMENT")
-    add_dependencies(merge-fdata-deps generate-bolt-fdata)
-  else()
-    # Convert perf profiles into fdata
-    add_custom_target(convert-perf-fdata
-      COMMAND "${Python3_EXECUTABLE}" ${CMAKE_CURRENT_SOURCE_DIR}/perf-helper.py perf2bolt $<TARGET_FILE:llvm-bolt> ${CMAKE_CURRENT_BINARY_DIR} $<TARGET_FILE:clang> ${BOLT_LBR}
-      COMMENT "Converting perf files to BOLT fdata"
-      DEPENDS llvm-bolt generate-bolt-fdata)
-    add_dependencies(merge-fdata-deps convert-perf-fdata)
-  endif()
-
-  # Merge profiles into one using merge-fdata
-  add_custom_target(clang-bolt-profile
-    COMMAND "${Python3_EXECUTABLE}" ${CMAKE_CURRENT_SOURCE_DIR}/perf-helper.py merge-fdata $<TARGET_FILE:merge-fdata> ${CMAKE_CURRENT_BINARY_DIR}/prof.fdata ${CMAKE_CURRENT_BINARY_DIR}
-    COMMENT "Merging BOLT fdata"
-    DEPENDS merge-fdata merge-fdata-deps)
 endif()
diff --git a/clang/utils/perf-training/perf-helper.py b/clang/utils/perf-training/perf-helper.py
index d76c6ede3fe5a..55c5160a71c4f 100644
--- a/clang/utils/perf-training/perf-helper.py
+++ b/clang/utils/perf-training/perf-helper.py
@@ -16,6 +16,8 @@
 import bisect
 import shlex
 import tempfile
+import re
+import shutil
 
 test_env = {"PATH": os.environ["PATH"]}
 
@@ -558,7 +560,103 @@ def genOrderFile(args):
     return 0
 
 
+def bolt_optimize(args):
+    parser = argparse.ArgumentParser("%prog  [options] ")
+    parser.add_argument("--method", choices=["INSTRUMENT", "PERF", "LBR"])
+    parser.add_argument("--input")
+    parser.add_argument("--instrumented-output")
+    parser.add_argument("--fdata")
+    parser.add_argument("--perf-training-binary-dir")
+    parser.add_argument("--readelf")
+    parser.add_argument("--bolt")
+    parser.add_argument("--lit")
+    parser.add_argument("--merge-fdata")
+
+    opts = parser.parse_args(args)
+
+    output = subprocess.check_output(
+        [opts.readelf, "-WS", opts.input], universal_newlines=True
+    )
+
+    # This binary has already been bolt-optimized, so skip further processing.
+    if re.search("\\.bolt\\.org\\.text", output, re.MULTILINE):
+        return 0
+
+    if opts.method == "INSTRUMENT":
+        process = subprocess.run(
+            [
+                opts.bolt,
+                opts.input,
+                "-o",
+                opts.instrumented_output,
+                "-instrument",
+                "--instrumentation-file-append-pid",
+                f"--instrumentation-file={opts.fdata}",
+            ],
+            stdout=subprocess.PIPE,
+            stderr=subprocess.STDOUT,
+            text=True,
+        )
+
+        print(process.args)
+        for line in process.stdout:
+            sys.stdout.write(line)
+        process.check_returncode()
+
+    process = subprocess.run(
+        [
+            sys.executable,
+            opts.lit,
+            os.path.join(opts.perf_training_binary_dir, "bolt-fdata"),
+        ],
+        stdout=subprocess.PIPE,
+        stderr=subprocess.STDOUT,
+        text=True,
+    )
+
+    print(process.args)
+    for line in process.stdout:
+        sys.stdout.write(line)
+    process.check_returncode()
+
+    if opts.method in ["PERF", "LBR"]:
+        perf2bolt([opts.bolt, opts.perf_training_binary_dir, opts.input])
+
+    merge_fdata([opts.merge_fdata, opts.fdata, opts.perf_training_binary_dir])
+
+    shutil.copy(opts.input, f"{opts.input}-prebolt")
+
+    process = subprocess.run(
+        [
+            opts.bolt,
+            f"{opts.input}-prebolt",
+            "-o",
+            opts.input,
+            "-data",
+            opts.fdata,
+            "-reorder-blocks=ext-tsp",
+            "-reorder-functions=cdsort",
+            "-split-functions",
+            "-split-all-cold",
+            "-split-eh",
+            "-dyno-stats",
+            "-use-gnu-stack",
+            "-update-debug-sections",
+            "-nl" if opts.method == "PERF" else "",
+        ],
+        stdout=subprocess.PIPE,
+        stderr=subprocess.STDOUT,
+        text=True,
+    )
+
+    print(process.args)
+    for line in process.stdout:
+        sys.stdout.write(line)
+    process.check_returncode()
+
+
 commands = {
+    "bolt-optimize": bolt_optimize,
     "clean": clean,
     "merge": merge,
     "dtrace": dtrace,
diff --git a/clang/www/analyzer/open_projects.html b/clang/www/analyzer/open_projects.html
index a7c99c6e25bb3..4d6e40f92f82c 100644
--- a/clang/www/analyzer/open_projects.html
+++ b/clang/www/analyzer/open_projects.html
@@ -35,19 +35,6 @@ <h1>Open Projects</h1>
     Such checkers should either be improved
     up to a point where they can be enabled by default,
     or removed from the analyzer entirely.
-
-    <ul>
-      <li><code>alpha.security.ArrayBound</code> and
-      <code>alpha.security.ArrayBoundV2</code>
-      <p>Array bounds checking is a desired feature,
-      but having an acceptable rate of false positives might not be possible
-      without a proper
-      <a href="https://en.wikipedia.org/wiki/Widening_(computer_science)">loop widening</a> support.
-      Additionally, it might be more promising to perform index checking based on
-      <a href="https://en.wikipedia.org/wiki/Taint_checking">tainted</a> index values.
-      <p><i>(Difficulty: Medium)</i></p></p>
-      </li>
-    </ul>
   </li>
 
   <li>Improve C++ support
diff --git a/clang/www/analyzer/potential_checkers.html b/clang/www/analyzer/potential_checkers.html
index ad789b83e71b7..2543db251be12 100644
--- a/clang/www/analyzer/potential_checkers.html
+++ b/clang/www/analyzer/potential_checkers.html
@@ -965,7 +965,7 @@ <h3>undefined behavior</h3>
 (C++03)</span><div class="descr">
 Undefined behavior: out-of-bound <code>basic_string</code> access/modification.
 <br>Note: possibly an enhancement to <span class="name">
-alpha.security.ArrayBoundV2</span>.
+security.ArrayBound</span>.
 <p>Source: C++03 21.3.4p1; C++11 behavior is defined
 (21.4.5p2).</p></div></div></td>
 <td><div class="exampleContainer expandable">
@@ -1692,24 +1692,6 @@ <h3>different</h3>
 <td class="aligned"></td></tr>
 
 
-<tr><td><div class="namedescr expandable"><span class="name">
-different.ArrayBound</span><span class="lang">
-(C++)</span><div class="descr">
-Out-of-bound dynamic array access.
-<br>Note: possibly an enhancement to <span class="name">
-alpha.security.ArrayBoundV2</span>.</div></div></td>
-<td><div class="exampleContainer expandable">
-<div class="example"><pre>
-void test() {
-  int *p = new int[1];
-  int i = 1;
-  if(p[i]) {}; // warn
-  delete[] p;
-}
-</pre></div></div></td>
-<td class="aligned"></td></tr>
-
-
 <tr><td><div class="namedescr expandable"><span class="name">
 different.StrcpyInputSize</span><span class="lang">
 (C)</span><div class="descr">
diff --git a/clang/www/cxx_status.html b/clang/www/cxx_status.html
index 575055117b35f..0fc3b1d314698 100755
--- a/clang/www/cxx_status.html
+++ b/clang/www/cxx_status.html
@@ -1204,8 +1204,8 @@ <h2 id="cxx17">C++17 implementation status</h2>
 <span id="p0522">(10): While this feature was initially implemented in Clang 4,
 it was not enabled by default prior to clang 19, but could be enabled with
 <tt>-frelaxed-template-template-args</tt>.
-Starting from Clang 19, the flag is deprecated and will be removed in a future
-version.
+In Clang 19, the flag was deprecated.
+In Clang 21, the flag was removed altogether.
 </span>
 </p>
 </details>
diff --git a/cmake/Modules/LLVMVersion.cmake b/cmake/Modules/LLVMVersion.cmake
index 897dd963bd9ab..f14aae172f077 100644
--- a/cmake/Modules/LLVMVersion.cmake
+++ b/cmake/Modules/LLVMVersion.cmake
@@ -1,7 +1,7 @@
 # The LLVM Version number information
 
 if(NOT DEFINED LLVM_VERSION_MAJOR)
-  set(LLVM_VERSION_MAJOR 20)
+  set(LLVM_VERSION_MAJOR 21)
 endif()
 if(NOT DEFINED LLVM_VERSION_MINOR)
   set(LLVM_VERSION_MINOR 0)
diff --git a/compiler-rt/cmake/Modules/AllSupportedArchDefs.cmake b/compiler-rt/cmake/Modules/AllSupportedArchDefs.cmake
index ee5be276f3df7..2683259e93e37 100644
--- a/compiler-rt/cmake/Modules/AllSupportedArchDefs.cmake
+++ b/compiler-rt/cmake/Modules/AllSupportedArchDefs.cmake
@@ -103,7 +103,7 @@ if(APPLE)
 set(ALL_XRAY_SUPPORTED_ARCH ${X86_64} ${ARM64})
 else()
 set(ALL_XRAY_SUPPORTED_ARCH ${X86_64} ${ARM32} ${ARM64} ${MIPS32} ${MIPS64}
-               powerpc64le ${HEXAGON} ${LOONGARCH64} ${RISCV32} ${RISCV64})
+    powerpc64le ${HEXAGON} ${LOONGARCH64} ${RISCV32} ${RISCV64} ${S390X})
 endif()
 set(ALL_XRAY_DSO_SUPPORTED_ARCH ${X86_64} ${ARM64})
 set(ALL_SHADOWCALLSTACK_SUPPORTED_ARCH ${ARM64})
diff --git a/compiler-rt/lib/asan/asan_allocator.cpp b/compiler-rt/lib/asan/asan_allocator.cpp
index 9e66f77217ec6..3a55c2af65653 100644
--- a/compiler-rt/lib/asan/asan_allocator.cpp
+++ b/compiler-rt/lib/asan/asan_allocator.cpp
@@ -21,6 +21,7 @@
 #include "asan_poisoning.h"
 #include "asan_report.h"
 #include "asan_stack.h"
+#include "asan_suppressions.h"
 #include "asan_thread.h"
 #include "lsan/lsan_common.h"
 #include "sanitizer_common/sanitizer_allocator_checks.h"
@@ -732,7 +733,8 @@ struct Allocator {
     if (!AtomicallySetQuarantineFlagIfAllocated(m, ptr, stack)) return;
 
     if (m->alloc_type != alloc_type) {
-      if (atomic_load(&alloc_dealloc_mismatch, memory_order_acquire)) {
+      if (atomic_load(&alloc_dealloc_mismatch, memory_order_acquire) &&
+          !IsAllocDeallocMismatchSuppressed(stack)) {
         ReportAllocTypeMismatch((uptr)ptr, stack, (AllocType)m->alloc_type,
                                 (AllocType)alloc_type);
       }
diff --git a/compiler-rt/lib/asan/asan_suppressions.cpp b/compiler-rt/lib/asan/asan_suppressions.cpp
index 94289d14d7e78..30de775235800 100644
--- a/compiler-rt/lib/asan/asan_suppressions.cpp
+++ b/compiler-rt/lib/asan/asan_suppressions.cpp
@@ -26,9 +26,10 @@ static const char kInterceptorName[] = "interceptor_name";
 static const char kInterceptorViaFunction[] = "interceptor_via_fun";
 static const char kInterceptorViaLibrary[] = "interceptor_via_lib";
 static const char kODRViolation[] = "odr_violation";
+static const char kAllocDeallocMismatch[] = "alloc_dealloc_mismatch";
 static const char *kSuppressionTypes[] = {
     kInterceptorName, kInterceptorViaFunction, kInterceptorViaLibrary,
-    kODRViolation};
+    kODRViolation, kAllocDeallocMismatch};
 
 SANITIZER_INTERFACE_WEAK_DEF(const char *, __asan_default_suppressions, void) {
   return "";
@@ -62,6 +63,44 @@ bool IsODRViolationSuppressed(const char *global_var_name) {
   return suppression_ctx->Match(global_var_name, kODRViolation, &s);
 }
 
+bool IsAddrSuppressed(const char *suppression, Symbolizer *symbolizer,
+                      uptr addr) {
+  CHECK(suppression_ctx);
+  CHECK(suppression_ctx->HasSuppressionType(suppression));
+  CHECK(symbolizer);
+  SymbolizedStackHolder symbolized_stack(symbolizer->SymbolizePC(addr));
+  const SymbolizedStack *frames = symbolized_stack.get();
+  CHECK(frames);
+  for (const SymbolizedStack *cur = frames; cur; cur = cur->next) {
+    const char *function_name = cur->info.function;
+    if (!function_name) {
+      continue;
+    }
+    // Match suppressions.
+    Suppression *s;
+    if (suppression_ctx->Match(function_name, suppression, &s)) {
+      return true;
+    }
+  }
+  return false;
+}
+
+bool IsAllocDeallocMismatchSuppressed(const StackTrace *stack) {
+  CHECK(suppression_ctx);
+  if (!suppression_ctx->HasSuppressionType(kAllocDeallocMismatch)) {
+    return false;
+  }
+  Symbolizer *symbolizer = Symbolizer::GetOrInit();
+  for (uptr i = 0; i < stack->size && stack->trace[i]; i++) {
+    uptr addr = stack->trace[i];
+    // Match "alloc_dealloc_mismatch" suppressions.
+    if (IsAddrSuppressed(kAllocDeallocMismatch, symbolizer, addr)) {
+      return true;
+    }
+  }
+  return false;
+}
+
 bool IsStackTraceSuppressed(const StackTrace *stack) {
   if (!HaveStackTraceBasedSuppressions())
     return false;
@@ -80,19 +119,9 @@ bool IsStackTraceSuppressed(const StackTrace *stack) {
     }
 
     if (suppression_ctx->HasSuppressionType(kInterceptorViaFunction)) {
-      SymbolizedStackHolder symbolized_stack(symbolizer->SymbolizePC(addr));
-      const SymbolizedStack *frames = symbolized_stack.get();
-      CHECK(frames);
-      for (const SymbolizedStack *cur = frames; cur; cur = cur->next) {
-        const char *function_name = cur->info.function;
-        if (!function_name) {
-          continue;
-        }
-        // Match "interceptor_via_fun" suppressions.
-        if (suppression_ctx->Match(function_name, kInterceptorViaFunction,
-                                   &s)) {
-          return true;
-        }
+      // Match "interceptor_via_func" suppressions.
+      if (IsAddrSuppressed(kInterceptorViaFunction, symbolizer, addr)) {
+        return true;
       }
     }
   }
diff --git a/compiler-rt/lib/asan/asan_suppressions.h b/compiler-rt/lib/asan/asan_suppressions.h
index 121d4ddf18756..4613a362f28e4 100644
--- a/compiler-rt/lib/asan/asan_suppressions.h
+++ b/compiler-rt/lib/asan/asan_suppressions.h
@@ -23,6 +23,7 @@ bool IsInterceptorSuppressed(const char *interceptor_name);
 bool HaveStackTraceBasedSuppressions();
 bool IsStackTraceSuppressed(const StackTrace *stack);
 bool IsODRViolationSuppressed(const char *global_var_name);
+bool IsAllocDeallocMismatchSuppressed(const StackTrace *stack);
 
 } // namespace __asan
 
diff --git a/compiler-rt/lib/builtins/cpu_model/riscv.c b/compiler-rt/lib/builtins/cpu_model/riscv.c
index 74534896057ef..6879c2ad48264 100644
--- a/compiler-rt/lib/builtins/cpu_model/riscv.c
+++ b/compiler-rt/lib/builtins/cpu_model/riscv.c
@@ -14,12 +14,6 @@ struct {
   unsigned long long features[RISCV_FEATURE_BITS_LENGTH];
 } __riscv_feature_bits __attribute__((visibility("hidden"), nocommon));
 
-#define RISCV_VENDOR_FEATURE_BITS_LENGTH 1
-struct {
-  unsigned length;
-  unsigned long long features[RISCV_VENDOR_FEATURE_BITS_LENGTH];
-} __riscv_vendor_feature_bits __attribute__((visibility("hidden"), nocommon));
-
 struct {
   unsigned mvendorid;
   unsigned long long marchid;
@@ -338,11 +332,11 @@ static int FeaturesBitCached = 0;
 void __init_riscv_feature_bits(void *);
 static void __init_riscv_feature_bits_ctor(void) CONSTRUCTOR_ATTRIBUTE;
 
-// A constructor function that sets __riscv_feature_bits, and
-// __riscv_vendor_feature_bits to the right values.  This needs to run
-// only once.  This constructor is given the highest priority and it should
-// run before constructors without the priority set.  However, it still runs
-// after ifunc initializers and needs to be called explicitly there.
+// A constructor function that sets __riscv_feature_bits
+// to the right values.  This needs to run only once.  This constructor is given
+// the highest priority and it should run before constructors without the
+// priority set.  However, it still runs after ifunc initializers and needs to
+// be called explicitly there.
 
 static void CONSTRUCTOR_ATTRIBUTE __init_riscv_feature_bits_ctor(void) {
   __init_riscv_feature_bits(0);
@@ -357,7 +351,6 @@ void __init_riscv_feature_bits(void *PlatformArgs) {
     return;
 
   __riscv_feature_bits.length = RISCV_FEATURE_BITS_LENGTH;
-  __riscv_vendor_feature_bits.length = RISCV_VENDOR_FEATURE_BITS_LENGTH;
 
 #if defined(__linux__)
   struct riscv_hwprobe Hwprobes[] = {
diff --git a/compiler-rt/lib/interception/interception_win.cpp b/compiler-rt/lib/interception/interception_win.cpp
index 64004c171d534..002b37468a200 100644
--- a/compiler-rt/lib/interception/interception_win.cpp
+++ b/compiler-rt/lib/interception/interception_win.cpp
@@ -661,6 +661,10 @@ static size_t GetInstructionSize(uptr address, size_t* rel_offset = nullptr) {
     case 0xC1F6:  // F6 C1 XX : test cl, XX
       return 3;
 
+    case 0x89FF:  // FF 89 XX XX XX XX : dec dword ptr [ecx + XX XX XX XX]
+    case 0xEC81:  // 81 EC XX XX XX XX : sub esp, XX XX XX XX
+      return 6;
+
     // Cannot overwrite control-instruction. Return 0 to indicate failure.
     case 0x25FF:  // FF 25 XX YY ZZ WW : jmp dword ptr ds:[WWZZYYXX]
       return 0;
@@ -739,6 +743,7 @@ static size_t GetInstructionSize(uptr address, size_t* rel_offset = nullptr) {
     case 0x058B:  // 8B 05 XX XX XX XX : mov eax, dword ptr [XX XX XX XX]
       if (rel_offset)
         *rel_offset = 2;
+    case 0xB841:  // 41 B8 XX XX XX XX : mov r8d, XX XX XX XX
       return 6;
 
     case 0x7E81:  // 81 7E YY XX XX XX XX  cmp DWORD PTR [rsi+YY], XX XX XX XX
@@ -852,6 +857,7 @@ static size_t GetInstructionSize(uptr address, size_t* rel_offset = nullptr) {
       return 6;
 
     case 0xec8148:    // 48 81 EC XX XX XX XX : sub rsp, XXXXXXXX
+    case 0xc0c748:    // 48 C7 C0 XX XX XX XX : mov rax, XX XX XX XX
       return 7;
 
     // clang-format off
@@ -913,6 +919,8 @@ static size_t GetInstructionSize(uptr address, size_t* rel_offset = nullptr) {
       return 5;
     case 0x24648348:  // 48 83 64 24 XX YY : and QWORD PTR [rsp + XX], YY
       return 6;
+    case 0x24A48D48:  // 48 8D A4 24 XX XX XX XX : lea rsp, [rsp + XX XX XX XX]
+      return 8;
   }
 
   switch (0xFFFFFFFFFFULL & *(u64 *)(address)) {
diff --git a/compiler-rt/lib/interception/tests/interception_win_test.cpp b/compiler-rt/lib/interception/tests/interception_win_test.cpp
index c5dcf26070f0d..2a7549d230ae2 100644
--- a/compiler-rt/lib/interception/tests/interception_win_test.cpp
+++ b/compiler-rt/lib/interception/tests/interception_win_test.cpp
@@ -870,6 +870,8 @@ const struct InstructionSizeData {
     { 5, {0xb8, 0x71, 0x72, 0x73, 0x74}, 0, "b8 XX XX XX XX : mov eax, XX XX XX XX"},
     { 5, {0xB9, 0x71, 0x72, 0x73, 0x74}, 0, "b9 XX XX XX XX : mov ecx, XX XX XX XX"},
     { 5, {0xBA, 0x71, 0x72, 0x73, 0x74}, 0, "ba XX XX XX XX : mov edx, XX XX XX XX"},
+    { 6, {0x81, 0xEC, 0x72, 0x73, 0x74, 0x75}, 0, "81 EC XX XX XX XX : sub esp, XX XX XX XX"},
+    { 6, {0xFF, 0x89, 0x72, 0x73, 0x74, 0x75}, 0, "FF 89 XX XX XX XX : dec dword ptr [ecx + XX XX XX XX]"},
     { 7, {0x8D, 0xA4, 0x24, 0x73, 0x74, 0x75, 0x76}, 0, "8D A4 24 XX XX XX XX : lea esp, [esp + XX XX XX XX]"},
 #if SANITIZER_WINDOWS_x64
     // sorted list
@@ -1000,6 +1002,7 @@ const struct InstructionSizeData {
     { 5, {0x66, 0x48, 0x0F, 0x7E, 0xC0}, 0, "66 48 0F 7E C0 : movq rax, xmm0"},
     { 5, {0x83, 0x44, 0x72, 0x73, 0x74}, 0, "83 44 72 XX YY : add DWORD PTR [rdx+rsi*2+XX],YY"},
     { 5, {0x83, 0x64, 0x24, 0x73, 0x74}, 0, "83 64 24 XX YY : and DWORD PTR [rsp+XX], YY"},
+    { 6, {0x41, 0xB8, 0x72, 0x73, 0x74, 0x75}, 0, "41 B8 XX XX XX XX : mov r8d, XX XX XX XX"},
     { 6, {0x48, 0x83, 0x64, 0x24, 0x74, 0x75}, 0, "48 83 64 24 XX YY : and QWORD PTR [rsp + XX], YY"},
     { 6, {0x66, 0x81, 0x78, 0x73, 0x74, 0x75}, 0, "66 81 78 XX YY YY : cmp WORD PTR [rax+XX], YY YY"},
     { 6, {0x66, 0x81, 0x79, 0x73, 0x74, 0x75}, 0, "66 81 79 XX YY YY : cmp WORD PTR [rcx+XX], YY YY"},
@@ -1019,6 +1022,7 @@ const struct InstructionSizeData {
     { 7, {0x48, 0x89, 0x15, 0x73, 0x74, 0x75, 0x76}, 3, "48 89 15 XX XX XX XX : mov QWORD PTR [rip + XXXXXXXX], rdx"},
     { 7, {0x48, 0x8b, 0x05, 0x73, 0x74, 0x75, 0x76}, 3, "48 8b 05 XX XX XX XX : mov rax, QWORD PTR [rip + XXXXXXXX]"},
     { 7, {0x48, 0x8d, 0x05, 0x73, 0x74, 0x75, 0x76}, 3, "48 8d 05 XX XX XX XX : lea rax, QWORD PTR [rip + XXXXXXXX]"},
+    { 7, {0x48, 0xc7, 0xc0, 0x73, 0x74, 0x75, 0x76}, 0, "48 C7 C0 XX XX XX XX : mov rax, XX XX XX XX"},
     { 7, {0x48, 0xff, 0x25, 0x73, 0x74, 0x75, 0x76}, 3, "48 ff 25 XX XX XX XX : rex.W jmp QWORD PTR [rip + XXXXXXXX]"},
     { 7, {0x4C, 0x8D, 0x15, 0x73, 0x74, 0x75, 0x76}, 3, "4c 8d 15 XX XX XX XX : lea r10, [rip + XX]"},
     { 7, {0x81, 0x78, 0x72, 0x73, 0x74, 0x75, 0x76}, 0, "81 78 YY XX XX XX XX : cmp DWORD PTR [rax+YY], XX XX XX XX"},
@@ -1034,6 +1038,7 @@ const struct InstructionSizeData {
     { 8, {0x41, 0x81, 0x7d, 0x73, 0x74, 0x75, 0x76, 0x77}, 0, "41 81 7d XX YY YY YY YY : cmp DWORD PTR [r13+YY], XX XX XX XX"},
     { 8, {0x41, 0x81, 0x7e, 0x73, 0x74, 0x75, 0x76, 0x77}, 0, "41 81 7e XX YY YY YY YY : cmp DWORD PTR [r14+YY], XX XX XX XX"},
     { 8, {0x41, 0x81, 0x7f, 0x73, 0x74, 0x75, 0x76, 0x77}, 0, "41 81 7f YY XX XX XX XX : cmp DWORD PTR [r15+YY], XX XX XX XX"},
+    { 8, {0x48, 0x8D, 0xA4, 0x24, 0x74, 0x75, 0x76, 0x77}, 0, "48 8D A4 24 XX XX XX XX : lea rsp, [rsp + XX XX XX XX]"},
     { 8, {0x81, 0x7c, 0x24, 0x73, 0x74, 0x75, 0x76, 0x77}, 0, "81 7c 24 YY XX XX XX XX : cmp DWORD PTR [rsp+YY], XX XX XX XX"},
     { 8, {0xc7, 0x44, 0x24, 0x73, 0x74, 0x75, 0x76, 0x77}, 0, "C7 44 24 XX YY YY YY YY : mov dword ptr [rsp + XX], YYYYYYYY"},
     { 9, {0x41, 0x81, 0x7c, 0x73, 0x74, 0x75, 0x76, 0x77, 0x78}, 0, "41 81 7c ZZ YY XX XX XX XX : cmp DWORD PTR [reg+reg*n+YY], XX XX XX XX"},
diff --git a/compiler-rt/lib/memprof/memprof_allocator.cpp b/compiler-rt/lib/memprof/memprof_allocator.cpp
index c3448c22532bb..60f5c853f9d76 100644
--- a/compiler-rt/lib/memprof/memprof_allocator.cpp
+++ b/compiler-rt/lib/memprof/memprof_allocator.cpp
@@ -291,10 +291,9 @@ struct Allocator {
 
   atomic_uint8_t destructing;
   atomic_uint8_t constructed;
-  bool print_text;
 
   // ------------------- Initialization ------------------------
-  explicit Allocator(LinkerInitialized) : print_text(flags()->print_text) {
+  explicit Allocator(LinkerInitialized) {
     atomic_store_relaxed(&destructing, 0);
     atomic_store_relaxed(&constructed, 1);
   }
@@ -350,13 +349,13 @@ struct Allocator {
   }
 
   void FinishAndWrite() {
-    if (print_text && common_flags()->print_module_map)
+    if (flags()->print_text && common_flags()->print_module_map)
       DumpProcessMap();
 
     allocator.ForceLock();
 
     InsertLiveBlocks();
-    if (print_text) {
+    if (flags()->print_text) {
       if (!flags()->print_terse)
         Printf("Recorded MIBs (incl. live on exit):\n");
       MIBMap.ForEach(PrintCallback,
diff --git a/compiler-rt/lib/orc/macho_platform.cpp b/compiler-rt/lib/orc/macho_platform.cpp
index 8ca68587aeb36..4b603fd95e316 100644
--- a/compiler-rt/lib/orc/macho_platform.cpp
+++ b/compiler-rt/lib/orc/macho_platform.cpp
@@ -557,6 +557,12 @@ Error MachOPlatformRuntimeState::registerObjectPlatformSections(
     return make_error<StringError>(ErrStream.str());
   }
 
+  ORC_RT_DEBUG({
+    printdbg("  UnwindInfo: %s, UseCallbackStyleUnwindInfo: %s\n",
+             UnwindInfo ? "true" : "false",
+             UseCallbackStyleUnwindInfo ? "true" : "false");
+  });
+
   if (UnwindInfo && UseCallbackStyleUnwindInfo) {
     ORC_RT_DEBUG({
       printdbg("  Registering new-style unwind info for:\n"
diff --git a/compiler-rt/lib/profile/InstrProfilingFile.c b/compiler-rt/lib/profile/InstrProfilingFile.c
index bad4cc71801ec..343063fd6b754 100644
--- a/compiler-rt/lib/profile/InstrProfilingFile.c
+++ b/compiler-rt/lib/profile/InstrProfilingFile.c
@@ -77,6 +77,7 @@ typedef struct lprofFilename {
   char Hostname[COMPILER_RT_MAX_HOSTLEN];
   unsigned NumPids;
   unsigned NumHosts;
+  unsigned NumBinaryIds;
   /* When in-process merging is enabled, this parameter specifies
    * the total number of profile data files shared by all the processes
    * spawned from the same binary. By default the value is 1. If merging
@@ -88,8 +89,8 @@ typedef struct lprofFilename {
   ProfileNameSpecifier PNS;
 } lprofFilename;
 
-static lprofFilename lprofCurFilename = {0,   0, 0, {0}, NULL,
-                                         {0}, 0, 0, 0,   PNS_unknown};
+static lprofFilename lprofCurFilename = {0, 0, 0, {0}, NULL,       {0},
+                                         0, 0, 0, 0,   PNS_unknown};
 
 static int ProfileMergeRequested = 0;
 static int getProfileFileSizeForMerging(FILE *ProfileFile,
@@ -790,7 +791,7 @@ static int checkBounds(int Idx, int Strlen) {
  * lprofcurFilename structure. */
 static int parseFilenamePattern(const char *FilenamePat,
                                 unsigned CopyFilenamePat) {
-  int NumPids = 0, NumHosts = 0, I;
+  int NumPids = 0, NumHosts = 0, NumBinaryIds = 0, I;
   char *PidChars = &lprofCurFilename.PidChars[0];
   char *Hostname = &lprofCurFilename.Hostname[0];
   int MergingEnabled = 0;
@@ -855,6 +856,16 @@ static int parseFilenamePattern(const char *FilenamePat,
                     FilenamePat);
           return -1;
         }
+      } else if (FilenamePat[I] == 'b') {
+        if (!NumBinaryIds++) {
+          /* Check if binary ID does not exist or if its size is 0. */
+          if (__llvm_write_binary_ids(NULL) <= 0) {
+            PROF_WARN("Unable to get binary ID for filename pattern %s. Using "
+                      "the default name.",
+                      FilenamePat);
+            return -1;
+          }
+        }
       } else if (FilenamePat[I] == 'c') {
         if (__llvm_profile_is_continuous_mode_enabled()) {
           PROF_WARN("%%c specifier can only be specified once in %s.\n",
@@ -887,6 +898,7 @@ static int parseFilenamePattern(const char *FilenamePat,
 
   lprofCurFilename.NumPids = NumPids;
   lprofCurFilename.NumHosts = NumHosts;
+  lprofCurFilename.NumBinaryIds = NumBinaryIds;
   return 0;
 }
 
@@ -934,24 +946,53 @@ static void parseAndSetFilename(const char *FilenamePat,
  * filename with PID and hostname substitutions. */
 /* The length to hold uint64_t followed by 3 digits pool id including '_' */
 #define SIGLEN 24
+/* The length to hold 160-bit hash in hexadecimal form */
+#define BINARY_ID_LEN 40
 static int getCurFilenameLength(void) {
   int Len;
   if (!lprofCurFilename.FilenamePat || !lprofCurFilename.FilenamePat[0])
     return 0;
 
   if (!(lprofCurFilename.NumPids || lprofCurFilename.NumHosts ||
-        lprofCurFilename.TmpDir || lprofCurFilename.MergePoolSize))
+        lprofCurFilename.NumBinaryIds || lprofCurFilename.TmpDir ||
+        lprofCurFilename.MergePoolSize))
     return strlen(lprofCurFilename.FilenamePat);
 
   Len = strlen(lprofCurFilename.FilenamePat) +
         lprofCurFilename.NumPids * (strlen(lprofCurFilename.PidChars) - 2) +
         lprofCurFilename.NumHosts * (strlen(lprofCurFilename.Hostname) - 2) +
+        lprofCurFilename.NumBinaryIds * BINARY_ID_LEN +
         (lprofCurFilename.TmpDir ? (strlen(lprofCurFilename.TmpDir) - 1) : 0);
   if (lprofCurFilename.MergePoolSize)
     Len += SIGLEN;
   return Len;
 }
 
+typedef struct lprofBinaryIdsBuffer {
+  char String[BINARY_ID_LEN + 1];
+  int Length;
+} lprofBinaryIdsBuffer;
+
+/* Reads binary ID length and then its data, writes it into lprofBinaryIdsBuffer
+ * in hexadecimal form. */
+static uint32_t binaryIdsStringWriter(ProfDataWriter *This,
+                                      ProfDataIOVec *IOVecs,
+                                      uint32_t NumIOVecs) {
+  if (NumIOVecs < 2 || IOVecs[0].ElmSize != sizeof(uint64_t))
+    return -1;
+  uint64_t BinaryIdLen = *(const uint64_t *)IOVecs[0].Data;
+  if (IOVecs[1].ElmSize != sizeof(uint8_t) || IOVecs[1].NumElm != BinaryIdLen)
+    return -1;
+  const uint8_t *BinaryIdData = (const uint8_t *)IOVecs[1].Data;
+  lprofBinaryIdsBuffer *Data = (lprofBinaryIdsBuffer *)This->WriterCtx;
+  for (uint64_t I = 0; I < BinaryIdLen; I++) {
+    Data->Length +=
+        snprintf(Data->String + Data->Length, BINARY_ID_LEN + 1 - Data->Length,
+                 "%02hhx", BinaryIdData[I]);
+  }
+  return 0;
+}
+
 /* Return the pointer to the current profile file name (after substituting
  * PIDs and Hostnames in filename pattern. \p FilenameBuf is the buffer
  * to store the resulting filename. If no substitution is needed, the
@@ -965,7 +1006,8 @@ static const char *getCurFilename(char *FilenameBuf, int ForceUseBuf) {
     return 0;
 
   if (!(lprofCurFilename.NumPids || lprofCurFilename.NumHosts ||
-        lprofCurFilename.TmpDir || lprofCurFilename.MergePoolSize ||
+        lprofCurFilename.NumBinaryIds || lprofCurFilename.TmpDir ||
+        lprofCurFilename.MergePoolSize ||
         __llvm_profile_is_continuous_mode_enabled())) {
     if (!ForceUseBuf)
       return lprofCurFilename.FilenamePat;
@@ -992,6 +1034,12 @@ static const char *getCurFilename(char *FilenameBuf, int ForceUseBuf) {
         memcpy(FilenameBuf + J, lprofCurFilename.TmpDir, TmpDirLength);
         FilenameBuf[J + TmpDirLength] = DIR_SEPARATOR;
         J += TmpDirLength + 1;
+      } else if (FilenamePat[I] == 'b') {
+        lprofBinaryIdsBuffer Data = {{0}, 0};
+        ProfDataWriter Writer = {binaryIdsStringWriter, &Data};
+        __llvm_write_binary_ids(&Writer);
+        memcpy(FilenameBuf + J, Data.String, Data.Length);
+        J += Data.Length;
       } else {
         if (!getMergePoolSize(FilenamePat, &I))
           continue;
diff --git a/compiler-rt/lib/rtsan/rtsan_interceptors_posix.cpp b/compiler-rt/lib/rtsan/rtsan_interceptors_posix.cpp
index 1b499f2194f21..410da0748b433 100644
--- a/compiler-rt/lib/rtsan/rtsan_interceptors_posix.cpp
+++ b/compiler-rt/lib/rtsan/rtsan_interceptors_posix.cpp
@@ -47,6 +47,7 @@ void OSSpinLockLock(volatile OSSpinLock *__lock);
 #include <stdarg.h>
 #include <stdio.h>
 #if SANITIZER_LINUX
+#include <linux/mman.h>
 #include <sys/inotify.h>
 #endif
 #include <sys/select.h>
@@ -243,6 +244,39 @@ INTERCEPTOR(int, close, int filedes) {
   return REAL(close)(filedes);
 }
 
+INTERCEPTOR(int, chdir, const char *path) {
+  __rtsan_notify_intercepted_call("chdir");
+  return REAL(chdir)(path);
+}
+
+INTERCEPTOR(int, fchdir, int fd) {
+  __rtsan_notify_intercepted_call("fchdir");
+  return REAL(fchdir)(fd);
+}
+
+#if SANITIZER_INTERCEPT_READLINK
+INTERCEPTOR(ssize_t, readlink, const char *pathname, char *buf, size_t size) {
+  __rtsan_notify_intercepted_call("readlink");
+  return REAL(readlink)(pathname, buf, size);
+}
+#define RTSAN_MAYBE_INTERCEPT_READLINK INTERCEPT_FUNCTION(readlink)
+#else
+#define RTSAN_MAYBE_INTERCEPT_READLINK
+#endif
+
+#if SANITIZER_INTERCEPT_READLINKAT
+INTERCEPTOR(ssize_t, readlinkat, int dirfd, const char *pathname, char *buf,
+            size_t size) {
+  __rtsan_notify_intercepted_call("readlinkat");
+  return REAL(readlinkat)(dirfd, pathname, buf, size);
+}
+#define RTSAN_MAYBE_INTERCEPT_READLINKAT INTERCEPT_FUNCTION(readlinkat)
+#else
+#define RTSAN_MAYBE_INTERCEPT_READLINKAT
+#endif
+
+// Streams
+
 INTERCEPTOR(FILE *, fopen, const char *path, const char *mode) {
   __rtsan_notify_intercepted_call("fopen");
   return REAL(fopen)(path, mode);
@@ -253,8 +287,6 @@ INTERCEPTOR(FILE *, freopen, const char *path, const char *mode, FILE *stream) {
   return REAL(freopen)(path, mode, stream);
 }
 
-// Streams
-
 #if SANITIZER_INTERCEPT_FOPEN64
 INTERCEPTOR(FILE *, fopen64, const char *path, const char *mode) {
   __rtsan_notify_intercepted_call("fopen64");
@@ -740,6 +772,26 @@ INTERCEPTOR(int, sched_yield, void) {
   return REAL(sched_yield)();
 }
 
+#if SANITIZER_LINUX
+INTERCEPTOR(int, sched_getaffinity, pid_t pid, size_t len, cpu_set_t *set) {
+  __rtsan_notify_intercepted_call("sched_getaffinity");
+  return REAL(sched_getaffinity)(pid, len, set);
+}
+
+INTERCEPTOR(int, sched_setaffinity, pid_t pid, size_t len,
+            const cpu_set_t *set) {
+  __rtsan_notify_intercepted_call("sched_setaffinity");
+  return REAL(sched_setaffinity)(pid, len, set);
+}
+#define RTSAN_MAYBE_INTERCEPT_SCHED_GETAFFINITY                                \
+  INTERCEPT_FUNCTION(sched_getaffinity)
+#define RTSAN_MAYBE_INTERCEPT_SCHED_SETAFFINITY                                \
+  INTERCEPT_FUNCTION(sched_setaffinity)
+#else
+#define RTSAN_MAYBE_INTERCEPT_SCHED_GETAFFINITY
+#define RTSAN_MAYBE_INTERCEPT_SCHED_SETAFFINITY
+#endif
+
 // Memory
 
 INTERCEPTOR(void *, calloc, SIZE_T num, SIZE_T size) {
@@ -850,6 +902,32 @@ INTERCEPTOR(void *, mmap64, void *addr, size_t length, int prot, int flags,
 #define RTSAN_MAYBE_INTERCEPT_MMAP64
 #endif // SANITIZER_INTERCEPT_MMAP64
 
+#if SANITIZER_LINUX
+// Note that even if rtsan is ported to netbsd, it has a slighty different
+// and non-variadic signature
+INTERCEPTOR(void *, mremap, void *oaddr, size_t olength, size_t nlength,
+            int flags, ...) {
+  __rtsan_notify_intercepted_call("mremap");
+
+  // the last optional argument is only used in this case
+  // as the new page region will be assigned to. Is ignored otherwise.
+  if (flags & MREMAP_FIXED) {
+    va_list args;
+
+    va_start(args, flags);
+    void *naddr = va_arg(args, void *);
+    va_end(args);
+
+    return REAL(mremap)(oaddr, olength, nlength, flags, naddr);
+  }
+
+  return REAL(mremap)(oaddr, olength, nlength, flags);
+}
+#define RTSAN_MAYBE_INTERCEPT_MREMAP INTERCEPT_FUNCTION(mremap)
+#else
+#define RTSAN_MAYBE_INTERCEPT_MREMAP
+#endif
+
 INTERCEPTOR(int, munmap, void *addr, size_t length) {
   __rtsan_notify_intercepted_call("munmap");
   return REAL(munmap)(addr, length);
@@ -1068,6 +1146,11 @@ INTERCEPTOR(int, setsockopt, int socket, int level, int option,
 #define RTSAN_MAYBE_INTERCEPT_SETSOCKOPT
 #endif
 
+INTERCEPTOR(int, socketpair, int domain, int type, int protocol, int pair[2]) {
+  __rtsan_notify_intercepted_call("socketpair");
+  return REAL(socketpair)(domain, type, protocol, pair);
+}
+
 // I/O Multiplexing
 
 INTERCEPTOR(int, poll, struct pollfd *fds, nfds_t nfds, int timeout) {
@@ -1321,6 +1404,7 @@ void __rtsan::InitializeInterceptors() {
   INTERCEPT_FUNCTION(posix_memalign);
   INTERCEPT_FUNCTION(mmap);
   RTSAN_MAYBE_INTERCEPT_MMAP64;
+  RTSAN_MAYBE_INTERCEPT_MREMAP;
   INTERCEPT_FUNCTION(munmap);
   RTSAN_MAYBE_INTERCEPT_MADVISE;
   RTSAN_MAYBE_INTERCEPT_POSIX_MADVISE;
@@ -1337,6 +1421,10 @@ void __rtsan::InitializeInterceptors() {
   INTERCEPT_FUNCTION(openat);
   RTSAN_MAYBE_INTERCEPT_OPENAT64;
   INTERCEPT_FUNCTION(close);
+  INTERCEPT_FUNCTION(chdir);
+  INTERCEPT_FUNCTION(fchdir);
+  RTSAN_MAYBE_INTERCEPT_READLINK;
+  RTSAN_MAYBE_INTERCEPT_READLINKAT;
   INTERCEPT_FUNCTION(fopen);
   RTSAN_MAYBE_INTERCEPT_FOPEN64;
   RTSAN_MAYBE_INTERCEPT_FREOPEN64;
@@ -1415,6 +1503,8 @@ void __rtsan::InitializeInterceptors() {
   INTERCEPT_FUNCTION(usleep);
   INTERCEPT_FUNCTION(nanosleep);
   INTERCEPT_FUNCTION(sched_yield);
+  RTSAN_MAYBE_INTERCEPT_SCHED_GETAFFINITY;
+  RTSAN_MAYBE_INTERCEPT_SCHED_SETAFFINITY;
 
   INTERCEPT_FUNCTION(accept);
   INTERCEPT_FUNCTION(bind);
@@ -1437,6 +1527,7 @@ void __rtsan::InitializeInterceptors() {
   RTSAN_MAYBE_INTERCEPT_GETPEERNAME;
   RTSAN_MAYBE_INTERCEPT_GETSOCKOPT;
   RTSAN_MAYBE_INTERCEPT_SETSOCKOPT;
+  INTERCEPT_FUNCTION(socketpair);
 
   RTSAN_MAYBE_INTERCEPT_SELECT;
   INTERCEPT_FUNCTION(pselect);
diff --git a/compiler-rt/lib/rtsan/tests/rtsan_test_interceptors_posix.cpp b/compiler-rt/lib/rtsan/tests/rtsan_test_interceptors_posix.cpp
index a4f2b92b7c494..98d27caae94b8 100644
--- a/compiler-rt/lib/rtsan/tests/rtsan_test_interceptors_posix.cpp
+++ b/compiler-rt/lib/rtsan/tests/rtsan_test_interceptors_posix.cpp
@@ -197,6 +197,16 @@ TEST(TestRtsanInterceptors, MmapDiesWhenRealtime) {
   ExpectNonRealtimeSurvival(Func);
 }
 
+#if SANITIZER_LINUX
+TEST(TestRtsanInterceptors, MremapDiesWhenRealtime) {
+  void *addr = mmap(nullptr, 8, PROT_READ | PROT_WRITE,
+                    MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
+  auto Func = [addr]() { void *_ = mremap(addr, 8, 16, 0); };
+  ExpectRealtimeDeath(Func, "mremap");
+  ExpectNonRealtimeSurvival(Func);
+}
+#endif
+
 TEST(TestRtsanInterceptors, MunmapDiesWhenRealtime) {
   void *ptr = mmap(nullptr, 8, PROT_READ | PROT_WRITE,
                    MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
@@ -323,6 +333,22 @@ TEST(TestRtsanInterceptors, SchedYieldDiesWhenRealtime) {
   ExpectNonRealtimeSurvival(Func);
 }
 
+#if SANITIZER_LINUX
+TEST(TestRtsanInterceptors, SchedGetaffinityDiesWhenRealtime) {
+  cpu_set_t set{};
+  auto Func = [&set]() { sched_getaffinity(0, sizeof(set), &set); };
+  ExpectRealtimeDeath(Func, "sched_getaffinity");
+  ExpectNonRealtimeSurvival(Func);
+}
+
+TEST(TestRtsanInterceptors, SchedSetaffinityDiesWhenRealtime) {
+  cpu_set_t set{};
+  auto Func = [&set]() { sched_setaffinity(0, sizeof(set), &set); };
+  ExpectRealtimeDeath(Func, "sched_setaffinity");
+  ExpectNonRealtimeSurvival(Func);
+}
+#endif
+
 /*
     Filesystem
 */
@@ -419,6 +445,36 @@ TEST(TestRtsanInterceptors, CloseDiesWhenRealtime) {
   ExpectNonRealtimeSurvival(Func);
 }
 
+TEST(TestRtsanInterceptors, ChdirDiesWhenRealtime) {
+  auto Func = []() { chdir("."); };
+  ExpectRealtimeDeath(Func, "chdir");
+  ExpectNonRealtimeSurvival(Func);
+}
+
+TEST(TestRtsanInterceptors, FchdirDiesWhenRealtime) {
+  auto Func = []() { fchdir(0); };
+  ExpectRealtimeDeath(Func, "fchdir");
+  ExpectNonRealtimeSurvival(Func);
+}
+
+#if SANITIZER_INTERCEPT_READLINK
+TEST(TestRtsanInterceptors, ReadlinkDiesWhenRealtime) {
+  char buf[1024];
+  auto Func = [&buf]() { readlink("/proc/self", buf, sizeof(buf)); };
+  ExpectRealtimeDeath(Func, "readlink");
+  ExpectNonRealtimeSurvival(Func);
+}
+#endif
+
+#if SANITIZER_INTERCEPT_READLINKAT
+TEST(TestRtsanInterceptors, ReadlinkatDiesWhenRealtime) {
+  char buf[1024];
+  auto Func = [&buf]() { readlinkat(0, "/proc/self", buf, sizeof(buf)); };
+  ExpectRealtimeDeath(Func, "readlinkat");
+  ExpectNonRealtimeSurvival(Func);
+}
+#endif
+
 TEST_F(RtsanFileTest, FopenDiesWhenRealtime) {
   auto Func = [this]() {
     FILE *f = fopen(GetTemporaryFilePath(), "w");
@@ -1335,6 +1391,13 @@ TEST(TestRtsanInterceptors, SetsockoptOnASocketDiesWhenRealtime) {
 }
 #endif
 
+TEST(TestRtsanInterceptors, SocketpairDiesWhenRealtime) {
+  int pair[2]{};
+  auto Func = [&pair]() { socketpair(0, 0, 0, pair); };
+  ExpectRealtimeDeath(Func, "socketpair");
+  ExpectNonRealtimeSurvival(Func);
+}
+
 /*
     I/O Multiplexing
 */
diff --git a/compiler-rt/lib/safestack/safestack_platform.h b/compiler-rt/lib/safestack/safestack_platform.h
index 41c7c25fdaf4d..3f6a69026548e 100644
--- a/compiler-rt/lib/safestack/safestack_platform.h
+++ b/compiler-rt/lib/safestack/safestack_platform.h
@@ -54,7 +54,8 @@ extern "C" void *__mmap(void *, size_t, int, int, int, int, off_t);
 // but it still needs to use 64-bit syscalls.
 #if SANITIZER_LINUX &&                                \
     (defined(__x86_64__) || defined(__powerpc64__) || \
-     SANITIZER_WORDSIZE == 64 || (defined(__mips__) && _MIPS_SIM == _ABIN32))
+     SANITIZER_WORDSIZE == 64 ||                      \
+     (defined(__mips__) && defined(_ABIN32) && _MIPS_SIM == _ABIN32))
 #  define SANITIZER_LINUX_USES_64BIT_SYSCALLS 1
 #else
 #  define SANITIZER_LINUX_USES_64BIT_SYSCALLS 0
diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_linux.cpp b/compiler-rt/lib/sanitizer_common/sanitizer_linux.cpp
index 997b95f343d41..1fa75214f4205 100644
--- a/compiler-rt/lib/sanitizer_common/sanitizer_linux.cpp
+++ b/compiler-rt/lib/sanitizer_common/sanitizer_linux.cpp
@@ -82,6 +82,12 @@
 #    include <sys/personality.h>
 #  endif
 
+#  if SANITIZER_ANDROID && __ANDROID_API__ < 35
+// The weak `strerrorname_np` (introduced in API level 35) definition,
+// allows for checking the API level at runtime.
+extern "C" SANITIZER_WEAK_ATTRIBUTE const char *strerrorname_np(int);
+#  endif
+
 #  if SANITIZER_LINUX && defined(__loongarch__)
 #    include <sys/sysmacros.h>
 #  endif
@@ -134,9 +140,10 @@ const int FUTEX_WAKE_PRIVATE = FUTEX_WAKE | FUTEX_PRIVATE_FLAG;
 // Are we using 32-bit or 64-bit Linux syscalls?
 // x32 (which defines __x86_64__) has SANITIZER_WORDSIZE == 32
 // but it still needs to use 64-bit syscalls.
-#  if SANITIZER_LINUX && (defined(__x86_64__) || defined(__powerpc64__) || \
-                          SANITIZER_WORDSIZE == 64 ||                      \
-                          (defined(__mips__) && _MIPS_SIM == _ABIN32))
+#  if SANITIZER_LINUX &&                                \
+      (defined(__x86_64__) || defined(__powerpc64__) || \
+       SANITIZER_WORDSIZE == 64 ||                      \
+       (defined(__mips__) && defined(_ABIN32) && _MIPS_SIM == _ABIN32))
 #    define SANITIZER_LINUX_USES_64BIT_SYSCALLS 1
 #  else
 #    define SANITIZER_LINUX_USES_64BIT_SYSCALLS 0
@@ -429,8 +436,9 @@ uptr internal_stat(const char *path, void *buf) {
                              AT_NO_AUTOMOUNT, STATX_BASIC_STATS, (uptr)&bufx);
   statx_to_stat(&bufx, (struct stat *)buf);
   return res;
-#      elif (SANITIZER_WORDSIZE == 64 || SANITIZER_X32 ||    \
-             (defined(__mips__) && _MIPS_SIM == _ABIN32)) && \
+#      elif (                                                                 \
+          SANITIZER_WORDSIZE == 64 || SANITIZER_X32 ||                        \
+          (defined(__mips__) && defined(_ABIN32) && _MIPS_SIM == _ABIN32)) && \
           !SANITIZER_SPARC
   return internal_syscall(SYSCALL(newfstatat), AT_FDCWD, (uptr)path, (uptr)buf,
                           0);
@@ -467,8 +475,9 @@ uptr internal_lstat(const char *path, void *buf) {
                              STATX_BASIC_STATS, (uptr)&bufx);
   statx_to_stat(&bufx, (struct stat *)buf);
   return res;
-#      elif (defined(_LP64) || SANITIZER_X32 ||              \
-             (defined(__mips__) && _MIPS_SIM == _ABIN32)) && \
+#      elif (                                                                 \
+          defined(_LP64) || SANITIZER_X32 ||                                  \
+          (defined(__mips__) && defined(_ABIN32) && _MIPS_SIM == _ABIN32)) && \
           !SANITIZER_SPARC
   return internal_syscall(SYSCALL(newfstatat), AT_FDCWD, (uptr)path, (uptr)buf,
                           AT_SYMLINK_NOFOLLOW);
@@ -1237,6 +1246,16 @@ uptr GetPageSize() {
   CHECK_EQ(rv, 0);
   return (uptr)pz;
 #    elif SANITIZER_USE_GETAUXVAL
+#      if SANITIZER_ANDROID && __ANDROID_API__ < 35
+  // The 16 KB page size was introduced in Android 15 (API level 35), while
+  // earlier versions of Android always used a 4 KB page size.
+  // We are checking the weak definition of `strerrorname_np` (introduced in API
+  // level 35) because some earlier API levels crashed when
+  // `getauxval(AT_PAGESZ)` was called from the `.preinit_array`.
+  if (!strerrorname_np)
+    return 4096;
+#      endif
+
   return getauxval(AT_PAGESZ);
 #    else
   return sysconf(_SC_PAGESIZE);  // EXEC_PAGESIZE may not be trustworthy.
diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_linux_libcdep.cpp b/compiler-rt/lib/sanitizer_common/sanitizer_linux_libcdep.cpp
index e11eff13cd326..e5e79d4e0521c 100644
--- a/compiler-rt/lib/sanitizer_common/sanitizer_linux_libcdep.cpp
+++ b/compiler-rt/lib/sanitizer_common/sanitizer_linux_libcdep.cpp
@@ -74,20 +74,6 @@ extern "C" int __sys_sigaction(int signum, const struct sigaction *act,
 #    include <thread.h>
 #  endif
 
-#  if SANITIZER_ANDROID
-#    include <android/api-level.h>
-#    if !defined(CPU_COUNT) && !defined(__aarch64__)
-#      include <dirent.h>
-#      include <fcntl.h>
-struct __sanitizer::linux_dirent {
-  long d_ino;
-  off_t d_off;
-  unsigned short d_reclen;
-  char d_name[];
-};
-#    endif
-#  endif
-
 #  if !SANITIZER_ANDROID
 #    include <elf.h>
 #    include <unistd.h>
@@ -857,42 +843,6 @@ u32 GetNumberOfCPUs() {
   req[1] = HW_NCPU;
   CHECK_EQ(internal_sysctl(req, 2, &ncpu, &len, NULL, 0), 0);
   return ncpu;
-#  elif SANITIZER_ANDROID && !defined(CPU_COUNT) && !defined(__aarch64__)
-  // Fall back to /sys/devices/system/cpu on Android when cpu_set_t doesn't
-  // exist in sched.h. That is the case for toolchains generated with older
-  // NDKs.
-  // This code doesn't work on AArch64 because internal_getdents makes use of
-  // the 64bit getdents syscall, but cpu_set_t seems to always exist on AArch64.
-  uptr fd = internal_open("/sys/devices/system/cpu", O_RDONLY | O_DIRECTORY);
-  if (internal_iserror(fd))
-    return 0;
-  InternalMmapVector<u8> buffer(4096);
-  uptr bytes_read = buffer.size();
-  uptr n_cpus = 0;
-  u8 *d_type;
-  struct linux_dirent *entry = (struct linux_dirent *)&buffer[bytes_read];
-  while (true) {
-    if ((u8 *)entry >= &buffer[bytes_read]) {
-      bytes_read = internal_getdents(fd, (struct linux_dirent *)buffer.data(),
-                                     buffer.size());
-      if (internal_iserror(bytes_read) || !bytes_read)
-        break;
-      entry = (struct linux_dirent *)buffer.data();
-    }
-    d_type = (u8 *)entry + entry->d_reclen - 1;
-    if (d_type >= &buffer[bytes_read] ||
-        (u8 *)&entry->d_name[3] >= &buffer[bytes_read])
-      break;
-    if (entry->d_ino != 0 && *d_type == DT_DIR) {
-      if (entry->d_name[0] == 'c' && entry->d_name[1] == 'p' &&
-          entry->d_name[2] == 'u' && entry->d_name[3] >= '0' &&
-          entry->d_name[3] <= '9')
-        n_cpus++;
-    }
-    entry = (struct linux_dirent *)(((u8 *)entry) + entry->d_reclen);
-  }
-  internal_close(fd);
-  return n_cpus;
 #  elif SANITIZER_SOLARIS
   return sysconf(_SC_NPROCESSORS_ONLN);
 #  else
diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_mac.cpp b/compiler-rt/lib/sanitizer_common/sanitizer_mac.cpp
index d15f30c61b586..0b8a75391136d 100644
--- a/compiler-rt/lib/sanitizer_common/sanitizer_mac.cpp
+++ b/compiler-rt/lib/sanitizer_common/sanitizer_mac.cpp
@@ -1203,13 +1203,14 @@ uptr MapDynamicShadow(uptr shadow_size_bytes, uptr shadow_scale,
   const uptr left_padding =
       Max<uptr>(granularity, 1ULL << min_shadow_base_alignment);
 
-  uptr space_size = shadow_size_bytes + left_padding;
+  uptr space_size = shadow_size_bytes;
 
   uptr largest_gap_found = 0;
   uptr max_occupied_addr = 0;
+
   VReport(2, "FindDynamicShadowStart, space_size = %p\n", (void *)space_size);
   uptr shadow_start =
-      FindAvailableMemoryRange(space_size, alignment, granularity,
+      FindAvailableMemoryRange(space_size, alignment, left_padding,
                                &largest_gap_found, &max_occupied_addr);
   // If the shadow doesn't fit, restrict the address space to make it fit.
   if (shadow_start == 0) {
@@ -1229,9 +1230,9 @@ uptr MapDynamicShadow(uptr shadow_size_bytes, uptr shadow_scale,
     }
     RestrictMemoryToMaxAddress(new_max_vm);
     high_mem_end = new_max_vm - 1;
-    space_size = (high_mem_end >> shadow_scale) + left_padding;
+    space_size = (high_mem_end >> shadow_scale);
     VReport(2, "FindDynamicShadowStart, space_size = %p\n", (void *)space_size);
-    shadow_start = FindAvailableMemoryRange(space_size, alignment, granularity,
+    shadow_start = FindAvailableMemoryRange(space_size, alignment, left_padding,
                                             nullptr, nullptr);
     if (shadow_start == 0) {
       Report("Unable to find a memory range after restricting VM.\n");
@@ -1272,10 +1273,15 @@ uptr FindAvailableMemoryRange(uptr size, uptr alignment, uptr left_padding,
     mach_msg_type_number_t count = kRegionInfoSize;
     kr = mach_vm_region_recurse(mach_task_self(), &address, &vmsize, &depth,
                                 (vm_region_info_t)&vminfo, &count);
-    if (kr == KERN_INVALID_ADDRESS) {
+
+    // There are cases where going beyond the processes' max vm does
+    // not return KERN_INVALID_ADDRESS so we check for going beyond that
+    // max address as well.
+    if (kr == KERN_INVALID_ADDRESS || address > max_vm_address) {
       // No more regions beyond "address", consider the gap at the end of VM.
       address = max_vm_address;
       vmsize = 0;
+      kr = -1;  // break after this iteration.
     } else {
       if (max_occupied_addr) *max_occupied_addr = address + vmsize;
     }
diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_posix.h b/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_posix.h
index 7e62dc0e0523e..1a7d9e64048eb 100644
--- a/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_posix.h
+++ b/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_posix.h
@@ -98,10 +98,13 @@ const unsigned struct_kernel_stat64_sz = 104;
 const unsigned struct_kernel_stat_sz = 144;
 const unsigned struct_kernel_stat64_sz = 104;
 #elif defined(__mips__)
-const unsigned struct_kernel_stat_sz =
-    SANITIZER_ANDROID
-        ? FIRST_32_SECOND_64(104, 128)
-        : FIRST_32_SECOND_64((_MIPS_SIM == _ABIN32) ? 176 : 160, 216);
+const unsigned struct_kernel_stat_sz = SANITIZER_ANDROID
+                                           ? FIRST_32_SECOND_64(104, 128)
+#      if defined(_ABIN32) && _MIPS_SIM == _ABIN32
+                                           : FIRST_32_SECOND_64(176, 216);
+#      else
+                                           : FIRST_32_SECOND_64(160, 216);
+#      endif
 const unsigned struct_kernel_stat64_sz = 104;
 #elif defined(__s390__) && !defined(__s390x__)
 const unsigned struct_kernel_stat_sz = 64;
diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_stoptheworld_linux_libcdep.cpp b/compiler-rt/lib/sanitizer_common/sanitizer_stoptheworld_linux_libcdep.cpp
index 945da99d41f4e..b05c67e354418 100644
--- a/compiler-rt/lib/sanitizer_common/sanitizer_stoptheworld_linux_libcdep.cpp
+++ b/compiler-rt/lib/sanitizer_common/sanitizer_stoptheworld_linux_libcdep.cpp
@@ -37,25 +37,28 @@
 # include <asm/ptrace.h>
 #endif
 #include <sys/user.h>  // for user_regs_struct
-#if SANITIZER_ANDROID && SANITIZER_MIPS
-# include <asm/reg.h>  // for mips SP register in sys/user.h
-#endif
-#include <sys/wait.h> // for signal-related stuff
-
-#ifdef sa_handler
-# undef sa_handler
-#endif
-
-#ifdef sa_sigaction
-# undef sa_sigaction
-#endif
-
-#include "sanitizer_common.h"
-#include "sanitizer_flags.h"
-#include "sanitizer_libc.h"
-#include "sanitizer_linux.h"
-#include "sanitizer_mutex.h"
-#include "sanitizer_placement_new.h"
+#  if SANITIZER_MIPS
+// clang-format off
+# include <asm/sgidefs.h>  // <asm/sgidefs.h> must be included before <asm/reg.h>
+# include <asm/reg.h>      // for mips SP register
+// clang-format on
+#  endif
+#  include <sys/wait.h>  // for signal-related stuff
+
+#  ifdef sa_handler
+#    undef sa_handler
+#  endif
+
+#  ifdef sa_sigaction
+#    undef sa_sigaction
+#  endif
+
+#  include "sanitizer_common.h"
+#  include "sanitizer_flags.h"
+#  include "sanitizer_libc.h"
+#  include "sanitizer_linux.h"
+#  include "sanitizer_mutex.h"
+#  include "sanitizer_placement_new.h"
 
 // Sufficiently old kernel headers don't provide this value, but we can still
 // call prctl with it. If the runtime kernel is new enough, the prctl call will
@@ -510,11 +513,7 @@ typedef pt_regs regs_struct;
 
 #elif defined(__mips__)
 typedef struct user regs_struct;
-# if SANITIZER_ANDROID
-#  define REG_SP regs[EF_R29]
-# else
-#  define REG_SP regs[EF_REG29]
-# endif
+#    define REG_SP regs[EF_R29]
 
 #elif defined(__aarch64__)
 typedef struct user_pt_regs regs_struct;
diff --git a/compiler-rt/lib/sanitizer_common/tests/sanitizer_libc_test.cpp b/compiler-rt/lib/sanitizer_common/tests/sanitizer_libc_test.cpp
index 272520fa51f01..1bec83be4c80b 100644
--- a/compiler-rt/lib/sanitizer_common/tests/sanitizer_libc_test.cpp
+++ b/compiler-rt/lib/sanitizer_common/tests/sanitizer_libc_test.cpp
@@ -352,6 +352,7 @@ TEST(SanitizerCommon, ReportFile) {
   // This will close tmpfile.
   report_file.SetReportPath("stderr");
   Unlink(tmpfile);
+  Unlink(path);
 }
 
 TEST(SanitizerCommon, FileExists) {
diff --git a/compiler-rt/lib/scudo/standalone/allocator_config.def b/compiler-rt/lib/scudo/standalone/allocator_config.def
index ce37b1cfaedcc..43893e9af3cf3 100644
--- a/compiler-rt/lib/scudo/standalone/allocator_config.def
+++ b/compiler-rt/lib/scudo/standalone/allocator_config.def
@@ -31,6 +31,9 @@
 #ifndef SECONDARY_REQUIRED_TEMPLATE_TYPE
 #define SECONDARY_REQUIRED_TEMPLATE_TYPE(...)
 #endif
+#ifndef SECONDARY_OPTIONAL
+#define SECONDARY_OPTIONAL(...)
+#endif
 #ifndef SECONDARY_CACHE_OPTIONAL
 #define SECONDARY_CACHE_OPTIONAL(...)
 #endif
@@ -109,6 +112,11 @@ PRIMARY_OPTIONAL_TYPE(CompactPtrT, uptr)
 // Defines the type of Secondary Cache to use.
 SECONDARY_REQUIRED_TEMPLATE_TYPE(CacheT)
 
+// SECONDARY_OPTIONAL(TYPE, NAME, DEFAULT)
+//
+// Add one guard page at the front and back for each allocation.
+SECONDARY_OPTIONAL(const bool, EnableGuardPages, true)
+
 // SECONDARY_CACHE_OPTIONAL(TYPE, NAME, DEFAULT)
 //
 // Defines the type of cache used by the Secondary. Some additional
@@ -122,6 +130,7 @@ SECONDARY_CACHE_OPTIONAL(const s32, MaxReleaseToOsIntervalMs, INT32_MAX)
 SECONDARY_CACHE_OPTIONAL(const s32, DefaultReleaseToOsIntervalMs, INT32_MIN)
 
 #undef SECONDARY_CACHE_OPTIONAL
+#undef SECONDARY_OPTIONAL
 #undef SECONDARY_REQUIRED_TEMPLATE_TYPE
 #undef PRIMARY_OPTIONAL_TYPE
 #undef PRIMARY_OPTIONAL
diff --git a/compiler-rt/lib/scudo/standalone/allocator_config_wrapper.h b/compiler-rt/lib/scudo/standalone/allocator_config_wrapper.h
index ea12a5b1447f6..ac639ee9dd943 100644
--- a/compiler-rt/lib/scudo/standalone/allocator_config_wrapper.h
+++ b/compiler-rt/lib/scudo/standalone/allocator_config_wrapper.h
@@ -95,6 +95,13 @@ template <typename AllocatorConfig> struct SecondaryConfig {
 #define SECONDARY_REQUIRED_TEMPLATE_TYPE(NAME)                                 \
   template <typename T>                                                        \
   using NAME = typename AllocatorConfig::Secondary::template NAME<T>;
+
+#define SECONDARY_OPTIONAL(TYPE, NAME, DEFAULT)                                \
+  OPTIONAL_TEMPLATE(TYPE, NAME, DEFAULT, NAME)                                 \
+  static constexpr removeConst<TYPE>::type get##NAME() {                       \
+    return NAME##State<typename AllocatorConfig::Secondary>::getValue();       \
+  }
+
 #include "allocator_config.def"
 
   struct CacheConfig {
diff --git a/compiler-rt/lib/scudo/standalone/secondary.h b/compiler-rt/lib/scudo/standalone/secondary.h
index 25b82356ca20f..f3f91c40e7a03 100644
--- a/compiler-rt/lib/scudo/standalone/secondary.h
+++ b/compiler-rt/lib/scudo/standalone/secondary.h
@@ -614,6 +614,12 @@ template <typename Config> class MapAllocator {
     return getBlockEnd(Ptr) - reinterpret_cast<uptr>(Ptr);
   }
 
+  static uptr getGuardPageSize() {
+    if (Config::getEnableGuardPages())
+      return getPageSizeCached();
+    return 0U;
+  }
+
   static constexpr uptr getHeadersSize() {
     return Chunk::getHeaderSize() + LargeBlock::getHeaderSize();
   }
@@ -763,11 +769,11 @@ void *MapAllocator<Config>::allocate(const Options &Options, uptr Size,
 
   uptr RoundedSize =
       roundUp(roundUp(Size, Alignment) + getHeadersSize(), PageSize);
-  if (Alignment > PageSize)
+  if (UNLIKELY(Alignment > PageSize))
     RoundedSize += Alignment - PageSize;
 
   ReservedMemoryT ReservedMemory;
-  const uptr MapSize = RoundedSize + 2 * PageSize;
+  const uptr MapSize = RoundedSize + 2 * getGuardPageSize();
   if (UNLIKELY(!ReservedMemory.create(/*Addr=*/0U, MapSize, nullptr,
                                       MAP_ALLOWNOMEM))) {
     return nullptr;
@@ -777,7 +783,7 @@ void *MapAllocator<Config>::allocate(const Options &Options, uptr Size,
   MemMapT MemMap = ReservedMemory.dispatch(ReservedMemory.getBase(),
                                            ReservedMemory.getCapacity());
   uptr MapBase = MemMap.getBase();
-  uptr CommitBase = MapBase + PageSize;
+  uptr CommitBase = MapBase + getGuardPageSize();
   uptr MapEnd = MapBase + MapSize;
 
   // In the unlikely event of alignments larger than a page, adjust the amount
@@ -786,25 +792,30 @@ void *MapAllocator<Config>::allocate(const Options &Options, uptr Size,
     // For alignments greater than or equal to a page, the user pointer (eg:
     // the pointer that is returned by the C or C++ allocation APIs) ends up
     // on a page boundary , and our headers will live in the preceding page.
-    CommitBase = roundUp(MapBase + PageSize + 1, Alignment) - PageSize;
-    const uptr NewMapBase = CommitBase - PageSize;
-    DCHECK_GE(NewMapBase, MapBase);
+    CommitBase =
+        roundUp(MapBase + getGuardPageSize() + 1, Alignment) - PageSize;
     // We only trim the extra memory on 32-bit platforms: 64-bit platforms
     // are less constrained memory wise, and that saves us two syscalls.
-    if (SCUDO_WORDSIZE == 32U && NewMapBase != MapBase) {
-      MemMap.unmap(MapBase, NewMapBase - MapBase);
-      MapBase = NewMapBase;
-    }
-    const uptr NewMapEnd =
-        CommitBase + PageSize + roundUp(Size, PageSize) + PageSize;
-    DCHECK_LE(NewMapEnd, MapEnd);
-    if (SCUDO_WORDSIZE == 32U && NewMapEnd != MapEnd) {
-      MemMap.unmap(NewMapEnd, MapEnd - NewMapEnd);
-      MapEnd = NewMapEnd;
+    if (SCUDO_WORDSIZE == 32U) {
+      const uptr NewMapBase = CommitBase - getGuardPageSize();
+      DCHECK_GE(NewMapBase, MapBase);
+      if (NewMapBase != MapBase) {
+        MemMap.unmap(MapBase, NewMapBase - MapBase);
+        MapBase = NewMapBase;
+      }
+      // CommitBase is past the first guard page, but this computation needs
+      // to include a page where the header lives.
+      const uptr NewMapEnd =
+          CommitBase + PageSize + roundUp(Size, PageSize) + getGuardPageSize();
+      DCHECK_LE(NewMapEnd, MapEnd);
+      if (NewMapEnd != MapEnd) {
+        MemMap.unmap(NewMapEnd, MapEnd - NewMapEnd);
+        MapEnd = NewMapEnd;
+      }
     }
   }
 
-  const uptr CommitSize = MapEnd - PageSize - CommitBase;
+  const uptr CommitSize = MapEnd - getGuardPageSize() - CommitBase;
   const uptr AllocPos = roundDown(CommitBase + CommitSize - Size, Alignment);
   if (!mapSecondary<Config>(Options, CommitBase, CommitSize, AllocPos, 0,
                             MemMap)) {
@@ -812,6 +823,8 @@ void *MapAllocator<Config>::allocate(const Options &Options, uptr Size,
     return nullptr;
   }
   const uptr HeaderPos = AllocPos - getHeadersSize();
+  // Make sure that the header is not in the guard page or before the base.
+  DCHECK_GE(HeaderPos, MapBase + getGuardPageSize());
   LargeBlock::Header *H = reinterpret_cast<LargeBlock::Header *>(
       LargeBlock::addHeaderTag<Config>(HeaderPos));
   if (useMemoryTagging<Config>(Options))
diff --git a/compiler-rt/lib/scudo/standalone/tests/combined_test.cpp b/compiler-rt/lib/scudo/standalone/tests/combined_test.cpp
index ff98eb3397ee0..9d665ef88c402 100644
--- a/compiler-rt/lib/scudo/standalone/tests/combined_test.cpp
+++ b/compiler-rt/lib/scudo/standalone/tests/combined_test.cpp
@@ -54,16 +54,18 @@ void checkMemoryTaggingMaybe(AllocatorT *Allocator, void *P, scudo::uptr Size,
                              scudo::uptr Alignment) {
   const scudo::uptr MinAlignment = 1UL << SCUDO_MIN_ALIGNMENT_LOG;
   Size = scudo::roundUp(Size, MinAlignment);
-  if (Allocator->useMemoryTaggingTestOnly())
+  if (Allocator->useMemoryTaggingTestOnly()) {
     EXPECT_DEATH(
         {
           disableDebuggerdMaybe();
           reinterpret_cast<char *>(P)[-1] = 'A';
         },
         "");
+  }
   if (isPrimaryAllocation<AllocatorT>(Size, Alignment)
           ? Allocator->useMemoryTaggingTestOnly()
-          : Alignment == MinAlignment) {
+          : Alignment == MinAlignment &&
+                AllocatorT::SecondaryT::getGuardPageSize() > 0) {
     EXPECT_DEATH(
         {
           disableDebuggerdMaybe();
diff --git a/compiler-rt/lib/scudo/standalone/tests/scudo_unit_test.h b/compiler-rt/lib/scudo/standalone/tests/scudo_unit_test.h
index f8b658c937889..27c0e591a2099 100644
--- a/compiler-rt/lib/scudo/standalone/tests/scudo_unit_test.h
+++ b/compiler-rt/lib/scudo/standalone/tests/scudo_unit_test.h
@@ -12,6 +12,7 @@
 #include <zxtest/zxtest.h>
 using Test = ::zxtest::Test;
 #define TEST_SKIP(message) ZXTEST_SKIP(message)
+#define TEST_HAS_FAILURE true
 #else
 #include "gtest/gtest.h"
 using Test = ::testing::Test;
@@ -19,6 +20,7 @@ using Test = ::testing::Test;
   do {                                                                         \
     GTEST_SKIP() << message;                                                   \
   } while (0)
+#define TEST_HAS_FAILURE Test::HasFailure()
 #endif
 
 // If EXPECT_DEATH isn't defined, make it a no-op.
diff --git a/compiler-rt/lib/scudo/standalone/tests/secondary_test.cpp b/compiler-rt/lib/scudo/standalone/tests/secondary_test.cpp
index 3638f1c36ddd9..d8a7f6bd66ed2 100644
--- a/compiler-rt/lib/scudo/standalone/tests/secondary_test.cpp
+++ b/compiler-rt/lib/scudo/standalone/tests/secondary_test.cpp
@@ -13,15 +13,19 @@
 #include "allocator_config_wrapper.h"
 #include "secondary.h"
 
+#include <string.h>
+
 #include <algorithm>
 #include <condition_variable>
 #include <memory>
 #include <mutex>
 #include <random>
-#include <stdio.h>
 #include <thread>
 #include <vector>
 
+// Get this once to use through-out the tests.
+const scudo::uptr PageSize = scudo::getPageSizeCached();
+
 template <typename Config> static scudo::Options getOptionsForConfig() {
   if (!Config::getMaySupportMemoryTagging() ||
       !scudo::archSupportsMemoryTagging() ||
@@ -32,60 +36,39 @@ template <typename Config> static scudo::Options getOptionsForConfig() {
   return AO.load();
 }
 
-template <typename Config> static void testSecondaryBasic(void) {
-  using SecondaryT = scudo::MapAllocator<scudo::SecondaryConfig<Config>>;
-  scudo::Options Options =
-      getOptionsForConfig<scudo::SecondaryConfig<Config>>();
+template <class Config> struct AllocatorInfoType {
+  std::unique_ptr<scudo::MapAllocator<scudo::SecondaryConfig<Config>>>
+      Allocator;
+  scudo::GlobalStats GlobalStats;
+  scudo::Options Options;
+
+  AllocatorInfoType(scudo::s32 ReleaseToOsInterval) {
+    using SecondaryT = scudo::MapAllocator<scudo::SecondaryConfig<Config>>;
+    Options = getOptionsForConfig<scudo::SecondaryConfig<Config>>();
+    GlobalStats.init();
+    Allocator.reset(new SecondaryT);
+    Allocator->init(&GlobalStats, ReleaseToOsInterval);
+  }
 
-  scudo::GlobalStats S;
-  S.init();
-  std::unique_ptr<SecondaryT> L(new SecondaryT);
-  L->init(&S);
-  const scudo::uptr Size = 1U << 16;
-  void *P = L->allocate(Options, Size);
-  EXPECT_NE(P, nullptr);
-  memset(P, 'A', Size);
-  EXPECT_GE(SecondaryT::getBlockSize(P), Size);
-  L->deallocate(Options, P);
+  AllocatorInfoType() : AllocatorInfoType(-1) {}
 
-  // If the Secondary can't cache that pointer, it will be unmapped.
-  if (!L->canCache(Size)) {
-    EXPECT_DEATH(
-        {
-          // Repeat few time to avoid missing crash if it's mmaped by unrelated
-          // code.
-          for (int i = 0; i < 10; ++i) {
-            P = L->allocate(Options, Size);
-            L->deallocate(Options, P);
-            memset(P, 'A', Size);
-          }
-        },
-        "");
-  }
+  ~AllocatorInfoType() {
+    if (Allocator == nullptr) {
+      return;
+    }
 
-  const scudo::uptr Align = 1U << 16;
-  P = L->allocate(Options, Size + Align, Align);
-  EXPECT_NE(P, nullptr);
-  void *AlignedP = reinterpret_cast<void *>(
-      scudo::roundUp(reinterpret_cast<scudo::uptr>(P), Align));
-  memset(AlignedP, 'A', Size);
-  L->deallocate(Options, P);
+    if (TEST_HAS_FAILURE) {
+      // Print all of the stats if the test fails.
+      scudo::ScopedString Str;
+      Allocator->getStats(&Str);
+      Str.output();
+    }
 
-  std::vector<void *> V;
-  for (scudo::uptr I = 0; I < 32U; I++)
-    V.push_back(L->allocate(Options, Size));
-  std::shuffle(V.begin(), V.end(), std::mt19937(std::random_device()()));
-  while (!V.empty()) {
-    L->deallocate(Options, V.back());
-    V.pop_back();
+    Allocator->unmapTestOnly();
   }
-  scudo::ScopedString Str;
-  L->getStats(&Str);
-  Str.output();
-  L->unmapTestOnly();
-}
+};
 
-struct NoCacheConfig {
+struct TestNoCacheConfig {
   static const bool MaySupportMemoryTagging = false;
   template <typename> using TSDRegistryT = void;
   template <typename> using PrimaryT = void;
@@ -97,7 +80,20 @@ struct NoCacheConfig {
   };
 };
 
-struct TestConfig {
+struct TestNoCacheNoGuardPageConfig {
+  static const bool MaySupportMemoryTagging = false;
+  template <typename> using TSDRegistryT = void;
+  template <typename> using PrimaryT = void;
+  template <typename Config> using SecondaryT = scudo::MapAllocator<Config>;
+
+  struct Secondary {
+    template <typename Config>
+    using CacheT = scudo::MapAllocatorNoCache<Config>;
+    static const bool EnableGuardPages = false;
+  };
+};
+
+struct TestCacheConfig {
   static const bool MaySupportMemoryTagging = false;
   template <typename> using TSDRegistryT = void;
   template <typename> using PrimaryT = void;
@@ -117,30 +113,85 @@ struct TestConfig {
   };
 };
 
-TEST(ScudoSecondaryTest, SecondaryBasic) {
-  testSecondaryBasic<NoCacheConfig>();
-  testSecondaryBasic<scudo::DefaultConfig>();
-  testSecondaryBasic<TestConfig>();
-}
+struct TestCacheNoGuardPageConfig {
+  static const bool MaySupportMemoryTagging = false;
+  template <typename> using TSDRegistryT = void;
+  template <typename> using PrimaryT = void;
+  template <typename> using SecondaryT = void;
 
-struct MapAllocatorTest : public Test {
-  using Config = scudo::DefaultConfig;
-  using LargeAllocator = scudo::MapAllocator<scudo::SecondaryConfig<Config>>;
+  struct Secondary {
+    struct Cache {
+      static const scudo::u32 EntriesArraySize = 128U;
+      static const scudo::u32 QuarantineSize = 0U;
+      static const scudo::u32 DefaultMaxEntriesCount = 64U;
+      static const scudo::uptr DefaultMaxEntrySize = 1UL << 20;
+      static const scudo::s32 MinReleaseToOsIntervalMs = INT32_MIN;
+      static const scudo::s32 MaxReleaseToOsIntervalMs = INT32_MAX;
+    };
 
-  void SetUp() override { Allocator->init(nullptr); }
+    template <typename Config> using CacheT = scudo::MapAllocatorCache<Config>;
+    static const bool EnableGuardPages = false;
+  };
+};
 
-  void TearDown() override { Allocator->unmapTestOnly(); }
+template <typename Config> static void testBasic() {
+  using SecondaryT = scudo::MapAllocator<scudo::SecondaryConfig<Config>>;
+  AllocatorInfoType<Config> Info;
 
-  std::unique_ptr<LargeAllocator> Allocator =
-      std::make_unique<LargeAllocator>();
-  scudo::Options Options =
-      getOptionsForConfig<scudo::SecondaryConfig<Config>>();
-};
+  const scudo::uptr Size = 1U << 16;
+  void *P = Info.Allocator->allocate(Info.Options, Size);
+  EXPECT_NE(P, nullptr);
+  memset(P, 'A', Size);
+  EXPECT_GE(SecondaryT::getBlockSize(P), Size);
+  Info.Allocator->deallocate(Info.Options, P);
+
+  // If the Secondary can't cache that pointer, it will be unmapped.
+  if (!Info.Allocator->canCache(Size)) {
+    EXPECT_DEATH(
+        {
+          // Repeat few time to avoid missing crash if it's mmaped by unrelated
+          // code.
+          for (int i = 0; i < 10; ++i) {
+            P = Info.Allocator->allocate(Info.Options, Size);
+            Info.Allocator->deallocate(Info.Options, P);
+            memset(P, 'A', Size);
+          }
+        },
+        "");
+  }
+
+  const scudo::uptr Align = 1U << 16;
+  P = Info.Allocator->allocate(Info.Options, Size + Align, Align);
+  EXPECT_NE(P, nullptr);
+  void *AlignedP = reinterpret_cast<void *>(
+      scudo::roundUp(reinterpret_cast<scudo::uptr>(P), Align));
+  memset(AlignedP, 'A', Size);
+  Info.Allocator->deallocate(Info.Options, P);
+
+  std::vector<void *> V;
+  for (scudo::uptr I = 0; I < 32U; I++)
+    V.push_back(Info.Allocator->allocate(Info.Options, Size));
+  std::shuffle(V.begin(), V.end(), std::mt19937(std::random_device()()));
+  while (!V.empty()) {
+    Info.Allocator->deallocate(Info.Options, V.back());
+    V.pop_back();
+  }
+}
+
+TEST(ScudoSecondaryTest, Basic) {
+  testBasic<TestNoCacheConfig>();
+  testBasic<TestNoCacheNoGuardPageConfig>();
+  testBasic<TestCacheConfig>();
+  testBasic<TestCacheNoGuardPageConfig>();
+  testBasic<scudo::DefaultConfig>();
+}
 
 // This exercises a variety of combinations of size and alignment for the
 // MapAllocator. The size computation done here mimic the ones done by the
 // combined allocator.
-TEST_F(MapAllocatorTest, SecondaryCombinations) {
+template <typename Config> void testAllocatorCombinations() {
+  AllocatorInfoType<Config> Info;
+
   constexpr scudo::uptr MinAlign = FIRST_32_SECOND_64(8, 16);
   constexpr scudo::uptr HeaderSize = scudo::roundUp(8, MinAlign);
   for (scudo::uptr SizeLog = 0; SizeLog <= 20; SizeLog++) {
@@ -154,106 +205,81 @@ TEST_F(MapAllocatorTest, SecondaryCombinations) {
             static_cast<scudo::uptr>((1LL << SizeLog) + Delta), MinAlign);
         const scudo::uptr Size =
             HeaderSize + UserSize + (Align > MinAlign ? Align - HeaderSize : 0);
-        void *P = Allocator->allocate(Options, Size, Align);
+        void *P = Info.Allocator->allocate(Info.Options, Size, Align);
         EXPECT_NE(P, nullptr);
         void *AlignedP = reinterpret_cast<void *>(
             scudo::roundUp(reinterpret_cast<scudo::uptr>(P), Align));
         memset(AlignedP, 0xff, UserSize);
-        Allocator->deallocate(Options, P);
+        Info.Allocator->deallocate(Info.Options, P);
       }
     }
   }
-  scudo::ScopedString Str;
-  Allocator->getStats(&Str);
-  Str.output();
 }
 
-TEST_F(MapAllocatorTest, SecondaryIterate) {
+TEST(ScudoSecondaryTest, AllocatorCombinations) {
+  testAllocatorCombinations<TestNoCacheConfig>();
+  testAllocatorCombinations<TestNoCacheNoGuardPageConfig>();
+}
+
+template <typename Config> void testAllocatorIterate() {
+  AllocatorInfoType<Config> Info;
+
   std::vector<void *> V;
-  const scudo::uptr PageSize = scudo::getPageSizeCached();
   for (scudo::uptr I = 0; I < 32U; I++)
-    V.push_back(Allocator->allocate(
-        Options, (static_cast<scudo::uptr>(std::rand()) % 16U) * PageSize));
+    V.push_back(Info.Allocator->allocate(
+        Info.Options,
+        (static_cast<scudo::uptr>(std::rand()) % 16U) * PageSize));
   auto Lambda = [&V](scudo::uptr Block) {
     EXPECT_NE(std::find(V.begin(), V.end(), reinterpret_cast<void *>(Block)),
               V.end());
   };
-  Allocator->disable();
-  Allocator->iterateOverBlocks(Lambda);
-  Allocator->enable();
+  Info.Allocator->disable();
+  Info.Allocator->iterateOverBlocks(Lambda);
+  Info.Allocator->enable();
   while (!V.empty()) {
-    Allocator->deallocate(Options, V.back());
+    Info.Allocator->deallocate(Info.Options, V.back());
     V.pop_back();
   }
-  scudo::ScopedString Str;
-  Allocator->getStats(&Str);
-  Str.output();
 }
 
-TEST_F(MapAllocatorTest, SecondaryCacheOptions) {
-  if (!Allocator->canCache(0U))
-    TEST_SKIP("Secondary Cache disabled");
-
-  // Attempt to set a maximum number of entries higher than the array size.
-  EXPECT_TRUE(Allocator->setOption(scudo::Option::MaxCacheEntriesCount, 4096U));
-
-  // Attempt to set an invalid (negative) number of entries
-  EXPECT_FALSE(Allocator->setOption(scudo::Option::MaxCacheEntriesCount, -1));
-
-  // Various valid combinations.
-  EXPECT_TRUE(Allocator->setOption(scudo::Option::MaxCacheEntriesCount, 4U));
-  EXPECT_TRUE(
-      Allocator->setOption(scudo::Option::MaxCacheEntrySize, 1UL << 20));
-  EXPECT_TRUE(Allocator->canCache(1UL << 18));
-  EXPECT_TRUE(
-      Allocator->setOption(scudo::Option::MaxCacheEntrySize, 1UL << 17));
-  EXPECT_FALSE(Allocator->canCache(1UL << 18));
-  EXPECT_TRUE(Allocator->canCache(1UL << 16));
-  EXPECT_TRUE(Allocator->setOption(scudo::Option::MaxCacheEntriesCount, 0U));
-  EXPECT_FALSE(Allocator->canCache(1UL << 16));
-  EXPECT_TRUE(Allocator->setOption(scudo::Option::MaxCacheEntriesCount, 4U));
-  EXPECT_TRUE(
-      Allocator->setOption(scudo::Option::MaxCacheEntrySize, 1UL << 20));
-  EXPECT_TRUE(Allocator->canCache(1UL << 16));
+TEST(ScudoSecondaryTest, AllocatorIterate) {
+  testAllocatorIterate<TestNoCacheConfig>();
+  testAllocatorIterate<TestNoCacheNoGuardPageConfig>();
 }
 
-struct MapAllocatorWithReleaseTest : public MapAllocatorTest {
-  void SetUp() override { Allocator->init(nullptr, /*ReleaseToOsInterval=*/0); }
-
-  void performAllocations() {
-    std::vector<void *> V;
-    const scudo::uptr PageSize = scudo::getPageSizeCached();
-    {
-      std::unique_lock<std::mutex> Lock(Mutex);
-      while (!Ready)
-        Cv.wait(Lock);
-    }
-    for (scudo::uptr I = 0; I < 128U; I++) {
-      // Deallocate 75% of the blocks.
-      const bool Deallocate = (std::rand() & 3) != 0;
-      void *P = Allocator->allocate(
-          Options, (static_cast<scudo::uptr>(std::rand()) % 16U) * PageSize);
-      if (Deallocate)
-        Allocator->deallocate(Options, P);
-      else
-        V.push_back(P);
-    }
-    while (!V.empty()) {
-      Allocator->deallocate(Options, V.back());
-      V.pop_back();
-    }
-  }
+template <typename Config> void testAllocatorWithReleaseThreadsRace() {
+  AllocatorInfoType<Config> Info(/*ReleaseToOsInterval=*/0);
 
   std::mutex Mutex;
   std::condition_variable Cv;
   bool Ready = false;
-};
 
-TEST_F(MapAllocatorWithReleaseTest, SecondaryThreadsRace) {
   std::thread Threads[16];
   for (scudo::uptr I = 0; I < ARRAY_SIZE(Threads); I++)
-    Threads[I] =
-        std::thread(&MapAllocatorWithReleaseTest::performAllocations, this);
+    Threads[I] = std::thread([&Mutex, &Cv, &Ready, &Info]() {
+      std::vector<void *> V;
+      {
+        std::unique_lock<std::mutex> Lock(Mutex);
+        while (!Ready)
+          Cv.wait(Lock);
+      }
+      for (scudo::uptr I = 0; I < 128U; I++) {
+        // Deallocate 75% of the blocks.
+        const bool Deallocate = (std::rand() & 3) != 0;
+        void *P = Info.Allocator->allocate(
+            Info.Options,
+            (static_cast<scudo::uptr>(std::rand()) % 16U) * PageSize);
+        if (Deallocate)
+          Info.Allocator->deallocate(Info.Options, P);
+        else
+          V.push_back(P);
+      }
+      while (!V.empty()) {
+        Info.Allocator->deallocate(Info.Options, V.back());
+        V.pop_back();
+      }
+    });
+
   {
     std::unique_lock<std::mutex> Lock(Mutex);
     Ready = true;
@@ -261,36 +287,101 @@ TEST_F(MapAllocatorWithReleaseTest, SecondaryThreadsRace) {
   }
   for (auto &T : Threads)
     T.join();
-  scudo::ScopedString Str;
-  Allocator->getStats(&Str);
-  Str.output();
 }
 
-struct MapAllocatorCacheTest : public Test {
-  static constexpr scudo::u32 UnmappedMarker = 0xDEADBEEF;
+TEST(ScudoSecondaryTest, AllocatorWithReleaseThreadsRace) {
+  testAllocatorWithReleaseThreadsRace<TestNoCacheConfig>();
+  testAllocatorWithReleaseThreadsRace<TestNoCacheNoGuardPageConfig>();
+}
+
+template <typename Config>
+void testGetMappedSize(scudo::uptr Size, scudo::uptr *mapped,
+                       scudo::uptr *guard_page_size) {
+  AllocatorInfoType<Config> Info;
 
-  static void testUnmapCallback(scudo::MemMapT &MemMap) {
+  scudo::uptr Stats[scudo::StatCount] = {};
+  Info.GlobalStats.get(Stats);
+  *mapped = Stats[scudo::StatMapped];
+  Stats[scudo::StatMapped] = 0;
+
+  // Make sure the allocation is aligned to a page boundary so that the checks
+  // in the tests can avoid problems due to allocations having different
+  // alignments.
+  void *Ptr = Info.Allocator->allocate(Info.Options, Size, PageSize);
+  EXPECT_NE(Ptr, nullptr);
+
+  Info.GlobalStats.get(Stats);
+  EXPECT_GE(Stats[scudo::StatMapped], *mapped);
+  *mapped = Stats[scudo::StatMapped] - *mapped;
+
+  Info.Allocator->deallocate(Info.Options, Ptr);
+
+  *guard_page_size = Info.Allocator->getGuardPageSize();
+}
+
+TEST(ScudoSecondaryTest, VerifyGuardPageOption) {
+  static scudo::uptr AllocSize = 1000 * PageSize;
+
+  // Verify that a config with guard pages enabled:
+  //  - Non-zero sized guard page
+  //  - Mapped in at least the size of the allocation plus 2 * guard page size
+  scudo::uptr guard_mapped = 0;
+  scudo::uptr guard_page_size = 0;
+  testGetMappedSize<TestNoCacheConfig>(AllocSize, &guard_mapped,
+                                       &guard_page_size);
+  EXPECT_GT(guard_page_size, 0U);
+  EXPECT_GE(guard_mapped, AllocSize + 2 * guard_page_size);
+
+  // Verify that a config with guard pages disabled:
+  //  - Zero sized guard page
+  //  - The total mapped in is greater than the allocation size
+  scudo::uptr no_guard_mapped = 0;
+  scudo::uptr no_guard_page_size = 0;
+  testGetMappedSize<TestNoCacheNoGuardPageConfig>(AllocSize, &no_guard_mapped,
+                                                  &no_guard_page_size);
+  EXPECT_EQ(no_guard_page_size, 0U);
+  EXPECT_GE(no_guard_mapped, AllocSize);
+
+  // Verify that a guard page config mapped in at least twice the size of
+  // their guard page when compared to a no guard page config.
+  EXPECT_GE(guard_mapped, no_guard_mapped + guard_page_size * 2);
+}
+
+// Value written to cache entries that are unmapped.
+static scudo::u32 UnmappedMarker = 0xDEADBEEF;
+
+template <class Config> struct CacheInfoType {
+  static void addMarkerToMapCallback(scudo::MemMapT &MemMap) {
+    // When a cache entry is unmaped, don't unmap it write a special marker
+    // to indicate the cache entry was released. The real unmap will happen
+    // in the destructor. It is assumed that all of these maps will be in
+    // the MemMaps vector.
     scudo::u32 *Ptr = reinterpret_cast<scudo::u32 *>(MemMap.getBase());
     *Ptr = UnmappedMarker;
   }
 
-  using SecondaryConfig = scudo::SecondaryConfig<TestConfig>;
+  using SecondaryConfig = scudo::SecondaryConfig<TestCacheConfig>;
   using CacheConfig = SecondaryConfig::CacheConfig;
-  using CacheT = scudo::MapAllocatorCache<CacheConfig, testUnmapCallback>;
-
+  using CacheT = scudo::MapAllocatorCache<CacheConfig, addMarkerToMapCallback>;
+  scudo::Options Options = getOptionsForConfig<SecondaryConfig>();
   std::unique_ptr<CacheT> Cache = std::make_unique<CacheT>();
-
-  const scudo::uptr PageSize = scudo::getPageSizeCached();
+  std::vector<scudo::MemMapT> MemMaps;
   // The current test allocation size is set to the maximum
   // cache entry size
   static constexpr scudo::uptr TestAllocSize =
       CacheConfig::getDefaultMaxEntrySize();
 
-  scudo::Options Options = getOptionsForConfig<SecondaryConfig>();
+  CacheInfoType() { Cache->init(/*ReleaseToOsInterval=*/-1); }
 
-  void SetUp() override { Cache->init(/*ReleaseToOsInterval=*/-1); }
+  ~CacheInfoType() {
+    if (Cache == nullptr) {
+      return;
+    }
 
-  void TearDown() override { Cache->unmapTestOnly(); }
+    // Clean up MemMaps
+    for (auto &MemMap : MemMaps)
+      MemMap.unmap();
+  }
 
   scudo::MemMapT allocate(scudo::uptr Size) {
     scudo::uptr MapSize = scudo::roundUp(Size, PageSize);
@@ -304,8 +395,7 @@ struct MapAllocatorCacheTest : public Test {
     return MemMap;
   }
 
-  void fillCacheWithSameSizeBlocks(std::vector<scudo::MemMapT> &MemMaps,
-                                   scudo::uptr NumEntries, scudo::uptr Size) {
+  void fillCacheWithSameSizeBlocks(scudo::uptr NumEntries, scudo::uptr Size) {
     for (scudo::uptr I = 0; I < NumEntries; I++) {
       MemMaps.emplace_back(allocate(Size));
       auto &MemMap = MemMaps[I];
@@ -315,58 +405,60 @@ struct MapAllocatorCacheTest : public Test {
   }
 };
 
-TEST_F(MapAllocatorCacheTest, CacheOrder) {
-  std::vector<scudo::MemMapT> MemMaps;
-  Cache->setOption(scudo::Option::MaxCacheEntriesCount,
-                   CacheConfig::getEntriesArraySize());
+TEST(ScudoSecondaryTest, AllocatorCacheEntryOrder) {
+  CacheInfoType<TestCacheConfig> Info;
+  using CacheConfig = CacheInfoType<TestCacheConfig>::CacheConfig;
 
-  fillCacheWithSameSizeBlocks(MemMaps, CacheConfig::getEntriesArraySize(),
-                              TestAllocSize);
+  Info.Cache->setOption(scudo::Option::MaxCacheEntriesCount,
+                        CacheConfig::getEntriesArraySize());
+
+  Info.fillCacheWithSameSizeBlocks(CacheConfig::getEntriesArraySize(),
+                                   Info.TestAllocSize);
 
   // Retrieval order should be the inverse of insertion order
   for (scudo::uptr I = CacheConfig::getEntriesArraySize(); I > 0; I--) {
     scudo::uptr EntryHeaderPos;
-    scudo::CachedBlock Entry =
-        Cache->retrieve(0, TestAllocSize, PageSize, 0, EntryHeaderPos);
-    EXPECT_EQ(Entry.MemMap.getBase(), MemMaps[I - 1].getBase());
+    scudo::CachedBlock Entry = Info.Cache->retrieve(
+        0, Info.TestAllocSize, PageSize, 0, EntryHeaderPos);
+    EXPECT_EQ(Entry.MemMap.getBase(), Info.MemMaps[I - 1].getBase());
   }
-
-  // Clean up MemMaps
-  for (auto &MemMap : MemMaps)
-    MemMap.unmap();
 }
 
-TEST_F(MapAllocatorCacheTest, PartialChunkHeuristicRetrievalTest) {
+TEST(ScudoSecondaryTest, AllocatorCachePartialChunkHeuristicRetrievalTest) {
+  CacheInfoType<TestCacheConfig> Info;
+
   const scudo::uptr FragmentedPages =
       1 + scudo::CachedBlock::MaxReleasedCachePages;
   scudo::uptr EntryHeaderPos;
   scudo::CachedBlock Entry;
-  scudo::MemMapT MemMap = allocate(PageSize + FragmentedPages * PageSize);
-  Cache->store(Options, MemMap.getBase(), MemMap.getCapacity(),
-               MemMap.getBase(), MemMap);
+  scudo::MemMapT MemMap = Info.allocate(PageSize + FragmentedPages * PageSize);
+  Info.Cache->store(Info.Options, MemMap.getBase(), MemMap.getCapacity(),
+                    MemMap.getBase(), MemMap);
 
   // FragmentedPages > MaxAllowedFragmentedPages so PageSize
   // cannot be retrieved from the cache
-  Entry = Cache->retrieve(/*MaxAllowedFragmentedPages=*/0, PageSize, PageSize,
-                          0, EntryHeaderPos);
+  Entry = Info.Cache->retrieve(/*MaxAllowedFragmentedPages=*/0, PageSize,
+                               PageSize, 0, EntryHeaderPos);
   EXPECT_FALSE(Entry.isValid());
 
   // FragmentedPages == MaxAllowedFragmentedPages so PageSize
   // can be retrieved from the cache
-  Entry =
-      Cache->retrieve(FragmentedPages, PageSize, PageSize, 0, EntryHeaderPos);
+  Entry = Info.Cache->retrieve(FragmentedPages, PageSize, PageSize, 0,
+                               EntryHeaderPos);
   EXPECT_TRUE(Entry.isValid());
 
   MemMap.unmap();
 }
 
-TEST_F(MapAllocatorCacheTest, MemoryLeakTest) {
-  std::vector<scudo::MemMapT> MemMaps;
+TEST(ScudoSecondaryTest, AllocatorCacheMemoryLeakTest) {
+  CacheInfoType<TestCacheConfig> Info;
+  using CacheConfig = CacheInfoType<TestCacheConfig>::CacheConfig;
+
   // Fill the cache above MaxEntriesCount to force an eviction
   // The first cache entry should be evicted (because it is the oldest)
   // due to the maximum number of entries being reached
-  fillCacheWithSameSizeBlocks(
-      MemMaps, CacheConfig::getDefaultMaxEntriesCount() + 1, TestAllocSize);
+  Info.fillCacheWithSameSizeBlocks(CacheConfig::getDefaultMaxEntriesCount() + 1,
+                                   Info.TestAllocSize);
 
   std::vector<scudo::CachedBlock> RetrievedEntries;
 
@@ -374,16 +466,40 @@ TEST_F(MapAllocatorCacheTest, MemoryLeakTest) {
   // inserted into the cache
   for (scudo::uptr I = CacheConfig::getDefaultMaxEntriesCount(); I > 0; I--) {
     scudo::uptr EntryHeaderPos;
-    RetrievedEntries.push_back(
-        Cache->retrieve(0, TestAllocSize, PageSize, 0, EntryHeaderPos));
-    EXPECT_EQ(MemMaps[I].getBase(), RetrievedEntries.back().MemMap.getBase());
+    RetrievedEntries.push_back(Info.Cache->retrieve(
+        0, Info.TestAllocSize, PageSize, 0, EntryHeaderPos));
+    EXPECT_EQ(Info.MemMaps[I].getBase(),
+              RetrievedEntries.back().MemMap.getBase());
   }
 
   // Evicted entry should be marked due to unmap callback
-  EXPECT_EQ(*reinterpret_cast<scudo::u32 *>(MemMaps[0].getBase()),
+  EXPECT_EQ(*reinterpret_cast<scudo::u32 *>(Info.MemMaps[0].getBase()),
             UnmappedMarker);
+}
+
+TEST(ScudoSecondaryTest, AllocatorCacheOptions) {
+  CacheInfoType<TestCacheConfig> Info;
 
-  // Clean up MemMaps
-  for (auto &MemMap : MemMaps)
-    MemMap.unmap();
+  // Attempt to set a maximum number of entries higher than the array size.
+  EXPECT_TRUE(
+      Info.Cache->setOption(scudo::Option::MaxCacheEntriesCount, 4096U));
+
+  // Attempt to set an invalid (negative) number of entries
+  EXPECT_FALSE(Info.Cache->setOption(scudo::Option::MaxCacheEntriesCount, -1));
+
+  // Various valid combinations.
+  EXPECT_TRUE(Info.Cache->setOption(scudo::Option::MaxCacheEntriesCount, 4U));
+  EXPECT_TRUE(
+      Info.Cache->setOption(scudo::Option::MaxCacheEntrySize, 1UL << 20));
+  EXPECT_TRUE(Info.Cache->canCache(1UL << 18));
+  EXPECT_TRUE(
+      Info.Cache->setOption(scudo::Option::MaxCacheEntrySize, 1UL << 17));
+  EXPECT_FALSE(Info.Cache->canCache(1UL << 18));
+  EXPECT_TRUE(Info.Cache->canCache(1UL << 16));
+  EXPECT_TRUE(Info.Cache->setOption(scudo::Option::MaxCacheEntriesCount, 0U));
+  EXPECT_FALSE(Info.Cache->canCache(1UL << 16));
+  EXPECT_TRUE(Info.Cache->setOption(scudo::Option::MaxCacheEntriesCount, 4U));
+  EXPECT_TRUE(
+      Info.Cache->setOption(scudo::Option::MaxCacheEntrySize, 1UL << 20));
+  EXPECT_TRUE(Info.Cache->canCache(1UL << 16));
 }
diff --git a/compiler-rt/lib/tsan/go/buildgo.sh b/compiler-rt/lib/tsan/go/buildgo.sh
index 78ba41a0bdc6d..6871b36c3f510 100755
--- a/compiler-rt/lib/tsan/go/buildgo.sh
+++ b/compiler-rt/lib/tsan/go/buildgo.sh
@@ -18,6 +18,8 @@ if [ "`uname -a | grep Linux`" != "" ]; then
 		else
 			HOST_GOARCH="mips64"
 		fi
+  	elif [ "`uname -a | grep riscv64`" != "" ]; then
+		HOST_GOARCH="riscv64"
 	elif [ "`uname -a | grep s390x`" != "" ]; then
 		HOST_GOARCH="s390x"
 	fi
diff --git a/compiler-rt/lib/tsan/rtl/tsan_interceptors_mac.cpp b/compiler-rt/lib/tsan/rtl/tsan_interceptors_mac.cpp
index e0e4c5b9d36cd..978664411fff4 100644
--- a/compiler-rt/lib/tsan/rtl/tsan_interceptors_mac.cpp
+++ b/compiler-rt/lib/tsan/rtl/tsan_interceptors_mac.cpp
@@ -25,7 +25,6 @@
 #  include "tsan_interceptors.h"
 #  include "tsan_interface.h"
 #  include "tsan_interface_ann.h"
-#  include "tsan_spinlock_defs_mac.h"
 
 #  if defined(__has_include) && __has_include(<xpc/xpc.h>)
 #    include <xpc/xpc.h>
@@ -96,8 +95,7 @@ static constexpr morder kMacFailureOrder = mo_relaxed;
                 m_orig(int32_t, uint32_t, a32, f##32##OrigBarrier,       \
                        __tsan_atomic32_##tsan_atomic_f, kMacOrderBarrier)
 
-#  pragma clang diagnostic push
-// OSAtomic* functions are deprecated.
+#  pragma clang diagnostic push  // OSAtomic* deprecation
 #  pragma clang diagnostic ignored "-Wdeprecated-declarations"
 OSATOMIC_INTERCEPTORS_ARITHMETIC(OSAtomicAdd, fetch_add,
                                  OSATOMIC_INTERCEPTOR_PLUS_X)
@@ -111,6 +109,7 @@ OSATOMIC_INTERCEPTORS_BITWISE(OSAtomicAnd, fetch_and,
                               OSATOMIC_INTERCEPTOR_PLUS_X, OSATOMIC_INTERCEPTOR)
 OSATOMIC_INTERCEPTORS_BITWISE(OSAtomicXor, fetch_xor,
                               OSATOMIC_INTERCEPTOR_PLUS_X, OSATOMIC_INTERCEPTOR)
+#  pragma clang diagnostic pop  // OSAtomic* deprecation
 
 #  define OSATOMIC_INTERCEPTORS_CAS(f, tsan_atomic_f, tsan_t, t)           \
     TSAN_INTERCEPTOR(bool, f, t old_value, t new_value, t volatile *ptr) { \
@@ -128,8 +127,7 @@ OSATOMIC_INTERCEPTORS_BITWISE(OSAtomicXor, fetch_xor,
           kMacOrderBarrier, kMacFailureOrder);                             \
     }
 
-#  pragma clang diagnostic push
-// OSAtomicCompareAndSwap* functions are deprecated.
+#  pragma clang diagnostic push  // OSAtomicCompareAndSwap* deprecation
 #  pragma clang diagnostic ignored "-Wdeprecated-declarations"
 OSATOMIC_INTERCEPTORS_CAS(OSAtomicCompareAndSwapInt, __tsan_atomic32, a32, int)
 OSATOMIC_INTERCEPTORS_CAS(OSAtomicCompareAndSwapLong, __tsan_atomic64, a64,
@@ -140,7 +138,7 @@ OSATOMIC_INTERCEPTORS_CAS(OSAtomicCompareAndSwap32, __tsan_atomic32, a32,
                           int32_t)
 OSATOMIC_INTERCEPTORS_CAS(OSAtomicCompareAndSwap64, __tsan_atomic64, a64,
                           int64_t)
-#  pragma clang diagnostic pop
+#  pragma clang diagnostic pop  // OSAtomicCompareAndSwap* deprecation
 
 #  define OSATOMIC_INTERCEPTOR_BITOP(f, op, clear, mo)             \
     TSAN_INTERCEPTOR(bool, f, uint32_t n, volatile void *ptr) {    \
@@ -156,9 +154,12 @@ OSATOMIC_INTERCEPTORS_CAS(OSAtomicCompareAndSwap64, __tsan_atomic64, a64,
     OSATOMIC_INTERCEPTOR_BITOP(f, op, clear, kMacOrderNonBarrier) \
     OSATOMIC_INTERCEPTOR_BITOP(f##Barrier, op, clear, kMacOrderBarrier)
 
+#  pragma clang diagnostic push  // OSAtomicTestAnd* deprecation
+#  pragma clang diagnostic ignored "-Wdeprecated-declarations"
 OSATOMIC_INTERCEPTORS_BITOP(OSAtomicTestAndSet, __tsan_atomic8_fetch_or, false)
 OSATOMIC_INTERCEPTORS_BITOP(OSAtomicTestAndClear, __tsan_atomic8_fetch_and,
                             true)
+#  pragma clang diagnostic pop  // OSAtomicTestAnd* deprecation
 
 TSAN_INTERCEPTOR(void, OSAtomicEnqueue, OSQueueHead *list, void *item,
                  size_t offset) {
@@ -196,6 +197,16 @@ TSAN_INTERCEPTOR(void *, OSAtomicFifoDequeue, OSFifoQueueHead *list,
 
 #  endif
 
+// If `OSSPINLOCK_USE_INLINED=1` is set, then SDK headers don't declare these
+// as functions, but macros that call non-deprecated APIs.  Undefine these
+// macros so they don't interfere with the interceptor machinery.
+#  undef OSSpinLockLock
+#  undef OSSpinLockTry
+#  undef OSSpinLockUnlock
+
+#  pragma clang diagnostic push  // OSSpinLock* deprecation
+#  pragma clang diagnostic ignored "-Wdeprecated-declarations"
+
 TSAN_INTERCEPTOR(void, OSSpinLockLock, volatile OSSpinLock *lock) {
   CHECK(!cur_thread()->is_dead);
   if (!cur_thread()->is_inited) {
@@ -227,6 +238,7 @@ TSAN_INTERCEPTOR(void, OSSpinLockUnlock, volatile OSSpinLock *lock) {
   Release(thr, pc, (uptr)lock);
   REAL(OSSpinLockUnlock)(lock);
 }
+#  pragma clang diagnostic pop  // OSSpinLock* deprecation
 
 TSAN_INTERCEPTOR(void, os_lock_lock, void *lock) {
   CHECK(!cur_thread()->is_dead);
diff --git a/compiler-rt/lib/tsan/rtl/tsan_spinlock_defs_mac.h b/compiler-rt/lib/tsan/rtl/tsan_spinlock_defs_mac.h
deleted file mode 100644
index 1a99a81c03023..0000000000000
--- a/compiler-rt/lib/tsan/rtl/tsan_spinlock_defs_mac.h
+++ /dev/null
@@ -1,45 +0,0 @@
-//===-- tsan_spinlock_defs_mac.h -------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-//
-// This file is a part of ThreadSanitizer (TSan), a race detector.
-//
-// Mac-specific forward-declared function defintions that may be
-// deprecated in later versions of the OS.
-// These are needed for interceptors.
-//
-//===----------------------------------------------------------------------===//
-
-#if SANITIZER_APPLE
-
-#ifndef TSAN_SPINLOCK_DEFS_MAC_H
-#define TSAN_SPINLOCK_DEFS_MAC_H
-
-#include <stdint.h>
-
-extern "C" {
-
-/*
-Provides forward declarations related to OSSpinLocks on Darwin. These functions are
-deprecated on macOS version 10.12 and later,
-and are no longer included in the system headers.
-
-However, the symbols are still available on the system, so we provide these forward
-declarations to prevent compilation errors in tsan_interceptors_mac.cpp, which
-references these functions when defining TSAN interceptor functions.
-*/
-
-typedef int32_t OSSpinLock;
-
-void OSSpinLockLock(volatile OSSpinLock *__lock);
-void OSSpinLockUnlock(volatile OSSpinLock *__lock);
-bool OSSpinLockTry(volatile OSSpinLock *__lock);
-
-}
-
-#endif //TSAN_SPINLOCK_DEFS_MAC_H
-#endif // SANITIZER_APPLE
diff --git a/compiler-rt/lib/xray/CMakeLists.txt b/compiler-rt/lib/xray/CMakeLists.txt
index e7f01a2f4f164..c79b0b634ddf1 100644
--- a/compiler-rt/lib/xray/CMakeLists.txt
+++ b/compiler-rt/lib/xray/CMakeLists.txt
@@ -106,6 +106,13 @@ set(riscv64_SOURCES
   xray_trampoline_riscv64.S
   )
 
+  set(s390x_SOURCES
+  xray_s390x.cpp
+  xray_trampoline_s390x.S
+  )
+# Enable vector instructions in the assembly file.
+set_source_files_properties(xray_trampoline_s390x.S PROPERTIES COMPILE_FLAGS -march=z13)
+
 set(XRAY_SOURCE_ARCHS
   arm
   armhf
@@ -116,6 +123,7 @@ set(XRAY_SOURCE_ARCHS
   mips64
   mips64el
   powerpc64le
+  s390x
   x86_64
   )
 
@@ -168,6 +176,7 @@ set(XRAY_ALL_SOURCE_FILES
   ${powerpc64le_SOURCES}
   ${riscv32_SOURCES}
   ${riscv64_SOURCES}
+  ${s390x_SOURCES}
   ${XRAY_IMPL_HEADERS}
   )
 list(REMOVE_DUPLICATES XRAY_ALL_SOURCE_FILES)
diff --git a/compiler-rt/lib/xray/xray_interface.cpp b/compiler-rt/lib/xray/xray_interface.cpp
index 4ec492c266d80..3f97827874a70 100644
--- a/compiler-rt/lib/xray/xray_interface.cpp
+++ b/compiler-rt/lib/xray/xray_interface.cpp
@@ -61,6 +61,8 @@ static const int16_t cSledLength = 20;
 static const int16_t cSledLength = 68;
 #elif defined(__riscv) && (__riscv_xlen == 32)
 static const int16_t cSledLength = 52;
+#elif defined(__s390x__)
+static const int16_t cSledLength = 18;
 #else
 #error "Unsupported CPU Architecture"
 #endif /* CPU architecture */
diff --git a/compiler-rt/lib/xray/xray_interface_internal.h b/compiler-rt/lib/xray/xray_interface_internal.h
index a8cfe0fde84dd..5dcccfe825cf5 100644
--- a/compiler-rt/lib/xray/xray_interface_internal.h
+++ b/compiler-rt/lib/xray/xray_interface_internal.h
@@ -29,6 +29,10 @@ extern void __xray_FunctionTailExit();
 extern void __xray_ArgLoggerEntry();
 extern void __xray_CustomEvent();
 extern void __xray_TypedEvent();
+#if defined(__s390x__)
+extern void __xray_FunctionEntryVec();
+extern void __xray_FunctionExitVec();
+#endif
 }
 
 extern "C" {
diff --git a/compiler-rt/lib/xray/xray_s390x.cpp b/compiler-rt/lib/xray/xray_s390x.cpp
new file mode 100644
index 0000000000000..599485435671b
--- /dev/null
+++ b/compiler-rt/lib/xray/xray_s390x.cpp
@@ -0,0 +1,104 @@
+//===-- xray_s390x.cpp ------------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of XRay, a dynamic runtime instrumentation system.
+//
+// Implementation of s390x routines.
+//
+//===----------------------------------------------------------------------===//
+#include "sanitizer_common/sanitizer_common.h"
+#include "xray_defs.h"
+#include "xray_interface_internal.h"
+#include <cassert>
+#include <cstring>
+
+bool __xray::patchFunctionEntry(const bool Enable, const uint32_t FuncId,
+                                const XRaySledEntry &Sled,
+                                const XRayTrampolines &Trampolines,
+                                bool LogArgs) XRAY_NEVER_INSTRUMENT {
+  uint32_t *Address = reinterpret_cast<uint32_t *>(Sled.address());
+  // TODO: Trampoline addresses are currently inserted at compile-time, using
+  //       __xray_FunctionEntry and __xray_FunctionExit only.
+  //       To support DSO instrumentation, trampolines have to be written during
+  //       patching (see implementation on X86_64, e.g.).
+  if (Enable) {
+    // The resulting code is:
+    //   stmg    %r2, %r15, 16(%r15)
+    //   llilf   %2, FuncID
+    //   brasl   %r14, __xray_FunctionEntry@GOT
+    // The FuncId and the stmg instruction must be written.
+
+    // Write FuncId into llilf.
+    Address[2] = FuncId;
+    // Write last part of stmg.
+    reinterpret_cast<uint16_t *>(Address)[2] = 0x24;
+    // Write first part of stmg.
+    Address[0] = 0xeb2ff010;
+  } else {
+    // j +16 instructions.
+    Address[0] = 0xa7f4000b;
+  }
+  return true;
+}
+
+bool __xray::patchFunctionExit(
+    const bool Enable, const uint32_t FuncId, const XRaySledEntry &Sled,
+    const XRayTrampolines &Trampolines) XRAY_NEVER_INSTRUMENT {
+  uint32_t *Address = reinterpret_cast<uint32_t *>(Sled.address());
+  // TODO: Trampoline addresses are currently inserted at compile-time, using
+  //       __xray_FunctionEntry and __xray_FunctionExit only.
+  //       To support DSO instrumentation, trampolines have to be written during
+  //       patching (see implementation on X86_64, e.g.).
+  if (Enable) {
+    // The resulting code is:
+    //   stmg    %r2, %r15, 24(%r15)
+    //   llilf   %2,FuncID
+    //   j       __xray_FunctionEntry@GOT
+    // The FuncId and the stmg instruction must be written.
+
+    // Write FuncId into llilf.
+    Address[2] = FuncId;
+    // Write last part of of stmg.
+    reinterpret_cast<uint16_t *>(Address)[2] = 0x24;
+    // Write first part of stmg.
+    Address[0] = 0xeb2ff010;
+  } else {
+    // br %14 instruction.
+    reinterpret_cast<uint16_t *>(Address)[0] = 0x07fe;
+  }
+  return true;
+}
+
+bool __xray::patchFunctionTailExit(
+    const bool Enable, const uint32_t FuncId, const XRaySledEntry &Sled,
+    const XRayTrampolines &Trampolines) XRAY_NEVER_INSTRUMENT {
+  return patchFunctionExit(Enable, FuncId, Sled, Trampolines);
+}
+
+bool __xray::patchCustomEvent(const bool Enable, const uint32_t FuncId,
+                              const XRaySledEntry &Sled) XRAY_NEVER_INSTRUMENT {
+  // TODO Implement.
+  return false;
+}
+
+bool __xray::patchTypedEvent(const bool Enable, const uint32_t FuncId,
+                             const XRaySledEntry &Sled) XRAY_NEVER_INSTRUMENT {
+  // TODO Implement.
+  return false;
+}
+
+extern "C" void __xray_ArgLoggerEntry() XRAY_NEVER_INSTRUMENT {
+  // TODO this will have to be implemented in the trampoline assembly file.
+}
+
+extern "C" void __xray_FunctionTailExit() XRAY_NEVER_INSTRUMENT {
+  // For PowerPC, calls to __xray_FunctionEntry and __xray_FunctionExit
+  // are statically inserted into the sled. Tail exits are handled like normal
+  // function exits. This trampoline is therefore not implemented.
+  // This stub is placed here to avoid linking issues.
+}
diff --git a/compiler-rt/lib/xray/xray_trampoline_s390x.S b/compiler-rt/lib/xray/xray_trampoline_s390x.S
new file mode 100644
index 0000000000000..4073943641b99
--- /dev/null
+++ b/compiler-rt/lib/xray/xray_trampoline_s390x.S
@@ -0,0 +1,176 @@
+//===-- xray_trampoline_s390x.s ---------------------------------*- ASM -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of XRay, a dynamic runtime instrumentation system.
+//
+// This implements the s390x-specific assembler for the trampolines.
+// 2 versions of the functions are provided: one which does not store the
+// vector registers, and one which does store them. The compiler decides
+// which to call based on the availability of the vector extension.
+//
+//===----------------------------------------------------------------------===//
+
+    .text
+
+// Minimal stack frame size
+#define STACKSZ  160
+
+// Minimal stack frame size (160) plus space for 8 vector registers a 16 bytes.
+#define STACKSZ_VEC  288
+
+//===----------------------------------------------------------------------===//
+
+    .globl  __xray_FunctionEntry
+    .p2align    4
+    .type   __xray_FunctionEntry,@function
+__xray_FunctionEntry:
+    # The registers r2-15 of the instrumented function are already saved in the
+    # stack frame. On entry, r2 contains the function id, and %r14 the address
+    # of the first instruction of the instrumented function.
+    # Register r14 will be stored in the slot reserved for compiler use.
+    stg     %r14, 8(%r15)
+    std     %f0, 128(%r15)
+    std     %f2, 136(%r15)
+    std     %f4, 144(%r15)
+    std     %f6, 152(%r15)
+    aghi    %r15, -STACKSZ
+
+    lgrl    %r1, _ZN6__xray19XRayPatchedFunctionE@GOT
+    ltg     %r1, 0(%r1)
+    je      .Lrestore0
+
+    # Set r3 to XRayEntryType::ENTRY = 0.
+    # The FuncId is still stored in r2.
+    lghi    %r3, 0
+    basr    %r14, %r1
+
+.Lrestore0:
+    ld      %f6, STACKSZ+152(%r15)
+    ld      %f4, STACKSZ+144(%r15)
+    ld      %f2, STACKSZ+136(%r15)
+    ld      %f0, STACKSZ+128(%r15)
+    lmg     %r1, %r15, STACKSZ+8(%r15)
+    br      %r1
+.Lfunc_end0:
+    .size    __xray_FunctionEntry, .Lfunc_end0-__xray_FunctionEntry
+
+//===----------------------------------------------------------------------===//
+
+    .globl  __xray_FunctionEntryVec
+    .p2align    4
+    .type   __xray_FunctionEntryVec,@function
+__xray_FunctionEntryVec:
+    # The registers r2-15 of the instrumented function are already saved in the
+    # stack frame. On entry, r2 contains the function id, and %r14 the address
+    # of the first instruction of the instrumented function.
+    # Register r14 will be stored in the slot reserved for compiler use.
+    stg     %r14, 8(%r15)
+    std     %f0, 128(%r15)
+    std     %f2, 136(%r15)
+    std     %f4, 144(%r15)
+    std     %f6, 152(%r15)
+    aghi    %r15, -STACKSZ_VEC
+    vstm    %v24, %v31, 160(%r15)
+
+    lgrl    %r1, _ZN6__xray19XRayPatchedFunctionE@GOT
+    ltg     %r1, 0(%r1)
+    je      .Lrestore1
+
+    # Set r3 to XRayEntryType::ENTRY = 0.
+    # The FuncId is still stored in r2.
+    lghi    %r3, 0
+    basr    %r14, %r1
+
+.Lrestore1:
+    vlm     %v24, %v31, 160(%r15)
+    ld      %f6, STACKSZ_VEC+152(%r15)
+    ld      %f4, STACKSZ_VEC+144(%r15)
+    ld      %f2, STACKSZ_VEC+136(%r15)
+    ld      %f0, STACKSZ_VEC+128(%r15)
+    lmg     %r1, %r15, STACKSZ_VEC+8(%r15)
+    br      %r1
+.Lfunc_end1:
+    .size    __xray_FunctionEntryVec, .Lfunc_end1-__xray_FunctionEntryVec
+
+//===----------------------------------------------------------------------===//
+
+    .globl  __xray_FunctionExit
+    .p2align    4
+    .type   __xray_FunctionExit,@function
+__xray_FunctionExit:
+    # The registers r2-15 of the instrumented function are already saved in the
+    # stack frame. On entry, the register r2 contains the function id.
+    # At the end, the function jumps to the address saved in the slot for r14,
+    # which contains the return address into the caller of the instrumented
+    # function.
+    std     %f0, 128(%r15)
+    std     %f2, 136(%r15)
+    std     %f4, 144(%r15)
+    std     %f6, 152(%r15)
+    aghi    %r15, -STACKSZ
+
+    lgrl    %r1, _ZN6__xray19XRayPatchedFunctionE@GOT
+    ltg     %r1, 0(%r1)
+    je      .Lrestore2
+
+    # Set r3 to XRayEntryType::EXIT = 1.
+    # The FuncId is still stored in r2.
+    lghi    %r3, 1
+    basr    %r14, %r1
+
+.Lrestore2:
+    ld      %f6, STACKSZ+152(%r15)
+    ld      %f4, STACKSZ+144(%r15)
+    ld      %f2, STACKSZ+136(%r15)
+    ld      %f0, STACKSZ+128(%r15)
+    lmg     %r2, %r15, STACKSZ+16(%r15)
+    br      %r14
+.Lfunc_end2:
+    .size    __xray_FunctionExit, .Lfunc_end2-__xray_FunctionExit
+
+//===----------------------------------------------------------------------===//
+
+    .globl  __xray_FunctionExitVec
+    .p2align    4
+    .type   __xray_FunctionExitVec,@function
+__xray_FunctionExitVec:
+    # The registers r2-15 of the instrumented function are already saved in the
+    # stack frame. On entry, the register r2 contains the function id.
+    # At the end, the function jumps to the address saved in the slot for r14,
+    # which contains the return address into the caller of the instrumented
+    # function.
+    std     %f0, 128(%r15)
+    std     %f2, 136(%r15)
+    std     %f4, 144(%r15)
+    std     %f6, 152(%r15)
+    aghi    %r15, -STACKSZ_VEC
+    vstm    %v24, %v31, 160(%r15)
+
+    lgrl    %r1, _ZN6__xray19XRayPatchedFunctionE@GOT
+    ltg     %r1, 0(%r1)
+    je      .Lrestore3
+
+    # Set r3 to XRayEntryType::EXIT = 1.
+    # The FuncId is still stored in r2.
+    lghi    %r3, 1
+    basr    %r14, %r1
+
+.Lrestore3:
+    vlm     %v24, %v31, 160(%r15)
+    ld      %f6, STACKSZ_VEC+152(%r15)
+    ld      %f4, STACKSZ_VEC+144(%r15)
+    ld      %f2, STACKSZ_VEC+136(%r15)
+    ld      %f0, STACKSZ_VEC+128(%r15)
+    lmg     %r2, %r15, STACKSZ_VEC+16(%r15)
+    br      %r14
+.Lfunc_end3:
+    .size    __xray_FunctionExit, .Lfunc_end3-__xray_FunctionExit
+
+//===----------------------------------------------------------------------===//
+
+    .section    ".note.GNU-stack","",@progbits
diff --git a/compiler-rt/lib/xray/xray_tsc.h b/compiler-rt/lib/xray/xray_tsc.h
index b62a686d6ce0f..c8a8b2f16fef8 100644
--- a/compiler-rt/lib/xray/xray_tsc.h
+++ b/compiler-rt/lib/xray/xray_tsc.h
@@ -83,6 +83,35 @@ inline uint64_t getTSCFrequency() XRAY_NEVER_INSTRUMENT {
 
 } // namespace __xray
 
+#elif defined(__s390x__)
+#include "sanitizer_common/sanitizer_common.h"
+#include "sanitizer_common/sanitizer_internal_defs.h"
+#include "xray_defs.h"
+#include <cerrno>
+#include <cstdint>
+#include <time.h>
+
+namespace __xray {
+
+inline bool probeRequiredCPUFeatures() XRAY_NEVER_INSTRUMENT { return true; }
+
+ALWAYS_INLINE uint64_t readTSC(uint8_t &CPU) XRAY_NEVER_INSTRUMENT {
+  CPU = 0;
+#if __has_builtin(__builtin_readcyclecounter)
+  return __builtin_readcyclecounter();
+#else
+  uint64_t Cycles;
+  asm volatile("stckf %0" : : "Q"(Cycles) : "cc");
+  return Cycles;
+#endif
+}
+
+inline uint64_t getTSCFrequency() XRAY_NEVER_INSTRUMENT {
+  return NanosecondsPerSecond;
+}
+
+} // namespace __xray
+
 #else
 #error Target architecture is not supported.
 #endif // CPU architecture
diff --git a/compiler-rt/test/asan/TestCases/Windows/coverage-basic.cpp b/compiler-rt/test/asan/TestCases/Windows/coverage-basic.cpp
index 71291e8a53158..1b79016988260 100644
--- a/compiler-rt/test/asan/TestCases/Windows/coverage-basic.cpp
+++ b/compiler-rt/test/asan/TestCases/Windows/coverage-basic.cpp
@@ -4,6 +4,7 @@
 // RUN: %env_asan_opts=coverage=1 %run ./test.exe
 //
 // RUN: %sancov print *.sancov | FileCheck %s
+// XFAIL: msvc
 
 #include <stdio.h>
 
diff --git a/compiler-rt/test/asan/TestCases/Windows/crash_read_write.cpp b/compiler-rt/test/asan/TestCases/Windows/crash_read_write.cpp
index 74200cca1521b..bafc40c2c5665 100644
--- a/compiler-rt/test/asan/TestCases/Windows/crash_read_write.cpp
+++ b/compiler-rt/test/asan/TestCases/Windows/crash_read_write.cpp
@@ -2,12 +2,13 @@
 // RUN: not %run %t       2>&1 | FileCheck %s --check-prefix=READ
 // RUN: not %run %t write 2>&1 | FileCheck %s --check-prefix=WRITE
 
-#include <windows.h>
+#include "../defines.h"
 #include <stdio.h>
+#include <windows.h>
 
 static volatile int sink;
-__attribute__((noinline)) void Read(int *ptr) { sink = *ptr; }
-__attribute__((noinline)) void Write(int *ptr) { *ptr = 0; }
+ATTRIBUTE_NOINLINE void Read(int *ptr) { sink = *ptr; }
+ATTRIBUTE_NOINLINE void Write(int *ptr) { *ptr = 0; }
 int main(int argc, char **argv) {
   // Writes to shadow are detected as reads from shadow gap (because of how the
   // shadow mapping works). This is kinda hard to fix. Test a random address in
diff --git a/compiler-rt/test/asan/TestCases/Windows/dll_host.cpp b/compiler-rt/test/asan/TestCases/Windows/dll_host.cpp
index 85b7967e86b51..773482783823f 100644
--- a/compiler-rt/test/asan/TestCases/Windows/dll_host.cpp
+++ b/compiler-rt/test/asan/TestCases/Windows/dll_host.cpp
@@ -1,4 +1,5 @@
 // UNSUPPORTED: target={{.*-windows-gnu}}
+// XFAIL: msvc
 
 // This is a host program for DLL tests.
 //
diff --git a/compiler-rt/test/asan/TestCases/Windows/dll_intercept_memcpy.cpp b/compiler-rt/test/asan/TestCases/Windows/dll_intercept_memcpy.cpp
index 0a96752f9d0bf..4fa437960f126 100644
--- a/compiler-rt/test/asan/TestCases/Windows/dll_intercept_memcpy.cpp
+++ b/compiler-rt/test/asan/TestCases/Windows/dll_intercept_memcpy.cpp
@@ -1,9 +1,9 @@
 // RUN: %clang_cl_asan %Od %p/dll_host.cpp %Fe%t
-// RUN: %clang_cl_asan -Wno-fortify-source %LD %Od %s %Fe%t.dll
+// RUN: %clang_cl_asan %if !MSVC %{ -Wno-fortify-source %} %LD %Od %s %Fe%t.dll
 // RUN: not %run %t %t.dll 2>&1 | FileCheck %s
 
 // Test that it works correctly even with ICF enabled.
-// RUN: %clang_cl_asan -Wno-fortify-source %LD %Od %s %Fe%t.dll \
+// RUN: %clang_cl_asan %if !MSVC %{ -Wno-fortify-source %} %LD %Od %s %Fe%t.dll \
 // RUN:   %if target={{.*-windows-gnu}} %{ -Wl,--gc-sections,--icf=all %} \
 // RUN:   %else %{ -link /OPT:REF /OPT:ICF %}
 // RUN: not %run %t %t.dll 2>&1 | FileCheck %s
@@ -23,12 +23,12 @@ int test_function() {
 // CHECK: Initial test OK
 
   memcpy(buff2, buff1, 6);
-// CHECK: AddressSanitizer: stack-buffer-overflow on address [[ADDR:0x[0-9a-f]+]]
-// CHECK: WRITE of size 6 at [[ADDR]] thread T0
-// CHECK-NEXT:  __asan_{{.*}}memcpy
-// CHECK-NEXT:  test_function {{.*}}dll_intercept_memcpy.cpp:[[@LINE-4]]
-// CHECK: Address [[ADDR]] is located in stack of thread T0 at offset {{.*}} in frame
-// CHECK-NEXT:  test_function {{.*}}dll_intercept_memcpy.cpp
-// CHECK: 'buff2'{{.*}} <== Memory access at offset {{.*}} overflows this variable
+  // CHECK: AddressSanitizer: stack-buffer-overflow on address [[ADDR:0x[0-9a-f]+]]
+  // CHECK: WRITE of size 6 at [[ADDR]] thread T0
+  // CHECK-NEXT:  __asan_{{.*}}{{(memcpy|memmove)}}
+  // CHECK-NEXT:  test_function {{.*}}dll_intercept_memcpy.cpp:[[@LINE-4]]
+  // CHECK: Address [[ADDR]] is located in stack of thread T0 at offset {{.*}} in frame
+  // CHECK-NEXT:  test_function {{.*}}dll_intercept_memcpy.cpp
+  // CHECK: 'buff2'{{.*}} <== Memory access at offset {{.*}} overflows this variable
   return 0;
 }
diff --git a/compiler-rt/test/asan/TestCases/Windows/dll_intercept_memset.cpp b/compiler-rt/test/asan/TestCases/Windows/dll_intercept_memset.cpp
index 27b343f439cb5..948a429e8c1a6 100644
--- a/compiler-rt/test/asan/TestCases/Windows/dll_intercept_memset.cpp
+++ b/compiler-rt/test/asan/TestCases/Windows/dll_intercept_memset.cpp
@@ -1,9 +1,9 @@
-// RUN: %clang_cl_asan %Od %p/dll_host.cpp %Fe%t
-// RUN: %clang_cl_asan -Wno-fortify-source %LD %Od %s %Fe%t.dll
+// RUN: %clang_cl_asan %Od %Oy- %p/dll_host.cpp %Fe%t
+// RUN: %clang_cl_asan %if !MSVC %{ -Wno-fortify-source %} %Oy- %LD %Od %s %Fe%t.dll
 // RUN: not %run %t %t.dll 2>&1 | FileCheck %s
 
 // Test that it works correctly even with ICF enabled.
-// RUN: %clang_cl_asan -Wno-fortify-source %LD %Od %s %Fe%t.dll \
+// RUN: %clang_cl_asan %if !MSVC %{ -Wno-fortify-source %} %Oy- %LD %Od %s %Fe%t.dll \
 // RUN:   %if target={{.*-windows-gnu}} %{ -Wl,--gc-sections,--icf=all %} \
 // RUN:   %else %{ -link /OPT:REF /OPT:ICF %}
 // RUN: not %run %t %t.dll 2>&1 | FileCheck %s
diff --git a/compiler-rt/test/asan/TestCases/Windows/dll_null_deref.cpp b/compiler-rt/test/asan/TestCases/Windows/dll_null_deref.cpp
index b65926941cd78..d0d9a8ff0f516 100644
--- a/compiler-rt/test/asan/TestCases/Windows/dll_null_deref.cpp
+++ b/compiler-rt/test/asan/TestCases/Windows/dll_null_deref.cpp
@@ -2,7 +2,9 @@
 // RUN: %clang_cl_asan %LD %Od %s %Fe%t.dll
 // RUN: not %run %t %t.dll 2>&1 | FileCheck %s
 
-__attribute__((noinline))
+#include "../defines.h"
+
+ATTRIBUTE_NOINLINE
 static void NullDeref(int *ptr) {
   // CHECK: ERROR: AddressSanitizer: access-violation on unknown address
   // CHECK:   {{0x0*000.. .*pc 0x.*}}
diff --git a/compiler-rt/test/asan/TestCases/Windows/dll_operator_array_new_with_dtor_left_oob.cpp b/compiler-rt/test/asan/TestCases/Windows/dll_operator_array_new_with_dtor_left_oob.cpp
index ca6b256654a23..1d279f279d768 100644
--- a/compiler-rt/test/asan/TestCases/Windows/dll_operator_array_new_with_dtor_left_oob.cpp
+++ b/compiler-rt/test/asan/TestCases/Windows/dll_operator_array_new_with_dtor_left_oob.cpp
@@ -2,12 +2,20 @@
 // RUN: %clang_cl_asan %LD %Od %s %Fe%t.dll
 // RUN: not %run %t %t.dll 2>&1 | FileCheck %s
 
+#include "../defines.h"
+
 struct C {
   int x;
   ~C() {}
 };
 
-int __attribute__((noinline, optnone)) hide(int x) { return x; }
+int ATTRIBUTE_NOINLINE
+#if defined(__clang__) || !defined(_MSC_VER)
+    __attribute__((optnone))
+#endif
+    hide(int x) {
+  return x;
+}
 
 extern "C" __declspec(dllexport)
 int test_function() {
diff --git a/compiler-rt/test/asan/TestCases/Windows/dll_stack_use_after_return.cpp b/compiler-rt/test/asan/TestCases/Windows/dll_stack_use_after_return.cpp
index be237044bb340..3a09566eb873f 100644
--- a/compiler-rt/test/asan/TestCases/Windows/dll_stack_use_after_return.cpp
+++ b/compiler-rt/test/asan/TestCases/Windows/dll_stack_use_after_return.cpp
@@ -4,6 +4,8 @@
 // RUN: %clang_cl_asan %LD %Od %s %Fe%t.dll -fsanitize-address-use-after-return=always
 // RUN: not %run %t %t.dll 2>&1 | FileCheck %s
 
+// UNSUPPORTED: msvc
+
 #include <malloc.h>
 
 char *x;
diff --git a/compiler-rt/test/asan/TestCases/Windows/global_const_string_oob.cpp b/compiler-rt/test/asan/TestCases/Windows/global_const_string_oob.cpp
index 9788609773644..9615ccfc2aa0e 100644
--- a/compiler-rt/test/asan/TestCases/Windows/global_const_string_oob.cpp
+++ b/compiler-rt/test/asan/TestCases/Windows/global_const_string_oob.cpp
@@ -13,7 +13,7 @@ int main(void) {
   // CHECK: AddressSanitizer: global-buffer-overflow on address [[ADDR:0x[0-9a-f]+]]
   // CHECK: READ of size 1 at [[ADDR]] thread T0
   // CHECK-NEXT:   {{#0 .* main .*global_const_string_oob.cpp:}}[[@LINE-5]]
-  // CHECK: [[ADDR]] is located 5 bytes after global variable [[STR:.*]] defined in {{'.*global_const_string_oob.cpp' .*}} of size 11
+  // CHECK: [[ADDR]] is located 5 bytes after global variable [[STR:.*]] defined in {{'.*global_const_string_oob.cpp.*}} of size 11
   // CHECK:   [[STR]] is ascii string 'foobarspam'
   return 0;
 }
diff --git a/compiler-rt/test/asan/TestCases/Windows/illegal_instruction.cpp b/compiler-rt/test/asan/TestCases/Windows/illegal_instruction.cpp
index 9b902c7536ca9..5c71c38a5113b 100644
--- a/compiler-rt/test/asan/TestCases/Windows/illegal_instruction.cpp
+++ b/compiler-rt/test/asan/TestCases/Windows/illegal_instruction.cpp
@@ -1,5 +1,7 @@
 // RUN: %clang_cl_asan %Od %s %Fe%t
 // RUN: %env_asan_opts=handle_sigill=1 not %run %t 2>&1 | FileCheck %s
+// msvc doesn't have a __builtin_trap equivalent
+// XFAIL: msvc
 
 // Test the error output from an illegal instruction.
 
diff --git a/compiler-rt/test/asan/TestCases/Windows/issue64990.cpp b/compiler-rt/test/asan/TestCases/Windows/issue64990.cpp
index b1b6e42148cb6..5222ec6e08191 100644
--- a/compiler-rt/test/asan/TestCases/Windows/issue64990.cpp
+++ b/compiler-rt/test/asan/TestCases/Windows/issue64990.cpp
@@ -1,9 +1,13 @@
 // Repro for the issue #64990: Asan with Windows EH generates __asan_xxx runtime calls without required funclet tokens
-// RUN: %clang_cl_asan %Od %s -EHsc %Fe%t
+// RUN: %clang_cl_asan %Od %if MSVC %{ /Oi %} %s -EHsc %Fe%t
 // RUN: not %run %t 2>&1 | FileCheck %s
 
 // UNSUPPORTED: target={{.*-windows-gnu}}
 
+#if defined(_MSC_VER) && !defined(__clang__)
+#  include <string.h>
+#endif
+
 char buff1[6] = "hello";
 char buff2[6] = "hello";
 
@@ -12,7 +16,11 @@ int main(int argc, char **argv) {
     throw 1;
   } catch (...) {
     // Make asan generate call to __asan_memcpy inside the EH pad.
+#if defined(_MSC_VER) && !defined(__clang__)
+    memcpy(buff1, buff2 + 3, 6);
+#else
     __builtin_memcpy(buff1, buff2 + 3, 6);
+#endif
   }
   return 0;
 }
diff --git a/compiler-rt/test/asan/TestCases/Windows/msvc/seh.cpp b/compiler-rt/test/asan/TestCases/Windows/msvc/seh.cpp
index 4cb0c55bc7730..f3c7e1a33a568 100644
--- a/compiler-rt/test/asan/TestCases/Windows/msvc/seh.cpp
+++ b/compiler-rt/test/asan/TestCases/Windows/msvc/seh.cpp
@@ -2,15 +2,15 @@
 // different EH personality depending on the -GS setting, so test both -GS+ and
 // -GS-.
 //
-// RUN: cl -c %s -Fo%t.obj -DCOMPILE_SEH
-// RUN: %clangxx_asan -o %t.exe %s %t.obj
+// RUN: cl /EHa %MD -c %s -Fo%t.obj -DCOMPILE_SEH
+// RUN: %clangxx_asan -o %t.exe %s %t.obj -DCOMPILE_MAIN
 // RUN: %run %t.exe
 //
-// RUN: cl -GS- -c %s -Fo%t.obj -DCOMPILE_SEH
-// RUN: %clangxx_asan -o %t.exe %s %t.obj
+// RUN: cl /EHa %MD -GS- -c %s -Fo%t.obj -DCOMPILE_SEH
+// RUN: %clangxx_asan -o %t.exe %s %t.obj -DCOMPILE_MAIN
 // RUN: %run %t.exe
 //
-// RUN: %clang_cl_asan %s -DCOMPILE_SEH -Fe%t.exe
+// RUN: %clang_cl_asan /EHa %MD %s -DCOMPILE_SEH -Fe%t.exe -DCOMPILE_MAIN
 // RUN: %run %t.exe
 
 #include <windows.h>
@@ -42,7 +42,7 @@ void ThrowAndCatch() {
 }
 #endif
 
-#if defined(__clang__)
+#if defined(COMPILE_MAIN)
 int main() {
   char x[32];
   fprintf(stderr, "Before: %p poisoned: %d\n", &x,
diff --git a/compiler-rt/test/asan/TestCases/Windows/null_deref.cpp b/compiler-rt/test/asan/TestCases/Windows/null_deref.cpp
deleted file mode 100644
index 6582bac9e69e3..0000000000000
--- a/compiler-rt/test/asan/TestCases/Windows/null_deref.cpp
+++ /dev/null
@@ -1,15 +0,0 @@
-// RUN: %clangxx_asan -O0 %s -o %t && not %run %t 2>&1 | FileCheck %s
-// FIXME: merge this with the common null_deref test when we can run common
-// tests on Windows.
-
-__attribute__((noinline))
-static void NullDeref(int *ptr) {
-  // CHECK: ERROR: AddressSanitizer: access-violation on unknown address
-  // CHECK:   {{0x0*000.. .*pc 0x.*}}
-  ptr[10]++;  // BOOM
-}
-int main() {
-  NullDeref((int*)0);
-  // CHECK: {{    #1 0x.* in main.*null_deref.cpp:}}[[@LINE-1]]:3
-  // CHECK: AddressSanitizer can not provide additional info.
-}
diff --git a/compiler-rt/test/asan/TestCases/Windows/null_deref_multiple_dlls.cpp b/compiler-rt/test/asan/TestCases/Windows/null_deref_multiple_dlls.cpp
index ea5140a13dc88..19e3beefa4e0f 100644
--- a/compiler-rt/test/asan/TestCases/Windows/null_deref_multiple_dlls.cpp
+++ b/compiler-rt/test/asan/TestCases/Windows/null_deref_multiple_dlls.cpp
@@ -9,6 +9,7 @@
 // RUN: %clang_cl_asan %Od -DEXE %s %t1.lib %t2.lib %Fe%t
 // RUN: not %run %t 2>&1 | FileCheck %s
 
+#include "../defines.h"
 #include <malloc.h>
 #include <string.h>
 
@@ -28,7 +29,8 @@ __declspec(dllexport) void foo1() {}
 }
 #elif defined(DLL2)
 extern "C" {
-__attribute__((noinline)) static void NullDeref(int *ptr) {
+ATTRIBUTE_NOINLINE
+static void NullDeref(int *ptr) {
   // CHECK: ERROR: AddressSanitizer: access-violation on unknown address
   // CHECK:   {{0x0*000.. .*pc 0x.*}}
   ptr[10]++; // BOOM
diff --git a/compiler-rt/test/asan/TestCases/Windows/operator_array_new_with_dtor_left_oob.cpp b/compiler-rt/test/asan/TestCases/Windows/operator_array_new_with_dtor_left_oob.cpp
index cad28ae8ace21..c66b201500e36 100644
--- a/compiler-rt/test/asan/TestCases/Windows/operator_array_new_with_dtor_left_oob.cpp
+++ b/compiler-rt/test/asan/TestCases/Windows/operator_array_new_with_dtor_left_oob.cpp
@@ -1,13 +1,21 @@
 // RUN: %clang_cl_asan %Od %s %Fe%t
 // RUN: not %run %t 2>&1 | FileCheck %s
 
+#include "../defines.h"
+
 struct C {
   int x;
   ~C() {}
 };
-
-int __attribute__((noinline, optnone)) hide(int x) { return x; }
-
+#if defined(_MSC_VER) && !defined(__clang__)
+#  pragma optimize("", off)
+#else
+__attribute__((optnone))
+#endif
+int ATTRIBUTE_NOINLINE hide(int x) { return x; }
+#if defined(_MSC_VER) && !defined(__clang__)
+#  pragma optimize("", on)
+#endif
 int main() {
   C *buffer = new C[42];
   buffer[hide(-(1 + (int)sizeof(void*) / 4))].x = 42;
diff --git a/compiler-rt/test/asan/TestCases/Windows/sse_misalignment.cpp b/compiler-rt/test/asan/TestCases/Windows/sse_misalignment.cpp
index 1424ee88129f6..ee953cd6e4bfe 100644
--- a/compiler-rt/test/asan/TestCases/Windows/sse_misalignment.cpp
+++ b/compiler-rt/test/asan/TestCases/Windows/sse_misalignment.cpp
@@ -3,6 +3,7 @@
 
 // FIXME: On MinGW frame #0 does not include the line number?
 // XFAIL: target={{.*-windows-gnu}}
+// XFAIL: msvc
 
 // Test the error output from misaligned SSE2 memory access. This is a READ
 // memory access. Windows appears to always provide an address of -1 for these
diff --git a/compiler-rt/test/asan/TestCases/Windows/stack_use_after_return.cpp b/compiler-rt/test/asan/TestCases/Windows/stack_use_after_return.cpp
index 9385e36a37b67..0b33ff3340cf6 100644
--- a/compiler-rt/test/asan/TestCases/Windows/stack_use_after_return.cpp
+++ b/compiler-rt/test/asan/TestCases/Windows/stack_use_after_return.cpp
@@ -3,7 +3,7 @@
 
 // RUN: %clang_cl_asan %Od %s %Fe%t -fsanitize-address-use-after-return=always
 // RUN: not %run %t 2>&1 | FileCheck %s
-
+// XFAIL: msvc
 char *x;
 
 void foo() {
diff --git a/compiler-rt/test/asan/TestCases/Windows/unsymbolized.cpp b/compiler-rt/test/asan/TestCases/Windows/unsymbolized.cpp
index 00428b809fccd..ade5e5b914be8 100644
--- a/compiler-rt/test/asan/TestCases/Windows/unsymbolized.cpp
+++ b/compiler-rt/test/asan/TestCases/Windows/unsymbolized.cpp
@@ -8,9 +8,10 @@
 // RUN: not %run %t.exe 2>&1 | FileCheck %s
 // REQUIRES: lld-available
 
-#include <stdlib.h>
+#include "../defines.h"
 #include <stdio.h>
-int __attribute__((noinline)) do_uaf(void);
+#include <stdlib.h>
+int ATTRIBUTE_NOINLINE do_uaf(void);
 int main() {
   int r = do_uaf();
   printf("r: %d\n", r);
diff --git a/compiler-rt/test/asan/TestCases/alloca_big_alignment.cpp b/compiler-rt/test/asan/TestCases/alloca_big_alignment.cpp
index a451e873f2e42..5ef52e7b6e0dd 100644
--- a/compiler-rt/test/asan/TestCases/alloca_big_alignment.cpp
+++ b/compiler-rt/test/asan/TestCases/alloca_big_alignment.cpp
@@ -1,12 +1,14 @@
 // RUN: %clangxx_asan -O0 -mllvm -asan-instrument-dynamic-allocas %s -o %t
 // RUN: not %run %t 2>&1 | FileCheck %s
 //
+// XFAIL: msvc
 
+#include "defines.h"
 #include <assert.h>
 #include <stdint.h>
 
-__attribute__((noinline)) void foo(int index, int len) {
-  volatile char str[len] __attribute__((aligned(128)));
+ATTRIBUTE_NOINLINE void foo(int index, int len) {
+  volatile char str[len] ATTRIBUTE_ALIGNED(128);
   assert(!(reinterpret_cast<uintptr_t>(str) & 127L));
   str[index] = '1'; // BOOM
 // CHECK: ERROR: AddressSanitizer: dynamic-stack-buffer-overflow on address [[ADDR:0x[0-9a-f]+]]
diff --git a/compiler-rt/test/asan/TestCases/alloca_constant_size.cpp b/compiler-rt/test/asan/TestCases/alloca_constant_size.cpp
index 8910ea9f8d8e2..91c3ca296ac1e 100644
--- a/compiler-rt/test/asan/TestCases/alloca_constant_size.cpp
+++ b/compiler-rt/test/asan/TestCases/alloca_constant_size.cpp
@@ -7,6 +7,9 @@
 #include <stdio.h>
 #include <string.h>
 #include <stdlib.h>
+#ifdef _MSC_VER
+#  include <malloc.h>
+#endif
 
 // MSVC provides _alloca instead of alloca.
 #if defined(_MSC_VER) && !defined(alloca)
diff --git a/compiler-rt/test/asan/TestCases/alloca_detect_custom_size_.cpp b/compiler-rt/test/asan/TestCases/alloca_detect_custom_size_.cpp
index 8b207aa07f2eb..e8e73d46f8a73 100644
--- a/compiler-rt/test/asan/TestCases/alloca_detect_custom_size_.cpp
+++ b/compiler-rt/test/asan/TestCases/alloca_detect_custom_size_.cpp
@@ -2,16 +2,24 @@
 // RUN: not %run %t 2>&1 | FileCheck %s
 //
 
+#include "defines.h"
 #include <assert.h>
 #include <stdint.h>
+#if defined(_MSC_VER) && !defined(__clang__)
+#  include <malloc.h>
+#endif
 
 struct A {
   char a[3];
   int b[3];
 };
 
-__attribute__((noinline)) void foo(int index, int len) {
-  volatile struct A str[len] __attribute__((aligned(32)));
+ATTRIBUTE_NOINLINE void foo(int index, int len) {
+#if !defined(_MSC_VER) || defined(__clang__)
+  volatile struct A str[len] ATTRIBUTE_ALIGNED(32);
+#else
+  volatile struct A *str = (volatile struct A *)_alloca(len * sizeof(struct A));
+#endif
   assert(!(reinterpret_cast<uintptr_t>(str) & 31L));
   str[index].a[0] = '1'; // BOOM
 // CHECK: ERROR: AddressSanitizer: dynamic-stack-buffer-overflow on address [[ADDR:0x[0-9a-f]+]]
diff --git a/compiler-rt/test/asan/TestCases/alloca_instruments_all_paddings.cpp b/compiler-rt/test/asan/TestCases/alloca_instruments_all_paddings.cpp
index 912c8b0abaabd..4688079dfe6fb 100644
--- a/compiler-rt/test/asan/TestCases/alloca_instruments_all_paddings.cpp
+++ b/compiler-rt/test/asan/TestCases/alloca_instruments_all_paddings.cpp
@@ -2,6 +2,8 @@
 // RUN: %clangxx_asan -O3 -mllvm -asan-instrument-dynamic-allocas %s -o %t
 // RUN: %run %t 2>&1
 //
+// MSVC does not support asan-instrament-dynamic-allocas yet
+// UNSUPPORTED: msvc
 
 #include "sanitizer/asan_interface.h"
 #include <assert.h>
diff --git a/compiler-rt/test/asan/TestCases/alloca_loop_unpoisoning.cpp b/compiler-rt/test/asan/TestCases/alloca_loop_unpoisoning.cpp
index 0967b34dc7dbf..7e77d963c1e65 100644
--- a/compiler-rt/test/asan/TestCases/alloca_loop_unpoisoning.cpp
+++ b/compiler-rt/test/asan/TestCases/alloca_loop_unpoisoning.cpp
@@ -5,10 +5,14 @@
 
 // This testcase checks that allocas and VLAs inside loop are correctly unpoisoned.
 
+// MSVC doesn't support VLAs in the first place.
+// UNSUPPORTED: msvc
+
+#include "defines.h"
+#include "sanitizer/asan_interface.h"
 #include <assert.h>
 #include <stdint.h>
 #include <stdlib.h>
-#include "sanitizer/asan_interface.h"
 
 // MSVC provides _alloca instead of alloca.
 #if defined(_MSC_VER) && !defined(alloca)
@@ -21,7 +25,7 @@
 
 void *top, *bot;
 
-__attribute__((noinline)) void foo(int len) {
+ATTRIBUTE_NOINLINE void foo(int len) {
   char x;
   top = &x;
   volatile char array[len];
diff --git a/compiler-rt/test/asan/TestCases/alloca_overflow_partial.cpp b/compiler-rt/test/asan/TestCases/alloca_overflow_partial.cpp
index 25c6d75be7a53..f06195bb7793f 100644
--- a/compiler-rt/test/asan/TestCases/alloca_overflow_partial.cpp
+++ b/compiler-rt/test/asan/TestCases/alloca_overflow_partial.cpp
@@ -1,6 +1,8 @@
 // RUN: %clangxx_asan -O0 -mllvm -asan-instrument-dynamic-allocas %s -o %t
 // RUN: not %run %t 2>&1 | FileCheck %s
 //
+// MSVC doesn't support VLAs
+// UNSUPPORTED: msvc
 
 #include <assert.h>
 #include <stdint.h>
diff --git a/compiler-rt/test/asan/TestCases/alloca_overflow_right.cpp b/compiler-rt/test/asan/TestCases/alloca_overflow_right.cpp
index 7ec4b86cdc715..626c12488858d 100644
--- a/compiler-rt/test/asan/TestCases/alloca_overflow_right.cpp
+++ b/compiler-rt/test/asan/TestCases/alloca_overflow_right.cpp
@@ -1,6 +1,8 @@
 // RUN: %clangxx_asan -O0 -mllvm -asan-instrument-dynamic-allocas %s -o %t
 // RUN: not %run %t 2>&1 | FileCheck %s
 //
+// MSVC doesn't support VLAs
+// UNSUPPORTED: msvc
 
 #include <assert.h>
 #include <stdint.h>
diff --git a/compiler-rt/test/asan/TestCases/alloca_safe_access.cpp b/compiler-rt/test/asan/TestCases/alloca_safe_access.cpp
index 98e31769812bf..8979b6ca726ee 100644
--- a/compiler-rt/test/asan/TestCases/alloca_safe_access.cpp
+++ b/compiler-rt/test/asan/TestCases/alloca_safe_access.cpp
@@ -1,6 +1,8 @@
 // RUN: %clangxx_asan -O0 -mllvm -asan-instrument-dynamic-allocas %s -o %t
 // RUN: %run %t 2>&1
 //
+// MSVC doesn't support VLAs
+// UNSUPPORTED: msvc
 
 #include <assert.h>
 #include <stdint.h>
diff --git a/compiler-rt/test/asan/TestCases/alloca_underflow_left.cpp b/compiler-rt/test/asan/TestCases/alloca_underflow_left.cpp
index 52cd781c4bb88..532344319b019 100644
--- a/compiler-rt/test/asan/TestCases/alloca_underflow_left.cpp
+++ b/compiler-rt/test/asan/TestCases/alloca_underflow_left.cpp
@@ -1,6 +1,8 @@
 // RUN: %clangxx_asan -O0 -mllvm -asan-instrument-dynamic-allocas %s -o %t
 // RUN: not %run %t 2>&1 | FileCheck %s
 //
+// MSVC doesn't support VLAs
+// UNSUPPORTED: msvc
 
 #include <assert.h>
 #include <stdint.h>
diff --git a/compiler-rt/test/asan/TestCases/alloca_vla_interact.cpp b/compiler-rt/test/asan/TestCases/alloca_vla_interact.cpp
index b98bb726dcebc..e83812242f88b 100644
--- a/compiler-rt/test/asan/TestCases/alloca_vla_interact.cpp
+++ b/compiler-rt/test/asan/TestCases/alloca_vla_interact.cpp
@@ -8,6 +8,9 @@
 
 // This testcase checks correct interaction between VLAs and allocas.
 
+// MSVC doesn't support VLA's
+// UNSUPPORTED: msvc
+
 #include <assert.h>
 #include <stdint.h>
 #include <stdlib.h>
diff --git a/compiler-rt/test/asan/TestCases/contiguous_container.cpp b/compiler-rt/test/asan/TestCases/contiguous_container.cpp
index 6288e752fd4ba..a2adb1951dde6 100644
--- a/compiler-rt/test/asan/TestCases/contiguous_container.cpp
+++ b/compiler-rt/test/asan/TestCases/contiguous_container.cpp
@@ -2,6 +2,7 @@
 //
 // Test __sanitizer_annotate_contiguous_container.
 
+#include "defines.h"
 #include <algorithm>
 #include <numeric>
 #include <vector>
@@ -247,9 +248,9 @@ void TestDoubleEndedContainer(size_t capacity, size_t off_begin,
   delete[] buffer;
 }
 
-__attribute__((noinline)) void Throw() { throw 1; }
+ATTRIBUTE_NOINLINE void Throw() { throw 1; }
 
-__attribute__((noinline)) void ThrowAndCatch() {
+ATTRIBUTE_NOINLINE void ThrowAndCatch() {
   try {
     Throw();
   } catch (...) {
diff --git a/compiler-rt/test/asan/TestCases/coverage-trace-pc.cpp b/compiler-rt/test/asan/TestCases/coverage-trace-pc.cpp
index c03a6f02f7719..e55c3acce9af3 100644
--- a/compiler-rt/test/asan/TestCases/coverage-trace-pc.cpp
+++ b/compiler-rt/test/asan/TestCases/coverage-trace-pc.cpp
@@ -2,6 +2,8 @@
 // RUN: %clangxx_asan -O0 -DTRACE_RT %s -o %t-rt.o -c
 // RUN: %clangxx_asan -O0 -fsanitize-coverage=edge,trace-pc,indirect-calls %s -o %t %t-rt.o
 // RUN: %run %t
+// XFAIL: msvc
+
 #ifdef TRACE_RT
 int pc_count;
 void *last_callee;
@@ -12,13 +14,14 @@ extern "C" void __sanitizer_cov_trace_pc_indir(void *callee) {
   last_callee = callee;
 }
 #else
-#include <stdio.h>
-#include <assert.h>
+#  include "defines.h"
+#  include <assert.h>
+#  include <stdio.h>
 extern int pc_count;
 extern void *last_callee;
 
-__attribute__((noinline)) void foo() { printf("foo\n"); }
-__attribute__((noinline)) void bar() { printf("bar\n"); }
+ATTRIBUTE_NOINLINE void foo() { printf("foo\n"); }
+ATTRIBUTE_NOINLINE void bar() { printf("bar\n"); }
 
 int main(int argc, char **argv) {
   void (*f)(void) = argc ? foo : bar;
diff --git a/compiler-rt/test/asan/TestCases/debug_locate.cpp b/compiler-rt/test/asan/TestCases/debug_locate.cpp
index 93d1af8b83916..2ccddd4739d4a 100644
--- a/compiler-rt/test/asan/TestCases/debug_locate.cpp
+++ b/compiler-rt/test/asan/TestCases/debug_locate.cpp
@@ -3,7 +3,7 @@
 // that it correctly finds out which region (and name and size) the address
 // belongs to.
 // RUN: %clangxx_asan -O0 %s -o %t && %run %t 2>&1
-
+// UNSUPPORTED: msvc
 #include <assert.h>
 #include <sanitizer/asan_interface.h>
 #include <stdio.h>
diff --git a/compiler-rt/test/asan/TestCases/debug_stacks.cpp b/compiler-rt/test/asan/TestCases/debug_stacks.cpp
index 67a9ac849c3aa..391e241023462 100644
--- a/compiler-rt/test/asan/TestCases/debug_stacks.cpp
+++ b/compiler-rt/test/asan/TestCases/debug_stacks.cpp
@@ -1,6 +1,6 @@
 // Check that the stack trace debugging API works and returns correct
 // malloc and free stacks.
-// RUN: %clangxx_asan -O0 %s -o %t && not %run %t 2>&1 | FileCheck %s
+// RUN: %clangxx_asan -O0 %if MSVC %{ /wd4477 %} %s -o %t && not %run %t 2>&1 | FileCheck %s
 
 // FIXME: Figure out why allocation/free stack traces may be too short on ARM.
 // REQUIRES: stable-runtime
diff --git a/compiler-rt/test/asan/TestCases/deep_tail_call.cpp b/compiler-rt/test/asan/TestCases/deep_tail_call.cpp
index 628ef06db1447..772fa48bab7dc 100644
--- a/compiler-rt/test/asan/TestCases/deep_tail_call.cpp
+++ b/compiler-rt/test/asan/TestCases/deep_tail_call.cpp
@@ -4,15 +4,16 @@
 // RUN: %clangxx_asan -O3 %s -o %t && not %run %t 2>&1 | FileCheck %s
 
 // CHECK: AddressSanitizer: global-buffer-overflow
+#include "defines.h"
 int global[10];
 // CHECK: {{#0.*call4}}
-void __attribute__((noinline)) call4(int i) { global[i+10]++; }
+void ATTRIBUTE_NOINLINE call4(int i) { global[i + 10]++; }
 // CHECK: {{#1.*call3}}
-void __attribute__((noinline)) call3(int i) { call4(i); }
+void ATTRIBUTE_NOINLINE call3(int i) { call4(i); }
 // CHECK: {{#2.*call2}}
-void __attribute__((noinline)) call2(int i) { call3(i); }
+void ATTRIBUTE_NOINLINE call2(int i) { call3(i); }
 // CHECK: {{#3.*call1}}
-void __attribute__((noinline)) call1(int i) { call2(i); }
+void ATTRIBUTE_NOINLINE call1(int i) { call2(i); }
 // CHECK: {{#4.*main}}
 int main(int argc, char **argv) {
   call1(argc);
diff --git a/compiler-rt/test/asan/TestCases/default_ignorelist.cpp b/compiler-rt/test/asan/TestCases/default_ignorelist.cpp
index 8297b51e7680e..c1277394680f1 100644
--- a/compiler-rt/test/asan/TestCases/default_ignorelist.cpp
+++ b/compiler-rt/test/asan/TestCases/default_ignorelist.cpp
@@ -1,5 +1,5 @@
 // FIXME: https://code.google.com/p/address-sanitizer/issues/detail?id=316
-// XFAIL: android
+// XFAIL: android, msvc
 // UNSUPPORTED: ios
 //
 // Test that ASan uses the default ignorelist from resource directory.
diff --git a/compiler-rt/test/asan/TestCases/default_options.cpp b/compiler-rt/test/asan/TestCases/default_options.cpp
index 845e8a5f1793e..33507f3c82ef1 100644
--- a/compiler-rt/test/asan/TestCases/default_options.cpp
+++ b/compiler-rt/test/asan/TestCases/default_options.cpp
@@ -1,13 +1,14 @@
 // RUN: %clangxx_asan -O2 %s -o %t
 // RUN: %run %t 2>&1 | FileCheck %s
 
+#include "defines.h"
+
 const char *kAsanDefaultOptions = "verbosity=1 help=1";
 // Required for dyld macOS 12.0+
 #if (__APPLE__)
 __attribute__((weak))
 #endif
-__attribute__((no_sanitize_address))
-extern "C" const char *
+ATTRIBUTE_NO_SANITIZE_ADDRESS extern "C" const char *
 __asan_default_options() {
   // CHECK: Available flags for AddressSanitizer:
   return kAsanDefaultOptions;
diff --git a/compiler-rt/test/asan/TestCases/defines.h b/compiler-rt/test/asan/TestCases/defines.h
new file mode 100644
index 0000000000000..db6e374d2a031
--- /dev/null
+++ b/compiler-rt/test/asan/TestCases/defines.h
@@ -0,0 +1,34 @@
+#ifndef SANITIZER_TEST_DEFINES_H
+#define SANITIZER_TEST_DEFINES_H
+
+#if defined(_MSC_VER) && !defined(__clang__)
+#  include <intrin.h>
+
+#  define ATTRIBUTE_NOINLINE __declspec(noinline)
+#  define ATTRIBUTE_ALIGNED(x) __declspec(align(x))
+#  define ATTRIBUTE_NO_SANITIZE_ADDRESS __declspec(no_sanitize_address)
+#  define ATTRIBUTE_USED /* FIXME: Is there a __declspec used? */
+#  define ATTRIBUTE_ALWAYS_INLINE __forceinline
+#  define VOLATILE volatile
+#  define EXTRACT_RETURN_ADDRESS _ReturnAddress()
+#  define ASM_CAUSE_SIDE_EFFECT(dest) __asm { mov eax, dest}
+#  define MULTIPLE_ATTRIBUTE_DECL(a, b) __declspec(a b)
+
+#else
+
+#  define ATTRIBUTE_NOINLINE __attribute__((noinline))
+#  define ATTRIBUTE_ALIGNED(x) __attribute__((aligned(x)))
+#  define ATTRIBUTE_NO_SANITIZE_ADDRESS __attribute__((no_sanitize_address))
+#  define ATTRIBUTE_USED __attribute__((used))
+#  define ATTRIBUTE_ALWAYS_INLINE __attribute__((always_inline))
+#  define INLINE_ASM(x) __asm__(x)
+#  define VOLATILE __volatile__
+#  define EXTRACT_RETURN_ADDRESS                                               \
+    __builtin_extract_return_addr(__builtin_return_address(0))
+#  define ASM_CAUSE_SIDE_EFFECT(dest)                                          \
+    __asm__ __volatile__("" : : "r"(dest) : "memory");
+#  define MULTIPLE_ATTRIBUTE_DECL(a, b) __attribute__((a, b))
+
+#endif // _MSC_VER
+
+#endif // SANITIZER_TEST_DEFINES
diff --git a/compiler-rt/test/asan/TestCases/error_report_callback.cpp b/compiler-rt/test/asan/TestCases/error_report_callback.cpp
index 8c5bbe418fc52..903947e729fb2 100644
--- a/compiler-rt/test/asan/TestCases/error_report_callback.cpp
+++ b/compiler-rt/test/asan/TestCases/error_report_callback.cpp
@@ -1,6 +1,7 @@
 // RUN: %clangxx_asan -O0 %s -o %t
 // RUN: not %run %t 0 2>&1 | FileCheck %s
 
+#include "defines.h"
 #include <sanitizer/asan_interface.h>
 #include <stdio.h>
 
@@ -11,9 +12,7 @@ static void ErrorReportCallbackOneToZ(const char *report) {
 
 int main(int argc, char **argv) {
   __asan_set_error_report_callback(ErrorReportCallbackOneToZ);
-  __asan_report_error(
-      (void *)__builtin_extract_return_addr(__builtin_return_address(0)), 0, 0,
-      0, true, 1);
+  __asan_report_error((void *)EXTRACT_RETURN_ADDRESS, 0, 0, 0, true, 1);
   // CHECK: ABCDEF
   // CHECK: ERROR: AddressSanitizer
   // CHECK: GHIJKL
diff --git a/compiler-rt/test/asan/TestCases/exitcode.cpp b/compiler-rt/test/asan/TestCases/exitcode.cpp
index e949436e1909c..21743bfbd7b8d 100644
--- a/compiler-rt/test/asan/TestCases/exitcode.cpp
+++ b/compiler-rt/test/asan/TestCases/exitcode.cpp
@@ -1,4 +1,4 @@
-// RUN: %clangxx_asan -g -Wno-deprecated-declarations %s -o %t
+// RUN: %clangxx_asan -g %if !MSVC %{ -Wno-deprecated-declarations %} %s -o %t
 // RUN: %env_asan_opts=exitcode=42 %run %t | FileCheck %s
 
 // Android doesn't have spawn.h or posix_spawn.
diff --git a/compiler-rt/test/asan/TestCases/force_inline_opt0.cpp b/compiler-rt/test/asan/TestCases/force_inline_opt0.cpp
index e6e5d26c7998e..0c721bcc5c809 100644
--- a/compiler-rt/test/asan/TestCases/force_inline_opt0.cpp
+++ b/compiler-rt/test/asan/TestCases/force_inline_opt0.cpp
@@ -1,8 +1,11 @@
 // This test checks that we are no instrumenting a memory access twice
 // (before and after inlining)
-// RUN: %clangxx_asan -O1 %s -o %t && %run %t
-// RUN: %clangxx_asan -O0 %s -o %t && %run %t
-__attribute__((always_inline))
+// RUN: %clangxx_asan -O1 %if MSVC %{ /Ob1 %} %s -o %t && %run %t
+// RUN: %clangxx_asan -O0 %if MSVC %{ /Ob1 %} %s -o %t && %run %t
+
+#include "defines.h"
+
+ATTRIBUTE_ALWAYS_INLINE
 void foo(int *x) {
   *x = 0;
 }
diff --git a/compiler-rt/test/asan/TestCases/global-location-nodebug.cpp b/compiler-rt/test/asan/TestCases/global-location-nodebug.cpp
index 07c8e63f439da..199816b8036ab 100644
--- a/compiler-rt/test/asan/TestCases/global-location-nodebug.cpp
+++ b/compiler-rt/test/asan/TestCases/global-location-nodebug.cpp
@@ -11,7 +11,7 @@
 /// Solaris ld -S has different semantics, so enforce -fuse-ld= for
 /// configurations that default to GNU ld.
 // XFAIL: target={{.*solaris.*}}
-
+// XFAIL: msvc
 // CHECK: AddressSanitizer: global-buffer-overflow
 // CLASS_STATIC-NO-G: 0x{{.*}} is located 4 bytes after global variable '{{.*}}C::array{{.*}}' defined in '{{.*}}global-location.cpp' {{.*}} of size 40
 // GLOB-NO-G: 0x{{.*}} is located 4 bytes after global variable '{{.*}}global{{.*}}' defined in '{{.*}}global-location.cpp' {{.*}} of size 40
diff --git a/compiler-rt/test/asan/TestCases/global-underflow.cpp b/compiler-rt/test/asan/TestCases/global-underflow.cpp
index 32b4ce82f9b04..fc9b14628aece 100644
--- a/compiler-rt/test/asan/TestCases/global-underflow.cpp
+++ b/compiler-rt/test/asan/TestCases/global-underflow.cpp
@@ -1,3 +1,4 @@
+// XFAIL: msvc
 // RUN: %clangxx_asan -O0 %s %p/Helpers/underflow.cpp -o %t && not %run %t 2>&1 | FileCheck %s
 // RUN: %clangxx_asan -O1 %s %p/Helpers/underflow.cpp -o %t && not %run %t 2>&1 | FileCheck %s
 // RUN: %clangxx_asan -O2 %s %p/Helpers/underflow.cpp -o %t && not %run %t 2>&1 | FileCheck %s
diff --git a/compiler-rt/test/asan/TestCases/halt_on_error-1.c b/compiler-rt/test/asan/TestCases/halt_on_error-1.c
index 63c65e58bb71b..d40a281987133 100644
--- a/compiler-rt/test/asan/TestCases/halt_on_error-1.c
+++ b/compiler-rt/test/asan/TestCases/halt_on_error-1.c
@@ -5,7 +5,7 @@
 // RUN: env not %run %t 2>&1 | FileCheck %s
 // RUN: %env_asan_opts=halt_on_error=true not %run %t 2>&1 | FileCheck %s
 // RUN: %env_asan_opts=halt_on_error=false %run %t 2>&1 | FileCheck %s --check-prefix CHECK-RECOVER
-
+// XFAIL: msvc
 #include <string.h>
 
 volatile int ten = 10;
diff --git a/compiler-rt/test/asan/TestCases/heavy_uar_test.cpp b/compiler-rt/test/asan/TestCases/heavy_uar_test.cpp
index 661867260d260..200d19cdbaf8a 100644
--- a/compiler-rt/test/asan/TestCases/heavy_uar_test.cpp
+++ b/compiler-rt/test/asan/TestCases/heavy_uar_test.cpp
@@ -15,29 +15,29 @@
 
 // UNSUPPORTED: ios
 
+#include "defines.h"
 #include <stdio.h>
-#include <string.h>
 #include <stdlib.h>
+#include <string.h>
 #ifdef _WIN32
 #  include <windows.h>
 #endif
 
-__attribute__((noinline))
+ATTRIBUTE_NOINLINE
 char *pretend_to_do_something(char *x) {
   __asm__ __volatile__("" : : "r" (x) : "memory");
   return x;
 }
 
-__attribute__((noinline))
+ATTRIBUTE_NOINLINE
 char *LeakStack() {
   char x[1024];
   memset(x, 0, sizeof(x));
   return pretend_to_do_something(x);
 }
 
-template<size_t kFrameSize>
-__attribute__((noinline))
-void RecursiveFunctionWithStackFrame(int depth) {
+template <size_t kFrameSize>
+ATTRIBUTE_NOINLINE void RecursiveFunctionWithStackFrame(int depth) {
   if (depth <= 0) return;
   char x[kFrameSize];
   x[0] = depth;
diff --git a/compiler-rt/test/asan/TestCases/ignorelist.cpp b/compiler-rt/test/asan/TestCases/ignorelist.cpp
index 348ea5d350bf1..5fe728bfe9c21 100644
--- a/compiler-rt/test/asan/TestCases/ignorelist.cpp
+++ b/compiler-rt/test/asan/TestCases/ignorelist.cpp
@@ -15,7 +15,10 @@
 // badGlobal is accessed improperly, but we ignorelisted it. Align
 // it to make sure memory past the end of badGlobal will be in
 // the same page.
-__attribute__((aligned(16))) int badGlobal;
+
+// XFAIL: msvc
+#include "defines.h"
+ATTRIBUTE_ALIGNED(16) int badGlobal;
 int readBadGlobal() {
   return (&badGlobal)[1];
 }
diff --git a/compiler-rt/test/asan/TestCases/ill.cpp b/compiler-rt/test/asan/TestCases/ill.cpp
index d7b5350916719..da457d6737dbd 100644
--- a/compiler-rt/test/asan/TestCases/ill.cpp
+++ b/compiler-rt/test/asan/TestCases/ill.cpp
@@ -2,12 +2,18 @@
 //
 // RUN: %clangxx_asan %s -o %t && %env_asan_opts=handle_sigill=0 not --crash %run %t 2>&1 | FileCheck %s --check-prefix=CHECK0
 // RUN: %clangxx_asan %s -o %t && %env_asan_opts=handle_sigill=1 not %run %t 2>&1 | FileCheck %s --check-prefix=CHECK1
-// REQUIRES: x86-target-arch
+// REQUIRES: x86-target-arch && (!MSVC || asan-32-bits)
 
 #ifdef _WIN32
 #include <windows.h>
 #endif
 
+// note: test is limited to i386 only ("asan-32-bits") when using "real" MSVC
+//  see the requires clause above
+#if defined(_MSC_VER) && !defined(__clang__)
+#  define __builtin_trap() __asm ud2;
+#endif
+
 int main(int argc, char **argv) {
 #ifdef _WIN32
   // Sometimes on Windows this test generates a WER fault dialog. Suppress that.
diff --git a/compiler-rt/test/asan/TestCases/initialization-bug.cpp b/compiler-rt/test/asan/TestCases/initialization-bug.cpp
index 2775f6ce356dd..1af6e256f0c24 100644
--- a/compiler-rt/test/asan/TestCases/initialization-bug.cpp
+++ b/compiler-rt/test/asan/TestCases/initialization-bug.cpp
@@ -8,6 +8,7 @@
 // FIXME: https://code.google.com/p/address-sanitizer/issues/detail?id=186
 // XFAIL: target={{.*windows-msvc.*}}
 
+#include "defines.h"
 #include <cstdio>
 
 // The structure of the test is:
@@ -27,7 +28,7 @@ int z = initZ();
 // result is undefined behavior, which should be caught by initialization order
 // checking.
 extern int y;
-int __attribute__((noinline)) initX() {
+int ATTRIBUTE_NOINLINE initX() {
   return y + 1;
   // CHECK: {{AddressSanitizer: initialization-order-fiasco}}
   // CHECK: {{READ of size .* at 0x.* thread T0}}
diff --git a/compiler-rt/test/asan/TestCases/inline.cpp b/compiler-rt/test/asan/TestCases/inline.cpp
index 12bd27e675844..42779a07ae2e6 100644
--- a/compiler-rt/test/asan/TestCases/inline.cpp
+++ b/compiler-rt/test/asan/TestCases/inline.cpp
@@ -2,10 +2,15 @@
 
 // Test that no_sanitize_address attribute applies even when the function would
 // be normally inlined.
+//
+// MSVC doesn't apply __declspec(no_sanitize_address) to inlined functions
+//    (i.e. it contains this bug)
+// XFAIL: msvc
 
+#include "defines.h"
 #include <stdlib.h>
 
-__attribute__((no_sanitize_address))
+ATTRIBUTE_NO_SANITIZE_ADDRESS
 int f(int *p) {
   return *p; // BOOOM?? Nope!
 }
diff --git a/compiler-rt/test/asan/TestCases/intercept-rethrow-exception.cpp b/compiler-rt/test/asan/TestCases/intercept-rethrow-exception.cpp
index bf51eed41fddf..24b93c56e730c 100644
--- a/compiler-rt/test/asan/TestCases/intercept-rethrow-exception.cpp
+++ b/compiler-rt/test/asan/TestCases/intercept-rethrow-exception.cpp
@@ -15,6 +15,7 @@
 // https://reviews.llvm.org/D111703 made compiler incompatible with released NDK.
 // UNSUPPORTED: android && arm-target-arch
 
+#include "defines.h"
 #include <assert.h>
 #include <exception>
 #include <sanitizer/asan_interface.h>
@@ -28,7 +29,7 @@ namespace {
 // [[noreturn]] because the scenario we're emulating doesn't always throw. If it
 // were [[noreturn]], the calling code would emit a call to
 // __asan_handle_no_return.
-void __attribute__((no_sanitize("address")))
+void ATTRIBUTE_NO_SANITIZE_ADDRESS
 uninstrumented_rethrow_exception(std::exception_ptr const &exc_ptr) {
   std::rethrow_exception(exc_ptr);
 }
diff --git a/compiler-rt/test/asan/TestCases/interception_failure_test.cpp b/compiler-rt/test/asan/TestCases/interception_failure_test.cpp
index 918fa8e82bf74..bdaf3eb3178f2 100644
--- a/compiler-rt/test/asan/TestCases/interception_failure_test.cpp
+++ b/compiler-rt/test/asan/TestCases/interception_failure_test.cpp
@@ -18,6 +18,10 @@
 // it works with the dynamic runtime.
 // XFAIL: target={{.*netbsd.*}} && !asan-dynamic-runtime
 
+#if defined(_MSC_VER) && !defined(__clang__)
+#  pragma warning(disable : 4273)
+#endif
+
 #include <stdlib.h>
 #include <stdio.h>
 #include <string.h>
diff --git a/compiler-rt/test/asan/TestCases/interface_test.cpp b/compiler-rt/test/asan/TestCases/interface_test.cpp
index 9419f07e91eae..bc347807bcc3a 100644
--- a/compiler-rt/test/asan/TestCases/interface_test.cpp
+++ b/compiler-rt/test/asan/TestCases/interface_test.cpp
@@ -1,8 +1,8 @@
 // Check that user may include ASan interface header.
 // RUN: %clang_asan %s -o %t && %run %t
 // RUN: %clang_asan -x c %s -o %t && %run %t
-// RUN: %clang %s -pie -o %t && %run %t
-// RUN: %clang -x c %s -pie -o %t && %run %t
+// RUN: %clang %s %pie -o %t && %run %t
+// RUN: %clang -x c %s %pie -o %t && %run %t
 #include <sanitizer/asan_interface.h>
 
 int main() {
diff --git a/compiler-rt/test/asan/TestCases/invalid-pointer-pairs-compare-errors.cpp b/compiler-rt/test/asan/TestCases/invalid-pointer-pairs-compare-errors.cpp
index 7b1479e2d05bd..84c264b305692 100644
--- a/compiler-rt/test/asan/TestCases/invalid-pointer-pairs-compare-errors.cpp
+++ b/compiler-rt/test/asan/TestCases/invalid-pointer-pairs-compare-errors.cpp
@@ -1,7 +1,7 @@
 // RUN: %clangxx_asan -O0 %s -o %t -mllvm -asan-detect-invalid-pointer-pair
 
 // RUN: %env_asan_opts=detect_invalid_pointer_pairs=2:halt_on_error=0 %run %t 2>&1 | FileCheck %s
-
+// XFAIL: msvc
 #include <assert.h>
 #include <stdlib.h>
 
diff --git a/compiler-rt/test/asan/TestCases/invalid-pointer-pairs-subtract-errors.cpp b/compiler-rt/test/asan/TestCases/invalid-pointer-pairs-subtract-errors.cpp
index 535833d05c89a..80742cddc0460 100644
--- a/compiler-rt/test/asan/TestCases/invalid-pointer-pairs-subtract-errors.cpp
+++ b/compiler-rt/test/asan/TestCases/invalid-pointer-pairs-subtract-errors.cpp
@@ -1,7 +1,7 @@
 // RUN: %clangxx_asan -O0 %s -o %t -mllvm -asan-detect-invalid-pointer-pair
 
 // RUN: %env_asan_opts=detect_invalid_pointer_pairs=2:halt_on_error=0 %run %t 2>&1 | FileCheck %s
-
+// XFAIL: msvc
 #include <assert.h>
 #include <stdlib.h>
 
diff --git a/compiler-rt/test/asan/TestCases/invalid-pointer-pairs.cpp b/compiler-rt/test/asan/TestCases/invalid-pointer-pairs.cpp
index accd9b7704d51..061a0a7b2b861 100644
--- a/compiler-rt/test/asan/TestCases/invalid-pointer-pairs.cpp
+++ b/compiler-rt/test/asan/TestCases/invalid-pointer-pairs.cpp
@@ -4,7 +4,7 @@
 // RUN: %env_asan_opts=detect_invalid_pointer_pairs=1 not %run %t g 2>&1 | FileCheck %s -check-prefix=CMP -check-prefix=ALL-ERRORS
 // RUN: %env_asan_opts=detect_invalid_pointer_pairs=1 not %run %t s 2>&1 | FileCheck %s -check-prefix=SUB -check-prefix=ALL-ERRORS
 // RUN: %env_asan_opts=detect_invalid_pointer_pairs=1 not %run %t f 2>&1 | FileCheck %s -check-prefix=FREE -check-prefix=ALL-ERRORS
-
+// XFAIL: msvc
 #include <assert.h>
 #include <stdlib.h>
 
diff --git a/compiler-rt/test/asan/TestCases/large_func_test.cpp b/compiler-rt/test/asan/TestCases/large_func_test.cpp
index c64fc7d3c7aa7..1c25d345cb5c4 100644
--- a/compiler-rt/test/asan/TestCases/large_func_test.cpp
+++ b/compiler-rt/test/asan/TestCases/large_func_test.cpp
@@ -7,8 +7,9 @@
 // Issue #108194: Incomplete .debug_line at -O1 and above.
 // XFAIL: target={{.*sparc.*}}
 
+#include "defines.h"
 #include <stdlib.h>
-__attribute__((noinline))
+ATTRIBUTE_NOINLINE
 static void LargeFunction(int *x, int zero) {
   x[0]++;
   x[1]++;
@@ -29,7 +30,7 @@ static void LargeFunction(int *x, int zero) {
   // Darwin.
   // CHECK-Linux:  {{#0 0x.* in LargeFunction.*large_func_test.cpp:}}[[@LINE-3]]
   // CHECK-SunOS:  {{#0 0x.* in LargeFunction.*large_func_test.cpp:}}[[@LINE-4]]
-  // CHECK-Windows:{{#0 0x.* in LargeFunction.*large_func_test.cpp:}}[[@LINE-5]]
+  // CHECK-Windows:{{#[0-1] 0x.* in LargeFunction.*large_func_test.cpp:}}[[@LINE-5]]
   // CHECK-FreeBSD:{{#0 0x.* in LargeFunction.*large_func_test.cpp:}}[[@LINE-6]]
   // CHECK-Darwin: {{#0 0x.* in .*LargeFunction.*large_func_test.cpp}}:[[@LINE-7]]
 
@@ -48,7 +49,7 @@ static void LargeFunction(int *x, int zero) {
 int main(int argc, char **argv) {
   int *x = new int[100];
   LargeFunction(x, argc - 1);
-  // CHECK: {{    #1 0x.* in main .*large_func_test.cpp:}}[[@LINE-1]]
+  // CHECK: {{    #[1-2] 0x.* in main .*large_func_test.cpp:}}[[@LINE-1]]
   // CHECK: {{0x.* is located 12 bytes after 400-byte region}}
   // CHECK: {{allocated by thread T0 here:}}
   // CHECK-Linux:  {{    #0 0x.* in operator new}}
diff --git a/compiler-rt/test/asan/TestCases/null_deref.cpp b/compiler-rt/test/asan/TestCases/null_deref.cpp
index a8947a4378032..3ccb475326369 100644
--- a/compiler-rt/test/asan/TestCases/null_deref.cpp
+++ b/compiler-rt/test/asan/TestCases/null_deref.cpp
@@ -3,7 +3,9 @@
 // RUN: %clangxx_asan -O2 %s -o %t && not %run %t 2>&1 | FileCheck %s
 // RUN: %clangxx_asan -O3 %s -o %t && not %run %t 2>&1 | FileCheck %s
 
-__attribute__((noinline))
+#include "defines.h"
+
+ATTRIBUTE_NOINLINE
 // FIXME: Static symbols don't show up in PDBs. We can remove this once we start
 // using DWARF.
 #ifndef _MSC_VER
diff --git a/compiler-rt/test/asan/TestCases/pass-struct-byval.cpp b/compiler-rt/test/asan/TestCases/pass-struct-byval.cpp
index ba49eccf41cab..69409bc6ace6b 100644
--- a/compiler-rt/test/asan/TestCases/pass-struct-byval.cpp
+++ b/compiler-rt/test/asan/TestCases/pass-struct-byval.cpp
@@ -1,6 +1,6 @@
 // RUN: %clangxx_asan -O0 %s -o %t
 // RUN: not %run %t 2>&1 | FileCheck %s
-
+// XFAIL: msvc
 struct A {
   int a[8];
 };
diff --git a/compiler-rt/test/asan/TestCases/report_error_summary.cpp b/compiler-rt/test/asan/TestCases/report_error_summary.cpp
index 9e024e35bed86..e10def7218634 100644
--- a/compiler-rt/test/asan/TestCases/report_error_summary.cpp
+++ b/compiler-rt/test/asan/TestCases/report_error_summary.cpp
@@ -2,8 +2,10 @@
 
 #include <stdio.h>
 
+#if !defined(_MSC_VER) || defined(__clang__)
 // Required for ld64 macOS 12.0+
 __attribute__((weak)) extern "C" void foo() {}
+#endif
 
 extern "C" void __sanitizer_report_error_summary(const char *summary) {
   fprintf(stderr, "test_report_error_summary\n");
diff --git a/compiler-rt/test/asan/TestCases/speculative_load.cpp b/compiler-rt/test/asan/TestCases/speculative_load.cpp
index fdf70eb39767a..27bd293d28a39 100644
--- a/compiler-rt/test/asan/TestCases/speculative_load.cpp
+++ b/compiler-rt/test/asan/TestCases/speculative_load.cpp
@@ -4,6 +4,9 @@
 // RUN: %clangxx_asan -O2 %s -o %t && %run %t 2>&1
 // RUN: %clangxx_asan -O3 %s -o %t && %run %t 2>&1
 
+// MSVC doesn't support GCC style inline assembly
+// UNSUPPORTED: msvc
+
 #include <sanitizer/asan_interface.h>
 
 struct S {
diff --git a/compiler-rt/test/asan/TestCases/stack-buffer-overflow-with-position.cpp b/compiler-rt/test/asan/TestCases/stack-buffer-overflow-with-position.cpp
index 0077663d6a176..d33c6246fd4fb 100644
--- a/compiler-rt/test/asan/TestCases/stack-buffer-overflow-with-position.cpp
+++ b/compiler-rt/test/asan/TestCases/stack-buffer-overflow-with-position.cpp
@@ -13,6 +13,7 @@
 // RUN: not %run %t 63 2>&1 | FileCheck --check-prefix=CHECK-63 %s
 // RUN: not %run %t 73 2>&1 | FileCheck --check-prefix=CHECK-73 %s
 // RUN: not %run %t 74 2>&1 | FileCheck --check-prefix=CHECK-74 %s
+// XFAIL: msvc
 #include <string.h>
 #include <stdio.h>
 #include <stdlib.h>
diff --git a/compiler-rt/test/asan/TestCases/stack-oob-frames.cpp b/compiler-rt/test/asan/TestCases/stack-oob-frames.cpp
index 3b5d511b26818..1e077513c2168 100644
--- a/compiler-rt/test/asan/TestCases/stack-oob-frames.cpp
+++ b/compiler-rt/test/asan/TestCases/stack-oob-frames.cpp
@@ -4,6 +4,9 @@
 // RUN: not %run %t 2 2>&1 | FileCheck %s --check-prefix=CHECK2
 // RUN: not %run %t 3 2>&1 | FileCheck %s --check-prefix=CHECK3
 
+// MSVC doesn't support GCC style inline ASM
+// UNSUPPORTED: msvc
+
 #define NOINLINE __attribute__((noinline))
 inline void break_optimization(void *arg) {
   __asm__ __volatile__("" : : "r" (arg) : "memory");
diff --git a/compiler-rt/test/asan/TestCases/strcat-overlap.cpp b/compiler-rt/test/asan/TestCases/strcat-overlap.cpp
index 76fd3e1f11d14..af78bc4440549 100644
--- a/compiler-rt/test/asan/TestCases/strcat-overlap.cpp
+++ b/compiler-rt/test/asan/TestCases/strcat-overlap.cpp
@@ -34,11 +34,12 @@
 // UNSUPPORTED: target={{.*windows-msvc.*}}
 // UNSUPPORTED: android
 
+#include "defines.h"
 #include <string.h>
 
 
 // Don't inline function otherwise stacktrace changes.
-__attribute__((noinline)) void bad_function() {
+ATTRIBUTE_NOINLINE void bad_function() {
   char buffer[] = "hello\0XXX";
   // CHECK: strcat-param-overlap: memory ranges
   // CHECK: [{{0x.*,[ ]*0x.*}}) and [{{0x.*,[ ]*0x.*}}) overlap
diff --git a/compiler-rt/test/asan/TestCases/strcpy-overlap.cpp b/compiler-rt/test/asan/TestCases/strcpy-overlap.cpp
index efd2e6b7521ef..89ac7a39b33ca 100644
--- a/compiler-rt/test/asan/TestCases/strcpy-overlap.cpp
+++ b/compiler-rt/test/asan/TestCases/strcpy-overlap.cpp
@@ -26,13 +26,14 @@
 // RUN: echo "interceptor_name:strcpy" > %t.supp
 // RUN: %env_asan_opts=suppressions='"%t.supp"' %run %t
 
-// UNSUPPORTED: android
+// UNSUPPORTED: android, MSVC
 
+#include "defines.h"
 #include <string.h>
 
 
 // Don't inline function otherwise stacktrace changes.
-__attribute__((noinline)) void bad_function() {
+ATTRIBUTE_NOINLINE void bad_function() {
   char buffer[] = "hello";
   // CHECK: strcpy-param-overlap: memory ranges
   // CHECK: [{{0x.*,[ ]*0x.*}}) and [{{0x.*,[ ]*0x.*}}) overlap
diff --git a/compiler-rt/test/asan/TestCases/strncat-overlap.cpp b/compiler-rt/test/asan/TestCases/strncat-overlap.cpp
index 3e3f7ee2723f5..e4f2fbca68abf 100644
--- a/compiler-rt/test/asan/TestCases/strncat-overlap.cpp
+++ b/compiler-rt/test/asan/TestCases/strncat-overlap.cpp
@@ -28,11 +28,12 @@
 
 // UNSUPPORTED: android
 
+#include "defines.h"
 #include <string.h>
 
 
 // Don't inline function otherwise stacktrace changes.
-__attribute__((noinline)) void bad_function() {
+ATTRIBUTE_NOINLINE void bad_function() {
   char buffer[] = "hello\0XXX";
   // CHECK: strncat-param-overlap: memory ranges
   // CHECK: [{{0x.*,[ ]*0x.*}}) and [{{0x.*,[ ]*0x.*}}) overlap
diff --git a/compiler-rt/test/asan/TestCases/strncpy-overflow.cpp b/compiler-rt/test/asan/TestCases/strncpy-overflow.cpp
index ff84052a94987..f75f8c5eb3ff6 100644
--- a/compiler-rt/test/asan/TestCases/strncpy-overflow.cpp
+++ b/compiler-rt/test/asan/TestCases/strncpy-overflow.cpp
@@ -6,14 +6,15 @@
 // REQUIRES: compiler-rt-optimized
 // REQUIRES: stable-runtime
 
-#include <string.h>
+#include "defines.h"
 #include <stdlib.h>
+#include <string.h>
 
 // We need a way to prevent the optimize from eliminating the
 // strncpy below (which otherwises writes to dead storage).  We
 // need the read to be out-of-line to prevent memory forwarding
 // from making the memory dead again.
-int sink_memory(int N, char *p) __attribute__((noinline));
+int ATTRIBUTE_NOINLINE sink_memory(int N, char *p);
 int sink_memory(int N, char *p) {
   int sum = 0;
   for (int i = 0; i < N; i++)
diff --git a/compiler-rt/test/asan/TestCases/strncpy-overlap.cpp b/compiler-rt/test/asan/TestCases/strncpy-overlap.cpp
index 860fc5d304e0e..9334a333c052a 100644
--- a/compiler-rt/test/asan/TestCases/strncpy-overlap.cpp
+++ b/compiler-rt/test/asan/TestCases/strncpy-overlap.cpp
@@ -28,11 +28,12 @@
 
 // UNSUPPORTED: android
 
+#include "defines.h"
 #include <string.h>
 
 
 // Don't inline function otherwise stacktrace changes.
-__attribute__((noinline)) void bad_function() {
+ATTRIBUTE_NOINLINE void bad_function() {
   char buffer[] = "hello";
   // CHECK: strncpy-param-overlap: memory ranges
   // CHECK: [{{0x.*,[ ]*0x.*}}) and [{{0x.*,[ ]*0x.*}}) overlap
diff --git a/compiler-rt/test/asan/TestCases/suppressions-alloc-dealloc-mismatch.cpp b/compiler-rt/test/asan/TestCases/suppressions-alloc-dealloc-mismatch.cpp
new file mode 100644
index 0000000000000..7f592c83216fd
--- /dev/null
+++ b/compiler-rt/test/asan/TestCases/suppressions-alloc-dealloc-mismatch.cpp
@@ -0,0 +1,34 @@
+// Check that without suppressions, we catch the issue.
+// RUN: %clangxx_asan -O0 %s -o %t
+// RUN: %env_asan_opts=alloc_dealloc_mismatch=1 not %run %t 2>&1 | FileCheck --check-prefix=CHECK-CRASH %s
+
+// RUN: echo "alloc_dealloc_mismatch:function" > %t.supp
+// RUN: %clangxx_asan -O0 %s -o %t && %env_asan_opts=alloc_dealloc_mismatch=1:suppressions='"%t.supp"' %run %t 2>&1 | FileCheck --check-prefix=CHECK-IGNORE %s
+// RUN: %clangxx_asan -O3 %s -o %t && %env_asan_opts=alloc_dealloc_mismatch=1:suppressions='"%t.supp"' %run %t 2>&1 | FileCheck --check-prefix=CHECK-IGNORE %s
+
+// FIXME: Upload suppressions to device.
+// XFAIL: android
+
+// FIXME: atos does not work for inlined functions, yet llvm-symbolizer
+// does not always work with debug info on Darwin. Behavior is similar on MSVC x86 outside of /Od.
+// UNSUPPORTED: darwin
+// UNSUPPORTED: target={{.*windows-msvc.*}} && asan-32-bits
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+void function() {
+  char *a = (char *)malloc(6);
+  a[0] = '\0';
+  size_t len = strlen(a);
+  delete a; // BOOM
+  fprintf(stderr, "strlen ignored, len = %zu\n", len);
+}
+
+int main() { function(); }
+
+// CHECK-CRASH: AddressSanitizer: alloc-dealloc-mismatch
+// CHECK-CRASH-NOT: strlen ignored
+// CHECK-IGNORE-NOT: AddressSanitizer: alloc-dealloc-mismatch
+// CHECK-IGNORE: strlen ignored
diff --git a/compiler-rt/test/asan/TestCases/suppressions-exec-relative-location.cpp b/compiler-rt/test/asan/TestCases/suppressions-exec-relative-location.cpp
index 58649c74fc4f9..0a028e2ebed6d 100644
--- a/compiler-rt/test/asan/TestCases/suppressions-exec-relative-location.cpp
+++ b/compiler-rt/test/asan/TestCases/suppressions-exec-relative-location.cpp
@@ -23,6 +23,7 @@
 // RUN: %env_asan_opts=suppressions='"folder/only/"' not %run %t 2>&1 | \
 // RUN:   FileCheck --check-prefix=CHECK-WRONG-FILE-NAME %s
 
+// FIXME: Upload suppressions to device.
 // XFAIL: android
 // UNSUPPORTED: ios
 
diff --git a/compiler-rt/test/asan/TestCases/suppressions-function.cpp b/compiler-rt/test/asan/TestCases/suppressions-function.cpp
index 9a0ef95ebb2ac..c8ed3fd4866da 100644
--- a/compiler-rt/test/asan/TestCases/suppressions-function.cpp
+++ b/compiler-rt/test/asan/TestCases/suppressions-function.cpp
@@ -6,12 +6,14 @@
 // RUN: %clangxx_asan -O0 %s -o %t && %env_asan_opts=suppressions='"%t.supp"' %run %t 2>&1 | FileCheck --check-prefix=CHECK-IGNORE %s
 // RUN: %clangxx_asan -O3 %s -o %t && %env_asan_opts=suppressions='"%t.supp"' %run %t 2>&1 | FileCheck --check-prefix=CHECK-IGNORE %s
 
+// FIXME: Upload suppressions to device.
 // XFAIL: android
 // UNSUPPORTED: ios
 
 // FIXME: atos does not work for inlined functions, yet llvm-symbolizer
-// does not always work with debug info on Darwin.
+// does not always work with debug info on Darwin. Behavior is similar on MSVC x86 outside of /Od.
 // UNSUPPORTED: darwin
+// UNSUPPORTED: target={{.*windows-msvc.*}} && asan-32-bits
 
 #include <stdio.h>
 #include <stdlib.h>
diff --git a/compiler-rt/test/asan/TestCases/suppressions-interceptor.cpp b/compiler-rt/test/asan/TestCases/suppressions-interceptor.cpp
index e44ccb8e6527d..02995cf6f0510 100644
--- a/compiler-rt/test/asan/TestCases/suppressions-interceptor.cpp
+++ b/compiler-rt/test/asan/TestCases/suppressions-interceptor.cpp
@@ -5,6 +5,7 @@
 // RUN: echo "interceptor_name:strlen" > %t.supp
 // RUN: %env_asan_opts=suppressions='"%t.supp"' %run %t 2>&1 | FileCheck --check-prefix=CHECK-IGNORE %s
 
+// FIXME: Upload suppressions to device.
 // XFAIL: android
 
 #include <stdio.h>
diff --git a/compiler-rt/test/asan/TestCases/suppressions-library.cpp b/compiler-rt/test/asan/TestCases/suppressions-library.cpp
index d11802a20ad3f..5427122eaa92f 100644
--- a/compiler-rt/test/asan/TestCases/suppressions-library.cpp
+++ b/compiler-rt/test/asan/TestCases/suppressions-library.cpp
@@ -9,6 +9,7 @@
 // RUN: echo "interceptor_via_lib:"%xdynamiclib_filename > %t.supp
 // RUN: %env_asan_opts=suppressions='"%t.supp"' %run %t 2>&1 | FileCheck --check-prefix=CHECK-IGNORE %s
 
+// FIXME: Upload suppressions to device.
 // XFAIL: android
 
 #include <stdio.h>
diff --git a/compiler-rt/test/asan/TestCases/throw_call_test.cpp b/compiler-rt/test/asan/TestCases/throw_call_test.cpp
index 20a7c0b766015..06e9422701406 100644
--- a/compiler-rt/test/asan/TestCases/throw_call_test.cpp
+++ b/compiler-rt/test/asan/TestCases/throw_call_test.cpp
@@ -2,6 +2,10 @@
 // http://code.google.com/p/address-sanitizer/issues/detail?id=147 (not fixed).
 // BROKEN: %clangxx_asan %s -o %t -static-libstdc++ && %run %t
 
+// MSVC doesn't support GCC style inline ASM
+// UNSUPPORTED: msvc
+
+#include "defines.h"
 #include <stdio.h>
 static volatile int zero = 0;
 inline void pretend_to_do_something(void *x) {
@@ -15,7 +19,7 @@ void ReallyThrow() {
     throw 42;
 }
 
-__attribute__((noinline))
+ATTRIBUTE_NOINLINE
 void Throw() {
   int a, b, c, d, e, f, g, h;
   pretend_to_do_something(&a);
@@ -30,7 +34,7 @@ void Throw() {
   ReallyThrow();
 }
 
-__attribute__((noinline))
+ATTRIBUTE_NOINLINE
 void CheckStack() {
   int ar[100];
   pretend_to_do_something(ar);
diff --git a/compiler-rt/test/asan/TestCases/throw_catch.cpp b/compiler-rt/test/asan/TestCases/throw_catch.cpp
index 2884e95f8a7e2..71e66e3fc4dd4 100644
--- a/compiler-rt/test/asan/TestCases/throw_catch.cpp
+++ b/compiler-rt/test/asan/TestCases/throw_catch.cpp
@@ -1,17 +1,18 @@
 // RUN: %clangxx_asan -fsanitize-address-use-after-return=never -O %s -o %t && %run %t
 
+#include "defines.h"
 #include <assert.h>
-#include <stdio.h>
 #include <sanitizer/asan_interface.h>
+#include <stdio.h>
 
-__attribute__((noinline))
+ATTRIBUTE_NOINLINE
 void Throw() {
   int local;
   fprintf(stderr, "Throw:  %p\n", &local);
   throw 1;
 }
 
-__attribute__((noinline))
+ATTRIBUTE_NOINLINE
 void ThrowAndCatch() {
   int local;
   try {
@@ -21,7 +22,7 @@ void ThrowAndCatch() {
   }
 }
 
-__attribute__((noinline))
+ATTRIBUTE_NOINLINE
 void TestThrow() {
   char x[32];
   fprintf(stderr, "Before: %p poisoned: %d\n", &x,
@@ -33,7 +34,7 @@ void TestThrow() {
   assert(!__asan_address_is_poisoned(x + 32));
 }
 
-__attribute__((noinline))
+ATTRIBUTE_NOINLINE
 void TestThrowInline() {
   char x[32];
   fprintf(stderr, "Before: %p poisoned: %d\n", &x,
diff --git a/compiler-rt/test/asan/TestCases/throw_invoke_test.cpp b/compiler-rt/test/asan/TestCases/throw_invoke_test.cpp
index a9069e1ca8d1a..2faf3efe9e6ec 100644
--- a/compiler-rt/test/asan/TestCases/throw_invoke_test.cpp
+++ b/compiler-rt/test/asan/TestCases/throw_invoke_test.cpp
@@ -2,8 +2,9 @@
 // RUN: %clangxx_asan %s -o %t -static-libstdc++ && %run %t
 
 // Investigate why it fails with NDK 21.
-// UNSUPPORTED: android
+// UNSUPPORTED: android, MSVC
 
+#include "defines.h"
 #include <stdio.h>
 static volatile int zero = 0;
 inline void pretend_to_do_something(void *x) {
@@ -34,7 +35,7 @@ void Throw() {
   ReallyThrow();
 }
 
-__attribute__((noinline))
+ATTRIBUTE_NOINLINE
 void CheckStack() {
   int ar[100];
   pretend_to_do_something(ar);
diff --git a/compiler-rt/test/asan/TestCases/use-after-scope-capture.cpp b/compiler-rt/test/asan/TestCases/use-after-scope-capture.cpp
index 0eca27181e54f..110d9e5222453 100644
--- a/compiler-rt/test/asan/TestCases/use-after-scope-capture.cpp
+++ b/compiler-rt/test/asan/TestCases/use-after-scope-capture.cpp
@@ -1,12 +1,13 @@
 // RUN: %clangxx_asan -O0 %s -o %t && not %run %t 2>&1 | FileCheck %s
 
+#include "defines.h"
 #include <functional>
 
 int main() {
   std::function<int()> f;
   {
     int x = 0;
-    f = [&x]() __attribute__((noinline)) {
+    f = [&x]() ATTRIBUTE_NOINLINE {
       return x;  // BOOM
       // CHECK: ERROR: AddressSanitizer: stack-use-after-scope
       // We cannot assert the line, after the argument promotion pass this crashes
diff --git a/compiler-rt/test/asan/TestCases/use-after-scope-dtor-order.cpp b/compiler-rt/test/asan/TestCases/use-after-scope-dtor-order.cpp
index 42f62c72397ec..c4a188d2c86ce 100644
--- a/compiler-rt/test/asan/TestCases/use-after-scope-dtor-order.cpp
+++ b/compiler-rt/test/asan/TestCases/use-after-scope-dtor-order.cpp
@@ -1,10 +1,11 @@
 // RUN: %clangxx_asan -O1 %s -o %t && not %run %t 2>&1 | FileCheck %s
 
+#include "defines.h"
 #include <stdio.h>
 
 struct IntHolder {
   explicit IntHolder(int *val = 0) : val_(val) { }
-  __attribute__((noinline)) ~IntHolder() {
+  ATTRIBUTE_NOINLINE ~IntHolder() {
     printf("Value: %d\n", *val_);  // BOOM
     // CHECK: ERROR: AddressSanitizer: stack-use-after-scope
     // CHECK:  #0 0x{{.*}} in IntHolder::~IntHolder{{.*}}.cpp:[[@LINE-2]]
diff --git a/compiler-rt/test/asan/TestCases/use-after-scope-inlined.cpp b/compiler-rt/test/asan/TestCases/use-after-scope-inlined.cpp
index 1014ff919b9ef..d0154ef744241 100644
--- a/compiler-rt/test/asan/TestCases/use-after-scope-inlined.cpp
+++ b/compiler-rt/test/asan/TestCases/use-after-scope-inlined.cpp
@@ -4,9 +4,13 @@
 //
 // RUN: %clangxx_asan -O2 %s -o %t && not %run %t 2>&1 | FileCheck %s
 
+// MSVC marks this as xfail because it doesn't generate the metadata to display the "x.i" offset.
+// XFAIL: msvc
+#include "defines.h"
+
 int *arr;
 
-__attribute__((always_inline))
+ATTRIBUTE_ALWAYS_INLINE
 void inlined(int arg) {
   int x[5];
   for (int i = 0; i < arg; i++) x[i] = i;
diff --git a/compiler-rt/test/asan/TestCases/use-after-scope-loop-bug.cpp b/compiler-rt/test/asan/TestCases/use-after-scope-loop-bug.cpp
index 286b6441a5698..b97fe6730ddc4 100644
--- a/compiler-rt/test/asan/TestCases/use-after-scope-loop-bug.cpp
+++ b/compiler-rt/test/asan/TestCases/use-after-scope-loop-bug.cpp
@@ -1,4 +1,4 @@
-// RUN: %clangxx_asan -O1 %s -o %t && not %run %t 2>&1 | FileCheck %s
+// RUN: %clangxx_asan %if MSVC %{ /Od %} %else %{ -O1 %} %s -o %t && not %run %t 2>&1 | FileCheck %s
 
 volatile int *p;
 
diff --git a/compiler-rt/test/asan/TestCases/use-after-scope-loop-removed.cpp b/compiler-rt/test/asan/TestCases/use-after-scope-loop-removed.cpp
index 8a8a7b60deb6b..8fedceb6cf330 100644
--- a/compiler-rt/test/asan/TestCases/use-after-scope-loop-removed.cpp
+++ b/compiler-rt/test/asan/TestCases/use-after-scope-loop-removed.cpp
@@ -1,4 +1,5 @@
-// RUN: %clangxx_asan -O1 %s -o %t && not %run %t 2>&1 | FileCheck %s
+// RUN: %clangxx_asan %if MSVC %{ /Od %} %else %{ -O1 %} \
+// RUN:     %s -o %t && not %run %t 2>&1 | FileCheck %s
 
 #include <stdlib.h>
 
diff --git a/compiler-rt/test/asan/TestCases/use-after-scope-loop.cpp b/compiler-rt/test/asan/TestCases/use-after-scope-loop.cpp
index 3e199056930cb..7254a785995b7 100644
--- a/compiler-rt/test/asan/TestCases/use-after-scope-loop.cpp
+++ b/compiler-rt/test/asan/TestCases/use-after-scope-loop.cpp
@@ -1,4 +1,5 @@
-// RUN: %clangxx_asan -O1 %s -o %t && not %run %t 2>&1 | FileCheck %s
+// RUN: %clangxx_asan %if MSVC %{ /Od %} %else %{ -O1 %} \
+// RUN:     %s -o %t && not %run %t 2>&1 | FileCheck %s
 
 int *p[3];
 
diff --git a/compiler-rt/test/asan/TestCases/use-after-scope-temp.cpp b/compiler-rt/test/asan/TestCases/use-after-scope-temp.cpp
index 29680f37bfa53..ab14e14e485d9 100644
--- a/compiler-rt/test/asan/TestCases/use-after-scope-temp.cpp
+++ b/compiler-rt/test/asan/TestCases/use-after-scope-temp.cpp
@@ -1,14 +1,14 @@
-// RUN: %clangxx_asan -O1 %s -o %t && not %run %t 2>&1 | FileCheck %s
+// RUN: %clangxx_asan %if MSVC %{ /Od %} %else %{ -O1 %} \
+// RUN:     %s -o %t && not %run %t 2>&1 | FileCheck %s
 
+#include "defines.h"
 struct IntHolder {
   int val;
 };
 
 const IntHolder *saved;
 
-__attribute__((noinline)) void save(const IntHolder &holder) {
-  saved = &holder;
-}
+ATTRIBUTE_NOINLINE void save(const IntHolder &holder) { saved = &holder; }
 
 int main(int argc, char *argv[]) {
   save({argc});
diff --git a/compiler-rt/test/asan/TestCases/use-after-scope-temp2.cpp b/compiler-rt/test/asan/TestCases/use-after-scope-temp2.cpp
index b98d6f12911a7..ff65f592ee208 100644
--- a/compiler-rt/test/asan/TestCases/use-after-scope-temp2.cpp
+++ b/compiler-rt/test/asan/TestCases/use-after-scope-temp2.cpp
@@ -1,9 +1,10 @@
-// RUN: %clangxx_asan -O1 %s -o %t && not %run %t 2>&1 | FileCheck %s
+// RUN: %clangxx_asan %if MSVC %{ /Od %} %else %{ -O1 %} \
+// RUN:     %s -o %t && not %run %t 2>&1 | FileCheck %s
+
+#include "defines.h"
 
 struct IntHolder {
-  __attribute__((noinline)) const IntHolder &Self() const {
-    return *this;
-  }
+  ATTRIBUTE_NOINLINE const IntHolder &Self() const { return *this; }
   int val = 3;
 };
 
diff --git a/compiler-rt/test/asan/TestCases/use-after-scope-types.cpp b/compiler-rt/test/asan/TestCases/use-after-scope-types.cpp
index 5751a0a09bb4e..3e740edfae250 100644
--- a/compiler-rt/test/asan/TestCases/use-after-scope-types.cpp
+++ b/compiler-rt/test/asan/TestCases/use-after-scope-types.cpp
@@ -11,6 +11,7 @@
 // RUN: not %run %t 9 2>&1 | FileCheck %s
 // RUN: not %run %t 10 2>&1 | FileCheck %s
 
+#include "defines.h"
 #include <stdlib.h>
 #include <string>
 #include <vector>
@@ -32,7 +33,7 @@ template <class T, size_t N> struct Ptr<T[N]> {
   T *t;
 };
 
-template <class T> __attribute__((noinline)) void test() {
+template <class T> ATTRIBUTE_NOINLINE void test() {
   Ptr<T> ptr;
   {
     T x;
diff --git a/compiler-rt/test/asan/TestCases/vla_chrome_testcase.cpp b/compiler-rt/test/asan/TestCases/vla_chrome_testcase.cpp
index 613d4bb2763fa..743e2ae71425b 100644
--- a/compiler-rt/test/asan/TestCases/vla_chrome_testcase.cpp
+++ b/compiler-rt/test/asan/TestCases/vla_chrome_testcase.cpp
@@ -1,20 +1,22 @@
 // RUN: %clangxx_asan -O0 -mllvm -asan-instrument-dynamic-allocas %s -o %t
 // RUN: not %run %t 2>&1 | FileCheck %s
 //
+// MSVC doesn't support VLAs
+// UNSUPPORTED: msvc
 
 // This is reduced testcase based on Chromium code.
 // See http://reviews.llvm.org/D6055?vs=on&id=15616&whitespace=ignore-all#toc.
 
-#include <stdint.h>
+#include "defines.h"
 #include <assert.h>
+#include <stdint.h>
 
 int a = 7;
 int b;
 int c;
 int *p;
 
-__attribute__((noinline)) void fn3(int *first, int second) {
-}
+ATTRIBUTE_NOINLINE void fn3(int *first, int second) {}
 
 int main() {
   int d = b && c;
diff --git a/compiler-rt/test/asan/TestCases/vla_condition_overflow.cpp b/compiler-rt/test/asan/TestCases/vla_condition_overflow.cpp
index 00df5d2919c46..5269a35e9796a 100644
--- a/compiler-rt/test/asan/TestCases/vla_condition_overflow.cpp
+++ b/compiler-rt/test/asan/TestCases/vla_condition_overflow.cpp
@@ -2,11 +2,14 @@
 // RUN: not %run %t 2>&1 | FileCheck %s
 //
 // REQUIRES: stable-runtime
+// MSVC doesn't support VLAs
+// UNSUPPORTED: msvc
 
+#include "defines.h"
 #include <assert.h>
 #include <stdint.h>
 
-__attribute__((noinline)) void foo(int index, int len) {
+ATTRIBUTE_NOINLINE void foo(int index, int len) {
   if (index > len) {
     char str[len];
     assert(!(reinterpret_cast<uintptr_t>(str) & 31L));
diff --git a/compiler-rt/test/asan/TestCases/vla_loop_overfow.cpp b/compiler-rt/test/asan/TestCases/vla_loop_overfow.cpp
index 370e0dc72c12f..d4cf9513d0fa6 100644
--- a/compiler-rt/test/asan/TestCases/vla_loop_overfow.cpp
+++ b/compiler-rt/test/asan/TestCases/vla_loop_overfow.cpp
@@ -3,6 +3,9 @@
 //
 // REQUIRES: stable-runtime
 
+// MSVC doesn't support VLAs
+// UNSUPPORTED: msvc
+
 #include <assert.h>
 #include <stdint.h>
 
diff --git a/compiler-rt/test/asan/TestCases/zero_page_pc.cpp b/compiler-rt/test/asan/TestCases/zero_page_pc.cpp
index a7d00ce9b6988..3af4d04a587c2 100644
--- a/compiler-rt/test/asan/TestCases/zero_page_pc.cpp
+++ b/compiler-rt/test/asan/TestCases/zero_page_pc.cpp
@@ -1,6 +1,10 @@
 // Check that ASan correctly detects SEGV on the zero page.
 // RUN: %clangxx_asan %s -o %t && not %run %t 2>&1 | FileCheck %s
 
+#if defined(_MSC_VER) && !defined(__CLANG__)
+#  define __has_feature(x) 0
+#endif
+
 #if __has_feature(ptrauth_calls)
 #  include <ptrauth.h>
 #endif
diff --git a/compiler-rt/test/asan/lit.cfg.py b/compiler-rt/test/asan/lit.cfg.py
index c57f5ca0fa652..2da511103da7c 100644
--- a/compiler-rt/test/asan/lit.cfg.py
+++ b/compiler-rt/test/asan/lit.cfg.py
@@ -183,6 +183,7 @@ def build_invocation(compile_flags, with_lto=False):
         config.substitutions.append(("%MD", "-MD"))
         config.substitutions.append(("%MT", "-MT"))
         config.substitutions.append(("%Gw", "-Gw"))
+        config.substitutions.append(("%Oy-", "-Oy-"))
 
         base_lib = os.path.join(
             config.compiler_rt_libdir, "clang_rt.asan%%s%s.lib" % config.target_suffix
@@ -220,6 +221,7 @@ def build_invocation(compile_flags, with_lto=False):
         config.substitutions.append(("%MD", ""))
         config.substitutions.append(("%MT", ""))
         config.substitutions.append(("%Gw", "-fdata-sections"))
+        config.substitutions.append(("%Oy-", "-fno-omit-frame-pointer"))
 
 # FIXME: De-hardcode this path.
 asan_source_dir = os.path.join(
diff --git a/compiler-rt/test/builtins/Unit/arm/aeabi_idivmod_test.c b/compiler-rt/test/builtins/Unit/arm/aeabi_idivmod_test.c
index ff767d754eb97..e4953bf51b037 100644
--- a/compiler-rt/test/builtins/Unit/arm/aeabi_idivmod_test.c
+++ b/compiler-rt/test/builtins/Unit/arm/aeabi_idivmod_test.c
@@ -14,8 +14,19 @@ int test__aeabi_idivmod(si_int a, si_int b,
 {
 	  si_int rem;
     du_int ret = __aeabi_idivmod(a, b);
+    // __aeabi_idivmod actually returns a struct { quotient; remainder; } using
+    // value_in_regs calling convention. Due to the ABI rules, struct fields
+    // come in the same order regardless of endianness. However since the
+    // result is received here as a 64-bit integer, in which endianness does
+    // matter, the position of each component (quotient and remainder) varies
+    // depending on endianness.
+#  if _YUGA_BIG_ENDIAN
+    rem = ret & 0xFFFFFFFF;
+    si_int result = ret >> 32;
+#  else
     rem = ret >> 32;
     si_int result = ret & 0xFFFFFFFF;
+#  endif
     if (result != expected_result) {
         printf("error in __aeabi_idivmod: %d / %d = %d, expected %d\n",
                a, b, result, expected_result);
diff --git a/compiler-rt/test/builtins/Unit/arm/aeabi_uidivmod_test.c b/compiler-rt/test/builtins/Unit/arm/aeabi_uidivmod_test.c
index de5a43d5b8143..a507267c44602 100644
--- a/compiler-rt/test/builtins/Unit/arm/aeabi_uidivmod_test.c
+++ b/compiler-rt/test/builtins/Unit/arm/aeabi_uidivmod_test.c
@@ -13,8 +13,19 @@ int test__aeabi_uidivmod(su_int a, su_int b,
 						su_int expected_result, su_int expected_rem)
 {
     du_int ret = __aeabi_uidivmod(a, b);
+    // __aeabi_uidivmod actually returns a struct { quotient; remainder; }
+    // using value_in_regs calling convention. Due to the ABI rules, struct
+    // fields come in the same order regardless of endianness. However since
+    // the result is received here as a 64-bit integer, in which endianness
+    // does matter, the position of each component (quotient and remainder)
+    // varies depending on endianness.
+#  if _YUGA_BIG_ENDIAN
+    su_int rem = ret & 0xFFFFFFFF;
+    si_int result = ret >> 32;
+#  else
     su_int rem = ret >> 32;
     si_int result = ret & 0xFFFFFFFF;
+#  endif
 
     if (result != expected_result) {
         printf("error in __aeabi_uidivmod: %u / %u = %u, expected %u\n",
diff --git a/compiler-rt/test/builtins/Unit/arm/aeabi_uldivmod_test.c b/compiler-rt/test/builtins/Unit/arm/aeabi_uldivmod_test.c
index 2ff65a8b9ec3f..02cfa2f38713b 100644
--- a/compiler-rt/test/builtins/Unit/arm/aeabi_uldivmod_test.c
+++ b/compiler-rt/test/builtins/Unit/arm/aeabi_uldivmod_test.c
@@ -12,20 +12,37 @@ COMPILER_RT_ABI void /* __value_in_regs */ __aeabi_uldivmod(du_int a, du_int b);
 int test_aeabi_uldivmod(du_int a, du_int b, du_int expected_q, du_int expected_r)
 {
     du_int q, r;
+    // __aeabi_uldivmod returns a struct { quotient; remainder; } using
+    // value_in_regs calling convention. Each field is a 64-bit integer, so the
+    // quotient resides in r0 and r1, while the remainder in r2 and r3. The
+    // byte order however depends on the endianness.
     __asm__(
+#  if _YUGA_BIG_ENDIAN
+        "movs r1, %Q[a] \n"
+        "movs r0, %R[a] \n"
+        "movs r3, %Q[b] \n"
+        "movs r2, %R[b] \n"
+#  else
         "movs r0, %Q[a] \n"
         "movs r1, %R[a] \n"
         "movs r2, %Q[b] \n"
         "movs r3, %R[b] \n"
+#  endif
         "bl __aeabi_uldivmod \n"
+#  if _YUGA_BIG_ENDIAN
+        "movs %Q[q], r1\n"
+        "movs %R[q], r0\n"
+        "movs %Q[r], r3\n"
+        "movs %R[r], r2\n"
+#  else
         "movs %Q[q], r0\n"
         "movs %R[q], r1\n"
         "movs %Q[r], r2\n"
         "movs %R[r], r3\n"
-        : [q] "=r" (q), [r] "=r"(r)
+#  endif
+        : [q] "=r"(q), [r] "=r"(r)
         : [a] "r"(a), [b] "r"(b)
-        : "lr", "r0", "r1", "r2", "r3"
-        );
+        : "lr", "r0", "r1", "r2", "r3");
     if (q != expected_q || r != expected_r)
         printf("error in aeabi_uldivmod: %llX / %llX = %llX, R = %llX, expected %llX, %llX\n",
                a, b, q, r, expected_q, expected_r);
diff --git a/compiler-rt/test/orc/TestCases/Darwin/Generic/exceptions-stress-test-tower.cpp b/compiler-rt/test/orc/TestCases/Darwin/Generic/exceptions-stress-test-tower.cpp
new file mode 100644
index 0000000000000..245afa084f5c0
--- /dev/null
+++ b/compiler-rt/test/orc/TestCases/Darwin/Generic/exceptions-stress-test-tower.cpp
@@ -0,0 +1,24 @@
+// RUN: %clangxx -c -o %t %s
+// RUN: %llvm_jitlink -slab-allocate=20Mb %t
+//
+// REQUIRES: system-darwin && host-arch-compatible
+
+// Test that we can throw and catch an exception through a large number of
+// stack frames. The number (1022) is chosen to force emission of multiple
+// unwind info second-level pages.
+
+template <int N>
+void f() { try { f<N - 1>(); } catch (...) { throw; } }
+
+template <>
+void f<0>() { throw 42; }
+
+int main(int argc, char *argv[]) {
+  try {
+    f<1020>();
+  } catch (int n) {
+    return 42 - n;
+  }
+  return 1;
+}
+
diff --git a/compiler-rt/test/orc/TestCases/Darwin/Generic/exceptions.cpp b/compiler-rt/test/orc/TestCases/Darwin/Generic/exceptions.cpp
new file mode 100644
index 0000000000000..304588a327386
--- /dev/null
+++ b/compiler-rt/test/orc/TestCases/Darwin/Generic/exceptions.cpp
@@ -0,0 +1,14 @@
+// RUN: %clangxx -c -o %t %s
+// RUN: %llvm_jitlink -slab-allocate=20Mb %t
+//
+// REQUIRES: system-darwin && host-arch-compatible
+
+// Test that trivial throw / catch works.
+int main(int argc, char *argv[]) {
+  try {
+    throw 42;
+  } catch (int E) {
+    return 42 - E;
+  }
+  return 1;
+}
diff --git a/compiler-rt/test/profile/Linux/binary-id-path.c b/compiler-rt/test/profile/Linux/binary-id-path.c
new file mode 100644
index 0000000000000..a845fd424bcb7
--- /dev/null
+++ b/compiler-rt/test/profile/Linux/binary-id-path.c
@@ -0,0 +1,42 @@
+// REQUIRES: linux
+// RUN: split-file %s %t.dir
+// RUN: %clang_profgen -Wl,--build-id=sha1 -o %t.dir/foo %t.dir/foo.c
+// RUN: %clang_profgen -Wl,--build-id=sha1 -o %t.dir/bar %t.dir/bar.c
+
+// Check that foo and bar have the same signatures.
+// RUN: rm -rf %t.profdir
+// RUN: env LLVM_PROFILE_FILE=%t.profdir/%m.profraw %run %t.dir/foo
+// RUN: env LLVM_PROFILE_FILE=%t.profdir/%m.profraw %run %t.dir/bar 2>&1 | FileCheck %s --check-prefix=MERGE-ERROR
+
+// Check that foo and bar have different binary IDs.
+// RUN: rm -rf %t.profdir %t.profdata
+// RUN: env LLVM_PROFILE_FILE=%t.profdir/%b.profraw %run %t.dir/foo
+// RUN: env LLVM_PROFILE_FILE=%t.profdir/%b.profraw %run %t.dir/bar
+// RUN: llvm-profdata merge -o %t.profdata %t.profdir
+// RUN: llvm-profdata show --binary-ids %t.profdata | FileCheck %s --check-prefix=BINARY-ID
+
+// Check fallback to the default name if binary ID is missing.
+// RUN: %clang_profgen -Wl,--build-id=none -o %t.dir/foo %t.dir/foo.c
+// RUN: env LLVM_PROFILE_FILE=%t.profdir/%b.profraw %run %t.dir/foo 2>&1 | FileCheck %s --check-prefix=MISSING
+
+// MERGE-ERROR: LLVM Profile Error: Profile Merging of file {{.*}}.profraw failed: File exists
+
+// BINARY-ID: Instrumentation level: Front-end
+// BINARY-ID-NEXT: Total functions: 3
+// BINARY-ID-NEXT: Maximum function count: 2
+// BINARY-ID-NEXT: Maximum internal block count: 0
+// BINARY-ID-NEXT: Total number of blocks: 3
+// BINARY-ID-NEXT: Total count: 2
+// BINARY-ID-NEXT: Binary IDs:
+// BINARY-ID-NEXT: {{[0-9a-f]+}}
+// BINARY-ID-NEXT: {{[0-9a-f]+}}
+
+// MISSING: Unable to get binary ID for filename pattern {{.*}}.profraw. Using the default name.
+
+//--- foo.c
+int main(void) { return 0; }
+void foo(void) {}
+
+//--- bar.c
+int main(void) { return 0; }
+void bar(int *a) { *a += 10; }
diff --git a/flang/CMakeLists.txt b/flang/CMakeLists.txt
index b619553ef8302..c012b884ae3be 100644
--- a/flang/CMakeLists.txt
+++ b/flang/CMakeLists.txt
@@ -79,6 +79,8 @@ if(CMAKE_SIZEOF_VOID_P EQUAL 4)
   message(FATAL_ERROR "flang isn't supported on 32 bit CPUs")
 endif()
 
+set(MLIR_MAIN_SRC_DIR "${CMAKE_CURRENT_SOURCE_DIR}/../mlir" CACHE PATH "Path to MLIR source tree")
+
 if (FLANG_STANDALONE_BUILD)
   set(FLANG_BINARY_DIR ${CMAKE_CURRENT_BINARY_DIR})
   set(CMAKE_INCLUDE_CURRENT_DIR ON)
@@ -240,13 +242,14 @@ else()
     set(FLANG_BINARY_DIR ${CMAKE_CURRENT_BINARY_DIR})
   endif()
 
-  set(MLIR_MAIN_SRC_DIR ${LLVM_MAIN_SRC_DIR}/../mlir ) # --src-root
-  set(MLIR_INCLUDE_DIR ${MLIR_MAIN_SRC_DIR}/include ) # --includedir
+  set(MLIR_INCLUDE_DIRS ${MLIR_MAIN_SRC_DIR}/include ) # --includedir
   set(MLIR_TABLEGEN_OUTPUT_DIR ${CMAKE_BINARY_DIR}/tools/mlir/include)
-  include_directories(SYSTEM ${MLIR_INCLUDE_DIR})
+  include_directories(SYSTEM ${MLIR_INCLUDE_DIRS})
   include_directories(SYSTEM ${MLIR_TABLEGEN_OUTPUT_DIR})
 endif()
 
+option(FLANG_INCLUDE_RUNTIME "Build the runtime in-tree (deprecated; to be replaced with LLVM_ENABLE_RUNTIMES=flang-rt)" ON)
+
 set(FLANG_TOOLS_INSTALL_DIR "${CMAKE_INSTALL_BINDIR}" CACHE PATH
     "Path for binary subdirectory (defaults to '${CMAKE_INSTALL_BINDIR}')")
 mark_as_advanced(FLANG_TOOLS_INSTALL_DIR)
@@ -299,7 +302,7 @@ set(FLANG_DEFAULT_LINKER "" CACHE STRING
   "Default linker to use (linker name or absolute path, empty for platform default)")
 
 set(FLANG_DEFAULT_RTLIB "" CACHE STRING
-   "Default Fortran runtime library to use (\"libFortranRuntime\"), leave empty for platform default.")
+   "Default Fortran runtime library to use (\"libflang_rt.runtime\"), leave empty for platform default.")
 
 if (NOT(FLANG_DEFAULT_RTLIB STREQUAL ""))
   message(WARNING "Resetting Flang's default runtime library to use platform default.")
@@ -471,6 +474,12 @@ if (FLANG_INCLUDE_TESTS)
   add_compile_definitions(FLANG_INCLUDE_TESTS=1)
 endif()
 
+option(FLANG_CUF_RUNTIME
+  "Compile CUDA Fortran runtime sources" OFF)
+if (FLANG_CUF_RUNTIME)
+  find_package(CUDAToolkit REQUIRED)
+endif()
+
 add_subdirectory(include)
 add_subdirectory(lib)
 add_subdirectory(cmake/modules)
@@ -481,13 +490,9 @@ if (FLANG_BUILD_TOOLS)
   add_subdirectory(tools)
 endif()
 
-option(FLANG_CUF_RUNTIME
-  "Compile CUDA Fortran runtime sources" OFF)
-if (FLANG_CUF_RUNTIME)
-  find_package(CUDAToolkit REQUIRED)
-endif()
-
-add_subdirectory(runtime)
+if (FLANG_INCLUDE_RUNTIME)
+  add_subdirectory(runtime)
+endif ()
 
 if (LLVM_INCLUDE_EXAMPLES)
   add_subdirectory(examples)
diff --git a/flang/cmake/modules/AddFlang.cmake b/flang/cmake/modules/AddFlang.cmake
index 1f178772067ed..c9f65eb73fef0 100644
--- a/flang/cmake/modules/AddFlang.cmake
+++ b/flang/cmake/modules/AddFlang.cmake
@@ -57,7 +57,7 @@ function(add_flang_library name)
     set(LIBTYPE SHARED)
   elseif(ARG_STATIC)
     # If BUILD_SHARED_LIBS and ARG_STATIC are both set, llvm_add_library prioritizes STATIC.
-    # This is required behavior for libFortranFloat128Math.
+    # This is required behavior for libflang_rt.quadmath.
     set(LIBTYPE STATIC)
   else()
     # Let llvm_add_library decide, taking BUILD_SHARED_LIBS into account.
diff --git a/flang/docs/Directives.md b/flang/docs/Directives.md
index f356f762b13a2..c6c2e29a420ea 100644
--- a/flang/docs/Directives.md
+++ b/flang/docs/Directives.md
@@ -39,15 +39,22 @@ A list of non-standard directives supported by Flang
 * `!dir$ vector always` forces vectorization on the following loop regardless
   of cost model decisions. The loop must still be vectorizable.
   [This directive currently only works on plain do loops without labels].
+* `!dir$ unroll [n]` specifies that the compiler ought to unroll the immediately
+  following loop `n` times. When `n` is `0` or `1`, the loop should not be unrolled
+  at all. When `n` is `2` or greater, the loop should be unrolled exactly `n`
+  times if possible. When `n` is omitted, the compiler should attempt to fully
+  unroll the loop. Some compilers accept an optional `=` before the `n` when `n`
+  is present in the directive. Flang does not.
 
 # Directive Details
 
 ## Introduction
 Directives are commonly used in Fortran programs to specify additional actions 
 to be performed by the compiler. The directives are always specified with the 
-`!dir$` or `cdir$` prefix. 
+`!dir$` or `cdir$` prefix.
 
 ## Loop Directives
+
 Some directives are associated with the following construct, for example loop
 directives. Directives on loops are used to specify additional transformation to
 be performed by the compiler like enabling vectorisation, unrolling, interchange
diff --git a/flang/docs/Extensions.md b/flang/docs/Extensions.md
index f25f0d1e0ca39..ffb7e90eff919 100644
--- a/flang/docs/Extensions.md
+++ b/flang/docs/Extensions.md
@@ -141,6 +141,10 @@ end
   This interpretation has usability advantages and is what six other
   Fortran compilers do, but is not conforming now that J3 approved an
   "interp" in June 2024 to the contrary.
+* When an Arm processor raises an `ieee_overflow` or `ieee_underflow`
+  exception, the `ieee_inexact` exception is also raised. This happens
+  for a call to `ieee_set_flag` as well as for floating point expression
+  evaluation.
 * Arm has processors that allow a user to control what happens when an
   arithmetic exception is signaled, as well as processors that do not
   have this capability. An Arm executable will run on either type of
@@ -411,6 +415,8 @@ end
   is accepted before an array specification (`ch*3(2)`) as well
   as afterwards.
 * A zero field width is allowed for logical formatted output (`L0`).
+* `OPEN(..., FORM='BINARY')` is accepted as a legacy synonym for
+  the standard `OPEN(..., FORM='UNFORMATTED', ACCESS='STREAM')`.
 
 ### Extensions supported when enabled by options
 
diff --git a/flang/docs/FlangDriver.md b/flang/docs/FlangDriver.md
index 23cbab30ee903..97744f0bee069 100644
--- a/flang/docs/FlangDriver.md
+++ b/flang/docs/FlangDriver.md
@@ -119,7 +119,7 @@ Internally, a `clangDriver` based compiler driver works by creating actions
 that correspond to various compilation phases, e.g. `PreprocessJobClass`,
 `CompileJobClass`, `BackendJobClass` or `LinkJobClass` from the
 `clang::driver::Action::ActionClass` enum. There are also other, more
-specialised actions, e.g. `MigrateJobClass` or `InputClass`, that do not map
+specialised actions, e.g. `InputClass`, that do not map
 directly to common compilation steps. The actions to run are determined from
 the supplied compiler flags, e.g.
 
@@ -175,19 +175,18 @@ like this:
 
 ```
 $ flang -v -o example example.o
-"/usr/bin/ld" [...] example.o [...] "-lFortranRuntime" "-lFortranDecimal" [...]
+"/usr/bin/ld" [...] example.o [...] "-lflang_rt.runtime" [...]
 ```
 
 The automatically added libraries are:
 
-* `FortranRuntime`: Provides most of the Flang runtime library.
-* `FortranDecimal`: Provides operations for decimal numbers.
+* `flang_rt.runtime`: Provides most of the Flang runtime library.
 
 If the code is C/C++ based and invokes Fortran routines, one can either use Clang
 or Flang as the linker driver.  If Clang is used, it will automatically all
 required runtime libraries needed by C++ (e.g., for STL) to the linker invocation.
-In this case, one has to explicitly provide the Fortran runtime libraries
-`FortranRuntime` and/or `FortranDecimal`.  An alternative is to use Flang to link.
+In this case, one has to explicitly provide the Fortran runtime library
+`flang_rt.runtime`.  An alternative is to use Flang to link.
 In this case, it may be required to explicitly supply C++ runtime libraries.
 
 On Darwin, the logical root where the system libraries are located (sysroot)
diff --git a/flang/docs/GettingStarted.md b/flang/docs/GettingStarted.md
index 1c85a6754b155..e422a31a0b402 100644
--- a/flang/docs/GettingStarted.md
+++ b/flang/docs/GettingStarted.md
@@ -216,7 +216,7 @@ cmake \
   -DCMAKE_CUDA_COMPILER=clang \
   -DCMAKE_CUDA_HOST_COMPILER=clang++ \
   ../runtime/
-make -j FortranRuntime
+make -j flang-rt
 ```
 
 Note that the used version of `clang` must [support](https://releases.llvm.org/16.0.0/tools/clang/docs/ReleaseNotes.html#cuda-support)
@@ -239,7 +239,7 @@ cmake \
   -DCMAKE_CUDA_HOST_COMPILER=clang++ \
   ../runtime/
 
-make -j FortranRuntime
+make -j flang-rt
 ```
 
 Note that `nvcc` might limit support to certain
@@ -294,7 +294,7 @@ cmake \
   -DFLANG_OMP_DEVICE_ARCHITECTURES="all" \
   ../runtime/
 
-make -j FortranRuntime
+make -j flang-rt
 ```
 
 The result of the build is a "device-only" library, i.e. the host
@@ -309,7 +309,7 @@ The same set of CMake variables works for Flang in-tree build.
 One may provide optional CMake variables to customize the build. Available options:
 
 * `-DFLANG_RUNTIME_F128_MATH_LIB=libquadmath`: enables build of
-  `FortranFloat128Math` library that provides `REAL(16)` math APIs
+  `flang_rt.quadmath` library that provides `REAL(16)` math APIs
   for intrinsics such as `SIN`, `COS`, etc. GCC `libquadmath`'s header file
   `quadmath.h` must be available to the build compiler.
   [More details](Real16MathSupport.md).
diff --git a/flang/docs/Intrinsics.md b/flang/docs/Intrinsics.md
index d0b7999fbd067..5b671d1b2c740 100644
--- a/flang/docs/Intrinsics.md
+++ b/flang/docs/Intrinsics.md
@@ -767,7 +767,7 @@ This phase currently supports all the intrinsic procedures listed above but the
 | Object characteristic inquiry functions | ALLOCATED, ASSOCIATED, EXTENDS_TYPE_OF, IS_CONTIGUOUS, PRESENT, RANK, SAME_TYPE, STORAGE_SIZE |
 | Type inquiry intrinsic functions | BIT_SIZE, DIGITS, EPSILON, HUGE, KIND, MAXEXPONENT, MINEXPONENT, NEW_LINE, PRECISION, RADIX, RANGE, TINY|
 | Non-standard intrinsic functions | AND, OR, XOR, SHIFT, ZEXT, IZEXT, COSD, SIND, TAND, ACOSD, ASIND, ATAND, ATAN2D, COMPL, EQV, NEQV, INT8, JINT, JNINT, KNINT, QCMPLX, DREAL, DFLOAT, QEXT, QFLOAT, QREAL, DNUM, NUM, JNUM, KNUM, QNUM, RNUM, RAN, RANF, ILEN, SIZEOF, MCLOCK, SECNDS, COTAN, IBCHNG, ISHA, ISHC, ISHL, IXOR, IARG, IARGC, NARGS, GETPID, NUMARG, BADDRESS, IADDR, CACHESIZE, EOF, FP_CLASS, INT_PTR_KIND, ISNAN, MALLOC, FREE, GETUID, GETGID |
-| Intrinsic subroutines |MVBITS (elemental), CPU_TIME, DATE_AND_TIME, EVENT_QUERY, EXECUTE_COMMAND_LINE, GET_COMMAND, GET_COMMAND_ARGUMENT, GET_ENVIRONMENT_VARIABLE, MOVE_ALLOC, RANDOM_INIT, RANDOM_NUMBER, RANDOM_SEED, SIGNAL, SLEEP, SYSTEM, SYSTEM_CLOCK |
+| Intrinsic subroutines |MVBITS (elemental), CHDIR, CPU_TIME, DATE_AND_TIME, EVENT_QUERY, EXECUTE_COMMAND_LINE, GET_COMMAND, GET_COMMAND_ARGUMENT, GET_ENVIRONMENT_VARIABLE, MOVE_ALLOC, RANDOM_INIT, RANDOM_NUMBER, RANDOM_SEED, SIGNAL, SLEEP, SYSTEM, SYSTEM_CLOCK |
 | Atomic intrinsic subroutines | ATOMIC_ADD |
 | Collective intrinsic subroutines | CO_REDUCE |
 | Library subroutines | BACKTRACE, FDATE, GETLOG, GETENV |
@@ -1064,3 +1064,45 @@ This intrinsic is an alias for `LEN_TRIM`, without the optional KIND argument.
 - **Arguments:** `TIME` - a REAL value into which the elapsed CPU time in
                           seconds is written
 - **RETURN value:** same as TIME argument
+
+### Non-Standard Intrinsics: CHDIR
+
+#### Description
+`CHDIR(NAME[, STATUS])` Change current working directory to a specified path.
+
+This intrinsic is provided in both subroutine and function forms; however, only one form can be used in any given program unit.
+*STATUS* is `INTENT(OUT)` and provide the following:
+
+|            |                                                                                                   |
+|------------|---------------------------------------------------------------------------------------------------|
+| `NAME`     | The type shall be `CHARACTER` of default kind and shall specify a valid path within the file system.       |
+| `STATUS`   | (Optional) Status flag. Returns 0 on success, a system specific and nonzero error code otherwise. The type shall be `INTEGER` and of the default kind. |
+
+#### Usage and Info
+
+- **Standard:** GNU extension
+- **Class:** Subroutine, function
+- **Syntax:** `CALL CHDIR(NAME[, STATUS])` and `STATUS = CHDIR(NAME)`
+
+#### Example
+```Fortran
+program chdir_func
+  character(len=) :: path
+  integer :: status 
+
+  call chdir("/tmp")
+  status = chdir("..")
+  print *, "status: ", status
+end program chdir_func
+```
+
+### Non-Standard Intrinsics: IERRNO
+
+#### Description
+`IERRNO()` returns the last system error number, as given by the C `errno` variable.
+
+#### Usage and Info
+
+- **Standard:** GNU extension
+- **Class:** function
+- **Syntax:** `RESULT = IERRNO()`
diff --git a/flang/docs/ModFiles.md b/flang/docs/ModFiles.md
index 7463454c8563a..a4c2395d308fb 100644
--- a/flang/docs/ModFiles.md
+++ b/flang/docs/ModFiles.md
@@ -164,3 +164,13 @@ a diagnostic but we still wouldn't have line numbers.
 To provide line numbers and character positions or source lines as the user
 wrote them we would have to save some amount of provenance information in the
 module file as well.
+
+## Hermetic modules files
+
+Top-level module files for libraries can be build with `-fhermetic-module-files`.
+This option causes these module files to contain copies of all of the non-intrinsic
+modules on which they depend, so that non-top-level local modules and the
+modules of dependent libraries need not also be packaged with the library.
+When the compiler reads a hermetic module file, the copies of the dependent
+modules are read into their own scope, and will not conflict with other modules
+of the same name that client code might `USE`.
diff --git a/flang/docs/OpenACC-descriptor-management.md b/flang/docs/OpenACC-descriptor-management.md
index 52d00ae4daef8..008c57937e23b 100644
--- a/flang/docs/OpenACC-descriptor-management.md
+++ b/flang/docs/OpenACC-descriptor-management.md
@@ -427,7 +427,7 @@ The implementation's behavior may be described as (OpenACC 2.7.2):
 
 All the "is-present" checks and the data actions for the auxiliary pointers must be performed atomically with regards to the present counters bookkeeping.
 
-The API relies on the primitives provided by `liboffload`, so it is provided by a new F18 runtime library, e.g. `FortranOffloadRuntime`, that depends on `FortranRuntime` and `liboffload`.  The F18 driver adds `FortranOffloadRuntime` for linking under `-fopenacc`/`-fopenmp` (and maybe additional switches like `-fopenmp-targets`).
+The API relies on the primitives provided by `liboffload`, so it is provided by a new F18 runtime library, e.g. `FortranOffloadRuntime`, that depends on `flang_rt.runtime` and `liboffload`.  The F18 driver adds `FortranOffloadRuntime` for linking under `-fopenacc`/`-fopenmp` (and maybe additional switches like `-fopenmp-targets`).
 
 ## TODOs:
 
diff --git a/flang/docs/Real16MathSupport.md b/flang/docs/Real16MathSupport.md
index 21482c7be21af..93492c8b767c3 100644
--- a/flang/docs/Real16MathSupport.md
+++ b/flang/docs/Real16MathSupport.md
@@ -12,9 +12,9 @@ To support most `REAL(16)` (i.e. 128-bit float) math intrinsics Flang relies
 on third-party libraries providing the implementation.
 
 `-DFLANG_RUNTIME_F128_MATH_LIB=libquadmath` CMake option can be used
-to build `FortranFloat128Math` library that has unresolved references
+to build `libflang_rt.quadmath` library that has unresolved references
 to GCC `libquadmath` library. A Flang driver built with this option
-will automatically link `FortranFloat128Math` and `libquadmath` libraries
+will automatically link `libflang_rt.quadmath` and `libquadmath` libraries
 to any Fortran program. This implies that `libquadmath` library
 has to be available in the standard library paths, so that linker
 can find it. The `libquadmath` library installation into Flang project
diff --git a/flang/docs/ReleaseNotes.md b/flang/docs/ReleaseNotes.md
index f0c956281915f..387d4b2e62e0f 100644
--- a/flang/docs/ReleaseNotes.md
+++ b/flang/docs/ReleaseNotes.md
@@ -36,6 +36,13 @@ page](https://llvm.org/releases/).
 
 ## Build System Changes
 
+ * The FortranRuntime library has been renamed to `flang_rt.runtime`.
+
+ * The FortranFloat128Math library has been renamed to `flang_rt.quadmath`.
+
+ * The CufRuntime_cuda_${version} library has been renamed to
+   `flang_rt.cuda_${version}`.
+
 ## New Issues Found
 
 
diff --git a/flang/examples/CMakeLists.txt b/flang/examples/CMakeLists.txt
index 8cc66ddbbbb0e..23c6e790791fb 100644
--- a/flang/examples/CMakeLists.txt
+++ b/flang/examples/CMakeLists.txt
@@ -1,4 +1,6 @@
-add_subdirectory(ExternalHelloWorld)
+if (FLANG_INCLUDE_RUNTIME)
+  add_subdirectory(ExternalHelloWorld)
+endif ()
 add_subdirectory(PrintFlangFunctionNames)
 add_subdirectory(FlangOmpReport)
 add_subdirectory(FeatureList)
diff --git a/flang/examples/ExternalHelloWorld/CMakeLists.txt b/flang/examples/ExternalHelloWorld/CMakeLists.txt
index 042d4b6238ba4..b61948718a5e3 100644
--- a/flang/examples/ExternalHelloWorld/CMakeLists.txt
+++ b/flang/examples/ExternalHelloWorld/CMakeLists.txt
@@ -5,5 +5,5 @@ add_llvm_example(external-hello-world
 
 target_link_libraries(external-hello-world
   PRIVATE
-  FortranRuntime
+  flang_rt.runtime
   )
diff --git a/flang/examples/FeatureList/FeatureList.cpp b/flang/examples/FeatureList/FeatureList.cpp
index 3a689c335c81c..e35f120d8661e 100644
--- a/flang/examples/FeatureList/FeatureList.cpp
+++ b/flang/examples/FeatureList/FeatureList.cpp
@@ -514,7 +514,6 @@ struct NodeVisitor {
   READ_FEATURE(OmpReductionClause)
   READ_FEATURE(OmpInReductionClause)
   READ_FEATURE(OmpReductionCombiner)
-  READ_FEATURE(OmpReductionCombiner::FunctionCombiner)
   READ_FEATURE(OmpReductionInitializerClause)
   READ_FEATURE(OmpReductionIdentifier)
   READ_FEATURE(OmpAllocateClause)
diff --git a/flang/examples/FlangOmpReport/FlangOmpReportVisitor.cpp b/flang/examples/FlangOmpReport/FlangOmpReportVisitor.cpp
index c78dd7f14e503..b0a632247fe10 100644
--- a/flang/examples/FlangOmpReport/FlangOmpReportVisitor.cpp
+++ b/flang/examples/FlangOmpReport/FlangOmpReportVisitor.cpp
@@ -112,6 +112,10 @@ std::string OpenMPCounterVisitor::getName(const OpenMPDeclarativeConstruct &c) {
             const CharBlock &source{o.source};
             return normalize_construct_name(source.ToString());
           },
+          [&](const OmpMetadirectiveDirective &o) -> std::string {
+            const CharBlock &source{o.source};
+            return normalize_construct_name(source.ToString());
+          },
           [&](const auto &o) -> std::string {
             const CharBlock &source{std::get<Verbatim>(o.t).source};
             return normalize_construct_name(source.ToString());
diff --git a/flang/include/flang/Common/ISO_Fortran_binding_wrapper.h b/flang/include/flang/Common/ISO_Fortran_binding_wrapper.h
new file mode 100644
index 0000000000000..890522953efe0
--- /dev/null
+++ b/flang/include/flang/Common/ISO_Fortran_binding_wrapper.h
@@ -0,0 +1,38 @@
+/*===-- include/flang/Common/ISO_Fortran_binding_wrapper.h --------*- C++ -*-===
+ *
+ * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+ * See https://llvm.org/LICENSE.txt for license information.
+ * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+ *
+ *===----------------------------------------------------------------------===*/
+
+#ifndef FORTRAN_COMMON_ISO_FORTRAN_BINDING_WRAPPER_H_
+#define FORTRAN_COMMON_ISO_FORTRAN_BINDING_WRAPPER_H_
+
+/* A thin wrapper around flang/include/ISO_Fortran_binding.h
+ * This header file must be included when ISO_Fortran_binding.h
+ * definitions/declarations are needed in Flang compiler/runtime
+ * sources. The inclusion of Common/api-attrs.h below sets up
+ * proper values for the macros used in ISO_Fortran_binding.h
+ * for the device offload builds.
+ * flang/include/ISO_Fortran_binding.h is made a standalone
+ * header file so that it can be used on its own in users'
+ * C/C++ programs.
+ */
+
+/* clang-format off */
+#include <stddef.h>
+#include "api-attrs.h"
+#ifdef __cplusplus
+namespace Fortran {
+namespace ISO {
+#define FORTRAN_ISO_NAMESPACE_ ::Fortran::ISO
+#endif /* __cplusplus */
+#include "flang/ISO_Fortran_binding.h"
+#ifdef __cplusplus
+} // namespace ISO
+} // namespace Fortran
+#endif /* __cplusplus */
+/* clang-format on */
+
+#endif /* FORTRAN_COMMON_ISO_FORTRAN_BINDING_WRAPPER_H_ */
diff --git a/flang/include/flang/Common/erfc-scaled.h b/flang/include/flang/Common/erfc-scaled.h
index a1bf3ea0f0925..88e2183c15d03 100644
--- a/flang/include/flang/Common/erfc-scaled.h
+++ b/flang/include/flang/Common/erfc-scaled.h
@@ -9,6 +9,9 @@
 #ifndef FORTRAN_COMMON_ERFC_SCALED_H_
 #define FORTRAN_COMMON_ERFC_SCALED_H_
 
+#include <cmath>
+#include <limits>
+
 namespace Fortran::common {
 template <typename T> inline T ErfcScaled(T arg) {
   // Coefficients for approximation to erfc in the first interval.
diff --git a/flang/include/flang/Common/fast-int-set.h b/flang/include/flang/Common/fast-int-set.h
index d1c9c756a44a6..1214bac75a4bf 100644
--- a/flang/include/flang/Common/fast-int-set.h
+++ b/flang/include/flang/Common/fast-int-set.h
@@ -24,7 +24,7 @@
 #ifndef FORTRAN_COMMON_FAST_INT_SET_H_
 #define FORTRAN_COMMON_FAST_INT_SET_H_
 
-#include <optional>
+#include "optional.h"
 
 namespace Fortran::common {
 
@@ -83,9 +83,9 @@ template <int N> class FastIntSet {
     }
   }
 
-  std::optional<int> PopValue() {
+  optional<int> PopValue() {
     if (IsEmpty()) {
-      return std::nullopt;
+      return nullopt;
     } else {
       return value_[--size_];
     }
diff --git a/flang/include/flang/Common/format.h b/flang/include/flang/Common/format.h
index c5e9fb06e260f..da416506ffb5d 100644
--- a/flang/include/flang/Common/format.h
+++ b/flang/include/flang/Common/format.h
@@ -9,8 +9,8 @@
 #ifndef FORTRAN_COMMON_FORMAT_H_
 #define FORTRAN_COMMON_FORMAT_H_
 
+#include "Fortran-consts.h"
 #include "enum-set.h"
-#include "flang/Common/Fortran-consts.h"
 #include <cstring>
 
 // Define a FormatValidator class template to validate a format expression
diff --git a/flang/include/flang/Common/optional.h b/flang/include/flang/Common/optional.h
index c0f4278009f40..c7c81f40cc8c8 100644
--- a/flang/include/flang/Common/optional.h
+++ b/flang/include/flang/Common/optional.h
@@ -26,7 +26,7 @@
 #ifndef FORTRAN_COMMON_OPTIONAL_H
 #define FORTRAN_COMMON_OPTIONAL_H
 
-#include "flang/Common/api-attrs.h"
+#include "api-attrs.h"
 #include <optional>
 #include <type_traits>
 
diff --git a/flang/include/flang/Common/real.h b/flang/include/flang/Common/real.h
index b527deda0e3b4..b47ba46581db6 100644
--- a/flang/include/flang/Common/real.h
+++ b/flang/include/flang/Common/real.h
@@ -13,7 +13,7 @@
 // The various representations are distinguished by their binary precisions
 // (number of explicit significand bits and any implicit MSB in the fraction).
 
-#include "flang/Common/api-attrs.h"
+#include "api-attrs.h"
 #include <cinttypes>
 
 namespace Fortran::common {
diff --git a/flang/include/flang/Common/reference-wrapper.h b/flang/include/flang/Common/reference-wrapper.h
index 2983754108f95..85b2c6d5abce3 100644
--- a/flang/include/flang/Common/reference-wrapper.h
+++ b/flang/include/flang/Common/reference-wrapper.h
@@ -25,7 +25,7 @@
 #ifndef FORTRAN_COMMON_REFERENCE_WRAPPER_H
 #define FORTRAN_COMMON_REFERENCE_WRAPPER_H
 
-#include "flang/Common/api-attrs.h"
+#include "api-attrs.h"
 #include <functional>
 #include <type_traits>
 
diff --git a/flang/include/flang/Common/restorer.h b/flang/include/flang/Common/restorer.h
index 0f1bc48620d37..c18c13e351c76 100644
--- a/flang/include/flang/Common/restorer.h
+++ b/flang/include/flang/Common/restorer.h
@@ -18,8 +18,8 @@
 
 #ifndef FORTRAN_COMMON_RESTORER_H_
 #define FORTRAN_COMMON_RESTORER_H_
+#include "api-attrs.h"
 #include "idioms.h"
-#include "flang/Common/api-attrs.h"
 namespace Fortran::common {
 template <typename A> class Restorer {
 public:
diff --git a/flang/include/flang/Common/target-rounding.h b/flang/include/flang/Common/target-rounding.h
index c0c9f6c49b26a..f503b22930ed4 100644
--- a/flang/include/flang/Common/target-rounding.h
+++ b/flang/include/flang/Common/target-rounding.h
@@ -9,8 +9,8 @@
 #ifndef FORTRAN_COMMON_TARGET_ROUNDING_H_
 #define FORTRAN_COMMON_TARGET_ROUNDING_H_
 
-#include "flang/Common/Fortran-consts.h"
-#include "flang/Common/enum-set.h"
+#include "Fortran-consts.h"
+#include "enum-set.h"
 
 namespace Fortran::common {
 
diff --git a/flang/include/flang/Common/template.h b/flang/include/flang/Common/template.h
index 51d09fb42ce36..6501994133759 100644
--- a/flang/include/flang/Common/template.h
+++ b/flang/include/flang/Common/template.h
@@ -9,8 +9,8 @@
 #ifndef FORTRAN_COMMON_TEMPLATE_H_
 #define FORTRAN_COMMON_TEMPLATE_H_
 
+#include "idioms.h"
 #include "variant.h"
-#include "flang/Common/idioms.h"
 #include <functional>
 #include <optional>
 #include <tuple>
diff --git a/flang/include/flang/Common/uint128.h b/flang/include/flang/Common/uint128.h
index 821c8c3b08a52..faecb88737a0b 100644
--- a/flang/include/flang/Common/uint128.h
+++ b/flang/include/flang/Common/uint128.h
@@ -19,8 +19,8 @@
 #define AVOID_NATIVE_UINT128_T 0
 #endif
 
+#include "api-attrs.h"
 #include "leading-zero-bit-count.h"
-#include "flang/Common/api-attrs.h"
 #include <cstdint>
 #include <type_traits>
 
diff --git a/flang/include/flang/Common/visit.h b/flang/include/flang/Common/visit.h
index ad66297650b09..1a84102b56642 100644
--- a/flang/include/flang/Common/visit.h
+++ b/flang/include/flang/Common/visit.h
@@ -21,8 +21,8 @@
 #ifndef FORTRAN_COMMON_VISIT_H_
 #define FORTRAN_COMMON_VISIT_H_
 
+#include "api-attrs.h"
 #include "variant.h"
-#include "flang/Common/api-attrs.h"
 #include <type_traits>
 
 namespace Fortran::common {
diff --git a/flang/include/flang/Evaluate/call.h b/flang/include/flang/Evaluate/call.h
index 63277438128eb..2a5929b873d74 100644
--- a/flang/include/flang/Evaluate/call.h
+++ b/flang/include/flang/Evaluate/call.h
@@ -13,11 +13,11 @@
 #include "constant.h"
 #include "formatting.h"
 #include "type.h"
-#include "flang/Common/Fortran.h"
 #include "flang/Common/indirection.h"
 #include "flang/Common/reference.h"
 #include "flang/Parser/char-block.h"
 #include "flang/Semantics/attr.h"
+#include "flang/Support/Fortran.h"
 #include <optional>
 #include <vector>
 
diff --git a/flang/include/flang/Evaluate/characteristics.h b/flang/include/flang/Evaluate/characteristics.h
index 357fc3e595243..3aa980db5d931 100644
--- a/flang/include/flang/Evaluate/characteristics.h
+++ b/flang/include/flang/Evaluate/characteristics.h
@@ -18,13 +18,13 @@
 #include "shape.h"
 #include "tools.h"
 #include "type.h"
-#include "flang/Common/Fortran-features.h"
-#include "flang/Common/Fortran.h"
 #include "flang/Common/enum-set.h"
 #include "flang/Common/idioms.h"
 #include "flang/Common/indirection.h"
 #include "flang/Parser/char-block.h"
 #include "flang/Semantics/symbol.h"
+#include "flang/Support/Fortran-features.h"
+#include "flang/Support/Fortran.h"
 #include <optional>
 #include <string>
 #include <variant>
@@ -60,8 +60,7 @@ bool ShapesAreCompatible(const std::optional<Shape> &,
 
 class TypeAndShape {
 public:
-  ENUM_CLASS(
-      Attr, AssumedRank, AssumedShape, AssumedSize, DeferredShape, Coarray)
+  ENUM_CLASS(Attr, AssumedRank, AssumedShape, AssumedSize, DeferredShape)
   using Attrs = common::EnumSet<Attr, Attr_enumSize>;
 
   explicit TypeAndShape(DynamicType t) : type_{t}, shape_{Shape{}} {
@@ -103,9 +102,6 @@ class TypeAndShape {
     if (auto type{x.GetType()}) {
       TypeAndShape result{*type, GetShape(context, x, invariantOnly)};
       result.corank_ = GetCorank(x);
-      if (result.corank_ > 0) {
-        result.attrs_.set(Attr::Coarray);
-      }
       if (type->category() == TypeCategory::Character) {
         if (const auto *chExpr{UnwrapExpr<Expr<SomeCharacter>>(x)}) {
           if (auto length{chExpr->LEN()}) {
@@ -179,14 +175,14 @@ class TypeAndShape {
   const std::optional<Shape> &shape() const { return shape_; }
   const Attrs &attrs() const { return attrs_; }
   int corank() const { return corank_; }
+  void set_corank(int n) { corank_ = n; }
 
   // Return -1 for assumed-rank as a safety.
   int Rank() const { return shape_ ? GetRank(*shape_) : -1; }
 
   // Can sequence association apply to this argument?
   bool CanBeSequenceAssociated() const {
-    constexpr Attrs notAssumedOrExplicitShape{
-        ~Attrs{Attr::AssumedSize, Attr::Coarray}};
+    constexpr Attrs notAssumedOrExplicitShape{~Attrs{Attr::AssumedSize}};
     return Rank() > 0 && (attrs() & notAssumedOrExplicitShape).none();
   }
 
diff --git a/flang/include/flang/Evaluate/check-expression.h b/flang/include/flang/Evaluate/check-expression.h
index 49b64468ffaa7..7eee1994a888c 100644
--- a/flang/include/flang/Evaluate/check-expression.h
+++ b/flang/include/flang/Evaluate/check-expression.h
@@ -99,29 +99,44 @@ extern template void CheckSpecificationExpr(
     FoldingContext &, bool forElementalFunctionResult);
 
 // Contiguity & "simple contiguity" (9.5.4)
+// Named constant sections are expressions, and as such their evaluation is
+// considered to be contiguous. This avoids funny situations where
+// IS_CONTIGUOUS(cst(1:10:2)) would fold to true because `cst(1:10:2)` is
+// folded into an array constructor literal, but IS_CONTIGUOUS(cst(i:i+9:2))
+// folds to false because the named constant reference cannot be folded.
+// Note that these IS_CONTIGUOUS usages are not portable (can probably be
+// considered to fall into F2023 8.5.7 (4)), and existing compilers are not
+// consistent here.
+// However, the compiler may very well decide to create a descriptor over
+// `cst(i:i+9:2)` when it can to avoid copies, and as such it needs internally
+// to be able to tell the actual contiguity of that array section over the
+// read-only data.
 template <typename A>
-std::optional<bool> IsContiguous(const A &, FoldingContext &);
+std::optional<bool> IsContiguous(const A &, FoldingContext &,
+    bool namedConstantSectionsAreContiguous = true);
+extern template std::optional<bool> IsContiguous(const Expr<SomeType> &,
+    FoldingContext &, bool namedConstantSectionsAreContiguous);
+extern template std::optional<bool> IsContiguous(const ArrayRef &,
+    FoldingContext &, bool namedConstantSectionsAreContiguous);
+extern template std::optional<bool> IsContiguous(const Substring &,
+    FoldingContext &, bool namedConstantSectionsAreContiguous);
+extern template std::optional<bool> IsContiguous(const Component &,
+    FoldingContext &, bool namedConstantSectionsAreContiguous);
+extern template std::optional<bool> IsContiguous(const ComplexPart &,
+    FoldingContext &, bool namedConstantSectionsAreContiguous);
+extern template std::optional<bool> IsContiguous(const CoarrayRef &,
+    FoldingContext &, bool namedConstantSectionsAreContiguous);
 extern template std::optional<bool> IsContiguous(
-    const Expr<SomeType> &, FoldingContext &);
-extern template std::optional<bool> IsContiguous(
-    const ArrayRef &, FoldingContext &);
-extern template std::optional<bool> IsContiguous(
-    const Substring &, FoldingContext &);
-extern template std::optional<bool> IsContiguous(
-    const Component &, FoldingContext &);
-extern template std::optional<bool> IsContiguous(
-    const ComplexPart &, FoldingContext &);
-extern template std::optional<bool> IsContiguous(
-    const CoarrayRef &, FoldingContext &);
-extern template std::optional<bool> IsContiguous(
-    const Symbol &, FoldingContext &);
-static inline std::optional<bool> IsContiguous(
-    const SymbolRef &s, FoldingContext &c) {
-  return IsContiguous(s.get(), c);
+    const Symbol &, FoldingContext &, bool namedConstantSectionsAreContiguous);
+static inline std::optional<bool> IsContiguous(const SymbolRef &s,
+    FoldingContext &c, bool namedConstantSectionsAreContiguous = true) {
+  return IsContiguous(s.get(), c, namedConstantSectionsAreContiguous);
 }
 template <typename A>
-bool IsSimplyContiguous(const A &x, FoldingContext &context) {
-  return IsContiguous(x, context).value_or(false);
+bool IsSimplyContiguous(const A &x, FoldingContext &context,
+    bool namedConstantSectionsAreContiguous = true) {
+  return IsContiguous(x, context, namedConstantSectionsAreContiguous)
+      .value_or(false);
 }
 
 template <typename A> bool IsErrorExpr(const A &);
diff --git a/flang/include/flang/Evaluate/common.h b/flang/include/flang/Evaluate/common.h
index 915e95169c7f8..993540aebca57 100644
--- a/flang/include/flang/Evaluate/common.h
+++ b/flang/include/flang/Evaluate/common.h
@@ -9,9 +9,6 @@
 #ifndef FORTRAN_EVALUATE_COMMON_H_
 #define FORTRAN_EVALUATE_COMMON_H_
 
-#include "flang/Common/Fortran-features.h"
-#include "flang/Common/Fortran.h"
-#include "flang/Common/default-kinds.h"
 #include "flang/Common/enum-set.h"
 #include "flang/Common/idioms.h"
 #include "flang/Common/indirection.h"
@@ -19,6 +16,9 @@
 #include "flang/Common/target-rounding.h"
 #include "flang/Parser/char-block.h"
 #include "flang/Parser/message.h"
+#include "flang/Support/Fortran-features.h"
+#include "flang/Support/Fortran.h"
+#include "flang/Support/default-kinds.h"
 #include <cinttypes>
 #include <map>
 #include <set>
diff --git a/flang/include/flang/Evaluate/constant.h b/flang/include/flang/Evaluate/constant.h
index 61a814446bbfd..6fc22e3b86aa2 100644
--- a/flang/include/flang/Evaluate/constant.h
+++ b/flang/include/flang/Evaluate/constant.h
@@ -11,8 +11,8 @@
 
 #include "formatting.h"
 #include "type.h"
-#include "flang/Common/default-kinds.h"
 #include "flang/Common/reference.h"
+#include "flang/Support/default-kinds.h"
 #include <map>
 #include <vector>
 
diff --git a/flang/include/flang/Evaluate/expression.h b/flang/include/flang/Evaluate/expression.h
index 04f4406fc8a2c..1203fca8640a6 100644
--- a/flang/include/flang/Evaluate/expression.h
+++ b/flang/include/flang/Evaluate/expression.h
@@ -21,11 +21,11 @@
 #include "formatting.h"
 #include "type.h"
 #include "variable.h"
-#include "flang/Common/Fortran.h"
 #include "flang/Common/idioms.h"
 #include "flang/Common/indirection.h"
 #include "flang/Common/template.h"
 #include "flang/Parser/char-block.h"
+#include "flang/Support/Fortran.h"
 #include <algorithm>
 #include <list>
 #include <tuple>
diff --git a/flang/include/flang/Evaluate/intrinsics.h b/flang/include/flang/Evaluate/intrinsics.h
index 15afb772ae767..dbe1ba7fe7ec1 100644
--- a/flang/include/flang/Evaluate/intrinsics.h
+++ b/flang/include/flang/Evaluate/intrinsics.h
@@ -12,9 +12,9 @@
 #include "call.h"
 #include "characteristics.h"
 #include "type.h"
-#include "flang/Common/default-kinds.h"
 #include "flang/Parser/char-block.h"
 #include "flang/Parser/message.h"
+#include "flang/Support/default-kinds.h"
 #include <memory>
 #include <optional>
 #include <string>
diff --git a/flang/include/flang/Evaluate/shape.h b/flang/include/flang/Evaluate/shape.h
index 3e42ec691158b..f0505cfcdf2d7 100644
--- a/flang/include/flang/Evaluate/shape.h
+++ b/flang/include/flang/Evaluate/shape.h
@@ -71,6 +71,9 @@ template <typename A>
 std::optional<Shape> GetShape(
     FoldingContext &, const A &, bool invariantOnly = true);
 template <typename A>
+std::optional<Shape> GetShape(
+    FoldingContext *, const A &, bool invariantOnly = true);
+template <typename A>
 std::optional<Shape> GetShape(const A &, bool invariantOnly = true);
 
 // The dimension argument to these inquiries is zero-based,
@@ -149,6 +152,8 @@ inline MaybeExtentExpr GetSize(const std::optional<Shape> &maybeShape) {
 // Utility predicate: does an expression reference any implied DO index?
 bool ContainsAnyImpliedDoIndex(const ExtentExpr &);
 
+// GetShape()
+
 class GetShapeHelper
     : public AnyTraverse<GetShapeHelper, std::optional<Shape>> {
 public:
@@ -261,23 +266,27 @@ class GetShapeHelper
 
 template <typename A>
 std::optional<Shape> GetShape(
-    FoldingContext &context, const A &x, bool invariantOnly) {
-  if (auto shape{GetShapeHelper{&context, invariantOnly}(x)}) {
-    return Fold(context, std::move(shape));
+    FoldingContext *context, const A &x, bool invariantOnly) {
+  if (auto shape{GetShapeHelper{context, invariantOnly}(x)}) {
+    if (context) {
+      return Fold(*context, std::move(shape));
+    } else {
+      return shape;
+    }
   } else {
     return std::nullopt;
   }
 }
 
 template <typename A>
-std::optional<Shape> GetShape(const A &x, bool invariantOnly) {
-  return GetShapeHelper{/*context=*/nullptr, invariantOnly}(x);
+std::optional<Shape> GetShape(
+    FoldingContext &context, const A &x, bool invariantOnly) {
+  return GetShape(&context, x, invariantOnly);
 }
 
 template <typename A>
-std::optional<Shape> GetShape(
-    FoldingContext *context, const A &x, bool invariantOnly = true) {
-  return GetShapeHelper{context, invariantOnly}(x);
+std::optional<Shape> GetShape(const A &x, bool invariantOnly) {
+  return GetShape(/*context=*/nullptr, x, invariantOnly);
 }
 
 template <typename A>
diff --git a/flang/include/flang/Evaluate/target.h b/flang/include/flang/Evaluate/target.h
index e07f916b875e0..ead4481c32e12 100644
--- a/flang/include/flang/Evaluate/target.h
+++ b/flang/include/flang/Evaluate/target.h
@@ -12,11 +12,11 @@
 #ifndef FORTRAN_EVALUATE_TARGET_H_
 #define FORTRAN_EVALUATE_TARGET_H_
 
-#include "flang/Common/Fortran.h"
 #include "flang/Common/enum-class.h"
 #include "flang/Common/enum-set.h"
 #include "flang/Common/target-rounding.h"
 #include "flang/Evaluate/common.h"
+#include "flang/Support/Fortran.h"
 #include <cstdint>
 
 namespace Fortran::evaluate {
@@ -143,9 +143,10 @@ class TargetCharacteristics {
   std::string compilerOptionsString_;
   std::string compilerVersionString_;
   IeeeFeatures ieeeFeatures_{IeeeFeature::Denormal, IeeeFeature::Divide,
-      IeeeFeature::Flags, IeeeFeature::Inf, IeeeFeature::Io, IeeeFeature::NaN,
-      IeeeFeature::Rounding, IeeeFeature::Sqrt, IeeeFeature::Standard,
-      IeeeFeature::Subnormal, IeeeFeature::UnderflowControl};
+      IeeeFeature::Flags, IeeeFeature::Halting, IeeeFeature::Inf,
+      IeeeFeature::Io, IeeeFeature::NaN, IeeeFeature::Rounding,
+      IeeeFeature::Sqrt, IeeeFeature::Standard, IeeeFeature::Subnormal,
+      IeeeFeature::UnderflowControl};
 };
 
 } // namespace Fortran::evaluate
diff --git a/flang/include/flang/Evaluate/tools.h b/flang/include/flang/Evaluate/tools.h
index 669efb41b0344..352f6b36458ce 100644
--- a/flang/include/flang/Evaluate/tools.h
+++ b/flang/include/flang/Evaluate/tools.h
@@ -321,28 +321,38 @@ template <typename A> const Symbol *ExtractBareLenParameter(const A &expr) {
 // of a substring or complex part.
 template <typename A>
 common::IfNoLvalue<std::optional<DataRef>, A> ExtractDataRef(
-    const A &, bool intoSubstring, bool intoComplexPart) {
-  return std::nullopt; // default base case
+    const A &x, bool intoSubstring, bool intoComplexPart) {
+  if constexpr (common::HasMember<decltype(x), decltype(DataRef::u)>) {
+    return DataRef{x};
+  } else {
+    return std::nullopt; // default base case
+  }
+}
+
+std::optional<DataRef> ExtractSubstringBase(const Substring &);
+
+inline std::optional<DataRef> ExtractDataRef(const Substring &x,
+    bool intoSubstring = false, bool intoComplexPart = false) {
+  if (intoSubstring) {
+    return ExtractSubstringBase(x);
+  } else {
+    return std::nullopt;
+  }
+}
+inline std::optional<DataRef> ExtractDataRef(const ComplexPart &x,
+    bool intoSubstring = false, bool intoComplexPart = false) {
+  if (intoComplexPart) {
+    return x.complex();
+  } else {
+    return std::nullopt;
+  }
 }
 template <typename T>
 std::optional<DataRef> ExtractDataRef(const Designator<T> &d,
     bool intoSubstring = false, bool intoComplexPart = false) {
   return common::visit(
       [=](const auto &x) -> std::optional<DataRef> {
-        if constexpr (common::HasMember<decltype(x), decltype(DataRef::u)>) {
-          return DataRef{x};
-        }
-        if constexpr (std::is_same_v<std::decay_t<decltype(x)>, Substring>) {
-          if (intoSubstring) {
-            return ExtractSubstringBase(x);
-          }
-        }
-        if constexpr (std::is_same_v<std::decay_t<decltype(x)>, ComplexPart>) {
-          if (intoComplexPart) {
-            return x.complex();
-          }
-        }
-        return std::nullopt; // w/o "else" to dodge bogus g++ 8.1 warning
+        return ExtractDataRef(x, intoSubstring, intoComplexPart);
       },
       d.u);
 }
@@ -376,8 +386,6 @@ std::optional<DataRef> ExtractDataRef(
 std::optional<DataRef> ExtractDataRef(const ActualArgument &,
     bool intoSubstring = false, bool intoComplexPart = false);
 
-std::optional<DataRef> ExtractSubstringBase(const Substring &);
-
 // Predicate: is an expression is an array element reference?
 template <typename T>
 bool IsArrayElement(const Expr<T> &expr, bool intoSubstring = true,
diff --git a/flang/include/flang/Evaluate/type.h b/flang/include/flang/Evaluate/type.h
index 1f9296ac4fea7..cfb162f040e8a 100644
--- a/flang/include/flang/Evaluate/type.h
+++ b/flang/include/flang/Evaluate/type.h
@@ -22,11 +22,11 @@
 #include "integer.h"
 #include "logical.h"
 #include "real.h"
-#include "flang/Common/Fortran-features.h"
-#include "flang/Common/Fortran.h"
 #include "flang/Common/idioms.h"
 #include "flang/Common/real.h"
 #include "flang/Common/template.h"
+#include "flang/Support/Fortran-features.h"
+#include "flang/Support/Fortran.h"
 #include <cinttypes>
 #include <optional>
 #include <string>
@@ -510,6 +510,8 @@ bool AreSameDerivedType(
     const semantics::DerivedTypeSpec &, const semantics::DerivedTypeSpec &);
 bool AreSameDerivedTypeIgnoringTypeParameters(
     const semantics::DerivedTypeSpec &, const semantics::DerivedTypeSpec &);
+bool AreSameDerivedTypeIgnoringSequence(
+    const semantics::DerivedTypeSpec &, const semantics::DerivedTypeSpec &);
 
 // For generating "[extern] template class", &c. boilerplate
 #define EXPAND_FOR_EACH_INTEGER_KIND(M, P, S) \
diff --git a/flang/include/flang/Frontend/CompilerInvocation.h b/flang/include/flang/Frontend/CompilerInvocation.h
index 7d3f0bdf2e510..9e6724be33033 100644
--- a/flang/include/flang/Frontend/CompilerInvocation.h
+++ b/flang/include/flang/Frontend/CompilerInvocation.h
@@ -13,7 +13,6 @@
 #ifndef FORTRAN_FRONTEND_COMPILERINVOCATION_H
 #define FORTRAN_FRONTEND_COMPILERINVOCATION_H
 
-#include "flang/Common/LangOptions.h"
 #include "flang/Frontend/CodeGenOptions.h"
 #include "flang/Frontend/FrontendOptions.h"
 #include "flang/Frontend/PreprocessorOptions.h"
@@ -21,6 +20,7 @@
 #include "flang/Lower/LoweringOptions.h"
 #include "flang/Parser/parsing.h"
 #include "flang/Semantics/semantics.h"
+#include "flang/Support/LangOptions.h"
 #include "mlir/Support/Timing.h"
 #include "clang/Basic/Diagnostic.h"
 #include "clang/Basic/DiagnosticOptions.h"
diff --git a/flang/include/flang/Frontend/FrontendOptions.h b/flang/include/flang/Frontend/FrontendOptions.h
index a4cb021e309d4..0bd2e621813ca 100644
--- a/flang/include/flang/Frontend/FrontendOptions.h
+++ b/flang/include/flang/Frontend/FrontendOptions.h
@@ -13,10 +13,10 @@
 #ifndef FORTRAN_FRONTEND_FRONTENDOPTIONS_H
 #define FORTRAN_FRONTEND_FRONTENDOPTIONS_H
 
-#include "flang/Common/Fortran-features.h"
 #include "flang/Lower/EnvironmentDefault.h"
 #include "flang/Parser/characters.h"
 #include "flang/Parser/unparse.h"
+#include "flang/Support/Fortran-features.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/Support/MemoryBuffer.h"
 #include <cstdint>
diff --git a/flang/include/flang/ISO_Fortran_binding.h b/flang/include/flang/ISO_Fortran_binding.h
index 945f8fef89f0b..f5b8d0d2ea610 100644
--- a/flang/include/flang/ISO_Fortran_binding.h
+++ b/flang/include/flang/ISO_Fortran_binding.h
@@ -14,7 +14,7 @@
  * it does so by means of a wrapper header that establishes namespaces and
  * a macro for extra function attributes (RT_API_ATTRS).
  */
-#ifndef FORTRAN_ISO_FORTRAN_BINDING_WRAPPER_H_
+#ifndef FORTRAN_COMMON_ISO_FORTRAN_BINDING_WRAPPER_H_
 #include <stddef.h>
 #define FORTRAN_ISO_NAMESPACE_
 #endif
diff --git a/flang/include/flang/ISO_Fortran_binding_wrapper.h b/flang/include/flang/ISO_Fortran_binding_wrapper.h
deleted file mode 100644
index 37289bdbabd03..0000000000000
--- a/flang/include/flang/ISO_Fortran_binding_wrapper.h
+++ /dev/null
@@ -1,39 +0,0 @@
-/*===-- include/flang/ISO_Fortran_binding_wrapper.h ---------------*- C++ -*-===
- *
- * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
- * See https://llvm.org/LICENSE.txt for license information.
- * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
- *
- * ===-----------------------------------------------------------------------===
- */
-
-#ifndef FORTRAN_ISO_FORTRAN_BINDING_WRAPPER_H_
-#define FORTRAN_ISO_FORTRAN_BINDING_WRAPPER_H_
-
-/* A thin wrapper around flang/include/ISO_Fortran_binding.h
- * This header file must be included when ISO_Fortran_binding.h
- * definitions/declarations are needed in Flang compiler/runtime
- * sources. The inclusion of Common/api-attrs.h below sets up
- * proper values for the macros used in ISO_Fortran_binding.h
- * for the device offload builds.
- * flang/include/ISO_Fortran_binding.h is made a standalone
- * header file so that it can be used on its own in users'
- * C/C++ programs.
- */
-
-/* clang-format off */
-#include <stddef.h>
-#include "Common/api-attrs.h"
-#ifdef __cplusplus
-namespace Fortran {
-namespace ISO {
-#define FORTRAN_ISO_NAMESPACE_ ::Fortran::ISO
-#endif /* __cplusplus */
-#include "ISO_Fortran_binding.h"
-#ifdef __cplusplus
-} // namespace ISO
-} // namespace Fortran
-#endif /* __cplusplus */
-/* clang-format on */
-
-#endif /* FORTRAN_ISO_FORTRAN_BINDING_WRAPPER_H_ */
diff --git a/flang/include/flang/Lower/AbstractConverter.h b/flang/include/flang/Lower/AbstractConverter.h
index c24f43737df50..3d2b805da6f47 100644
--- a/flang/include/flang/Lower/AbstractConverter.h
+++ b/flang/include/flang/Lower/AbstractConverter.h
@@ -13,12 +13,12 @@
 #ifndef FORTRAN_LOWER_ABSTRACTCONVERTER_H
 #define FORTRAN_LOWER_ABSTRACTCONVERTER_H
 
-#include "flang/Common/Fortran.h"
 #include "flang/Lower/LoweringOptions.h"
 #include "flang/Lower/PFTDefs.h"
 #include "flang/Optimizer/Builder/BoxValue.h"
 #include "flang/Optimizer/Dialect/FIRAttr.h"
 #include "flang/Semantics/symbol.h"
+#include "flang/Support/Fortran.h"
 #include "mlir/IR/Builders.h"
 #include "mlir/IR/BuiltinOps.h"
 #include "mlir/IR/Operation.h"
diff --git a/flang/include/flang/Lower/Bridge.h b/flang/include/flang/Lower/Bridge.h
index 6404a16f7785a..a8c2bcfda31c1 100644
--- a/flang/include/flang/Lower/Bridge.h
+++ b/flang/include/flang/Lower/Bridge.h
@@ -13,7 +13,6 @@
 #ifndef FORTRAN_LOWER_BRIDGE_H
 #define FORTRAN_LOWER_BRIDGE_H
 
-#include "flang/Common/Fortran.h"
 #include "flang/Frontend/CodeGenOptions.h"
 #include "flang/Frontend/TargetOptions.h"
 #include "flang/Lower/AbstractConverter.h"
@@ -22,6 +21,7 @@
 #include "flang/Lower/StatementContext.h"
 #include "flang/Optimizer/Builder/FIRBuilder.h"
 #include "flang/Optimizer/Dialect/Support/KindMapping.h"
+#include "flang/Support/Fortran.h"
 #include "mlir/IR/BuiltinOps.h"
 #include "mlir/IR/OwningOpRef.h"
 #include <set>
diff --git a/flang/include/flang/Lower/ConvertType.h b/flang/include/flang/Lower/ConvertType.h
index 7a3f92649a4e4..179a682584046 100644
--- a/flang/include/flang/Lower/ConvertType.h
+++ b/flang/include/flang/Lower/ConvertType.h
@@ -21,8 +21,8 @@
 #ifndef FORTRAN_LOWER_CONVERT_TYPE_H
 #define FORTRAN_LOWER_CONVERT_TYPE_H
 
-#include "flang/Common/Fortran.h"
 #include "flang/Evaluate/type.h"
+#include "flang/Support/Fortran.h"
 #include "mlir/IR/BuiltinTypes.h"
 
 namespace mlir {
diff --git a/flang/include/flang/Lower/DirectivesCommon.h b/flang/include/flang/Lower/DirectivesCommon.h
index c7cac1357b227..6e24343cebd3a 100644
--- a/flang/include/flang/Lower/DirectivesCommon.h
+++ b/flang/include/flang/Lower/DirectivesCommon.h
@@ -584,10 +584,11 @@ void createEmptyRegionBlocks(
 inline fir::factory::AddrAndBoundsInfo
 getDataOperandBaseAddr(Fortran::lower::AbstractConverter &converter,
                        fir::FirOpBuilder &builder,
-                       Fortran::lower::SymbolRef sym, mlir::Location loc) {
+                       Fortran::lower::SymbolRef sym, mlir::Location loc,
+                       bool unwrapFirBox = true) {
   return fir::factory::getDataOperandBaseAddr(
       builder, converter.getSymbolAddress(sym),
-      Fortran::semantics::IsOptional(sym), loc);
+      Fortran::semantics::IsOptional(sym), loc, unwrapFirBox);
 }
 
 namespace detail {
@@ -880,13 +881,15 @@ fir::factory::AddrAndBoundsInfo gatherDataOperandAddrAndBounds(
     Fortran::semantics::SymbolRef symbol,
     const Fortran::semantics::MaybeExpr &maybeDesignator,
     mlir::Location operandLocation, std::stringstream &asFortran,
-    llvm::SmallVector<mlir::Value> &bounds, bool treatIndexAsSection = false) {
+    llvm::SmallVector<mlir::Value> &bounds, bool treatIndexAsSection = false,
+    bool unwrapFirBox = true, bool genDefaultBounds = true) {
   using namespace Fortran;
 
   fir::factory::AddrAndBoundsInfo info;
 
   if (!maybeDesignator) {
-    info = getDataOperandBaseAddr(converter, builder, symbol, operandLocation);
+    info = getDataOperandBaseAddr(converter, builder, symbol, operandLocation,
+                                  unwrapFirBox);
     asFortran << symbol->name().ToString();
     return info;
   }
@@ -930,7 +933,8 @@ fir::factory::AddrAndBoundsInfo gatherDataOperandAddrAndBounds(
       const semantics::Symbol &sym = arrayRef->GetLastSymbol();
       dataExvIsAssumedSize =
           Fortran::semantics::IsAssumedSizeArray(sym.GetUltimate());
-      info = getDataOperandBaseAddr(converter, builder, sym, operandLocation);
+      info = getDataOperandBaseAddr(converter, builder, sym, operandLocation,
+                                    unwrapFirBox);
       dataExv = converter.getSymbolExtendedValue(sym);
       asFortran << sym.name().ToString();
     }
@@ -947,7 +951,8 @@ fir::factory::AddrAndBoundsInfo gatherDataOperandAddrAndBounds(
         converter.genExprAddr(operandLocation, designator, stmtCtx);
     info.addr = fir::getBase(compExv);
     info.rawInput = info.addr;
-    if (mlir::isa<fir::SequenceType>(fir::unwrapRefType(info.addr.getType())))
+    if (genDefaultBounds &&
+        mlir::isa<fir::SequenceType>(fir::unwrapRefType(info.addr.getType())))
       bounds = fir::factory::genBaseBoundsOps<BoundsOp, BoundsType>(
           builder, operandLocation, compExv,
           /*isAssumedSize=*/false);
@@ -958,14 +963,17 @@ fir::factory::AddrAndBoundsInfo gatherDataOperandAddrAndBounds(
           operandLocation, builder.getI1Type(), info.rawInput);
     }
 
-    if (auto loadOp =
-            mlir::dyn_cast_or_null<fir::LoadOp>(info.addr.getDefiningOp())) {
-      if (fir::isAllocatableType(loadOp.getType()) ||
-          fir::isPointerType(loadOp.getType())) {
-        info.boxType = info.addr.getType();
-        info.addr = builder.create<fir::BoxAddrOp>(operandLocation, info.addr);
+    if (unwrapFirBox) {
+      if (auto loadOp =
+              mlir::dyn_cast_or_null<fir::LoadOp>(info.addr.getDefiningOp())) {
+        if (fir::isAllocatableType(loadOp.getType()) ||
+            fir::isPointerType(loadOp.getType())) {
+          info.boxType = info.addr.getType();
+          info.addr =
+              builder.create<fir::BoxAddrOp>(operandLocation, info.addr);
+        }
+        info.rawInput = info.addr;
       }
-      info.rawInput = info.addr;
     }
 
     // If the component is an allocatable or pointer the result of
@@ -977,8 +985,9 @@ fir::factory::AddrAndBoundsInfo gatherDataOperandAddrAndBounds(
       info.addr = boxAddrOp.getVal();
       info.boxType = info.addr.getType();
       info.rawInput = info.addr;
-      bounds = fir::factory::genBoundsOpsFromBox<BoundsOp, BoundsType>(
-          builder, operandLocation, compExv, info);
+      if (genDefaultBounds)
+        bounds = fir::factory::genBoundsOpsFromBox<BoundsOp, BoundsType>(
+            builder, operandLocation, compExv, info);
     }
   } else {
     if (detail::getRef<evaluate::ArrayRef>(designator)) {
@@ -990,17 +999,18 @@ fir::factory::AddrAndBoundsInfo gatherDataOperandAddrAndBounds(
     } else if (auto symRef = detail::getRef<semantics::SymbolRef>(designator)) {
       // Scalar or full array.
       fir::ExtendedValue dataExv = converter.getSymbolExtendedValue(*symRef);
-      info =
-          getDataOperandBaseAddr(converter, builder, *symRef, operandLocation);
-      if (mlir::isa<fir::BaseBoxType>(
-              fir::unwrapRefType(info.addr.getType()))) {
+      info = getDataOperandBaseAddr(converter, builder, *symRef,
+                                    operandLocation, unwrapFirBox);
+      if (genDefaultBounds && mlir::isa<fir::BaseBoxType>(
+                                  fir::unwrapRefType(info.addr.getType()))) {
         info.boxType = fir::unwrapRefType(info.addr.getType());
         bounds = fir::factory::genBoundsOpsFromBox<BoundsOp, BoundsType>(
             builder, operandLocation, dataExv, info);
       }
       bool dataExvIsAssumedSize =
           Fortran::semantics::IsAssumedSizeArray(symRef->get().GetUltimate());
-      if (mlir::isa<fir::SequenceType>(fir::unwrapRefType(info.addr.getType())))
+      if (genDefaultBounds &&
+          mlir::isa<fir::SequenceType>(fir::unwrapRefType(info.addr.getType())))
         bounds = fir::factory::genBaseBoundsOps<BoundsOp, BoundsType>(
             builder, operandLocation, dataExv, dataExvIsAssumedSize);
       asFortran << symRef->get().name().ToString();
diff --git a/flang/include/flang/Lower/LoweringOptions.h b/flang/include/flang/Lower/LoweringOptions.h
index 8105ccd7ef6b1..171510393b816 100644
--- a/flang/include/flang/Lower/LoweringOptions.h
+++ b/flang/include/flang/Lower/LoweringOptions.h
@@ -15,7 +15,7 @@
 #ifndef FLANG_LOWER_LOWERINGOPTIONS_H
 #define FLANG_LOWER_LOWERINGOPTIONS_H
 
-#include "flang/Common/MathOptionsBase.h"
+#include "flang/Support/MathOptionsBase.h"
 
 namespace Fortran::lower {
 
diff --git a/flang/include/flang/Lower/Support/Utils.h b/flang/include/flang/Lower/Support/Utils.h
index 1cc74521e22d8..baaf644dd6efb 100644
--- a/flang/include/flang/Lower/Support/Utils.h
+++ b/flang/include/flang/Lower/Support/Utils.h
@@ -14,13 +14,13 @@
 #define FORTRAN_LOWER_SUPPORT_UTILS_H
 
 #include "flang/Common/indirection.h"
+#include "flang/Lower/IterationSpace.h"
 #include "flang/Parser/char-block.h"
 #include "flang/Semantics/tools.h"
 #include "mlir/Dialect/Arith/IR/Arith.h"
 #include "mlir/Dialect/Func/IR/FuncOps.h"
 #include "mlir/IR/BuiltinAttributes.h"
 #include "llvm/ADT/StringRef.h"
-#include <cstdint>
 
 namespace Fortran::lower {
 using SomeExpr = Fortran::evaluate::Expr<Fortran::evaluate::SomeType>;
@@ -87,555 +87,13 @@ A flatZip(const A &container1, const A &container2) {
 }
 
 namespace Fortran::lower {
-// Fortran::evaluate::Expr are functional values organized like an AST. A
-// Fortran::evaluate::Expr is meant to be moved and cloned. Using the front end
-// tools can often cause copies and extra wrapper classes to be added to any
-// Fortran::evaluate::Expr. These values should not be assumed or relied upon to
-// have an *object* identity. They are deeply recursive, irregular structures
-// built from a large number of classes which do not use inheritance and
-// necessitate a large volume of boilerplate code as a result.
-//
-// Contrastingly, LLVM data structures make ubiquitous assumptions about an
-// object's identity via pointers to the object. An object's location in memory
-// is thus very often an identifying relation.
-
-// This class defines a hash computation of a Fortran::evaluate::Expr tree value
-// so it can be used with llvm::DenseMap. The Fortran::evaluate::Expr need not
-// have the same address.
-class HashEvaluateExpr {
-public:
-  // A Se::Symbol is the only part of an Fortran::evaluate::Expr with an
-  // identity property.
-  static unsigned getHashValue(const Fortran::semantics::Symbol &x) {
-    return static_cast<unsigned>(reinterpret_cast<std::intptr_t>(&x));
-  }
-  template <typename A, bool COPY>
-  static unsigned getHashValue(const Fortran::common::Indirection<A, COPY> &x) {
-    return getHashValue(x.value());
-  }
-  template <typename A>
-  static unsigned getHashValue(const std::optional<A> &x) {
-    if (x.has_value())
-      return getHashValue(x.value());
-    return 0u;
-  }
-  static unsigned getHashValue(const Fortran::evaluate::Subscript &x) {
-    return Fortran::common::visit(
-        [&](const auto &v) { return getHashValue(v); }, x.u);
-  }
-  static unsigned getHashValue(const Fortran::evaluate::Triplet &x) {
-    return getHashValue(x.lower()) - getHashValue(x.upper()) * 5u -
-           getHashValue(x.stride()) * 11u;
-  }
-  static unsigned getHashValue(const Fortran::evaluate::Component &x) {
-    return getHashValue(x.base()) * 83u - getHashValue(x.GetLastSymbol());
-  }
-  static unsigned getHashValue(const Fortran::evaluate::ArrayRef &x) {
-    unsigned subs = 1u;
-    for (const Fortran::evaluate::Subscript &v : x.subscript())
-      subs -= getHashValue(v);
-    return getHashValue(x.base()) * 89u - subs;
-  }
-  static unsigned getHashValue(const Fortran::evaluate::CoarrayRef &x) {
-    unsigned subs = 1u;
-    for (const Fortran::evaluate::Subscript &v : x.subscript())
-      subs -= getHashValue(v);
-    unsigned cosubs = 3u;
-    for (const Fortran::evaluate::Expr<Fortran::evaluate::SubscriptInteger> &v :
-         x.cosubscript())
-      cosubs -= getHashValue(v);
-    unsigned syms = 7u;
-    for (const Fortran::evaluate::SymbolRef &v : x.base())
-      syms += getHashValue(v);
-    return syms * 97u - subs - cosubs + getHashValue(x.stat()) + 257u +
-           getHashValue(x.team());
-  }
-  static unsigned getHashValue(const Fortran::evaluate::NamedEntity &x) {
-    if (x.IsSymbol())
-      return getHashValue(x.GetFirstSymbol()) * 11u;
-    return getHashValue(x.GetComponent()) * 13u;
-  }
-  static unsigned getHashValue(const Fortran::evaluate::DataRef &x) {
-    return Fortran::common::visit(
-        [&](const auto &v) { return getHashValue(v); }, x.u);
-  }
-  static unsigned getHashValue(const Fortran::evaluate::ComplexPart &x) {
-    return getHashValue(x.complex()) - static_cast<unsigned>(x.part());
-  }
-  template <Fortran::common::TypeCategory TC1, int KIND,
-            Fortran::common::TypeCategory TC2>
-  static unsigned getHashValue(
-      const Fortran::evaluate::Convert<Fortran::evaluate::Type<TC1, KIND>, TC2>
-          &x) {
-    return getHashValue(x.left()) - (static_cast<unsigned>(TC1) + 2u) -
-           (static_cast<unsigned>(KIND) + 5u);
-  }
-  template <int KIND>
-  static unsigned
-  getHashValue(const Fortran::evaluate::ComplexComponent<KIND> &x) {
-    return getHashValue(x.left()) -
-           (static_cast<unsigned>(x.isImaginaryPart) + 1u) * 3u;
-  }
-  template <typename T>
-  static unsigned getHashValue(const Fortran::evaluate::Parentheses<T> &x) {
-    return getHashValue(x.left()) * 17u;
-  }
-  template <Fortran::common::TypeCategory TC, int KIND>
-  static unsigned getHashValue(
-      const Fortran::evaluate::Negate<Fortran::evaluate::Type<TC, KIND>> &x) {
-    return getHashValue(x.left()) - (static_cast<unsigned>(TC) + 5u) -
-           (static_cast<unsigned>(KIND) + 7u);
-  }
-  template <Fortran::common::TypeCategory TC, int KIND>
-  static unsigned getHashValue(
-      const Fortran::evaluate::Add<Fortran::evaluate::Type<TC, KIND>> &x) {
-    return (getHashValue(x.left()) + getHashValue(x.right())) * 23u +
-           static_cast<unsigned>(TC) + static_cast<unsigned>(KIND);
-  }
-  template <Fortran::common::TypeCategory TC, int KIND>
-  static unsigned getHashValue(
-      const Fortran::evaluate::Subtract<Fortran::evaluate::Type<TC, KIND>> &x) {
-    return (getHashValue(x.left()) - getHashValue(x.right())) * 19u +
-           static_cast<unsigned>(TC) + static_cast<unsigned>(KIND);
-  }
-  template <Fortran::common::TypeCategory TC, int KIND>
-  static unsigned getHashValue(
-      const Fortran::evaluate::Multiply<Fortran::evaluate::Type<TC, KIND>> &x) {
-    return (getHashValue(x.left()) + getHashValue(x.right())) * 29u +
-           static_cast<unsigned>(TC) + static_cast<unsigned>(KIND);
-  }
-  template <Fortran::common::TypeCategory TC, int KIND>
-  static unsigned getHashValue(
-      const Fortran::evaluate::Divide<Fortran::evaluate::Type<TC, KIND>> &x) {
-    return (getHashValue(x.left()) - getHashValue(x.right())) * 31u +
-           static_cast<unsigned>(TC) + static_cast<unsigned>(KIND);
-  }
-  template <Fortran::common::TypeCategory TC, int KIND>
-  static unsigned getHashValue(
-      const Fortran::evaluate::Power<Fortran::evaluate::Type<TC, KIND>> &x) {
-    return (getHashValue(x.left()) - getHashValue(x.right())) * 37u +
-           static_cast<unsigned>(TC) + static_cast<unsigned>(KIND);
-  }
-  template <Fortran::common::TypeCategory TC, int KIND>
-  static unsigned getHashValue(
-      const Fortran::evaluate::Extremum<Fortran::evaluate::Type<TC, KIND>> &x) {
-    return (getHashValue(x.left()) + getHashValue(x.right())) * 41u +
-           static_cast<unsigned>(TC) + static_cast<unsigned>(KIND) +
-           static_cast<unsigned>(x.ordering) * 7u;
-  }
-  template <Fortran::common::TypeCategory TC, int KIND>
-  static unsigned getHashValue(
-      const Fortran::evaluate::RealToIntPower<Fortran::evaluate::Type<TC, KIND>>
-          &x) {
-    return (getHashValue(x.left()) - getHashValue(x.right())) * 43u +
-           static_cast<unsigned>(TC) + static_cast<unsigned>(KIND);
-  }
-  template <int KIND>
-  static unsigned
-  getHashValue(const Fortran::evaluate::ComplexConstructor<KIND> &x) {
-    return (getHashValue(x.left()) - getHashValue(x.right())) * 47u +
-           static_cast<unsigned>(KIND);
-  }
-  template <int KIND>
-  static unsigned getHashValue(const Fortran::evaluate::Concat<KIND> &x) {
-    return (getHashValue(x.left()) - getHashValue(x.right())) * 53u +
-           static_cast<unsigned>(KIND);
-  }
-  template <int KIND>
-  static unsigned getHashValue(const Fortran::evaluate::SetLength<KIND> &x) {
-    return (getHashValue(x.left()) - getHashValue(x.right())) * 59u +
-           static_cast<unsigned>(KIND);
-  }
-  static unsigned getHashValue(const Fortran::semantics::SymbolRef &sym) {
-    return getHashValue(sym.get());
-  }
-  static unsigned getHashValue(const Fortran::evaluate::Substring &x) {
-    return 61u *
-               Fortran::common::visit(
-                   [&](const auto &p) { return getHashValue(p); }, x.parent()) -
-           getHashValue(x.lower()) - (getHashValue(x.lower()) + 1u);
-  }
-  static unsigned
-  getHashValue(const Fortran::evaluate::StaticDataObject::Pointer &x) {
-    return llvm::hash_value(x->name());
-  }
-  static unsigned getHashValue(const Fortran::evaluate::SpecificIntrinsic &x) {
-    return llvm::hash_value(x.name);
-  }
-  template <typename A>
-  static unsigned getHashValue(const Fortran::evaluate::Constant<A> &x) {
-    // FIXME: Should hash the content.
-    return 103u;
-  }
-  static unsigned getHashValue(const Fortran::evaluate::ActualArgument &x) {
-    if (const Fortran::evaluate::Symbol *sym = x.GetAssumedTypeDummy())
-      return getHashValue(*sym);
-    return getHashValue(*x.UnwrapExpr());
-  }
-  static unsigned
-  getHashValue(const Fortran::evaluate::ProcedureDesignator &x) {
-    return Fortran::common::visit(
-        [&](const auto &v) { return getHashValue(v); }, x.u);
-  }
-  static unsigned getHashValue(const Fortran::evaluate::ProcedureRef &x) {
-    unsigned args = 13u;
-    for (const std::optional<Fortran::evaluate::ActualArgument> &v :
-         x.arguments())
-      args -= getHashValue(v);
-    return getHashValue(x.proc()) * 101u - args;
-  }
-  template <typename A>
-  static unsigned
-  getHashValue(const Fortran::evaluate::ArrayConstructor<A> &x) {
-    // FIXME: hash the contents.
-    return 127u;
-  }
-  static unsigned getHashValue(const Fortran::evaluate::ImpliedDoIndex &x) {
-    return llvm::hash_value(toStringRef(x.name).str()) * 131u;
-  }
-  static unsigned getHashValue(const Fortran::evaluate::TypeParamInquiry &x) {
-    return getHashValue(x.base()) * 137u - getHashValue(x.parameter()) * 3u;
-  }
-  static unsigned getHashValue(const Fortran::evaluate::DescriptorInquiry &x) {
-    return getHashValue(x.base()) * 139u -
-           static_cast<unsigned>(x.field()) * 13u +
-           static_cast<unsigned>(x.dimension());
-  }
-  static unsigned
-  getHashValue(const Fortran::evaluate::StructureConstructor &x) {
-    // FIXME: hash the contents.
-    return 149u;
-  }
-  template <int KIND>
-  static unsigned getHashValue(const Fortran::evaluate::Not<KIND> &x) {
-    return getHashValue(x.left()) * 61u + static_cast<unsigned>(KIND);
-  }
-  template <int KIND>
-  static unsigned
-  getHashValue(const Fortran::evaluate::LogicalOperation<KIND> &x) {
-    unsigned result = getHashValue(x.left()) + getHashValue(x.right());
-    return result * 67u + static_cast<unsigned>(x.logicalOperator) * 5u;
-  }
-  template <Fortran::common::TypeCategory TC, int KIND>
-  static unsigned getHashValue(
-      const Fortran::evaluate::Relational<Fortran::evaluate::Type<TC, KIND>>
-          &x) {
-    return (getHashValue(x.left()) + getHashValue(x.right())) * 71u +
-           static_cast<unsigned>(TC) + static_cast<unsigned>(KIND) +
-           static_cast<unsigned>(x.opr) * 11u;
-  }
-  template <typename A>
-  static unsigned getHashValue(const Fortran::evaluate::Expr<A> &x) {
-    return Fortran::common::visit(
-        [&](const auto &v) { return getHashValue(v); }, x.u);
-  }
-  static unsigned getHashValue(
-      const Fortran::evaluate::Relational<Fortran::evaluate::SomeType> &x) {
-    return Fortran::common::visit(
-        [&](const auto &v) { return getHashValue(v); }, x.u);
-  }
-  template <typename A>
-  static unsigned getHashValue(const Fortran::evaluate::Designator<A> &x) {
-    return Fortran::common::visit(
-        [&](const auto &v) { return getHashValue(v); }, x.u);
-  }
-  template <int BITS>
-  static unsigned
-  getHashValue(const Fortran::evaluate::value::Integer<BITS> &x) {
-    return static_cast<unsigned>(x.ToSInt());
-  }
-  static unsigned getHashValue(const Fortran::evaluate::NullPointer &x) {
-    return ~179u;
-  }
-};
+unsigned getHashValue(const Fortran::lower::SomeExpr *x);
+unsigned getHashValue(const Fortran::lower::ExplicitIterSpace::ArrayBases &x);
 
-// Define the is equals test for using Fortran::evaluate::Expr values with
-// llvm::DenseMap.
-class IsEqualEvaluateExpr {
-public:
-  // A Se::Symbol is the only part of an Fortran::evaluate::Expr with an
-  // identity property.
-  static bool isEqual(const Fortran::semantics::Symbol &x,
-                      const Fortran::semantics::Symbol &y) {
-    return isEqual(&x, &y);
-  }
-  static bool isEqual(const Fortran::semantics::Symbol *x,
-                      const Fortran::semantics::Symbol *y) {
-    return x == y;
-  }
-  template <typename A, bool COPY>
-  static bool isEqual(const Fortran::common::Indirection<A, COPY> &x,
-                      const Fortran::common::Indirection<A, COPY> &y) {
-    return isEqual(x.value(), y.value());
-  }
-  template <typename A>
-  static bool isEqual(const std::optional<A> &x, const std::optional<A> &y) {
-    if (x.has_value() && y.has_value())
-      return isEqual(x.value(), y.value());
-    return !x.has_value() && !y.has_value();
-  }
-  template <typename A>
-  static bool isEqual(const std::vector<A> &x, const std::vector<A> &y) {
-    if (x.size() != y.size())
-      return false;
-    const std::size_t size = x.size();
-    for (std::remove_const_t<decltype(size)> i = 0; i < size; ++i)
-      if (!isEqual(x[i], y[i]))
-        return false;
-    return true;
-  }
-  static bool isEqual(const Fortran::evaluate::Subscript &x,
-                      const Fortran::evaluate::Subscript &y) {
-    return Fortran::common::visit(
-        [&](const auto &v, const auto &w) { return isEqual(v, w); }, x.u, y.u);
-  }
-  static bool isEqual(const Fortran::evaluate::Triplet &x,
-                      const Fortran::evaluate::Triplet &y) {
-    return isEqual(x.lower(), y.lower()) && isEqual(x.upper(), y.upper()) &&
-           isEqual(x.stride(), y.stride());
-  }
-  static bool isEqual(const Fortran::evaluate::Component &x,
-                      const Fortran::evaluate::Component &y) {
-    return isEqual(x.base(), y.base()) &&
-           isEqual(x.GetLastSymbol(), y.GetLastSymbol());
-  }
-  static bool isEqual(const Fortran::evaluate::ArrayRef &x,
-                      const Fortran::evaluate::ArrayRef &y) {
-    return isEqual(x.base(), y.base()) && isEqual(x.subscript(), y.subscript());
-  }
-  static bool isEqual(const Fortran::evaluate::CoarrayRef &x,
-                      const Fortran::evaluate::CoarrayRef &y) {
-    return isEqual(x.base(), y.base()) &&
-           isEqual(x.subscript(), y.subscript()) &&
-           isEqual(x.cosubscript(), y.cosubscript()) &&
-           isEqual(x.stat(), y.stat()) && isEqual(x.team(), y.team());
-  }
-  static bool isEqual(const Fortran::evaluate::NamedEntity &x,
-                      const Fortran::evaluate::NamedEntity &y) {
-    if (x.IsSymbol() && y.IsSymbol())
-      return isEqual(x.GetFirstSymbol(), y.GetFirstSymbol());
-    return !x.IsSymbol() && !y.IsSymbol() &&
-           isEqual(x.GetComponent(), y.GetComponent());
-  }
-  static bool isEqual(const Fortran::evaluate::DataRef &x,
-                      const Fortran::evaluate::DataRef &y) {
-    return Fortran::common::visit(
-        [&](const auto &v, const auto &w) { return isEqual(v, w); }, x.u, y.u);
-  }
-  static bool isEqual(const Fortran::evaluate::ComplexPart &x,
-                      const Fortran::evaluate::ComplexPart &y) {
-    return isEqual(x.complex(), y.complex()) && x.part() == y.part();
-  }
-  template <typename A, Fortran::common::TypeCategory TC2>
-  static bool isEqual(const Fortran::evaluate::Convert<A, TC2> &x,
-                      const Fortran::evaluate::Convert<A, TC2> &y) {
-    return isEqual(x.left(), y.left());
-  }
-  template <int KIND>
-  static bool isEqual(const Fortran::evaluate::ComplexComponent<KIND> &x,
-                      const Fortran::evaluate::ComplexComponent<KIND> &y) {
-    return isEqual(x.left(), y.left()) &&
-           x.isImaginaryPart == y.isImaginaryPart;
-  }
-  template <typename T>
-  static bool isEqual(const Fortran::evaluate::Parentheses<T> &x,
-                      const Fortran::evaluate::Parentheses<T> &y) {
-    return isEqual(x.left(), y.left());
-  }
-  template <typename A>
-  static bool isEqual(const Fortran::evaluate::Negate<A> &x,
-                      const Fortran::evaluate::Negate<A> &y) {
-    return isEqual(x.left(), y.left());
-  }
-  template <typename A>
-  static bool isBinaryEqual(const A &x, const A &y) {
-    return isEqual(x.left(), y.left()) && isEqual(x.right(), y.right());
-  }
-  template <typename A>
-  static bool isEqual(const Fortran::evaluate::Add<A> &x,
-                      const Fortran::evaluate::Add<A> &y) {
-    return isBinaryEqual(x, y);
-  }
-  template <typename A>
-  static bool isEqual(const Fortran::evaluate::Subtract<A> &x,
-                      const Fortran::evaluate::Subtract<A> &y) {
-    return isBinaryEqual(x, y);
-  }
-  template <typename A>
-  static bool isEqual(const Fortran::evaluate::Multiply<A> &x,
-                      const Fortran::evaluate::Multiply<A> &y) {
-    return isBinaryEqual(x, y);
-  }
-  template <typename A>
-  static bool isEqual(const Fortran::evaluate::Divide<A> &x,
-                      const Fortran::evaluate::Divide<A> &y) {
-    return isBinaryEqual(x, y);
-  }
-  template <typename A>
-  static bool isEqual(const Fortran::evaluate::Power<A> &x,
-                      const Fortran::evaluate::Power<A> &y) {
-    return isBinaryEqual(x, y);
-  }
-  template <typename A>
-  static bool isEqual(const Fortran::evaluate::Extremum<A> &x,
-                      const Fortran::evaluate::Extremum<A> &y) {
-    return isBinaryEqual(x, y);
-  }
-  template <typename A>
-  static bool isEqual(const Fortran::evaluate::RealToIntPower<A> &x,
-                      const Fortran::evaluate::RealToIntPower<A> &y) {
-    return isBinaryEqual(x, y);
-  }
-  template <int KIND>
-  static bool isEqual(const Fortran::evaluate::ComplexConstructor<KIND> &x,
-                      const Fortran::evaluate::ComplexConstructor<KIND> &y) {
-    return isBinaryEqual(x, y);
-  }
-  template <int KIND>
-  static bool isEqual(const Fortran::evaluate::Concat<KIND> &x,
-                      const Fortran::evaluate::Concat<KIND> &y) {
-    return isBinaryEqual(x, y);
-  }
-  template <int KIND>
-  static bool isEqual(const Fortran::evaluate::SetLength<KIND> &x,
-                      const Fortran::evaluate::SetLength<KIND> &y) {
-    return isBinaryEqual(x, y);
-  }
-  static bool isEqual(const Fortran::semantics::SymbolRef &x,
-                      const Fortran::semantics::SymbolRef &y) {
-    return isEqual(x.get(), y.get());
-  }
-  static bool isEqual(const Fortran::evaluate::Substring &x,
-                      const Fortran::evaluate::Substring &y) {
-    return Fortran::common::visit(
-               [&](const auto &p, const auto &q) { return isEqual(p, q); },
-               x.parent(), y.parent()) &&
-           isEqual(x.lower(), y.lower()) && isEqual(x.upper(), y.upper());
-  }
-  static bool isEqual(const Fortran::evaluate::StaticDataObject::Pointer &x,
-                      const Fortran::evaluate::StaticDataObject::Pointer &y) {
-    return x->name() == y->name();
-  }
-  static bool isEqual(const Fortran::evaluate::SpecificIntrinsic &x,
-                      const Fortran::evaluate::SpecificIntrinsic &y) {
-    return x.name == y.name;
-  }
-  template <typename A>
-  static bool isEqual(const Fortran::evaluate::Constant<A> &x,
-                      const Fortran::evaluate::Constant<A> &y) {
-    return x == y;
-  }
-  static bool isEqual(const Fortran::evaluate::ActualArgument &x,
-                      const Fortran::evaluate::ActualArgument &y) {
-    if (const Fortran::evaluate::Symbol *xs = x.GetAssumedTypeDummy()) {
-      if (const Fortran::evaluate::Symbol *ys = y.GetAssumedTypeDummy())
-        return isEqual(*xs, *ys);
-      return false;
-    }
-    return !y.GetAssumedTypeDummy() &&
-           isEqual(*x.UnwrapExpr(), *y.UnwrapExpr());
-  }
-  static bool isEqual(const Fortran::evaluate::ProcedureDesignator &x,
-                      const Fortran::evaluate::ProcedureDesignator &y) {
-    return Fortran::common::visit(
-        [&](const auto &v, const auto &w) { return isEqual(v, w); }, x.u, y.u);
-  }
-  static bool isEqual(const Fortran::evaluate::ProcedureRef &x,
-                      const Fortran::evaluate::ProcedureRef &y) {
-    return isEqual(x.proc(), y.proc()) && isEqual(x.arguments(), y.arguments());
-  }
-  template <typename A>
-  static bool isEqual(const Fortran::evaluate::ArrayConstructor<A> &x,
-                      const Fortran::evaluate::ArrayConstructor<A> &y) {
-    llvm::report_fatal_error("not implemented");
-  }
-  static bool isEqual(const Fortran::evaluate::ImpliedDoIndex &x,
-                      const Fortran::evaluate::ImpliedDoIndex &y) {
-    return toStringRef(x.name) == toStringRef(y.name);
-  }
-  static bool isEqual(const Fortran::evaluate::TypeParamInquiry &x,
-                      const Fortran::evaluate::TypeParamInquiry &y) {
-    return isEqual(x.base(), y.base()) && isEqual(x.parameter(), y.parameter());
-  }
-  static bool isEqual(const Fortran::evaluate::DescriptorInquiry &x,
-                      const Fortran::evaluate::DescriptorInquiry &y) {
-    return isEqual(x.base(), y.base()) && x.field() == y.field() &&
-           x.dimension() == y.dimension();
-  }
-  static bool isEqual(const Fortran::evaluate::StructureConstructor &x,
-                      const Fortran::evaluate::StructureConstructor &y) {
-    const auto &xValues = x.values();
-    const auto &yValues = y.values();
-    if (xValues.size() != yValues.size())
-      return false;
-    if (x.derivedTypeSpec() != y.derivedTypeSpec())
-      return false;
-    for (const auto &[xSymbol, xValue] : xValues) {
-      auto yIt = yValues.find(xSymbol);
-      // This should probably never happen, since the derived type
-      // should be the same.
-      if (yIt == yValues.end())
-        return false;
-      if (!isEqual(xValue, yIt->second))
-        return false;
-    }
-    return true;
-  }
-  template <int KIND>
-  static bool isEqual(const Fortran::evaluate::Not<KIND> &x,
-                      const Fortran::evaluate::Not<KIND> &y) {
-    return isEqual(x.left(), y.left());
-  }
-  template <int KIND>
-  static bool isEqual(const Fortran::evaluate::LogicalOperation<KIND> &x,
-                      const Fortran::evaluate::LogicalOperation<KIND> &y) {
-    return isEqual(x.left(), y.left()) && isEqual(x.right(), y.right());
-  }
-  template <typename A>
-  static bool isEqual(const Fortran::evaluate::Relational<A> &x,
-                      const Fortran::evaluate::Relational<A> &y) {
-    return isEqual(x.left(), y.left()) && isEqual(x.right(), y.right());
-  }
-  template <typename A>
-  static bool isEqual(const Fortran::evaluate::Expr<A> &x,
-                      const Fortran::evaluate::Expr<A> &y) {
-    return Fortran::common::visit(
-        [&](const auto &v, const auto &w) { return isEqual(v, w); }, x.u, y.u);
-  }
-  static bool
-  isEqual(const Fortran::evaluate::Relational<Fortran::evaluate::SomeType> &x,
-          const Fortran::evaluate::Relational<Fortran::evaluate::SomeType> &y) {
-    return Fortran::common::visit(
-        [&](const auto &v, const auto &w) { return isEqual(v, w); }, x.u, y.u);
-  }
-  template <typename A>
-  static bool isEqual(const Fortran::evaluate::Designator<A> &x,
-                      const Fortran::evaluate::Designator<A> &y) {
-    return Fortran::common::visit(
-        [&](const auto &v, const auto &w) { return isEqual(v, w); }, x.u, y.u);
-  }
-  template <int BITS>
-  static bool isEqual(const Fortran::evaluate::value::Integer<BITS> &x,
-                      const Fortran::evaluate::value::Integer<BITS> &y) {
-    return x == y;
-  }
-  static bool isEqual(const Fortran::evaluate::NullPointer &x,
-                      const Fortran::evaluate::NullPointer &y) {
-    return true;
-  }
-  template <typename A, typename B,
-            std::enable_if_t<!std::is_same_v<A, B>, bool> = true>
-  static bool isEqual(const A &, const B &) {
-    return false;
-  }
-};
-
-static inline unsigned getHashValue(const Fortran::lower::SomeExpr *x) {
-  return HashEvaluateExpr::getHashValue(*x);
-}
-
-static bool isEqual(const Fortran::lower::SomeExpr *x,
-                    const Fortran::lower::SomeExpr *y);
+bool isEqual(const Fortran::lower::SomeExpr *x,
+             const Fortran::lower::SomeExpr *y);
+bool isEqual(const Fortran::lower::ExplicitIterSpace::ArrayBases &x,
+             const Fortran::lower::ExplicitIterSpace::ArrayBases &y);
 } // end namespace Fortran::lower
 
 // DenseMapInfo for pointers to Fortran::lower::SomeExpr.
@@ -658,17 +116,4 @@ struct DenseMapInfo<const Fortran::lower::SomeExpr *> {
 };
 } // namespace llvm
 
-namespace Fortran::lower {
-static inline bool isEqual(const Fortran::lower::SomeExpr *x,
-                           const Fortran::lower::SomeExpr *y) {
-  const auto *empty =
-      llvm::DenseMapInfo<const Fortran::lower::SomeExpr *>::getEmptyKey();
-  const auto *tombstone =
-      llvm::DenseMapInfo<const Fortran::lower::SomeExpr *>::getTombstoneKey();
-  if (x == empty || y == empty || x == tombstone || y == tombstone)
-    return x == y;
-  return x == y || IsEqualEvaluateExpr::isEqual(*x, *y);
-}
-} // end namespace Fortran::lower
-
 #endif // FORTRAN_LOWER_SUPPORT_UTILS_H
diff --git a/flang/include/flang/Optimizer/Builder/DirectivesCommon.h b/flang/include/flang/Optimizer/Builder/DirectivesCommon.h
index 443b0ee59007f..c0ab557d26970 100644
--- a/flang/include/flang/Optimizer/Builder/DirectivesCommon.h
+++ b/flang/include/flang/Optimizer/Builder/DirectivesCommon.h
@@ -54,7 +54,8 @@ struct AddrAndBoundsInfo {
 inline AddrAndBoundsInfo getDataOperandBaseAddr(fir::FirOpBuilder &builder,
                                                 mlir::Value symAddr,
                                                 bool isOptional,
-                                                mlir::Location loc) {
+                                                mlir::Location loc,
+                                                bool unwrapFirBox = true) {
   mlir::Value rawInput = symAddr;
   if (auto declareOp =
           mlir::dyn_cast_or_null<hlfir::DeclareOp>(symAddr.getDefiningOp())) {
@@ -80,7 +81,8 @@ inline AddrAndBoundsInfo getDataOperandBaseAddr(fir::FirOpBuilder &builder,
     // all address/dimension retrievals. For Fortran optional though, leave
     // the load generation for later so it can be done in the appropriate
     // if branches.
-    if (mlir::isa<fir::ReferenceType>(symAddr.getType()) && !isOptional) {
+    if (unwrapFirBox && mlir::isa<fir::ReferenceType>(symAddr.getType()) &&
+        !isOptional) {
       mlir::Value addr = builder.create<fir::LoadOp>(loc, symAddr);
       return AddrAndBoundsInfo(addr, rawInput, isPresent, boxTy);
     }
diff --git a/flang/include/flang/Optimizer/Builder/FIRBuilder.h b/flang/include/flang/Optimizer/Builder/FIRBuilder.h
index ea658fb16a36c..93eca78424775 100644
--- a/flang/include/flang/Optimizer/Builder/FIRBuilder.h
+++ b/flang/include/flang/Optimizer/Builder/FIRBuilder.h
@@ -16,12 +16,12 @@
 #ifndef FORTRAN_OPTIMIZER_BUILDER_FIRBUILDER_H
 #define FORTRAN_OPTIMIZER_BUILDER_FIRBUILDER_H
 
-#include "flang/Common/MathOptionsBase.h"
 #include "flang/Optimizer/Dialect/FIROps.h"
 #include "flang/Optimizer/Dialect/FIROpsSupport.h"
 #include "flang/Optimizer/Dialect/FIRType.h"
 #include "flang/Optimizer/Dialect/Support/FIRContext.h"
 #include "flang/Optimizer/Dialect/Support/KindMapping.h"
+#include "flang/Support/MathOptionsBase.h"
 #include "mlir/IR/Builders.h"
 #include "mlir/IR/BuiltinOps.h"
 #include "llvm/ADT/DenseMap.h"
@@ -57,7 +57,13 @@ class FirOpBuilder : public mlir::OpBuilder, public mlir::OpBuilder::Listener {
   explicit FirOpBuilder(mlir::Operation *op, fir::KindMapping kindMap,
                         mlir::SymbolTable *symbolTable = nullptr)
       : OpBuilder{op, /*listener=*/this}, kindMap{std::move(kindMap)},
-        symbolTable{symbolTable} {}
+        symbolTable{symbolTable} {
+    auto fmi = mlir::dyn_cast<mlir::arith::ArithFastMathInterface>(*op);
+    if (fmi) {
+      // Set the builder with FastMathFlags attached to the operation.
+      setFastMathFlags(fmi.getFastMathFlagsAttr().getValue());
+    }
+  }
   explicit FirOpBuilder(mlir::OpBuilder &builder, fir::KindMapping kindMap,
                         mlir::SymbolTable *symbolTable = nullptr)
       : OpBuilder(builder), OpBuilder::Listener(), kindMap{std::move(kindMap)},
@@ -813,6 +819,18 @@ uint64_t getAllocaAddressSpace(mlir::DataLayout *dataLayout);
 llvm::SmallVector<mlir::Value> deduceOptimalExtents(mlir::ValueRange extents1,
                                                     mlir::ValueRange extents2);
 
+/// Given array extents generate code that sets them all to zeroes,
+/// if the array is empty, e.g.:
+///   %false = arith.constant false
+///   %c0 = arith.constant 0 : index
+///   %p1 = arith.cmpi eq, %e0, %c0 : index
+///   %p2 = arith.ori %false, %p1 : i1
+///   %p3 = arith.cmpi eq, %e1, %c0 : index
+///   %p4 = arith.ori %p1, %p2 : i1
+///   %result0 = arith.select %p4, %c0, %e0 : index
+///   %result1 = arith.select %p4, %c0, %e1 : index
+llvm::SmallVector<mlir::Value> updateRuntimeExtentsForEmptyArrays(
+    fir::FirOpBuilder &builder, mlir::Location loc, mlir::ValueRange extents);
 } // namespace fir::factory
 
 #endif // FORTRAN_OPTIMIZER_BUILDER_FIRBUILDER_H
diff --git a/flang/include/flang/Optimizer/Builder/HLFIRTools.h b/flang/include/flang/Optimizer/Builder/HLFIRTools.h
index 0684ad0f926ec..8b1235b50cc6f 100644
--- a/flang/include/flang/Optimizer/Builder/HLFIRTools.h
+++ b/flang/include/flang/Optimizer/Builder/HLFIRTools.h
@@ -125,7 +125,7 @@ class Entity : public mlir::Value {
   bool isSimplyContiguous() const {
     // If this can be described without a fir.box in FIR, this must
     // be contiguous.
-    if (!hlfir::isBoxAddressOrValueType(getFirBase().getType()))
+    if (!hlfir::isBoxAddressOrValueType(getFirBase().getType()) || isScalar())
       return true;
     // Otherwise, if this entity has a visible declaration in FIR,
     // or is the dereference of an allocatable or contiguous pointer
@@ -150,10 +150,7 @@ class Entity : public mlir::Value {
     return base.getDefiningOp<fir::FortranVariableOpInterface>();
   }
 
-  bool isOptional() const {
-    auto varIface = getIfVariableInterface();
-    return varIface ? varIface.isOptional() : false;
-  }
+  bool mayBeOptional() const;
 
   bool isParameter() const {
     auto varIface = getIfVariableInterface();
@@ -210,7 +207,8 @@ class EntityWithAttributes : public Entity {
 using CleanupFunction = std::function<void()>;
 std::pair<fir::ExtendedValue, std::optional<CleanupFunction>>
 translateToExtendedValue(mlir::Location loc, fir::FirOpBuilder &builder,
-                         Entity entity, bool contiguousHint = false);
+                         Entity entity, bool contiguousHint = false,
+                         bool keepScalarOptionalBoxed = false);
 
 /// Function to translate FortranVariableOpInterface to fir::ExtendedValue.
 /// It may generates IR to unbox fir.boxchar, but has otherwise no side effects
diff --git a/flang/include/flang/Optimizer/Builder/IntrinsicCall.h b/flang/include/flang/Optimizer/Builder/IntrinsicCall.h
index e2ea89483ef11..47e8a77fa6aec 100644
--- a/flang/include/flang/Optimizer/Builder/IntrinsicCall.h
+++ b/flang/include/flang/Optimizer/Builder/IntrinsicCall.h
@@ -186,6 +186,13 @@ struct IntrinsicLibrary {
   fir::ExtendedValue genAny(mlir::Type, llvm::ArrayRef<fir::ExtendedValue>);
   mlir::Value genAtanpi(mlir::Type, llvm::ArrayRef<mlir::Value>);
   mlir::Value genAtomicAdd(mlir::Type, llvm::ArrayRef<mlir::Value>);
+  mlir::Value genAtomicAnd(mlir::Type, llvm::ArrayRef<mlir::Value>);
+  mlir::Value genAtomicOr(mlir::Type, llvm::ArrayRef<mlir::Value>);
+  mlir::Value genAtomicDec(mlir::Type, llvm::ArrayRef<mlir::Value>);
+  mlir::Value genAtomicInc(mlir::Type, llvm::ArrayRef<mlir::Value>);
+  mlir::Value genAtomicMax(mlir::Type, llvm::ArrayRef<mlir::Value>);
+  mlir::Value genAtomicMin(mlir::Type, llvm::ArrayRef<mlir::Value>);
+  mlir::Value genAtomicSub(mlir::Type, llvm::ArrayRef<mlir::Value>);
   fir::ExtendedValue
       genCommandArgumentCount(mlir::Type, llvm::ArrayRef<fir::ExtendedValue>);
   mlir::Value genAsind(mlir::Type, llvm::ArrayRef<mlir::Value>);
@@ -203,6 +210,8 @@ struct IntrinsicLibrary {
   mlir::Value genBtest(mlir::Type, llvm::ArrayRef<mlir::Value>);
   mlir::Value genCeiling(mlir::Type, llvm::ArrayRef<mlir::Value>);
   fir::ExtendedValue genChar(mlir::Type, llvm::ArrayRef<fir::ExtendedValue>);
+  fir::ExtendedValue genChdir(std::optional<mlir::Type> resultType,
+                              llvm::ArrayRef<fir::ExtendedValue>);
   template <mlir::arith::CmpIPredicate pred>
   fir::ExtendedValue genCharacterCompare(mlir::Type,
                                          llvm::ArrayRef<fir::ExtendedValue>);
@@ -397,6 +406,7 @@ struct IntrinsicLibrary {
   mlir::Value genSyncThreadsAnd(mlir::Type, llvm::ArrayRef<mlir::Value>);
   mlir::Value genSyncThreadsCount(mlir::Type, llvm::ArrayRef<mlir::Value>);
   mlir::Value genSyncThreadsOr(mlir::Type, llvm::ArrayRef<mlir::Value>);
+  void genSyncWarp(llvm::ArrayRef<fir::ExtendedValue>);
   fir::ExtendedValue genSystem(std::optional<mlir::Type>,
                                mlir::ArrayRef<fir::ExtendedValue> args);
   void genSystemClock(llvm::ArrayRef<fir::ExtendedValue>);
diff --git a/flang/include/flang/Optimizer/Builder/Runtime/Intrinsics.h b/flang/include/flang/Optimizer/Builder/Runtime/Intrinsics.h
index 02b9b68da0db4..51d2dc82f98ae 100644
--- a/flang/include/flang/Optimizer/Builder/Runtime/Intrinsics.h
+++ b/flang/include/flang/Optimizer/Builder/Runtime/Intrinsics.h
@@ -90,6 +90,10 @@ void genSignal(fir::FirOpBuilder &builder, mlir::Location loc,
 void genSleep(fir::FirOpBuilder &builder, mlir::Location loc,
               mlir::Value seconds);
 
+/// generate chdir runtime call
+mlir::Value genChdir(fir::FirOpBuilder &builder, mlir::Location loc,
+                     mlir::Value name);
+
 } // namespace runtime
 } // namespace fir
 
diff --git a/flang/include/flang/Optimizer/Builder/Runtime/Main.h b/flang/include/flang/Optimizer/Builder/Runtime/Main.h
index e4c5dc914c700..a0586deade42a 100644
--- a/flang/include/flang/Optimizer/Builder/Runtime/Main.h
+++ b/flang/include/flang/Optimizer/Builder/Runtime/Main.h
@@ -24,7 +24,8 @@ class GlobalOp;
 namespace fir::runtime {
 
 void genMain(fir::FirOpBuilder &builder, mlir::Location loc,
-             const std::vector<Fortran::lower::EnvironmentDefault> &defs);
+             const std::vector<Fortran::lower::EnvironmentDefault> &defs,
+             bool initCuda = false);
 }
 
 #endif // FORTRAN_OPTIMIZER_BUILDER_RUNTIME_MAIN_H
diff --git a/flang/include/flang/Optimizer/Builder/Runtime/RTBuilder.h b/flang/include/flang/Optimizer/Builder/Runtime/RTBuilder.h
index eaa1de76154d9..1ffc354d6b80f 100644
--- a/flang/include/flang/Optimizer/Builder/Runtime/RTBuilder.h
+++ b/flang/include/flang/Optimizer/Builder/Runtime/RTBuilder.h
@@ -17,12 +17,12 @@
 #ifndef FORTRAN_OPTIMIZER_BUILDER_RUNTIME_RTBUILDER_H
 #define FORTRAN_OPTIMIZER_BUILDER_RUNTIME_RTBUILDER_H
 
-#include "flang/Common/Fortran.h"
 #include "flang/Common/uint128.h"
 #include "flang/Optimizer/Builder/FIRBuilder.h"
 #include "flang/Optimizer/Dialect/FIRDialect.h"
 #include "flang/Optimizer/Dialect/FIRType.h"
 #include "flang/Runtime/reduce.h"
+#include "flang/Support/Fortran.h"
 #include "mlir/IR/BuiltinTypes.h"
 #include "mlir/IR/MLIRContext.h"
 #include "llvm/ADT/SmallVector.h"
@@ -401,6 +401,13 @@ constexpr TypeBuilderFunc getModel<bool &>() {
   };
 }
 template <>
+constexpr TypeBuilderFunc getModel<bool *>() {
+  return [](mlir::MLIRContext *context) -> mlir::Type {
+    TypeBuilderFunc f{getModel<bool>()};
+    return fir::ReferenceType::get(f(context));
+  };
+}
+template <>
 constexpr TypeBuilderFunc getModel<unsigned short>() {
   return [](mlir::MLIRContext *context) -> mlir::Type {
     return mlir::IntegerType::get(
diff --git a/flang/include/flang/Optimizer/CodeGen/DescriptorModel.h b/flang/include/flang/Optimizer/CodeGen/DescriptorModel.h
index 9cccf8db87270..b9a8a670d0793 100644
--- a/flang/include/flang/Optimizer/CodeGen/DescriptorModel.h
+++ b/flang/include/flang/Optimizer/CodeGen/DescriptorModel.h
@@ -22,7 +22,7 @@
 #ifndef OPTIMIZER_DESCRIPTOR_MODEL_H
 #define OPTIMIZER_DESCRIPTOR_MODEL_H
 
-#include "flang/ISO_Fortran_binding_wrapper.h"
+#include "flang/Common/ISO_Fortran_binding_wrapper.h"
 #include "flang/Runtime/descriptor-consts.h"
 #include "mlir/Dialect/LLVMIR/LLVMTypes.h"
 #include "mlir/IR/BuiltinTypes.h"
diff --git a/flang/include/flang/Optimizer/Dialect/CMakeLists.txt b/flang/include/flang/Optimizer/Dialect/CMakeLists.txt
index 10ab213b30b02..73f388cbab6c9 100644
--- a/flang/include/flang/Optimizer/Dialect/CMakeLists.txt
+++ b/flang/include/flang/Optimizer/Dialect/CMakeLists.txt
@@ -37,7 +37,7 @@ set_target_properties(flang-doc PROPERTIES FOLDER "Flang/Docs")
 set(dialect_doc_filename "FIRLangRef")
 
 set(LLVM_TARGET_DEFINITIONS FIROps.td)
-tablegen(MLIR ${dialect_doc_filename}.md -gen-op-doc "-I${MLIR_INCLUDE_DIR}")
+tablegen(MLIR ${dialect_doc_filename}.md -gen-op-doc)
 set(GEN_DOC_FILE ${FLANG_BINARY_DIR}/docs/Dialect/${dialect_doc_filename}.md)
 add_custom_command(
         OUTPUT ${GEN_DOC_FILE}
diff --git a/flang/include/flang/Optimizer/Dialect/CUF/Attributes/CUFAttr.h b/flang/include/flang/Optimizer/Dialect/CUF/Attributes/CUFAttr.h
index f32e39b543e3f..85615a4d69157 100644
--- a/flang/include/flang/Optimizer/Dialect/CUF/Attributes/CUFAttr.h
+++ b/flang/include/flang/Optimizer/Dialect/CUF/Attributes/CUFAttr.h
@@ -13,7 +13,7 @@
 #ifndef FORTRAN_OPTIMIZER_DIALECT_CUF_CUFATTR_H
 #define FORTRAN_OPTIMIZER_DIALECT_CUF_CUFATTR_H
 
-#include "flang/Common/Fortran.h"
+#include "flang/Support/Fortran.h"
 #include "mlir/IR/BuiltinAttributes.h"
 
 namespace llvm {
diff --git a/flang/include/flang/Optimizer/Dialect/CUF/CUFOps.td b/flang/include/flang/Optimizer/Dialect/CUF/CUFOps.td
index a270e69b39410..c1021da0cfb21 100644
--- a/flang/include/flang/Optimizer/Dialect/CUF/CUFOps.td
+++ b/flang/include/flang/Optimizer/Dialect/CUF/CUFOps.td
@@ -207,7 +207,9 @@ def cuf_KernelLaunchOp : cuf_Op<"kernel_launch", [CallOpInterface,
     I32:$block_z,
     Optional<I32>:$bytes,
     Optional<I32>:$stream,
-    Variadic<AnyType>:$args
+    Variadic<AnyType>:$args,
+    OptionalAttr<DictArrayAttr>:$arg_attrs,
+    OptionalAttr<DictArrayAttr>:$res_attrs
   );
 
   let assemblyFormat = [{
diff --git a/flang/include/flang/Optimizer/Dialect/FIROps.td b/flang/include/flang/Optimizer/Dialect/FIROps.td
index 5f0f0b48e892b..8dbc9df9f553d 100644
--- a/flang/include/flang/Optimizer/Dialect/FIROps.td
+++ b/flang/include/flang/Optimizer/Dialect/FIROps.td
@@ -2432,6 +2432,8 @@ def fir_CallOp : fir_Op<"call",
   let arguments = (ins
     OptionalAttr<SymbolRefAttr>:$callee,
     Variadic<AnyType>:$args,
+    OptionalAttr<DictArrayAttr>:$arg_attrs,
+    OptionalAttr<DictArrayAttr>:$res_attrs,
     OptionalAttr<fir_FortranProcedureFlagsAttr>:$procedure_attrs,
     DefaultValuedAttr<Arith_FastMathAttr,
                       "::mlir::arith::FastMathFlags::none">:$fastmath
@@ -2518,6 +2520,8 @@ def fir_DispatchOp : fir_Op<"dispatch", []> {
     fir_ClassType:$object,
     Variadic<AnyType>:$args,
     OptionalAttr<I32Attr>:$pass_arg_pos,
+    OptionalAttr<DictArrayAttr>:$arg_attrs,
+    OptionalAttr<DictArrayAttr>:$res_attrs,
     OptionalAttr<fir_FortranProcedureFlagsAttr>:$procedure_attrs
   );
 
diff --git a/flang/include/flang/Optimizer/HLFIR/HLFIROpBase.td b/flang/include/flang/Optimizer/HLFIR/HLFIROpBase.td
index 404ab5f633bf7..1b1ac61d4550f 100644
--- a/flang/include/flang/Optimizer/HLFIR/HLFIROpBase.td
+++ b/flang/include/flang/Optimizer/HLFIR/HLFIROpBase.td
@@ -125,6 +125,11 @@ def IsFortranNumericalArrayObjectPred
 def AnyFortranNumericalArrayObject : Type<IsFortranNumericalArrayObjectPred,
     "any array-like object containing a numerical type">;
 
+def AnyFortranNumericalArrayEntity
+    : Type<And<[AnyFortranNumericalArrayObject.predicate,
+                AnyFortranEntity.predicate]>,
+           "any array-like entity containing a numerical type">;
+
 def IsFortranNumericalOrLogicalArrayObjectPred
         : CPred<"::hlfir::isFortranNumericalOrLogicalArrayObject($_self)">;
 def AnyFortranNumericalOrLogicalArrayObject : Type<IsFortranNumericalOrLogicalArrayObjectPred,
@@ -135,6 +140,10 @@ def IsFortranArrayObjectPred
 def AnyFortranArrayObject : Type<IsFortranArrayObjectPred,
     "any array-like object">;
 
+def AnyFortranArrayEntity
+    : Type<And<[AnyFortranArrayObject.predicate, AnyFortranEntity.predicate]>,
+           "any array-like entity">;
+
 def IsPassByRefOrIntegerTypePred
         : CPred<"::hlfir::isPassByRefOrIntegerType($_self)">;
 def AnyPassByRefOrIntegerType : Type<IsPassByRefOrIntegerTypePred,
diff --git a/flang/include/flang/Optimizer/HLFIR/HLFIROps.td b/flang/include/flang/Optimizer/HLFIR/HLFIROps.td
index 48764580d526d..f4102538efc3c 100644
--- a/flang/include/flang/Optimizer/HLFIR/HLFIROps.td
+++ b/flang/include/flang/Optimizer/HLFIR/HLFIROps.td
@@ -720,6 +720,32 @@ def hlfir_CShiftOp
   let hasVerifier = 1;
 }
 
+def hlfir_ReshapeOp
+    : hlfir_Op<
+          "reshape", [AttrSizedOperandSegments,
+                      DeclareOpInterfaceMethods<MemoryEffectsOpInterface>]> {
+  let summary = "RESHAPE transformational intrinsic";
+  let description = [{
+    Reshapes an ARRAY to correspond to the given SHAPE.
+    If PAD is specified the new array may be padded with elements
+    from PAD array.
+    If ORDER is specified the new array may be permuted accordingly.
+  }];
+
+  let arguments = (ins AnyFortranArrayEntity:$array,
+      AnyFortranNumericalArrayEntity:$shape,
+      Optional<AnyFortranArrayEntity>:$pad,
+      Optional<AnyFortranNumericalArrayEntity>:$order);
+
+  let results = (outs hlfir_ExprType);
+
+  let assemblyFormat = [{
+    $array $shape (`pad` $pad^)? (`order` $order^)? attr-dict `:` functional-type(operands, results)
+  }];
+
+  let hasVerifier = 1;
+}
+
 // An allocation effect is needed because the value produced by the associate
 // is "deallocated" by hlfir.end_associate (the end_associate must not be
 // removed, and there must be only one hlfir.end_associate).
diff --git a/flang/include/flang/Optimizer/HLFIR/Passes.td b/flang/include/flang/Optimizer/HLFIR/Passes.td
index 90cf6e74241bd..eae0c9ca2e366 100644
--- a/flang/include/flang/Optimizer/HLFIR/Passes.td
+++ b/flang/include/flang/Optimizer/HLFIR/Passes.td
@@ -19,6 +19,11 @@ def ConvertHLFIRtoFIR : Pass<"convert-hlfir-to-fir", "::mlir::ModuleOp"> {
 
 def BufferizeHLFIR : Pass<"bufferize-hlfir", "::mlir::ModuleOp"> {
   let summary = "Convert HLFIR operations operating on hlfir.expr into operations on memory";
+  let options = [Option<"optimizeEmptyElementals", "opt-empty-elementals",
+                        "bool", /*default=*/"false",
+                        "When converting hlfir.elemental into a loop nest, "
+                        "check if the resulting expression is an empty array, "
+                        "and make sure none of the loops is executed.">];
 }
 
 def OptimizedBufferization : Pass<"opt-bufferization"> {
diff --git a/flang/include/flang/Optimizer/Support/DataLayout.h b/flang/include/flang/Optimizer/Support/DataLayout.h
index 6072425b7d637..957ea99162c5b 100644
--- a/flang/include/flang/Optimizer/Support/DataLayout.h
+++ b/flang/include/flang/Optimizer/Support/DataLayout.h
@@ -18,10 +18,14 @@
 
 namespace mlir {
 class ModuleOp;
-}
+namespace gpu {
+class GPUModuleOp;
+} // namespace gpu
+} // namespace mlir
+
 namespace llvm {
 class DataLayout;
-}
+} // namespace llvm
 
 namespace fir::support {
 /// Create an mlir::DataLayoutSpecInterface attribute from an llvm::DataLayout
@@ -30,6 +34,8 @@ namespace fir::support {
 /// the llvm::DataLayout on the module.
 /// These attributes are replaced if they were already set.
 void setMLIRDataLayout(mlir::ModuleOp mlirModule, const llvm::DataLayout &dl);
+void setMLIRDataLayout(mlir::gpu::GPUModuleOp mlirModule,
+                       const llvm::DataLayout &dl);
 
 /// Create an mlir::DataLayoutSpecInterface from the llvm.data_layout attribute
 /// if one is provided. If such attribute is not available, create a default
@@ -37,6 +43,8 @@ void setMLIRDataLayout(mlir::ModuleOp mlirModule, const llvm::DataLayout &dl);
 /// nothing.
 void setMLIRDataLayoutFromAttributes(mlir::ModuleOp mlirModule,
                                      bool allowDefaultLayout);
+void setMLIRDataLayoutFromAttributes(mlir::gpu::GPUModuleOp mlirModule,
+                                     bool allowDefaultLayout);
 
 /// Create mlir::DataLayout from the data layout information on the
 /// mlir::Module. Creates the data layout information attributes with
@@ -44,7 +52,12 @@ void setMLIRDataLayoutFromAttributes(mlir::ModuleOp mlirModule,
 /// information is present at all and \p allowDefaultLayout is false, returns
 /// std::nullopt.
 std::optional<mlir::DataLayout>
-getOrSetDataLayout(mlir::ModuleOp mlirModule, bool allowDefaultLayout = false);
+getOrSetMLIRDataLayout(mlir::ModuleOp mlirModule,
+                       bool allowDefaultLayout = false);
+std::optional<mlir::DataLayout>
+getOrSetMLIRDataLayout(mlir::gpu::GPUModuleOp mlirModule,
+                       bool allowDefaultLayout = false);
+
 } // namespace fir::support
 
 #endif // FORTRAN_OPTIMIZER_SUPPORT_DATALAYOUT_H
diff --git a/flang/include/flang/Optimizer/Support/TypeCode.h b/flang/include/flang/Optimizer/Support/TypeCode.h
index 308c82118d502..ec21d672287e6 100644
--- a/flang/include/flang/Optimizer/Support/TypeCode.h
+++ b/flang/include/flang/Optimizer/Support/TypeCode.h
@@ -13,7 +13,7 @@
 #ifndef FORTRAN_OPTIMIZER_SUPPORT_TYPECODE_H
 #define FORTRAN_OPTIMIZER_SUPPORT_TYPECODE_H
 
-#include "flang/ISO_Fortran_binding_wrapper.h"
+#include "flang/Common/ISO_Fortran_binding_wrapper.h"
 #include "llvm/Support/ErrorHandling.h"
 
 namespace fir {
diff --git a/flang/include/flang/Optimizer/Support/Utils.h b/flang/include/flang/Optimizer/Support/Utils.h
index d507f4116efb7..ec73af6ec72e9 100644
--- a/flang/include/flang/Optimizer/Support/Utils.h
+++ b/flang/include/flang/Optimizer/Support/Utils.h
@@ -13,13 +13,13 @@
 #ifndef FORTRAN_OPTIMIZER_SUPPORT_UTILS_H
 #define FORTRAN_OPTIMIZER_SUPPORT_UTILS_H
 
-#include "flang/Common/default-kinds.h"
 #include "flang/Optimizer/Builder/FIRBuilder.h"
 #include "flang/Optimizer/Builder/Todo.h"
 #include "flang/Optimizer/Dialect/CUF/Attributes/CUFAttr.h"
 #include "flang/Optimizer/Dialect/FIROps.h"
 #include "flang/Optimizer/Dialect/FIRType.h"
 #include "flang/Optimizer/Support/FatalError.h"
+#include "flang/Support/default-kinds.h"
 #include "mlir/Dialect/Arith/IR/Arith.h"
 #include "mlir/Dialect/Func/IR/FuncOps.h"
 #include "mlir/IR/BuiltinAttributes.h"
diff --git a/flang/include/flang/Parser/dump-parse-tree.h b/flang/include/flang/Parser/dump-parse-tree.h
index 11725991e9c9a..21ee1d0517840 100644
--- a/flang/include/flang/Parser/dump-parse-tree.h
+++ b/flang/include/flang/Parser/dump-parse-tree.h
@@ -14,9 +14,9 @@
 #include "parse-tree.h"
 #include "tools.h"
 #include "unparse.h"
-#include "flang/Common/Fortran.h"
 #include "flang/Common/idioms.h"
 #include "flang/Common/indirection.h"
+#include "flang/Support/Fortran.h"
 #include "llvm/Support/raw_ostream.h"
 #include <string>
 #include <type_traits>
@@ -208,6 +208,7 @@ class ParseTreeDumper {
   NODE(CompilerDirective, NameValue)
   NODE(CompilerDirective, Unrecognized)
   NODE(CompilerDirective, VectorAlways)
+  NODE(CompilerDirective, Unroll)
   NODE(parser, ComplexLiteralConstant)
   NODE(parser, ComplexPart)
   NODE(parser, ComponentArraySpec)
@@ -476,10 +477,22 @@ class ParseTreeDumper {
   NODE(parser, NullInit)
   NODE(parser, ObjectDecl)
   NODE(parser, OldParameterStmt)
+  NODE(parser, OmpTypeSpecifier)
+  NODE(parser, OmpTypeNameList)
+  NODE(parser, OmpLocator)
+  NODE(parser, OmpLocatorList)
+  NODE(parser, OmpReductionSpecifier)
+  NODE(parser, OmpArgument)
+  NODE(parser, OmpMetadirectiveDirective)
+  NODE(parser, OmpMatchClause)
+  NODE(parser, OmpOtherwiseClause)
+  NODE(parser, OmpWhenClause)
+  NODE(OmpWhenClause, Modifier)
+  NODE(parser, OmpDirectiveSpecification)
   NODE(parser, OmpTraitPropertyName)
   NODE(parser, OmpTraitScore)
   NODE(parser, OmpTraitPropertyExtension)
-  NODE(OmpTraitPropertyExtension, ExtensionValue)
+  NODE(OmpTraitPropertyExtension, Complex)
   NODE(parser, OmpTraitProperty)
   NODE(parser, OmpTraitSelectorName)
   NODE_ENUM(OmpTraitSelectorName, Value)
@@ -535,7 +548,7 @@ class ParseTreeDumper {
   NODE(parser, OmpDeclareTargetSpecifier)
   NODE(parser, OmpDeclareTargetWithClause)
   NODE(parser, OmpDeclareTargetWithList)
-  NODE(parser, OmpDeclareMapperSpecifier)
+  NODE(parser, OmpMapperSpecifier)
   NODE(parser, OmpDefaultClause)
   NODE_ENUM(OmpDefaultClause, DataSharingAttribute)
   NODE(parser, OmpVariableCategory)
@@ -618,7 +631,6 @@ class ParseTreeDumper {
   NODE(parser, OmpReductionCombiner)
   NODE(parser, OmpTaskReductionClause)
   NODE(OmpTaskReductionClause, Modifier)
-  NODE(OmpReductionCombiner, FunctionCombiner)
   NODE(parser, OmpReductionInitializerClause)
   NODE(parser, OmpReductionIdentifier)
   NODE(parser, OmpAllocateClause)
@@ -679,6 +691,9 @@ class ParseTreeDumper {
   NODE_ENUM(common, OmpAtomicDefaultMemOrderType)
   NODE(parser, OpenMPDepobjConstruct)
   NODE(parser, OpenMPUtilityConstruct)
+  NODE(parser, OpenMPDispatchConstruct)
+  NODE(parser, OmpDispatchDirective)
+  NODE(parser, OmpEndDispatchDirective)
   NODE(parser, OpenMPFlushConstruct)
   NODE(parser, OpenMPLoopConstruct)
   NODE(parser, OpenMPExecutableAllocate)
diff --git a/flang/include/flang/Parser/message.h b/flang/include/flang/Parser/message.h
index bc38f571ca3df..e19b16c23b82b 100644
--- a/flang/include/flang/Parser/message.h
+++ b/flang/include/flang/Parser/message.h
@@ -15,10 +15,10 @@
 #include "char-block.h"
 #include "char-set.h"
 #include "provenance.h"
-#include "flang/Common/Fortran-features.h"
 #include "flang/Common/idioms.h"
 #include "flang/Common/reference-counted.h"
 #include "flang/Common/restorer.h"
+#include "flang/Support/Fortran-features.h"
 #include <cstddef>
 #include <cstring>
 #include <forward_list>
diff --git a/flang/include/flang/Parser/parse-state.h b/flang/include/flang/Parser/parse-state.h
index 76cbb3470dc08..36d70b81b7923 100644
--- a/flang/include/flang/Parser/parse-state.h
+++ b/flang/include/flang/Parser/parse-state.h
@@ -16,11 +16,11 @@
 // and recovery during parsing!
 
 #include "user-state.h"
-#include "flang/Common/Fortran-features.h"
 #include "flang/Common/idioms.h"
 #include "flang/Parser/characters.h"
 #include "flang/Parser/message.h"
 #include "flang/Parser/provenance.h"
+#include "flang/Support/Fortran-features.h"
 #include <cstddef>
 #include <cstring>
 #include <list>
diff --git a/flang/include/flang/Parser/parse-tree.h b/flang/include/flang/Parser/parse-tree.h
index 00d85aa05fb3a..c3a02fca5ade8 100644
--- a/flang/include/flang/Parser/parse-tree.h
+++ b/flang/include/flang/Parser/parse-tree.h
@@ -22,9 +22,9 @@
 #include "format-specification.h"
 #include "message.h"
 #include "provenance.h"
-#include "flang/Common/Fortran.h"
 #include "flang/Common/idioms.h"
 #include "flang/Common/indirection.h"
+#include "flang/Support/Fortran.h"
 #include "llvm/Frontend/OpenACC/ACC.h.inc"
 #include "llvm/Frontend/OpenMP/OMP.h"
 #include "llvm/Frontend/OpenMP/OMPConstants.h"
@@ -3368,10 +3368,13 @@ struct CompilerDirective {
     TUPLE_CLASS_BOILERPLATE(NameValue);
     std::tuple<Name, std::optional<std::uint64_t>> t;
   };
+  struct Unroll {
+    WRAPPER_CLASS_BOILERPLATE(Unroll, std::optional<std::uint64_t>);
+  };
   EMPTY_CLASS(Unrecognized);
   CharBlock source;
   std::variant<std::list<IgnoreTKR>, LoopCount, std::list<AssumeAligned>,
-      VectorAlways, std::list<NameValue>, Unrecognized>
+      VectorAlways, std::list<NameValue>, Unroll, Unrecognized>
       u;
 };
 
@@ -3454,7 +3457,7 @@ WRAPPER_CLASS(PauseStmt, std::optional<StopCode>);
 // --- Common definitions
 
 struct OmpClause;
-struct OmpClauseList;
+struct OmpDirectiveSpecification;
 
 // 2.1 Directives or clauses may accept a list or extended-list.
 //     A list item is a variable, array section or common block name (enclosed
@@ -3467,15 +3470,76 @@ struct OmpObject {
 
 WRAPPER_CLASS(OmpObjectList, std::list<OmpObject>);
 
-#define MODIFIER_BOILERPLATE(...) \
-  struct Modifier { \
-    using Variant = std::variant<__VA_ARGS__>; \
-    UNION_CLASS_BOILERPLATE(Modifier); \
-    CharBlock source; \
-    Variant u; \
-  }
+// Ref: [4.5:201-207], [5.0:293-299], [5.1:325-331], [5.2:124]
+//
+// reduction-identifier ->
+//    base-language-identifier |                    // since 4.5
+//    - |                                           // since 4.5, until 5.2
+//    + | * | .AND. | .OR. | .EQV. | .NEQV. |       // since 4.5
+//    MIN | MAX | IAND | IOR | IEOR                 // since 4.5
+struct OmpReductionIdentifier {
+  UNION_CLASS_BOILERPLATE(OmpReductionIdentifier);
+  std::variant<DefinedOperator, ProcedureDesignator> u;
+};
 
-#define MODIFIERS() std::optional<std::list<Modifier>>
+// Ref: [4.5:222:6], [5.0:305:27], [5.1:337:19], [5.2:126:3-4], [6.0:240:27-28]
+//
+// combiner-expression ->                           // since 4.5
+//    assignment-statement |
+//    function-reference
+struct OmpReductionCombiner {
+  UNION_CLASS_BOILERPLATE(OmpReductionCombiner);
+  std::variant<AssignmentStmt, FunctionReference> u;
+};
+
+inline namespace arguments {
+struct OmpTypeSpecifier {
+  UNION_CLASS_BOILERPLATE(OmpTypeSpecifier);
+  std::variant<TypeSpec, DeclarationTypeSpec> u;
+};
+
+WRAPPER_CLASS(OmpTypeNameList, std::list<OmpTypeSpecifier>);
+
+struct OmpLocator {
+  UNION_CLASS_BOILERPLATE(OmpLocator);
+  std::variant<OmpObject, FunctionReference> u;
+};
+
+WRAPPER_CLASS(OmpLocatorList, std::list<OmpLocator>);
+
+// Ref: [5.0:326:10-16], [5.1:359:5-11], [5.2:163:2-7], [6.0:293:16-21]
+//
+// mapper-specifier ->
+//    [mapper-identifier :] type :: var |           // since 5.0
+//    DEFAULT type :: var
+struct OmpMapperSpecifier {
+  // Absent mapper-identifier is equivalent to DEFAULT.
+  TUPLE_CLASS_BOILERPLATE(OmpMapperSpecifier);
+  std::tuple<std::optional<Name>, TypeSpec, Name> t;
+};
+
+// Ref: [4.5:222:1-5], [5.0:305:20-27], [5.1:337:11-19], [5.2:139:18-23],
+// [6.0:260:16-20]
+//
+// reduction-specifier ->
+//    reduction-identifier : typename-list
+//        : combiner-expression                     // since 4.5, until 5.2
+//    reduction-identifier : typename-list          // since 6.0
+struct OmpReductionSpecifier {
+  TUPLE_CLASS_BOILERPLATE(OmpReductionSpecifier);
+  std::tuple<OmpReductionIdentifier, OmpTypeNameList,
+      std::optional<OmpReductionCombiner>>
+      t;
+};
+
+struct OmpArgument {
+  CharBlock source;
+  UNION_CLASS_BOILERPLATE(OmpArgument);
+  std::variant<OmpLocator, // {variable, extended, locator}-list-item
+      OmpMapperSpecifier, OmpReductionSpecifier>
+      u;
+};
+} // namespace arguments
 
 inline namespace traits {
 // trait-property-name ->
@@ -3505,37 +3569,22 @@ struct OmpTraitScore {
 };
 
 // trait-property-extension ->
-//    trait-property-name (trait-property-value, ...)
-// trait-property-value ->
 //    trait-property-name |
-//    scalar-integer-expression |
-//    trait-property-extension
-//
-// The grammar in OpenMP 5.2+ spec is ambiguous, the above is a different
-// version (but equivalent) that doesn't have ambiguities.
-// The ambiguity is in
-//   trait-property:
-//      trait-property-name          <- (a)
-//      trait-property-clause
-//      trait-property-expression    <- (b)
-//      trait-property-extension     <- this conflicts with (a) and (b)
-//   trait-property-extension:
-//      trait-property-name          <- conflict with (a)
-//      identifier(trait-property-extension[, trait-property-extension[, ...]])
-//      constant integer expression  <- conflict with (b)
+//    scalar-expr |
+//    trait-property-name (trait-property-extension, ...)
 //
 struct OmpTraitPropertyExtension {
   CharBlock source;
-  TUPLE_CLASS_BOILERPLATE(OmpTraitPropertyExtension);
-  struct ExtensionValue {
+  UNION_CLASS_BOILERPLATE(OmpTraitPropertyExtension);
+  struct Complex { // name (prop-ext, prop-ext, ...)
     CharBlock source;
-    UNION_CLASS_BOILERPLATE(ExtensionValue);
-    std::variant<OmpTraitPropertyName, ScalarExpr,
-        common::Indirection<OmpTraitPropertyExtension>>
-        u;
+    TUPLE_CLASS_BOILERPLATE(Complex);
+    std::tuple<OmpTraitPropertyName,
+        std::list<common::Indirection<OmpTraitPropertyExtension>>>
+        t;
   };
-  using ExtensionList = std::list<ExtensionValue>;
-  std::tuple<OmpTraitPropertyName, ExtensionList> t;
+
+  std::variant<OmpTraitPropertyName, ScalarExpr, Complex> u;
 };
 
 // trait-property ->
@@ -3568,18 +3617,20 @@ struct OmpTraitProperty {
 //    UID |               T        // unique-string-id /impl-defined
 //    VENDOR |            I        // name-list (vendor-id /add-def-doc)
 //    EXTENSION |         I        // name-list (ext_name /impl-defined)
-//    ATOMIC_DEFAULT_MEM_ORDER I | // value of admo
+//    ATOMIC_DEFAULT_MEM_ORDER I | // clause-list (value of admo)
 //    REQUIRES |          I        // clause-list (from requires)
 //    CONDITION           U        // logical-expr
+//    <other name>        I        // treated as extension
 //
 // Trait-set-selectors:
 //    [D]evice, [T]arget_device, [C]onstruct, [I]mplementation, [U]ser.
 struct OmpTraitSelectorName {
+  std::string ToString() const;
   CharBlock source;
   UNION_CLASS_BOILERPLATE(OmpTraitSelectorName);
   ENUM_CLASS(Value, Arch, Atomic_Default_Mem_Order, Condition, Device_Num,
       Extension, Isa, Kind, Requires, Simd, Uid, Vendor)
-  std::variant<Value, llvm::omp::Directive> u;
+  std::variant<Value, llvm::omp::Directive, std::string> u;
 };
 
 // trait-selector ->
@@ -3599,6 +3650,7 @@ struct OmpTraitSelector {
 //    CONSTRUCT | DEVICE | IMPLEMENTATION | USER |  // since 5.0
 //    TARGET_DEVICE                                 // since 5.1
 struct OmpTraitSetSelectorName {
+  std::string ToString() const;
   CharBlock source;
   ENUM_CLASS(Value, Construct, Device, Implementation, Target_Device, User)
   WRAPPER_CLASS_BOILERPLATE(OmpTraitSetSelectorName, Value);
@@ -3621,6 +3673,16 @@ struct OmpContextSelectorSpecification { // Modifier
 };
 } // namespace traits
 
+#define MODIFIER_BOILERPLATE(...) \
+  struct Modifier { \
+    using Variant = std::variant<__VA_ARGS__>; \
+    UNION_CLASS_BOILERPLATE(Modifier); \
+    CharBlock source; \
+    Variant u; \
+  }
+
+#define MODIFIERS() std::optional<std::list<Modifier>>
+
 inline namespace modifier {
 // For uniformity, in all keyword modifiers the name of the type defined
 // by ENUM_CLASS is "Value", e.g.
@@ -3833,18 +3895,6 @@ struct OmpPrescriptiveness {
   WRAPPER_CLASS_BOILERPLATE(OmpPrescriptiveness, Value);
 };
 
-// Ref: [4.5:201-207], [5.0:293-299], [5.1:325-331], [5.2:124]
-//
-// reduction-identifier ->
-//    base-language-identifier |                    // since 4.5
-//    - |                                           // since 4.5, until 5.2
-//    + | * | .AND. | .OR. | .EQV. | .NEQV. |       // since 4.5
-//    MIN | MAX | IAND | IOR | IEOR                 // since 4.5
-struct OmpReductionIdentifier {
-  UNION_CLASS_BOILERPLATE(OmpReductionIdentifier);
-  std::variant<DefinedOperator, ProcedureDesignator> u;
-};
-
 // Ref: [5.0:300-302], [5.1:332-334], [5.2:134-137]
 //
 // reduction-modifier ->
@@ -3973,14 +4023,23 @@ struct OmpBindClause {
 
 // Ref: [4.5:46-50], [5.0:74-78], [5.1:92-96], [5.2:109]
 //
+// When used as a data-sharing clause:
 // default-clause ->
 //    DEFAULT(data-sharing-attribute)               // since 4.5
 // data-sharing-attribute ->
 //    SHARED | NONE |                               // since 4.5
 //    PRIVATE | FIRSTPRIVATE                        // since 5.0
+//
+// When used in METADIRECTIVE:
+// default-clause ->
+//    DEFAULT(directive-specification)              // since 5.0, until 5.1
+// See also otherwise-clause.
 struct OmpDefaultClause {
   ENUM_CLASS(DataSharingAttribute, Private, Firstprivate, Shared, None)
-  WRAPPER_CLASS_BOILERPLATE(OmpDefaultClause, DataSharingAttribute);
+  UNION_CLASS_BOILERPLATE(OmpDefaultClause);
+  std::variant<DataSharingAttribute,
+      common::Indirection<OmpDirectiveSpecification>>
+      u;
 };
 
 // Ref: [4.5:103-107], [5.0:324-325], [5.1:357-358], [5.2:161-162]
@@ -4198,6 +4257,16 @@ struct OmpMapClause {
   std::tuple<MODIFIERS(), OmpObjectList, /*CommaSeparated=*/bool> t;
 };
 
+// Ref: [5.0:58-60], [5.1:63-68], [5.2:194-195]
+//
+// match-clause ->
+//    MATCH (context-selector-specification)        // since 5.0
+struct OmpMatchClause {
+  // The context-selector is an argument.
+  WRAPPER_CLASS_BOILERPLATE(
+      OmpMatchClause, traits::OmpContextSelectorSpecification);
+};
+
 // Ref: [5.2:217-218]
 // message-clause ->
 //    MESSAGE("message-text")
@@ -4228,6 +4297,17 @@ struct OmpOrderClause {
   std::tuple<MODIFIERS(), Ordering> t;
 };
 
+// Ref: [5.0:56-57], [5.1:60-62], [5.2:191]
+//
+// otherwise-clause ->
+//    DEFAULT ([directive-specification])           // since 5.0, until 5.1
+// otherwise-clause ->
+//    OTHERWISE ([directive-specification])]        // since 5.2
+struct OmpOtherwiseClause {
+  WRAPPER_CLASS_BOILERPLATE(OmpOtherwiseClause,
+      std::optional<common::Indirection<OmpDirectiveSpecification>>);
+};
+
 // Ref: [4.5:46-50], [5.0:74-78], [5.1:92-96], [5.2:229-230]
 //
 // proc-bind-clause ->
@@ -4313,6 +4393,19 @@ struct OmpUpdateClause {
   std::variant<OmpDependenceType, OmpTaskDependenceType> u;
 };
 
+// Ref: [5.0:56-57], [5.1:60-62], [5.2:190-191]
+//
+// when-clause ->
+//    WHEN (context-selector :
+//        [directive-specification])                // since 5.0
+struct OmpWhenClause {
+  TUPLE_CLASS_BOILERPLATE(OmpWhenClause);
+  MODIFIER_BOILERPLATE(OmpContextSelector);
+  std::tuple<MODIFIERS(),
+      std::optional<common::Indirection<OmpDirectiveSpecification>>>
+      t;
+};
+
 // OpenMP Clauses
 struct OmpClause {
   UNION_CLASS_BOILERPLATE(OmpClause);
@@ -4337,6 +4430,20 @@ struct OmpClauseList {
 
 // --- Directives and constructs
 
+struct OmpDirectiveSpecification {
+  CharBlock source;
+  TUPLE_CLASS_BOILERPLATE(OmpDirectiveSpecification);
+  std::tuple<llvm::omp::Directive, std::optional<std::list<OmpArgument>>,
+      std::optional<OmpClauseList>>
+      t;
+};
+
+struct OmpMetadirectiveDirective {
+  TUPLE_CLASS_BOILERPLATE(OmpMetadirectiveDirective);
+  std::tuple<OmpClauseList> t;
+  CharBlock source;
+};
+
 // Ref: [5.1:89-90], [5.2:216]
 //
 // nothing-directive ->
@@ -4431,27 +4538,16 @@ struct OpenMPDeclareTargetConstruct {
   std::tuple<Verbatim, OmpDeclareTargetSpecifier> t;
 };
 
-struct OmpDeclareMapperSpecifier {
-  TUPLE_CLASS_BOILERPLATE(OmpDeclareMapperSpecifier);
-  std::tuple<std::optional<Name>, TypeSpec, Name> t;
-};
-
 // OMP v5.2: 5.8.8
 //  declare-mapper -> DECLARE MAPPER ([mapper-name :] type :: var) map-clauses
 struct OpenMPDeclareMapperConstruct {
   TUPLE_CLASS_BOILERPLATE(OpenMPDeclareMapperConstruct);
   CharBlock source;
-  std::tuple<Verbatim, OmpDeclareMapperSpecifier, OmpClauseList> t;
+  std::tuple<Verbatim, OmpMapperSpecifier, OmpClauseList> t;
 };
 
 // 2.16 declare-reduction -> DECLARE REDUCTION (reduction-identifier : type-list
 //                                              : combiner) [initializer-clause]
-struct OmpReductionCombiner {
-  UNION_CLASS_BOILERPLATE(OmpReductionCombiner);
-  WRAPPER_CLASS(FunctionCombiner, Call);
-  std::variant<AssignmentStmt, FunctionCombiner> u;
-};
-
 WRAPPER_CLASS(OmpReductionInitializerClause, Expr);
 
 struct OpenMPDeclareReductionConstruct {
@@ -4496,8 +4592,8 @@ struct OpenMPDeclarativeConstruct {
   CharBlock source;
   std::variant<OpenMPDeclarativeAllocate, OpenMPDeclareMapperConstruct,
       OpenMPDeclareReductionConstruct, OpenMPDeclareSimdConstruct,
-      OpenMPDeclareTargetConstruct, OpenMPThreadprivate,
-      OpenMPRequiresConstruct, OpenMPUtilityConstruct>
+      OpenMPThreadprivate, OpenMPRequiresConstruct, OpenMPUtilityConstruct,
+      OpenMPDeclareTargetConstruct, OmpMetadirectiveDirective>
       u;
 };
 
@@ -4685,6 +4781,31 @@ struct OpenMPDepobjConstruct {
   std::tuple<Verbatim, OmpObject, OmpClause> t;
 };
 
+// Ref: [5.2: 200-201]
+//
+// dispatch-construct -> DISPATCH dispatch-clause
+// dispatch-clause -> depend-clause |
+//                    device-clause |
+//                    is_device_ptr-clause |
+//                    nocontext-clause |
+//                    novariants-clause |
+//                    nowait-clause
+struct OmpDispatchDirective {
+  TUPLE_CLASS_BOILERPLATE(OmpDispatchDirective);
+  CharBlock source;
+  std::tuple<Verbatim, OmpClauseList> t;
+};
+
+EMPTY_CLASS(OmpEndDispatchDirective);
+
+struct OpenMPDispatchConstruct {
+  TUPLE_CLASS_BOILERPLATE(OpenMPDispatchConstruct);
+  CharBlock source;
+  std::tuple<OmpDispatchDirective, Block,
+      std::optional<OmpEndDispatchDirective>>
+      t;
+};
+
 // 2.17.8 flush -> FLUSH [memory-order-clause] [(variable-name-list)]
 struct OpenMPFlushConstruct {
   TUPLE_CLASS_BOILERPLATE(OpenMPFlushConstruct);
@@ -4710,7 +4831,7 @@ struct OpenMPStandaloneConstruct {
   CharBlock source;
   std::variant<OpenMPSimpleStandaloneConstruct, OpenMPFlushConstruct,
       OpenMPCancelConstruct, OpenMPCancellationPointConstruct,
-      OpenMPDepobjConstruct>
+      OpenMPDepobjConstruct, OmpMetadirectiveDirective>
       u;
 };
 
@@ -4757,9 +4878,9 @@ struct OpenMPConstruct {
   UNION_CLASS_BOILERPLATE(OpenMPConstruct);
   std::variant<OpenMPStandaloneConstruct, OpenMPSectionsConstruct,
       OpenMPSectionConstruct, OpenMPLoopConstruct, OpenMPBlockConstruct,
-      OpenMPAtomicConstruct, OpenMPDeclarativeAllocate, OpenMPUtilityConstruct,
-      OpenMPExecutableAllocate, OpenMPAllocatorsConstruct,
-      OpenMPCriticalConstruct>
+      OpenMPAtomicConstruct, OpenMPDeclarativeAllocate, OpenMPDispatchConstruct,
+      OpenMPUtilityConstruct, OpenMPExecutableAllocate,
+      OpenMPAllocatorsConstruct, OpenMPCriticalConstruct>
       u;
 };
 
diff --git a/flang/include/flang/Parser/parsing.h b/flang/include/flang/Parser/parsing.h
index 0c774decb16d3..116b6bd6f191f 100644
--- a/flang/include/flang/Parser/parsing.h
+++ b/flang/include/flang/Parser/parsing.h
@@ -14,8 +14,8 @@
 #include "message.h"
 #include "parse-tree.h"
 #include "provenance.h"
-#include "flang/Common/Fortran-features.h"
 #include "flang/Parser/preprocessor.h"
+#include "flang/Support/Fortran-features.h"
 #include "llvm/Support/raw_ostream.h"
 #include <optional>
 #include <string>
diff --git a/flang/include/flang/Parser/user-state.h b/flang/include/flang/Parser/user-state.h
index 61745a833c715..129f9fb8fee05 100644
--- a/flang/include/flang/Parser/user-state.h
+++ b/flang/include/flang/Parser/user-state.h
@@ -14,10 +14,10 @@
 // parse tree construction so as to avoid any need for representing
 // state in static data.
 
-#include "flang/Common/Fortran-features.h"
 #include "flang/Common/idioms.h"
 #include "flang/Parser/char-block.h"
 #include "flang/Parser/parse-tree.h"
+#include "flang/Support/Fortran-features.h"
 #include "llvm/Support/raw_ostream.h"
 #include <cinttypes>
 #include <optional>
diff --git a/flang/include/flang/Runtime/CUDA/allocatable.h b/flang/include/flang/Runtime/CUDA/allocatable.h
index 0a96f73b6be44..822f2d4a2b297 100644
--- a/flang/include/flang/Runtime/CUDA/allocatable.h
+++ b/flang/include/flang/Runtime/CUDA/allocatable.h
@@ -18,28 +18,30 @@ extern "C" {
 
 /// Perform allocation of the descriptor.
 int RTDECL(CUFAllocatableAllocate)(Descriptor &, int64_t stream = -1,
-    bool hasStat = false, const Descriptor *errMsg = nullptr,
-    const char *sourceFile = nullptr, int sourceLine = 0);
+    bool *pinned = nullptr, bool hasStat = false,
+    const Descriptor *errMsg = nullptr, const char *sourceFile = nullptr,
+    int sourceLine = 0);
 
 /// Perform allocation of the descriptor with synchronization of it when
 /// necessary.
 int RTDECL(CUFAllocatableAllocateSync)(Descriptor &, int64_t stream = -1,
-    bool hasStat = false, const Descriptor *errMsg = nullptr,
-    const char *sourceFile = nullptr, int sourceLine = 0);
+    bool *pinned = nullptr, bool hasStat = false,
+    const Descriptor *errMsg = nullptr, const char *sourceFile = nullptr,
+    int sourceLine = 0);
 
 /// Perform allocation of the descriptor without synchronization. Assign data
 /// from source.
 int RTDEF(CUFAllocatableAllocateSource)(Descriptor &alloc,
-    const Descriptor &source, int64_t stream = -1, bool hasStat = false,
-    const Descriptor *errMsg = nullptr, const char *sourceFile = nullptr,
-    int sourceLine = 0);
+    const Descriptor &source, int64_t stream = -1, bool *pinned = nullptr,
+    bool hasStat = false, const Descriptor *errMsg = nullptr,
+    const char *sourceFile = nullptr, int sourceLine = 0);
 
 /// Perform allocation of the descriptor with synchronization of it when
 /// necessary. Assign data from source.
 int RTDEF(CUFAllocatableAllocateSourceSync)(Descriptor &alloc,
-    const Descriptor &source, int64_t stream = -1, bool hasStat = false,
-    const Descriptor *errMsg = nullptr, const char *sourceFile = nullptr,
-    int sourceLine = 0);
+    const Descriptor &source, int64_t stream = -1, bool *pinned = nullptr,
+    bool hasStat = false, const Descriptor *errMsg = nullptr,
+    const char *sourceFile = nullptr, int sourceLine = 0);
 
 /// Perform deallocation of the descriptor with synchronization of it when
 /// necessary.
diff --git a/flang/include/flang/Runtime/CUDA/init.h b/flang/include/flang/Runtime/CUDA/init.h
new file mode 100644
index 0000000000000..d1c8fcbf7d587
--- /dev/null
+++ b/flang/include/flang/Runtime/CUDA/init.h
@@ -0,0 +1,19 @@
+//===-- include/flang/Runtime/CUDA/init.h -----------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef FORTRAN_RUNTIME_CUDA_INIT_H_
+#define FORTRAN_RUNTIME_CUDA_INIT_H_
+
+#include "flang/Runtime/entry-names.h"
+
+extern "C" {
+
+void RTDECL(CUFInit)();
+}
+
+#endif // FORTRAN_RUNTIME_CUDA_INIT_H_
diff --git a/flang/include/flang/Runtime/CUDA/kernel.h b/flang/include/flang/Runtime/CUDA/kernel.h
index 85afda09e347a..1f812b580327a 100644
--- a/flang/include/flang/Runtime/CUDA/kernel.h
+++ b/flang/include/flang/Runtime/CUDA/kernel.h
@@ -28,6 +28,10 @@ void RTDEF(CUFLaunchClusterKernel)(const void *kernelName, intptr_t clusterX,
     intptr_t gridZ, intptr_t blockX, intptr_t blockY, intptr_t blockZ,
     int32_t smem, void **params, void **extra);
 
+void RTDEF(CUFLaunchCooperativeKernel)(const void *kernelName, intptr_t gridX,
+    intptr_t gridY, intptr_t gridZ, intptr_t blockX, intptr_t blockY,
+    intptr_t blockZ, int32_t smem, void **params, void **extra);
+
 } // extern "C"
 
 #endif // FORTRAN_RUNTIME_CUDA_KERNEL_H_
diff --git a/flang/include/flang/Runtime/CUDA/pointer.h b/flang/include/flang/Runtime/CUDA/pointer.h
index 78c7a1a92b7ea..7fbd8f8e061f2 100644
--- a/flang/include/flang/Runtime/CUDA/pointer.h
+++ b/flang/include/flang/Runtime/CUDA/pointer.h
@@ -18,28 +18,30 @@ extern "C" {
 
 /// Perform allocation of the descriptor.
 int RTDECL(CUFPointerAllocate)(Descriptor &, int64_t stream = -1,
-    bool hasStat = false, const Descriptor *errMsg = nullptr,
-    const char *sourceFile = nullptr, int sourceLine = 0);
+    bool *pinned = nullptr, bool hasStat = false,
+    const Descriptor *errMsg = nullptr, const char *sourceFile = nullptr,
+    int sourceLine = 0);
 
 /// Perform allocation of the descriptor with synchronization of it when
 /// necessary.
 int RTDECL(CUFPointerAllocateSync)(Descriptor &, int64_t stream = -1,
-    bool hasStat = false, const Descriptor *errMsg = nullptr,
-    const char *sourceFile = nullptr, int sourceLine = 0);
+    bool *pinned = nullptr, bool hasStat = false,
+    const Descriptor *errMsg = nullptr, const char *sourceFile = nullptr,
+    int sourceLine = 0);
 
 /// Perform allocation of the descriptor without synchronization. Assign data
 /// from source.
 int RTDEF(CUFPointerAllocateSource)(Descriptor &pointer,
-    const Descriptor &source, int64_t stream = -1, bool hasStat = false,
-    const Descriptor *errMsg = nullptr, const char *sourceFile = nullptr,
-    int sourceLine = 0);
+    const Descriptor &source, int64_t stream = -1, bool *pinned = nullptr,
+    bool hasStat = false, const Descriptor *errMsg = nullptr,
+    const char *sourceFile = nullptr, int sourceLine = 0);
 
 /// Perform allocation of the descriptor with synchronization of it when
 /// necessary. Assign data from source.
 int RTDEF(CUFPointerAllocateSourceSync)(Descriptor &pointer,
-    const Descriptor &source, int64_t stream = -1, bool hasStat = false,
-    const Descriptor *errMsg = nullptr, const char *sourceFile = nullptr,
-    int sourceLine = 0);
+    const Descriptor &source, int64_t stream = -1, bool *pinned = nullptr,
+    bool hasStat = false, const Descriptor *errMsg = nullptr,
+    const char *sourceFile = nullptr, int sourceLine = 0);
 
 } // extern "C"
 
diff --git a/flang/include/flang/Runtime/descriptor-consts.h b/flang/include/flang/Runtime/descriptor-consts.h
index 3b2537579d586..abcdbc4a12002 100644
--- a/flang/include/flang/Runtime/descriptor-consts.h
+++ b/flang/include/flang/Runtime/descriptor-consts.h
@@ -9,8 +9,8 @@
 #ifndef FORTRAN_RUNTIME_DESCRIPTOR_CONSTS_H_
 #define FORTRAN_RUNTIME_DESCRIPTOR_CONSTS_H_
 
+#include "flang/Common/ISO_Fortran_binding_wrapper.h"
 #include "flang/Common/api-attrs.h"
-#include "flang/ISO_Fortran_binding_wrapper.h"
 #include <cstddef>
 #include <cstdint>
 
diff --git a/flang/include/flang/Runtime/descriptor.h b/flang/include/flang/Runtime/descriptor.h
index dd36fba157ca9..628ac8c927a51 100644
--- a/flang/include/flang/Runtime/descriptor.h
+++ b/flang/include/flang/Runtime/descriptor.h
@@ -18,7 +18,7 @@
 // User C code is welcome to depend on that ISO_Fortran_binding.h file,
 // but should never reference this internal header.
 
-#include "flang/ISO_Fortran_binding_wrapper.h"
+#include "flang/Common/ISO_Fortran_binding_wrapper.h"
 #include "flang/Runtime/descriptor-consts.h"
 #include "flang/Runtime/memory.h"
 #include "flang/Runtime/type-code.h"
diff --git a/flang/include/flang/Runtime/extensions.h b/flang/include/flang/Runtime/extensions.h
index a855c694e0090..133194dea87cf 100644
--- a/flang/include/flang/Runtime/extensions.h
+++ b/flang/include/flang/Runtime/extensions.h
@@ -69,5 +69,11 @@ std::int64_t FORTRAN_PROCEDURE_NAME(access)(const char *name,
     std::int64_t nameLength, const char *mode, std::int64_t modeLength);
 #endif
 
+// GNU extension subroutine CHDIR(NAME, [STATUS])
+int RTNAME(Chdir)(const char *name);
+
+// GNU extension function IERRNO()
+int FORTRAN_PROCEDURE_NAME(ierrno)();
+
 } // extern "C"
 #endif // FORTRAN_RUNTIME_EXTENSIONS_H_
diff --git a/flang/include/flang/Runtime/random.h b/flang/include/flang/Runtime/random.h
index 269297dff47db..1f01be1aa9ffa 100644
--- a/flang/include/flang/Runtime/random.h
+++ b/flang/include/flang/Runtime/random.h
@@ -8,6 +8,9 @@
 
 // Intrinsic subroutines RANDOM_INIT, RANDOM_NUMBER, and RANDOM_SEED.
 
+#ifndef FORTRAN_RUNTIME_RANDOM_H_
+#define FORTRAN_RUNTIME_RANDOM_H_
+
 #include "flang/Runtime/entry-names.h"
 #include <cstdint>
 
@@ -35,3 +38,5 @@ void RTNAME(RandomSeed)(const Descriptor *size, const Descriptor *put,
 
 } // extern "C"
 } // namespace Fortran::runtime
+
+#endif // FORTRAN_RUNTIME_RANDOM_H_
diff --git a/flang/include/flang/Runtime/support.h b/flang/include/flang/Runtime/support.h
index ba9c2598bb0dd..4a6d4357e8710 100644
--- a/flang/include/flang/Runtime/support.h
+++ b/flang/include/flang/Runtime/support.h
@@ -10,7 +10,7 @@
 #ifndef FORTRAN_RUNTIME_SUPPORT_H_
 #define FORTRAN_RUNTIME_SUPPORT_H_
 
-#include "flang/ISO_Fortran_binding_wrapper.h"
+#include "flang/Common/ISO_Fortran_binding_wrapper.h"
 #include "flang/Runtime/entry-names.h"
 #include <cstddef>
 #include <cstdint>
diff --git a/flang/include/flang/Runtime/type-code.h b/flang/include/flang/Runtime/type-code.h
index dd3a9f2690ee7..ae854ed2145e4 100644
--- a/flang/include/flang/Runtime/type-code.h
+++ b/flang/include/flang/Runtime/type-code.h
@@ -10,8 +10,8 @@
 #define FORTRAN_RUNTIME_TYPE_CODE_H_
 
 #include "flang/Common/Fortran-consts.h"
+#include "flang/Common/ISO_Fortran_binding_wrapper.h"
 #include "flang/Common/optional.h"
-#include "flang/ISO_Fortran_binding_wrapper.h"
 #include <utility>
 
 namespace Fortran::runtime {
diff --git a/flang/include/flang/Semantics/expression.h b/flang/include/flang/Semantics/expression.h
index bb1674a9f8877..ce253e6f52e8f 100644
--- a/flang/include/flang/Semantics/expression.h
+++ b/flang/include/flang/Semantics/expression.h
@@ -10,7 +10,6 @@
 #define FORTRAN_SEMANTICS_EXPRESSION_H_
 
 #include "semantics.h"
-#include "flang/Common/Fortran.h"
 #include "flang/Common/indirection.h"
 #include "flang/Common/restorer.h"
 #include "flang/Common/visit.h"
@@ -24,6 +23,7 @@
 #include "flang/Parser/parse-tree-visitor.h"
 #include "flang/Parser/parse-tree.h"
 #include "flang/Parser/tools.h"
+#include "flang/Support/Fortran.h"
 #include <map>
 #include <optional>
 #include <type_traits>
@@ -348,7 +348,8 @@ class ExpressionAnalyzer {
   bool CheckDataRef(const DataRef &); // ditto
   std::optional<Expr<SubscriptInteger>> GetSubstringBound(
       const std::optional<parser::ScalarIntExpr> &);
-  MaybeExpr AnalyzeDefinedOp(const parser::Name &, ActualArguments &&);
+  MaybeExpr AnalyzeDefinedOp(
+      const parser::Name &, ActualArguments &&, const Symbol *&);
   MaybeExpr FixMisparsedSubstring(const parser::Designator &);
 
   struct CalleeAndArguments {
diff --git a/flang/include/flang/Semantics/openmp-modifiers.h b/flang/include/flang/Semantics/openmp-modifiers.h
index 5d5c5e97faf41..7cdbf65adebe1 100644
--- a/flang/include/flang/Semantics/openmp-modifiers.h
+++ b/flang/include/flang/Semantics/openmp-modifiers.h
@@ -72,6 +72,7 @@ DECLARE_DESCRIPTOR(parser::OmpAlignModifier);
 DECLARE_DESCRIPTOR(parser::OmpAllocatorComplexModifier);
 DECLARE_DESCRIPTOR(parser::OmpAllocatorSimpleModifier);
 DECLARE_DESCRIPTOR(parser::OmpChunkModifier);
+DECLARE_DESCRIPTOR(parser::OmpContextSelector);
 DECLARE_DESCRIPTOR(parser::OmpDependenceType);
 DECLARE_DESCRIPTOR(parser::OmpDeviceModifier);
 DECLARE_DESCRIPTOR(parser::OmpDirectiveNameModifier);
diff --git a/flang/include/flang/Semantics/scope.h b/flang/include/flang/Semantics/scope.h
index b3b033a5a3ae3..b4046830522b8 100644
--- a/flang/include/flang/Semantics/scope.h
+++ b/flang/include/flang/Semantics/scope.h
@@ -11,11 +11,11 @@
 
 #include "attr.h"
 #include "symbol.h"
-#include "flang/Common/Fortran.h"
 #include "flang/Common/idioms.h"
 #include "flang/Common/reference.h"
 #include "flang/Parser/message.h"
 #include "flang/Parser/provenance.h"
+#include "flang/Support/Fortran.h"
 #include <list>
 #include <map>
 #include <optional>
diff --git a/flang/include/flang/Semantics/semantics.h b/flang/include/flang/Semantics/semantics.h
index c981d86fbd94c..730513dbe3232 100644
--- a/flang/include/flang/Semantics/semantics.h
+++ b/flang/include/flang/Semantics/semantics.h
@@ -13,12 +13,12 @@
 #include "program-tree.h"
 #include "scope.h"
 #include "symbol.h"
-#include "flang/Common/Fortran-features.h"
-#include "flang/Common/LangOptions.h"
 #include "flang/Evaluate/common.h"
 #include "flang/Evaluate/intrinsics.h"
 #include "flang/Evaluate/target.h"
 #include "flang/Parser/message.h"
+#include "flang/Support/Fortran-features.h"
+#include "flang/Support/LangOptions.h"
 #include <iosfwd>
 #include <set>
 #include <string>
@@ -110,6 +110,12 @@ class SemanticsContext {
   }
   Scope &globalScope() { return globalScope_; }
   Scope &intrinsicModulesScope() { return intrinsicModulesScope_; }
+  Scope *currentHermeticModuleFileScope() {
+    return currentHermeticModuleFileScope_;
+  }
+  void set_currentHermeticModuleFileScope(Scope *scope) {
+    currentHermeticModuleFileScope_ = scope;
+  }
   parser::Messages &messages() { return messages_; }
   evaluate::FoldingContext &foldingContext() { return foldingContext_; }
   parser::AllCookedSources &allCookedSources() { return allCookedSources_; }
@@ -313,6 +319,7 @@ class SemanticsContext {
   evaluate::TargetCharacteristics targetCharacteristics_;
   Scope globalScope_;
   Scope &intrinsicModulesScope_;
+  Scope *currentHermeticModuleFileScope_{nullptr};
   ScopeIndex scopeIndex_;
   parser::Messages messages_;
   evaluate::FoldingContext foldingContext_;
diff --git a/flang/include/flang/Semantics/symbol.h b/flang/include/flang/Semantics/symbol.h
index bc6abccac1bb8..4ae2775c0f849 100644
--- a/flang/include/flang/Semantics/symbol.h
+++ b/flang/include/flang/Semantics/symbol.h
@@ -10,11 +10,11 @@
 #define FORTRAN_SEMANTICS_SYMBOL_H_
 
 #include "type.h"
-#include "flang/Common/Fortran.h"
 #include "flang/Common/enum-set.h"
 #include "flang/Common/reference.h"
 #include "flang/Common/visit.h"
 #include "flang/Semantics/module-dependences.h"
+#include "flang/Support/Fortran.h"
 #include "llvm/ADT/DenseMapInfo.h"
 
 #include <array>
@@ -605,12 +605,12 @@ class UseDetails {
 class UseErrorDetails {
 public:
   UseErrorDetails(const UseDetails &);
-  UseErrorDetails &add_occurrence(const SourceName &, const Scope &);
-  using listType = std::list<std::pair<SourceName, const Scope *>>;
-  const listType occurrences() const { return occurrences_; };
+  UseErrorDetails &add_occurrence(const SourceName &, const Symbol &);
+  using ListType = std::list<std::pair<SourceName, const Symbol *>>;
+  const ListType occurrences() const { return occurrences_; };
 
 private:
-  listType occurrences_;
+  ListType occurrences_;
 };
 
 // A symbol host-associated from an enclosing scope.
diff --git a/flang/include/flang/Semantics/tools.h b/flang/include/flang/Semantics/tools.h
index 07103f98ff041..e82446a2ba884 100644
--- a/flang/include/flang/Semantics/tools.h
+++ b/flang/include/flang/Semantics/tools.h
@@ -12,7 +12,6 @@
 // Simple predicates and look-up functions that are best defined
 // canonically for use in semantic checking.
 
-#include "flang/Common/Fortran.h"
 #include "flang/Common/visit.h"
 #include "flang/Evaluate/expression.h"
 #include "flang/Evaluate/shape.h"
@@ -23,6 +22,7 @@
 #include "flang/Semantics/attr.h"
 #include "flang/Semantics/expression.h"
 #include "flang/Semantics/semantics.h"
+#include "flang/Support/Fortran.h"
 #include <functional>
 
 namespace Fortran::semantics {
@@ -529,6 +529,9 @@ template <ComponentKind componentKind> class ComponentIterator {
     // having to check against an end() iterator.
     explicit operator bool() const { return !componentPath_.empty(); }
 
+    // Returns the current sequence of components, including parent components.
+    SymbolVector GetComponentPath() const;
+
     // Builds a designator name of the referenced component for messages.
     // The designator helps when the component referred to by the iterator
     // may be "buried" into other components. This gives the full
@@ -626,7 +629,7 @@ using PotentialAndPointerComponentIterator =
 // is returned. Otherwise, the returned iterator casts to true and can be
 // dereferenced.
 PotentialComponentIterator::const_iterator FindEventOrLockPotentialComponent(
-    const DerivedTypeSpec &);
+    const DerivedTypeSpec &, bool ignoreCoarrays = false);
 UltimateComponentIterator::const_iterator FindCoarrayUltimateComponent(
     const DerivedTypeSpec &);
 UltimateComponentIterator::const_iterator FindPointerUltimateComponent(
diff --git a/flang/include/flang/Semantics/type.h b/flang/include/flang/Semantics/type.h
index 3522191502059..5d96f1e89bf52 100644
--- a/flang/include/flang/Semantics/type.h
+++ b/flang/include/flang/Semantics/type.h
@@ -9,10 +9,10 @@
 #ifndef FORTRAN_SEMANTICS_TYPE_H_
 #define FORTRAN_SEMANTICS_TYPE_H_
 
-#include "flang/Common/Fortran.h"
 #include "flang/Common/idioms.h"
 #include "flang/Evaluate/expression.h"
 #include "flang/Parser/char-block.h"
+#include "flang/Support/Fortran.h"
 #include <algorithm>
 #include <iosfwd>
 #include <map>
diff --git a/flang/include/flang/Common/Fortran-features.h b/flang/include/flang/Support/Fortran-features.h
similarity index 87%
rename from flang/include/flang/Common/Fortran-features.h
rename to flang/include/flang/Support/Fortran-features.h
index 44f88009f8f2c..44ba6428e6c93 100644
--- a/flang/include/flang/Common/Fortran-features.h
+++ b/flang/include/flang/Support/Fortran-features.h
@@ -1,4 +1,4 @@
-//===-- include/flang/Common/Fortran-features.h -----------------*- C++ -*-===//
+//===-- include/flang/Support/Fortran-features.h ----------------*- C++ -*-===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
@@ -6,10 +6,10 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef FORTRAN_COMMON_FORTRAN_FEATURES_H_
-#define FORTRAN_COMMON_FORTRAN_FEATURES_H_
+#ifndef FORTRAN_SUPPORT_FORTRAN_FEATURES_H_
+#define FORTRAN_SUPPORT_FORTRAN_FEATURES_H_
 
-#include "flang/Common/Fortran.h"
+#include "Fortran.h"
 #include "flang/Common/enum-set.h"
 #include "flang/Common/idioms.h"
 #include <optional>
@@ -34,13 +34,13 @@ ENUM_CLASS(LanguageFeature, BackslashEscapes, OldDebugLines,
     EquivalenceSameNonSequence, AdditionalIntrinsics, AnonymousParents,
     OldLabelDoEndStatements, LogicalIntegerAssignment, EmptySourceFile,
     ProgramReturn, ImplicitNoneTypeNever, ImplicitNoneTypeAlways,
-    ForwardRefImplicitNone, OpenAccessAppend, BOZAsDefaultInteger,
-    DistinguishableSpecifics, DefaultSave, PointerInSeqType, NonCharacterFormat,
-    SaveMainProgram, SaveBigMainProgramVariables,
-    DistinctArrayConstructorLengths, PPCVector, RelaxedIntentInChecking,
-    ForwardRefImplicitNoneData, NullActualForAllocatable,
-    ActualIntegerConvertedToSmallerKind, HollerithOrCharacterAsBOZ,
-    BindingAsProcedure, StatementFunctionExtensions,
+    ImplicitNoneExternal, ForwardRefImplicitNone, OpenAccessAppend,
+    BOZAsDefaultInteger, DistinguishableSpecifics, DefaultSave,
+    PointerInSeqType, NonCharacterFormat, SaveMainProgram,
+    SaveBigMainProgramVariables, DistinctArrayConstructorLengths, PPCVector,
+    RelaxedIntentInChecking, ForwardRefImplicitNoneData,
+    NullActualForAllocatable, ActualIntegerConvertedToSmallerKind,
+    HollerithOrCharacterAsBOZ, BindingAsProcedure, StatementFunctionExtensions,
     UseGenericIntrinsicWhenSpecificDoesntMatch, DataStmtExtensions,
     RedundantContiguous, RedundantAttribute, InitBlankCommon,
     EmptyBindCDerivedType, MiscSourceExtensions, AllocateToOtherLength,
@@ -71,9 +71,10 @@ ENUM_CLASS(UsageWarning, Portability, PointerToUndefinable,
     ZeroDoStep, UnusedForallIndex, OpenMPUsage, DataLength, IgnoredDirective,
     HomonymousSpecific, HomonymousResult, IgnoredIntrinsicFunctionType,
     PreviousScalarUse, RedeclaredInaccessibleComponent, ImplicitShared,
-    IndexVarRedefinition, IncompatibleImplicitInterfaces, BadTypeForTarget,
+    IndexVarRedefinition, IncompatibleImplicitInterfaces,
     VectorSubscriptFinalization, UndefinedFunctionResult, UselessIomsg,
-    MismatchingDummyProcedure, SubscriptedEmptyArray, UnsignedLiteralTruncation)
+    MismatchingDummyProcedure, SubscriptedEmptyArray, UnsignedLiteralTruncation,
+    CompatibleDeclarationsFromDistinctModules)
 
 using LanguageFeatures = EnumSet<LanguageFeature, LanguageFeature_enumSize>;
 using UsageWarnings = EnumSet<UsageWarning, UsageWarning_enumSize>;
@@ -125,4 +126,4 @@ class LanguageFeatureControl {
   bool warnAllUsage_{false};
 };
 } // namespace Fortran::common
-#endif // FORTRAN_COMMON_FORTRAN_FEATURES_H_
+#endif // FORTRAN_SUPPORT_FORTRAN_FEATURES_H_
diff --git a/flang/include/flang/Common/Fortran.h b/flang/include/flang/Support/Fortran.h
similarity index 94%
rename from flang/include/flang/Common/Fortran.h
rename to flang/include/flang/Support/Fortran.h
index e1922f7654bb1..6e6e37ba594be 100644
--- a/flang/include/flang/Common/Fortran.h
+++ b/flang/include/flang/Support/Fortran.h
@@ -1,4 +1,4 @@
-//===-- include/flang/Common/Fortran.h --------------------------*- C++ -*-===//
+//===-- include/flang/Support/Fortran.h -------------------------*- C++ -*-===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
@@ -6,15 +6,15 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef FORTRAN_COMMON_FORTRAN_H_
-#define FORTRAN_COMMON_FORTRAN_H_
+#ifndef FORTRAN_SUPPORT_FORTRAN_H_
+#define FORTRAN_SUPPORT_FORTRAN_H_
 
 // Fortran language concepts that are used in many phases are defined
 // once here to avoid redundancy and needless translation.
 
-#include "enum-set.h"
-#include "idioms.h"
 #include "flang/Common/Fortran-consts.h"
+#include "flang/Common/enum-set.h"
+#include "flang/Common/idioms.h"
 #include <cinttypes>
 #include <optional>
 #include <string>
@@ -109,4 +109,4 @@ inline std::string GetExternalAssemblyName(
 }
 
 } // namespace Fortran::common
-#endif // FORTRAN_COMMON_FORTRAN_H_
+#endif // FORTRAN_SUPPORT_FORTRAN_H_
diff --git a/flang/include/flang/Common/LangOptions.def b/flang/include/flang/Support/LangOptions.def
similarity index 100%
rename from flang/include/flang/Common/LangOptions.def
rename to flang/include/flang/Support/LangOptions.def
diff --git a/flang/include/flang/Common/LangOptions.h b/flang/include/flang/Support/LangOptions.h
similarity index 87%
rename from flang/include/flang/Common/LangOptions.h
rename to flang/include/flang/Support/LangOptions.h
index 83f25cfbe2614..fac6fb92df85a 100644
--- a/flang/include/flang/Common/LangOptions.h
+++ b/flang/include/flang/Support/LangOptions.h
@@ -1,4 +1,4 @@
-//===------ LangOptions.h ---------------------------------------*- C++ -*-===//
+//===-- include/flang/Support/LangOptions.h ---------------------*- C++ -*-===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
@@ -12,8 +12,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef FORTRAN_COMMON_LANGOPTIONS_H
-#define FORTRAN_COMMON_LANGOPTIONS_H
+#ifndef FORTRAN_SUPPORT_LANGOPTIONS_H_
+#define FORTRAN_SUPPORT_LANGOPTIONS_H_
 
 #include <string>
 #include <vector>
@@ -45,12 +45,12 @@ class LangOptionsBase {
 
 #define LANGOPT(Name, Bits, Default) unsigned Name : Bits;
 #define ENUM_LANGOPT(Name, Type, Bits, Default)
-#include "flang/Common/LangOptions.def"
+#include "LangOptions.def"
 
 protected:
 #define LANGOPT(Name, Bits, Default)
 #define ENUM_LANGOPT(Name, Type, Bits, Default) unsigned Name : Bits;
-#include "flang/Common/LangOptions.def"
+#include "LangOptions.def"
 };
 
 /// Tracks various options which control the dialect of Fortran that is
@@ -63,7 +63,7 @@ class LangOptions : public LangOptionsBase {
 #define ENUM_LANGOPT(Name, Type, Bits, Default) \
   Type get##Name() const { return static_cast<Type>(Name); } \
   void set##Name(Type Value) { Name = static_cast<unsigned>(Value); }
-#include "flang/Common/LangOptions.def"
+#include "LangOptions.def"
 
   /// Name of the IR file that contains the result of the OpenMP target
   /// host code generation.
@@ -77,4 +77,4 @@ class LangOptions : public LangOptionsBase {
 
 } // end namespace Fortran::common
 
-#endif // FORTRAN_COMMON_LANGOPTIONS_H
+#endif // FORTRAN_SUPPORT_LANGOPTIONS_H_
diff --git a/flang/include/flang/Common/MathOptionsBase.def b/flang/include/flang/Support/MathOptionsBase.def
similarity index 100%
rename from flang/include/flang/Common/MathOptionsBase.def
rename to flang/include/flang/Support/MathOptionsBase.def
diff --git a/flang/include/flang/Common/MathOptionsBase.h b/flang/include/flang/Support/MathOptionsBase.h
similarity index 77%
rename from flang/include/flang/Common/MathOptionsBase.h
rename to flang/include/flang/Support/MathOptionsBase.h
index 7f8ebdbee1987..a2bea3c598a8c 100644
--- a/flang/include/flang/Common/MathOptionsBase.h
+++ b/flang/include/flang/Support/MathOptionsBase.h
@@ -1,4 +1,4 @@
-//===- MathOptionsBase.h - Math options config ------------------*- C++ -*-===//
+//===-- include/flang/Support/MathOptionsBase.h -----------------*- C++ -*-===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
@@ -14,8 +14,8 @@
 ///
 //===----------------------------------------------------------------------===//
 
-#ifndef FORTRAN_COMMON_MATHOPTIONSBASE_H
-#define FORTRAN_COMMON_MATHOPTIONSBASE_H
+#ifndef FORTRAN_SUPPORT_MATHOPTIONSBASE_H_
+#define FORTRAN_SUPPORT_MATHOPTIONSBASE_H_
 
 namespace Fortran::common {
 
@@ -27,18 +27,18 @@ class MathOptionsBase {
     Name = static_cast<unsigned>(Value); \
     return *this; \
   }
-#include "flang/Common/MathOptionsBase.def"
+#include "MathOptionsBase.def"
 
   MathOptionsBase() {
 #define ENUM_MATHOPT(Name, Type, Bits, Default) set##Name(Default);
-#include "flang/Common/MathOptionsBase.def"
+#include "MathOptionsBase.def"
   }
 
 private:
 #define ENUM_MATHOPT(Name, Type, Bits, Default) unsigned Name : Bits;
-#include "flang/Common/MathOptionsBase.def"
+#include "MathOptionsBase.def"
 };
 
 } // namespace Fortran::common
 
-#endif // FORTRAN_COMMON_MATHOPTIONSBASE_H
+#endif // FORTRAN_SUPPORT_MATHOPTIONSBASE_H_
diff --git a/flang/include/flang/Common/OpenMP-features.h b/flang/include/flang/Support/OpenMP-features.h
similarity index 86%
rename from flang/include/flang/Common/OpenMP-features.h
rename to flang/include/flang/Support/OpenMP-features.h
index 480c8b4c1013f..1dd7ea560cc96 100644
--- a/flang/include/flang/Common/OpenMP-features.h
+++ b/flang/include/flang/Support/OpenMP-features.h
@@ -1,4 +1,4 @@
-//===-- include/flang/Common/OpenMP-features.h -----------------*- C++ -*-====//
+//===-- include/flang/Support/OpenMP-features.h -----------------*- C++ -*-===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef FORTRAN_COMMON_OPENMP_FEATURES_H_
-#define FORTRAN_COMMON_OPENMP_FEATURES_H_
+#ifndef FORTRAN_SUPPORT_OPENMP_FEATURES_H_
+#define FORTRAN_SUPPORT_OPENMP_FEATURES_H_
 
 namespace Fortran::common {
 
@@ -49,4 +49,4 @@ void setOpenMPMacro(int version, FortranPredefinitions &predefinitions) {
   }
 }
 } // namespace Fortran::common
-#endif // FORTRAN_COMMON_OPENMP_FEATURES_H_
+#endif // FORTRAN_SUPPORT_OPENMP_FEATURES_H_
diff --git a/flang/include/flang/Common/OpenMP-utils.h b/flang/include/flang/Support/OpenMP-utils.h
similarity index 92%
rename from flang/include/flang/Common/OpenMP-utils.h
rename to flang/include/flang/Support/OpenMP-utils.h
index 827f13bc4758e..d8f82e1cf94f1 100644
--- a/flang/include/flang/Common/OpenMP-utils.h
+++ b/flang/include/flang/Support/OpenMP-utils.h
@@ -1,4 +1,4 @@
-//===-- include/flang/Common/OpenMP-utils.h --------------------*- C++ -*-====//
+//===-- include/flang/Support/OpenMP-utils.h --------------------*- C++ -*-===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef FORTRAN_COMMON_OPENMP_UTILS_H_
-#define FORTRAN_COMMON_OPENMP_UTILS_H_
+#ifndef FORTRAN_SUPPORT_OPENMP_UTILS_H_
+#define FORTRAN_SUPPORT_OPENMP_UTILS_H_
 
 #include "flang/Semantics/symbol.h"
 
@@ -73,4 +73,4 @@ mlir::Block *genEntryBlock(
     mlir::OpBuilder &builder, const EntryBlockArgs &args, mlir::Region &region);
 } // namespace Fortran::common::openmp
 
-#endif // FORTRAN_COMMON_OPENMP_UTILS_H_
+#endif // FORTRAN_SUPPORT_OPENMP_UTILS_H_
diff --git a/flang/include/flang/Common/Version.h b/flang/include/flang/Support/Version.h
similarity index 90%
rename from flang/include/flang/Common/Version.h
rename to flang/include/flang/Support/Version.h
index 3257d4a4f6459..fe7f7395e1eba 100644
--- a/flang/include/flang/Common/Version.h
+++ b/flang/include/flang/Support/Version.h
@@ -1,4 +1,4 @@
-//===- Version.h - Flang Version Number ---------------------*- Fortran -*-===//
+//===-- include/flang/Support/Version.h -------------------------*- C++ -*-===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
@@ -12,8 +12,8 @@
 ///
 //===----------------------------------------------------------------------===//
 
-#ifndef FORTRAN_COMMON_VERSION_H
-#define FORTRAN_COMMON_VERSION_H
+#ifndef FORTRAN_SUPPORT_VERSION_H_
+#define FORTRAN_SUPPORT_VERSION_H_
 
 #include "flang/Version.inc"
 #include "llvm/ADT/StringRef.h"
@@ -53,4 +53,4 @@ std::string getFlangFullVersion();
 std::string getFlangToolFullVersion(llvm::StringRef ToolName);
 } // namespace Fortran::common
 
-#endif // FORTRAN_COMMON_VERSION_H
+#endif // FORTRAN_SUPPORT_VERSION_H_
diff --git a/flang/include/flang/Common/default-kinds.h b/flang/include/flang/Support/default-kinds.h
similarity index 91%
rename from flang/include/flang/Common/default-kinds.h
rename to flang/include/flang/Support/default-kinds.h
index 6c66b98e98a05..ab1422d63af30 100644
--- a/flang/include/flang/Common/default-kinds.h
+++ b/flang/include/flang/Support/default-kinds.h
@@ -1,4 +1,4 @@
-//===-- include/flang/Common/default-kinds.h --------------------*- C++ -*-===//
+//===-- include/flang/Support/default-kinds.h -------------------*- C++ -*-===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
@@ -6,10 +6,10 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef FORTRAN_COMMON_DEFAULT_KINDS_H_
-#define FORTRAN_COMMON_DEFAULT_KINDS_H_
+#ifndef FORTRAN_SUPPORT_DEFAULT_KINDS_H_
+#define FORTRAN_SUPPORT_DEFAULT_KINDS_H_
 
-#include "flang/Common/Fortran.h"
+#include "Fortran.h"
 #include <cstdint>
 
 namespace Fortran::common {
@@ -58,4 +58,4 @@ class IntrinsicTypeDefaultKinds {
   int defaultLogicalKind_{defaultIntegerKind_};
 };
 } // namespace Fortran::common
-#endif // FORTRAN_COMMON_DEFAULT_KINDS_H_
+#endif // FORTRAN_SUPPORT_DEFAULT_KINDS_H_
diff --git a/flang/include/flang/Testing/fp-testing.h b/flang/include/flang/Testing/fp-testing.h
new file mode 100644
index 0000000000000..c65766b8b47e8
--- /dev/null
+++ b/flang/include/flang/Testing/fp-testing.h
@@ -0,0 +1,35 @@
+//===-- include/flang/Testing/fp-testing.h ----------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef FORTRAN_TESTING_FP_TESTING_H_
+#define FORTRAN_TESTING_FP_TESTING_H_
+
+#include "flang/Common/target-rounding.h"
+#include <fenv.h>
+
+using Fortran::common::RealFlags;
+using Fortran::common::Rounding;
+using Fortran::common::RoundingMode;
+
+class ScopedHostFloatingPointEnvironment {
+public:
+  ScopedHostFloatingPointEnvironment(bool treatSubnormalOperandsAsZero = false,
+      bool flushSubnormalResultsToZero = false);
+  ~ScopedHostFloatingPointEnvironment();
+  void ClearFlags() const;
+  static RealFlags CurrentFlags();
+  static void SetRounding(Rounding rounding);
+
+private:
+  fenv_t originalFenv_;
+#if __x86_64__
+  unsigned int originalMxcsr;
+#endif
+};
+
+#endif // FORTRAN_TESTING_FP_TESTING_H_
diff --git a/flang/unittests/Evaluate/testing.h b/flang/include/flang/Testing/testing.h
similarity index 75%
rename from flang/unittests/Evaluate/testing.h
rename to flang/include/flang/Testing/testing.h
index 422e2853c05bc..03b589924518d 100644
--- a/flang/unittests/Evaluate/testing.h
+++ b/flang/include/flang/Testing/testing.h
@@ -1,5 +1,13 @@
-#ifndef FORTRAN_EVALUATE_TESTING_H_
-#define FORTRAN_EVALUATE_TESTING_H_
+//===-- include/flang/Testing/testing.h -------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef FORTRAN_TESTING_TESTING_H_
+#define FORTRAN_TESTING_TESTING_H_
 
 #include <cinttypes>
 #include <string>
@@ -33,4 +41,4 @@ FailureDetailPrinter Match(const char *file, int line, const std::string &want,
 FailureDetailPrinter Compare(const char *file, int line, const char *xs,
     const char *rel, const char *ys, std::uint64_t x, std::uint64_t y);
 } // namespace testing
-#endif // FORTRAN_EVALUATE_TESTING_H_
+#endif // FORTRAN_TESTING_TESTING_H_
diff --git a/flang/include/flang/Tools/CrossToolHelpers.h b/flang/include/flang/Tools/CrossToolHelpers.h
index 0286f2aa14519..d9401e753fb79 100644
--- a/flang/include/flang/Tools/CrossToolHelpers.h
+++ b/flang/include/flang/Tools/CrossToolHelpers.h
@@ -13,9 +13,9 @@
 #ifndef FORTRAN_TOOLS_CROSS_TOOL_HELPERS_H
 #define FORTRAN_TOOLS_CROSS_TOOL_HELPERS_H
 
-#include "flang/Common/LangOptions.h"
-#include "flang/Common/MathOptionsBase.h"
 #include "flang/Frontend/CodeGenOptions.h"
+#include "flang/Support/LangOptions.h"
+#include "flang/Support/MathOptionsBase.h"
 #include <cstdint>
 
 #include "mlir/Dialect/OpenMP/OpenMPDialect.h"
diff --git a/flang/include/flang/Tools/TargetSetup.h b/flang/include/flang/Tools/TargetSetup.h
index d1b0da3a42c89..d167f44fe2fd7 100644
--- a/flang/include/flang/Tools/TargetSetup.h
+++ b/flang/include/flang/Tools/TargetSetup.h
@@ -24,34 +24,35 @@ namespace Fortran::tools {
     const std::string &compilerVersion, const std::string &compilerOptions) {
 
   const llvm::Triple &targetTriple{targetMachine.getTargetTriple()};
-  // FIXME: Handle real(3) ?
-  if (targetTriple.getArch() != llvm::Triple::ArchType::x86_64) {
-    targetCharacteristics.DisableType(
-        Fortran::common::TypeCategory::Real, /*kind=*/10);
-  }
+
   if (targetTriple.getArch() == llvm::Triple::ArchType::x86_64) {
     targetCharacteristics.set_hasSubnormalFlushingControl(/*kind=*/3);
     targetCharacteristics.set_hasSubnormalFlushingControl(/*kind=*/4);
     targetCharacteristics.set_hasSubnormalFlushingControl(/*kind=*/8);
   }
+
   if (targetTriple.isARM() || targetTriple.isAArch64()) {
     targetCharacteristics.set_haltingSupportIsUnknownAtCompileTime();
     targetCharacteristics.set_ieeeFeature(
         evaluate::IeeeFeature::Halting, false);
-  } else {
-    targetCharacteristics.set_ieeeFeature(evaluate::IeeeFeature::Halting);
+    targetCharacteristics.set_ieeeFeature(
+        evaluate::IeeeFeature::Standard, false);
+    targetCharacteristics.set_hasSubnormalFlushingControl(/*kind=*/3);
+    targetCharacteristics.set_hasSubnormalFlushingControl(/*kind=*/4);
+    targetCharacteristics.set_hasSubnormalFlushingControl(/*kind=*/8);
+  }
+
+  if (targetTriple.getArch() != llvm::Triple::ArchType::x86_64) {
+    targetCharacteristics.DisableType(
+        Fortran::common::TypeCategory::Real, /*kind=*/10);
   }
 
-  // Figure out if we can support F128: see
-  // flang/runtime/Float128Math/math-entries.h
-  // TODO: this should be taken from TargetInfo::getLongDoubleFormat to support
-  // cross-compilation
+  // Check for kind=16 support. See flang/runtime/Float128Math/math-entries.h.
+  // TODO: Take this from TargetInfo::getLongDoubleFormat for cross compilation.
 #ifdef FLANG_RUNTIME_F128_MATH_LIB
-  // we can use libquadmath wrappers
-  constexpr bool f128Support = true;
+  constexpr bool f128Support = true; // use libquadmath wrappers
 #elif HAS_LDBL128
-  // we can use libm wrappers
-  constexpr bool f128Support = true;
+  constexpr bool f128Support = true; // use libm wrappers
 #else
   constexpr bool f128Support = false;
 #endif
diff --git a/flang/lib/CMakeLists.txt b/flang/lib/CMakeLists.txt
index 2182e845b6a79..8b201d9a758a8 100644
--- a/flang/lib/CMakeLists.txt
+++ b/flang/lib/CMakeLists.txt
@@ -1,4 +1,3 @@
-add_subdirectory(Common)
 add_subdirectory(Evaluate)
 add_subdirectory(Decimal)
 add_subdirectory(Lower)
@@ -9,3 +8,7 @@ add_subdirectory(Frontend)
 add_subdirectory(FrontendTool)
 
 add_subdirectory(Optimizer)
+
+if (FLANG_INCLUDE_TESTS)
+  add_subdirectory(Testing)
+endif ()
diff --git a/flang/lib/Common/CMakeLists.txt b/flang/lib/Common/CMakeLists.txt
deleted file mode 100644
index 4b5df0a49f403..0000000000000
--- a/flang/lib/Common/CMakeLists.txt
+++ /dev/null
@@ -1,52 +0,0 @@
-find_first_existing_vc_file("${LLVM_MAIN_SRC_DIR}" llvm_vc)
-find_first_existing_vc_file("${FLANG_SOURCE_DIR}" flang_vc)
-
-# The VC revision include that we want to generate.
-set(version_inc "${CMAKE_CURRENT_BINARY_DIR}/VCSVersion.inc")
-
-set(generate_vcs_version_script "${LLVM_CMAKE_DIR}/GenerateVersionFromVCS.cmake")
-
-if(llvm_vc AND LLVM_APPEND_VC_REV)
-  set(llvm_source_dir ${LLVM_MAIN_SRC_DIR})
-endif()
-if(flang_vc AND LLVM_APPEND_VC_REV)
-  set(flang_source_dir ${FLANG_SOURCE_DIR})
-endif()
-
-# Create custom target to generate the VC revision include.
-add_custom_command(OUTPUT "${version_inc}"
-  DEPENDS "${llvm_vc}" "${flang_vc}" "${generate_vcs_version_script}"
-  COMMAND ${CMAKE_COMMAND} "-DNAMES=\"LLVM;FLANG\""
-                           "-DLLVM_SOURCE_DIR=${llvm_source_dir}"
-                           "-DFLANG_SOURCE_DIR=${flang_source_dir}"
-                           "-DHEADER_FILE=${version_inc}"
-                           "-DLLVM_FORCE_VC_REVISION=${LLVM_FORCE_VC_REVISION}"
-                           "-DLLVM_FORCE_VC_REPOSITORY=${LLVM_FORCE_VC_REPOSITORY}"
-                           -P "${generate_vcs_version_script}")
-
-# Mark the generated header as being generated.
-set_source_files_properties("${version_inc}"
-  PROPERTIES GENERATED TRUE
-             HEADER_FILE_ONLY TRUE)
-
-if(FLANG_VENDOR)
-  set_source_files_properties(Version.cpp
-    PROPERTIES COMPILE_DEFINITIONS "FLANG_VENDOR=\"${FLANG_VENDOR} \"")
-endif()
-
-add_flang_library(FortranCommon
-  Fortran.cpp
-  Fortran-features.cpp
-  default-kinds.cpp
-  idioms.cpp
-  LangOptions.cpp
-  OpenMP-utils.cpp
-  Version.cpp
-  ${version_inc}
-
-  LINK_COMPONENTS
-  Support
-
-  MLIR_LIBS
-  MLIRIR
-)
diff --git a/flang/lib/Decimal/CMakeLists.txt b/flang/lib/Decimal/CMakeLists.txt
index 880b190f1c581..477d44e0565eb 100644
--- a/flang/lib/Decimal/CMakeLists.txt
+++ b/flang/lib/Decimal/CMakeLists.txt
@@ -1,86 +1,12 @@
-if (CMAKE_SOURCE_DIR STREQUAL CMAKE_CURRENT_SOURCE_DIR)
-  cmake_minimum_required(VERSION 3.20.0)
-
-  project(FortranDecimal C CXX)
-
-  set(CMAKE_CXX_STANDARD 17)
-  set(CMAKE_CXX_STANDARD_REQUIRED TRUE)
-  set(CMAKE_CXX_EXTENSIONS OFF)
-
-  set(FLANG_SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/../..")
-
-  set(LLVM_COMMON_CMAKE_UTILS "${FLANG_SOURCE_DIR}/../cmake")
-  set(LLVM_CMAKE_UTILS "${FLANG_SOURCE_DIR}/../llvm/cmake")
-  set(CLANG_CMAKE_UTILS "${FLANG_SOURCE_DIR}/../clang/cmake")
-
-  # Add path for custom modules
-  list(INSERT CMAKE_MODULE_PATH 0
-    "${FLANG_SOURCE_DIR}/cmake"
-    "${FLANG_SOURCE_DIR}/cmake/modules"
-    "${LLVM_COMMON_CMAKE_UTILS}"
-    "${LLVM_COMMON_CMAKE_UTILS}/Modules"
-    "${LLVM_CMAKE_UTILS}"
-    "${LLVM_CMAKE_UTILS}/modules"
-    "${CLANG_CMAKE_UTILS}/modules"
-    )
-
-  include(AddClang)
-  include(AddLLVM)
-  include(AddFlang)
-  include(HandleLLVMOptions)
-
-  include(TestBigEndian)
-  test_big_endian(IS_BIGENDIAN)
-  if (IS_BIGENDIAN)
-    add_compile_definitions(FLANG_BIG_ENDIAN=1)
-  else ()
-    add_compile_definitions(FLANG_LITTLE_ENDIAN=1)
-  endif ()
-  include_directories(BEFORE
-    ${FLANG_SOURCE_DIR}/include)
-endif()
-
-check_cxx_compiler_flag(-fno-lto FLANG_RUNTIME_HAS_FNO_LTO_FLAG)
-if (FLANG_RUNTIME_HAS_FNO_LTO_FLAG)
-  append("-fno-lto" CMAKE_CXX_FLAGS)
-endif()
-
-# Disable libstdc++ assertions, even in an LLVM_ENABLE_ASSERTIONS build, to
-# avoid an unwanted dependency on libstdc++.so.
-add_definitions(-U_GLIBCXX_ASSERTIONS)
-
-set(sources
+#===-- lib/Decimal/CMakeLists.txt ------------------------------------------===#
+#
+# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+# See https://llvm.org/LICENSE.txt for license information.
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+#
+#===------------------------------------------------------------------------===#
+
+add_flang_library(FortranDecimal
   binary-to-decimal.cpp
   decimal-to-binary.cpp
 )
-
-include(AddFlangOffloadRuntime)
-enable_cuda_compilation(FortranDecimal "${sources}")
-enable_omp_offload_compilation("${sources}")
-
-add_flang_library(FortranDecimal INSTALL_WITH_TOOLCHAIN ${sources})
-
-if (DEFINED MSVC)
-  set(CMAKE_MSVC_RUNTIME_LIBRARY MultiThreaded)
-  add_flang_library(FortranDecimal.static INSTALL_WITH_TOOLCHAIN
-          binary-to-decimal.cpp
-          decimal-to-binary.cpp
-  )
-  set(CMAKE_MSVC_RUNTIME_LIBRARY MultiThreadedDLL)
-  add_flang_library(FortranDecimal.dynamic INSTALL_WITH_TOOLCHAIN
-    binary-to-decimal.cpp
-    decimal-to-binary.cpp
-  )
-  set(CMAKE_MSVC_RUNTIME_LIBRARY MultiThreadedDebug)
-  add_flang_library(FortranDecimal.static_dbg INSTALL_WITH_TOOLCHAIN
-          binary-to-decimal.cpp
-          decimal-to-binary.cpp
-  )
-  set(CMAKE_MSVC_RUNTIME_LIBRARY MultiThreadedDebugDLL)
-  add_flang_library(FortranDecimal.dynamic_dbg INSTALL_WITH_TOOLCHAIN
-    binary-to-decimal.cpp
-    decimal-to-binary.cpp
-  )
-  add_dependencies(FortranDecimal FortranDecimal.static FortranDecimal.dynamic
-    FortranDecimal.static_dbg FortranDecimal.dynamic_dbg)
-endif()
diff --git a/flang/lib/Evaluate/CMakeLists.txt b/flang/lib/Evaluate/CMakeLists.txt
index b38f450d746ea..e0bdb6b2d24d8 100644
--- a/flang/lib/Evaluate/CMakeLists.txt
+++ b/flang/lib/Evaluate/CMakeLists.txt
@@ -60,7 +60,7 @@ add_flang_library(FortranEvaluate
   variable.cpp
 
   LINK_LIBS
-  FortranCommon
+  FortranSupport
   FortranDecimal
   FortranParser
   ${LIBPGMATH}
diff --git a/flang/lib/Evaluate/call.cpp b/flang/lib/Evaluate/call.cpp
index c5b50e806d249..f77df92a7597a 100644
--- a/flang/lib/Evaluate/call.cpp
+++ b/flang/lib/Evaluate/call.cpp
@@ -7,13 +7,13 @@
 //===----------------------------------------------------------------------===//
 
 #include "flang/Evaluate/call.h"
-#include "flang/Common/Fortran.h"
 #include "flang/Common/idioms.h"
 #include "flang/Evaluate/characteristics.h"
 #include "flang/Evaluate/check-expression.h"
 #include "flang/Evaluate/expression.h"
 #include "flang/Evaluate/tools.h"
 #include "flang/Semantics/symbol.h"
+#include "flang/Support/Fortran.h"
 
 namespace Fortran::evaluate {
 
diff --git a/flang/lib/Evaluate/characteristics.cpp b/flang/lib/Evaluate/characteristics.cpp
index 3912d1c4b4771..c5470df2622a5 100644
--- a/flang/lib/Evaluate/characteristics.cpp
+++ b/flang/lib/Evaluate/characteristics.cpp
@@ -227,9 +227,8 @@ void TypeAndShape::AcquireAttrs(const semantics::Symbol &symbol) {
   } else if (semantics::IsAssumedSizeArray(symbol)) {
     attrs_.set(Attr::AssumedSize);
   }
-  if (int n{GetCorank(symbol)}) {
-    corank_ = n;
-    attrs_.set(Attr::Coarray);
+  if (int corank{GetCorank(symbol)}; corank > 0) {
+    corank_ = corank;
   }
   if (const auto *object{
           symbol.GetUltimate().detailsIf<semantics::ObjectEntityDetails>()};
@@ -439,9 +438,9 @@ bool DummyDataObject::CanBePassedViaImplicitInterface(
     return false; // 15.4.2.2(3)(a)
   } else if ((type.attrs() &
                  TypeAndShape::Attrs{TypeAndShape::Attr::AssumedShape,
-                     TypeAndShape::Attr::AssumedRank,
-                     TypeAndShape::Attr::Coarray})
-                 .any()) {
+                     TypeAndShape::Attr::AssumedRank})
+                 .any() ||
+      type.corank() > 0) {
     if (whyNot) {
       *whyNot = "a dummy argument is assumed-shape, assumed-rank, or a coarray";
     }
@@ -471,14 +470,15 @@ bool DummyDataObject::CanBePassedViaImplicitInterface(
 }
 
 bool DummyDataObject::IsPassedByDescriptor(bool isBindC) const {
-  constexpr TypeAndShape::Attrs shapeRequiringBox = {
+  constexpr TypeAndShape::Attrs shapeRequiringBox{
       TypeAndShape::Attr::AssumedShape, TypeAndShape::Attr::DeferredShape,
-      TypeAndShape::Attr::AssumedRank, TypeAndShape::Attr::Coarray};
+      TypeAndShape::Attr::AssumedRank};
   if ((attrs & Attrs{Attr::Allocatable, Attr::Pointer}).any()) {
     return true;
   } else if ((type.attrs() & shapeRequiringBox).any()) {
-    // Need to pass shape/coshape info in a descriptor.
-    return true;
+    return true; // pass shape in descriptor
+  } else if (type.corank() > 0) {
+    return true; // pass coshape in descriptor
   } else if (type.type().IsPolymorphic() && !type.type().IsAssumedType()) {
     // Need to pass dynamic type info in a descriptor.
     return true;
diff --git a/flang/lib/Evaluate/check-expression.cpp b/flang/lib/Evaluate/check-expression.cpp
index 726a0ab35ede4..6ace5bbcd0c77 100644
--- a/flang/lib/Evaluate/check-expression.cpp
+++ b/flang/lib/Evaluate/check-expression.cpp
@@ -833,7 +833,10 @@ class IsContiguousHelper
 public:
   using Result = std::optional<bool>; // tri-state
   using Base = AnyTraverse<IsContiguousHelper, Result>;
-  explicit IsContiguousHelper(FoldingContext &c) : Base{*this}, context_{c} {}
+  explicit IsContiguousHelper(
+      FoldingContext &c, bool namedConstantSectionsAreContiguous)
+      : Base{*this}, context_{c}, namedConstantSectionsAreContiguous_{
+                                      namedConstantSectionsAreContiguous} {}
   using Base::operator();
 
   template <typename T> Result operator()(const Constant<T> &) const {
@@ -856,6 +859,11 @@ class IsContiguousHelper
       // RANK(*) associating entity is contiguous.
       if (details->IsAssumedSize()) {
         return true;
+      } else if (!IsVariable(details->expr()) &&
+          (namedConstantSectionsAreContiguous_ ||
+              !ExtractDataRef(details->expr(), true, true))) {
+        // Selector is associated to an expression value.
+        return true;
       } else {
         return Base::operator()(ultimate); // use expr
       }
@@ -1113,22 +1121,34 @@ class IsContiguousHelper
   }
 
   FoldingContext &context_;
+  bool namedConstantSectionsAreContiguous_{false};
 };
 
 template <typename A>
-std::optional<bool> IsContiguous(const A &x, FoldingContext &context) {
-  return IsContiguousHelper{context}(x);
+std::optional<bool> IsContiguous(const A &x, FoldingContext &context,
+    bool namedConstantSectionsAreContiguous) {
+  if (!IsVariable(x) &&
+      (namedConstantSectionsAreContiguous || !ExtractDataRef(x, true, true))) {
+    return true;
+  } else {
+    return IsContiguousHelper{context, namedConstantSectionsAreContiguous}(x);
+  }
 }
 
+template std::optional<bool> IsContiguous(const Expr<SomeType> &,
+    FoldingContext &, bool namedConstantSectionsAreContiguous);
+template std::optional<bool> IsContiguous(const ArrayRef &, FoldingContext &,
+    bool namedConstantSectionsAreContiguous);
+template std::optional<bool> IsContiguous(const Substring &, FoldingContext &,
+    bool namedConstantSectionsAreContiguous);
+template std::optional<bool> IsContiguous(const Component &, FoldingContext &,
+    bool namedConstantSectionsAreContiguous);
+template std::optional<bool> IsContiguous(const ComplexPart &, FoldingContext &,
+    bool namedConstantSectionsAreContiguous);
+template std::optional<bool> IsContiguous(const CoarrayRef &, FoldingContext &,
+    bool namedConstantSectionsAreContiguous);
 template std::optional<bool> IsContiguous(
-    const Expr<SomeType> &, FoldingContext &);
-template std::optional<bool> IsContiguous(const ArrayRef &, FoldingContext &);
-template std::optional<bool> IsContiguous(const Substring &, FoldingContext &);
-template std::optional<bool> IsContiguous(const Component &, FoldingContext &);
-template std::optional<bool> IsContiguous(
-    const ComplexPart &, FoldingContext &);
-template std::optional<bool> IsContiguous(const CoarrayRef &, FoldingContext &);
-template std::optional<bool> IsContiguous(const Symbol &, FoldingContext &);
+    const Symbol &, FoldingContext &, bool namedConstantSectionsAreContiguous);
 
 // IsErrorExpr()
 struct IsErrorExprHelper : public AnyTraverse<IsErrorExprHelper, bool> {
diff --git a/flang/lib/Evaluate/fold-implementation.h b/flang/lib/Evaluate/fold-implementation.h
index 31d043f490fd8..b0f39e63d0941 100644
--- a/flang/lib/Evaluate/fold-implementation.h
+++ b/flang/lib/Evaluate/fold-implementation.h
@@ -290,6 +290,9 @@ std::optional<Constant<T>> Folder<T>::ApplyComponent(
             auto *typedExpr{UnwrapExpr<Expr<T>>(expr.value())};
             CHECK(typedExpr);
             array = std::make_unique<ArrayConstructor<T>>(*typedExpr);
+            if constexpr (T::category == TypeCategory::Character) {
+              array->set_LEN(Expr<SubscriptInteger>{value->LEN()});
+            }
           }
           if (subscripts) {
             if (auto element{ApplySubscripts(*value, *subscripts)}) {
@@ -407,6 +410,7 @@ template <typename T> Expr<T> Folder<T>::Folding(Designator<T> &&designator) {
 template <typename T>
 Constant<T> *Folder<T>::Folding(std::optional<ActualArgument> &arg) {
   if (auto *expr{UnwrapExpr<Expr<SomeType>>(arg)}) {
+    *expr = Fold(context_, std::move(*expr));
     if constexpr (T::category != TypeCategory::Derived) {
       if (!UnwrapExpr<Expr<T>>(*expr)) {
         if (const Symbol *
@@ -1263,6 +1267,12 @@ template <typename T> class ArrayConstructorFolder {
   explicit ArrayConstructorFolder(FoldingContext &c) : context_{c} {}
 
   Expr<T> FoldArray(ArrayConstructor<T> &&array) {
+    if constexpr (T::category == TypeCategory::Character) {
+      if (const auto *len{array.LEN()}) {
+        charLength_ = ToInt64(Fold(context_, common::Clone(*len)));
+        knownCharLength_ = charLength_.has_value();
+      }
+    }
     // Calls FoldArray(const ArrayConstructorValues<T> &) below
     if (FoldArray(array)) {
       auto n{static_cast<ConstantSubscript>(elements_.size())};
@@ -1270,12 +1280,9 @@ template <typename T> class ArrayConstructorFolder {
         return Expr<T>{Constant<T>{array.GetType().GetDerivedTypeSpec(),
             std::move(elements_), ConstantSubscripts{n}}};
       } else if constexpr (T::category == TypeCategory::Character) {
-        if (const auto *len{array.LEN()}) {
-          auto length{Fold(context_, common::Clone(*len))};
-          if (std::optional<ConstantSubscript> lengthValue{ToInt64(length)}) {
-            return Expr<T>{Constant<T>{
-                *lengthValue, std::move(elements_), ConstantSubscripts{n}}};
-          }
+        if (charLength_) {
+          return Expr<T>{Constant<T>{
+              *charLength_, std::move(elements_), ConstantSubscripts{n}}};
         }
       } else {
         return Expr<T>{
@@ -1296,6 +1303,11 @@ template <typename T> class ArrayConstructorFolder {
           elements_.emplace_back(c->At(index));
         } while (c->IncrementSubscripts(index));
       }
+      if constexpr (T::category == TypeCategory::Character) {
+        if (!knownCharLength_) {
+          charLength_ = std::max(c->LEN(), charLength_.value_or(-1));
+        }
+      }
       return true;
     } else {
       return false;
@@ -1345,6 +1357,8 @@ template <typename T> class ArrayConstructorFolder {
 
   FoldingContext &context_;
   std::vector<Scalar<T>> elements_;
+  std::optional<ConstantSubscript> charLength_;
+  bool knownCharLength_{false};
 };
 
 template <typename T>
diff --git a/flang/lib/Evaluate/formatting.cpp b/flang/lib/Evaluate/formatting.cpp
index f3a53c1f983df..c4c3998257830 100644
--- a/flang/lib/Evaluate/formatting.cpp
+++ b/flang/lib/Evaluate/formatting.cpp
@@ -7,7 +7,6 @@
 //===----------------------------------------------------------------------===//
 
 #include "flang/Evaluate/formatting.h"
-#include "flang/Common/Fortran.h"
 #include "flang/Evaluate/call.h"
 #include "flang/Evaluate/constant.h"
 #include "flang/Evaluate/expression.h"
@@ -16,6 +15,7 @@
 #include "flang/Parser/characters.h"
 #include "flang/Semantics/semantics.h"
 #include "flang/Semantics/symbol.h"
+#include "flang/Support/Fortran.h"
 #include "llvm/Support/raw_ostream.h"
 
 namespace Fortran::evaluate {
diff --git a/flang/lib/Evaluate/intrinsics-library.cpp b/flang/lib/Evaluate/intrinsics-library.cpp
index c1b270f518c0e..d2c1be65dca44 100644
--- a/flang/lib/Evaluate/intrinsics-library.cpp
+++ b/flang/lib/Evaluate/intrinsics-library.cpp
@@ -260,6 +260,16 @@ struct HostRuntimeLibrary<HostT, LibraryVersion::Libm> {
   static_assert(map.Verify(), "map must be sorted");
 };
 
+#define COMPLEX_SIGNATURES(HOST_T) \
+  using F = FuncPointer<std::complex<HOST_T>, const std::complex<HOST_T> &>; \
+  using F2 = FuncPointer<std::complex<HOST_T>, const std::complex<HOST_T> &, \
+      const std::complex<HOST_T> &>; \
+  using F2A = FuncPointer<std::complex<HOST_T>, const HOST_T &, \
+      const std::complex<HOST_T> &>; \
+  using F2B = FuncPointer<std::complex<HOST_T>, const std::complex<HOST_T> &, \
+      const HOST_T &>;
+
+#ifndef _AIX
 // Helpers to map complex std::pow whose resolution in F2{std::pow} is
 // ambiguous as of clang++ 20.
 template <typename HostT>
@@ -267,98 +277,24 @@ static std::complex<HostT> StdPowF2(
     const std::complex<HostT> &x, const std::complex<HostT> &y) {
   return std::pow(x, y);
 }
+
 template <typename HostT>
 static std::complex<HostT> StdPowF2A(
     const HostT &x, const std::complex<HostT> &y) {
   return std::pow(x, y);
 }
+
 template <typename HostT>
 static std::complex<HostT> StdPowF2B(
     const std::complex<HostT> &x, const HostT &y) {
   return std::pow(x, y);
 }
 
-#ifdef _AIX
-#ifdef __clang_major__
-#pragma clang diagnostic ignored "-Wc99-extensions"
-#endif
-
-extern "C" {
-float _Complex cacosf(float _Complex);
-double _Complex cacos(double _Complex);
-float _Complex csqrtf(float _Complex);
-double _Complex csqrt(double _Complex);
-}
-
-enum CRI { Real, Imag };
-template <typename TR, typename TA> static TR &reIm(TA &x, CRI n) {
-  return reinterpret_cast<TR(&)[2]>(x)[n];
-}
-template <typename TR, typename T> static TR CppToC(const std::complex<T> &x) {
-  TR r;
-  reIm<T, TR>(r, CRI::Real) = x.real();
-  reIm<T, TR>(r, CRI::Imag) = x.imag();
-  return r;
-}
-template <typename T, typename TA> static std::complex<T> CToCpp(const TA &x) {
-  TA &z{const_cast<TA &>(x)};
-  return std::complex<T>(reIm<T, TA>(z, CRI::Real), reIm<T, TA>(z, CRI::Imag));
-}
-#endif
-
-template <typename HostT>
-static std::complex<HostT> CSqrt(const std::complex<HostT> &x) {
-  std::complex<HostT> res;
-#ifdef _AIX
-  // On AIX, the implementation of csqrt[f] and std::sqrt is different,
-  // use csqrt[f] in folding.
-  if constexpr (std::is_same_v<HostT, float>) {
-    float _Complex r{csqrtf(CppToC<float _Complex, float>(x))};
-    res = CToCpp<float, float _Complex>(r);
-  } else if constexpr (std::is_same_v<HostT, double>) {
-    double _Complex r{csqrt(CppToC<double _Complex, double>(x))};
-    res = CToCpp<double, double _Complex>(r);
-  } else {
-    DIE("bad complex component type");
-  }
-#else
-  res = std::sqrt(x);
-#endif
-  return res;
-}
-
-template <typename HostT>
-static std::complex<HostT> CAcos(const std::complex<HostT> &x) {
-  std::complex<HostT> res;
-#ifdef _AIX
-  // On AIX, the implementation of cacos[f] and std::acos is different,
-  // use cacos[f] in folding.
-  if constexpr (std::is_same_v<HostT, float>) {
-    float _Complex r{cacosf(CppToC<float _Complex, float>(x))};
-    res = CToCpp<float, float _Complex>(r);
-  } else if constexpr (std::is_same_v<HostT, double>) {
-    double _Complex r{cacos(CppToC<double _Complex, double>(x))};
-    res = CToCpp<double, double _Complex>(r);
-  } else {
-    DIE("bad complex component type");
-  }
-#else
-  res = std::acos(x);
-#endif
-  return res;
-}
-
 template <typename HostT>
 struct HostRuntimeLibrary<std::complex<HostT>, LibraryVersion::Libm> {
-  using F = FuncPointer<std::complex<HostT>, const std::complex<HostT> &>;
-  using F2 = FuncPointer<std::complex<HostT>, const std::complex<HostT> &,
-      const std::complex<HostT> &>;
-  using F2A = FuncPointer<std::complex<HostT>, const HostT &,
-      const std::complex<HostT> &>;
-  using F2B = FuncPointer<std::complex<HostT>, const std::complex<HostT> &,
-      const HostT &>;
+  COMPLEX_SIGNATURES(HostT)
   static constexpr HostRuntimeFunction table[]{
-      FolderFactory<F, F{CAcos}>::Create("acos"),
+      FolderFactory<F, F{std::acos}>::Create("acos"),
       FolderFactory<F, F{std::acosh}>::Create("acosh"),
       FolderFactory<F, F{std::asin}>::Create("asin"),
       FolderFactory<F, F{std::asinh}>::Create("asinh"),
@@ -373,13 +309,129 @@ struct HostRuntimeLibrary<std::complex<HostT>, LibraryVersion::Libm> {
       FolderFactory<F2B, F2B{StdPowF2B}>::Create("pow"),
       FolderFactory<F, F{std::sin}>::Create("sin"),
       FolderFactory<F, F{std::sinh}>::Create("sinh"),
-      FolderFactory<F, F{CSqrt}>::Create("sqrt"),
+      FolderFactory<F, F{std::sqrt}>::Create("sqrt"),
       FolderFactory<F, F{std::tan}>::Create("tan"),
       FolderFactory<F, F{std::tanh}>::Create("tanh"),
   };
   static constexpr HostRuntimeMap map{table};
   static_assert(map.Verify(), "map must be sorted");
 };
+#else
+// On AIX, call libm routines to preserve consistent value between
+// runtime and compile time evaluation.
+#ifdef __clang_major__
+#pragma clang diagnostic ignored "-Wc99-extensions"
+#endif
+
+extern "C" {
+float _Complex cacosf(float _Complex);
+double _Complex cacos(double _Complex);
+float _Complex cacoshf(float _Complex);
+double _Complex cacosh(double _Complex);
+float _Complex casinf(float _Complex);
+double _Complex casin(double _Complex);
+float _Complex casinhf(float _Complex);
+double _Complex casinh(double _Complex);
+float _Complex catanf(float _Complex);
+double _Complex catan(double _Complex);
+float _Complex catanhf(float _Complex);
+double _Complex catanh(double _Complex);
+float _Complex ccosf(float _Complex);
+double _Complex ccos(double _Complex);
+float _Complex ccoshf(float _Complex);
+double _Complex ccosh(double _Complex);
+float _Complex cexpf(float _Complex);
+double _Complex cexp(double _Complex);
+float _Complex clogf(float _Complex);
+double _Complex __clog(double _Complex);
+float _Complex cpowf(float _Complex, float _Complex);
+double _Complex cpow(double _Complex, double _Complex);
+float _Complex csinf(float _Complex);
+double _Complex csin(double _Complex);
+float _Complex csinhf(float _Complex);
+double _Complex csinh(double _Complex);
+float _Complex csqrtf(float _Complex);
+double _Complex csqrt(double _Complex);
+float _Complex ctanf(float _Complex);
+double _Complex ctan(double _Complex);
+float _Complex ctanhf(float _Complex);
+double _Complex ctanh(double _Complex);
+}
+
+template <typename T> struct ToStdComplex {
+  using Type = T;
+  using AType = Type;
+};
+template <> struct ToStdComplex<float _Complex> {
+  using Type = std::complex<float>;
+  using AType = const Type &;
+};
+template <> struct ToStdComplex<double _Complex> {
+  using Type = std::complex<double>;
+  using AType = const Type &;
+};
+
+template <typename F, F func> struct CComplexFunc {};
+template <typename R, typename... A, FuncPointer<R, A...> func>
+struct CComplexFunc<FuncPointer<R, A...>, func> {
+  static typename ToStdComplex<R>::Type wrapper(
+      typename ToStdComplex<A>::AType... args) {
+    R res{func(*reinterpret_cast<const A *>(&args)...)};
+    return *reinterpret_cast<typename ToStdComplex<R>::Type *>(&res);
+  }
+};
+#define C_COMPLEX_FUNC(func) CComplexFunc<decltype(&func), &func>::wrapper
+
+template <>
+struct HostRuntimeLibrary<std::complex<float>, LibraryVersion::Libm> {
+  COMPLEX_SIGNATURES(float)
+  static constexpr HostRuntimeFunction table[]{
+      FolderFactory<F, C_COMPLEX_FUNC(cacosf)>::Create("acos"),
+      FolderFactory<F, C_COMPLEX_FUNC(cacoshf)>::Create("acosh"),
+      FolderFactory<F, C_COMPLEX_FUNC(casinf)>::Create("asin"),
+      FolderFactory<F, C_COMPLEX_FUNC(casinhf)>::Create("asinh"),
+      FolderFactory<F, C_COMPLEX_FUNC(catanf)>::Create("atan"),
+      FolderFactory<F, C_COMPLEX_FUNC(catanhf)>::Create("atanh"),
+      FolderFactory<F, C_COMPLEX_FUNC(ccosf)>::Create("cos"),
+      FolderFactory<F, C_COMPLEX_FUNC(ccoshf)>::Create("cosh"),
+      FolderFactory<F, C_COMPLEX_FUNC(cexpf)>::Create("exp"),
+      FolderFactory<F, C_COMPLEX_FUNC(clogf)>::Create("log"),
+      FolderFactory<F2, C_COMPLEX_FUNC(cpowf)>::Create("pow"),
+      FolderFactory<F, C_COMPLEX_FUNC(csinf)>::Create("sin"),
+      FolderFactory<F, C_COMPLEX_FUNC(csinhf)>::Create("sinh"),
+      FolderFactory<F, C_COMPLEX_FUNC(csqrtf)>::Create("sqrt"),
+      FolderFactory<F, C_COMPLEX_FUNC(ctanf)>::Create("tan"),
+      FolderFactory<F, C_COMPLEX_FUNC(ctanhf)>::Create("tanh"),
+  };
+  static constexpr HostRuntimeMap map{table};
+  static_assert(map.Verify(), "map must be sorted");
+};
+template <>
+struct HostRuntimeLibrary<std::complex<double>, LibraryVersion::Libm> {
+  COMPLEX_SIGNATURES(double)
+  static constexpr HostRuntimeFunction table[]{
+      FolderFactory<F, C_COMPLEX_FUNC(cacos)>::Create("acos"),
+      FolderFactory<F, C_COMPLEX_FUNC(cacosh)>::Create("acosh"),
+      FolderFactory<F, C_COMPLEX_FUNC(casin)>::Create("asin"),
+      FolderFactory<F, C_COMPLEX_FUNC(casinh)>::Create("asinh"),
+      FolderFactory<F, C_COMPLEX_FUNC(catan)>::Create("atan"),
+      FolderFactory<F, C_COMPLEX_FUNC(catanh)>::Create("atanh"),
+      FolderFactory<F, C_COMPLEX_FUNC(ccos)>::Create("cos"),
+      FolderFactory<F, C_COMPLEX_FUNC(ccosh)>::Create("cosh"),
+      FolderFactory<F, C_COMPLEX_FUNC(cexp)>::Create("exp"),
+      FolderFactory<F, C_COMPLEX_FUNC(__clog)>::Create("log"),
+      FolderFactory<F2, C_COMPLEX_FUNC(cpow)>::Create("pow"),
+      FolderFactory<F, C_COMPLEX_FUNC(csin)>::Create("sin"),
+      FolderFactory<F, C_COMPLEX_FUNC(csinh)>::Create("sinh"),
+      FolderFactory<F, C_COMPLEX_FUNC(csqrt)>::Create("sqrt"),
+      FolderFactory<F, C_COMPLEX_FUNC(ctan)>::Create("tan"),
+      FolderFactory<F, C_COMPLEX_FUNC(ctanh)>::Create("tanh"),
+  };
+  static constexpr HostRuntimeMap map{table};
+  static_assert(map.Verify(), "map must be sorted");
+};
+#endif // _AIX
+
 // Note regarding cmath:
 //  - cmath does not have modulo and erfc_scaled equivalent
 //  - C++17 defined standard Bessel math functions std::cyl_bessel_j
diff --git a/flang/lib/Evaluate/intrinsics.cpp b/flang/lib/Evaluate/intrinsics.cpp
index f234241cfe14a..5c9527cec2f69 100644
--- a/flang/lib/Evaluate/intrinsics.cpp
+++ b/flang/lib/Evaluate/intrinsics.cpp
@@ -7,7 +7,6 @@
 //===----------------------------------------------------------------------===//
 
 #include "flang/Evaluate/intrinsics.h"
-#include "flang/Common/Fortran.h"
 #include "flang/Common/enum-set.h"
 #include "flang/Common/idioms.h"
 #include "flang/Evaluate/check-expression.h"
@@ -19,6 +18,7 @@
 #include "flang/Evaluate/type.h"
 #include "flang/Semantics/scope.h"
 #include "flang/Semantics/tools.h"
+#include "flang/Support/Fortran.h"
 #include "llvm/Support/raw_ostream.h"
 #include <algorithm>
 #include <cmath>
@@ -96,6 +96,7 @@ ENUM_CLASS(KindCode, none, defaultIntegerKind,
     typeless, // BOZ literals are INTEGER with this kind
     ieeeFlagType, // IEEE_FLAG_TYPE from ISO_FORTRAN_EXCEPTION
     ieeeRoundType, // IEEE_ROUND_TYPE from ISO_FORTRAN_ARITHMETIC
+    eventType, // EVENT_TYPE from module ISO_FORTRAN_ENV (for coarrays)
     teamType, // TEAM_TYPE from module ISO_FORTRAN_ENV (for coarrays)
     kindArg, // this argument is KIND=
     effectiveKind, // for function results: "kindArg" value, possibly defaulted
@@ -129,6 +130,7 @@ static constexpr TypePattern DefaultChar{CharType, KindCode::defaultCharKind};
 static constexpr TypePattern DefaultLogical{
     LogicalType, KindCode::defaultLogicalKind};
 static constexpr TypePattern BOZ{IntType, KindCode::typeless};
+static constexpr TypePattern EventType{DerivedType, KindCode::eventType};
 static constexpr TypePattern IeeeFlagType{DerivedType, KindCode::ieeeFlagType};
 static constexpr TypePattern IeeeRoundType{
     DerivedType, KindCode::ieeeRoundType};
@@ -404,6 +406,8 @@ static const IntrinsicInterface genericIntrinsicFunction[]{
         DefaultLogical},
     {"ceiling", {{"a", AnyReal}, DefaultingKIND}, KINDInt},
     {"char", {{"i", AnyInt, Rank::elementalOrBOZ}, DefaultingKIND}, KINDChar},
+    {"chdir", {{"name", DefaultChar, Rank::scalar, Optionality::required}},
+        DefaultInt},
     {"cmplx", {{"x", AnyComplex}, DefaultingKIND}, KINDComplex},
     {"cmplx",
         {{"x", AnyIntUnsignedOrReal, Rank::elementalOrBOZ},
@@ -1311,6 +1315,14 @@ static const SpecificIntrinsicInterface specificIntrinsicFunction[]{
 
 static const IntrinsicInterface intrinsicSubroutine[]{
     {"abort", {}, {}, Rank::elemental, IntrinsicClass::impureSubroutine},
+    {"atomic_add",
+        {{"atom", AtomicInt, Rank::atom, Optionality::required,
+             common::Intent::InOut},
+            {"value", AnyInt, Rank::scalar, Optionality::required,
+                common::Intent::In},
+            {"stat", AnyInt, Rank::scalar, Optionality::optional,
+                common::Intent::Out}},
+        {}, Rank::elemental, IntrinsicClass::atomicSubroutine},
     {"atomic_and",
         {{"atom", AtomicInt, Rank::atom, Optionality::required,
              common::Intent::InOut},
@@ -1403,6 +1415,11 @@ static const IntrinsicInterface intrinsicSubroutine[]{
             {"stat", AnyInt, Rank::scalar, Optionality::optional,
                 common::Intent::Out}},
         {}, Rank::elemental, IntrinsicClass::atomicSubroutine},
+    {"chdir",
+        {{"name", DefaultChar, Rank::scalar, Optionality::required},
+            {"status", AnyInt, Rank::scalar, Optionality::optional,
+                common::Intent::Out}},
+        {}, Rank::elemental, IntrinsicClass::impureSubroutine},
     {"co_broadcast",
         {{"a", AnyData, Rank::anyOrAssumedRank, Optionality::required,
              common::Intent::InOut},
@@ -1463,6 +1480,13 @@ static const IntrinsicInterface intrinsicSubroutine[]{
             {"time", TypePattern{RealType, KindCode::exactKind, 4},
                 Rank::scalar, Optionality::required, common::Intent::Out}},
         {}, Rank::elemental, IntrinsicClass::impureSubroutine},
+    {"event_query",
+        {{"event", EventType, Rank::scalar},
+            {"count", AnyInt, Rank::scalar, Optionality::required,
+                common::Intent::Out},
+            {"stat", AnyInt, Rank::scalar, Optionality::optional,
+                common::Intent::Out}},
+        {}, Rank::elemental, IntrinsicClass::impureSubroutine},
     {"execute_command_line",
         {{"command", DefaultChar, Rank::scalar},
             {"wait", AnyLogical, Rank::scalar, Optionality::optional},
@@ -1584,8 +1608,6 @@ static const IntrinsicInterface intrinsicSubroutine[]{
         {}, Rank::elemental, IntrinsicClass::impureSubroutine},
 };
 
-// TODO: Intrinsic subroutine EVENT_QUERY
-// TODO: Atomic intrinsic subroutines: ATOMIC_ADD
 // TODO: Collective intrinsic subroutines: co_reduce
 
 // Finds a built-in derived type and returns it as a DynamicType.
@@ -1713,8 +1735,8 @@ static bool CheckAndPushMinMaxArgument(ActualArgument &arg,
 }
 
 static bool CheckAtomicKind(const ActualArgument &arg,
-    const semantics::Scope *builtinsScope,
-    parser::ContextualMessages &messages) {
+    const semantics::Scope *builtinsScope, parser::ContextualMessages &messages,
+    const char *keyword) {
   std::string atomicKindStr;
   std::optional<DynamicType> type{arg.GetType()};
 
@@ -1727,11 +1749,12 @@ static bool CheckAtomicKind(const ActualArgument &arg,
                 "must be used with IntType or LogicalType");
   }
 
-  bool argOk = type->kind() ==
-      GetBuiltinKind(builtinsScope, ("__builtin_" + atomicKindStr).c_str());
+  bool argOk{type->kind() ==
+      GetBuiltinKind(builtinsScope, ("__builtin_" + atomicKindStr).c_str())};
   if (!argOk) {
     messages.Say(arg.sourceLocation(),
-        "Actual argument for 'atom=' must have kind=atomic_int_kind or atomic_logical_kind, but is '%s'"_err_en_US,
+        "Actual argument for '%s=' must have kind=atomic_%s_kind, but is '%s'"_err_en_US,
+        keyword, type->category() == TypeCategory::Integer ? "int" : "logical",
         type->AsFortran());
   }
   return argOk;
@@ -1960,6 +1983,11 @@ std::optional<SpecificCall> IntrinsicInterface::Match(
     case KindCode::typeless:
       argOk = false;
       break;
+    case KindCode::eventType:
+      argOk = !type->IsUnlimitedPolymorphic() &&
+          type->category() == TypeCategory::Derived &&
+          semantics::IsEventType(&type->GetDerivedTypeSpec());
+      break;
     case KindCode::ieeeFlagType:
       argOk = !type->IsUnlimitedPolymorphic() &&
           type->category() == TypeCategory::Derived &&
@@ -2008,11 +2036,16 @@ std::optional<SpecificCall> IntrinsicInterface::Match(
       if (!sameArg) {
         sameArg = arg;
       }
-      // Check both ways so that a CLASS(*) actuals to
-      // MOVE_ALLOC and EOSHIFT both work.
       auto sameType{sameArg->GetType().value()};
-      argOk = sameType.IsTkLenCompatibleWith(*type) ||
-          type->IsTkLenCompatibleWith(sameType);
+      if (name == "move_alloc"s) {
+        // second argument can be more general
+        argOk = type->IsTkLenCompatibleWith(sameType);
+      } else if (name == "merge"s) {
+        argOk = type->IsTkLenCompatibleWith(sameType) &&
+            sameType.IsTkLenCompatibleWith(*type);
+      } else {
+        argOk = sameType.IsTkLenCompatibleWith(*type);
+      }
     } break;
     case KindCode::sameKind:
       if (!sameArg) {
@@ -2052,7 +2085,7 @@ std::optional<SpecificCall> IntrinsicInterface::Match(
     case KindCode::sameAtom:
       if (!sameArg) {
         sameArg = arg;
-        argOk = CheckAtomicKind(DEREF(arg), builtinsScope, messages);
+        argOk = CheckAtomicKind(DEREF(arg), builtinsScope, messages, d.keyword);
       } else {
         argOk = type->IsTkCompatibleWith(sameArg->GetType().value());
         if (!argOk) {
@@ -2061,23 +2094,21 @@ std::optional<SpecificCall> IntrinsicInterface::Match(
               d.keyword, type->AsFortran());
         }
       }
-      if (!argOk)
+      if (!argOk) {
         return std::nullopt;
+      }
       break;
     case KindCode::atomicIntKind:
-      argOk = type->kind() ==
-          GetBuiltinKind(builtinsScope, "__builtin_atomic_int_kind");
+      argOk = CheckAtomicKind(DEREF(arg), builtinsScope, messages, d.keyword);
       if (!argOk) {
-        messages.Say(arg->sourceLocation(),
-            "Actual argument for '%s=' must have kind=atomic_int_kind, but is '%s'"_err_en_US,
-            d.keyword, type->AsFortran());
         return std::nullopt;
       }
       break;
     case KindCode::atomicIntOrLogicalKind:
-      argOk = CheckAtomicKind(DEREF(arg), builtinsScope, messages);
-      if (!argOk)
+      argOk = CheckAtomicKind(DEREF(arg), builtinsScope, messages, d.keyword);
+      if (!argOk) {
         return std::nullopt;
+      }
       break;
     default:
       CRASH_NO_CASE;
@@ -2338,7 +2369,7 @@ std::optional<SpecificCall> IntrinsicInterface::Match(
       if (kindArg) {
         if (auto *expr{kindArg->UnwrapExpr()}) {
           CHECK(expr->Rank() == 0);
-          if (auto code{ToInt64(*expr)}) {
+          if (auto code{ToInt64(Fold(context, common::Clone(*expr)))}) {
             if (context.targetCharacteristics().IsTypeEnabled(
                     *category, *code)) {
               if (*category == TypeCategory::Character) { // ACHAR & CHAR
@@ -2350,9 +2381,8 @@ std::optional<SpecificCall> IntrinsicInterface::Match(
             }
           }
         }
-        messages.Say("'kind=' argument must be a constant scalar integer "
-                     "whose value is a supported kind for the "
-                     "intrinsic result type"_err_en_US);
+        messages.Say(
+            "'kind=' argument must be a constant scalar integer whose value is a supported kind for the intrinsic result type"_err_en_US);
         // use default kind below for error recovery
       } else if (kindDummyArg->flags.test(ArgFlag::defaultsToSameKind)) {
         CHECK(sameArg);
@@ -2557,6 +2587,10 @@ std::optional<SpecificCall> IntrinsicInterface::Match(
             // Dummy procedures are never elemental.
             dummyProc->procedure.value().attrs.reset(
                 characteristics::Procedure::Attr::Elemental);
+          } else if (auto *dummyObject{
+                         std::get_if<characteristics::DummyDataObject>(
+                             &dc->u)}) {
+            dummyObject->type.set_corank(0);
           }
           dummyArgs.emplace_back(std::move(*dc));
           if (d.typePattern.kindCode == KindCode::same && !sameDummyArg) {
@@ -2719,8 +2753,8 @@ bool IntrinsicProcTable::Implementation::IsDualIntrinsic(
     const std::string &name) const {
   // Collection for some intrinsics with function and subroutine form,
   // in order to pass the semantic check.
-  static const std::string dualIntrinsic[]{
-      {"etime"s}, {"getcwd"s}, {"rename"s}, {"second"s}, {"system"s}};
+  static const std::string dualIntrinsic[]{{"chdir"s}, {"etime"s}, {"getcwd"s},
+      {"rename"s}, {"second"s}, {"system"s}};
 
   return llvm::is_contained(dualIntrinsic, name);
 }
@@ -3232,8 +3266,8 @@ static bool ApplySpecificChecks(SpecificCall &call, FoldingContext &context) {
           arg ? arg->sourceLocation() : context.messages().at(),
           "Argument of ALLOCATED() must be an ALLOCATABLE object or component"_err_en_US);
     }
-  } else if (name == "atomic_and" || name == "atomic_or" ||
-      name == "atomic_xor") {
+  } else if (name == "atomic_add" || name == "atomic_and" ||
+      name == "atomic_or" || name == "atomic_xor" || name == "event_query") {
     return CheckForCoindexedObject(
         context.messages(), call.arguments[2], name, "stat");
   } else if (name == "atomic_cas") {
diff --git a/flang/lib/Evaluate/shape.cpp b/flang/lib/Evaluate/shape.cpp
index 58b824d9b8e64..fa957cfc08495 100644
--- a/flang/lib/Evaluate/shape.cpp
+++ b/flang/lib/Evaluate/shape.cpp
@@ -16,6 +16,7 @@
 #include "flang/Evaluate/tools.h"
 #include "flang/Evaluate/type.h"
 #include "flang/Parser/message.h"
+#include "flang/Semantics/semantics.h"
 #include "flang/Semantics/symbol.h"
 #include <functional>
 
@@ -23,6 +24,10 @@ using namespace std::placeholders; // _1, _2, &c. for std::bind()
 
 namespace Fortran::evaluate {
 
+FoldingContext &GetFoldingContextFrom(const Symbol &symbol) {
+  return symbol.owner().context().foldingContext();
+}
+
 bool IsImpliedShape(const Symbol &original) {
   const Symbol &symbol{ResolveAssociations(original)};
   const auto *details{symbol.detailsIf<semantics::ObjectEntityDetails>()};
@@ -483,7 +488,7 @@ static MaybeExtentExpr GetAssociatedExtent(
     const Symbol &symbol, int dimension) {
   if (const auto *assoc{symbol.detailsIf<semantics::AssocEntityDetails>()};
       assoc && !assoc->rank()) { // not SELECT RANK case
-    if (auto shape{GetShape(assoc->expr())};
+    if (auto shape{GetShape(GetFoldingContextFrom(symbol), assoc->expr())};
         shape && dimension < static_cast<int>(shape->size())) {
       if (auto &extent{shape->at(dimension)};
           // Don't return a non-constant extent, as the variables that
@@ -519,7 +524,8 @@ MaybeExtentExpr GetExtent(
   }
   if (const auto *details{symbol.detailsIf<semantics::ObjectEntityDetails>()}) {
     if (IsImpliedShape(symbol) && details->init()) {
-      if (auto shape{GetShape(symbol, invariantOnly)}) {
+      if (auto shape{
+              GetShape(GetFoldingContextFrom(symbol), symbol, invariantOnly)}) {
         if (dimension < static_cast<int>(shape->size())) {
           return std::move(shape->at(dimension));
         }
@@ -568,7 +574,8 @@ MaybeExtentExpr GetExtent(const Subscript &subscript, const NamedEntity &base,
                 MaybeExtentExpr{triplet.stride()});
           },
           [&](const IndirectSubscriptIntegerExpr &subs) -> MaybeExtentExpr {
-            if (auto shape{GetShape(subs.value())};
+            if (auto shape{GetShape(
+                    GetFoldingContextFrom(base.GetLastSymbol()), subs.value())};
                 shape && GetRank(*shape) == 1) {
               // vector-valued subscript
               return std::move(shape->at(0));
diff --git a/flang/lib/Evaluate/tools.cpp b/flang/lib/Evaluate/tools.cpp
index 2d0e1996632fc..16b0260719097 100644
--- a/flang/lib/Evaluate/tools.cpp
+++ b/flang/lib/Evaluate/tools.cpp
@@ -212,6 +212,11 @@ ConvertRealOperandsResult ConvertRealOperands(
             return {AsSameKindExprs<TypeCategory::Real>(
                 ConvertTo(ry, std::move(bx)), std::move(ry))};
           },
+          [&](BOZLiteralConstant &&,
+              BOZLiteralConstant &&) -> ConvertRealOperandsResult {
+            messages.Say("operands cannot both be BOZ"_err_en_US);
+            return std::nullopt;
+          },
           [&](auto &&, auto &&) -> ConvertRealOperandsResult { // C718
             messages.Say(
                 "operands must be INTEGER, UNSIGNED, REAL, or BOZ"_err_en_US);
diff --git a/flang/lib/Evaluate/type.cpp b/flang/lib/Evaluate/type.cpp
index 0c2784d9cbe30..c8f75f91ed9c6 100644
--- a/flang/lib/Evaluate/type.cpp
+++ b/flang/lib/Evaluate/type.cpp
@@ -293,11 +293,13 @@ using SetOfDerivedTypePairs =
 
 static bool AreSameDerivedType(const semantics::DerivedTypeSpec &,
     const semantics::DerivedTypeSpec &, bool ignoreTypeParameterValues,
-    bool ignoreLenParameters, SetOfDerivedTypePairs &inProgress);
+    bool ignoreLenParameters, bool ignoreSequence,
+    SetOfDerivedTypePairs &inProgress);
 
 // F2023 7.5.3.2
 static bool AreSameComponent(const semantics::Symbol &x,
-    const semantics::Symbol &y, SetOfDerivedTypePairs &inProgress) {
+    const semantics::Symbol &y, bool ignoreSequence,
+    SetOfDerivedTypePairs &inProgress) {
   if (x.attrs() != y.attrs()) {
     return false;
   }
@@ -325,7 +327,8 @@ static bool AreSameComponent(const semantics::Symbol &x,
               !yType->IsUnlimitedPolymorphic() ||
           (!xType->IsUnlimitedPolymorphic() &&
               !AreSameDerivedType(xType->GetDerivedTypeSpec(),
-                  yType->GetDerivedTypeSpec(), false, false, inProgress))) {
+                  yType->GetDerivedTypeSpec(), false, false, ignoreSequence,
+                  inProgress))) {
         return false;
       }
     } else if (!xType->IsTkLenCompatibleWith(*yType)) {
@@ -449,7 +452,8 @@ static bool AreTypeParamCompatible(const semantics::DerivedTypeSpec &x,
 // F2023 7.5.3.2
 static bool AreSameDerivedType(const semantics::DerivedTypeSpec &x,
     const semantics::DerivedTypeSpec &y, bool ignoreTypeParameterValues,
-    bool ignoreLenParameters, SetOfDerivedTypePairs &inProgress) {
+    bool ignoreLenParameters, bool ignoreSequence,
+    SetOfDerivedTypePairs &inProgress) {
   if (&x == &y) {
     return true;
   }
@@ -472,7 +476,12 @@ static bool AreSameDerivedType(const semantics::DerivedTypeSpec &x,
   inProgress.insert(thisQuery);
   const auto &xDetails{xSymbol.get<semantics::DerivedTypeDetails>()};
   const auto &yDetails{ySymbol.get<semantics::DerivedTypeDetails>()};
-  if (!(xDetails.sequence() && yDetails.sequence()) &&
+  if (xDetails.sequence() != yDetails.sequence() ||
+      xSymbol.attrs().test(semantics::Attr::BIND_C) !=
+          ySymbol.attrs().test(semantics::Attr::BIND_C)) {
+    return false;
+  }
+  if (!ignoreSequence && !(xDetails.sequence() && yDetails.sequence()) &&
       !(xSymbol.attrs().test(semantics::Attr::BIND_C) &&
           ySymbol.attrs().test(semantics::Attr::BIND_C))) {
     // PGI does not enforce this requirement; all other Fortran
@@ -493,7 +502,8 @@ static bool AreSameDerivedType(const semantics::DerivedTypeSpec &x,
     const auto yLookup{ySymbol.scope()->find(*yComponentName)};
     if (xLookup == xSymbol.scope()->end() ||
         yLookup == ySymbol.scope()->end() ||
-        !AreSameComponent(*xLookup->second, *yLookup->second, inProgress)) {
+        !AreSameComponent(
+            *xLookup->second, *yLookup->second, ignoreSequence, inProgress)) {
       return false;
     }
   }
@@ -503,13 +513,19 @@ static bool AreSameDerivedType(const semantics::DerivedTypeSpec &x,
 bool AreSameDerivedType(
     const semantics::DerivedTypeSpec &x, const semantics::DerivedTypeSpec &y) {
   SetOfDerivedTypePairs inProgress;
-  return AreSameDerivedType(x, y, false, false, inProgress);
+  return AreSameDerivedType(x, y, false, false, false, inProgress);
 }
 
 bool AreSameDerivedTypeIgnoringTypeParameters(
     const semantics::DerivedTypeSpec &x, const semantics::DerivedTypeSpec &y) {
   SetOfDerivedTypePairs inProgress;
-  return AreSameDerivedType(x, y, true, true, inProgress);
+  return AreSameDerivedType(x, y, true, true, false, inProgress);
+}
+
+bool AreSameDerivedTypeIgnoringSequence(
+    const semantics::DerivedTypeSpec &x, const semantics::DerivedTypeSpec &y) {
+  SetOfDerivedTypePairs inProgress;
+  return AreSameDerivedType(x, y, false, false, true, inProgress);
 }
 
 static bool AreSameDerivedType(
@@ -536,7 +552,7 @@ static bool AreCompatibleDerivedTypes(const semantics::DerivedTypeSpec *x,
   } else {
     SetOfDerivedTypePairs inProgress;
     if (AreSameDerivedType(*x, *y, ignoreTypeParameterValues,
-            ignoreLenTypeParameters, inProgress)) {
+            ignoreLenTypeParameters, false, inProgress)) {
       return true;
     } else {
       return isPolymorphic &&
diff --git a/flang/lib/Frontend/CMakeLists.txt b/flang/lib/Frontend/CMakeLists.txt
index d063ed36d00b4..81eef2d468d8c 100644
--- a/flang/lib/Frontend/CMakeLists.txt
+++ b/flang/lib/Frontend/CMakeLists.txt
@@ -27,7 +27,7 @@ add_flang_library(flangFrontend
   FortranParser
   FortranSemantics
   FortranEvaluate
-  FortranCommon
+  FortranSupport
   FortranLower
   FortranSupport
   FIRDialect
diff --git a/flang/lib/Frontend/CompilerInstance.cpp b/flang/lib/Frontend/CompilerInstance.cpp
index 298790bae6655..b1fa32ecb4cfc 100644
--- a/flang/lib/Frontend/CompilerInstance.cpp
+++ b/flang/lib/Frontend/CompilerInstance.cpp
@@ -11,12 +11,12 @@
 //===----------------------------------------------------------------------===//
 
 #include "flang/Frontend/CompilerInstance.h"
-#include "flang/Common/Fortran-features.h"
 #include "flang/Frontend/CompilerInvocation.h"
 #include "flang/Frontend/TextDiagnosticPrinter.h"
 #include "flang/Parser/parsing.h"
 #include "flang/Parser/provenance.h"
 #include "flang/Semantics/semantics.h"
+#include "flang/Support/Fortran-features.h"
 #include "flang/Support/Timing.h"
 #include "mlir/Support/RawOstreamExtras.h"
 #include "clang/Basic/DiagnosticFrontend.h"
@@ -153,9 +153,6 @@ bool CompilerInstance::executeAction(FrontendAction &act) {
   CompilerInvocation &invoc = this->getInvocation();
 
   llvm::Triple targetTriple{llvm::Triple(invoc.getTargetOpts().triple)};
-  if (targetTriple.getArch() == llvm::Triple::ArchType::x86_64) {
-    invoc.getDefaultKinds().set_quadPrecisionKind(10);
-  }
 
   // Set some sane defaults for the frontend.
   invoc.setDefaultFortranOpts();
diff --git a/flang/lib/Frontend/CompilerInvocation.cpp b/flang/lib/Frontend/CompilerInvocation.cpp
index 3c6da4687f65d..3eb413e8a62a9 100644
--- a/flang/lib/Frontend/CompilerInvocation.cpp
+++ b/flang/lib/Frontend/CompilerInvocation.cpp
@@ -11,13 +11,13 @@
 //===----------------------------------------------------------------------===//
 
 #include "flang/Frontend/CompilerInvocation.h"
-#include "flang/Common/Fortran-features.h"
-#include "flang/Common/OpenMP-features.h"
-#include "flang/Common/Version.h"
 #include "flang/Frontend/CodeGenOptions.h"
 #include "flang/Frontend/PreprocessorOptions.h"
 #include "flang/Frontend/TargetOptions.h"
 #include "flang/Semantics/semantics.h"
+#include "flang/Support/Fortran-features.h"
+#include "flang/Support/OpenMP-features.h"
+#include "flang/Support/Version.h"
 #include "flang/Tools/TargetSetup.h"
 #include "flang/Version.inc"
 #include "clang/Basic/AllDiagnostics.h"
@@ -740,6 +740,12 @@ static bool parseFrontendArgs(FrontendOptions &opts, llvm::opt::ArgList &args,
       args.hasFlag(clang::driver::options::OPT_fimplicit_none,
                    clang::driver::options::OPT_fno_implicit_none, false));
 
+  // -f{no-}implicit-none-ext
+  opts.features.Enable(
+      Fortran::common::LanguageFeature::ImplicitNoneExternal,
+      args.hasFlag(clang::driver::options::OPT_fimplicit_none_ext,
+                   clang::driver::options::OPT_fno_implicit_none_ext, false));
+
   // -f{no-}backslash
   opts.features.Enable(Fortran::common::LanguageFeature::BackslashEscapes,
                        args.hasFlag(clang::driver::options::OPT_fbackslash,
@@ -770,10 +776,11 @@ static bool parseFrontendArgs(FrontendOptions &opts, llvm::opt::ArgList &args,
     opts.features.Enable(Fortran::common::LanguageFeature::DefaultSave);
   }
 
-  // -fsave-main-program
-  if (args.hasArg(clang::driver::options::OPT_fsave_main_program)) {
-    opts.features.Enable(Fortran::common::LanguageFeature::SaveMainProgram);
-  }
+  // -f{no}-save-main-program
+  opts.features.Enable(
+      Fortran::common::LanguageFeature::SaveMainProgram,
+      args.hasFlag(clang::driver::options::OPT_fsave_main_program,
+                   clang::driver::options::OPT_fno_save_main_program, false));
 
   if (args.hasArg(
           clang::driver::options::OPT_falternative_parameter_statement)) {
diff --git a/flang/lib/Frontend/FrontendActions.cpp b/flang/lib/Frontend/FrontendActions.cpp
index b0545a7ac2f99..b7674bd093f68 100644
--- a/flang/lib/Frontend/FrontendActions.cpp
+++ b/flang/lib/Frontend/FrontendActions.cpp
@@ -11,7 +11,6 @@
 //===----------------------------------------------------------------------===//
 
 #include "flang/Frontend/FrontendActions.h"
-#include "flang/Common/default-kinds.h"
 #include "flang/Frontend/CompilerInstance.h"
 #include "flang/Frontend/CompilerInvocation.h"
 #include "flang/Frontend/FrontendOptions.h"
@@ -34,6 +33,7 @@
 #include "flang/Semantics/runtime-type-info.h"
 #include "flang/Semantics/semantics.h"
 #include "flang/Semantics/unparse-with-symbols.h"
+#include "flang/Support/default-kinds.h"
 #include "flang/Tools/CrossToolHelpers.h"
 
 #include "mlir/IR/Dialect.h"
diff --git a/flang/lib/Frontend/TextDiagnosticPrinter.cpp b/flang/lib/Frontend/TextDiagnosticPrinter.cpp
index 2ab02c0b70ab5..65626827af3b3 100644
--- a/flang/lib/Frontend/TextDiagnosticPrinter.cpp
+++ b/flang/lib/Frontend/TextDiagnosticPrinter.cpp
@@ -39,7 +39,8 @@ static void printRemarkOption(llvm::raw_ostream &os,
                               clang::DiagnosticsEngine::Level level,
                               const clang::Diagnostic &info) {
   llvm::StringRef opt =
-      clang::DiagnosticIDs::getWarningOptionForDiag(info.getID());
+      info.getDiags()->getDiagnosticIDs()->getWarningOptionForDiag(
+          info.getID());
   if (!opt.empty()) {
     // We still need to check if the level is a Remark since, an unknown option
     // warning could be printed i.e. [-Wunknown-warning-option]
diff --git a/flang/lib/Lower/Bridge.cpp b/flang/lib/Lower/Bridge.cpp
index d92dc0cf9abd6..36e58e456dea3 100644
--- a/flang/lib/Lower/Bridge.cpp
+++ b/flang/lib/Lower/Bridge.cpp
@@ -12,7 +12,6 @@
 
 #include "flang/Lower/Bridge.h"
 
-#include "flang/Common/Version.h"
 #include "flang/Lower/Allocatable.h"
 #include "flang/Lower/CallInterface.h"
 #include "flang/Lower/Coarray.h"
@@ -62,7 +61,9 @@
 #include "flang/Semantics/runtime-type-info.h"
 #include "flang/Semantics/symbol.h"
 #include "flang/Semantics/tools.h"
+#include "flang/Support/Version.h"
 #include "mlir/Dialect/ControlFlow/IR/ControlFlowOps.h"
+#include "mlir/IR/BuiltinAttributes.h"
 #include "mlir/IR/Matchers.h"
 #include "mlir/IR/PatternMatch.h"
 #include "mlir/Parser/Parser.h"
@@ -459,7 +460,9 @@ class FirConverter : public Fortran::lower::AbstractConverter {
     if (hasMainProgram)
       createGlobalOutsideOfFunctionLowering([&]() {
         fir::runtime::genMain(*builder, toLocation(),
-                              bridge.getEnvironmentDefaults());
+                              bridge.getEnvironmentDefaults(),
+                              getFoldingContext().languageFeatures().IsEnabled(
+                                  Fortran::common::LanguageFeature::CUDA));
       });
 
     finalizeOpenACCLowering();
@@ -2168,14 +2171,64 @@ class FirConverter : public Fortran::lower::AbstractConverter {
     return builder->createIntegerConstant(loc, controlType, 1); // step
   }
 
-  void addLoopAnnotationAttr(IncrementLoopInfo &info) {
-    mlir::BoolAttr f = mlir::BoolAttr::get(builder->getContext(), false);
-    mlir::LLVM::LoopVectorizeAttr va = mlir::LLVM::LoopVectorizeAttr::get(
-        builder->getContext(), /*disable=*/f, {}, {}, {}, {}, {}, {});
+  // For unroll directives without a value, force full unrolling.
+  // For unroll directives with a value, if the value is greater than 1,
+  // force unrolling with the given factor. Otherwise, disable unrolling.
+  mlir::LLVM::LoopUnrollAttr
+  genLoopUnrollAttr(std::optional<std::uint64_t> directiveArg) {
+    mlir::BoolAttr falseAttr =
+        mlir::BoolAttr::get(builder->getContext(), false);
+    mlir::BoolAttr trueAttr = mlir::BoolAttr::get(builder->getContext(), true);
+    mlir::IntegerAttr countAttr;
+    mlir::BoolAttr fullUnrollAttr;
+    bool shouldUnroll = true;
+    if (directiveArg.has_value()) {
+      auto unrollingFactor = directiveArg.value();
+      if (unrollingFactor == 0 || unrollingFactor == 1) {
+        shouldUnroll = false;
+      } else {
+        countAttr =
+            builder->getIntegerAttr(builder->getI64Type(), unrollingFactor);
+      }
+    } else {
+      fullUnrollAttr = trueAttr;
+    }
+
+    mlir::BoolAttr disableAttr = shouldUnroll ? falseAttr : trueAttr;
+    return mlir::LLVM::LoopUnrollAttr::get(
+        builder->getContext(), /*disable=*/disableAttr, /*count=*/countAttr, {},
+        /*full=*/fullUnrollAttr, {}, {}, {});
+  }
+
+  void addLoopAnnotationAttr(
+      IncrementLoopInfo &info,
+      llvm::SmallVectorImpl<const Fortran::parser::CompilerDirective *> &dirs) {
+    mlir::LLVM::LoopVectorizeAttr va;
+    mlir::LLVM::LoopUnrollAttr ua;
+    bool has_attrs = false;
+    for (const auto *dir : dirs) {
+      Fortran::common::visit(
+          Fortran::common::visitors{
+              [&](const Fortran::parser::CompilerDirective::VectorAlways &) {
+                mlir::BoolAttr falseAttr =
+                    mlir::BoolAttr::get(builder->getContext(), false);
+                va = mlir::LLVM::LoopVectorizeAttr::get(builder->getContext(),
+                                                        /*disable=*/falseAttr,
+                                                        {}, {}, {}, {}, {}, {});
+                has_attrs = true;
+              },
+              [&](const Fortran::parser::CompilerDirective::Unroll &u) {
+                ua = genLoopUnrollAttr(u.v);
+                has_attrs = true;
+              },
+              [&](const auto &) {}},
+          dir->u);
+    }
     mlir::LLVM::LoopAnnotationAttr la = mlir::LLVM::LoopAnnotationAttr::get(
-        builder->getContext(), {}, /*vectorize=*/va, {}, {}, {}, {}, {}, {}, {},
-        {}, {}, {}, {}, {}, {});
-    info.doLoop.setLoopAnnotationAttr(la);
+        builder->getContext(), {}, /*vectorize=*/va, {}, /*unroll*/ ua, {}, {},
+        {}, {}, {}, {}, {}, {}, {}, {}, {});
+    if (has_attrs)
+      info.doLoop.setLoopAnnotationAttr(la);
   }
 
   /// Generate FIR to begin a structured or unstructured increment loop nest.
@@ -2274,14 +2327,7 @@ class FirConverter : public Fortran::lower::AbstractConverter {
         if (info.hasLocalitySpecs())
           handleLocalitySpecs(info);
 
-        for (const auto *dir : dirs) {
-          Fortran::common::visit(
-              Fortran::common::visitors{
-                  [&](const Fortran::parser::CompilerDirective::VectorAlways
-                          &d) { addLoopAnnotationAttr(info); },
-                  [&](const auto &) {}},
-              dir->u);
-        }
+        addLoopAnnotationAttr(info, dirs);
         continue;
       }
 
@@ -2833,6 +2879,9 @@ class FirConverter : public Fortran::lower::AbstractConverter {
             [&](const Fortran::parser::CompilerDirective::VectorAlways &) {
               attachDirectiveToLoop(dir, &eval);
             },
+            [&](const Fortran::parser::CompilerDirective::Unroll &) {
+              attachDirectiveToLoop(dir, &eval);
+            },
             [&](const auto &) {}},
         dir.u);
   }
diff --git a/flang/lib/Lower/CMakeLists.txt b/flang/lib/Lower/CMakeLists.txt
index f611010765cb5..c9b249781552e 100644
--- a/flang/lib/Lower/CMakeLists.txt
+++ b/flang/lib/Lower/CMakeLists.txt
@@ -34,6 +34,7 @@ add_flang_library(FortranLower
   OpenMP/Utils.cpp
   PFTBuilder.cpp
   Runtime.cpp
+  Support/Utils.cpp
   SymbolMap.cpp
   VectorSubscripts.cpp
   
@@ -55,7 +56,7 @@ add_flang_library(FortranLower
   FIRSupport
   FIRTransforms
   HLFIRDialect
-  FortranCommon
+  FortranSupport
   FortranParser
   FortranEvaluate
   FortranSemantics
diff --git a/flang/lib/Lower/CallInterface.cpp b/flang/lib/Lower/CallInterface.cpp
index e20b90b2ff1bc..226ba1e52c968 100644
--- a/flang/lib/Lower/CallInterface.cpp
+++ b/flang/lib/Lower/CallInterface.cpp
@@ -7,7 +7,6 @@
 //===----------------------------------------------------------------------===//
 
 #include "flang/Lower/CallInterface.h"
-#include "flang/Common/Fortran.h"
 #include "flang/Evaluate/fold.h"
 #include "flang/Lower/Bridge.h"
 #include "flang/Lower/Mangler.h"
@@ -23,6 +22,7 @@
 #include "flang/Optimizer/Support/Utils.h"
 #include "flang/Semantics/symbol.h"
 #include "flang/Semantics/tools.h"
+#include "flang/Support/Fortran.h"
 #include <optional>
 
 static mlir::FunctionType
@@ -1132,10 +1132,7 @@ class Fortran::lower::CallInterfaceImpl {
 
     // TODO: intents that require special care (e.g finalization)
 
-    using ShapeAttr = Fortran::evaluate::characteristics::TypeAndShape::Attr;
-    const Fortran::evaluate::characteristics::TypeAndShape::Attrs &shapeAttrs =
-        obj.type.attrs();
-    if (shapeAttrs.test(ShapeAttr::Coarray))
+    if (obj.type.corank() > 0)
       TODO(loc, "coarray: dummy argument coarray in procedure interface");
 
     // So far assume that if the argument cannot be passed by implicit interface
diff --git a/flang/lib/Lower/ConvertCall.cpp b/flang/lib/Lower/ConvertCall.cpp
index 40cd106e63018..6a0f4d1090adc 100644
--- a/flang/lib/Lower/ConvertCall.cpp
+++ b/flang/lib/Lower/ConvertCall.cpp
@@ -32,6 +32,7 @@
 #include "flang/Optimizer/Dialect/FIROpsSupport.h"
 #include "flang/Optimizer/HLFIR/HLFIROps.h"
 #include "mlir/IR/IRMapping.h"
+#include "llvm/ADT/TypeSwitch.h"
 #include "llvm/Support/CommandLine.h"
 #include "llvm/Support/Debug.h"
 #include <optional>
@@ -594,7 +595,8 @@ Fortran::lower::genCallOpAndResult(
 
     builder.create<cuf::KernelLaunchOp>(
         loc, funcType.getResults(), funcSymbolAttr, grid_x, grid_y, grid_z,
-        block_x, block_y, block_z, bytes, stream, operands);
+        block_x, block_y, block_z, bytes, stream, operands,
+        /*arg_attrs=*/nullptr, /*res_attrs=*/nullptr);
     callNumResults = 0;
   } else if (caller.requireDispatchCall()) {
     // Procedure call requiring a dynamic dispatch. Call is created with
@@ -621,7 +623,8 @@ Fortran::lower::genCallOpAndResult(
       dispatch = builder.create<fir::DispatchOp>(
           loc, funcType.getResults(), builder.getStringAttr(procName),
           caller.getInputs()[*passArg], operands,
-          builder.getI32IntegerAttr(*passArg), procAttrs);
+          builder.getI32IntegerAttr(*passArg), /*arg_attrs=*/nullptr,
+          /*res_attrs=*/nullptr, procAttrs);
     } else {
       // NOPASS
       const Fortran::evaluate::Component *component =
@@ -636,7 +639,8 @@ Fortran::lower::genCallOpAndResult(
         passObject = builder.create<fir::LoadOp>(loc, passObject);
       dispatch = builder.create<fir::DispatchOp>(
           loc, funcType.getResults(), builder.getStringAttr(procName),
-          passObject, operands, nullptr, procAttrs);
+          passObject, operands, nullptr, /*arg_attrs=*/nullptr,
+          /*res_attrs=*/nullptr, procAttrs);
     }
     callNumResults = dispatch.getNumResults();
     if (callNumResults != 0)
@@ -644,7 +648,8 @@ Fortran::lower::genCallOpAndResult(
   } else {
     // Standard procedure call with fir.call.
     auto call = builder.create<fir::CallOp>(
-        loc, funcType.getResults(), funcSymbolAttr, operands, procAttrs);
+        loc, funcType.getResults(), funcSymbolAttr, operands,
+        /*arg_attrs=*/nullptr, /*res_attrs=*/nullptr, procAttrs);
 
     callNumResults = call.getNumResults();
     if (callNumResults != 0)
@@ -1131,6 +1136,27 @@ isSimplyContiguous(const Fortran::evaluate::ActualArgument &arg,
          Fortran::evaluate::IsSimplyContiguous(*sym, foldingContext);
 }
 
+static bool isParameterObjectOrSubObject(hlfir::Entity entity) {
+  mlir::Value base = entity;
+  bool foundParameter = false;
+  while (mlir::Operation *op = base ? base.getDefiningOp() : nullptr) {
+    base =
+        llvm::TypeSwitch<mlir::Operation *, mlir::Value>(op)
+            .Case<hlfir::DeclareOp>([&](auto declare) -> mlir::Value {
+              foundParameter |= hlfir::Entity{declare}.isParameter();
+              return foundParameter ? mlir::Value{} : declare.getMemref();
+            })
+            .Case<hlfir::DesignateOp, hlfir::ParentComponentOp, fir::EmboxOp>(
+                [&](auto op) -> mlir::Value { return op.getMemref(); })
+            .Case<fir::ReboxOp>(
+                [&](auto rebox) -> mlir::Value { return rebox.getBox(); })
+            .Case<fir::ConvertOp>(
+                [&](auto convert) -> mlir::Value { return convert.getValue(); })
+            .Default([](mlir::Operation *) -> mlir::Value { return nullptr; });
+  }
+  return foundParameter;
+}
+
 /// When dummy is not ALLOCATABLE, POINTER and is not passed in register,
 /// prepare the actual argument according to the interface. Do as needed:
 /// - address element if this is an array argument in an elemental call.
@@ -1294,8 +1320,9 @@ static PreparedDummyArgument preparePresentUserCallActualArgument(
         // 'parameter' attribute. Even though the constant expressions
         // are not definable and explicit assignments to them are not
         // possible, we have to create a temporary copies when we pass
-        // them down the call stack.
-        entity.isParameter()) {
+        // them down the call stack because of potential compiler
+        // generated writes in copy-out.
+        isParameterObjectOrSubObject(entity)) {
       // Make a copy in a temporary.
       auto copy = builder.create<hlfir::AsExprOp>(loc, entity);
       mlir::Type storageType = entity.getType();
diff --git a/flang/lib/Lower/ConvertExpr.cpp b/flang/lib/Lower/ConvertExpr.cpp
index d9ae502edbafa..b33baf90582b8 100644
--- a/flang/lib/Lower/ConvertExpr.cpp
+++ b/flang/lib/Lower/ConvertExpr.cpp
@@ -11,7 +11,6 @@
 //===----------------------------------------------------------------------===//
 
 #include "flang/Lower/ConvertExpr.h"
-#include "flang/Common/default-kinds.h"
 #include "flang/Common/unwrap.h"
 #include "flang/Evaluate/fold.h"
 #include "flang/Evaluate/real.h"
@@ -52,6 +51,7 @@
 #include "flang/Semantics/symbol.h"
 #include "flang/Semantics/tools.h"
 #include "flang/Semantics/type.h"
+#include "flang/Support/default-kinds.h"
 #include "mlir/Dialect/Func/IR/FuncOps.h"
 #include "llvm/ADT/TypeSwitch.h"
 #include "llvm/Support/CommandLine.h"
diff --git a/flang/lib/Lower/ConvertExprToHLFIR.cpp b/flang/lib/Lower/ConvertExprToHLFIR.cpp
index 3e54cefb0974c..91daa6f0ad6ea 100644
--- a/flang/lib/Lower/ConvertExprToHLFIR.cpp
+++ b/flang/lib/Lower/ConvertExprToHLFIR.cpp
@@ -21,6 +21,7 @@
 #include "flang/Lower/ConvertProcedureDesignator.h"
 #include "flang/Lower/ConvertType.h"
 #include "flang/Lower/ConvertVariable.h"
+#include "flang/Lower/DumpEvaluateExpr.h"
 #include "flang/Lower/StatementContext.h"
 #include "flang/Lower/SymbolMap.h"
 #include "flang/Optimizer/Builder/Complex.h"
@@ -220,7 +221,8 @@ class HlfirDesignatorBuilder {
     // Non simply contiguous ref require a fir.box to carry the byte stride.
     if (mlir::isa<fir::SequenceType>(resultValueType) &&
         !Fortran::evaluate::IsSimplyContiguous(
-            designatorNode, getConverter().getFoldingContext()))
+            designatorNode, getConverter().getFoldingContext(),
+            /*namedConstantSectionsAreAlwaysContiguous=*/false))
       return fir::BoxType::get(resultValueType);
     // Other designators can be handled as raw addresses.
     return fir::ReferenceType::get(resultValueType);
diff --git a/flang/lib/Lower/ConvertVariable.cpp b/flang/lib/Lower/ConvertVariable.cpp
index 87236dc293ebb..81d14fbb1d777 100644
--- a/flang/lib/Lower/ConvertVariable.cpp
+++ b/flang/lib/Lower/ConvertVariable.cpp
@@ -956,7 +956,15 @@ static void instantiateLocal(Fortran::lower::AbstractConverter &converter,
                              Fortran::lower::SymMap &symMap) {
   assert(!var.isAlias());
   Fortran::lower::StatementContext stmtCtx;
+  // isUnusedEntryDummy must be computed before mapSymbolAttributes.
+  const bool isUnusedEntryDummy =
+      var.hasSymbol() && Fortran::semantics::IsDummy(var.getSymbol()) &&
+      !symMap.lookupSymbol(var.getSymbol()).getAddr();
   mapSymbolAttributes(converter, var, symMap, stmtCtx);
+  // Do not generate code to initialize/finalize/destroy dummy arguments that
+  // are nor part of the current ENTRY. They do not have backing storage.
+  if (isUnusedEntryDummy)
+    return;
   deallocateIntentOut(converter, var, symMap);
   if (needDummyIntentoutFinalization(var))
     finalizeAtRuntime(converter, var, symMap);
@@ -999,7 +1007,6 @@ static void instantiateLocal(Fortran::lower::AbstractConverter &converter,
                "trying to deallocate entity not lowered as allocatable");
         Fortran::lower::genDeallocateIfAllocated(*converterPtr, *mutableBox,
                                                  loc, sym);
-        
       });
     }
   }
diff --git a/flang/lib/Lower/HlfirIntrinsics.cpp b/flang/lib/Lower/HlfirIntrinsics.cpp
index 9d3cd3a5c8fa1..8b96b209ddb00 100644
--- a/flang/lib/Lower/HlfirIntrinsics.cpp
+++ b/flang/lib/Lower/HlfirIntrinsics.cpp
@@ -170,6 +170,17 @@ class HlfirCShiftLowering : public HlfirTransformationalIntrinsic {
             mlir::Type stmtResultType) override;
 };
 
+class HlfirReshapeLowering : public HlfirTransformationalIntrinsic {
+public:
+  using HlfirTransformationalIntrinsic::HlfirTransformationalIntrinsic;
+
+protected:
+  mlir::Value
+  lowerImpl(const Fortran::lower::PreparedActualArguments &loweredActuals,
+            const fir::IntrinsicArgumentLoweringRules *argLowering,
+            mlir::Type stmtResultType) override;
+};
+
 } // namespace
 
 mlir::Value HlfirTransformationalIntrinsic::loadBoxAddress(
@@ -419,6 +430,17 @@ mlir::Value HlfirCShiftLowering::lowerImpl(
   return createOp<hlfir::CShiftOp>(resultType, operands);
 }
 
+mlir::Value HlfirReshapeLowering::lowerImpl(
+    const Fortran::lower::PreparedActualArguments &loweredActuals,
+    const fir::IntrinsicArgumentLoweringRules *argLowering,
+    mlir::Type stmtResultType) {
+  auto operands = getOperandVector(loweredActuals, argLowering);
+  assert(operands.size() == 4);
+  mlir::Type resultType = computeResultType(operands[0], stmtResultType);
+  return createOp<hlfir::ReshapeOp>(resultType, operands[0], operands[1],
+                                    operands[2], operands[3]);
+}
+
 std::optional<hlfir::EntityWithAttributes> Fortran::lower::lowerHlfirIntrinsic(
     fir::FirOpBuilder &builder, mlir::Location loc, const std::string &name,
     const Fortran::lower::PreparedActualArguments &loweredActuals,
@@ -467,6 +489,9 @@ std::optional<hlfir::EntityWithAttributes> Fortran::lower::lowerHlfirIntrinsic(
   if (name == "cshift")
     return HlfirCShiftLowering{builder, loc}.lower(loweredActuals, argLowering,
                                                    stmtResultType);
+  if (name == "reshape")
+    return HlfirReshapeLowering{builder, loc}.lower(loweredActuals, argLowering,
+                                                    stmtResultType);
   if (mlir::isa<fir::CharacterType>(stmtResultType)) {
     if (name == "min")
       return HlfirCharExtremumLowering{builder, loc,
diff --git a/flang/lib/Lower/IterationSpace.cpp b/flang/lib/Lower/IterationSpace.cpp
index 63011483022b7..b011b3ab9a248 100644
--- a/flang/lib/Lower/IterationSpace.cpp
+++ b/flang/lib/Lower/IterationSpace.cpp
@@ -19,36 +19,6 @@
 
 #define DEBUG_TYPE "flang-lower-iteration-space"
 
-unsigned Fortran::lower::getHashValue(
-    const Fortran::lower::ExplicitIterSpace::ArrayBases &x) {
-  return Fortran::common::visit(
-      [&](const auto *p) { return HashEvaluateExpr::getHashValue(*p); }, x);
-}
-
-bool Fortran::lower::isEqual(
-    const Fortran::lower::ExplicitIterSpace::ArrayBases &x,
-    const Fortran::lower::ExplicitIterSpace::ArrayBases &y) {
-  return Fortran::common::visit(
-      Fortran::common::visitors{
-          // Fortran::semantics::Symbol * are the exception here. These pointers
-          // have identity; if two Symbol * values are the same (different) then
-          // they are the same (different) logical symbol.
-          [&](Fortran::lower::FrontEndSymbol p,
-              Fortran::lower::FrontEndSymbol q) { return p == q; },
-          [&](const auto *p, const auto *q) {
-            if constexpr (std::is_same_v<decltype(p), decltype(q)>) {
-              LLVM_DEBUG(llvm::dbgs()
-                         << "is equal: " << p << ' ' << q << ' '
-                         << IsEqualEvaluateExpr::isEqual(*p, *q) << '\n');
-              return IsEqualEvaluateExpr::isEqual(*p, *q);
-            } else {
-              // Different subtree types are never equal.
-              return false;
-            }
-          }},
-      x, y);
-}
-
 namespace {
 
 /// This class can recover the base array in an expression that contains
diff --git a/flang/lib/Lower/OpenACC.cpp b/flang/lib/Lower/OpenACC.cpp
index 456c30264d068..3dd35ed9ae481 100644
--- a/flang/lib/Lower/OpenACC.cpp
+++ b/flang/lib/Lower/OpenACC.cpp
@@ -26,17 +26,32 @@
 #include "flang/Optimizer/Builder/HLFIRTools.h"
 #include "flang/Optimizer/Builder/IntrinsicCall.h"
 #include "flang/Optimizer/Builder/Todo.h"
+#include "flang/Optimizer/Dialect/FIRType.h"
 #include "flang/Parser/parse-tree-visitor.h"
 #include "flang/Parser/parse-tree.h"
 #include "flang/Semantics/expression.h"
 #include "flang/Semantics/scope.h"
 #include "flang/Semantics/tools.h"
 #include "mlir/Dialect/ControlFlow/IR/ControlFlowOps.h"
+#include "mlir/Support/LLVM.h"
+#include "llvm/ADT/STLExtras.h"
 #include "llvm/Frontend/OpenACC/ACC.h.inc"
+#include "llvm/Support/CommandLine.h"
 #include "llvm/Support/Debug.h"
 
 #define DEBUG_TYPE "flang-lower-openacc"
 
+static llvm::cl::opt<bool> unwrapFirBox(
+    "openacc-unwrap-fir-box",
+    llvm::cl::desc(
+        "Whether to use the address from fix.box in data clause operations."),
+    llvm::cl::init(false));
+
+static llvm::cl::opt<bool> generateDefaultBounds(
+    "openacc-generate-default-bounds",
+    llvm::cl::desc("Whether to generate default bounds for arrays."),
+    llvm::cl::init(false));
+
 // Special value for * passed in device_type or gang clauses.
 static constexpr std::int64_t starCst = -1;
 
@@ -94,8 +109,9 @@ createDataEntryOp(fir::FirOpBuilder &builder, mlir::Location loc,
   // The data clause may apply to either the box reference itself or the
   // pointer to the data it holds. So use `unwrapBoxAddr` to decide.
   // When we have a box value - assume it refers to the data inside box.
-  if ((fir::isBoxAddress(baseAddr.getType()) && unwrapBoxAddr) ||
-      fir::isa_box_type(baseAddr.getType())) {
+  if (unwrapFirBox &&
+      ((fir::isBoxAddress(baseAddr.getType()) && unwrapBoxAddr) ||
+       fir::isa_box_type(baseAddr.getType()))) {
     if (isPresent) {
       mlir::Type ifRetTy =
           mlir::cast<fir::BaseBoxType>(fir::unwrapRefType(baseAddr.getType()))
@@ -140,8 +156,16 @@ createDataEntryOp(fir::FirOpBuilder &builder, mlir::Location loc,
   op.setStructured(structured);
   op.setImplicit(implicit);
   op.setDataClause(dataClause);
-  op.setVarType(mlir::cast<mlir::acc::PointerLikeType>(baseAddr.getType())
-                    .getElementType());
+  if (auto mappableTy =
+          mlir::dyn_cast<mlir::acc::MappableType>(baseAddr.getType())) {
+    op.setVarType(baseAddr.getType());
+  } else {
+    assert(mlir::isa<mlir::acc::PointerLikeType>(baseAddr.getType()) &&
+           "expected pointer-like");
+    op.setVarType(mlir::cast<mlir::acc::PointerLikeType>(baseAddr.getType())
+                      .getElementType());
+  }
+
   op->setAttr(Op::getOperandSegmentSizeAttr(),
               builder.getDenseI32ArrayAttr(operandSegments));
   if (!asyncDeviceTypes.empty())
@@ -208,7 +232,9 @@ static void createDeclareAllocFuncWithArg(mlir::OpBuilder &modBuilder,
 
   llvm::SmallVector<mlir::Value> bounds;
   std::stringstream asFortranDesc;
-  asFortranDesc << asFortran.str() << accFirDescriptorPostfix.str();
+  asFortranDesc << asFortran.str();
+  if (unwrapFirBox)
+    asFortranDesc << accFirDescriptorPostfix.str();
 
   // Updating descriptor must occur before the mapping of the data so that
   // attached data pointer is not overwritten.
@@ -222,17 +248,19 @@ static void createDeclareAllocFuncWithArg(mlir::OpBuilder &modBuilder,
   llvm::SmallVector<mlir::Value> operands{updateDeviceOp.getResult()};
   createSimpleOp<mlir::acc::UpdateOp>(builder, loc, operands, operandSegments);
 
-  mlir::Value desc =
-      builder.create<fir::LoadOp>(loc, registerFuncOp.getArgument(0));
-  fir::BoxAddrOp boxAddrOp = builder.create<fir::BoxAddrOp>(loc, desc);
-  addDeclareAttr(builder, boxAddrOp.getOperation(), clause);
-  EntryOp entryOp = createDataEntryOp<EntryOp>(
-      builder, loc, boxAddrOp.getResult(), asFortran, bounds,
-      /*structured=*/false, /*implicit=*/false, clause, boxAddrOp.getType(),
-      /*async=*/{}, /*asyncDeviceTypes=*/{}, /*asyncOnlyDeviceTypes=*/{});
-  builder.create<mlir::acc::DeclareEnterOp>(
-      loc, mlir::acc::DeclareTokenType::get(entryOp.getContext()),
-      mlir::ValueRange(entryOp.getAccPtr()));
+  if (unwrapFirBox) {
+    mlir::Value desc =
+        builder.create<fir::LoadOp>(loc, registerFuncOp.getArgument(0));
+    fir::BoxAddrOp boxAddrOp = builder.create<fir::BoxAddrOp>(loc, desc);
+    addDeclareAttr(builder, boxAddrOp.getOperation(), clause);
+    EntryOp entryOp = createDataEntryOp<EntryOp>(
+        builder, loc, boxAddrOp.getResult(), asFortran, bounds,
+        /*structured=*/false, /*implicit=*/false, clause, boxAddrOp.getType(),
+        /*async=*/{}, /*asyncDeviceTypes=*/{}, /*asyncOnlyDeviceTypes=*/{});
+    builder.create<mlir::acc::DeclareEnterOp>(
+        loc, mlir::acc::DeclareTokenType::get(entryOp.getContext()),
+        mlir::ValueRange(entryOp.getAccVar()));
+  }
 
   modBuilder.setInsertionPointAfter(registerFuncOp);
   builder.restoreInsertionPoint(crtInsPt);
@@ -252,31 +280,36 @@ static void createDeclareDeallocFuncWithArg(
     descTy = fir::ReferenceType::get(descTy);
   auto preDeallocOp = createDeclareFunc(
       modBuilder, builder, loc, preDeallocFuncName.str(), {descTy}, {loc});
-  mlir::Value loadOp =
-      builder.create<fir::LoadOp>(loc, preDeallocOp.getArgument(0));
-  fir::BoxAddrOp boxAddrOp = builder.create<fir::BoxAddrOp>(loc, loadOp);
-  addDeclareAttr(builder, boxAddrOp.getOperation(), clause);
+
+  mlir::Value var = preDeallocOp.getArgument(0);
+  if (unwrapFirBox) {
+    mlir::Value loadOp =
+        builder.create<fir::LoadOp>(loc, preDeallocOp.getArgument(0));
+    fir::BoxAddrOp boxAddrOp = builder.create<fir::BoxAddrOp>(loc, loadOp);
+    addDeclareAttr(builder, boxAddrOp.getOperation(), clause);
+    var = boxAddrOp.getResult();
+  }
 
   llvm::SmallVector<mlir::Value> bounds;
   mlir::acc::GetDevicePtrOp entryOp =
       createDataEntryOp<mlir::acc::GetDevicePtrOp>(
-          builder, loc, boxAddrOp.getResult(), asFortran, bounds,
-          /*structured=*/false, /*implicit=*/false, clause, boxAddrOp.getType(),
+          builder, loc, var, asFortran, bounds,
+          /*structured=*/false, /*implicit=*/false, clause, var.getType(),
           /*async=*/{}, /*asyncDeviceTypes=*/{}, /*asyncOnlyDeviceTypes=*/{});
   builder.create<mlir::acc::DeclareExitOp>(
-      loc, mlir::Value{}, mlir::ValueRange(entryOp.getAccPtr()));
+      loc, mlir::Value{}, mlir::ValueRange(entryOp.getAccVar()));
 
   if constexpr (std::is_same_v<ExitOp, mlir::acc::CopyoutOp> ||
                 std::is_same_v<ExitOp, mlir::acc::UpdateHostOp>)
-    builder.create<ExitOp>(entryOp.getLoc(), entryOp.getAccPtr(),
-                           entryOp.getVarPtr(), entryOp.getVarType(),
+    builder.create<ExitOp>(entryOp.getLoc(), entryOp.getAccVar(),
+                           entryOp.getVar(), entryOp.getVarType(),
                            entryOp.getBounds(), entryOp.getAsyncOperands(),
                            entryOp.getAsyncOperandsDeviceTypeAttr(),
                            entryOp.getAsyncOnlyAttr(), entryOp.getDataClause(),
                            /*structured=*/false, /*implicit=*/false,
                            builder.getStringAttr(*entryOp.getName()));
   else
-    builder.create<ExitOp>(entryOp.getLoc(), entryOp.getAccPtr(),
+    builder.create<ExitOp>(entryOp.getLoc(), entryOp.getAccVar(),
                            entryOp.getBounds(), entryOp.getAsyncOperands(),
                            entryOp.getAsyncOperandsDeviceTypeAttr(),
                            entryOp.getAsyncOnlyAttr(), entryOp.getDataClause(),
@@ -290,13 +323,18 @@ static void createDeclareDeallocFuncWithArg(
                       << Fortran::lower::declarePostDeallocSuffix.str();
   auto postDeallocOp = createDeclareFunc(
       modBuilder, builder, loc, postDeallocFuncName.str(), {descTy}, {loc});
-  loadOp = builder.create<fir::LoadOp>(loc, postDeallocOp.getArgument(0));
-  asFortran << accFirDescriptorPostfix.str();
+
+  var = postDeallocOp.getArgument(0);
+  if (unwrapFirBox) {
+    var = builder.create<fir::LoadOp>(loc, postDeallocOp.getArgument(0));
+    asFortran << accFirDescriptorPostfix.str();
+  }
+
   mlir::acc::UpdateDeviceOp updateDeviceOp =
       createDataEntryOp<mlir::acc::UpdateDeviceOp>(
-          builder, loc, loadOp, asFortran, bounds,
+          builder, loc, var, asFortran, bounds,
           /*structured=*/false, /*implicit=*/true,
-          mlir::acc::DataClause::acc_update_device, loadOp.getType(),
+          mlir::acc::DataClause::acc_update_device, var.getType(),
           /*async=*/{}, /*asyncDeviceTypes=*/{}, /*asyncOnlyDeviceTypes=*/{});
   llvm::SmallVector<int32_t> operandSegments{0, 0, 0, 1};
   llvm::SmallVector<mlir::Value> operands{updateDeviceOp.getResult()};
@@ -357,7 +395,8 @@ genDataOperandOperations(const Fortran::parser::AccObjectList &objectList,
             mlir::acc::DataBoundsOp, mlir::acc::DataBoundsType>(
             converter, builder, semanticsContext, stmtCtx, symbol, designator,
             operandLocation, asFortran, bounds,
-            /*treatIndexAsSection=*/true);
+            /*treatIndexAsSection=*/true, /*unwrapFirBox=*/unwrapFirBox,
+            /*genDefaultBounds=*/generateDefaultBounds);
     LLVM_DEBUG(llvm::dbgs() << __func__ << "\n"; info.dump(llvm::dbgs()));
 
     // If the input value is optional and is not a descriptor, we use the
@@ -371,7 +410,7 @@ genDataOperandOperations(const Fortran::parser::AccObjectList &objectList,
         builder, operandLocation, baseAddr, asFortran, bounds, structured,
         implicit, dataClause, baseAddr.getType(), async, asyncDeviceTypes,
         asyncOnlyDeviceTypes, /*unwrapBoxAddr=*/true, info.isPresent);
-    dataOperands.push_back(op.getAccPtr());
+    dataOperands.push_back(op.getAccVar());
   }
 }
 
@@ -396,14 +435,16 @@ static void genDeclareDataOperandOperations(
         Fortran::lower::gatherDataOperandAddrAndBounds<
             mlir::acc::DataBoundsOp, mlir::acc::DataBoundsType>(
             converter, builder, semanticsContext, stmtCtx, symbol, designator,
-            operandLocation, asFortran, bounds);
+            operandLocation, asFortran, bounds,
+            /*treatIndexAsSection=*/true, /*unwrapFirBox=*/unwrapFirBox,
+            /*genDefaultBounds=*/generateDefaultBounds);
     LLVM_DEBUG(llvm::dbgs() << __func__ << "\n"; info.dump(llvm::dbgs()));
     EntryOp op = createDataEntryOp<EntryOp>(
         builder, operandLocation, info.addr, asFortran, bounds, structured,
         implicit, dataClause, info.addr.getType(),
         /*async=*/{}, /*asyncDeviceTypes=*/{}, /*asyncOnlyDeviceTypes=*/{});
-    dataOperands.push_back(op.getAccPtr());
-    addDeclareAttr(builder, op.getVarPtr().getDefiningOp(), dataClause);
+    dataOperands.push_back(op.getAccVar());
+    addDeclareAttr(builder, op.getVar().getDefiningOp(), dataClause);
     if (mlir::isa<fir::BaseBoxType>(fir::unwrapRefType(info.addr.getType()))) {
       mlir::OpBuilder modBuilder(builder.getModule().getBodyRegion());
       modBuilder.setInsertionPointAfter(builder.getFunction());
@@ -452,14 +493,14 @@ static void genDataExitOperations(fir::FirOpBuilder &builder,
     if constexpr (std::is_same_v<ExitOp, mlir::acc::CopyoutOp> ||
                   std::is_same_v<ExitOp, mlir::acc::UpdateHostOp>)
       builder.create<ExitOp>(
-          entryOp.getLoc(), entryOp.getAccPtr(), entryOp.getVarPtr(),
+          entryOp.getLoc(), entryOp.getAccVar(), entryOp.getVar(),
           entryOp.getVarType(), entryOp.getBounds(), entryOp.getAsyncOperands(),
           entryOp.getAsyncOperandsDeviceTypeAttr(), entryOp.getAsyncOnlyAttr(),
           entryOp.getDataClause(), structured, entryOp.getImplicit(),
           builder.getStringAttr(*entryOp.getName()));
     else
       builder.create<ExitOp>(
-          entryOp.getLoc(), entryOp.getAccPtr(), entryOp.getBounds(),
+          entryOp.getLoc(), entryOp.getAccVar(), entryOp.getBounds(),
           entryOp.getAsyncOperands(), entryOp.getAsyncOperandsDeviceTypeAttr(),
           entryOp.getAsyncOnlyAttr(), entryOp.getDataClause(), structured,
           entryOp.getImplicit(), builder.getStringAttr(*entryOp.getName()));
@@ -480,41 +521,38 @@ template <typename RecipeOp>
 static void genPrivateLikeInitRegion(mlir::OpBuilder &builder, RecipeOp recipe,
                                      mlir::Type ty, mlir::Location loc) {
   mlir::Value retVal = recipe.getInitRegion().front().getArgument(0);
-  if (auto refTy = mlir::dyn_cast_or_null<fir::ReferenceType>(ty)) {
-    if (fir::isa_trivial(refTy.getEleTy())) {
-      auto alloca = builder.create<fir::AllocaOp>(loc, refTy.getEleTy());
+  ty = fir::unwrapRefType(ty);
+  if (fir::isa_trivial(ty)) {
+    auto alloca = builder.create<fir::AllocaOp>(loc, ty);
+    auto declareOp = builder.create<hlfir::DeclareOp>(
+        loc, alloca, accPrivateInitName, /*shape=*/nullptr,
+        llvm::ArrayRef<mlir::Value>{}, /*dummy_scope=*/nullptr,
+        fir::FortranVariableFlagsAttr{});
+    retVal = declareOp.getBase();
+  } else if (auto seqTy = mlir::dyn_cast_or_null<fir::SequenceType>(ty)) {
+    if (fir::isa_trivial(seqTy.getEleTy())) {
+      mlir::Value shape;
+      llvm::SmallVector<mlir::Value> extents;
+      if (seqTy.hasDynamicExtents()) {
+        // Extents are passed as block arguments. First argument is the
+        // original value.
+        for (unsigned i = 1; i < recipe.getInitRegion().getArguments().size();
+             ++i)
+          extents.push_back(recipe.getInitRegion().getArgument(i));
+        shape = builder.create<fir::ShapeOp>(loc, extents);
+      } else {
+        shape = genShapeOp(builder, seqTy, loc);
+      }
+      auto alloca = builder.create<fir::AllocaOp>(
+          loc, seqTy, /*typeparams=*/mlir::ValueRange{}, extents);
       auto declareOp = builder.create<hlfir::DeclareOp>(
-          loc, alloca, accPrivateInitName, /*shape=*/nullptr,
-          llvm::ArrayRef<mlir::Value>{}, /*dummy_scope=*/nullptr,
-          fir::FortranVariableFlagsAttr{});
+          loc, alloca, accPrivateInitName, shape, llvm::ArrayRef<mlir::Value>{},
+          /*dummy_scope=*/nullptr, fir::FortranVariableFlagsAttr{});
       retVal = declareOp.getBase();
-    } else if (auto seqTy = mlir::dyn_cast_or_null<fir::SequenceType>(
-                   refTy.getEleTy())) {
-      if (fir::isa_trivial(seqTy.getEleTy())) {
-        mlir::Value shape;
-        llvm::SmallVector<mlir::Value> extents;
-        if (seqTy.hasDynamicExtents()) {
-          // Extents are passed as block arguments. First argument is the
-          // original value.
-          for (unsigned i = 1; i < recipe.getInitRegion().getArguments().size();
-               ++i)
-            extents.push_back(recipe.getInitRegion().getArgument(i));
-          shape = builder.create<fir::ShapeOp>(loc, extents);
-        } else {
-          shape = genShapeOp(builder, seqTy, loc);
-        }
-        auto alloca = builder.create<fir::AllocaOp>(
-            loc, seqTy, /*typeparams=*/mlir::ValueRange{}, extents);
-        auto declareOp = builder.create<hlfir::DeclareOp>(
-            loc, alloca, accPrivateInitName, shape,
-            llvm::ArrayRef<mlir::Value>{}, /*dummy_scope=*/nullptr,
-            fir::FortranVariableFlagsAttr{});
-        retVal = declareOp.getBase();
-      }
     }
   } else if (auto boxTy = mlir::dyn_cast_or_null<fir::BaseBoxType>(ty)) {
-    mlir::Type innerTy = fir::extractSequenceType(boxTy);
-    if (!innerTy)
+    mlir::Type innerTy = fir::unwrapRefType(boxTy.getEleTy());
+    if (!fir::isa_trivial(innerTy) && !mlir::isa<fir::SequenceType>(innerTy))
       TODO(loc, "Unsupported boxed type in OpenACC privatization");
     fir::FirOpBuilder firBuilder{builder, recipe.getOperation()};
     hlfir::Entity source = hlfir::Entity{retVal};
@@ -608,11 +646,18 @@ genConstantBounds(fir::FirOpBuilder &builder, mlir::Location loc,
   return {lb, ub, step};
 }
 
-static fir::ShapeOp genShapeFromBoundsOrArgs(
+static mlir::Value genShapeFromBoundsOrArgs(
     mlir::Location loc, fir::FirOpBuilder &builder, fir::SequenceType seqTy,
     const llvm::SmallVector<mlir::Value> &bounds, mlir::ValueRange arguments) {
   llvm::SmallVector<mlir::Value> args;
-  if (areAllBoundConstant(bounds)) {
+  if (bounds.empty() && seqTy) {
+    if (seqTy.hasDynamicExtents()) {
+      assert(!arguments.empty() && "arguments must hold the entity");
+      auto entity = hlfir::Entity{arguments[0]};
+      return hlfir::genShape(loc, builder, entity);
+    }
+    return genShapeOp(builder, seqTy, loc).getResult();
+  } else if (areAllBoundConstant(bounds)) {
     for (auto bound : llvm::reverse(bounds)) {
       auto dataBound =
           mlir::cast<mlir::acc::DataBoundsOp>(bound.getDefiningOp());
@@ -859,7 +904,9 @@ genPrivatizations(const Fortran::parser::AccObjectList &objectList,
         Fortran::lower::gatherDataOperandAddrAndBounds<
             mlir::acc::DataBoundsOp, mlir::acc::DataBoundsType>(
             converter, builder, semanticsContext, stmtCtx, symbol, designator,
-            operandLocation, asFortran, bounds);
+            operandLocation, asFortran, bounds,
+            /*treatIndexAsSection=*/true, /*unwrapFirBox=*/unwrapFirBox,
+            /*genDefaultBounds=*/generateDefaultBounds);
     LLVM_DEBUG(llvm::dbgs() << __func__ << "\n"; info.dump(llvm::dbgs()));
 
     RecipeOp recipe;
@@ -874,7 +921,7 @@ genPrivatizations(const Fortran::parser::AccObjectList &objectList,
           builder, operandLocation, info.addr, asFortran, bounds, true,
           /*implicit=*/false, mlir::acc::DataClause::acc_private, retTy, async,
           asyncDeviceTypes, asyncOnlyDeviceTypes, /*unwrapBoxAddr=*/true);
-      dataOperands.push_back(op.getAccPtr());
+      dataOperands.push_back(op.getAccVar());
     } else {
       std::string suffix =
           areAllBoundConstant(bounds) ? getBoundsString(bounds) : "";
@@ -887,7 +934,7 @@ genPrivatizations(const Fortran::parser::AccObjectList &objectList,
           /*implicit=*/false, mlir::acc::DataClause::acc_firstprivate, retTy,
           async, asyncDeviceTypes, asyncOnlyDeviceTypes,
           /*unwrapBoxAddr=*/true);
-      dataOperands.push_back(op.getAccPtr());
+      dataOperands.push_back(op.getAccVar());
     }
     privatizations.push_back(mlir::SymbolRefAttr::get(
         builder.getContext(), recipe.getSymName().str()));
@@ -1074,7 +1121,7 @@ static mlir::Value genReductionInitRegion(fir::FirOpBuilder &builder,
         builder.create<hlfir::AssignOp>(loc, initValue, declareOp.getBase());
         return declareOp.getBase();
       }
-      for (auto ext : llvm::reverse(seqTy.getShape())) {
+      for (auto ext : seqTy.getShape()) {
         auto lb = builder.createIntegerConstant(loc, idxTy, 0);
         auto ub = builder.createIntegerConstant(loc, idxTy, ext - 1);
         auto step = builder.createIntegerConstant(loc, idxTy, 1);
@@ -1091,14 +1138,18 @@ static mlir::Value genReductionInitRegion(fir::FirOpBuilder &builder,
       return declareOp.getBase();
     }
   } else if (auto boxTy = mlir::dyn_cast_or_null<fir::BaseBoxType>(ty)) {
-    mlir::Type innerTy = fir::extractSequenceType(boxTy);
-    if (!mlir::isa<fir::SequenceType>(innerTy))
+    mlir::Type innerTy = fir::unwrapRefType(boxTy.getEleTy());
+    if (!fir::isa_trivial(innerTy) && !mlir::isa<fir::SequenceType>(innerTy))
       TODO(loc, "Unsupported boxed type for reduction");
     // Create the private copy from the initial fir.box.
     hlfir::Entity source = hlfir::Entity{builder.getBlock()->getArgument(0)};
     auto [temp, cleanup] = hlfir::createTempFromMold(loc, builder, source);
-    builder.create<hlfir::AssignOp>(loc, initValue, temp);
-    return temp;
+    mlir::Value newBox = temp;
+    if (!mlir::isa<fir::BaseBoxType>(temp.getType())) {
+      newBox = builder.create<fir::EmboxOp>(loc, boxTy, temp);
+    }
+    builder.create<hlfir::AssignOp>(loc, initValue, newBox);
+    return newBox;
   }
   llvm::report_fatal_error("Unsupported OpenACC reduction type");
 }
@@ -1261,9 +1312,27 @@ static void genCombiner(fir::FirOpBuilder &builder, mlir::Location loc,
       builder.create<hlfir::AssignOp>(loc, elemental, v1DeclareOp.getBase());
       return;
     }
-    if (allConstantBound) {
+    if (bounds.empty()) {
+      llvm::SmallVector<mlir::Value> extents;
+      mlir::Type idxTy = builder.getIndexType();
+      for (auto extent : seqTy.getShape()) {
+        mlir::Value lb = builder.create<mlir::arith::ConstantOp>(
+            loc, idxTy, builder.getIntegerAttr(idxTy, 0));
+        mlir::Value ub = builder.create<mlir::arith::ConstantOp>(
+            loc, idxTy, builder.getIntegerAttr(idxTy, extent - 1));
+        mlir::Value step = builder.create<mlir::arith::ConstantOp>(
+            loc, idxTy, builder.getIntegerAttr(idxTy, 1));
+        auto loop = builder.create<fir::DoLoopOp>(loc, lb, ub, step,
+                                                  /*unordered=*/false);
+        builder.setInsertionPointToStart(loop.getBody());
+        loops.push_back(loop);
+        ivs.push_back(loop.getInductionVar());
+      }
+    } else if (allConstantBound) {
       // Use the constant bound directly in the combiner region so they do not
       // need to be passed as block argument.
+      assert(!bounds.empty() &&
+             "seq type with constant bounds cannot have empty bounds");
       for (auto bound : llvm::reverse(bounds)) {
         auto dataBound =
             mlir::dyn_cast<mlir::acc::DataBoundsOp>(bound.getDefiningOp());
@@ -1303,38 +1372,64 @@ static void genCombiner(fir::FirOpBuilder &builder, mlir::Location loc,
     builder.create<fir::StoreOp>(loc, res, addr1);
     builder.setInsertionPointAfter(loops[0]);
   } else if (auto boxTy = mlir::dyn_cast<fir::BaseBoxType>(ty)) {
-    mlir::Type innerTy = fir::extractSequenceType(boxTy);
-    fir::SequenceType seqTy =
-        mlir::dyn_cast_or_null<fir::SequenceType>(innerTy);
-    if (!seqTy)
-      TODO(loc, "Unsupported boxed type in OpenACC reduction");
+    mlir::Type innerTy = fir::unwrapRefType(boxTy.getEleTy());
+    if (fir::isa_trivial(innerTy)) {
+      mlir::Value boxAddr1 = value1, boxAddr2 = value2;
+      if (fir::isBoxAddress(boxAddr1.getType()))
+        boxAddr1 = builder.create<fir::LoadOp>(loc, boxAddr1);
+      if (fir::isBoxAddress(boxAddr2.getType()))
+        boxAddr2 = builder.create<fir::LoadOp>(loc, boxAddr2);
+      boxAddr1 = builder.create<fir::BoxAddrOp>(loc, boxAddr1);
+      boxAddr2 = builder.create<fir::BoxAddrOp>(loc, boxAddr2);
+      auto leftEntity = hlfir::Entity{boxAddr1};
+      auto rightEntity = hlfir::Entity{boxAddr2};
+
+      auto leftVal = hlfir::loadTrivialScalar(loc, builder, leftEntity);
+      auto rightVal = hlfir::loadTrivialScalar(loc, builder, rightEntity);
+      mlir::Value res =
+          genScalarCombiner(builder, loc, op, innerTy, leftVal, rightVal);
+      builder.create<hlfir::AssignOp>(loc, res, boxAddr1);
+    } else {
+      mlir::Type innerTy = fir::extractSequenceType(boxTy);
+      fir::SequenceType seqTy =
+          mlir::dyn_cast_or_null<fir::SequenceType>(innerTy);
+      if (!seqTy)
+        TODO(loc, "Unsupported boxed type in OpenACC reduction combiner");
 
-    auto shape = genShapeFromBoundsOrArgs(
-        loc, builder, seqTy, bounds, recipe.getCombinerRegion().getArguments());
-    hlfir::DesignateOp::Subscripts triplets =
-        getSubscriptsFromArgs(recipe.getCombinerRegion().getArguments());
-    auto leftEntity = hlfir::Entity{value1};
-    auto left =
-        genDesignateWithTriplets(builder, loc, leftEntity, triplets, shape);
-    auto rightEntity = hlfir::Entity{value2};
-    auto right =
-        genDesignateWithTriplets(builder, loc, rightEntity, triplets, shape);
-
-    llvm::SmallVector<mlir::Value, 1> typeParams;
-    auto genKernel = [&builder, &loc, op, seqTy, &left, &right](
-                         mlir::Location l, fir::FirOpBuilder &b,
-                         mlir::ValueRange oneBasedIndices) -> hlfir::Entity {
-      auto leftElement = hlfir::getElementAt(l, b, left, oneBasedIndices);
-      auto rightElement = hlfir::getElementAt(l, b, right, oneBasedIndices);
-      auto leftVal = hlfir::loadTrivialScalar(l, b, leftElement);
-      auto rightVal = hlfir::loadTrivialScalar(l, b, rightElement);
-      return hlfir::Entity{genScalarCombiner(builder, loc, op, seqTy.getEleTy(),
-                                             leftVal, rightVal)};
-    };
-    mlir::Value elemental = hlfir::genElementalOp(
-        loc, builder, seqTy.getEleTy(), shape, typeParams, genKernel,
-        /*isUnordered=*/true);
-    builder.create<hlfir::AssignOp>(loc, elemental, value1);
+      auto shape =
+          genShapeFromBoundsOrArgs(loc, builder, seqTy, bounds,
+                                   recipe.getCombinerRegion().getArguments());
+      hlfir::DesignateOp::Subscripts triplets =
+          getSubscriptsFromArgs(recipe.getCombinerRegion().getArguments());
+      auto leftEntity = hlfir::Entity{value1};
+      if (fir::isBoxAddress(value1.getType()))
+        leftEntity =
+            hlfir::Entity{builder.create<fir::LoadOp>(loc, value1).getResult()};
+      auto left =
+          genDesignateWithTriplets(builder, loc, leftEntity, triplets, shape);
+      auto rightEntity = hlfir::Entity{value2};
+      if (fir::isBoxAddress(value2.getType()))
+        rightEntity =
+            hlfir::Entity{builder.create<fir::LoadOp>(loc, value2).getResult()};
+      auto right =
+          genDesignateWithTriplets(builder, loc, rightEntity, triplets, shape);
+
+      llvm::SmallVector<mlir::Value, 1> typeParams;
+      auto genKernel = [&builder, &loc, op, seqTy, &left, &right](
+                           mlir::Location l, fir::FirOpBuilder &b,
+                           mlir::ValueRange oneBasedIndices) -> hlfir::Entity {
+        auto leftElement = hlfir::getElementAt(l, b, left, oneBasedIndices);
+        auto rightElement = hlfir::getElementAt(l, b, right, oneBasedIndices);
+        auto leftVal = hlfir::loadTrivialScalar(l, b, leftElement);
+        auto rightVal = hlfir::loadTrivialScalar(l, b, rightElement);
+        return hlfir::Entity{genScalarCombiner(
+            builder, loc, op, seqTy.getEleTy(), leftVal, rightVal)};
+      };
+      mlir::Value elemental = hlfir::genElementalOp(
+          loc, builder, seqTy.getEleTy(), shape, typeParams, genKernel,
+          /*isUnordered=*/true);
+      builder.create<hlfir::AssignOp>(loc, elemental, value1);
+    }
   } else {
     mlir::Value res = genScalarCombiner(builder, loc, op, ty, value1, value2);
     builder.create<fir::StoreOp>(loc, res, value1);
@@ -1440,7 +1535,9 @@ genReductions(const Fortran::parser::AccObjectListWithReduction &objectList,
         Fortran::lower::gatherDataOperandAddrAndBounds<
             mlir::acc::DataBoundsOp, mlir::acc::DataBoundsType>(
             converter, builder, semanticsContext, stmtCtx, symbol, designator,
-            operandLocation, asFortran, bounds);
+            operandLocation, asFortran, bounds,
+            /*treatIndexAsSection=*/true, /*unwrapFirBox=*/unwrapFirBox,
+            /*genDefaultBounds=*/generateDefaultBounds);
     LLVM_DEBUG(llvm::dbgs() << __func__ << "\n"; info.dump(llvm::dbgs()));
 
     mlir::Type reductionTy = fir::unwrapRefType(info.addr.getType());
@@ -1455,7 +1552,7 @@ genReductions(const Fortran::parser::AccObjectListWithReduction &objectList,
         /*structured=*/true, /*implicit=*/false,
         mlir::acc::DataClause::acc_reduction, info.addr.getType(), async,
         asyncDeviceTypes, asyncOnlyDeviceTypes, /*unwrapBoxAddr=*/true);
-    mlir::Type ty = op.getAccPtr().getType();
+    mlir::Type ty = op.getAccVar().getType();
     if (!areAllBoundConstant(bounds) ||
         fir::isAssumedShape(info.addr.getType()) ||
         fir::isAllocatableOrPointerArray(info.addr.getType()))
@@ -1471,7 +1568,7 @@ genReductions(const Fortran::parser::AccObjectListWithReduction &objectList,
             builder, recipeName, operandLocation, ty, mlirOp, bounds);
     reductionRecipes.push_back(mlir::SymbolRefAttr::get(
         builder.getContext(), recipe.getSymName().str()));
-    reductionOperands.push_back(op.getAccPtr());
+    reductionOperands.push_back(op.getAccVar());
   }
 }
 
@@ -1706,19 +1803,20 @@ static void privatizeIv(Fortran::lower::AbstractConverter &converter,
       builder, recipeName, loc, ivValue.getType());
 
   std::stringstream asFortran;
+  asFortran << Fortran::lower::mangle::demangleName(toStringRef(sym.name()));
   auto op = createDataEntryOp<mlir::acc::PrivateOp>(
       builder, loc, ivValue, asFortran, {}, true, /*implicit=*/true,
       mlir::acc::DataClause::acc_private, ivValue.getType(),
       /*async=*/{}, /*asyncDeviceTypes=*/{}, /*asyncOnlyDeviceTypes=*/{});
 
-  privateOperands.push_back(op.getAccPtr());
+  privateOperands.push_back(op.getAccVar());
   privatizations.push_back(mlir::SymbolRefAttr::get(builder.getContext(),
                                                     recipe.getSymName().str()));
 
   // Map the new private iv to its symbol for the scope of the loop. bindSymbol
   // might create a hlfir.declare op, if so, we map its result in order to
   // use the sym value in the scope.
-  converter.bindSymbol(sym, op.getAccPtr());
+  converter.bindSymbol(sym, op.getAccVar());
   auto privateValue = converter.getSymbolAddress(sym);
   if (auto declareOp =
           mlir::dyn_cast<hlfir::DeclareOp>(privateValue.getDefiningOp()))
@@ -2184,8 +2282,8 @@ static Op createComputeOp(
   mlir::Value selfCond;
   llvm::SmallVector<mlir::Value> waitOperands, attachEntryOperands,
       copyEntryOperands, copyinEntryOperands, copyoutEntryOperands,
-      createEntryOperands, dataClauseOperands, numGangs, numWorkers,
-      vectorLength, async;
+      createEntryOperands, nocreateEntryOperands, presentEntryOperands,
+      dataClauseOperands, numGangs, numWorkers, vectorLength, async;
   llvm::SmallVector<mlir::Attribute> numGangsDeviceTypes, numWorkersDeviceTypes,
       vectorLengthDeviceTypes, asyncDeviceTypes, asyncOnlyDeviceTypes,
       waitOperandsDeviceTypes, waitOnlyDeviceTypes;
@@ -2360,19 +2458,25 @@ static Op createComputeOp(
     } else if (const auto *noCreateClause =
                    std::get_if<Fortran::parser::AccClause::NoCreate>(
                        &clause.u)) {
+      auto crtDataStart = dataClauseOperands.size();
       genDataOperandOperations<mlir::acc::NoCreateOp>(
           noCreateClause->v, converter, semanticsContext, stmtCtx,
           dataClauseOperands, mlir::acc::DataClause::acc_no_create,
           /*structured=*/true, /*implicit=*/false, async, asyncDeviceTypes,
           asyncOnlyDeviceTypes);
+      nocreateEntryOperands.append(dataClauseOperands.begin() + crtDataStart,
+                                   dataClauseOperands.end());
     } else if (const auto *presentClause =
                    std::get_if<Fortran::parser::AccClause::Present>(
                        &clause.u)) {
+      auto crtDataStart = dataClauseOperands.size();
       genDataOperandOperations<mlir::acc::PresentOp>(
           presentClause->v, converter, semanticsContext, stmtCtx,
           dataClauseOperands, mlir::acc::DataClause::acc_present,
           /*structured=*/true, /*implicit=*/false, async, asyncDeviceTypes,
           asyncOnlyDeviceTypes);
+      presentEntryOperands.append(dataClauseOperands.begin() + crtDataStart,
+                                  dataClauseOperands.end());
     } else if (const auto *devicePtrClause =
                    std::get_if<Fortran::parser::AccClause::Deviceptr>(
                        &clause.u)) {
@@ -2537,6 +2641,10 @@ static Op createComputeOp(
       builder, attachEntryOperands, /*structured=*/true);
   genDataExitOperations<mlir::acc::CreateOp, mlir::acc::DeleteOp>(
       builder, createEntryOperands, /*structured=*/true);
+  genDataExitOperations<mlir::acc::NoCreateOp, mlir::acc::DeleteOp>(
+      builder, nocreateEntryOperands, /*structured=*/true);
+  genDataExitOperations<mlir::acc::PresentOp, mlir::acc::DeleteOp>(
+      builder, presentEntryOperands, /*structured=*/true);
 
   builder.restoreInsertionPoint(insPt);
   return computeOp;
@@ -2551,7 +2659,8 @@ static void genACCDataOp(Fortran::lower::AbstractConverter &converter,
   mlir::Value ifCond;
   llvm::SmallVector<mlir::Value> attachEntryOperands, createEntryOperands,
       copyEntryOperands, copyinEntryOperands, copyoutEntryOperands,
-      dataClauseOperands, waitOperands, async;
+      nocreateEntryOperands, presentEntryOperands, dataClauseOperands,
+      waitOperands, async;
   llvm::SmallVector<mlir::Attribute> asyncDeviceTypes, asyncOnlyDeviceTypes,
       waitOperandsDeviceTypes, waitOnlyDeviceTypes;
   llvm::SmallVector<int32_t> waitOperandsSegments;
@@ -2648,19 +2757,25 @@ static void genACCDataOp(Fortran::lower::AbstractConverter &converter,
     } else if (const auto *noCreateClause =
                    std::get_if<Fortran::parser::AccClause::NoCreate>(
                        &clause.u)) {
+      auto crtDataStart = dataClauseOperands.size();
       genDataOperandOperations<mlir::acc::NoCreateOp>(
           noCreateClause->v, converter, semanticsContext, stmtCtx,
           dataClauseOperands, mlir::acc::DataClause::acc_no_create,
           /*structured=*/true, /*implicit=*/false, async, asyncDeviceTypes,
           asyncOnlyDeviceTypes);
+      nocreateEntryOperands.append(dataClauseOperands.begin() + crtDataStart,
+                                   dataClauseOperands.end());
     } else if (const auto *presentClause =
                    std::get_if<Fortran::parser::AccClause::Present>(
                        &clause.u)) {
+      auto crtDataStart = dataClauseOperands.size();
       genDataOperandOperations<mlir::acc::PresentOp>(
           presentClause->v, converter, semanticsContext, stmtCtx,
           dataClauseOperands, mlir::acc::DataClause::acc_present,
           /*structured=*/true, /*implicit=*/false, async, asyncDeviceTypes,
           asyncOnlyDeviceTypes);
+      presentEntryOperands.append(dataClauseOperands.begin() + crtDataStart,
+                                  dataClauseOperands.end());
     } else if (const auto *deviceptrClause =
                    std::get_if<Fortran::parser::AccClause::Deviceptr>(
                        &clause.u)) {
@@ -2740,6 +2855,10 @@ static void genACCDataOp(Fortran::lower::AbstractConverter &converter,
       builder, attachEntryOperands, /*structured=*/true);
   genDataExitOperations<mlir::acc::CreateOp, mlir::acc::DeleteOp>(
       builder, createEntryOperands, /*structured=*/true);
+  genDataExitOperations<mlir::acc::NoCreateOp, mlir::acc::DeleteOp>(
+      builder, nocreateEntryOperands, /*structured=*/true);
+  genDataExitOperations<mlir::acc::PresentOp, mlir::acc::DeleteOp>(
+      builder, presentEntryOperands, /*structured=*/true);
 
   builder.restoreInsertionPoint(insPt);
 }
@@ -3450,12 +3569,12 @@ static void createDeclareGlobalOp(mlir::OpBuilder &modBuilder,
   if constexpr (std::is_same_v<DeclareOp, mlir::acc::DeclareEnterOp>)
     builder.create<DeclareOp>(
         loc, mlir::acc::DeclareTokenType::get(entryOp.getContext()),
-        mlir::ValueRange(entryOp.getAccPtr()));
+        mlir::ValueRange(entryOp.getAccVar()));
   else
     builder.create<DeclareOp>(loc, mlir::Value{},
-                              mlir::ValueRange(entryOp.getAccPtr()));
+                              mlir::ValueRange(entryOp.getAccVar()));
   if constexpr (std::is_same_v<GlobalOp, mlir::acc::GlobalDestructorOp>) {
-    builder.create<ExitOp>(entryOp.getLoc(), entryOp.getAccPtr(),
+    builder.create<ExitOp>(entryOp.getLoc(), entryOp.getAccVar(),
                            entryOp.getBounds(), entryOp.getAsyncOperands(),
                            entryOp.getAsyncOperandsDeviceTypeAttr(),
                            entryOp.getAsyncOnlyAttr(), entryOp.getDataClause(),
@@ -3483,7 +3602,9 @@ static void createDeclareAllocFunc(mlir::OpBuilder &modBuilder,
   std::stringstream asFortran;
   asFortran << Fortran::lower::mangle::demangleName(globalOp.getSymName());
   std::stringstream asFortranDesc;
-  asFortranDesc << asFortran.str() << accFirDescriptorPostfix.str();
+  asFortranDesc << asFortran.str();
+  if (unwrapFirBox)
+    asFortranDesc << accFirDescriptorPostfix.str();
   llvm::SmallVector<mlir::Value> bounds;
 
   // Updating descriptor must occur before the mapping of the data so that
@@ -3498,16 +3619,18 @@ static void createDeclareAllocFunc(mlir::OpBuilder &modBuilder,
   llvm::SmallVector<mlir::Value> operands{updateDeviceOp.getResult()};
   createSimpleOp<mlir::acc::UpdateOp>(builder, loc, operands, operandSegments);
 
-  auto loadOp = builder.create<fir::LoadOp>(loc, addrOp.getResult());
-  fir::BoxAddrOp boxAddrOp = builder.create<fir::BoxAddrOp>(loc, loadOp);
-  addDeclareAttr(builder, boxAddrOp.getOperation(), clause);
-  EntryOp entryOp = createDataEntryOp<EntryOp>(
-      builder, loc, boxAddrOp.getResult(), asFortran, bounds,
-      /*structured=*/false, /*implicit=*/false, clause, boxAddrOp.getType(),
-      /*async=*/{}, /*asyncDeviceTypes=*/{}, /*asyncOnlyDeviceTypes=*/{});
-  builder.create<mlir::acc::DeclareEnterOp>(
-      loc, mlir::acc::DeclareTokenType::get(entryOp.getContext()),
-      mlir::ValueRange(entryOp.getAccPtr()));
+  if (unwrapFirBox) {
+    auto loadOp = builder.create<fir::LoadOp>(loc, addrOp.getResult());
+    fir::BoxAddrOp boxAddrOp = builder.create<fir::BoxAddrOp>(loc, loadOp);
+    addDeclareAttr(builder, boxAddrOp.getOperation(), clause);
+    EntryOp entryOp = createDataEntryOp<EntryOp>(
+        builder, loc, boxAddrOp.getResult(), asFortran, bounds,
+        /*structured=*/false, /*implicit=*/false, clause, boxAddrOp.getType(),
+        /*async=*/{}, /*asyncDeviceTypes=*/{}, /*asyncOnlyDeviceTypes=*/{});
+    builder.create<mlir::acc::DeclareEnterOp>(
+        loc, mlir::acc::DeclareTokenType::get(entryOp.getContext()),
+        mlir::ValueRange(entryOp.getAccVar()));
+  }
 
   modBuilder.setInsertionPointAfter(registerFuncOp);
 }
@@ -3522,59 +3645,69 @@ static void createDeclareDeallocFunc(mlir::OpBuilder &modBuilder,
                                      mlir::Location loc,
                                      fir::GlobalOp &globalOp,
                                      mlir::acc::DataClause clause) {
-
-  // Generate the pre dealloc function.
-  std::stringstream preDeallocFuncName;
-  preDeallocFuncName << globalOp.getSymName().str()
-                     << Fortran::lower::declarePreDeallocSuffix.str();
-  auto preDeallocOp =
-      createDeclareFunc(modBuilder, builder, loc, preDeallocFuncName.str());
-  fir::AddrOfOp addrOp = builder.create<fir::AddrOfOp>(
-      loc, fir::ReferenceType::get(globalOp.getType()), globalOp.getSymbol());
-  auto loadOp = builder.create<fir::LoadOp>(loc, addrOp.getResult());
-  fir::BoxAddrOp boxAddrOp = builder.create<fir::BoxAddrOp>(loc, loadOp);
-  addDeclareAttr(builder, boxAddrOp.getOperation(), clause);
-
   std::stringstream asFortran;
   asFortran << Fortran::lower::mangle::demangleName(globalOp.getSymName());
-  llvm::SmallVector<mlir::Value> bounds;
-  mlir::acc::GetDevicePtrOp entryOp =
-      createDataEntryOp<mlir::acc::GetDevicePtrOp>(
-          builder, loc, boxAddrOp.getResult(), asFortran, bounds,
-          /*structured=*/false, /*implicit=*/false, clause, boxAddrOp.getType(),
-          /*async=*/{}, /*asyncDeviceTypes=*/{}, /*asyncOnlyDeviceTypes=*/{});
 
-  builder.create<mlir::acc::DeclareExitOp>(
-      loc, mlir::Value{}, mlir::ValueRange(entryOp.getAccPtr()));
+  // If FIR box semantics are being unwrapped, then a pre-dealloc function
+  // needs generated to ensure to delete the device data pointed to by the
+  // descriptor before this information is lost.
+  if (unwrapFirBox) {
+    // Generate the pre dealloc function.
+    std::stringstream preDeallocFuncName;
+    preDeallocFuncName << globalOp.getSymName().str()
+                       << Fortran::lower::declarePreDeallocSuffix.str();
+    auto preDeallocOp =
+        createDeclareFunc(modBuilder, builder, loc, preDeallocFuncName.str());
+
+    fir::AddrOfOp addrOp = builder.create<fir::AddrOfOp>(
+        loc, fir::ReferenceType::get(globalOp.getType()), globalOp.getSymbol());
+    auto loadOp = builder.create<fir::LoadOp>(loc, addrOp.getResult());
+    fir::BoxAddrOp boxAddrOp = builder.create<fir::BoxAddrOp>(loc, loadOp);
+    mlir::Value var = boxAddrOp.getResult();
+    addDeclareAttr(builder, var.getDefiningOp(), clause);
 
-  if constexpr (std::is_same_v<ExitOp, mlir::acc::CopyoutOp> ||
-                std::is_same_v<ExitOp, mlir::acc::UpdateHostOp>)
-    builder.create<ExitOp>(entryOp.getLoc(), entryOp.getAccPtr(),
-                           entryOp.getVarPtr(), entryOp.getBounds(),
-                           entryOp.getAsyncOperands(),
-                           entryOp.getAsyncOperandsDeviceTypeAttr(),
-                           entryOp.getAsyncOnlyAttr(), entryOp.getDataClause(),
-                           /*structured=*/false, /*implicit=*/false,
-                           builder.getStringAttr(*entryOp.getName()));
-  else
-    builder.create<ExitOp>(entryOp.getLoc(), entryOp.getAccPtr(),
-                           entryOp.getBounds(), entryOp.getAsyncOperands(),
-                           entryOp.getAsyncOperandsDeviceTypeAttr(),
-                           entryOp.getAsyncOnlyAttr(), entryOp.getDataClause(),
-                           /*structured=*/false, /*implicit=*/false,
-                           builder.getStringAttr(*entryOp.getName()));
+    llvm::SmallVector<mlir::Value> bounds;
+    mlir::acc::GetDevicePtrOp entryOp =
+        createDataEntryOp<mlir::acc::GetDevicePtrOp>(
+            builder, loc, var, asFortran, bounds,
+            /*structured=*/false, /*implicit=*/false, clause, var.getType(),
+            /*async=*/{}, /*asyncDeviceTypes=*/{}, /*asyncOnlyDeviceTypes=*/{});
+
+    builder.create<mlir::acc::DeclareExitOp>(
+        loc, mlir::Value{}, mlir::ValueRange(entryOp.getAccVar()));
+
+    if constexpr (std::is_same_v<ExitOp, mlir::acc::CopyoutOp> ||
+                  std::is_same_v<ExitOp, mlir::acc::UpdateHostOp>)
+      builder.create<ExitOp>(
+          entryOp.getLoc(), entryOp.getAccVar(), entryOp.getVar(),
+          entryOp.getBounds(), entryOp.getAsyncOperands(),
+          entryOp.getAsyncOperandsDeviceTypeAttr(), entryOp.getAsyncOnlyAttr(),
+          entryOp.getDataClause(),
+          /*structured=*/false, /*implicit=*/false,
+          builder.getStringAttr(*entryOp.getName()));
+    else
+      builder.create<ExitOp>(
+          entryOp.getLoc(), entryOp.getAccVar(), entryOp.getBounds(),
+          entryOp.getAsyncOperands(), entryOp.getAsyncOperandsDeviceTypeAttr(),
+          entryOp.getAsyncOnlyAttr(), entryOp.getDataClause(),
+          /*structured=*/false, /*implicit=*/false,
+          builder.getStringAttr(*entryOp.getName()));
+
+    // Generate the post dealloc function.
+    modBuilder.setInsertionPointAfter(preDeallocOp);
+  }
 
-  // Generate the post dealloc function.
-  modBuilder.setInsertionPointAfter(preDeallocOp);
   std::stringstream postDeallocFuncName;
   postDeallocFuncName << globalOp.getSymName().str()
                       << Fortran::lower::declarePostDeallocSuffix.str();
   auto postDeallocOp =
       createDeclareFunc(modBuilder, builder, loc, postDeallocFuncName.str());
 
-  addrOp = builder.create<fir::AddrOfOp>(
+  fir::AddrOfOp addrOp = builder.create<fir::AddrOfOp>(
       loc, fir::ReferenceType::get(globalOp.getType()), globalOp.getSymbol());
-  asFortran << accFirDescriptorPostfix.str();
+  if (unwrapFirBox)
+    asFortran << accFirDescriptorPostfix.str();
+  llvm::SmallVector<mlir::Value> bounds;
   mlir::acc::UpdateDeviceOp updateDeviceOp =
       createDataEntryOp<mlir::acc::UpdateDeviceOp>(
           builder, loc, addrOp, asFortran, bounds,
@@ -3703,7 +3836,7 @@ genDeclareInFunction(Fortran::lower::AbstractConverter &converter,
                      const Fortran::parser::AccClauseList &accClauseList) {
   llvm::SmallVector<mlir::Value> dataClauseOperands, copyEntryOperands,
       copyinEntryOperands, createEntryOperands, copyoutEntryOperands,
-      deviceResidentEntryOperands;
+      presentEntryOperands, deviceResidentEntryOperands;
   Fortran::lower::StatementContext stmtCtx;
   fir::FirOpBuilder &builder = converter.getFirOpBuilder();
 
@@ -3734,11 +3867,14 @@ genDeclareInFunction(Fortran::lower::AbstractConverter &converter,
     } else if (const auto *presentClause =
                    std::get_if<Fortran::parser::AccClause::Present>(
                        &clause.u)) {
+      auto crtDataStart = dataClauseOperands.size();
       genDeclareDataOperandOperations<mlir::acc::PresentOp,
-                                      mlir::acc::PresentOp>(
+                                      mlir::acc::DeleteOp>(
           presentClause->v, converter, semanticsContext, stmtCtx,
           dataClauseOperands, mlir::acc::DataClause::acc_present,
           /*structured=*/true, /*implicit=*/false);
+      presentEntryOperands.append(dataClauseOperands.begin() + crtDataStart,
+                                  dataClauseOperands.end());
     } else if (const auto *copyinClause =
                    std::get_if<Fortran::parser::AccClause::Copyin>(&clause.u)) {
       auto crtDataStart = dataClauseOperands.size();
@@ -3817,14 +3953,15 @@ genDeclareInFunction(Fortran::lower::AbstractConverter &converter,
 
   openAccCtx.attachCleanup([&builder, loc, createEntryOperands,
                             copyEntryOperands, copyinEntryOperands,
-                            copyoutEntryOperands, deviceResidentEntryOperands,
-                            declareToken]() {
+                            copyoutEntryOperands, presentEntryOperands,
+                            deviceResidentEntryOperands, declareToken]() {
     llvm::SmallVector<mlir::Value> operands;
     operands.append(createEntryOperands);
     operands.append(deviceResidentEntryOperands);
     operands.append(copyEntryOperands);
     operands.append(copyinEntryOperands);
     operands.append(copyoutEntryOperands);
+    operands.append(presentEntryOperands);
 
     mlir::func::FuncOp funcOp = builder.getFunction();
     auto ops = funcOp.getOps<mlir::acc::DeclareExitOp>();
@@ -3846,6 +3983,8 @@ genDeclareInFunction(Fortran::lower::AbstractConverter &converter,
         builder, copyinEntryOperands, /*structured=*/true);
     genDataExitOperations<mlir::acc::CreateOp, mlir::acc::CopyoutOp>(
         builder, copyoutEntryOperands, /*structured=*/true);
+    genDataExitOperations<mlir::acc::PresentOp, mlir::acc::DeleteOp>(
+        builder, presentEntryOperands, /*structured=*/true);
   });
 }
 
diff --git a/flang/lib/Lower/OpenMP/ClauseProcessor.cpp b/flang/lib/Lower/OpenMP/ClauseProcessor.cpp
index 299d9d438f115..febc6adcf9d6f 100644
--- a/flang/lib/Lower/OpenMP/ClauseProcessor.cpp
+++ b/flang/lib/Lower/OpenMP/ClauseProcessor.cpp
@@ -344,6 +344,20 @@ bool ClauseProcessor::processDistSchedule(
   return false;
 }
 
+bool ClauseProcessor::processExclusive(
+    mlir::Location currentLocation,
+    mlir::omp::ExclusiveClauseOps &result) const {
+  if (auto *clause = findUniqueClause<omp::clause::Exclusive>()) {
+    for (const Object &object : clause->v) {
+      const semantics::Symbol *symbol = object.sym();
+      mlir::Value symVal = converter.getSymbolAddress(*symbol);
+      result.exclusiveVars.push_back(symVal);
+    }
+    return true;
+  }
+  return false;
+}
+
 bool ClauseProcessor::processFilter(lower::StatementContext &stmtCtx,
                                     mlir::omp::FilterClauseOps &result) const {
   if (auto *clause = findUniqueClause<omp::clause::Filter>()) {
@@ -380,6 +394,20 @@ bool ClauseProcessor::processHint(mlir::omp::HintClauseOps &result) const {
   return false;
 }
 
+bool ClauseProcessor::processInclusive(
+    mlir::Location currentLocation,
+    mlir::omp::InclusiveClauseOps &result) const {
+  if (auto *clause = findUniqueClause<omp::clause::Inclusive>()) {
+    for (const Object &object : clause->v) {
+      const semantics::Symbol *symbol = object.sym();
+      mlir::Value symVal = converter.getSymbolAddress(*symbol);
+      result.inclusiveVars.push_back(symVal);
+    }
+    return true;
+  }
+  return false;
+}
+
 bool ClauseProcessor::processMergeable(
     mlir::omp::MergeableClauseOps &result) const {
   return markClauseOccurrence<omp::clause::Mergeable>(result.mergeable);
@@ -1135,10 +1163,9 @@ bool ClauseProcessor::processReduction(
         llvm::SmallVector<mlir::Attribute> reductionDeclSymbols;
         llvm::SmallVector<const semantics::Symbol *> reductionSyms;
         ReductionProcessor rp;
-        rp.addDeclareReduction(currentLocation, converter, clause,
-                               reductionVars, reduceVarByRef,
-                               reductionDeclSymbols, reductionSyms);
-
+        rp.processReductionArguments(
+            currentLocation, converter, clause, reductionVars, reduceVarByRef,
+            reductionDeclSymbols, reductionSyms, result.reductionMod);
         // Copy local lists into the output.
         llvm::copy(reductionVars, std::back_inserter(result.reductionVars));
         llvm::copy(reduceVarByRef, std::back_inserter(result.reductionByref));
diff --git a/flang/lib/Lower/OpenMP/ClauseProcessor.h b/flang/lib/Lower/OpenMP/ClauseProcessor.h
index 7b047d4a7567a..e05f66c766684 100644
--- a/flang/lib/Lower/OpenMP/ClauseProcessor.h
+++ b/flang/lib/Lower/OpenMP/ClauseProcessor.h
@@ -64,6 +64,8 @@ class ClauseProcessor {
   bool processDeviceType(mlir::omp::DeviceTypeClauseOps &result) const;
   bool processDistSchedule(lower::StatementContext &stmtCtx,
                            mlir::omp::DistScheduleClauseOps &result) const;
+  bool processExclusive(mlir::Location currentLocation,
+                        mlir::omp::ExclusiveClauseOps &result) const;
   bool processFilter(lower::StatementContext &stmtCtx,
                      mlir::omp::FilterClauseOps &result) const;
   bool processFinal(lower::StatementContext &stmtCtx,
@@ -72,6 +74,8 @@ class ClauseProcessor {
       mlir::omp::HasDeviceAddrClauseOps &result,
       llvm::SmallVectorImpl<const semantics::Symbol *> &isDeviceSyms) const;
   bool processHint(mlir::omp::HintClauseOps &result) const;
+  bool processInclusive(mlir::Location currentLocation,
+                        mlir::omp::InclusiveClauseOps &result) const;
   bool processMergeable(mlir::omp::MergeableClauseOps &result) const;
   bool processNowait(mlir::omp::NowaitClauseOps &result) const;
   bool processNumTeams(lower::StatementContext &stmtCtx,
diff --git a/flang/lib/Lower/OpenMP/Clauses.cpp b/flang/lib/Lower/OpenMP/Clauses.cpp
index b424e209d56da..831ba23870360 100644
--- a/flang/lib/Lower/OpenMP/Clauses.cpp
+++ b/flang/lib/Lower/OpenMP/Clauses.cpp
@@ -202,6 +202,7 @@ MAKE_EMPTY_CLASS(Mergeable, Mergeable);
 MAKE_EMPTY_CLASS(Nogroup, Nogroup);
 MAKE_EMPTY_CLASS(NoOpenmp, NoOpenmp);
 MAKE_EMPTY_CLASS(NoOpenmpRoutines, NoOpenmpRoutines);
+MAKE_EMPTY_CLASS(NoOpenmpConstructs, NoOpenmpConstructs);
 MAKE_EMPTY_CLASS(NoParallelism, NoParallelism);
 MAKE_EMPTY_CLASS(Notinbranch, Notinbranch);
 MAKE_EMPTY_CLASS(Nowait, Nowait);
@@ -230,9 +231,6 @@ MAKE_EMPTY_CLASS(Threadprivate, Threadprivate);
 
 MAKE_INCOMPLETE_CLASS(AdjustArgs, AdjustArgs);
 MAKE_INCOMPLETE_CLASS(AppendArgs, AppendArgs);
-MAKE_INCOMPLETE_CLASS(Match, Match);
-// MAKE_INCOMPLETE_CLASS(Otherwise, );   // missing-in-parser
-MAKE_INCOMPLETE_CLASS(When, When);
 
 List<IteratorSpecifier>
 makeIteratorSpecifiers(const parser::OmpIteratorSpecifier &inp,
@@ -528,8 +526,13 @@ Copyprivate make(const parser::OmpClause::Copyprivate &inp,
   return Copyprivate{/*List=*/makeObjects(inp.v, semaCtx)};
 }
 
-Default make(const parser::OmpClause::Default &inp,
-             semantics::SemanticsContext &semaCtx) {
+// The Default clause is overloaded in OpenMP 5.0 and 5.1: it can be either
+// a data-sharing clause, or a METADIRECTIVE clause. In the latter case, it
+// has been superseded by the OTHERWISE clause.
+// Disambiguate this in this representation: for the DSA case, create Default,
+// and in the other case create Otherwise.
+Default makeDefault(const parser::OmpClause::Default &inp,
+                    semantics::SemanticsContext &semaCtx) {
   // inp.v -> parser::OmpDefaultClause
   using wrapped = parser::OmpDefaultClause;
 
@@ -543,7 +546,13 @@ Default make(const parser::OmpClause::Default &inp,
       // clang-format on
   );
 
-  return Default{/*DataSharingAttribute=*/convert(inp.v.v)};
+  auto dsa = std::get<wrapped::DataSharingAttribute>(inp.v.u);
+  return Default{/*DataSharingAttribute=*/convert(dsa)};
+}
+
+Otherwise makeOtherwise(const parser::OmpClause::Default &inp,
+                        semantics::SemanticsContext &semaCtx) {
+  return Otherwise{};
 }
 
 Defaultmap make(const parser::OmpClause::Defaultmap &inp,
@@ -728,8 +737,8 @@ Enter make(const parser::OmpClause::Enter &inp,
 
 Exclusive make(const parser::OmpClause::Exclusive &inp,
                semantics::SemanticsContext &semaCtx) {
-  // inp -> empty
-  llvm_unreachable("Empty: exclusive");
+  // inp.v -> parser::OmpObjectList
+  return Exclusive{makeObjects(/*List=*/inp.v, semaCtx)};
 }
 
 Fail make(const parser::OmpClause::Fail &inp,
@@ -838,8 +847,8 @@ If make(const parser::OmpClause::If &inp,
 
 Inclusive make(const parser::OmpClause::Inclusive &inp,
                semantics::SemanticsContext &semaCtx) {
-  // inp -> empty
-  llvm_unreachable("Empty: inclusive");
+  // inp.v -> parser::OmpObjectList
+  return Inclusive{makeObjects(/*List=*/inp.v, semaCtx)};
 }
 
 Indirect make(const parser::OmpClause::Indirect &inp,
@@ -997,7 +1006,11 @@ Map make(const parser::OmpClause::Map &inp,
               /*LocatorList=*/makeObjects(t4, semaCtx)}};
 }
 
-// Match: incomplete
+Match make(const parser::OmpClause::Match &inp,
+           semantics::SemanticsContext &semaCtx) {
+  return Match{};
+}
+
 // MemoryOrder: empty
 // Mergeable: empty
 
@@ -1023,6 +1036,7 @@ Nontemporal make(const parser::OmpClause::Nontemporal &inp,
 
 // NoOpenmp: empty
 // NoOpenmpRoutines: empty
+// NoOpenmpConstructs: empty
 // NoParallelism: empty
 // Notinbranch: empty
 
@@ -1101,7 +1115,11 @@ Ordered make(const parser::OmpClause::Ordered &inp,
   return Ordered{/*N=*/maybeApply(makeExprFn(semaCtx), inp.v)};
 }
 
-// Otherwise: incomplete, missing-in-parser
+// See also Default.
+Otherwise make(const parser::OmpClause::Otherwise &inp,
+               semantics::SemanticsContext &semaCtx) {
+  return Otherwise{};
+}
 
 Partial make(const parser::OmpClause::Partial &inp,
              semantics::SemanticsContext &semaCtx) {
@@ -1356,15 +1374,32 @@ UsesAllocators make(const parser::OmpClause::UsesAllocators &inp,
 }
 
 // Weak: empty
-// When: incomplete
+
+When make(const parser::OmpClause::When &inp,
+          semantics::SemanticsContext &semaCtx) {
+  return When{};
+}
+
 // Write: empty
 } // namespace clause
 
 Clause makeClause(const parser::OmpClause &cls,
                   semantics::SemanticsContext &semaCtx) {
-  return Fortran::common::visit(
-      [&](auto &&s) {
-        return makeClause(cls.Id(), clause::make(s, semaCtx), cls.source);
+  return Fortran::common::visit( //
+      common::visitors{
+          [&](const parser::OmpClause::Default &s) {
+            using DSA = parser::OmpDefaultClause::DataSharingAttribute;
+            if (std::holds_alternative<DSA>(s.v.u)) {
+              return makeClause(llvm::omp::Clause::OMPC_default,
+                                clause::makeDefault(s, semaCtx), cls.source);
+            } else {
+              return makeClause(llvm::omp::Clause::OMPC_otherwise,
+                                clause::makeOtherwise(s, semaCtx), cls.source);
+            }
+          },
+          [&](auto &&s) {
+            return makeClause(cls.Id(), clause::make(s, semaCtx), cls.source);
+          },
       },
       cls.u);
 }
diff --git a/flang/lib/Lower/OpenMP/Clauses.h b/flang/lib/Lower/OpenMP/Clauses.h
index 65282d243d87a..0f172e0acf626 100644
--- a/flang/lib/Lower/OpenMP/Clauses.h
+++ b/flang/lib/Lower/OpenMP/Clauses.h
@@ -242,6 +242,8 @@ using Mergeable = tomp::clause::MergeableT<TypeTy, IdTy, ExprTy>;
 using Message = tomp::clause::MessageT<TypeTy, IdTy, ExprTy>;
 using NoOpenmp = tomp::clause::NoOpenmpT<TypeTy, IdTy, ExprTy>;
 using NoOpenmpRoutines = tomp::clause::NoOpenmpRoutinesT<TypeTy, IdTy, ExprTy>;
+using NoOpenmpConstructs =
+    tomp::clause::NoOpenmpConstructsT<TypeTy, IdTy, ExprTy>;
 using NoParallelism = tomp::clause::NoParallelismT<TypeTy, IdTy, ExprTy>;
 using Nocontext = tomp::clause::NocontextT<TypeTy, IdTy, ExprTy>;
 using Nogroup = tomp::clause::NogroupT<TypeTy, IdTy, ExprTy>;
@@ -257,6 +259,7 @@ using OmpxBare = tomp::clause::OmpxBareT<TypeTy, IdTy, ExprTy>;
 using OmpxDynCgroupMem = tomp::clause::OmpxDynCgroupMemT<TypeTy, IdTy, ExprTy>;
 using Ordered = tomp::clause::OrderedT<TypeTy, IdTy, ExprTy>;
 using Order = tomp::clause::OrderT<TypeTy, IdTy, ExprTy>;
+using Otherwise = tomp::clause::OtherwiseT<TypeTy, IdTy, ExprTy>;
 using Partial = tomp::clause::PartialT<TypeTy, IdTy, ExprTy>;
 using Priority = tomp::clause::PriorityT<TypeTy, IdTy, ExprTy>;
 using Private = tomp::clause::PrivateT<TypeTy, IdTy, ExprTy>;
diff --git a/flang/lib/Lower/OpenMP/DataSharingProcessor.cpp b/flang/lib/Lower/OpenMP/DataSharingProcessor.cpp
index 5b89816850bed..55f543ca38178 100644
--- a/flang/lib/Lower/OpenMP/DataSharingProcessor.cpp
+++ b/flang/lib/Lower/OpenMP/DataSharingProcessor.cpp
@@ -12,13 +12,17 @@
 
 #include "DataSharingProcessor.h"
 
+#include "PrivateReductionUtils.h"
 #include "Utils.h"
 #include "flang/Lower/ConvertVariable.h"
 #include "flang/Lower/PFTBuilder.h"
 #include "flang/Lower/SymbolMap.h"
+#include "flang/Optimizer/Builder/BoxValue.h"
 #include "flang/Optimizer/Builder/HLFIRTools.h"
 #include "flang/Optimizer/Builder/Todo.h"
+#include "flang/Optimizer/HLFIR/HLFIRDialect.h"
 #include "flang/Optimizer/HLFIR/HLFIROps.h"
+#include "flang/Semantics/attr.h"
 #include "flang/Semantics/tools.h"
 
 namespace Fortran {
@@ -85,35 +89,65 @@ void DataSharingProcessor::insertDeallocs() {
         converter.createHostAssociateVarCloneDealloc(*sym);
         continue;
       }
-
-      lower::SymbolBox hsb = converter.lookupOneLevelUpSymbol(*sym);
-      assert(hsb && "Host symbol box not found");
-      mlir::Type symType = hsb.getAddr().getType();
-      mlir::Location symLoc = hsb.getAddr().getLoc();
-      fir::ExtendedValue symExV = converter.getSymbolExtendedValue(*sym);
-      mlir::omp::PrivateClauseOp privatizer = symToPrivatizer.at(sym);
-
-      lower::SymMapScope scope(symTable);
-      mlir::OpBuilder::InsertionGuard guard(firOpBuilder);
-
-      mlir::Region &deallocRegion = privatizer.getDeallocRegion();
-      fir::FirOpBuilder &firOpBuilder = converter.getFirOpBuilder();
-      mlir::Block *deallocEntryBlock = firOpBuilder.createBlock(
-          &deallocRegion, /*insertPt=*/{}, symType, symLoc);
-
-      firOpBuilder.setInsertionPointToEnd(deallocEntryBlock);
-      symTable.addSymbol(*sym,
-                         fir::substBase(symExV, deallocRegion.getArgument(0)));
-
-      converter.createHostAssociateVarCloneDealloc(*sym);
-      firOpBuilder.create<mlir::omp::YieldOp>(hsb.getAddr().getLoc());
+      // For delayed privatization deallocs are created by
+      // populateByRefInitAndCleanupRegions
     }
 }
 
 void DataSharingProcessor::cloneSymbol(const semantics::Symbol *sym) {
   bool isFirstPrivate = sym->test(semantics::Symbol::Flag::OmpFirstPrivate);
-  bool success = converter.createHostAssociateVarClone(
-      *sym, /*skipDefaultInit=*/isFirstPrivate);
+
+  // If we are doing eager-privatization on a symbol created using delayed
+  // privatization there could be incompatible types here e.g.
+  // fir.ref<fir.box<fir.array<>>>
+  bool success = [&]() -> bool {
+    const auto *details =
+        sym->detailsIf<Fortran::semantics::HostAssocDetails>();
+    assert(details && "No host-association found");
+    const Fortran::semantics::Symbol &hsym = details->symbol();
+    mlir::Value addr = converter.getSymbolAddress(hsym);
+
+    if (auto refTy = mlir::dyn_cast<fir::ReferenceType>(addr.getType())) {
+      if (auto boxTy = mlir::dyn_cast<fir::BoxType>(refTy.getElementType())) {
+        if (auto arrayTy =
+                mlir::dyn_cast<fir::SequenceType>(boxTy.getElementType())) {
+          // FirConverter/fir::ExtendedValue considers all references to boxes
+          // as mutable boxes. Outside of OpenMP it doesn't make sense to have a
+          // mutable box of an array. Work around this here by loading the
+          // reference so it is a normal boxed array.
+          fir::FirOpBuilder &builder = converter.getFirOpBuilder();
+          mlir::Location loc = converter.genLocation(hsym.name());
+          fir::ExtendedValue hexv = converter.getSymbolExtendedValue(hsym);
+
+          llvm::SmallVector<mlir::Value> extents =
+              fir::factory::getExtents(loc, builder, hexv);
+
+          // TODO: uniqName, name
+          mlir::Value allocVal =
+              builder.allocateLocal(loc, arrayTy, /*uniqName=*/"",
+                                    /*name=*/"", extents, /*typeParams=*/{},
+                                    sym->GetUltimate().attrs().test(
+                                        Fortran::semantics::Attr::TARGET));
+          mlir::Value shape = builder.genShape(loc, extents);
+          mlir::Value box = builder.createBox(loc, boxTy, allocVal, shape,
+                                              nullptr, {}, nullptr);
+
+          // This can't be a CharArrayBoxValue because otherwise
+          // boxTy.getElementType() would be a character type.
+          // Assume the array element type isn't polymorphic because we are
+          // privatizing.
+          fir::ExtendedValue newExv = fir::ArrayBoxValue{box, extents};
+
+          converter.bindSymbol(*sym, newExv);
+          return true;
+        }
+      }
+    }
+
+    // Normal case:
+    return converter.createHostAssociateVarClone(
+        *sym, /*skipDefaultInit=*/isFirstPrivate);
+  }();
   (void)success;
   assert(success && "Privatization failed due to existing binding");
 
@@ -132,7 +166,7 @@ void DataSharingProcessor::cloneSymbol(const semantics::Symbol *sym) {
 
   if (needInitClone()) {
     Fortran::lower::initializeCloneAtRuntime(converter, *sym, symTable);
-    callsInitClone = true;
+    mightHaveReadHostSym = true;
   }
 }
 
@@ -173,6 +207,9 @@ void DataSharingProcessor::collectSymbolsForPrivatization() {
     }
   }
 
+  // TODO For common blocks, add the underlying objects within the block. Doing
+  // so, we won't need to explicitely handle block objects (or forget to do
+  // so).
   for (auto *sym : explicitlyPrivatizedSymbols)
     allPrivatizedSymbols.insert(sym);
 }
@@ -184,7 +221,8 @@ bool DataSharingProcessor::needBarrier() {
   // Emit implicit barrier for linear clause. Maybe on somewhere else.
   for (const semantics::Symbol *sym : allPrivatizedSymbols) {
     if (sym->test(semantics::Symbol::Flag::OmpLastPrivate) &&
-        (sym->test(semantics::Symbol::Flag::OmpFirstPrivate) || callsInitClone))
+        (sym->test(semantics::Symbol::Flag::OmpFirstPrivate) ||
+         mightHaveReadHostSym))
       return true;
   }
   return false;
@@ -200,82 +238,85 @@ void DataSharingProcessor::insertLastPrivateCompare(mlir::Operation *op) {
   if (auto wrapper = mlir::dyn_cast<mlir::omp::LoopWrapperInterface>(op))
     loopOp = mlir::cast<mlir::omp::LoopNestOp>(wrapper.getWrappedLoop());
 
-  bool cmpCreated = false;
   mlir::OpBuilder::InsertionGuard guard(firOpBuilder);
-  for (const omp::Clause &clause : clauses) {
-    if (clause.id != llvm::omp::OMPC_lastprivate)
-      continue;
-    if (mlir::isa<mlir::omp::WsloopOp>(op) ||
-        mlir::isa<mlir::omp::SimdOp>(op)) {
-      // Update the original variable just before exiting the worksharing
-      // loop. Conversion as follows:
-      //
-      // omp.wsloop / omp.simd {    omp.wsloop / omp.simd {
-      //   omp.loop_nest {            omp.loop_nest {
-      //     ...                        ...
-      //     store          ===>        store
-      //     omp.yield                  %v = arith.addi %iv, %step
-      //   }                            %cmp = %step < 0 ? %v < %ub : %v > %ub
-      // }                              fir.if %cmp {
-      //                                  fir.store %v to %loopIV
-      //                                  ^%lpv_update_blk:
-      //                                }
-      //                                omp.yield
-      //                              }
-      //                            }
-
-      // Only generate the compare once in presence of multiple LastPrivate
-      // clauses.
-      if (cmpCreated)
-        continue;
-      cmpCreated = true;
-
-      mlir::Location loc = loopOp.getLoc();
-      mlir::Operation *lastOper = loopOp.getRegion().back().getTerminator();
-      firOpBuilder.setInsertionPoint(lastOper);
-
-      mlir::Value cmpOp;
-      llvm::SmallVector<mlir::Value> vs;
-      vs.reserve(loopOp.getIVs().size());
-      for (auto [iv, ub, step] :
-           llvm::zip_equal(loopOp.getIVs(), loopOp.getLoopUpperBounds(),
-                           loopOp.getLoopSteps())) {
-        // v = iv + step
-        // cmp = step < 0 ? v < ub : v > ub
-        mlir::Value v = firOpBuilder.create<mlir::arith::AddIOp>(loc, iv, step);
-        vs.push_back(v);
-        mlir::Value zero =
-            firOpBuilder.createIntegerConstant(loc, step.getType(), 0);
-        mlir::Value negativeStep = firOpBuilder.create<mlir::arith::CmpIOp>(
-            loc, mlir::arith::CmpIPredicate::slt, step, zero);
-        mlir::Value vLT = firOpBuilder.create<mlir::arith::CmpIOp>(
-            loc, mlir::arith::CmpIPredicate::slt, v, ub);
-        mlir::Value vGT = firOpBuilder.create<mlir::arith::CmpIOp>(
-            loc, mlir::arith::CmpIPredicate::sgt, v, ub);
-        mlir::Value icmpOp = firOpBuilder.create<mlir::arith::SelectOp>(
-            loc, negativeStep, vLT, vGT);
-
-        if (cmpOp) {
-          cmpOp = firOpBuilder.create<mlir::arith::AndIOp>(loc, cmpOp, icmpOp);
-        } else {
-          cmpOp = icmpOp;
-        }
-      }
+  bool hasLastPrivate = [&]() {
+    for (const semantics::Symbol *sym : allPrivatizedSymbols) {
+      if (const auto *commonDet =
+              sym->detailsIf<semantics::CommonBlockDetails>()) {
+        for (const auto &mem : commonDet->objects())
+          if (mem->test(semantics::Symbol::Flag::OmpLastPrivate))
+            return true;
+      } else if (sym->test(semantics::Symbol::Flag::OmpLastPrivate))
+        return true;
+    }
 
-      auto ifOp = firOpBuilder.create<fir::IfOp>(loc, cmpOp, /*else*/ false);
-      firOpBuilder.setInsertionPointToStart(&ifOp.getThenRegion().front());
-      for (auto [v, loopIV] : llvm::zip_equal(vs, loopIVs)) {
-        assert(loopIV && "loopIV was not set");
-        firOpBuilder.createStoreWithConvert(loc, v, loopIV);
-      }
-      lastPrivIP = firOpBuilder.saveInsertionPoint();
-    } else if (mlir::isa<mlir::omp::SectionsOp>(op)) {
-      // Already handled by genOMP()
-    } else {
-      TODO(converter.getCurrentLocation(),
-           "lastprivate clause in constructs other than "
-           "simd/worksharing-loop");
+    return false;
+  }();
+
+  if (!hasLastPrivate)
+    return;
+
+  if (mlir::isa<mlir::omp::WsloopOp>(op) || mlir::isa<mlir::omp::SimdOp>(op)) {
+    // Update the original variable just before exiting the worksharing
+    // loop. Conversion as follows:
+    //
+    // omp.wsloop / omp.simd {    omp.wsloop / omp.simd {
+    //   omp.loop_nest {            omp.loop_nest {
+    //     ...                        ...
+    //     store          ===>        store
+    //     omp.yield                  %v = arith.addi %iv, %step
+    //   }                            %cmp = %step < 0 ? %v < %ub : %v > %ub
+    // }                              fir.if %cmp {
+    //                                  fir.store %v to %loopIV
+    //                                  ^%lpv_update_blk:
+    //                                }
+    //                                omp.yield
+    //                              }
+    //                            }
+    mlir::Location loc = loopOp.getLoc();
+    mlir::Operation *lastOper = loopOp.getRegion().back().getTerminator();
+    firOpBuilder.setInsertionPoint(lastOper);
+
+    mlir::Value cmpOp;
+    llvm::SmallVector<mlir::Value> vs;
+    vs.reserve(loopOp.getIVs().size());
+    for (auto [iv, ub, step] :
+         llvm::zip_equal(loopOp.getIVs(), loopOp.getLoopUpperBounds(),
+                         loopOp.getLoopSteps())) {
+      // v = iv + step
+      // cmp = step < 0 ? v < ub : v > ub
+      mlir::Value v = firOpBuilder.create<mlir::arith::AddIOp>(loc, iv, step);
+      vs.push_back(v);
+      mlir::Value zero =
+          firOpBuilder.createIntegerConstant(loc, step.getType(), 0);
+      mlir::Value negativeStep = firOpBuilder.create<mlir::arith::CmpIOp>(
+          loc, mlir::arith::CmpIPredicate::slt, step, zero);
+      mlir::Value vLT = firOpBuilder.create<mlir::arith::CmpIOp>(
+          loc, mlir::arith::CmpIPredicate::slt, v, ub);
+      mlir::Value vGT = firOpBuilder.create<mlir::arith::CmpIOp>(
+          loc, mlir::arith::CmpIPredicate::sgt, v, ub);
+      mlir::Value icmpOp = firOpBuilder.create<mlir::arith::SelectOp>(
+          loc, negativeStep, vLT, vGT);
+
+      if (cmpOp)
+        cmpOp = firOpBuilder.create<mlir::arith::AndIOp>(loc, cmpOp, icmpOp);
+      else
+        cmpOp = icmpOp;
+    }
+
+    auto ifOp = firOpBuilder.create<fir::IfOp>(loc, cmpOp, /*else*/ false);
+    firOpBuilder.setInsertionPointToStart(&ifOp.getThenRegion().front());
+    for (auto [v, loopIV] : llvm::zip_equal(vs, loopIVs)) {
+      assert(loopIV && "loopIV was not set");
+      firOpBuilder.createStoreWithConvert(loc, v, loopIV);
     }
+    lastPrivIP = firOpBuilder.saveInsertionPoint();
+  } else if (mlir::isa<mlir::omp::SectionsOp>(op)) {
+    // Already handled by genOMP()
+  } else {
+    TODO(converter.getCurrentLocation(),
+         "lastprivate clause in constructs other than "
+         "simd/worksharing-loop");
   }
 }
 
@@ -468,15 +509,47 @@ void DataSharingProcessor::doPrivatize(const semantics::Symbol *sym,
   lower::SymbolBox hsb = converter.lookupOneLevelUpSymbol(*sym);
   assert(hsb && "Host symbol box not found");
 
-  mlir::Type symType = hsb.getAddr().getType();
   mlir::Location symLoc = hsb.getAddr().getLoc();
   std::string privatizerName = sym->name().ToString() + ".privatizer";
   bool isFirstPrivate = sym->test(semantics::Symbol::Flag::OmpFirstPrivate);
 
+  mlir::Value privVal = hsb.getAddr();
+  mlir::Type allocType = privVal.getType();
+  if (!mlir::isa<fir::PointerType>(privVal.getType()))
+    allocType = fir::unwrapRefType(privVal.getType());
+
+  if (auto poly = mlir::dyn_cast<fir::ClassType>(allocType)) {
+    if (!mlir::isa<fir::PointerType>(poly.getEleTy()) && isFirstPrivate)
+      TODO(symLoc, "create polymorphic host associated copy");
+  }
+
+  // fir.array<> cannot be converted to any single llvm type and fir helpers
+  // are not available in openmp to llvmir translation so we cannot generate
+  // an alloca for a fir.array type there. Get around this by boxing all
+  // arrays.
+  if (mlir::isa<fir::SequenceType>(allocType)) {
+    hlfir::Entity entity{hsb.getAddr()};
+    entity = genVariableBox(symLoc, firOpBuilder, entity);
+    privVal = entity.getBase();
+    allocType = privVal.getType();
+  }
+
+  if (mlir::isa<fir::BaseBoxType>(privVal.getType())) {
+    // Boxes should be passed by reference into nested regions:
+    auto oldIP = firOpBuilder.saveInsertionPoint();
+    firOpBuilder.setInsertionPointToStart(firOpBuilder.getAllocaBlock());
+    auto alloca = firOpBuilder.create<fir::AllocaOp>(symLoc, privVal.getType());
+    firOpBuilder.restoreInsertionPoint(oldIP);
+    firOpBuilder.create<fir::StoreOp>(symLoc, privVal, alloca);
+    privVal = alloca;
+  }
+
+  mlir::Type argType = privVal.getType();
+
   mlir::omp::PrivateClauseOp privatizerOp = [&]() {
     auto moduleOp = firOpBuilder.getModule();
     auto uniquePrivatizerName = fir::getTypeAsString(
-        symType, converter.getKindMap(),
+        allocType, converter.getKindMap(),
         converter.mangleName(*sym) +
             (isFirstPrivate ? "_firstprivate" : "_private"));
 
@@ -488,44 +561,40 @@ void DataSharingProcessor::doPrivatize(const semantics::Symbol *sym,
     mlir::OpBuilder::InsertionGuard guard(firOpBuilder);
     firOpBuilder.setInsertionPointToStart(moduleOp.getBody());
     auto result = firOpBuilder.create<mlir::omp::PrivateClauseOp>(
-        symLoc, uniquePrivatizerName, symType,
+        symLoc, uniquePrivatizerName, allocType,
         isFirstPrivate ? mlir::omp::DataSharingClauseType::FirstPrivate
                        : mlir::omp::DataSharingClauseType::Private);
     fir::ExtendedValue symExV = converter.getSymbolExtendedValue(*sym);
     lower::SymMapScope outerScope(symTable);
 
-    // Populate the `alloc` region.
-    {
-      mlir::Region &allocRegion = result.getAllocRegion();
-      mlir::Block *allocEntryBlock = firOpBuilder.createBlock(
-          &allocRegion, /*insertPt=*/{}, symType, symLoc);
-
-      firOpBuilder.setInsertionPointToEnd(allocEntryBlock);
-
-      fir::ExtendedValue localExV =
-          hlfir::translateToExtendedValue(
-              symLoc, firOpBuilder, hlfir::Entity{allocRegion.getArgument(0)},
-              /*contiguousHint=*/
-              evaluate::IsSimplyContiguous(*sym, converter.getFoldingContext()))
-              .first;
-
-      symTable.addSymbol(*sym, localExV);
-      lower::SymMapScope innerScope(symTable);
-      cloneSymbol(sym);
-      mlir::Value cloneAddr = symTable.shallowLookupSymbol(*sym).getAddr();
-      mlir::Type cloneType = cloneAddr.getType();
-
-      // A `convert` op is required for variables that are storage associated
-      // via `equivalence`. The problem is that these variables are declared as
-      // `fir.ptr`s while their privatized storage is declared as `fir.ref`,
-      // therefore we convert to proper symbol type.
-      mlir::Value yieldedValue =
-          (symType == cloneType) ? cloneAddr
-                                 : firOpBuilder.createConvert(
-                                       cloneAddr.getLoc(), symType, cloneAddr);
-
-      firOpBuilder.create<mlir::omp::YieldOp>(hsb.getAddr().getLoc(),
-                                              yieldedValue);
+    // Populate the `init` region.
+    // We need to initialize in the following cases:
+    // 1. The allocation was for a derived type which requires initialization
+    //    (this can be skipped if it will be initialized anyway by the copy
+    //    region, unless the derived type has allocatable components)
+    // 2. The allocation was for any kind of box
+    // 3. The allocation was for a boxed character
+    const bool needsInitialization =
+        (Fortran::lower::hasDefaultInitialization(sym->GetUltimate()) &&
+         (!isFirstPrivate || hlfir::mayHaveAllocatableComponent(allocType))) ||
+        mlir::isa<fir::BaseBoxType>(allocType) ||
+        mlir::isa<fir::BoxCharType>(allocType);
+    if (needsInitialization) {
+      mlir::Region &initRegion = result.getInitRegion();
+      mlir::Block *initBlock = firOpBuilder.createBlock(
+          &initRegion, /*insertPt=*/{}, {argType, argType}, {symLoc, symLoc});
+
+      populateByRefInitAndCleanupRegions(
+          converter, symLoc, argType, /*scalarInitValue=*/nullptr, initBlock,
+          result.getInitPrivateArg(), result.getInitMoldArg(),
+          result.getDeallocRegion(),
+          isFirstPrivate ? DeclOperationKind::FirstPrivate
+                         : DeclOperationKind::Private,
+          sym);
+      // TODO: currently there are false positives from dead uses of the mold
+      // arg
+      if (!result.getInitMoldArg().getUses().empty())
+        mightHaveReadHostSym = true;
     }
 
     // Populate the `copy` region if this is a `firstprivate`.
@@ -534,7 +603,7 @@ void DataSharingProcessor::doPrivatize(const semantics::Symbol *sym,
       // First block argument corresponding to the original/host value while
       // second block argument corresponding to the privatized value.
       mlir::Block *copyEntryBlock = firOpBuilder.createBlock(
-          &copyRegion, /*insertPt=*/{}, {symType, symType}, {symLoc, symLoc});
+          &copyRegion, /*insertPt=*/{}, {argType, argType}, {symLoc, symLoc});
       firOpBuilder.setInsertionPointToEnd(copyEntryBlock);
 
       auto addSymbol = [&](unsigned argIdx, bool force = false) {
@@ -565,7 +634,7 @@ void DataSharingProcessor::doPrivatize(const semantics::Symbol *sym,
 
   if (clauseOps) {
     clauseOps->privateSyms.push_back(mlir::SymbolRefAttr::get(privatizerOp));
-    clauseOps->privateVars.push_back(hsb.getAddr());
+    clauseOps->privateVars.push_back(privVal);
   }
 
   symToPrivatizer[sym] = privatizerOp;
diff --git a/flang/lib/Lower/OpenMP/DataSharingProcessor.h b/flang/lib/Lower/OpenMP/DataSharingProcessor.h
index 8c7a222ec939f..8e15c6d260389 100644
--- a/flang/lib/Lower/OpenMP/DataSharingProcessor.h
+++ b/flang/lib/Lower/OpenMP/DataSharingProcessor.h
@@ -86,7 +86,7 @@ class DataSharingProcessor {
   lower::pft::Evaluation &eval;
   bool shouldCollectPreDeterminedSymbols;
   bool useDelayedPrivatization;
-  bool callsInitClone = false;
+  bool mightHaveReadHostSym = false;
   lower::SymMap &symTable;
   OMPConstructSymbolVisitor visitor;
 
diff --git a/flang/lib/Lower/OpenMP/OpenMP.cpp b/flang/lib/Lower/OpenMP/OpenMP.cpp
index 1434bcd6330e0..bd794033cdf11 100644
--- a/flang/lib/Lower/OpenMP/OpenMP.cpp
+++ b/flang/lib/Lower/OpenMP/OpenMP.cpp
@@ -18,7 +18,6 @@
 #include "Decomposer.h"
 #include "ReductionProcessor.h"
 #include "Utils.h"
-#include "flang/Common/OpenMP-utils.h"
 #include "flang/Common/idioms.h"
 #include "flang/Lower/Bridge.h"
 #include "flang/Lower/ConvertExpr.h"
@@ -35,6 +34,7 @@
 #include "flang/Parser/parse-tree.h"
 #include "flang/Semantics/openmp-directive-sets.h"
 #include "flang/Semantics/tools.h"
+#include "flang/Support/OpenMP-utils.h"
 #include "mlir/Dialect/ControlFlow/IR/ControlFlowOps.h"
 #include "mlir/Dialect/OpenMP/OpenMPDialect.h"
 #include "mlir/Transforms/RegionUtils.h"
@@ -381,6 +381,9 @@ extractOmpDirective(const parser::OpenMPConstruct &ompConstruct) {
           [](const parser::OpenMPDeclarativeAllocate &c) {
             return llvm::omp::OMPD_allocate;
           },
+          [](const parser::OpenMPDispatchConstruct &c) {
+            return llvm::omp::OMPD_dispatch;
+          },
           [](const parser::OpenMPExecutableAllocate &c) {
             return llvm::omp::OMPD_allocate;
           },
@@ -413,6 +416,9 @@ extractOmpDirective(const parser::OpenMPConstruct &ompConstruct) {
                     [](const parser::OpenMPCancellationPointConstruct &c) {
                       return llvm::omp::OMPD_cancellation_point;
                     },
+                    [](const parser::OmpMetadirectiveDirective &c) {
+                      return llvm::omp::OMPD_metadirective;
+                    },
                     [](const parser::OpenMPDepobjConstruct &c) {
                       return llvm::omp::OMPD_depobj;
                     }},
@@ -1578,6 +1584,15 @@ static void genParallelClauses(
   cp.processReduction(loc, clauseOps, reductionSyms);
 }
 
+static void genScanClauses(lower::AbstractConverter &converter,
+                           semantics::SemanticsContext &semaCtx,
+                           const List<Clause> &clauses, mlir::Location loc,
+                           mlir::omp::ScanOperands &clauseOps) {
+  ClauseProcessor cp(converter, semaCtx, clauses);
+  cp.processInclusive(loc, clauseOps);
+  cp.processExclusive(loc, clauseOps);
+}
+
 static void genSectionsClauses(
     lower::AbstractConverter &converter, semantics::SemanticsContext &semaCtx,
     const List<Clause> &clauses, mlir::Location loc,
@@ -1975,6 +1990,16 @@ genParallelOp(lower::AbstractConverter &converter, lower::SymMap &symTable,
   return parallelOp;
 }
 
+static mlir::omp::ScanOp
+genScanOp(lower::AbstractConverter &converter, lower::SymMap &symTable,
+          semantics::SemanticsContext &semaCtx, mlir::Location loc,
+          const ConstructQueue &queue, ConstructQueue::const_iterator item) {
+  mlir::omp::ScanOperands clauseOps;
+  genScanClauses(converter, semaCtx, item->clauses, loc, clauseOps);
+  return converter.getFirOpBuilder().create<mlir::omp::ScanOp>(
+      converter.getCurrentLocation(), clauseOps);
+}
+
 /// This breaks the normal prototype of the gen*Op functions: adding the
 /// sectionBlocks argument so that the enclosed section constructs can be
 /// lowered here with correct reduction symbol remapping.
@@ -2083,7 +2108,13 @@ genSectionsOp(lower::AbstractConverter &converter, lower::SymMap &symTable,
       const auto &objList = std::get<ObjectList>(lastp->t);
       for (const Object &object : objList) {
         semantics::Symbol *sym = object.sym();
-        converter.copyHostAssociateVar(*sym, &insp, /*hostIsSource=*/false);
+        if (const auto *common =
+                sym->detailsIf<semantics::CommonBlockDetails>()) {
+          for (const auto &obj : common->objects())
+            converter.copyHostAssociateVar(*obj, &insp, /*hostIsSource=*/false);
+        } else {
+          converter.copyHostAssociateVar(*sym, &insp, /*hostIsSource=*/false);
+        }
       }
     }
   }
@@ -2513,7 +2544,7 @@ static void genStandaloneDo(lower::AbstractConverter &converter,
 
   DataSharingProcessor dsp(converter, semaCtx, item->clauses, eval,
                            /*shouldCollectPreDeterminedSymbols=*/true,
-                           enableDelayedPrivatizationStaging, symTable);
+                           enableDelayedPrivatization, symTable);
   dsp.processStep1(&wsloopClauseOps);
 
   mlir::omp::LoopNestOperands loopNestClauseOps;
@@ -2978,7 +3009,7 @@ static void genOMPDispatch(lower::AbstractConverter &converter,
     genStandaloneParallel(converter, symTable, semaCtx, eval, loc, queue, item);
     break;
   case llvm::omp::Directive::OMPD_scan:
-    TODO(loc, "Unhandled directive " + llvm::omp::getOpenMPDirectiveName(dir));
+    genScanOp(converter, symTable, semaCtx, loc, queue, item);
     break;
   case llvm::omp::Directive::OMPD_section:
     llvm_unreachable("genOMPDispatch: OMPD_section");
@@ -3136,6 +3167,13 @@ static void genOMP(lower::AbstractConverter &converter, lower::SymMap &symTable,
   // support the case of threadprivate variable declared in module.
 }
 
+static void genOMP(lower::AbstractConverter &converter, lower::SymMap &symTable,
+                   semantics::SemanticsContext &semaCtx,
+                   lower::pft::Evaluation &eval,
+                   const parser::OmpMetadirectiveDirective &meta) {
+  TODO(converter.getCurrentLocation(), "METADIRECTIVE");
+}
+
 static void genOMP(lower::AbstractConverter &converter, lower::SymMap &symTable,
                    semantics::SemanticsContext &semaCtx,
                    lower::pft::Evaluation &eval,
@@ -3388,6 +3426,13 @@ static void genOMP(lower::AbstractConverter &converter, lower::SymMap &symTable,
   TODO(converter.getCurrentLocation(), "OpenMPUtilityConstruct");
 }
 
+static void genOMP(lower::AbstractConverter &converter, lower::SymMap &symTable,
+                   semantics::SemanticsContext &semaCtx,
+                   lower::pft::Evaluation &eval,
+                   const parser::OpenMPDispatchConstruct &) {
+  TODO(converter.getCurrentLocation(), "OpenMPDispatchConstruct");
+}
+
 static void genOMP(lower::AbstractConverter &converter, lower::SymMap &symTable,
                    semantics::SemanticsContext &semaCtx,
                    lower::pft::Evaluation &eval,
diff --git a/flang/lib/Lower/OpenMP/PrivateReductionUtils.cpp b/flang/lib/Lower/OpenMP/PrivateReductionUtils.cpp
index 83f0d4e93ca54..951293b133677 100644
--- a/flang/lib/Lower/OpenMP/PrivateReductionUtils.cpp
+++ b/flang/lib/Lower/OpenMP/PrivateReductionUtils.cpp
@@ -12,17 +12,38 @@
 
 #include "PrivateReductionUtils.h"
 
+#include "flang/Lower/AbstractConverter.h"
+#include "flang/Lower/Allocatable.h"
+#include "flang/Lower/ConvertVariable.h"
+#include "flang/Optimizer/Builder/BoxValue.h"
+#include "flang/Optimizer/Builder/Character.h"
 #include "flang/Optimizer/Builder/FIRBuilder.h"
 #include "flang/Optimizer/Builder/HLFIRTools.h"
+#include "flang/Optimizer/Builder/Runtime/Derived.h"
 #include "flang/Optimizer/Builder/Todo.h"
+#include "flang/Optimizer/Dialect/FIROps.h"
+#include "flang/Optimizer/Dialect/FIRType.h"
+#include "flang/Optimizer/HLFIR/HLFIRDialect.h"
 #include "flang/Optimizer/HLFIR/HLFIROps.h"
 #include "flang/Optimizer/Support/FatalError.h"
+#include "flang/Semantics/symbol.h"
 #include "mlir/Dialect/OpenMP/OpenMPDialect.h"
 #include "mlir/IR/Location.h"
 
-static void createCleanupRegion(fir::FirOpBuilder &builder, mlir::Location loc,
-                                mlir::Type argType,
-                                mlir::Region &cleanupRegion) {
+static bool hasFinalization(const Fortran::semantics::Symbol &sym) {
+  if (sym.has<Fortran::semantics::ObjectEntityDetails>())
+    if (const Fortran::semantics::DeclTypeSpec *declTypeSpec = sym.GetType())
+      if (const Fortran::semantics::DerivedTypeSpec *derivedTypeSpec =
+              declTypeSpec->AsDerived())
+        return Fortran::semantics::IsFinalizable(*derivedTypeSpec);
+  return false;
+}
+
+static void createCleanupRegion(Fortran::lower::AbstractConverter &converter,
+                                mlir::Location loc, mlir::Type argType,
+                                mlir::Region &cleanupRegion,
+                                const Fortran::semantics::Symbol *sym) {
+  fir::FirOpBuilder &builder = converter.getFirOpBuilder();
   assert(cleanupRegion.empty());
   mlir::Block *block = builder.createBlock(&cleanupRegion, cleanupRegion.end(),
                                            {argType}, {loc});
@@ -37,12 +58,29 @@ static void createCleanupRegion(fir::FirOpBuilder &builder, mlir::Location loc,
 
   mlir::Type valTy = fir::unwrapRefType(argType);
   if (auto boxTy = mlir::dyn_cast_or_null<fir::BaseBoxType>(valTy)) {
-    if (!mlir::isa<fir::HeapType, fir::PointerType>(boxTy.getEleTy())) {
-      mlir::Type innerTy = fir::extractSequenceType(boxTy);
-      if (!mlir::isa<fir::SequenceType>(innerTy))
-        typeError();
+    // TODO: what about undoing init of unboxed derived types?
+    if (auto recTy = mlir::dyn_cast<fir::RecordType>(
+            fir::unwrapSequenceType(fir::dyn_cast_ptrOrBoxEleTy(boxTy)))) {
+      mlir::Type eleTy = boxTy.getEleTy();
+      if (mlir::isa<fir::PointerType, fir::HeapType>(eleTy)) {
+        mlir::Type mutableBoxTy =
+            fir::ReferenceType::get(fir::BoxType::get(eleTy));
+        mlir::Value converted =
+            builder.createConvert(loc, mutableBoxTy, block->getArgument(0));
+        if (recTy.getNumLenParams() > 0)
+          TODO(loc, "Deallocate box with length parameters");
+        fir::MutableBoxValue mutableBox{converted, /*lenParameters=*/{},
+                                        /*mutableProperties=*/{}};
+        Fortran::lower::genDeallocateIfAllocated(converter, mutableBox, loc);
+        builder.create<mlir::omp::YieldOp>(loc);
+        return;
+      }
     }
 
+    // TODO: just replace this whole body with
+    // Fortran::lower::genDeallocateIfAllocated (not done now to avoid test
+    // churn)
+
     mlir::Value arg = builder.loadIfRef(loc, block->getArgument(0));
     assert(mlir::isa<fir::BaseBoxType>(arg.getType()));
 
@@ -66,6 +104,21 @@ static void createCleanupRegion(fir::FirOpBuilder &builder, mlir::Location loc,
     return;
   }
 
+  if (auto boxCharTy = mlir::dyn_cast<fir::BoxCharType>(argType)) {
+    auto [addr, len] =
+        fir::factory::CharacterExprHelper{builder, loc}.createUnboxChar(
+            block->getArgument(0));
+
+    // convert addr to a heap type so it can be used with fir::FreeMemOp
+    auto refTy = mlir::cast<fir::ReferenceType>(addr.getType());
+    auto heapTy = fir::HeapType::get(refTy.getEleTy());
+    addr = builder.createConvert(loc, heapTy, addr);
+
+    builder.create<fir::FreeMemOp>(loc, addr);
+    builder.create<mlir::omp::YieldOp>(loc);
+    return;
+  }
+
   typeError();
 }
 
@@ -96,141 +149,480 @@ fir::ShapeShiftOp Fortran::lower::omp::getShapeShift(fir::FirOpBuilder &builder,
   return shapeShift;
 }
 
-void Fortran::lower::omp::populateByRefInitAndCleanupRegions(
-    fir::FirOpBuilder &builder, mlir::Location loc, mlir::Type argType,
-    mlir::Value scalarInitValue, mlir::Block *initBlock,
-    mlir::Value allocatedPrivVarArg, mlir::Value moldArg,
-    mlir::Region &cleanupRegion) {
-  mlir::Type ty = fir::unwrapRefType(argType);
-  builder.setInsertionPointToEnd(initBlock);
-  auto yield = [&](mlir::Value ret) {
+// Initialize box newBox using moldBox. These should both have the same type and
+// be boxes containing derived types e.g.
+// fir.box<!fir.type<>>
+// fir.box<!fir.heap<!fir.type<>>
+// fir.box<!fir.heap<!fir.array<fir.type<>>>
+// fir.class<...<!fir.type<>>>
+// If the type doesn't match , this does nothing
+static void initializeIfDerivedTypeBox(fir::FirOpBuilder &builder,
+                                       mlir::Location loc, mlir::Value newBox,
+                                       mlir::Value moldBox, bool hasInitializer,
+                                       bool isFirstPrivate) {
+  assert(moldBox.getType() == newBox.getType());
+  fir::BoxType boxTy = mlir::dyn_cast<fir::BoxType>(newBox.getType());
+  fir::ClassType classTy = mlir::dyn_cast<fir::ClassType>(newBox.getType());
+  if (!boxTy && !classTy)
+    return;
+
+  // remove pointer and array types in the middle
+  mlir::Type eleTy = boxTy ? boxTy.getElementType() : classTy.getEleTy();
+  mlir::Type derivedTy = fir::unwrapRefType(eleTy);
+  if (auto array = mlir::dyn_cast<fir::SequenceType>(derivedTy))
+    derivedTy = array.getElementType();
+
+  if (!fir::isa_derived(derivedTy))
+    return;
+
+  if (hasInitializer)
+    fir::runtime::genDerivedTypeInitialize(builder, loc, newBox);
+
+  if (hlfir::mayHaveAllocatableComponent(derivedTy) && !isFirstPrivate)
+    fir::runtime::genDerivedTypeInitializeClone(builder, loc, newBox, moldBox);
+}
+
+static void getLengthParameters(fir::FirOpBuilder &builder, mlir::Location loc,
+                                mlir::Value moldArg,
+                                llvm::SmallVectorImpl<mlir::Value> &lenParams) {
+  // We pass derived types unboxed and so are not self-contained entities.
+  // Assume that unboxed derived types won't need length paramters.
+  if (!hlfir::isFortranEntity(moldArg))
+    return;
+
+  hlfir::genLengthParameters(loc, builder, hlfir::Entity{moldArg}, lenParams);
+  if (lenParams.empty())
+    return;
+
+  // The verifier for EmboxOp doesn't allow length parameters when the the
+  // character already has static LEN. genLengthParameters may still return them
+  // in this case.
+  mlir::Type unwrappedType =
+      fir::unwrapRefType(fir::unwrapSeqOrBoxedSeqType(moldArg.getType()));
+  if (auto strTy = mlir::dyn_cast<fir::CharacterType>(unwrappedType)) {
+    if (strTy.hasConstantLen())
+      lenParams.resize(0);
+  }
+}
+
+static bool
+isDerivedTypeNeedingInitialization(const Fortran::semantics::Symbol &sym) {
+  // Fortran::lower::hasDefaultInitialization returns false for ALLOCATABLE, so
+  // re-implement here.
+  // ignorePointer=true because either the pointer points to the same target as
+  // the original variable, or it is uninitialized.
+  if (const Fortran::semantics::DeclTypeSpec *declTypeSpec = sym.GetType())
+    if (const Fortran::semantics::DerivedTypeSpec *derivedTypeSpec =
+            declTypeSpec->AsDerived())
+      return derivedTypeSpec->HasDefaultInitialization(
+          /*ignoreAllocatable=*/false, /*ignorePointer=*/true);
+  return false;
+}
+
+static mlir::Value generateZeroShapeForRank(fir::FirOpBuilder &builder,
+                                            mlir::Location loc,
+                                            mlir::Value moldArg) {
+  mlir::Type moldType = fir::unwrapRefType(moldArg.getType());
+  mlir::Type eleType = fir::dyn_cast_ptrOrBoxEleTy(moldType);
+  fir::SequenceType seqTy =
+      mlir::dyn_cast_if_present<fir::SequenceType>(eleType);
+  if (!seqTy)
+    return mlir::Value{};
+
+  unsigned rank = seqTy.getShape().size();
+  mlir::Value zero =
+      builder.createIntegerConstant(loc, builder.getIndexType(), 0);
+  mlir::SmallVector<mlir::Value> dims;
+  dims.resize(rank, zero);
+  mlir::Type shapeTy = fir::ShapeType::get(builder.getContext(), rank);
+  return builder.create<fir::ShapeOp>(loc, shapeTy, dims);
+}
+
+namespace {
+using namespace Fortran::lower::omp;
+/// Class to store shared data so we don't have to maintain so many function
+/// arguments
+class PopulateInitAndCleanupRegionsHelper {
+public:
+  PopulateInitAndCleanupRegionsHelper(
+      Fortran::lower::AbstractConverter &converter, mlir::Location loc,
+      mlir::Type argType, mlir::Value scalarInitValue,
+      mlir::Value allocatedPrivVarArg, mlir::Value moldArg,
+      mlir::Block *initBlock, mlir::Region &cleanupRegion,
+      DeclOperationKind kind, const Fortran::semantics::Symbol *sym)
+      : converter{converter}, builder{converter.getFirOpBuilder()}, loc{loc},
+        argType{argType}, scalarInitValue{scalarInitValue},
+        allocatedPrivVarArg{allocatedPrivVarArg}, moldArg{moldArg},
+        initBlock{initBlock}, cleanupRegion{cleanupRegion}, kind{kind},
+        sym{sym} {
+    valType = fir::unwrapRefType(argType);
+  }
+
+  void populateByRefInitAndCleanupRegions();
+
+private:
+  Fortran::lower::AbstractConverter &converter;
+  fir::FirOpBuilder &builder;
+
+  mlir::Location loc;
+
+  /// The type of the block arguments passed into the init and cleanup regions
+  mlir::Type argType;
+
+  /// argType stripped of any references
+  mlir::Type valType;
+
+  /// sclarInitValue:      The value scalars should be initialized to (only
+  ///                      valid for reductions).
+  /// allocatedPrivVarArg: The allocation for the private
+  ///                      variable.
+  /// moldArg:             The original variable.
+  /// loadedMoldArg:       The original variable, loaded. Access via
+  ///                      getLoadedMoldArg().
+  mlir::Value scalarInitValue, allocatedPrivVarArg, moldArg, loadedMoldArg;
+
+  /// The first block in the init region.
+  mlir::Block *initBlock;
+
+  /// The region to insert clanup code into.
+  mlir::Region &cleanupRegion;
+
+  /// The kind of operation we are generating init/cleanup regions for.
+  DeclOperationKind kind;
+
+  /// (optional) The symbol being privatized.
+  const Fortran::semantics::Symbol *sym;
+
+  /// Any length parameters which have been fetched for the type
+  mlir::SmallVector<mlir::Value> lenParams;
+
+  void createYield(mlir::Value ret) {
     builder.create<mlir::omp::YieldOp>(loc, ret);
-  };
+  }
 
-  if (fir::isa_trivial(ty)) {
+  void initTrivialType() {
     builder.setInsertionPointToEnd(initBlock);
-
     if (scalarInitValue)
       builder.createStoreWithConvert(loc, scalarInitValue, allocatedPrivVarArg);
-    yield(allocatedPrivVarArg);
-    return;
+    createYield(allocatedPrivVarArg);
   }
 
-  // check if an allocatable box is unallocated. If so, initialize the boxAlloca
-  // to be unallocated e.g.
-  // %box_alloca = fir.alloca !fir.box<!fir.heap<...>>
-  // %addr = fir.box_addr %box
-  // if (%addr == 0) {
-  //   %nullbox = fir.embox %addr
-  //   fir.store %nullbox to %box_alloca
-  // } else {
-  //   // ...
-  //   fir.store %something to %box_alloca
-  // }
-  // omp.yield %box_alloca
-  moldArg = builder.loadIfRef(loc, moldArg);
-  auto handleNullAllocatable = [&](mlir::Value boxAlloca) -> fir::IfOp {
-    mlir::Value addr = builder.create<fir::BoxAddrOp>(loc, moldArg);
-    mlir::Value isNotAllocated = builder.genIsNullAddr(loc, addr);
-    fir::IfOp ifOp = builder.create<fir::IfOp>(loc, isNotAllocated,
-                                               /*withElseRegion=*/true);
-    builder.setInsertionPointToStart(&ifOp.getThenRegion().front());
-    // just embox the null address and return
-    mlir::Value nullBox = builder.create<fir::EmboxOp>(loc, ty, addr);
-    builder.create<fir::StoreOp>(loc, nullBox, boxAlloca);
-    return ifOp;
-  };
+  void initBoxedPrivatePointer(fir::BaseBoxType boxTy);
 
-  // all arrays are boxed
-  if (auto boxTy = mlir::dyn_cast_or_null<fir::BaseBoxType>(ty)) {
-    bool isAllocatableOrPointer =
-        mlir::isa<fir::HeapType, fir::PointerType>(boxTy.getEleTy());
+  /// e.g. !fir.box<!fir.heap<i32>>, !fir.box<!fir.type<....>>,
+  /// !fir.box<!fir.char<...>>
+  void initAndCleanupBoxedScalar(fir::BaseBoxType boxTy,
+                                 bool needsInitialization);
 
-    builder.setInsertionPointToEnd(initBlock);
-    mlir::Value boxAlloca = allocatedPrivVarArg;
-    mlir::Type innerTy = fir::unwrapRefType(boxTy.getEleTy());
-    if (fir::isa_trivial(innerTy)) {
-      // boxed non-sequence value e.g. !fir.box<!fir.heap<i32>>
-      if (!isAllocatableOrPointer)
-        TODO(loc,
-             "Reduction/Privatization of non-allocatable trivial typed box");
+  void initAndCleanupBoxedArray(fir::BaseBoxType boxTy,
+                                bool needsInitialization);
+
+  void initAndCleanupBoxchar(fir::BoxCharType boxCharTy);
+
+  void initAndCleanupUnboxedDerivedType(bool needsInitialization);
+
+  fir::IfOp handleNullAllocatable();
 
-      fir::IfOp ifUnallocated = handleNullAllocatable(boxAlloca);
+  // Do this lazily so that we don't load it when it is not used.
+  inline mlir::Value getLoadedMoldArg() {
+    if (loadedMoldArg)
+      return loadedMoldArg;
+    loadedMoldArg = builder.loadIfRef(loc, moldArg);
+    return loadedMoldArg;
+  }
+};
+
+} // namespace
+
+/// The initial state of a private pointer is undefined so we don't need to
+/// match the mold argument (OpenMP 5.2 end of page 106).
+void PopulateInitAndCleanupRegionsHelper::initBoxedPrivatePointer(
+    fir::BaseBoxType boxTy) {
+  assert(isPrivatization(kind));
+  // we need a shape with the right rank so that the embox op is lowered
+  // to an llvm struct of the right type. This returns nullptr if the types
+  // aren't right.
+  mlir::Value shape = generateZeroShapeForRank(builder, loc, moldArg);
+  // Just incase, do initialize the box with a null value
+  mlir::Value null = builder.createNullConstant(loc, boxTy.getEleTy());
+  mlir::Value nullBox;
+  nullBox = builder.create<fir::EmboxOp>(loc, boxTy, null, shape,
+                                         /*slice=*/mlir::Value{}, lenParams);
+  builder.create<fir::StoreOp>(loc, nullBox, allocatedPrivVarArg);
+  createYield(allocatedPrivVarArg);
+}
+/// Check if an allocatable box is unallocated. If so, initialize the boxAlloca
+/// to be unallocated e.g.
+/// %box_alloca = fir.alloca !fir.box<!fir.heap<...>>
+/// %addr = fir.box_addr %box
+/// if (%addr == 0) {
+///   %nullbox = fir.embox %addr
+///   fir.store %nullbox to %box_alloca
+/// } else {
+///   // ...
+///   fir.store %something to %box_alloca
+/// }
+/// omp.yield %box_alloca
+fir::IfOp PopulateInitAndCleanupRegionsHelper::handleNullAllocatable() {
+  mlir::Value addr = builder.create<fir::BoxAddrOp>(loc, getLoadedMoldArg());
+  mlir::Value isNotAllocated = builder.genIsNullAddr(loc, addr);
+  fir::IfOp ifOp = builder.create<fir::IfOp>(loc, isNotAllocated,
+                                             /*withElseRegion=*/true);
+  builder.setInsertionPointToStart(&ifOp.getThenRegion().front());
+  // Just embox the null address and return.
+  // We have to give the embox a shape so that the LLVM box structure has the
+  // right rank. This returns an empty value if the types don't match.
+  mlir::Value shape = generateZeroShapeForRank(builder, loc, moldArg);
+
+  mlir::Value nullBox =
+      builder.create<fir::EmboxOp>(loc, valType, addr, shape,
+                                   /*slice=*/mlir::Value{}, lenParams);
+  builder.create<fir::StoreOp>(loc, nullBox, allocatedPrivVarArg);
+  return ifOp;
+}
+
+void PopulateInitAndCleanupRegionsHelper::initAndCleanupBoxedScalar(
+    fir::BaseBoxType boxTy, bool needsInitialization) {
+  bool isAllocatableOrPointer =
+      mlir::isa<fir::HeapType, fir::PointerType>(boxTy.getEleTy());
+  mlir::Type innerTy = fir::unwrapRefType(boxTy.getEleTy());
+  fir::IfOp ifUnallocated{nullptr};
+  if (isAllocatableOrPointer) {
+    ifUnallocated = handleNullAllocatable();
+    builder.setInsertionPointToStart(&ifUnallocated.getElseRegion().front());
+  }
 
-      builder.setInsertionPointToStart(&ifUnallocated.getElseRegion().front());
-      mlir::Value valAlloc = builder.create<fir::AllocMemOp>(loc, innerTy);
-      if (scalarInitValue)
-        builder.createStoreWithConvert(loc, scalarInitValue, valAlloc);
-      mlir::Value box = builder.create<fir::EmboxOp>(loc, ty, valAlloc);
-      builder.create<fir::StoreOp>(loc, box, boxAlloca);
+  mlir::Value valAlloc = builder.createHeapTemporary(loc, innerTy, /*name=*/{},
+                                                     /*shape=*/{}, lenParams);
+  if (scalarInitValue)
+    builder.createStoreWithConvert(loc, scalarInitValue, valAlloc);
+  mlir::Value box = builder.create<fir::EmboxOp>(
+      loc, valType, valAlloc, /*shape=*/mlir::Value{},
+      /*slice=*/mlir::Value{}, lenParams);
+  initializeIfDerivedTypeBox(
+      builder, loc, box, getLoadedMoldArg(), needsInitialization,
+      /*isFirstPrivate=*/kind == DeclOperationKind::FirstPrivate);
+  fir::StoreOp lastOp =
+      builder.create<fir::StoreOp>(loc, box, allocatedPrivVarArg);
 
-      createCleanupRegion(builder, loc, argType, cleanupRegion);
+  createCleanupRegion(converter, loc, argType, cleanupRegion, sym);
+
+  if (ifUnallocated)
+    builder.setInsertionPointAfter(ifUnallocated);
+  else
+    builder.setInsertionPointAfter(lastOp);
+
+  createYield(allocatedPrivVarArg);
+}
+
+void PopulateInitAndCleanupRegionsHelper::initAndCleanupBoxedArray(
+    fir::BaseBoxType boxTy, bool needsInitialization) {
+  bool isAllocatableOrPointer =
+      mlir::isa<fir::HeapType, fir::PointerType>(boxTy.getEleTy());
+  getLengthParameters(builder, loc, getLoadedMoldArg(), lenParams);
+
+  fir::IfOp ifUnallocated{nullptr};
+  if (isAllocatableOrPointer) {
+    ifUnallocated = handleNullAllocatable();
+    builder.setInsertionPointToStart(&ifUnallocated.getElseRegion().front());
+  }
+
+  // Create the private copy from the initial fir.box:
+  hlfir::Entity source = hlfir::Entity{getLoadedMoldArg()};
+
+  // Special case for (possibly allocatable) arrays of polymorphic types
+  // e.g. !fir.class<!fir.heap<!fir.array<?x!fir.type<>>>>
+  if (source.isPolymorphic()) {
+    fir::ShapeShiftOp shape = getShapeShift(builder, loc, source);
+    mlir::Type arrayType = source.getElementOrSequenceType();
+    mlir::Value allocatedArray = builder.create<fir::AllocMemOp>(
+        loc, arrayType, /*typeparams=*/mlir::ValueRange{}, shape.getExtents());
+    mlir::Value firClass = builder.create<fir::EmboxOp>(loc, source.getType(),
+                                                        allocatedArray, shape);
+    initializeIfDerivedTypeBox(
+        builder, loc, firClass, source, needsInitialization,
+        /*isFirstprivate=*/kind == DeclOperationKind::FirstPrivate);
+    builder.create<fir::StoreOp>(loc, firClass, allocatedPrivVarArg);
+    if (ifUnallocated)
       builder.setInsertionPointAfter(ifUnallocated);
-      yield(boxAlloca);
-      return;
-    }
-    innerTy = fir::extractSequenceType(boxTy);
-    if (!mlir::isa<fir::SequenceType>(innerTy))
-      TODO(loc, "Unsupported boxed type for reduction/privatization");
+    createYield(allocatedPrivVarArg);
+    mlir::OpBuilder::InsertionGuard guard(builder);
+    createCleanupRegion(converter, loc, argType, cleanupRegion, sym);
+    return;
+  }
+
+  // Allocating on the heap in case the whole reduction/privatization is nested
+  // inside of a loop
+  auto [temp, needsDealloc] = createTempFromMold(loc, builder, source);
+  // if needsDealloc isn't statically false, add cleanup region. Always
+  // do this for allocatable boxes because they might have been re-allocated
+  // in the body of the loop/parallel region
+
+  std::optional<int64_t> cstNeedsDealloc = fir::getIntIfConstant(needsDealloc);
+  assert(cstNeedsDealloc.has_value() &&
+         "createTempFromMold decides this statically");
+  if (cstNeedsDealloc.has_value() && *cstNeedsDealloc != false) {
+    mlir::OpBuilder::InsertionGuard guard(builder);
+    createCleanupRegion(converter, loc, argType, cleanupRegion, sym);
+  } else {
+    assert(!isAllocatableOrPointer &&
+           "Pointer-like arrays must be heap allocated");
+  }
+
+  // Put the temporary inside of a box:
+  // hlfir::genVariableBox doesn't handle non-default lower bounds
+  mlir::Value box;
+  fir::ShapeShiftOp shapeShift =
+      getShapeShift(builder, loc, getLoadedMoldArg());
+  mlir::Type boxType = getLoadedMoldArg().getType();
+  if (mlir::isa<fir::BaseBoxType>(temp.getType()))
+    // the box created by the declare form createTempFromMold is missing
+    // lower bounds info
+    box = builder.create<fir::ReboxOp>(loc, boxType, temp, shapeShift,
+                                       /*shift=*/mlir::Value{});
+  else
+    box = builder.create<fir::EmboxOp>(
+        loc, boxType, temp, shapeShift,
+        /*slice=*/mlir::Value{},
+        /*typeParams=*/llvm::ArrayRef<mlir::Value>{});
+
+  if (scalarInitValue)
+    builder.create<hlfir::AssignOp>(loc, scalarInitValue, box);
+
+  initializeIfDerivedTypeBox(
+      builder, loc, box, getLoadedMoldArg(), needsInitialization,
+      /*isFirstPrivate=*/kind == DeclOperationKind::FirstPrivate);
+
+  builder.create<fir::StoreOp>(loc, box, allocatedPrivVarArg);
+  if (ifUnallocated)
+    builder.setInsertionPointAfter(ifUnallocated);
+  createYield(allocatedPrivVarArg);
+}
+
+void PopulateInitAndCleanupRegionsHelper::initAndCleanupBoxchar(
+    fir::BoxCharType boxCharTy) {
+  mlir::Type eleTy = boxCharTy.getEleTy();
+  builder.setInsertionPointToStart(initBlock);
+  fir::factory::CharacterExprHelper charExprHelper{builder, loc};
+  auto [addr, len] = charExprHelper.createUnboxChar(moldArg);
+
+  // Using heap temporary so that
+  // 1) It is safe to use privatization inside of big loops.
+  // 2) The lifetime can outlive the current stack frame for delayed task
+  // execution.
+  // We can't always allocate a boxchar implicitly as the type of the
+  // omp.private because the allocation potentially needs the length
+  // parameters fetched above.
+  // TODO: this deviates from the intended design for delayed task
+  // execution.
+  mlir::Value privateAddr = builder.createHeapTemporary(
+      loc, eleTy, /*name=*/{}, /*shape=*/{}, /*lenParams=*/len);
+  mlir::Value boxChar = charExprHelper.createEmboxChar(privateAddr, len);
+
+  createCleanupRegion(converter, loc, argType, cleanupRegion, sym);
+
+  builder.setInsertionPointToEnd(initBlock);
+  createYield(boxChar);
+}
+
+void PopulateInitAndCleanupRegionsHelper::initAndCleanupUnboxedDerivedType(
+    bool needsInitialization) {
+  builder.setInsertionPointToStart(initBlock);
+  mlir::Type boxedTy = fir::BoxType::get(valType);
+  mlir::Value newBox =
+      builder.create<fir::EmboxOp>(loc, boxedTy, allocatedPrivVarArg);
+  mlir::Value moldBox = builder.create<fir::EmboxOp>(loc, boxedTy, moldArg);
+  initializeIfDerivedTypeBox(builder, loc, newBox, moldBox, needsInitialization,
+                             /*isFirstPrivate=*/kind ==
+                                 DeclOperationKind::FirstPrivate);
+
+  if (sym && hasFinalization(*sym))
+    createCleanupRegion(converter, loc, argType, cleanupRegion, sym);
+
+  builder.setInsertionPointToEnd(initBlock);
+  createYield(allocatedPrivVarArg);
+}
+
+/// This is the main driver deciding how to initialize the private variable.
+void PopulateInitAndCleanupRegionsHelper::populateByRefInitAndCleanupRegions() {
+  if (isPrivatization(kind)) {
+    assert(sym && "Symbol information is required to privatize derived types");
+    assert(!scalarInitValue && "ScalarInitvalue is unused for privatization");
+  }
+  mlir::Type valTy = fir::unwrapRefType(argType);
+
+  if (fir::isa_trivial(valTy)) {
+    initTrivialType();
+    return;
+  }
 
-    fir::IfOp ifUnallocated{nullptr};
-    if (isAllocatableOrPointer) {
-      ifUnallocated = handleNullAllocatable(boxAlloca);
-      builder.setInsertionPointToStart(&ifUnallocated.getElseRegion().front());
+  bool needsInitialization =
+      sym ? isDerivedTypeNeedingInitialization(sym->GetUltimate()) : false;
+
+  if (auto boxTy = mlir::dyn_cast_or_null<fir::BaseBoxType>(valTy)) {
+    builder.setInsertionPointToEnd(initBlock);
+
+    // TODO: don't do this unless it is needed
+    getLengthParameters(builder, loc, getLoadedMoldArg(), lenParams);
+
+    if (isPrivatization(kind) &&
+        mlir::isa<fir::PointerType>(boxTy.getEleTy())) {
+      initBoxedPrivatePointer(boxTy);
+      return;
     }
 
-    // Create the private copy from the initial fir.box:
-    mlir::Value loadedBox = builder.loadIfRef(loc, moldArg);
-    hlfir::Entity source = hlfir::Entity{loadedBox};
-
-    // Allocating on the heap in case the whole reduction is nested inside of a
-    // loop
-    // TODO: compare performance here to using allocas - this could be made to
-    // work by inserting stacksave/stackrestore around the reduction in
-    // openmpirbuilder
-    auto [temp, needsDealloc] = createTempFromMold(loc, builder, source);
-    // if needsDealloc isn't statically false, add cleanup region. Always
-    // do this for allocatable boxes because they might have been re-allocated
-    // in the body of the loop/parallel region
-
-    std::optional<int64_t> cstNeedsDealloc =
-        fir::getIntIfConstant(needsDealloc);
-    assert(cstNeedsDealloc.has_value() &&
-           "createTempFromMold decides this statically");
-    if (cstNeedsDealloc.has_value() && *cstNeedsDealloc != false) {
-      mlir::OpBuilder::InsertionGuard guard(builder);
-      createCleanupRegion(builder, loc, argType, cleanupRegion);
-    } else {
-      assert(!isAllocatableOrPointer &&
-             "Pointer-like arrays must be heap allocated");
+    mlir::Type innerTy = fir::unwrapRefType(boxTy.getEleTy());
+    bool isDerived = fir::isa_derived(innerTy);
+    bool isChar = fir::isa_char(innerTy);
+    if (fir::isa_trivial(innerTy) || isDerived || isChar) {
+      // boxed non-sequence value e.g. !fir.box<!fir.heap<i32>>
+      if ((isDerived || isChar) && (isReduction(kind) || scalarInitValue))
+        TODO(loc, "Reduction of an unsupported boxed type");
+      initAndCleanupBoxedScalar(boxTy, needsInitialization);
+      return;
     }
 
-    // Put the temporary inside of a box:
-    // hlfir::genVariableBox doesn't handle non-default lower bounds
-    mlir::Value box;
-    fir::ShapeShiftOp shapeShift = getShapeShift(builder, loc, loadedBox);
-    mlir::Type boxType = loadedBox.getType();
-    if (mlir::isa<fir::BaseBoxType>(temp.getType()))
-      // the box created by the declare form createTempFromMold is missing lower
-      // bounds info
-      box = builder.create<fir::ReboxOp>(loc, boxType, temp, shapeShift,
-                                         /*shift=*/mlir::Value{});
-    else
-      box = builder.create<fir::EmboxOp>(
-          loc, boxType, temp, shapeShift,
-          /*slice=*/mlir::Value{},
-          /*typeParams=*/llvm::ArrayRef<mlir::Value>{});
+    innerTy = fir::extractSequenceType(boxTy);
+    if (!innerTy || !mlir::isa<fir::SequenceType>(innerTy))
+      TODO(loc, "Unsupported boxed type for reduction/privatization");
+    initAndCleanupBoxedArray(boxTy, needsInitialization);
+    return;
+  }
 
-    if (scalarInitValue)
-      builder.create<hlfir::AssignOp>(loc, scalarInitValue, box);
-    builder.create<fir::StoreOp>(loc, box, boxAlloca);
-    if (ifUnallocated)
-      builder.setInsertionPointAfter(ifUnallocated);
-    yield(boxAlloca);
+  // Unboxed types:
+  if (auto boxCharTy = mlir::dyn_cast<fir::BoxCharType>(argType)) {
+    initAndCleanupBoxchar(boxCharTy);
+    return;
+  }
+  if (fir::isa_derived(valType)) {
+    initAndCleanupUnboxedDerivedType(needsInitialization);
     return;
   }
 
   TODO(loc,
        "creating reduction/privatization init region for unsupported type");
-  return;
+}
+
+void Fortran::lower::omp::populateByRefInitAndCleanupRegions(
+    Fortran::lower::AbstractConverter &converter, mlir::Location loc,
+    mlir::Type argType, mlir::Value scalarInitValue, mlir::Block *initBlock,
+    mlir::Value allocatedPrivVarArg, mlir::Value moldArg,
+    mlir::Region &cleanupRegion, DeclOperationKind kind,
+    const Fortran::semantics::Symbol *sym) {
+  PopulateInitAndCleanupRegionsHelper helper(
+      converter, loc, argType, scalarInitValue, allocatedPrivVarArg, moldArg,
+      initBlock, cleanupRegion, kind, sym);
+  helper.populateByRefInitAndCleanupRegions();
+
+  // Often we load moldArg to check something (e.g. length parameters, shape)
+  // but then those answers can be gotten statically without accessing the
+  // runtime value and so the only remaining use is a dead load. These loads can
+  // force us to insert additional barriers and so should be avoided where
+  // possible.
+  if (moldArg.hasOneUse()) {
+    mlir::Operation *user = *moldArg.getUsers().begin();
+    if (auto load = mlir::dyn_cast<fir::LoadOp>(user))
+      if (load.use_empty())
+        load.erase();
+  }
 }
diff --git a/flang/lib/Lower/OpenMP/PrivateReductionUtils.h b/flang/lib/Lower/OpenMP/PrivateReductionUtils.h
index b4abc40cd4b67..fcd36392a29e0 100644
--- a/flang/lib/Lower/OpenMP/PrivateReductionUtils.h
+++ b/flang/lib/Lower/OpenMP/PrivateReductionUtils.h
@@ -20,6 +20,12 @@ namespace mlir {
 class Region;
 } // namespace mlir
 
+namespace Fortran {
+namespace semantics {
+class Symbol;
+} // namespace semantics
+} // namespace Fortran
+
 namespace fir {
 class FirOpBuilder;
 class ShapeShiftOp;
@@ -27,18 +33,29 @@ class ShapeShiftOp;
 
 namespace Fortran {
 namespace lower {
+class AbstractConverter;
+
 namespace omp {
 
+enum class DeclOperationKind { Private, FirstPrivate, Reduction };
+inline bool isPrivatization(DeclOperationKind kind) {
+  return (kind == DeclOperationKind::FirstPrivate) ||
+         (kind == DeclOperationKind::Private);
+}
+inline bool isReduction(DeclOperationKind kind) {
+  return kind == DeclOperationKind::Reduction;
+}
+
 /// Generate init and cleanup regions suitable for reduction or privatizer
 /// declarations. `scalarInitValue` may be nullptr if there is no default
-/// initialization (for privatization).
-void populateByRefInitAndCleanupRegions(fir::FirOpBuilder &builder,
-                                        mlir::Location loc, mlir::Type argType,
-                                        mlir::Value scalarInitValue,
-                                        mlir::Block *initBlock,
-                                        mlir::Value allocatedPrivVarArg,
-                                        mlir::Value moldArg,
-                                        mlir::Region &cleanupRegion);
+/// initialization (for privatization). `kind` should be set to indicate
+/// what kind of operation definition this initialization belongs to.
+void populateByRefInitAndCleanupRegions(
+    AbstractConverter &converter, mlir::Location loc, mlir::Type argType,
+    mlir::Value scalarInitValue, mlir::Block *initBlock,
+    mlir::Value allocatedPrivVarArg, mlir::Value moldArg,
+    mlir::Region &cleanupRegion, DeclOperationKind kind,
+    const Fortran::semantics::Symbol *sym = nullptr);
 
 /// Generate a fir::ShapeShift op describing the provided boxed array.
 fir::ShapeShiftOp getShapeShift(fir::FirOpBuilder &builder, mlir::Location loc,
diff --git a/flang/lib/Lower/OpenMP/ReductionProcessor.cpp b/flang/lib/Lower/OpenMP/ReductionProcessor.cpp
index 2cd21107a916e..f83079eb68688 100644
--- a/flang/lib/Lower/OpenMP/ReductionProcessor.cpp
+++ b/flang/lib/Lower/OpenMP/ReductionProcessor.cpp
@@ -31,6 +31,9 @@ static llvm::cl::opt<bool> forceByrefReduction(
     llvm::cl::desc("Pass all reduction arguments by reference"),
     llvm::cl::Hidden);
 
+using ReductionModifier =
+    Fortran::lower::omp::clause::Reduction::ReductionModifier;
+
 namespace Fortran {
 namespace lower {
 namespace omp {
@@ -410,10 +413,11 @@ static mlir::Type unwrapSeqOrBoxedType(mlir::Type ty) {
 }
 
 static void createReductionAllocAndInitRegions(
-    fir::FirOpBuilder &builder, mlir::Location loc,
+    AbstractConverter &converter, mlir::Location loc,
     mlir::omp::DeclareReductionOp &reductionDecl,
     const ReductionProcessor::ReductionIdentifier redId, mlir::Type type,
     bool isByRef) {
+  fir::FirOpBuilder &builder = converter.getFirOpBuilder();
   auto yield = [&](mlir::Value ret) {
     builder.create<mlir::omp::YieldOp>(loc, ret);
   };
@@ -439,10 +443,11 @@ static void createReductionAllocAndInitRegions(
       loc, unwrapSeqOrBoxedType(ty), redId, builder);
 
   if (isByRef) {
-    populateByRefInitAndCleanupRegions(builder, loc, type, initValue, initBlock,
-                                       reductionDecl.getInitializerAllocArg(),
-                                       reductionDecl.getInitializerMoldArg(),
-                                       reductionDecl.getCleanupRegion());
+    populateByRefInitAndCleanupRegions(
+        converter, loc, type, initValue, initBlock,
+        reductionDecl.getInitializerAllocArg(),
+        reductionDecl.getInitializerMoldArg(), reductionDecl.getCleanupRegion(),
+        DeclOperationKind::Reduction);
   }
 
   if (fir::isa_trivial(ty)) {
@@ -466,9 +471,10 @@ static void createReductionAllocAndInitRegions(
 }
 
 mlir::omp::DeclareReductionOp ReductionProcessor::createDeclareReduction(
-    fir::FirOpBuilder &builder, llvm::StringRef reductionOpName,
+    AbstractConverter &converter, llvm::StringRef reductionOpName,
     const ReductionIdentifier redId, mlir::Type type, mlir::Location loc,
     bool isByRef) {
+  fir::FirOpBuilder &builder = converter.getFirOpBuilder();
   mlir::OpBuilder::InsertionGuard guard(builder);
   mlir::ModuleOp module = builder.getModule();
 
@@ -486,7 +492,8 @@ mlir::omp::DeclareReductionOp ReductionProcessor::createDeclareReduction(
 
   decl = modBuilder.create<mlir::omp::DeclareReductionOp>(loc, reductionOpName,
                                                           type);
-  createReductionAllocAndInitRegions(builder, loc, decl, redId, type, isByRef);
+  createReductionAllocAndInitRegions(converter, loc, decl, redId, type,
+                                     isByRef);
 
   builder.createBlock(&decl.getReductionRegion(),
                       decl.getReductionRegion().end(), {type, type},
@@ -514,18 +521,36 @@ static bool doReductionByRef(mlir::Value reductionVar) {
   return false;
 }
 
-void ReductionProcessor::addDeclareReduction(
+mlir::omp::ReductionModifier translateReductionModifier(ReductionModifier mod) {
+  switch (mod) {
+  case ReductionModifier::Default:
+    return mlir::omp::ReductionModifier::defaultmod;
+  case ReductionModifier::Inscan:
+    return mlir::omp::ReductionModifier::inscan;
+  case ReductionModifier::Task:
+    return mlir::omp::ReductionModifier::task;
+  }
+  return mlir::omp::ReductionModifier::defaultmod;
+}
+
+void ReductionProcessor::processReductionArguments(
     mlir::Location currentLocation, lower::AbstractConverter &converter,
     const omp::clause::Reduction &reduction,
     llvm::SmallVectorImpl<mlir::Value> &reductionVars,
     llvm::SmallVectorImpl<bool> &reduceVarByRef,
     llvm::SmallVectorImpl<mlir::Attribute> &reductionDeclSymbols,
-    llvm::SmallVectorImpl<const semantics::Symbol *> &reductionSymbols) {
+    llvm::SmallVectorImpl<const semantics::Symbol *> &reductionSymbols,
+    mlir::omp::ReductionModifierAttr &reductionMod) {
   fir::FirOpBuilder &firOpBuilder = converter.getFirOpBuilder();
 
-  if (std::get<std::optional<omp::clause::Reduction::ReductionModifier>>(
-          reduction.t))
-    TODO(currentLocation, "Reduction modifiers are not supported");
+  auto mod = std::get<std::optional<ReductionModifier>>(reduction.t);
+  if (mod.has_value()) {
+    if (mod.value() == ReductionModifier::Task)
+      TODO(currentLocation, "Reduction modifier `task` is not supported");
+    else
+      reductionMod = mlir::omp::ReductionModifierAttr::get(
+          firOpBuilder.getContext(), translateReductionModifier(mod.value()));
+  }
 
   mlir::omp::DeclareReductionOp decl;
   const auto &redOperatorList{
@@ -645,7 +670,7 @@ void ReductionProcessor::addDeclareReduction(
       TODO(currentLocation, "Unexpected reduction type");
     }
 
-    decl = createDeclareReduction(firOpBuilder, reductionName, redId, redType,
+    decl = createDeclareReduction(converter, reductionName, redId, redType,
                                   currentLocation, isByRef);
     reductionDeclSymbols.push_back(
         mlir::SymbolRefAttr::get(firOpBuilder.getContext(), decl.getSymName()));
diff --git a/flang/lib/Lower/OpenMP/ReductionProcessor.h b/flang/lib/Lower/OpenMP/ReductionProcessor.h
index 5f4d742b62cb1..11baa839c74b4 100644
--- a/flang/lib/Lower/OpenMP/ReductionProcessor.h
+++ b/flang/lib/Lower/OpenMP/ReductionProcessor.h
@@ -19,6 +19,7 @@
 #include "flang/Parser/parse-tree.h"
 #include "flang/Semantics/symbol.h"
 #include "flang/Semantics/type.h"
+#include "mlir/Dialect/OpenMP/OpenMPDialect.h"
 #include "mlir/IR/Location.h"
 #include "mlir/IR/Types.h"
 
@@ -113,20 +114,21 @@ class ReductionProcessor {
   /// value `initValue`, and the reduction combiner carried over from `reduce`.
   /// TODO: add atomic region.
   static mlir::omp::DeclareReductionOp
-  createDeclareReduction(fir::FirOpBuilder &builder,
+  createDeclareReduction(AbstractConverter &builder,
                          llvm::StringRef reductionOpName,
                          const ReductionIdentifier redId, mlir::Type type,
                          mlir::Location loc, bool isByRef);
 
   /// Creates a reduction declaration and associates it with an OpenMP block
   /// directive.
-  static void addDeclareReduction(
+  static void processReductionArguments(
       mlir::Location currentLocation, lower::AbstractConverter &converter,
       const omp::clause::Reduction &reduction,
       llvm::SmallVectorImpl<mlir::Value> &reductionVars,
       llvm::SmallVectorImpl<bool> &reduceVarByRef,
       llvm::SmallVectorImpl<mlir::Attribute> &reductionDeclSymbols,
-      llvm::SmallVectorImpl<const semantics::Symbol *> &reductionSymbols);
+      llvm::SmallVectorImpl<const semantics::Symbol *> &reductionSymbols,
+      mlir::omp::ReductionModifierAttr &reductionMod);
 };
 
 template <typename FloatOp, typename IntegerOp>
diff --git a/flang/lib/Lower/Support/Utils.cpp b/flang/lib/Lower/Support/Utils.cpp
new file mode 100644
index 0000000000000..5a9a839330364
--- /dev/null
+++ b/flang/lib/Lower/Support/Utils.cpp
@@ -0,0 +1,605 @@
+//===-- Lower/Support/Utils.cpp -- utilities --------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Coding style: https://mlir.llvm.org/getting_started/DeveloperGuide/
+//
+//===----------------------------------------------------------------------===//
+
+#include "flang/Lower/Support/Utils.h"
+
+#include "flang/Common/indirection.h"
+#include "flang/Lower/IterationSpace.h"
+#include "flang/Semantics/tools.h"
+#include <cstdint>
+#include <optional>
+#include <type_traits>
+
+namespace Fortran::lower {
+// Fortran::evaluate::Expr are functional values organized like an AST. A
+// Fortran::evaluate::Expr is meant to be moved and cloned. Using the front end
+// tools can often cause copies and extra wrapper classes to be added to any
+// Fortran::evaluate::Expr. These values should not be assumed or relied upon to
+// have an *object* identity. They are deeply recursive, irregular structures
+// built from a large number of classes which do not use inheritance and
+// necessitate a large volume of boilerplate code as a result.
+//
+// Contrastingly, LLVM data structures make ubiquitous assumptions about an
+// object's identity via pointers to the object. An object's location in memory
+// is thus very often an identifying relation.
+
+// This class defines a hash computation of a Fortran::evaluate::Expr tree value
+// so it can be used with llvm::DenseMap. The Fortran::evaluate::Expr need not
+// have the same address.
+class HashEvaluateExpr {
+public:
+  // A Se::Symbol is the only part of an Fortran::evaluate::Expr with an
+  // identity property.
+  static unsigned getHashValue(const Fortran::semantics::Symbol &x) {
+    return static_cast<unsigned>(reinterpret_cast<std::intptr_t>(&x));
+  }
+  template <typename A, bool COPY>
+  static unsigned getHashValue(const Fortran::common::Indirection<A, COPY> &x) {
+    return getHashValue(x.value());
+  }
+  template <typename A>
+  static unsigned getHashValue(const std::optional<A> &x) {
+    if (x.has_value())
+      return getHashValue(x.value());
+    return 0u;
+  }
+  static unsigned getHashValue(const Fortran::evaluate::Subscript &x) {
+    return Fortran::common::visit(
+        [&](const auto &v) { return getHashValue(v); }, x.u);
+  }
+  static unsigned getHashValue(const Fortran::evaluate::Triplet &x) {
+    return getHashValue(x.lower()) - getHashValue(x.upper()) * 5u -
+           getHashValue(x.stride()) * 11u;
+  }
+  static unsigned getHashValue(const Fortran::evaluate::Component &x) {
+    return getHashValue(x.base()) * 83u - getHashValue(x.GetLastSymbol());
+  }
+  static unsigned getHashValue(const Fortran::evaluate::ArrayRef &x) {
+    unsigned subs = 1u;
+    for (const Fortran::evaluate::Subscript &v : x.subscript())
+      subs -= getHashValue(v);
+    return getHashValue(x.base()) * 89u - subs;
+  }
+  static unsigned getHashValue(const Fortran::evaluate::CoarrayRef &x) {
+    unsigned subs = 1u;
+    for (const Fortran::evaluate::Subscript &v : x.subscript())
+      subs -= getHashValue(v);
+    unsigned cosubs = 3u;
+    for (const Fortran::evaluate::Expr<Fortran::evaluate::SubscriptInteger> &v :
+         x.cosubscript())
+      cosubs -= getHashValue(v);
+    unsigned syms = 7u;
+    for (const Fortran::evaluate::SymbolRef &v : x.base())
+      syms += getHashValue(v);
+    return syms * 97u - subs - cosubs + getHashValue(x.stat()) + 257u +
+           getHashValue(x.team());
+  }
+  static unsigned getHashValue(const Fortran::evaluate::NamedEntity &x) {
+    if (x.IsSymbol())
+      return getHashValue(x.GetFirstSymbol()) * 11u;
+    return getHashValue(x.GetComponent()) * 13u;
+  }
+  static unsigned getHashValue(const Fortran::evaluate::DataRef &x) {
+    return Fortran::common::visit(
+        [&](const auto &v) { return getHashValue(v); }, x.u);
+  }
+  static unsigned getHashValue(const Fortran::evaluate::ComplexPart &x) {
+    return getHashValue(x.complex()) - static_cast<unsigned>(x.part());
+  }
+  template <Fortran::common::TypeCategory TC1, int KIND,
+            Fortran::common::TypeCategory TC2>
+  static unsigned getHashValue(
+      const Fortran::evaluate::Convert<Fortran::evaluate::Type<TC1, KIND>, TC2>
+          &x) {
+    return getHashValue(x.left()) - (static_cast<unsigned>(TC1) + 2u) -
+           (static_cast<unsigned>(KIND) + 5u);
+  }
+  template <int KIND>
+  static unsigned
+  getHashValue(const Fortran::evaluate::ComplexComponent<KIND> &x) {
+    return getHashValue(x.left()) -
+           (static_cast<unsigned>(x.isImaginaryPart) + 1u) * 3u;
+  }
+  template <typename T>
+  static unsigned getHashValue(const Fortran::evaluate::Parentheses<T> &x) {
+    return getHashValue(x.left()) * 17u;
+  }
+  template <Fortran::common::TypeCategory TC, int KIND>
+  static unsigned getHashValue(
+      const Fortran::evaluate::Negate<Fortran::evaluate::Type<TC, KIND>> &x) {
+    return getHashValue(x.left()) - (static_cast<unsigned>(TC) + 5u) -
+           (static_cast<unsigned>(KIND) + 7u);
+  }
+  template <Fortran::common::TypeCategory TC, int KIND>
+  static unsigned getHashValue(
+      const Fortran::evaluate::Add<Fortran::evaluate::Type<TC, KIND>> &x) {
+    return (getHashValue(x.left()) + getHashValue(x.right())) * 23u +
+           static_cast<unsigned>(TC) + static_cast<unsigned>(KIND);
+  }
+  template <Fortran::common::TypeCategory TC, int KIND>
+  static unsigned getHashValue(
+      const Fortran::evaluate::Subtract<Fortran::evaluate::Type<TC, KIND>> &x) {
+    return (getHashValue(x.left()) - getHashValue(x.right())) * 19u +
+           static_cast<unsigned>(TC) + static_cast<unsigned>(KIND);
+  }
+  template <Fortran::common::TypeCategory TC, int KIND>
+  static unsigned getHashValue(
+      const Fortran::evaluate::Multiply<Fortran::evaluate::Type<TC, KIND>> &x) {
+    return (getHashValue(x.left()) + getHashValue(x.right())) * 29u +
+           static_cast<unsigned>(TC) + static_cast<unsigned>(KIND);
+  }
+  template <Fortran::common::TypeCategory TC, int KIND>
+  static unsigned getHashValue(
+      const Fortran::evaluate::Divide<Fortran::evaluate::Type<TC, KIND>> &x) {
+    return (getHashValue(x.left()) - getHashValue(x.right())) * 31u +
+           static_cast<unsigned>(TC) + static_cast<unsigned>(KIND);
+  }
+  template <Fortran::common::TypeCategory TC, int KIND>
+  static unsigned getHashValue(
+      const Fortran::evaluate::Power<Fortran::evaluate::Type<TC, KIND>> &x) {
+    return (getHashValue(x.left()) - getHashValue(x.right())) * 37u +
+           static_cast<unsigned>(TC) + static_cast<unsigned>(KIND);
+  }
+  template <Fortran::common::TypeCategory TC, int KIND>
+  static unsigned getHashValue(
+      const Fortran::evaluate::Extremum<Fortran::evaluate::Type<TC, KIND>> &x) {
+    return (getHashValue(x.left()) + getHashValue(x.right())) * 41u +
+           static_cast<unsigned>(TC) + static_cast<unsigned>(KIND) +
+           static_cast<unsigned>(x.ordering) * 7u;
+  }
+  template <Fortran::common::TypeCategory TC, int KIND>
+  static unsigned getHashValue(
+      const Fortran::evaluate::RealToIntPower<Fortran::evaluate::Type<TC, KIND>>
+          &x) {
+    return (getHashValue(x.left()) - getHashValue(x.right())) * 43u +
+           static_cast<unsigned>(TC) + static_cast<unsigned>(KIND);
+  }
+  template <int KIND>
+  static unsigned
+  getHashValue(const Fortran::evaluate::ComplexConstructor<KIND> &x) {
+    return (getHashValue(x.left()) - getHashValue(x.right())) * 47u +
+           static_cast<unsigned>(KIND);
+  }
+  template <int KIND>
+  static unsigned getHashValue(const Fortran::evaluate::Concat<KIND> &x) {
+    return (getHashValue(x.left()) - getHashValue(x.right())) * 53u +
+           static_cast<unsigned>(KIND);
+  }
+  template <int KIND>
+  static unsigned getHashValue(const Fortran::evaluate::SetLength<KIND> &x) {
+    return (getHashValue(x.left()) - getHashValue(x.right())) * 59u +
+           static_cast<unsigned>(KIND);
+  }
+  static unsigned getHashValue(const Fortran::semantics::SymbolRef &sym) {
+    return getHashValue(sym.get());
+  }
+  static unsigned getHashValue(const Fortran::evaluate::Substring &x) {
+    return 61u *
+               Fortran::common::visit(
+                   [&](const auto &p) { return getHashValue(p); }, x.parent()) -
+           getHashValue(x.lower()) - (getHashValue(x.lower()) + 1u);
+  }
+  static unsigned
+  getHashValue(const Fortran::evaluate::StaticDataObject::Pointer &x) {
+    return llvm::hash_value(x->name());
+  }
+  static unsigned getHashValue(const Fortran::evaluate::SpecificIntrinsic &x) {
+    return llvm::hash_value(x.name);
+  }
+  template <typename A>
+  static unsigned getHashValue(const Fortran::evaluate::Constant<A> &x) {
+    // FIXME: Should hash the content.
+    return 103u;
+  }
+  static unsigned getHashValue(const Fortran::evaluate::ActualArgument &x) {
+    if (const Fortran::evaluate::Symbol *sym = x.GetAssumedTypeDummy())
+      return getHashValue(*sym);
+    return getHashValue(*x.UnwrapExpr());
+  }
+  static unsigned
+  getHashValue(const Fortran::evaluate::ProcedureDesignator &x) {
+    return Fortran::common::visit(
+        [&](const auto &v) { return getHashValue(v); }, x.u);
+  }
+  static unsigned getHashValue(const Fortran::evaluate::ProcedureRef &x) {
+    unsigned args = 13u;
+    for (const std::optional<Fortran::evaluate::ActualArgument> &v :
+         x.arguments())
+      args -= getHashValue(v);
+    return getHashValue(x.proc()) * 101u - args;
+  }
+  template <typename A>
+  static unsigned
+  getHashValue(const Fortran::evaluate::ArrayConstructor<A> &x) {
+    // FIXME: hash the contents.
+    return 127u;
+  }
+  static unsigned getHashValue(const Fortran::evaluate::ImpliedDoIndex &x) {
+    return llvm::hash_value(toStringRef(x.name).str()) * 131u;
+  }
+  static unsigned getHashValue(const Fortran::evaluate::TypeParamInquiry &x) {
+    return getHashValue(x.base()) * 137u - getHashValue(x.parameter()) * 3u;
+  }
+  static unsigned getHashValue(const Fortran::evaluate::DescriptorInquiry &x) {
+    return getHashValue(x.base()) * 139u -
+           static_cast<unsigned>(x.field()) * 13u +
+           static_cast<unsigned>(x.dimension());
+  }
+  static unsigned
+  getHashValue(const Fortran::evaluate::StructureConstructor &x) {
+    // FIXME: hash the contents.
+    return 149u;
+  }
+  template <int KIND>
+  static unsigned getHashValue(const Fortran::evaluate::Not<KIND> &x) {
+    return getHashValue(x.left()) * 61u + static_cast<unsigned>(KIND);
+  }
+  template <int KIND>
+  static unsigned
+  getHashValue(const Fortran::evaluate::LogicalOperation<KIND> &x) {
+    unsigned result = getHashValue(x.left()) + getHashValue(x.right());
+    return result * 67u + static_cast<unsigned>(x.logicalOperator) * 5u;
+  }
+  template <Fortran::common::TypeCategory TC, int KIND>
+  static unsigned getHashValue(
+      const Fortran::evaluate::Relational<Fortran::evaluate::Type<TC, KIND>>
+          &x) {
+    return (getHashValue(x.left()) + getHashValue(x.right())) * 71u +
+           static_cast<unsigned>(TC) + static_cast<unsigned>(KIND) +
+           static_cast<unsigned>(x.opr) * 11u;
+  }
+  template <typename A>
+  static unsigned getHashValue(const Fortran::evaluate::Expr<A> &x) {
+    return Fortran::common::visit(
+        [&](const auto &v) { return getHashValue(v); }, x.u);
+  }
+  static unsigned getHashValue(
+      const Fortran::evaluate::Relational<Fortran::evaluate::SomeType> &x) {
+    return Fortran::common::visit(
+        [&](const auto &v) { return getHashValue(v); }, x.u);
+  }
+  template <typename A>
+  static unsigned getHashValue(const Fortran::evaluate::Designator<A> &x) {
+    return Fortran::common::visit(
+        [&](const auto &v) { return getHashValue(v); }, x.u);
+  }
+  template <int BITS>
+  static unsigned
+  getHashValue(const Fortran::evaluate::value::Integer<BITS> &x) {
+    return static_cast<unsigned>(x.ToSInt());
+  }
+  static unsigned getHashValue(const Fortran::evaluate::NullPointer &x) {
+    return ~179u;
+  }
+};
+
+// Define the is equals test for using Fortran::evaluate::Expr values with
+// llvm::DenseMap.
+class IsEqualEvaluateExpr {
+public:
+  // A Se::Symbol is the only part of an Fortran::evaluate::Expr with an
+  // identity property.
+  static bool isEqual(const Fortran::semantics::Symbol &x,
+                      const Fortran::semantics::Symbol &y) {
+    return isEqual(&x, &y);
+  }
+  static bool isEqual(const Fortran::semantics::Symbol *x,
+                      const Fortran::semantics::Symbol *y) {
+    return x == y;
+  }
+  template <typename A, bool COPY>
+  static bool isEqual(const Fortran::common::Indirection<A, COPY> &x,
+                      const Fortran::common::Indirection<A, COPY> &y) {
+    return isEqual(x.value(), y.value());
+  }
+  template <typename A>
+  static bool isEqual(const std::optional<A> &x, const std::optional<A> &y) {
+    if (x.has_value() && y.has_value())
+      return isEqual(x.value(), y.value());
+    return !x.has_value() && !y.has_value();
+  }
+  template <typename A>
+  static bool isEqual(const std::vector<A> &x, const std::vector<A> &y) {
+    if (x.size() != y.size())
+      return false;
+    const std::size_t size = x.size();
+    for (std::remove_const_t<decltype(size)> i = 0; i < size; ++i)
+      if (!isEqual(x[i], y[i]))
+        return false;
+    return true;
+  }
+  static bool isEqual(const Fortran::evaluate::Subscript &x,
+                      const Fortran::evaluate::Subscript &y) {
+    return Fortran::common::visit(
+        [&](const auto &v, const auto &w) { return isEqual(v, w); }, x.u, y.u);
+  }
+  static bool isEqual(const Fortran::evaluate::Triplet &x,
+                      const Fortran::evaluate::Triplet &y) {
+    return isEqual(x.lower(), y.lower()) && isEqual(x.upper(), y.upper()) &&
+           isEqual(x.stride(), y.stride());
+  }
+  static bool isEqual(const Fortran::evaluate::Component &x,
+                      const Fortran::evaluate::Component &y) {
+    return isEqual(x.base(), y.base()) &&
+           isEqual(x.GetLastSymbol(), y.GetLastSymbol());
+  }
+  static bool isEqual(const Fortran::evaluate::ArrayRef &x,
+                      const Fortran::evaluate::ArrayRef &y) {
+    return isEqual(x.base(), y.base()) && isEqual(x.subscript(), y.subscript());
+  }
+  static bool isEqual(const Fortran::evaluate::CoarrayRef &x,
+                      const Fortran::evaluate::CoarrayRef &y) {
+    return isEqual(x.base(), y.base()) &&
+           isEqual(x.subscript(), y.subscript()) &&
+           isEqual(x.cosubscript(), y.cosubscript()) &&
+           isEqual(x.stat(), y.stat()) && isEqual(x.team(), y.team());
+  }
+  static bool isEqual(const Fortran::evaluate::NamedEntity &x,
+                      const Fortran::evaluate::NamedEntity &y) {
+    if (x.IsSymbol() && y.IsSymbol())
+      return isEqual(x.GetFirstSymbol(), y.GetFirstSymbol());
+    return !x.IsSymbol() && !y.IsSymbol() &&
+           isEqual(x.GetComponent(), y.GetComponent());
+  }
+  static bool isEqual(const Fortran::evaluate::DataRef &x,
+                      const Fortran::evaluate::DataRef &y) {
+    return Fortran::common::visit(
+        [&](const auto &v, const auto &w) { return isEqual(v, w); }, x.u, y.u);
+  }
+  static bool isEqual(const Fortran::evaluate::ComplexPart &x,
+                      const Fortran::evaluate::ComplexPart &y) {
+    return isEqual(x.complex(), y.complex()) && x.part() == y.part();
+  }
+  template <typename A, Fortran::common::TypeCategory TC2>
+  static bool isEqual(const Fortran::evaluate::Convert<A, TC2> &x,
+                      const Fortran::evaluate::Convert<A, TC2> &y) {
+    return isEqual(x.left(), y.left());
+  }
+  template <int KIND>
+  static bool isEqual(const Fortran::evaluate::ComplexComponent<KIND> &x,
+                      const Fortran::evaluate::ComplexComponent<KIND> &y) {
+    return isEqual(x.left(), y.left()) &&
+           x.isImaginaryPart == y.isImaginaryPart;
+  }
+  template <typename T>
+  static bool isEqual(const Fortran::evaluate::Parentheses<T> &x,
+                      const Fortran::evaluate::Parentheses<T> &y) {
+    return isEqual(x.left(), y.left());
+  }
+  template <typename A>
+  static bool isEqual(const Fortran::evaluate::Negate<A> &x,
+                      const Fortran::evaluate::Negate<A> &y) {
+    return isEqual(x.left(), y.left());
+  }
+  template <typename A>
+  static bool isBinaryEqual(const A &x, const A &y) {
+    return isEqual(x.left(), y.left()) && isEqual(x.right(), y.right());
+  }
+  template <typename A>
+  static bool isEqual(const Fortran::evaluate::Add<A> &x,
+                      const Fortran::evaluate::Add<A> &y) {
+    return isBinaryEqual(x, y);
+  }
+  template <typename A>
+  static bool isEqual(const Fortran::evaluate::Subtract<A> &x,
+                      const Fortran::evaluate::Subtract<A> &y) {
+    return isBinaryEqual(x, y);
+  }
+  template <typename A>
+  static bool isEqual(const Fortran::evaluate::Multiply<A> &x,
+                      const Fortran::evaluate::Multiply<A> &y) {
+    return isBinaryEqual(x, y);
+  }
+  template <typename A>
+  static bool isEqual(const Fortran::evaluate::Divide<A> &x,
+                      const Fortran::evaluate::Divide<A> &y) {
+    return isBinaryEqual(x, y);
+  }
+  template <typename A>
+  static bool isEqual(const Fortran::evaluate::Power<A> &x,
+                      const Fortran::evaluate::Power<A> &y) {
+    return isBinaryEqual(x, y);
+  }
+  template <typename A>
+  static bool isEqual(const Fortran::evaluate::Extremum<A> &x,
+                      const Fortran::evaluate::Extremum<A> &y) {
+    return isBinaryEqual(x, y);
+  }
+  template <typename A>
+  static bool isEqual(const Fortran::evaluate::RealToIntPower<A> &x,
+                      const Fortran::evaluate::RealToIntPower<A> &y) {
+    return isBinaryEqual(x, y);
+  }
+  template <int KIND>
+  static bool isEqual(const Fortran::evaluate::ComplexConstructor<KIND> &x,
+                      const Fortran::evaluate::ComplexConstructor<KIND> &y) {
+    return isBinaryEqual(x, y);
+  }
+  template <int KIND>
+  static bool isEqual(const Fortran::evaluate::Concat<KIND> &x,
+                      const Fortran::evaluate::Concat<KIND> &y) {
+    return isBinaryEqual(x, y);
+  }
+  template <int KIND>
+  static bool isEqual(const Fortran::evaluate::SetLength<KIND> &x,
+                      const Fortran::evaluate::SetLength<KIND> &y) {
+    return isBinaryEqual(x, y);
+  }
+  static bool isEqual(const Fortran::semantics::SymbolRef &x,
+                      const Fortran::semantics::SymbolRef &y) {
+    return isEqual(x.get(), y.get());
+  }
+  static bool isEqual(const Fortran::evaluate::Substring &x,
+                      const Fortran::evaluate::Substring &y) {
+    return Fortran::common::visit(
+               [&](const auto &p, const auto &q) { return isEqual(p, q); },
+               x.parent(), y.parent()) &&
+           isEqual(x.lower(), y.lower()) && isEqual(x.upper(), y.upper());
+  }
+  static bool isEqual(const Fortran::evaluate::StaticDataObject::Pointer &x,
+                      const Fortran::evaluate::StaticDataObject::Pointer &y) {
+    return x->name() == y->name();
+  }
+  static bool isEqual(const Fortran::evaluate::SpecificIntrinsic &x,
+                      const Fortran::evaluate::SpecificIntrinsic &y) {
+    return x.name == y.name;
+  }
+  template <typename A>
+  static bool isEqual(const Fortran::evaluate::Constant<A> &x,
+                      const Fortran::evaluate::Constant<A> &y) {
+    return x == y;
+  }
+  static bool isEqual(const Fortran::evaluate::ActualArgument &x,
+                      const Fortran::evaluate::ActualArgument &y) {
+    if (const Fortran::evaluate::Symbol *xs = x.GetAssumedTypeDummy()) {
+      if (const Fortran::evaluate::Symbol *ys = y.GetAssumedTypeDummy())
+        return isEqual(*xs, *ys);
+      return false;
+    }
+    return !y.GetAssumedTypeDummy() &&
+           isEqual(*x.UnwrapExpr(), *y.UnwrapExpr());
+  }
+  static bool isEqual(const Fortran::evaluate::ProcedureDesignator &x,
+                      const Fortran::evaluate::ProcedureDesignator &y) {
+    return Fortran::common::visit(
+        [&](const auto &v, const auto &w) { return isEqual(v, w); }, x.u, y.u);
+  }
+  static bool isEqual(const Fortran::evaluate::ProcedureRef &x,
+                      const Fortran::evaluate::ProcedureRef &y) {
+    return isEqual(x.proc(), y.proc()) && isEqual(x.arguments(), y.arguments());
+  }
+  template <typename A>
+  static bool isEqual(const Fortran::evaluate::ArrayConstructor<A> &x,
+                      const Fortran::evaluate::ArrayConstructor<A> &y) {
+    llvm::report_fatal_error("not implemented");
+  }
+  static bool isEqual(const Fortran::evaluate::ImpliedDoIndex &x,
+                      const Fortran::evaluate::ImpliedDoIndex &y) {
+    return toStringRef(x.name) == toStringRef(y.name);
+  }
+  static bool isEqual(const Fortran::evaluate::TypeParamInquiry &x,
+                      const Fortran::evaluate::TypeParamInquiry &y) {
+    return isEqual(x.base(), y.base()) && isEqual(x.parameter(), y.parameter());
+  }
+  static bool isEqual(const Fortran::evaluate::DescriptorInquiry &x,
+                      const Fortran::evaluate::DescriptorInquiry &y) {
+    return isEqual(x.base(), y.base()) && x.field() == y.field() &&
+           x.dimension() == y.dimension();
+  }
+  static bool isEqual(const Fortran::evaluate::StructureConstructor &x,
+                      const Fortran::evaluate::StructureConstructor &y) {
+    const auto &xValues = x.values();
+    const auto &yValues = y.values();
+    if (xValues.size() != yValues.size())
+      return false;
+    if (x.derivedTypeSpec() != y.derivedTypeSpec())
+      return false;
+    for (const auto &[xSymbol, xValue] : xValues) {
+      auto yIt = yValues.find(xSymbol);
+      // This should probably never happen, since the derived type
+      // should be the same.
+      if (yIt == yValues.end())
+        return false;
+      if (!isEqual(xValue, yIt->second))
+        return false;
+    }
+    return true;
+  }
+  template <int KIND>
+  static bool isEqual(const Fortran::evaluate::Not<KIND> &x,
+                      const Fortran::evaluate::Not<KIND> &y) {
+    return isEqual(x.left(), y.left());
+  }
+  template <int KIND>
+  static bool isEqual(const Fortran::evaluate::LogicalOperation<KIND> &x,
+                      const Fortran::evaluate::LogicalOperation<KIND> &y) {
+    return isEqual(x.left(), y.left()) && isEqual(x.right(), y.right());
+  }
+  template <typename A>
+  static bool isEqual(const Fortran::evaluate::Relational<A> &x,
+                      const Fortran::evaluate::Relational<A> &y) {
+    return isEqual(x.left(), y.left()) && isEqual(x.right(), y.right());
+  }
+  template <typename A>
+  static bool isEqual(const Fortran::evaluate::Expr<A> &x,
+                      const Fortran::evaluate::Expr<A> &y) {
+    return Fortran::common::visit(
+        [&](const auto &v, const auto &w) { return isEqual(v, w); }, x.u, y.u);
+  }
+  static bool
+  isEqual(const Fortran::evaluate::Relational<Fortran::evaluate::SomeType> &x,
+          const Fortran::evaluate::Relational<Fortran::evaluate::SomeType> &y) {
+    return Fortran::common::visit(
+        [&](const auto &v, const auto &w) { return isEqual(v, w); }, x.u, y.u);
+  }
+  template <typename A>
+  static bool isEqual(const Fortran::evaluate::Designator<A> &x,
+                      const Fortran::evaluate::Designator<A> &y) {
+    return Fortran::common::visit(
+        [&](const auto &v, const auto &w) { return isEqual(v, w); }, x.u, y.u);
+  }
+  template <int BITS>
+  static bool isEqual(const Fortran::evaluate::value::Integer<BITS> &x,
+                      const Fortran::evaluate::value::Integer<BITS> &y) {
+    return x == y;
+  }
+  static bool isEqual(const Fortran::evaluate::NullPointer &x,
+                      const Fortran::evaluate::NullPointer &y) {
+    return true;
+  }
+  template <typename A, typename B,
+            std::enable_if_t<!std::is_same_v<A, B>, bool> = true>
+  static bool isEqual(const A &, const B &) {
+    return false;
+  }
+};
+
+unsigned getHashValue(const Fortran::lower::SomeExpr *x) {
+  return HashEvaluateExpr::getHashValue(*x);
+}
+
+unsigned getHashValue(const Fortran::lower::ExplicitIterSpace::ArrayBases &x) {
+  return Fortran::common::visit(
+      [&](const auto *p) { return HashEvaluateExpr::getHashValue(*p); }, x);
+}
+
+bool isEqual(const Fortran::lower::SomeExpr *x,
+             const Fortran::lower::SomeExpr *y) {
+  const auto *empty =
+      llvm::DenseMapInfo<const Fortran::lower::SomeExpr *>::getEmptyKey();
+  const auto *tombstone =
+      llvm::DenseMapInfo<const Fortran::lower::SomeExpr *>::getTombstoneKey();
+  if (x == empty || y == empty || x == tombstone || y == tombstone)
+    return x == y;
+  return x == y || IsEqualEvaluateExpr::isEqual(*x, *y);
+}
+
+bool isEqual(const Fortran::lower::ExplicitIterSpace::ArrayBases &x,
+             const Fortran::lower::ExplicitIterSpace::ArrayBases &y) {
+  return Fortran::common::visit(
+      Fortran::common::visitors{
+          // Fortran::semantics::Symbol * are the exception here. These pointers
+          // have identity; if two Symbol * values are the same (different) then
+          // they are the same (different) logical symbol.
+          [&](Fortran::lower::FrontEndSymbol p,
+              Fortran::lower::FrontEndSymbol q) { return p == q; },
+          [&](const auto *p, const auto *q) {
+            if constexpr (std::is_same_v<decltype(p), decltype(q)>) {
+              return IsEqualEvaluateExpr::isEqual(*p, *q);
+            } else {
+              // Different subtree types are never equal.
+              return false;
+            }
+          }},
+      x, y);
+}
+} // end namespace Fortran::lower
diff --git a/flang/lib/Optimizer/Analysis/AliasAnalysis.cpp b/flang/lib/Optimizer/Analysis/AliasAnalysis.cpp
index e33d8fa333e7a..5827d1c3c529e 100644
--- a/flang/lib/Optimizer/Analysis/AliasAnalysis.cpp
+++ b/flang/lib/Optimizer/Analysis/AliasAnalysis.cpp
@@ -31,8 +31,26 @@ using namespace mlir;
 // AliasAnalysis: alias
 //===----------------------------------------------------------------------===//
 
-/// Temporary function to skip through all the no op operations
-/// TODO: Generalize support of fir.load
+static fir::AliasAnalysis::Source::Attributes
+getAttrsFromVariable(fir::FortranVariableOpInterface var) {
+  fir::AliasAnalysis::Source::Attributes attrs;
+  if (var.isTarget())
+    attrs.set(fir::AliasAnalysis::Attribute::Target);
+  if (var.isPointer())
+    attrs.set(fir::AliasAnalysis::Attribute::Pointer);
+  if (var.isIntentIn())
+    attrs.set(fir::AliasAnalysis::Attribute::IntentIn);
+
+  return attrs;
+}
+
+static bool hasGlobalOpTargetAttr(mlir::Value v, fir::AddrOfOp op) {
+  auto globalOpName =
+      mlir::OperationName(fir::GlobalOp::getOperationName(), op->getContext());
+  return fir::valueHasFirAttribute(
+      v, fir::GlobalOp::getTargetAttrName(globalOpName));
+}
+
 static mlir::Value getOriginalDef(mlir::Value v) {
   mlir::Operation *defOp;
   bool breakFromLoop = false;
@@ -46,6 +64,27 @@ static mlir::Value getOriginalDef(mlir::Value v) {
   return v;
 }
 
+static bool isEvaluateInMemoryBlockArg(mlir::Value v) {
+  if (auto evalInMem = llvm::dyn_cast_or_null<hlfir::EvaluateInMemoryOp>(
+          v.getParentRegion()->getParentOp()))
+    return evalInMem.getMemory() == v;
+  return false;
+}
+
+template <typename OMPTypeOp, typename DeclTypeOp>
+static bool isPrivateArg(omp::BlockArgOpenMPOpInterface &argIface,
+                         OMPTypeOp &op, DeclTypeOp &declOp) {
+  if (!op.getPrivateSyms().has_value())
+    return false;
+  for (auto [opSym, blockArg] :
+       llvm::zip_equal(*op.getPrivateSyms(), argIface.getPrivateBlockArgs())) {
+    if (blockArg == declOp.getMemref()) {
+      return true;
+    }
+  }
+  return false;
+}
+
 namespace fir {
 
 void AliasAnalysis::Source::print(llvm::raw_ostream &os) const {
@@ -91,13 +130,6 @@ bool AliasAnalysis::Source::isDummyArgument() const {
   return false;
 }
 
-static bool isEvaluateInMemoryBlockArg(mlir::Value v) {
-  if (auto evalInMem = llvm::dyn_cast_or_null<hlfir::EvaluateInMemoryOp>(
-          v.getParentRegion()->getParentOp()))
-    return evalInMem.getMemory() == v;
-  return false;
-}
-
 bool AliasAnalysis::Source::isData() const { return origin.isData; }
 bool AliasAnalysis::Source::isBoxData() const {
   return mlir::isa<fir::BaseBoxType>(fir::unwrapRefType(valueType)) &&
@@ -491,33 +523,6 @@ ModRefResult AliasAnalysis::getModRef(mlir::Region &region,
   return result;
 }
 
-AliasAnalysis::Source::Attributes
-getAttrsFromVariable(fir::FortranVariableOpInterface var) {
-  AliasAnalysis::Source::Attributes attrs;
-  if (var.isTarget())
-    attrs.set(AliasAnalysis::Attribute::Target);
-  if (var.isPointer())
-    attrs.set(AliasAnalysis::Attribute::Pointer);
-  if (var.isIntentIn())
-    attrs.set(AliasAnalysis::Attribute::IntentIn);
-
-  return attrs;
-}
-
-template <typename OMPTypeOp, typename DeclTypeOp>
-static bool isPrivateArg(omp::BlockArgOpenMPOpInterface &argIface,
-                         OMPTypeOp &op, DeclTypeOp &declOp) {
-  if (!op.getPrivateSyms().has_value())
-    return false;
-  for (auto [opSym, blockArg] :
-       llvm::zip_equal(*op.getPrivateSyms(), argIface.getPrivateBlockArgs())) {
-    if (blockArg == declOp.getMemref()) {
-      return true;
-    }
-  }
-  return false;
-}
-
 AliasAnalysis::Source AliasAnalysis::getSource(mlir::Value v,
                                                bool getLastInstantiationPoint) {
   auto *defOp = v.getDefiningOp();
@@ -573,16 +578,6 @@ AliasAnalysis::Source AliasAnalysis::getSource(mlir::Value v,
             breakFromLoop = true;
         })
         .Case<fir::LoadOp>([&](auto op) {
-          // If the load is from a leaf source, return the leaf. Do not track
-          // through indirections otherwise.
-          // TODO: Add support to fir.alloca and fir.allocmem
-          auto def = getOriginalDef(op.getMemref());
-          if (isDummyArgument(def) ||
-              def.template getDefiningOp<fir::AddrOfOp>()) {
-            v = def;
-            defOp = v.getDefiningOp();
-            return;
-          }
           // If load is inside target and it points to mapped item,
           // continue tracking.
           Operation *loadMemrefOp = op.getMemref().getDefiningOp();
@@ -595,6 +590,40 @@ AliasAnalysis::Source AliasAnalysis::getSource(mlir::Value v,
             defOp = v.getDefiningOp();
             return;
           }
+
+          // If we are loading a box reference, but following the data,
+          // we gather the attributes of the box to populate the source
+          // and stop tracking.
+          if (auto boxTy = mlir::dyn_cast<fir::BaseBoxType>(ty);
+              boxTy && followingData) {
+
+            if (mlir::isa<fir::PointerType>(boxTy.getEleTy()))
+              attributes.set(Attribute::Pointer);
+
+            auto def = getOriginalDef(op.getMemref());
+            if (auto addrOfOp = def.template getDefiningOp<fir::AddrOfOp>()) {
+              global = addrOfOp.getSymbol();
+
+              if (hasGlobalOpTargetAttr(def, addrOfOp))
+                attributes.set(Attribute::Target);
+
+              type = SourceKind::Global;
+            }
+
+            // TODO: Add support to fir.alloca and fir.allocmem
+            // if (auto allocOp = def.template getDefiningOp<fir::AllocaOp>()) {
+            //   ...
+            // }
+
+            if (isDummyArgument(def)) {
+              defOp = nullptr;
+              v = def;
+            }
+
+            breakFromLoop = true;
+            return;
+          }
+
           // No further tracking for addresses loaded from memory for now.
           type = SourceKind::Indirect;
           breakFromLoop = true;
@@ -604,10 +633,7 @@ AliasAnalysis::Source AliasAnalysis::getSource(mlir::Value v,
           ty = v.getType();
           type = SourceKind::Global;
 
-          auto globalOpName = mlir::OperationName(
-              fir::GlobalOp::getOperationName(), defOp->getContext());
-          if (fir::valueHasFirAttribute(
-                  v, fir::GlobalOp::getTargetAttrName(globalOpName)))
+          if (hasGlobalOpTargetAttr(v, op))
             attributes.set(Attribute::Target);
 
           // TODO: Take followBoxData into account when setting the pointer
diff --git a/flang/lib/Optimizer/Analysis/CMakeLists.txt b/flang/lib/Optimizer/Analysis/CMakeLists.txt
index 6fe9c70f83765..c4dae898f8e57 100644
--- a/flang/lib/Optimizer/Analysis/CMakeLists.txt
+++ b/flang/lib/Optimizer/Analysis/CMakeLists.txt
@@ -12,6 +12,7 @@ add_flang_library(FIRAnalysis
   LINK_LIBS
   FIRBuilder
   FIRDialect
+  FIRSupport
   HLFIRDialect
 
   MLIR_LIBS
@@ -19,5 +20,4 @@ add_flang_library(FIRAnalysis
   MLIRLLVMDialect
   MLIRMathTransforms
   MLIROpenMPDialect
-  FIRSupport
 )
diff --git a/flang/lib/Optimizer/Builder/FIRBuilder.cpp b/flang/lib/Optimizer/Builder/FIRBuilder.cpp
index 64c540cfb95ae..d9779c46ae79e 100644
--- a/flang/lib/Optimizer/Builder/FIRBuilder.cpp
+++ b/flang/lib/Optimizer/Builder/FIRBuilder.cpp
@@ -1410,7 +1410,8 @@ static bool recordTypeCanBeMemCopied(fir::RecordType recordType) {
   for (auto [_, fieldType] : recordType.getTypeList()) {
     // Derived type component may have user assignment (so far, we cannot tell
     // in FIR, so assume it is always the case, TODO: get the actual info).
-    if (mlir::isa<fir::RecordType>(fir::unwrapSequenceType(fieldType)))
+    if (mlir::isa<fir::RecordType>(fir::unwrapSequenceType(fieldType)) &&
+        !fir::isa_builtin_c_devptr_type(fir::unwrapSequenceType(fieldType)))
       return false;
     // Allocatable components need deep copy.
     if (auto boxType = mlir::dyn_cast<fir::BaseBoxType>(fieldType))
@@ -1758,3 +1759,29 @@ fir::factory::deduceOptimalExtents(mlir::ValueRange extents1,
   }
   return extents;
 }
+
+llvm::SmallVector<mlir::Value> fir::factory::updateRuntimeExtentsForEmptyArrays(
+    fir::FirOpBuilder &builder, mlir::Location loc, mlir::ValueRange extents) {
+  if (extents.size() <= 1)
+    return extents;
+
+  mlir::Type i1Type = builder.getI1Type();
+  mlir::Value isEmpty = createZeroValue(builder, loc, i1Type);
+
+  llvm::SmallVector<mlir::Value, Fortran::common::maxRank> zeroes;
+  for (mlir::Value extent : extents) {
+    mlir::Type type = extent.getType();
+    mlir::Value zero = createZeroValue(builder, loc, type);
+    zeroes.push_back(zero);
+    mlir::Value isZero = builder.create<mlir::arith::CmpIOp>(
+        loc, mlir::arith::CmpIPredicate::eq, extent, zero);
+    isEmpty = builder.create<mlir::arith::OrIOp>(loc, isEmpty, isZero);
+  }
+
+  llvm::SmallVector<mlir::Value> newExtents;
+  for (auto [zero, extent] : llvm::zip_equal(zeroes, extents)) {
+    newExtents.push_back(
+        builder.create<mlir::arith::SelectOp>(loc, isEmpty, zero, extent));
+  }
+  return newExtents;
+}
diff --git a/flang/lib/Optimizer/Builder/HLFIRTools.cpp b/flang/lib/Optimizer/Builder/HLFIRTools.cpp
index f71adf123511d..8993065c2bb64 100644
--- a/flang/lib/Optimizer/Builder/HLFIRTools.cpp
+++ b/flang/lib/Optimizer/Builder/HLFIRTools.cpp
@@ -221,6 +221,25 @@ bool hlfir::Entity::mayHaveNonDefaultLowerBounds() const {
   return true;
 }
 
+mlir::Operation *traverseConverts(mlir::Operation *op) {
+  while (auto convert = llvm::dyn_cast_or_null<fir::ConvertOp>(op))
+    op = convert.getValue().getDefiningOp();
+  return op;
+}
+
+bool hlfir::Entity::mayBeOptional() const {
+  if (!isVariable())
+    return false;
+  // TODO: introduce a fir type to better identify optionals.
+  if (mlir::Operation *op = traverseConverts(getDefiningOp())) {
+    if (auto varIface = llvm::dyn_cast<fir::FortranVariableOpInterface>(op))
+      return varIface.isOptional();
+    return !llvm::isa<fir::AllocaOp, fir::AllocMemOp, fir::ReboxOp,
+                      fir::EmboxOp, fir::LoadOp>(op);
+  }
+  return true;
+}
+
 fir::FortranVariableOpInterface
 hlfir::genDeclare(mlir::Location loc, fir::FirOpBuilder &builder,
                   const fir::ExtendedValue &exv, llvm::StringRef name,
@@ -963,9 +982,69 @@ llvm::SmallVector<mlir::Value> hlfir::genLoopNestWithReductions(
   return outerLoop->getResults();
 }
 
+template <typename Lambda>
+static fir::ExtendedValue
+conditionallyEvaluate(mlir::Location loc, fir::FirOpBuilder &builder,
+                      mlir::Value condition, const Lambda &genIfTrue) {
+  mlir::OpBuilder::InsertPoint insertPt = builder.saveInsertionPoint();
+
+  // Evaluate in some region that will be moved into the actual ifOp (the actual
+  // ifOp can only be created when the result types are known).
+  auto badIfOp = builder.create<fir::IfOp>(loc, condition.getType(), condition,
+                                           /*withElseRegion=*/false);
+  mlir::Block *preparationBlock = &badIfOp.getThenRegion().front();
+  builder.setInsertionPointToStart(preparationBlock);
+  fir::ExtendedValue result = genIfTrue();
+  fir::ResultOp resultOp = result.match(
+      [&](const fir::CharBoxValue &box) -> fir::ResultOp {
+        return builder.create<fir::ResultOp>(
+            loc, mlir::ValueRange{box.getAddr(), box.getLen()});
+      },
+      [&](const mlir::Value &addr) -> fir::ResultOp {
+        return builder.create<fir::ResultOp>(loc, addr);
+      },
+      [&](const auto &) -> fir::ResultOp {
+        TODO(loc, "unboxing non scalar optional fir.box");
+      });
+  builder.restoreInsertionPoint(insertPt);
+
+  // Create actual fir.if operation.
+  auto ifOp =
+      builder.create<fir::IfOp>(loc, resultOp->getOperandTypes(), condition,
+                                /*withElseRegion=*/true);
+  // Move evaluation into Then block,
+  preparationBlock->moveBefore(&ifOp.getThenRegion().back());
+  ifOp.getThenRegion().back().erase();
+  // Create absent result in the Else block.
+  builder.setInsertionPointToStart(&ifOp.getElseRegion().front());
+  llvm::SmallVector<mlir::Value> absentValues;
+  for (mlir::Type resTy : ifOp->getResultTypes()) {
+    if (fir::isa_ref_type(resTy) || fir::isa_box_type(resTy))
+      absentValues.emplace_back(builder.create<fir::AbsentOp>(loc, resTy));
+    else
+      absentValues.emplace_back(builder.create<fir::ZeroOp>(loc, resTy));
+  }
+  builder.create<fir::ResultOp>(loc, absentValues);
+  badIfOp->erase();
+
+  // Build fir::ExtendedValue from the result values.
+  builder.setInsertionPointAfter(ifOp);
+  return result.match(
+      [&](const fir::CharBoxValue &box) -> fir::ExtendedValue {
+        return fir::CharBoxValue{ifOp.getResult(0), ifOp.getResult(1)};
+      },
+      [&](const mlir::Value &) -> fir::ExtendedValue {
+        return ifOp.getResult(0);
+      },
+      [&](const auto &) -> fir::ExtendedValue {
+        TODO(loc, "unboxing non scalar optional fir.box");
+      });
+}
+
 static fir::ExtendedValue translateVariableToExtendedValue(
     mlir::Location loc, fir::FirOpBuilder &builder, hlfir::Entity variable,
-    bool forceHlfirBase = false, bool contiguousHint = false) {
+    bool forceHlfirBase = false, bool contiguousHint = false,
+    bool keepScalarOptionalBoxed = false) {
   assert(variable.isVariable() && "must be a variable");
   // When going towards FIR, use the original base value to avoid
   // introducing descriptors at runtime when they are not required.
@@ -984,14 +1063,33 @@ static fir::ExtendedValue translateVariableToExtendedValue(
     const bool contiguous = variable.isSimplyContiguous() || contiguousHint;
     const bool isAssumedRank = variable.isAssumedRank();
     if (!contiguous || variable.isPolymorphic() ||
-        variable.isDerivedWithLengthParameters() || variable.isOptional() ||
-        isAssumedRank) {
+        variable.isDerivedWithLengthParameters() || isAssumedRank) {
       llvm::SmallVector<mlir::Value> nonDefaultLbounds;
       if (!isAssumedRank)
         nonDefaultLbounds = getNonDefaultLowerBounds(loc, builder, variable);
       return fir::BoxValue(base, nonDefaultLbounds,
                            getExplicitTypeParams(variable));
     }
+    if (variable.mayBeOptional()) {
+      if (!keepScalarOptionalBoxed && variable.isScalar()) {
+        mlir::Value isPresent = builder.create<fir::IsPresentOp>(
+            loc, builder.getI1Type(), variable);
+        return conditionallyEvaluate(
+            loc, builder, isPresent, [&]() -> fir::ExtendedValue {
+              mlir::Value base = genVariableRawAddress(loc, builder, variable);
+              if (variable.isCharacter()) {
+                mlir::Value len =
+                    genCharacterVariableLength(loc, builder, variable);
+                return fir::CharBoxValue{base, len};
+              }
+              return base;
+            });
+      }
+      llvm::SmallVector<mlir::Value> nonDefaultLbounds =
+          getNonDefaultLowerBounds(loc, builder, variable);
+      return fir::BoxValue(base, nonDefaultLbounds,
+                           getExplicitTypeParams(variable));
+    }
     // Otherwise, the variable can be represented in a fir::ExtendedValue
     // without the overhead of a fir.box.
     base = genVariableRawAddress(loc, builder, variable);
@@ -1035,10 +1133,12 @@ hlfir::translateToExtendedValue(mlir::Location loc, fir::FirOpBuilder &builder,
 
 std::pair<fir::ExtendedValue, std::optional<hlfir::CleanupFunction>>
 hlfir::translateToExtendedValue(mlir::Location loc, fir::FirOpBuilder &builder,
-                                hlfir::Entity entity, bool contiguousHint) {
+                                hlfir::Entity entity, bool contiguousHint,
+                                bool keepScalarOptionalBoxed) {
   if (entity.isVariable())
     return {translateVariableToExtendedValue(loc, builder, entity, false,
-                                             contiguousHint),
+                                             contiguousHint,
+                                             keepScalarOptionalBoxed),
             std::nullopt};
 
   if (entity.isProcedure()) {
@@ -1094,7 +1194,9 @@ hlfir::convertToBox(mlir::Location loc, fir::FirOpBuilder &builder,
   if (entity.isProcedurePointer())
     entity = hlfir::derefPointersAndAllocatables(loc, builder, entity);
 
-  auto [exv, cleanup] = translateToExtendedValue(loc, builder, entity);
+  auto [exv, cleanup] =
+      translateToExtendedValue(loc, builder, entity, /*contiguousHint=*/false,
+                               /*keepScalarOptionalBoxed=*/true);
   // Procedure entities should not go through createBoxValue that embox
   // object entities. Return the fir.boxproc directly.
   if (entity.isProcedure())
diff --git a/flang/lib/Optimizer/Builder/IntrinsicCall.cpp b/flang/lib/Optimizer/Builder/IntrinsicCall.cpp
index db9918c265164..9a80e36efe837 100644
--- a/flang/lib/Optimizer/Builder/IntrinsicCall.cpp
+++ b/flang/lib/Optimizer/Builder/IntrinsicCall.cpp
@@ -151,6 +151,22 @@ static constexpr IntrinsicHandler handlers[]{
     {"atomicaddf", &I::genAtomicAdd, {{{"a", asAddr}, {"v", asValue}}}, false},
     {"atomicaddi", &I::genAtomicAdd, {{{"a", asAddr}, {"v", asValue}}}, false},
     {"atomicaddl", &I::genAtomicAdd, {{{"a", asAddr}, {"v", asValue}}}, false},
+    {"atomicandi", &I::genAtomicAnd, {{{"a", asAddr}, {"v", asValue}}}, false},
+    {"atomicdeci", &I::genAtomicDec, {{{"a", asAddr}, {"v", asValue}}}, false},
+    {"atomicinci", &I::genAtomicInc, {{{"a", asAddr}, {"v", asValue}}}, false},
+    {"atomicmaxd", &I::genAtomicMax, {{{"a", asAddr}, {"v", asValue}}}, false},
+    {"atomicmaxf", &I::genAtomicMax, {{{"a", asAddr}, {"v", asValue}}}, false},
+    {"atomicmaxi", &I::genAtomicMax, {{{"a", asAddr}, {"v", asValue}}}, false},
+    {"atomicmaxl", &I::genAtomicMax, {{{"a", asAddr}, {"v", asValue}}}, false},
+    {"atomicmind", &I::genAtomicMin, {{{"a", asAddr}, {"v", asValue}}}, false},
+    {"atomicminf", &I::genAtomicMin, {{{"a", asAddr}, {"v", asValue}}}, false},
+    {"atomicmini", &I::genAtomicMin, {{{"a", asAddr}, {"v", asValue}}}, false},
+    {"atomicminl", &I::genAtomicMin, {{{"a", asAddr}, {"v", asValue}}}, false},
+    {"atomicori", &I::genAtomicOr, {{{"a", asAddr}, {"v", asValue}}}, false},
+    {"atomicsubd", &I::genAtomicSub, {{{"a", asAddr}, {"v", asValue}}}, false},
+    {"atomicsubf", &I::genAtomicSub, {{{"a", asAddr}, {"v", asValue}}}, false},
+    {"atomicsubi", &I::genAtomicSub, {{{"a", asAddr}, {"v", asValue}}}, false},
+    {"atomicsubl", &I::genAtomicSub, {{{"a", asAddr}, {"v", asValue}}}, false},
     {"bessel_jn",
      &I::genBesselJn,
      {{{"n1", asValue}, {"n2", asValue}, {"x", asValue}}},
@@ -189,6 +205,10 @@ static constexpr IntrinsicHandler handlers[]{
     {"c_ptr_ne", &I::genCPtrCompare<mlir::arith::CmpIPredicate::ne>},
     {"ceiling", &I::genCeiling},
     {"char", &I::genChar},
+    {"chdir",
+     &I::genChdir,
+     {{{"name", asAddr}, {"status", asAddr, handleDynamicOptional}}},
+     /*isElemental=*/false},
     {"cmplx",
      &I::genCmplx,
      {{{"x", asValue}, {"y", asValue, handleDynamicOptional}}}},
@@ -660,6 +680,7 @@ static constexpr IntrinsicHandler handlers[]{
     {"syncthreads_and", &I::genSyncThreadsAnd, {}, /*isElemental=*/false},
     {"syncthreads_count", &I::genSyncThreadsCount, {}, /*isElemental=*/false},
     {"syncthreads_or", &I::genSyncThreadsOr, {}, /*isElemental=*/false},
+    {"syncwarp", &I::genSyncWarp, {}, /*isElemental=*/false},
     {"system",
      &I::genSystem,
      {{{"command", asBox}, {"exitstat", asBox, handleDynamicOptional}}},
@@ -788,8 +809,8 @@ prettyPrintIntrinsicName(fir::FirOpBuilder &builder, mlir::Location loc,
 // Generate a call to the Fortran runtime library providing
 // support for 128-bit float math.
 // On 'HAS_LDBL128' targets the implementation
-// is provided by FortranRuntime, otherwise, it is done via
-// FortranFloat128Math library. In the latter case the compiler
+// is provided by flang_rt, otherwise, it is done via the
+// libflang_rt.quadmath library. In the latter case the compiler
 // has to be built with FLANG_RUNTIME_F128_MATH_LIB to guarantee
 // proper linking actions in the driver.
 static mlir::Value genLibF128Call(fir::FirOpBuilder &builder,
@@ -2600,6 +2621,75 @@ mlir::Value IntrinsicLibrary::genAtomicAdd(mlir::Type resultType,
   return genAtomBinOp(builder, loc, binOp, args[0], args[1]);
 }
 
+mlir::Value IntrinsicLibrary::genAtomicSub(mlir::Type resultType,
+                                           llvm::ArrayRef<mlir::Value> args) {
+  assert(args.size() == 2);
+
+  mlir::LLVM::AtomicBinOp binOp =
+      mlir::isa<mlir::IntegerType>(args[1].getType())
+          ? mlir::LLVM::AtomicBinOp::sub
+          : mlir::LLVM::AtomicBinOp::fsub;
+  return genAtomBinOp(builder, loc, binOp, args[0], args[1]);
+}
+
+mlir::Value IntrinsicLibrary::genAtomicAnd(mlir::Type resultType,
+                                           llvm::ArrayRef<mlir::Value> args) {
+  assert(args.size() == 2);
+  assert(mlir::isa<mlir::IntegerType>(args[1].getType()));
+
+  mlir::LLVM::AtomicBinOp binOp = mlir::LLVM::AtomicBinOp::_and;
+  return genAtomBinOp(builder, loc, binOp, args[0], args[1]);
+}
+
+mlir::Value IntrinsicLibrary::genAtomicOr(mlir::Type resultType,
+                                          llvm::ArrayRef<mlir::Value> args) {
+  assert(args.size() == 2);
+  assert(mlir::isa<mlir::IntegerType>(args[1].getType()));
+
+  mlir::LLVM::AtomicBinOp binOp = mlir::LLVM::AtomicBinOp::_or;
+  return genAtomBinOp(builder, loc, binOp, args[0], args[1]);
+}
+
+mlir::Value IntrinsicLibrary::genAtomicDec(mlir::Type resultType,
+                                           llvm::ArrayRef<mlir::Value> args) {
+  assert(args.size() == 2);
+  assert(mlir::isa<mlir::IntegerType>(args[1].getType()));
+
+  mlir::LLVM::AtomicBinOp binOp = mlir::LLVM::AtomicBinOp::udec_wrap;
+  return genAtomBinOp(builder, loc, binOp, args[0], args[1]);
+}
+
+mlir::Value IntrinsicLibrary::genAtomicInc(mlir::Type resultType,
+                                           llvm::ArrayRef<mlir::Value> args) {
+  assert(args.size() == 2);
+  assert(mlir::isa<mlir::IntegerType>(args[1].getType()));
+
+  mlir::LLVM::AtomicBinOp binOp = mlir::LLVM::AtomicBinOp::uinc_wrap;
+  return genAtomBinOp(builder, loc, binOp, args[0], args[1]);
+}
+
+mlir::Value IntrinsicLibrary::genAtomicMax(mlir::Type resultType,
+                                           llvm::ArrayRef<mlir::Value> args) {
+  assert(args.size() == 2);
+
+  mlir::LLVM::AtomicBinOp binOp =
+      mlir::isa<mlir::IntegerType>(args[1].getType())
+          ? mlir::LLVM::AtomicBinOp::max
+          : mlir::LLVM::AtomicBinOp::fmax;
+  return genAtomBinOp(builder, loc, binOp, args[0], args[1]);
+}
+
+mlir::Value IntrinsicLibrary::genAtomicMin(mlir::Type resultType,
+                                           llvm::ArrayRef<mlir::Value> args) {
+  assert(args.size() == 2);
+
+  mlir::LLVM::AtomicBinOp binOp =
+      mlir::isa<mlir::IntegerType>(args[1].getType())
+          ? mlir::LLVM::AtomicBinOp::min
+          : mlir::LLVM::AtomicBinOp::fmin;
+  return genAtomBinOp(builder, loc, binOp, args[0], args[1]);
+}
+
 // ASSOCIATED
 fir::ExtendedValue
 IntrinsicLibrary::genAssociated(mlir::Type resultType,
@@ -3101,6 +3191,35 @@ IntrinsicLibrary::genChar(mlir::Type type,
   return fir::CharBoxValue{cast, len};
 }
 
+// CHDIR
+fir::ExtendedValue
+IntrinsicLibrary::genChdir(std::optional<mlir::Type> resultType,
+                           llvm::ArrayRef<fir::ExtendedValue> args) {
+  assert((args.size() == 1 && resultType.has_value()) ||
+         (args.size() >= 1 && !resultType.has_value()));
+  mlir::Value name = fir::getBase(args[0]);
+  mlir::Value status = fir::runtime::genChdir(builder, loc, name);
+
+  if (resultType.has_value()) {
+    return status;
+  } else {
+    // Subroutine form, store status and return none.
+    if (!isStaticallyAbsent(args[1])) {
+      mlir::Value statusAddr = fir::getBase(args[1]);
+      statusAddr.dump();
+      mlir::Value statusIsPresentAtRuntime =
+          builder.genIsNotNullAddr(loc, statusAddr);
+      builder.genIfThen(loc, statusIsPresentAtRuntime)
+          .genThen([&]() {
+            builder.createStoreWithConvert(loc, status, statusAddr);
+          })
+          .end();
+    }
+  }
+
+  return {};
+}
+
 // CMPLX
 mlir::Value IntrinsicLibrary::genCmplx(mlir::Type resultType,
                                        llvm::ArrayRef<mlir::Value> args) {
@@ -6248,10 +6367,12 @@ mlir::Value IntrinsicLibrary::genNearest(mlir::Type resultType,
   mlir::FloatType xType = mlir::dyn_cast<mlir::FloatType>(x.getType());
   const unsigned xBitWidth = xType.getWidth();
   mlir::Type i1Ty = builder.getI1Type();
-  if constexpr (proc == NearestProc::NextAfter)
+  if constexpr (proc == NearestProc::NextAfter) {
     // If isNan(Y), set X to a qNaN that will propagate to the resultIsX result.
-    x = builder.create<mlir::arith::SelectOp>(
-        loc, genIsFPClass(i1Ty, args[1], nanTest), genQNan(xType), x);
+    mlir::Value qNan = genQNan(xType);
+    mlir::Value isFPClass = genIsFPClass(i1Ty, args[1], nanTest);
+    x = builder.create<mlir::arith::SelectOp>(loc, isFPClass, qNan, x);
+  }
   mlir::Value resultIsX = genIsFPClass(i1Ty, x, nanTest);
   mlir::Type intType = builder.getIntegerType(xBitWidth);
   mlir::Value one = builder.createIntegerConstant(loc, intType, 1);
@@ -6371,12 +6492,11 @@ mlir::Value IntrinsicLibrary::genNearest(mlir::Type resultType,
   } else {
     // Kind 2, 3, 4, 8, 16. Increment or decrement X cast to integer.
     mlir::Value intX = builder.create<mlir::arith::BitcastOp>(loc, intType, x);
+    mlir::Value add = builder.create<mlir::arith::AddIOp>(loc, intX, one);
+    mlir::Value sub = builder.create<mlir::arith::SubIOp>(loc, intX, one);
     result = builder.create<mlir::arith::BitcastOp>(
         loc, resultType,
-        builder.create<mlir::arith::SelectOp>(
-            loc, magnitudeUp,
-            builder.create<mlir::arith::AddIOp>(loc, intX, one),
-            builder.create<mlir::arith::SubIOp>(loc, intX, one)));
+        builder.create<mlir::arith::SelectOp>(loc, magnitudeUp, add, sub));
     if constexpr (proc == NearestProc::Nearest ||
                   proc == NearestProc::NextAfter) {
       genRaiseExcept(_FORTRAN_RUNTIME_IEEE_OVERFLOW |
@@ -7585,6 +7705,18 @@ IntrinsicLibrary::genSyncThreadsOr(mlir::Type resultType,
   return builder.create<fir::CallOp>(loc, funcOp, args).getResult(0);
 }
 
+// SYNCWARP
+void IntrinsicLibrary::genSyncWarp(llvm::ArrayRef<fir::ExtendedValue> args) {
+  assert(args.size() == 1);
+  constexpr llvm::StringLiteral funcName = "llvm.nvvm.bar.warp.sync";
+  mlir::Value mask = fir::getBase(args[0]);
+  mlir::FunctionType funcType =
+      mlir::FunctionType::get(builder.getContext(), {mask.getType()}, {});
+  auto funcOp = builder.createFunction(loc, funcName, funcType);
+  llvm::SmallVector<mlir::Value> argsList{mask};
+  builder.create<fir::CallOp>(loc, funcOp, argsList);
+}
+
 // SYSTEM
 fir::ExtendedValue
 IntrinsicLibrary::genSystem(std::optional<mlir::Type> resultType,
diff --git a/flang/lib/Optimizer/Builder/Runtime/Intrinsics.cpp b/flang/lib/Optimizer/Builder/Runtime/Intrinsics.cpp
index 963051ccdc379..2f46e7605fe91 100644
--- a/flang/lib/Optimizer/Builder/Runtime/Intrinsics.cpp
+++ b/flang/lib/Optimizer/Builder/Runtime/Intrinsics.cpp
@@ -384,3 +384,13 @@ void fir::runtime::genSleep(fir::FirOpBuilder &builder, mlir::Location loc,
       fir::runtime::getRuntimeFunc<mkRTKey(Sleep)>(loc, builder)};
   builder.create<fir::CallOp>(loc, func, seconds);
 }
+
+/// generate chdir runtime call
+mlir::Value fir::runtime::genChdir(fir::FirOpBuilder &builder,
+                                   mlir::Location loc, mlir::Value name) {
+  mlir::func::FuncOp func{
+      fir::runtime::getRuntimeFunc<mkRTKey(Chdir)>(loc, builder)};
+  llvm::SmallVector<mlir::Value> args =
+      fir::runtime::createArguments(builder, loc, func.getFunctionType(), name);
+  return builder.create<fir::CallOp>(loc, func, args).getResult(0);
+}
diff --git a/flang/lib/Optimizer/Builder/Runtime/Main.cpp b/flang/lib/Optimizer/Builder/Runtime/Main.cpp
index ab3c4ca81314c..973744837d378 100644
--- a/flang/lib/Optimizer/Builder/Runtime/Main.cpp
+++ b/flang/lib/Optimizer/Builder/Runtime/Main.cpp
@@ -14,6 +14,7 @@
 #include "flang/Optimizer/Builder/Runtime/RTBuilder.h"
 #include "flang/Optimizer/Dialect/FIROps.h"
 #include "flang/Optimizer/Dialect/FIRType.h"
+#include "flang/Runtime/CUDA/init.h"
 #include "flang/Runtime/main.h"
 #include "flang/Runtime/stop.h"
 
@@ -22,7 +23,8 @@ using namespace Fortran::runtime;
 /// Create a `int main(...)` that calls the Fortran entry point
 void fir::runtime::genMain(
     fir::FirOpBuilder &builder, mlir::Location loc,
-    const std::vector<Fortran::lower::EnvironmentDefault> &defs) {
+    const std::vector<Fortran::lower::EnvironmentDefault> &defs,
+    bool initCuda) {
   auto *context = builder.getContext();
   auto argcTy = builder.getDefaultIntegerType();
   auto ptrTy = mlir::LLVM::LLVMPointerType::get(context);
@@ -61,6 +63,13 @@ void fir::runtime::genMain(
   args.push_back(env);
 
   builder.create<fir::CallOp>(loc, startFn, args);
+
+  if (initCuda) {
+    auto initFn = builder.createFunction(
+        loc, RTNAME_STRING(CUFInit), mlir::FunctionType::get(context, {}, {}));
+    builder.create<fir::CallOp>(loc, initFn);
+  }
+
   builder.create<fir::CallOp>(loc, qqMainFn);
   builder.create<fir::CallOp>(loc, stopFn);
 
diff --git a/flang/lib/Optimizer/CodeGen/BoxedProcedure.cpp b/flang/lib/Optimizer/CodeGen/BoxedProcedure.cpp
index c7c2dcede8aa6..26f4aee21d8bd 100644
--- a/flang/lib/Optimizer/CodeGen/BoxedProcedure.cpp
+++ b/flang/lib/Optimizer/CodeGen/BoxedProcedure.cpp
@@ -270,10 +270,18 @@ class BoxedProcedurePass
             // Create the thunk.
             auto module = embox->getParentOfType<mlir::ModuleOp>();
             FirOpBuilder builder(rewriter, module);
+            const auto triple{fir::getTargetTriple(module)};
             auto loc = embox.getLoc();
             mlir::Type i8Ty = builder.getI8Type();
             mlir::Type i8Ptr = builder.getRefType(i8Ty);
-            mlir::Type buffTy = SequenceType::get({32}, i8Ty);
+            // For AArch64, PPC32 and PPC64, the thunk is populated by a call to
+            // __trampoline_setup, which is defined in
+            // compiler-rt/lib/builtins/trampoline_setup.c and requires the
+            // thunk size greater than 32 bytes.  For RISCV and x86_64, the
+            // thunk setup doesn't go through __trampoline_setup and fits in 32
+            // bytes.
+            fir::SequenceType::Extent thunkSize = triple.getTrampolineSize();
+            mlir::Type buffTy = SequenceType::get({thunkSize}, i8Ty);
             auto buffer = builder.create<AllocaOp>(loc, buffTy);
             mlir::Value closure =
                 builder.createConvert(loc, i8Ptr, embox.getHost());
diff --git a/flang/lib/Optimizer/CodeGen/CodeGen.cpp b/flang/lib/Optimizer/CodeGen/CodeGen.cpp
index 6ff2c20d74453..cb4eb8303a495 100644
--- a/flang/lib/Optimizer/CodeGen/CodeGen.cpp
+++ b/flang/lib/Optimizer/CodeGen/CodeGen.cpp
@@ -1190,7 +1190,7 @@ genCUFAllocDescriptor(mlir::Location loc,
                       mlir::ModuleOp mod, fir::BaseBoxType boxTy,
                       const fir::LLVMTypeConverter &typeConverter) {
   std::optional<mlir::DataLayout> dl =
-      fir::support::getOrSetDataLayout(mod, /*allowDefaultLayout=*/true);
+      fir::support::getOrSetMLIRDataLayout(mod, /*allowDefaultLayout=*/true);
   if (!dl)
     mlir::emitError(mod.getLoc(),
                     "module operation must carry a data layout attribute "
@@ -1675,22 +1675,26 @@ struct EmboxCommonConversion : public fir::FIROpConversion<OP> {
     this->attachTBAATag(storeOp, boxTy, boxTy, nullptr);
     return storage;
   }
-};
 
-/// Compute the extent of a triplet slice (lb:ub:step).
-static mlir::Value
-computeTripletExtent(mlir::ConversionPatternRewriter &rewriter,
-                     mlir::Location loc, mlir::Value lb, mlir::Value ub,
-                     mlir::Value step, mlir::Value zero, mlir::Type type) {
-  mlir::Value extent = rewriter.create<mlir::LLVM::SubOp>(loc, type, ub, lb);
-  extent = rewriter.create<mlir::LLVM::AddOp>(loc, type, extent, step);
-  extent = rewriter.create<mlir::LLVM::SDivOp>(loc, type, extent, step);
-  // If the resulting extent is negative (`ub-lb` and `step` have different
-  // signs), zero must be returned instead.
-  auto cmp = rewriter.create<mlir::LLVM::ICmpOp>(
-      loc, mlir::LLVM::ICmpPredicate::sgt, extent, zero);
-  return rewriter.create<mlir::LLVM::SelectOp>(loc, cmp, extent, zero);
-}
+  /// Compute the extent of a triplet slice (lb:ub:step).
+  mlir::Value computeTripletExtent(mlir::ConversionPatternRewriter &rewriter,
+                                   mlir::Location loc, mlir::Value lb,
+                                   mlir::Value ub, mlir::Value step,
+                                   mlir::Value zero, mlir::Type type) const {
+    lb = this->integerCast(loc, rewriter, type, lb);
+    ub = this->integerCast(loc, rewriter, type, ub);
+    step = this->integerCast(loc, rewriter, type, step);
+    zero = this->integerCast(loc, rewriter, type, zero);
+    mlir::Value extent = rewriter.create<mlir::LLVM::SubOp>(loc, type, ub, lb);
+    extent = rewriter.create<mlir::LLVM::AddOp>(loc, type, extent, step);
+    extent = rewriter.create<mlir::LLVM::SDivOp>(loc, type, extent, step);
+    // If the resulting extent is negative (`ub-lb` and `step` have different
+    // signs), zero must be returned instead.
+    auto cmp = rewriter.create<mlir::LLVM::ICmpOp>(
+        loc, mlir::LLVM::ICmpPredicate::sgt, extent, zero);
+    return rewriter.create<mlir::LLVM::SelectOp>(loc, cmp, extent, zero);
+  }
+};
 
 /// Create a generic box on a memory reference. This conversions lowers the
 /// abstract box to the appropriate, initialized descriptor.
@@ -1851,14 +1855,16 @@ struct XEmboxOpConversion : public EmboxCommonConversion<fir::cg::XEmboxOp> {
     // translating everything to values in the descriptor wherever the entity
     // has a dynamic array dimension.
     for (unsigned di = 0, descIdx = 0; di < rank; ++di) {
-      mlir::Value extent = operands[shapeOffset];
+      mlir::Value extent =
+          integerCast(loc, rewriter, i64Ty, operands[shapeOffset]);
       mlir::Value outerExtent = extent;
       bool skipNext = false;
       if (hasSlice) {
-        mlir::Value off = operands[sliceOffset];
+        mlir::Value off =
+            integerCast(loc, rewriter, i64Ty, operands[sliceOffset]);
         mlir::Value adj = one;
         if (hasShift)
-          adj = operands[shiftOffset];
+          adj = integerCast(loc, rewriter, i64Ty, operands[shiftOffset]);
         auto ao = rewriter.create<mlir::LLVM::SubOp>(loc, i64Ty, off, adj);
         if (constRows > 0) {
           cstInteriorIndices.push_back(ao);
@@ -1895,7 +1901,7 @@ struct XEmboxOpConversion : public EmboxCommonConversion<fir::cg::XEmboxOp> {
         // the lower bound.
         if (hasShift && !(hasSlice || hasSubcomp || hasSubstr) &&
             (isaPointerOrAllocatable || !normalizedLowerBound(xbox))) {
-          lb = operands[shiftOffset];
+          lb = integerCast(loc, rewriter, i64Ty, operands[shiftOffset]);
           auto extentIsEmpty = rewriter.create<mlir::LLVM::ICmpOp>(
               loc, mlir::LLVM::ICmpPredicate::eq, extent, zero);
           lb = rewriter.create<mlir::LLVM::SelectOp>(loc, extentIsEmpty, one,
@@ -1907,9 +1913,12 @@ struct XEmboxOpConversion : public EmboxCommonConversion<fir::cg::XEmboxOp> {
 
         // store step (scaled by shaped extent)
         mlir::Value step = prevDimByteStride;
-        if (hasSlice)
-          step = rewriter.create<mlir::LLVM::MulOp>(loc, i64Ty, step,
-                                                    operands[sliceOffset + 2]);
+        if (hasSlice) {
+          mlir::Value sliceStep =
+              integerCast(loc, rewriter, i64Ty, operands[sliceOffset + 2]);
+          step =
+              rewriter.create<mlir::LLVM::MulOp>(loc, i64Ty, step, sliceStep);
+        }
         dest = insertStride(rewriter, loc, dest, descIdx, step);
         ++descIdx;
       }
@@ -2996,11 +3005,13 @@ struct GlobalOpConversion : public fir::FIROpConversion<fir::GlobalOp> {
     llvm::SmallVector<mlir::Attribute> dbgExprs;
 
     if (auto fusedLoc = mlir::dyn_cast<mlir::FusedLoc>(global.getLoc())) {
-      if (auto gvAttr =
-              mlir::dyn_cast_or_null<mlir::LLVM::DIGlobalVariableAttr>(
-                  fusedLoc.getMetadata())) {
-        dbgExprs.push_back(mlir::LLVM::DIGlobalVariableExpressionAttr::get(
-            global.getContext(), gvAttr, mlir::LLVM::DIExpressionAttr()));
+      if (auto gvExprAttr = mlir::dyn_cast_if_present<mlir::ArrayAttr>(
+              fusedLoc.getMetadata())) {
+        for (auto attr : gvExprAttr.getAsRange<mlir::Attribute>())
+          if (auto dbgAttr =
+                  mlir::dyn_cast<mlir::LLVM::DIGlobalVariableExpressionAttr>(
+                      attr))
+            dbgExprs.push_back(dbgAttr);
       }
     }
 
@@ -3931,7 +3942,7 @@ class FIRToLLVMLowering
       return signalPassFailure();
 
     std::optional<mlir::DataLayout> dl =
-        fir::support::getOrSetDataLayout(mod, /*allowDefaultLayout=*/true);
+        fir::support::getOrSetMLIRDataLayout(mod, /*allowDefaultLayout=*/true);
     if (!dl) {
       mlir::emitError(mod.getLoc(),
                       "module operation must carry a data layout attribute "
diff --git a/flang/lib/Optimizer/CodeGen/CodeGenOpenMP.cpp b/flang/lib/Optimizer/CodeGen/CodeGenOpenMP.cpp
index da13ed648e44e..37f1c9f97e1ce 100644
--- a/flang/lib/Optimizer/CodeGen/CodeGenOpenMP.cpp
+++ b/flang/lib/Optimizer/CodeGen/CodeGenOpenMP.cpp
@@ -90,9 +90,45 @@ struct MapInfoOpConversion
     return mlir::success();
   }
 };
+
+// FIR op specific conversion for PrivateClauseOp that overwrites the default
+// OpenMP Dialect lowering, this allows FIR-aware lowering of types, required
+// for boxes because the OpenMP dialect conversion doesn't know anything about
+// FIR types.
+struct PrivateClauseOpConversion
+    : public OpenMPFIROpConversion<mlir::omp::PrivateClauseOp> {
+  using OpenMPFIROpConversion::OpenMPFIROpConversion;
+
+  llvm::LogicalResult
+  matchAndRewrite(mlir::omp::PrivateClauseOp curOp, OpAdaptor adaptor,
+                  mlir::ConversionPatternRewriter &rewriter) const override {
+    const fir::LLVMTypeConverter &converter = lowerTy();
+    mlir::Type convertedAllocType;
+    if (auto box = mlir::dyn_cast<fir::BaseBoxType>(curOp.getType())) {
+      // In LLVM codegen fir.box<> == fir.ref<fir.box<>> == llvm.ptr
+      // Here we really do want the actual structure
+      if (box.isAssumedRank())
+        TODO(curOp->getLoc(), "Privatize an assumed rank array");
+      unsigned rank = 0;
+      if (auto seqTy = mlir::dyn_cast<fir::SequenceType>(
+              fir::unwrapRefType(box.getEleTy())))
+        rank = seqTy.getShape().size();
+      convertedAllocType = converter.convertBoxTypeAsStruct(box, rank);
+    } else {
+      convertedAllocType = converter.convertType(adaptor.getType());
+    }
+    if (!convertedAllocType)
+      return mlir::failure();
+    rewriter.startOpModification(curOp);
+    curOp.setType(convertedAllocType);
+    rewriter.finalizeOpModification(curOp);
+    return mlir::success();
+  }
+};
 } // namespace
 
 void fir::populateOpenMPFIRToLLVMConversionPatterns(
     const LLVMTypeConverter &converter, mlir::RewritePatternSet &patterns) {
   patterns.add<MapInfoOpConversion>(converter);
+  patterns.add<PrivateClauseOpConversion>(converter);
 }
diff --git a/flang/lib/Optimizer/CodeGen/TargetRewrite.cpp b/flang/lib/Optimizer/CodeGen/TargetRewrite.cpp
index b0b9499557e2b..c099a08ffd30a 100644
--- a/flang/lib/Optimizer/CodeGen/TargetRewrite.cpp
+++ b/flang/lib/Optimizer/CodeGen/TargetRewrite.cpp
@@ -107,7 +107,7 @@ class TargetRewrite : public fir::impl::TargetRewritePassBase<TargetRewrite> {
     // TargetRewrite will require querying the type storage sizes, if it was
     // not set already, create a DataLayoutSpec for the ModuleOp now.
     std::optional<mlir::DataLayout> dl =
-        fir::support::getOrSetDataLayout(mod, /*allowDefaultLayout=*/true);
+        fir::support::getOrSetMLIRDataLayout(mod, /*allowDefaultLayout=*/true);
     if (!dl) {
       mlir::emitError(mod.getLoc(),
                       "module operation must carry a data layout attribute "
@@ -518,6 +518,7 @@ class TargetRewrite : public fir::impl::TargetRewritePassBase<TargetRewrite> {
     newOpers.insert(newOpers.end(), trailingOpers.begin(), trailingOpers.end());
 
     llvm::SmallVector<mlir::Value, 1> newCallResults;
+    // TODO propagate/update call argument and result attributes.
     if constexpr (std::is_same_v<std::decay_t<A>, mlir::gpu::LaunchFuncOp>) {
       auto newCall = rewriter->create<A>(
           loc, callOp.getKernel(), callOp.getGridSizeOperandValues(),
@@ -557,6 +558,7 @@ class TargetRewrite : public fir::impl::TargetRewritePassBase<TargetRewrite> {
           loc, newResTys, rewriter->getStringAttr(callOp.getMethod()),
           callOp.getOperands()[0], newOpers,
           rewriter->getI32IntegerAttr(*callOp.getPassArgPos() + passArgShift),
+          /*arg_attrs=*/nullptr, /*res_attrs=*/nullptr,
           callOp.getProcedureAttrsAttr());
       if (wrap)
         newCallResults.push_back((*wrap)(dispatchOp.getOperation()));
diff --git a/flang/lib/Optimizer/CodeGen/TypeConverter.cpp b/flang/lib/Optimizer/CodeGen/TypeConverter.cpp
index 0eace903720f0..89f498433806e 100644
--- a/flang/lib/Optimizer/CodeGen/TypeConverter.cpp
+++ b/flang/lib/Optimizer/CodeGen/TypeConverter.cpp
@@ -13,7 +13,6 @@
 #define DEBUG_TYPE "flang-type-conversion"
 
 #include "flang/Optimizer/CodeGen/TypeConverter.h"
-#include "flang/Common/Fortran.h"
 #include "flang/Optimizer/Builder/Todo.h" // remove when TODO's are done
 #include "flang/Optimizer/CodeGen/DescriptorModel.h"
 #include "flang/Optimizer/CodeGen/TBAABuilder.h"
@@ -22,6 +21,7 @@
 #include "flang/Optimizer/Dialect/Support/FIRContext.h"
 #include "flang/Optimizer/Dialect/Support/KindMapping.h"
 #include "flang/Optimizer/Support/InternalNames.h"
+#include "flang/Support/Fortran.h"
 #include "mlir/Conversion/LLVMCommon/TypeConverter.h"
 #include "llvm/ADT/ScopeExit.h"
 #include "llvm/Support/Debug.h"
diff --git a/flang/lib/Optimizer/Dialect/FIRType.cpp b/flang/lib/Optimizer/Dialect/FIRType.cpp
index 0b57a10a6c493..67d918cc0f41c 100644
--- a/flang/lib/Optimizer/Dialect/FIRType.cpp
+++ b/flang/lib/Optimizer/Dialect/FIRType.cpp
@@ -11,7 +11,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "flang/Optimizer/Dialect/FIRType.h"
-#include "flang/ISO_Fortran_binding_wrapper.h"
+#include "flang/Common/ISO_Fortran_binding_wrapper.h"
 #include "flang/Optimizer/Builder/Todo.h"
 #include "flang/Optimizer/Dialect/FIRDialect.h"
 #include "flang/Optimizer/Dialect/Support/KindMapping.h"
diff --git a/flang/lib/Optimizer/HLFIR/IR/HLFIROps.cpp b/flang/lib/Optimizer/HLFIR/IR/HLFIROps.cpp
index d93e25280237f..2fcfa1353f86b 100644
--- a/flang/lib/Optimizer/HLFIR/IR/HLFIROps.cpp
+++ b/flang/lib/Optimizer/HLFIR/IR/HLFIROps.cpp
@@ -67,6 +67,33 @@ getIntrinsicEffects(mlir::Operation *self,
   }
 }
 
+/// Verification helper for checking if two types are the same.
+/// Set \p allowCharacterLenMismatch to true, if character types
+/// of different known lengths should be treated as the same.
+template <typename Op>
+static llvm::LogicalResult areMatchingTypes(Op &op, mlir::Type type1,
+                                            mlir::Type type2,
+                                            bool allowCharacterLenMismatch) {
+  if (auto charType1 = mlir::dyn_cast<fir::CharacterType>(type1))
+    if (auto charType2 = mlir::dyn_cast<fir::CharacterType>(type2)) {
+      // Character kinds must match.
+      if (charType1.getFKind() != charType2.getFKind())
+        return op.emitOpError("character KIND mismatch");
+
+      // Constant propagation can result in mismatching lengths
+      // in the dead code, but we should not fail on this.
+      if (!allowCharacterLenMismatch)
+        if (charType1.getLen() != fir::CharacterType::unknownLen() &&
+            charType2.getLen() != fir::CharacterType::unknownLen() &&
+            charType1.getLen() != charType2.getLen())
+          return op.emitOpError("character LEN mismatch");
+
+      return mlir::success();
+    }
+
+  return type1 == type2 ? mlir::success() : mlir::failure();
+}
+
 //===----------------------------------------------------------------------===//
 // DeclareOp
 //===----------------------------------------------------------------------===//
@@ -1360,23 +1387,12 @@ llvm::LogicalResult hlfir::CShiftOp::verify() {
   mlir::Value shift = getShift();
   mlir::Type shiftTy = hlfir::getFortranElementOrSequenceType(shift.getType());
 
-  if (eleTy != resultEleTy) {
-    if (mlir::isa<fir::CharacterType>(eleTy) &&
-        mlir::isa<fir::CharacterType>(resultEleTy)) {
-      auto eleCharTy = mlir::cast<fir::CharacterType>(eleTy);
-      auto resultCharTy = mlir::cast<fir::CharacterType>(resultEleTy);
-      if (eleCharTy.getFKind() != resultCharTy.getFKind())
-        return emitOpError("kind mismatch between input and output arrays");
-      if (eleCharTy.getLen() != fir::CharacterType::unknownLen() &&
-          resultCharTy.getLen() != fir::CharacterType::unknownLen() &&
-          eleCharTy.getLen() != resultCharTy.getLen())
-        return emitOpError(
-            "character LEN mismatch between input and output arrays");
-    } else {
-      return emitOpError(
-          "input and output arrays should have the same element type");
-    }
-  }
+  // TODO: turn allowCharacterLenMismatch into true.
+  if (auto match = areMatchingTypes(*this, eleTy, resultEleTy,
+                                    /*allowCharacterLenMismatch=*/false);
+      match.failed())
+    return emitOpError(
+        "input and output arrays should have the same element type");
 
   if (arrayRank != resultRank)
     return emitOpError("input and output arrays should have the same rank");
@@ -1444,6 +1460,67 @@ void hlfir::CShiftOp::getEffects(
   getIntrinsicEffects(getOperation(), effects);
 }
 
+//===----------------------------------------------------------------------===//
+// ReshapeOp
+//===----------------------------------------------------------------------===//
+
+llvm::LogicalResult hlfir::ReshapeOp::verify() {
+  auto results = getOperation()->getResultTypes();
+  assert(results.size() == 1);
+  hlfir::ExprType resultType = mlir::cast<hlfir::ExprType>(results[0]);
+  mlir::Value array = getArray();
+  auto arrayType = mlir::cast<fir::SequenceType>(
+      hlfir::getFortranElementOrSequenceType(array.getType()));
+  if (auto match = areMatchingTypes(
+          *this, hlfir::getFortranElementType(resultType),
+          arrayType.getElementType(), /*allowCharacterLenMismatch=*/true);
+      match.failed())
+    return emitOpError("ARRAY and the result must have the same element type");
+  if (hlfir::isPolymorphicType(resultType) !=
+      hlfir::isPolymorphicType(array.getType()))
+    return emitOpError("ARRAY must be polymorphic iff result is polymorphic");
+
+  mlir::Value shape = getShape();
+  auto shapeArrayType = mlir::cast<fir::SequenceType>(
+      hlfir::getFortranElementOrSequenceType(shape.getType()));
+  if (shapeArrayType.getDimension() != 1)
+    return emitOpError("SHAPE must be an array of rank 1");
+  if (!mlir::isa<mlir::IntegerType>(shapeArrayType.getElementType()))
+    return emitOpError("SHAPE must be an integer array");
+  if (shapeArrayType.hasDynamicExtents())
+    return emitOpError("SHAPE must have known size");
+  if (shapeArrayType.getConstantArraySize() != resultType.getRank())
+    return emitOpError("SHAPE's extent must match the result rank");
+
+  if (mlir::Value pad = getPad()) {
+    auto padArrayType = mlir::cast<fir::SequenceType>(
+        hlfir::getFortranElementOrSequenceType(pad.getType()));
+    if (auto match = areMatchingTypes(*this, arrayType.getElementType(),
+                                      padArrayType.getElementType(),
+                                      /*allowCharacterLenMismatch=*/true);
+        match.failed())
+      return emitOpError("ARRAY and PAD must be of the same type");
+  }
+
+  if (mlir::Value order = getOrder()) {
+    auto orderArrayType = mlir::cast<fir::SequenceType>(
+        hlfir::getFortranElementOrSequenceType(order.getType()));
+    if (orderArrayType.getDimension() != 1)
+      return emitOpError("ORDER must be an array of rank 1");
+    if (!mlir::isa<mlir::IntegerType>(orderArrayType.getElementType()))
+      return emitOpError("ORDER must be an integer array");
+  }
+
+  return mlir::success();
+}
+
+void hlfir::ReshapeOp::getEffects(
+    llvm::SmallVectorImpl<
+        mlir::SideEffects::EffectInstance<mlir::MemoryEffects::Effect>>
+        &effects) {
+  getIntrinsicEffects(getOperation(), effects);
+}
+
 //===----------------------------------------------------------------------===//
 // AssociateOp
 //===----------------------------------------------------------------------===//
diff --git a/flang/lib/Optimizer/HLFIR/Transforms/BufferizeHLFIR.cpp b/flang/lib/Optimizer/HLFIR/Transforms/BufferizeHLFIR.cpp
index 48e3db0eb39b6..30e7ef7890953 100644
--- a/flang/lib/Optimizer/HLFIR/Transforms/BufferizeHLFIR.cpp
+++ b/flang/lib/Optimizer/HLFIR/Transforms/BufferizeHLFIR.cpp
@@ -362,6 +362,7 @@ struct GetLengthOpConversion
     if (!length)
       return rewriter.notifyMatchFailure(
           getLength, "could not deduce length from GetLengthOp operand");
+    length = builder.createConvert(loc, builder.getIndexType(), length);
     rewriter.replaceOp(getLength, length);
     return mlir::success();
   }
@@ -760,8 +761,10 @@ struct HLFIRListener : public mlir::OpBuilder::Listener {
 struct ElementalOpConversion
     : public mlir::OpConversionPattern<hlfir::ElementalOp> {
   using mlir::OpConversionPattern<hlfir::ElementalOp>::OpConversionPattern;
-  explicit ElementalOpConversion(mlir::MLIRContext *ctx)
-      : mlir::OpConversionPattern<hlfir::ElementalOp>{ctx} {
+  explicit ElementalOpConversion(mlir::MLIRContext *ctx,
+                                 bool optimizeEmptyElementals = false)
+      : mlir::OpConversionPattern<hlfir::ElementalOp>{ctx},
+        optimizeEmptyElementals(optimizeEmptyElementals) {
     // This pattern recursively converts nested ElementalOp's
     // by cloning and then converting them, so we have to allow
     // for recursive pattern application. The recursion is bounded
@@ -790,6 +793,10 @@ struct ElementalOpConversion
     // of the loop nest.
     temp = derefPointersAndAllocatables(loc, builder, temp);
 
+    if (optimizeEmptyElementals)
+      extents = fir::factory::updateRuntimeExtentsForEmptyArrays(builder, loc,
+                                                                 extents);
+
     // Generate a loop nest looping around the fir.elemental shape and clone
     // fir.elemental region inside the inner loop.
     hlfir::LoopNest loopNest =
@@ -860,6 +867,9 @@ struct ElementalOpConversion
     rewriter.replaceOp(elemental, bufferizedExpr);
     return mlir::success();
   }
+
+private:
+  bool optimizeEmptyElementals = false;
 };
 struct CharExtremumOpConversion
     : public mlir::OpConversionPattern<hlfir::CharExtremumOp> {
@@ -931,6 +941,8 @@ struct EvaluateInMemoryOpConversion
 
 class BufferizeHLFIR : public hlfir::impl::BufferizeHLFIRBase<BufferizeHLFIR> {
 public:
+  using BufferizeHLFIRBase<BufferizeHLFIR>::BufferizeHLFIRBase;
+
   void runOnOperation() override {
     // TODO: make this a pass operating on FuncOp. The issue is that
     // FirOpBuilder helpers may generate new FuncOp because of runtime/llvm
@@ -942,13 +954,13 @@ class BufferizeHLFIR : public hlfir::impl::BufferizeHLFIRBase<BufferizeHLFIR> {
     auto module = this->getOperation();
     auto *context = &getContext();
     mlir::RewritePatternSet patterns(context);
-    patterns
-        .insert<ApplyOpConversion, AsExprOpConversion, AssignOpConversion,
-                AssociateOpConversion, CharExtremumOpConversion,
-                ConcatOpConversion, DestroyOpConversion, ElementalOpConversion,
-                EndAssociateOpConversion, EvaluateInMemoryOpConversion,
-                NoReassocOpConversion, SetLengthOpConversion,
-                ShapeOfOpConversion, GetLengthOpConversion>(context);
+    patterns.insert<ApplyOpConversion, AsExprOpConversion, AssignOpConversion,
+                    AssociateOpConversion, CharExtremumOpConversion,
+                    ConcatOpConversion, DestroyOpConversion,
+                    EndAssociateOpConversion, EvaluateInMemoryOpConversion,
+                    NoReassocOpConversion, SetLengthOpConversion,
+                    ShapeOfOpConversion, GetLengthOpConversion>(context);
+    patterns.insert<ElementalOpConversion>(context, optimizeEmptyElementals);
     mlir::ConversionTarget target(*context);
     // Note that YieldElementOp is not marked as an illegal operation.
     // It must be erased by its parent converter and there is no explicit
diff --git a/flang/lib/Optimizer/HLFIR/Transforms/ConvertToFIR.cpp b/flang/lib/Optimizer/HLFIR/Transforms/ConvertToFIR.cpp
index 536f2077e4f70..496a5560ac615 100644
--- a/flang/lib/Optimizer/HLFIR/Transforms/ConvertToFIR.cpp
+++ b/flang/lib/Optimizer/HLFIR/Transforms/ConvertToFIR.cpp
@@ -735,6 +735,9 @@ class GetExtentOpConversion
       llvm::APInt dim = getExtentOp.getDim();
       uint64_t dimVal = dim.getLimitedValue(shapeTy.getRank());
       mlir::Value extent = s.getExtents()[dimVal];
+      fir::FirOpBuilder builder(rewriter, getExtentOp.getOperation());
+      extent = builder.createConvert(getExtentOp.getLoc(),
+                                     builder.getIndexType(), extent);
       rewriter.replaceOp(getExtentOp, extent);
       return mlir::success();
     }
diff --git a/flang/lib/Optimizer/HLFIR/Transforms/LowerHLFIRIntrinsics.cpp b/flang/lib/Optimizer/HLFIR/Transforms/LowerHLFIRIntrinsics.cpp
index 091ed7ed999df..7c0fcba806869 100644
--- a/flang/lib/Optimizer/HLFIR/Transforms/LowerHLFIRIntrinsics.cpp
+++ b/flang/lib/Optimizer/HLFIR/Transforms/LowerHLFIRIntrinsics.cpp
@@ -121,8 +121,14 @@ class HlfirIntrinsicConversion : public mlir::OpRewritePattern<OP> {
         // simplified since the fir.box lowered here are now guarenteed to
         // contain the local lower bounds thanks to the hlfir.declare (the extra
         // rebox can be removed).
-        auto [exv, cleanup] =
-            hlfir::translateToExtendedValue(loc, builder, entity);
+        // When taking arguments as descriptors, the runtime expect absent
+        // OPTIONAL to be a nullptr to a descriptor, lowering has already
+        // prepared such descriptors as needed, hence set
+        // keepScalarOptionalBoxed to avoid building descriptors with a null
+        // address for them.
+        auto [exv, cleanup] = hlfir::translateToExtendedValue(
+            loc, builder, entity, /*contiguous=*/false,
+            /*keepScalarOptionalBoxed=*/true);
         if (cleanup)
           cleanupFns.push_back(*cleanup);
         ret.emplace_back(exv);
@@ -494,6 +500,41 @@ class CShiftOpConversion : public HlfirIntrinsicConversion<hlfir::CShiftOp> {
   }
 };
 
+class ReshapeOpConversion : public HlfirIntrinsicConversion<hlfir::ReshapeOp> {
+  using HlfirIntrinsicConversion<hlfir::ReshapeOp>::HlfirIntrinsicConversion;
+
+  llvm::LogicalResult
+  matchAndRewrite(hlfir::ReshapeOp reshape,
+                  mlir::PatternRewriter &rewriter) const override {
+    fir::FirOpBuilder builder{rewriter, reshape.getOperation()};
+    const mlir::Location &loc = reshape->getLoc();
+
+    llvm::SmallVector<IntrinsicArgument, 4> inArgs;
+    mlir::Value array = reshape.getArray();
+    inArgs.push_back({array, array.getType()});
+    mlir::Value shape = reshape.getShape();
+    inArgs.push_back({shape, shape.getType()});
+    mlir::Type noneType = builder.getNoneType();
+    mlir::Value pad = reshape.getPad();
+    inArgs.push_back({pad, pad ? pad.getType() : noneType});
+    mlir::Value order = reshape.getOrder();
+    inArgs.push_back({order, order ? order.getType() : noneType});
+
+    auto *argLowering = fir::getIntrinsicArgumentLowering("reshape");
+    llvm::SmallVector<fir::ExtendedValue, 4> args =
+        lowerArguments(reshape, inArgs, rewriter, argLowering);
+
+    mlir::Type scalarResultType =
+        hlfir::getFortranElementType(reshape.getType());
+
+    auto [resultExv, mustBeFreed] =
+        fir::genIntrinsicCall(builder, loc, "reshape", scalarResultType, args);
+
+    processReturnValue(reshape, resultExv, mustBeFreed, builder, rewriter);
+    return mlir::success();
+  }
+};
+
 class LowerHLFIRIntrinsics
     : public hlfir::impl::LowerHLFIRIntrinsicsBase<LowerHLFIRIntrinsics> {
 public:
@@ -501,13 +542,12 @@ class LowerHLFIRIntrinsics
     mlir::ModuleOp module = this->getOperation();
     mlir::MLIRContext *context = &getContext();
     mlir::RewritePatternSet patterns(context);
-    patterns
-        .insert<MatmulOpConversion, MatmulTransposeOpConversion,
-                AllOpConversion, AnyOpConversion, SumOpConversion,
-                ProductOpConversion, TransposeOpConversion, CountOpConversion,
-                DotProductOpConversion, MaxvalOpConversion, MinvalOpConversion,
-                MinlocOpConversion, MaxlocOpConversion, CShiftOpConversion>(
-            context);
+    patterns.insert<
+        MatmulOpConversion, MatmulTransposeOpConversion, AllOpConversion,
+        AnyOpConversion, SumOpConversion, ProductOpConversion,
+        TransposeOpConversion, CountOpConversion, DotProductOpConversion,
+        MaxvalOpConversion, MinvalOpConversion, MinlocOpConversion,
+        MaxlocOpConversion, CShiftOpConversion, ReshapeOpConversion>(context);
 
     // While conceptually this pass is performing dialect conversion, we use
     // pattern rewrites here instead of dialect conversion because this pass
diff --git a/flang/lib/Optimizer/HLFIR/Transforms/OptimizedBufferization.cpp b/flang/lib/Optimizer/HLFIR/Transforms/OptimizedBufferization.cpp
index 0cfefc2d23ecb..3d506abbaa454 100644
--- a/flang/lib/Optimizer/HLFIR/Transforms/OptimizedBufferization.cpp
+++ b/flang/lib/Optimizer/HLFIR/Transforms/OptimizedBufferization.cpp
@@ -930,9 +930,7 @@ class ReductionConversion : public mlir::OpRewritePattern<Op> {
       llvm_unreachable("unsupported type");
     };
 
-    fir::KindMapping kindMap =
-        fir::getKindMapping(op->template getParentOfType<mlir::ModuleOp>());
-    fir::FirOpBuilder builder{op, kindMap};
+    fir::FirOpBuilder builder{rewriter, op.getOperation()};
 
     mlir::Value init;
     GenBodyFn genBodyFn;
diff --git a/flang/lib/Optimizer/HLFIR/Transforms/SimplifyHLFIRIntrinsics.cpp b/flang/lib/Optimizer/HLFIR/Transforms/SimplifyHLFIRIntrinsics.cpp
index fe7ae0eeed3cc..c1c3839c47e11 100644
--- a/flang/lib/Optimizer/HLFIR/Transforms/SimplifyHLFIRIntrinsics.cpp
+++ b/flang/lib/Optimizer/HLFIR/Transforms/SimplifyHLFIRIntrinsics.cpp
@@ -951,6 +951,219 @@ class DotProductConversion
   }
 };
 
+class ReshapeAsElementalConversion
+    : public mlir::OpRewritePattern<hlfir::ReshapeOp> {
+public:
+  using mlir::OpRewritePattern<hlfir::ReshapeOp>::OpRewritePattern;
+
+  llvm::LogicalResult
+  matchAndRewrite(hlfir::ReshapeOp reshape,
+                  mlir::PatternRewriter &rewriter) const override {
+    // Do not inline RESHAPE with ORDER yet. The runtime implementation
+    // may be good enough, unless the temporary creation overhead
+    // is high.
+    // TODO: If ORDER is constant, then we can still easily inline.
+    // TODO: If the result's rank is 1, then we can assume ORDER == (/1/).
+    if (reshape.getOrder())
+      return rewriter.notifyMatchFailure(reshape,
+                                         "RESHAPE with ORDER argument");
+
+    // Verify that the element types of ARRAY, PAD and the result
+    // match before doing any transformations. For example,
+    // the character types of different lengths may appear in the dead
+    // code, and it just does not make sense to inline hlfir.reshape
+    // in this case (a runtime call might have less code size footprint).
+    hlfir::Entity result = hlfir::Entity{reshape};
+    hlfir::Entity array = hlfir::Entity{reshape.getArray()};
+    mlir::Type elementType = array.getFortranElementType();
+    if (result.getFortranElementType() != elementType)
+      return rewriter.notifyMatchFailure(
+          reshape, "ARRAY and result have different types");
+    mlir::Value pad = reshape.getPad();
+    if (pad && hlfir::getFortranElementType(pad.getType()) != elementType)
+      return rewriter.notifyMatchFailure(reshape,
+                                         "ARRAY and PAD have different types");
+
+    // TODO: selecting between ARRAY and PAD of non-trivial element types
+    // requires more work. We have to select between two references
+    // to elements in ARRAY and PAD. This requires conditional
+    // bufferization of the element, if ARRAY/PAD is an expression.
+    if (pad && !fir::isa_trivial(elementType))
+      return rewriter.notifyMatchFailure(reshape,
+                                         "PAD present with non-trivial type");
+
+    mlir::Location loc = reshape.getLoc();
+    fir::FirOpBuilder builder{rewriter, reshape.getOperation()};
+    // Assume that all the indices arithmetic does not overflow
+    // the IndexType.
+    builder.setIntegerOverflowFlags(mlir::arith::IntegerOverflowFlags::nuw);
+
+    llvm::SmallVector<mlir::Value, 1> typeParams;
+    hlfir::genLengthParameters(loc, builder, array, typeParams);
+
+    // Fetch the extents of ARRAY, PAD and result beforehand.
+    llvm::SmallVector<mlir::Value, Fortran::common::maxRank> arrayExtents =
+        hlfir::genExtentsVector(loc, builder, array);
+
+    // If PAD is present, we have to use array size to start taking
+    // elements from the PAD array.
+    mlir::Value arraySize =
+        pad ? computeArraySize(loc, builder, arrayExtents) : nullptr;
+    hlfir::Entity shape = hlfir::Entity{reshape.getShape()};
+    llvm::SmallVector<mlir::Value, Fortran::common::maxRank> resultExtents;
+    mlir::Type indexType = builder.getIndexType();
+    for (int idx = 0; idx < result.getRank(); ++idx)
+      resultExtents.push_back(hlfir::loadElementAt(
+          loc, builder, shape,
+          builder.createIntegerConstant(loc, indexType, idx + 1)));
+    auto resultShape = builder.create<fir::ShapeOp>(loc, resultExtents);
+
+    auto genKernel = [&](mlir::Location loc, fir::FirOpBuilder &builder,
+                         mlir::ValueRange inputIndices) -> hlfir::Entity {
+      mlir::Value linearIndex =
+          computeLinearIndex(loc, builder, resultExtents, inputIndices);
+      fir::IfOp ifOp;
+      if (pad) {
+        // PAD is present. Check if this element comes from the PAD array.
+        mlir::Value isInsideArray = builder.create<mlir::arith::CmpIOp>(
+            loc, mlir::arith::CmpIPredicate::ult, linearIndex, arraySize);
+        ifOp = builder.create<fir::IfOp>(loc, elementType, isInsideArray,
+                                         /*withElseRegion=*/true);
+
+        // In the 'else' block, return an element from the PAD.
+        builder.setInsertionPointToStart(&ifOp.getElseRegion().front());
+        // PAD is dynamically optional, but we can unconditionally access it
+        // in the 'else' block. If we have to start taking elements from it,
+        // then it must be present in a valid program.
+        llvm::SmallVector<mlir::Value, Fortran::common::maxRank> padExtents =
+            hlfir::genExtentsVector(loc, builder, hlfir::Entity{pad});
+        // Subtract the ARRAY size from the zero-based linear index
+        // to get the zero-based linear index into PAD.
+        mlir::Value padLinearIndex =
+            builder.create<mlir::arith::SubIOp>(loc, linearIndex, arraySize);
+        llvm::SmallVector<mlir::Value, Fortran::common::maxRank> padIndices =
+            delinearizeIndex(loc, builder, padExtents, padLinearIndex,
+                             /*wrapAround=*/true);
+        mlir::Value padElement =
+            hlfir::loadElementAt(loc, builder, hlfir::Entity{pad}, padIndices);
+        builder.create<fir::ResultOp>(loc, padElement);
+
+        // In the 'then' block, return an element from the ARRAY.
+        builder.setInsertionPointToStart(&ifOp.getThenRegion().front());
+      }
+
+      llvm::SmallVector<mlir::Value, Fortran::common::maxRank> arrayIndices =
+          delinearizeIndex(loc, builder, arrayExtents, linearIndex,
+                           /*wrapAround=*/false);
+      mlir::Value arrayElement =
+          hlfir::loadElementAt(loc, builder, array, arrayIndices);
+
+      if (ifOp) {
+        builder.create<fir::ResultOp>(loc, arrayElement);
+        builder.setInsertionPointAfter(ifOp);
+        arrayElement = ifOp.getResult(0);
+      }
+
+      return hlfir::Entity{arrayElement};
+    };
+    hlfir::ElementalOp elementalOp = hlfir::genElementalOp(
+        loc, builder, elementType, resultShape, typeParams, genKernel,
+        /*isUnordered=*/true,
+        /*polymorphicMold=*/result.isPolymorphic() ? array : mlir::Value{},
+        reshape.getResult().getType());
+    assert(elementalOp.getResult().getType() == reshape.getResult().getType());
+    rewriter.replaceOp(reshape, elementalOp);
+    return mlir::success();
+  }
+
+private:
+  /// Compute zero-based linear index given an array extents
+  /// and one-based indices:
+  ///   \p extents: [e0, e1, ..., en]
+  ///   \p indices: [i0, i1, ..., in]
+  ///
+  /// linear-index :=
+  ///   (...((in-1)*e(n-1)+(i(n-1)-1))*e(n-2)+...)*e0+(i0-1)
+  static mlir::Value computeLinearIndex(mlir::Location loc,
+                                        fir::FirOpBuilder &builder,
+                                        mlir::ValueRange extents,
+                                        mlir::ValueRange indices) {
+    std::size_t rank = extents.size();
+    assert(rank == indices.size());
+    mlir::Type indexType = builder.getIndexType();
+    mlir::Value zero = builder.createIntegerConstant(loc, indexType, 0);
+    mlir::Value one = builder.createIntegerConstant(loc, indexType, 1);
+    mlir::Value linearIndex = zero;
+    std::size_t idx = 0;
+    for (auto index : llvm::reverse(indices)) {
+      mlir::Value tmp = builder.create<mlir::arith::SubIOp>(
+          loc, builder.createConvert(loc, indexType, index), one);
+      tmp = builder.create<mlir::arith::AddIOp>(loc, linearIndex, tmp);
+      if (idx + 1 < rank)
+        tmp = builder.create<mlir::arith::MulIOp>(
+            loc, tmp,
+            builder.createConvert(loc, indexType, extents[rank - idx - 2]));
+
+      linearIndex = tmp;
+      ++idx;
+    }
+    return linearIndex;
+  }
+
+  /// Compute one-based array indices from the given zero-based \p linearIndex
+  /// and the array \p extents [e0, e1, ..., en].
+  ///   i0 := linearIndex % e0 + 1
+  ///   linearIndex := linearIndex / e0
+  ///   i1 := linearIndex % e1 + 1
+  ///   linearIndex := linearIndex / e1
+  ///   ...
+  ///   i(n-1) := linearIndex % e(n-1) + 1
+  ///   linearIndex := linearIndex / e(n-1)
+  ///   if (wrapAround) {
+  ///     // If the index is allowed to wrap around, then
+  ///     // we need to modulo it by the last dimension's extent.
+  ///     in := linearIndex % en + 1
+  ///   } else {
+  ///     in := linearIndex + 1
+  ///   }
+  static llvm::SmallVector<mlir::Value, Fortran::common::maxRank>
+  delinearizeIndex(mlir::Location loc, fir::FirOpBuilder &builder,
+                   mlir::ValueRange extents, mlir::Value linearIndex,
+                   bool wrapAround) {
+    llvm::SmallVector<mlir::Value, Fortran::common::maxRank> indices;
+    mlir::Type indexType = builder.getIndexType();
+    mlir::Value one = builder.createIntegerConstant(loc, indexType, 1);
+    linearIndex = builder.createConvert(loc, indexType, linearIndex);
+
+    for (std::size_t dim = 0; dim < extents.size(); ++dim) {
+      mlir::Value extent = builder.createConvert(loc, indexType, extents[dim]);
+      // Avoid the modulo for the last index, unless wrap around is allowed.
+      mlir::Value currentIndex = linearIndex;
+      if (dim != extents.size() - 1 || wrapAround)
+        currentIndex =
+            builder.create<mlir::arith::RemUIOp>(loc, linearIndex, extent);
+      // The result of the last division is unused, so it will be DCEd.
+      linearIndex =
+          builder.create<mlir::arith::DivUIOp>(loc, linearIndex, extent);
+      indices.push_back(
+          builder.create<mlir::arith::AddIOp>(loc, currentIndex, one));
+    }
+    return indices;
+  }
+
+  /// Return size of an array given its extents.
+  static mlir::Value computeArraySize(mlir::Location loc,
+                                      fir::FirOpBuilder &builder,
+                                      mlir::ValueRange extents) {
+    mlir::Type indexType = builder.getIndexType();
+    mlir::Value size = builder.createIntegerConstant(loc, indexType, 1);
+    for (auto extent : extents)
+      size = builder.create<mlir::arith::MulIOp>(
+          loc, size, builder.createConvert(loc, indexType, extent));
+    return size;
+  }
+};
+
 class SimplifyHLFIRIntrinsics
     : public hlfir::impl::SimplifyHLFIRIntrinsicsBase<SimplifyHLFIRIntrinsics> {
 public:
@@ -987,6 +1200,7 @@ class SimplifyHLFIRIntrinsics
       patterns.insert<MatmulConversion<hlfir::MatmulOp>>(context);
 
     patterns.insert<DotProductConversion>(context);
+    patterns.insert<ReshapeAsElementalConversion>(context);
 
     if (mlir::failed(mlir::applyPatternsGreedily(
             getOperation(), std::move(patterns), config))) {
diff --git a/flang/lib/Optimizer/OpenMP/CMakeLists.txt b/flang/lib/Optimizer/OpenMP/CMakeLists.txt
index c8509ccc7fea5..86ae93f3207cc 100644
--- a/flang/lib/Optimizer/OpenMP/CMakeLists.txt
+++ b/flang/lib/Optimizer/OpenMP/CMakeLists.txt
@@ -21,7 +21,7 @@ add_flang_library(FlangOpenMPTransforms
   FIRDialect
   FIRDialectSupport
   FIRSupport
-  FortranCommon
+  FortranSupport
   HLFIRDialect
 
   MLIR_LIBS
diff --git a/flang/lib/Optimizer/OpenMP/GenericLoopConversion.cpp b/flang/lib/Optimizer/OpenMP/GenericLoopConversion.cpp
index 555601c5e92df..3512a537d38c3 100644
--- a/flang/lib/Optimizer/OpenMP/GenericLoopConversion.cpp
+++ b/flang/lib/Optimizer/OpenMP/GenericLoopConversion.cpp
@@ -6,7 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "flang/Common/OpenMP-utils.h"
+#include "flang/Support/OpenMP-utils.h"
 
 #include "mlir/Dialect/Func/IR/FuncOps.h"
 #include "mlir/Dialect/OpenMP/OpenMPDialect.h"
@@ -29,11 +29,7 @@ namespace {
 class GenericLoopConversionPattern
     : public mlir::OpConversionPattern<mlir::omp::LoopOp> {
 public:
-  enum class GenericLoopCombinedInfo {
-    Standalone,
-    TargetTeamsLoop,
-    TargetParallelLoop
-  };
+  enum class GenericLoopCombinedInfo { Standalone, TeamsLoop, ParallelLoop };
 
   using mlir::OpConversionPattern<mlir::omp::LoopOp>::OpConversionPattern;
 
@@ -53,12 +49,13 @@ class GenericLoopConversionPattern
 
     switch (combinedInfo) {
     case GenericLoopCombinedInfo::Standalone:
-      rewriteToSimdLoop(loopOp, rewriter);
+      rewriteStandaloneLoop(loopOp, rewriter);
       break;
-    case GenericLoopCombinedInfo::TargetParallelLoop:
-      llvm_unreachable("not yet implemented: `parallel loop` direcitve");
+    case GenericLoopCombinedInfo::ParallelLoop:
+      llvm_unreachable(
+          "not yet implemented: Combined `parallel loop` directive");
       break;
-    case GenericLoopCombinedInfo::TargetTeamsLoop:
+    case GenericLoopCombinedInfo::TeamsLoop:
       rewriteToDistributeParallelDo(loopOp, rewriter);
       break;
     }
@@ -74,10 +71,10 @@ class GenericLoopConversionPattern
     switch (combinedInfo) {
     case GenericLoopCombinedInfo::Standalone:
       break;
-    case GenericLoopCombinedInfo::TargetParallelLoop:
+    case GenericLoopCombinedInfo::ParallelLoop:
       return loopOp.emitError(
-          "not yet implemented: Combined `omp target parallel loop` directive");
-    case GenericLoopCombinedInfo::TargetTeamsLoop:
+          "not yet implemented: Combined `parallel loop` directive");
+    case GenericLoopCombinedInfo::TeamsLoop:
       break;
     }
 
@@ -87,7 +84,10 @@ class GenericLoopConversionPattern
              << loopOp->getName() << " operation";
     };
 
-    if (loopOp.getBindKind())
+    // For standalone directives, `bind` is already supported. Other combined
+    // forms will be supported in a follow-up PR.
+    if (combinedInfo != GenericLoopCombinedInfo::Standalone &&
+        loopOp.getBindKind())
       return todo("bind");
 
     if (loopOp.getOrder())
@@ -96,7 +96,7 @@ class GenericLoopConversionPattern
     if (!loopOp.getReductionVars().empty())
       return todo("reduction");
 
-    // TODO For `target teams loop`, check similar constrains to what is checked
+    // TODO For `teams loop`, check similar constrains to what is checked
     // by `TeamsLoopChecker` in SemaOpenMP.cpp.
     return mlir::success();
   }
@@ -108,18 +108,36 @@ class GenericLoopConversionPattern
     GenericLoopCombinedInfo result = GenericLoopCombinedInfo::Standalone;
 
     if (auto teamsOp = mlir::dyn_cast_if_present<mlir::omp::TeamsOp>(parentOp))
-      if (mlir::isa_and_present<mlir::omp::TargetOp>(teamsOp->getParentOp()))
-        result = GenericLoopCombinedInfo::TargetTeamsLoop;
+      result = GenericLoopCombinedInfo::TeamsLoop;
 
     if (auto parallelOp =
             mlir::dyn_cast_if_present<mlir::omp::ParallelOp>(parentOp))
-      if (mlir::isa_and_present<mlir::omp::TargetOp>(parallelOp->getParentOp()))
-        result = GenericLoopCombinedInfo::TargetParallelLoop;
+      result = GenericLoopCombinedInfo::ParallelLoop;
 
     return result;
   }
 
-  /// Rewrites standalone `loop` directives to equivalent `simd` constructs.
+  void rewriteStandaloneLoop(mlir::omp::LoopOp loopOp,
+                             mlir::ConversionPatternRewriter &rewriter) const {
+    using namespace mlir::omp;
+    std::optional<ClauseBindKind> bindKind = loopOp.getBindKind();
+
+    if (!bindKind.has_value())
+      return rewriteToSimdLoop(loopOp, rewriter);
+
+    switch (*loopOp.getBindKind()) {
+    case ClauseBindKind::Parallel:
+      return rewriteToWsloop(loopOp, rewriter);
+    case ClauseBindKind::Teams:
+      return rewriteToDistrbute(loopOp, rewriter);
+    case ClauseBindKind::Thread:
+      return rewriteToSimdLoop(loopOp, rewriter);
+    }
+  }
+
+  /// Rewrites standalone `loop` (without `bind` clause or with
+  /// `bind(parallel)`) directives to equivalent `simd` constructs.
+  ///
   /// The reasoning behind this decision is that according to the spec (version
   /// 5.2, section 11.7.1):
   ///
@@ -147,30 +165,57 @@ class GenericLoopConversionPattern
   /// the directive.
   void rewriteToSimdLoop(mlir::omp::LoopOp loopOp,
                          mlir::ConversionPatternRewriter &rewriter) const {
-    loopOp.emitWarning("Detected standalone OpenMP `loop` directive, the "
-                       "associated loop will be rewritten to `simd`.");
-    mlir::omp::SimdOperands simdClauseOps;
-    simdClauseOps.privateVars = loopOp.getPrivateVars();
+    loopOp.emitWarning(
+        "Detected standalone OpenMP `loop` directive with thread binding, "
+        "the associated loop will be rewritten to `simd`.");
+    rewriteToSingleWrapperOp<mlir::omp::SimdOp, mlir::omp::SimdOperands>(
+        loopOp, rewriter);
+  }
+
+  void rewriteToDistrbute(mlir::omp::LoopOp loopOp,
+                          mlir::ConversionPatternRewriter &rewriter) const {
+    rewriteToSingleWrapperOp<mlir::omp::DistributeOp,
+                             mlir::omp::DistributeOperands>(loopOp, rewriter);
+  }
+
+  void rewriteToWsloop(mlir::omp::LoopOp loopOp,
+                       mlir::ConversionPatternRewriter &rewriter) const {
+    rewriteToSingleWrapperOp<mlir::omp::WsloopOp, mlir::omp::WsloopOperands>(
+        loopOp, rewriter);
+  }
+
+  // TODO Suggestion by Sergio: tag auto-generated operations for constructs
+  // that weren't part of the original program, that would be useful
+  // information for debugging purposes later on. This new attribute could be
+  // used for `omp.loop`, but also for `do concurrent` transformations,
+  // `workshare`, `workdistribute`, etc. The tag could be used for all kinds of
+  // auto-generated operations using a dialect attribute (named something like
+  // `omp.origin` or `omp.derived`) and perhaps hold the name of the operation
+  // it was derived from, the reason it was transformed or something like that
+  // we could use when emitting any messages related to it later on.
+  template <typename OpTy, typename OpOperandsTy>
+  void
+  rewriteToSingleWrapperOp(mlir::omp::LoopOp loopOp,
+                           mlir::ConversionPatternRewriter &rewriter) const {
+    OpOperandsTy clauseOps;
+    clauseOps.privateVars = loopOp.getPrivateVars();
 
     auto privateSyms = loopOp.getPrivateSyms();
     if (privateSyms)
-      simdClauseOps.privateSyms.assign(privateSyms->begin(),
-                                       privateSyms->end());
+      clauseOps.privateSyms.assign(privateSyms->begin(), privateSyms->end());
 
-    Fortran::common::openmp::EntryBlockArgs simdArgs;
-    simdArgs.priv.vars = simdClauseOps.privateVars;
+    Fortran::common::openmp::EntryBlockArgs args;
+    args.priv.vars = clauseOps.privateVars;
 
-    auto simdOp =
-        rewriter.create<mlir::omp::SimdOp>(loopOp.getLoc(), simdClauseOps);
-    mlir::Block *simdBlock =
-        genEntryBlock(rewriter, simdArgs, simdOp.getRegion());
+    auto wrapperOp = rewriter.create<OpTy>(loopOp.getLoc(), clauseOps);
+    mlir::Block *opBlock = genEntryBlock(rewriter, args, wrapperOp.getRegion());
 
     mlir::IRMapping mapper;
     mlir::Block &loopBlock = *loopOp.getRegion().begin();
 
-    for (auto [loopOpArg, simdopArg] :
-         llvm::zip_equal(loopBlock.getArguments(), simdBlock->getArguments()))
-      mapper.map(loopOpArg, simdopArg);
+    for (auto [loopOpArg, opArg] :
+         llvm::zip_equal(loopBlock.getArguments(), opBlock->getArguments()))
+      mapper.map(loopOpArg, opArg);
 
     rewriter.clone(*loopOp.begin(), mapper);
   }
diff --git a/flang/lib/Optimizer/OpenMP/MapsForPrivatizedSymbols.cpp b/flang/lib/Optimizer/OpenMP/MapsForPrivatizedSymbols.cpp
index c990bebcabde4..963ae863c1fc5 100644
--- a/flang/lib/Optimizer/OpenMP/MapsForPrivatizedSymbols.cpp
+++ b/flang/lib/Optimizer/OpenMP/MapsForPrivatizedSymbols.cpp
@@ -55,15 +55,17 @@ class MapsForPrivatizedSymbolsPass
         std::underlying_type_t<llvm::omp::OpenMPOffloadMappingFlags>>(
         llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_TO);
     Operation *definingOp = var.getDefiningOp();
-    auto declOp = llvm::dyn_cast_or_null<hlfir::DeclareOp>(definingOp);
-    assert(declOp &&
-           "Expected defining Op of privatized var to be hlfir.declare");
 
+    Value varPtr = var;
     // We want the first result of the hlfir.declare op because our goal
     // is to map the descriptor (fir.box or fir.boxchar) and the first
     // result for hlfir.declare is the descriptor if a the symbol being
     // decalred needs a descriptor.
-    Value varPtr = declOp.getBase();
+    // Some types are boxed immediately before privatization. These have other
+    // operations in between the privatization and the declaration. It is safe
+    // to use var directly here because they will be boxed anyway.
+    if (auto declOp = llvm::dyn_cast_if_present<hlfir::DeclareOp>(definingOp))
+      varPtr = declOp.getBase();
 
     // If we do not have a reference to descritor, but the descriptor itself
     // then we need to store that on the stack so that we can map the
diff --git a/flang/lib/Optimizer/Passes/CMakeLists.txt b/flang/lib/Optimizer/Passes/CMakeLists.txt
index eb25beba309bf..1c19a5765aff1 100644
--- a/flang/lib/Optimizer/Passes/CMakeLists.txt
+++ b/flang/lib/Optimizer/Passes/CMakeLists.txt
@@ -5,14 +5,11 @@ add_flang_library(flangPasses
   CommandLineOpts.cpp
   Pipelines.cpp
 
-  DEPENDS
-  FortranCommon
-
   LINK_LIBS
   FIRCodeGen
   FIRTransforms
   FlangOpenMPTransforms
-  FortranCommon
+  FortranSupport
   HLFIRTransforms
 
   LINK_COMPONENTS
diff --git a/flang/lib/Optimizer/Passes/Pipelines.cpp b/flang/lib/Optimizer/Passes/Pipelines.cpp
index 1cc3f0b81c20a..d55ad9e603ffa 100644
--- a/flang/lib/Optimizer/Passes/Pipelines.cpp
+++ b/flang/lib/Optimizer/Passes/Pipelines.cpp
@@ -245,7 +245,15 @@ void createHLFIRToFIRPassPipeline(mlir::PassManager &pm, bool enableOpenMP,
   }
   pm.addPass(hlfir::createLowerHLFIROrderedAssignments());
   pm.addPass(hlfir::createLowerHLFIRIntrinsics());
-  pm.addPass(hlfir::createBufferizeHLFIR());
+
+  hlfir::BufferizeHLFIROptions bufferizeOptions;
+  // For opt-for-speed, avoid running any of the loops resulting
+  // from hlfir.elemental lowering, if the result is an empty array.
+  // This helps to avoid long running loops for elementals with
+  // shapes like (0, HUGE).
+  if (optLevel.isOptimizingForSpeed())
+    bufferizeOptions.optimizeEmptyElementals = true;
+  pm.addPass(hlfir::createBufferizeHLFIR(bufferizeOptions));
   // Run hlfir.assign inlining again after BufferizeHLFIR,
   // because the latter may introduce new hlfir.assign operations,
   // e.g. for copying an array into a temporary due to
diff --git a/flang/lib/Optimizer/Support/DataLayout.cpp b/flang/lib/Optimizer/Support/DataLayout.cpp
index 93a3b92d08105..f89ce5984b919 100644
--- a/flang/lib/Optimizer/Support/DataLayout.cpp
+++ b/flang/lib/Optimizer/Support/DataLayout.cpp
@@ -10,6 +10,7 @@
 #include "flang/Optimizer/Dialect/Support/FIRContext.h"
 #include "flang/Optimizer/Support/FatalError.h"
 #include "mlir/Dialect/DLTI/DLTI.h"
+#include "mlir/Dialect/GPU/IR/GPUDialect.h"
 #include "mlir/Dialect/LLVMIR/LLVMDialect.h"
 #include "mlir/IR/BuiltinOps.h"
 #include "mlir/Interfaces/DataLayoutInterfaces.h"
@@ -20,8 +21,9 @@
 #include "llvm/Support/TargetSelect.h"
 #include "llvm/Target/TargetMachine.h"
 
-void fir::support::setMLIRDataLayout(mlir::ModuleOp mlirModule,
-                                     const llvm::DataLayout &dl) {
+namespace {
+template <typename ModOpTy>
+static void setDataLayout(ModOpTy mlirModule, const llvm::DataLayout &dl) {
   mlir::MLIRContext *context = mlirModule.getContext();
   mlirModule->setAttr(
       mlir::LLVM::LLVMDialect::getDataLayoutAttrName(),
@@ -30,12 +32,14 @@ void fir::support::setMLIRDataLayout(mlir::ModuleOp mlirModule,
   mlirModule->setAttr(mlir::DLTIDialect::kDataLayoutAttrName, dlSpec);
 }
 
-void fir::support::setMLIRDataLayoutFromAttributes(mlir::ModuleOp mlirModule,
-                                                   bool allowDefaultLayout) {
+template <typename ModOpTy>
+static void setDataLayoutFromAttributes(ModOpTy mlirModule,
+                                        bool allowDefaultLayout) {
   if (mlirModule.getDataLayoutSpec())
     return; // Already set.
-  if (auto dataLayoutString = mlirModule->getAttrOfType<mlir::StringAttr>(
-          mlir::LLVM::LLVMDialect::getDataLayoutAttrName())) {
+  if (auto dataLayoutString =
+          mlirModule->template getAttrOfType<mlir::StringAttr>(
+              mlir::LLVM::LLVMDialect::getDataLayoutAttrName())) {
     llvm::DataLayout llvmDataLayout(dataLayoutString);
     fir::support::setMLIRDataLayout(mlirModule, llvmDataLayout);
     return;
@@ -46,15 +50,48 @@ void fir::support::setMLIRDataLayoutFromAttributes(mlir::ModuleOp mlirModule,
   fir::support::setMLIRDataLayout(mlirModule, llvmDataLayout);
 }
 
-std::optional<mlir::DataLayout>
-fir::support::getOrSetDataLayout(mlir::ModuleOp mlirModule,
-                                 bool allowDefaultLayout) {
-  if (!mlirModule.getDataLayoutSpec()) {
+template <typename ModOpTy>
+static std::optional<mlir::DataLayout>
+getOrSetDataLayout(ModOpTy mlirModule, bool allowDefaultLayout) {
+  if (!mlirModule.getDataLayoutSpec())
     fir::support::setMLIRDataLayoutFromAttributes(mlirModule,
                                                   allowDefaultLayout);
-    if (!mlirModule.getDataLayoutSpec()) {
-      return std::nullopt;
-    }
-  }
+  if (!mlirModule.getDataLayoutSpec() &&
+      !mlir::isa<mlir::gpu::GPUModuleOp>(mlirModule))
+    return std::nullopt;
   return mlir::DataLayout(mlirModule);
 }
+
+} // namespace
+
+void fir::support::setMLIRDataLayout(mlir::ModuleOp mlirModule,
+                                     const llvm::DataLayout &dl) {
+  setDataLayout(mlirModule, dl);
+}
+
+void fir::support::setMLIRDataLayout(mlir::gpu::GPUModuleOp mlirModule,
+                                     const llvm::DataLayout &dl) {
+  setDataLayout(mlirModule, dl);
+}
+
+void fir::support::setMLIRDataLayoutFromAttributes(mlir::ModuleOp mlirModule,
+                                                   bool allowDefaultLayout) {
+  setDataLayoutFromAttributes(mlirModule, allowDefaultLayout);
+}
+
+void fir::support::setMLIRDataLayoutFromAttributes(
+    mlir::gpu::GPUModuleOp mlirModule, bool allowDefaultLayout) {
+  setDataLayoutFromAttributes(mlirModule, allowDefaultLayout);
+}
+
+std::optional<mlir::DataLayout>
+fir::support::getOrSetMLIRDataLayout(mlir::ModuleOp mlirModule,
+                                     bool allowDefaultLayout) {
+  return getOrSetDataLayout(mlirModule, allowDefaultLayout);
+}
+
+std::optional<mlir::DataLayout>
+fir::support::getOrSetMLIRDataLayout(mlir::gpu::GPUModuleOp mlirModule,
+                                     bool allowDefaultLayout) {
+  return getOrSetDataLayout(mlirModule, allowDefaultLayout);
+}
diff --git a/flang/lib/Optimizer/Transforms/AbstractResult.cpp b/flang/lib/Optimizer/Transforms/AbstractResult.cpp
index b0327cc10e9de..f8badfa639f94 100644
--- a/flang/lib/Optimizer/Transforms/AbstractResult.cpp
+++ b/flang/lib/Optimizer/Transforms/AbstractResult.cpp
@@ -147,6 +147,7 @@ class CallConversion : public mlir::OpRewritePattern<Op> {
       newResultTypes.emplace_back(getVoidPtrType(result.getContext()));
 
     Op newOp;
+    // TODO: propagate argument and result attributes (need to be shifted).
     // fir::CallOp specific handling.
     if constexpr (std::is_same_v<Op, fir::CallOp>) {
       if (op.getCallee()) {
@@ -189,9 +190,11 @@ class CallConversion : public mlir::OpRewritePattern<Op> {
       if (op.getPassArgPos())
         passArgPos =
             rewriter.getI32IntegerAttr(*op.getPassArgPos() + passArgShift);
+      // TODO: propagate argument and result attributes (need to be shifted).
       newOp = rewriter.create<fir::DispatchOp>(
           loc, newResultTypes, rewriter.getStringAttr(op.getMethod()),
           op.getOperands()[0], newOperands, passArgPos,
+          /*arg_attrs=*/nullptr, /*res_attrs=*/nullptr,
           op.getProcedureAttrsAttr());
     }
 
diff --git a/flang/lib/Optimizer/Transforms/AddDebugInfo.cpp b/flang/lib/Optimizer/Transforms/AddDebugInfo.cpp
index a8e9d198ccb97..49c9a737b2c59 100644
--- a/flang/lib/Optimizer/Transforms/AddDebugInfo.cpp
+++ b/flang/lib/Optimizer/Transforms/AddDebugInfo.cpp
@@ -12,7 +12,6 @@
 //===----------------------------------------------------------------------===//
 
 #include "DebugTypeGenerator.h"
-#include "flang/Common/Version.h"
 #include "flang/Optimizer/Builder/FIRBuilder.h"
 #include "flang/Optimizer/Builder/Todo.h"
 #include "flang/Optimizer/CodeGen/CGOps.h"
@@ -23,6 +22,7 @@
 #include "flang/Optimizer/Dialect/Support/FIRContext.h"
 #include "flang/Optimizer/Support/InternalNames.h"
 #include "flang/Optimizer/Transforms/Passes.h"
+#include "flang/Support/Version.h"
 #include "mlir/Dialect/Func/IR/FuncOps.h"
 #include "mlir/Dialect/LLVMIR/LLVMDialect.h"
 #include "mlir/IR/Matchers.h"
@@ -59,10 +59,19 @@ class AddDebugInfoPass : public fir::impl::AddDebugInfoBase<AddDebugInfoPass> {
 
 private:
   llvm::StringMap<mlir::LLVM::DIModuleAttr> moduleMap;
+  llvm::StringMap<mlir::LLVM::DICommonBlockAttr> commonBlockMap;
+  // List of GlobalVariableExpressionAttr that are attached to a given global
+  // that represents the storage for common block.
+  llvm::DenseMap<fir::GlobalOp, llvm::SmallVector<mlir::Attribute>>
+      globalToGlobalExprsMap;
 
   mlir::LLVM::DIModuleAttr getOrCreateModuleAttr(
       const std::string &name, mlir::LLVM::DIFileAttr fileAttr,
       mlir::LLVM::DIScopeAttr scope, unsigned line, bool decl);
+  mlir::LLVM::DICommonBlockAttr
+  getOrCreateCommonBlockAttr(llvm::StringRef name,
+                             mlir::LLVM::DIFileAttr fileAttr,
+                             mlir::LLVM::DIScopeAttr scope, unsigned line);
 
   void handleGlobalOp(fir::GlobalOp glocalOp, mlir::LLVM::DIFileAttr fileAttr,
                       mlir::LLVM::DIScopeAttr scope,
@@ -73,6 +82,12 @@ class AddDebugInfoPass : public fir::impl::AddDebugInfoBase<AddDebugInfoPass> {
                     mlir::LLVM::DICompileUnitAttr cuAttr,
                     fir::DebugTypeGenerator &typeGen,
                     mlir::SymbolTable *symbolTable);
+  bool createCommonBlockGlobal(fir::cg::XDeclareOp declOp,
+                               const std::string &name,
+                               mlir::LLVM::DIFileAttr fileAttr,
+                               mlir::LLVM::DIScopeAttr scopeAttr,
+                               fir::DebugTypeGenerator &typeGen,
+                               mlir::SymbolTable *symbolTable);
   std::optional<mlir::LLVM::DIModuleAttr>
   getModuleAttrFromGlobalOp(fir::GlobalOp globalOp,
                             mlir::LLVM::DIFileAttr fileAttr,
@@ -90,6 +105,67 @@ bool debugInfoIsAlreadySet(mlir::Location loc) {
 
 } // namespace
 
+bool AddDebugInfoPass::createCommonBlockGlobal(
+    fir::cg::XDeclareOp declOp, const std::string &name,
+    mlir::LLVM::DIFileAttr fileAttr, mlir::LLVM::DIScopeAttr scopeAttr,
+    fir::DebugTypeGenerator &typeGen, mlir::SymbolTable *symbolTable) {
+  mlir::MLIRContext *context = &getContext();
+  mlir::OpBuilder builder(context);
+  std::optional<std::int64_t> optint;
+  mlir::Operation *op = declOp.getMemref().getDefiningOp();
+
+  if (auto conOp = mlir::dyn_cast_if_present<fir::ConvertOp>(op))
+    op = conOp.getValue().getDefiningOp();
+
+  if (auto cordOp = mlir::dyn_cast_if_present<fir::CoordinateOp>(op)) {
+    optint = fir::getIntIfConstant(cordOp.getOperand(1));
+    if (!optint)
+      return false;
+    op = cordOp.getRef().getDefiningOp();
+    if (auto conOp2 = mlir::dyn_cast_if_present<fir::ConvertOp>(op))
+      op = conOp2.getValue().getDefiningOp();
+
+    if (auto addrOfOp = mlir::dyn_cast_if_present<fir::AddrOfOp>(op)) {
+      mlir::SymbolRefAttr sym = addrOfOp.getSymbol();
+      if (auto global =
+              symbolTable->lookup<fir::GlobalOp>(sym.getRootReference())) {
+
+        unsigned line = getLineFromLoc(global.getLoc());
+        llvm::StringRef commonName(sym.getRootReference());
+        // FIXME: We are trying to extract the name of the common block from the
+        // name of the global. As part of mangling, GetCommonBlockObjectName can
+        // add a trailing _ in the name of that global. The demangle function
+        // does not seem to handle such cases. So the following hack is used to
+        // remove the trailing '_'.
+        if (commonName != Fortran::common::blankCommonObjectName &&
+            commonName.back() == '_')
+          commonName = commonName.drop_back();
+        mlir::LLVM::DICommonBlockAttr commonBlock =
+            getOrCreateCommonBlockAttr(commonName, fileAttr, scopeAttr, line);
+        mlir::LLVM::DITypeAttr diType = typeGen.convertType(
+            fir::unwrapRefType(declOp.getType()), fileAttr, scopeAttr, declOp);
+        line = getLineFromLoc(declOp.getLoc());
+        auto gvAttr = mlir::LLVM::DIGlobalVariableAttr::get(
+            context, commonBlock, mlir::StringAttr::get(context, name),
+            declOp.getUniqName(), fileAttr, line, diType,
+            /*isLocalToUnit*/ false, /*isDefinition*/ true, /* alignInBits*/ 0);
+        mlir::LLVM::DIExpressionAttr expr;
+        if (*optint != 0) {
+          llvm::SmallVector<mlir::LLVM::DIExpressionElemAttr> ops;
+          ops.push_back(mlir::LLVM::DIExpressionElemAttr::get(
+              context, llvm::dwarf::DW_OP_plus_uconst, *optint));
+          expr = mlir::LLVM::DIExpressionAttr::get(context, ops);
+        }
+        auto dbgExpr = mlir::LLVM::DIGlobalVariableExpressionAttr::get(
+            global.getContext(), gvAttr, expr);
+        globalToGlobalExprsMap[global].push_back(dbgExpr);
+        return true;
+      }
+    }
+  }
+  return false;
+}
+
 void AddDebugInfoPass::handleDeclareOp(fir::cg::XDeclareOp declOp,
                                        mlir::LLVM::DIFileAttr fileAttr,
                                        mlir::LLVM::DIScopeAttr scopeAttr,
@@ -101,16 +177,21 @@ void AddDebugInfoPass::handleDeclareOp(fir::cg::XDeclareOp declOp,
 
   if (result.first != fir::NameUniquer::NameKind::VARIABLE)
     return;
-  // If this DeclareOp actually represents a global then treat it as such.
-  if (auto global = symbolTable->lookup<fir::GlobalOp>(declOp.getUniqName())) {
-    handleGlobalOp(global, fileAttr, scopeAttr, typeGen, symbolTable, declOp);
-    return;
-  }
 
-  // Only accept local variables.
-  if (result.second.procs.empty())
+  if (createCommonBlockGlobal(declOp, result.second.name, fileAttr, scopeAttr,
+                              typeGen, symbolTable))
     return;
 
+  // If this DeclareOp actually represents a global then treat it as such.
+  mlir::Operation *defOp = declOp.getMemref().getDefiningOp();
+  if (defOp && llvm::isa<fir::AddrOfOp>(defOp)) {
+    if (auto global =
+            symbolTable->lookup<fir::GlobalOp>(declOp.getUniqName())) {
+      handleGlobalOp(global, fileAttr, scopeAttr, typeGen, symbolTable, declOp);
+      return;
+    }
+  }
+
   // FIXME: There may be cases where an argument is processed a bit before
   // DeclareOp is generated. In that case, DeclareOp may point to an
   // intermediate op and not to BlockArgument.
@@ -136,6 +217,22 @@ void AddDebugInfoPass::handleDeclareOp(fir::cg::XDeclareOp declOp,
   declOp->setLoc(builder.getFusedLoc({declOp->getLoc()}, localVarAttr));
 }
 
+mlir::LLVM::DICommonBlockAttr AddDebugInfoPass::getOrCreateCommonBlockAttr(
+    llvm::StringRef name, mlir::LLVM::DIFileAttr fileAttr,
+    mlir::LLVM::DIScopeAttr scope, unsigned line) {
+  mlir::MLIRContext *context = &getContext();
+  mlir::LLVM::DICommonBlockAttr cbAttr;
+  if (auto iter{commonBlockMap.find(name)}; iter != commonBlockMap.end()) {
+    cbAttr = iter->getValue();
+  } else {
+    cbAttr = mlir::LLVM::DICommonBlockAttr::get(
+        context, scope, nullptr, mlir::StringAttr::get(context, name), fileAttr,
+        line);
+    commonBlockMap[name] = cbAttr;
+  }
+  return cbAttr;
+}
+
 // The `module` does not have a first class representation in the `FIR`. We
 // extract information about it from the name of the identifiers and keep a
 // map to avoid duplication.
@@ -227,7 +324,10 @@ void AddDebugInfoPass::handleGlobalOp(fir::GlobalOp globalOp,
       mlir::StringAttr::get(context, globalOp.getName()), fileAttr, line,
       diType, /*isLocalToUnit*/ false,
       /*isDefinition*/ globalOp.isInitialized(), /* alignInBits*/ 0);
-  globalOp->setLoc(builder.getFusedLoc({globalOp->getLoc()}, gvAttr));
+  auto dbgExpr = mlir::LLVM::DIGlobalVariableExpressionAttr::get(
+      globalOp.getContext(), gvAttr, nullptr);
+  auto arrayAttr = mlir::ArrayAttr::get(context, {dbgExpr});
+  globalOp->setLoc(builder.getFusedLoc({globalOp.getLoc()}, arrayAttr));
 }
 
 void AddDebugInfoPass::handleFuncOp(mlir::func::FuncOp funcOp,
@@ -409,6 +509,11 @@ void AddDebugInfoPass::handleFuncOp(mlir::func::FuncOp funcOp,
     if (&funcOp.front() == declOp->getBlock())
       handleDeclareOp(declOp, fileAttr, spAttr, typeGen, symbolTable);
   });
+  // commonBlockMap ensures that we don't create multiple DICommonBlockAttr of
+  // the same name in one function. But it is ok (rather required) to create
+  // them in different functions if common block of the same name has been used
+  // there.
+  commonBlockMap.clear();
 }
 
 void AddDebugInfoPass::runOnOperation() {
@@ -418,7 +523,7 @@ void AddDebugInfoPass::runOnOperation() {
   llvm::StringRef fileName;
   std::string filePath;
   std::optional<mlir::DataLayout> dl =
-      fir::support::getOrSetDataLayout(module, /*allowDefaultLayout=*/true);
+      fir::support::getOrSetMLIRDataLayout(module, /*allowDefaultLayout=*/true);
   if (!dl) {
     mlir::emitError(module.getLoc(), "Missing data layout attribute in module");
     signalPassFailure();
@@ -461,6 +566,13 @@ void AddDebugInfoPass::runOnOperation() {
   module.walk([&](mlir::func::FuncOp funcOp) {
     handleFuncOp(funcOp, fileAttr, cuAttr, typeGen, &symbolTable);
   });
+  mlir::OpBuilder builder(context);
+  // We have processed all function. Attach common block variables to the
+  // global that represent the storage.
+  for (auto [global, exprs] : globalToGlobalExprsMap) {
+    auto arrayAttr = mlir::ArrayAttr::get(context, exprs);
+    global->setLoc(builder.getFusedLoc({global.getLoc()}, arrayAttr));
+  }
   // Process any global which was not processed through DeclareOp.
   if (debugLevel == mlir::LLVM::DIEmissionKind::Full) {
     // Process 'GlobalOp' only if full debug info is requested.
diff --git a/flang/lib/Optimizer/Transforms/AssumedRankOpConversion.cpp b/flang/lib/Optimizer/Transforms/AssumedRankOpConversion.cpp
index eb59045a5fde7..d0bd67a236419 100644
--- a/flang/lib/Optimizer/Transforms/AssumedRankOpConversion.cpp
+++ b/flang/lib/Optimizer/Transforms/AssumedRankOpConversion.cpp
@@ -6,7 +6,6 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "flang/Common/Fortran.h"
 #include "flang/Lower/BuiltinModules.h"
 #include "flang/Optimizer/Builder/FIRBuilder.h"
 #include "flang/Optimizer/Builder/Runtime/Support.h"
@@ -17,6 +16,7 @@
 #include "flang/Optimizer/Support/Utils.h"
 #include "flang/Optimizer/Transforms/Passes.h"
 #include "flang/Runtime/support.h"
+#include "flang/Support/Fortran.h"
 #include "mlir/Dialect/Func/IR/FuncOps.h"
 #include "mlir/Pass/Pass.h"
 #include "mlir/Transforms/DialectConversion.h"
diff --git a/flang/lib/Optimizer/Transforms/CMakeLists.txt b/flang/lib/Optimizer/Transforms/CMakeLists.txt
index 9c550f983434a..50994fbb21700 100644
--- a/flang/lib/Optimizer/Transforms/CMakeLists.txt
+++ b/flang/lib/Optimizer/Transforms/CMakeLists.txt
@@ -46,7 +46,7 @@ add_flang_library(FIRTransforms
   FIRDialect
   FIRDialectSupport
   FIRSupport
-  FortranCommon
+  FortranSupport
   HLFIRDialect
 
   MLIR_LIBS
diff --git a/flang/lib/Optimizer/Transforms/CUFAddConstructor.cpp b/flang/lib/Optimizer/Transforms/CUFAddConstructor.cpp
index 97551595db039..43ef6822de302 100644
--- a/flang/lib/Optimizer/Transforms/CUFAddConstructor.cpp
+++ b/flang/lib/Optimizer/Transforms/CUFAddConstructor.cpp
@@ -57,7 +57,7 @@ struct CUFAddConstructor
     auto funcTy =
         mlir::LLVM::LLVMFunctionType::get(voidTy, {}, /*isVarArg=*/false);
     std::optional<mlir::DataLayout> dl =
-        fir::support::getOrSetDataLayout(mod, /*allowDefaultLayout=*/false);
+        fir::support::getOrSetMLIRDataLayout(mod, /*allowDefaultLayout=*/false);
     if (!dl) {
       mlir::emitError(mod.getLoc(),
                       "data layout attribute is required to perform " +
diff --git a/flang/lib/Optimizer/Transforms/CUFDeviceGlobal.cpp b/flang/lib/Optimizer/Transforms/CUFDeviceGlobal.cpp
index 7486dde0e281e..3f13a182ad0c3 100644
--- a/flang/lib/Optimizer/Transforms/CUFDeviceGlobal.cpp
+++ b/flang/lib/Optimizer/Transforms/CUFDeviceGlobal.cpp
@@ -6,7 +6,6 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "flang/Common/Fortran.h"
 #include "flang/Optimizer/Builder/CUFCommon.h"
 #include "flang/Optimizer/Dialect/CUF/CUFOps.h"
 #include "flang/Optimizer/Dialect/FIRDialect.h"
@@ -15,6 +14,7 @@
 #include "flang/Optimizer/Support/InternalNames.h"
 #include "flang/Runtime/CUDA/common.h"
 #include "flang/Runtime/allocatable.h"
+#include "flang/Support/Fortran.h"
 #include "mlir/Dialect/LLVMIR/NVVMDialect.h"
 #include "mlir/IR/SymbolTable.h"
 #include "mlir/Pass/Pass.h"
diff --git a/flang/lib/Optimizer/Transforms/CUFGPUToLLVMConversion.cpp b/flang/lib/Optimizer/Transforms/CUFGPUToLLVMConversion.cpp
index 60aa401e1cc8c..2a95e41944f3f 100644
--- a/flang/lib/Optimizer/Transforms/CUFGPUToLLVMConversion.cpp
+++ b/flang/lib/Optimizer/Transforms/CUFGPUToLLVMConversion.cpp
@@ -7,10 +7,10 @@
 //===----------------------------------------------------------------------===//
 
 #include "flang/Optimizer/Transforms/CUFGPUToLLVMConversion.h"
-#include "flang/Common/Fortran.h"
 #include "flang/Optimizer/CodeGen/TypeConverter.h"
 #include "flang/Optimizer/Support/DataLayout.h"
 #include "flang/Runtime/CUDA/common.h"
+#include "flang/Support/Fortran.h"
 #include "mlir/Conversion/LLVMCommon/Pattern.h"
 #include "mlir/Dialect/GPU/IR/GPUDialect.h"
 #include "mlir/Pass/Pass.h"
@@ -139,20 +139,26 @@ struct GPULaunchKernelConversion
                            adaptor.getBlockSizeY(), adaptor.getBlockSizeZ(),
                            dynamicMemorySize, kernelArgs, nullPtr});
     } else {
-      auto funcOp = mod.lookupSymbol<mlir::LLVM::LLVMFuncOp>(
-          RTNAME_STRING(CUFLaunchKernel));
+      auto procAttr =
+          op->getAttrOfType<cuf::ProcAttributeAttr>(cuf::getProcAttrName());
+      bool isGridGlobal =
+          procAttr && procAttr.getValue() == cuf::ProcAttribute::GridGlobal;
+      llvm::StringRef fctName = isGridGlobal
+                                    ? RTNAME_STRING(CUFLaunchCooperativeKernel)
+                                    : RTNAME_STRING(CUFLaunchKernel);
+      auto funcOp = mod.lookupSymbol<mlir::LLVM::LLVMFuncOp>(fctName);
       auto funcTy = mlir::LLVM::LLVMFunctionType::get(
           voidTy,
           {ptrTy, llvmIntPtrType, llvmIntPtrType, llvmIntPtrType,
            llvmIntPtrType, llvmIntPtrType, llvmIntPtrType, i32Ty, ptrTy, ptrTy},
           /*isVarArg=*/false);
-      auto cufLaunchKernel = mlir::SymbolRefAttr::get(
-          mod.getContext(), RTNAME_STRING(CUFLaunchKernel));
+      auto cufLaunchKernel =
+          mlir::SymbolRefAttr::get(mod.getContext(), fctName);
       if (!funcOp) {
         mlir::OpBuilder::InsertionGuard insertGuard(rewriter);
         rewriter.setInsertionPointToStart(mod.getBody());
-        auto launchKernelFuncOp = rewriter.create<mlir::LLVM::LLVMFuncOp>(
-            loc, RTNAME_STRING(CUFLaunchKernel), funcTy);
+        auto launchKernelFuncOp =
+            rewriter.create<mlir::LLVM::LLVMFuncOp>(loc, fctName, funcTy);
         launchKernelFuncOp.setVisibility(
             mlir::SymbolTable::Visibility::Private);
       }
@@ -182,8 +188,8 @@ class CUFGPUToLLVMConversion
     if (!module)
       return signalPassFailure();
 
-    std::optional<mlir::DataLayout> dl =
-        fir::support::getOrSetDataLayout(module, /*allowDefaultLayout=*/false);
+    std::optional<mlir::DataLayout> dl = fir::support::getOrSetMLIRDataLayout(
+        module, /*allowDefaultLayout=*/false);
     fir::LLVMTypeConverter typeConverter(module, /*applyTBAA=*/false,
                                          /*forceUnifiedTBAATree=*/false, *dl);
     cuf::populateCUFGPUToLLVMConversionPatterns(typeConverter, patterns);
diff --git a/flang/lib/Optimizer/Transforms/CUFOpConversion.cpp b/flang/lib/Optimizer/Transforms/CUFOpConversion.cpp
index 7292ce741b85b..1f0576aa82f83 100644
--- a/flang/lib/Optimizer/Transforms/CUFOpConversion.cpp
+++ b/flang/lib/Optimizer/Transforms/CUFOpConversion.cpp
@@ -7,7 +7,6 @@
 //===----------------------------------------------------------------------===//
 
 #include "flang/Optimizer/Transforms/CUFOpConversion.h"
-#include "flang/Common/Fortran.h"
 #include "flang/Optimizer/Builder/CUFCommon.h"
 #include "flang/Optimizer/Builder/Runtime/RTBuilder.h"
 #include "flang/Optimizer/CodeGen/TypeConverter.h"
@@ -22,6 +21,7 @@
 #include "flang/Runtime/CUDA/memory.h"
 #include "flang/Runtime/CUDA/pointer.h"
 #include "flang/Runtime/allocatable.h"
+#include "flang/Support/Fortran.h"
 #include "mlir/Conversion/LLVMCommon/Pattern.h"
 #include "mlir/Dialect/GPU/IR/GPUDialect.h"
 #include "mlir/IR/Matchers.h"
@@ -103,7 +103,7 @@ static mlir::LogicalResult convertOpToCall(OpTy op,
   mlir::Value sourceLine;
   if constexpr (std::is_same_v<OpTy, cuf::AllocateOp>)
     sourceLine = fir::factory::locationToLineNo(
-        builder, loc, op.getSource() ? fTy.getInput(6) : fTy.getInput(5));
+        builder, loc, op.getSource() ? fTy.getInput(7) : fTy.getInput(6));
   else
     sourceLine = fir::factory::locationToLineNo(builder, loc, fTy.getInput(4));
 
@@ -119,22 +119,28 @@ static mlir::LogicalResult convertOpToCall(OpTy op,
   }
   llvm::SmallVector<mlir::Value> args;
   if constexpr (std::is_same_v<OpTy, cuf::AllocateOp>) {
+    mlir::Value pinned =
+        op.getPinned()
+            ? op.getPinned()
+            : builder.createNullConstant(
+                  loc, fir::ReferenceType::get(
+                           mlir::IntegerType::get(op.getContext(), 1)));
     if (op.getSource()) {
       mlir::Value stream =
           op.getStream()
               ? op.getStream()
               : builder.createIntegerConstant(loc, fTy.getInput(2), -1);
-      args = fir::runtime::createArguments(builder, loc, fTy, op.getBox(),
-                                           op.getSource(), stream, hasStat,
-                                           errmsg, sourceFile, sourceLine);
+      args = fir::runtime::createArguments(
+          builder, loc, fTy, op.getBox(), op.getSource(), stream, pinned,
+          hasStat, errmsg, sourceFile, sourceLine);
     } else {
       mlir::Value stream =
           op.getStream()
               ? op.getStream()
               : builder.createIntegerConstant(loc, fTy.getInput(1), -1);
       args = fir::runtime::createArguments(builder, loc, fTy, op.getBox(),
-                                           stream, hasStat, errmsg, sourceFile,
-                                           sourceLine);
+                                           stream, pinned, hasStat, errmsg,
+                                           sourceFile, sourceLine);
     }
   } else {
     args =
@@ -153,11 +159,6 @@ struct CUFAllocateOpConversion
   mlir::LogicalResult
   matchAndRewrite(cuf::AllocateOp op,
                   mlir::PatternRewriter &rewriter) const override {
-    // TODO: Pinned is a reference to a logical value that can be set to true
-    // when pinned allocation succeed. This will require a new entry point.
-    if (op.getPinned())
-      return mlir::failure();
-
     auto mod = op->getParentOfType<mlir::ModuleOp>();
     fir::FirOpBuilder builder(rewriter, mod);
     mlir::Location loc = op.getLoc();
@@ -294,19 +295,22 @@ struct CUFAllocOpConversion : public mlir::OpRewritePattern<cuf::AllocOp> {
   matchAndRewrite(cuf::AllocOp op,
                   mlir::PatternRewriter &rewriter) const override {
 
+    mlir::Location loc = op.getLoc();
+
     if (inDeviceContext(op.getOperation())) {
       // In device context just replace the cuf.alloc operation with a fir.alloc
       // the cuf.free will be removed.
-      rewriter.replaceOpWithNewOp<fir::AllocaOp>(
-          op, op.getInType(), op.getUniqName() ? *op.getUniqName() : "",
+      auto allocaOp = rewriter.create<fir::AllocaOp>(
+          loc, op.getInType(), op.getUniqName() ? *op.getUniqName() : "",
           op.getBindcName() ? *op.getBindcName() : "", op.getTypeparams(),
           op.getShape());
+      allocaOp->setAttr(cuf::getDataAttrName(), op.getDataAttrAttr());
+      rewriter.replaceOp(op, allocaOp);
       return mlir::success();
     }
 
     auto mod = op->getParentOfType<mlir::ModuleOp>();
     fir::FirOpBuilder builder(rewriter, mod);
-    mlir::Location loc = op.getLoc();
     mlir::Value sourceFile = fir::factory::locationToFilename(builder, loc);
 
     if (!mlir::dyn_cast_or_null<fir::BaseBoxType>(op.getInType())) {
@@ -359,6 +363,7 @@ struct CUFAllocOpConversion : public mlir::OpRewritePattern<cuf::AllocOp> {
       llvm::SmallVector<mlir::Value> args{fir::runtime::createArguments(
           builder, loc, fTy, bytes, memTy, sourceFile, sourceLine)};
       auto callOp = builder.create<fir::CallOp>(loc, func, args);
+      callOp->setAttr(cuf::getDataAttrName(), op.getDataAttrAttr());
       auto convOp = builder.createConvert(loc, op.getResult().getType(),
                                           callOp.getResult(0));
       rewriter.replaceOp(op, convOp);
@@ -381,6 +386,7 @@ struct CUFAllocOpConversion : public mlir::OpRewritePattern<cuf::AllocOp> {
     llvm::SmallVector<mlir::Value> args{fir::runtime::createArguments(
         builder, loc, fTy, sizeInBytes, sourceFile, sourceLine)};
     auto callOp = builder.create<fir::CallOp>(loc, func, args);
+    callOp->setAttr(cuf::getDataAttrName(), op.getDataAttrAttr());
     auto convOp = builder.createConvert(loc, op.getResult().getType(),
                                         callOp.getResult(0));
     rewriter.replaceOp(op, convOp);
@@ -508,7 +514,8 @@ struct CUFFreeOpConversion : public mlir::OpRewritePattern<cuf::FreeOp> {
         fir::factory::locationToLineNo(builder, loc, fTy.getInput(2));
     llvm::SmallVector<mlir::Value> args{fir::runtime::createArguments(
         builder, loc, fTy, op.getDevptr(), sourceFile, sourceLine)};
-    builder.create<fir::CallOp>(loc, func, args);
+    auto callOp = builder.create<fir::CallOp>(loc, func, args);
+    callOp->setAttr(cuf::getDataAttrName(), op.getDataAttrAttr());
     rewriter.eraseOp(op);
     return mlir::success();
   }
@@ -810,6 +817,7 @@ struct CUFLaunchOpConversion
             rewriter.getContext(),
             op.getCallee().getLeafReference().getValue())});
     mlir::Value clusterDimX, clusterDimY, clusterDimZ;
+    cuf::ProcAttributeAttr procAttr;
     if (auto funcOp = symTab.lookup<mlir::func::FuncOp>(
             op.getCallee().getLeafReference())) {
       if (auto clusterDimsAttr = funcOp->getAttrOfType<cuf::ClusterDimsAttr>(
@@ -821,6 +829,8 @@ struct CUFLaunchOpConversion
         clusterDimZ = rewriter.create<mlir::arith::ConstantIndexOp>(
             loc, clusterDimsAttr.getZ().getInt());
       }
+      procAttr =
+          funcOp->getAttrOfType<cuf::ProcAttributeAttr>(cuf::getProcAttrName());
     }
     llvm::SmallVector<mlir::Value> args;
     for (mlir::Value arg : op.getArgs()) {
@@ -855,6 +865,8 @@ struct CUFLaunchOpConversion
       gpuLaunchOp.getClusterSizeYMutable().assign(clusterDimY);
       gpuLaunchOp.getClusterSizeZMutable().assign(clusterDimZ);
     }
+    if (procAttr)
+      gpuLaunchOp->setAttr(cuf::getProcAttrName(), procAttr);
     rewriter.replaceOp(op, gpuLaunchOp);
     return mlir::success();
   }
@@ -908,8 +920,8 @@ class CUFOpConversion : public fir::impl::CUFOpConversionBase<CUFOpConversion> {
       return signalPassFailure();
     mlir::SymbolTable symtab(module);
 
-    std::optional<mlir::DataLayout> dl =
-        fir::support::getOrSetDataLayout(module, /*allowDefaultLayout=*/false);
+    std::optional<mlir::DataLayout> dl = fir::support::getOrSetMLIRDataLayout(
+        module, /*allowDefaultLayout=*/false);
     fir::LLVMTypeConverter typeConverter(module, /*applyTBAA=*/false,
                                          /*forceUnifiedTBAATree=*/false, *dl);
     target.addLegalDialect<fir::FIROpsDialect, mlir::arith::ArithDialect,
diff --git a/flang/lib/Optimizer/Transforms/DebugTypeGenerator.cpp b/flang/lib/Optimizer/Transforms/DebugTypeGenerator.cpp
index 555f354521c9b..cdd30dce183dd 100644
--- a/flang/lib/Optimizer/Transforms/DebugTypeGenerator.cpp
+++ b/flang/lib/Optimizer/Transforms/DebugTypeGenerator.cpp
@@ -48,7 +48,8 @@ DebugTypeGenerator::DebugTypeGenerator(mlir::ModuleOp m,
                                        mlir::SymbolTable *symbolTable_,
                                        const mlir::DataLayout &dl)
     : module(m), symbolTable(symbolTable_), dataLayout{&dl},
-      kindMapping(getKindMapping(m)), llvmTypeConverter(m, false, false, dl) {
+      kindMapping(getKindMapping(m)), llvmTypeConverter(m, false, false, dl),
+      derivedTypeDepth(0) {
   LLVM_DEBUG(llvm::dbgs() << "DITypeAttr generator\n");
 
   mlir::MLIRContext *context = module.getContext();
@@ -407,7 +408,10 @@ mlir::LLVM::DITypeAttr DebugTypeGenerator::convertRecordType(
       /*baseType=*/nullptr, mlir::LLVM::DIFlags::Zero, offset * 8,
       /*alignInBits=*/0, elements, /*dataLocation=*/nullptr, /*rank=*/nullptr,
       /*allocated=*/nullptr, /*associated=*/nullptr);
-  if (canCacheThisType) {
+
+  // derivedTypeDepth == 1 means that it is a top level type which is safe to
+  // cache.
+  if (canCacheThisType || derivedTypeDepth == 1) {
     typeCache[Ty] = finalAttr;
   } else {
     auto iter = typeCache.find(Ty);
@@ -663,7 +667,27 @@ DebugTypeGenerator::convertType(mlir::Type Ty, mlir::LLVM::DIFileAttr fileAttr,
     return convertCharacterType(charTy, fileAttr, scope, declOp,
                                 /*hasDescriptor=*/false);
   } else if (auto recTy = mlir::dyn_cast_if_present<fir::RecordType>(Ty)) {
-    return convertRecordType(recTy, fileAttr, scope, declOp);
+    // For nested derived types like shown below, the call sequence of the
+    // convertRecordType will look something like as follows:
+    // convertRecordType (t1)
+    //  convertRecordType (t2)
+    //    convertRecordType (t3)
+    // We need to recognize when we are processing the top level type like t1
+    // to make caching decision. The variable `derivedTypeDepth` is used for
+    // this purpose and maintains the current depth of derived type processing.
+    //  type t1
+    //   type(t2), pointer :: p1
+    // end type
+    // type t2
+    //   type(t3), pointer :: p2
+    // end type
+    // type t2
+    //   integer a
+    // end type
+    derivedTypeDepth++;
+    auto result = convertRecordType(recTy, fileAttr, scope, declOp);
+    derivedTypeDepth--;
+    return result;
   } else if (auto tupleTy = mlir::dyn_cast_if_present<mlir::TupleType>(Ty)) {
     return convertTupleType(tupleTy, fileAttr, scope, declOp);
   } else if (auto refTy = mlir::dyn_cast_if_present<fir::ReferenceType>(Ty)) {
diff --git a/flang/lib/Optimizer/Transforms/DebugTypeGenerator.h b/flang/lib/Optimizer/Transforms/DebugTypeGenerator.h
index 7daa0af166e69..cc4b5428ee1a9 100644
--- a/flang/lib/Optimizer/Transforms/DebugTypeGenerator.h
+++ b/flang/lib/Optimizer/Transforms/DebugTypeGenerator.h
@@ -91,6 +91,7 @@ class DebugTypeGenerator {
   std::uint64_t lenOffset;
   std::uint64_t rankOffset;
   std::uint64_t rankSize;
+  int32_t derivedTypeDepth;
   llvm::DenseMap<mlir::Type, mlir::LLVM::DITypeAttr> typeCache;
 };
 
diff --git a/flang/lib/Optimizer/Transforms/ExternalNameConversion.cpp b/flang/lib/Optimizer/Transforms/ExternalNameConversion.cpp
index 4f6974ee52695..3d84eaa4c1595 100644
--- a/flang/lib/Optimizer/Transforms/ExternalNameConversion.cpp
+++ b/flang/lib/Optimizer/Transforms/ExternalNameConversion.cpp
@@ -6,12 +6,12 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "flang/Common/Fortran.h"
 #include "flang/Optimizer/Dialect/FIRDialect.h"
 #include "flang/Optimizer/Dialect/FIROps.h"
 #include "flang/Optimizer/Dialect/FIROpsSupport.h"
 #include "flang/Optimizer/Support/InternalNames.h"
 #include "flang/Optimizer/Transforms/Passes.h"
+#include "flang/Support/Fortran.h"
 #include "mlir/Dialect/GPU/IR/GPUDialect.h"
 #include "mlir/IR/Attributes.h"
 #include "mlir/IR/SymbolTable.h"
diff --git a/flang/lib/Optimizer/Transforms/LoopVersioning.cpp b/flang/lib/Optimizer/Transforms/LoopVersioning.cpp
index b534ec160ce21..1f3495569c9dd 100644
--- a/flang/lib/Optimizer/Transforms/LoopVersioning.cpp
+++ b/flang/lib/Optimizer/Transforms/LoopVersioning.cpp
@@ -40,7 +40,7 @@
 ///       could be part of the cost analysis above.
 //===----------------------------------------------------------------------===//
 
-#include "flang/ISO_Fortran_binding_wrapper.h"
+#include "flang/Common/ISO_Fortran_binding_wrapper.h"
 #include "flang/Optimizer/Builder/BoxValue.h"
 #include "flang/Optimizer/Builder/FIRBuilder.h"
 #include "flang/Optimizer/Builder/Runtime/Inquiry.h"
@@ -312,8 +312,8 @@ void LoopVersioningPass::runOnOperation() {
   mlir::ModuleOp module = func->getParentOfType<mlir::ModuleOp>();
   fir::KindMapping kindMap = fir::getKindMapping(module);
   mlir::SmallVector<ArgInfo, 4> argsOfInterest;
-  std::optional<mlir::DataLayout> dl =
-      fir::support::getOrSetDataLayout(module, /*allowDefaultLayout=*/false);
+  std::optional<mlir::DataLayout> dl = fir::support::getOrSetMLIRDataLayout(
+      module, /*allowDefaultLayout=*/false);
   if (!dl)
     mlir::emitError(module.getLoc(),
                     "data layout attribute is required to perform " DEBUG_TYPE
diff --git a/flang/lib/Optimizer/Transforms/PolymorphicOpConversion.cpp b/flang/lib/Optimizer/Transforms/PolymorphicOpConversion.cpp
index 070889a284f48..0c78a878cdc53 100644
--- a/flang/lib/Optimizer/Transforms/PolymorphicOpConversion.cpp
+++ b/flang/lib/Optimizer/Transforms/PolymorphicOpConversion.cpp
@@ -205,8 +205,9 @@ struct DispatchOpConv : public OpConversionPattern<fir::DispatchOp> {
     // Make the call.
     llvm::SmallVector<mlir::Value> args{funcPtr};
     args.append(dispatch.getArgs().begin(), dispatch.getArgs().end());
-    rewriter.replaceOpWithNewOp<fir::CallOp>(dispatch, resTypes, nullptr, args,
-                                             dispatch.getProcedureAttrsAttr());
+    rewriter.replaceOpWithNewOp<fir::CallOp>(
+        dispatch, resTypes, nullptr, args, dispatch.getArgAttrsAttr(),
+        dispatch.getResAttrsAttr(), dispatch.getProcedureAttrsAttr());
     return mlir::success();
   }
 
diff --git a/flang/lib/Optimizer/Transforms/SimplifyIntrinsics.cpp b/flang/lib/Optimizer/Transforms/SimplifyIntrinsics.cpp
index fa6a7b23624e8..df2887ff1422e 100644
--- a/flang/lib/Optimizer/Transforms/SimplifyIntrinsics.cpp
+++ b/flang/lib/Optimizer/Transforms/SimplifyIntrinsics.cpp
@@ -22,7 +22,6 @@
 /// and small in size.
 //===----------------------------------------------------------------------===//
 
-#include "flang/Common/Fortran.h"
 #include "flang/Optimizer/Builder/BoxValue.h"
 #include "flang/Optimizer/Builder/CUFCommon.h"
 #include "flang/Optimizer/Builder/FIRBuilder.h"
@@ -35,6 +34,7 @@
 #include "flang/Optimizer/Transforms/Passes.h"
 #include "flang/Optimizer/Transforms/Utils.h"
 #include "flang/Runtime/entry-names.h"
+#include "flang/Support/Fortran.h"
 #include "mlir/Dialect/LLVMIR/LLVMDialect.h"
 #include "mlir/IR/Matchers.h"
 #include "mlir/IR/Operation.h"
diff --git a/flang/lib/Optimizer/Transforms/StackReclaim.cpp b/flang/lib/Optimizer/Transforms/StackReclaim.cpp
index bd3e49a47bc39..acb685850d173 100644
--- a/flang/lib/Optimizer/Transforms/StackReclaim.cpp
+++ b/flang/lib/Optimizer/Transforms/StackReclaim.cpp
@@ -6,11 +6,11 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "flang/Common/Fortran.h"
 #include "flang/Optimizer/Builder/FIRBuilder.h"
 #include "flang/Optimizer/Dialect/FIRDialect.h"
 #include "flang/Optimizer/Dialect/FIROps.h"
 #include "flang/Optimizer/Transforms/Passes.h"
+#include "flang/Support/Fortran.h"
 #include "mlir/Dialect/LLVMIR/LLVMDialect.h"
 #include "mlir/IR/Matchers.h"
 #include "mlir/Pass/Pass.h"
diff --git a/flang/lib/Optimizer/Transforms/VScaleAttr.cpp b/flang/lib/Optimizer/Transforms/VScaleAttr.cpp
index d311167c58b4d..54a2456e77366 100644
--- a/flang/lib/Optimizer/Transforms/VScaleAttr.cpp
+++ b/flang/lib/Optimizer/Transforms/VScaleAttr.cpp
@@ -14,7 +14,7 @@
 /// likely harmless to run it on something else, but it is also not valuable].
 //===----------------------------------------------------------------------===//
 
-#include "flang/ISO_Fortran_binding_wrapper.h"
+#include "flang/Common/ISO_Fortran_binding_wrapper.h"
 #include "flang/Optimizer/Builder/BoxValue.h"
 #include "flang/Optimizer/Builder/FIRBuilder.h"
 #include "flang/Optimizer/Builder/Runtime/Inquiry.h"
diff --git a/flang/lib/Parser/CMakeLists.txt b/flang/lib/Parser/CMakeLists.txt
index d364671d7a322..76fe3d7ce6ba4 100644
--- a/flang/lib/Parser/CMakeLists.txt
+++ b/flang/lib/Parser/CMakeLists.txt
@@ -25,7 +25,7 @@ add_flang_library(FortranParser
   user-state.cpp
 
   LINK_LIBS
-  FortranCommon
+  FortranSupport
 
   LINK_COMPONENTS
   Support
diff --git a/flang/lib/Parser/Fortran-parsers.cpp b/flang/lib/Parser/Fortran-parsers.cpp
index 7cb35c1f173bb..b5bcb53a12761 100644
--- a/flang/lib/Parser/Fortran-parsers.cpp
+++ b/flang/lib/Parser/Fortran-parsers.cpp
@@ -1293,6 +1293,7 @@ TYPE_PARSER(construct<StatOrErrmsg>("STAT =" >> statVariable) ||
 // !DIR$ IGNORE_TKR [ [(tkrdmac...)] name ]...
 // !DIR$ LOOP COUNT (n1[, n2]...)
 // !DIR$ name[=value] [, name[=value]]...
+// !DIR$ UNROLL [n]
 // !DIR$ <anything else>
 constexpr auto ignore_tkr{
     "IGNORE_TKR" >> optionalList(construct<CompilerDirective::IgnoreTKR>(
@@ -1305,11 +1306,14 @@ constexpr auto assumeAligned{"ASSUME_ALIGNED" >>
         indirect(designator), ":"_tok >> digitString64))};
 constexpr auto vectorAlways{
     "VECTOR ALWAYS" >> construct<CompilerDirective::VectorAlways>()};
+constexpr auto unroll{
+    "UNROLL" >> construct<CompilerDirective::Unroll>(maybe(digitString64))};
 TYPE_PARSER(beginDirective >> "DIR$ "_tok >>
     sourced((construct<CompilerDirective>(ignore_tkr) ||
                 construct<CompilerDirective>(loopCount) ||
                 construct<CompilerDirective>(assumeAligned) ||
                 construct<CompilerDirective>(vectorAlways) ||
+                construct<CompilerDirective>(unroll) ||
                 construct<CompilerDirective>(
                     many(construct<CompilerDirective::NameValue>(
                         name, maybe(("="_tok || ":"_tok) >> digitString64))))) /
diff --git a/flang/lib/Parser/basic-parsers.h b/flang/lib/Parser/basic-parsers.h
index 1a8c14e7048f6..7e69d41debfcd 100644
--- a/flang/lib/Parser/basic-parsers.h
+++ b/flang/lib/Parser/basic-parsers.h
@@ -22,7 +22,6 @@
 // This header defines the fundamental parser class templates and helper
 // template functions.  See parser-combinators.txt for documentation.
 
-#include "flang/Common/Fortran-features.h"
 #include "flang/Common/idioms.h"
 #include "flang/Common/indirection.h"
 #include "flang/Parser/char-block.h"
@@ -30,6 +29,7 @@
 #include "flang/Parser/parse-state.h"
 #include "flang/Parser/provenance.h"
 #include "flang/Parser/user-state.h"
+#include "flang/Support/Fortran-features.h"
 #include <cstring>
 #include <functional>
 #include <list>
diff --git a/flang/lib/Parser/openmp-parsers.cpp b/flang/lib/Parser/openmp-parsers.cpp
index 5ff91da082c85..2b6c77c08cc58 100644
--- a/flang/lib/Parser/openmp-parsers.cpp
+++ b/flang/lib/Parser/openmp-parsers.cpp
@@ -68,6 +68,8 @@ void OmpDirectiveNameParser::initTokens(NameWithId *table) const {
       [](auto &a, auto &b) { return a.first.size() > b.first.size(); });
 }
 
+// --- Modifier helpers -----------------------------------------------
+
 template <typename Clause, typename Separator> struct ModifierList {
   constexpr ModifierList(Separator sep) : sep_(sep) {}
   constexpr ModifierList(const ModifierList &) = default;
@@ -118,10 +120,8 @@ struct SpecificModifierParser {
   }
 };
 
-// OpenMP Clauses
+// --- Iterator helpers -----------------------------------------------
 
-// [5.0] 2.1.6 iterator-specifier -> type-declaration-stmt = subscript-triple |
-//                                   identifier = subscript-triple
 // [5.0:47:17-18] In an iterator-specifier, if the iterator-type is not
 // specified then the type of that iterator is default integer.
 // [5.0:49:14] The iterator-type must be an integer type.
@@ -153,82 +153,181 @@ static TypeDeclarationStmt makeIterSpecDecl(std::list<ObjectName> &&names) {
       makeEntityList(std::move(names)));
 }
 
+// --- Parsers for arguments ------------------------------------------
+
+// At the moment these are only directive arguments. This is needed for
+// parsing directive-specification.
+
+TYPE_PARSER( //
+    construct<OmpLocator>(Parser<OmpObject>{}) ||
+    construct<OmpLocator>(Parser<FunctionReference>{}))
+
+TYPE_PARSER(sourced( //
+    construct<OmpArgument>(Parser<OmpMapperSpecifier>{}) ||
+    construct<OmpArgument>(Parser<OmpReductionSpecifier>{}) ||
+    construct<OmpArgument>(Parser<OmpLocator>{})))
+
+TYPE_PARSER(construct<OmpLocatorList>(nonemptyList(Parser<OmpLocator>{})))
+
+TYPE_PARSER( //
+    construct<OmpTypeSpecifier>(Parser<TypeSpec>{}) ||
+    construct<OmpTypeSpecifier>(Parser<DeclarationTypeSpec>{}))
+
+TYPE_PARSER(construct<OmpReductionSpecifier>( //
+    Parser<OmpReductionIdentifier>{},
+    ":"_tok >> nonemptyList(Parser<OmpTypeSpecifier>{}),
+    maybe(":"_tok >> Parser<OmpReductionCombiner>{})))
+
 // --- Parsers for context traits -------------------------------------
 
 static std::string nameToString(Name &&name) { return name.ToString(); }
 
 TYPE_PARSER(sourced(construct<OmpTraitPropertyName>( //
-    (space >> charLiteralConstantWithoutKind) ||
-    applyFunction(nameToString, Parser<Name>{}))))
+    construct<OmpTraitPropertyName>(space >> charLiteralConstantWithoutKind) ||
+    construct<OmpTraitPropertyName>(
+        applyFunction(nameToString, Parser<Name>{})))))
 
 TYPE_PARSER(sourced(construct<OmpTraitScore>( //
-    "SCORE" >> parenthesized(scalarIntExpr))))
-
-TYPE_PARSER(sourced(construct<OmpTraitPropertyExtension::ExtensionValue>(
-    // Parse nested extension first.
-    construct<OmpTraitPropertyExtension::ExtensionValue>(
-        indirect(Parser<OmpTraitPropertyExtension>{})) ||
-    construct<OmpTraitPropertyExtension::ExtensionValue>(
-        Parser<OmpTraitPropertyName>{}) ||
-    construct<OmpTraitPropertyExtension::ExtensionValue>(scalarExpr))))
+    "SCORE"_id >> parenthesized(scalarIntExpr))))
 
-TYPE_PARSER(sourced(construct<OmpTraitPropertyExtension>( //
+TYPE_PARSER(sourced(construct<OmpTraitPropertyExtension::Complex>(
     Parser<OmpTraitPropertyName>{},
     parenthesized(nonemptySeparated(
-        Parser<OmpTraitPropertyExtension::ExtensionValue>{}, ","_tok)))))
+        indirect(Parser<OmpTraitPropertyExtension>{}), ",")))))
 
-TYPE_PARSER(sourced(construct<OmpTraitProperty>(
-    // Try clause first, then extension before OmpTraitPropertyName.
-    construct<OmpTraitProperty>(indirect(Parser<OmpClause>{})) ||
-    construct<OmpTraitProperty>(Parser<OmpTraitPropertyExtension>{}) ||
-    construct<OmpTraitProperty>(Parser<OmpTraitPropertyName>{}) ||
-    construct<OmpTraitProperty>(scalarExpr))))
+TYPE_PARSER(sourced(construct<OmpTraitPropertyExtension>(
+    construct<OmpTraitPropertyExtension>(
+        Parser<OmpTraitPropertyExtension::Complex>{}) ||
+    construct<OmpTraitPropertyExtension>(Parser<OmpTraitPropertyName>{}) ||
+    construct<OmpTraitPropertyExtension>(scalarExpr))))
 
 TYPE_PARSER(construct<OmpTraitSelectorName::Value>(
-    "ARCH" >> pure(OmpTraitSelectorName::Value::Arch) ||
-    "ATOMIC_DEFAULT_MEM_ORDER" >>
+    "ARCH"_id >> pure(OmpTraitSelectorName::Value::Arch) ||
+    "ATOMIC_DEFAULT_MEM_ORDER"_id >>
         pure(OmpTraitSelectorName::Value::Atomic_Default_Mem_Order) ||
-    "CONDITION" >> pure(OmpTraitSelectorName::Value::Condition) ||
-    "DEVICE_NUM" >> pure(OmpTraitSelectorName::Value::Device_Num) ||
-    "EXTENSION" >> pure(OmpTraitSelectorName::Value::Extension) ||
-    "ISA" >> pure(OmpTraitSelectorName::Value::Isa) ||
-    "KIND" >> pure(OmpTraitSelectorName::Value::Kind) ||
-    "REQUIRES" >> pure(OmpTraitSelectorName::Value::Requires) ||
-    "SIMD" >> pure(OmpTraitSelectorName::Value::Simd) ||
-    "UID" >> pure(OmpTraitSelectorName::Value::Uid) ||
-    "VENDOR" >> pure(OmpTraitSelectorName::Value::Vendor)))
+    "CONDITION"_id >> pure(OmpTraitSelectorName::Value::Condition) ||
+    "DEVICE_NUM"_id >> pure(OmpTraitSelectorName::Value::Device_Num) ||
+    "EXTENSION"_id >> pure(OmpTraitSelectorName::Value::Extension) ||
+    "ISA"_id >> pure(OmpTraitSelectorName::Value::Isa) ||
+    "KIND"_id >> pure(OmpTraitSelectorName::Value::Kind) ||
+    "REQUIRES"_id >> pure(OmpTraitSelectorName::Value::Requires) ||
+    "SIMD"_id >> pure(OmpTraitSelectorName::Value::Simd) ||
+    "UID"_id >> pure(OmpTraitSelectorName::Value::Uid) ||
+    "VENDOR"_id >> pure(OmpTraitSelectorName::Value::Vendor)))
 
 TYPE_PARSER(sourced(construct<OmpTraitSelectorName>(
     // Parse predefined names first (because of SIMD).
     construct<OmpTraitSelectorName>(Parser<OmpTraitSelectorName::Value>{}) ||
-    construct<OmpTraitSelectorName>(OmpDirectiveNameParser{}))))
+    construct<OmpTraitSelectorName>(OmpDirectiveNameParser{}) ||
+    // identifier-or-string for extensions
+    construct<OmpTraitSelectorName>(
+        applyFunction(nameToString, Parser<Name>{})) ||
+    construct<OmpTraitSelectorName>(space >> charLiteralConstantWithoutKind))))
+
+// Parser for OmpTraitSelector::Properties
+template <typename... PropParser>
+static constexpr auto propertyListParser(PropParser... pp) {
+  // Parse the property list "(score(expr): item1...)" in three steps:
+  // 1. Parse the "("
+  // 2. Parse the optional "score(expr):"
+  // 3. Parse the "item1, ...)", together with the ")".
+  // The reason for including the ")" in the 3rd step is to force parsing
+  // the entire list in each of the alternative property parsers. Otherwise,
+  // the name parser could stop after "foo" in "(foo, bar(1))", without
+  // allowing the next parser to give the list a try.
+  using P = OmpTraitProperty;
+  return maybe("(" >> //
+      construct<OmpTraitSelector::Properties>(
+          maybe(Parser<OmpTraitScore>{} / ":"),
+          (attempt(nonemptyList(sourced(construct<P>(pp))) / ")") || ...)));
+}
+
+// Parser for OmpTraitSelector
+struct TraitSelectorParser {
+  using resultType = OmpTraitSelector;
+
+  constexpr TraitSelectorParser(Parser<OmpTraitSelectorName> p) : np(p) {}
+
+  std::optional<resultType> Parse(ParseState &state) const {
+    auto name{attempt(np).Parse(state)};
+    if (!name.has_value()) {
+      return std::nullopt;
+    }
+
+    // Default fallback parser for lists that cannot be parser using the
+    // primary property parser.
+    auto extParser{Parser<OmpTraitPropertyExtension>{}};
+
+    if (auto *v{std::get_if<OmpTraitSelectorName::Value>(&name->u)}) {
+      // (*) The comments below show the sections of the OpenMP spec that
+      // describe given trait. The cases marked with a (*) are those where
+      // the spec doesn't assign any list-type to these traits, but for
+      // convenience they can be treated as if they were.
+      switch (*v) {
+      // name-list properties
+      case OmpTraitSelectorName::Value::Arch: // [6.0:319:18]
+      case OmpTraitSelectorName::Value::Extension: // [6.0:319:30]
+      case OmpTraitSelectorName::Value::Isa: // [6.0:319:15]
+      case OmpTraitSelectorName::Value::Kind: // [6.0:319:10]
+      case OmpTraitSelectorName::Value::Uid: // [6.0:319:23](*)
+      case OmpTraitSelectorName::Value::Vendor: { // [6.0:319:27]
+        auto pp{propertyListParser(Parser<OmpTraitPropertyName>{}, extParser)};
+        return OmpTraitSelector(std::move(*name), std::move(*pp.Parse(state)));
+      }
+      // clause-list
+      case OmpTraitSelectorName::Value::Atomic_Default_Mem_Order:
+        // [6.0:321:26-29](*)
+      case OmpTraitSelectorName::Value::Requires: // [6.0:319:33]
+      case OmpTraitSelectorName::Value::Simd: { // [6.0:318:31]
+        auto pp{propertyListParser(indirect(Parser<OmpClause>{}), extParser)};
+        return OmpTraitSelector(std::move(*name), std::move(*pp.Parse(state)));
+      }
+      // expr-list
+      case OmpTraitSelectorName::Value::Condition: // [6.0:321:33](*)
+      case OmpTraitSelectorName::Value::Device_Num: { // [6.0:321:23-24](*)
+        auto pp{propertyListParser(scalarExpr, extParser)};
+        return OmpTraitSelector(std::move(*name), std::move(*pp.Parse(state)));
+      }
+      } // switch
+    } else {
+      // The other alternatives are `llvm::omp::Directive`, and `std::string`.
+      // The former doesn't take any properties[1], the latter is a name of an
+      // extension[2].
+      // [1] [6.0:319:1-2]
+      // [2] [6.0:319:36-37]
+      auto pp{propertyListParser(extParser)};
+      return OmpTraitSelector(std::move(*name), std::move(*pp.Parse(state)));
+    }
 
-TYPE_PARSER(construct<OmpTraitSelector::Properties>(
-    maybe(Parser<OmpTraitScore>{} / ":"_tok),
-    nonemptySeparated(Parser<OmpTraitProperty>{}, ","_tok)))
+    llvm_unreachable("Unhandled trait name?");
+  }
+
+private:
+  const Parser<OmpTraitSelectorName> np;
+};
 
-TYPE_PARSER(sourced(construct<OmpTraitSelector>( //
-    Parser<OmpTraitSelectorName>{}, //
-    maybe(parenthesized(Parser<OmpTraitSelector::Properties>{})))))
+TYPE_PARSER(sourced(construct<OmpTraitSelector>(
+    sourced(TraitSelectorParser(Parser<OmpTraitSelectorName>{})))))
 
 TYPE_PARSER(construct<OmpTraitSetSelectorName::Value>(
-    "CONSTRUCT" >> pure(OmpTraitSetSelectorName::Value::Construct) ||
-    "DEVICE" >> pure(OmpTraitSetSelectorName::Value::Device) ||
-    "IMPLEMENTATION" >> pure(OmpTraitSetSelectorName::Value::Implementation) ||
-    "TARGET_DEVICE" >> pure(OmpTraitSetSelectorName::Value::Target_Device) ||
-    "USER" >> pure(OmpTraitSetSelectorName::Value::User)))
+    "CONSTRUCT"_id >> pure(OmpTraitSetSelectorName::Value::Construct) ||
+    "DEVICE"_id >> pure(OmpTraitSetSelectorName::Value::Device) ||
+    "IMPLEMENTATION"_id >>
+        pure(OmpTraitSetSelectorName::Value::Implementation) ||
+    "TARGET_DEVICE"_id >> pure(OmpTraitSetSelectorName::Value::Target_Device) ||
+    "USER"_id >> pure(OmpTraitSetSelectorName::Value::User)))
 
 TYPE_PARSER(sourced(construct<OmpTraitSetSelectorName>(
     Parser<OmpTraitSetSelectorName::Value>{})))
 
 TYPE_PARSER(sourced(construct<OmpTraitSetSelector>( //
     Parser<OmpTraitSetSelectorName>{},
-    "=" >> braced(nonemptySeparated(Parser<OmpTraitSelector>{}, ","_tok)))))
+    "=" >> braced(nonemptySeparated(Parser<OmpTraitSelector>{}, ",")))))
 
 TYPE_PARSER(sourced(construct<OmpContextSelectorSpecification>(
-    nonemptySeparated(Parser<OmpTraitSetSelector>{}, ","_tok))))
+    nonemptySeparated(Parser<OmpTraitSetSelector>{}, ","))))
 
-// Parser<OmpContextSelector> == Parser<traits::OmpContextSelectorSpecification>
+// Note: OmpContextSelector is a type alias.
 
 // --- Parsers for clause modifiers -----------------------------------
 
@@ -423,6 +522,9 @@ TYPE_PARSER(sourced(construct<OmpToClause::Modifier>(
         construct<OmpToClause::Modifier>(Parser<OmpMapper>{}) ||
         construct<OmpToClause::Modifier>(Parser<OmpIterator>{})))))
 
+TYPE_PARSER(sourced(construct<OmpWhenClause::Modifier>( //
+    Parser<OmpContextSelector>{})))
+
 // --- Parsers for clauses --------------------------------------------
 
 /// `MOBClause` is a clause that has a
@@ -449,13 +551,18 @@ TYPE_PARSER(construct<OmpAffinityClause>(
     Parser<OmpObjectList>{}))
 
 // 2.15.3.1 DEFAULT (PRIVATE | FIRSTPRIVATE | SHARED | NONE)
-TYPE_PARSER(construct<OmpDefaultClause>(
+TYPE_PARSER(construct<OmpDefaultClause::DataSharingAttribute>(
     "PRIVATE" >> pure(OmpDefaultClause::DataSharingAttribute::Private) ||
     "FIRSTPRIVATE" >>
         pure(OmpDefaultClause::DataSharingAttribute::Firstprivate) ||
     "SHARED" >> pure(OmpDefaultClause::DataSharingAttribute::Shared) ||
     "NONE" >> pure(OmpDefaultClause::DataSharingAttribute::None)))
 
+TYPE_PARSER(construct<OmpDefaultClause>(
+    construct<OmpDefaultClause>(
+        Parser<OmpDefaultClause::DataSharingAttribute>{}) ||
+    construct<OmpDefaultClause>(indirect(Parser<OmpDirectiveSpecification>{}))))
+
 // 2.5 PROC_BIND (MASTER | CLOSE | PRIMARY | SPREAD)
 TYPE_PARSER(construct<OmpProcBindClause>(
     "CLOSE" >> pure(OmpProcBindClause::AffinityPolicy::Close) ||
@@ -620,6 +727,16 @@ TYPE_PARSER(construct<OmpOrderClause>(
     maybe(nonemptyList(Parser<OmpOrderClause::Modifier>{}) / ":"),
     "CONCURRENT" >> pure(OmpOrderClause::Ordering::Concurrent)))
 
+TYPE_PARSER(construct<OmpMatchClause>(
+    Parser<traits::OmpContextSelectorSpecification>{}))
+
+TYPE_PARSER(construct<OmpOtherwiseClause>(
+    maybe(indirect(sourced(Parser<OmpDirectiveSpecification>{})))))
+
+TYPE_PARSER(construct<OmpWhenClause>(
+    maybe(nonemptyList(Parser<OmpWhenClause::Modifier>{}) / ":"),
+    maybe(indirect(sourced(Parser<OmpDirectiveSpecification>{})))))
+
 // OMP 5.2 12.6.1 grainsize([ prescriptiveness :] scalar-integer-expression)
 TYPE_PARSER(construct<OmpGrainsizeClause>(
     maybe(nonemptyList(Parser<OmpGrainsizeClause::Modifier>{}) / ":"),
@@ -737,14 +854,20 @@ TYPE_PARSER(
                   parenthesized(Parser<OmpObjectList>{}))) ||
     "MAP" >> construct<OmpClause>(construct<OmpClause::Map>(
                  parenthesized(Parser<OmpMapClause>{}))) ||
+    "MATCH" >> construct<OmpClause>(construct<OmpClause::Match>(
+                   parenthesized(Parser<OmpMatchClause>{}))) ||
     "MERGEABLE" >> construct<OmpClause>(construct<OmpClause::Mergeable>()) ||
     "MESSAGE" >> construct<OmpClause>(construct<OmpClause::Message>(
                      parenthesized(Parser<OmpMessageClause>{}))) ||
+    "NOCONTEXT" >> construct<OmpClause>(construct<OmpClause::Nocontext>(
+                       parenthesized(scalarLogicalExpr))) ||
     "NOGROUP" >> construct<OmpClause>(construct<OmpClause::Nogroup>()) ||
     "NONTEMPORAL" >> construct<OmpClause>(construct<OmpClause::Nontemporal>(
                          parenthesized(nonemptyList(name)))) ||
     "NOTINBRANCH" >>
         construct<OmpClause>(construct<OmpClause::Notinbranch>()) ||
+    "NOVARIANTS" >> construct<OmpClause>(construct<OmpClause::Novariants>(
+                        parenthesized(scalarLogicalExpr))) ||
     "NOWAIT" >> construct<OmpClause>(construct<OmpClause::Nowait>()) ||
     "NUM_TASKS" >> construct<OmpClause>(construct<OmpClause::NumTasks>(
                        parenthesized(Parser<OmpNumTasksClause>{}))) ||
@@ -757,6 +880,8 @@ TYPE_PARSER(
                    parenthesized(Parser<OmpOrderClause>{}))) ||
     "ORDERED" >> construct<OmpClause>(construct<OmpClause::Ordered>(
                      maybe(parenthesized(scalarIntConstantExpr)))) ||
+    "OTHERWISE" >> construct<OmpClause>(construct<OmpClause::Otherwise>(
+                       maybe(parenthesized(Parser<OmpOtherwiseClause>{})))) ||
     "PARTIAL" >> construct<OmpClause>(construct<OmpClause::Partial>(
                      maybe(parenthesized(scalarIntConstantExpr)))) ||
     "PRIORITY" >> construct<OmpClause>(construct<OmpClause::Priority>(
@@ -812,7 +937,9 @@ TYPE_PARSER(
                      parenthesized(nonemptyList(name)))) ||
     "UNTIED" >> construct<OmpClause>(construct<OmpClause::Untied>()) ||
     "UPDATE" >> construct<OmpClause>(construct<OmpClause::Update>(
-                    parenthesized(Parser<OmpUpdateClause>{}))))
+                    parenthesized(Parser<OmpUpdateClause>{}))) ||
+    "WHEN" >> construct<OmpClause>(construct<OmpClause::When>(
+                  parenthesized(Parser<OmpWhenClause>{}))))
 
 // [Clause, [Clause], ...]
 TYPE_PARSER(sourced(construct<OmpClauseList>(
@@ -824,6 +951,13 @@ TYPE_PARSER(construct<OmpObjectList>(nonemptyList(Parser<OmpObject>{})))
 TYPE_PARSER(sourced(construct<OmpErrorDirective>(
     verbatim("ERROR"_tok), Parser<OmpClauseList>{})))
 
+// --- Parsers for directives and constructs --------------------------
+
+TYPE_PARSER(sourced(construct<OmpDirectiveSpecification>( //
+    OmpDirectiveNameParser{},
+    maybe(parenthesized(nonemptyList(Parser<OmpArgument>{}))),
+    maybe(Parser<OmpClauseList>{}))))
+
 TYPE_PARSER(sourced(construct<OmpNothingDirective>("NOTHING" >> ok)))
 
 TYPE_PARSER(sourced(construct<OpenMPUtilityConstruct>(
@@ -832,6 +966,9 @@ TYPE_PARSER(sourced(construct<OpenMPUtilityConstruct>(
     sourced(construct<OpenMPUtilityConstruct>(
         sourced(Parser<OmpNothingDirective>{}))))))
 
+TYPE_PARSER(sourced(construct<OmpMetadirectiveDirective>(
+    "METADIRECTIVE" >> Parser<OmpClauseList>{})))
+
 // Omp directives enclosing do loop
 TYPE_PARSER(sourced(construct<OmpLoopDirective>(first(
     "DISTRIBUTE PARALLEL DO SIMD" >>
@@ -977,6 +1114,8 @@ TYPE_PARSER(
         construct<OpenMPStandaloneConstruct>(Parser<OpenMPCancelConstruct>{}) ||
         construct<OpenMPStandaloneConstruct>(
             Parser<OpenMPCancellationPointConstruct>{}) ||
+        construct<OpenMPStandaloneConstruct>(
+            Parser<OmpMetadirectiveDirective>{}) ||
         construct<OpenMPStandaloneConstruct>(Parser<OpenMPDepobjConstruct>{})) /
     endOfLine)
 
@@ -1031,20 +1170,17 @@ TYPE_PARSER(
 TYPE_PARSER(sourced(construct<OpenMPDeclareTargetConstruct>(
     verbatim("DECLARE TARGET"_tok), Parser<OmpDeclareTargetSpecifier>{})))
 
-// declare-mapper-specifier
-TYPE_PARSER(construct<OmpDeclareMapperSpecifier>(
+// mapper-specifier
+TYPE_PARSER(construct<OmpMapperSpecifier>(
     maybe(name / ":" / !":"_tok), typeSpec / "::", name))
 
 // OpenMP 5.2: 5.8.8 Declare Mapper Construct
-TYPE_PARSER(sourced(construct<OpenMPDeclareMapperConstruct>(
-    verbatim("DECLARE MAPPER"_tok),
-    "(" >> Parser<OmpDeclareMapperSpecifier>{} / ")", Parser<OmpClauseList>{})))
+TYPE_PARSER(sourced(
+    construct<OpenMPDeclareMapperConstruct>(verbatim("DECLARE MAPPER"_tok),
+        parenthesized(Parser<OmpMapperSpecifier>{}), Parser<OmpClauseList>{})))
 
 TYPE_PARSER(construct<OmpReductionCombiner>(Parser<AssignmentStmt>{}) ||
-    construct<OmpReductionCombiner>(
-        construct<OmpReductionCombiner::FunctionCombiner>(
-            construct<Call>(Parser<ProcedureDesignator>{},
-                parenthesized(optionalList(actualArgSpec))))))
+    construct<OmpReductionCombiner>(Parser<FunctionReference>{}))
 
 // 2.17.7 atomic -> ATOMIC [clause [,]] atomic-clause [[,] clause] |
 //                  ATOMIC [clause]
@@ -1119,6 +1255,16 @@ TYPE_PARSER(sourced(construct<OmpCriticalDirective>(verbatim("CRITICAL"_tok),
 TYPE_PARSER(construct<OpenMPCriticalConstruct>(
     Parser<OmpCriticalDirective>{}, block, Parser<OmpEndCriticalDirective>{}))
 
+TYPE_PARSER(sourced(construct<OmpDispatchDirective>(
+    verbatim("DISPATCH"_tok), Parser<OmpClauseList>{})))
+
+TYPE_PARSER(
+    construct<OmpEndDispatchDirective>(startOmpLine >> "END DISPATCH"_tok))
+
+TYPE_PARSER(sourced(construct<OpenMPDispatchConstruct>(
+    Parser<OmpDispatchDirective>{} / endOmpLine, block,
+    maybe(Parser<OmpEndDispatchDirective>{} / endOmpLine))))
+
 // 2.11.3 Executable Allocate directive
 TYPE_PARSER(
     sourced(construct<OpenMPExecutableAllocate>(verbatim("ALLOCATE"_tok),
@@ -1173,7 +1319,9 @@ TYPE_PARSER(startOmpLine >>
             construct<OpenMPDeclarativeConstruct>(
                 Parser<OpenMPThreadprivate>{}) ||
             construct<OpenMPDeclarativeConstruct>(
-                Parser<OpenMPUtilityConstruct>{})) /
+                Parser<OpenMPUtilityConstruct>{}) ||
+            construct<OpenMPDeclarativeConstruct>(
+                Parser<OmpMetadirectiveDirective>{})) /
             endOmpLine))
 
 // Block Construct
@@ -1219,6 +1367,7 @@ TYPE_CONTEXT_PARSER("OpenMP construct"_en_US,
                 construct<OpenMPConstruct>(Parser<OpenMPStandaloneConstruct>{}),
                 construct<OpenMPConstruct>(Parser<OpenMPAtomicConstruct>{}),
                 construct<OpenMPConstruct>(Parser<OpenMPUtilityConstruct>{}),
+                construct<OpenMPConstruct>(Parser<OpenMPDispatchConstruct>{}),
                 construct<OpenMPConstruct>(Parser<OpenMPExecutableAllocate>{}),
                 construct<OpenMPConstruct>(Parser<OpenMPAllocatorsConstruct>{}),
                 construct<OpenMPConstruct>(Parser<OpenMPDeclarativeAllocate>{}),
diff --git a/flang/lib/Parser/parse-tree.cpp b/flang/lib/Parser/parse-tree.cpp
index a414f226058e3..251b6919cf52f 100644
--- a/flang/lib/Parser/parse-tree.cpp
+++ b/flang/lib/Parser/parse-tree.cpp
@@ -281,6 +281,26 @@ OmpTaskDependenceType::Value OmpDependClause::TaskDep::GetTaskDepType() const {
   }
 }
 
+std::string OmpTraitSelectorName::ToString() const {
+  return common::visit( //
+      common::visitors{
+          [&](Value v) { //
+            return std::string(EnumToString(v));
+          },
+          [&](llvm::omp::Directive d) {
+            return llvm::omp::getOpenMPDirectiveName(d).str();
+          },
+          [&](const std::string &s) { //
+            return s;
+          },
+      },
+      u);
+}
+
+std::string OmpTraitSetSelectorName::ToString() const {
+  return std::string(EnumToString(v));
+}
+
 } // namespace Fortran::parser
 
 template <typename C> static llvm::omp::Clause getClauseIdForClass(C &&) {
diff --git a/flang/lib/Parser/prescan.h b/flang/lib/Parser/prescan.h
index 08041f93b14b6..e2440ad6fbc42 100644
--- a/flang/lib/Parser/prescan.h
+++ b/flang/lib/Parser/prescan.h
@@ -16,11 +16,11 @@
 // fixed form character literals on truncated card images, file
 // inclusion, and driving the Fortran source preprocessor.
 
-#include "flang/Common/Fortran-features.h"
 #include "flang/Parser/characters.h"
 #include "flang/Parser/message.h"
 #include "flang/Parser/provenance.h"
 #include "flang/Parser/token-sequence.h"
+#include "flang/Support/Fortran-features.h"
 #include <bitset>
 #include <optional>
 #include <string>
diff --git a/flang/lib/Parser/unparse.cpp b/flang/lib/Parser/unparse.cpp
index 7bf404bba2c3e..cd91fbe4ea5eb 100644
--- a/flang/lib/Parser/unparse.cpp
+++ b/flang/lib/Parser/unparse.cpp
@@ -10,13 +10,13 @@
 // traversal templates in parse-tree-visitor.h.
 
 #include "flang/Parser/unparse.h"
-#include "flang/Common/Fortran.h"
 #include "flang/Common/idioms.h"
 #include "flang/Common/indirection.h"
 #include "flang/Parser/characters.h"
 #include "flang/Parser/parse-tree-visitor.h"
 #include "flang/Parser/parse-tree.h"
 #include "flang/Parser/tools.h"
+#include "flang/Support/Fortran.h"
 #include "llvm/Support/raw_ostream.h"
 #include <algorithm>
 #include <cinttypes>
@@ -1847,6 +1847,10 @@ class UnparseVisitor {
             [&](const std::list<CompilerDirective::NameValue> &names) {
               Walk("!DIR$ ", names, " ");
             },
+            [&](const CompilerDirective::Unroll &unroll) {
+              Word("!DIR$ UNROLL");
+              Walk(" ", unroll.v);
+            },
             [&](const CompilerDirective::Unrecognized &) {
               Word("!DIR$ ");
               Word(x.source.ToString());
@@ -2067,18 +2071,44 @@ class UnparseVisitor {
   }
 
   // OpenMP Clauses & Directives
+  void Unparse(const OmpTypeNameList &x) { //
+    Walk(x.v, ",");
+  }
+  void Unparse(const OmpMapperSpecifier &x) {
+    Walk(std::get<std::optional<Name>>(x.t), ":");
+    Walk(std::get<TypeSpec>(x.t));
+    Put("::");
+    Walk(std::get<Name>(x.t));
+  }
+  void Unparse(const OmpReductionSpecifier &x) {
+    Walk(std::get<OmpReductionIdentifier>(x.t));
+    Put(":");
+    Walk(std::get<OmpTypeNameList>(x.t));
+    Walk(":", std::get<std::optional<OmpReductionCombiner>>(x.t));
+  }
   void Unparse(const llvm::omp::Directive &x) {
     Word(llvm::omp::getOpenMPDirectiveName(x).str());
   }
+  void Unparse(const OmpDirectiveSpecification &x) {
+    using ArgList = std::list<parser::OmpArgument>;
+    Walk(std::get<llvm::omp::Directive>(x.t));
+    if (auto &args{std::get<std::optional<ArgList>>(x.t)}) {
+      Put("(");
+      Walk(*args);
+      Put(")");
+    }
+    Walk(std::get<std::optional<OmpClauseList>>(x.t));
+  }
   void Unparse(const OmpTraitScore &x) {
     Word("SCORE(");
     Walk(x.v);
     Put(")");
   }
-  void Unparse(const OmpTraitPropertyExtension &x) {
+  void Unparse(const OmpTraitPropertyExtension::Complex &x) {
+    using PropList = std::list<common::Indirection<OmpTraitPropertyExtension>>;
     Walk(std::get<OmpTraitPropertyName>(x.t));
     Put("(");
-    Walk(std::get<OmpTraitPropertyExtension::ExtensionList>(x.t), ",");
+    Walk(std::get<PropList>(x.t), ",");
     Put(")");
   }
   void Unparse(const OmpTraitSelector &x) {
@@ -2290,6 +2320,12 @@ class UnparseVisitor {
     Walk(std::get<std::optional<std::list<Modifier>>>(x.t), ": ");
     Walk(std::get<OmpObjectList>(x.t));
   }
+  void Unparse(const OmpWhenClause &x) {
+    using Modifier = OmpWhenClause::Modifier;
+    using Directive = common::Indirection<OmpDirectiveSpecification>;
+    Walk(std::get<std::optional<std::list<Modifier>>>(x.t), ": ");
+    Walk(std::get<std::optional<Directive>>(x.t));
+  }
 #define GEN_FLANG_CLAUSE_UNPARSE
 #include "llvm/Frontend/OpenMP/OMP.inc"
   void Unparse(const OmpLoopDirective &x) {
@@ -2650,18 +2686,6 @@ class UnparseVisitor {
     Walk(x.v);
     Put(")");
   }
-  void Unparse(const OmpReductionCombiner::FunctionCombiner &x) {
-    const auto &pd = std::get<ProcedureDesignator>(x.v.t);
-    const auto &args = std::get<std::list<ActualArgSpec>>(x.v.t);
-    Walk(pd);
-    if (args.empty()) {
-      if (std::holds_alternative<ProcComponentRef>(pd.u)) {
-        Put("()");
-      }
-    } else {
-      Walk("(", args, ", ", ")");
-    }
-  }
   void Unparse(const OpenMPDeclareReductionConstruct &x) {
     BeginOpenMP();
     Word("!$OMP DECLARE REDUCTION ");
@@ -2677,7 +2701,7 @@ class UnparseVisitor {
   void Unparse(const OpenMPDeclareMapperConstruct &z) {
     BeginOpenMP();
     Word("!$OMP DECLARE MAPPER (");
-    const auto &spec{std::get<OmpDeclareMapperSpecifier>(z.t)};
+    const auto &spec{std::get<OmpMapperSpecifier>(z.t)};
     if (auto mapname{std::get<std::optional<Name>>(spec.t)}) {
       Walk(mapname);
       Put(":");
@@ -2725,6 +2749,15 @@ class UnparseVisitor {
     Walk(x.v);
     return false;
   }
+  void Unparse(const OmpDispatchDirective &x) {
+    Word("!$OMP DISPATCH");
+    Walk(x.t);
+    Put("\n");
+  }
+  void Unparse(const OmpEndDispatchDirective &) {
+    Word("!$OMP END DISPATCH");
+    Put("\n");
+  }
   void Unparse(const OmpErrorDirective &x) {
     Word("!$OMP ERROR ");
     Walk(x.t);
@@ -2799,6 +2832,13 @@ class UnparseVisitor {
                   },
         x.u);
   }
+  void Unparse(const OmpMetadirectiveDirective &x) {
+    BeginOpenMP();
+    Word("!$OMP METADIRECTIVE ");
+    Walk(std::get<OmpClauseList>(x.t));
+    Put("\n");
+    EndOpenMP();
+  }
   void Unparse(const OpenMPDepobjConstruct &x) {
     BeginOpenMP();
     Word("!$OMP DEPOBJ");
diff --git a/flang/lib/Semantics/CMakeLists.txt b/flang/lib/Semantics/CMakeLists.txt
index 7855ae7eed138..00108dde49dbd 100644
--- a/flang/lib/Semantics/CMakeLists.txt
+++ b/flang/lib/Semantics/CMakeLists.txt
@@ -53,7 +53,7 @@ add_flang_library(FortranSemantics
   omp_gen
 
   LINK_LIBS
-  FortranCommon
+  FortranSupport
   FortranParser
   FortranEvaluate
 
diff --git a/flang/lib/Semantics/assignment.cpp b/flang/lib/Semantics/assignment.cpp
index 0b57197fb8db8..627983d19a822 100644
--- a/flang/lib/Semantics/assignment.cpp
+++ b/flang/lib/Semantics/assignment.cpp
@@ -42,6 +42,7 @@ class AssignmentContext {
   void Analyze(const parser::AssignmentStmt &);
   void Analyze(const parser::PointerAssignmentStmt &);
   void Analyze(const parser::ConcurrentControl &);
+  int deviceConstructDepth_{0};
 
 private:
   bool CheckForPureContext(const SomeExpr &rhs, parser::CharBlock rhsSource);
@@ -90,6 +91,17 @@ void AssignmentContext::Analyze(const parser::AssignmentStmt &stmt) {
     if (whereDepth_ > 0) {
       CheckShape(lhsLoc, &lhs);
     }
+    if (context_.foldingContext().languageFeatures().IsEnabled(
+            common::LanguageFeature::CUDA)) {
+      const auto &scope{context_.FindScope(lhsLoc)};
+      const Scope &progUnit{GetProgramUnitContaining(scope)};
+      if (!IsCUDADeviceContext(&progUnit) && deviceConstructDepth_ == 0) {
+        if (Fortran::evaluate::HasCUDADeviceAttrs(lhs) &&
+            Fortran::evaluate::HasCUDAImplicitTransfer(rhs)) {
+          context_.Say(lhsLoc, "Unsupported CUDA data transfer"_err_en_US);
+        }
+      }
+    }
   }
 }
 
@@ -217,6 +229,46 @@ void AssignmentChecker::Enter(const parser::MaskedElsewhereStmt &x) {
 void AssignmentChecker::Leave(const parser::MaskedElsewhereStmt &) {
   context_.value().PopWhereContext();
 }
+void AssignmentChecker::Enter(const parser::CUFKernelDoConstruct &x) {
+  ++context_.value().deviceConstructDepth_;
+}
+void AssignmentChecker::Leave(const parser::CUFKernelDoConstruct &) {
+  --context_.value().deviceConstructDepth_;
+}
+static bool IsOpenACCComputeConstruct(const parser::OpenACCBlockConstruct &x) {
+  const auto &beginBlockDirective =
+      std::get<Fortran::parser::AccBeginBlockDirective>(x.t);
+  const auto &blockDirective =
+      std::get<Fortran::parser::AccBlockDirective>(beginBlockDirective.t);
+  if (blockDirective.v == llvm::acc::ACCD_parallel ||
+      blockDirective.v == llvm::acc::ACCD_serial ||
+      blockDirective.v == llvm::acc::ACCD_kernels) {
+    return true;
+  }
+  return false;
+}
+void AssignmentChecker::Enter(const parser::OpenACCBlockConstruct &x) {
+  if (IsOpenACCComputeConstruct(x)) {
+    ++context_.value().deviceConstructDepth_;
+  }
+}
+void AssignmentChecker::Leave(const parser::OpenACCBlockConstruct &x) {
+  if (IsOpenACCComputeConstruct(x)) {
+    --context_.value().deviceConstructDepth_;
+  }
+}
+void AssignmentChecker::Enter(const parser::OpenACCCombinedConstruct &) {
+  ++context_.value().deviceConstructDepth_;
+}
+void AssignmentChecker::Leave(const parser::OpenACCCombinedConstruct &) {
+  --context_.value().deviceConstructDepth_;
+}
+void AssignmentChecker::Enter(const parser::OpenACCLoopConstruct &) {
+  ++context_.value().deviceConstructDepth_;
+}
+void AssignmentChecker::Leave(const parser::OpenACCLoopConstruct &) {
+  --context_.value().deviceConstructDepth_;
+}
 
 } // namespace Fortran::semantics
 template class Fortran::common::Indirection<
diff --git a/flang/lib/Semantics/assignment.h b/flang/lib/Semantics/assignment.h
index 95d7b3cf91b17..a67bee4a03dfc 100644
--- a/flang/lib/Semantics/assignment.h
+++ b/flang/lib/Semantics/assignment.h
@@ -45,6 +45,14 @@ class AssignmentChecker : public virtual BaseChecker {
   void Leave(const parser::EndWhereStmt &);
   void Enter(const parser::MaskedElsewhereStmt &);
   void Leave(const parser::MaskedElsewhereStmt &);
+  void Enter(const parser::CUFKernelDoConstruct &);
+  void Leave(const parser::CUFKernelDoConstruct &);
+  void Enter(const parser::OpenACCBlockConstruct &);
+  void Leave(const parser::OpenACCBlockConstruct &);
+  void Enter(const parser::OpenACCCombinedConstruct &);
+  void Leave(const parser::OpenACCCombinedConstruct &);
+  void Enter(const parser::OpenACCLoopConstruct &);
+  void Leave(const parser::OpenACCLoopConstruct &);
 
 private:
   common::Indirection<AssignmentContext> context_;
diff --git a/flang/lib/Semantics/canonicalize-directives.cpp b/flang/lib/Semantics/canonicalize-directives.cpp
index 739bc3c1992ba..b27a27618808b 100644
--- a/flang/lib/Semantics/canonicalize-directives.cpp
+++ b/flang/lib/Semantics/canonicalize-directives.cpp
@@ -54,7 +54,9 @@ bool CanonicalizeDirectives(
 }
 
 static bool IsExecutionDirective(const parser::CompilerDirective &dir) {
-  return std::holds_alternative<parser::CompilerDirective::VectorAlways>(dir.u);
+  return std::holds_alternative<parser::CompilerDirective::VectorAlways>(
+             dir.u) ||
+      std::holds_alternative<parser::CompilerDirective::Unroll>(dir.u);
 }
 
 void CanonicalizationOfDirectives::Post(parser::SpecificationPart &spec) {
@@ -110,6 +112,9 @@ void CanonicalizationOfDirectives::Post(parser::Block &block) {
           common::visitors{[&](parser::CompilerDirective::VectorAlways &) {
                              CheckLoopDirective(*dir, block, it);
                            },
+              [&](parser::CompilerDirective::Unroll &) {
+                CheckLoopDirective(*dir, block, it);
+              },
               [&](auto &) {}},
           dir->u);
     }
diff --git a/flang/lib/Semantics/check-call.cpp b/flang/lib/Semantics/check-call.cpp
index ba68a0f898d46..e396ece303103 100644
--- a/flang/lib/Semantics/check-call.cpp
+++ b/flang/lib/Semantics/check-call.cpp
@@ -67,10 +67,6 @@ static void CheckImplicitInterfaceArg(evaluate::ActualArgument &arg,
           "Null pointer argument requires an explicit interface"_err_en_US);
     } else if (auto named{evaluate::ExtractNamedEntity(*expr)}) {
       const Symbol &symbol{named->GetLastSymbol()};
-      if (symbol.Corank() > 0) {
-        messages.Say(
-            "Coarray argument requires an explicit interface"_err_en_US);
-      }
       if (evaluate::IsAssumedRank(symbol)) {
         messages.Say(
             "Assumed rank argument requires an explicit interface"_err_en_US);
@@ -116,9 +112,9 @@ static bool CanAssociateWithStorageSequence(
              characteristics::TypeAndShape::Attr::AssumedRank) &&
       !dummy.type.attrs().test(
           characteristics::TypeAndShape::Attr::AssumedShape) &&
-      !dummy.type.attrs().test(characteristics::TypeAndShape::Attr::Coarray) &&
       !dummy.attrs.test(characteristics::DummyDataObject::Attr::Allocatable) &&
-      !dummy.attrs.test(characteristics::DummyDataObject::Attr::Pointer);
+      !dummy.attrs.test(characteristics::DummyDataObject::Attr::Pointer) &&
+      dummy.type.corank() == 0;
 }
 
 // When a CHARACTER actual argument is known to be short,
@@ -167,7 +163,9 @@ static void CheckCharacterActual(evaluate::Expr<evaluate::SomeType> &actual,
                   context.foldingContext(), /*getLastComponent=*/true};
               if (auto actualOffset{folder.FoldDesignator(actual)}) {
                 std::int64_t actualChars{*actualLength};
-                if (static_cast<std::size_t>(actualOffset->offset()) >=
+                if (IsAllocatableOrPointer(actualOffset->symbol())) {
+                  // don't use actualOffset->symbol().size()!
+                } else if (static_cast<std::size_t>(actualOffset->offset()) >=
                         actualOffset->symbol().size() ||
                     !evaluate::IsContiguous(
                         actualOffset->symbol(), foldingContext)) {
@@ -634,7 +632,9 @@ static void CheckExplicitDataArg(const characteristics::DummyDataObject &dummy,
                 context.foldingContext(), /*getLastComponent=*/true};
             if (auto actualOffset{folder.FoldDesignator(actual)}) {
               std::optional<std::int64_t> actualElements;
-              if (static_cast<std::size_t>(actualOffset->offset()) >=
+              if (IsAllocatableOrPointer(actualOffset->symbol())) {
+                // don't use actualOffset->symbol().size()!
+              } else if (static_cast<std::size_t>(actualOffset->offset()) >=
                       actualOffset->symbol().size() ||
                   !evaluate::IsContiguous(
                       actualOffset->symbol(), foldingContext)) {
@@ -1616,6 +1616,36 @@ static void CheckAssociated(evaluate::ActualArguments &arguments,
   }
 }
 
+// EVENT_QUERY (F'2023 16.9.82)
+static void CheckEvent_Query(evaluate::ActualArguments &arguments,
+    evaluate::FoldingContext &foldingContext) {
+  if (arguments.size() > 0 && arguments[0] &&
+      ExtractCoarrayRef(*arguments[0]).has_value()) {
+    foldingContext.messages().Say(arguments[0]->sourceLocation(),
+        "EVENT= argument to EVENT_QUERY must not be coindexed"_err_en_US);
+  }
+  if (arguments.size() > 1 && arguments[1]) {
+    if (auto dyType{arguments[1]->GetType()}) {
+      int defaultInt{
+          foldingContext.defaults().GetDefaultKind(TypeCategory::Integer)};
+      if (dyType->category() == TypeCategory::Integer &&
+          dyType->kind() < defaultInt) {
+        foldingContext.messages().Say(arguments[1]->sourceLocation(),
+            "COUNT= argument to EVENT_QUERY must be an integer with kind >= %d"_err_en_US,
+            defaultInt);
+      }
+    }
+  }
+  if (arguments.size() > 2 && arguments[2]) {
+    if (auto dyType{arguments[2]->GetType()}) {
+      if (dyType->category() == TypeCategory::Integer && dyType->kind() < 2) {
+        foldingContext.messages().Say(arguments[2]->sourceLocation(),
+            "STAT= argument to EVENT_QUERY must be an integer with kind >= 2 when present"_err_en_US);
+      }
+    }
+  }
+}
+
 // IMAGE_INDEX (F'2023 16.9.107)
 static void CheckImage_Index(evaluate::ActualArguments &arguments,
     parser::ContextualMessages &messages) {
@@ -1952,6 +1982,8 @@ static void CheckSpecificIntrinsic(const characteristics::Procedure &proc,
     const Scope *scope, const evaluate::SpecificIntrinsic &intrinsic) {
   if (intrinsic.name == "associated") {
     CheckAssociated(arguments, context, scope);
+  } else if (intrinsic.name == "event_query") {
+    CheckEvent_Query(arguments, context.foldingContext());
   } else if (intrinsic.name == "image_index") {
     CheckImage_Index(arguments, context.foldingContext().messages());
   } else if (intrinsic.name == "max" || intrinsic.name == "min") {
diff --git a/flang/lib/Semantics/check-coarray.cpp b/flang/lib/Semantics/check-coarray.cpp
index 6cf61a6b923db..6bed525d7f687 100644
--- a/flang/lib/Semantics/check-coarray.cpp
+++ b/flang/lib/Semantics/check-coarray.cpp
@@ -133,9 +133,6 @@ static void CheckEventVariable(
     if (!IsEventType(evaluate::GetDerivedTypeSpec(expr->GetType()))) { // C1176
       context.Say(parser::FindSourceLocation(eventVar),
           "The event-variable must be of type EVENT_TYPE from module ISO_FORTRAN_ENV"_err_en_US);
-    } else if (!evaluate::IsCoarray(*expr)) { // C1604
-      context.Say(parser::FindSourceLocation(eventVar),
-          "The event-variable must be a coarray"_err_en_US);
     }
   }
 }
diff --git a/flang/lib/Semantics/check-declarations.cpp b/flang/lib/Semantics/check-declarations.cpp
index a7e6cf32e85ee..bf4dc16a15b4a 100644
--- a/flang/lib/Semantics/check-declarations.cpp
+++ b/flang/lib/Semantics/check-declarations.cpp
@@ -683,7 +683,20 @@ void CheckHelper::CheckObjectEntity(
   const DeclTypeSpec *type{details.type()};
   const DerivedTypeSpec *derived{type ? type->AsDerived() : nullptr};
   bool isComponent{symbol.owner().IsDerivedType()};
-  if (!details.coshape().empty()) {
+  if (details.coshape().empty()) { // not a coarray
+    if (!isComponent && !IsPointer(symbol) && derived) {
+      if (IsEventTypeOrLockType(derived)) {
+        messages_.Say(
+            "Variable '%s' with EVENT_TYPE or LOCK_TYPE must be a coarray"_err_en_US,
+            symbol.name());
+      } else if (auto component{FindEventOrLockPotentialComponent(
+                     *derived, /*ignoreCoarrays=*/true)}) {
+        messages_.Say(
+            "Variable '%s' with EVENT_TYPE or LOCK_TYPE potential component '%s' must be a coarray"_err_en_US,
+            symbol.name(), component.BuildResultDesignatorName());
+      }
+    }
+  } else { // it's a coarray
     bool isDeferredCoshape{details.coshape().CanBeDeferredShape()};
     if (IsAllocatable(symbol)) {
       if (!isDeferredCoshape) { // C827
@@ -976,9 +989,9 @@ void CheckHelper::CheckObjectEntity(
       }
       break;
     case common::CUDADataAttr::Device:
-      if (isComponent && !IsAllocatable(symbol)) {
+      if (isComponent && !IsAllocatable(symbol) && !IsPointer(symbol)) {
         messages_.Say(
-            "Component '%s' with ATTRIBUTES(DEVICE) must also be allocatable"_err_en_US,
+            "Component '%s' with ATTRIBUTES(DEVICE) must also be allocatable or pointer"_err_en_US,
             symbol.name());
       }
       break;
@@ -3076,16 +3089,17 @@ parser::Messages CheckHelper::WhyNotInteroperableObject(
       }
     }
     if (type->IsAssumedType()) { // ok
-    } else if (IsAssumedLengthCharacter(symbol)) {
+    } else if (IsAssumedLengthCharacter(symbol) &&
+        !IsAllocatableOrPointer(symbol)) {
     } else if (IsAllocatableOrPointer(symbol) &&
         type->category() == DeclTypeSpec::Character &&
         type->characterTypeSpec().length().isDeferred()) {
       // ok; F'2023 18.3.7 p2(6)
     } else if (derived) { // type has been checked
     } else if (auto dyType{evaluate::DynamicType::From(*type)}; dyType &&
-               evaluate::IsInteroperableIntrinsicType(*dyType,
-                   InModuleFile() ? nullptr : &context_.languageFeatures())
-                   .value_or(false)) {
+        evaluate::IsInteroperableIntrinsicType(
+            *dyType, InModuleFile() ? nullptr : &context_.languageFeatures())
+            .value_or(false)) {
       // F'2023 18.3.7 p2(4,5)
       // N.B. Language features are not passed to IsInteroperableIntrinsicType
       // when processing a module file, since the module file might have been
diff --git a/flang/lib/Semantics/check-io.cpp b/flang/lib/Semantics/check-io.cpp
index 3c99163c1f134..42c3b9e11efc1 100644
--- a/flang/lib/Semantics/check-io.cpp
+++ b/flang/lib/Semantics/check-io.cpp
@@ -920,7 +920,7 @@ void IoChecker::CheckStringValue(IoSpecKind specKind, const std::string &value,
       {IoSpecKind::Decimal, {"COMMA", "POINT"}},
       {IoSpecKind::Delim, {"APOSTROPHE", "NONE", "QUOTE"}},
       {IoSpecKind::Encoding, {"DEFAULT", "UTF-8"}},
-      {IoSpecKind::Form, {"FORMATTED", "UNFORMATTED"}},
+      {IoSpecKind::Form, {"FORMATTED", "UNFORMATTED", "BINARY"}},
       {IoSpecKind::Pad, {"NO", "YES"}},
       {IoSpecKind::Position, {"APPEND", "ASIS", "REWIND"}},
       {IoSpecKind::Round,
diff --git a/flang/lib/Semantics/check-omp-structure.cpp b/flang/lib/Semantics/check-omp-structure.cpp
index d3f2d3fd2f9dc..fd2893998205c 100644
--- a/flang/lib/Semantics/check-omp-structure.cpp
+++ b/flang/lib/Semantics/check-omp-structure.cpp
@@ -9,6 +9,8 @@
 #include "check-omp-structure.h"
 #include "definable.h"
 #include "flang/Evaluate/check-expression.h"
+#include "flang/Evaluate/expression.h"
+#include "flang/Evaluate/type.h"
 #include "flang/Parser/parse-tree.h"
 #include "flang/Semantics/expression.h"
 #include "flang/Semantics/openmp-modifiers.h"
@@ -238,6 +240,16 @@ class OmpUnitedTaskDesignatorChecker {
 };
 
 bool OmpStructureChecker::CheckAllowedClause(llvmOmpClause clause) {
+  // Do not do clause checks while processing METADIRECTIVE.
+  // Context selectors can contain clauses that are not given as a part
+  // of a construct, but as trait properties. Testing whether they are
+  // valid or not is deferred to the checks of the context selectors.
+  // As it stands now, these clauses would appear as if they were present
+  // on METADIRECTIVE, leading to incorrect diagnostics.
+  if (GetDirectiveNest(ContextSelectorNest) > 0) {
+    return true;
+  }
+
   unsigned version{context_.langOptions().OpenMPVersion};
   DirectiveContext &dirCtx = GetContext();
   llvm::omp::Directive dir{dirCtx.directive};
@@ -614,6 +626,22 @@ void OmpStructureChecker::CheckHintClause(
   }
 }
 
+void OmpStructureChecker::Enter(const parser::OmpDirectiveSpecification &x) {
+  PushContextAndClauseSets(x.source, std::get<llvm::omp::Directive>(x.t));
+}
+
+void OmpStructureChecker::Leave(const parser::OmpDirectiveSpecification &) {
+  dirContext_.pop_back();
+}
+
+void OmpStructureChecker::Enter(const parser::OmpMetadirectiveDirective &x) {
+  PushContextAndClauseSets(x.source, llvm::omp::Directive::OMPD_metadirective);
+}
+
+void OmpStructureChecker::Leave(const parser::OmpMetadirectiveDirective &) {
+  dirContext_.pop_back();
+}
+
 void OmpStructureChecker::Enter(const parser::OpenMPConstruct &x) {
   // Simd Construct with Ordered Construct Nesting check
   // We cannot use CurrentDirectiveIsNested() here because
@@ -1612,7 +1640,7 @@ void OmpStructureChecker::Enter(const parser::OpenMPDeclareMapperConstruct &x) {
   const auto &dir{std::get<parser::Verbatim>(x.t)};
   PushContextAndClauseSets(
       dir.source, llvm::omp::Directive::OMPD_declare_mapper);
-  const auto &spec{std::get<parser::OmpDeclareMapperSpecifier>(x.t)};
+  const auto &spec{std::get<parser::OmpMapperSpecifier>(x.t)};
   const auto &type = std::get<parser::TypeSpec>(spec.t);
   if (!std::get_if<parser::DerivedTypeSpec>(&type.u)) {
     context_.Say(dir.source, "Type is not a derived type"_err_en_US);
@@ -1748,6 +1776,36 @@ void OmpStructureChecker::Enter(const parser::OmpErrorDirective &x) {
   PushContextAndClauseSets(dir.source, llvm::omp::Directive::OMPD_error);
 }
 
+void OmpStructureChecker::Enter(const parser::OpenMPDispatchConstruct &x) {
+  PushContextAndClauseSets(x.source, llvm::omp::Directive::OMPD_dispatch);
+  const auto &block{std::get<parser::Block>(x.t)};
+  if (block.empty() || block.size() > 1) {
+    context_.Say(x.source,
+        "The DISPATCH construct is empty or contains more than one statement"_err_en_US);
+    return;
+  }
+
+  auto it{block.begin()};
+  bool passChecks{false};
+  if (const parser::AssignmentStmt *
+      assignStmt{parser::Unwrap<parser::AssignmentStmt>(*it)}) {
+    if (parser::Unwrap<parser::FunctionReference>(assignStmt->t)) {
+      passChecks = true;
+    }
+  } else if (parser::Unwrap<parser::CallStmt>(*it)) {
+    passChecks = true;
+  }
+
+  if (!passChecks) {
+    context_.Say(x.source,
+        "The DISPATCH construct does not contain a SUBROUTINE or FUNCTION"_err_en_US);
+  }
+}
+
+void OmpStructureChecker::Leave(const parser::OpenMPDispatchConstruct &x) {
+  dirContext_.pop_back();
+}
+
 void OmpStructureChecker::Leave(const parser::OmpErrorDirective &x) {
   dirContext_.pop_back();
 }
@@ -2908,6 +2966,7 @@ CHECK_SIMPLE_CLAUSE(Indirect, OMPC_indirect)
 CHECK_SIMPLE_CLAUSE(Mergeable, OMPC_mergeable)
 CHECK_SIMPLE_CLAUSE(NoOpenmp, OMPC_no_openmp)
 CHECK_SIMPLE_CLAUSE(NoOpenmpRoutines, OMPC_no_openmp_routines)
+CHECK_SIMPLE_CLAUSE(NoOpenmpConstructs, OMPC_no_openmp_constructs)
 CHECK_SIMPLE_CLAUSE(NoParallelism, OMPC_no_parallelism)
 CHECK_SIMPLE_CLAUSE(Nogroup, OMPC_nogroup)
 CHECK_SIMPLE_CLAUSE(Notinbranch, OMPC_notinbranch)
@@ -2928,7 +2987,7 @@ CHECK_SIMPLE_CLAUSE(Nocontext, OMPC_nocontext)
 CHECK_SIMPLE_CLAUSE(Severity, OMPC_severity)
 CHECK_SIMPLE_CLAUSE(Message, OMPC_message)
 CHECK_SIMPLE_CLAUSE(Filter, OMPC_filter)
-CHECK_SIMPLE_CLAUSE(When, OMPC_when)
+CHECK_SIMPLE_CLAUSE(Otherwise, OMPC_otherwise)
 CHECK_SIMPLE_CLAUSE(AdjustArgs, OMPC_adjust_args)
 CHECK_SIMPLE_CLAUSE(AppendArgs, OMPC_append_args)
 CHECK_SIMPLE_CLAUSE(MemoryOrder, OMPC_memory_order)
@@ -3167,9 +3226,12 @@ void OmpStructureChecker::CheckReductionObjects(
     }
   }
 
+  // Denied in all current versions of the standard because structure components
+  // are not definable (i.e. they are expressions not variables).
+  // Object cannot be a part of another object (except array elements).
+  CheckStructureComponent(objects, clauseId);
+
   if (version >= 50) {
-    // Object cannot be a part of another object (except array elements)
-    CheckStructureComponent(objects, clauseId);
     // If object is an array section or element, the base expression must be
     // a language identifier.
     for (const parser::OmpObject &object : objects.v) {
@@ -3408,6 +3470,7 @@ void OmpStructureChecker::Enter(const parser::OmpClause::Ordered &x) {
 void OmpStructureChecker::Enter(const parser::OmpClause::Shared &x) {
   CheckAllowedClause(llvm::omp::Clause::OMPC_shared);
   CheckIsVarPartOfAnotherVar(GetContext().clauseSource, x.v, "SHARED");
+  CheckCrayPointee(x.v, "SHARED");
 }
 void OmpStructureChecker::Enter(const parser::OmpClause::Private &x) {
   SymbolSourceMap symbols;
@@ -3415,6 +3478,7 @@ void OmpStructureChecker::Enter(const parser::OmpClause::Private &x) {
   CheckAllowedClause(llvm::omp::Clause::OMPC_private);
   CheckIsVarPartOfAnotherVar(GetContext().clauseSource, x.v, "PRIVATE");
   CheckIntentInPointer(symbols, llvm::omp::Clause::OMPC_private);
+  CheckCrayPointee(x.v, "PRIVATE");
 }
 
 void OmpStructureChecker::Enter(const parser::OmpClause::Nowait &x) {
@@ -3494,6 +3558,7 @@ void OmpStructureChecker::Enter(const parser::OmpClause::Firstprivate &x) {
   CheckAllowedClause(llvm::omp::Clause::OMPC_firstprivate);
 
   CheckIsVarPartOfAnotherVar(GetContext().clauseSource, x.v, "FIRSTPRIVATE");
+  CheckCrayPointee(x.v, "FIRSTPRIVATE");
   CheckIsLoopIvPartOfClause(llvmOmpClause::OMPC_firstprivate, x.v);
 
   SymbolSourceMap currSymbols;
@@ -3728,6 +3793,7 @@ void OmpStructureChecker::Enter(const parser::OmpClause::Linear &x) {
 
   SymbolSourceMap symbols;
   auto &objects{std::get<parser::OmpObjectList>(x.v.t)};
+  CheckCrayPointee(objects, "LINEAR", false);
   GetSymbolsInObjectList(objects, symbols);
 
   auto CheckIntegerNoRef{[&](const Symbol *symbol, parser::CharBlock source) {
@@ -4173,6 +4239,7 @@ void OmpStructureChecker::Enter(const parser::OmpClause::Lastprivate &x) {
   const auto &objectList{std::get<parser::OmpObjectList>(x.v.t)};
   CheckIsVarPartOfAnotherVar(
       GetContext().clauseSource, objectList, "LASTPRIVATE");
+  CheckCrayPointee(objectList, "LASTPRIVATE");
 
   DirectivesClauseTriple dirClauseTriple;
   SymbolSourceMap currSymbols;
@@ -4475,6 +4542,518 @@ void OmpStructureChecker::Enter(const parser::OmpClause::OmpxBare &x) {
   }
 }
 
+void OmpStructureChecker::Enter(const parser::OmpClause::When &x) {
+  CheckAllowedClause(llvm::omp::Clause::OMPC_when);
+  OmpVerifyModifiers(
+      x.v, llvm::omp::OMPC_when, GetContext().clauseSource, context_);
+}
+
+void OmpStructureChecker::Enter(const parser::OmpContextSelector &ctx) {
+  EnterDirectiveNest(ContextSelectorNest);
+
+  using SetName = parser::OmpTraitSetSelectorName;
+  std::map<SetName::Value, const SetName *> visited;
+
+  for (const parser::OmpTraitSetSelector &traitSet : ctx.v) {
+    auto &name{std::get<SetName>(traitSet.t)};
+    auto [prev, unique]{visited.insert(std::make_pair(name.v, &name))};
+    if (!unique) {
+      std::string showName{parser::ToUpperCaseLetters(name.ToString())};
+      parser::MessageFormattedText txt(
+          "Repeated trait set name %s in a context specifier"_err_en_US,
+          showName);
+      parser::Message message(name.source, txt);
+      message.Attach(prev->second->source,
+          "Previous trait set %s provided here"_en_US, showName);
+      context_.Say(std::move(message));
+    }
+    CheckTraitSetSelector(traitSet);
+  }
+}
+
+void OmpStructureChecker::Leave(const parser::OmpContextSelector &) {
+  ExitDirectiveNest(ContextSelectorNest);
+}
+
+std::optional<evaluate::DynamicType> OmpStructureChecker::GetDynamicType(
+    const common::Indirection<parser::Expr> &parserExpr) {
+  // Indirection<parser::Expr>      parserExpr
+  //  `- parser::Expr               ^.value()
+  const parser::TypedExpr &typedExpr{parserExpr.value().typedExpr};
+  // ForwardOwningPointer           typedExpr
+  // `- GenericExprWrapper          ^.get()
+  //    `- std::optional<Expr>      ^->v
+  if (auto maybeExpr{typedExpr.get()->v}) {
+    return maybeExpr->GetType();
+  } else {
+    return std::nullopt;
+  }
+}
+
+const std::list<parser::OmpTraitProperty> &
+OmpStructureChecker::GetTraitPropertyList(
+    const parser::OmpTraitSelector &trait) {
+  static const std::list<parser::OmpTraitProperty> empty{};
+  auto &[_, maybeProps]{trait.t};
+  if (maybeProps) {
+    using PropertyList = std::list<parser::OmpTraitProperty>;
+    return std::get<PropertyList>(maybeProps->t);
+  } else {
+    return empty;
+  }
+}
+
+std::optional<llvm::omp::Clause> OmpStructureChecker::GetClauseFromProperty(
+    const parser::OmpTraitProperty &property) {
+  using MaybeClause = std::optional<llvm::omp::Clause>;
+
+  // The parser for OmpClause will only succeed if the clause was
+  // given with all required arguments.
+  // If this is a string or complex extension with a clause name,
+  // treat it as a clause and let the trait checker deal with it.
+
+  auto getClauseFromString{[&](const std::string &s) -> MaybeClause {
+    auto id{llvm::omp::getOpenMPClauseKind(parser::ToLowerCaseLetters(s))};
+    if (id != llvm::omp::Clause::OMPC_unknown) {
+      return id;
+    } else {
+      return std::nullopt;
+    }
+  }};
+
+  return common::visit( //
+      common::visitors{
+          [&](const parser::OmpTraitPropertyName &x) -> MaybeClause {
+            return getClauseFromString(x.v);
+          },
+          [&](const common::Indirection<parser::OmpClause> &x) -> MaybeClause {
+            return x.value().Id();
+          },
+          [&](const parser::ScalarExpr &x) -> MaybeClause {
+            return std::nullopt;
+          },
+          [&](const parser::OmpTraitPropertyExtension &x) -> MaybeClause {
+            using ExtProperty = parser::OmpTraitPropertyExtension;
+            if (auto *name{std::get_if<parser::OmpTraitPropertyName>(&x.u)}) {
+              return getClauseFromString(name->v);
+            } else if (auto *cpx{std::get_if<ExtProperty::Complex>(&x.u)}) {
+              return getClauseFromString(
+                  std::get<parser::OmpTraitPropertyName>(cpx->t).v);
+            }
+            return std::nullopt;
+          },
+      },
+      property.u);
+}
+
+void OmpStructureChecker::CheckTraitSelectorList(
+    const std::list<parser::OmpTraitSelector> &traits) {
+  // [6.0:322:20]
+  // Each trait-selector-name may only be specified once in a trait selector
+  // set.
+
+  // Cannot store OmpTraitSelectorName directly, because it's not copyable.
+  using TraitName = parser::OmpTraitSelectorName;
+  using BareName = decltype(TraitName::u);
+  std::map<BareName, const TraitName *> visited;
+
+  for (const parser::OmpTraitSelector &trait : traits) {
+    auto &name{std::get<TraitName>(trait.t)};
+
+    auto [prev, unique]{visited.insert(std::make_pair(name.u, &name))};
+    if (!unique) {
+      std::string showName{parser::ToUpperCaseLetters(name.ToString())};
+      parser::MessageFormattedText txt(
+          "Repeated trait name %s in a trait set"_err_en_US, showName);
+      parser::Message message(name.source, txt);
+      message.Attach(prev->second->source,
+          "Previous trait %s provided here"_en_US, showName);
+      context_.Say(std::move(message));
+    }
+  }
+}
+
+void OmpStructureChecker::CheckTraitSetSelector(
+    const parser::OmpTraitSetSelector &traitSet) {
+
+  // Trait Set      |           Allowed traits | D-traits | X-traits | Score |
+  //
+  // Construct      |     Simd, directive-name |      Yes |       No |    No |
+  // Device         |          Arch, Isa, Kind |       No |      Yes |    No |
+  // Implementation | Atomic_Default_Mem_Order |       No |      Yes |   Yes |
+  //                |      Extension, Requires |          |          |       |
+  //                |                   Vendor |          |          |       |
+  // Target_Device  |    Arch, Device_Num, Isa |       No |      Yes |    No |
+  //                |                Kind, Uid |          |          |       |
+  // User           |                Condition |       No |       No |   Yes |
+
+  struct TraitSetConfig {
+    std::set<parser::OmpTraitSelectorName::Value> allowed;
+    bool allowsDirectiveTraits;
+    bool allowsExtensionTraits;
+    bool allowsScore;
+  };
+
+  using SName = parser::OmpTraitSetSelectorName::Value;
+  using TName = parser::OmpTraitSelectorName::Value;
+
+  static const std::map<SName, TraitSetConfig> configs{
+      {SName::Construct, //
+          {{TName::Simd}, true, false, false}},
+      {SName::Device, //
+          {{TName::Arch, TName::Isa, TName::Kind}, false, true, false}},
+      {SName::Implementation, //
+          {{TName::Atomic_Default_Mem_Order, TName::Extension, TName::Requires,
+               TName::Vendor},
+              false, true, true}},
+      {SName::Target_Device, //
+          {{TName::Arch, TName::Device_Num, TName::Isa, TName::Kind,
+               TName::Uid},
+              false, true, false}},
+      {SName::User, //
+          {{TName::Condition}, false, false, true}},
+  };
+
+  auto checkTraitSet{[&](const TraitSetConfig &config) {
+    auto &[setName, traits]{traitSet.t};
+    auto usn{parser::ToUpperCaseLetters(setName.ToString())};
+
+    // Check if there are any duplicate traits.
+    CheckTraitSelectorList(traits);
+
+    for (const parser::OmpTraitSelector &trait : traits) {
+      // Don't use structured bindings here, because they cannot be captured
+      // before C++20.
+      auto &traitName = std::get<parser::OmpTraitSelectorName>(trait.t);
+      auto &maybeProps =
+          std::get<std::optional<parser::OmpTraitSelector::Properties>>(
+              trait.t);
+
+      // Check allowed traits
+      common::visit( //
+          common::visitors{
+              [&](parser::OmpTraitSelectorName::Value v) {
+                if (!config.allowed.count(v)) {
+                  context_.Say(traitName.source,
+                      "%s is not a valid trait for %s trait set"_err_en_US,
+                      parser::ToUpperCaseLetters(traitName.ToString()), usn);
+                }
+              },
+              [&](llvm::omp::Directive) {
+                if (!config.allowsDirectiveTraits) {
+                  context_.Say(traitName.source,
+                      "Directive name is not a valid trait for %s trait set"_err_en_US,
+                      usn);
+                }
+              },
+              [&](const std::string &) {
+                if (!config.allowsExtensionTraits) {
+                  context_.Say(traitName.source,
+                      "Extension traits are not valid for %s trait set"_err_en_US,
+                      usn);
+                }
+              },
+          },
+          traitName.u);
+
+      // Check score
+      if (maybeProps) {
+        auto &[maybeScore, _]{maybeProps->t};
+        if (maybeScore) {
+          CheckTraitScore(*maybeScore);
+        }
+      }
+
+      // Check the properties of the individual traits
+      CheckTraitSelector(traitSet, trait);
+    }
+  }};
+
+  checkTraitSet(
+      configs.at(std::get<parser::OmpTraitSetSelectorName>(traitSet.t).v));
+}
+
+void OmpStructureChecker::CheckTraitScore(const parser::OmpTraitScore &score) {
+  // [6.0:322:23]
+  // A score-expression must be a non-negative constant integer expression.
+  if (auto value{GetIntValue(score)}; !value || value <= 0) {
+    context_.Say(score.source,
+        "SCORE expression must be a non-negative constant integer expression"_err_en_US);
+  }
+}
+
+bool OmpStructureChecker::VerifyTraitPropertyLists(
+    const parser::OmpTraitSetSelector &traitSet,
+    const parser::OmpTraitSelector &trait) {
+  using TraitName = parser::OmpTraitSelectorName;
+  using PropertyList = std::list<parser::OmpTraitProperty>;
+  auto &[traitName, maybeProps]{trait.t};
+
+  auto checkPropertyList{[&](const PropertyList &properties, auto isValid,
+                             const std::string &message) {
+    bool foundInvalid{false};
+    for (const parser::OmpTraitProperty &prop : properties) {
+      if (!isValid(prop)) {
+        if (foundInvalid) {
+          context_.Say(
+              prop.source, "More invalid properties are present"_err_en_US);
+          break;
+        }
+        context_.Say(prop.source, "%s"_err_en_US, message);
+        foundInvalid = true;
+      }
+    }
+    return !foundInvalid;
+  }};
+
+  bool invalid{false};
+
+  if (std::holds_alternative<llvm::omp::Directive>(traitName.u)) {
+    // Directive-name traits don't have properties.
+    if (maybeProps) {
+      context_.Say(trait.source,
+          "Directive-name traits cannot have properties"_err_en_US);
+      invalid = true;
+    }
+  }
+  // Ignore properties on extension traits.
+
+  // See `TraitSelectorParser` in openmp-parser.cpp
+  if (auto *v{std::get_if<TraitName::Value>(&traitName.u)}) {
+    switch (*v) {
+    // name-list properties
+    case parser::OmpTraitSelectorName::Value::Arch:
+    case parser::OmpTraitSelectorName::Value::Extension:
+    case parser::OmpTraitSelectorName::Value::Isa:
+    case parser::OmpTraitSelectorName::Value::Kind:
+    case parser::OmpTraitSelectorName::Value::Uid:
+    case parser::OmpTraitSelectorName::Value::Vendor:
+      if (maybeProps) {
+        auto isName{[](const parser::OmpTraitProperty &prop) {
+          return std::holds_alternative<parser::OmpTraitPropertyName>(prop.u);
+        }};
+        invalid = !checkPropertyList(std::get<PropertyList>(maybeProps->t),
+            isName, "Trait property should be a name");
+      }
+      break;
+    // clause-list
+    case parser::OmpTraitSelectorName::Value::Atomic_Default_Mem_Order:
+    case parser::OmpTraitSelectorName::Value::Requires:
+    case parser::OmpTraitSelectorName::Value::Simd:
+      if (maybeProps) {
+        auto isClause{[&](const parser::OmpTraitProperty &prop) {
+          return GetClauseFromProperty(prop).has_value();
+        }};
+        invalid = !checkPropertyList(std::get<PropertyList>(maybeProps->t),
+            isClause, "Trait property should be a clause");
+      }
+      break;
+    // expr-list
+    case parser::OmpTraitSelectorName::Value::Condition:
+    case parser::OmpTraitSelectorName::Value::Device_Num:
+      if (maybeProps) {
+        auto isExpr{[](const parser::OmpTraitProperty &prop) {
+          return std::holds_alternative<parser::ScalarExpr>(prop.u);
+        }};
+        invalid = !checkPropertyList(std::get<PropertyList>(maybeProps->t),
+            isExpr, "Trait property should be a scalar expression");
+      }
+      break;
+    } // switch
+  }
+
+  return !invalid;
+}
+
+void OmpStructureChecker::CheckTraitSelector(
+    const parser::OmpTraitSetSelector &traitSet,
+    const parser::OmpTraitSelector &trait) {
+  using TraitName = parser::OmpTraitSelectorName;
+  auto &[traitName, maybeProps]{trait.t};
+
+  // Only do the detailed checks if the property lists are valid.
+  if (VerifyTraitPropertyLists(traitSet, trait)) {
+    if (std::holds_alternative<llvm::omp::Directive>(traitName.u) ||
+        std::holds_alternative<std::string>(traitName.u)) {
+      // No properties here: directives don't have properties, and
+      // we don't implement any extension traits now.
+      return;
+    }
+
+    // Specific traits we want to check.
+    // Limitations:
+    // (1) The properties for these traits are defined in "Additional
+    // Definitions for the OpenMP API Specification". It's not clear how
+    // to define them in a portable way, and how to verify their validity,
+    // especially if they get replaced by their integer values (in case
+    // they are defined as enums).
+    // (2) These are entirely implementation-defined, and at the moment
+    // there is no known schema to validate these values.
+    auto v{std::get<TraitName::Value>(traitName.u)};
+    switch (v) {
+    case TraitName::Value::Arch:
+      // Unchecked, TBD(1)
+      break;
+    case TraitName::Value::Atomic_Default_Mem_Order:
+      CheckTraitADMO(traitSet, trait);
+      break;
+    case TraitName::Value::Condition:
+      CheckTraitCondition(traitSet, trait);
+      break;
+    case TraitName::Value::Device_Num:
+      CheckTraitDeviceNum(traitSet, trait);
+      break;
+    case TraitName::Value::Extension:
+      // Ignore
+      break;
+    case TraitName::Value::Isa:
+      // Unchecked, TBD(1)
+      break;
+    case TraitName::Value::Kind:
+      // Unchecked, TBD(1)
+      break;
+    case TraitName::Value::Requires:
+      CheckTraitRequires(traitSet, trait);
+      break;
+    case TraitName::Value::Simd:
+      CheckTraitSimd(traitSet, trait);
+      break;
+    case TraitName::Value::Uid:
+      // Unchecked, TBD(2)
+      break;
+    case TraitName::Value::Vendor:
+      // Unchecked, TBD(1)
+      break;
+    }
+  }
+}
+
+void OmpStructureChecker::CheckTraitADMO(
+    const parser::OmpTraitSetSelector &traitSet,
+    const parser::OmpTraitSelector &trait) {
+  auto &traitName{std::get<parser::OmpTraitSelectorName>(trait.t)};
+  auto &properties{GetTraitPropertyList(trait)};
+
+  if (properties.size() != 1) {
+    context_.Say(trait.source,
+        "%s trait requires a single clause property"_err_en_US,
+        parser::ToUpperCaseLetters(traitName.ToString()));
+  } else {
+    const parser::OmpTraitProperty &property{properties.front()};
+    auto clauseId{*GetClauseFromProperty(property)};
+    // Check that the clause belongs to the memory-order clause-set.
+    // Clause sets will hopefully be autogenerated at some point.
+    switch (clauseId) {
+    case llvm::omp::Clause::OMPC_acq_rel:
+    case llvm::omp::Clause::OMPC_acquire:
+    case llvm::omp::Clause::OMPC_relaxed:
+    case llvm::omp::Clause::OMPC_release:
+    case llvm::omp::Clause::OMPC_seq_cst:
+      break;
+    default:
+      context_.Say(property.source,
+          "%s trait requires a clause from the memory-order clause set"_err_en_US,
+          parser::ToUpperCaseLetters(traitName.ToString()));
+    }
+
+    using ClauseProperty = common::Indirection<parser::OmpClause>;
+    if (!std::holds_alternative<ClauseProperty>(property.u)) {
+      context_.Say(property.source,
+          "Invalid clause specification for %s"_err_en_US,
+          parser::ToUpperCaseLetters(getClauseName(clauseId)));
+    }
+  }
+}
+
+void OmpStructureChecker::CheckTraitCondition(
+    const parser::OmpTraitSetSelector &traitSet,
+    const parser::OmpTraitSelector &trait) {
+  auto &traitName{std::get<parser::OmpTraitSelectorName>(trait.t)};
+  auto &properties{GetTraitPropertyList(trait)};
+
+  if (properties.size() != 1) {
+    context_.Say(trait.source,
+        "%s trait requires a single expression property"_err_en_US,
+        parser::ToUpperCaseLetters(traitName.ToString()));
+  } else {
+    const parser::OmpTraitProperty &property{properties.front()};
+    auto &scalarExpr{std::get<parser::ScalarExpr>(property.u)};
+
+    auto maybeType{GetDynamicType(scalarExpr.thing)};
+    if (!maybeType || maybeType->category() != TypeCategory::Logical) {
+      context_.Say(property.source,
+          "%s trait requires a single LOGICAL expression"_err_en_US,
+          parser::ToUpperCaseLetters(traitName.ToString()));
+    }
+  }
+}
+
+void OmpStructureChecker::CheckTraitDeviceNum(
+    const parser::OmpTraitSetSelector &traitSet,
+    const parser::OmpTraitSelector &trait) {
+  auto &traitName{std::get<parser::OmpTraitSelectorName>(trait.t)};
+  auto &properties{GetTraitPropertyList(trait)};
+
+  if (properties.size() != 1) {
+    context_.Say(trait.source,
+        "%s trait requires a single expression property"_err_en_US,
+        parser::ToUpperCaseLetters(traitName.ToString()));
+  }
+  // No other checks at the moment.
+}
+
+void OmpStructureChecker::CheckTraitRequires(
+    const parser::OmpTraitSetSelector &traitSet,
+    const parser::OmpTraitSelector &trait) {
+  unsigned version{context_.langOptions().OpenMPVersion};
+  auto &traitName{std::get<parser::OmpTraitSelectorName>(trait.t)};
+  auto &properties{GetTraitPropertyList(trait)};
+
+  for (const parser::OmpTraitProperty &property : properties) {
+    auto clauseId{*GetClauseFromProperty(property)};
+    if (!llvm::omp::isAllowedClauseForDirective(
+            llvm::omp::OMPD_requires, clauseId, version)) {
+      context_.Say(property.source,
+          "%s trait requires a clause from the requirement clause set"_err_en_US,
+          parser::ToUpperCaseLetters(traitName.ToString()));
+    }
+
+    using ClauseProperty = common::Indirection<parser::OmpClause>;
+    if (!std::holds_alternative<ClauseProperty>(property.u)) {
+      context_.Say(property.source,
+          "Invalid clause specification for %s"_err_en_US,
+          parser::ToUpperCaseLetters(getClauseName(clauseId)));
+    }
+  }
+}
+
+void OmpStructureChecker::CheckTraitSimd(
+    const parser::OmpTraitSetSelector &traitSet,
+    const parser::OmpTraitSelector &trait) {
+  unsigned version{context_.langOptions().OpenMPVersion};
+  auto &traitName{std::get<parser::OmpTraitSelectorName>(trait.t)};
+  auto &properties{GetTraitPropertyList(trait)};
+
+  for (const parser::OmpTraitProperty &property : properties) {
+    auto clauseId{*GetClauseFromProperty(property)};
+    if (!llvm::omp::isAllowedClauseForDirective(
+            llvm::omp::OMPD_declare_simd, clauseId, version)) {
+      context_.Say(property.source,
+          "%s trait requires a clause that is allowed on the %s directive"_err_en_US,
+          parser::ToUpperCaseLetters(traitName.ToString()),
+          parser::ToUpperCaseLetters(
+              getDirectiveName(llvm::omp::OMPD_declare_simd)));
+    }
+
+    using ClauseProperty = common::Indirection<parser::OmpClause>;
+    if (!std::holds_alternative<ClauseProperty>(property.u)) {
+      context_.Say(property.source,
+          "Invalid clause specification for %s"_err_en_US,
+          parser::ToUpperCaseLetters(getClauseName(clauseId)));
+    }
+  }
+}
+
 llvm::StringRef OmpStructureChecker::getClauseName(llvm::omp::Clause clause) {
   return llvm::omp::getOpenMPClauseName(clause);
 }
@@ -4590,6 +5169,26 @@ void OmpStructureChecker::CheckProcedurePointer(
   }
 }
 
+void OmpStructureChecker::CheckCrayPointee(
+    const parser::OmpObjectList &objectList, llvm::StringRef clause,
+    bool suggestToUseCrayPointer) {
+  SymbolSourceMap symbols;
+  GetSymbolsInObjectList(objectList, symbols);
+  for (auto it{symbols.begin()}; it != symbols.end(); ++it) {
+    const auto *symbol{it->first};
+    const auto source{it->second};
+    if (symbol->test(Symbol::Flag::CrayPointee)) {
+      std::string suggestionMsg = "";
+      if (suggestToUseCrayPointer)
+        suggestionMsg = ", use Cray Pointer '" +
+            semantics::GetCrayPointer(*symbol).name().ToString() + "' instead";
+      context_.Say(source,
+          "Cray Pointee '%s' may not appear in %s clause%s"_err_en_US,
+          symbol->name(), clause.str(), suggestionMsg);
+    }
+  }
+}
+
 void OmpStructureChecker::GetSymbolsInObjectList(
     const parser::OmpObjectList &objectList, SymbolSourceMap &symbols) {
   for (const auto &ompObject : objectList.v) {
diff --git a/flang/lib/Semantics/check-omp-structure.h b/flang/lib/Semantics/check-omp-structure.h
index dc360957c873b..a9ac93a9149d4 100644
--- a/flang/lib/Semantics/check-omp-structure.h
+++ b/flang/lib/Semantics/check-omp-structure.h
@@ -105,6 +105,8 @@ class OmpStructureChecker
   void Enter(const parser::OmpDeclareTargetWithList &);
   void Enter(const parser::OmpDeclareTargetWithClause &);
   void Leave(const parser::OmpDeclareTargetWithClause &);
+  void Enter(const parser::OpenMPDispatchConstruct &);
+  void Leave(const parser::OpenMPDispatchConstruct &);
   void Enter(const parser::OmpErrorDirective &);
   void Leave(const parser::OmpErrorDirective &);
   void Enter(const parser::OpenMPExecutableAllocate &);
@@ -144,6 +146,15 @@ class OmpStructureChecker
   void Enter(const parser::DoConstruct &);
   void Leave(const parser::DoConstruct &);
 
+  void Enter(const parser::OmpDirectiveSpecification &);
+  void Leave(const parser::OmpDirectiveSpecification &);
+
+  void Enter(const parser::OmpMetadirectiveDirective &);
+  void Leave(const parser::OmpMetadirectiveDirective &);
+
+  void Enter(const parser::OmpContextSelector &);
+  void Leave(const parser::OmpContextSelector &);
+
 #define GEN_FLANG_CLAUSE_CHECK_ENTER
 #include "llvm/Frontend/OpenMP/OMP.inc"
 
@@ -173,6 +184,32 @@ class OmpStructureChecker
   // specific clause related
   void CheckAllowedMapTypes(const parser::OmpMapType::Value &,
       const std::list<parser::OmpMapType::Value> &);
+
+  std::optional<evaluate::DynamicType> GetDynamicType(
+      const common::Indirection<parser::Expr> &);
+  const std::list<parser::OmpTraitProperty> &GetTraitPropertyList(
+      const parser::OmpTraitSelector &);
+  std::optional<llvm::omp::Clause> GetClauseFromProperty(
+      const parser::OmpTraitProperty &);
+
+  void CheckTraitSelectorList(const std::list<parser::OmpTraitSelector> &);
+  void CheckTraitSetSelector(const parser::OmpTraitSetSelector &);
+  void CheckTraitScore(const parser::OmpTraitScore &);
+  bool VerifyTraitPropertyLists(
+      const parser::OmpTraitSetSelector &, const parser::OmpTraitSelector &);
+  void CheckTraitSelector(
+      const parser::OmpTraitSetSelector &, const parser::OmpTraitSelector &);
+  void CheckTraitADMO(
+      const parser::OmpTraitSetSelector &, const parser::OmpTraitSelector &);
+  void CheckTraitCondition(
+      const parser::OmpTraitSetSelector &, const parser::OmpTraitSelector &);
+  void CheckTraitDeviceNum(
+      const parser::OmpTraitSetSelector &, const parser::OmpTraitSelector &);
+  void CheckTraitRequires(
+      const parser::OmpTraitSetSelector &, const parser::OmpTraitSelector &);
+  void CheckTraitSimd(
+      const parser::OmpTraitSetSelector &, const parser::OmpTraitSelector &);
+
   llvm::StringRef getClauseName(llvm::omp::Clause clause) override;
   llvm::StringRef getDirectiveName(llvm::omp::Directive directive) override;
 
@@ -197,6 +234,8 @@ class OmpStructureChecker
       const parser::CharBlock &source, const parser::OmpObjectList &objList);
   void CheckIntentInPointer(SymbolSourceMap &, const llvm::omp::Clause);
   void CheckProcedurePointer(SymbolSourceMap &, const llvm::omp::Clause);
+  void CheckCrayPointee(const parser::OmpObjectList &objectList,
+      llvm::StringRef clause, bool suggestToUseCrayPointer = true);
   void GetSymbolsInObjectList(const parser::OmpObjectList &, SymbolSourceMap &);
   void CheckDefinableObjects(SymbolSourceMap &, const llvm::omp::Clause);
   void CheckCopyingPolymorphicAllocatable(
@@ -280,7 +319,8 @@ class OmpStructureChecker
     TargetBlockOnlyTeams,
     TargetNest,
     DeclarativeNest,
-    LastType = DeclarativeNest,
+    ContextSelectorNest,
+    LastType = ContextSelectorNest,
   };
   int directiveNest_[LastType + 1] = {0};
 
diff --git a/flang/lib/Semantics/check-return.cpp b/flang/lib/Semantics/check-return.cpp
index 22729f659ed5b..52220a566bbdc 100644
--- a/flang/lib/Semantics/check-return.cpp
+++ b/flang/lib/Semantics/check-return.cpp
@@ -7,11 +7,11 @@
 //===----------------------------------------------------------------------===//
 
 #include "check-return.h"
-#include "flang/Common/Fortran-features.h"
 #include "flang/Parser/message.h"
 #include "flang/Parser/parse-tree.h"
 #include "flang/Semantics/semantics.h"
 #include "flang/Semantics/tools.h"
+#include "flang/Support/Fortran-features.h"
 
 namespace Fortran::semantics {
 
diff --git a/flang/lib/Semantics/check-select-rank.cpp b/flang/lib/Semantics/check-select-rank.cpp
index 2e602d307013c..b227bbaaef4ba 100644
--- a/flang/lib/Semantics/check-select-rank.cpp
+++ b/flang/lib/Semantics/check-select-rank.cpp
@@ -7,11 +7,11 @@
 //===----------------------------------------------------------------------===//
 
 #include "check-select-rank.h"
-#include "flang/Common/Fortran.h"
 #include "flang/Common/idioms.h"
 #include "flang/Parser/message.h"
 #include "flang/Parser/tools.h"
 #include "flang/Semantics/tools.h"
+#include "flang/Support/Fortran.h"
 #include <list>
 #include <optional>
 #include <set>
diff --git a/flang/lib/Semantics/check-stop.cpp b/flang/lib/Semantics/check-stop.cpp
index 43535b07f029e..aabd52098e63b 100644
--- a/flang/lib/Semantics/check-stop.cpp
+++ b/flang/lib/Semantics/check-stop.cpp
@@ -7,11 +7,11 @@
 //===----------------------------------------------------------------------===//
 
 #include "check-stop.h"
-#include "flang/Common/Fortran.h"
 #include "flang/Evaluate/expression.h"
 #include "flang/Parser/parse-tree.h"
 #include "flang/Semantics/semantics.h"
 #include "flang/Semantics/tools.h"
+#include "flang/Support/Fortran.h"
 #include <optional>
 
 namespace Fortran::semantics {
diff --git a/flang/lib/Semantics/data-to-inits.h b/flang/lib/Semantics/data-to-inits.h
index d8cc4601de26f..7486ac8113e90 100644
--- a/flang/lib/Semantics/data-to-inits.h
+++ b/flang/lib/Semantics/data-to-inits.h
@@ -9,10 +9,10 @@
 #ifndef FORTRAN_SEMANTICS_DATA_TO_INITS_H_
 #define FORTRAN_SEMANTICS_DATA_TO_INITS_H_
 
-#include "flang/Common/default-kinds.h"
 #include "flang/Common/interval.h"
 #include "flang/Evaluate/fold-designator.h"
 #include "flang/Evaluate/initial-image.h"
+#include "flang/Support/default-kinds.h"
 #include <list>
 #include <map>
 
diff --git a/flang/lib/Semantics/expression.cpp b/flang/lib/Semantics/expression.cpp
index 3ec6f385ceb86..6949e5693d08f 100644
--- a/flang/lib/Semantics/expression.cpp
+++ b/flang/lib/Semantics/expression.cpp
@@ -11,7 +11,6 @@
 #include "pointer-assignment.h"
 #include "resolve-names-utils.h"
 #include "resolve-names.h"
-#include "flang/Common/Fortran.h"
 #include "flang/Common/idioms.h"
 #include "flang/Evaluate/common.h"
 #include "flang/Evaluate/fold.h"
@@ -24,6 +23,7 @@
 #include "flang/Semantics/semantics.h"
 #include "flang/Semantics/symbol.h"
 #include "flang/Semantics/tools.h"
+#include "flang/Support/Fortran.h"
 #include "llvm/Support/raw_ostream.h"
 #include <algorithm>
 #include <functional>
@@ -2834,13 +2834,12 @@ std::pair<const Symbol *, bool> ExpressionAnalyzer::ResolveGeneric(
   // Check for generic or explicit INTRINSIC of the same name in outer scopes.
   // See 15.5.5.2 for details.
   if (!symbol.owner().IsGlobal() && !symbol.owner().IsDerivedType()) {
-    for (const std::string &n : GetAllNames(context_, symbol.name())) {
-      if (const Symbol *outer{symbol.owner().parent().FindSymbol(n)}) {
-        auto pair{ResolveGeneric(*outer, actuals, adjustActuals, isSubroutine,
-            mightBeStructureConstructor)};
-        if (pair.first) {
-          return pair;
-        }
+    if (const Symbol *
+        outer{symbol.owner().parent().FindSymbol(symbol.name())}) {
+      auto pair{ResolveGeneric(*outer, actuals, adjustActuals, isSubroutine,
+          mightBeStructureConstructor)};
+      if (pair.first) {
+        return pair;
       }
     }
   }
@@ -3635,13 +3634,13 @@ MaybeExpr ExpressionAnalyzer::Analyze(const parser::Expr::Concat &x) {
 // The Name represents a user-defined intrinsic operator.
 // If the actuals match one of the specific procedures, return a function ref.
 // Otherwise report the error in messages.
-MaybeExpr ExpressionAnalyzer::AnalyzeDefinedOp(
-    const parser::Name &name, ActualArguments &&actuals) {
+MaybeExpr ExpressionAnalyzer::AnalyzeDefinedOp(const parser::Name &name,
+    ActualArguments &&actuals, const Symbol *&symbol) {
   if (auto callee{GetCalleeAndArguments(name, std::move(actuals))}) {
-    CHECK(std::holds_alternative<ProcedureDesignator>(callee->u));
-    return MakeFunctionRef(name.source,
-        std::move(std::get<ProcedureDesignator>(callee->u)),
-        std::move(callee->arguments));
+    auto &proc{std::get<evaluate::ProcedureDesignator>(callee->u)};
+    symbol = proc.GetSymbol();
+    return MakeFunctionRef(
+        name.source, std::move(proc), std::move(callee->arguments));
   } else {
     return std::nullopt;
   }
@@ -4079,8 +4078,7 @@ bool ExpressionAnalyzer::CheckIntrinsicKind(
     return true;
   } else if (foldingContext_.targetCharacteristics().CanSupportType(
                  category, kind)) {
-    Warn(common::UsageWarning::BadTypeForTarget,
-        "%s(KIND=%jd) is not an enabled type for this target"_warn_en_US,
+    Say("%s(KIND=%jd) is not an enabled type for this target"_err_en_US,
         ToUpperCase(EnumToString(category)), kind);
     return true;
   } else {
@@ -4102,20 +4100,7 @@ bool ExpressionAnalyzer::CheckIntrinsicSize(
       return false;
     }
   }
-  if (foldingContext_.targetCharacteristics().IsTypeEnabled(
-          category, kind)) { // C712, C714, C715, C727
-    return true;
-  } else if (foldingContext_.targetCharacteristics().CanSupportType(
-                 category, kind)) {
-    Warn(common::UsageWarning::BadTypeForTarget,
-        "%s*%jd is not an enabled type for this target"_warn_en_US,
-        ToUpperCase(EnumToString(category)), size);
-    return true;
-  } else {
-    Say("%s*%jd is not a supported type"_err_en_US,
-        ToUpperCase(EnumToString(category)), size);
-    return false;
-  }
+  return CheckIntrinsicKind(category, kind);
 }
 
 bool ExpressionAnalyzer::AddImpliedDo(parser::CharBlock name, int kind) {
@@ -4453,38 +4438,45 @@ MaybeExpr ArgumentAnalyzer::TryDefinedOp(
     parser::Messages buffer;
     auto restorer{context_.GetContextualMessages().SetMessages(buffer)};
     const auto &scope{context_.context().FindScope(source_)};
-    if (Symbol *symbol{scope.FindSymbol(oprName)}) {
+
+    auto FoundOne{[&](MaybeExpr &&thisResult, const Symbol &generic,
+                      const Symbol *resolution) {
       anyPossibilities = true;
-      parser::Name name{symbol->name(), symbol};
-      if (!fatalErrors_) {
-        result = context_.AnalyzeDefinedOp(name, GetActuals());
-      }
-      if (result) {
-        inaccessible = CheckAccessibleSymbol(scope, *symbol);
-        if (inaccessible) {
-          result.reset();
+      if (thisResult) {
+        if (auto thisInaccessible{CheckAccessibleSymbol(scope, generic)}) {
+          inaccessible = thisInaccessible;
         } else {
-          hit.push_back(symbol);
-          hitBuffer = std::move(buffer);
+          bool isElemental{IsElementalProcedure(DEREF(resolution))};
+          bool hitsAreNonElemental{
+              !hit.empty() && !IsElementalProcedure(DEREF(hit[0]))};
+          if (isElemental && hitsAreNonElemental) {
+            // ignore elemental resolutions in favor of a non-elemental one
+          } else {
+            if (!isElemental && !hitsAreNonElemental) {
+              hit.clear();
+            }
+            result = std::move(thisResult);
+            hit.push_back(resolution);
+            hitBuffer = std::move(buffer);
+          }
         }
       }
+    }};
+
+    if (Symbol * generic{scope.FindSymbol(oprName)}; generic && !fatalErrors_) {
+      parser::Name name{generic->name(), generic};
+      const Symbol *resultSymbol{nullptr};
+      MaybeExpr possibleResult{context_.AnalyzeDefinedOp(
+          name, ActualArguments{actuals_}, resultSymbol)};
+      FoundOne(std::move(possibleResult), *generic, resultSymbol);
     }
     for (std::size_t passIndex{0}; passIndex < actuals_.size(); ++passIndex) {
       buffer.clear();
       const Symbol *generic{nullptr};
-      if (const Symbol *binding{
-              FindBoundOp(oprName, passIndex, generic, false)}) {
-        anyPossibilities = true;
-        if (MaybeExpr thisResult{TryBoundOp(*binding, passIndex)}) {
-          if (auto thisInaccessible{
-                  CheckAccessibleSymbol(scope, DEREF(generic))}) {
-            inaccessible = thisInaccessible;
-          } else {
-            result = std::move(thisResult);
-            hit.push_back(binding);
-            hitBuffer = std::move(buffer);
-          }
-        }
+      if (const Symbol *
+          binding{FindBoundOp(
+              oprName, passIndex, generic, /*isSubroutine=*/false)}) {
+        FoundOne(TryBoundOp(*binding, passIndex), DEREF(generic), binding);
       }
     }
   }
@@ -4655,7 +4647,8 @@ std::optional<ProcedureRef> ArgumentAnalyzer::GetDefinedAssignmentProc() {
     }
     for (std::size_t i{0}; !proc && i < actuals_.size(); ++i) {
       const Symbol *generic{nullptr};
-      if (const Symbol *binding{FindBoundOp(oprName, i, generic, true)}) {
+      if (const Symbol *
+          binding{FindBoundOp(oprName, i, generic, /*isSubroutine=*/true)}) {
         if (CheckAccessibleSymbol(scope, DEREF(generic))) {
           // ignore inaccessible type-bound ASSIGNMENT(=) generic
         } else if (const Symbol *
diff --git a/flang/lib/Semantics/mod-file.cpp b/flang/lib/Semantics/mod-file.cpp
index 51ff70c3ed834..bef934beaacfa 100644
--- a/flang/lib/Semantics/mod-file.cpp
+++ b/flang/lib/Semantics/mod-file.cpp
@@ -306,8 +306,10 @@ void ModFileWriter::PrepareRenamings(const Scope &scope) {
   // to their names in this scope, creating those new names when needed.
   auto &renamings{context_.moduleFileOutputRenamings()};
   for (SymbolRef s : symbolsNeeded) {
-    if (s->owner().kind() == Scope::Kind::DerivedType) {
-      continue; // component or binding: ok
+    if (s->owner().kind() != Scope::Kind::Module) {
+      // Not a USE'able name from a module's top scope;
+      // component, binding, dummy argument, &c.
+      continue;
     }
     const Scope *sMod{FindModuleContaining(s->owner())};
     if (!sMod || sMod == &scope) {
@@ -1366,6 +1368,12 @@ Scope *ModFileReader::Read(SourceName name, std::optional<bool> isIntrinsic,
       name.ToString(), isIntrinsic.value_or(false))};
   if (!isIntrinsic.value_or(false) && !ancestor) {
     // Already present in the symbol table as a usable non-intrinsic module?
+    if (Scope * hermeticScope{context_.currentHermeticModuleFileScope()}) {
+      auto it{hermeticScope->find(name)};
+      if (it != hermeticScope->end()) {
+        return it->second->scope();
+      }
+    }
     auto it{context_.globalScope().find(name)};
     if (it != context_.globalScope().end()) {
       Scope *scope{it->second->scope()};
@@ -1544,9 +1552,22 @@ Scope *ModFileReader::Read(SourceName name, std::optional<bool> isIntrinsic,
   // Process declarations from the module file
   auto wasModuleFileName{context_.foldingContext().moduleFileName()};
   context_.foldingContext().set_moduleFileName(name);
+  // Are there multiple modules in the module file due to it having been
+  // created under -fhermetic-module-files?  If so, process them first in
+  // their own nested scope that will be visible only to USE statements
+  // within the module file.
+  if (parseTree.v.size() > 1) {
+    parser::Program hermeticModules{std::move(parseTree.v)};
+    parseTree.v.emplace_back(std::move(hermeticModules.v.front()));
+    hermeticModules.v.pop_front();
+    Scope &hermeticScope{topScope.MakeScope(Scope::Kind::Global)};
+    context_.set_currentHermeticModuleFileScope(&hermeticScope);
+    ResolveNames(context_, hermeticModules, hermeticScope);
+  }
   GetModuleDependences(context_.moduleDependences(), sourceFile->content());
   ResolveNames(context_, parseTree, topScope);
   context_.foldingContext().set_moduleFileName(wasModuleFileName);
+  context_.set_currentHermeticModuleFileScope(nullptr);
   if (!moduleSymbol) {
     // Submodule symbols' storage are owned by their parents' scopes,
     // but their names are not in their parents' dictionaries -- we
@@ -1780,8 +1801,8 @@ bool SubprogramSymbolCollector::NeedImport(
     return found->has<UseDetails>() && found->owner() != scope_;
   } else {
     // "found" can be null in the case of a use-associated derived type's
-    // parent type
-    CHECK(symbol.has<DerivedTypeDetails>());
+    // parent type, and also in the case of an object (like a dummy argument)
+    // used to define a length or bound of a nested interface.
     return false;
   }
 }
diff --git a/flang/lib/Semantics/openmp-modifiers.cpp b/flang/lib/Semantics/openmp-modifiers.cpp
index 9f2896229bb7f..73ad7751ee517 100644
--- a/flang/lib/Semantics/openmp-modifiers.cpp
+++ b/flang/lib/Semantics/openmp-modifiers.cpp
@@ -156,6 +156,23 @@ const OmpModifierDescriptor &OmpGetDescriptor<parser::OmpChunkModifier>() {
   return desc;
 }
 
+template <>
+const OmpModifierDescriptor &OmpGetDescriptor<parser::OmpContextSelector>() {
+  static const OmpModifierDescriptor desc{
+      /*name=*/"context-selector",
+      /*props=*/
+      {
+          {50, {OmpProperty::Required, OmpProperty::Unique}},
+      },
+      /*clauses=*/
+      {
+          // The MATCH clause takes a selector as an argument, not modifier.
+          {50, {Clause::OMPC_when}},
+      },
+  };
+  return desc;
+}
+
 template <>
 const OmpModifierDescriptor &OmpGetDescriptor<parser::OmpDependenceType>() {
   static const OmpModifierDescriptor desc{
diff --git a/flang/lib/Semantics/resolve-directives.cpp b/flang/lib/Semantics/resolve-directives.cpp
index ea102371334a6..91a1b3061e1f9 100644
--- a/flang/lib/Semantics/resolve-directives.cpp
+++ b/flang/lib/Semantics/resolve-directives.cpp
@@ -351,6 +351,17 @@ class OmpAttributeVisitor : DirectiveAttributeVisitor<llvm::omp::Directive> {
     return true;
   }
 
+  bool Pre(const parser::OmpDirectiveSpecification &x) {
+    PushContext(x.source, std::get<llvm::omp::Directive>(x.t));
+    return true;
+  }
+  void Post(const parser::OmpDirectiveSpecification &) { PopContext(); }
+  bool Pre(const parser::OmpMetadirectiveDirective &x) {
+    PushContext(x.source, llvm::omp::Directive::OMPD_metadirective);
+    return true;
+  }
+  void Post(const parser::OmpMetadirectiveDirective &) { PopContext(); }
+
   bool Pre(const parser::OpenMPBlockConstruct &);
   void Post(const parser::OpenMPBlockConstruct &);
 
@@ -441,6 +452,9 @@ class OmpAttributeVisitor : DirectiveAttributeVisitor<llvm::omp::Directive> {
   bool Pre(const parser::OpenMPDeclarativeAllocate &);
   void Post(const parser::OpenMPDeclarativeAllocate &) { PopContext(); }
 
+  bool Pre(const parser::OpenMPDispatchConstruct &);
+  void Post(const parser::OpenMPDispatchConstruct &) { PopContext(); }
+
   bool Pre(const parser::OpenMPExecutableAllocate &);
   void Post(const parser::OpenMPExecutableAllocate &);
 
@@ -1976,6 +1990,11 @@ bool OmpAttributeVisitor::Pre(const parser::OpenMPDeclarativeAllocate &x) {
   return false;
 }
 
+bool OmpAttributeVisitor::Pre(const parser::OpenMPDispatchConstruct &x) {
+  PushContext(x.source, llvm::omp::Directive::OMPD_dispatch);
+  return true;
+}
+
 bool OmpAttributeVisitor::Pre(const parser::OpenMPExecutableAllocate &x) {
   PushContext(x.source, llvm::omp::Directive::OMPD_allocate);
   const auto &list{std::get<std::optional<parser::OmpObjectList>>(x.t)};
@@ -1999,20 +2018,25 @@ bool OmpAttributeVisitor::Pre(const parser::OpenMPAllocatorsConstruct &x) {
 }
 
 void OmpAttributeVisitor::Post(const parser::OmpDefaultClause &x) {
-  if (!dirContext_.empty()) {
-    switch (x.v) {
-    case parser::OmpDefaultClause::DataSharingAttribute::Private:
-      SetContextDefaultDSA(Symbol::Flag::OmpPrivate);
-      break;
-    case parser::OmpDefaultClause::DataSharingAttribute::Firstprivate:
-      SetContextDefaultDSA(Symbol::Flag::OmpFirstPrivate);
-      break;
-    case parser::OmpDefaultClause::DataSharingAttribute::Shared:
-      SetContextDefaultDSA(Symbol::Flag::OmpShared);
-      break;
-    case parser::OmpDefaultClause::DataSharingAttribute::None:
-      SetContextDefaultDSA(Symbol::Flag::OmpNone);
-      break;
+  // The DEFAULT clause may also be used on METADIRECTIVE. In that case
+  // there is nothing to do.
+  using DataSharingAttribute = parser::OmpDefaultClause::DataSharingAttribute;
+  if (auto *dsa{std::get_if<DataSharingAttribute>(&x.u)}) {
+    if (!dirContext_.empty()) {
+      switch (*dsa) {
+      case DataSharingAttribute::Private:
+        SetContextDefaultDSA(Symbol::Flag::OmpPrivate);
+        break;
+      case DataSharingAttribute::Firstprivate:
+        SetContextDefaultDSA(Symbol::Flag::OmpFirstPrivate);
+        break;
+      case DataSharingAttribute::Shared:
+        SetContextDefaultDSA(Symbol::Flag::OmpShared);
+        break;
+      case DataSharingAttribute::None:
+        SetContextDefaultDSA(Symbol::Flag::OmpNone);
+        break;
+      }
     }
   }
 }
@@ -2097,8 +2121,11 @@ void OmpAttributeVisitor::Post(const parser::OpenMPAllocatorsConstruct &x) {
 static bool IsPrivatizable(const Symbol *sym) {
   auto *misc{sym->detailsIf<MiscDetails>()};
   return IsVariableName(*sym) && !IsProcedure(*sym) && !IsNamedConstant(*sym) &&
-      !semantics::IsAssumedSizeArray(
-          *sym) && /* OpenMP 5.2, 5.1.1: Assumed-size arrays are shared*/
+      ( // OpenMP 5.2, 5.1.1: Assumed-size arrays are shared
+          !semantics::IsAssumedSizeArray(*sym) ||
+          // If CrayPointer is among the DSA list then the
+          // CrayPointee is Privatizable
+          sym->test(Symbol::Flag::CrayPointee)) &&
       !sym->owner().IsDerivedType() &&
       sym->owner().kind() != Scope::Kind::ImpliedDos &&
       sym->owner().kind() != Scope::Kind::Forall &&
@@ -2265,10 +2292,18 @@ void OmpAttributeVisitor::Post(const parser::Name &name) {
             // the scope of the parallel region, and not in this scope.
             // TODO: check whether this should be caught in IsObjectWithDSA
             !symbol->test(Symbol::Flag::OmpPrivate)) {
-          context_.Say(name.source,
-              "The DEFAULT(NONE) clause requires that '%s' must be listed in "
-              "a data-sharing attribute clause"_err_en_US,
-              symbol->name());
+          if (symbol->test(Symbol::Flag::CrayPointee)) {
+            std::string crayPtrName{
+                semantics::GetCrayPointer(*symbol).name().ToString()};
+            if (!IsObjectWithDSA(*currScope().FindSymbol(crayPtrName)))
+              context_.Say(name.source,
+                  "The DEFAULT(NONE) clause requires that the Cray Pointer '%s' must be listed in a data-sharing attribute clause"_err_en_US,
+                  crayPtrName);
+          } else {
+            context_.Say(name.source,
+                "The DEFAULT(NONE) clause requires that '%s' must be listed in a data-sharing attribute clause"_err_en_US,
+                symbol->name());
+          }
         }
       }
     }
diff --git a/flang/lib/Semantics/resolve-labels.cpp b/flang/lib/Semantics/resolve-labels.cpp
index 04e4b142efedf..b0cbc4b56e889 100644
--- a/flang/lib/Semantics/resolve-labels.cpp
+++ b/flang/lib/Semantics/resolve-labels.cpp
@@ -122,6 +122,8 @@ constexpr Legality IsLegalBranchTarget(const parser::Statement<A> &) {
       std::is_same_v<A, parser::EndCriticalStmt> ||
       std::is_same_v<A, parser::ForallConstructStmt> ||
       std::is_same_v<A, parser::WhereConstructStmt> ||
+      std::is_same_v<A, parser::ChangeTeamStmt> ||
+      std::is_same_v<A, parser::EndChangeTeamStmt> ||
       std::is_same_v<A, parser::EndFunctionStmt> ||
       std::is_same_v<A, parser::EndMpSubprogramStmt> ||
       std::is_same_v<A, parser::EndProgramStmt> ||
@@ -210,8 +212,9 @@ class ParseTreeAnalyzer {
         // subprograms.  Visit that statement in advance so that results
         // are placed in the correct programUnits_ slot.
         auto targetFlags{ConstructBranchTargetFlags(endStmt)};
-        AddTargetLabelDefinition(
-            endStmt.label.value(), targetFlags, currentScope_);
+        AddTargetLabelDefinition(endStmt.label.value(), targetFlags,
+            currentScope_,
+            /*isExecutableConstructEndStmt=*/false);
       }
     }
     return true;
@@ -238,18 +241,20 @@ class ParseTreeAnalyzer {
             parser::EndProgramStmt, parser::EndSubroutineStmt>;
     auto targetFlags{ConstructBranchTargetFlags(statement)};
     if constexpr (common::HasMember<A, LabeledConstructStmts>) {
-      AddTargetLabelDefinition(label.value(), targetFlags, ParentScope());
+      AddTargetLabelDefinition(label.value(), targetFlags, ParentScope(),
+          /*isExecutableConstructEndStmt=*/false);
     } else if constexpr (std::is_same_v<A, parser::EndIfStmt> ||
         std::is_same_v<A, parser::EndSelectStmt>) {
       // the label on an END IF/SELECT is not in the last part/case
-      AddTargetLabelDefinition(label.value(), targetFlags, ParentScope(), true);
+      AddTargetLabelDefinition(label.value(), targetFlags, ParentScope(),
+          /*isExecutableConstructEndStmt=*/true);
     } else if constexpr (common::HasMember<A, LabeledConstructEndStmts>) {
-      constexpr bool isExecutableConstructEndStmt{true};
       AddTargetLabelDefinition(label.value(), targetFlags, currentScope_,
-          isExecutableConstructEndStmt);
+          /*isExecutableConstructEndStmt=*/true);
     } else if constexpr (!common::HasMember<A, LabeledProgramUnitEndStmts>) {
       // Program unit END statements have already been processed.
-      AddTargetLabelDefinition(label.value(), targetFlags, currentScope_);
+      AddTargetLabelDefinition(label.value(), targetFlags, currentScope_,
+          /*isExecutableConstructEndStmt=*/false);
     }
     return true;
   }
@@ -826,7 +831,7 @@ class ParseTreeAnalyzer {
   // 6.2.5., paragraph 2
   void AddTargetLabelDefinition(parser::Label label,
       LabeledStmtClassificationSet labeledStmtClassificationSet,
-      ProxyForScope scope, bool isExecutableConstructEndStmt = false) {
+      ProxyForScope scope, bool isExecutableConstructEndStmt) {
     CheckLabelInRange(label);
     TargetStmtMap &targetStmtMap{disposableMaps_.empty()
             ? programUnits_.back().targetStmts
@@ -912,7 +917,7 @@ bool InBody(const parser::CharBlock &position,
   return false;
 }
 
-LabeledStatementInfoTuplePOD GetLabel(
+static LabeledStatementInfoTuplePOD GetLabel(
     const TargetStmtMap &labels, const parser::Label &label) {
   const auto iter{labels.find(label)};
   if (iter == labels.cend()) {
diff --git a/flang/lib/Semantics/resolve-names-utils.cpp b/flang/lib/Semantics/resolve-names-utils.cpp
index a838d49c06104..742bb748b7ff3 100644
--- a/flang/lib/Semantics/resolve-names-utils.cpp
+++ b/flang/lib/Semantics/resolve-names-utils.cpp
@@ -7,8 +7,6 @@
 //===----------------------------------------------------------------------===//
 
 #include "resolve-names-utils.h"
-#include "flang/Common/Fortran-features.h"
-#include "flang/Common/Fortran.h"
 #include "flang/Common/idioms.h"
 #include "flang/Common/indirection.h"
 #include "flang/Evaluate/fold.h"
@@ -20,6 +18,8 @@
 #include "flang/Semantics/expression.h"
 #include "flang/Semantics/semantics.h"
 #include "flang/Semantics/tools.h"
+#include "flang/Support/Fortran-features.h"
+#include "flang/Support/Fortran.h"
 #include <initializer_list>
 #include <variant>
 
@@ -762,7 +762,11 @@ void SymbolMapper::MapSymbolExprs(Symbol &symbol) {
               proc.set_procInterfaces(
                   *mappedSymbol, BypassGeneric(mappedSymbol->GetUltimate()));
             } else if (const DeclTypeSpec * mappedType{MapType(proc.type())}) {
-              proc.set_type(*mappedType);
+              if (proc.type()) {
+                CHECK(*proc.type() == *mappedType);
+              } else {
+                proc.set_type(*mappedType);
+              }
             }
             if (proc.init()) {
               if (const Symbol * mapped{MapSymbol(*proc.init())}) {
diff --git a/flang/lib/Semantics/resolve-names.cpp b/flang/lib/Semantics/resolve-names.cpp
index aee0357333159..e64abe6b50e78 100644
--- a/flang/lib/Semantics/resolve-names.cpp
+++ b/flang/lib/Semantics/resolve-names.cpp
@@ -13,8 +13,6 @@
 #include "resolve-directives.h"
 #include "resolve-names-utils.h"
 #include "rewrite-parse-tree.h"
-#include "flang/Common/Fortran.h"
-#include "flang/Common/default-kinds.h"
 #include "flang/Common/indirection.h"
 #include "flang/Common/restorer.h"
 #include "flang/Common/visit.h"
@@ -38,6 +36,8 @@
 #include "flang/Semantics/symbol.h"
 #include "flang/Semantics/tools.h"
 #include "flang/Semantics/type.h"
+#include "flang/Support/Fortran.h"
+#include "flang/Support/default-kinds.h"
 #include "llvm/Support/raw_ostream.h"
 #include <list>
 #include <map>
@@ -87,7 +87,8 @@ class ImplicitRules {
   bool inheritFromParent_{false}; // look in parent if not specified here
   bool isImplicitNoneType_{
       context_.IsEnabled(common::LanguageFeature::ImplicitNoneTypeAlways)};
-  bool isImplicitNoneExternal_{false};
+  bool isImplicitNoneExternal_{
+      context_.IsEnabled(common::LanguageFeature::ImplicitNoneExternal)};
   // map_ contains the mapping between letters and types that were defined
   // by the IMPLICIT statements of the related scope. It does not contain
   // the default Fortran mappings nor the mapping defined in parents.
@@ -1474,7 +1475,12 @@ class OmpVisitor : public virtual DeclarationVisitor {
     return true;
   }
 
-  bool Pre(const parser::OpenMPDeclareMapperConstruct &);
+  bool Pre(const parser::OpenMPDeclareMapperConstruct &x) {
+    AddOmpSourceRange(x.source);
+    ProcessMapperSpecifier(std::get<parser::OmpMapperSpecifier>(x.t),
+        std::get<parser::OmpClauseList>(x.t));
+    return false;
+  }
 
   bool Pre(const parser::OmpMapClause &);
 
@@ -1616,6 +1622,21 @@ class OmpVisitor : public virtual DeclarationVisitor {
       PopScope();
     }
   }
+  bool Pre(const parser::OmpDirectiveSpecification &x);
+
+  bool Pre(const parser::OmpTypeSpecifier &x) {
+    BeginDeclTypeSpec();
+    return true;
+  }
+  void Post(const parser::OmpTypeSpecifier &x) { //
+    EndDeclTypeSpec();
+  }
+
+private:
+  void ProcessMapperSpecifier(const parser::OmpMapperSpecifier &spec,
+      const parser::OmpClauseList &clauses);
+  void ProcessReductionSpecifier(const parser::OmpReductionSpecifier &spec,
+      const parser::OmpClauseList &clauses);
 };
 
 bool OmpVisitor::NeedsScope(const parser::OpenMPBlockConstruct &x) {
@@ -1655,37 +1676,6 @@ void OmpVisitor::Post(const parser::OpenMPBlockConstruct &x) {
   }
 }
 
-// This "manually" walks the tree of the construct, because we need
-// to resolve the type before the map clauses are processed - when
-// just following the natural flow, the map clauses gets processed before
-// the type has been fully processed.
-bool OmpVisitor::Pre(const parser::OpenMPDeclareMapperConstruct &x) {
-  AddOmpSourceRange(x.source);
-  BeginDeclTypeSpec();
-  const auto &spec{std::get<parser::OmpDeclareMapperSpecifier>(x.t)};
-  Symbol *mapperSym{nullptr};
-  if (const auto &mapperName{std::get<std::optional<parser::Name>>(spec.t)}) {
-    mapperSym =
-        &MakeSymbol(*mapperName, MiscDetails{MiscDetails::Kind::ConstructName});
-    mapperName->symbol = mapperSym;
-  } else {
-    const parser::CharBlock defaultName{"default", 7};
-    mapperSym = &MakeSymbol(
-        defaultName, Attrs{}, MiscDetails{MiscDetails::Kind::ConstructName});
-  }
-
-  PushScope(Scope::Kind::OtherConstruct, nullptr);
-  Walk(std::get<parser::TypeSpec>(spec.t));
-  const auto &varName{std::get<parser::ObjectName>(spec.t)};
-  DeclareObjectEntity(varName);
-
-  Walk(std::get<parser::OmpClauseList>(x.t));
-
-  EndDeclTypeSpec();
-  PopScope();
-  return false;
-}
-
 bool OmpVisitor::Pre(const parser::OmpMapClause &x) {
   auto &mods{OmpGetModifiers(x)};
   if (auto *mapper{OmpGetUniqueModifier<parser::OmpMapper>(mods)}) {
@@ -1713,6 +1703,83 @@ bool OmpVisitor::Pre(const parser::OmpMapClause &x) {
   return true;
 }
 
+void OmpVisitor::ProcessMapperSpecifier(const parser::OmpMapperSpecifier &spec,
+    const parser::OmpClauseList &clauses) {
+  // This "manually" walks the tree of the construct, because we need
+  // to resolve the type before the map clauses are processed - when
+  // just following the natural flow, the map clauses gets processed before
+  // the type has been fully processed.
+  BeginDeclTypeSpec();
+  if (auto &mapperName{std::get<std::optional<parser::Name>>(spec.t)}) {
+    mapperName->symbol =
+        &MakeSymbol(*mapperName, MiscDetails{MiscDetails::Kind::ConstructName});
+  } else {
+    const parser::CharBlock defaultName{"default", 7};
+    MakeSymbol(
+        defaultName, Attrs{}, MiscDetails{MiscDetails::Kind::ConstructName});
+  }
+
+  PushScope(Scope::Kind::OtherConstruct, nullptr);
+  Walk(std::get<parser::TypeSpec>(spec.t));
+  auto &varName{std::get<parser::Name>(spec.t)};
+  DeclareObjectEntity(varName);
+
+  Walk(clauses);
+  EndDeclTypeSpec();
+  PopScope();
+}
+
+void OmpVisitor::ProcessReductionSpecifier(
+    const parser::OmpReductionSpecifier &spec,
+    const parser::OmpClauseList &clauses) {
+  // Creating a new scope in case the combiner expression (or clauses) use
+  // reerved identifiers, like "omp_in". This is a temporary solution until
+  // we deal with these in a more thorough way.
+  PushScope(Scope::Kind::OtherConstruct, nullptr);
+  Walk(std::get<parser::OmpReductionIdentifier>(spec.t));
+  Walk(std::get<parser::OmpTypeNameList>(spec.t));
+  Walk(std::get<std::optional<parser::OmpReductionCombiner>>(spec.t));
+  Walk(clauses);
+  PopScope();
+}
+
+bool OmpVisitor::Pre(const parser::OmpDirectiveSpecification &x) {
+  // OmpDirectiveSpecification is only used in METADIRECTIVE at the moment.
+  // Since it contains directives and clauses, some semantic checks may
+  // not be applicable.
+  // Disable the semantic analysis for it for now to allow the compiler to
+  // parse METADIRECTIVE without flagging errors.
+  AddOmpSourceRange(x.source);
+  auto dirId{std::get<llvm::omp::Directive>(x.t)};
+  auto &maybeArgs{std::get<std::optional<std::list<parser::OmpArgument>>>(x.t)};
+  auto &maybeClauses{std::get<std::optional<parser::OmpClauseList>>(x.t)};
+
+  switch (dirId) {
+  case llvm::omp::Directive::OMPD_declare_mapper:
+    if (maybeArgs && maybeClauses) {
+      const parser::OmpArgument &first{maybeArgs->front()};
+      if (auto *spec{std::get_if<parser::OmpMapperSpecifier>(&first.u)}) {
+        ProcessMapperSpecifier(*spec, *maybeClauses);
+      }
+    }
+    break;
+  case llvm::omp::Directive::OMPD_declare_reduction:
+    if (maybeArgs && maybeClauses) {
+      const parser::OmpArgument &first{maybeArgs->front()};
+      if (auto *spec{std::get_if<parser::OmpReductionSpecifier>(&first.u)}) {
+        ProcessReductionSpecifier(*spec, *maybeClauses);
+      }
+    }
+    break;
+  default:
+    // Default processing.
+    Walk(maybeArgs);
+    Walk(maybeClauses);
+    break;
+  }
+  return false;
+}
+
 // Walk the parse tree and resolve names to symbols.
 class ResolveNamesVisitor : public virtual ScopeHandler,
                             public ModuleVisitor,
@@ -2591,9 +2658,11 @@ void ScopeHandler::PopScope() {
     ConvertToObjectEntity(*pair.second);
   }
   funcResultStack_.Pop();
-  // If popping back into a global scope, pop back to the main global scope.
-  SetScope(currScope_->parent().IsGlobal() ? context().globalScope()
-                                           : currScope_->parent());
+  // If popping back into a global scope, pop back to the top scope.
+  Scope *hermetic{context().currentHermeticModuleFileScope()};
+  SetScope(currScope_->parent().IsGlobal()
+          ? (hermetic ? *hermetic : context().globalScope())
+          : currScope_->parent());
 }
 void ScopeHandler::SetScope(Scope &scope) {
   currScope_ = &scope;
@@ -3204,9 +3273,9 @@ ModuleVisitor::SymbolRename ModuleVisitor::AddUse(
 // symbol must be either a Use or a Generic formed by merging two uses.
 // Convert it to a UseError with this additional location.
 static bool ConvertToUseError(
-    Symbol &symbol, const SourceName &location, const Scope &module) {
+    Symbol &symbol, const SourceName &location, const Symbol &used) {
   if (auto *ued{symbol.detailsIf<UseErrorDetails>()}) {
-    ued->add_occurrence(location, module);
+    ued->add_occurrence(location, used);
     return true;
   }
   const auto *useDetails{symbol.detailsIf<UseDetails>()};
@@ -3219,18 +3288,104 @@ static bool ConvertToUseError(
   }
   if (useDetails) {
     symbol.set_details(
-        UseErrorDetails{*useDetails}.add_occurrence(location, module));
+        UseErrorDetails{*useDetails}.add_occurrence(location, used));
     return true;
   } else {
     return false;
   }
 }
 
+// Two ultimate symbols are distinct, but they have the same name and come
+// from modules with the same name.  At link time, their mangled names
+// would conflict, so they had better resolve to the same definition.
+// Check whether the two ultimate symbols have compatible definitions.
+// Returns true if no further processing is required in DoAddUse().
+static bool CheckCompatibleDistinctUltimates(SemanticsContext &context,
+    SourceName location, SourceName localName, const Symbol &localSymbol,
+    const Symbol &localUltimate, const Symbol &useUltimate, bool &isError) {
+  isError = false;
+  if (localUltimate.has<GenericDetails>()) {
+    if (useUltimate.has<GenericDetails>() ||
+        useUltimate.has<SubprogramDetails>() ||
+        useUltimate.has<DerivedTypeDetails>()) {
+      return false; // can try to merge them
+    } else {
+      isError = true;
+    }
+  } else if (useUltimate.has<GenericDetails>()) {
+    if (localUltimate.has<SubprogramDetails>() ||
+        localUltimate.has<DerivedTypeDetails>()) {
+      return false; // can try to merge them
+    } else {
+      isError = true;
+    }
+  } else if (localUltimate.has<SubprogramDetails>()) {
+    if (useUltimate.has<SubprogramDetails>()) {
+      auto localCharacteristics{
+          evaluate::characteristics::Procedure::Characterize(
+              localUltimate, context.foldingContext())};
+      auto useCharacteristics{
+          evaluate::characteristics::Procedure::Characterize(
+              useUltimate, context.foldingContext())};
+      if ((localCharacteristics &&
+              (!useCharacteristics ||
+                  *localCharacteristics != *useCharacteristics)) ||
+          (!localCharacteristics && useCharacteristics)) {
+        isError = true;
+      }
+    } else {
+      isError = true;
+    }
+  } else if (useUltimate.has<SubprogramDetails>()) {
+    isError = true;
+  } else if (const auto *localObject{
+                 localUltimate.detailsIf<ObjectEntityDetails>()}) {
+    if (const auto *useObject{useUltimate.detailsIf<ObjectEntityDetails>()}) {
+      auto localType{evaluate::DynamicType::From(localUltimate)};
+      auto useType{evaluate::DynamicType::From(useUltimate)};
+      if (localUltimate.size() != useUltimate.size() ||
+          (localType &&
+              (!useType || !localType->IsTkLenCompatibleWith(*useType) ||
+                  !useType->IsTkLenCompatibleWith(*localType))) ||
+          (!localType && useType)) {
+        isError = true;
+      } else if (IsNamedConstant(localUltimate)) {
+        isError = !IsNamedConstant(useUltimate) ||
+            !(*localObject->init() == *useObject->init());
+      } else {
+        isError = IsNamedConstant(useUltimate);
+      }
+    } else {
+      isError = true;
+    }
+  } else if (useUltimate.has<ObjectEntityDetails>()) {
+    isError = true;
+  } else if (IsProcedurePointer(localUltimate)) {
+    isError = !IsProcedurePointer(useUltimate);
+  } else if (IsProcedurePointer(useUltimate)) {
+    isError = true;
+  } else if (localUltimate.has<DerivedTypeDetails>()) {
+    isError = !(useUltimate.has<DerivedTypeDetails>() &&
+        evaluate::AreSameDerivedTypeIgnoringSequence(
+            DerivedTypeSpec{localUltimate.name(), localUltimate},
+            DerivedTypeSpec{useUltimate.name(), useUltimate}));
+  } else if (useUltimate.has<DerivedTypeDetails>()) {
+    isError = true;
+  } else if (localUltimate.has<NamelistDetails>() &&
+      useUltimate.has<NamelistDetails>()) {
+  } else if (localUltimate.has<CommonBlockDetails>() &&
+      useUltimate.has<CommonBlockDetails>()) {
+  } else {
+    isError = true;
+  }
+  return true; // don't try to merge generics (or whatever)
+}
+
 void ModuleVisitor::DoAddUse(SourceName location, SourceName localName,
     Symbol &originalLocal, const Symbol &useSymbol) {
   Symbol *localSymbol{&originalLocal};
   if (auto *details{localSymbol->detailsIf<UseErrorDetails>()}) {
-    details->add_occurrence(location, *useModuleScope_);
+    details->add_occurrence(location, useSymbol);
     return;
   }
   const Symbol &useUltimate{useSymbol.GetUltimate()};
@@ -3266,6 +3421,49 @@ void ModuleVisitor::DoAddUse(SourceName location, SourceName localName,
     // use-associating the same symbol again -- ok
     return;
   }
+  if (useUltimate.owner().IsModule() && localUltimate.owner().IsSubmodule() &&
+      DoesScopeContain(&useUltimate.owner(), localUltimate)) {
+    // Within a submodule, USE'ing a symbol that comes indirectly
+    // from the ancestor module, e.g. foo in:
+    //  MODULE m1; INTERFACE; MODULE SUBROUTINE foo; END INTERFACE; END
+    //  MODULE m2; USE m1; END
+    //  SUBMODULE m1(sm); USE m2; CONTAINS; MODULE PROCEDURE foo; END; END
+    return; // ok, ignore it
+  }
+
+  if (localUltimate.name() == useUltimate.name() &&
+      localUltimate.owner().IsModule() && useUltimate.owner().IsModule() &&
+      localUltimate.owner().GetName() &&
+      localUltimate.owner().GetName() == useUltimate.owner().GetName()) {
+    bool isError{false};
+    if (CheckCompatibleDistinctUltimates(context(), location, localName,
+            *localSymbol, localUltimate, useUltimate, isError)) {
+      if (isError) {
+        // Convert the local symbol to a UseErrorDetails, if possible;
+        // otherwise emit a fatal error.
+        if (!ConvertToUseError(*localSymbol, location, useSymbol)) {
+          context()
+              .Say(location,
+                  "'%s' use-associated from '%s' in module '%s' is incompatible with '%s' from another module"_err_en_US,
+                  localName, useUltimate.name(),
+                  useUltimate.owner().GetName().value(), localUltimate.name())
+              .Attach(useUltimate.name(), "First declaration"_en_US)
+              .Attach(localUltimate.name(), "Other declaration"_en_US);
+          return;
+        }
+      }
+      if (auto *msg{context().Warn(
+              common::UsageWarning::CompatibleDeclarationsFromDistinctModules,
+              location,
+              "'%s' is use-associated from '%s' in two distinct instances of module '%s'"_warn_en_US,
+              localName, localUltimate.name(),
+              localUltimate.owner().GetName().value())}) {
+        msg->Attach(localUltimate.name(), "Previous declaration"_en_US)
+            .Attach(useUltimate.name(), "Later declaration"_en_US);
+      }
+      return;
+    }
+  }
 
   // There are many possible combinations of symbol types that could arrive
   // with the same (local) name vie USE association from distinct modules.
@@ -3328,7 +3526,7 @@ void ModuleVisitor::DoAddUse(SourceName location, SourceName localName,
     EraseSymbol(*localSymbol);
     CHECK(localSymbol->has<UseDetails>());
     UseErrorDetails details{localSymbol->get<UseDetails>()};
-    details.add_occurrence(location, *useModuleScope_);
+    details.add_occurrence(location, useSymbol);
     Symbol *newSymbol{&MakeSymbol(localName, Attrs{}, std::move(details))};
     // Restore *localSymbol in currScope
     auto iter{currScope().find(localName)};
@@ -3365,7 +3563,7 @@ void ModuleVisitor::DoAddUse(SourceName location, SourceName localName,
       if (localGeneric) {
         combinedDerivedType = CreateLocalUseError();
       } else {
-        ConvertToUseError(*localSymbol, location, *useModuleScope_);
+        ConvertToUseError(*localSymbol, location, useSymbol);
         localDerivedType = nullptr;
         localGeneric = nullptr;
         combinedDerivedType = localSymbol;
@@ -3473,7 +3671,7 @@ void ModuleVisitor::DoAddUse(SourceName location, SourceName localName,
 
   // If symbols are not combinable, create a use error.
   if (cantCombine) {
-    if (!ConvertToUseError(*localSymbol, location, *useModuleScope_)) {
+    if (!ConvertToUseError(*localSymbol, location, useSymbol)) {
       Say(location,
           "Cannot use-associate '%s'; it is already declared in this scope"_err_en_US,
           localName)
@@ -8508,8 +8706,11 @@ void DeclarationVisitor::PointerInitialization(
     if (!context().HasError(ultimate)) {
       if (IsProcedurePointer(ultimate)) {
         auto &details{ultimate.get<ProcEntityDetails>()};
-        CHECK(!details.init());
-        if (const auto *targetName{std::get_if<parser::Name>(&target.u)}) {
+        if (details.init()) {
+          Say(name, "'%s' was previously initialized"_err_en_US);
+          context().SetError(ultimate);
+        } else if (const auto *targetName{
+                       std::get_if<parser::Name>(&target.u)}) {
           Walk(target);
           if (!CheckUseError(*targetName) && targetName->symbol) {
             // Validation is done in declaration checking.
@@ -8520,8 +8721,7 @@ void DeclarationVisitor::PointerInitialization(
         }
       } else {
         Say(name,
-            "'%s' is not a procedure pointer but is initialized "
-            "like one"_err_en_US);
+            "'%s' is not a procedure pointer but is initialized like one"_err_en_US);
         context().SetError(ultimate);
       }
     }
@@ -9334,7 +9534,8 @@ void ResolveNamesVisitor::Post(const parser::AssignedGotoStmt &x) {
 }
 
 void ResolveNamesVisitor::Post(const parser::CompilerDirective &x) {
-  if (std::holds_alternative<parser::CompilerDirective::VectorAlways>(x.u)) {
+  if (std::holds_alternative<parser::CompilerDirective::VectorAlways>(x.u) ||
+      std::holds_alternative<parser::CompilerDirective::Unroll>(x.u)) {
     return;
   }
   if (const auto *tkr{
@@ -9465,6 +9666,12 @@ template <typename A> std::set<SourceName> GetUses(const A &x) {
 }
 
 bool ResolveNamesVisitor::Pre(const parser::Program &x) {
+  if (Scope * hermetic{context().currentHermeticModuleFileScope()}) {
+    // Processing either the dependent modules or first module of a
+    // hermetic module file; ensure that the hermetic module scope has
+    // its implicit rules map entry.
+    ImplicitRulesVisitor::BeginScope(*hermetic);
+  }
   std::map<SourceName, const parser::ProgramUnit *> modules;
   std::set<SourceName> uses;
   bool disordered{false};
@@ -9606,7 +9813,7 @@ void ResolveNamesVisitor::ResolveSpecificationParts(ProgramTree &node) {
       },
       node.stmt());
   Walk(node.spec());
-  bool inDeviceSubprogram = false;
+  bool inDeviceSubprogram{false};
   // If this is a function, convert result to an object. This is to prevent the
   // result from being converted later to a function symbol if it is called
   // inside the function.
diff --git a/flang/lib/Semantics/semantics.cpp b/flang/lib/Semantics/semantics.cpp
index 58dc1f218b56f..10a01039ea0ae 100644
--- a/flang/lib/Semantics/semantics.cpp
+++ b/flang/lib/Semantics/semantics.cpp
@@ -37,12 +37,12 @@
 #include "resolve-labels.h"
 #include "resolve-names.h"
 #include "rewrite-parse-tree.h"
-#include "flang/Common/default-kinds.h"
 #include "flang/Parser/parse-tree-visitor.h"
 #include "flang/Parser/tools.h"
 #include "flang/Semantics/expression.h"
 #include "flang/Semantics/scope.h"
 #include "flang/Semantics/symbol.h"
+#include "flang/Support/default-kinds.h"
 #include "llvm/Support/raw_ostream.h"
 #include "llvm/TargetParser/Host.h"
 #include "llvm/TargetParser/Triple.h"
diff --git a/flang/lib/Semantics/symbol.cpp b/flang/lib/Semantics/symbol.cpp
index 14d6564664f2c..61982295f323a 100644
--- a/flang/lib/Semantics/symbol.cpp
+++ b/flang/lib/Semantics/symbol.cpp
@@ -177,11 +177,11 @@ ProcEntityDetails::ProcEntityDetails(EntityDetails &&d)
     : EntityDetails(std::move(d)) {}
 
 UseErrorDetails::UseErrorDetails(const UseDetails &useDetails) {
-  add_occurrence(useDetails.location(), *GetUsedModule(useDetails).scope());
+  add_occurrence(useDetails.location(), useDetails.symbol());
 }
 UseErrorDetails &UseErrorDetails::add_occurrence(
-    const SourceName &location, const Scope &module) {
-  occurrences_.push_back(std::make_pair(location, &module));
+    const SourceName &location, const Symbol &used) {
+  occurrences_.push_back(std::make_pair(location, &used));
   return *this;
 }
 
@@ -557,9 +557,8 @@ llvm::raw_ostream &operator<<(llvm::raw_ostream &os, const Details &details) {
           [&](const UseErrorDetails &x) {
             os << " uses:";
             char sep{':'};
-            for (const auto &[location, module] : x.occurrences()) {
-              os << sep << " from " << module->GetName().value() << " at "
-                 << location;
+            for (const auto &[location, sym] : x.occurrences()) {
+              os << sep << " from " << sym->name() << " at " << location;
               sep = ',';
             }
           },
diff --git a/flang/lib/Semantics/tools.cpp b/flang/lib/Semantics/tools.cpp
index 052d71be43472..7544731a682ec 100644
--- a/flang/lib/Semantics/tools.cpp
+++ b/flang/lib/Semantics/tools.cpp
@@ -7,7 +7,6 @@
 //===----------------------------------------------------------------------===//
 
 #include "flang/Parser/tools.h"
-#include "flang/Common/Fortran.h"
 #include "flang/Common/indirection.h"
 #include "flang/Parser/dump-parse-tree.h"
 #include "flang/Parser/message.h"
@@ -17,6 +16,7 @@
 #include "flang/Semantics/symbol.h"
 #include "flang/Semantics/tools.h"
 #include "flang/Semantics/type.h"
+#include "flang/Support/Fortran.h"
 #include "llvm/Support/raw_ostream.h"
 #include <algorithm>
 #include <set>
@@ -137,12 +137,6 @@ Tristate IsDefinedAssignment(
   if (!lhsType || !rhsType) {
     return Tristate::No; // error or rhs is untyped
   }
-  if (lhsType->IsUnlimitedPolymorphic()) {
-    return Tristate::No;
-  }
-  if (rhsType->IsUnlimitedPolymorphic()) {
-    return Tristate::Maybe;
-  }
   TypeCategory lhsCat{lhsType->category()};
   TypeCategory rhsCat{rhsType->category()};
   if (rhsRank > 0 && lhsRank != rhsRank) {
@@ -1364,13 +1358,23 @@ void ComponentIterator<componentKind>::const_iterator::Increment() {
   }
 }
 
+template <ComponentKind componentKind>
+SymbolVector
+ComponentIterator<componentKind>::const_iterator::GetComponentPath() const {
+  SymbolVector result;
+  for (const auto &node : componentPath_) {
+    result.push_back(DEREF(node.component()));
+  }
+  return result;
+}
+
 template <ComponentKind componentKind>
 std::string
 ComponentIterator<componentKind>::const_iterator::BuildResultDesignatorName()
     const {
   std::string designator;
-  for (const auto &node : componentPath_) {
-    designator += "%"s + DEREF(node.component()).name().ToString();
+  for (const Symbol &component : GetComponentPath()) {
+    designator += "%"s + component.name().ToString();
   }
   return designator;
 }
@@ -1396,16 +1400,29 @@ UltimateComponentIterator::const_iterator FindPointerUltimateComponent(
 }
 
 PotentialComponentIterator::const_iterator FindEventOrLockPotentialComponent(
-    const DerivedTypeSpec &derived) {
+    const DerivedTypeSpec &derived, bool ignoreCoarrays) {
   PotentialComponentIterator potentials{derived};
-  return std::find_if(
-      potentials.begin(), potentials.end(), [](const Symbol &component) {
-        if (const auto *details{component.detailsIf<ObjectEntityDetails>()}) {
-          const DeclTypeSpec *type{details->type()};
-          return type && IsEventTypeOrLockType(type->AsDerived());
+  auto iter{potentials.begin()};
+  for (auto end{potentials.end()}; iter != end; ++iter) {
+    const Symbol &component{*iter};
+    if (const auto *object{component.detailsIf<ObjectEntityDetails>()}) {
+      if (const DeclTypeSpec * type{object->type()}) {
+        if (IsEventTypeOrLockType(type->AsDerived())) {
+          if (!ignoreCoarrays) {
+            break; // found one
+          }
+          auto path{iter.GetComponentPath()};
+          path.pop_back();
+          if (std::find_if(path.begin(), path.end(), [](const Symbol &sym) {
+                return evaluate::IsCoarray(sym);
+              }) == path.end()) {
+            break; // found one not in a coarray
+          }
         }
-        return false;
-      });
+      }
+    }
+  }
+  return iter;
 }
 
 UltimateComponentIterator::const_iterator FindAllocatableUltimateComponent(
@@ -1764,9 +1781,19 @@ bool HadUseError(
           symbol ? symbol->detailsIf<UseErrorDetails>() : nullptr}) {
     auto &msg{context.Say(
         at, "Reference to '%s' is ambiguous"_err_en_US, symbol->name())};
-    for (const auto &[location, module] : details->occurrences()) {
-      msg.Attach(location, "'%s' was use-associated from module '%s'"_en_US, at,
-          module->GetName().value());
+    for (const auto &[location, sym] : details->occurrences()) {
+      const Symbol &ultimate{sym->GetUltimate()};
+      auto &attachment{
+          msg.Attach(location, "'%s' was use-associated from module '%s'"_en_US,
+              at, sym->owner().GetName().value())};
+      if (&*sym != &ultimate) {
+        // For incompatible definitions where one comes from a hermetic
+        // module file's incorporated dependences and the other from another
+        // module of the same name.
+        attachment.Attach(ultimate.name(),
+            "ultimately from '%s' in module '%s'"_en_US, ultimate.name(),
+            ultimate.owner().GetName().value());
+      }
     }
     context.SetError(*symbol);
     return true;
diff --git a/flang/lib/Support/CMakeLists.txt b/flang/lib/Support/CMakeLists.txt
index 12183f590316d..4ee381589a208 100644
--- a/flang/lib/Support/CMakeLists.txt
+++ b/flang/lib/Support/CMakeLists.txt
@@ -1,9 +1,62 @@
+#===-- lib/Support/CMakeLists.txt ------------------------------------------===#
+#
+# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+# See https://llvm.org/LICENSE.txt for license information.
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+#
+#===------------------------------------------------------------------------===#
+
+find_first_existing_vc_file("${LLVM_MAIN_SRC_DIR}" llvm_vc)
+find_first_existing_vc_file("${FLANG_SOURCE_DIR}" flang_vc)
+
+# The VC revision include that we want to generate.
+set(version_inc "${CMAKE_CURRENT_BINARY_DIR}/VCSVersion.inc")
+
+set(generate_vcs_version_script "${LLVM_CMAKE_DIR}/GenerateVersionFromVCS.cmake")
+
+if(llvm_vc AND LLVM_APPEND_VC_REV)
+  set(llvm_source_dir ${LLVM_MAIN_SRC_DIR})
+endif()
+if(flang_vc AND LLVM_APPEND_VC_REV)
+  set(flang_source_dir ${FLANG_SOURCE_DIR})
+endif()
+
+# Create custom target to generate the VC revision include.
+add_custom_command(OUTPUT "${version_inc}"
+  DEPENDS "${llvm_vc}" "${flang_vc}" "${generate_vcs_version_script}"
+  COMMAND ${CMAKE_COMMAND} "-DNAMES=\"LLVM;FLANG\""
+                           "-DLLVM_SOURCE_DIR=${llvm_source_dir}"
+                           "-DFLANG_SOURCE_DIR=${flang_source_dir}"
+                           "-DHEADER_FILE=${version_inc}"
+                           "-DLLVM_FORCE_VC_REVISION=${LLVM_FORCE_VC_REVISION}"
+                           "-DLLVM_FORCE_VC_REPOSITORY=${LLVM_FORCE_VC_REPOSITORY}"
+                           -P "${generate_vcs_version_script}")
+
+# Mark the generated header as being generated.
+set_source_files_properties("${version_inc}"
+  PROPERTIES GENERATED TRUE
+             HEADER_FILE_ONLY TRUE)
+
+if(FLANG_VENDOR)
+  set_source_files_properties(Version.cpp
+    PROPERTIES COMPILE_DEFINITIONS "FLANG_VENDOR=\"${FLANG_VENDOR} \"")
+endif()
+
 add_flang_library(FortranSupport
+  default-kinds.cpp
+  Fortran.cpp
+  Fortran-features.cpp
+  idioms.cpp
+  LangOptions.cpp
+  OpenMP-utils.cpp
   Timing.cpp
+  Version.cpp
+  ${version_inc}
 
   LINK_COMPONENTS
   Support
 
   MLIR_LIBS
+  MLIRIR
   MLIRSupport
 )
diff --git a/flang/lib/Common/Fortran-features.cpp b/flang/lib/Support/Fortran-features.cpp
similarity index 97%
rename from flang/lib/Common/Fortran-features.cpp
rename to flang/lib/Support/Fortran-features.cpp
index 3565275915a31..bbeb4b15a0486 100644
--- a/flang/lib/Common/Fortran-features.cpp
+++ b/flang/lib/Support/Fortran-features.cpp
@@ -1,4 +1,4 @@
-//===-- lib/Common/Fortran-features.cpp -----------------------------------===//
+//===-- lib/Support/Fortran-features.cpp ------------------------*- C++ -*-===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
@@ -6,9 +6,9 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "flang/Common/Fortran-features.h"
-#include "flang/Common/Fortran.h"
+#include "flang/Support/Fortran-features.h"
 #include "flang/Common/idioms.h"
+#include "flang/Support/Fortran.h"
 
 namespace Fortran::common {
 
@@ -22,6 +22,7 @@ LanguageFeatureControl::LanguageFeatureControl() {
   disable_.set(LanguageFeature::CudaUnified);
   disable_.set(LanguageFeature::ImplicitNoneTypeNever);
   disable_.set(LanguageFeature::ImplicitNoneTypeAlways);
+  disable_.set(LanguageFeature::ImplicitNoneExternal);
   disable_.set(LanguageFeature::DefaultSave);
   disable_.set(LanguageFeature::SaveMainProgram);
   // These features, if enabled, conflict with valid standard usage,
@@ -79,7 +80,6 @@ LanguageFeatureControl::LanguageFeatureControl() {
   warnUsage_.set(UsageWarning::ImplicitShared);
   warnUsage_.set(UsageWarning::IndexVarRedefinition);
   warnUsage_.set(UsageWarning::IncompatibleImplicitInterfaces);
-  warnUsage_.set(UsageWarning::BadTypeForTarget);
   warnUsage_.set(UsageWarning::VectorSubscriptFinalization);
   warnUsage_.set(UsageWarning::UndefinedFunctionResult);
   warnUsage_.set(UsageWarning::UselessIomsg);
diff --git a/flang/lib/Common/Fortran.cpp b/flang/lib/Support/Fortran.cpp
similarity index 96%
rename from flang/lib/Common/Fortran.cpp
rename to flang/lib/Support/Fortran.cpp
index eec83419f9eb2..746b7c9740728 100644
--- a/flang/lib/Common/Fortran.cpp
+++ b/flang/lib/Support/Fortran.cpp
@@ -1,4 +1,4 @@
-//===-- lib/Common/Fortran.cpp --------------------------------------------===//
+//===-- lib/Support/Fortran.cpp ---------------------------------*- C++ -*-===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "flang/Common/Fortran.h"
-#include "flang/Common/Fortran-features.h"
+#include "flang/Support/Fortran.h"
+#include "flang/Support/Fortran-features.h"
 
 namespace Fortran::common {
 
diff --git a/flang/lib/Common/LangOptions.cpp b/flang/lib/Support/LangOptions.cpp
similarity index 80%
rename from flang/lib/Common/LangOptions.cpp
rename to flang/lib/Support/LangOptions.cpp
index 415c715156e7b..273274fc07215 100644
--- a/flang/lib/Common/LangOptions.cpp
+++ b/flang/lib/Support/LangOptions.cpp
@@ -1,4 +1,4 @@
-//===------ LangOptions.cpp -----------------------------------------------===//
+//===-- lib/Support/LangOptions.cpp -----------------------------*- C++ -*-===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
@@ -10,7 +10,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "flang/Common/LangOptions.h"
+#include "flang/Support/LangOptions.h"
 #include <string.h>
 
 namespace Fortran::common {
@@ -18,7 +18,7 @@ namespace Fortran::common {
 LangOptions::LangOptions() {
 #define LANGOPT(Name, Bits, Default) Name = Default;
 #define ENUM_LANGOPT(Name, Type, Bits, Default) set##Name(Default);
-#include "flang/Common/LangOptions.def"
+#include "flang/Support/LangOptions.def"
 }
 
 } // end namespace Fortran::common
diff --git a/flang/lib/Common/OpenMP-utils.cpp b/flang/lib/Support/OpenMP-utils.cpp
similarity index 93%
rename from flang/lib/Common/OpenMP-utils.cpp
rename to flang/lib/Support/OpenMP-utils.cpp
index 47e89fe6dd1ee..178a6e38dc0f2 100644
--- a/flang/lib/Common/OpenMP-utils.cpp
+++ b/flang/lib/Support/OpenMP-utils.cpp
@@ -1,4 +1,4 @@
-//===-- include/flang/Common/OpenMP-utils.cpp ------------------*- C++ -*-====//
+//===-- lib/Support/OpenMP-utils.cpp ----------------------------*- C++ -*-===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
@@ -6,7 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "flang/Common/OpenMP-utils.h"
+#include "flang/Support/OpenMP-utils.h"
 
 #include "mlir/IR/OpDefinition.h"
 
diff --git a/flang/lib/Common/Version.cpp b/flang/lib/Support/Version.cpp
similarity index 95%
rename from flang/lib/Common/Version.cpp
rename to flang/lib/Support/Version.cpp
index d67255f353dfd..8ee4908f28f62 100644
--- a/flang/lib/Common/Version.cpp
+++ b/flang/lib/Support/Version.cpp
@@ -1,4 +1,4 @@
-//===- Version.cpp - Flang Version Number -------------------*- Fortran -*-===//
+//===-- lib/Support/Version.cpp ---------------------------------*- C++ -*-===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
@@ -10,7 +10,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "flang/Common/Version.h"
+#include "flang/Support/Version.h"
 #include "llvm/Support/raw_ostream.h"
 #include <cstdlib>
 #include <cstring>
diff --git a/flang/lib/Common/default-kinds.cpp b/flang/lib/Support/default-kinds.cpp
similarity index 89%
rename from flang/lib/Common/default-kinds.cpp
rename to flang/lib/Support/default-kinds.cpp
index fbafd827ff0d0..b763f8ec19418 100644
--- a/flang/lib/Common/default-kinds.cpp
+++ b/flang/lib/Support/default-kinds.cpp
@@ -1,4 +1,4 @@
-//===-- lib/Common/default-kinds.cpp --------------------------------------===//
+//===-- lib/Support/default-kinds.cpp ---------------------------*- C++ -*-===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
@@ -6,16 +6,12 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "flang/Common/default-kinds.h"
+#include "flang/Support/default-kinds.h"
 #include "flang/Common/idioms.h"
 
 namespace Fortran::common {
 
-IntrinsicTypeDefaultKinds::IntrinsicTypeDefaultKinds() {
-#if __x86_64__
-  quadPrecisionKind_ = 10;
-#endif
-}
+IntrinsicTypeDefaultKinds::IntrinsicTypeDefaultKinds() {}
 
 IntrinsicTypeDefaultKinds &IntrinsicTypeDefaultKinds::set_defaultIntegerKind(
     int k) {
diff --git a/flang/lib/Common/idioms.cpp b/flang/lib/Support/idioms.cpp
similarity index 89%
rename from flang/lib/Common/idioms.cpp
rename to flang/lib/Support/idioms.cpp
index 536a5c2e5479d..906a4c1bcd9b0 100644
--- a/flang/lib/Common/idioms.cpp
+++ b/flang/lib/Support/idioms.cpp
@@ -1,4 +1,4 @@
-//===-- lib/Common/idioms.cpp ---------------------------------------------===//
+//===-- lib/Support/idioms.cpp ----------------------------------*- C++ -*-===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
diff --git a/flang/lib/Testing/CMakeLists.txt b/flang/lib/Testing/CMakeLists.txt
new file mode 100644
index 0000000000000..8051bc09736d1
--- /dev/null
+++ b/flang/lib/Testing/CMakeLists.txt
@@ -0,0 +1,20 @@
+#===-- lib/Testing/CMakeLists.txt ------------------------------------------===#
+#
+# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+# See https://llvm.org/LICENSE.txt for license information.
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+#
+#===------------------------------------------------------------------------===#
+
+add_library(NonGTestTesting EXCLUDE_FROM_ALL
+    testing.cpp
+    fp-testing.cpp
+)
+set_target_properties(NonGTestTesting PROPERTIES FOLDER "Flang/Tests")
+
+if (LLVM_LINK_LLVM_DYLIB)
+  set(llvm_libs LLVM)
+else()
+  llvm_map_components_to_libnames(llvm_libs Support)
+endif()
+target_link_libraries(NonGTestTesting ${llvm_libs})
diff --git a/flang/unittests/Evaluate/fp-testing.cpp b/flang/lib/Testing/fp-testing.cpp
similarity index 87%
rename from flang/unittests/Evaluate/fp-testing.cpp
rename to flang/lib/Testing/fp-testing.cpp
index 1a1d7425d5824..5e1728e8df5e4 100644
--- a/flang/unittests/Evaluate/fp-testing.cpp
+++ b/flang/lib/Testing/fp-testing.cpp
@@ -1,4 +1,12 @@
-#include "fp-testing.h"
+//===-- lib/Testing/fp-testing.cpp ------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "flang/Testing/fp-testing.h"
 #include "llvm/Support/Errno.h"
 #include <cstdio>
 #include <cstdlib>
diff --git a/flang/unittests/Evaluate/testing.cpp b/flang/lib/Testing/testing.cpp
similarity index 88%
rename from flang/unittests/Evaluate/testing.cpp
rename to flang/lib/Testing/testing.cpp
index b2f73bf1b265a..37f9d88f4886b 100644
--- a/flang/unittests/Evaluate/testing.cpp
+++ b/flang/lib/Testing/testing.cpp
@@ -1,4 +1,12 @@
-#include "testing.h"
+//===-- lib/Testing/testing.cpp ---------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "flang/Testing/testing.h"
 #include "llvm/Support/raw_ostream.h"
 #include <cstdarg>
 #include <cstdio>
diff --git a/flang/module/cudadevice.f90 b/flang/module/cudadevice.f90
index 53b6beaaf1ad8..45b9f2c838638 100644
--- a/flang/module/cudadevice.f90
+++ b/flang/module/cudadevice.f90
@@ -29,28 +29,28 @@ attributes(device) subroutine syncthreads()
 
   interface
     attributes(device) integer function syncthreads_and(value)
-      integer :: value
+      integer, value :: value
     end function
   end interface
   public :: syncthreads_and
 
   interface
     attributes(device) integer function syncthreads_count(value)
-      integer :: value
+      integer, value :: value
     end function
   end interface
   public :: syncthreads_count
 
   interface
     attributes(device) integer function syncthreads_or(value)
-      integer :: value
+      integer, value :: value
     end function
   end interface
   public :: syncthreads_or
 
   interface
-    attributes(device) subroutine syncwarp(mask) bind(c, name='__syncwarp')
-      integer :: mask
+    attributes(device) subroutine syncwarp(mask)
+      integer, value :: mask
     end subroutine
   end interface
   public :: syncwarp
@@ -77,6 +77,22 @@ attributes(device) subroutine threadfence_system()
 
   ! Math API
 
+  interface __fadd_rn
+   attributes(device) real function __fadd_rn(a,b) bind(c, name='__nv_fadd_rn')
+  !dir$ ignore_tkr (d) a, (d) b
+    real, value :: a, b
+   end function
+  end interface
+  public :: __fadd_rn
+  
+  interface __fadd_rz
+   attributes(device) real function __fadd_rz(a,b) bind(c, name='__nv_fadd_rz')
+  !dir$ ignore_tkr (d) a, (d) b
+    real, value :: a, b
+   end function
+  end interface
+  public :: __fadd_rz
+
   interface
     attributes(device) function __fadd_rd(x, y) bind(c, name='__nv_fadd_rd')
       real, intent(in), value :: x, y
@@ -93,6 +109,391 @@ attributes(device) subroutine threadfence_system()
   end interface
   public :: __fadd_ru
 
+  interface __fmul_rn
+   attributes(device) real function __fmul_rn(a,b) bind(c, name='__nv_fmul_rn')
+  !dir$ ignore_tkr (d) a, (d) b
+    real, value :: a, b
+   end function
+  end interface
+  public :: __fmul_rn
+
+  interface __fmul_rz
+   attributes(device) real function __fmul_rz(a,b) bind(c, name='__nv_fmul_rz')
+  !dir$ ignore_tkr (d) a, (d) b
+    real, value :: a, b
+   end function
+  end interface
+  public :: __fmul_rz
+
+  interface __fmul_ru
+   attributes(device) real function __fmul_ru(a,b) bind(c, name='__nv_fmul_ru')
+  !dir$ ignore_tkr (d) a, (d) b
+    real, value :: a, b
+   end function
+  end interface
+  public :: __fmul_ru
+
+  interface __fmul_rd
+   attributes(device) real function __fmul_rd(a,b) bind(c, name='__nv_fmul_rd')
+  !dir$ ignore_tkr (d) a, (d) b
+    real, value :: a, b
+   end function
+  end interface
+  public :: __fmul_rd
+
+  interface __fmaf_rn
+   attributes(device) real function __fmaf_rn(a,b,c) bind(c, name='__nv_fmaf_rn')
+  !dir$ ignore_tkr (d) a, (d) b, (d) c
+    real, value :: a, b, c
+   end function
+  end interface
+  public :: __fmaf_rn
+
+  interface __fmaf_rz
+   attributes(device) real function __fmaf_rz(a,b,c) bind(c, name='__nv_fmaf_rz')
+  !dir$ ignore_tkr (d) a, (d) b, (d) c
+    real, value :: a, b, c
+   end function
+  end interface
+  public :: __fmaf_rz
+  
+  interface __fmaf_ru
+   attributes(device) real function __fmaf_ru(a,b,c) bind(c, name='__nv_fmaf_ru')
+  !dir$ ignore_tkr (d) a, (d) b, (d) c
+    real, value :: a, b, c
+   end function
+  end interface
+  public :: __fmaf_ru
+  
+  interface __fmaf_rd
+   attributes(device) real function __fmaf_rd(a,b,c) bind(c, name='__nv_fmaf_rd')
+  !dir$ ignore_tkr (d) a, (d) b, (d) c
+    real, value :: a, b, c
+   end function
+  end interface
+  public :: __fmaf_rd
+
+  interface __frcp_rn
+   attributes(device) real function __frcp_rn(a) bind(c, name='__nv_frcp_rn')
+  !dir$ ignore_tkr (d) a
+    real, value :: a
+   end function
+  end interface
+  public :: __frcp_rn
+
+  interface __frcp_rz
+   attributes(device) real function __frcp_rz(a) bind(c, name='__nv_frcp_rz')
+  !dir$ ignore_tkr (d) a
+    real, value :: a
+   end function
+  end interface
+  public :: __frcp_rz
+
+  interface __frcp_ru
+   attributes(device) real function __frcp_ru(a) bind(c, name='__nv_frcp_ru')
+  !dir$ ignore_tkr (d) a
+    real, value :: a
+   end function
+  end interface
+  public :: __frcp_ru
+
+  interface __frcp_rd
+   attributes(device) real function __frcp_rd(a) bind(c, name='__nv_frcp_rd')
+  !dir$ ignore_tkr (d) a
+    real, value :: a
+   end function
+  end interface
+  public :: __frcp_rd
+
+  interface __fsqrt_rn
+   attributes(device) real function __fsqrt_rn(a) bind(c, name='__nv_fsqrt_rn')
+  !dir$ ignore_tkr (d) a
+    real, value :: a
+   end function
+  end interface
+  public :: __fsqrt_rn
+
+  interface __fsqrt_rz
+   attributes(device) real function __fsqrt_rz(a) bind(c, name='__nv_fsqrt_rz')
+  !dir$ ignore_tkr (d) a
+    real, value :: a
+   end function
+  end interface
+  public :: __fsqrt_rz
+
+  interface __fsqrt_ru
+   attributes(device) real function __fsqrt_ru(a) bind(c, name='__nv_fsqrt_ru')
+  !dir$ ignore_tkr (d) a
+    real, value :: a
+   end function
+  end interface
+  public :: __fsqrt_ru
+
+  interface __fsqrt_rd
+   attributes(device) real function __fsqrt_rd(a) bind(c, name='__nv_fsqrt_rd')
+  !dir$ ignore_tkr (d) a
+    real, value :: a
+   end function
+  end interface
+  public :: __fsqrt_rd
+
+  interface __fdiv_rn
+   attributes(device) real function __fdiv_rn(a,b) bind(c, name='__nv_fdiv_rn')
+  !dir$ ignore_tkr (d) a, (d) b
+    real, value :: a, b
+   end function
+  end interface
+  public :: __fdiv_rn
+
+  interface __fdiv_rz
+   attributes(device) real function __fdiv_rz(a,b) bind(c, name='__nv_fdiv_rz')
+  !dir$ ignore_tkr (d) a, (d) b
+    real, value :: a, b
+   end function
+  end interface
+  public :: __fdiv_rz
+
+  interface __fdiv_ru
+   attributes(device) real function __fdiv_ru(a,b) bind(c, name='__nv_fdiv_ru')
+  !dir$ ignore_tkr (d) a, (d) b
+    real, value :: a, b
+   end function
+  end interface
+  public :: __fdiv_ru
+
+  interface __fdiv_rd
+   attributes(device) real function __fdiv_rd(a,b) bind(c, name='__nv_fdiv_rd')
+  !dir$ ignore_tkr (d) a, (d) b
+    real, value :: a, b
+   end function
+  end interface
+  public :: __fdiv_rd
+
+  interface __dadd_rn
+   attributes(device) real(8) function __dadd_rn(a,b) bind(c, name='__nv_dadd_rn')
+  !dir$ ignore_tkr (d) a, (d) b
+    real(8), value :: a, b
+   end function
+  end interface
+  public :: __dadd_rn
+
+  interface __dadd_rz
+   attributes(device) real(8) function __dadd_rz(a,b) bind(c, name='__nv_dadd_rz')
+  !dir$ ignore_tkr (d) a, (d) b
+    real(8), value :: a, b
+   end function
+  end interface
+  public :: __dadd_rz
+
+  interface __dadd_ru
+   attributes(device) real(8) function __dadd_ru(a,b) bind(c, name='__nv_dadd_ru')
+  !dir$ ignore_tkr (d) a, (d) b
+    real(8), value :: a, b
+   end function
+  end interface
+  public :: __dadd_ru
+
+  interface __dadd_rd
+   attributes(device) real(8) function __dadd_rd(a,b) bind(c, name='__nv_dadd_rd')
+  !dir$ ignore_tkr (d) a, (d) b
+    real(8), value :: a, b
+   end function
+  end interface
+  public :: __dadd_rd
+
+  interface __dmul_rn
+   attributes(device) real(8) function __dmul_rn(a,b) bind(c, name='__nv_dmul_rn')
+  !dir$ ignore_tkr (d) a, (d) b
+    real(8), value :: a, b
+   end function
+  end interface
+  public :: __dmul_rn
+
+  interface __dmul_rz
+   attributes(device) real(8) function __dmul_rz(a,b) bind(c, name='__nv_dmul_rz')
+  !dir$ ignore_tkr (d) a, (d) b
+    real(8), value :: a, b
+   end function
+  end interface
+  public :: __dmul_rz
+
+  interface __dmul_ru
+   attributes(device) real(8) function __dmul_ru(a,b) bind(c, name='__nv_dmul_ru')
+  !dir$ ignore_tkr (d) a, (d) b
+    real(8), value :: a, b
+   end function
+  end interface
+  public :: __dmul_ru
+
+  interface __dmul_rd
+   attributes(device) real(8) function __dmul_rd(a,b) bind(c, name='__nv_dmul_rd')
+  !dir$ ignore_tkr (d) a, (d) b
+    real(8), value :: a, b
+   end function
+  end interface
+  public :: __dmul_rd
+
+  interface __fma_rn
+   attributes(device) real(8) function __fma_rn(a,b,c) bind(c, name='__nv_fma_rn')
+  !dir$ ignore_tkr (d) a, (d) b
+    real(8), value :: a, b, c
+   end function
+  end interface
+  public :: __fma_rn
+
+  interface __fma_rz
+   attributes(device) real(8) function __fma_rz(a,b,c) bind(c, name='__nv_fma_rz')
+  !dir$ ignore_tkr (d) a, (d) b
+    real(8), value :: a, b, c
+   end function
+  end interface
+  public :: __fma_rz
+
+  interface __fma_ru
+   attributes(device) real(8) function __fma_ru(a,b,c) bind(c, name='__nv_fma_ru')
+  !dir$ ignore_tkr (d) a, (d) b
+    real(8), value :: a, b, c
+   end function
+  end interface
+  public :: __fma_ru
+
+  interface __fma_rd
+   attributes(device) real(8) function __fma_rd(a,b,c) bind(c, name='__nv_fma_rd')
+  !dir$ ignore_tkr (d) a, (d) b
+    real(8), value :: a, b, c
+   end function
+  end interface
+  public :: __fma_rd
+
+  interface rsqrt
+    attributes(device) real(4) function rsqrtf(x) bind(c,name='__nv_rsqrtf')
+      real(4), value :: x
+    end function
+    attributes(device) real(8) function rsqrt(x) bind(c,name='__nv_rsqrt')
+      real(8), value :: x
+    end function
+  end interface
+  public :: rsqrt
+  
+  interface signbit
+    attributes(device) integer(4) function signbitf(x) bind(c,name='__nv_signbitf')
+      real(4), value :: x
+    end function
+    attributes(device) integer(4) function signbit(x) bind(c,name='__nv_signbitd')
+      real(8), value :: x
+    end function
+  end interface
+  public :: signbit
+
+  interface
+    attributes(device) subroutine sincosf(x, y, z) bind(c,name='__nv_sincosf')
+      real(4), value :: x
+      real(4), device :: y
+      real(4), device :: z
+    end subroutine
+  end interface
+  public :: sincosf
+  interface
+    attributes(device) subroutine sincos(x, y, z) bind(c,name='__nv_sincos')
+      real(8), value :: x
+      real(8), device :: y
+      real(8), device :: z
+    end subroutine
+  end interface
+  interface sincos
+    procedure :: sincosf
+    procedure :: sincos
+  end interface
+  public :: sincos
+
+  interface
+    attributes(device) subroutine sincospif(x, y, z) bind(c,name='__nv_sincospif')
+      real(4), value :: x
+      real(4), device :: y
+      real(4), device :: z
+    end subroutine
+  end interface
+  public :: sincospif
+  interface
+    attributes(device) subroutine sincospi(x, y, z) bind(c,name='__nv_sincospi')
+      real(8), value :: x
+      real(8), device :: y
+      real(8), device :: z
+    end subroutine
+  end interface
+  interface sincospi
+    procedure :: sincospif
+    procedure :: sincospi
+  end interface
+  public :: sincospi
+
+  interface
+    attributes(device) real(4) function cospif(x) bind(c,name='__nv_cospif')
+      real(4), value :: x
+    end function
+  end interface
+  public :: cospif
+  interface
+    attributes(device) real(8) function cospi(x) bind(c,name='__nv_cospi')
+      real(8), value :: x
+    end function
+  end interface
+  interface cospi
+    procedure :: cospif
+    procedure :: cospi
+  end interface
+  public :: cospi
+
+  interface  
+    attributes(device) real(4) function sinpif(x) bind(c,name='__nv_sinpif')
+      real(4), value :: x
+    end function
+  end interface
+  public :: sinpif
+  interface
+    attributes(device) real(8) function sinpi(x) bind(c,name='__nv_sinpi')
+      real(8), value :: x
+    end function
+  end interface
+  interface sinpi
+    procedure :: sinpif
+    procedure :: sinpi
+  end interface
+  public :: sinpi
+  
+  interface mulhi
+   attributes(device) integer function __mulhi(i,j) bind(c,name='__nv_mulhi')
+  !dir$ ignore_tkr (d) i, (d) j
+    integer, value :: i,j
+   end function
+  end interface
+  public :: mulhi
+
+  interface umulhi
+   attributes(device) integer function __umulhi(i,j) bind(c,name='__nv_umulhi')
+  !dir$ ignore_tkr (d) i, (d) j
+    integer, value :: i,j
+   end function
+  end interface
+  public :: umulhi
+  
+  interface mul64hi
+   attributes(device) integer(8) function __mul64hi(i,j) bind(c,name='__nv_mul64hi')
+  !dir$ ignore_tkr (d) i, (d) j
+    integer(8), value :: i,j
+   end function
+  end interface
+  public :: mul64hi
+  
+  interface umul64hi
+   attributes(device) integer(8) function __umul64hi(i,j) bind(c,name='__nv_umul64hi')
+  !dir$ ignore_tkr (d) i, (d) j
+    integer(8), value :: i,j
+   end function
+  end interface
+  public :: umul64hi
+
+
   ! Atomic Operations
 
   interface atomicadd
@@ -106,10 +507,10 @@ attributes(device) pure real function atomicaddf(address, val)
     real, intent(inout) :: address
     real, value :: val
     end function
-    attributes(device) pure real*8 function atomicaddd(address, val)
+    attributes(device) pure real(8) function atomicaddd(address, val)
   !dir$ ignore_tkr (d) address, (d) val
-    real*8, intent(inout) :: address
-    real*8, value :: val
+    real(8), intent(inout) :: address
+    real(8), value :: val
     end function
     attributes(device) pure integer(8) function atomicaddl(address, val)
   !dir$ ignore_tkr (d) address, (d) val
@@ -117,6 +518,115 @@ attributes(device) pure integer(8) function atomicaddl(address, val)
     integer(8), value :: val
     end function
   end interface 
-public :: atomicadd
+  public :: atomicadd
+
+  interface atomicsub
+    attributes(device) pure integer function atomicsubi(address, val)
+  !dir$ ignore_tkr (d) address, (d) val
+    integer, intent(inout) :: address
+    integer, value :: val
+    end function
+    attributes(device) pure real function atomicsubf(address, val)
+  !dir$ ignore_tkr (d) address, (d) val
+    real, intent(inout) :: address
+    real, value :: val
+    end function
+    attributes(device) pure real(8) function atomicsubd(address, val)
+  !dir$ ignore_tkr (d) address, (d) val
+    real(8), intent(inout) :: address
+    real(8), value :: val
+    end function
+    attributes(device) pure integer(8) function atomicsubl(address, val)
+  !dir$ ignore_tkr (d) address, (dk) val
+    integer(8), intent(inout) :: address
+    integer(8), value :: val
+    end function
+  end interface
+  public :: atomicsub
+  
+  interface atomicmax
+    attributes(device) pure integer function atomicmaxi(address, val)
+  !dir$ ignore_tkr (d) address, (d) val
+    integer, intent(inout) :: address
+    integer, value :: val
+    end function
+    attributes(device) pure real function atomicmaxf(address, val)
+  !dir$ ignore_tkr (d) address, (d) val
+    real, intent(inout) :: address
+    real, value :: val
+    end function
+    attributes(device) pure real(8) function atomicmaxd(address, val)
+  !dir$ ignore_tkr (d) address, (d) val
+    real(8), intent(inout) :: address
+    real(8), value :: val
+    end function
+    attributes(device) pure integer(8) function atomicmaxl(address, val)
+  !dir$ ignore_tkr (d) address, (dk) val
+    integer(8), intent(inout) :: address
+    integer(8), value :: val
+    end function
+  end interface
+  public :: atomicmax
+  
+  interface atomicmin
+    attributes(device) pure integer function atomicmini(address, val)
+  !dir$ ignore_tkr (d) address, (d) val
+    integer, intent(inout) :: address
+    integer, value :: val
+    end function
+    attributes(device) pure real function atomicminf(address, val)
+  !dir$ ignore_tkr (d) address, (d) val
+    real, intent(inout) :: address
+    real, value :: val
+    end function
+    attributes(device) pure real(8) function atomicmind(address, val)
+  !dir$ ignore_tkr (d) address, (d) val
+    real(8), intent(inout) :: address
+    real(8), value :: val
+    end function
+    attributes(device) pure integer(8) function atomicminl(address, val)
+  !dir$ ignore_tkr (d) address, (dk) val
+    integer(8), intent(inout) :: address
+    integer(8), value :: val
+    end function
+  end interface
+  public :: atomicmin
+  
+  interface atomicand
+    attributes(device) pure integer function atomicandi(address, val)
+  !dir$ ignore_tkr (d) address, (d) val
+    integer, intent(inout) :: address
+    integer, value :: val
+    end function
+  end interface
+  public :: atomicand
+  
+  interface atomicor
+    attributes(device) pure integer function atomicori(address, val)
+  !dir$ ignore_tkr (d) address, (d) val
+    integer, intent(inout) :: address
+    integer, value :: val
+    end function
+  end interface
+  public :: atomicor
+
+  interface atomicinc
+    attributes(device) pure integer function atomicinci(address, val)
+  !dir$ ignore_tkr (d) address, (d) val
+    integer, intent(inout) :: address
+    integer, value :: val
+    end function
+  end interface
+  public :: atomicinc
+  
+  interface atomicdec
+    attributes(device) pure integer function atomicdeci(address, val)
+  !dir$ ignore_tkr (d) address, (d) val
+    integer, intent(inout) :: address
+    integer, value :: val
+    end function
+  end interface
+  public :: atomicdec
+
 
 end module
diff --git a/flang/runtime/CMakeLists.txt b/flang/runtime/CMakeLists.txt
index fbfaae9a88064..f651ea83664eb 100644
--- a/flang/runtime/CMakeLists.txt
+++ b/flang/runtime/CMakeLists.txt
@@ -59,10 +59,15 @@ if (CMAKE_SOURCE_DIR STREQUAL CMAKE_CURRENT_SOURCE_DIR)
     )
 endif()
 
+set(linked_libraries "")
+
 # function checks
 find_package(Backtrace)
 set(HAVE_BACKTRACE ${Backtrace_FOUND})
 set(BACKTRACE_HEADER ${Backtrace_HEADER})
+if(HAVE_BACKTRACE)
+  list(APPEND linked_libraries ${Backtrace_LIBRARY})
+endif()
 
 include(CheckCXXSymbolExists)
 include(CheckCXXSourceCompiles)
@@ -111,6 +116,8 @@ add_definitions(-U_LIBCPP_ENABLE_ASSERTIONS)
 add_subdirectory(Float128Math)
 
 set(sources
+  ${FLANG_SOURCE_DIR}/lib/Decimal/binary-to-decimal.cpp
+  ${FLANG_SOURCE_DIR}/lib/Decimal/decimal-to-binary.cpp
   ISO_Fortran_binding.cpp
   allocator-registry.cpp
   allocatable.cpp
@@ -182,6 +189,8 @@ include(AddFlangOffloadRuntime)
 
 # List of files that are buildable for all devices.
 set(supported_files
+  ${FLANG_SOURCE_DIR}/lib/Decimal/binary-to-decimal.cpp
+  ${FLANG_SOURCE_DIR}/lib/Decimal/decimal-to-binary.cpp
   ISO_Fortran_binding.cpp
   allocatable.cpp
   allocator-registry.cpp
@@ -234,13 +243,13 @@ set(supported_files
   utf.cpp
   )
 
-enable_cuda_compilation(FortranRuntime "${supported_files}")
+enable_cuda_compilation(flang_rt "${supported_files}")
 enable_omp_offload_compilation("${supported_files}")
 
-if (NOT TARGET FortranFloat128Math)
-  # If FortranFloat128Math is not defined, then we are not building
-  # standalone FortranFloat128Math library. Instead, include
-  # the relevant sources into FortranRuntime itself.
+if (NOT TARGET flang_rt.quadmath)
+  # If flang_rt.quadmath is not defined, then we are not building
+  # standalone flang_rt.quadmath library. Instead, include
+  # the relevant sources into flang_rt.runtime itself.
   # The information is provided via FortranFloat128MathILib
   # interface library.
   get_target_property(f128_sources
@@ -268,49 +277,44 @@ if (NOT TARGET FortranFloat128Math)
 endif()
 
 if (NOT DEFINED MSVC)
-  add_flang_library(FortranRuntime
+  add_flang_library(flang_rt.runtime
     ${sources}
     LINK_LIBS
-    FortranDecimal
+    ${linked_libraries}
 
     INSTALL_WITH_TOOLCHAIN
   )
 else()
-  add_flang_library(FortranRuntime
+  add_flang_library(flang_rt.runtime
     ${sources}
     LINK_LIBS
-    FortranDecimal
+    ${linked_libraries}
   )
   set(CMAKE_MSVC_RUNTIME_LIBRARY MultiThreaded)
-  add_flang_library(FortranRuntime.static ${sources}
-    LINK_LIBS
-    FortranDecimal.static
+  add_flang_library(flang_rt.runtime.static ${sources}
     INSTALL_WITH_TOOLCHAIN)
-  set_target_properties(FortranRuntime.static PROPERTIES FOLDER "Flang/Runtime Libraries")
+  set_target_properties(flang_rt.runtime.static PROPERTIES FOLDER "Flang/Runtime Libraries")
   set(CMAKE_MSVC_RUNTIME_LIBRARY MultiThreadedDLL)
-  add_flang_library(FortranRuntime.dynamic ${sources}
-    LINK_LIBS
-    FortranDecimal.dynamic
+  add_flang_library(flang_rt.runtime.dynamic ${sources}
     INSTALL_WITH_TOOLCHAIN)
-  set_target_properties(FortranRuntime.dynamic PROPERTIES FOLDER "Flang/Runtime Libraries")
+  set_target_properties(flang_rt.runtime.dynamic PROPERTIES FOLDER "Flang/Runtime Libraries")
   set(CMAKE_MSVC_RUNTIME_LIBRARY MultiThreadedDebug)
-  add_flang_library(FortranRuntime.static_dbg ${sources}
-    LINK_LIBS
-    FortranDecimal.static_dbg
+  add_flang_library(flang_rt.runtime.static_dbg ${sources}
     INSTALL_WITH_TOOLCHAIN)
-  set_target_properties(FortranRuntime.static_dbg PROPERTIES FOLDER "Flang/Runtime Libraries")
+  set_target_properties(flang_rt.runtime.static_dbg PROPERTIES FOLDER "Flang/Runtime Libraries")
   set(CMAKE_MSVC_RUNTIME_LIBRARY MultiThreadedDebugDLL)
-  add_flang_library(FortranRuntime.dynamic_dbg ${sources}
-    LINK_LIBS
-    FortranDecimal.dynamic_dbg
+  add_flang_library(flang_rt.runtime.dynamic_dbg ${sources}
     INSTALL_WITH_TOOLCHAIN)
-  set_target_properties(FortranRuntime.dynamic_dbg PROPERTIES FOLDER "Flang/Runtime Libraries")
-  add_dependencies(FortranRuntime FortranRuntime.static FortranRuntime.dynamic
-    FortranRuntime.static_dbg FortranRuntime.dynamic_dbg)
+  set_target_properties(flang_rt.runtime.dynamic_dbg PROPERTIES FOLDER "Flang/Runtime Libraries")
+  add_dependencies(flang_rt.runtime
+    flang_rt.runtime.static
+    flang_rt.runtime.dynamic
+    flang_rt.runtime.static_dbg
+    flang_rt.runtime.dynamic_dbg)
 endif()
-set_target_properties(FortranRuntime PROPERTIES FOLDER "Flang/Runtime Libraries")
+set_target_properties(flang_rt.runtime PROPERTIES FOLDER "Flang/Runtime Libraries")
 
-# If FortranRuntime is part of a Flang build (and not a separate build) then
+# If flang_rt is part of a Flang build (and not a separate build) then
 # add dependency to make sure that Fortran runtime library is being built after
 # we have the Flang compiler available.  This also includes the MODULE files
 # that compile when the 'flang' target is built.
@@ -318,9 +322,21 @@ set_target_properties(FortranRuntime PROPERTIES FOLDER "Flang/Runtime Libraries"
 # TODO: This is a workaround and should be updated when runtime build procedure
 # is changed to a regular runtime build.  See discussion in PR #95388.
 if (TARGET flang AND TARGET module_files)
-  add_dependencies(FortranRuntime flang module_files)
+  add_dependencies(flang_rt.runtime flang module_files)
 endif()
 
 if (FLANG_CUF_RUNTIME)
   add_subdirectory(CUDA)
 endif()
+
+# Compatibility targets.
+add_custom_target(flang-rt)
+add_dependencies(flang-rt flang_rt.runtime)
+if (TARGET flang_rt.quadmath)
+  add_dependencies(flang-rt flang_rt.quadmath)
+endif ()
+if (TARGET flang_rt.cuda_${CUDAToolkit_VERSION_MAJOR})
+  add_dependencies(flang-rt flang_rt.cuda_${CUDAToolkit_VERSION_MAJOR})
+endif ()
+add_custom_target(FortranRuntime)
+add_dependencies(FortranRuntime flang_rt.runtime)
diff --git a/flang/runtime/CUDA/CMakeLists.txt b/flang/runtime/CUDA/CMakeLists.txt
index 23e01da72eded..1fd3bf22a83cf 100644
--- a/flang/runtime/CUDA/CMakeLists.txt
+++ b/flang/runtime/CUDA/CMakeLists.txt
@@ -8,15 +8,16 @@
 
 include_directories(${CUDAToolkit_INCLUDE_DIRS})
 
-# libCufRuntime depends on a certain version of CUDA. To be able to have
+# libflang_rt.cuda depends on a certain version of CUDA. To be able to have
 # multiple build of this library with different CUDA version, the version is
 # added to the library name.
-set(CUFRT_LIBNAME CufRuntime_cuda_${CUDAToolkit_VERSION_MAJOR})
+set(CUFRT_LIBNAME flang_rt.cuda_${CUDAToolkit_VERSION_MAJOR})
 
 add_flang_library(${CUFRT_LIBNAME}
   allocator.cpp
   allocatable.cpp
   descriptor.cpp
+  init.cpp
   kernel.cpp
   memmove-function.cpp
   memory.cpp
@@ -32,6 +33,6 @@ endif()
 
 target_link_libraries(${CUFRT_LIBNAME}
   PRIVATE
-  FortranRuntime
+  flang_rt.runtime
   ${CUDA_RT_TARGET}
 )
diff --git a/flang/runtime/CUDA/allocatable.cpp b/flang/runtime/CUDA/allocatable.cpp
index 9be54e8906903..6df3b06793b3e 100644
--- a/flang/runtime/CUDA/allocatable.cpp
+++ b/flang/runtime/CUDA/allocatable.cpp
@@ -23,10 +23,10 @@ extern "C" {
 RT_EXT_API_GROUP_BEGIN
 
 int RTDEF(CUFAllocatableAllocateSync)(Descriptor &desc, int64_t stream,
-    bool hasStat, const Descriptor *errMsg, const char *sourceFile,
-    int sourceLine) {
+    bool *pinned, bool hasStat, const Descriptor *errMsg,
+    const char *sourceFile, int sourceLine) {
   int stat{RTNAME(CUFAllocatableAllocate)(
-      desc, stream, hasStat, errMsg, sourceFile, sourceLine)};
+      desc, stream, pinned, hasStat, errMsg, sourceFile, sourceLine)};
 #ifndef RT_DEVICE_COMPILATION
   // Descriptor synchronization is only done when the allocation is done
   // from the host.
@@ -41,8 +41,8 @@ int RTDEF(CUFAllocatableAllocateSync)(Descriptor &desc, int64_t stream,
 }
 
 int RTDEF(CUFAllocatableAllocate)(Descriptor &desc, int64_t stream,
-    bool hasStat, const Descriptor *errMsg, const char *sourceFile,
-    int sourceLine) {
+    bool *pinned, bool hasStat, const Descriptor *errMsg,
+    const char *sourceFile, int sourceLine) {
   if (desc.HasAddendum()) {
     Terminator terminator{sourceFile, sourceLine};
     // TODO: This require a bit more work to set the correct type descriptor
@@ -53,14 +53,19 @@ int RTDEF(CUFAllocatableAllocate)(Descriptor &desc, int64_t stream,
   // Perform the standard allocation.
   int stat{RTNAME(AllocatableAllocate)(
       desc, hasStat, errMsg, sourceFile, sourceLine)};
+  if (pinned) {
+    // Set pinned according to stat. More infrastructre is needed to set it
+    // closer to the actual allocation call.
+    *pinned = (stat == StatOk);
+  }
   return stat;
 }
 
 int RTDEF(CUFAllocatableAllocateSource)(Descriptor &alloc,
-    const Descriptor &source, int64_t stream, bool hasStat,
+    const Descriptor &source, int64_t stream, bool *pinned, bool hasStat,
     const Descriptor *errMsg, const char *sourceFile, int sourceLine) {
   int stat{RTNAME(CUFAllocatableAllocate)(
-      alloc, stream, hasStat, errMsg, sourceFile, sourceLine)};
+      alloc, stream, pinned, hasStat, errMsg, sourceFile, sourceLine)};
   if (stat == StatOk) {
     Terminator terminator{sourceFile, sourceLine};
     Fortran::runtime::DoFromSourceAssign(
@@ -70,10 +75,10 @@ int RTDEF(CUFAllocatableAllocateSource)(Descriptor &alloc,
 }
 
 int RTDEF(CUFAllocatableAllocateSourceSync)(Descriptor &alloc,
-    const Descriptor &source, int64_t stream, bool hasStat,
+    const Descriptor &source, int64_t stream, bool *pinned, bool hasStat,
     const Descriptor *errMsg, const char *sourceFile, int sourceLine) {
   int stat{RTNAME(CUFAllocatableAllocateSync)(
-      alloc, stream, hasStat, errMsg, sourceFile, sourceLine)};
+      alloc, stream, pinned, hasStat, errMsg, sourceFile, sourceLine)};
   if (stat == StatOk) {
     Terminator terminator{sourceFile, sourceLine};
     Fortran::runtime::DoFromSourceAssign(
diff --git a/flang/runtime/CUDA/allocator.cpp b/flang/runtime/CUDA/allocator.cpp
index 85b3daf65a8ba..368c1124ef70a 100644
--- a/flang/runtime/CUDA/allocator.cpp
+++ b/flang/runtime/CUDA/allocator.cpp
@@ -11,10 +11,10 @@
 #include "../stat.h"
 #include "../terminator.h"
 #include "../type-info.h"
-#include "flang/Common/Fortran.h"
-#include "flang/ISO_Fortran_binding_wrapper.h"
+#include "flang/Common/ISO_Fortran_binding_wrapper.h"
 #include "flang/Runtime/CUDA/common.h"
 #include "flang/Runtime/allocator-registry.h"
+#include "flang/Support/Fortran.h"
 
 #include "cuda_runtime.h"
 
diff --git a/flang/runtime/CUDA/init.cpp b/flang/runtime/CUDA/init.cpp
new file mode 100644
index 0000000000000..2bffce842b952
--- /dev/null
+++ b/flang/runtime/CUDA/init.cpp
@@ -0,0 +1,25 @@
+//===-- runtime/CUDA/init.cpp ---------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "flang/Runtime/CUDA/init.h"
+#include "../environment.h"
+#include "../terminator.h"
+#include "flang/Runtime/CUDA/common.h"
+
+#include "cuda_runtime.h"
+
+extern "C" {
+
+void RTDEF(CUFInit)() {
+  // Perform ctx initialization based on execution environment if necessary.
+  if (Fortran::runtime::executionEnvironment.cudaStackLimit) {
+    CUDA_REPORT_IF_ERROR(cudaDeviceSetLimit(cudaLimitStackSize,
+        Fortran::runtime::executionEnvironment.cudaStackLimit));
+  }
+}
+}
diff --git a/flang/runtime/CUDA/kernel.cpp b/flang/runtime/CUDA/kernel.cpp
index bdc04ccb17672..02d89fb8423a5 100644
--- a/flang/runtime/CUDA/kernel.cpp
+++ b/flang/runtime/CUDA/kernel.cpp
@@ -151,4 +151,69 @@ void RTDEF(CUFLaunchClusterKernel)(const void *kernel, intptr_t clusterX,
   CUDA_REPORT_IF_ERROR(cudaLaunchKernelExC(&config, kernel, params));
 }
 
+void RTDEF(CUFLaunchCooperativeKernel)(const void *kernel, intptr_t gridX,
+    intptr_t gridY, intptr_t gridZ, intptr_t blockX, intptr_t blockY,
+    intptr_t blockZ, int32_t smem, void **params, void **extra) {
+  dim3 gridDim;
+  gridDim.x = gridX;
+  gridDim.y = gridY;
+  gridDim.z = gridZ;
+  dim3 blockDim;
+  blockDim.x = blockX;
+  blockDim.y = blockY;
+  blockDim.z = blockZ;
+  unsigned nbNegGridDim{0};
+  if (gridX < 0) {
+    ++nbNegGridDim;
+  }
+  if (gridY < 0) {
+    ++nbNegGridDim;
+  }
+  if (gridZ < 0) {
+    ++nbNegGridDim;
+  }
+  if (nbNegGridDim == 1) {
+    int maxBlocks, nbBlocks, dev, multiProcCount;
+    cudaError_t err1, err2;
+    nbBlocks = blockDim.x * blockDim.y * blockDim.z;
+    cudaGetDevice(&dev);
+    err1 = cudaDeviceGetAttribute(
+        &multiProcCount, cudaDevAttrMultiProcessorCount, dev);
+    err2 = cudaOccupancyMaxActiveBlocksPerMultiprocessor(
+        &maxBlocks, kernel, nbBlocks, smem);
+    if (err1 == cudaSuccess && err2 == cudaSuccess) {
+      maxBlocks = multiProcCount * maxBlocks;
+    }
+    if (maxBlocks > 0) {
+      if (gridX > 0) {
+        maxBlocks = maxBlocks / gridDim.x;
+      }
+      if (gridY > 0) {
+        maxBlocks = maxBlocks / gridDim.y;
+      }
+      if (gridZ > 0) {
+        maxBlocks = maxBlocks / gridDim.z;
+      }
+      if (maxBlocks < 1) {
+        maxBlocks = 1;
+      }
+      if (gridX < 0) {
+        gridDim.x = maxBlocks;
+      }
+      if (gridY < 0) {
+        gridDim.y = maxBlocks;
+      }
+      if (gridZ < 0) {
+        gridDim.z = maxBlocks;
+      }
+    }
+  } else if (nbNegGridDim > 1) {
+    Fortran::runtime::Terminator terminator{__FILE__, __LINE__};
+    terminator.Crash("Too many invalid grid dimensions");
+  }
+  cudaStream_t stream = 0; // TODO stream managment
+  CUDA_REPORT_IF_ERROR(cudaLaunchCooperativeKernel(
+      kernel, gridDim, blockDim, params, smem, stream));
+}
+
 } // extern "C"
diff --git a/flang/runtime/CUDA/pointer.cpp b/flang/runtime/CUDA/pointer.cpp
index 3252410bd8d2c..d3ebe97b4e4ac 100644
--- a/flang/runtime/CUDA/pointer.cpp
+++ b/flang/runtime/CUDA/pointer.cpp
@@ -21,8 +21,9 @@ namespace Fortran::runtime::cuda {
 extern "C" {
 RT_EXT_API_GROUP_BEGIN
 
-int RTDEF(CUFPointerAllocate)(Descriptor &desc, int64_t stream, bool hasStat,
-    const Descriptor *errMsg, const char *sourceFile, int sourceLine) {
+int RTDEF(CUFPointerAllocate)(Descriptor &desc, int64_t stream, bool *pinned,
+    bool hasStat, const Descriptor *errMsg, const char *sourceFile,
+    int sourceLine) {
   if (desc.HasAddendum()) {
     Terminator terminator{sourceFile, sourceLine};
     // TODO: This require a bit more work to set the correct type descriptor
@@ -33,14 +34,19 @@ int RTDEF(CUFPointerAllocate)(Descriptor &desc, int64_t stream, bool hasStat,
   // Perform the standard allocation.
   int stat{
       RTNAME(PointerAllocate)(desc, hasStat, errMsg, sourceFile, sourceLine)};
+  if (pinned) {
+    // Set pinned according to stat. More infrastructre is needed to set it
+    // closer to the actual allocation call.
+    *pinned = (stat == StatOk);
+  }
   return stat;
 }
 
 int RTDEF(CUFPointerAllocateSync)(Descriptor &desc, int64_t stream,
-    bool hasStat, const Descriptor *errMsg, const char *sourceFile,
-    int sourceLine) {
+    bool *pinned, bool hasStat, const Descriptor *errMsg,
+    const char *sourceFile, int sourceLine) {
   int stat{RTNAME(CUFPointerAllocate)(
-      desc, stream, hasStat, errMsg, sourceFile, sourceLine)};
+      desc, stream, pinned, hasStat, errMsg, sourceFile, sourceLine)};
 #ifndef RT_DEVICE_COMPILATION
   // Descriptor synchronization is only done when the allocation is done
   // from the host.
@@ -55,10 +61,10 @@ int RTDEF(CUFPointerAllocateSync)(Descriptor &desc, int64_t stream,
 }
 
 int RTDEF(CUFPointerAllocateSource)(Descriptor &pointer,
-    const Descriptor &source, int64_t stream, bool hasStat,
+    const Descriptor &source, int64_t stream, bool *pinned, bool hasStat,
     const Descriptor *errMsg, const char *sourceFile, int sourceLine) {
   int stat{RTNAME(CUFPointerAllocate)(
-      pointer, stream, hasStat, errMsg, sourceFile, sourceLine)};
+      pointer, stream, pinned, hasStat, errMsg, sourceFile, sourceLine)};
   if (stat == StatOk) {
     Terminator terminator{sourceFile, sourceLine};
     Fortran::runtime::DoFromSourceAssign(
@@ -68,10 +74,10 @@ int RTDEF(CUFPointerAllocateSource)(Descriptor &pointer,
 }
 
 int RTDEF(CUFPointerAllocateSourceSync)(Descriptor &pointer,
-    const Descriptor &source, int64_t stream, bool hasStat,
+    const Descriptor &source, int64_t stream, bool *pinned, bool hasStat,
     const Descriptor *errMsg, const char *sourceFile, int sourceLine) {
   int stat{RTNAME(CUFPointerAllocateSync)(
-      pointer, stream, hasStat, errMsg, sourceFile, sourceLine)};
+      pointer, stream, pinned, hasStat, errMsg, sourceFile, sourceLine)};
   if (stat == StatOk) {
     Terminator terminator{sourceFile, sourceLine};
     Fortran::runtime::DoFromSourceAssign(
diff --git a/flang/runtime/Float128Math/CMakeLists.txt b/flang/runtime/Float128Math/CMakeLists.txt
index 703f85fcaf8da..3c382d16a21cd 100644
--- a/flang/runtime/Float128Math/CMakeLists.txt
+++ b/flang/runtime/Float128Math/CMakeLists.txt
@@ -12,7 +12,7 @@
 # It is distributed as a static library only.
 # Fortran programs/libraries that end up linking any of the provided
 # will have a dependency on the third-party library that is being
-# used for building this FortranFloat128Math library.
+# used for building this flang_rt.quadmath library.
 
 include(CheckLibraryExists)
 
@@ -93,20 +93,20 @@ if (FLANG_RUNTIME_F128_MATH_LIB)
       )
   endif()
 
-  add_flang_library(FortranFloat128Math STATIC INSTALL_WITH_TOOLCHAIN
+  add_flang_library(flang_rt.quadmath STATIC INSTALL_WITH_TOOLCHAIN
     ${sources})
 
   if (DEFINED MSVC)
     set(CMAKE_MSVC_RUNTIME_LIBRARY MultiThreaded)
-    add_flang_library(FortranFloat128Math.static STATIC INSTALL_WITH_TOOLCHAIN
+    add_flang_library(flang_rt.quadmath.static STATIC INSTALL_WITH_TOOLCHAIN
       ${sources}
       )
     set(CMAKE_MSVC_RUNTIME_LIBRARY MultiThreadedDebug)
-    add_flang_library(FortranFloat128Math.static_dbg STATIC INSTALL_WITH_TOOLCHAIN
+    add_flang_library(flang_rt.quadmath.static_dbg STATIC INSTALL_WITH_TOOLCHAIN
       ${sources}
       )
-    add_dependencies(FortranFloat128Math FortranFloat128Math.static
-      FortranFloat128Math.static_dbg
+    add_dependencies(flang_rt.quadmath flang_rt.quadmath.static
+      flang_rt.quadmath.static_dbg
       )
   endif()
 elseif (HAVE_LDBL_MANT_DIG_113)
@@ -118,7 +118,7 @@ elseif (HAVE_LDBL_MANT_DIG_113)
       )
     target_sources(FortranFloat128MathILib INTERFACE ${sources})
   else()
-    message(FATAL_ERROR "FortranRuntime cannot build without libm")
+    message(FATAL_ERROR "flang_rt.quadmath cannot build without libm")
   endif()
 else()
   # We can use '__float128' version from libc, if it has them.
diff --git a/flang/runtime/ISO_Fortran_binding.cpp b/flang/runtime/ISO_Fortran_binding.cpp
index fe22026f31f55..64e239f498230 100644
--- a/flang/runtime/ISO_Fortran_binding.cpp
+++ b/flang/runtime/ISO_Fortran_binding.cpp
@@ -11,7 +11,7 @@
 
 #include "ISO_Fortran_util.h"
 #include "terminator.h"
-#include "flang/ISO_Fortran_binding_wrapper.h"
+#include "flang/Common/ISO_Fortran_binding_wrapper.h"
 #include "flang/Runtime/descriptor.h"
 #include "flang/Runtime/pointer.h"
 #include "flang/Runtime/type-code.h"
diff --git a/flang/runtime/ISO_Fortran_util.h b/flang/runtime/ISO_Fortran_util.h
index dd0eeef80bb89..aca9aee8c5718 100644
--- a/flang/runtime/ISO_Fortran_util.h
+++ b/flang/runtime/ISO_Fortran_util.h
@@ -12,7 +12,7 @@
 // Internal utils for establishing CFI_cdesc_t descriptors.
 
 #include "terminator.h"
-#include "flang/ISO_Fortran_binding_wrapper.h"
+#include "flang/Common/ISO_Fortran_binding_wrapper.h"
 #include "flang/Runtime/descriptor.h"
 #include "flang/Runtime/type-code.h"
 #include <cstdlib>
diff --git a/flang/runtime/allocatable.cpp b/flang/runtime/allocatable.cpp
index 5e065f47636a8..686114bf86eaf 100644
--- a/flang/runtime/allocatable.cpp
+++ b/flang/runtime/allocatable.cpp
@@ -12,7 +12,7 @@
 #include "stat.h"
 #include "terminator.h"
 #include "type-info.h"
-#include "flang/ISO_Fortran_binding_wrapper.h"
+#include "flang/Common/ISO_Fortran_binding_wrapper.h"
 #include "flang/Runtime/assign.h"
 #include "flang/Runtime/descriptor.h"
 
diff --git a/flang/runtime/environment.cpp b/flang/runtime/environment.cpp
index 52b1d99ba536e..678d8745c9fd7 100644
--- a/flang/runtime/environment.cpp
+++ b/flang/runtime/environment.cpp
@@ -143,6 +143,18 @@ void ExecutionEnvironment::Configure(int ac, const char *av[],
     }
   }
 
+  if (auto *x{std::getenv("ACC_OFFLOAD_STACK_SIZE")}) {
+    char *end;
+    auto n{std::strtoul(x, &end, 10)};
+    if (n > 0 && n < std::numeric_limits<std::size_t>::max() && *end == '\0') {
+      cudaStackLimit = n;
+    } else {
+      std::fprintf(stderr,
+          "Fortran runtime: ACC_OFFLOAD_STACK_SIZE=%s is invalid; ignored\n",
+          x);
+    }
+  }
+
   // TODO: Set RP/ROUND='PROCESSOR_DEFINED' from environment
 }
 
diff --git a/flang/runtime/environment.h b/flang/runtime/environment.h
index b8b9f10e4e57f..500aa925a625b 100644
--- a/flang/runtime/environment.h
+++ b/flang/runtime/environment.h
@@ -56,6 +56,9 @@ struct ExecutionEnvironment {
   bool noStopMessage{false}; // NO_STOP_MESSAGE=1 inhibits "Fortran STOP"
   bool defaultUTF8{false}; // DEFAULT_UTF8
   bool checkPointerDeallocation{true}; // FORT_CHECK_POINTER_DEALLOCATION
+
+  // CUDA related variables
+  std::size_t cudaStackLimit{0}; // ACC_OFFLOAD_STACK_SIZE
 };
 
 RT_OFFLOAD_VAR_GROUP_BEGIN
diff --git a/flang/runtime/exceptions.cpp b/flang/runtime/exceptions.cpp
index f541b8e844ade..344e7216cfaae 100644
--- a/flang/runtime/exceptions.cpp
+++ b/flang/runtime/exceptions.cpp
@@ -11,11 +11,16 @@
 #include "flang/Runtime/exceptions.h"
 #include "terminator.h"
 #include <cfenv>
-#if __x86_64__
+#if defined(__aarch64__) && defined(__GLIBC__)
+#include <fpu_control.h>
+#elif defined(__x86_64__) && !defined(_WIN32)
 #include <xmmintrin.h>
 #endif
 
-// fenv.h may not define exception macros.
+// File fenv.h usually, but not always, defines standard exceptions as both
+// enumerator values and preprocessor #defines. Some x86 environments also
+// define a nonstandard __FE_DENORM enumerator, but without a corresponding
+// #define, which makes it more difficult to determine if it is present or not.
 #ifndef FE_INVALID
 #define FE_INVALID 0
 #endif
@@ -31,6 +36,12 @@
 #ifndef FE_INEXACT
 #define FE_INEXACT 0
 #endif
+#if FE_INVALID == 1 && FE_DIVBYZERO == 4 && FE_OVERFLOW == 8 && \
+    FE_UNDERFLOW == 16 && FE_INEXACT == 32
+#define __FE_DENORM 2
+#else
+#define __FE_DENORM 0
+#endif
 
 namespace Fortran::runtime {
 
@@ -42,11 +53,7 @@ uint32_t RTNAME(MapException)(uint32_t excepts) {
   Terminator terminator{__FILE__, __LINE__};
 
   static constexpr uint32_t v{FE_INVALID};
-#if __x86_64__
-  static constexpr uint32_t s{__FE_DENORM}; // nonstandard, not a #define
-#else
-  static constexpr uint32_t s{0};
-#endif
+  static constexpr uint32_t s{__FE_DENORM};
   static constexpr uint32_t z{FE_DIVBYZERO};
   static constexpr uint32_t o{FE_OVERFLOW};
   static constexpr uint32_t u{FE_UNDERFLOW};
@@ -90,20 +97,40 @@ bool RTNAME(SupportHalting)([[maybe_unused]] uint32_t except) {
 #endif
 }
 
+// A hardware FZ (flush to zero) bit is the negation of the
+// ieee_[get|set]_underflow_mode GRADUAL argument.
+#if defined(_MM_FLUSH_ZERO_MASK)
+// The x86_64 MXCSR FZ bit affects computations of real kinds 3, 4, and 8.
+#elif defined(_FPU_GETCW)
+// The aarch64 FPCR FZ bit affects computations of real kinds 3, 4, and 8.
+// bit 24: FZ   -- single, double precision flush to zero bit
+// bit 19: FZ16 -- half precision flush to zero bit [not currently relevant]
+#define _FPU_FPCR_FZ_MASK_ 0x01080000
+#endif
+
 bool RTNAME(GetUnderflowMode)(void) {
-#if _MM_FLUSH_ZERO_MASK
-  // The MXCSR Flush to Zero flag is the negation of the ieee_get_underflow_mode
-  // GRADUAL argument. It affects real computations of kinds 3, 4, and 8.
+#if defined(_MM_FLUSH_ZERO_MASK)
   return _MM_GET_FLUSH_ZERO_MODE() == _MM_FLUSH_ZERO_OFF;
+#elif defined(_FPU_GETCW)
+  uint64_t fpcr;
+  _FPU_GETCW(fpcr);
+  return (fpcr & _FPU_FPCR_FZ_MASK_) == 0;
 #else
   return false;
 #endif
 }
 void RTNAME(SetUnderflowMode)(bool flag) {
-#if _MM_FLUSH_ZERO_MASK
-  // The MXCSR Flush to Zero flag is the negation of the ieee_set_underflow_mode
-  // GRADUAL argument. It affects real computations of kinds 3, 4, and 8.
+#if defined(_MM_FLUSH_ZERO_MASK)
   _MM_SET_FLUSH_ZERO_MODE(flag ? _MM_FLUSH_ZERO_OFF : _MM_FLUSH_ZERO_ON);
+#elif defined(_FPU_GETCW)
+  uint64_t fpcr;
+  _FPU_GETCW(fpcr);
+  if (flag) {
+    fpcr &= ~_FPU_FPCR_FZ_MASK_;
+  } else {
+    fpcr |= _FPU_FPCR_FZ_MASK_;
+  }
+  _FPU_SETCW(fpcr);
 #endif
 }
 
diff --git a/flang/runtime/extensions.cpp b/flang/runtime/extensions.cpp
index 50d3c72fe650d..ac19ba7b31d4c 100644
--- a/flang/runtime/extensions.cpp
+++ b/flang/runtime/extensions.cpp
@@ -51,7 +51,9 @@ inline void CtimeBuffer(char *buffer, size_t bufsize, const time_t cur_time,
 
 #ifndef _WIN32
 // posix-compliant and has getlogin_r and F_OK
-#include <unistd.h> 
+#include <unistd.h>
+#else
+#include <direct.h>
 #endif
 
 extern "C" {
@@ -248,5 +250,17 @@ std::int64_t FORTRAN_PROCEDURE_NAME(access)(const char *name,
 }
 #endif
 
+// CHDIR(DIR)
+int RTNAME(Chdir)(const char *name) {
+// chdir alias seems to be deprecated on Windows.
+#ifndef _WIN32
+  return chdir(name);
+#else
+  return _chdir(name);
+#endif
+}
+
+int FORTRAN_PROCEDURE_NAME(ierrno)() { return errno; }
+
 } // namespace Fortran::runtime
 } // extern "C"
diff --git a/flang/runtime/io-api.cpp b/flang/runtime/io-api.cpp
index 7023f61ba34de..dc3f6f87fc21b 100644
--- a/flang/runtime/io-api.cpp
+++ b/flang/runtime/io-api.cpp
@@ -770,18 +770,18 @@ bool IODEF(SetAsynchronous)(
           "SetAsynchronous() called after GetNewUnit() for an OPEN statement");
     }
     open->unit().set_mayAsynchronous(isYes);
+  } else if (!isYes) {
+    // ASYNCHRONOUS='NO' is the default, so this is a no-op
   } else if (auto *ext{io.get_if<ExternalIoStatementBase>()}) {
-    if (isYes) {
-      if (ext->unit().mayAsynchronous()) {
-        ext->SetAsynchronous();
-      } else {
-        handler.SignalError(IostatBadAsynchronous);
-      }
+    if (ext->unit().mayAsynchronous()) {
+      ext->SetAsynchronous();
+    } else {
+      handler.SignalError(IostatBadAsynchronous);
     }
   } else if (!io.get_if<NoopStatementState>() &&
       !io.get_if<ErroneousIoStatementState>()) {
-    handler.Crash("SetAsynchronous() called when not in an OPEN or external "
-                  "I/O statement");
+    handler.Crash("SetAsynchronous('YES') called when not in an OPEN or "
+                  "external I/O statement");
   }
   return !handler.InError();
 }
@@ -887,7 +887,7 @@ bool IODEF(SetForm)(Cookie cookie, const char *keyword, std::size_t length) {
     io.GetIoErrorHandler().Crash(
         "SetForm() called after GetNewUnit() for an OPEN statement");
   }
-  static const char *keywords[]{"FORMATTED", "UNFORMATTED", nullptr};
+  static const char *keywords[]{"FORMATTED", "UNFORMATTED", "BINARY", nullptr};
   switch (IdentifyValue(keyword, length, keywords)) {
   case 0:
     open->set_isUnformatted(false);
@@ -895,6 +895,10 @@ bool IODEF(SetForm)(Cookie cookie, const char *keyword, std::size_t length) {
   case 1:
     open->set_isUnformatted(true);
     break;
+  case 2: // legacy FORM='BINARY' means an unformatted stream
+    open->set_isUnformatted(true);
+    open->set_access(Access::Stream);
+    break;
   default:
     open->SignalError(IostatErrorInKeyword, "Invalid FORM='%.*s'",
         static_cast<int>(length), keyword);
diff --git a/flang/runtime/stat.h b/flang/runtime/stat.h
index 71faeb027d908..572cb6d10b489 100644
--- a/flang/runtime/stat.h
+++ b/flang/runtime/stat.h
@@ -11,8 +11,8 @@
 
 #ifndef FORTRAN_RUNTIME_STAT_H_
 #define FORTRAN_RUNTIME_STAT_H_
+#include "flang/Common/ISO_Fortran_binding_wrapper.h"
 #include "flang/Common/api-attrs.h"
-#include "flang/ISO_Fortran_binding_wrapper.h"
 #include "flang/Runtime/magic-numbers.h"
 namespace Fortran::runtime {
 
diff --git a/flang/runtime/temporary-stack.cpp b/flang/runtime/temporary-stack.cpp
index 667b10e04dbd2..93340266b1b44 100644
--- a/flang/runtime/temporary-stack.cpp
+++ b/flang/runtime/temporary-stack.cpp
@@ -11,7 +11,7 @@
 
 #include "flang/Runtime/temporary-stack.h"
 #include "terminator.h"
-#include "flang/ISO_Fortran_binding_wrapper.h"
+#include "flang/Common/ISO_Fortran_binding_wrapper.h"
 #include "flang/Runtime/assign.h"
 #include "flang/Runtime/descriptor.h"
 #include "flang/Runtime/memory.h"
diff --git a/flang/runtime/time-intrinsic.cpp b/flang/runtime/time-intrinsic.cpp
index e6f6e81c7b50c..942604a92aaad 100644
--- a/flang/runtime/time-intrinsic.cpp
+++ b/flang/runtime/time-intrinsic.cpp
@@ -62,7 +62,7 @@ template <typename Unused = void> double GetCpuTime(fallback_implementation) {
 
 #if defined __MINGW32__
 // clock_gettime is implemented in the pthread library for MinGW.
-// Using it here would mean that all programs that link libFortranRuntime are
+// Using it here would mean that all programs that link libflang_rt are
 // required to also link to pthread. Instead, don't use the function.
 #undef CLOCKID_CPU_TIME
 #undef CLOCKID_ELAPSED_TIME
diff --git a/flang/runtime/tools.h b/flang/runtime/tools.h
index facbd23161057..75544098d47ab 100644
--- a/flang/runtime/tools.h
+++ b/flang/runtime/tools.h
@@ -348,7 +348,7 @@ inline RT_API_ATTRS RESULT ApplyFloatingPointKind(
     if constexpr (HasCppTypeFor<TypeCategory::Real, 16>) {
       // If FUNC implemenation relies on FP math functions,
       // then we should not be here. The compiler should have
-      // generated a call to an entry in FortranFloat128Math
+      // generated a call to an entry in flang_rt.quadmath
       // library.
       if constexpr (!NEEDSMATH) {
         return FUNC<16>{}(std::forward<A>(x)...);
diff --git a/flang/test/Analysis/AliasAnalysis/alias-analysis-2.fir b/flang/test/Analysis/AliasAnalysis/alias-analysis-2.fir
index d03348efd2a68..24cfaf6ed7ecc 100644
--- a/flang/test/Analysis/AliasAnalysis/alias-analysis-2.fir
+++ b/flang/test/Analysis/AliasAnalysis/alias-analysis-2.fir
@@ -47,6 +47,16 @@
 // CHECK-DAG: arg2.load#0 <-> arg2.addr#0: MustAlias
 // CHECK-DAG: boxp1.addr#0 <-> arg2.addr#0: MayAlias
 
+// TODO: Can the address in a pointer alias the address of a pointer, when the
+// pointer has no box. Should this be NoAlias?
+// T3 from <https://github.com/llvm/llvm-project/pull/117785#discussion_r1924348480>.
+// CHECK-DAG: p1.addr#0 <-> p1.tgt#0: MayAlias
+
+// The addresses stored in two different pointers can alias, even if one has no
+// box.  In this program, they happen to be the same address.
+// T4: 
+// CHECK-DAG: p1.tgt#0 <-> boxp1.addr#0: MayAlias
+
 func.func @_QFPtest(%arg0: !fir.ref<f32> {fir.bindc_name = "v1", fir.target}, %arg1: !fir.ref<f32> {fir.bindc_name = "v2", fir.target}, %arg2: !fir.ref<!fir.box<!fir.ptr<f32>>> ) attributes {test.ptr = "func"} {
 
   %1 = fir.alloca !fir.ptr<f32> {test.ptr = "p1.addr"}
@@ -63,7 +73,7 @@ func.func @_QFPtest(%arg0: !fir.ref<f32> {fir.bindc_name = "v1", fir.target}, %a
   fir.store %6 to %4 : !fir.ref<!fir.ptr<f32>>
 
   %0 = fir.alloca !fir.box<!fir.ptr<f32>> {bindc_name = "p1", uniq_name = "_QFtestEp1"}
-  %7 = fir.load %1 : !fir.ref<!fir.ptr<f32>>
+  %7 = fir.load %1 {test.ptr="p1.tgt"} : !fir.ref<!fir.ptr<f32>>
   %8 = fir.embox %7 : (!fir.ptr<f32>) -> !fir.box<!fir.ptr<f32>>
   fir.store %8 to %0 : !fir.ref<!fir.box<!fir.ptr<f32>>>
 
diff --git a/flang/test/Analysis/AliasAnalysis/alias-analysis-omp-private-allocatable.mlir b/flang/test/Analysis/AliasAnalysis/alias-analysis-omp-private-allocatable.mlir
index 5116622364fad..e19885c71a9f8 100644
--- a/flang/test/Analysis/AliasAnalysis/alias-analysis-omp-private-allocatable.mlir
+++ b/flang/test/Analysis/AliasAnalysis/alias-analysis-omp-private-allocatable.mlir
@@ -20,15 +20,7 @@
 // CHECK: ar2#0 <-> ar1#1: NoAlias
 // CHECK: ar2#1 <-> ar1#1: NoAlias
 
-omp.private {type = private} @_QFmysubEar1_private_ref_box_heap_Uxf64 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xf64>>>> alloc {
-^bb0(%arg0: !fir.ref<!fir.box<!fir.heap<!fir.array<?xf64>>>>):
-  %0 = fir.alloca !fir.box<!fir.heap<!fir.array<?xf64>>> {bindc_name = "ar1", pinned, uniq_name = "_QFmysubEar1"}
-  %5:2 = hlfir.declare %0 {fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "_QFmysubEar1"} : (!fir.ref<!fir.box<!fir.heap<!fir.array<?xf64>>>>) -> (!fir.ref<!fir.box<!fir.heap<!fir.array<?xf64>>>>, !fir.ref<!fir.box<!fir.heap<!fir.array<?xf64>>>>)
-  omp.yield(%5#0 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xf64>>>>)
-} dealloc {
-^bb0(%arg0: !fir.ref<!fir.box<!fir.heap<!fir.array<?xf64>>>>):
-  omp.yield
-}
+omp.private {type = private} @_QFmysubEar1_private_ref_box_heap_Uxf64 : !fir.box<!fir.heap<!fir.array<?xf64>>>
 func.func @testPrivateAllocatable(%arg0: !fir.ref<i32> {fir.bindc_name = "ns"}, %arg1: !fir.ref<i32> {fir.bindc_name = "ne"}) {
   %0 = fir.dummy_scope : !fir.dscope
   %1 = fir.alloca !fir.box<!fir.heap<!fir.array<?xf64>>> {bindc_name = "ar1", uniq_name = "_QFmysubEar1"}
diff --git a/flang/test/Analysis/AliasAnalysis/alias-analysis-omp-teams-distribute-private-ptr.mlir b/flang/test/Analysis/AliasAnalysis/alias-analysis-omp-teams-distribute-private-ptr.mlir
index 78207d21c45bf..b60fbe4152fc1 100644
--- a/flang/test/Analysis/AliasAnalysis/alias-analysis-omp-teams-distribute-private-ptr.mlir
+++ b/flang/test/Analysis/AliasAnalysis/alias-analysis-omp-teams-distribute-private-ptr.mlir
@@ -17,18 +17,8 @@
 // CHECK-LABEL: Testing : "_QQmain"
 // CHECK-DAG:   ptrA#0 <-> ArrayA#0: MayAlias
 
-omp.private {type = private} @_QFEi_private_ref_i32 : !fir.ref<i32> alloc {
-^bb0(%arg0: !fir.ref<i32>):
-  %0 = fir.alloca i32 {bindc_name = "i", pinned, uniq_name = "_QFEi"}
-  %1:2 = hlfir.declare %0 {uniq_name = "_QFEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-  omp.yield(%1#0 : !fir.ref<i32>)
-}
-omp.private {type = firstprivate} @_QFEptra_firstprivate_ref_box_ptr_Uxi32 : !fir.ref<!fir.box<!fir.ptr<!fir.array<?xi32>>>> alloc {
-^bb0(%arg0: !fir.ref<!fir.box<!fir.ptr<!fir.array<?xi32>>>>):
-  %0 = fir.alloca !fir.box<!fir.ptr<!fir.array<?xi32>>> {bindc_name = "ptra", pinned, uniq_name = "_QFEptra"}
-  %1:2 = hlfir.declare %0 {fortran_attrs = #fir.var_attrs<pointer>, uniq_name = "_QFEptra"} : (!fir.ref<!fir.box<!fir.ptr<!fir.array<?xi32>>>>) -> (!fir.ref<!fir.box<!fir.ptr<!fir.array<?xi32>>>>, !fir.ref<!fir.box<!fir.ptr<!fir.array<?xi32>>>>)
-  omp.yield(%1#0 : !fir.ref<!fir.box<!fir.ptr<!fir.array<?xi32>>>>)
-} copy {
+omp.private {type = private} @_QFEi_private_ref_i32 : i32
+omp.private {type = firstprivate} @_QFEptra_firstprivate_ref_box_ptr_Uxi32 : !fir.box<!fir.ptr<!fir.array<?xi32>>> copy {
 ^bb0(%arg0: !fir.ref<!fir.box<!fir.ptr<!fir.array<?xi32>>>>, %arg1: !fir.ref<!fir.box<!fir.ptr<!fir.array<?xi32>>>>):
   %0 = fir.load %arg0 : !fir.ref<!fir.box<!fir.ptr<!fir.array<?xi32>>>>
   fir.store %0 to %arg1 : !fir.ref<!fir.box<!fir.ptr<!fir.array<?xi32>>>>
diff --git a/flang/test/Analysis/AliasAnalysis/alias-analysis-omp-teams-distribute-private.mlir b/flang/test/Analysis/AliasAnalysis/alias-analysis-omp-teams-distribute-private.mlir
index 4668b2c215c8c..7f60a6fa0803a 100644
--- a/flang/test/Analysis/AliasAnalysis/alias-analysis-omp-teams-distribute-private.mlir
+++ b/flang/test/Analysis/AliasAnalysis/alias-analysis-omp-teams-distribute-private.mlir
@@ -21,26 +21,10 @@
 // CHECK-DAG: tmp_private_array#0 <-> unnamed_array#0: NoAlias
 // CHECK-DAG: tmp_private_array#1 <-> unnamed_array#0: NoAlias
 
-omp.private {type = private} @_QFEi_private_ref_i32 : !fir.ref<i32> alloc {
-^bb0(%arg0: !fir.ref<i32>):
-  %0 = fir.alloca i32 {bindc_name = "i", pinned, uniq_name = "_QFEi"}
-  %1:2 = hlfir.declare %0 {uniq_name = "_QFEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-  omp.yield(%1#0 : !fir.ref<i32>)
-}
-omp.private {type = private} @_QFEj_private_ref_i32 : !fir.ref<i32> alloc {
-^bb0(%arg0: !fir.ref<i32>):
-  %0 = fir.alloca i32 {bindc_name = "j", pinned, uniq_name = "_QFEj"}
-  %1:2 = hlfir.declare %0 {uniq_name = "_QFEj"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-  omp.yield(%1#0 : !fir.ref<i32>)
-}
-omp.private {type = private} @_QFEtmp_private_ref_2xi32 : !fir.ref<!fir.array<2xi32>> alloc {
-^bb0(%arg0: !fir.ref<!fir.array<2xi32>>):
-  %c2 = arith.constant 2 : index
-  %0 = fir.alloca !fir.array<2xi32> {bindc_name = "tmp", pinned, uniq_name = "_QFEtmp"}
-  %1 = fir.shape %c2 : (index) -> !fir.shape<1>
-  %2:2 = hlfir.declare %0(%1) {uniq_name = "_QFEtmp"} : (!fir.ref<!fir.array<2xi32>>, !fir.shape<1>) -> (!fir.ref<!fir.array<2xi32>>, !fir.ref<!fir.array<2xi32>>)
-  omp.yield(%2#0 : !fir.ref<!fir.array<2xi32>>)
-}
+omp.private {type = private} @_QFEi_private_ref_i32 : i32
+omp.private {type = private} @_QFEj_private_ref_i32 : i32
+omp.private {type = private} @_QFEtmp_private_ref_2xi32 : !fir.array<2xi32>
+
 func.func @_QQmain() attributes {fir.bindc_name = "main"} {
   %0 = fir.address_of(@_QFEarraya) : !fir.ref<!fir.array<10x10xi32>>
   %c10 = arith.constant 10 : index
diff --git a/flang/test/Analysis/AliasAnalysis/alias-analysis-target.fir b/flang/test/Analysis/AliasAnalysis/alias-analysis-target.fir
new file mode 100644
index 0000000000000..8e88b508d56e3
--- /dev/null
+++ b/flang/test/Analysis/AliasAnalysis/alias-analysis-target.fir
@@ -0,0 +1,82 @@
+// RUN: fir-opt %s -pass-pipeline='builtin.module(func.func(test-fir-alias-analysis))'  2>&1 | FileCheck %s
+
+// The test was obtained from 
+// bbc test.f90 -emit-fir
+// module mod
+//   real, pointer :: p0
+//   real, allocatable :: alloc
+//   real, allocatable, target :: t_alloc
+//   real, target :: t
+//   real :: v
+// end module
+// 
+// subroutine test(n)
+//   use mod
+//   integer :: n
+//   real r1
+//   p0 => t_alloc
+//   v = alloc
+//   r1 = p0
+// end subroutine test
+
+// Checking that aliasing can only happen with an entity with the target attribute
+// 
+// CHECK-DAG: r1#0 <-> t_alloc#0: NoAlias
+// CHECK-DAG: r1#0 <-> alloc#0: NoAlias
+// CHECK-DAG: t_alloc#0 <-> alloc#0: NoAlias
+// CHECK-DAG: r1#0 <-> p0.ptr#0: NoAlias
+// CHECK-DAG: t_alloc#0 <-> p0.ptr#0: MayAlias
+// CHECK-DAG: alloc#0 <-> p0.ptr#0: NoAlias
+
+fir.global @_QMmodEalloc : !fir.box<!fir.heap<f32>> {
+  %0 = fir.zero_bits !fir.heap<f32>
+  %1 = fir.embox %0 : (!fir.heap<f32>) -> !fir.box<!fir.heap<f32>>
+  fir.has_value %1 : !fir.box<!fir.heap<f32>>
+}
+fir.global @_QMmodEp0 : !fir.box<!fir.ptr<f32>> {
+  %0 = fir.zero_bits !fir.ptr<f32>
+  %1 = fir.embox %0 : (!fir.ptr<f32>) -> !fir.box<!fir.ptr<f32>>
+  fir.has_value %1 : !fir.box<!fir.ptr<f32>>
+}
+fir.global @_QMmodEt target : f32 {
+  %0 = fir.zero_bits f32
+  fir.has_value %0 : f32
+}
+fir.global @_QMmodEt_alloc target : !fir.box<!fir.heap<f32>> {
+  %0 = fir.zero_bits !fir.heap<f32>
+  %1 = fir.embox %0 : (!fir.heap<f32>) -> !fir.box<!fir.heap<f32>>
+  fir.has_value %1 : !fir.box<!fir.heap<f32>>
+}
+fir.global @_QMmodEv : f32 {
+  %0 = fir.zero_bits f32
+  fir.has_value %0 : f32
+}
+func.func @_QPtest(%arg0: !fir.ref<i32> {fir.bindc_name = "n"}) {
+  %0 = fir.dummy_scope : !fir.dscope
+  %1 = fir.address_of(@_QMmodEalloc) : !fir.ref<!fir.box<!fir.heap<f32>>>
+  %2 = fir.declare %1 {fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "_QMmodEalloc"} : (!fir.ref<!fir.box<!fir.heap<f32>>>) -> !fir.ref<!fir.box<!fir.heap<f32>>>
+  %3 = fir.declare %arg0 dummy_scope %0 {uniq_name = "_QFtestEn"} : (!fir.ref<i32>, !fir.dscope) -> !fir.ref<i32>
+  %4 = fir.address_of(@_QMmodEp0) : !fir.ref<!fir.box<!fir.ptr<f32>>>
+  %5 = fir.declare %4 {fortran_attrs = #fir.var_attrs<pointer>, uniq_name = "_QMmodEp0"} : (!fir.ref<!fir.box<!fir.ptr<f32>>>) -> !fir.ref<!fir.box<!fir.ptr<f32>>>
+  %6 = fir.alloca f32 {bindc_name = "r1", uniq_name = "_QFtestEr1"}
+  %7 = fir.declare %6 {test.ptr="r1", uniq_name = "_QFtestEr1"} : (!fir.ref<f32>) -> !fir.ref<f32>
+  %8 = fir.address_of(@_QMmodEt) : !fir.ref<f32>
+  %9 = fir.declare %8 {fortran_attrs = #fir.var_attrs<target>, uniq_name = "_QMmodEt"} : (!fir.ref<f32>) -> !fir.ref<f32>
+  %10 = fir.address_of(@_QMmodEt_alloc) : !fir.ref<!fir.box<!fir.heap<f32>>>
+  %11 = fir.declare %10 {fortran_attrs = #fir.var_attrs<allocatable, target>, uniq_name = "_QMmodEt_alloc"} : (!fir.ref<!fir.box<!fir.heap<f32>>>) -> !fir.ref<!fir.box<!fir.heap<f32>>>
+  %12 = fir.address_of(@_QMmodEv) : !fir.ref<f32>
+  %13 = fir.declare %12 {uniq_name = "_QMmodEv"} : (!fir.ref<f32>) -> !fir.ref<f32>
+  %14 = fir.load %11 : !fir.ref<!fir.box<!fir.heap<f32>>>
+  %15 = fir.box_addr %14 {test.ptr="t_alloc"}: (!fir.box<!fir.heap<f32>>) -> !fir.heap<f32>
+  %16 = fir.embox %15 : (!fir.heap<f32>) -> !fir.box<!fir.ptr<f32>>
+  fir.store %16 to %5 : !fir.ref<!fir.box<!fir.ptr<f32>>>
+  %17 = fir.load %2 : !fir.ref<!fir.box<!fir.heap<f32>>>
+  %18 = fir.box_addr %17 {test.ptr="alloc"} : (!fir.box<!fir.heap<f32>>) -> !fir.heap<f32>
+  %19 = fir.load %18 : !fir.heap<f32>
+  fir.store %19 to %13 : !fir.ref<f32>
+  %20 = fir.load %5 : !fir.ref<!fir.box<!fir.ptr<f32>>>
+  %21 = fir.box_addr %20 {test.ptr="p0.ptr"} : (!fir.box<!fir.ptr<f32>>) -> !fir.ptr<f32>
+  %22 = fir.load %21 : !fir.ptr<f32>
+  fir.store %22 to %7 : !fir.ref<f32>
+  return
+}
diff --git a/flang/test/CMakeLists.txt b/flang/test/CMakeLists.txt
index cab214c2ef4c8..3fac8717e9bd9 100644
--- a/flang/test/CMakeLists.txt
+++ b/flang/test/CMakeLists.txt
@@ -71,9 +71,13 @@ set(FLANG_TEST_DEPENDS
   llvm-objdump
   llvm-readobj
   split-file
-  FortranRuntime
   FortranDecimal
 )
+
+if (FLANG_INCLUDE_RUNTIME)
+  list(APPEND FLANG_TEST_DEPENDS flang_rt.runtime)
+endif ()
+
 if (LLVM_ENABLE_PLUGINS AND NOT WIN32)
   list(APPEND FLANG_TEST_DEPENDS Bye)
 endif()
@@ -120,3 +124,9 @@ if (DEFINED FLANG_TEST_TARGET_TRIPLE)
                         "to use FLANG_TEST_TARGET_TRIPLE.")
   endif()
 endif()
+
+# Compatibility targets.
+if (FLANG_INCLUDE_RUNTIME)
+  add_custom_target(check-flang-rt)
+  add_dependencies(check-flang-rt check-flang)
+endif ()
diff --git a/flang/test/Driver/ctofortran.f90 b/flang/test/Driver/ctofortran.f90
index 78eac32133b18..10c7adaccc958 100644
--- a/flang/test/Driver/ctofortran.f90
+++ b/flang/test/Driver/ctofortran.f90
@@ -1,4 +1,5 @@
 ! UNSUPPORTED: system-windows
+! REQUIRES: flang-rt
 ! RUN: split-file %s %t
 ! RUN: chmod +x %t/runtest.sh
 ! RUN: %t/runtest.sh %t %t/ffile.f90 %t/cfile.c %flang | FileCheck %s
diff --git a/flang/test/Driver/dump-pft.f90 b/flang/test/Driver/dump-pft.f90
index 6d98fc9f10b89..5e940c769449f 100644
--- a/flang/test/Driver/dump-pft.f90
+++ b/flang/test/Driver/dump-pft.f90
@@ -4,8 +4,8 @@
 
 ! PFT: 1 Subroutine test_routine: subroutine test_routine(a, b, n)
 ! PFT-NEXT:  1 EndSubroutineStmt: end subroutine
-! PRF-NEXT: End Subroutine test_routine
-! PFT-NO: Program -> ProgramUnit -> SubroutineSubprogram
+! PFT-NEXT: End Subroutine test_routine
+! PFT-NOT: Program -> ProgramUnit -> SubroutineSubprogram
 
 ! PARSE_TREE: Program -> ProgramUnit -> SubroutineSubprogram
 ! PARSE_TREE-NEXT: | SubroutineStmt
@@ -17,7 +17,7 @@
 ! PARSE_TREE-NEXT: | | ImplicitPart ->
 ! PARSE_TREE-NEXT: | ExecutionPart -> Block
 ! PARSE_TREE-NEXT: | EndSubroutineStmt ->
-! PARSE_TREE-NO: Subroutine test_routine: subroutine test_routine(a, b, n)
+! PARSE_TREE-NOT: Subroutine test_routine: subroutine test_routine(a, b, n)
 
 subroutine test_routine(a, b, n)
 end subroutine
diff --git a/flang/test/Driver/exec.f90 b/flang/test/Driver/exec.f90
index fd174005ddf62..9ca91ee24011c 100644
--- a/flang/test/Driver/exec.f90
+++ b/flang/test/Driver/exec.f90
@@ -1,4 +1,5 @@
 ! UNSUPPORTED: system-windows
+! REQUIRES: flang-rt
 ! Verify that flang can correctly build executables.
 
 ! RUN: %flang %s -o %t
diff --git a/flang/test/Driver/fsave-main-program.f90 b/flang/test/Driver/fsave-main-program.f90
index bffdfd97911e8..e7a2f9d8b470e 100644
--- a/flang/test/Driver/fsave-main-program.f90
+++ b/flang/test/Driver/fsave-main-program.f90
@@ -1,5 +1,9 @@
 ! Check that the driver passes through -fsave-main-program:
 ! RUN: %flang -### -S -fsave-main-program %s -o - 2>&1 | FileCheck %s
+! CHECK: "-fc1"{{.*}}"-fsave-main-program"
+
+! RUN: %flang -### -S -fno-save-main-program %s -o - 2>&1 | FileCheck %s --check-prefix=CHECK2
+! CHECK2: "-fc1"{{.*}}"-fno-save-main-program"
+
 ! Check that the compiler accepts -fsave-main-program:
 ! RUN: %flang_fc1 -emit-hlfir -fsave-main-program %s -o -
-! CHECK: "-fc1"{{.*}}"-fsave-main-program"
diff --git a/flang/test/Driver/func-attr-fast-math.f90 b/flang/test/Driver/func-attr-fast-math.f90
index 05824a6078a09..c21f385fe690f 100644
--- a/flang/test/Driver/func-attr-fast-math.f90
+++ b/flang/test/Driver/func-attr-fast-math.f90
@@ -7,7 +7,7 @@ end subroutine func
 
 ! CHECK-NOFASTMATH-LABEL: define void @func_() local_unnamed_addr
 ! CHECK-NOFASTMATH-SAME: #[[ATTRS:[0-9]+]]
-! CHECK-NOT fp-math"=
+! CHECK-NOT: fp-math"=
 
 ! CHECK-OFAST-LABEL: define void @func_() local_unnamed_addr
 ! CHECK-OFAST-SAME: #[[ATTRS:[0-9]+]]
diff --git a/flang/test/Driver/gcc-toolchain-install-dir.f90 b/flang/test/Driver/gcc-toolchain-install-dir.f90
index 5a073b0c51712..e195bdde6d2c9 100644
--- a/flang/test/Driver/gcc-toolchain-install-dir.f90
+++ b/flang/test/Driver/gcc-toolchain-install-dir.f90
@@ -1,5 +1,5 @@
 !! Test that --gcc-toolchain and --gcc-install-dir options are working as expected.
-!! It does not test cross-compiling (--sysroot), so crtbegin.o, libgcc/compiler-rt, libc, libFortranRuntime, etc. are not supposed to be affected.
+!! It does not test cross-compiling (--sysroot), so crtbegin.o, libgcc/compiler-rt, libc, libflang_rt.runtime, etc. are not supposed to be affected.
 !! PREFIX is captured twice because the driver escapes backslashes (occuring in Windows paths) in the -### output, but not on the "Selected GCC installation:" line.
 
 ! RUN: %flang 2>&1 -### -v -o %t %s -no-integrated-as -fuse-ld=ld --target=i386-unknown-linux-gnu --gcc-install-dir=%S/Inputs/basic_cross_linux_tree/usr/lib/gcc/i386-unknown-linux-gnu/10.2.0 | FileCheck %s --check-prefix=CHECK-I386
diff --git a/flang/test/Driver/linker-flags.f90 b/flang/test/Driver/linker-flags.f90
index ac9500d7c45ce..4e62a8c32d360 100644
--- a/flang/test/Driver/linker-flags.f90
+++ b/flang/test/Driver/linker-flags.f90
@@ -5,10 +5,10 @@
 ! RUN: %flang -### --target=ppc64le-linux-gnu %S/Inputs/hello.f90 2>&1 | FileCheck %s --check-prefixes=CHECK,UNIX,UNIX-F128%f128-lib
 ! RUN: %flang -### --target=aarch64-apple-darwin %S/Inputs/hello.f90 2>&1 | FileCheck %s --check-prefixes=CHECK,DARWIN,DARWIN-F128%f128-lib
 ! RUN: %flang -### --target=sparc-sun-solaris2.11 %S/Inputs/hello.f90 2>&1 | FileCheck %s --check-prefixes=CHECK,UNIX,SOLARIS-F128%f128-lib
-! RUN: %flang -### --target=x86_64-unknown-freebsd %S/Inputs/hello.f90 2>&1 | FileCheck %s --check-prefixes=CHECK,UNIX,UNIX-F128%f128-lib
-! RUN: %flang -### --target=x86_64-unknown-netbsd %S/Inputs/hello.f90 2>&1 | FileCheck %s --check-prefixes=CHECK,UNIX,UNIX-F128%f128-lib
-! RUN: %flang -### --target=x86_64-unknown-openbsd %S/Inputs/hello.f90 2>&1 | FileCheck %s --check-prefixes=CHECK,UNIX,UNIX-F128%f128-lib
-! RUN: %flang -### --target=x86_64-unknown-dragonfly %S/Inputs/hello.f90 2>&1 | FileCheck %s --check-prefixes=CHECK,UNIX,UNIX-F128%f128-lib
+! RUN: %flang -### --target=x86_64-unknown-freebsd %S/Inputs/hello.f90 2>&1 | FileCheck %s --check-prefixes=CHECK,BSD,BSD-F128%f128-lib
+! RUN: %flang -### --target=x86_64-unknown-netbsd %S/Inputs/hello.f90 2>&1 | FileCheck %s --check-prefixes=CHECK,BSD,BSD-F128%f128-lib
+! RUN: %flang -### --target=x86_64-unknown-openbsd %S/Inputs/hello.f90 2>&1 | FileCheck %s --check-prefixes=CHECK,BSD,BSD-F128%f128-lib
+! RUN: %flang -### --target=x86_64-unknown-dragonfly %S/Inputs/hello.f90 2>&1 | FileCheck %s --check-prefixes=CHECK,BSD,BSD-F128%f128-lib
 ! RUN: %flang -### --target=x86_64-unknown-haiku %S/Inputs/hello.f90 2>&1 | FileCheck %s --check-prefixes=CHECK,HAIKU,HAIKU-F128%f128-lib
 ! RUN: %flang -### --target=x86_64-windows-gnu %S/Inputs/hello.f90 2>&1 | FileCheck %s --check-prefixes=CHECK,MINGW,MINGW-F128%f128-lib
 ! RUN: %flang -### -rtlib=compiler-rt --target=aarch64-linux-gnu %S/Inputs/hello.f90 2>&1 | FileCheck %s --check-prefixes=CHECK,UNIX,COMPILER-RT
@@ -29,32 +29,37 @@
 !       executable and may find the GNU linker from MinGW or Cygwin.
 ! UNIX-LABEL:  "{{.*}}ld{{(\.exe)?}}"
 ! UNIX-SAME: "[[object_file]]"
-! UNIX-F128NONE-NOT: FortranFloat128Math
-! SOLARIS-F128NONE-NOT: FortranFloat128Math
-! UNIX-F128LIBQUADMATH-SAME: "-lFortranFloat128Math" "--as-needed" "-lquadmath" "--no-as-needed"
-! SOLARIS-F128LIBQUADMATH-SAME: "-lFortranFloat128Math" "-z" "ignore" "-lquadmath" "-z" "record"
-! UNIX-SAME: "-lFortranRuntime" "-lFortranDecimal" "-lm"
+! UNIX-F128NONE-NOT: lang_rt.quadmath
+! SOLARIS-F128NONE-NOT: flang_rt.quadmath
+! UNIX-F128LIBQUADMATH-SAME: "-lflang_rt.quadmath" "--as-needed" "-lquadmath" "--no-as-needed"
+! SOLARIS-F128LIBQUADMATH-SAME: "-lflang_rt.quadmath" "-z" "ignore" "-lquadmath" "-z" "record"
+! UNIX-SAME: "-lflang_rt.runtime" "-lm"
 ! COMPILER-RT: "{{.*}}{{\\|/}}libclang_rt.builtins.a"
 
+! BSD-LABEL:  "{{.*}}ld{{(\.exe)?}}"
+! BSD-SAME: "[[object_file]]"
+! BSD-F128NONE-NOT: flang_rt.quadmath
+! BSD-F128LIBQUADMATH-SAME: "-lflang_rt.quadmath" "--as-needed" "-lquadmath" "--no-as-needed"
+! BSD-SAME: -lflang_rt.runtime
+! BSD-SAME: -lexecinfo
+
 ! DARWIN-LABEL:  "{{.*}}ld{{(\.exe)?}}"
 ! DARWIN-SAME: "[[object_file]]"
-! DARWIN-F128NONE-NOT: FortranFloat128Math
-! DARWIN-F128LIBQUADMATH-SAME: "-lFortranFloat128Math" "--as-needed" "-lquadmath" "--no-as-needed"
-! DARWIN-SAME: -lFortranRuntime
-! DARWIN-SAME: -lFortranDecimal
+! DARWIN-F128NONE-NOT: libflang_rt.quadmath
+! DARWIN-F128LIBQUADMATH-SAME: "-lflang_rt.quadmath" "--as-needed" "-lquadmath" "--no-as-needed"
+! DARWIN-SAME: -lflang_rt.runtime
 
 ! HAIKU-LABEL:  "{{.*}}ld{{(\.exe)?}}"
 ! HAIKU-SAME: "[[object_file]]"
-! HAIKU-F128NONE-NOT: FortranFloat128Math
-! HAIKU-F128LIBQUADMATH-SAME: "-lFortranFloat128Math" "--as-needed" "-lquadmath" "--no-as-needed"
-! HAIKU-SAME: "-lFortranRuntime" "-lFortranDecimal"
+! HAIKU-F128NONE-NOT: libflang_rt.quadmath
+! HAIKU-F128LIBQUADMATH-SAME: "-lflang_rt.quadmath" "--as-needed" "-lquadmath" "--no-as-needed"
+! HAIKU-SAME: "-lflang_rt.runtime"
 
 ! MINGW-LABEL:  "{{.*}}ld{{(\.exe)?}}"
 ! MINGW-SAME: "[[object_file]]"
-! MINGW-F128NONE-NOT: FortranFloat128Math
-! MINGW-F128LIBQUADMATH-SAME: "-lFortranFloat128Math" "--as-needed" "-lquadmath" "--no-as-needed"
-! MINGW-SAME: -lFortranRuntime
-! MINGW-SAME: -lFortranDecimal
+! MINGW-F128NONE-NOT: libflang_rt.quadmath
+! MINGW-F128LIBQUADMATH-SAME: "-lflang_rt.quadmath" "--as-needed" "-lquadmath" "--no-as-needed"
+! MINGW-SAME: -lflang_rt.runtime
 
 ! NOTE: This also matches lld-link (when CLANG_DEFAULT_LINKER=lld) and
 !       any .exe suffix that is added when resolving to the full path of
diff --git a/flang/test/Driver/msvc-dependent-lib-flags.f90 b/flang/test/Driver/msvc-dependent-lib-flags.f90
index 765917f07d8e7..641c73912c4d1 100644
--- a/flang/test/Driver/msvc-dependent-lib-flags.f90
+++ b/flang/test/Driver/msvc-dependent-lib-flags.f90
@@ -7,24 +7,21 @@
 ! MSVC-SAME: --dependent-lib=clang_rt.builtins.lib
 ! MSVC-SAME: -D_MT
 ! MSVC-SAME: --dependent-lib=libcmt
-! MSVC-SAME: --dependent-lib=FortranRuntime.static.lib
-! MSVC-SAME: --dependent-lib=FortranDecimal.static.lib
+! MSVC-SAME: --dependent-lib=flang_rt.runtime.static.lib
 
 ! MSVC-DEBUG: -fc1
 ! MSVC-DEBUG-SAME: --dependent-lib=clang_rt.builtins.lib
 ! MSVC-DEBUG-SAME: -D_MT
 ! MSVC-DEBUG-SAME: -D_DEBUG
 ! MSVC-DEBUG-SAME: --dependent-lib=libcmtd
-! MSVC-DEBUG-SAME: --dependent-lib=FortranRuntime.static_dbg.lib
-! MSVC-DEBUG-SAME: --dependent-lib=FortranDecimal.static_dbg.lib
+! MSVC-DEBUG-SAME: --dependent-lib=flang_rt.runtime.static_dbg.lib
 
 ! MSVC-DLL: -fc1
 ! MSVC-DLL-SAME: --dependent-lib=clang_rt.builtins.lib
 ! MSVC-DLL-SAME: -D_MT
 ! MSVC-DLL-SAME: -D_DLL
 ! MSVC-DLL-SAME: --dependent-lib=msvcrt
-! MSVC-DLL-SAME: --dependent-lib=FortranRuntime.dynamic.lib
-! MSVC-DLL-SAME: --dependent-lib=FortranDecimal.dynamic.lib
+! MSVC-DLL-SAME: --dependent-lib=flang_rt.runtime.dynamic.lib
 
 ! MSVC-DLL-DEBUG: -fc1
 ! MSVC-DLL-DEBUG-SAME: --dependent-lib=clang_rt.builtins.lib
@@ -32,5 +29,4 @@
 ! MSVC-DLL-DEBUG-SAME: -D_DEBUG
 ! MSVC-DLL-DEBUG-SAME: -D_DLL
 ! MSVC-DLL-DEBUG-SAME: --dependent-lib=msvcrtd
-! MSVC-DLL-DEBUG-SAME: --dependent-lib=FortranRuntime.dynamic_dbg.lib
-! MSVC-DLL-DEBUG-SAME: --dependent-lib=FortranDecimal.dynamic_dbg.lib
+! MSVC-DLL-DEBUG-SAME: --dependent-lib=flang_rt.runtime.dynamic_dbg.lib
diff --git a/flang/test/Driver/nostdlib.f90 b/flang/test/Driver/nostdlib.f90
index cd707e632a45f..dc23be6462376 100644
--- a/flang/test/Driver/nostdlib.f90
+++ b/flang/test/Driver/nostdlib.f90
@@ -24,6 +24,5 @@
 ! in certain cases. But it is not clear that it is worth checking for each
 ! platform individually.
 
-! CHECK-NOT: "-lFortranRuntime"
-! CHECK-NOT: "-lFortranDecimal"
+! CHECK-NOT: "-lflang_rt.runtime"
 ! CHECK-NOT: "-lgcc"
diff --git a/flang/test/Driver/omp-cse-region-boundary.f90 b/flang/test/Driver/omp-cse-region-boundary.f90
deleted file mode 100644
index 726ac4899ba70..0000000000000
--- a/flang/test/Driver/omp-cse-region-boundary.f90
+++ /dev/null
@@ -1,28 +0,0 @@
-!This test checks that when compiling an OpenMP program for the target device
-!CSE is not done across target op region boundaries. It also checks that when
-!compiling for the host CSE is done.
-!RUN: %flang_fc1 -fopenmp-is-target-device -emit-mlir -fopenmp %s -o - | fir-opt -cse | FileCheck %s -check-prefix=CHECK-DEVICE
-!RUN: %flang_fc1 -emit-mlir -fopenmp %s -o - | fir-opt -cse | FileCheck %s -check-prefix=CHECK-HOST
-!RUN: bbc -fopenmp-is-target-device -emit-fir -fopenmp %s -o - | fir-opt -cse | FileCheck %s -check-prefix=CHECK-DEVICE
-!RUN: bbc -emit-fir -fopenmp %s -o - | fir-opt -cse | FileCheck %s -check-prefix=CHECK-HOST
-
-!Constant should be present inside target region.
-!CHECK-DEVICE: omp.target
-!CHECK-DEVICE: arith.constant 10
-!CHECK-DEVICE: omp.terminator
-
-!Constant should not be present inside target region.
-!CHECK-HOST: omp.target
-!CHECK-NOT-HOST: arith.constant 10
-!CHECK-HOST: omp.terminator
-
-subroutine writeIndex(sum)
-        integer :: sum
-        integer :: myconst1
-        integer :: myconst2
-        myconst1 = 10
-!$omp target map(from:myconst2)
-        myconst2 = 10
-!$omp end target
-        sum = myconst2 + myconst2
-end subroutine writeIndex
diff --git a/flang/test/Driver/omp-driver-offload.f90 b/flang/test/Driver/omp-driver-offload.f90
index 7c51656f0001a..a6e3b87a605d5 100644
--- a/flang/test/Driver/omp-driver-offload.f90
+++ b/flang/test/Driver/omp-driver-offload.f90
@@ -130,13 +130,13 @@
 ! RUN: %flang -### %s -o %t 2>&1 \
 ! RUN: -fopenmp --offload-arch=gfx90a \
 ! RUN: -fopenmp-targets=amdgcn-amd-amdhsa \
-! RUN: -fopenmp-target-debug -nogpulib\
-! RUN: | FileCheck %s --check-prefixes=CHECK-TARGET-DEBUG
+! RUN: -fopenmp-target-debug=111 -nogpulib\
+! RUN: | FileCheck %s --check-prefixes=CHECK-TARGET-DEBUG-EQ
 ! RUN: %flang -### %s -o %t 2>&1 \
 ! RUN: -fopenmp --offload-arch=sm_70 \
 ! RUN: -fopenmp-targets=nvptx64-nvidia-cuda \
-! RUN: -fopenmp-target-debug \
-! RUN: | FileCheck %s --check-prefixes=CHECK-TARGET-DEBUG
+! RUN: -fopenmp-target-debug=111 \
+! RUN: | FileCheck %s --check-prefixes=CHECK-TARGET-DEBUG-EQ
 ! CHECK-TARGET-DEBUG-EQ: "{{[^"]*}}flang" "-fc1" {{.*}} "-fopenmp" {{.*}} "-fopenmp-is-target-device" "-fopenmp-target-debug=111" {{.*}}.f90"
 
 ! RUN: %flang -S -### %s -o %t 2>&1 \
diff --git a/flang/test/Evaluate/bug123766.f90 b/flang/test/Evaluate/bug123766.f90
new file mode 100644
index 0000000000000..b58989e6c26d8
--- /dev/null
+++ b/flang/test/Evaluate/bug123766.f90
@@ -0,0 +1,5 @@
+! RUN: %python %S/test_folding.py %s %flang_fc1
+character(10), parameter :: a = '0123456789'
+character(3), parameter :: arr(3) = [(a(1:i), i=1,3)]
+logical, parameter :: test1 = all(arr == ["0", "01", "012"])
+end
diff --git a/flang/test/Evaluate/bug124191.f90 b/flang/test/Evaluate/bug124191.f90
new file mode 100644
index 0000000000000..27d08032efa2f
--- /dev/null
+++ b/flang/test/Evaluate/bug124191.f90
@@ -0,0 +1,6 @@
+! RUN: %flang_fc1 -fsyntax-only -pedantic %s 2>&1 | FileCheck --allow-empty %s
+! CHECK-NOT: error:
+! Regression test for https://github.com/llvm/llvm-project/issues/124191
+character(3) :: arr(5) = ['aa.', 'bb.', 'cc.', 'dd.', 'ee.']
+arr([(mod(iachar(arr(i:i-1:-1)(1:1)),5)+1, i=2,5,3)]) = arr(5:2:-1)
+end
diff --git a/flang/test/Evaluate/bug124618.f90 b/flang/test/Evaluate/bug124618.f90
new file mode 100644
index 0000000000000..939985e588af2
--- /dev/null
+++ b/flang/test/Evaluate/bug124618.f90
@@ -0,0 +1,5 @@
+! RUN: %flang_fc1 -fsyntax-only %s 2>&1 | FileCheck --allow-empty %s
+!CHECK-NOT: error:
+real x
+print *, char(48, kind=size([x])) ! folds down to 1
+end
diff --git a/flang/test/Evaluate/errors01.f90 b/flang/test/Evaluate/errors01.f90
index 33bda3babf900..283c246393dcd 100644
--- a/flang/test/Evaluate/errors01.f90
+++ b/flang/test/Evaluate/errors01.f90
@@ -145,9 +145,9 @@ subroutine s11
   subroutine s12(x,y)
     class(t), intent(in) :: x
     class(*), intent(in) :: y
-    !CHERK: error: Must be a constant value
+    !CHECK: error: Must be a constant value
     integer, parameter :: bad1 = storage_size(x)
-    !CHERK: error: Must be a constant value
+    !CHECK: error: Must be a constant value
     integer, parameter :: bad2 = storage_size(y)
   end subroutine
   subroutine s13
diff --git a/flang/test/Evaluate/fold-arr-char-component.f90 b/flang/test/Evaluate/fold-arr-char-component.f90
new file mode 100644
index 0000000000000..9835db960d4ae
--- /dev/null
+++ b/flang/test/Evaluate/fold-arr-char-component.f90
@@ -0,0 +1,11 @@
+! RUN: %python %S/test_folding.py %s %flang_fc1
+! Ensure that array-valued component references have lengths
+! (see https://github.com/llvm/llvm-project/issues/123362)
+module m
+  type cdt
+    character(7) :: a = "ibm704", b = "cdc6600"
+  end type
+  type(cdt), parameter :: arr(2) = cdt()
+  integer, parameter :: check(*) = scan(arr%a, arr%b)
+  logical, parameter :: test1 = all(check == 5) ! the '0'
+end
diff --git a/flang/test/Evaluate/fold-ieee.f90 b/flang/test/Evaluate/fold-ieee.f90
index 99f8526fd23db..a393fcc6b4297 100644
--- a/flang/test/Evaluate/fold-ieee.f90
+++ b/flang/test/Evaluate/fold-ieee.f90
@@ -54,9 +54,6 @@ module m
   logical, parameter :: test_sq_all = ieee_support_sqrt()
   logical, parameter :: test_sq_4 = ieee_support_sqrt(1.)
   logical, parameter :: test_sq_8 = ieee_support_sqrt(1.d0)
-  logical, parameter :: test_std_all = ieee_support_standard()
-  logical, parameter :: test_std_4 = ieee_support_standard(1.)
-  logical, parameter :: test_std_8 = ieee_support_standard(1.d0)
   logical, parameter :: test_sn_all = ieee_support_subnormal()
   logical, parameter :: test_sn_4 = ieee_support_subnormal(1.)
   logical, parameter :: test_sn_8 = ieee_support_subnormal(1.d0)
@@ -64,5 +61,8 @@ module m
   logical, parameter :: test_uc_all = .not. ieee_support_underflow_control()
   logical, parameter :: test_uc_4 = ieee_support_underflow_control(1.)
   logical, parameter :: test_uc_8 = ieee_support_underflow_control(1.d0)
+  logical, parameter :: test_std_all = ieee_support_standard()
+  logical, parameter :: test_std_4 = ieee_support_standard(1.)
+  logical, parameter :: test_std_8 = ieee_support_standard(1.d0)
 #endif
 end
diff --git a/flang/test/Evaluate/folding09.f90 b/flang/test/Evaluate/folding09.f90
index 534ff1a89a174..864f38b37ee79 100644
--- a/flang/test/Evaluate/folding09.f90
+++ b/flang/test/Evaluate/folding09.f90
@@ -9,7 +9,7 @@ module m
   logical, parameter :: test_param1 = is_contiguous(cst(:,1))
   logical, parameter :: test_param2 = is_contiguous(cst(1,:))
   logical, parameter :: test_param3 = is_contiguous(cst(:,n))
-  logical, parameter :: test_param4 = .not. is_contiguous(cst(n,:))
+  logical, parameter :: test_param4 = is_contiguous(cst(n,:))
   logical, parameter :: test_param5 = is_contiguous(empty_cst(n,-1:n:2))
  contains
   function f()
diff --git a/flang/test/Evaluate/folding18.f90 b/flang/test/Evaluate/folding18.f90
index 52aeb6a3532d0..a27eeabefae55 100644
--- a/flang/test/Evaluate/folding18.f90
+++ b/flang/test/Evaluate/folding18.f90
@@ -51,6 +51,7 @@ module m
     .and. ieee_support_sqrt(1.0_8) &
     .and. ieee_support_sqrt(1.0_10) &
     .and. ieee_support_sqrt(1.0_16)
+#if __x86_64__
   logical, parameter :: test_ieee_support_standard = ieee_support_standard() &
     .and. ieee_support_standard(1.0_2) &
     .and. ieee_support_standard(1.0_3) &
@@ -58,6 +59,7 @@ module m
     .and. ieee_support_standard(1.0_8) &
     .and. ieee_support_standard(1.0_10) &
     .and. ieee_support_standard(1.0_16)
+#endif
   logical, parameter :: test_ieee_support_subnormal = ieee_support_subnormal() &
     .and. ieee_support_subnormal(1.0_2) &
     .and. ieee_support_subnormal(1.0_3) &
diff --git a/flang/test/Evaluate/rewrite-out_of_range.F90 b/flang/test/Evaluate/rewrite-out_of_range.F90
index b5df610ff2fbb..fcfe3eb8f6bd1 100644
--- a/flang/test/Evaluate/rewrite-out_of_range.F90
+++ b/flang/test/Evaluate/rewrite-out_of_range.F90
@@ -1,5 +1,7 @@
 ! Tests rewriting of OUT_OF_RANGE()
-! RUN: %flang_fc1 -fdebug-unparse -cpp %s 2>&1 | FileCheck %s --check-prefixes=CHECK%if target=x86_64{{.*}} %{,CHECK-X86-64%}
+! REQUIRES: target=x86_64{{.*}}
+! REQUIRES: system-linux
+! RUN: %flang_fc1 -fdebug-unparse -cpp %s 2>&1 | FileCheck %s --check-prefixes=CHECK%if target=x86_64{{.*}} %{%if system-linux %{,CHECK-X86-64%}%}%if flang-supports-f128-math %{,CHECK-KIND16%}
 
 logical round
 
@@ -30,7 +32,7 @@
   M(XT,XK,real,4); \
   M(XT,XK,real,8); \
   M(XT,XK,real,10); \
-  M(XT,XK,real,16)
+  M(XT,XK,real, merge(16, 4, selected_real_kind(p=33).eq.16))
 
 #define INTXS(M1,M2) \
   M1(M2, integer, 1); \
@@ -45,7 +47,7 @@
   M1(M2, real, 4); \
   M1(M2, real, 8); \
   M1(M2, real, 10); \
-  M1(M2, real, 16)
+  M1(M2, real, merge(16, 4, selected_real_kind(p=33).eq.16))
 
 INTXS(INTMOLDS, T1)
 INTXS(REALMOLDS, T1)
@@ -85,31 +87,31 @@
 !CHECK:   PRINT *, " integer", 1_4, "real", 4_4, .false._4
 !CHECK:   PRINT *, " integer", 1_4, "real", 8_4, .false._4
 !CHECK:   PRINT *, " integer", 1_4, "real", 10_4, .false._4
-!CHECK:   PRINT *, " integer", 1_4, "real", 16_4, .false._4
+!CHECK-KIND16:   PRINT *, " integer", 1_4, "real", 16_4, .false._4
 !CHECK:   PRINT *, " integer", 2_4, "real", 2_4, .false._4
 !CHECK:   PRINT *, " integer", 2_4, "real", 3_4, .false._4
 !CHECK:   PRINT *, " integer", 2_4, "real", 4_4, .false._4
 !CHECK:   PRINT *, " integer", 2_4, "real", 8_4, .false._4
 !CHECK:   PRINT *, " integer", 2_4, "real", 10_4, .false._4
-!CHECK:   PRINT *, " integer", 2_4, "real", 16_4, .false._4
+!CHECK-KIND16:   PRINT *, " integer", 2_4, "real", 16_4, .false._4
 !CHECK:   PRINT *, " integer", 4_4, "real", 2_4, bgt(x--65519_4,131038_4)
 !CHECK:   PRINT *, " integer", 4_4, "real", 3_4, .false._4
 !CHECK:   PRINT *, " integer", 4_4, "real", 4_4, .false._4
 !CHECK:   PRINT *, " integer", 4_4, "real", 8_4, .false._4
 !CHECK:   PRINT *, " integer", 4_4, "real", 10_4, .false._4
-!CHECK:   PRINT *, " integer", 4_4, "real", 16_4, .false._4
+!CHECK-KIND16:   PRINT *, " integer", 4_4, "real", 16_4, .false._4
 !CHECK:   PRINT *, " integer", 8_4, "real", 2_4, bgt(x--65519_8,131038_8)
 !CHECK:   PRINT *, " integer", 8_4, "real", 3_4, .false._4
 !CHECK:   PRINT *, " integer", 8_4, "real", 4_4, .false._4
 !CHECK:   PRINT *, " integer", 8_4, "real", 8_4, .false._4
 !CHECK:   PRINT *, " integer", 8_4, "real", 10_4, .false._4
-!CHECK:   PRINT *, " integer", 8_4, "real", 16_4, .false._4
+!CHECK-KIND16:   PRINT *, " integer", 8_4, "real", 16_4, .false._4
 !CHECK:   PRINT *, " integer", 16_4, "real", 2_4, bgt(x--65519_16,131038_16)
 !CHECK:   PRINT *, " integer", 16_4, "real", 3_4, .false._4
 !CHECK:   PRINT *, " integer", 16_4, "real", 4_4, .false._4
 !CHECK:   PRINT *, " integer", 16_4, "real", 8_4, .false._4
 !CHECK:   PRINT *, " integer", 16_4, "real", 10_4, .false._4
-!CHECK:   PRINT *, " integer", 16_4, "real", 16_4, .false._4
+!CHECK-KIND16:   PRINT *, " integer", 16_4, "real", 16_4, .false._4
 !CHECK:   PRINT *, " real", 2_4, "integer", 1_4, bgt(transfer(real(x,kind=4)--1.28875e2_4,0_4),1132488704_4)
 !CHECK:   PRINT *, " real", 2_4, "integer", 2_4, bgt(transfer(real(x,kind=4)--3.2768e4_4,0_4),1199566848_4)
 !CHECK:   PRINT *, " real", 2_4, "integer", 4_4, bgt(transfer(real(x,kind=4)--6.5504e4_4,0_4),1207951360_4)
@@ -135,11 +137,11 @@
 !CHECK:   PRINT *, " real", 10_4, "integer", 4_4, bgt(transfer(real(x,kind=16)--2.14748364899999999976716935634613037109375e9_16,0_16),85231552932850404448147289660857516032_16)
 !CHECK:   PRINT *, " real", 10_4, "integer", 8_4, bgt(transfer(real(x,kind=16)--9.223372036854775808e18_16,0_16),85397706432322310006440791651706208256_16)
 !CHECK:   PRINT *, " real", 10_4, "integer", 16_4, bgt(transfer(real(x,kind=16)--1.70141183460469231731687303715884105728e38_16,0_16),85730013431268538974666743416776294400_16)
-!CHECK:   PRINT *, " real", 16_4, "integer", 1_4, bgt(transfer(x--1.28999999999999999999999999999999975348096711843381080883482334912930322712298902843031100928783416748046875e2_16,0_16),85106958090653963310913789279497879551_16)
-!CHECK:   PRINT *, " real", 16_4, "integer", 2_4, bgt(transfer(x--3.27689999999999999999999999999999936891127582319055567061714777377101626143485191278159618377685546875e4_16,0_16),85148476262340800794535946896424304639_16)
-!CHECK:   PRINT *, " real", 16_4, "integer", 4_4, bgt(transfer(x--2.147483648999999999999999999999999586409693723486162564295653965018573217093944549560546875e9_16,0_16),85231552932850404448147711873322582015_16)
-!CHECK:   PRINT *, " real", 16_4, "integer", 8_4, bgt(transfer(x--9.2233720368547758089999999999999982236431605997495353221893310546875e18_16,0_16),85397706432322310006441354601659629567_16)
-!CHECK:   PRINT *, " real", 16_4, "integer", 16_4, bgt(transfer(x--1.70141183460469231731687303715884105728e38_16,0_16),85730013431268538974667024891753005055_16)
+!CHECK-KIND16:   PRINT *, " real", 16_4, "integer", 1_4, bgt(transfer(x--1.28999999999999999999999999999999975348096711843381080883482334912930322712298902843031100928783416748046875e2_16,0_16),85106958090653963310913789279497879551_16)
+!CHECK-KIND16:   PRINT *, " real", 16_4, "integer", 2_4, bgt(transfer(x--3.27689999999999999999999999999999936891127582319055567061714777377101626143485191278159618377685546875e4_16,0_16),85148476262340800794535946896424304639_16)
+!CHECK-KIND16:   PRINT *, " real", 16_4, "integer", 4_4, bgt(transfer(x--2.147483648999999999999999999999999586409693723486162564295653965018573217093944549560546875e9_16,0_16),85231552932850404448147711873322582015_16)
+!CHECK-KIND16:   PRINT *, " real", 16_4, "integer", 8_4, bgt(transfer(x--9.2233720368547758089999999999999982236431605997495353221893310546875e18_16,0_16),85397706432322310006441354601659629567_16)
+!CHECK-KIND16:   PRINT *, " real", 16_4, "integer", 16_4, bgt(transfer(x--1.70141183460469231731687303715884105728e38_16,0_16),85730013431268538974667024891753005055_16)
 !CHECK:   PRINT *, " real", 2_4, "integer", 1_4, "round", bgt(transfer(real(x,kind=4)-real(merge(-1.28375e2_2,-1.28875e2_2,round),kind=4),0_4),transfer(real(merge(1.274375e2_2,1.279375e2_2,round),kind=4)-real(merge(-1.28375e2_2,-1.28875e2_2,round),kind=4),0_4))
 !CHECK:   PRINT *, " real", 2_4, "integer", 2_4, "round", bgt(transfer(real(x,kind=4)-real(merge(-3.2768e4_2,-3.2768e4_2,round),kind=4),0_4),transfer(real(merge(3.2752e4_2,3.2752e4_2,round),kind=4)-real(merge(-3.2768e4_2,-3.2768e4_2,round),kind=4),0_4))
 !CHECK:   PRINT *, " real", 2_4, "integer", 4_4, "round", bgt(transfer(real(x,kind=4)-real(merge(-6.5504e4_2,-6.5504e4_2,round),kind=4),0_4),transfer(real(merge(6.5504e4_2,6.5504e4_2,round),kind=4)-real(merge(-6.5504e4_2,-6.5504e4_2,round),kind=4),0_4))
@@ -165,44 +167,44 @@
 !CHECK:   PRINT *, " real", 10_4, "integer", 4_4, "round", bgt(transfer(real(x,kind=16)-real(merge(-2.14748364849999999976716935634613037109375e9_10,-2.14748364899999999976716935634613037109375e9_10,round),kind=16),0_16),transfer(real(merge(2.147483647499999999883584678173065185546875e9_10,2.147483647999999999883584678173065185546875e9_10,round),kind=16)-real(merge(-2.14748364849999999976716935634613037109375e9_10,-2.14748364899999999976716935634613037109375e9_10,round),kind=16),0_16))
 !CHECK:   PRINT *, " real", 10_4, "integer", 8_4, "round", bgt(transfer(real(x,kind=16)-real(merge(-9.223372036854775808e18_10,-9.223372036854775808e18_10,round),kind=16),0_16),transfer(real(merge(9.223372036854775807e18_10,9.2233720368547758075e18_10,round),kind=16)-real(merge(-9.223372036854775808e18_10,-9.223372036854775808e18_10,round),kind=16),0_16))
 !CHECK:   PRINT *, " real", 10_4, "integer", 16_4, "round", bgt(transfer(real(x,kind=16)-real(merge(-1.70141183460469231731687303715884105728e38_10,-1.70141183460469231731687303715884105728e38_10,round),kind=16),0_16),transfer(real(merge(1.7014118346046923172246393167902932992e38_10,1.7014118346046923172246393167902932992e38_10,round),kind=16)-real(merge(-1.70141183460469231731687303715884105728e38_10,-1.70141183460469231731687303715884105728e38_10,round),kind=16),0_16))
-!CHECK:   PRINT *, " real", 16_4, "integer", 1_4, "round", bgt(transfer(x-merge(-1.28499999999999999999999999999999975348096711843381080883482334912930322712298902843031100928783416748046875e2_16,-1.28999999999999999999999999999999975348096711843381080883482334912930322712298902843031100928783416748046875e2_16,round),0_16),transfer(merge(1.274999999999999999999999999999999876740483559216905404417411674564651613561494514215155504643917083740234375e2_16,1.279999999999999999999999999999999876740483559216905404417411674564651613561494514215155504643917083740234375e2_16,round)-merge(-1.28499999999999999999999999999999975348096711843381080883482334912930322712298902843031100928783416748046875e2_16,-1.28999999999999999999999999999999975348096711843381080883482334912930322712298902843031100928783416748046875e2_16,round),0_16))
-!CHECK:   PRINT *, " real", 16_4, "integer", 2_4, "round", bgt(transfer(x-merge(-3.27684999999999999999999999999999936891127582319055567061714777377101626143485191278159618377685546875e4_16,-3.27689999999999999999999999999999936891127582319055567061714777377101626143485191278159618377685546875e4_16,round),0_16),transfer(merge(3.276749999999999999999999999999999684455637911595277835308573886885508130717425956390798091888427734375e4_16,3.276799999999999999999999999999999684455637911595277835308573886885508130717425956390798091888427734375e4_16,round)-merge(-3.27684999999999999999999999999999936891127582319055567061714777377101626143485191278159618377685546875e4_16,-3.27689999999999999999999999999999936891127582319055567061714777377101626143485191278159618377685546875e4_16,round),0_16))
-!CHECK:   PRINT *, " real", 16_4, "integer", 4_4, "round", bgt(transfer(x-merge(-2.147483648499999999999999999999999586409693723486162564295653965018573217093944549560546875e9_16,-2.147483648999999999999999999999999586409693723486162564295653965018573217093944549560546875e9_16,round),0_16),transfer(merge(2.1474836474999999999999999999999997932048468617430812821478269825092866085469722747802734375e9_16,2.1474836479999999999999999999999997932048468617430812821478269825092866085469722747802734375e9_16,round)-merge(-2.147483648499999999999999999999999586409693723486162564295653965018573217093944549560546875e9_16,-2.147483648999999999999999999999999586409693723486162564295653965018573217093944549560546875e9_16,round),0_16))
-!CHECK:   PRINT *, " real", 16_4, "integer", 8_4, "round", bgt(transfer(x-merge(-9.2233720368547758084999999999999982236431605997495353221893310546875e18_16,-9.2233720368547758089999999999999982236431605997495353221893310546875e18_16,round),0_16),transfer(merge(9.22337203685477580749999999999999911182158029987476766109466552734375e18_16,9.22337203685477580799999999999999911182158029987476766109466552734375e18_16,round)-merge(-9.2233720368547758084999999999999982236431605997495353221893310546875e18_16,-9.2233720368547758089999999999999982236431605997495353221893310546875e18_16,round),0_16))
-!CHECK:   PRINT *, " real", 16_4, "integer", 16_4, "round", bgt(transfer(x-merge(-1.70141183460469231731687303715884105728e38_16,-1.70141183460469231731687303715884105728e38_16,round),0_16),transfer(merge(1.70141183460469231731687303715884089344e38_16,1.70141183460469231731687303715884089344e38_16,round)-merge(-1.70141183460469231731687303715884105728e38_16,-1.70141183460469231731687303715884105728e38_16,round),0_16))
+!CHECK-KIND16:   PRINT *, " real", 16_4, "integer", 1_4, "round", bgt(transfer(x-merge(-1.28499999999999999999999999999999975348096711843381080883482334912930322712298902843031100928783416748046875e2_16,-1.28999999999999999999999999999999975348096711843381080883482334912930322712298902843031100928783416748046875e2_16,round),0_16),transfer(merge(1.274999999999999999999999999999999876740483559216905404417411674564651613561494514215155504643917083740234375e2_16,1.279999999999999999999999999999999876740483559216905404417411674564651613561494514215155504643917083740234375e2_16,round)-merge(-1.28499999999999999999999999999999975348096711843381080883482334912930322712298902843031100928783416748046875e2_16,-1.28999999999999999999999999999999975348096711843381080883482334912930322712298902843031100928783416748046875e2_16,round),0_16))
+!CHECK-KIND16:   PRINT *, " real", 16_4, "integer", 2_4, "round", bgt(transfer(x-merge(-3.27684999999999999999999999999999936891127582319055567061714777377101626143485191278159618377685546875e4_16,-3.27689999999999999999999999999999936891127582319055567061714777377101626143485191278159618377685546875e4_16,round),0_16),transfer(merge(3.276749999999999999999999999999999684455637911595277835308573886885508130717425956390798091888427734375e4_16,3.276799999999999999999999999999999684455637911595277835308573886885508130717425956390798091888427734375e4_16,round)-merge(-3.27684999999999999999999999999999936891127582319055567061714777377101626143485191278159618377685546875e4_16,-3.27689999999999999999999999999999936891127582319055567061714777377101626143485191278159618377685546875e4_16,round),0_16))
+!CHECK-KIND16:   PRINT *, " real", 16_4, "integer", 4_4, "round", bgt(transfer(x-merge(-2.147483648499999999999999999999999586409693723486162564295653965018573217093944549560546875e9_16,-2.147483648999999999999999999999999586409693723486162564295653965018573217093944549560546875e9_16,round),0_16),transfer(merge(2.1474836474999999999999999999999997932048468617430812821478269825092866085469722747802734375e9_16,2.1474836479999999999999999999999997932048468617430812821478269825092866085469722747802734375e9_16,round)-merge(-2.147483648499999999999999999999999586409693723486162564295653965018573217093944549560546875e9_16,-2.147483648999999999999999999999999586409693723486162564295653965018573217093944549560546875e9_16,round),0_16))
+!CHECK-KIND16:   PRINT *, " real", 16_4, "integer", 8_4, "round", bgt(transfer(x-merge(-9.2233720368547758084999999999999982236431605997495353221893310546875e18_16,-9.2233720368547758089999999999999982236431605997495353221893310546875e18_16,round),0_16),transfer(merge(9.22337203685477580749999999999999911182158029987476766109466552734375e18_16,9.22337203685477580799999999999999911182158029987476766109466552734375e18_16,round)-merge(-9.2233720368547758084999999999999982236431605997495353221893310546875e18_16,-9.2233720368547758089999999999999982236431605997495353221893310546875e18_16,round),0_16))
+!CHECK-KIND16:   PRINT *, " real", 16_4, "integer", 16_4, "round", bgt(transfer(x-merge(-1.70141183460469231731687303715884105728e38_16,-1.70141183460469231731687303715884105728e38_16,round),0_16),transfer(merge(1.70141183460469231731687303715884089344e38_16,1.70141183460469231731687303715884089344e38_16,round)-merge(-1.70141183460469231731687303715884105728e38_16,-1.70141183460469231731687303715884105728e38_16,round),0_16))
 !CHECK:   PRINT *, " real", 2_4, "real", 2_4, .false._4
 !CHECK:   PRINT *, " real", 2_4, "real", 3_4, .false._4
 !CHECK:   PRINT *, " real", 2_4, "real", 4_4, .false._4
 !CHECK:   PRINT *, " real", 2_4, "real", 8_4, .false._4
 !CHECK:   PRINT *, " real", 2_4, "real", 10_4, .false._4
-!CHECK:   PRINT *, " real", 2_4, "real", 16_4, .false._4
+!CHECK-KIND16:   PRINT *, " real", 2_4, "real", 16_4, .false._4
 !CHECK:   PRINT *, " real", 3_4, "real", 2_4, blt(int(transfer(abs(x)-6.5536e4_3,0_2),kind=8)-1_8,32639_2)
 !CHECK:   PRINT *, " real", 3_4, "real", 3_4, .false._4
 !CHECK:   PRINT *, " real", 3_4, "real", 4_4, .false._4
 !CHECK:   PRINT *, " real", 3_4, "real", 8_4, .false._4
 !CHECK:   PRINT *, " real", 3_4, "real", 10_4, .false._4
-!CHECK:   PRINT *, " real", 3_4, "real", 16_4, .false._4
+!CHECK-KIND16:   PRINT *, " real", 3_4, "real", 16_4, .false._4
 !CHECK:   PRINT *, " real", 4_4, "real", 2_4, blt(int(transfer(abs(x)-6.5504e4_4,0_4),kind=8)-1_8,2139095039_4)
 !CHECK:   PRINT *, " real", 4_4, "real", 3_4, blt(int(transfer(abs(x)-3.3895313892515354759047080037148786688e38_4,0_4),kind=8)-1_8,2139095039_4)
 !CHECK:   PRINT *, " real", 4_4, "real", 4_4, .false._4
 !CHECK:   PRINT *, " real", 4_4, "real", 8_4, .false._4
 !CHECK:   PRINT *, " real", 4_4, "real", 10_4, .false._4
-!CHECK:   PRINT *, " real", 4_4, "real", 16_4, .false._4
+!CHECK-KIND16:   PRINT *, " real", 4_4, "real", 16_4, .false._4
 !CHECK:   PRINT *, " real", 8_4, "real", 2_4, blt(transfer(abs(x)-6.5504e4_8,0_8)-1_8,9218868437227405311_8)
 !CHECK:   PRINT *, " real", 8_4, "real", 3_4, blt(transfer(abs(x)-3.3895313892515354759047080037148786688e38_8,0_8)-1_8,9218868437227405311_8)
 !CHECK:   PRINT *, " real", 8_4, "real", 4_4, blt(transfer(abs(x)-3.4028234663852885981170418348451692544e38_8,0_8)-1_8,9218868437227405311_8)
 !CHECK:   PRINT *, " real", 8_4, "real", 8_4, .false._4
 !CHECK:   PRINT *, " real", 8_4, "real", 10_4, .false._4
-!CHECK:   PRINT *, " real", 8_4, "real", 16_4, .false._4
+!CHECK-KIND16:   PRINT *, " real", 8_4, "real", 16_4, .false._4
 !CHECK-X86-64:   PRINT *, " real", 10_4, "real", 2_4, blt(transfer(abs(x)-6.5504e4_10,0_16)-1_16,604444463063240877801471_16)
 !CHECK-X86-64:   PRINT *, " real", 10_4, "real", 3_4, blt(transfer(abs(x)-3.3895313892515354759047080037148786688e38_10,0_16)-1_16,604444463063240877801471_16)
 !CHECK-X86-64:   PRINT *, " real", 10_4, "real", 4_4, blt(transfer(abs(x)-3.4028234663852885981170418348451692544e38_10,0_16)-1_16,604444463063240877801471_16)
 !CHECK-X86-64:   PRINT *, " real", 10_4, "real", 8_4, blt(transfer(abs(x)-1.79769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368e308_10,0_16)-1_16,604444463063240877801471_16)
 !CHECK-X86-64:   PRINT *, " real", 10_4, "real", 10_4, .false._4
-!CHECK-X86-64:   PRINT *, " real", 10_4, "real", 16_4, .false._4
-!CHECK:   PRINT *, " real", 16_4, "real", 2_4, blt(transfer(abs(x)-6.5504e4_16,0_16)-1_16,170135991163610696904058773219554885631_16)
-!CHECK:   PRINT *, " real", 16_4, "real", 3_4, blt(transfer(abs(x)-3.3895313892515354759047080037148786688e38_16,0_16)-1_16,170135991163610696904058773219554885631_16)
-!CHECK:   PRINT *, " real", 16_4, "real", 4_4, blt(transfer(abs(x)-3.4028234663852885981170418348451692544e38_16,0_16)-1_16,170135991163610696904058773219554885631_16)
-!CHECK:   PRINT *, " real", 16_4, "real", 8_4, blt(transfer(abs(x)-1.79769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368e308_16,0_16)-1_16,170135991163610696904058773219554885631_16)
-!CHECK:   PRINT *, " real", 16_4, "real", 10_4, blt(transfer(abs(x)-1.18973149535723176502126385303097020516906332229462420044032373389173700552297072261641029033652888285354569780749557731442744315367028843419812557385374367867359320070697326320191591828296152436552951064679108661431179063216977883889613478656060039914875343321145491116008867984515486651285234014977303760000912547939396622315138362241783854274391783813871780588948754057516822634765923557697480511372564902088485522249479139937758502601177354918009979622602685950855888360815984690023564513234659447638493985927645628457966177293040780660922910271504608538808795932778162298682754783076808004015069494230341172895777710033571401055977524212405734700738625166011082837911962300846927720096515350020847447079244384854591288672300061908512647211195136146752763351956292759795725027800298079590419313960302147099703527646744553092202267965628099149823208332964124103850923918473478612192169721054348428704835340811304257300221642134891734717423480071488075100206439051723424765600472176809648610799494341570347632064355862420744350442438056613601760883747816538902780957697597728686007148702828795556714140463261583262360276289631617397848425448686060994827086796804807870251185893083854658422304090880599629459458620190376604844679092600222541053077590106576067134720012584640695703025713896098375799892695455305236856075868317922311363951946885088077187210470520395758748001314313144425494391994017575316933939236688185618912993172910425292123683515992232205099800167710278403536014082929639811512287776813570604578934353545169653956125404884644716978689321167108722908808277835051822885764606221873970285165508372099234948333443522898475123275372663606621390228126470623407535207172405866507951821730346378263135339370677490195019784169044182473806316282858685774143258116536404021840272491339332094921949842244273042701987304453662035026238695780468200360144729199712309553005720614186697485284685618651483271597448120312194675168637934309618961510733006555242148519520176285859509105183947250286387163249416761380499631979144187025430270675849519200883791516940158174004671147787720145964446117520405945350476472180797576111172084627363927960033967047003761337450955318415007379641260504792325166135484129188421134082301547330475406707281876350361733290800595189632520707167390454777712968226520622565143991937680440029238090311243791261477625596469422198137514696707944687035800439250765945161837981185939204954403611491531078225107269148697980924094677214272701240437718740921675661363493890045123235166814608932240069799317601780533819184998193300841098599393876029260139091141452600372028487213241195542428210183120421610446740462163533690058366460659115629876474552506814500393294140413149540067760295100596225302282300363147382468105964844244132486457313743759509641616804802412935187620466813563687753281467553879887177183651289394719533506188500326760735438867336800207438784965701457609034985757124304510203873049485425670247933932280911052604153852899484920399109194612991249163328991799809438033787952209313146694614970593966415237594928589096048991612194498998638483702248667224914892467841020618336462741696957630763248023558797524525373703543388296086275342774001633343405508353704850737454481975472222897528108302089868263302028525992308416805453968791141829762998896457648276528750456285492426516521775079951625966922911497778896235667095662713848201819134832168799586365263762097828507009933729439678463987902491451422274252700636394232799848397673998715441855420156224415492665301451550468548925862027608576183712976335876121538256512963353814166394951655600026415918655485005705261143195291991880795452239464962763563017858089669222640623538289853586759599064700838568712381032959192649484625076899225841930548076362021508902214922052806984201835084058693849381549890944546197789302911357651677540623227829831403347327660395223160342282471752818181884430488092132193355086987339586127607367086665237555567580317149010847732009642431878007000879734603290627894355374356444885190719161645514115576193939969076741515640282654366402676009508752394550734155613586793306603174472092444651353236664764973540085196704077110364053815007348689179836404957060618953500508984091382686953509006678332447257871219660441528492484004185093281190896363417573989716659600075948780061916409485433875852065711654107226099628815012314437794400874930194474433078438899570184271000480830501217712356062289507626904285680004771889315808935851559386317665294808903126774702966254511086154895839508779675546413794489596052797520987481383976257859210575628440175934932416214833956535018919681138909184379573470326940634289008780584694035245347939808067427323629788710086717580253156130235606487870925986528841635097252953709111431720488774740553905400942537542411931794417513706468964386151771884986701034153254238591108962471088538580868883777725864856414593426212108664758848926003176234596076950884914966244415660441955208681198977024e4932_16,0_16)-1_16,170135991163610696904058773219554885631_16)
-!CHECK:   PRINT *, " real", 16_4, "real", 16_4, .false._4
+!CHECK-KIND16:   PRINT *, " real", 10_4, "real", 16_4, .false._4
+!CHECK-KIND16:   PRINT *, " real", 16_4, "real", 2_4, blt(transfer(abs(x)-6.5504e4_16,0_16)-1_16,170135991163610696904058773219554885631_16)
+!CHECK-KIND16:   PRINT *, " real", 16_4, "real", 3_4, blt(transfer(abs(x)-3.3895313892515354759047080037148786688e38_16,0_16)-1_16,170135991163610696904058773219554885631_16)
+!CHECK-KIND16:   PRINT *, " real", 16_4, "real", 4_4, blt(transfer(abs(x)-3.4028234663852885981170418348451692544e38_16,0_16)-1_16,170135991163610696904058773219554885631_16)
+!CHECK-KIND16:   PRINT *, " real", 16_4, "real", 8_4, blt(transfer(abs(x)-1.79769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368e308_16,0_16)-1_16,170135991163610696904058773219554885631_16)
+!CHECK-KIND16:   PRINT *, " real", 16_4, "real", 10_4, blt(transfer(abs(x)-1.18973149535723176502126385303097020516906332229462420044032373389173700552297072261641029033652888285354569780749557731442744315367028843419812557385374367867359320070697326320191591828296152436552951064679108661431179063216977883889613478656060039914875343321145491116008867984515486651285234014977303760000912547939396622315138362241783854274391783813871780588948754057516822634765923557697480511372564902088485522249479139937758502601177354918009979622602685950855888360815984690023564513234659447638493985927645628457966177293040780660922910271504608538808795932778162298682754783076808004015069494230341172895777710033571401055977524212405734700738625166011082837911962300846927720096515350020847447079244384854591288672300061908512647211195136146752763351956292759795725027800298079590419313960302147099703527646744553092202267965628099149823208332964124103850923918473478612192169721054348428704835340811304257300221642134891734717423480071488075100206439051723424765600472176809648610799494341570347632064355862420744350442438056613601760883747816538902780957697597728686007148702828795556714140463261583262360276289631617397848425448686060994827086796804807870251185893083854658422304090880599629459458620190376604844679092600222541053077590106576067134720012584640695703025713896098375799892695455305236856075868317922311363951946885088077187210470520395758748001314313144425494391994017575316933939236688185618912993172910425292123683515992232205099800167710278403536014082929639811512287776813570604578934353545169653956125404884644716978689321167108722908808277835051822885764606221873970285165508372099234948333443522898475123275372663606621390228126470623407535207172405866507951821730346378263135339370677490195019784169044182473806316282858685774143258116536404021840272491339332094921949842244273042701987304453662035026238695780468200360144729199712309553005720614186697485284685618651483271597448120312194675168637934309618961510733006555242148519520176285859509105183947250286387163249416761380499631979144187025430270675849519200883791516940158174004671147787720145964446117520405945350476472180797576111172084627363927960033967047003761337450955318415007379641260504792325166135484129188421134082301547330475406707281876350361733290800595189632520707167390454777712968226520622565143991937680440029238090311243791261477625596469422198137514696707944687035800439250765945161837981185939204954403611491531078225107269148697980924094677214272701240437718740921675661363493890045123235166814608932240069799317601780533819184998193300841098599393876029260139091141452600372028487213241195542428210183120421610446740462163533690058366460659115629876474552506814500393294140413149540067760295100596225302282300363147382468105964844244132486457313743759509641616804802412935187620466813563687753281467553879887177183651289394719533506188500326760735438867336800207438784965701457609034985757124304510203873049485425670247933932280911052604153852899484920399109194612991249163328991799809438033787952209313146694614970593966415237594928589096048991612194498998638483702248667224914892467841020618336462741696957630763248023558797524525373703543388296086275342774001633343405508353704850737454481975472222897528108302089868263302028525992308416805453968791141829762998896457648276528750456285492426516521775079951625966922911497778896235667095662713848201819134832168799586365263762097828507009933729439678463987902491451422274252700636394232799848397673998715441855420156224415492665301451550468548925862027608576183712976335876121538256512963353814166394951655600026415918655485005705261143195291991880795452239464962763563017858089669222640623538289853586759599064700838568712381032959192649484625076899225841930548076362021508902214922052806984201835084058693849381549890944546197789302911357651677540623227829831403347327660395223160342282471752818181884430488092132193355086987339586127607367086665237555567580317149010847732009642431878007000879734603290627894355374356444885190719161645514115576193939969076741515640282654366402676009508752394550734155613586793306603174472092444651353236664764973540085196704077110364053815007348689179836404957060618953500508984091382686953509006678332447257871219660441528492484004185093281190896363417573989716659600075948780061916409485433875852065711654107226099628815012314437794400874930194474433078438899570184271000480830501217712356062289507626904285680004771889315808935851559386317665294808903126774702966254511086154895839508779675546413794489596052797520987481383976257859210575628440175934932416214833956535018919681138909184379573470326940634289008780584694035245347939808067427323629788710086717580253156130235606487870925986528841635097252953709111431720488774740553905400942537542411931794417513706468964386151771884986701034153254238591108962471088538580868883777725864856414593426212108664758848926003176234596076950884914966244415660441955208681198977024e4932_16,0_16)-1_16,170135991163610696904058773219554885631_16)
+!CHECK-KIND16:   PRINT *, " real", 16_4, "real", 16_4, .false._4
diff --git a/flang/test/Fir/CUDA/cuda-alloc-free.fir b/flang/test/Fir/CUDA/cuda-alloc-free.fir
index 6194f0071cd79..31f2ed022b6c4 100644
--- a/flang/test/Fir/CUDA/cuda-alloc-free.fir
+++ b/flang/test/Fir/CUDA/cuda-alloc-free.fir
@@ -11,7 +11,7 @@ func.func @_QPsub1() {
 
 // CHECK-LABEL: func.func @_QPsub1()
 // CHECK: %[[BYTES:.*]] = fir.convert %c4{{.*}} : (index) -> i64
-// CHECK: %[[ALLOC:.*]] = fir.call @_FortranACUFMemAlloc(%[[BYTES]], %c0{{.*}}, %{{.*}}, %{{.*}}) : (i64, i32, !fir.ref<i8>, i32) -> !fir.llvm_ptr<i8>
+// CHECK: %[[ALLOC:.*]] = fir.call @_FortranACUFMemAlloc(%[[BYTES]], %c0{{.*}}, %{{.*}}, %{{.*}}) {cuf.data_attr = #cuf.cuda<device>} : (i64, i32, !fir.ref<i8>, i32) -> !fir.llvm_ptr<i8>
 // CHECK: %[[CONV:.*]] = fir.convert %3 : (!fir.llvm_ptr<i8>) -> !fir.ref<i32>
 // CHECK: %[[DECL:.*]]:2 = hlfir.declare %[[CONV]] {data_attr = #cuf.cuda<device>, uniq_name = "_QFsub1Eidev"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 // CHECK: %[[DEVPTR:.*]] = fir.convert %[[DECL]]#1 : (!fir.ref<i32>) -> !fir.llvm_ptr<i8>
@@ -26,7 +26,7 @@ func.func @_QPsub2() {
 // CHECK-LABEL: func.func @_QPsub2()
 // CHECK: %[[BYTES:.*]] = arith.muli %c10{{.*}}, %c4{{.*}} : index
 // CHECK: %[[CONV_BYTES:.*]] = fir.convert %[[BYTES]] : (index) -> i64 
-// CHECK: %{{.*}} = fir.call @_FortranACUFMemAlloc(%[[CONV_BYTES]], %c0{{.*}}, %{{.*}}, %{{.*}}) : (i64, i32, !fir.ref<i8>, i32) -> !fir.llvm_ptr<i8>
+// CHECK: %{{.*}} = fir.call @_FortranACUFMemAlloc(%[[CONV_BYTES]], %c0{{.*}}, %{{.*}}, %{{.*}}) {cuf.data_attr = #cuf.cuda<device>} : (i64, i32, !fir.ref<i8>, i32) -> !fir.llvm_ptr<i8>
 // CHECK: fir.call @_FortranACUFMemFree
 
 func.func @_QPsub3(%arg0: !fir.ref<i32> {fir.bindc_name = "n"}, %arg1: !fir.ref<i32> {fir.bindc_name = "m"}) {
@@ -58,7 +58,7 @@ func.func @_QPsub3(%arg0: !fir.ref<i32> {fir.bindc_name = "n"}, %arg1: !fir.ref<
 // CHECK: %[[NBELEM:.*]] = arith.muli %[[N]], %[[M]] : index
 // CHECK: %[[BYTES:.*]] = arith.muli %[[NBELEM]], %c4{{.*}} : index
 // CHECK: %[[CONV_BYTES:.*]] = fir.convert %[[BYTES]] : (index) -> i64
-// CHECK: %{{.*}} = fir.call @_FortranACUFMemAlloc(%[[CONV_BYTES]], %c0{{.*}}, %{{.*}}, %{{.*}}) : (i64, i32, !fir.ref<i8>, i32) -> !fir.llvm_ptr<i8>
+// CHECK: %{{.*}} = fir.call @_FortranACUFMemAlloc(%[[CONV_BYTES]], %c0{{.*}}, %{{.*}}, %{{.*}}) {cuf.data_attr = #cuf.cuda<device>} : (i64, i32, !fir.ref<i8>, i32) -> !fir.llvm_ptr<i8>
 // CHECK: fir.call @_FortranACUFMemFree
 
 func.func @_QPtest_type() {
@@ -71,7 +71,7 @@ func.func @_QPtest_type() {
 // CHECK-LABEL: func.func @_QPtest_type()
 // CHECK: %[[BYTES:.*]] = arith.constant 12 : index
 // CHECK: %[[CONV_BYTES:.*]] = fir.convert %[[BYTES]] : (index) -> i64
-// CHECK: fir.call @_FortranACUFMemAlloc(%[[CONV_BYTES]], %c0{{.*}}, %{{.*}}, %{{.*}}) : (i64, i32, !fir.ref<i8>, i32) -> !fir.llvm_ptr<i8>
+// CHECK: fir.call @_FortranACUFMemAlloc(%[[CONV_BYTES]], %c0{{.*}}, %{{.*}}, %{{.*}}) {cuf.data_attr = #cuf.cuda<device>} : (i64, i32, !fir.ref<i8>, i32) -> !fir.llvm_ptr<i8>
 
 gpu.module @cuda_device_mod {
   gpu.func @_QMalloc() kernel {
@@ -81,7 +81,7 @@ gpu.module @cuda_device_mod {
 }
 
 // CHECK-LABEL: gpu.func @_QMalloc() kernel
-// CHECK: fir.alloca !fir.box<!fir.heap<!fir.array<?xf32>>> {bindc_name = "a", uniq_name = "_QMallocEa"}
+// CHECK: fir.alloca !fir.box<!fir.heap<!fir.array<?xf32>>> {bindc_name = "a", cuf.data_attr = #cuf.cuda<device>, uniq_name = "_QMallocEa"}
 
 func.func @_QQalloc_char() attributes {fir.bindc_name = "alloc_char"} {
   %c1 = arith.constant 1 : index
@@ -92,6 +92,6 @@ func.func @_QQalloc_char() attributes {fir.bindc_name = "alloc_char"} {
 // CHECK-LABEL: func.func @_QQalloc_char()
 // CHECK: %[[BYTES:.*]] = arith.muli %c10{{.*}}, %c1{{.*}} : index
 // CHECK: %[[BYTES_CONV:.*]] = fir.convert %[[BYTES]] : (index) -> i64
-// CHECK: fir.call @_FortranACUFMemAlloc(%[[BYTES_CONV]], %c0{{.*}}, %{{.*}}, %{{.*}}) : (i64, i32, !fir.ref<i8>, i32) -> !fir.llvm_ptr<i8>
+// CHECK: fir.call @_FortranACUFMemAlloc(%[[BYTES_CONV]], %c0{{.*}}, %{{.*}}, %{{.*}}) {cuf.data_attr = #cuf.cuda<device>} : (i64, i32, !fir.ref<i8>, i32) -> !fir.llvm_ptr<i8>
 
 } // end module
diff --git a/flang/test/Fir/CUDA/cuda-allocate.fir b/flang/test/Fir/CUDA/cuda-allocate.fir
index b8457b846716e..095ad92d5deb5 100644
--- a/flang/test/Fir/CUDA/cuda-allocate.fir
+++ b/flang/test/Fir/CUDA/cuda-allocate.fir
@@ -15,16 +15,16 @@ func.func @_QPsub1() {
 }
 
 // CHECK-LABEL: func.func @_QPsub1()
-// CHECK: %[[DESC_RT_CALL:.*]] = fir.call @_FortranACUFAllocDescriptor(%{{.*}}, %{{.*}}, %{{.*}}) : (i64, !fir.ref<i8>, i32) -> !fir.ref<!fir.box<none>>
+// CHECK: %[[DESC_RT_CALL:.*]] = fir.call @_FortranACUFAllocDescriptor(%{{.*}}, %{{.*}}, %{{.*}}) {cuf.data_attr = #cuf.cuda<device>} : (i64, !fir.ref<i8>, i32) -> !fir.ref<!fir.box<none>>
 // CHECK: %[[DESC:.*]] = fir.convert %[[DESC_RT_CALL]] : (!fir.ref<!fir.box<none>>) -> !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>
 // CHECK: %[[DECL_DESC:.*]]:2 = hlfir.declare %[[DESC]] {data_attr = #cuf.cuda<device>, fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "_QFsub1Ea"} : (!fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>) -> (!fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>, !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>)
 // CHECK: %[[BOX_NONE:.*]] = fir.convert %[[DECL_DESC]]#1 : (!fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>) -> !fir.ref<!fir.box<none>>
-// CHECK: %{{.*}} = fir.call @_FortranACUFAllocatableAllocate(%[[BOX_NONE]], %{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}) : (!fir.ref<!fir.box<none>>, i64, i1, !fir.box<none>, !fir.ref<i8>, i32) -> i32
+// CHECK: %{{.*}} = fir.call @_FortranACUFAllocatableAllocate(%[[BOX_NONE]], %{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}) : (!fir.ref<!fir.box<none>>, i64, !fir.ref<i1>, i1, !fir.box<none>, !fir.ref<i8>, i32) -> i32
 
 // CHECK: %[[BOX_NONE:.*]] = fir.convert %[[DECL_DESC]]#1 : (!fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>) -> !fir.ref<!fir.box<none>>
 // CHECK: %{{.*}} = fir.call @_FortranAAllocatableDeallocate(%[[BOX_NONE]], %{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}) : (!fir.ref<!fir.box<none>>, i1, !fir.box<none>, !fir.ref<i8>, i32) -> i32
 // CHECK: %[[BOX_NONE:.*]] = fir.convert %[[DECL_DESC]]#1 : (!fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>) -> !fir.ref<!fir.box<none>>
-// CHECK: fir.call @_FortranACUFFreeDescriptor(%[[BOX_NONE]], %{{.*}}, %{{.*}}) : (!fir.ref<!fir.box<none>>, !fir.ref<i8>, i32) -> ()
+// CHECK: fir.call @_FortranACUFFreeDescriptor(%[[BOX_NONE]], %{{.*}}, %{{.*}}) {cuf.data_attr = #cuf.cuda<device>} : (!fir.ref<!fir.box<none>>, !fir.ref<i8>, i32) -> ()
 
 fir.global @_QMmod1Ea {data_attr = #cuf.cuda<device>} : !fir.box<!fir.heap<!fir.array<?xf32>>> {
     %0 = fir.zero_bits !fir.heap<!fir.array<?xf32>>
@@ -47,7 +47,7 @@ func.func @_QPsub3() {
 // CHECK: %[[A:.*]]:2 = hlfir.declare %[[A_ADDR]] {data_attr = #cuf.cuda<device>, fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "_QMmod1Ea"} : (!fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>) -> (!fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>, !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>)
 
 // CHECK: %[[A_BOX:.*]] = fir.convert %[[A]]#1 : (!fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>) -> !fir.ref<!fir.box<none>>
-// CHECK: fir.call @_FortranACUFAllocatableAllocateSync(%[[A_BOX]], %{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}) : (!fir.ref<!fir.box<none>>, i64, i1, !fir.box<none>, !fir.ref<i8>, i32) -> i32
+// CHECK: fir.call @_FortranACUFAllocatableAllocateSync(%[[A_BOX]], %{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}) : (!fir.ref<!fir.box<none>>, i64, !fir.ref<i1>, i1, !fir.box<none>, !fir.ref<i8>, i32) -> i32
 
 // CHECK: %[[A_BOX:.*]] = fir.convert %[[A]]#1 : (!fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>) -> !fir.ref<!fir.box<none>>
 // CHECK: fir.call @_FortranACUFAllocatableDeallocate(%[[A_BOX]], %{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}) : (!fir.ref<!fir.box<none>>, i1, !fir.box<none>, !fir.ref<i8>, i32) -> i32
@@ -87,7 +87,7 @@ func.func @_QPsub5() {
 }
 
 // CHECK-LABEL: func.func @_QPsub5()
-// CHECK: fir.call @_FortranACUFAllocatableAllocate({{.*}}) : (!fir.ref<!fir.box<none>>, i64, i1, !fir.box<none>, !fir.ref<i8>, i32) -> i32
+// CHECK: fir.call @_FortranACUFAllocatableAllocate({{.*}}) : (!fir.ref<!fir.box<none>>, i64, !fir.ref<i1>, i1, !fir.box<none>, !fir.ref<i8>, i32) -> i32
 // CHECK: fir.call @_FortranAAllocatableDeallocate({{.*}}) : (!fir.ref<!fir.box<none>>, i1, !fir.box<none>, !fir.ref<i8>, i32) -> i32
 
 
@@ -118,7 +118,7 @@ func.func @_QQsub6() attributes {fir.bindc_name = "test"} {
 // CHECK: %[[B:.*]]:2 = hlfir.declare %[[B_ADDR]] {data_attr = #cuf.cuda<device>, fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "_QMdataEb"} : (!fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>) -> (!fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>, !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>)
 // CHECK: _FortranAAllocatableSetBounds
 // CHECK: %[[B_BOX:.*]] = fir.convert %[[B]]#1 : (!fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>) -> !fir.ref<!fir.box<none>>
-// CHECK: fir.call @_FortranACUFAllocatableAllocateSync(%[[B_BOX]], %{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}) : (!fir.ref<!fir.box<none>>, i64, i1, !fir.box<none>, !fir.ref<i8>, i32) -> i32
+// CHECK: fir.call @_FortranACUFAllocatableAllocateSync(%[[B_BOX]], %{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}) : (!fir.ref<!fir.box<none>>, i64, !fir.ref<i1>, i1, !fir.box<none>, !fir.ref<i8>, i32) -> i32
 
 
 func.func @_QPallocate_source() {
@@ -142,7 +142,7 @@ func.func @_QPallocate_source() {
 // CHECK: %[[SOURCE:.*]] = fir.load %[[DECL_HOST]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?x?xf32>>>>
 // CHECK: %[[DEV_CONV:.*]] = fir.convert %[[DECL_DEV]] : (!fir.ref<!fir.box<!fir.heap<!fir.array<?x?xf32>>>>) -> !fir.ref<!fir.box<none>>
 // CHECK: %[[SOURCE_CONV:.*]] = fir.convert %[[SOURCE]] : (!fir.box<!fir.heap<!fir.array<?x?xf32>>>) -> !fir.box<none>
-// CHECK: %{{.*}} = fir.call @_FortranACUFAllocatableAllocateSource(%[[DEV_CONV]], %[[SOURCE_CONV]], %{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}) : (!fir.ref<!fir.box<none>>, !fir.box<none>, i64, i1, !fir.box<none>, !fir.ref<i8>, i32) -> i32
+// CHECK: %{{.*}} = fir.call @_FortranACUFAllocatableAllocateSource(%[[DEV_CONV]], %[[SOURCE_CONV]], %{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}) : (!fir.ref<!fir.box<none>>, !fir.box<none>, i64, !fir.ref<i1>, i1, !fir.box<none>, !fir.ref<i8>, i32) -> i32
 
 
 fir.global @_QMmod1Ea_d {data_attr = #cuf.cuda<device>} : !fir.box<!fir.heap<!fir.array<?x?xf32>>> {
@@ -179,7 +179,7 @@ func.func @_QQallocate_stream() {
 // CHECK: %[[STREAM_ALLOCA:.*]] = fir.alloca i64 {bindc_name = "stream1", uniq_name = "_QFEstream1"}
 // CHECK: %[[STREAM:.*]] = fir.declare %[[STREAM_ALLOCA]] {uniq_name = "_QFEstream1"} : (!fir.ref<i64>) -> !fir.ref<i64>
 // CHECK: %[[STREAM_LOAD:.*]] = fir.load %[[STREAM]] : !fir.ref<i64>
-// CHECK: fir.call @_FortranACUFAllocatableAllocate(%{{.*}}, %[[STREAM_LOAD]], %{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}) : (!fir.ref<!fir.box<none>>, i64, i1, !fir.box<none>, !fir.ref<i8>, i32) -> i32
+// CHECK: fir.call @_FortranACUFAllocatableAllocate(%{{.*}}, %[[STREAM_LOAD]], %{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}) : (!fir.ref<!fir.box<none>>, i64, !fir.ref<i1>, i1, !fir.box<none>, !fir.ref<i8>, i32) -> i32
 
 
 func.func @_QPp_alloc() {
@@ -255,4 +255,19 @@ func.func @_QMmod1Ppointer_source_global() {
 // CHECK-LABEL: func.func @_QMmod1Ppointer_source_global()
 // CHECK: fir.call @_FortranACUFPointerAllocateSourceSync
 
+func.func @_QQpinned() attributes {fir.bindc_name = "testasync"} {
+  %0 = cuf.alloc !fir.box<!fir.heap<!fir.array<?xf32>>> {bindc_name = "a", data_attr = #cuf.cuda<pinned>, uniq_name = "_QFEa"} -> !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>
+  %4 = fir.declare %0 {data_attr = #cuf.cuda<pinned>, fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "_QFEa"} : (!fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>) -> !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>
+  %13 = fir.alloca !fir.logical<4> {bindc_name = "pinnedflag", uniq_name = "_QFEpinnedflag"}
+  %14 = fir.declare %13 {uniq_name = "_QFEpinnedflag"} : (!fir.ref<!fir.logical<4>>) -> !fir.ref<!fir.logical<4>>
+  %18 = cuf.allocate %4 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>> pinned(%14 : !fir.ref<!fir.logical<4>>) {data_attr = #cuf.cuda<pinned>, hasStat} -> i32
+  return
+}
+
+// CHECK-LABEL: func.func @_QQpinned() attributes {fir.bindc_name = "testasync"} {
+// CHECK: %[[PINNED:.*]] = fir.alloca !fir.logical<4> {bindc_name = "pinnedflag", uniq_name = "_QFEpinnedflag"}
+// CHECK: %[[DECL_PINNED:.*]] = fir.declare %[[PINNED]] {uniq_name = "_QFEpinnedflag"} : (!fir.ref<!fir.logical<4>>) -> !fir.ref<!fir.logical<4>>
+// CHECK: %[[CONV_PINNED:.*]] = fir.convert %[[DECL_PINNED]] : (!fir.ref<!fir.logical<4>>) -> !fir.ref<i1>
+// CHECK: fir.call @_FortranACUFAllocatableAllocate(%{{.*}}, %{{.*}}, %[[CONV_PINNED]], %{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}) : (!fir.ref<!fir.box<none>>, i64, !fir.ref<i1>, i1, !fir.box<none>, !fir.ref<i8>, i32) -> i32
+
 } // end of module
diff --git a/flang/test/Fir/CUDA/cuda-data-transfer.fir b/flang/test/Fir/CUDA/cuda-data-transfer.fir
index 415d0015918bb..b62c500f4a2d3 100644
--- a/flang/test/Fir/CUDA/cuda-data-transfer.fir
+++ b/flang/test/Fir/CUDA/cuda-data-transfer.fir
@@ -329,7 +329,7 @@ func.func @_QPtest_array_type() {
 // CHECK-LABEL: func.func @_QPtest_array_type()
 // CHECK: %[[BYTES:.*]] = arith.muli %c10{{.*}}, %c12 : index
 // CHECK: %[[CONV_BYTES:.*]] = fir.convert %[[BYTES]] : (index) -> i64
-// CHECK: fir.call @_FortranACUFMemAlloc(%[[CONV_BYTES]], %c0{{.*}}, %{{.*}}, %{{.*}}) : (i64, i32, !fir.ref<i8>, i32) -> !fir.llvm_ptr<i8>
+// CHECK: fir.call @_FortranACUFMemAlloc(%[[CONV_BYTES]], %c0{{.*}}, %{{.*}}, %{{.*}}) {cuf.data_attr = #cuf.cuda<device>} : (i64, i32, !fir.ref<i8>, i32) -> !fir.llvm_ptr<i8>
 // CHECK: %[[BYTES:.*]] = arith.muli %c10{{.*}}, %c12{{.*}} : i64
 // CHECK: fir.call @_FortranACUFDataTransferPtrPtr(%{{.*}}, %{{.*}}, %[[BYTES]], %c0{{.*}}, %{{.*}}, %{{.*}}) : (!fir.llvm_ptr<i8>, !fir.llvm_ptr<i8>, i64, i32, !fir.ref<i8>, i32) -> ()
 
diff --git a/flang/test/Fir/CUDA/cuda-gpu-launch-func.mlir b/flang/test/Fir/CUDA/cuda-gpu-launch-func.mlir
index 3db2336c90a7d..0827e378c7c07 100644
--- a/flang/test/Fir/CUDA/cuda-gpu-launch-func.mlir
+++ b/flang/test/Fir/CUDA/cuda-gpu-launch-func.mlir
@@ -131,3 +131,38 @@ module attributes {dlti.dl_spec = #dlti.dl_spec<#dlti.dl_entry<!llvm.ptr<272>, d
 // CHECK-LABEL: llvm.func @_QQmain()
 // CHECK: %[[KERNEL_PTR:.*]] = llvm.mlir.addressof @_QMmod1Psub1
 // CHECK: llvm.call @_FortranACUFLaunchClusterKernel(%[[KERNEL_PTR]], {{.*}})
+
+// -----
+
+module attributes {dlti.dl_spec = #dlti.dl_spec<#dlti.dl_entry<i1, dense<8> : vector<2xi64>>, #dlti.dl_entry<!llvm.ptr, dense<64> : vector<4xi64>>, #dlti.dl_entry<!llvm.ptr<270>, dense<32> : vector<4xi64>>, #dlti.dl_entry<!llvm.ptr<271>, dense<32> : vector<4xi64>>, #dlti.dl_entry<i8, dense<8> : vector<2xi64>>, #dlti.dl_entry<i16, dense<16> : vector<2xi64>>, #dlti.dl_entry<!llvm.ptr<272>, dense<64> : vector<4xi64>>, #dlti.dl_entry<i64, dense<64> : vector<2xi64>>, #dlti.dl_entry<i32, dense<32> : vector<2xi64>>, #dlti.dl_entry<f128, dense<128> : vector<2xi64>>, #dlti.dl_entry<i128, dense<128> : vector<2xi64>>, #dlti.dl_entry<f64, dense<64> : vector<2xi64>>, #dlti.dl_entry<f80, dense<128> : vector<2xi64>>, #dlti.dl_entry<f16, dense<16> : vector<2xi64>>, #dlti.dl_entry<"dlti.endianness", "little">, #dlti.dl_entry<"dlti.stack_alignment", 128 : i64>>, fir.defaultkind = "a1c4d8i4l4r4", fir.kindmap = "", gpu.container_module, llvm.data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", llvm.ident = "flang version 20.0.0 (git@github.com:clementval/llvm-project.git ddcfd4d2dc17bf66cee8c3ef6284118684a2b0e6)", llvm.target_triple = "x86_64-unknown-linux-gnu"} {
+  llvm.func @_QMmod1Phost_sub() {
+    %0 = llvm.mlir.constant(1 : i32) : i32
+    %1 = llvm.alloca %0 x !llvm.struct<(ptr, i64, i32, i8, i8, i8, i8, array<1 x array<3 x i64>>)> {alignment = 8 : i64} : (i32) -> !llvm.ptr
+    %2 = llvm.mlir.constant(40 : i64) : i64
+    %3 = llvm.mlir.constant(16 : i32) : i32
+    %4 = llvm.mlir.constant(25 : i32) : i32
+    %5 = llvm.mlir.constant(21 : i32) : i32
+    %6 = llvm.mlir.constant(17 : i32) : i32
+    %7 = llvm.mlir.constant(1 : index) : i64
+    %8 = llvm.mlir.constant(27 : i32) : i32
+    %9 = llvm.mlir.constant(6 : i32) : i32
+    %10 = llvm.mlir.constant(1 : i32) : i32
+    %11 = llvm.mlir.constant(0 : i32) : i32
+    %12 = llvm.mlir.constant(10 : index) : i64
+    %13 = llvm.mlir.addressof @_QQclX91d13f6e74caa2f03965d7a7c6a8fdd5 : !llvm.ptr
+    %14 = llvm.call @_FortranACUFMemAlloc(%2, %11, %13, %6) : (i64, i32, !llvm.ptr, i32) -> !llvm.ptr
+    gpu.launch_func  @cuda_device_mod::@_QMmod1Psub1 blocks in (%7, %7, %7) threads in (%12, %7, %7) : i64 dynamic_shared_memory_size %11 args(%14 : !llvm.ptr) {cuf.proc_attr = #cuf.cuda_proc<grid_global>}
+    llvm.return
+  }
+  llvm.func @_QMmod1Psub1(!llvm.ptr) -> ()
+  llvm.mlir.global linkonce constant @_QQclX91d13f6e74caa2f03965d7a7c6a8fdd5()  {addr_space = 0 : i32} : !llvm.array<2 x i8> {
+    %0 = llvm.mlir.constant("a\00") : !llvm.array<2 x i8>
+    llvm.return %0 : !llvm.array<2 x i8>
+  }
+  llvm.func @_FortranACUFMemAlloc(i64, i32, !llvm.ptr, i32) -> !llvm.ptr attributes {fir.runtime, sym_visibility = "private"}
+  llvm.func @_FortranACUFMemFree(!llvm.ptr, i32, !llvm.ptr, i32) -> !llvm.struct<()> attributes {fir.runtime, sym_visibility = "private"}
+  gpu.binary @cuda_device_mod  [#gpu.object<#nvvm.target, "">]
+}
+
+// CHECK-LABEL: llvm.func @_QMmod1Phost_sub()
+// CHECK: llvm.call @_FortranACUFLaunchCooperativeKernel
diff --git a/flang/test/Fir/CUDA/cuda-launch.fir b/flang/test/Fir/CUDA/cuda-launch.fir
index 8432b9ec926e3..7833fc7b490bf 100644
--- a/flang/test/Fir/CUDA/cuda-launch.fir
+++ b/flang/test/Fir/CUDA/cuda-launch.fir
@@ -104,3 +104,24 @@ module attributes {gpu.container_module, dlti.dl_spec = #dlti.dl_spec<#dlti.dl_e
 // CHECK: %[[DEVADDR:.*]] = fir.call @_FortranACUFGetDeviceAddress(%[[CONV_ADDR]], %{{.*}}, %{{.*}}) : (!fir.llvm_ptr<i8>, !fir.ref<i8>, i32) -> !fir.llvm_ptr<i8>
 // CHECK: %[[CONV_DEVADDR:.*]] = fir.convert %[[DEVADDR]] : (!fir.llvm_ptr<i8>) -> !fir.ref<!fir.box<!fir.ptr<!fir.array<?xf32>>>>
 // CHECK: gpu.launch_func  @cuda_device_mod::@_QMdevptrPtest blocks in (%{{.*}}, %{{.*}}, %{{.*}}) threads in (%{{.*}}, %{{.*}}, %{{.*}})  dynamic_shared_memory_size %{{.*}} args(%[[CONV_DEVADDR]] : !fir.ref<!fir.box<!fir.ptr<!fir.array<?xf32>>>>)
+
+// -----
+
+module attributes {gpu.container_module, dlti.dl_spec = #dlti.dl_spec<#dlti.dl_entry<f80, dense<128> : vector<2xi64>>, #dlti.dl_entry<i128, dense<128> : vector<2xi64>>, #dlti.dl_entry<i64, dense<64> : vector<2xi64>>, #dlti.dl_entry<!llvm.ptr<272>, dense<64> : vector<4xi64>>, #dlti.dl_entry<!llvm.ptr<271>, dense<32> : vector<4xi64>>, #dlti.dl_entry<!llvm.ptr<270>, dense<32> : vector<4xi64>>, #dlti.dl_entry<f128, dense<128> : vector<2xi64>>, #dlti.dl_entry<f64, dense<64> : vector<2xi64>>, #dlti.dl_entry<f16, dense<16> : vector<2xi64>>, #dlti.dl_entry<i32, dense<32> : vector<2xi64>>, #dlti.dl_entry<i16, dense<16> : vector<2xi64>>, #dlti.dl_entry<i8, dense<8> : vector<2xi64>>, #dlti.dl_entry<i1, dense<8> : vector<2xi64>>, #dlti.dl_entry<!llvm.ptr, dense<64> : vector<4xi64>>, #dlti.dl_entry<"dlti.endianness", "little">, #dlti.dl_entry<"dlti.stack_alignment", 128 : i64>>} {
+  gpu.module @cuda_device_mod {
+    gpu.func @_QMdevptrPtest() kernel {
+      gpu.return
+    }
+  }
+  func.func @_QMdevptrPtest() attributes {cuf.proc_attr = #cuf.cuda_proc<grid_global>} {
+    return
+  }
+  func.func @_QQmain() {
+    %c1_i32 = arith.constant 1 : i32
+    cuf.kernel_launch @_QMdevptrPtest<<<%c1_i32, %c1_i32, %c1_i32, %c1_i32, %c1_i32, %c1_i32>>>()
+    return
+  }
+}
+
+// CHECK-LABEL: func.func @_QQmain()
+// CHECK: gpu.launch_func  @cuda_device_mod::@_QMdevptrPtest blocks in (%{{.*}}, %{{.*}}, %{{.*}}) threads in (%{{.*}}, %{{.*}}, %{{.*}})  dynamic_shared_memory_size %{{.*}}  {cuf.proc_attr = #cuf.cuda_proc<grid_global>}
diff --git a/flang/test/Fir/arrayset.fir b/flang/test/Fir/arrayset.fir
index d201e34ce72af..dab939aba1702 100644
--- a/flang/test/Fir/arrayset.fir
+++ b/flang/test/Fir/arrayset.fir
@@ -1,7 +1,7 @@
 // RUN: tco %s | FileCheck %s
 // RUN: %flang_fc1 -emit-llvm %s -o - | FileCheck %s
 
-// CHECK-LABEL: define void @x(ptr nocapture %0)
+// CHECK-LABEL: define void @x(ptr captures(none) %0)
 func.func @x(%arr : !fir.ref<!fir.array<10xf32>>) {
   %1 = arith.constant 0 : index
   %2 = arith.constant 9 : index
diff --git a/flang/test/Fir/arrexp.fir b/flang/test/Fir/arrexp.fir
index 3363ead15d8a8..69fc77deb57e3 100644
--- a/flang/test/Fir/arrexp.fir
+++ b/flang/test/Fir/arrexp.fir
@@ -1,7 +1,7 @@
 // RUN: tco %s | FileCheck %s
 
-// CHECK-LINE: define void @f1
-// CHECK: (ptr nocapture %[[A:[^,]*]], {{.*}}, float %[[F:.*]])
+// CHECK-LABEL: define void @f1
+// CHECK: (ptr captures(none) %[[A:[^,]*]], {{.*}}, float %[[F:.*]])
 func.func @f1(%a : !fir.ref<!fir.array<?x?xf32>>, %n : index, %m : index, %o : index, %p : index, %f : f32) {
   %c1 = arith.constant 1 : index
   %s = fir.shape_shift %o, %n, %p, %m : (index, index, index, index) -> !fir.shapeshift<2>
@@ -22,8 +22,8 @@ func.func @f1(%a : !fir.ref<!fir.array<?x?xf32>>, %n : index, %m : index, %o : i
   return
 }
 
-// CHECK-LINE: define void @f2
-// CHECK: (ptr nocapture %[[A:[^,]*]], {{.*}}, float %[[F:.*]])
+// CHECK-LABEL: define void @f2
+// CHECK: (ptr captures(none) %[[A:[^,]*]], {{.*}}, float %[[F:.*]])
 func.func @f2(%a : !fir.ref<!fir.array<?x?xf32>>, %b : !fir.ref<!fir.array<?x?xf32>>, %n : index, %m : index, %o : index, %p : index, %f : f32) {
   %c1 = arith.constant 1 : index
   %s = fir.shape_shift %o, %n, %p, %m : (index, index, index, index) -> !fir.shapeshift<2>
@@ -46,8 +46,8 @@ func.func @f2(%a : !fir.ref<!fir.array<?x?xf32>>, %b : !fir.ref<!fir.array<?x?xf
   return
 }
 
-// CHECK-LINE: define void @f3
-// CHECK: (ptr nocapture %[[A:[^,]*]], {{.*}}, float %[[F:.*]])
+// CHECK-LABEL: define void @f3
+// CHECK: (ptr captures(none) %[[A:[^,]*]], {{.*}}, float %[[F:.*]])
 func.func @f3(%a : !fir.ref<!fir.array<?x?xf32>>, %b : !fir.ref<!fir.array<?x?xf32>>, %n : index, %m : index, %o : index, %p : index, %f : f32) {
   %c1 = arith.constant 1 : index
   %s = fir.shape_shift %o, %n, %p, %m : (index, index, index, index) -> !fir.shapeshift<2>
@@ -71,8 +71,8 @@ func.func @f3(%a : !fir.ref<!fir.array<?x?xf32>>, %b : !fir.ref<!fir.array<?x?xf
   return
 }
 
-// CHECK-LINE: define void @f4
-// CHECK: (ptr nocapture %[[A:[^,]*]], {{.*}}, float %[[F:.*]])
+// CHECK-LABEL: define void @f4
+// CHECK: (ptr captures(none) %[[A:[^,]*]], {{.*}}, float %[[F:.*]])
 func.func @f4(%a : !fir.ref<!fir.array<?x?xf32>>, %b : !fir.ref<!fir.array<?x?xf32>>, %n : index, %m : index, %o : index, %p : index, %f : f32) {
   %c1 = arith.constant 1 : index
   %s = fir.shape_shift %o, %n, %p, %m : (index, index, index, index) -> !fir.shapeshift<2>
@@ -101,7 +101,7 @@ func.func @f4(%a : !fir.ref<!fir.array<?x?xf32>>, %b : !fir.ref<!fir.array<?x?xf
 // Array expression assignment with potentially non contiguous arrays (e.g.
 // `a = b + f`, with and v assumed shapes.
 // Tests that the stride from the descriptor is used.
-// CHECK-LINE: define void @f5
+// CHECK-LABEL: define void @f5
 // CHECK: (ptr %[[A:.*]], ptr %[[B:.*]], float %[[F:.*]])
 func.func @f5(%arg0: !fir.box<!fir.array<?xf32>>, %arg1: !fir.box<!fir.array<?xf32>>, %arg2: f32) {
   %c0 = arith.constant 0 : index
@@ -134,7 +134,7 @@ func.func @f5(%arg0: !fir.box<!fir.array<?xf32>>, %arg1: !fir.box<!fir.array<?xf
 // Overlapping array expression assignment with a potentially non
 // contiguous array (e.g. `a(2:10:1) = a(1:9:1) + f`, with a assumed shape).
 // Test that a temp is created.
-// CHECK-LINE: define void @f6
+// CHECK-LABEL: define void @f6
 // CHECK: (ptr %[[A:[^,]*]], float %[[F:.*]])
 func.func @f6(%arg0: !fir.box<!fir.array<?xf32>>, %arg1: f32) {
   %c0 = arith.constant 0 : index
@@ -165,7 +165,7 @@ func.func @f6(%arg0: !fir.box<!fir.array<?xf32>>, %arg1: f32) {
 // Non contiguous array with lower bounds (x = y(100), with y(4:))
 // Test array_coor offset computation.
 // CHECK-LABEL:  define void @f7(
-// CHECK: ptr nocapture %[[X:[^,]*]], ptr %[[Y:.*]])
+// CHECK: ptr captures(none) %[[X:[^,]*]], ptr %[[Y:.*]])
 func.func @f7(%arg0: !fir.ref<f32>, %arg1: !fir.box<!fir.array<?xf32>>) {
   %c4 = arith.constant 4 : index
   %c100 = arith.constant 100 : index
@@ -181,7 +181,7 @@ func.func @f7(%arg0: !fir.ref<f32>, %arg1: !fir.box<!fir.array<?xf32>>) {
 
 // Test A(:, :)%x reference codegen with A constant shape.
 // CHECK-LABEL:  define void @f8(
-// CHECK-SAME: ptr nocapture %[[A:.*]], i32 %[[I:.*]])
+// CHECK-SAME: ptr captures(none) %[[A:.*]], i32 %[[I:.*]])
 func.func @f8(%a : !fir.ref<!fir.array<2x2x!fir.type<t{i:i32}>>>, %i : i32) {
   %c0 = arith.constant 0 : index
   %c1 = arith.constant 1 : index
@@ -198,7 +198,7 @@ func.func @f8(%a : !fir.ref<!fir.array<2x2x!fir.type<t{i:i32}>>>, %i : i32) {
 
 // Test casts in in array_coor offset computation when type parameters are not i64
 // CHECK-LABEL: define ptr @f9(
-// CHECK-SAME: i32 %[[I:.*]], i64 %{{.*}}, i64 %{{.*}}, ptr nocapture %[[C:.*]])
+// CHECK-SAME: i32 %[[I:.*]], i64 %{{.*}}, i64 %{{.*}}, ptr captures(none) %[[C:.*]])
 func.func @f9(%i: i32, %e : i64, %j: i64, %c: !fir.ref<!fir.array<?x?x!fir.char<1,?>>>) -> !fir.ref<!fir.char<1,?>> {
   %s = fir.shape %e, %e : (i64, i64) -> !fir.shape<2>
   // CHECK: %[[CAST:.*]] = sext i32 %[[I]] to i64
diff --git a/flang/test/Fir/box-offset-codegen.fir b/flang/test/Fir/box-offset-codegen.fir
index 5be3ad1f3eeed..15c9a11e5aefe 100644
--- a/flang/test/Fir/box-offset-codegen.fir
+++ b/flang/test/Fir/box-offset-codegen.fir
@@ -7,7 +7,7 @@ func.func @scalar_addr(%scalar : !fir.ref<!fir.box<!fir.type<t>>>) -> !fir.llvm_
   return %addr : !fir.llvm_ptr<!fir.ref<!fir.type<t>>>
 }
 // CHECK-LABEL: define ptr @scalar_addr(
-// CHECK-SAME: ptr nocapture %[[BOX:.*]]){{.*}}{
+// CHECK-SAME: ptr captures(none) %[[BOX:.*]]){{.*}}{
 // CHECK:    %[[VAL_0:.*]] = getelementptr { ptr, i64, i32, i8, i8, i8, i8, ptr, [1 x i64] }, ptr %[[BOX]], i32 0, i32 0
 // CHECK:    ret ptr %[[VAL_0]]
 
@@ -16,7 +16,7 @@ func.func @scalar_tdesc(%scalar : !fir.ref<!fir.box<!fir.type<t>>>) -> !fir.llvm
   return %tdesc : !fir.llvm_ptr<!fir.tdesc<!fir.type<t>>>
 }
 // CHECK-LABEL: define ptr @scalar_tdesc(
-// CHECK-SAME: ptr nocapture %[[BOX:.*]]){{.*}}{
+// CHECK-SAME: ptr captures(none) %[[BOX:.*]]){{.*}}{
 // CHECK:    %[[VAL_0:.*]] = getelementptr { ptr, i64, i32, i8, i8, i8, i8, ptr, [1 x i64] }, ptr %[[BOX]], i32 0, i32 7
 // CHECK:    ret ptr %[[VAL_0]]
 
@@ -25,7 +25,7 @@ func.func @array_addr(%array : !fir.ref<!fir.class<!fir.ptr<!fir.array<?x!fir.ty
   return %addr : !fir.llvm_ptr<!fir.ptr<!fir.array<?x!fir.type<t>>>>
 }
 // CHECK-LABEL: define ptr @array_addr(
-// CHECK-SAME: ptr nocapture %[[BOX:.*]]){{.*}}{
+// CHECK-SAME: ptr captures(none) %[[BOX:.*]]){{.*}}{
 // CHECK:    %[[VAL_0:.*]] = getelementptr { ptr, i64, i32, i8, i8, i8, i8, [1 x [3 x i64]], ptr, [1 x i64] }, ptr %[[BOX]], i32 0, i32 0
 // CHECK:    ret ptr %[[VAL_0]]
 
@@ -34,6 +34,6 @@ func.func @array_tdesc(%array : !fir.ref<!fir.class<!fir.ptr<!fir.array<?x!fir.t
   return %tdesc : !fir.llvm_ptr<!fir.tdesc<!fir.type<t>>>
 }
 // CHECK-LABEL: define ptr @array_tdesc(
-// CHECK-SAME: ptr nocapture %[[BOX:.*]]){{.*}}{
+// CHECK-SAME: ptr captures(none) %[[BOX:.*]]){{.*}}{
 // CHECK:    %[[VAL_0:.*]] = getelementptr { ptr, i64, i32, i8, i8, i8, i8, [1 x [3 x i64]], ptr, [1 x i64] }, ptr %[[BOX]], i32 0, i32 8
 // CHECK:    ret ptr %[[VAL_0]]
diff --git a/flang/test/Fir/box.fir b/flang/test/Fir/box.fir
index 841e10d3debbc..d4a5157888367 100644
--- a/flang/test/Fir/box.fir
+++ b/flang/test/Fir/box.fir
@@ -24,7 +24,7 @@ func.func private @g(%b : !fir.box<f32>)
 func.func private @ga(%b : !fir.box<!fir.array<?xf32>>)
 
 // CHECK-LABEL: define void @f
-// CHECK: (ptr nocapture %[[ARG:.*]])
+// CHECK: (ptr captures(none) %[[ARG:.*]])
 func.func @f(%a : !fir.ref<f32>) {
   // CHECK: %[[DESC:.*]] = alloca { ptr, i64, i32, i8, i8, i8, i8 }
   // CHECK: %[[INS0:.*]] = insertvalue {{.*}} { ptr undef, i64 ptrtoint (ptr getelementptr (float, ptr null, i32 1) to i64), i32 20240719, i8 0, i8 27, i8 0, i8 0 }, ptr %[[ARG]], 0
@@ -38,7 +38,7 @@ func.func @f(%a : !fir.ref<f32>) {
 }
 
 // CHECK-LABEL: define void @fa
-// CHECK: (ptr nocapture %[[ARG:.*]])
+// CHECK: (ptr captures(none) %[[ARG:.*]])
 func.func @fa(%a : !fir.ref<!fir.array<100xf32>>) {
   %c = fir.convert %a : (!fir.ref<!fir.array<100xf32>>) -> !fir.ref<!fir.array<?xf32>>
   %c1 = arith.constant 1 : index
@@ -54,7 +54,7 @@ func.func @fa(%a : !fir.ref<!fir.array<100xf32>>) {
 
 // Boxing of a scalar character of dynamic length
 // CHECK-LABEL: define void @b1(
-// CHECK-SAME: ptr nocapture %[[res:.*]], ptr nocapture %[[arg0:.*]], i64 %[[arg1:.*]])
+// CHECK-SAME: ptr captures(none) %[[res:.*]], ptr captures(none) %[[arg0:.*]], i64 %[[arg1:.*]])
 func.func @b1(%arg0 : !fir.ref<!fir.char<1,?>>, %arg1 : index) -> !fir.box<!fir.char<1,?>> {
   // CHECK: %[[alloca:.*]] = alloca { ptr, i64, i32, i8, i8, i8, i8 }
   // CHECK: %[[size:.*]] = mul i64 ptrtoint (ptr getelementptr (i8, ptr null, i32 1) to i64), %[[arg1]]
@@ -69,8 +69,8 @@ func.func @b1(%arg0 : !fir.ref<!fir.char<1,?>>, %arg1 : index) -> !fir.box<!fir.
 
 // Boxing of a dynamic array of character with static length (5)
 // CHECK-LABEL: define void @b2(
-// CHECK-SAME: ptr nocapture %[[res]],
-// CHECK-SAME: ptr nocapture %[[arg0:.*]], i64 %[[arg1:.*]])
+// CHECK-SAME: ptr captures(none) %[[res]],
+// CHECK-SAME: ptr captures(none) %[[arg0:.*]], i64 %[[arg1:.*]])
 func.func @b2(%arg0 : !fir.ref<!fir.array<?x!fir.char<1,5>>>, %arg1 : index) -> !fir.box<!fir.array<?x!fir.char<1,5>>> {
   %1 = fir.shape %arg1 : (index) -> !fir.shape<1>
   // CHECK: %[[alloca:.*]] = alloca { ptr, i64, i32, i8, i8, i8, i8, [1 x [3 x i64]] }
@@ -85,7 +85,7 @@ func.func @b2(%arg0 : !fir.ref<!fir.array<?x!fir.char<1,5>>>, %arg1 : index) ->
 
 // Boxing of a dynamic array of character of dynamic length
 // CHECK-LABEL: define void @b3(
-// CHECK-SAME: ptr nocapture %[[res:.*]], ptr nocapture %[[arg0:.*]], i64 %[[arg1:.*]], i64 %[[arg2:.*]])
+// CHECK-SAME: ptr captures(none) %[[res:.*]], ptr captures(none) %[[arg0:.*]], i64 %[[arg1:.*]], i64 %[[arg2:.*]])
 func.func @b3(%arg0 : !fir.ref<!fir.array<?x!fir.char<1,?>>>, %arg1 : index, %arg2 : index) -> !fir.box<!fir.array<?x!fir.char<1,?>>> {
   %1 = fir.shape %arg2 : (index) -> !fir.shape<1>
   // CHECK: %[[alloca:.*]] = alloca { ptr, i64, i32, i8, i8, i8, i8, [1 x [3 x i64]] }
@@ -103,7 +103,7 @@ func.func @b3(%arg0 : !fir.ref<!fir.array<?x!fir.char<1,?>>>, %arg1 : index, %ar
 
 // Boxing of a static array of character of dynamic length
 // CHECK-LABEL: define void @b4(
-// CHECK-SAME: ptr nocapture %[[res:.*]], ptr nocapture %[[arg0:.*]], i64 %[[arg1:.*]])
+// CHECK-SAME: ptr captures(none) %[[res:.*]], ptr captures(none) %[[arg0:.*]], i64 %[[arg1:.*]])
 func.func @b4(%arg0 : !fir.ref<!fir.array<7x!fir.char<1,?>>>, %arg1 : index) -> !fir.box<!fir.array<7x!fir.char<1,?>>> {
   %c_7 = arith.constant 7 : index
   %1 = fir.shape %c_7 : (index) -> !fir.shape<1>
@@ -122,7 +122,7 @@ func.func @b4(%arg0 : !fir.ref<!fir.array<7x!fir.char<1,?>>>, %arg1 : index) ->
 
 // Storing a fir.box into a fir.ref<fir.box> (modifying descriptors).
 // CHECK-LABEL: define void @b5(
-// CHECK-SAME: ptr nocapture %[[arg0:.*]], ptr %[[arg1:.*]])
+// CHECK-SAME: ptr captures(none) %[[arg0:.*]], ptr %[[arg1:.*]])
 func.func @b5(%arg0 : !fir.ref<!fir.box<!fir.heap<!fir.array<?x?xf32>>>>, %arg1 : !fir.box<!fir.heap<!fir.array<?x?xf32>>>) {
   fir.store %arg1 to %arg0 : !fir.ref<!fir.box<!fir.heap<!fir.array<?x?xf32>>>>
   // CHECK: call void @llvm.memcpy.p0.p0.i32(ptr %0, ptr %1, i32 72, i1 false)
@@ -132,7 +132,7 @@ func.func @b5(%arg0 : !fir.ref<!fir.box<!fir.heap<!fir.array<?x?xf32>>>>, %arg1
 func.func private @callee6(!fir.box<none>) -> i32
 
 // CHECK-LABEL: define i32 @box6(
-// CHECK-SAME: ptr nocapture %[[ARG0:.*]], i64 %[[ARG1:.*]], i64 %[[ARG2:.*]])
+// CHECK-SAME: ptr captures(none) %[[ARG0:.*]], i64 %[[ARG1:.*]], i64 %[[ARG2:.*]])
 func.func @box6(%0 : !fir.ref<!fir.array<?x?x?x?xf32>>, %1 : index, %2 : index) -> i32 {
   %c100 = arith.constant 100 : index
   %c50 = arith.constant 50 : index
diff --git a/flang/test/Fir/boxproc-openmp.fir b/flang/test/Fir/boxproc-openmp.fir
index 9db053ad93c66..4f62b0a4a42b2 100644
--- a/flang/test/Fir/boxproc-openmp.fir
+++ b/flang/test/Fir/boxproc-openmp.fir
@@ -3,26 +3,13 @@
 // Check minimally, only arguments, yields and the private types.
 
 // Test a private declaration with one region (alloc)
-//CHECK: omp.private {type = private}  @_QFsub1Et1_private_ref_rec__QFsub1Tt : !fir.ref<!fir.type<_QFsub1TtUnboxProc{p1:() -> ()}>> alloc {
-omp.private {type = private} @_QFsub1Et1_private_ref_rec__QFsub1Tt : !fir.ref<!fir.type<_QFsub1Tt{p1:!fir.boxproc<() -> ()>}>> alloc {
-//CHECK: ^bb0(%{{.*}}: !fir.ref<!fir.type<_QFsub1TtUnboxProc{p1:() -> ()}>>):
-^bb0(%arg0: !fir.ref<!fir.type<_QFsub1Tt{p1:!fir.boxproc<() -> ()>}>>):
-  %c1_i32 = arith.constant 1 : i32
-  %0 = fir.alloca !fir.type<_QFsub1Tt{p1:!fir.boxproc<() -> ()>}> {bindc_name = "t1", pinned, uniq_name = "_QFsub1Et1"}
-  %1 = fir.declare %0 {uniq_name = "_QFsub1Et1"} : (!fir.ref<!fir.type<_QFsub1Tt{p1:!fir.boxproc<() -> ()>}>>) -> !fir.ref<!fir.type<_QFsub1Tt{p1:!fir.boxproc<() -> ()>}>>
-  %2 = fir.embox %1 : (!fir.ref<!fir.type<_QFsub1Tt{p1:!fir.boxproc<() -> ()>}>>) -> !fir.box<!fir.type<_QFsub1Tt{p1:!fir.boxproc<() -> ()>}>>
-  %3 = fir.address_of(@_QQclXea6256ba131ddd9c2210e68030a0edd3) : !fir.ref<!fir.char<1,49>>
-  %4 = fir.convert %2 : (!fir.box<!fir.type<_QFsub1Tt{p1:!fir.boxproc<() -> ()>}>>) -> !fir.box<none>
-  %5 = fir.convert %3 : (!fir.ref<!fir.char<1,49>>) -> !fir.ref<i8>
-  fir.call @_FortranAInitialize(%4, %5, %c1_i32) fastmath<contract> : (!fir.box<none>, !fir.ref<i8>, i32) -> ()
-//CHECK: omp.yield(%{{.*}} : !fir.ref<!fir.type<_QFsub1TtUnboxProc{p1:() -> ()}>>)
-  omp.yield(%1 : !fir.ref<!fir.type<_QFsub1Tt{p1:!fir.boxproc<() -> ()>}>>)
-}
+//CHECK: omp.private {type = private}  @_QFsub1Et1_private_rec__QFsub1Tt : !fir.type<_QFsub1TtUnboxProc{p1:() -> ()}>{{$}}
+omp.private {type = private} @_QFsub1Et1_private_rec__QFsub1Tt : !fir.type<_QFsub1Tt{p1:!fir.boxproc<() -> ()>}>
 func.func @_QPsub1() {
   %0 = fir.alloca !fir.type<_QFsub1Tt{p1:!fir.boxproc<() -> ()>}> {bindc_name = "t1", uniq_name = "_QFsub1Et1"}
   %1 = fir.declare %0 {uniq_name = "_QFsub1Et1"} : (!fir.ref<!fir.type<_QFsub1Tt{p1:!fir.boxproc<() -> ()>}>>) -> !fir.ref<!fir.type<_QFsub1Tt{p1:!fir.boxproc<() -> ()>}>>
-//CHECK: omp.parallel private(@_QFsub1Et1_private_ref_rec__QFsub1Tt %{{.*}} -> %{{.*}} : !fir.ref<!fir.type<_QFsub1TtUnboxProc{p1:() -> ()}>>) {
-  omp.parallel private(@_QFsub1Et1_private_ref_rec__QFsub1Tt %1 -> %arg0 : !fir.ref<!fir.type<_QFsub1Tt{p1:!fir.boxproc<() -> ()>}>>) {
+//CHECK: omp.parallel private(@_QFsub1Et1_private_rec__QFsub1Tt %{{.*}} -> %{{.*}} : !fir.ref<!fir.type<_QFsub1TtUnboxProc{p1:() -> ()}>>) {
+  omp.parallel private(@_QFsub1Et1_private_rec__QFsub1Tt %1 -> %arg0 : !fir.ref<!fir.type<_QFsub1Tt{p1:!fir.boxproc<() -> ()>}>>) {
     %2 = fir.declare %arg0 {uniq_name = "_QFsub1Et1"} : (!fir.ref<!fir.type<_QFsub1Tt{p1:!fir.boxproc<() -> ()>}>>) -> !fir.ref<!fir.type<_QFsub1Tt{p1:!fir.boxproc<() -> ()>}>>
     omp.terminator
   }
@@ -31,11 +18,11 @@ func.func @_QPsub1() {
 
 
 // Test a private declaration with all regions (alloc, copy, dealloc)
-//CHECK: omp.private {type = firstprivate} @_QFsub2Et1_firstprivate_ref_box_heap_rec__QFsub2Tt : 
-//CHECK-SAME: !fir.ref<!fir.box<!fir.heap<!fir.type<_QFsub2TtUnboxProc{p1:() -> ()}>>>> alloc {
-omp.private {type = firstprivate} @_QFsub2Et1_firstprivate_ref_box_heap_rec__QFsub2Tt : !fir.ref<!fir.box<!fir.heap<!fir.type<_QFsub2Tt{p1:!fir.boxproc<() -> ()>}>>>> alloc {
-//CHECK: ^bb0(%{{.*}}: !fir.ref<!fir.box<!fir.heap<!fir.type<_QFsub2TtUnboxProc{p1:() -> ()}>>>>):
-^bb0(%arg0: !fir.ref<!fir.box<!fir.heap<!fir.type<_QFsub2Tt{p1:!fir.boxproc<() -> ()>}>>>>):
+//CHECK: omp.private {type = firstprivate} @_QFsub2Et1_firstprivate_box_heap_rec__QFsub2Tt :
+//CHECK-SAME: [[TYPE:!fir.box<!fir.heap<!fir.type<_QFsub2TtUnboxProc\{p1:\(\) -> \(\)\}>>>]] init {
+omp.private {type = firstprivate} @_QFsub2Et1_firstprivate_box_heap_rec__QFsub2Tt : !fir.box<!fir.heap<!fir.type<_QFsub2Tt{p1:!fir.boxproc<() -> ()>}>>> init {
+//CHECK: ^bb0(%{{.*}}: !fir.ref<[[TYPE]]>, %{{.*}}: !fir.ref<[[TYPE]]>):
+^bb0(%arg0: !fir.ref<!fir.box<!fir.heap<!fir.type<_QFsub2Tt{p1:!fir.boxproc<() -> ()>}>>>>, %arg1:!fir.ref<!fir.box<!fir.heap<!fir.type<_QFsub2Tt{p1:!fir.boxproc<() -> ()>}>>>>):
   %0 = fir.alloca !fir.box<!fir.heap<!fir.type<_QFsub2Tt{p1:!fir.boxproc<() -> ()>}>>> {bindc_name = "t1", pinned, uniq_name = "_QFsub2Et1"}
   %1 = fir.declare %0 {fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "_QFsub2Et1"} : (!fir.ref<!fir.box<!fir.heap<!fir.type<_QFsub2Tt{p1:!fir.boxproc<() -> ()>}>>>>) -> !fir.ref<!fir.box<!fir.heap<!fir.type<_QFsub2Tt{p1:!fir.boxproc<() -> ()>}>>>>
 //CHECK: omp.yield(%{{.*}} : !fir.ref<!fir.box<!fir.heap<!fir.type<_QFsub2TtUnboxProc{p1:() -> ()}>>>>)
@@ -70,9 +57,9 @@ omp.private {type = firstprivate} @_QFsub2Et1_firstprivate_ref_box_heap_rec__QFs
 func.func @_QPsub2() {
   %0 = fir.alloca !fir.box<!fir.heap<!fir.type<_QFsub2Tt{p1:!fir.boxproc<() -> ()>}>>> {bindc_name = "t1", uniq_name = "_QFsub2Et1"}
   %1 = fir.declare %0 {fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "_QFsub2Et1"} : (!fir.ref<!fir.box<!fir.heap<!fir.type<_QFsub2Tt{p1:!fir.boxproc<() -> ()>}>>>>) -> !fir.ref<!fir.box<!fir.heap<!fir.type<_QFsub2Tt{p1:!fir.boxproc<() -> ()>}>>>>
-//CHECK: omp.parallel private(@_QFsub2Et1_firstprivate_ref_box_heap_rec__QFsub2Tt %{{.*}} -> %{{.*}} :
+//CHECK: omp.parallel private(@_QFsub2Et1_firstprivate_box_heap_rec__QFsub2Tt %{{.*}} -> %{{.*}} :
 //CHECK-SAME: !fir.ref<!fir.box<!fir.heap<!fir.type<_QFsub2TtUnboxProc{p1:() -> ()}>>>>) {
-  omp.parallel private(@_QFsub2Et1_firstprivate_ref_box_heap_rec__QFsub2Tt %1 -> %arg0 : !fir.ref<!fir.box<!fir.heap<!fir.type<_QFsub2Tt{p1:!fir.boxproc<() -> ()>}>>>>) {
+  omp.parallel private(@_QFsub2Et1_firstprivate_box_heap_rec__QFsub2Tt %1 -> %arg0 : !fir.ref<!fir.box<!fir.heap<!fir.type<_QFsub2Tt{p1:!fir.boxproc<() -> ()>}>>>>) {
     %2 = fir.declare %arg0 {fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "_QFsub2Et1"} : (!fir.ref<!fir.box<!fir.heap<!fir.type<_QFsub2Tt{p1:!fir.boxproc<() -> ()>}>>>>) -> !fir.ref<!fir.box<!fir.heap<!fir.type<_QFsub2Tt{p1:!fir.boxproc<() -> ()>}>>>>
     omp.terminator
   }
diff --git a/flang/test/Fir/boxproc.fir b/flang/test/Fir/boxproc.fir
index 27d8953236e72..e99dfd0b92afd 100644
--- a/flang/test/Fir/boxproc.fir
+++ b/flang/test/Fir/boxproc.fir
@@ -1,7 +1,11 @@
-// RUN: tco %s | FileCheck %s
+// RUN: %if aarch64-registered-target %{tco --target=aarch64-unknown-linux-gnu %s | FileCheck %s --check-prefixes=CHECK,CHECK-AARCH64 %}
+// RUN: %if x86-registered-target %{tco --target=x86_64-unknown-linux-gnu %s | FileCheck %s --check-prefixes=CHECK,CHECK-X86 %}
+// RUN: %if powerpc-registered-target %{tco --target=powerpc64le-unknown-linux-gnu %s | FileCheck %s --check-prefixes=CHECK,CHECK-PPC %}
 
 // CHECK-LABEL: define void @_QPtest_proc_dummy()
-// CHECK:         %[[VAL_3:.*]] = alloca [32 x i8], i64 1, align 1
+// CHECK-AARCH64: %[[VAL_3:.*]] = alloca [36 x i8], i64 1, align 1
+// CHECK-X86:     %[[VAL_3:.*]] = alloca [32 x i8], i64 1, align 1
+// CHECK-PPC:     %[[VAL_3:.*]] = alloca [4{{[0-8]+}} x i8], i64 1, align 1
 // CHECK:         %[[VAL_1:.*]] = alloca { ptr }, i64 1, align 8
 // CHECK:         %[[VAL_0:.*]] = alloca i32, i64 1, align 4
 // CHECK:         %[[VAL_2:.*]] = getelementptr { ptr }, ptr %[[VAL_1]], i32 0, i32 0
@@ -12,7 +16,7 @@
 // CHECK:         call void @_QPtest_proc_dummy_other(ptr %[[VAL_6]])
 
 // CHECK-LABEL: define void @_QFtest_proc_dummyPtest_proc_dummy_a(ptr
-// CHECK-SAME:              nocapture %[[VAL_0:.*]], ptr nest nocapture %[[VAL_1:.*]])
+// CHECK-SAME:              captures(none) %[[VAL_0:.*]], ptr nest captures(none) %[[VAL_1:.*]])
 
 // CHECK-LABEL: define void @_QPtest_proc_dummy_other(ptr
 // CHECK-SAME:              %[[VAL_0:.*]])
@@ -59,7 +63,9 @@ func.func @_QPtest_proc_dummy_other(%arg0: !fir.boxproc<() -> ()>) {
 }
 
 // CHECK-LABEL: define void @_QPtest_proc_dummy_char()
-// CHECK:         %[[VAL_20:.*]] = alloca [32 x i8], i64 1, align 1
+// CHECK-AARCH64: %[[VAL_20:.*]] = alloca [36 x i8], i64 1, align 1
+// CHECK-X86:     %[[VAL_20:.*]] = alloca [32 x i8], i64 1, align 1
+// CHECK-PPC:     %[[VAL_20:.*]] = alloca [4{{[0-8]+}} x i8], i64 1, align 1
 // CHECK:         %[[VAL_2:.*]] = alloca { { ptr, i64 } }, i64 1, align 8
 // CHECK:         %[[VAL_1:.*]] = alloca [10 x i8], i64 1, align 1
 // CHECK:         %[[VAL_0:.*]] = alloca [40 x i8], i64 1, align 1
@@ -86,7 +92,7 @@ func.func @_QPtest_proc_dummy_other(%arg0: !fir.boxproc<() -> ()>) {
 // CHECK:         call void @llvm.stackrestore.p0(ptr %[[VAL_27]])
 
 // CHECK-LABEL: define { ptr, i64 } @_QFtest_proc_dummy_charPgen_message(ptr
-// CHECK-SAME:                nocapture %[[VAL_0:.*]], i64 %[[VAL_1:.*]], ptr nest nocapture %[[VAL_2:.*]])
+// CHECK-SAME:                captures(none) %[[VAL_0:.*]], i64 %[[VAL_1:.*]], ptr nest captures(none) %[[VAL_2:.*]])
 // CHECK:         %[[VAL_3:.*]] = getelementptr { { ptr, i64 } }, ptr %[[VAL_2]], i32 0, i32 0
 // CHECK:         %[[VAL_4:.*]] = load { ptr, i64 }, ptr %[[VAL_3]], align 8
 // CHECK:         %[[VAL_5:.*]] = extractvalue { ptr, i64 } %[[VAL_4]], 0
diff --git a/flang/test/Fir/commute.fir b/flang/test/Fir/commute.fir
index 75338161d2d99..a857ba55b00c5 100644
--- a/flang/test/Fir/commute.fir
+++ b/flang/test/Fir/commute.fir
@@ -11,7 +11,7 @@ func.func @f1(%a : i32, %b : i32) -> i32 {
   return %3 : i32
 }
 
-// CHECK-LABEL: define i32 @f2(ptr nocapture %0)
+// CHECK-LABEL: define i32 @f2(ptr captures(none) %0)
 func.func @f2(%a : !fir.ref<i32>) -> i32 {
   %1 = fir.load %a : !fir.ref<i32>
   // CHECK: %[[r2:.*]] = load
diff --git a/flang/test/Fir/convert-to-llvm.fir b/flang/test/Fir/convert-to-llvm.fir
index 4c9f965e1241a..6d7a4a09918e5 100644
--- a/flang/test/Fir/convert-to-llvm.fir
+++ b/flang/test/Fir/convert-to-llvm.fir
@@ -1909,6 +1909,67 @@ func.func @xembox0(%arg0: !fir.ref<!fir.array<?xi32>>) {
 // CHECK:         %[[BOX10:.*]] = llvm.insertvalue %[[BASE_PTR]], %[[BOX9]][0] : !llvm.struct<(ptr, i{{.*}}, i{{.*}}, i{{.*}}, i{{.*}}, i{{.*}}, i{{.*}}, array<1 x array<3 x i64>>)>
 // CHECK:         llvm.store %[[BOX10]], %[[ALLOCA]] : !llvm.struct<(ptr, i{{.*}}, i{{.*}}, i{{.*}}, i{{.*}}, i{{.*}}, i{{.*}}, array<1 x array<3 x i64>>)>, !llvm.ptr
 
+// Test i32 shape/shift/slice:
+func.func @xembox0_i32(%arg0: !fir.ref<!fir.array<?xi32>>) {
+  %c0 = arith.constant 0 : i32
+  %c0_i64 = arith.constant 0 : i64
+  %0 = fircg.ext_embox %arg0(%c0) origin %c0[%c0, %c0, %c0] : (!fir.ref<!fir.array<?xi32>>, i32, i32, i32, i32, i32) -> !fir.box<!fir.array<?xi32>>
+  return
+}
+// CHECK-LABEL: llvm.func @xembox0_i32(
+// CHECK-SAME:                     %[[ARG0:.*]]: !llvm.ptr
+// CHECK:         %[[ALLOCA_SIZE:.*]] = llvm.mlir.constant(1 : i32) : i32
+// GENERIC:       %[[ALLOCA:.*]] = llvm.alloca %[[ALLOCA_SIZE]] x !llvm.struct<(ptr, i{{.*}}, i{{.*}}, i{{.*}}, i{{.*}}, i{{.*}}, i{{.*}}, array<1 x array<3 x i64>>)> {alignment = 8 : i64} : (i32) -> !llvm.ptr
+// AMDGPU:        %[[AA:.*]] = llvm.alloca %[[ALLOCA_SIZE]] x !llvm.struct<(ptr, i{{.*}}, i{{.*}}, i{{.*}}, i{{.*}}, i{{.*}}, i{{.*}}, array<1 x array<3 x i64>>)> {alignment = 8 : i64} : (i32) -> !llvm.ptr<5>
+// AMDGPU:        %[[ALLOCA:.*]] = llvm.addrspacecast %[[AA]] : !llvm.ptr<5> to !llvm.ptr
+// CHECK:         %[[C0_I32:.*]] = llvm.mlir.constant(0 : i32) : i32
+// CHECK:         %[[C0:.*]] = llvm.mlir.constant(0 : i64) : i64
+// CHECK:         %[[TYPE:.*]] = llvm.mlir.constant(9 : i32) : i32
+// CHECK:         %[[NULL:.*]] = llvm.mlir.zero : !llvm.ptr
+// CHECK:         %[[GEP:.*]] = llvm.getelementptr %[[NULL]][1]
+// CHECK:         %[[ELEM_LEN_I64:.*]] = llvm.ptrtoint %[[GEP]] : !llvm.ptr to i64
+// CHECK:         %[[BOX0:.*]] = llvm.mlir.undef : !llvm.struct<(ptr, i{{.*}}, i{{.*}}, i{{.*}}, i{{.*}}, i{{.*}}, i{{.*}}, array<1 x array<3 x i64>>)>
+// CHECK:         %[[BOX1:.*]] = llvm.insertvalue %[[ELEM_LEN_I64]], %[[BOX0]][1] : !llvm.struct<(ptr, i{{.*}}, i{{.*}}, i{{.*}}, i{{.*}}, i{{.*}}, i{{.*}}, array<1 x array<3 x i64>>)>
+// CHECK:         %[[VERSION:.*]] = llvm.mlir.constant(20240719 : i32) : i32
+// CHECK:         %[[BOX2:.*]] = llvm.insertvalue %[[VERSION]], %[[BOX1]][2] : !llvm.struct<(ptr, i{{.*}}, i{{.*}}, i{{.*}}, i{{.*}}, i{{.*}}, i{{.*}}, array<1 x array<3 x i64>>)>
+// CHECK:         %[[RANK:.*]] = llvm.mlir.constant(1 : i32) : i32
+// CHECK:         %[[RANK_I8:.*]] = llvm.trunc %[[RANK]] : i32 to i8
+// CHECK:         %[[BOX3:.*]] = llvm.insertvalue %[[RANK_I8]], %[[BOX2]][3] : !llvm.struct<(ptr, i{{.*}}, i{{.*}}, i{{.*}}, i{{.*}}, i{{.*}}, i{{.*}}, array<1 x array<3 x i64>>)>
+// CHECK:         %[[TYPE_I8:.*]] = llvm.trunc %[[TYPE]] : i32 to i8
+// CHECK:         %[[BOX4:.*]] = llvm.insertvalue %[[TYPE_I8]], %[[BOX3]][4] : !llvm.struct<(ptr, i{{.*}}, i{{.*}}, i{{.*}}, i{{.*}}, i{{.*}}, i{{.*}}, array<1 x array<3 x i64>>)>
+// CHECK:         %[[ATTR:.*]] = llvm.mlir.constant(0 : i32) : i32
+// CHECK:         %[[ATTR_I8:.*]] = llvm.trunc %[[ATTR]] : i32 to i8
+// CHECK:         %[[BOX5:.*]] = llvm.insertvalue %[[ATTR_I8]], %[[BOX4]][5] : !llvm.struct<(ptr, i{{.*}}, i{{.*}}, i{{.*}}, i{{.*}}, i{{.*}}, i{{.*}}, array<1 x array<3 x i64>>)>
+// CHECK:         %[[F18ADDENDUM:.*]] = llvm.mlir.constant(0 : i32) : i32
+// CHECK:         %[[F18ADDENDUM_I8:.*]] = llvm.trunc %[[F18ADDENDUM]] : i32 to i8
+// CHECK:         %[[BOX6:.*]] = llvm.insertvalue %[[F18ADDENDUM_I8]], %[[BOX5]][6] : !llvm.struct<(ptr, i{{.*}}, i{{.*}}, i{{.*}}, i{{.*}}, i{{.*}}, i{{.*}}, array<1 x array<3 x i64>>)>
+// CHECK:         %[[ZERO:.*]] = llvm.mlir.constant(0 : i64) : i64
+// CHECK:         %[[ONE:.*]] = llvm.mlir.constant(1 : i64) : i64
+// CHECK:         %[[C0_1:.*]] = llvm.sext %[[C0_I32]] : i32 to i64
+// CHECK:         %[[C0_2:.*]] = llvm.sext %[[C0_I32]] : i32 to i64
+// CHECK:         %[[C0_3:.*]] = llvm.sext %[[C0_I32]] : i32 to i64
+// CHECK:         %[[ADJUSTED_OFFSET:.*]] = llvm.sub %[[C0_2]], %[[C0_3]]  : i64
+// CHECK:         %[[DIM_OFFSET:.*]] = llvm.mul %[[ADJUSTED_OFFSET]], %[[ONE]]  : i64
+// CHECK:         %[[PTR_OFFSET:.*]] = llvm.add %[[DIM_OFFSET]], %[[ZERO]]  : i64
+// CHECK:         %[[C0_4:.*]] = llvm.sext %[[C0_I32]] : i32 to i64
+// CHECK:         %[[C0_5:.*]] = llvm.sext %[[C0_I32]] : i32 to i64
+// CHECK:         %[[C0_6:.*]] = llvm.sext %[[C0_I32]] : i32 to i64
+// CHECK:         %[[EXTENT0:.*]] = llvm.sub %[[C0_5]], %[[C0_4]]  : i64
+// CHECK:         %[[EXTENT1:.*]] = llvm.add %[[EXTENT0]], %[[C0_6]]  : i64
+// CHECK:         %[[EXTENT2:.*]] = llvm.sdiv %[[EXTENT1]], %[[C0_6]]  : i64
+// CHECK:         %[[EXTENT_CMP:.*]] = llvm.icmp "sgt" %[[EXTENT2]], %[[ZERO]] : i64
+// CHECK:         %[[EXTENT:.*]] = llvm.select %[[EXTENT_CMP]], %[[EXTENT2]], %[[ZERO]] : i1, i64
+// CHECK:         %[[BOX7:.*]] = llvm.insertvalue %[[ONE]], %[[BOX6]][7, 0, 0] : !llvm.struct<(ptr, i{{.*}}, i{{.*}}, i{{.*}}, i{{.*}}, i{{.*}}, i{{.*}}, array<1 x array<3 x i64>>)>
+// CHECK:         %[[BOX8:.*]] = llvm.insertvalue %[[EXTENT]], %[[BOX7]][7, 0, 1] : !llvm.struct<(ptr, i{{.*}}, i{{.*}}, i{{.*}}, i{{.*}}, i{{.*}}, i{{.*}}, array<1 x array<3 x i64>>)>
+// CHECK:         %[[C0_7:.*]] = llvm.sext %[[C0_I32]] : i32 to i64
+// CHECK:         %[[STRIDE:.*]] = llvm.mul %[[ELEM_LEN_I64]], %[[C0_7]]  : i64
+// CHECK:         %[[BOX9:.*]] = llvm.insertvalue %[[STRIDE]], %[[BOX8]][7, 0, 2] : !llvm.struct<(ptr, i{{.*}}, i{{.*}}, i{{.*}}, i{{.*}}, i{{.*}}, i{{.*}}, array<1 x array<3 x i64>>)>
+// CHECK:         %[[PREV_DIM:.*]] = llvm.mul %[[ELEM_LEN_I64]], %[[C0_1]]  : i64
+// CHECK:         %[[PREV_PTROFF:.*]] = llvm.mul %[[ONE]], %[[C0_1]]  : i64
+// CHECK:         %[[BASE_PTR:.*]] = llvm.getelementptr %[[ARG0]][%[[PTR_OFFSET]]] : (!llvm.ptr, i64) -> !llvm.ptr, i32
+// CHECK:         %[[BOX10:.*]] = llvm.insertvalue %[[BASE_PTR]], %[[BOX9]][0] : !llvm.struct<(ptr, i{{.*}}, i{{.*}}, i{{.*}}, i{{.*}}, i{{.*}}, i{{.*}}, array<1 x array<3 x i64>>)>
+// CHECK:         llvm.store %[[BOX10]], %[[ALLOCA]] : !llvm.struct<(ptr, i{{.*}}, i{{.*}}, i{{.*}}, i{{.*}}, i{{.*}}, i{{.*}}, array<1 x array<3 x i64>>)>, !llvm.ptr
+
 // Check adjustment of element scaling factor.
 
 func.func @xembox1(%arg0: !fir.ref<!fir.array<?x!fir.char<1, 10>>>) {
diff --git a/flang/test/Fir/coordinateof.fir b/flang/test/Fir/coordinateof.fir
index f96f5b48bb355..693bdf716ba1d 100644
--- a/flang/test/Fir/coordinateof.fir
+++ b/flang/test/Fir/coordinateof.fir
@@ -62,7 +62,7 @@ func.func @foo5(%box : !fir.box<!fir.ptr<!fir.array<?xi32>>>, %i : index) -> i32
 }
 
 // CHECK-LABEL: @foo6
-// CHECK-SAME: (ptr %[[box:.*]], i64 %{{.*}}, ptr nocapture %{{.*}}) 
+// CHECK-SAME: (ptr %[[box:.*]], i64 %{{.*}}, ptr captures(none) %{{.*}}) 
 func.func @foo6(%box : !fir.box<!fir.ptr<!fir.array<?x!fir.char<1>>>>, %i : i64 , %res : !fir.ref<!fir.char<1>>) {
   // CHECK: %[[addr_gep:.*]] = getelementptr { ptr, i64, i32, i8, i8, i8, i8, [1 x [3 x i64]] }, ptr %[[box]], i32 0, i32 0
   // CHECK: %[[addr:.*]] = load ptr, ptr %[[addr_gep]]
diff --git a/flang/test/Fir/embox.fir b/flang/test/Fir/embox.fir
index b21e1349b0552..0b8bc3fd4be14 100644
--- a/flang/test/Fir/embox.fir
+++ b/flang/test/Fir/embox.fir
@@ -64,7 +64,7 @@ func.func @_QPtest_dt_slice() {
 func.func private @takesRank2CharBox(!fir.box<!fir.array<?x?x!fir.char<1,?>>>)
 
 // CHECK-LABEL: define void @emboxSubstring(
-// CHECK-SAME: ptr nocapture %[[arg0:.*]])
+// CHECK-SAME: ptr captures(none) %[[arg0:.*]])
 func.func @emboxSubstring(%arg0: !fir.ref<!fir.array<2x3x!fir.char<1,4>>>) {
   %c2 = arith.constant 2 : index
   %c3 = arith.constant 3 : index
@@ -85,7 +85,7 @@ func.func @emboxSubstring(%arg0: !fir.ref<!fir.array<2x3x!fir.char<1,4>>>) {
 
 func.func private @do_something(!fir.box<!fir.array<?xf32>>) -> ()
 // CHECK: define void @fir_dev_issue_1416
-// CHECK-SAME: ptr nocapture %[[base_addr:.*]], i64 %[[low:.*]], i64 %[[up:.*]], i64 %[[at:.*]])
+// CHECK-SAME: ptr captures(none) %[[base_addr:.*]], i64 %[[low:.*]], i64 %[[up:.*]], i64 %[[at:.*]])
 func.func @fir_dev_issue_1416(%arg0: !fir.ref<!fir.array<40x?xf32>>, %low: index, %up: index, %at : index) {
     // Test fir.embox with a constant interior array shape.
     %c1 = arith.constant 1 : index
diff --git a/flang/test/Fir/field-index.fir b/flang/test/Fir/field-index.fir
index 08653752d71d3..4f2551b380d55 100644
--- a/flang/test/Fir/field-index.fir
+++ b/flang/test/Fir/field-index.fir
@@ -7,7 +7,7 @@
 // CHECK-DAG: %[[c:.*]] = type { float, %[[b]] }
 
 // CHECK-LABEL: @simple_field
-// CHECK-SAME: (ptr nocapture %[[arg0:.*]])
+// CHECK-SAME: (ptr captures(none) %[[arg0:.*]])
 func.func @simple_field(%arg0: !fir.ref<!fir.type<a{x:f32,i:i32}>>) -> i32 {
   %1 = fir.field_index i, !fir.type<a{x:f32,i:i32}>
   // CHECK: %[[GEP:.*]] = getelementptr %a, ptr %[[arg0]], i32 0, i32 1
@@ -18,7 +18,7 @@ func.func @simple_field(%arg0: !fir.ref<!fir.type<a{x:f32,i:i32}>>) -> i32 {
 }
 
 // CHECK-LABEL: @derived_field
-// CHECK-SAME: (ptr nocapture %[[arg0:.*]])
+// CHECK-SAME: (ptr captures(none) %[[arg0:.*]])
 func.func @derived_field(%arg0: !fir.ref<!fir.type<c{x:f32,some_b:!fir.type<b{x:f32,i:i32}>}>>) -> i32 {
   %1 = fir.field_index some_b, !fir.type<c{x:f32,some_b:!fir.type<b{x:f32,i:i32}>}>
   %2 = fir.field_index i, !fir.type<b{x:f32,i:i32}>
diff --git a/flang/test/Fir/ignore-missing-type-descriptor.fir b/flang/test/Fir/ignore-missing-type-descriptor.fir
index 009e110a22b30..2958918156410 100644
--- a/flang/test/Fir/ignore-missing-type-descriptor.fir
+++ b/flang/test/Fir/ignore-missing-type-descriptor.fir
@@ -15,7 +15,7 @@ func.func @test_embox(%addr: !fir.ref<!some_freestyle_type>) {
   return
 }
 // CHECK-LABEL: define void @test_embox(
-// CHECK-SAME: ptr nocapture %[[ADDR:.*]])
+// CHECK-SAME: ptr captures(none) %[[ADDR:.*]])
 // CHECK: insertvalue { ptr, i64, i32, i8, i8, i8, i8, ptr, [1 x i64] }
 // CHECK-SAME: { ptr undef, i64 ptrtoint (ptr getelementptr (%some_not_mangled_type, ptr null, i32 1) to i64),
 // CHECK-SAME: i32 20240719, i8 0, i8 42, i8 0, i8 1, ptr null, [1 x i64] zeroinitializer },
diff --git a/flang/test/Fir/pdt.fir b/flang/test/Fir/pdt.fir
index 4464b897414a1..ce1fb7a379b8b 100644
--- a/flang/test/Fir/pdt.fir
+++ b/flang/test/Fir/pdt.fir
@@ -1,6 +1,6 @@
 // RUN: tco %s | FileCheck %s
 
-// CHECK-LINE: define i64 @_QTtP.mem.size(i32 %0, i16 %1)
+// CHECK-LABEL: define i64 @_QTtP.mem.size(i32 %0, i16 %1)
 func.func @_QTtP.mem.size(%0 : i32, %1 : i16) -> index {
   %2 = call @_QTtP.f1.size(%0, %1) : (i32, i16) -> index
   %3 = call @_QTtP.f2.size(%0, %1) : (i32, i16) -> index
@@ -8,25 +8,25 @@ func.func @_QTtP.mem.size(%0 : i32, %1 : i16) -> index {
   // CHECK: ret i64 8
   return %4 : index
 }
-// CHECK-LINE: define i64 @_QTtP.f1.size(i32 %0, i16 %1)
+// CHECK-LABEL: define i64 @_QTtP.f1.size(i32 %0, i16 %1)
 func.func @_QTtP.f1.size(%0 : i32, %1 : i16) -> index {
   %2 = arith.constant 4 : index
   // CHECK: ret i64 4
   return %2 : index
 }
-// CHECK-LINE: define i64 @_QTtP.f2.size(i32 %0, i16 %1)
+// CHECK-LABEL: define i64 @_QTtP.f2.size(i32 %0, i16 %1)
 func.func @_QTtP.f2.size(%0 : i32, %1 : i16) -> index {
   %2 = arith.constant 4 : index
   // CHECK: ret i64 4
   return %2 : index
 }
-// CHECK-LINE: define i32 @_QTtP.f1.offset(i32 %0, i16 %1)
+// CHECK-LABEL: define i32 @_QTtP.f1.offset(i32 %0, i16 %1)
 func.func @_QTtP.f1.offset(%0 : i32, %1 : i16) -> i32 {
   %2 = arith.constant 0 : i32
   // CHECK: ret i32 0
   return %2 : i32
 }
-// CHECK-LINE: define i32 @_QTtP.f2.offset(i32 %0, i16 %1)
+// CHECK-LABEL: define i32 @_QTtP.f2.offset(i32 %0, i16 %1)
 func.func @_QTtP.f2.offset(%0 : i32, %1 : i16) -> i32 {
   %2 = arith.constant 4 : i32
   // CHECK: ret i32 4
@@ -44,7 +44,7 @@ func.func @_QTtP.f2.offset(%0 : i32, %1 : i16) -> i32 {
 //   var%f1 = 4
 // end program p
 
-// CHECK-LINE: define void @_QQmain(i32 %0, i16 %1)
+// CHECK-LABEL: define void @_QQmain(i32 %0, i16 %1)
 func.func @_QQmain(%arg0 : i32, %arg1 : i16) {
   // CHECK: %[[size:.*]] = call i64 @_QTtP.mem.size(i32 %0, i16 %1)
   // CHECK: %[[alloc:.*]] = alloca i8, i64 %[[size]]
@@ -56,7 +56,7 @@ func.func @_QQmain(%arg0 : i32, %arg1 : i16) {
   return
 }
 
-// CHECK-LINE: define i64 @_QTt1P.mem.size(i32 %0, i32 %1)
+// CHECK-LABEL: define i64 @_QTt1P.mem.size(i32 %0, i32 %1)
 func.func @_QTt1P.mem.size(%0 : i32, %1 : i32) -> index {
   // CHECK: call i64 @_QTt1P.f1.size
   %2 = call @_QTt1P.f1.size(%0, %1) : (i32, i32) -> index
@@ -65,22 +65,22 @@ func.func @_QTt1P.mem.size(%0 : i32, %1 : i32) -> index {
   %4 = arith.addi %2, %3 : index
   return %4 : index
 }
-// CHECK-LINE: define i64 @_QTt1P.f1.size(i32 %0, i32 %1)
+// CHECK-LABEL: define i64 @_QTt1P.f1.size(i32 %0, i32 %1)
 func.func @_QTt1P.f1.size(%0 : i32, %1 : i32) -> index {
   %2 = fir.convert %0 : (i32) -> index
   return %2 : index
 }
-// CHECK-LINE: define i64 @_QTt1P.f2.size(i32 %0, i32 %1)
+// CHECK-LABEL: define i64 @_QTt1P.f2.size(i32 %0, i32 %1)
 func.func @_QTt1P.f2.size(%0 : i32, %1 : i32) -> index {
   %2 = fir.convert %1 : (i32) -> index
   return %2 : index
 }
-// CHECK-LINE: define i32 @_QTt1P.f1.offset(i32 %0, i32 %1)
+// CHECK-LABEL: define i32 @_QTt1P.f1.offset(i32 %0, i32 %1)
 func.func @_QTt1P.f1.offset(%0 : i32, %1 : i32) -> i32 {
   %2 = arith.constant 0 : i32
   return %2 : i32
 }
-// CHECK-LINE: define i32 @_QTt1P.f2.offset(i32 %0, i32 %1)
+// CHECK-LABEL: define i32 @_QTt1P.f2.offset(i32 %0, i32 %1)
 func.func @_QTt1P.f2.offset(%0 : i32, %1 : i32) -> i32 {
   return %0 : i32
 }
@@ -97,7 +97,7 @@ func.func @_QTt1P.f2.offset(%0 : i32, %1 : i32) -> i32 {
 
 func.func private @bar(!fir.ref<!fir.char<1,?>>)
 
-// CHECK-LINE: define i8* @_QPfoo(i32 %0, i32 %1)
+// CHECK-LABEL: define void @_QPfoo(i32 %0, i32 %1)
 func.func @_QPfoo(%arg0 : i32, %arg1 : i32) {
   // CHECK: %[[size:.*]] = call i64 @_QTt1P.mem.size(i32 %0, i32 %1)
   // CHECK: %[[alloc:.*]] = alloca i8, i64 %[[size]]
diff --git a/flang/test/Fir/polymorphic.fir b/flang/test/Fir/polymorphic.fir
index f9cf6fab6b707..c65841ea54cee 100644
--- a/flang/test/Fir/polymorphic.fir
+++ b/flang/test/Fir/polymorphic.fir
@@ -175,7 +175,7 @@ func.func @_QMmod2Pinitp(%arg0: !fir.ref<!fir.class<!fir.ptr<none>>> {fir.bindc_
 func.func private @_FortranAPointerAssociate(!fir.ref<!fir.box<none>>, !fir.box<none>) -> () attributes {fir.runtime}
 
 // CHECK-LABEL: define void @_QMmod2Pinitp(
-// CHECK-SAME: ptr nocapture %[[ARG0:.*]]){{.*}}{
+// CHECK-SAME: ptr captures(none) %[[ARG0:.*]]){{.*}}{
 // CHECK: %[[ALLOCA_CLASS_NONE:.*]] = alloca { ptr, i64, i32, i8, i8, i8, i8, ptr, [1 x i64] }
 // CHECK: call void @llvm.memcpy.p0.p0.i32(ptr %[[ALLOCA_CLASS_NONE]], ptr %[[ARG0]], i32 40, i1 false)
 // CHECK: call void @_FortranAPointerAssociate(ptr @_QMmod2Ep, ptr %[[ALLOCA_CLASS_NONE]])
diff --git a/flang/test/Fir/simplify-region-lite-after-inliner.fir b/flang/test/Fir/simplify-region-lite-after-inliner.fir
index c09612b5dabed..1a41c8d38ce19 100644
--- a/flang/test/Fir/simplify-region-lite-after-inliner.fir
+++ b/flang/test/Fir/simplify-region-lite-after-inliner.fir
@@ -33,4 +33,4 @@ func.func private @foo() -> none
 
 
 // CHECK: define void @repro(i8 %0, i8 %1)
-//   CHECK-NEXT  ret void
+// CHECK-NEXT: ret void
diff --git a/flang/test/Fir/struct-passing-x86-64-byval.fir b/flang/test/Fir/struct-passing-x86-64-byval.fir
index 44256c0c0a048..13cf52de581f7 100644
--- a/flang/test/Fir/struct-passing-x86-64-byval.fir
+++ b/flang/test/Fir/struct-passing-x86-64-byval.fir
@@ -81,27 +81,27 @@ func.func @not_enough_int_reg_3(%arg0: i32, %arg1: i32, %arg2: i32, %arg3: i32,
 }
 }
 
-// CHECK: define void @takes_toobig(ptr nocapture byval(%toobig) align 8 %{{.*}}) {
-// CHECK: define void @takes_toobig_align16(ptr nocapture byval(%toobig_align16) align 16 %{{.*}}) {
-// CHECK: define void @not_enough_int_reg_1(i32 %{{.*}}, i32 %{{.*}}, i32 %{{.*}}, i32 %{{.*}}, i32 %{{.*}}, i32 %{{.*}}, ptr nocapture byval(%fits_in_1_int_reg) align 8 %{{.*}}) {
-// CHECK: define void @not_enough_int_reg_1b(ptr nocapture %{{.*}}, ptr nocapture %{{.*}}, ptr nocapture %{{.*}}, ptr nocapture %{{.*}}, ptr nocapture %{{.*}}, ptr nocapture %{{.*}}, ptr nocapture byval(%fits_in_1_int_reg) align 8 %{{.*}}) {
-// CHECK: define void @not_enough_int_reg_2(i32 %{{.*}}, i32 %{{.*}}, i32 %{{.*}}, i32 %{{.*}}, i32 %{{.*}}, ptr nocapture byval(%fits_in_2_int_reg) align 8 %{{.*}}) {
-// CHECK: define void @ftakes_toobig(ptr nocapture byval(%ftoobig) align 8 %{{.*}}) {
-// CHECK: define void @ftakes_toobig_align16(ptr nocapture byval(%ftoobig_align16) align 16 %{{.*}}) {
-// CHECK: define void @not_enough_sse_reg_1(float %{{.*}}, float %{{.*}}, float %{{.*}}, float %{{.*}}, float %{{.*}}, float %{{.*}}, float %{{.*}}, float %{{.*}}, ptr nocapture byval(%fits_in_1_sse_reg) align 8 %{{.*}}) {
-// CHECK: define void @not_enough_sse_reg_1b(<2 x float> %{{.*}}, <2 x float> %{{.*}}, <2 x float> %{{.*}}, <2 x float> %{{.*}}, <2 x float> %{{.*}}, <2 x float> %{{.*}}, <2 x float> %{{.*}}, <2 x float> %{{.*}}, ptr nocapture byval(%fits_in_1_sse_reg) align 8 %{{.*}}) {
-// CHECK: define void @not_enough_sse_reg_1c(double %{{.*}}, double %{{.*}}, double %{{.*}}, double %{{.*}}, double %{{.*}}, double %{{.*}}, double %{{.*}}, double %{{.*}}, ptr nocapture byval(%fits_in_1_sse_reg) align 8 %{{.*}}) {
-// CHECK: define void @not_enough_sse_reg_2(float %{{.*}}, float %{{.*}}, float %{{.*}}, float %{{.*}}, float %{{.*}}, float %{{.*}}, float %{{.*}}, ptr nocapture byval(%fits_in_2_sse_reg) align 8 %{{.*}}) {
-// CHECK: define void @test_contains_x87(ptr nocapture byval(%contains_x87) align 16 %{{.*}}) {
-// CHECK: define void @test_contains_complex_x87(ptr nocapture byval(%contains_complex_x87) align 16 %{{.*}}) {
-// CHECK: define void @test_nested_toobig(ptr nocapture byval(%nested_toobig) align 8 %{{.*}}) {
-// CHECK: define void @test_badly_aligned(ptr nocapture byval(%badly_aligned) align 8 %{{.*}}) {
-// CHECK: define void @test_logical_toobig(ptr nocapture byval(%logical_too_big) align 8 %{{.*}}) {
-// CHECK: define void @l_not_enough_int_reg(i32 %{{.*}}, i32 %{{.*}}, i32 %{{.*}}, i32 %{{.*}}, i32 %{{.*}}, ptr nocapture byval(%l_fits_in_2_int_reg) align 8 %{{.*}}) {
-// CHECK: define void @test_complex_toobig(ptr nocapture byval(%complex_too_big) align 8 %{{.*}}) {
-// CHECK: define void @cplx_not_enough_sse_reg_1(float %{{.*}}, float %{{.*}}, float %{{.*}}, float %{{.*}}, float %{{.*}}, float %{{.*}}, float %{{.*}}, float %{{.*}}, ptr nocapture byval(%cplx_fits_in_1_sse_reg) align 8 %{{.*}}) {
-// CHECK: define void @test_char_to_big(ptr nocapture byval(%char_too_big) align 8 %{{.*}}) {
-// CHECK: define void @char_not_enough_int_reg_1(i32 %{{.*}}, i32 %{{.*}}, i32 %{{.*}}, i32 %{{.*}}, i32 %{{.*}}, i32 %{{.*}}, ptr nocapture byval(%char_fits_in_1_int_reg) align 8 %{{.*}}) {
-// CHECK: define void @mix_not_enough_int_reg_1(i32 %{{.*}}, i32 %{{.*}}, i32 %{{.*}}, i32 %{{.*}}, i32 %{{.*}}, i32 %{{.*}}, ptr nocapture byval(%mix_in_1_int_reg) align 8 %{{.*}}) {
-// CHECK: define void @mix_not_enough_sse_reg_2(float %{{.*}}, float %{{.*}}, float %{{.*}}, float %{{.*}}, float %{{.*}}, float %{{.*}}, float %{{.*}}, float %{{.*}}, ptr nocapture byval(%mix_in_1_int_reg_1_sse_reg) align 8 %{{.*}}) {
-// CHECK: define void @not_enough_int_reg_3(ptr nocapture sret({ fp128, fp128 }) align 16 %{{.*}}, i32 %{{.*}}, i32 %{{.*}}, i32 %{{.*}}, i32 %{{.*}}, i32 %{{.*}}, ptr nocapture byval(%fits_in_1_int_reg) align 8 %{{.*}})
+// CHECK: define void @takes_toobig(ptr byval(%toobig) align 8 captures(none) %{{.*}}) {
+// CHECK: define void @takes_toobig_align16(ptr byval(%toobig_align16) align 16 captures(none) %{{.*}}) {
+// CHECK: define void @not_enough_int_reg_1(i32 %{{.*}}, i32 %{{.*}}, i32 %{{.*}}, i32 %{{.*}}, i32 %{{.*}}, i32 %{{.*}}, ptr byval(%fits_in_1_int_reg) align 8 captures(none) %{{.*}}) {
+// CHECK: define void @not_enough_int_reg_1b(ptr captures(none) %{{.*}}, ptr captures(none) %{{.*}}, ptr captures(none) %{{.*}}, ptr captures(none) %{{.*}}, ptr captures(none) %{{.*}}, ptr captures(none) %{{.*}}, ptr byval(%fits_in_1_int_reg) align 8 captures(none) %{{.*}}) {
+// CHECK: define void @not_enough_int_reg_2(i32 %{{.*}}, i32 %{{.*}}, i32 %{{.*}}, i32 %{{.*}}, i32 %{{.*}}, ptr byval(%fits_in_2_int_reg) align 8 captures(none) %{{.*}}) {
+// CHECK: define void @ftakes_toobig(ptr byval(%ftoobig) align 8 captures(none) %{{.*}}) {
+// CHECK: define void @ftakes_toobig_align16(ptr byval(%ftoobig_align16) align 16 captures(none) %{{.*}}) {
+// CHECK: define void @not_enough_sse_reg_1(float %{{.*}}, float %{{.*}}, float %{{.*}}, float %{{.*}}, float %{{.*}}, float %{{.*}}, float %{{.*}}, float %{{.*}}, ptr byval(%fits_in_1_sse_reg) align 8 captures(none) %{{.*}}) {
+// CHECK: define void @not_enough_sse_reg_1b(<2 x float> %{{.*}}, <2 x float> %{{.*}}, <2 x float> %{{.*}}, <2 x float> %{{.*}}, <2 x float> %{{.*}}, <2 x float> %{{.*}}, <2 x float> %{{.*}}, <2 x float> %{{.*}}, ptr byval(%fits_in_1_sse_reg) align 8 captures(none) %{{.*}}) {
+// CHECK: define void @not_enough_sse_reg_1c(double %{{.*}}, double %{{.*}}, double %{{.*}}, double %{{.*}}, double %{{.*}}, double %{{.*}}, double %{{.*}}, double %{{.*}}, ptr byval(%fits_in_1_sse_reg) align 8 captures(none) %{{.*}}) {
+// CHECK: define void @not_enough_sse_reg_2(float %{{.*}}, float %{{.*}}, float %{{.*}}, float %{{.*}}, float %{{.*}}, float %{{.*}}, float %{{.*}}, ptr byval(%fits_in_2_sse_reg) align 8 captures(none) %{{.*}}) {
+// CHECK: define void @test_contains_x87(ptr byval(%contains_x87) align 16 captures(none) %{{.*}}) {
+// CHECK: define void @test_contains_complex_x87(ptr byval(%contains_complex_x87) align 16 captures(none) %{{.*}}) {
+// CHECK: define void @test_nested_toobig(ptr byval(%nested_toobig) align 8 captures(none) %{{.*}}) {
+// CHECK: define void @test_badly_aligned(ptr byval(%badly_aligned) align 8 captures(none) %{{.*}}) {
+// CHECK: define void @test_logical_toobig(ptr byval(%logical_too_big) align 8 captures(none) %{{.*}}) {
+// CHECK: define void @l_not_enough_int_reg(i32 %{{.*}}, i32 %{{.*}}, i32 %{{.*}}, i32 %{{.*}}, i32 %{{.*}}, ptr byval(%l_fits_in_2_int_reg) align 8 captures(none) %{{.*}}) {
+// CHECK: define void @test_complex_toobig(ptr byval(%complex_too_big) align 8 captures(none) %{{.*}}) {
+// CHECK: define void @cplx_not_enough_sse_reg_1(float %{{.*}}, float %{{.*}}, float %{{.*}}, float %{{.*}}, float %{{.*}}, float %{{.*}}, float %{{.*}}, float %{{.*}}, ptr byval(%cplx_fits_in_1_sse_reg) align 8 captures(none) %{{.*}}) {
+// CHECK: define void @test_char_to_big(ptr byval(%char_too_big) align 8 captures(none) %{{.*}}) {
+// CHECK: define void @char_not_enough_int_reg_1(i32 %{{.*}}, i32 %{{.*}}, i32 %{{.*}}, i32 %{{.*}}, i32 %{{.*}}, i32 %{{.*}}, ptr byval(%char_fits_in_1_int_reg) align 8 captures(none) %{{.*}}) {
+// CHECK: define void @mix_not_enough_int_reg_1(i32 %{{.*}}, i32 %{{.*}}, i32 %{{.*}}, i32 %{{.*}}, i32 %{{.*}}, i32 %{{.*}}, ptr byval(%mix_in_1_int_reg) align 8 captures(none) %{{.*}}) {
+// CHECK: define void @mix_not_enough_sse_reg_2(float %{{.*}}, float %{{.*}}, float %{{.*}}, float %{{.*}}, float %{{.*}}, float %{{.*}}, float %{{.*}}, float %{{.*}}, ptr byval(%mix_in_1_int_reg_1_sse_reg) align 8 captures(none) %{{.*}}) {
+// CHECK: define void @not_enough_int_reg_3(ptr sret({ fp128, fp128 }) align 16 captures(none) %{{.*}}, i32 %{{.*}}, i32 %{{.*}}, i32 %{{.*}}, i32 %{{.*}}, i32 %{{.*}}, ptr byval(%fits_in_1_int_reg) align 8 captures(none) %{{.*}})
diff --git a/flang/test/Fir/target-rewrite-complex-10-x86.fir b/flang/test/Fir/target-rewrite-complex-10-x86.fir
index 651d1e78fdc93..a6dd178cfe70a 100644
--- a/flang/test/Fir/target-rewrite-complex-10-x86.fir
+++ b/flang/test/Fir/target-rewrite-complex-10-x86.fir
@@ -29,4 +29,4 @@ func.func @takecomplex10(%z: complex<f80>) {
 // AMD64:    %[[VAL_3:.*]] = fir.alloca complex<f80>
 // AMD64:    fir.store %[[VAL_2]] to %[[VAL_3]] : !fir.ref<complex<f80>>
 
-// AMD64_LLVM: define void @takecomplex10(ptr nocapture byval({ x86_fp80, x86_fp80 }) align 16 %0)
+// AMD64_LLVM: define void @takecomplex10(ptr byval({ x86_fp80, x86_fp80 }) align 16 captures(none) %0)
diff --git a/flang/test/Fir/target.fir b/flang/test/Fir/target.fir
index d101db47c8051..62ce488a3ca2f 100644
--- a/flang/test/Fir/target.fir
+++ b/flang/test/Fir/target.fir
@@ -26,7 +26,7 @@ func.func @gen4() -> complex<f32> {
   return %6 : complex<f32>
 }
 
-// I32-LABEL: define void @gen8(ptr nocapture sret({ double, double }) align 4 %
+// I32-LABEL: define void @gen8(ptr sret({ double, double }) align 4 captures(none) %
 // X64-LABEL: define { double, double } @gen8()
 // AARCH64-LABEL: define { double, double } @gen8()
 // PPC-LABEL: define { double, double } @gen8()
@@ -39,9 +39,9 @@ func.func @gen8() -> complex<f64> {
   %c1 = arith.constant 1 : i32
   %5 = fir.insert_value %4, %2, [1 : index] : (complex<f64>, f64) -> complex<f64>
   // I32: store { double, double } { double -4.000000e+00, double 1.000000e+00 }
-  // I64: store { double, double } { double -4.000000e+00, double 1.000000e+00 }
-  // I64: %[[load:.*]] = load { double, double }
-  // I64: ret { double, double } %[[load]]
+  // X64: store { double, double } { double -4.000000e+00, double 1.000000e+00 }
+  // X64: %[[load:.*]] = load { double, double }
+  // X64: ret { double, double } %[[load]]
   // AARCH64: ret { double, double }
   // PPC: ret { double, double }
   return %5 : complex<f64>
@@ -93,9 +93,9 @@ func.func @call8() {
   return
 }
 
-// I32-LABEL: define i64 @char1lensum(ptr nocapture %0, ptr nocapture %1, i32 %2, i32 %3)
-// X64-LABEL: define i64 @char1lensum(ptr nocapture %0, ptr nocapture %1, i64 %2, i64 %3)
-// PPC-LABEL: define i64 @char1lensum(ptr nocapture %0, ptr nocapture %1, i64 %2, i64 %3)
+// I32-LABEL: define i64 @char1lensum(ptr captures(none) %0, ptr captures(none) %1, i32 %2, i32 %3)
+// X64-LABEL: define i64 @char1lensum(ptr captures(none) %0, ptr captures(none) %1, i64 %2, i64 %3)
+// PPC-LABEL: define i64 @char1lensum(ptr captures(none) %0, ptr captures(none) %1, i64 %2, i64 %3)
 func.func @char1lensum(%arg0 : !fir.boxchar<1>, %arg1 : !fir.boxchar<1>) -> i64 {
   // X64-DAG: %[[p0:.*]] = insertvalue { ptr, i64 } undef, ptr %1, 0
   // X64-DAG: = insertvalue { ptr, i64 } %[[p0]], i64 %3, 1
diff --git a/flang/test/HLFIR/assign-codegen.fir b/flang/test/HLFIR/assign-codegen.fir
index 581d1ab0e7739..7e03aa0bd464d 100644
--- a/flang/test/HLFIR/assign-codegen.fir
+++ b/flang/test/HLFIR/assign-codegen.fir
@@ -427,3 +427,57 @@ func.func @test_upoly_expr_assignment(%arg0: !fir.class<!fir.array<?xnone>> {fir
 // CHECK:           }
 // CHECK:           return
 // CHECK:         }
+
+func.func @test_scalar_box(%arg0: f32, %arg1: !fir.box<!fir.ptr<f32>>) {
+  %x = fir.declare %arg1 {uniq_name = "x"} : (!fir.box<!fir.ptr<f32>>) -> !fir.box<!fir.ptr<f32>>
+  hlfir.assign %arg0 to %x : f32, !fir.box<!fir.ptr<f32>>
+  return
+}
+// CHECK-LABEL:   func.func @test_scalar_box(
+// CHECK-SAME:                               %[[VAL_0:.*]]: f32,
+// CHECK-SAME:                               %[[VAL_1:.*]]: !fir.box<!fir.ptr<f32>>) {
+// CHECK:           %[[VAL_2:.*]] = fir.declare %[[VAL_1]] {uniq_name = "x"} : (!fir.box<!fir.ptr<f32>>) -> !fir.box<!fir.ptr<f32>>
+// CHECK:           %[[VAL_3:.*]] = fir.box_addr %[[VAL_2]] : (!fir.box<!fir.ptr<f32>>) -> !fir.ptr<f32>
+// CHECK:           fir.store %[[VAL_0]] to %[[VAL_3]] : !fir.ptr<f32>
+
+func.func @test_scalar_opt_box(%arg0: f32, %arg1: !fir.box<!fir.ptr<f32>>) {
+  %x = fir.declare %arg1 {fortran_attrs = #fir.var_attrs<optional>, uniq_name = "x"} : (!fir.box<!fir.ptr<f32>>) -> !fir.box<!fir.ptr<f32>>
+  hlfir.assign %arg0 to %x : f32, !fir.box<!fir.ptr<f32>>
+  return
+}
+// CHECK-LABEL:   func.func @test_scalar_opt_box(
+// CHECK-SAME:                                   %[[VAL_0:.*]]: f32,
+// CHECK-SAME:                                   %[[VAL_1:.*]]: !fir.box<!fir.ptr<f32>>) {
+// CHECK:           %[[VAL_2:.*]] = fir.declare %[[VAL_1]] {fortran_attrs = #fir.var_attrs<optional>, uniq_name = "x"} : (!fir.box<!fir.ptr<f32>>) -> !fir.box<!fir.ptr<f32>>
+// CHECK:           %[[VAL_3:.*]] = fir.is_present %[[VAL_2]] : (!fir.box<!fir.ptr<f32>>) -> i1
+// CHECK:           %[[VAL_4:.*]] = fir.if %[[VAL_3]] -> (!fir.ptr<f32>) {
+// CHECK:             %[[VAL_5:.*]] = fir.box_addr %[[VAL_2]] : (!fir.box<!fir.ptr<f32>>) -> !fir.ptr<f32>
+// CHECK:             fir.result %[[VAL_5]] : !fir.ptr<f32>
+// CHECK:           } else {
+// CHECK:             %[[VAL_6:.*]] = fir.absent !fir.ptr<f32>
+// CHECK:             fir.result %[[VAL_6]] : !fir.ptr<f32>
+// CHECK:           }
+// CHECK:           fir.store %[[VAL_0]] to %[[VAL_4]] : !fir.ptr<f32>
+
+func.func @test_scalar_opt_char_box(%arg0: !fir.ref<!fir.char<1,10>>, %arg1: !fir.box<!fir.char<1,?>>) {
+  %x = fir.declare %arg1 {fortran_attrs = #fir.var_attrs<optional>, uniq_name = "x"} : (!fir.box<!fir.char<1,?>>) -> !fir.box<!fir.char<1,?>>
+  hlfir.assign %arg0 to %x : !fir.ref<!fir.char<1,10>>, !fir.box<!fir.char<1,?>>
+  return
+}
+// CHECK-LABEL:   func.func @test_scalar_opt_char_box(
+// CHECK-SAME:                                        %[[VAL_0:.*]]: !fir.ref<!fir.char<1,10>>,
+// CHECK-SAME:                                        %[[VAL_1:.*]]: !fir.box<!fir.char<1,?>>) {
+// CHECK:           %[[VAL_2:.*]] = fir.declare %[[VAL_1]] {fortran_attrs = #fir.var_attrs<optional>, uniq_name = "x"} : (!fir.box<!fir.char<1,?>>) -> !fir.box<!fir.char<1,?>>
+// CHECK:           %[[VAL_3:.*]] = arith.constant 10 : index
+// CHECK:           %[[VAL_4:.*]] = fir.is_present %[[VAL_2]] : (!fir.box<!fir.char<1,?>>) -> i1
+// CHECK:           %[[VAL_5:.*]]:2 = fir.if %[[VAL_4]] -> (!fir.ref<!fir.char<1,?>>, index) {
+// CHECK:             %[[VAL_6:.*]] = fir.box_addr %[[VAL_2]] : (!fir.box<!fir.char<1,?>>) -> !fir.ref<!fir.char<1,?>>
+// CHECK:             %[[VAL_7:.*]] = fir.box_elesize %[[VAL_2]] : (!fir.box<!fir.char<1,?>>) -> index
+// CHECK:             fir.result %[[VAL_6]], %[[VAL_7]] : !fir.ref<!fir.char<1,?>>, index
+// CHECK:           } else {
+// CHECK:             %[[VAL_8:.*]] = fir.absent !fir.ref<!fir.char<1,?>>
+// CHECK:             %[[VAL_9:.*]] = fir.zero_bits index
+// CHECK:             fir.result %[[VAL_8]], %[[VAL_9]] : !fir.ref<!fir.char<1,?>>, index
+// CHECK:           }
+// ...
+// CHECK:           fir.call @llvm.memmove.p0.p0.i64(
diff --git a/flang/test/HLFIR/elemental-with-empty-check-codegen.fir b/flang/test/HLFIR/elemental-with-empty-check-codegen.fir
new file mode 100644
index 0000000000000..051127cf67ac7
--- /dev/null
+++ b/flang/test/HLFIR/elemental-with-empty-check-codegen.fir
@@ -0,0 +1,56 @@
+// Test hlfir.elemental code generation with a dynamic check
+// for empty result array
+// RUN: fir-opt %s --bufferize-hlfir=opt-empty-elementals=true | FileCheck %s
+
+func.func @test(%v: i32, %e0: i32, %e1: i32, %e2: i64, %e3: i64) {
+  %shape = fir.shape %e0, %e1, %e2, %e3 : (i32, i32, i64, i64) -> !fir.shape<4>
+  %result = hlfir.elemental %shape : (!fir.shape<4>) -> !hlfir.expr<?x?x?x?xi32> {
+  ^bb0(%arg0: index, %arg1: index, %arg2: index, %arg3: index):
+    hlfir.yield_element %v : i32
+  }
+  return
+}
+// CHECK-LABEL:   func.func @test(
+// CHECK-SAME:                    %[[VAL_0:.*]]: i32, %[[VAL_1:.*]]: i32, %[[VAL_2:.*]]: i32,
+// CHECK-SAME:                    %[[VAL_3:.*]]: i64, %[[VAL_4:.*]]: i64) {
+// CHECK:           %[[VAL_5:.*]] = fir.shape %[[VAL_1]], %[[VAL_2]], %[[VAL_3]], %[[VAL_4]] : (i32, i32, i64, i64) -> !fir.shape<4>
+// CHECK:           %[[VAL_6:.*]] = fir.convert %[[VAL_1]] : (i32) -> index
+// CHECK:           %[[VAL_7:.*]] = fir.convert %[[VAL_2]] : (i32) -> index
+// CHECK:           %[[VAL_8:.*]] = fir.convert %[[VAL_3]] : (i64) -> index
+// CHECK:           %[[VAL_9:.*]] = fir.convert %[[VAL_4]] : (i64) -> index
+// CHECK:           %[[VAL_10:.*]] = fir.allocmem !fir.array<?x?x?x?xi32>, %[[VAL_6]], %[[VAL_7]], %[[VAL_8]], %[[VAL_9]] {bindc_name = ".tmp.array", uniq_name = ""}
+// CHECK:           %[[VAL_11:.*]]:2 = hlfir.declare %[[VAL_10]](%[[VAL_5]]) {uniq_name = ".tmp.array"} : (!fir.heap<!fir.array<?x?x?x?xi32>>, !fir.shape<4>) -> (!fir.box<!fir.array<?x?x?x?xi32>>, !fir.heap<!fir.array<?x?x?x?xi32>>)
+// CHECK:           %[[VAL_12:.*]] = arith.constant true
+// CHECK:           %[[VAL_13:.*]] = arith.constant false
+// CHECK:           %[[C0_1:.*]] = arith.constant 0 : index
+// CHECK:           %[[VAL_15:.*]] = arith.cmpi eq, %[[VAL_6]], %[[C0_1]] : index
+// CHECK:           %[[VAL_16:.*]] = arith.ori %[[VAL_13]], %[[VAL_15]] : i1
+// CHECK:           %[[C0_2:.*]] = arith.constant 0 : index
+// CHECK:           %[[VAL_17:.*]] = arith.cmpi eq, %[[VAL_7]], %[[C0_2]] : index
+// CHECK:           %[[VAL_18:.*]] = arith.ori %[[VAL_16]], %[[VAL_17]] : i1
+// CHECK:           %[[C0_3:.*]] = arith.constant 0 : index
+// CHECK:           %[[VAL_19:.*]] = arith.cmpi eq, %[[VAL_8]], %[[C0_3]] : index
+// CHECK:           %[[VAL_20:.*]] = arith.ori %[[VAL_18]], %[[VAL_19]] : i1
+// CHECK:           %[[C0_4:.*]] = arith.constant 0 : index
+// CHECK:           %[[VAL_21:.*]] = arith.cmpi eq, %[[VAL_9]], %[[C0_4]] : index
+// CHECK:           %[[VAL_22:.*]] = arith.ori %[[VAL_20]], %[[VAL_21]] : i1
+// CHECK:           %[[VAL_23:.*]] = arith.select %[[VAL_22]], %[[C0_1]], %[[VAL_6]] : index
+// CHECK:           %[[VAL_24:.*]] = arith.select %[[VAL_22]], %[[C0_2]], %[[VAL_7]] : index
+// CHECK:           %[[VAL_25:.*]] = arith.select %[[VAL_22]], %[[C0_3]], %[[VAL_8]] : index
+// CHECK:           %[[VAL_26:.*]] = arith.select %[[VAL_22]], %[[C0_4]], %[[VAL_9]] : index
+// CHECK:           %[[VAL_27:.*]] = arith.constant 1 : index
+// CHECK:           fir.do_loop %[[VAL_28:.*]] = %[[VAL_27]] to %[[VAL_26]] step %[[VAL_27]] {
+// CHECK:             fir.do_loop %[[VAL_29:.*]] = %[[VAL_27]] to %[[VAL_25]] step %[[VAL_27]] {
+// CHECK:               fir.do_loop %[[VAL_30:.*]] = %[[VAL_27]] to %[[VAL_24]] step %[[VAL_27]] {
+// CHECK:                 fir.do_loop %[[VAL_31:.*]] = %[[VAL_27]] to %[[VAL_23]] step %[[VAL_27]] {
+// CHECK:                   %[[VAL_32:.*]] = hlfir.designate %[[VAL_11]]#0 (%[[VAL_31]], %[[VAL_30]], %[[VAL_29]], %[[VAL_28]])  : (!fir.box<!fir.array<?x?x?x?xi32>>, index, index, index, index) -> !fir.ref<i32>
+// CHECK:                   hlfir.assign %[[VAL_0]] to %[[VAL_32]] temporary_lhs : i32, !fir.ref<i32>
+// CHECK:                 }
+// CHECK:               }
+// CHECK:             }
+// CHECK:           }
+// CHECK:           %[[VAL_33:.*]] = fir.undefined tuple<!fir.box<!fir.array<?x?x?x?xi32>>, i1>
+// CHECK:           %[[VAL_34:.*]] = fir.insert_value %[[VAL_33]], %[[VAL_12]], [1 : index] : (tuple<!fir.box<!fir.array<?x?x?x?xi32>>, i1>, i1) -> tuple<!fir.box<!fir.array<?x?x?x?xi32>>, i1>
+// CHECK:           %[[VAL_35:.*]] = fir.insert_value %[[VAL_34]], %[[VAL_11]]#0, [0 : index] : (tuple<!fir.box<!fir.array<?x?x?x?xi32>>, i1>, !fir.box<!fir.array<?x?x?x?xi32>>) -> tuple<!fir.box<!fir.array<?x?x?x?xi32>>, i1>
+// CHECK:           return
+// CHECK:         }
diff --git a/flang/test/HLFIR/get_length_codegen.fir b/flang/test/HLFIR/get_length_codegen.fir
index 7cb36d2cd3c6c..c828aa5653d65 100644
--- a/flang/test/HLFIR/get_length_codegen.fir
+++ b/flang/test/HLFIR/get_length_codegen.fir
@@ -30,3 +30,22 @@ fir.global linkonce @_QQclX616263 constant : !fir.char<1,3> {
 // CHECK:           %[[VAL_33:.*]]:2 = hlfir.declare %[[VAL_31:.*]] typeparams %[[VAL_9]] {uniq_name = ".tmp"} : (!fir.ref<!fir.char<1,?>>, index) -> (!fir.boxchar<1>, !fir.ref<!fir.char<1,?>>)
 // CHECK:           return %[[VAL_9]] : index
 // CHECK:         }
+
+// Test get_length taking the length from an expression with i32 length parameter.
+func.func @i32_length(%char: !fir.boxchar<1>, %shape : i32, %len : i32) -> index {
+  %14 = fir.shape %shape : (i32) -> !fir.shape<1>
+  %15 = hlfir.elemental %14 typeparams %len unordered : (!fir.shape<1>, i32) -> !hlfir.expr<?x!fir.char<1,?>> {
+  ^bb0(%arg0: index):
+    hlfir.yield_element %char : !fir.boxchar<1>
+  }
+  %18 = hlfir.get_length %15 : (!hlfir.expr<?x!fir.char<1,?>>) -> index
+  hlfir.destroy %15 : !hlfir.expr<?x!fir.char<1,?>>
+  return %18 : index
+}
+// CHECK-LABEL:   func.func @i32_length(
+// CHECK-SAME:                          %[[VAL_0:.*]]: !fir.boxchar<1>,
+// CHECK-SAME:                          %[[VAL_1:.*]]: i32,
+// CHECK-SAME:                          %[[VAL_2:.*]]: i32) -> index {
+// CHECK:           %[[VAL_14:.*]] = fir.convert %[[VAL_2]] : (i32) -> index
+// CHECK:           return %[[VAL_14]] : index
+// CHECK:         }
diff --git a/flang/test/HLFIR/getextent-codegen.fir b/flang/test/HLFIR/getextent-codegen.fir
new file mode 100644
index 0000000000000..4f48569424e1b
--- /dev/null
+++ b/flang/test/HLFIR/getextent-codegen.fir
@@ -0,0 +1,22 @@
+// Test hlfir.get_extent code generation to FIR
+// RUN: fir-opt %s -convert-hlfir-to-fir | FileCheck %s
+
+func.func @test1(%arg0: i1, %arg1: i32, %arg2: i64, %arg3: index) -> (index, index, index, index) {
+  %0 = fir.shape %arg0, %arg1, %arg2, %arg3 : (i1, i32, i64, index) -> !fir.shape<4>
+  %1 = hlfir.get_extent %0 {dim = 0 : index} : (!fir.shape<4>) -> index
+  %2 = hlfir.get_extent %0 {dim = 1 : index} : (!fir.shape<4>) -> index
+  %3 = hlfir.get_extent %0 {dim = 2 : index} : (!fir.shape<4>) -> index
+  %4 = hlfir.get_extent %0 {dim = 3 : index} : (!fir.shape<4>) -> index
+  return %1, %2, %3, %4 : index, index, index, index
+}
+// CHECK-LABEL:   func.func @test1(
+// CHECK-SAME:                     %[[VAL_0:.*]]: i1,
+// CHECK-SAME:                     %[[VAL_1:.*]]: i32,
+// CHECK-SAME:                     %[[VAL_2:.*]]: i64,
+// CHECK-SAME:                     %[[VAL_3:.*]]: index) -> (index, index, index, index) {
+// CHECK:           %[[VAL_4:.*]] = fir.shape %[[VAL_0]], %[[VAL_1]], %[[VAL_2]], %[[VAL_3]] : (i1, i32, i64, index) -> !fir.shape<4>
+// CHECK:           %[[VAL_5:.*]] = fir.convert %[[VAL_0]] : (i1) -> index
+// CHECK:           %[[VAL_6:.*]] = fir.convert %[[VAL_1]] : (i32) -> index
+// CHECK:           %[[VAL_7:.*]] = fir.convert %[[VAL_2]] : (i64) -> index
+// CHECK:           return %[[VAL_5]], %[[VAL_6]], %[[VAL_7]], %[[VAL_3]] : index, index, index, index
+// CHECK:         }
diff --git a/flang/test/HLFIR/invalid.fir b/flang/test/HLFIR/invalid.fir
index b35bec4b2a899..d61efe0062e69 100644
--- a/flang/test/HLFIR/invalid.fir
+++ b/flang/test/HLFIR/invalid.fir
@@ -1411,7 +1411,8 @@ func.func @bad_cshift7(%arg0: !hlfir.expr<?x2xi32>, %arg1: !hlfir.expr<3xi32>) {
 // -----
 
 func.func @bad_cshift8(%arg0: !hlfir.expr<?x!fir.char<1,?>>, %arg1: i32) {
-  // expected-error@+1 {{'hlfir.cshift' op kind mismatch between input and output arrays}}
+  // expected-error@+2 {{'hlfir.cshift' op character KIND mismatch}}
+  // expected-error@+1 {{'hlfir.cshift' op input and output arrays should have the same element type}}
   %0 = hlfir.cshift %arg0 %arg1 : (!hlfir.expr<?x!fir.char<1,?>>, i32) -> !hlfir.expr<?x!fir.char<2,?>>
   return
 }
@@ -1419,7 +1420,138 @@ func.func @bad_cshift8(%arg0: !hlfir.expr<?x!fir.char<1,?>>, %arg1: i32) {
 // -----
 
 func.func @bad_cshift9(%arg0: !hlfir.expr<?x!fir.char<1,1>>, %arg1: i32) {
-  // expected-error@+1 {{'hlfir.cshift' op character LEN mismatch between input and output arrays}}
+  // expected-error@+2 {{'hlfir.cshift' op character LEN mismatch}}
+  // expected-error@+1 {{'hlfir.cshift' op input and output arrays should have the same element type}}
   %0 = hlfir.cshift %arg0 %arg1 : (!hlfir.expr<?x!fir.char<1,1>>, i32) -> !hlfir.expr<?x!fir.char<1,2>>
   return
 }
+
+// -----
+
+func.func @bad_reshape(%arg0: !hlfir.expr<1xi32>) {
+  // expected-error@+1 {{'hlfir.reshape' op ARRAY and the result must have the same element type}}
+  %0 = hlfir.reshape %arg0 %arg0 : (!hlfir.expr<1xi32>, !hlfir.expr<1xi32>) -> !hlfir.expr<?xf32>
+  return
+}
+
+// -----
+
+func.func @bad_reshape(%arg0: !hlfir.expr<?x!fir.type<whatever>?>, %arg1: !hlfir.expr<1xi32>) {
+  // expected-error@+1 {{'hlfir.reshape' op ARRAY must be polymorphic iff result is polymorphic}}
+  %0 = hlfir.reshape %arg0 %arg1 : (!hlfir.expr<?x!fir.type<whatever>?>, !hlfir.expr<1xi32>) -> !hlfir.expr<?x!fir.type<whatever>>
+  return
+}
+
+// -----
+
+func.func @bad_reshape(%arg0: !hlfir.expr<?x!fir.type<whatever>>, %arg1: !hlfir.expr<1xi32>) {
+  // expected-error@+1 {{'hlfir.reshape' op ARRAY must be polymorphic iff result is polymorphic}}
+  %0 = hlfir.reshape %arg0 %arg1 : (!hlfir.expr<?x!fir.type<whatever>>, !hlfir.expr<1xi32>) -> !hlfir.expr<?x!fir.type<whatever>?>
+  return
+}
+
+// -----
+
+func.func @bad_reshape(%arg0: !hlfir.expr<1x1xi32>) {
+  // expected-error@+1 {{'hlfir.reshape' op SHAPE must be an array of rank 1}}
+  %0 = hlfir.reshape %arg0 %arg0 : (!hlfir.expr<1x1xi32>, !hlfir.expr<1x1xi32>) -> !hlfir.expr<?xi32>
+  return
+}
+
+// -----
+
+func.func @bad_reshape(%arg0: !hlfir.expr<1xf32>) {
+  // expected-error@+1 {{'hlfir.reshape' op SHAPE must be an integer array}}
+  %0 = hlfir.reshape %arg0 %arg0 : (!hlfir.expr<1xf32>, !hlfir.expr<1xf32>) -> !hlfir.expr<?xf32>
+  return
+}
+
+// -----
+
+func.func @bad_reshape(%arg0: !hlfir.expr<?xi32>) {
+  // expected-error@+1 {{'hlfir.reshape' op SHAPE must have known size}}
+  %0 = hlfir.reshape %arg0 %arg0 : (!hlfir.expr<?xi32>, !hlfir.expr<?xi32>) -> !hlfir.expr<?xi32>
+  return
+}
+
+// -----
+
+func.func @bad_reshape(%arg0: !hlfir.expr<1xi32>) {
+  // expected-error@+1 {{'hlfir.reshape' op SHAPE's extent must match the result rank}}
+  %0 = hlfir.reshape %arg0 %arg0 : (!hlfir.expr<1xi32>, !hlfir.expr<1xi32>) -> !hlfir.expr<?x?xi32>
+  return
+}
+
+// -----
+
+func.func @bad_reshape(%arg0: !hlfir.expr<1xi32>, %arg1: !hlfir.expr<?xi16>) {
+  // expected-error@+1 {{'hlfir.reshape' op ARRAY and PAD must be of the same type}}
+  %0 = hlfir.reshape %arg0 %arg0 pad %arg1 : (!hlfir.expr<1xi32>, !hlfir.expr<1xi32>, !hlfir.expr<?xi16>) -> !hlfir.expr<?xi32>
+  return
+}
+
+// -----
+
+func.func @bad_reshape(%arg0: !hlfir.expr<1xi32>, %arg1: !hlfir.expr<?x?xi16>) {
+  // expected-error@+1 {{'hlfir.reshape' op ORDER must be an array of rank 1}}
+  %0 = hlfir.reshape %arg0 %arg0 order %arg1 : (!hlfir.expr<1xi32>, !hlfir.expr<1xi32>, !hlfir.expr<?x?xi16>) -> !hlfir.expr<?xi32>
+  return
+}
+
+// -----
+
+func.func @bad_reshape(%arg0: !hlfir.expr<1xi32>, %arg1: !hlfir.expr<?xf16>) {
+  // expected-error@+1 {{'hlfir.reshape' op ORDER must be an integer array}}
+  %0 = hlfir.reshape %arg0 %arg0 order %arg1 : (!hlfir.expr<1xi32>, !hlfir.expr<1xi32>, !hlfir.expr<?xf16>) -> !hlfir.expr<?xi32>
+  return
+}
+
+// -----
+
+func.func @bad_reshape(%arg0: !fir.ref<!fir.array<?xi32>>, %arg1: !hlfir.expr<1xi32>) {
+  // expected-error@+1 {{'hlfir.reshape' op operand #0 must be any array-like entity}}
+  %0 = hlfir.reshape %arg0 %arg1 : (!fir.ref<!fir.array<?xi32>>, !hlfir.expr<1xi32>) -> !hlfir.expr<?xi32>
+  return
+}
+
+// -----
+
+func.func @bad_reshape(%arg0: !fir.ref<!fir.array<?xi32>>, %arg1: !hlfir.expr<?xi32>) {
+  // expected-error@+1 {{'hlfir.reshape' op operand #1 must be any array-like entity containing a numerical type}}
+  %0 = hlfir.reshape %arg1 %arg0 : (!hlfir.expr<?xi32>, !fir.ref<!fir.array<?xi32>>) -> !hlfir.expr<?xi32>
+  return
+}
+
+// -----
+
+func.func @bad_reshape(%arg0: !fir.ref<!fir.array<?xi32>>, %arg1: !hlfir.expr<1xi32>) {
+  // expected-error@+1 {{'hlfir.reshape' op operand #2 must be any array-like entity}}
+  %0 = hlfir.reshape %arg1 %arg1 pad %arg0 : (!hlfir.expr<1xi32>, !hlfir.expr<1xi32>, !fir.ref<!fir.array<?xi32>>) -> !hlfir.expr<?xi32>
+  return
+}
+
+// -----
+
+func.func @bad_reshape(%arg0: !fir.ref<!fir.array<?xi32>>, %arg1: !hlfir.expr<1xi32>) {
+  // expected-error@+1 {{'hlfir.reshape' op operand #3 must be any array-like entity containing a numerical type}}
+  %0 = hlfir.reshape %arg1 %arg1 pad %arg1 order %arg0 : (!hlfir.expr<1xi32>, !hlfir.expr<1xi32>, !hlfir.expr<1xi32>, !fir.ref<!fir.array<?xi32>>) -> !hlfir.expr<?xi32>
+  return
+}
+
+// -----
+
+func.func @bad_reshape(%arg0: !hlfir.expr<1x!fir.char<1,2>>, %arg1: !hlfir.expr<1xi32>) {
+  // expected-error@+2 {{'hlfir.reshape' op character KIND mismatch}}
+  // expected-error@+1 {{'hlfir.reshape' op ARRAY and the result must have the same element type}}
+  %0 = hlfir.reshape %arg0 %arg1 : (!hlfir.expr<1x!fir.char<1,2>>, !hlfir.expr<1xi32>) -> !hlfir.expr<?x!fir.char<2,?>>
+  return
+}
+
+// -----
+
+func.func @bad_reshape(%arg0: !hlfir.expr<1x!fir.char<1,2>>, %arg1: !hlfir.expr<1xi32>, %arg2: !hlfir.expr<1x!fir.char<2,?>>) {
+  // expected-error@+2 {{'hlfir.reshape' op character KIND mismatch}}
+  // expected-error@+1 {{'hlfir.reshape' op ARRAY and PAD must be of the same type}}
+  %0 = hlfir.reshape %arg0 %arg1 pad %arg2 : (!hlfir.expr<1x!fir.char<1,2>>, !hlfir.expr<1xi32>, !hlfir.expr<1x!fir.char<2,?>>) -> !hlfir.expr<?x!fir.char<1,?>>
+  return
+}
diff --git a/flang/test/HLFIR/maxval-elemental.fir b/flang/test/HLFIR/maxval-elemental.fir
index aa642253b0832..9dc028abe8da3 100644
--- a/flang/test/HLFIR/maxval-elemental.fir
+++ b/flang/test/HLFIR/maxval-elemental.fir
@@ -82,9 +82,9 @@ func.func @_QPtest_float(%arg0: !fir.box<!fir.array<?xf32>> {fir.bindc_name = "a
 // CHECK-NEXT:  %[[V6:.*]] = fir.do_loop %arg1 = %c1 to %c4 step %c1 iter_args(%arg2 = %cst) -> (f32) {
 // CHECK-NEXT:      %[[V7:.*]] = hlfir.designate %[[V5]] (%arg1)  : (!fir.box<!fir.array<4xf32>>, index) -> !fir.ref<f32>
 // CHECK-NEXT:      %[[V8:.*]] = fir.load %[[V7]] : !fir.ref<f32>
-// CHECK-NEXT:      %[[V9:.*]] = arith.cmpf ogt, %[[V8]], %arg2 : f32
-// CHECK-NEXT:      %[[V10:.*]] = arith.cmpf une, %arg2, %arg2 : f32
-// CHECK-NEXT:      %[[V11:.*]] = arith.cmpf oeq, %[[V8]], %[[V8]] : f32
+// CHECK-NEXT:      %[[V9:.*]] = arith.cmpf ogt, %[[V8]], %arg2 fastmath<contract> : f32
+// CHECK-NEXT:      %[[V10:.*]] = arith.cmpf une, %arg2, %arg2 fastmath<contract> : f32
+// CHECK-NEXT:      %[[V11:.*]] = arith.cmpf oeq, %[[V8]], %[[V8]] fastmath<contract> : f32
 // CHECK-NEXT:      %[[V12:.*]] = arith.andi %[[V10]], %[[V11]] : i1
 // CHECK-NEXT:      %[[V13:.*]] = arith.ori %[[V9]], %[[V12]] : i1
 // CHECK-NEXT:      %[[V14:.*]] = arith.select %[[V13]], %[[V8]], %arg2 : f32
diff --git a/flang/test/HLFIR/maxval-lowering.fir b/flang/test/HLFIR/maxval-lowering.fir
index 7e025c41c6aeb..fbf75d9050544 100644
--- a/flang/test/HLFIR/maxval-lowering.fir
+++ b/flang/test/HLFIR/maxval-lowering.fir
@@ -216,3 +216,25 @@ func.func @_QPmaxval6(%arg0: !fir.box<!fir.array<?x!fir.char<1,?>>> {fir.bindc_n
 // CHECK:         hlfir.destroy %[[ASEXPR]]
 // CHECK-NEXT:    return
 // CHECK-NEXT:  }
+
+func.func @_QPmaxval_opt_mask(%arg0: !fir.box<!fir.array<?x?xf32>> {fir.bindc_name = "input"}, %arg1: !fir.ref<!fir.logical<4>> {fir.bindc_name = "mask", fir.optional}) -> f32 {
+  %0 = fir.dummy_scope : !fir.dscope
+  %1:2 = hlfir.declare %arg0 dummy_scope %0 {fortran_attrs = #fir.var_attrs<intent_in>, uniq_name = "_QFmaxval_opt_maskEinput"} : (!fir.box<!fir.array<?x?xf32>>, !fir.dscope) -> (!fir.box<!fir.array<?x?xf32>>, !fir.box<!fir.array<?x?xf32>>)
+  %2:2 = hlfir.declare %arg1 dummy_scope %0 {fortran_attrs = #fir.var_attrs<intent_in, optional>, uniq_name = "_QFmaxval_opt_maskEmask"} : (!fir.ref<!fir.logical<4>>, !fir.dscope) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
+  %3 = fir.alloca f32 {bindc_name = "maxval_1", uniq_name = "_QFmaxval_opt_maskEmaxval_1"}
+  %4:2 = hlfir.declare %3 {uniq_name = "_QFmaxval_opt_maskEmaxval_1"} : (!fir.ref<f32>) -> (!fir.ref<f32>, !fir.ref<f32>)
+  %5 = fir.is_present %2#0 : (!fir.ref<!fir.logical<4>>) -> i1
+  %6 = fir.embox %2#1 : (!fir.ref<!fir.logical<4>>) -> !fir.box<!fir.logical<4>>
+  %7 = fir.absent !fir.box<!fir.logical<4>>
+  %8 = arith.select %5, %6, %7 : !fir.box<!fir.logical<4>>
+  %9 = hlfir.maxval %1#0 mask %8 : (!fir.box<!fir.array<?x?xf32>>, !fir.box<!fir.logical<4>>) -> f32
+  hlfir.assign %9 to %4#0 : f32, !fir.ref<f32>
+  %10 = fir.load %4#1 : !fir.ref<f32>
+  return %10 : f32
+}
+// CHECK-LABEL:   func.func @_QPmaxval_opt_mask(
+// CHECK:           %[[VAL_10:.*]] = fir.embox %{{.*}} : (!fir.ref<!fir.logical<4>>) -> !fir.box<!fir.logical<4>>
+// CHECK:           %[[VAL_11:.*]] = fir.absent !fir.box<!fir.logical<4>>
+// CHECK:           %[[VAL_12:.*]] = arith.select %{{.*}}, %[[VAL_10]], %[[VAL_11]] : !fir.box<!fir.logical<4>>
+// CHECK:           %[[VAL_17:.*]] = fir.convert %[[VAL_12]] : (!fir.box<!fir.logical<4>>) -> !fir.box<none>                                                                  
+// CHECK:           %[[VAL_18:.*]] = fir.call @_FortranAMaxvalReal4(%{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}) : (!fir.box<none>, !fir.ref<i8>, i32, i32, !fir.box<none>) -> f32
diff --git a/flang/test/HLFIR/minval-elemental.fir b/flang/test/HLFIR/minval-elemental.fir
index 8da1b1bdf515b..64cd5403ec558 100644
--- a/flang/test/HLFIR/minval-elemental.fir
+++ b/flang/test/HLFIR/minval-elemental.fir
@@ -82,9 +82,9 @@ func.func @_QPtest_float(%arg0: !fir.box<!fir.array<?xf32>> {fir.bindc_name = "a
 // CHECK-NEXT:  %[[V6:.*]] = fir.do_loop %arg1 = %c1 to %c4 step %c1 iter_args(%arg2 = %cst) -> (f32) {
 // CHECK-NEXT:      %[[V7:.*]] = hlfir.designate %[[V5]] (%arg1)  : (!fir.box<!fir.array<4xf32>>, index) -> !fir.ref<f32>
 // CHECK-NEXT:      %[[V8:.*]] = fir.load %[[V7]] : !fir.ref<f32>
-// CHECK-NEXT:      %[[V9:.*]] = arith.cmpf olt, %[[V8]], %arg2 : f32
-// CHECK-NEXT:      %[[V10:.*]] = arith.cmpf une, %arg2, %arg2 : f32
-// CHECK-NEXT:      %[[V11:.*]] = arith.cmpf oeq, %[[V8]], %[[V8]] : f32
+// CHECK-NEXT:      %[[V9:.*]] = arith.cmpf olt, %[[V8]], %arg2 fastmath<contract> : f32
+// CHECK-NEXT:      %[[V10:.*]] = arith.cmpf une, %arg2, %arg2 fastmath<contract> : f32
+// CHECK-NEXT:      %[[V11:.*]] = arith.cmpf oeq, %[[V8]], %[[V8]] fastmath<contract> : f32
 // CHECK-NEXT:      %[[V12:.*]] = arith.andi %[[V10]], %[[V11]] : i1
 // CHECK-NEXT:      %[[V13:.*]] = arith.ori %[[V9]], %[[V12]] : i1
 // CHECK-NEXT:      %[[V14:.*]] = arith.select %[[V13]], %[[V8]], %arg2 : f32
diff --git a/flang/test/HLFIR/opt-variable-assign-omp.fir b/flang/test/HLFIR/opt-variable-assign-omp.fir
index 10cb2b4408fb8..f3ba53283c746 100755
--- a/flang/test/HLFIR/opt-variable-assign-omp.fir
+++ b/flang/test/HLFIR/opt-variable-assign-omp.fir
@@ -7,8 +7,8 @@
 // TODO: we can't currently optimize this assign because alias analysis doesn't
 // know that the block arguments of the copy region cannot alias.
 
-omp.private {type = firstprivate} @_QFFbEl_firstprivate_box_Uxi32 : !fir.ref<!fir.box<!fir.array<?xi32>>> alloc {
-^bb0(%arg0: !fir.ref<!fir.box<!fir.array<?xi32>>>):
+omp.private {type = firstprivate} @_QFFbEl_firstprivate_box_Uxi32 : !fir.box<!fir.array<?xi32>> init {
+^bb0(%arg0: !fir.ref<!fir.box<!fir.array<?xi32>>>, %arg1: !fir.ref<!fir.box<!fir.array<?xi32>>>):
   %0 = fir.load %arg0 : !fir.ref<!fir.box<!fir.array<?xi32>>>
   %c0 = arith.constant 0 : index
   %1:3 = fir.box_dims %0, %c0 : (!fir.box<!fir.array<?xi32>>, index) -> (index, index, index)
@@ -20,9 +20,8 @@ omp.private {type = firstprivate} @_QFFbEl_firstprivate_box_Uxi32 : !fir.ref<!fi
   %5:3 = fir.box_dims %0, %c0_0 : (!fir.box<!fir.array<?xi32>>, index) -> (index, index, index)
   %6 = fir.shape_shift %5#0, %5#1 : (index, index) -> !fir.shapeshift<1>
   %7 = fir.rebox %4#0(%6) : (!fir.box<!fir.array<?xi32>>, !fir.shapeshift<1>) -> !fir.box<!fir.array<?xi32>>
-  %8 = fir.alloca !fir.box<!fir.array<?xi32>>
-  fir.store %7 to %8 : !fir.ref<!fir.box<!fir.array<?xi32>>>
-  omp.yield(%8 : !fir.ref<!fir.box<!fir.array<?xi32>>>)
+  fir.store %7 to %arg1 : !fir.ref<!fir.box<!fir.array<?xi32>>>
+  omp.yield(%arg1 : !fir.ref<!fir.box<!fir.array<?xi32>>>)
 } copy {
 ^bb0(%arg0: !fir.ref<!fir.box<!fir.array<?xi32>>>, %arg1 : !fir.ref<!fir.box<!fir.array<?xi32>>>):
   %0 = fir.load %arg0 {test.ptr = "load_from_block_arg"} : !fir.ref<!fir.box<!fir.array<?xi32>>>
diff --git a/flang/test/HLFIR/reshape-lowering.fir b/flang/test/HLFIR/reshape-lowering.fir
new file mode 100644
index 0000000000000..c2f060efc5044
--- /dev/null
+++ b/flang/test/HLFIR/reshape-lowering.fir
@@ -0,0 +1,443 @@
+// Test hlfir.reshape operation lowering to fir runtime call
+// RUN: fir-opt %s -lower-hlfir-intrinsics | FileCheck %s
+
+// reshape(x, y)
+func.func @_QPreshape1(%arg0: !fir.box<!fir.array<?xf32>> {fir.bindc_name = "x"}, %arg1: !fir.ref<!fir.array<1xi32>> {fir.bindc_name = "y"}) {
+  %c1 = arith.constant 1 : index
+  %0 = fir.dummy_scope : !fir.dscope
+  %1:2 = hlfir.declare %arg0 dummy_scope %0 {uniq_name = "_QFreshape1Ex"} : (!fir.box<!fir.array<?xf32>>, !fir.dscope) -> (!fir.box<!fir.array<?xf32>>, !fir.box<!fir.array<?xf32>>)
+  %2 = fir.shape %c1 : (index) -> !fir.shape<1>
+  %3:2 = hlfir.declare %arg1(%2) dummy_scope %0 {uniq_name = "_QFreshape1Ey"} : (!fir.ref<!fir.array<1xi32>>, !fir.shape<1>, !fir.dscope) -> (!fir.ref<!fir.array<1xi32>>, !fir.ref<!fir.array<1xi32>>)
+  %4 = hlfir.reshape %1#0 %3#0 : (!fir.box<!fir.array<?xf32>>, !fir.ref<!fir.array<1xi32>>) -> !hlfir.expr<?xf32>
+  hlfir.assign %4 to %1#0 : !hlfir.expr<?xf32>, !fir.box<!fir.array<?xf32>>
+  hlfir.destroy %4 : !hlfir.expr<?xf32>
+  return
+}
+// CHECK-LABEL:   func.func @_QPreshape1(
+// CHECK-SAME:                           %[[VAL_0:.*]]: !fir.box<!fir.array<?xf32>> {fir.bindc_name = "x"},
+// CHECK-SAME:                           %[[VAL_1:.*]]: !fir.ref<!fir.array<1xi32>> {fir.bindc_name = "y"}) {
+// CHECK:           %[[VAL_2:.*]] = arith.constant true
+// CHECK:           %[[VAL_3:.*]] = arith.constant {{[0-9]*}} : i32
+// CHECK:           %[[VAL_4:.*]] = arith.constant 0 : index
+// CHECK:           %[[VAL_5:.*]] = arith.constant 1 : index
+// CHECK:           %[[VAL_6:.*]] = fir.alloca !fir.box<!fir.heap<!fir.array<?xf32>>>
+// CHECK:           %[[VAL_7:.*]] = fir.dummy_scope : !fir.dscope
+// CHECK:           %[[VAL_8:.*]]:2 = hlfir.declare %[[VAL_0]] dummy_scope %[[VAL_7]] {uniq_name = "_QFreshape1Ex"} : (!fir.box<!fir.array<?xf32>>, !fir.dscope) -> (!fir.box<!fir.array<?xf32>>, !fir.box<!fir.array<?xf32>>)
+// CHECK:           %[[VAL_9:.*]] = fir.shape %[[VAL_5]] : (index) -> !fir.shape<1>
+// CHECK:           %[[VAL_10:.*]]:2 = hlfir.declare %[[VAL_1]](%[[VAL_9]]) dummy_scope %[[VAL_7]] {uniq_name = "_QFreshape1Ey"} : (!fir.ref<!fir.array<1xi32>>, !fir.shape<1>, !fir.dscope) -> (!fir.ref<!fir.array<1xi32>>, !fir.ref<!fir.array<1xi32>>)
+// CHECK:           %[[VAL_11:.*]] = fir.shape %[[VAL_5]] : (index) -> !fir.shape<1>
+// CHECK:           %[[VAL_12:.*]] = fir.embox %[[VAL_10]]#1(%[[VAL_11]]) : (!fir.ref<!fir.array<1xi32>>, !fir.shape<1>) -> !fir.box<!fir.array<1xi32>>
+// CHECK:           %[[VAL_13:.*]] = fir.absent !fir.box<i1>
+// CHECK:           %[[VAL_14:.*]] = fir.absent !fir.box<i1>
+// CHECK:           %[[VAL_15:.*]] = fir.zero_bits !fir.heap<!fir.array<?xf32>>
+// CHECK:           %[[VAL_16:.*]] = fir.shape %[[VAL_4]] : (index) -> !fir.shape<1>
+// CHECK:           %[[VAL_17:.*]] = fir.embox %[[VAL_15]](%[[VAL_16]]) : (!fir.heap<!fir.array<?xf32>>, !fir.shape<1>) -> !fir.box<!fir.heap<!fir.array<?xf32>>>
+// CHECK:           fir.store %[[VAL_17]] to %[[VAL_6]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>
+// CHECK:           %[[VAL_18:.*]] = fir.address_of(@_QQcl
+// CHECK:           %[[VAL_19:.*]] = fir.convert %[[VAL_6]] : (!fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>) -> !fir.ref<!fir.box<none>>
+// CHECK:           %[[VAL_20:.*]] = fir.convert %[[VAL_8]]#1 : (!fir.box<!fir.array<?xf32>>) -> !fir.box<none>
+// CHECK:           %[[VAL_21:.*]] = fir.convert %[[VAL_12]] : (!fir.box<!fir.array<1xi32>>) -> !fir.box<none>
+// CHECK:           %[[VAL_22:.*]] = fir.convert %[[VAL_13]] : (!fir.box<i1>) -> !fir.box<none>
+// CHECK:           %[[VAL_23:.*]] = fir.convert %[[VAL_14]] : (!fir.box<i1>) -> !fir.box<none>
+// CHECK:           %[[VAL_24:.*]] = fir.convert %[[VAL_18]] : (!fir.ref<!fir.char<1,{{[0-9]*}}>>) -> !fir.ref<i8>
+// CHECK:           fir.call @_FortranAReshape(%[[VAL_19]], %[[VAL_20]], %[[VAL_21]], %[[VAL_22]], %[[VAL_23]], %[[VAL_24]], %[[VAL_3]]) : (!fir.ref<!fir.box<none>>, !fir.box<none>, !fir.box<none>, !fir.box<none>, !fir.box<none>, !fir.ref<i8>, i32) -> ()
+// CHECK:           %[[VAL_25:.*]] = fir.load %[[VAL_6]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>
+// CHECK:           %[[VAL_26:.*]]:3 = fir.box_dims %[[VAL_25]], %[[VAL_4]] : (!fir.box<!fir.heap<!fir.array<?xf32>>>, index) -> (index, index, index)
+// CHECK:           %[[VAL_27:.*]] = fir.box_addr %[[VAL_25]] : (!fir.box<!fir.heap<!fir.array<?xf32>>>) -> !fir.heap<!fir.array<?xf32>>
+// CHECK:           %[[VAL_28:.*]] = fir.shape_shift %[[VAL_26]]#0, %[[VAL_26]]#1 : (index, index) -> !fir.shapeshift<1>
+// CHECK:           %[[VAL_29:.*]]:2 = hlfir.declare %[[VAL_27]](%[[VAL_28]]) {uniq_name = ".tmp.intrinsic_result"} : (!fir.heap<!fir.array<?xf32>>, !fir.shapeshift<1>) -> (!fir.box<!fir.array<?xf32>>, !fir.heap<!fir.array<?xf32>>)
+// CHECK:           %[[VAL_30:.*]] = hlfir.as_expr %[[VAL_29]]#0 move %[[VAL_2]] : (!fir.box<!fir.array<?xf32>>, i1) -> !hlfir.expr<?xf32>
+// CHECK:           hlfir.assign %[[VAL_30]] to %[[VAL_8]]#0 : !hlfir.expr<?xf32>, !fir.box<!fir.array<?xf32>>
+// CHECK:           hlfir.destroy %[[VAL_30]] : !hlfir.expr<?xf32>
+// CHECK:           return
+// CHECK:         }
+
+// reshape(x, y, pad)
+func.func @_QPreshape2(%arg0: !fir.box<!fir.array<?xf32>> {fir.bindc_name = "x"}, %arg1: !fir.ref<!fir.array<1xi32>> {fir.bindc_name = "y"}, %arg2: !fir.box<!fir.array<?xf32>> {fir.bindc_name = "pad"}) {
+  %c1 = arith.constant 1 : index
+  %0 = fir.dummy_scope : !fir.dscope
+  %1:2 = hlfir.declare %arg2 dummy_scope %0 {uniq_name = "_QFreshape2Epad"} : (!fir.box<!fir.array<?xf32>>, !fir.dscope) -> (!fir.box<!fir.array<?xf32>>, !fir.box<!fir.array<?xf32>>)
+  %2:2 = hlfir.declare %arg0 dummy_scope %0 {uniq_name = "_QFreshape2Ex"} : (!fir.box<!fir.array<?xf32>>, !fir.dscope) -> (!fir.box<!fir.array<?xf32>>, !fir.box<!fir.array<?xf32>>)
+  %3 = fir.shape %c1 : (index) -> !fir.shape<1>
+  %4:2 = hlfir.declare %arg1(%3) dummy_scope %0 {uniq_name = "_QFreshape2Ey"} : (!fir.ref<!fir.array<1xi32>>, !fir.shape<1>, !fir.dscope) -> (!fir.ref<!fir.array<1xi32>>, !fir.ref<!fir.array<1xi32>>)
+  %5 = hlfir.reshape %2#0 %4#0 pad %1#0 : (!fir.box<!fir.array<?xf32>>, !fir.ref<!fir.array<1xi32>>, !fir.box<!fir.array<?xf32>>) -> !hlfir.expr<?xf32>
+  hlfir.assign %5 to %2#0 : !hlfir.expr<?xf32>, !fir.box<!fir.array<?xf32>>
+  hlfir.destroy %5 : !hlfir.expr<?xf32>
+  return
+}
+// CHECK-LABEL:   func.func @_QPreshape2(
+// CHECK-SAME:                           %[[VAL_0:.*]]: !fir.box<!fir.array<?xf32>> {fir.bindc_name = "x"},
+// CHECK-SAME:                           %[[VAL_1:.*]]: !fir.ref<!fir.array<1xi32>> {fir.bindc_name = "y"},
+// CHECK-SAME:                           %[[VAL_2:.*]]: !fir.box<!fir.array<?xf32>> {fir.bindc_name = "pad"}) {
+// CHECK:           %[[VAL_3:.*]] = arith.constant true
+// CHECK:           %[[VAL_4:.*]] = arith.constant {{[0-9]*}} : i32
+// CHECK:           %[[VAL_5:.*]] = arith.constant 0 : index
+// CHECK:           %[[VAL_6:.*]] = arith.constant 1 : index
+// CHECK:           %[[VAL_7:.*]] = fir.alloca !fir.box<!fir.heap<!fir.array<?xf32>>>
+// CHECK:           %[[VAL_8:.*]] = fir.dummy_scope : !fir.dscope
+// CHECK:           %[[VAL_9:.*]]:2 = hlfir.declare %[[VAL_2]] dummy_scope %[[VAL_8]] {uniq_name = "_QFreshape2Epad"} : (!fir.box<!fir.array<?xf32>>, !fir.dscope) -> (!fir.box<!fir.array<?xf32>>, !fir.box<!fir.array<?xf32>>)
+// CHECK:           %[[VAL_10:.*]]:2 = hlfir.declare %[[VAL_0]] dummy_scope %[[VAL_8]] {uniq_name = "_QFreshape2Ex"} : (!fir.box<!fir.array<?xf32>>, !fir.dscope) -> (!fir.box<!fir.array<?xf32>>, !fir.box<!fir.array<?xf32>>)
+// CHECK:           %[[VAL_11:.*]] = fir.shape %[[VAL_6]] : (index) -> !fir.shape<1>
+// CHECK:           %[[VAL_12:.*]]:2 = hlfir.declare %[[VAL_1]](%[[VAL_11]]) dummy_scope %[[VAL_8]] {uniq_name = "_QFreshape2Ey"} : (!fir.ref<!fir.array<1xi32>>, !fir.shape<1>, !fir.dscope) -> (!fir.ref<!fir.array<1xi32>>, !fir.ref<!fir.array<1xi32>>)
+// CHECK:           %[[VAL_13:.*]] = fir.shape %[[VAL_6]] : (index) -> !fir.shape<1>
+// CHECK:           %[[VAL_14:.*]] = fir.embox %[[VAL_12]]#1(%[[VAL_13]]) : (!fir.ref<!fir.array<1xi32>>, !fir.shape<1>) -> !fir.box<!fir.array<1xi32>>
+// CHECK:           %[[VAL_15:.*]] = fir.absent !fir.box<i1>
+// CHECK:           %[[VAL_16:.*]] = fir.zero_bits !fir.heap<!fir.array<?xf32>>
+// CHECK:           %[[VAL_17:.*]] = fir.shape %[[VAL_5]] : (index) -> !fir.shape<1>
+// CHECK:           %[[VAL_18:.*]] = fir.embox %[[VAL_16]](%[[VAL_17]]) : (!fir.heap<!fir.array<?xf32>>, !fir.shape<1>) -> !fir.box<!fir.heap<!fir.array<?xf32>>>
+// CHECK:           fir.store %[[VAL_18]] to %[[VAL_7]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>
+// CHECK:           %[[VAL_19:.*]] = fir.address_of(@_QQcl
+// CHECK:           %[[VAL_20:.*]] = fir.convert %[[VAL_7]] : (!fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>) -> !fir.ref<!fir.box<none>>
+// CHECK:           %[[VAL_21:.*]] = fir.convert %[[VAL_10]]#1 : (!fir.box<!fir.array<?xf32>>) -> !fir.box<none>
+// CHECK:           %[[VAL_22:.*]] = fir.convert %[[VAL_14]] : (!fir.box<!fir.array<1xi32>>) -> !fir.box<none>
+// CHECK:           %[[VAL_23:.*]] = fir.convert %[[VAL_9]]#1 : (!fir.box<!fir.array<?xf32>>) -> !fir.box<none>
+// CHECK:           %[[VAL_24:.*]] = fir.convert %[[VAL_15]] : (!fir.box<i1>) -> !fir.box<none>
+// CHECK:           %[[VAL_25:.*]] = fir.convert %[[VAL_19]] : (!fir.ref<!fir.char<1,{{[0-9]*}}>>) -> !fir.ref<i8>
+// CHECK:           fir.call @_FortranAReshape(%[[VAL_20]], %[[VAL_21]], %[[VAL_22]], %[[VAL_23]], %[[VAL_24]], %[[VAL_25]], %[[VAL_4]]) : (!fir.ref<!fir.box<none>>, !fir.box<none>, !fir.box<none>, !fir.box<none>, !fir.box<none>, !fir.ref<i8>, i32) -> ()
+// CHECK:           %[[VAL_26:.*]] = fir.load %[[VAL_7]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>
+// CHECK:           %[[VAL_27:.*]]:3 = fir.box_dims %[[VAL_26]], %[[VAL_5]] : (!fir.box<!fir.heap<!fir.array<?xf32>>>, index) -> (index, index, index)
+// CHECK:           %[[VAL_28:.*]] = fir.box_addr %[[VAL_26]] : (!fir.box<!fir.heap<!fir.array<?xf32>>>) -> !fir.heap<!fir.array<?xf32>>
+// CHECK:           %[[VAL_29:.*]] = fir.shape_shift %[[VAL_27]]#0, %[[VAL_27]]#1 : (index, index) -> !fir.shapeshift<1>
+// CHECK:           %[[VAL_30:.*]]:2 = hlfir.declare %[[VAL_28]](%[[VAL_29]]) {uniq_name = ".tmp.intrinsic_result"} : (!fir.heap<!fir.array<?xf32>>, !fir.shapeshift<1>) -> (!fir.box<!fir.array<?xf32>>, !fir.heap<!fir.array<?xf32>>)
+// CHECK:           %[[VAL_31:.*]] = hlfir.as_expr %[[VAL_30]]#0 move %[[VAL_3]] : (!fir.box<!fir.array<?xf32>>, i1) -> !hlfir.expr<?xf32>
+// CHECK:           hlfir.assign %[[VAL_31]] to %[[VAL_10]]#0 : !hlfir.expr<?xf32>, !fir.box<!fir.array<?xf32>>
+// CHECK:           hlfir.destroy %[[VAL_31]] : !hlfir.expr<?xf32>
+// CHECK:           return
+// CHECK:         }
+
+// reshape(x, y, order=order)
+func.func @_QPreshape3(%arg0: !fir.box<!fir.array<?xf32>> {fir.bindc_name = "x"}, %arg1: !fir.ref<!fir.array<1xi32>> {fir.bindc_name = "y"}, %arg2: !fir.box<!fir.array<?xi32>> {fir.bindc_name = "order"}) {
+  %c1 = arith.constant 1 : index
+  %0 = fir.dummy_scope : !fir.dscope
+  %1:2 = hlfir.declare %arg2 dummy_scope %0 {uniq_name = "_QFreshape3Eorder"} : (!fir.box<!fir.array<?xi32>>, !fir.dscope) -> (!fir.box<!fir.array<?xi32>>, !fir.box<!fir.array<?xi32>>)
+  %2:2 = hlfir.declare %arg0 dummy_scope %0 {uniq_name = "_QFreshape3Ex"} : (!fir.box<!fir.array<?xf32>>, !fir.dscope) -> (!fir.box<!fir.array<?xf32>>, !fir.box<!fir.array<?xf32>>)
+  %3 = fir.shape %c1 : (index) -> !fir.shape<1>
+  %4:2 = hlfir.declare %arg1(%3) dummy_scope %0 {uniq_name = "_QFreshape3Ey"} : (!fir.ref<!fir.array<1xi32>>, !fir.shape<1>, !fir.dscope) -> (!fir.ref<!fir.array<1xi32>>, !fir.ref<!fir.array<1xi32>>)
+  %5 = hlfir.reshape %2#0 %4#0 order %1#0 : (!fir.box<!fir.array<?xf32>>, !fir.ref<!fir.array<1xi32>>, !fir.box<!fir.array<?xi32>>) -> !hlfir.expr<?xf32>
+  hlfir.assign %5 to %2#0 : !hlfir.expr<?xf32>, !fir.box<!fir.array<?xf32>>
+  hlfir.destroy %5 : !hlfir.expr<?xf32>
+  return
+}
+// CHECK-LABEL:   func.func @_QPreshape3(
+// CHECK-SAME:                           %[[VAL_0:.*]]: !fir.box<!fir.array<?xf32>> {fir.bindc_name = "x"},
+// CHECK-SAME:                           %[[VAL_1:.*]]: !fir.ref<!fir.array<1xi32>> {fir.bindc_name = "y"},
+// CHECK-SAME:                           %[[VAL_2:.*]]: !fir.box<!fir.array<?xi32>> {fir.bindc_name = "order"}) {
+// CHECK:           %[[VAL_3:.*]] = arith.constant true
+// CHECK:           %[[VAL_4:.*]] = arith.constant {{[0-9]*}} : i32
+// CHECK:           %[[VAL_5:.*]] = arith.constant 0 : index
+// CHECK:           %[[VAL_6:.*]] = arith.constant 1 : index
+// CHECK:           %[[VAL_7:.*]] = fir.alloca !fir.box<!fir.heap<!fir.array<?xf32>>>
+// CHECK:           %[[VAL_8:.*]] = fir.dummy_scope : !fir.dscope
+// CHECK:           %[[VAL_9:.*]]:2 = hlfir.declare %[[VAL_2]] dummy_scope %[[VAL_8]] {uniq_name = "_QFreshape3Eorder"} : (!fir.box<!fir.array<?xi32>>, !fir.dscope) -> (!fir.box<!fir.array<?xi32>>, !fir.box<!fir.array<?xi32>>)
+// CHECK:           %[[VAL_10:.*]]:2 = hlfir.declare %[[VAL_0]] dummy_scope %[[VAL_8]] {uniq_name = "_QFreshape3Ex"} : (!fir.box<!fir.array<?xf32>>, !fir.dscope) -> (!fir.box<!fir.array<?xf32>>, !fir.box<!fir.array<?xf32>>)
+// CHECK:           %[[VAL_11:.*]] = fir.shape %[[VAL_6]] : (index) -> !fir.shape<1>
+// CHECK:           %[[VAL_12:.*]]:2 = hlfir.declare %[[VAL_1]](%[[VAL_11]]) dummy_scope %[[VAL_8]] {uniq_name = "_QFreshape3Ey"} : (!fir.ref<!fir.array<1xi32>>, !fir.shape<1>, !fir.dscope) -> (!fir.ref<!fir.array<1xi32>>, !fir.ref<!fir.array<1xi32>>)
+// CHECK:           %[[VAL_13:.*]] = fir.shape %[[VAL_6]] : (index) -> !fir.shape<1>
+// CHECK:           %[[VAL_14:.*]] = fir.embox %[[VAL_12]]#1(%[[VAL_13]]) : (!fir.ref<!fir.array<1xi32>>, !fir.shape<1>) -> !fir.box<!fir.array<1xi32>>
+// CHECK:           %[[VAL_15:.*]] = fir.absent !fir.box<i1>
+// CHECK:           %[[VAL_16:.*]] = fir.zero_bits !fir.heap<!fir.array<?xf32>>
+// CHECK:           %[[VAL_17:.*]] = fir.shape %[[VAL_5]] : (index) -> !fir.shape<1>
+// CHECK:           %[[VAL_18:.*]] = fir.embox %[[VAL_16]](%[[VAL_17]]) : (!fir.heap<!fir.array<?xf32>>, !fir.shape<1>) -> !fir.box<!fir.heap<!fir.array<?xf32>>>
+// CHECK:           fir.store %[[VAL_18]] to %[[VAL_7]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>
+// CHECK:           %[[VAL_19:.*]] = fir.address_of(@_QQcl
+// CHECK:           %[[VAL_20:.*]] = fir.convert %[[VAL_7]] : (!fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>) -> !fir.ref<!fir.box<none>>
+// CHECK:           %[[VAL_21:.*]] = fir.convert %[[VAL_10]]#1 : (!fir.box<!fir.array<?xf32>>) -> !fir.box<none>
+// CHECK:           %[[VAL_22:.*]] = fir.convert %[[VAL_14]] : (!fir.box<!fir.array<1xi32>>) -> !fir.box<none>
+// CHECK:           %[[VAL_23:.*]] = fir.convert %[[VAL_15]] : (!fir.box<i1>) -> !fir.box<none>
+// CHECK:           %[[VAL_24:.*]] = fir.convert %[[VAL_9]]#1 : (!fir.box<!fir.array<?xi32>>) -> !fir.box<none>
+// CHECK:           %[[VAL_25:.*]] = fir.convert %[[VAL_19]] : (!fir.ref<!fir.char<1,{{[0-9]*}}>>) -> !fir.ref<i8>
+// CHECK:           fir.call @_FortranAReshape(%[[VAL_20]], %[[VAL_21]], %[[VAL_22]], %[[VAL_23]], %[[VAL_24]], %[[VAL_25]], %[[VAL_4]]) : (!fir.ref<!fir.box<none>>, !fir.box<none>, !fir.box<none>, !fir.box<none>, !fir.box<none>, !fir.ref<i8>, i32) -> ()
+// CHECK:           %[[VAL_26:.*]] = fir.load %[[VAL_7]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>
+// CHECK:           %[[VAL_27:.*]]:3 = fir.box_dims %[[VAL_26]], %[[VAL_5]] : (!fir.box<!fir.heap<!fir.array<?xf32>>>, index) -> (index, index, index)
+// CHECK:           %[[VAL_28:.*]] = fir.box_addr %[[VAL_26]] : (!fir.box<!fir.heap<!fir.array<?xf32>>>) -> !fir.heap<!fir.array<?xf32>>
+// CHECK:           %[[VAL_29:.*]] = fir.shape_shift %[[VAL_27]]#0, %[[VAL_27]]#1 : (index, index) -> !fir.shapeshift<1>
+// CHECK:           %[[VAL_30:.*]]:2 = hlfir.declare %[[VAL_28]](%[[VAL_29]]) {uniq_name = ".tmp.intrinsic_result"} : (!fir.heap<!fir.array<?xf32>>, !fir.shapeshift<1>) -> (!fir.box<!fir.array<?xf32>>, !fir.heap<!fir.array<?xf32>>)
+// CHECK:           %[[VAL_31:.*]] = hlfir.as_expr %[[VAL_30]]#0 move %[[VAL_3]] : (!fir.box<!fir.array<?xf32>>, i1) -> !hlfir.expr<?xf32>
+// CHECK:           hlfir.assign %[[VAL_31]] to %[[VAL_10]]#0 : !hlfir.expr<?xf32>, !fir.box<!fir.array<?xf32>>
+// CHECK:           hlfir.destroy %[[VAL_31]] : !hlfir.expr<?xf32>
+// CHECK:           return
+// CHECK:         }
+
+// reshape(x, y, pad, order)
+func.func @_QPreshape4(%arg0: !fir.box<!fir.array<?xf32>> {fir.bindc_name = "x"}, %arg1: !fir.ref<!fir.array<1xi32>> {fir.bindc_name = "y"}, %arg2: !fir.box<!fir.array<?xf32>> {fir.bindc_name = "pad"}, %arg3: !fir.box<!fir.array<?xi32>> {fir.bindc_name = "order"}) {
+  %c1 = arith.constant 1 : index
+  %0 = fir.dummy_scope : !fir.dscope
+  %1:2 = hlfir.declare %arg3 dummy_scope %0 {uniq_name = "_QFreshape4Eorder"} : (!fir.box<!fir.array<?xi32>>, !fir.dscope) -> (!fir.box<!fir.array<?xi32>>, !fir.box<!fir.array<?xi32>>)
+  %2:2 = hlfir.declare %arg2 dummy_scope %0 {uniq_name = "_QFreshape4Epad"} : (!fir.box<!fir.array<?xf32>>, !fir.dscope) -> (!fir.box<!fir.array<?xf32>>, !fir.box<!fir.array<?xf32>>)
+  %3:2 = hlfir.declare %arg0 dummy_scope %0 {uniq_name = "_QFreshape4Ex"} : (!fir.box<!fir.array<?xf32>>, !fir.dscope) -> (!fir.box<!fir.array<?xf32>>, !fir.box<!fir.array<?xf32>>)
+  %4 = fir.shape %c1 : (index) -> !fir.shape<1>
+  %5:2 = hlfir.declare %arg1(%4) dummy_scope %0 {uniq_name = "_QFreshape4Ey"} : (!fir.ref<!fir.array<1xi32>>, !fir.shape<1>, !fir.dscope) -> (!fir.ref<!fir.array<1xi32>>, !fir.ref<!fir.array<1xi32>>)
+  %6 = hlfir.reshape %3#0 %5#0 pad %2#0 order %1#0 : (!fir.box<!fir.array<?xf32>>, !fir.ref<!fir.array<1xi32>>, !fir.box<!fir.array<?xf32>>, !fir.box<!fir.array<?xi32>>) -> !hlfir.expr<?xf32>
+  hlfir.assign %6 to %3#0 : !hlfir.expr<?xf32>, !fir.box<!fir.array<?xf32>>
+  hlfir.destroy %6 : !hlfir.expr<?xf32>
+  return
+}
+// CHECK-LABEL:   func.func @_QPreshape4(
+// CHECK-SAME:                           %[[VAL_0:.*]]: !fir.box<!fir.array<?xf32>> {fir.bindc_name = "x"},
+// CHECK-SAME:                           %[[VAL_1:.*]]: !fir.ref<!fir.array<1xi32>> {fir.bindc_name = "y"},
+// CHECK-SAME:                           %[[VAL_2:.*]]: !fir.box<!fir.array<?xf32>> {fir.bindc_name = "pad"},
+// CHECK-SAME:                           %[[VAL_3:.*]]: !fir.box<!fir.array<?xi32>> {fir.bindc_name = "order"}) {
+// CHECK:           %[[VAL_4:.*]] = arith.constant true
+// CHECK:           %[[VAL_5:.*]] = arith.constant {{[0-9]*}} : i32
+// CHECK:           %[[VAL_6:.*]] = arith.constant 0 : index
+// CHECK:           %[[VAL_7:.*]] = arith.constant 1 : index
+// CHECK:           %[[VAL_8:.*]] = fir.alloca !fir.box<!fir.heap<!fir.array<?xf32>>>
+// CHECK:           %[[VAL_9:.*]] = fir.dummy_scope : !fir.dscope
+// CHECK:           %[[VAL_10:.*]]:2 = hlfir.declare %[[VAL_3]] dummy_scope %[[VAL_9]] {uniq_name = "_QFreshape4Eorder"} : (!fir.box<!fir.array<?xi32>>, !fir.dscope) -> (!fir.box<!fir.array<?xi32>>, !fir.box<!fir.array<?xi32>>)
+// CHECK:           %[[VAL_11:.*]]:2 = hlfir.declare %[[VAL_2]] dummy_scope %[[VAL_9]] {uniq_name = "_QFreshape4Epad"} : (!fir.box<!fir.array<?xf32>>, !fir.dscope) -> (!fir.box<!fir.array<?xf32>>, !fir.box<!fir.array<?xf32>>)
+// CHECK:           %[[VAL_12:.*]]:2 = hlfir.declare %[[VAL_0]] dummy_scope %[[VAL_9]] {uniq_name = "_QFreshape4Ex"} : (!fir.box<!fir.array<?xf32>>, !fir.dscope) -> (!fir.box<!fir.array<?xf32>>, !fir.box<!fir.array<?xf32>>)
+// CHECK:           %[[VAL_13:.*]] = fir.shape %[[VAL_7]] : (index) -> !fir.shape<1>
+// CHECK:           %[[VAL_14:.*]]:2 = hlfir.declare %[[VAL_1]](%[[VAL_13]]) dummy_scope %[[VAL_9]] {uniq_name = "_QFreshape4Ey"} : (!fir.ref<!fir.array<1xi32>>, !fir.shape<1>, !fir.dscope) -> (!fir.ref<!fir.array<1xi32>>, !fir.ref<!fir.array<1xi32>>)
+// CHECK:           %[[VAL_15:.*]] = fir.shape %[[VAL_7]] : (index) -> !fir.shape<1>
+// CHECK:           %[[VAL_16:.*]] = fir.embox %[[VAL_14]]#1(%[[VAL_15]]) : (!fir.ref<!fir.array<1xi32>>, !fir.shape<1>) -> !fir.box<!fir.array<1xi32>>
+// CHECK:           %[[VAL_17:.*]] = fir.zero_bits !fir.heap<!fir.array<?xf32>>
+// CHECK:           %[[VAL_18:.*]] = fir.shape %[[VAL_6]] : (index) -> !fir.shape<1>
+// CHECK:           %[[VAL_19:.*]] = fir.embox %[[VAL_17]](%[[VAL_18]]) : (!fir.heap<!fir.array<?xf32>>, !fir.shape<1>) -> !fir.box<!fir.heap<!fir.array<?xf32>>>
+// CHECK:           fir.store %[[VAL_19]] to %[[VAL_8]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>
+// CHECK:           %[[VAL_20:.*]] = fir.address_of(@_QQcl
+// CHECK:           %[[VAL_21:.*]] = fir.convert %[[VAL_8]] : (!fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>) -> !fir.ref<!fir.box<none>>
+// CHECK:           %[[VAL_22:.*]] = fir.convert %[[VAL_12]]#1 : (!fir.box<!fir.array<?xf32>>) -> !fir.box<none>
+// CHECK:           %[[VAL_23:.*]] = fir.convert %[[VAL_16]] : (!fir.box<!fir.array<1xi32>>) -> !fir.box<none>
+// CHECK:           %[[VAL_24:.*]] = fir.convert %[[VAL_11]]#1 : (!fir.box<!fir.array<?xf32>>) -> !fir.box<none>
+// CHECK:           %[[VAL_25:.*]] = fir.convert %[[VAL_10]]#1 : (!fir.box<!fir.array<?xi32>>) -> !fir.box<none>
+// CHECK:           %[[VAL_26:.*]] = fir.convert %[[VAL_20]] : (!fir.ref<!fir.char<1,{{[0-9]*}}>>) -> !fir.ref<i8>
+// CHECK:           fir.call @_FortranAReshape(%[[VAL_21]], %[[VAL_22]], %[[VAL_23]], %[[VAL_24]], %[[VAL_25]], %[[VAL_26]], %[[VAL_5]]) : (!fir.ref<!fir.box<none>>, !fir.box<none>, !fir.box<none>, !fir.box<none>, !fir.box<none>, !fir.ref<i8>, i32) -> ()
+// CHECK:           %[[VAL_27:.*]] = fir.load %[[VAL_8]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>
+// CHECK:           %[[VAL_28:.*]]:3 = fir.box_dims %[[VAL_27]], %[[VAL_6]] : (!fir.box<!fir.heap<!fir.array<?xf32>>>, index) -> (index, index, index)
+// CHECK:           %[[VAL_29:.*]] = fir.box_addr %[[VAL_27]] : (!fir.box<!fir.heap<!fir.array<?xf32>>>) -> !fir.heap<!fir.array<?xf32>>
+// CHECK:           %[[VAL_30:.*]] = fir.shape_shift %[[VAL_28]]#0, %[[VAL_28]]#1 : (index, index) -> !fir.shapeshift<1>
+// CHECK:           %[[VAL_31:.*]]:2 = hlfir.declare %[[VAL_29]](%[[VAL_30]]) {uniq_name = ".tmp.intrinsic_result"} : (!fir.heap<!fir.array<?xf32>>, !fir.shapeshift<1>) -> (!fir.box<!fir.array<?xf32>>, !fir.heap<!fir.array<?xf32>>)
+// CHECK:           %[[VAL_32:.*]] = hlfir.as_expr %[[VAL_31]]#0 move %[[VAL_4]] : (!fir.box<!fir.array<?xf32>>, i1) -> !hlfir.expr<?xf32>
+// CHECK:           hlfir.assign %[[VAL_32]] to %[[VAL_12]]#0 : !hlfir.expr<?xf32>, !fir.box<!fir.array<?xf32>>
+// CHECK:           hlfir.destroy %[[VAL_32]] : !hlfir.expr<?xf32>
+// CHECK:           return
+// CHECK:         }
+
+// subroutine reshape5(x, y)
+//   use types
+//   integer :: y(1)
+//   type(t) :: x(:)
+//   x = reshape(x, y)
+// end subroutine reshape5
+func.func @_QPreshape5(%arg0: !fir.box<!fir.array<?x!fir.type<_QMtypesTt>>> {fir.bindc_name = "x"}, %arg1: !fir.ref<!fir.array<1xi32>> {fir.bindc_name = "y"}) {
+  %c1 = arith.constant 1 : index
+  %0 = fir.dummy_scope : !fir.dscope
+  %1:2 = hlfir.declare %arg0 dummy_scope %0 {uniq_name = "_QFreshape5Ex"} : (!fir.box<!fir.array<?x!fir.type<_QMtypesTt>>>, !fir.dscope) -> (!fir.box<!fir.array<?x!fir.type<_QMtypesTt>>>, !fir.box<!fir.array<?x!fir.type<_QMtypesTt>>>)
+  %2 = fir.shape %c1 : (index) -> !fir.shape<1>
+  %3:2 = hlfir.declare %arg1(%2) dummy_scope %0 {uniq_name = "_QFreshape5Ey"} : (!fir.ref<!fir.array<1xi32>>, !fir.shape<1>, !fir.dscope) -> (!fir.ref<!fir.array<1xi32>>, !fir.ref<!fir.array<1xi32>>)
+  %4 = hlfir.reshape %1#0 %3#0 : (!fir.box<!fir.array<?x!fir.type<_QMtypesTt>>>, !fir.ref<!fir.array<1xi32>>) -> !hlfir.expr<?x!fir.type<_QMtypesTt>>
+  hlfir.assign %4 to %1#0 : !hlfir.expr<?x!fir.type<_QMtypesTt>>, !fir.box<!fir.array<?x!fir.type<_QMtypesTt>>>
+  hlfir.destroy %4 : !hlfir.expr<?x!fir.type<_QMtypesTt>>
+  return
+}
+// CHECK-LABEL:   func.func @_QPreshape5(
+// CHECK-SAME:                           %[[VAL_0:.*]]: !fir.box<!fir.array<?x!fir.type<_QMtypesTt>>> {fir.bindc_name = "x"},
+// CHECK-SAME:                           %[[VAL_1:.*]]: !fir.ref<!fir.array<1xi32>> {fir.bindc_name = "y"}) {
+// CHECK:           %[[VAL_2:.*]] = arith.constant true
+// CHECK:           %[[VAL_3:.*]] = arith.constant {{[0-9]*}} : i32
+// CHECK:           %[[VAL_4:.*]] = arith.constant 0 : index
+// CHECK:           %[[VAL_5:.*]] = arith.constant 1 : index
+// CHECK:           %[[VAL_6:.*]] = fir.alloca !fir.box<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt>>>>
+// CHECK:           %[[VAL_7:.*]] = fir.dummy_scope : !fir.dscope
+// CHECK:           %[[VAL_8:.*]]:2 = hlfir.declare %[[VAL_0]] dummy_scope %[[VAL_7]] {uniq_name = "_QFreshape5Ex"} : (!fir.box<!fir.array<?x!fir.type<_QMtypesTt>>>, !fir.dscope) -> (!fir.box<!fir.array<?x!fir.type<_QMtypesTt>>>, !fir.box<!fir.array<?x!fir.type<_QMtypesTt>>>)
+// CHECK:           %[[VAL_9:.*]] = fir.shape %[[VAL_5]] : (index) -> !fir.shape<1>
+// CHECK:           %[[VAL_10:.*]]:2 = hlfir.declare %[[VAL_1]](%[[VAL_9]]) dummy_scope %[[VAL_7]] {uniq_name = "_QFreshape5Ey"} : (!fir.ref<!fir.array<1xi32>>, !fir.shape<1>, !fir.dscope) -> (!fir.ref<!fir.array<1xi32>>, !fir.ref<!fir.array<1xi32>>)
+// CHECK:           %[[VAL_11:.*]] = fir.shape %[[VAL_5]] : (index) -> !fir.shape<1>
+// CHECK:           %[[VAL_12:.*]] = fir.embox %[[VAL_10]]#1(%[[VAL_11]]) : (!fir.ref<!fir.array<1xi32>>, !fir.shape<1>) -> !fir.box<!fir.array<1xi32>>
+// CHECK:           %[[VAL_13:.*]] = fir.absent !fir.box<i1>
+// CHECK:           %[[VAL_14:.*]] = fir.absent !fir.box<i1>
+// CHECK:           %[[VAL_15:.*]] = fir.zero_bits !fir.heap<!fir.array<?x!fir.type<_QMtypesTt>>>
+// CHECK:           %[[VAL_16:.*]] = fir.shape %[[VAL_4]] : (index) -> !fir.shape<1>
+// CHECK:           %[[VAL_17:.*]] = fir.embox %[[VAL_15]](%[[VAL_16]]) : (!fir.heap<!fir.array<?x!fir.type<_QMtypesTt>>>, !fir.shape<1>) -> !fir.box<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt>>>>
+// CHECK:           fir.store %[[VAL_17]] to %[[VAL_6]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt>>>>>
+// CHECK:           %[[VAL_18:.*]] = fir.address_of(@_QQcl
+// CHECK:           %[[VAL_19:.*]] = fir.convert %[[VAL_6]] : (!fir.ref<!fir.box<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt>>>>>) -> !fir.ref<!fir.box<none>>
+// CHECK:           %[[VAL_20:.*]] = fir.convert %[[VAL_8]]#1 : (!fir.box<!fir.array<?x!fir.type<_QMtypesTt>>>) -> !fir.box<none>
+// CHECK:           %[[VAL_21:.*]] = fir.convert %[[VAL_12]] : (!fir.box<!fir.array<1xi32>>) -> !fir.box<none>
+// CHECK:           %[[VAL_22:.*]] = fir.convert %[[VAL_13]] : (!fir.box<i1>) -> !fir.box<none>
+// CHECK:           %[[VAL_23:.*]] = fir.convert %[[VAL_14]] : (!fir.box<i1>) -> !fir.box<none>
+// CHECK:           %[[VAL_24:.*]] = fir.convert %[[VAL_18]] : (!fir.ref<!fir.char<1,{{[0-9]*}}>>) -> !fir.ref<i8>
+// CHECK:           fir.call @_FortranAReshape(%[[VAL_19]], %[[VAL_20]], %[[VAL_21]], %[[VAL_22]], %[[VAL_23]], %[[VAL_24]], %[[VAL_3]]) : (!fir.ref<!fir.box<none>>, !fir.box<none>, !fir.box<none>, !fir.box<none>, !fir.box<none>, !fir.ref<i8>, i32) -> ()
+// CHECK:           %[[VAL_25:.*]] = fir.load %[[VAL_6]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt>>>>>
+// CHECK:           %[[VAL_26:.*]]:3 = fir.box_dims %[[VAL_25]], %[[VAL_4]] : (!fir.box<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt>>>>, index) -> (index, index, index)
+// CHECK:           %[[VAL_27:.*]] = fir.shift %[[VAL_26]]#0 : (index) -> !fir.shift<1>
+// CHECK:           %[[VAL_28:.*]]:2 = hlfir.declare %[[VAL_25]](%[[VAL_27]]) {uniq_name = ".tmp.intrinsic_result"} : (!fir.box<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt>>>>, !fir.shift<1>) -> (!fir.box<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt>>>>, !fir.box<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt>>>>)
+// CHECK:           %[[VAL_29:.*]] = hlfir.as_expr %[[VAL_28]]#0 move %[[VAL_2]] : (!fir.box<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt>>>>, i1) -> !hlfir.expr<?x!fir.type<_QMtypesTt>>
+// CHECK:           hlfir.assign %[[VAL_29]] to %[[VAL_8]]#0 : !hlfir.expr<?x!fir.type<_QMtypesTt>>, !fir.box<!fir.array<?x!fir.type<_QMtypesTt>>>
+// CHECK:           hlfir.destroy %[[VAL_29]] : !hlfir.expr<?x!fir.type<_QMtypesTt>>
+// CHECK:           return
+// CHECK:         }
+
+// subroutine reshape6(x, y)
+//   use types
+//   integer :: y(1)
+//   class(t), allocatable :: x(:)
+//   x = reshape(x, y)
+// end subroutine reshape6
+func.func @_QPreshape6(%arg0: !fir.ref<!fir.class<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt>>>>> {fir.bindc_name = "x"}, %arg1: !fir.ref<!fir.array<1xi32>> {fir.bindc_name = "y"}) {
+  %c1 = arith.constant 1 : index
+  %0 = fir.dummy_scope : !fir.dscope
+  %1:2 = hlfir.declare %arg0 dummy_scope %0 {fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "_QFreshape6Ex"} : (!fir.ref<!fir.class<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt>>>>>, !fir.dscope) -> (!fir.ref<!fir.class<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt>>>>>, !fir.ref<!fir.class<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt>>>>>)
+  %2 = fir.shape %c1 : (index) -> !fir.shape<1>
+  %3:2 = hlfir.declare %arg1(%2) dummy_scope %0 {uniq_name = "_QFreshape6Ey"} : (!fir.ref<!fir.array<1xi32>>, !fir.shape<1>, !fir.dscope) -> (!fir.ref<!fir.array<1xi32>>, !fir.ref<!fir.array<1xi32>>)
+  %4 = fir.load %1#0 : !fir.ref<!fir.class<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt>>>>>
+  %5 = hlfir.reshape %4 %3#0 : (!fir.class<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt>>>>, !fir.ref<!fir.array<1xi32>>) -> !hlfir.expr<?x!fir.type<_QMtypesTt>?>
+  hlfir.assign %5 to %1#0 realloc : !hlfir.expr<?x!fir.type<_QMtypesTt>?>, !fir.ref<!fir.class<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt>>>>>
+  hlfir.destroy %5 : !hlfir.expr<?x!fir.type<_QMtypesTt>?>
+  return
+}
+// CHECK-LABEL:   func.func @_QPreshape6(
+// CHECK-SAME:                           %[[VAL_0:.*]]: !fir.ref<!fir.class<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt>>>>> {fir.bindc_name = "x"},
+// CHECK-SAME:                           %[[VAL_1:.*]]: !fir.ref<!fir.array<1xi32>> {fir.bindc_name = "y"}) {
+// CHECK:           %[[VAL_2:.*]] = arith.constant true
+// CHECK:           %[[VAL_3:.*]] = arith.constant {{[0-9]*}} : i32
+// CHECK:           %[[VAL_4:.*]] = arith.constant 0 : index
+// CHECK:           %[[VAL_5:.*]] = arith.constant 1 : index
+// CHECK:           %[[VAL_6:.*]] = fir.alloca !fir.class<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt>>>>
+// CHECK:           %[[VAL_7:.*]] = fir.dummy_scope : !fir.dscope
+// CHECK:           %[[VAL_8:.*]]:2 = hlfir.declare %[[VAL_0]] dummy_scope %[[VAL_7]] {fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "_QFreshape6Ex"} : (!fir.ref<!fir.class<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt>>>>>, !fir.dscope) -> (!fir.ref<!fir.class<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt>>>>>, !fir.ref<!fir.class<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt>>>>>)
+// CHECK:           %[[VAL_9:.*]] = fir.shape %[[VAL_5]] : (index) -> !fir.shape<1>
+// CHECK:           %[[VAL_10:.*]]:2 = hlfir.declare %[[VAL_1]](%[[VAL_9]]) dummy_scope %[[VAL_7]] {uniq_name = "_QFreshape6Ey"} : (!fir.ref<!fir.array<1xi32>>, !fir.shape<1>, !fir.dscope) -> (!fir.ref<!fir.array<1xi32>>, !fir.ref<!fir.array<1xi32>>)
+// CHECK:           %[[VAL_11:.*]] = fir.load %[[VAL_8]]#0 : !fir.ref<!fir.class<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt>>>>>
+// CHECK:           %[[VAL_12:.*]] = fir.shape %[[VAL_5]] : (index) -> !fir.shape<1>
+// CHECK:           %[[VAL_13:.*]] = fir.embox %[[VAL_10]]#1(%[[VAL_12]]) : (!fir.ref<!fir.array<1xi32>>, !fir.shape<1>) -> !fir.box<!fir.array<1xi32>>
+// CHECK:           %[[VAL_14:.*]] = fir.absent !fir.box<i1>
+// CHECK:           %[[VAL_15:.*]] = fir.absent !fir.box<i1>
+// CHECK:           %[[VAL_16:.*]] = fir.zero_bits !fir.heap<!fir.array<?x!fir.type<_QMtypesTt>>>
+// CHECK:           %[[VAL_17:.*]] = fir.shape %[[VAL_4]] : (index) -> !fir.shape<1>
+// CHECK:           %[[VAL_18:.*]] = fir.embox %[[VAL_16]](%[[VAL_17]]) source_box %[[VAL_11]] : (!fir.heap<!fir.array<?x!fir.type<_QMtypesTt>>>, !fir.shape<1>, !fir.class<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt>>>>) -> !fir.class<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt>>>>
+// CHECK:           fir.store %[[VAL_18]] to %[[VAL_6]] : !fir.ref<!fir.class<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt>>>>>
+// CHECK:           %[[VAL_19:.*]] = fir.address_of(@_QQcl
+// CHECK:           %[[VAL_20:.*]] = fir.convert %[[VAL_6]] : (!fir.ref<!fir.class<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt>>>>>) -> !fir.ref<!fir.box<none>>
+// CHECK:           %[[VAL_21:.*]] = fir.convert %[[VAL_11]] : (!fir.class<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt>>>>) -> !fir.box<none>
+// CHECK:           %[[VAL_22:.*]] = fir.convert %[[VAL_13]] : (!fir.box<!fir.array<1xi32>>) -> !fir.box<none>
+// CHECK:           %[[VAL_23:.*]] = fir.convert %[[VAL_14]] : (!fir.box<i1>) -> !fir.box<none>
+// CHECK:           %[[VAL_24:.*]] = fir.convert %[[VAL_15]] : (!fir.box<i1>) -> !fir.box<none>
+// CHECK:           %[[VAL_25:.*]] = fir.convert %[[VAL_19]] : (!fir.ref<!fir.char<1,{{[0-9]*}}>>) -> !fir.ref<i8>
+// CHECK:           fir.call @_FortranAReshape(%[[VAL_20]], %[[VAL_21]], %[[VAL_22]], %[[VAL_23]], %[[VAL_24]], %[[VAL_25]], %[[VAL_3]]) : (!fir.ref<!fir.box<none>>, !fir.box<none>, !fir.box<none>, !fir.box<none>, !fir.box<none>, !fir.ref<i8>, i32) -> ()
+// CHECK:           %[[VAL_26:.*]] = fir.load %[[VAL_6]] : !fir.ref<!fir.class<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt>>>>>
+// CHECK:           %[[VAL_27:.*]]:3 = fir.box_dims %[[VAL_26]], %[[VAL_4]] : (!fir.class<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt>>>>, index) -> (index, index, index)
+// CHECK:           %[[VAL_28:.*]] = fir.shift %[[VAL_27]]#0 : (index) -> !fir.shift<1>
+// CHECK:           %[[VAL_29:.*]]:2 = hlfir.declare %[[VAL_26]](%[[VAL_28]]) {uniq_name = ".tmp.intrinsic_result"} : (!fir.class<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt>>>>, !fir.shift<1>) -> (!fir.class<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt>>>>, !fir.class<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt>>>>)
+// CHECK:           %[[VAL_30:.*]] = hlfir.as_expr %[[VAL_29]]#0 move %[[VAL_2]] : (!fir.class<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt>>>>, i1) -> !hlfir.expr<?x!fir.type<_QMtypesTt>?>
+// CHECK:           hlfir.assign %[[VAL_30]] to %[[VAL_8]]#0 realloc : !hlfir.expr<?x!fir.type<_QMtypesTt>?>, !fir.ref<!fir.class<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt>>>>>
+// CHECK:           hlfir.destroy %[[VAL_30]] : !hlfir.expr<?x!fir.type<_QMtypesTt>?>
+// CHECK:           return
+// CHECK:         }
+
+// subroutine reshape7(x, y, pad)
+//   use types
+//   integer :: y(1)
+//   type(t) :: x(:), pad(:)
+//   x = reshape(x, y, pad)
+// end subroutine reshape7
+func.func @_QPreshape7(%arg0: !fir.box<!fir.array<?x!fir.type<_QMtypesTt>>> {fir.bindc_name = "x"}, %arg1: !fir.ref<!fir.array<1xi32>> {fir.bindc_name = "y"}, %arg2: !fir.box<!fir.array<?x!fir.type<_QMtypesTt>>> {fir.bindc_name = "pad"}) {
+  %c1 = arith.constant 1 : index
+  %0 = fir.dummy_scope : !fir.dscope
+  %1:2 = hlfir.declare %arg2 dummy_scope %0 {uniq_name = "_QFreshape7Epad"} : (!fir.box<!fir.array<?x!fir.type<_QMtypesTt>>>, !fir.dscope) -> (!fir.box<!fir.array<?x!fir.type<_QMtypesTt>>>, !fir.box<!fir.array<?x!fir.type<_QMtypesTt>>>)
+  %2:2 = hlfir.declare %arg0 dummy_scope %0 {uniq_name = "_QFreshape7Ex"} : (!fir.box<!fir.array<?x!fir.type<_QMtypesTt>>>, !fir.dscope) -> (!fir.box<!fir.array<?x!fir.type<_QMtypesTt>>>, !fir.box<!fir.array<?x!fir.type<_QMtypesTt>>>)
+  %3 = fir.shape %c1 : (index) -> !fir.shape<1>
+  %4:2 = hlfir.declare %arg1(%3) dummy_scope %0 {uniq_name = "_QFreshape7Ey"} : (!fir.ref<!fir.array<1xi32>>, !fir.shape<1>, !fir.dscope) -> (!fir.ref<!fir.array<1xi32>>, !fir.ref<!fir.array<1xi32>>)
+  %5 = hlfir.reshape %2#0 %4#0 pad %1#0 : (!fir.box<!fir.array<?x!fir.type<_QMtypesTt>>>, !fir.ref<!fir.array<1xi32>>, !fir.box<!fir.array<?x!fir.type<_QMtypesTt>>>) -> !hlfir.expr<?x!fir.type<_QMtypesTt>>
+  hlfir.assign %5 to %2#0 : !hlfir.expr<?x!fir.type<_QMtypesTt>>, !fir.box<!fir.array<?x!fir.type<_QMtypesTt>>>
+  hlfir.destroy %5 : !hlfir.expr<?x!fir.type<_QMtypesTt>>
+  return
+}
+// CHECK-LABEL:   func.func @_QPreshape7(
+// CHECK-SAME:                           %[[VAL_0:.*]]: !fir.box<!fir.array<?x!fir.type<_QMtypesTt>>> {fir.bindc_name = "x"},
+// CHECK-SAME:                           %[[VAL_1:.*]]: !fir.ref<!fir.array<1xi32>> {fir.bindc_name = "y"},
+// CHECK-SAME:                           %[[VAL_2:.*]]: !fir.box<!fir.array<?x!fir.type<_QMtypesTt>>> {fir.bindc_name = "pad"}) {
+// CHECK:           %[[VAL_3:.*]] = arith.constant true
+// CHECK:           %[[VAL_4:.*]] = arith.constant {{[0-9]*}} : i32
+// CHECK:           %[[VAL_5:.*]] = arith.constant 0 : index
+// CHECK:           %[[VAL_6:.*]] = arith.constant 1 : index
+// CHECK:           %[[VAL_7:.*]] = fir.alloca !fir.box<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt>>>>
+// CHECK:           %[[VAL_8:.*]] = fir.dummy_scope : !fir.dscope
+// CHECK:           %[[VAL_9:.*]]:2 = hlfir.declare %[[VAL_2]] dummy_scope %[[VAL_8]] {uniq_name = "_QFreshape7Epad"} : (!fir.box<!fir.array<?x!fir.type<_QMtypesTt>>>, !fir.dscope) -> (!fir.box<!fir.array<?x!fir.type<_QMtypesTt>>>, !fir.box<!fir.array<?x!fir.type<_QMtypesTt>>>)
+// CHECK:           %[[VAL_10:.*]]:2 = hlfir.declare %[[VAL_0]] dummy_scope %[[VAL_8]] {uniq_name = "_QFreshape7Ex"} : (!fir.box<!fir.array<?x!fir.type<_QMtypesTt>>>, !fir.dscope) -> (!fir.box<!fir.array<?x!fir.type<_QMtypesTt>>>, !fir.box<!fir.array<?x!fir.type<_QMtypesTt>>>)
+// CHECK:           %[[VAL_11:.*]] = fir.shape %[[VAL_6]] : (index) -> !fir.shape<1>
+// CHECK:           %[[VAL_12:.*]]:2 = hlfir.declare %[[VAL_1]](%[[VAL_11]]) dummy_scope %[[VAL_8]] {uniq_name = "_QFreshape7Ey"} : (!fir.ref<!fir.array<1xi32>>, !fir.shape<1>, !fir.dscope) -> (!fir.ref<!fir.array<1xi32>>, !fir.ref<!fir.array<1xi32>>)
+// CHECK:           %[[VAL_13:.*]] = fir.shape %[[VAL_6]] : (index) -> !fir.shape<1>
+// CHECK:           %[[VAL_14:.*]] = fir.embox %[[VAL_12]]#1(%[[VAL_13]]) : (!fir.ref<!fir.array<1xi32>>, !fir.shape<1>) -> !fir.box<!fir.array<1xi32>>
+// CHECK:           %[[VAL_15:.*]] = fir.absent !fir.box<i1>
+// CHECK:           %[[VAL_16:.*]] = fir.zero_bits !fir.heap<!fir.array<?x!fir.type<_QMtypesTt>>>
+// CHECK:           %[[VAL_17:.*]] = fir.shape %[[VAL_5]] : (index) -> !fir.shape<1>
+// CHECK:           %[[VAL_18:.*]] = fir.embox %[[VAL_16]](%[[VAL_17]]) : (!fir.heap<!fir.array<?x!fir.type<_QMtypesTt>>>, !fir.shape<1>) -> !fir.box<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt>>>>
+// CHECK:           fir.store %[[VAL_18]] to %[[VAL_7]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt>>>>>
+// CHECK:           %[[VAL_19:.*]] = fir.address_of(@_QQcl
+// CHECK:           %[[VAL_20:.*]] = fir.convert %[[VAL_7]] : (!fir.ref<!fir.box<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt>>>>>) -> !fir.ref<!fir.box<none>>
+// CHECK:           %[[VAL_21:.*]] = fir.convert %[[VAL_10]]#1 : (!fir.box<!fir.array<?x!fir.type<_QMtypesTt>>>) -> !fir.box<none>
+// CHECK:           %[[VAL_22:.*]] = fir.convert %[[VAL_14]] : (!fir.box<!fir.array<1xi32>>) -> !fir.box<none>
+// CHECK:           %[[VAL_23:.*]] = fir.convert %[[VAL_9]]#1 : (!fir.box<!fir.array<?x!fir.type<_QMtypesTt>>>) -> !fir.box<none>
+// CHECK:           %[[VAL_24:.*]] = fir.convert %[[VAL_15]] : (!fir.box<i1>) -> !fir.box<none>
+// CHECK:           %[[VAL_25:.*]] = fir.convert %[[VAL_19]] : (!fir.ref<!fir.char<1,{{[0-9]*}}>>) -> !fir.ref<i8>
+// CHECK:           fir.call @_FortranAReshape(%[[VAL_20]], %[[VAL_21]], %[[VAL_22]], %[[VAL_23]], %[[VAL_24]], %[[VAL_25]], %[[VAL_4]]) : (!fir.ref<!fir.box<none>>, !fir.box<none>, !fir.box<none>, !fir.box<none>, !fir.box<none>, !fir.ref<i8>, i32) -> ()
+// CHECK:           %[[VAL_26:.*]] = fir.load %[[VAL_7]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt>>>>>
+// CHECK:           %[[VAL_27:.*]]:3 = fir.box_dims %[[VAL_26]], %[[VAL_5]] : (!fir.box<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt>>>>, index) -> (index, index, index)
+// CHECK:           %[[VAL_28:.*]] = fir.shift %[[VAL_27]]#0 : (index) -> !fir.shift<1>
+// CHECK:           %[[VAL_29:.*]]:2 = hlfir.declare %[[VAL_26]](%[[VAL_28]]) {uniq_name = ".tmp.intrinsic_result"} : (!fir.box<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt>>>>, !fir.shift<1>) -> (!fir.box<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt>>>>, !fir.box<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt>>>>)
+// CHECK:           %[[VAL_30:.*]] = hlfir.as_expr %[[VAL_29]]#0 move %[[VAL_3]] : (!fir.box<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt>>>>, i1) -> !hlfir.expr<?x!fir.type<_QMtypesTt>>
+// CHECK:           hlfir.assign %[[VAL_30]] to %[[VAL_10]]#0 : !hlfir.expr<?x!fir.type<_QMtypesTt>>, !fir.box<!fir.array<?x!fir.type<_QMtypesTt>>>
+// CHECK:           hlfir.destroy %[[VAL_30]] : !hlfir.expr<?x!fir.type<_QMtypesTt>>
+// CHECK:           return
+// CHECK:         }
+
+// subroutine reshape8(x, y, pad)
+//   use types
+//   integer :: y(1)
+//   class(t), allocatable :: x(:)
+//   class(t) :: pad(:)
+//   x = reshape(x, y, pad)
+// end subroutine reshape8
+func.func @_QPreshape8(%arg0: !fir.ref<!fir.class<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt>>>>> {fir.bindc_name = "x"}, %arg1: !fir.ref<!fir.array<1xi32>> {fir.bindc_name = "y"}, %arg2: !fir.class<!fir.array<?x!fir.type<_QMtypesTt>>> {fir.bindc_name = "pad"}) {
+  %c1 = arith.constant 1 : index
+  %0 = fir.dummy_scope : !fir.dscope
+  %1:2 = hlfir.declare %arg2 dummy_scope %0 {uniq_name = "_QFreshape8Epad"} : (!fir.class<!fir.array<?x!fir.type<_QMtypesTt>>>, !fir.dscope) -> (!fir.class<!fir.array<?x!fir.type<_QMtypesTt>>>, !fir.class<!fir.array<?x!fir.type<_QMtypesTt>>>)
+  %2:2 = hlfir.declare %arg0 dummy_scope %0 {fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "_QFreshape8Ex"} : (!fir.ref<!fir.class<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt>>>>>, !fir.dscope) -> (!fir.ref<!fir.class<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt>>>>>, !fir.ref<!fir.class<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt>>>>>)
+  %3 = fir.shape %c1 : (index) -> !fir.shape<1>
+  %4:2 = hlfir.declare %arg1(%3) dummy_scope %0 {uniq_name = "_QFreshape8Ey"} : (!fir.ref<!fir.array<1xi32>>, !fir.shape<1>, !fir.dscope) -> (!fir.ref<!fir.array<1xi32>>, !fir.ref<!fir.array<1xi32>>)
+  %5 = fir.load %2#0 : !fir.ref<!fir.class<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt>>>>>
+  %6 = hlfir.reshape %5 %4#0 pad %1#0 : (!fir.class<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt>>>>, !fir.ref<!fir.array<1xi32>>, !fir.class<!fir.array<?x!fir.type<_QMtypesTt>>>) -> !hlfir.expr<?x!fir.type<_QMtypesTt>?>
+  hlfir.assign %6 to %2#0 realloc : !hlfir.expr<?x!fir.type<_QMtypesTt>?>, !fir.ref<!fir.class<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt>>>>>
+  hlfir.destroy %6 : !hlfir.expr<?x!fir.type<_QMtypesTt>?>
+  return
+}
+// CHECK-LABEL:   func.func @_QPreshape8(
+// CHECK-SAME:                           %[[VAL_0:.*]]: !fir.ref<!fir.class<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt>>>>> {fir.bindc_name = "x"},
+// CHECK-SAME:                           %[[VAL_1:.*]]: !fir.ref<!fir.array<1xi32>> {fir.bindc_name = "y"},
+// CHECK-SAME:                           %[[VAL_2:.*]]: !fir.class<!fir.array<?x!fir.type<_QMtypesTt>>> {fir.bindc_name = "pad"}) {
+// CHECK:           %[[VAL_3:.*]] = arith.constant true
+// CHECK:           %[[VAL_4:.*]] = arith.constant {{[0-9]*}} : i32
+// CHECK:           %[[VAL_5:.*]] = arith.constant 0 : index
+// CHECK:           %[[VAL_6:.*]] = arith.constant 1 : index
+// CHECK:           %[[VAL_7:.*]] = fir.alloca !fir.class<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt>>>>
+// CHECK:           %[[VAL_8:.*]] = fir.dummy_scope : !fir.dscope
+// CHECK:           %[[VAL_9:.*]]:2 = hlfir.declare %[[VAL_2]] dummy_scope %[[VAL_8]] {uniq_name = "_QFreshape8Epad"} : (!fir.class<!fir.array<?x!fir.type<_QMtypesTt>>>, !fir.dscope) -> (!fir.class<!fir.array<?x!fir.type<_QMtypesTt>>>, !fir.class<!fir.array<?x!fir.type<_QMtypesTt>>>)
+// CHECK:           %[[VAL_10:.*]]:2 = hlfir.declare %[[VAL_0]] dummy_scope %[[VAL_8]] {fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "_QFreshape8Ex"} : (!fir.ref<!fir.class<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt>>>>>, !fir.dscope) -> (!fir.ref<!fir.class<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt>>>>>, !fir.ref<!fir.class<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt>>>>>)
+// CHECK:           %[[VAL_11:.*]] = fir.shape %[[VAL_6]] : (index) -> !fir.shape<1>
+// CHECK:           %[[VAL_12:.*]]:2 = hlfir.declare %[[VAL_1]](%[[VAL_11]]) dummy_scope %[[VAL_8]] {uniq_name = "_QFreshape8Ey"} : (!fir.ref<!fir.array<1xi32>>, !fir.shape<1>, !fir.dscope) -> (!fir.ref<!fir.array<1xi32>>, !fir.ref<!fir.array<1xi32>>)
+// CHECK:           %[[VAL_13:.*]] = fir.load %[[VAL_10]]#0 : !fir.ref<!fir.class<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt>>>>>
+// CHECK:           %[[VAL_14:.*]] = fir.shape %[[VAL_6]] : (index) -> !fir.shape<1>
+// CHECK:           %[[VAL_15:.*]] = fir.embox %[[VAL_12]]#1(%[[VAL_14]]) : (!fir.ref<!fir.array<1xi32>>, !fir.shape<1>) -> !fir.box<!fir.array<1xi32>>
+// CHECK:           %[[VAL_16:.*]] = fir.absent !fir.box<i1>
+// CHECK:           %[[VAL_17:.*]] = fir.zero_bits !fir.heap<!fir.array<?x!fir.type<_QMtypesTt>>>
+// CHECK:           %[[VAL_18:.*]] = fir.shape %[[VAL_5]] : (index) -> !fir.shape<1>
+// CHECK:           %[[VAL_19:.*]] = fir.embox %[[VAL_17]](%[[VAL_18]]) source_box %[[VAL_13]] : (!fir.heap<!fir.array<?x!fir.type<_QMtypesTt>>>, !fir.shape<1>, !fir.class<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt>>>>) -> !fir.class<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt>>>>
+// CHECK:           fir.store %[[VAL_19]] to %[[VAL_7]] : !fir.ref<!fir.class<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt>>>>>
+// CHECK:           %[[VAL_20:.*]] = fir.address_of(@_QQcl
+// CHECK:           %[[VAL_21:.*]] = fir.convert %[[VAL_7]] : (!fir.ref<!fir.class<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt>>>>>) -> !fir.ref<!fir.box<none>>
+// CHECK:           %[[VAL_22:.*]] = fir.convert %[[VAL_13]] : (!fir.class<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt>>>>) -> !fir.box<none>
+// CHECK:           %[[VAL_23:.*]] = fir.convert %[[VAL_15]] : (!fir.box<!fir.array<1xi32>>) -> !fir.box<none>
+// CHECK:           %[[VAL_24:.*]] = fir.convert %[[VAL_9]]#1 : (!fir.class<!fir.array<?x!fir.type<_QMtypesTt>>>) -> !fir.box<none>
+// CHECK:           %[[VAL_25:.*]] = fir.convert %[[VAL_16]] : (!fir.box<i1>) -> !fir.box<none>
+// CHECK:           %[[VAL_26:.*]] = fir.convert %[[VAL_20]] : (!fir.ref<!fir.char<1,{{[0-9]*}}>>) -> !fir.ref<i8>
+// CHECK:           fir.call @_FortranAReshape(%[[VAL_21]], %[[VAL_22]], %[[VAL_23]], %[[VAL_24]], %[[VAL_25]], %[[VAL_26]], %[[VAL_4]]) : (!fir.ref<!fir.box<none>>, !fir.box<none>, !fir.box<none>, !fir.box<none>, !fir.box<none>, !fir.ref<i8>, i32) -> ()
+// CHECK:           %[[VAL_27:.*]] = fir.load %[[VAL_7]] : !fir.ref<!fir.class<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt>>>>>
+// CHECK:           %[[VAL_28:.*]]:3 = fir.box_dims %[[VAL_27]], %[[VAL_5]] : (!fir.class<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt>>>>, index) -> (index, index, index)
+// CHECK:           %[[VAL_29:.*]] = fir.shift %[[VAL_28]]#0 : (index) -> !fir.shift<1>
+// CHECK:           %[[VAL_30:.*]]:2 = hlfir.declare %[[VAL_27]](%[[VAL_29]]) {uniq_name = ".tmp.intrinsic_result"} : (!fir.class<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt>>>>, !fir.shift<1>) -> (!fir.class<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt>>>>, !fir.class<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt>>>>)
+// CHECK:           %[[VAL_31:.*]] = hlfir.as_expr %[[VAL_30]]#0 move %[[VAL_3]] : (!fir.class<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt>>>>, i1) -> !hlfir.expr<?x!fir.type<_QMtypesTt>?>
+// CHECK:           hlfir.assign %[[VAL_31]] to %[[VAL_10]]#0 realloc : !hlfir.expr<?x!fir.type<_QMtypesTt>?>, !fir.ref<!fir.class<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt>>>>>
+// CHECK:           hlfir.destroy %[[VAL_31]] : !hlfir.expr<?x!fir.type<_QMtypesTt>?>
+// CHECK:           return
+// CHECK:         }
diff --git a/flang/test/HLFIR/reshape.fir b/flang/test/HLFIR/reshape.fir
new file mode 100644
index 0000000000000..4f586386e0909
--- /dev/null
+++ b/flang/test/HLFIR/reshape.fir
@@ -0,0 +1,86 @@
+// Test hlfir.reshape operation parse, verify (no errors), and unparse
+// RUN: fir-opt %s | fir-opt | FileCheck %s
+
+// Operands are expressions of known shape
+func.func @reshape1(%arg0: !hlfir.expr<42xi32>, %arg1: !hlfir.expr<1xi32>) -> !hlfir.expr<?xi32> {
+  %0 = hlfir.reshape %arg0 %arg1 : (!hlfir.expr<42xi32>, !hlfir.expr<1xi32>) -> !hlfir.expr<?xi32>
+  return %0 : !hlfir.expr<?xi32>
+}
+// CHECK-LABEL:   func.func @reshape1(
+// CHECK-SAME:                        %[[VAL_0:.*]]: !hlfir.expr<42xi32>,
+// CHECK-SAME:                        %[[VAL_1:.*]]: !hlfir.expr<1xi32>) -> !hlfir.expr<?xi32> {
+// CHECK:           %[[VAL_2:.*]] = hlfir.reshape %[[VAL_0]] %[[VAL_1]] : (!hlfir.expr<42xi32>, !hlfir.expr<1xi32>) -> !hlfir.expr<?xi32>
+// CHECK:           return %[[VAL_2]] : !hlfir.expr<?xi32>
+// CHECK:         }
+
+// Operands are expressions of assumed shape
+func.func @reshape2(%arg0: !hlfir.expr<?xi32>, %arg1: !hlfir.expr<1xi32>) -> !hlfir.expr<4xi32> {
+  %0 = hlfir.reshape %arg0 %arg1 pad %arg0 order %arg0 : (!hlfir.expr<?xi32>, !hlfir.expr<1xi32>, !hlfir.expr<?xi32>, !hlfir.expr<?xi32>) -> !hlfir.expr<4xi32>
+  return %0 : !hlfir.expr<4xi32>
+}
+// CHECK-LABEL:   func.func @reshape2(
+// CHECK-SAME:                        %[[VAL_0:.*]]: !hlfir.expr<?xi32>,
+// CHECK-SAME:                        %[[VAL_1:.*]]: !hlfir.expr<1xi32>) -> !hlfir.expr<4xi32> {
+// CHECK:           %[[VAL_2:.*]] = hlfir.reshape %[[VAL_0]] %[[VAL_1]] pad %[[VAL_0]] order %[[VAL_0]] : (!hlfir.expr<?xi32>, !hlfir.expr<1xi32>, !hlfir.expr<?xi32>, !hlfir.expr<?xi32>) -> !hlfir.expr<4xi32>
+// CHECK:           return %[[VAL_2]] : !hlfir.expr<4xi32>
+// CHECK:         }
+
+// Operands are boxed array
+func.func @reshape3(%arg0: !fir.box<!fir.array<42xi32>>, %arg1: !fir.box<!fir.array<2xi32>>) -> !hlfir.expr<?x?xi32> {
+  %0 = hlfir.reshape %arg0 %arg1 : (!fir.box<!fir.array<42xi32>>, !fir.box<!fir.array<2xi32>>) -> !hlfir.expr<?x?xi32>
+  return %0 : !hlfir.expr<?x?xi32>
+}
+// CHECK-LABEL:   func.func @reshape3(
+// CHECK-SAME:                        %[[VAL_0:.*]]: !fir.box<!fir.array<42xi32>>,
+// CHECK-SAME:                        %[[VAL_1:.*]]: !fir.box<!fir.array<2xi32>>) -> !hlfir.expr<?x?xi32> {
+// CHECK:           %[[VAL_2:.*]] = hlfir.reshape %[[VAL_0]] %[[VAL_1]] : (!fir.box<!fir.array<42xi32>>, !fir.box<!fir.array<2xi32>>) -> !hlfir.expr<?x?xi32>
+// CHECK:           return %[[VAL_2]] : !hlfir.expr<?x?xi32>
+// CHECK:         }
+
+// Operands are assumed shape boxed arrays
+func.func @reshape4(%arg0: !fir.box<!fir.array<?xi32>>, %arg1: !fir.box<!fir.array<1xi32>>) -> !hlfir.expr<?xi32> {
+  %0 = hlfir.reshape %arg0 %arg1 pad %arg0 order %arg0 : (!fir.box<!fir.array<?xi32>>, !fir.box<!fir.array<1xi32>>, !fir.box<!fir.array<?xi32>>, !fir.box<!fir.array<?xi32>>) -> !hlfir.expr<?xi32>
+  return %0 : !hlfir.expr<?xi32>
+}
+// CHECK-LABEL:   func.func @reshape4(
+// CHECK-SAME:                        %[[VAL_0:.*]]: !fir.box<!fir.array<?xi32>>,
+// CHECK-SAME:                        %[[VAL_1:.*]]: !fir.box<!fir.array<1xi32>>) -> !hlfir.expr<?xi32> {
+// CHECK:           %[[VAL_2:.*]] = hlfir.reshape %[[VAL_0]] %[[VAL_1]] pad %[[VAL_0]] order %[[VAL_0]] : (!fir.box<!fir.array<?xi32>>, !fir.box<!fir.array<1xi32>>, !fir.box<!fir.array<?xi32>>, !fir.box<!fir.array<?xi32>>) -> !hlfir.expr<?xi32>
+// CHECK:           return %[[VAL_2]] : !hlfir.expr<?xi32>
+// CHECK:         }
+
+// Operands are ref<array<>> of known shape
+func.func @reshape5(%arg0: !fir.ref<!fir.array<1xi32>>) -> !hlfir.expr<?xi32> {
+  %0 = hlfir.reshape %arg0 %arg0 pad %arg0 order %arg0 : (!fir.ref<!fir.array<1xi32>>, !fir.ref<!fir.array<1xi32>>, !fir.ref<!fir.array<1xi32>>, !fir.ref<!fir.array<1xi32>>) -> !hlfir.expr<?xi32>
+  return %0 : !hlfir.expr<?xi32>
+}
+// CHECK-LABEL:   func.func @reshape5(
+// CHECK-SAME:                        %[[VAL_0:.*]]: !fir.ref<!fir.array<1xi32>>) -> !hlfir.expr<?xi32> {
+// CHECK:           %[[VAL_1:.*]] = hlfir.reshape %[[VAL_0]] %[[VAL_0]] pad %[[VAL_0]] order %[[VAL_0]] : (!fir.ref<!fir.array<1xi32>>, !fir.ref<!fir.array<1xi32>>, !fir.ref<!fir.array<1xi32>>, !fir.ref<!fir.array<1xi32>>) -> !hlfir.expr<?xi32>
+// CHECK:           return %[[VAL_1]] : !hlfir.expr<?xi32>
+// CHECK:         }
+
+// Polymorphic operands
+func.func @reshape6(%arg0: !fir.class<!fir.heap<!fir.array<?x!fir.type<whatever>>>>, %arg1: !hlfir.expr<1xi32>) -> !hlfir.expr<?x!fir.type<whatever>?> {
+  %0 = hlfir.reshape %arg0 %arg1 pad %arg0 : (!fir.class<!fir.heap<!fir.array<?x!fir.type<whatever>>>>, !hlfir.expr<1xi32>, !fir.class<!fir.heap<!fir.array<?x!fir.type<whatever>>>>) -> !hlfir.expr<?x!fir.type<whatever>?>
+  return %0 : !hlfir.expr<?x!fir.type<whatever>?>
+}
+// CHECK-LABEL:   func.func @reshape6(
+// CHECK-SAME:                        %[[VAL_0:.*]]: !fir.class<!fir.heap<!fir.array<?x!fir.type<whatever>>>>,
+// CHECK-SAME:                        %[[VAL_1:.*]]: !hlfir.expr<1xi32>) -> !hlfir.expr<?x!fir.type<whatever>?> {
+// CHECK:           %[[VAL_2:.*]] = hlfir.reshape %[[VAL_0]] %[[VAL_1]] pad %[[VAL_0]] : (!fir.class<!fir.heap<!fir.array<?x!fir.type<whatever>>>>, !hlfir.expr<1xi32>, !fir.class<!fir.heap<!fir.array<?x!fir.type<whatever>>>>) -> !hlfir.expr<?x!fir.type<whatever>?>
+// CHECK:           return %[[VAL_2]] : !hlfir.expr<?x!fir.type<whatever>?>
+// CHECK:         }
+
+// Allow character LEN mismatch for ARRAY/PAD and the result
+func.func @reshape7(%arg0: !hlfir.expr<1x!fir.char<2,2>>, %arg1: !hlfir.expr<1xi32>, %arg2: !hlfir.expr<1x!fir.char<2,1>>) -> !hlfir.expr<?x!fir.char<2,3>> {
+  %0 = hlfir.reshape %arg0 %arg1 pad %arg2 : (!hlfir.expr<1x!fir.char<2,2>>, !hlfir.expr<1xi32>, !hlfir.expr<1x!fir.char<2,1>>) -> !hlfir.expr<?x!fir.char<2,3>>
+  return %0 : !hlfir.expr<?x!fir.char<2,3>>
+}
+// CHECK-LABEL:   func.func @reshape7(
+// CHECK-SAME:                        %[[VAL_0:.*]]: !hlfir.expr<1x!fir.char<2,2>>,
+// CHECK-SAME:                        %[[VAL_1:.*]]: !hlfir.expr<1xi32>,
+// CHECK-SAME:                        %[[VAL_2:.*]]: !hlfir.expr<1x!fir.char<2>>) -> !hlfir.expr<?x!fir.char<2,3>> {
+// CHECK:           %[[VAL_3:.*]] = hlfir.reshape %[[VAL_0]] %[[VAL_1]] pad %[[VAL_2]] : (!hlfir.expr<1x!fir.char<2,2>>, !hlfir.expr<1xi32>, !hlfir.expr<1x!fir.char<2>>) -> !hlfir.expr<?x!fir.char<2,3>>
+// CHECK:           return %[[VAL_3]] : !hlfir.expr<?x!fir.char<2,3>>
+// CHECK:         }
diff --git a/flang/test/HLFIR/simplify-hlfir-intrinsics-reshape.fir b/flang/test/HLFIR/simplify-hlfir-intrinsics-reshape.fir
new file mode 100644
index 0000000000000..afbd3bcd6d98c
--- /dev/null
+++ b/flang/test/HLFIR/simplify-hlfir-intrinsics-reshape.fir
@@ -0,0 +1,228 @@
+// Test hlfir.reshape simplification to hlfir.elemental:
+// RUN: fir-opt --simplify-hlfir-intrinsics %s | FileCheck %s
+
+func.func @reshape_simple(%arg0: !fir.box<!fir.array<?xf32>>, %arg1: !fir.ref<!fir.array<1xi32>>) -> !hlfir.expr<?xf32> {
+  %res = hlfir.reshape %arg0 %arg1 : (!fir.box<!fir.array<?xf32>>, !fir.ref<!fir.array<1xi32>>) -> !hlfir.expr<?xf32>
+  return %res : !hlfir.expr<?xf32>
+}
+// CHECK-LABEL:   func.func @reshape_simple(
+// CHECK-SAME:                              %[[VAL_0:.*]]: !fir.box<!fir.array<?xf32>>,
+// CHECK-SAME:                              %[[VAL_1:.*]]: !fir.ref<!fir.array<1xi32>>) -> !hlfir.expr<?xf32> {
+// CHECK:           %[[VAL_2:.*]] = arith.constant 1 : index
+// CHECK:           %[[VAL_3:.*]] = arith.constant 0 : index
+// CHECK:           %[[VAL_4:.*]] = hlfir.designate %[[VAL_1]] (%[[VAL_2]])  : (!fir.ref<!fir.array<1xi32>>, index) -> !fir.ref<i32>
+// CHECK:           %[[VAL_5:.*]] = fir.load %[[VAL_4]] : !fir.ref<i32>
+// CHECK:           %[[VAL_6:.*]] = fir.shape %[[VAL_5]] : (i32) -> !fir.shape<1>
+// CHECK:           %[[VAL_7:.*]] = hlfir.elemental %[[VAL_6]] unordered : (!fir.shape<1>) -> !hlfir.expr<?xf32> {
+// CHECK:           ^bb0(%[[VAL_8:.*]]: index):
+// CHECK:             %[[VAL_9:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_3]] : (!fir.box<!fir.array<?xf32>>, index) -> (index, index, index)
+// CHECK:             %[[VAL_10:.*]] = arith.subi %[[VAL_9]]#0, %[[VAL_2]] overflow<nuw> : index
+// CHECK:             %[[VAL_11:.*]] = arith.addi %[[VAL_8]], %[[VAL_10]] overflow<nuw> : index
+// CHECK:             %[[VAL_12:.*]] = hlfir.designate %[[VAL_0]] (%[[VAL_11]])  : (!fir.box<!fir.array<?xf32>>, index) -> !fir.ref<f32>
+// CHECK:             %[[VAL_13:.*]] = fir.load %[[VAL_12]] : !fir.ref<f32>
+// CHECK:             hlfir.yield_element %[[VAL_13]] : f32
+// CHECK:           }
+// CHECK:           return %[[VAL_7]] : !hlfir.expr<?xf32>
+// CHECK:         }
+
+func.func @reshape_with_pad(%arg0: !fir.box<!fir.array<?x?x?xf32>>, %arg1: !fir.ref<!fir.array<2xi32>>, %arg2: !fir.box<!fir.array<?x?x?xf32>>) -> !hlfir.expr<?x?xf32> {
+  %res = hlfir.reshape %arg0 %arg1 pad %arg2 : (!fir.box<!fir.array<?x?x?xf32>>, !fir.ref<!fir.array<2xi32>>, !fir.box<!fir.array<?x?x?xf32>>) -> !hlfir.expr<?x?xf32>
+  return %res : !hlfir.expr<?x?xf32>
+}
+// CHECK-LABEL:   func.func @reshape_with_pad(
+// CHECK-SAME:                                %[[VAL_0:.*]]: !fir.box<!fir.array<?x?x?xf32>>,
+// CHECK-SAME:                                %[[VAL_1:.*]]: !fir.ref<!fir.array<2xi32>>,
+// CHECK-SAME:                                %[[VAL_2:.*]]: !fir.box<!fir.array<?x?x?xf32>>) -> !hlfir.expr<?x?xf32> {
+// CHECK:           %[[VAL_3:.*]] = arith.constant 2 : index
+// CHECK:           %[[VAL_4:.*]] = arith.constant 1 : index
+// CHECK:           %[[VAL_5:.*]] = arith.constant 0 : index
+// CHECK:           %[[ARRAY_DIM0:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_5]] : (!fir.box<!fir.array<?x?x?xf32>>, index) -> (index, index, index)
+// CHECK:           %[[ARRAY_DIM1:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_4]] : (!fir.box<!fir.array<?x?x?xf32>>, index) -> (index, index, index)
+// CHECK:           %[[ARRAY_DIM2:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_3]] : (!fir.box<!fir.array<?x?x?xf32>>, index) -> (index, index, index)
+// CHECK:           %[[VAL_9:.*]] = arith.muli %[[ARRAY_DIM0]]#1, %[[ARRAY_DIM1]]#1 overflow<nuw> : index
+// CHECK:           %[[ARRAY_SIZE:.*]] = arith.muli %[[VAL_9]], %[[ARRAY_DIM2]]#1 overflow<nuw> : index
+// CHECK:           %[[VAL_16:.*]] = hlfir.designate %[[VAL_1]] (%[[VAL_4]])  : (!fir.ref<!fir.array<2xi32>>, index) -> !fir.ref<i32>
+// CHECK:           %[[VAL_17:.*]] = fir.load %[[VAL_16]] : !fir.ref<i32>
+// CHECK:           %[[VAL_18:.*]] = hlfir.designate %[[VAL_1]] (%[[VAL_3]])  : (!fir.ref<!fir.array<2xi32>>, index) -> !fir.ref<i32>
+// CHECK:           %[[VAL_19:.*]] = fir.load %[[VAL_18]] : !fir.ref<i32>
+// CHECK:           %[[VAL_20:.*]] = fir.shape %[[VAL_17]], %[[VAL_19]] : (i32, i32) -> !fir.shape<2>
+// CHECK:           %[[VAL_21:.*]] = hlfir.elemental %[[VAL_20]] unordered : (!fir.shape<2>) -> !hlfir.expr<?x?xf32> {
+// CHECK:           ^bb0(%[[VAL_22:.*]]: index, %[[VAL_23:.*]]: index):
+// CHECK:             %[[VAL_24:.*]] = arith.subi %[[VAL_23]], %[[VAL_4]] overflow<nuw> : index
+// CHECK:             %[[VAL_25:.*]] = fir.convert %[[VAL_17]] : (i32) -> index
+// CHECK:             %[[VAL_26:.*]] = arith.muli %[[VAL_24]], %[[VAL_25]] overflow<nuw> : index
+// CHECK:             %[[VAL_27:.*]] = arith.subi %[[VAL_22]], %[[VAL_4]] overflow<nuw> : index
+// CHECK:             %[[LINEAR_INDEX:.*]] = arith.addi %[[VAL_26]], %[[VAL_27]] overflow<nuw> : index
+// CHECK:             %[[IS_WITHIN_ARRAY:.*]] = arith.cmpi ult, %[[LINEAR_INDEX]], %[[ARRAY_SIZE]] : index
+// CHECK:             %[[VAL_30:.*]] = fir.if %[[IS_WITHIN_ARRAY]] -> (f32) {
+// CHECK:               %[[VAL_31:.*]] = arith.remui %[[LINEAR_INDEX]], %[[ARRAY_DIM0]]#1 : index
+// CHECK:               %[[VAL_32:.*]] = arith.divui %[[LINEAR_INDEX]], %[[ARRAY_DIM0]]#1 : index
+// CHECK:               %[[ARRAY_IDX0:.*]] = arith.addi %[[VAL_31]], %[[VAL_4]] overflow<nuw> : index
+// CHECK:               %[[VAL_34:.*]] = arith.remui %[[VAL_32]], %[[ARRAY_DIM1]]#1 : index
+// CHECK:               %[[VAL_35:.*]] = arith.divui %[[VAL_32]], %[[ARRAY_DIM1]]#1 : index
+// CHECK:               %[[ARRAY_IDX1:.*]] = arith.addi %[[VAL_34]], %[[VAL_4]] overflow<nuw> : index
+// CHECK:               %[[ARRAY_IDX2:.*]] = arith.addi %[[VAL_35]], %[[VAL_4]] overflow<nuw> : index
+// CHECK:               %[[VAL_38:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_5]] : (!fir.box<!fir.array<?x?x?xf32>>, index) -> (index, index, index)
+// CHECK:               %[[VAL_39:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_4]] : (!fir.box<!fir.array<?x?x?xf32>>, index) -> (index, index, index)
+// CHECK:               %[[VAL_40:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_3]] : (!fir.box<!fir.array<?x?x?xf32>>, index) -> (index, index, index)
+// CHECK:               %[[VAL_41:.*]] = arith.subi %[[VAL_38]]#0, %[[VAL_4]] overflow<nuw> : index
+// CHECK:               %[[VAL_42:.*]] = arith.addi %[[ARRAY_IDX0]], %[[VAL_41]] overflow<nuw> : index
+// CHECK:               %[[VAL_43:.*]] = arith.subi %[[VAL_39]]#0, %[[VAL_4]] overflow<nuw> : index
+// CHECK:               %[[VAL_44:.*]] = arith.addi %[[ARRAY_IDX1]], %[[VAL_43]] overflow<nuw> : index
+// CHECK:               %[[VAL_45:.*]] = arith.subi %[[VAL_40]]#0, %[[VAL_4]] overflow<nuw> : index
+// CHECK:               %[[VAL_46:.*]] = arith.addi %[[ARRAY_IDX2]], %[[VAL_45]] overflow<nuw> : index
+// CHECK:               %[[VAL_47:.*]] = hlfir.designate %[[VAL_0]] (%[[VAL_42]], %[[VAL_44]], %[[VAL_46]])  : (!fir.box<!fir.array<?x?x?xf32>>, index, index, index) -> !fir.ref<f32>
+// CHECK:               %[[VAL_48:.*]] = fir.load %[[VAL_47]] : !fir.ref<f32>
+// CHECK:               fir.result %[[VAL_48]] : f32
+// CHECK:             } else {
+// CHECK:               %[[PAD_DIM0:.*]]:3 = fir.box_dims %[[VAL_2]], %[[VAL_5]] : (!fir.box<!fir.array<?x?x?xf32>>, index) -> (index, index, index)
+// CHECK:               %[[PAD_DIM1:.*]]:3 = fir.box_dims %[[VAL_2]], %[[VAL_4]] : (!fir.box<!fir.array<?x?x?xf32>>, index) -> (index, index, index)
+// CHECK:               %[[PAD_DIM2:.*]]:3 = fir.box_dims %[[VAL_2]], %[[VAL_3]] : (!fir.box<!fir.array<?x?x?xf32>>, index) -> (index, index, index)
+// CHECK:               %[[PAD_LINEAR_INDEX:.*]] = arith.subi %[[LINEAR_INDEX]], %[[ARRAY_SIZE]] overflow<nuw> : index
+// CHECK:               %[[VAL_51:.*]] = arith.remui %[[PAD_LINEAR_INDEX]], %[[PAD_DIM0]]#1 : index
+// CHECK:               %[[VAL_52:.*]] = arith.divui %[[PAD_LINEAR_INDEX]], %[[PAD_DIM0]]#1 : index
+// CHECK:               %[[PAD_IDX0:.*]] = arith.addi %[[VAL_51]], %[[VAL_4]] overflow<nuw> : index
+// CHECK:               %[[VAL_54:.*]] = arith.remui %[[VAL_52]], %[[PAD_DIM1]]#1 : index
+// CHECK:               %[[VAL_55:.*]] = arith.divui %[[VAL_52]], %[[PAD_DIM1]]#1 : index
+// CHECK:               %[[PAD_IDX1:.*]] = arith.addi %[[VAL_54]], %[[VAL_4]] overflow<nuw> : index
+// CHECK:               %[[VAL_56:.*]] = arith.remui %[[VAL_55]], %[[PAD_DIM2]]#1 : index
+// CHECK:               %[[PAD_IDX2:.*]] = arith.addi %[[VAL_56]], %[[VAL_4]] overflow<nuw> : index
+// CHECK:               %[[VAL_58:.*]]:3 = fir.box_dims %[[VAL_2]], %[[VAL_5]] : (!fir.box<!fir.array<?x?x?xf32>>, index) -> (index, index, index)
+// CHECK:               %[[VAL_59:.*]]:3 = fir.box_dims %[[VAL_2]], %[[VAL_4]] : (!fir.box<!fir.array<?x?x?xf32>>, index) -> (index, index, index)
+// CHECK:               %[[VAL_60:.*]]:3 = fir.box_dims %[[VAL_2]], %[[VAL_3]] : (!fir.box<!fir.array<?x?x?xf32>>, index) -> (index, index, index)
+// CHECK:               %[[VAL_61:.*]] = arith.subi %[[VAL_58]]#0, %[[VAL_4]] overflow<nuw> : index
+// CHECK:               %[[VAL_62:.*]] = arith.addi %[[PAD_IDX0]], %[[VAL_61]] overflow<nuw> : index
+// CHECK:               %[[VAL_63:.*]] = arith.subi %[[VAL_59]]#0, %[[VAL_4]] overflow<nuw> : index
+// CHECK:               %[[VAL_64:.*]] = arith.addi %[[PAD_IDX1]], %[[VAL_63]] overflow<nuw> : index
+// CHECK:               %[[VAL_65:.*]] = arith.subi %[[VAL_60]]#0, %[[VAL_4]] overflow<nuw> : index
+// CHECK:               %[[VAL_66:.*]] = arith.addi %[[PAD_IDX2]], %[[VAL_65]] overflow<nuw> : index
+// CHECK:               %[[VAL_67:.*]] = hlfir.designate %[[VAL_2]] (%[[VAL_62]], %[[VAL_64]], %[[VAL_66]])  : (!fir.box<!fir.array<?x?x?xf32>>, index, index, index) -> !fir.ref<f32>
+// CHECK:               %[[VAL_68:.*]] = fir.load %[[VAL_67]] : !fir.ref<f32>
+// CHECK:               fir.result %[[VAL_68]] : f32
+// CHECK:             }
+// CHECK:             hlfir.yield_element %[[VAL_30]] : f32
+// CHECK:           }
+// CHECK:           return %[[VAL_21]] : !hlfir.expr<?x?xf32>
+// CHECK:         }
+
+func.func @reshape_derived_obj(%arg0: !fir.ref<!fir.array<10x!fir.type<whatever>>>, %arg1: !fir.ref<!fir.array<1xi32>>) -> !hlfir.expr<?x!fir.type<whatever>> {
+  %res = hlfir.reshape %arg0 %arg1 : (!fir.ref<!fir.array<10x!fir.type<whatever>>>, !fir.ref<!fir.array<1xi32>>) -> !hlfir.expr<?x!fir.type<whatever>>
+  return %res : !hlfir.expr<?x!fir.type<whatever>>
+}
+// CHECK-LABEL:   func.func @reshape_derived_obj(
+// CHECK-SAME:                                   %[[VAL_0:.*]]: !fir.ref<!fir.array<10x!fir.type<whatever>>>,
+// CHECK-SAME:                                   %[[VAL_1:.*]]: !fir.ref<!fir.array<1xi32>>) -> !hlfir.expr<?x!fir.type<whatever>> {
+// CHECK:           %[[VAL_2:.*]] = arith.constant 1 : index
+// CHECK:           %[[VAL_3:.*]] = hlfir.designate %[[VAL_1]] (%[[VAL_2]])  : (!fir.ref<!fir.array<1xi32>>, index) -> !fir.ref<i32>
+// CHECK:           %[[VAL_4:.*]] = fir.load %[[VAL_3]] : !fir.ref<i32>
+// CHECK:           %[[VAL_5:.*]] = fir.shape %[[VAL_4]] : (i32) -> !fir.shape<1>
+// CHECK:           %[[VAL_6:.*]] = hlfir.elemental %[[VAL_5]] unordered : (!fir.shape<1>) -> !hlfir.expr<?x!fir.type<whatever>> {
+// CHECK:           ^bb0(%[[VAL_7:.*]]: index):
+// CHECK:             %[[VAL_8:.*]] = hlfir.designate %[[VAL_0]] (%[[VAL_7]])  : (!fir.ref<!fir.array<10x!fir.type<whatever>>>, index) -> !fir.ref<!fir.type<whatever>>
+// CHECK:             hlfir.yield_element %[[VAL_8]] : !fir.ref<!fir.type<whatever>>
+// CHECK:           }
+// CHECK:           return %[[VAL_6]] : !hlfir.expr<?x!fir.type<whatever>>
+// CHECK:         }
+
+func.func @reshape_derived_expr(%arg0: !hlfir.expr<?x!fir.type<whatever>>, %arg1: !fir.ref<!fir.array<1xi32>>) -> !hlfir.expr<?x!fir.type<whatever>> {
+  %res = hlfir.reshape %arg0 %arg1 : (!hlfir.expr<?x!fir.type<whatever>>, !fir.ref<!fir.array<1xi32>>) -> !hlfir.expr<?x!fir.type<whatever>>
+  return %res : !hlfir.expr<?x!fir.type<whatever>>
+}
+// CHECK-LABEL:   func.func @reshape_derived_expr(
+// CHECK-SAME:                                    %[[VAL_0:.*]]: !hlfir.expr<?x!fir.type<whatever>>,
+// CHECK-SAME:                                    %[[VAL_1:.*]]: !fir.ref<!fir.array<1xi32>>) -> !hlfir.expr<?x!fir.type<whatever>> {
+// CHECK:           %[[VAL_2:.*]] = arith.constant 1 : index
+// CHECK:           %[[VAL_3:.*]] = hlfir.designate %[[VAL_1]] (%[[VAL_2]])  : (!fir.ref<!fir.array<1xi32>>, index) -> !fir.ref<i32>
+// CHECK:           %[[VAL_4:.*]] = fir.load %[[VAL_3]] : !fir.ref<i32>
+// CHECK:           %[[VAL_5:.*]] = fir.shape %[[VAL_4]] : (i32) -> !fir.shape<1>
+// CHECK:           %[[VAL_6:.*]] = hlfir.elemental %[[VAL_5]] unordered : (!fir.shape<1>) -> !hlfir.expr<?x!fir.type<whatever>> {
+// CHECK:           ^bb0(%[[VAL_7:.*]]: index):
+// CHECK:             %[[VAL_8:.*]] = hlfir.apply %[[VAL_0]], %[[VAL_7]] : (!hlfir.expr<?x!fir.type<whatever>>, index) -> !hlfir.expr<!fir.type<whatever>>
+// CHECK:             hlfir.yield_element %[[VAL_8]] : !hlfir.expr<!fir.type<whatever>>
+// CHECK:           }
+// CHECK:           return %[[VAL_6]] : !hlfir.expr<?x!fir.type<whatever>>
+// CHECK:         }
+
+func.func @reshape_poly_obj(%arg0: !fir.class<!fir.array<?x!fir.type<whatever>>>, %arg1: !fir.ref<!fir.array<1xi32>>) -> !hlfir.expr<?x!fir.type<whatever>?> {
+  %res = hlfir.reshape %arg0 %arg1 : (!fir.class<!fir.array<?x!fir.type<whatever>>>, !fir.ref<!fir.array<1xi32>>) -> !hlfir.expr<?x!fir.type<whatever>?>
+  return %res : !hlfir.expr<?x!fir.type<whatever>?>
+}
+// CHECK-LABEL:   func.func @reshape_poly_obj(
+// CHECK-SAME:                                %[[VAL_0:.*]]: !fir.class<!fir.array<?x!fir.type<whatever>>>,
+// CHECK-SAME:                                %[[VAL_1:.*]]: !fir.ref<!fir.array<1xi32>>) -> !hlfir.expr<?x!fir.type<whatever>?> {
+// CHECK:           %[[VAL_2:.*]] = arith.constant 1 : index
+// CHECK:           %[[VAL_3:.*]] = arith.constant 0 : index
+// CHECK:           %[[VAL_4:.*]] = hlfir.designate %[[VAL_1]] (%[[VAL_2]])  : (!fir.ref<!fir.array<1xi32>>, index) -> !fir.ref<i32>
+// CHECK:           %[[VAL_5:.*]] = fir.load %[[VAL_4]] : !fir.ref<i32>
+// CHECK:           %[[VAL_6:.*]] = fir.shape %[[VAL_5]] : (i32) -> !fir.shape<1>
+// CHECK:           %[[VAL_7:.*]] = hlfir.elemental %[[VAL_6]] mold %[[VAL_0]] unordered : (!fir.shape<1>, !fir.class<!fir.array<?x!fir.type<whatever>>>) -> !hlfir.expr<?x!fir.type<whatever>?> {
+// CHECK:           ^bb0(%[[VAL_8:.*]]: index):
+// CHECK:             %[[VAL_9:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_3]] : (!fir.class<!fir.array<?x!fir.type<whatever>>>, index) -> (index, index, index)
+// CHECK:             %[[VAL_10:.*]] = arith.subi %[[VAL_9]]#0, %[[VAL_2]] overflow<nuw> : index
+// CHECK:             %[[VAL_11:.*]] = arith.addi %[[VAL_8]], %[[VAL_10]] overflow<nuw> : index
+// CHECK:             %[[VAL_12:.*]] = hlfir.designate %[[VAL_0]] (%[[VAL_11]])  : (!fir.class<!fir.array<?x!fir.type<whatever>>>, index) -> !fir.class<!fir.type<whatever>>
+// CHECK:             hlfir.yield_element %[[VAL_12]] : !fir.class<!fir.type<whatever>>
+// CHECK:           }
+// CHECK:           return %[[VAL_7]] : !hlfir.expr<?x!fir.type<whatever>?>
+// CHECK:         }
+
+func.func @reshape_poly_expr(%arg0: !hlfir.expr<?x!fir.type<whatever>?>, %arg1: !fir.ref<!fir.array<1xi32>>) -> !hlfir.expr<?x!fir.type<whatever>?> {
+  %res = hlfir.reshape %arg0 %arg1 : (!hlfir.expr<?x!fir.type<whatever>?>, !fir.ref<!fir.array<1xi32>>) -> !hlfir.expr<?x!fir.type<whatever>?>
+  return %res : !hlfir.expr<?x!fir.type<whatever>?>
+}
+// CHECK-LABEL:   func.func @reshape_poly_expr(
+// CHECK-SAME:                                 %[[VAL_0:.*]]: !hlfir.expr<?x!fir.type<whatever>?>,
+// CHECK-SAME:                                 %[[VAL_1:.*]]: !fir.ref<!fir.array<1xi32>>) -> !hlfir.expr<?x!fir.type<whatever>?> {
+// CHECK:           %[[VAL_2:.*]] = arith.constant 1 : index
+// CHECK:           %[[VAL_3:.*]] = hlfir.designate %[[VAL_1]] (%[[VAL_2]])  : (!fir.ref<!fir.array<1xi32>>, index) -> !fir.ref<i32>
+// CHECK:           %[[VAL_4:.*]] = fir.load %[[VAL_3]] : !fir.ref<i32>
+// CHECK:           %[[VAL_5:.*]] = fir.shape %[[VAL_4]] : (i32) -> !fir.shape<1>
+// CHECK:           %[[VAL_6:.*]] = hlfir.elemental %[[VAL_5]] mold %[[VAL_0]] unordered : (!fir.shape<1>, !hlfir.expr<?x!fir.type<whatever>?>) -> !hlfir.expr<?x!fir.type<whatever>?> {
+// CHECK:           ^bb0(%[[VAL_7:.*]]: index):
+// CHECK:             %[[VAL_8:.*]] = hlfir.apply %[[VAL_0]], %[[VAL_7]] : (!hlfir.expr<?x!fir.type<whatever>?>, index) -> !hlfir.expr<!fir.type<whatever>?>
+// CHECK:             hlfir.yield_element %[[VAL_8]] : !hlfir.expr<!fir.type<whatever>?>
+// CHECK:           }
+// CHECK:           return %[[VAL_6]] : !hlfir.expr<?x!fir.type<whatever>?>
+// CHECK:         }
+
+func.func @reshape_char(%arg0: !fir.box<!fir.array<?x!fir.char<2,?>>>, %arg1: !fir.ref<!fir.array<1xi32>>) -> !hlfir.expr<?x!fir.char<2,?>> {
+  %res = hlfir.reshape %arg0 %arg1 : (!fir.box<!fir.array<?x!fir.char<2,?>>>, !fir.ref<!fir.array<1xi32>>) -> !hlfir.expr<?x!fir.char<2,?>>
+  return %res : !hlfir.expr<?x!fir.char<2,?>>
+}
+// CHECK-LABEL:   func.func @reshape_char(
+// CHECK-SAME:                            %[[VAL_0:.*]]: !fir.box<!fir.array<?x!fir.char<2,?>>>,
+// CHECK-SAME:                            %[[VAL_1:.*]]: !fir.ref<!fir.array<1xi32>>) -> !hlfir.expr<?x!fir.char<2,?>> {
+// CHECK:           %[[VAL_2:.*]] = arith.constant 1 : index
+// CHECK:           %[[VAL_3:.*]] = arith.constant 0 : index
+// CHECK:           %[[VAL_4:.*]] = arith.constant 2 : index
+// CHECK:           %[[VAL_5:.*]] = fir.box_elesize %[[VAL_0]] : (!fir.box<!fir.array<?x!fir.char<2,?>>>) -> index
+// CHECK:           %[[VAL_6:.*]] = arith.divsi %[[VAL_5]], %[[VAL_4]] : index
+// CHECK:           %[[VAL_7:.*]] = hlfir.designate %[[VAL_1]] (%[[VAL_2]])  : (!fir.ref<!fir.array<1xi32>>, index) -> !fir.ref<i32>
+// CHECK:           %[[VAL_8:.*]] = fir.load %[[VAL_7]] : !fir.ref<i32>
+// CHECK:           %[[VAL_9:.*]] = fir.shape %[[VAL_8]] : (i32) -> !fir.shape<1>
+// CHECK:           %[[VAL_10:.*]] = hlfir.elemental %[[VAL_9]] typeparams %[[VAL_6]] unordered : (!fir.shape<1>, index) -> !hlfir.expr<?x!fir.char<2,?>> {
+// CHECK:           ^bb0(%[[VAL_11:.*]]: index):
+// CHECK:             %[[VAL_12:.*]] = fir.box_elesize %[[VAL_0]] : (!fir.box<!fir.array<?x!fir.char<2,?>>>) -> index
+// CHECK:             %[[VAL_13:.*]] = arith.divsi %[[VAL_12]], %[[VAL_4]] : index
+// CHECK:             %[[VAL_14:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_3]] : (!fir.box<!fir.array<?x!fir.char<2,?>>>, index) -> (index, index, index)
+// CHECK:             %[[VAL_15:.*]] = arith.subi %[[VAL_14]]#0, %[[VAL_2]] overflow<nuw> : index
+// CHECK:             %[[VAL_16:.*]] = arith.addi %[[VAL_11]], %[[VAL_15]] overflow<nuw> : index
+// CHECK:             %[[VAL_17:.*]] = hlfir.designate %[[VAL_0]] (%[[VAL_16]])  typeparams %[[VAL_13]] : (!fir.box<!fir.array<?x!fir.char<2,?>>>, index, index) -> !fir.boxchar<2>
+// CHECK:             hlfir.yield_element %[[VAL_17]] : !fir.boxchar<2>
+// CHECK:           }
+// CHECK:           return %[[VAL_10]] : !hlfir.expr<?x!fir.char<2,?>>
+// CHECK:         }
+
+func.func @reshape_negative_result_array_have_different_types(%arg0: !fir.box<!fir.array<?x!fir.char<2,1>>>, %arg1: !fir.ref<!fir.array<1xi32>>) -> !hlfir.expr<?x!fir.char<2,2>> {
+  %res = hlfir.reshape %arg0 %arg1 : (!fir.box<!fir.array<?x!fir.char<2,1>>>, !fir.ref<!fir.array<1xi32>>) -> !hlfir.expr<?x!fir.char<2,2>>
+  return %res : !hlfir.expr<?x!fir.char<2,2>>
+}
+// CHECK-LABEL:   func.func @reshape_negative_result_array_have_different_types(
+// CHECK:           hlfir.reshape %{{.*}} %{{.*}} : (!fir.box<!fir.array<?x!fir.char<2>>>, !fir.ref<!fir.array<1xi32>>) -> !hlfir.expr<?x!fir.char<2,2>>
+
+func.func @reshape_negative_array_pad_have_different_types(%arg0: !fir.box<!fir.array<?x!fir.char<2,2>>>, %arg1: !fir.ref<!fir.array<1xi32>>, %arg2: !fir.box<!fir.array<?x!fir.char<2,1>>>) -> !hlfir.expr<?x!fir.char<2,2>> {
+  %res = hlfir.reshape %arg0 %arg1 pad %arg2 : (!fir.box<!fir.array<?x!fir.char<2,2>>>, !fir.ref<!fir.array<1xi32>>, !fir.box<!fir.array<?x!fir.char<2,1>>>) -> !hlfir.expr<?x!fir.char<2,2>>
+  return %res : !hlfir.expr<?x!fir.char<2,2>>
+}
+// CHECK-LABEL:   func.func @reshape_negative_array_pad_have_different_types(
+// CHECK:           hlfir.reshape %{{.*}} %{{.*}} pad %{{.*}} : (!fir.box<!fir.array<?x!fir.char<2,2>>>, !fir.ref<!fir.array<1xi32>>, !fir.box<!fir.array<?x!fir.char<2>>>) -> !hlfir.expr<?x!fir.char<2,2>>
diff --git a/flang/test/HLFIR/unroll-loops.fir b/flang/test/HLFIR/unroll-loops.fir
deleted file mode 100644
index 4494cfa570dd7..0000000000000
--- a/flang/test/HLFIR/unroll-loops.fir
+++ /dev/null
@@ -1,42 +0,0 @@
-// RUN: %flang_fc1 -emit-llvm -O1 -funroll-loops -mllvm -force-vector-width=2 -o- %s | FileCheck %s --check-prefixes=CHECK,UNROLL
-// RUN: %flang_fc1 -emit-llvm -O2 -mllvm -force-vector-width=2 -o- %s | FileCheck %s --check-prefixes=CHECK,UNROLL
-// RUN: %flang_fc1 -emit-llvm -O1 -fno-unroll-loops -mllvm -force-vector-width=2 -o- %s | FileCheck %s --check-prefixes=CHECK,NO-UNROLL
-// RUN: %flang_fc1 -emit-llvm -O1 -mllvm -force-vector-width=2 -o- %s | FileCheck %s --check-prefixes=CHECK,NO-UNROLL
-
-// FIXME: https://github.com/llvm/llvm-project/issues/123668
-// XFAIL: target=powerpc64{{.*}}
-
-// CHECK-LABEL: @unroll
-// CHECK-SAME: (ptr nocapture writeonly %[[ARG0:.*]])
-func.func @unroll(%arg0: !fir.ref<!fir.array<1000 x index>> {fir.bindc_name = "a"}) {
-  %scope = fir.dummy_scope : !fir.dscope
-  %c1000 = arith.constant 1000 : index
-  %shape = fir.shape %c1000 : (index) -> !fir.shape<1>
-  %a:2 = hlfir.declare %arg0(%shape) dummy_scope %scope {uniq_name = "unrollEa"} : (!fir.ref<!fir.array<1000xindex>>, !fir.shape<1>, !fir.dscope) -> (!fir.ref<!fir.array<1000 x index>>, !fir.ref<!fir.array<1000 x index>>)
-  %c1 = arith.constant 1 : index
-  fir.do_loop %arg1 = %c1 to %c1000 step %c1 {
-    // CHECK: br label %[[BLK:.*]]
-    // CHECK: [[BLK]]:
-    // CHECK-NEXT: %[[IND:.*]] = phi i64 [ 0, %{{.*}} ], [ %[[NIV:.*]], %[[BLK]] ]
-    // CHECK-NEXT: %[[VIND:.*]] = phi <2 x i64> [ <i64 1, i64 2>, %{{.*}} ], [ %[[NVIND:.*]], %[[BLK]] ]
-
-    // NO-UNROLL-NEXT: %[[GEP:.*]] = getelementptr i64, ptr %[[ARG0]], i64 %[[IND]]
-    // NO-UNROLL-NEXT: store <2 x i64> %[[VIND]], ptr %[[GEP]]
-    // NO-UNROLL-NEXT: %[[NIV:.*]] = add nuw i64 %{{.*}}, 2
-    // NO-UNROLL-NEXT: %[[NVIND]] = add <2 x i64> %[[VIND]], splat (i64 2)
-
-    // UNROLL-NEXT: %[[VIND1:.*]] = add <2 x i64> %[[VIND]], splat (i64 2)
-    // UNROLL-NEXT: %[[GEP0:.*]] = getelementptr i64, ptr %[[ARG0]], i64 %[[IND]]
-    // UNROLL-NEXT: %[[GEP1:.*]] = getelementptr i8, ptr %[[GEP0]], i64 16
-    // UNROLL-NEXT: store <2 x i64> %[[VIND]], ptr %[[GEP0]]
-    // UNROLL-NEXT: store <2 x i64> %[[VIND1]], ptr %[[GEP1]]
-    // UNROLL-NEXT: %[[NIV:.*]] = add nuw i64 %[[IND]], 4
-    // UNROLL-NEXT: %[[NVIND:.*]] = add <2 x i64> %[[VIND]], splat (i64 4)
-
-    // CHECK-NEXT: %[[EXIT:.*]] = icmp eq i64 %[[NIV]], 1000
-    // CHECK-NEXT: br i1 %[[EXIT]], label %{{.*}}, label %[[BLK]]
-    %ai = hlfir.designate %a#0 (%arg1)  : (!fir.ref<!fir.array<1000 x index>>, index) -> !fir.ref<index>
-    hlfir.assign %arg1 to %ai : index, !fir.ref<index>
-  }
-  return
-}
diff --git a/flang/test/Integration/OpenMP/copyprivate.f90 b/flang/test/Integration/OpenMP/copyprivate.f90
index d38fd20020f34..3bae003ea8d83 100644
--- a/flang/test/Integration/OpenMP/copyprivate.f90
+++ b/flang/test/Integration/OpenMP/copyprivate.f90
@@ -8,33 +8,37 @@
 
 !RUN: %flang_fc1 -emit-llvm -fopenmp %s -o - | FileCheck %s
 
-!CHECK-DAG: define internal void @_copy_box_Uxi32(ptr nocapture %{{.*}}, ptr nocapture %{{.*}})
-!CHECK-DAG: define internal void @_copy_10xi32(ptr nocapture %{{.*}}, ptr nocapture %{{.*}})
-!CHECK-DAG: define internal void @_copy_i64(ptr nocapture %{{.*}}, ptr nocapture %{{.*}})
-!CHECK-DAG: define internal void @_copy_box_Uxi64(ptr nocapture %{{.*}}, ptr nocapture %{{.*}})
-!CHECK-DAG: define internal void @_copy_f32(ptr nocapture %{{.*}}, ptr nocapture %{{.*}})
-!CHECK-DAG: define internal void @_copy_2x3xf32(ptr nocapture %{{.*}}, ptr nocapture %{{.*}})
-!CHECK-DAG: define internal void @_copy_z32(ptr nocapture %{{.*}}, ptr nocapture %{{.*}})
-!CHECK-DAG: define internal void @_copy_10xz32(ptr nocapture %{{.*}}, ptr nocapture %{{.*}})
-!CHECK-DAG: define internal void @_copy_l32(ptr nocapture %{{.*}}, ptr nocapture %{{.*}})
-!CHECK-DAG: define internal void @_copy_5xl32(ptr nocapture %{{.*}}, ptr nocapture %{{.*}})
-!CHECK-DAG: define internal void @_copy_c8x8(ptr nocapture %{{.*}}, ptr nocapture %{{.*}})
-!CHECK-DAG: define internal void @_copy_10xc8x8(ptr nocapture %{{.*}}, ptr nocapture %{{.*}})
-!CHECK-DAG: define internal void @_copy_c16x5(ptr nocapture %{{.*}}, ptr nocapture %{{.*}})
-!CHECK-DAG: define internal void @_copy_rec__QFtest_typesTdt(ptr nocapture %{{.*}}, ptr nocapture %{{.*}})
-!CHECK-DAG: define internal void @_copy_box_heap_Uxi32(ptr nocapture %{{.*}}, ptr nocapture %{{.*}})
-!CHECK-DAG: define internal void @_copy_box_ptr_Uxc8x9(ptr nocapture %{{.*}}, ptr nocapture %{{.*}})
+!CHECK-DAG: define internal void @_copy_box_Uxi32(ptr captures(none) %{{.*}}, ptr captures(none) %{{.*}})
+!CHECK-DAG: define internal void @_copy_box_10xi32(ptr captures(none) %{{.*}}, ptr captures(none) %{{.*}})
+!CHECK-DAG: define internal void @_copy_i64(ptr captures(none) %{{.*}}, ptr captures(none) %{{.*}})
+!CHECK-DAG: define internal void @_copy_box_Uxi64(ptr captures(none) %{{.*}}, ptr captures(none) %{{.*}})
+!CHECK-DAG: define internal void @_copy_f32(ptr captures(none) %{{.*}}, ptr captures(none) %{{.*}})
+!CHECK-DAG: define internal void @_copy_box_2x3xf32(ptr captures(none) %{{.*}}, ptr captures(none) %{{.*}})
+!CHECK-DAG: define internal void @_copy_z32(ptr captures(none) %{{.*}}, ptr captures(none) %{{.*}})
+!CHECK-DAG: define internal void @_copy_box_10xz32(ptr captures(none) %{{.*}}, ptr captures(none) %{{.*}})
+!CHECK-DAG: define internal void @_copy_l32(ptr captures(none) %{{.*}}, ptr captures(none) %{{.*}})
+!CHECK-DAG: define internal void @_copy_box_5xl32(ptr captures(none) %{{.*}}, ptr captures(none) %{{.*}})
+!CHECK-DAG: define internal void @_copy_c8x8(ptr captures(none) %{{.*}}, ptr captures(none) %{{.*}})
+!CHECK-DAG: define internal void @_copy_box_10xc8x8(ptr captures(none) %{{.*}}, ptr captures(none) %{{.*}})
+!CHECK-DAG: define internal void @_copy_c16x5(ptr captures(none) %{{.*}}, ptr captures(none) %{{.*}})
+!CHECK-DAG: define internal void @_copy_rec__QFtest_typesTdt(ptr captures(none) %{{.*}}, ptr captures(none) %{{.*}})
+!CHECK-DAG: define internal void @_copy_box_heap_Uxi32(ptr captures(none) %{{.*}}, ptr captures(none) %{{.*}})
+!CHECK-DAG: define internal void @_copy_box_ptr_Uxc8x9(ptr captures(none) %{{.*}}, ptr captures(none) %{{.*}})
 
 !CHECK-LABEL: define internal void @_copy_i32(
-!CHECK-SAME:                         ptr nocapture %[[DST:.*]], ptr nocapture %[[SRC:.*]]){{.*}} {
+!CHECK-SAME:                         ptr captures(none) %[[DST:.*]], ptr captures(none) %[[SRC:.*]]){{.*}} {
 !CHECK-NEXT:    %[[SRC_VAL:.*]] = load i32, ptr %[[SRC]]
 !CHECK-NEXT:    store i32 %[[SRC_VAL]], ptr %[[DST]]
 !CHECK-NEXT:    ret void
 !CHECK-NEXT:  }
 
 !CHECK-LABEL: define internal void @test_scalar_..omp_par({{.*}})
-!CHECK:         %[[I:.*]] = alloca i32, i64 1
-!CHECK:         %[[J:.*]] = alloca i32, i64 1
+!CHECK-NEXT: omp.par.entry:
+!CHECK:         %[[TID_ADDR:.*]] = alloca i32, align 4
+!CHECK:         %[[I:.*]] = alloca i32, align 4
+!CHECK:         %[[J:.*]] = alloca i32, align 4
+!CHECK:         br label %[[OMP_REDUCTION_INIT:.*]]
+
 !CHECK:         %[[DID_IT:.*]] = alloca i32
 !CHECK:         store i32 0, ptr %[[DID_IT]]
 !CHECK:         %[[THREAD_NUM1:.*]] = call i32 @__kmpc_global_thread_num(ptr @[[LOC:.*]])
diff --git a/flang/test/Integration/OpenMP/map-types-and-sizes.f90 b/flang/test/Integration/OpenMP/map-types-and-sizes.f90
index dac4938157a60..4ff338e79aecc 100644
--- a/flang/test/Integration/OpenMP/map-types-and-sizes.f90
+++ b/flang/test/Integration/OpenMP/map-types-and-sizes.f90
@@ -30,7 +30,7 @@ subroutine mapType_array
   !$omp end target
 end subroutine mapType_array
 
-!CHECK: @.offload_sizes{{.*}} = private unnamed_addr constant [4 x i64] [i64 0, i64 24, i64 8, i64 4]
+!CHECK: @.offload_sizes{{.*}} = private unnamed_addr constant [4 x i64] [i64 0, i64 24, i64 8, i64 0]
 !CHECK: @.offload_maptypes{{.*}} = private unnamed_addr constant [4 x i64] [i64 32, i64 281474976711169, i64 281474976711171, i64 281474976711187]
 subroutine mapType_ptr
   integer, pointer :: a
@@ -39,7 +39,7 @@ subroutine mapType_ptr
   !$omp end target
 end subroutine mapType_ptr
 
-!CHECK: @.offload_sizes{{.*}} = private unnamed_addr constant [4 x i64] [i64 0, i64 24, i64 8, i64 4]
+!CHECK: @.offload_sizes{{.*}} = private unnamed_addr constant [4 x i64] [i64 0, i64 24, i64 8, i64 0]
 !CHECK: @.offload_maptypes{{.*}} = private unnamed_addr constant [4 x i64] [i64 32, i64 281474976711169, i64 281474976711171, i64 281474976711187]
 subroutine mapType_allocatable
   integer, allocatable :: a
@@ -50,7 +50,7 @@ subroutine mapType_allocatable
   deallocate(a)
 end subroutine mapType_allocatable
 
-!CHECK: @.offload_sizes{{.*}} = private unnamed_addr constant [4 x i64] [i64 0, i64 24, i64 8, i64 4]
+!CHECK: @.offload_sizes{{.*}} = private unnamed_addr constant [4 x i64] [i64 0, i64 24, i64 8, i64 0]
 !CHECK: @.offload_maptypes{{.*}} = private unnamed_addr constant [4 x i64] [i64 32, i64 281474976710657, i64 281474976710659, i64 281474976710675]
 subroutine mapType_ptr_explicit
   integer, pointer :: a
@@ -59,7 +59,7 @@ subroutine mapType_ptr_explicit
   !$omp end target
 end subroutine mapType_ptr_explicit
 
-!CHECK: @.offload_sizes{{.*}} = private unnamed_addr constant [4 x i64] [i64 0, i64 24, i64 8, i64 4]
+!CHECK: @.offload_sizes{{.*}} = private unnamed_addr constant [4 x i64] [i64 0, i64 24, i64 8, i64 0]
 !CHECK: @.offload_maptypes{{.*}} = private unnamed_addr constant [4 x i64] [i64 32, i64 281474976710657, i64 281474976710659, i64 281474976710675]
 subroutine mapType_allocatable_explicit
   integer, allocatable :: a
@@ -232,7 +232,7 @@ subroutine mapType_derived_type_alloca()
   !$omp end target
 end subroutine
 
-!CHECK: @.offload_sizes{{.*}} = private unnamed_addr constant [8 x i64] [i64 0, i64 40, i64 8, i64 136, i64 48, i64 8, i64 0, i64 4]
+!CHECK: @.offload_sizes{{.*}} = private unnamed_addr constant [8 x i64] [i64 0, i64 40, i64 8, i64 0, i64 48, i64 8, i64 0, i64 4]
 !CHECK: @.offload_maptypes{{.*}} = private unnamed_addr constant [8 x i64] [i64 32, i64 281474976710657, i64 281474976710659, i64 281474976710675, i64 281474976710657, i64 281474976710659, i64 281474976710675, i64 281474976710659]
 subroutine mapType_alloca_derived_type()
   type :: one_layer
@@ -255,7 +255,7 @@ subroutine mapType_alloca_derived_type()
   !$omp end target
 end subroutine
 
-!CHECK: @.offload_sizes{{.*}} = private unnamed_addr constant [8 x i64] [i64 0, i64 40, i64 8, i64 240, i64 48, i64 8, i64 0, i64 4]
+!CHECK: @.offload_sizes{{.*}} = private unnamed_addr constant [8 x i64] [i64 0, i64 40, i64 8, i64 0, i64 48, i64 8, i64 0, i64 4]
 !CHECK: @.offload_maptypes{{.*}} = private unnamed_addr constant [8 x i64] [i64 32, i64 281474976710657, i64 281474976710659, i64 281474976710675, i64 281474976710657, i64 281474976710659, i64 281474976710675, i64 281474976710659]
 subroutine mapType_alloca_nested_derived_type()
   type :: middle_layer
@@ -513,11 +513,15 @@ end subroutine mapType_common_block_members
 !CHECK: %[[RESTORE_OFFSET:.*]] = add i64 %[[CALCULATE_DIM_SIZE]], 1
 !CHECK: %[[MEMBER_BASE_ADDR_SIZE:.*]] = mul i64 1, %[[RESTORE_OFFSET]]
 !CHECK: %[[DESC_BASE_ADDR_DATA_SIZE:.*]] = mul i64 %[[MEMBER_BASE_ADDR_SIZE]], 4
+!CHECK: %[[LOAD_ADDR_DATA:.*]] = load ptr, ptr %[[MEMBER_DESCRIPTOR_BASE_ADDR]], align 8
+!CHECK: %[[GEP_ADDR_DATA:.*]] = getelementptr inbounds i32, ptr %[[LOAD_ADDR_DATA]], i64 0
 !CHECK: %[[MEMBER_ACCESS_ADDR_END:.*]] = getelementptr { ptr, i64, i32, i8, i8, i8, i8, [1 x [3 x i64]] }, ptr %[[MEMBER_ACCESS]], i64 1
 !CHECK: %[[MEMBER_ACCESS_ADDR_INT:.*]] = ptrtoint ptr %[[MEMBER_ACCESS_ADDR_END]] to i64
 !CHECK: %[[MEMBER_ACCESS_ADDR_BEGIN:.*]] = ptrtoint ptr %[[MEMBER_ACCESS]] to i64
 !CHECK: %[[DTYPE_SEGMENT_SIZE:.*]] = sub i64 %[[MEMBER_ACCESS_ADDR_INT]], %[[MEMBER_ACCESS_ADDR_BEGIN]]
 !CHECK: %[[DTYPE_SIZE_CALC:.*]] = sdiv exact i64 %[[DTYPE_SEGMENT_SIZE]], ptrtoint (ptr getelementptr (i8, ptr null, i32 1) to i64)
+!CHECK: %[[DTYPE_CMP:.*]] = icmp eq ptr %[[GEP_ADDR_DATA]], null
+!CHECK: %[[DTYPE_SEL:.*]] = select i1 %[[DTYPE_CMP]], i64 0, i64 %[[DESC_BASE_ADDR_DATA_SIZE]]
 !CHECK: %[[BASE_PTR_ARR:.*]] = getelementptr inbounds [4 x ptr], ptr %.offload_baseptrs, i32 0, i32 0
 !CHECK: store ptr %[[ALLOCA]], ptr %[[BASE_PTR_ARR]], align 8
 !CHECK: %[[OFFLOAD_PTR_ARR:.*]] = getelementptr inbounds [4 x ptr], ptr %.offload_ptrs, i32 0, i32 0
@@ -537,7 +541,7 @@ end subroutine mapType_common_block_members
 !CHECK: %[[OFFLOAD_PTR_ARR:.*]] = getelementptr inbounds [4 x ptr], ptr %.offload_ptrs, i32 0, i32 3
 !CHECK: store ptr %array_offset, ptr %[[OFFLOAD_PTR_ARR]], align 8
 !CHECK: %[[OFFLOAD_SIZE_ARR:.*]] = getelementptr inbounds [4 x i64], ptr %.offload_sizes, i32 0, i32 3
-!CHECK: store i64 %[[DESC_BASE_ADDR_DATA_SIZE]], ptr %[[OFFLOAD_SIZE_ARR]], align 8
+!CHECK: store i64 %[[DTYPE_SEL]], ptr %[[OFFLOAD_SIZE_ARR]], align 8
 
 !CHECK-LABEL: define {{.*}} @{{.*}}maptype_alloca_derived_type_{{.*}}
 !CHECK: %{{.*}} = alloca { ptr, i64, i32, i8, i8, i8, i8, ptr, [1 x i64] }, align 8
@@ -568,6 +572,8 @@ end subroutine mapType_common_block_members
 !CHECK: %[[DTYPE_BEGIN:.*]] = ptrtoint ptr %[[DTYPE_DESC_ALLOCA_3]] to i64
 !CHECK: %[[DTYPE_DESC_SZ_CALC:.*]] = sub i64 %[[DTYPE_END]], %[[DTYPE_BEGIN]]
 !CHECK: %[[DTYPE_DESC_SZ:.*]] = sdiv exact i64 %[[DTYPE_DESC_SZ_CALC]], ptrtoint (ptr getelementptr (i8, ptr null, i32 1) to i64)
+!CHECK: %[[SIZE_CMP:.*]] = icmp eq ptr %[[MEMBER_ARRAY_OFFSET]], null
+!CHECK: %[[SIZE_SEL:.*]] = select i1 %[[SIZE_CMP]], i64 0, i64 %[[MEMBER_SIZE_CALC_4]]
 !CHECK: %[[BASE_PTR_ARR:.*]] = getelementptr inbounds [8 x ptr], ptr %.offload_baseptrs, i32 0, i32 0
 !CHECK: store ptr %[[DTYPE_DESC_ALLOCA_3]], ptr %[[BASE_PTR_ARR]], align 8
 !CHECK: %[[OFFLOAD_PTR_ARR:.*]] = getelementptr inbounds [8 x ptr], ptr %.offload_ptrs, i32 0, i32 0
@@ -599,7 +605,7 @@ end subroutine mapType_common_block_members
 !CHECK: %[[OFFLOAD_PTR_ARR:.*]] = getelementptr inbounds [8 x ptr], ptr %.offload_ptrs, i32 0, i32 6
 !CHECK: store ptr %[[MEMBER_ARRAY_OFFSET]], ptr %[[OFFLOAD_PTR_ARR]], align 8
 !CHECK: %[[OFFLOAD_SIZE_ARR:.*]] = getelementptr inbounds [8 x i64], ptr %.offload_sizes, i32 0, i32 6
-!CHECK: store i64 %[[MEMBER_SIZE_CALC_4]], ptr %[[OFFLOAD_SIZE_ARR]], align 8
+!CHECK: store i64 %[[SIZE_SEL]], ptr %[[OFFLOAD_SIZE_ARR]], align 8
 !CHECK: %[[BASE_PTR_ARR:.*]] = getelementptr inbounds [8 x ptr], ptr %.offload_baseptrs, i32 0, i32 7
 !CHECK: store ptr %[[DTYPE_DESC_ALLOCA_3]], ptr %[[BASE_PTR_ARR]], align 8
 !CHECK: %[[OFFLOAD_PTR_ARR:.*]] = getelementptr inbounds [8 x ptr], ptr %.offload_ptrs, i32 0, i32 7
@@ -636,6 +642,8 @@ end subroutine mapType_common_block_members
 !CHECK: %[[DTYPE_DESC_SIZE_CALC_3:.*]] = ptrtoint ptr %[[DTYPE_DESC_ALLOCA_3]] to i64
 !CHECK: %[[DTYPE_DESC_SIZE_CALC_4:.*]] = sub i64 %[[DTYPE_DESC_SIZE_CALC_2]], %[[DTYPE_DESC_SIZE_CALC_3]]
 !CHECK: %[[DTYPE_DESC_SIZE_CALC_5:.*]] = sdiv exact i64 %[[DTYPE_DESC_SIZE_CALC_4]], ptrtoint (ptr getelementptr (i8, ptr null, i32 1) to i64)
+!CHECK: %[[DATA_CMP:.*]] = icmp eq ptr %[[ARRAY_OFFSET]], null
+!CHECK: %[[DATA_SEL:.*]] = select i1 %[[DATA_CMP]], i64 0, i64 %[[ALLOCATABLE_MEMBER_SIZE_CALC_5]]
 !CHECK: %[[BASE_PTR_ARR:.*]] = getelementptr inbounds [8 x ptr], ptr %.offload_baseptrs, i32 0, i32 0
 !CHECK: store ptr %[[DTYPE_DESC_ALLOCA_3]], ptr %[[BASE_PTR_ARR]], align 8
 !CHECK: %[[OFFLOAD_PTR_ARR:.*]] = getelementptr inbounds [8 x ptr], ptr %.offload_ptrs, i32 0, i32 0
@@ -667,7 +675,7 @@ end subroutine mapType_common_block_members
 !CHECK: %[[OFFLOAD_PTR_ARR:.*]] = getelementptr inbounds [8 x ptr], ptr %.offload_ptrs, i32 0, i32 6
 !CHECK: store ptr %[[ARRAY_OFFSET]], ptr %[[OFFLOAD_PTR_ARR]], align 8
 !CHECK: %[[OFFLOAD_SIZE_ARR:.*]] = getelementptr inbounds [8 x i64], ptr %.offload_sizes, i32 0, i32 6
-!CHECK: store i64 %[[ALLOCATABLE_MEMBER_SIZE_CALC_5]], ptr %[[OFFLOAD_SIZE_ARR]], align 8
+!CHECK: store i64 %[[DATA_SEL]], ptr %[[OFFLOAD_SIZE_ARR]], align 8
 !CHECK: %[[BASE_PTR_ARR:.*]] = getelementptr inbounds [8 x ptr], ptr %.offload_baseptrs, i32 0, i32 7
 !CHECK: store ptr %[[DTYPE_DESC_ALLOCA_3]], ptr %[[BASE_PTR_ARR]], align 8
 !CHECK: %[[OFFLOAD_PTR_ARR:.*]] = getelementptr inbounds [8 x ptr], ptr %.offload_ptrs, i32 0, i32 7
@@ -693,6 +701,8 @@ end subroutine mapType_common_block_members
 !CHECK: %[[DTYPE_SEGMENT_SIZE_CALC_2:.*]] = ptrtoint ptr %[[NESTED_MEMBER_ACCESS]] to i64
 !CHECK: %[[DTYPE_SEGMENT_SIZE_CALC_3:.*]] = sub i64 %[[DTYPE_SEGMENT_SIZE_CALC_1]], %[[DTYPE_SEGMENT_SIZE_CALC_2]]
 !CHECK: %[[DTYPE_SEGMENT_SIZE_CALC_4:.*]] = sdiv exact i64 %[[DTYPE_SEGMENT_SIZE_CALC_3]], ptrtoint (ptr getelementptr (i8, ptr null, i32 1) to i64)
+!CHECK: %[[DATA_CMP:.*]] = icmp eq ptr %[[ARR_OFFS]], null
+!CHECK: %[[DATA_SEL:.*]] = select i1 %[[DATA_CMP]], i64 0, i64 %[[ALLOCATABLE_MEMBER_SIZE_CALC_5]]
 !CHECK: %[[BASE_PTR_ARR:.*]] = getelementptr inbounds [4 x ptr], ptr %.offload_baseptrs, i32 0, i32 0
 !CHECK: store ptr %[[ALLOCA]], ptr %[[BASE_PTR_ARR]], align 8
 !CHECK: %[[OFFLOAD_PTR_ARR:.*]] = getelementptr inbounds [4 x ptr], ptr %.offload_ptrs, i32 0, i32 0
@@ -712,7 +722,7 @@ end subroutine mapType_common_block_members
 !CHECK: %[[OFFLOAD_PTR_ARR:.*]] = getelementptr inbounds [4 x ptr], ptr %.offload_ptrs, i32 0, i32 3
 !CHECK: store ptr %[[ARR_OFFS]], ptr %[[OFFLOAD_PTR_ARR]], align 8
 !CHECK: %[[OFFLOAD_SIZE_ARR:.*]] = getelementptr inbounds [4 x i64], ptr %.offload_sizes, i32 0, i32 3
-!CHECK: store i64 %[[ALLOCATABLE_MEMBER_SIZE_CALC_5]], ptr %[[OFFLOAD_SIZE_ARR]], align 8
+!CHECK: store i64 %[[DATA_SEL]], ptr %[[OFFLOAD_SIZE_ARR]], align 8
 
 !CHECK-LABEL: define {{.*}} @{{.*}}maptype_nested_derived_type_member_idx{{.*}}
 !CHECK: %[[ALLOCA_0:.*]] = alloca { ptr, i64, i32, i8, i8, i8, i8, [1 x [3 x i64]], ptr, [1 x i64] }, align 8
@@ -754,6 +764,10 @@ end subroutine mapType_common_block_members
 !CHECK: %[[SZ_CALC_3:.*]] = ptrtoint ptr %[[OFF_PTR_1]] to i64
 !CHECK: %[[SZ_CALC_4:.*]] = sub i64 %[[SZ_CALC_2]], %[[SZ_CALC_3]]
 !CHECK: %[[SZ_CALC_5:.*]] = sdiv exact i64 %[[SZ_CALC_4]], ptrtoint (ptr getelementptr (i8, ptr null, i32 1) to i64)
+!CHECK: %[[SIZE_CMP:.*]] = icmp eq ptr %[[ARR_OFFS]], null
+!CHECK: %[[SIZE_SEL:.*]] = select i1 %[[SIZE_CMP]], i64 0, i64 %[[OFF_PTR_3]]
+!CHECK: %[[SIZE_CMP2:.*]] = icmp eq ptr %[[ARR_OFFS_1]], null
+!CHECK: %[[SIZE_SEL2:.*]] = select i1 %[[SIZE_CMP2]], i64 0, i64 %[[SZ_CALC_4_2]]
 !CHECK: %[[BASE_PTR_ARR:.*]] = getelementptr inbounds [7 x ptr], ptr %.offload_baseptrs, i32 0, i32 0
 !CHECK: store ptr %[[BASE_PTR_1]], ptr %[[BASE_PTR_ARR]], align 8
 !CHECK: %[[OFFLOAD_PTR_ARR:.*]] = getelementptr inbounds [7 x ptr], ptr %.offload_ptrs, i32 0, i32 0
@@ -773,7 +787,7 @@ end subroutine mapType_common_block_members
 !CHECK: %[[OFFLOAD_PTR_ARR:.*]] = getelementptr inbounds [7 x ptr], ptr %.offload_ptrs, i32 0, i32 3
 !CHECK: store ptr %[[ARR_OFFS]], ptr %[[OFFLOAD_PTR_ARR]], align 8
 !CHECK: %[[OFFLOAD_SIZE_ARR:.*]] = getelementptr inbounds [7 x i64], ptr %.offload_sizes, i32 0, i32 3
-!CHECK: store i64 %[[OFF_PTR_3]], ptr %[[OFFLOAD_SIZE_ARR]], align 8
+!CHECK: store i64 %[[SIZE_SEL]], ptr %[[OFFLOAD_SIZE_ARR]], align 8
 !CHECK: %[[BASE_PTR_ARR:.*]] = getelementptr inbounds [7 x ptr], ptr %.offload_baseptrs, i32 0, i32 4
 !CHECK: store ptr %[[BASE_PTR_1]], ptr %[[BASE_PTR_ARR]], align 8
 !CHECK: %[[OFFLOAD_PTR_ARR:.*]] = getelementptr inbounds [7 x ptr], ptr %.offload_ptrs, i32 0, i32 4
@@ -787,8 +801,7 @@ end subroutine mapType_common_block_members
 !CHECK: %[[OFFLOAD_PTR_ARR:.*]] = getelementptr inbounds [7 x ptr], ptr %.offload_ptrs, i32 0, i32 6
 !CHECK: store ptr %[[ARR_OFFS_1]], ptr %[[OFFLOAD_PTR_ARR]], align 8
 !CHECK: %[[OFFLOAD_SIZE_ARR:.*]] = getelementptr inbounds [7 x i64], ptr %.offload_sizes, i32 0, i32 6
-!CHECK: store i64 %[[SZ_CALC_4_2]], ptr %[[OFFLOAD_SIZE_ARR]], align 8
-
+!CHECK: store i64 %[[SIZE_SEL2]], ptr %[[OFFLOAD_SIZE_ARR]], align 8
 !CHECK-LABEL: define {{.*}} @{{.*}}maptype_common_block_{{.*}}
 !CHECK: %[[BASE_PTR_ARR:.*]] = getelementptr inbounds [1 x ptr], ptr %.offload_baseptrs, i32 0, i32 0
 !CHECK: store ptr @var_common_, ptr %[[BASE_PTR_ARR]], align 8
diff --git a/flang/test/Integration/OpenMP/parallel-private-reduction-worstcase.f90 b/flang/test/Integration/OpenMP/parallel-private-reduction-worstcase.f90
index 0173847b73235..7e735b6499504 100644
--- a/flang/test/Integration/OpenMP/parallel-private-reduction-worstcase.f90
+++ b/flang/test/Integration/OpenMP/parallel-private-reduction-worstcase.f90
@@ -32,123 +32,123 @@ subroutine worst_case(a, b, c, d)
 ! CHECK-LABEL: define internal void @worst_case_..omp_par
 ! CHECK-NEXT:  omp.par.entry:
 !                [reduction alloc regions inlined here]
-! CHECK:         br label %omp.private.latealloc
+! CHECK:         br label %omp.region.after_alloca
 
-! CHECK:       omp.private.latealloc:                            ; preds = %omp.par.entry
-! CHECK-NEXT:  br label %omp.private.alloc5
+! CHECK:       omp.region.after_alloca:
+! CHECK-NEXT:    br label %omp.par.region
+
+! CHECK:       omp.par.region:
+! CHECK-NEXT:    br label %omp.private.init
+
+! CHECK:       omp.private.init:
+! CHECK-NEXT:  br label %omp.private.init2
 
-! CHECK:       omp.private.alloc5:                               ; preds = %omp.private.latealloc
+! CHECK:       omp.private.init2:
 !                [begin private alloc for first var]
 !                [read the length from the mold argument]
 !                [if it is non-zero...]
-! CHECK:         br i1 {{.*}}, label %omp.private.alloc6, label %omp.private.alloc7
+! CHECK:         br i1 %{{.*}}, label %omp.private.init3, label %omp.private.init4
 
-! CHECK:       omp.private.alloc7:                               ; preds = %omp.private.alloc5
-!                [finish private alloc for first var with zero extent]
-! CHECK:         br label %omp.private.alloc8
+! CHECK:       omp.private.init4:                               ; preds = %omp.private.init2
+!                [finish private alloc for second var with zero extent]
+! CHECK:         br label %omp.private.init5
 
-! CHECK:       omp.private.alloc8:                               ; preds = %omp.private.alloc6, %omp.private.alloc7
-! CHECK-NEXT:    br label %omp.region.cont4
+! CHECK:       omp.private.init5:                               ; preds = %omp.private.init3, %omp.private.init4
+! CHECK-NEXT:    br label %omp.region.cont
 
-! CHECK:       omp.region.cont4:                                 ; preds = %omp.private.alloc8
+! CHECK:       omp.region.cont:                                  ; preds = %omp.private.init5
 ! CHECK-NEXT:    %{{.*}} = phi ptr
-! CHECK-NEXT:    br label %omp.private.alloc
+! CHECK-NEXT:    br label %omp.private.init7
 
-! CHECK:       omp.private.alloc:                                ; preds = %omp.region.cont4
+! CHECK:       omp.private.init7:
 !                [begin private alloc for first var]
 !                [read the length from the mold argument]
 !                [if it is non-zero...]
-! CHECK:         br i1 %{{.*}}, label %omp.private.alloc1, label %omp.private.alloc2
+! CHECK:         br i1 {{.*}}, label %omp.private.init8, label %omp.private.init9
 
-! CHECK:       omp.private.alloc2:                               ; preds = %omp.private.alloc
-!                [finish private alloc for second var with zero extent]
-! CHECK:         br label %omp.private.alloc3
+! CHECK:       omp.private.init9:                               ; preds = %omp.private.init7
+!                [finish private alloc for first var with zero extent]
+! CHECK:         br label %omp.private.init10
 
-! CHECK:       omp.private.alloc3:                               ; preds = %omp.private.alloc1, %omp.private.alloc2
-! CHECK-NEXT:    br label %omp.region.cont
+! CHECK:       omp.private.init10:                               ; preds = %omp.private.init8, %omp.private.init9
+! CHECK-NEXT:    br label %omp.region.cont6
 
-! CHECK:       omp.region.cont:                                  ; preds = %omp.private.alloc3
+! CHECK:       omp.region.cont6:                                 ; preds = %omp.private.init10
 ! CHECK-NEXT:    %{{.*}} = phi ptr
 ! CHECK-NEXT:    br label %omp.private.copy
 
-! CHECK:       omp.private.copy:                                 ; preds = %omp.region.cont
-! CHECK-NEXT:    br label %omp.private.copy10
+! CHECK:       omp.private.copy:
+! CHECK-NEXT:    br label %omp.private.copy12
 
-! CHECK:       omp.private.copy10:                               ; preds = %omp.private.copy
+! CHECK:       omp.private.copy12:                               ; preds = %omp.private.copy
 !                [begin firstprivate copy for first var]
 !                [read the length, is it non-zero?]
-! CHECK:         br i1 %{{.*}}, label %omp.private.copy11, label %omp.private.copy12
+! CHECK:         br i1 %{{.*}}, label %omp.private.copy13, label %omp.private.copy14
 
-! CHECK:       omp.private.copy12:                               ; preds = %omp.private.copy11, %omp.private.copy10
-! CHECK-NEXT:    br label %omp.region.cont9
+! CHECK:       omp.private.copy14:                               ; preds = %omp.private.copy13, %omp.private.copy12
+! CHECK-NEXT:    br label %omp.region.cont11
 
-! CHECK:       omp.region.cont9:                                 ; preds = %omp.private.copy12
+! CHECK:       omp.region.cont11:                                 ; preds = %omp.private.copy14
 ! CHECK-NEXT:    %{{.*}} = phi ptr
-! CHECK-NEXT:    br label %omp.private.copy14
+! CHECK-NEXT:    br label %omp.private.copy16
 
-! CHECK:       omp.private.copy14:                               ; preds = %omp.region.cont9
+! CHECK:       omp.private.copy16:                               ; preds = %omp.region.cont11
 !                [begin firstprivate copy for second var]
 !                [read the length, is it non-zero?]
-! CHECK:         br i1 %{{.*}}, label %omp.private.copy15, label %omp.private.copy16
+! CHECK:         br i1 %{{.*}}, label %omp.private.copy17, label %omp.private.copy18
 
-! CHECK:       omp.private.copy16:                               ; preds = %omp.private.copy15, %omp.private.copy14
-! CHECK-NEXT:    br label %omp.region.cont13
+! CHECK:       omp.private.copy18:                               ; preds = %omp.private.copy17, %omp.private.copy16
+! CHECK-NEXT:    br label %omp.region.cont15
 
-! CHECK:       omp.region.cont13:                                ; preds = %omp.private.copy16
+! CHECK:       omp.region.cont15:                                ; preds = %omp.private.copy18
 ! CHECK-NEXT:    %{{.*}} = phi ptr
-! CHECK-NEXT:    br label %omp.region.after_alloca
-
-! CHECK:       omp.region.after_alloca:
-! CHECK-NEXT:    br label %omp.par.region
-
-! CHECK:       omp.par.region:                                   ; preds = %omp.region.after_alloca
 ! CHECK-NEXT:    br label %omp.reduction.init
 
-! CHECK:       omp.reduction.init:                               ; preds = %omp.par.region
+! CHECK:       omp.reduction.init:                               ; preds = %omp.region.cont15
 !                [deffered stores for results of reduction alloc regions]
 ! CHECK:         br label %[[VAL_96:.*]]
 
 ! CHECK:       omp.reduction.neutral:                            ; preds = %omp.reduction.init
 !                [start of reduction initialization region]
 !                [null check:]
-! CHECK:         br i1 %{{.*}}, label %omp.reduction.neutral18, label %omp.reduction.neutral19
+! CHECK:         br i1 %{{.*}}, label %omp.reduction.neutral20, label %omp.reduction.neutral21
 
-! CHECK:       omp.reduction.neutral19:                          ; preds = %omp.reduction.neutral
+! CHECK:       omp.reduction.neutral21:                          ; preds = %omp.reduction.neutral
 !                [malloc and assign the default value to the reduction variable]
-! CHECK:         br label %omp.reduction.neutral20
+! CHECK:         br label %omp.reduction.neutral22
 
-! CHECK:       omp.reduction.neutral20:                          ; preds = %omp.reduction.neutral18, %omp.reduction.neutral19
-! CHECK-NEXT:    br label %omp.region.cont17
+! CHECK:       omp.reduction.neutral22:                          ; preds = %omp.reduction.neutral20, %omp.reduction.neutral21
+! CHECK-NEXT:    br label %omp.region.cont19
 
-! CHECK:       omp.region.cont17:                                ; preds = %omp.reduction.neutral20
+! CHECK:       omp.region.cont19:                                ; preds = %omp.reduction.neutral22
 ! CHECK-NEXT:    %{{.*}} = phi ptr
-! CHECK-NEXT:    br label %omp.reduction.neutral22
+! CHECK-NEXT:    br label %omp.reduction.neutral24
 
-! CHECK:       omp.reduction.neutral22:                          ; preds = %omp.region.cont17
+! CHECK:       omp.reduction.neutral24:                          ; preds = %omp.region.cont19
 !                [start of reduction initialization region]
 !                [null check:]
-! CHECK:         br i1 %{{.*}}, label %omp.reduction.neutral23, label %omp.reduction.neutral24
+! CHECK:         br i1 %{{.*}}, label %omp.reduction.neutral25, label %omp.reduction.neutral26
 
-! CHECK:       omp.reduction.neutral24:                          ; preds = %omp.reduction.neutral22
+! CHECK:       omp.reduction.neutral26:                          ; preds = %omp.reduction.neutral24
 !                [malloc and assign the default value to the reduction variable]
-! CHECK:         br label %omp.reduction.neutral25
+! CHECK:         br label %omp.reduction.neutral27
 
-! CHECK:       omp.reduction.neutral25:                          ; preds = %omp.reduction.neutral23, %omp.reduction.neutral24
-! CHECK-NEXT:    br label %omp.region.cont21
+! CHECK:       omp.reduction.neutral27:                          ; preds = %omp.reduction.neutral25, %omp.reduction.neutral26
+! CHECK-NEXT:    br label %omp.region.cont23
 
-! CHECK:       omp.region.cont21:                                ; preds = %omp.reduction.neutral25
+! CHECK:       omp.region.cont23:                                ; preds = %omp.reduction.neutral27
 ! CHECK-NEXT:    %{{.*}} = phi ptr
-! CHECK-NEXT:    br label %omp.par.region27
+! CHECK-NEXT:    br label %omp.par.region29
 
-! CHECK:       omp.par.region27:                                 ; preds = %omp.region.cont21
+! CHECK:       omp.par.region29:                                 ; preds = %omp.region.cont23
 !                [call SUM runtime function]
 !                [if (sum(a) == 1)]
-! CHECK:         br i1 %{{.*}}, label %omp.par.region28, label %omp.par.region29
+! CHECK:         br i1 %{{.*}}, label %omp.par.region30, label %omp.par.region31
 
-! CHECK:       omp.par.region29:                                 ; preds = %omp.par.region27
-! CHECK-NEXT:    br label %omp.region.cont26
+! CHECK:       omp.par.region31:                                 ; preds = %omp.par.region29
+! CHECK-NEXT:    br label %omp.region.cont28
 
-! CHECK:       omp.region.cont26:                                ; preds = %omp.par.region28, %omp.par.region29
+! CHECK:       omp.region.cont28:                                ; preds = %omp.par.region30, %omp.par.region31
 !                [omp parallel region done, call into the runtime to complete reduction]
 ! CHECK:         %[[VAL_233:.*]] = call i32 @__kmpc_reduce(
 ! CHECK:         switch i32 %[[VAL_233]], label %reduce.finalize [
@@ -156,16 +156,16 @@ subroutine worst_case(a, b, c, d)
 ! CHECK-NEXT:      i32 2, label %reduce.switch.atomic
 ! CHECK-NEXT:    ]
 
-! CHECK:       reduce.switch.atomic:                             ; preds = %omp.region.cont26
+! CHECK:       reduce.switch.atomic:                             ; preds = %omp.region.cont28
 ! CHECK-NEXT:    unreachable
 
-! CHECK:       reduce.switch.nonatomic:                          ; preds = %omp.region.cont26
+! CHECK:       reduce.switch.nonatomic:                          ; preds = %omp.region.cont28
 ! CHECK-NEXT:    %[[red_private_value_0:.*]] = load ptr, ptr %{{.*}}, align 8
 ! CHECK-NEXT:    br label %omp.reduction.nonatomic.body
 
 !              [various blocks implementing the reduction]
 
-! CHECK:       omp.region.cont35:                                ; preds =
+! CHECK:       omp.region.cont37:                                ; preds =
 ! CHECK-NEXT:    %{{.*}} = phi ptr
 ! CHECK-NEXT:    call void @__kmpc_end_reduce(
 ! CHECK-NEXT:    br label %reduce.finalize
@@ -179,87 +179,45 @@ subroutine worst_case(a, b, c, d)
 
 ! CHECK:       omp.reduction.cleanup:                            ; preds = %omp.par.pre_finalize
 !                [null check]
-! CHECK:         br i1 %{{.*}}, label %omp.reduction.cleanup41, label %omp.reduction.cleanup42
+! CHECK:         br i1 %{{.*}}, label %omp.reduction.cleanup43, label %omp.reduction.cleanup44
 
-! CHECK:       omp.reduction.cleanup42:                          ; preds = %omp.reduction.cleanup41, %omp.reduction.cleanup
-! CHECK-NEXT:    br label %omp.region.cont40
+! CHECK:       omp.reduction.cleanup44:                          ; preds = %omp.reduction.cleanup43, %omp.reduction.cleanup
+! CHECK-NEXT:    br label %omp.region.cont42
 
-! CHECK:       omp.region.cont40:                                ; preds = %omp.reduction.cleanup42
+! CHECK:       omp.region.cont42:                                ; preds = %omp.reduction.cleanup44
 ! CHECK-NEXT:    %{{.*}} = load ptr, ptr
-! CHECK-NEXT:    br label %omp.reduction.cleanup44
-
-! CHECK:       omp.reduction.cleanup44:                          ; preds = %omp.region.cont40
-!                [null check]
-! CHECK:         br i1 %{{.*}}, label %omp.reduction.cleanup45, label %omp.reduction.cleanup46
-
-! CHECK:       omp.reduction.cleanup46:                          ; preds = %omp.reduction.cleanup45, %omp.reduction.cleanup44
-! CHECK-NEXT:    br label %omp.region.cont43
-
-! CHECK:       omp.region.cont43:                                ; preds = %omp.reduction.cleanup46
-! CHECK-NEXT:    br label %omp.private.dealloc
-
-! CHECK:       omp.private.dealloc:                              ; preds = %omp.region.cont43
-!                [null check]
-! CHECK:         br i1 %{{.*}}, label %omp.private.dealloc48, label %omp.private.dealloc49
-
-! CHECK:       omp.private.dealloc49:                            ; preds = %omp.private.dealloc48, %omp.private.dealloc
-! CHECK-NEXT:    br label %omp.region.cont47
-
-! CHECK:       omp.region.cont47:                                ; preds = %omp.private.dealloc49
-! CHECK-NEXT:    br label %omp.private.dealloc51
-
-! CHECK:       omp.private.dealloc51:                            ; preds = %omp.region.cont47
-!                [null check]
-! CHECK:         br i1 %{{.*}}, label %omp.private.dealloc52, label %omp.private.dealloc53
-
-! CHECK:       omp.private.dealloc53:                            ; preds = %omp.private.dealloc52, %omp.private.dealloc51
-! CHECK-NEXT:    br label %omp.region.cont50
-
-! CHECK:       omp.region.cont50:                                ; preds = %omp.private.dealloc53
-! CHECK-NEXT:    br label %omp.par.outlined.exit.exitStub
-
-! CHECK:       omp.private.dealloc52:                            ; preds = %omp.private.dealloc51
-!                [dealloc memory]
-! CHECK:         br label %omp.private.dealloc53
-
-! CHECK:       omp.private.dealloc48:                            ; preds = %omp.private.dealloc
-!                [dealloc memory]
-! CHECK:         br label %omp.private.dealloc49
-
-! CHECK:       omp.reduction.cleanup45:                          ; preds = %omp.reduction.cleanup44
-! CHECK-NEXT:    call void @free(
 ! CHECK-NEXT:    br label %omp.reduction.cleanup46
 
-! CHECK:       omp.reduction.cleanup41:                          ; preds = %omp.reduction.cleanup
-! CHECK-NEXT:    call void @free(
-! CHECK-NEXT:    br label %omp.reduction.cleanup42
+! CHECK:       omp.reduction.cleanup46:                          ; preds = %omp.region.cont42
+!                [null check]
+! CHECK:         br i1 %{{.*}}, label %omp.reduction.cleanup47, label %omp.reduction.cleanup48
 
-! CHECK:       omp.par.region28:                                 ; preds = %omp.par.region27
+! CHECK:       omp.par.region30:                                 ; preds = %omp.par.region29
 ! CHECK-NEXT:    call void @_FortranAStopStatement
 
-! CHECK:       omp.reduction.neutral23:                          ; preds = %omp.reduction.neutral22
+! CHECK:       omp.reduction.neutral25:                          ; preds = %omp.reduction.neutral24
 !                [source length was zero: finish initializing array]
-! CHECK:         br label %omp.reduction.neutral25
+! CHECK:         br label %omp.reduction.neutral27
 
-! CHECK:       omp.reduction.neutral18:                          ; preds = %omp.reduction.neutral
+! CHECK:       omp.reduction.neutral20:                          ; preds = %omp.reduction.neutral
 !                [source length was zero: finish initializing array]
-! CHECK:         br label %omp.reduction.neutral20
+! CHECK:         br label %omp.reduction.neutral22
 
-! CHECK:       omp.private.copy15:                               ; preds = %omp.private.copy14
+! CHECK:       omp.private.copy17:                               ; preds = %omp.private.copy16
 !                [source length was non-zero: call assign runtime]
-! CHECK:         br label %omp.private.copy16
+! CHECK:         br label %omp.private.copy18
 
-! CHECK:       omp.private.copy11:                               ; preds = %omp.private.copy10
+! CHECK:       omp.private.copy13:                               ; preds = %omp.private.copy12
 !                [source length was non-zero: call assign runtime]
-! CHECK:         br label %omp.private.copy12
+! CHECK:         br label %omp.private.copy14
 
-! CHECK:       omp.private.alloc1:                               ; preds = %omp.private.alloc
+! CHECK:       omp.private.init8:                               ; preds = %omp.private.init7
 !                [var extent was non-zero: malloc a private array]
-! CHECK:         br label %omp.private.alloc3
+! CHECK:         br label %omp.private.init10
 
-! CHECK:       omp.private.alloc6:                               ; preds = %omp.private.alloc5
+! CHECK:       omp.private.init3:                               ; preds = %omp.private.init2
 !                [var extent was non-zero: malloc a private array]
-! CHECK:         br label %omp.private.alloc8
+! CHECK:         br label %omp.private.init5
 
-! CHECK:       omp.par.outlined.exit.exitStub:                   ; preds = %omp.region.cont50
+! CHECK:       omp.par.outlined.exit.exitStub:                   ; preds = %omp.region.cont52
 ! CHECK-NEXT:    ret void
diff --git a/flang/test/Integration/OpenMP/private-global.f90 b/flang/test/Integration/OpenMP/private-global.f90
index 07dbe86e5ec93..39d7e2274cff9 100644
--- a/flang/test/Integration/OpenMP/private-global.f90
+++ b/flang/test/Integration/OpenMP/private-global.f90
@@ -21,20 +21,24 @@ program bug
 ! CHECK:         %[[VAL_10:.*]] = load i32, ptr %[[VAL_11:.*]], align 4
 ! CHECK:         store i32 %[[VAL_10]], ptr %[[VAL_9]], align 4
 ! CHECK:         %[[VAL_12:.*]] = load i32, ptr %[[VAL_9]], align 4
-! CHECK:         %[[PRIV_TABLE:.*]] = alloca [10 x i32], i64 1, align 4
+! CHECK:         %[[PRIV_BOX_ALLOC:.*]] = alloca { ptr, i64, i32, i8, i8, i8, i8, [1 x [3 x i64]] }, align 8
 ! ...
 ! check that we use the private copy of table for the assignment
 ! CHECK:       omp.par.region1:
 ! CHECK:         %[[ELEMENTAL_TMP:.*]] = alloca { ptr, i64, i32, i8, i8, i8, i8, [1 x [3 x i64]] }, align 8
 ! CHECK:         %[[TABLE_BOX_ADDR:.*]] = alloca { ptr, i64, i32, i8, i8, i8, i8, [1 x [3 x i64]] }, align 8
 ! CHECK:         %[[BOXED_FIFTY:.*]] = alloca { ptr, i64, i32, i8, i8, i8, i8 }, align 8
+! CHECK:         %[[FIFTY:.*]] = alloca i32, i64 1, align 4
+! CHECK:         %[[INTERMEDIATE:.*]] = alloca { ptr, i64, i32, i8, i8, i8, i8, [1 x [3 x i64]] }, align 8
 ! CHECK:         %[[TABLE_BOX_ADDR2:.*]] = alloca { ptr, i64, i32, i8, i8, i8, i8, [1 x [3 x i64]] }, i64 1, align 8
-! CHECK:         %[[TABLE_BOX_VAL:.*]] = insertvalue { ptr, i64, i32, i8, i8, i8, i8, [1 x [3 x i64]] } { ptr undef, i64 ptrtoint (ptr getelementptr (i32, ptr null, i32 1) to i64), i32 20240719, i8 1, i8 9, i8 0, i8 0, [1 x [3 x i64]] {{\[\[}}3 x i64] [i64 1, i64 10, i64 ptrtoint (ptr getelementptr (i32, ptr null, i32 1) to i64)]] }, ptr %[[PRIV_TABLE]], 0
-! CHECK:         store { ptr, i64, i32, i8, i8, i8, i8, [1 x [3 x i64]] } %[[TABLE_BOX_VAL]], ptr %[[TABLE_BOX_ADDR]], align 8
-! CHECK :         %[[TABLE_BOX_VAL2:.*]] = load { ptr, i64, i32, i8, i8, i8, i8, [1 x [3 x i64]] }, ptr %[[TABLE_BOX_ADDR]], align 8
-! CHECK :         store { ptr, i64, i32, i8, i8, i8, i8, [1 x [3 x i64]] } %[[TABLE_BOX_VAL2]], ptr %[[TABLE_BOX_ADDR2]], align 8
-! CHECK:         call void @llvm.memcpy.p0.p0.i32(ptr %[[TABLE_BOX_ADDR2]], ptr %[[TABLE_BOX_ADDR]], i32 48, i1 false)
+! CHECK:         call void @llvm.memcpy.p0.p0.i32(ptr %[[INTERMEDIATE]], ptr %[[PRIV_BOX_ALLOC]], i32 48, i1 false)
+! CHECK:         store i32 50, ptr %[[FIFTY]], align 4
+! CHECK:         %[[FIFTY_BOX_VAL:.*]] = insertvalue { ptr, i64, i32, i8, i8, i8, i8 } { ptr undef, i64 ptrtoint (ptr getelementptr (i32, ptr null, i32 1) to i64), i32 20240719, i8 0, i8 9, i8 0, i8 0 }, ptr %[[FIFTY]], 0
+! CHECK:         store { ptr, i64, i32, i8, i8, i8, i8 } %[[FIFTY_BOX_VAL]], ptr %[[BOXED_FIFTY]], align 8
+! CHECK:         call void @llvm.memcpy.p0.p0.i32(ptr %[[TABLE_BOX_ADDR2]], ptr %[[INTERMEDIATE]], i32 48, i1 false)
 ! CHECK:         call void @_FortranAAssign(ptr %[[TABLE_BOX_ADDR2]], ptr %[[BOXED_FIFTY]], ptr @{{.*}}, i32 9)
+! CHECK:         call void @llvm.memcpy.p0.p0.i32(ptr %[[TABLE_BOX_ADDR]], ptr %[[PRIV_BOX_ALLOC]], i32 48, i1 false)
+! CHECK:         %[[PRIV_TABLE:.*]] = call ptr @malloc(i64 ptrtoint (ptr getelementptr ([10 x i32], ptr null, i32 1) to i64))
 ! ...
 ! check that we use the private copy of table for table/=50
 ! CHECK:       omp.par.region3:
@@ -43,5 +47,3 @@ program bug
 ! CHECK:         %[[VAL_46:.*]] = mul nsw i64 %[[VAL_45]], 1
 ! CHECK:         %[[VAL_47:.*]] = add nsw i64 %[[VAL_46]], 0
 ! CHECK:         %[[VAL_48:.*]] = getelementptr i32, ptr %[[PRIV_TABLE]], i64 %[[VAL_47]]
-! CHECK:         %[[VAL_49:.*]] = load i32, ptr %[[VAL_48]], align 4
-! CHECK:         %[[VAL_50:.*]] = icmp ne i32 %[[VAL_49]], 50
diff --git a/flang/test/Integration/debug-common-block-1.f90 b/flang/test/Integration/debug-common-block-1.f90
new file mode 100644
index 0000000000000..18217637be0fa
--- /dev/null
+++ b/flang/test/Integration/debug-common-block-1.f90
@@ -0,0 +1,138 @@
+! RUN: %flang_fc1 -emit-llvm -debug-info-kind=standalone %s -o - | FileCheck  %s
+
+subroutine f1
+  real(kind=4) :: x, y, xa, ya
+  common // x, y
+  common /a/ xa, ya
+  x = 1.1
+  y = 2.2
+  xa = 3.3
+  ya = 4.4
+  print *, x, y, xa, ya
+end subroutine
+! CHECK-DAG: ![[XF1:[0-9]+]] = {{.*}}!DIGlobalVariable(name: "x", linkageName: "_QFf1Ex", scope: ![[CBF1:[0-9]+]], file: !5, line: [[@LINE-9]], type: ![[REAL:[0-9]+]]{{.*}})
+! CHECK-DAG: ![[EXPXF1:[0-9]+]] = !DIGlobalVariableExpression(var: ![[XF1]], expr: !DIExpression())
+! CHECK-DAG: ![[YF1:[0-9]+]] = {{.*}}!DIGlobalVariable(name: "y", linkageName: "_QFf1Ey", scope: ![[CBF1]], file: !{{[0-9]+}}, line: [[@LINE-11]], type: ![[REAL]]{{.*}})
+! CHECK-DAG: ![[EXPYF1:[0-9]+]] = !DIGlobalVariableExpression(var: ![[YF1]], expr: !DIExpression(DW_OP_plus_uconst, 4))
+! CHECK-DAG: ![[XAF1:[0-9]+]] = {{.*}}!DIGlobalVariable(name: "xa", linkageName: "_QFf1Exa", scope: ![[CBAF1:[0-9]+]], file: !{{[0-9]+}}, line: [[@LINE-13]], type: ![[REAL]]{{.*}})
+! CHECK-DAG: ![[EXPXAF1:[0-9]+]] = !DIGlobalVariableExpression(var: ![[XAF1]], expr: !DIExpression())
+! CHECK-DAG: ![[YAF1:[0-9]+]] = {{.*}}!DIGlobalVariable(name: "ya", linkageName: "_QFf1Eya", scope: ![[CBAF1]], file: !{{[0-9]+}}, line: [[@LINE-15]], type: ![[REAL]]{{.*}})
+! CHECK-DAG: ![[EXPYAF1:[0-9]+]] = !DIGlobalVariableExpression(var: ![[YAF1]], expr: !DIExpression(DW_OP_plus_uconst, 4))
+
+
+subroutine f2
+  real(kind=4) :: x, y, z, xa, ya, za
+  common // x, y, z
+  common /a/ xa, ya, za
+  print *, x, y, z, xa, ya, za
+end subroutine
+! CHECK-DAG: ![[XF2:[0-9]+]] = {{.*}}!DIGlobalVariable(name: "x", linkageName: "_QFf2Ex", scope: ![[CBF2:[0-9]+]], file: !{{[0-9]+}}, line: [[@LINE-5]], type: ![[REAL]]{{.*}})
+! CHECK-DAG: ![[EXPXF2:[0-9]+]] = !DIGlobalVariableExpression(var: ![[XF2]], expr: !DIExpression())
+! CHECK-DAG: ![[YF2:[0-9]+]] = {{.*}}!DIGlobalVariable(name: "y", linkageName: "_QFf2Ey", scope: ![[CBF2]], file: !{{[0-9]+}}, line: [[@LINE-7]], type: ![[REAL]]{{.*}})
+! CHECK-DAG: ![[EXPYF2:[0-9]+]] = !DIGlobalVariableExpression(var: ![[YF2]], expr: !DIExpression(DW_OP_plus_uconst, 4))
+! CHECK-DAG: ![[ZF2:[0-9]+]] = {{.*}}!DIGlobalVariable(name: "z", linkageName: "_QFf2Ez", scope: ![[CBF2]], file: !{{[0-9]+}}, line: [[@LINE-9]], type: ![[REAL]]{{.*}})
+! CHECK-DAG: ![[EXPZF2:[0-9]+]] = !DIGlobalVariableExpression(var: ![[ZF2]], expr: !DIExpression(DW_OP_plus_uconst, 8))
+! CHECK-DAG: ![[XAF2:[0-9]+]] = {{.*}}!DIGlobalVariable(name: "xa", linkageName: "_QFf2Exa", scope: ![[CBAF2:[0-9]+]], file: !{{[0-9]+}}, line: [[@LINE-11]], type: ![[REAL]]{{.*}})
+! CHECK-DAG: ![[EXPXAF2:[0-9]+]] = !DIGlobalVariableExpression(var: ![[XAF2]], expr: !DIExpression())
+! CHECK-DAG: ![[YAF2:[0-9]+]] = {{.*}}!DIGlobalVariable(name: "ya", linkageName: "_QFf2Eya", scope: ![[CBAF2]], file: !{{[0-9]+}}, line: [[@LINE-13]], type: ![[REAL]]{{.*}})
+! CHECK-DAG: ![[EXPYAF2:[0-9]+]] = !DIGlobalVariableExpression(var: ![[YAF2]], expr: !DIExpression(DW_OP_plus_uconst, 4))
+! CHECK-DAG: ![[ZAF2:[0-9]+]] = {{.*}}!DIGlobalVariable(name: "za", linkageName: "_QFf2Eza", scope: ![[CBAF2]], file: !{{[0-9]+}}, line: [[@LINE-15]], type: ![[REAL]]{{.*}})
+! CHECK-DAG: ![[EXPZAF2:[0-9]+]] = !DIGlobalVariableExpression(var: ![[ZAF2]], expr: !DIExpression(DW_OP_plus_uconst, 8))
+
+subroutine f3
+  integer(kind=4) :: x = 42, xa = 42
+  common // x
+  common /a/ xa
+  print *, x
+  print *, xa
+end subroutine
+! CHECK-DAG: ![[XF3:[0-9]+]] = {{.*}}!DIGlobalVariable(name: "x", linkageName: "_QFf3Ex", scope: ![[CBF3:[0-9]+]], file: !{{[0-9]+}}, line: [[@LINE-6]], type: ![[INT:[0-9]+]]{{.*}})
+! CHECK-DAG: ![[EXPXF3:[0-9]+]] = !DIGlobalVariableExpression(var: ![[XF3]], expr: !DIExpression())
+! CHECK-DAG: ![[XAF3:[0-9]+]] = {{.*}}!DIGlobalVariable(name: "xa", linkageName: "_QFf3Exa", scope: ![[CBAF3:[0-9]+]], file: !{{[0-9]+}}, line: [[@LINE-8]], type: ![[INT]]{{.*}})
+! CHECK-DAG: ![[EXPXAF3:[0-9]+]] = !DIGlobalVariableExpression(var: ![[XAF3]], expr: !DIExpression())
+
+program test
+  real(kind=4) :: v1, v2, v3, va1, va2, va3
+  common // v1, v2, v3
+  common /a/ va1, va2, va3
+  call f1()
+  call f2()
+  call f3()
+  print *, v1, va1, va3
+END
+! CHECK-DAG: ![[V1:[0-9]+]] = {{.*}}!DIGlobalVariable(name: "v1", linkageName: "_QFEv1", scope: ![[CBM:[0-9]+]], file: !{{[0-9]+}}, line: [[@LINE-8]], type: ![[REAL]]{{.*}})
+! CHECK-DAG: ![[EXPV1:[0-9]+]] = !DIGlobalVariableExpression(var: ![[V1]], expr: !DIExpression())
+! CHECK-DAG: ![[V2:[0-9]+]] = {{.*}}!DIGlobalVariable(name: "v2", linkageName: "_QFEv2", scope: ![[CBM]], file: !{{[0-9]+}}, line: [[@LINE-10]], type: ![[REAL]]{{.*}})
+! CHECK-DAG: ![[EXPV2:[0-9]+]] = !DIGlobalVariableExpression(var: ![[V2]], expr: !DIExpression(DW_OP_plus_uconst, 4))
+! CHECK-DAG: ![[V3:[0-9]+]] = {{.*}}!DIGlobalVariable(name: "v3", linkageName: "_QFEv3", scope: ![[CBM]], file: !{{[0-9]+}}, line: [[@LINE-12]], type: ![[REAL]]{{.*}})
+! CHECK-DAG: ![[EXPV3:[0-9]+]] = !DIGlobalVariableExpression(var: ![[V3]], expr: !DIExpression(DW_OP_plus_uconst, 8))
+! CHECK-DAG: ![[VA1:[0-9]+]] = {{.*}}!DIGlobalVariable(name: "va1", linkageName: "_QFEva1", scope: ![[CBAM:[0-9]+]], file: !{{[0-9]+}}, line: [[@LINE-14]], type: ![[REAL]]{{.*}})
+! CHECK-DAG: ![[EXPVA1:[0-9]+]] = !DIGlobalVariableExpression(var: ![[VA1]], expr: !DIExpression())
+! CHECK-DAG: ![[VA2:[0-9]+]] = {{.*}}!DIGlobalVariable(name: "va2", linkageName: "_QFEva2", scope: ![[CBAM]], file: !{{[0-9]+}}, line: [[@LINE-16]], type: ![[REAL]]{{.*}})
+! CHECK-DAG: ![[EXPVA2:[0-9]+]] = !DIGlobalVariableExpression(var: ![[VA2]], expr: !DIExpression(DW_OP_plus_uconst, 4))
+! CHECK-DAG: ![[VA3:[0-9]+]] = {{.*}}!DIGlobalVariable(name: "va3", linkageName: "_QFEva3", scope: ![[CBAM]], file: !{{[0-9]+}}, line: [[@LINE-18]], type: ![[REAL]]{{.*}})
+! CHECK-DAG: ![[EXPVA3:[0-9]+]] = !DIGlobalVariableExpression(var: ![[VA3]], expr: !DIExpression(DW_OP_plus_uconst, 8))
+
+
+! CHECK-DAG: ![[REAL]] = !DIBasicType(name: "real", size: 32, encoding: DW_ATE_float)
+! CHECK-DAG: ![[INT]] = !DIBasicType(name: "integer", size: 32, encoding: DW_ATE_signed)
+
+! CHECK-DAG: ![[F1:[0-9]+]] = {{.*}}!DISubprogram(name: "f1"{{.*}})
+! CHECK-DAG: ![[CBF1]] = !DICommonBlock(scope: ![[F1]], declaration: null, name: "__BLNK__"{{.*}})
+! CHECK-DAG: ![[CBAF1]] = !DICommonBlock(scope: ![[F1]], declaration: null, name: "a"{{.*}})
+
+! CHECK-DAG: ![[F2:[0-9]+]] = {{.*}}!DISubprogram(name: "f2"{{.*}})
+! CHECK-DAG: ![[CBF2]] = !DICommonBlock(scope: ![[F2]], declaration: null, name: "__BLNK__"{{.*}})
+! CHECK-DAG: ![[CBAF2]] = !DICommonBlock(scope: ![[F2]], declaration: null, name: "a"{{.*}})
+
+! CHECK-DAG: ![[F3:[0-9]+]] = {{.*}}!DISubprogram(name: "f3"{{.*}})
+! CHECK-DAG: ![[CBF3]] = !DICommonBlock(scope: ![[F3]], declaration: null, name: "__BLNK__"{{.*}})
+! CHECK-DAG: ![[CBAF3]] = !DICommonBlock(scope: ![[F3]], declaration: null, name: "a"{{.*}})
+
+! CHECK-DAG: ![[MAIN:[0-9]+]] = {{.*}}!DISubprogram(name: "test"{{.*}})
+! CHECK-DAG: ![[CBM]] = !DICommonBlock(scope: ![[MAIN]], declaration: null, name: "__BLNK__"{{.*}})
+! CHECK-DAG: ![[CBAM]] = !DICommonBlock(scope: ![[MAIN]], declaration: null, name: "a"{{.*}})
+
+! Using CHECK-DAG-SAME so that we are not dependent on order of variable in these lists.
+! CHECK-DAG: @__BLNK__ = global{{.*}}
+! CHECK-DAG-SAME: !dbg ![[EXPXF1]]
+! CHECK-DAG-SAME: !dbg ![[EXPYF1]]
+! CHECK-DAG-SAME: !dbg ![[EXPXF2]]
+! CHECK-DAG-SAME: !dbg ![[EXPYF2]]
+! CHECK-DAG-SAME: !dbg ![[EXPZF2]]
+! CHECK-DAG-SAME: !dbg ![[EXPXF3]]
+! CHECK-DAG-SAME: !dbg ![[EXPV1]]
+! CHECK-DAG-SAME: !dbg ![[EXPV2]]
+! CHECK-DAG-SAME: !dbg ![[EXPV3]]
+
+! CHECK-DAG: @a_ = global{{.*}}
+! CHECK-DAG-SAME: !dbg ![[EXPXAF1]]
+! CHECK-DAG-SAME: !dbg ![[EXPYAF1]]
+! CHECK-DAG-SAME: !dbg ![[EXPXAF2]]
+! CHECK-DAG-SAME: !dbg ![[EXPYAF2]]
+! CHECK-DAG-SAME: !dbg ![[EXPZAF2]]
+! CHECK-DAG-SAME: !dbg ![[EXPXAF3]]
+! CHECK-DAG-SAME: !dbg ![[EXPVA1]]
+! CHECK-DAG-SAME: !dbg ![[EXPVA2]]
+! CHECK-DAG-SAME: !dbg ![[EXPVA3]]
+
+! CHECK-DAG: !DICompileUnit({{.*}}, globals: ![[GLOBALS:[0-9]+]])
+! CHECK-DAG: ![[GLOBALS]]
+! CHECK-DAG-SAME: ![[EXPXF1]]
+! CHECK-DAG-SAME: ![[EXPYF1]]
+! CHECK-DAG-SAME: ![[EXPXAF1]]
+! CHECK-DAG-SAME: ![[EXPYAF1]]
+! CHECK-DAG-SAME: ![[EXPXF2]]
+! CHECK-DAG-SAME: ![[EXPYF2]]
+! CHECK-DAG-SAME: ![[EXPZF2]]
+! CHECK-DAG-SAME: ![[EXPXAF2]]
+! CHECK-DAG-SAME: ![[EXPYAF2]]
+! CHECK-DAG-SAME: ![[EXPZAF2]]
+! CHECK-DAG-SAME: ![[EXPXF3]]
+! CHECK-DAG-SAME: ![[EXPXAF3]]
+! CHECK-DAG-SAME: ![[EXPV1]]
+! CHECK-DAG-SAME: ![[EXPV2]]
+! CHECK-DAG-SAME: ![[EXPV3]]
+! CHECK-DAG-SAME: ![[EXPVA1]]
+! CHECK-DAG-SAME: ![[EXPVA2]]
+! CHECK-DAG-SAME: ![[EXPVA3]]
diff --git a/flang/test/Integration/debug-cyclic-derived-type-3.f90 b/flang/test/Integration/debug-cyclic-derived-type-3.f90
index ef9aed13cc514..d91c635576e26 100644
--- a/flang/test/Integration/debug-cyclic-derived-type-3.f90
+++ b/flang/test/Integration/debug-cyclic-derived-type-3.f90
@@ -1,4 +1,4 @@
-! RUN: %flang_fc1 -emit-llvm -debug-info-kind=standalone %s -o -
+! RUN: %flang_fc1 -emit-llvm -debug-info-kind=standalone %s -o - | FileCheck %s
 
 ! mainly test that this program does not cause an assertion failure
 ! testcase for issue 122024
@@ -17,7 +17,7 @@ module m1
 
 program test
   use m1
-  type(t1),pointer :: foo
+  type(t1),pointer :: foo, foo2
   allocate(foo)
   allocate(foo%x1)
   allocate(foo%x1%x2)
@@ -30,3 +30,7 @@ subroutine sub1(bar)
   use m1
   type(t2) :: bar
 end subroutine
+
+! Test that file compiles ok and there is only one DICompositeType for "t1".
+!CHECK: !DICompositeType(tag: DW_TAG_structure_type, name: "t1"{{.*}})
+!CHECK-NOT: !DICompositeType(tag: DW_TAG_structure_type, name: "t1"{{.*}})
diff --git a/flang/test/Integration/debug-local-var-2.f90 b/flang/test/Integration/debug-local-var-2.f90
index fe4144a3dd46e..20f9a9c622890 100644
--- a/flang/test/Integration/debug-local-var-2.f90
+++ b/flang/test/Integration/debug-local-var-2.f90
@@ -27,7 +27,7 @@
 ! BOTH-LABEL: }
 
 ! BOTH-LABEL: define {{.*}}i64 @_QFPfn1
-! BOTH-SAME: (ptr nocapture %[[ARG1:.*]], ptr nocapture %[[ARG2:.*]], ptr nocapture %[[ARG3:.*]])
+! BOTH-SAME: (ptr captures(none) %[[ARG1:.*]], ptr captures(none) %[[ARG2:.*]], ptr captures(none) %[[ARG3:.*]])
 ! INTRINSICS-DAG: call void @llvm.dbg.declare(metadata ptr %[[ARG1]], metadata ![[A1:.*]], metadata !DIExpression())
 ! INTRINSICS-DAG: call void @llvm.dbg.declare(metadata ptr %[[ARG2]], metadata ![[B1:.*]], metadata !DIExpression())
 ! INTRINSICS-DAG: call void @llvm.dbg.declare(metadata ptr %[[ARG3]], metadata ![[C1:.*]], metadata !DIExpression())
@@ -40,7 +40,7 @@
 ! BOTH-LABEL: }
 
 ! BOTH-LABEL: define {{.*}}i32 @_QFPfn2
-! BOTH-SAME: (ptr nocapture %[[FN2ARG1:.*]], ptr nocapture %[[FN2ARG2:.*]], ptr nocapture %[[FN2ARG3:.*]])
+! BOTH-SAME: (ptr captures(none) %[[FN2ARG1:.*]], ptr captures(none) %[[FN2ARG2:.*]], ptr captures(none) %[[FN2ARG3:.*]])
 ! INTRINSICS-DAG: call void @llvm.dbg.declare(metadata ptr %[[FN2ARG1]], metadata ![[A2:.*]], metadata !DIExpression())
 ! INTRINSICS-DAG: call void @llvm.dbg.declare(metadata ptr %[[FN2ARG2]], metadata ![[B2:.*]], metadata !DIExpression())
 ! INTRINSICS-DAG: call void @llvm.dbg.declare(metadata ptr %[[FN2ARG3]], metadata ![[C2:.*]], metadata !DIExpression())
diff --git a/flang/test/Integration/unroll-loops.f90 b/flang/test/Integration/unroll-loops.f90
index 4b4a394502881..c3fcf1c3a7cf3 100644
--- a/flang/test/Integration/unroll-loops.f90
+++ b/flang/test/Integration/unroll-loops.f90
@@ -1,13 +1,19 @@
-! RUN: %flang_fc1 -emit-llvm -O1 -funroll-loops -mllvm -force-vector-width=2 -o- %s | FileCheck %s --check-prefixes=CHECK,UNROLL
-! RUN: %flang_fc1 -emit-llvm -O2 -mllvm -force-vector-width=2 -o- %s | FileCheck %s --check-prefixes=CHECK,UNROLL
-! RUN: %flang_fc1 -emit-llvm -O1 -fno-unroll-loops -mllvm -force-vector-width=2 -o- %s | FileCheck %s --check-prefixes=CHECK,NO-UNROLL
-! RUN: %flang_fc1 -emit-llvm -O1 -mllvm -force-vector-width=2 -o- %s | FileCheck %s --check-prefixes=CHECK,NO-UNROLL
-
 ! FIXME: https://github.com/llvm/llvm-project/issues/123668
-! XFAIL: target=powerpc64{{.*}}
-
+!
+! DEFINE: %{triple} =
+! DEFINE: %{check-unroll} = %flang_fc1 -emit-llvm -O1 -funroll-loops -mllvm -force-vector-width=2 -triple %{triple} -o- %s | FileCheck %s --check-prefixes=CHECK,UNROLL
+! DEFINE: %{check-nounroll} = %flang_fc1 -emit-llvm -O1 -mllvm -force-vector-width=2 -triple %{triple} -o- %s | FileCheck %s --check-prefixes=CHECK,NO-UNROLL
+!
+! REDEFINE: %{triple} = aarch64-unknown-linux-gnu
+! RUN: %if aarch64-registered-target %{ %{check-unroll} %}
+! RUN: %if aarch64-registered-target %{ %{check-nounroll} %}
+!
+! REDEFINE: %{triple} = x86_64-unknown-linux-gnu
+! RUN: %if x86-registered-target %{ %{check-unroll} %}
+! RUN: %if x86-registered-target %{ %{check-nounroll} %}
+!
 ! CHECK-LABEL: @unroll
-! CHECK-SAME: (ptr nocapture writeonly %[[ARG0:.*]])
+! CHECK-SAME: (ptr writeonly captures(none) %[[ARG0:.*]])
 subroutine unroll(a)
   integer(kind=8), intent(out) :: a(1000)
   integer(kind=8) :: i
diff --git a/flang/test/Integration/unroll.f90 b/flang/test/Integration/unroll.f90
new file mode 100644
index 0000000000000..aa47e465b63fc
--- /dev/null
+++ b/flang/test/Integration/unroll.f90
@@ -0,0 +1,49 @@
+! RUN: %flang_fc1 -emit-llvm -o - %s | FileCheck %s
+
+! CHECK-LABEL: unroll_dir
+subroutine unroll_dir
+  integer :: a(10)
+  !dir$ unroll
+  ! CHECK:   br i1 {{.*}}, label {{.*}}, label {{.*}}, !llvm.loop ![[UNROLL_ENABLE_FULL_ANNO:.*]]
+  do i=1,10
+  a(i)=i
+  end do
+end subroutine unroll_dir
+
+! CHECK-LABEL: unroll_dir_0
+subroutine unroll_dir_0
+  integer :: a(10)
+  !dir$ unroll 0
+  ! CHECK:   br i1 {{.*}}, label {{.*}}, label {{.*}}, !llvm.loop ![[UNROLL_DISABLE_ANNO:.*]]
+  do i=1,10
+  a(i)=i
+  end do
+end subroutine unroll_dir_0
+
+! CHECK-LABEL: unroll_dir_1
+subroutine unroll_dir_1
+  integer :: a(10)
+  !dir$ unroll 1
+  ! CHECK:   br i1 {{.*}}, label {{.*}}, label {{.*}}, !llvm.loop ![[UNROLL_DISABLE_ANNO]]
+  do i=1,10
+  a(i)=i
+  end do
+end subroutine unroll_dir_1
+
+! CHECK-LABEL: unroll_dir_2
+subroutine unroll_dir_2
+  integer :: a(10)
+  !dir$ unroll 2
+  ! CHECK:   br i1 {{.*}}, label {{.*}}, label {{.*}}, !llvm.loop ![[UNROLL_ENABLE_COUNT_2:.*]]
+  do i=1,10
+  a(i)=i
+  end do
+end subroutine unroll_dir_2
+
+! CHECK: ![[UNROLL_ENABLE_FULL_ANNO]] = distinct !{![[UNROLL_ENABLE_FULL_ANNO]], ![[UNROLL_ENABLE:.*]], ![[UNROLL_FULL:.*]]}
+! CHECK: ![[UNROLL_ENABLE:.*]] = !{!"llvm.loop.unroll.enable"}
+! CHECK: ![[UNROLL_FULL:.*]] = !{!"llvm.loop.unroll.full"}
+! CHECK: ![[UNROLL_DISABLE_ANNO]] = distinct !{![[UNROLL_DISABLE_ANNO]], ![[UNROLL_DISABLE:.*]]}
+! CHECK: ![[UNROLL_DISABLE]] = !{!"llvm.loop.unroll.disable"}
+! CHECK: ![[UNROLL_ENABLE_COUNT_2]] = distinct !{![[UNROLL_ENABLE_COUNT_2]], ![[UNROLL_ENABLE]], ![[UNROLL_COUNT_2:.*]]}
+! CHECK: ![[UNROLL_COUNT_2]] = !{!"llvm.loop.unroll.count", i32 2}
diff --git a/flang/test/Intrinsics/math-codegen.fir b/flang/test/Intrinsics/math-codegen.fir
index 2165a13b10de2..c45c6b23e897e 100644
--- a/flang/test/Intrinsics/math-codegen.fir
+++ b/flang/test/Intrinsics/math-codegen.fir
@@ -1803,10 +1803,10 @@ func.func private @sinh(f64) -> f64
 //--- tanh_fast.fir
 // RUN: fir-opt %t/tanh_fast.fir --fir-to-llvm-ir="target=x86_64-unknown-linux-gnu" | FileCheck %t/tanh_fast.fir
 // CHECK: @_QPtest_real4
-// CHECK: {{%[A-Za-z0-9._]+}} = llvm.call @tanhf({{%[A-Za-z0-9._]+}}) : (f32) -> f32
+// CHECK: {{%[A-Za-z0-9._]+}} = llvm.intr.tanh({{%[A-Za-z0-9._]+}}) : (f32) -> f32
 
 // CHECK: @_QPtest_real8
-// CHECK: {{%[A-Za-z0-9._]+}} = llvm.call @tanh({{%[A-Za-z0-9._]+}}) : (f64) -> f64
+// CHECK: {{%[A-Za-z0-9._]+}} = llvm.intr.tanh({{%[A-Za-z0-9._]+}}) : (f64) -> f64
 
 func.func @_QPtest_real4(%arg0: !fir.ref<f32> {fir.bindc_name = "x"}) -> f32 {
   %0 = fir.alloca f32 {bindc_name = "test_real4", uniq_name = "_QFtest_real4Etest_real4"}
@@ -1828,10 +1828,10 @@ func.func @_QPtest_real8(%arg0: !fir.ref<f64> {fir.bindc_name = "x"}) -> f64 {
 //--- tanh_relaxed.fir
 // RUN: fir-opt %t/tanh_relaxed.fir --fir-to-llvm-ir="target=x86_64-unknown-linux-gnu" | FileCheck %t/tanh_relaxed.fir
 // CHECK: @_QPtest_real4
-// CHECK: {{%[A-Za-z0-9._]+}} = llvm.call @tanhf({{%[A-Za-z0-9._]+}}) : (f32) -> f32
+// CHECK: {{%[A-Za-z0-9._]+}} = llvm.intr.tanh({{%[A-Za-z0-9._]+}}) : (f32) -> f32
 
 // CHECK: @_QPtest_real8
-// CHECK: {{%[A-Za-z0-9._]+}} = llvm.call @tanh({{%[A-Za-z0-9._]+}}) : (f64) -> f64
+// CHECK: {{%[A-Za-z0-9._]+}} = llvm.intr.tanh({{%[A-Za-z0-9._]+}}) : (f64) -> f64
 
 func.func @_QPtest_real4(%arg0: !fir.ref<f32> {fir.bindc_name = "x"}) -> f32 {
   %0 = fir.alloca f32 {bindc_name = "test_real4", uniq_name = "_QFtest_real4Etest_real4"}
@@ -1880,10 +1880,10 @@ func.func private @tanh(f64) -> f64
 //--- tan_fast.fir
 // RUN: fir-opt %t/tan_fast.fir --fir-to-llvm-ir="target=x86_64-unknown-linux-gnu" | FileCheck %t/tan_fast.fir
 // CHECK: @_QPtest_real4
-// CHECK: {{%[A-Za-z0-9._]+}} = llvm.call @tanf({{%[A-Za-z0-9._]+}}) : (f32) -> f32
+// CHECK: {{%[A-Za-z0-9._]+}} = llvm.intr.tan({{%[A-Za-z0-9._]+}}) : (f32) -> f32
 
 // CHECK: @_QPtest_real8
-// CHECK: {{%[A-Za-z0-9._]+}} = llvm.call @tan({{%[A-Za-z0-9._]+}}) : (f64) -> f64
+// CHECK: {{%[A-Za-z0-9._]+}} = llvm.intr.tan({{%[A-Za-z0-9._]+}}) : (f64) -> f64
 
 func.func @_QPtest_real4(%arg0: !fir.ref<f32> {fir.bindc_name = "x"}) -> f32 {
   %0 = fir.alloca f32 {bindc_name = "test_real4", uniq_name = "_QFtest_real4Etest_real4"}
@@ -1905,10 +1905,10 @@ func.func @_QPtest_real8(%arg0: !fir.ref<f64> {fir.bindc_name = "x"}) -> f64 {
 //--- tan_relaxed.fir
 // RUN: fir-opt %t/tan_relaxed.fir --fir-to-llvm-ir="target=x86_64-unknown-linux-gnu" | FileCheck %t/tan_relaxed.fir
 // CHECK: @_QPtest_real4
-// CHECK: {{%[A-Za-z0-9._]+}} = llvm.call @tanf({{%[A-Za-z0-9._]+}}) : (f32) -> f32
+// CHECK: {{%[A-Za-z0-9._]+}} = llvm.intr.tan({{%[A-Za-z0-9._]+}}) : (f32) -> f32
 
 // CHECK: @_QPtest_real8
-// CHECK: {{%[A-Za-z0-9._]+}} = llvm.call @tan({{%[A-Za-z0-9._]+}}) : (f64) -> f64
+// CHECK: {{%[A-Za-z0-9._]+}} = llvm.intr.tan({{%[A-Za-z0-9._]+}}) : (f64) -> f64
 
 func.func @_QPtest_real4(%arg0: !fir.ref<f32> {fir.bindc_name = "x"}) -> f32 {
   %0 = fir.alloca f32 {bindc_name = "test_real4", uniq_name = "_QFtest_real4Etest_real4"}
diff --git a/flang/test/Lower/CUDA/cuda-allocatable.cuf b/flang/test/Lower/CUDA/cuda-allocatable.cuf
index ed78bec1b8f08..4aa86165252a7 100644
--- a/flang/test/Lower/CUDA/cuda-allocatable.cuf
+++ b/flang/test/Lower/CUDA/cuda-allocatable.cuf
@@ -81,7 +81,7 @@ end subroutine
 ! CHECK: %[[BOX_DECL:.*]]:2 = hlfir.declare %[[BOX]] {data_attr = #cuf.cuda<pinned>, fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "_QFsub3Ea"} : (!fir.ref<!fir.box<!fir.heap<!fir.array<?x?xi32>>>>) -> (!fir.ref<!fir.box<!fir.heap<!fir.array<?x?xi32>>>>, !fir.ref<!fir.box<!fir.heap<!fir.array<?x?xi32>>>>)
 ! CHECK: %[[PLOG:.*]] = fir.alloca !fir.logical<4> {bindc_name = "plog", uniq_name = "_QFsub3Eplog"}
 ! CHECK: %[[PLOG_DECL:.*]]:2 = hlfir.declare %{{.*}} {uniq_name = "_QFsub3Eplog"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
-! CHECK-2: fir.call @_FortranAAllocatableSetBounds
+! CHECK-COUNT-2: fir.call @_FortranAAllocatableSetBounds
 ! CHECK: %{{.*}} = cuf.allocate %[[BOX_DECL]]#1 : !fir.ref<!fir.box<!fir.heap<!fir.array<?x?xi32>>>> pinned(%[[PLOG_DECL]]#1 : !fir.ref<!fir.logical<4>>) {data_attr = #cuf.cuda<pinned>} -> i32
 ! CHECK: fir.if %{{.*}} {
 ! CHECK:   %{{.*}} = cuf.deallocate %[[BOX_DECL]]#1 : !fir.ref<!fir.box<!fir.heap<!fir.array<?x?xi32>>>> {data_attr = #cuf.cuda<pinned>} -> i32
diff --git a/flang/test/Lower/CUDA/cuda-device-proc.cuf b/flang/test/Lower/CUDA/cuda-device-proc.cuf
index 661e5728bf85b..17a6a1d965640 100644
--- a/flang/test/Lower/CUDA/cuda-device-proc.cuf
+++ b/flang/test/Lower/CUDA/cuda-device-proc.cuf
@@ -23,11 +23,31 @@ attributes(global) subroutine devsub()
   al = atomicadd(al, 1_8)
   af = atomicadd(af, 1.0_4)
   ad = atomicadd(ad, 1.0_8)
+  
+  ai = atomicsub(ai, 1_4)
+  al = atomicsub(al, 1_8)
+  af = atomicsub(af, 1.0_4)
+  ad = atomicsub(ad, 1.0_8)
+  
+  ai = atomicmax(ai, 1_4)
+  al = atomicmax(al, 1_8)
+  af = atomicmax(af, 1.0_4)
+  ad = atomicmax(ad, 1.0_8)
+  
+  ai = atomicmin(ai, 1_4)
+  al = atomicmin(al, 1_8)
+  af = atomicmin(af, 1.0_4)
+  ad = atomicmin(ad, 1.0_8)
+  
+  ai = atomicand(ai, 1_4)
+  ai = atomicor(ai, 1_4)
+  ai = atomicinc(ai, 1_4)
+  ai = atomicdec(ai, 1_4)
 end
 
 ! CHECK-LABEL: func.func @_QPdevsub() attributes {cuf.proc_attr = #cuf.cuda_proc<global>}
 ! CHECK: fir.call @llvm.nvvm.barrier0() fastmath<contract> : () -> ()
-! CHECK: fir.call @__syncwarp(%{{.*}}) proc_attrs<bind_c> fastmath<contract> : (!fir.ref<i32>) -> ()
+! CHECK: fir.call @llvm.nvvm.bar.warp.sync(%c1{{.*}}) fastmath<contract> : (i32) -> ()
 ! CHECK: fir.call @llvm.nvvm.membar.gl() fastmath<contract> : () -> ()
 ! CHECK: fir.call @llvm.nvvm.membar.cta() fastmath<contract> : () -> ()
 ! CHECK: fir.call @llvm.nvvm.membar.sys() fastmath<contract> : () -> ()
@@ -39,8 +59,56 @@ end
 ! CHECK: %{{.*}} = llvm.atomicrmw fadd %{{.*}}, %{{.*}} seq_cst : !llvm.ptr, f32
 ! CHECK: %{{.*}} = llvm.atomicrmw fadd %{{.*}}, %{{.*}} seq_cst : !llvm.ptr, f64
 
+! CHECK: %{{.*}} = llvm.atomicrmw sub  %{{.*}}, %{{.*}} seq_cst : !llvm.ptr, i32
+! CHECK: %{{.*}} = llvm.atomicrmw sub  %{{.*}}, %{{.*}} seq_cst : !llvm.ptr, i64
+! CHECK: %{{.*}} = llvm.atomicrmw fsub %{{.*}}, %{{.*}} seq_cst : !llvm.ptr, f32
+! CHECK: %{{.*}} = llvm.atomicrmw fsub %{{.*}}, %{{.*}} seq_cst : !llvm.ptr, f64
+
+! CHECK: %{{.*}} = llvm.atomicrmw max  %{{.*}}, %{{.*}} seq_cst : !llvm.ptr, i32
+! CHECK: %{{.*}} = llvm.atomicrmw max  %{{.*}}, %{{.*}} seq_cst : !llvm.ptr, i64
+! CHECK: %{{.*}} = llvm.atomicrmw fmax %{{.*}}, %{{.*}} seq_cst : !llvm.ptr, f32
+! CHECK: %{{.*}} = llvm.atomicrmw fmax %{{.*}}, %{{.*}} seq_cst : !llvm.ptr, f64
+
+! CHECK: %{{.*}} = llvm.atomicrmw min  %{{.*}}, %{{.*}} seq_cst : !llvm.ptr, i32
+! CHECK: %{{.*}} = llvm.atomicrmw min  %{{.*}}, %{{.*}} seq_cst : !llvm.ptr, i64
+! CHECK: %{{.*}} = llvm.atomicrmw fmin %{{.*}}, %{{.*}} seq_cst : !llvm.ptr, f32
+! CHECK: %{{.*}} = llvm.atomicrmw fmin %{{.*}}, %{{.*}} seq_cst : !llvm.ptr, f64
+
+! CHECK: %{{.*}} = llvm.atomicrmw _and  %{{.*}}, %{{.*}} seq_cst : !llvm.ptr, i32
+! CHECK: %{{.*}} = llvm.atomicrmw _or  %{{.*}}, %{{.*}} seq_cst : !llvm.ptr, i32
+! CHECK: %{{.*}} = llvm.atomicrmw uinc_wrap  %{{.*}}, %{{.*}} seq_cst : !llvm.ptr, i32
+! CHECK: %{{.*}} = llvm.atomicrmw udec_wrap  %{{.*}}, %{{.*}} seq_cst : !llvm.ptr, i32
+
+subroutine host1()
+  integer, device :: a(32)
+  integer, device :: ret
+  integer :: i, j
+
+block; use cudadevice
+  !$cuf kernel do(1) <<<*,32>>>
+  do i = 1, 32
+    a(i) = a(i) * 2.0
+    call syncthreads()
+    a(i) = a(i) + a(j) - 34.0
+
+    call syncwarp(1)
+    ret = syncthreads_and(1)
+    ret = syncthreads_count(1)
+    ret = syncthreads_or(1)
+  end do
+end block
+end 
+
+! CHECK-LABEL: func.func @_QPhost1()
+! CHECK: cuf.kernel
+! CHECK: fir.call @llvm.nvvm.barrier0() fastmath<contract> : () -> ()
+! CHECK: fir.call @llvm.nvvm.bar.warp.sync(%c1{{.*}}) fastmath<contract> : (i32) -> ()
+! CHECK: fir.call @llvm.nvvm.barrier0.and(%c1{{.*}}) fastmath<contract> : (i32) -> i32
+! CHECK: fir.call @llvm.nvvm.barrier0.popc(%c1{{.*}}) fastmath<contract> : (i32) -> i32
+! CHECK: fir.call @llvm.nvvm.barrier0.or(%c1{{.*}}) fastmath<contract> : (i32) -> i32
+
 ! CHECK: func.func private @llvm.nvvm.barrier0()
-! CHECK: func.func private @__syncwarp(!fir.ref<i32> {cuf.data_attr = #cuf.cuda<device>}) attributes {cuf.proc_attr = #cuf.cuda_proc<device>, fir.bindc_name = "__syncwarp", fir.proc_attrs = #fir.proc_attrs<bind_c>}
+! CHECK: func.func private @llvm.nvvm.bar.warp.sync(i32)
 ! CHECK: func.func private @llvm.nvvm.membar.gl()
 ! CHECK: func.func private @llvm.nvvm.membar.cta()
 ! CHECK: func.func private @llvm.nvvm.membar.sys()
diff --git a/flang/test/Lower/CUDA/cuda-devptr.cuf b/flang/test/Lower/CUDA/cuda-devptr.cuf
index d61d84d9bc750..0a9087cf6c133 100644
--- a/flang/test/Lower/CUDA/cuda-devptr.cuf
+++ b/flang/test/Lower/CUDA/cuda-devptr.cuf
@@ -4,6 +4,12 @@
 
 module cudafct
   use __fortran_builtins, only : c_devptr => __builtin_c_devptr
+  
+  type :: t1
+    type(c_devptr) :: devp
+    integer :: a
+  end type
+
 contains
   function c_devloc(x)
     use iso_c_binding, only: c_loc
@@ -12,6 +18,10 @@ contains
     real, target, device :: x
     c_devloc%cptr = c_loc(x)
   end function
+
+  attributes(device) function get_t1()
+    type(t1) :: get_t1
+  end
 end
 
 subroutine sub1()
@@ -68,3 +78,12 @@ end subroutine
 ! CHECK: %[[P_ADDR_COORD:.*]] = fir.coordinate_of %[[P_CPTR_COORD]], %[[ADDRESS_FIELD]] : (!fir.ref<!fir.type<_QM__fortran_builtinsT__builtin_c_ptr{__address:i64}>>, !fir.field) -> !fir.ref<i64>
 ! CHECK: %[[ADDR:.*]] = fir.load %[[RES_ADDR_COORD]] : !fir.ref<i64>
 ! CHECK: fir.store %[[ADDR]] to %[[P_ADDR_COORD]] : !fir.ref<i64>
+
+attributes(global) subroutine assign_nested_c_devptr(p, a)
+  use cudafct
+  type(t1), device :: p
+  p = get_t1()
+end subroutine
+
+! CHECK-LABEL: func.func @_QPassign_nested_c_devptr
+! CHECK-NOT: fir.call @_FortranAAssign
diff --git a/flang/test/Lower/CUDA/cuda-init.cuf b/flang/test/Lower/CUDA/cuda-init.cuf
new file mode 100644
index 0000000000000..71ec1e3f9df56
--- /dev/null
+++ b/flang/test/Lower/CUDA/cuda-init.cuf
@@ -0,0 +1,11 @@
+! RUN: bbc -emit-fir -hlfir -fcuda %s -o - | FileCheck %s --check-prefixes=ALL,CUDA
+! RUN: bbc -emit-fir -hlfir %s -o - | FileCheck %s --check-prefixes=ALL,NOCUDA
+
+program test_init
+
+end 
+
+! ALL-LABEL: func.func @main
+! ALL: fir.call @_FortranAProgramStart
+! CUDA: fir.call @_FortranACUFInit() fastmath<contract> : () -> ()
+! NOCUDA-NOT: fir.call @_FortranACUFInit() fastmath<contract> : () -> ()
diff --git a/flang/test/Lower/CUDA/cuda-intrinsic.cuf b/flang/test/Lower/CUDA/cuda-intrinsic.cuf
index 9723afc532387..77682ad7139a7 100644
--- a/flang/test/Lower/CUDA/cuda-intrinsic.cuf
+++ b/flang/test/Lower/CUDA/cuda-intrinsic.cuf
@@ -1,17 +1,194 @@
 ! RUN: bbc -emit-hlfir -fcuda %s -o - | FileCheck %s
 
 module mod1
-  type int
-    real :: inf, sup 
-  end type int
+  type retf
+    real :: inf, sup, near, zero 
+  end type retf
+  type retd
+    real(8) :: inf, sup, near, zero 
+  end type retd
 contains
   attributes(global) subroutine fadd(c, a, b)
-    type (int) :: c, a, b
-    c%inf = __fadd_rd(a%inf, b%inf)
-    c%sup = __fadd_ru(a%sup, b%sup)
+    real(4) :: a, b
+    type (retf) :: c
+    c%near = __fadd_rn(a, b)
+    c%zero = __fadd_rz(a, b)
+    c%inf =  __fadd_rd(a, b)
+    c%sup =  __fadd_ru(a, b)
   end subroutine
+  attributes(global) subroutine dadd(c, a, b)
+    real(8) :: a, b
+    type (retd) :: c
+    c%near = __dadd_rn(a, b)
+    c%zero = __dadd_rz(a, b)
+    c%inf =  __dadd_rd(a, b)
+    c%sup =  __dadd_ru(a, b)
+  end subroutine
+  attributes(global) subroutine fmul(c, a, b)
+    real(4) :: a, b
+    type (retf) :: c
+    c%near = __fmul_rn(a, b)
+    c%zero = __fmul_rz(a, b)
+    c%inf  = __fmul_rd(a, b)
+    c%sup  = __fmul_ru(a, b)
+  end subroutine
+  attributes(global) subroutine dmul(c, a, b)
+    real(8) :: a, b
+    type (retf) :: c
+    c%near = __dmul_rn(a, b)
+    c%zero = __dmul_rz(a, b)
+    c%inf  = __dmul_rd(a, b)
+    c%sup  = __dmul_ru(a, b)
+  end subroutine
+  attributes(global) subroutine fmaf(c, a, b)
+    real(4) :: a, b
+    type (retf) :: c
+    c%near = __fmaf_rn(a, b, b)
+    c%zero = __fmaf_rz(a, b, b)
+    c%inf  = __fmaf_rd(a, b, b)
+    c%sup  = __fmaf_ru(a, b, b)
+  end subroutine
+  attributes(global) subroutine fma(c, a, b)
+    real(8) :: a, b
+    type (retd) :: c
+    c%near = __fma_rn(a, b, b)
+    c%zero = __fma_rz(a, b, b)
+    c%inf  = __fma_rd(a, b, b)
+    c%sup  = __fma_ru(a, b, b)
+  end subroutine
+  attributes(global) subroutine frcp(c,a)
+    real(4) :: a
+    type (retf) :: c
+    c%near = __frcp_rn(a)
+    c%zero = __frcp_rz(a)
+    c%inf  = __frcp_rd(a)
+    c%sup  = __frcp_ru(a)
+  end subroutine
+  attributes(global) subroutine fsqrt(c,a)
+    real(4) :: a
+    type (retf) :: c
+    c%near = __fsqrt_rn(a)
+    c%zero = __fsqrt_rz(a)
+    c%inf  = __fsqrt_rd(a)
+    c%sup  = __fsqrt_ru(a)
+  end subroutine
+  attributes(global) subroutine fdiv(c, a, b)
+    real(4) :: a, b
+    type (retf) :: c
+    c%near = __fdiv_rn(a, b)
+    c%zero = __fdiv_rz(a, b)
+    c%inf  = __fdiv_rd(a, b)
+    c%sup  = __fdiv_ru(a, b)
+  end subroutine
+  attributes(global) subroutine testsincosf(c, a, b)
+    real(4) :: a, b, c
+    call sincos(a, b, c)
+  end subroutine
+  attributes(global) subroutine testsincosd(c, a, b)
+    real(8) :: a, b, c
+    call sincos(a, b, c)
+  end subroutine
+  attributes(global) subroutine testsincospif(c, a, b)
+    real(4) :: a, b, c
+    call sincospi(a, b, c)
+  end subroutine
+  attributes(global) subroutine testsincospid(c, a, b)
+    real(8) :: a, b, c
+    call sincospi(a, b, c)
+  end subroutine
+  attributes(global) subroutine testmulhi(c, a, b)
+    integer(4) :: a, b, c
+    c = __mulhi(a, b)
+  end subroutine
+  attributes(global) subroutine testumulhi(c, a, b)
+    integer(4) :: a, b, c
+    c = __umulhi(a, b)
+  end subroutine
+  attributes(global) subroutine testmul64hi(c, a, b)
+    integer(8) :: a, b, c
+    c = __mul64hi(a, b)
+  end subroutine
+  attributes(global) subroutine testumul64hi(c, a, b)
+    integer(8) :: a, b, c
+    c = __umul64hi(a, b)
+  end subroutine
+
 end
 
 ! CHECK-LABEL: func.func @_QMmod1Pfadd
+! CHECK: fir.call @__nv_fadd_rn
+! CHECK: fir.call @__nv_fadd_rz
 ! CHECK: fir.call @__nv_fadd_rd
 ! CHECK: fir.call @__nv_fadd_ru
+
+! CHECK-LABEL: func.func @_QMmod1Pdadd
+! CHECK: fir.call @__nv_dadd_rn
+! CHECK: fir.call @__nv_dadd_rz
+! CHECK: fir.call @__nv_dadd_rd
+! CHECK: fir.call @__nv_dadd_ru
+
+! CHECK-LABEL: func.func @_QMmod1Pfmul
+! CHECK: fir.call @__nv_fmul_rn
+! CHECK: fir.call @__nv_fmul_rz
+! CHECK: fir.call @__nv_fmul_rd
+! CHECK: fir.call @__nv_fmul_ru
+
+! CHECK-LABEL: func.func @_QMmod1Pdmul
+! CHECK: fir.call @__nv_dmul_rn
+! CHECK: fir.call @__nv_dmul_rz
+! CHECK: fir.call @__nv_dmul_rd
+! CHECK: fir.call @__nv_dmul_ru
+
+! CHECK-LABEL: func.func @_QMmod1Pfmaf
+! CHECK: fir.call @__nv_fmaf_rn
+! CHECK: fir.call @__nv_fmaf_rz
+! CHECK: fir.call @__nv_fmaf_rd
+! CHECK: fir.call @__nv_fmaf_ru
+
+! CHECK-LABEL: func.func @_QMmod1Pfma
+! CHECK: fir.call @__nv_fma_rn
+! CHECK: fir.call @__nv_fma_rz
+! CHECK: fir.call @__nv_fma_rd
+! CHECK: fir.call @__nv_fma_ru
+
+! CHECK-LABEL: func.func @_QMmod1Pfrcp
+! CHECK: fir.call @__nv_frcp_rn
+! CHECK: fir.call @__nv_frcp_rz
+! CHECK: fir.call @__nv_frcp_rd
+! CHECK: fir.call @__nv_frcp_ru
+
+! CHECK-LABEL: func.func @_QMmod1Pfsqrt
+! CHECK: fir.call @__nv_fsqrt_rn
+! CHECK: fir.call @__nv_fsqrt_rz
+! CHECK: fir.call @__nv_fsqrt_rd
+! CHECK: fir.call @__nv_fsqrt_ru
+
+! CHECK-LABEL: func.func @_QMmod1Pfdiv
+! CHECK: fir.call @__nv_fdiv_rn
+! CHECK: fir.call @__nv_fdiv_rz
+! CHECK: fir.call @__nv_fdiv_rd
+! CHECK: fir.call @__nv_fdiv_ru
+
+! CHECK-LABEL: func.func @_QMmod1Ptestsincosf
+! CHECK: fir.call @__nv_sincosf
+
+! CHECK-LABEL: func.func @_QMmod1Ptestsincosd
+! CHECK: fir.call @__nv_sincos
+
+! CHECK-LABEL: func.func @_QMmod1Ptestsincospif
+! CHECK: fir.call @__nv_sincospif
+
+! CHECK-LABEL: func.func @_QMmod1Ptestsincospid
+! CHECK: fir.call @__nv_sincospi
+
+! CHECK-LABEL: func.func @_QMmod1Ptestmulhi
+! CHECK: fir.call @__nv_mulhi
+
+! CHECK-LABEL: func.func @_QMmod1Ptestumulhi
+! CHECK: fir.call @__nv_umulhi
+
+! CHECK-LABEL: func.func @_QMmod1Ptestmul64hi
+! CHECK: fir.call @__nv_mul64hi
+
+! CHECK-LABEL: func.func @_QMmod1Ptestumul64hi
+! CHECK: fir.call @__nv_umul64hi
diff --git a/flang/test/Lower/HLFIR/call-issue-124043.f90 b/flang/test/Lower/HLFIR/call-issue-124043.f90
new file mode 100644
index 0000000000000..441063c1d296e
--- /dev/null
+++ b/flang/test/Lower/HLFIR/call-issue-124043.f90
@@ -0,0 +1,15 @@
+! Reproducer for https://github.com/llvm/llvm-project/issues/124043 lowering
+! crash.
+! RUN: bbc -emit-hlfir -o - %s | FileCheck %s
+
+subroutine repro(a)
+  integer a(10)
+  associate (b => a(::2)+1)
+    call bar(b)
+  end associate
+end
+! CHECK-LABEL:   func.func @_QPrepro(
+! CHECK:           %[[VAL_11:.*]] = hlfir.elemental
+! CHECK:           %[[VAL_16:.*]]:3 = hlfir.associate %[[VAL_11]]
+! CHECK:           %[[VAL_18:.*]]:2 = hlfir.declare %[[VAL_16]]#1
+! CHECK:           fir.call @_QPbar(%[[VAL_18]]#1)
diff --git a/flang/test/Lower/HLFIR/calls-constant-expr-arg.f90 b/flang/test/Lower/HLFIR/calls-constant-expr-arg.f90
index 61e7ef959d33f..3e8e10ee83054 100644
--- a/flang/test/Lower/HLFIR/calls-constant-expr-arg.f90
+++ b/flang/test/Lower/HLFIR/calls-constant-expr-arg.f90
@@ -62,3 +62,31 @@ end subroutine test
 ! CHECK:           hlfir.end_associate %[[VAL_7]]#1, %[[VAL_7]]#2 : !fir.ref<i32>, i1
 ! CHECK:           return
 ! CHECK:         }
+
+subroutine test_associate(i)
+  interface
+   subroutine foo(x)
+     real :: x(:)
+   end subroutine
+  end interface
+  real, parameter :: p(*) = [1.,2.,3.,4.]
+  integer(8) :: i
+  associate(a => p(1:i))
+    associate(b => a(1:1:2))
+      call foo(b)
+    end associate
+  end associate
+end subroutine
+! CHECK-LABEL:   func.func @_QPtest_associate(
+! CHECK:           %[[VAL_3:.*]] = fir.address_of(@_QFtest_associateECp) : !fir.ref<!fir.array<4xf32>>
+! CHECK:           %[[VAL_6:.*]]:2 = hlfir.declare %[[VAL_3]](%{{.*}}) {fortran_attrs = #fir.var_attrs<parameter>, uniq_name = "_QFtest_associateECp"} : (!fir.ref<!fir.array<4xf32>>, !fir.shape<1>) -> (!fir.ref<!fir.array<4xf32>>, !fir.ref<!fir.array<4xf32>>)
+! CHECK:           %[[VAL_18:.*]] = hlfir.designate %[[VAL_6]]#0 {{.*}}
+! CHECK:           %[[VAL_19:.*]]:2 = hlfir.declare %[[VAL_18]] {{.*}}
+! CHECK:           %[[VAL_25:.*]] = hlfir.designate %[[VAL_19]]#0 {{.*}}
+! CHECK:           %[[VAL_26:.*]]:2 = hlfir.declare %[[VAL_25]] {uniq_name = "_QFtest_associateEb"} : (!fir.box<!fir.array<1xf32>>) -> (!fir.box<!fir.array<1xf32>>, !fir.box<!fir.array<1xf32>>)
+! CHECK:           %[[VAL_27:.*]] = hlfir.as_expr %[[VAL_26]]#0 : (!fir.box<!fir.array<1xf32>>) -> !hlfir.expr<1xf32>
+! CHECK:           %[[VAL_30:.*]]:3 = hlfir.associate %[[VAL_27]]({{.*}}) {adapt.valuebyref} : (!hlfir.expr<1xf32>, !fir.shape<1>) -> (!fir.ref<!fir.array<1xf32>>, !fir.ref<!fir.array<1xf32>>, i1)
+! CHECK:           %[[VAL_31:.*]] = fir.embox %[[VAL_30]]
+! CHECK:           %[[VAL_32:.*]] = fir.convert %[[VAL_31]] : (!fir.box<!fir.array<1xf32>>) -> !fir.box<!fir.array<?xf32>>
+! CHECK:           fir.call @_QPfoo(%[[VAL_32]]) {{.*}} : (!fir.box<!fir.array<?xf32>>) -> ()
+! CHECK:           hlfir.end_associate %[[VAL_30]]#1, %[[VAL_30]]#2 : !fir.ref<!fir.array<1xf32>>, i1
diff --git a/flang/test/Lower/HLFIR/calls-f77.f90 b/flang/test/Lower/HLFIR/calls-f77.f90
index fad32d5fd0278..b312d16c091ba 100644
--- a/flang/test/Lower/HLFIR/calls-f77.f90
+++ b/flang/test/Lower/HLFIR/calls-f77.f90
@@ -191,11 +191,11 @@ subroutine alternate_return_call(n1, n2, k)
 !     Test calls to user procedures with intrinsic interfaces
 ! -----------------------------------------------------------------------------
 
-! CHECK-NAME: func.func @_QPintrinsic_iface()
+! CHECK-LABEL: func.func @_QPintrinsic_iface()
 subroutine intrinsic_iface()
   intrinsic acos
   real :: x
   procedure(acos) :: proc
   x = proc(1.0)
 end subroutine
-! CHECK" fir.call @_QPproc(%{{.*}}) {{.*}}: (!fir.ref<f32>) -> f32
+! CHECK: fir.call @_QPproc(%{{.*}}) {{.*}}: (!fir.ref<f32>) -> f32
diff --git a/flang/test/Lower/HLFIR/convert-variable.f90 b/flang/test/Lower/HLFIR/convert-variable.f90
index 7acb1be578b93..07b91d0f34a07 100644
--- a/flang/test/Lower/HLFIR/convert-variable.f90
+++ b/flang/test/Lower/HLFIR/convert-variable.f90
@@ -1,5 +1,5 @@
 ! Test lowering of variables to fir.declare
-! RUN: bbc -emit-hlfir %s -o - | FileCheck %s
+! RUN: bbc -emit-hlfir %s -o - | FileCheck %s --check-prefixes=CHECK,%if flang-supports-f128-math %{F128%} %else %{F64%}
 
 subroutine scalar_numeric(x)
   integer :: x
@@ -68,13 +68,16 @@ subroutine scalar_numeric_attributes(x)
 ! CHECK:  %[[VAL_1:.*]] = hlfir.declare %[[VAL_0]] dummy_scope %{{[0-9]+}} {fortran_attrs = #fir.var_attrs<intent_in, optional, target>, uniq_name = "_QFscalar_numeric_attributesEx"} : (!fir.ref<i32>, !fir.dscope) -> (!fir.ref<i32>, !fir.ref<i32>)
 
 subroutine scalar_numeric_attributes_2(x)
-  real(16), value :: x(100)
+  integer, parameter :: rk = merge(16, 8, selected_real_kind(33, 4931)==16)
+  real(rk), value :: x(100)
 end subroutine
 ! CHECK-LABEL: func.func @_QPscalar_numeric_attributes_2(
-! CHECK-SAME:    %[[VAL_0:.*]]: !fir.ref<!fir.array<100xf128>>
+! F128-SAME:    %[[VAL_0:.*]]: !fir.ref<!fir.array<100xf128>>
+! F64-SAME:    %[[VAL_0:.*]]: !fir.ref<!fir.array<100xf64>>
 ! CHECK:  %[[VAL_1:.*]] = arith.constant 100 : index
 ! CHECK:  %[[VAL_2:.*]] = fir.shape %[[VAL_1]] : (index) -> !fir.shape<1>
-! CHECK:  %[[VAL_3:.*]] = hlfir.declare %[[VAL_0]](%[[VAL_2]]) dummy_scope %{{[0-9]+}} {fortran_attrs = #fir.var_attrs<value>, uniq_name = "_QFscalar_numeric_attributes_2Ex"} : (!fir.ref<!fir.array<100xf128>>, !fir.shape<1>, !fir.dscope) -> (!fir.ref<!fir.array<100xf128>>, !fir.ref<!fir.array<100xf128>>)
+! F128:  %[[VAL_3:.*]] = hlfir.declare %[[VAL_0]](%[[VAL_2]]) dummy_scope %{{[0-9]+}} {fortran_attrs = #fir.var_attrs<value>, uniq_name = "_QFscalar_numeric_attributes_2Ex"} : (!fir.ref<!fir.array<100xf128>>, !fir.shape<1>, !fir.dscope) -> (!fir.ref<!fir.array<100xf128>>, !fir.ref<!fir.array<100xf128>>)
+! F64:  %[[VAL_3:.*]] = hlfir.declare %[[VAL_0]](%[[VAL_2]]) dummy_scope %{{[0-9]+}} {fortran_attrs = #fir.var_attrs<value>, uniq_name = "_QFscalar_numeric_attributes_2Ex"} : (!fir.ref<!fir.array<100xf64>>, !fir.shape<1>, !fir.dscope) -> (!fir.ref<!fir.array<100xf64>>, !fir.ref<!fir.array<100xf64>>)
 
 subroutine scalar_numeric_attributes_3(x)
   real, intent(in) :: x
diff --git a/flang/test/Lower/HLFIR/maxloc.f90 b/flang/test/Lower/HLFIR/maxloc.f90
index 539affad2d7df..1d13f2fc6305b 100644
--- a/flang/test/Lower/HLFIR/maxloc.f90
+++ b/flang/test/Lower/HLFIR/maxloc.f90
@@ -86,7 +86,7 @@ subroutine maxloc5(s)
 ! CHECK-NEXT:    hlfir.assign %[[EXPR]] to %[[OUT]]#0 : !hlfir.expr<2xi32>, !fir.ref<!fir.array<2xi32>>
 ! CHECK-NEXT:    hlfir.destroy %[[EXPR]] : !hlfir.expr<2xi32>
 ! CHECK-NEXT:    return
-! CHECK-nEXT:  }
+! CHECK-NEXT:  }
 
 ! back argument as .true.
 subroutine maxloc_back(a, s)
diff --git a/flang/test/Lower/HLFIR/maxval.f90 b/flang/test/Lower/HLFIR/maxval.f90
index 32e1a80417a27..adc9ea8191beb 100644
--- a/flang/test/Lower/HLFIR/maxval.f90
+++ b/flang/test/Lower/HLFIR/maxval.f90
@@ -83,7 +83,7 @@ subroutine maxval5(s)
 ! CHECK-NEXT:    hlfir.assign %[[EXPR]] to %[[OUT]]#0 : !hlfir.expr<2xi32>, !fir.ref<!fir.array<2xi32>>
 ! CHECK-NEXT:    hlfir.destroy %[[EXPR]] : !hlfir.expr<2xi32>
 ! CHECK-NEXT:    return
-! CHECK-nEXT:  }
+! CHECK-NEXT:  }
 
 subroutine maxval6(a, s, d)
   integer, pointer :: d
diff --git a/flang/test/Lower/HLFIR/minloc.f90 b/flang/test/Lower/HLFIR/minloc.f90
index ce149ffcfb54f..3d33a9328238b 100644
--- a/flang/test/Lower/HLFIR/minloc.f90
+++ b/flang/test/Lower/HLFIR/minloc.f90
@@ -86,7 +86,7 @@ subroutine minloc5(s)
 ! CHECK-NEXT:    hlfir.assign %[[EXPR]] to %[[OUT]]#0 : !hlfir.expr<2xi32>, !fir.ref<!fir.array<2xi32>>
 ! CHECK-NEXT:    hlfir.destroy %[[EXPR]] : !hlfir.expr<2xi32>
 ! CHECK-NEXT:    return
-! CHECK-nEXT:  }
+! CHECK-NEXT:  }
 
 ! back argument as .true.
 subroutine minloc_back(a, s)
diff --git a/flang/test/Lower/HLFIR/minval.f90 b/flang/test/Lower/HLFIR/minval.f90
index 2ac9aba850b6f..d7e603ece0610 100644
--- a/flang/test/Lower/HLFIR/minval.f90
+++ b/flang/test/Lower/HLFIR/minval.f90
@@ -83,7 +83,7 @@ subroutine minval5(s)
 ! CHECK-NEXT:    hlfir.assign %[[EXPR]] to %[[OUT]]#0 : !hlfir.expr<2xi32>, !fir.ref<!fir.array<2xi32>>
 ! CHECK-NEXT:    hlfir.destroy %[[EXPR]] : !hlfir.expr<2xi32>
 ! CHECK-NEXT:    return
-! CHECK-nEXT:  }
+! CHECK-NEXT:  }
 
 subroutine minval6(a, s, d)
   integer, pointer :: d
diff --git a/flang/test/Lower/HLFIR/reshape.f90 b/flang/test/Lower/HLFIR/reshape.f90
new file mode 100644
index 0000000000000..91a08dfe514ec
--- /dev/null
+++ b/flang/test/Lower/HLFIR/reshape.f90
@@ -0,0 +1,143 @@
+! Test lowering of RESHAPE intrinsic to HLFIR
+! RUN: bbc -emit-hlfir %s -o - | FileCheck %s
+
+module types
+  type t
+  end type t
+end module types
+
+subroutine reshape_test(x, source, pd, sh, ord)
+  integer :: x(:,:)
+  integer :: source(:,:,:)
+  integer :: pd(:,:,:)
+  integer :: sh(2)
+  integer :: ord(2)
+  x = reshape(source, sh, pd, ord)
+end subroutine reshape_test
+! CHECK-LABEL:   func.func @_QPreshape_test(
+! CHECK:           %[[VAL_8:.*]]:2 = hlfir.declare {{.*}}{uniq_name = "_QFreshape_testEord"} : (!fir.ref<!fir.array<2xi32>>, !fir.shape<1>, !fir.dscope) -> (!fir.ref<!fir.array<2xi32>>, !fir.ref<!fir.array<2xi32>>)
+! CHECK:           %[[VAL_9:.*]]:2 = hlfir.declare {{.*}}{uniq_name = "_QFreshape_testEpd"} : (!fir.box<!fir.array<?x?x?xi32>>, !fir.dscope) -> (!fir.box<!fir.array<?x?x?xi32>>, !fir.box<!fir.array<?x?x?xi32>>)
+! CHECK:           %[[VAL_12:.*]]:2 = hlfir.declare {{.*}}{uniq_name = "_QFreshape_testEsh"} : (!fir.ref<!fir.array<2xi32>>, !fir.shape<1>, !fir.dscope) -> (!fir.ref<!fir.array<2xi32>>, !fir.ref<!fir.array<2xi32>>)
+! CHECK:           %[[VAL_13:.*]]:2 = hlfir.declare {{.*}}{uniq_name = "_QFreshape_testEsource"} : (!fir.box<!fir.array<?x?x?xi32>>, !fir.dscope) -> (!fir.box<!fir.array<?x?x?xi32>>, !fir.box<!fir.array<?x?x?xi32>>)
+! CHECK:           %[[VAL_14:.*]]:2 = hlfir.declare {{.*}}{uniq_name = "_QFreshape_testEx"} : (!fir.box<!fir.array<?x?xi32>>, !fir.dscope) -> (!fir.box<!fir.array<?x?xi32>>, !fir.box<!fir.array<?x?xi32>>)
+! CHECK:           %[[VAL_15:.*]] = hlfir.reshape %[[VAL_13]]#0 %[[VAL_12]]#0 pad %[[VAL_9]]#0 order %[[VAL_8]]#0 : (!fir.box<!fir.array<?x?x?xi32>>, !fir.ref<!fir.array<2xi32>>, !fir.box<!fir.array<?x?x?xi32>>, !fir.ref<!fir.array<2xi32>>) -> !hlfir.expr<?x?xi32>
+! CHECK:           hlfir.assign %[[VAL_15]] to %[[VAL_14]]#0 : !hlfir.expr<?x?xi32>, !fir.box<!fir.array<?x?xi32>>
+! CHECK:           hlfir.destroy %[[VAL_15]] : !hlfir.expr<?x?xi32>
+
+subroutine reshape_test_noorder(x, source, pd, sh)
+  integer :: x(:,:)
+  integer :: source(:,:,:)
+  integer :: pd(:,:,:)
+  integer :: sh(2)
+  x = reshape(source, sh, pd)
+end subroutine reshape_test_noorder
+! CHECK-LABEL:   func.func @_QPreshape_test_noorder(
+! CHECK:           %[[VAL_5:.*]]:2 = hlfir.declare {{.*}}{uniq_name = "_QFreshape_test_noorderEpd"} : (!fir.box<!fir.array<?x?x?xi32>>, !fir.dscope) -> (!fir.box<!fir.array<?x?x?xi32>>, !fir.box<!fir.array<?x?x?xi32>>)
+! CHECK:           %[[VAL_8:.*]]:2 = hlfir.declare {{.*}}{uniq_name = "_QFreshape_test_noorderEsh"} : (!fir.ref<!fir.array<2xi32>>, !fir.shape<1>, !fir.dscope) -> (!fir.ref<!fir.array<2xi32>>, !fir.ref<!fir.array<2xi32>>)
+! CHECK:           %[[VAL_9:.*]]:2 = hlfir.declare {{.*}}{uniq_name = "_QFreshape_test_noorderEsource"} : (!fir.box<!fir.array<?x?x?xi32>>, !fir.dscope) -> (!fir.box<!fir.array<?x?x?xi32>>, !fir.box<!fir.array<?x?x?xi32>>)
+! CHECK:           %[[VAL_11:.*]] = hlfir.reshape %[[VAL_9]]#0 %[[VAL_8]]#0 pad %[[VAL_5]]#0 : (!fir.box<!fir.array<?x?x?xi32>>, !fir.ref<!fir.array<2xi32>>, !fir.box<!fir.array<?x?x?xi32>>) -> !hlfir.expr<?x?xi32>
+
+subroutine reshape_test_nopad(x, source, sh, ord)
+  integer :: x(:,:)
+  integer :: source(:,:,:)
+  integer :: sh(2)
+  integer :: ord(2)
+  x = reshape(source, sh, ORDER=ord)
+end subroutine reshape_test_nopad
+! CHECK-LABEL:   func.func @_QPreshape_test_nopad(
+! CHECK:           %[[VAL_7:.*]]:2 = hlfir.declare {{.*}}{uniq_name = "_QFreshape_test_nopadEord"} : (!fir.ref<!fir.array<2xi32>>, !fir.shape<1>, !fir.dscope) -> (!fir.ref<!fir.array<2xi32>>, !fir.ref<!fir.array<2xi32>>)
+! CHECK:           %[[VAL_10:.*]]:2 = hlfir.declare {{.*}}{uniq_name = "_QFreshape_test_nopadEsh"} : (!fir.ref<!fir.array<2xi32>>, !fir.shape<1>, !fir.dscope) -> (!fir.ref<!fir.array<2xi32>>, !fir.ref<!fir.array<2xi32>>)
+! CHECK:           %[[VAL_11:.*]]:2 = hlfir.declare {{.*}}{uniq_name = "_QFreshape_test_nopadEsource"} : (!fir.box<!fir.array<?x?x?xi32>>, !fir.dscope) -> (!fir.box<!fir.array<?x?x?xi32>>, !fir.box<!fir.array<?x?x?xi32>>)
+! CHECK:           %[[VAL_13:.*]] = hlfir.reshape %[[VAL_11]]#0 %[[VAL_10]]#0 order %[[VAL_7]]#0 : (!fir.box<!fir.array<?x?x?xi32>>, !fir.ref<!fir.array<2xi32>>, !fir.ref<!fir.array<2xi32>>) -> !hlfir.expr<?x?xi32>
+  
+subroutine test_reshape_optional1(pad, order, source, shape)
+  real, pointer :: pad(:, :)
+  integer, pointer :: order(:)
+  real :: source(:, :, :)
+  integer :: shape(4)
+  print *, reshape(source=source, shape=shape, pad=pad, order=order)
+end subroutine test_reshape_optional1
+! CHECK-LABEL:   func.func @_QPtest_reshape_optional1(
+! CHECK:           %[[VAL_5:.*]]:2 = hlfir.declare {{.*}}{fortran_attrs = #fir.var_attrs<pointer>, uniq_name = "_QFtest_reshape_optional1Eorder"} : (!fir.ref<!fir.box<!fir.ptr<!fir.array<?xi32>>>>, !fir.dscope) -> (!fir.ref<!fir.box<!fir.ptr<!fir.array<?xi32>>>>, !fir.ref<!fir.box<!fir.ptr<!fir.array<?xi32>>>>)
+! CHECK:           %[[VAL_6:.*]]:2 = hlfir.declare {{.*}}{fortran_attrs = #fir.var_attrs<pointer>, uniq_name = "_QFtest_reshape_optional1Epad"} : (!fir.ref<!fir.box<!fir.ptr<!fir.array<?x?xf32>>>>, !fir.dscope) -> (!fir.ref<!fir.box<!fir.ptr<!fir.array<?x?xf32>>>>, !fir.ref<!fir.box<!fir.ptr<!fir.array<?x?xf32>>>>)
+! CHECK:           %[[VAL_9:.*]]:2 = hlfir.declare {{.*}}{uniq_name = "_QFtest_reshape_optional1Eshape"} : (!fir.ref<!fir.array<4xi32>>, !fir.shape<1>, !fir.dscope) -> (!fir.ref<!fir.array<4xi32>>, !fir.ref<!fir.array<4xi32>>)
+! CHECK:           %[[VAL_10:.*]]:2 = hlfir.declare {{.*}}{uniq_name = "_QFtest_reshape_optional1Esource"} : (!fir.box<!fir.array<?x?x?xf32>>, !fir.dscope) -> (!fir.box<!fir.array<?x?x?xf32>>, !fir.box<!fir.array<?x?x?xf32>>)
+! CHECK:           %[[VAL_16:.*]] = fir.load %[[VAL_6]]#1 : !fir.ref<!fir.box<!fir.ptr<!fir.array<?x?xf32>>>>
+! CHECK:           %[[VAL_17:.*]] = fir.box_addr %[[VAL_16]] : (!fir.box<!fir.ptr<!fir.array<?x?xf32>>>) -> !fir.ptr<!fir.array<?x?xf32>>
+! CHECK:           %[[VAL_18:.*]] = fir.convert %[[VAL_17]] : (!fir.ptr<!fir.array<?x?xf32>>) -> i64
+! CHECK:           %[[VAL_19:.*]] = arith.constant 0 : i64
+! CHECK:           %[[VAL_20:.*]] = arith.cmpi ne, %[[VAL_18]], %[[VAL_19]] : i64
+! CHECK:           %[[VAL_21:.*]] = fir.load %[[VAL_5]]#1 : !fir.ref<!fir.box<!fir.ptr<!fir.array<?xi32>>>>
+! CHECK:           %[[VAL_22:.*]] = fir.box_addr %[[VAL_21]] : (!fir.box<!fir.ptr<!fir.array<?xi32>>>) -> !fir.ptr<!fir.array<?xi32>>
+! CHECK:           %[[VAL_23:.*]] = fir.convert %[[VAL_22]] : (!fir.ptr<!fir.array<?xi32>>) -> i64
+! CHECK:           %[[VAL_24:.*]] = arith.constant 0 : i64
+! CHECK:           %[[VAL_25:.*]] = arith.cmpi ne, %[[VAL_23]], %[[VAL_24]] : i64
+! CHECK:           %[[VAL_26:.*]] = fir.load %[[VAL_6]]#1 : !fir.ref<!fir.box<!fir.ptr<!fir.array<?x?xf32>>>>
+! CHECK:           %[[VAL_27:.*]] = fir.absent !fir.box<!fir.ptr<!fir.array<?x?xf32>>>
+! CHECK:           %[[VAL_28:.*]] = arith.select %[[VAL_20]], %[[VAL_26]], %[[VAL_27]] : !fir.box<!fir.ptr<!fir.array<?x?xf32>>>
+! CHECK:           %[[VAL_29:.*]] = fir.load %[[VAL_5]]#1 : !fir.ref<!fir.box<!fir.ptr<!fir.array<?xi32>>>>
+! CHECK:           %[[VAL_30:.*]] = fir.absent !fir.box<!fir.ptr<!fir.array<?xi32>>>
+! CHECK:           %[[VAL_31:.*]] = arith.select %[[VAL_25]], %[[VAL_29]], %[[VAL_30]] : !fir.box<!fir.ptr<!fir.array<?xi32>>>
+! CHECK:           %[[VAL_32:.*]] = hlfir.reshape %[[VAL_10]]#0 %[[VAL_9]]#0 pad %[[VAL_28]] order %[[VAL_31]] : (!fir.box<!fir.array<?x?x?xf32>>, !fir.ref<!fir.array<4xi32>>, !fir.box<!fir.ptr<!fir.array<?x?xf32>>>, !fir.box<!fir.ptr<!fir.array<?xi32>>>) -> !hlfir.expr<?x?x?x?xf32>
+
+subroutine test_reshape_optional2(pad, order, source, shape)
+  real, optional :: pad(:, :)
+  integer, pointer, optional :: order(:)
+  real :: source(:, :, :)
+  integer :: shape(4)
+  print *, reshape(source=source, shape=shape, pad=pad, order=order)
+end subroutine test_reshape_optional2
+! CHECK-LABEL:   func.func @_QPtest_reshape_optional2(
+! CHECK:           %[[VAL_5:.*]]:2 = hlfir.declare {{.*}}{fortran_attrs = #fir.var_attrs<optional, pointer>, uniq_name = "_QFtest_reshape_optional2Eorder"} : (!fir.ref<!fir.box<!fir.ptr<!fir.array<?xi32>>>>, !fir.dscope) -> (!fir.ref<!fir.box<!fir.ptr<!fir.array<?xi32>>>>, !fir.ref<!fir.box<!fir.ptr<!fir.array<?xi32>>>>)
+! CHECK:           %[[VAL_6:.*]]:2 = hlfir.declare {{.*}}{fortran_attrs = #fir.var_attrs<optional>, uniq_name = "_QFtest_reshape_optional2Epad"} : (!fir.box<!fir.array<?x?xf32>>, !fir.dscope) -> (!fir.box<!fir.array<?x?xf32>>, !fir.box<!fir.array<?x?xf32>>)
+! CHECK:           %[[VAL_9:.*]]:2 = hlfir.declare {{.*}}{uniq_name = "_QFtest_reshape_optional2Eshape"} : (!fir.ref<!fir.array<4xi32>>, !fir.shape<1>, !fir.dscope) -> (!fir.ref<!fir.array<4xi32>>, !fir.ref<!fir.array<4xi32>>)
+! CHECK:           %[[VAL_10:.*]]:2 = hlfir.declare {{.*}}{uniq_name = "_QFtest_reshape_optional2Esource"} : (!fir.box<!fir.array<?x?x?xf32>>, !fir.dscope) -> (!fir.box<!fir.array<?x?x?xf32>>, !fir.box<!fir.array<?x?x?xf32>>)
+! CHECK:           %[[VAL_16:.*]] = fir.is_present %[[VAL_6]]#0 : (!fir.box<!fir.array<?x?xf32>>) -> i1
+! CHECK:           %[[VAL_17:.*]] = fir.load %[[VAL_5]]#1 : !fir.ref<!fir.box<!fir.ptr<!fir.array<?xi32>>>>
+! CHECK:           %[[VAL_18:.*]] = fir.box_addr %[[VAL_17]] : (!fir.box<!fir.ptr<!fir.array<?xi32>>>) -> !fir.ptr<!fir.array<?xi32>>
+! CHECK:           %[[VAL_19:.*]] = fir.convert %[[VAL_18]] : (!fir.ptr<!fir.array<?xi32>>) -> i64
+! CHECK:           %[[VAL_20:.*]] = arith.constant 0 : i64
+! CHECK:           %[[VAL_21:.*]] = arith.cmpi ne, %[[VAL_19]], %[[VAL_20]] : i64
+! CHECK:           %[[VAL_22:.*]] = fir.absent !fir.box<!fir.array<?x?xf32>>
+! CHECK:           %[[VAL_23:.*]] = arith.select %[[VAL_16]], %[[VAL_6]]#1, %[[VAL_22]] : !fir.box<!fir.array<?x?xf32>>
+! CHECK:           %[[VAL_24:.*]] = fir.load %[[VAL_5]]#1 : !fir.ref<!fir.box<!fir.ptr<!fir.array<?xi32>>>>
+! CHECK:           %[[VAL_25:.*]] = fir.absent !fir.box<!fir.ptr<!fir.array<?xi32>>>
+! CHECK:           %[[VAL_26:.*]] = arith.select %[[VAL_21]], %[[VAL_24]], %[[VAL_25]] : !fir.box<!fir.ptr<!fir.array<?xi32>>>
+! CHECK:           %[[VAL_27:.*]] = hlfir.reshape %[[VAL_10]]#0 %[[VAL_9]]#0 pad %[[VAL_23]] order %[[VAL_26]] : (!fir.box<!fir.array<?x?x?xf32>>, !fir.ref<!fir.array<4xi32>>, !fir.box<!fir.array<?x?xf32>>, !fir.box<!fir.ptr<!fir.array<?xi32>>>) -> !hlfir.expr<?x?x?x?xf32>
+
+subroutine test_reshape_shape_expr(source, shape)
+  integer :: source(:), shape(2)
+  print *, reshape(source, shape + 1)
+end subroutine test_reshape_shape_expr
+! CHECK-LABEL:   func.func @_QPtest_reshape_shape_expr(
+! CHECK:           %[[VAL_13:.*]] = hlfir.elemental
+! CHECK:           %[[VAL_18:.*]] = hlfir.reshape %{{.*}} %[[VAL_13]] : (!fir.box<!fir.array<?xi32>>, !hlfir.expr<2xi32>) -> !hlfir.expr<?x?xi32>
+
+subroutine test_reshape_polymorphic1(source, shape)
+  use types
+  class(t), allocatable :: source(:)
+  integer :: shape(1)
+  source = reshape(source, shape)
+end subroutine test_reshape_polymorphic1
+! CHECK-LABEL:   func.func @_QPtest_reshape_polymorphic1(
+! CHECK:           hlfir.reshape %{{.*}} %{{.*}} : (!fir.class<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt>>>>, !fir.ref<!fir.array<1xi32>>) -> !hlfir.expr<?x!fir.type<_QMtypesTt>?>
+
+subroutine test_reshape_polymorphic2(source, shape, pad)
+  use types
+  class(t), allocatable :: source(:)
+  type(t) :: pad(:)
+  integer :: shape(1)
+  source = reshape(source, shape, pad)
+end subroutine test_reshape_polymorphic2
+! CHECK-LABEL:   func.func @_QPtest_reshape_polymorphic2(
+! CHECK:           hlfir.reshape %{{.*}} %{{.*}} pad %{{.*}} : (!fir.class<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt>>>>, !fir.ref<!fir.array<1xi32>>, !fir.box<!fir.array<?x!fir.type<_QMtypesTt>>>) -> !hlfir.expr<?x!fir.type<_QMtypesTt>?>
+
+subroutine test_reshape_polymorphic3(source, shape, pad)
+  use types
+  type(t) :: source(:)
+  class(t) :: pad(:)
+  integer :: shape(1)
+  source = reshape(source, shape, pad)
+end subroutine test_reshape_polymorphic3
+! CHECK-LABEL:   func.func @_QPtest_reshape_polymorphic3(
+! CHECK:           hlfir.reshape %{{.*}} %{{.*}} pad %{{.*}} : (!fir.box<!fir.array<?x!fir.type<_QMtypesTt>>>, !fir.ref<!fir.array<1xi32>>, !fir.class<!fir.array<?x!fir.type<_QMtypesTt>>>) -> !hlfir.expr<?x!fir.type<_QMtypesTt>>
diff --git a/flang/test/Lower/HLFIR/sum.f90 b/flang/test/Lower/HLFIR/sum.f90
index 339582088b032..aed567f73b6b1 100644
--- a/flang/test/Lower/HLFIR/sum.f90
+++ b/flang/test/Lower/HLFIR/sum.f90
@@ -83,7 +83,7 @@ subroutine sum5(s)
 ! CHECK-NEXT:    hlfir.assign %[[EXPR]] to %[[OUT]]#0 : !hlfir.expr<2xi32>, !fir.ref<!fir.array<2xi32>>
 ! CHECK-NEXT:    hlfir.destroy %[[EXPR]] : !hlfir.expr<2xi32>
 ! CHECK-NEXT:    return
-! CHECK-nEXT:  }
+! CHECK-NEXT:  }
 
 subroutine sum6(a, s, d)
   integer, pointer :: d
diff --git a/flang/test/Lower/HLFIR/unroll-loops.fir b/flang/test/Lower/HLFIR/unroll-loops.fir
new file mode 100644
index 0000000000000..6a9dd28a37b6d
--- /dev/null
+++ b/flang/test/Lower/HLFIR/unroll-loops.fir
@@ -0,0 +1,46 @@
+// DEFINE: %{triple} =
+// DEFINE: %{check-unroll} = %flang_fc1 -emit-llvm -O1 -funroll-loops -mllvm -force-vector-width=2 -triple %{triple} -o- %s | FileCheck %s --check-prefixes=CHECK,UNROLL
+// DEFINE: %{check-nounroll} = %flang_fc1 -emit-llvm -O1 -mllvm -force-vector-width=2 -triple %{triple} -o- %s | FileCheck %s --check-prefixes=CHECK,NO-UNROLL
+
+// REDEFINE: %{triple} = aarch64-unknown-linux-gnu
+// RUN: %if aarch64-registered-target %{ %{check-unroll} %}
+// RUN: %if aarch64-registered-target %{ %{check-nounroll} %}
+
+// REDEFINE: %{triple} = x86_64-unknown-linux-gnu
+// RUN: %if x86-registered-target %{ %{check-unroll} %}
+// RUN: %if x86-registered-target %{ %{check-nounroll} %}
+
+// CHECK-LABEL: @unroll
+// CHECK-SAME: (ptr writeonly captures(none) %[[ARG0:.*]])
+func.func @unroll(%arg0: !fir.ref<!fir.array<1000 x index>> {fir.bindc_name = "a"}) {
+  %scope = fir.dummy_scope : !fir.dscope
+  %c1000 = arith.constant 1000 : index
+  %shape = fir.shape %c1000 : (index) -> !fir.shape<1>
+  %a:2 = hlfir.declare %arg0(%shape) dummy_scope %scope {uniq_name = "unrollEa"} : (!fir.ref<!fir.array<1000xindex>>, !fir.shape<1>, !fir.dscope) -> (!fir.ref<!fir.array<1000 x index>>, !fir.ref<!fir.array<1000 x index>>)
+  %c1 = arith.constant 1 : index
+  fir.do_loop %arg1 = %c1 to %c1000 step %c1 {
+    // CHECK: br label %[[BLK:.*]]
+    // CHECK: [[BLK]]:
+    // CHECK-NEXT: %[[IND:.*]] = phi i64 [ 0, %{{.*}} ], [ %[[NIV:.*]], %[[BLK]] ]
+    // CHECK-NEXT: %[[VIND:.*]] = phi <2 x i64> [ <i64 1, i64 2>, %{{.*}} ], [ %[[NVIND:.*]], %[[BLK]] ]
+
+    // NO-UNROLL-NEXT: %[[GEP:.*]] = getelementptr i64, ptr %[[ARG0]], i64 %[[IND]]
+    // NO-UNROLL-NEXT: store <2 x i64> %[[VIND]], ptr %[[GEP]]
+    // NO-UNROLL-NEXT: %[[NIV:.*]] = add nuw i64 %{{.*}}, 2
+    // NO-UNROLL-NEXT: %[[NVIND]] = add <2 x i64> %[[VIND]], splat (i64 2)
+
+    // UNROLL-NEXT: %[[VIND1:.*]] = add <2 x i64> %[[VIND]], splat (i64 2)
+    // UNROLL-NEXT: %[[GEP0:.*]] = getelementptr i64, ptr %[[ARG0]], i64 %[[IND]]
+    // UNROLL-NEXT: %[[GEP1:.*]] = getelementptr i8, ptr %[[GEP0]], i64 16
+    // UNROLL-NEXT: store <2 x i64> %[[VIND]], ptr %[[GEP0]]
+    // UNROLL-NEXT: store <2 x i64> %[[VIND1]], ptr %[[GEP1]]
+    // UNROLL-NEXT: %[[NIV:.*]] = add nuw i64 %[[IND]], 4
+    // UNROLL-NEXT: %[[NVIND:.*]] = add <2 x i64> %[[VIND]], splat (i64 4)
+
+    // CHECK-NEXT: %[[EXIT:.*]] = icmp eq i64 %[[NIV]], 1000
+    // CHECK-NEXT: br i1 %[[EXIT]], label %{{.*}}, label %[[BLK]]
+    %ai = hlfir.designate %a#0 (%arg1)  : (!fir.ref<!fir.array<1000 x index>>, index) -> !fir.ref<index>
+    hlfir.assign %arg1 to %ai : index, !fir.ref<index>
+  }
+  return
+}
diff --git a/flang/test/Lower/Intrinsics/abs.f90 b/flang/test/Lower/Intrinsics/abs.f90
index e5e4b79e9f79e..7150cb2d352fd 100644
--- a/flang/test/Lower/Intrinsics/abs.f90
+++ b/flang/test/Lower/Intrinsics/abs.f90
@@ -1,7 +1,7 @@
-! RUN: bbc -emit-fir -hlfir=false %s -o - | FileCheck %s --check-prefixes="CHECK,CMPLX,CMPLX-PRECISE"
+! RUN: bbc -emit-fir -hlfir=false %s -o - | FileCheck %s --check-prefixes=CHECK,CMPLX,CMPLX-PRECISE,%if flang-supports-f128-math %{F128%} %else %{F64%}
 ! RUN: bbc -emit-fir -hlfir=false --math-runtime=precise %s -o - | FileCheck %s --check-prefixes="CMPLX,CMPLX-PRECISE"
 ! RUN: bbc --force-mlir-complex -emit-fir -hlfir=false %s -o - | FileCheck %s --check-prefixes="CMPLX,CMPLX-FAST"
-! RUN: %flang_fc1 -emit-fir -flang-deprecated-no-hlfir %s -o - | FileCheck %s --check-prefixes="CHECK,CMPLX,CMPLX-PRECISE"
+! RUN: %flang_fc1 -emit-fir -flang-deprecated-no-hlfir %s -o - | FileCheck %s --check-prefixes=CHECK,CMPLX,CMPLX-PRECISE,%if flang-supports-f128-math %{F128%} %else %{F64%}
 ! RUN: %flang_fc1 -emit-fir -flang-deprecated-no-hlfir -mllvm --math-runtime=precise %s -o - | FileCheck %s --check-prefixes="CMPLX,CMPLX-PRECISE"
 ! RUN: %flang_fc1 -emit-fir -flang-deprecated-no-hlfir -mllvm --force-mlir-complex %s -o - | FileCheck %s --check-prefixes="CMPLX,CMPLX-FAST"
 ! RUN: %flang_fc1 -fapprox-func -emit-fir -flang-deprecated-no-hlfir %s -o - | FileCheck %s --check-prefixes="CMPLX,CMPLX-APPROX"
@@ -85,13 +85,18 @@ subroutine abs_testd(a, b)
 end subroutine
 
 ! CHECK-LABEL: func @_QPabs_testr16(
-! CHECK-SAME:  %[[VAL_0:.*]]: !fir.ref<f128>{{.*}}, %[[VAL_1:.*]]: !fir.ref<f128>{{.*}}) {
+! F128-SAME:  %[[VAL_0:.*]]: !fir.ref<f128>{{.*}}, %[[VAL_1:.*]]: !fir.ref<f128>{{.*}}) {
+! F64-SAME:  %[[VAL_0:.*]]: !fir.ref<f64>{{.*}}, %[[VAL_1:.*]]: !fir.ref<f64>{{.*}}) {
 subroutine abs_testr16(a, b)
-! CHECK: %[[VAL_2:.*]] = fir.load %[[VAL_0]] : !fir.ref<f128>
-! CHECK: %[[VAL_3:.*]] = math.absf %[[VAL_2]] {{.*}}: f128
-! CHECK: fir.store %[[VAL_3]] to %[[VAL_1]] : !fir.ref<f128>
+! F128: %[[VAL_2:.*]] = fir.load %[[VAL_0]] : !fir.ref<f128>
+! F64: %[[VAL_2:.*]] = fir.load %[[VAL_0]] : !fir.ref<f64>
+! F128: %[[VAL_3:.*]] = math.absf %[[VAL_2]] {{.*}}: f128
+! F64: %[[VAL_3:.*]] = math.absf %[[VAL_2]] {{.*}}: f64
+! F128: fir.store %[[VAL_3]] to %[[VAL_1]] : !fir.ref<f128>
+! F64: fir.store %[[VAL_3]] to %[[VAL_1]] : !fir.ref<f64>
 ! CHECK: return
-  real(kind=16) :: a, b
+  integer, parameter :: rk = merge(16, 8, selected_real_kind(33, 4931)==16)
+  real(kind=rk) :: a, b
   b = abs(a)
 end subroutine
 
diff --git a/flang/test/Lower/Intrinsics/adjustl.f90 b/flang/test/Lower/Intrinsics/adjustl.f90
index 56c93996015f4..a742f58db5c48 100644
--- a/flang/test/Lower/Intrinsics/adjustl.f90
+++ b/flang/test/Lower/Intrinsics/adjustl.f90
@@ -1,6 +1,6 @@
 ! RUN: bbc -emit-fir -hlfir=false %s -o - | FileCheck %s
 
-! CHECK-LABLE: adjustl_test
+! CHECK-LABEL: adjustl_test
 subroutine adjustl_test
     character(len=12) :: adjust_str = '  0123456789'
   ! CHECK: %[[strBox:.*]] = fir.alloca !fir.box<!fir.heap<!fir.char<1,?>>>
diff --git a/flang/test/Lower/Intrinsics/adjustr.f90 b/flang/test/Lower/Intrinsics/adjustr.f90
index 17c2a1647bb8d..a929ab17ab9ff 100644
--- a/flang/test/Lower/Intrinsics/adjustr.f90
+++ b/flang/test/Lower/Intrinsics/adjustr.f90
@@ -1,6 +1,6 @@
 ! RUN: bbc -emit-fir -hlfir=false %s -o - | FileCheck %s
 
-! CHECK-LABLE: adjustr_test
+! CHECK-LABEL: adjustr_test
 subroutine adjustr_test
     character(len=12) :: adjust_str = '0123456789  '
   ! CHECK: %[[strBox:.*]] = fir.alloca !fir.box<!fir.heap<!fir.char<1,?>>>
diff --git a/flang/test/Lower/Intrinsics/aint.f90 b/flang/test/Lower/Intrinsics/aint.f90
index fb459953a06c3..b82b63abbf616 100644
--- a/flang/test/Lower/Intrinsics/aint.f90
+++ b/flang/test/Lower/Intrinsics/aint.f90
@@ -1,41 +1,31 @@
-! RUN: bbc -emit-fir -hlfir=false %s -o - | FileCheck %s
+! RUN: bbc -emit-hlfir %s -o - | FileCheck %s --check-prefixes=CHECK%if target=x86_64{{.*}} %{,CHECK-KIND10%}
 
 ! CHECK-LABEL: func @_QPaint_test(
-! CHECK-SAME: %[[VAL_0:.*]]: !fir.ref<f32>{{.*}}, %[[VAL_1:.*]]: !fir.ref<f32>{{.*}}) {
+! CHECK-SAME: %[[VAL_0_b:.*]]: !fir.ref<f32>{{.*}}, %[[VAL_1_b:.*]]: !fir.ref<f32>{{.*}}) {
 subroutine aint_test(a, b)
-! CHECK: %[[VAL_2:.*]] = fir.load %[[VAL_0]] : !fir.ref<f32>
+! CHECK: %[[VAL_0:.*]]:2 = hlfir.declare %[[VAL_0_b]]
+! CHECK: %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_1_b]]
+! CHECK: %[[VAL_2:.*]] = fir.load %[[VAL_0]]#0 : !fir.ref<f32>
 ! CHECK: %[[VAL_3:.*]] = fir.call @llvm.trunc.f32(%[[VAL_2]]) {{.*}}: (f32) -> f32
-! CHECK: fir.store %[[VAL_3]] to %[[VAL_1]] : !fir.ref<f32>
+! CHECK: hlfir.assign %[[VAL_3]] to %[[VAL_1]]#0 : f32, !fir.ref<f32>
 ! CHECK: return
   real :: a, b
   b = aint(a)
 end subroutine
 
 ! CHECK-LABEL: func.func @_QPaint_test_real8(
-! CHECK-SAME:                                %[[VAL_0:.*]]: !fir.ref<f64> {fir.bindc_name = "a"},
-! CHECK-SAME:                                %[[VAL_1:.*]]: !fir.ref<f64> {fir.bindc_name = "b"}) {
-! CHECK:         %[[VAL_2:.*]] = fir.load %[[VAL_0]] : !fir.ref<f64>
-! CHECK:         %[[VAL_3:.*]] = fir.call @llvm.trunc.f64(%[[VAL_2]]) {{.*}}: (f64) -> f64
-! CHECK:         fir.store %[[VAL_3]] to %[[VAL_1]] : !fir.ref<f64>
-! CHECK:         return
-! CHECK:       }
+! CHECK:   fir.call @llvm.trunc.f64({{.*}}) {{.*}}: (f64) -> f64
 
 subroutine aint_test_real8(a, b)
   real(8) :: a, b
   b = aint(a)
 end subroutine
 
-! CHECK-LABEL: func.func @_QPaint_test_real10(
-! CHECK-SAME:                                 %[[VAL_0:.*]]: !fir.ref<f80> {fir.bindc_name = "a"},
-! CHECK-SAME:                                 %[[VAL_1:.*]]: !fir.ref<f80> {fir.bindc_name = "b"}) {
-! CHECK:         %[[VAL_2:.*]] = fir.load %[[VAL_0]] : !fir.ref<f80>
-! CHECK:         %[[VAL_3:.*]] = fir.call @llvm.trunc.f80(%[[VAL_2]]) {{.*}}: (f80) -> f80
-! CHECK:         fir.store %[[VAL_3]] to %[[VAL_1]] : !fir.ref<f80>
-! CHECK:         return
-! CHECK:       }
-
+! CHECK-KIND10-LABEL: func.func @_QPaint_test_real10(
+! CHECK-KIND10:   fir.call @llvm.trunc.f80({{.*}}) {{.*}}: (f80) -> f80
 subroutine aint_test_real10(a, b)
-  real(10) :: a, b
+  integer, parameter :: kind10 = merge(10, 4, selected_real_kind(p=18).eq.10)
+  real(kind10) :: a, b
   b = aint(a)
 end subroutine
 
diff --git a/flang/test/Lower/Intrinsics/anint.f90 b/flang/test/Lower/Intrinsics/anint.f90
index 4148d18f15b30..a7b24648ca0b6 100644
--- a/flang/test/Lower/Intrinsics/anint.f90
+++ b/flang/test/Lower/Intrinsics/anint.f90
@@ -1,11 +1,13 @@
-! RUN: bbc -emit-fir -hlfir=false %s -o - | FileCheck %s
+! RUN: bbc -emit-hlfir %s -o - | FileCheck %s --check-prefixes=CHECK%if target=x86_64{{.*}} %{,CHECK-KIND10%}
 
 ! CHECK-LABEL: func.func @_QPanint_test(
-! CHECK-SAME:                           %[[VAL_0:.*]]: !fir.ref<f32> {fir.bindc_name = "a"},
-! CHECK-SAME:                           %[[VAL_1:.*]]: !fir.ref<f32> {fir.bindc_name = "b"}) {
-! CHECK:         %[[VAL_2:.*]] = fir.load %[[VAL_0]] : !fir.ref<f32>
+! CHECK-SAME:                           %[[VAL_0_b:.*]]: !fir.ref<f32> {fir.bindc_name = "a"},
+! CHECK-SAME:                           %[[VAL_1_b:.*]]: !fir.ref<f32> {fir.bindc_name = "b"}) {
+! CHECK:         %[[VAL_0:.*]]:2 = hlfir.declare %[[VAL_0_b]]
+! CHECK:         %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_1_b]]
+! CHECK:         %[[VAL_2:.*]] = fir.load %[[VAL_0]]#0 : !fir.ref<f32>
 ! CHECK:         %[[VAL_3:.*]] = llvm.intr.round(%[[VAL_2]]) : (f32) -> f32
-! CHECK:         fir.store %[[VAL_3]] to %[[VAL_1]] : !fir.ref<f32>
+! CHECK:         hlfir.assign %[[VAL_3]] to %[[VAL_1]]#0 : f32, !fir.ref<f32>
 ! CHECK:         return
 ! CHECK:       }
 
@@ -15,30 +17,19 @@ subroutine anint_test(a, b)
 end subroutine
 
 ! CHECK-LABEL: func.func @_QPanint_test_real8(
-! CHECK-SAME:                                 %[[VAL_0:.*]]: !fir.ref<f64> {fir.bindc_name = "a"},
-! CHECK-SAME:                                 %[[VAL_1:.*]]: !fir.ref<f64> {fir.bindc_name = "b"}) {
-! CHECK:         %[[VAL_2:.*]] = fir.load %[[VAL_0]] : !fir.ref<f64>
-! CHECK:         %[[VAL_3:.*]] = llvm.intr.round(%[[VAL_2]]) : (f64) -> f64
-! CHECK:         fir.store %[[VAL_3]] to %[[VAL_1]] : !fir.ref<f64>
-! CHECK:         return
-! CHECK:       }
+! CHECK:    llvm.intr.round(%{{.*}}) : (f64) -> f64
 
 subroutine anint_test_real8(a, b)
   real(8) :: a, b
   b = anint(a)
 end subroutine
 
-! CHECK-LABEL: func.func @_QPanint_test_real10(
-! CHECK-SAME:                                  %[[VAL_0:.*]]: !fir.ref<f80> {fir.bindc_name = "a"},
-! CHECK-SAME:                                  %[[VAL_1:.*]]: !fir.ref<f80> {fir.bindc_name = "b"}) {
-! CHECK:         %[[VAL_2:.*]] = fir.load %[[VAL_0]] : !fir.ref<f80>
-! CHECK:         %[[VAL_3:.*]] = llvm.intr.round(%[[VAL_2]]) : (f80) -> f80
-! CHECK:         fir.store %[[VAL_3]] to %[[VAL_1]] : !fir.ref<f80>
-! CHECK:         return
-! CHECK:       }
+! CHECK-KIND10-LABEL: func.func @_QPanint_test_real10(
+! CHECK-KIND10:    llvm.intr.round(%{{.*}}) : (f80) -> f80
 
 subroutine anint_test_real10(a, b)
-  real(10) :: a, b
+  integer, parameter :: kind10 = merge(10, 4, selected_real_kind(p=18).eq.10)
+  real(kind10) :: a, b
   b = anint(a)
 end subroutine
 
diff --git a/flang/test/Lower/Intrinsics/any.f90 b/flang/test/Lower/Intrinsics/any.f90
index e4dc20e3de2f3..2c0b92fcc1d74 100644
--- a/flang/test/Lower/Intrinsics/any.f90
+++ b/flang/test/Lower/Intrinsics/any.f90
@@ -27,5 +27,5 @@ subroutine any_test2(mask, d, rslt)
 ! CHECK:  fir.call @_FortranAAnyDim(%[[a6:.*]], %[[a7:.*]], %[[a1:.*]], %{{.*}}, %{{.*}}) {{.*}}: (!fir.ref<!fir.box<none>>, !fir.box<none>, i32, !fir.ref<i8>, i32) -> ()
 ! CHECK-DAG:  %[[a10:.*]] = fir.load %[[a0:.*]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?x!fir.logical<4>>>>>
 ! CHECK-DAG:  %[[a12:.*]] = fir.box_addr %[[a10:.*]] : (!fir.box<!fir.heap<!fir.array<?x!fir.logical<4>>>>) -> !fir.heap<!fir.array<?x!fir.logical<4>>>
-! CHECK-DAG  fir.freemem %[[a12:.*]]
+! CHECK-DAG:  fir.freemem %[[a12:.*]]
 end subroutine
diff --git a/flang/test/Lower/Intrinsics/chdir.f90 b/flang/test/Lower/Intrinsics/chdir.f90
new file mode 100644
index 0000000000000..3c47b41b95e8d
--- /dev/null
+++ b/flang/test/Lower/Intrinsics/chdir.f90
@@ -0,0 +1,94 @@
+! RUN: bbc -emit-hlfir %s -o - | FileCheck %s
+
+subroutine test_chdir()
+  implicit none
+! CHECK-LABEL:   func.func @_QPtest_chdir() {
+
+  call chdir("..")
+! CHECK:  %[[VAL_0:.*]] = fir.address_of(@_QQclX2E2E) : !fir.ref<!fir.char<1,2>>
+! CHECK:  %[[C_2:.*]] = arith.constant 2 : index
+! CHECK:  %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0]] typeparams %[[C_2]] {fortran_attrs = #fir.var_attrs<parameter>, uniq_name = "_QQclX2E2E"} : (!fir.ref<!fir.char<1,2>>, index) -> (!fir.ref<!fir.char<1,2>>, !fir.ref<!fir.char<1,2>>)
+! CHECK:  %[[VAL_2:.*]] = fir.convert %[[VAL_1]]#1 : (!fir.ref<!fir.char<1,2>>) -> !fir.ref<i8>
+! CHECK:  %[[VAL_3:.*]] = fir.call @_FortranAChdir(%[[VAL_2]]) fastmath<contract> : (!fir.ref<i8>) -> i32
+end subroutine
+
+subroutine test_chdir_subroutine_status_i4()
+  implicit none
+  integer(4) :: stat
+! CHECK-LABEL:   func.func @_QPtest_chdir_subroutine_status_i4() {
+
+  call chdir("..", STATUS=stat)
+! CHECK:  %[[VAL_0:.*]] = fir.alloca i32 {bindc_name = "stat", uniq_name = "_QFtest_chdir_subroutine_status_i4Estat"}
+! CHECK:  %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0]] {uniq_name = "_QFtest_chdir_subroutine_status_i4Estat"} : (!fir.ref<i32>) ->
+! (!fir.ref<i32>, !fir.ref<i32>)
+! CHECK:  %[[VAL_2:.*]] = fir.address_of(@_QQclX2E2E) : !fir.ref<!fir.char<1,2>>
+! CHECK:  %[[C_2:.*]] = arith.constant 2 : index
+! CHECK:  %[[VAL_3:.*]]:2 = hlfir.declare %[[VAL_2]] typeparams %[[C_2]] {fortran_attrs = #fir.var_attrs<parameter>, uniq_name = {{.*}} : (!fir.ref<!fir.char<1,2>>, index) -> (!fir.ref<!fir.char<1,2>>, !fir.ref<!fir.char<1,2>>)
+! CHECK:  %[[VAL_4:.*]] = fir.convert %[[VAL_3]]#1 : (!fir.ref<!fir.char<1,2>>) -> !fir.ref<i8>
+! CHECK:  %[[VAL_5:.*]] = fir.call @_FortranAChdir(%[[VAL_4]]) fastmath<contract> : (!fir.ref<i8>) -> i32
+! CHECK:  %[[VAL_6:.*]] = fir.convert %{{.*}} : (!fir.ref<i32>) -> i64
+! CHECK:  %[[C_0_I64:.*]] = arith.constant 0 : i64
+! CHECK:  %[[VAL_7:.*]] = arith.cmpi ne, %[[VAL_6]], %[[C_0_I64]] : i64
+! CHECK: fir.if %[[VAL_7]] {
+! CHECK:   fir.store %[[VAL_5]] to %[[VAL_1]]#1 : !fir.ref<i32>
+! CHECK: }
+end subroutine
+
+subroutine test_chdir_function_status_i4()
+  implicit none
+  integer(4) :: stat
+! CHECK-LABEL:   func.func @_QPtest_chdir_function_status_i4() {
+
+  stat = chdir("..")
+! CHECK:  %[[VAL_0:.*]] = fir.alloca i32 {bindc_name = "stat", uniq_name = "_QFtest_chdir_function_status_i4Estat"}
+! CHECK:  %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0]] {uniq_name = "_QFtest_chdir_function_status_i4Estat"} : (!fir.ref<i32>) ->
+! (!fir.ref<i32>, !fir.ref<i32>)
+! CHECK:  %[[VAL_2:.*]] = fir.address_of(@_QQclX2E2E) : !fir.ref<!fir.char<1,2>>
+! CHECK:  %[[C_2:.*]] = arith.constant 2 : index
+! CHECK:  %[[VAL_3:.*]]:2 = hlfir.declare %[[VAL_2]] typeparams %[[C_2]] {fortran_attrs = #fir.var_attrs<parameter>, uniq_name = {{.*}} : (!fir.ref<!fir.char<1,2>>, index) -> (!fir.ref<!fir.char<1,2>>, !fir.ref<!fir.char<1,2>>)
+! CHECK:  %[[VAL_4:.*]] = fir.convert %[[VAL_3]]#1 : (!fir.ref<!fir.char<1,2>>) -> !fir.ref<i8>
+! CHECK:  %[[VAL_5:.*]] = fir.call @_FortranAChdir(%[[VAL_4]]) fastmath<contract> : (!fir.ref<i8>) -> i32
+! CHECK: hlfir.assign %[[VAL_5]] to %[[VAL_1]]#0 : i32, !fir.ref<i32>
+end subroutine
+
+subroutine test_chdir_subroutine_status_i8()
+  implicit none
+  integer(8) :: stat
+! CHECK-LABEL:   func.func @_QPtest_chdir_subroutine_status_i8() {
+
+  call chdir("..", STATUS=stat)
+! CHECK:  %[[VAL_0:.*]] = fir.alloca i64 {bindc_name = "stat", uniq_name = "_QFtest_chdir_subroutine_status_i8Estat"}
+! CHECK:  %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0]] {uniq_name = "_QFtest_chdir_subroutine_status_i8Estat"} : (!fir.ref<i64>) ->
+! (!fir.ref<i64>, !fir.ref<i64>)
+! CHECK:  %[[VAL_2:.*]] = fir.address_of(@_QQclX2E2E) : !fir.ref<!fir.char<1,2>>
+! CHECK:  %[[C_2:.*]] = arith.constant 2 : index
+! CHECK:  %[[VAL_3:.*]]:2 = hlfir.declare %[[VAL_2]] typeparams %[[C_2]] {fortran_attrs = #fir.var_attrs<parameter>, uniq_name = {{.*}} : (!fir.ref<!fir.char<1,2>>, index) -> (!fir.ref<!fir.char<1,2>>, !fir.ref<!fir.char<1,2>>)
+! CHECK:  %[[VAL_4:.*]] = fir.convert %{{.*}} : (!fir.ref<!fir.char<1,2>>) -> !fir.ref<i8>
+! CHECK:  %[[VAL_5:.*]] = fir.call @_FortranAChdir(%[[VAL_4]]) fastmath<contract> : (!fir.ref<i8>) -> i32
+! CHECK:  %[[VAL_6:.*]] = fir.convert %[[VAL_1]]#1 : (!fir.ref<i64>) -> i64
+! CHECK:  %[[C_0_I64:.*]] = arith.constant 0 : i64
+! CHECK:  %[[VAL_7:.*]] = arith.cmpi ne, %[[VAL_6]], %[[C_0_I64]] : i64
+! CHECK: fir.if %[[VAL_7]] {
+! CHECK:  %[[VAL_8:.*]] = fir.convert %[[VAL_5]] : (i32) -> i64
+! CHECK:   fir.store %[[VAL_8]] to %[[VAL_1]]#1 : !fir.ref<i64>
+! CHECK: }
+end subroutine
+
+subroutine test_chdir_function_status_i8()
+  implicit none
+  integer(8) :: stat
+! CHECK-LABEL:   func.func @_QPtest_chdir_function_status_i8() {
+
+  stat = chdir("..")
+! CHECK:  %[[VAL_0:.*]] = fir.alloca i64 {bindc_name = "stat", uniq_name = "_QFtest_chdir_function_status_i8Estat"}
+! CHECK:  %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0]] {uniq_name = "_QFtest_chdir_function_status_i8Estat"} : (!fir.ref<i64>) ->
+! (!fir.ref<i64>, !fir.ref<i64>)
+! CHECK:  %[[VAL_2:.*]] = fir.address_of(@_QQclX2E2E) : !fir.ref<!fir.char<1,2>>
+! CHECK:  %[[C_2:.*]] = arith.constant 2 : index
+! CHECK:  %[[VAL_3:.*]]:2 = hlfir.declare %[[VAL_2]] typeparams %[[C_2]] {fortran_attrs = #fir.var_attrs<parameter>, uniq_name = {{.*}} : (!fir.ref<!fir.char<1,2>>, index) -> (!fir.ref<!fir.char<1,2>>, !fir.ref<!fir.char<1,2>>)
+! CHECK:  %[[VAL_4:.*]] = fir.convert %[[VAL_3]]#1 : (!fir.ref<!fir.char<1,2>>) -> !fir.ref<i8>
+! CHECK:  %[[VAL_5:.*]] = fir.call @_FortranAChdir(%[[VAL_4]]) fastmath<contract> : (!fir.ref<i8>) -> i32
+! CHECK:  %[[VAL_6:.*]] = fir.convert %[[VAL_5]] : (i32) -> i64
+! CHECK: hlfir.assign %[[VAL_6]] to %[[VAL_1]]#0 : i64, !fir.ref<i64>
+end subroutine
+
diff --git a/flang/test/Lower/Intrinsics/dot_product.f90 b/flang/test/Lower/Intrinsics/dot_product.f90
index 9a825c4b9acf1..62694a70555df 100644
--- a/flang/test/Lower/Intrinsics/dot_product.f90
+++ b/flang/test/Lower/Intrinsics/dot_product.f90
@@ -1,14 +1,16 @@
-! RUN: bbc -emit-fir -hlfir=false %s -o - | FileCheck %s
-! RUN: %flang_fc1 -emit-fir -flang-deprecated-no-hlfir %s -o - | FileCheck %s
+! RUN: %flang_fc1 -emit-fir -O0 %s -o - | FileCheck %s --check-prefixes=CHECK%if target=x86_64{{.*}} %{,CHECK-KIND10%}%if flang-supports-f128-math %{,CHECK-KIND16%}
 
 ! DOT_PROD
 ! CHECK-LABEL: dot_prod_int_default
-! CHECK-SAME: %[[x:arg0]]: !fir.box<!fir.array<?xi32>>
-! CHECK-SAME: %[[y:arg1]]: !fir.box<!fir.array<?xi32>>
-! CHECK-SAME: %[[z:arg2]]: !fir.box<!fir.array<?xi32>>
 subroutine dot_prod_int_default (x, y, z)
   integer, dimension(1:) :: x,y
   integer, dimension(1:) :: z
+  ! CHECK: %[[x1:.*]] = fir.declare{{.*}}x"
+  ! CHECK: %[[x:.*]] = fir.rebox %[[x1]]{{.*}}
+  ! CHECK: %[[y1:.*]] = fir.declare{{.*}}y"
+  ! CHECK: %[[y:.*]] = fir.rebox %[[y1]]{{.*}}
+  ! CHECK: %[[z1:.*]] = fir.declare{{.*}}z"
+  ! CHECK: %[[z:.*]] = fir.rebox %[[z1]]{{.*}}
   ! CHECK-DAG: %[[x_conv:.*]] = fir.convert %[[x]] : (!fir.box<!fir.array<?xi32>>) -> !fir.box<none>
   ! CHECK-DAG: %[[y_conv:.*]] = fir.convert %[[y]] : (!fir.box<!fir.array<?xi32>>) -> !fir.box<none>
   ! CHECK-DAG: %[[res:.*]] = fir.call @_FortranADotProductInteger4(%[[x_conv]], %[[y_conv]], %{{[0-9]+}}, %{{.*}}) {{.*}}: (!fir.box<none>, !fir.box<none>, !fir.ref<i8>, i32) -> i32
@@ -16,268 +18,172 @@ subroutine dot_prod_int_default (x, y, z)
 end subroutine
 
 ! CHECK-LABEL: dot_prod_int_kind_1
-! CHECK-SAME: %[[x:arg0]]: !fir.box<!fir.array<?xi8>>
-! CHECK-SAME: %[[y:arg1]]: !fir.box<!fir.array<?xi8>>
-! CHECK-SAME: %[[z:arg2]]: !fir.box<!fir.array<?xi8>>
 subroutine dot_prod_int_kind_1 (x, y, z)
   integer(kind=1), dimension(1:) :: x,y
   integer(kind=1), dimension(1:) :: z
-  ! CHECK-DAG: %[[x_conv:.*]] = fir.convert %[[x]] : (!fir.box<!fir.array<?xi8>>) -> !fir.box<none>
-  ! CHECK-DAG: %[[y_conv:.*]] = fir.convert %[[y]] : (!fir.box<!fir.array<?xi8>>) -> !fir.box<none>
-  ! CHECK-DAG: %[[res:.*]] = fir.call @_FortranADotProductInteger1(%[[x_conv]], %[[y_conv]], %{{[0-9]+}}, %{{.*}}) {{.*}}: (!fir.box<none>, !fir.box<none>, !fir.ref<i8>, i32) -> i8
+  ! CHECK: fir.call @_FortranADotProductInteger1(%{{.*}}, %{{.*}}, %{{[0-9]+}}, %{{.*}}) {{.*}}: (!fir.box<none>, !fir.box<none>, !fir.ref<i8>, i32) -> i8
   z = dot_product(x,y)
 end subroutine
 
 ! CHECK-LABEL: dot_prod_int_kind_2
-! CHECK-SAME: %[[x:arg0]]: !fir.box<!fir.array<?xi16>>
-! CHECK-SAME: %[[y:arg1]]: !fir.box<!fir.array<?xi16>>
-! CHECK-SAME: %[[z:arg2]]: !fir.box<!fir.array<?xi16>>
 subroutine dot_prod_int_kind_2 (x, y, z)
   integer(kind=2), dimension(1:) :: x,y
   integer(kind=2), dimension(1:) :: z
-  ! CHECK-DAG: %[[x_conv:.*]] = fir.convert %[[x]] : (!fir.box<!fir.array<?xi16>>) -> !fir.box<none>
-  ! CHECK-DAG: %[[y_conv:.*]] = fir.convert %[[y]] : (!fir.box<!fir.array<?xi16>>) -> !fir.box<none>
-  ! CHECK-DAG: %[[res:.*]] = fir.call @_FortranADotProductInteger2(%[[x_conv]], %[[y_conv]], %{{[0-9]+}}, %{{.*}}) {{.*}}: (!fir.box<none>, !fir.box<none>, !fir.ref<i8>, i32) -> i16
+  ! CHECK: fir.call @_FortranADotProductInteger2(%{{.*}}, %{{.*}}, %{{[0-9]+}}, %{{.*}}) {{.*}}: (!fir.box<none>, !fir.box<none>, !fir.ref<i8>, i32) -> i16
   z = dot_product(x,y)
 end subroutine
 
 ! CHECK-LABEL: dot_prod_int_kind_4
-! CHECK-SAME: %[[x:arg0]]: !fir.box<!fir.array<?xi32>>
-! CHECK-SAME: %[[y:arg1]]: !fir.box<!fir.array<?xi32>>
-! CHECK-SAME: %[[z:arg2]]: !fir.box<!fir.array<?xi32>>
 subroutine dot_prod_int_kind_4 (x, y, z)
   integer(kind=4), dimension(1:) :: x,y
   integer(kind=4), dimension(1:) :: z
-  ! CHECK-DAG: %[[x_conv:.*]] = fir.convert %[[x]] : (!fir.box<!fir.array<?xi32>>) -> !fir.box<none>
-  ! CHECK-DAG: %[[y_conv:.*]] = fir.convert %[[y]] : (!fir.box<!fir.array<?xi32>>) -> !fir.box<none>
-  ! CHECK-DAG: %[[res:.*]] = fir.call @_FortranADotProductInteger4(%[[x_conv]], %[[y_conv]], %{{[0-9]+}}, %{{.*}}) {{.*}}: (!fir.box<none>, !fir.box<none>, !fir.ref<i8>, i32) -> i32
+  ! CHECK: fir.call @_FortranADotProductInteger4(%{{.*}}, %{{.*}}, %{{[0-9]+}}, %{{.*}}) {{.*}}: (!fir.box<none>, !fir.box<none>, !fir.ref<i8>, i32) -> i32
   z = dot_product(x,y)
 end subroutine
 
 ! CHECK-LABEL: dot_prod_int_kind_8
-! CHECK-SAME: %[[x:arg0]]: !fir.box<!fir.array<?xi64>>
-! CHECK-SAME: %[[y:arg1]]: !fir.box<!fir.array<?xi64>>
-! CHECK-SAME: %[[z:arg2]]: !fir.box<!fir.array<?xi64>>
 subroutine dot_prod_int_kind_8 (x, y, z)
   integer(kind=8), dimension(1:) :: x,y
   integer(kind=8), dimension(1:) :: z
-  ! CHECK-DAG: %[[x_conv:.*]] = fir.convert %[[x]] : (!fir.box<!fir.array<?xi64>>) -> !fir.box<none>
-  ! CHECK-DAG: %[[y_conv:.*]] = fir.convert %[[y]] : (!fir.box<!fir.array<?xi64>>) -> !fir.box<none>
-  ! CHECK-DAG: %[[res:.*]] = fir.call @_FortranADotProductInteger8(%[[x_conv]], %[[y_conv]], %{{[0-9]+}}, %{{.*}}) {{.*}}: (!fir.box<none>, !fir.box<none>, !fir.ref<i8>, i32) -> i64
+  ! CHECK: fir.call @_FortranADotProductInteger8(%{{.*}}, %{{.*}}, %{{[0-9]+}}, %{{.*}}) {{.*}}: (!fir.box<none>, !fir.box<none>, !fir.ref<i8>, i32) -> i64
   z = dot_product(x,y)
 end subroutine
 
 ! CHECK-LABEL: dot_prod_int_kind_16
-! CHECK-SAME: %[[x:arg0]]: !fir.box<!fir.array<?xi128>>
-! CHECK-SAME: %[[y:arg1]]: !fir.box<!fir.array<?xi128>>
-! CHECK-SAME: %[[z:arg2]]: !fir.box<!fir.array<?xi128>>
 subroutine dot_prod_int_kind_16 (x, y, z)
   integer(kind=16), dimension(1:) :: x,y
   integer(kind=16), dimension(1:) :: z
-  ! CHECK-DAG: %[[x_conv:.*]] = fir.convert %[[x]] : (!fir.box<!fir.array<?xi128>>) -> !fir.box<none>
-  ! CHECK-DAG: %[[y_conv:.*]] = fir.convert %[[y]] : (!fir.box<!fir.array<?xi128>>) -> !fir.box<none>
-  ! CHECK-DAG: %[[res:.*]] = fir.call @_FortranADotProductInteger16(%[[x_conv]], %[[y_conv]], %{{[0-9]+}}, %{{.*}}) {{.*}}: (!fir.box<none>, !fir.box<none>, !fir.ref<i8>, i32) -> i128
+  ! CHECK: fir.call @_FortranADotProductInteger16(%{{.*}}, %{{.*}}, %{{[0-9]+}}, %{{.*}}) {{.*}}: (!fir.box<none>, !fir.box<none>, !fir.ref<i8>, i32) -> i128
   z = dot_product(x,y)
 end subroutine
 
 ! CHECK-LABEL: dot_prod_real_kind_default
-! CHECK-SAME: %[[x:arg0]]: !fir.box<!fir.array<?xf32>>
-! CHECK-SAME: %[[y:arg1]]: !fir.box<!fir.array<?xf32>>
-! CHECK-SAME: %[[z:arg2]]: !fir.box<!fir.array<?xf32>>
 subroutine dot_prod_real_kind_default (x, y, z)
   real, dimension(1:) :: x,y
   real, dimension(1:) :: z
-  ! CHECK-DAG: %[[x_conv:.*]] = fir.convert %[[x]] : (!fir.box<!fir.array<?xf32>>) -> !fir.box<none>
-  ! CHECK-DAG: %[[y_conv:.*]] = fir.convert %[[y]] : (!fir.box<!fir.array<?xf32>>) -> !fir.box<none>
-  ! CHECK-DAG: %[[res:.*]] = fir.call @_FortranADotProductReal4(%[[x_conv]], %[[y_conv]], %{{[0-9]+}}, %{{.*}}) {{.*}}: (!fir.box<none>, !fir.box<none>, !fir.ref<i8>, i32) -> f32
+  ! CHECK: fir.call @_FortranADotProductReal4(%{{.*}}, %{{.*}}, %{{[0-9]+}}, %{{.*}}) {{.*}}: (!fir.box<none>, !fir.box<none>, !fir.ref<i8>, i32) -> f32
   z = dot_product(x,y)
 end subroutine
 
 ! CHECK-LABEL: dot_prod_real_kind_4
-! CHECK-SAME: %[[x:arg0]]: !fir.box<!fir.array<?xf32>>
-! CHECK-SAME: %[[y:arg1]]: !fir.box<!fir.array<?xf32>>
-! CHECK-SAME: %[[z:arg2]]: !fir.box<!fir.array<?xf32>>
 subroutine dot_prod_real_kind_4 (x, y, z)
   real(kind=4), dimension(1:) :: x,y
   real(kind=4), dimension(1:) :: z
-  ! CHECK-DAG: %[[x_conv:.*]] = fir.convert %[[x]] : (!fir.box<!fir.array<?xf32>>) -> !fir.box<none>
-  ! CHECK-DAG: %[[y_conv:.*]] = fir.convert %[[y]] : (!fir.box<!fir.array<?xf32>>) -> !fir.box<none>
-  ! CHECK-DAG: %[[res:.*]] = fir.call @_FortranADotProductReal4(%[[x_conv]], %[[y_conv]], %{{[0-9]+}}, %{{.*}}) {{.*}}: (!fir.box<none>, !fir.box<none>, !fir.ref<i8>, i32) -> f32
+  ! CHECK: fir.call @_FortranADotProductReal4(%{{.*}}, %{{.*}}, %{{[0-9]+}}, %{{.*}}) {{.*}}: (!fir.box<none>, !fir.box<none>, !fir.ref<i8>, i32) -> f32
   z = dot_product(x,y)
 end subroutine
 
 ! CHECK-LABEL: dot_prod_real_kind_8
-! CHECK-SAME: %[[x:arg0]]: !fir.box<!fir.array<?xf64>>
-! CHECK-SAME: %[[y:arg1]]: !fir.box<!fir.array<?xf64>>
-! CHECK-SAME: %[[z:arg2]]: !fir.box<!fir.array<?xf64>>
 subroutine dot_prod_real_kind_8 (x, y, z)
   real(kind=8), dimension(1:) :: x,y
   real(kind=8), dimension(1:) :: z
-  ! CHECK-DAG: %[[x_conv:.*]] = fir.convert %[[x]] : (!fir.box<!fir.array<?xf64>>) -> !fir.box<none>
-  ! CHECK-DAG: %[[y_conv:.*]] = fir.convert %[[y]] : (!fir.box<!fir.array<?xf64>>) -> !fir.box<none>
-  ! CHECK-DAG: %[[res:.*]] = fir.call @_FortranADotProductReal8(%[[x_conv]], %[[y_conv]], %{{[0-9]+}}, %{{.*}}) {{.*}}: (!fir.box<none>, !fir.box<none>, !fir.ref<i8>, i32) -> f64
+  ! CHECK: fir.call @_FortranADotProductReal8(%{{.*}}, %{{.*}}, %{{[0-9]+}}, %{{.*}}) {{.*}}: (!fir.box<none>, !fir.box<none>, !fir.ref<i8>, i32) -> f64
   z = dot_product(x,y)
 end subroutine
 
-! CHECK-LABEL: dot_prod_real_kind_10
-! CHECK-SAME: %[[x:arg0]]: !fir.box<!fir.array<?xf80>>
-! CHECK-SAME: %[[y:arg1]]: !fir.box<!fir.array<?xf80>>
-! CHECK-SAME: %[[z:arg2]]: !fir.box<!fir.array<?xf80>>
+! CHECK-KIND10-LABEL: dot_prod_real_kind_10
 subroutine dot_prod_real_kind_10 (x, y, z)
-  real(kind=10), dimension(1:) :: x,y
-  real(kind=10), dimension(1:) :: z
-  ! CHECK-DAG: %[[x_conv:.*]] = fir.convert %[[x]] : (!fir.box<!fir.array<?xf80>>) -> !fir.box<none>
-  ! CHECK-DAG: %[[y_conv:.*]] = fir.convert %[[y]] : (!fir.box<!fir.array<?xf80>>) -> !fir.box<none>
-  ! CHECK-DAG: %[[res:.*]] = fir.call @_FortranADotProductReal10(%[[x_conv]], %[[y_conv]], %{{[0-9]+}}, %{{.*}}) {{.*}}: (!fir.box<none>, !fir.box<none>, !fir.ref<i8>, i32) -> f80
+  integer, parameter :: kind10 = merge(10, 4, selected_real_kind(p=18).eq.10)
+  real(kind=kind10), dimension(1:) :: x,y
+  real(kind=kind10), dimension(1:) :: z
+  ! CHECK-KIND10: fir.call @_FortranADotProductReal10(%{{.*}}, %{{.*}}, %{{[0-9]+}}, %{{.*}}) {{.*}}: (!fir.box<none>, !fir.box<none>, !fir.ref<i8>, i32) -> f80
   z = dot_product(x,y)
 end subroutine
 
-! CHECK-LABEL: dot_prod_real_kind_16
-! CHECK-SAME: %[[x:arg0]]: !fir.box<!fir.array<?xf128>>
-! CHECK-SAME: %[[y:arg1]]: !fir.box<!fir.array<?xf128>>
-! CHECK-SAME: %[[z:arg2]]: !fir.box<!fir.array<?xf128>>
+! CHECK-KIND16-LABEL: dot_prod_real_kind_16
 subroutine dot_prod_real_kind_16 (x, y, z)
-  real(kind=16), dimension(1:) :: x,y
-  real(kind=16), dimension(1:) :: z
-  ! CHECK-DAG: %[[x_conv:.*]] = fir.convert %[[x]] : (!fir.box<!fir.array<?xf128>>) -> !fir.box<none>
-  ! CHECK-DAG: %[[y_conv:.*]] = fir.convert %[[y]] : (!fir.box<!fir.array<?xf128>>) -> !fir.box<none>
-  ! CHECK-DAG: %[[res:.*]] = fir.call @_FortranADotProductReal16(%[[x_conv]], %[[y_conv]], %{{[0-9]+}}, %{{.*}}) {{.*}}: (!fir.box<none>, !fir.box<none>, !fir.ref<i8>, i32) -> f128
+  integer, parameter :: kind16 = merge(16, 4, selected_real_kind(p=33).eq.16)
+  real(kind=kind16), dimension(1:) :: x,y
+  real(kind=kind16), dimension(1:) :: z
+  ! CHECK-KIND16: fir.call @_FortranADotProductReal16(%{{.*}}, %{{.*}}, %{{[0-9]+}}, %{{.*}}) {{.*}}: (!fir.box<none>, !fir.box<none>, !fir.ref<i8>, i32) -> f128
   z = dot_product(x,y)
 end subroutine
 
 ! CHECK-LABEL: dot_prod_double_default
-! CHECK-SAME: %[[x:arg0]]: !fir.box<!fir.array<?xf64>>
-! CHECK-SAME: %[[y:arg1]]: !fir.box<!fir.array<?xf64>>
-! CHECK-SAME: %[[z:arg2]]: !fir.box<!fir.array<?xf64>>
 subroutine dot_prod_double_default (x, y, z)
   double precision, dimension(1:) :: x,y
   double precision, dimension(1:) :: z
-  ! CHECK-DAG: %[[x_conv:.*]] = fir.convert %[[x]] : (!fir.box<!fir.array<?xf64>>) -> !fir.box<none>
-  ! CHECK-DAG: %[[y_conv:.*]] = fir.convert %[[y]] : (!fir.box<!fir.array<?xf64>>) -> !fir.box<none>
-  ! CHECK-DAG: %[[res:.*]] = fir.call @_FortranADotProductReal8(%[[x_conv]], %[[y_conv]], %{{[0-9]+}}, %{{.*}}) {{.*}}: (!fir.box<none>, !fir.box<none>, !fir.ref<i8>, i32) -> f64
+  ! CHECK: fir.call @_FortranADotProductReal8(%{{.*}}, %{{.*}}, %{{[0-9]+}}, %{{.*}}) {{.*}}: (!fir.box<none>, !fir.box<none>, !fir.ref<i8>, i32) -> f64
   z = dot_product(x,y)
 end subroutine
 
 ! CHECK-LABEL: dot_prod_complex_default
-! CHECK-SAME: %[[x:arg0]]: !fir.box<!fir.array<?xcomplex<f32>>>
-! CHECK-SAME: %[[y:arg1]]: !fir.box<!fir.array<?xcomplex<f32>>>
-! CHECK-SAME: %[[z:arg2]]: !fir.box<!fir.array<?xcomplex<f32>>>
 subroutine dot_prod_complex_default (x, y, z)
   complex, dimension(1:) :: x,y
   complex, dimension(1:) :: z
-  ! CHECK-DAG: %[[res:.*]] = fir.alloca complex<f32>
-  ! CHECK-DAG: %[[x_conv:.*]] = fir.convert %[[x]] : (!fir.box<!fir.array<?xcomplex<f32>>>) -> !fir.box<none>
-  ! CHECK-DAG: %[[y_conv:.*]] = fir.convert %[[y]] : (!fir.box<!fir.array<?xcomplex<f32>>>) -> !fir.box<none>
-  ! CHECK-DAG: fir.call @_FortranACppDotProductComplex4(%[[res]], %[[x_conv]], %[[y_conv]], %{{[0-9]+}}, %{{.*}}) {{.*}}: (!fir.ref<complex<f32>>, !fir.box<none>, !fir.box<none>, !fir.ref<i8>, i32) -> ()
+  ! CHECK: %[[res:.*]] = fir.alloca complex<f32>
+  ! CHECK: fir.call @_FortranACppDotProductComplex4(%[[res]], %{{.*}}, %{{.*}}, %{{[0-9]+}}, %{{.*}}) {{.*}}: (!fir.ref<complex<f32>>, !fir.box<none>, !fir.box<none>, !fir.ref<i8>, i32) -> ()
   z = dot_product(x,y)
 end subroutine
 
 ! CHECK-LABEL: dot_prod_complex_kind_4
-! CHECK-SAME: %[[x:arg0]]: !fir.box<!fir.array<?xcomplex<f32>>>
-! CHECK-SAME: %[[y:arg1]]: !fir.box<!fir.array<?xcomplex<f32>>>
-! CHECK-SAME: %[[z:arg2]]: !fir.box<!fir.array<?xcomplex<f32>>>
 subroutine dot_prod_complex_kind_4 (x, y, z)
   complex(kind=4), dimension(1:) :: x,y
   complex(kind=4), dimension(1:) :: z
-  ! CHECK-DAG: %[[res:.*]] = fir.alloca complex<f32>
-  ! CHECK-DAG: %[[x_conv:.*]] = fir.convert %[[x]] : (!fir.box<!fir.array<?xcomplex<f32>>>) -> !fir.box<none>
-  ! CHECK-DAG: %[[y_conv:.*]] = fir.convert %[[y]] : (!fir.box<!fir.array<?xcomplex<f32>>>) -> !fir.box<none>
-  ! CHECK-DAG: fir.call @_FortranACppDotProductComplex4(%[[res]], %[[x_conv]], %[[y_conv]], %{{[0-9]+}}, %{{.*}}) {{.*}}: (!fir.ref<complex<f32>>, !fir.box<none>, !fir.box<none>, !fir.ref<i8>, i32) -> ()
+  ! CHECK: %[[res:.*]] = fir.alloca complex<f32>
+  ! CHECK: fir.call @_FortranACppDotProductComplex4(%[[res]], %{{.*}}, %{{.*}}, %{{[0-9]+}}, %{{.*}}) {{.*}}: (!fir.ref<complex<f32>>, !fir.box<none>, !fir.box<none>, !fir.ref<i8>, i32) -> ()
   z = dot_product(x,y)
 end subroutine
 
 ! CHECK-LABEL: dot_prod_complex_kind_8
-! CHECK-SAME: %[[x:arg0]]: !fir.box<!fir.array<?xcomplex<f64>>>
-! CHECK-SAME: %[[y:arg1]]: !fir.box<!fir.array<?xcomplex<f64>>>
-! CHECK-SAME: %[[z:arg2]]: !fir.box<!fir.array<?xcomplex<f64>>>
 subroutine dot_prod_complex_kind_8 (x, y, z)
   complex(kind=8), dimension(1:) :: x,y
   complex(kind=8), dimension(1:) :: z
-  ! CHECK-DAG: %[[res:.*]] = fir.alloca complex<f64>
-  ! CHECK-DAG: %[[x_conv:.*]] = fir.convert %[[x]] : (!fir.box<!fir.array<?xcomplex<f64>>>) -> !fir.box<none>
-  ! CHECK-DAG: %[[y_conv:.*]] = fir.convert %[[y]] : (!fir.box<!fir.array<?xcomplex<f64>>>) -> !fir.box<none>
-  ! CHECK-DAG: fir.call @_FortranACppDotProductComplex8(%[[res]], %[[x_conv]], %[[y_conv]], %{{[0-9]+}}, %{{.*}}) {{.*}}: (!fir.ref<complex<f64>>, !fir.box<none>, !fir.box<none>, !fir.ref<i8>, i32) -> ()
+  ! CHECK: %[[res:.*]] = fir.alloca complex<f64>
+  ! CHECK: fir.call @_FortranACppDotProductComplex8(%[[res]], %{{.*}}, %{{.*}}, %{{[0-9]+}}, %{{.*}}) {{.*}}: (!fir.ref<complex<f64>>, !fir.box<none>, !fir.box<none>, !fir.ref<i8>, i32) -> ()
   z = dot_product(x,y)
 end subroutine
 
-! CHECK-LABEL: dot_prod_complex_kind_10
-! CHECK-SAME: %[[x:arg0]]: !fir.box<!fir.array<?xcomplex<f80>>>
-! CHECK-SAME: %[[y:arg1]]: !fir.box<!fir.array<?xcomplex<f80>>>
-! CHECK-SAME: %[[z:arg2]]: !fir.box<!fir.array<?xcomplex<f80>>>
+! CHECK-KIND10-LABEL: dot_prod_complex_kind_10
 subroutine dot_prod_complex_kind_10 (x, y, z)
-  complex(kind=10), dimension(1:) :: x,y
-  complex(kind=10), dimension(1:) :: z
-  ! CHECK-DAG: %[[res:.*]] = fir.alloca complex<f80>
-  ! CHECK-DAG: %[[x_conv:.*]] = fir.convert %[[x]] : (!fir.box<!fir.array<?xcomplex<f80>>>) -> !fir.box<none>
-  ! CHECK-DAG: %[[y_conv:.*]] = fir.convert %[[y]] : (!fir.box<!fir.array<?xcomplex<f80>>>) -> !fir.box<none>
-  ! CHECK-DAG: fir.call @_FortranACppDotProductComplex10(%[[res]], %[[x_conv]], %[[y_conv]], %{{[0-9]+}}, %{{.*}}) {{.*}}: (!fir.ref<complex<f80>>, !fir.box<none>, !fir.box<none>, !fir.ref<i8>, i32) -> ()
+  integer, parameter :: kind10 = merge(10, 4, selected_real_kind(p=18).eq.10)
+  complex(kind=kind10), dimension(1:) :: x,y
+  complex(kind=kind10), dimension(1:) :: z
+  ! CHECK-KIND10: %[[res:.*]] = fir.alloca complex<f80>
+  ! CHECK-KIND10: fir.call @_FortranACppDotProductComplex10(%[[res]], %{{.*}}, %{{.*}}, %{{[0-9]+}}, %{{.*}}) {{.*}}: (!fir.ref<complex<f80>>, !fir.box<none>, !fir.box<none>, !fir.ref<i8>, i32) -> ()
   z = dot_product(x,y)
 end subroutine
 
-! CHECK-LABEL: dot_prod_complex_kind_16
-! CHECK-SAME: %[[x:arg0]]: !fir.box<!fir.array<?xcomplex<f128>>>
-! CHECK-SAME: %[[y:arg1]]: !fir.box<!fir.array<?xcomplex<f128>>>
-! CHECK-SAME: %[[z:arg2]]: !fir.box<!fir.array<?xcomplex<f128>>>
+! CHECK-KIND16-LABEL: dot_prod_complex_kind_16
 subroutine dot_prod_complex_kind_16 (x, y, z)
-  complex(kind=16), dimension(1:) :: x,y
-  complex(kind=16), dimension(1:) :: z
-  ! CHECK-DAG: %[[res:.*]] = fir.alloca complex<f128>
-  ! CHECK-DAG: %[[x_conv:.*]] = fir.convert %[[x]] : (!fir.box<!fir.array<?xcomplex<f128>>>) -> !fir.box<none>
-  ! CHECK-DAG: %[[y_conv:.*]] = fir.convert %[[y]] : (!fir.box<!fir.array<?xcomplex<f128>>>) -> !fir.box<none>
-  ! CHECK-DAG: fir.call @_FortranACppDotProductComplex16(%[[res]], %[[x_conv]], %[[y_conv]], %{{[0-9]+}}, %{{.*}}) {{.*}}: (!fir.ref<complex<f128>>, !fir.box<none>, !fir.box<none>, !fir.ref<i8>, i32) -> ()
+  integer, parameter :: kind16 = merge(16, 4, selected_real_kind(p=33).eq.16)
+  complex(kind=kind16), dimension(1:) :: x,y
+  complex(kind=kind16), dimension(1:) :: z
+  ! CHECK-KIND16: %[[res:.*]] = fir.alloca complex<f128>
+  ! CHECK-KIND16: fir.call @_FortranACppDotProductComplex16(%[[res]], %{{.*}}, %{{.*}}, %{{[0-9]+}}, %{{.*}}) {{.*}}: (!fir.ref<complex<f128>>, !fir.box<none>, !fir.box<none>, !fir.ref<i8>, i32) -> ()
   z = dot_product(x,y)
 end subroutine
 
 ! CHECK-LABEL: dot_prod_logical
-! CHECK-SAME: %[[x:arg0]]: !fir.box<!fir.array<?x!fir.logical<4>>>
-! CHECK-SAME: %[[y:arg1]]: !fir.box<!fir.array<?x!fir.logical<4>>>
-! CHECK-SAME: %[[z:arg2]]: !fir.box<!fir.array<?x!fir.logical<4>>>
 subroutine dot_prod_logical (x, y, z)
   logical, dimension(1:) :: x,y
   logical, dimension(1:) :: z
-  ! CHECK-DAG: %[[x_conv:.*]] = fir.convert %[[x]] : (!fir.box<!fir.array<?x!fir.logical<4>>>) -> !fir.box<none>
-  ! CHECK-DAG: %[[y_conv:.*]] = fir.convert %[[y]] : (!fir.box<!fir.array<?x!fir.logical<4>>>) -> !fir.box<none>
-  ! CHECK-DAG: %[[res:.*]] = fir.call @_FortranADotProductLogical(%[[x_conv]], %[[y_conv]], %{{[0-9]+}}, %{{.*}}) {{.*}}: (!fir.box<none>, !fir.box<none>, !fir.ref<i8>, i32) -> i1
+  ! CHECK: fir.call @_FortranADotProductLogical(%{{.*}}, %{{.*}}, %{{[0-9]+}}, %{{.*}}) {{.*}}: (!fir.box<none>, !fir.box<none>, !fir.ref<i8>, i32) -> i1
   z = dot_product(x,y)
 end subroutine
 
 ! CHECK-LABEL: dot_product_mixed_int_real
-! CHECK-SAME: %[[x:arg0]]: !fir.box<!fir.array<?xi32>>
-! CHECK-SAME: %[[y:arg1]]: !fir.box<!fir.array<?xf32>>
-! CHECK-SAME: %[[z:arg2]]: !fir.box<!fir.array<?xf32>>
 subroutine dot_product_mixed_int_real(x, y, z)
   integer, dimension(1:) :: x
   real, dimension(1:) :: y, z
-  ! CHECK-DAG: %[[x_conv:.*]] = fir.convert %[[x]] : (!fir.box<!fir.array<?xi32>>) -> !fir.box<none>
-  ! CHECK-DAG: %[[y_conv:.*]] = fir.convert %[[y]] : (!fir.box<!fir.array<?xf32>>) -> !fir.box<none>
-  ! CHECK-DAG: %[[res:.*]] = fir.call @_FortranADotProductReal4(%[[x_conv]], %[[y_conv]], %{{[0-9]+}}, %{{.*}}) {{.*}}: (!fir.box<none>, !fir.box<none>, !fir.ref<i8>, i32) -> f32
+  ! CHECK: fir.call @_FortranADotProductReal4(%{{.*}}, %{{.*}}, %{{[0-9]+}}, %{{.*}}) {{.*}}: (!fir.box<none>, !fir.box<none>, !fir.ref<i8>, i32) -> f32
   z = dot_product(x,y)
 end subroutine
 
 ! CHECK-LABEL: dot_product_mixed_int_complex
-! CHECK-SAME: %[[x:arg0]]: !fir.box<!fir.array<?xi32>>
-! CHECK-SAME: %[[y:arg1]]: !fir.box<!fir.array<?xcomplex<f32>>>
-! CHECK-SAME: %[[z:arg2]]: !fir.box<!fir.array<?xcomplex<f32>>>
 subroutine dot_product_mixed_int_complex(x, y, z)
   integer, dimension(1:) :: x
   complex, dimension(1:) :: y, z
-  ! CHECK-DAG: %[[res:.*]] = fir.alloca complex<f32>
-  ! CHECK-DAG: %[[x_conv:.*]] = fir.convert %[[x]] : (!fir.box<!fir.array<?xi32>>) -> !fir.box<none>
-  ! CHECK-DAG: %[[y_conv:.*]] = fir.convert %[[y]] : (!fir.box<!fir.array<?xcomplex<f32>>>) -> !fir.box<none>
-  ! CHECK-DAG: fir.call @_FortranACppDotProductComplex4(%[[res]], %[[x_conv]], %[[y_conv]], %{{[0-9]+}}, %{{.*}}) {{.*}}: (!fir.ref<complex<f32>>, !fir.box<none>, !fir.box<none>, !fir.ref<i8>, i32) -> ()
+  ! CHECK: %[[res:.*]] = fir.alloca complex<f32>
+  ! CHECK: fir.call @_FortranACppDotProductComplex4(%[[res]], %{{.*}}, %{{.*}}, %{{[0-9]+}}, %{{.*}}) {{.*}}: (!fir.ref<complex<f32>>, !fir.box<none>, !fir.box<none>, !fir.ref<i8>, i32) -> ()
   z = dot_product(x,y)
 end subroutine
 
 ! CHECK-LABEL: dot_product_mixed_real_complex
-! CHECK-SAME: %[[x:arg0]]: !fir.box<!fir.array<?xf32>>
-! CHECK-SAME: %[[y:arg1]]: !fir.box<!fir.array<?xcomplex<f32>>>
-! CHECK-SAME: %[[z:arg2]]: !fir.box<!fir.array<?xcomplex<f32>>>
 subroutine dot_product_mixed_real_complex(x, y, z)
   real, dimension(1:) :: x
   complex, dimension(1:) :: y, z
-  ! CHECK-DAG: %[[res:.*]] = fir.alloca complex<f32>
-  ! CHECK-DAG: %[[x_conv:.*]] = fir.convert %[[x]] : (!fir.box<!fir.array<?xf32>>) -> !fir.box<none>
-  ! CHECK-DAG: %[[y_conv:.*]] = fir.convert %[[y]] : (!fir.box<!fir.array<?xcomplex<f32>>>) -> !fir.box<none>
-  ! CHECK-DAG: fir.call @_FortranACppDotProductComplex4(%[[res]], %[[x_conv]], %[[y_conv]], %{{[0-9]+}}, %{{.*}}) {{.*}}: (!fir.ref<complex<f32>>, !fir.box<none>, !fir.box<none>, !fir.ref<i8>, i32) -> ()
+  ! CHECK: %[[res:.*]] = fir.alloca complex<f32>
+  ! CHECK: fir.call @_FortranACppDotProductComplex4(%[[res]], %{{.*}}, %{{.*}}, %{{[0-9]+}}, %{{.*}}) {{.*}}: (!fir.ref<complex<f32>>, !fir.box<none>, !fir.box<none>, !fir.ref<i8>, i32) -> ()
   z = dot_product(x,y)
 end subroutine
diff --git a/flang/test/Lower/Intrinsics/exponent.f90 b/flang/test/Lower/Intrinsics/exponent.f90
index e4db238c6d5a2..f5f0603ee13c8 100644
--- a/flang/test/Lower/Intrinsics/exponent.f90
+++ b/flang/test/Lower/Intrinsics/exponent.f90
@@ -1,41 +1,38 @@
-! RUN: bbc -emit-fir %s -o - | FileCheck %s
+! RUN: bbc -emit-hlfir %s -o - | FileCheck %s --check-prefixes=CHECK%if target=x86_64{{.*}} %{,CHECK-KIND10%}%if flang-supports-f128-math %{,CHECK-KIND16%}
 
 ! EXPONENT
-! CHECK-LABEL: exponent_test
-subroutine exponent_test
 
-    integer :: i1, i2, i3, i4
-  ! CHECK: %[[i0:.*]] = fir.alloca i32 {bindc_name = "i1", uniq_name = "_QFexponent_testEi1"}
-  ! CHECK: %[[i1:.*]] = fir.alloca i32 {bindc_name = "i2", uniq_name = "_QFexponent_testEi2"}
-  ! CHECK: %[[i2:.*]] = fir.alloca i32 {bindc_name = "i3", uniq_name = "_QFexponent_testEi3"}
-  ! CHECK: %[[i3:.*]] = fir.alloca i32 {bindc_name = "i4", uniq_name = "_QFexponent_testEi4"}
-  
-    real(kind = 4) :: x1
-    real(kind = 8) :: x2
-    real(kind = 10) :: x3
-    real(kind = 16) :: x4
-  ! CHECK: %[[x0:.*]] = fir.alloca f32 {bindc_name = "x1", uniq_name = "_QFexponent_testEx1"}
-  ! CHECK: %[[x1:.*]] = fir.alloca f64 {bindc_name = "x2", uniq_name = "_QFexponent_testEx2"}
-  ! CHECK: %[[x2:.*]] = fir.alloca f80 {bindc_name = "x3", uniq_name = "_QFexponent_testEx3"}
-  ! CHECK: %[[x3:.*]] = fir.alloca f128 {bindc_name = "x4", uniq_name = "_QFexponent_testEx4"}
-  
-    i1 = exponent(x1)
-  ! CHECK: %[[temp0:.*]] = fir.load %[[x0:.*]] : !fir.ref<f32>
-  ! CHECK: %[[result0:.*]] = fir.call @_FortranAExponent4_4(%[[temp0:.*]]) {{.*}}: (f32) -> i32
-  ! CHECK: fir.store %[[result0:.*]] to %[[i0:.*]] : !fir.ref<i32>
-  
-    i2 = exponent(x2)
-  ! CHECK: %[[temp1:.*]] = fir.load %[[x1:.*]] : !fir.ref<f64>
-  ! CHECK: %[[result1:.*]] = fir.call @_FortranAExponent8_4(%[[temp1:.*]]) {{.*}}: (f64) -> i32
-  ! CHECK: fir.store %[[result1:.*]] to %[[i1:.*]] : !fir.ref<i32>
-  
-    i3 = exponent(x3)
-  ! CHECK: %[[temp2:.*]] = fir.load %[[x2:.*]] : !fir.ref<f80>
-  ! CHECK: %[[result2:.*]] = fir.call @_FortranAExponent10_4(%[[temp2:.*]]) {{.*}}: (f80) -> i32
-  ! CHECK: fir.store %[[result2:.*]] to %[[i2:.*]] : !fir.ref<i32>
-  
-    i4 = exponent(x4)
-  ! CHECK: %[[temp3:.*]] = fir.load %[[x3:.*]] : !fir.ref<f128>
-  ! CHECK: %[[result3:.*]] = fir.call @_FortranAExponent16_4(%[[temp3:.*]]) {{.*}}: (f128) -> i32
-  ! CHECK: fir.store %[[result3:.*]] to %[[i3:.*]] : !fir.ref<i32>
-  end subroutine exponent_test
+! CHECK-LABEL: exponent_test(
+subroutine exponent_test(i1, i2, x4, x8)
+    integer :: i1, i2, i3
+    real(kind = 4) :: x4
+    real(kind = 8) :: x8
+
+    i1 = exponent(x4)
+  ! CHECK: %[[temp0:.*]] = fir.load %{{.*}} : !fir.ref<f32>
+  ! CHECK: fir.call @_FortranAExponent4_4(%[[temp0:.*]]) {{.*}}: (f32) -> i32
+
+    i2 = exponent(x8)
+  ! CHECK: %[[temp1:.*]] = fir.load %{{.*}} : !fir.ref<f64>
+  ! CHECK: fir.call @_FortranAExponent8_4(%[[temp1:.*]]) {{.*}}: (f64) -> i32
+end subroutine exponent_test
+
+! CHECK-KIND10-LABEL: exponent_10(
+subroutine exponent_10(i, x10)
+    integer :: i
+    integer, parameter :: kind10 = merge(10, 4, selected_real_kind(p=18).eq.10)
+    real(kind = kind10) :: x10
+    i = exponent(x10)
+  ! CHECK-KIND10: %[[temp2:.*]] = fir.load %{{.*}} : !fir.ref<f80>
+  ! CHECK-KIND10: fir.call @_FortranAExponent10_4(%[[temp2:.*]]) {{.*}}: (f80) -> i32
+end subroutine
+
+! CHECK-KIND16-LABEL: exponent_16(
+subroutine exponent_16(i, x16)
+    integer :: i
+    integer, parameter :: kind16 = merge(16, 4, selected_real_kind(p=33).eq.16)
+    real(kind = kind16) :: x16
+    i = exponent(x16)
+  ! CHECK-KIND16: %[[temp2:.*]] = fir.load %{{.*}} : !fir.ref<f128>
+  ! CHECK-KIND16: fir.call @_FortranAExponent16_4(%[[temp2:.*]]) {{.*}}: (f128) -> i32
+end subroutine
diff --git a/flang/test/Lower/Intrinsics/fma_real16.f90 b/flang/test/Lower/Intrinsics/fma_real16.f90
index 62cf2fbcefbf1..ecc32635be9dc 100644
--- a/flang/test/Lower/Intrinsics/fma_real16.f90
+++ b/flang/test/Lower/Intrinsics/fma_real16.f90
@@ -1,3 +1,4 @@
+! REQUIRES: flang-supports-f128-math
 ! RUN: bbc -emit-fir %s -o - | FileCheck %s
 ! RUN: bbc --math-runtime=precise -emit-fir %s -o - | FileCheck %s
 ! RUN: %flang_fc1 -emit-fir %s -o - | FileCheck %s
diff --git a/flang/test/Lower/Intrinsics/fraction.f90 b/flang/test/Lower/Intrinsics/fraction.f90
index f9fff725eb27a..594beb97cc5a3 100644
--- a/flang/test/Lower/Intrinsics/fraction.f90
+++ b/flang/test/Lower/Intrinsics/fraction.f90
@@ -1,35 +1,35 @@
-! RUN: bbc -emit-fir %s -o - | FileCheck %s
- 
+! RUN: bbc -emit-hlfir %s -o - | FileCheck %s --check-prefixes=CHECK%if target=x86_64{{.*}} %{,CHECK-KIND10%}%if flang-supports-f128-math %{,CHECK-KIND16%}
+
 ! FRACTION
-! CHECK-LABE: fraction_test
-subroutine fraction_test
 
-    real(kind=4) :: x1 = 178.1387e-4
-    real(kind=8) :: x2 = 178.1387e-4
-    real(kind=10) :: x3 = 178.1387e-4
-    real(kind=16) :: x4 = 178.1387e-4
-  ! CHECK: %[[r0:.*]] = fir.address_of(@_QFfraction_testEx1) : !fir.ref<f32>
-  ! CHECK: %[[r1:.*]] = fir.address_of(@_QFfraction_testEx2) : !fir.ref<f64>
-  ! CHECK: %[[r2:.*]] = fir.address_of(@_QFfraction_testEx3) : !fir.ref<f80>
-  ! CHECK: %[[r3:.*]] = fir.address_of(@_QFfraction_testEx4) : !fir.ref<f128>
-  
-    x1 = fraction(x1)
-  ! CHECK: %[[temp0:.*]] = fir.load %[[r0:.*]] : !fir.ref<f32>
-  ! CHECK: %[[result0:.*]] = fir.call @_FortranAFraction4(%[[temp0:.*]]) {{.*}}: (f32) -> f32
-  ! CHECK: fir.store %[[result0:.*]] to %[[r0:.*]] : !fir.ref<f32>
-  
-    x2 = fraction(x2)
-  ! CHECK: %[[temp1:.*]] = fir.load %[[r1:.*]] : !fir.ref<f64>
-  ! CHECK: %[[result1:.*]] = fir.call @_FortranAFraction8(%[[temp1:.*]]) {{.*}}: (f64) -> f64
-  ! CHECK: fir.store %[[result1:.*]] to %[[r1:.*]] : !fir.ref<f64>
-  
-    x3 = fraction(x3)
-  ! CHECK: %[[temp2:.*]] = fir.load %[[r2:.*]] : !fir.ref<f80>
-  ! CHECK: %[[result2:.*]] = fir.call @_FortranAFraction10(%[[temp2:.*]]) {{.*}}: (f80) -> f80
-  ! CHECK: fir.store %[[result2:.*]] to %[[r2:.*]] : !fir.ref<f80>
-  
-    x4 = fraction(x4)
-  ! CHECK: %[[temp3:.*]] = fir.load %[[r3:.*]] : !fir.ref<f128>
-  ! CHECK: %[[result3:.*]] = fir.call @_FortranAFraction16(%[[temp3:.*]]) {{.*}}: (f128) -> f128
-  ! CHECK: fir.store %[[result3:.*]] to %[[r3:.*]] : !fir.ref<f128>
-  end subroutine fraction_test
+! CHECK-LABEL: fraction_test(
+subroutine fraction_test(res4, res8, x4, x8)
+    real(kind = 4) :: x4, res4
+    real(kind = 8) :: x8, res8
+
+    res4 = fraction(x4)
+  ! CHECK: %[[temp0:.*]] = fir.load %{{.*}} : !fir.ref<f32>
+  ! CHECK: fir.call @_FortranAFraction4(%[[temp0:.*]]) {{.*}}: (f32) -> f32
+
+    res8 = fraction(x8)
+  ! CHECK: %[[temp1:.*]] = fir.load %{{.*}} : !fir.ref<f64>
+  ! CHECK: fir.call @_FortranAFraction8(%[[temp1:.*]]) {{.*}}: (f64) -> f64
+end subroutine fraction_test
+
+! CHECK-KIND10-LABEL: fraction_10(
+subroutine fraction_10(res10, x10)
+    integer, parameter :: kind10 = merge(10, 4, selected_real_kind(p=18).eq.10)
+    real(kind = kind10) :: x10, res10
+    res10 = fraction(x10)
+  ! CHECK-KIND10: %[[temp2:.*]] = fir.load %{{.*}} : !fir.ref<f80>
+  ! CHECK-KIND10: fir.call @_FortranAFraction10(%[[temp2:.*]]) {{.*}}: (f80) -> f80
+end subroutine
+
+! CHECK-KIND16-LABEL: fraction_16(
+subroutine fraction_16(res16, x16)
+    integer, parameter :: kind16 = merge(16, 4, selected_real_kind(p=33).eq.16)
+    real(kind = kind16) :: x16, res16
+    res16 = fraction(x16)
+  ! CHECK-KIND16: %[[temp2:.*]] = fir.load %{{.*}} : !fir.ref<f128>
+  ! CHECK-KIND16: fir.call @_FortranAFraction16(%[[temp2:.*]]) {{.*}}: (f128) -> f128
+end subroutine
diff --git a/flang/test/Lower/Intrinsics/ieee_class_queries.f90 b/flang/test/Lower/Intrinsics/ieee_class_queries.f90
index b2f9df83a902a..aae8071b6a8b5 100644
--- a/flang/test/Lower/Intrinsics/ieee_class_queries.f90
+++ b/flang/test/Lower/Intrinsics/ieee_class_queries.f90
@@ -1,5 +1,6 @@
 ! REQUIRES: flang-supports-f128-math
-! RUN: bbc -emit-fir -o - %s | FileCheck %s
+! REQUIRES: x86_64-registered-target
+! RUN: bbc -target x86_64-unknown-linux-gnu -emit-fir -o - %s | FileCheck %s
 
   ! CHECK-LABEL: func @_QQmain
   use ieee_arithmetic, only: ieee_is_finite, ieee_is_nan, ieee_is_negative, &
@@ -8,7 +9,6 @@
   real(3) :: x3 = -3.0
   real(4) :: x4 = -4.0
   real(8) :: x8 = -8.0
-  real(10) :: x10 = -10.0
   real(16) :: x16 = -16.0
 
   ! CHECK:     "llvm.intr.is.fpclass"(%{{.*}}) <{bit = 504 : i32}> : (f16) -> i1
diff --git a/flang/test/Lower/Intrinsics/ieee_is_normal.f90 b/flang/test/Lower/Intrinsics/ieee_is_normal.f90
index 9b864c9a9849c..d55b2e3c08561 100644
--- a/flang/test/Lower/Intrinsics/ieee_is_normal.f90
+++ b/flang/test/Lower/Intrinsics/ieee_is_normal.f90
@@ -1,5 +1,4 @@
-! RUN: bbc -emit-fir %s -o - | FileCheck %s
-! RUN: flang -fc1 -emit-fir %s -o - | FileCheck %s
+! RUN: bbc -emit-fir %s -o - | FileCheck %s --check-prefixes=CHECK%if target=x86_64{{.*}} %{,CHECK-KIND10%}%if flang-supports-f128-math %{,CHECK-KIND16%}
 
 ! CHECK-LABEL: ieee_is_normal_f16
 subroutine ieee_is_normal_f16(r)
@@ -39,20 +38,22 @@ subroutine ieee_is_normal_f64(r)
   ! CHECK: fir.convert %[[l]] : (i1) -> !fir.logical<4>
 end subroutine ieee_is_normal_f64
 
-! CHECK-LABEL: ieee_is_normal_f80
+! CHECK-KIND10-LABEL: ieee_is_normal_f80
 subroutine ieee_is_normal_f80(r)
   use ieee_arithmetic
-  real(KIND=10) :: r
+  integer, parameter :: kind10 = merge(10, 4, selected_real_kind(p=18).eq.10)
+  real(KIND=kind10) :: r
   i = ieee_is_normal(r)
-  ! CHECK: %[[l:.*]] = "llvm.intr.is.fpclass"(%{{.*}}) <{bit = 360 : i32}> : (f80) -> i1
-  ! CHECK: fir.convert %[[l]] : (i1) -> !fir.logical<4>
+  ! CHECK-KIND10: %[[l:.*]] = "llvm.intr.is.fpclass"(%{{.*}}) <{bit = 360 : i32}> : (f80) -> i1
+  ! CHECK-KIND10: fir.convert %[[l]] : (i1) -> !fir.logical<4>
 end subroutine ieee_is_normal_f80
 
-! CHECK-LABEL: ieee_is_normal_f128
+! CHECK-KIND16-LABEL: ieee_is_normal_f128
 subroutine ieee_is_normal_f128(r)
   use ieee_arithmetic
-  real(KIND=16) :: r
+  integer, parameter :: kind16 = merge(16, 4, selected_real_kind(p=33).eq.16)
+  real(KIND=kind16) :: r
   i = ieee_is_normal(r)
-  ! CHECK: %[[l:.*]] = "llvm.intr.is.fpclass"(%{{.*}}) <{bit = 360 : i32}> : (f128) -> i1
-  ! CHECK: fir.convert %[[l]] : (i1) -> !fir.logical<4>
+  ! CHECK-KIND16: %[[l:.*]] = "llvm.intr.is.fpclass"(%{{.*}}) <{bit = 360 : i32}> : (f128) -> i1
+  ! CHECK-KIND16: fir.convert %[[l]] : (i1) -> !fir.logical<4>
 end subroutine ieee_is_normal_f128
diff --git a/flang/test/Lower/Intrinsics/ieee_next.f90 b/flang/test/Lower/Intrinsics/ieee_next.f90
index fa9692b83bc87..aa545967f9bcd 100644
--- a/flang/test/Lower/Intrinsics/ieee_next.f90
+++ b/flang/test/Lower/Intrinsics/ieee_next.f90
@@ -1,284 +1,394 @@
-! RUN: bbc -emit-fir -o - %s | FileCheck %s
+! RUN: bbc -emit-hlfir %s -o - | FileCheck %s --check-prefixes=CHECK%if target=x86_64{{.*}} %{,CHECK-KIND10%}%if flang-supports-f128-math %{,CHECK-KIND16%}
 
-! CHECK-LABEL: c.func @_QQmain
-program p
+module ieee_next_tests
   use ieee_arithmetic, only: ieee_value, ieee_negative_inf, ieee_positive_inf
   use ieee_arithmetic, only: ieee_next_after, ieee_next_down, ieee_next_up
   implicit none
-  ! CHECK-DAG: %[[V_4:[0-9]+]] = fir.alloca f80 {bindc_name = "r10", uniq_name = "_QFEr10"}
-  ! CHECK-DAG: %[[V_5:[0-9]+]] = fir.declare %[[V_4]] {uniq_name = "_QFEr10"} : (!fir.ref<f80>) -> !fir.ref<f80>
-  ! CHECK-DAG: %[[V_6:[0-9]+]] = fir.alloca f128 {bindc_name = "r16", uniq_name = "_QFEr16"}
-  ! CHECK-DAG: %[[V_7:[0-9]+]] = fir.declare %[[V_6]] {uniq_name = "_QFEr16"} : (!fir.ref<f128>) -> !fir.ref<f128>
-  ! CHECK-DAG: %[[V_8:[0-9]+]] = fir.alloca f16 {bindc_name = "r2", uniq_name = "_QFEr2"}
-  ! CHECK-DAG: %[[V_9:[0-9]+]] = fir.declare %[[V_8]] {uniq_name = "_QFEr2"} : (!fir.ref<f16>) -> !fir.ref<f16>
-  ! CHECK-DAG: %[[V_10:[0-9]+]] = fir.alloca bf16 {bindc_name = "r3", uniq_name = "_QFEr3"}
-  ! CHECK-DAG: %[[V_11:[0-9]+]] = fir.declare %[[V_10]] {uniq_name = "_QFEr3"} : (!fir.ref<bf16>) -> !fir.ref<bf16>
-  ! CHECK-DAG: %[[V_12:[0-9]+]] = fir.alloca f32 {bindc_name = "r4", uniq_name = "_QFEr4"}
-  ! CHECK-DAG: %[[V_13:[0-9]+]] = fir.declare %[[V_12]] {uniq_name = "_QFEr4"} : (!fir.ref<f32>) -> !fir.ref<f32>
-  ! CHECK-DAG: %[[V_14:[0-9]+]] = fir.alloca f64 {bindc_name = "r8", uniq_name = "_QFEr8"}
-  ! CHECK-DAG: %[[V_15:[0-9]+]] = fir.declare %[[V_14]] {uniq_name = "_QFEr8"} : (!fir.ref<f64>) -> !fir.ref<f64>
-  ! CHECK-DAG: %[[V_16:[0-9]+]] = fir.address_of(@_QFEx10) : !fir.ref<f80>
-  ! CHECK-DAG: %[[V_17:[0-9]+]] = fir.declare %[[V_16]] {uniq_name = "_QFEx10"} : (!fir.ref<f80>) -> !fir.ref<f80>
-  ! CHECK-DAG: %[[V_18:[0-9]+]] = fir.alloca f128 {bindc_name = "x16", uniq_name = "_QFEx16"}
-  ! CHECK-DAG: %[[V_19:[0-9]+]] = fir.declare %[[V_18]] {uniq_name = "_QFEx16"} : (!fir.ref<f128>) -> !fir.ref<f128>
-  ! CHECK-DAG: %[[V_20:[0-9]+]] = fir.alloca f16 {bindc_name = "x2", uniq_name = "_QFEx2"}
-  ! CHECK-DAG: %[[V_21:[0-9]+]] = fir.declare %[[V_20]] {uniq_name = "_QFEx2"} : (!fir.ref<f16>) -> !fir.ref<f16>
-  ! CHECK-DAG: %[[V_22:[0-9]+]] = fir.address_of(@_QFEx3) : !fir.ref<bf16>
-  ! CHECK-DAG: %[[V_23:[0-9]+]] = fir.declare %[[V_22]] {uniq_name = "_QFEx3"} : (!fir.ref<bf16>) -> !fir.ref<bf16>
-  ! CHECK-DAG: %[[V_24:[0-9]+]] = fir.address_of(@_QFEx4) : !fir.ref<f32>
-  ! CHECK-DAG: %[[V_25:[0-9]+]] = fir.declare %[[V_24]] {uniq_name = "_QFEx4"} : (!fir.ref<f32>) -> !fir.ref<f32>
-  ! CHECK-DAG: %[[V_26:[0-9]+]] = fir.address_of(@_QFEx8) : !fir.ref<f64>
-  ! CHECK-DAG: %[[V_27:[0-9]+]] = fir.declare %[[V_26]] {uniq_name = "_QFEx8"} : (!fir.ref<f64>) -> !fir.ref<f64>
-  real(2)  ::  r2,  x2
-  real(3)  ::  r3,  x3 = -huge(x3)
-  real(4)  ::  r4,  x4 = -0.
-  real(8)  ::  r8,  x8 =  0.
-  real(10) :: r10, x10 =  huge(x10)
-  real(16) :: r16, x16
-
-  x2  = ieee_value(x2, ieee_negative_inf)
-  x16 = ieee_value(x2, ieee_positive_inf)
+  integer, parameter :: kind10 = merge(10, 4, selected_real_kind(p=18).eq.10)
+  integer, parameter :: kind16 = merge(16, 4, selected_real_kind(p=33).eq.16)
+contains
 
-  ! CHECK:     %[[V_45:[0-9]+]] = fir.load %[[V_21]] : !fir.ref<f16>
-  ! CHECK:     %[[V_46:[0-9]+]] = fir.load %[[V_17]] : !fir.ref<f80>
-  ! CHECK-DAG: %[[V_47:[0-9]+]] = fir.coordinate_of %{{.*}}, %c2{{.*}} : (!fir.ref<!fir.array<12xi16>>, i8) -> !fir.ref<i16>
-  ! CHECK-DAG: %[[V_48:[0-9]+]] = fir.load %[[V_47]] : !fir.ref<i16>
-  ! CHECK-DAG: %[[V_49:[0-9]+]] = arith.bitcast %[[V_48]] : i16 to f16
-  ! CHECK-DAG: %[[V_50:[0-9]+]] = "llvm.intr.is.fpclass"(%[[V_46]]) <{bit = 3 : i32}> : (f80) -> i1
-  ! CHECK:     %[[V_51:[0-9]+]] = arith.select %[[V_50]], %[[V_49]], %[[V_45]] : f16
-  ! CHECK:     %[[V_52:[0-9]+]] = "llvm.intr.is.fpclass"(%[[V_51]]) <{bit = 3 : i32}> : (f16) -> i1
-  ! CHECK:     %[[V_53:[0-9]+]] = fir.convert %[[V_51]] : (f16) -> f80
-  ! CHECK:     %[[V_54:[0-9]+]] = arith.cmpf oeq, %[[V_53]], %[[V_46]] fastmath<contract> : f80
-  ! CHECK:     %[[V_55:[0-9]+]] = arith.ori %[[V_52]], %[[V_54]] : i1
-  ! CHECK:     %[[V_56:[0-9]+]] = arith.cmpf olt, %[[V_53]], %[[V_46]] fastmath<contract> : f80
-  ! CHECK:     %[[V_57:[0-9]+]] = arith.bitcast %[[V_45]] : f16 to i16
-  ! CHECK:     %[[V_58:[0-9]+]] = arith.shrui %[[V_57]], %c15{{.*}} : i16
-  ! CHECK:     %[[V_59:[0-9]+]] = fir.convert %[[V_58]] : (i16) -> i1
-  ! CHECK:     %[[V_60:[0-9]+]] = arith.cmpi ne, %[[V_56]], %[[V_59]] : i1
-  ! CHECK:     %[[V_61:[0-9]+]] = "llvm.intr.is.fpclass"(%[[V_51]]) <{bit = 516 : i32}> : (f16) -> i1
-  ! CHECK:     %[[V_62:[0-9]+]] = arith.andi %[[V_61]], %[[V_60]] : i1
-  ! CHECK:     %[[V_63:[0-9]+]] = arith.ori %[[V_55]], %[[V_62]] : i1
-  ! CHECK:     %[[V_64:[0-9]+]] = fir.if %[[V_63]] -> (f16) {
-  ! CHECK:       fir.result %[[V_51]] : f16
-  ! CHECK:     } else {
-  ! CHECK:       %[[V_202:[0-9]+]] = arith.cmpf oeq, %[[V_51]], %cst{{[_0-9]*}} fastmath<contract> : f16
-  ! CHECK:       %[[V_203:[0-9]+]] = fir.if %[[V_202]] -> (f16) {
-  ! CHECK:         %[[V_204:[0-9]+]] = arith.select %[[V_56]], %cst{{[_0-9]*}}, %cst{{[_0-9]*}} : f16
-  ! CHECK:         %[[V_205:[0-9]+]] = fir.call @_FortranAMapException(%c48{{.*}}) fastmath<contract> : (i32) -> i32
-  ! CHECK:                             fir.call @feraiseexcept(%[[V_205]]) fastmath<contract> : (i32) -> i32
-  ! CHECK:         fir.result %[[V_204]] : f16
-  ! CHECK:       } else {
-  ! CHECK:         %[[V_204:[0-9]+]] = arith.bitcast %[[V_51]] : f16 to i16
-  ! CHECK-DAG:     %[[V_205:[0-9]+]] = arith.subi %[[V_204]], %c1{{.*}} : i16
-  ! CHECK-DAG:     %[[V_206:[0-9]+]] = arith.addi %[[V_204]], %c1{{.*}} : i16
-  ! CHECK:         %[[V_207:[0-9]+]] = arith.select %[[V_60]], %[[V_206]], %[[V_205]] : i16
-  ! CHECK:         %[[V_208:[0-9]+]] = arith.bitcast %[[V_207]] : i16 to f16
-  ! CHECK:         %[[V_209:[0-9]+]] = "llvm.intr.is.fpclass"(%[[V_208]]) <{bit = 516 : i32}> : (f16) -> i1
-  ! CHECK:         fir.if %[[V_209]] {
-  ! CHECK:           %[[V_211:[0-9]+]] = fir.call @_FortranAMapException(%c40{{.*}}) fastmath<contract> : (i32) -> i32
-  ! CHECK:                               fir.call @feraiseexcept(%[[V_211]]) fastmath<contract> : (i32) -> i32
-  ! CHECK:         }
-  ! CHECK:         %[[V_210:[0-9]+]] = "llvm.intr.is.fpclass"(%[[V_208]]) <{bit = 144 : i32}> : (f16) -> i1
-  ! CHECK:         fir.if %[[V_210]] {
-  ! CHECK:           %[[V_211:[0-9]+]] = fir.call @_FortranAMapException(%c48{{.*}}) fastmath<contract> : (i32) -> i32
-  ! CHECK:                               fir.call @feraiseexcept(%[[V_211]]) fastmath<contract> : (i32) -> i32
-  ! CHECK:         }
-  ! CHECK:         fir.result %[[V_208]] : f16
-  ! CHECK:       }
-  ! CHECK:       fir.result %[[V_203]] : f16
-  ! CHECK:     }
-  ! CHECK:     fir.store %[[V_64]] to %[[V_9]] : !fir.ref<f16>
+subroutine test1(r2, x2, x10)
+  real(2)  ::  r2,  x2
+  real(kind10) :: x10
   r2 = ieee_next_after(x2, x10)
-  print "('after:  ', z4.4, ' -> ', z4.4, ' = ', g0)", x2, r2, r2
+end subroutine
+!CHECK-KIND10-LABEL:   func.func @_QMieee_next_testsPtest1(
+!CHECK-KIND10:           %[[VAL_12:.*]]:2 = hlfir.declare {{.*}}r2"
+!CHECK-KIND10:           %[[VAL_13:.*]]:2 = hlfir.declare {{.*}}x10"
+!CHECK-KIND10:           %[[VAL_14:.*]]:2 = hlfir.declare {{.*}}x2"
+!CHECK-KIND10:           %[[VAL_15:.*]] = fir.load %[[VAL_14]]#0 : !fir.ref<f16>
+!CHECK-KIND10:           %[[VAL_16:.*]] = fir.load %[[VAL_13]]#0 : !fir.ref<f80>
+!CHECK-KIND10-DAG:       %[[VAL_17:.*]] = "llvm.intr.is.fpclass"(%[[VAL_16]]) <{bit = 3 : i32}> : (f80) -> i1
+!CHECK-KIND10-DAG:       %[[VAL_18:.*]] = arith.constant 2 : i8
+!CHECK-KIND10-DAG:       %[[VAL_19:.*]] = fir.address_of(@_FortranAIeeeValueTable_2) : !fir.ref<!fir.array<12xi16>>
+!CHECK-KIND10-DAG:       %[[VAL_20:.*]] = fir.coordinate_of %[[VAL_19]], %[[VAL_18]] : (!fir.ref<!fir.array<12xi16>>, i8) -> !fir.ref<i16>
+!CHECK-KIND10-DAG:       %[[VAL_21:.*]] = fir.load %[[VAL_20]] : !fir.ref<i16>
+!CHECK-KIND10-DAG:       %[[VAL_22:.*]] = arith.bitcast %[[VAL_21]] : i16 to f16
+!CHECK-KIND10:           %[[VAL_23:.*]] = arith.select %[[VAL_17]], %[[VAL_22]], %[[VAL_15]] : f16
+!CHECK-KIND10:           %[[VAL_24:.*]] = "llvm.intr.is.fpclass"(%[[VAL_23]]) <{bit = 3 : i32}> : (f16) -> i1
+!CHECK-KIND10:           %[[VAL_25:.*]] = arith.constant 1 : i16
+!CHECK-KIND10:           %[[VAL_26:.*]] = fir.convert %[[VAL_23]] : (f16) -> f32
+!CHECK-KIND10:           %[[VAL_27:.*]] = fir.convert %[[VAL_26]] : (f32) -> f80
+!CHECK-KIND10:           %[[VAL_28:.*]] = arith.cmpf oeq, %[[VAL_27]], %[[VAL_16]] fastmath<contract> : f80
+!CHECK-KIND10:           %[[VAL_29:.*]] = arith.ori %[[VAL_24]], %[[VAL_28]] : i1
+!CHECK-KIND10:           %[[VAL_30:.*]] = arith.cmpf olt, %[[VAL_27]], %[[VAL_16]] fastmath<contract> : f80
+!CHECK-KIND10:           %[[VAL_31:.*]] = arith.bitcast %[[VAL_15]] : f16 to i16
+!CHECK-KIND10:           %[[VAL_32:.*]] = arith.constant 15 : i16
+!CHECK-KIND10:           %[[VAL_33:.*]] = arith.shrui %[[VAL_31]], %[[VAL_32]] : i16
+!CHECK-KIND10:           %[[VAL_34:.*]] = fir.convert %[[VAL_33]] : (i16) -> i1
+!CHECK-KIND10:           %[[VAL_35:.*]] = arith.cmpi ne, %[[VAL_30]], %[[VAL_34]] : i1
+!CHECK-KIND10:           %[[VAL_36:.*]] = "llvm.intr.is.fpclass"(%[[VAL_23]]) <{bit = 516 : i32}> : (f16) -> i1
+!CHECK-KIND10:           %[[VAL_37:.*]] = arith.andi %[[VAL_36]], %[[VAL_35]] : i1
+!CHECK-KIND10:           %[[VAL_38:.*]] = arith.ori %[[VAL_29]], %[[VAL_37]] : i1
+!CHECK-KIND10:           %[[VAL_39:.*]] = fir.if %[[VAL_38]] -> (f16) {
+!CHECK-KIND10:             fir.result %[[VAL_23]] : f16
+!CHECK-KIND10:           } else {
+!CHECK-KIND10:             %[[VAL_40:.*]] = arith.constant 0.000000e+00 : f16
+!CHECK-KIND10:             %[[VAL_41:.*]] = arith.cmpf oeq, %[[VAL_23]], %[[VAL_40]] fastmath<contract> : f16
+!CHECK-KIND10:             %[[VAL_42:.*]] = fir.if %[[VAL_41]] -> (f16) {
+!CHECK-KIND10:               %[[VAL_43:.*]] = arith.bitcast %[[VAL_25]] : i16 to f16
+!CHECK-KIND10:               %[[VAL_44:.*]] = arith.constant -32767 : i16
+!CHECK-KIND10:               %[[VAL_45:.*]] = arith.bitcast %[[VAL_44]] : i16 to f16
+!CHECK-KIND10:               %[[VAL_46:.*]] = arith.select %[[VAL_30]], %[[VAL_43]], %[[VAL_45]] : f16
+!CHECK-KIND10:               %[[VAL_47:.*]] = arith.constant 48 : i32
+!CHECK-KIND10:               %[[VAL_48:.*]] = fir.call @_FortranAMapException(%[[VAL_47]]) fastmath<contract> : (i32) -> i32
+!CHECK-KIND10:               %[[VAL_49:.*]] = fir.call @feraiseexcept(%[[VAL_48]]) fastmath<contract> : (i32) -> i32
+!CHECK-KIND10:               fir.result %[[VAL_46]] : f16
+!CHECK-KIND10:             } else {
+!CHECK-KIND10:               %[[VAL_50:.*]] = arith.bitcast %[[VAL_23]] : f16 to i16
+!CHECK-KIND10-DAG:           %[[VAL_51:.*]] = arith.addi %[[VAL_50]], %[[VAL_25]] : i16
+!CHECK-KIND10-DAG:           %[[VAL_52:.*]] = arith.subi %[[VAL_50]], %[[VAL_25]] : i16
+!CHECK-KIND10:               %[[VAL_53:.*]] = arith.select %[[VAL_35]], %[[VAL_51]], %[[VAL_52]] : i16
+!CHECK-KIND10:               %[[VAL_54:.*]] = arith.bitcast %[[VAL_53]] : i16 to f16
+!CHECK-KIND10:               %[[VAL_55:.*]] = "llvm.intr.is.fpclass"(%[[VAL_54]]) <{bit = 516 : i32}> : (f16) -> i1
+!CHECK-KIND10:               fir.if %[[VAL_55]] {
+!CHECK-KIND10:                 %[[VAL_56:.*]] = arith.constant 40 : i32
+!CHECK-KIND10:                 %[[VAL_57:.*]] = fir.call @_FortranAMapException(%[[VAL_56]]) fastmath<contract> : (i32) -> i32
+!CHECK-KIND10:                 %[[VAL_58:.*]] = fir.call @feraiseexcept(%[[VAL_57]]) fastmath<contract> : (i32) -> i32
+!CHECK-KIND10:               }
+!CHECK-KIND10:               %[[VAL_59:.*]] = "llvm.intr.is.fpclass"(%[[VAL_54]]) <{bit = 144 : i32}> : (f16) -> i1
+!CHECK-KIND10:               fir.if %[[VAL_59]] {
+!CHECK-KIND10:                 %[[VAL_60:.*]] = arith.constant 48 : i32
+!CHECK-KIND10:                 %[[VAL_61:.*]] = fir.call @_FortranAMapException(%[[VAL_60]]) fastmath<contract> : (i32) -> i32
+!CHECK-KIND10:                 %[[VAL_62:.*]] = fir.call @feraiseexcept(%[[VAL_61]]) fastmath<contract> : (i32) -> i32
+!CHECK-KIND10:               }
+!CHECK-KIND10:               fir.result %[[VAL_54]] : f16
+!CHECK-KIND10:             }
+!CHECK-KIND10:             fir.result %[[VAL_42]] : f16
+!CHECK-KIND10:           }
+!CHECK-KIND10:           hlfir.assign %[[VAL_39]] to %[[VAL_12]]#0 : f16, !fir.ref<f16>
+!CHECK-KIND10:           return
+!CHECK-KIND10:         }
 
-  ! CHECK:     %[[V_81:[0-9]+]] = fir.load %[[V_23]] : !fir.ref<bf16>
-  ! CHECK:     %[[V_82:[0-9]+]] = "llvm.intr.is.fpclass"(%[[V_81]]) <{bit = 3 : i32}> : (bf16) -> i1
-  ! CHECK:     %[[V_83:[0-9]+]] = fir.convert %[[V_81]] : (bf16) -> f32
-  ! CHECK:     %[[V_84:[0-9]+]] = arith.bitcast %[[V_83]] : f32 to i32
-  ! CHECK:     %[[V_85:[0-9]+]] = arith.shrui %[[V_84]], %c31{{.*}} : i32
-  ! CHECK:     %[[V_86:[0-9]+]] = fir.convert %[[V_85]] : (i32) -> i1
-  ! CHECK:     %[[V_87:[0-9]+]] = arith.cmpi ne, %[[V_86]], %true{{[_0-9]*}} : i1
-  ! CHECK:     %[[V_88:[0-9]+]] = "llvm.intr.is.fpclass"(%[[V_81]]) <{bit = 516 : i32}> : (bf16) -> i1
-  ! CHECK:     %[[V_89:[0-9]+]] = arith.andi %[[V_88]], %[[V_87]] : i1
-  ! CHECK:     %[[V_90:[0-9]+]] = arith.ori %[[V_82]], %[[V_89]] : i1
-  ! CHECK:     %[[V_91:[0-9]+]] = fir.if %[[V_90]] -> (bf16) {
-  ! CHECK:       %[[V_202:[0-9]+]] = "llvm.intr.is.fpclass"(%[[V_81]]) <{bit = 1 : i32}> : (bf16) -> i1
-  ! CHECK:       fir.if %[[V_202]] {
-  ! CHECK:         %[[V_203:[0-9]+]] = fir.call @_FortranAMapException(%c1{{.*}}) fastmath<contract> : (i32) -> i32
-  ! CHECK:                             fir.call @feraiseexcept(%[[V_203]]) fastmath<contract> : (i32) -> i32
-  ! CHECK:       }
-  ! CHECK:       fir.result %[[V_81]] : bf16
-  ! CHECK:     } else {
-  ! CHECK:       %[[V_202:[0-9]+]] = arith.cmpf oeq, %[[V_81]], %cst{{[_0-9]*}} fastmath<contract> : bf16
-  ! CHECK:       %[[V_203:[0-9]+]] = fir.if %[[V_202]] -> (bf16) {
-  ! CHECK:         fir.result %cst{{[_0-9]*}} : bf16
-  ! CHECK:       } else {
-  ! CHECK:         %[[V_204:[0-9]+]] = arith.bitcast %[[V_81]] : bf16 to i16
-  ! CHECK-DAG:     %[[V_205:[0-9]+]] = arith.subi %[[V_204]], %c1{{.*}} : i16
-  ! CHECK-DAG:     %[[V_206:[0-9]+]] = arith.addi %[[V_204]], %c1{{.*}} : i16
-  ! CHECK:         %[[V_207:[0-9]+]] = arith.select %[[V_87]], %[[V_206]], %[[V_205]] : i16
-  ! CHECK:         %[[V_208:[0-9]+]] = arith.bitcast %[[V_207]] : i16 to bf16
-  ! CHECK:         fir.result %[[V_208]] : bf16
-  ! CHECK:       }
-  ! CHECK:       fir.result %[[V_203]] : bf16
-  ! CHECK:     }
-  ! CHECK:     fir.store %[[V_91]] to %[[V_11]] : !fir.ref<bf16>
+subroutine test2(r3, x3)
+  real(3)  ::  r3,  x3
   r3 = ieee_next_up(x3)
-  print "('up:     ', z4.4, ' -> ', z4.4, ' = ', g0)", x3, r3, r3
+end subroutine
+!CHECK-LABEL:   func.func @_QMieee_next_testsPtest2(
+!CHECK:           %[[VAL_11:.*]]:2 = hlfir.declare {{.*}}r3"
+!CHECK:           %[[VAL_12:.*]]:2 = hlfir.declare {{.*}}x3"
+!CHECK:           %[[VAL_13:.*]] = fir.load %[[VAL_12]]#0 : !fir.ref<bf16>
+!CHECK:           %[[VAL_14:.*]] = "llvm.intr.is.fpclass"(%[[VAL_13]]) <{bit = 3 : i32}> : (bf16) -> i1
+!CHECK:           %[[VAL_15:.*]] = arith.constant 1 : i16
+!CHECK:           %[[VAL_16:.*]] = arith.constant true
+!CHECK:           %[[VAL_17:.*]] = fir.convert %[[VAL_13]] : (bf16) -> f32
+!CHECK:           %[[VAL_18:.*]] = arith.bitcast %[[VAL_17]] : f32 to i32
+!CHECK:           %[[VAL_19:.*]] = arith.constant 31 : i32
+!CHECK:           %[[VAL_20:.*]] = arith.shrui %[[VAL_18]], %[[VAL_19]] : i32
+!CHECK:           %[[VAL_21:.*]] = fir.convert %[[VAL_20]] : (i32) -> i1
+!CHECK:           %[[VAL_22:.*]] = arith.cmpi ne, %[[VAL_16]], %[[VAL_21]] : i1
+!CHECK:           %[[VAL_23:.*]] = "llvm.intr.is.fpclass"(%[[VAL_13]]) <{bit = 516 : i32}> : (bf16) -> i1
+!CHECK:           %[[VAL_24:.*]] = arith.andi %[[VAL_23]], %[[VAL_22]] : i1
+!CHECK:           %[[VAL_25:.*]] = arith.ori %[[VAL_14]], %[[VAL_24]] : i1
+!CHECK:           %[[VAL_26:.*]] = fir.if %[[VAL_25]] -> (bf16) {
+!CHECK:             %[[VAL_27:.*]] = "llvm.intr.is.fpclass"(%[[VAL_13]]) <{bit = 1 : i32}> : (bf16) -> i1
+!CHECK:             fir.if %[[VAL_27]] {
+!CHECK:               %[[VAL_28:.*]] = arith.constant 1 : i32
+!CHECK:               %[[VAL_29:.*]] = fir.call @_FortranAMapException(%[[VAL_28]]) fastmath<contract> : (i32) -> i32
+!CHECK:               %[[VAL_30:.*]] = fir.call @feraiseexcept(%[[VAL_29]]) fastmath<contract> : (i32) -> i32
+!CHECK:             }
+!CHECK:             fir.result %[[VAL_13]] : bf16
+!CHECK:           } else {
+!CHECK:             %[[VAL_31:.*]] = arith.constant 0.000000e+00 : bf16
+!CHECK:             %[[VAL_32:.*]] = arith.cmpf oeq, %[[VAL_13]], %[[VAL_31]] fastmath<contract> : bf16
+!CHECK:             %[[VAL_33:.*]] = fir.if %[[VAL_32]] -> (bf16) {
+!CHECK:               %[[VAL_34:.*]] = arith.bitcast %[[VAL_15]] : i16 to bf16
+!CHECK:               %[[VAL_35:.*]] = arith.constant -32767 : i16
+!CHECK:               %[[VAL_36:.*]] = arith.bitcast %[[VAL_35]] : i16 to bf16
+!CHECK:               %[[VAL_37:.*]] = arith.select %[[VAL_16]], %[[VAL_34]], %[[VAL_36]] : bf16
+!CHECK:               fir.result %[[VAL_37]] : bf16
+!CHECK:             } else {
+!CHECK:               %[[VAL_38:.*]] = arith.bitcast %[[VAL_13]] : bf16 to i16
+!CHECK-DAG:           %[[VAL_39:.*]] = arith.addi %[[VAL_38]], %[[VAL_15]] : i16
+!CHECK-DAG:           %[[VAL_40:.*]] = arith.subi %[[VAL_38]], %[[VAL_15]] : i16
+!CHECK:               %[[VAL_41:.*]] = arith.select %[[VAL_22]], %[[VAL_39]], %[[VAL_40]] : i16
+!CHECK:               %[[VAL_42:.*]] = arith.bitcast %[[VAL_41]] : i16 to bf16
+!CHECK:               fir.result %[[VAL_42]] : bf16
+!CHECK:             }
+!CHECK:             fir.result %[[VAL_33]] : bf16
+!CHECK:           }
+!CHECK:           hlfir.assign %[[VAL_26]] to %[[VAL_11]]#0 : bf16, !fir.ref<bf16>
+!CHECK:           return
+!CHECK:         }
 
-  ! CHECK:     %[[V_104:[0-9]+]] = fir.load %[[V_25]] : !fir.ref<f32>
-  ! CHECK:     %[[V_105:[0-9]+]] = "llvm.intr.is.fpclass"(%[[V_104]]) <{bit = 3 : i32}> : (f32) -> i1
-  ! CHECK:     %[[V_106:[0-9]+]] = arith.bitcast %[[V_104]] : f32 to i32
-  ! CHECK:     %[[V_107:[0-9]+]] = arith.shrui %[[V_106]], %c31{{.*}} : i32
-  ! CHECK:     %[[V_108:[0-9]+]] = fir.convert %[[V_107]] : (i32) -> i1
-  ! CHECK:     %[[V_109:[0-9]+]] = arith.cmpi ne, %[[V_108]], %false{{[_0-9]*}} : i1
-  ! CHECK:     %[[V_110:[0-9]+]] = "llvm.intr.is.fpclass"(%[[V_104]]) <{bit = 516 : i32}> : (f32) -> i1
-  ! CHECK:     %[[V_111:[0-9]+]] = arith.andi %[[V_110]], %[[V_109]] : i1
-  ! CHECK:     %[[V_112:[0-9]+]] = arith.ori %[[V_105]], %[[V_111]] : i1
-  ! CHECK:     %[[V_113:[0-9]+]] = fir.if %[[V_112]] -> (f32) {
-  ! CHECK:       %[[V_202:[0-9]+]] = "llvm.intr.is.fpclass"(%[[V_104]]) <{bit = 1 : i32}> : (f32) -> i1
-  ! CHECK:       fir.if %[[V_202]] {
-  ! CHECK:         %[[V_203:[0-9]+]] = fir.call @_FortranAMapException(%c1{{.*}}) fastmath<contract> : (i32) -> i32
-  ! CHECK:                             fir.call @feraiseexcept(%[[V_203]]) fastmath<contract> : (i32) -> i32
-  ! CHECK:       }
-  ! CHECK:       fir.result %[[V_104]] : f32
-  ! CHECK:     } else {
-  ! CHECK:       %[[V_202:[0-9]+]] = arith.cmpf oeq, %[[V_104]], %cst{{[_0-9]*}} fastmath<contract> : f32
-  ! CHECK:       %[[V_203:[0-9]+]] = fir.if %[[V_202]] -> (f32) {
-  ! CHECK:         fir.result %cst{{[_0-9]*}} : f32
-  ! CHECK:       } else {
-  ! CHECK-DAG:     %[[V_204:[0-9]+]] = arith.subi %[[V_106]], %c1{{.*}} : i32
-  ! CHECK-DAG:     %[[V_205:[0-9]+]] = arith.addi %[[V_106]], %c1{{.*}} : i32
-  ! CHECK:         %[[V_206:[0-9]+]] = arith.select %[[V_109]], %[[V_205]], %[[V_204]] : i32
-  ! CHECK:         %[[V_207:[0-9]+]] = arith.bitcast %[[V_206]] : i32 to f32
-  ! CHECK:         fir.result %[[V_207]] : f32
-  ! CHECK:       }
-  ! CHECK:       fir.result %[[V_203]] : f32
-  ! CHECK:     }
-  ! CHECK:     fir.store %[[V_113]] to %[[V_13]] : !fir.ref<f32>
+subroutine test3(r4, x4)
+  real(4)  ::  r4,  x4
   r4 = ieee_next_down(x4)
-  print "('down:   ', z8.8, ' -> ', z8.8, ' = ', g0)", x4, r4, r4
+end subroutine
+!CHECK-LABEL:   func.func @_QMieee_next_testsPtest3(
+!CHECK:           %[[VAL_11:.*]]:2 = hlfir.declare {{.*}}r4"
+!CHECK:           %[[VAL_12:.*]]:2 = hlfir.declare {{.*}}x4"
+!CHECK:           %[[VAL_13:.*]] = fir.load %[[VAL_12]]#0 : !fir.ref<f32>
+!CHECK:           %[[VAL_14:.*]] = "llvm.intr.is.fpclass"(%[[VAL_13]]) <{bit = 3 : i32}> : (f32) -> i1
+!CHECK:           %[[VAL_15:.*]] = arith.constant 1 : i32
+!CHECK:           %[[VAL_16:.*]] = arith.constant false
+!CHECK:           %[[VAL_17:.*]] = arith.bitcast %[[VAL_13]] : f32 to i32
+!CHECK:           %[[VAL_18:.*]] = arith.constant 31 : i32
+!CHECK:           %[[VAL_19:.*]] = arith.shrui %[[VAL_17]], %[[VAL_18]] : i32
+!CHECK:           %[[VAL_20:.*]] = fir.convert %[[VAL_19]] : (i32) -> i1
+!CHECK:           %[[VAL_21:.*]] = arith.cmpi ne, %[[VAL_16]], %[[VAL_20]] : i1
+!CHECK:           %[[VAL_22:.*]] = "llvm.intr.is.fpclass"(%[[VAL_13]]) <{bit = 516 : i32}> : (f32) -> i1
+!CHECK:           %[[VAL_23:.*]] = arith.andi %[[VAL_22]], %[[VAL_21]] : i1
+!CHECK:           %[[VAL_24:.*]] = arith.ori %[[VAL_14]], %[[VAL_23]] : i1
+!CHECK:           %[[VAL_25:.*]] = fir.if %[[VAL_24]] -> (f32) {
+!CHECK:             %[[VAL_26:.*]] = "llvm.intr.is.fpclass"(%[[VAL_13]]) <{bit = 1 : i32}> : (f32) -> i1
+!CHECK:             fir.if %[[VAL_26]] {
+!CHECK:               %[[VAL_27:.*]] = arith.constant 1 : i32
+!CHECK:               %[[VAL_28:.*]] = fir.call @_FortranAMapException(%[[VAL_27]]) fastmath<contract> : (i32) -> i32
+!CHECK:               %[[VAL_29:.*]] = fir.call @feraiseexcept(%[[VAL_28]]) fastmath<contract> : (i32) -> i32
+!CHECK:             }
+!CHECK:             fir.result %[[VAL_13]] : f32
+!CHECK:           } else {
+!CHECK:             %[[VAL_30:.*]] = arith.constant 0.000000e+00 : f32
+!CHECK:             %[[VAL_31:.*]] = arith.cmpf oeq, %[[VAL_13]], %[[VAL_30]] fastmath<contract> : f32
+!CHECK:             %[[VAL_32:.*]] = fir.if %[[VAL_31]] -> (f32) {
+!CHECK:               %[[VAL_33:.*]] = arith.bitcast %[[VAL_15]] : i32 to f32
+!CHECK:               %[[VAL_34:.*]] = arith.constant -2147483647 : i32
+!CHECK:               %[[VAL_35:.*]] = arith.bitcast %[[VAL_34]] : i32 to f32
+!CHECK:               %[[VAL_36:.*]] = arith.select %[[VAL_16]], %[[VAL_33]], %[[VAL_35]] : f32
+!CHECK:               fir.result %[[VAL_36]] : f32
+!CHECK:             } else {
+!CHECK:               %[[VAL_37:.*]] = arith.bitcast %[[VAL_13]] : f32 to i32
+!CHECK-DAG:           %[[VAL_38:.*]] = arith.addi %[[VAL_37]], %[[VAL_15]] : i32
+!CHECK-DAG:           %[[VAL_39:.*]] = arith.subi %[[VAL_37]], %[[VAL_15]] : i32
+!CHECK:               %[[VAL_40:.*]] = arith.select %[[VAL_21]], %[[VAL_38]], %[[VAL_39]] : i32
+!CHECK:               %[[VAL_41:.*]] = arith.bitcast %[[VAL_40]] : i32 to f32
+!CHECK:               fir.result %[[VAL_41]] : f32
+!CHECK:             }
+!CHECK:             fir.result %[[VAL_32]] : f32
+!CHECK:           }
+!CHECK:           hlfir.assign %[[VAL_25]] to %[[VAL_11]]#0 : f32, !fir.ref<f32>
+!CHECK:           return
+!CHECK:         }
 
-  ! CHECK:     %[[V_125:[0-9]+]] = fir.load %[[V_27]] : !fir.ref<f64>
-  ! CHECK:     %[[V_126:[0-9]+]] = fir.load %[[V_21]] : !fir.ref<f16>
-  ! CHECK-DAG: %[[V_127:[0-9]+]] = fir.address_of(@_FortranAIeeeValueTable_8) : !fir.ref<!fir.array<12xi64>>
-  ! CHECK-DAG: %[[V_128:[0-9]+]] = fir.coordinate_of %[[V_127]], %c2{{.*}} : (!fir.ref<!fir.array<12xi64>>, i8) -> !fir.ref<i64>
-  ! CHECK-DAG: %[[V_129:[0-9]+]] = fir.load %[[V_128]] : !fir.ref<i64>
-  ! CHECK-DAG: %[[V_130:[0-9]+]] = arith.bitcast %[[V_129]] : i64 to f64
-  ! CHECK-DAG: %[[V_131:[0-9]+]] = "llvm.intr.is.fpclass"(%[[V_126]]) <{bit = 3 : i32}> : (f16) -> i1
-  ! CHECK:     %[[V_132:[0-9]+]] = arith.select %[[V_131]], %[[V_130]], %[[V_125]] : f64
-  ! CHECK:     %[[V_133:[0-9]+]] = "llvm.intr.is.fpclass"(%[[V_132]]) <{bit = 3 : i32}> : (f64) -> i1
-  ! CHECK:     %[[V_134:[0-9]+]] = fir.convert %[[V_126]] : (f16) -> f64
-  ! CHECK:     %[[V_135:[0-9]+]] = arith.cmpf oeq, %[[V_132]], %[[V_134]] fastmath<contract> : f64
-  ! CHECK:     %[[V_136:[0-9]+]] = arith.ori %[[V_133]], %[[V_135]] : i1
-  ! CHECK:     %[[V_137:[0-9]+]] = arith.cmpf olt, %[[V_132]], %[[V_134]] fastmath<contract> : f64
-  ! CHECK:     %[[V_138:[0-9]+]] = arith.bitcast %[[V_125]] : f64 to i64
-  ! CHECK:     %[[V_139:[0-9]+]] = arith.shrui %[[V_138]], %c63{{.*}} : i64
-  ! CHECK:     %[[V_140:[0-9]+]] = fir.convert %[[V_139]] : (i64) -> i1
-  ! CHECK:     %[[V_141:[0-9]+]] = arith.cmpi ne, %[[V_137]], %[[V_140]] : i1
-  ! CHECK:     %[[V_142:[0-9]+]] = "llvm.intr.is.fpclass"(%[[V_132]]) <{bit = 516 : i32}> : (f64) -> i1
-  ! CHECK:     %[[V_143:[0-9]+]] = arith.andi %[[V_142]], %[[V_141]] : i1
-  ! CHECK:     %[[V_144:[0-9]+]] = arith.ori %[[V_136]], %[[V_143]] : i1
-  ! CHECK:     %[[V_145:[0-9]+]] = fir.if %[[V_144]] -> (f64) {
-  ! CHECK:       fir.result %[[V_132]] : f64
-  ! CHECK:     } else {
-  ! CHECK:       %[[V_202:[0-9]+]] = arith.cmpf oeq, %[[V_132]], %cst{{[_0-9]*}} fastmath<contract> : f64
-  ! CHECK:       %[[V_203:[0-9]+]] = fir.if %[[V_202]] -> (f64) {
-  ! CHECK:         %[[V_204:[0-9]+]] = arith.select %[[V_137]], %cst{{[_0-9]*}}, %cst{{[_0-9]*}} : f64
-  ! CHECK:         %[[V_205:[0-9]+]] = fir.call @_FortranAMapException(%c48{{.*}}) fastmath<contract> : (i32) -> i32
-  ! CHECK:                             fir.call @feraiseexcept(%[[V_205]]) fastmath<contract> : (i32) -> i32
-  ! CHECK:         fir.result %[[V_204]] : f64
-  ! CHECK:       } else {
-  ! CHECK:         %[[V_204:[0-9]+]] = arith.bitcast %[[V_132]] : f64 to i64
-  ! CHECK-DAG:     %[[V_205:[0-9]+]] = arith.subi %[[V_204]], %c1{{.*}} : i64
-  ! CHECK-DAG:     %[[V_206:[0-9]+]] = arith.addi %[[V_204]], %c1{{.*}} : i64
-  ! CHECK:         %[[V_207:[0-9]+]] = arith.select %[[V_141]], %[[V_206]], %[[V_205]] : i64
-  ! CHECK:         %[[V_208:[0-9]+]] = arith.bitcast %[[V_207]] : i64 to f64
-  ! CHECK:         %[[V_209:[0-9]+]] = "llvm.intr.is.fpclass"(%[[V_208]]) <{bit = 516 : i32}> : (f64) -> i1
-  ! CHECK:         fir.if %[[V_209]] {
-  ! CHECK:           %[[V_211:[0-9]+]] = fir.call @_FortranAMapException(%c40{{.*}}) fastmath<contract> : (i32) -> i32
-  ! CHECK:                               fir.call @feraiseexcept(%[[V_211]]) fastmath<contract> : (i32) -> i32
-  ! CHECK:         }
-  ! CHECK:         %[[V_210:[0-9]+]] = "llvm.intr.is.fpclass"(%[[V_208]]) <{bit = 144 : i32}> : (f64) -> i1
-  ! CHECK:         fir.if %[[V_210]] {
-  ! CHECK:           %[[V_211:[0-9]+]] = fir.call @_FortranAMapException(%c48{{.*}}) fastmath<contract> : (i32) -> i32
-  ! CHECK:                               fir.call @feraiseexcept(%[[V_211]]) fastmath<contract> : (i32) -> i32
-  ! CHECK:         }
-  ! CHECK:         fir.result %[[V_208]] : f64
-  ! CHECK:       }
-  ! CHECK:       fir.result %[[V_203]] : f64
-  ! CHECK:     }
-  ! CHECK:     fir.store %[[V_145]] to %[[V_15]] : !fir.ref<f64>
+subroutine test4(r8, x8, x2)
+  real(2)  ::  x2
+  real(8)  ::  r8,  x8
   r8 = ieee_next_after(x8, x2)
-  print "('after:  ', z16.16, ' -> ', z16.16, ' = ', g0)", x8, r8, r8
+end subroutine
+!CHECK-LABEL:   func.func @_QMieee_next_testsPtest4(
+!CHECK:           %[[VAL_12:.*]]:2 = hlfir.declare {{.*}}r8"
+!CHECK:           %[[VAL_13:.*]]:2 = hlfir.declare {{.*}}x2"
+!CHECK:           %[[VAL_14:.*]]:2 = hlfir.declare {{.*}}x8"
+!CHECK:           %[[VAL_15:.*]] = fir.load %[[VAL_14]]#0 : !fir.ref<f64>
+!CHECK:           %[[VAL_16:.*]] = fir.load %[[VAL_13]]#0 : !fir.ref<f16>
+!CHECK-DAG:       %[[VAL_17:.*]] = "llvm.intr.is.fpclass"(%[[VAL_16]]) <{bit = 3 : i32}> : (f16) -> i1
+!CHECK-DAG:       %[[VAL_18:.*]] = arith.constant 2 : i8
+!CHECK-DAG:       %[[VAL_19:.*]] = fir.address_of(@_FortranAIeeeValueTable_8) : !fir.ref<!fir.array<12xi64>>
+!CHECK-DAG:       %[[VAL_20:.*]] = fir.coordinate_of %[[VAL_19]], %[[VAL_18]] : (!fir.ref<!fir.array<12xi64>>, i8) -> !fir.ref<i64>
+!CHECK-DAG:       %[[VAL_21:.*]] = fir.load %[[VAL_20]] : !fir.ref<i64>
+!CHECK-DAG:       %[[VAL_22:.*]] = arith.bitcast %[[VAL_21]] : i64 to f64
+!CHECK:           %[[VAL_23:.*]] = arith.select %[[VAL_17]], %[[VAL_22]], %[[VAL_15]] : f64
+!CHECK:           %[[VAL_24:.*]] = "llvm.intr.is.fpclass"(%[[VAL_23]]) <{bit = 3 : i32}> : (f64) -> i1
+!CHECK:           %[[VAL_25:.*]] = arith.constant 1 : i64
+!CHECK:           %[[VAL_26:.*]] = fir.convert %[[VAL_16]] : (f16) -> f32
+!CHECK:           %[[VAL_27:.*]] = fir.convert %[[VAL_26]] : (f32) -> f64
+!CHECK:           %[[VAL_28:.*]] = arith.cmpf oeq, %[[VAL_23]], %[[VAL_27]] fastmath<contract> : f64
+!CHECK:           %[[VAL_29:.*]] = arith.ori %[[VAL_24]], %[[VAL_28]] : i1
+!CHECK:           %[[VAL_30:.*]] = arith.cmpf olt, %[[VAL_23]], %[[VAL_27]] fastmath<contract> : f64
+!CHECK:           %[[VAL_31:.*]] = arith.bitcast %[[VAL_15]] : f64 to i64
+!CHECK:           %[[VAL_32:.*]] = arith.constant 63 : i64
+!CHECK:           %[[VAL_33:.*]] = arith.shrui %[[VAL_31]], %[[VAL_32]] : i64
+!CHECK:           %[[VAL_34:.*]] = fir.convert %[[VAL_33]] : (i64) -> i1
+!CHECK:           %[[VAL_35:.*]] = arith.cmpi ne, %[[VAL_30]], %[[VAL_34]] : i1
+!CHECK:           %[[VAL_36:.*]] = "llvm.intr.is.fpclass"(%[[VAL_23]]) <{bit = 516 : i32}> : (f64) -> i1
+!CHECK:           %[[VAL_37:.*]] = arith.andi %[[VAL_36]], %[[VAL_35]] : i1
+!CHECK:           %[[VAL_38:.*]] = arith.ori %[[VAL_29]], %[[VAL_37]] : i1
+!CHECK:           %[[VAL_39:.*]] = fir.if %[[VAL_38]] -> (f64) {
+!CHECK:             fir.result %[[VAL_23]] : f64
+!CHECK:           } else {
+!CHECK:             %[[VAL_40:.*]] = arith.constant 0.000000e+00 : f64
+!CHECK:             %[[VAL_41:.*]] = arith.cmpf oeq, %[[VAL_23]], %[[VAL_40]] fastmath<contract> : f64
+!CHECK:             %[[VAL_42:.*]] = fir.if %[[VAL_41]] -> (f64) {
+!CHECK:               %[[VAL_43:.*]] = arith.bitcast %[[VAL_25]] : i64 to f64
+!CHECK:               %[[VAL_44:.*]] = arith.constant -9223372036854775807 : i64
+!CHECK:               %[[VAL_45:.*]] = arith.bitcast %[[VAL_44]] : i64 to f64
+!CHECK:               %[[VAL_46:.*]] = arith.select %[[VAL_30]], %[[VAL_43]], %[[VAL_45]] : f64
+!CHECK:               %[[VAL_47:.*]] = arith.constant 48 : i32
+!CHECK:               %[[VAL_48:.*]] = fir.call @_FortranAMapException(%[[VAL_47]]) fastmath<contract> : (i32) -> i32
+!CHECK:               %[[VAL_49:.*]] = fir.call @feraiseexcept(%[[VAL_48]]) fastmath<contract> : (i32) -> i32
+!CHECK:               fir.result %[[VAL_46]] : f64
+!CHECK:             } else {
+!CHECK:               %[[VAL_50:.*]] = arith.bitcast %[[VAL_23]] : f64 to i64
+!CHECK-DAG:           %[[VAL_51:.*]] = arith.addi %[[VAL_50]], %[[VAL_25]] : i64
+!CHECK-DAG:           %[[VAL_52:.*]] = arith.subi %[[VAL_50]], %[[VAL_25]] : i64
+!CHECK:               %[[VAL_53:.*]] = arith.select %[[VAL_35]], %[[VAL_51]], %[[VAL_52]] : i64
+!CHECK:               %[[VAL_54:.*]] = arith.bitcast %[[VAL_53]] : i64 to f64
+!CHECK:               %[[VAL_55:.*]] = "llvm.intr.is.fpclass"(%[[VAL_54]]) <{bit = 516 : i32}> : (f64) -> i1
+!CHECK:               fir.if %[[VAL_55]] {
+!CHECK:                 %[[VAL_56:.*]] = arith.constant 40 : i32
+!CHECK:                 %[[VAL_57:.*]] = fir.call @_FortranAMapException(%[[VAL_56]]) fastmath<contract> : (i32) -> i32
+!CHECK:                 %[[VAL_58:.*]] = fir.call @feraiseexcept(%[[VAL_57]]) fastmath<contract> : (i32) -> i32
+!CHECK:               }
+!CHECK:               %[[VAL_59:.*]] = "llvm.intr.is.fpclass"(%[[VAL_54]]) <{bit = 144 : i32}> : (f64) -> i1
+!CHECK:               fir.if %[[VAL_59]] {
+!CHECK:                 %[[VAL_60:.*]] = arith.constant 48 : i32
+!CHECK:                 %[[VAL_61:.*]] = fir.call @_FortranAMapException(%[[VAL_60]]) fastmath<contract> : (i32) -> i32
+!CHECK:                 %[[VAL_62:.*]] = fir.call @feraiseexcept(%[[VAL_61]]) fastmath<contract> : (i32) -> i32
+!CHECK:               }
+!CHECK:               fir.result %[[VAL_54]] : f64
+!CHECK:             }
+!CHECK:             fir.result %[[VAL_42]] : f64
+!CHECK:           }
+!CHECK:           hlfir.assign %[[VAL_39]] to %[[VAL_12]]#0 : f64, !fir.ref<f64>
+!CHECK:           return
+!CHECK:         }
 
-  ! CHECK:     %[[V_158:[0-9]+]] = fir.load %[[V_17]] : !fir.ref<f80>
-  ! CHECK:     %[[V_159:[0-9]+]] = "llvm.intr.is.fpclass"(%[[V_158]]) <{bit = 3 : i32}> : (f80) -> i1
-  ! CHECK:     %[[V_160:[0-9]+]] = arith.bitcast %[[V_158]] : f80 to i80
-  ! CHECK:     %[[V_161:[0-9]+]] = arith.shrui %[[V_160]], %c79{{.*}} : i80
-  ! CHECK:     %[[V_162:[0-9]+]] = fir.convert %[[V_161]] : (i80) -> i1
-  ! CHECK:     %[[V_163:[0-9]+]] = arith.cmpi ne, %[[V_162]], %true{{[_0-9]*}} : i1
-  ! CHECK:     %[[V_164:[0-9]+]] = "llvm.intr.is.fpclass"(%[[V_158]]) <{bit = 516 : i32}> : (f80) -> i1
-  ! CHECK:     %[[V_165:[0-9]+]] = arith.andi %[[V_164]], %[[V_163]] : i1
-  ! CHECK:     %[[V_166:[0-9]+]] = arith.ori %[[V_159]], %[[V_165]] : i1
-  ! CHECK:     %[[V_167:[0-9]+]] = fir.if %[[V_166]] -> (f80) {
-  ! CHECK:       %[[V_202:[0-9]+]] = "llvm.intr.is.fpclass"(%[[V_158]]) <{bit = 1 : i32}> : (f80) -> i1
-  ! CHECK:       fir.if %[[V_202]] {
-  ! CHECK:         %[[V_203:[0-9]+]] = fir.call @_FortranAMapException(%c1{{.*}}) fastmath<contract> : (i32) -> i32
-  ! CHECK:                             fir.call @feraiseexcept(%[[V_203]]) fastmath<contract> : (i32) -> i32
-  ! CHECK:       }
-  ! CHECK:       fir.result %[[V_158]] : f80
-  ! CHECK:     } else {
-  ! CHECK:       %[[V_202:[0-9]+]] = arith.cmpf oeq, %[[V_158]], %cst{{[_0-9]*}} fastmath<contract> : f80
-  ! CHECK:       %[[V_203:[0-9]+]] = fir.if %[[V_202]] -> (f80) {
-  ! CHECK:         fir.result %cst{{[_0-9]*}} : f80
-  ! CHECK:       } else {
-  ! CHECK:         %[[V_204:[0-9]+]] = fir.call @_FortranAMapException(%c63{{.*}}) fastmath<contract> : (i32) -> i32
-  ! CHECK:         %[[V_205:[0-9]+]] = fir.call @fetestexcept(%[[V_204]]) fastmath<contract> : (i32) -> i32
-  ! CHECK:         %[[V_206:[0-9]+]] = fir.call @fedisableexcept(%[[V_204]]) fastmath<contract> : (i32) -> i32
-  ! CHECK:         %[[V_207:[0-9]+]] = fir.call @_FortranANearest10(%[[V_158]], %true{{[_0-9]*}}) fastmath<contract> : (f80, i1) -> f80
-  ! CHECK:         %[[V_208:[0-9]+]] = fir.call @feclearexcept(%[[V_204]]) fastmath<contract> : (i32) -> i32
-  ! CHECK:         %[[V_209:[0-9]+]] = fir.call @feraiseexcept(%[[V_205]]) fastmath<contract> : (i32) -> i32
-  ! CHECK:         %[[V_210:[0-9]+]] = fir.call @feenableexcept(%[[V_206]]) fastmath<contract> : (i32) -> i32
-  ! CHECK:         fir.result %[[V_207]] : f80
-  ! CHECK:       }
-  ! CHECK:       fir.result %[[V_203]] : f80
-  ! CHECK:     }
-  ! CHECK:     fir.store %[[V_167]] to %[[V_5]] : !fir.ref<f80>
+subroutine test5(r10, x10)
+  real(kind10) :: x10, r10
   r10 = ieee_next_up(x10)
-  print "('up:     ', z20.20, ' -> ', z20.20, ' = ', g0)", x10, r10, r10
-
-  ! CHECK:     %[[V_180:[0-9]+]] = fir.load %[[V_19]] : !fir.ref<f128>
-  ! CHECK:     %[[V_181:[0-9]+]] = "llvm.intr.is.fpclass"(%[[V_180]]) <{bit = 3 : i32}> : (f128) -> i1
-  ! CHECK:     %[[V_182:[0-9]+]] = arith.bitcast %[[V_180]] : f128 to i128
-  ! CHECK:     %[[V_183:[0-9]+]] = arith.shrui %[[V_182]], %c127{{.*}} : i128
-  ! CHECK:     %[[V_184:[0-9]+]] = fir.convert %[[V_183]] : (i128) -> i1
-  ! CHECK:     %[[V_185:[0-9]+]] = arith.cmpi ne, %[[V_184]], %false{{[_0-9]*}} : i1
-  ! CHECK:     %[[V_186:[0-9]+]] = "llvm.intr.is.fpclass"(%[[V_180]]) <{bit = 516 : i32}> : (f128) -> i1
-  ! CHECK:     %[[V_187:[0-9]+]] = arith.andi %[[V_186]], %[[V_185]] : i1
-  ! CHECK:     %[[V_188:[0-9]+]] = arith.ori %[[V_181]], %[[V_187]] : i1
-  ! CHECK:     %[[V_189:[0-9]+]] = fir.if %[[V_188]] -> (f128) {
-  ! CHECK:       %[[V_202:[0-9]+]] = "llvm.intr.is.fpclass"(%[[V_180]]) <{bit = 1 : i32}> : (f128) -> i1
-  ! CHECK:       fir.if %[[V_202]] {
-  ! CHECK:         %[[V_203:[0-9]+]] = fir.call @_FortranAMapException(%c1{{.*}}) fastmath<contract> : (i32) -> i32
-  ! CHECK:                             fir.call @feraiseexcept(%[[V_203]]) fastmath<contract> : (i32) -> i32
-  ! CHECK:       }
-  ! CHECK:       fir.result %[[V_180]] : f128
-  ! CHECK:     } else {
-  ! CHECK:       %[[V_202:[0-9]+]] = arith.cmpf oeq, %[[V_180]], %cst{{[_0-9]*}} fastmath<contract> : f128
-  ! CHECK:       %[[V_203:[0-9]+]] = fir.if %[[V_202]] -> (f128) {
-  ! CHECK:         fir.result %cst{{[_0-9]*}} : f128
-  ! CHECK:       } else {
-  ! CHECK-DAG:     %[[V_204:[0-9]+]] = arith.subi %[[V_182]], %c1{{.*}} : i128
-  ! CHECK-DAG:     %[[V_205:[0-9]+]] = arith.addi %[[V_182]], %c1{{.*}} : i128
-  ! CHECK:         %[[V_206:[0-9]+]] = arith.select %[[V_185]], %[[V_205]], %[[V_204]] : i128
-  ! CHECK:         %[[V_207:[0-9]+]] = arith.bitcast %[[V_206]] : i128 to f128
-  ! CHECK:         fir.result %[[V_207]] : f128
-  ! CHECK:       }
-  ! CHECK:       fir.result %[[V_203]] : f128
-  ! CHECK:     }
-  ! CHECK:     fir.store %[[V_189]] to %[[V_7]] : !fir.ref<f128>
+end subroutine
+!CHECK-KIND10-LABEL:   func.func @_QMieee_next_testsPtest5(
+!CHECK-KIND10:           %[[VAL_11:.*]]:2 = hlfir.declare {{.*}}r10"
+!CHECK-KIND10:           %[[VAL_12:.*]]:2 = hlfir.declare {{.*}}x10"
+!CHECK-KIND10:           %[[VAL_13:.*]] = fir.load %[[VAL_12]]#0 : !fir.ref<f80>
+!CHECK-KIND10:           %[[VAL_14:.*]] = "llvm.intr.is.fpclass"(%[[VAL_13]]) <{bit = 3 : i32}> : (f80) -> i1
+!CHECK-KIND10:           %[[VAL_15:.*]] = arith.constant 1 : i80
+!CHECK-KIND10:           %[[VAL_16:.*]] = arith.constant true
+!CHECK-KIND10:           %[[VAL_17:.*]] = arith.bitcast %[[VAL_13]] : f80 to i80
+!CHECK-KIND10:           %[[VAL_18:.*]] = arith.constant 79 : i80
+!CHECK-KIND10:           %[[VAL_19:.*]] = arith.shrui %[[VAL_17]], %[[VAL_18]] : i80
+!CHECK-KIND10:           %[[VAL_20:.*]] = fir.convert %[[VAL_19]] : (i80) -> i1
+!CHECK-KIND10:           %[[VAL_21:.*]] = arith.cmpi ne, %[[VAL_16]], %[[VAL_20]] : i1
+!CHECK-KIND10:           %[[VAL_22:.*]] = "llvm.intr.is.fpclass"(%[[VAL_13]]) <{bit = 516 : i32}> : (f80) -> i1
+!CHECK-KIND10:           %[[VAL_23:.*]] = arith.andi %[[VAL_22]], %[[VAL_21]] : i1
+!CHECK-KIND10:           %[[VAL_24:.*]] = arith.ori %[[VAL_14]], %[[VAL_23]] : i1
+!CHECK-KIND10:           %[[VAL_25:.*]] = fir.if %[[VAL_24]] -> (f80) {
+!CHECK-KIND10:             %[[VAL_26:.*]] = "llvm.intr.is.fpclass"(%[[VAL_13]]) <{bit = 1 : i32}> : (f80) -> i1
+!CHECK-KIND10:             fir.if %[[VAL_26]] {
+!CHECK-KIND10:               %[[VAL_27:.*]] = arith.constant 1 : i32
+!CHECK-KIND10:               %[[VAL_28:.*]] = fir.call @_FortranAMapException(%[[VAL_27]]) fastmath<contract> : (i32) -> i32
+!CHECK-KIND10:               %[[VAL_29:.*]] = fir.call @feraiseexcept(%[[VAL_28]]) fastmath<contract> : (i32) -> i32
+!CHECK-KIND10:             }
+!CHECK-KIND10:             fir.result %[[VAL_13]] : f80
+!CHECK-KIND10:           } else {
+!CHECK-KIND10:             %[[VAL_30:.*]] = arith.constant 0.000000e+00 : f80
+!CHECK-KIND10:             %[[VAL_31:.*]] = arith.cmpf oeq, %[[VAL_13]], %[[VAL_30]] fastmath<contract> : f80
+!CHECK-KIND10:             %[[VAL_32:.*]] = fir.if %[[VAL_31]] -> (f80) {
+!CHECK-KIND10:               %[[VAL_33:.*]] = arith.bitcast %[[VAL_15]] : i80 to f80
+!CHECK-KIND10:               %[[VAL_34:.*]] = arith.constant -604462909807314587353087 : i80
+!CHECK-KIND10:               %[[VAL_35:.*]] = arith.bitcast %[[VAL_34]] : i80 to f80
+!CHECK-KIND10:               %[[VAL_36:.*]] = arith.select %[[VAL_16]], %[[VAL_33]], %[[VAL_35]] : f80
+!CHECK-KIND10:               fir.result %[[VAL_36]] : f80
+!CHECK-KIND10:             } else {
+!CHECK-KIND10:               %[[VAL_37:.*]] = arith.constant 63 : i32
+!CHECK-KIND10:               %[[VAL_38:.*]] = fir.call @_FortranAMapException(%[[VAL_37]]) fastmath<contract> : (i32) -> i32
+!CHECK-KIND10:               %[[VAL_39:.*]] = fir.call @fetestexcept(%[[VAL_38]]) fastmath<contract> : (i32) -> i32
+!CHECK-KIND10:               %[[VAL_40:.*]] = fir.call @fedisableexcept(%[[VAL_38]]) fastmath<contract> : (i32) -> i32
+!CHECK-KIND10:               %[[VAL_41:.*]] = fir.call @_FortranANearest10(%[[VAL_13]], %[[VAL_16]]) fastmath<contract> : (f80, i1) -> f80
+!CHECK-KIND10:               %[[VAL_42:.*]] = fir.call @feclearexcept(%[[VAL_38]]) fastmath<contract> : (i32) -> i32
+!CHECK-KIND10:               %[[VAL_43:.*]] = fir.call @feraiseexcept(%[[VAL_39]]) fastmath<contract> : (i32) -> i32
+!CHECK-KIND10:               %[[VAL_44:.*]] = fir.call @feenableexcept(%[[VAL_40]]) fastmath<contract> : (i32) -> i32
+!CHECK-KIND10:               fir.result %[[VAL_41]] : f80
+!CHECK-KIND10:             }
+!CHECK-KIND10:             fir.result %[[VAL_32]] : f80
+!CHECK-KIND10:           }
+!CHECK-KIND10:           hlfir.assign %[[VAL_25]] to %[[VAL_11]]#0 : f80, !fir.ref<f80>
+!CHECK-KIND10:           return
+!CHECK-KIND10:         }
 
+subroutine test6(r16, x16)
+  real(kind16) :: r16, x16
   r16 = ieee_next_down(x16)
+end subroutine
+!CHECK-KIND16-LABEL:   func.func @_QMieee_next_testsPtest6(
+!CHECK-KIND16:           %[[VAL_11:.*]]:2 = hlfir.declare {{.*}}r16"
+!CHECK-KIND16:           %[[VAL_12:.*]]:2 = hlfir.declare {{.*}}x16"
+!CHECK-KIND16:           %[[VAL_13:.*]] = fir.load %[[VAL_12]]#0 : !fir.ref<f128>
+!CHECK-KIND16:           %[[VAL_14:.*]] = "llvm.intr.is.fpclass"(%[[VAL_13]]) <{bit = 3 : i32}> : (f128) -> i1
+!CHECK-KIND16:           %[[VAL_15:.*]] = arith.constant 1 : i128
+!CHECK-KIND16:           %[[VAL_16:.*]] = arith.constant false
+!CHECK-KIND16:           %[[VAL_17:.*]] = arith.bitcast %[[VAL_13]] : f128 to i128
+!CHECK-KIND16:           %[[VAL_18:.*]] = arith.constant 127 : i128
+!CHECK-KIND16:           %[[VAL_19:.*]] = arith.shrui %[[VAL_17]], %[[VAL_18]] : i128
+!CHECK-KIND16:           %[[VAL_20:.*]] = fir.convert %[[VAL_19]] : (i128) -> i1
+!CHECK-KIND16:           %[[VAL_21:.*]] = arith.cmpi ne, %[[VAL_16]], %[[VAL_20]] : i1
+!CHECK-KIND16:           %[[VAL_22:.*]] = "llvm.intr.is.fpclass"(%[[VAL_13]]) <{bit = 516 : i32}> : (f128) -> i1
+!CHECK-KIND16:           %[[VAL_23:.*]] = arith.andi %[[VAL_22]], %[[VAL_21]] : i1
+!CHECK-KIND16:           %[[VAL_24:.*]] = arith.ori %[[VAL_14]], %[[VAL_23]] : i1
+!CHECK-KIND16:           %[[VAL_25:.*]] = fir.if %[[VAL_24]] -> (f128) {
+!CHECK-KIND16:             %[[VAL_26:.*]] = "llvm.intr.is.fpclass"(%[[VAL_13]]) <{bit = 1 : i32}> : (f128) -> i1
+!CHECK-KIND16:             fir.if %[[VAL_26]] {
+!CHECK-KIND16:               %[[VAL_27:.*]] = arith.constant 1 : i32
+!CHECK-KIND16:               %[[VAL_28:.*]] = fir.call @_FortranAMapException(%[[VAL_27]]) fastmath<contract> : (i32) -> i32
+!CHECK-KIND16:               %[[VAL_29:.*]] = fir.call @feraiseexcept(%[[VAL_28]]) fastmath<contract> : (i32) -> i32
+!CHECK-KIND16:             }
+!CHECK-KIND16:             fir.result %[[VAL_13]] : f128
+!CHECK-KIND16:           } else {
+!CHECK-KIND16:             %[[VAL_30:.*]] = arith.constant 0.000000e+00 : f128
+!CHECK-KIND16:             %[[VAL_31:.*]] = arith.cmpf oeq, %[[VAL_13]], %[[VAL_30]] fastmath<contract> : f128
+!CHECK-KIND16:             %[[VAL_32:.*]] = fir.if %[[VAL_31]] -> (f128) {
+!CHECK-KIND16:               %[[VAL_33:.*]] = arith.bitcast %[[VAL_15]] : i128 to f128
+!CHECK-KIND16:               %[[VAL_34:.*]] = arith.constant -170141183460469231731687303715884105727 : i128
+!CHECK-KIND16:               %[[VAL_35:.*]] = arith.bitcast %[[VAL_34]] : i128 to f128
+!CHECK-KIND16:               %[[VAL_36:.*]] = arith.select %[[VAL_16]], %[[VAL_33]], %[[VAL_35]] : f128
+!CHECK-KIND16:               fir.result %[[VAL_36]] : f128
+!CHECK-KIND16:             } else {
+!CHECK-KIND16:               %[[VAL_37:.*]] = arith.bitcast %[[VAL_13]] : f128 to i128
+!CHECK-KIND16-DAG:           %[[VAL_38:.*]] = arith.addi %[[VAL_37]], %[[VAL_15]] : i128
+!CHECK-KIND16-DAG:           %[[VAL_39:.*]] = arith.subi %[[VAL_37]], %[[VAL_15]] : i128
+!CHECK-KIND16:               %[[VAL_40:.*]] = arith.select %[[VAL_21]], %[[VAL_38]], %[[VAL_39]] : i128
+!CHECK-KIND16:               %[[VAL_41:.*]] = arith.bitcast %[[VAL_40]] : i128 to f128
+!CHECK-KIND16:               fir.result %[[VAL_41]] : f128
+!CHECK-KIND16:             }
+!CHECK-KIND16:             fir.result %[[VAL_32]] : f128
+!CHECK-KIND16:           }
+!CHECK-KIND16:           hlfir.assign %[[VAL_25]] to %[[VAL_11]]#0 : f128, !fir.ref<f128>
+!CHECK-KIND16:           return
+!CHECK-KIND16:         }
+end module
+
+! Expected end-to-end output when both kind10 and kind16 enabled (not part of lit
+! test, only provided for debug help):
+!
+! after:  FC00 -> FBFF = -.655E+5
+! up:     FF7F -> FF7E = -.337E+39
+! down:   80000000 -> 80000001 = -.1E-44
+! after:  0000000000000000 -> 8000000000000001 = -.5E-323
+! up:     7FFEFFFFFFFFFFFFFFFF -> 7FFF8000000000000000 = Inf
+! down:   7FFF0000000000000000000000000000 -> 7FFEFFFFFFFFFFFFFFFFFFFFFFFFFFFF = .1189731495357231765085759326628007E+4933
+program p
+  use ieee_next_tests
+  real(2)  ::  r2,  x2
+  real(3)  ::  r3,  x3 = -huge(x3)
+  real(4)  ::  r4,  x4 = -0.
+  real(8)  ::  r8,  x8 =  0.
+  real(kind10) :: r10, x10 =  huge(x10)
+  real(kind16) :: r16, x16
+
+  x2  = ieee_value(x2, ieee_negative_inf)
+  x16 = ieee_value(x2, ieee_positive_inf)
+  call test1(r2, x2, x10)
+  print "('after:  ', z4.4, ' -> ', z4.4, ' = ', g0)", x2, r2, r2
+  call test2(r3, x3)
+  print "('up:     ', z4.4, ' -> ', z4.4, ' = ', g0)", x3, r3, r3
+  call test3(r4, x4)
+  print "('down:   ', z8.8, ' -> ', z8.8, ' = ', g0)", x4, r4, r4
+  call test4(r8, x8, x2)
+  print "('after:  ', z16.16, ' -> ', z16.16, ' = ', g0)", x8, r8, r8
+  call test5(r10, x10)
+  print "('up:     ', z20.20, ' -> ', z20.20, ' = ', g0)", x10, r10, r10
+  call test6(r16, x16)
   print "('down:   ', z32.32, ' -> ', z32.32, ' = ', g0)", x16, r16, r16
 end
diff --git a/flang/test/Lower/Intrinsics/ierrno.f90 b/flang/test/Lower/Intrinsics/ierrno.f90
new file mode 100644
index 0000000000000..00e04989f4d59
--- /dev/null
+++ b/flang/test/Lower/Intrinsics/ierrno.f90
@@ -0,0 +1,13 @@
+! RUN: bbc -emit-fir %s -o - | FileCheck --check-prefixes=CHECK %s
+! RUN: %flang_fc1 -emit-fir %s -o - | FileCheck --check-prefixes=CHECK %s
+
+! CHECK-LABEL: func @_QPtest_ierrno(
+subroutine test_ierrno()
+    integer :: i
+    i = ierrno()
+! CHECK: %[[VAL_0:.*]] = fir.alloca i32 {bindc_name = "i", uniq_name = "_QFtest_ierrnoEi"}
+! CHECK: %[[VAL_1:.*]] = fir.declare %[[VAL_0]] {uniq_name = "_QFtest_ierrnoEi"} : (!fir.ref<i32>) -> !fir.ref<i32>
+! CHECK: %[[VAL_2:.*]] = fir.call @_QPierrno() fastmath<contract> : () -> i32
+! CHECK: fir.store %[[VAL_2]] to %[[VAL_1]] : !fir.ref<i32>
+! CHECK: return
+end subroutine test_ierrno
diff --git a/flang/test/Lower/Intrinsics/isnan.f90 b/flang/test/Lower/Intrinsics/isnan.f90
index 62b98c8ea98be..6535724b2ce3b 100644
--- a/flang/test/Lower/Intrinsics/isnan.f90
+++ b/flang/test/Lower/Intrinsics/isnan.f90
@@ -1,5 +1,4 @@
-! RUN: bbc -emit-fir %s -o - | FileCheck %s
-! RUN: flang -fc1 -emit-fir %s -o - | FileCheck %s
+! RUN: bbc -emit-fir %s -o - | FileCheck %s --check-prefixes=CHECK%if target=x86_64{{.*}} %{,CHECK-KIND10%}%if flang-supports-f128-math %{,CHECK-KIND16%}
 
 ! CHECK-LABEL: isnan_f32
 subroutine isnan_f32(r)
@@ -35,36 +34,40 @@ subroutine ieee_is_nan_f64(r)
   ! CHECK: fir.convert %[[l]] : (i1) -> !fir.logical<4>
 end subroutine ieee_is_nan_f64
 
-! CHECK-LABEL: isnan_f80
+! CHECK-KIND10-LABEL: isnan_f80
 subroutine isnan_f80(r)
-  real(KIND=10) :: r
+  integer, parameter :: kind10 = merge(10, 4, selected_real_kind(p=18).eq.10)
+  real(KIND=kind10) :: r
   i = isnan(r)
-  ! CHECK: %[[l:.*]] = "llvm.intr.is.fpclass"(%{{.*}}) <{bit = 3 : i32}> : (f80) -> i1
-  ! CHECK: fir.convert %[[l]] : (i1) -> !fir.logical<4>
+  ! CHECK-KIND10: %[[l:.*]] = "llvm.intr.is.fpclass"(%{{.*}}) <{bit = 3 : i32}> : (f80) -> i1
+  ! CHECK-KIND10: fir.convert %[[l]] : (i1) -> !fir.logical<4>
 end subroutine isnan_f80
 
-! CHECK-LABEL: ieee_is_nan_f80
+! CHECK-KIND10-LABEL: ieee_is_nan_f80
 subroutine ieee_is_nan_f80(r)
   use ieee_arithmetic
-  real(KIND=10) :: r
+  integer, parameter :: kind10 = merge(10, 4, selected_real_kind(p=18).eq.10)
+  real(KIND=kind10) :: r
   i = ieee_is_nan(r)
-  ! CHECK: %[[l:.*]] = "llvm.intr.is.fpclass"(%{{.*}}) <{bit = 3 : i32}> : (f80) -> i1
-  ! CHECK: fir.convert %[[l]] : (i1) -> !fir.logical<4>
+  ! CHECK-KIND10: %[[l:.*]] = "llvm.intr.is.fpclass"(%{{.*}}) <{bit = 3 : i32}> : (f80) -> i1
+  ! CHECK-KIND10: fir.convert %[[l]] : (i1) -> !fir.logical<4>
 end subroutine ieee_is_nan_f80
 
-! CHECK-LABEL: isnan_f128
+! CHECK-KIND16-LABEL: isnan_f128
 subroutine isnan_f128(r)
-  real(KIND=16) :: r
+  integer, parameter :: kind16 = merge(16, 4, selected_real_kind(p=33).eq.16)
+  real(KIND=kind16) :: r
   i = isnan(r)
-  ! CHECK: %[[l:.*]] = "llvm.intr.is.fpclass"(%{{.*}}) <{bit = 3 : i32}> : (f128) -> i1
-  ! CHECK: fir.convert %[[l]] : (i1) -> !fir.logical<4>
+  ! CHECK-KIND16: %[[l:.*]] = "llvm.intr.is.fpclass"(%{{.*}}) <{bit = 3 : i32}> : (f128) -> i1
+  ! CHECK-KIND16: fir.convert %[[l]] : (i1) -> !fir.logical<4>
 end subroutine isnan_f128
 
-! CHECK-LABEL: ieee_is_nan_f128
+! CHECK-KIND16-LABEL: ieee_is_nan_f128
 subroutine ieee_is_nan_f128(r)
   use ieee_arithmetic
-  real(KIND=16) :: r
+  integer, parameter :: kind16 = merge(16, 4, selected_real_kind(p=33).eq.16)
+  real(KIND=kind16) :: r
   i = ieee_is_nan(r)
-  ! CHECK: %[[l:.*]] = "llvm.intr.is.fpclass"(%{{.*}}) <{bit = 3 : i32}> : (f128) -> i1
-  ! CHECK: fir.convert %[[l]] : (i1) -> !fir.logical<4>
+  ! CHECK-KIND16: %[[l:.*]] = "llvm.intr.is.fpclass"(%{{.*}}) <{bit = 3 : i32}> : (f128) -> i1
+  ! CHECK-KIND16: fir.convert %[[l]] : (i1) -> !fir.logical<4>
 end subroutine ieee_is_nan_f128
diff --git a/flang/test/Lower/Intrinsics/mod.f90 b/flang/test/Lower/Intrinsics/mod.f90
index 3f5385ac303ab..5bc81d923b800 100644
--- a/flang/test/Lower/Intrinsics/mod.f90
+++ b/flang/test/Lower/Intrinsics/mod.f90
@@ -1,54 +1,38 @@
-! RUN: bbc -emit-fir -hlfir=false %s -o - | FileCheck %s
-! RUN: %flang_fc1 -emit-fir -flang-deprecated-no-hlfir %s -o - | FileCheck %s
+! RUN: bbc -emit-fir %s -o - | FileCheck %s --check-prefixes=CHECK%if target=x86_64{{.*}} %{,CHECK-KIND10%}%if flang-supports-f128-math %{,CHECK-KIND16%}
 
 ! CHECK-LABEL: func @_QPmod_testr4(
-! CHECK-SAME: %[[arg0:.*]]: !fir.ref<f32>{{.*}}, %[[arg1:.*]]: !fir.ref<f32>{{.*}}, %[[arg2:.*]]: !fir.ref<f32>{{.*}}) {
 subroutine mod_testr4(r, a, p)
   real(4) :: r, a, p
-! CHECK: %[[V1:.*]] = fir.load %[[arg1]] : !fir.ref<f32>
-! CHECK: %[[V2:.*]] = fir.load %[[arg2]] : !fir.ref<f32>
-! CHECK: %[[FILE:.*]] = fir.address_of(@{{.*}}) : !fir.ref<!fir.char<1,{{.*}}>>
 ! CHECK: %[[LINE:.*]] = arith.constant {{[0-9]*}} : i32
+! CHECK: %[[A:.*]] = fir.declare{{.*}}a"
+! CHECK: %[[P:.*]] = fir.declare{{.*}}p"
+! CHECK: %[[A_LOAD:.*]] = fir.load %[[A]]
+! CHECK: %[[P_LOAD:.*]] = fir.load %[[P]]
+! CHECK: %[[FILE:.*]] = fir.address_of(@{{.*}}) : !fir.ref<!fir.char<1,{{.*}}>>
 ! CHECK: %[[FILEARG:.*]] = fir.convert %[[FILE]] : (!fir.ref<!fir.char<1,{{.*}}>>) -> !fir.ref<i8>
-! CHECK: fir.call @_FortranAModReal4(%[[V1]], %[[V2]], %[[FILEARG]], %[[LINE]]) {{.*}}: (f32, f32, !fir.ref<i8>, i32) -> f32
+! CHECK: fir.call @_FortranAModReal4(%[[A_LOAD]], %[[P_LOAD]], %[[FILEARG]], %[[LINE]]) {{.*}}: (f32, f32, !fir.ref<i8>, i32) -> f32
   r = mod(a, p)
 end subroutine
 
 ! CHECK-LABEL: func @_QPmod_testr8(
-! CHECK-SAME: %[[arg0:.*]]: !fir.ref<f64>{{.*}}, %[[arg1:.*]]: !fir.ref<f64>{{.*}}, %[[arg2:.*]]: !fir.ref<f64>{{.*}}) {
 subroutine mod_testr8(r, a, p)
   real(8) :: r, a, p
-! CHECK: %[[V1:.*]] = fir.load %[[arg1]] : !fir.ref<f64>
-! CHECK: %[[V2:.*]] = fir.load %[[arg2]] : !fir.ref<f64>
-! CHECK: %[[FILE:.*]] = fir.address_of(@{{.*}}) : !fir.ref<!fir.char<1,{{.*}}>>
-! CHECK: %[[LINE:.*]] = arith.constant {{[0-9]*}} : i32
-! CHECK: %[[FILEARG:.*]] = fir.convert %[[FILE]] : (!fir.ref<!fir.char<1,{{.*}}>>) -> !fir.ref<i8>
-! CHECK: fir.call @_FortranAModReal8(%[[V1]], %[[V2]], %[[FILEARG]], %[[LINE]]) {{.*}}: (f64, f64, !fir.ref<i8>, i32) -> f64
+! CHECK: fir.call @_FortranAModReal8(%{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}) {{.*}}: (f64, f64, !fir.ref<i8>, i32) -> f64
   r = mod(a, p)
 end subroutine
 
-! CHECK-LABEL: func @_QPmod_testr10(
-! CHECK-SAME: %[[arg0:.*]]: !fir.ref<f80>{{.*}}, %[[arg1:.*]]: !fir.ref<f80>{{.*}}, %[[arg2:.*]]: !fir.ref<f80>{{.*}}) {
+! CHECK-KIND10-LABEL: func @_QPmod_testr10(
 subroutine mod_testr10(r, a, p)
-  real(10) :: r, a, p
-! CHECK: %[[V1:.*]] = fir.load %[[arg1]] : !fir.ref<f80>
-! CHECK: %[[V2:.*]] = fir.load %[[arg2]] : !fir.ref<f80>
-! CHECK: %[[FILE:.*]] = fir.address_of(@{{.*}}) : !fir.ref<!fir.char<1,{{.*}}>>
-! CHECK: %[[LINE:.*]] = arith.constant {{[0-9]*}} : i32
-! CHECK: %[[FILEARG:.*]] = fir.convert %[[FILE]] : (!fir.ref<!fir.char<1,{{.*}}>>) -> !fir.ref<i8>
-! CHECK: fir.call @_FortranAModReal10(%[[V1]], %[[V2]], %[[FILEARG]], %[[LINE]]) {{.*}}: (f80, f80, !fir.ref<i8>, i32) -> f80
+  integer, parameter :: kind10 = merge(10, 4, selected_real_kind(p=18).eq.10)
+  real(kind10) :: r, a, p
+! CHECK-KIND10: fir.call @_FortranAModReal10(%{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}) {{.*}}: (f80, f80, !fir.ref<i8>, i32) -> f80
   r = mod(a, p)
 end subroutine
 
-! CHECK-LABEL: func @_QPmod_testr16(
-! CHECK-SAME: %[[arg0:.*]]: !fir.ref<f128>{{.*}}, %[[arg1:.*]]: !fir.ref<f128>{{.*}}, %[[arg2:.*]]: !fir.ref<f128>{{.*}}) {
+! CHECK-KIND16-LABEL: func @_QPmod_testr16(
 subroutine mod_testr16(r, a, p)
-  real(16) :: r, a, p
-! CHECK: %[[V1:.*]] = fir.load %[[arg1]] : !fir.ref<f128>
-! CHECK: %[[V2:.*]] = fir.load %[[arg2]] : !fir.ref<f128>
-! CHECK: %[[FILE:.*]] = fir.address_of(@{{.*}}) : !fir.ref<!fir.char<1,{{.*}}>>
-! CHECK: %[[LINE:.*]] = arith.constant {{[0-9]*}} : i32
-! CHECK: %[[FILEARG:.*]] = fir.convert %[[FILE]] : (!fir.ref<!fir.char<1,{{.*}}>>) -> !fir.ref<i8>
-! CHECK: fir.call @_FortranAModReal16(%[[V1]], %[[V2]], %[[FILEARG]], %[[LINE]]) {{.*}}: (f128, f128, !fir.ref<i8>, i32) -> f128
+  integer, parameter :: kind16 = merge(16, 4, selected_real_kind(p=33).eq.16)
+  real(kind16) :: r, a, p
+! CHECK-KIND16: fir.call @_FortranAModReal16(%{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}) {{.*}}: (f128, f128, !fir.ref<i8>, i32) -> f128
   r = mod(a, p)
 end subroutine
diff --git a/flang/test/Lower/Intrinsics/modulo.f90 b/flang/test/Lower/Intrinsics/modulo.f90
index 781ef8296a2b7..37c4cd1a94ca2 100644
--- a/flang/test/Lower/Intrinsics/modulo.f90
+++ b/flang/test/Lower/Intrinsics/modulo.f90
@@ -1,5 +1,5 @@
 ! RUN: bbc -emit-fir -hlfir=false %s -o - | FileCheck %s -check-prefixes=HONORINF,ALL
-! RUN: flang -fc1 -menable-no-infs -emit-fir -flang-deprecated-no-hlfir %s -o - | FileCheck %s -check-prefixes=CHECK,ALL
+! RUN: flang -fc1 -menable-no-infs -emit-fir -flang-deprecated-no-hlfir %s -o - | FileCheck %s -check-prefixes=CHECK,ALL,%if flang-supports-f128-math %{F128%} %else %{F64%}
 
 ! ALL-LABEL: func @_QPmodulo_testr(
 ! ALL-SAME: %[[arg0:.*]]: !fir.ref<f64>{{.*}}, %[[arg1:.*]]: !fir.ref<f64>{{.*}}, %[[arg2:.*]]: !fir.ref<f64>{{.*}}) {
@@ -39,9 +39,12 @@ subroutine modulo_testi(r, a, p)
 end subroutine
 
 ! CHECK-LABEL: func @_QPmodulo_testr16(
-! CHECK-SAME: %[[arg0:.*]]: !fir.ref<f128>{{.*}}, %[[arg1:.*]]: !fir.ref<f128>{{.*}}, %[[arg2:.*]]: !fir.ref<f128>{{.*}}) {
+! F128-SAME: %[[arg0:.*]]: !fir.ref<f128>{{.*}}, %[[arg1:.*]]: !fir.ref<f128>{{.*}}, %[[arg2:.*]]: !fir.ref<f128>{{.*}}) {
+! F64-SAME: %[[arg0:.*]]: !fir.ref<f64>{{.*}}, %[[arg1:.*]]: !fir.ref<f64>{{.*}}, %[[arg2:.*]]: !fir.ref<f64>{{.*}}) {
 subroutine modulo_testr16(r, a, p)
-  real(16) :: r, a, p
-  ! CHECK: fir.call @_FortranAModuloReal16({{.*}}){{.*}}: (f128, f128, !fir.ref<i8>, i32) -> f128
+  integer, parameter :: rk = merge(16, 8, selected_real_kind(33, 4931)==16)
+  real(rk) :: r, a, p
+  !F128: fir.call @_FortranAModuloReal16({{.*}}){{.*}}: (f128, f128, !fir.ref<i8>, i32) -> f128
+  !F64: arith.remf %0, %1 fastmath<ninf,contract> : f64
   r = modulo(a, p)
 end subroutine
diff --git a/flang/test/Lower/Intrinsics/nearest.f90 b/flang/test/Lower/Intrinsics/nearest.f90
index 5920d299d5fdf..d4859cfe90e54 100644
--- a/flang/test/Lower/Intrinsics/nearest.f90
+++ b/flang/test/Lower/Intrinsics/nearest.f90
@@ -1,4 +1,4 @@
-! RUN: bbc -emit-fir %s -o - | FileCheck %s
+! RUN: bbc -emit-fir %s -o - | FileCheck %s --check-prefixes=CHECK%if target=x86_64{{.*}} %{,CHECK-KIND10%}%if flang-supports-f128-math %{,CHECK-KIND16%}
 
 ! CHECK-LABEL: c.func @_QPnearest_test1
   ! CHECK:     %[[V_0:[0-9]+]] = fir.dummy_scope : !fir.dscope
@@ -240,168 +240,171 @@ subroutine nearest_test4(x, s)
   res = nearest(x, s)
 end
 
-! CHECK-LABEL: c.func @_QPnearest_test5
-  ! CHECK:     %[[V_0:[0-9]+]] = fir.dummy_scope : !fir.dscope
-  ! CHECK:     %[[V_1:[0-9]+]] = fir.alloca f80 {bindc_name = "res", uniq_name = "_QFnearest_test5Eres"}
-  ! CHECK:     %[[V_2:[0-9]+]] = fir.declare %[[V_1]] {uniq_name = "_QFnearest_test5Eres"} : (!fir.ref<f80>) -> !fir.ref<f80>
-  ! CHECK:     %[[V_3:[0-9]+]] = fir.declare %arg1 dummy_scope %[[V_0]] {uniq_name = "_QFnearest_test5Es"} : (!fir.ref<f80>, !fir.dscope) -> !fir.ref<f80>
-  ! CHECK:     %[[V_4:[0-9]+]] = fir.declare %arg0 dummy_scope %[[V_0]] {uniq_name = "_QFnearest_test5Ex"} : (!fir.ref<f80>, !fir.dscope) -> !fir.ref<f80>
-  ! CHECK:     %[[V_5:[0-9]+]] = fir.load %[[V_4]] : !fir.ref<f80>
-  ! CHECK:     %[[V_6:[0-9]+]] = fir.load %[[V_3]] : !fir.ref<f80>
-  ! CHECK:     %[[V_7:[0-9]+]] = "llvm.intr.is.fpclass"(%[[V_5]]) <{bit = 3 : i32}> : (f80) -> i1
-  ! CHECK:     %[[V_8:[0-9]+]] = "llvm.intr.is.fpclass"(%[[V_6]]) <{bit = 96 : i32}> : (f80) -> i1
-  ! CHECK:     fir.if %[[V_8]] {
-  ! CHECK:       fir.call @_FortranAReportFatalUserError
-  ! CHECK:     }
-  ! CHECK:     %[[V_9:[0-9]+]] = arith.bitcast %[[V_6]] : f80 to i80
-  ! CHECK:     %[[V_10:[0-9]+]] = arith.shrui %[[V_9]], %c79{{.*}} : i80
-  ! CHECK:     %[[V_11:[0-9]+]] = arith.cmpi ne, %[[V_10]], %c1{{.*}} : i80
-  ! CHECK:     %[[V_12:[0-9]+]] = arith.bitcast %[[V_5]] : f80 to i80
-  ! CHECK:     %[[V_13:[0-9]+]] = arith.shrui %[[V_12]], %c79{{.*}} : i80
-  ! CHECK:     %[[V_14:[0-9]+]] = fir.convert %[[V_13]] : (i80) -> i1
-  ! CHECK:     %[[V_15:[0-9]+]] = arith.cmpi ne, %[[V_11]], %[[V_14]] : i1
-  ! CHECK:     %[[V_16:[0-9]+]] = "llvm.intr.is.fpclass"(%[[V_5]]) <{bit = 516 : i32}> : (f80) -> i1
-  ! CHECK:     %[[V_17:[0-9]+]] = arith.andi %[[V_16]], %[[V_15]] : i1
-  ! CHECK:     %[[V_18:[0-9]+]] = arith.ori %[[V_7]], %[[V_17]] : i1
-  ! CHECK:     %[[V_19:[0-9]+]] = fir.if %[[V_18]] -> (f80) {
-  ! CHECK:       fir.result %[[V_5]] : f80
-  ! CHECK:     } else {
-  ! CHECK:       %[[V_20:[0-9]+]] = arith.cmpf oeq, %[[V_5]], %cst{{[_0-9]*}} fastmath<contract> : f80
-  ! CHECK:       %[[V_21:[0-9]+]] = fir.if %[[V_20]] -> (f80) {
-  ! CHECK:         %[[V_22:[0-9]+]] = arith.select %[[V_11]], %cst{{[_0-9]*}}, %cst{{[_0-9]*}} : f80
-  ! CHECK:         %[[V_23:[0-9]+]] = fir.call @_FortranAMapException(%c48{{.*}}) fastmath<contract> : (i32) -> i32
-  ! CHECK:                            fir.call @feraiseexcept(%[[V_23]]) fastmath<contract> : (i32) -> i32
-  ! CHECK:         fir.result %[[V_22]] : f80
-  ! CHECK:       } else {
-  ! CHECK:         %[[V_22:[0-9]+]] = fir.call @_FortranANearest10(%[[V_5]], %[[V_11]]) fastmath<contract> : (f80, i1) -> f80
-  ! CHECK:         fir.result %[[V_22]] : f80
-  ! CHECK:       }
-  ! CHECK:       fir.result %[[V_21]] : f80
-  ! CHECK:     }
-  ! CHECK:     fir.store %[[V_19]] to %[[V_2]] : !fir.ref<f80>
-  ! CHECK:     return
-  ! CHECK:   }
+! CHECK-KIND10-LABEL: c.func @_QPnearest_test5
+  ! CHECK-KIND10:     %[[V_0:[0-9]+]] = fir.dummy_scope : !fir.dscope
+  ! CHECK-KIND10:     %[[V_1:[0-9]+]] = fir.alloca f80 {bindc_name = "res", uniq_name = "_QFnearest_test5Eres"}
+  ! CHECK-KIND10:     %[[V_2:[0-9]+]] = fir.declare %[[V_1]] {uniq_name = "_QFnearest_test5Eres"} : (!fir.ref<f80>) -> !fir.ref<f80>
+  ! CHECK-KIND10:     %[[V_3:[0-9]+]] = fir.declare %arg1 dummy_scope %[[V_0]] {uniq_name = "_QFnearest_test5Es"} : (!fir.ref<f80>, !fir.dscope) -> !fir.ref<f80>
+  ! CHECK-KIND10:     %[[V_4:[0-9]+]] = fir.declare %arg0 dummy_scope %[[V_0]] {uniq_name = "_QFnearest_test5Ex"} : (!fir.ref<f80>, !fir.dscope) -> !fir.ref<f80>
+  ! CHECK-KIND10:     %[[V_5:[0-9]+]] = fir.load %[[V_4]] : !fir.ref<f80>
+  ! CHECK-KIND10:     %[[V_6:[0-9]+]] = fir.load %[[V_3]] : !fir.ref<f80>
+  ! CHECK-KIND10:     %[[V_7:[0-9]+]] = "llvm.intr.is.fpclass"(%[[V_5]]) <{bit = 3 : i32}> : (f80) -> i1
+  ! CHECK-KIND10:     %[[V_8:[0-9]+]] = "llvm.intr.is.fpclass"(%[[V_6]]) <{bit = 96 : i32}> : (f80) -> i1
+  ! CHECK-KIND10:     fir.if %[[V_8]] {
+  ! CHECK-KIND10:       fir.call @_FortranAReportFatalUserError
+  ! CHECK-KIND10:     }
+  ! CHECK-KIND10:     %[[V_9:[0-9]+]] = arith.bitcast %[[V_6]] : f80 to i80
+  ! CHECK-KIND10:     %[[V_10:[0-9]+]] = arith.shrui %[[V_9]], %c79{{.*}} : i80
+  ! CHECK-KIND10:     %[[V_11:[0-9]+]] = arith.cmpi ne, %[[V_10]], %c1{{.*}} : i80
+  ! CHECK-KIND10:     %[[V_12:[0-9]+]] = arith.bitcast %[[V_5]] : f80 to i80
+  ! CHECK-KIND10:     %[[V_13:[0-9]+]] = arith.shrui %[[V_12]], %c79{{.*}} : i80
+  ! CHECK-KIND10:     %[[V_14:[0-9]+]] = fir.convert %[[V_13]] : (i80) -> i1
+  ! CHECK-KIND10:     %[[V_15:[0-9]+]] = arith.cmpi ne, %[[V_11]], %[[V_14]] : i1
+  ! CHECK-KIND10:     %[[V_16:[0-9]+]] = "llvm.intr.is.fpclass"(%[[V_5]]) <{bit = 516 : i32}> : (f80) -> i1
+  ! CHECK-KIND10:     %[[V_17:[0-9]+]] = arith.andi %[[V_16]], %[[V_15]] : i1
+  ! CHECK-KIND10:     %[[V_18:[0-9]+]] = arith.ori %[[V_7]], %[[V_17]] : i1
+  ! CHECK-KIND10:     %[[V_19:[0-9]+]] = fir.if %[[V_18]] -> (f80) {
+  ! CHECK-KIND10:       fir.result %[[V_5]] : f80
+  ! CHECK-KIND10:     } else {
+  ! CHECK-KIND10:       %[[V_20:[0-9]+]] = arith.cmpf oeq, %[[V_5]], %cst{{[_0-9]*}} fastmath<contract> : f80
+  ! CHECK-KIND10:       %[[V_21:[0-9]+]] = fir.if %[[V_20]] -> (f80) {
+  ! CHECK-KIND10:         %[[V_22:[0-9]+]] = arith.select %[[V_11]], %cst{{[_0-9]*}}, %cst{{[_0-9]*}} : f80
+  ! CHECK-KIND10:         %[[V_23:[0-9]+]] = fir.call @_FortranAMapException(%c48{{.*}}) fastmath<contract> : (i32) -> i32
+  ! CHECK-KIND10:                            fir.call @feraiseexcept(%[[V_23]]) fastmath<contract> : (i32) -> i32
+  ! CHECK-KIND10:         fir.result %[[V_22]] : f80
+  ! CHECK-KIND10:       } else {
+  ! CHECK-KIND10:         %[[V_22:[0-9]+]] = fir.call @_FortranANearest10(%[[V_5]], %[[V_11]]) fastmath<contract> : (f80, i1) -> f80
+  ! CHECK-KIND10:         fir.result %[[V_22]] : f80
+  ! CHECK-KIND10:       }
+  ! CHECK-KIND10:       fir.result %[[V_21]] : f80
+  ! CHECK-KIND10:     }
+  ! CHECK-KIND10:     fir.store %[[V_19]] to %[[V_2]] : !fir.ref<f80>
+  ! CHECK-KIND10:     return
+  ! CHECK-KIND10:   }
 subroutine nearest_test5(x, s)
-  real(kind=10) :: x, s, res
+  integer, parameter :: kind10 = merge(10, 4, selected_real_kind(p=18).eq.10)
+  real(kind=kind10) :: x, s, res
   res = nearest(x, s)
 end
 
-! CHECK-LABEL: c.func @_QPnearest_test6
-  ! CHECK:     %[[V_0:[0-9]+]] = fir.dummy_scope : !fir.dscope
-  ! CHECK:     %[[V_1:[0-9]+]] = fir.alloca f128 {bindc_name = "res", uniq_name = "_QFnearest_test6Eres"}
-  ! CHECK:     %[[V_2:[0-9]+]] = fir.declare %[[V_1]] {uniq_name = "_QFnearest_test6Eres"} : (!fir.ref<f128>) -> !fir.ref<f128>
-  ! CHECK:     %[[V_3:[0-9]+]] = fir.declare %arg1 dummy_scope %[[V_0]] {uniq_name = "_QFnearest_test6Es"} : (!fir.ref<f128>, !fir.dscope) -> !fir.ref<f128>
-  ! CHECK:     %[[V_4:[0-9]+]] = fir.declare %arg0 dummy_scope %[[V_0]] {uniq_name = "_QFnearest_test6Ex"} : (!fir.ref<f128>, !fir.dscope) -> !fir.ref<f128>
-  ! CHECK:     %[[V_5:[0-9]+]] = fir.load %[[V_4]] : !fir.ref<f128>
-  ! CHECK:     %[[V_6:[0-9]+]] = fir.load %[[V_3]] : !fir.ref<f128>
-  ! CHECK:     %[[V_7:[0-9]+]] = "llvm.intr.is.fpclass"(%[[V_5]]) <{bit = 3 : i32}> : (f128) -> i1
-  ! CHECK:     %[[V_8:[0-9]+]] = "llvm.intr.is.fpclass"(%[[V_6]]) <{bit = 96 : i32}> : (f128) -> i1
-  ! CHECK:     fir.if %[[V_8]] {
-  ! CHECK:       fir.call @_FortranAReportFatalUserError
-  ! CHECK:     }
-  ! CHECK:     %[[V_9:[0-9]+]] = arith.bitcast %[[V_6]] : f128 to i128
-  ! CHECK:     %[[V_10:[0-9]+]] = arith.shrui %[[V_9]], %c127{{.*}} : i128
-  ! CHECK:     %[[V_11:[0-9]+]] = arith.cmpi ne, %[[V_10]], %c1{{.*}} : i128
-  ! CHECK:     %[[V_12:[0-9]+]] = arith.bitcast %[[V_5]] : f128 to i128
-  ! CHECK:     %[[V_13:[0-9]+]] = arith.shrui %[[V_12]], %c127{{.*}} : i128
-  ! CHECK:     %[[V_14:[0-9]+]] = fir.convert %[[V_13]] : (i128) -> i1
-  ! CHECK:     %[[V_15:[0-9]+]] = arith.cmpi ne, %[[V_11]], %[[V_14]] : i1
-  ! CHECK:     %[[V_16:[0-9]+]] = "llvm.intr.is.fpclass"(%[[V_5]]) <{bit = 516 : i32}> : (f128) -> i1
-  ! CHECK:     %[[V_17:[0-9]+]] = arith.andi %[[V_16]], %[[V_15]] : i1
-  ! CHECK:     %[[V_18:[0-9]+]] = arith.ori %[[V_7]], %[[V_17]] : i1
-  ! CHECK:     %[[V_19:[0-9]+]] = fir.if %[[V_18]] -> (f128) {
-  ! CHECK:       fir.result %[[V_5]] : f128
-  ! CHECK:     } else {
-  ! CHECK:       %[[V_20:[0-9]+]] = arith.cmpf oeq, %[[V_5]], %cst{{[_0-9]*}} fastmath<contract> : f128
-  ! CHECK:       %[[V_21:[0-9]+]] = fir.if %[[V_20]] -> (f128) {
-  ! CHECK:         %[[V_22:[0-9]+]] = arith.select %[[V_11]], %cst{{[_0-9]*}}, %cst{{[_0-9]*}} : f128
-  ! CHECK:         %[[V_23:[0-9]+]] = fir.call @_FortranAMapException(%c48{{.*}}) fastmath<contract> : (i32) -> i32
-  ! CHECK:                            fir.call @feraiseexcept(%[[V_23]]) fastmath<contract> : (i32) -> i32
-  ! CHECK:         fir.result %[[V_22]] : f128
-  ! CHECK:       } else {
-  ! CHECK-DAG:     %[[V_22:[0-9]+]] = arith.subi %[[V_12]], %c1{{.*}} : i128
-  ! CHECK-DAG:     %[[V_23:[0-9]+]] = arith.addi %[[V_12]], %c1{{.*}} : i128
-  ! CHECK:         %[[V_24:[0-9]+]] = arith.select %[[V_15]], %[[V_23]], %[[V_22]] : i128
-  ! CHECK:         %[[V_25:[0-9]+]] = arith.bitcast %[[V_24]] : i128 to f128
-  ! CHECK:         %[[V_26:[0-9]+]] = "llvm.intr.is.fpclass"(%[[V_25]]) <{bit = 516 : i32}> : (f128) -> i1
-  ! CHECK:         fir.if %[[V_26]] {
-  ! CHECK:           %[[V_28:[0-9]+]] = fir.call @_FortranAMapException(%c40{{.*}}) fastmath<contract> : (i32) -> i32
-  ! CHECK:                              fir.call @feraiseexcept(%[[V_28]]) fastmath<contract> : (i32) -> i32
-  ! CHECK:         }
-  ! CHECK:         %[[V_27:[0-9]+]] = "llvm.intr.is.fpclass"(%[[V_25]]) <{bit = 144 : i32}> : (f128) -> i1
-  ! CHECK:         fir.if %[[V_27]] {
-  ! CHECK:           %[[V_28:[0-9]+]] = fir.call @_FortranAMapException(%c48{{.*}}) fastmath<contract> : (i32) -> i32
-  ! CHECK:                              fir.call @feraiseexcept(%[[V_28]]) fastmath<contract> : (i32) -> i32
-  ! CHECK:         }
-  ! CHECK:         fir.result %[[V_25]] : f128
-  ! CHECK:       }
-  ! CHECK:       fir.result %[[V_21]] : f128
-  ! CHECK:     }
-  ! CHECK:     fir.store %[[V_19]] to %[[V_2]] : !fir.ref<f128>
-  ! CHECK:     return
-  ! CHECK:   }
+! CHECK-KIND16-LABEL: c.func @_QPnearest_test6
+  ! CHECK-KIND16:     %[[V_0:[0-9]+]] = fir.dummy_scope : !fir.dscope
+  ! CHECK-KIND16:     %[[V_1:[0-9]+]] = fir.alloca f128 {bindc_name = "res", uniq_name = "_QFnearest_test6Eres"}
+  ! CHECK-KIND16:     %[[V_2:[0-9]+]] = fir.declare %[[V_1]] {uniq_name = "_QFnearest_test6Eres"} : (!fir.ref<f128>) -> !fir.ref<f128>
+  ! CHECK-KIND16:     %[[V_3:[0-9]+]] = fir.declare %arg1 dummy_scope %[[V_0]] {uniq_name = "_QFnearest_test6Es"} : (!fir.ref<f128>, !fir.dscope) -> !fir.ref<f128>
+  ! CHECK-KIND16:     %[[V_4:[0-9]+]] = fir.declare %arg0 dummy_scope %[[V_0]] {uniq_name = "_QFnearest_test6Ex"} : (!fir.ref<f128>, !fir.dscope) -> !fir.ref<f128>
+  ! CHECK-KIND16:     %[[V_5:[0-9]+]] = fir.load %[[V_4]] : !fir.ref<f128>
+  ! CHECK-KIND16:     %[[V_6:[0-9]+]] = fir.load %[[V_3]] : !fir.ref<f128>
+  ! CHECK-KIND16:     %[[V_7:[0-9]+]] = "llvm.intr.is.fpclass"(%[[V_5]]) <{bit = 3 : i32}> : (f128) -> i1
+  ! CHECK-KIND16:     %[[V_8:[0-9]+]] = "llvm.intr.is.fpclass"(%[[V_6]]) <{bit = 96 : i32}> : (f128) -> i1
+  ! CHECK-KIND16:     fir.if %[[V_8]] {
+  ! CHECK-KIND16:       fir.call @_FortranAReportFatalUserError
+  ! CHECK-KIND16:     }
+  ! CHECK-KIND16:     %[[V_9:[0-9]+]] = arith.bitcast %[[V_6]] : f128 to i128
+  ! CHECK-KIND16:     %[[V_10:[0-9]+]] = arith.shrui %[[V_9]], %c127{{.*}} : i128
+  ! CHECK-KIND16:     %[[V_11:[0-9]+]] = arith.cmpi ne, %[[V_10]], %c1{{.*}} : i128
+  ! CHECK-KIND16:     %[[V_12:[0-9]+]] = arith.bitcast %[[V_5]] : f128 to i128
+  ! CHECK-KIND16:     %[[V_13:[0-9]+]] = arith.shrui %[[V_12]], %c127{{.*}} : i128
+  ! CHECK-KIND16:     %[[V_14:[0-9]+]] = fir.convert %[[V_13]] : (i128) -> i1
+  ! CHECK-KIND16:     %[[V_15:[0-9]+]] = arith.cmpi ne, %[[V_11]], %[[V_14]] : i1
+  ! CHECK-KIND16:     %[[V_16:[0-9]+]] = "llvm.intr.is.fpclass"(%[[V_5]]) <{bit = 516 : i32}> : (f128) -> i1
+  ! CHECK-KIND16:     %[[V_17:[0-9]+]] = arith.andi %[[V_16]], %[[V_15]] : i1
+  ! CHECK-KIND16:     %[[V_18:[0-9]+]] = arith.ori %[[V_7]], %[[V_17]] : i1
+  ! CHECK-KIND16:     %[[V_19:[0-9]+]] = fir.if %[[V_18]] -> (f128) {
+  ! CHECK-KIND16:       fir.result %[[V_5]] : f128
+  ! CHECK-KIND16:     } else {
+  ! CHECK-KIND16:       %[[V_20:[0-9]+]] = arith.cmpf oeq, %[[V_5]], %cst{{[_0-9]*}} fastmath<contract> : f128
+  ! CHECK-KIND16:       %[[V_21:[0-9]+]] = fir.if %[[V_20]] -> (f128) {
+  ! CHECK-KIND16:         %[[V_22:[0-9]+]] = arith.select %[[V_11]], %cst{{[_0-9]*}}, %cst{{[_0-9]*}} : f128
+  ! CHECK-KIND16:         %[[V_23:[0-9]+]] = fir.call @_FortranAMapException(%c48{{.*}}) fastmath<contract> : (i32) -> i32
+  ! CHECK-KIND16:                            fir.call @feraiseexcept(%[[V_23]]) fastmath<contract> : (i32) -> i32
+  ! CHECK-KIND16:         fir.result %[[V_22]] : f128
+  ! CHECK-KIND16:       } else {
+  ! CHECK-KIND16-DAG:     %[[V_22:[0-9]+]] = arith.subi %[[V_12]], %c1{{.*}} : i128
+  ! CHECK-KIND16-DAG:     %[[V_23:[0-9]+]] = arith.addi %[[V_12]], %c1{{.*}} : i128
+  ! CHECK-KIND16:         %[[V_24:[0-9]+]] = arith.select %[[V_15]], %[[V_23]], %[[V_22]] : i128
+  ! CHECK-KIND16:         %[[V_25:[0-9]+]] = arith.bitcast %[[V_24]] : i128 to f128
+  ! CHECK-KIND16:         %[[V_26:[0-9]+]] = "llvm.intr.is.fpclass"(%[[V_25]]) <{bit = 516 : i32}> : (f128) -> i1
+  ! CHECK-KIND16:         fir.if %[[V_26]] {
+  ! CHECK-KIND16:           %[[V_28:[0-9]+]] = fir.call @_FortranAMapException(%c40{{.*}}) fastmath<contract> : (i32) -> i32
+  ! CHECK-KIND16:                              fir.call @feraiseexcept(%[[V_28]]) fastmath<contract> : (i32) -> i32
+  ! CHECK-KIND16:         }
+  ! CHECK-KIND16:         %[[V_27:[0-9]+]] = "llvm.intr.is.fpclass"(%[[V_25]]) <{bit = 144 : i32}> : (f128) -> i1
+  ! CHECK-KIND16:         fir.if %[[V_27]] {
+  ! CHECK-KIND16:           %[[V_28:[0-9]+]] = fir.call @_FortranAMapException(%c48{{.*}}) fastmath<contract> : (i32) -> i32
+  ! CHECK-KIND16:                              fir.call @feraiseexcept(%[[V_28]]) fastmath<contract> : (i32) -> i32
+  ! CHECK-KIND16:         }
+  ! CHECK-KIND16:         fir.result %[[V_25]] : f128
+  ! CHECK-KIND16:       }
+  ! CHECK-KIND16:       fir.result %[[V_21]] : f128
+  ! CHECK-KIND16:     }
+  ! CHECK-KIND16:     fir.store %[[V_19]] to %[[V_2]] : !fir.ref<f128>
+  ! CHECK-KIND16:     return
+  ! CHECK-KIND16:   }
 subroutine nearest_test6(x, s)
-  real(kind=16) :: x, s, res
+  integer, parameter :: kind16 = merge(16, 4, selected_real_kind(p=33).eq.16)
+  real(kind=kind16) :: x, s, res
   res = nearest(x, s)
 end
 
-! CHECK-LABEL: c.func @_QPnearest_test7
-  ! CHECK:     %[[V_0:[0-9]+]] = fir.dummy_scope : !fir.dscope
-  ! CHECK:     %[[V_1:[0-9]+]] = fir.alloca f128 {bindc_name = "res", uniq_name = "_QFnearest_test7Eres"}
-  ! CHECK:     %[[V_2:[0-9]+]] = fir.declare %[[V_1]] {uniq_name = "_QFnearest_test7Eres"} : (!fir.ref<f128>) -> !fir.ref<f128>
-  ! CHECK:     %[[V_3:[0-9]+]] = fir.declare %arg1 dummy_scope %[[V_0]] {uniq_name = "_QFnearest_test7Es"} : (!fir.ref<f32>, !fir.dscope) -> !fir.ref<f32>
-  ! CHECK:     %[[V_4:[0-9]+]] = fir.declare %arg0 dummy_scope %[[V_0]] {uniq_name = "_QFnearest_test7Ex"} : (!fir.ref<f128>, !fir.dscope) -> !fir.ref<f128>
-  ! CHECK:     %[[V_5:[0-9]+]] = fir.load %[[V_4]] : !fir.ref<f128>
-  ! CHECK:     %[[V_6:[0-9]+]] = fir.load %[[V_3]] : !fir.ref<f32>
-  ! CHECK:     %[[V_7:[0-9]+]] = "llvm.intr.is.fpclass"(%[[V_5]]) <{bit = 3 : i32}> : (f128) -> i1
-  ! CHECK:     %[[V_8:[0-9]+]] = "llvm.intr.is.fpclass"(%[[V_6]]) <{bit = 96 : i32}> : (f32) -> i1
-  ! CHECK:     fir.if %[[V_8]] {
-  ! CHECK:       fir.call @_FortranAReportFatalUserError
-  ! CHECK:     }
-  ! CHECK:     %[[V_9:[0-9]+]] = arith.bitcast %[[V_6]] : f32 to i32
-  ! CHECK:     %[[V_10:[0-9]+]] = arith.shrui %[[V_9]], %c31{{.*}} : i32
-  ! CHECK:     %[[V_11:[0-9]+]] = fir.convert %[[V_10]] : (i32) -> i128
-  ! CHECK:     %[[V_12:[0-9]+]] = arith.cmpi ne, %[[V_11]], %c1{{.*}} : i128
-  ! CHECK:     %[[V_13:[0-9]+]] = arith.bitcast %[[V_5]] : f128 to i128
-  ! CHECK:     %[[V_14:[0-9]+]] = arith.shrui %[[V_13]], %c127{{.*}} : i128
-  ! CHECK:     %[[V_15:[0-9]+]] = fir.convert %[[V_14]] : (i128) -> i1
-  ! CHECK:     %[[V_16:[0-9]+]] = arith.cmpi ne, %[[V_12]], %[[V_15]] : i1
-  ! CHECK:     %[[V_17:[0-9]+]] = "llvm.intr.is.fpclass"(%[[V_5]]) <{bit = 516 : i32}> : (f128) -> i1
-  ! CHECK:     %[[V_18:[0-9]+]] = arith.andi %[[V_17]], %[[V_16]] : i1
-  ! CHECK:     %[[V_19:[0-9]+]] = arith.ori %[[V_7]], %[[V_18]] : i1
-  ! CHECK:     %[[V_20:[0-9]+]] = fir.if %[[V_19]] -> (f128) {
-  ! CHECK:       fir.result %[[V_5]] : f128
-  ! CHECK:     } else {
-  ! CHECK:       %[[V_21:[0-9]+]] = arith.cmpf oeq, %[[V_5]], %cst{{[_0-9]*}} fastmath<contract> : f128
-  ! CHECK:       %[[V_22:[0-9]+]] = fir.if %[[V_21]] -> (f128) {
-  ! CHECK:         %[[V_23:[0-9]+]] = arith.select %[[V_12]], %cst{{[_0-9]*}}, %cst{{[_0-9]*}} : f128
-  ! CHECK:         %[[V_24:[0-9]+]] = fir.call @_FortranAMapException(%c48{{.*}}) fastmath<contract> : (i32) -> i32
-  ! CHECK:                            fir.call @feraiseexcept(%[[V_24]]) fastmath<contract> : (i32) -> i32
-  ! CHECK:         fir.result %[[V_23]] : f128
-  ! CHECK:       } else {
-  ! CHECK-DAG:     %[[V_23:[0-9]+]] = arith.subi %[[V_13]], %c1{{.*}} : i128
-  ! CHECK-DAG:     %[[V_24:[0-9]+]] = arith.addi %[[V_13]], %c1{{.*}} : i128
-  ! CHECK:         %[[V_25:[0-9]+]] = arith.select %[[V_16]], %[[V_24]], %[[V_23]] : i128
-  ! CHECK:         %[[V_26:[0-9]+]] = arith.bitcast %[[V_25]] : i128 to f128
-  ! CHECK:         %[[V_27:[0-9]+]] = "llvm.intr.is.fpclass"(%[[V_26]]) <{bit = 516 : i32}> : (f128) -> i1
-  ! CHECK:         fir.if %[[V_27]] {
-  ! CHECK:           %[[V_29:[0-9]+]] = fir.call @_FortranAMapException(%c40{{.*}}) fastmath<contract> : (i32) -> i32
-  ! CHECK:                              fir.call @feraiseexcept(%[[V_29]]) fastmath<contract> : (i32) -> i32
-  ! CHECK:         }
-  ! CHECK:         %[[V_28:[0-9]+]] = "llvm.intr.is.fpclass"(%[[V_26]]) <{bit = 144 : i32}> : (f128) -> i1
-  ! CHECK:         fir.if %[[V_28]] {
-  ! CHECK:           %[[V_29:[0-9]+]] = fir.call @_FortranAMapException(%c48{{.*}}) fastmath<contract> : (i32) -> i32
-  ! CHECK:                              fir.call @feraiseexcept(%[[V_29]]) fastmath<contract> : (i32) -> i32
-  ! CHECK:         }
-  ! CHECK:         fir.result %[[V_26]] : f128
-  ! CHECK:       }
-  ! CHECK:       fir.result %[[V_22]] : f128
-  ! CHECK:     }
-  ! CHECK:     fir.store %[[V_20]] to %[[V_2]] : !fir.ref<f128>
-  ! CHECK:     return
-  ! CHECK:   }
+! CHECK-KIND16-LABEL: c.func @_QPnearest_test7
+  ! CHECK-KIND16:     %[[V_0:[0-9]+]] = fir.dummy_scope : !fir.dscope
+  ! CHECK-KIND16:     %[[V_1:[0-9]+]] = fir.alloca f128 {bindc_name = "res", uniq_name = "_QFnearest_test7Eres"}
+  ! CHECK-KIND16:     %[[V_2:[0-9]+]] = fir.declare %[[V_1]] {uniq_name = "_QFnearest_test7Eres"} : (!fir.ref<f128>) -> !fir.ref<f128>
+  ! CHECK-KIND16:     %[[V_3:[0-9]+]] = fir.declare %arg1 dummy_scope %[[V_0]] {uniq_name = "_QFnearest_test7Es"} : (!fir.ref<f32>, !fir.dscope) -> !fir.ref<f32>
+  ! CHECK-KIND16:     %[[V_4:[0-9]+]] = fir.declare %arg0 dummy_scope %[[V_0]] {uniq_name = "_QFnearest_test7Ex"} : (!fir.ref<f128>, !fir.dscope) -> !fir.ref<f128>
+  ! CHECK-KIND16:     %[[V_5:[0-9]+]] = fir.load %[[V_4]] : !fir.ref<f128>
+  ! CHECK-KIND16:     %[[V_6:[0-9]+]] = fir.load %[[V_3]] : !fir.ref<f32>
+  ! CHECK-KIND16:     %[[V_7:[0-9]+]] = "llvm.intr.is.fpclass"(%[[V_5]]) <{bit = 3 : i32}> : (f128) -> i1
+  ! CHECK-KIND16:     %[[V_8:[0-9]+]] = "llvm.intr.is.fpclass"(%[[V_6]]) <{bit = 96 : i32}> : (f32) -> i1
+  ! CHECK-KIND16:     fir.if %[[V_8]] {
+  ! CHECK-KIND16:       fir.call @_FortranAReportFatalUserError
+  ! CHECK-KIND16:     }
+  ! CHECK-KIND16:     %[[V_9:[0-9]+]] = arith.bitcast %[[V_6]] : f32 to i32
+  ! CHECK-KIND16:     %[[V_10:[0-9]+]] = arith.shrui %[[V_9]], %c31{{.*}} : i32
+  ! CHECK-KIND16:     %[[V_11:[0-9]+]] = fir.convert %[[V_10]] : (i32) -> i128
+  ! CHECK-KIND16:     %[[V_12:[0-9]+]] = arith.cmpi ne, %[[V_11]], %c1{{.*}} : i128
+  ! CHECK-KIND16:     %[[V_13:[0-9]+]] = arith.bitcast %[[V_5]] : f128 to i128
+  ! CHECK-KIND16:     %[[V_14:[0-9]+]] = arith.shrui %[[V_13]], %c127{{.*}} : i128
+  ! CHECK-KIND16:     %[[V_15:[0-9]+]] = fir.convert %[[V_14]] : (i128) -> i1
+  ! CHECK-KIND16:     %[[V_16:[0-9]+]] = arith.cmpi ne, %[[V_12]], %[[V_15]] : i1
+  ! CHECK-KIND16:     %[[V_17:[0-9]+]] = "llvm.intr.is.fpclass"(%[[V_5]]) <{bit = 516 : i32}> : (f128) -> i1
+  ! CHECK-KIND16:     %[[V_18:[0-9]+]] = arith.andi %[[V_17]], %[[V_16]] : i1
+  ! CHECK-KIND16:     %[[V_19:[0-9]+]] = arith.ori %[[V_7]], %[[V_18]] : i1
+  ! CHECK-KIND16:     %[[V_20:[0-9]+]] = fir.if %[[V_19]] -> (f128) {
+  ! CHECK-KIND16:       fir.result %[[V_5]] : f128
+  ! CHECK-KIND16:     } else {
+  ! CHECK-KIND16:       %[[V_21:[0-9]+]] = arith.cmpf oeq, %[[V_5]], %cst{{[_0-9]*}} fastmath<contract> : f128
+  ! CHECK-KIND16:       %[[V_22:[0-9]+]] = fir.if %[[V_21]] -> (f128) {
+  ! CHECK-KIND16:         %[[V_23:[0-9]+]] = arith.select %[[V_12]], %cst{{[_0-9]*}}, %cst{{[_0-9]*}} : f128
+  ! CHECK-KIND16:         %[[V_24:[0-9]+]] = fir.call @_FortranAMapException(%c48{{.*}}) fastmath<contract> : (i32) -> i32
+  ! CHECK-KIND16:                            fir.call @feraiseexcept(%[[V_24]]) fastmath<contract> : (i32) -> i32
+  ! CHECK-KIND16:         fir.result %[[V_23]] : f128
+  ! CHECK-KIND16:       } else {
+  ! CHECK-KIND16-DAG:     %[[V_23:[0-9]+]] = arith.subi %[[V_13]], %c1{{.*}} : i128
+  ! CHECK-KIND16-DAG:     %[[V_24:[0-9]+]] = arith.addi %[[V_13]], %c1{{.*}} : i128
+  ! CHECK-KIND16:         %[[V_25:[0-9]+]] = arith.select %[[V_16]], %[[V_24]], %[[V_23]] : i128
+  ! CHECK-KIND16:         %[[V_26:[0-9]+]] = arith.bitcast %[[V_25]] : i128 to f128
+  ! CHECK-KIND16:         %[[V_27:[0-9]+]] = "llvm.intr.is.fpclass"(%[[V_26]]) <{bit = 516 : i32}> : (f128) -> i1
+  ! CHECK-KIND16:         fir.if %[[V_27]] {
+  ! CHECK-KIND16:           %[[V_29:[0-9]+]] = fir.call @_FortranAMapException(%c40{{.*}}) fastmath<contract> : (i32) -> i32
+  ! CHECK-KIND16:                              fir.call @feraiseexcept(%[[V_29]]) fastmath<contract> : (i32) -> i32
+  ! CHECK-KIND16:         }
+  ! CHECK-KIND16:         %[[V_28:[0-9]+]] = "llvm.intr.is.fpclass"(%[[V_26]]) <{bit = 144 : i32}> : (f128) -> i1
+  ! CHECK-KIND16:         fir.if %[[V_28]] {
+  ! CHECK-KIND16:           %[[V_29:[0-9]+]] = fir.call @_FortranAMapException(%c48{{.*}}) fastmath<contract> : (i32) -> i32
+  ! CHECK-KIND16:                              fir.call @feraiseexcept(%[[V_29]]) fastmath<contract> : (i32) -> i32
+  ! CHECK-KIND16:         }
+  ! CHECK-KIND16:         fir.result %[[V_26]] : f128
+  ! CHECK-KIND16:       }
+  ! CHECK-KIND16:       fir.result %[[V_22]] : f128
+  ! CHECK-KIND16:     }
+  ! CHECK-KIND16:     fir.store %[[V_20]] to %[[V_2]] : !fir.ref<f128>
+  ! CHECK-KIND16:     return
+  ! CHECK-KIND16:   }
 subroutine nearest_test7(x, s)
-  real(kind=16) :: x, res
+  integer, parameter :: kind16 = merge(16, 4, selected_real_kind(p=33).eq.16)
+  real(kind=kind16) :: x, res
   real :: s
   res = nearest(x, s)
 end
diff --git a/flang/test/Lower/Intrinsics/norm2.f90 b/flang/test/Lower/Intrinsics/norm2.f90
index ac761ae3f5381..ec89caa51d1a5 100644
--- a/flang/test/Lower/Intrinsics/norm2.f90
+++ b/flang/test/Lower/Intrinsics/norm2.f90
@@ -1,94 +1,70 @@
-! RUN: bbc -emit-fir -hlfir=false %s -o - | FileCheck %s
-! RUN: %flang_fc1 -emit-fir -flang-deprecated-no-hlfir %s -o - | FileCheck %s
+! RUN: bbc -emit-fir %s -o - | FileCheck %s --check-prefixes=CHECK%if target=x86_64{{.*}} %{,CHECK-KIND10%}%if flang-supports-f128-math %{,CHECK-KIND16%}
 
 ! CHECK-LABEL: func @_QPnorm2_test_4(
-! CHECK-SAME: %[[arg0:.*]]: !fir.box<!fir.array<?xf32>>{{.*}}) -> f32
 real(4) function norm2_test_4(a)
   real(4) :: a(:)
-  ! CHECK-DAG:  %[[c0:.*]] = arith.constant 0 : index
-  ! CHECK-DAG:  %[[arr:.*]] = fir.convert %[[arg0]] : (!fir.box<!fir.array<?xf32>>) -> !fir.box<none>
+  ! CHECK:  %[[c0:.*]] = arith.constant 0 : index
+  ! CHECK: %[[a1:.*]] = fir.declare{{.*}}a"
+  ! CHECK: %[[a:.*]] = fir.rebox %[[a1]]{{.*}}
+  ! CHECK-DAG:  %[[arr:.*]] = fir.convert %[[a]] : (!fir.box<!fir.array<?xf32>>) -> !fir.box<none>
   ! CHECK:  %[[dim:.*]] = fir.convert %[[c0]] : (index) -> i32
   norm2_test_4 = norm2(a)
   ! CHECK:  %{{.*}} = fir.call @_FortranANorm2_4(%[[arr]], %{{.*}}, %{{.*}}, %[[dim]]) {{.*}} : (!fir.box<none>, !fir.ref<i8>, i32, i32) -> f32
 end function norm2_test_4
 
 ! CHECK-LABEL: func @_QPnorm2_test_8(
-! CHECK-SAME: %[[arg0:.*]]: !fir.box<!fir.array<?x?xf64>>{{.*}}) -> f64
 real(8) function norm2_test_8(a)
   real(8) :: a(:,:)
-  ! CHECK-DAG:  %[[c0:.*]] = arith.constant 0 : index
-  ! CHECK-DAG:  %[[arr:.*]] = fir.convert %[[arg0]] : (!fir.box<!fir.array<?x?xf64>>) -> !fir.box<none>
-  ! CHECK:  %[[dim:.*]] = fir.convert %[[c0]] : (index) -> i32
   norm2_test_8 = norm2(a)
-  ! CHECK:  %{{.*}} = fir.call @_FortranANorm2_8(%[[arr]], %{{.*}}, %{{.*}}, %[[dim]]) {{.*}} : (!fir.box<none>, !fir.ref<i8>, i32, i32) -> f64
+  ! CHECK: fir.call @_FortranANorm2_8(%{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}) {{.*}} : (!fir.box<none>, !fir.ref<i8>, i32, i32) -> f64
 end function norm2_test_8
 
-! CHECK-LABEL: func @_QPnorm2_test_10(
-! CHECK-SAME: %[[arg0:.*]]: !fir.box<!fir.array<?x?x?xf80>>{{.*}}) -> f80
-real(10) function norm2_test_10(a)
-  real(10) :: a(:,:,:)
-  ! CHECK-DAG:  %[[c0:.*]] = arith.constant 0 : index
-  ! CHECK-DAG:  %[[arr:.*]] = fir.convert %[[arg0]] : (!fir.box<!fir.array<?x?x?xf80>>) -> !fir.box<none>
-  ! CHECK:  %[[dim:.*]] = fir.convert %[[c0]] : (index) -> i32
+! CHECK-KIND10-LABEL: func @_QPnorm2_test_10(
+function norm2_test_10(a)
+  integer, parameter :: kind10 = merge(10, 4, selected_real_kind(p=18).eq.10)
+  real(kind10) :: a(:,:,:), norm2_test_10
   norm2_test_10 = norm2(a)
-  ! CHECK:  %{{.*}} = fir.call @_FortranANorm2_10(%[[arr]], %{{.*}}, %{{.*}}, %[[dim]]) {{.*}} : (!fir.box<none>, !fir.ref<i8>, i32, i32) -> f80
+  ! CHECK-KIND10: fir.call @_FortranANorm2_10(%{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}) {{.*}} : (!fir.box<none>, !fir.ref<i8>, i32, i32) -> f80
 end function norm2_test_10
 
-! CHECK-LABEL: func @_QPnorm2_test_16(
-! CHECK-SAME: %[[arg0:.*]]: !fir.box<!fir.array<?x?x?xf128>>{{.*}}) -> f128
-real(16) function norm2_test_16(a)
-  real(16) :: a(:,:,:)
-  ! CHECK-DAG:  %[[c0:.*]] = arith.constant 0 : index
-  ! CHECK-DAG:  %[[arr:.*]] = fir.convert %[[arg0]] : (!fir.box<!fir.array<?x?x?xf128>>) -> !fir.box<none>
-  ! CHECK:  %[[dim:.*]] = fir.convert %[[c0]] : (index) -> i32
+! CHECK-KIND16-LABEL: func @_QPnorm2_test_16(
+function norm2_test_16(a)
+  integer, parameter :: kind16 = merge(16, 4, selected_real_kind(p=33).eq.16)
+  real(kind16) :: a(:,:,:), norm2_test_16
   norm2_test_16 = norm2(a)
-  ! CHECK:  %{{.*}} = fir.call @_FortranANorm2_16(%[[arr]], %{{.*}}, %{{.*}}, %[[dim]]) {{.*}} : (!fir.box<none>, !fir.ref<i8>, i32, i32) -> f128
+  ! CHECK-KIND16:  %{{.*}} = fir.call @_FortranANorm2_16(%{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}) {{.*}} : (!fir.box<none>, !fir.ref<i8>, i32, i32) -> f128
 end function norm2_test_16
 
 ! CHECK-LABEL: func @_QPnorm2_test_dim_2(
-! CHECK-SAME: %[[arg0:.*]]: !fir.box<!fir.array<?x?xf32>>{{.*}}, %[[arg1:.*]]: !fir.box<!fir.array<?xf32>>{{.*}})
 subroutine norm2_test_dim_2(a,r)
   real :: a(:,:)
   real :: r(:)
   ! CHECK-DAG:  %[[dim:.*]] = arith.constant 1 : i32
+  ! CHECK-DAG: %[[a1:.*]] = fir.declare{{.*}}a"
+  ! CHECK-DAG: %[[a:.*]] = fir.rebox %[[a1]]{{.*}}
   ! CHECK-DAG:  %[[r:.*]] = fir.alloca !fir.box<!fir.heap<!fir.array<?xf32>>>
   ! CHECK-DAG:  %[[res:.*]] = fir.convert %[[r]] : (!fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>) -> !fir.ref<!fir.box<none>>
-  ! CHECK:  %[[arr:.*]] = fir.convert %[[arg0]] : (!fir.box<!fir.array<?x?xf32>>) -> !fir.box<none>
+  ! CHECK:  %[[arr:.*]] = fir.convert %[[a]] : (!fir.box<!fir.array<?x?xf32>>) -> !fir.box<none>
   r = norm2(a,dim=1)
   ! CHECK:  fir.call @_FortranANorm2Dim(%[[res]], %[[arr]], %[[dim]], %{{.*}}, %{{.*}}) {{.*}} : (!fir.ref<!fir.box<none>>, !fir.box<none>, i32, !fir.ref<i8>, i32) -> ()
-  ! CHECK:  %[[box:.*]] = fir.load %[[r]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>
-  ! CHECK-DAG:  %[[addr:.*]] = fir.box_addr %[[box]] : (!fir.box<!fir.heap<!fir.array<?xf32>>>) -> !fir.heap<!fir.array<?xf32>>
-  ! CHECK-DAG:  fir.freemem %[[addr]]
+  ! CHECK-DAG:  fir.freemem
 end subroutine norm2_test_dim_2
 
 ! CHECK-LABEL: func @_QPnorm2_test_dim_3(
-! CHECK-SAME: %[[arg0:.*]]: !fir.box<!fir.array<?x?x?xf32>>{{.*}}, %[[arg1:.*]]: !fir.box<!fir.array<?x?xf32>>{{.*}})
 subroutine norm2_test_dim_3(a,r)
   real :: a(:,:,:)
   real :: r(:,:)
   ! CHECK-DAG:  %[[dim:.*]] = arith.constant 3 : i32
-  ! CHECK-DAG:  %[[r:.*]] = fir.alloca !fir.box<!fir.heap<!fir.array<?x?xf32>>>
-  ! CHECK-DAG:  %[[res:.*]] = fir.convert %[[r]] : (!fir.ref<!fir.box<!fir.heap<!fir.array<?x?xf32>>>>) -> !fir.ref<!fir.box<none>>
-  ! CHECK:  %[[arr:.*]] = fir.convert %[[arg0]] : (!fir.box<!fir.array<?x?x?xf32>>) -> !fir.box<none>
   r = norm2(a,dim=3)
-  ! CHECK:  fir.call @_FortranANorm2Dim(%[[res]], %[[arr]], %[[dim]], %{{.*}}, %{{.*}}) {{.*}} : (!fir.ref<!fir.box<none>>, !fir.box<none>, i32, !fir.ref<i8>, i32) -> ()
-  ! CHECK:  %[[box:.*]] = fir.load %[[r]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?x?xf32>>>>
-  ! CHECK-DAG:  %[[addr:.*]] = fir.box_addr %[[box]] : (!fir.box<!fir.heap<!fir.array<?x?xf32>>>) -> !fir.heap<!fir.array<?x?xf32>>
-  ! CHECK-DAG:  fir.freemem %[[addr]]
+  ! CHECK:  fir.call @_FortranANorm2Dim(%{{.*}}, %{{.*}}, %[[dim]], %{{.*}}, %{{.*}}) {{.*}} : (!fir.ref<!fir.box<none>>, !fir.box<none>, i32, !fir.ref<i8>, i32) -> ()
 end subroutine norm2_test_dim_3
 
 ! CHECK-LABEL: func @_QPnorm2_test_real16(
-! CHECK-SAME: %[[arg0:.*]]: !fir.box<!fir.array<?x?x?xf128>>{{.*}}, %[[arg1:.*]]: !fir.box<!fir.array<?x?xf128>>{{.*}})
 subroutine norm2_test_real16(a,r)
-  real(16) :: a(:,:,:)
-  real(16) :: r(:,:)
-  ! CHECK-DAG:  %[[dim:.*]] = arith.constant 3 : i32
-  ! CHECK-DAG:  %[[r:.*]] = fir.alloca !fir.box<!fir.heap<!fir.array<?x?xf128>>>
-  ! CHECK-DAG:  %[[res:.*]] = fir.convert %[[r]] : (!fir.ref<!fir.box<!fir.heap<!fir.array<?x?xf128>>>>) -> !fir.ref<!fir.box<none>>
-  ! CHECK:  %[[arr:.*]] = fir.convert %[[arg0]] : (!fir.box<!fir.array<?x?x?xf128>>) -> !fir.box<none>
+  integer, parameter :: kind16 = merge(16, 4, selected_real_kind(p=33).eq.16)
+  real(kind16) :: a(:,:,:)
+  real(kind16) :: r(:,:)
   r = norm2(a,dim=3)
-  ! CHECK:  fir.call @_FortranANorm2DimReal16(%[[res]], %[[arr]], %[[dim]], %{{.*}}, %{{.*}}) {{.*}} : (!fir.ref<!fir.box<none>>, !fir.box<none>, i32, !fir.ref<i8>, i32) -> ()
-  ! CHECK:  %[[box:.*]] = fir.load %[[r]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?x?xf128>>>>
-  ! CHECK-DAG:  %[[addr:.*]] = fir.box_addr %[[box]] : (!fir.box<!fir.heap<!fir.array<?x?xf128>>>) -> !fir.heap<!fir.array<?x?xf128>>
-  ! CHECK-DAG:  fir.freemem %[[addr]]
+  ! CHECK-KIND16:  fir.call @_FortranANorm2DimReal16(%{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}) {{.*}} : (!fir.ref<!fir.box<none>>, !fir.box<none>, i32, !fir.ref<i8>, i32) -> ()
+  ! CHECK-KIND16:  fir.freemem
 end subroutine norm2_test_real16
diff --git a/flang/test/Lower/Intrinsics/parity.f90 b/flang/test/Lower/Intrinsics/parity.f90
index 91b168ee5662d..d8ff01b7be975 100644
--- a/flang/test/Lower/Intrinsics/parity.f90
+++ b/flang/test/Lower/Intrinsics/parity.f90
@@ -28,5 +28,5 @@ subroutine parity_test2(mask, d, rslt)
   ! CHECK:  fir.call @_FortranAParityDim(%[[a6:.*]], %[[a7:.*]], %[[a1:.*]], %{{.*}}, %{{.*}}) {{.*}}: (!fir.ref<!fir.box<none>>, !fir.box<none>, i32, !fir.ref<i8>, i32) -> ()
   ! CHECK-DAG:  %[[a10:.*]] = fir.load %[[a0:.*]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?x!fir.logical<4>>>>>
   ! CHECK-DAG:  %[[a12:.*]] = fir.box_addr %[[a10:.*]] : (!fir.box<!fir.heap<!fir.array<?x!fir.logical<4>>>>) -> !fir.heap<!fir.array<?x!fir.logical<4>>>
-  ! CHECK-DAG  fir.freemem %[[a12:.*]]
+  ! CHECK-DAG:  fir.freemem %[[a12:.*]]
 end subroutine parity_test2
diff --git a/flang/test/Lower/Intrinsics/powi_real16.f90 b/flang/test/Lower/Intrinsics/powi_real16.f90
index 9e7d0f828b5cd..dc19b32742c0b 100644
--- a/flang/test/Lower/Intrinsics/powi_real16.f90
+++ b/flang/test/Lower/Intrinsics/powi_real16.f90
@@ -1,3 +1,4 @@
+! REQUIRES: flang-supports-f128-math
 ! RUN: bbc -emit-fir %s -o - | FileCheck %s --check-prefix=CHECK-FAST
 ! RUN: bbc --math-runtime=precise -emit-fir %s -o - | FileCheck %s --check-prefix=CHECK-PRECISE
 ! RUN: %flang_fc1 -emit-fir %s -o - | FileCheck %s --check-prefix=CHECK-FAST
diff --git a/flang/test/Lower/Intrinsics/random_number_real16.f90 b/flang/test/Lower/Intrinsics/random_number_real16.f90
index 060574d5b3b3f..1597d9bd5397f 100644
--- a/flang/test/Lower/Intrinsics/random_number_real16.f90
+++ b/flang/test/Lower/Intrinsics/random_number_real16.f90
@@ -1,3 +1,4 @@
+! REQUIRES: flang-supports-f128-math
 ! RUN: bbc -emit-fir %s -o - | FileCheck %s
 ! RUN: %flang_fc1 -emit-fir %s -o - | FileCheck %s
 
diff --git a/flang/test/Lower/Intrinsics/reduce.f90 b/flang/test/Lower/Intrinsics/reduce.f90
index 8d7ec89d27474..4851fa20906d4 100644
--- a/flang/test/Lower/Intrinsics/reduce.f90
+++ b/flang/test/Lower/Intrinsics/reduce.f90
@@ -1,4 +1,4 @@
-! RUN: bbc -emit-hlfir %s -o - | FileCheck %s
+! RUN: bbc -emit-hlfir %s -o - | FileCheck %s --check-prefixes=CHECK%if target=x86_64{{.*}} %{,CHECK-KIND10%}%if flang-supports-f128-math %{,CHECK-KIND16%}
 
 module reduce_mod
 
@@ -17,6 +17,10 @@ pure function red_int1_interface_value(a, b)
     end function
   end interface
 
+  integer, parameter :: kind10 = merge(10, 4, selected_real_kind(p=18).eq.10)
+  integer, parameter :: kind16 = merge(16, 4, selected_real_kind(p=33).eq.16)
+  
+
 contains
 
 pure function red_int1(a,b)
@@ -270,48 +274,48 @@ subroutine real8(a)
 ! CHECK: fir.call @_FortranAReduceReal8Value
 
 pure function red_real10(a,b)
-  real(10), intent(in) :: a, b
-  real(10) :: red_real10
+  real(kind10), intent(in) :: a, b
+  real(kind10) :: red_real10
   red_real10 = a + b
 end function
 
 pure function red_real10_value(a,b)
-  real(10), value, intent(in) :: a, b
-  real(10) :: red_real10_value
+  real(kind10), value, intent(in) :: a, b
+  real(kind10) :: red_real10_value
   red_real10_value = a + b
 end function
 
 subroutine real10(a)
-  real(10), intent(in) :: a(:)
-  real(10) :: res
+  real(kind10), intent(in) :: a(:)
+  real(kind10) :: res
   res = reduce(a, red_real10)
   res = reduce(a, red_real10_value)
 end subroutine
 
-! CHECK: fir.call @_FortranAReduceReal10Ref
-! CHECK: fir.call @_FortranAReduceReal10Value
+! CHECK-KIND10: fir.call @_FortranAReduceReal10Ref
+! CHECK-KIND10: fir.call @_FortranAReduceReal10Value
 
 pure function red_real16(a,b)
-  real(16), intent(in) :: a, b
-  real(16) :: red_real16
+  real(kind16), intent(in) :: a, b
+  real(kind16) :: red_real16
   red_real16 = a + b
 end function
 
 pure function red_real16_value(a,b)
-  real(16), value, intent(in) :: a, b
-  real(16) :: red_real16_value
+  real(kind16), value, intent(in) :: a, b
+  real(kind16) :: red_real16_value
   red_real16_value = a + b
 end function
 
 subroutine real16(a)
-  real(16), intent(in) :: a(:)
-  real(16) :: res
+  real(kind16), intent(in) :: a(:)
+  real(kind16) :: res
   res = reduce(a, red_real16)
   res = reduce(a, red_real16_value)
 end subroutine
 
-! CHECK: fir.call @_FortranAReduceReal16Ref
-! CHECK: fir.call @_FortranAReduceReal16Value
+! CHECK-KIND16: fir.call @_FortranAReduceReal16Ref
+! CHECK-KIND16: fir.call @_FortranAReduceReal16Value
 
 pure function red_complex2(a,b)
   complex(2), intent(in) :: a, b
@@ -402,48 +406,48 @@ subroutine complex8(a)
 ! CHECK: fir.call @_FortranACppReduceComplex8Value
 
 pure function red_complex10(a,b)
-  complex(10), intent(in) :: a, b
-  complex(10) :: red_complex10
+  complex(kind10), intent(in) :: a, b
+  complex(kind10) :: red_complex10
   red_complex10 = a + b
 end function
 
 pure function red_complex10_value(a,b)
-  complex(10), value, intent(in) :: a, b
-  complex(10) :: red_complex10_value
+  complex(kind10), value, intent(in) :: a, b
+  complex(kind10) :: red_complex10_value
   red_complex10_value = a + b
 end function
 
 subroutine complex10(a)
-  complex(10), intent(in) :: a(:)
-  complex(10) :: res
+  complex(kind10), intent(in) :: a(:)
+  complex(kind10) :: res
   res = reduce(a, red_complex10)
   res = reduce(a, red_complex10_value)
 end subroutine
 
-! CHECK: fir.call @_FortranACppReduceComplex10Ref
-! CHECK: fir.call @_FortranACppReduceComplex10Value
+! CHECK-KIND10: fir.call @_FortranACppReduceComplex10Ref
+! CHECK-KIND10: fir.call @_FortranACppReduceComplex10Value
 
 pure function red_complex16(a,b)
-  complex(16), intent(in) :: a, b
-  complex(16) :: red_complex16
+  complex(kind16), intent(in) :: a, b
+  complex(kind16) :: red_complex16
   red_complex16 = a + b
 end function
 
 pure function red_complex16_value(a,b)
-  complex(16), value, intent(in) :: a, b
-  complex(16) :: red_complex16_value
+  complex(kind16), value, intent(in) :: a, b
+  complex(kind16) :: red_complex16_value
   red_complex16_value = a + b
 end function
 
 subroutine complex16(a)
-  complex(16), intent(in) :: a(:)
-  complex(16) :: res
+  complex(kind16), intent(in) :: a(:)
+  complex(kind16) :: res
   res = reduce(a, red_complex16)
   res = reduce(a, red_complex16_value)
 end subroutine
 
-! CHECK: fir.call @_FortranACppReduceComplex16Ref
-! CHECK: fir.call @_FortranACppReduceComplex16Value
+! CHECK-KIND16: fir.call @_FortranACppReduceComplex16Ref
+! CHECK-KIND16: fir.call @_FortranACppReduceComplex16Value
 
 pure function red_log1(a,b)
   logical(1), intent(in) :: a, b
@@ -693,26 +697,26 @@ subroutine real8dim(a, id)
 ! CHECK: fir.call @_FortranAReduceReal8DimValue
 
 subroutine real10dim(a, id)
-  real(10), intent(in) :: a(:,:)
-  real(10), allocatable :: res(:)
+  real(kind10), intent(in) :: a(:,:)
+  real(kind10), allocatable :: res(:)
 
   res = reduce(a, red_real10, 2)
   res = reduce(a, red_real10_value, 2)
 end subroutine
 
-! CHECK: fir.call @_FortranAReduceReal10DimRef
-! CHECK: fir.call @_FortranAReduceReal10DimValue
+! CHECK-KIND10: fir.call @_FortranAReduceReal10DimRef
+! CHECK-KIND10: fir.call @_FortranAReduceReal10DimValue
 
 subroutine real16dim(a, id)
-  real(16), intent(in) :: a(:,:)
-  real(16), allocatable :: res(:)
+  real(kind16), intent(in) :: a(:,:)
+  real(kind16), allocatable :: res(:)
 
   res = reduce(a, red_real16, 2)
   res = reduce(a, red_real16_value, 2)
 end subroutine
 
-! CHECK: fir.call @_FortranAReduceReal16DimRef
-! CHECK: fir.call @_FortranAReduceReal16DimValue
+! CHECK-KIND16: fir.call @_FortranAReduceReal16DimRef
+! CHECK-KIND16: fir.call @_FortranAReduceReal16DimValue
 
 subroutine complex2dim(a, id)
   complex(2), intent(in) :: a(:,:)
@@ -759,26 +763,26 @@ subroutine complex8dim(a, id)
 ! CHECK: fir.call @_FortranACppReduceComplex8DimValue
 
 subroutine complex10dim(a, id)
-  complex(10), intent(in) :: a(:,:)
-  complex(10), allocatable :: res(:)
+  complex(kind10), intent(in) :: a(:,:)
+  complex(kind10), allocatable :: res(:)
 
   res = reduce(a, red_complex10, 2)
   res = reduce(a, red_complex10_value, 2)
 end subroutine
 
-! CHECK: fir.call @_FortranACppReduceComplex10DimRef
-! CHECK: fir.call @_FortranACppReduceComplex10DimValue
+! CHECK-KIND10: fir.call @_FortranACppReduceComplex10DimRef
+! CHECK-KIND10: fir.call @_FortranACppReduceComplex10DimValue
 
 subroutine complex16dim(a, id)
-  complex(16), intent(in) :: a(:,:)
-  complex(16), allocatable :: res(:)
+  complex(kind16), intent(in) :: a(:,:)
+  complex(kind16), allocatable :: res(:)
 
   res = reduce(a, red_complex16, 2)
   res = reduce(a, red_complex16_value, 2)
 end subroutine
 
-! CHECK: fir.call @_FortranACppReduceComplex16DimRef
-! CHECK: fir.call @_FortranACppReduceComplex16DimValue
+! CHECK-KIND16: fir.call @_FortranACppReduceComplex16DimRef
+! CHECK-KIND16: fir.call @_FortranACppReduceComplex16DimValue
 
 subroutine logical1dim(a, id)
   logical(1), intent(in) :: a(:,:)
diff --git a/flang/test/Lower/Intrinsics/rrspacing.f90 b/flang/test/Lower/Intrinsics/rrspacing.f90
index 7125f6bf319b8..5cfd7c7befdc4 100644
--- a/flang/test/Lower/Intrinsics/rrspacing.f90
+++ b/flang/test/Lower/Intrinsics/rrspacing.f90
@@ -1,3 +1,4 @@
+! REQUIRES: flang-supports-f128-math
 ! RUN: bbc -emit-fir -hlfir=false %s -o - | FileCheck %s
 ! RUN: %flang_fc1 -emit-fir -flang-deprecated-no-hlfir %s -o - | FileCheck %s
 
diff --git a/flang/test/Lower/Intrinsics/scale.f90 b/flang/test/Lower/Intrinsics/scale.f90
index 91892838ea513..9c97349d1dd57 100644
--- a/flang/test/Lower/Intrinsics/scale.f90
+++ b/flang/test/Lower/Intrinsics/scale.f90
@@ -1,53 +1,42 @@
-! RUN: bbc -emit-fir -hlfir=false %s -o - | FileCheck %s
+! RUN: bbc -emit-hlfir %s -o - | FileCheck %s --check-prefixes=CHECK%if target=x86_64{{.*}} %{,CHECK-KIND10%}%if flang-supports-f128-math %{,CHECK-KIND16%}
 
 ! CHECK-LABEL: scale_test1
 subroutine scale_test1(x, i)
     real :: x, res
-  ! CHECK: %[[res:.*]] = fir.alloca f32 {bindc_name = "res", uniq_name = "_QFscale_test1Eres"}
-  ! CHECK: %[[x:.*]] = fir.load %arg0 : !fir.ref<f32>
+  ! CHECK: %[[i:.*]]:2 = hlfir.declare{{.*}}i"
+  ! CHECK: %[[res:.*]]:2 = hlfir.declare{{.*}}res"
+  ! CHECK: %[[x:.*]]:2 = hlfir.declare{{.*}}x"
+  ! CHECK: %[[x_val:.*]] = fir.load %[[x]]#0 : !fir.ref<f32>
     integer :: i
-  ! CHECK: %[[i0:.*]] = fir.load %arg1 : !fir.ref<i32>
+  ! CHECK: %[[i_val:.*]] = fir.load %[[i]]#0 : !fir.ref<i32>
     res = scale(x, i)
-  ! CHECK: %[[i1:.*]] = fir.convert %[[i0]] : (i32) -> i64
-  ! CHECK: %[[tmp:.*]] = fir.call @_FortranAScale4(%[[x]], %[[i1]]) {{.*}}: (f32, i64) -> f32
-  ! CHECK: fir.store %[[tmp]] to %[[res]] : !fir.ref<f32>
-  end subroutine scale_test1
+  ! CHECK: %[[i_cast:.*]] = fir.convert %[[i_val]] : (i32) -> i64
+  ! CHECK: %[[tmp:.*]] = fir.call @_FortranAScale4(%[[x_val]], %[[i_cast]]) {{.*}}: (f32, i64) -> f32
+  ! CHECK: hlfir.assign %[[tmp]] to %[[res]]#0 : f32, !fir.ref<f32>
+end subroutine scale_test1
   
-  ! CHECK-LABEL: scale_test2
-  subroutine scale_test2(x, i)
-    real(kind=8) :: x, res
-  ! CHECK: %[[res:.*]] = fir.alloca f64 {bindc_name = "res", uniq_name = "_QFscale_test2Eres"}
-  ! CHECK: %[[x:.*]] = fir.load %arg0 : !fir.ref<f64>
-    integer :: i
-  ! CHECK: %[[i0:.*]] = fir.load %arg1 : !fir.ref<i32>
-    res = scale(x, i)
-  ! CHECK: %[[i1:.*]] = fir.convert %[[i0]] : (i32) -> i64
-  ! CHECK: %[[tmp:.*]] = fir.call @_FortranAScale8(%[[x]], %[[i1]]) {{.*}}: (f64, i64) -> f64
-  ! CHECK: fir.store %[[tmp]] to %[[res]] : !fir.ref<f64>
-  end subroutine scale_test2
-  
-  ! CHECK-LABEL: scale_test3
-  subroutine scale_test3(x, i)
-    real(kind=10) :: x, res
-  ! CHECK: %[[res:.*]] = fir.alloca f80 {bindc_name = "res", uniq_name = "_QFscale_test3Eres"}
-  ! CHECK: %[[x:.*]] = fir.load %arg0 : !fir.ref<f80>
-    integer :: i
-  ! CHECK: %[[i0:.*]] = fir.load %arg1 : !fir.ref<i32>
-    res = scale(x, i)
-  ! CHECK: %[[i1:.*]] = fir.convert %[[i0]] : (i32) -> i64
-  ! CHECK: %[[tmp:.*]] = fir.call @_FortranAScale10(%[[x]], %[[i1]]) {{.*}}: (f80, i64) -> f80
-  ! CHECK: fir.store %[[tmp]] to %[[res]] : !fir.ref<f80>
-  end subroutine scale_test3
-  
-  ! CHECK-LABEL: scale_test4
-  subroutine scale_test4(x, i)
-    real(kind=16) :: x, res
-  ! CHECK: %[[res:.*]] = fir.alloca f128 {bindc_name = "res", uniq_name = "_QFscale_test4Eres"}
-  ! CHECK: %[[x:.*]] = fir.load %arg0 : !fir.ref<f128>
-    integer :: i
-  ! CHECK: %[[i0:.*]] = fir.load %arg1 : !fir.ref<i32>
-    res = scale(x, i)
-  ! CHECK: %[[i1:.*]] = fir.convert %[[i0]] : (i32) -> i64
-  ! CHECK: %[[tmp:.*]] = fir.call @_FortranAScale16(%[[x]], %[[i1]]) {{.*}}: (f128, i64) -> f128
-  ! CHECK: fir.store %[[tmp]] to %[[res]] : !fir.ref<f128>
-  end subroutine scale_test4
+! CHECK-LABEL: scale_test2
+subroutine scale_test2(x, i)
+  real(kind=8) :: x, res
+  integer :: i
+  res = scale(x, i)
+! CHECK: fir.call @_FortranAScale8(%{{.*}}, %{{.*}}) {{.*}}: (f64, i64) -> f64
+end subroutine scale_test2
+
+! CHECK-KIND10-LABEL: scale_test3
+subroutine scale_test3(x, i)
+  integer, parameter :: kind10 = merge(10, 4, selected_real_kind(p=18).eq.10)
+  real(kind=kind10) :: x, res
+  integer :: i
+  res = scale(x, i)
+! CHECK-KIND10: fir.call @_FortranAScale10(%{{.*}}, %{{.*}}) {{.*}}: (f80, i64) -> f80
+end subroutine scale_test3
+
+! CHECK-KIND16-LABEL: scale_test4
+subroutine scale_test4(x, i)
+  integer, parameter :: kind16 = merge(16, 4, selected_real_kind(p=33).eq.16)
+  real(kind=kind16) :: x, res
+  integer :: i
+  res = scale(x, i)
+! CHECK-KIND16: fir.call @_FortranAScale16(%{{.*}}, %{{.*}}) {{.*}}: (f128, i64) -> f128
+end subroutine scale_test4
diff --git a/flang/test/Lower/Intrinsics/set_exponent.f90 b/flang/test/Lower/Intrinsics/set_exponent.f90
index fedbad78747a9..a06be6aac4419 100644
--- a/flang/test/Lower/Intrinsics/set_exponent.f90
+++ b/flang/test/Lower/Intrinsics/set_exponent.f90
@@ -1,47 +1,43 @@
-! RUN: bbc -emit-fir -hlfir=false %s -o - | FileCheck %s
-! RUN: %flang_fc1 -emit-fir -flang-deprecated-no-hlfir %s -o - | FileCheck %s
+! RUN: bbc -emit-hlfir %s -o - | FileCheck %s --check-prefixes=CHECK%if target=x86_64{{.*}} %{,CHECK-KIND10%}%if flang-supports-f128-math %{,CHECK-KIND16%}
 
 ! SET_EXPONENT
-! CHECK-LABEL: set_exponent_test
-subroutine set_exponent_test
+! CHECK-LABEL: set_exponent_test_4
+subroutine set_exponent_test_4(x, i)
+  real(kind = 4) :: x
+  integer :: i
+  x = set_exponent(x, i)
+! CHECK:  %[[VAL_3:.*]]:2 = hlfir.declare {{.*}}"_QFset_exponent_test_4Ei"
+! CHECK:  %[[VAL_4:.*]]:2 = hlfir.declare {{.*}}"_QFset_exponent_test_4Ex"
+! CHECK:  %[[VAL_5:.*]] = fir.load %[[VAL_4]]#0 : !fir.ref<f32>
+! CHECK:  %[[VAL_6:.*]] = fir.load %[[VAL_3]]#0 : !fir.ref<i32>
+! CHECK:  %[[VAL_7:.*]] = fir.convert %[[VAL_6]] : (i32) -> i64
+! CHECK:  %[[VAL_8:.*]] = fir.call @_FortranASetExponent4(%[[VAL_5]], %[[VAL_7]]) fastmath<contract> : (f32, i64) -> f32
+! CHECK:  hlfir.assign %[[VAL_8]] to %[[VAL_4]]#0 : f32, !fir.ref<f32>
+end subroutine
 
-  real(kind = 4) :: x1 = 178.1378e-4
-  real(kind = 8) :: x2 = 178.1378e-4
-  real(kind = 10) :: x3 = 178.1378e-4
-  real(kind = 16) :: x4 = 178.1378e-4
-  integer :: i = 17
-! CHECK: %[[addri:.*]] = fir.address_of(@_QFset_exponent_testEi) : !fir.ref<i32>
-! CHECK: %[[addrx1:.*]] = fir.address_of(@_QFset_exponent_testEx1) : !fir.ref<f32>
-! CHECK: %[[addrx2:.*]] = fir.address_of(@_QFset_exponent_testEx2) : !fir.ref<f64>
-! CHECK: %[[addrx3:.*]] = fir.address_of(@_QFset_exponent_testEx3) : !fir.ref<f80>
-! CHECK: %[[addrx4:.*]] = fir.address_of(@_QFset_exponent_testEx4) : !fir.ref<f128>
 
-  x1 = set_exponent(x1, i)
-! CHECK: %[[x1:.*]] = fir.load %[[addrx1:.*]] : !fir.ref<f32>
-! CHECK: %[[i1:.*]] = fir.load %[[addri:.*]] : !fir.ref<i32>
-! CHECK: %[[i64v1:.*]] = fir.convert %[[i1:.*]] : (i32) -> i64
-! CHECK: %[[result1:.*]] = fir.call @_FortranASetExponent4(%[[x1:.*]], %[[i64v1:.*]]) {{.*}}: (f32, i64) -> f32
-! CHECK: fir.store %[[result1:.*]] to %[[addrx1:.*]] : !fir.ref<f32>
+! CHECK-LABEL: set_exponent_test_8
+subroutine set_exponent_test_8(x, i)
+  real(kind = 8) :: x
+  integer :: i
+  x = set_exponent(x, i)
+! CHECK: fir.call @_FortranASetExponent8(%{{.*}}, %{{.*}}) {{.*}}: (f64, i64) -> f64
+end subroutine
 
-  x2 = set_exponent(x2, i)
-! CHECK: %[[x2:.*]] = fir.load %[[addrx2:.*]] : !fir.ref<f64>
-! CHECK: %[[i2:.*]] = fir.load %[[addri:.*]] : !fir.ref<i32>
-! CHECK: %[[i64v2:.*]] = fir.convert %[[i2:.*]] : (i32) -> i64
-! CHECK: %[[result2:.*]] = fir.call @_FortranASetExponent8(%[[x2:.*]], %[[i64v2:.*]]) {{.*}}: (f64, i64) -> f64
-! CHECK: fir.store %[[result2:.*]] to %[[addrx2:.*]] : !fir.ref<f64>
-
-  x3 = set_exponent(x3, i)
-! CHECK: %[[x3:.*]] = fir.load %[[addrx3:.*]] : !fir.ref<f80>
-! CHECK: %[[i3:.*]] = fir.load %[[addri:.*]] : !fir.ref<i32>
-! CHECK: %[[i64v3:.*]] = fir.convert %[[i3:.*]] : (i32) -> i64
-! CHECK: %[[result3:.*]] = fir.call @_FortranASetExponent10(%[[x3:.*]], %[[i64v3:.*]]) {{.*}}: (f80, i64) -> f80
-! CHECK: fir.store %[[result3:.*]] to %[[addrx3:.*]] : !fir.ref<f80>
-
-  x4 = set_exponent(x4, i)
-! CHECK: %[[x4:.*]] = fir.load %[[addrx4:.*]] : !fir.ref<f128>
-! CHECK: %[[i4:.*]] = fir.load %[[addri:.*]] : !fir.ref<i32>
-! CHECK: %[[i64v4:.*]] = fir.convert %18 : (i32) -> i64
-! CHECK: %[[result4:.*]] = fir.call @_FortranASetExponent16(%[[x4:.*]], %[[i64v4:.*]]) {{.*}}: (f128, i64) -> f128
-! CHECK: fir.store %[[result4:.*]] to %[[addrx4:.*]] : !fir.ref<f128>
-end subroutine set_exponent_test
+! CHECK-KIND10-LABEL: set_exponent_test_10
+subroutine set_exponent_test_10(x, i)
+  integer, parameter :: kind10 = merge(10, 4, selected_real_kind(p=18).eq.10)
+  real(kind = kind10) :: x
+  integer :: i
+  x = set_exponent(x, i)
+! CHECK-KIND10: fir.call @_FortranASetExponent10(%{{.*}}, %{{.*}}) {{.*}}: (f80, i64) -> f80
+end subroutine
 
+! CHECK-KIND16-LABEL: set_exponent_test_16
+subroutine set_exponent_test_16(x, i)
+  integer, parameter :: kind16 = merge(16, 4, selected_real_kind(p=33).eq.16)
+  real(kind = kind16) :: x
+  integer :: i
+  x = set_exponent(x, i)
+! CHECK-KIND16: fir.call @_FortranASetExponent16(%{{.*}}, %{{.*}}) {{.*}}: (f128, i64) -> f128
+end subroutine
diff --git a/flang/test/Lower/Intrinsics/sign.f90 b/flang/test/Lower/Intrinsics/sign.f90
index 218080f0d49b6..965bda3d5b834 100644
--- a/flang/test/Lower/Intrinsics/sign.f90
+++ b/flang/test/Lower/Intrinsics/sign.f90
@@ -1,4 +1,4 @@
-! RUN: bbc %s -o - | FileCheck %s
+! RUN: bbc %s -o - | FileCheck %s --check-prefixes=CHECK,%if flang-supports-f128-math %{F128%} %else %{F64%}
 
 ! CHECK-LABEL: sign_testi
 subroutine sign_testi(a, b, c)
@@ -22,8 +22,10 @@ subroutine sign_testr(a, b, c)
 
 ! CHECK-LABEL: sign_testr2
 subroutine sign_testr2(a, b, c)
-  real(KIND=16) a, b, c
+  integer, parameter :: rk = merge(16, 8, selected_real_kind(33, 4931)==16)
+  real(KIND=rk) a, b, c
   ! CHECK-NOT: fir.call @{{.*}}fabs
-  ! CHECK: math.copysign{{.*}} : f128
+  ! F128: math.copysign{{.*}} : f128
+  ! F64: math.copysign{{.*}} : f64
   c = sign(a, b)
 end subroutine
diff --git a/flang/test/Lower/Intrinsics/spacing.f90 b/flang/test/Lower/Intrinsics/spacing.f90
index 151f4e2a6d236..9f597a2149792 100644
--- a/flang/test/Lower/Intrinsics/spacing.f90
+++ b/flang/test/Lower/Intrinsics/spacing.f90
@@ -1,5 +1,4 @@
-! RUN: bbc -emit-fir %s -o - | FileCheck %s
-! RUN: %flang_fc1 -emit-fir %s -o - | FileCheck %s
+! RUN: %flang_fc1 -emit-hlfir %s -o - | FileCheck %s --check-prefixes=CHECK%if target=x86_64{{.*}} %{,CHECK-KIND10%}
 
 ! CHECK-LABEL: func @_QPspacing_test(
 real*4 function spacing_test(x)
@@ -9,12 +8,13 @@ real*4 function spacing_test(x)
 ! CHECK: %{{.*}} = fir.call @_FortranASpacing4(%[[a1]]) {{.*}}: (f32) -> f32
 end function
 
-! CHECK-LABEL: func @_QPspacing_test2(
-real*10 function spacing_test2(x)
-  real*10 :: x
+! CHECK-KIND10-LABEL: func @_QPspacing_test2(
+function spacing_test2(x)
+  integer, parameter :: kind10 = merge(10, 4, selected_real_kind(p=18).eq.10)
+  real(kind10) :: x, spacing_test2
   spacing_test2 = spacing(x)
-! CHECK: %[[a1:.*]] = fir.load %{{.*}} : !fir.ref<f80>
-! CHECK: %{{.*}} = fir.call @_FortranASpacing10(%[[a1]]) {{.*}}: (f80) -> f80
+! CHECK-KIND10: %[[a1:.*]] = fir.load %{{.*}} : !fir.ref<f80>
+! CHECK-KIND10: %{{.*}} = fir.call @_FortranASpacing10(%[[a1]]) {{.*}}: (f80) -> f80
 end function
 
 ! CHECK-LABEL: test_real2
diff --git a/flang/test/Lower/Intrinsics/storage_size-2.f90 b/flang/test/Lower/Intrinsics/storage_size-2.f90
index d6fb68df70ea9..208070e04e6e6 100644
--- a/flang/test/Lower/Intrinsics/storage_size-2.f90
+++ b/flang/test/Lower/Intrinsics/storage_size-2.f90
@@ -1,7 +1,7 @@
 ! Test storage_size with characters
 ! RUN: bbc -emit-hlfir %s -o - | FileCheck %s
 
-! check-label: func.func @_QPtest_storage_size
+! CHECK-LABEL: func.func @_QPtest_storage_size
 subroutine test_storage_size(n)
   interface
     function return_char(l)
diff --git a/flang/test/Lower/OpenACC/acc-bounds.f90 b/flang/test/Lower/OpenACC/acc-bounds.f90
index e44c786e62964..8fea357f116a2 100644
--- a/flang/test/Lower/OpenACC/acc-bounds.f90
+++ b/flang/test/Lower/OpenACC/acc-bounds.f90
@@ -1,6 +1,6 @@
 ! This test checks lowering of OpenACC data bounds operation.
 
-! RUN: bbc -fopenacc -emit-hlfir %s -o - | FileCheck %s
+! RUN: bbc -fopenacc -emit-hlfir --openacc-unwrap-fir-box=true --openacc-generate-default-bounds=true %s -o - | FileCheck %s
 
 module openacc_bounds
 
diff --git a/flang/test/Lower/OpenACC/acc-data-operands-unwrap-defaultbounds.f90 b/flang/test/Lower/OpenACC/acc-data-operands-unwrap-defaultbounds.f90
new file mode 100644
index 0000000000000..ccd32f16b54e3
--- /dev/null
+++ b/flang/test/Lower/OpenACC/acc-data-operands-unwrap-defaultbounds.f90
@@ -0,0 +1,152 @@
+! This test checks lowering of complex OpenACC data operands and checks
+! that default bounds are generated.
+
+! RUN: bbc -fopenacc -emit-hlfir --openacc-unwrap-fir-box=true --openacc-generate-default-bounds=true %s -o - | FileCheck %s
+
+module acc_data_operand
+
+  type wrapper
+    real :: data(100)
+  end type
+
+contains
+
+! Testing derived-type component without section
+subroutine acc_operand_derived_type_component()
+  type(wrapper) :: w
+
+  !$acc data copy(w%data)
+  !$acc end data
+end subroutine
+
+! CHECK-LABEL: func.func @_QMacc_data_operandPacc_operand_derived_type_component() {
+! CHECK: %[[W:.*]] = fir.alloca !fir.type<_QMacc_data_operandTwrapper{data:!fir.array<100xf32>}> {bindc_name = "w", uniq_name = "_QMacc_data_operandFacc_operand_derived_type_componentEw"}
+! CHECK: %[[DECLW:.*]]:2 = hlfir.declare %[[W]]
+! CHECK: %[[EXT:.*]] = arith.constant 100 : index
+! CHECK: %[[COORD_DATA:.*]] = hlfir.designate %[[DECLW]]#0{"data"}   shape %{{.*}} : (!fir.ref<!fir.type<_QMacc_data_operandTwrapper{data:!fir.array<100xf32>}>>, !fir.shape<1>) -> !fir.ref<!fir.array<100xf32>>
+! CHECK: %[[ONE:.*]] = arith.constant 1 : index
+! CHECK: %[[LB:.*]] = arith.constant 0 : index
+! CHECK: %[[UB:.*]] = arith.subi %[[EXT]], %[[ONE]] : index
+! CHECK: %[[BOUND:.*]] = acc.bounds lowerbound(%[[LB]] : index) upperbound(%[[UB]] : index) extent(%[[EXT]] : index) stride(%[[ONE]] : index) startIdx(%[[ONE]] : index)
+! CHECK: %[[COPY_COPYIN:.*]] = acc.copyin varPtr(%[[COORD_DATA]] : !fir.ref<!fir.array<100xf32>>) bounds(%[[BOUND]]) -> !fir.ref<!fir.array<100xf32>> {dataClause = #acc<data_clause acc_copy>, name = "w%data"}
+! CHECK: acc.data dataOperands(%[[COPY_COPYIN]] : !fir.ref<!fir.array<100xf32>>) {
+! CHECK:   acc.terminator
+! CHECK: }
+! CHECK: acc.copyout accPtr(%[[COPY_COPYIN]] : !fir.ref<!fir.array<100xf32>>) bounds(%[[BOUND]]) to varPtr(%[[COORD_DATA]] : !fir.ref<!fir.array<100xf32>>) {dataClause = #acc<data_clause acc_copy>, name = "w%data"}
+
+
+! Testing array of derived-type component without section
+subroutine acc_operand_array_derived_type_component()
+  type(wrapper) :: w(10)
+
+  !$acc data copy(w(1)%data)
+  !$acc end data
+end subroutine
+
+! CHECK-LABEL: func.func @_QMacc_data_operandPacc_operand_array_derived_type_component() {
+! CHECK: %[[W:.*]] = fir.alloca !fir.array<10x!fir.type<_QMacc_data_operandTwrapper{data:!fir.array<100xf32>}>> {bindc_name = "w", uniq_name = "_QMacc_data_operandFacc_operand_array_derived_type_componentEw"}
+! CHECK: %[[DECLW:.*]]:2 = hlfir.declare %[[W]]
+! CHECK: %[[C1:.*]] = arith.constant 1 : index
+! CHECK: %[[W_1:.*]] = hlfir.designate %[[DECLW]]#0 (%[[C1]])  : (!fir.ref<!fir.array<10x!fir.type<_QMacc_data_operandTwrapper{data:!fir.array<100xf32>}>>>, index) -> !fir.ref<!fir.type<_QMacc_data_operandTwrapper{data:!fir.array<100xf32>}>>
+! CHECK: %[[EXT:.*]] = arith.constant 100 : index
+! CHECK: %[[COORD_W1_DATA:.*]] = hlfir.designate %[[W_1]]{"data"}   shape %{{.*}} : (!fir.ref<!fir.type<_QMacc_data_operandTwrapper{data:!fir.array<100xf32>}>>, !fir.shape<1>) -> !fir.ref<!fir.array<100xf32>>
+! CHECK: %[[ONE:.*]] = arith.constant 1 : index
+! CHECK: %[[LB:.*]] = arith.constant 0 : index
+! CHECK: %[[UB:.*]] = arith.subi %[[EXT]], %[[ONE]] : index
+! CHECK: %[[BOUND:.*]] = acc.bounds lowerbound(%[[LB]] : index) upperbound(%[[UB]] : index) extent(%[[EXT]] : index) stride(%[[ONE]] : index) startIdx(%[[ONE]] : index)
+! CHECK: %[[COPY_COPYIN:.*]] = acc.copyin varPtr(%[[COORD_W1_DATA]] : !fir.ref<!fir.array<100xf32>>) bounds(%[[BOUND]]) -> !fir.ref<!fir.array<100xf32>> {dataClause = #acc<data_clause acc_copy>, name = "w(1_8)%data"}
+! CHECK: acc.data dataOperands(%[[COPY_COPYIN]] : !fir.ref<!fir.array<100xf32>>) {
+! CHECK:   acc.terminator
+! CHECK: }
+! CHECK: acc.copyout accPtr(%[[COPY_COPYIN]] : !fir.ref<!fir.array<100xf32>>) bounds(%[[BOUND]]) to varPtr(%[[COORD_W1_DATA]] : !fir.ref<!fir.array<100xf32>>) {dataClause = #acc<data_clause acc_copy>, name = "w(1_8)%data"}
+
+! Testing array sections on allocatable array
+subroutine acc_operand_array_section_allocatable()
+  real, allocatable :: a(:)
+
+  allocate(a(100))
+
+  !$acc data copyin(a(1:50)) copyout(a(51:100))
+  !$acc end data
+
+  deallocate(a)
+end subroutine
+
+! CHECK-LABEL: func.func @_QMacc_data_operandPacc_operand_array_section_allocatable() {
+! CHECK: %[[A:.*]] = fir.alloca !fir.box<!fir.heap<!fir.array<?xf32>>> {bindc_name = "a", uniq_name = "_QMacc_data_operandFacc_operand_array_section_allocatableEa"}
+! CHECK: %[[DECLA:.*]]:2 = hlfir.declare %[[A]] {fortran_attrs = #fir.var_attrs<allocatable>
+! CHECK: %[[LOAD_BOX_A_0:.*]] = fir.load %[[DECLA]]#0 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>
+! CHECK: %[[LOAD_BOX_A_1:.*]] = fir.load %[[DECLA]]#1 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>
+! CHECK: %[[C0:.*]] = arith.constant 0 : index
+! CHECK: %[[DIMS0_0:.*]]:3 = fir.box_dims %[[LOAD_BOX_A_1]], %[[C0]] : (!fir.box<!fir.heap<!fir.array<?xf32>>>, index) -> (index, index, index)
+! CHECK: %[[C0:.*]] = arith.constant 0 : index
+! CHECK: %[[DIMS0_1:.*]]:3 = fir.box_dims %[[LOAD_BOX_A_0]], %[[C0]] : (!fir.box<!fir.heap<!fir.array<?xf32>>>, index) -> (index, index, index)
+! CHECK: %[[C1:.*]] = arith.constant 1 : index
+! CHECK: %[[LB:.*]] = arith.subi %[[C1]], %[[DIMS0_0]]#0 : index
+! CHECK: %[[C50:.*]] = arith.constant 50 : index
+! CHECK: %[[UB:.*]] = arith.subi %[[C50]], %[[DIMS0_0]]#0 : index
+! CHECK: %[[LOAD_BOX_A_2:.*]] = fir.load %[[DECLA]]#1 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>
+! CHECK: %[[C0:.*]] = arith.constant 0 : index
+! CHECK: %[[DIMS0_2:.*]]:3 = fir.box_dims %[[LOAD_BOX_A_2]], %[[C0]] : (!fir.box<!fir.heap<!fir.array<?xf32>>>, index) -> (index, index, index)
+! CHECK: %[[BOUND_1_50:.*]] = acc.bounds lowerbound(%[[LB]] : index) upperbound(%[[UB]] : index) extent(%[[DIMS0_2]]#1 : index) stride(%[[DIMS0_1]]#2 : index) startIdx(%[[DIMS0_0]]#0 : index) {strideInBytes = true}
+! CHECK: %[[BOX_ADDR:.*]] = fir.box_addr %[[LOAD_BOX_A_0]] : (!fir.box<!fir.heap<!fir.array<?xf32>>>) -> !fir.heap<!fir.array<?xf32>>
+! CHECK: %[[COPYIN:.*]] = acc.copyin varPtr(%[[BOX_ADDR]] : !fir.heap<!fir.array<?xf32>>) bounds(%[[BOUND_1_50]]) -> !fir.heap<!fir.array<?xf32>> {name = "a(1:50)"}
+! CHECK: %[[LOAD_BOX_A_0:.*]] = fir.load %[[DECLA]]#0 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>
+! CHECK: %[[LOAD_BOX_A_1:.*]] = fir.load %[[DECLA]]#1 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>
+! CHECK: %[[C0:.*]] = arith.constant 0 : index
+! CHECK: %[[DIMS0_0:.*]]:3 = fir.box_dims %[[LOAD_BOX_A_1]], %[[C0]] : (!fir.box<!fir.heap<!fir.array<?xf32>>>, index) -> (index, index, index)
+! CHECK: %[[C0:.*]] = arith.constant 0 : index
+! CHECK: %[[DIMS0_1:.*]]:3 = fir.box_dims %[[LOAD_BOX_A_0]], %[[C0]] : (!fir.box<!fir.heap<!fir.array<?xf32>>>, index) -> (index, index, index)
+! CHECK: %[[C51:.*]] = arith.constant 51 : index
+! CHECK: %[[LB:.*]] = arith.subi %[[C51]], %[[DIMS0_0]]#0 : index
+! CHECK: %[[C100:.*]] = arith.constant 100 : index
+! CHECK: %[[UB:.*]] = arith.subi %[[C100]], %[[DIMS0_0]]#0 : index
+! CHECK: %[[LOAD_BOX_A_2:.*]] = fir.load %[[DECLA]]#1 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>
+! CHECK: %[[C0:.*]] = arith.constant 0 : index
+! CHECK: %[[DIMS0_2:.*]]:3 = fir.box_dims %[[LOAD_BOX_A_2]], %[[C0]] : (!fir.box<!fir.heap<!fir.array<?xf32>>>, index) -> (index, index, index)
+! CHECK: %[[BOUND_51_100:.*]] = acc.bounds lowerbound(%[[LB]] : index) upperbound(%[[UB]] : index) extent(%[[DIMS0_2]]#1 : index) stride(%[[DIMS0_1]]#2 : index) startIdx(%[[DIMS0_0]]#0 : index) {strideInBytes = true}
+! CHECK: %[[BOX_ADDR:.*]] = fir.box_addr %[[LOAD_BOX_A_0]] : (!fir.box<!fir.heap<!fir.array<?xf32>>>) -> !fir.heap<!fir.array<?xf32>>
+! CHECK: %[[COPYOUT_CREATE:.*]] = acc.create varPtr(%[[BOX_ADDR]] : !fir.heap<!fir.array<?xf32>>) bounds(%[[BOUND_51_100]]) -> !fir.heap<!fir.array<?xf32>> {dataClause = #acc<data_clause acc_copyout>, name = "a(51:100)"}
+! CHECK: acc.data dataOperands(%[[COPYIN]], %[[COPYOUT_CREATE]] : !fir.heap<!fir.array<?xf32>>, !fir.heap<!fir.array<?xf32>>) {
+! CHECK:   acc.terminator
+! CHECK: }
+! CHECK: acc.delete accPtr(%[[COPYIN]] : !fir.heap<!fir.array<?xf32>>) bounds(%[[BOUND_1_50]]) {dataClause = #acc<data_clause acc_copyin>, name = "a(1:50)"}
+! CHECK: acc.copyout accPtr(%[[COPYOUT_CREATE]] : !fir.heap<!fir.array<?xf32>>) bounds(%[[BOUND_51_100]]) to varPtr(%[[BOX_ADDR]] : !fir.heap<!fir.array<?xf32>>) {name = "a(51:100)"}
+
+
+! Testing array sections on pointer array
+subroutine acc_operand_array_section_pointer()
+  real, target :: a(100)
+  real, pointer :: p(:)
+
+  p => a
+
+  !$acc data copyin(p(1:50))
+  !$acc end data
+end subroutine
+
+! CHECK-LABEL: func.func @_QMacc_data_operandPacc_operand_array_section_pointer() {
+! CHECK: %[[P:.*]] = fir.alloca !fir.box<!fir.ptr<!fir.array<?xf32>>> {bindc_name = "p", uniq_name = "_QMacc_data_operandFacc_operand_array_section_pointerEp"}
+! CHECK: %[[DECLP:.*]]:2 = hlfir.declare %[[P]] {fortran_attrs = #fir.var_attrs<pointer>
+! CHECK: %[[LOAD_BOX_P_0:.*]] = fir.load %[[DECLP]]#0 : !fir.ref<!fir.box<!fir.ptr<!fir.array<?xf32>>>>
+! CHECK: %[[LOAD_BOX_P_1:.*]] = fir.load %[[DECLP]]#1 : !fir.ref<!fir.box<!fir.ptr<!fir.array<?xf32>>>>
+! CHECK: %[[C0:.*]] = arith.constant 0 : index
+! CHECK: %[[DIMS0_0:.*]]:3 = fir.box_dims %[[LOAD_BOX_P_1]], %[[C0:.*]] : (!fir.box<!fir.ptr<!fir.array<?xf32>>>, index) -> (index, index, index)
+! CHECK: %[[C0:.*]] = arith.constant 0 : index
+! CHECK: %[[DIMS0_1:.*]]:3 = fir.box_dims %[[LOAD_BOX_P_0]], %[[C0]] : (!fir.box<!fir.ptr<!fir.array<?xf32>>>, index) -> (index, index, index)
+! CHECK: %[[C1:.*]] = arith.constant 1 : index
+! CHECK: %[[LB:.*]] = arith.subi %[[C1]], %[[DIMS0_0]]#0 : index
+! CHECK: %[[C50:.*]] = arith.constant 50 : index
+! CHECK: %[[UB:.*]] = arith.subi %[[C50]], %[[DIMS0_0]]#0 : index
+! CHECK: %[[LOAD_BOX_P_2:.*]] = fir.load %[[DECLP]]#1 : !fir.ref<!fir.box<!fir.ptr<!fir.array<?xf32>>>>
+! CHECK: %[[C0:.*]] = arith.constant 0 : index
+! CHECK: %[[DIMS0_2:.*]]:3 = fir.box_dims %[[LOAD_BOX_P_2]], %[[C0]] : (!fir.box<!fir.ptr<!fir.array<?xf32>>>, index) -> (index, index, index)
+! CHECK: %[[BOUND:.*]] = acc.bounds lowerbound(%[[LB]] : index) upperbound(%[[UB]] : index) extent(%[[DIMS0_2]]#1 : index) stride(%[[DIMS0_1]]#2 : index) startIdx(%[[DIMS0_0]]#0 : index) {strideInBytes = true}
+! CHECK: %[[BOX_ADDR:.*]] = fir.box_addr %[[LOAD_BOX_P_0]] : (!fir.box<!fir.ptr<!fir.array<?xf32>>>) -> !fir.ptr<!fir.array<?xf32>>
+! CHECK: %[[COPYIN:.*]] = acc.copyin varPtr(%[[BOX_ADDR]] : !fir.ptr<!fir.array<?xf32>>) bounds(%[[BOUND]]) -> !fir.ptr<!fir.array<?xf32>> {name = "p(1:50)"}
+! CHECK: acc.data dataOperands(%[[COPYIN]] : !fir.ptr<!fir.array<?xf32>>) {
+! CHECK:   acc.terminator
+! CHECK: }
+! CHECK: acc.delete accPtr(%[[COPYIN]] : !fir.ptr<!fir.array<?xf32>>) bounds(%[[BOUND]]) {dataClause = #acc<data_clause acc_copyin>, name = "p(1:50)"}
+
+end module
diff --git a/flang/test/Lower/OpenACC/acc-data-operands.f90 b/flang/test/Lower/OpenACC/acc-data-operands.f90
index 4341ea687070b..78b5963363a95 100644
--- a/flang/test/Lower/OpenACC/acc-data-operands.f90
+++ b/flang/test/Lower/OpenACC/acc-data-operands.f90
@@ -73,17 +73,12 @@ subroutine acc_operand_derived_type_component()
 ! CHECK-LABEL: func.func @_QMacc_data_operandPacc_operand_derived_type_component() {
 ! CHECK: %[[W:.*]] = fir.alloca !fir.type<_QMacc_data_operandTwrapper{data:!fir.array<100xf32>}> {bindc_name = "w", uniq_name = "_QMacc_data_operandFacc_operand_derived_type_componentEw"}
 ! CHECK: %[[DECLW:.*]]:2 = hlfir.declare %[[W]]
-! CHECK: %[[EXT:.*]] = arith.constant 100 : index
 ! CHECK: %[[COORD_DATA:.*]] = hlfir.designate %[[DECLW]]#0{"data"}   shape %{{.*}} : (!fir.ref<!fir.type<_QMacc_data_operandTwrapper{data:!fir.array<100xf32>}>>, !fir.shape<1>) -> !fir.ref<!fir.array<100xf32>>
-! CHECK: %[[ONE:.*]] = arith.constant 1 : index
-! CHECK: %[[LB:.*]] = arith.constant 0 : index
-! CHECK: %[[UB:.*]] = arith.subi %[[EXT]], %[[ONE]] : index
-! CHECK: %[[BOUND:.*]] = acc.bounds lowerbound(%[[LB]] : index) upperbound(%[[UB]] : index) extent(%[[EXT]] : index) stride(%[[ONE]] : index) startIdx(%[[ONE]] : index)
-! CHECK: %[[COPY_COPYIN:.*]] = acc.copyin varPtr(%[[COORD_DATA]] : !fir.ref<!fir.array<100xf32>>) bounds(%[[BOUND]]) -> !fir.ref<!fir.array<100xf32>> {dataClause = #acc<data_clause acc_copy>, name = "w%data"}
+! CHECK: %[[COPY_COPYIN:.*]] = acc.copyin varPtr(%[[COORD_DATA]] : !fir.ref<!fir.array<100xf32>>) -> !fir.ref<!fir.array<100xf32>> {dataClause = #acc<data_clause acc_copy>, name = "w%data"}
 ! CHECK: acc.data dataOperands(%[[COPY_COPYIN]] : !fir.ref<!fir.array<100xf32>>) {
 ! CHECK:   acc.terminator
 ! CHECK: }
-! CHECK: acc.copyout accPtr(%[[COPY_COPYIN]] : !fir.ref<!fir.array<100xf32>>) bounds(%[[BOUND]]) to varPtr(%[[COORD_DATA]] : !fir.ref<!fir.array<100xf32>>) {dataClause = #acc<data_clause acc_copy>, name = "w%data"}
+! CHECK: acc.copyout accPtr(%[[COPY_COPYIN]] : !fir.ref<!fir.array<100xf32>>) to varPtr(%[[COORD_DATA]] : !fir.ref<!fir.array<100xf32>>) {dataClause = #acc<data_clause acc_copy>, name = "w%data"}
 
 
 ! Testing array of derived-type component without section
@@ -99,17 +94,12 @@ subroutine acc_operand_array_derived_type_component()
 ! CHECK: %[[DECLW:.*]]:2 = hlfir.declare %[[W]]
 ! CHECK: %[[C1:.*]] = arith.constant 1 : index
 ! CHECK: %[[W_1:.*]] = hlfir.designate %[[DECLW]]#0 (%[[C1]])  : (!fir.ref<!fir.array<10x!fir.type<_QMacc_data_operandTwrapper{data:!fir.array<100xf32>}>>>, index) -> !fir.ref<!fir.type<_QMacc_data_operandTwrapper{data:!fir.array<100xf32>}>>
-! CHECK: %[[EXT:.*]] = arith.constant 100 : index
 ! CHECK: %[[COORD_W1_DATA:.*]] = hlfir.designate %[[W_1]]{"data"}   shape %{{.*}} : (!fir.ref<!fir.type<_QMacc_data_operandTwrapper{data:!fir.array<100xf32>}>>, !fir.shape<1>) -> !fir.ref<!fir.array<100xf32>>
-! CHECK: %[[ONE:.*]] = arith.constant 1 : index
-! CHECK: %[[LB:.*]] = arith.constant 0 : index
-! CHECK: %[[UB:.*]] = arith.subi %[[EXT]], %[[ONE]] : index
-! CHECK: %[[BOUND:.*]] = acc.bounds lowerbound(%[[LB]] : index) upperbound(%[[UB]] : index) extent(%[[EXT]] : index) stride(%[[ONE]] : index) startIdx(%[[ONE]] : index)
-! CHECK: %[[COPY_COPYIN:.*]] = acc.copyin varPtr(%[[COORD_W1_DATA]] : !fir.ref<!fir.array<100xf32>>) bounds(%[[BOUND]]) -> !fir.ref<!fir.array<100xf32>> {dataClause = #acc<data_clause acc_copy>, name = "w(1_8)%data"}
+! CHECK: %[[COPY_COPYIN:.*]] = acc.copyin varPtr(%[[COORD_W1_DATA]] : !fir.ref<!fir.array<100xf32>>) -> !fir.ref<!fir.array<100xf32>> {dataClause = #acc<data_clause acc_copy>, name = "w(1_8)%data"}
 ! CHECK: acc.data dataOperands(%[[COPY_COPYIN]] : !fir.ref<!fir.array<100xf32>>) {
 ! CHECK:   acc.terminator
 ! CHECK: }
-! CHECK: acc.copyout accPtr(%[[COPY_COPYIN]] : !fir.ref<!fir.array<100xf32>>) bounds(%[[BOUND]]) to varPtr(%[[COORD_W1_DATA]] : !fir.ref<!fir.array<100xf32>>) {dataClause = #acc<data_clause acc_copy>, name = "w(1_8)%data"}
+! CHECK: acc.copyout accPtr(%[[COPY_COPYIN]] : !fir.ref<!fir.array<100xf32>>) to varPtr(%[[COORD_W1_DATA]] : !fir.ref<!fir.array<100xf32>>) {dataClause = #acc<data_clause acc_copy>, name = "w(1_8)%data"}
 
 ! Testing array sections on allocatable array
 subroutine acc_operand_array_section_allocatable()
@@ -126,10 +116,10 @@ subroutine acc_operand_array_section_allocatable()
 ! CHECK-LABEL: func.func @_QMacc_data_operandPacc_operand_array_section_allocatable() {
 ! CHECK: %[[A:.*]] = fir.alloca !fir.box<!fir.heap<!fir.array<?xf32>>> {bindc_name = "a", uniq_name = "_QMacc_data_operandFacc_operand_array_section_allocatableEa"}
 ! CHECK: %[[DECLA:.*]]:2 = hlfir.declare %[[A]] {fortran_attrs = #fir.var_attrs<allocatable>
-! CHECK: %[[LOAD_BOX_A_0:.*]] = fir.load %[[DECLA]]#0 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>
 ! CHECK: %[[LOAD_BOX_A_1:.*]] = fir.load %[[DECLA]]#1 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>
 ! CHECK: %[[C0:.*]] = arith.constant 0 : index
 ! CHECK: %[[DIMS0_0:.*]]:3 = fir.box_dims %[[LOAD_BOX_A_1]], %[[C0]] : (!fir.box<!fir.heap<!fir.array<?xf32>>>, index) -> (index, index, index)
+! CHECK: %[[LOAD_BOX_A_0:.*]] = fir.load %[[DECLA]]#0 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>
 ! CHECK: %[[C0:.*]] = arith.constant 0 : index
 ! CHECK: %[[DIMS0_1:.*]]:3 = fir.box_dims %[[LOAD_BOX_A_0]], %[[C0]] : (!fir.box<!fir.heap<!fir.array<?xf32>>>, index) -> (index, index, index)
 ! CHECK: %[[C1:.*]] = arith.constant 1 : index
@@ -140,12 +130,11 @@ subroutine acc_operand_array_section_allocatable()
 ! CHECK: %[[C0:.*]] = arith.constant 0 : index
 ! CHECK: %[[DIMS0_2:.*]]:3 = fir.box_dims %[[LOAD_BOX_A_2]], %[[C0]] : (!fir.box<!fir.heap<!fir.array<?xf32>>>, index) -> (index, index, index)
 ! CHECK: %[[BOUND_1_50:.*]] = acc.bounds lowerbound(%[[LB]] : index) upperbound(%[[UB]] : index) extent(%[[DIMS0_2]]#1 : index) stride(%[[DIMS0_1]]#2 : index) startIdx(%[[DIMS0_0]]#0 : index) {strideInBytes = true}
-! CHECK: %[[BOX_ADDR:.*]] = fir.box_addr %[[LOAD_BOX_A_0]] : (!fir.box<!fir.heap<!fir.array<?xf32>>>) -> !fir.heap<!fir.array<?xf32>>
-! CHECK: %[[COPYIN:.*]] = acc.copyin varPtr(%[[BOX_ADDR]] : !fir.heap<!fir.array<?xf32>>) bounds(%[[BOUND_1_50]]) -> !fir.heap<!fir.array<?xf32>> {name = "a(1:50)"}
-! CHECK: %[[LOAD_BOX_A_0:.*]] = fir.load %[[DECLA]]#0 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>
+! CHECK: %[[COPYIN:.*]] = acc.copyin varPtr(%[[DECLA]]#0 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>) bounds(%[[BOUND_1_50]]) -> !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>> {name = "a(1:50)"}
 ! CHECK: %[[LOAD_BOX_A_1:.*]] = fir.load %[[DECLA]]#1 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>
 ! CHECK: %[[C0:.*]] = arith.constant 0 : index
 ! CHECK: %[[DIMS0_0:.*]]:3 = fir.box_dims %[[LOAD_BOX_A_1]], %[[C0]] : (!fir.box<!fir.heap<!fir.array<?xf32>>>, index) -> (index, index, index)
+! CHECK: %[[LOAD_BOX_A_0:.*]] = fir.load %[[DECLA]]#0 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>
 ! CHECK: %[[C0:.*]] = arith.constant 0 : index
 ! CHECK: %[[DIMS0_1:.*]]:3 = fir.box_dims %[[LOAD_BOX_A_0]], %[[C0]] : (!fir.box<!fir.heap<!fir.array<?xf32>>>, index) -> (index, index, index)
 ! CHECK: %[[C51:.*]] = arith.constant 51 : index
@@ -156,13 +145,12 @@ subroutine acc_operand_array_section_allocatable()
 ! CHECK: %[[C0:.*]] = arith.constant 0 : index
 ! CHECK: %[[DIMS0_2:.*]]:3 = fir.box_dims %[[LOAD_BOX_A_2]], %[[C0]] : (!fir.box<!fir.heap<!fir.array<?xf32>>>, index) -> (index, index, index)
 ! CHECK: %[[BOUND_51_100:.*]] = acc.bounds lowerbound(%[[LB]] : index) upperbound(%[[UB]] : index) extent(%[[DIMS0_2]]#1 : index) stride(%[[DIMS0_1]]#2 : index) startIdx(%[[DIMS0_0]]#0 : index) {strideInBytes = true}
-! CHECK: %[[BOX_ADDR:.*]] = fir.box_addr %[[LOAD_BOX_A_0]] : (!fir.box<!fir.heap<!fir.array<?xf32>>>) -> !fir.heap<!fir.array<?xf32>>
-! CHECK: %[[COPYOUT_CREATE:.*]] = acc.create varPtr(%[[BOX_ADDR]] : !fir.heap<!fir.array<?xf32>>) bounds(%[[BOUND_51_100]]) -> !fir.heap<!fir.array<?xf32>> {dataClause = #acc<data_clause acc_copyout>, name = "a(51:100)"}
-! CHECK: acc.data dataOperands(%[[COPYIN]], %[[COPYOUT_CREATE]] : !fir.heap<!fir.array<?xf32>>, !fir.heap<!fir.array<?xf32>>) {
+! CHECK: %[[COPYOUT_CREATE:.*]] = acc.create varPtr(%[[DECLA]]#0 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>) bounds(%[[BOUND_51_100]]) -> !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>> {dataClause = #acc<data_clause acc_copyout>, name = "a(51:100)"}
+! CHECK: acc.data dataOperands(%[[COPYIN]], %[[COPYOUT_CREATE]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>, !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>) {
 ! CHECK:   acc.terminator
 ! CHECK: }
-! CHECK: acc.delete accPtr(%[[COPYIN]] : !fir.heap<!fir.array<?xf32>>) bounds(%[[BOUND_1_50]]) {dataClause = #acc<data_clause acc_copyin>, name = "a(1:50)"}
-! CHECK: acc.copyout accPtr(%[[COPYOUT_CREATE]] : !fir.heap<!fir.array<?xf32>>) bounds(%[[BOUND_51_100]]) to varPtr(%[[BOX_ADDR]] : !fir.heap<!fir.array<?xf32>>) {name = "a(51:100)"}
+! CHECK: acc.delete accPtr(%[[COPYIN]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>) bounds(%[[BOUND_1_50]]) {dataClause = #acc<data_clause acc_copyin>, name = "a(1:50)"}
+! CHECK: acc.copyout accPtr(%[[COPYOUT_CREATE]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>) bounds(%[[BOUND_51_100]]) to varPtr(%[[DECLA]]#0 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>) {name = "a(51:100)"}
 
 
 ! Testing array sections on pointer array
@@ -179,10 +167,10 @@ subroutine acc_operand_array_section_pointer()
 ! CHECK-LABEL: func.func @_QMacc_data_operandPacc_operand_array_section_pointer() {
 ! CHECK: %[[P:.*]] = fir.alloca !fir.box<!fir.ptr<!fir.array<?xf32>>> {bindc_name = "p", uniq_name = "_QMacc_data_operandFacc_operand_array_section_pointerEp"}
 ! CHECK: %[[DECLP:.*]]:2 = hlfir.declare %[[P]] {fortran_attrs = #fir.var_attrs<pointer>
-! CHECK: %[[LOAD_BOX_P_0:.*]] = fir.load %[[DECLP]]#0 : !fir.ref<!fir.box<!fir.ptr<!fir.array<?xf32>>>>
 ! CHECK: %[[LOAD_BOX_P_1:.*]] = fir.load %[[DECLP]]#1 : !fir.ref<!fir.box<!fir.ptr<!fir.array<?xf32>>>>
 ! CHECK: %[[C0:.*]] = arith.constant 0 : index
 ! CHECK: %[[DIMS0_0:.*]]:3 = fir.box_dims %[[LOAD_BOX_P_1]], %[[C0:.*]] : (!fir.box<!fir.ptr<!fir.array<?xf32>>>, index) -> (index, index, index)
+! CHECK: %[[LOAD_BOX_P_0:.*]] = fir.load %[[DECLP]]#0 : !fir.ref<!fir.box<!fir.ptr<!fir.array<?xf32>>>>
 ! CHECK: %[[C0:.*]] = arith.constant 0 : index
 ! CHECK: %[[DIMS0_1:.*]]:3 = fir.box_dims %[[LOAD_BOX_P_0]], %[[C0]] : (!fir.box<!fir.ptr<!fir.array<?xf32>>>, index) -> (index, index, index)
 ! CHECK: %[[C1:.*]] = arith.constant 1 : index
@@ -193,12 +181,10 @@ subroutine acc_operand_array_section_pointer()
 ! CHECK: %[[C0:.*]] = arith.constant 0 : index
 ! CHECK: %[[DIMS0_2:.*]]:3 = fir.box_dims %[[LOAD_BOX_P_2]], %[[C0]] : (!fir.box<!fir.ptr<!fir.array<?xf32>>>, index) -> (index, index, index)
 ! CHECK: %[[BOUND:.*]] = acc.bounds lowerbound(%[[LB]] : index) upperbound(%[[UB]] : index) extent(%[[DIMS0_2]]#1 : index) stride(%[[DIMS0_1]]#2 : index) startIdx(%[[DIMS0_0]]#0 : index) {strideInBytes = true}
-! CHECK: %[[BOX_ADDR:.*]] = fir.box_addr %[[LOAD_BOX_P_0]] : (!fir.box<!fir.ptr<!fir.array<?xf32>>>) -> !fir.ptr<!fir.array<?xf32>>
-! CHECK: %[[COPYIN:.*]] = acc.copyin varPtr(%[[BOX_ADDR]] : !fir.ptr<!fir.array<?xf32>>) bounds(%[[BOUND]]) -> !fir.ptr<!fir.array<?xf32>> {name = "p(1:50)"}
-! CHECK: acc.data dataOperands(%[[COPYIN]] : !fir.ptr<!fir.array<?xf32>>) {
+! CHECK: %[[COPYIN:.*]] = acc.copyin varPtr(%[[DECLP]]#0 : !fir.ref<!fir.box<!fir.ptr<!fir.array<?xf32>>>>) bounds(%[[BOUND]]) -> !fir.ref<!fir.box<!fir.ptr<!fir.array<?xf32>>>> {name = "p(1:50)"}
+! CHECK: acc.data dataOperands(%[[COPYIN]] : !fir.ref<!fir.box<!fir.ptr<!fir.array<?xf32>>>>) {
 ! CHECK:   acc.terminator
 ! CHECK: }
-! CHECK: acc.delete accPtr(%[[COPYIN]] : !fir.ptr<!fir.array<?xf32>>) bounds(%[[BOUND]]) {dataClause = #acc<data_clause acc_copyin>, name = "p(1:50)"}
-
+! CHECK: acc.delete accPtr(%[[COPYIN]] : !fir.ref<!fir.box<!fir.ptr<!fir.array<?xf32>>>>) bounds(%[[BOUND]]) {dataClause = #acc<data_clause acc_copyin>, name = "p(1:50)"}
 
 end module
diff --git a/flang/test/Lower/OpenACC/acc-data-unwrap-defaultbounds.f90 b/flang/test/Lower/OpenACC/acc-data-unwrap-defaultbounds.f90
new file mode 100644
index 0000000000000..d010d39cef4eb
--- /dev/null
+++ b/flang/test/Lower/OpenACC/acc-data-unwrap-defaultbounds.f90
@@ -0,0 +1,205 @@
+! This test checks lowering of OpenACC data directive.
+
+! RUN: bbc -fopenacc -emit-hlfir --openacc-unwrap-fir-box=true --openacc-generate-default-bounds=true %s -o - | FileCheck %s
+
+subroutine acc_data
+  real, dimension(10, 10) :: a, b, c
+  real, pointer :: d, e
+  logical :: ifCondition = .TRUE.
+
+! CHECK: %[[A:.*]] = fir.alloca !fir.array<10x10xf32> {{{.*}}uniq_name = "{{.*}}Ea"}
+! CHECK:%[[DECLA:.*]]:2 = hlfir.declare %[[A]]
+! CHECK: %[[B:.*]] = fir.alloca !fir.array<10x10xf32> {{{.*}}uniq_name = "{{.*}}Eb"}
+! CHECK:%[[DECLB:.*]]:2 = hlfir.declare %[[B]]
+! CHECK: %[[C:.*]] = fir.alloca !fir.array<10x10xf32> {{{.*}}uniq_name = "{{.*}}Ec"}
+! CHECK:%[[DECLC:.*]]:2 = hlfir.declare %[[C]]
+! CHECK: %[[D:.*]] = fir.alloca !fir.box<!fir.ptr<f32>> {bindc_name = "d", uniq_name = "{{.*}}Ed"}
+! CHECK:%[[DECLD:.*]]:2 = hlfir.declare %[[D]]
+! CHECK: %[[E:.*]] = fir.alloca !fir.box<!fir.ptr<f32>> {bindc_name = "e", uniq_name = "{{.*}}Ee"}
+! CHECK:%[[DECLE:.*]]:2 = hlfir.declare %[[E]]
+
+  !$acc data if(.TRUE.) copy(a)
+  !$acc end data
+
+! CHECK:      %[[IF1:.*]] = arith.constant true
+! CHECK:     %[[COPYIN:.*]] = acc.copyin varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = #acc<data_clause acc_copy>, name = "a"}
+! CHECK:      acc.data if(%[[IF1]]) dataOperands(%[[COPYIN]] : !fir.ref<!fir.array<10x10xf32>>)  {
+! CHECK:        acc.terminator
+! CHECK-NEXT: }{{$}}
+! CHECK:acc.copyout accPtr(%[[COPYIN]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) to varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) {dataClause = #acc<data_clause acc_copy>, name = "a"}
+
+  !$acc data copy(a) if(ifCondition)
+  !$acc end data
+
+! CHECK:     %[[COPYIN:.*]] = acc.copyin varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = #acc<data_clause acc_copy>, name = "a"}
+! CHECK:      %[[IFCOND:.*]] = fir.load %{{.*}} : !fir.ref<!fir.logical<4>>
+! CHECK:      %[[IF2:.*]] = fir.convert %[[IFCOND]] : (!fir.logical<4>) -> i1
+! CHECK:      acc.data if(%[[IF2]]) dataOperands(%[[COPYIN]] : !fir.ref<!fir.array<10x10xf32>>) {
+! CHECK:        acc.terminator
+! CHECK-NEXT: }{{$}}
+! CHECK:acc.copyout accPtr(%[[COPYIN]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) to varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) {dataClause = #acc<data_clause acc_copy>, name = "a"}
+
+  !$acc data copy(a, b, c)
+  !$acc end data
+
+! CHECK:     %[[COPYIN_A:.*]] = acc.copyin varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = #acc<data_clause acc_copy>, name = "a"}
+! CHECK:     %[[COPYIN_B:.*]] = acc.copyin varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = #acc<data_clause acc_copy>, name = "b"}
+! CHECK:     %[[COPYIN_C:.*]] = acc.copyin varPtr(%[[DECLC]]#0 : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = #acc<data_clause acc_copy>, name = "c"}
+! CHECK:      acc.data dataOperands(%[[COPYIN_A]], %[[COPYIN_B]], %[[COPYIN_C]] : !fir.ref<!fir.array<10x10xf32>>, !fir.ref<!fir.array<10x10xf32>>, !fir.ref<!fir.array<10x10xf32>>) {
+! CHECK:        acc.terminator
+! CHECK-NEXT: }{{$}}
+! CHECK:acc.copyout accPtr(%[[COPYIN_A]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) to varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) {dataClause = #acc<data_clause acc_copy>, name = "a"}
+! CHECK:acc.copyout accPtr(%[[COPYIN_B]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) to varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<10x10xf32>>) {dataClause = #acc<data_clause acc_copy>, name = "b"}
+! CHECK:acc.copyout accPtr(%[[COPYIN_C]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) to varPtr(%[[DECLC]]#0 : !fir.ref<!fir.array<10x10xf32>>) {dataClause = #acc<data_clause acc_copy>, name = "c"}
+
+  !$acc data copy(a) copy(b) copy(c)
+  !$acc end data
+
+! CHECK:     %[[COPYIN_A:.*]] = acc.copyin varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = #acc<data_clause acc_copy>, name = "a"}
+! CHECK:     %[[COPYIN_B:.*]] = acc.copyin varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = #acc<data_clause acc_copy>, name = "b"}
+! CHECK:     %[[COPYIN_C:.*]] = acc.copyin varPtr(%[[DECLC]]#0 : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = #acc<data_clause acc_copy>, name = "c"}
+! CHECK:      acc.data dataOperands(%[[COPYIN_A]], %[[COPYIN_B]], %[[COPYIN_C]] : !fir.ref<!fir.array<10x10xf32>>, !fir.ref<!fir.array<10x10xf32>>, !fir.ref<!fir.array<10x10xf32>>) {
+! CHECK:        acc.terminator
+! CHECK-NEXT: }{{$}}
+! CHECK:acc.copyout accPtr(%[[COPYIN_A]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) to varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) {dataClause = #acc<data_clause acc_copy>, name = "a"}
+! CHECK:acc.copyout accPtr(%[[COPYIN_B]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) to varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<10x10xf32>>) {dataClause = #acc<data_clause acc_copy>, name = "b"}
+! CHECK:acc.copyout accPtr(%[[COPYIN_C]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) to varPtr(%[[DECLC]]#0 : !fir.ref<!fir.array<10x10xf32>>) {dataClause = #acc<data_clause acc_copy>, name = "c"}
+
+  !$acc data copyin(a) copyin(readonly: b, c)
+  !$acc end data
+
+! CHECK:     %[[COPYIN_A:.*]] = acc.copyin varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {name = "a"}
+! CHECK:     %[[COPYIN_B:.*]] = acc.copyin varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = #acc<data_clause acc_copyin_readonly>, name = "b"}
+! CHECK:     %[[COPYIN_C:.*]] = acc.copyin varPtr(%[[DECLC]]#0 : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = #acc<data_clause acc_copyin_readonly>, name = "c"}
+! CHECK:      acc.data dataOperands(%[[COPYIN_A]], %[[COPYIN_B]], %[[COPYIN_C]] : !fir.ref<!fir.array<10x10xf32>>, !fir.ref<!fir.array<10x10xf32>>, !fir.ref<!fir.array<10x10xf32>>) {
+! CHECK:        acc.terminator
+! CHECK-NEXT: }{{$}}
+! CHECK: acc.delete accPtr(%[[COPYIN_A]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) {dataClause = #acc<data_clause acc_copyin>, name = "a"}
+! CHECK: acc.delete accPtr(%[[COPYIN_B]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) {dataClause = #acc<data_clause acc_copyin_readonly>, name = "b"}
+! CHECK: acc.delete accPtr(%[[COPYIN_C]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) {dataClause = #acc<data_clause acc_copyin_readonly>, name = "c"}
+
+  !$acc data copyout(a) copyout(zero: b) copyout(c)
+  !$acc end data
+
+! CHECK:     %[[CREATE_A:.*]] = acc.create varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = #acc<data_clause acc_copyout>, name = "a"}
+! CHECK:     %[[CREATE_B:.*]] = acc.create varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = #acc<data_clause acc_copyout_zero>, name = "b"}
+! CHECK:     %[[CREATE_C:.*]] = acc.create varPtr(%[[DECLC]]#0 : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = #acc<data_clause acc_copyout>, name = "c"}
+! CHECK:      acc.data dataOperands(%[[CREATE_A]], %[[CREATE_B]], %[[CREATE_C]] : !fir.ref<!fir.array<10x10xf32>>, !fir.ref<!fir.array<10x10xf32>>, !fir.ref<!fir.array<10x10xf32>>) {
+! CHECK:        acc.terminator
+! CHECK-NEXT: }{{$}}
+! CHECK:acc.copyout accPtr(%[[CREATE_A]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) to varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) {name = "a"}
+! CHECK:acc.copyout accPtr(%[[CREATE_B]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) to varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<10x10xf32>>) {dataClause = #acc<data_clause acc_copyout_zero>, name = "b"}
+! CHECK:acc.copyout accPtr(%[[CREATE_C]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) to varPtr(%[[DECLC]]#0 : !fir.ref<!fir.array<10x10xf32>>) {name = "c"}
+
+  !$acc data create(a, b) create(zero: c)
+  !$acc end data
+
+! CHECK:     %[[CREATE_A:.*]] = acc.create varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {name = "a"}
+! CHECK:     %[[CREATE_B:.*]] = acc.create varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {name = "b"}
+! CHECK:     %[[CREATE_C:.*]] = acc.create varPtr(%[[DECLC]]#0 : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = #acc<data_clause acc_create_zero>, name = "c"}
+! CHECK:      acc.data dataOperands(%[[CREATE_A]], %[[CREATE_B]], %[[CREATE_C]] : !fir.ref<!fir.array<10x10xf32>>, !fir.ref<!fir.array<10x10xf32>>, !fir.ref<!fir.array<10x10xf32>>) {
+! CHECK:        acc.terminator
+! CHECK-NEXT: }{{$}}
+! CHECK: acc.delete accPtr(%[[CREATE_A]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) {dataClause = #acc<data_clause acc_create>, name = "a"}
+! CHECK: acc.delete accPtr(%[[CREATE_B]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) {dataClause = #acc<data_clause acc_create>, name = "b"}
+! CHECK: acc.delete accPtr(%[[CREATE_C]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) {dataClause = #acc<data_clause acc_create_zero>, name = "c"}
+
+  !$acc data create(c) copy(b) create(a)
+  !$acc end data
+! CHECK:%[[CREATE_C:.*]] = acc.create varPtr(%[[DECLC]]#0 : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {name = "c"}
+! CHECK:%[[COPY_B:.*]] = acc.copyin varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = #acc<data_clause acc_copy>, name = "b"}
+! CHECK:%[[CREATE_A:.*]] = acc.create varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {name = "a"}
+! CHECK: acc.data dataOperands(%[[CREATE_C]], %[[COPY_B]], %[[CREATE_A]] : !fir.ref<!fir.array<10x10xf32>>, !fir.ref<!fir.array<10x10xf32>>, !fir.ref<!fir.array<10x10xf32>>) {
+
+  !$acc data no_create(a, b) create(zero: c)
+  !$acc end data
+
+! CHECK:     %[[NO_CREATE_A:.*]] = acc.nocreate varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {name = "a"}
+! CHECK:     %[[NO_CREATE_B:.*]] = acc.nocreate varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {name = "b"}
+! CHECK:     %[[CREATE_C:.*]] = acc.create varPtr(%[[DECLC]]#0 : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = #acc<data_clause acc_create_zero>, name = "c"}
+! CHECK:      acc.data dataOperands(%[[NO_CREATE_A]], %[[NO_CREATE_B]], %[[CREATE_C]] : !fir.ref<!fir.array<10x10xf32>>, !fir.ref<!fir.array<10x10xf32>>, !fir.ref<!fir.array<10x10xf32>>) {
+! CHECK:        acc.terminator
+! CHECK-NEXT: }{{$}}
+! CHECK: acc.delete accPtr(%[[CREATE_C]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) {dataClause = #acc<data_clause acc_create_zero>, name = "c"}
+
+  !$acc data present(a, b, c)
+  !$acc end data
+
+! CHECK:     %[[PRESENT_A:.*]] = acc.present varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {name = "a"}
+! CHECK:     %[[PRESENT_B:.*]] = acc.present varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {name = "b"}
+! CHECK:     %[[PRESENT_C:.*]] = acc.present varPtr(%[[DECLC]]#0 : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {name = "c"}
+! CHECK:      acc.data dataOperands(%[[PRESENT_A]], %[[PRESENT_B]], %[[PRESENT_C]] : !fir.ref<!fir.array<10x10xf32>>, !fir.ref<!fir.array<10x10xf32>>, !fir.ref<!fir.array<10x10xf32>>) {
+! CHECK:        acc.terminator
+! CHECK-NEXT: }{{$}}
+
+  !$acc data deviceptr(b, c)
+  !$acc end data
+
+! CHECK:     %[[DEVICEPTR_B:.*]] = acc.deviceptr varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {name = "b"}
+! CHECK:     %[[DEVICEPTR_C:.*]] = acc.deviceptr varPtr(%[[DECLC]]#0 : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {name = "c"}
+! CHECK:      acc.data dataOperands(%[[DEVICEPTR_B]], %[[DEVICEPTR_C]] : !fir.ref<!fir.array<10x10xf32>>, !fir.ref<!fir.array<10x10xf32>>) {
+! CHECK:        acc.terminator
+! CHECK-NEXT: }{{$}}
+
+  !$acc data attach(d, e)
+  !$acc end data
+
+! CHECK:      %[[ATTACH_D:.*]] = acc.attach varPtr(%{{.*}} : !fir.ptr<f32>) -> !fir.ptr<f32> {name = "d"}
+! CHECK:      %[[ATTACH_E:.*]] = acc.attach varPtr(%{{.*}} : !fir.ptr<f32>) -> !fir.ptr<f32> {name = "e"}
+! CHECK:      acc.data dataOperands(%[[ATTACH_D]], %[[ATTACH_E]] : !fir.ptr<f32>, !fir.ptr<f32>) {
+! CHECK:        acc.terminator
+! CHECK-NEXT: }{{$}}
+! CHECK: acc.detach accPtr(%[[ATTACH_D]] : !fir.ptr<f32>) {dataClause = #acc<data_clause acc_attach>, name = "d"}
+! CHECK: acc.detach accPtr(%[[ATTACH_E]] : !fir.ptr<f32>) {dataClause = #acc<data_clause acc_attach>, name = "e"}
+
+  !$acc data present(a) async
+  !$acc end data
+
+! CHECK: acc.data dataOperands(%{{.*}}) {
+! CHECK: } attributes {asyncOnly = [#acc.device_type<none>]}
+
+  !$acc data copy(a) async(1)
+  !$acc end data
+
+! CHECK: %[[COPYIN:.*]] = acc.copyin varPtr(%{{.*}} : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) async(%[[ASYNC:.*]] : i32) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = #acc<data_clause acc_copy>, name = "a"}
+! CHECK: acc.data async(%[[ASYNC]] : i32) dataOperands(%[[COPYIN]] : !fir.ref<!fir.array<10x10xf32>>) {
+! CHECK: }{{$}}
+! CHECK: acc.copyout accPtr(%[[COPYIN]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) async(%[[ASYNC]] : i32) to varPtr(%{{.*}} : !fir.ref<!fir.array<10x10xf32>>) {dataClause = #acc<data_clause acc_copy>, name = "a"}
+
+  !$acc data present(a) wait
+  !$acc end data
+
+! CHECK: acc.data dataOperands(%{{.*}}) wait {
+! CHECK: }
+
+  !$acc data present(a) wait(1)
+  !$acc end data
+
+! CHECK: acc.data dataOperands(%{{.*}}) wait({%{{.*}} : i32}) {
+! CHECK: }{{$}}
+
+  !$acc data present(a) wait(devnum: 0: 1)
+  !$acc end data
+
+! CHECK: acc.data dataOperands(%{{.*}}) wait({devnum: %{{.*}} : i32, %{{.*}} : i32}) {
+! CHECK: }{{$}}
+
+  !$acc data default(none)
+  !$acc end data
+
+! CHECK: acc.data {
+! CHECK:   acc.terminator
+! CHECK: } attributes {defaultAttr = #acc<defaultvalue none>}
+
+  !$acc data default(present)
+  !$acc end data
+
+! CHECK: acc.data {
+! CHECK:   acc.terminator
+! CHECK: } attributes {defaultAttr = #acc<defaultvalue present>}
+
+  !$acc data
+  !$acc end data
+! CHECK-NOT: acc.data
+
+end subroutine acc_data
diff --git a/flang/test/Lower/OpenACC/acc-data.f90 b/flang/test/Lower/OpenACC/acc-data.f90
index 074f4d1135f10..7965fdc0ac707 100644
--- a/flang/test/Lower/OpenACC/acc-data.f90
+++ b/flang/test/Lower/OpenACC/acc-data.f90
@@ -22,112 +22,112 @@ subroutine acc_data
   !$acc end data
 
 ! CHECK:      %[[IF1:.*]] = arith.constant true
-! CHECK:     %[[COPYIN:.*]] = acc.copyin varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = #acc<data_clause acc_copy>, name = "a"}
+! CHECK:     %[[COPYIN:.*]] = acc.copyin varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = #acc<data_clause acc_copy>, name = "a"}
 ! CHECK:      acc.data if(%[[IF1]]) dataOperands(%[[COPYIN]] : !fir.ref<!fir.array<10x10xf32>>)  {
 ! CHECK:        acc.terminator
 ! CHECK-NEXT: }{{$}}
-! CHECK:acc.copyout accPtr(%[[COPYIN]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) to varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) {dataClause = #acc<data_clause acc_copy>, name = "a"}
+! CHECK:acc.copyout accPtr(%[[COPYIN]] : !fir.ref<!fir.array<10x10xf32>>) to varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) {dataClause = #acc<data_clause acc_copy>, name = "a"}
 
   !$acc data copy(a) if(ifCondition)
   !$acc end data
 
-! CHECK:     %[[COPYIN:.*]] = acc.copyin varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = #acc<data_clause acc_copy>, name = "a"}
+! CHECK:     %[[COPYIN:.*]] = acc.copyin varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = #acc<data_clause acc_copy>, name = "a"}
 ! CHECK:      %[[IFCOND:.*]] = fir.load %{{.*}} : !fir.ref<!fir.logical<4>>
 ! CHECK:      %[[IF2:.*]] = fir.convert %[[IFCOND]] : (!fir.logical<4>) -> i1
 ! CHECK:      acc.data if(%[[IF2]]) dataOperands(%[[COPYIN]] : !fir.ref<!fir.array<10x10xf32>>) {
 ! CHECK:        acc.terminator
 ! CHECK-NEXT: }{{$}}
-! CHECK:acc.copyout accPtr(%[[COPYIN]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) to varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) {dataClause = #acc<data_clause acc_copy>, name = "a"}
+! CHECK:acc.copyout accPtr(%[[COPYIN]] : !fir.ref<!fir.array<10x10xf32>>) to varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) {dataClause = #acc<data_clause acc_copy>, name = "a"}
 
   !$acc data copy(a, b, c)
   !$acc end data
 
-! CHECK:     %[[COPYIN_A:.*]] = acc.copyin varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = #acc<data_clause acc_copy>, name = "a"}
-! CHECK:     %[[COPYIN_B:.*]] = acc.copyin varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = #acc<data_clause acc_copy>, name = "b"}
-! CHECK:     %[[COPYIN_C:.*]] = acc.copyin varPtr(%[[DECLC]]#0 : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = #acc<data_clause acc_copy>, name = "c"}
+! CHECK:     %[[COPYIN_A:.*]] = acc.copyin varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = #acc<data_clause acc_copy>, name = "a"}
+! CHECK:     %[[COPYIN_B:.*]] = acc.copyin varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<10x10xf32>>) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = #acc<data_clause acc_copy>, name = "b"}
+! CHECK:     %[[COPYIN_C:.*]] = acc.copyin varPtr(%[[DECLC]]#0 : !fir.ref<!fir.array<10x10xf32>>) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = #acc<data_clause acc_copy>, name = "c"}
 ! CHECK:      acc.data dataOperands(%[[COPYIN_A]], %[[COPYIN_B]], %[[COPYIN_C]] : !fir.ref<!fir.array<10x10xf32>>, !fir.ref<!fir.array<10x10xf32>>, !fir.ref<!fir.array<10x10xf32>>) {
 ! CHECK:        acc.terminator
 ! CHECK-NEXT: }{{$}}
-! CHECK:acc.copyout accPtr(%[[COPYIN_A]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) to varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) {dataClause = #acc<data_clause acc_copy>, name = "a"}
-! CHECK:acc.copyout accPtr(%[[COPYIN_B]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) to varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<10x10xf32>>) {dataClause = #acc<data_clause acc_copy>, name = "b"}
-! CHECK:acc.copyout accPtr(%[[COPYIN_C]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) to varPtr(%[[DECLC]]#0 : !fir.ref<!fir.array<10x10xf32>>) {dataClause = #acc<data_clause acc_copy>, name = "c"}
+! CHECK:acc.copyout accPtr(%[[COPYIN_A]] : !fir.ref<!fir.array<10x10xf32>>) to varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) {dataClause = #acc<data_clause acc_copy>, name = "a"}
+! CHECK:acc.copyout accPtr(%[[COPYIN_B]] : !fir.ref<!fir.array<10x10xf32>>) to varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<10x10xf32>>) {dataClause = #acc<data_clause acc_copy>, name = "b"}
+! CHECK:acc.copyout accPtr(%[[COPYIN_C]] : !fir.ref<!fir.array<10x10xf32>>) to varPtr(%[[DECLC]]#0 : !fir.ref<!fir.array<10x10xf32>>) {dataClause = #acc<data_clause acc_copy>, name = "c"}
 
   !$acc data copy(a) copy(b) copy(c)
   !$acc end data
 
-! CHECK:     %[[COPYIN_A:.*]] = acc.copyin varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = #acc<data_clause acc_copy>, name = "a"}
-! CHECK:     %[[COPYIN_B:.*]] = acc.copyin varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = #acc<data_clause acc_copy>, name = "b"}
-! CHECK:     %[[COPYIN_C:.*]] = acc.copyin varPtr(%[[DECLC]]#0 : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = #acc<data_clause acc_copy>, name = "c"}
+! CHECK:     %[[COPYIN_A:.*]] = acc.copyin varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = #acc<data_clause acc_copy>, name = "a"}
+! CHECK:     %[[COPYIN_B:.*]] = acc.copyin varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<10x10xf32>>) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = #acc<data_clause acc_copy>, name = "b"}
+! CHECK:     %[[COPYIN_C:.*]] = acc.copyin varPtr(%[[DECLC]]#0 : !fir.ref<!fir.array<10x10xf32>>) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = #acc<data_clause acc_copy>, name = "c"}
 ! CHECK:      acc.data dataOperands(%[[COPYIN_A]], %[[COPYIN_B]], %[[COPYIN_C]] : !fir.ref<!fir.array<10x10xf32>>, !fir.ref<!fir.array<10x10xf32>>, !fir.ref<!fir.array<10x10xf32>>) {
 ! CHECK:        acc.terminator
 ! CHECK-NEXT: }{{$}}
-! CHECK:acc.copyout accPtr(%[[COPYIN_A]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) to varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) {dataClause = #acc<data_clause acc_copy>, name = "a"}
-! CHECK:acc.copyout accPtr(%[[COPYIN_B]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) to varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<10x10xf32>>) {dataClause = #acc<data_clause acc_copy>, name = "b"}
-! CHECK:acc.copyout accPtr(%[[COPYIN_C]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) to varPtr(%[[DECLC]]#0 : !fir.ref<!fir.array<10x10xf32>>) {dataClause = #acc<data_clause acc_copy>, name = "c"}
+! CHECK:acc.copyout accPtr(%[[COPYIN_A]] : !fir.ref<!fir.array<10x10xf32>>) to varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) {dataClause = #acc<data_clause acc_copy>, name = "a"}
+! CHECK:acc.copyout accPtr(%[[COPYIN_B]] : !fir.ref<!fir.array<10x10xf32>>) to varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<10x10xf32>>) {dataClause = #acc<data_clause acc_copy>, name = "b"}
+! CHECK:acc.copyout accPtr(%[[COPYIN_C]] : !fir.ref<!fir.array<10x10xf32>>) to varPtr(%[[DECLC]]#0 : !fir.ref<!fir.array<10x10xf32>>) {dataClause = #acc<data_clause acc_copy>, name = "c"}
 
   !$acc data copyin(a) copyin(readonly: b, c)
   !$acc end data
 
-! CHECK:     %[[COPYIN_A:.*]] = acc.copyin varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {name = "a"}
-! CHECK:     %[[COPYIN_B:.*]] = acc.copyin varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = #acc<data_clause acc_copyin_readonly>, name = "b"}
-! CHECK:     %[[COPYIN_C:.*]] = acc.copyin varPtr(%[[DECLC]]#0 : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = #acc<data_clause acc_copyin_readonly>, name = "c"}
+! CHECK:     %[[COPYIN_A:.*]] = acc.copyin varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) -> !fir.ref<!fir.array<10x10xf32>> {name = "a"}
+! CHECK:     %[[COPYIN_B:.*]] = acc.copyin varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<10x10xf32>>) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = #acc<data_clause acc_copyin_readonly>, name = "b"}
+! CHECK:     %[[COPYIN_C:.*]] = acc.copyin varPtr(%[[DECLC]]#0 : !fir.ref<!fir.array<10x10xf32>>) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = #acc<data_clause acc_copyin_readonly>, name = "c"}
 ! CHECK:      acc.data dataOperands(%[[COPYIN_A]], %[[COPYIN_B]], %[[COPYIN_C]] : !fir.ref<!fir.array<10x10xf32>>, !fir.ref<!fir.array<10x10xf32>>, !fir.ref<!fir.array<10x10xf32>>) {
 ! CHECK:        acc.terminator
 ! CHECK-NEXT: }{{$}}
-! CHECK: acc.delete accPtr(%[[COPYIN_A]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) {dataClause = #acc<data_clause acc_copyin>, name = "a"}
-! CHECK: acc.delete accPtr(%[[COPYIN_B]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) {dataClause = #acc<data_clause acc_copyin_readonly>, name = "b"}
-! CHECK: acc.delete accPtr(%[[COPYIN_C]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) {dataClause = #acc<data_clause acc_copyin_readonly>, name = "c"}
+! CHECK: acc.delete accPtr(%[[COPYIN_A]] : !fir.ref<!fir.array<10x10xf32>>) {dataClause = #acc<data_clause acc_copyin>, name = "a"}
+! CHECK: acc.delete accPtr(%[[COPYIN_B]] : !fir.ref<!fir.array<10x10xf32>>) {dataClause = #acc<data_clause acc_copyin_readonly>, name = "b"}
+! CHECK: acc.delete accPtr(%[[COPYIN_C]] : !fir.ref<!fir.array<10x10xf32>>) {dataClause = #acc<data_clause acc_copyin_readonly>, name = "c"}
 
   !$acc data copyout(a) copyout(zero: b) copyout(c)
   !$acc end data
 
-! CHECK:     %[[CREATE_A:.*]] = acc.create varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = #acc<data_clause acc_copyout>, name = "a"}
-! CHECK:     %[[CREATE_B:.*]] = acc.create varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = #acc<data_clause acc_copyout_zero>, name = "b"}
-! CHECK:     %[[CREATE_C:.*]] = acc.create varPtr(%[[DECLC]]#0 : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = #acc<data_clause acc_copyout>, name = "c"}
+! CHECK:     %[[CREATE_A:.*]] = acc.create varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = #acc<data_clause acc_copyout>, name = "a"}
+! CHECK:     %[[CREATE_B:.*]] = acc.create varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<10x10xf32>>) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = #acc<data_clause acc_copyout_zero>, name = "b"}
+! CHECK:     %[[CREATE_C:.*]] = acc.create varPtr(%[[DECLC]]#0 : !fir.ref<!fir.array<10x10xf32>>) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = #acc<data_clause acc_copyout>, name = "c"}
 ! CHECK:      acc.data dataOperands(%[[CREATE_A]], %[[CREATE_B]], %[[CREATE_C]] : !fir.ref<!fir.array<10x10xf32>>, !fir.ref<!fir.array<10x10xf32>>, !fir.ref<!fir.array<10x10xf32>>) {
 ! CHECK:        acc.terminator
 ! CHECK-NEXT: }{{$}}
-! CHECK:acc.copyout accPtr(%[[CREATE_A]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) to varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) {name = "a"}
-! CHECK:acc.copyout accPtr(%[[CREATE_B]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) to varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<10x10xf32>>) {dataClause = #acc<data_clause acc_copyout_zero>, name = "b"}
-! CHECK:acc.copyout accPtr(%[[CREATE_C]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) to varPtr(%[[DECLC]]#0 : !fir.ref<!fir.array<10x10xf32>>) {name = "c"}
+! CHECK:acc.copyout accPtr(%[[CREATE_A]] : !fir.ref<!fir.array<10x10xf32>>) to varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) {name = "a"}
+! CHECK:acc.copyout accPtr(%[[CREATE_B]] : !fir.ref<!fir.array<10x10xf32>>) to varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<10x10xf32>>) {dataClause = #acc<data_clause acc_copyout_zero>, name = "b"}
+! CHECK:acc.copyout accPtr(%[[CREATE_C]] : !fir.ref<!fir.array<10x10xf32>>) to varPtr(%[[DECLC]]#0 : !fir.ref<!fir.array<10x10xf32>>) {name = "c"}
 
   !$acc data create(a, b) create(zero: c)
   !$acc end data
 
-! CHECK:     %[[CREATE_A:.*]] = acc.create varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {name = "a"}
-! CHECK:     %[[CREATE_B:.*]] = acc.create varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {name = "b"}
-! CHECK:     %[[CREATE_C:.*]] = acc.create varPtr(%[[DECLC]]#0 : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = #acc<data_clause acc_create_zero>, name = "c"}
+! CHECK:     %[[CREATE_A:.*]] = acc.create varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) -> !fir.ref<!fir.array<10x10xf32>> {name = "a"}
+! CHECK:     %[[CREATE_B:.*]] = acc.create varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<10x10xf32>>) -> !fir.ref<!fir.array<10x10xf32>> {name = "b"}
+! CHECK:     %[[CREATE_C:.*]] = acc.create varPtr(%[[DECLC]]#0 : !fir.ref<!fir.array<10x10xf32>>) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = #acc<data_clause acc_create_zero>, name = "c"}
 ! CHECK:      acc.data dataOperands(%[[CREATE_A]], %[[CREATE_B]], %[[CREATE_C]] : !fir.ref<!fir.array<10x10xf32>>, !fir.ref<!fir.array<10x10xf32>>, !fir.ref<!fir.array<10x10xf32>>) {
 ! CHECK:        acc.terminator
 ! CHECK-NEXT: }{{$}}
-! CHECK: acc.delete accPtr(%[[CREATE_A]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) {dataClause = #acc<data_clause acc_create>, name = "a"}
-! CHECK: acc.delete accPtr(%[[CREATE_B]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) {dataClause = #acc<data_clause acc_create>, name = "b"}
-! CHECK: acc.delete accPtr(%[[CREATE_C]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) {dataClause = #acc<data_clause acc_create_zero>, name = "c"}
+! CHECK: acc.delete accPtr(%[[CREATE_A]] : !fir.ref<!fir.array<10x10xf32>>) {dataClause = #acc<data_clause acc_create>, name = "a"}
+! CHECK: acc.delete accPtr(%[[CREATE_B]] : !fir.ref<!fir.array<10x10xf32>>) {dataClause = #acc<data_clause acc_create>, name = "b"}
+! CHECK: acc.delete accPtr(%[[CREATE_C]] : !fir.ref<!fir.array<10x10xf32>>) {dataClause = #acc<data_clause acc_create_zero>, name = "c"}
 
   !$acc data create(c) copy(b) create(a)
   !$acc end data
-! CHECK:%[[CREATE_C:.*]] = acc.create varPtr(%[[DECLC]]#0 : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {name = "c"}
-! CHECK:%[[COPY_B:.*]] = acc.copyin varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = #acc<data_clause acc_copy>, name = "b"}
-! CHECK:%[[CREATE_A:.*]] = acc.create varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {name = "a"}
+! CHECK:%[[CREATE_C:.*]] = acc.create varPtr(%[[DECLC]]#0 : !fir.ref<!fir.array<10x10xf32>>) -> !fir.ref<!fir.array<10x10xf32>> {name = "c"}
+! CHECK:%[[COPY_B:.*]] = acc.copyin varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<10x10xf32>>) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = #acc<data_clause acc_copy>, name = "b"}
+! CHECK:%[[CREATE_A:.*]] = acc.create varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) -> !fir.ref<!fir.array<10x10xf32>> {name = "a"}
 ! CHECK: acc.data dataOperands(%[[CREATE_C]], %[[COPY_B]], %[[CREATE_A]] : !fir.ref<!fir.array<10x10xf32>>, !fir.ref<!fir.array<10x10xf32>>, !fir.ref<!fir.array<10x10xf32>>) {
 
   !$acc data no_create(a, b) create(zero: c)
   !$acc end data
 
-! CHECK:     %[[NO_CREATE_A:.*]] = acc.nocreate varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {name = "a"}
-! CHECK:     %[[NO_CREATE_B:.*]] = acc.nocreate varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {name = "b"}
-! CHECK:     %[[CREATE_C:.*]] = acc.create varPtr(%[[DECLC]]#0 : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = #acc<data_clause acc_create_zero>, name = "c"}
+! CHECK:     %[[NO_CREATE_A:.*]] = acc.nocreate varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) -> !fir.ref<!fir.array<10x10xf32>> {name = "a"}
+! CHECK:     %[[NO_CREATE_B:.*]] = acc.nocreate varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<10x10xf32>>) -> !fir.ref<!fir.array<10x10xf32>> {name = "b"}
+! CHECK:     %[[CREATE_C:.*]] = acc.create varPtr(%[[DECLC]]#0 : !fir.ref<!fir.array<10x10xf32>>) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = #acc<data_clause acc_create_zero>, name = "c"}
 ! CHECK:      acc.data dataOperands(%[[NO_CREATE_A]], %[[NO_CREATE_B]], %[[CREATE_C]] : !fir.ref<!fir.array<10x10xf32>>, !fir.ref<!fir.array<10x10xf32>>, !fir.ref<!fir.array<10x10xf32>>) {
 ! CHECK:        acc.terminator
 ! CHECK-NEXT: }{{$}}
-! CHECK: acc.delete accPtr(%[[CREATE_C]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) {dataClause = #acc<data_clause acc_create_zero>, name = "c"}
+! CHECK: acc.delete accPtr(%[[CREATE_C]] : !fir.ref<!fir.array<10x10xf32>>) {dataClause = #acc<data_clause acc_create_zero>, name = "c"}
 
   !$acc data present(a, b, c)
   !$acc end data
 
-! CHECK:     %[[PRESENT_A:.*]] = acc.present varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {name = "a"}
-! CHECK:     %[[PRESENT_B:.*]] = acc.present varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {name = "b"}
-! CHECK:     %[[PRESENT_C:.*]] = acc.present varPtr(%[[DECLC]]#0 : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {name = "c"}
+! CHECK:     %[[PRESENT_A:.*]] = acc.present varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) -> !fir.ref<!fir.array<10x10xf32>> {name = "a"}
+! CHECK:     %[[PRESENT_B:.*]] = acc.present varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<10x10xf32>>) -> !fir.ref<!fir.array<10x10xf32>> {name = "b"}
+! CHECK:     %[[PRESENT_C:.*]] = acc.present varPtr(%[[DECLC]]#0 : !fir.ref<!fir.array<10x10xf32>>) -> !fir.ref<!fir.array<10x10xf32>> {name = "c"}
 ! CHECK:      acc.data dataOperands(%[[PRESENT_A]], %[[PRESENT_B]], %[[PRESENT_C]] : !fir.ref<!fir.array<10x10xf32>>, !fir.ref<!fir.array<10x10xf32>>, !fir.ref<!fir.array<10x10xf32>>) {
 ! CHECK:        acc.terminator
 ! CHECK-NEXT: }{{$}}
@@ -135,8 +135,8 @@ subroutine acc_data
   !$acc data deviceptr(b, c)
   !$acc end data
 
-! CHECK:     %[[DEVICEPTR_B:.*]] = acc.deviceptr varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {name = "b"}
-! CHECK:     %[[DEVICEPTR_C:.*]] = acc.deviceptr varPtr(%[[DECLC]]#0 : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {name = "c"}
+! CHECK:     %[[DEVICEPTR_B:.*]] = acc.deviceptr varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<10x10xf32>>) -> !fir.ref<!fir.array<10x10xf32>> {name = "b"}
+! CHECK:     %[[DEVICEPTR_C:.*]] = acc.deviceptr varPtr(%[[DECLC]]#0 : !fir.ref<!fir.array<10x10xf32>>) -> !fir.ref<!fir.array<10x10xf32>> {name = "c"}
 ! CHECK:      acc.data dataOperands(%[[DEVICEPTR_B]], %[[DEVICEPTR_C]] : !fir.ref<!fir.array<10x10xf32>>, !fir.ref<!fir.array<10x10xf32>>) {
 ! CHECK:        acc.terminator
 ! CHECK-NEXT: }{{$}}
@@ -144,13 +144,13 @@ subroutine acc_data
   !$acc data attach(d, e)
   !$acc end data
 
-! CHECK:      %[[ATTACH_D:.*]] = acc.attach varPtr(%{{.*}} : !fir.ptr<f32>) -> !fir.ptr<f32> {name = "d"}
-! CHECK:      %[[ATTACH_E:.*]] = acc.attach varPtr(%{{.*}} : !fir.ptr<f32>) -> !fir.ptr<f32> {name = "e"}
-! CHECK:      acc.data dataOperands(%[[ATTACH_D]], %[[ATTACH_E]] : !fir.ptr<f32>, !fir.ptr<f32>) {
+! CHECK:      %[[ATTACH_D:.*]] = acc.attach varPtr(%{{.*}} : !fir.ref<!fir.box<!fir.ptr<f32>>>) -> !fir.ref<!fir.box<!fir.ptr<f32>>> {name = "d"}
+! CHECK:      %[[ATTACH_E:.*]] = acc.attach varPtr(%{{.*}} : !fir.ref<!fir.box<!fir.ptr<f32>>>) -> !fir.ref<!fir.box<!fir.ptr<f32>>> {name = "e"}
+! CHECK:      acc.data dataOperands(%[[ATTACH_D]], %[[ATTACH_E]] : !fir.ref<!fir.box<!fir.ptr<f32>>>, !fir.ref<!fir.box<!fir.ptr<f32>>>) {
 ! CHECK:        acc.terminator
 ! CHECK-NEXT: }{{$}}
-! CHECK: acc.detach accPtr(%[[ATTACH_D]] : !fir.ptr<f32>) {dataClause = #acc<data_clause acc_attach>, name = "d"}
-! CHECK: acc.detach accPtr(%[[ATTACH_E]] : !fir.ptr<f32>) {dataClause = #acc<data_clause acc_attach>, name = "e"}
+! CHECK: acc.detach accPtr(%[[ATTACH_D]] : !fir.ref<!fir.box<!fir.ptr<f32>>>) {dataClause = #acc<data_clause acc_attach>, name = "d"}
+! CHECK: acc.detach accPtr(%[[ATTACH_E]] : !fir.ref<!fir.box<!fir.ptr<f32>>>) {dataClause = #acc<data_clause acc_attach>, name = "e"}
 
   !$acc data present(a) async
   !$acc end data
@@ -161,10 +161,10 @@ subroutine acc_data
   !$acc data copy(a) async(1)
   !$acc end data
 
-! CHECK: %[[COPYIN:.*]] = acc.copyin varPtr(%{{.*}} : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) async(%[[ASYNC:.*]] : i32) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = #acc<data_clause acc_copy>, name = "a"}
+! CHECK: %[[COPYIN:.*]] = acc.copyin varPtr(%{{.*}} : !fir.ref<!fir.array<10x10xf32>>) async(%[[ASYNC:.*]] : i32) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = #acc<data_clause acc_copy>, name = "a"}
 ! CHECK: acc.data async(%[[ASYNC]] : i32) dataOperands(%[[COPYIN]] : !fir.ref<!fir.array<10x10xf32>>) {
 ! CHECK: }{{$}}
-! CHECK: acc.copyout accPtr(%[[COPYIN]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) async(%[[ASYNC]] : i32) to varPtr(%{{.*}} : !fir.ref<!fir.array<10x10xf32>>) {dataClause = #acc<data_clause acc_copy>, name = "a"}
+! CHECK: acc.copyout accPtr(%[[COPYIN]] : !fir.ref<!fir.array<10x10xf32>>) async(%[[ASYNC]] : i32) to varPtr(%{{.*}} : !fir.ref<!fir.array<10x10xf32>>) {dataClause = #acc<data_clause acc_copy>, name = "a"}
 
   !$acc data present(a) wait
   !$acc end data
diff --git a/flang/test/Lower/OpenACC/acc-declare-unwrap-defaultbounds.f90 b/flang/test/Lower/OpenACC/acc-declare-unwrap-defaultbounds.f90
new file mode 100644
index 0000000000000..065033431137d
--- /dev/null
+++ b/flang/test/Lower/OpenACC/acc-declare-unwrap-defaultbounds.f90
@@ -0,0 +1,478 @@
+! This test checks lowering of OpenACC declare directive in function and
+! subroutine specification parts.
+
+! RUN: bbc -fopenacc -emit-hlfir --openacc-unwrap-fir-box=true --openacc-generate-default-bounds=true %s -o - | FileCheck %s
+
+module acc_declare
+  contains
+
+  subroutine acc_declare_copy()
+    integer :: a(100), i
+    !$acc declare copy(a)
+
+    do i = 1, 100
+      a(i) = i
+    end do
+  end subroutine
+
+! CHECK-LABEL: func.func @_QMacc_declarePacc_declare_copy()
+! CHECK-DAG: %[[C1:.*]] = arith.constant 1 : index
+! CHECK-DAG: %[[ALLOCA:.*]] = fir.alloca !fir.array<100xi32> {bindc_name = "a", uniq_name = "_QMacc_declareFacc_declare_copyEa"}
+! CHECK-DAG: %[[DECL:.*]]:2 = hlfir.declare %[[ALLOCA]](%{{.*}}) {acc.declare = #acc.declare<dataClause =  acc_copy>, uniq_name = "_QMacc_declareFacc_declare_copyEa"} : (!fir.ref<!fir.array<100xi32>>, !fir.shape<1>) -> (!fir.ref<!fir.array<100xi32>>, !fir.ref<!fir.array<100xi32>>)
+! CHECK: %[[BOUND:.*]] = acc.bounds   lowerbound(%{{.*}} : index) upperbound(%{{.*}} : index) extent(%{{.*}} : index) stride(%[[C1]] : index) startIdx(%[[C1]] : index)
+! CHECK: %[[COPYIN:.*]] = acc.copyin varPtr(%[[DECL]]#0 : !fir.ref<!fir.array<100xi32>>) bounds(%[[BOUND]]) -> !fir.ref<!fir.array<100xi32>> {dataClause = #acc<data_clause acc_copy>, name = "a"}
+! CHECK: %[[TOKEN:.*]] = acc.declare_enter dataOperands(%[[COPYIN]] : !fir.ref<!fir.array<100xi32>>)
+! CHECK: %{{.*}}:2 = fir.do_loop %{{.*}} = %{{.*}} to %{{.*}} step %{{.*}} iter_args(%{{.*}} = %{{.*}}) -> (index, i32) {
+! CHECK: }
+! CHECK: acc.declare_exit token(%[[TOKEN]]) dataOperands(%[[COPYIN]] : !fir.ref<!fir.array<100xi32>>)
+! CHECK: acc.copyout accPtr(%[[COPYIN]] : !fir.ref<!fir.array<100xi32>>) bounds(%[[BOUND]]) to varPtr(%[[DECL]]#0 : !fir.ref<!fir.array<100xi32>>) {dataClause = #acc<data_clause acc_copy>, name = "a"}
+! CHECK: return
+
+  subroutine acc_declare_create()
+    integer :: a(100), i
+    !$acc declare create(a)
+
+    do i = 1, 100
+      a(i) = i
+    end do
+  end subroutine
+
+! CHECK-LABEL: func.func @_QMacc_declarePacc_declare_create() {
+! CHECK-DAG: %[[C1:.*]] = arith.constant 1 : index
+! CHECK-DAG: %[[ALLOCA:.*]] = fir.alloca !fir.array<100xi32> {bindc_name = "a", uniq_name = "_QMacc_declareFacc_declare_createEa"}
+! CHECK-DAG: %[[DECL:.*]]:2 = hlfir.declare %[[ALLOCA]](%{{.*}}) {acc.declare = #acc.declare<dataClause =  acc_create>, uniq_name = "_QMacc_declareFacc_declare_createEa"} : (!fir.ref<!fir.array<100xi32>>, !fir.shape<1>) -> (!fir.ref<!fir.array<100xi32>>, !fir.ref<!fir.array<100xi32>>)
+! CHECK: %[[BOUND:.*]] = acc.bounds   lowerbound(%{{.*}} : index) upperbound(%{{.*}} : index) extent(%{{.*}} : index) stride(%[[C1]] : index) startIdx(%[[C1]] : index)
+! CHECK: %[[CREATE:.*]] = acc.create varPtr(%[[DECL]]#0 : !fir.ref<!fir.array<100xi32>>)   bounds(%[[BOUND]]) -> !fir.ref<!fir.array<100xi32>> {name = "a"}
+! CHECK: %[[TOKEN:.*]] = acc.declare_enter dataOperands(%[[CREATE]] : !fir.ref<!fir.array<100xi32>>)
+! CHECK: %{{.*}}:2 = fir.do_loop %{{.*}} = %{{.*}} to %{{.*}} step %{{.*}} iter_args(%{{.*}} = %{{.*}}) -> (index, i32) {
+! CHECK: }
+! CHECK: acc.declare_exit token(%[[TOKEN]]) dataOperands(%[[CREATE]] : !fir.ref<!fir.array<100xi32>>)
+! CHECK: acc.delete accPtr(%[[CREATE]] : !fir.ref<!fir.array<100xi32>>) bounds(%[[BOUND]]) {dataClause = #acc<data_clause acc_create>, name = "a"}
+! CHECK: return
+
+  subroutine acc_declare_present(a)
+    integer :: a(100), i
+    !$acc declare present(a)
+
+    do i = 1, 100
+      a(i) = i
+    end do
+  end subroutine
+
+! CHECK-LABEL: func.func @_QMacc_declarePacc_declare_present(
+! CHECK-SAME: %[[ARG0:.*]]: !fir.ref<!fir.array<100xi32>> {fir.bindc_name = "a"})
+! CHECK-DAG: %[[C1:.*]] = arith.constant 1 : index
+! CHECK-DAG: %[[DECL:.*]]:2 = hlfir.declare %[[ARG0]](%{{.*}}) dummy_scope %{{[0-9]+}} {acc.declare = #acc.declare<dataClause =  acc_present>, uniq_name = "_QMacc_declareFacc_declare_presentEa"} : (!fir.ref<!fir.array<100xi32>>, !fir.shape<1>, !fir.dscope) -> (!fir.ref<!fir.array<100xi32>>, !fir.ref<!fir.array<100xi32>>)
+! CHECK: %[[BOUND:.*]] = acc.bounds lowerbound(%{{.*}} : index) upperbound(%{{.*}} : index) extent(%{{.*}} : index) stride(%{{.*}} : index) startIdx(%[[C1]] : index)
+! CHECK: %[[PRESENT:.*]] = acc.present varPtr(%[[DECL]]#0 : !fir.ref<!fir.array<100xi32>>)   bounds(%[[BOUND]]) -> !fir.ref<!fir.array<100xi32>> {name = "a"}
+! CHECK: %[[TOKEN:.*]] = acc.declare_enter dataOperands(%[[PRESENT]] : !fir.ref<!fir.array<100xi32>>)
+! CHECK: %{{.*}}:2 = fir.do_loop %{{.*}} = %{{.*}} to %{{.*}} step %{{.*}} iter_args(%arg{{.*}} = %{{.*}}) -> (index, i32)
+! CHECK: acc.declare_exit token(%[[TOKEN]]) dataOperands(%[[PRESENT]] : !fir.ref<!fir.array<100xi32>>)
+! CHECK: acc.delete accPtr(%[[PRESENT]] : !fir.ref<!fir.array<100xi32>>) bounds(%[[BOUND]]) {dataClause = #acc<data_clause acc_present>, name = "a"}
+
+  subroutine acc_declare_copyin()
+    integer :: a(100), b(10), i
+    !$acc declare copyin(a) copyin(readonly: b)
+
+    do i = 1, 100
+      a(i) = i
+    end do
+  end subroutine
+
+! CHECK-LABEL: func.func @_QMacc_declarePacc_declare_copyin()
+! CHECK: %[[A:.*]] = fir.alloca !fir.array<100xi32> {bindc_name = "a", uniq_name = "_QMacc_declareFacc_declare_copyinEa"}
+! CHECK: %[[ADECL:.*]]:2 = hlfir.declare %[[A]](%{{.*}}) {acc.declare = #acc.declare<dataClause =  acc_copyin>, uniq_name = "_QMacc_declareFacc_declare_copyinEa"} : (!fir.ref<!fir.array<100xi32>>, !fir.shape<1>) -> (!fir.ref<!fir.array<100xi32>>, !fir.ref<!fir.array<100xi32>>)
+! CHECK: %[[B:.*]] = fir.alloca !fir.array<10xi32> {bindc_name = "b", uniq_name = "_QMacc_declareFacc_declare_copyinEb"}
+! CHECK: %[[BDECL:.*]]:2 = hlfir.declare %[[B]](%{{.*}}) {acc.declare = #acc.declare<dataClause =  acc_copyin_readonly>, uniq_name = "_QMacc_declareFacc_declare_copyinEb"} : (!fir.ref<!fir.array<10xi32>>, !fir.shape<1>) -> (!fir.ref<!fir.array<10xi32>>, !fir.ref<!fir.array<10xi32>>)
+! CHECK: %[[BOUND_A:.*]] = acc.bounds lowerbound(%{{.*}} : index) upperbound(%{{.*}} : index) extent(%{{.*}} : index) stride(%{{.*}} : index) startIdx(%{{.*}} : index)
+! CHECK: %[[COPYIN_A:.*]] = acc.copyin varPtr(%[[ADECL]]#0 : !fir.ref<!fir.array<100xi32>>)   bounds(%[[BOUND_A]]) -> !fir.ref<!fir.array<100xi32>> {name = "a"}
+! CHECK: %[[BOUND_B:.*]] = acc.bounds lowerbound(%{{.*}} : index) upperbound(%{{.*}} : index) extent(%{{.*}} : index) stride(%{{.*}} : index) startIdx(%{{.*}} : index)
+! CHECK: %[[COPYIN_B:.*]] = acc.copyin varPtr(%[[BDECL]]#0 : !fir.ref<!fir.array<10xi32>>)   bounds(%[[BOUND_B]]) -> !fir.ref<!fir.array<10xi32>> {dataClause = #acc<data_clause acc_copyin_readonly>, name = "b"}
+! CHECK: acc.declare_enter dataOperands(%[[COPYIN_A]], %[[COPYIN_B]] : !fir.ref<!fir.array<100xi32>>, !fir.ref<!fir.array<10xi32>>)
+! CHECK: %{{.*}}:2 = fir.do_loop %{{.*}} = %{{.*}} to %{{.*}} step %{{.*}} iter_args(%arg{{.*}} = %{{.*}}) -> (index, i32)
+! CHECK: acc.delete accPtr(%[[COPYIN_A]] : !fir.ref<!fir.array<100xi32>>)   bounds(%[[BOUND_A]]) {dataClause = #acc<data_clause acc_copyin>, name = "a"}
+! CHECK: acc.delete accPtr(%[[COPYIN_B]] : !fir.ref<!fir.array<10xi32>>)   bounds(%[[BOUND_B]]) {dataClause = #acc<data_clause acc_copyin_readonly>, name = "b"}
+
+  subroutine acc_declare_copyout()
+    integer :: a(100), i
+    !$acc declare copyout(a)
+
+    do i = 1, 100
+      a(i) = i
+    end do
+  end subroutine
+
+! CHECK-LABEL: func.func @_QMacc_declarePacc_declare_copyout()
+! CHECK: %[[A:.*]] = fir.alloca !fir.array<100xi32> {bindc_name = "a", uniq_name = "_QMacc_declareFacc_declare_copyoutEa"}
+! CHECK: %[[ADECL:.*]]:2 = hlfir.declare %[[A]](%{{.*}}) {acc.declare = #acc.declare<dataClause =  acc_copyout>, uniq_name = "_QMacc_declareFacc_declare_copyoutEa"} : (!fir.ref<!fir.array<100xi32>>, !fir.shape<1>) -> (!fir.ref<!fir.array<100xi32>>, !fir.ref<!fir.array<100xi32>>)
+! CHECK: %[[CREATE:.*]] = acc.create varPtr(%[[ADECL]]#0 : !fir.ref<!fir.array<100xi32>>)   bounds(%{{.*}}) -> !fir.ref<!fir.array<100xi32>> {dataClause = #acc<data_clause acc_copyout>, name = "a"}
+! CHECK: %[[TOKEN:.*]] = acc.declare_enter dataOperands(%[[CREATE]] : !fir.ref<!fir.array<100xi32>>)
+! CHECK: %{{.*}}:2 = fir.do_loop %{{.*}} = %{{.*}} to %{{.*}} step %{{.*}} iter_args(%arg{{.*}} = %{{.*}}) -> (index, i32)
+! CHECK: acc.declare_exit token(%[[TOKEN]]) dataOperands(%[[CREATE]] : !fir.ref<!fir.array<100xi32>>)
+! CHECK: acc.copyout accPtr(%[[CREATE]] : !fir.ref<!fir.array<100xi32>>)   bounds(%{{.*}}) to varPtr(%[[ADECL]]#0 : !fir.ref<!fir.array<100xi32>>) {name = "a"}
+! CHECK: return
+
+  subroutine acc_declare_deviceptr(a)
+    integer :: a(100), i
+    !$acc declare deviceptr(a)
+
+    do i = 1, 100
+      a(i) = i
+    end do
+  end subroutine
+
+! CHECK-LABEL: func.func @_QMacc_declarePacc_declare_deviceptr(
+! CHECK-SAME: %[[ARG0:.*]]: !fir.ref<!fir.array<100xi32>> {fir.bindc_name = "a"}) {
+! CHECK: %[[DECL:.*]]:2 = hlfir.declare %[[ARG0]](%{{.*}}) dummy_scope %{{[0-9]+}} {acc.declare = #acc.declare<dataClause =  acc_deviceptr>, uniq_name = "_QMacc_declareFacc_declare_deviceptrEa"} : (!fir.ref<!fir.array<100xi32>>, !fir.shape<1>, !fir.dscope) -> (!fir.ref<!fir.array<100xi32>>, !fir.ref<!fir.array<100xi32>>)
+! CHECK: %[[DEVICEPTR:.*]] = acc.deviceptr varPtr(%[[DECL]]#0 : !fir.ref<!fir.array<100xi32>>)   bounds(%{{.*}}) -> !fir.ref<!fir.array<100xi32>> {name = "a"}
+! CHECK: acc.declare_enter dataOperands(%[[DEVICEPTR]] : !fir.ref<!fir.array<100xi32>>)
+! CHECK: %{{.*}}:2 = fir.do_loop %{{.*}} = %{{.*}} to %{{.*}} step %{{.*}} iter_args(%arg{{.*}} = %{{.*}}) -> (index, i32)
+
+  subroutine acc_declare_link(a)
+    integer :: a(100), i
+    !$acc declare link(a)
+
+    do i = 1, 100
+      a(i) = i
+    end do
+  end subroutine
+
+! CHECK-LABEL: func.func @_QMacc_declarePacc_declare_link(
+! CHECK-SAME: %[[ARG0:.*]]: !fir.ref<!fir.array<100xi32>> {fir.bindc_name = "a"})
+! CHECK: %[[DECL:.*]]:2 = hlfir.declare %[[ARG0]](%{{.*}}) dummy_scope %{{[0-9]+}} {acc.declare = #acc.declare<dataClause =  acc_declare_link>, uniq_name = "_QMacc_declareFacc_declare_linkEa"} : (!fir.ref<!fir.array<100xi32>>, !fir.shape<1>, !fir.dscope) -> (!fir.ref<!fir.array<100xi32>>, !fir.ref<!fir.array<100xi32>>)
+! CHECK: %[[LINK:.*]] = acc.declare_link varPtr(%[[DECL]]#0 : !fir.ref<!fir.array<100xi32>>)   bounds(%{{.*}}) -> !fir.ref<!fir.array<100xi32>> {name = "a"}
+! CHECK: acc.declare_enter dataOperands(%[[LINK]] : !fir.ref<!fir.array<100xi32>>)
+! CHECK: %{{.*}}:2 = fir.do_loop %{{.*}} = %{{.*}} to %{{.*}} step %{{.*}} iter_args(%arg{{.*}} = %{{.*}}) -> (index, i32)
+
+  subroutine acc_declare_device_resident(a)
+    integer :: a(100), i
+    !$acc declare device_resident(a)
+
+    do i = 1, 100
+      a(i) = i
+    end do
+  end subroutine
+
+! CHECK-LABEL: func.func @_QMacc_declarePacc_declare_device_resident(
+! CHECK-SAME: %[[ARG0:.*]]: !fir.ref<!fir.array<100xi32>> {fir.bindc_name = "a"})
+! CHECK: %[[DECL:.*]]:2 = hlfir.declare %[[ARG0]](%{{.*}}) dummy_scope %{{[0-9]+}} {acc.declare = #acc.declare<dataClause =  acc_declare_device_resident>, uniq_name = "_QMacc_declareFacc_declare_device_residentEa"} : (!fir.ref<!fir.array<100xi32>>, !fir.shape<1>, !fir.dscope) -> (!fir.ref<!fir.array<100xi32>>, !fir.ref<!fir.array<100xi32>>)
+! CHECK: %[[DEVICERES:.*]] = acc.declare_device_resident varPtr(%[[DECL]]#0 : !fir.ref<!fir.array<100xi32>>)   bounds(%{{.*}}) -> !fir.ref<!fir.array<100xi32>> {name = "a"}
+! CHECK: %[[TOKEN:.*]] = acc.declare_enter dataOperands(%[[DEVICERES]] : !fir.ref<!fir.array<100xi32>>)
+! CHECK: %{{.*}}:2 = fir.do_loop %{{.*}} = %{{.*}} to %{{.*}} step %{{.*}} iter_args(%arg{{.*}} = %{{.*}}) -> (index, i32)
+! CHECK: acc.declare_exit token(%[[TOKEN]]) dataOperands(%[[DEVICERES]] : !fir.ref<!fir.array<100xi32>>)
+! CHECK: acc.delete accPtr(%[[DEVICERES]] : !fir.ref<!fir.array<100xi32>>) bounds(%{{.*}}) {dataClause = #acc<data_clause acc_declare_device_resident>, name = "a"}
+
+  subroutine acc_declare_device_resident2()
+    integer, parameter :: n = 100
+    real, dimension(n) :: dataparam
+    !$acc declare device_resident(dataparam)
+  end subroutine
+
+! CHECK-LABEL: func.func @_QMacc_declarePacc_declare_device_resident2()
+! CHECK: %[[ALLOCA:.*]] = fir.alloca !fir.array<100xf32> {bindc_name = "dataparam", uniq_name = "_QMacc_declareFacc_declare_device_resident2Edataparam"}
+! CHECK: %[[DECL:.*]]:2 = hlfir.declare %[[ALLOCA]](%{{.*}}) {acc.declare = #acc.declare<dataClause =  acc_declare_device_resident>, uniq_name = "_QMacc_declareFacc_declare_device_resident2Edataparam"} : (!fir.ref<!fir.array<100xf32>>, !fir.shape<1>) -> (!fir.ref<!fir.array<100xf32>>, !fir.ref<!fir.array<100xf32>>)
+! CHECK: %[[DEVICERES:.*]] = acc.declare_device_resident varPtr(%[[DECL]]#0 : !fir.ref<!fir.array<100xf32>>)   bounds(%{{.*}}) -> !fir.ref<!fir.array<100xf32>> {name = "dataparam"}
+! CHECK: %[[TOKEN:.*]] = acc.declare_enter dataOperands(%[[DEVICERES]] : !fir.ref<!fir.array<100xf32>>)
+! CHECK: acc.declare_exit token(%[[TOKEN]]) dataOperands(%[[DEVICERES]] : !fir.ref<!fir.array<100xf32>>)
+! CHECK: acc.delete accPtr(%[[DEVICERES]] : !fir.ref<!fir.array<100xf32>>) bounds(%{{.*}}) {dataClause = #acc<data_clause acc_declare_device_resident>, name = "dataparam"}
+
+  subroutine acc_declare_link2()
+    integer, parameter :: n = 100
+    real, dimension(n) :: dataparam
+    !$acc declare link(dataparam)
+  end subroutine
+
+! CHECK-LABEL: func.func @_QMacc_declarePacc_declare_link2()
+! CHECK: %[[ALLOCA:.*]] = fir.alloca !fir.array<100xf32> {bindc_name = "dataparam", uniq_name = "_QMacc_declareFacc_declare_link2Edataparam"}
+! CHECK: %[[DECL:.*]]:2 = hlfir.declare %[[ALLOCA]](%{{.*}}) {acc.declare = #acc.declare<dataClause =  acc_declare_link>, uniq_name = "_QMacc_declareFacc_declare_link2Edataparam"} : (!fir.ref<!fir.array<100xf32>>, !fir.shape<1>) -> (!fir.ref<!fir.array<100xf32>>, !fir.ref<!fir.array<100xf32>>)
+! CHECK: %[[LINK:.*]] = acc.declare_link varPtr(%[[DECL]]#0 : !fir.ref<!fir.array<100xf32>>)   bounds(%{{.*}}) -> !fir.ref<!fir.array<100xf32>> {name = "dataparam"}
+! CHECK: acc.declare_enter dataOperands(%[[LINK]] : !fir.ref<!fir.array<100xf32>>)
+
+  subroutine acc_declare_deviceptr2()
+    integer, parameter :: n = 100
+    real, dimension(n) :: dataparam
+    !$acc declare deviceptr(dataparam)
+  end subroutine
+
+! CHECK-LABEL: func.func @_QMacc_declarePacc_declare_deviceptr2()
+! CHECK: %[[ALLOCA:.*]] = fir.alloca !fir.array<100xf32> {bindc_name = "dataparam", uniq_name = "_QMacc_declareFacc_declare_deviceptr2Edataparam"}
+! CHECK: %[[DECL:.*]]:2 = hlfir.declare %[[ALLOCA]](%{{.*}}) {acc.declare = #acc.declare<dataClause =  acc_deviceptr>, uniq_name = "_QMacc_declareFacc_declare_deviceptr2Edataparam"} : (!fir.ref<!fir.array<100xf32>>, !fir.shape<1>) -> (!fir.ref<!fir.array<100xf32>>, !fir.ref<!fir.array<100xf32>>)
+! CHECK: %[[DEVICEPTR:.*]] = acc.deviceptr varPtr(%[[DECL]]#0 : !fir.ref<!fir.array<100xf32>>)   bounds(%{{.*}}) -> !fir.ref<!fir.array<100xf32>> {name = "dataparam"}
+! CHECK: acc.declare_enter dataOperands(%[[DEVICEPTR]] : !fir.ref<!fir.array<100xf32>>)
+
+  function acc_declare_in_func()
+    real :: a(1024)
+    !$acc declare device_resident(a)
+  end function
+
+! CHECK-LABEL: func.func @_QMacc_declarePacc_declare_in_func() -> f32 {
+! CHECK: %[[DEVICE_RESIDENT:.*]] = acc.declare_device_resident varPtr(%{{.*}}#0 : !fir.ref<!fir.array<1024xf32>>) bounds(%{{.*}}) -> !fir.ref<!fir.array<1024xf32>> {name = "a"}
+! CHECK: %[[TOKEN:.*]] = acc.declare_enter dataOperands(%[[DEVICE_RESIDENT]] : !fir.ref<!fir.array<1024xf32>>)
+! CHECK: %[[LOAD:.*]] = fir.load %{{.*}}#1 : !fir.ref<f32>
+! CHECK: acc.declare_exit token(%[[TOKEN]]) dataOperands(%[[DEVICE_RESIDENT]] : !fir.ref<!fir.array<1024xf32>>)
+! CHECK: acc.delete accPtr(%[[DEVICE_RESIDENT]] : !fir.ref<!fir.array<1024xf32>>) bounds(%6) {dataClause = #acc<data_clause acc_declare_device_resident>, name = "a"}
+! CHECK: return %[[LOAD]] : f32
+! CHECK: }
+
+  function acc_declare_in_func2(i)
+    real :: a(1024)
+    integer :: i
+    !$acc declare create(a)
+    return
+  end function
+
+! CHECK-LABEL: func.func @_QMacc_declarePacc_declare_in_func2(%arg0: !fir.ref<i32> {fir.bindc_name = "i"}) -> f32 {
+! CHECK: %[[ALLOCA_A:.*]] = fir.alloca !fir.array<1024xf32> {bindc_name = "a", uniq_name = "_QMacc_declareFacc_declare_in_func2Ea"}
+! CHECK: %[[DECL_A:.*]]:2 = hlfir.declare %[[ALLOCA_A]](%{{.*}}) {acc.declare = #acc.declare<dataClause =  acc_create>, uniq_name = "_QMacc_declareFacc_declare_in_func2Ea"} : (!fir.ref<!fir.array<1024xf32>>, !fir.shape<1>) -> (!fir.ref<!fir.array<1024xf32>>, !fir.ref<!fir.array<1024xf32>>)
+! CHECK: %[[CREATE:.*]] = acc.create varPtr(%[[DECL_A]]#0 : !fir.ref<!fir.array<1024xf32>>) bounds(%{{[0-9]+}}) -> !fir.ref<!fir.array<1024xf32>> {name = "a"}
+! CHECK: %[[TOKEN:.*]] = acc.declare_enter dataOperands(%[[CREATE]] : !fir.ref<!fir.array<1024xf32>>)
+! CHECK:   cf.br ^bb1
+! CHECK: ^bb1:
+! CHECK: acc.declare_exit token(%[[TOKEN]]) dataOperands(%[[CREATE]] : !fir.ref<!fir.array<1024xf32>>)
+! CHECK: acc.delete accPtr(%[[CREATE]] : !fir.ref<!fir.array<1024xf32>>) bounds(%{{[0-9]+}}) {dataClause = #acc<data_clause acc_create>, name = "a"}
+! CHECK:   return %{{.*}} : f32
+! CHECK: }
+
+  subroutine acc_declare_allocate()
+    integer, allocatable :: a(:)
+    !$acc declare create(a)
+
+    allocate(a(100))
+
+! CHECK: %{{.*}} = fir.allocmem !fir.array<?xi32>, %{{.*}} {fir.must_be_heap = true, uniq_name = "_QMacc_declareFacc_declare_allocateEa.alloc"}
+! CHECK: fir.store %{{.*}} to %{{.*}} {acc.declare_action = #acc.declare_action<postAlloc = @_QMacc_declareFacc_declare_allocateEa_acc_declare_update_desc_post_alloc>} : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>
+
+    deallocate(a)
+
+! CHECK: %{{.*}} = fir.box_addr %{{.*}} {acc.declare_action = #acc.declare_action<preDealloc = @_QMacc_declareFacc_declare_allocateEa_acc_declare_update_desc_pre_dealloc>} : (!fir.box<!fir.heap<!fir.array<?xi32>>>) -> !fir.heap<!fir.array<?xi32>>
+
+! CHECK: fir.freemem %{{.*}} : !fir.heap<!fir.array<?xi32>>
+! CHECK: fir.store %{{.*}} to %{{.*}} {acc.declare_action = #acc.declare_action<postDealloc = @_QMacc_declareFacc_declare_allocateEa_acc_declare_update_desc_post_dealloc>} : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>
+
+! CHECK: fir.if
+! CHECK: fir.freemem %{{.*}} : !fir.heap<!fir.array<?xi32>>
+! CHECK: fir.store %{{.*}} to %{{.*}}#1 {acc.declare_action = #acc.declare_action<postDealloc = @_QMacc_declareFacc_declare_allocateEa_acc_declare_update_desc_post_dealloc>} : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>
+! CHECK: }
+
+  end subroutine
+
+! CHECK-LABEL: func.func private @_QMacc_declareFacc_declare_allocateEa_acc_declare_update_desc_post_alloc(
+! CHECK-SAME:    %[[ARG0:.*]]: !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>) {
+! CHECK:         %[[UPDATE:.*]] = acc.update_device varPtr(%[[ARG0]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>) -> !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>> {implicit = true, name = "a_desc", structured = false}
+! CHECK:         acc.update dataOperands(%[[UPDATE]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>)
+! CHECK:         %[[LOAD:.*]] = fir.load %[[ARG0]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>
+! CHECK:         %[[BOX_ADDR:.*]] = fir.box_addr %[[LOAD]] {acc.declare = #acc.declare<dataClause =  acc_create>} : (!fir.box<!fir.heap<!fir.array<?xi32>>>) -> !fir.heap<!fir.array<?xi32>>
+! CHECK:         %[[CREATE:.*]] = acc.create varPtr(%[[BOX_ADDR]] : !fir.heap<!fir.array<?xi32>>) -> !fir.heap<!fir.array<?xi32>> {name = "a", structured = false}
+! CHECK:         acc.declare_enter dataOperands(%[[CREATE]] : !fir.heap<!fir.array<?xi32>>)
+! CHECK:         return
+! CHECK:       }
+
+! CHECK-LABEL: func.func private @_QMacc_declareFacc_declare_allocateEa_acc_declare_update_desc_pre_dealloc(
+! CHECK-SAME:    %[[ARG0:.*]]: !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>) {
+! CHECK:         %[[LOAD:.*]] = fir.load %[[ARG0]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>
+! CHECK:         %[[BOX_ADDR:.*]] = fir.box_addr %[[LOAD]] {acc.declare = #acc.declare<dataClause =  acc_create>} : (!fir.box<!fir.heap<!fir.array<?xi32>>>) -> !fir.heap<!fir.array<?xi32>>
+! CHECK:         %[[GETDEVICEPTR:.*]] = acc.getdeviceptr varPtr(%[[BOX_ADDR]] : !fir.heap<!fir.array<?xi32>>) -> !fir.heap<!fir.array<?xi32>> {dataClause = #acc<data_clause acc_create>, name = "a", structured = false}
+! CHECK:         acc.declare_exit dataOperands(%[[GETDEVICEPTR]] : !fir.heap<!fir.array<?xi32>>)
+! CHECK:         acc.delete accPtr(%[[GETDEVICEPTR]] : !fir.heap<!fir.array<?xi32>>) {dataClause = #acc<data_clause acc_create>, name = "a", structured = false}
+! CHECK:         return
+! CHECK:       }
+
+! CHECK-LABEL: func.func private @_QMacc_declareFacc_declare_allocateEa_acc_declare_update_desc_post_dealloc(
+! CHECK-SAME:    %[[ARG0:.*]]: !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>) {
+! CHECK:         %[[LOAD:.*]] = fir.load %[[ARG0]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>
+! CHECK:         %[[BOX_ADDR:.*]] = fir.box_addr %[[LOAD]] : (!fir.box<!fir.heap<!fir.array<?xi32>>>) -> !fir.heap<!fir.array<?xi32>>
+! CHECK:         %[[UPDATE:.*]] = acc.update_device varPtr(%[[BOX_ADDR]] : !fir.heap<!fir.array<?xi32>>) -> !fir.heap<!fir.array<?xi32>> {implicit = true, name = "a_desc", structured = false}
+! CHECK:         acc.update dataOperands(%[[UPDATE]] : !fir.heap<!fir.array<?xi32>>)
+! CHECK:         return
+! CHECK:       }
+
+  subroutine acc_declare_multiple_directive(a, b)
+    integer :: a(100), b(100), i
+    !$acc declare copy(a)
+    !$acc declare copyout(b)
+
+    do i = 1, 100
+      a(i) = i
+    end do
+  end subroutine
+
+! CHECK-LABEL: func.func @_QMacc_declarePacc_declare_multiple_directive(
+! CHECK-SAME: %[[ARG0:.*]]: !fir.ref<!fir.array<100xi32>> {fir.bindc_name = "a"}, %[[ARG1:.*]]: !fir.ref<!fir.array<100xi32>> {fir.bindc_name = "b"}) {
+! CHECK: %[[DECL_A:.*]]:2 = hlfir.declare %[[ARG0]](%{{.*}}) dummy_scope %{{[0-9]+}} {acc.declare = #acc.declare<dataClause =  acc_copy>, uniq_name = "_QMacc_declareFacc_declare_multiple_directiveEa"} : (!fir.ref<!fir.array<100xi32>>, !fir.shape<1>, !fir.dscope) -> (!fir.ref<!fir.array<100xi32>>, !fir.ref<!fir.array<100xi32>>)
+! CHECK: %[[DECL_B:.*]]:2 = hlfir.declare %[[ARG1]](%{{.*}}) dummy_scope %{{[0-9]+}} {acc.declare = #acc.declare<dataClause =  acc_copyout>, uniq_name = "_QMacc_declareFacc_declare_multiple_directiveEb"} : (!fir.ref<!fir.array<100xi32>>, !fir.shape<1>, !fir.dscope) -> (!fir.ref<!fir.array<100xi32>>, !fir.ref<!fir.array<100xi32>>)
+! CHECK: %[[COPYIN:.*]] = acc.copyin varPtr(%[[DECL_A]]#0 : !fir.ref<!fir.array<100xi32>>) bounds(%{{.*}}) -> !fir.ref<!fir.array<100xi32>> {dataClause = #acc<data_clause acc_copy>, name = "a"}
+! CHECK: %[[CREATE:.*]] = acc.create varPtr(%[[DECL_B]]#0 : !fir.ref<!fir.array<100xi32>>) bounds(%{{.*}}) -> !fir.ref<!fir.array<100xi32>> {dataClause = #acc<data_clause acc_copyout>, name = "b"}
+! CHECK: acc.declare_enter dataOperands(%[[COPYIN]], %[[CREATE]] : !fir.ref<!fir.array<100xi32>>, !fir.ref<!fir.array<100xi32>>)
+! CHECK: %{{.*}}:{{.*}} = fir.do_loop %{{.*}} = %{{.*}} to %{{.*}} step %{{.*}} iter_args(%{{.*}} = %{{.*}}) -> (index, i32) {
+
+
+! CHECK: acc.copyout accPtr(%[[CREATE]] : !fir.ref<!fir.array<100xi32>>) bounds(%{{.*}}) to varPtr(%[[DECL_B]]#0 : !fir.ref<!fir.array<100xi32>>) {name = "b"}
+! CHECK: acc.copyout accPtr(%[[COPYIN]] : !fir.ref<!fir.array<100xi32>>) bounds(%{{.*}}) to varPtr(%[[DECL_A]]#0 : !fir.ref<!fir.array<100xi32>>) {dataClause = #acc<data_clause acc_copy>, name = "a"}
+
+  subroutine acc_declare_array_section(a)
+    integer :: a(:)
+    !$acc declare copy(a(1:10))
+
+    do i = 1, 100
+      a(i) = i
+    end do
+  end subroutine
+
+! CHECK-LABEL: func.func @_QMacc_declarePacc_declare_array_section(
+! CHECK-SAME:    %[[ARG0:.*]]: !fir.box<!fir.array<?xi32>> {fir.bindc_name = "a"}) {
+! CHECK: %[[DECL_A:.*]]:2 = hlfir.declare %[[ARG0]] dummy_scope %{{[0-9]+}} {uniq_name = "_QMacc_declareFacc_declare_array_sectionEa"} : (!fir.box<!fir.array<?xi32>>, !fir.dscope) -> (!fir.box<!fir.array<?xi32>>, !fir.box<!fir.array<?xi32>>)
+! CHECK: %[[BOX_ADDR:.*]] = fir.box_addr %[[DECL_A]]#0 {acc.declare = #acc.declare<dataClause =  acc_copy>} : (!fir.box<!fir.array<?xi32>>) -> !fir.ref<!fir.array<?xi32>>
+! CHECK: %[[COPYIN:.*]] = acc.copyin varPtr(%[[BOX_ADDR]] : !fir.ref<!fir.array<?xi32>>) bounds(%{{.*}}) -> !fir.ref<!fir.array<?xi32>> {dataClause = #acc<data_clause acc_copy>, name = "a(1:10)"}
+! CHECK: acc.declare_enter dataOperands(%[[COPYIN]] : !fir.ref<!fir.array<?xi32>>)
+
+! CHECK: acc.copyout accPtr(%[[COPYIN]] : !fir.ref<!fir.array<?xi32>>) bounds(%{{.*}}) to varPtr(%[[BOX_ADDR]] : !fir.ref<!fir.array<?xi32>>) {dataClause = #acc<data_clause acc_copy>, name = "a(1:10)"}
+
+  subroutine acc_declare_allocate_with_stat()
+    integer :: status
+    real, pointer, dimension(:) :: localptr
+    !$acc declare create(localptr)
+    allocate(localptr(n), stat=status)
+
+    deallocate(localptr, stat=status)
+  end subroutine
+
+! CHECK-LABEL: func.func @_QMacc_declarePacc_declare_allocate_with_stat()
+! CHECK: fir.call @_FortranAPointerAllocate(%{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}) {{.*}} {acc.declare_action = #acc.declare_action<postAlloc = @_QMacc_declareFacc_declare_allocate_with_statElocalptr_acc_declare_update_desc_post_alloc>}
+! CHECK: fir.call @_FortranAPointerDeallocate(%{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}) {{.*}} {acc.declare_action = #acc.declare_action<preDealloc = @_QMacc_declareFacc_declare_allocate_with_statElocalptr_acc_declare_update_desc_pre_dealloc, postDealloc = @_QMacc_declareFacc_declare_allocate_with_statElocalptr_acc_declare_update_desc_post_dealloc>}
+end module
+
+module acc_declare_allocatable_test
+ integer, allocatable :: data1(:)
+ !$acc declare create(data1)
+end module
+
+! CHECK-LABEL: acc.global_ctor @_QMacc_declare_allocatable_testEdata1_acc_ctor {
+! CHECK:         %[[GLOBAL_ADDR:.*]] = fir.address_of(@_QMacc_declare_allocatable_testEdata1) {acc.declare = #acc.declare<dataClause =  acc_create>} : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>
+! CHECK:         %[[COPYIN:.*]] = acc.copyin varPtr(%[[GLOBAL_ADDR]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>)   -> !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>> {dataClause = #acc<data_clause acc_create>, implicit = true, name = "data1", structured = false}
+! CHECK:         acc.declare_enter dataOperands(%[[COPYIN]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>)
+! CHECK:         acc.terminator
+! CHECK:       }
+
+! CHECK-LABEL: func.func private @_QMacc_declare_allocatable_testEdata1_acc_declare_update_desc_post_alloc() {
+! CHECK:         %[[GLOBAL_ADDR:.*]] = fir.address_of(@_QMacc_declare_allocatable_testEdata1) : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>
+! CHECK:         %[[UPDATE:.*]] = acc.update_device varPtr(%[[GLOBAL_ADDR]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>) -> !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>> {implicit = true, name = "data1_desc", structured = false}
+! CHECK:         acc.update dataOperands(%[[UPDATE]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>)
+! CHECK:         %[[LOAD:.*]] = fir.load %[[GLOBAL_ADDR]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>
+! CHECK:         %[[BOXADDR:.*]] = fir.box_addr %[[LOAD]] {acc.declare = #acc.declare<dataClause =  acc_create>} : (!fir.box<!fir.heap<!fir.array<?xi32>>>) -> !fir.heap<!fir.array<?xi32>>
+! CHECK:         %[[CREATE:.*]] = acc.create varPtr(%[[BOXADDR]] : !fir.heap<!fir.array<?xi32>>) -> !fir.heap<!fir.array<?xi32>> {name = "data1", structured = false}
+! CHECK:         acc.declare_enter dataOperands(%[[CREATE]] : !fir.heap<!fir.array<?xi32>>)
+! CHECK:         return
+! CHECK:       }
+
+! CHECK-LABEL: func.func private @_QMacc_declare_allocatable_testEdata1_acc_declare_update_desc_pre_dealloc() {
+! CHECK:         %[[GLOBAL_ADDR:.*]] = fir.address_of(@_QMacc_declare_allocatable_testEdata1) : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>
+! CHECK:         %[[LOAD:.*]] = fir.load %[[GLOBAL_ADDR]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>
+! CHECK:         %[[BOXADDR:.*]] = fir.box_addr %[[LOAD]] {acc.declare = #acc.declare<dataClause =  acc_create>} : (!fir.box<!fir.heap<!fir.array<?xi32>>>) -> !fir.heap<!fir.array<?xi32>>
+! CHECK:         %[[DEVPTR:.*]] = acc.getdeviceptr varPtr(%[[BOXADDR]] : !fir.heap<!fir.array<?xi32>>)   -> !fir.heap<!fir.array<?xi32>> {dataClause = #acc<data_clause acc_create>, name = "data1", structured = false}
+! CHECK:         acc.declare_exit dataOperands(%[[DEVPTR]] : !fir.heap<!fir.array<?xi32>>)
+! CHECK:         acc.delete accPtr(%[[DEVPTR]] : !fir.heap<!fir.array<?xi32>>) {dataClause = #acc<data_clause acc_create>, name = "data1", structured = false}
+! CHECK:         return
+! CHECK:       }
+
+! CHECK-LABEL: func.func private @_QMacc_declare_allocatable_testEdata1_acc_declare_update_desc_post_dealloc() {
+! CHECK:         %[[GLOBAL_ADDR:.*]] = fir.address_of(@_QMacc_declare_allocatable_testEdata1) : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>
+! CHECK:         %[[UPDATE:.*]] = acc.update_device varPtr(%[[GLOBAL_ADDR]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>)   -> !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>> {implicit = true, name = "data1_desc", structured = false}
+! CHECK:         acc.update dataOperands(%[[UPDATE]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>)
+! CHECK:         return
+! CHECK:       }
+
+! CHECK-LABEL: acc.global_dtor @_QMacc_declare_allocatable_testEdata1_acc_dtor {
+! CHECK:         %[[GLOBAL_ADDR:.*]] = fir.address_of(@_QMacc_declare_allocatable_testEdata1) {acc.declare = #acc.declare<dataClause =  acc_create>} : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>
+! CHECK:         %[[DEVICEPTR:.*]] = acc.getdeviceptr varPtr(%[[GLOBAL_ADDR]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>)   -> !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>> {dataClause = #acc<data_clause acc_create>, name = "data1", structured = false}
+! CHECK:         acc.declare_exit dataOperands(%[[DEVICEPTR]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>)
+! CHECK:         acc.delete accPtr(%[[DEVICEPTR]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>) {dataClause = #acc<data_clause acc_create>, name = "data1", structured = false}
+! CHECK:         acc.terminator
+! CHECK:       }
+
+
+module acc_declare_equivalent
+  integer, parameter :: n = 10
+  real :: v1(n)
+  real :: v2(n)
+  equivalence(v1(1), v2(1))
+  !$acc declare create(v2)
+end module
+
+! CHECK-LABEL: acc.global_ctor @_QMacc_declare_equivalentEv2_acc_ctor {
+! CHECK:         %[[ADDR:.*]] = fir.address_of(@_QMacc_declare_equivalentEv1) {acc.declare = #acc.declare<dataClause = acc_create>} : !fir.ref<!fir.array<40xi8>>
+! CHECK:         %[[CREATE:.*]] = acc.create varPtr(%[[ADDR]] : !fir.ref<!fir.array<40xi8>>) -> !fir.ref<!fir.array<40xi8>> {name = "v2", structured = false}
+! CHECK:         acc.declare_enter dataOperands(%[[CREATE]] : !fir.ref<!fir.array<40xi8>>)
+! CHECK:         acc.terminator
+! CHECK:       }
+! CHECK-LABEL: acc.global_dtor @_QMacc_declare_equivalentEv2_acc_dtor {
+! CHECK:         %[[ADDR:.*]] = fir.address_of(@_QMacc_declare_equivalentEv1) {acc.declare = #acc.declare<dataClause =  acc_create>} : !fir.ref<!fir.array<40xi8>>
+! CHECK:         %[[DEVICEPTR:.*]] = acc.getdeviceptr varPtr(%[[ADDR]] : !fir.ref<!fir.array<40xi8>>) -> !fir.ref<!fir.array<40xi8>> {dataClause = #acc<data_clause acc_create>, name = "v2", structured = false}
+! CHECK:         acc.declare_exit dataOperands(%[[DEVICEPTR]] : !fir.ref<!fir.array<40xi8>>)
+! CHECK:         acc.delete accPtr(%[[DEVICEPTR]] : !fir.ref<!fir.array<40xi8>>) {dataClause = #acc<data_clause acc_create>, name = "v2", structured = false}
+! CHECK:         acc.terminator
+! CHECK:       }
+
+module acc_declare_equivalent2
+  real :: v1(10)
+  real :: v2(5)
+  equivalence(v1(6), v2(1))
+  !$acc declare create(v2)
+end module
+
+! CHECK-LABEL: acc.global_ctor @_QMacc_declare_equivalent2Ev2_acc_ctor {
+! CHECK:         %[[ADDR:.*]] = fir.address_of(@_QMacc_declare_equivalent2Ev1) {acc.declare = #acc.declare<dataClause =  acc_create>} : !fir.ref<!fir.array<40xi8>>
+! CHECK:         %[[CREATE:.*]] = acc.create varPtr(%[[ADDR]] : !fir.ref<!fir.array<40xi8>>) -> !fir.ref<!fir.array<40xi8>> {name = "v2", structured = false}
+! CHECK:         acc.declare_enter dataOperands(%[[CREATE]] : !fir.ref<!fir.array<40xi8>>)
+! CHECK:         acc.terminator
+! CHECK:       }
+! CHECK-LABEL: acc.global_dtor @_QMacc_declare_equivalent2Ev2_acc_dtor {
+! CHECK:         %[[ADDR:.*]] = fir.address_of(@_QMacc_declare_equivalent2Ev1) {acc.declare = #acc.declare<dataClause =  acc_create>} : !fir.ref<!fir.array<40xi8>>
+! CHECK:         %[[DEVICEPTR:.*]] = acc.getdeviceptr varPtr(%[[ADDR]] : !fir.ref<!fir.array<40xi8>>) -> !fir.ref<!fir.array<40xi8>> {dataClause = #acc<data_clause acc_create>, name = "v2", structured = false}
+! CHECK:         acc.declare_exit dataOperands(%[[DEVICEPTR]] : !fir.ref<!fir.array<40xi8>>)
+! CHECK:         acc.delete accPtr(%[[DEVICEPTR]] : !fir.ref<!fir.array<40xi8>>) {dataClause = #acc<data_clause acc_create>, name = "v2", structured = false}
+! CHECK:         acc.terminator
+! CHECK:       }
+
+! Test that the pre/post alloc/dealloc attributes are set when the
+! allocate/deallocate statement are in a different module.
+module acc_declare_allocatable_test2
+contains
+  subroutine init()
+    use acc_declare_allocatable_test
+    allocate(data1(100))
+! CHECK: fir.store %{{.*}} to %{{.*}} {acc.declare_action = #acc.declare_action<postAlloc = @_QMacc_declare_allocatable_testEdata1_acc_declare_update_desc_post_alloc>} : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>
+  end subroutine
+
+  subroutine finalize()
+    use acc_declare_allocatable_test
+    deallocate(data1)
+! CHECK: %{{.*}} = fir.box_addr %{{.*}} {acc.declare_action = #acc.declare_action<preDealloc = @_QMacc_declare_allocatable_testEdata1_acc_declare_update_desc_pre_dealloc>} : (!fir.box<!fir.heap<!fir.array<?xi32>>>) -> !fir.heap<!fir.array<?xi32>>
+! CHECK: fir.store %{{.*}} to %{{.*}} {acc.declare_action = #acc.declare_action<postDealloc = @_QMacc_declare_allocatable_testEdata1_acc_declare_update_desc_post_dealloc>} : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>
+  end subroutine
+end module
+
+module acc_declare_allocatable_test3
+  integer, allocatable :: data1(:)
+  integer, allocatable :: data2(:)
+  !$acc declare create(data1, data2, data1)
+end module
+
+! CHECK-LABEL: acc.global_ctor @_QMacc_declare_allocatable_test3Edata1_acc_ctor
+! CHECK-LABEL: acc.global_ctor @_QMacc_declare_allocatable_test3Edata2_acc_ctor
+
+module acc_declare_post_action_stat
+  real, dimension(:), allocatable :: x, y
+  !$acc declare create(x,y)
+
+contains
+
+  subroutine init()
+    integer :: stat
+    allocate(x(10), y(10), stat=stat)
+  end subroutine
+end module
+
+! CHECK-LABEL: func.func @_QMacc_declare_post_action_statPinit()
+! CHECK: fir.call @_FortranAAllocatableAllocate({{.*}}) fastmath<contract> {acc.declare_action = #acc.declare_action<postAlloc = @_QMacc_declare_post_action_statEx_acc_declare_update_desc_post_alloc>} : (!fir.ref<!fir.box<none>>, i1, !fir.box<none>, !fir.ref<i8>, i32) -> i32
+! CHECK: fir.if
+! CHECK: fir.call @_FortranAAllocatableAllocate({{.*}}) fastmath<contract> {acc.declare_action = #acc.declare_action<postAlloc = @_QMacc_declare_post_action_statEy_acc_declare_update_desc_post_alloc>} : (!fir.ref<!fir.box<none>>, i1, !fir.box<none>, !fir.ref<i8>, i32) -> i32
diff --git a/flang/test/Lower/OpenACC/acc-declare.f90 b/flang/test/Lower/OpenACC/acc-declare.f90
index f40216e2c9fe8..8727c0722e474 100644
--- a/flang/test/Lower/OpenACC/acc-declare.f90
+++ b/flang/test/Lower/OpenACC/acc-declare.f90
@@ -16,16 +16,14 @@ subroutine acc_declare_copy()
   end subroutine
 
 ! CHECK-LABEL: func.func @_QMacc_declarePacc_declare_copy()
-! CHECK-DAG: %[[C1:.*]] = arith.constant 1 : index
-! CHECK-DAG: %[[ALLOCA:.*]] = fir.alloca !fir.array<100xi32> {bindc_name = "a", uniq_name = "_QMacc_declareFacc_declare_copyEa"}
-! CHECK-DAG: %[[DECL:.*]]:2 = hlfir.declare %[[ALLOCA]](%{{.*}}) {acc.declare = #acc.declare<dataClause =  acc_copy>, uniq_name = "_QMacc_declareFacc_declare_copyEa"} : (!fir.ref<!fir.array<100xi32>>, !fir.shape<1>) -> (!fir.ref<!fir.array<100xi32>>, !fir.ref<!fir.array<100xi32>>)
-! CHECK: %[[BOUND:.*]] = acc.bounds   lowerbound(%{{.*}} : index) upperbound(%{{.*}} : index) extent(%{{.*}} : index) stride(%[[C1]] : index) startIdx(%[[C1]] : index)
-! CHECK: %[[COPYIN:.*]] = acc.copyin varPtr(%[[DECL]]#0 : !fir.ref<!fir.array<100xi32>>) bounds(%[[BOUND]]) -> !fir.ref<!fir.array<100xi32>> {dataClause = #acc<data_clause acc_copy>, name = "a"}
+! CHECK: %[[ALLOCA:.*]] = fir.alloca !fir.array<100xi32> {bindc_name = "a", uniq_name = "_QMacc_declareFacc_declare_copyEa"}
+! CHECK: %[[DECL:.*]]:2 = hlfir.declare %[[ALLOCA]](%{{.*}}) {acc.declare = #acc.declare<dataClause =  acc_copy>, uniq_name = "_QMacc_declareFacc_declare_copyEa"} : (!fir.ref<!fir.array<100xi32>>, !fir.shape<1>) -> (!fir.ref<!fir.array<100xi32>>, !fir.ref<!fir.array<100xi32>>)
+! CHECK: %[[COPYIN:.*]] = acc.copyin varPtr(%[[DECL]]#0 : !fir.ref<!fir.array<100xi32>>) -> !fir.ref<!fir.array<100xi32>> {dataClause = #acc<data_clause acc_copy>, name = "a"}
 ! CHECK: %[[TOKEN:.*]] = acc.declare_enter dataOperands(%[[COPYIN]] : !fir.ref<!fir.array<100xi32>>)
 ! CHECK: %{{.*}}:2 = fir.do_loop %{{.*}} = %{{.*}} to %{{.*}} step %{{.*}} iter_args(%{{.*}} = %{{.*}}) -> (index, i32) {
 ! CHECK: }
 ! CHECK: acc.declare_exit token(%[[TOKEN]]) dataOperands(%[[COPYIN]] : !fir.ref<!fir.array<100xi32>>)
-! CHECK: acc.copyout accPtr(%[[COPYIN]] : !fir.ref<!fir.array<100xi32>>) bounds(%[[BOUND]]) to varPtr(%[[DECL]]#0 : !fir.ref<!fir.array<100xi32>>) {dataClause = #acc<data_clause acc_copy>, name = "a"}
+! CHECK: acc.copyout accPtr(%[[COPYIN]] : !fir.ref<!fir.array<100xi32>>) to varPtr(%[[DECL]]#0 : !fir.ref<!fir.array<100xi32>>) {dataClause = #acc<data_clause acc_copy>, name = "a"}
 ! CHECK: return
 
   subroutine acc_declare_create()
@@ -38,16 +36,14 @@ subroutine acc_declare_create()
   end subroutine
 
 ! CHECK-LABEL: func.func @_QMacc_declarePacc_declare_create() {
-! CHECK-DAG: %[[C1:.*]] = arith.constant 1 : index
-! CHECK-DAG: %[[ALLOCA:.*]] = fir.alloca !fir.array<100xi32> {bindc_name = "a", uniq_name = "_QMacc_declareFacc_declare_createEa"}
-! CHECK-DAG: %[[DECL:.*]]:2 = hlfir.declare %[[ALLOCA]](%{{.*}}) {acc.declare = #acc.declare<dataClause =  acc_create>, uniq_name = "_QMacc_declareFacc_declare_createEa"} : (!fir.ref<!fir.array<100xi32>>, !fir.shape<1>) -> (!fir.ref<!fir.array<100xi32>>, !fir.ref<!fir.array<100xi32>>)
-! CHECK: %[[BOUND:.*]] = acc.bounds   lowerbound(%{{.*}} : index) upperbound(%{{.*}} : index) extent(%{{.*}} : index) stride(%[[C1]] : index) startIdx(%[[C1]] : index)
-! CHECK: %[[CREATE:.*]] = acc.create varPtr(%[[DECL]]#0 : !fir.ref<!fir.array<100xi32>>)   bounds(%[[BOUND]]) -> !fir.ref<!fir.array<100xi32>> {name = "a"}
+! CHECK: %[[ALLOCA:.*]] = fir.alloca !fir.array<100xi32> {bindc_name = "a", uniq_name = "_QMacc_declareFacc_declare_createEa"}
+! CHECK: %[[DECL:.*]]:2 = hlfir.declare %[[ALLOCA]](%{{.*}}) {acc.declare = #acc.declare<dataClause =  acc_create>, uniq_name = "_QMacc_declareFacc_declare_createEa"} : (!fir.ref<!fir.array<100xi32>>, !fir.shape<1>) -> (!fir.ref<!fir.array<100xi32>>, !fir.ref<!fir.array<100xi32>>)
+! CHECK: %[[CREATE:.*]] = acc.create varPtr(%[[DECL]]#0 : !fir.ref<!fir.array<100xi32>>) -> !fir.ref<!fir.array<100xi32>> {name = "a"}
 ! CHECK: %[[TOKEN:.*]] = acc.declare_enter dataOperands(%[[CREATE]] : !fir.ref<!fir.array<100xi32>>)
 ! CHECK: %{{.*}}:2 = fir.do_loop %{{.*}} = %{{.*}} to %{{.*}} step %{{.*}} iter_args(%{{.*}} = %{{.*}}) -> (index, i32) {
 ! CHECK: }
 ! CHECK: acc.declare_exit token(%[[TOKEN]]) dataOperands(%[[CREATE]] : !fir.ref<!fir.array<100xi32>>)
-! CHECK: acc.delete accPtr(%[[CREATE]] : !fir.ref<!fir.array<100xi32>>) bounds(%[[BOUND]]) {dataClause = #acc<data_clause acc_create>, name = "a"}
+! CHECK: acc.delete accPtr(%[[CREATE]] : !fir.ref<!fir.array<100xi32>>) {dataClause = #acc<data_clause acc_create>, name = "a"}
 ! CHECK: return
 
   subroutine acc_declare_present(a)
@@ -61,12 +57,12 @@ subroutine acc_declare_present(a)
 
 ! CHECK-LABEL: func.func @_QMacc_declarePacc_declare_present(
 ! CHECK-SAME: %[[ARG0:.*]]: !fir.ref<!fir.array<100xi32>> {fir.bindc_name = "a"})
-! CHECK-DAG: %[[C1:.*]] = arith.constant 1 : index
-! CHECK-DAG: %[[DECL:.*]]:2 = hlfir.declare %[[ARG0]](%{{.*}}) dummy_scope %{{[0-9]+}} {acc.declare = #acc.declare<dataClause =  acc_present>, uniq_name = "_QMacc_declareFacc_declare_presentEa"} : (!fir.ref<!fir.array<100xi32>>, !fir.shape<1>, !fir.dscope) -> (!fir.ref<!fir.array<100xi32>>, !fir.ref<!fir.array<100xi32>>)
-! CHECK: %[[BOUND:.*]] = acc.bounds lowerbound(%{{.*}} : index) upperbound(%{{.*}} : index) extent(%{{.*}} : index) stride(%{{.*}} : index) startIdx(%[[C1]] : index)
-! CHECK: %[[PRESENT:.*]] = acc.present varPtr(%[[DECL]]#0 : !fir.ref<!fir.array<100xi32>>)   bounds(%[[BOUND]]) -> !fir.ref<!fir.array<100xi32>> {name = "a"}
-! CHECK: acc.declare_enter dataOperands(%[[PRESENT]] : !fir.ref<!fir.array<100xi32>>)
+! CHECK: %[[DECL:.*]]:2 = hlfir.declare %[[ARG0]](%{{.*}}) dummy_scope %{{[0-9]+}} {acc.declare = #acc.declare<dataClause =  acc_present>, uniq_name = "_QMacc_declareFacc_declare_presentEa"} : (!fir.ref<!fir.array<100xi32>>, !fir.shape<1>, !fir.dscope) -> (!fir.ref<!fir.array<100xi32>>, !fir.ref<!fir.array<100xi32>>)
+! CHECK: %[[PRESENT:.*]] = acc.present varPtr(%[[DECL]]#0 : !fir.ref<!fir.array<100xi32>>) -> !fir.ref<!fir.array<100xi32>> {name = "a"}
+! CHECK: %[[TOKEN:.*]] = acc.declare_enter dataOperands(%[[PRESENT]] : !fir.ref<!fir.array<100xi32>>)
 ! CHECK: %{{.*}}:2 = fir.do_loop %{{.*}} = %{{.*}} to %{{.*}} step %{{.*}} iter_args(%arg{{.*}} = %{{.*}}) -> (index, i32)
+! CHECK: acc.declare_exit token(%[[TOKEN]]) dataOperands(%[[PRESENT]] : !fir.ref<!fir.array<100xi32>>)
+! CHECK: acc.delete accPtr(%[[PRESENT]] : !fir.ref<!fir.array<100xi32>>) {dataClause = #acc<data_clause acc_present>, name = "a"}
 
   subroutine acc_declare_copyin()
     integer :: a(100), b(10), i
@@ -82,14 +78,12 @@ subroutine acc_declare_copyin()
 ! CHECK: %[[ADECL:.*]]:2 = hlfir.declare %[[A]](%{{.*}}) {acc.declare = #acc.declare<dataClause =  acc_copyin>, uniq_name = "_QMacc_declareFacc_declare_copyinEa"} : (!fir.ref<!fir.array<100xi32>>, !fir.shape<1>) -> (!fir.ref<!fir.array<100xi32>>, !fir.ref<!fir.array<100xi32>>)
 ! CHECK: %[[B:.*]] = fir.alloca !fir.array<10xi32> {bindc_name = "b", uniq_name = "_QMacc_declareFacc_declare_copyinEb"}
 ! CHECK: %[[BDECL:.*]]:2 = hlfir.declare %[[B]](%{{.*}}) {acc.declare = #acc.declare<dataClause =  acc_copyin_readonly>, uniq_name = "_QMacc_declareFacc_declare_copyinEb"} : (!fir.ref<!fir.array<10xi32>>, !fir.shape<1>) -> (!fir.ref<!fir.array<10xi32>>, !fir.ref<!fir.array<10xi32>>)
-! CHECK: %[[BOUND_A:.*]] = acc.bounds lowerbound(%{{.*}} : index) upperbound(%{{.*}} : index) extent(%{{.*}} : index) stride(%{{.*}} : index) startIdx(%{{.*}} : index)
-! CHECK: %[[COPYIN_A:.*]] = acc.copyin varPtr(%[[ADECL]]#0 : !fir.ref<!fir.array<100xi32>>)   bounds(%[[BOUND_A]]) -> !fir.ref<!fir.array<100xi32>> {name = "a"}
-! CHECK: %[[BOUND_B:.*]] = acc.bounds lowerbound(%{{.*}} : index) upperbound(%{{.*}} : index) extent(%{{.*}} : index) stride(%{{.*}} : index) startIdx(%{{.*}} : index)
-! CHECK: %[[COPYIN_B:.*]] = acc.copyin varPtr(%[[BDECL]]#0 : !fir.ref<!fir.array<10xi32>>)   bounds(%[[BOUND_B]]) -> !fir.ref<!fir.array<10xi32>> {dataClause = #acc<data_clause acc_copyin_readonly>, name = "b"}
+! CHECK: %[[COPYIN_A:.*]] = acc.copyin varPtr(%[[ADECL]]#0 : !fir.ref<!fir.array<100xi32>>) -> !fir.ref<!fir.array<100xi32>> {name = "a"}
+! CHECK: %[[COPYIN_B:.*]] = acc.copyin varPtr(%[[BDECL]]#0 : !fir.ref<!fir.array<10xi32>>) -> !fir.ref<!fir.array<10xi32>> {dataClause = #acc<data_clause acc_copyin_readonly>, name = "b"}
 ! CHECK: acc.declare_enter dataOperands(%[[COPYIN_A]], %[[COPYIN_B]] : !fir.ref<!fir.array<100xi32>>, !fir.ref<!fir.array<10xi32>>)
 ! CHECK: %{{.*}}:2 = fir.do_loop %{{.*}} = %{{.*}} to %{{.*}} step %{{.*}} iter_args(%arg{{.*}} = %{{.*}}) -> (index, i32)
-! CHECK: acc.delete accPtr(%[[COPYIN_A]] : !fir.ref<!fir.array<100xi32>>)   bounds(%[[BOUND_A]]) {dataClause = #acc<data_clause acc_copyin>, name = "a"}
-! CHECK: acc.delete accPtr(%[[COPYIN_B]] : !fir.ref<!fir.array<10xi32>>)   bounds(%[[BOUND_B]]) {dataClause = #acc<data_clause acc_copyin_readonly>, name = "b"}
+! CHECK: acc.delete accPtr(%[[COPYIN_A]] : !fir.ref<!fir.array<100xi32>>) {dataClause = #acc<data_clause acc_copyin>, name = "a"}
+! CHECK: acc.delete accPtr(%[[COPYIN_B]] : !fir.ref<!fir.array<10xi32>>) {dataClause = #acc<data_clause acc_copyin_readonly>, name = "b"}
 
   subroutine acc_declare_copyout()
     integer :: a(100), i
@@ -103,11 +97,11 @@ subroutine acc_declare_copyout()
 ! CHECK-LABEL: func.func @_QMacc_declarePacc_declare_copyout()
 ! CHECK: %[[A:.*]] = fir.alloca !fir.array<100xi32> {bindc_name = "a", uniq_name = "_QMacc_declareFacc_declare_copyoutEa"}
 ! CHECK: %[[ADECL:.*]]:2 = hlfir.declare %[[A]](%{{.*}}) {acc.declare = #acc.declare<dataClause =  acc_copyout>, uniq_name = "_QMacc_declareFacc_declare_copyoutEa"} : (!fir.ref<!fir.array<100xi32>>, !fir.shape<1>) -> (!fir.ref<!fir.array<100xi32>>, !fir.ref<!fir.array<100xi32>>)
-! CHECK: %[[CREATE:.*]] = acc.create varPtr(%[[ADECL]]#0 : !fir.ref<!fir.array<100xi32>>)   bounds(%{{.*}}) -> !fir.ref<!fir.array<100xi32>> {dataClause = #acc<data_clause acc_copyout>, name = "a"}
+! CHECK: %[[CREATE:.*]] = acc.create varPtr(%[[ADECL]]#0 : !fir.ref<!fir.array<100xi32>>) -> !fir.ref<!fir.array<100xi32>> {dataClause = #acc<data_clause acc_copyout>, name = "a"}
 ! CHECK: %[[TOKEN:.*]] = acc.declare_enter dataOperands(%[[CREATE]] : !fir.ref<!fir.array<100xi32>>)
 ! CHECK: %{{.*}}:2 = fir.do_loop %{{.*}} = %{{.*}} to %{{.*}} step %{{.*}} iter_args(%arg{{.*}} = %{{.*}}) -> (index, i32)
 ! CHECK: acc.declare_exit token(%[[TOKEN]]) dataOperands(%[[CREATE]] : !fir.ref<!fir.array<100xi32>>)
-! CHECK: acc.copyout accPtr(%[[CREATE]] : !fir.ref<!fir.array<100xi32>>)   bounds(%{{.*}}) to varPtr(%[[ADECL]]#0 : !fir.ref<!fir.array<100xi32>>) {name = "a"}
+! CHECK: acc.copyout accPtr(%[[CREATE]] : !fir.ref<!fir.array<100xi32>>) to varPtr(%[[ADECL]]#0 : !fir.ref<!fir.array<100xi32>>) {name = "a"}
 ! CHECK: return
 
   subroutine acc_declare_deviceptr(a)
@@ -122,7 +116,7 @@ subroutine acc_declare_deviceptr(a)
 ! CHECK-LABEL: func.func @_QMacc_declarePacc_declare_deviceptr(
 ! CHECK-SAME: %[[ARG0:.*]]: !fir.ref<!fir.array<100xi32>> {fir.bindc_name = "a"}) {
 ! CHECK: %[[DECL:.*]]:2 = hlfir.declare %[[ARG0]](%{{.*}}) dummy_scope %{{[0-9]+}} {acc.declare = #acc.declare<dataClause =  acc_deviceptr>, uniq_name = "_QMacc_declareFacc_declare_deviceptrEa"} : (!fir.ref<!fir.array<100xi32>>, !fir.shape<1>, !fir.dscope) -> (!fir.ref<!fir.array<100xi32>>, !fir.ref<!fir.array<100xi32>>)
-! CHECK: %[[DEVICEPTR:.*]] = acc.deviceptr varPtr(%[[DECL]]#0 : !fir.ref<!fir.array<100xi32>>)   bounds(%{{.*}}) -> !fir.ref<!fir.array<100xi32>> {name = "a"}
+! CHECK: %[[DEVICEPTR:.*]] = acc.deviceptr varPtr(%[[DECL]]#0 : !fir.ref<!fir.array<100xi32>>) -> !fir.ref<!fir.array<100xi32>> {name = "a"}
 ! CHECK: acc.declare_enter dataOperands(%[[DEVICEPTR]] : !fir.ref<!fir.array<100xi32>>)
 ! CHECK: %{{.*}}:2 = fir.do_loop %{{.*}} = %{{.*}} to %{{.*}} step %{{.*}} iter_args(%arg{{.*}} = %{{.*}}) -> (index, i32)
 
@@ -138,7 +132,7 @@ subroutine acc_declare_link(a)
 ! CHECK-LABEL: func.func @_QMacc_declarePacc_declare_link(
 ! CHECK-SAME: %[[ARG0:.*]]: !fir.ref<!fir.array<100xi32>> {fir.bindc_name = "a"})
 ! CHECK: %[[DECL:.*]]:2 = hlfir.declare %[[ARG0]](%{{.*}}) dummy_scope %{{[0-9]+}} {acc.declare = #acc.declare<dataClause =  acc_declare_link>, uniq_name = "_QMacc_declareFacc_declare_linkEa"} : (!fir.ref<!fir.array<100xi32>>, !fir.shape<1>, !fir.dscope) -> (!fir.ref<!fir.array<100xi32>>, !fir.ref<!fir.array<100xi32>>)
-! CHECK: %[[LINK:.*]] = acc.declare_link varPtr(%[[DECL]]#0 : !fir.ref<!fir.array<100xi32>>)   bounds(%{{.*}}) -> !fir.ref<!fir.array<100xi32>> {name = "a"}
+! CHECK: %[[LINK:.*]] = acc.declare_link varPtr(%[[DECL]]#0 : !fir.ref<!fir.array<100xi32>>) -> !fir.ref<!fir.array<100xi32>> {name = "a"}
 ! CHECK: acc.declare_enter dataOperands(%[[LINK]] : !fir.ref<!fir.array<100xi32>>)
 ! CHECK: %{{.*}}:2 = fir.do_loop %{{.*}} = %{{.*}} to %{{.*}} step %{{.*}} iter_args(%arg{{.*}} = %{{.*}}) -> (index, i32)
 
@@ -154,11 +148,11 @@ subroutine acc_declare_device_resident(a)
 ! CHECK-LABEL: func.func @_QMacc_declarePacc_declare_device_resident(
 ! CHECK-SAME: %[[ARG0:.*]]: !fir.ref<!fir.array<100xi32>> {fir.bindc_name = "a"})
 ! CHECK: %[[DECL:.*]]:2 = hlfir.declare %[[ARG0]](%{{.*}}) dummy_scope %{{[0-9]+}} {acc.declare = #acc.declare<dataClause =  acc_declare_device_resident>, uniq_name = "_QMacc_declareFacc_declare_device_residentEa"} : (!fir.ref<!fir.array<100xi32>>, !fir.shape<1>, !fir.dscope) -> (!fir.ref<!fir.array<100xi32>>, !fir.ref<!fir.array<100xi32>>)
-! CHECK: %[[DEVICERES:.*]] = acc.declare_device_resident varPtr(%[[DECL]]#0 : !fir.ref<!fir.array<100xi32>>)   bounds(%{{.*}}) -> !fir.ref<!fir.array<100xi32>> {name = "a"}
+! CHECK: %[[DEVICERES:.*]] = acc.declare_device_resident varPtr(%[[DECL]]#0 : !fir.ref<!fir.array<100xi32>>) -> !fir.ref<!fir.array<100xi32>> {name = "a"}
 ! CHECK: %[[TOKEN:.*]] = acc.declare_enter dataOperands(%[[DEVICERES]] : !fir.ref<!fir.array<100xi32>>)
 ! CHECK: %{{.*}}:2 = fir.do_loop %{{.*}} = %{{.*}} to %{{.*}} step %{{.*}} iter_args(%arg{{.*}} = %{{.*}}) -> (index, i32)
 ! CHECK: acc.declare_exit token(%[[TOKEN]]) dataOperands(%[[DEVICERES]] : !fir.ref<!fir.array<100xi32>>)
-! CHECK: acc.delete accPtr(%[[DEVICERES]] : !fir.ref<!fir.array<100xi32>>) bounds(%{{.*}}) {dataClause = #acc<data_clause acc_declare_device_resident>, name = "a"}
+! CHECK: acc.delete accPtr(%[[DEVICERES]] : !fir.ref<!fir.array<100xi32>>) {dataClause = #acc<data_clause acc_declare_device_resident>, name = "a"}
 
   subroutine acc_declare_device_resident2()
     integer, parameter :: n = 100
@@ -169,10 +163,10 @@ subroutine acc_declare_device_resident2()
 ! CHECK-LABEL: func.func @_QMacc_declarePacc_declare_device_resident2()
 ! CHECK: %[[ALLOCA:.*]] = fir.alloca !fir.array<100xf32> {bindc_name = "dataparam", uniq_name = "_QMacc_declareFacc_declare_device_resident2Edataparam"}
 ! CHECK: %[[DECL:.*]]:2 = hlfir.declare %[[ALLOCA]](%{{.*}}) {acc.declare = #acc.declare<dataClause =  acc_declare_device_resident>, uniq_name = "_QMacc_declareFacc_declare_device_resident2Edataparam"} : (!fir.ref<!fir.array<100xf32>>, !fir.shape<1>) -> (!fir.ref<!fir.array<100xf32>>, !fir.ref<!fir.array<100xf32>>)
-! CHECK: %[[DEVICERES:.*]] = acc.declare_device_resident varPtr(%[[DECL]]#0 : !fir.ref<!fir.array<100xf32>>)   bounds(%{{.*}}) -> !fir.ref<!fir.array<100xf32>> {name = "dataparam"}
+! CHECK: %[[DEVICERES:.*]] = acc.declare_device_resident varPtr(%[[DECL]]#0 : !fir.ref<!fir.array<100xf32>>) -> !fir.ref<!fir.array<100xf32>> {name = "dataparam"}
 ! CHECK: %[[TOKEN:.*]] = acc.declare_enter dataOperands(%[[DEVICERES]] : !fir.ref<!fir.array<100xf32>>)
 ! CHECK: acc.declare_exit token(%[[TOKEN]]) dataOperands(%[[DEVICERES]] : !fir.ref<!fir.array<100xf32>>)
-! CHECK: acc.delete accPtr(%[[DEVICERES]] : !fir.ref<!fir.array<100xf32>>) bounds(%{{.*}}) {dataClause = #acc<data_clause acc_declare_device_resident>, name = "dataparam"}
+! CHECK: acc.delete accPtr(%[[DEVICERES]] : !fir.ref<!fir.array<100xf32>>) {dataClause = #acc<data_clause acc_declare_device_resident>, name = "dataparam"}
 
   subroutine acc_declare_link2()
     integer, parameter :: n = 100
@@ -183,7 +177,7 @@ subroutine acc_declare_link2()
 ! CHECK-LABEL: func.func @_QMacc_declarePacc_declare_link2()
 ! CHECK: %[[ALLOCA:.*]] = fir.alloca !fir.array<100xf32> {bindc_name = "dataparam", uniq_name = "_QMacc_declareFacc_declare_link2Edataparam"}
 ! CHECK: %[[DECL:.*]]:2 = hlfir.declare %[[ALLOCA]](%{{.*}}) {acc.declare = #acc.declare<dataClause =  acc_declare_link>, uniq_name = "_QMacc_declareFacc_declare_link2Edataparam"} : (!fir.ref<!fir.array<100xf32>>, !fir.shape<1>) -> (!fir.ref<!fir.array<100xf32>>, !fir.ref<!fir.array<100xf32>>)
-! CHECK: %[[LINK:.*]] = acc.declare_link varPtr(%[[DECL]]#0 : !fir.ref<!fir.array<100xf32>>)   bounds(%{{.*}}) -> !fir.ref<!fir.array<100xf32>> {name = "dataparam"}
+! CHECK: %[[LINK:.*]] = acc.declare_link varPtr(%[[DECL]]#0 : !fir.ref<!fir.array<100xf32>>) -> !fir.ref<!fir.array<100xf32>> {name = "dataparam"}
 ! CHECK: acc.declare_enter dataOperands(%[[LINK]] : !fir.ref<!fir.array<100xf32>>)
 
   subroutine acc_declare_deviceptr2()
@@ -195,7 +189,7 @@ subroutine acc_declare_deviceptr2()
 ! CHECK-LABEL: func.func @_QMacc_declarePacc_declare_deviceptr2()
 ! CHECK: %[[ALLOCA:.*]] = fir.alloca !fir.array<100xf32> {bindc_name = "dataparam", uniq_name = "_QMacc_declareFacc_declare_deviceptr2Edataparam"}
 ! CHECK: %[[DECL:.*]]:2 = hlfir.declare %[[ALLOCA]](%{{.*}}) {acc.declare = #acc.declare<dataClause =  acc_deviceptr>, uniq_name = "_QMacc_declareFacc_declare_deviceptr2Edataparam"} : (!fir.ref<!fir.array<100xf32>>, !fir.shape<1>) -> (!fir.ref<!fir.array<100xf32>>, !fir.ref<!fir.array<100xf32>>)
-! CHECK: %[[DEVICEPTR:.*]] = acc.deviceptr varPtr(%[[DECL]]#0 : !fir.ref<!fir.array<100xf32>>)   bounds(%{{.*}}) -> !fir.ref<!fir.array<100xf32>> {name = "dataparam"}
+! CHECK: %[[DEVICEPTR:.*]] = acc.deviceptr varPtr(%[[DECL]]#0 : !fir.ref<!fir.array<100xf32>>) -> !fir.ref<!fir.array<100xf32>> {name = "dataparam"}
 ! CHECK: acc.declare_enter dataOperands(%[[DEVICEPTR]] : !fir.ref<!fir.array<100xf32>>)
 
   function acc_declare_in_func()
@@ -204,11 +198,11 @@ function acc_declare_in_func()
   end function
 
 ! CHECK-LABEL: func.func @_QMacc_declarePacc_declare_in_func() -> f32 {
-! CHECK: %[[DEVICE_RESIDENT:.*]] = acc.declare_device_resident varPtr(%{{.*}}#0 : !fir.ref<!fir.array<1024xf32>>) bounds(%{{.*}}) -> !fir.ref<!fir.array<1024xf32>> {name = "a"}
+! CHECK: %[[DEVICE_RESIDENT:.*]] = acc.declare_device_resident varPtr(%{{.*}}#0 : !fir.ref<!fir.array<1024xf32>>) -> !fir.ref<!fir.array<1024xf32>> {name = "a"}
 ! CHECK: %[[TOKEN:.*]] = acc.declare_enter dataOperands(%[[DEVICE_RESIDENT]] : !fir.ref<!fir.array<1024xf32>>)
 ! CHECK: %[[LOAD:.*]] = fir.load %{{.*}}#1 : !fir.ref<f32>
 ! CHECK: acc.declare_exit token(%[[TOKEN]]) dataOperands(%[[DEVICE_RESIDENT]] : !fir.ref<!fir.array<1024xf32>>)
-! CHECK: acc.delete accPtr(%[[DEVICE_RESIDENT]] : !fir.ref<!fir.array<1024xf32>>) bounds(%6) {dataClause = #acc<data_clause acc_declare_device_resident>, name = "a"}
+! CHECK: acc.delete accPtr(%[[DEVICE_RESIDENT]] : !fir.ref<!fir.array<1024xf32>>) {dataClause = #acc<data_clause acc_declare_device_resident>, name = "a"}
 ! CHECK: return %[[LOAD]] : f32
 ! CHECK: }
 
@@ -222,12 +216,12 @@ function acc_declare_in_func2(i)
 ! CHECK-LABEL: func.func @_QMacc_declarePacc_declare_in_func2(%arg0: !fir.ref<i32> {fir.bindc_name = "i"}) -> f32 {
 ! CHECK: %[[ALLOCA_A:.*]] = fir.alloca !fir.array<1024xf32> {bindc_name = "a", uniq_name = "_QMacc_declareFacc_declare_in_func2Ea"}
 ! CHECK: %[[DECL_A:.*]]:2 = hlfir.declare %[[ALLOCA_A]](%{{.*}}) {acc.declare = #acc.declare<dataClause =  acc_create>, uniq_name = "_QMacc_declareFacc_declare_in_func2Ea"} : (!fir.ref<!fir.array<1024xf32>>, !fir.shape<1>) -> (!fir.ref<!fir.array<1024xf32>>, !fir.ref<!fir.array<1024xf32>>)
-! CHECK: %[[CREATE:.*]] = acc.create varPtr(%[[DECL_A]]#0 : !fir.ref<!fir.array<1024xf32>>) bounds(%{{[0-9]+}}) -> !fir.ref<!fir.array<1024xf32>> {name = "a"}
+! CHECK: %[[CREATE:.*]] = acc.create varPtr(%[[DECL_A]]#0 : !fir.ref<!fir.array<1024xf32>>) -> !fir.ref<!fir.array<1024xf32>> {name = "a"}
 ! CHECK: %[[TOKEN:.*]] = acc.declare_enter dataOperands(%[[CREATE]] : !fir.ref<!fir.array<1024xf32>>)
 ! CHECK:   cf.br ^bb1
 ! CHECK: ^bb1:
 ! CHECK: acc.declare_exit token(%[[TOKEN]]) dataOperands(%[[CREATE]] : !fir.ref<!fir.array<1024xf32>>)
-! CHECK: acc.delete accPtr(%[[CREATE]] : !fir.ref<!fir.array<1024xf32>>) bounds(%{{[0-9]+}}) {dataClause = #acc<data_clause acc_create>, name = "a"}
+! CHECK: acc.delete accPtr(%[[CREATE]] : !fir.ref<!fir.array<1024xf32>>) {dataClause = #acc<data_clause acc_create>, name = "a"}
 ! CHECK:   return %{{.*}} : f32
 ! CHECK: }
 
@@ -256,31 +250,15 @@ subroutine acc_declare_allocate()
 
 ! CHECK-LABEL: func.func private @_QMacc_declareFacc_declare_allocateEa_acc_declare_update_desc_post_alloc(
 ! CHECK-SAME:    %[[ARG0:.*]]: !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>) {
-! CHECK:         %[[UPDATE:.*]] = acc.update_device varPtr(%[[ARG0]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>) -> !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>> {implicit = true, name = "a_desc", structured = false}
+! CHECK:         %[[UPDATE:.*]] = acc.update_device varPtr(%[[ARG0]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>) -> !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>> {implicit = true, name = "a", structured = false}
 ! CHECK:         acc.update dataOperands(%[[UPDATE]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>)
-! CHECK:         %[[LOAD:.*]] = fir.load %[[ARG0]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>
-! CHECK:         %[[BOX_ADDR:.*]] = fir.box_addr %[[LOAD]] {acc.declare = #acc.declare<dataClause =  acc_create>} : (!fir.box<!fir.heap<!fir.array<?xi32>>>) -> !fir.heap<!fir.array<?xi32>>
-! CHECK:         %[[CREATE:.*]] = acc.create varPtr(%[[BOX_ADDR]] : !fir.heap<!fir.array<?xi32>>) -> !fir.heap<!fir.array<?xi32>> {name = "a", structured = false}
-! CHECK:         acc.declare_enter dataOperands(%[[CREATE]] : !fir.heap<!fir.array<?xi32>>)
-! CHECK:         return
-! CHECK:       }
-
-! CHECK-LABEL: func.func private @_QMacc_declareFacc_declare_allocateEa_acc_declare_update_desc_pre_dealloc(
-! CHECK-SAME:    %[[ARG0:.*]]: !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>) {
-! CHECK:         %[[LOAD:.*]] = fir.load %[[ARG0]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>
-! CHECK:         %[[BOX_ADDR:.*]] = fir.box_addr %[[LOAD]] {acc.declare = #acc.declare<dataClause =  acc_create>} : (!fir.box<!fir.heap<!fir.array<?xi32>>>) -> !fir.heap<!fir.array<?xi32>>
-! CHECK:         %[[GETDEVICEPTR:.*]] = acc.getdeviceptr varPtr(%[[BOX_ADDR]] : !fir.heap<!fir.array<?xi32>>) -> !fir.heap<!fir.array<?xi32>> {dataClause = #acc<data_clause acc_create>, name = "a", structured = false}
-! CHECK:         acc.declare_exit dataOperands(%[[GETDEVICEPTR]] : !fir.heap<!fir.array<?xi32>>)
-! CHECK:         acc.delete accPtr(%[[GETDEVICEPTR]] : !fir.heap<!fir.array<?xi32>>) {dataClause = #acc<data_clause acc_create>, name = "a", structured = false}
 ! CHECK:         return
 ! CHECK:       }
 
 ! CHECK-LABEL: func.func private @_QMacc_declareFacc_declare_allocateEa_acc_declare_update_desc_post_dealloc(
 ! CHECK-SAME:    %[[ARG0:.*]]: !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>) {
-! CHECK:         %[[LOAD:.*]] = fir.load %[[ARG0]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>
-! CHECK:         %[[BOX_ADDR:.*]] = fir.box_addr %[[LOAD]] : (!fir.box<!fir.heap<!fir.array<?xi32>>>) -> !fir.heap<!fir.array<?xi32>>
-! CHECK:         %[[UPDATE:.*]] = acc.update_device varPtr(%[[BOX_ADDR]] : !fir.heap<!fir.array<?xi32>>) -> !fir.heap<!fir.array<?xi32>> {implicit = true, name = "a_desc", structured = false}
-! CHECK:         acc.update dataOperands(%[[UPDATE]] : !fir.heap<!fir.array<?xi32>>)
+! CHECK:         %[[UPDATE:.*]] = acc.update_device varPtr(%[[ARG0]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>) -> !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>> {implicit = true, name = "a", structured = false}
+! CHECK:         acc.update dataOperands(%[[UPDATE]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>)
 ! CHECK:         return
 ! CHECK:       }
 
@@ -298,14 +276,14 @@ subroutine acc_declare_multiple_directive(a, b)
 ! CHECK-SAME: %[[ARG0:.*]]: !fir.ref<!fir.array<100xi32>> {fir.bindc_name = "a"}, %[[ARG1:.*]]: !fir.ref<!fir.array<100xi32>> {fir.bindc_name = "b"}) {
 ! CHECK: %[[DECL_A:.*]]:2 = hlfir.declare %[[ARG0]](%{{.*}}) dummy_scope %{{[0-9]+}} {acc.declare = #acc.declare<dataClause =  acc_copy>, uniq_name = "_QMacc_declareFacc_declare_multiple_directiveEa"} : (!fir.ref<!fir.array<100xi32>>, !fir.shape<1>, !fir.dscope) -> (!fir.ref<!fir.array<100xi32>>, !fir.ref<!fir.array<100xi32>>)
 ! CHECK: %[[DECL_B:.*]]:2 = hlfir.declare %[[ARG1]](%{{.*}}) dummy_scope %{{[0-9]+}} {acc.declare = #acc.declare<dataClause =  acc_copyout>, uniq_name = "_QMacc_declareFacc_declare_multiple_directiveEb"} : (!fir.ref<!fir.array<100xi32>>, !fir.shape<1>, !fir.dscope) -> (!fir.ref<!fir.array<100xi32>>, !fir.ref<!fir.array<100xi32>>)
-! CHECK: %[[COPYIN:.*]] = acc.copyin varPtr(%[[DECL_A]]#0 : !fir.ref<!fir.array<100xi32>>) bounds(%{{.*}}) -> !fir.ref<!fir.array<100xi32>> {dataClause = #acc<data_clause acc_copy>, name = "a"}
-! CHECK: %[[CREATE:.*]] = acc.create varPtr(%[[DECL_B]]#0 : !fir.ref<!fir.array<100xi32>>) bounds(%{{.*}}) -> !fir.ref<!fir.array<100xi32>> {dataClause = #acc<data_clause acc_copyout>, name = "b"}
+! CHECK: %[[COPYIN:.*]] = acc.copyin varPtr(%[[DECL_A]]#0 : !fir.ref<!fir.array<100xi32>>) -> !fir.ref<!fir.array<100xi32>> {dataClause = #acc<data_clause acc_copy>, name = "a"}
+! CHECK: %[[CREATE:.*]] = acc.create varPtr(%[[DECL_B]]#0 : !fir.ref<!fir.array<100xi32>>) -> !fir.ref<!fir.array<100xi32>> {dataClause = #acc<data_clause acc_copyout>, name = "b"}
 ! CHECK: acc.declare_enter dataOperands(%[[COPYIN]], %[[CREATE]] : !fir.ref<!fir.array<100xi32>>, !fir.ref<!fir.array<100xi32>>)
 ! CHECK: %{{.*}}:{{.*}} = fir.do_loop %{{.*}} = %{{.*}} to %{{.*}} step %{{.*}} iter_args(%{{.*}} = %{{.*}}) -> (index, i32) {
 
 
-! CHECK: acc.copyout accPtr(%[[CREATE]] : !fir.ref<!fir.array<100xi32>>) bounds(%{{.*}}) to varPtr(%[[DECL_B]]#0 : !fir.ref<!fir.array<100xi32>>) {name = "b"}
-! CHECK: acc.copyout accPtr(%[[COPYIN]] : !fir.ref<!fir.array<100xi32>>) bounds(%{{.*}}) to varPtr(%[[DECL_A]]#0 : !fir.ref<!fir.array<100xi32>>) {dataClause = #acc<data_clause acc_copy>, name = "a"}
+! CHECK: acc.copyout accPtr(%[[CREATE]] : !fir.ref<!fir.array<100xi32>>) to varPtr(%[[DECL_B]]#0 : !fir.ref<!fir.array<100xi32>>) {name = "b"}
+! CHECK: acc.copyout accPtr(%[[COPYIN]] : !fir.ref<!fir.array<100xi32>>) to varPtr(%[[DECL_A]]#0 : !fir.ref<!fir.array<100xi32>>) {dataClause = #acc<data_clause acc_copy>, name = "a"}
 
   subroutine acc_declare_array_section(a)
     integer :: a(:)
@@ -318,12 +296,11 @@ subroutine acc_declare_array_section(a)
 
 ! CHECK-LABEL: func.func @_QMacc_declarePacc_declare_array_section(
 ! CHECK-SAME:    %[[ARG0:.*]]: !fir.box<!fir.array<?xi32>> {fir.bindc_name = "a"}) {
-! CHECK: %[[DECL_A:.*]]:2 = hlfir.declare %[[ARG0]] dummy_scope %{{[0-9]+}} {uniq_name = "_QMacc_declareFacc_declare_array_sectionEa"} : (!fir.box<!fir.array<?xi32>>, !fir.dscope) -> (!fir.box<!fir.array<?xi32>>, !fir.box<!fir.array<?xi32>>)
-! CHECK: %[[BOX_ADDR:.*]] = fir.box_addr %[[DECL_A]]#0 {acc.declare = #acc.declare<dataClause =  acc_copy>} : (!fir.box<!fir.array<?xi32>>) -> !fir.ref<!fir.array<?xi32>>
-! CHECK: %[[COPYIN:.*]] = acc.copyin varPtr(%[[BOX_ADDR]] : !fir.ref<!fir.array<?xi32>>) bounds(%{{.*}}) -> !fir.ref<!fir.array<?xi32>> {dataClause = #acc<data_clause acc_copy>, name = "a(1:10)"}
-! CHECK: acc.declare_enter dataOperands(%[[COPYIN]] : !fir.ref<!fir.array<?xi32>>)
+! CHECK: %[[DECL_A:.*]]:2 = hlfir.declare %[[ARG0]] dummy_scope %{{[0-9]+}} {acc.declare = #acc.declare<dataClause = acc_copy>, uniq_name = "_QMacc_declareFacc_declare_array_sectionEa"} : (!fir.box<!fir.array<?xi32>>, !fir.dscope) -> (!fir.box<!fir.array<?xi32>>, !fir.box<!fir.array<?xi32>>)
+! CHECK: %[[COPYIN:.*]] = acc.copyin var(%[[DECL_A]]#0 : !fir.box<!fir.array<?xi32>>) bounds(%{{.*}}) -> !fir.box<!fir.array<?xi32>> {dataClause = #acc<data_clause acc_copy>, name = "a(1:10)"}
+! CHECK: acc.declare_enter dataOperands(%[[COPYIN]] : !fir.box<!fir.array<?xi32>>)
 
-! CHECK: acc.copyout accPtr(%[[COPYIN]] : !fir.ref<!fir.array<?xi32>>) bounds(%{{.*}}) to varPtr(%[[BOX_ADDR]] : !fir.ref<!fir.array<?xi32>>) {dataClause = #acc<data_clause acc_copy>, name = "a(1:10)"}
+! CHECK: acc.copyout accVar(%[[COPYIN]] : !fir.box<!fir.array<?xi32>>) bounds(%{{.*}}) to var(%[[DECL_A]]#0 : !fir.box<!fir.array<?xi32>>) {dataClause = #acc<data_clause acc_copy>, name = "a(1:10)"}
 
   subroutine acc_declare_allocate_with_stat()
     integer :: status
@@ -353,28 +330,14 @@ module acc_declare_allocatable_test
 
 ! CHECK-LABEL: func.func private @_QMacc_declare_allocatable_testEdata1_acc_declare_update_desc_post_alloc() {
 ! CHECK:         %[[GLOBAL_ADDR:.*]] = fir.address_of(@_QMacc_declare_allocatable_testEdata1) : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>
-! CHECK:         %[[UPDATE:.*]] = acc.update_device varPtr(%[[GLOBAL_ADDR]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>) -> !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>> {implicit = true, name = "data1_desc", structured = false}
+! CHECK:         %[[UPDATE:.*]] = acc.update_device varPtr(%[[GLOBAL_ADDR]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>) -> !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>> {implicit = true, name = "data1", structured = false}
 ! CHECK:         acc.update dataOperands(%[[UPDATE]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>)
-! CHECK:         %[[LOAD:.*]] = fir.load %[[GLOBAL_ADDR]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>
-! CHECK:         %[[BOXADDR:.*]] = fir.box_addr %[[LOAD]] {acc.declare = #acc.declare<dataClause =  acc_create>} : (!fir.box<!fir.heap<!fir.array<?xi32>>>) -> !fir.heap<!fir.array<?xi32>>
-! CHECK:         %[[CREATE:.*]] = acc.create varPtr(%[[BOXADDR]] : !fir.heap<!fir.array<?xi32>>) -> !fir.heap<!fir.array<?xi32>> {name = "data1", structured = false}
-! CHECK:         acc.declare_enter dataOperands(%[[CREATE]] : !fir.heap<!fir.array<?xi32>>)
-! CHECK:         return
-! CHECK:       }
-
-! CHECK-LABEL: func.func private @_QMacc_declare_allocatable_testEdata1_acc_declare_update_desc_pre_dealloc() {
-! CHECK:         %[[GLOBAL_ADDR:.*]] = fir.address_of(@_QMacc_declare_allocatable_testEdata1) : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>
-! CHECK:         %[[LOAD:.*]] = fir.load %[[GLOBAL_ADDR]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>
-! CHECK:         %[[BOXADDR:.*]] = fir.box_addr %[[LOAD]] {acc.declare = #acc.declare<dataClause =  acc_create>} : (!fir.box<!fir.heap<!fir.array<?xi32>>>) -> !fir.heap<!fir.array<?xi32>>
-! CHECK:         %[[DEVPTR:.*]] = acc.getdeviceptr varPtr(%[[BOXADDR]] : !fir.heap<!fir.array<?xi32>>)   -> !fir.heap<!fir.array<?xi32>> {dataClause = #acc<data_clause acc_create>, name = "data1", structured = false}
-! CHECK:         acc.declare_exit dataOperands(%[[DEVPTR]] : !fir.heap<!fir.array<?xi32>>)
-! CHECK:         acc.delete accPtr(%[[DEVPTR]] : !fir.heap<!fir.array<?xi32>>) {dataClause = #acc<data_clause acc_create>, name = "data1", structured = false}
 ! CHECK:         return
 ! CHECK:       }
 
 ! CHECK-LABEL: func.func private @_QMacc_declare_allocatable_testEdata1_acc_declare_update_desc_post_dealloc() {
 ! CHECK:         %[[GLOBAL_ADDR:.*]] = fir.address_of(@_QMacc_declare_allocatable_testEdata1) : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>
-! CHECK:         %[[UPDATE:.*]] = acc.update_device varPtr(%[[GLOBAL_ADDR]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>)   -> !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>> {implicit = true, name = "data1_desc", structured = false}
+! CHECK:         %[[UPDATE:.*]] = acc.update_device varPtr(%[[GLOBAL_ADDR]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>)   -> !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>> {implicit = true, name = "data1", structured = false}
 ! CHECK:         acc.update dataOperands(%[[UPDATE]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>)
 ! CHECK:         return
 ! CHECK:       }
diff --git a/flang/test/Lower/OpenACC/acc-enter-data-unwrap-defaultbounds.f90 b/flang/test/Lower/OpenACC/acc-enter-data-unwrap-defaultbounds.f90
new file mode 100644
index 0000000000000..6bdd1031eeb4e
--- /dev/null
+++ b/flang/test/Lower/OpenACC/acc-enter-data-unwrap-defaultbounds.f90
@@ -0,0 +1,818 @@
+! This test checks lowering of OpenACC enter data directive.
+
+! RUN: bbc -fopenacc -emit-hlfir --openacc-unwrap-fir-box=true --openacc-generate-default-bounds=true %s -o - | FileCheck %s
+
+subroutine acc_enter_data
+  integer :: async = 1
+  real, dimension(10, 10) :: a, b, c
+  real, pointer :: d
+  logical :: ifCondition = .TRUE.
+
+!CHECK: %[[C10:.*]] = arith.constant 10 : index
+!CHECK: %[[EXTENT_C10:.*]] = arith.constant 10 : index
+!CHECK: %[[A:.*]] = fir.alloca !fir.array<10x10xf32> {{{.*}}uniq_name = "{{.*}}Ea"}
+!CHECK: %[[DECLA:.*]]:2 = hlfir.declare %[[A]]
+!CHECK: %[[B:.*]] = fir.alloca !fir.array<10x10xf32> {{{.*}}uniq_name = "{{.*}}Eb"}
+!CHECK: %[[DECLB:.*]]:2 = hlfir.declare %[[B]]
+!CHECK: %[[C:.*]] = fir.alloca !fir.array<10x10xf32> {{{.*}}uniq_name = "{{.*}}Ec"}
+!CHECK: %[[DECLC:.*]]:2 = hlfir.declare %[[C]]
+!CHECK: %[[D:.*]] = fir.alloca !fir.box<!fir.ptr<f32>> {bindc_name = "d", uniq_name = "{{.*}}Ed"}
+!CHECK: %[[DECLD:.*]]:2 = hlfir.declare %[[D]]
+
+  !$acc enter data create(a)
+!CHECK: %[[ONE:.*]] = arith.constant 1 : index
+!CHECK: %[[LB:.*]] = arith.constant 0 : index
+!CHECK: %[[UB:.*]] = arith.subi %[[C10]], %[[ONE]] : index
+!CHECK: %[[BOUND0:.*]] = acc.bounds lowerbound(%[[LB]] : index) upperbound(%[[UB]] : index) extent(%[[C10]] : index) stride(%[[ONE]] : index) startIdx(%[[ONE]] : index)
+!CHECK: %[[LB:.*]] = arith.constant 0 : index
+!CHECK: %[[UB:.*]] = arith.subi %[[EXTENT_C10]], %[[ONE]] : index
+!CHECK: %[[BOUND1:.*]] = acc.bounds lowerbound(%[[LB]] : index) upperbound(%[[UB]] : index) extent(%[[EXTENT_C10]] : index) stride(%[[ONE]] : index) startIdx(%[[ONE]] : index)
+!CHECK: %[[CREATE_A:.*]] = acc.create varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) bounds(%[[BOUND0]], %[[BOUND1]]) -> !fir.ref<!fir.array<10x10xf32>> {name = "a", structured = false}
+!CHECK: acc.enter_data dataOperands(%[[CREATE_A]] : !fir.ref<!fir.array<10x10xf32>>){{$}}
+
+  !$acc enter data create(a) if(.true.)
+!CHECK: %[[ONE:.*]] = arith.constant 1 : index
+!CHECK: %[[LB:.*]] = arith.constant 0 : index
+!CHECK: %[[UB:.*]] = arith.subi %[[C10]], %[[ONE]] : index
+!CHECK: %[[BOUND0:.*]] = acc.bounds lowerbound(%[[LB]] : index) upperbound(%[[UB]] : index) extent(%[[C10]] : index) stride(%[[ONE]] : index) startIdx(%[[ONE]] : index)
+!CHECK: %[[LB:.*]] = arith.constant 0 : index
+!CHECK: %[[UB:.*]] = arith.subi %[[EXTENT_C10]], %[[ONE]] : index
+!CHECK: %[[BOUND1:.*]] = acc.bounds lowerbound(%[[LB]] : index) upperbound(%[[UB]] : index) extent(%[[EXTENT_C10]] : index) stride(%[[ONE]] : index) startIdx(%[[ONE]] : index)
+!CHECK: %[[CREATE_A:.*]] = acc.create varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) bounds(%[[BOUND0]], %[[BOUND1]]) -> !fir.ref<!fir.array<10x10xf32>> {name = "a", structured = false}
+!CHECK: [[IF1:%.*]] = arith.constant true
+!CHECK: acc.enter_data if([[IF1]]) dataOperands(%[[CREATE_A]] : !fir.ref<!fir.array<10x10xf32>>){{$}}
+
+  !$acc enter data create(a) if(ifCondition)
+!CHECK: %[[ONE:.*]] = arith.constant 1 : index
+!CHECK: %[[LB:.*]] = arith.constant 0 : index
+!CHECK: %[[UB:.*]] = arith.subi %[[C10]], %[[ONE]] : index
+!CHECK: %[[BOUND0:.*]] = acc.bounds lowerbound(%[[LB]] : index) upperbound(%[[UB]] : index) extent(%[[C10]] : index) stride(%[[ONE]] : index) startIdx(%[[ONE]] : index)
+!CHECK: %[[LB:.*]] = arith.constant 0 : index
+!CHECK: %[[UB:.*]] = arith.subi %[[EXTENT_C10]], %[[ONE]] : index
+!CHECK: %[[BOUND1:.*]] = acc.bounds lowerbound(%[[LB]] : index) upperbound(%[[UB]] : index) extent(%[[EXTENT_C10]] : index) stride(%[[ONE]] : index) startIdx(%[[ONE]] : index)
+!CHECK: %[[CREATE_A:.*]] = acc.create varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) bounds(%[[BOUND0]], %[[BOUND1]]) -> !fir.ref<!fir.array<10x10xf32>> {name = "a", structured = false}
+!CHECK: [[IFCOND:%.*]] = fir.load %{{.*}} : !fir.ref<!fir.logical<4>>
+!CHECK: [[IF2:%.*]] = fir.convert [[IFCOND]] : (!fir.logical<4>) -> i1
+!CHECK: acc.enter_data if([[IF2]]) dataOperands(%[[CREATE_A]] : !fir.ref<!fir.array<10x10xf32>>){{$}}
+
+  !$acc enter data create(a) create(b) create(c)
+!CHECK: %[[BOUND0:.*]] = acc.bounds lowerbound(%{{.*}} : index) upperbound(%{{.*}} : index) extent(%[[C10]] : index) stride(%c1{{.*}} : index) startIdx(%{{.*}} : index)
+!CHECK: %[[BOUND1:.*]] = acc.bounds lowerbound(%{{.*}} : index) upperbound(%{{.*}} : index) extent(%[[EXTENT_C10]] : index) stride(%c1{{.*}} : index) startIdx(%{{.*}} : index)
+!CHECK: %[[CREATE_A:.*]] = acc.create varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) bounds(%[[BOUND0]], %[[BOUND1]]) -> !fir.ref<!fir.array<10x10xf32>> {name = "a", structured = false}
+!CHECK: %[[BOUND0:.*]] = acc.bounds lowerbound(%{{.*}} : index) upperbound(%{{.*}} : index) extent(%c10_{{.*}} : index) stride(%c1{{.*}} : index) startIdx(%{{.*}} : index)
+!CHECK: %[[BOUND1:.*]] = acc.bounds lowerbound(%{{.*}} : index) upperbound(%{{.*}} : index) extent(%c10_{{.*}} : index) stride(%c1{{.*}} : index) startIdx(%{{.*}} : index)
+!CHECK: %[[CREATE_B:.*]] = acc.create varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<10x10xf32>>) bounds(%[[BOUND0]], %[[BOUND1]]) -> !fir.ref<!fir.array<10x10xf32>> {name = "b", structured = false}
+!CHECK: %[[BOUND0:.*]] = acc.bounds lowerbound(%{{.*}} : index) upperbound(%{{.*}} : index) extent(%c10_{{.*}} : index) stride(%c1{{.*}} : index) startIdx(%{{.*}} : index)
+!CHECK: %[[BOUND1:.*]] = acc.bounds lowerbound(%{{.*}} : index) upperbound(%{{.*}} : index) extent(%c10_{{.*}} : index) stride(%c1{{.*}} : index) startIdx(%{{.*}} : index)
+!CHECK: %[[CREATE_C:.*]] = acc.create varPtr(%[[DECLC]]#0 : !fir.ref<!fir.array<10x10xf32>>) bounds(%[[BOUND0]], %[[BOUND1]]) -> !fir.ref<!fir.array<10x10xf32>> {name = "c", structured = false}
+!CHECK: acc.enter_data dataOperands(%[[CREATE_A]], %[[CREATE_B]], %[[CREATE_C]] : !fir.ref<!fir.array<10x10xf32>>, !fir.ref<!fir.array<10x10xf32>>, !fir.ref<!fir.array<10x10xf32>>){{$}}
+
+  !$acc enter data create(a) create(b) create(zero: c)
+!CHECK: %[[BOUND0:.*]] = acc.bounds lowerbound(%{{.*}} : index) upperbound(%{{.*}} : index) extent(%[[C10]] : index) stride(%c1{{.*}} : index) startIdx(%{{.*}} : index)
+!CHECK: %[[BOUND1:.*]] = acc.bounds lowerbound(%{{.*}} : index) upperbound(%{{.*}} : index) extent(%[[EXTENT_C10]] : index) stride(%c1{{.*}} : index) startIdx(%{{.*}} : index)
+!CHECK: %[[CREATE_A:.*]] = acc.create varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) bounds(%[[BOUND0]], %[[BOUND1]]) -> !fir.ref<!fir.array<10x10xf32>> {name = "a", structured = false}
+!CHECK: %[[BOUND0:.*]] = acc.bounds lowerbound(%{{.*}} : index) upperbound(%{{.*}} : index) extent(%c10_{{.*}} : index) stride(%c1{{.*}} : index) startIdx(%{{.*}} : index)
+!CHECK: %[[BOUND1:.*]] = acc.bounds lowerbound(%{{.*}} : index) upperbound(%{{.*}} : index) extent(%c10_{{.*}} : index) stride(%c1{{.*}} : index) startIdx(%{{.*}} : index)
+!CHECK: %[[CREATE_B:.*]] = acc.create varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<10x10xf32>>) bounds(%[[BOUND0]], %[[BOUND1]]) -> !fir.ref<!fir.array<10x10xf32>> {name = "b", structured = false}
+!CHECK: %[[BOUND0:.*]] = acc.bounds lowerbound(%{{.*}} : index) upperbound(%{{.*}} : index) extent(%c10_{{.*}} : index) stride(%c1{{.*}} : index) startIdx(%{{.*}} : index)
+!CHECK: %[[BOUND1:.*]] = acc.bounds lowerbound(%{{.*}} : index) upperbound(%{{.*}} : index) extent(%c10_{{.*}} : index) stride(%c1{{.*}} : index) startIdx(%{{.*}} : index)
+!CHECK: %[[CREATE_C:.*]] = acc.create varPtr(%[[DECLC]]#0 : !fir.ref<!fir.array<10x10xf32>>) bounds(%[[BOUND0]], %[[BOUND1]]) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = #acc<data_clause acc_create_zero>, name = "c", structured = false}
+!CHECK: acc.enter_data dataOperands(%[[CREATE_A]], %[[CREATE_B]], %[[CREATE_C]] : !fir.ref<!fir.array<10x10xf32>>, !fir.ref<!fir.array<10x10xf32>>, !fir.ref<!fir.array<10x10xf32>>){{$}}
+
+  !$acc enter data copyin(a) create(b) attach(d)
+!CHECK: %[[BOUND0:.*]] = acc.bounds lowerbound(%{{.*}} : index) upperbound(%{{.*}} : index) extent(%[[C10]] : index) stride(%c1{{.*}} : index) startIdx(%{{.*}} : index)
+!CHECK: %[[BOUND1:.*]] = acc.bounds lowerbound(%{{.*}} : index) upperbound(%{{.*}} : index) extent(%[[EXTENT_C10]] : index) stride(%c1{{.*}} : index) startIdx(%{{.*}} : index)
+!CHECK: %[[COPYIN_A:.*]] = acc.copyin varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) bounds(%[[BOUND0]], %[[BOUND1]])  -> !fir.ref<!fir.array<10x10xf32>> {name = "a", structured = false}
+!CHECK: %[[BOUND0:.*]] = acc.bounds lowerbound(%{{.*}} : index) upperbound(%{{.*}} : index) extent(%c10_{{.*}} : index) stride(%c1{{.*}} : index) startIdx(%{{.*}} : index)
+!CHECK: %[[BOUND1:.*]] = acc.bounds lowerbound(%{{.*}} : index) upperbound(%{{.*}} : index) extent(%c10_{{.*}} : index) stride(%c1{{.*}} : index) startIdx(%{{.*}} : index)
+!CHECK: %[[CREATE_B:.*]] = acc.create varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<10x10xf32>>) bounds(%[[BOUND0]], %[[BOUND1]]) -> !fir.ref<!fir.array<10x10xf32>> {name = "b", structured = false}
+!CHECK: %[[BOX_D:.*]] = fir.load %[[DECLD]]#0 : !fir.ref<!fir.box<!fir.ptr<f32>>>
+!CHECK: %[[BOX_ADDR_D:.*]] = fir.box_addr %[[BOX_D]] : (!fir.box<!fir.ptr<f32>>) -> !fir.ptr<f32>
+!CHECK: %[[ATTACH_D:.*]] = acc.attach varPtr(%[[BOX_ADDR_D]] : !fir.ptr<f32>) -> !fir.ptr<f32> {name = "d", structured = false}
+!CHECK: acc.enter_data dataOperands(%[[COPYIN_A]], %[[CREATE_B]], %[[ATTACH_D]] : !fir.ref<!fir.array<10x10xf32>>, !fir.ref<!fir.array<10x10xf32>>, !fir.ptr<f32>){{$}}
+
+  !$acc enter data create(a) async
+!CHECK: %[[BOUND0:.*]] = acc.bounds lowerbound(%{{.*}} : index) upperbound(%{{.*}} : index) extent(%[[C10]] : index) stride(%c1{{.*}} : index) startIdx(%{{.*}} : index)
+!CHECK: %[[BOUND1:.*]] = acc.bounds lowerbound(%{{.*}} : index) upperbound(%{{.*}} : index) extent(%[[EXTENT_C10]] : index) stride(%c1{{.*}} : index) startIdx(%{{.*}} : index)
+!CHECK: %[[CREATE_A:.*]] = acc.create varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) bounds(%[[BOUND0]], %[[BOUND1]]) -> !fir.ref<!fir.array<10x10xf32>> {asyncOnly = [#acc.device_type<none>], name = "a", structured = false}
+!CHECK: acc.enter_data dataOperands(%[[CREATE_A]] : !fir.ref<!fir.array<10x10xf32>>) attributes {async}
+
+  !$acc enter data create(a) wait
+!CHECK: %[[BOUND0:.*]] = acc.bounds lowerbound(%{{.*}} : index) upperbound(%{{.*}} : index) extent(%[[C10]] : index) stride(%c1{{.*}} : index) startIdx(%{{.*}} : index)
+!CHECK: %[[BOUND1:.*]] = acc.bounds lowerbound(%{{.*}} : index) upperbound(%{{.*}} : index) extent(%[[EXTENT_C10]] : index) stride(%c1{{.*}} : index) startIdx(%{{.*}} : index)
+!CHECK: %[[CREATE_A:.*]] = acc.create varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) bounds(%[[BOUND0]], %[[BOUND1]]) -> !fir.ref<!fir.array<10x10xf32>> {name = "a", structured = false}
+!CHECK: acc.enter_data dataOperands(%[[CREATE_A]] : !fir.ref<!fir.array<10x10xf32>>) attributes {wait}
+
+  !$acc enter data create(a) async wait
+!CHECK: %[[BOUND0:.*]] = acc.bounds lowerbound(%{{.*}} : index) upperbound(%{{.*}} : index) extent(%[[C10]] : index) stride(%c1{{.*}} : index) startIdx(%{{.*}} : index)
+!CHECK: %[[BOUND1:.*]] = acc.bounds lowerbound(%{{.*}} : index) upperbound(%{{.*}} : index) extent(%[[EXTENT_C10]] : index) stride(%c1{{.*}} : index) startIdx(%{{.*}} : index)
+!CHECK: %[[CREATE_A:.*]] = acc.create varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) bounds(%[[BOUND0]], %[[BOUND1]]) -> !fir.ref<!fir.array<10x10xf32>> {asyncOnly = [#acc.device_type<none>], name = "a", structured = false}
+!CHECK: acc.enter_data dataOperands(%[[CREATE_A]] : !fir.ref<!fir.array<10x10xf32>>) attributes {async, wait}
+
+  !$acc enter data create(a) async(1)
+!CHECK: %[[ASYNC1:.*]] = arith.constant 1 : i32
+!CHECK: %[[BOUND0:.*]] = acc.bounds lowerbound(%{{.*}} : index) upperbound(%{{.*}} : index) extent(%[[C10]] : index) stride(%c1{{.*}} : index) startIdx(%{{.*}} : index)
+!CHECK: %[[BOUND1:.*]] = acc.bounds lowerbound(%{{.*}} : index) upperbound(%{{.*}} : index) extent(%[[EXTENT_C10]] : index) stride(%c1{{.*}} : index) startIdx(%{{.*}} : index)
+!CHECK: %[[CREATE_A:.*]] = acc.create varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) bounds(%[[BOUND0]], %[[BOUND1]]) async(%[[ASYNC1]] : i32) -> !fir.ref<!fir.array<10x10xf32>> {name = "a", structured = false}
+!CHECK: acc.enter_data async(%[[ASYNC1]] : i32) dataOperands(%[[CREATE_A]] : !fir.ref<!fir.array<10x10xf32>>)
+
+  !$acc enter data create(a) async(async)
+!CHECK: %[[ASYNC2:.*]] = fir.load %{{.*}} : !fir.ref<i32>
+!CHECK: %[[BOUND0:.*]] = acc.bounds lowerbound(%{{.*}} : index) upperbound(%{{.*}} : index) extent(%[[C10]] : index) stride(%c1{{.*}} : index) startIdx(%{{.*}} : index)
+!CHECK: %[[BOUND1:.*]] = acc.bounds lowerbound(%{{.*}} : index) upperbound(%{{.*}} : index) extent(%[[EXTENT_C10]] : index) stride(%c1{{.*}} : index) startIdx(%{{.*}} : index)
+
+!CHECK: %[[CREATE_A:.*]] = acc.create varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) bounds(%[[BOUND0]], %[[BOUND1]]) async(%[[ASYNC2]] : i32) -> !fir.ref<!fir.array<10x10xf32>> {name = "a", structured = false}
+!CHECK: acc.enter_data async(%[[ASYNC2]] : i32) dataOperands(%[[CREATE_A]] : !fir.ref<!fir.array<10x10xf32>>)
+
+  !$acc enter data create(a) wait(1)
+!CHECK: %[[BOUND0:.*]] = acc.bounds lowerbound(%{{.*}} : index) upperbound(%{{.*}} : index) extent(%[[C10]] : index) stride(%c1{{.*}} : index) startIdx(%{{.*}} : index)
+!CHECK: %[[BOUND1:.*]] = acc.bounds lowerbound(%{{.*}} : index) upperbound(%{{.*}} : index) extent(%[[EXTENT_C10]] : index) stride(%c1{{.*}} : index) startIdx(%{{.*}} : index)
+!CHECK: %[[CREATE_A:.*]] = acc.create varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) bounds(%[[BOUND0]], %[[BOUND1]]) -> !fir.ref<!fir.array<10x10xf32>> {name = "a", structured = false}
+!CHECK: %[[WAIT1:.*]] = arith.constant 1 : i32
+!CHECK: acc.enter_data wait(%[[WAIT1]] : i32) dataOperands(%[[CREATE_A]] : !fir.ref<!fir.array<10x10xf32>>)
+
+  !$acc enter data create(a) wait(queues: 1, 2)
+!CHECK: %[[BOUND0:.*]] = acc.bounds lowerbound(%{{.*}} : index) upperbound(%{{.*}} : index) extent(%[[C10]] : index) stride(%c1{{.*}} : index) startIdx(%{{.*}} : index)
+!CHECK: %[[BOUND1:.*]] = acc.bounds lowerbound(%{{.*}} : index) upperbound(%{{.*}} : index) extent(%[[EXTENT_C10]] : index) stride(%c1{{.*}} : index) startIdx(%{{.*}} : index)
+!CHECK: %[[CREATE_A:.*]] = acc.create varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) bounds(%[[BOUND0]], %[[BOUND1]]) -> !fir.ref<!fir.array<10x10xf32>> {name = "a", structured = false}
+!CHECK: %[[WAIT2:.*]] = arith.constant 1 : i32
+!CHECK: %[[WAIT3:.*]] = arith.constant 2 : i32
+!CHECK: acc.enter_data wait(%[[WAIT2]], %[[WAIT3]] : i32, i32) dataOperands(%[[CREATE_A]] : !fir.ref<!fir.array<10x10xf32>>)
+
+  !$acc enter data create(a) wait(devnum: 1: queues: 1, 2)
+!CHECK: %[[BOUND0:.*]] = acc.bounds lowerbound(%{{.*}} : index) upperbound(%{{.*}} : index) extent(%[[C10]] : index) stride(%c1{{.*}} : index) startIdx(%{{.*}} : index)
+!CHECK: %[[BOUND1:.*]] = acc.bounds lowerbound(%{{.*}} : index) upperbound(%{{.*}} : index) extent(%[[EXTENT_C10]] : index) stride(%c1{{.*}} : index) startIdx(%{{.*}} : index)
+!CHECK: %[[CREATE_A:.*]] = acc.create varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) bounds(%[[BOUND0]], %[[BOUND1]]) -> !fir.ref<!fir.array<10x10xf32>> {name = "a", structured = false}
+!CHECK: %[[WAIT4:.*]] = arith.constant 1 : i32
+!CHECK: %[[WAIT5:.*]] = arith.constant 2 : i32
+!CHECK: %[[WAIT6:.*]] = arith.constant 1 : i32
+!CHECK: acc.enter_data wait_devnum(%[[WAIT6]] : i32) wait(%[[WAIT4]], %[[WAIT5]] : i32, i32) dataOperands(%[[CREATE_A]] : !fir.ref<!fir.array<10x10xf32>>)
+
+  !$acc enter data copyin(a(1:10,1:5))
+!CHECK: %[[BOUND0:.*]] = acc.bounds lowerbound(%{{.*}} : index) upperbound(%{{.*}} : index) extent(%[[C10]] : index) stride(%c1{{.*}} : index) startIdx(%{{.*}} : index)
+!CHECK: %[[BOUND1:.*]] = acc.bounds lowerbound(%{{.*}} : index) upperbound(%{{.*}} : index) extent(%[[EXTENT_C10]] : index) stride(%c1{{.*}} : index) startIdx(%{{.*}} : index)
+!CHECK: %[[COPYIN_A:.*]] = acc.copyin varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) bounds(%[[BOUND0]], %[[BOUND1]]) -> !fir.ref<!fir.array<10x10xf32>> {name = "a(1:10,1:5)", structured = false}
+!CHECK: acc.enter_data dataOperands(%[[COPYIN_A]] : !fir.ref<!fir.array<10x10xf32>>)
+
+  !$acc enter data copyin(a(1:,1:5))
+!CHECK: %[[ONE:.*]] = arith.constant 1 : index
+!CHECK: %[[LB1:.*]] = arith.constant 0 : index
+!CHECK: %[[UB1:.*]] = arith.subi %c10{{.*}}, %[[ONE]] : index
+!CHECK: %[[BOUND1:.*]] = acc.bounds lowerbound(%[[LB1]] : index) upperbound(%[[UB1]] : index) extent(%c10{{.*}} : index) stride(%[[ONE]] : index) startIdx(%c1{{.*}} : index)
+!CHECK: %[[LB2:.*]] = arith.constant 0 : index
+!CHECK: %[[UB2:.*]] = arith.constant 4 : index
+!CHECK: %[[BOUND2:.*]] = acc.bounds lowerbound(%[[LB2]] : index) upperbound(%[[UB2]] : index) extent(%[[EXTENT_C10]] : index) stride(%[[ONE]] : index) startIdx(%c1{{.*}} : index)
+!CHECK: %[[COPYIN_A:.*]] = acc.copyin varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) bounds(%[[BOUND1]], %[[BOUND2]]) -> !fir.ref<!fir.array<10x10xf32>> {name = "a(1:,1:5)", structured = false}
+!CHECK: acc.enter_data   dataOperands(%[[COPYIN_A]] : !fir.ref<!fir.array<10x10xf32>>)
+
+  !$acc enter data copyin(a(:10,1:5))
+!CHECK: %[[LB:.*]] = arith.constant 0 : index
+!CHECK: %[[ONE:.*]] = arith.constant 1 : index
+!CHECK: %[[UB1:.*]] = arith.constant 9 : index
+!CHECK: %[[BOUND1:.*]] = acc.bounds lowerbound(%[[LB]] : index) upperbound(%[[UB1]] : index) extent(%[[C10]] : index) stride(%[[ONE]] : index) startIdx(%[[ONE]] : index)
+!CHECK: %[[LB:.*]] = arith.constant 0 : index
+!CHECK: %[[UB2:.*]] = arith.constant 4 : index
+!CHECK: %[[BOUND2:.*]] = acc.bounds lowerbound(%[[LB]] : index) upperbound(%[[UB2]] : index) extent(%[[EXTENT_C10]] : index) stride(%[[ONE]] : index) startIdx(%[[ONE]] : index)
+!CHECK: %[[COPYIN_A:.*]] = acc.copyin varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) bounds(%[[BOUND1]], %[[BOUND2]]) -> !fir.ref<!fir.array<10x10xf32>> {name = "a(:10,1:5)", structured = false}
+!CHECK: acc.enter_data dataOperands(%[[COPYIN_A]] : !fir.ref<!fir.array<10x10xf32>>)
+
+  !$acc enter data copyin(a(:,:))
+!CHECK: %[[LB:.*]] = arith.constant 0 : index
+!CHECK: %[[ONE:.*]] = arith.constant 1 : index
+!CHECK: %[[UB:.*]] = arith.subi %c10{{.*}}, %[[ONE]] : index
+!CHECK: %[[BOUND1:.*]] = acc.bounds lowerbound(%[[LB]] : index) upperbound(%[[UB]] : index) extent(%c10{{.*}} : index) stride(%[[ONE]] : index) startIdx(%[[ONE]] : index)
+!CHECK: %[[UB:.*]] = arith.subi %c10{{.*}}, %[[ONE]] : index
+!CHECK: %[[BOUND2:.*]] = acc.bounds lowerbound(%[[LB]] : index) upperbound(%[[UB]] : index) extent(%c10{{.*}} : index) stride(%[[ONE]] : index) startIdx(%[[ONE]] : index)
+!CHECK: %[[COPYIN_A:.*]] = acc.copyin varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) bounds(%[[BOUND1]], %[[BOUND2]]) -> !fir.ref<!fir.array<10x10xf32>> {name = "a(:,:)", structured = false}
+end subroutine acc_enter_data
+
+subroutine acc_enter_data_dummy(a, b, n, m)
+  integer :: n, m
+  real :: a(1:10)
+  real :: b(n:m)
+
+!CHECK-LABEL: func.func @_QPacc_enter_data_dummy
+!CHECK-SAME:    %[[A:.*]]: !fir.ref<!fir.array<10xf32>> {fir.bindc_name = "a"}, %[[B:.*]]: !fir.ref<!fir.array<?xf32>> {fir.bindc_name = "b"}, %[[N:.*]]: !fir.ref<i32> {fir.bindc_name = "n"}, %[[M:.*]]: !fir.ref<i32> {fir.bindc_name = "m"}
+!CHECK: %[[C10:.*]] = arith.constant 10 : index
+!CHECK: %[[DECLA:.*]]:2 = hlfir.declare %[[A]]
+!CHECK: %[[DECLN:.*]]:2 = hlfir.declare %[[N]]
+!CHECK: %[[DECLM:.*]]:2 = hlfir.declare %[[M]]
+!CHECK: %[[LOAD_N:.*]] = fir.load %[[DECLN]]#0 : !fir.ref<i32>
+!CHECK: %[[N_I64:.*]] = fir.convert %[[LOAD_N]] : (i32) -> i64
+!CHECK: %[[N_IDX:.*]] = fir.convert %[[N_I64]] : (i64) -> index
+!CHECK: %[[LOAD_M:.*]] = fir.load %[[DECLM]]#0 : !fir.ref<i32>
+!CHECK: %[[M_I64:.*]] = fir.convert %[[LOAD_M]] : (i32) -> i64
+!CHECK: %[[M_IDX:.*]] = fir.convert %[[M_I64]] : (i64) -> index
+!CHECK: %[[M_N:.*]] = arith.subi %[[M_IDX]], %[[N_IDX]] : index
+!CHECK: %[[C1:.*]] = arith.constant 1 : index
+!CHECK: %[[M_N_1:.*]] = arith.addi %[[M_N]], %[[C1]] : index
+!CHECK: %[[C0:.*]] = arith.constant 0 : index
+!CHECK: %[[CMP:.*]] = arith.cmpi sgt, %[[M_N_1]], %[[C0]] : index
+!CHECK: %[[EXT_B:.*]] = arith.select %[[CMP]], %[[M_N_1]], %[[C0]] : index
+!CHECK: %[[DECLB:.*]]:2 = hlfir.declare %[[B]]
+
+  !$acc enter data create(a)
+!CHECK: %[[BOUND:.*]] = acc.bounds lowerbound(%{{.*}} : index) upperbound(%{{.*}} : index) extent(%c10{{.*}} : index) stride(%c1{{.*}} : index) startIdx(%{{.*}} : index)
+!CHECK: %[[CREATE:.*]] = acc.create varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10xf32>>) bounds(%[[BOUND]]) -> !fir.ref<!fir.array<10xf32>> {name = "a", structured = false}
+!CHECK: acc.enter_data dataOperands(%[[CREATE]] : !fir.ref<!fir.array<10xf32>>)
+
+  !$acc enter data create(b)
+!CHECK: %[[DIMS:.*]]:3 = fir.box_dims %[[DECLB]]#0, %c0{{.*}} : (!fir.box<!fir.array<?xf32>>, index) -> (index, index, index)
+!CHECK: %[[BOUND:.*]] = acc.bounds lowerbound(%{{.*}} : index) upperbound(%{{.*}} : index) extent(%[[DIMS]]#1 : index) stride(%[[DIMS]]#2 : index) startIdx(%{{.*}} : index) {strideInBytes = true}
+!CHECK: %[[ADDR:.*]] = fir.box_addr %[[DECLB]]#0 : (!fir.box<!fir.array<?xf32>>) -> !fir.ref<!fir.array<?xf32>>
+!CHECK: %[[CREATE:.*]] = acc.create varPtr(%[[ADDR]] : !fir.ref<!fir.array<?xf32>>) bounds(%[[BOUND]]) -> !fir.ref<!fir.array<?xf32>> {name = "b", structured = false}
+!CHECK: acc.enter_data dataOperands(%[[CREATE]] : !fir.ref<!fir.array<?xf32>>)
+
+  !$acc enter data create(a(5:10))
+!CHECK: %[[ONE:.*]] = arith.constant 1 : index
+!CHECK: %[[LB1:.*]] = arith.constant 4 : index
+!CHECK: %[[UB1:.*]] = arith.constant 9 : index
+!CHECK: %[[BOUND1:.*]] = acc.bounds lowerbound(%[[LB1]] : index) upperbound(%[[UB1]] : index) extent(%c10{{.*}} : index) stride(%[[ONE]] : index) startIdx(%c1{{.*}} : index)
+!CHECK: %[[CREATE1:.*]] = acc.create varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10xf32>>) bounds(%[[BOUND1]]) -> !fir.ref<!fir.array<10xf32>> {name = "a(5:10)", structured = false}
+!CHECK: acc.enter_data dataOperands(%[[CREATE1]] : !fir.ref<!fir.array<10xf32>>)
+
+  !$acc enter data create(b(n:m))
+!CHECK: %[[DIMS0:.*]]:3 = fir.box_dims %[[DECLB]]#0, %c0{{.*}} : (!fir.box<!fir.array<?xf32>>, index) -> (index, index, index)
+!CHECK: %[[LOAD_N:.*]] = fir.load %[[DECLN]]#0 : !fir.ref<i32>
+!CHECK: %[[N_CONV1:.*]] = fir.convert %[[LOAD_N]] : (i32) -> i64
+!CHECK: %[[N_CONV2:.*]] = fir.convert %[[N_CONV1]] : (i64) -> index
+!CHECK: %[[LB:.*]] = arith.subi %[[N_CONV2]], %[[N_IDX]] : index
+!CHECK: %[[LOAD_M:.*]] = fir.load %[[DECLM]]#0 : !fir.ref<i32>
+!CHECK: %[[M_CONV1:.*]] = fir.convert %[[LOAD_M]] : (i32) -> i64
+!CHECK: %[[M_CONV2:.*]] = fir.convert %[[M_CONV1]] : (i64) -> index
+!CHECK: %[[UB:.*]] = arith.subi %[[M_CONV2]], %[[N_IDX]] : index
+!CHECK: %[[BOUND1:.*]] = acc.bounds lowerbound(%[[LB]] : index) upperbound(%[[UB]] : index) extent(%[[EXT_B]] : index) stride(%[[DIMS0]]#2 : index) startIdx(%[[N_IDX]] : index) {strideInBytes = true}
+!CHECK: %[[ADDR:.*]] = fir.box_addr %[[DECLB]]#0 : (!fir.box<!fir.array<?xf32>>) -> !fir.ref<!fir.array<?xf32>>
+!CHECK: %[[CREATE1:.*]] = acc.create varPtr(%[[ADDR]] : !fir.ref<!fir.array<?xf32>>) bounds(%[[BOUND1]]) -> !fir.ref<!fir.array<?xf32>> {name = "b(n:m)", structured = false}
+!CHECK: acc.enter_data dataOperands(%[[CREATE1]] : !fir.ref<!fir.array<?xf32>>)
+
+  !$acc enter data create(b(n:))
+!CHECK: %[[ONE:.*]] = arith.constant 1 : index
+!CHECK: %[[DIMS0:.*]]:3 = fir.box_dims %[[DECLB]]#0, %c0_8 : (!fir.box<!fir.array<?xf32>>, index) -> (index, index, index)
+!CHECK: %[[LOAD_N:.*]] = fir.load %[[DECLN]]#0 : !fir.ref<i32>
+!CHECK: %[[CONVERT1_N:.*]] = fir.convert %[[LOAD_N]] : (i32) -> i64
+!CHECK: %[[CONVERT2_N:.*]] = fir.convert %[[CONVERT1_N]] : (i64) -> index
+!CHECK: %[[LB:.*]] = arith.subi %[[CONVERT2_N]], %[[N_IDX]] : index
+!CHECK: %[[UB:.*]] = arith.subi %[[EXT_B]], %c1{{.*}} : index
+!CHECK: %[[BOUND1:.*]] = acc.bounds lowerbound(%[[LB]] : index) upperbound(%[[UB]] : index) extent(%[[EXT_B]] : index) stride(%[[DIMS0]]#2 : index) startIdx(%[[N_IDX]] : index) {strideInBytes = true}
+!CHECK: %[[ADDR:.*]] = fir.box_addr %[[DECLB]]#0 : (!fir.box<!fir.array<?xf32>>) -> !fir.ref<!fir.array<?xf32>>
+!CHECK: %[[CREATE1:.*]] = acc.create varPtr(%[[ADDR]] : !fir.ref<!fir.array<?xf32>>) bounds(%[[BOUND1]]) -> !fir.ref<!fir.array<?xf32>> {name = "b(n:)", structured = false}
+!CHECK: acc.enter_data dataOperands(%[[CREATE1]] : !fir.ref<!fir.array<?xf32>>)
+
+  !$acc enter data create(b(:))
+!CHECK: %[[ZERO:.*]] = arith.constant 0 : index
+!CHECK: %[[ONE:.*]] = arith.constant 1 : index
+!CHECK: %[[DIMS0:.*]]:3 = fir.box_dims %[[DECLB]]#0, %c0{{.*}} : (!fir.box<!fir.array<?xf32>>, index) -> (index, index, index)
+!CHECK: %[[UB:.*]] = arith.subi %[[EXT_B]], %[[ONE]] : index
+!CHECK: %[[BOUND1:.*]] = acc.bounds lowerbound(%[[ZERO]] : index) upperbound(%[[UB]] : index) extent(%[[EXT_B]] : index) stride(%[[DIMS0]]#2 : index) startIdx(%[[N_IDX]] : index) {strideInBytes = true}
+!CHECK: %[[ADDR:.*]] = fir.box_addr %[[DECLB]]#0 : (!fir.box<!fir.array<?xf32>>) -> !fir.ref<!fir.array<?xf32>>
+!CHECK: %[[CREATE1:.*]] = acc.create varPtr(%[[ADDR]] : !fir.ref<!fir.array<?xf32>>) bounds(%[[BOUND1]]) -> !fir.ref<!fir.array<?xf32>> {name = "b(:)", structured = false}
+!CHECK: acc.enter_data dataOperands(%[[CREATE1]] : !fir.ref<!fir.array<?xf32>>)
+
+end subroutine
+
+! Test lowering of array section for non default lower bound.
+subroutine acc_enter_data_non_default_lb()
+  integer :: a(0:9)
+  integer :: b(11:20)
+
+!CHECK-LABEL: func.func @_QPacc_enter_data_non_default_lb() {
+!CHECK: %[[BASELB:.*]] = arith.constant 0 : index
+!CHECK: %[[EXTENT_C10:.*]] = arith.constant 10 : index
+!CHECK: %[[A:.*]] = fir.alloca !fir.array<10xi32> {bindc_name = "a", uniq_name = "_QFacc_enter_data_non_default_lbEa"}
+!CHECK: %[[DECLA:.*]]:2 = hlfir.declare %[[A]]
+!CHECK: %[[B:.*]] = fir.alloca !fir.array<10xi32> {bindc_name = "b", uniq_name = "_QFacc_enter_data_non_default_lbEb"}
+!CHECK: %[[DECLB:.*]]:2 = hlfir.declare %[[B]]
+
+  !$acc enter data create(a(5:9))
+!CHECK: %[[SECTIONLB:.*]] = arith.constant 5 : index
+!CHECK: %[[LB:.*]] = arith.subi %[[SECTIONLB]], %[[BASELB]] : index
+!CHECK: %[[SECTIONUB:.*]] = arith.constant 9 : index
+!CHECK: %[[UB:.*]] = arith.subi %[[SECTIONUB]], %[[BASELB]] : index
+!CHECK: %[[BOUND:.*]] = acc.bounds lowerbound(%[[LB]] : index) upperbound(%[[UB]] : index) extent(%c10{{.*}} : index) stride(%{{.*}} : index) startIdx(%[[BASELB]] : index)
+!CHECK: %[[ADDR:.*]] = fir.box_addr %[[DECLA]]#0 : (!fir.box<!fir.array<10xi32>>) -> !fir.ref<!fir.array<10xi32>>
+!CHECK: %[[CREATE:.*]] = acc.create varPtr(%[[ADDR]] : !fir.ref<!fir.array<10xi32>>) bounds(%[[BOUND]]) -> !fir.ref<!fir.array<10xi32>> {name = "a(5:9)", structured = false}
+!CHECK: acc.enter_data dataOperands(%[[CREATE]] : !fir.ref<!fir.array<10xi32>>)
+
+  !$acc enter data create(a(:))
+!CHECK: %[[ZERO:.*]] = arith.constant 0 : index
+!CHECK: %[[ONE:.*]] = arith.constant 1 : index
+!CHECK: %[[UB:.*]] = arith.subi %[[EXTENT_C10]], %[[ONE]] : index
+!CHECK: %[[BOUND:.*]] = acc.bounds lowerbound(%[[ZERO]] : index) upperbound(%[[UB]] : index) extent(%[[EXTENT_C10]] : index) stride(%{{.*}} : index) startIdx(%[[BASELB]] : index)
+!CHECK: %[[ADDR:.*]] = fir.box_addr %[[DECLA]]#0 : (!fir.box<!fir.array<10xi32>>) -> !fir.ref<!fir.array<10xi32>>
+!CHECK: %[[CREATE:.*]] = acc.create varPtr(%[[ADDR]] : !fir.ref<!fir.array<10xi32>>) bounds(%[[BOUND]]) -> !fir.ref<!fir.array<10xi32>> {name = "a(:)", structured = false}
+!CHECK: acc.enter_data dataOperands(%[[CREATE]] : !fir.ref<!fir.array<10xi32>>)
+
+  !$acc enter data create(a(:6))
+!CHECK: %[[ZERO:.*]] = arith.constant 0 : index
+!CHECK: %[[SECTIONUB:.*]] = arith.constant 6 : index
+!CHECK: %[[UB:.*]] = arith.subi %[[SECTIONUB]], %[[BASELB]] : index
+!CHECK: %[[BOUND:.*]] = acc.bounds lowerbound(%[[ZERO]] : index) upperbound(%[[UB]] : index) extent(%c10{{.*}} : index) stride(%{{.*}} : index) startIdx(%[[BASELB]] : index)
+!CHECK: %[[ADDR:.*]] = fir.box_addr %[[DECLA]]#0 : (!fir.box<!fir.array<10xi32>>) -> !fir.ref<!fir.array<10xi32>>
+!CHECK: %[[CREATE:.*]] = acc.create varPtr(%[[ADDR]] : !fir.ref<!fir.array<10xi32>>) bounds(%[[BOUND]]) -> !fir.ref<!fir.array<10xi32>> {name = "a(:6)", structured = false}
+!CHECK: acc.enter_data dataOperands(%[[CREATE]] : !fir.ref<!fir.array<10xi32>>)
+
+  !$acc enter data create(a(4:))
+!CHECK: %[[ONE:.*]] = arith.constant 1 : index
+!CHECK: %[[SECTIONLB:.*]] = arith.constant 4 : index
+!CHECK: %[[LB:.*]] = arith.subi %[[SECTIONLB]], %[[BASELB]] : index
+!CHECK: %[[UB:.*]] = arith.subi %[[EXTENT_C10]], %[[ONE]] : index
+!CHECK: %[[BOUND:.*]] = acc.bounds lowerbound(%[[LB]] : index) upperbound(%[[UB]] : index) extent(%[[EXTENT_C10]] : index) stride(%{{.*}} : index) startIdx(%[[BASELB]] : index)
+!CHECK: %[[ADDR:.*]] = fir.box_addr %[[DECLA]]#0 : (!fir.box<!fir.array<10xi32>>) -> !fir.ref<!fir.array<10xi32>>
+!CHECK: %[[CREATE:.*]] = acc.create varPtr(%[[ADDR]] : !fir.ref<!fir.array<10xi32>>) bounds(%[[BOUND]]) -> !fir.ref<!fir.array<10xi32>> {name = "a(4:)", structured = false}
+!CHECK: acc.enter_data dataOperands(%[[CREATE]] : !fir.ref<!fir.array<10xi32>>)
+
+  !$acc enter data create(b)
+!CHECK: %[[ONE:.*]] = arith.constant 1 : index
+!CHECK: %[[DIMS0:.*]]:3 = fir.box_dims %[[DECLB]]#0, %c0{{.*}} : (!fir.box<!fir.array<10xi32>>, index) -> (index, index, index)
+!CHECK: %[[LB:.*]] = arith.constant 0 : index
+!CHECK: %[[UB:.*]] = arith.subi %[[DIMS0]]#1, %[[ONE]] : index
+!CHECK: %[[BOUND:.*]] = acc.bounds lowerbound(%c0{{.*}} : index) upperbound(%[[UB]] : index) extent(%[[DIMS0]]#1 : index) stride(%{{.*}} : index) startIdx(%c11{{.*}} : index) {strideInBytes = true}
+!CHECK: %[[ADDR:.*]] = fir.box_addr %[[DECLB]]#0 : (!fir.box<!fir.array<10xi32>>) -> !fir.ref<!fir.array<10xi32>>
+!CHECK: %[[CREATE:.*]] = acc.create varPtr(%[[ADDR]] : !fir.ref<!fir.array<10xi32>>) bounds(%[[BOUND]]) -> !fir.ref<!fir.array<10xi32>> {name = "b", structured = false}
+!CHECK: acc.enter_data dataOperands(%[[CREATE]] : !fir.ref<!fir.array<10xi32>>)
+
+end subroutine
+
+! Test lowering of assumed size arrays.
+subroutine acc_enter_data_assumed(a, b, n, m)
+  integer :: n, m
+  real :: a(:)
+  real :: b(10:)
+
+!CHECK-LABEL: func.func @_QPacc_enter_data_assumed(
+!CHECK-SAME: %[[A:.*]]: !fir.box<!fir.array<?xf32>> {fir.bindc_name = "a"}, %[[B:.*]]: !fir.box<!fir.array<?xf32>> {fir.bindc_name = "b"}, %[[N:.*]]: !fir.ref<i32> {fir.bindc_name = "n"}, %[[M:.*]]: !fir.ref<i32> {fir.bindc_name = "m"}) {
+!CHECK: %[[DECLA:.*]]:2 = hlfir.declare %[[A]]
+!CHECK: %[[LB_C10:.*]] = arith.constant 10 : i64
+!CHECK: %[[LB_C10_IDX:.*]] = fir.convert %[[LB_C10]] : (i64) -> index
+!CHECK: %[[DECLB:.*]]:2 = hlfir.declare %[[B]]
+!CHECK: %[[DECLM:.*]]:2 = hlfir.declare %[[M]]
+!CHECK: %[[DECLN:.*]]:2 = hlfir.declare %[[N]]
+
+  !$acc enter data create(a)
+!CHECK: %[[C1:.*]] = arith.constant 1 : index
+!CHECK: %[[C0:.*]] = arith.constant 0 : index
+!CHECK: %[[DIMS:.*]]:3 = fir.box_dims %[[DECLA]]#0, %[[C0]] : (!fir.box<!fir.array<?xf32>>, index) -> (index, index, index)
+!CHECK: %[[LB:.*]] = arith.constant 0 : index
+!CHECK: %[[UB:.*]] = arith.subi %[[DIMS]]#1, %[[C1]] : index
+!CHECK: %[[BOUND:.*]] = acc.bounds lowerbound(%[[LB]] : index) upperbound(%[[UB]] : index) extent(%[[DIMS]]#1 : index) stride(%[[DIMS]]#2 : index) startIdx(%[[C1]] : index) {strideInBytes = true}
+!CHECK: %[[BOX_ADDR:.*]] = fir.box_addr %[[DECLA]]#0 : (!fir.box<!fir.array<?xf32>>) -> !fir.ref<!fir.array<?xf32>>
+!CHECK: %[[CREATE:.*]] = acc.create varPtr(%[[BOX_ADDR]] : !fir.ref<!fir.array<?xf32>>) bounds(%[[BOUND]]) -> !fir.ref<!fir.array<?xf32>> {name = "a", structured = false}
+!CHECK: acc.enter_data dataOperands(%[[CREATE]] : !fir.ref<!fir.array<?xf32>>)
+
+  !$acc enter data create(a(:))
+!CHECK: %[[LB:.*]] = arith.constant 0 : index
+!CHECK: %[[ONE:.*]] = arith.constant 1 : index
+!CHECK: %[[C0:.*]] = arith.constant 0 : index
+
+!CHECK: %[[DIMS0:.*]]:3 = fir.box_dims %[[DECLA]]#0, %[[C0]] : (!fir.box<!fir.array<?xf32>>, index) -> (index, index, index)
+!CHECK: %[[C0:.*]] = arith.constant 0 : index
+
+!CHECK: %[[DIMS1:.*]]:3 = fir.box_dims %[[DECLA]]#1, %[[C0]] : (!fir.box<!fir.array<?xf32>>, index) -> (index, index, index)
+!CHECK: %[[UB:.*]] = arith.subi %[[DIMS1]]#1, %[[ONE]] : index
+!CHECK: %[[BOUND:.*]] = acc.bounds lowerbound(%[[LB]] : index) upperbound(%[[UB]] : index) extent(%[[DIMS1]]#1 : index) stride(%[[DIMS0]]#2 : index) startIdx(%[[ONE]] : index) {strideInBytes = true}
+
+!CHECK: %[[BOX_ADDR:.*]] = fir.box_addr %[[DECLA]]#0 : (!fir.box<!fir.array<?xf32>>) -> !fir.ref<!fir.array<?xf32>>
+!CHECK: %[[CREATE:.*]] = acc.create varPtr(%[[BOX_ADDR]] : !fir.ref<!fir.array<?xf32>>) bounds(%[[BOUND]]) -> !fir.ref<!fir.array<?xf32>> {name = "a(:)", structured = false}
+!CHECK: acc.enter_data dataOperands(%[[CREATE]] : !fir.ref<!fir.array<?xf32>>)
+
+  !$acc enter data create(a(2:))
+!CHECK: %[[ONE:.*]] = arith.constant 1 : index
+!CHECK: %[[C0:.*]] = arith.constant 0 : index
+
+!CHECK: %[[DIMS0:.*]]:3 = fir.box_dims %[[DECLA]]#0, %[[C0]] : (!fir.box<!fir.array<?xf32>>, index) -> (index, index, index)
+!CHECK: %[[LB:.*]] = arith.constant 1 : index
+!CHECK: %[[C0:.*]] = arith.constant 0 : index
+
+!CHECK: %[[DIMS1:.*]]:3 = fir.box_dims %[[DECLA]]#1, %[[C0]] : (!fir.box<!fir.array<?xf32>>, index) -> (index, index, index)
+!CHECK: %[[UB:.*]] = arith.subi %[[DIMS1]]#1, %[[ONE]] : index
+!CHECK: %[[BOUND:.*]] = acc.bounds lowerbound(%[[LB]] : index) upperbound(%[[UB]] : index) extent(%[[DIMS1]]#1 : index) stride(%[[DIMS0]]#2 : index) startIdx(%[[ONE]] : index) {strideInBytes = true}
+
+!CHECK: %[[BOX_ADDR:.*]] = fir.box_addr %[[DECLA]]#0 : (!fir.box<!fir.array<?xf32>>) -> !fir.ref<!fir.array<?xf32>>
+!CHECK: %[[CREATE:.*]] = acc.create varPtr(%[[BOX_ADDR]] : !fir.ref<!fir.array<?xf32>>) bounds(%[[BOUND]]) -> !fir.ref<!fir.array<?xf32>> {name = "a(2:)", structured = false}
+!CHECK: acc.enter_data dataOperands(%[[CREATE]] : !fir.ref<!fir.array<?xf32>>)
+
+  !$acc enter data create(a(:4))
+!CHECK: %[[LB:.*]] = arith.constant 0 : index
+!CHECK: %[[ONE:.*]] = arith.constant 1 : index
+!CHECK: %[[C0:.*]] = arith.constant 0 : index
+
+!CHECK: %[[DIMS0:.*]]:3 = fir.box_dims %[[DECLA]]#0, %[[C0]] : (!fir.box<!fir.array<?xf32>>, index) -> (index, index, index)
+!CHECK: %[[UB:.*]] = arith.constant 3 : index
+!CHECK: %[[DIMS1:.*]]:3 = fir.box_dims %[[DECLA]]#1, %{{.*}} : (!fir.box<!fir.array<?xf32>>, index) -> (index, index, index)
+!CHECK: %[[BOUND:.*]] = acc.bounds lowerbound(%[[LB]] : index) upperbound(%[[UB]] : index) extent(%[[DIMS1]]#1 : index) stride(%[[DIMS0]]#2 : index) startIdx(%[[ONE]] : index) {strideInBytes = true}
+
+!CHECK: %[[BOX_ADDR:.*]] = fir.box_addr %[[DECLA]]#0 : (!fir.box<!fir.array<?xf32>>) -> !fir.ref<!fir.array<?xf32>>
+!CHECK: %[[CREATE:.*]] = acc.create varPtr(%[[BOX_ADDR]] : !fir.ref<!fir.array<?xf32>>) bounds(%[[BOUND]]) -> !fir.ref<!fir.array<?xf32>> {name = "a(:4)", structured = false}
+!CHECK: acc.enter_data dataOperands(%[[CREATE]] : !fir.ref<!fir.array<?xf32>>)
+
+  !$acc enter data create(a(6:10))
+!CHECK: %[[ONE:.*]] = arith.constant 1 : index
+!CHECK: %[[C0:.*]] = arith.constant 0 : index
+
+!CHECK: %[[DIMS0:.*]]:3 = fir.box_dims %[[DECLA]]#0, %[[C0]] : (!fir.box<!fir.array<?xf32>>, index) -> (index, index, index)
+!CHECK: %[[LB:.*]] = arith.constant 5 : index
+!CHECK: %[[UB:.*]] = arith.constant 9 : index
+!CHECK: %[[DIMS1:.*]]:3 = fir.box_dims %[[DECLA]]#1, %{{.*}} : (!fir.box<!fir.array<?xf32>>, index) -> (index, index, index)
+!CHECK: %[[BOUND:.*]] = acc.bounds lowerbound(%[[LB]] : index) upperbound(%[[UB]] : index) extent(%[[DIMS1]]#1 : index) stride(%[[DIMS0]]#2 : index) startIdx(%[[ONE]] : index) {strideInBytes = true}
+
+!CHECK: %[[BOX_ADDR:.*]] = fir.box_addr %[[DECLA]]#0 : (!fir.box<!fir.array<?xf32>>) -> !fir.ref<!fir.array<?xf32>>
+!CHECK: %[[CREATE:.*]] = acc.create varPtr(%[[BOX_ADDR]] : !fir.ref<!fir.array<?xf32>>) bounds(%[[BOUND]]) -> !fir.ref<!fir.array<?xf32>> {name = "a(6:10)", structured = false}
+!CHECK: acc.enter_data dataOperands(%[[CREATE]] : !fir.ref<!fir.array<?xf32>>)
+
+  !$acc enter data create(a(n:))
+!CHECK: %[[ONE:.*]] = arith.constant 1 : index
+!CHECK: %[[C0:.*]] = arith.constant 0 : index
+
+!CHECK: %[[DIMS0:.*]]:3 = fir.box_dims %[[DECLA]]#0, %[[C0]] : (!fir.box<!fir.array<?xf32>>, index) -> (index, index, index)
+
+!CHECK: %[[LOAD_N:.*]] = fir.load %[[DECLN]]#0 : !fir.ref<i32>
+!CHECK: %[[CONVERT1_N:.*]] = fir.convert %[[LOAD_N]] : (i32) -> i64
+!CHECK: %[[CONVERT2_N:.*]] = fir.convert %[[CONVERT1_N]] : (i64) -> index
+!CHECK: %[[LB:.*]] = arith.subi %[[CONVERT2_N]], %[[ONE]] : index
+!CHECK: %[[C0:.*]] = arith.constant 0 : index
+
+!CHECK: %[[DIMS:.*]]:3 = fir.box_dims %[[DECLA]]#1, %[[C0]] : (!fir.box<!fir.array<?xf32>>, index) -> (index, index, index)
+!CHECK: %[[UB:.*]] = arith.subi %[[DIMS]]#1, %[[ONE]] : index
+!CHECK: %[[BOUND:.*]] = acc.bounds lowerbound(%[[LB]] : index) upperbound(%[[UB]] : index) extent(%[[DIMS]]#1 : index) stride(%[[DIMS0]]#2 : index) startIdx(%[[ONE]] : index) {strideInBytes = true}
+
+!CHECK: %[[BOX_ADDR:.*]] = fir.box_addr %[[DECLA]]#0 : (!fir.box<!fir.array<?xf32>>) -> !fir.ref<!fir.array<?xf32>>
+!CHECK: %[[CREATE:.*]] = acc.create varPtr(%[[BOX_ADDR]] : !fir.ref<!fir.array<?xf32>>) bounds(%[[BOUND]]) -> !fir.ref<!fir.array<?xf32>> {name = "a(n:)", structured = false}
+!CHECK: acc.enter_data dataOperands(%[[CREATE]] : !fir.ref<!fir.array<?xf32>>)
+
+  !$acc enter data create(a(:m))
+!CHECK: %[[BASELB:.*]] = arith.constant 0 : index
+!CHECK: %[[ONE:.*]] = arith.constant 1 : index
+!CHECK: %[[C0:.*]] = arith.constant 0 : index
+
+!CHECK: %[[DIMS0:.*]]:3 = fir.box_dims %[[DECLA]]#0, %[[C0]] : (!fir.box<!fir.array<?xf32>>, index) -> (index, index, index)
+
+!CHECK: %[[LOAD_M:.*]] = fir.load %[[DECLM]]#0 : !fir.ref<i32>
+!CHECK: %[[CONVERT1_M:.*]] = fir.convert %[[LOAD_M]] : (i32) -> i64
+!CHECK: %[[CONVERT2_M:.*]] = fir.convert %[[CONVERT1_M]] : (i64) -> index
+!CHECK: %[[UB:.*]] = arith.subi %[[CONVERT2_M]], %[[ONE]] : index
+!CHECK: %[[DIMS1:.*]]:3 = fir.box_dims %[[DECLA]]#1, %{{.*}} : (!fir.box<!fir.array<?xf32>>, index) -> (index, index, index)
+!CHECK: %[[BOUND:.*]] = acc.bounds lowerbound(%[[BASELB]] : index) upperbound(%[[UB]] : index) extent(%[[DIMS1]]#1 : index) stride(%[[DIMS0]]#2 : index) startIdx(%[[ONE]] : index) {strideInBytes = true}
+
+!CHECK: %[[BOX_ADDR:.*]] = fir.box_addr %[[DECLA]]#0 : (!fir.box<!fir.array<?xf32>>) -> !fir.ref<!fir.array<?xf32>>
+!CHECK: %[[CREATE:.*]] = acc.create varPtr(%[[BOX_ADDR]] : !fir.ref<!fir.array<?xf32>>) bounds(%[[BOUND]]) -> !fir.ref<!fir.array<?xf32>> {name = "a(:m)", structured = false}
+!CHECK: acc.enter_data dataOperands(%[[CREATE]] : !fir.ref<!fir.array<?xf32>>)
+
+  !$acc enter data create(a(n:m))
+!CHECK: %[[ONE:.*]] = arith.constant 1 : index
+!CHECK: %[[C0:.*]] = arith.constant 0 : index
+
+!CHECK: %[[DIMS0:.*]]:3 = fir.box_dims %[[DECLA]]#0, %[[C0]] : (!fir.box<!fir.array<?xf32>>, index) -> (index, index, index)
+
+!CHECK: %[[LOAD_N:.*]] = fir.load %[[DECLN]]#0 : !fir.ref<i32>
+!CHECK: %[[CONVERT1_N:.*]] = fir.convert %[[LOAD_N]] : (i32) -> i64
+!CHECK: %[[CONVERT2_N:.*]] = fir.convert %[[CONVERT1_N]] : (i64) -> index
+!CHECK: %[[LB:.*]] = arith.subi %[[CONVERT2_N]], %[[ONE]] : index
+
+!CHECK: %[[LOAD_M:.*]] = fir.load %[[DECLM]]#0 : !fir.ref<i32>
+!CHECK: %[[CONVERT1_M:.*]] = fir.convert %[[LOAD_M]] : (i32) -> i64
+!CHECK: %[[CONVERT2_M:.*]] = fir.convert %[[CONVERT1_M]] : (i64) -> index
+!CHECK: %[[UB:.*]] = arith.subi %[[CONVERT2_M]], %[[ONE]] : index
+!CHECK: %[[DIMS1:.*]]:3 = fir.box_dims %[[DECLA]]#1, %{{.*}} : (!fir.box<!fir.array<?xf32>>, index) -> (index, index, index)
+!CHECK: %[[BOUND:.*]] = acc.bounds lowerbound(%[[LB]] : index) upperbound(%[[UB]] : index) extent(%[[DIMS1]]#1 : index) stride(%[[DIMS0]]#2 : index) startIdx(%[[ONE]] : index) {strideInBytes = true}
+
+!CHECK: %[[BOX_ADDR:.*]] = fir.box_addr %[[DECLA]]#0 : (!fir.box<!fir.array<?xf32>>) -> !fir.ref<!fir.array<?xf32>>
+!CHECK: %[[CREATE:.*]] = acc.create varPtr(%[[BOX_ADDR]] : !fir.ref<!fir.array<?xf32>>) bounds(%[[BOUND]]) -> !fir.ref<!fir.array<?xf32>> {name = "a(n:m)", structured = false}
+!CHECK: acc.enter_data dataOperands(%[[CREATE]] : !fir.ref<!fir.array<?xf32>>)
+
+  !$acc enter data create(b(:m))
+!CHECK: %[[ZERO:.*]] = arith.constant 0 : index
+!CHECK: %[[C0:.*]] = arith.constant 0 : index
+
+!CHECK: %[[DIMS0:.*]]:3 = fir.box_dims %[[DECLB]]#0, %[[C0]] : (!fir.box<!fir.array<?xf32>>, index) -> (index, index, index)
+
+!CHECK: %[[LOAD_M:.*]] = fir.load %[[DECLM]]#0 : !fir.ref<i32>
+!CHECK: %[[CONVERT1_M:.*]] = fir.convert %[[LOAD_M]] : (i32) -> i64
+!CHECK: %[[CONVERT2_M:.*]] = fir.convert %[[CONVERT1_M]] : (i64) -> index
+!CHECK: %[[UB:.*]] = arith.subi %[[CONVERT2_M]], %[[LB_C10_IDX]] : index
+!CHECK: %[[DIMS1:.*]]:3 = fir.box_dims %[[DECLB]]#1, %{{.*}} : (!fir.box<!fir.array<?xf32>>, index) -> (index, index, index)
+!CHECK: %[[BOUND:.*]] = acc.bounds lowerbound(%[[ZERO]] : index) upperbound(%[[UB]] : index) extent(%[[DIMS1]]#1 : index) stride(%[[DIMS0]]#2 : index) startIdx(%[[LB_C10_IDX]] : index) {strideInBytes = true}
+
+!CHECK: %[[BOX_ADDR:.*]] = fir.box_addr %[[DECLB]]#0 : (!fir.box<!fir.array<?xf32>>) -> !fir.ref<!fir.array<?xf32>>
+!CHECK: %[[CREATE:.*]] = acc.create varPtr(%[[BOX_ADDR]] : !fir.ref<!fir.array<?xf32>>) bounds(%[[BOUND]]) -> !fir.ref<!fir.array<?xf32>> {name = "b(:m)", structured = false}
+!CHECK: acc.enter_data dataOperands(%[[CREATE]] : !fir.ref<!fir.array<?xf32>>)
+
+  !$acc enter data create(b)
+!CHECK: %[[ONE:.*]] = arith.constant 1 : index
+!CHECK: %[[C0:.*]] = arith.constant 0 : index
+
+!CHECK: %[[DIMS0:.*]]:3 = fir.box_dims %[[DECLB]]#0, %[[C0]] : (!fir.box<!fir.array<?xf32>>, index) -> (index, index, index)
+!CHECK: %[[C0:.*]] = arith.constant 0 : index
+!CHECK: %[[UB:.*]] = arith.subi %[[DIMS0]]#1, %[[ONE]] : index
+!CHECK: %[[BOUND:.*]] = acc.bounds lowerbound(%[[C0]] : index) upperbound(%[[UB]] : index) extent(%[[DIMS0]]#1 : index) stride(%[[DIMS0]]#2 : index) startIdx(%[[LB_C10_IDX]] : index) {strideInBytes = true}
+
+!CHECK: %[[BOX_ADDR:.*]] = fir.box_addr %[[DECLB]]#0 : (!fir.box<!fir.array<?xf32>>) -> !fir.ref<!fir.array<?xf32>>
+!CHECK: %[[CREATE:.*]] = acc.create varPtr(%[[BOX_ADDR]] : !fir.ref<!fir.array<?xf32>>) bounds(%[[BOUND]]) -> !fir.ref<!fir.array<?xf32>> {name = "b", structured = false}
+!CHECK: acc.enter_data dataOperands(%[[CREATE]] : !fir.ref<!fir.array<?xf32>>)
+
+end subroutine
+
+subroutine acc_enter_data_allocatable()
+  real, allocatable :: a(:)
+  integer, allocatable :: i
+
+!CHECK-LABEL: func.func @_QPacc_enter_data_allocatable() {
+!CHECK: %[[A:.*]] = fir.alloca !fir.box<!fir.heap<!fir.array<?xf32>>> {bindc_name = "a", uniq_name = "_QFacc_enter_data_allocatableEa"}
+!CHECK: %[[DECLA:.*]]:2 = hlfir.declare %[[A]]
+!CHECK: %[[I:.*]] = fir.alloca !fir.box<!fir.heap<i32>> {bindc_name = "i", uniq_name = "_QFacc_enter_data_allocatableEi"}
+!CHECK: %[[DECLI:.*]]:2 = hlfir.declare %[[I]]
+
+  !$acc enter data create(a)
+
+!CHECK: %[[BOX_A_0:.*]] = fir.load %[[DECLA]]#0 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>
+!CHECK: %[[C0_0:.*]] = arith.constant 0 : index
+!CHECK: %[[BOX_A_1:.*]] = fir.load %[[DECLA]]#1 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>
+!CHECK: %[[C0_1:.*]] = arith.constant 0 : index
+!CHECK: %[[DIMS0:.*]]:3 = fir.box_dims %[[BOX_A_1]], %c0{{.*}} : (!fir.box<!fir.heap<!fir.array<?xf32>>>, index) -> (index, index, index)
+!CHECK: %[[DIMS1:.*]]:3 = fir.box_dims %[[BOX_A_0]], %c0{{.*}} : (!fir.box<!fir.heap<!fir.array<?xf32>>>, index) -> (index, index, index)
+!CHECK: %[[UB:.*]] = arith.subi %[[DIMS1]]#1, %c1{{.*}} : index
+!CHECK: %[[BOUND:.*]] = acc.bounds lowerbound(%c0{{.*}} : index) upperbound(%[[UB]] : index) extent(%[[DIMS1]]#1 : index) stride(%[[DIMS1]]#2 : index) startIdx(%[[DIMS0]]#0 : index) {strideInBytes = true}
+!CHECK: %[[BOX_ADDR:.*]] = fir.box_addr %[[BOX_A_0]] : (!fir.box<!fir.heap<!fir.array<?xf32>>>) -> !fir.heap<!fir.array<?xf32>>
+!CHECK: %[[CREATE:.*]] = acc.create varPtr(%[[BOX_ADDR]] : !fir.heap<!fir.array<?xf32>>) bounds(%[[BOUND]]) -> !fir.heap<!fir.array<?xf32>> {name = "a", structured = false}
+!CHECK: acc.enter_data dataOperands(%[[CREATE]] : !fir.heap<!fir.array<?xf32>>)
+
+  !$acc enter data create(a(:))
+
+!CHECK: %[[BOX_A_0:.*]] = fir.load %[[DECLA]]#0 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>
+!CHECK: %[[ZERO:.*]] = arith.constant 0 : index
+!CHECK: %[[ONE:.*]] = arith.constant 1 : index
+
+!CHECK: %[[BOX_A_1:.*]] = fir.load %[[DECLA]]#1 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>
+!CHECK: %[[C0:.*]] = arith.constant 0 : index
+!CHECK: %[[DIMS0:.*]]:3 = fir.box_dims %[[BOX_A_1]], %[[C0]] : (!fir.box<!fir.heap<!fir.array<?xf32>>>, index) -> (index, index, index)
+!CHECK: %[[C0:.*]] = arith.constant 0 : index
+!CHECK: %[[DIMS1:.*]]:3 = fir.box_dims %[[BOX_A_0]], %[[C0]] : (!fir.box<!fir.heap<!fir.array<?xf32>>>, index) -> (index, index, index)
+
+!CHECK: %[[BOX_A_2:.*]] = fir.load %[[DECLA]]#1 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>
+!CHECK: %[[C0:.*]] = arith.constant 0 : index
+!CHECK: %[[DIMS2:.*]]:3 = fir.box_dims %[[BOX_A_2]], %[[C0]] : (!fir.box<!fir.heap<!fir.array<?xf32>>>, index) -> (index, index, index)
+!CHECK: %[[UB:.*]] = arith.subi %[[DIMS2]]#1, %[[ONE]] : index
+!CHECK: %[[BOUND:.*]] = acc.bounds lowerbound(%[[ZERO]] : index) upperbound(%[[UB:.*]] : index) extent(%[[DIMS2]]#1 : index) stride(%[[DIMS1]]#2 : index) startIdx(%[[DIMS0]]#0 : index) {strideInBytes = true}
+!CHECK: %[[BOX_ADDR:.*]] = fir.box_addr %[[BOX_A_0]] : (!fir.box<!fir.heap<!fir.array<?xf32>>>) -> !fir.heap<!fir.array<?xf32>>
+!CHECK: %[[CREATE:.*]] = acc.create varPtr(%[[BOX_ADDR]] : !fir.heap<!fir.array<?xf32>>) bounds(%[[BOUND]]) -> !fir.heap<!fir.array<?xf32>> {name = "a(:)", structured = false}
+!CHECK: acc.enter_data dataOperands(%[[CREATE]] : !fir.heap<!fir.array<?xf32>>)
+
+  !$acc enter data create(a(2:5))
+
+!CHECK: %[[BOX_A_0:.*]] = fir.load %[[DECLA]]#0 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>
+
+!CHECK: %[[BOX_A_1:.*]] = fir.load %[[DECLA]]#1 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>
+!CHECK: %[[C0:.*]] = arith.constant 0 : index
+!CHECK: %[[DIMS0:.*]]:3 = fir.box_dims %[[BOX_A_1]], %[[C0]] : (!fir.box<!fir.heap<!fir.array<?xf32>>>, index) -> (index, index, index)
+!CHECK: %[[C0:.*]] = arith.constant 0 : index
+!CHECK: %[[DIMS1:.*]]:3 = fir.box_dims %[[BOX_A_0]], %[[C0]] : (!fir.box<!fir.heap<!fir.array<?xf32>>>, index) -> (index, index, index)
+!CHECK: %[[C2:.*]] = arith.constant 2 : index
+!CHECK: %[[LB:.*]] = arith.subi %[[C2]], %[[DIMS0]]#0 : index
+!CHECK: %[[C5:.*]] = arith.constant 5 : index
+!CHECK: %[[UB:.*]] = arith.subi %[[C5]], %[[DIMS0]]#0 : index
+!CHECK: %[[BOX_A_2:.*]] = fir.load %[[DECLA]]#1 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>
+!CHECK: %[[C0:.*]] = arith.constant 0 : index
+!CHECK: %[[DIMS2:.*]]:3 = fir.box_dims %[[BOX_A_2]], %[[C0]] : (!fir.box<!fir.heap<!fir.array<?xf32>>>, index) -> (index, index, index)
+!CHECK: %[[BOUND:.*]] = acc.bounds lowerbound(%[[LB]] : index) upperbound(%[[UB]] : index) extent(%[[DIMS2]]#1 : index) stride(%[[DIMS1]]#2 : index) startIdx(%[[DIMS0]]#0 : index) {strideInBytes = true}
+!CHECK: %[[BOX_ADDR:.*]] = fir.box_addr %[[BOX_A_0]] : (!fir.box<!fir.heap<!fir.array<?xf32>>>) -> !fir.heap<!fir.array<?xf32>>
+!CHECK: %[[CREATE:.*]] = acc.create varPtr(%[[BOX_ADDR]] : !fir.heap<!fir.array<?xf32>>) bounds(%[[BOUND]]) -> !fir.heap<!fir.array<?xf32>> {name = "a(2:5)", structured = false}
+!CHECK: acc.enter_data dataOperands(%[[CREATE]] : !fir.heap<!fir.array<?xf32>>)
+
+  !$acc enter data create(a(3:))
+
+!CHECK: %[[BOX_A_0:.*]] = fir.load %[[DECLA]]#0 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>
+!CHECK: %[[ONE:.*]] = arith.constant 1 : index
+
+!CHECK: %[[BOX_A_1:.*]] = fir.load %[[DECLA]]#1 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>
+!CHECK: %[[C0:.*]] = arith.constant 0 : index
+!CHECK: %[[DIMS0:.*]]:3 = fir.box_dims %[[BOX_A_1]], %[[C0]] : (!fir.box<!fir.heap<!fir.array<?xf32>>>, index) -> (index, index, index)
+!CHECK: %[[C0:.*]] = arith.constant 0 : index
+!CHECK: %[[DIMS1:.*]]:3 = fir.box_dims %[[BOX_A_0]], %[[C0]] : (!fir.box<!fir.heap<!fir.array<?xf32>>>, index) -> (index, index, index)
+!CHECK: %[[C3:.*]] = arith.constant 3 : index
+!CHECK: %[[LB:.*]] = arith.subi %[[C3]], %[[DIMS0]]#0 : index
+
+!CHECK: %[[BOX_A_2:.*]] = fir.load %[[DECLA]]#1 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>
+!CHECK: %[[C0:.*]] = arith.constant 0 : index
+!CHECK: %[[DIMS2:.*]]:3 = fir.box_dims %[[BOX_A_2]], %[[C0]] : (!fir.box<!fir.heap<!fir.array<?xf32>>>, index) -> (index, index, index)
+!CHECK: %[[UB:.*]] = arith.subi %[[DIMS2]]#1, %[[ONE]] : index
+!CHECK: %[[BOUND:.*]] = acc.bounds lowerbound(%[[LB]] : index) upperbound(%[[UB]] : index) extent(%[[DIMS2]]#1 : index) stride(%[[DIMS1]]#2 : index) startIdx(%[[DIMS0]]#0 : index) {strideInBytes = true}
+!CHECK: %[[BOX_ADDR:.*]] = fir.box_addr %[[BOX_A_0]] : (!fir.box<!fir.heap<!fir.array<?xf32>>>) -> !fir.heap<!fir.array<?xf32>>
+!CHECK: %[[CREATE:.*]] = acc.create varPtr(%[[BOX_ADDR]] : !fir.heap<!fir.array<?xf32>>) bounds(%[[BOUND]]) -> !fir.heap<!fir.array<?xf32>> {name = "a(3:)", structured = false}
+!CHECK: acc.enter_data dataOperands(%[[CREATE]] : !fir.heap<!fir.array<?xf32>>)
+
+  !$acc enter data create(a(:7))
+
+!CHECK: %[[BOX_A_0:.*]] = fir.load %[[DECLA]]#0 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>
+!CHECK: %[[ZERO:.*]] = arith.constant 0 : index
+
+!CHECK: %[[BOX_A_1:.*]] = fir.load %[[DECLA]]#1 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>
+!CHECK: %[[C0:.*]] = arith.constant 0 : index
+!CHECK: %[[DIMS0:.*]]:3 = fir.box_dims %[[BOX_A_1]], %[[C0]] : (!fir.box<!fir.heap<!fir.array<?xf32>>>, index) -> (index, index, index)
+!CHECK: %[[C0:.*]] = arith.constant 0 : index
+!CHECK: %[[DIMS1:.*]]:3 = fir.box_dims %[[BOX_A_0]], %[[C0]] : (!fir.box<!fir.heap<!fir.array<?xf32>>>, index) -> (index, index, index)
+!CHECK: %[[C7:.*]] = arith.constant 7 : index
+!CHECK: %[[UB:.*]] = arith.subi %[[C7]], %[[DIMS0]]#0 : index
+!CHECK: %[[BOX_A_2:.*]] = fir.load %[[DECLA]]#1 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>
+!CHECK: %[[C0:.*]] = arith.constant 0 : index
+!CHECK: %[[DIMS2:.*]]:3 = fir.box_dims %[[BOX_A_2]], %[[C0]] : (!fir.box<!fir.heap<!fir.array<?xf32>>>, index) -> (index, index, index)
+!CHECK: %[[BOUND:.*]] = acc.bounds lowerbound(%[[ZERO]] : index) upperbound(%[[UB]] : index) extent(%[[DIMS2]]#1 : index) stride(%[[DIMS1]]#2 : index) startIdx(%[[DIMS0]]#0 : index) {strideInBytes = true}
+!CHECK: %[[BOX_ADDR:.*]] = fir.box_addr %[[BOX_A_0]] : (!fir.box<!fir.heap<!fir.array<?xf32>>>) -> !fir.heap<!fir.array<?xf32>>
+!CHECK: %[[CREATE:.*]] = acc.create varPtr(%[[BOX_ADDR]] : !fir.heap<!fir.array<?xf32>>) bounds(%[[BOUND]]) -> !fir.heap<!fir.array<?xf32>> {name = "a(:7)", structured = false}
+!CHECK: acc.enter_data dataOperands(%[[CREATE]] : !fir.heap<!fir.array<?xf32>>)
+
+  !$acc enter data create(i)
+
+!CHECK: %[[BOX_I:.*]] = fir.load %[[DECLI]]#0 : !fir.ref<!fir.box<!fir.heap<i32>>>
+!CHECK: %[[BOX_ADDR:.*]] = fir.box_addr %[[BOX_I]] : (!fir.box<!fir.heap<i32>>) -> !fir.heap<i32>
+!CHECK: %[[CREATE:.*]] = acc.create varPtr(%[[BOX_ADDR]] : !fir.heap<i32>) -> !fir.heap<i32> {name = "i", structured = false}
+!CHECK: acc.enter_data dataOperands(%[[CREATE]] : !fir.heap<i32>)
+
+end subroutine
+
+subroutine acc_enter_data_derived_type()
+  type :: dt
+    real :: data
+    real :: array(1:10)
+  end type
+
+  type :: t
+    type(dt) :: d
+  end type
+
+  type :: z
+    integer, allocatable :: data(:)
+  end type
+
+  type :: tt
+    type(dt) :: d(10)
+  end type
+
+  type(dt) :: a
+  type(t) :: b
+  type(dt) :: aa(10)
+  type(z) :: c
+  type(tt) :: d
+
+!CHECK-LABEL: func.func @_QPacc_enter_data_derived_type() {
+!CHECK: %[[A:.*]] = fir.alloca !fir.type<_QFacc_enter_data_derived_typeTdt{data:f32,array:!fir.array<10xf32>}> {bindc_name = "a", uniq_name = "_QFacc_enter_data_derived_typeEa"}
+!CHECK: %[[DECLA:.*]]:2 = hlfir.declare %[[A]]
+!CHECK: %[[AA:.*]] = fir.alloca !fir.array<10x!fir.type<_QFacc_enter_data_derived_typeTdt{data:f32,array:!fir.array<10xf32>}>> {bindc_name = "aa", uniq_name = "_QFacc_enter_data_derived_typeEaa"}
+!CHECK: %[[DECLAA:.*]]:2 = hlfir.declare %[[AA]]
+!CHECK: %[[B:.*]] = fir.alloca !fir.type<_QFacc_enter_data_derived_typeTt{d:!fir.type<_QFacc_enter_data_derived_typeTdt{data:f32,array:!fir.array<10xf32>}>}> {bindc_name = "b", uniq_name = "_QFacc_enter_data_derived_typeEb"}
+!CHECK: %[[DECLB:.*]]:2 = hlfir.declare %[[B]]
+!CHECK: %[[C:.*]] = fir.alloca !fir.type<_QFacc_enter_data_derived_typeTz{data:!fir.box<!fir.heap<!fir.array<?xi32>>>}> {bindc_name = "c", uniq_name = "_QFacc_enter_data_derived_typeEc"}
+!CHECK: %[[DECLC:.*]]:2 = hlfir.declare %[[C]]
+!CHECK: %[[D:.*]] = fir.alloca !fir.type<_QFacc_enter_data_derived_typeTtt{d:!fir.array<10x!fir.type<_QFacc_enter_data_derived_typeTdt{data:f32,array:!fir.array<10xf32>}>>}> {bindc_name = "d", uniq_name = "_QFacc_enter_data_derived_typeEd"}
+!CHECK: %[[DECLD:.*]]:2 = hlfir.declare %[[D]]
+
+  !$acc enter data create(a%data)
+
+
+!CHECK: %[[DATA_COORD:.*]] = hlfir.designate %[[DECLA]]#0{"data"}   : (!fir.ref<!fir.type<_QFacc_enter_data_derived_typeTdt{data:f32,array:!fir.array<10xf32>}>>) -> !fir.ref<f32>
+!CHECK: %[[CREATE:.*]] = acc.create varPtr(%[[DATA_COORD]] : !fir.ref<f32>) -> !fir.ref<f32> {name = "a%data", structured = false}
+!CHECK: acc.enter_data dataOperands(%[[CREATE]] : !fir.ref<f32>)
+
+  !$acc enter data create(b%d%data)
+
+
+
+!CHECK: %[[D_COORD:.*]] = hlfir.designate %[[DECLB]]#0{"d"}   : (!fir.ref<!fir.type<_QFacc_enter_data_derived_typeTt{d:!fir.type<_QFacc_enter_data_derived_typeTdt{data:f32,array:!fir.array<10xf32>}>}>>) -> !fir.ref<!fir.type<_QFacc_enter_data_derived_typeTdt{data:f32,array:!fir.array<10xf32>}>>
+!CHECK: %[[DATA_COORD:.*]] = hlfir.designate %[[D_COORD]]{"data"}   : (!fir.ref<!fir.type<_QFacc_enter_data_derived_typeTdt{data:f32,array:!fir.array<10xf32>}>>) -> !fir.ref<f32>
+!CHECK: %[[CREATE:.*]] = acc.create varPtr(%[[DATA_COORD]] : !fir.ref<f32>) -> !fir.ref<f32> {name = "b%d%data", structured = false}
+!CHECK: acc.enter_data dataOperands(%[[CREATE]] : !fir.ref<f32>)
+
+  !$acc enter data create(a%array)
+
+
+!CHECK: %[[C10:.*]] = arith.constant 10 : index
+!CHECK: %[[ARRAY_COORD:.*]] = hlfir.designate %[[DECLA]]#0{"array"}   shape %{{.*}} : (!fir.ref<!fir.type<_QFacc_enter_data_derived_typeTdt{data:f32,array:!fir.array<10xf32>}>>, !fir.shape<1>) -> !fir.ref<!fir.array<10xf32>>
+!CHECK: %[[C1:.*]] = arith.constant 1 : index
+!CHECK: %[[LB:.*]] = arith.constant 0 : index
+!CHECK: %[[UB:.*]] = arith.subi %[[C10]], %[[C1]] : index
+!CHECK: %[[BOUND:.*]] = acc.bounds lowerbound(%[[LB]] : index) upperbound(%[[UB]] : index) extent(%[[C10]] : index) stride(%[[C1]] : index) startIdx(%[[C1]] : index)
+!CHECK: %[[CREATE:.*]] = acc.create varPtr(%[[ARRAY_COORD]] : !fir.ref<!fir.array<10xf32>>) bounds(%[[BOUND]]) -> !fir.ref<!fir.array<10xf32>> {name = "a%array", structured = false}
+!CHECK: acc.enter_data dataOperands(%[[CREATE]] : !fir.ref<!fir.array<10xf32>>)
+
+  !$acc enter data create(a%array(:))
+
+
+!CHECK: %[[C10:.*]] = arith.constant 10 : index
+!CHECK: %[[ARRAY_COORD:.*]] = hlfir.designate %[[DECLA]]#0{"array"}   shape %{{.*}} : (!fir.ref<!fir.type<_QFacc_enter_data_derived_typeTdt{data:f32,array:!fir.array<10xf32>}>>, !fir.shape<1>) -> !fir.ref<!fir.array<10xf32>>
+!CHECK: %[[LB:.*]] = arith.constant 0 : index
+!CHECK: %[[C1:.*]] = arith.constant 1 : index
+!CHECK: %[[UB:.*]] = arith.subi %[[C10]], %[[C1]] : index
+!CHECK: %[[BOUND:.*]] = acc.bounds lowerbound(%[[LB]] : index) upperbound(%[[UB]] : index) extent(%[[C10]] : index) stride(%[[C1]] : index) startIdx(%[[C1]] : index)
+!CHECK: %[[CREATE:.*]] = acc.create varPtr(%[[ARRAY_COORD]] : !fir.ref<!fir.array<10xf32>>) bounds(%[[BOUND]]) -> !fir.ref<!fir.array<10xf32>> {name = "a%array(:)", structured = false}
+!CHECK: acc.enter_data dataOperands(%[[CREATE]] : !fir.ref<!fir.array<10xf32>>)
+
+  !$acc enter data create(a%array(1:5))
+
+!CHECK: %[[C10:.*]] = arith.constant 10 : index
+!CHECK: %[[ARRAY_COORD:.*]] = hlfir.designate %[[DECLA]]#0{"array"}   shape %{{.*}} : (!fir.ref<!fir.type<_QFacc_enter_data_derived_typeTdt{data:f32,array:!fir.array<10xf32>}>>, !fir.shape<1>) -> !fir.ref<!fir.array<10xf32>>
+!CHECK: %[[C1:.*]] = arith.constant 1 : index
+!CHECK: %[[C0:.*]] = arith.constant 0 : index
+!CHECK: %[[C4:.*]] = arith.constant 4 : index
+!CHECK: %[[BOUND:.*]] = acc.bounds lowerbound(%[[C0]] : index) upperbound(%[[C4]] : index)  extent(%[[C10]] : index) stride(%[[C1]] : index) startIdx(%[[C1]] : index)
+!CHECK: %[[CREATE:.*]] = acc.create varPtr(%[[ARRAY_COORD]] : !fir.ref<!fir.array<10xf32>>) bounds(%[[BOUND]]) -> !fir.ref<!fir.array<10xf32>> {name = "a%array(1:5)", structured = false}
+!CHECK: acc.enter_data dataOperands(%[[CREATE]] : !fir.ref<!fir.array<10xf32>>)
+
+  !$acc enter data create(a%array(:5))
+
+!CHECK: %[[C10:.*]] = arith.constant 10 : index
+!CHECK: %[[ARRAY_COORD:.*]] = hlfir.designate %[[DECLA]]#0{"array"}   shape %{{.*}} : (!fir.ref<!fir.type<_QFacc_enter_data_derived_typeTdt{data:f32,array:!fir.array<10xf32>}>>, !fir.shape<1>) -> !fir.ref<!fir.array<10xf32>>
+!CHECK: %[[LB:.*]] = arith.constant 0 : index
+!CHECK: %[[C1:.*]] = arith.constant 1 : index
+!CHECK: %[[C4:.*]] = arith.constant 4 : index
+!CHECK: %[[BOUND:.*]] = acc.bounds lowerbound(%[[LB]] : index) upperbound(%[[C4]] : index) extent(%[[C10]] : index) stride(%[[C1]] : index) startIdx(%[[C1]] : index)
+!CHECK: %[[CREATE:.*]] = acc.create varPtr(%[[ARRAY_COORD]] : !fir.ref<!fir.array<10xf32>>) bounds(%[[BOUND]]) -> !fir.ref<!fir.array<10xf32>> {name = "a%array(:5)", structured = false}
+!CHECK: acc.enter_data dataOperands(%[[CREATE]] : !fir.ref<!fir.array<10xf32>>)
+
+  !$acc enter data create(a%array(2:))
+
+
+!CHECK: %[[C10:.*]] = arith.constant 10 : index
+!CHECK: %[[ARRAY_COORD:.*]] = hlfir.designate %[[DECLA]]#0{"array"}   shape %{{.*}} : (!fir.ref<!fir.type<_QFacc_enter_data_derived_typeTdt{data:f32,array:!fir.array<10xf32>}>>, !fir.shape<1>) -> !fir.ref<!fir.array<10xf32>>
+!CHECK: %[[ONE:.*]] = arith.constant 1 : index
+!CHECK: %[[LB:.*]] = arith.constant 1 : index
+!CHECK: %[[UB:.*]] = arith.subi %[[C10]], %[[ONE]] : index
+!CHECK: %[[BOUND:.*]] = acc.bounds lowerbound(%[[LB]] : index) upperbound(%[[UB]] : index) extent(%[[C10]] : index) stride(%[[ONE]] : index) startIdx(%[[ONE]] : index)
+!CHECK: %[[CREATE:.*]] = acc.create varPtr(%[[ARRAY_COORD]] : !fir.ref<!fir.array<10xf32>>) bounds(%[[BOUND]]) -> !fir.ref<!fir.array<10xf32>> {name = "a%array(2:)", structured = false}
+!CHECK: acc.enter_data dataOperands(%[[CREATE]] : !fir.ref<!fir.array<10xf32>>)
+
+!$acc enter data create(b%d%array)
+
+
+
+!CHECK: %[[D_COORD:.*]] = hlfir.designate %[[DECLB]]#0{"d"}   : (!fir.ref<!fir.type<_QFacc_enter_data_derived_typeTt{d:!fir.type<_QFacc_enter_data_derived_typeTdt{data:f32,array:!fir.array<10xf32>}>}>>) -> !fir.ref<!fir.type<_QFacc_enter_data_derived_typeTdt{data:f32,array:!fir.array<10xf32>}>>
+!CHECK: %[[C10:.*]] = arith.constant 10 : index
+!CHECK: %[[ARRAY_COORD:.*]] = hlfir.designate %[[D_COORD]]{"array"}   shape %{{.*}} : (!fir.ref<!fir.type<_QFacc_enter_data_derived_typeTdt{data:f32,array:!fir.array<10xf32>}>>, !fir.shape<1>) -> !fir.ref<!fir.array<10xf32>>
+!CHECK: %[[C1:.*]] = arith.constant 1 : index
+!CHECK: %[[LB:.*]] = arith.constant 0 : index
+!CHECK: %[[UB:.*]] = arith.subi %[[C10]], %[[C1]] : index
+!CHECK: %[[BOUND:.*]] = acc.bounds lowerbound(%[[LB]] : index) upperbound(%[[UB]] : index) extent(%[[C10]] : index) stride(%[[C1]] : index) startIdx(%[[C1]] : index)
+!CHECK: %[[CREATE:.*]] = acc.create varPtr(%[[ARRAY_COORD]] : !fir.ref<!fir.array<10xf32>>) bounds(%[[BOUND]]) -> !fir.ref<!fir.array<10xf32>> {name = "b%d%array", structured = false}
+
+  !$acc enter data create(c%data)
+
+
+!CHECK: %[[DATA_COORD:.*]] = hlfir.designate %[[DECLC]]#0{"data"}   {fortran_attrs = #fir.var_attrs<allocatable>} : (!fir.ref<!fir.type<_QFacc_enter_data_derived_typeTz{data:!fir.box<!fir.heap<!fir.array<?xi32>>>}>>) -> !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>
+!CHECK: %[[DATA_BOX:.*]] = fir.load %[[DATA_COORD]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>
+!CHECK: %[[DIM0:.*]] = arith.constant 0 : index
+!CHECK: %[[DIMS0:.*]]:3 = fir.box_dims %[[DATA_BOX]], %[[DIM0]] : (!fir.box<!fir.heap<!fir.array<?xi32>>>, index) -> (index, index, index)
+!CHECK: %[[ONE:.*]] = arith.constant 1 : index
+!CHECK: %[[DIM0_1:.*]] = arith.constant 0 : index
+!CHECK: %[[DIMS0_1:.*]]:3 = fir.box_dims %[[DATA_BOX]], %[[DIM0_1]] : (!fir.box<!fir.heap<!fir.array<?xi32>>>, index) -> (index, index, index)
+!CHECK: %[[UB:.*]] = arith.subi %[[DIMS0_1]]#1, %[[ONE]] : index
+!CHECK: %[[BOUND:.*]] = acc.bounds lowerbound(%c0{{.*}} : index) upperbound(%[[UB]] : index) extent(%[[DIMS0_1]]#1 : index) stride(%[[DIMS0_1]]#2 : index) startIdx(%[[DIMS0]]#0 : index) {strideInBytes = true}
+!CHECK: %[[BOX_ADDR:.*]] = fir.box_addr %[[DATA_BOX]] : (!fir.box<!fir.heap<!fir.array<?xi32>>>) -> !fir.heap<!fir.array<?xi32>>
+!CHECK: %[[CREATE:.*]] = acc.create varPtr(%[[BOX_ADDR]] : !fir.heap<!fir.array<?xi32>>) bounds(%[[BOUND]]) -> !fir.heap<!fir.array<?xi32>> {name = "c%data", structured = false}
+!CHECK: acc.enter_data dataOperands(%[[CREATE]] : !fir.heap<!fir.array<?xi32>>)
+
+  !$acc enter data create (d%d(1)%array)
+
+
+
+
+
+
+!CHECK: %[[ONE:.*]] = arith.constant 1 : index
+!CHECK: %[[D1_COORD:.*]] = hlfir.designate %[[DECLD]]#0{"d"} <%{{.*}}> (%[[ONE]])  : (!fir.ref<!fir.type<_QFacc_enter_data_derived_typeTtt{d:!fir.array<10x!fir.type<_QFacc_enter_data_derived_typeTdt{data:f32,array:!fir.array<10xf32>}>>}>>, !fir.shape<1>, index) -> !fir.ref<!fir.type<_QFacc_enter_data_derived_typeTdt{data:f32,array:!fir.array<10xf32>}>>
+
+
+!CHECK: %[[C10:.*]] = arith.constant 10 : index
+!CHECK: %[[ARRAY_COORD:.*]] = hlfir.designate %[[D1_COORD]]{"array"}   shape %{{.*}} : (!fir.ref<!fir.type<_QFacc_enter_data_derived_typeTdt{data:f32,array:!fir.array<10xf32>}>>, !fir.shape<1>) -> !fir.ref<!fir.array<10xf32>>
+!CHECK: %[[C1:.*]] = arith.constant 1 : index
+!CHECK: %[[LB:.*]] = arith.constant 0 : index
+!CHECK: %[[UB:.*]] = arith.subi %[[C10]], %[[C1]] : index
+!CHECK: %[[BOUND:.*]] = acc.bounds lowerbound(%[[LB]] : index) upperbound(%[[UB]] : index) extent(%[[C10]] : index) stride(%[[C1]] : index) startIdx(%[[C1]] : index)
+!CHECK: %[[CREATE:.*]] = acc.create varPtr(%[[ARRAY_COORD]] : !fir.ref<!fir.array<10xf32>>) bounds(%[[BOUND]]) -> !fir.ref<!fir.array<10xf32>> {name = "d%d(1_8)%array", structured = false}
+!CHECK: acc.enter_data dataOperands(%[[CREATE]] : !fir.ref<!fir.array<10xf32>>)
+
+end subroutine
+
+subroutine acc_enter_data_single_array_element()
+  type t1
+    real, allocatable :: a(:, :)
+  end type t1
+  type(t1), allocatable :: e(:)
+  allocate(e(10)%a(5,5))
+
+  !$acc enter data create(e(2)%a(1,2))
+
+!CHECK-LABEL:   func.func @_QPacc_enter_data_single_array_element() {
+!CHECK-DAG:       %[[VAL_38:.*]]:3 = fir.box_dims %[[BOX:.*]], %[[VAL_37:.*]] : (!fir.box<!fir.heap<!fir.array<?x?xf32>>>, index) -> (index, index, index)
+!CHECK-DAG:       %[[VAL_37]] = arith.constant 0 : index
+!CHECK-DAG:       %[[VAL_40:.*]]:3 = fir.box_dims %[[BOX]], %[[VAL_39:.*]] : (!fir.box<!fir.heap<!fir.array<?x?xf32>>>, index) -> (index, index, index)
+!CHECK-DAG:       %[[VAL_39]] = arith.constant 1 : index
+!CHECK-DAG:       %[[VAL_41:.*]] = fir.box_addr %[[BOX]] : (!fir.box<!fir.heap<!fir.array<?x?xf32>>>) -> !fir.heap<!fir.array<?x?xf32>>
+!CHECK:           %[[VAL_42:.*]] = arith.constant 1 : index
+!CHECK:           %[[VAL_43:.*]] = arith.constant 1 : index
+!CHECK:           %[[VAL_44:.*]] = arith.subi %[[VAL_43]], %[[VAL_38]]#0 : index
+!CHECK:           %[[VAL_45:.*]] = acc.bounds lowerbound(%[[VAL_44]] : index) upperbound(%[[VAL_44]] : index) extent(%[[VAL_42]] : index) stride(%[[VAL_42]] : index) startIdx(%[[VAL_38]]#0 : index)
+!CHECK:           %[[VAL_46:.*]] = arith.constant 2 : index
+!CHECK:           %[[VAL_47:.*]] = arith.subi %[[VAL_46]], %[[VAL_40]]#0 : index
+!CHECK:           %[[VAL_48:.*]] = acc.bounds lowerbound(%[[VAL_47]] : index) upperbound(%[[VAL_47]] : index) extent(%[[VAL_42]] : index) stride(%[[VAL_42]] : index) startIdx(%[[VAL_40]]#0 : index)
+!CHECK:           %[[CREATE:.*]] = acc.create varPtr(%[[VAL_41]] : !fir.heap<!fir.array<?x?xf32>>) bounds(%[[VAL_45]], %[[VAL_48]]) -> !fir.heap<!fir.array<?x?xf32>> {name = "e(2_8)%a(1,2)", structured = false}
+!CHECK:           acc.enter_data dataOperands(%[[CREATE]] : !fir.heap<!fir.array<?x?xf32>>)
+
+end subroutine
diff --git a/flang/test/Lower/OpenACC/acc-enter-data.f90 b/flang/test/Lower/OpenACC/acc-enter-data.f90
index 80326a1012376..8892dec7d1197 100644
--- a/flang/test/Lower/OpenACC/acc-enter-data.f90
+++ b/flang/test/Lower/OpenACC/acc-enter-data.f90
@@ -20,129 +20,73 @@ subroutine acc_enter_data
 !CHECK: %[[DECLD:.*]]:2 = hlfir.declare %[[D]]
 
   !$acc enter data create(a)
-!CHECK: %[[ONE:.*]] = arith.constant 1 : index
-!CHECK: %[[LB:.*]] = arith.constant 0 : index
-!CHECK: %[[UB:.*]] = arith.subi %[[C10]], %[[ONE]] : index
-!CHECK: %[[BOUND0:.*]] = acc.bounds lowerbound(%[[LB]] : index) upperbound(%[[UB]] : index) extent(%[[C10]] : index) stride(%[[ONE]] : index) startIdx(%[[ONE]] : index)
-!CHECK: %[[LB:.*]] = arith.constant 0 : index
-!CHECK: %[[UB:.*]] = arith.subi %[[EXTENT_C10]], %[[ONE]] : index
-!CHECK: %[[BOUND1:.*]] = acc.bounds lowerbound(%[[LB]] : index) upperbound(%[[UB]] : index) extent(%[[EXTENT_C10]] : index) stride(%[[ONE]] : index) startIdx(%[[ONE]] : index)
-!CHECK: %[[CREATE_A:.*]] = acc.create varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) bounds(%[[BOUND0]], %[[BOUND1]]) -> !fir.ref<!fir.array<10x10xf32>> {name = "a", structured = false}
+!CHECK: %[[CREATE_A:.*]] = acc.create varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) -> !fir.ref<!fir.array<10x10xf32>> {name = "a", structured = false}
 !CHECK: acc.enter_data dataOperands(%[[CREATE_A]] : !fir.ref<!fir.array<10x10xf32>>){{$}}
 
   !$acc enter data create(a) if(.true.)
-!CHECK: %[[ONE:.*]] = arith.constant 1 : index
-!CHECK: %[[LB:.*]] = arith.constant 0 : index
-!CHECK: %[[UB:.*]] = arith.subi %[[C10]], %[[ONE]] : index
-!CHECK: %[[BOUND0:.*]] = acc.bounds lowerbound(%[[LB]] : index) upperbound(%[[UB]] : index) extent(%[[C10]] : index) stride(%[[ONE]] : index) startIdx(%[[ONE]] : index)
-!CHECK: %[[LB:.*]] = arith.constant 0 : index
-!CHECK: %[[UB:.*]] = arith.subi %[[EXTENT_C10]], %[[ONE]] : index
-!CHECK: %[[BOUND1:.*]] = acc.bounds lowerbound(%[[LB]] : index) upperbound(%[[UB]] : index) extent(%[[EXTENT_C10]] : index) stride(%[[ONE]] : index) startIdx(%[[ONE]] : index)
-!CHECK: %[[CREATE_A:.*]] = acc.create varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) bounds(%[[BOUND0]], %[[BOUND1]]) -> !fir.ref<!fir.array<10x10xf32>> {name = "a", structured = false}
+!CHECK: %[[CREATE_A:.*]] = acc.create varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) -> !fir.ref<!fir.array<10x10xf32>> {name = "a", structured = false}
 !CHECK: [[IF1:%.*]] = arith.constant true
 !CHECK: acc.enter_data if([[IF1]]) dataOperands(%[[CREATE_A]] : !fir.ref<!fir.array<10x10xf32>>){{$}}
 
   !$acc enter data create(a) if(ifCondition)
-!CHECK: %[[ONE:.*]] = arith.constant 1 : index
-!CHECK: %[[LB:.*]] = arith.constant 0 : index
-!CHECK: %[[UB:.*]] = arith.subi %[[C10]], %[[ONE]] : index
-!CHECK: %[[BOUND0:.*]] = acc.bounds lowerbound(%[[LB]] : index) upperbound(%[[UB]] : index) extent(%[[C10]] : index) stride(%[[ONE]] : index) startIdx(%[[ONE]] : index)
-!CHECK: %[[LB:.*]] = arith.constant 0 : index
-!CHECK: %[[UB:.*]] = arith.subi %[[EXTENT_C10]], %[[ONE]] : index
-!CHECK: %[[BOUND1:.*]] = acc.bounds lowerbound(%[[LB]] : index) upperbound(%[[UB]] : index) extent(%[[EXTENT_C10]] : index) stride(%[[ONE]] : index) startIdx(%[[ONE]] : index)
-!CHECK: %[[CREATE_A:.*]] = acc.create varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) bounds(%[[BOUND0]], %[[BOUND1]]) -> !fir.ref<!fir.array<10x10xf32>> {name = "a", structured = false}
+!CHECK: %[[CREATE_A:.*]] = acc.create varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) -> !fir.ref<!fir.array<10x10xf32>> {name = "a", structured = false}
 !CHECK: [[IFCOND:%.*]] = fir.load %{{.*}} : !fir.ref<!fir.logical<4>>
 !CHECK: [[IF2:%.*]] = fir.convert [[IFCOND]] : (!fir.logical<4>) -> i1
 !CHECK: acc.enter_data if([[IF2]]) dataOperands(%[[CREATE_A]] : !fir.ref<!fir.array<10x10xf32>>){{$}}
 
   !$acc enter data create(a) create(b) create(c)
-!CHECK: %[[BOUND0:.*]] = acc.bounds lowerbound(%{{.*}} : index) upperbound(%{{.*}} : index) extent(%[[C10]] : index) stride(%c1{{.*}} : index) startIdx(%{{.*}} : index)
-!CHECK: %[[BOUND1:.*]] = acc.bounds lowerbound(%{{.*}} : index) upperbound(%{{.*}} : index) extent(%[[EXTENT_C10]] : index) stride(%c1{{.*}} : index) startIdx(%{{.*}} : index)
-!CHECK: %[[CREATE_A:.*]] = acc.create varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) bounds(%[[BOUND0]], %[[BOUND1]]) -> !fir.ref<!fir.array<10x10xf32>> {name = "a", structured = false}
-!CHECK: %[[BOUND0:.*]] = acc.bounds lowerbound(%{{.*}} : index) upperbound(%{{.*}} : index) extent(%c10_{{.*}} : index) stride(%c1{{.*}} : index) startIdx(%{{.*}} : index)
-!CHECK: %[[BOUND1:.*]] = acc.bounds lowerbound(%{{.*}} : index) upperbound(%{{.*}} : index) extent(%c10_{{.*}} : index) stride(%c1{{.*}} : index) startIdx(%{{.*}} : index)
-!CHECK: %[[CREATE_B:.*]] = acc.create varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<10x10xf32>>) bounds(%[[BOUND0]], %[[BOUND1]]) -> !fir.ref<!fir.array<10x10xf32>> {name = "b", structured = false}
-!CHECK: %[[BOUND0:.*]] = acc.bounds lowerbound(%{{.*}} : index) upperbound(%{{.*}} : index) extent(%c10_{{.*}} : index) stride(%c1{{.*}} : index) startIdx(%{{.*}} : index)
-!CHECK: %[[BOUND1:.*]] = acc.bounds lowerbound(%{{.*}} : index) upperbound(%{{.*}} : index) extent(%c10_{{.*}} : index) stride(%c1{{.*}} : index) startIdx(%{{.*}} : index)
-!CHECK: %[[CREATE_C:.*]] = acc.create varPtr(%[[DECLC]]#0 : !fir.ref<!fir.array<10x10xf32>>) bounds(%[[BOUND0]], %[[BOUND1]]) -> !fir.ref<!fir.array<10x10xf32>> {name = "c", structured = false}
+!CHECK: %[[CREATE_A:.*]] = acc.create varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) -> !fir.ref<!fir.array<10x10xf32>> {name = "a", structured = false}
+!CHECK: %[[CREATE_B:.*]] = acc.create varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<10x10xf32>>) -> !fir.ref<!fir.array<10x10xf32>> {name = "b", structured = false}
+!CHECK: %[[CREATE_C:.*]] = acc.create varPtr(%[[DECLC]]#0 : !fir.ref<!fir.array<10x10xf32>>) -> !fir.ref<!fir.array<10x10xf32>> {name = "c", structured = false}
 !CHECK: acc.enter_data dataOperands(%[[CREATE_A]], %[[CREATE_B]], %[[CREATE_C]] : !fir.ref<!fir.array<10x10xf32>>, !fir.ref<!fir.array<10x10xf32>>, !fir.ref<!fir.array<10x10xf32>>){{$}}
 
   !$acc enter data create(a) create(b) create(zero: c)
-!CHECK: %[[BOUND0:.*]] = acc.bounds lowerbound(%{{.*}} : index) upperbound(%{{.*}} : index) extent(%[[C10]] : index) stride(%c1{{.*}} : index) startIdx(%{{.*}} : index)
-!CHECK: %[[BOUND1:.*]] = acc.bounds lowerbound(%{{.*}} : index) upperbound(%{{.*}} : index) extent(%[[EXTENT_C10]] : index) stride(%c1{{.*}} : index) startIdx(%{{.*}} : index)
-!CHECK: %[[CREATE_A:.*]] = acc.create varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) bounds(%[[BOUND0]], %[[BOUND1]]) -> !fir.ref<!fir.array<10x10xf32>> {name = "a", structured = false}
-!CHECK: %[[BOUND0:.*]] = acc.bounds lowerbound(%{{.*}} : index) upperbound(%{{.*}} : index) extent(%c10_{{.*}} : index) stride(%c1{{.*}} : index) startIdx(%{{.*}} : index)
-!CHECK: %[[BOUND1:.*]] = acc.bounds lowerbound(%{{.*}} : index) upperbound(%{{.*}} : index) extent(%c10_{{.*}} : index) stride(%c1{{.*}} : index) startIdx(%{{.*}} : index)
-!CHECK: %[[CREATE_B:.*]] = acc.create varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<10x10xf32>>) bounds(%[[BOUND0]], %[[BOUND1]]) -> !fir.ref<!fir.array<10x10xf32>> {name = "b", structured = false}
-!CHECK: %[[BOUND0:.*]] = acc.bounds lowerbound(%{{.*}} : index) upperbound(%{{.*}} : index) extent(%c10_{{.*}} : index) stride(%c1{{.*}} : index) startIdx(%{{.*}} : index)
-!CHECK: %[[BOUND1:.*]] = acc.bounds lowerbound(%{{.*}} : index) upperbound(%{{.*}} : index) extent(%c10_{{.*}} : index) stride(%c1{{.*}} : index) startIdx(%{{.*}} : index)
-!CHECK: %[[CREATE_C:.*]] = acc.create varPtr(%[[DECLC]]#0 : !fir.ref<!fir.array<10x10xf32>>) bounds(%[[BOUND0]], %[[BOUND1]]) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = #acc<data_clause acc_create_zero>, name = "c", structured = false}
+!CHECK: %[[CREATE_A:.*]] = acc.create varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) -> !fir.ref<!fir.array<10x10xf32>> {name = "a", structured = false}
+!CHECK: %[[CREATE_B:.*]] = acc.create varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<10x10xf32>>) -> !fir.ref<!fir.array<10x10xf32>> {name = "b", structured = false}
+!CHECK: %[[CREATE_C:.*]] = acc.create varPtr(%[[DECLC]]#0 : !fir.ref<!fir.array<10x10xf32>>) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = #acc<data_clause acc_create_zero>, name = "c", structured = false}
 !CHECK: acc.enter_data dataOperands(%[[CREATE_A]], %[[CREATE_B]], %[[CREATE_C]] : !fir.ref<!fir.array<10x10xf32>>, !fir.ref<!fir.array<10x10xf32>>, !fir.ref<!fir.array<10x10xf32>>){{$}}
 
   !$acc enter data copyin(a) create(b) attach(d)
-!CHECK: %[[BOUND0:.*]] = acc.bounds lowerbound(%{{.*}} : index) upperbound(%{{.*}} : index) extent(%[[C10]] : index) stride(%c1{{.*}} : index) startIdx(%{{.*}} : index)
-!CHECK: %[[BOUND1:.*]] = acc.bounds lowerbound(%{{.*}} : index) upperbound(%{{.*}} : index) extent(%[[EXTENT_C10]] : index) stride(%c1{{.*}} : index) startIdx(%{{.*}} : index)
-!CHECK: %[[COPYIN_A:.*]] = acc.copyin varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) bounds(%[[BOUND0]], %[[BOUND1]])  -> !fir.ref<!fir.array<10x10xf32>> {name = "a", structured = false}
-!CHECK: %[[BOUND0:.*]] = acc.bounds lowerbound(%{{.*}} : index) upperbound(%{{.*}} : index) extent(%c10_{{.*}} : index) stride(%c1{{.*}} : index) startIdx(%{{.*}} : index)
-!CHECK: %[[BOUND1:.*]] = acc.bounds lowerbound(%{{.*}} : index) upperbound(%{{.*}} : index) extent(%c10_{{.*}} : index) stride(%c1{{.*}} : index) startIdx(%{{.*}} : index)
-!CHECK: %[[CREATE_B:.*]] = acc.create varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<10x10xf32>>) bounds(%[[BOUND0]], %[[BOUND1]]) -> !fir.ref<!fir.array<10x10xf32>> {name = "b", structured = false}
-!CHECK: %[[BOX_D:.*]] = fir.load %[[DECLD]]#0 : !fir.ref<!fir.box<!fir.ptr<f32>>>
-!CHECK: %[[BOX_ADDR_D:.*]] = fir.box_addr %[[BOX_D]] : (!fir.box<!fir.ptr<f32>>) -> !fir.ptr<f32>
-!CHECK: %[[ATTACH_D:.*]] = acc.attach varPtr(%[[BOX_ADDR_D]] : !fir.ptr<f32>) -> !fir.ptr<f32> {name = "d", structured = false}
-!CHECK: acc.enter_data dataOperands(%[[COPYIN_A]], %[[CREATE_B]], %[[ATTACH_D]] : !fir.ref<!fir.array<10x10xf32>>, !fir.ref<!fir.array<10x10xf32>>, !fir.ptr<f32>){{$}}
+!CHECK: %[[COPYIN_A:.*]] = acc.copyin varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) -> !fir.ref<!fir.array<10x10xf32>> {name = "a", structured = false}
+!CHECK: %[[CREATE_B:.*]] = acc.create varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<10x10xf32>>) -> !fir.ref<!fir.array<10x10xf32>> {name = "b", structured = false}
+!CHECK: %[[ATTACH_D:.*]] = acc.attach varPtr(%[[DECLD]]#0 : !fir.ref<!fir.box<!fir.ptr<f32>>>) -> !fir.ref<!fir.box<!fir.ptr<f32>>> {name = "d", structured = false}
+!CHECK: acc.enter_data dataOperands(%[[COPYIN_A]], %[[CREATE_B]], %[[ATTACH_D]] : !fir.ref<!fir.array<10x10xf32>>, !fir.ref<!fir.array<10x10xf32>>, !fir.ref<!fir.box<!fir.ptr<f32>>>){{$}}
 
   !$acc enter data create(a) async
-!CHECK: %[[BOUND0:.*]] = acc.bounds lowerbound(%{{.*}} : index) upperbound(%{{.*}} : index) extent(%[[C10]] : index) stride(%c1{{.*}} : index) startIdx(%{{.*}} : index)
-!CHECK: %[[BOUND1:.*]] = acc.bounds lowerbound(%{{.*}} : index) upperbound(%{{.*}} : index) extent(%[[EXTENT_C10]] : index) stride(%c1{{.*}} : index) startIdx(%{{.*}} : index)
-!CHECK: %[[CREATE_A:.*]] = acc.create varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) bounds(%[[BOUND0]], %[[BOUND1]]) -> !fir.ref<!fir.array<10x10xf32>> {asyncOnly = [#acc.device_type<none>], name = "a", structured = false}
+!CHECK: %[[CREATE_A:.*]] = acc.create varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) -> !fir.ref<!fir.array<10x10xf32>> {asyncOnly = [#acc.device_type<none>], name = "a", structured = false}
 !CHECK: acc.enter_data dataOperands(%[[CREATE_A]] : !fir.ref<!fir.array<10x10xf32>>) attributes {async}
 
   !$acc enter data create(a) wait
-!CHECK: %[[BOUND0:.*]] = acc.bounds lowerbound(%{{.*}} : index) upperbound(%{{.*}} : index) extent(%[[C10]] : index) stride(%c1{{.*}} : index) startIdx(%{{.*}} : index)
-!CHECK: %[[BOUND1:.*]] = acc.bounds lowerbound(%{{.*}} : index) upperbound(%{{.*}} : index) extent(%[[EXTENT_C10]] : index) stride(%c1{{.*}} : index) startIdx(%{{.*}} : index)
-!CHECK: %[[CREATE_A:.*]] = acc.create varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) bounds(%[[BOUND0]], %[[BOUND1]]) -> !fir.ref<!fir.array<10x10xf32>> {name = "a", structured = false}
+!CHECK: %[[CREATE_A:.*]] = acc.create varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) -> !fir.ref<!fir.array<10x10xf32>> {name = "a", structured = false}
 !CHECK: acc.enter_data dataOperands(%[[CREATE_A]] : !fir.ref<!fir.array<10x10xf32>>) attributes {wait}
 
   !$acc enter data create(a) async wait
-!CHECK: %[[BOUND0:.*]] = acc.bounds lowerbound(%{{.*}} : index) upperbound(%{{.*}} : index) extent(%[[C10]] : index) stride(%c1{{.*}} : index) startIdx(%{{.*}} : index)
-!CHECK: %[[BOUND1:.*]] = acc.bounds lowerbound(%{{.*}} : index) upperbound(%{{.*}} : index) extent(%[[EXTENT_C10]] : index) stride(%c1{{.*}} : index) startIdx(%{{.*}} : index)
-!CHECK: %[[CREATE_A:.*]] = acc.create varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) bounds(%[[BOUND0]], %[[BOUND1]]) -> !fir.ref<!fir.array<10x10xf32>> {asyncOnly = [#acc.device_type<none>], name = "a", structured = false}
+!CHECK: %[[CREATE_A:.*]] = acc.create varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) -> !fir.ref<!fir.array<10x10xf32>> {asyncOnly = [#acc.device_type<none>], name = "a", structured = false}
 !CHECK: acc.enter_data dataOperands(%[[CREATE_A]] : !fir.ref<!fir.array<10x10xf32>>) attributes {async, wait}
 
   !$acc enter data create(a) async(1)
 !CHECK: %[[ASYNC1:.*]] = arith.constant 1 : i32
-!CHECK: %[[BOUND0:.*]] = acc.bounds lowerbound(%{{.*}} : index) upperbound(%{{.*}} : index) extent(%[[C10]] : index) stride(%c1{{.*}} : index) startIdx(%{{.*}} : index)
-!CHECK: %[[BOUND1:.*]] = acc.bounds lowerbound(%{{.*}} : index) upperbound(%{{.*}} : index) extent(%[[EXTENT_C10]] : index) stride(%c1{{.*}} : index) startIdx(%{{.*}} : index)
-!CHECK: %[[CREATE_A:.*]] = acc.create varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) bounds(%[[BOUND0]], %[[BOUND1]]) async(%[[ASYNC1]] : i32) -> !fir.ref<!fir.array<10x10xf32>> {name = "a", structured = false}
+!CHECK: %[[CREATE_A:.*]] = acc.create varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) async(%[[ASYNC1]] : i32) -> !fir.ref<!fir.array<10x10xf32>> {name = "a", structured = false}
 !CHECK: acc.enter_data async(%[[ASYNC1]] : i32) dataOperands(%[[CREATE_A]] : !fir.ref<!fir.array<10x10xf32>>)
 
   !$acc enter data create(a) async(async)
 !CHECK: %[[ASYNC2:.*]] = fir.load %{{.*}} : !fir.ref<i32>
-!CHECK: %[[BOUND0:.*]] = acc.bounds lowerbound(%{{.*}} : index) upperbound(%{{.*}} : index) extent(%[[C10]] : index) stride(%c1{{.*}} : index) startIdx(%{{.*}} : index)
-!CHECK: %[[BOUND1:.*]] = acc.bounds lowerbound(%{{.*}} : index) upperbound(%{{.*}} : index) extent(%[[EXTENT_C10]] : index) stride(%c1{{.*}} : index) startIdx(%{{.*}} : index)
-
-!CHECK: %[[CREATE_A:.*]] = acc.create varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) bounds(%[[BOUND0]], %[[BOUND1]]) async(%[[ASYNC2]] : i32) -> !fir.ref<!fir.array<10x10xf32>> {name = "a", structured = false}
+!CHECK: %[[CREATE_A:.*]] = acc.create varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) async(%[[ASYNC2]] : i32) -> !fir.ref<!fir.array<10x10xf32>> {name = "a", structured = false}
 !CHECK: acc.enter_data async(%[[ASYNC2]] : i32) dataOperands(%[[CREATE_A]] : !fir.ref<!fir.array<10x10xf32>>)
 
   !$acc enter data create(a) wait(1)
-!CHECK: %[[BOUND0:.*]] = acc.bounds lowerbound(%{{.*}} : index) upperbound(%{{.*}} : index) extent(%[[C10]] : index) stride(%c1{{.*}} : index) startIdx(%{{.*}} : index)
-!CHECK: %[[BOUND1:.*]] = acc.bounds lowerbound(%{{.*}} : index) upperbound(%{{.*}} : index) extent(%[[EXTENT_C10]] : index) stride(%c1{{.*}} : index) startIdx(%{{.*}} : index)
-!CHECK: %[[CREATE_A:.*]] = acc.create varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) bounds(%[[BOUND0]], %[[BOUND1]]) -> !fir.ref<!fir.array<10x10xf32>> {name = "a", structured = false}
+!CHECK: %[[CREATE_A:.*]] = acc.create varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) -> !fir.ref<!fir.array<10x10xf32>> {name = "a", structured = false}
 !CHECK: %[[WAIT1:.*]] = arith.constant 1 : i32
 !CHECK: acc.enter_data wait(%[[WAIT1]] : i32) dataOperands(%[[CREATE_A]] : !fir.ref<!fir.array<10x10xf32>>)
 
   !$acc enter data create(a) wait(queues: 1, 2)
-!CHECK: %[[BOUND0:.*]] = acc.bounds lowerbound(%{{.*}} : index) upperbound(%{{.*}} : index) extent(%[[C10]] : index) stride(%c1{{.*}} : index) startIdx(%{{.*}} : index)
-!CHECK: %[[BOUND1:.*]] = acc.bounds lowerbound(%{{.*}} : index) upperbound(%{{.*}} : index) extent(%[[EXTENT_C10]] : index) stride(%c1{{.*}} : index) startIdx(%{{.*}} : index)
-!CHECK: %[[CREATE_A:.*]] = acc.create varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) bounds(%[[BOUND0]], %[[BOUND1]]) -> !fir.ref<!fir.array<10x10xf32>> {name = "a", structured = false}
+!CHECK: %[[CREATE_A:.*]] = acc.create varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) -> !fir.ref<!fir.array<10x10xf32>> {name = "a", structured = false}
 !CHECK: %[[WAIT2:.*]] = arith.constant 1 : i32
 !CHECK: %[[WAIT3:.*]] = arith.constant 2 : i32
 !CHECK: acc.enter_data wait(%[[WAIT2]], %[[WAIT3]] : i32, i32) dataOperands(%[[CREATE_A]] : !fir.ref<!fir.array<10x10xf32>>)
 
   !$acc enter data create(a) wait(devnum: 1: queues: 1, 2)
-!CHECK: %[[BOUND0:.*]] = acc.bounds lowerbound(%{{.*}} : index) upperbound(%{{.*}} : index) extent(%[[C10]] : index) stride(%c1{{.*}} : index) startIdx(%{{.*}} : index)
-!CHECK: %[[BOUND1:.*]] = acc.bounds lowerbound(%{{.*}} : index) upperbound(%{{.*}} : index) extent(%[[EXTENT_C10]] : index) stride(%c1{{.*}} : index) startIdx(%{{.*}} : index)
-!CHECK: %[[CREATE_A:.*]] = acc.create varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) bounds(%[[BOUND0]], %[[BOUND1]]) -> !fir.ref<!fir.array<10x10xf32>> {name = "a", structured = false}
+!CHECK: %[[CREATE_A:.*]] = acc.create varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) -> !fir.ref<!fir.array<10x10xf32>> {name = "a", structured = false}
 !CHECK: %[[WAIT4:.*]] = arith.constant 1 : i32
 !CHECK: %[[WAIT5:.*]] = arith.constant 2 : i32
 !CHECK: %[[WAIT6:.*]] = arith.constant 1 : i32
@@ -212,16 +156,12 @@ subroutine acc_enter_data_dummy(a, b, n, m)
 !CHECK: %[[DECLB:.*]]:2 = hlfir.declare %[[B]]
 
   !$acc enter data create(a)
-!CHECK: %[[BOUND:.*]] = acc.bounds lowerbound(%{{.*}} : index) upperbound(%{{.*}} : index) extent(%c10{{.*}} : index) stride(%c1{{.*}} : index) startIdx(%{{.*}} : index)
-!CHECK: %[[CREATE:.*]] = acc.create varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10xf32>>) bounds(%[[BOUND]]) -> !fir.ref<!fir.array<10xf32>> {name = "a", structured = false}
+!CHECK: %[[CREATE:.*]] = acc.create varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10xf32>>) -> !fir.ref<!fir.array<10xf32>> {name = "a", structured = false}
 !CHECK: acc.enter_data dataOperands(%[[CREATE]] : !fir.ref<!fir.array<10xf32>>)
 
   !$acc enter data create(b)
-!CHECK: %[[DIMS:.*]]:3 = fir.box_dims %[[DECLB]]#0, %c0{{.*}} : (!fir.box<!fir.array<?xf32>>, index) -> (index, index, index)
-!CHECK: %[[BOUND:.*]] = acc.bounds lowerbound(%{{.*}} : index) upperbound(%{{.*}} : index) extent(%[[DIMS]]#1 : index) stride(%[[DIMS]]#2 : index) startIdx(%{{.*}} : index) {strideInBytes = true}
-!CHECK: %[[ADDR:.*]] = fir.box_addr %[[DECLB]]#0 : (!fir.box<!fir.array<?xf32>>) -> !fir.ref<!fir.array<?xf32>>
-!CHECK: %[[CREATE:.*]] = acc.create varPtr(%[[ADDR]] : !fir.ref<!fir.array<?xf32>>) bounds(%[[BOUND]]) -> !fir.ref<!fir.array<?xf32>> {name = "b", structured = false}
-!CHECK: acc.enter_data dataOperands(%[[CREATE]] : !fir.ref<!fir.array<?xf32>>)
+!CHECK: %[[CREATE:.*]] = acc.create var(%[[DECLB]]#0 : !fir.box<!fir.array<?xf32>>) -> !fir.box<!fir.array<?xf32>> {name = "b", structured = false}
+!CHECK: acc.enter_data dataOperands(%[[CREATE]] : !fir.box<!fir.array<?xf32>>)
 
   !$acc enter data create(a(5:10))
 !CHECK: %[[ONE:.*]] = arith.constant 1 : index
@@ -242,22 +182,20 @@ subroutine acc_enter_data_dummy(a, b, n, m)
 !CHECK: %[[M_CONV2:.*]] = fir.convert %[[M_CONV1]] : (i64) -> index
 !CHECK: %[[UB:.*]] = arith.subi %[[M_CONV2]], %[[N_IDX]] : index
 !CHECK: %[[BOUND1:.*]] = acc.bounds lowerbound(%[[LB]] : index) upperbound(%[[UB]] : index) extent(%[[EXT_B]] : index) stride(%[[DIMS0]]#2 : index) startIdx(%[[N_IDX]] : index) {strideInBytes = true}
-!CHECK: %[[ADDR:.*]] = fir.box_addr %[[DECLB]]#0 : (!fir.box<!fir.array<?xf32>>) -> !fir.ref<!fir.array<?xf32>>
-!CHECK: %[[CREATE1:.*]] = acc.create varPtr(%[[ADDR]] : !fir.ref<!fir.array<?xf32>>) bounds(%[[BOUND1]]) -> !fir.ref<!fir.array<?xf32>> {name = "b(n:m)", structured = false}
-!CHECK: acc.enter_data dataOperands(%[[CREATE1]] : !fir.ref<!fir.array<?xf32>>)
+!CHECK: %[[CREATE1:.*]] = acc.create var(%[[DECLB]]#0 : !fir.box<!fir.array<?xf32>>) bounds(%[[BOUND1]]) -> !fir.box<!fir.array<?xf32>> {name = "b(n:m)", structured = false}
+!CHECK: acc.enter_data dataOperands(%[[CREATE1]] : !fir.box<!fir.array<?xf32>>)
 
   !$acc enter data create(b(n:))
 !CHECK: %[[ONE:.*]] = arith.constant 1 : index
-!CHECK: %[[DIMS0:.*]]:3 = fir.box_dims %[[DECLB]]#0, %c0_8 : (!fir.box<!fir.array<?xf32>>, index) -> (index, index, index)
+!CHECK: %[[DIMS0:.*]]:3 = fir.box_dims %[[DECLB]]#0, %c0{{.*}} : (!fir.box<!fir.array<?xf32>>, index) -> (index, index, index)
 !CHECK: %[[LOAD_N:.*]] = fir.load %[[DECLN]]#0 : !fir.ref<i32>
 !CHECK: %[[CONVERT1_N:.*]] = fir.convert %[[LOAD_N]] : (i32) -> i64
 !CHECK: %[[CONVERT2_N:.*]] = fir.convert %[[CONVERT1_N]] : (i64) -> index
 !CHECK: %[[LB:.*]] = arith.subi %[[CONVERT2_N]], %[[N_IDX]] : index
 !CHECK: %[[UB:.*]] = arith.subi %[[EXT_B]], %c1{{.*}} : index
 !CHECK: %[[BOUND1:.*]] = acc.bounds lowerbound(%[[LB]] : index) upperbound(%[[UB]] : index) extent(%[[EXT_B]] : index) stride(%[[DIMS0]]#2 : index) startIdx(%[[N_IDX]] : index) {strideInBytes = true}
-!CHECK: %[[ADDR:.*]] = fir.box_addr %[[DECLB]]#0 : (!fir.box<!fir.array<?xf32>>) -> !fir.ref<!fir.array<?xf32>>
-!CHECK: %[[CREATE1:.*]] = acc.create varPtr(%[[ADDR]] : !fir.ref<!fir.array<?xf32>>) bounds(%[[BOUND1]]) -> !fir.ref<!fir.array<?xf32>> {name = "b(n:)", structured = false}
-!CHECK: acc.enter_data dataOperands(%[[CREATE1]] : !fir.ref<!fir.array<?xf32>>)
+!CHECK: %[[CREATE1:.*]] = acc.create var(%[[DECLB]]#0 : !fir.box<!fir.array<?xf32>>) bounds(%[[BOUND1]]) -> !fir.box<!fir.array<?xf32>> {name = "b(n:)", structured = false}
+!CHECK: acc.enter_data dataOperands(%[[CREATE1]] : !fir.box<!fir.array<?xf32>>)
 
   !$acc enter data create(b(:))
 !CHECK: %[[ZERO:.*]] = arith.constant 0 : index
@@ -265,9 +203,8 @@ subroutine acc_enter_data_dummy(a, b, n, m)
 !CHECK: %[[DIMS0:.*]]:3 = fir.box_dims %[[DECLB]]#0, %c0{{.*}} : (!fir.box<!fir.array<?xf32>>, index) -> (index, index, index)
 !CHECK: %[[UB:.*]] = arith.subi %[[EXT_B]], %[[ONE]] : index
 !CHECK: %[[BOUND1:.*]] = acc.bounds lowerbound(%[[ZERO]] : index) upperbound(%[[UB]] : index) extent(%[[EXT_B]] : index) stride(%[[DIMS0]]#2 : index) startIdx(%[[N_IDX]] : index) {strideInBytes = true}
-!CHECK: %[[ADDR:.*]] = fir.box_addr %[[DECLB]]#0 : (!fir.box<!fir.array<?xf32>>) -> !fir.ref<!fir.array<?xf32>>
-!CHECK: %[[CREATE1:.*]] = acc.create varPtr(%[[ADDR]] : !fir.ref<!fir.array<?xf32>>) bounds(%[[BOUND1]]) -> !fir.ref<!fir.array<?xf32>> {name = "b(:)", structured = false}
-!CHECK: acc.enter_data dataOperands(%[[CREATE1]] : !fir.ref<!fir.array<?xf32>>)
+!CHECK: %[[CREATE1:.*]] = acc.create var(%[[DECLB]]#0 : !fir.box<!fir.array<?xf32>>) bounds(%[[BOUND1]]) -> !fir.box<!fir.array<?xf32>> {name = "b(:)", structured = false}
+!CHECK: acc.enter_data dataOperands(%[[CREATE1]] : !fir.box<!fir.array<?xf32>>)
 
 end subroutine
 
@@ -290,27 +227,24 @@ subroutine acc_enter_data_non_default_lb()
 !CHECK: %[[SECTIONUB:.*]] = arith.constant 9 : index
 !CHECK: %[[UB:.*]] = arith.subi %[[SECTIONUB]], %[[BASELB]] : index
 !CHECK: %[[BOUND:.*]] = acc.bounds lowerbound(%[[LB]] : index) upperbound(%[[UB]] : index) extent(%c10{{.*}} : index) stride(%{{.*}} : index) startIdx(%[[BASELB]] : index)
-!CHECK: %[[ADDR:.*]] = fir.box_addr %[[DECLA]]#0 : (!fir.box<!fir.array<10xi32>>) -> !fir.ref<!fir.array<10xi32>>
-!CHECK: %[[CREATE:.*]] = acc.create varPtr(%[[ADDR]] : !fir.ref<!fir.array<10xi32>>) bounds(%[[BOUND]]) -> !fir.ref<!fir.array<10xi32>> {name = "a(5:9)", structured = false}
-!CHECK: acc.enter_data dataOperands(%[[CREATE]] : !fir.ref<!fir.array<10xi32>>)
+!CHECK: %[[CREATE:.*]] = acc.create var(%[[DECLA]]#0 : !fir.box<!fir.array<10xi32>>) bounds(%[[BOUND]]) -> !fir.box<!fir.array<10xi32>> {name = "a(5:9)", structured = false}
+!CHECK: acc.enter_data dataOperands(%[[CREATE]] : !fir.box<!fir.array<10xi32>>)
 
   !$acc enter data create(a(:))
 !CHECK: %[[ZERO:.*]] = arith.constant 0 : index
 !CHECK: %[[ONE:.*]] = arith.constant 1 : index
 !CHECK: %[[UB:.*]] = arith.subi %[[EXTENT_C10]], %[[ONE]] : index
 !CHECK: %[[BOUND:.*]] = acc.bounds lowerbound(%[[ZERO]] : index) upperbound(%[[UB]] : index) extent(%[[EXTENT_C10]] : index) stride(%{{.*}} : index) startIdx(%[[BASELB]] : index)
-!CHECK: %[[ADDR:.*]] = fir.box_addr %[[DECLA]]#0 : (!fir.box<!fir.array<10xi32>>) -> !fir.ref<!fir.array<10xi32>>
-!CHECK: %[[CREATE:.*]] = acc.create varPtr(%[[ADDR]] : !fir.ref<!fir.array<10xi32>>) bounds(%[[BOUND]]) -> !fir.ref<!fir.array<10xi32>> {name = "a(:)", structured = false}
-!CHECK: acc.enter_data dataOperands(%[[CREATE]] : !fir.ref<!fir.array<10xi32>>)
+!CHECK: %[[CREATE:.*]] = acc.create var(%[[DECLA]]#0 : !fir.box<!fir.array<10xi32>>) bounds(%[[BOUND]]) -> !fir.box<!fir.array<10xi32>> {name = "a(:)", structured = false}
+!CHECK: acc.enter_data dataOperands(%[[CREATE]] : !fir.box<!fir.array<10xi32>>)
 
   !$acc enter data create(a(:6))
 !CHECK: %[[ZERO:.*]] = arith.constant 0 : index
 !CHECK: %[[SECTIONUB:.*]] = arith.constant 6 : index
 !CHECK: %[[UB:.*]] = arith.subi %[[SECTIONUB]], %[[BASELB]] : index
 !CHECK: %[[BOUND:.*]] = acc.bounds lowerbound(%[[ZERO]] : index) upperbound(%[[UB]] : index) extent(%c10{{.*}} : index) stride(%{{.*}} : index) startIdx(%[[BASELB]] : index)
-!CHECK: %[[ADDR:.*]] = fir.box_addr %[[DECLA]]#0 : (!fir.box<!fir.array<10xi32>>) -> !fir.ref<!fir.array<10xi32>>
-!CHECK: %[[CREATE:.*]] = acc.create varPtr(%[[ADDR]] : !fir.ref<!fir.array<10xi32>>) bounds(%[[BOUND]]) -> !fir.ref<!fir.array<10xi32>> {name = "a(:6)", structured = false}
-!CHECK: acc.enter_data dataOperands(%[[CREATE]] : !fir.ref<!fir.array<10xi32>>)
+!CHECK: %[[CREATE:.*]] = acc.create var(%[[DECLA]]#0 : !fir.box<!fir.array<10xi32>>) bounds(%[[BOUND]]) -> !fir.box<!fir.array<10xi32>> {name = "a(:6)", structured = false}
+!CHECK: acc.enter_data dataOperands(%[[CREATE]] : !fir.box<!fir.array<10xi32>>)
 
   !$acc enter data create(a(4:))
 !CHECK: %[[ONE:.*]] = arith.constant 1 : index
@@ -318,19 +252,12 @@ subroutine acc_enter_data_non_default_lb()
 !CHECK: %[[LB:.*]] = arith.subi %[[SECTIONLB]], %[[BASELB]] : index
 !CHECK: %[[UB:.*]] = arith.subi %[[EXTENT_C10]], %[[ONE]] : index
 !CHECK: %[[BOUND:.*]] = acc.bounds lowerbound(%[[LB]] : index) upperbound(%[[UB]] : index) extent(%[[EXTENT_C10]] : index) stride(%{{.*}} : index) startIdx(%[[BASELB]] : index)
-!CHECK: %[[ADDR:.*]] = fir.box_addr %[[DECLA]]#0 : (!fir.box<!fir.array<10xi32>>) -> !fir.ref<!fir.array<10xi32>>
-!CHECK: %[[CREATE:.*]] = acc.create varPtr(%[[ADDR]] : !fir.ref<!fir.array<10xi32>>) bounds(%[[BOUND]]) -> !fir.ref<!fir.array<10xi32>> {name = "a(4:)", structured = false}
-!CHECK: acc.enter_data dataOperands(%[[CREATE]] : !fir.ref<!fir.array<10xi32>>)
+!CHECK: %[[CREATE:.*]] = acc.create var(%[[DECLA]]#0 : !fir.box<!fir.array<10xi32>>) bounds(%[[BOUND]]) -> !fir.box<!fir.array<10xi32>> {name = "a(4:)", structured = false}
+!CHECK: acc.enter_data dataOperands(%[[CREATE]] : !fir.box<!fir.array<10xi32>>)
 
   !$acc enter data create(b)
-!CHECK: %[[ONE:.*]] = arith.constant 1 : index
-!CHECK: %[[DIMS0:.*]]:3 = fir.box_dims %[[DECLB]]#0, %c0{{.*}} : (!fir.box<!fir.array<10xi32>>, index) -> (index, index, index)
-!CHECK: %[[LB:.*]] = arith.constant 0 : index
-!CHECK: %[[UB:.*]] = arith.subi %[[DIMS0]]#1, %[[ONE]] : index
-!CHECK: %[[BOUND:.*]] = acc.bounds lowerbound(%c0{{.*}} : index) upperbound(%[[UB]] : index) extent(%[[DIMS0]]#1 : index) stride(%{{.*}} : index) startIdx(%c11{{.*}} : index) {strideInBytes = true}
-!CHECK: %[[ADDR:.*]] = fir.box_addr %[[DECLB]]#0 : (!fir.box<!fir.array<10xi32>>) -> !fir.ref<!fir.array<10xi32>>
-!CHECK: %[[CREATE:.*]] = acc.create varPtr(%[[ADDR]] : !fir.ref<!fir.array<10xi32>>) bounds(%[[BOUND]]) -> !fir.ref<!fir.array<10xi32>> {name = "b", structured = false}
-!CHECK: acc.enter_data dataOperands(%[[CREATE]] : !fir.ref<!fir.array<10xi32>>)
+!CHECK: %[[CREATE:.*]] = acc.create var(%[[DECLB]]#0 : !fir.box<!fir.array<10xi32>>) -> !fir.box<!fir.array<10xi32>> {name = "b", structured = false}
+!CHECK: acc.enter_data dataOperands(%[[CREATE]] : !fir.box<!fir.array<10xi32>>)
 
 end subroutine
 
@@ -350,15 +277,8 @@ subroutine acc_enter_data_assumed(a, b, n, m)
 !CHECK: %[[DECLN:.*]]:2 = hlfir.declare %[[N]]
 
   !$acc enter data create(a)
-!CHECK: %[[C1:.*]] = arith.constant 1 : index
-!CHECK: %[[C0:.*]] = arith.constant 0 : index
-!CHECK: %[[DIMS:.*]]:3 = fir.box_dims %[[DECLA]]#0, %[[C0]] : (!fir.box<!fir.array<?xf32>>, index) -> (index, index, index)
-!CHECK: %[[LB:.*]] = arith.constant 0 : index
-!CHECK: %[[UB:.*]] = arith.subi %[[DIMS]]#1, %[[C1]] : index
-!CHECK: %[[BOUND:.*]] = acc.bounds lowerbound(%[[LB]] : index) upperbound(%[[UB]] : index) extent(%[[DIMS]]#1 : index) stride(%[[DIMS]]#2 : index) startIdx(%[[C1]] : index) {strideInBytes = true}
-!CHECK: %[[BOX_ADDR:.*]] = fir.box_addr %[[DECLA]]#0 : (!fir.box<!fir.array<?xf32>>) -> !fir.ref<!fir.array<?xf32>>
-!CHECK: %[[CREATE:.*]] = acc.create varPtr(%[[BOX_ADDR]] : !fir.ref<!fir.array<?xf32>>) bounds(%[[BOUND]]) -> !fir.ref<!fir.array<?xf32>> {name = "a", structured = false}
-!CHECK: acc.enter_data dataOperands(%[[CREATE]] : !fir.ref<!fir.array<?xf32>>)
+!CHECK: %[[CREATE:.*]] = acc.create var(%[[DECLA]]#0 : !fir.box<!fir.array<?xf32>>) -> !fir.box<!fir.array<?xf32>> {name = "a", structured = false}
+!CHECK: acc.enter_data dataOperands(%[[CREATE]] : !fir.box<!fir.array<?xf32>>)
 
   !$acc enter data create(a(:))
 !CHECK: %[[LB:.*]] = arith.constant 0 : index
@@ -372,9 +292,8 @@ subroutine acc_enter_data_assumed(a, b, n, m)
 !CHECK: %[[UB:.*]] = arith.subi %[[DIMS1]]#1, %[[ONE]] : index
 !CHECK: %[[BOUND:.*]] = acc.bounds lowerbound(%[[LB]] : index) upperbound(%[[UB]] : index) extent(%[[DIMS1]]#1 : index) stride(%[[DIMS0]]#2 : index) startIdx(%[[ONE]] : index) {strideInBytes = true}
 
-!CHECK: %[[BOX_ADDR:.*]] = fir.box_addr %[[DECLA]]#0 : (!fir.box<!fir.array<?xf32>>) -> !fir.ref<!fir.array<?xf32>>
-!CHECK: %[[CREATE:.*]] = acc.create varPtr(%[[BOX_ADDR]] : !fir.ref<!fir.array<?xf32>>) bounds(%[[BOUND]]) -> !fir.ref<!fir.array<?xf32>> {name = "a(:)", structured = false}
-!CHECK: acc.enter_data dataOperands(%[[CREATE]] : !fir.ref<!fir.array<?xf32>>)
+!CHECK: %[[CREATE:.*]] = acc.create var(%[[DECLA]]#0 : !fir.box<!fir.array<?xf32>>) bounds(%[[BOUND]]) -> !fir.box<!fir.array<?xf32>> {name = "a(:)", structured = false}
+!CHECK: acc.enter_data dataOperands(%[[CREATE]] : !fir.box<!fir.array<?xf32>>)
 
   !$acc enter data create(a(2:))
 !CHECK: %[[ONE:.*]] = arith.constant 1 : index
@@ -388,9 +307,8 @@ subroutine acc_enter_data_assumed(a, b, n, m)
 !CHECK: %[[UB:.*]] = arith.subi %[[DIMS1]]#1, %[[ONE]] : index
 !CHECK: %[[BOUND:.*]] = acc.bounds lowerbound(%[[LB]] : index) upperbound(%[[UB]] : index) extent(%[[DIMS1]]#1 : index) stride(%[[DIMS0]]#2 : index) startIdx(%[[ONE]] : index) {strideInBytes = true}
 
-!CHECK: %[[BOX_ADDR:.*]] = fir.box_addr %[[DECLA]]#0 : (!fir.box<!fir.array<?xf32>>) -> !fir.ref<!fir.array<?xf32>>
-!CHECK: %[[CREATE:.*]] = acc.create varPtr(%[[BOX_ADDR]] : !fir.ref<!fir.array<?xf32>>) bounds(%[[BOUND]]) -> !fir.ref<!fir.array<?xf32>> {name = "a(2:)", structured = false}
-!CHECK: acc.enter_data dataOperands(%[[CREATE]] : !fir.ref<!fir.array<?xf32>>)
+!CHECK: %[[CREATE:.*]] = acc.create var(%[[DECLA]]#0 : !fir.box<!fir.array<?xf32>>) bounds(%[[BOUND]]) -> !fir.box<!fir.array<?xf32>> {name = "a(2:)", structured = false}
+!CHECK: acc.enter_data dataOperands(%[[CREATE]] : !fir.box<!fir.array<?xf32>>)
 
   !$acc enter data create(a(:4))
 !CHECK: %[[LB:.*]] = arith.constant 0 : index
@@ -401,114 +319,82 @@ subroutine acc_enter_data_assumed(a, b, n, m)
 !CHECK: %[[UB:.*]] = arith.constant 3 : index
 !CHECK: %[[DIMS1:.*]]:3 = fir.box_dims %[[DECLA]]#1, %{{.*}} : (!fir.box<!fir.array<?xf32>>, index) -> (index, index, index)
 !CHECK: %[[BOUND:.*]] = acc.bounds lowerbound(%[[LB]] : index) upperbound(%[[UB]] : index) extent(%[[DIMS1]]#1 : index) stride(%[[DIMS0]]#2 : index) startIdx(%[[ONE]] : index) {strideInBytes = true}
-
-!CHECK: %[[BOX_ADDR:.*]] = fir.box_addr %[[DECLA]]#0 : (!fir.box<!fir.array<?xf32>>) -> !fir.ref<!fir.array<?xf32>>
-!CHECK: %[[CREATE:.*]] = acc.create varPtr(%[[BOX_ADDR]] : !fir.ref<!fir.array<?xf32>>) bounds(%[[BOUND]]) -> !fir.ref<!fir.array<?xf32>> {name = "a(:4)", structured = false}
-!CHECK: acc.enter_data dataOperands(%[[CREATE]] : !fir.ref<!fir.array<?xf32>>)
+!CHECK: %[[CREATE:.*]] = acc.create var(%[[DECLA]]#0 : !fir.box<!fir.array<?xf32>>) bounds(%[[BOUND]]) -> !fir.box<!fir.array<?xf32>> {name = "a(:4)", structured = false}
+!CHECK: acc.enter_data dataOperands(%[[CREATE]] : !fir.box<!fir.array<?xf32>>)
 
   !$acc enter data create(a(6:10))
 !CHECK: %[[ONE:.*]] = arith.constant 1 : index
 !CHECK: %[[C0:.*]] = arith.constant 0 : index
-
 !CHECK: %[[DIMS0:.*]]:3 = fir.box_dims %[[DECLA]]#0, %[[C0]] : (!fir.box<!fir.array<?xf32>>, index) -> (index, index, index)
 !CHECK: %[[LB:.*]] = arith.constant 5 : index
 !CHECK: %[[UB:.*]] = arith.constant 9 : index
 !CHECK: %[[DIMS1:.*]]:3 = fir.box_dims %[[DECLA]]#1, %{{.*}} : (!fir.box<!fir.array<?xf32>>, index) -> (index, index, index)
 !CHECK: %[[BOUND:.*]] = acc.bounds lowerbound(%[[LB]] : index) upperbound(%[[UB]] : index) extent(%[[DIMS1]]#1 : index) stride(%[[DIMS0]]#2 : index) startIdx(%[[ONE]] : index) {strideInBytes = true}
-
-!CHECK: %[[BOX_ADDR:.*]] = fir.box_addr %[[DECLA]]#0 : (!fir.box<!fir.array<?xf32>>) -> !fir.ref<!fir.array<?xf32>>
-!CHECK: %[[CREATE:.*]] = acc.create varPtr(%[[BOX_ADDR]] : !fir.ref<!fir.array<?xf32>>) bounds(%[[BOUND]]) -> !fir.ref<!fir.array<?xf32>> {name = "a(6:10)", structured = false}
-!CHECK: acc.enter_data dataOperands(%[[CREATE]] : !fir.ref<!fir.array<?xf32>>)
+!CHECK: %[[CREATE:.*]] = acc.create var(%[[DECLA]]#0 : !fir.box<!fir.array<?xf32>>) bounds(%[[BOUND]]) -> !fir.box<!fir.array<?xf32>> {name = "a(6:10)", structured = false}
+!CHECK: acc.enter_data dataOperands(%[[CREATE]] : !fir.box<!fir.array<?xf32>>)
 
   !$acc enter data create(a(n:))
 !CHECK: %[[ONE:.*]] = arith.constant 1 : index
 !CHECK: %[[C0:.*]] = arith.constant 0 : index
-
 !CHECK: %[[DIMS0:.*]]:3 = fir.box_dims %[[DECLA]]#0, %[[C0]] : (!fir.box<!fir.array<?xf32>>, index) -> (index, index, index)
-
 !CHECK: %[[LOAD_N:.*]] = fir.load %[[DECLN]]#0 : !fir.ref<i32>
 !CHECK: %[[CONVERT1_N:.*]] = fir.convert %[[LOAD_N]] : (i32) -> i64
 !CHECK: %[[CONVERT2_N:.*]] = fir.convert %[[CONVERT1_N]] : (i64) -> index
 !CHECK: %[[LB:.*]] = arith.subi %[[CONVERT2_N]], %[[ONE]] : index
 !CHECK: %[[C0:.*]] = arith.constant 0 : index
-
 !CHECK: %[[DIMS:.*]]:3 = fir.box_dims %[[DECLA]]#1, %[[C0]] : (!fir.box<!fir.array<?xf32>>, index) -> (index, index, index)
 !CHECK: %[[UB:.*]] = arith.subi %[[DIMS]]#1, %[[ONE]] : index
 !CHECK: %[[BOUND:.*]] = acc.bounds lowerbound(%[[LB]] : index) upperbound(%[[UB]] : index) extent(%[[DIMS]]#1 : index) stride(%[[DIMS0]]#2 : index) startIdx(%[[ONE]] : index) {strideInBytes = true}
-
-!CHECK: %[[BOX_ADDR:.*]] = fir.box_addr %[[DECLA]]#0 : (!fir.box<!fir.array<?xf32>>) -> !fir.ref<!fir.array<?xf32>>
-!CHECK: %[[CREATE:.*]] = acc.create varPtr(%[[BOX_ADDR]] : !fir.ref<!fir.array<?xf32>>) bounds(%[[BOUND]]) -> !fir.ref<!fir.array<?xf32>> {name = "a(n:)", structured = false}
-!CHECK: acc.enter_data dataOperands(%[[CREATE]] : !fir.ref<!fir.array<?xf32>>)
+!CHECK: %[[CREATE:.*]] = acc.create var(%[[DECLA]]#0 : !fir.box<!fir.array<?xf32>>) bounds(%[[BOUND]]) -> !fir.box<!fir.array<?xf32>> {name = "a(n:)", structured = false}
+!CHECK: acc.enter_data dataOperands(%[[CREATE]] : !fir.box<!fir.array<?xf32>>)
 
   !$acc enter data create(a(:m))
 !CHECK: %[[BASELB:.*]] = arith.constant 0 : index
 !CHECK: %[[ONE:.*]] = arith.constant 1 : index
 !CHECK: %[[C0:.*]] = arith.constant 0 : index
-
 !CHECK: %[[DIMS0:.*]]:3 = fir.box_dims %[[DECLA]]#0, %[[C0]] : (!fir.box<!fir.array<?xf32>>, index) -> (index, index, index)
-
 !CHECK: %[[LOAD_M:.*]] = fir.load %[[DECLM]]#0 : !fir.ref<i32>
 !CHECK: %[[CONVERT1_M:.*]] = fir.convert %[[LOAD_M]] : (i32) -> i64
 !CHECK: %[[CONVERT2_M:.*]] = fir.convert %[[CONVERT1_M]] : (i64) -> index
 !CHECK: %[[UB:.*]] = arith.subi %[[CONVERT2_M]], %[[ONE]] : index
 !CHECK: %[[DIMS1:.*]]:3 = fir.box_dims %[[DECLA]]#1, %{{.*}} : (!fir.box<!fir.array<?xf32>>, index) -> (index, index, index)
 !CHECK: %[[BOUND:.*]] = acc.bounds lowerbound(%[[BASELB]] : index) upperbound(%[[UB]] : index) extent(%[[DIMS1]]#1 : index) stride(%[[DIMS0]]#2 : index) startIdx(%[[ONE]] : index) {strideInBytes = true}
-
-!CHECK: %[[BOX_ADDR:.*]] = fir.box_addr %[[DECLA]]#0 : (!fir.box<!fir.array<?xf32>>) -> !fir.ref<!fir.array<?xf32>>
-!CHECK: %[[CREATE:.*]] = acc.create varPtr(%[[BOX_ADDR]] : !fir.ref<!fir.array<?xf32>>) bounds(%[[BOUND]]) -> !fir.ref<!fir.array<?xf32>> {name = "a(:m)", structured = false}
-!CHECK: acc.enter_data dataOperands(%[[CREATE]] : !fir.ref<!fir.array<?xf32>>)
+!CHECK: %[[CREATE:.*]] = acc.create var(%[[DECLA]]#0 : !fir.box<!fir.array<?xf32>>) bounds(%[[BOUND]]) -> !fir.box<!fir.array<?xf32>> {name = "a(:m)", structured = false}
+!CHECK: acc.enter_data dataOperands(%[[CREATE]] : !fir.box<!fir.array<?xf32>>)
 
   !$acc enter data create(a(n:m))
 !CHECK: %[[ONE:.*]] = arith.constant 1 : index
 !CHECK: %[[C0:.*]] = arith.constant 0 : index
-
 !CHECK: %[[DIMS0:.*]]:3 = fir.box_dims %[[DECLA]]#0, %[[C0]] : (!fir.box<!fir.array<?xf32>>, index) -> (index, index, index)
-
 !CHECK: %[[LOAD_N:.*]] = fir.load %[[DECLN]]#0 : !fir.ref<i32>
 !CHECK: %[[CONVERT1_N:.*]] = fir.convert %[[LOAD_N]] : (i32) -> i64
 !CHECK: %[[CONVERT2_N:.*]] = fir.convert %[[CONVERT1_N]] : (i64) -> index
 !CHECK: %[[LB:.*]] = arith.subi %[[CONVERT2_N]], %[[ONE]] : index
-
 !CHECK: %[[LOAD_M:.*]] = fir.load %[[DECLM]]#0 : !fir.ref<i32>
 !CHECK: %[[CONVERT1_M:.*]] = fir.convert %[[LOAD_M]] : (i32) -> i64
 !CHECK: %[[CONVERT2_M:.*]] = fir.convert %[[CONVERT1_M]] : (i64) -> index
 !CHECK: %[[UB:.*]] = arith.subi %[[CONVERT2_M]], %[[ONE]] : index
 !CHECK: %[[DIMS1:.*]]:3 = fir.box_dims %[[DECLA]]#1, %{{.*}} : (!fir.box<!fir.array<?xf32>>, index) -> (index, index, index)
 !CHECK: %[[BOUND:.*]] = acc.bounds lowerbound(%[[LB]] : index) upperbound(%[[UB]] : index) extent(%[[DIMS1]]#1 : index) stride(%[[DIMS0]]#2 : index) startIdx(%[[ONE]] : index) {strideInBytes = true}
-
-!CHECK: %[[BOX_ADDR:.*]] = fir.box_addr %[[DECLA]]#0 : (!fir.box<!fir.array<?xf32>>) -> !fir.ref<!fir.array<?xf32>>
-!CHECK: %[[CREATE:.*]] = acc.create varPtr(%[[BOX_ADDR]] : !fir.ref<!fir.array<?xf32>>) bounds(%[[BOUND]]) -> !fir.ref<!fir.array<?xf32>> {name = "a(n:m)", structured = false}
-!CHECK: acc.enter_data dataOperands(%[[CREATE]] : !fir.ref<!fir.array<?xf32>>)
+!CHECK: %[[CREATE:.*]] = acc.create var(%[[DECLA]]#0 : !fir.box<!fir.array<?xf32>>) bounds(%[[BOUND]]) -> !fir.box<!fir.array<?xf32>> {name = "a(n:m)", structured = false}
+!CHECK: acc.enter_data dataOperands(%[[CREATE]] : !fir.box<!fir.array<?xf32>>)
 
   !$acc enter data create(b(:m))
 !CHECK: %[[ZERO:.*]] = arith.constant 0 : index
 !CHECK: %[[C0:.*]] = arith.constant 0 : index
-
 !CHECK: %[[DIMS0:.*]]:3 = fir.box_dims %[[DECLB]]#0, %[[C0]] : (!fir.box<!fir.array<?xf32>>, index) -> (index, index, index)
-
 !CHECK: %[[LOAD_M:.*]] = fir.load %[[DECLM]]#0 : !fir.ref<i32>
 !CHECK: %[[CONVERT1_M:.*]] = fir.convert %[[LOAD_M]] : (i32) -> i64
 !CHECK: %[[CONVERT2_M:.*]] = fir.convert %[[CONVERT1_M]] : (i64) -> index
 !CHECK: %[[UB:.*]] = arith.subi %[[CONVERT2_M]], %[[LB_C10_IDX]] : index
 !CHECK: %[[DIMS1:.*]]:3 = fir.box_dims %[[DECLB]]#1, %{{.*}} : (!fir.box<!fir.array<?xf32>>, index) -> (index, index, index)
 !CHECK: %[[BOUND:.*]] = acc.bounds lowerbound(%[[ZERO]] : index) upperbound(%[[UB]] : index) extent(%[[DIMS1]]#1 : index) stride(%[[DIMS0]]#2 : index) startIdx(%[[LB_C10_IDX]] : index) {strideInBytes = true}
-
-!CHECK: %[[BOX_ADDR:.*]] = fir.box_addr %[[DECLB]]#0 : (!fir.box<!fir.array<?xf32>>) -> !fir.ref<!fir.array<?xf32>>
-!CHECK: %[[CREATE:.*]] = acc.create varPtr(%[[BOX_ADDR]] : !fir.ref<!fir.array<?xf32>>) bounds(%[[BOUND]]) -> !fir.ref<!fir.array<?xf32>> {name = "b(:m)", structured = false}
-!CHECK: acc.enter_data dataOperands(%[[CREATE]] : !fir.ref<!fir.array<?xf32>>)
+!CHECK: %[[CREATE:.*]] = acc.create var(%[[DECLB]]#0 : !fir.box<!fir.array<?xf32>>) bounds(%[[BOUND]]) -> !fir.box<!fir.array<?xf32>> {name = "b(:m)", structured = false}
+!CHECK: acc.enter_data dataOperands(%[[CREATE]] : !fir.box<!fir.array<?xf32>>)
 
   !$acc enter data create(b)
-!CHECK: %[[ONE:.*]] = arith.constant 1 : index
-!CHECK: %[[C0:.*]] = arith.constant 0 : index
-
-!CHECK: %[[DIMS0:.*]]:3 = fir.box_dims %[[DECLB]]#0, %[[C0]] : (!fir.box<!fir.array<?xf32>>, index) -> (index, index, index)
-!CHECK: %[[C0:.*]] = arith.constant 0 : index
-!CHECK: %[[UB:.*]] = arith.subi %[[DIMS0]]#1, %[[ONE]] : index
-!CHECK: %[[BOUND:.*]] = acc.bounds lowerbound(%[[C0]] : index) upperbound(%[[UB]] : index) extent(%[[DIMS0]]#1 : index) stride(%[[DIMS0]]#2 : index) startIdx(%[[LB_C10_IDX]] : index) {strideInBytes = true}
-
-!CHECK: %[[BOX_ADDR:.*]] = fir.box_addr %[[DECLB]]#0 : (!fir.box<!fir.array<?xf32>>) -> !fir.ref<!fir.array<?xf32>>
-!CHECK: %[[CREATE:.*]] = acc.create varPtr(%[[BOX_ADDR]] : !fir.ref<!fir.array<?xf32>>) bounds(%[[BOUND]]) -> !fir.ref<!fir.array<?xf32>> {name = "b", structured = false}
-!CHECK: acc.enter_data dataOperands(%[[CREATE]] : !fir.ref<!fir.array<?xf32>>)
+!CHECK: %[[CREATE:.*]] = acc.create var(%[[DECLB]]#0 : !fir.box<!fir.array<?xf32>>) -> !fir.box<!fir.array<?xf32>> {name = "b", structured = false}
+!CHECK: acc.enter_data dataOperands(%[[CREATE]] : !fir.box<!fir.array<?xf32>>)
 
 end subroutine
 
@@ -524,46 +410,33 @@ subroutine acc_enter_data_allocatable()
 
   !$acc enter data create(a)
 
-!CHECK: %[[BOX_A_0:.*]] = fir.load %[[DECLA]]#0 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>
-!CHECK: %[[C0_0:.*]] = arith.constant 0 : index
-!CHECK: %[[BOX_A_1:.*]] = fir.load %[[DECLA]]#1 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>
-!CHECK: %[[C0_1:.*]] = arith.constant 0 : index
-!CHECK: %[[DIMS0:.*]]:3 = fir.box_dims %[[BOX_A_1]], %c0{{.*}} : (!fir.box<!fir.heap<!fir.array<?xf32>>>, index) -> (index, index, index)
-!CHECK: %[[DIMS1:.*]]:3 = fir.box_dims %[[BOX_A_0]], %c0{{.*}} : (!fir.box<!fir.heap<!fir.array<?xf32>>>, index) -> (index, index, index)
-!CHECK: %[[UB:.*]] = arith.subi %[[DIMS1]]#1, %c1{{.*}} : index
-!CHECK: %[[BOUND:.*]] = acc.bounds lowerbound(%c0{{.*}} : index) upperbound(%[[UB]] : index) extent(%[[DIMS1]]#1 : index) stride(%[[DIMS1]]#2 : index) startIdx(%[[DIMS0]]#0 : index) {strideInBytes = true}
-!CHECK: %[[BOX_ADDR:.*]] = fir.box_addr %[[BOX_A_0]] : (!fir.box<!fir.heap<!fir.array<?xf32>>>) -> !fir.heap<!fir.array<?xf32>>
-!CHECK: %[[CREATE:.*]] = acc.create varPtr(%[[BOX_ADDR]] : !fir.heap<!fir.array<?xf32>>) bounds(%[[BOUND]]) -> !fir.heap<!fir.array<?xf32>> {name = "a", structured = false}
-!CHECK: acc.enter_data dataOperands(%[[CREATE]] : !fir.heap<!fir.array<?xf32>>)
+!CHECK: %[[CREATE:.*]] = acc.create varPtr(%[[DECLA]]#0 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>) -> !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>> {name = "a", structured = false}
+!CHECK: acc.enter_data dataOperands(%[[CREATE]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>)
 
   !$acc enter data create(a(:))
 
-!CHECK: %[[BOX_A_0:.*]] = fir.load %[[DECLA]]#0 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>
 !CHECK: %[[ZERO:.*]] = arith.constant 0 : index
 !CHECK: %[[ONE:.*]] = arith.constant 1 : index
-
 !CHECK: %[[BOX_A_1:.*]] = fir.load %[[DECLA]]#1 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>
 !CHECK: %[[C0:.*]] = arith.constant 0 : index
 !CHECK: %[[DIMS0:.*]]:3 = fir.box_dims %[[BOX_A_1]], %[[C0]] : (!fir.box<!fir.heap<!fir.array<?xf32>>>, index) -> (index, index, index)
+!CHECK: %[[BOX_A_0:.*]] = fir.load %[[DECLA]]#0 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>
 !CHECK: %[[C0:.*]] = arith.constant 0 : index
 !CHECK: %[[DIMS1:.*]]:3 = fir.box_dims %[[BOX_A_0]], %[[C0]] : (!fir.box<!fir.heap<!fir.array<?xf32>>>, index) -> (index, index, index)
-
 !CHECK: %[[BOX_A_2:.*]] = fir.load %[[DECLA]]#1 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>
 !CHECK: %[[C0:.*]] = arith.constant 0 : index
 !CHECK: %[[DIMS2:.*]]:3 = fir.box_dims %[[BOX_A_2]], %[[C0]] : (!fir.box<!fir.heap<!fir.array<?xf32>>>, index) -> (index, index, index)
 !CHECK: %[[UB:.*]] = arith.subi %[[DIMS2]]#1, %[[ONE]] : index
 !CHECK: %[[BOUND:.*]] = acc.bounds lowerbound(%[[ZERO]] : index) upperbound(%[[UB:.*]] : index) extent(%[[DIMS2]]#1 : index) stride(%[[DIMS1]]#2 : index) startIdx(%[[DIMS0]]#0 : index) {strideInBytes = true}
-!CHECK: %[[BOX_ADDR:.*]] = fir.box_addr %[[BOX_A_0]] : (!fir.box<!fir.heap<!fir.array<?xf32>>>) -> !fir.heap<!fir.array<?xf32>>
-!CHECK: %[[CREATE:.*]] = acc.create varPtr(%[[BOX_ADDR]] : !fir.heap<!fir.array<?xf32>>) bounds(%[[BOUND]]) -> !fir.heap<!fir.array<?xf32>> {name = "a(:)", structured = false}
-!CHECK: acc.enter_data dataOperands(%[[CREATE]] : !fir.heap<!fir.array<?xf32>>)
+!CHECK: %[[CREATE:.*]] = acc.create varPtr(%[[DECLA]]#0 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>) bounds(%[[BOUND]]) -> !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>> {name = "a(:)", structured = false}
+!CHECK: acc.enter_data dataOperands(%[[CREATE]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>)
 
   !$acc enter data create(a(2:5))
 
-!CHECK: %[[BOX_A_0:.*]] = fir.load %[[DECLA]]#0 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>
-
 !CHECK: %[[BOX_A_1:.*]] = fir.load %[[DECLA]]#1 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>
 !CHECK: %[[C0:.*]] = arith.constant 0 : index
 !CHECK: %[[DIMS0:.*]]:3 = fir.box_dims %[[BOX_A_1]], %[[C0]] : (!fir.box<!fir.heap<!fir.array<?xf32>>>, index) -> (index, index, index)
+!CHECK: %[[BOX_A_0:.*]] = fir.load %[[DECLA]]#0 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>
 !CHECK: %[[C0:.*]] = arith.constant 0 : index
 !CHECK: %[[DIMS1:.*]]:3 = fir.box_dims %[[BOX_A_0]], %[[C0]] : (!fir.box<!fir.heap<!fir.array<?xf32>>>, index) -> (index, index, index)
 !CHECK: %[[C2:.*]] = arith.constant 2 : index
@@ -574,40 +447,35 @@ subroutine acc_enter_data_allocatable()
 !CHECK: %[[C0:.*]] = arith.constant 0 : index
 !CHECK: %[[DIMS2:.*]]:3 = fir.box_dims %[[BOX_A_2]], %[[C0]] : (!fir.box<!fir.heap<!fir.array<?xf32>>>, index) -> (index, index, index)
 !CHECK: %[[BOUND:.*]] = acc.bounds lowerbound(%[[LB]] : index) upperbound(%[[UB]] : index) extent(%[[DIMS2]]#1 : index) stride(%[[DIMS1]]#2 : index) startIdx(%[[DIMS0]]#0 : index) {strideInBytes = true}
-!CHECK: %[[BOX_ADDR:.*]] = fir.box_addr %[[BOX_A_0]] : (!fir.box<!fir.heap<!fir.array<?xf32>>>) -> !fir.heap<!fir.array<?xf32>>
-!CHECK: %[[CREATE:.*]] = acc.create varPtr(%[[BOX_ADDR]] : !fir.heap<!fir.array<?xf32>>) bounds(%[[BOUND]]) -> !fir.heap<!fir.array<?xf32>> {name = "a(2:5)", structured = false}
-!CHECK: acc.enter_data dataOperands(%[[CREATE]] : !fir.heap<!fir.array<?xf32>>)
+!CHECK: %[[CREATE:.*]] = acc.create varPtr(%[[DECLA]]#0 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>) bounds(%[[BOUND]]) -> !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>> {name = "a(2:5)", structured = false}
+!CHECK: acc.enter_data dataOperands(%[[CREATE]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>)
 
   !$acc enter data create(a(3:))
 
-!CHECK: %[[BOX_A_0:.*]] = fir.load %[[DECLA]]#0 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>
 !CHECK: %[[ONE:.*]] = arith.constant 1 : index
-
 !CHECK: %[[BOX_A_1:.*]] = fir.load %[[DECLA]]#1 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>
 !CHECK: %[[C0:.*]] = arith.constant 0 : index
 !CHECK: %[[DIMS0:.*]]:3 = fir.box_dims %[[BOX_A_1]], %[[C0]] : (!fir.box<!fir.heap<!fir.array<?xf32>>>, index) -> (index, index, index)
+!CHECK: %[[BOX_A_0:.*]] = fir.load %[[DECLA]]#0 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>
 !CHECK: %[[C0:.*]] = arith.constant 0 : index
 !CHECK: %[[DIMS1:.*]]:3 = fir.box_dims %[[BOX_A_0]], %[[C0]] : (!fir.box<!fir.heap<!fir.array<?xf32>>>, index) -> (index, index, index)
 !CHECK: %[[C3:.*]] = arith.constant 3 : index
 !CHECK: %[[LB:.*]] = arith.subi %[[C3]], %[[DIMS0]]#0 : index
-
 !CHECK: %[[BOX_A_2:.*]] = fir.load %[[DECLA]]#1 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>
 !CHECK: %[[C0:.*]] = arith.constant 0 : index
 !CHECK: %[[DIMS2:.*]]:3 = fir.box_dims %[[BOX_A_2]], %[[C0]] : (!fir.box<!fir.heap<!fir.array<?xf32>>>, index) -> (index, index, index)
 !CHECK: %[[UB:.*]] = arith.subi %[[DIMS2]]#1, %[[ONE]] : index
 !CHECK: %[[BOUND:.*]] = acc.bounds lowerbound(%[[LB]] : index) upperbound(%[[UB]] : index) extent(%[[DIMS2]]#1 : index) stride(%[[DIMS1]]#2 : index) startIdx(%[[DIMS0]]#0 : index) {strideInBytes = true}
-!CHECK: %[[BOX_ADDR:.*]] = fir.box_addr %[[BOX_A_0]] : (!fir.box<!fir.heap<!fir.array<?xf32>>>) -> !fir.heap<!fir.array<?xf32>>
-!CHECK: %[[CREATE:.*]] = acc.create varPtr(%[[BOX_ADDR]] : !fir.heap<!fir.array<?xf32>>) bounds(%[[BOUND]]) -> !fir.heap<!fir.array<?xf32>> {name = "a(3:)", structured = false}
-!CHECK: acc.enter_data dataOperands(%[[CREATE]] : !fir.heap<!fir.array<?xf32>>)
+!CHECK: %[[CREATE:.*]] = acc.create varPtr(%[[DECLA]]#0 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>) bounds(%[[BOUND]]) -> !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>> {name = "a(3:)", structured = false}
+!CHECK: acc.enter_data dataOperands(%[[CREATE]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>)
 
   !$acc enter data create(a(:7))
 
-!CHECK: %[[BOX_A_0:.*]] = fir.load %[[DECLA]]#0 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>
 !CHECK: %[[ZERO:.*]] = arith.constant 0 : index
-
 !CHECK: %[[BOX_A_1:.*]] = fir.load %[[DECLA]]#1 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>
 !CHECK: %[[C0:.*]] = arith.constant 0 : index
 !CHECK: %[[DIMS0:.*]]:3 = fir.box_dims %[[BOX_A_1]], %[[C0]] : (!fir.box<!fir.heap<!fir.array<?xf32>>>, index) -> (index, index, index)
+!CHECK: %[[BOX_A_0:.*]] = fir.load %[[DECLA]]#0 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>
 !CHECK: %[[C0:.*]] = arith.constant 0 : index
 !CHECK: %[[DIMS1:.*]]:3 = fir.box_dims %[[BOX_A_0]], %[[C0]] : (!fir.box<!fir.heap<!fir.array<?xf32>>>, index) -> (index, index, index)
 !CHECK: %[[C7:.*]] = arith.constant 7 : index
@@ -616,16 +484,13 @@ subroutine acc_enter_data_allocatable()
 !CHECK: %[[C0:.*]] = arith.constant 0 : index
 !CHECK: %[[DIMS2:.*]]:3 = fir.box_dims %[[BOX_A_2]], %[[C0]] : (!fir.box<!fir.heap<!fir.array<?xf32>>>, index) -> (index, index, index)
 !CHECK: %[[BOUND:.*]] = acc.bounds lowerbound(%[[ZERO]] : index) upperbound(%[[UB]] : index) extent(%[[DIMS2]]#1 : index) stride(%[[DIMS1]]#2 : index) startIdx(%[[DIMS0]]#0 : index) {strideInBytes = true}
-!CHECK: %[[BOX_ADDR:.*]] = fir.box_addr %[[BOX_A_0]] : (!fir.box<!fir.heap<!fir.array<?xf32>>>) -> !fir.heap<!fir.array<?xf32>>
-!CHECK: %[[CREATE:.*]] = acc.create varPtr(%[[BOX_ADDR]] : !fir.heap<!fir.array<?xf32>>) bounds(%[[BOUND]]) -> !fir.heap<!fir.array<?xf32>> {name = "a(:7)", structured = false}
-!CHECK: acc.enter_data dataOperands(%[[CREATE]] : !fir.heap<!fir.array<?xf32>>)
+!CHECK: %[[CREATE:.*]] = acc.create varPtr(%[[DECLA]]#0 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>) bounds(%[[BOUND]]) -> !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>> {name = "a(:7)", structured = false}
+!CHECK: acc.enter_data dataOperands(%[[CREATE]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>)
 
   !$acc enter data create(i)
 
-!CHECK: %[[BOX_I:.*]] = fir.load %[[DECLI]]#0 : !fir.ref<!fir.box<!fir.heap<i32>>>
-!CHECK: %[[BOX_ADDR:.*]] = fir.box_addr %[[BOX_I]] : (!fir.box<!fir.heap<i32>>) -> !fir.heap<i32>
-!CHECK: %[[CREATE:.*]] = acc.create varPtr(%[[BOX_ADDR]] : !fir.heap<i32>) -> !fir.heap<i32> {name = "i", structured = false}
-!CHECK: acc.enter_data dataOperands(%[[CREATE]] : !fir.heap<i32>)
+!CHECK: %[[CREATE:.*]] = acc.create varPtr(%[[DECLI]]#0 : !fir.ref<!fir.box<!fir.heap<i32>>>) -> !fir.ref<!fir.box<!fir.heap<i32>>> {name = "i", structured = false}
+!CHECK: acc.enter_data dataOperands(%[[CREATE]] : !fir.ref<!fir.box<!fir.heap<i32>>>)
 
 end subroutine
 
@@ -684,13 +549,8 @@ subroutine acc_enter_data_derived_type()
   !$acc enter data create(a%array)
 
 
-!CHECK: %[[C10:.*]] = arith.constant 10 : index
 !CHECK: %[[ARRAY_COORD:.*]] = hlfir.designate %[[DECLA]]#0{"array"}   shape %{{.*}} : (!fir.ref<!fir.type<_QFacc_enter_data_derived_typeTdt{data:f32,array:!fir.array<10xf32>}>>, !fir.shape<1>) -> !fir.ref<!fir.array<10xf32>>
-!CHECK: %[[C1:.*]] = arith.constant 1 : index
-!CHECK: %[[LB:.*]] = arith.constant 0 : index
-!CHECK: %[[UB:.*]] = arith.subi %[[C10]], %[[C1]] : index
-!CHECK: %[[BOUND:.*]] = acc.bounds lowerbound(%[[LB]] : index) upperbound(%[[UB]] : index) extent(%[[C10]] : index) stride(%[[C1]] : index) startIdx(%[[C1]] : index)
-!CHECK: %[[CREATE:.*]] = acc.create varPtr(%[[ARRAY_COORD]] : !fir.ref<!fir.array<10xf32>>) bounds(%[[BOUND]]) -> !fir.ref<!fir.array<10xf32>> {name = "a%array", structured = false}
+!CHECK: %[[CREATE:.*]] = acc.create varPtr(%[[ARRAY_COORD]] : !fir.ref<!fir.array<10xf32>>) -> !fir.ref<!fir.array<10xf32>> {name = "a%array", structured = false}
 !CHECK: acc.enter_data dataOperands(%[[CREATE]] : !fir.ref<!fir.array<10xf32>>)
 
   !$acc enter data create(a%array(:))
@@ -744,48 +604,23 @@ subroutine acc_enter_data_derived_type()
 
 
 !CHECK: %[[D_COORD:.*]] = hlfir.designate %[[DECLB]]#0{"d"}   : (!fir.ref<!fir.type<_QFacc_enter_data_derived_typeTt{d:!fir.type<_QFacc_enter_data_derived_typeTdt{data:f32,array:!fir.array<10xf32>}>}>>) -> !fir.ref<!fir.type<_QFacc_enter_data_derived_typeTdt{data:f32,array:!fir.array<10xf32>}>>
-!CHECK: %[[C10:.*]] = arith.constant 10 : index
 !CHECK: %[[ARRAY_COORD:.*]] = hlfir.designate %[[D_COORD]]{"array"}   shape %{{.*}} : (!fir.ref<!fir.type<_QFacc_enter_data_derived_typeTdt{data:f32,array:!fir.array<10xf32>}>>, !fir.shape<1>) -> !fir.ref<!fir.array<10xf32>>
-!CHECK: %[[C1:.*]] = arith.constant 1 : index
-!CHECK: %[[LB:.*]] = arith.constant 0 : index
-!CHECK: %[[UB:.*]] = arith.subi %[[C10]], %[[C1]] : index
-!CHECK: %[[BOUND:.*]] = acc.bounds lowerbound(%[[LB]] : index) upperbound(%[[UB]] : index) extent(%[[C10]] : index) stride(%[[C1]] : index) startIdx(%[[C1]] : index)
-!CHECK: %[[CREATE:.*]] = acc.create varPtr(%[[ARRAY_COORD]] : !fir.ref<!fir.array<10xf32>>) bounds(%[[BOUND]]) -> !fir.ref<!fir.array<10xf32>> {name = "b%d%array", structured = false}
+!CHECK: %[[CREATE:.*]] = acc.create varPtr(%[[ARRAY_COORD]] : !fir.ref<!fir.array<10xf32>>) -> !fir.ref<!fir.array<10xf32>> {name = "b%d%array", structured = false}
 
   !$acc enter data create(c%data)
 
 
 !CHECK: %[[DATA_COORD:.*]] = hlfir.designate %[[DECLC]]#0{"data"}   {fortran_attrs = #fir.var_attrs<allocatable>} : (!fir.ref<!fir.type<_QFacc_enter_data_derived_typeTz{data:!fir.box<!fir.heap<!fir.array<?xi32>>>}>>) -> !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>
 !CHECK: %[[DATA_BOX:.*]] = fir.load %[[DATA_COORD]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>
-!CHECK: %[[DIM0:.*]] = arith.constant 0 : index
-!CHECK: %[[DIMS0:.*]]:3 = fir.box_dims %[[DATA_BOX]], %[[DIM0]] : (!fir.box<!fir.heap<!fir.array<?xi32>>>, index) -> (index, index, index)
-!CHECK: %[[ONE:.*]] = arith.constant 1 : index
-!CHECK: %[[DIM0_1:.*]] = arith.constant 0 : index
-!CHECK: %[[DIMS0_1:.*]]:3 = fir.box_dims %[[DATA_BOX]], %[[DIM0_1]] : (!fir.box<!fir.heap<!fir.array<?xi32>>>, index) -> (index, index, index)
-!CHECK: %[[UB:.*]] = arith.subi %[[DIMS0_1]]#1, %[[ONE]] : index
-!CHECK: %[[BOUND:.*]] = acc.bounds lowerbound(%c0{{.*}} : index) upperbound(%[[UB]] : index) extent(%[[DIMS0_1]]#1 : index) stride(%[[DIMS0_1]]#2 : index) startIdx(%[[DIMS0]]#0 : index) {strideInBytes = true}
-!CHECK: %[[BOX_ADDR:.*]] = fir.box_addr %[[DATA_BOX]] : (!fir.box<!fir.heap<!fir.array<?xi32>>>) -> !fir.heap<!fir.array<?xi32>>
-!CHECK: %[[CREATE:.*]] = acc.create varPtr(%[[BOX_ADDR]] : !fir.heap<!fir.array<?xi32>>) bounds(%[[BOUND]]) -> !fir.heap<!fir.array<?xi32>> {name = "c%data", structured = false}
-!CHECK: acc.enter_data dataOperands(%[[CREATE]] : !fir.heap<!fir.array<?xi32>>)
+!CHECK: %[[CREATE:.*]] = acc.create var(%[[DATA_BOX]] : !fir.box<!fir.heap<!fir.array<?xi32>>>) -> !fir.box<!fir.heap<!fir.array<?xi32>>> {name = "c%data", structured = false}
+!CHECK: acc.enter_data dataOperands(%[[CREATE]] : !fir.box<!fir.heap<!fir.array<?xi32>>>)
 
   !$acc enter data create (d%d(1)%array)
 
-
-
-
-
-
 !CHECK: %[[ONE:.*]] = arith.constant 1 : index
 !CHECK: %[[D1_COORD:.*]] = hlfir.designate %[[DECLD]]#0{"d"} <%{{.*}}> (%[[ONE]])  : (!fir.ref<!fir.type<_QFacc_enter_data_derived_typeTtt{d:!fir.array<10x!fir.type<_QFacc_enter_data_derived_typeTdt{data:f32,array:!fir.array<10xf32>}>>}>>, !fir.shape<1>, index) -> !fir.ref<!fir.type<_QFacc_enter_data_derived_typeTdt{data:f32,array:!fir.array<10xf32>}>>
-
-
-!CHECK: %[[C10:.*]] = arith.constant 10 : index
 !CHECK: %[[ARRAY_COORD:.*]] = hlfir.designate %[[D1_COORD]]{"array"}   shape %{{.*}} : (!fir.ref<!fir.type<_QFacc_enter_data_derived_typeTdt{data:f32,array:!fir.array<10xf32>}>>, !fir.shape<1>) -> !fir.ref<!fir.array<10xf32>>
-!CHECK: %[[C1:.*]] = arith.constant 1 : index
-!CHECK: %[[LB:.*]] = arith.constant 0 : index
-!CHECK: %[[UB:.*]] = arith.subi %[[C10]], %[[C1]] : index
-!CHECK: %[[BOUND:.*]] = acc.bounds lowerbound(%[[LB]] : index) upperbound(%[[UB]] : index) extent(%[[C10]] : index) stride(%[[C1]] : index) startIdx(%[[C1]] : index)
-!CHECK: %[[CREATE:.*]] = acc.create varPtr(%[[ARRAY_COORD]] : !fir.ref<!fir.array<10xf32>>) bounds(%[[BOUND]]) -> !fir.ref<!fir.array<10xf32>> {name = "d%d(1_8)%array", structured = false}
+!CHECK: %[[CREATE:.*]] = acc.create varPtr(%[[ARRAY_COORD]] : !fir.ref<!fir.array<10xf32>>) -> !fir.ref<!fir.array<10xf32>> {name = "d%d(1_8)%array", structured = false}
 !CHECK: acc.enter_data dataOperands(%[[CREATE]] : !fir.ref<!fir.array<10xf32>>)
 
 end subroutine
diff --git a/flang/test/Lower/OpenACC/acc-exit-data-unwrap-defaultbounds.f90 b/flang/test/Lower/OpenACC/acc-exit-data-unwrap-defaultbounds.f90
new file mode 100644
index 0000000000000..7999a7647f49b
--- /dev/null
+++ b/flang/test/Lower/OpenACC/acc-exit-data-unwrap-defaultbounds.f90
@@ -0,0 +1,107 @@
+! This test checks lowering of OpenACC exit data directive.
+
+! RUN: bbc -fopenacc -emit-hlfir --openacc-unwrap-fir-box=true --openacc-generate-default-bounds=true %s -o - | FileCheck %s
+
+subroutine acc_exit_data
+  integer :: async = 1
+  real, dimension(10, 10) :: a, b, c
+  real, pointer :: d
+  logical :: ifCondition = .TRUE.
+
+!CHECK: %[[A:.*]] = fir.alloca !fir.array<10x10xf32> {{{.*}}uniq_name = "{{.*}}Ea"}
+!CHECK: %[[DECLA:.*]]:2 = hlfir.declare %[[A]]
+!CHECK: %[[B:.*]] = fir.alloca !fir.array<10x10xf32> {{{.*}}uniq_name = "{{.*}}Eb"}
+!CHECK: %[[DECLB:.*]]:2 = hlfir.declare %[[B]]
+!CHECK: %[[C:.*]] = fir.alloca !fir.array<10x10xf32> {{{.*}}uniq_name = "{{.*}}Ec"}
+!CHECK: %[[DECLC:.*]]:2 = hlfir.declare %[[C]]
+!CHECK: %[[D:.*]] = fir.alloca !fir.box<!fir.ptr<f32>> {bindc_name = "d", uniq_name = "{{.*}}Ed"}
+!CHECK: %[[DECLD:.*]]:2 = hlfir.declare %[[D]]
+
+  !$acc exit data delete(a)
+!CHECK: %[[DEVPTR:.*]] = acc.getdeviceptr varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = #acc<data_clause acc_delete>, name = "a", structured = false}
+!CHECK: acc.exit_data dataOperands(%[[DEVPTR]] : !fir.ref<!fir.array<10x10xf32>>)
+!CHECK: acc.delete accPtr(%[[DEVPTR]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) {name = "a", structured = false}
+
+  !$acc exit data delete(a) if(.true.)
+!CHECK: %[[DEVPTR:.*]] = acc.getdeviceptr varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>)   bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = #acc<data_clause acc_delete>, name = "a", structured = false}
+!CHECK: %[[IF1:.*]] = arith.constant true
+!CHECK: acc.exit_data if(%[[IF1]]) dataOperands(%[[DEVPTR]] : !fir.ref<!fir.array<10x10xf32>>)
+!CHECK: acc.delete accPtr(%[[DEVPTR]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) {name = "a", structured = false}
+
+  !$acc exit data delete(a) if(ifCondition)
+!CHECK: %[[DEVPTR:.*]] = acc.getdeviceptr varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>)   bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = #acc<data_clause acc_delete>, name = "a", structured = false}
+!CHECK: %[[IFCOND:.*]] = fir.load %{{.*}} : !fir.ref<!fir.logical<4>>
+!CHECK: %[[IF2:.*]] = fir.convert %[[IFCOND]] : (!fir.logical<4>) -> i1
+!CHECK: acc.exit_data if(%[[IF2]]) dataOperands(%[[DEVPTR]] : !fir.ref<!fir.array<10x10xf32>>){{$}}
+!CHECK: acc.delete accPtr(%[[DEVPTR]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) {name = "a", structured = false}
+
+  !$acc exit data delete(a) delete(b) delete(c)
+!CHECK: %[[DEVPTR_A:.*]] = acc.getdeviceptr varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>)   bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = #acc<data_clause acc_delete>, name = "a", structured = false}
+!CHECK: %[[DEVPTR_B:.*]] = acc.getdeviceptr varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<10x10xf32>>)   bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = #acc<data_clause acc_delete>, name = "b", structured = false}
+!CHECK: %[[DEVPTR_C:.*]] = acc.getdeviceptr varPtr(%[[DECLC]]#0 : !fir.ref<!fir.array<10x10xf32>>)   bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = #acc<data_clause acc_delete>, name = "c", structured = false}
+!CHECK: acc.exit_data dataOperands(%[[DEVPTR_A]], %[[DEVPTR_B]], %[[DEVPTR_C]] : !fir.ref<!fir.array<10x10xf32>>, !fir.ref<!fir.array<10x10xf32>>, !fir.ref<!fir.array<10x10xf32>>){{$}}
+!CHECK: acc.delete accPtr(%[[DEVPTR_A]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) {name = "a", structured = false}
+!CHECK: acc.delete accPtr(%[[DEVPTR_B]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) {name = "b", structured = false}
+!CHECK: acc.delete accPtr(%[[DEVPTR_C]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) {name = "c", structured = false}
+
+  !$acc exit data copyout(a) delete(b) detach(d)
+!CHECK: %[[DEVPTR_A:.*]] = acc.getdeviceptr varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = #acc<data_clause acc_copyout>, name = "a", structured = false}
+!CHECK: %[[DEVPTR_B:.*]] = acc.getdeviceptr varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<10x10xf32>>)   bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = #acc<data_clause acc_delete>, name = "b", structured = false}
+!CHECK: %[[BOX_D:.*]] = fir.load %[[DECLD]]#0 : !fir.ref<!fir.box<!fir.ptr<f32>>>
+!CHECK: %[[D_ADDR:.*]] = fir.box_addr %[[BOX_D]] : (!fir.box<!fir.ptr<f32>>) -> !fir.ptr<f32>
+!CHECK: %[[DEVPTR_D:.*]] = acc.getdeviceptr varPtr(%[[D_ADDR]] : !fir.ptr<f32>) -> !fir.ptr<f32> {dataClause = #acc<data_clause acc_detach>, name = "d", structured = false}
+!CHECK: acc.exit_data dataOperands(%[[DEVPTR_A]], %[[DEVPTR_B]], %[[DEVPTR_D]] : !fir.ref<!fir.array<10x10xf32>>, !fir.ref<!fir.array<10x10xf32>>, !fir.ptr<f32>)
+!CHECK: acc.copyout accPtr(%[[DEVPTR_A]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) to varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) {name = "a", structured = false}
+!CHECK: acc.delete accPtr(%[[DEVPTR_B]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) {name = "b", structured = false}
+!CHECK: acc.detach accPtr(%[[DEVPTR_D]] : !fir.ptr<f32>) {name = "d", structured = false}
+
+  !$acc exit data delete(a) async
+!CHECK: %[[DEVPTR:.*]] = acc.getdeviceptr varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {asyncOnly = [#acc.device_type<none>], dataClause = #acc<data_clause acc_delete>, name = "a", structured = false}
+!CHECK: acc.exit_data dataOperands(%[[DEVPTR]] : !fir.ref<!fir.array<10x10xf32>>) attributes {async}
+!CHECK: acc.delete accPtr(%[[DEVPTR]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) {asyncOnly = [#acc.device_type<none>], name = "a", structured = false}
+
+  !$acc exit data delete(a) wait
+!CHECK: %[[DEVPTR:.*]] = acc.getdeviceptr varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = #acc<data_clause acc_delete>, name = "a", structured = false}
+!CHECK: acc.exit_data dataOperands(%[[DEVPTR]] : !fir.ref<!fir.array<10x10xf32>>) attributes {wait}
+!CHECK: acc.delete accPtr(%[[DEVPTR]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) {name = "a", structured = false}
+
+  !$acc exit data delete(a) async wait
+!CHECK: %[[DEVPTR:.*]] = acc.getdeviceptr varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {asyncOnly = [#acc.device_type<none>], dataClause = #acc<data_clause acc_delete>, name = "a", structured = false}
+!CHECK: acc.exit_data dataOperands(%[[DEVPTR]] : !fir.ref<!fir.array<10x10xf32>>) attributes {async, wait}
+!CHECK: acc.delete accPtr(%[[DEVPTR]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) {asyncOnly = [#acc.device_type<none>], name = "a", structured = false}
+
+  !$acc exit data delete(a) async(1)
+!CHECK: %[[ASYNC1:.*]] = arith.constant 1 : i32
+!CHECK: %[[DEVPTR:.*]] = acc.getdeviceptr varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) async(%[[ASYNC1]] : i32) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = #acc<data_clause acc_delete>, name = "a", structured = false}
+!CHECK: acc.exit_data async(%[[ASYNC1]] : i32) dataOperands(%[[DEVPTR]] : !fir.ref<!fir.array<10x10xf32>>)
+!CHECK: acc.delete accPtr(%[[DEVPTR]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) async(%[[ASYNC1]] : i32) {name = "a", structured = false}
+
+
+  !$acc exit data delete(a) async(async)
+!CHECK: %[[ASYNC2:.*]] = fir.load %{{.*}} : !fir.ref<i32>
+!CHECK: %[[DEVPTR:.*]] = acc.getdeviceptr varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) async(%[[ASYNC2]] : i32) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = #acc<data_clause acc_delete>, name = "a", structured = false}
+!CHECK: acc.exit_data async(%[[ASYNC2]] : i32) dataOperands(%[[DEVPTR]] : !fir.ref<!fir.array<10x10xf32>>)
+!CHECK: acc.delete accPtr(%[[DEVPTR]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) async(%[[ASYNC2]] : i32) {name = "a", structured = false}
+
+  !$acc exit data delete(a) wait(1)
+!CHECK: %[[DEVPTR:.*]] = acc.getdeviceptr varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = #acc<data_clause acc_delete>, name = "a", structured = false}
+!CHECK: %[[WAIT1:.*]] = arith.constant 1 : i32
+!CHECK: acc.exit_data wait(%[[WAIT1]] : i32) dataOperands(%[[DEVPTR]] : !fir.ref<!fir.array<10x10xf32>>)
+!CHECK: acc.delete accPtr(%[[DEVPTR]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) {name = "a", structured = false}
+
+  !$acc exit data delete(a) wait(queues: 1, 2)
+!CHECK: %[[DEVPTR:.*]] = acc.getdeviceptr varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = #acc<data_clause acc_delete>, name = "a", structured = false}
+!CHECK: %[[WAIT2:.*]] = arith.constant 1 : i32
+!CHECK: %[[WAIT3:.*]] = arith.constant 2 : i32
+!CHECK: acc.exit_data wait(%[[WAIT2]], %[[WAIT3]] : i32, i32) dataOperands(%[[DEVPTR]] : !fir.ref<!fir.array<10x10xf32>>)
+!CHECK: acc.delete accPtr(%[[DEVPTR]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) {name = "a", structured = false}
+
+  !$acc exit data delete(a) wait(devnum: 1: queues: 1, 2)
+!CHECK: %[[DEVPTR:.*]] = acc.getdeviceptr varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = #acc<data_clause acc_delete>, name = "a", structured = false}
+!CHECK: %[[WAIT4:.*]] = arith.constant 1 : i32
+!CHECK: %[[WAIT5:.*]] = arith.constant 2 : i32
+!CHECK: %[[WAIT6:.*]] = arith.constant 1 : i32
+!CHECK: acc.exit_data wait_devnum(%[[WAIT6]] : i32) wait(%[[WAIT4]], %[[WAIT5]] : i32, i32) dataOperands(%[[DEVPTR]] : !fir.ref<!fir.array<10x10xf32>>)
+!CHECK: acc.delete accPtr(%[[DEVPTR]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) {name = "a", structured = false}
+
+end subroutine acc_exit_data
diff --git a/flang/test/Lower/OpenACC/acc-exit-data.f90 b/flang/test/Lower/OpenACC/acc-exit-data.f90
index 017f1f38f8397..fbd1edfb6afe4 100644
--- a/flang/test/Lower/OpenACC/acc-exit-data.f90
+++ b/flang/test/Lower/OpenACC/acc-exit-data.f90
@@ -18,90 +18,88 @@ subroutine acc_exit_data
 !CHECK: %[[DECLD:.*]]:2 = hlfir.declare %[[D]]
 
   !$acc exit data delete(a)
-!CHECK: %[[DEVPTR:.*]] = acc.getdeviceptr varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = #acc<data_clause acc_delete>, name = "a", structured = false}
+!CHECK: %[[DEVPTR:.*]] = acc.getdeviceptr varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = #acc<data_clause acc_delete>, name = "a", structured = false}
 !CHECK: acc.exit_data dataOperands(%[[DEVPTR]] : !fir.ref<!fir.array<10x10xf32>>)
-!CHECK: acc.delete accPtr(%[[DEVPTR]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) {name = "a", structured = false}
+!CHECK: acc.delete accPtr(%[[DEVPTR]] : !fir.ref<!fir.array<10x10xf32>>) {name = "a", structured = false}
 
   !$acc exit data delete(a) if(.true.)
-!CHECK: %[[DEVPTR:.*]] = acc.getdeviceptr varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>)   bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = #acc<data_clause acc_delete>, name = "a", structured = false}
+!CHECK: %[[DEVPTR:.*]] = acc.getdeviceptr varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = #acc<data_clause acc_delete>, name = "a", structured = false}
 !CHECK: %[[IF1:.*]] = arith.constant true
 !CHECK: acc.exit_data if(%[[IF1]]) dataOperands(%[[DEVPTR]] : !fir.ref<!fir.array<10x10xf32>>)
-!CHECK: acc.delete accPtr(%[[DEVPTR]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) {name = "a", structured = false}
+!CHECK: acc.delete accPtr(%[[DEVPTR]] : !fir.ref<!fir.array<10x10xf32>>) {name = "a", structured = false}
 
   !$acc exit data delete(a) if(ifCondition)
-!CHECK: %[[DEVPTR:.*]] = acc.getdeviceptr varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>)   bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = #acc<data_clause acc_delete>, name = "a", structured = false}
+!CHECK: %[[DEVPTR:.*]] = acc.getdeviceptr varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = #acc<data_clause acc_delete>, name = "a", structured = false}
 !CHECK: %[[IFCOND:.*]] = fir.load %{{.*}} : !fir.ref<!fir.logical<4>>
 !CHECK: %[[IF2:.*]] = fir.convert %[[IFCOND]] : (!fir.logical<4>) -> i1
 !CHECK: acc.exit_data if(%[[IF2]]) dataOperands(%[[DEVPTR]] : !fir.ref<!fir.array<10x10xf32>>){{$}}
-!CHECK: acc.delete accPtr(%[[DEVPTR]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) {name = "a", structured = false}
+!CHECK: acc.delete accPtr(%[[DEVPTR]] : !fir.ref<!fir.array<10x10xf32>>) {name = "a", structured = false}
 
   !$acc exit data delete(a) delete(b) delete(c)
-!CHECK: %[[DEVPTR_A:.*]] = acc.getdeviceptr varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>)   bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = #acc<data_clause acc_delete>, name = "a", structured = false}
-!CHECK: %[[DEVPTR_B:.*]] = acc.getdeviceptr varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<10x10xf32>>)   bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = #acc<data_clause acc_delete>, name = "b", structured = false}
-!CHECK: %[[DEVPTR_C:.*]] = acc.getdeviceptr varPtr(%[[DECLC]]#0 : !fir.ref<!fir.array<10x10xf32>>)   bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = #acc<data_clause acc_delete>, name = "c", structured = false}
+!CHECK: %[[DEVPTR_A:.*]] = acc.getdeviceptr varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = #acc<data_clause acc_delete>, name = "a", structured = false}
+!CHECK: %[[DEVPTR_B:.*]] = acc.getdeviceptr varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<10x10xf32>>) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = #acc<data_clause acc_delete>, name = "b", structured = false}
+!CHECK: %[[DEVPTR_C:.*]] = acc.getdeviceptr varPtr(%[[DECLC]]#0 : !fir.ref<!fir.array<10x10xf32>>) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = #acc<data_clause acc_delete>, name = "c", structured = false}
 !CHECK: acc.exit_data dataOperands(%[[DEVPTR_A]], %[[DEVPTR_B]], %[[DEVPTR_C]] : !fir.ref<!fir.array<10x10xf32>>, !fir.ref<!fir.array<10x10xf32>>, !fir.ref<!fir.array<10x10xf32>>){{$}}
-!CHECK: acc.delete accPtr(%[[DEVPTR_A]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) {name = "a", structured = false}
-!CHECK: acc.delete accPtr(%[[DEVPTR_B]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) {name = "b", structured = false}
-!CHECK: acc.delete accPtr(%[[DEVPTR_C]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) {name = "c", structured = false}
+!CHECK: acc.delete accPtr(%[[DEVPTR_A]] : !fir.ref<!fir.array<10x10xf32>>) {name = "a", structured = false}
+!CHECK: acc.delete accPtr(%[[DEVPTR_B]] : !fir.ref<!fir.array<10x10xf32>>) {name = "b", structured = false}
+!CHECK: acc.delete accPtr(%[[DEVPTR_C]] : !fir.ref<!fir.array<10x10xf32>>) {name = "c", structured = false}
 
   !$acc exit data copyout(a) delete(b) detach(d)
-!CHECK: %[[DEVPTR_A:.*]] = acc.getdeviceptr varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = #acc<data_clause acc_copyout>, name = "a", structured = false}
-!CHECK: %[[DEVPTR_B:.*]] = acc.getdeviceptr varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<10x10xf32>>)   bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = #acc<data_clause acc_delete>, name = "b", structured = false}
-!CHECK: %[[BOX_D:.*]] = fir.load %[[DECLD]]#0 : !fir.ref<!fir.box<!fir.ptr<f32>>>
-!CHECK: %[[D_ADDR:.*]] = fir.box_addr %[[BOX_D]] : (!fir.box<!fir.ptr<f32>>) -> !fir.ptr<f32>
-!CHECK: %[[DEVPTR_D:.*]] = acc.getdeviceptr varPtr(%[[D_ADDR]] : !fir.ptr<f32>) -> !fir.ptr<f32> {dataClause = #acc<data_clause acc_detach>, name = "d", structured = false}
-!CHECK: acc.exit_data dataOperands(%[[DEVPTR_A]], %[[DEVPTR_B]], %[[DEVPTR_D]] : !fir.ref<!fir.array<10x10xf32>>, !fir.ref<!fir.array<10x10xf32>>, !fir.ptr<f32>)
-!CHECK: acc.copyout accPtr(%[[DEVPTR_A]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) to varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) {name = "a", structured = false}
-!CHECK: acc.delete accPtr(%[[DEVPTR_B]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) {name = "b", structured = false}
-!CHECK: acc.detach accPtr(%[[DEVPTR_D]] : !fir.ptr<f32>) {name = "d", structured = false}
+!CHECK: %[[DEVPTR_A:.*]] = acc.getdeviceptr varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = #acc<data_clause acc_copyout>, name = "a", structured = false}
+!CHECK: %[[DEVPTR_B:.*]] = acc.getdeviceptr varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<10x10xf32>>) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = #acc<data_clause acc_delete>, name = "b", structured = false}
+!CHECK: %[[DEVPTR_D:.*]] = acc.getdeviceptr varPtr(%[[DECLD]]#0 : !fir.ref<!fir.box<!fir.ptr<f32>>>) -> !fir.ref<!fir.box<!fir.ptr<f32>>> {dataClause = #acc<data_clause acc_detach>, name = "d", structured = false}
+!CHECK: acc.exit_data dataOperands(%[[DEVPTR_A]], %[[DEVPTR_B]], %[[DEVPTR_D]] : !fir.ref<!fir.array<10x10xf32>>, !fir.ref<!fir.array<10x10xf32>>, !fir.ref<!fir.box<!fir.ptr<f32>>>)
+!CHECK: acc.copyout accPtr(%[[DEVPTR_A]] : !fir.ref<!fir.array<10x10xf32>>) to varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) {name = "a", structured = false}
+!CHECK: acc.delete accPtr(%[[DEVPTR_B]] : !fir.ref<!fir.array<10x10xf32>>) {name = "b", structured = false}
+!CHECK: acc.detach accPtr(%[[DEVPTR_D]] : !fir.ref<!fir.box<!fir.ptr<f32>>>) {name = "d", structured = false}
 
   !$acc exit data delete(a) async
-!CHECK: %[[DEVPTR:.*]] = acc.getdeviceptr varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {asyncOnly = [#acc.device_type<none>], dataClause = #acc<data_clause acc_delete>, name = "a", structured = false}
+!CHECK: %[[DEVPTR:.*]] = acc.getdeviceptr varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) -> !fir.ref<!fir.array<10x10xf32>> {asyncOnly = [#acc.device_type<none>], dataClause = #acc<data_clause acc_delete>, name = "a", structured = false}
 !CHECK: acc.exit_data dataOperands(%[[DEVPTR]] : !fir.ref<!fir.array<10x10xf32>>) attributes {async}
-!CHECK: acc.delete accPtr(%[[DEVPTR]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) {asyncOnly = [#acc.device_type<none>], name = "a", structured = false}
+!CHECK: acc.delete accPtr(%[[DEVPTR]] : !fir.ref<!fir.array<10x10xf32>>) {asyncOnly = [#acc.device_type<none>], name = "a", structured = false}
 
   !$acc exit data delete(a) wait
-!CHECK: %[[DEVPTR:.*]] = acc.getdeviceptr varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = #acc<data_clause acc_delete>, name = "a", structured = false}
+!CHECK: %[[DEVPTR:.*]] = acc.getdeviceptr varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = #acc<data_clause acc_delete>, name = "a", structured = false}
 !CHECK: acc.exit_data dataOperands(%[[DEVPTR]] : !fir.ref<!fir.array<10x10xf32>>) attributes {wait}
-!CHECK: acc.delete accPtr(%[[DEVPTR]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) {name = "a", structured = false}
+!CHECK: acc.delete accPtr(%[[DEVPTR]] : !fir.ref<!fir.array<10x10xf32>>) {name = "a", structured = false}
 
   !$acc exit data delete(a) async wait
-!CHECK: %[[DEVPTR:.*]] = acc.getdeviceptr varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {asyncOnly = [#acc.device_type<none>], dataClause = #acc<data_clause acc_delete>, name = "a", structured = false}
+!CHECK: %[[DEVPTR:.*]] = acc.getdeviceptr varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) -> !fir.ref<!fir.array<10x10xf32>> {asyncOnly = [#acc.device_type<none>], dataClause = #acc<data_clause acc_delete>, name = "a", structured = false}
 !CHECK: acc.exit_data dataOperands(%[[DEVPTR]] : !fir.ref<!fir.array<10x10xf32>>) attributes {async, wait}
-!CHECK: acc.delete accPtr(%[[DEVPTR]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) {asyncOnly = [#acc.device_type<none>], name = "a", structured = false}
+!CHECK: acc.delete accPtr(%[[DEVPTR]] : !fir.ref<!fir.array<10x10xf32>>) {asyncOnly = [#acc.device_type<none>], name = "a", structured = false}
 
   !$acc exit data delete(a) async(1)
 !CHECK: %[[ASYNC1:.*]] = arith.constant 1 : i32
-!CHECK: %[[DEVPTR:.*]] = acc.getdeviceptr varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) async(%[[ASYNC1]] : i32) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = #acc<data_clause acc_delete>, name = "a", structured = false}
+!CHECK: %[[DEVPTR:.*]] = acc.getdeviceptr varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) async(%[[ASYNC1]] : i32) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = #acc<data_clause acc_delete>, name = "a", structured = false}
 !CHECK: acc.exit_data async(%[[ASYNC1]] : i32) dataOperands(%[[DEVPTR]] : !fir.ref<!fir.array<10x10xf32>>)
-!CHECK: acc.delete accPtr(%[[DEVPTR]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) async(%[[ASYNC1]] : i32) {name = "a", structured = false}
+!CHECK: acc.delete accPtr(%[[DEVPTR]] : !fir.ref<!fir.array<10x10xf32>>) async(%[[ASYNC1]] : i32) {name = "a", structured = false}
 
 
   !$acc exit data delete(a) async(async)
 !CHECK: %[[ASYNC2:.*]] = fir.load %{{.*}} : !fir.ref<i32>
-!CHECK: %[[DEVPTR:.*]] = acc.getdeviceptr varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) async(%[[ASYNC2]] : i32) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = #acc<data_clause acc_delete>, name = "a", structured = false}
+!CHECK: %[[DEVPTR:.*]] = acc.getdeviceptr varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) async(%[[ASYNC2]] : i32) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = #acc<data_clause acc_delete>, name = "a", structured = false}
 !CHECK: acc.exit_data async(%[[ASYNC2]] : i32) dataOperands(%[[DEVPTR]] : !fir.ref<!fir.array<10x10xf32>>)
-!CHECK: acc.delete accPtr(%[[DEVPTR]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) async(%[[ASYNC2]] : i32) {name = "a", structured = false}
+!CHECK: acc.delete accPtr(%[[DEVPTR]] : !fir.ref<!fir.array<10x10xf32>>) async(%[[ASYNC2]] : i32) {name = "a", structured = false}
 
   !$acc exit data delete(a) wait(1)
-!CHECK: %[[DEVPTR:.*]] = acc.getdeviceptr varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = #acc<data_clause acc_delete>, name = "a", structured = false}
+!CHECK: %[[DEVPTR:.*]] = acc.getdeviceptr varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = #acc<data_clause acc_delete>, name = "a", structured = false}
 !CHECK: %[[WAIT1:.*]] = arith.constant 1 : i32
 !CHECK: acc.exit_data wait(%[[WAIT1]] : i32) dataOperands(%[[DEVPTR]] : !fir.ref<!fir.array<10x10xf32>>)
-!CHECK: acc.delete accPtr(%[[DEVPTR]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) {name = "a", structured = false}
+!CHECK: acc.delete accPtr(%[[DEVPTR]] : !fir.ref<!fir.array<10x10xf32>>) {name = "a", structured = false}
 
   !$acc exit data delete(a) wait(queues: 1, 2)
-!CHECK: %[[DEVPTR:.*]] = acc.getdeviceptr varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = #acc<data_clause acc_delete>, name = "a", structured = false}
+!CHECK: %[[DEVPTR:.*]] = acc.getdeviceptr varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = #acc<data_clause acc_delete>, name = "a", structured = false}
 !CHECK: %[[WAIT2:.*]] = arith.constant 1 : i32
 !CHECK: %[[WAIT3:.*]] = arith.constant 2 : i32
 !CHECK: acc.exit_data wait(%[[WAIT2]], %[[WAIT3]] : i32, i32) dataOperands(%[[DEVPTR]] : !fir.ref<!fir.array<10x10xf32>>)
-!CHECK: acc.delete accPtr(%[[DEVPTR]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) {name = "a", structured = false}
+!CHECK: acc.delete accPtr(%[[DEVPTR]] : !fir.ref<!fir.array<10x10xf32>>) {name = "a", structured = false}
 
   !$acc exit data delete(a) wait(devnum: 1: queues: 1, 2)
-!CHECK: %[[DEVPTR:.*]] = acc.getdeviceptr varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = #acc<data_clause acc_delete>, name = "a", structured = false}
+!CHECK: %[[DEVPTR:.*]] = acc.getdeviceptr varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = #acc<data_clause acc_delete>, name = "a", structured = false}
 !CHECK: %[[WAIT4:.*]] = arith.constant 1 : i32
 !CHECK: %[[WAIT5:.*]] = arith.constant 2 : i32
 !CHECK: %[[WAIT6:.*]] = arith.constant 1 : i32
 !CHECK: acc.exit_data wait_devnum(%[[WAIT6]] : i32) wait(%[[WAIT4]], %[[WAIT5]] : i32, i32) dataOperands(%[[DEVPTR]] : !fir.ref<!fir.array<10x10xf32>>)
-!CHECK: acc.delete accPtr(%[[DEVPTR]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) {name = "a", structured = false}
+!CHECK: acc.delete accPtr(%[[DEVPTR]] : !fir.ref<!fir.array<10x10xf32>>) {name = "a", structured = false}
 
 end subroutine acc_exit_data
diff --git a/flang/test/Lower/OpenACC/acc-host-data-unwrap-defaultbounds.f90 b/flang/test/Lower/OpenACC/acc-host-data-unwrap-defaultbounds.f90
new file mode 100644
index 0000000000000..164eb32a8f684
--- /dev/null
+++ b/flang/test/Lower/OpenACC/acc-host-data-unwrap-defaultbounds.f90
@@ -0,0 +1,52 @@
+! This test checks lowering of OpenACC host_data directive.
+
+! RUN: bbc -fopenacc -emit-hlfir --openacc-unwrap-fir-box=true --openacc-generate-default-bounds=true %s -o - | FileCheck %s
+
+subroutine acc_host_data()
+  real, dimension(10) :: a
+  logical :: ifCondition = .TRUE.
+
+! CHECK: %[[A:.*]] = fir.alloca !fir.array<10xf32> {bindc_name = "a", uniq_name = "_QFacc_host_dataEa"}
+! CHECK: %[[DECLA:.*]]:2 = hlfir.declare %[[A]]
+! CHECK: %[[IFCOND:.*]] = fir.address_of(@_QFacc_host_dataEifcondition) : !fir.ref<!fir.logical<4>>
+! CHECK: %[[DECLIFCOND:.*]]:2 = hlfir.declare %[[IFCOND]]
+
+  !$acc host_data use_device(a)
+  !$acc end host_data
+
+! CHECK: %[[BOUND:.*]] = acc.bounds lowerbound(%{{.*}} : index) upperbound(%{{.*}} : index) stride(%{{.*}} : index) startIdx(%{{.*}} : index)
+! CHECK: %[[DA:.*]] = acc.use_device varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10xf32>>) bounds(%[[BOUND]]) -> !fir.ref<!fir.array<10xf32>> {name = "a"}
+! CHECK: acc.host_data dataOperands(%[[DA]] : !fir.ref<!fir.array<10xf32>>)
+
+  !$acc host_data use_device(a) if_present
+  !$acc end host_data
+
+! CHECK: %[[BOUND:.*]] = acc.bounds lowerbound(%{{.*}} : index) upperbound(%{{.*}} : index) stride(%{{.*}} : index) startIdx(%{{.*}} : index)
+! CHECK: %[[DA:.*]] = acc.use_device varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10xf32>>) bounds(%[[BOUND]]) -> !fir.ref<!fir.array<10xf32>> {name = "a"}
+! CHECK: acc.host_data dataOperands(%[[DA]] : !fir.ref<!fir.array<10xf32>>) {
+! CHECK: } attributes {ifPresent}
+
+  !$acc host_data use_device(a) if(ifCondition)
+  !$acc end host_data
+
+! CHECK: %[[BOUND:.*]] = acc.bounds lowerbound(%{{.*}} : index) upperbound(%{{.*}} : index) stride(%{{.*}} : index) startIdx(%{{.*}} : index)
+! CHECK: %[[DA:.*]] = acc.use_device varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10xf32>>) bounds(%[[BOUND]]) -> !fir.ref<!fir.array<10xf32>> {name = "a"}
+! CHECK: %[[LOAD_IFCOND:.*]] = fir.load %[[DECLIFCOND]]#0 : !fir.ref<!fir.logical<4>>
+! CHECK: %[[IFCOND_I1:.*]] = fir.convert %[[LOAD_IFCOND]] : (!fir.logical<4>) -> i1
+! CHECK: acc.host_data if(%[[IFCOND_I1]]) dataOperands(%[[DA]] : !fir.ref<!fir.array<10xf32>>)
+
+  !$acc host_data use_device(a) if(.true.)
+  !$acc end host_data
+
+! CHECK: %[[BOUND:.*]] = acc.bounds lowerbound(%{{.*}} : index) upperbound(%{{.*}} : index) stride(%{{.*}} : index) startIdx(%{{.*}} : index)
+! CHECK: %[[DA:.*]] = acc.use_device varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10xf32>>) bounds(%[[BOUND]]) -> !fir.ref<!fir.array<10xf32>> {name = "a"}
+! CHECK: acc.host_data dataOperands(%[[DA]] : !fir.ref<!fir.array<10xf32>>)
+
+  !$acc host_data use_device(a) if(.false.)
+    a = 1.0
+  !$acc end host_data
+
+! CHECK-NOT: acc.host_data
+! CHECK: hlfir.assign %{{.*}} to %[[DECLA]]#0
+
+end subroutine
diff --git a/flang/test/Lower/OpenACC/acc-host-data.f90 b/flang/test/Lower/OpenACC/acc-host-data.f90
index bcce84dc7dca4..7e06ebbc0ea04 100644
--- a/flang/test/Lower/OpenACC/acc-host-data.f90
+++ b/flang/test/Lower/OpenACC/acc-host-data.f90
@@ -14,23 +14,20 @@ subroutine acc_host_data()
   !$acc host_data use_device(a)
   !$acc end host_data
 
-! CHECK: %[[BOUND:.*]] = acc.bounds lowerbound(%{{.*}} : index) upperbound(%{{.*}} : index) stride(%{{.*}} : index) startIdx(%{{.*}} : index)
-! CHECK: %[[DA:.*]] = acc.use_device varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10xf32>>) bounds(%[[BOUND]]) -> !fir.ref<!fir.array<10xf32>> {name = "a"}
+! CHECK: %[[DA:.*]] = acc.use_device varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10xf32>>) -> !fir.ref<!fir.array<10xf32>> {name = "a"}
 ! CHECK: acc.host_data dataOperands(%[[DA]] : !fir.ref<!fir.array<10xf32>>)
 
   !$acc host_data use_device(a) if_present
   !$acc end host_data
 
-! CHECK: %[[BOUND:.*]] = acc.bounds lowerbound(%{{.*}} : index) upperbound(%{{.*}} : index) stride(%{{.*}} : index) startIdx(%{{.*}} : index)
-! CHECK: %[[DA:.*]] = acc.use_device varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10xf32>>) bounds(%[[BOUND]]) -> !fir.ref<!fir.array<10xf32>> {name = "a"}
+! CHECK: %[[DA:.*]] = acc.use_device varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10xf32>>) -> !fir.ref<!fir.array<10xf32>> {name = "a"}
 ! CHECK: acc.host_data dataOperands(%[[DA]] : !fir.ref<!fir.array<10xf32>>) {
 ! CHECK: } attributes {ifPresent}
 
   !$acc host_data use_device(a) if(ifCondition)
   !$acc end host_data
 
-! CHECK: %[[BOUND:.*]] = acc.bounds lowerbound(%{{.*}} : index) upperbound(%{{.*}} : index) stride(%{{.*}} : index) startIdx(%{{.*}} : index)
-! CHECK: %[[DA:.*]] = acc.use_device varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10xf32>>) bounds(%[[BOUND]]) -> !fir.ref<!fir.array<10xf32>> {name = "a"}
+! CHECK: %[[DA:.*]] = acc.use_device varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10xf32>>) -> !fir.ref<!fir.array<10xf32>> {name = "a"}
 ! CHECK: %[[LOAD_IFCOND:.*]] = fir.load %[[DECLIFCOND]]#0 : !fir.ref<!fir.logical<4>>
 ! CHECK: %[[IFCOND_I1:.*]] = fir.convert %[[LOAD_IFCOND]] : (!fir.logical<4>) -> i1
 ! CHECK: acc.host_data if(%[[IFCOND_I1]]) dataOperands(%[[DA]] : !fir.ref<!fir.array<10xf32>>)
@@ -38,8 +35,7 @@ subroutine acc_host_data()
   !$acc host_data use_device(a) if(.true.)
   !$acc end host_data
 
-! CHECK: %[[BOUND:.*]] = acc.bounds lowerbound(%{{.*}} : index) upperbound(%{{.*}} : index) stride(%{{.*}} : index) startIdx(%{{.*}} : index)
-! CHECK: %[[DA:.*]] = acc.use_device varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10xf32>>) bounds(%[[BOUND]]) -> !fir.ref<!fir.array<10xf32>> {name = "a"}
+! CHECK: %[[DA:.*]] = acc.use_device varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10xf32>>) -> !fir.ref<!fir.array<10xf32>> {name = "a"}
 ! CHECK: acc.host_data dataOperands(%[[DA]] : !fir.ref<!fir.array<10xf32>>)
 
   !$acc host_data use_device(a) if(.false.)
diff --git a/flang/test/Lower/OpenACC/acc-kernels-loop.f90 b/flang/test/Lower/OpenACC/acc-kernels-loop.f90
index 388a14d278d6c..aa5d65afd57ec 100644
--- a/flang/test/Lower/OpenACC/acc-kernels-loop.f90
+++ b/flang/test/Lower/OpenACC/acc-kernels-loop.f90
@@ -305,116 +305,120 @@ subroutine acc_kernels_loop
   END DO
 
 
-! CHECK:      %[[COPYIN_A:.*]] = acc.copyin varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10xf32>>) bounds(%{{.*}}) -> !fir.ref<!fir.array<10xf32>> {dataClause = #acc<data_clause acc_copy>, name = "a"}
-! CHECK:      %[[COPYIN_B:.*]] = acc.copyin varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<10xf32>>) bounds(%{{.*}}) -> !fir.ref<!fir.array<10xf32>> {dataClause = #acc<data_clause acc_copy>, name = "b"}
+! CHECK:      %[[COPYIN_A:.*]] = acc.copyin varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10xf32>>) -> !fir.ref<!fir.array<10xf32>> {dataClause = #acc<data_clause acc_copy>, name = "a"}
+! CHECK:      %[[COPYIN_B:.*]] = acc.copyin varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<10xf32>>) -> !fir.ref<!fir.array<10xf32>> {dataClause = #acc<data_clause acc_copy>, name = "b"}
 ! CHECK:      acc.kernels {{.*}} dataOperands(%[[COPYIN_A]], %[[COPYIN_B]] : !fir.ref<!fir.array<10xf32>>, !fir.ref<!fir.array<10xf32>>) {
 ! CHECK:        acc.loop {{.*}} {
 ! CHECK:          acc.yield
 ! CHECK-NEXT:   }{{$}}
 ! CHECK:        acc.terminator
 ! CHECK-NEXT: }{{$}}
-! CHECK:      acc.copyout accPtr(%[[COPYIN_A]] : !fir.ref<!fir.array<10xf32>>) bounds(%{{.*}}) to varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10xf32>>) {dataClause = #acc<data_clause acc_copy>, name = "a"}
-! CHECK:      acc.copyout accPtr(%[[COPYIN_B]] : !fir.ref<!fir.array<10xf32>>) bounds(%{{.*}}) to varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<10xf32>>) {dataClause = #acc<data_clause acc_copy>, name = "b"}
+! CHECK:      acc.copyout accPtr(%[[COPYIN_A]] : !fir.ref<!fir.array<10xf32>>) to varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10xf32>>) {dataClause = #acc<data_clause acc_copy>, name = "a"}
+! CHECK:      acc.copyout accPtr(%[[COPYIN_B]] : !fir.ref<!fir.array<10xf32>>) to varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<10xf32>>) {dataClause = #acc<data_clause acc_copy>, name = "b"}
 
   !$acc kernels loop copy(a) copy(b)
   DO i = 1, n
     a(i) = b(i)
   END DO
 
-! CHECK:      %[[COPYIN_A:.*]] = acc.copyin varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10xf32>>) bounds(%{{.*}}) -> !fir.ref<!fir.array<10xf32>> {dataClause = #acc<data_clause acc_copy>, name = "a"}
-! CHECK:      %[[COPYIN_B:.*]] = acc.copyin varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<10xf32>>) bounds(%{{.*}}) -> !fir.ref<!fir.array<10xf32>> {dataClause = #acc<data_clause acc_copy>, name = "b"}
+! CHECK:      %[[COPYIN_A:.*]] = acc.copyin varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10xf32>>) -> !fir.ref<!fir.array<10xf32>> {dataClause = #acc<data_clause acc_copy>, name = "a"}
+! CHECK:      %[[COPYIN_B:.*]] = acc.copyin varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<10xf32>>) -> !fir.ref<!fir.array<10xf32>> {dataClause = #acc<data_clause acc_copy>, name = "b"}
 ! CHECK:      acc.kernels {{.*}} dataOperands(%[[COPYIN_A]], %[[COPYIN_B]] : !fir.ref<!fir.array<10xf32>>, !fir.ref<!fir.array<10xf32>>) {
 ! CHECK:        acc.loop {{.*}} {
 ! CHECK:          acc.yield
 ! CHECK-NEXT:   }{{$}}
 ! CHECK:        acc.terminator
 ! CHECK-NEXT: }{{$}}
-! CHECK:      acc.copyout accPtr(%[[COPYIN_A]] : !fir.ref<!fir.array<10xf32>>) bounds(%{{.*}}) to varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10xf32>>) {dataClause = #acc<data_clause acc_copy>, name = "a"}
-! CHECK:      acc.copyout accPtr(%[[COPYIN_B]] : !fir.ref<!fir.array<10xf32>>) bounds(%{{.*}}) to varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<10xf32>>) {dataClause = #acc<data_clause acc_copy>, name = "b"}
+! CHECK:      acc.copyout accPtr(%[[COPYIN_A]] : !fir.ref<!fir.array<10xf32>>) to varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10xf32>>) {dataClause = #acc<data_clause acc_copy>, name = "a"}
+! CHECK:      acc.copyout accPtr(%[[COPYIN_B]] : !fir.ref<!fir.array<10xf32>>) to varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<10xf32>>) {dataClause = #acc<data_clause acc_copy>, name = "b"}
 
   !$acc kernels loop copyin(a) copyin(readonly: b)
   DO i = 1, n
     a(i) = b(i)
   END DO
 
-! CHECK:      %[[COPYIN_A:.*]] = acc.copyin varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10xf32>>) bounds(%{{.*}}) -> !fir.ref<!fir.array<10xf32>> {name = "a"}
-! CHECK:      %[[COPYIN_B:.*]] = acc.copyin varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<10xf32>>) bounds(%{{.*}}) -> !fir.ref<!fir.array<10xf32>> {dataClause = #acc<data_clause acc_copyin_readonly>, name = "b"}
+! CHECK:      %[[COPYIN_A:.*]] = acc.copyin varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10xf32>>) -> !fir.ref<!fir.array<10xf32>> {name = "a"}
+! CHECK:      %[[COPYIN_B:.*]] = acc.copyin varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<10xf32>>) -> !fir.ref<!fir.array<10xf32>> {dataClause = #acc<data_clause acc_copyin_readonly>, name = "b"}
 ! CHECK:      acc.kernels {{.*}} dataOperands(%[[COPYIN_A]], %[[COPYIN_B]] : !fir.ref<!fir.array<10xf32>>, !fir.ref<!fir.array<10xf32>>) {
 ! CHECK:        acc.loop {{.*}} {
 ! CHECK:          acc.yield
 ! CHECK-NEXT:   }{{$}}
 ! CHECK:        acc.terminator
 ! CHECK-NEXT: }{{$}}
-! CHECK:      acc.delete accPtr(%[[COPYIN_A]] : !fir.ref<!fir.array<10xf32>>) bounds(%{{.*}}) {dataClause = #acc<data_clause acc_copyin>, name = "a"}
-! CHECK:      acc.delete accPtr(%[[COPYIN_B]] : !fir.ref<!fir.array<10xf32>>) bounds(%{{.*}}) {dataClause = #acc<data_clause acc_copyin_readonly>, name = "b"}
+! CHECK:      acc.delete accPtr(%[[COPYIN_A]] : !fir.ref<!fir.array<10xf32>>) {dataClause = #acc<data_clause acc_copyin>, name = "a"}
+! CHECK:      acc.delete accPtr(%[[COPYIN_B]] : !fir.ref<!fir.array<10xf32>>) {dataClause = #acc<data_clause acc_copyin_readonly>, name = "b"}
 
   !$acc kernels loop copyout(a) copyout(zero: b)
   DO i = 1, n
     a(i) = b(i)
   END DO
 
-! CHECK:      %[[CREATE_A:.*]] = acc.create varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10xf32>>) bounds(%{{.*}}) -> !fir.ref<!fir.array<10xf32>> {dataClause = #acc<data_clause acc_copyout>, name = "a"}
-! CHECK:      %[[CREATE_B:.*]] = acc.create varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<10xf32>>) bounds(%{{.*}}) -> !fir.ref<!fir.array<10xf32>> {dataClause = #acc<data_clause acc_copyout>, name = "b"}
+! CHECK:      %[[CREATE_A:.*]] = acc.create varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10xf32>>) -> !fir.ref<!fir.array<10xf32>> {dataClause = #acc<data_clause acc_copyout>, name = "a"}
+! CHECK:      %[[CREATE_B:.*]] = acc.create varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<10xf32>>) -> !fir.ref<!fir.array<10xf32>> {dataClause = #acc<data_clause acc_copyout>, name = "b"}
 ! CHECK:      acc.kernels {{.*}} dataOperands(%[[CREATE_A]], %[[CREATE_B]] : !fir.ref<!fir.array<10xf32>>, !fir.ref<!fir.array<10xf32>>) {
 ! CHECK:        acc.loop {{.*}} {
 ! CHECK:          acc.yield
 ! CHECK-NEXT:   }{{$}}
 ! CHECK:        acc.terminator
 ! CHECK-NEXT: }{{$}}
-! CHECK:      acc.copyout accPtr(%[[CREATE_A]] : !fir.ref<!fir.array<10xf32>>) bounds(%{{.*}}) to varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10xf32>>) {name = "a"}
-! CHECK:      acc.copyout accPtr(%[[CREATE_B]] : !fir.ref<!fir.array<10xf32>>) bounds(%{{.*}}) to varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<10xf32>>) {name = "b"}
+! CHECK:      acc.copyout accPtr(%[[CREATE_A]] : !fir.ref<!fir.array<10xf32>>) to varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10xf32>>) {name = "a"}
+! CHECK:      acc.copyout accPtr(%[[CREATE_B]] : !fir.ref<!fir.array<10xf32>>) to varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<10xf32>>) {name = "b"}
 
   !$acc kernels loop create(b) create(zero: a)
   DO i = 1, n
     a(i) = b(i)
   END DO
 
-! CHECK:      %[[CREATE_B:.*]] = acc.create varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<10xf32>>) bounds(%{{.*}}) -> !fir.ref<!fir.array<10xf32>> {name = "b"}
-! CHECK:      %[[CREATE_A:.*]] = acc.create varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10xf32>>) bounds(%{{.*}}) -> !fir.ref<!fir.array<10xf32>> {dataClause = #acc<data_clause acc_create_zero>, name = "a"}
+! CHECK:      %[[CREATE_B:.*]] = acc.create varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<10xf32>>) -> !fir.ref<!fir.array<10xf32>> {name = "b"}
+! CHECK:      %[[CREATE_A:.*]] = acc.create varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10xf32>>) -> !fir.ref<!fir.array<10xf32>> {dataClause = #acc<data_clause acc_create_zero>, name = "a"}
 ! CHECK:      acc.kernels {{.*}} dataOperands(%[[CREATE_B]], %[[CREATE_A]] : !fir.ref<!fir.array<10xf32>>, !fir.ref<!fir.array<10xf32>>) {
 ! CHECK:        acc.loop {{.*}} {
 ! CHECK:          acc.yield
 ! CHECK-NEXT:   }{{$}}
 ! CHECK:        acc.terminator
 ! CHECK-NEXT: }{{$}}
-! CHECK:      acc.delete accPtr(%[[CREATE_B]] : !fir.ref<!fir.array<10xf32>>) bounds(%{{.*}}) {dataClause = #acc<data_clause acc_create>, name = "b"}
-! CHECK:      acc.delete accPtr(%[[CREATE_A]] : !fir.ref<!fir.array<10xf32>>) bounds(%{{.*}}) {dataClause = #acc<data_clause acc_create_zero>, name = "a"}
+! CHECK:      acc.delete accPtr(%[[CREATE_B]] : !fir.ref<!fir.array<10xf32>>) {dataClause = #acc<data_clause acc_create>, name = "b"}
+! CHECK:      acc.delete accPtr(%[[CREATE_A]] : !fir.ref<!fir.array<10xf32>>) {dataClause = #acc<data_clause acc_create_zero>, name = "a"}
 
   !$acc kernels loop no_create(a, b)
   DO i = 1, n
     a(i) = b(i)
   END DO
 
-! CHECK:      %[[NOCREATE_A:.*]] = acc.nocreate varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10xf32>>) bounds(%{{.*}}) -> !fir.ref<!fir.array<10xf32>> {name = "a"}
-! CHECK:      %[[NOCREATE_B:.*]] = acc.nocreate varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<10xf32>>) bounds(%{{.*}}) -> !fir.ref<!fir.array<10xf32>> {name = "b"}
+! CHECK:      %[[NOCREATE_A:.*]] = acc.nocreate varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10xf32>>) -> !fir.ref<!fir.array<10xf32>> {name = "a"}
+! CHECK:      %[[NOCREATE_B:.*]] = acc.nocreate varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<10xf32>>) -> !fir.ref<!fir.array<10xf32>> {name = "b"}
 ! CHECK:      acc.kernels {{.*}} dataOperands(%[[NOCREATE_A]], %[[NOCREATE_B]] : !fir.ref<!fir.array<10xf32>>, !fir.ref<!fir.array<10xf32>>) {
 ! CHECK:        acc.loop {{.*}} {
 ! CHECK:          acc.yield
 ! CHECK-NEXT:   }{{$}}
 ! CHECK:        acc.terminator
 ! CHECK-NEXT: }{{$}}
+! CHECK:      acc.delete accPtr(%[[NOCREATE_A]] : !fir.ref<!fir.array<10xf32>>) {dataClause = #acc<data_clause acc_no_create>, name = "a"}
+! CHECK:      acc.delete accPtr(%[[NOCREATE_B]] : !fir.ref<!fir.array<10xf32>>) {dataClause = #acc<data_clause acc_no_create>, name = "b"}
 
   !$acc kernels loop present(a, b)
   DO i = 1, n
     a(i) = b(i)
   END DO
 
-! CHECK:      %[[PRESENT_A:.*]] = acc.present varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10xf32>>) bounds(%{{.*}}) -> !fir.ref<!fir.array<10xf32>> {name = "a"}
-! CHECK:      %[[PRESENT_B:.*]] = acc.present varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<10xf32>>) bounds(%{{.*}}) -> !fir.ref<!fir.array<10xf32>> {name = "b"}
+! CHECK:      %[[PRESENT_A:.*]] = acc.present varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10xf32>>) -> !fir.ref<!fir.array<10xf32>> {name = "a"}
+! CHECK:      %[[PRESENT_B:.*]] = acc.present varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<10xf32>>) -> !fir.ref<!fir.array<10xf32>> {name = "b"}
 ! CHECK:      acc.kernels {{.*}} dataOperands(%[[PRESENT_A]], %[[PRESENT_B]] : !fir.ref<!fir.array<10xf32>>, !fir.ref<!fir.array<10xf32>>) {
 ! CHECK:        acc.loop {{.*}} {
 ! CHECK:          acc.yield
 ! CHECK-NEXT:   }{{$}}
 ! CHECK:        acc.terminator
 ! CHECK-NEXT: }{{$}}
+! CHECK:      acc.delete accPtr(%[[PRESENT_A]] : !fir.ref<!fir.array<10xf32>>) {dataClause = #acc<data_clause acc_present>, name = "a"}
+! CHECK:      acc.delete accPtr(%[[PRESENT_B]] : !fir.ref<!fir.array<10xf32>>) {dataClause = #acc<data_clause acc_present>, name = "b"}
 
   !$acc kernels loop deviceptr(a) deviceptr(b)
   DO i = 1, n
     a(i) = b(i)
   END DO
 
-! CHECK:      %[[DEVICEPTR_A:.*]] = acc.deviceptr varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10xf32>>) bounds(%{{.*}}) -> !fir.ref<!fir.array<10xf32>> {name = "a"}
-! CHECK:      %[[DEVICEPTR_B:.*]] = acc.deviceptr varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<10xf32>>) bounds(%{{.*}}) -> !fir.ref<!fir.array<10xf32>> {name = "b"}
+! CHECK:      %[[DEVICEPTR_A:.*]] = acc.deviceptr varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10xf32>>) -> !fir.ref<!fir.array<10xf32>> {name = "a"}
+! CHECK:      %[[DEVICEPTR_B:.*]] = acc.deviceptr varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<10xf32>>) -> !fir.ref<!fir.array<10xf32>> {name = "b"}
 ! CHECK:      acc.kernels {{.*}} dataOperands(%[[DEVICEPTR_A]], %[[DEVICEPTR_B]] : !fir.ref<!fir.array<10xf32>>, !fir.ref<!fir.array<10xf32>>) {
 ! CHECK:        acc.loop {{.*}} {
 ! CHECK:          acc.yield
@@ -427,13 +431,9 @@ subroutine acc_kernels_loop
     a(i) = b(i)
   END DO
 
-! CHECK:      %[[BOX_F:.*]] = fir.load %[[DECLF]]#0 : !fir.ref<!fir.box<!fir.ptr<f32>>>
-! CHECK:      %[[BOX_ADDR_F:.*]] = fir.box_addr %[[BOX_F]] : (!fir.box<!fir.ptr<f32>>) -> !fir.ptr<f32>
-! CHECK:      %[[ATTACH_F:.*]] = acc.attach varPtr(%[[BOX_ADDR_F]] : !fir.ptr<f32>) -> !fir.ptr<f32> {name = "f"}
-! CHECK:      %[[BOX_G:.*]] = fir.load %[[DECLG]]#0 : !fir.ref<!fir.box<!fir.ptr<f32>>>
-! CHECK:      %[[BOX_ADDR_G:.*]] = fir.box_addr %[[BOX_G]] : (!fir.box<!fir.ptr<f32>>) -> !fir.ptr<f32>
-! CHECK:      %[[ATTACH_G:.*]] = acc.attach varPtr(%[[BOX_ADDR_G]] : !fir.ptr<f32>) -> !fir.ptr<f32> {name = "g"}
-! CHECK:      acc.kernels {{.*}} dataOperands(%[[ATTACH_F]], %[[ATTACH_G]] : !fir.ptr<f32>, !fir.ptr<f32>) {
+! CHECK:      %[[ATTACH_F:.*]] = acc.attach varPtr(%[[DECLF]]#0 : !fir.ref<!fir.box<!fir.ptr<f32>>>) -> !fir.ref<!fir.box<!fir.ptr<f32>>> {name = "f"}
+! CHECK:      %[[ATTACH_G:.*]] = acc.attach varPtr(%[[DECLG]]#0 : !fir.ref<!fir.box<!fir.ptr<f32>>>) -> !fir.ref<!fir.box<!fir.ptr<f32>>> {name = "g"}
+! CHECK:      acc.kernels {{.*}} dataOperands(%[[ATTACH_F]], %[[ATTACH_G]] : !fir.ref<!fir.box<!fir.ptr<f32>>>, !fir.ref<!fir.box<!fir.ptr<f32>>>) {
 ! CHECK:        acc.loop {{.*}} {
 ! CHECK:          acc.yield
 ! CHECK-NEXT:   }{{$}}
diff --git a/flang/test/Lower/OpenACC/acc-kernels.f90 b/flang/test/Lower/OpenACC/acc-kernels.f90
index 7282fee689cad..b333bb981a79b 100644
--- a/flang/test/Lower/OpenACC/acc-kernels.f90
+++ b/flang/test/Lower/OpenACC/acc-kernels.f90
@@ -182,93 +182,99 @@ subroutine acc_kernels
   !$acc kernels copy(a, b, c)
   !$acc end kernels
 
-! CHECK:      %[[COPYIN_A:.*]] = acc.copyin varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = #acc<data_clause acc_copy>, name = "a"}
-! CHECK:      %[[COPYIN_B:.*]] = acc.copyin varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = #acc<data_clause acc_copy>, name = "b"}
-! CHECK:      %[[COPYIN_C:.*]] = acc.copyin varPtr(%[[DECLC]]#0 : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = #acc<data_clause acc_copy>, name = "c"}
+! CHECK:      %[[COPYIN_A:.*]] = acc.copyin varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = #acc<data_clause acc_copy>, name = "a"}
+! CHECK:      %[[COPYIN_B:.*]] = acc.copyin varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<10x10xf32>>) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = #acc<data_clause acc_copy>, name = "b"}
+! CHECK:      %[[COPYIN_C:.*]] = acc.copyin varPtr(%[[DECLC]]#0 : !fir.ref<!fir.array<10x10xf32>>) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = #acc<data_clause acc_copy>, name = "c"}
 ! CHECK:      acc.kernels  dataOperands(%[[COPYIN_A]], %[[COPYIN_B]], %[[COPYIN_C]] : !fir.ref<!fir.array<10x10xf32>>, !fir.ref<!fir.array<10x10xf32>>, !fir.ref<!fir.array<10x10xf32>>) {
 ! CHECK:        acc.terminator
 ! CHECK-NEXT: }{{$}}
-! CHECK:      acc.copyout accPtr(%[[COPYIN_A]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) to varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) {dataClause = #acc<data_clause acc_copy>, name = "a"}
-! CHECK:      acc.copyout accPtr(%[[COPYIN_B]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) to varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<10x10xf32>>) {dataClause = #acc<data_clause acc_copy>, name = "b"}
-! CHECK:      acc.copyout accPtr(%[[COPYIN_C]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) to varPtr(%[[DECLC]]#0 : !fir.ref<!fir.array<10x10xf32>>) {dataClause = #acc<data_clause acc_copy>, name = "c"}
+! CHECK:      acc.copyout accPtr(%[[COPYIN_A]] : !fir.ref<!fir.array<10x10xf32>>) to varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) {dataClause = #acc<data_clause acc_copy>, name = "a"}
+! CHECK:      acc.copyout accPtr(%[[COPYIN_B]] : !fir.ref<!fir.array<10x10xf32>>) to varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<10x10xf32>>) {dataClause = #acc<data_clause acc_copy>, name = "b"}
+! CHECK:      acc.copyout accPtr(%[[COPYIN_C]] : !fir.ref<!fir.array<10x10xf32>>) to varPtr(%[[DECLC]]#0 : !fir.ref<!fir.array<10x10xf32>>) {dataClause = #acc<data_clause acc_copy>, name = "c"}
 
   !$acc kernels copy(a) copy(b) copy(c)
   !$acc end kernels
 
-! CHECK:      %[[COPYIN_A:.*]] = acc.copyin varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = #acc<data_clause acc_copy>, name = "a"}
-! CHECK:      %[[COPYIN_B:.*]] = acc.copyin varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = #acc<data_clause acc_copy>, name = "b"}
-! CHECK:      %[[COPYIN_C:.*]] = acc.copyin varPtr(%[[DECLC]]#0 : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = #acc<data_clause acc_copy>, name = "c"}
+! CHECK:      %[[COPYIN_A:.*]] = acc.copyin varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = #acc<data_clause acc_copy>, name = "a"}
+! CHECK:      %[[COPYIN_B:.*]] = acc.copyin varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<10x10xf32>>) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = #acc<data_clause acc_copy>, name = "b"}
+! CHECK:      %[[COPYIN_C:.*]] = acc.copyin varPtr(%[[DECLC]]#0 : !fir.ref<!fir.array<10x10xf32>>) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = #acc<data_clause acc_copy>, name = "c"}
 ! CHECK:      acc.kernels dataOperands(%[[COPYIN_A]], %[[COPYIN_B]], %[[COPYIN_C]] : !fir.ref<!fir.array<10x10xf32>>, !fir.ref<!fir.array<10x10xf32>>, !fir.ref<!fir.array<10x10xf32>>) {
 ! CHECK:        acc.terminator
 ! CHECK-NEXT: }{{$}}
-! CHECK:      acc.copyout accPtr(%[[COPYIN_A]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) to varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) {dataClause = #acc<data_clause acc_copy>, name = "a"}
-! CHECK:      acc.copyout accPtr(%[[COPYIN_B]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) to varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<10x10xf32>>) {dataClause = #acc<data_clause acc_copy>, name = "b"}
-! CHECK:      acc.copyout accPtr(%[[COPYIN_C]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) to varPtr(%[[DECLC]]#0 : !fir.ref<!fir.array<10x10xf32>>) {dataClause = #acc<data_clause acc_copy>, name = "c"}
+! CHECK:      acc.copyout accPtr(%[[COPYIN_A]] : !fir.ref<!fir.array<10x10xf32>>) to varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) {dataClause = #acc<data_clause acc_copy>, name = "a"}
+! CHECK:      acc.copyout accPtr(%[[COPYIN_B]] : !fir.ref<!fir.array<10x10xf32>>) to varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<10x10xf32>>) {dataClause = #acc<data_clause acc_copy>, name = "b"}
+! CHECK:      acc.copyout accPtr(%[[COPYIN_C]] : !fir.ref<!fir.array<10x10xf32>>) to varPtr(%[[DECLC]]#0 : !fir.ref<!fir.array<10x10xf32>>) {dataClause = #acc<data_clause acc_copy>, name = "c"}
 
   !$acc kernels copyin(a) copyin(readonly: b, c)
   !$acc end kernels
 
-! CHECK:      %[[COPYIN_A:.*]] = acc.copyin varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {name = "a"}
-! CHECK:      %[[COPYIN_B:.*]] = acc.copyin varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = #acc<data_clause acc_copyin_readonly>, name = "b"}
-! CHECK:      %[[COPYIN_C:.*]] = acc.copyin varPtr(%[[DECLC]]#0 : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = #acc<data_clause acc_copyin_readonly>, name = "c"}
+! CHECK:      %[[COPYIN_A:.*]] = acc.copyin varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) -> !fir.ref<!fir.array<10x10xf32>> {name = "a"}
+! CHECK:      %[[COPYIN_B:.*]] = acc.copyin varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<10x10xf32>>) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = #acc<data_clause acc_copyin_readonly>, name = "b"}
+! CHECK:      %[[COPYIN_C:.*]] = acc.copyin varPtr(%[[DECLC]]#0 : !fir.ref<!fir.array<10x10xf32>>) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = #acc<data_clause acc_copyin_readonly>, name = "c"}
 ! CHECK:      acc.kernels dataOperands(%[[COPYIN_A]], %[[COPYIN_B]], %[[COPYIN_C]] : !fir.ref<!fir.array<10x10xf32>>, !fir.ref<!fir.array<10x10xf32>>, !fir.ref<!fir.array<10x10xf32>>) {
 ! CHECK:        acc.terminator
 ! CHECK-NEXT: }{{$}}
-! CHECK:      acc.delete accPtr(%[[COPYIN_A]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) {dataClause = #acc<data_clause acc_copyin>, name = "a"}
-! CHECK:      acc.delete accPtr(%[[COPYIN_B]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) {dataClause = #acc<data_clause acc_copyin_readonly>, name = "b"}
-! CHECK:      acc.delete accPtr(%[[COPYIN_C]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) {dataClause = #acc<data_clause acc_copyin_readonly>, name = "c"}
+! CHECK:      acc.delete accPtr(%[[COPYIN_A]] : !fir.ref<!fir.array<10x10xf32>>) {dataClause = #acc<data_clause acc_copyin>, name = "a"}
+! CHECK:      acc.delete accPtr(%[[COPYIN_B]] : !fir.ref<!fir.array<10x10xf32>>) {dataClause = #acc<data_clause acc_copyin_readonly>, name = "b"}
+! CHECK:      acc.delete accPtr(%[[COPYIN_C]] : !fir.ref<!fir.array<10x10xf32>>) {dataClause = #acc<data_clause acc_copyin_readonly>, name = "c"}
 
   !$acc kernels copyout(a) copyout(zero: b) copyout(c)
   !$acc end kernels
 
-! CHECK: %[[CREATE_A:.*]] = acc.create varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = #acc<data_clause acc_copyout>, name = "a"}
-! CHECK: %[[CREATE_B:.*]] = acc.create varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = #acc<data_clause acc_copyout>, name = "b"}
-! CHECK: %[[CREATE_C:.*]] = acc.create varPtr(%[[DECLC]]#0 : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = #acc<data_clause acc_copyout>, name = "c"}
+! CHECK: %[[CREATE_A:.*]] = acc.create varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = #acc<data_clause acc_copyout>, name = "a"}
+! CHECK: %[[CREATE_B:.*]] = acc.create varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<10x10xf32>>) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = #acc<data_clause acc_copyout>, name = "b"}
+! CHECK: %[[CREATE_C:.*]] = acc.create varPtr(%[[DECLC]]#0 : !fir.ref<!fir.array<10x10xf32>>) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = #acc<data_clause acc_copyout>, name = "c"}
 ! CHECK:      acc.kernels dataOperands(%[[CREATE_A]], %[[CREATE_B]], %[[CREATE_C]] : !fir.ref<!fir.array<10x10xf32>>, !fir.ref<!fir.array<10x10xf32>>, !fir.ref<!fir.array<10x10xf32>>) {
 ! CHECK:        acc.terminator
 ! CHECK-NEXT: }{{$}}
-! CHECK: acc.copyout accPtr(%[[CREATE_A]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) to varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) {name = "a"}
-! CHECK: acc.copyout accPtr(%[[CREATE_B]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) to varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<10x10xf32>>) {name = "b"}
-! CHECK: acc.copyout accPtr(%[[CREATE_C]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) to varPtr(%[[DECLC]]#0 : !fir.ref<!fir.array<10x10xf32>>) {name = "c"}
+! CHECK: acc.copyout accPtr(%[[CREATE_A]] : !fir.ref<!fir.array<10x10xf32>>) to varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) {name = "a"}
+! CHECK: acc.copyout accPtr(%[[CREATE_B]] : !fir.ref<!fir.array<10x10xf32>>) to varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<10x10xf32>>) {name = "b"}
+! CHECK: acc.copyout accPtr(%[[CREATE_C]] : !fir.ref<!fir.array<10x10xf32>>) to varPtr(%[[DECLC]]#0 : !fir.ref<!fir.array<10x10xf32>>) {name = "c"}
 
   !$acc kernels create(a, b) create(zero: c)
   !$acc end kernels
 
-! CHECK: %[[CREATE_A:.*]] = acc.create varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {name = "a"}
-! CHECK: %[[CREATE_B:.*]] = acc.create varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {name = "b"}
-! CHECK: %[[CREATE_C:.*]] = acc.create varPtr(%[[DECLC]]#0 : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = #acc<data_clause acc_create_zero>, name = "c"}
+! CHECK: %[[CREATE_A:.*]] = acc.create varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) -> !fir.ref<!fir.array<10x10xf32>> {name = "a"}
+! CHECK: %[[CREATE_B:.*]] = acc.create varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<10x10xf32>>) -> !fir.ref<!fir.array<10x10xf32>> {name = "b"}
+! CHECK: %[[CREATE_C:.*]] = acc.create varPtr(%[[DECLC]]#0 : !fir.ref<!fir.array<10x10xf32>>) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = #acc<data_clause acc_create_zero>, name = "c"}
 ! CHECK:      acc.kernels dataOperands(%[[CREATE_A]], %[[CREATE_B]], %[[CREATE_C]] : !fir.ref<!fir.array<10x10xf32>>, !fir.ref<!fir.array<10x10xf32>>, !fir.ref<!fir.array<10x10xf32>>) {
 ! CHECK:        acc.terminator
 ! CHECK-NEXT: }{{$}}
-! CHECK:   acc.delete accPtr(%[[CREATE_A]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) {dataClause = #acc<data_clause acc_create>, name = "a"}
-! CHECK: acc.delete accPtr(%[[CREATE_B]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) {dataClause = #acc<data_clause acc_create>, name = "b"}
-! CHECK: acc.delete accPtr(%[[CREATE_C]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) {dataClause = #acc<data_clause acc_create_zero>, name = "c"}
+! CHECK:   acc.delete accPtr(%[[CREATE_A]] : !fir.ref<!fir.array<10x10xf32>>) {dataClause = #acc<data_clause acc_create>, name = "a"}
+! CHECK: acc.delete accPtr(%[[CREATE_B]] : !fir.ref<!fir.array<10x10xf32>>) {dataClause = #acc<data_clause acc_create>, name = "b"}
+! CHECK: acc.delete accPtr(%[[CREATE_C]] : !fir.ref<!fir.array<10x10xf32>>) {dataClause = #acc<data_clause acc_create_zero>, name = "c"}
 
   !$acc kernels no_create(a, b) create(zero: c)
   !$acc end kernels
 
-! CHECK: %[[NO_CREATE_A:.*]] = acc.nocreate varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {name = "a"}
-! CHECK: %[[NO_CREATE_B:.*]] = acc.nocreate varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {name = "b"}
-! CHECK: %[[CREATE_C:.*]] = acc.create varPtr(%[[DECLC]]#0 : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = #acc<data_clause acc_create_zero>, name = "c"}
+! CHECK: %[[NO_CREATE_A:.*]] = acc.nocreate varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) -> !fir.ref<!fir.array<10x10xf32>> {name = "a"}
+! CHECK: %[[NO_CREATE_B:.*]] = acc.nocreate varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<10x10xf32>>) -> !fir.ref<!fir.array<10x10xf32>> {name = "b"}
+! CHECK: %[[CREATE_C:.*]] = acc.create varPtr(%[[DECLC]]#0 : !fir.ref<!fir.array<10x10xf32>>) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = #acc<data_clause acc_create_zero>, name = "c"}
 ! CHECK:      acc.kernels dataOperands(%[[NO_CREATE_A]], %[[NO_CREATE_B]], %[[CREATE_C]] : !fir.ref<!fir.array<10x10xf32>>, !fir.ref<!fir.array<10x10xf32>>, !fir.ref<!fir.array<10x10xf32>>) {
 ! CHECK:        acc.terminator
 ! CHECK-NEXT: }{{$}}
+! CHECK: acc.delete accPtr(%[[CREATE_C]] : !fir.ref<!fir.array<10x10xf32>>) {dataClause = #acc<data_clause acc_create_zero>, name = "c"}
+! CHECK: acc.delete accPtr(%[[NO_CREATE_A]] : !fir.ref<!fir.array<10x10xf32>>) {dataClause = #acc<data_clause acc_no_create>, name = "a"}
+! CHECK: acc.delete accPtr(%[[NO_CREATE_B]] : !fir.ref<!fir.array<10x10xf32>>) {dataClause = #acc<data_clause acc_no_create>, name = "b"}
 
   !$acc kernels present(a, b, c)
   !$acc end kernels
 
-! CHECK: %[[PRESENT_A:.*]] = acc.present varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {name = "a"}
-! CHECK: %[[PRESENT_B:.*]] = acc.present varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {name = "b"}
-! CHECK: %[[PRESENT_C:.*]] = acc.present varPtr(%[[DECLC]]#0 : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {name = "c"}
+! CHECK: %[[PRESENT_A:.*]] = acc.present varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) -> !fir.ref<!fir.array<10x10xf32>> {name = "a"}
+! CHECK: %[[PRESENT_B:.*]] = acc.present varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<10x10xf32>>) -> !fir.ref<!fir.array<10x10xf32>> {name = "b"}
+! CHECK: %[[PRESENT_C:.*]] = acc.present varPtr(%[[DECLC]]#0 : !fir.ref<!fir.array<10x10xf32>>) -> !fir.ref<!fir.array<10x10xf32>> {name = "c"}
 ! CHECK:      acc.kernels dataOperands(%[[PRESENT_A]], %[[PRESENT_B]], %[[PRESENT_C]] : !fir.ref<!fir.array<10x10xf32>>, !fir.ref<!fir.array<10x10xf32>>, !fir.ref<!fir.array<10x10xf32>>) {
 ! CHECK:        acc.terminator
 ! CHECK-NEXT: }{{$}}
+! CHECK: acc.delete accPtr(%[[PRESENT_A]] : !fir.ref<!fir.array<10x10xf32>>) {dataClause = #acc<data_clause acc_present>, name = "a"}
+! CHECK: acc.delete accPtr(%[[PRESENT_B]] : !fir.ref<!fir.array<10x10xf32>>) {dataClause = #acc<data_clause acc_present>, name = "b"}
+! CHECK: acc.delete accPtr(%[[PRESENT_C]] : !fir.ref<!fir.array<10x10xf32>>) {dataClause = #acc<data_clause acc_present>, name = "c"}
 
   !$acc kernels deviceptr(a) deviceptr(c)
   !$acc end kernels
 
-! CHECK:      %[[DEVICEPTR_A:.*]] = acc.deviceptr varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {name = "a"}
-! CHECK:      %[[DEVICEPTR_C:.*]] = acc.deviceptr varPtr(%[[DECLC]]#0 : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {name = "c"}
+! CHECK:      %[[DEVICEPTR_A:.*]] = acc.deviceptr varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) -> !fir.ref<!fir.array<10x10xf32>> {name = "a"}
+! CHECK:      %[[DEVICEPTR_C:.*]] = acc.deviceptr varPtr(%[[DECLC]]#0 : !fir.ref<!fir.array<10x10xf32>>) -> !fir.ref<!fir.array<10x10xf32>> {name = "c"}
 ! CHECK:      acc.kernels dataOperands(%[[DEVICEPTR_A]], %[[DEVICEPTR_C]] : !fir.ref<!fir.array<10x10xf32>>, !fir.ref<!fir.array<10x10xf32>>) {
 ! CHECK:        acc.terminator
 ! CHECK-NEXT: }{{$}}
@@ -276,13 +282,9 @@ subroutine acc_kernels
   !$acc kernels attach(d, e)
   !$acc end kernels
 
-! CHECK:      %[[BOX_D:.*]] = fir.load %[[DECLD]]#0 : !fir.ref<!fir.box<!fir.ptr<f32>>>
-! CHECK:      %[[BOX_ADDR_D:.*]] = fir.box_addr %[[BOX_D]] : (!fir.box<!fir.ptr<f32>>) -> !fir.ptr<f32>
-! CHECK:      %[[ATTACH_D:.*]] = acc.attach varPtr(%[[BOX_ADDR_D]] : !fir.ptr<f32>) -> !fir.ptr<f32> {name = "d"}
-! CHECK:      %[[BOX_E:.*]] = fir.load %[[DECLE]]#0 : !fir.ref<!fir.box<!fir.ptr<f32>>>
-! CHECK:      %[[BOX_ADDR_E:.*]] = fir.box_addr %[[BOX_E]] : (!fir.box<!fir.ptr<f32>>) -> !fir.ptr<f32>
-! CHECK:      %[[ATTACH_E:.*]] = acc.attach varPtr(%[[BOX_ADDR_E]] : !fir.ptr<f32>) -> !fir.ptr<f32> {name = "e"}
-! CHECK:      acc.kernels dataOperands(%[[ATTACH_D]], %[[ATTACH_E]] : !fir.ptr<f32>, !fir.ptr<f32>) {
+! CHECK:      %[[ATTACH_D:.*]] = acc.attach varPtr(%[[DECLD]]#0 : !fir.ref<!fir.box<!fir.ptr<f32>>>) -> !fir.ref<!fir.box<!fir.ptr<f32>>> {name = "d"}
+! CHECK:      %[[ATTACH_E:.*]] = acc.attach varPtr(%[[DECLE]]#0 : !fir.ref<!fir.box<!fir.ptr<f32>>>) -> !fir.ref<!fir.box<!fir.ptr<f32>>> {name = "e"}
+! CHECK:      acc.kernels dataOperands(%[[ATTACH_D]], %[[ATTACH_E]] : !fir.ref<!fir.box<!fir.ptr<f32>>>, !fir.ref<!fir.box<!fir.ptr<f32>>>) {
 ! CHECK:        acc.terminator
 ! CHECK-NEXT: }{{$}}
 
diff --git a/flang/test/Lower/OpenACC/acc-loop.f90 b/flang/test/Lower/OpenACC/acc-loop.f90
index 0d2594c86a6eb..f77aefcc2c314 100644
--- a/flang/test/Lower/OpenACC/acc-loop.f90
+++ b/flang/test/Lower/OpenACC/acc-loop.f90
@@ -285,7 +285,7 @@ program acc_loop
     a(i) = b(i)
   END DO
 
-! CHECK: %[[CACHE:.*]] = acc.cache varPtr(%{{.*}} : !fir.ref<!fir.array<10xf32>>) bounds(%{{.*}}) -> !fir.ref<!fir.array<10xf32>> {name = "b"}
+! CHECK: %[[CACHE:.*]] = acc.cache varPtr(%{{.*}} : !fir.ref<!fir.array<10xf32>>) -> !fir.ref<!fir.array<10xf32>> {name = "b"}
 ! CHECK: acc.loop {{.*}} cache(%[[CACHE]] : !fir.ref<!fir.array<10xf32>>)
 
   !$acc loop
@@ -321,8 +321,8 @@ subroutine sub1(i, j, k)
 ! CHECK: %[[DC_K:.*]] = fir.alloca i32 {bindc_name = "k"}
 ! CHECK: %[[DC_J:.*]] = fir.alloca i32 {bindc_name = "j"}
 ! CHECK: %[[DC_I:.*]] = fir.alloca i32 {bindc_name = "i"}
-! CHECK: %[[P_I:.*]] = acc.private varPtr(%[[DC_I]] : !fir.ref<i32>) -> !fir.ref<i32> {implicit = true, name = ""}
-! CHECK: %[[P_J:.*]] = acc.private varPtr(%[[DC_J]] : !fir.ref<i32>) -> !fir.ref<i32> {implicit = true, name = ""}
-! CHECK: %[[P_K:.*]] = acc.private varPtr(%[[DC_K]] : !fir.ref<i32>) -> !fir.ref<i32> {implicit = true, name = ""}
+! CHECK: %[[P_I:.*]] = acc.private varPtr(%[[DC_I]] : !fir.ref<i32>) -> !fir.ref<i32> {implicit = true, name = "i"}
+! CHECK: %[[P_J:.*]] = acc.private varPtr(%[[DC_J]] : !fir.ref<i32>) -> !fir.ref<i32> {implicit = true, name = "j"}
+! CHECK: %[[P_K:.*]] = acc.private varPtr(%[[DC_K]] : !fir.ref<i32>) -> !fir.ref<i32> {implicit = true, name = "k"}
 ! CHECK: acc.loop combined(parallel) private(@privatization_ref_i32 -> %[[P_I]] : !fir.ref<i32>, @privatization_ref_i32 -> %[[P_J]] : !fir.ref<i32>, @privatization_ref_i32 -> %[[P_K]] : !fir.ref<i32>) control(%{{.*}} : i32, %{{.*}} : i32, %{{.*}} : i32) = (%c1{{.*}}, %c1{{.*}}, %c1{{.*}} : i32, i32, i32) to (%c10{{.*}}, %c100{{.*}}, %c200{{.*}} : i32, i32, i32)  step (%c1{{.*}}, %c1{{.*}}, %c1{{.*}} : i32, i32, i32)
 ! CHECK: } attributes {inclusiveUpperbound = array<i1: true, true, true>}
diff --git a/flang/test/Lower/OpenACC/acc-parallel-loop.f90 b/flang/test/Lower/OpenACC/acc-parallel-loop.f90
index c4fc21a307b32..7a41da320b955 100644
--- a/flang/test/Lower/OpenACC/acc-parallel-loop.f90
+++ b/flang/test/Lower/OpenACC/acc-parallel-loop.f90
@@ -305,116 +305,120 @@ subroutine acc_parallel_loop
     a(i) = b(i)
   END DO
 
-! CHECK:      %[[COPYIN_A:.*]] = acc.copyin varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10xf32>>) bounds(%{{.*}}) -> !fir.ref<!fir.array<10xf32>> {dataClause = #acc<data_clause acc_copy>, name = "a"}
-! CHECK:      %[[COPYIN_B:.*]] = acc.copyin varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<10xf32>>) bounds(%{{.*}}) -> !fir.ref<!fir.array<10xf32>> {dataClause = #acc<data_clause acc_copy>, name = "b"}
+! CHECK:      %[[COPYIN_A:.*]] = acc.copyin varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10xf32>>) -> !fir.ref<!fir.array<10xf32>> {dataClause = #acc<data_clause acc_copy>, name = "a"}
+! CHECK:      %[[COPYIN_B:.*]] = acc.copyin varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<10xf32>>) -> !fir.ref<!fir.array<10xf32>> {dataClause = #acc<data_clause acc_copy>, name = "b"}
 ! CHECK:      acc.parallel {{.*}} dataOperands(%[[COPYIN_A]], %[[COPYIN_B]] : !fir.ref<!fir.array<10xf32>>, !fir.ref<!fir.array<10xf32>>) {
 ! CHECK:        acc.loop {{.*}} {
 ! CHECK:          acc.yield
 ! CHECK-NEXT:   }{{$}}
 ! CHECK:        acc.yield
 ! CHECK-NEXT: }{{$}}
-! CHECK:      acc.copyout accPtr(%[[COPYIN_A]] : !fir.ref<!fir.array<10xf32>>) bounds(%{{.*}}) to varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10xf32>>) {dataClause = #acc<data_clause acc_copy>, name = "a"}
-! CHECK:      acc.copyout accPtr(%[[COPYIN_B]] : !fir.ref<!fir.array<10xf32>>) bounds(%{{.*}}) to varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<10xf32>>) {dataClause = #acc<data_clause acc_copy>, name = "b"}
+! CHECK:      acc.copyout accPtr(%[[COPYIN_A]] : !fir.ref<!fir.array<10xf32>>) to varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10xf32>>) {dataClause = #acc<data_clause acc_copy>, name = "a"}
+! CHECK:      acc.copyout accPtr(%[[COPYIN_B]] : !fir.ref<!fir.array<10xf32>>) to varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<10xf32>>) {dataClause = #acc<data_clause acc_copy>, name = "b"}
 
   !$acc parallel loop copy(a) copy(b)
   DO i = 1, n
     a(i) = b(i)
   END DO
 
-! CHECK:      %[[COPYIN_A:.*]] = acc.copyin varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10xf32>>) bounds(%{{.*}}) -> !fir.ref<!fir.array<10xf32>> {dataClause = #acc<data_clause acc_copy>, name = "a"}
-! CHECK:      %[[COPYIN_B:.*]] = acc.copyin varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<10xf32>>) bounds(%{{.*}}) -> !fir.ref<!fir.array<10xf32>> {dataClause = #acc<data_clause acc_copy>, name = "b"}
+! CHECK:      %[[COPYIN_A:.*]] = acc.copyin varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10xf32>>) -> !fir.ref<!fir.array<10xf32>> {dataClause = #acc<data_clause acc_copy>, name = "a"}
+! CHECK:      %[[COPYIN_B:.*]] = acc.copyin varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<10xf32>>) -> !fir.ref<!fir.array<10xf32>> {dataClause = #acc<data_clause acc_copy>, name = "b"}
 ! CHECK:      acc.parallel {{.*}} dataOperands(%[[COPYIN_A]], %[[COPYIN_B]] : !fir.ref<!fir.array<10xf32>>, !fir.ref<!fir.array<10xf32>>) {
 ! CHECK:        acc.loop {{.*}} {
 ! CHECK:          acc.yield
 ! CHECK-NEXT:   }{{$}}
 ! CHECK:        acc.yield
 ! CHECK-NEXT: }{{$}}
-! CHECK:      acc.copyout accPtr(%[[COPYIN_A]] : !fir.ref<!fir.array<10xf32>>) bounds(%{{.*}}) to varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10xf32>>) {dataClause = #acc<data_clause acc_copy>, name = "a"}
-! CHECK:      acc.copyout accPtr(%[[COPYIN_B]] : !fir.ref<!fir.array<10xf32>>) bounds(%{{.*}}) to varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<10xf32>>) {dataClause = #acc<data_clause acc_copy>, name = "b"}
+! CHECK:      acc.copyout accPtr(%[[COPYIN_A]] : !fir.ref<!fir.array<10xf32>>) to varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10xf32>>) {dataClause = #acc<data_clause acc_copy>, name = "a"}
+! CHECK:      acc.copyout accPtr(%[[COPYIN_B]] : !fir.ref<!fir.array<10xf32>>) to varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<10xf32>>) {dataClause = #acc<data_clause acc_copy>, name = "b"}
 
   !$acc parallel loop copyin(a) copyin(readonly: b)
   DO i = 1, n
     a(i) = b(i)
   END DO
 
-! CHECK:      %[[COPYIN_A:.*]] = acc.copyin varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10xf32>>) bounds(%{{.*}}) -> !fir.ref<!fir.array<10xf32>> {name = "a"}
-! CHECK:      %[[COPYIN_B:.*]] = acc.copyin varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<10xf32>>) bounds(%{{.*}}) -> !fir.ref<!fir.array<10xf32>> {dataClause = #acc<data_clause acc_copyin_readonly>, name = "b"}
+! CHECK:      %[[COPYIN_A:.*]] = acc.copyin varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10xf32>>) -> !fir.ref<!fir.array<10xf32>> {name = "a"}
+! CHECK:      %[[COPYIN_B:.*]] = acc.copyin varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<10xf32>>) -> !fir.ref<!fir.array<10xf32>> {dataClause = #acc<data_clause acc_copyin_readonly>, name = "b"}
 ! CHECK:      acc.parallel {{.*}} dataOperands(%[[COPYIN_A]], %[[COPYIN_B]] : !fir.ref<!fir.array<10xf32>>, !fir.ref<!fir.array<10xf32>>) {
 ! CHECK:        acc.loop {{.*}} {
 ! CHECK:          acc.yield
 ! CHECK-NEXT:   }{{$}}
 ! CHECK:        acc.yield
 ! CHECK-NEXT: }{{$}}
-! CHECK:      acc.delete accPtr(%[[COPYIN_A]] : !fir.ref<!fir.array<10xf32>>) bounds(%{{.*}}) {dataClause = #acc<data_clause acc_copyin>, name = "a"}
-! CHECK:      acc.delete accPtr(%[[COPYIN_B]] : !fir.ref<!fir.array<10xf32>>) bounds(%{{.*}}) {dataClause = #acc<data_clause acc_copyin_readonly>, name = "b"}
+! CHECK:      acc.delete accPtr(%[[COPYIN_A]] : !fir.ref<!fir.array<10xf32>>) {dataClause = #acc<data_clause acc_copyin>, name = "a"}
+! CHECK:      acc.delete accPtr(%[[COPYIN_B]] : !fir.ref<!fir.array<10xf32>>) {dataClause = #acc<data_clause acc_copyin_readonly>, name = "b"}
 
   !$acc parallel loop copyout(a) copyout(zero: b)
   DO i = 1, n
     a(i) = b(i)
   END DO
 
-! CHECK:      %[[CREATE_A:.*]] = acc.create varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10xf32>>) bounds(%{{.*}}) -> !fir.ref<!fir.array<10xf32>> {dataClause = #acc<data_clause acc_copyout>, name = "a"}
-! CHECK:      %[[CREATE_B:.*]] = acc.create varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<10xf32>>) bounds(%{{.*}}) -> !fir.ref<!fir.array<10xf32>> {dataClause = #acc<data_clause acc_copyout>, name = "b"}
+! CHECK:      %[[CREATE_A:.*]] = acc.create varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10xf32>>) -> !fir.ref<!fir.array<10xf32>> {dataClause = #acc<data_clause acc_copyout>, name = "a"}
+! CHECK:      %[[CREATE_B:.*]] = acc.create varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<10xf32>>) -> !fir.ref<!fir.array<10xf32>> {dataClause = #acc<data_clause acc_copyout>, name = "b"}
 ! CHECK:      acc.parallel {{.*}} dataOperands(%[[CREATE_A]], %[[CREATE_B]] : !fir.ref<!fir.array<10xf32>>, !fir.ref<!fir.array<10xf32>>) {
 ! CHECK:        acc.loop {{.*}} {
 ! CHECK:          acc.yield
 ! CHECK-NEXT:   }{{$}}
 ! CHECK:        acc.yield
 ! CHECK-NEXT: }{{$}}
-! CHECK:      acc.copyout accPtr(%[[CREATE_A]] : !fir.ref<!fir.array<10xf32>>) bounds(%{{.*}}) to varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10xf32>>) {name = "a"}
-! CHECK:      acc.copyout accPtr(%[[CREATE_B]] : !fir.ref<!fir.array<10xf32>>) bounds(%{{.*}}) to varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<10xf32>>) {name = "b"}
+! CHECK:      acc.copyout accPtr(%[[CREATE_A]] : !fir.ref<!fir.array<10xf32>>) to varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10xf32>>) {name = "a"}
+! CHECK:      acc.copyout accPtr(%[[CREATE_B]] : !fir.ref<!fir.array<10xf32>>) to varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<10xf32>>) {name = "b"}
 
   !$acc parallel loop create(b) create(zero: a)
   DO i = 1, n
     a(i) = b(i)
   END DO
 
-! CHECK:      %[[CREATE_B:.*]] = acc.create varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<10xf32>>) bounds(%{{.*}}) -> !fir.ref<!fir.array<10xf32>> {name = "b"}
-! CHECK:      %[[CREATE_A:.*]] = acc.create varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10xf32>>) bounds(%{{.*}}) -> !fir.ref<!fir.array<10xf32>> {dataClause = #acc<data_clause acc_create_zero>, name = "a"}
+! CHECK:      %[[CREATE_B:.*]] = acc.create varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<10xf32>>) -> !fir.ref<!fir.array<10xf32>> {name = "b"}
+! CHECK:      %[[CREATE_A:.*]] = acc.create varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10xf32>>) -> !fir.ref<!fir.array<10xf32>> {dataClause = #acc<data_clause acc_create_zero>, name = "a"}
 ! CHECK:      acc.parallel {{.*}} dataOperands(%[[CREATE_B]], %[[CREATE_A]] : !fir.ref<!fir.array<10xf32>>, !fir.ref<!fir.array<10xf32>>) {
 ! CHECK:        acc.loop {{.*}} {
 ! CHECK:          acc.yield
 ! CHECK-NEXT:   }{{$}}
 ! CHECK:        acc.yield
 ! CHECK-NEXT: }{{$}}
-! CHECK:      acc.delete accPtr(%[[CREATE_B]] : !fir.ref<!fir.array<10xf32>>) bounds(%{{.*}}) {dataClause = #acc<data_clause acc_create>, name = "b"}
-! CHECK:      acc.delete accPtr(%[[CREATE_A]] : !fir.ref<!fir.array<10xf32>>) bounds(%{{.*}}) {dataClause = #acc<data_clause acc_create_zero>, name = "a"}
+! CHECK:      acc.delete accPtr(%[[CREATE_B]] : !fir.ref<!fir.array<10xf32>>) {dataClause = #acc<data_clause acc_create>, name = "b"}
+! CHECK:      acc.delete accPtr(%[[CREATE_A]] : !fir.ref<!fir.array<10xf32>>) {dataClause = #acc<data_clause acc_create_zero>, name = "a"}
 
   !$acc parallel loop no_create(a, b)
   DO i = 1, n
     a(i) = b(i)
   END DO
 
-! CHECK:      %[[NOCREATE_A:.*]] = acc.nocreate varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10xf32>>) bounds(%{{.*}}) -> !fir.ref<!fir.array<10xf32>> {name = "a"}
-! CHECK:      %[[NOCREATE_B:.*]] = acc.nocreate varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<10xf32>>) bounds(%{{.*}}) -> !fir.ref<!fir.array<10xf32>> {name = "b"}
+! CHECK:      %[[NOCREATE_A:.*]] = acc.nocreate varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10xf32>>) -> !fir.ref<!fir.array<10xf32>> {name = "a"}
+! CHECK:      %[[NOCREATE_B:.*]] = acc.nocreate varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<10xf32>>) -> !fir.ref<!fir.array<10xf32>> {name = "b"}
 ! CHECK:      acc.parallel {{.*}} dataOperands(%[[NOCREATE_A]], %[[NOCREATE_B]] : !fir.ref<!fir.array<10xf32>>, !fir.ref<!fir.array<10xf32>>) {
 ! CHECK:        acc.loop {{.*}} {
 ! CHECK:          acc.yield
 ! CHECK-NEXT:   }{{$}}
 ! CHECK:        acc.yield
 ! CHECK-NEXT: }{{$}}
+! CHECK:      acc.delete accPtr(%[[NOCREATE_A]] : !fir.ref<!fir.array<10xf32>>) {dataClause = #acc<data_clause acc_no_create>, name = "a"}
+! CHECK:      acc.delete accPtr(%[[NOCREATE_B]] : !fir.ref<!fir.array<10xf32>>) {dataClause = #acc<data_clause acc_no_create>, name = "b"}
 
   !$acc parallel loop present(a, b)
   DO i = 1, n
     a(i) = b(i)
   END DO
 
-! CHECK:      %[[PRESENT_A:.*]] = acc.present varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10xf32>>) bounds(%{{.*}}) -> !fir.ref<!fir.array<10xf32>> {name = "a"}
-! CHECK:      %[[PRESENT_B:.*]] = acc.present varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<10xf32>>) bounds(%{{.*}}) -> !fir.ref<!fir.array<10xf32>> {name = "b"}
+! CHECK:      %[[PRESENT_A:.*]] = acc.present varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10xf32>>) -> !fir.ref<!fir.array<10xf32>> {name = "a"}
+! CHECK:      %[[PRESENT_B:.*]] = acc.present varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<10xf32>>) -> !fir.ref<!fir.array<10xf32>> {name = "b"}
 ! CHECK:      acc.parallel {{.*}} dataOperands(%[[PRESENT_A]], %[[PRESENT_B]] : !fir.ref<!fir.array<10xf32>>, !fir.ref<!fir.array<10xf32>>) {
 ! CHECK:        acc.loop {{.*}} {
 ! CHECK:          acc.yield
 ! CHECK-NEXT:   }{{$}}
 ! CHECK:        acc.yield
 ! CHECK-NEXT: }{{$}}
+! CHECK:      acc.delete accPtr(%[[PRESENT_A]] : !fir.ref<!fir.array<10xf32>>) {dataClause = #acc<data_clause acc_present>, name = "a"}
+! CHECK:      acc.delete accPtr(%[[PRESENT_B]] : !fir.ref<!fir.array<10xf32>>) {dataClause = #acc<data_clause acc_present>, name = "b"}
 
   !$acc parallel loop deviceptr(a) deviceptr(b)
   DO i = 1, n
     a(i) = b(i)
   END DO
 
-! CHECK:      %[[DEVICEPTR_A:.*]] = acc.deviceptr varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10xf32>>) bounds(%{{.*}}) -> !fir.ref<!fir.array<10xf32>> {name = "a"}
-! CHECK:      %[[DEVICEPTR_B:.*]] = acc.deviceptr varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<10xf32>>) bounds(%{{.*}}) -> !fir.ref<!fir.array<10xf32>> {name = "b"}
+! CHECK:      %[[DEVICEPTR_A:.*]] = acc.deviceptr varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10xf32>>) -> !fir.ref<!fir.array<10xf32>> {name = "a"}
+! CHECK:      %[[DEVICEPTR_B:.*]] = acc.deviceptr varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<10xf32>>) -> !fir.ref<!fir.array<10xf32>> {name = "b"}
 ! CHECK:      acc.parallel {{.*}} dataOperands(%[[DEVICEPTR_A]], %[[DEVICEPTR_B]] : !fir.ref<!fir.array<10xf32>>, !fir.ref<!fir.array<10xf32>>) {
 ! CHECK:        acc.loop {{.*}} {
 ! CHECK:          acc.yield
@@ -427,13 +431,9 @@ subroutine acc_parallel_loop
     a(i) = b(i)
   END DO
 
-! CHECK:      %[[BOX_F:.*]] = fir.load %[[DECLF]]#0 : !fir.ref<!fir.box<!fir.ptr<f32>>>
-! CHECK:      %[[BOX_ADDR_F:.*]] = fir.box_addr %[[BOX_F]] : (!fir.box<!fir.ptr<f32>>) -> !fir.ptr<f32>
-! CHECK:      %[[ATTACH_F:.*]] = acc.attach varPtr(%[[BOX_ADDR_F]] : !fir.ptr<f32>) -> !fir.ptr<f32> {name = "f"}
-! CHECK:      %[[BOX_G:.*]] = fir.load %[[DECLG]]#0 : !fir.ref<!fir.box<!fir.ptr<f32>>>
-! CHECK:      %[[BOX_ADDR_G:.*]] = fir.box_addr %[[BOX_G]] : (!fir.box<!fir.ptr<f32>>) -> !fir.ptr<f32>
-! CHECK:      %[[ATTACH_G:.*]] = acc.attach varPtr(%[[BOX_ADDR_G]] : !fir.ptr<f32>) -> !fir.ptr<f32> {name = "g"}
-! CHECK:      acc.parallel {{.*}} dataOperands(%[[ATTACH_F]], %[[ATTACH_G]] : !fir.ptr<f32>, !fir.ptr<f32>) {
+! CHECK:      %[[ATTACH_F:.*]] = acc.attach varPtr(%[[DECLF]]#0 : !fir.ref<!fir.box<!fir.ptr<f32>>>) -> !fir.ref<!fir.box<!fir.ptr<f32>>> {name = "f"}
+! CHECK:      %[[ATTACH_G:.*]] = acc.attach varPtr(%[[DECLG]]#0 : !fir.ref<!fir.box<!fir.ptr<f32>>>) -> !fir.ref<!fir.box<!fir.ptr<f32>>> {name = "g"}
+! CHECK:      acc.parallel {{.*}} dataOperands(%[[ATTACH_F]], %[[ATTACH_G]] : !fir.ref<!fir.box<!fir.ptr<f32>>>, !fir.ref<!fir.box<!fir.ptr<f32>>>) {
 ! CHECK:        acc.loop {{.*}} {
 ! CHECK:          acc.yield
 ! CHECK-NEXT:   }{{$}}
@@ -445,9 +445,9 @@ subroutine acc_parallel_loop
     a(i) = b(i)
   END DO
 
-! CHECK:      %[[ACC_PRIVATE_B:.*]] = acc.firstprivate varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<10xf32>>) bounds(%{{.*}}) -> !fir.ref<!fir.array<10xf32>> {name = "b"}
-! CHECK:      acc.parallel {{.*}} firstprivate(@firstprivatization_section_ext10_ref_10xf32 -> %[[ACC_PRIVATE_B]] : !fir.ref<!fir.array<10xf32>>) {
-! CHECK:      %[[ACC_PRIVATE_A:.*]] = acc.private varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10xf32>>) bounds(%{{.*}}) -> !fir.ref<!fir.array<10xf32>> {name = "a"}
+! CHECK:      %[[ACC_PRIVATE_B:.*]] = acc.firstprivate varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<10xf32>>) -> !fir.ref<!fir.array<10xf32>> {name = "b"}
+! CHECK:      acc.parallel {{.*}} firstprivate(@firstprivatization_ref_10xf32 -> %[[ACC_PRIVATE_B]] : !fir.ref<!fir.array<10xf32>>) {
+! CHECK:      %[[ACC_PRIVATE_A:.*]] = acc.private varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10xf32>>) -> !fir.ref<!fir.array<10xf32>> {name = "a"}
 ! CHECK:        acc.loop {{.*}} private({{.*}}@privatization_ref_10xf32 -> %[[ACC_PRIVATE_A]] : !fir.ref<!fir.array<10xf32>>)
 ! CHECK-NOT:      fir.do_loop
 ! CHECK:          acc.yield
diff --git a/flang/test/Lower/OpenACC/acc-parallel.f90 b/flang/test/Lower/OpenACC/acc-parallel.f90
index 6f6e3208d753a..e16c1b218878c 100644
--- a/flang/test/Lower/OpenACC/acc-parallel.f90
+++ b/flang/test/Lower/OpenACC/acc-parallel.f90
@@ -2,7 +2,7 @@
 
 ! RUN: bbc -fopenacc -emit-hlfir %s -o - | FileCheck %s
 
-! CHECK-LABEL: acc.firstprivate.recipe @firstprivatization_section_ext10xext10_ref_10x10xf32 : !fir.ref<!fir.array<10x10xf32>> init {
+! CHECK-LABEL: acc.firstprivate.recipe @firstprivatization_ref_10x10xf32 : !fir.ref<!fir.array<10x10xf32>> init {
 ! CHECK: ^bb0(%{{.*}}: !fir.ref<!fir.array<10x10xf32>>):
 ! CHECK:   %[[SHAPE:.*]] = fir.shape %{{.*}}, %{{.*}} : (index, index) -> !fir.shape<2>
 ! CHECK:   %[[ALLOCA:.*]] = fir.alloca !fir.array<10x10xf32>
@@ -211,103 +211,107 @@ subroutine acc_parallel
   !$acc parallel copy(a, b, c)
   !$acc end parallel
 
-! CHECK:      %[[COPYIN_A:.*]] = acc.copyin varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = #acc<data_clause acc_copy>, name = "a"}
-! CHECK:      %[[COPYIN_B:.*]] = acc.copyin varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = #acc<data_clause acc_copy>, name = "b"}
-! CHECK:      %[[COPYIN_C:.*]] = acc.copyin varPtr(%[[DECLC]]#0 : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = #acc<data_clause acc_copy>, name = "c"}
+! CHECK:      %[[COPYIN_A:.*]] = acc.copyin varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = #acc<data_clause acc_copy>, name = "a"}
+! CHECK:      %[[COPYIN_B:.*]] = acc.copyin varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<10x10xf32>>) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = #acc<data_clause acc_copy>, name = "b"}
+! CHECK:      %[[COPYIN_C:.*]] = acc.copyin varPtr(%[[DECLC]]#0 : !fir.ref<!fir.array<10x10xf32>>) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = #acc<data_clause acc_copy>, name = "c"}
 ! CHECK:      acc.parallel dataOperands(%[[COPYIN_A]], %[[COPYIN_B]], %[[COPYIN_C]] : !fir.ref<!fir.array<10x10xf32>>, !fir.ref<!fir.array<10x10xf32>>, !fir.ref<!fir.array<10x10xf32>>) {
 ! CHECK:        acc.yield
 ! CHECK-NEXT: }{{$}}
-! CHECK:      acc.copyout accPtr(%[[COPYIN_A]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) to varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) {dataClause = #acc<data_clause acc_copy>, name = "a"}
-! CHECK:      acc.copyout accPtr(%[[COPYIN_B]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) to varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<10x10xf32>>) {dataClause = #acc<data_clause acc_copy>, name = "b"}
-! CHECK:      acc.copyout accPtr(%[[COPYIN_C]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) to varPtr(%[[DECLC]]#0 : !fir.ref<!fir.array<10x10xf32>>) {dataClause = #acc<data_clause acc_copy>, name = "c"}
+! CHECK:      acc.copyout accPtr(%[[COPYIN_A]] : !fir.ref<!fir.array<10x10xf32>>) to varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) {dataClause = #acc<data_clause acc_copy>, name = "a"}
+! CHECK:      acc.copyout accPtr(%[[COPYIN_B]] : !fir.ref<!fir.array<10x10xf32>>) to varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<10x10xf32>>) {dataClause = #acc<data_clause acc_copy>, name = "b"}
+! CHECK:      acc.copyout accPtr(%[[COPYIN_C]] : !fir.ref<!fir.array<10x10xf32>>) to varPtr(%[[DECLC]]#0 : !fir.ref<!fir.array<10x10xf32>>) {dataClause = #acc<data_clause acc_copy>, name = "c"}
 
   !$acc parallel copy(a) copy(b) copy(c)
   !$acc end parallel
 
 
-! CHECK:      %[[COPYIN_A:.*]] = acc.copyin varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = #acc<data_clause acc_copy>, name = "a"}
-! CHECK:      %[[COPYIN_B:.*]] = acc.copyin varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = #acc<data_clause acc_copy>, name = "b"}
-! CHECK:      %[[COPYIN_C:.*]] = acc.copyin varPtr(%[[DECLC]]#0 : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = #acc<data_clause acc_copy>, name = "c"}
+! CHECK:      %[[COPYIN_A:.*]] = acc.copyin varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = #acc<data_clause acc_copy>, name = "a"}
+! CHECK:      %[[COPYIN_B:.*]] = acc.copyin varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<10x10xf32>>) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = #acc<data_clause acc_copy>, name = "b"}
+! CHECK:      %[[COPYIN_C:.*]] = acc.copyin varPtr(%[[DECLC]]#0 : !fir.ref<!fir.array<10x10xf32>>) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = #acc<data_clause acc_copy>, name = "c"}
 ! CHECK:      acc.parallel dataOperands(%[[COPYIN_A]], %[[COPYIN_B]], %[[COPYIN_C]] : !fir.ref<!fir.array<10x10xf32>>, !fir.ref<!fir.array<10x10xf32>>, !fir.ref<!fir.array<10x10xf32>>) {
 ! CHECK:        acc.yield
 ! CHECK-NEXT: }{{$}}
-! CHECK:      acc.copyout accPtr(%[[COPYIN_A]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) to varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) {dataClause = #acc<data_clause acc_copy>, name = "a"}
-! CHECK:      acc.copyout accPtr(%[[COPYIN_B]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) to varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<10x10xf32>>) {dataClause = #acc<data_clause acc_copy>, name = "b"}
-! CHECK:      acc.copyout accPtr(%[[COPYIN_C]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) to varPtr(%[[DECLC]]#0 : !fir.ref<!fir.array<10x10xf32>>) {dataClause = #acc<data_clause acc_copy>, name = "c"}
+! CHECK:      acc.copyout accPtr(%[[COPYIN_A]] : !fir.ref<!fir.array<10x10xf32>>) to varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) {dataClause = #acc<data_clause acc_copy>, name = "a"}
+! CHECK:      acc.copyout accPtr(%[[COPYIN_B]] : !fir.ref<!fir.array<10x10xf32>>) to varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<10x10xf32>>) {dataClause = #acc<data_clause acc_copy>, name = "b"}
+! CHECK:      acc.copyout accPtr(%[[COPYIN_C]] : !fir.ref<!fir.array<10x10xf32>>) to varPtr(%[[DECLC]]#0 : !fir.ref<!fir.array<10x10xf32>>) {dataClause = #acc<data_clause acc_copy>, name = "c"}
 
   !$acc parallel copyin(a) copyin(readonly: b, c)
   !$acc end parallel
 
-! CHECK:      %[[COPYIN_A:.*]] = acc.copyin varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {name = "a"}
-! CHECK:      %[[COPYIN_B:.*]] = acc.copyin varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = #acc<data_clause acc_copyin_readonly>, name = "b"}
-! CHECK:      %[[COPYIN_C:.*]] = acc.copyin varPtr(%[[DECLC]]#0 : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = #acc<data_clause acc_copyin_readonly>, name = "c"}
+! CHECK:      %[[COPYIN_A:.*]] = acc.copyin varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) -> !fir.ref<!fir.array<10x10xf32>> {name = "a"}
+! CHECK:      %[[COPYIN_B:.*]] = acc.copyin varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<10x10xf32>>) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = #acc<data_clause acc_copyin_readonly>, name = "b"}
+! CHECK:      %[[COPYIN_C:.*]] = acc.copyin varPtr(%[[DECLC]]#0 : !fir.ref<!fir.array<10x10xf32>>) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = #acc<data_clause acc_copyin_readonly>, name = "c"}
 ! CHECK:      acc.parallel dataOperands(%[[COPYIN_A]], %[[COPYIN_B]], %[[COPYIN_C]] : !fir.ref<!fir.array<10x10xf32>>, !fir.ref<!fir.array<10x10xf32>>, !fir.ref<!fir.array<10x10xf32>>) {
 ! CHECK:        acc.yield
 ! CHECK-NEXT: }{{$}}
-! CHECK:      acc.delete accPtr(%[[COPYIN_A]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) {dataClause = #acc<data_clause acc_copyin>, name = "a"}
-! CHECK:      acc.delete accPtr(%[[COPYIN_B]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) {dataClause = #acc<data_clause acc_copyin_readonly>, name = "b"}
-! CHECK:      acc.delete accPtr(%[[COPYIN_C]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) {dataClause = #acc<data_clause acc_copyin_readonly>, name = "c"}
+! CHECK:      acc.delete accPtr(%[[COPYIN_A]] : !fir.ref<!fir.array<10x10xf32>>) {dataClause = #acc<data_clause acc_copyin>, name = "a"}
+! CHECK:      acc.delete accPtr(%[[COPYIN_B]] : !fir.ref<!fir.array<10x10xf32>>) {dataClause = #acc<data_clause acc_copyin_readonly>, name = "b"}
+! CHECK:      acc.delete accPtr(%[[COPYIN_C]] : !fir.ref<!fir.array<10x10xf32>>) {dataClause = #acc<data_clause acc_copyin_readonly>, name = "c"}
 
   !$acc parallel copyout(a) copyout(zero: b) copyout(c)
   !$acc end parallel
 
-! CHECK: %[[CREATE_A:.*]] = acc.create varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = #acc<data_clause acc_copyout>, name = "a"}
-! CHECK: %[[CREATE_B:.*]] = acc.create varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = #acc<data_clause acc_copyout>, name = "b"}
-! CHECK: %[[CREATE_C:.*]] = acc.create varPtr(%[[DECLC]]#0 : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = #acc<data_clause acc_copyout>, name = "c"}
+! CHECK: %[[CREATE_A:.*]] = acc.create varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = #acc<data_clause acc_copyout>, name = "a"}
+! CHECK: %[[CREATE_B:.*]] = acc.create varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<10x10xf32>>) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = #acc<data_clause acc_copyout>, name = "b"}
+! CHECK: %[[CREATE_C:.*]] = acc.create varPtr(%[[DECLC]]#0 : !fir.ref<!fir.array<10x10xf32>>) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = #acc<data_clause acc_copyout>, name = "c"}
 ! CHECK:      acc.parallel dataOperands(%[[CREATE_A]], %[[CREATE_B]], %[[CREATE_C]] : !fir.ref<!fir.array<10x10xf32>>, !fir.ref<!fir.array<10x10xf32>>, !fir.ref<!fir.array<10x10xf32>>) {
 ! CHECK:        acc.yield
 ! CHECK-NEXT: }{{$}}
-! CHECK: acc.copyout accPtr(%[[CREATE_A]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) to varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) {name = "a"}
-! CHECK: acc.copyout accPtr(%[[CREATE_B]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) to varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<10x10xf32>>) {name = "b"}
-! CHECK: acc.copyout accPtr(%[[CREATE_C]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) to varPtr(%[[DECLC]]#0 : !fir.ref<!fir.array<10x10xf32>>) {name = "c"}
+! CHECK: acc.copyout accPtr(%[[CREATE_A]] : !fir.ref<!fir.array<10x10xf32>>) to varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) {name = "a"}
+! CHECK: acc.copyout accPtr(%[[CREATE_B]] : !fir.ref<!fir.array<10x10xf32>>) to varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<10x10xf32>>) {name = "b"}
+! CHECK: acc.copyout accPtr(%[[CREATE_C]] : !fir.ref<!fir.array<10x10xf32>>) to varPtr(%[[DECLC]]#0 : !fir.ref<!fir.array<10x10xf32>>) {name = "c"}
 
   !$acc parallel create(a, b) create(zero: c)
   !$acc end parallel
 
-! CHECK: %[[CREATE_A:.*]] = acc.create varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {name = "a"}
-! CHECK: %[[CREATE_B:.*]] = acc.create varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {name = "b"}
-! CHECK: %[[CREATE_C:.*]] = acc.create varPtr(%[[DECLC]]#0 : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = #acc<data_clause acc_create_zero>, name = "c"}
+! CHECK: %[[CREATE_A:.*]] = acc.create varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) -> !fir.ref<!fir.array<10x10xf32>> {name = "a"}
+! CHECK: %[[CREATE_B:.*]] = acc.create varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<10x10xf32>>) -> !fir.ref<!fir.array<10x10xf32>> {name = "b"}
+! CHECK: %[[CREATE_C:.*]] = acc.create varPtr(%[[DECLC]]#0 : !fir.ref<!fir.array<10x10xf32>>) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = #acc<data_clause acc_create_zero>, name = "c"}
 ! CHECK:      acc.parallel dataOperands(%[[CREATE_A]], %[[CREATE_B]], %[[CREATE_C]] : !fir.ref<!fir.array<10x10xf32>>, !fir.ref<!fir.array<10x10xf32>>, !fir.ref<!fir.array<10x10xf32>>) {
 ! CHECK:        acc.yield
 ! CHECK-NEXT: }{{$}}
-! CHECK: acc.delete accPtr(%[[CREATE_A]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) {dataClause = #acc<data_clause acc_create>, name = "a"}
-! CHECK: acc.delete accPtr(%[[CREATE_B]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) {dataClause = #acc<data_clause acc_create>, name = "b"}
-! CHECK: acc.delete accPtr(%[[CREATE_C]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) {dataClause = #acc<data_clause acc_create_zero>, name = "c"}
+! CHECK: acc.delete accPtr(%[[CREATE_A]] : !fir.ref<!fir.array<10x10xf32>>) {dataClause = #acc<data_clause acc_create>, name = "a"}
+! CHECK: acc.delete accPtr(%[[CREATE_B]] : !fir.ref<!fir.array<10x10xf32>>) {dataClause = #acc<data_clause acc_create>, name = "b"}
+! CHECK: acc.delete accPtr(%[[CREATE_C]] : !fir.ref<!fir.array<10x10xf32>>) {dataClause = #acc<data_clause acc_create_zero>, name = "c"}
 
   !$acc parallel create(c) copy(b) create(a)
   !$acc end parallel
-! CHECK: %[[CREATE_C:.*]] = acc.create varPtr(%[[DECLC]]#0 : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {name = "c"}
-! CHECK: %[[COPY_B:.*]] = acc.copyin varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = #acc<data_clause acc_copy>, name = "b"}
-! CHECK: %[[CREATE_A:.*]] = acc.create varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {name = "a"}
+! CHECK: %[[CREATE_C:.*]] = acc.create varPtr(%[[DECLC]]#0 : !fir.ref<!fir.array<10x10xf32>>) -> !fir.ref<!fir.array<10x10xf32>> {name = "c"}
+! CHECK: %[[COPY_B:.*]] = acc.copyin varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<10x10xf32>>) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = #acc<data_clause acc_copy>, name = "b"}
+! CHECK: %[[CREATE_A:.*]] = acc.create varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) -> !fir.ref<!fir.array<10x10xf32>> {name = "a"}
 ! CHECK: acc.parallel   dataOperands(%[[CREATE_C]], %[[COPY_B]], %[[CREATE_A]] : !fir.ref<!fir.array<10x10xf32>>, !fir.ref<!fir.array<10x10xf32>>, !fir.ref<!fir.array<10x10xf32>>) {
 
   !$acc parallel no_create(a, b) create(zero: c)
   !$acc end parallel
 
-! CHECK: %[[NO_CREATE_A:.*]] = acc.nocreate varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {name = "a"}
-! CHECK: %[[NO_CREATE_B:.*]] = acc.nocreate varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {name = "b"}
-! CHECK: %[[CREATE_C:.*]] = acc.create varPtr(%[[DECLC]]#0 : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = #acc<data_clause acc_create_zero>, name = "c"}
+! CHECK: %[[NO_CREATE_A:.*]] = acc.nocreate varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) -> !fir.ref<!fir.array<10x10xf32>> {name = "a"}
+! CHECK: %[[NO_CREATE_B:.*]] = acc.nocreate varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<10x10xf32>>) -> !fir.ref<!fir.array<10x10xf32>> {name = "b"}
+! CHECK: %[[CREATE_C:.*]] = acc.create varPtr(%[[DECLC]]#0 : !fir.ref<!fir.array<10x10xf32>>) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = #acc<data_clause acc_create_zero>, name = "c"}
 ! CHECK:      acc.parallel dataOperands(%[[NO_CREATE_A]], %[[NO_CREATE_B]], %[[CREATE_C]] : !fir.ref<!fir.array<10x10xf32>>, !fir.ref<!fir.array<10x10xf32>>, !fir.ref<!fir.array<10x10xf32>>) {
 ! CHECK:        acc.yield
 ! CHECK-NEXT: }{{$}}
-! CHECK: acc.delete accPtr(%[[CREATE_C]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) {dataClause = #acc<data_clause acc_create_zero>, name = "c"}
-
+! CHECK: acc.delete accPtr(%[[CREATE_C]] : !fir.ref<!fir.array<10x10xf32>>) {dataClause = #acc<data_clause acc_create_zero>, name = "c"}
+! CHECK: acc.delete accPtr(%[[NO_CREATE_A]] : !fir.ref<!fir.array<10x10xf32>>) {dataClause = #acc<data_clause acc_no_create>, name = "a"}
+! CHECK: acc.delete accPtr(%[[NO_CREATE_B]] : !fir.ref<!fir.array<10x10xf32>>) {dataClause = #acc<data_clause acc_no_create>, name = "b"}
 
   !$acc parallel present(a, b, c)
   !$acc end parallel
 
-! CHECK: %[[PRESENT_A:.*]] = acc.present varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {name = "a"}
-! CHECK: %[[PRESENT_B:.*]] = acc.present varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {name = "b"}
-! CHECK: %[[PRESENT_C:.*]] = acc.present varPtr(%[[DECLC]]#0 : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {name = "c"}
+! CHECK: %[[PRESENT_A:.*]] = acc.present varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) -> !fir.ref<!fir.array<10x10xf32>> {name = "a"}
+! CHECK: %[[PRESENT_B:.*]] = acc.present varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<10x10xf32>>) -> !fir.ref<!fir.array<10x10xf32>> {name = "b"}
+! CHECK: %[[PRESENT_C:.*]] = acc.present varPtr(%[[DECLC]]#0 : !fir.ref<!fir.array<10x10xf32>>) -> !fir.ref<!fir.array<10x10xf32>> {name = "c"}
 ! CHECK:      acc.parallel dataOperands(%[[PRESENT_A]], %[[PRESENT_B]], %[[PRESENT_C]] : !fir.ref<!fir.array<10x10xf32>>, !fir.ref<!fir.array<10x10xf32>>, !fir.ref<!fir.array<10x10xf32>>) {
 ! CHECK:        acc.yield
 ! CHECK-NEXT: }{{$}}
+! CHECK: acc.delete accPtr(%[[PRESENT_A]] : !fir.ref<!fir.array<10x10xf32>>) {dataClause = #acc<data_clause acc_present>, name = "a"}
+! CHECK: acc.delete accPtr(%[[PRESENT_B]] : !fir.ref<!fir.array<10x10xf32>>) {dataClause = #acc<data_clause acc_present>, name = "b"}
+! CHECK: acc.delete accPtr(%[[PRESENT_C]] : !fir.ref<!fir.array<10x10xf32>>) {dataClause = #acc<data_clause acc_present>, name = "c"}
 
   !$acc parallel deviceptr(a) deviceptr(c)
   !$acc end parallel
 
-! CHECK: %[[DEVICEPTR_A:.*]] = acc.deviceptr varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {name = "a"}
-! CHECK: %[[DEVICEPTR_C:.*]] = acc.deviceptr varPtr(%[[DECLC]]#0 : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {name = "c"}
+! CHECK: %[[DEVICEPTR_A:.*]] = acc.deviceptr varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) -> !fir.ref<!fir.array<10x10xf32>> {name = "a"}
+! CHECK: %[[DEVICEPTR_C:.*]] = acc.deviceptr varPtr(%[[DECLC]]#0 : !fir.ref<!fir.array<10x10xf32>>) -> !fir.ref<!fir.array<10x10xf32>> {name = "c"}
 ! CHECK:      acc.parallel dataOperands(%[[DEVICEPTR_A]], %[[DEVICEPTR_C]] : !fir.ref<!fir.array<10x10xf32>>, !fir.ref<!fir.array<10x10xf32>>) {
 ! CHECK:        acc.yield
 ! CHECK-NEXT: }{{$}}
@@ -315,25 +319,21 @@ subroutine acc_parallel
   !$acc parallel attach(d, e)
   !$acc end parallel
 
-! CHECK: %[[BOX_D:.*]] = fir.load %[[DECLD]]#0 : !fir.ref<!fir.box<!fir.ptr<f32>>>
-! CHECK: %[[BOX_ADDR_D:.*]] = fir.box_addr %[[BOX_D]] : (!fir.box<!fir.ptr<f32>>) -> !fir.ptr<f32>
-! CHECK: %[[ATTACH_D:.*]] = acc.attach varPtr(%[[BOX_ADDR_D]] : !fir.ptr<f32>) -> !fir.ptr<f32> {name = "d"}
-! CHECK: %[[BOX_E:.*]] = fir.load %[[DECLE]]#0 : !fir.ref<!fir.box<!fir.ptr<f32>>>
-! CHECK: %[[BOX_ADDR_E:.*]] = fir.box_addr %[[BOX_E]] : (!fir.box<!fir.ptr<f32>>) -> !fir.ptr<f32>
-! CHECK: %[[ATTACH_E:.*]] = acc.attach varPtr(%[[BOX_ADDR_E]] : !fir.ptr<f32>) -> !fir.ptr<f32> {name = "e"}
-! CHECK: acc.parallel dataOperands(%[[ATTACH_D]], %[[ATTACH_E]] : !fir.ptr<f32>, !fir.ptr<f32>) {
+! CHECK: %[[ATTACH_D:.*]] = acc.attach varPtr(%[[DECLD]]#0 : !fir.ref<!fir.box<!fir.ptr<f32>>>) -> !fir.ref<!fir.box<!fir.ptr<f32>>> {name = "d"}
+! CHECK: %[[ATTACH_E:.*]] = acc.attach varPtr(%[[DECLE]]#0 : !fir.ref<!fir.box<!fir.ptr<f32>>>) -> !fir.ref<!fir.box<!fir.ptr<f32>>> {name = "e"}
+! CHECK: acc.parallel dataOperands(%[[ATTACH_D]], %[[ATTACH_E]] : !fir.ref<!fir.box<!fir.ptr<f32>>>, !fir.ref<!fir.box<!fir.ptr<f32>>>) {
 ! CHECK:        acc.yield
 ! CHECK-NEXT: }{{$}}
-! CHECK: acc.detach accPtr(%[[ATTACH_D]] : !fir.ptr<f32>) {dataClause = #acc<data_clause acc_attach>, name = "d"}
-! CHECK: acc.detach accPtr(%[[ATTACH_E]] : !fir.ptr<f32>) {dataClause = #acc<data_clause acc_attach>, name = "e"}
+! CHECK: acc.detach accPtr(%[[ATTACH_D]] : !fir.ref<!fir.box<!fir.ptr<f32>>>) {dataClause = #acc<data_clause acc_attach>, name = "d"}
+! CHECK: acc.detach accPtr(%[[ATTACH_E]] : !fir.ref<!fir.box<!fir.ptr<f32>>>) {dataClause = #acc<data_clause acc_attach>, name = "e"}
 
 !$acc parallel private(a) firstprivate(b) private(c) async(1)
 !$acc end parallel
 
-! CHECK:      %[[ACC_PRIVATE_A:.*]] = acc.private varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) async([[ASYNC3:%.*]]) -> !fir.ref<!fir.array<10x10xf32>> {name = "a"}
-! CHECK:      %[[ACC_FPRIVATE_B:.*]] = acc.firstprivate varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) async([[ASYNC3]]) -> !fir.ref<!fir.array<10x10xf32>> {name = "b"}
-! CHECK:      %[[ACC_PRIVATE_C:.*]] = acc.private varPtr(%[[DECLC]]#0 : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) async([[ASYNC3]]) -> !fir.ref<!fir.array<10x10xf32>> {name = "c"}
-! CHECK:      acc.parallel async([[ASYNC3]]) firstprivate(@firstprivatization_section_ext10xext10_ref_10x10xf32 -> %[[ACC_FPRIVATE_B]] : !fir.ref<!fir.array<10x10xf32>>) private(@privatization_ref_10x10xf32 -> %[[ACC_PRIVATE_A]] : !fir.ref<!fir.array<10x10xf32>>, @privatization_ref_10x10xf32 -> %[[ACC_PRIVATE_C]] : !fir.ref<!fir.array<10x10xf32>>) {
+! CHECK:      %[[ACC_PRIVATE_A:.*]] = acc.private varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) async([[ASYNC3:%.*]]) -> !fir.ref<!fir.array<10x10xf32>> {name = "a"}
+! CHECK:      %[[ACC_FPRIVATE_B:.*]] = acc.firstprivate varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<10x10xf32>>) async([[ASYNC3]]) -> !fir.ref<!fir.array<10x10xf32>> {name = "b"}
+! CHECK:      %[[ACC_PRIVATE_C:.*]] = acc.private varPtr(%[[DECLC]]#0 : !fir.ref<!fir.array<10x10xf32>>) async([[ASYNC3]]) -> !fir.ref<!fir.array<10x10xf32>> {name = "c"}
+! CHECK:      acc.parallel async([[ASYNC3]]) firstprivate(@firstprivatization_ref_10x10xf32 -> %[[ACC_FPRIVATE_B]] : !fir.ref<!fir.array<10x10xf32>>) private(@privatization_ref_10x10xf32 -> %[[ACC_PRIVATE_A]] : !fir.ref<!fir.array<10x10xf32>>, @privatization_ref_10x10xf32 -> %[[ACC_PRIVATE_C]] : !fir.ref<!fir.array<10x10xf32>>) {
 ! CHECK:        acc.yield
 ! CHECK-NEXT: }{{$}}
 
diff --git a/flang/test/Lower/OpenACC/acc-private-unwrap-defaultbounds.f90 b/flang/test/Lower/OpenACC/acc-private-unwrap-defaultbounds.f90
new file mode 100644
index 0000000000000..d86f82dae8d03
--- /dev/null
+++ b/flang/test/Lower/OpenACC/acc-private-unwrap-defaultbounds.f90
@@ -0,0 +1,403 @@
+! This test checks lowering of OpenACC loop directive.
+
+! RUN: bbc -fopenacc -emit-hlfir --openacc-unwrap-fir-box=true --openacc-generate-default-bounds=true %s -o - | FileCheck %s
+
+! CHECK-LABEL: acc.private.recipe @privatization_ref_10xf32 : !fir.ref<!fir.array<10xf32>> init {
+! CHECK:   ^bb0(%[[ARG0:.*]]: !fir.ref<!fir.array<10xf32>>):
+! CHECK:   %[[C10:.*]] = arith.constant 10 : index
+! CHECK:   %[[SHAPE:.*]] = fir.shape %[[C10]] : (index) -> !fir.shape<1>
+! CHECK:   %[[ALLOCA:.*]] = fir.alloca !fir.array<10xf32>
+! CHECK:   %[[DECLARE:.*]]:2 = hlfir.declare %[[ALLOCA]](%[[SHAPE]]) {uniq_name = "acc.private.init"} : (!fir.ref<!fir.array<10xf32>>, !fir.shape<1>) -> (!fir.ref<!fir.array<10xf32>>, !fir.ref<!fir.array<10xf32>>)
+! CHECK:   acc.yield %[[DECLARE]]#0 : !fir.ref<!fir.array<10xf32>>
+! CHECK: }
+
+! CHECK-LABEL: acc.firstprivate.recipe @firstprivatization_box_UxUx2xi32 : !fir.box<!fir.array<?x?x2xi32>> init {
+! CHECK: ^bb0(%[[ARG0:.*]]: !fir.box<!fir.array<?x?x2xi32>>):
+! CHECK:   %[[DIM0:.*]]:3 = fir.box_dims %arg0, %c0{{.*}} : (!fir.box<!fir.array<?x?x2xi32>>, index) -> (index, index, index)
+! CHECK:   %[[DIM1:.*]]:3 = fir.box_dims %arg0, %c1{{.*}} : (!fir.box<!fir.array<?x?x2xi32>>, index) -> (index, index, index)
+! CHECK:   %[[SHAPE:.*]] = fir.shape %[[DIM0]]#1, %[[DIM1]]#1, %c2{{.*}} : (index, index, index) -> !fir.shape<3>
+! CHECK:   %[[TEMP:.*]] = fir.allocmem !fir.array<?x?x2xi32>, %[[DIM0]]#1, %[[DIM1]]#1 {bindc_name = ".tmp", uniq_name = ""}
+! CHECK:   %[[DECL:.*]]:2 = hlfir.declare %[[TEMP]](%[[SHAPE]]) {uniq_name = ".tmp"} : (!fir.heap<!fir.array<?x?x2xi32>>, !fir.shape<3>) -> (!fir.box<!fir.array<?x?x2xi32>>, !fir.heap<!fir.array<?x?x2xi32>>)
+! CHECK:   acc.yield %[[DECL]]#0 : !fir.box<!fir.array<?x?x2xi32>>
+! CHECK: } copy {
+! CHECK: ^bb0(%[[ARG0:.*]]: !fir.box<!fir.array<?x?x2xi32>>, %[[ARG1:.*]]: !fir.box<!fir.array<?x?x2xi32>>, %[[LB0:.*]]: index, %[[UB0:.*]]: index, %[[STEP0:.*]]: index, %[[LB1:.*]]: index, %[[UB1:.*]]: index, %[[STEP1:.*]]: index, %[[LB2:.*]]: index, %[[UB2:.*]]: index, %[[STEP2:.*]]: index):
+! CHECK:   %[[SHAPE:.*]] = fir.shape %{{.*}}, %{{.*}}, %{{.*}} : (index, index, index) -> !fir.shape<3>
+! CHECK:   %[[DES_SRC:.*]] = hlfir.designate %[[ARG0]] (%[[LB0]]:%[[UB0]]:%[[STEP0]], %[[LB1]]:%[[UB1]]:%[[STEP1]], %[[LB2]]:%[[UB2]]:%[[STEP2]]) shape %[[SHAPE]] : (!fir.box<!fir.array<?x?x2xi32>>, index, index, index, index, index, index, index, index, index, !fir.shape<3>) -> !fir.box<!fir.array<?x?x2xi32>>
+! CHECK:   %[[DES_DST:.*]] = hlfir.designate %[[ARG1]] (%[[LB0]]:%[[UB0]]:%[[STEP0]], %[[LB1]]:%[[UB1]]:%[[STEP1]], %[[LB2]]:%[[UB2]]:%[[STEP2]]) shape %[[SHAPE]] : (!fir.box<!fir.array<?x?x2xi32>>, index, index, index, index, index, index, index, index, index, !fir.shape<3>) -> !fir.box<!fir.array<?x?x2xi32>>
+! CHECK:   hlfir.assign %[[DES_SRC]] to %[[DES_DST]] : !fir.box<!fir.array<?x?x2xi32>>, !fir.box<!fir.array<?x?x2xi32>>
+! CHECK:   acc.terminator
+! CHECK: }
+
+! CHECK-LABEL: acc.firstprivate.recipe @firstprivatization_section_lb4.ub9_box_Uxi32 : !fir.box<!fir.array<?xi32>> init {
+! CHECK: ^bb0(%{{.*}}: !fir.box<!fir.array<?xi32>>):
+! CHECK: } copy {
+! CHECK: ^bb0(%[[ARG0:.*]]: !fir.box<!fir.array<?xi32>>, %[[ARG1:.*]]: !fir.box<!fir.array<?xi32>>):
+! CHECK:   %[[LB:.*]] = arith.constant 4 : index
+! CHECK:   %[[UB:.*]] = arith.constant 9 : index
+! CHECK:   %[[STEP:.*]] = arith.constant 1 : index
+! CHECK:   %[[C1:.*]] = arith.constant 1 : index
+! CHECK:   %[[C0:.*]] = arith.constant 0 : index
+! CHECK:   %[[EXT0:.*]] = arith.subi %[[UB]], %[[LB]] : index
+! CHECK:   %[[EXT1:.*]] = arith.addi %[[EXT0]], %[[C1]] : index
+! CHECK:   %[[EXT2:.*]] = arith.divsi %[[EXT1]], %[[STEP]] : index
+! CHECK:   %[[CMP:.*]] = arith.cmpi sgt, %[[EXT2]], %[[C0]] : index
+! CHECK:   %[[SELECT:.*]] = arith.select %[[CMP]], %[[EXT2]], %[[C0]] : index
+! CHECK:   %[[SHAPE:.*]] = fir.shape %[[SELECT]] : (index) -> !fir.shape<1>
+! CHECK:   %[[LEFT:.*]] = hlfir.designate %[[ARG0]] shape %[[SHAPE]] : (!fir.box<!fir.array<?xi32>>, !fir.shape<1>) -> !fir.box<!fir.array<?xi32>>
+! CHECK:   %[[RIGHT:.*]] = hlfir.designate %[[ARG1]] shape %[[SHAPE]] : (!fir.box<!fir.array<?xi32>>, !fir.shape<1>) -> !fir.box<!fir.array<?xi32>>
+! CHECK:   hlfir.assign %[[LEFT]] to %[[RIGHT]] : !fir.box<!fir.array<?xi32>>, !fir.box<!fir.array<?xi32>>
+! CHECK:   acc.terminator
+! CHECK: }
+
+! CHECK-LABEL: acc.firstprivate.recipe @firstprivatization_box_Uxi32 : !fir.box<!fir.array<?xi32>> init {
+! CHECK: ^bb0(%[[ARG0:.*]]: !fir.box<!fir.array<?xi32>>):
+! CHECK:   %[[C0:.*]] = arith.constant 0 : index
+! CHECK:   %[[BOX_DIMS:.*]]:3 = fir.box_dims %[[ARG0]], %c0 : (!fir.box<!fir.array<?xi32>>, index) -> (index, index, index)
+! CHECK:   %[[SHAPE:.*]] = fir.shape %[[BOX_DIMS]]#1 : (index) -> !fir.shape<1>
+! CHECK:   %[[TEMP:.*]] = fir.allocmem !fir.array<?xi32>, %[[BOX_DIMS]]#1 {bindc_name = ".tmp", uniq_name = ""}
+! CHECK:   %[[DECL:.*]]:2 = hlfir.declare %[[TEMP]](%[[SHAPE]]) {uniq_name = ".tmp"} : (!fir.heap<!fir.array<?xi32>>, !fir.shape<1>) -> (!fir.box<!fir.array<?xi32>>, !fir.heap<!fir.array<?xi32>>)
+! CHECK:   acc.yield %[[DECL]]#0 : !fir.box<!fir.array<?xi32>>
+! CHECK: } copy {
+! CHECK: ^bb0(%[[ARG0:.*]]: !fir.box<!fir.array<?xi32>>, %[[ARG1:.*]]: !fir.box<!fir.array<?xi32>>, %[[ARG2:.*]]: index, %[[ARG3:.*]]: index, %[[ARG4:.*]]: index):
+! CHECK:   %[[SHAPE:.*]] = fir.shape %{{.*}} : (index) -> !fir.shape<1>
+! CHECK:   %[[DES_V1:.*]] = hlfir.designate %[[ARG0]] (%{{.*}}:%{{.*}}:%{{.*}}) shape %[[SHAPE]] : (!fir.box<!fir.array<?xi32>>, index, index, index, !fir.shape<1>) -> !fir.box<!fir.array<?xi32>>
+! CHECK:   %[[DES_V2:.*]] = hlfir.designate %[[ARG1]] (%{{.*}}:%{{.*}}:%{{.*}}) shape %[[SHAPE]] : (!fir.box<!fir.array<?xi32>>, index, index, index, !fir.shape<1>) -> !fir.box<!fir.array<?xi32>>
+! CHECK:   hlfir.assign %[[DES_V1]] to %[[DES_V2]] : !fir.box<!fir.array<?xi32>>, !fir.box<!fir.array<?xi32>>
+! CHECK:   acc.terminator
+! CHECK: }
+
+! CHECK-LABEL: acc.private.recipe @privatization_box_UxUx2xi32 : !fir.box<!fir.array<?x?x2xi32>> init {
+! CHECK: ^bb0(%[[ARG0:.*]]: !fir.box<!fir.array<?x?x2xi32>>):
+! CHECK:   %[[DIM0:.*]]:3 = fir.box_dims %arg0, %c0{{.*}} : (!fir.box<!fir.array<?x?x2xi32>>, index) -> (index, index, index)
+! CHECK:   %[[DIM1:.*]]:3 = fir.box_dims %arg0, %c1{{.*}} : (!fir.box<!fir.array<?x?x2xi32>>, index) -> (index, index, index)
+! CHECK:   %[[SHAPE:.*]] = fir.shape %[[DIM0]]#1, %[[DIM1]]#1, %c2{{.*}} : (index, index, index) -> !fir.shape<3>
+! CHECK:   %[[TEMP:.*]] = fir.allocmem !fir.array<?x?x2xi32>, %[[DIM0]]#1, %[[DIM1]]#1 {bindc_name = ".tmp", uniq_name = ""}
+! CHECK:   %[[DECL:.*]]:2 = hlfir.declare %[[TEMP]](%[[SHAPE]]) {uniq_name = ".tmp"} : (!fir.heap<!fir.array<?x?x2xi32>>, !fir.shape<3>) -> (!fir.box<!fir.array<?x?x2xi32>>, !fir.heap<!fir.array<?x?x2xi32>>)
+! CHECK:   acc.yield %[[DECL]]#0 : !fir.box<!fir.array<?x?x2xi32>>
+! CHECK: }
+
+! CHECK-LABEL: acc.private.recipe @privatization_box_ptr_Uxi32 : !fir.box<!fir.ptr<!fir.array<?xi32>>> init {
+! CHECK: ^bb0(%[[ARG0:.*]]: !fir.box<!fir.ptr<!fir.array<?xi32>>>):
+! CHECK:   %[[C0:.*]] = arith.constant 0 : index
+! CHECK:   %[[BOX_DIMS:.*]]:3 = fir.box_dims %arg0, %c0 : (!fir.box<!fir.ptr<!fir.array<?xi32>>>, index) -> (index, index, index)
+! CHECK:   %[[SHAPE:.*]] = fir.shape %[[BOX_DIMS]]#1 : (index) -> !fir.shape<1>
+! CHECK:   %[[TEMP:.*]] = fir.allocmem !fir.array<?xi32>, %0#1 {bindc_name = ".tmp", uniq_name = ""}
+! CHECK:   %[[DECLARE:.*]]:2 = hlfir.declare %[[TEMP]](%[[SHAPE]]) {uniq_name = ".tmp"} : (!fir.heap<!fir.array<?xi32>>, !fir.shape<1>) -> (!fir.box<!fir.array<?xi32>>, !fir.heap<!fir.array<?xi32>>)
+! CHECK:   acc.yield %[[DECLARE]]#0 : !fir.box<!fir.array<?xi32>>
+! CHECK: }
+
+! CHECK-LABEL: acc.private.recipe @privatization_box_heap_Uxi32 : !fir.box<!fir.heap<!fir.array<?xi32>>> init {
+! CHECK: ^bb0(%[[ARG0:.*]]: !fir.box<!fir.heap<!fir.array<?xi32>>>):
+! CHECK:   %[[C0:.*]] = arith.constant 0 : index
+! CHECK:   %[[BOX_DIMS:.*]]:3 = fir.box_dims %[[ARG0]], %[[C0]] : (!fir.box<!fir.heap<!fir.array<?xi32>>>, index) -> (index, index, index)
+! CHECK:   %[[SHAPE:.*]] = fir.shape %[[BOX_DIMS]]#1 : (index) -> !fir.shape<1>
+! CHECK:   %[[TEMP:.*]] = fir.allocmem !fir.array<?xi32>, %[[BOX_DIMS]]#1 {bindc_name = ".tmp", uniq_name = ""}
+! CHECK:   %[[DECLARE:.*]]:2 = hlfir.declare %[[TEMP]](%[[SHAPE]]) {uniq_name = ".tmp"} : (!fir.heap<!fir.array<?xi32>>, !fir.shape<1>) -> (!fir.box<!fir.array<?xi32>>, !fir.heap<!fir.array<?xi32>>)
+! CHECK:   acc.yield %[[DECLARE]]#0 : !fir.box<!fir.array<?xi32>>
+! CHECK: }
+
+! CHECK-LABEL: acc.private.recipe @privatization_box_Uxi32 : !fir.box<!fir.array<?xi32>> init {
+! CHECK: ^bb0(%[[ARG0:.*]]: !fir.box<!fir.array<?xi32>>):
+! CHECK:   %[[C0:.*]] = arith.constant 0 : index
+! CHECK:   %[[BOX_DIMS:.*]]:3 = fir.box_dims %[[ARG0]], %[[C0]] : (!fir.box<!fir.array<?xi32>>, index) -> (index, index, index)
+! CHECK:   %[[SHAPE:.*]] = fir.shape %[[BOX_DIMS]]#1 : (index) -> !fir.shape<1>
+! CHECK:   %[[TEMP:.*]] = fir.allocmem !fir.array<?xi32>, %0#1 {bindc_name = ".tmp", uniq_name = ""}
+! CHECK:   %[[DECLARE:.*]]:2 = hlfir.declare %[[TEMP]](%[[SHAPE]]) {uniq_name = ".tmp"} : (!fir.heap<!fir.array<?xi32>>, !fir.shape<1>) -> (!fir.box<!fir.array<?xi32>>, !fir.heap<!fir.array<?xi32>>)
+! CHECK:   acc.yield %[[DECLARE:.*]]#0 : !fir.box<!fir.array<?xi32>>
+! CHECK: }
+
+! CHECK-LABEL: acc.firstprivate.recipe @firstprivatization_section_lb50.ub99_ref_50xf32 : !fir.ref<!fir.array<50xf32>> init {
+! CHECK: ^bb0(%{{.*}}: !fir.ref<!fir.array<50xf32>>):
+! CHECK:   %[[SHAPE:.*]] = fir.shape %{{.*}} : (index) -> !fir.shape<1>
+! CHECK:   %[[ALLOCA:.*]] = fir.alloca !fir.array<50xf32>
+! CHECK:   %[[DECLARE:.*]]:2 = hlfir.declare %[[ALLOCA]](%[[SHAPE]]) {uniq_name = "acc.private.init"} : (!fir.ref<!fir.array<50xf32>>, !fir.shape<1>) -> (!fir.ref<!fir.array<50xf32>>, !fir.ref<!fir.array<50xf32>>)
+! CHECK:   acc.yield %[[DECLARE]]#0 : !fir.ref<!fir.array<50xf32>>
+! CHECK: } copy {
+! CHECK: ^bb0(%[[SRC:.*]]: !fir.ref<!fir.array<50xf32>>, %[[DST:.*]]: !fir.ref<!fir.array<50xf32>>):
+! CHECK:   %[[SHAPE:.*]] = fir.shape %{{.*}} : (index) -> !fir.shape<1>
+! CHECK:   %[[DECL_SRC:.*]]:2 = hlfir.declare %[[SRC]](%[[SHAPE]]) {uniq_name = ""} : (!fir.ref<!fir.array<50xf32>>, !fir.shape<1>) -> (!fir.ref<!fir.array<50xf32>>, !fir.ref<!fir.array<50xf32>>)
+! CHECK:   %[[DECL_DST:.*]]:2 = hlfir.declare %[[DST]](%[[SHAPE]]) {uniq_name = ""} : (!fir.ref<!fir.array<50xf32>>, !fir.shape<1>) -> (!fir.ref<!fir.array<50xf32>>, !fir.ref<!fir.array<50xf32>>)
+! CHECK:   %[[DES_SRC:.*]] = hlfir.designate %[[DECL_SRC]]#0 shape %[[SHAPE:.*]] : (!fir.ref<!fir.array<50xf32>>, !fir.shape<1>) -> !fir.ref<!fir.array<50xf32>>
+! CHECK:   %[[DES_DST:.*]] = hlfir.designate %[[DECL_DST]]#0 shape %[[SHAPE:.*]] : (!fir.ref<!fir.array<50xf32>>, !fir.shape<1>) -> !fir.ref<!fir.array<50xf32>>
+! CHECK:   hlfir.assign %[[DES_SRC]] to %[[DES_DST]] : !fir.ref<!fir.array<50xf32>>, !fir.ref<!fir.array<50xf32>>
+! CHECK: }
+
+! CHECK-LABEL: acc.firstprivate.recipe @firstprivatization_section_ext100_ref_100xf32 : !fir.ref<!fir.array<100xf32>> init {
+! CHECK: ^bb0(%{{.*}}: !fir.ref<!fir.array<100xf32>>):
+! CHECK:   %[[SHAPE:.*]] = fir.shape %{{.*}} : (index) -> !fir.shape<1>
+! CHECK:   %[[ALLOCA:.*]] = fir.alloca !fir.array<100xf32>
+! CHECK:   %[[DECLARE:.*]]:2 = hlfir.declare %[[ALLOCA]](%[[SHAPE]]) {uniq_name = "acc.private.init"} : (!fir.ref<!fir.array<100xf32>>, !fir.shape<1>) -> (!fir.ref<!fir.array<100xf32>>, !fir.ref<!fir.array<100xf32>>)
+! CHECK:   acc.yield %[[DECLARE]]#0 : !fir.ref<!fir.array<100xf32>>
+! CHECK: } copy {
+! CHECK: ^bb0(%[[SRC:.*]]: !fir.ref<!fir.array<100xf32>>, %[[DST:.*]]: !fir.ref<!fir.array<100xf32>>):
+! CHECK:   %[[SHAPE:.*]] = fir.shape %{{.*}} : (index) -> !fir.shape<1>
+! CHECK:   %[[DECL_SRC:.*]]:2 = hlfir.declare %[[SRC]](%[[SHAPE]]) {uniq_name = ""} : (!fir.ref<!fir.array<100xf32>>, !fir.shape<1>) -> (!fir.ref<!fir.array<100xf32>>, !fir.ref<!fir.array<100xf32>>)
+! CHECK:   %[[DECL_DST:.*]]:2 = hlfir.declare %[[DST]](%[[SHAPE]]) {uniq_name = ""} : (!fir.ref<!fir.array<100xf32>>, !fir.shape<1>) -> (!fir.ref<!fir.array<100xf32>>, !fir.ref<!fir.array<100xf32>>)
+! CHECK:   %[[DES_SRC:.*]] = hlfir.designate %[[DECL_SRC]]#0 shape %[[SHAPE]] : (!fir.ref<!fir.array<100xf32>>, !fir.shape<1>) -> !fir.ref<!fir.array<100xf32>>
+! CHECK:   %[[DES_DST:.*]] = hlfir.designate %[[DECL_DST]]#0 shape %[[SHAPE]] : (!fir.ref<!fir.array<100xf32>>, !fir.shape<1>) -> !fir.ref<!fir.array<100xf32>>
+! CHECK:   hlfir.assign %[[DES_SRC]] to %[[DES_DST]] : !fir.ref<!fir.array<100xf32>>, !fir.ref<!fir.array<100xf32>>
+! CHECK:   acc.terminator
+! CHECK: }
+
+! CHECK-LABEL: acc.firstprivate.recipe @firstprivatization_ref_i32 : !fir.ref<i32> init {
+! CHECK: ^bb0(%{{.*}}: !fir.ref<i32>):
+! CHECK:   %[[ALLOCA:.*]] = fir.alloca i32
+! CHECK:   %[[DECLARE:.*]]:2 = hlfir.declare %[[ALLOCA]] {uniq_name = "acc.private.init"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>) 
+! CHECK:   acc.yield %[[DECLARE]]#0 : !fir.ref<i32> 
+! CHECK: } copy {
+! CHECK: ^bb0(%[[SRC:.*]]: !fir.ref<i32>, %[[DST:.*]]: !fir.ref<i32>):
+! CHECK:   %[[VALUE:.*]] = fir.load %[[SRC]] : !fir.ref<i32>
+! CHECK:   fir.store %[[VALUE]] to %[[DST]] : !fir.ref<i32>
+! CHECK:   acc.terminator
+! CHECK: }
+
+! CHECK-LABEL: acc.private.recipe @privatization_ref_50xf32 : !fir.ref<!fir.array<50xf32>> init {
+! CHECK: ^bb0(%{{.*}}: !fir.ref<!fir.array<50xf32>>):
+! CHECK:   %[[SHAPE:.*]] = fir.shape %{{.*}} : (index) -> !fir.shape<1>
+! CHECK:   %[[ALLOCA:.*]] = fir.alloca !fir.array<50xf32>
+! CHECK:   %[[DECLARE:.*]]:2 = hlfir.declare %[[ALLOCA]](%[[SHAPE]]) {uniq_name = "acc.private.init"} : (!fir.ref<!fir.array<50xf32>>, !fir.shape<1>) -> (!fir.ref<!fir.array<50xf32>>, !fir.ref<!fir.array<50xf32>>)
+! CHECK:   acc.yield %[[DECLARE]]#0 : !fir.ref<!fir.array<50xf32>>
+! CHECK: }
+
+! CHECK-LABEL: acc.private.recipe @privatization_ref_100xf32 : !fir.ref<!fir.array<100xf32>> init {
+! CHECK: ^bb0(%{{.*}}: !fir.ref<!fir.array<100xf32>>):
+! CHECK:   %[[SHAPE:.*]] = fir.shape %{{.*}} : (index) -> !fir.shape<1>
+! CHECK:   %[[ALLOCA:.*]] = fir.alloca !fir.array<100xf32>
+! CHECK:   %[[DECLARE:.*]]:2 = hlfir.declare %[[ALLOCA]](%[[SHAPE]]) {uniq_name = "acc.private.init"} : (!fir.ref<!fir.array<100xf32>>, !fir.shape<1>) -> (!fir.ref<!fir.array<100xf32>>, !fir.ref<!fir.array<100xf32>>)
+! CHECK:   acc.yield %[[DECLARE]]#0 : !fir.ref<!fir.array<100xf32>>
+! CHECK: }
+
+! CHECK-LABEL: acc.private.recipe @privatization_ref_i32 : !fir.ref<i32> init {
+! CHECK: ^bb0(%{{.*}}: !fir.ref<i32>):
+! CHECK:   %[[ALLOCA:.*]] = fir.alloca i32
+! CHECK:   %[[DECLARE:.*]]:2 = hlfir.declare %[[ALLOCA]] {uniq_name = "acc.private.init"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>) 
+! CHECK:   acc.yield %[[DECLARE]]#0 : !fir.ref<i32> 
+! CHECK: }
+
+program acc_private
+  integer :: i, c
+  integer, parameter :: n = 100
+  real, dimension(n) :: a, b
+
+! CHECK: %[[B:.*]] = fir.address_of(@_QFEb) : !fir.ref<!fir.array<100xf32>>
+! CHECK: %[[DECLB:.*]]:2 = hlfir.declare %[[B]]
+! CHECK: %[[C:.*]] = fir.alloca i32 {bindc_name = "c", uniq_name = "_QFEc"}
+! CHECK: %[[DECLC:.*]]:2 = hlfir.declare %[[C]]
+
+  !$acc loop private(c)
+  DO i = 1, n
+    c = i
+    a(i) = b(i) + c
+  END DO
+
+! CHECK: %[[C_PRIVATE:.*]] = acc.private varPtr(%[[DECLC]]#0 : !fir.ref<i32>) -> !fir.ref<i32> {name = "c"}
+! CHECK: acc.loop private({{.*}}@privatization_ref_i32 -> %[[C_PRIVATE]] : !fir.ref<i32>)
+! CHECK: acc.yield
+
+  !$acc loop private(b)
+  DO i = 1, n
+    c = i
+    a(i) = b(i) + c
+  END DO
+
+! CHECK: %[[C1:.*]] = arith.constant 1 : index
+! CHECK: %[[LB:.*]] = arith.constant 0 : index
+! CHECK: %[[UB:.*]] = arith.subi %{{.*}}, %[[C1]] : index
+! CHECK: %[[BOUND:.*]] = acc.bounds lowerbound(%[[LB]] : index) upperbound(%[[UB]] : index) extent(%{{.*}} : index) stride(%[[C1]] : index) startIdx(%[[C1]] : index)
+! CHECK: %[[B_PRIVATE:.*]] = acc.private varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<100xf32>>) bounds(%[[BOUND]]) -> !fir.ref<!fir.array<100xf32>> {name = "b"}
+! CHECK: acc.loop private({{.*}}@privatization_ref_100xf32 -> %[[B_PRIVATE]] : !fir.ref<!fir.array<100xf32>>)
+! CHECK: acc.yield
+
+  !$acc loop private(b(1:50))
+  DO i = 1, n
+    c = i
+    a(i) = b(i) + c
+  END DO
+
+! CHECK: %[[C1:.*]] = arith.constant 1 : index
+! CHECK: %[[LB:.*]] = arith.constant 0 : index
+! CHECK: %[[UB:.*]] = arith.constant 49 : index
+! CHECK: %[[BOUND:.*]] = acc.bounds lowerbound(%[[LB]] : index) upperbound(%[[UB]] : index) extent(%{{.*}} : index) stride(%[[C1]] : index) startIdx(%[[C1]] : index)
+! CHECK: %[[B_PRIVATE:.*]] = acc.private varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<100xf32>>) bounds(%[[BOUND]]) -> !fir.ref<!fir.array<50xf32>> {name = "b(1:50)"}
+! CHECK: acc.loop private({{.*}}@privatization_ref_50xf32 -> %[[B_PRIVATE]] : !fir.ref<!fir.array<50xf32>>)
+
+  !$acc parallel loop firstprivate(c)
+  DO i = 1, n
+    c = i
+    a(i) = b(i) + c
+  END DO
+
+! CHECK: %[[FP_C:.*]] = acc.firstprivate varPtr(%[[DECLC]]#0 : !fir.ref<i32>)   -> !fir.ref<i32> {name = "c"}
+! CHECK: acc.parallel {{.*}} firstprivate(@firstprivatization_ref_i32 -> %[[FP_C]] : !fir.ref<i32>)
+! CHECK: acc.yield
+
+  !$acc parallel loop firstprivate(b)
+  DO i = 1, n
+    c = i
+    a(i) = b(i) + c
+  END DO
+
+! CHECK: %[[C1:.*]] = arith.constant 1 : index
+! CHECK: %[[LB:.*]] = arith.constant 0 : index
+! CHECK: %[[UB:.*]] = arith.subi %{{.*}}, %[[C1]] : index
+! CHECK: %[[BOUND:.*]] = acc.bounds lowerbound(%[[LB]] : index) upperbound(%[[UB]] : index) extent(%{{.*}} : index) stride(%[[C1]] : index) startIdx(%[[C1]] : index)
+! CHECK: %[[FP_B:.*]] = acc.firstprivate varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<100xf32>>) bounds(%[[BOUND]]) -> !fir.ref<!fir.array<100xf32>> {name = "b"}
+! CHECK: acc.parallel {{.*}} firstprivate(@firstprivatization_section_ext100_ref_100xf32 -> %[[FP_B]] : !fir.ref<!fir.array<100xf32>>)
+! CHECK: acc.yield
+
+  !$acc parallel loop firstprivate(b(51:100))
+  DO i = 1, n
+    c = i
+    a(i) = b(i) + c
+  END DO
+
+! CHECK: %[[C1:.*]] = arith.constant 1 : index
+! CHECK: %[[LB:.*]] = arith.constant 50 : index
+! CHECK: %[[UB:.*]] = arith.constant 99 : index
+! CHECK: %[[BOUND:.*]] = acc.bounds lowerbound(%[[LB]] : index) upperbound(%[[UB]] : index) extent(%{{.*}} : index) stride(%[[C1]] : index) startIdx(%[[C1]] : index)
+! CHECK: %[[FP_B:.*]] = acc.firstprivate varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<100xf32>>) bounds(%[[BOUND]]) -> !fir.ref<!fir.array<50xf32>> {name = "b(51:100)"}
+! CHECK: acc.parallel {{.*}} firstprivate(@firstprivatization_section_lb50.ub99_ref_50xf32 -> %[[FP_B]] : !fir.ref<!fir.array<50xf32>>)
+
+end program
+
+subroutine acc_private_assumed_shape(a, n)
+  integer :: a(:), i, n
+
+  !$acc parallel loop private(a)
+  do i = 1, n
+    a(i) = i
+  end do
+end subroutine
+
+! CHECK-LABEL: func.func @_QPacc_private_assumed_shape(
+! CHECK-SAME:    %[[ARG0:.*]]: !fir.box<!fir.array<?xi32>> {fir.bindc_name = "a"}
+! CHECK: %[[DECL_A:.*]]:2 = hlfir.declare %[[ARG0]] dummy_scope %{{[0-9]+}} {uniq_name = "_QFacc_private_assumed_shapeEa"} : (!fir.box<!fir.array<?xi32>>, !fir.dscope) -> (!fir.box<!fir.array<?xi32>>, !fir.box<!fir.array<?xi32>>)
+! CHECK: acc.parallel {{.*}} {
+! CHECK: %[[ADDR:.*]] = fir.box_addr %[[DECL_A]]#0 : (!fir.box<!fir.array<?xi32>>) -> !fir.ref<!fir.array<?xi32>>
+! CHECK: %[[PRIVATE:.*]] = acc.private varPtr(%[[ADDR]] : !fir.ref<!fir.array<?xi32>>) bounds(%{{.*}}) -> !fir.ref<!fir.array<?xi32>> {name = "a"}
+! CHECK: acc.loop {{.*}} private({{.*}}@privatization_box_Uxi32 -> %[[PRIVATE]] : !fir.ref<!fir.array<?xi32>>)
+
+subroutine acc_private_allocatable_array(a, n)
+  integer, allocatable :: a(:)
+  integer :: i, n
+
+  !$acc parallel loop private(a)
+  do i = 1, n
+    a(i) = i
+  end do
+
+  !$acc serial private(a)
+  a(i) = 1
+  !$acc end serial
+end subroutine
+
+! CHECK-LABEL: func.func @_QPacc_private_allocatable_array(
+! CHECK-SAME: %[[ARG0:.*]]: !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>> {fir.bindc_name = "a"}
+! CHECK: %[[DECLA_A:.*]]:2 = hlfir.declare %[[ARG0]] dummy_scope %{{[0-9]+}} {fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "_QFacc_private_allocatable_arrayEa"} : (!fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>, !fir.dscope) -> (!fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>, !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>)
+! CHECK: acc.parallel {{.*}} {
+! CHECK: %[[BOX:.*]] = fir.load %[[DECLA_A]]#0 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>
+! CHECK: %[[BOX_ADDR:.*]] = fir.box_addr %[[BOX]] : (!fir.box<!fir.heap<!fir.array<?xi32>>>) -> !fir.heap<!fir.array<?xi32>>
+! CHECK: %[[PRIVATE:.*]] = acc.private varPtr(%[[BOX_ADDR]] : !fir.heap<!fir.array<?xi32>>) bounds(%{{.*}}) -> !fir.heap<!fir.array<?xi32>> {name = "a"}
+! CHECK: acc.loop {{.*}} private({{.*}}@privatization_box_heap_Uxi32 -> %[[PRIVATE]] : !fir.heap<!fir.array<?xi32>>)
+! CHECK: acc.serial private(@privatization_box_heap_Uxi32 -> %{{.*}} : !fir.heap<!fir.array<?xi32>>)
+
+subroutine acc_private_pointer_array(a, n)
+  integer, pointer :: a(:)
+  integer :: i, n
+
+  !$acc parallel loop private(a)
+  do i = 1, n
+    a(i) = i
+  end do
+end subroutine
+
+! CHECK-LABEL: func.func @_QPacc_private_pointer_array(
+! CHECK-SAME: %[[ARG0:.*]]: !fir.ref<!fir.box<!fir.ptr<!fir.array<?xi32>>>> {fir.bindc_name = "a"}, %arg1: !fir.ref<i32> {fir.bindc_name = "n"}) {
+! CHECK: %[[DECL_A:.*]]:2 = hlfir.declare %arg0 dummy_scope %{{[0-9]+}} {fortran_attrs = #fir.var_attrs<pointer>, uniq_name = "_QFacc_private_pointer_arrayEa"} : (!fir.ref<!fir.box<!fir.ptr<!fir.array<?xi32>>>>, !fir.dscope) -> (!fir.ref<!fir.box<!fir.ptr<!fir.array<?xi32>>>>, !fir.ref<!fir.box<!fir.ptr<!fir.array<?xi32>>>>)
+! CHECK: acc.parallel {{.*}} {
+! CHECK: %[[BOX:.*]] = fir.load %[[DECLA_A]]#0 : !fir.ref<!fir.box<!fir.ptr<!fir.array<?xi32>>>>
+! CHECK: %[[BOX_ADDR:.*]] = fir.box_addr %[[BOX]] : (!fir.box<!fir.ptr<!fir.array<?xi32>>>) -> !fir.ptr<!fir.array<?xi32>>
+! CHECK: %[[PRIVATE:.*]] = acc.private varPtr(%[[BOX_ADDR]] : !fir.ptr<!fir.array<?xi32>>) bounds(%{{.*}}) -> !fir.ptr<!fir.array<?xi32>> {name = "a"}
+! CHECK: acc.loop {{.*}} private({{.*}}@privatization_box_ptr_Uxi32 -> %[[PRIVATE]] : !fir.ptr<!fir.array<?xi32>>)
+
+subroutine acc_private_dynamic_extent(a, n)
+  integer :: n, i
+  integer :: a(n, n, 2)
+
+  !$acc parallel loop private(a)
+  do i = 1, n
+    a(i, i, 1) = i
+  end do
+end subroutine
+
+! CHECK-LABEL: func.func @_QPacc_private_dynamic_extent(
+! CHECK-SAME: %[[ARG0:.*]]: !fir.ref<!fir.array<?x?x2xi32>> {fir.bindc_name = "a"}, %[[ARG1:.*]]: !fir.ref<i32> {fir.bindc_name = "n"}) {
+! CHECK: %[[DECL_N:.*]]:2 = hlfir.declare %[[ARG1]] dummy_scope %{{[0-9]+}} {uniq_name = "_QFacc_private_dynamic_extentEn"} : (!fir.ref<i32>, !fir.dscope) -> (!fir.ref<i32>, !fir.ref<i32>)
+! CHECK: %[[DECL_A:.*]]:2 = hlfir.declare %[[ARG0]](%{{.*}}) dummy_scope %{{[0-9]+}} {uniq_name = "_QFacc_private_dynamic_extentEa"} : (!fir.ref<!fir.array<?x?x2xi32>>, !fir.shape<3>, !fir.dscope) -> (!fir.box<!fir.array<?x?x2xi32>>, !fir.ref<!fir.array<?x?x2xi32>>)
+! CHECK: acc.parallel {{.*}} {
+! CHECK: %[[BOX_ADDR:.*]] = fir.box_addr %[[DECL_A]]#0 : (!fir.box<!fir.array<?x?x2xi32>>) -> !fir.ref<!fir.array<?x?x2xi32>>
+! CHECK: %[[PRIV:.*]] = acc.private varPtr(%[[BOX_ADDR]] : !fir.ref<!fir.array<?x?x2xi32>>) bounds(%{{.*}}, %{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<?x?x2xi32>> {name = "a"}
+! CHECK: acc.loop {{.*}} private({{.*}}@privatization_box_UxUx2xi32 -> %[[PRIV]] : !fir.ref<!fir.array<?x?x2xi32>>)
+
+subroutine acc_firstprivate_assumed_shape(a, n)
+  integer :: a(:), i, n
+
+  !$acc parallel loop firstprivate(a)
+  do i = 1, n
+    a(i) = i
+  end do
+end subroutine
+
+subroutine acc_firstprivate_assumed_shape_with_section(a, n)
+  integer :: a(:), i, n
+
+  !$acc parallel loop firstprivate(a(5:10))
+  do i = 1, n
+    a(i) = i
+  end do
+end subroutine
+
+subroutine acc_firstprivate_dynamic_extent(a, n)
+  integer :: n, i
+  integer :: a(n, n, 2)
+
+  !$acc parallel loop firstprivate(a)
+  do i = 1, n
+    a(i, i, 1) = i
+  end do
+end subroutine
+
+! CHECK: acc.parallel {{.*}} firstprivate(@firstprivatization_box_UxUx2xi32 -> %{{.*}} : !fir.ref<!fir.array<?x?x2xi32>>)
+
+module acc_declare_equivalent
+  integer, parameter :: n = 10
+  real :: v1(n)
+  real :: v2(n)
+  equivalence(v1(1), v2(1))
+contains
+  subroutine sub1()
+    !$acc parallel private(v2)
+    !$acc end parallel
+  end subroutine
+end module
+
+! CHECK: acc.parallel private(@privatization_ref_10xf32 -> %{{.*}} : !fir.ref<!fir.array<10xf32>>)
+
+subroutine acc_private_use()
+  integer :: i, j
+
+  !$acc parallel loop
+  do i = 1, 10
+    j = i
+  end do
+end
+
+! CHECK-LABEL: func.func @_QPacc_private_use()
+! CHECK: %[[I:.*]] = fir.alloca i32 {bindc_name = "i", uniq_name = "_QFacc_private_useEi"}
+! CHECK: %[[DECL_I:.*]]:2 = hlfir.declare %0 {uniq_name = "_QFacc_private_useEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+! CHECK: acc.parallel
+! CHECK: %[[PRIV_I:.*]] = acc.private varPtr(%[[DECL_I]]#1 : !fir.ref<i32>) -> !fir.ref<i32> {implicit = true, name = "i"}
+! CHECK: %[[DECL_PRIV_I:.*]]:2 = hlfir.declare %[[PRIV_I]] {uniq_name = "_QFacc_private_useEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+! CHECK: acc.loop {{.*}} private(@privatization_ref_i32 -> %[[PRIV_I]] : !fir.ref<i32>) control(%[[IV0:.*]] : i32) = (%c1{{.*}} : i32) to (%c10{{.*}} : i32) step (%c1{{.*}} : i32)
+! CHECK:   fir.store %[[IV0]] to %[[DECL_PRIV_I]]#0 : !fir.ref<i32>
+! CHECK:   %{{.*}} = fir.load %[[DECL_PRIV_I]]#0 : !fir.ref<i32>
diff --git a/flang/test/Lower/OpenACC/acc-private.f90 b/flang/test/Lower/OpenACC/acc-private.f90
index a299a7486c3ba..c86da8c001b55 100644
--- a/flang/test/Lower/OpenACC/acc-private.f90
+++ b/flang/test/Lower/OpenACC/acc-private.f90
@@ -2,8 +2,8 @@
 
 ! RUN: bbc -fopenacc -emit-hlfir %s -o - | FileCheck %s
 
-! CHECK-LABEL: acc.private.recipe @privatization_ref_10xf32 : !fir.ref<!fir.array<10xf32>> init {
-! CHECK:   ^bb0(%[[ARG0:.*]]: !fir.ref<!fir.array<10xf32>>):
+! CHECK-LABEL: acc.private.recipe @privatization_ptr_10xf32 : !fir.ptr<!fir.array<10xf32>> init {
+! CHECK:   ^bb0(%[[ARG0:.*]]: !fir.ptr<!fir.array<10xf32>>):
 ! CHECK:   %[[C10:.*]] = arith.constant 10 : index
 ! CHECK:   %[[SHAPE:.*]] = fir.shape %[[C10]] : (index) -> !fir.shape<1>
 ! CHECK:   %[[ALLOCA:.*]] = fir.alloca !fir.array<10xf32>
@@ -20,10 +20,10 @@
 ! CHECK:   %[[DECL:.*]]:2 = hlfir.declare %[[TEMP]](%[[SHAPE]]) {uniq_name = ".tmp"} : (!fir.heap<!fir.array<?x?x2xi32>>, !fir.shape<3>) -> (!fir.box<!fir.array<?x?x2xi32>>, !fir.heap<!fir.array<?x?x2xi32>>)
 ! CHECK:   acc.yield %[[DECL]]#0 : !fir.box<!fir.array<?x?x2xi32>>
 ! CHECK: } copy {
-! CHECK: ^bb0(%[[ARG0:.*]]: !fir.box<!fir.array<?x?x2xi32>>, %[[ARG1:.*]]: !fir.box<!fir.array<?x?x2xi32>>, %[[LB0:.*]]: index, %[[UB0:.*]]: index, %[[STEP0:.*]]: index, %[[LB1:.*]]: index, %[[UB1:.*]]: index, %[[STEP1:.*]]: index, %[[LB2:.*]]: index, %[[UB2:.*]]: index, %[[STEP2:.*]]: index):
+! CHECK: ^bb0(%[[ARG0:.*]]: !fir.box<!fir.array<?x?x2xi32>>, %[[ARG1:.*]]: !fir.box<!fir.array<?x?x2xi32>>):
 ! CHECK:   %[[SHAPE:.*]] = fir.shape %{{.*}}, %{{.*}}, %{{.*}} : (index, index, index) -> !fir.shape<3>
-! CHECK:   %[[DES_SRC:.*]] = hlfir.designate %[[ARG0]] (%[[LB0]]:%[[UB0]]:%[[STEP0]], %[[LB1]]:%[[UB1]]:%[[STEP1]], %[[LB2]]:%[[UB2]]:%[[STEP2]]) shape %[[SHAPE]] : (!fir.box<!fir.array<?x?x2xi32>>, index, index, index, index, index, index, index, index, index, !fir.shape<3>) -> !fir.box<!fir.array<?x?x2xi32>>
-! CHECK:   %[[DES_DST:.*]] = hlfir.designate %[[ARG1]] (%[[LB0]]:%[[UB0]]:%[[STEP0]], %[[LB1]]:%[[UB1]]:%[[STEP1]], %[[LB2]]:%[[UB2]]:%[[STEP2]]) shape %[[SHAPE]] : (!fir.box<!fir.array<?x?x2xi32>>, index, index, index, index, index, index, index, index, index, !fir.shape<3>) -> !fir.box<!fir.array<?x?x2xi32>>
+! CHECK:   %[[DES_SRC:.*]] = hlfir.designate %[[ARG0]] shape %[[SHAPE]] : (!fir.box<!fir.array<?x?x2xi32>>, !fir.shape<3>) -> !fir.box<!fir.array<?x?x2xi32>>
+! CHECK:   %[[DES_DST:.*]] = hlfir.designate %[[ARG1]] shape %[[SHAPE]] : (!fir.box<!fir.array<?x?x2xi32>>, !fir.shape<3>) -> !fir.box<!fir.array<?x?x2xi32>>
 ! CHECK:   hlfir.assign %[[DES_SRC]] to %[[DES_DST]] : !fir.box<!fir.array<?x?x2xi32>>, !fir.box<!fir.array<?x?x2xi32>>
 ! CHECK:   acc.terminator
 ! CHECK: }
@@ -58,10 +58,10 @@
 ! CHECK:   %[[DECL:.*]]:2 = hlfir.declare %[[TEMP]](%[[SHAPE]]) {uniq_name = ".tmp"} : (!fir.heap<!fir.array<?xi32>>, !fir.shape<1>) -> (!fir.box<!fir.array<?xi32>>, !fir.heap<!fir.array<?xi32>>)
 ! CHECK:   acc.yield %[[DECL]]#0 : !fir.box<!fir.array<?xi32>>
 ! CHECK: } copy {
-! CHECK: ^bb0(%[[ARG0:.*]]: !fir.box<!fir.array<?xi32>>, %[[ARG1:.*]]: !fir.box<!fir.array<?xi32>>, %[[ARG2:.*]]: index, %[[ARG3:.*]]: index, %[[ARG4:.*]]: index):
+! CHECK: ^bb0(%[[ARG0:.*]]: !fir.box<!fir.array<?xi32>>, %[[ARG1:.*]]: !fir.box<!fir.array<?xi32>>):
 ! CHECK:   %[[SHAPE:.*]] = fir.shape %{{.*}} : (index) -> !fir.shape<1>
-! CHECK:   %[[DES_V1:.*]] = hlfir.designate %[[ARG0]] (%{{.*}}:%{{.*}}:%{{.*}}) shape %[[SHAPE]] : (!fir.box<!fir.array<?xi32>>, index, index, index, !fir.shape<1>) -> !fir.box<!fir.array<?xi32>>
-! CHECK:   %[[DES_V2:.*]] = hlfir.designate %[[ARG1]] (%{{.*}}:%{{.*}}:%{{.*}}) shape %[[SHAPE]] : (!fir.box<!fir.array<?xi32>>, index, index, index, !fir.shape<1>) -> !fir.box<!fir.array<?xi32>>
+! CHECK:   %[[DES_V1:.*]] = hlfir.designate %[[ARG0]] shape %[[SHAPE]] : (!fir.box<!fir.array<?xi32>>, !fir.shape<1>) -> !fir.box<!fir.array<?xi32>>
+! CHECK:   %[[DES_V2:.*]] = hlfir.designate %[[ARG1]] shape %[[SHAPE]] : (!fir.box<!fir.array<?xi32>>, !fir.shape<1>) -> !fir.box<!fir.array<?xi32>>
 ! CHECK:   hlfir.assign %[[DES_V1]] to %[[DES_V2]] : !fir.box<!fir.array<?xi32>>, !fir.box<!fir.array<?xi32>>
 ! CHECK:   acc.terminator
 ! CHECK: }
@@ -76,20 +76,22 @@
 ! CHECK:   acc.yield %[[DECL]]#0 : !fir.box<!fir.array<?x?x2xi32>>
 ! CHECK: }
 
-! CHECK-LABEL: acc.private.recipe @privatization_box_ptr_Uxi32 : !fir.box<!fir.ptr<!fir.array<?xi32>>> init {
-! CHECK: ^bb0(%[[ARG0:.*]]: !fir.box<!fir.ptr<!fir.array<?xi32>>>):
+! CHECK-LABEL: acc.private.recipe @privatization_ref_box_ptr_Uxi32 : !fir.ref<!fir.box<!fir.ptr<!fir.array<?xi32>>>> init {
+! CHECK: ^bb0(%[[ARG0:.*]]: !fir.ref<!fir.box<!fir.ptr<!fir.array<?xi32>>>>):
+! CHECK:   %[[LOADBOX:.*]] = fir.load %[[ARG0]] : !fir.ref<!fir.box<!fir.ptr<!fir.array<?xi32>>>>
 ! CHECK:   %[[C0:.*]] = arith.constant 0 : index
-! CHECK:   %[[BOX_DIMS:.*]]:3 = fir.box_dims %arg0, %c0 : (!fir.box<!fir.ptr<!fir.array<?xi32>>>, index) -> (index, index, index)
+! CHECK:   %[[BOX_DIMS:.*]]:3 = fir.box_dims %[[LOADBOX]], %c0 : (!fir.box<!fir.ptr<!fir.array<?xi32>>>, index) -> (index, index, index)
 ! CHECK:   %[[SHAPE:.*]] = fir.shape %[[BOX_DIMS]]#1 : (index) -> !fir.shape<1>
-! CHECK:   %[[TEMP:.*]] = fir.allocmem !fir.array<?xi32>, %0#1 {bindc_name = ".tmp", uniq_name = ""}
+! CHECK:   %[[TEMP:.*]] = fir.allocmem !fir.array<?xi32>, %[[BOX_DIMS]]#1 {bindc_name = ".tmp", uniq_name = ""}
 ! CHECK:   %[[DECLARE:.*]]:2 = hlfir.declare %[[TEMP]](%[[SHAPE]]) {uniq_name = ".tmp"} : (!fir.heap<!fir.array<?xi32>>, !fir.shape<1>) -> (!fir.box<!fir.array<?xi32>>, !fir.heap<!fir.array<?xi32>>)
 ! CHECK:   acc.yield %[[DECLARE]]#0 : !fir.box<!fir.array<?xi32>>
 ! CHECK: }
 
-! CHECK-LABEL: acc.private.recipe @privatization_box_heap_Uxi32 : !fir.box<!fir.heap<!fir.array<?xi32>>> init {
-! CHECK: ^bb0(%[[ARG0:.*]]: !fir.box<!fir.heap<!fir.array<?xi32>>>):
+! CHECK-LABEL: acc.private.recipe @privatization_ref_box_heap_Uxi32 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>> init {
+! CHECK: ^bb0(%[[ARG0:.*]]: !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>):
+! CHECK:   %[[LOADBOX:.*]] = fir.load %[[ARG0]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>
 ! CHECK:   %[[C0:.*]] = arith.constant 0 : index
-! CHECK:   %[[BOX_DIMS:.*]]:3 = fir.box_dims %[[ARG0]], %[[C0]] : (!fir.box<!fir.heap<!fir.array<?xi32>>>, index) -> (index, index, index)
+! CHECK:   %[[BOX_DIMS:.*]]:3 = fir.box_dims %[[LOADBOX]], %[[C0]] : (!fir.box<!fir.heap<!fir.array<?xi32>>>, index) -> (index, index, index)
 ! CHECK:   %[[SHAPE:.*]] = fir.shape %[[BOX_DIMS]]#1 : (index) -> !fir.shape<1>
 ! CHECK:   %[[TEMP:.*]] = fir.allocmem !fir.array<?xi32>, %[[BOX_DIMS]]#1 {bindc_name = ".tmp", uniq_name = ""}
 ! CHECK:   %[[DECLARE:.*]]:2 = hlfir.declare %[[TEMP]](%[[SHAPE]]) {uniq_name = ".tmp"} : (!fir.heap<!fir.array<?xi32>>, !fir.shape<1>) -> (!fir.box<!fir.array<?xi32>>, !fir.heap<!fir.array<?xi32>>)
@@ -122,7 +124,7 @@
 ! CHECK:   hlfir.assign %[[DES_SRC]] to %[[DES_DST]] : !fir.ref<!fir.array<50xf32>>, !fir.ref<!fir.array<50xf32>>
 ! CHECK: }
 
-! CHECK-LABEL: acc.firstprivate.recipe @firstprivatization_section_ext100_ref_100xf32 : !fir.ref<!fir.array<100xf32>> init {
+! CHECK-LABEL: acc.firstprivate.recipe @firstprivatization_ref_100xf32 : !fir.ref<!fir.array<100xf32>> init {
 ! CHECK: ^bb0(%{{.*}}: !fir.ref<!fir.array<100xf32>>):
 ! CHECK:   %[[SHAPE:.*]] = fir.shape %{{.*}} : (index) -> !fir.shape<1>
 ! CHECK:   %[[ALLOCA:.*]] = fir.alloca !fir.array<100xf32>
@@ -200,11 +202,7 @@ program acc_private
     a(i) = b(i) + c
   END DO
 
-! CHECK: %[[C1:.*]] = arith.constant 1 : index
-! CHECK: %[[LB:.*]] = arith.constant 0 : index
-! CHECK: %[[UB:.*]] = arith.subi %{{.*}}, %[[C1]] : index
-! CHECK: %[[BOUND:.*]] = acc.bounds lowerbound(%[[LB]] : index) upperbound(%[[UB]] : index) extent(%{{.*}} : index) stride(%[[C1]] : index) startIdx(%[[C1]] : index)
-! CHECK: %[[B_PRIVATE:.*]] = acc.private varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<100xf32>>) bounds(%[[BOUND]]) -> !fir.ref<!fir.array<100xf32>> {name = "b"}
+! CHECK: %[[B_PRIVATE:.*]] = acc.private varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<100xf32>>) -> !fir.ref<!fir.array<100xf32>> {name = "b"}
 ! CHECK: acc.loop private({{.*}}@privatization_ref_100xf32 -> %[[B_PRIVATE]] : !fir.ref<!fir.array<100xf32>>)
 ! CHECK: acc.yield
 
@@ -237,12 +235,8 @@ program acc_private
     a(i) = b(i) + c
   END DO
 
-! CHECK: %[[C1:.*]] = arith.constant 1 : index
-! CHECK: %[[LB:.*]] = arith.constant 0 : index
-! CHECK: %[[UB:.*]] = arith.subi %{{.*}}, %[[C1]] : index
-! CHECK: %[[BOUND:.*]] = acc.bounds lowerbound(%[[LB]] : index) upperbound(%[[UB]] : index) extent(%{{.*}} : index) stride(%[[C1]] : index) startIdx(%[[C1]] : index)
-! CHECK: %[[FP_B:.*]] = acc.firstprivate varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<100xf32>>) bounds(%[[BOUND]]) -> !fir.ref<!fir.array<100xf32>> {name = "b"}
-! CHECK: acc.parallel {{.*}} firstprivate(@firstprivatization_section_ext100_ref_100xf32 -> %[[FP_B]] : !fir.ref<!fir.array<100xf32>>)
+! CHECK: %[[FP_B:.*]] = acc.firstprivate varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<100xf32>>) -> !fir.ref<!fir.array<100xf32>> {name = "b"}
+! CHECK: acc.parallel {{.*}} firstprivate(@firstprivatization_ref_100xf32 -> %[[FP_B]] : !fir.ref<!fir.array<100xf32>>)
 ! CHECK: acc.yield
 
   !$acc parallel loop firstprivate(b(51:100))
@@ -273,9 +267,8 @@ subroutine acc_private_assumed_shape(a, n)
 ! CHECK-SAME:    %[[ARG0:.*]]: !fir.box<!fir.array<?xi32>> {fir.bindc_name = "a"}
 ! CHECK: %[[DECL_A:.*]]:2 = hlfir.declare %[[ARG0]] dummy_scope %{{[0-9]+}} {uniq_name = "_QFacc_private_assumed_shapeEa"} : (!fir.box<!fir.array<?xi32>>, !fir.dscope) -> (!fir.box<!fir.array<?xi32>>, !fir.box<!fir.array<?xi32>>)
 ! CHECK: acc.parallel {{.*}} {
-! CHECK: %[[ADDR:.*]] = fir.box_addr %[[DECL_A]]#0 : (!fir.box<!fir.array<?xi32>>) -> !fir.ref<!fir.array<?xi32>>
-! CHECK: %[[PRIVATE:.*]] = acc.private varPtr(%[[ADDR]] : !fir.ref<!fir.array<?xi32>>) bounds(%{{.*}}) -> !fir.ref<!fir.array<?xi32>> {name = "a"}
-! CHECK: acc.loop {{.*}} private({{.*}}@privatization_box_Uxi32 -> %[[PRIVATE]] : !fir.ref<!fir.array<?xi32>>)
+! CHECK: %[[PRIVATE:.*]] = acc.private var(%[[DECL_A]]#0 : !fir.box<!fir.array<?xi32>>) -> !fir.box<!fir.array<?xi32>> {name = "a"}
+! CHECK: acc.loop {{.*}} private({{.*}}@privatization_box_Uxi32 -> %[[PRIVATE]] : !fir.box<!fir.array<?xi32>>)
 
 subroutine acc_private_allocatable_array(a, n)
   integer, allocatable :: a(:)
@@ -295,11 +288,9 @@ subroutine acc_private_allocatable_array(a, n)
 ! CHECK-SAME: %[[ARG0:.*]]: !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>> {fir.bindc_name = "a"}
 ! CHECK: %[[DECLA_A:.*]]:2 = hlfir.declare %[[ARG0]] dummy_scope %{{[0-9]+}} {fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "_QFacc_private_allocatable_arrayEa"} : (!fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>, !fir.dscope) -> (!fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>, !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>)
 ! CHECK: acc.parallel {{.*}} {
-! CHECK: %[[BOX:.*]] = fir.load %[[DECLA_A]]#0 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>
-! CHECK: %[[BOX_ADDR:.*]] = fir.box_addr %[[BOX]] : (!fir.box<!fir.heap<!fir.array<?xi32>>>) -> !fir.heap<!fir.array<?xi32>>
-! CHECK: %[[PRIVATE:.*]] = acc.private varPtr(%[[BOX_ADDR]] : !fir.heap<!fir.array<?xi32>>) bounds(%{{.*}}) -> !fir.heap<!fir.array<?xi32>> {name = "a"}
-! CHECK: acc.loop {{.*}} private({{.*}}@privatization_box_heap_Uxi32 -> %[[PRIVATE]] : !fir.heap<!fir.array<?xi32>>)
-! CHECK: acc.serial private(@privatization_box_heap_Uxi32 -> %{{.*}} : !fir.heap<!fir.array<?xi32>>)
+! CHECK: %[[PRIVATE:.*]] = acc.private varPtr(%[[DECLA_A]]#0 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>) -> !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>> {name = "a"}
+! CHECK: acc.loop {{.*}} private({{.*}}@privatization_ref_box_heap_Uxi32 -> %[[PRIVATE]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>)
+! CHECK: acc.serial private(@privatization_ref_box_heap_Uxi32 -> %{{.*}} : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>)
 
 subroutine acc_private_pointer_array(a, n)
   integer, pointer :: a(:)
@@ -315,10 +306,8 @@ subroutine acc_private_pointer_array(a, n)
 ! CHECK-SAME: %[[ARG0:.*]]: !fir.ref<!fir.box<!fir.ptr<!fir.array<?xi32>>>> {fir.bindc_name = "a"}, %arg1: !fir.ref<i32> {fir.bindc_name = "n"}) {
 ! CHECK: %[[DECL_A:.*]]:2 = hlfir.declare %arg0 dummy_scope %{{[0-9]+}} {fortran_attrs = #fir.var_attrs<pointer>, uniq_name = "_QFacc_private_pointer_arrayEa"} : (!fir.ref<!fir.box<!fir.ptr<!fir.array<?xi32>>>>, !fir.dscope) -> (!fir.ref<!fir.box<!fir.ptr<!fir.array<?xi32>>>>, !fir.ref<!fir.box<!fir.ptr<!fir.array<?xi32>>>>)
 ! CHECK: acc.parallel {{.*}} {
-! CHECK: %[[BOX:.*]] = fir.load %[[DECLA_A]]#0 : !fir.ref<!fir.box<!fir.ptr<!fir.array<?xi32>>>>
-! CHECK: %[[BOX_ADDR:.*]] = fir.box_addr %[[BOX]] : (!fir.box<!fir.ptr<!fir.array<?xi32>>>) -> !fir.ptr<!fir.array<?xi32>>
-! CHECK: %[[PRIVATE:.*]] = acc.private varPtr(%[[BOX_ADDR]] : !fir.ptr<!fir.array<?xi32>>) bounds(%{{.*}}) -> !fir.ptr<!fir.array<?xi32>> {name = "a"}
-! CHECK: acc.loop {{.*}} private({{.*}}@privatization_box_ptr_Uxi32 -> %[[PRIVATE]] : !fir.ptr<!fir.array<?xi32>>)
+! CHECK: %[[PRIVATE:.*]] = acc.private varPtr(%[[DECLA_A]]#0 : !fir.ref<!fir.box<!fir.ptr<!fir.array<?xi32>>>>) -> !fir.ref<!fir.box<!fir.ptr<!fir.array<?xi32>>>> {name = "a"}
+! CHECK: acc.loop {{.*}} private({{.*}}@privatization_ref_box_ptr_Uxi32 -> %[[PRIVATE]] : !fir.ref<!fir.box<!fir.ptr<!fir.array<?xi32>>>>)
 
 subroutine acc_private_dynamic_extent(a, n)
   integer :: n, i
@@ -335,9 +324,8 @@ subroutine acc_private_dynamic_extent(a, n)
 ! CHECK: %[[DECL_N:.*]]:2 = hlfir.declare %[[ARG1]] dummy_scope %{{[0-9]+}} {uniq_name = "_QFacc_private_dynamic_extentEn"} : (!fir.ref<i32>, !fir.dscope) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK: %[[DECL_A:.*]]:2 = hlfir.declare %[[ARG0]](%{{.*}}) dummy_scope %{{[0-9]+}} {uniq_name = "_QFacc_private_dynamic_extentEa"} : (!fir.ref<!fir.array<?x?x2xi32>>, !fir.shape<3>, !fir.dscope) -> (!fir.box<!fir.array<?x?x2xi32>>, !fir.ref<!fir.array<?x?x2xi32>>)
 ! CHECK: acc.parallel {{.*}} {
-! CHECK: %[[BOX_ADDR:.*]] = fir.box_addr %[[DECL_A]]#0 : (!fir.box<!fir.array<?x?x2xi32>>) -> !fir.ref<!fir.array<?x?x2xi32>>
-! CHECK: %[[PRIV:.*]] = acc.private varPtr(%[[BOX_ADDR]] : !fir.ref<!fir.array<?x?x2xi32>>) bounds(%{{.*}}, %{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<?x?x2xi32>> {name = "a"}
-! CHECK: acc.loop {{.*}} private({{.*}}@privatization_box_UxUx2xi32 -> %[[PRIV]] : !fir.ref<!fir.array<?x?x2xi32>>)
+! CHECK: %[[PRIV:.*]] = acc.private var(%[[DECL_A]]#0 : !fir.box<!fir.array<?x?x2xi32>>) -> !fir.box<!fir.array<?x?x2xi32>> {name = "a"}
+! CHECK: acc.loop {{.*}} private({{.*}}@privatization_box_UxUx2xi32 -> %[[PRIV]] : !fir.box<!fir.array<?x?x2xi32>>)
 
 subroutine acc_firstprivate_assumed_shape(a, n)
   integer :: a(:), i, n
@@ -367,7 +355,7 @@ subroutine acc_firstprivate_dynamic_extent(a, n)
   end do
 end subroutine
 
-! CHECK: acc.parallel {{.*}} firstprivate(@firstprivatization_box_UxUx2xi32 -> %{{.*}} : !fir.ref<!fir.array<?x?x2xi32>>)
+! CHECK: acc.parallel {{.*}} firstprivate(@firstprivatization_box_UxUx2xi32 -> %{{.*}} : !fir.box<!fir.array<?x?x2xi32>>)
 
 module acc_declare_equivalent
   integer, parameter :: n = 10
@@ -381,7 +369,7 @@ subroutine sub1()
   end subroutine
 end module
 
-! CHECK: acc.parallel private(@privatization_ref_10xf32 -> %{{.*}} : !fir.ref<!fir.array<10xf32>>)
+! CHECK: acc.parallel private(@privatization_ptr_10xf32 -> %{{.*}} : !fir.ptr<!fir.array<10xf32>>)
 
 subroutine acc_private_use()
   integer :: i, j
@@ -396,7 +384,7 @@ subroutine acc_private_use()
 ! CHECK: %[[I:.*]] = fir.alloca i32 {bindc_name = "i", uniq_name = "_QFacc_private_useEi"}
 ! CHECK: %[[DECL_I:.*]]:2 = hlfir.declare %0 {uniq_name = "_QFacc_private_useEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK: acc.parallel
-! CHECK: %[[PRIV_I:.*]] = acc.private varPtr(%[[DECL_I]]#1 : !fir.ref<i32>) -> !fir.ref<i32> {implicit = true, name = ""}
+! CHECK: %[[PRIV_I:.*]] = acc.private varPtr(%[[DECL_I]]#1 : !fir.ref<i32>) -> !fir.ref<i32> {implicit = true, name = "i"}
 ! CHECK: %[[DECL_PRIV_I:.*]]:2 = hlfir.declare %[[PRIV_I]] {uniq_name = "_QFacc_private_useEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK: acc.loop {{.*}} private(@privatization_ref_i32 -> %[[PRIV_I]] : !fir.ref<i32>) control(%[[IV0:.*]] : i32) = (%c1{{.*}} : i32) to (%c10{{.*}} : i32) step (%c1{{.*}} : i32)
 ! CHECK:   fir.store %[[IV0]] to %[[DECL_PRIV_I]]#0 : !fir.ref<i32>
diff --git a/flang/test/Lower/OpenACC/acc-reduction-unwrap-defaultbounds.f90 b/flang/test/Lower/OpenACC/acc-reduction-unwrap-defaultbounds.f90
new file mode 100644
index 0000000000000..5bb751678ed53
--- /dev/null
+++ b/flang/test/Lower/OpenACC/acc-reduction-unwrap-defaultbounds.f90
@@ -0,0 +1,1213 @@
+! This test checks lowering of OpenACC reduction clause.
+
+! RUN: bbc -fopenacc -emit-hlfir --openacc-unwrap-fir-box=true --openacc-generate-default-bounds=true %s -o - | FileCheck %s
+
+! CHECK-LABEL: acc.reduction.recipe @reduction_max_box_UxUxf32 : !fir.box<!fir.array<?x?xf32>> reduction_operator <max> init {
+! CHECK: ^bb0(%[[ARG0:.*]]: !fir.box<!fir.array<?x?xf32>>):
+! CHECK:   %[[CST:.*]] = arith.constant -1.401300e-45 : f32
+! CHECK:   %[[DIMS0:.*]]:3 = fir.box_dims %[[ARG0]], %c0{{.*}} : (!fir.box<!fir.array<?x?xf32>>, index) -> (index, index, index)
+! CHECK:   %[[DIMS1:.*]]:3 = fir.box_dims %[[ARG0]], %c1 : (!fir.box<!fir.array<?x?xf32>>, index) -> (index, index, index)
+! CHECK:   %[[SHAPE:.*]] = fir.shape %[[DIMS0]]#1, %[[DIMS1]]#1 : (index, index) -> !fir.shape<2>
+! CHECK:   %[[TEMP:.*]] = fir.allocmem !fir.array<?x?xf32>, %[[DIMS0]]#1, %[[DIMS1]]#1 {bindc_name = ".tmp", uniq_name = ""} 
+! CHECK:   %[[DECL:.*]]:2 = hlfir.declare %[[TEMP]](%[[SHAPE]]) {uniq_name = ".tmp"} : (!fir.heap<!fir.array<?x?xf32>>, !fir.shape<2>) -> (!fir.box<!fir.array<?x?xf32>>, !fir.heap<!fir.array<?x?xf32>>)
+! CHECK:   hlfir.assign %[[CST]] to %[[DECL]]#0 : f32, !fir.box<!fir.array<?x?xf32>>
+! CHECK:   acc.yield %[[DECL]]#0 : !fir.box<!fir.array<?x?xf32>>
+! CHECK: } combiner {
+! CHECK: ^bb0(%[[V1:.*]]: !fir.box<!fir.array<?x?xf32>>, %[[V2:.*]]: !fir.box<!fir.array<?x?xf32>>, %[[LB0:.*]]: index, %[[UB0:.*]]: index, %[[STEP0:.*]]: index, %[[LB1:.*]]: index, %[[UB1:.*]]: index, %[[STEP1:.*]]: index):
+ 
+! CHECK:   %[[SHAPE:.*]] = fir.shape %{{.*}}, %{{.*}} : (index, index) -> !fir.shape<2>
+! CHECK:   %[[DES_V1:.*]] = hlfir.designate %[[V1]] (%[[LB0]]:%[[UB0]]:%[[STEP0]], %[[LB1]]:%[[UB1]]:%[[STEP1]])  shape %[[SHAPE]] : (!fir.box<!fir.array<?x?xf32>>, index, index, index, index, index, index, !fir.shape<2>) -> !fir.box<!fir.array<?x?xf32>>
+! CHECK:   %[[DES_V2:.*]] = hlfir.designate %[[V2]] (%[[LB0]]:%[[UB0]]:%[[STEP0]], %[[LB1]]:%[[UB1]]:%[[STEP1]])  shape %[[SHAPE]] : (!fir.box<!fir.array<?x?xf32>>, index, index, index, index, index, index, !fir.shape<2>) -> !fir.box<!fir.array<?x?xf32>>
+! CHECK:   %[[ELEMENTAL:.*]] = hlfir.elemental %[[SHAPE]] unordered : (!fir.shape<2>) -> !hlfir.expr<?x?xf32> {
+! CHECK:   ^bb0(%[[ARG0:.*]]: index, %[[ARG1:.*]]: index):
+! CHECK:     %[[D1:.*]] = hlfir.designate %[[DES_V1]] (%[[ARG0]], %[[ARG1]])  : (!fir.box<!fir.array<?x?xf32>>, index, index) -> !fir.ref<f32>
+! CHECK:     %[[D2:.*]] = hlfir.designate %[[DES_V2]] (%[[ARG0]], %[[ARG1]])  : (!fir.box<!fir.array<?x?xf32>>, index, index) -> !fir.ref<f32>
+! CHECK:     %[[LOAD1:.*]] = fir.load %[[D1]] : !fir.ref<f32>
+! CHECK:     %[[LOAD2:.*]] = fir.load %[[D2]] : !fir.ref<f32>
+! CHECK:     %[[CMP:.*]] = arith.cmpf ogt, %[[LOAD1]], %[[LOAD2]] {{.*}} : f32
+! CHECK:     %[[SELECT:.*]] = arith.select %[[CMP]], %[[LOAD1]], %[[LOAD2]] : f32
+! CHECK:     hlfir.yield_element %[[SELECT]] : f32
+! CHECK:   }
+! CHECK:   hlfir.assign %[[ELEMENTAL]] to %[[V1]] : !hlfir.expr<?x?xf32>, !fir.box<!fir.array<?x?xf32>>
+! CHECK:   acc.yield %[[V1]] : !fir.box<!fir.array<?x?xf32>>
+! CHECK: }
+
+! CHECK-LABEL: acc.reduction.recipe @reduction_max_box_ptr_Uxf32 : !fir.box<!fir.ptr<!fir.array<?xf32>>> reduction_operator <max> init {
+! CHECK: ^bb0(%{{.*}}: !fir.box<!fir.ptr<!fir.array<?xf32>>>):
+! CHECK: } combiner {
+! CHECK: ^bb0(%{{.*}}: !fir.box<!fir.ptr<!fir.array<?xf32>>>, %{{.*}}: !fir.box<!fir.ptr<!fir.array<?xf32>>>, %{{.*}}: index, %{{.*}}: index, %{{.*}}: index):
+! CHECK: }
+
+! CHECK-LABEL: acc.reduction.recipe @reduction_max_box_heap_Uxf32 : !fir.box<!fir.heap<!fir.array<?xf32>>> reduction_operator <max> init {
+! CHECK: ^bb0(%[[ARG0:.*]]: !fir.box<!fir.heap<!fir.array<?xf32>>>):
+! CHECK:   %[[CST:.*]] = arith.constant -1.401300e-45 : f32
+! CHECK:   %[[C0:.*]] = arith.constant 0 : index
+! CHECK:   %[[BOX_DIMS:.*]]:3 = fir.box_dims %[[ARG0]], %[[C0]] : (!fir.box<!fir.heap<!fir.array<?xf32>>>, index) -> (index, index, index)
+! CHECK:   %[[SHAPE:.*]] = fir.shape %[[BOX_DIMS]]#1 : (index) -> !fir.shape<1>
+! CHECK:   %[[TEMP:.*]] = fir.allocmem !fir.array<?xf32>, %[[BOX_DIMS]]#1 {bindc_name = ".tmp", uniq_name = ""}
+! CHECK:   %[[DECLARE:.*]]:2 = hlfir.declare %2(%1) {uniq_name = ".tmp"} : (!fir.heap<!fir.array<?xf32>>, !fir.shape<1>) -> (!fir.box<!fir.array<?xf32>>, !fir.heap<!fir.array<?xf32>>)
+! CHECK:   hlfir.assign %[[CST]] to %[[DECLARE]]#0 : f32, !fir.box<!fir.array<?xf32>>
+! CHECK:   acc.yield %[[DECLARE]]#0 : !fir.box<!fir.array<?xf32>>
+! CHECK: } combiner {
+! CHECK: ^bb0(%[[ARG0:.*]]: !fir.box<!fir.heap<!fir.array<?xf32>>>, %[[ARG1:.*]]: !fir.box<!fir.heap<!fir.array<?xf32>>>, %[[ARG2:.*]]: index, %[[ARG3:.*]]: index, %[[ARG4:.*]]: index):
+! CHECK:   %[[SHAPE:.*]] = fir.shape %{{.*}} : (index) -> !fir.shape<1>
+! CHECK:   %[[DES_V1:.*]] = hlfir.designate %[[ARG0]] (%[[ARG2]]:%[[ARG3]]:%[[ARG4]]) shape %[[SHAPE]] : (!fir.box<!fir.heap<!fir.array<?xf32>>>, index, index, index, !fir.shape<1>) -> !fir.box<!fir.heap<!fir.array<?xf32>>>
+! CHECK:   %[[DES_V2:.*]] = hlfir.designate %[[ARG1]] (%[[ARG2]]:%[[ARG3]]:%[[ARG4]]) shape %[[SHAPE]] : (!fir.box<!fir.heap<!fir.array<?xf32>>>, index, index, index, !fir.shape<1>) -> !fir.box<!fir.heap<!fir.array<?xf32>>>
+! CHECK:   %[[ELEMENTAL:.*]] = hlfir.elemental %[[SHAPE]] unordered : (!fir.shape<1>) -> !hlfir.expr<?xf32> {
+! CHECK:   ^bb0(%[[IV:.*]]: index):
+! CHECK:     %[[V1:.*]] = hlfir.designate %[[DES_V1]] (%[[IV]])  : (!fir.box<!fir.heap<!fir.array<?xf32>>>, index) -> !fir.ref<f32>
+! CHECK:     %[[V2:.*]] = hlfir.designate %[[DES_V2]] (%[[IV]])  : (!fir.box<!fir.heap<!fir.array<?xf32>>>, index) -> !fir.ref<f32>
+! CHECK:     %[[LOAD_V1:.*]] = fir.load %[[V1]] : !fir.ref<f32>
+! CHECK:     %[[LOAD_V2:.*]] = fir.load %[[V2]] : !fir.ref<f32>
+! CHECK:     %[[CMP:.*]] = arith.cmpf ogt, %[[LOAD_V1]], %[[LOAD_V2]] {{.*}} : f32
+! CHECK:     %[[SELECT:.*]] = arith.select %[[CMP]], %[[LOAD_V1]], %[[LOAD_V2]] : f32
+! CHECK:     hlfir.yield_element %[[SELECT]] : f32
+! CHECK:   }
+! CHECK:   hlfir.assign %[[ELEMENTAL]] to %[[ARG0]] : !hlfir.expr<?xf32>, !fir.box<!fir.heap<!fir.array<?xf32>>>
+! CHECK:   acc.yield %[[ARG0]] : !fir.box<!fir.heap<!fir.array<?xf32>>>
+! CHECK: }
+
+! CHECK-LABEL: acc.reduction.recipe @reduction_add_section_lb1.ub3_box_Uxi32 : !fir.box<!fir.array<?xi32>> reduction_operator <add> init {
+! CHECK: ^bb0(%[[ARG0:.*]]: !fir.box<!fir.array<?xi32>>):
+! CHECK:   %[[BOX_DIMS:.*]]:3 = fir.box_dims %[[ARG0]], %c0{{.*}} : (!fir.box<!fir.array<?xi32>>, index) -> (index, index, index)
+! CHECK:   %[[SHAPE:.*]] = fir.shape %[[BOX_DIMS]]#1 : (index) -> !fir.shape<1>
+! CHECK:   %[[TEMP:.*]] = fir.allocmem !fir.array<?xi32>, %0#1 {bindc_name = ".tmp", uniq_name = ""}
+! CHECK:   %[[DECLARE:.*]]:2 = hlfir.declare %[[TEMP]](%[[SHAPE]]) {uniq_name = ".tmp"} : (!fir.heap<!fir.array<?xi32>>, !fir.shape<1>) -> (!fir.box<!fir.array<?xi32>>, !fir.heap<!fir.array<?xi32>>)
+! CHECK:   hlfir.assign %c0{{.*}} to %[[DECLARE]]#0 : i32, !fir.box<!fir.array<?xi32>>
+! CHECK:   acc.yield %[[DECLARE]]#0 : !fir.box<!fir.array<?xi32>>
+! CHECK: } combiner {
+! CHECK: ^bb0(%[[ARG0:.*]]: !fir.box<!fir.array<?xi32>>, %[[ARG1:.*]]: !fir.box<!fir.array<?xi32>>):
+! CHECK:   %[[SHAPE:.*]] = fir.shape %{{.*}} : (index) -> !fir.shape<1>
+! CHECK:   %[[DES1:.*]] = hlfir.designate %[[ARG0]] shape %[[SHAPE]] : (!fir.box<!fir.array<?xi32>>, !fir.shape<1>) -> !fir.box<!fir.array<?xi32>>
+! CHECK:   %[[DES2:.*]] = hlfir.designate %[[ARG1]] shape %[[SHAPE]] : (!fir.box<!fir.array<?xi32>>, !fir.shape<1>) -> !fir.box<!fir.array<?xi32>>
+! CHECK:   %[[ELEMENTAL:.*]] = hlfir.elemental %[[SHAPE]] unordered : (!fir.shape<1>) -> !hlfir.expr<?xi32> {
+! CHECK:   ^bb0(%[[IV:.*]]: index):
+! CHECK:     %[[DES_V1:.*]] = hlfir.designate %[[DES1]] (%[[IV]])  : (!fir.box<!fir.array<?xi32>>, index) -> !fir.ref<i32>
+! CHECK:     %[[DES_V2:.*]] = hlfir.designate %[[DES2]] (%[[IV]])  : (!fir.box<!fir.array<?xi32>>, index) -> !fir.ref<i32>
+! CHECK:     %[[LOAD_V1:.*]] = fir.load %[[DES_V1]] : !fir.ref<i32>
+! CHECK:     %[[LOAD_V2:.*]] = fir.load %[[DES_V2]] : !fir.ref<i32>
+! CHECK:     %[[COMBINED:.*]] = arith.addi %[[LOAD_V1]], %[[LOAD_V2]] : i32
+! CHECK:     hlfir.yield_element %[[COMBINED]] : i32
+! CHECK:   }
+! CHECK:   hlfir.assign %[[ELEMENTAL]] to %[[ARG0]] : !hlfir.expr<?xi32>, !fir.box<!fir.array<?xi32>>
+! CHECK:   acc.yield %[[ARG0]] : !fir.box<!fir.array<?xi32>>
+! CHECK: }
+
+! CHECK-LABEL: acc.reduction.recipe @reduction_max_box_Uxf32 : !fir.box<!fir.array<?xf32>> reduction_operator <max> init {
+! CHECK: ^bb0(%[[ARG0:.*]]: !fir.box<!fir.array<?xf32>>):
+! CHECK:   %[[INIT_VALUE:.*]] = arith.constant -1.401300e-45 : f32
+! CHECK:   %[[C0:.*]] = arith.constant 0 : index
+! CHECK:   %[[BOX_DIMS:.*]]:3 = fir.box_dims %[[ARG0]], %[[C0]] : (!fir.box<!fir.array<?xf32>>, index) -> (index, index, index)
+! CHECK:   %[[SHAPE:.*]] = fir.shape %[[BOX_DIMS]]#1 : (index) -> !fir.shape<1>
+! CHECK:   %[[TEMP:.*]] = fir.allocmem !fir.array<?xf32>, %0#1 {bindc_name = ".tmp", uniq_name = ""}
+! CHECK:   %[[DECLARE:.*]]:2 = hlfir.declare %[[TEMP]](%[[SHAPE]]) {uniq_name = ".tmp"} : (!fir.heap<!fir.array<?xf32>>, !fir.shape<1>) -> (!fir.box<!fir.array<?xf32>>, !fir.heap<!fir.array<?xf32>>)
+! CHECK:   hlfir.assign %[[INIT_VALUE]] to %[[DECLARE]]#0 : f32, !fir.box<!fir.array<?xf32>>
+! CHECK:   acc.yield %[[DECLARE]]#0 : !fir.box<!fir.array<?xf32>>
+! CHECK: } combiner {
+! CHECK: ^bb0(%[[ARG0:.*]]: !fir.box<!fir.array<?xf32>>, %[[ARG1:.*]]: !fir.box<!fir.array<?xf32>>
+! CHECK:   %[[LEFT:.*]] = hlfir.designate %[[ARG0]] (%{{.*}}:%{{.*}}:%{{.*}}) shape %{{.*}} : (!fir.box<!fir.array<?xf32>>, index, index, index, !fir.shape<1>) -> !fir.box<!fir.array<?xf32>>
+! CHECK:   %[[RIGHT:.*]] = hlfir.designate %[[ARG1]] (%{{.*}}:%{{.*}}:%{{.*}}) shape %{{.*}} : (!fir.box<!fir.array<?xf32>>, index, index, index, !fir.shape<1>) -> !fir.box<!fir.array<?xf32>>
+! CHECK:   %[[ELEMENTAL:.*]] = hlfir.elemental %{{.*}} unordered : (!fir.shape<1>) -> !hlfir.expr<?xf32> {
+! CHECK:   ^bb0(%{{.*}}: index):
+! CHECK:     %[[DES_V1:.*]] = hlfir.designate %[[LEFT]] (%{{.*}})  : (!fir.box<!fir.array<?xf32>>, index) -> !fir.ref<f32>
+! CHECK:     %[[DES_V2:.*]] = hlfir.designate %[[RIGHT]] (%{{.*}})  : (!fir.box<!fir.array<?xf32>>, index) -> !fir.ref<f32>
+! CHECK:     %[[LOAD_V1:.*]] = fir.load %[[DES_V1]] : !fir.ref<f32>
+! CHECK:     %[[LOAD_V2:.*]] = fir.load %[[DES_V2]] : !fir.ref<f32>
+! CHECK:     %[[CMPF:.*]] = arith.cmpf ogt, %[[LOAD_V1]], %[[LOAD_V2]] {{.*}} : f32
+! CHECK:     %[[SELECT:.*]] = arith.select %[[CMPF]], %[[LOAD_V1]], %[[LOAD_V2]] : f32
+! CHECK:     hlfir.yield_element %[[SELECT]] : f32
+! CHECK:   }
+! CHECK:   hlfir.assign %[[ELEMENTAL]] to %[[ARG0]] : !hlfir.expr<?xf32>, !fir.box<!fir.array<?xf32>>
+! CHECK: acc.yield %[[ARG0]] : !fir.box<!fir.array<?xf32>>
+! CHECK: }
+
+! CHECK-LABEL: acc.reduction.recipe @reduction_add_box_Uxi32 : !fir.box<!fir.array<?xi32>> reduction_operator <add> init {
+! CHECK: ^bb0(%[[ARG0:.*]]: !fir.box<!fir.array<?xi32>>):
+! CHECK:   %[[INIT_VALUE:.*]] = arith.constant 0 : i32
+! CHECK:   %[[C0:.*]] = arith.constant 0 : index
+! CHECK:   %[[BOX_DIMS:.*]]:3 = fir.box_dims %[[ARG0]], %[[C0]] : (!fir.box<!fir.array<?xi32>>, index) -> (index, index, index)
+! CHECK:   %[[SHAPE:.*]] = fir.shape %[[BOX_DIMS]]#1 : (index) -> !fir.shape<1>
+! CHECK:   %[[TEMP:.*]] = fir.allocmem !fir.array<?xi32>, %[[BOX_DIMS]]#1 {bindc_name = ".tmp", uniq_name = ""}
+! CHECK:   %[[DECLARE:.*]]:2 = hlfir.declare %[[TEMP]](%[[SHAPE]]) {uniq_name = ".tmp"} : (!fir.heap<!fir.array<?xi32>>, !fir.shape<1>) -> (!fir.box<!fir.array<?xi32>>, !fir.heap<!fir.array<?xi32>>)
+! CHECK:   hlfir.assign %[[INIT_VALUE]] to %[[DECLARE]]#0 : i32, !fir.box<!fir.array<?xi32>>
+! CHECK:   acc.yield %[[DECLARE]]#0 : !fir.box<!fir.array<?xi32>>
+! CHECK: } combiner {
+! CHECK: ^bb0(%[[V1:.*]]: !fir.box<!fir.array<?xi32>>, %[[V2:.*]]: !fir.box<!fir.array<?xi32>>
+! CHECK:   %[[LEFT:.*]] = hlfir.designate %[[ARG0]] (%{{.*}}:%{{.*}}:%{{.*}}) shape %{{.*}} : (!fir.box<!fir.array<?xi32>>, index, index, index, !fir.shape<1>) -> !fir.box<!fir.array<?xi32>>
+! CHECK:   %[[RIGHT:.*]] = hlfir.designate %[[ARG1]] (%{{.*}}:%{{.*}}:%{{.*}}) shape %{{.*}} : (!fir.box<!fir.array<?xi32>>, index, index, index, !fir.shape<1>) -> !fir.box<!fir.array<?xi32>>
+! CHECK:   %[[ELEMENTAL:.*]] = hlfir.elemental %{{.*}} unordered : (!fir.shape<1>) -> !hlfir.expr<?xi32> {
+! CHECK:   ^bb0(%{{.*}}: index):
+! CHECK:     %[[DES_V1:.*]] = hlfir.designate %[[LEFT]] (%{{.*}})  : (!fir.box<!fir.array<?xi32>>, index) -> !fir.ref<i32>
+! CHECK:     %[[DES_V2:.*]] = hlfir.designate %[[RIGHT]] (%{{.*}})  : (!fir.box<!fir.array<?xi32>>, index) -> !fir.ref<i32>
+! CHECK:     %[[LOAD_V1:.*]] = fir.load %[[DES_V1]] : !fir.ref<i32>
+! CHECK:     %[[LOAD_V2:.*]] = fir.load %[[DES_V2]] : !fir.ref<i32>
+! CHECK:     %[[COMBINED:.*]] = arith.addi %[[LOAD_V1]], %[[LOAD_V2]] : i32
+! CHECK:     hlfir.yield_element %[[COMBINED]] : i32
+! CHECK:   }
+! CHECK:   hlfir.assign %[[ELEMENTAL]] to %[[V1]] : !hlfir.expr<?xi32>, !fir.box<!fir.array<?xi32>>
+! CHECK:   acc.yield %arg0 : !fir.box<!fir.array<?xi32>>
+! CHECK: }
+
+! CHECK-LABEL: acc.reduction.recipe @reduction_mul_ref_z32 : !fir.ref<complex<f32>> reduction_operator <mul> init {
+! CHECK: ^bb0(%{{.*}}: !fir.ref<complex<f32>>):
+! CHECK:   %[[REAL:.*]] = arith.constant 1.000000e+00 : f32
+! CHECK:   %[[IMAG:.*]] = arith.constant 0.000000e+00 : f32
+! CHECK:   %[[UNDEF:.*]] = fir.undefined complex<f32>
+! CHECK:   %[[UNDEF1:.*]] = fir.insert_value %[[UNDEF]], %[[REAL]], [0 : index] : (complex<f32>, f32) -> complex<f32>
+! CHECK:   %[[UNDEF2:.*]] = fir.insert_value %[[UNDEF1]], %[[IMAG]], [1 : index] : (complex<f32>, f32) -> complex<f32>
+! CHECK:   %[[ALLOCA:.*]] = fir.alloca complex<f32>
+! CHECK:   %[[DECLARE:.*]]:2 = hlfir.declare %[[ALLOCA]] {uniq_name = "acc.reduction.init"} : (!fir.ref<complex<f32>>) -> (!fir.ref<complex<f32>>, !fir.ref<complex<f32>>)
+! CHECK:   fir.store %[[UNDEF2]] to %[[DECLARE]]#0 : !fir.ref<complex<f32>>
+! CHECK:   acc.yield %[[DECLARE]]#0 : !fir.ref<complex<f32>>
+! CHECK: } combiner {
+! CHECK: ^bb0(%[[ARG0:.*]]: !fir.ref<complex<f32>>, %[[ARG1:.*]]: !fir.ref<complex<f32>>):
+! CHECK:   %[[LOAD0:.*]] = fir.load %[[ARG0]] : !fir.ref<complex<f32>>
+! CHECK:   %[[LOAD1:.*]] = fir.load %[[ARG1]] : !fir.ref<complex<f32>>
+! CHECK:   %[[COMBINED:.*]] = fir.mulc %[[LOAD0]], %[[LOAD1]] {fastmath = #arith.fastmath<contract>} : complex<f32>
+! CHECK:   fir.store %[[COMBINED]] to %[[ARG0]] : !fir.ref<complex<f32>>
+! CHECK:   acc.yield %[[ARG0]] : !fir.ref<complex<f32>>
+! CHECK: }
+
+! CHECK-LABEL: acc.reduction.recipe @reduction_add_ref_z32 : !fir.ref<complex<f32>> reduction_operator <add> init {
+! CHECK: ^bb0(%{{.*}}: !fir.ref<complex<f32>>):
+! CHECK:   %[[REAL:.*]] = arith.constant 0.000000e+00 : f32
+! CHECK:   %[[IMAG:.*]] = arith.constant 0.000000e+00 : f32
+! CHECK:   %[[UNDEF:.*]] = fir.undefined complex<f32>
+! CHECK:   %[[UNDEF1:.*]] = fir.insert_value %[[UNDEF]], %[[REAL]], [0 : index] : (complex<f32>, f32) -> complex<f32>
+! CHECK:   %[[UNDEF2:.*]] = fir.insert_value %[[UNDEF1]], %[[IMAG]], [1 : index] : (complex<f32>, f32) -> complex<f32>
+! CHECK:   %[[ALLOCA:.*]] = fir.alloca complex<f32>
+! CHECK:   %[[DECLARE:.*]]:2 = hlfir.declare %[[ALLOCA]] {uniq_name = "acc.reduction.init"} : (!fir.ref<complex<f32>>) -> (!fir.ref<complex<f32>>, !fir.ref<complex<f32>>)
+! CHECK:   fir.store %[[UNDEF2]] to %[[DECLARE]]#0 : !fir.ref<complex<f32>>
+! CHECK:   acc.yield %[[DECLARE]]#0 : !fir.ref<complex<f32>>
+! CHECK: } combiner {
+! CHECK: ^bb0(%[[ARG0:.*]]: !fir.ref<complex<f32>>, %[[ARG1:.*]]: !fir.ref<complex<f32>>):
+! CHECK:   %[[LOAD0:.*]] = fir.load %[[ARG0]] : !fir.ref<complex<f32>>
+! CHECK:   %[[LOAD1:.*]] = fir.load %[[ARG1]] : !fir.ref<complex<f32>>
+! CHECK:   %[[COMBINED:.*]] = fir.addc %[[LOAD0]], %[[LOAD1]] {fastmath = #arith.fastmath<contract>} : complex<f32>
+! CHECK:   fir.store %[[COMBINED]] to %[[ARG0]] : !fir.ref<complex<f32>>
+! CHECK:   acc.yield %[[ARG0]] : !fir.ref<complex<f32>>
+! CHECK: }
+
+! CHECK-LABEL: acc.reduction.recipe @reduction_neqv_ref_l32 : !fir.ref<!fir.logical<4>> reduction_operator <neqv> init {
+! CHECK: ^bb0(%{{.*}}: !fir.ref<!fir.logical<4>>):
+! CHECK:   %[[CST:.*]] = arith.constant false
+! CHECK:   %[[ALLOCA:.*]] = fir.alloca !fir.logical<4>
+! CHECK:   %[[DECLARE:.*]]:2 = hlfir.declare %[[ALLOCA]] {uniq_name = "acc.reduction.init"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
+! CHECK:   %[[CONVERT:.*]] = fir.convert %[[CST]] : (i1) -> !fir.logical<4>
+! CHECK:   fir.store %[[CONVERT]] to %[[DECLARE]]#0 : !fir.ref<!fir.logical<4>>
+! CHECK:   acc.yield %[[DECLARE]]#0 : !fir.ref<!fir.logical<4>>
+! CHECK: } combiner {
+! CHECK: ^bb0(%[[ARG0:.*]]: !fir.ref<!fir.logical<4>>, %[[ARG1:.*]]: !fir.ref<!fir.logical<4>>):
+! CHECK:   %[[LOAD0:.*]] = fir.load %[[ARG0]] : !fir.ref<!fir.logical<4>>
+! CHECK:   %[[LOAD1:.*]] = fir.load %[[ARG1]] : !fir.ref<!fir.logical<4>>
+! CHECK:   %[[CONV0:.*]] = fir.convert %[[LOAD0]] : (!fir.logical<4>) -> i1
+! CHECK:   %[[CONV1:.*]] = fir.convert %[[LOAD1]] : (!fir.logical<4>) -> i1
+! CHECK:   %[[CMP:.*]] = arith.cmpi ne, %[[CONV0]], %[[CONV1]] : i1
+! CHECK:   %[[CMP_CONV:.*]] = fir.convert %[[CMP]] : (i1) -> !fir.logical<4>
+! CHECK:   fir.store %[[CMP_CONV]] to %[[ARG0]] : !fir.ref<!fir.logical<4>>
+! CHECK:   acc.yield %[[ARG0]] : !fir.ref<!fir.logical<4>>
+! CHECK: }
+
+! CHECK-LABEL: acc.reduction.recipe @reduction_eqv_ref_l32 : !fir.ref<!fir.logical<4>> reduction_operator <eqv> init {
+! CHECK: ^bb0(%{{.*}}: !fir.ref<!fir.logical<4>>):
+! CHECK:   %[[CST:.*]] = arith.constant true
+! CHECK:   %[[ALLOCA:.*]] = fir.alloca !fir.logical<4>
+! CHECK:   %[[DECLARE:.*]]:2 = hlfir.declare %[[ALLOCA]] {uniq_name = "acc.reduction.init"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
+! CHECK:   %[[CONVERT:.*]] = fir.convert %[[CST]] : (i1) -> !fir.logical<4>
+! CHECK:   fir.store %[[CONVERT]] to %[[DECLARE]]#0 : !fir.ref<!fir.logical<4>>
+! CHECK:   acc.yield %[[DECLARE]]#0 : !fir.ref<!fir.logical<4>>
+! CHECK: } combiner {
+! CHECK: ^bb0(%[[ARG0:.*]]: !fir.ref<!fir.logical<4>>, %[[ARG1:.*]]: !fir.ref<!fir.logical<4>>):
+! CHECK:   %[[LOAD0:.*]] = fir.load %[[ARG0]] : !fir.ref<!fir.logical<4>>
+! CHECK:   %[[LOAD1:.*]] = fir.load %[[ARG1]] : !fir.ref<!fir.logical<4>>
+! CHECK:   %[[CONV0:.*]] = fir.convert %[[LOAD0]] : (!fir.logical<4>) -> i1
+! CHECK:   %[[CONV1:.*]] = fir.convert %[[LOAD1]] : (!fir.logical<4>) -> i1
+! CHECK:   %[[CMP:.*]] = arith.cmpi eq, %[[CONV0]], %[[CONV1]] : i1
+! CHECK:   %[[CMP_CONV:.*]] = fir.convert %[[CMP]] : (i1) -> !fir.logical<4>
+! CHECK:   fir.store %[[CMP_CONV]] to %[[ARG0]] : !fir.ref<!fir.logical<4>>
+! CHECK:   acc.yield %[[ARG0]] : !fir.ref<!fir.logical<4>>
+! CHECK: }
+
+! CHECK-LABEL: acc.reduction.recipe @reduction_lor_ref_l32 : !fir.ref<!fir.logical<4>> reduction_operator <lor> init {
+! CHECK: ^bb0(%{{.*}}: !fir.ref<!fir.logical<4>>):
+! CHECK:   %[[CST:.*]] = arith.constant false
+! CHECK:   %[[ALLOCA:.*]] = fir.alloca !fir.logical<4>
+! CHECK:   %[[DECLARE:.*]]:2 = hlfir.declare %[[ALLOCA]] {uniq_name = "acc.reduction.init"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
+! CHECK:   %[[CONVERT:.*]] = fir.convert %[[CST]] : (i1) -> !fir.logical<4>
+! CHECK:   fir.store %[[CONVERT]] to %[[DECLARE]]#0 : !fir.ref<!fir.logical<4>>
+! CHECK:   acc.yield %[[DECLARE]]#0 : !fir.ref<!fir.logical<4>>
+! CHECK: } combiner {
+! CHECK: ^bb0(%[[ARG0:.*]]: !fir.ref<!fir.logical<4>>, %[[ARG1:.*]]: !fir.ref<!fir.logical<4>>):
+! CHECK:   %[[LOAD0:.*]] = fir.load %[[ARG0]] : !fir.ref<!fir.logical<4>>
+! CHECK:   %[[LOAD1:.*]] = fir.load %[[ARG1]] : !fir.ref<!fir.logical<4>>
+! CHECK:   %[[CONV0:.*]] = fir.convert %[[LOAD0]] : (!fir.logical<4>) -> i1
+! CHECK:   %[[CONV1:.*]] = fir.convert %[[LOAD1]] : (!fir.logical<4>) -> i1
+! CHECK:   %[[CMP:.*]] = arith.ori %[[CONV0]], %[[CONV1]] : i1
+! CHECK:   %[[CMP_CONV:.*]] = fir.convert %[[CMP]] : (i1) -> !fir.logical<4>
+! CHECK:   fir.store %[[CMP_CONV]] to %[[ARG0]] : !fir.ref<!fir.logical<4>>
+! CHECK:   acc.yield %[[ARG0]] : !fir.ref<!fir.logical<4>>
+! CHECK: }
+
+! CHECK-LABEL: acc.reduction.recipe @reduction_land_ref_l32 : !fir.ref<!fir.logical<4>> reduction_operator <land> init {
+! CHECK: ^bb0(%{{.*}}: !fir.ref<!fir.logical<4>>):
+! CHECK:   %[[CST:.*]] = arith.constant true
+! CHECK:   %[[ALLOCA:.*]] = fir.alloca !fir.logical<4>
+! CHECK:   %[[DECLARE:.*]]:2 = hlfir.declare %[[ALLOCA]] {uniq_name = "acc.reduction.init"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
+! CHECK:   %[[CONVERT:.*]] = fir.convert %[[CST]] : (i1) -> !fir.logical<4>
+! CHECK:   fir.store %[[CONVERT]] to %[[DECLARE]]#0 : !fir.ref<!fir.logical<4>>
+! CHECK:   acc.yield %[[DECLARE]]#0 : !fir.ref<!fir.logical<4>>
+! CHECK: } combiner {
+! CHECK: ^bb0(%[[ARG0:.*]]: !fir.ref<!fir.logical<4>>, %[[ARG1:.*]]: !fir.ref<!fir.logical<4>>):
+! CHECK:   %[[LOAD0:.*]] = fir.load %[[ARG0]] : !fir.ref<!fir.logical<4>>
+! CHECK:   %[[LOAD1:.*]] = fir.load %[[ARG1]] : !fir.ref<!fir.logical<4>>
+! CHECK:   %[[CONV0:.*]] = fir.convert %[[LOAD0]] : (!fir.logical<4>) -> i1
+! CHECK:   %[[CONV1:.*]] = fir.convert %[[LOAD1]] : (!fir.logical<4>) -> i1
+! CHECK:   %[[CMP:.*]] = arith.andi %[[CONV0]], %[[CONV1]] : i1
+! CHECK:   %[[CMP_CONV:.*]] = fir.convert %[[CMP]] : (i1) -> !fir.logical<4>
+! CHECK:   fir.store %[[CMP_CONV]] to %[[ARG0]] : !fir.ref<!fir.logical<4>>
+! CHECK:   acc.yield %[[ARG0]] : !fir.ref<!fir.logical<4>>
+! CHECK: }
+
+! CHECK-LABEL: acc.reduction.recipe @reduction_xor_ref_i32 : !fir.ref<i32> reduction_operator <xor> init {
+! CHECK: ^bb0(%{{.*}}: !fir.ref<i32>):
+! CHECK:   %[[CST:.*]] = arith.constant 0 : i32
+! CHECK:   %[[ALLOCA:.*]] = fir.alloca i32
+! CHECK:   %[[DECLARE]]:2 = hlfir.declare %[[ALLOCA]] {uniq_name = "acc.reduction.init"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+! CHECK:   fir.store %[[CST]] to %[[DECLARE]]#0 : !fir.ref<i32>
+! CHECK:   acc.yield %[[DECLARE]]#0 : !fir.ref<i32>
+! CHECK: } combiner {
+! CHECK: ^bb0(%[[ARG0:.*]]: !fir.ref<i32>, %[[ARG1:.*]]: !fir.ref<i32>):
+! CHECK:   %[[LOAD0:.*]] = fir.load %[[ARG0]] : !fir.ref<i32>
+! CHECK:   %[[LOAD1:.*]] = fir.load %[[ARG1]] : !fir.ref<i32>
+! CHECK:   %[[COMBINED:.*]] = arith.xori %[[LOAD0]], %[[LOAD1]] : i32
+! CHECK:   fir.store %[[COMBINED]] to %[[ARG0]] : !fir.ref<i32>
+! CHECK:   acc.yield %[[ARG0]] : !fir.ref<i32>
+! CHECK: }
+
+! CHECK-LABEL: acc.reduction.recipe @reduction_ior_ref_i32 : !fir.ref<i32> reduction_operator <ior> init {
+! CHECK: ^bb0(%{{.*}}: !fir.ref<i32>):
+! CHECK:   %[[CST:.*]] = arith.constant 0 : i32
+! CHECK:   %[[ALLOCA:.*]] = fir.alloca i32
+! CHECK:   %[[DECLARE:.*]]:2 = hlfir.declare %[[ALLOCA]] {uniq_name = "acc.reduction.init"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+! CHECK:   fir.store %[[CST]] to %[[DECLARE:.*]]#0 : !fir.ref<i32>
+! CHECK:   acc.yield %[[DECLARE:.*]]#0 : !fir.ref<i32>
+! CHECK: } combiner {
+! CHECK: ^bb0(%[[ARG0:.*]]: !fir.ref<i32>, %[[ARG1:.*]]: !fir.ref<i32>):
+! CHECK:   %[[LOAD0:.*]] = fir.load %[[ARG0]] : !fir.ref<i32>
+! CHECK:   %[[LOAD1:.*]] = fir.load %[[ARG1]] : !fir.ref<i32>
+! CHECK:   %[[COMBINED:.*]] = arith.ori %[[LOAD0]], %[[LOAD1]] : i32
+! CHECK:   fir.store %[[COMBINED]] to %[[ARG0]] : !fir.ref<i32>
+! CHECK:   acc.yield %[[ARG0]] : !fir.ref<i32>
+! CHECK: }
+
+! CHECK-LABEL: acc.reduction.recipe @reduction_iand_ref_i32 : !fir.ref<i32> reduction_operator <iand> init {
+! CHECK: ^bb0(%{{.*}}: !fir.ref<i32>):
+! CHECK:   %[[CST:.*]] = arith.constant -1 : i32
+! CHECK:   %[[ALLOCA:.*]] = fir.alloca i32
+! CHECK:   %[[DECLARE:.*]]:2 = hlfir.declare %[[ALLOCA]] {uniq_name = "acc.reduction.init"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+! CHECK:   fir.store %[[CST]] to %[[DECLARE]]#0 : !fir.ref<i32>
+! CHECK:   acc.yield %[[DECLARE]]#0 : !fir.ref<i32>
+! CHECK: } combiner {
+! CHECK: ^bb0(%[[ARG0:.*]]: !fir.ref<i32>, %[[ARG1:.*]]: !fir.ref<i32>):
+! CHECK:   %[[LOAD0:.*]] = fir.load %[[ARG0]] : !fir.ref<i32>
+! CHECK:   %[[LOAD1:.*]] = fir.load %[[ARG1]] : !fir.ref<i32>
+! CHECK:   %[[COMBINED:.*]] = arith.andi %[[LOAD0]], %[[LOAD1]] : i32
+! CHECK:   fir.store %[[COMBINED]] to %[[ARG0]] : !fir.ref<i32>
+! CHECK:   acc.yield %[[ARG0]] : !fir.ref<i32>
+! CHECK: }
+
+! CHECK-LABEL: acc.reduction.recipe @reduction_max_section_ext100_ref_100xf32 : !fir.ref<!fir.array<100xf32>> reduction_operator <max> init {
+! CHECK: ^bb0(%{{.*}}: !fir.ref<!fir.array<100xf32>>):
+! CHECK:   %[[INIT:.*]] = arith.constant -1.401300e-45 : f32
+! CHECK:   %[[SHAPE:.*]] = fir.shape %{{.*}} : (index) -> !fir.shape<1>
+! CHECK:   %[[ALLOCA:.*]] = fir.alloca !fir.array<100xf32>
+! CHECK:   %[[DECLARE:.*]]:2 = hlfir.declare %[[ALLOCA]](%[[SHAPE]]) {uniq_name = "acc.reduction.init"} : (!fir.ref<!fir.array<100xf32>>, !fir.shape<1>) -> (!fir.ref<!fir.array<100xf32>>, !fir.ref<!fir.array<100xf32>>)
+! CHECK:   %[[LB:.*]] = arith.constant 0 : index
+! CHECK:   %[[UB:.*]] = arith.constant 99 : index
+! CHECK:   %[[STEP:.*]] = arith.constant 1 : index
+! CHECK:   fir.do_loop %[[IV:.*]] = %[[LB]] to %[[UB]] step %[[STEP]] {
+! CHECK:     %[[COORD:.*]] = fir.coordinate_of %[[DECLARE]]#0, %[[IV]] : (!fir.ref<!fir.array<100xf32>>, index) -> !fir.ref<f32>
+! CHECK:     fir.store %[[INIT]] to %[[COORD]] : !fir.ref<f32>
+! CHECK:   }
+! CHECK:   acc.yield %[[DECLARE]]#0 : !fir.ref<!fir.array<100xf32>>
+! CHECK: } combiner {
+! CHECK: ^bb0(%[[ARG0:.*]]: !fir.ref<!fir.array<100xf32>>, %[[ARG1:.*]]: !fir.ref<!fir.array<100xf32>>):
+! CHECK:   %[[LB0:.*]] = arith.constant 0 : index
+! CHECK:   %[[UB0:.*]] = arith.constant 99 : index
+! CHECK:   %[[STEP0:.*]] = arith.constant 1 : index
+! CHECK:   fir.do_loop %[[IV0:.*]] = %[[LB0]] to %[[UB0]] step %[[STEP0]] {
+! CHECK:     %[[COORD1:.*]] = fir.coordinate_of %[[ARG0]], %[[IV0]] : (!fir.ref<!fir.array<100xf32>>, index) -> !fir.ref<f32>
+! CHECK:     %[[COORD2:.*]] = fir.coordinate_of %[[ARG1]], %[[IV0]] : (!fir.ref<!fir.array<100xf32>>, index) -> !fir.ref<f32>
+! CHECK:     %[[LOAD1:.*]] = fir.load %[[COORD1]] : !fir.ref<f32>
+! CHECK:     %[[LOAD2:.*]] = fir.load %[[COORD2]] : !fir.ref<f32>
+! CHECK:     %[[CMP:.*]] = arith.cmpf ogt, %[[LOAD1]], %[[LOAD2]] {{.*}} : f32
+! CHECK:     %[[SELECT:.*]] = arith.select %[[CMP]], %[[LOAD1]], %[[LOAD2]] : f32
+! CHECK:     fir.store %[[SELECT]] to %[[COORD1]] : !fir.ref<f32>
+! CHECK:   }
+! CHECK:   acc.yield %[[ARG0]] : !fir.ref<!fir.array<100xf32>>
+! CHECK: }
+
+! CHECK-LABEL: acc.reduction.recipe @reduction_max_ref_f32 : !fir.ref<f32> reduction_operator <max> init {
+! CHECK: ^bb0(%{{.*}}: !fir.ref<f32>):
+! CHECK:   %[[INIT:.*]] = arith.constant -1.401300e-45 : f32
+! CHECK:   %[[ALLOCA:.*]] = fir.alloca f32
+! CHECK:   %[[DECLARE:.*]]:2 = hlfir.declare %0 {uniq_name = "acc.reduction.init"} : (!fir.ref<f32>) -> (!fir.ref<f32>, !fir.ref<f32>)
+! CHECK:   fir.store %[[INIT]] to %[[DECLARE]]#0 : !fir.ref<f32>
+! CHECK:   acc.yield %[[DECLARE]]#0 : !fir.ref<f32>
+! CHECK: } combiner {
+! CHECK: ^bb0(%[[ARG0:.*]]: !fir.ref<f32>, %[[ARG1:.*]]: !fir.ref<f32>):
+! CHECK:   %[[LOAD0:.*]] = fir.load %[[ARG0]] : !fir.ref<f32>
+! CHECK:   %[[LOAD1:.*]] = fir.load %[[ARG1]] : !fir.ref<f32>
+! CHECK:   %[[CMP:.*]] = arith.cmpf ogt, %[[LOAD0]], %[[LOAD1]] {{.*}} : f32
+! CHECK:   %[[SELECT:.*]] = arith.select %[[CMP]], %[[LOAD0]], %[[LOAD1]] : f32
+! CHECK:   fir.store %[[SELECT]] to %[[ARG0]] : !fir.ref<f32>
+! CHECK:   acc.yield %[[ARG0]] : !fir.ref<f32>
+! CHECK: }
+
+! CHECK-LABEL: acc.reduction.recipe @reduction_max_section_ext100xext10_ref_100x10xi32 : !fir.ref<!fir.array<100x10xi32>> reduction_operator <max> init {
+! CHECK: ^bb0(%arg0: !fir.ref<!fir.array<100x10xi32>>):
+! CHECK:   %[[INIT:.*]] = arith.constant -2147483648 : i32
+! CHECK:   %[[SHAPE:.*]] = fir.shape %{{.*}}, %{{.*}} : (index, index) -> !fir.shape<2>
+! CHECK:   %[[ALLOCA:.*]] = fir.alloca !fir.array<100x10xi32>
+! CHECK:   %[[DECLARE:.*]]:2 = hlfir.declare %[[ALLOCA]](%[[SHAPE]]) {uniq_name = "acc.reduction.init"} : (!fir.ref<!fir.array<100x10xi32>>, !fir.shape<2>) -> (!fir.ref<!fir.array<100x10xi32>>, !fir.ref<!fir.array<100x10xi32>>)
+! CHECK:   acc.yield %[[DECLARE]]#0 : !fir.ref<!fir.array<100x10xi32>>
+! CHECK: } combiner {
+! CHECK: ^bb0(%[[ARG0:.*]]: !fir.ref<!fir.array<100x10xi32>>, %[[ARG1:.*]]: !fir.ref<!fir.array<100x10xi32>>):
+! CHECK:   %[[LB0:.*]] = arith.constant 0 : index
+! CHECK:   %[[UB0:.*]] = arith.constant 9 : index
+! CHECK:   %[[STEP0:.*]] = arith.constant 1 : index
+! CHECK:   fir.do_loop %[[IV0:.*]] = %[[LB0]] to %[[UB0]] step %[[STEP0]] {
+! CHECK:     %[[LB1:.*]] = arith.constant 0 : index
+! CHECK:     %[[UB1:.*]] = arith.constant 99 : index
+! CHECK:     %[[STEP1:.*]] = arith.constant 1 : index
+! CHECK:     fir.do_loop %[[IV1:.*]] = %[[LB1]] to %[[UB1]] step %[[STEP1]] {
+! CHECK:       %[[COORD1:.*]] = fir.coordinate_of %[[ARG0:.*]], %[[IV0]], %[[IV1]] : (!fir.ref<!fir.array<100x10xi32>>, index, index) -> !fir.ref<i32>
+! CHECK:       %[[COORD2:.*]] = fir.coordinate_of %[[ARG1:.*]], %[[IV0]], %[[IV1]] : (!fir.ref<!fir.array<100x10xi32>>, index, index) -> !fir.ref<i32>
+! CHECK:       %[[LOAD1:.*]] = fir.load %[[COORD1]] : !fir.ref<i32>
+! CHECK:       %[[LOAD2:.*]] = fir.load %[[COORD2]] : !fir.ref<i32>
+! CHECK:       %[[CMP:.*]] = arith.cmpi sgt, %[[LOAD1]], %[[LOAD2]] : i32
+! CHECK:       %[[SELECT:.*]] = arith.select %[[CMP]], %[[LOAD1]], %[[LOAD2]] : i32
+! CHECK:       fir.store %[[SELECT]] to %[[COORD1]] : !fir.ref<i32>
+! CHECK:     }
+! CHECK:   }
+! CHECK:   acc.yield %[[ARG0]] : !fir.ref<!fir.array<100x10xi32>>
+! CHECK: }
+
+! CHECK-LABEL: acc.reduction.recipe @reduction_max_ref_i32 : !fir.ref<i32> reduction_operator <max> init {
+! CHECK: ^bb0(%arg0: !fir.ref<i32>):
+! CHECK:   %[[INIT:.*]] = arith.constant -2147483648 : i32
+! CHECK:   %[[ALLOCA:.*]] = fir.alloca i32
+! CHECK:   %[[DECLARE:.*]]:2 = hlfir.declare %[[ALLOCA]] {uniq_name = "acc.reduction.init"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+! CHECK:   fir.store %[[INIT]] to %[[DECLARE]]#0 : !fir.ref<i32>
+! CHECK:   acc.yield %[[DECLARE]]#0 : !fir.ref<i32>
+! CHECK: } combiner {
+! CHECK: ^bb0(%[[ARG0:.*]]: !fir.ref<i32>, %[[ARG1:.*]]: !fir.ref<i32>):
+! CHECK:   %[[LOAD0:.*]] = fir.load %[[ARG0]] : !fir.ref<i32>
+! CHECK:   %[[LOAD1:.*]] = fir.load %[[ARG1]] : !fir.ref<i32>
+! CHECK:   %[[CMP:.*]] = arith.cmpi sgt, %[[LOAD0]], %[[LOAD1]] : i32
+! CHECK:   %[[SELECT:.*]] = arith.select %[[CMP]], %[[LOAD0]], %[[LOAD1]] : i32
+! CHECK:   fir.store %[[SELECT]] to %[[ARG0]] : !fir.ref<i32>
+! CHECK:   acc.yield %[[ARG0]] : !fir.ref<i32>
+! CHECK: }
+
+! CHECK-LABEL: acc.reduction.recipe @reduction_min_section_ext100xext10_ref_100x10xf32 : !fir.ref<!fir.array<100x10xf32>> reduction_operator <min> init {
+! CHECK: ^bb0(%{{.*}}: !fir.ref<!fir.array<100x10xf32>>):
+! CHECK:   %[[INIT:.*]] = arith.constant 3.40282347E+38 : f32
+! CHECK:   %[[SHAPE:.*]] = fir.shape %{{.*}}, %{{.*}} : (index, index) -> !fir.shape<2>
+! CHECK:   %[[ALLOCA:.*]] = fir.alloca !fir.array<100x10xf32>
+! CHECK:   %[[DECLARE:.*]]:2 = hlfir.declare %[[ALLOCA]](%[[SHAPE]]) {uniq_name = "acc.reduction.init"} : (!fir.ref<!fir.array<100x10xf32>>, !fir.shape<2>) -> (!fir.ref<!fir.array<100x10xf32>>, !fir.ref<!fir.array<100x10xf32>>)
+! CHECK:   acc.yield %[[DECLARE]]#0 : !fir.ref<!fir.array<100x10xf32>>
+! CHECK: } combiner {
+! CHECK: ^bb0(%[[ARG0:.*]]: !fir.ref<!fir.array<100x10xf32>>, %[[ARG1:.*]]: !fir.ref<!fir.array<100x10xf32>>):
+! CHECK:   %[[LB0:.*]] = arith.constant 0 : index
+! CHECK:   %[[UB0:.*]] = arith.constant 9 : index
+! CHECK:   %[[STEP0:.*]] = arith.constant 1 : index
+! CHECK:   fir.do_loop %[[IV0:.*]] = %[[LB0]] to %[[UB0]] step %[[STEP0]] {
+! CHECK:     %[[LB1:.*]] = arith.constant 0 : index
+! CHECK:     %[[UB1:.*]] = arith.constant 99 : index
+! CHECK:     %[[STEP1:.*]] = arith.constant 1 : index
+! CHECK:     fir.do_loop %[[IV1:.*]] = %[[LB1]] to %[[UB1]] step %[[STEP1]] {
+! CHECK:       %[[COORD1:.*]] = fir.coordinate_of %[[ARG0]], %[[IV0]], %[[IV1]] : (!fir.ref<!fir.array<100x10xf32>>, index, index) -> !fir.ref<f32>
+! CHECK:       %[[COORD2:.*]] = fir.coordinate_of %[[ARG1]], %[[IV0]], %[[IV1]] : (!fir.ref<!fir.array<100x10xf32>>, index, index) -> !fir.ref<f32>
+! CHECK:       %[[LOAD1:.*]] = fir.load %[[COORD1]] : !fir.ref<f32>
+! CHECK:       %[[LOAD2:.*]] = fir.load %[[COORD2]] : !fir.ref<f32>
+! CHECK:       %[[CMP:.*]] = arith.cmpf olt, %[[LOAD1]], %[[LOAD2]] {{.*}} : f32
+! CHECK:       %[[SELECT:.*]] = arith.select %[[CMP]], %[[LOAD1]], %[[LOAD2]] : f32
+! CHECK:       fir.store %[[SELECT]] to %[[COORD1]] : !fir.ref<f32>
+! CHECK:     }
+! CHECK:   }
+! CHECK:   acc.yield %[[ARG0]] : !fir.ref<!fir.array<100x10xf32>>
+! CHECK: }
+
+! CHECK-LABEL: acc.reduction.recipe @reduction_min_ref_f32 : !fir.ref<f32> reduction_operator <min> init {
+! CHECK: ^bb0(%{{.*}}: !fir.ref<f32>):
+! CHECK:   %[[INIT:.*]] = arith.constant 3.40282347E+38 : f32
+! CHECK:   %[[ALLOCA:.*]] = fir.alloca f32
+! CHECK:   %[[DECLARE:.*]]:2 = hlfir.declare %[[ALLOCA]] {uniq_name = "acc.reduction.init"} : (!fir.ref<f32>) -> (!fir.ref<f32>, !fir.ref<f32>)
+! CHECK:   fir.store %[[INIT]] to %[[DECLARE]]#0 : !fir.ref<f32>
+! CHECK:   acc.yield %[[DECLARE]]#0 : !fir.ref<f32>
+! CHECK: } combiner {
+! CHECK: ^bb0(%[[ARG0:.*]]: !fir.ref<f32>, %[[ARG1:.*]]: !fir.ref<f32>):
+! CHECK:   %[[LOAD0:.*]] = fir.load %[[ARG0]] : !fir.ref<f32>
+! CHECK:   %[[LOAD1:.*]] = fir.load %[[ARG1]] : !fir.ref<f32>
+! CHECK:   %[[CMP:.*]] = arith.cmpf olt, %[[LOAD0]], %[[LOAD1]] {{.*}} : f32
+! CHECK:   %[[SELECT:.*]] = arith.select %[[CMP]], %[[LOAD0]], %[[LOAD1]] : f32
+! CHECK:   fir.store %[[SELECT]] to %[[ARG0]] : !fir.ref<f32>
+! CHECK:   acc.yield %[[ARG0]] : !fir.ref<f32>
+! CHECK: }
+
+! CHECK-LABEL: acc.reduction.recipe @reduction_min_section_ext100_ref_100xi32 : !fir.ref<!fir.array<100xi32>> reduction_operator <min> init {
+! CHECK: ^bb0(%{{.*}}: !fir.ref<!fir.array<100xi32>>):
+! CHECK:   %[[INIT:.*]] = arith.constant 2147483647 : i32
+! CHECK:   %[[SHAPE:.*]] = fir.shape %{{.*}} : (index) -> !fir.shape<1>
+! CHECK:   %[[ALLOCA:.*]] = fir.alloca !fir.array<100xi32>
+! CHECK:   %[[DECLARE:.*]]:2 = hlfir.declare %[[ALLOCA]](%[[SHAPE]]) {uniq_name = "acc.reduction.init"} : (!fir.ref<!fir.array<100xi32>>, !fir.shape<1>) -> (!fir.ref<!fir.array<100xi32>>, !fir.ref<!fir.array<100xi32>>)
+! CHECK:   acc.yield %[[DECLARE]]#0 : !fir.ref<!fir.array<100xi32>>
+! CHECK: } combiner {
+! CHECK: ^bb0(%[[ARG0:.*]]: !fir.ref<!fir.array<100xi32>>, %[[ARG1:.*]]: !fir.ref<!fir.array<100xi32>>):
+! CHECK:   %[[LB0:.*]] = arith.constant 0 : index
+! CHECK:   %[[UB0:.*]] = arith.constant 99 : index
+! CHECK:   %[[STEP0:.*]] = arith.constant 1 : index
+! CHECK:   fir.do_loop %[[IV0:.*]] = %[[LB0]] to %[[UB0]] step %[[STEP0]] {
+! CHECK:     %[[COORD1:.*]] = fir.coordinate_of %[[ARG0]], %[[IV0]] : (!fir.ref<!fir.array<100xi32>>, index) -> !fir.ref<i32>
+! CHECK:     %[[COORD2:.*]] = fir.coordinate_of %[[ARG1]], %[[IV0]] : (!fir.ref<!fir.array<100xi32>>, index) -> !fir.ref<i32>
+! CHECK:     %[[LOAD1:.*]] = fir.load %[[COORD1]] : !fir.ref<i32>
+! CHECK:     %[[LOAD2:.*]] = fir.load %[[COORD2]] : !fir.ref<i32>
+! CHECK:     %[[CMP:.*]] = arith.cmpi slt, %[[LOAD1]], %[[LOAD2]] : i32
+! CHECK:     %[[SELECT:.*]] = arith.select %[[CMP]], %[[LOAD1]], %[[LOAD2]] : i32
+! CHECK:     fir.store %[[SELECT]] to %[[COORD1]] : !fir.ref<i32>
+! CHECK:   }
+! CHECK:   acc.yield %[[ARG0]] : !fir.ref<!fir.array<100xi32>>
+! CHECK: }
+
+! CHECK-LABEL: acc.reduction.recipe @reduction_min_ref_i32 : !fir.ref<i32> reduction_operator <min> init {
+! CHECK: ^bb0(%{{.*}}: !fir.ref<i32>):
+! CHECK:   %[[INIT:.*]] = arith.constant 2147483647 : i32
+! CHECK:   %[[ALLOCA:.*]] = fir.alloca i32
+! CHECK:   %[[DECLARE:.*]]:2 = hlfir.declare %[[ALLOCA]] {uniq_name = "acc.reduction.init"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+! CHECK:   fir.store %[[INIT]] to %[[DECLARE]]#0 : !fir.ref<i32>
+! CHECK:   acc.yield %[[DECLARE]]#0 : !fir.ref<i32>
+! CHECK: } combiner {
+! CHECK: ^bb0(%[[ARG0:.*]]: !fir.ref<i32>, %[[ARG1:.*]]: !fir.ref<i32>):
+! CHECK:   %[[LOAD0:.*]] = fir.load %[[ARG0]] : !fir.ref<i32>
+! CHECK:   %[[LOAD1:.*]] = fir.load %[[ARG1]] : !fir.ref<i32>
+! CHECK:   %[[CMP:.*]] = arith.cmpi slt, %[[LOAD0]], %[[LOAD1]] : i32
+! CHECK:   %[[SELECT:.*]] = arith.select %[[CMP]], %[[LOAD0]], %[[LOAD1]] : i32
+! CHECK:   fir.store %[[SELECT]] to %[[ARG0]] : !fir.ref<i32>
+! CHECK:   acc.yield %[[ARG0]] : !fir.ref<i32>
+! CHECK: }
+
+! CHECK-LABEL: acc.reduction.recipe @reduction_mul_ref_f32 : !fir.ref<f32> reduction_operator <mul> init {
+! CHECK: ^bb0(%{{.*}}: !fir.ref<f32>):
+! CHECK:   %[[INIT:.*]] = arith.constant 1.000000e+00 : f32
+! CHECK:   %[[ALLOCA:.*]] = fir.alloca f32
+! CHECK:   %[[DECLARE:.*]]:2 = hlfir.declare %[[ALLOCA]] {uniq_name = "acc.reduction.init"} : (!fir.ref<f32>) -> (!fir.ref<f32>, !fir.ref<f32>)
+! CHECK:   fir.store %[[INIT]] to %[[DECLARE]]#0 : !fir.ref<f32>
+! CHECK:   acc.yield %[[DECLARE]]#0 : !fir.ref<f32>
+! CHECK: } combiner {
+! CHECK: ^bb0(%[[ARG0:.*]]: !fir.ref<f32>, %[[ARG1:.*]]: !fir.ref<f32>):
+! CHECK:   %[[LOAD0:.*]] = fir.load %[[ARG0]] : !fir.ref<f32>
+! CHECK:   %[[LOAD1:.*]] = fir.load %[[ARG1]] : !fir.ref<f32>
+! CHECK:   %[[COMBINED:.*]] = arith.mulf %[[LOAD0]], %[[LOAD1]] fastmath<contract> : f32
+! CHECK:   fir.store %[[COMBINED]] to %[[ARG0]] : !fir.ref<f32>
+! CHECK:   acc.yield %[[ARG0]] : !fir.ref<f32>
+! CHECK: }
+
+! CHECK-LABEL: acc.reduction.recipe @reduction_mul_section_ext100_ref_100xi32 : !fir.ref<!fir.array<100xi32>> reduction_operator <mul> init {
+! CHECK: ^bb0(%{{.*}}: !fir.ref<!fir.array<100xi32>>):
+! CHECK:   %[[INIT:.*]] = arith.constant 1 : i32
+! CHECK:   %[[SHAPE:.*]] = fir.shape %{{.*}} : (index) -> !fir.shape<1>
+! CHECK:   %[[ALLOCA:.*]] = fir.alloca !fir.array<100xi32>
+! CHECK:   %[[DECLARE:.*]]:2 = hlfir.declare %[[ALLOCA]](%[[SHAPE]]) {uniq_name = "acc.reduction.init"} : (!fir.ref<!fir.array<100xi32>>, !fir.shape<1>) -> (!fir.ref<!fir.array<100xi32>>, !fir.ref<!fir.array<100xi32>>)
+! CHECK:   acc.yield %[[DECLARE]]#0 : !fir.ref<!fir.array<100xi32>>
+! CHECK: } combiner {
+! CHECK: ^bb0(%[[ARG0:.*]]: !fir.ref<!fir.array<100xi32>>, %[[ARG1:.*]]: !fir.ref<!fir.array<100xi32>>):
+! CHECK:   %[[LB:.*]] = arith.constant 0 : index
+! CHECK:   %[[UB:.*]] = arith.constant 99 : index
+! CHECK:   %[[STEP:.*]] = arith.constant 1 : index
+! CHECK:   fir.do_loop %[[IV:.*]] = %[[LB]] to %[[UB]] step %[[STEP]] {
+! CHECK:     %[[COORD1:.*]] = fir.coordinate_of %[[ARG0]], %[[IV]] : (!fir.ref<!fir.array<100xi32>>, index) -> !fir.ref<i32>
+! CHECK:     %[[COORD2:.*]] = fir.coordinate_of %[[ARG1]], %[[IV]] : (!fir.ref<!fir.array<100xi32>>, index) -> !fir.ref<i32>
+! CHECK:     %[[LOAD1:.*]] = fir.load %[[COORD1]] : !fir.ref<i32>
+! CHECK:     %[[LOAD2:.*]] = fir.load %[[COORD2]] : !fir.ref<i32>
+! CHECK:     %[[COMBINED:.*]] = arith.muli %[[LOAD1]], %[[LOAD2]] : i32
+! CHECK:     fir.store %[[COMBINED]] to %[[COORD1]] : !fir.ref<i32>
+! CHECK:   }
+! CHECK:   acc.yield %[[ARG0]] : !fir.ref<!fir.array<100xi32>>
+! CHECK: }
+
+! CHECK-LABEL: acc.reduction.recipe @reduction_mul_ref_i32 : !fir.ref<i32> reduction_operator <mul> init {
+! CHECK: ^bb0(%{{.*}}: !fir.ref<i32>):
+! CHECK:   %[[INIT:.*]] = arith.constant 1 : i32
+! CHECK:   %[[ALLOCA:.*]] = fir.alloca i32
+! CHECK:   %[[DECLARE:.*]]:2 = hlfir.declare %[[ALLOCA]] {uniq_name = "acc.reduction.init"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+! CHECK:   fir.store %[[INIT]] to %[[DECLARE]]#0 : !fir.ref<i32>
+! CHECK:   acc.yield %[[DECLARE]]#0 : !fir.ref<i32>
+! CHECK: } combiner {
+! CHECK: ^bb0(%[[ARG0:.*]]: !fir.ref<i32>, %[[ARG1:.*]]: !fir.ref<i32>):
+! CHECK:   %[[LOAD0:.*]] = fir.load %[[ARG0]] : !fir.ref<i32>
+! CHECK:   %[[LOAD1:.*]] = fir.load %[[ARG1]] : !fir.ref<i32>
+! CHECK:   %[[COMBINED:.*]] = arith.muli %[[LOAD0]], %[[LOAD1]] : i32
+! CHECK:   fir.store %[[COMBINED]] to %[[ARG0]] : !fir.ref<i32>
+! CHECK:   acc.yield %[[ARG0]] : !fir.ref<i32>
+! CHECK: }
+
+! CHECK-LABEL: acc.reduction.recipe @reduction_add_section_ext100_ref_100xf32 : !fir.ref<!fir.array<100xf32>> reduction_operator <add> init {
+! CHECK: ^bb0(%{{.*}}: !fir.ref<!fir.array<100xf32>>):
+! CHECK:   %[[INIT:.*]] = arith.constant 0.000000e+00 : f32
+! CHECK:   %[[SHAPE:.*]] = fir.shape %{{.*}} : (index) -> !fir.shape<1>
+! CHECK:   %[[ALLOCA:.*]] = fir.alloca !fir.array<100xf32>
+! CHECK:   %[[DECLARE:.*]]:2 = hlfir.declare %[[ALLOCA]](%[[SHAPE]]) {uniq_name = "acc.reduction.init"} : (!fir.ref<!fir.array<100xf32>>, !fir.shape<1>) -> (!fir.ref<!fir.array<100xf32>>, !fir.ref<!fir.array<100xf32>>)
+! CHECK:   acc.yield %[[DECLARE]]#0 : !fir.ref<!fir.array<100xf32>>
+! CHECK: } combiner {
+! CHECK: ^bb0(%[[ARG0:.*]]: !fir.ref<!fir.array<100xf32>>, %[[ARG1:.*]]: !fir.ref<!fir.array<100xf32>>):
+! CHECK:   %[[LB:.*]] = arith.constant 0 : index
+! CHECK:   %[[UB:.*]] = arith.constant 99 : index
+! CHECK:   %[[STEP:.*]] = arith.constant 1 : index
+! CHECK:   fir.do_loop %[[IV:.*]] = %[[LB]] to %[[UB]] step %[[STEP]] {
+! CHECK:   %[[COORD1:.*]] = fir.coordinate_of %[[ARG0]], %[[IV]] : (!fir.ref<!fir.array<100xf32>>, index) -> !fir.ref<f32>
+! CHECK:   %[[COORD2:.*]] = fir.coordinate_of %[[ARG1]], %[[IV]] : (!fir.ref<!fir.array<100xf32>>, index) -> !fir.ref<f32>
+! CHECK:   %[[LOAD1:.*]] = fir.load %[[COORD1]] : !fir.ref<f32>
+! CHECK:   %[[LOAD2:.*]] = fir.load %[[COORD2]] : !fir.ref<f32>
+! CHECK:   %[[COMBINED:.*]] = arith.addf %[[LOAD1]], %[[LOAD2]] fastmath<contract> : f32
+! CHECK:   fir.store %[[COMBINED]] to %[[COORD1]] : !fir.ref<f32>
+! CHECK:   }
+! CHECK:   acc.yield %[[ARG0]] : !fir.ref<!fir.array<100xf32>>
+! CHECK: }
+
+! CHECK-LABEL: acc.reduction.recipe @reduction_add_ref_f32 : !fir.ref<f32> reduction_operator <add> init {
+! CHECK: ^bb0(%{{.*}}: !fir.ref<f32>):
+! CHECK:   %[[INIT:.*]] = arith.constant 0.000000e+00 : f32
+! CHECK:   %[[ALLOCA:.*]] = fir.alloca f32
+! CHECK:   %[[DECLARE:.*]]:2 = hlfir.declare %[[ALLOCA]] {uniq_name = "acc.reduction.init"} : (!fir.ref<f32>) -> (!fir.ref<f32>, !fir.ref<f32>)
+! CHECK:   fir.store %[[INIT]] to %[[DECLARE]]#0 : !fir.ref<f32>
+! CHECK:   acc.yield %[[DECLARE]]#0 : !fir.ref<f32>
+! CHECK: } combiner {
+! CHECK: ^bb0(%[[ARG0:.*]]: !fir.ref<f32>, %[[ARG1:.*]]: !fir.ref<f32>):
+! CHECK:   %[[LOAD0:.*]] = fir.load %[[ARG0]] : !fir.ref<f32>
+! CHECK:   %[[LOAD1:.*]] = fir.load %[[ARG1]] : !fir.ref<f32>
+! CHECK:   %[[COMBINED:.*]] = arith.addf %[[LOAD0]], %[[LOAD1]] fastmath<contract> : f32
+! CHECK:   fir.store %[[COMBINED]] to %[[ARG0]] : !fir.ref<f32>
+! CHECK:   acc.yield %[[ARG0]] : !fir.ref<f32>
+! CHECK: }
+
+! CHECK-LABEL: acc.reduction.recipe @reduction_add_section_ext100xext10xext2_ref_100x10x2xi32 : !fir.ref<!fir.array<100x10x2xi32>> reduction_operator <add> init {
+! CHECK: ^bb0(%{{.*}}: !fir.ref<!fir.array<100x10x2xi32>>):
+! CHECK:   %[[INIT:.*]] = arith.constant 0 : i32
+! CHECK:   %[[SHAPE:.*]] = fir.shape %{{.*}}, %{{.*}}, %{{.*}} : (index, index, index) -> !fir.shape<3>
+! CHECK:   %[[ALLOCA:.*]] = fir.alloca !fir.array<100x10x2xi32>
+! CHECK:   %[[DECLARE:.*]]:2 = hlfir.declare %[[ALLOCA]](%[[SHAPE]]) {uniq_name = "acc.reduction.init"} : (!fir.ref<!fir.array<100x10x2xi32>>, !fir.shape<3>) -> (!fir.ref<!fir.array<100x10x2xi32>>, !fir.ref<!fir.array<100x10x2xi32>>)
+! CHECK:   acc.yield %[[DECLARE]]#0 : !fir.ref<!fir.array<100x10x2xi32>>
+! CHECK: } combiner {
+! CHECK: ^bb0(%[[ARG0:.*]]: !fir.ref<!fir.array<100x10x2xi32>>, %[[ARG1:.*]]: !fir.ref<!fir.array<100x10x2xi32>>):
+! CHECK:   %[[LB0:.*]] = arith.constant 0 : index
+! CHECK:   %[[UB0:.*]] = arith.constant 1 : index
+! CHECK:   %[[STEP0:.*]] = arith.constant 1 : index
+! CHECK:   fir.do_loop %[[IV0:.*]] = %[[LB0]] to %[[UB0]] step %[[STEP0]] {
+! CHECK:     %[[LB1:.*]] = arith.constant 0 : index
+! CHECK:     %[[UB1:.*]] = arith.constant 9 : index
+! CHECK:     %[[STEP1:.*]] = arith.constant 1 : index
+! CHECK:     fir.do_loop %[[IV1:.*]] = %[[LB1]] to %[[UB1]] step %[[STEP1]] {
+! CHECK:       %[[LB2:.*]] = arith.constant 0 : index
+! CHECK:       %[[UB2:.*]] = arith.constant 99 : index
+! CHECK:       %[[STEP2:.*]] = arith.constant 1 : index
+! CHECK:       fir.do_loop %[[IV2:.*]] = %[[LB2]] to %[[UB2]] step %[[STEP2]] {
+! CHECK:         %[[COORD1:.*]] = fir.coordinate_of %[[ARG0]], %[[IV0]], %[[IV1]], %[[IV2]] : (!fir.ref<!fir.array<100x10x2xi32>>, index, index, index) -> !fir.ref<i32>
+! CHECK:         %[[COORD2:.*]] = fir.coordinate_of %[[ARG1]], %[[IV0]], %[[IV1]], %[[IV2]] : (!fir.ref<!fir.array<100x10x2xi32>>, index, index, index) -> !fir.ref<i32>
+! CHECK:         %[[LOAD1:.*]] = fir.load %[[COORD1]] : !fir.ref<i32>
+! CHECK:         %[[LOAD2:.*]] = fir.load %[[COORD2]] : !fir.ref<i32>
+! CHECK:         %[[COMBINED:.*]] = arith.addi %[[LOAD1]], %[[LOAD2]] : i32
+! CHECK:         fir.store %[[COMBINED]] to %[[COORD1]] : !fir.ref<i32>
+! CHECK:       }
+! CHECK:     }
+! CHECK:   }
+! CHECK:   acc.yield %[[ARG0]] : !fir.ref<!fir.array<100x10x2xi32>>
+! CHECK: }
+
+! CHECK-LABEL: acc.reduction.recipe @reduction_add_section_ext100xext10_ref_100x10xi32 : !fir.ref<!fir.array<100x10xi32>> reduction_operator <add> init {
+! CHECK: ^bb0(%{{.*}}: !fir.ref<!fir.array<100x10xi32>>):
+! CHECK:   %[[INIT:.*]] = arith.constant 0 : i32
+! CHECK:   %[[SHAPE:.*]] = fir.shape %{{.*}}, %{{.*}} : (index, index) -> !fir.shape<2>
+! CHECK:   %[[ALLOCA:.*]] = fir.alloca !fir.array<100x10xi32>
+! CHECK:   %[[DECLARE:.*]]:2 = hlfir.declare %[[ALLOCA]](%[[SHAPE]]) {uniq_name = "acc.reduction.init"} : (!fir.ref<!fir.array<100x10xi32>>, !fir.shape<2>) -> (!fir.ref<!fir.array<100x10xi32>>, !fir.ref<!fir.array<100x10xi32>>)
+! CHECK:   acc.yield %[[DECLARE]]#0 : !fir.ref<!fir.array<100x10xi32>>
+! CHECK: } combiner {
+! CHECK: ^bb0(%[[ARG0:.*]]: !fir.ref<!fir.array<100x10xi32>>, %[[ARG1:.*]]: !fir.ref<!fir.array<100x10xi32>>):
+! CHECK:   %[[LB0:.*]] = arith.constant 0 : index
+! CHECK:   %[[UB0:.*]] = arith.constant 9 : index
+! CHECK:   %[[STEP0:.*]] = arith.constant 1 : index
+! CHECK:   fir.do_loop %[[IV0:.*]] = %[[LB0]] to %[[UB0]] step %[[STEP0]] {
+! CHECK:     %[[LB1:.*]] = arith.constant 0 : index
+! CHECK:     %[[UB1:.*]] = arith.constant 99 : index
+! CHECK:     %[[STEP1:.*]] = arith.constant 1 : index
+! CHECK:     fir.do_loop %[[IV1:.*]] = %[[LB1]] to %[[UB1]] step %[[STEP1]] {
+! CHECK:       %[[COORD1:.*]] = fir.coordinate_of %[[ARG0]], %[[IV0]], %[[IV1]] : (!fir.ref<!fir.array<100x10xi32>>, index, index) -> !fir.ref<i32>
+! CHECK:       %[[COORD2:.*]] = fir.coordinate_of %[[ARG1]], %[[IV0]], %[[IV1]] : (!fir.ref<!fir.array<100x10xi32>>, index, index) -> !fir.ref<i32>
+! CHECK:       %[[LOAD1]] = fir.load %[[COORD1]] : !fir.ref<i32>
+! CHECK:       %[[LOAD2]] = fir.load %[[COORD2]] : !fir.ref<i32>
+! CHECK:       %[[COMBINED:.*]] = arith.addi %[[LOAD1]], %[[LOAD2]] : i32
+! CHECK:       fir.store %[[COMBINED]] to %[[COORD1]] : !fir.ref<i32>
+! CHECK:     }
+! CHECK:   }
+! CHECK:   acc.yield %[[ARG0]] : !fir.ref<!fir.array<100x10xi32>>
+! CHECK: }
+
+! CHECK-LABEL: acc.reduction.recipe @reduction_add_section_ext100_ref_100xi32 : !fir.ref<!fir.array<100xi32>> reduction_operator <add> init {
+! CHECK: ^bb0(%{{.*}}: !fir.ref<!fir.array<100xi32>>):
+! CHECK:   %[[INIT:.*]] = arith.constant 0 : i32
+! CHECK:   %[[SHAPE:.*]] = fir.shape %{{.*}} : (index) -> !fir.shape<1>
+! CHECK:   %[[ALLOCA:.*]] = fir.alloca !fir.array<100xi32>
+! CHECK:   %[[DECLARE:.*]]:2 = hlfir.declare %[[ALLOCA]](%[[SHAPE]]) {uniq_name = "acc.reduction.init"} : (!fir.ref<!fir.array<100xi32>>, !fir.shape<1>) -> (!fir.ref<!fir.array<100xi32>>, !fir.ref<!fir.array<100xi32>>)
+! HFLIR:   acc.yield %[[DECLARE]]#0 : !fir.ref<!fir.array<100xi32>>
+! CHECK: } combiner {
+! CHECK: ^bb0(%[[ARG0:.*]]: !fir.ref<!fir.array<100xi32>>, %[[ARG1:.*]]: !fir.ref<!fir.array<100xi32>>):
+! CHECK:   %[[LB:.*]] = arith.constant 0 : index
+! CHECK:   %[[UB:.*]] = arith.constant 99 : index
+! CHECK:   %[[STEP:.*]] = arith.constant 1 : index
+! CHECK:   fir.do_loop %[[IV:.*]] = %[[LB]] to %[[UB]] step %[[STEP]] {
+! CHECK:     %[[COORD1:.*]] = fir.coordinate_of %[[ARG0]], %[[IV]] : (!fir.ref<!fir.array<100xi32>>, index) -> !fir.ref<i32>
+! CHECK:     %[[COORD2:.*]] = fir.coordinate_of %[[ARG1]], %[[IV]] : (!fir.ref<!fir.array<100xi32>>, index) -> !fir.ref<i32>
+! CHECK:     %[[LOAD1:.*]] = fir.load %[[COORD1]] : !fir.ref<i32>
+! CHECK:     %[[LOAD2:.*]] = fir.load %[[COORD2]] : !fir.ref<i32>
+! CHECK:     %[[COMBINED:.*]] = arith.addi %[[LOAD1]], %[[LOAD2]] : i32
+! CHECK:     fir.store %[[COMBINED]] to %[[COORD1]] : !fir.ref<i32>
+! CHECK:   }
+! CHECK:   acc.yield %[[ARG0]] : !fir.ref<!fir.array<100xi32>>
+! CHECK: }
+
+! CHECK-LABEL: acc.reduction.recipe @reduction_add_ref_i32 : !fir.ref<i32> reduction_operator <add> init {
+! CHECK: ^bb0(%{{.*}}: !fir.ref<i32>):
+! CHECK:   %[[INIT:.*]] = arith.constant 0 : i32
+! CHECK:   %[[ALLOCA:.*]] = fir.alloca i32
+! CHECK:   %[[DECLARE:.*]]:2 = hlfir.declare %[[ALLOCA]] {uniq_name = "acc.reduction.init"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+! CHECK:   fir.store %[[INIT]] to %[[DECLARE]]#0 : !fir.ref<i32>
+! CHECK:   acc.yield %[[DECLARE]]#0 : !fir.ref<i32>
+! CHECK: } combiner {
+! CHECK: ^bb0(%[[ARG0:.*]]: !fir.ref<i32>, %[[ARG1:.*]]: !fir.ref<i32>):
+! CHECK:   %[[LOAD0:.*]] = fir.load %[[ARG0]] : !fir.ref<i32>
+! CHECK:   %[[LOAD1:.*]] = fir.load %[[ARG1]] : !fir.ref<i32>
+! CHECK:   %[[COMBINED:.*]] = arith.addi %[[LOAD0]], %[[LOAD1]] : i32
+! CHECK:   fir.store %[[COMBINED]] to %[[ARG0]] : !fir.ref<i32>
+! CHECK:   acc.yield %[[ARG0]] : !fir.ref<i32>
+! CHECK: }
+
+subroutine acc_reduction_add_int(a, b)
+  integer :: a(100)
+  integer :: i, b
+
+  !$acc loop reduction(+:b)
+  do i = 1, 100
+    b = b + a(i)
+  end do
+end subroutine
+
+! CHECK-LABEL: func.func @_QPacc_reduction_add_int(
+! CHECK-SAME:  %{{.*}}: !fir.ref<!fir.array<100xi32>> {fir.bindc_name = "a"}, %[[B:.*]]: !fir.ref<i32> {fir.bindc_name = "b"})
+! CHECK:       %[[DECLB:.*]]:2 = hlfir.declare %[[B]]
+! CHECK:       %[[RED_B:.*]] = acc.reduction varPtr(%[[DECLB]]#0 : !fir.ref<i32>) -> !fir.ref<i32> {name = "b"}
+! CHECK:       acc.loop {{.*}} reduction(@reduction_add_ref_i32 -> %[[RED_B]] : !fir.ref<i32>)
+
+subroutine acc_reduction_add_int_array_1d(a, b)
+  integer :: a(100)
+  integer :: i, b(100)
+
+  !$acc loop reduction(+:b)
+  do i = 1, 100
+    b(i) = b(i) + a(i)
+  end do
+end subroutine
+
+! CHECK-LABEL: func.func @_QPacc_reduction_add_int_array_1d(
+! CHECK-SAME:  %{{.*}}: !fir.ref<!fir.array<100xi32>> {fir.bindc_name = "a"}, %[[B:.*]]: !fir.ref<!fir.array<100xi32>> {fir.bindc_name = "b"})
+! CHECK:       %[[DECLB:.*]]:2 = hlfir.declare %[[B]]
+! CHECK:       %[[RED_B:.*]] = acc.reduction varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<100xi32>>) bounds(%{{.*}}) -> !fir.ref<!fir.array<100xi32>> {name = "b"}
+! CHECK:       acc.loop {{.*}} reduction(@reduction_add_section_ext100_ref_100xi32 -> %[[RED_B]] : !fir.ref<!fir.array<100xi32>>)
+
+subroutine acc_reduction_add_int_array_2d(a, b)
+  integer :: a(100, 10), b(100, 10)
+  integer :: i, j
+
+  !$acc loop collapse(2) reduction(+:b)
+  do i = 1, 100
+    do j = 1, 10
+      b(i, j) = b(i, j) + a(i, j)
+    end do
+  end do
+end subroutine
+
+! CHECK-LABEL: func.func @_QPacc_reduction_add_int_array_2d(
+! CHECK-SAME:  %[[ARG0:.*]]: !fir.ref<!fir.array<100x10xi32>> {fir.bindc_name = "a"}, %[[ARG1:.*]]: !fir.ref<!fir.array<100x10xi32>> {fir.bindc_name = "b"}) {
+! CHECK:       %[[DECLARG1:.*]]:2 = hlfir.declare %[[ARG1]]
+! CHECK:       %[[RED_ARG1:.*]] = acc.reduction varPtr(%[[DECLARG1]]#0 : !fir.ref<!fir.array<100x10xi32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<100x10xi32>> {name = "b"}
+! CHECK:       acc.loop {{.*}} reduction(@reduction_add_section_ext100xext10_ref_100x10xi32 -> %[[RED_ARG1]] : !fir.ref<!fir.array<100x10xi32>>)
+! CHECK: } attributes {collapse = [2]{{.*}}
+
+subroutine acc_reduction_add_int_array_3d(a, b)
+  integer :: a(100, 10, 2), b(100, 10, 2)
+  integer :: i, j, k
+
+  !$acc loop collapse(3) reduction(+:b)
+  do i = 1, 100
+    do j = 1, 10
+      do k = 1, 2
+        b(i, j, k) = b(i, j, k) + a(i, j, k)
+      end do
+    end do
+  end do
+end subroutine
+
+! CHECK-LABEL: func.func @_QPacc_reduction_add_int_array_3d(
+! CHECK-SAME: %{{.*}}: !fir.ref<!fir.array<100x10x2xi32>> {fir.bindc_name = "a"}, %[[ARG1:.*]]: !fir.ref<!fir.array<100x10x2xi32>> {fir.bindc_name = "b"})
+! CHECK: %[[DECLARG1:.*]]:2 = hlfir.declare %[[ARG1]]
+! CHECK: %[[RED_ARG1:.*]] = acc.reduction varPtr(%[[DECLARG1]]#0 : !fir.ref<!fir.array<100x10x2xi32>>) bounds(%{{.*}}, %{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<100x10x2xi32>> {name = "b"}
+! CHECK: acc.loop {{.*}} reduction(@reduction_add_section_ext100xext10xext2_ref_100x10x2xi32 -> %[[RED_ARG1]] : !fir.ref<!fir.array<100x10x2xi32>>)
+! CHECK: } attributes {collapse = [3]{{.*}}
+
+subroutine acc_reduction_add_float(a, b)
+  real :: a(100), b
+  integer :: i
+
+  !$acc loop reduction(+:b)
+  do i = 1, 100
+    b = b + a(i)
+  end do
+end subroutine
+
+! CHECK-LABEL: func.func @_QPacc_reduction_add_float(
+! CHECK-SAME:  %{{.*}}: !fir.ref<!fir.array<100xf32>> {fir.bindc_name = "a"}, %[[B:.*]]: !fir.ref<f32> {fir.bindc_name = "b"})
+! CHECK:       %[[DECLB:.*]]:2 = hlfir.declare %[[B]]
+! CHECK:       %[[RED_B:.*]] = acc.reduction varPtr(%[[DECLB]]#0 : !fir.ref<f32>) -> !fir.ref<f32> {name = "b"}
+! CHECK:       acc.loop {{.*}} reduction(@reduction_add_ref_f32 -> %[[RED_B]] : !fir.ref<f32>)
+
+subroutine acc_reduction_add_float_array_1d(a, b)
+  real :: a(100), b(100)
+  integer :: i
+
+  !$acc loop reduction(+:b)
+  do i = 1, 100
+    b(i) = b(i) + a(i)
+  end do
+end subroutine
+
+! CHECK-LABEL: func.func @_QPacc_reduction_add_float_array_1d(
+! CHECK-SAME:  %{{.*}}: !fir.ref<!fir.array<100xf32>> {fir.bindc_name = "a"}, %[[B:.*]]: !fir.ref<!fir.array<100xf32>> {fir.bindc_name = "b"})
+! CHECK: %[[DECLB:.*]]:2 = hlfir.declare %[[B]]
+! CHECK: %[[RED_B:.*]] = acc.reduction varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<100xf32>>) bounds(%{{.*}}) -> !fir.ref<!fir.array<100xf32>> {name = "b"}
+! CHECK:       acc.loop {{.*}} reduction(@reduction_add_section_ext100_ref_100xf32 -> %[[RED_B]] : !fir.ref<!fir.array<100xf32>>)
+
+subroutine acc_reduction_mul_int(a, b)
+  integer :: a(100)
+  integer :: i, b
+
+  !$acc loop reduction(*:b)
+  do i = 1, 100
+    b = b * a(i)
+  end do
+end subroutine
+
+! CHECK-LABEL: func.func @_QPacc_reduction_mul_int(
+! CHECK-SAME:  %{{.*}}: !fir.ref<!fir.array<100xi32>> {fir.bindc_name = "a"}, %[[B:.*]]: !fir.ref<i32> {fir.bindc_name = "b"})
+! CHECK:       %[[DECLB:.*]]:2 = hlfir.declare %[[B]]
+! CHECK:       %[[RED_B:.*]] = acc.reduction varPtr(%[[DECLB]]#0 : !fir.ref<i32>) -> !fir.ref<i32> {name = "b"}
+! CHECK:       acc.loop {{.*}} reduction(@reduction_mul_ref_i32 -> %[[RED_B]] : !fir.ref<i32>)
+
+subroutine acc_reduction_mul_int_array_1d(a, b)
+  integer :: a(100)
+  integer :: i, b(100)
+
+  !$acc loop reduction(*:b)
+  do i = 1, 100
+    b(i) = b(i) * a(i)
+  end do
+end subroutine
+
+! CHECK-LABEL: func.func @_QPacc_reduction_mul_int_array_1d(
+! CHECK-SAME:  %{{.*}}: !fir.ref<!fir.array<100xi32>> {fir.bindc_name = "a"}, %[[B:.*]]: !fir.ref<!fir.array<100xi32>> {fir.bindc_name = "b"})
+! CHECK:       %[[DECLB:.*]]:2 = hlfir.declare %[[B]]
+! CHECK:       %[[RED_B:.*]] = acc.reduction varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<100xi32>>) bounds(%{{.*}}) -> !fir.ref<!fir.array<100xi32>> {name = "b"}
+! CHECK:       acc.loop {{.*}} reduction(@reduction_mul_section_ext100_ref_100xi32 -> %[[RED_B]] : !fir.ref<!fir.array<100xi32>>)
+
+subroutine acc_reduction_mul_float(a, b)
+  real :: a(100), b
+  integer :: i
+
+  !$acc loop reduction(*:b)
+  do i = 1, 100
+    b = b * a(i)
+  end do
+end subroutine
+
+! CHECK-LABEL: func.func @_QPacc_reduction_mul_float(
+! CHECK-SAME:  %{{.*}}: !fir.ref<!fir.array<100xf32>> {fir.bindc_name = "a"}, %[[B:.*]]: !fir.ref<f32> {fir.bindc_name = "b"})
+! CHECK:       %[[DECLB:.*]]:2 = hlfir.declare %[[B]]
+! CHECK:       %[[RED_B:.*]] = acc.reduction varPtr(%[[DECLB]]#0 : !fir.ref<f32>) -> !fir.ref<f32> {name = "b"}
+! CHECK:       acc.loop {{.*}} reduction(@reduction_mul_ref_f32 -> %[[RED_B]] : !fir.ref<f32>)
+
+subroutine acc_reduction_mul_float_array_1d(a, b)
+  real :: a(100), b(100)
+  integer :: i
+
+  !$acc loop reduction(*:b)
+  do i = 1, 100
+    b(i) = b(i) * a(i)
+  end do
+end subroutine
+
+! CHECK-LABEL: func.func @_QPacc_reduction_mul_float_array_1d(
+! CHECK-SAME:  %{{.*}}: !fir.ref<!fir.array<100xf32>> {fir.bindc_name = "a"}, %[[B:.*]]: !fir.ref<!fir.array<100xf32>> {fir.bindc_name = "b"})
+! CHECK:       %[[DECLB:.*]]:2 = hlfir.declare %[[B]]
+! CHECK:       %[[RED_B:.*]] = acc.reduction varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<100xf32>>) bounds(%{{.*}}) -> !fir.ref<!fir.array<100xf32>> {name = "b"}
+! CHECK:       acc.loop {{.*}} reduction(@reduction_mul_section_ext100_ref_100xf32 -> %[[RED_B]] : !fir.ref<!fir.array<100xf32>>)
+
+subroutine acc_reduction_min_int(a, b)
+  integer :: a(100)
+  integer :: i, b
+
+  !$acc loop reduction(min:b)
+  do i = 1, 100
+    b = min(b, a(i))
+  end do
+end subroutine
+
+! CHECK-LABEL: func.func @_QPacc_reduction_min_int(
+! CHECK-SAME:  %{{.*}}: !fir.ref<!fir.array<100xi32>> {fir.bindc_name = "a"}, %[[B:.*]]: !fir.ref<i32> {fir.bindc_name = "b"})
+! CHECK:       %[[DECLB:.*]]:2 = hlfir.declare %[[B]]
+! CHECK:       %[[RED_B:.*]] = acc.reduction varPtr(%[[DECLB]]#0 : !fir.ref<i32>) -> !fir.ref<i32> {name = "b"}
+! CHECK:       acc.loop {{.*}} reduction(@reduction_min_ref_i32 -> %[[RED_B]] : !fir.ref<i32>)
+
+subroutine acc_reduction_min_int_array_1d(a, b)
+  integer :: a(100), b(100)
+  integer :: i
+
+  !$acc loop reduction(min:b)
+  do i = 1, 100
+    b(i) = min(b(i), a(i))
+  end do
+end subroutine
+
+! CHECK-LABEL: func.func @_QPacc_reduction_min_int_array_1d(
+! CHECK-SAME: %{{.*}}: !fir.ref<!fir.array<100xi32>> {fir.bindc_name = "a"}, %[[ARG1:.*]]: !fir.ref<!fir.array<100xi32>> {fir.bindc_name = "b"})
+! CHECK: %[[DECLARG1:.*]]:2 = hlfir.declare %[[ARG1]]
+! CHECK: %[[RED_ARG1:.*]] = acc.reduction varPtr(%[[DECLARG1]]#0 : !fir.ref<!fir.array<100xi32>>) bounds(%{{.*}}) -> !fir.ref<!fir.array<100xi32>> {name = "b"}
+! CHECK: acc.loop {{.*}} reduction(@reduction_min_section_ext100_ref_100xi32 -> %[[RED_ARG1]] : !fir.ref<!fir.array<100xi32>>)
+
+subroutine acc_reduction_min_float(a, b)
+  real :: a(100), b
+  integer :: i
+
+  !$acc loop reduction(min:b)
+  do i = 1, 100
+    b = min(b, a(i))
+  end do
+end subroutine
+
+! CHECK-LABEL: func.func @_QPacc_reduction_min_float(
+! CHECK-SAME:  %{{.*}}: !fir.ref<!fir.array<100xf32>> {fir.bindc_name = "a"}, %[[B:.*]]: !fir.ref<f32> {fir.bindc_name = "b"})
+! CHECK:       %[[DECLB:.*]]:2 = hlfir.declare %[[B]]
+! CHECK:       %[[RED_B:.*]] = acc.reduction varPtr(%[[DECLB]]#0 : !fir.ref<f32>) -> !fir.ref<f32> {name = "b"}
+! CHECK:       acc.loop {{.*}} reduction(@reduction_min_ref_f32 -> %[[RED_B]] : !fir.ref<f32>)
+
+subroutine acc_reduction_min_float_array2d(a, b)
+  real :: a(100, 10), b(100, 10)
+  integer :: i, j
+
+  !$acc loop reduction(min:b) collapse(2)
+  do i = 1, 100
+    do j = 1, 10
+      b(i, j) = min(b(i, j), a(i, j))
+    end do
+  end do
+end subroutine
+
+! CHECK-LABEL: func.func @_QPacc_reduction_min_float_array2d(
+! CHECK-SAME: %{{.*}}: !fir.ref<!fir.array<100x10xf32>> {fir.bindc_name = "a"}, %[[ARG1:.*]]: !fir.ref<!fir.array<100x10xf32>> {fir.bindc_name = "b"})
+! CHECK: %[[DECLARG1:.*]]:2 = hlfir.declare %[[ARG1]]
+! CHECK: %[[RED_ARG1:.*]] = acc.reduction varPtr(%[[DECLARG1]]#0 : !fir.ref<!fir.array<100x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<100x10xf32>> {name = "b"}
+! CHECK: acc.loop {{.*}} reduction(@reduction_min_section_ext100xext10_ref_100x10xf32 -> %[[RED_ARG1]] : !fir.ref<!fir.array<100x10xf32>>)
+! CHECK: attributes {collapse = [2]{{.*}}
+
+subroutine acc_reduction_max_int(a, b)
+  integer :: a(100)
+  integer :: i, b
+
+  !$acc loop reduction(max:b)
+  do i = 1, 100
+    b = max(b, a(i))
+  end do
+end subroutine
+
+! CHECK-LABEL: func.func @_QPacc_reduction_max_int(
+! CHECK-SAME:  %{{.*}}: !fir.ref<!fir.array<100xi32>> {fir.bindc_name = "a"}, %[[B:.*]]: !fir.ref<i32> {fir.bindc_name = "b"})
+! CHECK:       %[[DECLB:.*]]:2 = hlfir.declare %[[B]]
+! CHECK:       %[[RED_B:.*]] = acc.reduction varPtr(%[[DECLB]]#0 : !fir.ref<i32>) -> !fir.ref<i32> {name = "b"}
+! CHECK:       acc.loop {{.*}} reduction(@reduction_max_ref_i32 -> %[[RED_B]] : !fir.ref<i32>)
+
+subroutine acc_reduction_max_int_array2d(a, b)
+  integer :: a(100, 10), b(100, 10)
+  integer :: i, j
+
+  !$acc loop reduction(max:b) collapse(2)
+  do i = 1, 100
+    do j = 1, 10
+      b(i, j) = max(b(i, j), a(i, j))
+    end do
+  end do
+end subroutine
+
+! CHECK-LABEL: func.func @_QPacc_reduction_max_int_array2d(
+! CHECK-SAME: %{{.*}}: !fir.ref<!fir.array<100x10xi32>> {fir.bindc_name = "a"}, %[[ARG1:.*]]: !fir.ref<!fir.array<100x10xi32>> {fir.bindc_name = "b"})
+! CHECK: %[[DECLARG1:.*]]:2 = hlfir.declare %[[ARG1]]
+! CHECK: %[[RED_ARG1:.*]] = acc.reduction varPtr(%[[DECLARG1]]#0 : !fir.ref<!fir.array<100x10xi32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<100x10xi32>> {name = "b"}
+! CHECK: acc.loop {{.*}} reduction(@reduction_max_section_ext100xext10_ref_100x10xi32 -> %[[RED_ARG1]] : !fir.ref<!fir.array<100x10xi32>>)
+
+subroutine acc_reduction_max_float(a, b)
+  real :: a(100), b
+  integer :: i
+
+  !$acc loop reduction(max:b)
+  do i = 1, 100
+    b = max(b, a(i))
+  end do
+end subroutine
+
+! CHECK-LABEL: func.func @_QPacc_reduction_max_float(
+! CHECK-SAME:  %{{.*}}: !fir.ref<!fir.array<100xf32>> {fir.bindc_name = "a"}, %[[B:.*]]: !fir.ref<f32> {fir.bindc_name = "b"})
+! CHECK:       %[[DECLB:.*]]:2 = hlfir.declare %[[B]]
+! CHECK:       %[[RED_B:.*]] = acc.reduction varPtr(%[[DECLB]]#0 : !fir.ref<f32>) -> !fir.ref<f32> {name = "b"}
+! CHECK:       acc.loop {{.*}} reduction(@reduction_max_ref_f32 -> %[[RED_B]] : !fir.ref<f32>)
+
+subroutine acc_reduction_max_float_array1d(a, b)
+  real :: a(100), b(100)
+  integer :: i
+
+  !$acc loop reduction(max:b)
+  do i = 1, 100
+    b(i) = max(b(i), a(i))
+  end do
+end subroutine
+
+! CHECK-LABEL: func.func @_QPacc_reduction_max_float_array1d(
+! CHECK-SAME:  %{{.*}}: !fir.ref<!fir.array<100xf32>> {fir.bindc_name = "a"}, %[[ARG1:.*]]: !fir.ref<!fir.array<100xf32>> {fir.bindc_name = "b"})
+! CHECK:       %[[DECLARG1:.*]]:2 = hlfir.declare %[[ARG1]]
+! CHECK:       %[[RED_ARG1:.*]] = acc.reduction varPtr(%[[DECLARG1]]#0 : !fir.ref<!fir.array<100xf32>>) bounds(%{{.*}}) -> !fir.ref<!fir.array<100xf32>> {name = "b"}
+! CHECK:       acc.loop {{.*}} reduction(@reduction_max_section_ext100_ref_100xf32 -> %[[RED_ARG1]] : !fir.ref<!fir.array<100xf32>>)
+
+subroutine acc_reduction_iand()
+  integer :: i
+  !$acc parallel reduction(iand:i)
+  !$acc end parallel
+end subroutine
+
+! CHECK-LABEL: func.func @_QPacc_reduction_iand()
+! CHECK: %[[RED:.*]] = acc.reduction varPtr(%{{.*}} : !fir.ref<i32>)   -> !fir.ref<i32> {name = "i"}
+! CHECK: acc.parallel   reduction(@reduction_iand_ref_i32 -> %[[RED]] : !fir.ref<i32>)
+
+subroutine acc_reduction_ior()
+  integer :: i
+  !$acc parallel reduction(ior:i)
+  !$acc end parallel
+end subroutine
+
+! CHECK-LABEL: func.func @_QPacc_reduction_ior()
+! CHECK: %[[RED:.*]] = acc.reduction varPtr(%{{.*}} : !fir.ref<i32>)   -> !fir.ref<i32> {name = "i"}
+! CHECK: acc.parallel reduction(@reduction_ior_ref_i32 -> %[[RED]] : !fir.ref<i32>)
+
+subroutine acc_reduction_ieor()
+  integer :: i
+  !$acc parallel reduction(ieor:i)
+  !$acc end parallel
+end subroutine
+
+! CHECK-LABEL: func.func @_QPacc_reduction_ieor()
+! CHECK: %[[RED:.*]] = acc.reduction varPtr(%{{.*}} : !fir.ref<i32>) -> !fir.ref<i32> {name = "i"}
+! CHECK: acc.parallel reduction(@reduction_xor_ref_i32 -> %[[RED]] : !fir.ref<i32>)
+
+subroutine acc_reduction_and()
+  logical :: l
+  !$acc parallel reduction(.and.:l)
+  !$acc end parallel
+end subroutine
+
+! CHECK-LABEL: func.func @_QPacc_reduction_and()
+! CHECK: %[[L:.*]] = fir.alloca !fir.logical<4> {bindc_name = "l", uniq_name = "_QFacc_reduction_andEl"}
+! CHECK: %[[DECLL:.*]]:2 = hlfir.declare %[[L]]
+! CHECK: %[[RED:.*]] = acc.reduction varPtr(%[[DECLL]]#0 : !fir.ref<!fir.logical<4>>) -> !fir.ref<!fir.logical<4>> {name = "l"}
+! CHECK: acc.parallel reduction(@reduction_land_ref_l32 -> %[[RED]] : !fir.ref<!fir.logical<4>>)
+
+subroutine acc_reduction_or()
+  logical :: l
+  !$acc parallel reduction(.or.:l)
+  !$acc end parallel
+end subroutine
+
+! CHECK-LABEL: func.func @_QPacc_reduction_or()
+! CHECK: %[[RED:.*]] = acc.reduction varPtr(%{{.*}} : !fir.ref<!fir.logical<4>>) -> !fir.ref<!fir.logical<4>> {name = "l"}
+! CHECK: acc.parallel reduction(@reduction_lor_ref_l32 -> %[[RED]] : !fir.ref<!fir.logical<4>>)
+
+subroutine acc_reduction_eqv()
+  logical :: l
+  !$acc parallel reduction(.eqv.:l)
+  !$acc end parallel
+end subroutine
+
+! CHECK-LABEL: func.func @_QPacc_reduction_eqv()
+! CHECK: %[[RED:.*]] = acc.reduction varPtr(%{{.*}} : !fir.ref<!fir.logical<4>>) -> !fir.ref<!fir.logical<4>> {name = "l"}
+! CHECK: acc.parallel reduction(@reduction_eqv_ref_l32 -> %[[RED]] : !fir.ref<!fir.logical<4>>)
+
+subroutine acc_reduction_neqv()
+  logical :: l
+  !$acc parallel reduction(.neqv.:l)
+  !$acc end parallel
+end subroutine
+
+! CHECK-LABEL: func.func @_QPacc_reduction_neqv()
+! CHECK: %[[RED:.*]] = acc.reduction varPtr(%{{.*}} : !fir.ref<!fir.logical<4>>) -> !fir.ref<!fir.logical<4>> {name = "l"}
+! CHECK: acc.parallel reduction(@reduction_neqv_ref_l32 -> %[[RED]] : !fir.ref<!fir.logical<4>>)
+
+subroutine acc_reduction_add_cmplx()
+  complex :: c
+  !$acc parallel reduction(+:c)
+  !$acc end parallel
+end subroutine
+
+! CHECK-LABEL: func.func @_QPacc_reduction_add_cmplx()
+! CHECK: %[[RED:.*]] = acc.reduction varPtr(%{{.*}} : !fir.ref<complex<f32>>) -> !fir.ref<complex<f32>> {name = "c"}
+! CHECK: acc.parallel reduction(@reduction_add_ref_z32 -> %[[RED]] : !fir.ref<complex<f32>>)
+
+subroutine acc_reduction_mul_cmplx()
+  complex :: c
+  !$acc parallel reduction(*:c)
+  !$acc end parallel
+end subroutine
+
+! CHECK-LABEL: func.func @_QPacc_reduction_mul_cmplx()
+! CHECK: %[[RED:.*]] = acc.reduction varPtr(%{{.*}} : !fir.ref<complex<f32>>) -> !fir.ref<complex<f32>> {name = "c"}
+! CHECK: acc.parallel reduction(@reduction_mul_ref_z32 -> %[[RED]] : !fir.ref<complex<f32>>)
+
+subroutine acc_reduction_add_alloc()
+  integer, allocatable :: i
+  allocate(i)
+  !$acc parallel reduction(+:i)
+  !$acc end parallel
+end subroutine
+
+! CHECK-LABEL: func.func @_QPacc_reduction_add_alloc()
+! CHECK: %[[ALLOCA:.*]] = fir.alloca !fir.box<!fir.heap<i32>> {bindc_name = "i", uniq_name = "_QFacc_reduction_add_allocEi"}
+! CHECK: %[[DECL:.*]]:2 = hlfir.declare %[[ALLOCA]]
+! CHECK: %[[LOAD:.*]] = fir.load %[[DECL]]#0 : !fir.ref<!fir.box<!fir.heap<i32>>>
+! CHECK: %[[BOX_ADDR:.*]] = fir.box_addr %[[LOAD]] : (!fir.box<!fir.heap<i32>>) -> !fir.heap<i32>
+! CHECK: %[[RED:.*]] = acc.reduction varPtr(%[[BOX_ADDR]] : !fir.heap<i32>) -> !fir.heap<i32> {name = "i"}
+! CHECK: acc.parallel reduction(@reduction_add_heap_i32 -> %[[RED]] : !fir.heap<i32>)
+
+subroutine acc_reduction_add_pointer(i)
+  integer, pointer :: i
+  !$acc parallel reduction(+:i)
+  !$acc end parallel
+end subroutine
+
+! CHECK-LABEL: func.func @_QPacc_reduction_add_pointer(
+! CHECK-SAME: %[[ARG0:.*]]: !fir.ref<!fir.box<!fir.ptr<i32>>> {fir.bindc_name = "i"})
+! CHECK: %[[DECLARG0:.*]]:2 = hlfir.declare %[[ARG0]]
+! CHECK: %[[LOAD:.*]] = fir.load %[[DECLARG0]]#0 : !fir.ref<!fir.box<!fir.ptr<i32>>>
+! CHECK: %[[BOX_ADDR:.*]] = fir.box_addr %[[LOAD]] : (!fir.box<!fir.ptr<i32>>) -> !fir.ptr<i32>
+! CHECK: %[[RED:.*]] = acc.reduction varPtr(%[[BOX_ADDR]] : !fir.ptr<i32>) -> !fir.ptr<i32> {name = "i"}
+! CHECK: acc.parallel reduction(@reduction_add_ptr_i32 -> %[[RED]] : !fir.ptr<i32>)
+
+subroutine acc_reduction_add_static_slice(a)
+  integer :: a(100)
+  !$acc parallel reduction(+:a(11:20))
+  !$acc end parallel
+end subroutine
+
+! CHECK-LABEL: func.func @_QPacc_reduction_add_static_slice(
+! CHECK-SAME: %[[ARG0:.*]]: !fir.ref<!fir.array<100xi32>> {fir.bindc_name = "a"})
+! CHECK: %[[C100:.*]] = arith.constant 100 : index
+! CHECK: %[[DECLARG0:.*]]:2 = hlfir.declare %[[ARG0]]
+! CHECK: %[[C1:.*]] = arith.constant 1 : index
+! CHECK: %[[LB:.*]] = arith.constant 10 : index
+! CHECK: %[[UB:.*]] = arith.constant 19 : index
+! CHECK: %[[BOUND:.*]] = acc.bounds lowerbound(%[[LB]] : index) upperbound(%[[UB]] : index) extent(%[[C100]] : index) stride(%[[C1]] : index) startIdx(%[[C1]] : index)
+! CHECK: %[[RED:.*]] = acc.reduction varPtr(%[[DECLARG0]]#0 : !fir.ref<!fir.array<100xi32>>) bounds(%[[BOUND]]) -> !fir.ref<!fir.array<100xi32>> {name = "a(11:20)"}
+! CHECK: acc.parallel reduction(@reduction_add_section_lb10.ub19_ref_100xi32 -> %[[RED]] : !fir.ref<!fir.array<100xi32>>)
+
+subroutine acc_reduction_add_dynamic_extent_add(a)
+  integer :: a(:)
+  !$acc parallel reduction(+:a)
+  !$acc end parallel
+end subroutine
+
+! CHECK-LABEL: func.func @_QPacc_reduction_add_dynamic_extent_add(
+! CHECK-SAME: %[[ARG0:.*]]: !fir.box<!fir.array<?xi32>> {fir.bindc_name = "a"})
+! CHECK: %[[DECLARG0:.*]]:2 = hlfir.declare %[[ARG0]]
+! CHECK: %[[RED:.*]] = acc.reduction varPtr(%{{.*}} : !fir.ref<!fir.array<?xi32>>) bounds(%{{.*}}) -> !fir.ref<!fir.array<?xi32>> {name = "a"}
+! CHECK: acc.parallel reduction(@reduction_add_box_Uxi32 -> %[[RED:.*]] : !fir.ref<!fir.array<?xi32>>)
+
+subroutine acc_reduction_add_assumed_shape_max(a)
+  real :: a(:)
+  !$acc parallel reduction(max:a)
+  !$acc end parallel
+end subroutine
+
+! CHECK-LABEL: func.func @_QPacc_reduction_add_assumed_shape_max(
+! CHECK-SAME: %[[ARG0:.*]]: !fir.box<!fir.array<?xf32>> {fir.bindc_name = "a"})
+! CHECK: %[[DECLARG0:.*]]:2 = hlfir.declare %[[ARG0]]
+! CHECK: %[[RED:.*]] = acc.reduction varPtr(%{{.*}} : !fir.ref<!fir.array<?xf32>>) bounds(%{{.*}}) -> !fir.ref<!fir.array<?xf32>> {name = "a"}
+! CHECK: acc.parallel reduction(@reduction_max_box_Uxf32 -> %[[RED]] : !fir.ref<!fir.array<?xf32>>) {
+
+subroutine acc_reduction_add_dynamic_extent_add_with_section(a)
+  integer :: a(:)
+  !$acc parallel reduction(+:a(2:4))
+  !$acc end parallel
+end subroutine
+
+! CHECK-LABEL: func.func @_QPacc_reduction_add_dynamic_extent_add_with_section(
+! CHECK-SAME: %[[ARG0:.*]]: !fir.box<!fir.array<?xi32>> {fir.bindc_name = "a"})
+! CHECK: %[[DECL:.*]]:2 = hlfir.declare %[[ARG0]] dummy_scope %{{[0-9]+}} {uniq_name = "_QFacc_reduction_add_dynamic_extent_add_with_sectionEa"} : (!fir.box<!fir.array<?xi32>>, !fir.dscope) -> (!fir.box<!fir.array<?xi32>>, !fir.box<!fir.array<?xi32>>)
+! CHECK: %[[BOUND:.*]] = acc.bounds lowerbound(%c1{{.*}} : index) upperbound(%c3{{.*}} : index) extent(%{{.*}}#1 : index) stride(%{{.*}}#2 : index) startIdx(%{{.*}} : index) {strideInBytes = true}
+! CHECK: %[[BOX_ADDR:.*]] = fir.box_addr %[[DECL]]#0 : (!fir.box<!fir.array<?xi32>>) -> !fir.ref<!fir.array<?xi32>>
+! CHECK: %[[RED:.*]] = acc.reduction varPtr(%[[BOX_ADDR]] : !fir.ref<!fir.array<?xi32>>) bounds(%[[BOUND]]) -> !fir.ref<!fir.array<?xi32>> {name = "a(2:4)"}
+! CHECK: acc.parallel reduction(@reduction_add_section_lb1.ub3_box_Uxi32 -> %[[RED]] : !fir.ref<!fir.array<?xi32>>)
+
+subroutine acc_reduction_add_allocatable(a)
+  real, allocatable :: a(:)
+  !$acc parallel reduction(max:a)
+  !$acc end parallel
+end subroutine
+
+! CHECK-LABEL: func.func @_QPacc_reduction_add_allocatable(
+! CHECK-SAME: %[[ARG0:.*]]: !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>> {fir.bindc_name = "a"})
+! CHECK: %[[DECL:.*]]:2 = hlfir.declare %[[ARG0]] dummy_scope %{{[0-9]+}} {fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "_QFacc_reduction_add_allocatableEa"} : (!fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>, !fir.dscope) -> (!fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>, !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>)
+! CHECK: %[[BOX:.*]] = fir.load %[[DECL]]#0 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>
+! CHECK: %[[BOUND:.*]] = acc.bounds lowerbound(%c0{{.*}} : index) upperbound(%{{.*}} : index) extent(%{{.*}}#1 : index) stride(%{{.*}}#2 : index) startIdx(%{{.*}}#0 : index) {strideInBytes = true}
+! CHECK: %[[BOX_ADDR:.*]] = fir.box_addr %[[BOX]] : (!fir.box<!fir.heap<!fir.array<?xf32>>>) -> !fir.heap<!fir.array<?xf32>>
+! CHECK: %[[RED:.*]] = acc.reduction varPtr(%[[BOX_ADDR]] : !fir.heap<!fir.array<?xf32>>)   bounds(%{{[0-9]+}}) -> !fir.heap<!fir.array<?xf32>> {name = "a"}
+! CHECK: acc.parallel reduction(@reduction_max_box_heap_Uxf32 -> %[[RED]] : !fir.heap<!fir.array<?xf32>>)
+
+subroutine acc_reduction_add_pointer_array(a)
+  real, pointer :: a(:)
+  !$acc parallel reduction(max:a)
+  !$acc end parallel
+end subroutine
+
+! CHECK-LABEL: func.func @_QPacc_reduction_add_pointer_array(
+! CHECK-SAME: %[[ARG0:.*]]: !fir.ref<!fir.box<!fir.ptr<!fir.array<?xf32>>>> {fir.bindc_name = "a"})
+! CHECK: %[[DECL:.*]]:2 = hlfir.declare %[[ARG0]] dummy_scope %{{[0-9]+}} {fortran_attrs = #fir.var_attrs<pointer>, uniq_name = "_QFacc_reduction_add_pointer_arrayEa"} : (!fir.ref<!fir.box<!fir.ptr<!fir.array<?xf32>>>>, !fir.dscope) -> (!fir.ref<!fir.box<!fir.ptr<!fir.array<?xf32>>>>, !fir.ref<!fir.box<!fir.ptr<!fir.array<?xf32>>>>)
+! CHECK: %[[BOX:.*]] = fir.load %[[DECL]]#0 : !fir.ref<!fir.box<!fir.ptr<!fir.array<?xf32>>>>
+! CHECK: %[[BOUND:.*]] = acc.bounds lowerbound(%c0{{.*}} : index) upperbound(%{{.*}} : index) extent(%{{.*}}#1 : index) stride(%{{.*}}#2 : index) startIdx(%{{.*}}#0 : index) {strideInBytes = true}
+! CHECK: %[[BOX_ADDR:.*]] = fir.box_addr %[[BOX]] : (!fir.box<!fir.ptr<!fir.array<?xf32>>>) -> !fir.ptr<!fir.array<?xf32>>
+! CHECK: %[[RED:.*]] = acc.reduction varPtr(%[[BOX_ADDR]] : !fir.ptr<!fir.array<?xf32>>) bounds(%[[BOUND]]) -> !fir.ptr<!fir.array<?xf32>> {name = "a"}
+! CHECK: acc.parallel reduction(@reduction_max_box_ptr_Uxf32 -> %[[RED]] : !fir.ptr<!fir.array<?xf32>>)
+
+subroutine acc_reduction_max_dynamic_extent_max(a, n)
+  integer :: n
+  real :: a(n, n)
+  !$acc parallel reduction(max:a)
+  !$acc end parallel
+end subroutine
+
+! CHECK-LABEL: func.func @_QPacc_reduction_max_dynamic_extent_max(
+! CHECK-SAME: %[[ARG0:.*]]: !fir.ref<!fir.array<?x?xf32>> {fir.bindc_name = "a"}, %{{.*}}: !fir.ref<i32> {fir.bindc_name = "n"})
+! CHECK: %[[DECL_A:.*]]:2 = hlfir.declare %[[ARG0]](%{{.*}}) dummy_scope %{{[0-9]+}} {uniq_name = "_QFacc_reduction_max_dynamic_extent_maxEa"} : (!fir.ref<!fir.array<?x?xf32>>, !fir.shape<2>, !fir.dscope) -> (!fir.box<!fir.array<?x?xf32>>, !fir.ref<!fir.array<?x?xf32>>)
+! CHECK: %[[ADDR:.*]] = fir.box_addr %[[DECL_A]]#0 : (!fir.box<!fir.array<?x?xf32>>) -> !fir.ref<!fir.array<?x?xf32>>
+! CHECK: %[[RED:.*]] = acc.reduction varPtr(%[[ADDR]] : !fir.ref<!fir.array<?x?xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<?x?xf32>> {name = "a"}
+! CHECK: acc.parallel reduction(@reduction_max_box_UxUxf32 -> %[[RED]] : !fir.ref<!fir.array<?x?xf32>>)
diff --git a/flang/test/Lower/OpenACC/acc-reduction.f90 b/flang/test/Lower/OpenACC/acc-reduction.f90
index 88c60a22b0fe8..0d97c298f8d24 100644
--- a/flang/test/Lower/OpenACC/acc-reduction.f90
+++ b/flang/test/Lower/OpenACC/acc-reduction.f90
@@ -13,11 +13,10 @@
 ! CHECK:   hlfir.assign %[[CST]] to %[[DECL]]#0 : f32, !fir.box<!fir.array<?x?xf32>>
 ! CHECK:   acc.yield %[[DECL]]#0 : !fir.box<!fir.array<?x?xf32>>
 ! CHECK: } combiner {
-! CHECK: ^bb0(%[[V1:.*]]: !fir.box<!fir.array<?x?xf32>>, %[[V2:.*]]: !fir.box<!fir.array<?x?xf32>>, %[[LB0:.*]]: index, %[[UB0:.*]]: index, %[[STEP0:.*]]: index, %[[LB1:.*]]: index, %[[UB1:.*]]: index, %[[STEP1:.*]]: index):
- 
+! CHECK: ^bb0(%[[V1:.*]]: !fir.box<!fir.array<?x?xf32>>, %[[V2:.*]]: !fir.box<!fir.array<?x?xf32>>):
 ! CHECK:   %[[SHAPE:.*]] = fir.shape %{{.*}}, %{{.*}} : (index, index) -> !fir.shape<2>
-! CHECK:   %[[DES_V1:.*]] = hlfir.designate %[[V1]] (%[[LB0]]:%[[UB0]]:%[[STEP0]], %[[LB1]]:%[[UB1]]:%[[STEP1]])  shape %[[SHAPE]] : (!fir.box<!fir.array<?x?xf32>>, index, index, index, index, index, index, !fir.shape<2>) -> !fir.box<!fir.array<?x?xf32>>
-! CHECK:   %[[DES_V2:.*]] = hlfir.designate %[[V2]] (%[[LB0]]:%[[UB0]]:%[[STEP0]], %[[LB1]]:%[[UB1]]:%[[STEP1]])  shape %[[SHAPE]] : (!fir.box<!fir.array<?x?xf32>>, index, index, index, index, index, index, !fir.shape<2>) -> !fir.box<!fir.array<?x?xf32>>
+! CHECK:   %[[DES_V1:.*]] = hlfir.designate %[[V1]] shape %[[SHAPE]] : (!fir.box<!fir.array<?x?xf32>>, !fir.shape<2>) -> !fir.box<!fir.array<?x?xf32>>
+! CHECK:   %[[DES_V2:.*]] = hlfir.designate %[[V2]] shape %[[SHAPE]] : (!fir.box<!fir.array<?x?xf32>>, !fir.shape<2>) -> !fir.box<!fir.array<?x?xf32>>
 ! CHECK:   %[[ELEMENTAL:.*]] = hlfir.elemental %[[SHAPE]] unordered : (!fir.shape<2>) -> !hlfir.expr<?x?xf32> {
 ! CHECK:   ^bb0(%[[ARG0:.*]]: index, %[[ARG1:.*]]: index):
 ! CHECK:     %[[D1:.*]] = hlfir.designate %[[DES_V1]] (%[[ARG0]], %[[ARG1]])  : (!fir.box<!fir.array<?x?xf32>>, index, index) -> !fir.ref<f32>
@@ -32,27 +31,62 @@
 ! CHECK:   acc.yield %[[V1]] : !fir.box<!fir.array<?x?xf32>>
 ! CHECK: }
 
-! CHECK-LABEL: acc.reduction.recipe @reduction_max_box_ptr_Uxf32 : !fir.box<!fir.ptr<!fir.array<?xf32>>> reduction_operator <max> init {
-! CHECK: ^bb0(%{{.*}}: !fir.box<!fir.ptr<!fir.array<?xf32>>>):
+! CHECK-LABEL: acc.reduction.recipe @reduction_max_ref_box_ptr_Uxf32 : !fir.ref<!fir.box<!fir.ptr<!fir.array<?xf32>>>> reduction_operator <max> init {
+! CHECK: ^bb0(%[[ARG0:.*]]: !fir.ref<!fir.box<!fir.ptr<!fir.array<?xf32>>>>):
+! CHECK:   %[[CST:.*]] = arith.constant -1.401300e-45 : f32
+! CHECK:   %[[BOX:.*]] = fir.load %[[ARG0]] : !fir.ref<!fir.box<!fir.ptr<!fir.array<?xf32>>>>
+! CHECK:   %[[C0:.*]] = arith.constant 0 : index
+! CHECK:   %[[BOX_DIMS:.*]]:3 = fir.box_dims %[[BOX]], %[[C0]] : (!fir.box<!fir.ptr<!fir.array<?xf32>>>, index) -> (index, index, index)
+! CHECK:   %[[SHAPE:.*]] = fir.shape %[[BOX_DIMS]]#1 : (index) -> !fir.shape<1>
+! CHECK:   %[[TEMP:.*]] = fir.allocmem !fir.array<?xf32>, %[[BOX_DIMS]]#1 {bindc_name = ".tmp", uniq_name = ""}
+! CHECK:   %[[DECLARE:.*]]:2 = hlfir.declare %[[TEMP]](%[[SHAPE]]) {uniq_name = ".tmp"} : (!fir.heap<!fir.array<?xf32>>, !fir.shape<1>) -> (!fir.box<!fir.array<?xf32>>, !fir.heap<!fir.array<?xf32>>)
+! CHECK:   hlfir.assign %[[CST]] to %[[DECLARE]]#0 : f32, !fir.box<!fir.array<?xf32>>
+! CHECK:   acc.yield %[[DECLARE]]#0 : !fir.box<!fir.array<?xf32>>
 ! CHECK: } combiner {
-! CHECK: ^bb0(%{{.*}}: !fir.box<!fir.ptr<!fir.array<?xf32>>>, %{{.*}}: !fir.box<!fir.ptr<!fir.array<?xf32>>>, %{{.*}}: index, %{{.*}}: index, %{{.*}}: index):
+! CHECK: ^bb0(%[[ARG0:.*]]: !fir.ref<!fir.box<!fir.ptr<!fir.array<?xf32>>>>, %[[ARG1:.*]]: !fir.ref<!fir.box<!fir.ptr<!fir.array<?xf32>>>>):
+! CHECK:   %[[BOX0:.*]] = fir.load %[[ARG0]] : !fir.ref<!fir.box<!fir.ptr<!fir.array<?xf32>>>>
+! CHECK:   %[[C0:.*]] = arith.constant 0 : index
+! CHECK:   %[[BOX_DIMS:.*]]:3 = fir.box_dims %[[BOX0]], %[[C0]] : (!fir.box<!fir.ptr<!fir.array<?xf32>>>, index) -> (index, index, index)
+! CHECK:   %[[SHAPE:.*]] = fir.shape %[[BOX_DIMS]]#1 : (index) -> !fir.shape<1>
+! CHECK:   %[[BOX0:.*]] = fir.load %[[ARG0]] : !fir.ref<!fir.box<!fir.ptr<!fir.array<?xf32>>>>
+! CHECK:   %[[DES_V1:.*]] = hlfir.designate %[[BOX0]] shape %[[SHAPE]] : (!fir.box<!fir.ptr<!fir.array<?xf32>>>, !fir.shape<1>) -> !fir.box<!fir.ptr<!fir.array<?xf32>>>
+! CHECK:   %[[BOX1:.*]] = fir.load %[[ARG1]] : !fir.ref<!fir.box<!fir.ptr<!fir.array<?xf32>>>>
+! CHECK:   %[[DES_V2:.*]] = hlfir.designate %[[BOX1]] shape %[[SHAPE]] : (!fir.box<!fir.ptr<!fir.array<?xf32>>>, !fir.shape<1>) -> !fir.box<!fir.ptr<!fir.array<?xf32>>>
+! CHECK:   %[[ELEMENTAL:.*]] = hlfir.elemental %[[SHAPE]] unordered : (!fir.shape<1>) -> !hlfir.expr<?xf32> {
+! CHECK:   ^bb0(%[[IV:.*]]: index):
+! CHECK:     %[[V1:.*]] = hlfir.designate %[[DES_V1]] (%[[IV]])  : (!fir.box<!fir.ptr<!fir.array<?xf32>>>, index) -> !fir.ref<f32>
+! CHECK:     %[[V2:.*]] = hlfir.designate %[[DES_V2]] (%[[IV]])  : (!fir.box<!fir.ptr<!fir.array<?xf32>>>, index) -> !fir.ref<f32>
+! CHECK:     %[[LOAD_V1:.*]] = fir.load %[[V1]] : !fir.ref<f32>
+! CHECK:     %[[LOAD_V2:.*]] = fir.load %[[V2]] : !fir.ref<f32>
+! CHECK:     %[[CMP:.*]] = arith.cmpf ogt, %[[LOAD_V1]], %[[LOAD_V2]] {{.*}} : f32
+! CHECK:     %[[SELECT:.*]] = arith.select %[[CMP]], %[[LOAD_V1]], %[[LOAD_V2]] : f32
+! CHECK:     hlfir.yield_element %[[SELECT]] : f32
+! CHECK:   }
+! CHECK:   hlfir.assign %[[ELEMENTAL]] to %[[ARG0]] : !hlfir.expr<?xf32>, !fir.ref<!fir.box<!fir.ptr<!fir.array<?xf32>>>>
+! CHECK:   acc.yield %[[ARG0]] : !fir.ref<!fir.box<!fir.ptr<!fir.array<?xf32>>>>
 ! CHECK: }
 
-! CHECK-LABEL: acc.reduction.recipe @reduction_max_box_heap_Uxf32 : !fir.box<!fir.heap<!fir.array<?xf32>>> reduction_operator <max> init {
-! CHECK: ^bb0(%[[ARG0:.*]]: !fir.box<!fir.heap<!fir.array<?xf32>>>):
+! CHECK-LABEL: acc.reduction.recipe @reduction_max_ref_box_heap_Uxf32 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>> reduction_operator <max> init {
+! CHECK: ^bb0(%[[ARG0:.*]]: !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>):
 ! CHECK:   %[[CST:.*]] = arith.constant -1.401300e-45 : f32
+! CHECK:   %[[BOX:.*]] = fir.load %[[ARG0]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>
 ! CHECK:   %[[C0:.*]] = arith.constant 0 : index
-! CHECK:   %[[BOX_DIMS:.*]]:3 = fir.box_dims %[[ARG0]], %[[C0]] : (!fir.box<!fir.heap<!fir.array<?xf32>>>, index) -> (index, index, index)
+! CHECK:   %[[BOX_DIMS:.*]]:3 = fir.box_dims %[[BOX]], %[[C0]] : (!fir.box<!fir.heap<!fir.array<?xf32>>>, index) -> (index, index, index)
 ! CHECK:   %[[SHAPE:.*]] = fir.shape %[[BOX_DIMS]]#1 : (index) -> !fir.shape<1>
 ! CHECK:   %[[TEMP:.*]] = fir.allocmem !fir.array<?xf32>, %[[BOX_DIMS]]#1 {bindc_name = ".tmp", uniq_name = ""}
-! CHECK:   %[[DECLARE:.*]]:2 = hlfir.declare %2(%1) {uniq_name = ".tmp"} : (!fir.heap<!fir.array<?xf32>>, !fir.shape<1>) -> (!fir.box<!fir.array<?xf32>>, !fir.heap<!fir.array<?xf32>>)
+! CHECK:   %[[DECLARE:.*]]:2 = hlfir.declare %[[TEMP]](%[[SHAPE]]) {uniq_name = ".tmp"} : (!fir.heap<!fir.array<?xf32>>, !fir.shape<1>) -> (!fir.box<!fir.array<?xf32>>, !fir.heap<!fir.array<?xf32>>)
 ! CHECK:   hlfir.assign %[[CST]] to %[[DECLARE]]#0 : f32, !fir.box<!fir.array<?xf32>>
 ! CHECK:   acc.yield %[[DECLARE]]#0 : !fir.box<!fir.array<?xf32>>
 ! CHECK: } combiner {
-! CHECK: ^bb0(%[[ARG0:.*]]: !fir.box<!fir.heap<!fir.array<?xf32>>>, %[[ARG1:.*]]: !fir.box<!fir.heap<!fir.array<?xf32>>>, %[[ARG2:.*]]: index, %[[ARG3:.*]]: index, %[[ARG4:.*]]: index):
-! CHECK:   %[[SHAPE:.*]] = fir.shape %{{.*}} : (index) -> !fir.shape<1>
-! CHECK:   %[[DES_V1:.*]] = hlfir.designate %[[ARG0]] (%[[ARG2]]:%[[ARG3]]:%[[ARG4]]) shape %[[SHAPE]] : (!fir.box<!fir.heap<!fir.array<?xf32>>>, index, index, index, !fir.shape<1>) -> !fir.box<!fir.heap<!fir.array<?xf32>>>
-! CHECK:   %[[DES_V2:.*]] = hlfir.designate %[[ARG1]] (%[[ARG2]]:%[[ARG3]]:%[[ARG4]]) shape %[[SHAPE]] : (!fir.box<!fir.heap<!fir.array<?xf32>>>, index, index, index, !fir.shape<1>) -> !fir.box<!fir.heap<!fir.array<?xf32>>>
+! CHECK: ^bb0(%[[ARG0:.*]]: !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>, %[[ARG1:.*]]: !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>):
+! CHECK:   %[[BOX0:.*]] = fir.load %[[ARG0]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>
+! CHECK:   %[[C0:.*]] = arith.constant 0 : index
+! CHECK:   %[[BOX_DIMS:.*]]:3 = fir.box_dims %[[BOX0]], %[[C0]] : (!fir.box<!fir.heap<!fir.array<?xf32>>>, index) -> (index, index, index)
+! CHECK:   %[[SHAPE:.*]] = fir.shape %[[BOX_DIMS]]#1 : (index) -> !fir.shape<1>
+! CHECK:   %[[BOX0:.*]] = fir.load %[[ARG0]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>
+! CHECK:   %[[DES_V1:.*]] = hlfir.designate %[[BOX0]] shape %[[SHAPE]] : (!fir.box<!fir.heap<!fir.array<?xf32>>>, !fir.shape<1>) -> !fir.box<!fir.heap<!fir.array<?xf32>>>
+! CHECK:   %[[BOX1:.*]] = fir.load %[[ARG1]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>
+! CHECK:   %[[DES_V2:.*]] = hlfir.designate %[[BOX1]] shape %[[SHAPE]] : (!fir.box<!fir.heap<!fir.array<?xf32>>>, !fir.shape<1>) -> !fir.box<!fir.heap<!fir.array<?xf32>>>
 ! CHECK:   %[[ELEMENTAL:.*]] = hlfir.elemental %[[SHAPE]] unordered : (!fir.shape<1>) -> !hlfir.expr<?xf32> {
 ! CHECK:   ^bb0(%[[IV:.*]]: index):
 ! CHECK:     %[[V1:.*]] = hlfir.designate %[[DES_V1]] (%[[IV]])  : (!fir.box<!fir.heap<!fir.array<?xf32>>>, index) -> !fir.ref<f32>
@@ -63,8 +97,8 @@
 ! CHECK:     %[[SELECT:.*]] = arith.select %[[CMP]], %[[LOAD_V1]], %[[LOAD_V2]] : f32
 ! CHECK:     hlfir.yield_element %[[SELECT]] : f32
 ! CHECK:   }
-! CHECK:   hlfir.assign %[[ELEMENTAL]] to %[[ARG0]] : !hlfir.expr<?xf32>, !fir.box<!fir.heap<!fir.array<?xf32>>>
-! CHECK:   acc.yield %[[ARG0]] : !fir.box<!fir.heap<!fir.array<?xf32>>>
+! CHECK:   hlfir.assign %[[ELEMENTAL]] to %[[ARG0]] : !hlfir.expr<?xf32>, !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>
+! CHECK:   acc.yield %[[ARG0]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>
 ! CHECK: }
 
 ! CHECK-LABEL: acc.reduction.recipe @reduction_add_section_lb1.ub3_box_Uxi32 : !fir.box<!fir.array<?xi32>> reduction_operator <add> init {
@@ -105,8 +139,8 @@
 ! CHECK:   acc.yield %[[DECLARE]]#0 : !fir.box<!fir.array<?xf32>>
 ! CHECK: } combiner {
 ! CHECK: ^bb0(%[[ARG0:.*]]: !fir.box<!fir.array<?xf32>>, %[[ARG1:.*]]: !fir.box<!fir.array<?xf32>>
-! CHECK:   %[[LEFT:.*]] = hlfir.designate %[[ARG0]] (%{{.*}}:%{{.*}}:%{{.*}}) shape %{{.*}} : (!fir.box<!fir.array<?xf32>>, index, index, index, !fir.shape<1>) -> !fir.box<!fir.array<?xf32>>
-! CHECK:   %[[RIGHT:.*]] = hlfir.designate %[[ARG1]] (%{{.*}}:%{{.*}}:%{{.*}}) shape %{{.*}} : (!fir.box<!fir.array<?xf32>>, index, index, index, !fir.shape<1>) -> !fir.box<!fir.array<?xf32>>
+! CHECK:   %[[LEFT:.*]] = hlfir.designate %[[ARG0]] shape %{{.*}} : (!fir.box<!fir.array<?xf32>>, !fir.shape<1>) -> !fir.box<!fir.array<?xf32>>
+! CHECK:   %[[RIGHT:.*]] = hlfir.designate %[[ARG1]] shape %{{.*}} : (!fir.box<!fir.array<?xf32>>, !fir.shape<1>) -> !fir.box<!fir.array<?xf32>>
 ! CHECK:   %[[ELEMENTAL:.*]] = hlfir.elemental %{{.*}} unordered : (!fir.shape<1>) -> !hlfir.expr<?xf32> {
 ! CHECK:   ^bb0(%{{.*}}: index):
 ! CHECK:     %[[DES_V1:.*]] = hlfir.designate %[[LEFT]] (%{{.*}})  : (!fir.box<!fir.array<?xf32>>, index) -> !fir.ref<f32>
@@ -132,9 +166,12 @@
 ! CHECK:   hlfir.assign %[[INIT_VALUE]] to %[[DECLARE]]#0 : i32, !fir.box<!fir.array<?xi32>>
 ! CHECK:   acc.yield %[[DECLARE]]#0 : !fir.box<!fir.array<?xi32>>
 ! CHECK: } combiner {
-! CHECK: ^bb0(%[[V1:.*]]: !fir.box<!fir.array<?xi32>>, %[[V2:.*]]: !fir.box<!fir.array<?xi32>>
-! CHECK:   %[[LEFT:.*]] = hlfir.designate %[[ARG0]] (%{{.*}}:%{{.*}}:%{{.*}}) shape %{{.*}} : (!fir.box<!fir.array<?xi32>>, index, index, index, !fir.shape<1>) -> !fir.box<!fir.array<?xi32>>
-! CHECK:   %[[RIGHT:.*]] = hlfir.designate %[[ARG1]] (%{{.*}}:%{{.*}}:%{{.*}}) shape %{{.*}} : (!fir.box<!fir.array<?xi32>>, index, index, index, !fir.shape<1>) -> !fir.box<!fir.array<?xi32>>
+! CHECK: ^bb0(%[[ARG0:.*]]: !fir.box<!fir.array<?xi32>>, %[[ARG1:.*]]: !fir.box<!fir.array<?xi32>>
+! CHECK:   %[[C0:.*]] = arith.constant 0 : index
+! CHECK:   %[[BOX_DIMS:.*]]:3 = fir.box_dims %[[ARG0]], %[[C0]] : (!fir.box<!fir.array<?xi32>>, index) -> (index, index, index)
+! CHECK:   %[[SHAPE:.*]] = fir.shape %[[BOX_DIMS]]#1 : (index) -> !fir.shape<1>
+! CHECK:   %[[LEFT:.*]] = hlfir.designate %[[ARG0]] shape %[[SHAPE]] : (!fir.box<!fir.array<?xi32>>, !fir.shape<1>) -> !fir.box<!fir.array<?xi32>>
+! CHECK:   %[[RIGHT:.*]] = hlfir.designate %[[ARG1]] shape %[[SHAPE]] : (!fir.box<!fir.array<?xi32>>, !fir.shape<1>) -> !fir.box<!fir.array<?xi32>>
 ! CHECK:   %[[ELEMENTAL:.*]] = hlfir.elemental %{{.*}} unordered : (!fir.shape<1>) -> !hlfir.expr<?xi32> {
 ! CHECK:   ^bb0(%{{.*}}: index):
 ! CHECK:     %[[DES_V1:.*]] = hlfir.designate %[[LEFT]] (%{{.*}})  : (!fir.box<!fir.array<?xi32>>, index) -> !fir.ref<i32>
@@ -144,7 +181,7 @@
 ! CHECK:     %[[COMBINED:.*]] = arith.addi %[[LOAD_V1]], %[[LOAD_V2]] : i32
 ! CHECK:     hlfir.yield_element %[[COMBINED]] : i32
 ! CHECK:   }
-! CHECK:   hlfir.assign %[[ELEMENTAL]] to %[[V1]] : !hlfir.expr<?xi32>, !fir.box<!fir.array<?xi32>>
+! CHECK:   hlfir.assign %[[ELEMENTAL]] to %[[ARG0]] : !hlfir.expr<?xi32>, !fir.box<!fir.array<?xi32>>
 ! CHECK:   acc.yield %arg0 : !fir.box<!fir.array<?xi32>>
 ! CHECK: }
 
@@ -316,7 +353,7 @@
 ! CHECK:   acc.yield %[[ARG0]] : !fir.ref<i32>
 ! CHECK: }
 
-! CHECK-LABEL: acc.reduction.recipe @reduction_max_section_ext100_ref_100xf32 : !fir.ref<!fir.array<100xf32>> reduction_operator <max> init {
+! CHECK-LABEL: acc.reduction.recipe @reduction_max_ref_100xf32 : !fir.ref<!fir.array<100xf32>> reduction_operator <max> init {
 ! CHECK: ^bb0(%{{.*}}: !fir.ref<!fir.array<100xf32>>):
 ! CHECK:   %[[INIT:.*]] = arith.constant -1.401300e-45 : f32
 ! CHECK:   %[[SHAPE:.*]] = fir.shape %{{.*}} : (index) -> !fir.shape<1>
@@ -364,7 +401,7 @@
 ! CHECK:   acc.yield %[[ARG0]] : !fir.ref<f32>
 ! CHECK: }
 
-! CHECK-LABEL: acc.reduction.recipe @reduction_max_section_ext100xext10_ref_100x10xi32 : !fir.ref<!fir.array<100x10xi32>> reduction_operator <max> init {
+! CHECK-LABEL: acc.reduction.recipe @reduction_max_ref_100x10xi32 : !fir.ref<!fir.array<100x10xi32>> reduction_operator <max> init {
 ! CHECK: ^bb0(%arg0: !fir.ref<!fir.array<100x10xi32>>):
 ! CHECK:   %[[INIT:.*]] = arith.constant -2147483648 : i32
 ! CHECK:   %[[SHAPE:.*]] = fir.shape %{{.*}}, %{{.*}} : (index, index) -> !fir.shape<2>
@@ -374,11 +411,11 @@
 ! CHECK: } combiner {
 ! CHECK: ^bb0(%[[ARG0:.*]]: !fir.ref<!fir.array<100x10xi32>>, %[[ARG1:.*]]: !fir.ref<!fir.array<100x10xi32>>):
 ! CHECK:   %[[LB0:.*]] = arith.constant 0 : index
-! CHECK:   %[[UB0:.*]] = arith.constant 9 : index
+! CHECK:   %[[UB0:.*]] = arith.constant 99 : index
 ! CHECK:   %[[STEP0:.*]] = arith.constant 1 : index
 ! CHECK:   fir.do_loop %[[IV0:.*]] = %[[LB0]] to %[[UB0]] step %[[STEP0]] {
 ! CHECK:     %[[LB1:.*]] = arith.constant 0 : index
-! CHECK:     %[[UB1:.*]] = arith.constant 99 : index
+! CHECK:     %[[UB1:.*]] = arith.constant 9 : index
 ! CHECK:     %[[STEP1:.*]] = arith.constant 1 : index
 ! CHECK:     fir.do_loop %[[IV1:.*]] = %[[LB1]] to %[[UB1]] step %[[STEP1]] {
 ! CHECK:       %[[COORD1:.*]] = fir.coordinate_of %[[ARG0:.*]], %[[IV0]], %[[IV1]] : (!fir.ref<!fir.array<100x10xi32>>, index, index) -> !fir.ref<i32>
@@ -410,7 +447,7 @@
 ! CHECK:   acc.yield %[[ARG0]] : !fir.ref<i32>
 ! CHECK: }
 
-! CHECK-LABEL: acc.reduction.recipe @reduction_min_section_ext100xext10_ref_100x10xf32 : !fir.ref<!fir.array<100x10xf32>> reduction_operator <min> init {
+! CHECK-LABEL: acc.reduction.recipe @reduction_min_ref_100x10xf32 : !fir.ref<!fir.array<100x10xf32>> reduction_operator <min> init {
 ! CHECK: ^bb0(%{{.*}}: !fir.ref<!fir.array<100x10xf32>>):
 ! CHECK:   %[[INIT:.*]] = arith.constant 3.40282347E+38 : f32
 ! CHECK:   %[[SHAPE:.*]] = fir.shape %{{.*}}, %{{.*}} : (index, index) -> !fir.shape<2>
@@ -420,11 +457,11 @@
 ! CHECK: } combiner {
 ! CHECK: ^bb0(%[[ARG0:.*]]: !fir.ref<!fir.array<100x10xf32>>, %[[ARG1:.*]]: !fir.ref<!fir.array<100x10xf32>>):
 ! CHECK:   %[[LB0:.*]] = arith.constant 0 : index
-! CHECK:   %[[UB0:.*]] = arith.constant 9 : index
+! CHECK:   %[[UB0:.*]] = arith.constant 99 : index
 ! CHECK:   %[[STEP0:.*]] = arith.constant 1 : index
 ! CHECK:   fir.do_loop %[[IV0:.*]] = %[[LB0]] to %[[UB0]] step %[[STEP0]] {
 ! CHECK:     %[[LB1:.*]] = arith.constant 0 : index
-! CHECK:     %[[UB1:.*]] = arith.constant 99 : index
+! CHECK:     %[[UB1:.*]] = arith.constant 9 : index
 ! CHECK:     %[[STEP1:.*]] = arith.constant 1 : index
 ! CHECK:     fir.do_loop %[[IV1:.*]] = %[[LB1]] to %[[UB1]] step %[[STEP1]] {
 ! CHECK:       %[[COORD1:.*]] = fir.coordinate_of %[[ARG0]], %[[IV0]], %[[IV1]] : (!fir.ref<!fir.array<100x10xf32>>, index, index) -> !fir.ref<f32>
@@ -456,7 +493,7 @@
 ! CHECK:   acc.yield %[[ARG0]] : !fir.ref<f32>
 ! CHECK: }
 
-! CHECK-LABEL: acc.reduction.recipe @reduction_min_section_ext100_ref_100xi32 : !fir.ref<!fir.array<100xi32>> reduction_operator <min> init {
+! CHECK-LABEL: acc.reduction.recipe @reduction_min_ref_100xi32 : !fir.ref<!fir.array<100xi32>> reduction_operator <min> init {
 ! CHECK: ^bb0(%{{.*}}: !fir.ref<!fir.array<100xi32>>):
 ! CHECK:   %[[INIT:.*]] = arith.constant 2147483647 : i32
 ! CHECK:   %[[SHAPE:.*]] = fir.shape %{{.*}} : (index) -> !fir.shape<1>
@@ -513,7 +550,7 @@
 ! CHECK:   acc.yield %[[ARG0]] : !fir.ref<f32>
 ! CHECK: }
 
-! CHECK-LABEL: acc.reduction.recipe @reduction_mul_section_ext100_ref_100xi32 : !fir.ref<!fir.array<100xi32>> reduction_operator <mul> init {
+! CHECK-LABEL: acc.reduction.recipe @reduction_mul_ref_100xi32 : !fir.ref<!fir.array<100xi32>> reduction_operator <mul> init {
 ! CHECK: ^bb0(%{{.*}}: !fir.ref<!fir.array<100xi32>>):
 ! CHECK:   %[[INIT:.*]] = arith.constant 1 : i32
 ! CHECK:   %[[SHAPE:.*]] = fir.shape %{{.*}} : (index) -> !fir.shape<1>
@@ -552,7 +589,7 @@
 ! CHECK:   acc.yield %[[ARG0]] : !fir.ref<i32>
 ! CHECK: }
 
-! CHECK-LABEL: acc.reduction.recipe @reduction_add_section_ext100_ref_100xf32 : !fir.ref<!fir.array<100xf32>> reduction_operator <add> init {
+! CHECK-LABEL: acc.reduction.recipe @reduction_add_ref_100xf32 : !fir.ref<!fir.array<100xf32>> reduction_operator <add> init {
 ! CHECK: ^bb0(%{{.*}}: !fir.ref<!fir.array<100xf32>>):
 ! CHECK:   %[[INIT:.*]] = arith.constant 0.000000e+00 : f32
 ! CHECK:   %[[SHAPE:.*]] = fir.shape %{{.*}} : (index) -> !fir.shape<1>
@@ -591,7 +628,7 @@
 ! CHECK:   acc.yield %[[ARG0]] : !fir.ref<f32>
 ! CHECK: }
 
-! CHECK-LABEL: acc.reduction.recipe @reduction_add_section_ext100xext10xext2_ref_100x10x2xi32 : !fir.ref<!fir.array<100x10x2xi32>> reduction_operator <add> init {
+! CHECK-LABEL: acc.reduction.recipe @reduction_add_ref_100x10x2xi32 : !fir.ref<!fir.array<100x10x2xi32>> reduction_operator <add> init {
 ! CHECK: ^bb0(%{{.*}}: !fir.ref<!fir.array<100x10x2xi32>>):
 ! CHECK:   %[[INIT:.*]] = arith.constant 0 : i32
 ! CHECK:   %[[SHAPE:.*]] = fir.shape %{{.*}}, %{{.*}}, %{{.*}} : (index, index, index) -> !fir.shape<3>
@@ -601,7 +638,7 @@
 ! CHECK: } combiner {
 ! CHECK: ^bb0(%[[ARG0:.*]]: !fir.ref<!fir.array<100x10x2xi32>>, %[[ARG1:.*]]: !fir.ref<!fir.array<100x10x2xi32>>):
 ! CHECK:   %[[LB0:.*]] = arith.constant 0 : index
-! CHECK:   %[[UB0:.*]] = arith.constant 1 : index
+! CHECK:   %[[UB0:.*]] = arith.constant 99 : index
 ! CHECK:   %[[STEP0:.*]] = arith.constant 1 : index
 ! CHECK:   fir.do_loop %[[IV0:.*]] = %[[LB0]] to %[[UB0]] step %[[STEP0]] {
 ! CHECK:     %[[LB1:.*]] = arith.constant 0 : index
@@ -609,7 +646,7 @@
 ! CHECK:     %[[STEP1:.*]] = arith.constant 1 : index
 ! CHECK:     fir.do_loop %[[IV1:.*]] = %[[LB1]] to %[[UB1]] step %[[STEP1]] {
 ! CHECK:       %[[LB2:.*]] = arith.constant 0 : index
-! CHECK:       %[[UB2:.*]] = arith.constant 99 : index
+! CHECK:       %[[UB2:.*]] = arith.constant 1 : index
 ! CHECK:       %[[STEP2:.*]] = arith.constant 1 : index
 ! CHECK:       fir.do_loop %[[IV2:.*]] = %[[LB2]] to %[[UB2]] step %[[STEP2]] {
 ! CHECK:         %[[COORD1:.*]] = fir.coordinate_of %[[ARG0]], %[[IV0]], %[[IV1]], %[[IV2]] : (!fir.ref<!fir.array<100x10x2xi32>>, index, index, index) -> !fir.ref<i32>
@@ -624,7 +661,7 @@
 ! CHECK:   acc.yield %[[ARG0]] : !fir.ref<!fir.array<100x10x2xi32>>
 ! CHECK: }
 
-! CHECK-LABEL: acc.reduction.recipe @reduction_add_section_ext100xext10_ref_100x10xi32 : !fir.ref<!fir.array<100x10xi32>> reduction_operator <add> init {
+! CHECK-LABEL: acc.reduction.recipe @reduction_add_ref_100x10xi32 : !fir.ref<!fir.array<100x10xi32>> reduction_operator <add> init {
 ! CHECK: ^bb0(%{{.*}}: !fir.ref<!fir.array<100x10xi32>>):
 ! CHECK:   %[[INIT:.*]] = arith.constant 0 : i32
 ! CHECK:   %[[SHAPE:.*]] = fir.shape %{{.*}}, %{{.*}} : (index, index) -> !fir.shape<2>
@@ -634,11 +671,11 @@
 ! CHECK: } combiner {
 ! CHECK: ^bb0(%[[ARG0:.*]]: !fir.ref<!fir.array<100x10xi32>>, %[[ARG1:.*]]: !fir.ref<!fir.array<100x10xi32>>):
 ! CHECK:   %[[LB0:.*]] = arith.constant 0 : index
-! CHECK:   %[[UB0:.*]] = arith.constant 9 : index
+! CHECK:   %[[UB0:.*]] = arith.constant 99 : index
 ! CHECK:   %[[STEP0:.*]] = arith.constant 1 : index
 ! CHECK:   fir.do_loop %[[IV0:.*]] = %[[LB0]] to %[[UB0]] step %[[STEP0]] {
 ! CHECK:     %[[LB1:.*]] = arith.constant 0 : index
-! CHECK:     %[[UB1:.*]] = arith.constant 99 : index
+! CHECK:     %[[UB1:.*]] = arith.constant 9 : index
 ! CHECK:     %[[STEP1:.*]] = arith.constant 1 : index
 ! CHECK:     fir.do_loop %[[IV1:.*]] = %[[LB1]] to %[[UB1]] step %[[STEP1]] {
 ! CHECK:       %[[COORD1:.*]] = fir.coordinate_of %[[ARG0]], %[[IV0]], %[[IV1]] : (!fir.ref<!fir.array<100x10xi32>>, index, index) -> !fir.ref<i32>
@@ -652,7 +689,7 @@
 ! CHECK:   acc.yield %[[ARG0]] : !fir.ref<!fir.array<100x10xi32>>
 ! CHECK: }
 
-! CHECK-LABEL: acc.reduction.recipe @reduction_add_section_ext100_ref_100xi32 : !fir.ref<!fir.array<100xi32>> reduction_operator <add> init {
+! CHECK-LABEL: acc.reduction.recipe @reduction_add_ref_100xi32 : !fir.ref<!fir.array<100xi32>> reduction_operator <add> init {
 ! CHECK: ^bb0(%{{.*}}: !fir.ref<!fir.array<100xi32>>):
 ! CHECK:   %[[INIT:.*]] = arith.constant 0 : i32
 ! CHECK:   %[[SHAPE:.*]] = fir.shape %{{.*}} : (index) -> !fir.shape<1>
@@ -720,8 +757,8 @@ subroutine acc_reduction_add_int_array_1d(a, b)
 ! CHECK-LABEL: func.func @_QPacc_reduction_add_int_array_1d(
 ! CHECK-SAME:  %{{.*}}: !fir.ref<!fir.array<100xi32>> {fir.bindc_name = "a"}, %[[B:.*]]: !fir.ref<!fir.array<100xi32>> {fir.bindc_name = "b"})
 ! CHECK:       %[[DECLB:.*]]:2 = hlfir.declare %[[B]]
-! CHECK:       %[[RED_B:.*]] = acc.reduction varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<100xi32>>) bounds(%{{.*}}) -> !fir.ref<!fir.array<100xi32>> {name = "b"}
-! CHECK:       acc.loop {{.*}} reduction(@reduction_add_section_ext100_ref_100xi32 -> %[[RED_B]] : !fir.ref<!fir.array<100xi32>>)
+! CHECK:       %[[RED_B:.*]] = acc.reduction varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<100xi32>>) -> !fir.ref<!fir.array<100xi32>> {name = "b"}
+! CHECK:       acc.loop {{.*}} reduction(@reduction_add_ref_100xi32 -> %[[RED_B]] : !fir.ref<!fir.array<100xi32>>)
 
 subroutine acc_reduction_add_int_array_2d(a, b)
   integer :: a(100, 10), b(100, 10)
@@ -738,8 +775,8 @@ subroutine acc_reduction_add_int_array_2d(a, b)
 ! CHECK-LABEL: func.func @_QPacc_reduction_add_int_array_2d(
 ! CHECK-SAME:  %[[ARG0:.*]]: !fir.ref<!fir.array<100x10xi32>> {fir.bindc_name = "a"}, %[[ARG1:.*]]: !fir.ref<!fir.array<100x10xi32>> {fir.bindc_name = "b"}) {
 ! CHECK:       %[[DECLARG1:.*]]:2 = hlfir.declare %[[ARG1]]
-! CHECK:       %[[RED_ARG1:.*]] = acc.reduction varPtr(%[[DECLARG1]]#0 : !fir.ref<!fir.array<100x10xi32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<100x10xi32>> {name = "b"}
-! CHECK:       acc.loop {{.*}} reduction(@reduction_add_section_ext100xext10_ref_100x10xi32 -> %[[RED_ARG1]] : !fir.ref<!fir.array<100x10xi32>>)
+! CHECK:       %[[RED_ARG1:.*]] = acc.reduction varPtr(%[[DECLARG1]]#0 : !fir.ref<!fir.array<100x10xi32>>) -> !fir.ref<!fir.array<100x10xi32>> {name = "b"}
+! CHECK:       acc.loop {{.*}} reduction(@reduction_add_ref_100x10xi32 -> %[[RED_ARG1]] : !fir.ref<!fir.array<100x10xi32>>)
 ! CHECK: } attributes {collapse = [2]{{.*}}
 
 subroutine acc_reduction_add_int_array_3d(a, b)
@@ -759,8 +796,8 @@ subroutine acc_reduction_add_int_array_3d(a, b)
 ! CHECK-LABEL: func.func @_QPacc_reduction_add_int_array_3d(
 ! CHECK-SAME: %{{.*}}: !fir.ref<!fir.array<100x10x2xi32>> {fir.bindc_name = "a"}, %[[ARG1:.*]]: !fir.ref<!fir.array<100x10x2xi32>> {fir.bindc_name = "b"})
 ! CHECK: %[[DECLARG1:.*]]:2 = hlfir.declare %[[ARG1]]
-! CHECK: %[[RED_ARG1:.*]] = acc.reduction varPtr(%[[DECLARG1]]#0 : !fir.ref<!fir.array<100x10x2xi32>>) bounds(%{{.*}}, %{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<100x10x2xi32>> {name = "b"}
-! CHECK: acc.loop {{.*}} reduction(@reduction_add_section_ext100xext10xext2_ref_100x10x2xi32 -> %[[RED_ARG1]] : !fir.ref<!fir.array<100x10x2xi32>>)
+! CHECK: %[[RED_ARG1:.*]] = acc.reduction varPtr(%[[DECLARG1]]#0 : !fir.ref<!fir.array<100x10x2xi32>>) -> !fir.ref<!fir.array<100x10x2xi32>> {name = "b"}
+! CHECK: acc.loop {{.*}} reduction(@reduction_add_ref_100x10x2xi32 -> %[[RED_ARG1]] : !fir.ref<!fir.array<100x10x2xi32>>)
 ! CHECK: } attributes {collapse = [3]{{.*}}
 
 subroutine acc_reduction_add_float(a, b)
@@ -792,8 +829,8 @@ subroutine acc_reduction_add_float_array_1d(a, b)
 ! CHECK-LABEL: func.func @_QPacc_reduction_add_float_array_1d(
 ! CHECK-SAME:  %{{.*}}: !fir.ref<!fir.array<100xf32>> {fir.bindc_name = "a"}, %[[B:.*]]: !fir.ref<!fir.array<100xf32>> {fir.bindc_name = "b"})
 ! CHECK: %[[DECLB:.*]]:2 = hlfir.declare %[[B]]
-! CHECK: %[[RED_B:.*]] = acc.reduction varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<100xf32>>) bounds(%{{.*}}) -> !fir.ref<!fir.array<100xf32>> {name = "b"}
-! CHECK:       acc.loop {{.*}} reduction(@reduction_add_section_ext100_ref_100xf32 -> %[[RED_B]] : !fir.ref<!fir.array<100xf32>>)
+! CHECK: %[[RED_B:.*]] = acc.reduction varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<100xf32>>) -> !fir.ref<!fir.array<100xf32>> {name = "b"}
+! CHECK:       acc.loop {{.*}} reduction(@reduction_add_ref_100xf32 -> %[[RED_B]] : !fir.ref<!fir.array<100xf32>>)
 
 subroutine acc_reduction_mul_int(a, b)
   integer :: a(100)
@@ -824,8 +861,8 @@ subroutine acc_reduction_mul_int_array_1d(a, b)
 ! CHECK-LABEL: func.func @_QPacc_reduction_mul_int_array_1d(
 ! CHECK-SAME:  %{{.*}}: !fir.ref<!fir.array<100xi32>> {fir.bindc_name = "a"}, %[[B:.*]]: !fir.ref<!fir.array<100xi32>> {fir.bindc_name = "b"})
 ! CHECK:       %[[DECLB:.*]]:2 = hlfir.declare %[[B]]
-! CHECK:       %[[RED_B:.*]] = acc.reduction varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<100xi32>>) bounds(%{{.*}}) -> !fir.ref<!fir.array<100xi32>> {name = "b"}
-! CHECK:       acc.loop {{.*}} reduction(@reduction_mul_section_ext100_ref_100xi32 -> %[[RED_B]] : !fir.ref<!fir.array<100xi32>>)
+! CHECK:       %[[RED_B:.*]] = acc.reduction varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<100xi32>>) -> !fir.ref<!fir.array<100xi32>> {name = "b"}
+! CHECK:       acc.loop {{.*}} reduction(@reduction_mul_ref_100xi32 -> %[[RED_B]] : !fir.ref<!fir.array<100xi32>>)
 
 subroutine acc_reduction_mul_float(a, b)
   real :: a(100), b
@@ -856,8 +893,8 @@ subroutine acc_reduction_mul_float_array_1d(a, b)
 ! CHECK-LABEL: func.func @_QPacc_reduction_mul_float_array_1d(
 ! CHECK-SAME:  %{{.*}}: !fir.ref<!fir.array<100xf32>> {fir.bindc_name = "a"}, %[[B:.*]]: !fir.ref<!fir.array<100xf32>> {fir.bindc_name = "b"})
 ! CHECK:       %[[DECLB:.*]]:2 = hlfir.declare %[[B]]
-! CHECK:       %[[RED_B:.*]] = acc.reduction varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<100xf32>>) bounds(%{{.*}}) -> !fir.ref<!fir.array<100xf32>> {name = "b"}
-! CHECK:       acc.loop {{.*}} reduction(@reduction_mul_section_ext100_ref_100xf32 -> %[[RED_B]] : !fir.ref<!fir.array<100xf32>>)
+! CHECK:       %[[RED_B:.*]] = acc.reduction varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<100xf32>>) -> !fir.ref<!fir.array<100xf32>> {name = "b"}
+! CHECK:       acc.loop {{.*}} reduction(@reduction_mul_ref_100xf32 -> %[[RED_B]] : !fir.ref<!fir.array<100xf32>>)
 
 subroutine acc_reduction_min_int(a, b)
   integer :: a(100)
@@ -888,8 +925,8 @@ subroutine acc_reduction_min_int_array_1d(a, b)
 ! CHECK-LABEL: func.func @_QPacc_reduction_min_int_array_1d(
 ! CHECK-SAME: %{{.*}}: !fir.ref<!fir.array<100xi32>> {fir.bindc_name = "a"}, %[[ARG1:.*]]: !fir.ref<!fir.array<100xi32>> {fir.bindc_name = "b"})
 ! CHECK: %[[DECLARG1:.*]]:2 = hlfir.declare %[[ARG1]]
-! CHECK: %[[RED_ARG1:.*]] = acc.reduction varPtr(%[[DECLARG1]]#0 : !fir.ref<!fir.array<100xi32>>) bounds(%{{.*}}) -> !fir.ref<!fir.array<100xi32>> {name = "b"}
-! CHECK: acc.loop {{.*}} reduction(@reduction_min_section_ext100_ref_100xi32 -> %[[RED_ARG1]] : !fir.ref<!fir.array<100xi32>>)
+! CHECK: %[[RED_ARG1:.*]] = acc.reduction varPtr(%[[DECLARG1]]#0 : !fir.ref<!fir.array<100xi32>>) -> !fir.ref<!fir.array<100xi32>> {name = "b"}
+! CHECK: acc.loop {{.*}} reduction(@reduction_min_ref_100xi32 -> %[[RED_ARG1]] : !fir.ref<!fir.array<100xi32>>)
 
 subroutine acc_reduction_min_float(a, b)
   real :: a(100), b
@@ -922,8 +959,8 @@ subroutine acc_reduction_min_float_array2d(a, b)
 ! CHECK-LABEL: func.func @_QPacc_reduction_min_float_array2d(
 ! CHECK-SAME: %{{.*}}: !fir.ref<!fir.array<100x10xf32>> {fir.bindc_name = "a"}, %[[ARG1:.*]]: !fir.ref<!fir.array<100x10xf32>> {fir.bindc_name = "b"})
 ! CHECK: %[[DECLARG1:.*]]:2 = hlfir.declare %[[ARG1]]
-! CHECK: %[[RED_ARG1:.*]] = acc.reduction varPtr(%[[DECLARG1]]#0 : !fir.ref<!fir.array<100x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<100x10xf32>> {name = "b"}
-! CHECK: acc.loop {{.*}} reduction(@reduction_min_section_ext100xext10_ref_100x10xf32 -> %[[RED_ARG1]] : !fir.ref<!fir.array<100x10xf32>>)
+! CHECK: %[[RED_ARG1:.*]] = acc.reduction varPtr(%[[DECLARG1]]#0 : !fir.ref<!fir.array<100x10xf32>>) -> !fir.ref<!fir.array<100x10xf32>> {name = "b"}
+! CHECK: acc.loop {{.*}} reduction(@reduction_min_ref_100x10xf32 -> %[[RED_ARG1]] : !fir.ref<!fir.array<100x10xf32>>)
 ! CHECK: attributes {collapse = [2]{{.*}}
 
 subroutine acc_reduction_max_int(a, b)
@@ -957,8 +994,8 @@ subroutine acc_reduction_max_int_array2d(a, b)
 ! CHECK-LABEL: func.func @_QPacc_reduction_max_int_array2d(
 ! CHECK-SAME: %{{.*}}: !fir.ref<!fir.array<100x10xi32>> {fir.bindc_name = "a"}, %[[ARG1:.*]]: !fir.ref<!fir.array<100x10xi32>> {fir.bindc_name = "b"})
 ! CHECK: %[[DECLARG1:.*]]:2 = hlfir.declare %[[ARG1]]
-! CHECK: %[[RED_ARG1:.*]] = acc.reduction varPtr(%[[DECLARG1]]#0 : !fir.ref<!fir.array<100x10xi32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<100x10xi32>> {name = "b"}
-! CHECK: acc.loop {{.*}} reduction(@reduction_max_section_ext100xext10_ref_100x10xi32 -> %[[RED_ARG1]] : !fir.ref<!fir.array<100x10xi32>>)
+! CHECK: %[[RED_ARG1:.*]] = acc.reduction varPtr(%[[DECLARG1]]#0 : !fir.ref<!fir.array<100x10xi32>>) -> !fir.ref<!fir.array<100x10xi32>> {name = "b"}
+! CHECK: acc.loop {{.*}} reduction(@reduction_max_ref_100x10xi32 -> %[[RED_ARG1]] : !fir.ref<!fir.array<100x10xi32>>)
 
 subroutine acc_reduction_max_float(a, b)
   real :: a(100), b
@@ -989,8 +1026,8 @@ subroutine acc_reduction_max_float_array1d(a, b)
 ! CHECK-LABEL: func.func @_QPacc_reduction_max_float_array1d(
 ! CHECK-SAME:  %{{.*}}: !fir.ref<!fir.array<100xf32>> {fir.bindc_name = "a"}, %[[ARG1:.*]]: !fir.ref<!fir.array<100xf32>> {fir.bindc_name = "b"})
 ! CHECK:       %[[DECLARG1:.*]]:2 = hlfir.declare %[[ARG1]]
-! CHECK:       %[[RED_ARG1:.*]] = acc.reduction varPtr(%[[DECLARG1]]#0 : !fir.ref<!fir.array<100xf32>>) bounds(%{{.*}}) -> !fir.ref<!fir.array<100xf32>> {name = "b"}
-! CHECK:       acc.loop {{.*}} reduction(@reduction_max_section_ext100_ref_100xf32 -> %[[RED_ARG1]] : !fir.ref<!fir.array<100xf32>>)
+! CHECK:       %[[RED_ARG1:.*]] = acc.reduction varPtr(%[[DECLARG1]]#0 : !fir.ref<!fir.array<100xf32>>) -> !fir.ref<!fir.array<100xf32>> {name = "b"}
+! CHECK:       acc.loop {{.*}} reduction(@reduction_max_ref_100xf32 -> %[[RED_ARG1]] : !fir.ref<!fir.array<100xf32>>)
 
 subroutine acc_reduction_iand()
   integer :: i
@@ -1094,10 +1131,8 @@ subroutine acc_reduction_add_alloc()
 ! CHECK-LABEL: func.func @_QPacc_reduction_add_alloc()
 ! CHECK: %[[ALLOCA:.*]] = fir.alloca !fir.box<!fir.heap<i32>> {bindc_name = "i", uniq_name = "_QFacc_reduction_add_allocEi"}
 ! CHECK: %[[DECL:.*]]:2 = hlfir.declare %[[ALLOCA]]
-! CHECK: %[[LOAD:.*]] = fir.load %[[DECL]]#0 : !fir.ref<!fir.box<!fir.heap<i32>>>
-! CHECK: %[[BOX_ADDR:.*]] = fir.box_addr %[[LOAD]] : (!fir.box<!fir.heap<i32>>) -> !fir.heap<i32>
-! CHECK: %[[RED:.*]] = acc.reduction varPtr(%[[BOX_ADDR]] : !fir.heap<i32>) -> !fir.heap<i32> {name = "i"}
-! CHECK: acc.parallel reduction(@reduction_add_heap_i32 -> %[[RED]] : !fir.heap<i32>)
+! CHECK: %[[RED:.*]] = acc.reduction varPtr(%[[DECL]]#0 : !fir.ref<!fir.box<!fir.heap<i32>>>) -> !fir.ref<!fir.box<!fir.heap<i32>>> {name = "i"}
+! CHECK: acc.parallel reduction(@reduction_add_ref_box_heap_i32 -> %[[RED]] : !fir.ref<!fir.box<!fir.heap<i32>>>)
 
 subroutine acc_reduction_add_pointer(i)
   integer, pointer :: i
@@ -1108,10 +1143,8 @@ subroutine acc_reduction_add_pointer(i)
 ! CHECK-LABEL: func.func @_QPacc_reduction_add_pointer(
 ! CHECK-SAME: %[[ARG0:.*]]: !fir.ref<!fir.box<!fir.ptr<i32>>> {fir.bindc_name = "i"})
 ! CHECK: %[[DECLARG0:.*]]:2 = hlfir.declare %[[ARG0]]
-! CHECK: %[[LOAD:.*]] = fir.load %[[DECLARG0]]#0 : !fir.ref<!fir.box<!fir.ptr<i32>>>
-! CHECK: %[[BOX_ADDR:.*]] = fir.box_addr %[[LOAD]] : (!fir.box<!fir.ptr<i32>>) -> !fir.ptr<i32>
-! CHECK: %[[RED:.*]] = acc.reduction varPtr(%[[BOX_ADDR]] : !fir.ptr<i32>) -> !fir.ptr<i32> {name = "i"}
-! CHECK: acc.parallel reduction(@reduction_add_ptr_i32 -> %[[RED]] : !fir.ptr<i32>)
+! CHECK: %[[RED:.*]] = acc.reduction varPtr(%[[DECLARG0]]#0 : !fir.ref<!fir.box<!fir.ptr<i32>>>) -> !fir.ref<!fir.box<!fir.ptr<i32>>> {name = "i"}
+! CHECK: acc.parallel reduction(@reduction_add_ref_box_ptr_i32 -> %[[RED]] : !fir.ref<!fir.box<!fir.ptr<i32>>>)
 
 subroutine acc_reduction_add_static_slice(a)
   integer :: a(100)
@@ -1139,8 +1172,8 @@ subroutine acc_reduction_add_dynamic_extent_add(a)
 ! CHECK-LABEL: func.func @_QPacc_reduction_add_dynamic_extent_add(
 ! CHECK-SAME: %[[ARG0:.*]]: !fir.box<!fir.array<?xi32>> {fir.bindc_name = "a"})
 ! CHECK: %[[DECLARG0:.*]]:2 = hlfir.declare %[[ARG0]]
-! CHECK: %[[RED:.*]] = acc.reduction varPtr(%{{.*}} : !fir.ref<!fir.array<?xi32>>) bounds(%{{.*}}) -> !fir.ref<!fir.array<?xi32>> {name = "a"}
-! CHECK: acc.parallel reduction(@reduction_add_box_Uxi32 -> %[[RED:.*]] : !fir.ref<!fir.array<?xi32>>)
+! CHECK: %[[RED:.*]] = acc.reduction var(%{{.*}} : !fir.box<!fir.array<?xi32>>) -> !fir.box<!fir.array<?xi32>> {name = "a"}
+! CHECK: acc.parallel reduction(@reduction_add_box_Uxi32 -> %[[RED:.*]] : !fir.box<!fir.array<?xi32>>)
 
 subroutine acc_reduction_add_assumed_shape_max(a)
   real :: a(:)
@@ -1151,8 +1184,8 @@ subroutine acc_reduction_add_assumed_shape_max(a)
 ! CHECK-LABEL: func.func @_QPacc_reduction_add_assumed_shape_max(
 ! CHECK-SAME: %[[ARG0:.*]]: !fir.box<!fir.array<?xf32>> {fir.bindc_name = "a"})
 ! CHECK: %[[DECLARG0:.*]]:2 = hlfir.declare %[[ARG0]]
-! CHECK: %[[RED:.*]] = acc.reduction varPtr(%{{.*}} : !fir.ref<!fir.array<?xf32>>) bounds(%{{.*}}) -> !fir.ref<!fir.array<?xf32>> {name = "a"}
-! CHECK: acc.parallel reduction(@reduction_max_box_Uxf32 -> %[[RED]] : !fir.ref<!fir.array<?xf32>>) {
+! CHECK: %[[RED:.*]] = acc.reduction var(%{{.*}} : !fir.box<!fir.array<?xf32>>) -> !fir.box<!fir.array<?xf32>> {name = "a"}
+! CHECK: acc.parallel reduction(@reduction_max_box_Uxf32 -> %[[RED]] : !fir.box<!fir.array<?xf32>>) {
 
 subroutine acc_reduction_add_dynamic_extent_add_with_section(a)
   integer :: a(:)
@@ -1164,9 +1197,8 @@ subroutine acc_reduction_add_dynamic_extent_add_with_section(a)
 ! CHECK-SAME: %[[ARG0:.*]]: !fir.box<!fir.array<?xi32>> {fir.bindc_name = "a"})
 ! CHECK: %[[DECL:.*]]:2 = hlfir.declare %[[ARG0]] dummy_scope %{{[0-9]+}} {uniq_name = "_QFacc_reduction_add_dynamic_extent_add_with_sectionEa"} : (!fir.box<!fir.array<?xi32>>, !fir.dscope) -> (!fir.box<!fir.array<?xi32>>, !fir.box<!fir.array<?xi32>>)
 ! CHECK: %[[BOUND:.*]] = acc.bounds lowerbound(%c1{{.*}} : index) upperbound(%c3{{.*}} : index) extent(%{{.*}}#1 : index) stride(%{{.*}}#2 : index) startIdx(%{{.*}} : index) {strideInBytes = true}
-! CHECK: %[[BOX_ADDR:.*]] = fir.box_addr %[[DECL]]#0 : (!fir.box<!fir.array<?xi32>>) -> !fir.ref<!fir.array<?xi32>>
-! CHECK: %[[RED:.*]] = acc.reduction varPtr(%[[BOX_ADDR]] : !fir.ref<!fir.array<?xi32>>) bounds(%[[BOUND]]) -> !fir.ref<!fir.array<?xi32>> {name = "a(2:4)"}
-! CHECK: acc.parallel reduction(@reduction_add_section_lb1.ub3_box_Uxi32 -> %[[RED]] : !fir.ref<!fir.array<?xi32>>)
+! CHECK: %[[RED:.*]] = acc.reduction var(%[[DECL]]#0 : !fir.box<!fir.array<?xi32>>) bounds(%[[BOUND]]) -> !fir.box<!fir.array<?xi32>> {name = "a(2:4)"}
+! CHECK: acc.parallel reduction(@reduction_add_section_lb1.ub3_box_Uxi32 -> %[[RED]] : !fir.box<!fir.array<?xi32>>)
 
 subroutine acc_reduction_add_allocatable(a)
   real, allocatable :: a(:)
@@ -1177,11 +1209,8 @@ subroutine acc_reduction_add_allocatable(a)
 ! CHECK-LABEL: func.func @_QPacc_reduction_add_allocatable(
 ! CHECK-SAME: %[[ARG0:.*]]: !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>> {fir.bindc_name = "a"})
 ! CHECK: %[[DECL:.*]]:2 = hlfir.declare %[[ARG0]] dummy_scope %{{[0-9]+}} {fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "_QFacc_reduction_add_allocatableEa"} : (!fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>, !fir.dscope) -> (!fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>, !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>)
-! CHECK: %[[BOX:.*]] = fir.load %[[DECL]]#0 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>
-! CHECK: %[[BOUND:.*]] = acc.bounds lowerbound(%c0{{.*}} : index) upperbound(%{{.*}} : index) extent(%{{.*}}#1 : index) stride(%{{.*}}#2 : index) startIdx(%{{.*}}#0 : index) {strideInBytes = true}
-! CHECK: %[[BOX_ADDR:.*]] = fir.box_addr %[[BOX]] : (!fir.box<!fir.heap<!fir.array<?xf32>>>) -> !fir.heap<!fir.array<?xf32>>
-! CHECK: %[[RED:.*]] = acc.reduction varPtr(%[[BOX_ADDR]] : !fir.heap<!fir.array<?xf32>>)   bounds(%{{[0-9]+}}) -> !fir.heap<!fir.array<?xf32>> {name = "a"}
-! CHECK: acc.parallel reduction(@reduction_max_box_heap_Uxf32 -> %[[RED]] : !fir.heap<!fir.array<?xf32>>)
+! CHECK: %[[RED:.*]] = acc.reduction varPtr(%[[DECL]]#0 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>) -> !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>> {name = "a"}
+! CHECK: acc.parallel reduction(@reduction_max_ref_box_heap_Uxf32 -> %[[RED]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>)
 
 subroutine acc_reduction_add_pointer_array(a)
   real, pointer :: a(:)
@@ -1192,11 +1221,8 @@ subroutine acc_reduction_add_pointer_array(a)
 ! CHECK-LABEL: func.func @_QPacc_reduction_add_pointer_array(
 ! CHECK-SAME: %[[ARG0:.*]]: !fir.ref<!fir.box<!fir.ptr<!fir.array<?xf32>>>> {fir.bindc_name = "a"})
 ! CHECK: %[[DECL:.*]]:2 = hlfir.declare %[[ARG0]] dummy_scope %{{[0-9]+}} {fortran_attrs = #fir.var_attrs<pointer>, uniq_name = "_QFacc_reduction_add_pointer_arrayEa"} : (!fir.ref<!fir.box<!fir.ptr<!fir.array<?xf32>>>>, !fir.dscope) -> (!fir.ref<!fir.box<!fir.ptr<!fir.array<?xf32>>>>, !fir.ref<!fir.box<!fir.ptr<!fir.array<?xf32>>>>)
-! CHECK: %[[BOX:.*]] = fir.load %[[DECL]]#0 : !fir.ref<!fir.box<!fir.ptr<!fir.array<?xf32>>>>
-! CHECK: %[[BOUND:.*]] = acc.bounds lowerbound(%c0{{.*}} : index) upperbound(%{{.*}} : index) extent(%{{.*}}#1 : index) stride(%{{.*}}#2 : index) startIdx(%{{.*}}#0 : index) {strideInBytes = true}
-! CHECK: %[[BOX_ADDR:.*]] = fir.box_addr %[[BOX]] : (!fir.box<!fir.ptr<!fir.array<?xf32>>>) -> !fir.ptr<!fir.array<?xf32>>
-! CHECK: %[[RED:.*]] = acc.reduction varPtr(%[[BOX_ADDR]] : !fir.ptr<!fir.array<?xf32>>) bounds(%[[BOUND]]) -> !fir.ptr<!fir.array<?xf32>> {name = "a"}
-! CHECK: acc.parallel reduction(@reduction_max_box_ptr_Uxf32 -> %[[RED]] : !fir.ptr<!fir.array<?xf32>>)
+! CHECK: %[[RED:.*]] = acc.reduction varPtr(%[[DECL]]#0 : !fir.ref<!fir.box<!fir.ptr<!fir.array<?xf32>>>>) -> !fir.ref<!fir.box<!fir.ptr<!fir.array<?xf32>>>> {name = "a"}
+! CHECK: acc.parallel reduction(@reduction_max_ref_box_ptr_Uxf32 -> %[[RED]] : !fir.ref<!fir.box<!fir.ptr<!fir.array<?xf32>>>>)
 
 subroutine acc_reduction_max_dynamic_extent_max(a, n)
   integer :: n
@@ -1208,6 +1234,5 @@ subroutine acc_reduction_max_dynamic_extent_max(a, n)
 ! CHECK-LABEL: func.func @_QPacc_reduction_max_dynamic_extent_max(
 ! CHECK-SAME: %[[ARG0:.*]]: !fir.ref<!fir.array<?x?xf32>> {fir.bindc_name = "a"}, %{{.*}}: !fir.ref<i32> {fir.bindc_name = "n"})
 ! CHECK: %[[DECL_A:.*]]:2 = hlfir.declare %[[ARG0]](%{{.*}}) dummy_scope %{{[0-9]+}} {uniq_name = "_QFacc_reduction_max_dynamic_extent_maxEa"} : (!fir.ref<!fir.array<?x?xf32>>, !fir.shape<2>, !fir.dscope) -> (!fir.box<!fir.array<?x?xf32>>, !fir.ref<!fir.array<?x?xf32>>)
-! CHECK: %[[ADDR:.*]] = fir.box_addr %[[DECL_A]]#0 : (!fir.box<!fir.array<?x?xf32>>) -> !fir.ref<!fir.array<?x?xf32>>
-! CHECK: %[[RED:.*]] = acc.reduction varPtr(%[[ADDR]] : !fir.ref<!fir.array<?x?xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<?x?xf32>> {name = "a"}
-! CHECK: acc.parallel reduction(@reduction_max_box_UxUxf32 -> %[[RED]] : !fir.ref<!fir.array<?x?xf32>>)
+! CHECK: %[[RED:.*]] = acc.reduction var(%[[DECL_A]]#0 : !fir.box<!fir.array<?x?xf32>>) -> !fir.box<!fir.array<?x?xf32>> {name = "a"}
+! CHECK: acc.parallel reduction(@reduction_max_box_UxUxf32 -> %[[RED]] : !fir.box<!fir.array<?x?xf32>>)
diff --git a/flang/test/Lower/OpenACC/acc-serial-loop.f90 b/flang/test/Lower/OpenACC/acc-serial-loop.f90
index 167c05986a546..a94f0e8e8583e 100644
--- a/flang/test/Lower/OpenACC/acc-serial-loop.f90
+++ b/flang/test/Lower/OpenACC/acc-serial-loop.f90
@@ -10,7 +10,7 @@
 ! CHECK:   acc.yield %[[DECLARE]]#0 : !fir.ref<!fir.array<10xf32>>
 ! CHECK: }
 
-! CHECK-LABEL: acc.firstprivate.recipe @firstprivatization_section_ext10_ref_10xf32 : !fir.ref<!fir.array<10xf32>> init {
+! CHECK-LABEL: acc.firstprivate.recipe @firstprivatization_ref_10xf32 : !fir.ref<!fir.array<10xf32>> init {
 ! CHECK: ^bb0(%{{.*}}: !fir.ref<!fir.array<10xf32>>):
 ! CHECK:   %[[SHAPE:.*]] = fir.shape %{{.*}} : (index) -> !fir.shape<1>
 ! CHECK:   %[[ALLOCA:.*]] = fir.alloca !fir.array<10xf32>
@@ -246,116 +246,120 @@ subroutine acc_serial_loop
     a(i) = b(i)
   END DO
 
-! CHECK:      %[[COPYIN_A:.*]] = acc.copyin varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10xf32>>) bounds(%{{.*}}) -> !fir.ref<!fir.array<10xf32>> {dataClause = #acc<data_clause acc_copy>, name = "a"}
-! CHECK:      %[[COPYIN_B:.*]] = acc.copyin varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<10xf32>>) bounds(%{{.*}}) -> !fir.ref<!fir.array<10xf32>> {dataClause = #acc<data_clause acc_copy>, name = "b"}
+! CHECK:      %[[COPYIN_A:.*]] = acc.copyin varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10xf32>>) -> !fir.ref<!fir.array<10xf32>> {dataClause = #acc<data_clause acc_copy>, name = "a"}
+! CHECK:      %[[COPYIN_B:.*]] = acc.copyin varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<10xf32>>) -> !fir.ref<!fir.array<10xf32>> {dataClause = #acc<data_clause acc_copy>, name = "b"}
 ! CHECK:      acc.serial {{.*}} dataOperands(%[[COPYIN_A]], %[[COPYIN_B]] : !fir.ref<!fir.array<10xf32>>, !fir.ref<!fir.array<10xf32>>) {
 ! CHECK:        acc.loop {{.*}} {
 ! CHECK:          acc.yield
 ! CHECK-NEXT:   }{{$}}
 ! CHECK:        acc.yield
 ! CHECK-NEXT: }{{$}}
-! CHECK:      acc.copyout accPtr(%[[COPYIN_A]] : !fir.ref<!fir.array<10xf32>>) bounds(%{{.*}}) to varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10xf32>>) {dataClause = #acc<data_clause acc_copy>, name = "a"}
-! CHECK:      acc.copyout accPtr(%[[COPYIN_B]] : !fir.ref<!fir.array<10xf32>>) bounds(%{{.*}}) to varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<10xf32>>) {dataClause = #acc<data_clause acc_copy>, name = "b"}
+! CHECK:      acc.copyout accPtr(%[[COPYIN_A]] : !fir.ref<!fir.array<10xf32>>) to varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10xf32>>) {dataClause = #acc<data_clause acc_copy>, name = "a"}
+! CHECK:      acc.copyout accPtr(%[[COPYIN_B]] : !fir.ref<!fir.array<10xf32>>) to varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<10xf32>>) {dataClause = #acc<data_clause acc_copy>, name = "b"}
 
   !$acc serial loop copy(a) copy(b)
   DO i = 1, n
     a(i) = b(i)
   END DO
 
-! CHECK:      %[[COPYIN_A:.*]] = acc.copyin varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10xf32>>) bounds(%{{.*}}) -> !fir.ref<!fir.array<10xf32>> {dataClause = #acc<data_clause acc_copy>, name = "a"}
-! CHECK:      %[[COPYIN_B:.*]] = acc.copyin varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<10xf32>>) bounds(%{{.*}}) -> !fir.ref<!fir.array<10xf32>> {dataClause = #acc<data_clause acc_copy>, name = "b"}
+! CHECK:      %[[COPYIN_A:.*]] = acc.copyin varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10xf32>>) -> !fir.ref<!fir.array<10xf32>> {dataClause = #acc<data_clause acc_copy>, name = "a"}
+! CHECK:      %[[COPYIN_B:.*]] = acc.copyin varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<10xf32>>) -> !fir.ref<!fir.array<10xf32>> {dataClause = #acc<data_clause acc_copy>, name = "b"}
 ! CHECK:      acc.serial {{.*}} dataOperands(%[[COPYIN_A]], %[[COPYIN_B]] : !fir.ref<!fir.array<10xf32>>, !fir.ref<!fir.array<10xf32>>) {
 ! CHECK:        acc.loop {{.*}} {
 ! CHECK:          acc.yield
 ! CHECK-NEXT:   }{{$}}
 ! CHECK:        acc.yield
 ! CHECK-NEXT: }{{$}}
-! CHECK:      acc.copyout accPtr(%[[COPYIN_A]] : !fir.ref<!fir.array<10xf32>>) bounds(%{{.*}}) to varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10xf32>>) {dataClause = #acc<data_clause acc_copy>, name = "a"}
-! CHECK:      acc.copyout accPtr(%[[COPYIN_B]] : !fir.ref<!fir.array<10xf32>>) bounds(%{{.*}}) to varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<10xf32>>) {dataClause = #acc<data_clause acc_copy>, name = "b"}
+! CHECK:      acc.copyout accPtr(%[[COPYIN_A]] : !fir.ref<!fir.array<10xf32>>) to varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10xf32>>) {dataClause = #acc<data_clause acc_copy>, name = "a"}
+! CHECK:      acc.copyout accPtr(%[[COPYIN_B]] : !fir.ref<!fir.array<10xf32>>) to varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<10xf32>>) {dataClause = #acc<data_clause acc_copy>, name = "b"}
 
   !$acc serial loop copyin(a) copyin(readonly: b)
   DO i = 1, n
     a(i) = b(i)
   END DO
 
-! CHECK:      %[[COPYIN_A:.*]] = acc.copyin varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10xf32>>) bounds(%{{.*}}) -> !fir.ref<!fir.array<10xf32>> {name = "a"}
-! CHECK:      %[[COPYIN_B:.*]] = acc.copyin varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<10xf32>>) bounds(%{{.*}}) -> !fir.ref<!fir.array<10xf32>> {dataClause = #acc<data_clause acc_copyin_readonly>, name = "b"}
+! CHECK:      %[[COPYIN_A:.*]] = acc.copyin varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10xf32>>) -> !fir.ref<!fir.array<10xf32>> {name = "a"}
+! CHECK:      %[[COPYIN_B:.*]] = acc.copyin varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<10xf32>>) -> !fir.ref<!fir.array<10xf32>> {dataClause = #acc<data_clause acc_copyin_readonly>, name = "b"}
 ! CHECK:      acc.serial {{.*}} dataOperands(%[[COPYIN_A]], %[[COPYIN_B]] : !fir.ref<!fir.array<10xf32>>, !fir.ref<!fir.array<10xf32>>) {
 ! CHECK:        acc.loop {{.*}} {
 ! CHECK:          acc.yield
 ! CHECK-NEXT:   }{{$}}
 ! CHECK:        acc.yield
 ! CHECK-NEXT: }{{$}}
-! CHECK:      acc.delete accPtr(%[[COPYIN_A]] : !fir.ref<!fir.array<10xf32>>) bounds(%{{.*}}) {dataClause = #acc<data_clause acc_copyin>, name = "a"}
-! CHECK:      acc.delete accPtr(%[[COPYIN_B]] : !fir.ref<!fir.array<10xf32>>) bounds(%{{.*}}) {dataClause = #acc<data_clause acc_copyin_readonly>, name = "b"}
+! CHECK:      acc.delete accPtr(%[[COPYIN_A]] : !fir.ref<!fir.array<10xf32>>) {dataClause = #acc<data_clause acc_copyin>, name = "a"}
+! CHECK:      acc.delete accPtr(%[[COPYIN_B]] : !fir.ref<!fir.array<10xf32>>) {dataClause = #acc<data_clause acc_copyin_readonly>, name = "b"}
 
   !$acc serial loop copyout(a) copyout(zero: b)
   DO i = 1, n
     a(i) = b(i)
   END DO
 
-! CHECK:      %[[CREATE_A:.*]] = acc.create varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10xf32>>) bounds(%{{.*}}) -> !fir.ref<!fir.array<10xf32>> {dataClause = #acc<data_clause acc_copyout>, name = "a"}
-! CHECK:      %[[CREATE_B:.*]] = acc.create varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<10xf32>>) bounds(%{{.*}}) -> !fir.ref<!fir.array<10xf32>> {dataClause = #acc<data_clause acc_copyout>, name = "b"}
+! CHECK:      %[[CREATE_A:.*]] = acc.create varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10xf32>>) -> !fir.ref<!fir.array<10xf32>> {dataClause = #acc<data_clause acc_copyout>, name = "a"}
+! CHECK:      %[[CREATE_B:.*]] = acc.create varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<10xf32>>) -> !fir.ref<!fir.array<10xf32>> {dataClause = #acc<data_clause acc_copyout>, name = "b"}
 ! CHECK:      acc.serial {{.*}} dataOperands(%[[CREATE_A]], %[[CREATE_B]] : !fir.ref<!fir.array<10xf32>>, !fir.ref<!fir.array<10xf32>>) {
 ! CHECK:        acc.loop {{.*}} {
 ! CHECK:          acc.yield
 ! CHECK-NEXT:   }{{$}}
 ! CHECK:        acc.yield
 ! CHECK-NEXT: }{{$}}
-! CHECK:      acc.copyout accPtr(%[[CREATE_A]] : !fir.ref<!fir.array<10xf32>>) bounds(%{{.*}}) to varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10xf32>>) {name = "a"}
-! CHECK:      acc.copyout accPtr(%[[CREATE_B]] : !fir.ref<!fir.array<10xf32>>) bounds(%{{.*}}) to varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<10xf32>>) {name = "b"}
+! CHECK:      acc.copyout accPtr(%[[CREATE_A]] : !fir.ref<!fir.array<10xf32>>) to varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10xf32>>) {name = "a"}
+! CHECK:      acc.copyout accPtr(%[[CREATE_B]] : !fir.ref<!fir.array<10xf32>>) to varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<10xf32>>) {name = "b"}
 
   !$acc serial loop create(b) create(zero: a)
   DO i = 1, n
     a(i) = b(i)
   END DO
 
-! CHECK:      %[[CREATE_B:.*]] = acc.create varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<10xf32>>) bounds(%{{.*}}) -> !fir.ref<!fir.array<10xf32>> {name = "b"}
-! CHECK:      %[[CREATE_A:.*]] = acc.create varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10xf32>>) bounds(%{{.*}}) -> !fir.ref<!fir.array<10xf32>> {dataClause = #acc<data_clause acc_create_zero>, name = "a"}
+! CHECK:      %[[CREATE_B:.*]] = acc.create varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<10xf32>>) -> !fir.ref<!fir.array<10xf32>> {name = "b"}
+! CHECK:      %[[CREATE_A:.*]] = acc.create varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10xf32>>) -> !fir.ref<!fir.array<10xf32>> {dataClause = #acc<data_clause acc_create_zero>, name = "a"}
 ! CHECK:      acc.serial {{.*}} dataOperands(%[[CREATE_B]], %[[CREATE_A]] : !fir.ref<!fir.array<10xf32>>, !fir.ref<!fir.array<10xf32>>) {
 ! CHECK:        acc.loop {{.*}} {
 ! CHECK:          acc.yield
 ! CHECK-NEXT:   }{{$}}
 ! CHECK:        acc.yield
 ! CHECK-NEXT: }{{$}}
-! CHECK:      acc.delete accPtr(%[[CREATE_B]] : !fir.ref<!fir.array<10xf32>>) bounds(%{{.*}}) {dataClause = #acc<data_clause acc_create>, name = "b"}
-! CHECK:      acc.delete accPtr(%[[CREATE_A]] : !fir.ref<!fir.array<10xf32>>) bounds(%{{.*}}) {dataClause = #acc<data_clause acc_create_zero>, name = "a"}
+! CHECK:      acc.delete accPtr(%[[CREATE_B]] : !fir.ref<!fir.array<10xf32>>) {dataClause = #acc<data_clause acc_create>, name = "b"}
+! CHECK:      acc.delete accPtr(%[[CREATE_A]] : !fir.ref<!fir.array<10xf32>>) {dataClause = #acc<data_clause acc_create_zero>, name = "a"}
 
   !$acc serial loop no_create(a, b)
   DO i = 1, n
     a(i) = b(i)
   END DO
 
-! CHECK:      %[[NOCREATE_A:.*]] = acc.nocreate varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10xf32>>) bounds(%{{.*}}) -> !fir.ref<!fir.array<10xf32>> {name = "a"}
-! CHECK:      %[[NOCREATE_B:.*]] = acc.nocreate varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<10xf32>>) bounds(%{{.*}}) -> !fir.ref<!fir.array<10xf32>> {name = "b"}
+! CHECK:      %[[NOCREATE_A:.*]] = acc.nocreate varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10xf32>>) -> !fir.ref<!fir.array<10xf32>> {name = "a"}
+! CHECK:      %[[NOCREATE_B:.*]] = acc.nocreate varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<10xf32>>) -> !fir.ref<!fir.array<10xf32>> {name = "b"}
 ! CHECK:      acc.serial {{.*}} dataOperands(%[[NOCREATE_A]], %[[NOCREATE_B]] : !fir.ref<!fir.array<10xf32>>, !fir.ref<!fir.array<10xf32>>) {
 ! CHECK:        acc.loop {{.*}} {
 ! CHECK:          acc.yield
 ! CHECK-NEXT:   }{{$}}
 ! CHECK:        acc.yield
 ! CHECK-NEXT: }{{$}}
+! CHECK:      acc.delete accPtr(%[[NOCREATE_A]] : !fir.ref<!fir.array<10xf32>>) {dataClause = #acc<data_clause acc_no_create>, name = "a"}
+! CHECK:      acc.delete accPtr(%[[NOCREATE_B]] : !fir.ref<!fir.array<10xf32>>) {dataClause = #acc<data_clause acc_no_create>, name = "b"}
 
   !$acc serial loop present(a, b)
   DO i = 1, n
     a(i) = b(i)
   END DO
 
-! CHECK:      %[[PRESENT_A:.*]] = acc.present varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10xf32>>) bounds(%{{.*}}) -> !fir.ref<!fir.array<10xf32>> {name = "a"}
-! CHECK:      %[[PRESENT_B:.*]] = acc.present varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<10xf32>>) bounds(%{{.*}}) -> !fir.ref<!fir.array<10xf32>> {name = "b"}
+! CHECK:      %[[PRESENT_A:.*]] = acc.present varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10xf32>>) -> !fir.ref<!fir.array<10xf32>> {name = "a"}
+! CHECK:      %[[PRESENT_B:.*]] = acc.present varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<10xf32>>) -> !fir.ref<!fir.array<10xf32>> {name = "b"}
 ! CHECK:      acc.serial {{.*}} dataOperands(%[[PRESENT_A]], %[[PRESENT_B]] : !fir.ref<!fir.array<10xf32>>, !fir.ref<!fir.array<10xf32>>) {
 ! CHECK:        acc.loop {{.*}} {
 ! CHECK:          acc.yield
 ! CHECK-NEXT:   }{{$}}
 ! CHECK:        acc.yield
 ! CHECK-NEXT: }{{$}}
+! CHECK:      acc.delete accPtr(%[[PRESENT_A]] : !fir.ref<!fir.array<10xf32>>) {dataClause = #acc<data_clause acc_present>, name = "a"}
+! CHECK:      acc.delete accPtr(%[[PRESENT_B]] : !fir.ref<!fir.array<10xf32>>) {dataClause = #acc<data_clause acc_present>, name = "b"}
 
   !$acc serial loop deviceptr(a) deviceptr(b)
   DO i = 1, n
     a(i) = b(i)
   END DO
 
-! CHECK:      %[[DEVICEPTR_A:.*]] = acc.deviceptr varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10xf32>>) bounds(%{{.*}}) -> !fir.ref<!fir.array<10xf32>> {name = "a"}
-! CHECK:      %[[DEVICEPTR_B:.*]] = acc.deviceptr varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<10xf32>>) bounds(%{{.*}}) -> !fir.ref<!fir.array<10xf32>> {name = "b"}
+! CHECK:      %[[DEVICEPTR_A:.*]] = acc.deviceptr varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10xf32>>) -> !fir.ref<!fir.array<10xf32>> {name = "a"}
+! CHECK:      %[[DEVICEPTR_B:.*]] = acc.deviceptr varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<10xf32>>) -> !fir.ref<!fir.array<10xf32>> {name = "b"}
 ! CHECK:      acc.serial {{.*}} dataOperands(%[[DEVICEPTR_A]], %[[DEVICEPTR_B]] : !fir.ref<!fir.array<10xf32>>, !fir.ref<!fir.array<10xf32>>) {
 ! CHECK:        acc.loop {{.*}} {
 ! CHECK:          acc.yield
@@ -368,13 +372,9 @@ subroutine acc_serial_loop
     a(i) = b(i)
   END DO
 
-! CHECK:      %[[BOX_F:.*]] = fir.load %[[DECLF]]#0 : !fir.ref<!fir.box<!fir.ptr<f32>>>
-! CHECK:      %[[BOX_ADDR_F:.*]] = fir.box_addr %[[BOX_F]] : (!fir.box<!fir.ptr<f32>>) -> !fir.ptr<f32>
-! CHECK:      %[[ATTACH_F:.*]] = acc.attach varPtr(%[[BOX_ADDR_F]] : !fir.ptr<f32>) -> !fir.ptr<f32> {name = "f"}
-! CHECK:      %[[BOX_G:.*]] = fir.load %[[DECLG]]#0 : !fir.ref<!fir.box<!fir.ptr<f32>>>
-! CHECK:      %[[BOX_ADDR_G:.*]] = fir.box_addr %[[BOX_G]] : (!fir.box<!fir.ptr<f32>>) -> !fir.ptr<f32>
-! CHECK:      %[[ATTACH_G:.*]] = acc.attach varPtr(%[[BOX_ADDR_G]] : !fir.ptr<f32>) -> !fir.ptr<f32> {name = "g"}
-! CHECK:      acc.serial {{.*}} dataOperands(%[[ATTACH_F]], %[[ATTACH_G]] : !fir.ptr<f32>, !fir.ptr<f32>) {
+! CHECK:      %[[ATTACH_F:.*]] = acc.attach varPtr(%[[DECLF]]#0 : !fir.ref<!fir.box<!fir.ptr<f32>>>) -> !fir.ref<!fir.box<!fir.ptr<f32>>> {name = "f"}
+! CHECK:      %[[ATTACH_G:.*]] = acc.attach varPtr(%[[DECLG]]#0 : !fir.ref<!fir.box<!fir.ptr<f32>>>) -> !fir.ref<!fir.box<!fir.ptr<f32>>> {name = "g"}
+! CHECK:      acc.serial {{.*}} dataOperands(%[[ATTACH_F]], %[[ATTACH_G]] : !fir.ref<!fir.box<!fir.ptr<f32>>>, !fir.ref<!fir.box<!fir.ptr<f32>>>) {
 ! CHECK:        acc.loop {{.*}} {
 ! CHECK:          acc.yield
 ! CHECK-NEXT:   }{{$}}
@@ -386,9 +386,9 @@ subroutine acc_serial_loop
     a(i) = b(i)
   END DO
 
-! CHECK:      %[[ACC_FPRIVATE_B:.*]] = acc.firstprivate varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<10xf32>>) bounds(%{{.*}}) -> !fir.ref<!fir.array<10xf32>> {name = "b"}
-! CHECK:      acc.serial {{.*}} firstprivate(@firstprivatization_section_ext10_ref_10xf32 -> %[[ACC_FPRIVATE_B]] : !fir.ref<!fir.array<10xf32>>) {
-! CHECK:      %[[ACC_PRIVATE_A:.*]] = acc.private varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10xf32>>) bounds(%{{.*}}) -> !fir.ref<!fir.array<10xf32>> {name = "a"}
+! CHECK:      %[[ACC_FPRIVATE_B:.*]] = acc.firstprivate varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<10xf32>>) -> !fir.ref<!fir.array<10xf32>> {name = "b"}
+! CHECK:      acc.serial {{.*}} firstprivate(@firstprivatization_ref_10xf32 -> %[[ACC_FPRIVATE_B]] : !fir.ref<!fir.array<10xf32>>) {
+! CHECK:      %[[ACC_PRIVATE_A:.*]] = acc.private varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10xf32>>) -> !fir.ref<!fir.array<10xf32>> {name = "a"}
 ! CHECK:        acc.loop {{.*}} private({{.*}}@privatization_ref_10xf32 -> %[[ACC_PRIVATE_A]] : !fir.ref<!fir.array<10xf32>>)
 ! CHECK-NOT:          fir.do_loop
 ! CHECK:          acc.yield
diff --git a/flang/test/Lower/OpenACC/acc-serial.f90 b/flang/test/Lower/OpenACC/acc-serial.f90
index 6f79e18c71442..f2f73876af877 100644
--- a/flang/test/Lower/OpenACC/acc-serial.f90
+++ b/flang/test/Lower/OpenACC/acc-serial.f90
@@ -2,7 +2,7 @@
 
 ! RUN: bbc -fopenacc -emit-hlfir %s -o - | FileCheck %s
 
-! CHECK-LABEL: acc.firstprivate.recipe @firstprivatization_section_ext10xext10_ref_10x10xf32 : !fir.ref<!fir.array<10x10xf32>> init {
+! CHECK-LABEL: acc.firstprivate.recipe @firstprivatization_ref_10x10xf32 : !fir.ref<!fir.array<10x10xf32>> init {
 ! CHECK: ^bb0(%{{.*}}: !fir.ref<!fir.array<10x10xf32>>):
 ! CHECK:   %[[SHAPE:.*]] = fir.shape %{{.*}}, %{{.*}} : (index, index) -> !fir.shape<2>
 ! CHECK:   %[[ALLOCA:.*]] = fir.alloca !fir.array<10x10xf32>
@@ -156,93 +156,99 @@ subroutine acc_serial
   !$acc serial copy(a, b, c)
   !$acc end serial
 
-! CHECK:      %[[COPYIN_A:.*]] = acc.copyin varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = #acc<data_clause acc_copy>, name = "a"}
-! CHECK:      %[[COPYIN_B:.*]] = acc.copyin varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = #acc<data_clause acc_copy>, name = "b"}
-! CHECK:      %[[COPYIN_C:.*]] = acc.copyin varPtr(%[[DECLC]]#0 : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = #acc<data_clause acc_copy>, name = "c"}
+! CHECK:      %[[COPYIN_A:.*]] = acc.copyin varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = #acc<data_clause acc_copy>, name = "a"}
+! CHECK:      %[[COPYIN_B:.*]] = acc.copyin varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<10x10xf32>>) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = #acc<data_clause acc_copy>, name = "b"}
+! CHECK:      %[[COPYIN_C:.*]] = acc.copyin varPtr(%[[DECLC]]#0 : !fir.ref<!fir.array<10x10xf32>>) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = #acc<data_clause acc_copy>, name = "c"}
 ! CHECK:      acc.serial dataOperands(%[[COPYIN_A]], %[[COPYIN_B]], %[[COPYIN_C]] : !fir.ref<!fir.array<10x10xf32>>, !fir.ref<!fir.array<10x10xf32>>, !fir.ref<!fir.array<10x10xf32>>) {
 ! CHECK:        acc.yield
 ! CHECK-NEXT: }{{$}}
-! CHECK:      acc.copyout accPtr(%[[COPYIN_A]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) to varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) {dataClause = #acc<data_clause acc_copy>, name = "a"}
-! CHECK:      acc.copyout accPtr(%[[COPYIN_B]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) to varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<10x10xf32>>) {dataClause = #acc<data_clause acc_copy>, name = "b"}
-! CHECK:      acc.copyout accPtr(%[[COPYIN_C]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) to varPtr(%[[DECLC]]#0 : !fir.ref<!fir.array<10x10xf32>>) {dataClause = #acc<data_clause acc_copy>, name = "c"}
+! CHECK:      acc.copyout accPtr(%[[COPYIN_A]] : !fir.ref<!fir.array<10x10xf32>>) to varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) {dataClause = #acc<data_clause acc_copy>, name = "a"}
+! CHECK:      acc.copyout accPtr(%[[COPYIN_B]] : !fir.ref<!fir.array<10x10xf32>>) to varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<10x10xf32>>) {dataClause = #acc<data_clause acc_copy>, name = "b"}
+! CHECK:      acc.copyout accPtr(%[[COPYIN_C]] : !fir.ref<!fir.array<10x10xf32>>) to varPtr(%[[DECLC]]#0 : !fir.ref<!fir.array<10x10xf32>>) {dataClause = #acc<data_clause acc_copy>, name = "c"}
 
   !$acc serial copy(a) copy(b) copy(c)
   !$acc end serial
 
-! CHECK:      %[[COPYIN_A:.*]] = acc.copyin varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = #acc<data_clause acc_copy>, name = "a"}
-! CHECK:      %[[COPYIN_B:.*]] = acc.copyin varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = #acc<data_clause acc_copy>, name = "b"}
-! CHECK:      %[[COPYIN_C:.*]] = acc.copyin varPtr(%[[DECLC]]#0 : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = #acc<data_clause acc_copy>, name = "c"}
+! CHECK:      %[[COPYIN_A:.*]] = acc.copyin varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = #acc<data_clause acc_copy>, name = "a"}
+! CHECK:      %[[COPYIN_B:.*]] = acc.copyin varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<10x10xf32>>) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = #acc<data_clause acc_copy>, name = "b"}
+! CHECK:      %[[COPYIN_C:.*]] = acc.copyin varPtr(%[[DECLC]]#0 : !fir.ref<!fir.array<10x10xf32>>) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = #acc<data_clause acc_copy>, name = "c"}
 ! CHECK:      acc.serial dataOperands(%[[COPYIN_A]], %[[COPYIN_B]], %[[COPYIN_C]] : !fir.ref<!fir.array<10x10xf32>>, !fir.ref<!fir.array<10x10xf32>>, !fir.ref<!fir.array<10x10xf32>>) {
 ! CHECK:        acc.yield
 ! CHECK-NEXT: }{{$}}
-! CHECK:      acc.copyout accPtr(%[[COPYIN_A]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) to varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) {dataClause = #acc<data_clause acc_copy>, name = "a"}
-! CHECK:      acc.copyout accPtr(%[[COPYIN_B]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) to varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<10x10xf32>>) {dataClause = #acc<data_clause acc_copy>, name = "b"}
-! CHECK:      acc.copyout accPtr(%[[COPYIN_C]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) to varPtr(%[[DECLC]]#0 : !fir.ref<!fir.array<10x10xf32>>) {dataClause = #acc<data_clause acc_copy>, name = "c"}
+! CHECK:      acc.copyout accPtr(%[[COPYIN_A]] : !fir.ref<!fir.array<10x10xf32>>) to varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) {dataClause = #acc<data_clause acc_copy>, name = "a"}
+! CHECK:      acc.copyout accPtr(%[[COPYIN_B]] : !fir.ref<!fir.array<10x10xf32>>) to varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<10x10xf32>>) {dataClause = #acc<data_clause acc_copy>, name = "b"}
+! CHECK:      acc.copyout accPtr(%[[COPYIN_C]] : !fir.ref<!fir.array<10x10xf32>>) to varPtr(%[[DECLC]]#0 : !fir.ref<!fir.array<10x10xf32>>) {dataClause = #acc<data_clause acc_copy>, name = "c"}
 
   !$acc serial copyin(a) copyin(readonly: b, c)
   !$acc end serial
 
-! CHECK:      %[[COPYIN_A:.*]] = acc.copyin varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {name = "a"}
-! CHECK:      %[[COPYIN_B:.*]] = acc.copyin varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = #acc<data_clause acc_copyin_readonly>, name = "b"}
-! CHECK:      %[[COPYIN_C:.*]] = acc.copyin varPtr(%[[DECLC]]#0 : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = #acc<data_clause acc_copyin_readonly>, name = "c"}
+! CHECK:      %[[COPYIN_A:.*]] = acc.copyin varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) -> !fir.ref<!fir.array<10x10xf32>> {name = "a"}
+! CHECK:      %[[COPYIN_B:.*]] = acc.copyin varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<10x10xf32>>) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = #acc<data_clause acc_copyin_readonly>, name = "b"}
+! CHECK:      %[[COPYIN_C:.*]] = acc.copyin varPtr(%[[DECLC]]#0 : !fir.ref<!fir.array<10x10xf32>>) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = #acc<data_clause acc_copyin_readonly>, name = "c"}
 ! CHECK:      acc.serial dataOperands(%[[COPYIN_A]], %[[COPYIN_B]], %[[COPYIN_C]] : !fir.ref<!fir.array<10x10xf32>>, !fir.ref<!fir.array<10x10xf32>>, !fir.ref<!fir.array<10x10xf32>>) {
 ! CHECK:        acc.yield
 ! CHECK-NEXT: }{{$}}
-! CHECK: acc.delete accPtr(%[[COPYIN_A]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) {dataClause = #acc<data_clause acc_copyin>, name = "a"}
-! CHECK: acc.delete accPtr(%[[COPYIN_B]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) {dataClause = #acc<data_clause acc_copyin_readonly>, name = "b"}
-! CHECK: acc.delete accPtr(%[[COPYIN_C]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) {dataClause = #acc<data_clause acc_copyin_readonly>, name = "c"}
+! CHECK: acc.delete accPtr(%[[COPYIN_A]] : !fir.ref<!fir.array<10x10xf32>>) {dataClause = #acc<data_clause acc_copyin>, name = "a"}
+! CHECK: acc.delete accPtr(%[[COPYIN_B]] : !fir.ref<!fir.array<10x10xf32>>) {dataClause = #acc<data_clause acc_copyin_readonly>, name = "b"}
+! CHECK: acc.delete accPtr(%[[COPYIN_C]] : !fir.ref<!fir.array<10x10xf32>>) {dataClause = #acc<data_clause acc_copyin_readonly>, name = "c"}
 
   !$acc serial copyout(a) copyout(zero: b) copyout(c)
   !$acc end serial
 
-! CHECK: %[[CREATE_A:.*]] = acc.create varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = #acc<data_clause acc_copyout>, name = "a"}
-! CHECK: %[[CREATE_B:.*]] = acc.create varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = #acc<data_clause acc_copyout>, name = "b"}
-! CHECK: %[[CREATE_C:.*]] = acc.create varPtr(%[[DECLC]]#0 : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = #acc<data_clause acc_copyout>, name = "c"}
+! CHECK: %[[CREATE_A:.*]] = acc.create varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = #acc<data_clause acc_copyout>, name = "a"}
+! CHECK: %[[CREATE_B:.*]] = acc.create varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<10x10xf32>>) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = #acc<data_clause acc_copyout>, name = "b"}
+! CHECK: %[[CREATE_C:.*]] = acc.create varPtr(%[[DECLC]]#0 : !fir.ref<!fir.array<10x10xf32>>) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = #acc<data_clause acc_copyout>, name = "c"}
 ! CHECK:      acc.serial dataOperands(%[[CREATE_A]], %[[CREATE_B]], %[[CREATE_C]] : !fir.ref<!fir.array<10x10xf32>>, !fir.ref<!fir.array<10x10xf32>>, !fir.ref<!fir.array<10x10xf32>>) {
 ! CHECK:        acc.yield
 ! CHECK-NEXT: }{{$}}
-! CHECK: acc.copyout accPtr(%[[CREATE_A]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) to varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) {name = "a"}
-! CHECK: acc.copyout accPtr(%[[CREATE_B]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) to varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<10x10xf32>>) {name = "b"}
-! CHECK: acc.copyout accPtr(%[[CREATE_C]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) to varPtr(%[[DECLC]]#0 : !fir.ref<!fir.array<10x10xf32>>) {name = "c"}
+! CHECK: acc.copyout accPtr(%[[CREATE_A]] : !fir.ref<!fir.array<10x10xf32>>) to varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) {name = "a"}
+! CHECK: acc.copyout accPtr(%[[CREATE_B]] : !fir.ref<!fir.array<10x10xf32>>) to varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<10x10xf32>>) {name = "b"}
+! CHECK: acc.copyout accPtr(%[[CREATE_C]] : !fir.ref<!fir.array<10x10xf32>>) to varPtr(%[[DECLC]]#0 : !fir.ref<!fir.array<10x10xf32>>) {name = "c"}
 
   !$acc serial create(a, b) create(zero: c)
   !$acc end serial
 
-! CHECK: %[[CREATE_A:.*]] = acc.create varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {name = "a"}
-! CHECK: %[[CREATE_B:.*]] = acc.create varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {name = "b"}
-! CHECK: %[[CREATE_C:.*]] = acc.create varPtr(%[[DECLC]]#0 : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = #acc<data_clause acc_create_zero>, name = "c"}
+! CHECK: %[[CREATE_A:.*]] = acc.create varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) -> !fir.ref<!fir.array<10x10xf32>> {name = "a"}
+! CHECK: %[[CREATE_B:.*]] = acc.create varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<10x10xf32>>) -> !fir.ref<!fir.array<10x10xf32>> {name = "b"}
+! CHECK: %[[CREATE_C:.*]] = acc.create varPtr(%[[DECLC]]#0 : !fir.ref<!fir.array<10x10xf32>>) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = #acc<data_clause acc_create_zero>, name = "c"}
 ! CHECK:      acc.serial dataOperands(%[[CREATE_A]], %[[CREATE_B]], %[[CREATE_C]] : !fir.ref<!fir.array<10x10xf32>>, !fir.ref<!fir.array<10x10xf32>>, !fir.ref<!fir.array<10x10xf32>>) {
 ! CHECK:        acc.yield
 ! CHECK-NEXT: }{{$}}
-! CHECK: acc.delete accPtr(%[[CREATE_A]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) {dataClause = #acc<data_clause acc_create>, name = "a"}
-! CHECK: acc.delete accPtr(%[[CREATE_B]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) {dataClause = #acc<data_clause acc_create>, name = "b"}
-! CHECK: acc.delete accPtr(%[[CREATE_C]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) {dataClause = #acc<data_clause acc_create_zero>, name = "c"}
+! CHECK: acc.delete accPtr(%[[CREATE_A]] : !fir.ref<!fir.array<10x10xf32>>) {dataClause = #acc<data_clause acc_create>, name = "a"}
+! CHECK: acc.delete accPtr(%[[CREATE_B]] : !fir.ref<!fir.array<10x10xf32>>) {dataClause = #acc<data_clause acc_create>, name = "b"}
+! CHECK: acc.delete accPtr(%[[CREATE_C]] : !fir.ref<!fir.array<10x10xf32>>) {dataClause = #acc<data_clause acc_create_zero>, name = "c"}
 
   !$acc serial no_create(a, b) create(zero: c)
   !$acc end serial
 
-! CHECK: %[[NO_CREATE_A:.*]] = acc.nocreate varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {name = "a"}
-! CHECK: %[[NO_CREATE_B:.*]] = acc.nocreate varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {name = "b"}
-! CHECK: %[[CREATE_C:.*]] = acc.create varPtr(%[[DECLC]]#0 : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = #acc<data_clause acc_create_zero>, name = "c"}
+! CHECK: %[[NO_CREATE_A:.*]] = acc.nocreate varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) -> !fir.ref<!fir.array<10x10xf32>> {name = "a"}
+! CHECK: %[[NO_CREATE_B:.*]] = acc.nocreate varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<10x10xf32>>) -> !fir.ref<!fir.array<10x10xf32>> {name = "b"}
+! CHECK: %[[CREATE_C:.*]] = acc.create varPtr(%[[DECLC]]#0 : !fir.ref<!fir.array<10x10xf32>>) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = #acc<data_clause acc_create_zero>, name = "c"}
 ! CHECK:      acc.serial dataOperands(%[[NO_CREATE_A]], %[[NO_CREATE_B]], %[[CREATE_C]] : !fir.ref<!fir.array<10x10xf32>>, !fir.ref<!fir.array<10x10xf32>>, !fir.ref<!fir.array<10x10xf32>>) {
 ! CHECK:        acc.yield
 ! CHECK-NEXT: }{{$}}
+! CHECK: acc.delete accPtr(%[[CREATE_C]] : !fir.ref<!fir.array<10x10xf32>>) {dataClause = #acc<data_clause acc_create_zero>, name = "c"}
+! CHECK: acc.delete accPtr(%[[NO_CREATE_A]] : !fir.ref<!fir.array<10x10xf32>>) {dataClause = #acc<data_clause acc_no_create>, name = "a"}
+! CHECK: acc.delete accPtr(%[[NO_CREATE_B]] : !fir.ref<!fir.array<10x10xf32>>) {dataClause = #acc<data_clause acc_no_create>, name = "b"}
 
   !$acc serial present(a, b, c)
   !$acc end serial
 
-! CHECK: %[[PRESENT_A:.*]] = acc.present varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {name = "a"}
-! CHECK: %[[PRESENT_B:.*]] = acc.present varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {name = "b"}
-! CHECK: %[[PRESENT_C:.*]] = acc.present varPtr(%[[DECLC]]#0 : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {name = "c"}
+! CHECK: %[[PRESENT_A:.*]] = acc.present varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) -> !fir.ref<!fir.array<10x10xf32>> {name = "a"}
+! CHECK: %[[PRESENT_B:.*]] = acc.present varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<10x10xf32>>) -> !fir.ref<!fir.array<10x10xf32>> {name = "b"}
+! CHECK: %[[PRESENT_C:.*]] = acc.present varPtr(%[[DECLC]]#0 : !fir.ref<!fir.array<10x10xf32>>) -> !fir.ref<!fir.array<10x10xf32>> {name = "c"}
 ! CHECK:      acc.serial dataOperands(%[[PRESENT_A]], %[[PRESENT_B]], %[[PRESENT_C]] : !fir.ref<!fir.array<10x10xf32>>, !fir.ref<!fir.array<10x10xf32>>, !fir.ref<!fir.array<10x10xf32>>) {
 ! CHECK:        acc.yield
 ! CHECK-NEXT: }{{$}}
+! CHECK: acc.delete accPtr(%[[PRESENT_A]] : !fir.ref<!fir.array<10x10xf32>>) {dataClause = #acc<data_clause acc_present>, name = "a"}
+! CHECK: acc.delete accPtr(%[[PRESENT_B]] : !fir.ref<!fir.array<10x10xf32>>) {dataClause = #acc<data_clause acc_present>, name = "b"}
+! CHECK: acc.delete accPtr(%[[PRESENT_C]] : !fir.ref<!fir.array<10x10xf32>>) {dataClause = #acc<data_clause acc_present>, name = "c"}
 
   !$acc serial deviceptr(a) deviceptr(c)
   !$acc end serial
 
-! CHECK: %[[DEVICEPTR_A:.*]] = acc.deviceptr varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {name = "a"}
-! CHECK: %[[DEVICEPTR_C:.*]] = acc.deviceptr varPtr(%[[DECLC]]#0 : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {name = "c"}
+! CHECK: %[[DEVICEPTR_A:.*]] = acc.deviceptr varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) -> !fir.ref<!fir.array<10x10xf32>> {name = "a"}
+! CHECK: %[[DEVICEPTR_C:.*]] = acc.deviceptr varPtr(%[[DECLC]]#0 : !fir.ref<!fir.array<10x10xf32>>) -> !fir.ref<!fir.array<10x10xf32>> {name = "c"}
 ! CHECK:      acc.serial dataOperands(%[[DEVICEPTR_A]], %[[DEVICEPTR_C]] : !fir.ref<!fir.array<10x10xf32>>, !fir.ref<!fir.array<10x10xf32>>) {
 ! CHECK:        acc.yield
 ! CHECK-NEXT: }{{$}}
@@ -250,25 +256,21 @@ subroutine acc_serial
   !$acc serial attach(d, e)
   !$acc end serial
 
-! CHECK: %[[BOX_D:.*]] = fir.load %[[DECLD]]#0 : !fir.ref<!fir.box<!fir.ptr<f32>>>
-! CHECK: %[[BOX_ADDR_D:.*]] = fir.box_addr %[[BOX_D]] : (!fir.box<!fir.ptr<f32>>) -> !fir.ptr<f32>
-! CHECK: %[[ATTACH_D:.*]] = acc.attach varPtr(%[[BOX_ADDR_D]] : !fir.ptr<f32>) -> !fir.ptr<f32> {name = "d"}
-! CHECK: %[[BOX_E:.*]] = fir.load %[[DECLE]]#0 : !fir.ref<!fir.box<!fir.ptr<f32>>>
-! CHECK: %[[BOX_ADDR_E:.*]] = fir.box_addr %[[BOX_E]] : (!fir.box<!fir.ptr<f32>>) -> !fir.ptr<f32>
-! CHECK: %[[ATTACH_E:.*]] = acc.attach varPtr(%[[BOX_ADDR_E]] : !fir.ptr<f32>) -> !fir.ptr<f32> {name = "e"}
-! CHECK:      acc.serial dataOperands(%[[ATTACH_D]], %[[ATTACH_E]] : !fir.ptr<f32>, !fir.ptr<f32>) {
+! CHECK: %[[ATTACH_D:.*]] = acc.attach varPtr(%[[DECLD]]#0 : !fir.ref<!fir.box<!fir.ptr<f32>>>) -> !fir.ref<!fir.box<!fir.ptr<f32>>> {name = "d"}
+! CHECK: %[[ATTACH_E:.*]] = acc.attach varPtr(%[[DECLE]]#0 : !fir.ref<!fir.box<!fir.ptr<f32>>>) -> !fir.ref<!fir.box<!fir.ptr<f32>>> {name = "e"}
+! CHECK:      acc.serial dataOperands(%[[ATTACH_D]], %[[ATTACH_E]] : !fir.ref<!fir.box<!fir.ptr<f32>>>, !fir.ref<!fir.box<!fir.ptr<f32>>>) {
 ! CHECK:        acc.yield
 ! CHECK-NEXT: }{{$}}
-! CHECK: acc.detach accPtr(%[[ATTACH_D]] : !fir.ptr<f32>) {dataClause = #acc<data_clause acc_attach>, name = "d"}
-! CHECK: acc.detach accPtr(%[[ATTACH_E]] : !fir.ptr<f32>) {dataClause = #acc<data_clause acc_attach>, name = "e"}
+! CHECK: acc.detach accPtr(%[[ATTACH_D]] : !fir.ref<!fir.box<!fir.ptr<f32>>>) {dataClause = #acc<data_clause acc_attach>, name = "d"}
+! CHECK: acc.detach accPtr(%[[ATTACH_E]] : !fir.ref<!fir.box<!fir.ptr<f32>>>) {dataClause = #acc<data_clause acc_attach>, name = "e"}
 
   !$acc serial private(a) firstprivate(b) private(c)
   !$acc end serial
 
-! CHECK:      %[[ACC_PRIVATE_A:.*]] = acc.private varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {name = "a"}
-! CHECK:      %[[ACC_FPRIVATE_B:.*]] = acc.firstprivate varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {name = "b"}
-! CHECK:      %[[ACC_PRIVATE_C:.*]] = acc.private varPtr(%[[DECLC]]#0 : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {name = "c"}
-! CHECK:      acc.serial firstprivate(@firstprivatization_section_ext10xext10_ref_10x10xf32 -> %[[ACC_FPRIVATE_B]] : !fir.ref<!fir.array<10x10xf32>>) private(@privatization_ref_10x10xf32 -> %[[ACC_PRIVATE_A]] : !fir.ref<!fir.array<10x10xf32>>, @privatization_ref_10x10xf32 -> %[[ACC_PRIVATE_C]] : !fir.ref<!fir.array<10x10xf32>>) {
+! CHECK:      %[[ACC_PRIVATE_A:.*]] = acc.private varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) -> !fir.ref<!fir.array<10x10xf32>> {name = "a"}
+! CHECK:      %[[ACC_FPRIVATE_B:.*]] = acc.firstprivate varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<10x10xf32>>) -> !fir.ref<!fir.array<10x10xf32>> {name = "b"}
+! CHECK:      %[[ACC_PRIVATE_C:.*]] = acc.private varPtr(%[[DECLC]]#0 : !fir.ref<!fir.array<10x10xf32>>) -> !fir.ref<!fir.array<10x10xf32>> {name = "c"}
+! CHECK:      acc.serial firstprivate(@firstprivatization_ref_10x10xf32 -> %[[ACC_FPRIVATE_B]] : !fir.ref<!fir.array<10x10xf32>>) private(@privatization_ref_10x10xf32 -> %[[ACC_PRIVATE_A]] : !fir.ref<!fir.array<10x10xf32>>, @privatization_ref_10x10xf32 -> %[[ACC_PRIVATE_C]] : !fir.ref<!fir.array<10x10xf32>>) {
 ! CHECK:        acc.yield
 ! CHECK-NEXT: }{{$}}
 
diff --git a/flang/test/Lower/OpenACC/acc-set.f90 b/flang/test/Lower/OpenACC/acc-set.f90
index 31c1912a87f57..b249a53533e6e 100644
--- a/flang/test/Lower/OpenACC/acc-set.f90
+++ b/flang/test/Lower/OpenACC/acc-set.f90
@@ -25,7 +25,6 @@ program test_acc_set
 ! CHECK: acc.set default_async(%[[C1]] : i32)
 
 ! CHECK: %[[C1:.*]] = arith.constant 1 : i32
-! FIR: %[[LOAD_L:.*]] = fir.load %[[L]] : !fir.ref<!fir.logical<4>>
 ! HLFIR: %[[LOAD_L:.*]] = fir.load %[[DECLL]]#0 : !fir.ref<!fir.logical<4>>
 ! CHECK: %[[CONV_L:.*]] = fir.convert %[[LOAD_L]] : (!fir.logical<4>) -> i1
 ! CHECK: acc.set default_async(%[[C1]] : i32) if(%[[CONV_L]])
diff --git a/flang/test/Lower/OpenACC/acc-update.f90 b/flang/test/Lower/OpenACC/acc-update.f90
index 0964fd91457f9..1ae06a8c1d0e8 100644
--- a/flang/test/Lower/OpenACC/acc-update.f90
+++ b/flang/test/Lower/OpenACC/acc-update.f90
@@ -15,98 +15,98 @@ subroutine acc_update
 ! CHECK: %[[DECLC:.*]]:2 = hlfir.declare %[[C]]
 
   !$acc update host(a)
-! CHECK: %[[DEVPTR_A:.*]] = acc.getdeviceptr varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = #acc<data_clause acc_update_host>, name = "a", structured = false}
+! CHECK: %[[DEVPTR_A:.*]] = acc.getdeviceptr varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = #acc<data_clause acc_update_host>, name = "a", structured = false}
 ! CHECK: acc.update dataOperands(%[[DEVPTR_A]] : !fir.ref<!fir.array<10x10xf32>>){{$}}
-! CHECK: acc.update_host accPtr(%[[DEVPTR_A]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) to varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) {name = "a", structured = false}
+! CHECK: acc.update_host accPtr(%[[DEVPTR_A]] : !fir.ref<!fir.array<10x10xf32>>) to varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) {name = "a", structured = false}
 
   !$acc update host(a) if_present
-! CHECK: %[[DEVPTR_A:.*]] = acc.getdeviceptr varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = #acc<data_clause acc_update_host>, name = "a", structured = false}
+! CHECK: %[[DEVPTR_A:.*]] = acc.getdeviceptr varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = #acc<data_clause acc_update_host>, name = "a", structured = false}
 ! CHECK: acc.update dataOperands(%[[DEVPTR_A]] : !fir.ref<!fir.array<10x10xf32>>) attributes {ifPresent}{{$}}
-! CHECK: acc.update_host accPtr(%[[DEVPTR_A]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) to varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) {name = "a", structured = false}
+! CHECK: acc.update_host accPtr(%[[DEVPTR_A]] : !fir.ref<!fir.array<10x10xf32>>) to varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) {name = "a", structured = false}
 
   !$acc update self(a)
-! CHECK: %[[DEVPTR_A:.*]] = acc.getdeviceptr varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = #acc<data_clause acc_update_self>, name = "a", structured = false}
+! CHECK: %[[DEVPTR_A:.*]] = acc.getdeviceptr varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = #acc<data_clause acc_update_self>, name = "a", structured = false}
 ! CHECK: acc.update dataOperands(%[[DEVPTR_A]] : !fir.ref<!fir.array<10x10xf32>>){{$}}
-! CHECK: acc.update_host accPtr(%[[DEVPTR_A]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) to varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) {dataClause = #acc<data_clause acc_update_self>, name = "a", structured = false}
+! CHECK: acc.update_host accPtr(%[[DEVPTR_A]] : !fir.ref<!fir.array<10x10xf32>>) to varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) {dataClause = #acc<data_clause acc_update_self>, name = "a", structured = false}
 
   !$acc update host(a) if(.true.)
-! CHECK: %[[DEVPTR_A:.*]] = acc.getdeviceptr varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = #acc<data_clause acc_update_host>, name = "a", structured = false}
+! CHECK: %[[DEVPTR_A:.*]] = acc.getdeviceptr varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = #acc<data_clause acc_update_host>, name = "a", structured = false}
 ! CHECK: %[[IF1:.*]] = arith.constant true
 ! CHECK: acc.update if(%[[IF1]]) dataOperands(%[[DEVPTR_A]] : !fir.ref<!fir.array<10x10xf32>>){{$}}
-! CHECK: acc.update_host accPtr(%[[DEVPTR_A]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) to varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) {name = "a", structured = false}
+! CHECK: acc.update_host accPtr(%[[DEVPTR_A]] : !fir.ref<!fir.array<10x10xf32>>) to varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) {name = "a", structured = false}
 
   !$acc update host(a) if(ifCondition)
-! CHECK: %[[DEVPTR_A:.*]] = acc.getdeviceptr varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = #acc<data_clause acc_update_host>, name = "a", structured = false}
+! CHECK: %[[DEVPTR_A:.*]] = acc.getdeviceptr varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = #acc<data_clause acc_update_host>, name = "a", structured = false}
 ! CHECK: %[[IFCOND:.*]] = fir.load %{{.*}} : !fir.ref<!fir.logical<4>>
 ! CHECK: %[[IF2:.*]] = fir.convert %[[IFCOND]] : (!fir.logical<4>) -> i1
 ! CHECK: acc.update if(%[[IF2]]) dataOperands(%[[DEVPTR_A]] : !fir.ref<!fir.array<10x10xf32>>){{$}}
-! CHECK: acc.update_host accPtr(%[[DEVPTR_A]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) to varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) {name = "a", structured = false}
+! CHECK: acc.update_host accPtr(%[[DEVPTR_A]] : !fir.ref<!fir.array<10x10xf32>>) to varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) {name = "a", structured = false}
 
   !$acc update host(a) host(b) host(c)
-! CHECK: %[[DEVPTR_A:.*]] = acc.getdeviceptr varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = #acc<data_clause acc_update_host>, name = "a", structured = false}
-! CHECK: %[[DEVPTR_B:.*]] = acc.getdeviceptr varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = #acc<data_clause acc_update_host>, name = "b", structured = false}
-! CHECK: %[[DEVPTR_C:.*]] = acc.getdeviceptr varPtr(%[[DECLC]]#0 : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = #acc<data_clause acc_update_host>, name = "c", structured = false}
+! CHECK: %[[DEVPTR_A:.*]] = acc.getdeviceptr varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = #acc<data_clause acc_update_host>, name = "a", structured = false}
+! CHECK: %[[DEVPTR_B:.*]] = acc.getdeviceptr varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<10x10xf32>>) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = #acc<data_clause acc_update_host>, name = "b", structured = false}
+! CHECK: %[[DEVPTR_C:.*]] = acc.getdeviceptr varPtr(%[[DECLC]]#0 : !fir.ref<!fir.array<10x10xf32>>) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = #acc<data_clause acc_update_host>, name = "c", structured = false}
 ! CHECK: acc.update dataOperands(%[[DEVPTR_A]], %[[DEVPTR_B]], %[[DEVPTR_C]] : !fir.ref<!fir.array<10x10xf32>>, !fir.ref<!fir.array<10x10xf32>>, !fir.ref<!fir.array<10x10xf32>>){{$}}
-! CHECK: acc.update_host accPtr(%[[DEVPTR_A]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) to varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) {name = "a", structured = false}
-! CHECK: acc.update_host accPtr(%[[DEVPTR_B]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) to varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<10x10xf32>>) {name = "b", structured = false}
-! CHECK: acc.update_host accPtr(%[[DEVPTR_C]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) to varPtr(%[[DECLC]]#0 : !fir.ref<!fir.array<10x10xf32>>) {name = "c", structured = false}
+! CHECK: acc.update_host accPtr(%[[DEVPTR_A]] : !fir.ref<!fir.array<10x10xf32>>) to varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) {name = "a", structured = false}
+! CHECK: acc.update_host accPtr(%[[DEVPTR_B]] : !fir.ref<!fir.array<10x10xf32>>) to varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<10x10xf32>>) {name = "b", structured = false}
+! CHECK: acc.update_host accPtr(%[[DEVPTR_C]] : !fir.ref<!fir.array<10x10xf32>>) to varPtr(%[[DECLC]]#0 : !fir.ref<!fir.array<10x10xf32>>) {name = "c", structured = false}
 
   !$acc update host(a) host(b) device(c)
-! CHECK: %[[DEVPTR_A:.*]] = acc.getdeviceptr varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = #acc<data_clause acc_update_host>, name = "a", structured = false}
-! CHECK: %[[DEVPTR_B:.*]] = acc.getdeviceptr varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = #acc<data_clause acc_update_host>, name = "b", structured = false}
-! CHECK: %[[DEVPTR_C:.*]] = acc.update_device varPtr(%[[DECLC]]#0 : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {name = "c", structured = false}
+! CHECK: %[[DEVPTR_A:.*]] = acc.getdeviceptr varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = #acc<data_clause acc_update_host>, name = "a", structured = false}
+! CHECK: %[[DEVPTR_B:.*]] = acc.getdeviceptr varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<10x10xf32>>) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = #acc<data_clause acc_update_host>, name = "b", structured = false}
+! CHECK: %[[DEVPTR_C:.*]] = acc.update_device varPtr(%[[DECLC]]#0 : !fir.ref<!fir.array<10x10xf32>>) -> !fir.ref<!fir.array<10x10xf32>> {name = "c", structured = false}
 ! CHECK: acc.update dataOperands(%[[DEVPTR_C]], %[[DEVPTR_A]], %[[DEVPTR_B]] : !fir.ref<!fir.array<10x10xf32>>, !fir.ref<!fir.array<10x10xf32>>, !fir.ref<!fir.array<10x10xf32>>){{$}}
-! CHECK: acc.update_host accPtr(%[[DEVPTR_A]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) to varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) {name = "a", structured = false}
-! CHECK: acc.update_host accPtr(%[[DEVPTR_B]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) to varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<10x10xf32>>) {name = "b", structured = false}
+! CHECK: acc.update_host accPtr(%[[DEVPTR_A]] : !fir.ref<!fir.array<10x10xf32>>) to varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) {name = "a", structured = false}
+! CHECK: acc.update_host accPtr(%[[DEVPTR_B]] : !fir.ref<!fir.array<10x10xf32>>) to varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<10x10xf32>>) {name = "b", structured = false}
 
   !$acc update host(a) async
-! CHECK: %[[DEVPTR_A:.*]] = acc.getdeviceptr varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {asyncOnly = [#acc.device_type<none>], dataClause = #acc<data_clause acc_update_host>, name = "a", structured = false}
+! CHECK: %[[DEVPTR_A:.*]] = acc.getdeviceptr varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) -> !fir.ref<!fir.array<10x10xf32>> {asyncOnly = [#acc.device_type<none>], dataClause = #acc<data_clause acc_update_host>, name = "a", structured = false}
 ! CHECK: acc.update async dataOperands(%[[DEVPTR_A]] : !fir.ref<!fir.array<10x10xf32>>)
-! CHECK: acc.update_host accPtr(%[[DEVPTR_A]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) to varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) {asyncOnly = [#acc.device_type<none>], name = "a", structured = false}
+! CHECK: acc.update_host accPtr(%[[DEVPTR_A]] : !fir.ref<!fir.array<10x10xf32>>) to varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) {asyncOnly = [#acc.device_type<none>], name = "a", structured = false}
 
   !$acc update host(a) wait
-! CHECK: %[[DEVPTR_A:.*]] = acc.getdeviceptr varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = #acc<data_clause acc_update_host>, name = "a", structured = false}
+! CHECK: %[[DEVPTR_A:.*]] = acc.getdeviceptr varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = #acc<data_clause acc_update_host>, name = "a", structured = false}
 ! CHECK: acc.update wait dataOperands(%[[DEVPTR_A]] : !fir.ref<!fir.array<10x10xf32>>)
-! CHECK: acc.update_host accPtr(%[[DEVPTR_A]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) to varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) {name = "a", structured = false}
+! CHECK: acc.update_host accPtr(%[[DEVPTR_A]] : !fir.ref<!fir.array<10x10xf32>>) to varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) {name = "a", structured = false}
 
   !$acc update host(a) async wait
-! CHECK: %[[DEVPTR_A:.*]] = acc.getdeviceptr varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {asyncOnly = [#acc.device_type<none>], dataClause = #acc<data_clause acc_update_host>, name = "a", structured = false}
+! CHECK: %[[DEVPTR_A:.*]] = acc.getdeviceptr varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) -> !fir.ref<!fir.array<10x10xf32>> {asyncOnly = [#acc.device_type<none>], dataClause = #acc<data_clause acc_update_host>, name = "a", structured = false}
 ! CHECK: acc.update async wait dataOperands(%[[DEVPTR_A]] : !fir.ref<!fir.array<10x10xf32>>)
-! CHECK: acc.update_host accPtr(%[[DEVPTR_A]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) to varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) {asyncOnly = [#acc.device_type<none>], name = "a", structured = false}
+! CHECK: acc.update_host accPtr(%[[DEVPTR_A]] : !fir.ref<!fir.array<10x10xf32>>) to varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) {asyncOnly = [#acc.device_type<none>], name = "a", structured = false}
 
   !$acc update host(a) async(1)
 ! CHECK: [[ASYNC1:%.*]] = arith.constant 1 : i32
-! CHECK: %[[DEVPTR_A:.*]] = acc.getdeviceptr varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) async([[ASYNC1]] : i32) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = #acc<data_clause acc_update_host>, name = "a", structured = false}
+! CHECK: %[[DEVPTR_A:.*]] = acc.getdeviceptr varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) async([[ASYNC1]] : i32) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = #acc<data_clause acc_update_host>, name = "a", structured = false}
 ! CHECK: acc.update async([[ASYNC1]] : i32) dataOperands(%[[DEVPTR_A]] : !fir.ref<!fir.array<10x10xf32>>)
-! CHECK: acc.update_host accPtr(%[[DEVPTR_A]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) async([[ASYNC1]] : i32) to varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) {name = "a", structured = false}
+! CHECK: acc.update_host accPtr(%[[DEVPTR_A]] : !fir.ref<!fir.array<10x10xf32>>) async([[ASYNC1]] : i32) to varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) {name = "a", structured = false}
 
   !$acc update host(a) async(async)
 ! CHECK: [[ASYNC2:%.*]] = fir.load %{{.*}} : !fir.ref<i32>
-! CHECK: %[[DEVPTR_A:.*]] = acc.getdeviceptr varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) async([[ASYNC2]] : i32) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = #acc<data_clause acc_update_host>, name = "a", structured = false}
+! CHECK: %[[DEVPTR_A:.*]] = acc.getdeviceptr varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) async([[ASYNC2]] : i32) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = #acc<data_clause acc_update_host>, name = "a", structured = false}
 ! CHECK: acc.update async([[ASYNC2]] : i32) dataOperands(%[[DEVPTR_A]] : !fir.ref<!fir.array<10x10xf32>>)
-! CHECK: acc.update_host accPtr(%[[DEVPTR_A]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) async([[ASYNC2]] : i32) to varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) {name = "a", structured = false}
+! CHECK: acc.update_host accPtr(%[[DEVPTR_A]] : !fir.ref<!fir.array<10x10xf32>>) async([[ASYNC2]] : i32) to varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) {name = "a", structured = false}
 
   !$acc update host(a) wait(1)
 ! CHECK: [[WAIT1:%.*]] = arith.constant 1 : i32
-! CHECK: %[[DEVPTR_A:.*]] = acc.getdeviceptr varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = #acc<data_clause acc_update_host>, name = "a", structured = false}
+! CHECK: %[[DEVPTR_A:.*]] = acc.getdeviceptr varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = #acc<data_clause acc_update_host>, name = "a", structured = false}
 ! CHECK: acc.update wait({[[WAIT1]] : i32}) dataOperands(%[[DEVPTR_A]] : !fir.ref<!fir.array<10x10xf32>>)
-! CHECK: acc.update_host accPtr(%[[DEVPTR_A]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) to varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) {name = "a", structured = false}
+! CHECK: acc.update_host accPtr(%[[DEVPTR_A]] : !fir.ref<!fir.array<10x10xf32>>) to varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) {name = "a", structured = false}
 
   !$acc update host(a) wait(queues: 1, 2)
 ! CHECK: [[WAIT2:%.*]] = arith.constant 1 : i32
 ! CHECK: [[WAIT3:%.*]] = arith.constant 2 : i32
-! CHECK: %[[DEVPTR_A:.*]] = acc.getdeviceptr varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = #acc<data_clause acc_update_host>, name = "a", structured = false}
+! CHECK: %[[DEVPTR_A:.*]] = acc.getdeviceptr varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = #acc<data_clause acc_update_host>, name = "a", structured = false}
 ! CHECK: acc.update wait({[[WAIT2]] : i32, [[WAIT3]] : i32}) dataOperands(%[[DEVPTR_A]] : !fir.ref<!fir.array<10x10xf32>>)
-! CHECK: acc.update_host accPtr(%[[DEVPTR_A]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) to varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) {name = "a", structured = false}
+! CHECK: acc.update_host accPtr(%[[DEVPTR_A]] : !fir.ref<!fir.array<10x10xf32>>) to varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) {name = "a", structured = false}
 
   !$acc update host(a) wait(devnum: 1: queues: 1, 2)
-! CHECK: %[[DEVPTR_A:.*]] = acc.getdeviceptr varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = #acc<data_clause acc_update_host>, name = "a", structured = false}
+! CHECK: %[[DEVPTR_A:.*]] = acc.getdeviceptr varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = #acc<data_clause acc_update_host>, name = "a", structured = false}
 ! CHECK: acc.update wait({devnum: %c1{{.*}} : i32, %c1{{.*}} : i32, %c2{{.*}} : i32}) dataOperands(%[[DEVPTR_A]] : !fir.ref<!fir.array<10x10xf32>>)
-! CHECK: acc.update_host accPtr(%[[DEVPTR_A]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) to varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) {name = "a", structured = false}
+! CHECK: acc.update_host accPtr(%[[DEVPTR_A]] : !fir.ref<!fir.array<10x10xf32>>) to varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) {name = "a", structured = false}
 
   !$acc update host(a) device_type(host, nvidia) async
-! CHECK: %[[DEVPTR_A:.*]] = acc.getdeviceptr varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {asyncOnly = [#acc.device_type<host>, #acc.device_type<nvidia>], dataClause = #acc<data_clause acc_update_host>, name = "a", structured = false}
+! CHECK: %[[DEVPTR_A:.*]] = acc.getdeviceptr varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) -> !fir.ref<!fir.array<10x10xf32>> {asyncOnly = [#acc.device_type<host>, #acc.device_type<nvidia>], dataClause = #acc<data_clause acc_update_host>, name = "a", structured = false}
 ! CHECK: acc.update async([#acc.device_type<host>, #acc.device_type<nvidia>]) dataOperands(%[[DEVPTR_A]] : !fir.ref<!fir.array<10x10xf32>>)
-! CHECK: acc.update_host accPtr(%[[DEVPTR_A]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) to varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) {asyncOnly = [#acc.device_type<host>, #acc.device_type<nvidia>], name = "a", structured = false}
+! CHECK: acc.update_host accPtr(%[[DEVPTR_A]] : !fir.ref<!fir.array<10x10xf32>>) to varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) {asyncOnly = [#acc.device_type<host>, #acc.device_type<nvidia>], name = "a", structured = false}
 
 end subroutine acc_update
diff --git a/flang/test/Lower/OpenMP/DelayedPrivatization/distribute-standalone-private.f90 b/flang/test/Lower/OpenMP/DelayedPrivatization/distribute-standalone-private.f90
index 9c2ff8b528485..8098cd53e9d2e 100644
--- a/flang/test/Lower/OpenMP/DelayedPrivatization/distribute-standalone-private.f90
+++ b/flang/test/Lower/OpenMP/DelayedPrivatization/distribute-standalone-private.f90
@@ -16,8 +16,8 @@ subroutine standalone_distribute
     !$omp end teams
 end subroutine standalone_distribute
 
-! CHECK: omp.private {type = private} @[[I_PRIVATIZER_SYM:.*]] : !fir.ref<i32>
-! CHECK: omp.private {type = private} @[[VAR_PRIVATIZER_SYM:.*]] : !fir.ref<i32>
+! CHECK: omp.private {type = private} @[[I_PRIVATIZER_SYM:.*]] : i32
+! CHECK: omp.private {type = private} @[[VAR_PRIVATIZER_SYM:.*]] : i32
 
 
 ! CHECK-LABEL: func.func @_QPstandalone_distribute() {
diff --git a/flang/test/Lower/OpenMP/DelayedPrivatization/equivalence.f90 b/flang/test/Lower/OpenMP/DelayedPrivatization/equivalence.f90
index 2307c09513795..721bfff012f14 100644
--- a/flang/test/Lower/OpenMP/DelayedPrivatization/equivalence.f90
+++ b/flang/test/Lower/OpenMP/DelayedPrivatization/equivalence.f90
@@ -13,13 +13,15 @@ subroutine private_common
   !$omp end parallel
 end subroutine
 
-! CHECK:  omp.private {type = firstprivate} @[[X_PRIVATIZER:.*]] : ![[X_TYPE:fir.ptr<f32>]] alloc {
-! CHECK:  ^bb0(%{{.*}}: ![[X_TYPE]]):
-! CHECK:    %[[PRIV_ALLOC:.*]] = fir.alloca f32 {bindc_name = "x", {{.*}}}
-! CHECK:    %[[PRIV_DECL:.*]]:2 = hlfir.declare %[[PRIV_ALLOC]] {{{.*}}} : (![[PRIV_TYPE:fir.ref<f32>]]) -> ({{.*}})
-! CHECK:    %[[PRIV_CONV:.*]] = fir.convert %[[PRIV_DECL]]#0 : (![[PRIV_TYPE]]) -> ![[X_TYPE]]
-! CHECK:    omp.yield(%[[PRIV_CONV]] : ![[X_TYPE]])
-! CHECK:  } copy {
+! TODO: the copy region for pointers is incorrect. OpenMP 5.2 says
+!
+! > If the original list item has the POINTER attribute, the new list items
+! > receive the same association status as the original list item
+!
+! Currently the original pointer is unconditionally loaded, which is undefined
+! behavior if that pointer is not associated.
+
+! CHECK:  omp.private {type = firstprivate} @[[X_PRIVATIZER:.*]] : ![[X_TYPE:fir.ptr<f32>]] copy {
 ! CHECK:  ^bb0(%[[ORIG_PTR:.*]]: ![[X_TYPE]], %[[PRIV_REF:.*]]: ![[X_TYPE]]):
 ! CHECK:    %[[ORIG_VAL:.*]] = fir.load %[[ORIG_PTR]] : !fir.ptr<f32>
 ! CHECK:    hlfir.assign %[[ORIG_VAL]] to %[[PRIV_REF]] : f32, ![[X_TYPE]]
diff --git a/flang/test/Lower/OpenMP/DelayedPrivatization/target-private-allocatable.f90 b/flang/test/Lower/OpenMP/DelayedPrivatization/target-private-allocatable.f90
index e11525c569ffb..3e2b3e59018b8 100644
--- a/flang/test/Lower/OpenMP/DelayedPrivatization/target-private-allocatable.f90
+++ b/flang/test/Lower/OpenMP/DelayedPrivatization/target-private-allocatable.f90
@@ -16,28 +16,25 @@ end subroutine target_allocatable
 
 ! CHECK-LABEL: omp.private {type = private}
 ! CHECK-SAME:    @[[VAR_PRIVATIZER_SYM:.*]] :
-! CHECK-SAME:      [[TYPE:!fir.ref<!fir.box<!fir.heap<i32>>>]] alloc {
-! CHECK:  ^bb0(%[[PRIV_ARG:.*]]: [[TYPE]]):
-! CHECK:    %[[PRIV_ALLOC:.*]] = fir.alloca [[DESC_TYPE:!fir.box<!fir.heap<i32>>]] {bindc_name = "alloc_var", {{.*}}}
+! CHECK-SAME:      [[DESC_TYPE:!fir.box<!fir.heap<i32>>]] init {
+! CHECK:  ^bb0(%[[PRIV_ARG:.*]]: [[TYPE:!fir.ref<!fir.box<!fir.heap<i32>>>]], %[[PRIV_ALLOC:.*]]: [[TYPE]]):
 
 ! CHECK-NEXT:   %[[PRIV_ARG_VAL:.*]] = fir.load %[[PRIV_ARG]] : [[TYPE]]
 ! CHECK-NEXT:   %[[PRIV_ARG_BOX:.*]] = fir.box_addr %[[PRIV_ARG_VAL]] : ([[DESC_TYPE]]) -> !fir.heap<i32>
 ! CHECK-NEXT:   %[[PRIV_ARG_ADDR:.*]] = fir.convert %[[PRIV_ARG_BOX]] : (!fir.heap<i32>) -> i64
 ! CHECK-NEXT:   %[[C0:.*]] = arith.constant 0 : i64
-! CHECK-NEXT:   %[[ALLOC_COND:.*]] = arith.cmpi ne, %[[PRIV_ARG_ADDR]], %[[C0]] : i64
+! CHECK-NEXT:   %[[ALLOC_COND:.*]] = arith.cmpi eq, %[[PRIV_ARG_ADDR]], %[[C0]] : i64
 
 ! CHECK-NEXT:   fir.if %[[ALLOC_COND]] {
-! CHECK:          %[[PRIV_ALLOCMEM:.*]] = fir.allocmem i32 {fir.must_be_heap = true, {{.*}}}
+! CHECK-NEXT:     %[[ZERO_BOX:.*]] = fir.embox %[[PRIV_ARG_BOX]] : (!fir.heap<i32>) -> [[DESC_TYPE]]
+! CHECK-NEXT:     fir.store %[[ZERO_BOX]] to %[[PRIV_ALLOC]] : [[TYPE]]
+! CHECK-NEXT:   } else {
+! CHECK-NEXT:     %[[PRIV_ALLOCMEM:.*]] = fir.allocmem i32
 ! CHECK-NEXT:     %[[PRIV_ALLOCMEM_BOX:.*]] = fir.embox %[[PRIV_ALLOCMEM]] : (!fir.heap<i32>) -> [[DESC_TYPE]]
 ! CHECK-NEXT:     fir.store %[[PRIV_ALLOCMEM_BOX]] to %[[PRIV_ALLOC]] : [[TYPE]]
-! CHECK-NEXT:   } else {
-! CHECK-NEXT:     %[[ZERO_BITS:.*]] = fir.zero_bits !fir.heap<i32>
-! CHECK-NEXT:     %[[ZERO_BOX:.*]] = fir.embox %[[ZERO_BITS]] : (!fir.heap<i32>) -> [[DESC_TYPE]]
-! CHECK-NEXT:     fir.store %[[ZERO_BOX]] to %[[PRIV_ALLOC]] : [[TYPE]]
 ! CHECK-NEXT:   }
 
-! CHECK-NEXT:   %[[PRIV_DECL:.*]]:2 = hlfir.declare %[[PRIV_ALLOC]]
-! CHECK-NEXT:   omp.yield(%[[PRIV_DECL]]#0 : [[TYPE]])
+! CHECK-NEXT:   omp.yield(%[[PRIV_ALLOC]] : [[TYPE]])
 
 ! CHECK-NEXT: } dealloc {
 ! CHECK-NEXT: ^bb0(%[[PRIV_ARG:.*]]: [[TYPE]]):
@@ -49,12 +46,7 @@ end subroutine target_allocatable
 ! CHECK-NEXT:   %[[PRIV_NULL_COND:.*]] = arith.cmpi ne, %[[PRIV_ADDR_I64]], %[[C0]] : i64
 
 ! CHECK-NEXT:   fir.if %[[PRIV_NULL_COND]] {
-! CHECK:          %[[PRIV_VAL_2:.*]] = fir.load %[[PRIV_ARG]]
-! CHECK-NEXT:     %[[PRIV_ADDR_2:.*]] = fir.box_addr %[[PRIV_VAL_2]]
-! CHECK-NEXT:     fir.freemem %[[PRIV_ADDR_2]]
-! CHECK-NEXT:     %[[ZEROS:.*]] = fir.zero_bits
-! CHECK-NEXT:     %[[ZEROS_BOX:.*]]  = fir.embox %[[ZEROS]]
-! CHECK-NEXT:     fir.store %[[ZEROS_BOX]] to %[[PRIV_ARG]]
+! CHECK-NEXT:     fir.freemem %[[PRIV_ADDR]]
 ! CHECK-NEXT:   }
 
 ! CHECK-NEXT:   omp.yield
diff --git a/flang/test/Lower/OpenMP/DelayedPrivatization/target-private-multiple-variables.f90 b/flang/test/Lower/OpenMP/DelayedPrivatization/target-private-multiple-variables.f90
index f3f9bbe4a76a2..5d31de10d74f8 100644
--- a/flang/test/Lower/OpenMP/DelayedPrivatization/target-private-multiple-variables.f90
+++ b/flang/test/Lower/OpenMP/DelayedPrivatization/target-private-multiple-variables.f90
@@ -38,95 +38,81 @@ end subroutine target_allocatable
 !
 ! CHECK:      omp.private {type = private}
 ! CHECK-SAME:   @[[CHAR_PRIVATIZER_SYM:[^[:space:]]+char_var[^[:space:]]+]]
-! CHECK-SAME:   : [[CHAR_TYPE:!fir.boxchar<1>]] alloc {
+! CHECK-SAME:   : [[CHAR_TYPE:!fir.boxchar<1>]] init {
 !
-! CHECK-NEXT: ^bb0(%[[PRIV_ARG:.*]]: [[CHAR_TYPE]]):
+! CHECK-NEXT: ^bb0(%[[PRIV_ARG:.*]]: [[CHAR_TYPE]], %[[UNUSED:.*]]: [[CHAR_TYPE]]):
 ! CHECK-NEXT:   %[[UNBOX:.*]]:2 = fir.unboxchar %[[PRIV_ARG]]
-! CHECK:        %[[PRIV_ALLOC:.*]] = fir.alloca !fir.char<1,?>(%[[UNBOX]]#1 : index)
-! CHECK-NEXT:   %[[PRIV_DECL:.*]]:2 = hlfir.declare %[[PRIV_ALLOC]] typeparams %[[UNBOX]]#1
-! CHECK-NEXT:   omp.yield(%[[PRIV_DECL]]#0 : [[CHAR_TYPE]])
-! CHECK-NEXT: }
+! CHECK:        %[[PRIV_ALLOC:.*]] = fir.allocmem !fir.char<1,?>(%[[UNBOX]]#1 : index)
+! CHECK:        %[[BOXCHAR:.*]] = fir.emboxchar %[[PRIV_ALLOC]], %[[UNBOX]]#1
+! CHECK-NEXT:   omp.yield(%[[BOXCHAR]] : [[CHAR_TYPE]])
+! CHECK-NEXT: } dealloc {
 
 ! Test the privatizer for `complex`
 !
 ! CHECK:      omp.private {type = private}
 ! CHECK-SAME:   @[[COMP_PRIVATIZER_SYM:[^[:space:]]+comp_var[^[:space:]]+]]
-! CHECK-SAME:   : [[COMP_TYPE:!fir.ref<complex<f32>>]] alloc {
-!
-! CHECK-NEXT: ^bb0(%[[PRIV_ARG:.*]]: [[COMP_TYPE]]):
-! CHECK-NEXT:   %[[PRIV_ALLOC:.*]] = fir.alloca complex<f32>
-! CHECK-NEXT:   %[[PRIV_DECL:.*]]:2 = hlfir.declare %[[PRIV_ALLOC]]
-! CHECK-NEXT:   omp.yield(%[[PRIV_DECL]]#0 : [[COMP_TYPE]])
-! CHECK-NEXT: }
+! CHECK-SAME:   : [[COMP_TYPE:complex<f32>]]{{$}}
 
 ! Test the privatizer for `real(:)`
 !
 ! CHECK:      omp.private {type = private}
 ! CHECK-SAME:   @[[ARR_PRIVATIZER_SYM:[^[:space:]]+real_arr[^[:space:]]+]]
-! CHECK-SAME:   : [[ARR_TYPE:!fir.box<!fir.array<\?xf32>>]] alloc {
-!
-! CHECK-NEXT: ^bb0(%[[PRIV_ARG:.*]]: [[ARR_TYPE]]):
-! CHECK:        %[[C0:.*]] = arith.constant 0 : index
-! CHECK-NEXT:   %[[DIMS:.*]]:3 = fir.box_dims %[[PRIV_ARG]], %[[C0]] : ([[ARR_TYPE]], index)
-! CHECK:        %[[PRIV_ALLOCA:.*]] = fir.alloca !fir.array<{{\?}}xf32>
-! CHECK-NEXT:   %[[SHAPE_SHIFT:.*]] = fir.shape_shift %[[DIMS]]#0, %[[DIMS]]#1
-! CHECK-NEXT:   %[[PRIV_DECL:.*]]:2 = hlfir.declare %[[PRIV_ALLOCA]](%[[SHAPE_SHIFT]])
-! CHECK-NEXT:  omp.yield(%[[PRIV_DECL]]#0 : [[ARR_TYPE]])
+! CHECK-SAME:   : [[ARR_TYPE:!fir.box<!fir.array<\?xf32>>]] init {
+!
+! CHECK-NEXT: ^bb0(%[[PRIV_ARG:.*]]: !fir.ref<[[ARR_TYPE]]>, %[[PRIV_ALLOC:.*]]: !fir.ref<[[ARR_TYPE]]>):
+! CHECK-NEXT:   %[[MOLD:.*]] = fir.load %[[PRIV_ARG]]
+! CHECK-NEXT:   %[[C0:.*]] = arith.constant 0 : index
+! CHECK-NEXT:   %[[BOX_DIMS:.*]]:3 = fir.box_dims %[[MOLD]], %[[C0]]
+! CHECK-NEXT:   %[[SHAPE:.*]] = fir.shape %[[BOX_DIMS]]#1
+! CHECK-NEXT:   %[[DATA_ALLOC:.*]] = fir.allocmem !fir.array<?xf32>, %[[BOX_DIMS]]#1
+! CHECK-NEXT:   %[[TRUE:.*]] = arith.constant true
+! CHECK-NEXT:   %[[DECL:.*]]:2 = hlfir.declare %[[DATA_ALLOC:.*]](%[[SHAPE]])
+! CHECK-NEXT:   %[[C0_2:.*]] = arith.constant 0 : index
+! CHECK-NEXT:   %[[BOX_DIMS_2:.*]]:3 = fir.box_dims %[[MOLD]], %[[C0_2]]
+! CHECK-NEXT:   %[[SHAPE_SHIFT:.*]] = fir.shape_shift %[[BOX_DIMS_2]]#0, %[[BOX_DIMS_2]]#1
+! CHECK-NEXT:   %[[BOX:.*]] = fir.rebox %[[DECL]]#0(%[[SHAPE_SHIFT]])
+! CHECK-NEXT:   fir.store %[[BOX]] to %[[PRIV_ALLOC]]
+! CHECK-NEXT:   omp.yield(%[[PRIV_ALLOC]] : !fir.ref<[[ARR_TYPE]]>)
 ! CHECK-NEXT: }
 
 ! Test the privatizer for `real(:)`'s lower bound
 !
 ! CHECK:      omp.private {type = private}
 ! CHECK-SAME:   @[[LB_PRIVATIZER_SYM:[^[:space:]]+lb[^[:space:]]+]]
-! CHECK-SAME:   : [[LB_TYPE:!fir.ref<i64>]] alloc {
-
-! CHECK-NEXT: ^bb0(%[[PRIV_ARG:.*]]: [[LB_TYPE]]):
-! CHECK-NEXT:   %[[PRIV_ALLOCA:.*]] = fir.alloca i64
-! CHECK-NEXT:   %[[PRIV_DECL:.*]]:2 = hlfir.declare %[[PRIV_ALLOCA]]
-! CHECK-NEXT:  omp.yield(%[[PRIV_DECL]]#0 : [[LB_TYPE]])
-! CHECK-NEXT: }
+! CHECK-SAME:   : [[LB_TYPE:i64]]{{$}}
 
 ! Test the privatizer for `real`
 !
 ! CHECK:      omp.private {type = private}
 ! CHECK-SAME:   @[[REAL_PRIVATIZER_SYM:[^[:space:]]+real_var[^[:space:]]+]]
-! CHECK-SAME:   : [[REAL_TYPE:!fir.ref<f32>]] alloc {
-
-! CHECK-NEXT: ^bb0(%[[PRIV_ARG:.*]]: [[REAL_TYPE]]):
-! CHECK-NEXT:   %[[PRIV_ALLOCA:.*]] = fir.alloca f32
-! CHECK-NEXT:   %[[PRIV_DECL:.*]]:2 = hlfir.declare %[[PRIV_ALLOCA]]
-! CHECK-NEXT:  omp.yield(%[[PRIV_DECL]]#0 : [[REAL_TYPE]])
-! CHECK-NEXT: }
+! CHECK-SAME:   : [[REAL_TYPE:f32]]{{$}}
 
 ! Test the privatizer for `allocatable`
 !
 ! CHECK:      omp.private {type = private}
 ! CHECK-SAME:   @[[ALLOC_PRIVATIZER_SYM:[^[:space:]]+alloc_var[^[:space:]]+]]
-! CHECK-SAME:   : [[ALLOC_TYPE:!fir.ref<!fir.box<!fir.heap<i32>>>]] alloc {
+! CHECK-SAME:   : [[ALLOC_TYPE:!fir.box<!fir.heap<i32>>]] init {
 !
-! CHECK-NEXT: ^bb0(%[[PRIV_ARG:.*]]: [[ALLOC_TYPE]]):
-! CHECK:        %[[PRIV_ALLOC:.*]] = fir.alloca !fir.box<!fir.heap<i32>>
+! CHECK-NEXT: ^bb0(%[[PRIV_ARG:.*]]: !fir.ref<[[ALLOC_TYPE]]>, %[[PRIV_ALLOC:.*]]: !fir.ref<[[ALLOC_TYPE]]>):
 ! CHECK-NEXT:   %[[PRIV_ARG_VAL:.*]] = fir.load %[[PRIV_ARG]] : !fir.ref<!fir.box<!fir.heap<i32>>>
 ! CHECK-NEXT:   %[[PRIV_ARG_BOX:.*]] = fir.box_addr %[[PRIV_ARG_VAL]] : (!fir.box<!fir.heap<i32>>) -> !fir.heap<i32>
 ! CHECK-NEXT:   %[[PRIV_ARG_ADDR:.*]] = fir.convert %[[PRIV_ARG_BOX]] : (!fir.heap<i32>) -> i64
 ! CHECK-NEXT:   %[[C0:.*]] = arith.constant 0 : i64
-! CHECK-NEXT:   %[[ALLOC_COND:.*]] = arith.cmpi ne, %[[PRIV_ARG_ADDR]], %[[C0]] : i64
+! CHECK-NEXT:   %[[ALLOC_COND:.*]] = arith.cmpi eq, %[[PRIV_ARG_ADDR]], %[[C0]] : i64
 !
 ! CHECK-NEXT:   fir.if %[[ALLOC_COND]] {
-! CHECK:          %[[PRIV_ALLOCMEM:.*]] = fir.allocmem i32 {fir.must_be_heap = true, {{.*}}}
+! CHECK-NEXT:     %[[ZERO_BOX:.*]] = fir.embox %[[PRIV_ARG_BOX]] : (!fir.heap<i32>) -> !fir.box<!fir.heap<i32>>
+! CHECK-NEXT:     fir.store %[[ZERO_BOX]] to %[[PRIV_ALLOC]] : !fir.ref<!fir.box<!fir.heap<i32>>>
+! CHECK-NEXT:   } else {
+! CHECK:          %[[PRIV_ALLOCMEM:.*]] = fir.allocmem i32
 ! CHECK-NEXT:     %[[PRIV_ALLOCMEM_BOX:.*]] = fir.embox %[[PRIV_ALLOCMEM]] : (!fir.heap<i32>) -> !fir.box<!fir.heap<i32>>
 ! CHECK-NEXT:     fir.store %[[PRIV_ALLOCMEM_BOX]] to %[[PRIV_ALLOC]] : !fir.ref<!fir.box<!fir.heap<i32>>>
-! CHECK-NEXT:   } else {
-! CHECK-NEXT:     %[[ZERO_BITS:.*]] = fir.zero_bits !fir.heap<i32>
-! CHECK-NEXT:     %[[ZERO_BOX:.*]] = fir.embox %[[ZERO_BITS]] : (!fir.heap<i32>) -> !fir.box<!fir.heap<i32>>
-! CHECK-NEXT:     fir.store %[[ZERO_BOX]] to %[[PRIV_ALLOC]] : !fir.ref<!fir.box<!fir.heap<i32>>>
 ! CHECK-NEXT:   }
 !
-! CHECK-NEXT:   %[[PRIV_DECL:.*]]:2 = hlfir.declare %[[PRIV_ALLOC]]
-! CHECK-NEXT:   omp.yield(%[[PRIV_DECL]]#0 : [[ALLOC_TYPE]])
+! CHECK-NEXT:   omp.yield(%[[PRIV_ALLOC]] : !fir.ref<[[ALLOC_TYPE]]>)
 !
 ! CHECK-NEXT: } dealloc {
-! CHECK-NEXT: ^bb0(%[[PRIV_ARG:.*]]: [[ALLOC_TYPE]]):
+! CHECK-NEXT: ^bb0(%[[PRIV_ARG:.*]]: !fir.ref<[[ALLOC_TYPE]]>):
 !
 ! CHECK-NEXT:   %[[PRIV_VAL:.*]] = fir.load %[[PRIV_ARG]]
 ! CHECK-NEXT:   %[[PRIV_ADDR:.*]] = fir.box_addr %[[PRIV_VAL]]
@@ -135,12 +121,7 @@ end subroutine target_allocatable
 ! CHECK-NEXT:   %[[PRIV_NULL_COND:.*]] = arith.cmpi ne, %[[PRIV_ADDR_I64]], %[[C0]] : i64
 !
 ! CHECK-NEXT:   fir.if %[[PRIV_NULL_COND]] {
-! CHECK:          %[[PRIV_VAL_2:.*]] = fir.load %[[PRIV_ARG]]
-! CHECK-NEXT:     %[[PRIV_ADDR_2:.*]] = fir.box_addr %[[PRIV_VAL_2]]
-! CHECK-NEXT:     fir.freemem %[[PRIV_ADDR_2]]
-! CHECK-NEXT:     %[[ZEROS:.*]] = fir.zero_bits
-! CHECK-NEXT:     %[[ZEROS_BOX:.*]]  = fir.embox %[[ZEROS]]
-! CHECK-NEXT:     fir.store %[[ZEROS_BOX]] to %[[PRIV_ARG]]
+! CHECK-NEXT:     fir.freemem %[[PRIV_ADDR]]
 ! CHECK-NEXT:   }
 !
 ! CHECK-NEXT:   omp.yield
@@ -157,9 +138,9 @@ end subroutine target_allocatable
 ! CHECK:        %[[CHAR_VAR_DECL:.*]]:2 = hlfir.declare %[[CHAR_VAR_ALLOC]] typeparams
 ! CHECK:        %[[REAL_ARR_ALLOC:.*]] = fir.alloca !fir.array<?xf32>, {{.*}} {bindc_name = "real_arr", {{.*}}}
 ! CHECK:        %[[REAL_ARR_DECL:.*]]:2 = hlfir.declare %[[REAL_ARR_ALLOC]]({{.*}})
+! CHECK:        fir.store %[[REAL_ARR_DECL]]#0 to %[[REAL_ARR_DESC_ALLOCA]] : !fir.ref<!fir.box<!fir.array<?xf32>>>
 ! CHECK:        %[[MAPPED_MI0:.*]] = omp.map.info var_ptr(%[[MAPPED_DECL]]#1 : !fir.ref<i32>, i32) {{.*}}
 ! CHECK:        %[[ALLOC_VAR_MAP:.*]] = omp.map.info var_ptr(%[[ALLOC_VAR_DECL]]#0 : !fir.ref<!fir.box<!fir.heap<i32>>>, !fir.box<!fir.heap<i32>>)
-! CHECK:        fir.store %[[REAL_ARR_DECL]]#0 to %[[REAL_ARR_DESC_ALLOCA]] : !fir.ref<!fir.box<!fir.array<?xf32>>>
 ! CHECK:        %[[REAL_ARR_DESC_MAP:.*]] = omp.map.info var_ptr(%[[REAL_ARR_DESC_ALLOCA]] : !fir.ref<!fir.box<!fir.array<?xf32>>>, !fir.box<!fir.array<?xf32>>)
 ! CHECK:        fir.store %[[CHAR_VAR_DECL]]#0 to %[[CHAR_VAR_DESC_ALLOCA]] : !fir.ref<!fir.boxchar<1>>
 ! CHECK:        %[[CHAR_VAR_DESC_MAP:.*]] = omp.map.info var_ptr(%[[CHAR_VAR_DESC_ALLOCA]] : !fir.ref<!fir.boxchar<1>>, !fir.boxchar<1>)
@@ -174,16 +155,15 @@ end subroutine target_allocatable
 ! CHECK-SAME:       @[[ALLOC_PRIVATIZER_SYM]] %{{[^[:space:]]+}}#0 -> %[[ALLOC_ARG:[^,]+]] [map_idx=1],
 ! CHECK-SAME:       @[[REAL_PRIVATIZER_SYM]] %{{[^[:space:]]+}}#0 -> %[[REAL_ARG:[^,]+]],
 ! CHECK-SAME:       @[[LB_PRIVATIZER_SYM]] %{{[^[:space:]]+}}#0 -> %[[LB_ARG:[^,]+]],
-! CHECK-SAME:       @[[ARR_PRIVATIZER_SYM]] %{{[^[:space:]]+}}#0 -> %[[ARR_ARG:[^,]+]] [map_idx=2],
+! CHECK-SAME:       @[[ARR_PRIVATIZER_SYM]] %{{[^[:space:]]+}} -> %[[ARR_ARG:[^,]+]] [map_idx=2],
 ! CHECK-SAME:       @[[COMP_PRIVATIZER_SYM]] %{{[^[:space:]]+}}#0 -> %[[COMP_ARG:[^,]+]],
 ! CHECK-SAME:       @[[CHAR_PRIVATIZER_SYM]] %{{[^[:space:]]+}}#0 -> %[[CHAR_ARG:[^,]+]] [map_idx=3] :
-! CHECK-SAME:       !fir.ref<!fir.box<!fir.heap<i32>>>, !fir.ref<f32>, !fir.ref<i64>, !fir.box<!fir.array<?xf32>>, !fir.ref<complex<f32>>, !fir.boxchar<1>) {
+! CHECK-SAME:       !fir.ref<!fir.box<!fir.heap<i32>>>, !fir.ref<f32>, !fir.ref<i64>, !fir.ref<!fir.box<!fir.array<?xf32>>>, !fir.ref<complex<f32>>, !fir.boxchar<1>) {
 ! CHECK-NOT:      fir.alloca
 ! CHECK:          hlfir.declare %[[ALLOC_ARG]]
 ! CHECK:          hlfir.declare %[[REAL_ARG]]
 ! CHECK:          hlfir.declare %[[LB_ARG]]
-! CHECK:          %[[ARR_ARG_ADDR:.*]] = fir.box_addr %[[ARR_ARG]]
-! CHECK:          hlfir.declare %[[ARR_ARG_ADDR]]
+! CHECK:          hlfir.declare %[[ARR_ARG]]
 ! CHECK:          hlfir.declare %[[COMP_ARG]]
 ! CHECK:          %[[CHAR_ARG_UNBOX:.*]]:2 = fir.unboxchar %[[CHAR_ARG]]
 ! CHECK:          hlfir.declare %[[CHAR_ARG_UNBOX]]
diff --git a/flang/test/Lower/OpenMP/DelayedPrivatization/target-private-simple.f90 b/flang/test/Lower/OpenMP/DelayedPrivatization/target-private-simple.f90
index 3c6836e81abe1..5abf2cbb15c92 100644
--- a/flang/test/Lower/OpenMP/DelayedPrivatization/target-private-simple.f90
+++ b/flang/test/Lower/OpenMP/DelayedPrivatization/target-private-simple.f90
@@ -15,12 +15,7 @@ subroutine target_simple
 end subroutine target_simple
 
 ! CHECK-LABEL: omp.private {type = private}
-! CHECK-SAME:              @[[VAR_PRIVATIZER_SYM:.*]] : !fir.ref<i32> alloc {
-! CHECK:  ^bb0(%[[PRIV_ARG:.*]]: !fir.ref<i32>):
-! CHECK:    %[[PRIV_ALLOC:.*]] = fir.alloca i32 {bindc_name = "simple_var", {{.*}}}
-! CHECK:    %[[PRIV_DECL:.*]]:2 = hlfir.declare %[[PRIV_ALLOC]]
-! CHECK:    omp.yield(%[[PRIV_DECL]]#0 : !fir.ref<i32>)
-! CHECK: }
+! CHECK-SAME:              @[[VAR_PRIVATIZER_SYM:.*]] : i32
 
 ! CHECK-LABEL: func.func @_QPtarget_simple() {
 ! CHECK:  %[[VAR_ALLOC:.*]] = fir.alloca i32 {bindc_name = "simple_var", {{.*}}}
diff --git a/flang/test/Lower/OpenMP/DelayedPrivatization/wsloop.f90 b/flang/test/Lower/OpenMP/DelayedPrivatization/wsloop.f90
index 66fd120085c78..221d08669f899 100644
--- a/flang/test/Lower/OpenMP/DelayedPrivatization/wsloop.f90
+++ b/flang/test/Lower/OpenMP/DelayedPrivatization/wsloop.f90
@@ -1,6 +1,6 @@
-! RUN: %flang_fc1 -emit-hlfir -fopenmp -mmlir --openmp-enable-delayed-privatization-staging \
+! RUN: %flang_fc1 -emit-hlfir -fopenmp \
 ! RUN:   -o - %s 2>&1 | FileCheck %s
-! RUN: bbc -emit-hlfir -fopenmp --openmp-enable-delayed-privatization-staging -o - %s 2>&1 \
+! RUN: bbc -emit-hlfir -fopenmp  -o - %s 2>&1 \
 ! RUN:   | FileCheck %s
 
 subroutine wsloop_private
@@ -13,8 +13,8 @@ subroutine wsloop_private
     end do
 end subroutine wsloop_private
 
-! CHECK: omp.private {type = private} @[[I_PRIVATIZER:.*i_private_ref_i32]]
-! CHECK: omp.private {type = firstprivate} @[[X_PRIVATIZER:.*x_firstprivate_ref_i32]]
+! CHECK: omp.private {type = private} @[[I_PRIVATIZER:.*i_private_i32]]
+! CHECK: omp.private {type = firstprivate} @[[X_PRIVATIZER:.*x_firstprivate_i32]]
 
 ! CHECK: func.func @{{.*}}() {
 ! CHECK:   %[[I_DECL:.*]]:2 = hlfir.declare %{{.*}} {uniq_name = "{{.*}}i"}
diff --git a/flang/test/Lower/OpenMP/Todo/dispatch.f90 b/flang/test/Lower/OpenMP/Todo/dispatch.f90
new file mode 100644
index 0000000000000..380dfa14eaae1
--- /dev/null
+++ b/flang/test/Lower/OpenMP/Todo/dispatch.f90
@@ -0,0 +1,12 @@
+! RUN: %not_todo_cmd %flang_fc1 -emit-fir -fopenmp -fopenmp-version=51 -o - %s 2>&1 | FileCheck %s
+
+! CHECK: not yet implemented: OpenMPDispatchConstruct
+program p
+  integer r
+  r = 1
+!$omp dispatch nowait
+  call foo()
+contains
+  subroutine foo
+  end subroutine
+end program p
diff --git a/flang/test/Lower/OpenMP/Todo/metadirective-exec.f90 b/flang/test/Lower/OpenMP/Todo/metadirective-exec.f90
new file mode 100644
index 0000000000000..2e160a1896616
--- /dev/null
+++ b/flang/test/Lower/OpenMP/Todo/metadirective-exec.f90
@@ -0,0 +1,9 @@
+!RUN: %not_todo_cmd bbc -emit-hlfir -fopenmp -fopenmp-version=52 -o - %s 2>&1 | FileCheck %s
+!RUN: %not_todo_cmd %flang_fc1 -emit-hlfir -fopenmp -fopenmp-version=52 -o - %s 2>&1 | FileCheck %s
+
+!CHECK: not yet implemented: METADIRECTIVE
+subroutine f00
+  continue
+  !Executable
+  !$omp metadirective when(user={condition(.true.)}: nothing)
+end
diff --git a/flang/test/Lower/OpenMP/Todo/metadirective-spec.f90 b/flang/test/Lower/OpenMP/Todo/metadirective-spec.f90
new file mode 100644
index 0000000000000..a00612a92218a
--- /dev/null
+++ b/flang/test/Lower/OpenMP/Todo/metadirective-spec.f90
@@ -0,0 +1,9 @@
+!RUN: %not_todo_cmd bbc -emit-hlfir -fopenmp -fopenmp-version=52 -o - %s 2>&1 | FileCheck %s
+!RUN: %not_todo_cmd %flang_fc1 -emit-hlfir -fopenmp -fopenmp-version=52 -o - %s 2>&1 | FileCheck %s
+
+!CHECK: not yet implemented: METADIRECTIVE
+subroutine f00
+  !Specification
+  !$omp metadirective when(user={condition(.true.)}: nothing)
+  implicit none
+end
diff --git a/flang/test/Lower/OpenMP/Todo/omp-default-clause-inner-loop.f90 b/flang/test/Lower/OpenMP/Todo/omp-default-clause-inner-loop.f90
index 77a1304f39a48..10879c53dc0c5 100644
--- a/flang/test/Lower/OpenMP/Todo/omp-default-clause-inner-loop.f90
+++ b/flang/test/Lower/OpenMP/Todo/omp-default-clause-inner-loop.f90
@@ -9,11 +9,10 @@
 ! The string "EXPECTED" denotes the expected FIR
 
 ! CHECK: omp.parallel  private(@{{.*}} %{{.*}} -> %[[PRIVATE_Y:.*]], @{{.*}} %{{.*}} -> %[[PRIVATE_Y:.*]] : !fir.ref<i32>, !fir.ref<i32>) {
-! CHECK: %[[TEMP:.*]] = fir.alloca i32 {bindc_name = "x", pinned, {{.*}}}
 ! CHECK: %[[const_1:.*]] = arith.constant 1 : i32
 ! CHECK: %[[const_2:.*]] = arith.constant 10 : i32
 ! CHECK: %[[const_3:.*]] = arith.constant 1 : i32
-! CHECK: omp.wsloop {
+! CHECK: omp.wsloop private(@{{.*}} %{{.*}} -> %[[TEMP:.*]] : !fir.ref<i32>) {
 ! CHECK-NEXT: omp.loop_nest (%[[ARG:.*]]) : i32 = (%[[const_1]]) to (%[[const_2]]) inclusive step (%[[const_3]]) {
 ! CHECK: fir.store %[[ARG]] to %[[TEMP]] : !fir.ref<i32>
 ! EXPECTED: %[[temp_1:.*]] = fir.load %[[PRIVATE_Z]] : !fir.ref<i32>
diff --git a/flang/test/Lower/OpenMP/Todo/reduction-inscan.f90 b/flang/test/Lower/OpenMP/Todo/reduction-inscan.f90
deleted file mode 100644
index 152d91a16f80f..0000000000000
--- a/flang/test/Lower/OpenMP/Todo/reduction-inscan.f90
+++ /dev/null
@@ -1,15 +0,0 @@
-! RUN: %not_todo_cmd bbc -emit-fir -fopenmp -o - %s 2>&1 | FileCheck %s
-! RUN: %not_todo_cmd %flang_fc1 -emit-fir -fopenmp -o - %s 2>&1 | FileCheck %s
-
-! CHECK: not yet implemented: Reduction modifiers are not supported
-subroutine reduction_inscan()
-  integer :: i,j
-  i = 0
-
-  !$omp do reduction(inscan, +:i)
-  do j=1,10
-     !$omp scan inclusive(i)
-     i = i + 1
-  end do
-  !$omp end do
-end subroutine reduction_inscan
diff --git a/flang/test/Lower/OpenMP/Todo/reduction-modifiers.f90 b/flang/test/Lower/OpenMP/Todo/reduction-modifiers.f90
deleted file mode 100644
index 82625ed8c5f31..0000000000000
--- a/flang/test/Lower/OpenMP/Todo/reduction-modifiers.f90
+++ /dev/null
@@ -1,14 +0,0 @@
-! RUN: %not_todo_cmd bbc -emit-fir -fopenmp -o - %s 2>&1 | FileCheck %s
-! RUN: %not_todo_cmd %flang_fc1 -emit-fir -fopenmp -o - %s 2>&1 | FileCheck %s
-
-! CHECK: not yet implemented: Reduction modifiers are not supported
-
-subroutine foo()
-  integer :: i, j
-  j = 0
-  !$omp do reduction (inscan, *: j)
-  do i = 1, 10
-    !$omp scan inclusive(j)
-    j = j + 1
-  end do
-end subroutine
diff --git a/flang/test/Lower/OpenMP/Todo/reduction-task.f90 b/flang/test/Lower/OpenMP/Todo/reduction-task.f90
index 6707f65e1a4cc..b8bfc37d1758f 100644
--- a/flang/test/Lower/OpenMP/Todo/reduction-task.f90
+++ b/flang/test/Lower/OpenMP/Todo/reduction-task.f90
@@ -1,7 +1,7 @@
 ! RUN: %not_todo_cmd bbc -emit-fir -fopenmp -o - %s 2>&1 | FileCheck %s
 ! RUN: %not_todo_cmd %flang_fc1 -emit-fir -fopenmp -o - %s 2>&1 | FileCheck %s
 
-! CHECK: not yet implemented: Reduction modifiers are not supported
+! CHECK: not yet implemented: Reduction modifier `task` is not supported
 subroutine reduction_task()
   integer :: i
   i = 0
diff --git a/flang/test/Lower/OpenMP/associate.f90 b/flang/test/Lower/OpenMP/associate.f90
index 4964890a6842c..d497b4ade782e 100644
--- a/flang/test/Lower/OpenMP/associate.f90
+++ b/flang/test/Lower/OpenMP/associate.f90
@@ -6,12 +6,12 @@
 !CHECK:         omp.parallel {
 !CHECK-NOT:       hlfir.declare {{.*}} {uniq_name = "_QFtest_parallel_assocEa"}
 !CHECK-NOT:       hlfir.declare {{.*}} {uniq_name = "_QFtest_parallel_assocEb"}
-!CHECK:           omp.wsloop {
+!CHECK:           omp.wsloop private({{.*}}) {
 !CHECK:           }
 !CHECK:         }
 !CHECK:         omp.parallel {{.*}} {
 !CHECK-NOT:       hlfir.declare {{.*}} {uniq_name = "_QFtest_parallel_assocEb"}
-!CHECK:           omp.wsloop {
+!CHECK:           omp.wsloop private({{.*}}) {
 !CHECK:           }
 !CHECK:         }
 subroutine test_parallel_assoc()
diff --git a/flang/test/Lower/OpenMP/cfg-conversion-omp.private.f90 b/flang/test/Lower/OpenMP/cfg-conversion-omp.private.f90
index 44036492f5595..8b8adf2b140c7 100644
--- a/flang/test/Lower/OpenMP/cfg-conversion-omp.private.f90
+++ b/flang/test/Lower/OpenMP/cfg-conversion-omp.private.f90
@@ -21,34 +21,27 @@ subroutine delayed_privatization_allocatable
 end subroutine
 
 ! CFGConv-LABEL: omp.private {type = private}
-! CFGConv-SAME: @[[PRIVATIZER_SYM:.*]] : [[TYPE:!fir.ref<!fir.box<!fir.heap<i32>>>]] alloc {
+! CFGConv-SAME: @[[PRIVATIZER_SYM:.*]] : [[BOX_TYPE:!fir.box<!fir.heap<i32>>]] init {
 
-! CFGConv-NEXT: ^bb0(%[[PRIV_ARG:.*]]: [[TYPE]]):
-
-! CFGConv-NEXT:   %[[PRIV_ALLOC:.*]] = fir.alloca !fir.box<!fir.heap<i32>> {bindc_name = "var1", pinned, uniq_name = "_QFdelayed_privatization_allocatableEvar1"}
+! CFGConv-NEXT: ^bb0(%[[PRIV_ARG:.*]]: [[TYPE:!fir.ref<!fir.box<!fir.heap<i32>>>]], %[[PRIV_ALLOC:.*]]: [[TYPE]]):
 
 ! CFGConv-NEXT:   %[[PRIV_ARG_VAL:.*]] = fir.load %[[PRIV_ARG]] : !fir.ref<!fir.box<!fir.heap<i32>>>
 ! CFGConv-NEXT:   %[[PRIV_ARG_BOX:.*]] = fir.box_addr %[[PRIV_ARG_VAL]] : (!fir.box<!fir.heap<i32>>) -> !fir.heap<i32>
 ! CFGConv-NEXT:   %[[PRIV_ARG_ADDR:.*]] = fir.convert %[[PRIV_ARG_BOX]] : (!fir.heap<i32>) -> i64
 ! CFGConv-NEXT:   %[[C0:.*]] = arith.constant 0 : i64
-! CFGConv-NEXT:   %[[ALLOC_COND:.*]] = arith.cmpi ne, %[[PRIV_ARG_ADDR]], %[[C0]] : i64
-! CFGConv-NEXT:   cf.cond_br %[[ALLOC_COND]], ^[[ALLOC_MEM_BB:.*]], ^[[ZERO_MEM_BB:.*]]
-! CFGConv-NEXT: ^[[ALLOC_MEM_BB]]:
-! CFGConv:        fir.allocmem
+! CFGConv-NEXT:   %[[ALLOC_COND:.*]] = arith.cmpi eq, %[[PRIV_ARG_ADDR]], %[[C0]] : i64
+! CFGConv-NEXT:   cf.cond_br %[[ALLOC_COND]], ^[[ZERO_MEM_BB:.*]], ^[[ALLOC_MEM_BB:.*]]
+! CFGConv-NEXT: ^[[ZERO_MEM_BB]]:
 ! CFGConv:        cf.br ^[[DECL_BB:.*]]
-! CFGConv:      ^[[ZERO_MEM_BB]]:
-! CFGConv-NEXT:   fir.zero_bits
+! CFGConv:      ^[[ALLOC_MEM_BB]]:
+! CFGConv:        fir.allocmem
 ! CFGConv:        cf.br ^[[DECL_BB:.*]]
 ! CFGConv-NEXT: ^[[DECL_BB]]:
-! CFGConv-NEXT:   hlfir.declare
 ! CFGConv-NEXT:   omp.yield
 
 
 ! LLVMDialect-LABEL: omp.private {type = private}
-! LLVMDialect-SAME: @[[PRIVATIZER_SYM:.*]] : [[TYPE:!llvm.ptr]] alloc {
-
-! LLVMDialect-NEXT: ^bb0(%[[PRIV_ARG:.*]]: [[TYPE]]):
-! LLVMDialect:        llvm.alloca
+! LLVMDialect-SAME: @[[PRIVATIZER_SYM:.*]] : !llvm.struct<(ptr, i64, i32, i8, i8, i8, i8)> init {
+! LLVMDialect-NEXT: ^bb0(%[[PRIV_ARG:.*]]: !llvm.ptr, %[[PRIV_ALLOC:.*]]: !llvm.ptr):
 ! LLVMDialect:        llvm.call @malloc
-
 ! LLVMDialect-NOT:    hlfir.declare
diff --git a/flang/test/Lower/OpenMP/copyin.f90 b/flang/test/Lower/OpenMP/copyin.f90
index ec4c544495c60..0e8fe0eaeed87 100644
--- a/flang/test/Lower/OpenMP/copyin.f90
+++ b/flang/test/Lower/OpenMP/copyin.f90
@@ -154,14 +154,13 @@ subroutine copyin_derived_type()
 
 ! CHECK:             omp.barrier
 
-! CHECK:             %[[VAL_6:.*]] = fir.alloca i32 {bindc_name = "i", pinned, {{.*}}}
-! CHECK:             %[[VAL_7:.*]]:2 = hlfir.declare %[[VAL_6]] {uniq_name = "_QFcombined_parallel_worksharing_loopEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 
 ! CHECK:             %[[VAL_11:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_12:.*]] = fir.load %[[VAL_9]]#0 : !fir.ref<i32>
 ! CHECK:             %[[VAL_13:.*]] = arith.constant 1 : i32
-! CHECK:             omp.wsloop {
+! CHECK:             omp.wsloop private(@{{.*}} %{{.*}} -> %[[VAL_6:.*]] : !fir.ref<i32>) {
 ! CHECK-NEXT:          omp.loop_nest (%[[VAL_14:.*]]) : i32 = (%[[VAL_11]]) to (%[[VAL_12]]) inclusive step (%[[VAL_13]]) {
+! CHECK:                 %[[VAL_7:.*]]:2 = hlfir.declare %[[VAL_6]] {uniq_name = "_QFcombined_parallel_worksharing_loopEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:                 fir.store %[[VAL_14]] to %[[VAL_7]]#1 : !fir.ref<i32>
 ! CHECK:                 fir.call @_QPsub4(%[[VAL_9]]#1) fastmath<contract> : (!fir.ref<i32>) -> ()
 ! CHECK:                 omp.yield
@@ -321,15 +320,12 @@ subroutine common_1()
 ! CHECK:             %[[VAL_33:.*]] = fir.load %[[VAL_18]]#0 : !fir.ref<i32>
 ! CHECK:             hlfir.assign %[[VAL_33]] to %[[VAL_31]]#0 : i32, !fir.ref<i32>
 ! CHECK:             omp.barrier
-
-! CHECK:             %[[VAL_19:.*]] = fir.alloca i32 {bindc_name = "i", pinned, {{.*}}}
-! CHECK:             %[[VAL_20:.*]]:2 = hlfir.declare %[[VAL_19]] {uniq_name = "_QFcommon_2Ei"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-
 ! CHECK:             %[[VAL_34:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_35:.*]] = fir.load %[[VAL_26]]#0 : !fir.ref<i32>
 ! CHECK:             %[[VAL_36:.*]] = arith.constant 1 : i32
-! CHECK:             omp.wsloop {
+! CHECK:             omp.wsloop private(@{{.*}} %{{.*}} -> %[[VAL_19:.*]] : !fir.ref<i32>) {
 ! CHECK-NEXT:          omp.loop_nest (%[[VAL_37:.*]]) : i32 = (%[[VAL_34]]) to (%[[VAL_35]]) inclusive step (%[[VAL_36]]) {
+! CHECK:             %[[VAL_20:.*]]:2 = hlfir.declare %[[VAL_19]] {uniq_name = "_QFcommon_2Ei"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:                 fir.store %[[VAL_37]] to %[[VAL_20]]#1 : !fir.ref<i32>
 ! CHECK:                 %[[VAL_38:.*]] = fir.load %[[VAL_31]]#0 : !fir.ref<i32>
 ! CHECK:                 %[[VAL_39:.*]] = fir.load %[[VAL_20]]#0 : !fir.ref<i32>
diff --git a/flang/test/Lower/OpenMP/copyprivate.f90 b/flang/test/Lower/OpenMP/copyprivate.f90
index 761e6190ed6ef..4c3ed9389369f 100644
--- a/flang/test/Lower/OpenMP/copyprivate.f90
+++ b/flang/test/Lower/OpenMP/copyprivate.f90
@@ -14,13 +14,13 @@
 !CHECK-DAG: func private @_copy_c16x8(%{{.*}}: !fir.ref<!fir.char<2,8>>, %{{.*}}: !fir.ref<!fir.char<2,8>>)
 
 !CHECK-DAG: func private @_copy_box_Uxi32(%{{.*}}: !fir.ref<!fir.box<!fir.array<?xi32>>>, %{{.*}}: !fir.ref<!fir.box<!fir.array<?xi32>>>)
-!CHECK-DAG: func private @_copy_10xi32(%{{.*}}: !fir.ref<!fir.array<10xi32>>, %{{.*}}: !fir.ref<!fir.array<10xi32>>)
-!CHECK-DAG: func private @_copy_3x4xi32(%{{.*}}: !fir.ref<!fir.array<3x4xi32>>, %{{.*}}: !fir.ref<!fir.array<3x4xi32>>)
-!CHECK-DAG: func private @_copy_10xf32(%{{.*}}: !fir.ref<!fir.array<10xf32>>, %{{.*}}: !fir.ref<!fir.array<10xf32>>)
-!CHECK-DAG: func private @_copy_3x4xz32(%{{.*}}: !fir.ref<!fir.array<3x4xcomplex<f32>>>, %{{.*}}: !fir.ref<!fir.array<3x4xcomplex<f32>>>)
-!CHECK-DAG: func private @_copy_10xl32(%{{.*}}: !fir.ref<!fir.array<10x!fir.logical<4>>>, %{{.*}}: !fir.ref<!fir.array<10x!fir.logical<4>>>)
-!CHECK-DAG: func private @_copy_3xc8x8(%{{.*}}: !fir.ref<!fir.array<3x!fir.char<1,8>>>, %{{.*}}: !fir.ref<!fir.array<3x!fir.char<1,8>>>)
-!CHECK-DAG: func private @_copy_3xc16x5(%{{.*}}: !fir.ref<!fir.array<3x!fir.char<2,5>>>, %{{.*}}: !fir.ref<!fir.array<3x!fir.char<2,5>>>)
+!CHECK-DAG: func private @_copy_box_10xi32(%{{.*}}: !fir.ref<!fir.box<!fir.array<10xi32>>>, %{{.*}}: !fir.ref<!fir.box<!fir.array<10xi32>>>)
+!CHECK-DAG: func private @_copy_box_3x4xi32(%{{.*}}: !fir.ref<!fir.box<!fir.array<3x4xi32>>>, %{{.*}}: !fir.ref<!fir.box<!fir.array<3x4xi32>>>)
+!CHECK-DAG: func private @_copy_box_10xf32(%{{.*}}: !fir.ref<!fir.box<!fir.array<10xf32>>>, %{{.*}}: !fir.ref<!fir.box<!fir.array<10xf32>>>)
+!CHECK-DAG: func private @_copy_box_3x4xz32(%{{.*}}: !fir.ref<!fir.box<!fir.array<3x4xcomplex<f32>>>>, %{{.*}}: !fir.ref<!fir.box<!fir.array<3x4xcomplex<f32>>>>)
+!CHECK-DAG: func private @_copy_box_10xl32(%{{.*}}: !fir.ref<!fir.box<!fir.array<10x!fir.logical<4>>>>, %{{.*}}: !fir.ref<!fir.box<!fir.array<10x!fir.logical<4>>>>)
+!CHECK-DAG: func private @_copy_box_3xc8x8(%{{.*}}: !fir.ref<!fir.box<!fir.array<3x!fir.char<1,8>>>>, %{{.*}}: !fir.ref<!fir.box<!fir.array<3x!fir.char<1,8>>>>)
+!CHECK-DAG: func private @_copy_box_3xc16x5(%{{.*}}: !fir.ref<!fir.box<!fir.array<3x!fir.char<2,5>>>>, %{{.*}}: !fir.ref<!fir.box<!fir.array<3x!fir.char<2,5>>>>)
 
 !CHECK-DAG: func private @_copy_rec__QFtest_dtTdt(%{{.*}}: !fir.ref<!fir.type<_QFtest_dtTdt{i:i32,r:f32}>>, %{{.*}}: !fir.ref<!fir.type<_QFtest_dtTdt{i:i32,r:f32}>>)
 !CHECK-DAG: func private @_copy_box_heap_Uxi32(%{{.*}}: !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>, %{{.*}}: !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>)
@@ -95,20 +95,16 @@ subroutine test_scalar()
 
 !CHECK-LABEL: func @_QPtest_array
 !CHECK:         omp.parallel
-!CHECK:           %[[A:.*]]:2 = hlfir.declare %{{.*}}(%{{.*}}) {uniq_name = "_QFtest_arrayEa"} : (!fir.box<!fir.array<?xi32>>, !fir.shift<1>) -> (!fir.box<!fir.array<?xi32>>, !fir.box<!fir.array<?xi32>>)
-!CHECK:           %[[I1:.*]]:2 = hlfir.declare %{{.*}}(%{{.*}}) {uniq_name = "_QFtest_arrayEi1"} : (!fir.ref<!fir.array<10xi32>>, !fir.shape<1>) -> (!fir.ref<!fir.array<10xi32>>, !fir.ref<!fir.array<10xi32>>)
-!CHECK:           %[[I2:.*]]:2 = hlfir.declare %{{.*}}(%{{.*}}) {uniq_name = "_QFtest_arrayEi2"} : (!fir.ref<!fir.array<3x4xi32>>, !fir.shape<2>) -> (!fir.ref<!fir.array<3x4xi32>>, !fir.ref<!fir.array<3x4xi32>>)
-!CHECK:           %[[I3:.*]]:2 = hlfir.declare %{{.*}}(%{{.*}}) {uniq_name = "_QFtest_arrayEi3"} : (!fir.ref<!fir.array<?xi32>>, !fir.shapeshift<1>) -> (!fir.box<!fir.array<?xi32>>, !fir.ref<!fir.array<?xi32>>)
-!CHECK:           %[[R1:.*]]:2 = hlfir.declare %{{.*}}(%{{.*}}) {uniq_name = "_QFtest_arrayEr1"} : (!fir.ref<!fir.array<10xf32>>, !fir.shape<1>) -> (!fir.ref<!fir.array<10xf32>>, !fir.ref<!fir.array<10xf32>>)
-!CHECK:           %[[C1:.*]]:2 = hlfir.declare %{{.*}}(%{{.*}}) {uniq_name = "_QFtest_arrayEc1"} : (!fir.ref<!fir.array<3x4xcomplex<f32>>>, !fir.shape<2>) -> (!fir.ref<!fir.array<3x4xcomplex<f32>>>, !fir.ref<!fir.array<3x4xcomplex<f32>>>)
-!CHECK:           %[[L1:.*]]:2 = hlfir.declare %{{.*}}(%{{.*}}) {uniq_name = "_QFtest_arrayEl1"} : (!fir.ref<!fir.array<10x!fir.logical<4>>>, !fir.shape<1>) -> (!fir.ref<!fir.array<10x!fir.logical<4>>>, !fir.ref<!fir.array<10x!fir.logical<4>>>)
-!CHECK:           %[[S1:.*]]:2 = hlfir.declare {{.*}} {uniq_name = "_QFtest_arrayEs1"} : (!fir.ref<!fir.array<3x!fir.char<1,8>>>, !fir.shape<1>, index) -> (!fir.ref<!fir.array<3x!fir.char<1,8>>>, !fir.ref<!fir.array<3x!fir.char<1,8>>>)
-!CHECK:           %[[S2:.*]]:2 = hlfir.declare {{.*}} {uniq_name = "_QFtest_arrayEs2"} : (!fir.ref<!fir.array<3x!fir.char<2,5>>>, !fir.shape<1>, index) -> (!fir.ref<!fir.array<3x!fir.char<2,5>>>, !fir.ref<!fir.array<3x!fir.char<2,5>>>)
-!CHECK:           %[[A_REF:.*]] = fir.alloca !fir.box<!fir.array<?xi32>>
-!CHECK:           fir.store %[[A]]#0 to %[[A_REF]] : !fir.ref<!fir.box<!fir.array<?xi32>>>
-!CHECK:           %[[I3_REF:.*]] = fir.alloca !fir.box<!fir.array<?xi32>>
-!CHECK:           fir.store %[[I3]]#0 to %[[I3_REF]] : !fir.ref<!fir.box<!fir.array<?xi32>>>
-!CHECK:           omp.single copyprivate(%[[A_REF]] -> @_copy_box_Uxi32 : !fir.ref<!fir.box<!fir.array<?xi32>>>, %[[I1]]#0 -> @_copy_10xi32 : !fir.ref<!fir.array<10xi32>>, %[[I2]]#0 -> @_copy_3x4xi32 : !fir.ref<!fir.array<3x4xi32>>, %[[I3_REF]] -> @_copy_box_Uxi32 : !fir.ref<!fir.box<!fir.array<?xi32>>>, %[[R1]]#0 -> @_copy_10xf32 : !fir.ref<!fir.array<10xf32>>, %[[C1]]#0 -> @_copy_3x4xz32 : !fir.ref<!fir.array<3x4xcomplex<f32>>>, %[[L1]]#0 -> @_copy_10xl32 : !fir.ref<!fir.array<10x!fir.logical<4>>>, %[[S1]]#0 -> @_copy_3xc8x8 : !fir.ref<!fir.array<3x!fir.char<1,8>>>, %[[S2]]#0 -> @_copy_3xc16x5 : !fir.ref<!fir.array<3x!fir.char<2,5>>>)
+!CHECK:           %[[A:.*]]:2 = hlfir.declare %{{.*}} {uniq_name = "_QFtest_arrayEa"}
+!CHECK:           %[[I1:.*]]:2 = hlfir.declare %{{.*}} {uniq_name = "_QFtest_arrayEi1"}
+!CHECK:           %[[I2:.*]]:2 = hlfir.declare %{{.*}} {uniq_name = "_QFtest_arrayEi2"}
+!CHECK:           %[[I3:.*]]:2 = hlfir.declare %{{.*}} {uniq_name = "_QFtest_arrayEi3"}
+!CHECK:           %[[R1:.*]]:2 = hlfir.declare %{{.*}} {uniq_name = "_QFtest_arrayEr1"}
+!CHECK:           %[[C1:.*]]:2 = hlfir.declare %{{.*}} {uniq_name = "_QFtest_arrayEc1"}
+!CHECK:           %[[L1:.*]]:2 = hlfir.declare %{{.*}} {uniq_name = "_QFtest_arrayEl1"}
+!CHECK:           %[[S1:.*]]:2 = hlfir.declare {{.*}} {uniq_name = "_QFtest_arrayEs1"}
+!CHECK:           %[[S2:.*]]:2 = hlfir.declare {{.*}} {uniq_name = "_QFtest_arrayEs2"}
+!CHECK:           omp.single copyprivate(%[[A]]#0 -> @_copy_box_Uxi32 : {{.*}}, %[[I1]]#0 -> @_copy_box_10xi32 : {{.*}}, %[[I2]]#0 -> @_copy_box_3x4xi32 : {{.*}}, %[[I3]]#0 -> @_copy_box_Uxi32 : {{.*}}, %[[R1]]#0 -> @_copy_box_10xf32 : {{.*}}, %[[C1]]#0 -> @_copy_box_3x4xz32 : {{.*}}, %[[L1]]#0 -> @_copy_box_10xl32 : {{.*}}, %[[S1]]#0 -> @_copy_box_3xc8x8 : {{.*}}, %[[S2]]#0 -> @_copy_box_3xc16x5 : {{.*}})
 subroutine test_array(a, n)
   integer :: a(:), n
   integer :: i1(10), i2(3, 4), i3(n)
diff --git a/flang/test/Lower/OpenMP/critical.f90 b/flang/test/Lower/OpenMP/critical.f90
index 051d378210646..99a4426ab0453 100644
--- a/flang/test/Lower/OpenMP/critical.f90
+++ b/flang/test/Lower/OpenMP/critical.f90
@@ -38,11 +38,10 @@ subroutine predetermined_privatization()
   !CHECK: omp.parallel
   !$omp parallel do
 
-  !CHECK: %[[PRIV_I_ALLOC:.*]] = fir.alloca i32 {bindc_name = "i", pinned, {{.*}}}
-  !CHECK: %[[PRIV_I_DECL:.*]]:2 = hlfir.declare %[[PRIV_I_ALLOC]]
   do i = 2, 10
-    !CHECK: omp.wsloop
+    !CHECK: omp.wsloop private(@{{.*}} %{{.*}} -> %[[PRIV_I_ALLOC:.*]] : !fir.ref<i32>)
     !CHECK: omp.loop_nest (%[[IV:[^[:space:]]+]])
+    !CHECK: %[[PRIV_I_DECL:.*]]:2 = hlfir.declare %[[PRIV_I_ALLOC]]
     !CHECK: fir.store %[[IV]] to %[[PRIV_I_DECL]]#1
     !CHECK: omp.critical
     !$omp critical
diff --git a/flang/test/Lower/OpenMP/default-clause-byref.f90 b/flang/test/Lower/OpenMP/default-clause-byref.f90
index 654c13ada9e39..c44c6bb966580 100644
--- a/flang/test/Lower/OpenMP/default-clause-byref.f90
+++ b/flang/test/Lower/OpenMP/default-clause-byref.f90
@@ -7,57 +7,27 @@
 ! RUN: bbc -fopenmp -emit-hlfir --force-byref-reduction %s -o - \
 ! RUN: | FileCheck %s
 
-!CHECK:  omp.private {type = firstprivate} @[[W_FIRSTPRIVATIZER:_QFEw_firstprivate_ref_i32]] : !fir.ref<i32> alloc {
-!CHECK:  ^bb0(%{{.*}}: !fir.ref<i32>):
-!CHECK:    %[[PRIV_W_ALLOC:.*]] = fir.alloca i32 {bindc_name = "w", {{.*}}}
-!CHECK:    %[[PRIV_W_DECL:.*]]:2 = hlfir.declare %[[PRIV_W_ALLOC]] {uniq_name = "_QFEw"}
-!CHECK:    omp.yield(%[[PRIV_W_DECL]]#0 : !fir.ref<i32>)
-!CHECK:  } copy {
+!CHECK:  omp.private {type = firstprivate} @[[W_FIRSTPRIVATIZER:_QFEw_firstprivate_i32]] : i32 copy {
 !CHECK:  ^bb0(%[[ORIG_W:.*]]: !fir.ref<i32>, %[[PRIV_W:.*]]: !fir.ref<i32>):
 !CHECK:    %[[ORIG_W_VAL:.*]] = fir.load %[[ORIG_W]]
 !CHECK:    hlfir.assign %[[ORIG_W_VAL]] to %[[PRIV_W]]
 !CHECK:    omp.yield(%[[PRIV_W]] : !fir.ref<i32>)
 !CHECK:  }
 
-!CHECK:  omp.private {type = firstprivate} @[[Y_FIRSTPRIVATIZER:_QFEy_firstprivate_ref_i32]] : !fir.ref<i32> alloc {
-!CHECK:  ^bb0(%{{.*}}: !fir.ref<i32>):
-!CHECK:    %[[PRIV_Y_ALLOC:.*]] = fir.alloca i32 {bindc_name = "y", {{.*}}}
-!CHECK:    %[[PRIV_Y_DECL:.*]]:2 = hlfir.declare %[[PRIV_Y_ALLOC]] {uniq_name = "_QFEy"}
-!CHECK:    omp.yield(%[[PRIV_Y_DECL]]#0 : !fir.ref<i32>)
-!CHECK:  } copy {
+!CHECK:  omp.private {type = firstprivate} @[[Y_FIRSTPRIVATIZER:_QFEy_firstprivate_i32]] : i32 copy {
 !CHECK:  ^bb0(%[[ORIG_Y:.*]]: !fir.ref<i32>, %[[PRIV_Y:.*]]: !fir.ref<i32>):
 !CHECK:    %[[ORIG_Y_VAL:.*]] = fir.load %[[ORIG_Y]]
 !CHECK:    hlfir.assign %[[ORIG_Y_VAL]] to %[[PRIV_Y]]
 !CHECK:    omp.yield(%[[PRIV_Y]] : !fir.ref<i32>)
 !CHECK:  }
 
-!CHECK:  omp.private {type = private} @[[X_PRIVATIZER:_QFEx_private_ref_i32]] : !fir.ref<i32> alloc {
-!CHECK:  ^bb0(%{{.*}}: !fir.ref<i32>):
-!CHECK:    %[[PRIV_X_ALLOC:.*]] = fir.alloca i32 {bindc_name = "x", {{.*}}}
-!CHECK:    %[[PRIV_X_DECL:.*]]:2 = hlfir.declare %[[PRIV_X_ALLOC]] {uniq_name = "_QFEx"}
-!CHECK:    omp.yield(%[[PRIV_X_DECL]]#0 : !fir.ref<i32>)
-!CHECK:  }
+!CHECK:  omp.private {type = private} @[[X_PRIVATIZER:_QFEx_private_i32]] : i32
 
-!CHECK:  omp.private {type = private} @[[W_PRIVATIZER:_QFEw_private_ref_i32]] : !fir.ref<i32> alloc {
-!CHECK:  ^bb0(%{{.*}}: !fir.ref<i32>):
-!CHECK:    %[[PRIV_W_ALLOC:.*]] = fir.alloca i32 {bindc_name = "w", {{.*}}}
-!CHECK:    %[[PRIV_W_DECL:.*]]:2 = hlfir.declare %[[PRIV_W_ALLOC]] {uniq_name = "_QFEw"}
-!CHECK:    omp.yield(%[[PRIV_W_DECL]]#0 : !fir.ref<i32>)
-!CHECK:  }
+!CHECK:  omp.private {type = private} @[[W_PRIVATIZER:_QFEw_private_i32]] : i32
 
-!CHECK:  omp.private {type = private} @[[Y_PRIVATIZER:_QFEy_private_ref_i32]] : !fir.ref<i32> alloc {
-!CHECK:  ^bb0(%{{.*}}: !fir.ref<i32>):
-!CHECK:    %[[PRIV_Y_ALLOC:.*]] = fir.alloca i32 {bindc_name = "y", {{.*}}}
-!CHECK:    %[[PRIV_Y_DECL:.*]]:2 = hlfir.declare %[[PRIV_Y_ALLOC]] {uniq_name = "_QFEy"}
-!CHECK:    omp.yield(%[[PRIV_Y_DECL]]#0 : !fir.ref<i32>)
-!CHECK:  }
+!CHECK:  omp.private {type = private} @[[Y_PRIVATIZER:_QFEy_private_i32]] : i32
 
-!CHECK:  omp.private {type = firstprivate} @[[X_FIRSTPRIVATIZER:_QFEx_firstprivate_ref_i32]] : !fir.ref<i32> alloc {
-!CHECK:  ^bb0(%{{.*}}: !fir.ref<i32>):
-!CHECK:    %[[PRIV_X_ALLOC:.*]] = fir.alloca i32 {bindc_name = "x", {{.*}}}
-!CHECK:    %[[PRIV_X_DECL:.*]]:2 = hlfir.declare %[[PRIV_X_ALLOC]] {uniq_name = "_QFEx"}
-!CHECK:    omp.yield(%[[PRIV_X_DECL]]#0 : !fir.ref<i32>)
-!CHECK:  } copy {
+!CHECK:  omp.private {type = firstprivate} @[[X_FIRSTPRIVATIZER:_QFEx_firstprivate_i32]] : i32 copy {
 !CHECK:  ^bb0(%[[ORIG_X:.*]]: !fir.ref<i32>, %[[PRIV_X:.*]]: !fir.ref<i32>):
 !CHECK:    %[[ORIG_X_VAL:.*]] = fir.load %[[ORIG_X]]
 !CHECK:    hlfir.assign %[[ORIG_X_VAL]] to %[[PRIV_X]]
@@ -346,7 +316,7 @@ subroutine skipped_default_clause_checks()
        type(it)::iii
 
 !CHECK: omp.parallel {{.*}} {
-!CHECK: omp.wsloop reduction(byref @min_byref_i32 %[[VAL_Z_DECLARE]]#0 -> %[[PRV:.+]] : !fir.ref<i32>) {
+!CHECK: omp.wsloop private({{.*}}) reduction(byref @min_byref_i32 %[[VAL_Z_DECLARE]]#0 -> %[[PRV:.+]] : !fir.ref<i32>) {
 !CHECK-NEXT: omp.loop_nest (%[[ARG:.*]]) {{.*}} {
 !CHECK: omp.yield
 !CHECK: }
diff --git a/flang/test/Lower/OpenMP/default-clause.f90 b/flang/test/Lower/OpenMP/default-clause.f90
index c004813a911f7..fcc8d033eea0f 100644
--- a/flang/test/Lower/OpenMP/default-clause.f90
+++ b/flang/test/Lower/OpenMP/default-clause.f90
@@ -284,16 +284,13 @@ subroutine nested_default_clause_test4
 !CHECK-LABEL: func @_QPnested_default_clause_test5
 !CHECK: omp.parallel {
 
-!CHECK: %[[X_ALLOCA:.*]] = fir.alloca i32 {bindc_name = "x", pinned, uniq_name = "_QFnested_default_clause_test5Ex"}
-!CHECK: %[[X_DECLARE:.*]]:2 = hlfir.declare %[[X_ALLOCA]] {{.*}}
-
-!CHECK: %[[LOOP_VAR_ALLOCA:.*]] = fir.alloca i32 {bindc_name = "i", pinned, {{.*}}}
-!CHECK: %[[LOOP_VAR_DECLARE:.*]]:2 = hlfir.declare %[[LOOP_VAR_ALLOCA]] {{.*}}
-
 !CHECK: %[[CONST_LB:.*]] = arith.constant 1 : i32
 !CHECK: %[[CONST_UB:.*]] = arith.constant 50 : i32
 !CHECK: %[[CONST_STEP:.*]] = arith.constant 1 : i32
+! CHECK: omp.wsloop private(@{{.*}} %{{.*}} -> %[[X_ALLOCA:.*]], @{{.*}} %{{.*}} -> %[[LOOP_VAR_ALLOCA:.*]] : !fir.ref<i32>, !fir.ref<i32>) {
 !CHECK: omp.loop_nest (%[[ARG:.*]]) : i32 = (%[[CONST_LB]]) to (%[[CONST_UB]]) inclusive step (%[[CONST_STEP]]) {
+!CHECK: %[[X_DECLARE:.*]]:2 = hlfir.declare %[[X_ALLOCA]] {{.*}}
+!CHECK: %[[LOOP_VAR_DECLARE:.*]]:2 = hlfir.declare %[[LOOP_VAR_ALLOCA]] {{.*}}
 !CHECK: fir.store %[[ARG]] to %[[LOOP_VAR_DECLARE]]#1 : !fir.ref<i32>
 !CHECK: %[[LOADED_X:.*]] = fir.load %[[X_DECLARE]]#0 : !fir.ref<i32>
 !CHECK: %[[CONST:.*]] = arith.constant 1 : i32
@@ -321,13 +318,12 @@ subroutine nested_default_clause_test5
 
 !CHECK: %[[Z_VAR_DECLARE:.*]]:2 = hlfir.declare %[[Z_VAR]] {{.*}}
 
-!CHECK: %[[LOOP_VAR:.*]] = fir.alloca i32 {bindc_name = "i", pinned, {{.*}}}
-!CHECK: %[[LOOP_VAR_DECLARE:.*]]:2 = hlfir.declare %[[LOOP_VAR]] {{.*}}
-
 !CHECK: %[[CONST_LB:.*]] = arith.constant 1 : i32
 !CHECK: %[[CONST_UB:.*]] = arith.constant 10 : i32
 !CHECK: %[[CONST_STEP:.*]] = arith.constant 1 : i32
+! CHECK: omp.wsloop private(@{{.*}} %{{.*}} -> %[[LOOP_VAR:.*]] : !fir.ref<i32>) {
 !CHECK: omp.loop_nest (%[[ARG:.*]]) : i32 = (%[[CONST_LB]]) to (%[[CONST_UB]]) inclusive step (%[[CONST_STEP]]) {
+!CHECK: %[[LOOP_VAR_DECLARE:.*]]:2 = hlfir.declare %[[LOOP_VAR]] {{.*}}
 !CHECK: fir.store %[[ARG]] to %[[LOOP_VAR_DECLARE]]#1 : !fir.ref<i32>
 !CHECK: %[[LOADED_X:.*]] = fir.load %[[X_VAR_DECLARE]]#0 : !fir.ref<i32>
 !CHECK: %[[CONST:.*]] = arith.constant 1 : i32
@@ -386,7 +382,7 @@ subroutine skipped_default_clause_checks()
        type(it)::iii
 
 !CHECK: omp.parallel {{.*}} {
-!CHECK: omp.wsloop reduction(@min_i32 %[[VAL_Z_DECLARE]]#0 -> %[[PRV:.+]] : !fir.ref<i32>) {
+!CHECK: omp.wsloop private({{.*}}) reduction(@min_i32 %[[VAL_Z_DECLARE]]#0 -> %[[PRV:.+]] : !fir.ref<i32>) {
 !CHECK-NEXT: omp.loop_nest (%[[ARG:.*]]) {{.*}} {
 !CHECK: omp.yield
 !CHECK: }
diff --git a/flang/test/Lower/OpenMP/delayed-privatization-allocatable-array.f90 b/flang/test/Lower/OpenMP/delayed-privatization-allocatable-array.f90
index 759d80cf45b2a..9b6dbabf0c6ff 100644
--- a/flang/test/Lower/OpenMP/delayed-privatization-allocatable-array.f90
+++ b/flang/test/Lower/OpenMP/delayed-privatization-allocatable-array.f90
@@ -16,38 +16,35 @@ subroutine delayed_privatization_private(var1, l1)
 end subroutine
 
 ! CHECK-LABEL: omp.private {type = firstprivate}
-! CHECK-SAME: @[[PRIVATIZER_SYM:.*]] : [[TYPE:!fir.ref<!fir.box<!fir.heap<!fir.array<\?xi32>>>>]] alloc {
+! CHECK-SAME: @[[PRIVATIZER_SYM:.*]] : [[BOX_TYPE:!fir.box<!fir.heap<!fir.array<\?xi32>>>]] init {
 
-! CHECK-NEXT: ^bb0(%[[PRIV_ARG:.*]]: [[TYPE]]):
-! CHECK-NEXT:   %[[PRIV_ALLOC:.*]] = fir.alloca !fir.box<!fir.heap<!fir.array<{{\?}}xi32>>> {bindc_name = "var1", pinned, uniq_name = "_QFdelayed_privatization_privateEvar1"}
+! CHECK-NEXT: ^bb0(%[[PRIV_ARG:.*]]: [[TYPE:!fir.ref<!fir.box<!fir.heap<!fir.array<\?xi32>>>>]], %[[PRIV_ALLOC:.*]]: [[TYPE]]):
 
 ! CHECK-NEXT:   %[[PRIV_ARG_VAL:.*]] = fir.load %[[PRIV_ARG]]
 ! CHECK-NEXT:   %[[PRIV_ARG_BOX:.*]] = fir.box_addr %[[PRIV_ARG_VAL]]
 ! CHECK-NEXT:   %[[PRIV_ARG_ADDR:.*]] = fir.convert %[[PRIV_ARG_BOX]]
 ! CHECK-NEXT:   %[[C0:.*]] = arith.constant 0 : i64
-! CHECK-NEXT:   %[[ALLOC_COND:.*]] = arith.cmpi ne, %[[PRIV_ARG_ADDR]], %[[C0]] : i64
+! CHECK-NEXT:   %[[ALLOC_COND:.*]] = arith.cmpi eq, %[[PRIV_ARG_ADDR]], %[[C0]] : i64
 
 ! CHECK-NEXT:   fir.if %[[ALLOC_COND]] {
-! CHECK-NEXT:     %[[PRIV_ARG_VAL:.*]] = fir.load %[[PRIV_ARG]] : [[TYPE]]
+! CHECK-NEXT:     %[[C0_2:.*]] = arith.constant 0 : index
+! CHECK-NEXT:     %[[SHAPE:.*]] = fir.shape %[[C0_2]]
+! CHECK-NEXT:     %[[EMBOX_2:.*]] = fir.embox %[[PRIV_ARG_BOX]](%[[SHAPE]])
+! CHECK-NEXT:     fir.store %[[EMBOX_2]] to %[[PRIV_ALLOC]]
+! CHECK-NEXT:   } else {
 ! CHECK-NEXT:     %[[C0:.*]] = arith.constant 0 : index
 ! CHECK-NEXT:     %[[DIMS:.*]]:3 = fir.box_dims %[[PRIV_ARG_VAL]], %[[C0]]
-! CHECK-NEXT:     fir.box_addr %[[PRIV_ARG_VAL]]
-! CHECK-NEXT:     %[[C0_2:.*]] = arith.constant 0 : index 
-! CHECK-NEXT:     %[[CMP:.*]] = arith.cmpi sgt, %[[DIMS]]#1, %[[C0_2]] : index
-! CHECK-NEXT:     %[[SELECT:.*]] = arith.select %[[CMP]], %[[DIMS]]#1, %[[C0_2]] : index
-! CHECK-NEXT:     %[[MEM:.*]] = fir.allocmem !fir.array<?xi32>, %[[SELECT]]
-! CHECK-NEXT:     %[[SHAPE_SHIFT:.*]] = fir.shape_shift %[[DIMS]]#0, %[[SELECT]] : (index, index) -> !fir.shapeshift<1>
-! CHECK-NEXT:     %[[EMBOX:.*]] = fir.embox %[[MEM]](%[[SHAPE_SHIFT]])
+! CHECK-NEXT:     %[[SHAPE:.*]] = fir.shape %[[DIMS]]#1
+! CHECK-NEXT:     %[[MEM:.*]] = fir.allocmem !fir.array<?xi32>, %[[DIMS]]#1
+! CHECK-NEXT:     %[[TRUE:.*]] = arith.constant true
+! CHECK-NEXT:     %[[DECL:.*]]:2 = hlfir.declare %[[MEM]](%[[SHAPE]])
+! CHECK-NEXT:     %[[C0_2:.*]] = arith.constant 0 : index
+! CHECK-NEXT:     %[[DIMS_2:.*]]:3 = fir.box_dims %[[PRIV_ARG_VAL]], %[[C0_2]]
+! CHECK-NEXT:     %[[SHAPE_SHIFT:.*]] = fir.shape_shift %[[DIMS_2]]#0, %[[DIMS_2]]#1
+! CHECK-NEXT:     %[[EMBOX:.*]] = fir.rebox %[[DECL]]#0(%[[SHAPE_SHIFT]])
 ! CHECK-NEXT:     fir.store %[[EMBOX]] to %[[PRIV_ALLOC]]
-! CHECK-NEXT:   } else {
-! CHECK-NEXT:     %[[ZEROS:.*]] = fir.zero_bits
-! CHECK-NEXT:     %[[C0_3:.*]] = arith.constant 0 : index
-! CHECK-NEXT:     %[[SHAPE:.*]] = fir.shape %[[C0_3]] : (index) -> !fir.shape<1>
-! CHECK-NEXT:     %[[EMBOX_2:.*]] = fir.embox %[[ZEROS]](%[[SHAPE]])
-! CHECK-NEXT:     fir.store %[[EMBOX_2]] to %[[PRIV_ALLOC]]
 ! CHECK-NEXT:   }
 
-! CHECK-NEXT:   hlfir.declare
 ! CHECK-NEXT:   omp.yield
 
 ! CHECK-NEXT: } copy {
diff --git a/flang/test/Lower/OpenMP/delayed-privatization-allocatable-firstprivate.f90 b/flang/test/Lower/OpenMP/delayed-privatization-allocatable-firstprivate.f90
index b3a668018df1d..01ca1073ae849 100644
--- a/flang/test/Lower/OpenMP/delayed-privatization-allocatable-firstprivate.f90
+++ b/flang/test/Lower/OpenMP/delayed-privatization-allocatable-firstprivate.f90
@@ -18,9 +18,9 @@ subroutine delayed_privatization_allocatable
 end subroutine
 
 ! CHECK-LABEL: omp.private {type = firstprivate}
-! CHECK-SAME: @[[PRIVATIZER_SYM:.*]] : [[TYPE:!fir.ref<!fir.box<!fir.heap<i32>>>]] alloc {
+! CHECK-SAME: @[[PRIVATIZER_SYM:.*]] : [[BOX_TYPE:!fir.box<!fir.heap<i32>>]] init {
 
-! CHECK-NEXT: ^bb0(%[[PRIV_ARG:.*]]: [[TYPE]]):
+! CHECK-NEXT: ^bb0(%[[PRIV_ARG:.*]]: [[TYPE:!fir.ref<!fir.box<!fir.heap<i32>>>]], %[[PRIV_ALLOC:.*]]: [[TYPE]]):
 
 ! CHECK: } copy {
 ! CHECK: ^bb0(%[[PRIV_ORIG_ARG:.*]]: [[TYPE]], %[[PRIV_PRIV_ARG:.*]]: [[TYPE]]):
diff --git a/flang/test/Lower/OpenMP/delayed-privatization-allocatable-private.f90 b/flang/test/Lower/OpenMP/delayed-privatization-allocatable-private.f90
index f1fae2540aa4d..4ce66f52110e0 100644
--- a/flang/test/Lower/OpenMP/delayed-privatization-allocatable-private.f90
+++ b/flang/test/Lower/OpenMP/delayed-privatization-allocatable-private.f90
@@ -15,30 +15,26 @@ subroutine delayed_privatization_allocatable
 end subroutine
 
 ! CHECK-LABEL: omp.private {type = private}
-! CHECK-SAME: @[[PRIVATIZER_SYM:.*]] : [[TYPE:!fir.ref<!fir.box<!fir.heap<i32>>>]] alloc {
+! CHECK-SAME: @[[PRIVATIZER_SYM:.*]] : [[BOX_TYPE:!fir.box<!fir.heap<i32>>]] init {
 
-! CHECK-NEXT: ^bb0(%[[PRIV_ARG:.*]]: [[TYPE]]):
-
-! CHECK-NEXT:   %[[PRIV_ALLOC:.*]] = fir.alloca !fir.box<!fir.heap<i32>> {bindc_name = "var1", pinned, uniq_name = "_QFdelayed_privatization_allocatableEvar1"}
+! CHECK-NEXT: ^bb0(%[[PRIV_ARG:.*]]: [[TYPE:!fir.ref<!fir.box<!fir.heap<i32>>>]], %[[PRIV_ALLOC:.*]]: [[TYPE]]):
 
 ! CHECK-NEXT:   %[[PRIV_ARG_VAL:.*]] = fir.load %[[PRIV_ARG]] : !fir.ref<!fir.box<!fir.heap<i32>>>
 ! CHECK-NEXT:   %[[PRIV_ARG_BOX:.*]] = fir.box_addr %[[PRIV_ARG_VAL]] : (!fir.box<!fir.heap<i32>>) -> !fir.heap<i32>
 ! CHECK-NEXT:   %[[PRIV_ARG_ADDR:.*]] = fir.convert %[[PRIV_ARG_BOX]] : (!fir.heap<i32>) -> i64
 ! CHECK-NEXT:   %[[C0:.*]] = arith.constant 0 : i64
-! CHECK-NEXT:   %[[ALLOC_COND:.*]] = arith.cmpi ne, %[[PRIV_ARG_ADDR]], %[[C0]] : i64
+! CHECK-NEXT:   %[[ALLOC_COND:.*]] = arith.cmpi eq, %[[PRIV_ARG_ADDR]], %[[C0]] : i64
 
 ! CHECK-NEXT:   fir.if %[[ALLOC_COND]] {
-! CHECK:          %[[PRIV_ALLOCMEM:.*]] = fir.allocmem i32 {fir.must_be_heap = true, uniq_name = "_QFdelayed_privatization_allocatableEvar1.alloc"}
+! CHECK-NEXT:     %[[ZERO_BOX:.*]] = fir.embox %[[PRIV_ARG_BOX]] : (!fir.heap<i32>) -> !fir.box<!fir.heap<i32>>
+! CHECK-NEXT:     fir.store %[[ZERO_BOX]] to %[[PRIV_ALLOC]] : !fir.ref<!fir.box<!fir.heap<i32>>>
+! CHECK-NEXT:   } else {
+! CHECK-NEXT:     %[[PRIV_ALLOCMEM:.*]] = fir.allocmem i32
 ! CHECK-NEXT:     %[[PRIV_ALLOCMEM_BOX:.*]] = fir.embox %[[PRIV_ALLOCMEM]] : (!fir.heap<i32>) -> !fir.box<!fir.heap<i32>>
 ! CHECK-NEXT:     fir.store %[[PRIV_ALLOCMEM_BOX]] to %[[PRIV_ALLOC]] : !fir.ref<!fir.box<!fir.heap<i32>>>
-! CHECK-NEXT:   } else {
-! CHECK-NEXT:     %[[ZERO_BITS:.*]] = fir.zero_bits !fir.heap<i32>
-! CHECK-NEXT:     %[[ZERO_BOX:.*]] = fir.embox %[[ZERO_BITS]] : (!fir.heap<i32>) -> !fir.box<!fir.heap<i32>>
-! CHECK-NEXT:     fir.store %[[ZERO_BOX]] to %[[PRIV_ALLOC]] : !fir.ref<!fir.box<!fir.heap<i32>>>
 ! CHECK-NEXT:   }
 
-! CHECK-NEXT:   %[[PRIV_DECL:.*]]:2 = hlfir.declare %[[PRIV_ALLOC]]
-! CHECK-NEXT:   omp.yield(%[[PRIV_DECL]]#0 : [[TYPE]])
+! CHECK-NEXT:   omp.yield(%[[PRIV_ALLOC]] : [[TYPE]])
 
 ! CHECK-NEXT: } dealloc {
 ! CHECK-NEXT: ^bb0(%[[PRIV_ARG:.*]]: [[TYPE]]):
@@ -50,12 +46,7 @@ subroutine delayed_privatization_allocatable
 ! CHECK-NEXT:   %[[PRIV_NULL_COND:.*]] = arith.cmpi ne, %[[PRIV_ADDR_I64]], %[[C0]] : i64
 
 ! CHECK-NEXT:   fir.if %[[PRIV_NULL_COND]] {
-! CHECK:          %[[PRIV_VAL_2:.*]] = fir.load %[[PRIV_ARG]]
-! CHECK-NEXT:     %[[PRIV_ADDR_2:.*]] = fir.box_addr %[[PRIV_VAL_2]]
-! CHECK-NEXT:     fir.freemem %[[PRIV_ADDR_2]]
-! CHECK-NEXT:     %[[ZEROS:.*]] = fir.zero_bits
-! CHECK-NEXT:     %[[ZEROS_BOX:.*]]  = fir.embox %[[ZEROS]]
-! CHECK-NEXT:     fir.store %[[ZEROS_BOX]] to %[[PRIV_ARG]]
+! CHECK-NEXT:     fir.freemem %[[PRIV_ADDR]]
 ! CHECK-NEXT:   }
 
 ! CHECK-NEXT:   omp.yield
diff --git a/flang/test/Lower/OpenMP/delayed-privatization-array.f90 b/flang/test/Lower/OpenMP/delayed-privatization-array.f90
index 3d641a0d69689..95fa3f9e03052 100644
--- a/flang/test/Lower/OpenMP/delayed-privatization-array.f90
+++ b/flang/test/Lower/OpenMP/delayed-privatization-array.f90
@@ -29,20 +29,28 @@ subroutine delayed_privatization_private_1d(var1, l1, u1)
 end subroutine
 
 ! ONE_DIM-LABEL: omp.private {type = firstprivate}
-! ONE_DIM-SAME: @[[PRIVATIZER_SYM:.*]] : [[TYPE:!fir.box<!fir.array<\?xi32>>]] alloc {
-
-! ONE_DIM-NEXT: ^bb0(%[[PRIV_ARG:.*]]: [[TYPE]]):
-
-! ONE_DIM:   %[[C0:.*]] = arith.constant 0 : index
-! ONE_DIM-NEXT:   %[[DIMS:.*]]:3 = fir.box_dims %[[PRIV_ARG]], %[[C0]] : ([[TYPE]], index) -> (index, index, index)
-! ONE_DIM:   %[[PRIV_ALLOCA:.*]] = fir.alloca !fir.array<{{\?}}xi32>
-! ONE_DIM-NEXT:   %[[SHAPE_SHIFT:.*]] = fir.shape_shift %[[DIMS]]#0, %[[DIMS]]#1 : (index, index) -> !fir.shapeshift<1>
-! ONE_DIM-NEXT:   %[[PRIV_DECL:.*]]:2 = hlfir.declare %[[PRIV_ALLOCA]](%[[SHAPE_SHIFT]]) {uniq_name = "_QFdelayed_privatization_private_1dEvar1"}
-! ONE_DIM-NEXT:  omp.yield(%[[PRIV_DECL]]#0 : [[TYPE]])
+! ONE_DIM-SAME: @[[PRIVATIZER_SYM:.*]] : [[BOX_TYPE:!fir.box<!fir.array<\?xi32>>]] init {
+
+! ONE_DIM-NEXT: ^bb0(%[[PRIV_ARG:.*]]: [[TYPE:!fir.ref<!fir.box<!fir.array<\?xi32>>>]], %[[PRIV_BOX_ALLOC:.*]]: [[TYPE]]):
+
+! ONE_DIM-NEXT:   %[[PRIV_ARG_VAL:.*]] = fir.load %[[PRIV_ARG]]
+! ONE_DIM-NEXT:   %[[C0:.*]] = arith.constant 0 : index
+! ONE_DIM-NEXT:   %[[DIMS:.*]]:3 = fir.box_dims %[[PRIV_ARG_VAL]], %[[C0]]
+! ONE_DIM-NEXT:   %[[SHAPE:.*]] = fir.shape %[[DIMS]]#1
+! ONE_DIM-NEXT:   %[[ARRAY_ALLOC:.*]] = fir.allocmem !fir.array<?xi32>, %[[DIMS]]#1
+! ONE_DIM-NEXT:   %[[TRUE:.*]] = arith.constant true
+! ONE_DIM-NEXT:   %[[DECL:.*]]:2 = hlfir.declare %[[ARRAY_ALLOC]](%[[SHAPE]])
+! ONE_DIM-NEXT:   %[[C0_0:.*]] = arith.constant 0
+! ONE_DIM-NEXT:   %[[DIMS2:.*]]:3 = fir.box_dims %[[PRIV_ARG_VAL]], %[[C0_0]]
+! ONE_DIM-NEXT:   %[[SHAPE_SHIFT:.*]] = fir.shape_shift %[[DIMS2]]#0, %[[DIMS2]]#1
+! ONE_DIM-NEXT:   %[[REBOX:.*]] = fir.rebox %[[DECL]]#0(%[[SHAPE_SHIFT]])
+! ONE_DIM-NEXT:   fir.store %[[REBOX]] to %[[PRIV_BOX_ALLOC]]
+! ONE_DIM-NEXT:   omp.yield(%[[PRIV_BOX_ALLOC]] : [[TYPE]])
 
 ! ONE_DIM-NEXT: } copy {
 ! ONE_DIM-NEXT: ^bb0(%[[PRIV_ORIG_ARG:.*]]: [[TYPE]], %[[PRIV_PRIV_ARG:.*]]: [[TYPE]]):
-! ONE_DIM-NEXT:  hlfir.assign %[[PRIV_ORIG_ARG]] to %[[PRIV_PRIV_ARG]]
+! ONE_DIM-NEXT:  %[[PRIV_ORIG_ARG_VAL:.*]] = fir.load %[[PRIV_ORIG_ARG:.*]] : [[TYPE]]
+! ONE_DIM-NEXT:  hlfir.assign %[[PRIV_ORIG_ARG_VAL]] to %[[PRIV_PRIV_ARG]]
 ! ONE_DIM-NEXT:   omp.yield(%[[PRIV_PRIV_ARG]] : [[TYPE]])
 ! ONE_DIM-NEXT: }
 
@@ -58,24 +66,31 @@ subroutine delayed_privatization_private_2d(var1, l1, u1, l2, u2)
 end subroutine
 
 ! TWO_DIM-LABEL: omp.private {type = firstprivate}
-! TWO_DIM-SAME: @[[PRIVATIZER_SYM:.*]] : [[TYPE:!fir.box<!fir.array<\?x\?xi32>>]] alloc {
-
-! TWO_DIM-NEXT: ^bb0(%[[PRIV_ARG:.*]]: [[TYPE]]):
-! TWO_DIM:        %[[C0:.*]] = arith.constant 0 : index
-! TWO_DIM-NEXT:   %[[DIMS0:.*]]:3 = fir.box_dims %[[PRIV_ARG]], %[[C0]] : ([[TYPE]], index) -> (index, index, index)
+! TWO_DIM-SAME: @[[PRIVATIZER_SYM:.*]] : [[BOX_TYPE:!fir.box<!fir.array<\?x\?xi32>>]] init {
 
+! TWO_DIM-NEXT: ^bb0(%[[PRIV_ARG:.*]]: [[TYPE:!fir.ref<!fir.box<!fir.array<\?x\?xi32>>>]], %[[PRIV_BOX_ALLOC:.*]]: [[TYPE]]):
+! TWO_DIM-NEXT:   %[[PRIV_ARG_VAL:.*]] = fir.load %[[PRIV_ARG]]
+! TWO_DIM-NEXT:   %[[C0:.*]] = arith.constant 0 : index
+! TWO_DIM-NEXT:   %[[DIMS_0:.*]]:3 = fir.box_dims %[[PRIV_ARG_VAL]], %[[C0]]
 ! TWO_DIM-NEXT:   %[[C1:.*]] = arith.constant 1 : index
-! TWO_DIM-NEXT:   %[[DIMS1:.*]]:3 = fir.box_dims %[[PRIV_ARG]], %[[C1]] : ([[TYPE]], index) -> (index, index, index)
-
-! TWO_DIM-NEXT:   %[[PRIV_ALLOCA:.*]] = fir.alloca !fir.array<{{\?}}x{{\?}}xi32>, %[[DIMS0]]#1, %[[DIMS1]]#1 {bindc_name = "var1", pinned, uniq_name = "_QFdelayed_privatization_private_2dEvar1"}
-! TWO_DIM-NEXT:   %[[SHAPE_SHIFT:.*]] = fir.shape_shift %[[DIMS0]]#0, %[[DIMS0]]#1, %[[DIMS1]]#0, %[[DIMS1]]#1 : (index, index, index, index) -> !fir.shapeshift<2>
-
-! TWO_DIM-NEXT:   %[[PRIV_DECL:.*]]:2 = hlfir.declare %[[PRIV_ALLOCA]](%[[SHAPE_SHIFT]]) {uniq_name = "_QFdelayed_privatization_private_2dEvar1"}
-! TWO_DIM-NEXT:  omp.yield(%[[PRIV_DECL]]#0 : [[TYPE]])
+! TWO_DIM-NEXT:   %[[DIMS_1:.*]]:3 = fir.box_dims %[[PRIV_ARG_VAL]], %[[C1]]
+! TWO_DIM-NEXT:   %[[SHAPE:.*]] = fir.shape %[[DIMS_0]]#1, %[[DIMS_1]]#1
+! TWO_DIM-NEXT:   %[[ARRAY_ALLOC:.*]] = fir.allocmem !fir.array<?x?xi32>, %[[DIMS_0]]#1, %[[DIMS_1]]#1
+! TWO_DIM-NEXT:   %[[TRUE:.*]] = arith.constant true
+! TWO_DIM-NEXT:   %[[DECL:.*]]:2 = hlfir.declare %[[ARRAY_ALLOC]](%[[SHAPE]])
+! TWO_DIM-NEXT:   %[[C0_0:.*]] = arith.constant 0
+! TWO_DIM-NEXT:   %[[DIMS2_0:.*]]:3 = fir.box_dims %[[PRIV_ARG_VAL]], %[[C0_0]]
+! TWO_DIM-NEXT:   %[[C1_0:.*]] = arith.constant 1
+! TWO_DIM-NEXT:   %[[DIMS2_1:.*]]:3 = fir.box_dims %[[PRIV_ARG_VAL]], %[[C1_0]]
+! TWO_DIM-NEXT:   %[[SHAPE_SHIFT:.*]] = fir.shape_shift %[[DIMS2_0]]#0, %[[DIMS2_0]]#1, %[[DIMS2_1]]#0, %[[DIMS2_1]]#1
+! TWO_DIM-NEXT:   %[[REBOX:.*]] = fir.rebox %[[DECL]]#0(%[[SHAPE_SHIFT]])
+! TWO_DIM-NEXT:   fir.store %[[REBOX]] to %[[PRIV_BOX_ALLOC]]
+! TWO_DIM-NEXT:   omp.yield(%[[PRIV_BOX_ALLOC]] : [[TYPE]])
 
 ! TWO_DIM-NEXT: } copy {
 ! TWO_DIM-NEXT: ^bb0(%[[PRIV_ORIG_ARG:.*]]: [[TYPE]], %[[PRIV_PRIV_ARG:.*]]: [[TYPE]]):
-! TWO_DIM-NEXT:  hlfir.assign %[[PRIV_ORIG_ARG]] to %[[PRIV_PRIV_ARG]]
+! TWO_DIM-NEXT:  %[[PRIV_ORIG_ARG_VAL:.*]] = fir.load %[[PRIV_ORIG_ARG:.*]] : [[TYPE]]
+! TWO_DIM-NEXT:  hlfir.assign %[[PRIV_ORIG_ARG_VAL]] to %[[PRIV_PRIV_ARG]]
 ! TWO_DIM-NEXT:   omp.yield(%[[PRIV_PRIV_ARG]] : [[TYPE]])
 ! TWO_DIM-NEXT: }
 
@@ -90,11 +105,18 @@ program main
 end program
 
 ! ONE_DIM_DEFAULT_LB-LABEL: omp.private {type = private}
-! ONE_DIM_DEFAULT_LB-SAME: @[[PRIVATIZER_SYM:.*]] : [[TYPE:!fir.ref<!fir.array<10xi32>>]] alloc {
-
-! ONE_DIM_DEFAULT_LB-NEXT: ^bb0(%[[PRIV_ARG:.*]]: [[TYPE]]):
-
-! ONE_DIM_DEFAULT_LB:   %[[C10:.*]] = arith.constant 10 : index
-! ONE_DIM_DEFAULT_LB:   %[[PRIV_ALLOCA:.*]] = fir.alloca !fir.array<10xi32>
-! ONE_DIM_DEFAULT_LB:   %[[SHAPE:.*]] = fir.shape %[[C10]] : (index) -> !fir.shape<1>
-! ONE_DIM_DEFAULT_LB:   hlfir.declare %[[PRIV_ALLOCA]](%[[SHAPE]])
+! ONE_DIM_DEFAULT_LB-SAME: @[[PRIVATIZER_SYM:.*]] : [[BOX_TYPE:!fir.box<!fir.array<10xi32>>]] init {
+
+! ONE_DIM_DEFAULT_LB-NEXT: ^bb0(%[[PRIV_ARG:.*]]: [[TYPE:!fir.ref<!fir.box<!fir.array<10xi32>>>]], %[[PRIV_BOX_ALLOC:.*]]: [[TYPE]]):
+! ONE_DIM_DEFAULT_LB-NEXT:   %[[PRIV_ARG_VAL:.*]] = fir.load %[[PRIV_ARG]]
+! ONE_DIM_DEFAULT_LB-NEXT:   %[[C10:.*]] = arith.constant 10 : index
+! ONE_DIM_DEFAULT_LB-NEXT:   %[[SHAPE:.*]] = fir.shape %[[C10]]
+! ONE_DIM_DEFAULT_LB-NEXT:   %[[ARRAY_ALLOC:.*]] = fir.allocmem !fir.array<10xi32>
+! ONE_DIM_DEFAULT_LB-NEXT:   %[[TRUE:.*]] = arith.constant true
+! ONE_DIM_DEFAULT_LB-NEXT:   %[[DECL:.*]]:2 = hlfir.declare %[[ARRAY_ALLOC]](%[[SHAPE]])
+! ONE_DIM_DEFAULT_LB-NEXT:   %[[C0_0:.*]] = arith.constant 0
+! ONE_DIM_DEFAULT_LB-NEXT:   %[[DIMS2:.*]]:3 = fir.box_dims %[[PRIV_ARG_VAL]], %[[C0_0]]
+! ONE_DIM_DEFAULT_LB-NEXT:   %[[SHAPE_SHIFT:.*]] = fir.shape_shift %[[DIMS2]]#0, %[[DIMS2]]#1
+! ONE_DIM_DEFAULT_LB-NEXT:   %[[EMBOX:.*]] = fir.embox %[[DECL]]#0(%[[SHAPE_SHIFT]])
+! ONE_DIM_DEFAULT_LB-NEXT:   fir.store %[[EMBOX]] to %[[PRIV_BOX_ALLOC]]
+! ONE_DIM_DEFAULT_LB-NEXT:   omp.yield(%[[PRIV_BOX_ALLOC]] : [[TYPE]])
diff --git a/flang/test/Lower/OpenMP/delayed-privatization-character-array.f90 b/flang/test/Lower/OpenMP/delayed-privatization-character-array.f90
index 9a9d0c01212c8..4c7287283c7ad 100644
--- a/flang/test/Lower/OpenMP/delayed-privatization-character-array.f90
+++ b/flang/test/Lower/OpenMP/delayed-privatization-character-array.f90
@@ -23,19 +23,14 @@ subroutine delayed_privatization_character_array_static_len(var1)
 end subroutine
 
 ! STATIC_LEN-LABEL: omp.private {type = firstprivate}
-! STATIC_LEN-SAME: @[[PRIVATIZER_SYM:.*]] : [[TYPE:!fir.ref<!fir.array<5x!fir.char<1,10>>>]] alloc {
+! STATIC_LEN-SAME: @[[PRIVATIZER_SYM:.*]] : [[TYPE:!fir.box<!fir.array<5x!fir.char<1,10>>>]] init {
 
-! STATIC_LEN-NEXT: ^bb0(%[[PRIV_ARG:.*]]: [[TYPE]]):
-! STATIC_LEN-DAG:    %[[C5:.*]] = arith.constant 5 : index
-! STATIC_LEN-DAG:    %[[C10:.*]] = arith.constant 10 : index
-! STATIC_LEN-NEXT:   %[[PRIV_ALLOC:.*]] = fir.alloca !fir.array<5x!fir.char<1,10>>
-! STATIC_LEN-NEXT:   %[[ARRAY_SHAPE:.*]] = fir.shape %[[C5]]
-! STATIC_LEN-NEXT:   %[[PRIV_DECL:.*]]:2 = hlfir.declare %[[PRIV_ALLOC]](%[[ARRAY_SHAPE]]) typeparams %[[C10]]
-! STATIC_LEN-NEXT:   omp.yield(%[[PRIV_DECL]]#0
-
-! STATIC_LEN-NEXT: } copy {
-! STATIC_LEN-NEXT: ^bb0(%[[PRIV_ORIG_ARG:.*]]: [[TYPE]], %[[PRIV_PRIV_ARG:.*]]: [[TYPE]]):
-! STATIC_LEN-NEXT:   hlfir.assign %[[PRIV_ORIG_ARG]] to %[[PRIV_PRIV_ARG]]
+! STATIC_LEN-NEXT: ^bb0(%[[MOLD_REF:.*]]: !fir.ref<[[TYPE]]>, %[[ALLOC:.*]]: !fir.ref<[[TYPE]]>):
+!                    [init region]
+! STATIC_LEN:      } copy {
+! STATIC_LEN-NEXT: ^bb0(%[[PRIV_ORIG_ARG:.*]]: !fir.ref<[[TYPE]]>, %[[PRIV_PRIV_ARG:.*]]: !fir.ref<[[TYPE]]>):
+! STATIC_LEN-NEXT:   %[[ORIG:.*]] = fir.load %[[PRIV_ORIG_ARG]] : !fir.ref<[[TYPE]]>
+! STATIC_LEN-NEXT:   hlfir.assign %[[ORIG]] to %[[PRIV_PRIV_ARG]]
 
 ! STATIC_LEN-NEXT:   omp.yield(%[[PRIV_PRIV_ARG]]
 ! STATIC_LEN-NEXT: }
@@ -53,15 +48,5 @@ subroutine delayed_privatization_character_array_dynamic_len(var1, char_len, arr
 end subroutine
 
 ! DYN_LEN-LABEL: omp.private {type = private}
-! DYN_LEN-SAME: @[[PRIVATIZER_SYM:.*]] : [[TYPE:!fir.box<!fir.array<\?x!fir.char<1,\?>>>]] alloc {
-
-! DYN_LEN-NEXT: ^bb0(%[[PRIV_ARG:.*]]: [[TYPE]]):
-
-! DYN_LEN:        %[[C0:.*]] = arith.constant 0 : index
-! DYN_LEN-NEXT:   %[[BOX_DIM:.*]]:3 = fir.box_dims %[[PRIV_ARG]], %[[C0]]
-! DYN_LEN:        %[[CHAR_LEN:.*]] = fir.box_elesize %[[PRIV_ARG]]
-! DYN_LEN-NEXT:   %[[PRIV_ALLOC:.*]] = fir.alloca !fir.array<?x!fir.char<1,?>>(%[[CHAR_LEN]] : index)
-! DYN_LEN-NEXT:   %[[ARRAY_SHAPE:.*]] = fir.shape
-! DYN_LEN-NEXT:   %[[PRIV_DECL:.*]]:2 = hlfir.declare %[[PRIV_ALLOC]](%[[ARRAY_SHAPE]]) typeparams %[[CHAR_LEN]]
-
-! DYN_LEN-NEXT:   omp.yield(%[[PRIV_DECL]]#0
+! DYN_LEN-SAME: @[[PRIVATIZER_SYM:.*]] : [[TYPE:!fir.box<!fir.array<\?x!fir.char<1,\?>>>]] init {
+! DYN_LEN-NEXT: ^bb0(%[[MOLD_ARG:.*]]: !fir.ref<!fir.box<!fir.array<?x!fir.char<1,?>>>>, %[[ALLOC_ARG:.*]]: !fir.ref<!fir.box<!fir.array<?x!fir.char<1,?>>>>)
diff --git a/flang/test/Lower/OpenMP/delayed-privatization-character.f90 b/flang/test/Lower/OpenMP/delayed-privatization-character.f90
index db678ab13bbe6..3d1a312963371 100644
--- a/flang/test/Lower/OpenMP/delayed-privatization-character.f90
+++ b/flang/test/Lower/OpenMP/delayed-privatization-character.f90
@@ -24,13 +24,13 @@ subroutine delayed_privatization_character(var1, l)
 end subroutine
 
 ! DYN_LEN-LABEL: omp.private {type = firstprivate}
-! DYN_LEN-SAME: @[[PRIVATIZER_SYM:.*]] : [[TYPE:!fir.boxchar<1>]] alloc {
+! DYN_LEN-SAME: @[[PRIVATIZER_SYM:.*]] : [[TYPE:!fir.boxchar<1>]] init {
 
-! DYN_LEN-NEXT: ^bb0(%[[PRIV_ARG:.*]]: [[TYPE]]):
+! DYN_LEN-NEXT: ^bb0(%[[PRIV_ARG:.*]]: [[TYPE]], %[[ALLOC_ARG:.*]]: [[TYPE]]):
 ! DYN_LEN-NEXT:   %[[UNBOX:.*]]:2 = fir.unboxchar %[[PRIV_ARG]]
-! DYN_LEN:        %[[PRIV_ALLOC:.*]] = fir.alloca !fir.char<1,?>(%[[UNBOX]]#1 : index)
-! DYN_LEN-NEXT:   %[[PRIV_DECL:.*]]:2 = hlfir.declare %[[PRIV_ALLOC]] typeparams %[[UNBOX]]#1
-! DYN_LEN-NEXT:   omp.yield(%[[PRIV_DECL]]#0 : !fir.boxchar<1>)
+! DYN_LEN-NEXT:   %[[PRIV_ALLOC:.*]] = fir.allocmem !fir.char<1,?>(%[[UNBOX]]#1 : index)
+! DYN_LEN-NEXT:   %[[EMBOXCHAR:.*]] = fir.emboxchar %[[PRIV_ALLOC]], %[[UNBOX]]#1
+! DYN_LEN:        omp.yield(%[[EMBOXCHAR]] : !fir.boxchar<1>)
 
 ! DYN_LEN-NEXT: } copy {
 ! DYN_LEN-NEXT: ^bb0(%[[PRIV_ORIG_ARG:.*]]: [[TYPE]], %[[PRIV_PRIV_ARG:.*]]: [[TYPE]]):
@@ -51,9 +51,4 @@ subroutine delayed_privatization_character_static_len(var1)
 end subroutine
 
 ! STATIC_LEN-LABEL: omp.private {type = private}
-! STATIC_LEN-SAME: @[[PRIVATIZER_SYM:.*]] : [[TYPE:!fir.ref<!fir.char<1,10>>]] alloc {
-
-! STATIC_LEN-NEXT: ^bb0(%[[PRIV_ARG:.*]]: [[TYPE]]):
-! STATIC_LEN-NEXT:   %[[C10:.*]] = arith.constant 10 : index
-! STATIC_LEN-NEXT:   %[[PRIV_ALLOC:.*]] = fir.alloca !fir.char<1,10>
-! STATIC_LEN-NEXT:   %[[PRIV_DECL:.*]]:2 = hlfir.declare %[[PRIV_ALLOC]] typeparams %[[C10]]
+! STATIC_LEN-SAME: @[[PRIVATIZER_SYM:.*]] : [[TYPE:!fir.char<1,10>]]
diff --git a/flang/test/Lower/OpenMP/delayed-privatization-default-init.f90 b/flang/test/Lower/OpenMP/delayed-privatization-default-init.f90
index 022b592db74b8..87d4605217a8a 100644
--- a/flang/test/Lower/OpenMP/delayed-privatization-default-init.f90
+++ b/flang/test/Lower/OpenMP/delayed-privatization-default-init.f90
@@ -29,19 +29,16 @@ subroutine delayed_privatization_default_init_firstprivate
   !$omp end parallel
 end subroutine
 
-! CHECK-LABEL:   omp.private {type = firstprivate} @_QFdelayed_privatization_default_init_firstprivateEa_firstprivate_ref_rec__QFdelayed_privatization_default_init_firstprivateTt : !fir.ref<!fir.type<_QFdelayed_privatization_default_init_firstprivateTt{i:i32}>> alloc {
-! CHECK:         ^bb0(%[[VAL_0:.*]]: !fir.ref<!fir.type<_QFdelayed_privatization_default_init_firstprivateTt{i:i32}>>):
-! CHECK:           %[[VAL_1:.*]] = fir.alloca !fir.type<_QFdelayed_privatization_default_init_firstprivateTt{i:i32}> {bindc_name = "a", pinned, uniq_name = "_QFdelayed_privatization_default_init_firstprivateEa"}
-! CHECK-NEXT:      %[[VAL_9:.*]]:2 = hlfir.declare %[[VAL_1]] {uniq_name = "_QFdelayed_privatization_default_init_firstprivateEa"} : (!fir.ref<!fir.type<_QFdelayed_privatization_default_init_firstprivateTt{i:i32}>>) -> (!fir.ref<!fir.type<_QFdelayed_privatization_default_init_firstprivateTt{i:i32}>>, !fir.ref<!fir.type<_QFdelayed_privatization_default_init_firstprivateTt{i:i32}>>)
-! CHECK:           omp.yield(%[[VAL_9]]#0 : !fir.ref<!fir.type<_QFdelayed_privatization_default_init_firstprivateTt{i:i32}>>)
-! CHECK:   }
+! CHECK-LABEL:   omp.private {type = firstprivate}
+! CHECK-SAME:        @_QFdelayed_privatization_default_init_firstprivateEa_firstprivate_rec__QFdelayed_privatization_default_init_firstprivateTt :
+! CHECK-SAME:        [[TYPE:!fir.type<_QFdelayed_privatization_default_init_firstprivateTt{i:i32}>]] copy {
 
-! CHECK-LABEL:   omp.private {type = private} @_QFdelayed_privatization_default_initEa_private_ref_rec__QFdelayed_privatization_default_initTt : !fir.ref<!fir.type<_QFdelayed_privatization_default_initTt{i:i32}>> alloc {
-! CHECK:         ^bb0(%[[VAL_0:.*]]: !fir.ref<!fir.type<_QFdelayed_privatization_default_initTt{i:i32}>>):
-! CHECK:           %[[VAL_1:.*]] = fir.alloca !fir.type<_QFdelayed_privatization_default_initTt{i:i32}> {bindc_name = "a", pinned, uniq_name = "_QFdelayed_privatization_default_initEa"}
+! CHECK-LABEL:   omp.private {type = private}
+! CHECK-SAME:        @_QFdelayed_privatization_default_initEa_private_rec__QFdelayed_privatization_default_initTt :
+! CHECK-SAME:        [[TYPE:!fir.type<_QFdelayed_privatization_default_initTt{i:i32}>]] init {
+! CHECK:         ^bb0(%[[VAL_0:.*]]: !fir.ref<[[TYPE]]>, %[[VAL_1:.*]]: !fir.ref<[[TYPE]]>):
 ! CHECK:           %[[VAL_2:.*]] = fir.embox %[[VAL_1]] : (!fir.ref<!fir.type<_QFdelayed_privatization_default_initTt{i:i32}>>) -> !fir.box<!fir.type<_QFdelayed_privatization_default_initTt{i:i32}>>
 ! CHECK:           %[[VAL_6:.*]] = fir.convert %[[VAL_2]] : (!fir.box<!fir.type<_QFdelayed_privatization_default_initTt{i:i32}>>) -> !fir.box<none>
 ! CHECK:           fir.call @_FortranAInitialize(%[[VAL_6]],{{.*}}
-! CHECK-NEXT:      %[[VAL_9:.*]]:2 = hlfir.declare %[[VAL_1]] {uniq_name = "_QFdelayed_privatization_default_initEa"} : (!fir.ref<!fir.type<_QFdelayed_privatization_default_initTt{i:i32}>>) -> (!fir.ref<!fir.type<_QFdelayed_privatization_default_initTt{i:i32}>>, !fir.ref<!fir.type<_QFdelayed_privatization_default_initTt{i:i32}>>)
-! CHECK:           omp.yield(%[[VAL_9]]#0 : !fir.ref<!fir.type<_QFdelayed_privatization_default_initTt{i:i32}>>)
+! CHECK:           omp.yield(%[[VAL_1]] : !fir.ref<!fir.type<_QFdelayed_privatization_default_initTt{i:i32}>>)
 ! CHECK:   }
diff --git a/flang/test/Lower/OpenMP/delayed-privatization-firstprivate.f90 b/flang/test/Lower/OpenMP/delayed-privatization-firstprivate.f90
index 119f77ea26626..904ea783ad5b4 100644
--- a/flang/test/Lower/OpenMP/delayed-privatization-firstprivate.f90
+++ b/flang/test/Lower/OpenMP/delayed-privatization-firstprivate.f90
@@ -15,12 +15,7 @@ subroutine delayed_privatization_firstprivate
 end subroutine
 
 ! CHECK-LABEL: omp.private {type = firstprivate}
-! CHECK-SAME: @[[VAR1_PRIVATIZER_SYM:.*]] : !fir.ref<i32> alloc {
-! CHECK-NEXT: ^bb0(%[[PRIV_ARG:.*]]: !fir.ref<i32>):
-! CHECK-NEXT:   %[[PRIV_ALLOC:.*]] = fir.alloca i32 {bindc_name = "var1", pinned, uniq_name = "_QFdelayed_privatization_firstprivateEvar1"}
-! CHECK-NEXT:   %[[PRIV_DECL:.*]]:2 = hlfir.declare %[[PRIV_ALLOC]] {uniq_name = "_QFdelayed_privatization_firstprivateEvar1"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-! CHECK-NEXT:   omp.yield(%[[PRIV_DECL]]#0 : !fir.ref<i32>)
-! CHECK: } copy {
+! CHECK-SAME: @[[VAR1_PRIVATIZER_SYM:.*]] : i32 copy {
 ! CHECK: ^bb0(%[[PRIV_ORIG_ARG:.*]]: !fir.ref<i32>, %[[PRIV_PRIV_ARG:.*]]: !fir.ref<i32>):
 ! CHECK:    %[[ORIG_VAL:.*]] = fir.load %[[PRIV_ORIG_ARG]] : !fir.ref<i32>
 ! CHECK:    hlfir.assign %[[ORIG_VAL]] to %[[PRIV_PRIV_ARG]] : i32, !fir.ref<i32>
diff --git a/flang/test/Lower/OpenMP/delayed-privatization-lastprivate-of-private.f90 b/flang/test/Lower/OpenMP/delayed-privatization-lastprivate-of-private.f90
new file mode 100644
index 0000000000000..d188f6e28b560
--- /dev/null
+++ b/flang/test/Lower/OpenMP/delayed-privatization-lastprivate-of-private.f90
@@ -0,0 +1,22 @@
+! RUN: %flang_fc1 -emit-hlfir -fopenmp -o - %s | FileCheck %s
+! RUN: bbc -emit-hlfir -fopenmp -o - %s | FileCheck %s
+
+! Check that we can lower this without crashing
+
+! CHECK: func.func @_QPlastprivate_of_private
+subroutine lastprivate_of_private(a)
+  real :: a(100)
+  integer i
+  ! CHECK: omp.parallel private({{.*}}) {
+  !$omp parallel private(a)
+    ! CHECK: omp.parallel {
+    !$omp parallel shared(a)
+    ! CHECK: omp.wsloop private({{.*}}) {
+    !$omp do lastprivate(a)
+    ! CHECK: omp.loop_nest
+      do i=1,100
+        a(i) = 1.0
+      end do
+    !$omp end parallel
+  !$omp end parallel
+end subroutine
diff --git a/flang/test/Lower/OpenMP/delayed-privatization-pointer.f90 b/flang/test/Lower/OpenMP/delayed-privatization-pointer.f90
index c96b0b49fd530..f39ac9199e8bd 100644
--- a/flang/test/Lower/OpenMP/delayed-privatization-pointer.f90
+++ b/flang/test/Lower/OpenMP/delayed-privatization-pointer.f90
@@ -14,21 +14,41 @@ subroutine delayed_privatization_pointer
 !$omp end parallel
 end subroutine
 
-! CHECK-LABEL: omp.private {type = firstprivate}
-! CHECK-SAME: @[[PRIVATIZER_SYM:.*]] : [[TYPE:!fir.ref<!fir.box<!fir.ptr<i32>>>]] alloc {
+subroutine delayed_privatization_lenparams(length)
+  integer, intent(in) :: length
+  character(length), pointer :: var
 
-! CHECK-NEXT: ^bb0(%[[PRIV_ARG:.*]]: [[TYPE]]):
+  !$omp parallel firstprivate(var)
+    var = 'a'
+  !$omp end parallel
+end subroutine
 
-! CHECK-NEXT:   %[[PRIV_ALLOC:.*]] = fir.alloca !fir.box<!fir.ptr<i32>> {bindc_name = "var1", pinned, uniq_name = "_QFdelayed_privatization_pointerEvar1"}
+! CHECK-LABEL: omp.private {type = firstprivate}
+! CHECK-SAME: @[[PRIVATIZER_SYM2:.*]] : [[TYPE:!fir.box<!fir.ptr<!fir.char<1,\?>>>]] init {
+! CHECK-NEXT: ^bb0(%[[PRIV_ARG:.*]]: !fir.ref<[[TYPE]]>, %[[PRIV_ALLOC:.*]]: !fir.ref<[[TYPE]]>):
+! CHECK-NEXT:   %[[ARG:.*]] = fir.load %[[PRIV_ARG]]
+! CHECK-NEXT:   %[[SIZE:.*]] = fir.box_elesize %[[ARG]]
+! CHECK-NEXT:   %[[NULL:.*]] = fir.zero_bits !fir.ptr<!fir.char<1,?>>
+! CHECK-NEXT:   %[[INIT:.*]] = fir.embox %[[NULL]] typeparams %[[SIZE]]
+! CHECK-NEXT:   fir.store %[[INIT]] to %[[PRIV_ALLOC]]
+! CHECK-NEXT:   omp.yield(%[[PRIV_ALLOC]] : !fir.ref<[[TYPE]]>)
+! CHECK-NEXT: } copy {
+! CHECK: ^bb0(%[[PRIV_ORIG_ARG:.*]]: !fir.ref<[[TYPE]]>, %[[PRIV_PRIV_ARG:.*]]: !fir.ref<[[TYPE]]>):
+! CHECK-NEXT:    %[[ORIG_BASE_VAL:.*]] = fir.load %[[PRIV_ORIG_ARG]]
+! CHECK-NEXT:   fir.store %[[ORIG_BASE_VAL]] to %[[PRIV_PRIV_ARG]]
+! CHECK-NEXT:   omp.yield(%[[PRIV_PRIV_ARG]] : !fir.ref<[[TYPE]]>)
+! CHECK-NEXT: }
+
+! CHECK-LABEL: omp.private {type = firstprivate}
+! CHECK-SAME: @[[PRIVATIZER_SYM:.*]] : [[TYPE:!fir.box<!fir.ptr<i32>>]] init {
+! CHECK-NEXT: ^bb0(%[[PRIV_ARG:.*]]: !fir.ref<[[TYPE]]>, %[[PRIV_ALLOC:.*]]: !fir.ref<[[TYPE]]>):
 ! CHECK-NEXT:   %[[NULL:.*]] = fir.zero_bits !fir.ptr<i32>
 ! CHECK-NEXT:   %[[INIT:.*]] = fir.embox %[[NULL]] : (!fir.ptr<i32>) -> !fir.box<!fir.ptr<i32>>
 ! CHECK-NEXT:   fir.store %[[INIT]] to %[[PRIV_ALLOC]] : !fir.ref<!fir.box<!fir.ptr<i32>>>
-! CHECK-NEXT:   %[[PRIV_DECL:.*]]:2 = hlfir.declare %[[PRIV_ALLOC]]
-! CHECK-NEXT:   omp.yield(%[[PRIV_DECL]]#0 : [[TYPE]])
-
+! CHECK-NEXT:   omp.yield(%[[PRIV_ALLOC]] : !fir.ref<[[TYPE]]>)
 ! CHECK-NEXT: } copy {
-! CHECK: ^bb0(%[[PRIV_ORIG_ARG:.*]]: [[TYPE]], %[[PRIV_PRIV_ARG:.*]]: [[TYPE]]):
+! CHECK: ^bb0(%[[PRIV_ORIG_ARG:.*]]: !fir.ref<[[TYPE]]>, %[[PRIV_PRIV_ARG:.*]]: !fir.ref<[[TYPE]]>):
 ! CHECK-NEXT:    %[[ORIG_BASE_VAL:.*]] = fir.load %[[PRIV_ORIG_ARG]]
- ! CHECK-NEXT:   fir.store %[[ORIG_BASE_VAL]] to %[[PRIV_PRIV_ARG]] : !fir.ref<!fir.box<!fir.ptr<i32>>>
-! CHECK-NEXT:   omp.yield(%[[PRIV_PRIV_ARG]] : [[TYPE]])
+! CHECK-NEXT:   fir.store %[[ORIG_BASE_VAL]] to %[[PRIV_PRIV_ARG]] : !fir.ref<!fir.box<!fir.ptr<i32>>>
+! CHECK-NEXT:   omp.yield(%[[PRIV_PRIV_ARG]] : !fir.ref<[[TYPE]]>)
 ! CHECK-NEXT: }
diff --git a/flang/test/Lower/OpenMP/delayed-privatization-private-firstprivate.f90 b/flang/test/Lower/OpenMP/delayed-privatization-private-firstprivate.f90
index 7d202f46c09d3..d961210dcbc38 100644
--- a/flang/test/Lower/OpenMP/delayed-privatization-private-firstprivate.f90
+++ b/flang/test/Lower/OpenMP/delayed-privatization-private-firstprivate.f90
@@ -17,13 +17,11 @@ subroutine delayed_privatization_private_firstprivate
 end subroutine
 
 ! CHECK-LABEL: omp.private {type = firstprivate}
-! CHECK-SAME: @[[VAR2_PRIVATIZER_SYM:.*]] : !fir.ref<i32> alloc {
-! CHECK: } copy {
+! CHECK-SAME: @[[VAR2_PRIVATIZER_SYM:.*]] : i32 copy {
 ! CHECK: }
 
 ! CHECK-LABEL: omp.private {type = private}
-! CHECK-SAME: @[[VAR1_PRIVATIZER_SYM:.*]] : !fir.ref<i32> alloc {
-! CHECK: }
+! CHECK-SAME: @[[VAR1_PRIVATIZER_SYM:.*]] : i32
 
 ! CHECK-LABEL: func.func @_QPdelayed_privatization_private_firstprivate() {
 ! CHECK:  %[[VAR1_ALLOC:.*]] = fir.alloca i32 {bindc_name = "var1"
diff --git a/flang/test/Lower/OpenMP/delayed-privatization-private.f90 b/flang/test/Lower/OpenMP/delayed-privatization-private.f90
index 7208521bcd77e..69c362e4828bf 100644
--- a/flang/test/Lower/OpenMP/delayed-privatization-private.f90
+++ b/flang/test/Lower/OpenMP/delayed-privatization-private.f90
@@ -15,12 +15,8 @@ subroutine delayed_privatization_private
 end subroutine
 
 ! CHECK-LABEL: omp.private {type = private}
-! CHECK-SAME: @[[PRIVATIZER_SYM:.*]] : !fir.ref<i32> alloc {
-! CHECK-NEXT: ^bb0(%[[PRIV_ARG:.*]]: !fir.ref<i32>):
-! CHECK-NEXT:   %[[PRIV_ALLOC:.*]] = fir.alloca i32 {bindc_name = "var1", pinned, uniq_name = "_QFdelayed_privatization_privateEvar1"}
-! CHECK-NEXT:   %[[PRIV_DECL:.*]]:2 = hlfir.declare %[[PRIV_ALLOC]] {uniq_name = "_QFdelayed_privatization_privateEvar1"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-! CHECK-NEXT:   omp.yield(%[[PRIV_DECL]]#0 : !fir.ref<i32>)
-! CHECK-NOT: } copy {
+! CHECK-SAME: @[[PRIVATIZER_SYM:.*]] : i32
+! CHECK-NOT: copy {
 
 ! CHECK-LABEL: @_QPdelayed_privatization_private
 ! CHECK: %[[ORIG_ALLOC:.*]] = fir.alloca i32 {bindc_name = "var1", uniq_name = "_QFdelayed_privatization_privateEvar1"}
diff --git a/flang/test/Lower/OpenMP/delayed-privatization-reduction-byref.f90 b/flang/test/Lower/OpenMP/delayed-privatization-reduction-byref.f90
index 6c00bb23f15b9..f463f2b4630ae 100644
--- a/flang/test/Lower/OpenMP/delayed-privatization-reduction-byref.f90
+++ b/flang/test/Lower/OpenMP/delayed-privatization-reduction-byref.f90
@@ -19,7 +19,7 @@ subroutine red_and_delayed_private
 end subroutine
 
 ! CHECK-LABEL: omp.private {type = private}
-! CHECK-SAME: @[[PRIVATIZER_SYM:.*]] : !fir.ref<i32> alloc {
+! CHECK-SAME: @[[PRIVATIZER_SYM:.*]] : i32
 
 ! CHECK-LABEL: omp.declare_reduction
 ! CHECK-SAME: @[[REDUCTION_SYM:.*]] : !fir.ref<i32> alloc
diff --git a/flang/test/Lower/OpenMP/delayed-privatization-reduction.f90 b/flang/test/Lower/OpenMP/delayed-privatization-reduction.f90
index 38139e52ce95c..a1ddbc30d6e46 100644
--- a/flang/test/Lower/OpenMP/delayed-privatization-reduction.f90
+++ b/flang/test/Lower/OpenMP/delayed-privatization-reduction.f90
@@ -22,7 +22,7 @@ subroutine red_and_delayed_private
 end subroutine
 
 ! CHECK-LABEL: omp.private {type = private}
-! CHECK-SAME: @[[PRIVATIZER_SYM:.*]] : !fir.ref<i32> alloc {
+! CHECK-SAME: @[[PRIVATIZER_SYM:.*]] : i32
 
 ! CHECK-LABEL: omp.declare_reduction
 ! CHECK-SAME: @[[REDUCTION_SYM:.*]] : i32 init
diff --git a/flang/test/Lower/OpenMP/derived-type-allocatable.f90 b/flang/test/Lower/OpenMP/derived-type-allocatable.f90
index 1d6e22212eedd..81ede56d7f7ed 100644
--- a/flang/test/Lower/OpenMP/derived-type-allocatable.f90
+++ b/flang/test/Lower/OpenMP/derived-type-allocatable.f90
@@ -13,32 +13,55 @@ module m1
 
 contains
 
+!CHECK-LABEL: omp.private {type = private} @_QMm1Ftest_class_allocatable_array
+!CHECK:       fir.call @_FortranAInitialize
+!CHECK-NOT:   omp.barrier
+!CHECK:       omp.yield
+
+!CHECK-LABEL: omp.private {type = private} @_QMm1Ftest_class_allocatable
+!CHECK:       fir.call @_FortranAInitialize
+!CHECK-NOT:   omp.barrier
+!CHECK:       omp.yield
+
+!CHECK-LABEL: omp.private {type = private} @_QMm1Ftest_allocatable
+!CHECK:       fir.call @_FortranAInitialize
+!CHECK-NOT:   omp.barrier
+!CHECK:       omp.yield
+
 !CHECK-LABEL: omp.private {type = private} @_QMm1Ftest_pointer
 !CHECK-NOT:   fir.call @_FortranAInitializeClone
+!CHECK-NOT:   omp.barrier
 !CHECK:       omp.yield
 
 !CHECK-LABEL: omp.private {type = private} @_QMm1Ftest_nested
 !CHECK:       fir.call @_FortranAInitializeClone
-!CHECK-NEXT:  omp.yield
+!CHECK-NOT:   omp.barrier
+!CHECK:       omp.yield
 
 !CHECK-LABEL: omp.private {type = private} @_QMm1Ftest_array_of_allocs
 !CHECK:       fir.call @_FortranAInitializeClone
-!CHECK-NEXT:  omp.yield
+!CHECK-NOT:   omp.barrier
+!CHECK:       omp.yield
 !CHECK:       } dealloc {
 !CHECK:       fir.call @_FortranAAllocatableDeallocate
 !CHECK:       omp.yield
 
 !CHECK-LABEL: omp.private {type = firstprivate} @_QMm1Ftest_array
+!CHECK:       fir.call @_FortranAInitialize(
 !CHECK-NOT:   fir.call @_FortranAInitializeClone
+!CHECK-NOT:   omp.barrier
 !CHECK:       omp.yield
 
 !CHECK-LABEL: omp.private {type = private} @_QMm1Ftest_array
+!CHECK:       fir.call @_FortranAInitialize(
 !CHECK:       fir.call @_FortranAInitializeClone
-!CHECK-NEXT:  omp.yield
+!CHECK-NOT:   omp.barrier
+!CHECK:       omp.yield
 
 !CHECK-LABEL: omp.private {type = private} @_QMm1Ftest_scalar
 !CHECK:       fir.call @_FortranAInitializeClone
-!CHECK-NEXT:  omp.yield
+!CHECK-NOT:   omp.barrier
+!CHECK:       omp.yield
 
   subroutine test_scalar()
     type(x) :: v
@@ -105,4 +128,34 @@ subroutine test_pointer()
     !$omp parallel private(ptr)
     !$omp end parallel
   end subroutine
+
+  subroutine test_allocatable()
+    type needs_init
+      integer :: i = 1
+    end type
+    type(needs_init), allocatable :: a
+
+    !$omp parallel private(a)
+    !$omp end parallel
+  end subroutine
+
+  subroutine test_class_allocatable()
+    type needs_init
+      integer :: i = 1
+    end type
+    class(needs_init), allocatable :: a
+
+    !$omp parallel private(a)
+    !$omp end parallel
+  end subroutine
+
+  subroutine test_class_allocatable_array()
+    type needs_init
+      integer :: i = 1
+    end type
+    class(needs_init), allocatable :: a(:)
+
+    !$omp parallel private(a)
+    !$omp end parallel
+  end subroutine
 end module
diff --git a/flang/test/Lower/OpenMP/firstprivate-alloc-comp.f90 b/flang/test/Lower/OpenMP/firstprivate-alloc-comp.f90
index 2453fe2c5208b..4d0a2a0b90243 100644
--- a/flang/test/Lower/OpenMP/firstprivate-alloc-comp.f90
+++ b/flang/test/Lower/OpenMP/firstprivate-alloc-comp.f90
@@ -13,7 +13,7 @@ subroutine firstprivate_alloc_comp
 
   call firstprivate_alloc_comp()
 end
-! CHECK-LABEL:   omp.private {type = firstprivate} @_QFfirstprivate_alloc_compEx_firstprivate_ref_rec__QFfirstprivate_alloc_compTt1 : !fir.ref<!fir.type<_QFfirstprivate_alloc_compTt1{c:!fir.box<!fir.heap<!fir.array<?xi32>>>}>> alloc {
+! CHECK-LABEL:   omp.private {type = firstprivate} @_QFfirstprivate_alloc_compEx_firstprivate_rec__QFfirstprivate_alloc_compTt1 : !fir.type<_QFfirstprivate_alloc_compTt1{c:!fir.box<!fir.heap<!fir.array<?xi32>>>}> init {
 ! CHECK:     fir.call @_FortranAInitialize(
 ! CHECK:   } copy {
 ! ...
diff --git a/flang/test/Lower/OpenMP/hlfir-wsloop.f90 b/flang/test/Lower/OpenMP/hlfir-wsloop.f90
index f7b0ba681efeb..786ab916d000c 100644
--- a/flang/test/Lower/OpenMP/hlfir-wsloop.f90
+++ b/flang/test/Lower/OpenMP/hlfir-wsloop.f90
@@ -10,12 +10,11 @@ subroutine simple_loop
   ! CHECK-DAG:     %[[WS_END:.*]] = arith.constant 9 : i32
   ! CHECK:  omp.parallel
   !$OMP PARALLEL
-  ! CHECK-DAG:     %[[ALLOCA_IV:.*]] = fir.alloca i32 {{{.*}}, pinned, {{.*}}}
-  ! CHECK:         %[[IV:.*]]    = fir.declare %[[ALLOCA_IV]] {uniq_name = "_QFsimple_loopEi"} : (!fir.ref<i32>) -> !fir.ref<i32>
-  ! CHECK:         omp.wsloop {
+  ! CHECK:         omp.wsloop private(@{{.*}} %{{.*}} -> %[[ALLOCA_IV:.*]] : !fir.ref<i32>) {
   ! CHECK-NEXT:      omp.loop_nest (%[[I:.*]]) : i32 = (%[[WS_ST]]) to (%[[WS_END]]) inclusive step (%[[WS_ST]]) {
   !$OMP DO
   do i=1, 9
+  ! CHECK:         %[[IV:.*]]    = fir.declare %[[ALLOCA_IV]] {uniq_name = "_QFsimple_loopEi"} : (!fir.ref<i32>) -> !fir.ref<i32>
   ! CHECK:             fir.store %[[I]] to %[[IV:.*]] : !fir.ref<i32>
   ! CHECK:             %[[LOAD_IV:.*]] = fir.load %[[IV]] : !fir.ref<i32>
   ! CHECK:             fir.call @_FortranAioOutputInteger32({{.*}}, %[[LOAD_IV]]) {{.*}}: (!fir.ref<i8>, i32) -> i1
diff --git a/flang/test/Lower/OpenMP/implicit-dsa.f90 b/flang/test/Lower/OpenMP/implicit-dsa.f90
index a1912a46f9ae7..f0f149bb415b0 100644
--- a/flang/test/Lower/OpenMP/implicit-dsa.f90
+++ b/flang/test/Lower/OpenMP/implicit-dsa.f90
@@ -6,99 +6,82 @@
 ! Privatizers
 
 ! CHECK-LABEL: omp.private
-! CHECK-SAME:      {type = private} @[[TEST6_Y_PRIV:.*]] : !fir.ref<i32>
-! CHECK:         fir.alloca i32 {bindc_name = "y"
-! CHECK-NOT:   } copy {
+! CHECK-SAME:      {type = private} @[[TEST6_Y_PRIV:.*]] : i32
+! CHECK-NOT:   copy {
 
 ! CHECK-LABEL: omp.private
-! CHECK-SAME:      {type = private} @[[TEST6_X_PRIV:.*]] : !fir.ref<i32>
-! CHECK:         fir.alloca i32 {bindc_name = "x"
-! CHECK-NOT:   } copy {
+! CHECK-SAME:      {type = private} @[[TEST6_X_PRIV:.*]] : i32
+! CHECK-NOT:   copy {
 
 ! CHECK-LABEL: omp.private
-! CHECK-SAME:      {type = firstprivate} @[[TEST6_Z_FIRSTPRIV:.*]] : !fir.ref<i32>
-! CHECK:         fir.alloca i32 {bindc_name = "z"
-! CHECK:       } copy {
+! CHECK-SAME:      {type = firstprivate} @[[TEST6_Z_FIRSTPRIV:.*]] : i32
+! CHECK-SAME:  copy {
 ! CHECK:         hlfir.assign
 
 ! CHECK-LABEL: omp.private
-! CHECK-SAME:      {type = firstprivate} @[[TEST6_Y_FIRSTPRIV:.*]] : !fir.ref<i32>
-! CHECK:         fir.alloca i32 {bindc_name = "y"
-! CHECK:       } copy {
+! CHECK-SAME:      {type = firstprivate} @[[TEST6_Y_FIRSTPRIV:.*]] : i32
+! CHECK-SAME:  copy {
 ! CHECK:         hlfir.assign
 
 ! CHECK-LABEL: omp.private
-! CHECK-SAME:      {type = firstprivate} @[[TEST6_X_FIRSTPRIV:.*]] : !fir.ref<i32>
-! CHECK:         fir.alloca i32 {bindc_name = "x"
-! CHECK:       } copy {
+! CHECK-SAME:      {type = firstprivate} @[[TEST6_X_FIRSTPRIV:.*]] : i32
+! CHECK-SAME:  copy {
 ! CHECK:         hlfir.assign
 
 ! CHECK-LABEL: omp.private
-! CHECK-SAME:      {type = firstprivate} @[[TEST5_X_FIRSTPRIV:.*]] : !fir.ref<i32>
-! CHECK:         fir.alloca i32 {bindc_name = "x"
-! CHECK:       } copy {
+! CHECK-SAME:      {type = firstprivate} @[[TEST5_X_FIRSTPRIV:.*]] : i32
+! CHECK-SAME:  copy {
 ! CHECK:         hlfir.assign
 
 ! CHECK-LABEL: omp.private
-! CHECK-SAME:      {type = private} @[[TEST5_X_PRIV:.*]] : !fir.ref<i32>
-! CHECK:         fir.alloca i32 {bindc_name = "x"
-! CHECK-NOT:   } copy {
+! CHECK-SAME:      {type = private} @[[TEST5_X_PRIV:.*]] : i32
+! CHECK-NOT:   copy {
 
 ! CHECK-LABEL: omp.private
-! CHECK-SAME:      {type = firstprivate} @[[TEST4_Y_FIRSTPRIV:.*]] : !fir.ref<i32>
-! CHECK:         fir.alloca i32 {bindc_name = "y"
-! CHECK:       } copy {
+! CHECK-SAME:      {type = firstprivate} @[[TEST4_Y_FIRSTPRIV:.*]] : i32
+! CHECK-SAME:  copy {
 ! CHECK:         hlfir.assign
 
 ! CHECK-LABEL: omp.private
-! CHECK-SAME:      {type = firstprivate} @[[TEST4_Z_FIRSTPRIV:.*]] : !fir.ref<i32>
-! CHECK:         fir.alloca i32 {bindc_name = "z"
-! CHECK:       } copy {
+! CHECK-SAME:      {type = firstprivate} @[[TEST4_Z_FIRSTPRIV:.*]] : i32
+! CHECK-SAME:  copy {
 ! CHECK:         hlfir.assign
 
 ! CHECK-LABEL: omp.private
-! CHECK-SAME:      {type = firstprivate} @[[TEST4_X_FIRSTPRIV:.*]] : !fir.ref<i32>
-! CHECK:         fir.alloca i32 {bindc_name = "x"
-! CHECK:       } copy {
+! CHECK-SAME:      {type = firstprivate} @[[TEST4_X_FIRSTPRIV:.*]] : i32
+! CHECK-SAME:  copy {
 ! CHECK:         hlfir.assign
 
 ! CHECK-LABEL: omp.private
-! CHECK-SAME:      {type = private} @[[TEST4_Y_PRIV:.*]] : !fir.ref<i32>
-! CHECK:         fir.alloca i32 {bindc_name = "y"
-! CHECK-NOT:   } copy {
+! CHECK-SAME:      {type = private} @[[TEST4_Y_PRIV:.*]] : i32
+! CHECK-NOT:   copy {
 
 ! CHECK-LABEL: omp.private
-! CHECK-SAME:      {type = private} @[[TEST4_Z_PRIV:.*]] : !fir.ref<i32>
-! CHECK:         fir.alloca i32 {bindc_name = "z"
-! CHECK-NOT:   } copy {
+! CHECK-SAME:      {type = private} @[[TEST4_Z_PRIV:.*]] : i32
+! CHECK-NOT:   copy {
 
 ! CHECK-LABEL: omp.private
-! CHECK-SAME:      {type = private} @[[TEST4_X_PRIV:.*]] : !fir.ref<i32>
-! CHECK:         fir.alloca i32 {bindc_name = "x"
-! CHECK-NOT:   } copy {
+! CHECK-SAME:      {type = private} @[[TEST4_X_PRIV:.*]] : i32
+! CHECK-NOT:   copy {
 
 ! CHECK-LABEL: omp.private
-! CHECK-SAME:      {type = firstprivate} @[[TEST3_X_FIRSTPRIV:.*]] : !fir.ref<i32>
-! CHECK:         fir.alloca i32 {bindc_name = "x"
-! CHECK:       } copy {
+! CHECK-SAME:      {type = firstprivate} @[[TEST3_X_FIRSTPRIV:.*]] : i32
+! CHECK:       copy {
 ! CHECK:         hlfir.assign
 
 ! CHECK-LABEL: omp.private
-! CHECK-SAME:      {type = firstprivate} @[[TEST2_X_FIRSTPRIV:.*]] : !fir.ref<i32>
-! CHECK:         fir.alloca i32 {bindc_name = "x"
-! CHECK:       } copy {
+! CHECK-SAME:      {type = firstprivate} @[[TEST2_X_FIRSTPRIV:.*]] : i32
+! CHECK:       copy {
 ! CHECK:         hlfir.assign
 
 ! CHECK-LABEL: omp.private
-! CHECK-SAME:      {type = firstprivate} @[[TEST1_X_FIRSTPRIV:.*]] : !fir.ref<i32>
-! CHECK:         fir.alloca i32 {bindc_name = "x"
-! CHECK:       } copy {
+! CHECK-SAME:      {type = firstprivate} @[[TEST1_X_FIRSTPRIV:.*]] : i32
+! CHECK:       copy {
 ! CHECK:         hlfir.assign
 
 ! CHECK-LABEL: omp.private
-! CHECK-SAME:      {type = private} @[[TEST1_Y_PRIV:.*]] : !fir.ref<i32>
-! CHECK:         fir.alloca i32 {bindc_name = "y"
-! CHECK-NOT:   } copy {
+! CHECK-SAME:      {type = private} @[[TEST1_Y_PRIV:.*]] : i32
+! CHECK-NOT:   copy {
 
 ! Basic cases.
 !CHECK-LABEL: func @_QPimplicit_dsa_test1
diff --git a/flang/test/Lower/OpenMP/lastprivate-allocatable.f90 b/flang/test/Lower/OpenMP/lastprivate-allocatable.f90
index 6b7d849fde93c..fd8338393dd88 100644
--- a/flang/test/Lower/OpenMP/lastprivate-allocatable.f90
+++ b/flang/test/Lower/OpenMP/lastprivate-allocatable.f90
@@ -8,12 +8,11 @@
 ! CHECK:           fir.store %[[VAL_2]] to %[[VAL_0]] : !fir.ref<!fir.box<!fir.heap<i32>>>
 ! CHECK:           %[[VAL_3:.*]]:2 = hlfir.declare %[[VAL_0]] {fortran_attrs = {{.*}}<allocatable>, uniq_name = "_QFEa"} : (!fir.ref<!fir.box<!fir.heap<i32>>>) -> (!fir.ref<!fir.box<!fir.heap<i32>>>, !fir.ref<!fir.box<!fir.heap<i32>>>)
 ! CHECK:           omp.parallel {
-!                    create original copy of private variable
-! CHECK:             %[[VAL_16:.*]]:2 = hlfir.declare %{{.*}} {fortran_attrs = {{.*}}<allocatable>, uniq_name = "_QFEa"} : (!fir.ref<!fir.box<!fir.heap<i32>>>) -> (!fir.ref<!fir.box<!fir.heap<i32>>>, !fir.ref<!fir.box<!fir.heap<i32>>>)
-! CHECK:             %[[VAL_17:.*]] = fir.alloca i32 {bindc_name = "i", pinned, uniq_name = "_QFEi"}
-! CHECK:             %[[VAL_18:.*]]:2 = hlfir.declare %[[VAL_17]] {uniq_name = "_QFEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-! CHECK:             omp.wsloop {
+! CHECK:             omp.wsloop private(@{{.*}} %{{.*}} -> %{{.*}}, @{{.*}} %{{.*}} -> %[[VAL_17:.*]] : !fir.ref<!fir.box<!fir.heap<i32>>>, !fir.ref<i32>) {
 ! CHECK:               omp.loop_nest
+! CHECK:                   %[[VAL_16:.*]]:2 = hlfir.declare %{{.*}} {fortran_attrs = {{.*}}<allocatable>, uniq_name = "_QFEa"} : (!fir.ref<!fir.box<!fir.heap<i32>>>) -> (!fir.ref<!fir.box<!fir.heap<i32>>>, !fir.ref<!fir.box<!fir.heap<i32>>>)
+! CHECK:                   %[[VAL_18:.*]]:2 = hlfir.declare %[[VAL_17]] {uniq_name = "_QFEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+
 !                        [...]
 !                        if this is the last iteration
 ! CHECK:                 fir.if %{{.*}} {
diff --git a/flang/test/Lower/OpenMP/lastprivate-commonblock.f90 b/flang/test/Lower/OpenMP/lastprivate-commonblock.f90
index faa3d3e053f34..c059382bf634c 100644
--- a/flang/test/Lower/OpenMP/lastprivate-commonblock.f90
+++ b/flang/test/Lower/OpenMP/lastprivate-commonblock.f90
@@ -11,12 +11,10 @@
 !CHECK:      %[[CB_C_Y_COOR:.*]] = fir.coordinate_of %[[CB_C_REF_CVT]], %{{.*}} : (!fir.ref<!fir.array<?xi8>>, index) -> !fir.ref<i8>
 !CHECK:      %[[CB_C_Y_ADDR:.*]] = fir.convert %[[CB_C_Y_COOR]] : (!fir.ref<i8>) -> !fir.ref<f32>
 !CHECK:      %[[Y_DECL:.*]]:2 = hlfir.declare %[[CB_C_Y_ADDR]] {uniq_name = "_QFlastprivate_commonEy"} : (!fir.ref<f32>) -> (!fir.ref<f32>, !fir.ref<f32>)
-!CHECK:      %[[PRIVATE_X_REF:.*]] = fir.alloca f32 {bindc_name = "x", pinned, uniq_name = "_QFlastprivate_commonEx"}
+!CHECK:      omp.wsloop private(@{{.*}} %{{.*}} -> %[[PRIVATE_X_REF:.*]], @{{.*}} %{{.*}} -> %[[PRIVATE_Y_REF:.*]], @{{.*}} %{{.*}} -> %{{.*}} : !{{.*}}, !{{.*}}, !{{.*}}) {
+!CHECK-NEXT:   omp.loop_nest (%[[I:.*]]) : i32 = (%{{.*}}) to (%{{.*}}) inclusive step (%{{.*}}) {
 !CHECK:      %[[PRIVATE_X_DECL:.*]]:2 = hlfir.declare %[[PRIVATE_X_REF]] {uniq_name = "_QFlastprivate_commonEx"} : (!fir.ref<f32>) -> (!fir.ref<f32>, !fir.ref<f32>)
-!CHECK:      %[[PRIVATE_Y_REF:.*]] = fir.alloca f32 {bindc_name = "y", pinned, uniq_name = "_QFlastprivate_commonEy"}
 !CHECK:      %[[PRIVATE_Y_DECL:.*]]:2 = hlfir.declare %[[PRIVATE_Y_REF]] {uniq_name = "_QFlastprivate_commonEy"} : (!fir.ref<f32>) -> (!fir.ref<f32>, !fir.ref<f32>)
-!CHECK:      omp.wsloop {
-!CHECK-NEXT:   omp.loop_nest (%[[I:.*]]) : i32 = (%{{.*}}) to (%{{.*}}) inclusive step (%{{.*}}) {
 !CHECK:          %[[V:.*]] = arith.addi %[[I]], %{{.*}} : i32
 !CHECK:          %[[C0:.*]] = arith.constant 0 : i32
 !CHECK:          %[[NEG_STEP:.*]] = arith.cmpi slt, %{{.*}}, %[[C0]] : i32
diff --git a/flang/test/Lower/OpenMP/lastprivate-iv.f90 b/flang/test/Lower/OpenMP/lastprivate-iv.f90
index 63a81e818bc8b..aacefd8b59c0f 100644
--- a/flang/test/Lower/OpenMP/lastprivate-iv.f90
+++ b/flang/test/Lower/OpenMP/lastprivate-iv.f90
@@ -6,14 +6,12 @@
 !CHECK:      %[[I2_MEM:.*]] = fir.alloca i32 {bindc_name = "i", uniq_name = "_QFlastprivate_iv_incEi"}
 !CHECK:      %[[I2:.*]]:2 = hlfir.declare %[[I2_MEM]] {uniq_name = "_QFlastprivate_iv_incEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 
-!CHECK:      %[[I_MEM:.*]] = fir.alloca i32 {bindc_name = "i", pinned, {{.*}}}
-!CHECK:      %[[I:.*]]:2 = hlfir.declare %[[I_MEM]] {uniq_name = "_QFlastprivate_iv_incEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-
 !CHECK:      %[[LB:.*]] = arith.constant 4 : i32
 !CHECK:      %[[UB:.*]] = arith.constant 10 : i32
 !CHECK:      %[[STEP:.*]]  = arith.constant 3 : i32
-!CHECK:      omp.wsloop {
+!CHECK:      omp.wsloop private(@{{.*}} %{{.*}} -> %[[I_MEM:.*]] : !fir.ref<i32>) {
 !CHECK-NEXT:   omp.loop_nest (%[[IV:.*]]) : i32 = (%[[LB]]) to (%[[UB]]) inclusive step (%[[STEP]]) {
+!CHECK:          %[[I:.*]]:2 = hlfir.declare %[[I_MEM]] {uniq_name = "_QFlastprivate_iv_incEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 !CHECK:          fir.store %[[IV]] to %[[I]]#1 : !fir.ref<i32>
 !CHECK:          %[[V:.*]] = arith.addi %[[IV]], %[[STEP]] : i32
 !CHECK:          %[[C0:.*]] = arith.constant 0 : i32
@@ -42,15 +40,12 @@ subroutine lastprivate_iv_inc()
 
 !CHECK:      %[[I2_MEM:.*]] = fir.alloca i32 {bindc_name = "i", uniq_name = "_QFlastprivate_iv_decEi"}
 !CHECK:      %[[I2:.*]]:2 = hlfir.declare %[[I2_MEM]] {uniq_name = "_QFlastprivate_iv_decEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-
-!CHECK:      %[[I_MEM:.*]] = fir.alloca i32 {bindc_name = "i", pinned, {{.*}}}
-!CHECK:      %[[I:.*]]:2 = hlfir.declare %[[I_MEM]] {uniq_name = "_QFlastprivate_iv_decEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-
 !CHECK:      %[[LB:.*]] = arith.constant 10 : i32
 !CHECK:      %[[UB:.*]] = arith.constant 1 : i32
 !CHECK:      %[[STEP:.*]]  = arith.constant -3 : i32
-!CHECK:      omp.wsloop {
+!CHECK:      omp.wsloop private(@{{.*}} %{{.*}} -> %[[I_MEM:.*]] : !fir.ref<i32>) {
 !CHECK-NEXT:   omp.loop_nest (%[[IV:.*]]) : i32 = (%[[LB]]) to (%[[UB]]) inclusive step (%[[STEP]]) {
+!CHECK:          %[[I:.*]]:2 = hlfir.declare %[[I_MEM]] {uniq_name = "_QFlastprivate_iv_decEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 !CHECK:          fir.store %[[IV]] to %[[I]]#1 : !fir.ref<i32>
 !CHECK:          %[[V:.*]] = arith.addi %[[IV]], %[[STEP]] : i32
 !CHECK:          %[[C0:.*]] = arith.constant 0 : i32
@@ -80,7 +75,7 @@ subroutine lastprivate_iv_dec()
 subroutine lastprivate_iv_i1
   integer*1 :: i1
   i1=0
-!CHECK:    omp.wsloop
+!CHECK:    omp.wsloop private({{.*}})
 !CHECK:      omp.loop_nest
 !CHECK:        fir.if %{{.*}} {
 !CHECK:          %[[I8_VAL:.*]] = fir.convert %{{.*}} : (i32) -> i8
diff --git a/flang/test/Lower/OpenMP/lastprivate-simd.f90 b/flang/test/Lower/OpenMP/lastprivate-simd.f90
new file mode 100644
index 0000000000000..75f5ebfe42f39
--- /dev/null
+++ b/flang/test/Lower/OpenMP/lastprivate-simd.f90
@@ -0,0 +1,60 @@
+! RUN: %flang_fc1 -emit-hlfir -fopenmp -o - %s 2>&1 | FileCheck %s
+
+subroutine simd_ivs
+  implicit none
+  integer :: ido1 = 1
+  integer :: ido2 = 2
+  integer :: ido3 = 3
+  integer :: ido4 = 4
+
+  !$omp parallel
+  !$omp simd collapse(3)
+  do ido1 = 1, 10
+    do ido2 = 1, 10
+      do ido3 = 1, 10
+        do ido4 = 1, 10
+        end do
+      end do
+    end do
+  end do
+  !$omp end simd
+  !$omp end parallel
+end subroutine
+
+! CHECK: func.func @_QPsimd_ivs() {
+! CHECK:   %[[IDO1_HOST_DECL:.*]]:2 = hlfir.declare %{{.*}} {uniq_name = "{{.*}}Eido1"}
+! CHECK:   %[[IDO2_HOST_DECL:.*]]:2 = hlfir.declare %{{.*}} {uniq_name = "{{.*}}Eido2"}
+! CHECK:   %[[IDO3_HOST_DECL:.*]]:2 = hlfir.declare %{{.*}} {uniq_name = "{{.*}}Eido3"}
+
+! CHECK:   omp.parallel {
+! CHECK:     omp.simd private(
+! CHECK-SAME:  @{{.*}}do1_private{{.*}} %[[IDO1_HOST_DECL]]#0 -> %[[IDO1_PRIV_ARG:[^[:space:]]*]],
+! CHECK-SAME:  @{{.*}}do2_private{{.*}} %[[IDO2_HOST_DECL]]#0 -> %[[IDO2_PRIV_ARG:[^[:space:]]*]],
+! CHECK-SAME:  @{{.*}}do3_private{{.*}} %[[IDO3_HOST_DECL]]#0 -> %[[IDO3_PRIV_ARG:[^[:space:]]*]]
+! CHECK-SAME:  : {{.*}}) {
+
+! CHECK:       omp.loop_nest (%[[IV1:.*]], %[[IV2:.*]], %[[IV3:.*]]) : {{.*}} {
+! CHECK:         %[[IDO1_PRIV_DECL:.*]]:2 = hlfir.declare %[[IDO1_PRIV_ARG]] {uniq_name = "{{.*}}Eido1"}
+! CHECK:         %[[IDO2_PRIV_DECL:.*]]:2 = hlfir.declare %[[IDO2_PRIV_ARG]] {uniq_name = "{{.*}}Eido2"}
+! CHECK:         %[[IDO3_PRIV_DECL:.*]]:2 = hlfir.declare %[[IDO3_PRIV_ARG]] {uniq_name = "{{.*}}Eido3"}
+
+! CHECK:         fir.if %33 {
+! CHECK:           fir.store %{{.*}} to %[[IDO1_PRIV_DECL]]#1
+! CHECK:           fir.store %{{.*}} to %[[IDO2_PRIV_DECL]]#1
+! CHECK:           fir.store %{{.*}} to %[[IDO3_PRIV_DECL]]#1
+! CHECK:           %[[IDO1_VAL:.*]] = fir.load %[[IDO1_PRIV_DECL]]#0
+! CHECK:           hlfir.assign %[[IDO1_VAL]] to %[[IDO1_HOST_DECL]]#0
+! CHECK:           %[[IDO2_VAL:.*]] = fir.load %[[IDO2_PRIV_DECL]]#0
+! CHECK:           hlfir.assign %[[IDO2_VAL]] to %[[IDO2_HOST_DECL]]#0
+! CHECK:           %[[IDO3_VAL:.*]] = fir.load %[[IDO3_PRIV_DECL]]#0
+! CHECK:           hlfir.assign %[[IDO3_VAL]] to %[[IDO3_HOST_DECL]]#0
+! CHECK:         }
+! CHECK-NEXT:    omp.yield
+! CHECK:       }
+
+! CHECK:     }
+
+! CHECK:     omp.terminator
+! CHECK:   }
+
+! CHECK: }
diff --git a/flang/test/Lower/OpenMP/location.f90 b/flang/test/Lower/OpenMP/location.f90
index 2dab22a1c1f90..fc7dd43499863 100644
--- a/flang/test/Lower/OpenMP/location.f90
+++ b/flang/test/Lower/OpenMP/location.f90
@@ -28,7 +28,7 @@ subroutine sub_target()
 
 !CHECK-LABEL: sub_loop
 subroutine sub_loop()
-!CHECK: omp.wsloop {
+!CHECK: omp.wsloop private({{.*}}) {
 !CHECK-NEXT: omp.loop_nest {{.*}} {
   !$omp do
   do i=1,10
diff --git a/flang/test/Lower/OpenMP/loop-directive.f90 b/flang/test/Lower/OpenMP/loop-directive.f90
index 9fa0de3bfe171..785f732e1b4f5 100644
--- a/flang/test/Lower/OpenMP/loop-directive.f90
+++ b/flang/test/Lower/OpenMP/loop-directive.f90
@@ -4,8 +4,8 @@
 ! RUN: %flang_fc1 -emit-hlfir -fopenmp -fopenmp-version=50 -o - %s 2>&1 | FileCheck %s
 
 ! CHECK: omp.declare_reduction @[[RED:add_reduction_i32]] : i32
-! CHECK: omp.private {type = private} @[[DUMMY_PRIV:.*test_privateEdummy_private.*]] : !fir.ref<i32>
-! CHECK: omp.private {type = private} @[[I_PRIV:.*test_no_clausesEi.*]] : !fir.ref<i32>
+! CHECK: omp.private {type = private} @[[DUMMY_PRIV:.*test_privateEdummy_private.*]] : i32
+! CHECK: omp.private {type = private} @[[I_PRIV:.*test_no_clausesEi.*]] : i32
 
 ! CHECK-LABEL: func.func @_QPtest_no_clauses
 subroutine test_no_clauses()
@@ -92,7 +92,7 @@ subroutine test_reduction()
 ! CHECK-LABEL: func.func @_QPtest_bind
 subroutine test_bind()
   integer :: i, dummy = 1
-  ! CHECK: omp.loop bind(thread) private(@{{.*}} %{{.*}}#0 -> %{{.*}} : {{.*}}) {
+  ! CHECK: omp.simd private(@{{.*}} %{{.*}}#0 -> %{{.*}} : {{.*}}) {
   ! CHECK: }
   !$omp loop bind(thread)
   do i=1,10
@@ -139,3 +139,43 @@ subroutine test_nested_directives
   end do
   !$omp end target teams
 end subroutine
+
+! CHECK-LABEL: func.func @_QPtest_standalone_bind_teams
+subroutine test_standalone_bind_teams
+  implicit none
+  integer, parameter :: N = 100000
+  integer a(N), b(N), c(N)
+  integer j,i, num, flag;
+  num = N
+
+  ! CHECK:     omp.distribute
+  ! CHECK-SAME:  private(@{{.*}}Ea_private_box_100000xi32 {{[^,]*}},
+  ! CHECK-SAME:          @{{.*}}Ei_private_i32 {{.*}} : {{.*}}) {
+  ! CHECK:       omp.loop_nest {{.*}} {
+  ! CHECK:       }
+  ! CHECK:     }
+  !$omp loop bind(teams) private(a)
+  do i=1,N
+    c(i) = a(i) * b(i)
+  end do
+end subroutine
+
+! CHECK-LABEL: func.func @_QPtest_standalone_bind_parallel
+subroutine test_standalone_bind_parallel
+  implicit none
+  integer, parameter :: N = 100000
+  integer a(N), b(N), c(N)
+  integer j,i, num, flag;
+  num = N
+
+  ! CHECK:     omp.wsloop
+  ! CHECK-SAME:  private(@{{.*}}Ea_private_box_100000xi32 {{[^,]*}},
+  ! CHECK-SAME:          @{{.*}}Ei_private_i32 {{.*}} : {{.*}}) {
+  ! CHECK:       omp.loop_nest {{.*}} {
+  ! CHECK:       }
+  ! CHECK:     }
+  !$omp loop bind(parallel) private(a)
+  do i=1,N
+    c(i) = a(i) * b(i)
+  end do
+end subroutine
diff --git a/flang/test/Lower/OpenMP/order-clause.f90 b/flang/test/Lower/OpenMP/order-clause.f90
index a30d82979021d..1f678e02708da 100644
--- a/flang/test/Lower/OpenMP/order-clause.f90
+++ b/flang/test/Lower/OpenMP/order-clause.f90
@@ -20,15 +20,15 @@ end subroutine simd_order
 
 !CHECK-LABEL:   func.func @_QPdo_order() {
 subroutine do_order
-   !CHECK: omp.wsloop order(reproducible:concurrent) {
+   !CHECK: omp.wsloop order(reproducible:concurrent) private({{.*}}) {
    !$omp do order(concurrent)
    do i = 1, 10
    end do
-   !CHECK: omp.wsloop order(reproducible:concurrent) {
+   !CHECK: omp.wsloop order(reproducible:concurrent) private({{.*}}) {
    !$omp do order(reproducible:concurrent)
    do i = 1, 10
    end do
-   !CHECK: omp.wsloop order(unconstrained:concurrent) {
+   !CHECK: omp.wsloop order(unconstrained:concurrent) private({{.*}}) {
    !$omp do order(unconstrained:concurrent)
    do i = 1, 10
    end do
diff --git a/flang/test/Lower/OpenMP/parallel-firstprivate-clause-scalar.f90 b/flang/test/Lower/OpenMP/parallel-firstprivate-clause-scalar.f90
index f0bee355543a6..017b2a3f2edd2 100644
--- a/flang/test/Lower/OpenMP/parallel-firstprivate-clause-scalar.f90
+++ b/flang/test/Lower/OpenMP/parallel-firstprivate-clause-scalar.f90
@@ -1,31 +1,22 @@
 ! This test checks lowering of `FIRSTPRIVATE` clause for scalar types.
 
+! REQUIRES: x86_64-registered-target
 ! REQUIRES: shell
 ! RUN: bbc -fopenmp -emit-hlfir %s -o - \
-! RUN: | FileCheck %s --check-prefix=CHECK
+! RUN: | FileCheck %s --check-prefixes=CHECK%if target=x86_64{{.*}} %{,CHECK-KIND10%}%if flang-supports-f128-math %{,CHECK-KIND16%}
 
-!CHECK:  omp.private {type = firstprivate} @[[ARG2_LOGICAL_PRIVATIZER:_QFfirstprivate_logicalEarg2_firstprivate_ref_l8]] : !fir.ref<!fir.logical<1>> alloc
+!CHECK:  omp.private {type = firstprivate} @[[ARG2_LOGICAL_PRIVATIZER:_QFfirstprivate_logicalEarg2_firstprivate_l8]] : !fir.logical<1>
 
-!CHECK:  omp.private {type = firstprivate} @[[ARG1_LOGICAL_PRIVATIZER:_QFfirstprivate_logicalEarg1_firstprivate_ref_l32]] : !fir.ref<!fir.logical<4>> alloc {
-!CHECK:  ^bb0(%{{.*}}: !fir.ref<!fir.logical<4>>):
-!CHECK:    %[[PVT_ALLOC:.*]] = fir.alloca !fir.logical<4> {{.*}}
-!CHECK:    %[[PVT_DECL:.*]]:2 = hlfir.declare %[[PVT_ALLOC]] {{.*}}
-!CHECK:    omp.yield(%[[PVT_DECL]]#0 : !fir.ref<!fir.logical<4>>)
-!CHECK:  } copy {
+!CHECK:  omp.private {type = firstprivate} @[[ARG1_LOGICAL_PRIVATIZER:_QFfirstprivate_logicalEarg1_firstprivate_l32]] : !fir.logical<4> copy {
 !CHECK:  ^bb0(%[[ORIG_REF:.*]]: !fir.ref<!fir.logical<4>>, %[[PVT_REF:.*]]: !fir.ref<!fir.logical<4>>):
 !CHECK:    %[[ORIG_VAL:.*]] = fir.load %[[ORIG_REF]] : {{.*}}
 !CHECK:    hlfir.assign %[[ORIG_VAL]] to %[[PVT_REF]] {{.*}}
 !CHECK:    omp.yield(%[[PVT_REF]] : !fir.ref<!fir.logical<4>>)
 !CHECK:  }
 
-!CHECK:  omp.private {type = firstprivate} @[[ARG2_COMPLEX_PRIVATIZER:_QFfirstprivate_complexEarg2_firstprivate_ref_z64]] : !fir.ref<complex<f64>> alloc
+!CHECK:  omp.private {type = firstprivate} @[[ARG2_COMPLEX_PRIVATIZER:_QFfirstprivate_complexEarg2_firstprivate_z64]] : complex<f64>
 
-!CHECK:  omp.private {type = firstprivate} @[[ARG1_COMPLEX_PRIVATIZER:_QFfirstprivate_complexEarg1_firstprivate_ref_z32]] : !fir.ref<complex<f32>> alloc {
-!CHECK:  ^bb0(%{{.*}}: !fir.ref<complex<f32>>):
-!CHECK:    %[[PVT_ALLOC:.*]] = fir.alloca complex<f32> {bindc_name = "arg1", {{.*}}}
-!CHECK:    %[[PVT_DECL:.*]]:2 = hlfir.declare %[[PVT_ALLOC]] {{.*}}
-!CHECK:    omp.yield(%[[PVT_DECL]]#0 : !fir.ref<complex<f32>>)
-!CHECK:  } copy {
+!CHECK:  omp.private {type = firstprivate} @[[ARG1_COMPLEX_PRIVATIZER:_QFfirstprivate_complexEarg1_firstprivate_z32]] : complex<f32> copy {
 !CHECK:  ^bb0(%[[ORIG_REF:.*]]: !fir.ref<complex<f32>>, %[[PVT_REF:.*]]: !fir.ref<complex<f32>>):
 !CHECK:    %[[ORIG_VAL:.*]] = fir.load %[[ORIG_REF]] : {{.*}}
 !CHECK:    hlfir.assign %[[ORIG_VAL]] to %[[PVT_REF]] {{.*}}
@@ -114,38 +105,64 @@ subroutine firstprivate_logical(arg1, arg2, arg3, arg4, arg5)
 
 end subroutine
 
-!CHECK-DAG: func @_QPfirstprivate_real(%[[ARG1:.*]]: !fir.ref<f32>{{.*}}, %[[ARG2:.*]]: !fir.ref<f16>{{.*}}, %[[ARG3:.*]]: !fir.ref<f32>{{.*}}, %[[ARG4:.*]]: !fir.ref<f64>{{.*}}, %[[ARG5:.*]]: !fir.ref<f80>{{.*}}, %[[ARG6:.*]]: !fir.ref<f128>{{.*}}) {
+!CHECK-LABEL: func @_QPfirstprivate_real(
+!CHECK-SAME: %[[ARG1:.*]]: !fir.ref<f32>{{.*}}, %[[ARG2:.*]]: !fir.ref<f16>{{.*}}, %[[ARG3:.*]]: !fir.ref<f32>{{.*}}, %[[ARG4:.*]]: !fir.ref<f64>{{.*}}) {
 !CHECK:   %[[ARG1_DECL:.*]]:2 = hlfir.declare %[[ARG1]] dummy_scope %{{[0-9]+}} {uniq_name = "_QFfirstprivate_realEarg1"} : (!fir.ref<f32>, !fir.dscope) -> (!fir.ref<f32>, !fir.ref<f32>)
 !CHECK:   %[[ARG2_DECL:.*]]:2 = hlfir.declare %[[ARG2]] dummy_scope %{{[0-9]+}} {uniq_name = "_QFfirstprivate_realEarg2"} : (!fir.ref<f16>, !fir.dscope) -> (!fir.ref<f16>, !fir.ref<f16>)
 !CHECK:   %[[ARG3_DECL:.*]]:2 = hlfir.declare %[[ARG3]] dummy_scope %{{[0-9]+}} {uniq_name = "_QFfirstprivate_realEarg3"} : (!fir.ref<f32>, !fir.dscope) -> (!fir.ref<f32>, !fir.ref<f32>)
 !CHECK:   %[[ARG4_DECL:.*]]:2 = hlfir.declare %[[ARG4]] dummy_scope %{{[0-9]+}} {uniq_name = "_QFfirstprivate_realEarg4"} : (!fir.ref<f64>, !fir.dscope) -> (!fir.ref<f64>, !fir.ref<f64>)
-!CHECK:   %[[ARG5_DECL:.*]]:2 = hlfir.declare %[[ARG5]] dummy_scope %{{[0-9]+}} {uniq_name = "_QFfirstprivate_realEarg5"} : (!fir.ref<f80>, !fir.dscope) -> (!fir.ref<f80>, !fir.ref<f80>)
-!CHECK:   %[[ARG6_DECL:.*]]:2 = hlfir.declare %[[ARG6]] dummy_scope %{{[0-9]+}} {uniq_name = "_QFfirstprivate_realEarg6"} : (!fir.ref<f128>, !fir.dscope) -> (!fir.ref<f128>, !fir.ref<f128>)
-!CHECK:  omp.parallel private({{.*firstprivate.*}} {{.*}}#0 -> %[[ARG1_PVT:.*]], {{.*firstprivate.*}} {{.*}}#0 -> %[[ARG2_PVT:.*]], {{.*firstprivate.*}} {{.*}}#0 -> %[[ARG3_PVT:.*]], {{.*firstprivate.*}} {{.*}}#0 -> %[[ARG4_PVT:.*]], {{.*firstprivate.*}} {{.*}}#0 -> %[[ARG5_PVT:.*]], {{.*firstprivate.*}} {{.*}}#0 -> %[[ARG6_PVT:.*]] : {{.*}}) {
+!CHECK:  omp.parallel private({{.*firstprivate.*}} {{.*}}#0 -> %[[ARG1_PVT:.*]], {{.*firstprivate.*}} {{.*}}#0 -> %[[ARG2_PVT:.*]], {{.*firstprivate.*}} {{.*}}#0 -> %[[ARG3_PVT:.*]], {{.*firstprivate.*}} {{.*}}#0 -> %[[ARG4_PVT:.*]] : {{.*}}) {
 !CHECK:     %[[ARG1_PVT_DECL:.*]]:2 = hlfir.declare %[[ARG1_PVT]] {uniq_name = "_QFfirstprivate_realEarg1"} : (!fir.ref<f32>) -> (!fir.ref<f32>, !fir.ref<f32>)
 !CHECK:     %[[ARG2_PVT_DECL:.*]]:2 = hlfir.declare %[[ARG2_PVT]] {uniq_name = "_QFfirstprivate_realEarg2"} : (!fir.ref<f16>) -> (!fir.ref<f16>, !fir.ref<f16>)
 !CHECK:     %[[ARG3_PVT_DECL:.*]]:2 = hlfir.declare %[[ARG3_PVT]] {uniq_name = "_QFfirstprivate_realEarg3"} : (!fir.ref<f32>) -> (!fir.ref<f32>, !fir.ref<f32>)
 !CHECK:     %[[ARG4_PVT_DECL:.*]]:2 = hlfir.declare %[[ARG4_PVT]] {uniq_name = "_QFfirstprivate_realEarg4"} : (!fir.ref<f64>) -> (!fir.ref<f64>, !fir.ref<f64>)
-!CHECK:     %[[ARG5_PVT_DECL:.*]]:2 = hlfir.declare %[[ARG5_PVT]] {uniq_name = "_QFfirstprivate_realEarg5"} : (!fir.ref<f80>) -> (!fir.ref<f80>, !fir.ref<f80>)
-!CHECK:     %[[ARG6_PVT_DECL:.*]]:2 = hlfir.declare %[[ARG6_PVT]] {uniq_name = "_QFfirstprivate_realEarg6"} : (!fir.ref<f128>) -> (!fir.ref<f128>, !fir.ref<f128>)
-!CHECK:     fir.call @_QPqux(%[[ARG1_PVT_DECL]]#1, %[[ARG2_PVT_DECL]]#1, %[[ARG3_PVT_DECL]]#1, %[[ARG4_PVT_DECL]]#1, %[[ARG5_PVT_DECL]]#1, %[[ARG6_PVT_DECL]]#1) {{.*}}: (!fir.ref<f32>, !fir.ref<f16>, !fir.ref<f32>, !fir.ref<f64>, !fir.ref<f80>, !fir.ref<f128>) -> ()
+!CHECK:     fir.call @_QPqux(%[[ARG1_PVT_DECL]]#1, %[[ARG2_PVT_DECL]]#1, %[[ARG3_PVT_DECL]]#1, %[[ARG4_PVT_DECL]]#1) {{.*}}: (!fir.ref<f32>, !fir.ref<f16>, !fir.ref<f32>, !fir.ref<f64>) -> ()
 !CHECK:     omp.terminator
 !CHECK:   }
-
 subroutine firstprivate_real(arg1, arg2, arg3, arg4, arg5, arg6)
         real :: arg1
         real(kind=2) :: arg2
         real(kind=4) :: arg3
         real(kind=8) :: arg4
-        real(kind=10) :: arg5
-        real(kind=16) :: arg6
 
-!$OMP PARALLEL FIRSTPRIVATE(arg1, arg2, arg3, arg4, arg5, arg6)
-        call qux(arg1, arg2, arg3, arg4, arg5, arg6)
+!$OMP PARALLEL FIRSTPRIVATE(arg1, arg2, arg3, arg4)
+        call qux(arg1, arg2, arg3, arg4)
 !$OMP END PARALLEL
 
 end subroutine
 
+!CHECK-KIND10-LABEL: func @_QPfirstprivate_real10(
+!CHECK-KIND10-SAME: %[[ARG1:.*]]: !fir.ref<f80>{{.*}}) {
+!CHECK-KIND10:   %[[ARG1_DECL:.*]]:2 = hlfir.declare %[[ARG1]] dummy_scope %{{[0-9]+}} {uniq_name = "_QFfirstprivate_real10Earg1"} : (!fir.ref<f80>, !fir.dscope) -> (!fir.ref<f80>, !fir.ref<f80>)
+!CHECK-KIND10:  omp.parallel private({{.*firstprivate.*}} {{.*}}#0 -> %[[ARG1_PVT:.*]] : {{.*}}) {
+!CHECK-KIND10:     %[[ARG1_PVT_DECL:.*]]:2 = hlfir.declare %[[ARG1_PVT]] {uniq_name = "_QFfirstprivate_real10Earg1"} : (!fir.ref<f80>) -> (!fir.ref<f80>, !fir.ref<f80>)
+!CHECK-KIND10:     fir.call @_QPqux10(%[[ARG1_PVT_DECL]]#1) {{.*}} : (!fir.ref<f80>) -> ()
+!CHECK-KIND10:     omp.terminator
+!CHECK-KIND10:   }
+subroutine firstprivate_real10(arg1)
+        integer, parameter :: kind10 = merge(10, 4, selected_real_kind(p=18).eq.10)
+        real(kind=kind10) :: arg1
+!$OMP PARALLEL FIRSTPRIVATE(arg1)
+        call qux10(arg1)
+!$OMP END PARALLEL
+end subroutine
+
+!CHECK-KIND16-LABEL: func @_QPfirstprivate_real16(
+!CHECK-KIND16-SAME: %[[ARG1:.*]]: !fir.ref<f128>{{.*}}) {
+!CHECK-KIND16:   %[[ARG1_DECL:.*]]:2 = hlfir.declare %[[ARG1]] dummy_scope %{{[0-9]+}} {uniq_name = "_QFfirstprivate_real16Earg1"} : (!fir.ref<f128>, !fir.dscope) -> (!fir.ref<f128>, !fir.ref<f128>)
+!CHECK-KIND16:  omp.parallel private({{.*firstprivate.*}} {{.*}}#0 -> %[[ARG1_PVT:.*]] : {{.*}}) {
+!CHECK-KIND16:     %[[ARG1_PVT_DECL:.*]]:2 = hlfir.declare %[[ARG1_PVT]] {uniq_name = "_QFfirstprivate_real16Earg1"} : (!fir.ref<f128>) -> (!fir.ref<f128>, !fir.ref<f128>)
+!CHECK-KIND16:     fir.call @_QPqux16(%[[ARG1_PVT_DECL]]#1) {{.*}} : (!fir.ref<f128>) -> ()
+!CHECK-KIND16:     omp.terminator
+!CHECK-KIND16:   }
+subroutine firstprivate_real16(arg1)
+        integer, parameter :: kind16 = merge(16, 4, selected_real_kind(p=33).eq.16)
+        real(kind=kind16) :: arg1
+!$OMP PARALLEL FIRSTPRIVATE(arg1)
+        call qux16(arg1)
+!$OMP END PARALLEL
+end subroutine
+
 !CHECK-LABEL:   func.func @_QPmultiple_firstprivate(
 !CHECK-SAME:                                        %[[A_ADDR:.*]]: !fir.ref<i32> {fir.bindc_name = "a"},
 !CHECK-SAME:                                        %[[B_ADDR:.*]]: !fir.ref<i32> {fir.bindc_name = "b"}) {
diff --git a/flang/test/Lower/OpenMP/parallel-lastprivate-clause-scalar.f90 b/flang/test/Lower/OpenMP/parallel-lastprivate-clause-scalar.f90
index 86309a24f91a0..531413c124f81 100644
--- a/flang/test/Lower/OpenMP/parallel-lastprivate-clause-scalar.f90
+++ b/flang/test/Lower/OpenMP/parallel-lastprivate-clause-scalar.f90
@@ -10,12 +10,12 @@
 !CHECK-DAG: %[[ARG1_DECL:.*]]:2 = hlfir.declare %[[ARG1_REF]] typeparams %[[FIVE]] dummy_scope %{{[0-9]+}} {uniq_name = "_QFlastprivate_characterEarg1"} : (!fir.ref<!fir.char<1,5>>, index, !fir.dscope) -> (!fir.ref<!fir.char<1,5>>, !fir.ref<!fir.char<1,5>>)
 
 !CHECK: omp.parallel {
-!CHECK-DAG: %[[ARG1_PVT:.*]] = fir.alloca !fir.char<1,5> {bindc_name = "arg1", pinned, {{.*}}}
-!CHECK-DAG: %[[ARG1_PVT_DECL:.*]]:2 = hlfir.declare %[[ARG1_PVT]] typeparams %[[FIVE]] {uniq_name = "_QFlastprivate_characterEarg1"} : (!fir.ref<!fir.char<1,5>>, index) -> (!fir.ref<!fir.char<1,5>>, !fir.ref<!fir.char<1,5>>)
 
 ! Check that we are accessing the clone inside the loop
-!CHECK: omp.wsloop {
+!CHECK: omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[ARG1_PVT:.*]], @{{.*}} %{{.*}}#0 -> %{{.*}} : !fir.ref<!fir.char<1,5>>, !{{.*}}) {
 !CHECK-NEXT: omp.loop_nest (%[[INDX_WS:.*]]) : {{.*}} {
+!CHECK: %[[FIVE:.*]] = arith.constant 5 : index
+!CHECK: %[[ARG1_PVT_DECL:.*]]:2 = hlfir.declare %[[ARG1_PVT]] typeparams %[[FIVE]] {uniq_name = "_QFlastprivate_characterEarg1"} : (!fir.ref<!fir.char<1,5>>, index) -> (!fir.ref<!fir.char<1,5>>, !fir.ref<!fir.char<1,5>>)
 !CHECK: %[[UNIT:.*]] = arith.constant 6 : i32
 !CHECK-NEXT: %[[ADDR:.*]] = fir.address_of(@_QQclX
 !CHECK-NEXT: %[[CVT0:.*]] = fir.convert %[[ADDR]] 
@@ -58,10 +58,9 @@ subroutine lastprivate_character(arg1)
 !CHECK: func @_QPlastprivate_int(%[[ARG1:.*]]: !fir.ref<i32> {fir.bindc_name = "arg1"}) {
 !CHECK: %[[ARG1_DECL:.*]]:2 = hlfir.declare %[[ARG1]] dummy_scope %{{[0-9]+}} {uniq_name = "_QFlastprivate_intEarg1"} : (!fir.ref<i32>, !fir.dscope) -> (!fir.ref<i32>, !fir.ref<i32>)
 !CHECK-DAG: omp.parallel  {
-!CHECK-DAG: %[[CLONE:.*]] = fir.alloca i32 {bindc_name = "arg1", pinned, {{.*}}}
-!CHECK-DAG: %[[CLONE_DECL:.*]]:2 = hlfir.declare %[[CLONE]] {uniq_name = "_QFlastprivate_intEarg1"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-!CHECK: omp.wsloop {
+!CHECK: omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[CLONE:.*]], @{{.*}} %{{.*}}#0 -> %{{.*}} : !fir.ref<i32>, !{{.*}}) {
 !CHECK-NEXT: omp.loop_nest (%[[INDX_WS:.*]]) : {{.*}} {
+!CHECK:      %[[CLONE_DECL:.*]]:2 = hlfir.declare %[[CLONE]] {uniq_name = "_QFlastprivate_intEarg1"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 
 ! Testing last iteration check
 !CHECK: %[[V:.*]] = arith.addi %[[INDX_WS]], %{{.*}} : i32
@@ -98,12 +97,10 @@ subroutine lastprivate_int(arg1)
 !CHECK: %[[ARG1_DECL:.*]]:2 = hlfir.declare %[[ARG1]] dummy_scope %{{[0-9]+}} {uniq_name = "_QFmult_lastprivate_intEarg1"} : (!fir.ref<i32>, !fir.dscope) -> (!fir.ref<i32>, !fir.ref<i32>)
 !CHECK: %[[ARG2_DECL:.*]]:2 = hlfir.declare %[[ARG2]] dummy_scope %{{[0-9]+}} {uniq_name = "_QFmult_lastprivate_intEarg2"} : (!fir.ref<i32>, !fir.dscope) -> (!fir.ref<i32>, !fir.ref<i32>)
 !CHECK: omp.parallel  {
-!CHECK-DAG: %[[CLONE1:.*]] = fir.alloca i32 {bindc_name = "arg1", pinned, {{.*}}}
+!CHECK: omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[CLONE1:.*]], @{{.*}} %{{.*}}#0 -> %[[CLONE2:.*]], @{{.*}} %{{.*}}#0 -> %{{.*}} : !fir.ref<i32>, !fir.ref<i32>, !{{.*}}) {
+!CHECK-NEXT: omp.loop_nest (%[[INDX_WS:.*]]) : {{.*}} {
 !CHECK-DAG: %[[CLONE1_DECL:.*]]:2 = hlfir.declare %[[CLONE1]] {uniq_name = "_QFmult_lastprivate_intEarg1"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-!CHECK-DAG: %[[CLONE2:.*]] = fir.alloca i32 {bindc_name = "arg2", pinned, {{.*}}}
 !CHECK-DAG: %[[CLONE2_DECL:.*]]:2 = hlfir.declare %[[CLONE2]] {uniq_name = "_QFmult_lastprivate_intEarg2"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-!CHECK: omp.wsloop {
-!CHECK-NEXT: omp.loop_nest (%[[INDX_WS:.*]]) : {{.*}} {
 
 ! Testing last iteration check
 !CHECK: %[[V:.*]] = arith.addi %[[INDX_WS]], %{{.*}} : i32
@@ -142,12 +139,10 @@ subroutine mult_lastprivate_int(arg1, arg2)
 !CHECK: %[[ARG1_DECL:.*]]:2 = hlfir.declare %[[ARG1]] dummy_scope %{{[0-9]+}} {uniq_name = "_QFmult_lastprivate_int2Earg1"} : (!fir.ref<i32>, !fir.dscope) -> (!fir.ref<i32>, !fir.ref<i32>)
 !CHECK: %[[ARG2_DECL:.*]]:2 = hlfir.declare %[[ARG2]] dummy_scope %{{[0-9]+}} {uniq_name = "_QFmult_lastprivate_int2Earg2"} : (!fir.ref<i32>, !fir.dscope) -> (!fir.ref<i32>, !fir.ref<i32>)
 !CHECK: omp.parallel  {
-!CHECK-DAG: %[[CLONE2:.*]] = fir.alloca i32 {bindc_name = "arg2", pinned, {{.*}}}
-!CHECK-DAG: %[[CLONE2_DECL:.*]]:2 = hlfir.declare %[[CLONE2]] {uniq_name = "_QFmult_lastprivate_int2Earg2"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-!CHECK-DAG: %[[CLONE1:.*]] = fir.alloca i32 {bindc_name = "arg1", pinned, {{.*}}}
-!CHECK-DAG: %[[CLONE1_DECL:.*]]:2 = hlfir.declare %[[CLONE1]] {uniq_name = "_QFmult_lastprivate_int2Earg1"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-!CHECK: omp.wsloop {
+!CHECK: omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[CLONE1:.*]], @{{.*}} %{{.*}}#0 -> %[[CLONE2:.*]], @{{.*}} %{{.*}}#0 -> %{{.*}} : !fir.ref<i32>, !fir.ref<i32>, !{{.*}}) {
 !CHECK-NEXT: omp.loop_nest (%[[INDX_WS:.*]]) : {{.*}} {
+!CHECK-DAG: %[[CLONE1_DECL:.*]]:2 = hlfir.declare %[[CLONE1]] {uniq_name = "_QFmult_lastprivate_int2Earg1"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+!CHECK-DAG: %[[CLONE2_DECL:.*]]:2 = hlfir.declare %[[CLONE2]] {uniq_name = "_QFmult_lastprivate_int2Earg2"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 
 !Testing last iteration check
 !CHECK: %[[V:.*]] = arith.addi %[[INDX_WS]], %{{.*}} : i32
@@ -187,16 +182,11 @@ subroutine mult_lastprivate_int2(arg1, arg2)
 !CHECK:    %[[ARG2_DECL:.*]]:2 = hlfir.declare %[[ARG2]] dummy_scope %{{[0-9]+}} {uniq_name = "_QFfirstpriv_lastpriv_intEarg2"} : (!fir.ref<i32>, !fir.dscope) -> (!fir.ref<i32>, !fir.ref<i32>)
 !CHECK: omp.parallel  {
 ! Firstprivate update
-!CHECK: %[[CLONE1:.*]] = fir.alloca i32 {bindc_name = "arg1", pinned, {{.*}}}
-!CHECK: %[[CLONE1_DECL:.*]]:2 = hlfir.declare %[[CLONE1]] {uniq_name = "_QFfirstpriv_lastpriv_intEarg1"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-!CHECK: %[[FPV_LD:.*]] = fir.load %[[ARG1_DECL]]#0 : !fir.ref<i32>
-!CHECK: hlfir.assign %[[FPV_LD]] to %[[CLONE1_DECL]]#0 : i32, !fir.ref<i32>
-! Lastprivate Allocation
-!CHECK: %[[CLONE2:.*]] = fir.alloca i32 {bindc_name = "arg2", pinned, {{.*}}}
-!CHECK: %[[CLONE2_DECL:.*]]:2 = hlfir.declare %[[CLONE2]] {uniq_name = "_QFfirstpriv_lastpriv_intEarg2"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 !CHECK-NOT: omp.barrier
-!CHECK: omp.wsloop {
+!CHECK: omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[CLONE1:.*]], @{{.*}} %{{.*}}#0 -> %[[CLONE2:.*]], @{{.*}} %{{.*}}#0 -> %{{.*}} : !fir.ref<i32>, !fir.ref<i32>, !{{.*}}) {
 !CHECK-NEXT: omp.loop_nest (%[[INDX_WS:.*]]) : {{.*}} {
+!CHECK: %[[CLONE1_DECL:.*]]:2 = hlfir.declare %[[CLONE1]] {uniq_name = "_QFfirstpriv_lastpriv_intEarg1"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+!CHECK: %[[CLONE2_DECL:.*]]:2 = hlfir.declare %[[CLONE2]] {uniq_name = "_QFfirstpriv_lastpriv_intEarg2"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 
 ! Testing last iteration check
 !CHECK: %[[V:.*]] = arith.addi %[[INDX_WS]], %{{.*}} : i32
@@ -234,17 +224,14 @@ subroutine firstpriv_lastpriv_int(arg1, arg2)
 !CHECK: omp.parallel  {
 
 ! Firstprivate update
-!CHECK: %[[CLONE1:.*]] = fir.alloca i32 {bindc_name = "arg1", pinned, {{.*}}}
-!CHECK: %[[CLONE1_DECL:.*]]:2 = hlfir.declare %[[CLONE1]] {uniq_name = "_QFfirstpriv_lastpriv_int2Earg1"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-!CHECK-NEXT: %[[FPV_LD:.*]] = fir.load %[[ARG1_DECL]]#0 : !fir.ref<i32>
-!CHECK-NEXT: hlfir.assign %[[FPV_LD]] to %[[CLONE1_DECL]]#0 : i32, !fir.ref<i32>
 
-!CHECK-NEXT: %[[IV:.*]] = fir.alloca i32 {bindc_name = "n", pinned, {{.*}}}
-!CHECK-NEXT: hlfir.declare %[[IV]]
 
 !CHECK-NEXT: omp.barrier
-!CHECK: omp.wsloop {
+!CHECK: omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[CLONE1:.*]], @{{.*}} %{{.*}}#0 -> %[[IV:.*]] : !fir.ref<i32>, !fir.ref<i32>) {
 !CHECK-NEXT: omp.loop_nest (%[[INDX_WS:.*]]) : {{.*}} {
+!CHECK: %[[CLONE1_DECL:.*]]:2 = hlfir.declare %[[CLONE1]] {uniq_name = "_QFfirstpriv_lastpriv_int2Earg1"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+
+!CHECK-NEXT: hlfir.declare %[[IV]]
 ! Testing last iteration check
 !CHECK: %[[V:.*]] = arith.addi %[[INDX_WS]], %{{.*}} : i32
 !CHECK: %[[C0:.*]] = arith.constant 0 : i32
diff --git a/flang/test/Lower/OpenMP/parallel-private-clause-fixes.f90 b/flang/test/Lower/OpenMP/parallel-private-clause-fixes.f90
index 99323e69113bc..2c34e33a0f4c1 100644
--- a/flang/test/Lower/OpenMP/parallel-private-clause-fixes.f90
+++ b/flang/test/Lower/OpenMP/parallel-private-clause-fixes.f90
@@ -3,30 +3,23 @@
 ! RUN: bbc -fopenmp -emit-hlfir %s -o - \
 ! RUN: | FileCheck %s
 
-! CHECK:  omp.private {type = private} @[[BOX_HEAP_CHAR_PRIVATIZER:_QFsub01Eaaa_private_ref_box_heap_c8xU]] : !fir.ref<!fir.box<!fir.heap<!fir.char<1,?>>>> alloc {
-! CHECK:  ^bb0(%[[ORIG_REF:.*]]: !fir.ref<!fir.box<!fir.heap<!fir.char<1,?>>>>):
-! CHECK:             %[[VAL_4:.*]] = fir.alloca !fir.box<!fir.heap<!fir.char<1,?>>> {bindc_name = "aaa", pinned, uniq_name = "_QFsub01Eaaa"}
+! CHECK:  omp.private {type = private} @[[BOX_HEAP_CHAR_PRIVATIZER:_QFsub01Eaaa_private_box_heap_c8xU]] : !fir.box<!fir.heap<!fir.char<1,?>>> init {
+! CHECK:  ^bb0(%[[ORIG_REF:.*]]: !fir.ref<!fir.box<!fir.heap<!fir.char<1,?>>>>, %[[VAL_4:.*]]: !fir.ref<!fir.box<!fir.heap<!fir.char<1,?>>>>):
 ! CHECK:             %[[VAL_5:.*]] = fir.load %[[ORIG_REF]] : !fir.ref<!fir.box<!fir.heap<!fir.char<1,?>>>>
+! CHECK:             %[[ELESIZE:.*]] = fir.box_elesize %[[VAL_5]]
 ! CHECK:             %[[VAL_6:.*]] = fir.box_addr %[[VAL_5]] : (!fir.box<!fir.heap<!fir.char<1,?>>>) -> !fir.heap<!fir.char<1,?>>
 ! CHECK:             %[[VAL_7:.*]] = fir.convert %[[VAL_6]] : (!fir.heap<!fir.char<1,?>>) -> i64
 ! CHECK:             %[[VAL_8:.*]] = arith.constant 0 : i64
-! CHECK:             %[[VAL_9:.*]] = arith.cmpi ne, %[[VAL_7]], %[[VAL_8]] : i64
+! CHECK:             %[[VAL_9:.*]] = arith.cmpi eq, %[[VAL_7]], %[[VAL_8]] : i64
 ! CHECK:             fir.if %[[VAL_9]] {
-! CHECK:               %[[ELEM_SIZE:.*]] = fir.box_elesize %{{.*}} : (!fir.box<!fir.heap<!fir.char<1,?>>>) -> index
-! CHECK:               %[[VAL_10:.*]] = arith.constant 0 : index
-! CHECK:               %[[VAL_11:.*]] = arith.cmpi sgt, %[[ELEM_SIZE]], %[[VAL_10]] : index
-! CHECK:               %[[VAL_12:.*]] = arith.select %[[VAL_11]], %[[ELEM_SIZE]], %[[VAL_10]] : index
-! CHECK:               %[[VAL_13:.*]] = fir.allocmem !fir.char<1,?>(%[[VAL_12]] : index) {fir.must_be_heap = true, uniq_name = "_QFsub01Eaaa.alloc"}
-! CHECK:               %[[VAL_14:.*]] = fir.embox %[[VAL_13]] typeparams %[[VAL_12]] : (!fir.heap<!fir.char<1,?>>, index) -> !fir.box<!fir.heap<!fir.char<1,?>>>
-! CHECK:               fir.store %[[VAL_14]] to %[[VAL_4]] : !fir.ref<!fir.box<!fir.heap<!fir.char<1,?>>>>
-! CHECK:             } else {
-! CHECK:               %[[VAL_15:.*]] = fir.zero_bits !fir.heap<!fir.char<1,?>>
-! CHECK:               %[[VAL_16:.*]] = arith.constant 0 : index
-! CHECK:               %[[VAL_17:.*]] = fir.embox %[[VAL_15]] typeparams %[[VAL_16]] : (!fir.heap<!fir.char<1,?>>, index) -> !fir.box<!fir.heap<!fir.char<1,?>>>
+! CHECK:               %[[VAL_17:.*]] = fir.embox %[[VAL_6]] typeparams %[[ELESIZE]] : (!fir.heap<!fir.char<1,?>>, index) -> !fir.box<!fir.heap<!fir.char<1,?>>>
 ! CHECK:               fir.store %[[VAL_17]] to %[[VAL_4]] : !fir.ref<!fir.box<!fir.heap<!fir.char<1,?>>>>
+! CHECK:             } else {
+! CHECK:               %[[VAL_13:.*]] = fir.allocmem !fir.char<1,?>(%[[ELESIZE]] : index)
+! CHECK:               %[[VAL_14:.*]] = fir.embox %[[VAL_13]] typeparams %[[ELESIZE]] : (!fir.heap<!fir.char<1,?>>, index) -> !fir.box<!fir.heap<!fir.char<1,?>>>
+! CHECK:               fir.store %[[VAL_14]] to %[[VAL_4]] : !fir.ref<!fir.box<!fir.heap<!fir.char<1,?>>>>
 ! CHECK:             }
-! CHECK:             %[[VAL_18:.*]]:2 = hlfir.declare %[[VAL_4]] {fortran_attrs = #{{.*}}<allocatable>, uniq_name = "_QFsub01Eaaa"} : (!fir.ref<!fir.box<!fir.heap<!fir.char<1,?>>>>) -> (!fir.ref<!fir.box<!fir.heap<!fir.char<1,?>>>>, !fir.ref<!fir.box<!fir.heap<!fir.char<1,?>>>>)
-!CHECK:              omp.yield(%[[VAL_18]]#0 : !fir.ref<!fir.box<!fir.heap<!fir.char<1,?>>>>)
+!CHECK:              omp.yield(%[[VAL_4]] : !fir.ref<!fir.box<!fir.heap<!fir.char<1,?>>>>)
 !CHECK:  } dealloc {
 !CHECK:  ^bb0(%[[ORIG_REF:.*]]: !fir.ref<!fir.box<!fir.heap<!fir.char<1,?>>>>):
 ! CHECK:             %[[VAL_19:.*]] = fir.load %[[ORIG_REF]] : !fir.ref<!fir.box<!fir.heap<!fir.char<1,?>>>>
@@ -35,13 +28,7 @@
 ! CHECK:             %[[VAL_22:.*]] = arith.constant 0 : i64
 ! CHECK:             %[[VAL_23:.*]] = arith.cmpi ne, %[[VAL_21]], %[[VAL_22]] : i64
 ! CHECK:             fir.if %[[VAL_23]] {
-! CHECK:               %[[VAL_24:.*]] = fir.load %[[ORIG_REF]] : !fir.ref<!fir.box<!fir.heap<!fir.char<1,?>>>>
-! CHECK:               %[[VAL_25:.*]] = fir.box_addr %[[VAL_24]] : (!fir.box<!fir.heap<!fir.char<1,?>>>) -> !fir.heap<!fir.char<1,?>>
-! CHECK:               fir.freemem %[[VAL_25]] : !fir.heap<!fir.char<1,?>>
-! CHECK:               %[[VAL_26:.*]] = fir.zero_bits !fir.heap<!fir.char<1,?>>
-! CHECK:               %[[VAL_27:.*]] = arith.constant 0 : index
-! CHECK:               %[[VAL_28:.*]] = fir.embox %[[VAL_26]] typeparams %[[VAL_27]] : (!fir.heap<!fir.char<1,?>>, index) -> !fir.box<!fir.heap<!fir.char<1,?>>>
-! CHECK:               fir.store %[[VAL_28]] to %[[ORIG_REF]] : !fir.ref<!fir.box<!fir.heap<!fir.char<1,?>>>>
+! CHECK:               fir.freemem %[[VAL_20]] : !fir.heap<!fir.char<1,?>>
 !CHECK:    }
 !CHECK:    omp.yield
 !CHECK:  }
@@ -56,17 +43,15 @@
 ! CHECK-DAG:         %[[VAL_2:.*]] = fir.alloca i32 {bindc_name = "x", uniq_name = "_QFmultiple_private_fixEx"}
 ! CHECK-DAG:         %[[X_DECL:.*]]:2 = hlfir.declare %[[VAL_2]] {uniq_name = "_QFmultiple_private_fixEx"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:         omp.parallel {
-! CHECK-DAG:           %[[PRIV_I:.*]] = fir.alloca i32 {bindc_name = "i", pinned, {{.*}}}
-! CHECK-DAG:           %[[PRIV_I_DECL:.*]]:2 = hlfir.declare %[[PRIV_I]] {uniq_name = "_QFmultiple_private_fixEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-! CHECK-DAG:           %[[PRIV_J:.*]] = fir.alloca i32 {bindc_name = "j", pinned, uniq_name = "_QFmultiple_private_fixEj"}
-! CHECK-DAG:           %[[PRIV_J_DECL:.*]]:2 = hlfir.declare %[[PRIV_J]] {uniq_name = "_QFmultiple_private_fixEj"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-! CHECK-DAG:           %[[PRIV_X:.*]] = fir.alloca i32 {bindc_name = "x", pinned, {{.*}}}
-! CHECK-DAG:           %[[PRIV_X_DECL:.*]]:2 = hlfir.declare %[[PRIV_X]] {uniq_name = "_QFmultiple_private_fixEx"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+
 ! CHECK:           %[[ONE:.*]] = arith.constant 1 : i32
 ! CHECK:           %[[VAL_3:.*]] = fir.load %[[GAMA_DECL]]#0 : !fir.ref<i32>
 ! CHECK:           %[[VAL_5:.*]] = arith.constant 1 : i32
-! CHECK:           omp.wsloop {
+! CHECK:           omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[PRIV_J:.*]], @{{.*}} %{{.*}}#0 -> %[[PRIV_X:.*]], @{{.*}} %{{.*}}#0 -> %[[PRIV_I:.*]] : !fir.ref<i32>, !fir.ref<i32>, !fir.ref<i32>) {
 ! CHECK-NEXT:        omp.loop_nest (%[[VAL_6:.*]]) : i32 = (%[[ONE]]) to (%[[VAL_3]]) inclusive step (%[[VAL_5]]) {
+! CHECK-DAG:           %[[PRIV_I_DECL:.*]]:2 = hlfir.declare %[[PRIV_I]] {uniq_name = "_QFmultiple_private_fixEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+! CHECK-DAG:           %[[PRIV_J_DECL:.*]]:2 = hlfir.declare %[[PRIV_J]] {uniq_name = "_QFmultiple_private_fixEj"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+! CHECK-DAG:           %[[PRIV_X_DECL:.*]]:2 = hlfir.declare %[[PRIV_X]] {uniq_name = "_QFmultiple_private_fixEx"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:               fir.store %[[VAL_6]] to %[[PRIV_I_DECL]]#1 : !fir.ref<i32>
 ! CHECK:               %[[VAL_7:.*]] = arith.constant 1 : i32
 ! CHECK:               %[[VAL_8:.*]] = fir.convert %[[VAL_7]] : (i32) -> index
diff --git a/flang/test/Lower/OpenMP/parallel-private-clause-str.f90 b/flang/test/Lower/OpenMP/parallel-private-clause-str.f90
index 70cb4a9fde3bd..44cb08e029aa1 100644
--- a/flang/test/Lower/OpenMP/parallel-private-clause-str.f90
+++ b/flang/test/Lower/OpenMP/parallel-private-clause-str.f90
@@ -8,45 +8,38 @@
 ! RUN: %flang_fc1 -emit-hlfir -fopenmp -o - %s 2>&1 \
 ! RUN: | FileCheck %s
 
-!CHECK:  omp.private {type = private} @[[STR_ARR_PRIVATIZER:_QFtest_allocatable_string_arrayEc_private_ref_box_heap_Uxc8xU]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?x!fir.char<1,?>>>>> alloc {
-!CHECK:  ^bb0(%[[ORIG_REF:.*]]: !fir.ref<!fir.box<!fir.heap<!fir.array<?x!fir.char<1,?>>>>>):
-!CHECK:      %[[C_PVT_BOX_REF:.*]] = fir.alloca !fir.box<!fir.heap<!fir.array<?x!fir.char<1,?>>>> {bindc_name = "c", pinned, uniq_name = "_QFtest_allocatable_string_arrayEc"}
+!CHECK:  omp.private {type = private} @[[STR_ARR_PRIVATIZER:_QFtest_allocatable_string_arrayEc_private_box_heap_Uxc8xU]] : [[TYPE:.*]] init {
+!CHECK:  ^bb0(%[[ORIG_REF:.*]]: !fir.ref<[[TYPE]]>, %[[C_PVT_BOX_REF:.*]]: !fir.ref<[[TYPE]]>):
 !CHECK:      %{{.*}} = fir.load %[[ORIG_REF]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?x!fir.char<1,?>>>>>
 !CHECK:      fir.if %{{.*}} {
-!CHECK:        %[[C_PVT_ALLOC:.*]] = fir.allocmem !fir.array<?x!fir.char<1,?>>(%{{.*}} : index), %{{.*}} {fir.must_be_heap = true, uniq_name = "_QFtest_allocatable_string_arrayEc.alloc"}
-!CHECK:        %[[C_PVT_BOX:.*]] = fir.embox %[[C_PVT_ALLOC]](%{{.*}}) typeparams %{{.*}} : (!fir.heap<!fir.array<?x!fir.char<1,?>>>, !fir.shapeshift<1>, index) -> !fir.box<!fir.heap<!fir.array<?x!fir.char<1,?>>>>
+!CHECK:      } else {
+!CHECK:        %[[C_PVT_ALLOC:.*]] = fir.allocmem !fir.array<?x!fir.char<1,?>>(%{{.*}} : index), %{{.*}}
+!CHECK:        %[[C_PVT_BOX:.*]] = fir.rebox
 !CHECK:        fir.store %[[C_PVT_BOX]] to %[[C_PVT_BOX_REF]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?x!fir.char<1,?>>>>>
 !CHECK:      }
-!CHECK:      %[[C_PVT_DECL:.*]]:2 = hlfir.declare %[[C_PVT_BOX_REF]] {fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "_QFtest_allocatable_string_arrayEc"} : (!fir.ref<!fir.box<!fir.heap<!fir.array<?x!fir.char<1,?>>>>>) -> (!fir.ref<!fir.box<!fir.heap<!fir.array<?x!fir.char<1,?>>>>>, !fir.ref<!fir.box<!fir.heap<!fir.array<?x!fir.char<1,?>>>>>)
-!CHECK:      omp.yield(%[[C_PVT_DECL]]#0 : !fir.ref<!fir.box<!fir.heap<!fir.array<?x!fir.char<1,?>>>>>)
+!CHECK:      omp.yield(%[[C_PVT_BOX_REF]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?x!fir.char<1,?>>>>>)
 !CHECK:  } dealloc {
 !CHECK:  ^bb0(%[[ORIG_REF:.*]]: !fir.ref<!fir.box<!fir.heap<!fir.array<?x!fir.char<1,?>>>>>):
 !CHECK:      %{{.*}} = fir.load %[[ORIG_REF]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?x!fir.char<1,?>>>>>
 !CHECK:      fir.if %{{.*}} {
-!CHECK:        %[[C_PVT_BOX:.*]] = fir.load %[[ORIG_REF]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?x!fir.char<1,?>>>>>
-!CHECK:        %[[C_PVT_ADDR:.*]] = fir.box_addr %[[C_PVT_BOX]] : (!fir.box<!fir.heap<!fir.array<?x!fir.char<1,?>>>>) -> !fir.heap<!fir.array<?x!fir.char<1,?>>>
-!CHECK:        fir.freemem %[[C_PVT_ADDR]] : !fir.heap<!fir.array<?x!fir.char<1,?>>>
+!CHECK:        fir.freemem %{{.*}} : !fir.heap<!fir.array<?x!fir.char<1,?>>>
 !CHECK:      }
 !CHECK:      omp.yield
 !CHECK:  }
 
-!CHECK:  omp.private {type = private} @[[STR_PRIVATIZER:_QFtest_allocatable_stringEc_private_ref_box_heap_c8xU]] : !fir.ref<!fir.box<!fir.heap<!fir.char<1,?>>>> alloc {
-!CHECK:  ^bb0(%[[ORIG_REF:.*]]: !fir.ref<!fir.box<!fir.heap<!fir.char<1,?>>>>):
-!CHECK:      %[[C_PVT_BOX_REF:.*]] = fir.alloca !fir.box<!fir.heap<!fir.char<1,?>>> {bindc_name = "c", pinned, uniq_name = "_QFtest_allocatable_stringEc"}
+!CHECK:  omp.private {type = private} @[[STR_PRIVATIZER:_QFtest_allocatable_stringEc_private_box_heap_c8xU]] : [[TYPE:!fir.box<!fir.heap<!fir.char<1,\?>>>]] init {
+!CHECK:  ^bb0(%[[ORIG_REF:.*]]: !fir.ref<[[TYPE]]>, %[[C_PVT_BOX_REF:.*]]: !fir.ref<[[TYPE]]>):
 !CHECK:      %[[C_BOX:.*]] = fir.load %[[ORIG_REF]] : !fir.ref<!fir.box<!fir.heap<!fir.char<1,?>>>>
 !CHECK:      fir.if %{{.*}} {
-!CHECK:        %[[C_PVT_MEM:.*]] = fir.allocmem !fir.char<1,?>(%{{.*}} : index) {fir.must_be_heap = true, uniq_name = "_QFtest_allocatable_stringEc.alloc"}
+!CHECK:        %[[C_PVT_MEM:.*]] = fir.allocmem !fir.char<1,?>(%{{.*}} : index)
 !CHECK:        %[[C_PVT_BOX:.*]] = fir.embox %[[C_PVT_MEM]] typeparams %{{.*}} : (!fir.heap<!fir.char<1,?>>, index) -> !fir.box<!fir.heap<!fir.char<1,?>>>
 !CHECK:        fir.store %[[C_PVT_BOX]] to %[[C_PVT_BOX_REF]] : !fir.ref<!fir.box<!fir.heap<!fir.char<1,?>>>>
 !CHECK:      }
-!CHECK:      %[[C_PVT_DECL:.*]]:2 = hlfir.declare %[[C_PVT_BOX_REF]] {fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "_QFtest_allocatable_stringEc"} : (!fir.ref<!fir.box<!fir.heap<!fir.char<1,?>>>>) -> (!fir.ref<!fir.box<!fir.heap<!fir.char<1,?>>>>, !fir.ref<!fir.box<!fir.heap<!fir.char<1,?>>>>)
-!CHECK:      omp.yield(%[[C_PVT_DECL]]#0 : !fir.ref<!fir.box<!fir.heap<!fir.char<1,?>>>>)
+!CHECK:      omp.yield(%[[C_PVT_BOX_REF]] : !fir.ref<!fir.box<!fir.heap<!fir.char<1,?>>>>)
 !CHECK:  } dealloc {
 !CHECK:  ^bb0(%[[ORIG_REF:.*]]: !fir.ref<!fir.box<!fir.heap<!fir.char<1,?>>>>):
 !CHECK:      fir.if %{{.*}} {
-!CHECK:        %[[C_PVT_BOX:.*]] = fir.load %[[ORIG_REF]] : !fir.ref<!fir.box<!fir.heap<!fir.char<1,?>>>>
-!CHECK:        %[[C_PVT_BOX_ADDR:.*]] = fir.box_addr %[[C_PVT_BOX]] : (!fir.box<!fir.heap<!fir.char<1,?>>>) -> !fir.heap<!fir.char<1,?>>
-!CHECK:        fir.freemem %[[C_PVT_BOX_ADDR]] : !fir.heap<!fir.char<1,?>>
+!CHECK:        fir.freemem %{{.*}} : !fir.heap<!fir.char<1,?>>
 !CHECK:      }
 !CHECK:      omp.yield
 !CHECK:  }
diff --git a/flang/test/Lower/OpenMP/parallel-private-clause.f90 b/flang/test/Lower/OpenMP/parallel-private-clause.f90
index 7114314df05d3..e4dd4189c5e87 100644
--- a/flang/test/Lower/OpenMP/parallel-private-clause.f90
+++ b/flang/test/Lower/OpenMP/parallel-private-clause.f90
@@ -5,12 +5,10 @@
 ! RUN: bbc --use-desc-for-alloc=false -fopenmp -emit-hlfir %s -o - \
 ! RUN: | FileCheck %s --check-prefix=FIRDialect
 
-! FIRDialect: omp.private {type = private} @_QFsimd_loop_1Er_private_ref_box_heap_f32 {{.*}} alloc {
-! FIRDialect:     [[R:%.*]] = fir.alloca !fir.box<!fir.heap<f32>> {bindc_name = "r", pinned, uniq_name = "{{.*}}Er"}
+! FIRDialect: omp.private {type = private} @_QFsimd_loop_1Er_private_box_heap_f32 : !fir.box<!fir.heap<f32>> init {
+! FIRDialect:     fir.store {{%.*}} to [[R:.*]] : !fir.ref<!fir.box<!fir.heap<f32>>>
 ! FIRDialect:     fir.store {{%.*}} to [[R]] : !fir.ref<!fir.box<!fir.heap<f32>>>
-! FIRDialect:     fir.store {{%.*}} to [[R]] : !fir.ref<!fir.box<!fir.heap<f32>>>
-! FIRDialect:     [[R_DECL:%.*]]:2 = hlfir.declare [[R]] {fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "{{.*}}r"} : (!fir.ref<!fir.box<!fir.heap<f32>>>) -> (!fir.ref<!fir.box<!fir.heap<f32>>>, !fir.ref<!fir.box<!fir.heap<f32>>>)
-! FIRDialect:     omp.yield([[R_DECL]]#0 : !fir.ref<!fir.box<!fir.heap<f32>>>)
+! FIRDialect:     omp.yield([[R]] : !fir.ref<!fir.box<!fir.heap<f32>>>)
 ! FIRDialect:   } dealloc {
 ! FIRDialect:  ^bb0([[R_DECL:%.*]]: !fir.ref<!fir.box<!fir.heap<f32>>>):
 ! FIRDialect:     {{%.*}} = fir.load [[R_DECL]] : !fir.ref<!fir.box<!fir.heap<f32>>>
@@ -18,36 +16,31 @@
 ! FIRDialect:     [[LD:%.*]] = fir.load [[R_DECL]] : !fir.ref<!fir.box<!fir.heap<f32>>>
 ! FIRDialect:     [[AD:%.*]] = fir.box_addr [[LD]] : (!fir.box<!fir.heap<f32>>) -> !fir.heap<f32>
 ! FIRDialect:     fir.freemem [[AD]] : !fir.heap<f32>
-! FIRDialect:     fir.store {{%.*}} to [[R_DECL]] : !fir.ref<!fir.box<!fir.heap<f32>>>
 ! FIRDialect:     omp.yield
 ! FIRDialect:   }
 
-!FIRDialect: omp.private {type = private} @[[DERIVED_PRIVATIZER:_QFprivate_clause_derived_typeEt_private_ref_rec__QFprivate_clause_derived_typeTmy_type]] : !fir.ref<!fir.type<_QFprivate_clause_derived_typeTmy_type{t_i:i32,t_arr:!fir.array<5xi32>}>> alloc {
-!FIRDialect:   ^bb0(%{{.*}}: !fir.ref<!fir.type<_QFprivate_clause_derived_typeTmy_type{t_i:i32,t_arr:!fir.array<5xi32>}>>):
-!FIRDialect:     %[[PRIV_ALLOC:.*]] = fir.alloca !fir.type<_QFprivate_clause_derived_typeTmy_type{t_i:i32,t_arr:!fir.array<5xi32>}> {bindc_name = "t", pinned, uniq_name = "_QFprivate_clause_derived_typeEt"}
-!FIRDialect:     %[[PRIV_DECL:.*]]:2 = hlfir.declare %[[PRIV_ALLOC]] {uniq_name = "_QFprivate_clause_derived_typeEt"} : (!fir.ref<!fir.type<_QFprivate_clause_derived_typeTmy_type{t_i:i32,t_arr:!fir.array<5xi32>}>>) -> (!fir.ref<!fir.type<_QFprivate_clause_derived_typeTmy_type{t_i:i32,t_arr:!fir.array<5xi32>}>>, !fir.ref<!fir.type<_QFprivate_clause_derived_typeTmy_type{t_i:i32,t_arr:!fir.array<5xi32>}>>)
-!FIRDialect:     omp.yield(%[[PRIV_DECL]]#0 : !fir.ref<!fir.type<_QFprivate_clause_derived_typeTmy_type{t_i:i32,t_arr:!fir.array<5xi32>}>>)
-!FIRDialect: }
+!FIRDialect: omp.private {type = private} @[[DERIVED_PRIVATIZER:_QFprivate_clause_derived_typeEt_private_rec__QFprivate_clause_derived_typeTmy_type]] : !fir.type<_QFprivate_clause_derived_typeTmy_type{t_i:i32,t_arr:!fir.array<5xi32>}>
 
 !FIRDialect: func @_QPprivate_clause(%[[ARG1:.*]]: !fir.ref<i32> {fir.bindc_name = "arg1"}, %[[ARG2:.*]]: !fir.ref<!fir.array<10xi32>> {fir.bindc_name = "arg2"}, %[[ARG3:.*]]: !fir.boxchar<1> {fir.bindc_name = "arg3"}, %[[ARG4:.*]]: !fir.boxchar<1> {fir.bindc_name = "arg4"}) {
 !FIRDialect-DAG: %[[ALPHA:.*]] = fir.alloca i32 {bindc_name = "alpha", uniq_name = "{{.*}}alpha"}
-!FIRDialect-DAG: %[[ALPHA_DECL:.*]]:2 = hlfir.declare %[[ALPHA]] {uniq_name = "{{.*}}alpha"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 !FIRDialect-DAG: %[[ALPHA_ARRAY:.*]] = fir.alloca !fir.array<10xi32> {bindc_name = "alpha_array", uniq_name = "{{.*}}alpha_array"}
-!FIRDialect-DAG: %[[ALPHA_ARRAY_DECL:.*]]:2 = hlfir.declare %[[ALPHA_ARRAY]]({{.*}}) {uniq_name = "{{.*}}alpha_array"} : (!fir.ref<!fir.array<10xi32>>, !fir.shape<1>) -> (!fir.ref<!fir.array<10xi32>>, !fir.ref<!fir.array<10xi32>>)
 !FIRDialect-DAG: %[[BETA:.*]] = fir.alloca !fir.char<1,5> {bindc_name = "beta", uniq_name = "{{.*}}beta"}
-!FIRDialect-DAG: %[[BETA_DECL:.*]]:2 = hlfir.declare %[[BETA]] typeparams {{.*}} {uniq_name = "{{.*}}beta"} : (!fir.ref<!fir.char<1,5>>, index) -> (!fir.ref<!fir.char<1,5>>, !fir.ref<!fir.char<1,5>>)
 !FIRDialect-DAG: %[[BETA_ARRAY:.*]] = fir.alloca !fir.array<10x!fir.char<1,5>> {bindc_name = "beta_array", uniq_name = "{{.*}}beta_array"}
+
+!FIRDialect-DAG: %[[ALPHA_DECL:.*]]:2 = hlfir.declare %[[ALPHA]] {uniq_name = "{{.*}}alpha"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+!FIRDialect-DAG: %[[ALPHA_ARRAY_DECL:.*]]:2 = hlfir.declare %[[ALPHA_ARRAY]]({{.*}}) {uniq_name = "{{.*}}alpha_array"} : (!fir.ref<!fir.array<10xi32>>, !fir.shape<1>) -> (!fir.ref<!fir.array<10xi32>>, !fir.ref<!fir.array<10xi32>>)
+!FIRDialect-DAG: %[[BETA_DECL:.*]]:2 = hlfir.declare %[[BETA]] typeparams {{.*}} {uniq_name = "{{.*}}beta"} : (!fir.ref<!fir.char<1,5>>, index) -> (!fir.ref<!fir.char<1,5>>, !fir.ref<!fir.char<1,5>>)
 !FIRDialect-DAG: %[[BETA_ARRAY_DECL:.*]]:2 = hlfir.declare %[[BETA_ARRAY]]({{.*}}) typeparams {{.*}} {uniq_name = "{{.*}}beta_array"} : (!fir.ref<!fir.array<10x!fir.char<1,5>>>, !fir.shape<1>, index) -> (!fir.ref<!fir.array<10x!fir.char<1,5>>>, !fir.ref<!fir.array<10x!fir.char<1,5>>>)
 
-!FIRDialect-DAG: omp.parallel private(@{{.*}} %{{.*}}#0 -> %[[ALPHA_PVT:.*]], @{{.*}} %{{.*}}#0 -> %[[ALPHA_ARRAY_PVT:.*]], @{{.*}} %{{.*}}#0 -> %[[BETA_PVT:.*]], @{{.*}} %{{.*}}#0 -> %[[BETA_ARRAY_PVT:.*]], @{{.*}} %{{.*}}#0 -> %[[ARG1_PVT:.*]], @{{.*}} %{{.*}}#0 -> %[[ARG2_PVT:.*]], @{{.*}} %{{.*}}#0 -> %[[ARG3_PVT:.*]], @{{.*}} %{{.*}}#0 -> %[[ARG4_PVT:.*]] : {{.*}}) {
+!FIRDialect:     omp.parallel private(@{{.*}} %[[ALPHA_DECL]]#0 -> %[[ALPHA_PVT:.*]], @{{.*}} %{{.*}} -> %[[ALPHA_ARRAY_PVT:.*]], @{{.*}} %{{.*}}#0 -> %[[BETA_PVT:.*]], @{{.*}} %{{.*}} -> %[[BETA_ARRAY_PVT:.*]], @{{.*}} %{{.*}}#0 -> %[[ARG1_PVT:.*]], @{{.*}} %{{.*}} -> %[[ARG2_PVT:.*]], @{{.*}} %{{.*}}#0 -> %[[ARG3_PVT:.*]], @{{.*}} %{{.*}} -> %[[ARG4_PVT:.*]] : {{.*}}) {
 !FIRDialect-DAG:  %[[ALPHA_PVT_DECL:.*]]:2 = hlfir.declare %[[ALPHA_PVT]] {uniq_name = "{{.*}}alpha"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-!FIRDialect-DAG:  %[[ALPHA_ARRAY_PVT_DECL:.*]]:2 = hlfir.declare %[[ALPHA_ARRAY_PVT]]({{.*}}) {uniq_name = "{{.*}}alpha_array"} : (!fir.ref<!fir.array<10xi32>>, !fir.shape<1>) -> (!fir.ref<!fir.array<10xi32>>, !fir.ref<!fir.array<10xi32>>)
+!FIRDialect-DAG:  %[[ALPHA_ARRAY_PVT_DECL:.*]]:2 = hlfir.declare %[[ALPHA_ARRAY_PVT]] {uniq_name = "{{.*}}alpha_array"} :
 !FIRDialect-DAG:  %[[BETA_PVT_DECL:.*]]:2 = hlfir.declare %[[BETA_PVT]] typeparams {{.*}} {uniq_name = "{{.*}}beta"} : (!fir.ref<!fir.char<1,5>>, index) -> (!fir.ref<!fir.char<1,5>>, !fir.ref<!fir.char<1,5>>)
-!FIRDialect-DAG:  %[[BETA_ARRAY_PVT_DECL:.*]]:2 = hlfir.declare %[[BETA_ARRAY_PVT]]({{.*}}) typeparams {{.*}} {uniq_name = "{{.*}}beta_array"} : (!fir.ref<!fir.array<10x!fir.char<1,5>>>, !fir.shape<1>, index) -> (!fir.ref<!fir.array<10x!fir.char<1,5>>>, !fir.ref<!fir.array<10x!fir.char<1,5>>>)
+!FIRDialect-DAG:  %[[BETA_ARRAY_PVT_DECL:.*]]:2 = hlfir.declare %[[BETA_ARRAY_PVT]] {uniq_name = "{{.*}}beta_array"} :
 !FIRDialect-DAG:  %[[ARG1_PVT_DECL:.*]]:2 = hlfir.declare %[[ARG1_PVT]] {uniq_name = "{{.*}}arg1"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-!FIRDialect-DAG:  %[[ARG2_PVT_DECL:.*]]:2 = hlfir.declare %[[ARG2_PVT]]({{.*}}) {uniq_name = "{{.*}}arg2"} : (!fir.ref<!fir.array<10xi32>>, !fir.shape<1>) -> (!fir.ref<!fir.array<10xi32>>, !fir.ref<!fir.array<10xi32>>)
+!FIRDialect-DAG:  %[[ARG2_PVT_DECL:.*]]:2 = hlfir.declare %[[ARG2_PVT]] {uniq_name = "{{.*}}arg2"} :
 !FIRDialect-DAG:  %[[ARG3_PVT_DECL:.*]]:2 = hlfir.declare %[[ARG3_PVT]] typeparams {{.*}} {uniq_name = "{{.*}}arg3"} : (!fir.ref<!fir.char<1,5>>, index) -> (!fir.ref<!fir.char<1,5>>, !fir.ref<!fir.char<1,5>>)
-!FIRDialect-DAG:  %[[ARG4_PVT_DECL:.*]]:2 = hlfir.declare %[[ARG4_PVT]]({{.*}}) typeparams {{.*}} {uniq_name = "{{.*}}arg4"} : (!fir.ref<!fir.array<10x!fir.char<1,5>>>, !fir.shape<1>, index) -> (!fir.ref<!fir.array<10x!fir.char<1,5>>>, !fir.ref<!fir.array<10x!fir.char<1,5>>>)
+!FIRDialect-DAG:  %[[ARG4_PVT_DECL:.*]]:2 = hlfir.declare %[[ARG4_PVT]] {uniq_name = "{{.*}}arg4"} :
 !FIRDialect:     omp.terminator
 !FIRDialect:  }
 
@@ -271,17 +264,16 @@ subroutine simple_loop_1
   !$OMP PARALLEL PRIVATE(r)
   ! FIRDialect:      %[[R_DECL:.*]]:2 = hlfir.declare %[[R]] {fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "_QFsimple_loop_1Er"} : (!fir.ref<!fir.box<!fir.heap<f32>>>) -> (!fir.ref<!fir.box<!fir.heap<f32>>>, !fir.ref<!fir.box<!fir.heap<f32>>>)
 
-  ! FIRDialect:      %[[ALLOCA_IV:.*]] = fir.alloca i32 {{{.*}}, pinned, {{.*}}}
-  ! FIRDialect:      %[[ALLOCA_IV_DECL:.*]]:2 = hlfir.declare %[[ALLOCA_IV]] {uniq_name = "_QFsimple_loop_1Ei"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 
   ! FIRDialect:      %[[WS_LB:.*]] = arith.constant 1 : i32
   ! FIRDialect:      %[[WS_UB:.*]] = arith.constant 9 : i32
   ! FIRDialect:      %[[WS_STEP:.*]] = arith.constant 1 : i32
 
-  ! FIRDialect:      omp.wsloop {
+  ! FIRDialect:      omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[ALLOCA_IV:.*]] : !fir.ref<i32>) {
   ! FIRDialect-NEXT: omp.loop_nest (%[[I:.*]]) : i32 = (%[[WS_LB]]) to (%[[WS_UB]]) inclusive step (%[[WS_STEP]]) {
   !$OMP DO
   do i=1, 9
+  ! FIRDialect:      %[[ALLOCA_IV_DECL:.*]]:2 = hlfir.declare %[[ALLOCA_IV]] {uniq_name = "_QFsimple_loop_1Ei"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
   ! FIRDialect:      fir.store %[[I]] to %[[ALLOCA_IV_DECL]]#1 : !fir.ref<i32>
   ! FIRDialect:      %[[LOAD_IV:.*]] = fir.load %[[ALLOCA_IV_DECL]]#0 : !fir.ref<i32>
   ! FIRDialect:      fir.call @_FortranAioOutputInteger32({{.*}}, %[[LOAD_IV]]) {{.*}} : (!fir.ref<i8>, i32) -> i1
@@ -299,34 +291,23 @@ subroutine simple_loop_2
   real, allocatable :: r;
   ! FIRDialect:  omp.parallel
   !$OMP PARALLEL
-  ! FIRDialect:      [[R:%.*]] = fir.alloca !fir.box<!fir.heap<f32>> {bindc_name = "r", pinned, uniq_name = "{{.*}}Er"}
-  ! FIRDialect:      fir.store {{%.*}} to [[R]] : !fir.ref<!fir.box<!fir.heap<f32>>>
-  ! FIRDialect:      fir.store {{%.*}} to [[R]] : !fir.ref<!fir.box<!fir.heap<f32>>>
-  ! FIRDialect:      %[[R_DECL:.*]]:2 = hlfir.declare [[R]] {fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "{{.*}}Er"} : (!fir.ref<!fir.box<!fir.heap<f32>>>) -> (!fir.ref<!fir.box<!fir.heap<f32>>>, !fir.ref<!fir.box<!fir.heap<f32>>>)
-
-  ! FIRDialect:      %[[ALLOCA_IV:.*]] = fir.alloca i32 {{{.*}}, pinned, {{.*}}}
-  ! FIRDialect:      %[[ALLOCA_IV_DECL:.*]]:2 = hlfir.declare %[[ALLOCA_IV]] {uniq_name = "{{.*}}Ei"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 
   ! FIRDialect:      %[[WS_LB:.*]] = arith.constant 1 : i32
   ! FIRDialect:      %[[WS_UB:.*]] = arith.constant 9 : i32
   ! FIRDialect:      %[[WS_STEP:.*]] = arith.constant 1 : i32
 
-  ! FIRDialect:      omp.wsloop {
+  ! FIRDialect:      omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[R:.*]], @{{.*}} %{{.*}}#0 -> %[[ALLOCA_IV:.*]] : !fir.ref<!fir.box<!fir.heap<f32>>>, !fir.ref<i32>) {
   ! FIRDialect-NEXT: omp.loop_nest (%[[I:.*]]) : i32 = (%[[WS_LB]]) to (%[[WS_UB]]) inclusive step (%[[WS_STEP]]) {
   !$OMP DO PRIVATE(r)
   do i=1, 9
+  ! FIRDialect:     %[[R_DECL:.*]]:2 = hlfir.declare %[[R]] {fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "{{.*}}Er"} : (!fir.ref<!fir.box<!fir.heap<f32>>>) -> (!fir.ref<!fir.box<!fir.heap<f32>>>, !fir.ref<!fir.box<!fir.heap<f32>>>)
+  ! FIRDialect:     %[[ALLOCA_IV_DECL:.*]]:2 = hlfir.declare %[[ALLOCA_IV]] {uniq_name = "{{.*}}Ei"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
   ! FIRDialect:     fir.store %[[I]] to %[[ALLOCA_IV_DECL]]#1 : !fir.ref<i32>
   ! FIRDialect:     %[[LOAD_IV:.*]] = fir.load %[[ALLOCA_IV_DECL]]#0 : !fir.ref<i32>
   ! FIRDialect:     fir.call @_FortranAioOutputInteger32({{.*}}, %[[LOAD_IV]]) {{.*}}: (!fir.ref<i8>, i32) -> i1
     print*, i
   end do
   ! FIRDialect:     omp.yield
-  ! FIRDialect:     {{%.*}} = fir.load %[[R_DECL]]#0 : !fir.ref<!fir.box<!fir.heap<f32>>>
-  ! FIRDialect:     fir.if {{%.*}} {
-  ! FIRDialect:     [[LD:%.*]] = fir.load %[[R_DECL]]#0 : !fir.ref<!fir.box<!fir.heap<f32>>>
-  ! FIRDialect:     [[AD:%.*]] = fir.box_addr [[LD]] : (!fir.box<!fir.heap<f32>>) -> !fir.heap<f32>
-  ! FIRDialect:     fir.freemem [[AD]] : !fir.heap<f32>
-  ! FIRDialect:     fir.store {{%.*}} to %[[R_DECL]]#0 : !fir.ref<!fir.box<!fir.heap<f32>>>
   !$OMP END DO
   ! FIRDialect:  omp.terminator
   !$OMP END PARALLEL
@@ -337,35 +318,24 @@ subroutine simple_loop_3
   integer :: i
   real, allocatable :: r;
   ! FIRDialect:  omp.parallel
-
-  ! FIRDialect:      [[R:%.*]] = fir.alloca !fir.box<!fir.heap<f32>> {bindc_name = "r", pinned, uniq_name = "{{.*}}Er"}
-  ! FIRDialect:      fir.store {{%.*}} to [[R]] : !fir.ref<!fir.box<!fir.heap<f32>>>
-  ! FIRDialect:      fir.store {{%.*}} to [[R]] : !fir.ref<!fir.box<!fir.heap<f32>>>
-  ! FIRDialect:      [[R_DECL:%.*]]:2 = hlfir.declare [[R]] {fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "{{.*}}Er"} : (!fir.ref<!fir.box<!fir.heap<f32>>>) -> (!fir.ref<!fir.box<!fir.heap<f32>>>, !fir.ref<!fir.box<!fir.heap<f32>>>)
-
-  ! FIRDialect:      %[[ALLOCA_IV:.*]] = fir.alloca i32 {{{.*}}, pinned, {{.*}}}
-  ! FIRDialect:      %[[ALLOCA_IV_DECL:.*]]:2 = hlfir.declare %[[ALLOCA_IV]] {uniq_name = "{{.*}}Ei"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-
   ! FIRDialect:      %[[WS_LB:.*]] = arith.constant 1 : i32
   ! FIRDialect:      %[[WS_UB:.*]] = arith.constant 9 : i32
   ! FIRDialect:      %[[WS_STEP:.*]] = arith.constant 1 : i32
 
-  ! FIRDialect:      omp.wsloop {
+  ! FIRDialect:      omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[R:.*]], @{{.*}} %{{.*}}#0 -> %[[ALLOCA_IV:.*]] : !fir.ref<!fir.box<!fir.heap<f32>>>, !fir.ref<i32>) {
   ! FIRDialect-NEXT: omp.loop_nest (%[[I:.*]]) : i32 = (%[[WS_LB]]) to (%[[WS_UB]]) inclusive step (%[[WS_STEP]]) {
   !$OMP PARALLEL DO PRIVATE(r)
   do i=1, 9
+  ! FIRDialect:      [[R_DECL:%.*]]:2 = hlfir.declare %[[R]] {fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "{{.*}}Er"} : (!fir.ref<!fir.box<!fir.heap<f32>>>) -> (!fir.ref<!fir.box<!fir.heap<f32>>>, !fir.ref<!fir.box<!fir.heap<f32>>>)
+
+  ! FIRDialect:      %[[ALLOCA_IV_DECL:.*]]:2 = hlfir.declare %[[ALLOCA_IV]] {uniq_name = "{{.*}}Ei"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+
   ! FIRDialect:      fir.store %[[I]] to %[[ALLOCA_IV_DECL:.*]]#1 : !fir.ref<i32>
   ! FIRDialect:      %[[LOAD_IV:.*]] = fir.load %[[ALLOCA_IV_DECL]]#0 : !fir.ref<i32>
   ! FIRDialect:      fir.call @_FortranAioOutputInteger32({{.*}}, %[[LOAD_IV]]) {{.*}}: (!fir.ref<i8>, i32) -> i1
     print*, i
   end do
   ! FIRDialect:      omp.yield
-  ! FIRDialect:      {{%.*}} = fir.load [[R_DECL]]#0 : !fir.ref<!fir.box<!fir.heap<f32>>>
-  ! FIRDialect:      fir.if {{%.*}} {
-  ! FIRDialect:      [[LD:%.*]] = fir.load [[R_DECL]]#0 : !fir.ref<!fir.box<!fir.heap<f32>>>
-  ! FIRDialect:      [[AD:%.*]] = fir.box_addr [[LD]] : (!fir.box<!fir.heap<f32>>) -> !fir.heap<f32>
-  ! FIRDialect:      fir.freemem [[AD]] : !fir.heap<f32>
-  ! FIRDialect:      fir.store {{%.*}} to [[R_DECL]]#0 : !fir.ref<!fir.box<!fir.heap<f32>>>
   !$OMP END PARALLEL DO
   ! FIRDialect:  omp.terminator
 end subroutine
diff --git a/flang/test/Lower/OpenMP/parallel-reduction-allocatable-array.f90 b/flang/test/Lower/OpenMP/parallel-reduction-allocatable-array.f90
index dabd495d733b5..021f5d1ba2ef7 100644
--- a/flang/test/Lower/OpenMP/parallel-reduction-allocatable-array.f90
+++ b/flang/test/Lower/OpenMP/parallel-reduction-allocatable-array.f90
@@ -30,7 +30,9 @@ program reduce
 ! CHECK:           %[[C0_I64:.*]] = arith.constant 0 : i64
 ! CHECK:           %[[IS_NULL:.*]] = arith.cmpi eq, %[[ADDRI]], %[[C0_I64]] : i64
 ! CHECK:           fir.if %[[IS_NULL]] {
-! CHECK:             %[[NULL_BOX:.*]] = fir.embox %[[ADDR]] : (!fir.heap<!fir.array<?xi32>>) -> !fir.box<!fir.heap<!fir.array<?xi32>>>
+! CHECK:             %[[C0_INDEX:.*]] = arith.constant 0 : index
+! CHECK:             %[[SHAPE:.*]] = fir.shape %[[C0_INDEX]]
+! CHECK:             %[[NULL_BOX:.*]] = fir.embox %[[ADDR]](%[[SHAPE]]) : (!fir.heap<!fir.array<?xi32>>, !fir.shape<1>) -> !fir.box<!fir.heap<!fir.array<?xi32>>>
 ! CHECK:             fir.store %[[NULL_BOX]] to %[[ALLOC]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>
 ! CHECK:           } else {
 ! CHECK:             %[[VAL_3:.*]] = arith.constant 0 : index
@@ -92,13 +94,12 @@ program reduce
 ! CHECK:           %[[VAL_11:.*]] = fir.embox %[[VAL_9]](%[[VAL_10]]) : (!fir.heap<!fir.array<?xi32>>, !fir.shape<1>) -> !fir.box<!fir.heap<!fir.array<?xi32>>>
 ! CHECK:           fir.store %[[VAL_11]] to %[[VAL_3]]#1 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>
 ! CHECK:           omp.parallel {
-! CHECK:             %[[VAL_12:.*]] = fir.alloca i32 {bindc_name = "i", pinned, {{.*}}}
-! CHECK:             %[[VAL_13:.*]]:2 = hlfir.declare %[[VAL_12]] {uniq_name = "_QFEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:             %[[VAL_14:.*]] = arith.constant 0 : i32
 ! CHECK:             %[[VAL_15:.*]] = arith.constant 10 : i32
 ! CHECK:             %[[VAL_16:.*]] = arith.constant 1 : i32
-! CHECK:             omp.wsloop reduction(byref @add_reduction_byref_box_heap_Uxi32 %[[VAL_3]]#0 -> %[[VAL_17:.*]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>) {
+! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_12:.*]] : !fir.ref<i32>) reduction(byref @add_reduction_byref_box_heap_Uxi32 %[[VAL_3]]#0 -> %[[VAL_17:.*]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>) {
 ! CHECK-NEXT:          omp.loop_nest (%[[VAL_18:.*]]) : i32 = (%[[VAL_14]]) to (%[[VAL_15]]) inclusive step (%[[VAL_16]]) {
+! CHECK:                 %[[VAL_13:.*]]:2 = hlfir.declare %[[VAL_12]] {uniq_name = "_QFEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:                 %[[VAL_19:.*]]:2 = hlfir.declare %[[VAL_17]] {fortran_attrs = {{.*}}<allocatable>, uniq_name = "_QFEr"} : (!fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>) -> (!fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>, !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>)
 ! CHECK:                 fir.store %[[VAL_18]] to %[[VAL_13]]#1 : !fir.ref<i32>
 ! CHECK:                 %[[VAL_20:.*]] = fir.load %[[VAL_13]]#0 : !fir.ref<i32>
diff --git a/flang/test/Lower/OpenMP/parallel-reduction-pointer-array.f90 b/flang/test/Lower/OpenMP/parallel-reduction-pointer-array.f90
index 1e07018a68877..fd620c4080dbc 100644
--- a/flang/test/Lower/OpenMP/parallel-reduction-pointer-array.f90
+++ b/flang/test/Lower/OpenMP/parallel-reduction-pointer-array.f90
@@ -31,7 +31,9 @@ program reduce
 ! CHECK:           %[[VAL_6:.*]] = arith.constant 0 : i64
 ! CHECK:           %[[VAL_7:.*]] = arith.cmpi eq, %[[VAL_5]], %[[VAL_6]] : i64
 ! CHECK:           fir.if %[[VAL_7]] {
-! CHECK:             %[[VAL_8:.*]] = fir.embox %[[VAL_4]] : (!fir.ptr<!fir.array<?xi32>>) -> !fir.box<!fir.ptr<!fir.array<?xi32>>>
+! CHECK:             %[[C0:.*]] = arith.constant 0 : index
+! CHECK:             %[[SHAPE:.*]] = fir.shape %[[C0]]
+! CHECK:             %[[VAL_8:.*]] = fir.embox %[[VAL_4]](%[[SHAPE]]) : (!fir.ptr<!fir.array<?xi32>>, !fir.shape<1>) -> !fir.box<!fir.ptr<!fir.array<?xi32>>>
 ! CHECK:             fir.store %[[VAL_8]] to %[[ALLOC]] : !fir.ref<!fir.box<!fir.ptr<!fir.array<?xi32>>>>
 ! CHECK:           } else {
 ! CHECK:             %[[VAL_9:.*]] = arith.constant 0 : index
@@ -104,13 +106,12 @@ program reduce
 ! CHECK:           %[[VAL_20:.*]] = fir.convert %[[VAL_6]] : (!fir.ref<!fir.char<{{.*}}>>) -> !fir.ref<i8>
 ! CHECK:           %[[VAL_21:.*]] = fir.call @_FortranAPointerAllocate(%[[VAL_19]], %[[VAL_4]], %[[VAL_5]], %[[VAL_20]], %[[VAL_7]]) fastmath<contract> : (!fir.ref<!fir.box<none>>, i1, !fir.box<none>, !fir.ref<i8>, i32) -> i32
 ! CHECK:           omp.parallel {
-! CHECK:             %[[VAL_22:.*]] = fir.alloca i32 {bindc_name = "i", pinned, uniq_name = "_QFEi"}
-! CHECK:             %[[VAL_23:.*]]:2 = hlfir.declare %[[VAL_22]] {uniq_name = "_QFEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:             %[[VAL_24:.*]] = arith.constant 0 : i32
 ! CHECK:             %[[VAL_25:.*]] = arith.constant 10 : i32
 ! CHECK:             %[[VAL_26:.*]] = arith.constant 1 : i32
-! CHECK:             omp.wsloop reduction(byref @add_reduction_byref_box_ptr_Uxi32 %[[VAL_3]]#0 -> %[[VAL_27:.*]] : !fir.ref<!fir.box<!fir.ptr<!fir.array<?xi32>>>>) {
+! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_22:.*]] : !fir.ref<i32>) reduction(byref @add_reduction_byref_box_ptr_Uxi32 %[[VAL_3]]#0 -> %[[VAL_27:.*]] : !fir.ref<!fir.box<!fir.ptr<!fir.array<?xi32>>>>) {
 ! CHECK:               omp.loop_nest (%[[VAL_28:.*]]) : i32 = (%[[VAL_24]]) to (%[[VAL_25]]) inclusive step (%[[VAL_26]]) {
+! CHECK:                 %[[VAL_23:.*]]:2 = hlfir.declare %[[VAL_22]] {uniq_name = "_QFEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:                 %[[VAL_29:.*]]:2 = hlfir.declare %[[VAL_27]] {fortran_attrs = {{.*}}<pointer>, uniq_name = "_QFEr"} : (!fir.ref<!fir.box<!fir.ptr<!fir.array<?xi32>>>>) -> (!fir.ref<!fir.box<!fir.ptr<!fir.array<?xi32>>>>, !fir.ref<!fir.box<!fir.ptr<!fir.array<?xi32>>>>)
 ! CHECK:                 fir.store %[[VAL_28]] to %[[VAL_23]]#1 : !fir.ref<i32>
 ! CHECK:                 %[[VAL_30:.*]] = fir.load %[[VAL_23]]#0 : !fir.ref<i32>
diff --git a/flang/test/Lower/OpenMP/parallel-reduction3.f90 b/flang/test/Lower/OpenMP/parallel-reduction3.f90
index bb45206e36ee5..194b3fdd98201 100644
--- a/flang/test/Lower/OpenMP/parallel-reduction3.f90
+++ b/flang/test/Lower/OpenMP/parallel-reduction3.f90
@@ -71,13 +71,12 @@
 ! CHECK:           omp.parallel {
 ! CHECK:             %[[VAL_14:.*]] = fir.alloca !fir.box<!fir.array<?xi32>>
 ! CHECK:             fir.store %[[VAL_12]]#0 to %[[VAL_14]] : !fir.ref<!fir.box<!fir.array<?xi32>>>
-! CHECK:             %[[VAL_15:.*]] = fir.alloca i32 {bindc_name = "i", pinned, {{.*}}}
-! CHECK:             %[[VAL_16:.*]]:2 = hlfir.declare %[[VAL_15]] {uniq_name = "_QFsEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:             %[[VAL_17:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_18:.*]] = arith.constant 100 : i32
 ! CHECK:             %[[VAL_19:.*]] = arith.constant 1 : i32
-! CHECK:             omp.wsloop reduction(byref @add_reduction_byref_box_Uxi32 %[[VAL_14]] -> %[[VAL_20:.*]] : !fir.ref<!fir.box<!fir.array<?xi32>>>) {
+! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_15:.*]] : !fir.ref<i32>) reduction(byref @add_reduction_byref_box_Uxi32 %[[VAL_14]] -> %[[VAL_20:.*]] : !fir.ref<!fir.box<!fir.array<?xi32>>>) {
 ! CHECK-NEXT:          omp.loop_nest (%[[VAL_21:.*]]) : i32 = (%[[VAL_17]]) to (%[[VAL_18]]) inclusive step (%[[VAL_19]]) {
+! CHECK:                 %[[VAL_16:.*]]:2 = hlfir.declare %[[VAL_15]] {uniq_name = "_QFsEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:                 %[[VAL_22:.*]]:2 = hlfir.declare %[[VAL_20]] {uniq_name = "_QFsEc"} : (!fir.ref<!fir.box<!fir.array<?xi32>>>) -> (!fir.ref<!fir.box<!fir.array<?xi32>>>, !fir.ref<!fir.box<!fir.array<?xi32>>>)
 ! CHECK:                 fir.store %[[VAL_21]] to %[[VAL_16]]#1 : !fir.ref<i32>
 ! CHECK:                 %[[VAL_23:.*]] = fir.load %[[VAL_22]]#0 : !fir.ref<!fir.box<!fir.array<?xi32>>>
diff --git a/flang/test/Lower/OpenMP/parallel-wsloop-firstpriv.f90 b/flang/test/Lower/OpenMP/parallel-wsloop-firstpriv.f90
index 4e7c2c15df743..3acc21678b9c0 100644
--- a/flang/test/Lower/OpenMP/parallel-wsloop-firstpriv.f90
+++ b/flang/test/Lower/OpenMP/parallel-wsloop-firstpriv.f90
@@ -11,20 +11,13 @@ subroutine omp_do_firstprivate(a)
   n = a+1
   !$omp parallel do firstprivate(a)
   ! CHECK:  omp.parallel {
-
-  ! CHECK: %[[A_PVT_REF:.*]] = fir.alloca i32 {bindc_name = "a", pinned, uniq_name = "_QFomp_do_firstprivateEa"}
-  ! CHECK: %[[A_PVT_DECL:.*]]:2 = hlfir.declare %[[A_PVT_REF]] {uniq_name = "_QFomp_do_firstprivateEa"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-  ! CHECK-NEXT: %[[LD:.*]] = fir.load %[[ARG0_DECL]]#0 : !fir.ref<i32>
-  ! CHECK-NEXT: hlfir.assign %[[LD]] to %[[A_PVT_DECL]]#0 : i32, !fir.ref<i32>
-
-  ! CHECK: %[[I_PVT_REF:.*]] = fir.alloca i32 {bindc_name = "i", pinned, {{.*}}}
-  ! CHECK: %[[I_PVT_DECL:.*]]:2 = hlfir.declare %[[I_PVT_REF]] {uniq_name = "_QFomp_do_firstprivateEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-
   ! CHECK: %[[LB:.*]] = arith.constant 1 : i32
-  ! CHECK-NEXT: %[[UB:.*]] = fir.load %[[A_PVT_DECL]]#0 : !fir.ref<i32>
+  ! CHECK-NEXT: %[[UB:.*]] = fir.load %[[ARG0_DECL]]#0 : !fir.ref<i32>
   ! CHECK-NEXT: %[[STEP:.*]] = arith.constant 1 : i32
-  ! CHECK-NEXT: omp.wsloop {
+  ! CHECK-NEXT: omp.wsloop private(@{{.*a_firstprivate.*}} %{{.*}}#0 -> %[[A_PVT_REF:.*]], @{{.*i_private.*}} %{{.*}}#0 -> %[[I_PVT_REF:.*]] : !fir.ref<i32>, !fir.ref<i32>) {
   ! CHECK-NEXT: omp.loop_nest (%[[ARG1:.*]]) : i32 = (%[[LB]]) to (%[[UB]]) inclusive step (%[[STEP]]) {
+  ! CHECK: %[[A_PVT_DECL:.*]]:2 = hlfir.declare %[[A_PVT_REF]] {uniq_name = "_QFomp_do_firstprivateEa"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+  ! CHECK: %[[I_PVT_DECL:.*]]:2 = hlfir.declare %[[I_PVT_REF]] {uniq_name = "_QFomp_do_firstprivateEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
   ! CHECK-NEXT: fir.store %[[ARG1]] to %[[I_PVT_DECL]]#1 : !fir.ref<i32>
   ! CHECK-NEXT: fir.call @_QPfoo(%[[I_PVT_DECL]]#1, %[[A_PVT_DECL]]#1) {{.*}}: (!fir.ref<i32>, !fir.ref<i32>) -> ()
   ! CHECK-NEXT: omp.yield
@@ -47,25 +40,14 @@ subroutine omp_do_firstprivate2(a, n)
   n = a+1
   !$omp parallel do firstprivate(a, n)
   ! CHECK:  omp.parallel {
-
-  ! CHECK: %[[A_PVT_REF:.*]] = fir.alloca i32 {bindc_name = "a", pinned, {{.*}}}
+  ! CHECK: %[[LB:.*]] = fir.load %[[ARG0_DECL]]#0 : !fir.ref<i32>
+  ! CHECK: %[[UB:.*]] = fir.load %[[ARG1_DECL]]#0 : !fir.ref<i32>
+  ! CHECK: %[[STEP:.*]] = arith.constant 1 : i32
+  ! CHECK: omp.wsloop private(@{{.*a_firstprivate.*}} %{{.*}}#0 -> %[[A_PVT_REF:.*]], @{{.*n_firstprivate.*}} %{{.*}}#0 -> %[[N_PVT_REF:.*]], @{{.*i_private.*}} %{{.*}}#0 -> %[[I_PVT_REF:.*]] : !fir.ref<i32>, !fir.ref<i32>, !fir.ref<i32>) {
+  ! CHECK-NEXT: omp.loop_nest (%[[ARG2:.*]]) : i32 = (%[[LB]]) to (%[[UB]]) inclusive step (%[[STEP]]) {
   ! CHECK: %[[A_PVT_DECL:.*]]:2 = hlfir.declare %[[A_PVT_REF]] {uniq_name = "_QFomp_do_firstprivate2Ea"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-  ! CHECK: %[[LD:.*]] = fir.load %[[ARG0_DECL]]#0 : !fir.ref<i32>
-  ! CHECK: hlfir.assign %[[LD]] to %[[A_PVT_DECL]]#0 : i32, !fir.ref<i32>
-
-  ! CHECK: %[[N_PVT_REF:.*]] = fir.alloca i32 {bindc_name = "n", pinned, uniq_name = "_QFomp_do_firstprivate2En"}
   ! CHECK: %[[N_PVT_DECL:.*]]:2 = hlfir.declare %[[N_PVT_REF]] {uniq_name = "_QFomp_do_firstprivate2En"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-  ! CHECK: %[[LD1:.*]] = fir.load %[[ARG1_DECL]]#0 : !fir.ref<i32>
-  ! CHECK: hlfir.assign %[[LD1]] to %[[N_PVT_DECL]]#0 : i32, !fir.ref<i32>
-
-  ! CHECK: %[[I_PVT_REF:.*]] = fir.alloca i32 {bindc_name = "i", pinned, {{.*}}}
   ! CHECK: %[[I_PVT_DECL:.*]]:2 = hlfir.declare %[[I_PVT_REF]] {uniq_name = "_QFomp_do_firstprivate2Ei"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-
-  ! CHECK: %[[LB:.*]] = fir.load %[[A_PVT_DECL]]#0 : !fir.ref<i32>
-  ! CHECK: %[[UB:.*]] = fir.load %[[N_PVT_DECL]]#0 : !fir.ref<i32>
-  ! CHECK: %[[STEP:.*]] = arith.constant 1 : i32
-  ! CHECK: omp.wsloop {
-  ! CHECK-NEXT: omp.loop_nest (%[[ARG2:.*]]) : i32 = (%[[LB]]) to (%[[UB]]) inclusive step (%[[STEP]]) {
   ! CHECK: fir.store %[[ARG2]] to %[[I_PVT_DECL]]#1 : !fir.ref<i32>
   ! CHECK: fir.call @_QPfoo(%[[I_PVT_DECL]]#1, %[[A_PVT_DECL]]#1) {{.*}}: (!fir.ref<i32>, !fir.ref<i32>) -> ()
   ! CHECK: omp.yield
diff --git a/flang/test/Lower/OpenMP/parallel-wsloop-lastpriv.f90 b/flang/test/Lower/OpenMP/parallel-wsloop-lastpriv.f90
index dbde5291c01c8..a7c0dc3b1b406 100644
--- a/flang/test/Lower/OpenMP/parallel-wsloop-lastpriv.f90
+++ b/flang/test/Lower/OpenMP/parallel-wsloop-lastpriv.f90
@@ -12,17 +12,15 @@ subroutine omp_do_lastprivate(a)
   !$omp parallel do lastprivate(a)
   ! CHECK:  omp.parallel {
 
-  ! CHECK: %[[A_PVT_REF:.*]] = fir.alloca i32 {bindc_name = "a", pinned, uniq_name = "_QFomp_do_lastprivateEa"}
-  ! CHECK: %[[A_PVT_DECL:.*]]:2 = hlfir.declare %[[A_PVT_REF]] {uniq_name = "_QFomp_do_lastprivateEa"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-
-  ! CHECK: %[[I_PVT_REF:.*]] = fir.alloca i32 {bindc_name = "i", pinned, {{.*}}}
-  ! CHECK: %[[I_PVT_DECL:.*]]:2 = hlfir.declare %[[I_PVT_REF]] {uniq_name = "_QFomp_do_lastprivateEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 
   ! CHECK: %[[LB:.*]] = arith.constant 1 : i32
-  ! CHECK-NEXT: %[[UB:.*]] = fir.load %[[A_PVT_DECL]]#0 : !fir.ref<i32>
+  ! CHECK-NEXT: %[[UB:.*]] = fir.load %[[ARG0_DECL]]#0 : !fir.ref<i32>
   ! CHECK-NEXT: %[[STEP:.*]] = arith.constant 1 : i32
-  ! CHECK-NEXT: omp.wsloop {
+  ! CHECK-NEXT: omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[A_PVT_REF:.*]], @{{.*}} %{{.*}}#0 -> %[[I_PVT_REF:.*]] : !fir.ref<i32>, !fir.ref<i32>) {
   ! CHECK-NEXT: omp.loop_nest (%[[ARG1:.*]]) : i32 = (%[[LB]]) to (%[[UB]]) inclusive step (%[[STEP]]) {
+  ! CHECK:      %[[A_PVT_DECL:.*]]:2 = hlfir.declare %[[A_PVT_REF]] {uniq_name = "_QFomp_do_lastprivateEa"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+  ! CHECK:      %[[I_PVT_DECL:.*]]:2 = hlfir.declare %[[I_PVT_REF]] {uniq_name = "_QFomp_do_lastprivateEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+
   ! CHECK-NEXT: fir.store %[[ARG1]] to %[[I_PVT_DECL]]#1 : !fir.ref<i32>
   ! CHECK-NEXT: fir.call @_QPfoo(%[[I_PVT_DECL]]#1, %[[A_PVT_DECL]]#1) {{.*}}: (!fir.ref<i32>, !fir.ref<i32>) -> ()
   ! CHECK:      %[[NEXT_ARG1:.*]] = arith.addi %[[ARG1]], %[[STEP]] : i32
@@ -58,20 +56,15 @@ subroutine omp_do_lastprivate2(a, n)
   !$omp parallel do lastprivate(a, n)
   ! CHECK:  omp.parallel {
 
-  ! CHECK: %[[A_PVT_REF:.*]] = fir.alloca i32 {bindc_name = "a", pinned, {{.*}}}
+  ! CHECK: %[[LB:.*]] = fir.load %[[ARG0_DECL]]#0 : !fir.ref<i32>
+  ! CHECK: %[[UB:.*]] = fir.load %[[ARG1_DECL]]#0 : !fir.ref<i32>
+  ! CHECK: %[[STEP:.*]] = arith.constant 1 : i32
+  ! CHECK: omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[A_PVT_REF:.*]], @{{.*}} %{{.*}}#0 -> %[[N_PVT_REF:.*]], @{{.*}} %{{.*}}#0 -> %[[I_PVT_REF:.*]] : !fir.ref<i32>, !fir.ref<i32>, !fir.ref<i32>) {
+  ! CHECK-NEXT: omp.loop_nest (%[[ARG2:.*]]) : i32 = (%[[LB]]) to (%[[UB]]) inclusive step (%[[STEP]]) {
   ! CHECK: %[[A_PVT_DECL:.*]]:2 = hlfir.declare %[[A_PVT_REF]] {uniq_name = "_QFomp_do_lastprivate2Ea"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-
-  ! CHECK: %[[N_PVT_REF:.*]] = fir.alloca i32 {bindc_name = "n", pinned, uniq_name = "_QFomp_do_lastprivate2En"}
   ! CHECK: %[[N_PVT_DECL:.*]]:2 = hlfir.declare %[[N_PVT_REF]] {uniq_name = "_QFomp_do_lastprivate2En"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-
-  ! CHECK: %[[I_PVT_REF:.*]] = fir.alloca i32 {bindc_name = "i", pinned, {{.*}}}
   ! CHECK: %[[I_PVT_DECL:.*]]:2 = hlfir.declare %[[I_PVT_REF]] {uniq_name = "_QFomp_do_lastprivate2Ei"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 
-  ! CHECK: %[[LB:.*]] = fir.load %[[A_PVT_DECL]]#0 : !fir.ref<i32>
-  ! CHECK: %[[UB:.*]] = fir.load %[[N_PVT_DECL]]#0 : !fir.ref<i32>
-  ! CHECK: %[[STEP:.*]] = arith.constant 1 : i32
-  ! CHECK: omp.wsloop {
-  ! CHECK-NEXT: omp.loop_nest (%[[ARG2:.*]]) : i32 = (%[[LB]]) to (%[[UB]]) inclusive step (%[[STEP]]) {
   ! CHECK: fir.store %[[ARG2]] to %[[I_PVT_DECL]]#1 : !fir.ref<i32>
   ! CHECK: fir.call @_QPfoo(%[[I_PVT_DECL]]#1, %[[A_PVT_DECL]]#1) {{.*}}: (!fir.ref<i32>, !fir.ref<i32>) -> ()
   ! CHECK: %[[NEXT_ARG2:.*]] = arith.addi %[[ARG2]], %[[STEP]] : i32
@@ -104,23 +97,18 @@ subroutine omp_do_lastprivate_collapse2(a)
   !$omp parallel do lastprivate(a) collapse(2)
   ! CHECK:  omp.parallel {
 
-  ! CHECK: %[[A_PVT_REF:.*]] = fir.alloca i32 {bindc_name = "a", pinned, uniq_name = "_QFomp_do_lastprivate_collapse2Ea"}
-  ! CHECK: %[[A_PVT_DECL:.*]]:2 = hlfir.declare %[[A_PVT_REF]] {uniq_name = "_QFomp_do_lastprivate_collapse2Ea"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-
-  ! CHECK: %[[I_PVT_REF:.*]] = fir.alloca i32 {bindc_name = "i", pinned, {{.*}}}
-  ! CHECK: %[[I_PVT_DECL:.*]]:2 = hlfir.declare %[[I_PVT_REF]] {uniq_name = "_QFomp_do_lastprivate_collapse2Ei"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-  !
-  ! CHECK: %[[J_PVT_REF:.*]] = fir.alloca i32 {bindc_name = "j", pinned, {{.*}}}
-  ! CHECK: %[[J_PVT_DECL:.*]]:2 = hlfir.declare %[[J_PVT_REF]] {uniq_name = "_QFomp_do_lastprivate_collapse2Ej"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-
   ! CHECK: %[[LB1:.*]] = arith.constant 1 : i32
-  ! CHECK-NEXT: %[[UB1:.*]] = fir.load %[[A_PVT_DECL]]#0 : !fir.ref<i32>
+  ! CHECK-NEXT: %[[UB1:.*]] = fir.load %[[ARG0_DECL]]#0 : !fir.ref<i32>
   ! CHECK-NEXT: %[[STEP1:.*]] = arith.constant 1 : i32
   ! CHECK: %[[LB2:.*]] = arith.constant 1 : i32
-  ! CHECK-NEXT: %[[UB2:.*]] = fir.load %[[A_PVT_DECL]]#0 : !fir.ref<i32>
+  ! CHECK-NEXT: %[[UB2:.*]] = fir.load %[[ARG0_DECL]]#0 : !fir.ref<i32>
   ! CHECK-NEXT: %[[STEP2:.*]] = arith.constant 1 : i32
-  ! CHECK-NEXT: omp.wsloop {
+  ! CHECK-NEXT: omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[A_PVT_REF:.*]], @{{.*}} %{{.*}}#0 -> %[[I_PVT_REF:.*]], @{{.*}} %{{.*}}#0 -> %[[J_PVT_REF:.*]] : !fir.ref<i32>, !fir.ref<i32>, !fir.ref<i32>) {
   ! CHECK-NEXT: omp.loop_nest (%[[ARG1:.*]], %[[ARG2:.*]]) : i32 = (%[[LB1]], %[[LB2]]) to (%[[UB1]], %[[UB2]]) inclusive step (%[[STEP1]], %[[STEP2]]) {
+  ! CHECK:      %[[A_PVT_DECL:.*]]:2 = hlfir.declare %[[A_PVT_REF]] {uniq_name = "_QFomp_do_lastprivate_collapse2Ea"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+  ! CHECK:      %[[I_PVT_DECL:.*]]:2 = hlfir.declare %[[I_PVT_REF]] {uniq_name = "_QFomp_do_lastprivate_collapse2Ei"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+  ! CHECK:      %[[J_PVT_DECL:.*]]:2 = hlfir.declare %[[J_PVT_REF]] {uniq_name = "_QFomp_do_lastprivate_collapse2Ej"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+
   ! CHECK-NEXT: fir.store %[[ARG1]] to %[[I_PVT_DECL]]#1 : !fir.ref<i32>
   ! CHECK-NEXT: fir.store %[[ARG2]] to %[[J_PVT_DECL]]#1 : !fir.ref<i32>
   ! CHECK-NEXT: fir.call @_QPfoo(%[[I_PVT_DECL]]#1, %[[A_PVT_DECL]]#1) {{.*}}: (!fir.ref<i32>, !fir.ref<i32>) -> ()
@@ -164,29 +152,23 @@ subroutine omp_do_lastprivate_collapse3(a)
   !$omp parallel do lastprivate(a) collapse(3)
   ! CHECK:  omp.parallel {
 
-  ! CHECK: %[[A_PVT_REF:.*]] = fir.alloca i32 {bindc_name = "a", pinned, uniq_name = "_QFomp_do_lastprivate_collapse3Ea"}
-  ! CHECK: %[[A_PVT_DECL:.*]]:2 = hlfir.declare %[[A_PVT_REF]] {uniq_name = "_QFomp_do_lastprivate_collapse3Ea"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-
-  ! CHECK: %[[I_PVT_REF:.*]] = fir.alloca i32 {bindc_name = "i", pinned, {{.*}}}
-  ! CHECK: %[[I_PVT_DECL:.*]]:2 = hlfir.declare %[[I_PVT_REF]] {uniq_name = "_QFomp_do_lastprivate_collapse3Ei"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-
-  ! CHECK: %[[J_PVT_REF:.*]] = fir.alloca i32 {bindc_name = "j", pinned, {{.*}}}
-  ! CHECK: %[[J_PVT_DECL:.*]]:2 = hlfir.declare %[[J_PVT_REF]] {uniq_name = "_QFomp_do_lastprivate_collapse3Ej"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-
-  ! CHECK: %[[K_PVT_REF:.*]] = fir.alloca i32 {bindc_name = "k", pinned, {{.*}}}
-  ! CHECK: %[[K_PVT_DECL:.*]]:2 = hlfir.declare %[[K_PVT_REF]] {uniq_name = "_QFomp_do_lastprivate_collapse3Ek"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 
   ! CHECK: %[[LB1:.*]] = arith.constant 1 : i32
-  ! CHECK-NEXT: %[[UB1:.*]] = fir.load %[[A_PVT_DECL]]#0 : !fir.ref<i32>
+  ! CHECK-NEXT: %[[UB1:.*]] = fir.load %[[ARG0_DECL]]#0 : !fir.ref<i32>
   ! CHECK-NEXT: %[[STEP1:.*]] = arith.constant 1 : i32
   ! CHECK: %[[LB2:.*]] = arith.constant 1 : i32
-  ! CHECK-NEXT: %[[UB2:.*]] = fir.load %[[A_PVT_DECL]]#0 : !fir.ref<i32>
+  ! CHECK-NEXT: %[[UB2:.*]] = fir.load %[[ARG0_DECL]]#0 : !fir.ref<i32>
   ! CHECK-NEXT: %[[STEP2:.*]] = arith.constant 1 : i32
   ! CHECK: %[[LB3:.*]] = arith.constant 1 : i32
-  ! CHECK-NEXT: %[[UB3:.*]] = fir.load %[[A_PVT_DECL]]#0 : !fir.ref<i32>
+  ! CHECK-NEXT: %[[UB3:.*]] = fir.load %[[ARG0_DECL]]#0 : !fir.ref<i32>
   ! CHECK-NEXT: %[[STEP3:.*]] = arith.constant 1 : i32
-  ! CHECK-NEXT: omp.wsloop {
+  ! CHECK-NEXT: omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[A_PVT_REF:.*]], @{{.*}} %{{.*}}#0 -> %[[I_PVT_REF:.*]], @{{.*}} %{{.*}}#0 -> %[[J_PVT_REF:.*]], @{{.*}} %{{.*}}#0 -> %[[K_PVT_REF:.*]] : !fir.ref<i32>, !fir.ref<i32>, !fir.ref<i32>, !fir.ref<i32>) {
   ! CHECK-NEXT: omp.loop_nest (%[[ARG1:.*]], %[[ARG2:.*]], %[[ARG3:.*]]) : i32 = (%[[LB1]], %[[LB2]], %[[LB3]]) to (%[[UB1]], %[[UB2]], %[[UB3]]) inclusive step (%[[STEP1]], %[[STEP2]], %[[STEP3]]) {
+  ! CHECK:      %[[A_PVT_DECL:.*]]:2 = hlfir.declare %[[A_PVT_REF]] {uniq_name = "_QFomp_do_lastprivate_collapse3Ea"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+  ! CHECK:      %[[I_PVT_DECL:.*]]:2 = hlfir.declare %[[I_PVT_REF]] {uniq_name = "_QFomp_do_lastprivate_collapse3Ei"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+  ! CHECK:      %[[J_PVT_DECL:.*]]:2 = hlfir.declare %[[J_PVT_REF]] {uniq_name = "_QFomp_do_lastprivate_collapse3Ej"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+  ! CHECK:      %[[K_PVT_DECL:.*]]:2 = hlfir.declare %[[K_PVT_REF]] {uniq_name = "_QFomp_do_lastprivate_collapse3Ek"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+
   ! CHECK-NEXT: fir.store %[[ARG1]] to %[[I_PVT_DECL]]#1 : !fir.ref<i32>
   ! CHECK-NEXT: fir.store %[[ARG2]] to %[[J_PVT_DECL]]#1 : !fir.ref<i32>
   ! CHECK-NEXT: fir.store %[[ARG3]] to %[[K_PVT_DECL]]#1 : !fir.ref<i32>
diff --git a/flang/test/Lower/OpenMP/parallel-wsloop-reduction-byref.f90 b/flang/test/Lower/OpenMP/parallel-wsloop-reduction-byref.f90
index 99c521406a777..3f44f292cb6a0 100644
--- a/flang/test/Lower/OpenMP/parallel-wsloop-reduction-byref.f90
+++ b/flang/test/Lower/OpenMP/parallel-wsloop-reduction-byref.f90
@@ -4,7 +4,7 @@
 ! RUN: flang -fc1 -fopenmp -mmlir --force-byref-reduction -emit-hlfir %s -o - | FileCheck %s
 
 ! CHECK: omp.parallel {
-! CHECK: omp.wsloop reduction(byref @add_reduction_byref_i32
+! CHECK: omp.wsloop private({{.*}}) reduction(byref @add_reduction_byref_i32
 subroutine sb
   integer :: x
   x = 0
diff --git a/flang/test/Lower/OpenMP/parallel-wsloop-reduction.f90 b/flang/test/Lower/OpenMP/parallel-wsloop-reduction.f90
index cfeb5de83f4e8..a206eef52da5a 100644
--- a/flang/test/Lower/OpenMP/parallel-wsloop-reduction.f90
+++ b/flang/test/Lower/OpenMP/parallel-wsloop-reduction.f90
@@ -4,7 +4,7 @@
 ! RUN: flang -fc1 -fopenmp -emit-hlfir %s -o - | FileCheck %s
 
 ! CHECK: omp.parallel {
-! CHECK: omp.wsloop reduction(@add_reduction_i32
+! CHECK: omp.wsloop private({{.*}}) reduction(@add_reduction_i32
 subroutine sb
   integer :: x
   x = 0
diff --git a/flang/test/Lower/OpenMP/parallel-wsloop.f90 b/flang/test/Lower/OpenMP/parallel-wsloop.f90
index cba5209f85989..7116069e8daa6 100644
--- a/flang/test/Lower/OpenMP/parallel-wsloop.f90
+++ b/flang/test/Lower/OpenMP/parallel-wsloop.f90
@@ -10,7 +10,7 @@ subroutine simple_parallel_do
   ! CHECK:      %[[WS_LB:.*]] = arith.constant 1 : i32
   ! CHECK:      %[[WS_UB:.*]] = arith.constant 9 : i32
   ! CHECK:      %[[WS_STEP:.*]] = arith.constant 1 : i32
-  ! CHECK:      omp.wsloop {
+  ! CHECK:      omp.wsloop private({{.*}}) {
   ! CHECK-NEXT: omp.loop_nest (%[[I:.*]]) : i32 = (%[[WS_LB]]) to (%[[WS_UB]]) inclusive step (%[[WS_STEP]]) {
   !$OMP PARALLEL DO
   do i=1, 9
@@ -39,7 +39,7 @@ subroutine parallel_do_with_parallel_clauses(cond, nt)
   ! CHECK:      %[[WS_LB:.*]] = arith.constant 1 : i32
   ! CHECK:      %[[WS_UB:.*]] = arith.constant 9 : i32
   ! CHECK:      %[[WS_STEP:.*]] = arith.constant 1 : i32
-  ! CHECK:      omp.wsloop {
+  ! CHECK:      omp.wsloop private({{.*}}) {
   ! CHECK-NEXT: omp.loop_nest (%[[I:.*]]) : i32 = (%[[WS_LB]]) to (%[[WS_UB]]) inclusive step (%[[WS_STEP]]) {
   !$OMP PARALLEL DO IF(cond) NUM_THREADS(nt) PROC_BIND(close)
   do i=1, 9
@@ -64,7 +64,7 @@ subroutine parallel_do_with_clauses(nt)
   ! CHECK:      %[[WS_LB:.*]] = arith.constant 1 : i32
   ! CHECK:      %[[WS_UB:.*]] = arith.constant 9 : i32
   ! CHECK:      %[[WS_STEP:.*]] = arith.constant 1 : i32
-  ! CHECK:      omp.wsloop schedule(dynamic) {
+  ! CHECK:      omp.wsloop schedule(dynamic) private({{.*}}) {
   ! CHECK-NEXT: omp.loop_nest (%[[I:.*]]) : i32 = (%[[WS_LB]]) to (%[[WS_UB]]) inclusive step (%[[WS_STEP]]) {
   !$OMP PARALLEL DO NUM_THREADS(nt) SCHEDULE(dynamic)
   do i=1, 9
@@ -92,19 +92,16 @@ subroutine parallel_do_with_privatisation_clauses(cond,nt)
   integer :: nt
   integer :: i
   ! CHECK:  omp.parallel
-  ! CHECK:      %[[PRIVATE_COND_REF:.*]] = fir.alloca !fir.logical<4> {bindc_name = "cond", pinned, uniq_name = "_QFparallel_do_with_privatisation_clausesEcond"}
-  ! CHECK:      %[[PRIVATE_COND_DECL:.*]]:2 = hlfir.declare %[[PRIVATE_COND_REF]] {uniq_name = "_QFparallel_do_with_privatisation_clausesEcond"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
-  ! CHECK:      %[[PRIVATE_NT_REF:.*]] = fir.alloca i32 {bindc_name = "nt", pinned, uniq_name = "_QFparallel_do_with_privatisation_clausesEnt"}
-  ! CHECK:      %[[PRIVATE_NT_DECL:.*]]:2 = hlfir.declare %[[PRIVATE_NT_REF]] {uniq_name = "_QFparallel_do_with_privatisation_clausesEnt"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-  ! CHECK:      %[[NT_VAL:.*]] = fir.load %[[NT_DECL]]#0 : !fir.ref<i32>
-  ! CHECK:      hlfir.assign %[[NT_VAL]] to %[[PRIVATE_NT_DECL]]#0 : i32, !fir.ref<i32>
+
   ! CHECK:      %[[WS_LB:.*]] = arith.constant 1 : i32
   ! CHECK:      %[[WS_UB:.*]] = arith.constant 9 : i32
   ! CHECK:      %[[WS_STEP:.*]] = arith.constant 1 : i32
-  ! CHECK:      omp.wsloop {
+  ! CHECK:      omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[PRIVATE_COND_REF:.*]], @{{.*}} %{{.*}}#0 -> %[[PRIVATE_NT_REF:.*]], @{{.*}} %3#0 -> %{{.*}} : !fir.ref<!fir.logical<4>>, !fir.ref<i32>, !fir.ref<i32>) {
   ! CHECK-NEXT: omp.loop_nest (%[[I:.*]]) : i32 = (%[[WS_LB]]) to (%[[WS_UB]]) inclusive step (%[[WS_STEP]]) {
   !$OMP PARALLEL DO PRIVATE(cond) FIRSTPRIVATE(nt)
   do i=1, 9
+  ! CHECK:      %[[PRIVATE_COND_DECL:.*]]:2 = hlfir.declare %[[PRIVATE_COND_REF]] {uniq_name = "_QFparallel_do_with_privatisation_clausesEcond"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
+  ! CHECK:      %[[PRIVATE_NT_DECL:.*]]:2 = hlfir.declare %[[PRIVATE_NT_REF]] {uniq_name = "_QFparallel_do_with_privatisation_clausesEnt"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
   ! CHECK:      fir.store %[[I]] to %[[IV_ADDR:.*]]#1 : !fir.ref<i32>
   ! CHECK:      %[[LOAD_IV:.*]] = fir.load %[[IV_ADDR]]#0 : !fir.ref<i32>
   ! CHECK:      fir.call @_FortranAioOutputInteger32({{.*}}, %[[LOAD_IV]]) {{.*}}: (!fir.ref<i8>, i32) -> i1
@@ -149,14 +146,12 @@ end subroutine parallel_private_do
 
 ! CHECK:             %[[NT_PRIV_DECL:.*]]:2 = hlfir.declare %[[NT_PRIV_ADDR]] {uniq_name = "_QFparallel_private_doEnt"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 
-! CHECK:             %[[I_PRIV:.*]] = fir.alloca i32 {bindc_name = "i", pinned, {{.*}}}
-! CHECK:             %[[I_PRIV_DECL:.*]]:2 = hlfir.declare %[[I_PRIV]] {uniq_name = "_QFparallel_private_doEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-
 ! CHECK:             %[[VAL_7:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_8:.*]] = arith.constant 9 : i32
 ! CHECK:             %[[VAL_9:.*]] = arith.constant 1 : i32
-! CHECK:             omp.wsloop {
+! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[I_PRIV:.*]] : !fir.ref<i32>) {
 ! CHECK-NEXT:          omp.loop_nest (%[[I:.*]]) : i32 = (%[[VAL_7]]) to (%[[VAL_8]]) inclusive step (%[[VAL_9]]) {
+! CHECK:                 %[[I_PRIV_DECL:.*]]:2 = hlfir.declare %[[I_PRIV]] {uniq_name = "_QFparallel_private_doEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:                 fir.store %[[I]] to %[[I_PRIV_DECL]]#1 : !fir.ref<i32>
 ! CHECK:                 fir.call @_QPfoo(%[[I_PRIV_DECL]]#1, %[[COND_DECL]]#1, %[[NT_PRIV_DECL]]#1) {{.*}}: (!fir.ref<i32>, !fir.ref<!fir.logical<4>>, !fir.ref<i32>) -> ()
 ! CHECK:                 omp.yield
@@ -195,14 +190,13 @@ end subroutine omp_parallel_multiple_firstprivate_do
 
 ! CHECK:             %[[B_PRIV_DECL:.*]]:2 = hlfir.declare %[[B_PRIV_ADDR]] {uniq_name = "_QFomp_parallel_multiple_firstprivate_doEb"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 
-! CHECK:             %[[I_PRIV_ADDR:.*]] = fir.alloca i32 {bindc_name = "i", pinned, {{.*}}}
-! CHECK:             %[[I_PRIV_DECL:.*]]:2 = hlfir.declare %[[I_PRIV_ADDR]] {uniq_name = "_QFomp_parallel_multiple_firstprivate_doEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 
 ! CHECK:             %[[VAL_8:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_9:.*]] = arith.constant 10 : i32
 ! CHECK:             %[[VAL_10:.*]] = arith.constant 1 : i32
-! CHECK:             omp.wsloop {
+! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[I_PRIV_ADDR:.*]] : !fir.ref<i32>) {
 ! CHECK-NEXT:          omp.loop_nest (%[[I:.*]]) : i32 = (%[[VAL_8]]) to (%[[VAL_9]]) inclusive step (%[[VAL_10]]) {
+! CHECK:                 %[[I_PRIV_DECL:.*]]:2 = hlfir.declare %[[I_PRIV_ADDR]] {uniq_name = "_QFomp_parallel_multiple_firstprivate_doEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:                 fir.store %[[I]] to %[[I_PRIV_DECL]]#1 : !fir.ref<i32>
 ! CHECK:                 fir.call @_QPbar(%[[I_PRIV_DECL]]#1, %[[A_PRIV_DECL]]#1) {{.*}}: (!fir.ref<i32>, !fir.ref<i32>) -> ()
 ! CHECK:                 omp.yield
@@ -237,23 +231,15 @@ end subroutine parallel_do_private
 ! CHECK-SAME:                                      %[[VAL_1:.*]]: !fir.ref<i32> {fir.bindc_name = "nt"}) {
 ! CHECK:           %[[NT_DECL:.*]]:2 = hlfir.declare %[[VAL_1]] dummy_scope %{{[0-9]+}} {uniq_name = "_QFparallel_do_privateEnt"} : (!fir.ref<i32>, !fir.dscope) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:           omp.parallel   {
-
-! CHECK:             %[[COND_PRIV_ADDR:.*]] = fir.alloca !fir.logical<4> {bindc_name = "cond", pinned, uniq_name = "_QFparallel_do_privateEcond"}
-! CHECK:             %[[COND_PRIV_DECL:.*]]:2 = hlfir.declare %[[COND_PRIV_ADDR]] {uniq_name = "_QFparallel_do_privateEcond"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
-
-! CHECK:             %[[NT_PRIV_ADDR:.*]] = fir.alloca i32 {bindc_name = "nt", pinned, uniq_name = "_QFparallel_do_privateEnt"}
-! CHECK:             %[[NT_PRIV_DECL:.*]]:2 = hlfir.declare %[[NT_PRIV_ADDR]] {uniq_name = "_QFparallel_do_privateEnt"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-! CHECK:             %[[NT_VAL:.*]] = fir.load %[[NT_DECL]]#0 : !fir.ref<i32>
-! CHECK:             hlfir.assign %[[NT_VAL]] to %[[NT_PRIV_DECL]]#0 : i32, !fir.ref<i32>
-
-! CHECK:             %[[I_PRIV_ADDR:.*]] = fir.alloca i32 {bindc_name = "i", pinned, {{.*}}}
-! CHECK:             %[[I_PRIV_DECL:.*]]:2 = hlfir.declare %[[I_PRIV_ADDR]] {uniq_name = "_QFparallel_do_privateEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-
 ! CHECK:             %[[VAL_7:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_8:.*]] = arith.constant 9 : i32
 ! CHECK:             %[[VAL_9:.*]] = arith.constant 1 : i32
-! CHECK:             omp.wsloop {
+! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[COND_PRIV_ADDR:.*]], @{{.*}} %{{.*}}#0 -> %[[NT_PRIV_ADDR:.*]], @{{.*}} %3#0 -> %[[I_PRIV_ADDR:.*]] : !fir.ref<!fir.logical<4>>, !fir.ref<i32>, !fir.ref<i32>) {
 ! CHECK-NEXT:          omp.loop_nest (%[[I:.*]]) : i32 = (%[[VAL_7]]) to (%[[VAL_8]]) inclusive step (%[[VAL_9]]) {
+! CHECK:                 %[[COND_PRIV_DECL:.*]]:2 = hlfir.declare %[[COND_PRIV_ADDR]] {uniq_name = "_QFparallel_do_privateEcond"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
+! CHECK:                 %[[NT_PRIV_DECL:.*]]:2 = hlfir.declare %[[NT_PRIV_ADDR]] {uniq_name = "_QFparallel_do_privateEnt"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+! CHECK:                 %[[I_PRIV_DECL:.*]]:2 = hlfir.declare %[[I_PRIV_ADDR]] {uniq_name = "_QFparallel_do_privateEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+
 ! CHECK:                 fir.store %[[I]] to %[[I_PRIV_DECL]]#1 : !fir.ref<i32>
 ! CHECK:                 fir.call @_QPfoo(%[[I_PRIV_DECL]]#1, %[[COND_PRIV_DECL]]#1, %[[NT_PRIV_DECL]]#1) {{.*}}: (!fir.ref<i32>, !fir.ref<!fir.logical<4>>, !fir.ref<i32>) -> ()
 ! CHECK:                 omp.yield
@@ -287,25 +273,15 @@ end subroutine omp_parallel_do_multiple_firstprivate
 ! CHECK:           %[[A_DECL:.*]]:2 = hlfir.declare %[[A_ADDR]] dummy_scope %{{[0-9]+}} {uniq_name = "_QFomp_parallel_do_multiple_firstprivateEa"} : (!fir.ref<i32>, !fir.dscope) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:           %[[B_DECL:.*]]:2 = hlfir.declare %[[B_ADDR]] dummy_scope %{{[0-9]+}} {uniq_name = "_QFomp_parallel_do_multiple_firstprivateEb"} : (!fir.ref<i32>, !fir.dscope) -> (!fir.ref<i32>, !fir.ref<i32>
 ! CHECK:           omp.parallel {
-
-! CHECK:             %[[A_PRIV_ADDR:.*]] = fir.alloca i32 {bindc_name = "a", pinned, uniq_name = "_QFomp_parallel_do_multiple_firstprivateEa"}
-! CHECK:             %[[A_PRIV_DECL:.*]]:2 = hlfir.declare %[[A_PRIV_ADDR]] {uniq_name = "_QFomp_parallel_do_multiple_firstprivateEa"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-! CHECK:             %[[A:.*]] = fir.load %[[A_DECL]]#0 : !fir.ref<i32>
-! CHECK:             hlfir.assign %[[A]] to %[[A_PRIV_DECL]]#0 : i32, !fir.ref<i32>
-
-! CHECK:             %[[B_PRIV_ADDR:.*]] = fir.alloca i32 {bindc_name = "b", pinned, uniq_name = "_QFomp_parallel_do_multiple_firstprivateEb"}
-! CHECK:             %[[B_PRIV_DECL:.*]]:2 = hlfir.declare %[[B_PRIV_ADDR]] {uniq_name = "_QFomp_parallel_do_multiple_firstprivateEb"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-! CHECK:             %[[B:.*]] = fir.load %[[B_DECL]]#0 : !fir.ref<i32>
-! CHECK:             hlfir.assign %[[B]] to %[[B_PRIV_DECL]]#0 : i32, !fir.ref<i32>
-
-! CHECK:             %[[I_PRIV_ADDR:.*]] = fir.alloca i32 {bindc_name = "i", pinned, {{.*}}}
-! CHECK:             %[[I_PRIV_DECL:.*]]:2 = hlfir.declare %[[I_PRIV_ADDR]] {uniq_name = "_QFomp_parallel_do_multiple_firstprivateEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-
 ! CHECK:             %[[VAL_8:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_9:.*]] = arith.constant 10 : i32
 ! CHECK:             %[[VAL_10:.*]] = arith.constant 1 : i32
-! CHECK:             omp.wsloop {
+! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[A_PRIV_ADDR:.*]], @{{.*}} %{{.}}#0 -> %[[B_PRIV_ADDR:.*]], @{{.*}} %{{.}}#0 -> %[[I_PRIV_ADDR:.*]] : !fir.ref<i32>, !fir.ref<i32>, !fir.ref<i32>) {
 ! CHECK-NEXT:         omp.loop_nest (%[[I:.*]]) : i32 = (%[[VAL_8]]) to (%[[VAL_9]]) inclusive step (%[[VAL_10]]) {
+! CHECK:                 %[[A_PRIV_DECL:.*]]:2 = hlfir.declare %[[A_PRIV_ADDR]] {uniq_name = "_QFomp_parallel_do_multiple_firstprivateEa"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+! CHECK:                 %[[B_PRIV_DECL:.*]]:2 = hlfir.declare %[[B_PRIV_ADDR]] {uniq_name = "_QFomp_parallel_do_multiple_firstprivateEb"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+! CHECK:                 %[[I_PRIV_DECL:.*]]:2 = hlfir.declare %[[I_PRIV_ADDR]] {uniq_name = "_QFomp_parallel_do_multiple_firstprivateEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+
 ! CHECK:                 fir.store %[[I]] to %[[I_PRIV_DECL]]#1 : !fir.ref<i32>
 ! CHECK:                 fir.call @_QPbar(%[[I_PRIV_DECL]]#1, %[[A_PRIV_DECL]]#1) {{.*}}: (!fir.ref<i32>, !fir.ref<i32>) -> ()
 ! CHECK:                 omp.yield
diff --git a/flang/test/Lower/OpenMP/pointer-to-array.f90 b/flang/test/Lower/OpenMP/pointer-to-array.f90
new file mode 100644
index 0000000000000..1861b3907bcf0
--- /dev/null
+++ b/flang/test/Lower/OpenMP/pointer-to-array.f90
@@ -0,0 +1,43 @@
+! Regression test for crash compiling privatizer for a pointer to an array.
+! The crash was because the fir.embox was not given a shape but it needs one.
+
+!RUN: %flang_fc1 -emit-hlfir -fopenmp %s -o - | FileCheck %s
+
+! ALLOCATABLE case (2nd subroutine)
+!CHECK-LABEL: omp.private {type = firstprivate}
+!CHECK-SAME: @{{.*}} : !fir.box<!fir.heap<!fir.array<?x!fir.type<{{.*}}>>>> init {
+!CHECK:        if %{{.*}} {
+!CHECK:        %[[SHAPE:.*]] = fir.shape
+!CHECK:        %[[BOX:.*]] = fir.embox %{{.*}}(%[[SHAPE]])
+!CHECK:        } else {
+
+! POINTER case (1st subroutine)
+!CHECK-LABEL: omp.private {type = firstprivate}
+!CHECK-SAME: @{{.*}} : !fir.box<!fir.ptr<!fir.array<?x!fir.type<{{.*}}>>>> init {
+!CHECK:        %[[SHAPE:.*]] = fir.shape
+!CHECK:        %[[ADDR:.*]] = fir.zero_bits
+!CHECK:        %[[BOX:.*]] = fir.embox %[[ADDR]](%[[SHAPE]])
+
+subroutine pointer_to_array_derived
+  type t
+    integer :: i
+  end type
+  type(t), pointer :: a(:)
+  allocate(a(1))
+  a(1)%i = 2
+  !$omp parallel firstprivate(a)
+  if (a(1)%i/=2) stop 2
+  !$omp end parallel
+end subroutine
+
+subroutine allocatable_array_derived
+  type t
+    integer :: i
+  end type
+  type(t), allocatable :: a(:)
+  allocate(a(1))
+  a(1)%i = 2
+  !$omp parallel firstprivate(a)
+  if (a(1)%i/=2) stop 2
+  !$omp end parallel
+end subroutine
diff --git a/flang/test/Lower/OpenMP/private-commonblock.f90 b/flang/test/Lower/OpenMP/private-commonblock.f90
index f6d285a3b011e..84a604cf10992 100644
--- a/flang/test/Lower/OpenMP/private-commonblock.f90
+++ b/flang/test/Lower/OpenMP/private-commonblock.f90
@@ -17,6 +17,8 @@ subroutine private_common
   !$omp end parallel
 end subroutine
 
+!CHECK:    %[[D_BOX_ADDR:.*]] = fir.alloca !fir.box<!fir.array<5x!fir.char<1,5>>>
+!CHECK:    %[[B_BOX_ADDR:.*]] = fir.alloca !fir.box<!fir.array<10xf32>>
 !CHECK:    %[[BLK_ADDR:.*]] = fir.address_of(@blk_) : !fir.ref<!fir.array<74xi8>>
 !CHECK:    %[[I8_ARR:.*]] = fir.convert %[[BLK_ADDR]] : (!fir.ref<!fir.array<74xi8>>) -> !fir.ref<!fir.array<?xi8>>
 !CHECK:    %[[C0:.*]] = arith.constant 0 : index
@@ -48,17 +50,24 @@ subroutine private_common
 !CHECK:    %[[D_REF:.*]] = fir.convert %[[D_DECL]]#1 : (!fir.ref<!fir.array<5x!fir.char<1,5>>>) -> !fir.ref<!fir.char<1,5>>
 !CHECK:    %[[D_BOX:.*]] = fir.emboxchar %[[D_REF]], %[[TP5]] : (!fir.ref<!fir.char<1,5>>, index) -> !fir.boxchar<1>
 !CHECK:    fir.call @_QPsub1(%[[A_DECL]]#1, %[[B_DECL]]#1, %[[C_BOX]], %[[D_BOX]]) fastmath<contract> : (!fir.ref<i32>, !fir.ref<!fir.array<10xf32>>, !fir.boxchar<1>, !fir.boxchar<1>) -> ()
-!CHECK:    omp.parallel private(@{{.*}} %{{.*}}#0 -> %[[A_PVT_REF:.*]], @{{.*}} %{{.*}}#0 -> %[[B_PVT_REF:.*]], @{{.*}} %{{.*}}#0 -> %[[C_PVT_REF:.*]], @{{.*}} %{{.*}}#0 -> %[[D_PVT_REF:.*]] : {{.*}}) {
+!CHECK:    %[[B_BOX:.*]] = fir.embox %[[B_DECL]]#0(%[[SH10]])
+!CHECK:    fir.store %[[B_BOX]] to %[[B_BOX_ADDR]]
+!CHECK:    %[[D_BOX:.*]] = fir.embox %[[D_DECL]]#0(%[[SH5]])
+!CHECK:    fir.store %[[D_BOX]] to %[[D_BOX_ADDR]]
+!CHECK:    omp.parallel private(@{{.*}} %{{.*}}#0 -> %[[A_PVT_REF:.*]], @{{.*}} %[[B_BOX_ADDR]] -> %[[B_PVT_REF:.*]], @{{.*}} %{{.*}}#0 -> %[[C_PVT_REF:.*]], @{{.*}} %[[D_BOX_ADDR]] -> %[[D_PVT_REF:.*]] : {{.*}}) {
 !CHECK:      %[[A_PVT_DECL:.*]]:2 = hlfir.declare %[[A_PVT_REF]] {uniq_name = "_QFprivate_clause_commonblockEa"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-!CHECK:      %[[SH10:.*]] = fir.shape %c10{{.*}} : (index) -> !fir.shape<1>
-!CHECK:      %[[B_PVT_DECL:.*]]:2 = hlfir.declare %[[B_PVT_REF]](%[[SH10]]) {uniq_name = "_QFprivate_clause_commonblockEb"} : (!fir.ref<!fir.array<10xf32>>, !fir.shape<1>) -> (!fir.ref<!fir.array<10xf32>>, !fir.ref<!fir.array<10xf32>>)
+!CHECK:      %[[B_PVT_DECL:.*]]:2 = hlfir.declare %[[B_PVT_REF]] {uniq_name = "_QFprivate_clause_commonblockEb"} :
 !CHECK:      %[[C_PVT_DECL:.*]]:2 = hlfir.declare %[[C_PVT_REF]] typeparams %{{.*}} {uniq_name = "_QFprivate_clause_commonblockEc"} : (!fir.ref<!fir.char<1,5>>, index) -> (!fir.ref<!fir.char<1,5>>, !fir.ref<!fir.char<1,5>>)
-!CHECK:      %[[SH5:.*]] = fir.shape %c5{{.*}} : (index) -> !fir.shape<1>
-!CHECK:      %[[D_PVT_DECL:.*]]:2 = hlfir.declare %[[D_PVT_REF]](%[[SH5]]) typeparams %c5{{.*}} {uniq_name = "_QFprivate_clause_commonblockEd"} : (!fir.ref<!fir.array<5x!fir.char<1,5>>>, !fir.shape<1>, index) -> (!fir.ref<!fir.array<5x!fir.char<1,5>>>, !fir.ref<!fir.array<5x!fir.char<1,5>>>)
+!CHECK:      %[[D_PVT_DECL:.*]]:2 = hlfir.declare %[[D_PVT_REF]]
+!CHECK:      %[[B_LOADED:.*]] = fir.load %[[B_PVT_DECL]]#0
+!CHECK:      %[[B_ADDR:.*]] = fir.box_addr %[[B_LOADED]]
 !CHECK:      %[[C_PVT_BOX:.*]] = fir.emboxchar %[[C_PVT_DECL]]#1, %{{.*}} : (!fir.ref<!fir.char<1,5>>, index) -> !fir.boxchar<1>
-!CHECK:      %[[D_PVT_REF:.*]] = fir.convert %[[D_PVT_DECL]]#1 : (!fir.ref<!fir.array<5x!fir.char<1,5>>>) -> !fir.ref<!fir.char<1,5>>
+
+!CHECK:      %[[D_LOADED:.*]] = fir.load %[[D_PVT_DECL]]#0
+!CHECK:      %[[D_ADDR:.*]] = fir.box_addr %[[D_LOADED]]
+!CHECK:      %[[D_PVT_REF:.*]] = fir.convert %[[D_ADDR]] : (!fir.ref<!fir.array<5x!fir.char<1,5>>>) -> !fir.ref<!fir.char<1,5>>
 !CHECK:      %[[D_PVT_BOX:.*]] = fir.emboxchar %[[D_PVT_REF]], %{{.*}} : (!fir.ref<!fir.char<1,5>>, index) -> !fir.boxchar<1>
-!CHECK:      fir.call @_QPsub2(%[[A_PVT_DECL]]#1, %[[B_PVT_DECL]]#1, %[[C_PVT_BOX]], %[[D_PVT_BOX]]) fastmath<contract> : (!fir.ref<i32>, !fir.ref<!fir.array<10xf32>>, !fir.boxchar<1>, !fir.boxchar<1>) -> ()
+!CHECK:      fir.call @_QPsub2(%[[A_PVT_DECL]]#1, %[[B_ADDR]], %[[C_PVT_BOX]], %[[D_PVT_BOX]]) fastmath<contract> : (!fir.ref<i32>, !fir.ref<!fir.array<10xf32>>, !fir.boxchar<1>, !fir.boxchar<1>) -> ()
 !CHECK:      omp.terminator
 !CHECK:    }
 !CHECK:    %[[C_BOX:.*]] = fir.emboxchar %[[C_DECL]]#1, %{{.*}} : (!fir.ref<!fir.char<1,5>>, index) -> !fir.boxchar<1>
diff --git a/flang/test/Lower/OpenMP/private-derived-type.f90 b/flang/test/Lower/OpenMP/private-derived-type.f90
index df1c7c3f92227..91d8fa753f2ec 100644
--- a/flang/test/Lower/OpenMP/private-derived-type.f90
+++ b/flang/test/Lower/OpenMP/private-derived-type.f90
@@ -15,16 +15,14 @@ subroutine s4
   !$omp end parallel
 end subroutine s4
 
-! CHECK:  omp.private {type = private} @[[DERIVED_PRIV:.*]] : !fir.ref<!fir.type<{{.*}}y3{x:!fir.box<!fir.heap<i32>>}>> alloc {
-! CHECK:             %[[VAL_23:.*]] = fir.alloca !fir.type<_QFs4Ty3{x:!fir.box<!fir.heap<i32>>}> {bindc_name = "v", pinned, uniq_name = "_QFs4Ev"}
-! CHECK:             %[[VAL_25:.*]] = fir.embox %[[VAL_23]] : (!fir.ref<!fir.type<_QFs4Ty3{x:!fir.box<!fir.heap<i32>>}>>) -> !fir.box<!fir.type<_QFs4Ty3{x:!fir.box<!fir.heap<i32>>}>>
+! CHECK:  omp.private {type = private} @[[DERIVED_PRIV:.*]] : !fir.type<{{.*}}y3{x:!fir.box<!fir.heap<i32>>}> init {
+! CHECK:             %[[VAL_25:.*]] = fir.embox %[[VAL_23:.*]] : (!fir.ref<!fir.type<_QFs4Ty3{x:!fir.box<!fir.heap<i32>>}>>) -> !fir.box<!fir.type<_QFs4Ty3{x:!fir.box<!fir.heap<i32>>}>>
 ! CHECK:             %[[VAL_26:.*]] = fir.address_of
 ! CHECK:             %[[VAL_27:.*]] = arith.constant 8 : i32
 ! CHECK:             %[[VAL_28:.*]] = fir.convert %[[VAL_25]] : (!fir.box<!fir.type<_QFs4Ty3{x:!fir.box<!fir.heap<i32>>}>>) -> !fir.box<none>
 ! CHECK:             %[[VAL_29:.*]] = fir.convert %[[VAL_26]] : (!fir.ref<!fir.char<1,{{.*}}>>) -> !fir.ref<i8>
 !                    Check we do call FortranAInitialize on the derived type
 ! CHECK:             fir.call @_FortranAInitialize(%[[VAL_28]], %[[VAL_29]], %[[VAL_27]]) fastmath<contract> : (!fir.box<none>, !fir.ref<i8>, i32) -> ()
-! CHECK:             %[[VAL_24:.*]]:2 = hlfir.declare %[[VAL_23]] {uniq_name = "_QFs4Ev"} : (!fir.ref<!fir.type<_QFs4Ty3{x:!fir.box<!fir.heap<i32>>}>>) -> (!fir.ref<!fir.type<_QFs4Ty3{x:!fir.box<!fir.heap<i32>>}>>, !fir.ref<!fir.type<_QFs4Ty3{x:!fir.box<!fir.heap<i32>>}>>)
 ! CHECK:  }
 
 ! CHECK-LABEL:   func.func @_QPs4() {
diff --git a/flang/test/Lower/OpenMP/same_var_first_lastprivate.f90 b/flang/test/Lower/OpenMP/same_var_first_lastprivate.f90
index e8e4a0802e00d..ee914f23aacf3 100644
--- a/flang/test/Lower/OpenMP/same_var_first_lastprivate.f90
+++ b/flang/test/Lower/OpenMP/same_var_first_lastprivate.f90
@@ -10,11 +10,7 @@ subroutine first_and_lastprivate
   !$omp end parallel do
 end subroutine
 
-! CHECK:  omp.private {type = firstprivate} @{{.*}}Evar_firstprivate_ref_i32 : {{.*}} alloc {
-! CHECK:    %[[ALLOC:.*]] = fir.alloca i32 {{.*}}
-! CHECK:    %[[ALLOC_DECL:.*]]:2 = hlfir.declare %[[ALLOC]]
-! CHECK:    omp.yield(%[[ALLOC_DECL]]#0 : !fir.ref<i32>)
-! CHECK:  } copy {
+! CHECK:  omp.private {type = firstprivate} @{{.*}}Evar_firstprivate_i32 : {{.*}} copy {
 ! CHECK: ^{{.*}}(%[[ORIG_REF:.*]]: {{.*}}, %[[PRIV_REF:.*]]: {{.*}}):
 ! CHECK:    %[[ORIG_VAL:.*]] = fir.load %[[ORIG_REF]]
 ! CHECK:    hlfir.assign %[[ORIG_VAL]] to %[[PRIV_REF]]
@@ -25,7 +21,7 @@ subroutine first_and_lastprivate
 ! CHECK:    %[[ORIG_VAR_DECL:.*]]:2 = hlfir.declare %{{.*}} {uniq_name = "{{.*}}Evar"}
 ! CHECK:    omp.parallel {
 ! CHECK:      omp.barrier
-! CHECK:      omp.wsloop private(@{{.*}}var_firstprivate_ref_i32 {{.*}}) {
+! CHECK:      omp.wsloop private(@{{.*}}var_firstprivate_i32 {{.*}}) {
 ! CHECK:        omp.loop_nest {{.*}} {
 ! CHECK:          %[[PRIV_VAR_DECL:.*]]:2 = hlfir.declare %{{.*}} {uniq_name = "{{.*}}Evar"}
 ! CHECK:          fir.if %{{.*}} {
diff --git a/flang/test/Lower/OpenMP/scan.f90 b/flang/test/Lower/OpenMP/scan.f90
new file mode 100644
index 0000000000000..383c1865c50d7
--- /dev/null
+++ b/flang/test/Lower/OpenMP/scan.f90
@@ -0,0 +1,36 @@
+! RUN: bbc -emit-hlfir -fopenmp %s -o - | FileCheck %s
+! RUN: %flang_fc1 -emit-hlfir -fopenmp %s -o - | FileCheck %s
+
+! CHECK: omp.wsloop private({{.*}}) reduction(mod: inscan, @add_reduction_i32 %{{.*}} -> %[[RED_ARG_1:.*]] : {{.*}}) {
+! CHECK: %[[RED_DECL_1:.*]]:2 = hlfir.declare %[[RED_ARG_1]]
+! CHECK: omp.scan inclusive(%[[RED_DECL_1]]#1 : {{.*}})
+
+subroutine inclusive_scan(a, b, n)
+ implicit none
+ integer a(:), b(:)
+ integer x, k, n
+
+ !$omp parallel do reduction(inscan, +: x)
+ do k = 1, n
+   x = x + a(k)
+   !$omp scan inclusive(x)
+   b(k) = x
+ end do
+end subroutine inclusive_scan
+
+
+! CHECK: omp.wsloop private({{.*}}) reduction(mod: inscan, @add_reduction_i32 %{{.*}} -> %[[RED_ARG_2:.*]] : {{.*}}) {
+! CHECK: %[[RED_DECL_2:.*]]:2 = hlfir.declare %[[RED_ARG_2]]
+! CHECK: omp.scan exclusive(%[[RED_DECL_2]]#1 : {{.*}})
+subroutine exclusive_scan(a, b, n)
+ implicit none
+ integer a(:), b(:)
+ integer x, k, n
+
+ !$omp parallel do reduction(inscan, +: x)
+ do k = 1, n
+   x = x + a(k)
+   !$omp scan exclusive(x)
+   b(k) = x
+ end do
+end subroutine exclusive_scan
diff --git a/flang/test/Lower/OpenMP/sections.f90 b/flang/test/Lower/OpenMP/sections.f90
index 2287918528635..5900eef6ea280 100644
--- a/flang/test/Lower/OpenMP/sections.f90
+++ b/flang/test/Lower/OpenMP/sections.f90
@@ -2,7 +2,7 @@
 
 ! This test checks the lowering of OpenMP sections construct with several clauses present
 
-! RUN: %flang_fc1 -flang-experimental-hlfir -emit-hlfir %openmp_flags %s -o - | FileCheck %s
+! RUN: %flang_fc1 -emit-hlfir %openmp_flags %s -o - | FileCheck %s
 ! RUN: bbc -hlfir -emit-hlfir %openmp_flags %s -o - | FileCheck %s
 
 !CHECK: func @_QQmain() attributes {fir.bindc_name = "sample"} {
@@ -263,6 +263,23 @@ subroutine lastprivate2()
     !$omp end sections
 end subroutine
 
+!CHECK-LABEL: func @_QPlastprivate_common
+!CHECK: %[[I:.*]]:2 = hlfir.declare %{{.*}} {uniq_name = "_QFlastprivate_commonEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+!CHECK: %[[I_PRIV:.*]]:2 = hlfir.declare %{{.*}} {uniq_name = "_QFlastprivate_commonEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+!CHECK: omp.sections
+!CHECK: omp.section
+!CHECK: %[[TMP:.*]] = fir.load %[[I_PRIV]]#0 : !fir.ref<i32>
+!CHECK: hlfir.assign %[[TMP]] to %[[I]]#0 : i32, !fir.ref<i32>
+subroutine lastprivate_common()
+    integer :: i
+    common /com/ i
+
+    i = 1
+    !$omp sections lastprivate(/com/)
+        i = 2
+    !$omp end sections
+end subroutine
+
 !CHECK-LABEL: func @_QPunstructured_sections_privatization
 subroutine unstructured_sections_privatization()
 !CHECK: %[[X:.*]] = fir.alloca f32 {bindc_name = "x", uniq_name = "_QFunstructured_sections_privatizationEx"}
diff --git a/flang/test/Lower/OpenMP/simd.f90 b/flang/test/Lower/OpenMP/simd.f90
index 0345ace24aaa0..fc3d908801ff1 100644
--- a/flang/test/Lower/OpenMP/simd.f90
+++ b/flang/test/Lower/OpenMP/simd.f90
@@ -254,7 +254,7 @@ subroutine lastprivate_with_simd
   real :: sum
 
   
-!CHECK: omp.simd private(@_QFlastprivate_with_simdEsum_private_ref_f32 %[[VAR_SUM_DECLARE]]#0 -> %[[VAR_SUM_PINNED:.*]], @{{.*}}) {
+!CHECK: omp.simd private(@_QFlastprivate_with_simdEsum_private_f32 %[[VAR_SUM_DECLARE]]#0 -> %[[VAR_SUM_PINNED:.*]], @{{.*}}) {
 !CHECK: omp.loop_nest (%[[ARG:.*]]) : i32 = ({{.*}} to ({{.*}}) inclusive step ({{.*}}) {
 !CHECK: %[[VAR_SUM_PINNED_DECLARE:.*]]:2 = hlfir.declare %[[VAR_SUM_PINNED]] {{.*}}
 !CHECK: %[[ADD_RESULT:.*]] = arith.addi {{.*}}
diff --git a/flang/test/Lower/OpenMP/stop-stmt-in-region.f90 b/flang/test/Lower/OpenMP/stop-stmt-in-region.f90
index f174caa7fa4fd..d817c4e771b31 100644
--- a/flang/test/Lower/OpenMP/stop-stmt-in-region.f90
+++ b/flang/test/Lower/OpenMP/stop-stmt-in-region.f90
@@ -80,14 +80,13 @@ subroutine test_stop_in_region3()
 ! CHECK:         %[[VAL_2:.*]] = fir.alloca i32 {bindc_name = "x", uniq_name = "_QFtest_stop_in_region4Ex"}
 ! CHECK:         %[[VAL_2_DECL:.*]]:2 = hlfir.declare %[[VAL_2]] {uniq_name = "_QFtest_stop_in_region4Ex"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 
-! CHECK:         %[[VAL_0:.*]] = fir.alloca i32 {bindc_name = "i", pinned, {{.*}}}
-! CHECK:         %[[VAL_0_DECL:.*]]:2 = hlfir.declare %[[VAL_0]] {uniq_name = "_QFtest_stop_in_region4Ei"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 
 ! CHECK:         %[[VAL_3:.*]] = arith.constant 1 : i32
 ! CHECK:         %[[VAL_4:.*]] = arith.constant 10 : i32
 ! CHECK:         %[[VAL_5:.*]] = arith.constant 1 : i32
-! CHECK:         omp.wsloop {
+! CHECK:         omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_0:.*]] : !fir.ref<i32>) {
 ! CHECK-NEXT:      omp.loop_nest (%[[VAL_6:.*]]) : i32 = (%[[VAL_3]]) to (%[[VAL_4]]) inclusive step (%[[VAL_5]]) {
+! CHECK:             %[[VAL_0_DECL:.*]]:2 = hlfir.declare %[[VAL_0]] {uniq_name = "_QFtest_stop_in_region4Ei"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:             fir.store %[[VAL_6]] to %[[VAL_0_DECL]]#1 : !fir.ref<i32>
 ! CHECK:             cf.br ^bb1
 ! CHECK:           ^bb1:
diff --git a/flang/test/Lower/OpenMP/target.f90 b/flang/test/Lower/OpenMP/target.f90
index 04764be2293c1..bf801e69405b9 100644
--- a/flang/test/Lower/OpenMP/target.f90
+++ b/flang/test/Lower/OpenMP/target.f90
@@ -586,11 +586,10 @@ subroutine omp_target_parallel_do
       !CHECK: %[[VAL_0_DECL:.*]]:2 = hlfir.declare %[[ARG_0]](%{{.*}}) {uniq_name = "_QFomp_target_parallel_doEa"} : (!fir.ref<!fir.array<1024xi32>>, !fir.shape<1>) -> (!fir.ref<!fir.array<1024xi32>>, !fir.ref<!fir.array<1024xi32>>)
       !CHECK: omp.parallel
       !$omp target parallel do map(tofrom: a)
-         !CHECK: %[[I_PVT_ALLOCA:.*]] = fir.alloca i32 {bindc_name = "i", pinned, {{.*}}}
-         !CHECK: %[[I_PVT_DECL:.*]]:2 = hlfir.declare %[[I_PVT_ALLOCA]] {uniq_name = "_QFomp_target_parallel_doEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-         !CHECK: omp.wsloop {
+         !CHECK: omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[I_PVT_ALLOCA:.*]] : !fir.ref<i32>) {
          !CHECK-NEXT: omp.loop_nest (%[[I_VAL:.*]]) : i32
          do i = 1, 1024
+         !CHECK:     %[[I_PVT_DECL:.*]]:2 = hlfir.declare %[[I_PVT_ALLOCA]] {uniq_name = "_QFomp_target_parallel_doEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
            !CHECK:   fir.store %[[I_VAL]] to %[[I_PVT_DECL]]#1 : !fir.ref<i32>
            !CHECK:   %[[C10:.*]] = arith.constant 10 : i32
            !CHECK:   %[[I_PVT_VAL:.*]] = fir.load %[[I_PVT_DECL]]#0 : !fir.ref<i32>
diff --git a/flang/test/Lower/OpenMP/task2.f90 b/flang/test/Lower/OpenMP/task2.f90
index 734e75c5bba06..85f934f109aff 100644
--- a/flang/test/Lower/OpenMP/task2.f90
+++ b/flang/test/Lower/OpenMP/task2.f90
@@ -2,7 +2,7 @@
 
 
 !CHECK-LABEL: omp.private
-!CHECK-SAME:      {type = firstprivate} @[[PRIVATIZER:.*]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>> alloc {
+!CHECK-SAME:      {type = firstprivate} @[[PRIVATIZER:.*]] : !fir.box<!fir.heap<!fir.array<?xi32>>> init {
 !CHECK:         fir.if
 !CHECK:       } copy {
 !CHECK:         fir.if
diff --git a/flang/test/Lower/OpenMP/unstructured.f90 b/flang/test/Lower/OpenMP/unstructured.f90
index a0955c8440c1c..febefef82ad1a 100644
--- a/flang/test/Lower/OpenMP/unstructured.f90
+++ b/flang/test/Lower/OpenMP/unstructured.f90
@@ -69,10 +69,9 @@ subroutine ss2(n) ! unstructured OpenMP construct; loop exit inside construct
 ! CHECK:     cond_br %{{[0-9]*}}, ^bb2, ^bb4
 ! CHECK:   ^bb2:  // pred: ^bb1
 
-! CHECK:     %[[ALLOCA_2:.*]] = fir.alloca i32 {{{.*}}, pinned, {{.*}}}
-! CHECK:     %[[OMP_LOOP_K_DECL:.*]]:2 = hlfir.declare %[[ALLOCA_2]] {uniq_name = "_QFss3Ek"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-! CHECK:     omp.wsloop {
+! CHECK:     omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[ALLOCA_2:.*]] : !fir.ref<i32>) {
 ! CHECK:       omp.loop_nest (%[[ARG1:.*]]) : {{.*}} {
+! CHECK:         %[[OMP_LOOP_K_DECL:.*]]:2 = hlfir.declare %[[ALLOCA_2]] {uniq_name = "_QFss3Ek"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:         fir.store %[[ARG1]] to %[[OMP_LOOP_K_DECL]]#1 : !fir.ref<i32>
 ! CHECK:         @_FortranAioBeginExternalListOutput
 ! CHECK:         %[[LOAD_1:.*]] = fir.load %[[OMP_LOOP_K_DECL]]#0 : !fir.ref<i32>
@@ -81,11 +80,9 @@ subroutine ss2(n) ! unstructured OpenMP construct; loop exit inside construct
 ! CHECK:       }
 ! CHECK:     }
 
-! CHECK:     %[[ALLOCA_1:.*]] = fir.alloca i32 {{{.*}}, pinned, {{.*}}}
-! CHECK:     %[[OMP_LOOP_J_DECL:.*]]:2 = hlfir.declare %[[ALLOCA_1]] {uniq_name = "_QFss3Ej"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-
-! CHECK:     omp.wsloop {
+! CHECK:     omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[ALLOCA_1:.*]] : !fir.ref<i32>) {
 ! CHECK:       omp.loop_nest (%[[ARG2:.*]]) : {{.*}} {
+! CHECK:         %[[OMP_LOOP_J_DECL:.*]]:2 = hlfir.declare %[[ALLOCA_1]] {uniq_name = "_QFss3Ej"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:         fir.store %[[ARG2]] to %[[OMP_LOOP_J_DECL]]#1 : !fir.ref<i32>
 ! CHECK:         br ^bb1
 ! CHECK:       ^bb2:  // 2 preds: ^bb1, ^bb5
@@ -128,10 +125,9 @@ subroutine ss3(n) ! nested unstructured OpenMP constructs
 
 ! CHECK-LABEL: func @_QPss4{{.*}} {
 ! CHECK:       omp.parallel private(@{{.*}} %{{.*}}#0 -> %{{.*}} : {{.*}}) {
-! CHECK:         %[[ALLOCA:.*]] = fir.alloca i32 {{{.*}}, pinned, uniq_name = "_QFss4Ej"}
-! CHECK:         %[[OMP_LOOP_J_DECL:.*]]:2 = hlfir.declare %[[ALLOCA]] {uniq_name = "_QFss4Ej"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-! CHECK:         omp.wsloop {
+! CHECK:         omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[ALLOCA:.*]] : !fir.ref<i32>) {
 ! CHECK-NEXT:      omp.loop_nest (%[[ARG:.*]]) : {{.*}} {
+! CHECK:             %[[OMP_LOOP_J_DECL:.*]]:2 = hlfir.declare %[[ALLOCA]] {uniq_name = "_QFss4Ej"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:             fir.store %[[ARG]] to %[[OMP_LOOP_J_DECL]]#1 : !fir.ref<i32>
 ! CHECK:             %[[COND:.*]] = arith.cmpi eq, %{{.*}}, %{{.*}}
 ! CHECK:             %[[COND_XOR:.*]] = arith.xori %[[COND]], %{{.*}}
@@ -160,7 +156,7 @@ subroutine ss4(n) ! CYCLE in OpenMP wsloop constructs
 
 ! CHECK-LABEL: func @_QPss5() {
 ! CHECK:  omp.parallel private(@{{.*}} %{{.*}}#0 -> %{{.*}} : {{.*}}) {
-! CHECK:    omp.wsloop {
+! CHECK:    omp.wsloop private({{.*}}) {
 ! CHECK:      omp.loop_nest {{.*}} {
 ! CHECK:        br ^[[BB1:.*]]
 ! CHECK:      ^[[BB1]]:
@@ -202,7 +198,7 @@ subroutine ss5() ! EXIT inside OpenMP wsloop (inside parallel)
 ! CHECK:  ^[[BB1_OUTER]]:
 ! CHECK:    cond_br %{{.*}}, ^[[BB2_OUTER:.*]], ^[[BB3_OUTER:.*]]
 ! CHECK:  ^[[BB2_OUTER]]:
-! CHECK:    omp.wsloop {
+! CHECK:    omp.wsloop private({{.*}}) {
 ! CHECK:      omp.loop_nest {{.*}} {
 ! CHECK:        br ^[[BB1:.*]]
 ! CHECK:      ^[[BB1]]:
@@ -248,7 +244,7 @@ subroutine ss6() ! EXIT inside OpenMP wsloop in a do loop (inside parallel)
 ! CHECK:   cond_br %{{.*}}, ^[[BB2_OUTER:.*]], ^[[BB3_OUTER:.*]]
 ! CHECK-NEXT: ^[[BB2_OUTER:.*]]:
 ! CHECK:   omp.parallel  {
-! CHECK:     omp.wsloop {
+! CHECK:     omp.wsloop private({{.*}}) {
 ! CHECK:       omp.loop_nest {{.*}} {
 ! CHECK:         br ^[[BB1:.*]]
 ! CHECK-NEXT:       ^[[BB1]]:
@@ -288,7 +284,7 @@ subroutine ss7() ! EXIT inside OpenMP parallel do (inside do loop)
 
 ! CHECK-LABEL: func @_QPss8() {
 ! CHECK:  omp.parallel  {
-! CHECK:    omp.wsloop {
+! CHECK:    omp.wsloop private({{.*}}) {
 ! CHECK:      omp.loop_nest {{.*}} {
 ! CHECK:        br ^[[BB1:.*]]
 ! CHECK-NEXT:      ^[[BB1]]:
@@ -337,7 +333,7 @@ subroutine ss8() ! EXIT inside OpenMP parallel do
 ! CHECK:      omp.terminator
 ! CHECK-NEXT:    }
 ! CHECK:    omp.terminator
-! CHECK-NEXT  }
+! CHECK-NEXT: }
 ! CHECK: }
 subroutine ss9() ! EXIT inside OpenMP parallel (inside parallel)
   integer :: x
diff --git a/flang/test/Lower/OpenMP/wsloop-chunks.f90 b/flang/test/Lower/OpenMP/wsloop-chunks.f90
index 0fb7d6f1b64fa..0a2b962761acb 100644
--- a/flang/test/Lower/OpenMP/wsloop-chunks.f90
+++ b/flang/test/Lower/OpenMP/wsloop-chunks.f90
@@ -20,7 +20,7 @@ program wsloop
 ! CHECK:         %[[VAL_3:.*]] = arith.constant 1 : i32
 ! CHECK:         %[[VAL_4:.*]] = arith.constant 9 : i32
 ! CHECK:         %[[VAL_5:.*]] = arith.constant 1 : i32
-! CHECK:         omp.wsloop nowait schedule(static = %[[VAL_2]] : i32) {
+! CHECK:         omp.wsloop nowait schedule(static = %[[VAL_2]] : i32) private({{.*}}) {
 ! CHECK-NEXT:      omp.loop_nest (%[[ARG0:.*]]) : i32 = (%[[VAL_3]]) to (%[[VAL_4]]) inclusive step (%[[VAL_5]]) {
 ! CHECK:             fir.store %[[ARG0]] to %[[STORE_IV:.*]]#1 : !fir.ref<i32>
 ! CHECK:             %[[LOAD_IV:.*]] = fir.load %[[STORE_IV]]#0 : !fir.ref<i32>
@@ -40,7 +40,7 @@ program wsloop
 ! CHECK:         %[[VAL_15:.*]] = arith.constant 1 : i32
 ! CHECK:         %[[VAL_16:.*]] = arith.constant 9 : i32
 ! CHECK:         %[[VAL_17:.*]] = arith.constant 1 : i32
-! CHECK:         omp.wsloop nowait schedule(static = %[[VAL_14]] : i32) {
+! CHECK:         omp.wsloop nowait schedule(static = %[[VAL_14]] : i32) private({{.*}}) {
 ! CHECK-NEXT:      omp.loop_nest (%[[ARG1:.*]]) : i32 = (%[[VAL_15]]) to (%[[VAL_16]]) inclusive step (%[[VAL_17]]) {
 ! CHECK:             fir.store %[[ARG1]] to %[[STORE_IV1:.*]]#1 : !fir.ref<i32>
 ! CHECK:             %[[VAL_24:.*]] = arith.constant 2 : i32
@@ -66,7 +66,7 @@ program wsloop
 ! CHECK:         %[[VAL_30:.*]] = arith.constant 1 : i32
 ! CHECK:         %[[VAL_31:.*]] = arith.constant 9 : i32
 ! CHECK:         %[[VAL_32:.*]] = arith.constant 1 : i32
-! CHECK:         omp.wsloop nowait schedule(static = %[[VAL_29]] : i32) {
+! CHECK:         omp.wsloop nowait schedule(static = %[[VAL_29]] : i32) private({{.*}}) {
 ! CHECK-NEXT:      omp.loop_nest (%[[ARG2:.*]]) : i32 = (%[[VAL_30]]) to (%[[VAL_31]]) inclusive step (%[[VAL_32]]) {
 ! CHECK:             fir.store %[[ARG2]] to %[[STORE_IV2:.*]]#1 : !fir.ref<i32>
 ! CHECK:             %[[VAL_39:.*]] = arith.constant 3 : i32
diff --git a/flang/test/Lower/OpenMP/wsloop-collapse.f90 b/flang/test/Lower/OpenMP/wsloop-collapse.f90
index 61ee76d589107..6d9862e625400 100644
--- a/flang/test/Lower/OpenMP/wsloop-collapse.f90
+++ b/flang/test/Lower/OpenMP/wsloop-collapse.f90
@@ -38,15 +38,6 @@ program wsloop_collapse
 !CHECK:           %[[VAL_23:.*]] = arith.constant 0 : i32
 !CHECK:           hlfir.assign %[[VAL_23]] to %[[VAL_19]]#0 : i32, !fir.ref<i32>
 
-!CHECK:           %[[VAL_4:.*]] = fir.alloca i32 {bindc_name = "i", pinned, {{.*}}}
-!CHECK:           %[[VAL_5:.*]]:2 = hlfir.declare %[[VAL_4]] {uniq_name = "_QFEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-
-!CHECK:           %[[VAL_2:.*]] = fir.alloca i32 {bindc_name = "j", pinned, {{.*}}}
-!CHECK:           %[[VAL_3:.*]]:2 = hlfir.declare %[[VAL_2]] {uniq_name = "_QFEj"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-
-!CHECK:           %[[VAL_0:.*]] = fir.alloca i32 {bindc_name = "k", pinned, {{.*}}}
-!CHECK:           %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0]] {uniq_name = "_QFEk"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-
   integer :: i, j, k
   integer :: a, b, c
   integer :: x
@@ -65,12 +56,17 @@ program wsloop_collapse
 !CHECK:           %[[VAL_30:.*]] = arith.constant 1 : i32
 !CHECK:           %[[VAL_31:.*]] = fir.load %[[VAL_11]]#0 : !fir.ref<i32>
 !CHECK:           %[[VAL_32:.*]] = arith.constant 1 : i32
-!CHECK:           omp.wsloop {
+!CHECK:           omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_4:.*]], @{{.*}} %{{.*}}#0 -> %[[VAL_2:.*]], @{{.*}} %{{.*}}#0 -> %[[VAL_0:.*]] : !fir.ref<i32>, !fir.ref<i32>, !fir.ref<i32>) {
 !CHECK-NEXT:        omp.loop_nest (%[[VAL_33:.*]], %[[VAL_34:.*]], %[[VAL_35:.*]]) : i32 = (%[[VAL_24]], %[[VAL_27]], %[[VAL_30]]) to (%[[VAL_25]], %[[VAL_28]], %[[VAL_31]]) inclusive step (%[[VAL_26]], %[[VAL_29]], %[[VAL_32]]) {
   !$omp do collapse(3)
   do i = 1, a
      do j= 1, b
         do k = 1, c
+
+!CHECK:               %[[VAL_5:.*]]:2 = hlfir.declare %[[VAL_4]] {uniq_name = "_QFEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+!CHECK:               %[[VAL_3:.*]]:2 = hlfir.declare %[[VAL_2]] {uniq_name = "_QFEj"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+!CHECK:               %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0]] {uniq_name = "_QFEk"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+
 !CHECK:               fir.store %[[VAL_33]] to %[[VAL_5]]#1 : !fir.ref<i32>
 !CHECK:               fir.store %[[VAL_34]] to %[[VAL_3]]#1 : !fir.ref<i32>
 !CHECK:               fir.store %[[VAL_35]] to %[[VAL_1]]#1 : !fir.ref<i32>
diff --git a/flang/test/Lower/OpenMP/wsloop-monotonic.f90 b/flang/test/Lower/OpenMP/wsloop-monotonic.f90
index 9659fff2d42e7..e21aa4c678f42 100644
--- a/flang/test/Lower/OpenMP/wsloop-monotonic.f90
+++ b/flang/test/Lower/OpenMP/wsloop-monotonic.f90
@@ -11,11 +11,10 @@ program wsloop_dynamic
 !CHECK:  omp.parallel {
 
 !$OMP DO SCHEDULE(monotonic:dynamic)
-!CHECK:     %[[ALLOCA_IV:.*]] = fir.alloca i32 {{{.*}}, pinned, {{.*}}}
 !CHECK:     %[[WS_LB:.*]] = arith.constant 1 : i32
 !CHECK:     %[[WS_UB:.*]] = arith.constant 9 : i32
 !CHECK:     %[[WS_STEP:.*]] = arith.constant 1 : i32
-!CHECK:     omp.wsloop nowait schedule(dynamic, monotonic) {
+!CHECK:     omp.wsloop nowait schedule(dynamic, monotonic) private({{.*}}) {
 !CHECK-NEXT:  omp.loop_nest (%[[I:.*]]) : i32 = (%[[WS_LB]]) to (%[[WS_UB]]) inclusive step (%[[WS_STEP]]) {
 !CHECK:         fir.store %[[I]] to %[[ALLOCA_IV:.*]]#1 : !fir.ref<i32>
 
diff --git a/flang/test/Lower/OpenMP/wsloop-nonmonotonic.f90 b/flang/test/Lower/OpenMP/wsloop-nonmonotonic.f90
index b1bea525ff489..23d3c49c00786 100644
--- a/flang/test/Lower/OpenMP/wsloop-nonmonotonic.f90
+++ b/flang/test/Lower/OpenMP/wsloop-nonmonotonic.f90
@@ -12,13 +12,12 @@ program wsloop_dynamic
 !CHECK:  omp.parallel {
 
 !$OMP DO SCHEDULE(nonmonotonic:dynamic)
-!CHECK:     %[[I_REF:.*]] = fir.alloca i32 {{{.*}}, pinned, {{.*}}}
-!CHECK:     %[[ALLOCA_IV:.*]]:2 = hlfir.declare %[[I_REF]] {uniq_name = "_QFEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 !CHECK:     %[[WS_LB:.*]] = arith.constant 1 : i32
 !CHECK:     %[[WS_UB:.*]] = arith.constant 9 : i32
 !CHECK:     %[[WS_STEP:.*]] = arith.constant 1 : i32
-!CHECK:     omp.wsloop nowait schedule(dynamic, nonmonotonic) {
+!CHECK:     omp.wsloop nowait schedule(dynamic, nonmonotonic) private(@{{.*}} %{{.*}}#0 -> %[[I_REF:.*]] : !fir.ref<i32>) {
 !CHECK-NEXT:  omp.loop_nest (%[[I:.*]]) : i32 = (%[[WS_LB]]) to (%[[WS_UB]]) inclusive step (%[[WS_STEP]]) {
+!CHECK:         %[[ALLOCA_IV:.*]]:2 = hlfir.declare %[[I_REF]] {uniq_name = "_QFEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 !CHECK:         fir.store %[[I]] to %[[ALLOCA_IV]]#1 : !fir.ref<i32>
 
   do i=1, 9
diff --git a/flang/test/Lower/OpenMP/wsloop-ordered.f90 b/flang/test/Lower/OpenMP/wsloop-ordered.f90
index 5fa53f7b28447..4862b7296a9bc 100644
--- a/flang/test/Lower/OpenMP/wsloop-ordered.f90
+++ b/flang/test/Lower/OpenMP/wsloop-ordered.f90
@@ -6,7 +6,7 @@
 subroutine wsloop_ordered_no_para()
   integer :: a(10), i
 
-! CHECK:  omp.wsloop ordered(0) {
+! CHECK:  omp.wsloop ordered(0) private({{.*}}) {
 ! CHECK-NEXT: omp.loop_nest (%{{.*}}) : i32 = (%{{.*}}) to (%{{.*}}) inclusive step (%{{.*}}) {
 ! CHECK:        omp.yield
 ! CHECK:      }
@@ -27,7 +27,7 @@ subroutine wsloop_ordered_with_para()
   integer :: a(10), i
 
 ! CHECK: func @_QPwsloop_ordered_with_para() {
-! CHECK:  omp.wsloop ordered(1) {
+! CHECK:  omp.wsloop ordered(1) private({{.*}}) {
 ! CHECK-NEXT: omp.loop_nest (%{{.*}}) : i32 = (%{{.*}}) to (%{{.*}}) inclusive step (%{{.*}}) {
 ! CHECK:        omp.yield
 ! CHECK:      }
diff --git a/flang/test/Lower/OpenMP/wsloop-reduction-add-byref.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-add-byref.f90
index c38a79191bc4e..bc021e7a3b273 100644
--- a/flang/test/Lower/OpenMP/wsloop-reduction-add-byref.f90
+++ b/flang/test/Lower/OpenMP/wsloop-reduction-add-byref.f90
@@ -85,13 +85,12 @@
 ! CHECK:           %[[VAL_4:.*]] = arith.constant 0 : i32
 ! CHECK:           hlfir.assign %[[VAL_4]] to %[[VAL_3]]#0 : i32, !fir.ref<i32>
 ! CHECK:           omp.parallel {
-! CHECK:             %[[VAL_5:.*]] = fir.alloca i32 {bindc_name = "i", pinned, {{.*}}}
-! CHECK:             %[[VAL_6:.*]]:2 = hlfir.declare %[[VAL_5]] {uniq_name = "_QFsimple_int_reductionEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:             %[[VAL_7:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_8:.*]] = arith.constant 100 : i32
 ! CHECK:             %[[VAL_9:.*]] = arith.constant 1 : i32
-! CHECK:             omp.wsloop reduction(byref @add_reduction_byref_i32 %[[VAL_3]]#0 -> %[[VAL_10:.*]] : !fir.ref<i32>) {
+! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_5:.*]] : !fir.ref<i32>) reduction(byref @add_reduction_byref_i32 %[[VAL_3]]#0 -> %[[VAL_10:.*]] : !fir.ref<i32>) {
 ! CHECK-NEXT:          omp.loop_nest (%[[VAL_11:.*]]) : i32 = (%[[VAL_7]]) to (%[[VAL_8]]) inclusive step (%[[VAL_9]]) {
+! CHECK:                 %[[VAL_6:.*]]:2 = hlfir.declare %[[VAL_5]] {uniq_name = "_QFsimple_int_reductionEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:                 %[[VAL_12:.*]]:2 = hlfir.declare %[[VAL_10]] {uniq_name = "_QFsimple_int_reductionEx"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:                 fir.store %[[VAL_11]] to %[[VAL_6]]#1 : !fir.ref<i32>
 ! CHECK:                 %[[VAL_13:.*]] = fir.load %[[VAL_12]]#0 : !fir.ref<i32>
@@ -127,13 +126,12 @@ subroutine simple_int_reduction
 ! CHECK:           %[[VAL_4:.*]] = arith.constant 0.000000e+00 : f32
 ! CHECK:           hlfir.assign %[[VAL_4]] to %[[VAL_3]]#0 : f32, !fir.ref<f32>
 ! CHECK:           omp.parallel {
-! CHECK:             %[[VAL_5:.*]] = fir.alloca i32 {bindc_name = "i", pinned, {{.*}}}
-! CHECK:             %[[VAL_6:.*]]:2 = hlfir.declare %[[VAL_5]] {uniq_name = "_QFsimple_real_reductionEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:             %[[VAL_7:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_8:.*]] = arith.constant 100 : i32
 ! CHECK:             %[[VAL_9:.*]] = arith.constant 1 : i32
-! CHECK:             omp.wsloop reduction(byref @add_reduction_byref_f32 %[[VAL_3]]#0 -> %[[VAL_10:.*]] : !fir.ref<f32>) {
+! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_5:.*]] : !fir.ref<i32>) reduction(byref @add_reduction_byref_f32 %[[VAL_3]]#0 -> %[[VAL_10:.*]] : !fir.ref<f32>) {
 ! CHECK-NEXT:          omp.loop_nest (%[[VAL_11:.*]]) : i32 = (%[[VAL_7]]) to (%[[VAL_8]]) inclusive step (%[[VAL_9]]) {
+! CHECK:                 %[[VAL_6:.*]]:2 = hlfir.declare %[[VAL_5]] {uniq_name = "_QFsimple_real_reductionEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:                 %[[VAL_12:.*]]:2 = hlfir.declare %[[VAL_10]] {uniq_name = "_QFsimple_real_reductionEx"} : (!fir.ref<f32>) -> (!fir.ref<f32>, !fir.ref<f32>)
 ! CHECK:                 fir.store %[[VAL_11]] to %[[VAL_6]]#1 : !fir.ref<i32>
 ! CHECK:                 %[[VAL_13:.*]] = fir.load %[[VAL_12]]#0 : !fir.ref<f32>
@@ -170,13 +168,12 @@ subroutine simple_real_reduction
 ! CHECK:           %[[VAL_4:.*]] = arith.constant 0 : i32
 ! CHECK:           hlfir.assign %[[VAL_4]] to %[[VAL_3]]#0 : i32, !fir.ref<i32>
 ! CHECK:           omp.parallel {
-! CHECK:             %[[VAL_5:.*]] = fir.alloca i32 {bindc_name = "i", pinned, {{.*}}}
-! CHECK:             %[[VAL_6:.*]]:2 = hlfir.declare %[[VAL_5]] {uniq_name = "_QFsimple_int_reduction_switch_orderEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:             %[[VAL_7:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_8:.*]] = arith.constant 100 : i32
 ! CHECK:             %[[VAL_9:.*]] = arith.constant 1 : i32
-! CHECK:             omp.wsloop reduction(byref @add_reduction_byref_i32 %[[VAL_3]]#0 -> %[[VAL_10:.*]] : !fir.ref<i32>) {
+! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_5:.*]] : !fir.ref<i32>) reduction(byref @add_reduction_byref_i32 %[[VAL_3]]#0 -> %[[VAL_10:.*]] : !fir.ref<i32>) {
 ! CHECK-NEXT:          omp.loop_nest (%[[VAL_11:.*]]) : i32 = (%[[VAL_7]]) to (%[[VAL_8]]) inclusive step (%[[VAL_9]]) {
+! CHECK:                 %[[VAL_6:.*]]:2 = hlfir.declare %[[VAL_5]] {uniq_name = "_QFsimple_int_reduction_switch_orderEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:                 %[[VAL_12:.*]]:2 = hlfir.declare %[[VAL_10]] {uniq_name = "_QFsimple_int_reduction_switch_orderEx"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:                 fir.store %[[VAL_11]] to %[[VAL_6]]#1 : !fir.ref<i32>
 ! CHECK:                 %[[VAL_13:.*]] = fir.load %[[VAL_6]]#0 : !fir.ref<i32>
@@ -211,13 +208,12 @@ subroutine simple_int_reduction_switch_order
 ! CHECK:           %[[VAL_4:.*]] = arith.constant 0.000000e+00 : f32
 ! CHECK:           hlfir.assign %[[VAL_4]] to %[[VAL_3]]#0 : f32, !fir.ref<f32>
 ! CHECK:           omp.parallel {
-! CHECK:             %[[VAL_5:.*]] = fir.alloca i32 {bindc_name = "i", pinned, {{.*}}}
-! CHECK:             %[[VAL_6:.*]]:2 = hlfir.declare %[[VAL_5]] {uniq_name = "_QFsimple_real_reduction_switch_orderEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:             %[[VAL_7:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_8:.*]] = arith.constant 100 : i32
 ! CHECK:             %[[VAL_9:.*]] = arith.constant 1 : i32
-! CHECK:             omp.wsloop reduction(byref @add_reduction_byref_f32 %[[VAL_3]]#0 -> %[[VAL_10:.*]] : !fir.ref<f32>) {
+! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_5:.*]] : !fir.ref<i32>) reduction(byref @add_reduction_byref_f32 %[[VAL_3]]#0 -> %[[VAL_10:.*]] : !fir.ref<f32>) {
 ! CHECK-NEXT:          omp.loop_nest (%[[VAL_11:.*]]) : i32 = (%[[VAL_7]]) to (%[[VAL_8]]) inclusive step (%[[VAL_9]]) {
+! CHECK:                 %[[VAL_6:.*]]:2 = hlfir.declare %[[VAL_5]] {uniq_name = "_QFsimple_real_reduction_switch_orderEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:                 %[[VAL_12:.*]]:2 = hlfir.declare %[[VAL_10]] {uniq_name = "_QFsimple_real_reduction_switch_orderEx"} : (!fir.ref<f32>) -> (!fir.ref<f32>, !fir.ref<f32>)
 ! CHECK:                 fir.store %[[VAL_11]] to %[[VAL_6]]#1 : !fir.ref<i32>
 ! CHECK:                 %[[VAL_13:.*]] = fir.load %[[VAL_6]]#0 : !fir.ref<i32>
@@ -261,13 +257,12 @@ subroutine simple_real_reduction_switch_order
 ! CHECK:           %[[VAL_10:.*]] = arith.constant 0 : i32
 ! CHECK:           hlfir.assign %[[VAL_10]] to %[[VAL_7]]#0 : i32, !fir.ref<i32>
 ! CHECK:           omp.parallel {
-! CHECK:             %[[VAL_11:.*]] = fir.alloca i32 {bindc_name = "i", pinned, {{.*}}}
-! CHECK:             %[[VAL_12:.*]]:2 = hlfir.declare %[[VAL_11]] {uniq_name = "_QFmultiple_int_reductions_same_typeEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:             %[[VAL_13:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_14:.*]] = arith.constant 100 : i32
 ! CHECK:             %[[VAL_15:.*]] = arith.constant 1 : i32
-! CHECK:             omp.wsloop reduction(byref @add_reduction_byref_i32 %[[VAL_3]]#0 -> %[[VAL_16:.*]], byref @add_reduction_byref_i32 %[[VAL_5]]#0 -> %[[VAL_17:.*]], byref @add_reduction_byref_i32 %[[VAL_7]]#0 -> %[[VAL_18:.*]] : !fir.ref<i32>, !fir.ref<i32>, !fir.ref<i32>) {
+! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_11:.*]] : !fir.ref<i32>) reduction(byref @add_reduction_byref_i32 %[[VAL_3]]#0 -> %[[VAL_16:.*]], byref @add_reduction_byref_i32 %[[VAL_5]]#0 -> %[[VAL_17:.*]], byref @add_reduction_byref_i32 %[[VAL_7]]#0 -> %[[VAL_18:.*]] : !fir.ref<i32>, !fir.ref<i32>, !fir.ref<i32>) {
 ! CHECK-NEXT:          omp.loop_nest (%[[VAL_19:.*]]) : i32 = (%[[VAL_13]]) to (%[[VAL_14]]) inclusive step (%[[VAL_15]]) {
+! CHECK:                 %[[VAL_12:.*]]:2 = hlfir.declare %[[VAL_11]] {uniq_name = "_QFmultiple_int_reductions_same_typeEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:                 %[[VAL_20:.*]]:2 = hlfir.declare %[[VAL_16]] {uniq_name = "_QFmultiple_int_reductions_same_typeEx"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:                 %[[VAL_21:.*]]:2 = hlfir.declare %[[VAL_17]] {uniq_name = "_QFmultiple_int_reductions_same_typeEy"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:                 %[[VAL_22:.*]]:2 = hlfir.declare %[[VAL_18]] {uniq_name = "_QFmultiple_int_reductions_same_typeEz"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
@@ -324,13 +319,12 @@ subroutine multiple_int_reductions_same_type
 ! CHECK:           %[[VAL_10:.*]] = arith.constant 0.000000e+00 : f32
 ! CHECK:           hlfir.assign %[[VAL_10]] to %[[VAL_7]]#0 : f32, !fir.ref<f32>
 ! CHECK:           omp.parallel {
-! CHECK:             %[[VAL_11:.*]] = fir.alloca i32 {bindc_name = "i", pinned, {{.*}}}
-! CHECK:             %[[VAL_12:.*]]:2 = hlfir.declare %[[VAL_11]] {uniq_name = "_QFmultiple_real_reductions_same_typeEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:             %[[VAL_13:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_14:.*]] = arith.constant 100 : i32
 ! CHECK:             %[[VAL_15:.*]] = arith.constant 1 : i32
-! CHECK:             omp.wsloop reduction(byref @add_reduction_byref_f32 %[[VAL_3]]#0 -> %[[VAL_16:.*]], byref @add_reduction_byref_f32 %[[VAL_5]]#0 -> %[[VAL_17:.*]], byref @add_reduction_byref_f32 %[[VAL_7]]#0 -> %[[VAL_18:.*]] : !fir.ref<f32>, !fir.ref<f32>, !fir.ref<f32>) {
+! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_11:.*]] : !fir.ref<i32>) reduction(byref @add_reduction_byref_f32 %[[VAL_3]]#0 -> %[[VAL_16:.*]], byref @add_reduction_byref_f32 %[[VAL_5]]#0 -> %[[VAL_17:.*]], byref @add_reduction_byref_f32 %[[VAL_7]]#0 -> %[[VAL_18:.*]] : !fir.ref<f32>, !fir.ref<f32>, !fir.ref<f32>) {
 ! CHECK-NEXT:          omp.loop_nest (%[[VAL_19:.*]]) : i32 = (%[[VAL_13]]) to (%[[VAL_14]]) inclusive step (%[[VAL_15]]) {
+! CHECK:                 %[[VAL_12:.*]]:2 = hlfir.declare %[[VAL_11]] {uniq_name = "_QFmultiple_real_reductions_same_typeEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:                 %[[VAL_20:.*]]:2 = hlfir.declare %[[VAL_16]] {uniq_name = "_QFmultiple_real_reductions_same_typeEx"} : (!fir.ref<f32>) -> (!fir.ref<f32>, !fir.ref<f32>)
 ! CHECK:                 %[[VAL_21:.*]]:2 = hlfir.declare %[[VAL_17]] {uniq_name = "_QFmultiple_real_reductions_same_typeEy"} : (!fir.ref<f32>) -> (!fir.ref<f32>, !fir.ref<f32>)
 ! CHECK:                 %[[VAL_22:.*]]:2 = hlfir.declare %[[VAL_18]] {uniq_name = "_QFmultiple_real_reductions_same_typeEz"} : (!fir.ref<f32>) -> (!fir.ref<f32>, !fir.ref<f32>)
@@ -394,13 +388,12 @@ subroutine multiple_real_reductions_same_type
 ! CHECK:           %[[VAL_13:.*]] = arith.constant 0.000000e+00 : f64
 ! CHECK:           hlfir.assign %[[VAL_13]] to %[[VAL_3]]#0 : f64, !fir.ref<f64>
 ! CHECK:           omp.parallel {
-! CHECK:             %[[VAL_14:.*]] = fir.alloca i32 {bindc_name = "i", pinned, {{.*}}}
-! CHECK:             %[[VAL_15:.*]]:2 = hlfir.declare %[[VAL_14]] {uniq_name = "_QFmultiple_reductions_different_typeEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:             %[[VAL_16:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_17:.*]] = arith.constant 100 : i32
 ! CHECK:             %[[VAL_18:.*]] = arith.constant 1 : i32
-! CHECK:             omp.wsloop reduction(byref @add_reduction_byref_i32 %[[VAL_5]]#0 -> %[[VAL_19:.*]], byref @add_reduction_byref_i64 %[[VAL_7]]#0 -> %[[VAL_20:.*]], byref @add_reduction_byref_f32 %[[VAL_9]]#0 -> %[[VAL_21:.*]], byref @add_reduction_byref_f64 %[[VAL_3]]#0 -> %[[VAL_22:.*]] : !fir.ref<i32>, !fir.ref<i64>, !fir.ref<f32>, !fir.ref<f64>) {
+! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_14:.*]] : !fir.ref<i32>) reduction(byref @add_reduction_byref_i32 %[[VAL_5]]#0 -> %[[VAL_19:.*]], byref @add_reduction_byref_i64 %[[VAL_7]]#0 -> %[[VAL_20:.*]], byref @add_reduction_byref_f32 %[[VAL_9]]#0 -> %[[VAL_21:.*]], byref @add_reduction_byref_f64 %[[VAL_3]]#0 -> %[[VAL_22:.*]] : !fir.ref<i32>, !fir.ref<i64>, !fir.ref<f32>, !fir.ref<f64>) {
 ! CHECK-NEXT:          omp.loop_nest (%[[VAL_23:.*]]) : i32 = (%[[VAL_16]]) to (%[[VAL_17]]) inclusive step (%[[VAL_18]]) {
+! CHECK:                 %[[VAL_15:.*]]:2 = hlfir.declare %[[VAL_14]] {uniq_name = "_QFmultiple_reductions_different_typeEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:                 %[[VAL_24:.*]]:2 = hlfir.declare %[[VAL_19]] {uniq_name = "_QFmultiple_reductions_different_typeEx"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:                 %[[VAL_25:.*]]:2 = hlfir.declare %[[VAL_20]] {uniq_name = "_QFmultiple_reductions_different_typeEy"} : (!fir.ref<i64>) -> (!fir.ref<i64>, !fir.ref<i64>)
 ! CHECK:                 %[[VAL_26:.*]]:2 = hlfir.declare %[[VAL_21]] {uniq_name = "_QFmultiple_reductions_different_typeEz"} : (!fir.ref<f32>) -> (!fir.ref<f32>, !fir.ref<f32>)
diff --git a/flang/test/Lower/OpenMP/wsloop-reduction-add.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-add.f90
index c5278e0ef8815..a355e968b4146 100644
--- a/flang/test/Lower/OpenMP/wsloop-reduction-add.f90
+++ b/flang/test/Lower/OpenMP/wsloop-reduction-add.f90
@@ -53,13 +53,12 @@
 ! CHECK:           %[[VAL_4:.*]] = arith.constant 0 : i32
 ! CHECK:           hlfir.assign %[[VAL_4]] to %[[VAL_3]]#0 : i32, !fir.ref<i32>
 ! CHECK:           omp.parallel {
-! CHECK:             %[[VAL_5:.*]] = fir.alloca i32 {bindc_name = "i", pinned, {{.*}}}
-! CHECK:             %[[VAL_6:.*]]:2 = hlfir.declare %[[VAL_5]] {uniq_name = "_QFsimple_int_reductionEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:             %[[VAL_7:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_8:.*]] = arith.constant 100 : i32
 ! CHECK:             %[[VAL_9:.*]] = arith.constant 1 : i32
-! CHECK:             omp.wsloop reduction(@add_reduction_i32 %[[VAL_3]]#0 -> %[[VAL_10:.*]] : !fir.ref<i32>) {
+! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_5:.*]] : !fir.ref<i32>) reduction(@add_reduction_i32 %[[VAL_3]]#0 -> %[[VAL_10:.*]] : !fir.ref<i32>) {
 ! CHECK-NEXT:          omp.loop_nest (%[[VAL_11:.*]]) : i32 = (%[[VAL_7]]) to (%[[VAL_8]]) inclusive step (%[[VAL_9]]) {
+! CHECK:                 %[[VAL_6:.*]]:2 = hlfir.declare %[[VAL_5]] {uniq_name = "_QFsimple_int_reductionEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:                 %[[VAL_12:.*]]:2 = hlfir.declare %[[VAL_10]] {uniq_name = "_QFsimple_int_reductionEx"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:                 fir.store %[[VAL_11]] to %[[VAL_6]]#1 : !fir.ref<i32>
 ! CHECK:                 %[[VAL_13:.*]] = fir.load %[[VAL_12]]#0 : !fir.ref<i32>
@@ -95,13 +94,12 @@ subroutine simple_int_reduction
 ! CHECK:           %[[VAL_4:.*]] = arith.constant 0.000000e+00 : f32
 ! CHECK:           hlfir.assign %[[VAL_4]] to %[[VAL_3]]#0 : f32, !fir.ref<f32>
 ! CHECK:           omp.parallel {
-! CHECK:             %[[VAL_5:.*]] = fir.alloca i32 {bindc_name = "i", pinned, {{.*}}}
-! CHECK:             %[[VAL_6:.*]]:2 = hlfir.declare %[[VAL_5]] {uniq_name = "_QFsimple_real_reductionEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:             %[[VAL_7:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_8:.*]] = arith.constant 100 : i32
 ! CHECK:             %[[VAL_9:.*]] = arith.constant 1 : i32
-! CHECK:             omp.wsloop reduction(@add_reduction_f32 %[[VAL_3]]#0 -> %[[VAL_10:.*]] : !fir.ref<f32>) {
+! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_5:.*]] : !fir.ref<i32>) reduction(@add_reduction_f32 %[[VAL_3]]#0 -> %[[VAL_10:.*]] : !fir.ref<f32>) {
 ! CHECK-NEXT:          omp.loop_nest (%[[VAL_11:.*]]) : i32 = (%[[VAL_7]]) to (%[[VAL_8]]) inclusive step (%[[VAL_9]]) {
+! CHECK:                 %[[VAL_6:.*]]:2 = hlfir.declare %[[VAL_5]] {uniq_name = "_QFsimple_real_reductionEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:                 %[[VAL_12:.*]]:2 = hlfir.declare %[[VAL_10]] {uniq_name = "_QFsimple_real_reductionEx"} : (!fir.ref<f32>) -> (!fir.ref<f32>, !fir.ref<f32>)
 ! CHECK:                 fir.store %[[VAL_11]] to %[[VAL_6]]#1 : !fir.ref<i32>
 ! CHECK:                 %[[VAL_13:.*]] = fir.load %[[VAL_12]]#0 : !fir.ref<f32>
@@ -138,13 +136,12 @@ subroutine simple_real_reduction
 ! CHECK:           %[[VAL_4:.*]] = arith.constant 0 : i32
 ! CHECK:           hlfir.assign %[[VAL_4]] to %[[VAL_3]]#0 : i32, !fir.ref<i32>
 ! CHECK:           omp.parallel {
-! CHECK:             %[[VAL_5:.*]] = fir.alloca i32 {bindc_name = "i", pinned, {{.*}}}
-! CHECK:             %[[VAL_6:.*]]:2 = hlfir.declare %[[VAL_5]] {uniq_name = "_QFsimple_int_reduction_switch_orderEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:             %[[VAL_7:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_8:.*]] = arith.constant 100 : i32
 ! CHECK:             %[[VAL_9:.*]] = arith.constant 1 : i32
-! CHECK:             omp.wsloop reduction(@add_reduction_i32 %[[VAL_3]]#0 -> %[[VAL_10:.*]] : !fir.ref<i32>) {
+! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_5:.*]] : !fir.ref<i32>) reduction(@add_reduction_i32 %[[VAL_3]]#0 -> %[[VAL_10:.*]] : !fir.ref<i32>) {
 ! CHECK-NEXT:          omp.loop_nest (%[[VAL_11:.*]]) : i32 = (%[[VAL_7]]) to (%[[VAL_8]]) inclusive step (%[[VAL_9]]) {
+! CHECK:                 %[[VAL_6:.*]]:2 = hlfir.declare %[[VAL_5]] {uniq_name = "_QFsimple_int_reduction_switch_orderEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:                 %[[VAL_12:.*]]:2 = hlfir.declare %[[VAL_10]] {uniq_name = "_QFsimple_int_reduction_switch_orderEx"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:                 fir.store %[[VAL_11]] to %[[VAL_6]]#1 : !fir.ref<i32>
 ! CHECK:                 %[[VAL_13:.*]] = fir.load %[[VAL_6]]#0 : !fir.ref<i32>
@@ -179,13 +176,12 @@ subroutine simple_int_reduction_switch_order
 ! CHECK:           %[[VAL_4:.*]] = arith.constant 0.000000e+00 : f32
 ! CHECK:           hlfir.assign %[[VAL_4]] to %[[VAL_3]]#0 : f32, !fir.ref<f32>
 ! CHECK:           omp.parallel {
-! CHECK:             %[[VAL_5:.*]] = fir.alloca i32 {bindc_name = "i", pinned, {{.*}}}
-! CHECK:             %[[VAL_6:.*]]:2 = hlfir.declare %[[VAL_5]] {uniq_name = "_QFsimple_real_reduction_switch_orderEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:             %[[VAL_7:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_8:.*]] = arith.constant 100 : i32
 ! CHECK:             %[[VAL_9:.*]] = arith.constant 1 : i32
-! CHECK:             omp.wsloop reduction(@add_reduction_f32 %[[VAL_3]]#0 -> %[[VAL_10:.*]] : !fir.ref<f32>) {
+! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_5:.*]] : !fir.ref<i32>) reduction(@add_reduction_f32 %[[VAL_3]]#0 -> %[[VAL_10:.*]] : !fir.ref<f32>) {
 ! CHECK-NEXT:          omp.loop_nest (%[[VAL_11:.*]]) : i32 = (%[[VAL_7]]) to (%[[VAL_8]]) inclusive step (%[[VAL_9]]) {
+! CHECK:                 %[[VAL_6:.*]]:2 = hlfir.declare %[[VAL_5]] {uniq_name = "_QFsimple_real_reduction_switch_orderEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:                 %[[VAL_12:.*]]:2 = hlfir.declare %[[VAL_10]] {uniq_name = "_QFsimple_real_reduction_switch_orderEx"} : (!fir.ref<f32>) -> (!fir.ref<f32>, !fir.ref<f32>)
 ! CHECK:                 fir.store %[[VAL_11]] to %[[VAL_6]]#1 : !fir.ref<i32>
 ! CHECK:                 %[[VAL_13:.*]] = fir.load %[[VAL_6]]#0 : !fir.ref<i32>
@@ -229,13 +225,12 @@ subroutine simple_real_reduction_switch_order
 ! CHECK:           %[[VAL_10:.*]] = arith.constant 0 : i32
 ! CHECK:           hlfir.assign %[[VAL_10]] to %[[VAL_7]]#0 : i32, !fir.ref<i32>
 ! CHECK:           omp.parallel {
-! CHECK:             %[[VAL_11:.*]] = fir.alloca i32 {bindc_name = "i", pinned, {{.*}}}
-! CHECK:             %[[VAL_12:.*]]:2 = hlfir.declare %[[VAL_11]] {uniq_name = "_QFmultiple_int_reductions_same_typeEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:             %[[VAL_13:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_14:.*]] = arith.constant 100 : i32
 ! CHECK:             %[[VAL_15:.*]] = arith.constant 1 : i32
-! CHECK:             omp.wsloop reduction(@add_reduction_i32 %[[VAL_3]]#0 -> %[[VAL_16:.*]], @add_reduction_i32 %[[VAL_5]]#0 -> %[[VAL_17:.*]], @add_reduction_i32 %[[VAL_7]]#0 -> %[[VAL_18:.*]] : !fir.ref<i32>, !fir.ref<i32>, !fir.ref<i32>) {
+! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_11:.*]] : !fir.ref<i32>) reduction(@add_reduction_i32 %[[VAL_3]]#0 -> %[[VAL_16:.*]], @add_reduction_i32 %[[VAL_5]]#0 -> %[[VAL_17:.*]], @add_reduction_i32 %[[VAL_7]]#0 -> %[[VAL_18:.*]] : !fir.ref<i32>, !fir.ref<i32>, !fir.ref<i32>) {
 ! CHECK-NEXT:          omp.loop_nest (%[[VAL_19:.*]]) : i32 = (%[[VAL_13]]) to (%[[VAL_14]]) inclusive step (%[[VAL_15]]) {
+! CHECK:                 %[[VAL_12:.*]]:2 = hlfir.declare %[[VAL_11]] {uniq_name = "_QFmultiple_int_reductions_same_typeEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:                 %[[VAL_20:.*]]:2 = hlfir.declare %[[VAL_16]] {uniq_name = "_QFmultiple_int_reductions_same_typeEx"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:                 %[[VAL_21:.*]]:2 = hlfir.declare %[[VAL_17]] {uniq_name = "_QFmultiple_int_reductions_same_typeEy"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:                 %[[VAL_22:.*]]:2 = hlfir.declare %[[VAL_18]] {uniq_name = "_QFmultiple_int_reductions_same_typeEz"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
@@ -292,13 +287,12 @@ subroutine multiple_int_reductions_same_type
 ! CHECK:           %[[VAL_10:.*]] = arith.constant 0.000000e+00 : f32
 ! CHECK:           hlfir.assign %[[VAL_10]] to %[[VAL_7]]#0 : f32, !fir.ref<f32>
 ! CHECK:           omp.parallel {
-! CHECK:             %[[VAL_11:.*]] = fir.alloca i32 {bindc_name = "i", pinned, {{.*}}}
-! CHECK:             %[[VAL_12:.*]]:2 = hlfir.declare %[[VAL_11]] {uniq_name = "_QFmultiple_real_reductions_same_typeEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:             %[[VAL_13:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_14:.*]] = arith.constant 100 : i32
 ! CHECK:             %[[VAL_15:.*]] = arith.constant 1 : i32
-! CHECK:             omp.wsloop reduction(@add_reduction_f32 %[[VAL_3]]#0 -> %[[VAL_16:.*]], @add_reduction_f32 %[[VAL_5]]#0 -> %[[VAL_17:.*]], @add_reduction_f32 %[[VAL_7]]#0 -> %[[VAL_18:.*]] : !fir.ref<f32>, !fir.ref<f32>, !fir.ref<f32>) {
+! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_11:.*]] : !fir.ref<i32>) reduction(@add_reduction_f32 %[[VAL_3]]#0 -> %[[VAL_16:.*]], @add_reduction_f32 %[[VAL_5]]#0 -> %[[VAL_17:.*]], @add_reduction_f32 %[[VAL_7]]#0 -> %[[VAL_18:.*]] : !fir.ref<f32>, !fir.ref<f32>, !fir.ref<f32>) {
 ! CHECK-NEXT:          omp.loop_nest (%[[VAL_19:.*]]) : i32 = (%[[VAL_13]]) to (%[[VAL_14]]) inclusive step (%[[VAL_15]]) {
+! CHECK:                 %[[VAL_12:.*]]:2 = hlfir.declare %[[VAL_11]] {uniq_name = "_QFmultiple_real_reductions_same_typeEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:                 %[[VAL_20:.*]]:2 = hlfir.declare %[[VAL_16]] {uniq_name = "_QFmultiple_real_reductions_same_typeEx"} : (!fir.ref<f32>) -> (!fir.ref<f32>, !fir.ref<f32>)
 ! CHECK:                 %[[VAL_21:.*]]:2 = hlfir.declare %[[VAL_17]] {uniq_name = "_QFmultiple_real_reductions_same_typeEy"} : (!fir.ref<f32>) -> (!fir.ref<f32>, !fir.ref<f32>)
 ! CHECK:                 %[[VAL_22:.*]]:2 = hlfir.declare %[[VAL_18]] {uniq_name = "_QFmultiple_real_reductions_same_typeEz"} : (!fir.ref<f32>) -> (!fir.ref<f32>, !fir.ref<f32>)
@@ -362,13 +356,12 @@ subroutine multiple_real_reductions_same_type
 ! CHECK:           %[[VAL_13:.*]] = arith.constant 0.000000e+00 : f64
 ! CHECK:           hlfir.assign %[[VAL_13]] to %[[VAL_3]]#0 : f64, !fir.ref<f64>
 ! CHECK:           omp.parallel {
-! CHECK:             %[[VAL_14:.*]] = fir.alloca i32 {bindc_name = "i", pinned, {{.*}}}
-! CHECK:             %[[VAL_15:.*]]:2 = hlfir.declare %[[VAL_14]] {uniq_name = "_QFmultiple_reductions_different_typeEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:             %[[VAL_16:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_17:.*]] = arith.constant 100 : i32
 ! CHECK:             %[[VAL_18:.*]] = arith.constant 1 : i32
-! CHECK:             omp.wsloop reduction(@add_reduction_i32 %[[VAL_5]]#0 -> %[[VAL_19:.*]], @add_reduction_i64 %[[VAL_7]]#0 -> %[[VAL_20:.*]], @add_reduction_f32 %[[VAL_9]]#0 -> %[[VAL_21:.*]], @add_reduction_f64 %[[VAL_3]]#0 -> %[[VAL_22:.*]] : !fir.ref<i32>, !fir.ref<i64>, !fir.ref<f32>, !fir.ref<f64>) {
+! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_14:.*]] : !fir.ref<i32>) reduction(@add_reduction_i32 %[[VAL_5]]#0 -> %[[VAL_19:.*]], @add_reduction_i64 %[[VAL_7]]#0 -> %[[VAL_20:.*]], @add_reduction_f32 %[[VAL_9]]#0 -> %[[VAL_21:.*]], @add_reduction_f64 %[[VAL_3]]#0 -> %[[VAL_22:.*]] : !fir.ref<i32>, !fir.ref<i64>, !fir.ref<f32>, !fir.ref<f64>) {
 ! CHECK-NEXT:          omp.loop_nest (%[[VAL_23:.*]]) : i32 = (%[[VAL_16]]) to (%[[VAL_17]]) inclusive step (%[[VAL_18]]) {
+! CHECK:                 %[[VAL_15:.*]]:2 = hlfir.declare %[[VAL_14]] {uniq_name = "_QFmultiple_reductions_different_typeEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:                 %[[VAL_24:.*]]:2 = hlfir.declare %[[VAL_19]] {uniq_name = "_QFmultiple_reductions_different_typeEx"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:                 %[[VAL_25:.*]]:2 = hlfir.declare %[[VAL_20]] {uniq_name = "_QFmultiple_reductions_different_typeEy"} : (!fir.ref<i64>) -> (!fir.ref<i64>, !fir.ref<i64>)
 ! CHECK:                 %[[VAL_26:.*]]:2 = hlfir.declare %[[VAL_21]] {uniq_name = "_QFmultiple_reductions_different_typeEz"} : (!fir.ref<f32>) -> (!fir.ref<f32>, !fir.ref<f32>)
diff --git a/flang/test/Lower/OpenMP/wsloop-reduction-allocatable-array-minmax.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-allocatable-array-minmax.f90
index ce45d09d77a22..d9c75af1fe62c 100644
--- a/flang/test/Lower/OpenMP/wsloop-reduction-allocatable-array-minmax.f90
+++ b/flang/test/Lower/OpenMP/wsloop-reduction-allocatable-array-minmax.f90
@@ -44,7 +44,9 @@ program reduce15
 ! CHECK:           %[[VAL_6:.*]] = arith.constant 0 : i64
 ! CHECK:           %[[VAL_7:.*]] = arith.cmpi eq, %[[VAL_5]], %[[VAL_6]] : i64
 ! CHECK:           fir.if %[[VAL_7]] {
-! CHECK:             %[[VAL_8:.*]] = fir.embox %[[VAL_4]] : (!fir.heap<!fir.array<?xi32>>) -> !fir.box<!fir.heap<!fir.array<?xi32>>>
+! CHECK:             %[[C0:.*]] = arith.constant 0 : index
+! CHECK:             %[[SHAPE:.*]] = fir.shape %[[C0]]
+! CHECK:             %[[VAL_8:.*]] = fir.embox %[[VAL_4]](%[[SHAPE]]) : (!fir.heap<!fir.array<?xi32>>, !fir.shape<1>) -> !fir.box<!fir.heap<!fir.array<?xi32>>>
 ! CHECK:             fir.store %[[VAL_8]] to %[[ALLOC]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>
 ! CHECK:           } else {
 ! CHECK:             %[[VAL_9:.*]] = arith.constant 0 : index
@@ -103,7 +105,9 @@ program reduce15
 ! CHECK:           %[[VAL_6:.*]] = arith.constant 0 : i64
 ! CHECK:           %[[VAL_7:.*]] = arith.cmpi eq, %[[VAL_5]], %[[VAL_6]] : i64
 ! CHECK:           fir.if %[[VAL_7]] {
-! CHECK:             %[[VAL_8:.*]] = fir.embox %[[VAL_4]] : (!fir.heap<!fir.array<?xi32>>) -> !fir.box<!fir.heap<!fir.array<?xi32>>>
+! CHECK:             %[[C0:.*]] = arith.constant 0 : index
+! CHECK:             %[[SHAPE:.*]] = fir.shape %[[C0]]
+! CHECK:             %[[VAL_8:.*]] = fir.embox %[[VAL_4]](%[[SHAPE]]) : (!fir.heap<!fir.array<?xi32>>, !fir.shape<1>) -> !fir.box<!fir.heap<!fir.array<?xi32>>>
 ! CHECK:             fir.store %[[VAL_8]] to %[[ALLOC]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>
 ! CHECK:           } else {
 ! CHECK:             %[[VAL_9:.*]] = arith.constant 0 : index
@@ -214,13 +218,12 @@ program reduce15
 ! CHECK:           }
 ! CHECK:           fir.store %[[VAL_54:.*]]#1 to %[[VAL_3]]#1 : !fir.ref<i32>
 ! CHECK:           omp.parallel {
-! CHECK:             %[[VAL_55:.*]] = fir.alloca i32 {bindc_name = "i", pinned, uniq_name = "_QFEi"}
-! CHECK:             %[[VAL_56:.*]]:2 = hlfir.declare %[[VAL_55]] {uniq_name = "_QFEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:             %[[VAL_57:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_58:.*]] = arith.constant 10 : i32
 ! CHECK:             %[[VAL_59:.*]] = arith.constant 1 : i32
-! CHECK:             omp.wsloop reduction(byref @max_byref_box_heap_Uxi32 %[[VAL_5]]#0 -> %[[VAL_60:.*]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>) {
+! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_55:.*]] : !fir.ref<i32>) reduction(byref @max_byref_box_heap_Uxi32 %[[VAL_5]]#0 -> %[[VAL_60:.*]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>) {
 ! CHECK:               omp.loop_nest (%[[VAL_61:.*]]) : i32 = (%[[VAL_57]]) to (%[[VAL_58]]) inclusive step (%[[VAL_59]]) {
+! CHECK:                 %[[VAL_56:.*]]:2 = hlfir.declare %[[VAL_55]] {uniq_name = "_QFEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:                 %[[VAL_62:.*]]:2 = hlfir.declare %[[VAL_60]] {fortran_attrs = {{.*}}<allocatable>, uniq_name = "_QFEmaxes"} : (!fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>) -> (!fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>, !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>)
 ! CHECK:                 fir.store %[[VAL_61]] to %[[VAL_56]]#1 : !fir.ref<i32>
 ! CHECK:                 %[[VAL_63:.*]] = fir.load %[[VAL_1]]#0 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>
@@ -256,13 +259,12 @@ program reduce15
 ! CHECK:             omp.terminator
 ! CHECK:           }
 ! CHECK:           omp.parallel {
-! CHECK:             %[[VAL_87:.*]] = fir.alloca i32 {bindc_name = "i", pinned, uniq_name = "_QFEi"}
-! CHECK:             %[[VAL_88:.*]]:2 = hlfir.declare %[[VAL_87]] {uniq_name = "_QFEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:             %[[VAL_89:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_90:.*]] = arith.constant 10 : i32
 ! CHECK:             %[[VAL_91:.*]] = arith.constant 1 : i32
-! CHECK:             omp.wsloop reduction(byref @min_byref_box_heap_Uxi32 %[[VAL_7]]#0 -> %[[VAL_92:.*]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>) {
+! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_87:.*]] : !fir.ref<i32>) reduction(byref @min_byref_box_heap_Uxi32 %[[VAL_7]]#0 -> %[[VAL_92:.*]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>) {
 ! CHECK:               omp.loop_nest (%[[VAL_93:.*]]) : i32 = (%[[VAL_89]]) to (%[[VAL_90]]) inclusive step (%[[VAL_91]]) {
+! CHECK:                 %[[VAL_88:.*]]:2 = hlfir.declare %[[VAL_87]] {uniq_name = "_QFEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:                 %[[VAL_94:.*]]:2 = hlfir.declare %[[VAL_92]] {fortran_attrs = {{.*}}<allocatable>, uniq_name = "_QFEmins"} : (!fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>) -> (!fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>, !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>)
 ! CHECK:                 fir.store %[[VAL_93]] to %[[VAL_88]]#1 : !fir.ref<i32>
 ! CHECK:                 %[[VAL_95:.*]] = fir.load %[[VAL_1]]#0 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>
diff --git a/flang/test/Lower/OpenMP/wsloop-reduction-allocatable.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-allocatable.f90
index ba7aea0d96c5b..7e6d7fddff5a1 100644
--- a/flang/test/Lower/OpenMP/wsloop-reduction-allocatable.f90
+++ b/flang/test/Lower/OpenMP/wsloop-reduction-allocatable.f90
@@ -77,13 +77,12 @@ program reduce
 ! CHECK:           %[[VAL_8:.*]] = arith.constant 0 : i32
 ! CHECK:           hlfir.assign %[[VAL_8]] to %[[VAL_5]]#0 realloc : i32, !fir.ref<!fir.box<!fir.heap<i32>>>
 ! CHECK:           omp.parallel {
-! CHECK:             %[[VAL_9:.*]] = fir.alloca i32 {bindc_name = "i", pinned, {{.*}}}
-! CHECK:             %[[VAL_10:.*]]:2 = hlfir.declare %[[VAL_9]] {uniq_name = "_QFEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:             %[[VAL_11:.*]] = arith.constant 0 : i32
 ! CHECK:             %[[VAL_12:.*]] = arith.constant 10 : i32
 ! CHECK:             %[[VAL_13:.*]] = arith.constant 1 : i32
-! CHECK:             omp.wsloop reduction(byref @add_reduction_byref_box_heap_i32 %[[VAL_5]]#0 -> %[[VAL_14:.*]] : !fir.ref<!fir.box<!fir.heap<i32>>>) {
+! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_9:.*]] : !fir.ref<i32>) reduction(byref @add_reduction_byref_box_heap_i32 %[[VAL_5]]#0 -> %[[VAL_14:.*]] : !fir.ref<!fir.box<!fir.heap<i32>>>) {
 ! CHECK-NEXT:          omp.loop_nest (%[[VAL_15:.*]]) : i32 = (%[[VAL_11]]) to (%[[VAL_12]]) inclusive step (%[[VAL_13]]) {
+! CHECK:                 %[[VAL_10:.*]]:2 = hlfir.declare %[[VAL_9]] {uniq_name = "_QFEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:                 %[[VAL_16:.*]]:2 = hlfir.declare %[[VAL_14]] {fortran_attrs = {{.*}}<allocatable>, uniq_name = "_QFEr"} : (!fir.ref<!fir.box<!fir.heap<i32>>>) -> (!fir.ref<!fir.box<!fir.heap<i32>>>, !fir.ref<!fir.box<!fir.heap<i32>>>)
 ! CHECK:                 fir.store %[[VAL_15]] to %[[VAL_10]]#1 : !fir.ref<i32>
 ! CHECK:                 %[[VAL_17:.*]] = fir.load %[[VAL_10]]#0 : !fir.ref<i32>
diff --git a/flang/test/Lower/OpenMP/wsloop-reduction-array-assumed-shape.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-array-assumed-shape.f90
index 9785f77c0e091..0e2fc3a24ee1b 100644
--- a/flang/test/Lower/OpenMP/wsloop-reduction-array-assumed-shape.f90
+++ b/flang/test/Lower/OpenMP/wsloop-reduction-array-assumed-shape.f90
@@ -81,13 +81,12 @@ subroutine reduce(r)
 ! CHECK:           omp.parallel {
 ! CHECK:             %[[VAL_4:.*]] = fir.alloca !fir.box<!fir.array<?xf64>>
 ! CHECK:             fir.store %[[VAL_3]]#1 to %[[VAL_4]] : !fir.ref<!fir.box<!fir.array<?xf64>>>
-! CHECK:             %[[VAL_5:.*]] = fir.alloca i32 {bindc_name = "i", pinned, {{.*}}}
-! CHECK:             %[[VAL_6:.*]]:2 = hlfir.declare %[[VAL_5]] {uniq_name = "_QFFreduceEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:             %[[VAL_7:.*]] = arith.constant 0 : i32
 ! CHECK:             %[[VAL_8:.*]] = arith.constant 10 : i32
 ! CHECK:             %[[VAL_9:.*]] = arith.constant 1 : i32
-! CHECK:             omp.wsloop reduction(byref @add_reduction_byref_box_Uxf64 %[[VAL_4]] -> %[[VAL_10:.*]] : !fir.ref<!fir.box<!fir.array<?xf64>>>) {
+! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_5:.*]] : !fir.ref<i32>) reduction(byref @add_reduction_byref_box_Uxf64 %[[VAL_4]] -> %[[VAL_10:.*]] : !fir.ref<!fir.box<!fir.array<?xf64>>>) {
 ! CHECK-NEXT:          omp.loop_nest (%[[VAL_11:.*]]) : i32 = (%[[VAL_7]]) to (%[[VAL_8]]) inclusive step (%[[VAL_9]]) {
+! CHECK:                 %[[VAL_6:.*]]:2 = hlfir.declare %[[VAL_5]] {uniq_name = "_QFFreduceEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:                 %[[VAL_12:.*]]:2 = hlfir.declare %[[VAL_10]] {fortran_attrs = {{.*}}, uniq_name = "_QFFreduceEr"} : (!fir.ref<!fir.box<!fir.array<?xf64>>>) -> (!fir.ref<!fir.box<!fir.array<?xf64>>>, !fir.ref<!fir.box<!fir.array<?xf64>>>)
 ! CHECK:                 fir.store %[[VAL_11]] to %[[VAL_6]]#1 : !fir.ref<i32>
 ! CHECK:                 %[[VAL_13:.*]] = fir.load %[[VAL_6]]#0 : !fir.ref<i32>
diff --git a/flang/test/Lower/OpenMP/wsloop-reduction-array.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-array.f90
index ea5df5a836972..065923ae3cf31 100644
--- a/flang/test/Lower/OpenMP/wsloop-reduction-array.f90
+++ b/flang/test/Lower/OpenMP/wsloop-reduction-array.f90
@@ -34,7 +34,7 @@ program reduce
 ! CHECK:           fir.store %[[VAL_7]] to %[[ALLOC]] : !fir.ref<!fir.box<!fir.array<2xi32>>>
 ! CHECK:           omp.yield(%[[ALLOC]] : !fir.ref<!fir.box<!fir.array<2xi32>>>)
 
-! CHECK-LABEL   } combiner {
+! CHECK-LABEL:   } combiner {
 ! CHECK:         ^bb0(%[[VAL_0:.*]]: !fir.ref<!fir.box<!fir.array<2xi32>>>, %[[VAL_1:.*]]: !fir.ref<!fir.box<!fir.array<2xi32>>>):
 ! CHECK:           %[[VAL_2:.*]] = fir.load %[[VAL_0]] : !fir.ref<!fir.box<!fir.array<2xi32>>>
 ! CHECK:           %[[VAL_3:.*]] = fir.load %[[VAL_1]] : !fir.ref<!fir.box<!fir.array<2xi32>>>
@@ -65,7 +65,7 @@ program reduce
 ! CHECK:           omp.yield
 ! CHECK:         }
 
-! CHECK-LABEL   func.func @_QQmain() attributes {fir.bindc_name = "reduce"} {
+! CHECK-LABEL:   func.func @_QQmain() attributes {fir.bindc_name = "reduce"} {
 ! CHECK:           %[[VAL_0:.*]] = fir.address_of(@_QFEi) : !fir.ref<i32>
 ! CHECK:           %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0]] {uniq_name = "_QFEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:           %[[VAL_2:.*]] = fir.address_of(@_QFEr) : !fir.ref<!fir.array<2xi32>>
@@ -76,13 +76,12 @@ program reduce
 ! CHECK:             %[[VAL_6:.*]] = fir.embox %[[VAL_5]]#0(%[[VAL_4]]) : (!fir.ref<!fir.array<2xi32>>, !fir.shape<1>) -> !fir.box<!fir.array<2xi32>>
 ! CHECK:             %[[VAL_7:.*]] = fir.alloca !fir.box<!fir.array<2xi32>>
 ! CHECK:             fir.store %[[VAL_6]] to %[[VAL_7]] : !fir.ref<!fir.box<!fir.array<2xi32>>>
-! CHECK:             %[[VAL_8:.*]] = fir.alloca i32 {bindc_name = "i", pinned, {{.*}}}
-! CHECK:             %[[VAL_9:.*]]:2 = hlfir.declare %[[VAL_8]] {uniq_name = "_QFEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:             %[[VAL_10:.*]] = arith.constant 0 : i32
 ! CHECK:             %[[VAL_11:.*]] = arith.constant 10 : i32
 ! CHECK:             %[[VAL_12:.*]] = arith.constant 1 : i32
-! CHECK:             omp.wsloop reduction(byref @add_reduction_byref_box_2xi32 %[[VAL_7]] -> %[[VAL_13:.*]] : !fir.ref<!fir.box<!fir.array<2xi32>>>) {
+! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_8:.*]] : !fir.ref<i32>) reduction(byref @add_reduction_byref_box_2xi32 %[[VAL_7]] -> %[[VAL_13:.*]] : !fir.ref<!fir.box<!fir.array<2xi32>>>) {
 ! CHECK-NEXT:          omp.loop_nest (%[[VAL_14:.*]]) : i32 = (%[[VAL_10]]) to (%[[VAL_11]]) inclusive step (%[[VAL_12]]) {
+! CHECK:                 %[[VAL_9:.*]]:2 = hlfir.declare %[[VAL_8]] {uniq_name = "_QFEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:                 %[[VAL_15:.*]]:2 = hlfir.declare %[[VAL_13]] {uniq_name = "_QFEr"} : (!fir.ref<!fir.box<!fir.array<2xi32>>>) -> (!fir.ref<!fir.box<!fir.array<2xi32>>>, !fir.ref<!fir.box<!fir.array<2xi32>>>)
 ! CHECK:                 fir.store %[[VAL_14]] to %[[VAL_9]]#1 : !fir.ref<i32>
 ! CHECK:                 %[[VAL_16:.*]] = fir.load %[[VAL_9]]#0 : !fir.ref<i32>
diff --git a/flang/test/Lower/OpenMP/wsloop-reduction-array2.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-array2.f90
index 9815cfa9c3150..a25bedb359f4e 100644
--- a/flang/test/Lower/OpenMP/wsloop-reduction-array2.f90
+++ b/flang/test/Lower/OpenMP/wsloop-reduction-array2.f90
@@ -76,13 +76,12 @@ program reduce
 ! CHECK:             %[[VAL_6:.*]] = fir.embox %[[VAL_5]]#0(%[[VAL_4]]) : (!fir.ref<!fir.array<2xi32>>, !fir.shape<1>) -> !fir.box<!fir.array<2xi32>>
 ! CHECK:             %[[VAL_7:.*]] = fir.alloca !fir.box<!fir.array<2xi32>>
 ! CHECK:             fir.store %[[VAL_6]] to %[[VAL_7]] : !fir.ref<!fir.box<!fir.array<2xi32>>>
-! CHECK:             %[[VAL_8:.*]] = fir.alloca i32 {bindc_name = "i", pinned, {{.*}}}
-! CHECK:             %[[VAL_9:.*]]:2 = hlfir.declare %[[VAL_8]] {uniq_name = "_QFEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:             %[[VAL_10:.*]] = arith.constant 0 : i32
 ! CHECK:             %[[VAL_11:.*]] = arith.constant 10 : i32
 ! CHECK:             %[[VAL_12:.*]] = arith.constant 1 : i32
-! CHECK:             omp.wsloop reduction(byref @add_reduction_byref_box_2xi32 %[[VAL_7]] -> %[[VAL_13:.*]] : !fir.ref<!fir.box<!fir.array<2xi32>>>) {
+! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_8:.*]] : !fir.ref<i32>) reduction(byref @add_reduction_byref_box_2xi32 %[[VAL_7]] -> %[[VAL_13:.*]] : !fir.ref<!fir.box<!fir.array<2xi32>>>) {
 ! CHECK-NEXT:          omp.loop_nest (%[[VAL_14:.*]]) : i32 = (%[[VAL_10]]) to (%[[VAL_11]]) inclusive step (%[[VAL_12]]) {
+! CHECK:                 %[[VAL_9:.*]]:2 = hlfir.declare %[[VAL_8]] {uniq_name = "_QFEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:                 %[[VAL_15:.*]]:2 = hlfir.declare %[[VAL_13]] {uniq_name = "_QFEr"} : (!fir.ref<!fir.box<!fir.array<2xi32>>>) -> (!fir.ref<!fir.box<!fir.array<2xi32>>>, !fir.ref<!fir.box<!fir.array<2xi32>>>)
 ! CHECK:                 fir.store %[[VAL_14]] to %[[VAL_9]]#1 : !fir.ref<i32>
 ! CHECK:                 %[[VAL_16:.*]] = fir.load %[[VAL_15]]#0 : !fir.ref<!fir.box<!fir.array<2xi32>>>
diff --git a/flang/test/Lower/OpenMP/wsloop-reduction-iand-byref.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-iand-byref.f90
index 829229807698a..18dcc3d722886 100644
--- a/flang/test/Lower/OpenMP/wsloop-reduction-iand-byref.f90
+++ b/flang/test/Lower/OpenMP/wsloop-reduction-iand-byref.f90
@@ -32,13 +32,12 @@
 ! CHECK:           %[[VAL_6:.*]] = arith.constant 0 : i32
 ! CHECK:           hlfir.assign %[[VAL_6]] to %[[VAL_4]]#0 : i32, !fir.ref<i32>
 ! CHECK:           omp.parallel {
-! CHECK:             %[[VAL_7:.*]] = fir.alloca i32 {bindc_name = "i", pinned, {{.*}}}
-! CHECK:             %[[VAL_8:.*]]:2 = hlfir.declare %[[VAL_7]] {uniq_name = "_QFreduction_iandEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:             %[[VAL_9:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_10:.*]] = arith.constant 100 : i32
 ! CHECK:             %[[VAL_11:.*]] = arith.constant 1 : i32
-! CHECK:             omp.wsloop reduction(byref @iand_byref_i32 %[[VAL_4]]#0 -> %[[VAL_12:.*]] : !fir.ref<i32>) {
+! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_7:.*]] : !fir.ref<i32>) reduction(byref @iand_byref_i32 %[[VAL_4]]#0 -> %[[VAL_12:.*]] : !fir.ref<i32>) {
 ! CHECK-NEXT:          omp.loop_nest (%[[VAL_13:.*]]) : i32 = (%[[VAL_9]]) to (%[[VAL_10]]) inclusive step (%[[VAL_11]]) {
+! CHECK:                 %[[VAL_8:.*]]:2 = hlfir.declare %[[VAL_7]] {uniq_name = "_QFreduction_iandEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:                 %[[VAL_14:.*]]:2 = hlfir.declare %[[VAL_12]] {uniq_name = "_QFreduction_iandEx"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:                 fir.store %[[VAL_13]] to %[[VAL_8]]#1 : !fir.ref<i32>
 ! CHECK:                 %[[VAL_15:.*]] = fir.load %[[VAL_8]]#0 : !fir.ref<i32>
diff --git a/flang/test/Lower/OpenMP/wsloop-reduction-iand.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-iand.f90
index 6c060f2e5292a..eaf07f93c7474 100644
--- a/flang/test/Lower/OpenMP/wsloop-reduction-iand.f90
+++ b/flang/test/Lower/OpenMP/wsloop-reduction-iand.f90
@@ -24,13 +24,12 @@
 ! CHECK:           %[[VAL_6:.*]] = arith.constant 0 : i32
 ! CHECK:           hlfir.assign %[[VAL_6]] to %[[VAL_4]]#0 : i32, !fir.ref<i32>
 ! CHECK:           omp.parallel {
-! CHECK:             %[[VAL_7:.*]] = fir.alloca i32 {bindc_name = "i", pinned, {{.*}}}
-! CHECK:             %[[VAL_8:.*]]:2 = hlfir.declare %[[VAL_7]] {uniq_name = "_QFreduction_iandEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:             %[[VAL_9:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_10:.*]] = arith.constant 100 : i32
 ! CHECK:             %[[VAL_11:.*]] = arith.constant 1 : i32
-! CHECK:             omp.wsloop reduction(@iand_i32 %[[VAL_4]]#0 -> %[[VAL_12:.*]] : !fir.ref<i32>) {
+! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_7:.*]] : !fir.ref<i32>) reduction(@iand_i32 %[[VAL_4]]#0 -> %[[VAL_12:.*]] : !fir.ref<i32>) {
 ! CHECK-NEXT:          omp.loop_nest (%[[VAL_13:.*]]) : i32 = (%[[VAL_9]]) to (%[[VAL_10]]) inclusive step (%[[VAL_11]]) {
+! CHECK:                 %[[VAL_8:.*]]:2 = hlfir.declare %[[VAL_7]] {uniq_name = "_QFreduction_iandEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:                 %[[VAL_14:.*]]:2 = hlfir.declare %[[VAL_12]] {uniq_name = "_QFreduction_iandEx"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:                 fir.store %[[VAL_13]] to %[[VAL_8]]#1 : !fir.ref<i32>
 ! CHECK:                 %[[VAL_15:.*]] = fir.load %[[VAL_8]]#0 : !fir.ref<i32>
diff --git a/flang/test/Lower/OpenMP/wsloop-reduction-ieor-byref.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-ieor-byref.f90
index 284ada404bd60..6be6913f91a33 100644
--- a/flang/test/Lower/OpenMP/wsloop-reduction-ieor-byref.f90
+++ b/flang/test/Lower/OpenMP/wsloop-reduction-ieor-byref.f90
@@ -28,10 +28,9 @@
 
 
 !CHECK: omp.parallel
-!CHECK: %[[I_REF:.*]] = fir.alloca i32 {bindc_name = "i", pinned, {{.*}}}
-!CHECK: %[[I_DECL:.*]]:2 = hlfir.declare %[[I_REF]] {uniq_name = "_QFreduction_ieorEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-!CHECK: omp.wsloop reduction(byref @ieor_byref_i32 %[[X_DECL]]#0 -> %[[PRV:.+]] : !fir.ref<i32>)
+!CHECK: omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[I_REF:.*]] : !fir.ref<i32>) reduction(byref @ieor_byref_i32 %[[X_DECL]]#0 -> %[[PRV:.+]] : !fir.ref<i32>)
 !CHECK-NEXT: omp.loop_nest
+!CHECK: %[[I_DECL:.*]]:2 = hlfir.declare %[[I_REF]] {uniq_name = "_QFreduction_ieorEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 !CHECK: %[[PRV_DECL:.+]]:2 = hlfir.declare %[[PRV]] {{.*}} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 !CHECK: fir.store %{{.*}} to %[[I_DECL]]#1 : !fir.ref<i32>
 !CHECK: %[[I_32:.*]] = fir.load %[[I_DECL]]#0 : !fir.ref<i32>
diff --git a/flang/test/Lower/OpenMP/wsloop-reduction-ieor.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-ieor.f90
index e67253a413ce2..632dbcf1348ec 100644
--- a/flang/test/Lower/OpenMP/wsloop-reduction-ieor.f90
+++ b/flang/test/Lower/OpenMP/wsloop-reduction-ieor.f90
@@ -17,10 +17,9 @@
 
 
 !CHECK: omp.parallel
-!CHECK: %[[I_REF:.*]] = fir.alloca i32 {bindc_name = "i", pinned, {{.*}}}
-!CHECK: %[[I_DECL:.*]]:2 = hlfir.declare %[[I_REF]] {uniq_name = "_QFreduction_ieorEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-!CHECK: omp.wsloop reduction(@[[IEOR_DECLARE_I]] %[[X_DECL]]#0 -> %[[PRV:.+]] : !fir.ref<i32>)
+!CHECK: omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[I_REF:.*]] : !fir.ref<i32>) reduction(@[[IEOR_DECLARE_I]] %[[X_DECL]]#0 -> %[[PRV:.+]] : !fir.ref<i32>)
 !CHECK-NEXT: omp.loop_nest
+!CHECK: %[[I_DECL:.*]]:2 = hlfir.declare %[[I_REF]] {uniq_name = "_QFreduction_ieorEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 !CHECK: %[[PRV_DECL:.+]]:2 = hlfir.declare %[[PRV]] {{.*}} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 !CHECK: fir.store %{{.*}} to %[[I_DECL]]#1 : !fir.ref<i32>
 !CHECK: %[[I_32:.*]] = fir.load %[[I_DECL]]#0 : !fir.ref<i32>
diff --git a/flang/test/Lower/OpenMP/wsloop-reduction-ior-byref.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-ior-byref.f90
index 315121cc7beb7..90b9d2f61f930 100644
--- a/flang/test/Lower/OpenMP/wsloop-reduction-ior-byref.f90
+++ b/flang/test/Lower/OpenMP/wsloop-reduction-ior-byref.f90
@@ -30,13 +30,12 @@
 ! CHECK:           %[[VAL_6:.*]] = arith.constant 0 : i32
 ! CHECK:           hlfir.assign %[[VAL_6]] to %[[VAL_4]]#0 : i32, !fir.ref<i32>
 ! CHECK:           omp.parallel
-! CHECK:             %[[VAL_7:.*]] = fir.alloca i32 {bindc_name = "i", pinned, {{.*}}}
-! CHECK:             %[[VAL_8:.*]]:2 = hlfir.declare %[[VAL_7]] {uniq_name = "_QFreduction_iorEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:             %[[VAL_9:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_10:.*]] = arith.constant 100 : i32
 ! CHECK:             %[[VAL_11:.*]] = arith.constant 1 : i32
-! CHECK:             omp.wsloop reduction(byref @ior_byref_i32 %[[VAL_4]]#0 -> %[[VAL_12:.*]] : !fir.ref<i32>) {
+! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_7:.*]] : !fir.ref<i32>) reduction(byref @ior_byref_i32 %[[VAL_4]]#0 -> %[[VAL_12:.*]] : !fir.ref<i32>) {
 ! CHECK-NEXT:          omp.loop_nest (%[[VAL_13:.*]]) : i32 = (%[[VAL_9]]) to (%[[VAL_10]]) inclusive step (%[[VAL_11]]) {
+! CHECK:                 %[[VAL_8:.*]]:2 = hlfir.declare %[[VAL_7]] {uniq_name = "_QFreduction_iorEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:                 %[[VAL_14:.*]]:2 = hlfir.declare %[[VAL_12]] {uniq_name = "_QFreduction_iorEx"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:                 fir.store %[[VAL_13]] to %[[VAL_8]]#1 : !fir.ref<i32>
 ! CHECK:                 %[[VAL_15:.*]] = fir.load %[[VAL_8]]#0 : !fir.ref<i32>
diff --git a/flang/test/Lower/OpenMP/wsloop-reduction-ior.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-ior.f90
index 3da250da9703d..144bc17cf8b31 100644
--- a/flang/test/Lower/OpenMP/wsloop-reduction-ior.f90
+++ b/flang/test/Lower/OpenMP/wsloop-reduction-ior.f90
@@ -24,13 +24,12 @@
 ! CHECK:           %[[VAL_6:.*]] = arith.constant 0 : i32
 ! CHECK:           hlfir.assign %[[VAL_6]] to %[[VAL_4]]#0 : i32, !fir.ref<i32>
 ! CHECK:           omp.parallel
-! CHECK:             %[[VAL_7:.*]] = fir.alloca i32 {bindc_name = "i", pinned, {{.*}}}
-! CHECK:             %[[VAL_8:.*]]:2 = hlfir.declare %[[VAL_7]] {uniq_name = "_QFreduction_iorEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:             %[[VAL_9:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_10:.*]] = arith.constant 100 : i32
 ! CHECK:             %[[VAL_11:.*]] = arith.constant 1 : i32
-! CHECK:             omp.wsloop reduction(@ior_i32 %[[VAL_4]]#0 -> %[[VAL_12:.*]] : !fir.ref<i32>) {
+! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_7:.*]] : !fir.ref<i32>) reduction(@ior_i32 %[[VAL_4]]#0 -> %[[VAL_12:.*]] : !fir.ref<i32>) {
 ! CHECK-NEXT:          omp.loop_nest (%[[VAL_13:.*]]) : i32 = (%[[VAL_9]]) to (%[[VAL_10]]) inclusive step (%[[VAL_11]]) {
+! CHECK:                 %[[VAL_8:.*]]:2 = hlfir.declare %[[VAL_7]] {uniq_name = "_QFreduction_iorEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:                 %[[VAL_14:.*]]:2 = hlfir.declare %[[VAL_12]] {uniq_name = "_QFreduction_iorEx"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:                 fir.store %[[VAL_13]] to %[[VAL_8]]#1 : !fir.ref<i32>
 ! CHECK:                 %[[VAL_15:.*]] = fir.load %[[VAL_8]]#0 : !fir.ref<i32>
diff --git a/flang/test/Lower/OpenMP/wsloop-reduction-logical-and-byref.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-logical-and-byref.f90
index 30908b6bdd4ce..e73540a93a71b 100644
--- a/flang/test/Lower/OpenMP/wsloop-reduction-logical-and-byref.f90
+++ b/flang/test/Lower/OpenMP/wsloop-reduction-logical-and-byref.f90
@@ -39,13 +39,12 @@
 ! CHECK:           %[[VAL_9:.*]] = fir.convert %[[VAL_8]] : (i1) -> !fir.logical<4>
 ! CHECK:           hlfir.assign %[[VAL_9]] to %[[VAL_4]]#0 : !fir.logical<4>, !fir.ref<!fir.logical<4>>
 ! CHECK:           omp.parallel {
-! CHECK:             %[[VAL_10:.*]] = fir.alloca i32 {bindc_name = "i", pinned, {{.*}}}
-! CHECK:             %[[VAL_11:.*]]:2 = hlfir.declare %[[VAL_10]] {uniq_name = "_QFsimple_reductionEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:             %[[VAL_12:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_13:.*]] = arith.constant 100 : i32
 ! CHECK:             %[[VAL_14:.*]] = arith.constant 1 : i32
-! CHECK:             omp.wsloop reduction(byref @and_reduction %[[VAL_4]]#0 -> %[[VAL_15:.*]] : !fir.ref<!fir.logical<4>>) {
+! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_10:.*]] : !fir.ref<i32>) reduction(byref @and_reduction %[[VAL_4]]#0 -> %[[VAL_15:.*]] : !fir.ref<!fir.logical<4>>) {
 ! CHECK-NEXT:          omp.loop_nest (%[[VAL_16:.*]]) : i32 = (%[[VAL_12]]) to (%[[VAL_13]]) inclusive step (%[[VAL_14]]) {
+! CHECK:                 %[[VAL_11:.*]]:2 = hlfir.declare %[[VAL_10]] {uniq_name = "_QFsimple_reductionEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:                 %[[VAL_17:.*]]:2 = hlfir.declare %[[VAL_15]] {uniq_name = "_QFsimple_reductionEx"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
 ! CHECK:                 fir.store %[[VAL_16]] to %[[VAL_11]]#1 : !fir.ref<i32>
 ! CHECK:                 %[[VAL_18:.*]] = fir.load %[[VAL_17]]#0 : !fir.ref<!fir.logical<4>>
@@ -88,13 +87,12 @@ end subroutine simple_reduction
 ! CHECK:           %[[VAL_9:.*]] = fir.convert %[[VAL_8]] : (i1) -> !fir.logical<4>
 ! CHECK:           hlfir.assign %[[VAL_9]] to %[[VAL_4]]#0 : !fir.logical<4>, !fir.ref<!fir.logical<4>>
 ! CHECK:           omp.parallel {
-! CHECK:             %[[VAL_10:.*]] = fir.alloca i32 {bindc_name = "i", pinned, {{.*}}}
-! CHECK:             %[[VAL_11:.*]]:2 = hlfir.declare %[[VAL_10]] {uniq_name = "_QFsimple_reduction_switch_orderEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:             %[[VAL_12:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_13:.*]] = arith.constant 100 : i32
 ! CHECK:             %[[VAL_14:.*]] = arith.constant 1 : i32
-! CHECK:             omp.wsloop reduction(byref @and_reduction %[[VAL_4]]#0 -> %[[VAL_15:.*]] : !fir.ref<!fir.logical<4>>) {
+! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_10:.*]] : !fir.ref<i32>) reduction(byref @and_reduction %[[VAL_4]]#0 -> %[[VAL_15:.*]] : !fir.ref<!fir.logical<4>>) {
 ! CHECK-NEXT:          omp.loop_nest (%[[VAL_16:.*]]) : i32 = (%[[VAL_12]]) to (%[[VAL_13]]) inclusive step (%[[VAL_14]]) {
+! CHECK:                 %[[VAL_11:.*]]:2 = hlfir.declare %[[VAL_10]] {uniq_name = "_QFsimple_reduction_switch_orderEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:                 %[[VAL_17:.*]]:2 = hlfir.declare %[[VAL_15]] {uniq_name = "_QFsimple_reduction_switch_orderEx"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
 ! CHECK:                 fir.store %[[VAL_16]] to %[[VAL_11]]#1 : !fir.ref<i32>
 ! CHECK:                 %[[VAL_18:.*]] = fir.load %[[VAL_11]]#0 : !fir.ref<i32>
@@ -146,13 +144,12 @@ subroutine simple_reduction_switch_order(y)
 ! CHECK:           %[[VAL_17:.*]] = fir.convert %[[VAL_16]] : (i1) -> !fir.logical<4>
 ! CHECK:           hlfir.assign %[[VAL_17]] to %[[VAL_11]]#0 : !fir.logical<4>, !fir.ref<!fir.logical<4>>
 ! CHECK:           omp.parallel {
-! CHECK:             %[[VAL_18:.*]] = fir.alloca i32 {bindc_name = "i", pinned, {{.*}}}
-! CHECK:             %[[VAL_19:.*]]:2 = hlfir.declare %[[VAL_18]] {uniq_name = "_QFmultiple_reductionsEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:             %[[VAL_20:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_21:.*]] = arith.constant 100 : i32
 ! CHECK:             %[[VAL_22:.*]] = arith.constant 1 : i32
-! CHECK:             omp.wsloop reduction(byref @and_reduction %[[VAL_7]]#0 -> %[[VAL_23:.*]], byref @and_reduction %[[VAL_9]]#0 -> %[[VAL_24:.*]], byref @and_reduction %[[VAL_11]]#0 -> %[[VAL_25:.*]] : !fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>) {
+! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_18:.*]] : !fir.ref<i32>) reduction(byref @and_reduction %[[VAL_7]]#0 -> %[[VAL_23:.*]], byref @and_reduction %[[VAL_9]]#0 -> %[[VAL_24:.*]], byref @and_reduction %[[VAL_11]]#0 -> %[[VAL_25:.*]] : !fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>) {
 ! CHECK-NEXT:          omp.loop_nest (%[[VAL_26:.*]]) : i32 = (%[[VAL_20]]) to (%[[VAL_21]]) inclusive step (%[[VAL_22]]) {
+! CHECK:                 %[[VAL_19:.*]]:2 = hlfir.declare %[[VAL_18]] {uniq_name = "_QFmultiple_reductionsEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:                 %[[VAL_27:.*]]:2 = hlfir.declare %[[VAL_23]] {uniq_name = "_QFmultiple_reductionsEx"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
 ! CHECK:                 %[[VAL_28:.*]]:2 = hlfir.declare %[[VAL_24]] {uniq_name = "_QFmultiple_reductionsEy"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
 ! CHECK:                 %[[VAL_29:.*]]:2 = hlfir.declare %[[VAL_25]] {uniq_name = "_QFmultiple_reductionsEz"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
diff --git a/flang/test/Lower/OpenMP/wsloop-reduction-logical-and.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-logical-and.f90
index 367683de02080..c059dab5bff5a 100644
--- a/flang/test/Lower/OpenMP/wsloop-reduction-logical-and.f90
+++ b/flang/test/Lower/OpenMP/wsloop-reduction-logical-and.f90
@@ -31,13 +31,12 @@
 ! CHECK:           %[[VAL_9:.*]] = fir.convert %[[VAL_8]] : (i1) -> !fir.logical<4>
 ! CHECK:           hlfir.assign %[[VAL_9]] to %[[VAL_4]]#0 : !fir.logical<4>, !fir.ref<!fir.logical<4>>
 ! CHECK:           omp.parallel {
-! CHECK:             %[[VAL_10:.*]] = fir.alloca i32 {bindc_name = "i", pinned, {{.*}}}
-! CHECK:             %[[VAL_11:.*]]:2 = hlfir.declare %[[VAL_10]] {uniq_name = "_QFsimple_reductionEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:             %[[VAL_12:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_13:.*]] = arith.constant 100 : i32
 ! CHECK:             %[[VAL_14:.*]] = arith.constant 1 : i32
-! CHECK:             omp.wsloop reduction(@and_reduction %[[VAL_4]]#0 -> %[[VAL_15:.*]] : !fir.ref<!fir.logical<4>>) {
+! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_10:.*]] : !fir.ref<i32>) reduction(@and_reduction %[[VAL_4]]#0 -> %[[VAL_15:.*]] : !fir.ref<!fir.logical<4>>) {
 ! CHECK-NEXT:          omp.loop_nest (%[[VAL_16:.*]]) : i32 = (%[[VAL_12]]) to (%[[VAL_13]]) inclusive step (%[[VAL_14]]) {
+! CHECK:                 %[[VAL_11:.*]]:2 = hlfir.declare %[[VAL_10]] {uniq_name = "_QFsimple_reductionEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:                 %[[VAL_17:.*]]:2 = hlfir.declare %[[VAL_15]] {uniq_name = "_QFsimple_reductionEx"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
 ! CHECK:                 fir.store %[[VAL_16]] to %[[VAL_11]]#1 : !fir.ref<i32>
 ! CHECK:                 %[[VAL_18:.*]] = fir.load %[[VAL_17]]#0 : !fir.ref<!fir.logical<4>>
@@ -80,13 +79,12 @@ end subroutine simple_reduction
 ! CHECK:           %[[VAL_9:.*]] = fir.convert %[[VAL_8]] : (i1) -> !fir.logical<4>
 ! CHECK:           hlfir.assign %[[VAL_9]] to %[[VAL_4]]#0 : !fir.logical<4>, !fir.ref<!fir.logical<4>>
 ! CHECK:           omp.parallel {
-! CHECK:             %[[VAL_10:.*]] = fir.alloca i32 {bindc_name = "i", pinned, {{.*}}}
-! CHECK:             %[[VAL_11:.*]]:2 = hlfir.declare %[[VAL_10]] {uniq_name = "_QFsimple_reduction_switch_orderEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:             %[[VAL_12:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_13:.*]] = arith.constant 100 : i32
 ! CHECK:             %[[VAL_14:.*]] = arith.constant 1 : i32
-! CHECK:             omp.wsloop reduction(@and_reduction %[[VAL_4]]#0 -> %[[VAL_15:.*]] : !fir.ref<!fir.logical<4>>) {
+! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_10:.*]] : !fir.ref<i32>) reduction(@and_reduction %[[VAL_4]]#0 -> %[[VAL_15:.*]] : !fir.ref<!fir.logical<4>>) {
 ! CHECK-NEXT:          omp.loop_nest (%[[VAL_16:.*]]) : i32 = (%[[VAL_12]]) to (%[[VAL_13]]) inclusive step (%[[VAL_14]]) {
+! CHECK:                 %[[VAL_11:.*]]:2 = hlfir.declare %[[VAL_10]] {uniq_name = "_QFsimple_reduction_switch_orderEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:                 %[[VAL_17:.*]]:2 = hlfir.declare %[[VAL_15]] {uniq_name = "_QFsimple_reduction_switch_orderEx"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
 ! CHECK:                 fir.store %[[VAL_16]] to %[[VAL_11]]#1 : !fir.ref<i32>
 ! CHECK:                 %[[VAL_18:.*]] = fir.load %[[VAL_11]]#0 : !fir.ref<i32>
@@ -138,13 +136,12 @@ subroutine simple_reduction_switch_order(y)
 ! CHECK:           %[[VAL_17:.*]] = fir.convert %[[VAL_16]] : (i1) -> !fir.logical<4>
 ! CHECK:           hlfir.assign %[[VAL_17]] to %[[VAL_11]]#0 : !fir.logical<4>, !fir.ref<!fir.logical<4>>
 ! CHECK:           omp.parallel {
-! CHECK:             %[[VAL_18:.*]] = fir.alloca i32 {bindc_name = "i", pinned, {{.*}}}
-! CHECK:             %[[VAL_19:.*]]:2 = hlfir.declare %[[VAL_18]] {uniq_name = "_QFmultiple_reductionsEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:             %[[VAL_20:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_21:.*]] = arith.constant 100 : i32
 ! CHECK:             %[[VAL_22:.*]] = arith.constant 1 : i32
-! CHECK:             omp.wsloop reduction(@and_reduction %[[VAL_7]]#0 -> %[[VAL_23:.*]], @and_reduction %[[VAL_9]]#0 -> %[[VAL_24:.*]], @and_reduction %[[VAL_11]]#0 -> %[[VAL_25:.*]] : !fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>) {
+! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_18:.*]] : !fir.ref<i32>) reduction(@and_reduction %[[VAL_7]]#0 -> %[[VAL_23:.*]], @and_reduction %[[VAL_9]]#0 -> %[[VAL_24:.*]], @and_reduction %[[VAL_11]]#0 -> %[[VAL_25:.*]] : !fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>) {
 ! CHECK-NEXT:          omp.loop_nest (%[[VAL_26:.*]]) : i32 = (%[[VAL_20]]) to (%[[VAL_21]]) inclusive step (%[[VAL_22]]) {
+! CHECK:                 %[[VAL_19:.*]]:2 = hlfir.declare %[[VAL_18]] {uniq_name = "_QFmultiple_reductionsEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:                 %[[VAL_27:.*]]:2 = hlfir.declare %[[VAL_23]] {uniq_name = "_QFmultiple_reductionsEx"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
 ! CHECK:                 %[[VAL_28:.*]]:2 = hlfir.declare %[[VAL_24]] {uniq_name = "_QFmultiple_reductionsEy"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
 ! CHECK:                 %[[VAL_29:.*]]:2 = hlfir.declare %[[VAL_25]] {uniq_name = "_QFmultiple_reductionsEz"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
diff --git a/flang/test/Lower/OpenMP/wsloop-reduction-logical-eqv-byref.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-logical-eqv-byref.f90
index 9137dd8ff4454..5e24ad6f7bb63 100644
--- a/flang/test/Lower/OpenMP/wsloop-reduction-logical-eqv-byref.f90
+++ b/flang/test/Lower/OpenMP/wsloop-reduction-logical-eqv-byref.f90
@@ -39,13 +39,12 @@
 ! CHECK:           %[[VAL_9:.*]] = fir.convert %[[VAL_8]] : (i1) -> !fir.logical<4>
 ! CHECK:           hlfir.assign %[[VAL_9]] to %[[VAL_4]]#0 : !fir.logical<4>, !fir.ref<!fir.logical<4>>
 ! CHECK:           omp.parallel {
-! CHECK:             %[[VAL_10:.*]] = fir.alloca i32 {bindc_name = "i", pinned, {{.*}}}
-! CHECK:             %[[VAL_11:.*]]:2 = hlfir.declare %[[VAL_10]] {uniq_name = "_QFsimple_reductionEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:             %[[VAL_12:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_13:.*]] = arith.constant 100 : i32
 ! CHECK:             %[[VAL_14:.*]] = arith.constant 1 : i32
-! CHECK:             omp.wsloop reduction(byref @eqv_reduction %[[VAL_4]]#0 -> %[[VAL_15:.*]] : !fir.ref<!fir.logical<4>>) {
+! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_10:.*]] : !fir.ref<i32>) reduction(byref @eqv_reduction %[[VAL_4]]#0 -> %[[VAL_15:.*]] : !fir.ref<!fir.logical<4>>) {
 ! CHECK-NEXT:          omp.loop_nest (%[[VAL_16:.*]]) : i32 = (%[[VAL_12]]) to (%[[VAL_13]]) inclusive step (%[[VAL_14]]) {
+! CHECK:                 %[[VAL_11:.*]]:2 = hlfir.declare %[[VAL_10]] {uniq_name = "_QFsimple_reductionEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:                 %[[VAL_17:.*]]:2 = hlfir.declare %[[VAL_15]] {uniq_name = "_QFsimple_reductionEx"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
 ! CHECK:                 fir.store %[[VAL_16]] to %[[VAL_11]]#1 : !fir.ref<i32>
 ! CHECK:                 %[[VAL_18:.*]] = fir.load %[[VAL_17]]#0 : !fir.ref<!fir.logical<4>>
@@ -87,13 +86,12 @@ subroutine simple_reduction(y)
 ! CHECK:           %[[VAL_9:.*]] = fir.convert %[[VAL_8]] : (i1) -> !fir.logical<4>
 ! CHECK:           hlfir.assign %[[VAL_9]] to %[[VAL_4]]#0 : !fir.logical<4>, !fir.ref<!fir.logical<4>>
 ! CHECK:           omp.parallel {
-! CHECK:             %[[VAL_10:.*]] = fir.alloca i32 {bindc_name = "i", pinned, {{.*}}}
-! CHECK:             %[[VAL_11:.*]]:2 = hlfir.declare %[[VAL_10]] {uniq_name = "_QFsimple_reduction_switch_orderEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:             %[[VAL_12:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_13:.*]] = arith.constant 100 : i32
 ! CHECK:             %[[VAL_14:.*]] = arith.constant 1 : i32
-! CHECK:             omp.wsloop reduction(byref @eqv_reduction %[[VAL_4]]#0 -> %[[VAL_15:.*]] : !fir.ref<!fir.logical<4>>) {
+! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_10:.*]] : !fir.ref<i32>) reduction(byref @eqv_reduction %[[VAL_4]]#0 -> %[[VAL_15:.*]] : !fir.ref<!fir.logical<4>>) {
 ! CHECK-NEXT:          omp.loop_nest (%[[VAL_16:.*]]) : i32 = (%[[VAL_12]]) to (%[[VAL_13]]) inclusive step (%[[VAL_14]]) {
+! CHECK:                 %[[VAL_11:.*]]:2 = hlfir.declare %[[VAL_10]] {uniq_name = "_QFsimple_reduction_switch_orderEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:                 %[[VAL_17:.*]]:2 = hlfir.declare %[[VAL_15]] {uniq_name = "_QFsimple_reduction_switch_orderEx"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
 ! CHECK:                 fir.store %[[VAL_16]] to %[[VAL_11]]#1 : !fir.ref<i32>
 ! CHECK:                 %[[VAL_18:.*]] = fir.load %[[VAL_11]]#0 : !fir.ref<i32>
@@ -145,13 +143,12 @@ subroutine simple_reduction_switch_order(y)
 ! CHECK:           %[[VAL_17:.*]] = fir.convert %[[VAL_16]] : (i1) -> !fir.logical<4>
 ! CHECK:           hlfir.assign %[[VAL_17]] to %[[VAL_11]]#0 : !fir.logical<4>, !fir.ref<!fir.logical<4>>
 ! CHECK:           omp.parallel {
-! CHECK:             %[[VAL_18:.*]] = fir.alloca i32 {bindc_name = "i", pinned, {{.*}}}
-! CHECK:             %[[VAL_19:.*]]:2 = hlfir.declare %[[VAL_18]] {uniq_name = "_QFmultiple_reductionsEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:             %[[VAL_20:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_21:.*]] = arith.constant 100 : i32
 ! CHECK:             %[[VAL_22:.*]] = arith.constant 1 : i32
-! CHECK:             omp.wsloop reduction(byref @eqv_reduction %[[VAL_7]]#0 -> %[[VAL_23:.*]], byref @eqv_reduction %[[VAL_9]]#0 -> %[[VAL_24:.*]], byref @eqv_reduction %[[VAL_11]]#0 -> %[[VAL_25:.*]] : !fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>) {
+! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_18:.*]] : !fir.ref<i32>) reduction(byref @eqv_reduction %[[VAL_7]]#0 -> %[[VAL_23:.*]], byref @eqv_reduction %[[VAL_9]]#0 -> %[[VAL_24:.*]], byref @eqv_reduction %[[VAL_11]]#0 -> %[[VAL_25:.*]] : !fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>) {
 ! CHECK-NEXT:          omp.loop_nest (%[[VAL_26:.*]]) : i32 = (%[[VAL_20]]) to (%[[VAL_21]]) inclusive step (%[[VAL_22]]) {
+! CHECK:                 %[[VAL_19:.*]]:2 = hlfir.declare %[[VAL_18]] {uniq_name = "_QFmultiple_reductionsEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:                 %[[VAL_27:.*]]:2 = hlfir.declare %[[VAL_23]] {uniq_name = "_QFmultiple_reductionsEx"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
 ! CHECK:                 %[[VAL_28:.*]]:2 = hlfir.declare %[[VAL_24]] {uniq_name = "_QFmultiple_reductionsEy"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
 ! CHECK:                 %[[VAL_29:.*]]:2 = hlfir.declare %[[VAL_25]] {uniq_name = "_QFmultiple_reductionsEz"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
diff --git a/flang/test/Lower/OpenMP/wsloop-reduction-logical-eqv.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-logical-eqv.f90
index d1ef676c37407..ad9e869984eac 100644
--- a/flang/test/Lower/OpenMP/wsloop-reduction-logical-eqv.f90
+++ b/flang/test/Lower/OpenMP/wsloop-reduction-logical-eqv.f90
@@ -31,13 +31,12 @@
 ! CHECK:           %[[VAL_9:.*]] = fir.convert %[[VAL_8]] : (i1) -> !fir.logical<4>
 ! CHECK:           hlfir.assign %[[VAL_9]] to %[[VAL_4]]#0 : !fir.logical<4>, !fir.ref<!fir.logical<4>>
 ! CHECK:           omp.parallel {
-! CHECK:             %[[VAL_10:.*]] = fir.alloca i32 {bindc_name = "i", pinned, {{.*}}}
-! CHECK:             %[[VAL_11:.*]]:2 = hlfir.declare %[[VAL_10]] {uniq_name = "_QFsimple_reductionEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:             %[[VAL_12:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_13:.*]] = arith.constant 100 : i32
 ! CHECK:             %[[VAL_14:.*]] = arith.constant 1 : i32
-! CHECK:             omp.wsloop reduction(@eqv_reduction %[[VAL_4]]#0 -> %[[VAL_15:.*]] : !fir.ref<!fir.logical<4>>) {
+! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_10:.*]] : !fir.ref<i32>) reduction(@eqv_reduction %[[VAL_4]]#0 -> %[[VAL_15:.*]] : !fir.ref<!fir.logical<4>>) {
 ! CHECK-NEXT:          omp.loop_nest (%[[VAL_16:.*]]) : i32 = (%[[VAL_12]]) to (%[[VAL_13]]) inclusive step (%[[VAL_14]]) {
+! CHECK:                 %[[VAL_11:.*]]:2 = hlfir.declare %[[VAL_10]] {uniq_name = "_QFsimple_reductionEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:                 %[[VAL_17:.*]]:2 = hlfir.declare %[[VAL_15]] {uniq_name = "_QFsimple_reductionEx"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
 ! CHECK:                 fir.store %[[VAL_16]] to %[[VAL_11]]#1 : !fir.ref<i32>
 ! CHECK:                 %[[VAL_18:.*]] = fir.load %[[VAL_17]]#0 : !fir.ref<!fir.logical<4>>
@@ -79,13 +78,12 @@ subroutine simple_reduction(y)
 ! CHECK:           %[[VAL_9:.*]] = fir.convert %[[VAL_8]] : (i1) -> !fir.logical<4>
 ! CHECK:           hlfir.assign %[[VAL_9]] to %[[VAL_4]]#0 : !fir.logical<4>, !fir.ref<!fir.logical<4>>
 ! CHECK:           omp.parallel {
-! CHECK:             %[[VAL_10:.*]] = fir.alloca i32 {bindc_name = "i", pinned, {{.*}}}
-! CHECK:             %[[VAL_11:.*]]:2 = hlfir.declare %[[VAL_10]] {uniq_name = "_QFsimple_reduction_switch_orderEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:             %[[VAL_12:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_13:.*]] = arith.constant 100 : i32
 ! CHECK:             %[[VAL_14:.*]] = arith.constant 1 : i32
-! CHECK:             omp.wsloop reduction(@eqv_reduction %[[VAL_4]]#0 -> %[[VAL_15:.*]] : !fir.ref<!fir.logical<4>>) {
+! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_10:.*]] : !fir.ref<i32>) reduction(@eqv_reduction %[[VAL_4]]#0 -> %[[VAL_15:.*]] : !fir.ref<!fir.logical<4>>) {
 ! CHECK-NEXT:          omp.loop_nest (%[[VAL_16:.*]]) : i32 = (%[[VAL_12]]) to (%[[VAL_13]]) inclusive step (%[[VAL_14]]) {
+! CHECK:                 %[[VAL_11:.*]]:2 = hlfir.declare %[[VAL_10]] {uniq_name = "_QFsimple_reduction_switch_orderEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:                 %[[VAL_17:.*]]:2 = hlfir.declare %[[VAL_15]] {uniq_name = "_QFsimple_reduction_switch_orderEx"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
 ! CHECK:                 fir.store %[[VAL_16]] to %[[VAL_11]]#1 : !fir.ref<i32>
 ! CHECK:                 %[[VAL_18:.*]] = fir.load %[[VAL_11]]#0 : !fir.ref<i32>
@@ -137,13 +135,12 @@ subroutine simple_reduction_switch_order(y)
 ! CHECK:           %[[VAL_17:.*]] = fir.convert %[[VAL_16]] : (i1) -> !fir.logical<4>
 ! CHECK:           hlfir.assign %[[VAL_17]] to %[[VAL_11]]#0 : !fir.logical<4>, !fir.ref<!fir.logical<4>>
 ! CHECK:           omp.parallel {
-! CHECK:             %[[VAL_18:.*]] = fir.alloca i32 {bindc_name = "i", pinned, {{.*}}}
-! CHECK:             %[[VAL_19:.*]]:2 = hlfir.declare %[[VAL_18]] {uniq_name = "_QFmultiple_reductionsEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:             %[[VAL_20:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_21:.*]] = arith.constant 100 : i32
 ! CHECK:             %[[VAL_22:.*]] = arith.constant 1 : i32
-! CHECK:             omp.wsloop reduction(@eqv_reduction %[[VAL_7]]#0 -> %[[VAL_23:.*]], @eqv_reduction %[[VAL_9]]#0 -> %[[VAL_24:.*]], @eqv_reduction %[[VAL_11]]#0 -> %[[VAL_25:.*]] : !fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>) {
+! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_18:.*]] : !fir.ref<i32>) reduction(@eqv_reduction %[[VAL_7]]#0 -> %[[VAL_23:.*]], @eqv_reduction %[[VAL_9]]#0 -> %[[VAL_24:.*]], @eqv_reduction %[[VAL_11]]#0 -> %[[VAL_25:.*]] : !fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>) {
 ! CHECK-NEXT:          omp.loop_nest (%[[VAL_26:.*]]) : i32 = (%[[VAL_20]]) to (%[[VAL_21]]) inclusive step (%[[VAL_22]]) {
+! CHECK:                 %[[VAL_19:.*]]:2 = hlfir.declare %[[VAL_18]] {uniq_name = "_QFmultiple_reductionsEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:                 %[[VAL_27:.*]]:2 = hlfir.declare %[[VAL_23]] {uniq_name = "_QFmultiple_reductionsEx"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
 ! CHECK:                 %[[VAL_28:.*]]:2 = hlfir.declare %[[VAL_24]] {uniq_name = "_QFmultiple_reductionsEy"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
 ! CHECK:                 %[[VAL_29:.*]]:2 = hlfir.declare %[[VAL_25]] {uniq_name = "_QFmultiple_reductionsEz"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
diff --git a/flang/test/Lower/OpenMP/wsloop-reduction-logical-neqv-byref.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-logical-neqv-byref.f90
index d1491a0f5561d..b5bf1d0d0b589 100644
--- a/flang/test/Lower/OpenMP/wsloop-reduction-logical-neqv-byref.f90
+++ b/flang/test/Lower/OpenMP/wsloop-reduction-logical-neqv-byref.f90
@@ -39,13 +39,12 @@
 ! CHECK:           %[[VAL_9:.*]] = fir.convert %[[VAL_8]] : (i1) -> !fir.logical<4>
 ! CHECK:           hlfir.assign %[[VAL_9]] to %[[VAL_4]]#0 : !fir.logical<4>, !fir.ref<!fir.logical<4>>
 ! CHECK:           omp.parallel {
-! CHECK:             %[[VAL_10:.*]] = fir.alloca i32 {bindc_name = "i", pinned, {{.*}}}
-! CHECK:             %[[VAL_11:.*]]:2 = hlfir.declare %[[VAL_10]] {uniq_name = "_QFsimple_reductionEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:             %[[VAL_12:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_13:.*]] = arith.constant 100 : i32
 ! CHECK:             %[[VAL_14:.*]] = arith.constant 1 : i32
-! CHECK:             omp.wsloop reduction(byref @neqv_reduction %[[VAL_4]]#0 -> %[[VAL_15:.*]] : !fir.ref<!fir.logical<4>>) {
+! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_10:.*]] : !fir.ref<i32>) reduction(byref @neqv_reduction %[[VAL_4]]#0 -> %[[VAL_15:.*]] : !fir.ref<!fir.logical<4>>) {
 ! CHECK-NEXT:          omp.loop_nest (%[[VAL_16:.*]]) : i32 = (%[[VAL_12]]) to (%[[VAL_13]]) inclusive step (%[[VAL_14]]) {
+! CHECK:                 %[[VAL_11:.*]]:2 = hlfir.declare %[[VAL_10]] {uniq_name = "_QFsimple_reductionEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:                 %[[VAL_17:.*]]:2 = hlfir.declare %[[VAL_15]] {uniq_name = "_QFsimple_reductionEx"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
 ! CHECK:                 fir.store %[[VAL_16]] to %[[VAL_11]]#1 : !fir.ref<i32>
 ! CHECK:                 %[[VAL_18:.*]] = fir.load %[[VAL_17]]#0 : !fir.ref<!fir.logical<4>>
@@ -88,13 +87,12 @@ subroutine simple_reduction(y)
 ! CHECK:           %[[VAL_9:.*]] = fir.convert %[[VAL_8]] : (i1) -> !fir.logical<4>
 ! CHECK:           hlfir.assign %[[VAL_9]] to %[[VAL_4]]#0 : !fir.logical<4>, !fir.ref<!fir.logical<4>>
 ! CHECK:           omp.parallel {
-! CHECK:             %[[VAL_10:.*]] = fir.alloca i32 {bindc_name = "i", pinned, {{.*}}}
-! CHECK:             %[[VAL_11:.*]]:2 = hlfir.declare %[[VAL_10]] {uniq_name = "_QFsimple_reduction_switch_orderEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:             %[[VAL_12:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_13:.*]] = arith.constant 100 : i32
 ! CHECK:             %[[VAL_14:.*]] = arith.constant 1 : i32
-! CHECK:             omp.wsloop reduction(byref @neqv_reduction %[[VAL_4]]#0 -> %[[VAL_15:.*]] : !fir.ref<!fir.logical<4>>) {
+! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_10:.*]] : !fir.ref<i32>) reduction(byref @neqv_reduction %[[VAL_4]]#0 -> %[[VAL_15:.*]] : !fir.ref<!fir.logical<4>>) {
 ! CHECK-NEXT:          omp.loop_nest (%[[VAL_16:.*]]) : i32 = (%[[VAL_12]]) to (%[[VAL_13]]) inclusive step (%[[VAL_14]]) {
+! CHECK:                 %[[VAL_11:.*]]:2 = hlfir.declare %[[VAL_10]] {uniq_name = "_QFsimple_reduction_switch_orderEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:                 %[[VAL_17:.*]]:2 = hlfir.declare %[[VAL_15]] {uniq_name = "_QFsimple_reduction_switch_orderEx"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
 ! CHECK:                 fir.store %[[VAL_16]] to %[[VAL_11]]#1 : !fir.ref<i32>
 ! CHECK:                 %[[VAL_18:.*]] = fir.load %[[VAL_11]]#0 : !fir.ref<i32>
@@ -148,13 +146,12 @@ subroutine simple_reduction_switch_order(y)
 ! CHECK:           %[[VAL_17:.*]] = fir.convert %[[VAL_16]] : (i1) -> !fir.logical<4>
 ! CHECK:           hlfir.assign %[[VAL_17]] to %[[VAL_11]]#0 : !fir.logical<4>, !fir.ref<!fir.logical<4>>
 ! CHECK:           omp.parallel {
-! CHECK:             %[[VAL_18:.*]] = fir.alloca i32 {bindc_name = "i", pinned, {{.*}}}
-! CHECK:             %[[VAL_19:.*]]:2 = hlfir.declare %[[VAL_18]] {uniq_name = "_QFmultiple_reductionsEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:             %[[VAL_20:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_21:.*]] = arith.constant 100 : i32
 ! CHECK:             %[[VAL_22:.*]] = arith.constant 1 : i32
-! CHECK:             omp.wsloop reduction(byref @neqv_reduction %[[VAL_7]]#0 -> %[[VAL_23:.*]], byref @neqv_reduction %[[VAL_9]]#0 -> %[[VAL_24:.*]], byref @neqv_reduction %[[VAL_11]]#0 -> %[[VAL_25:.*]] : !fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>) {
+! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_18:.*]] : !fir.ref<i32>) reduction(byref @neqv_reduction %[[VAL_7]]#0 -> %[[VAL_23:.*]], byref @neqv_reduction %[[VAL_9]]#0 -> %[[VAL_24:.*]], byref @neqv_reduction %[[VAL_11]]#0 -> %[[VAL_25:.*]] : !fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>) {
 ! CHECK-NEXT:          omp.loop_nest (%[[VAL_26:.*]]) : i32 = (%[[VAL_20]]) to (%[[VAL_21]]) inclusive step (%[[VAL_22]]) {
+! CHECK:                 %[[VAL_19:.*]]:2 = hlfir.declare %[[VAL_18]] {uniq_name = "_QFmultiple_reductionsEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:                 %[[VAL_27:.*]]:2 = hlfir.declare %[[VAL_23]] {uniq_name = "_QFmultiple_reductionsEx"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
 ! CHECK:                 %[[VAL_28:.*]]:2 = hlfir.declare %[[VAL_24]] {uniq_name = "_QFmultiple_reductionsEy"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
 ! CHECK:                 %[[VAL_29:.*]]:2 = hlfir.declare %[[VAL_25]] {uniq_name = "_QFmultiple_reductionsEz"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
diff --git a/flang/test/Lower/OpenMP/wsloop-reduction-logical-neqv.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-logical-neqv.f90
index b4df699c49ffa..ac9fc7f051d88 100644
--- a/flang/test/Lower/OpenMP/wsloop-reduction-logical-neqv.f90
+++ b/flang/test/Lower/OpenMP/wsloop-reduction-logical-neqv.f90
@@ -31,13 +31,12 @@
 ! CHECK:           %[[VAL_9:.*]] = fir.convert %[[VAL_8]] : (i1) -> !fir.logical<4>
 ! CHECK:           hlfir.assign %[[VAL_9]] to %[[VAL_4]]#0 : !fir.logical<4>, !fir.ref<!fir.logical<4>>
 ! CHECK:           omp.parallel {
-! CHECK:             %[[VAL_10:.*]] = fir.alloca i32 {bindc_name = "i", pinned, {{.*}}}
-! CHECK:             %[[VAL_11:.*]]:2 = hlfir.declare %[[VAL_10]] {uniq_name = "_QFsimple_reductionEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:             %[[VAL_12:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_13:.*]] = arith.constant 100 : i32
 ! CHECK:             %[[VAL_14:.*]] = arith.constant 1 : i32
-! CHECK:             omp.wsloop reduction(@neqv_reduction %[[VAL_4]]#0 -> %[[VAL_15:.*]] : !fir.ref<!fir.logical<4>>) {
+! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_10:.*]] : !fir.ref<i32>) reduction(@neqv_reduction %[[VAL_4]]#0 -> %[[VAL_15:.*]] : !fir.ref<!fir.logical<4>>) {
 ! CHECK-NEXT:          omp.loop_nest (%[[VAL_16:.*]]) : i32 = (%[[VAL_12]]) to (%[[VAL_13]]) inclusive step (%[[VAL_14]]) {
+! CHECK:                 %[[VAL_11:.*]]:2 = hlfir.declare %[[VAL_10]] {uniq_name = "_QFsimple_reductionEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:                 %[[VAL_17:.*]]:2 = hlfir.declare %[[VAL_15]] {uniq_name = "_QFsimple_reductionEx"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
 ! CHECK:                 fir.store %[[VAL_16]] to %[[VAL_11]]#1 : !fir.ref<i32>
 ! CHECK:                 %[[VAL_18:.*]] = fir.load %[[VAL_17]]#0 : !fir.ref<!fir.logical<4>>
@@ -80,13 +79,12 @@ subroutine simple_reduction(y)
 ! CHECK:           %[[VAL_9:.*]] = fir.convert %[[VAL_8]] : (i1) -> !fir.logical<4>
 ! CHECK:           hlfir.assign %[[VAL_9]] to %[[VAL_4]]#0 : !fir.logical<4>, !fir.ref<!fir.logical<4>>
 ! CHECK:           omp.parallel {
-! CHECK:             %[[VAL_10:.*]] = fir.alloca i32 {bindc_name = "i", pinned, {{.*}}}
-! CHECK:             %[[VAL_11:.*]]:2 = hlfir.declare %[[VAL_10]] {uniq_name = "_QFsimple_reduction_switch_orderEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:             %[[VAL_12:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_13:.*]] = arith.constant 100 : i32
 ! CHECK:             %[[VAL_14:.*]] = arith.constant 1 : i32
-! CHECK:             omp.wsloop reduction(@neqv_reduction %[[VAL_4]]#0 -> %[[VAL_15:.*]] : !fir.ref<!fir.logical<4>>) {
+! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_10:.*]] : !fir.ref<i32>) reduction(@neqv_reduction %[[VAL_4]]#0 -> %[[VAL_15:.*]] : !fir.ref<!fir.logical<4>>) {
 ! CHECK-NEXT:          omp.loop_nest (%[[VAL_16:.*]]) : i32 = (%[[VAL_12]]) to (%[[VAL_13]]) inclusive step (%[[VAL_14]]) {
+! CHECK:                 %[[VAL_11:.*]]:2 = hlfir.declare %[[VAL_10]] {uniq_name = "_QFsimple_reduction_switch_orderEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:                 %[[VAL_17:.*]]:2 = hlfir.declare %[[VAL_15]] {uniq_name = "_QFsimple_reduction_switch_orderEx"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
 ! CHECK:                 fir.store %[[VAL_16]] to %[[VAL_11]]#1 : !fir.ref<i32>
 ! CHECK:                 %[[VAL_18:.*]] = fir.load %[[VAL_11]]#0 : !fir.ref<i32>
@@ -140,13 +138,12 @@ subroutine simple_reduction_switch_order(y)
 ! CHECK:           %[[VAL_17:.*]] = fir.convert %[[VAL_16]] : (i1) -> !fir.logical<4>
 ! CHECK:           hlfir.assign %[[VAL_17]] to %[[VAL_11]]#0 : !fir.logical<4>, !fir.ref<!fir.logical<4>>
 ! CHECK:           omp.parallel {
-! CHECK:             %[[VAL_18:.*]] = fir.alloca i32 {bindc_name = "i", pinned, {{.*}}}
-! CHECK:             %[[VAL_19:.*]]:2 = hlfir.declare %[[VAL_18]] {uniq_name = "_QFmultiple_reductionsEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:             %[[VAL_20:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_21:.*]] = arith.constant 100 : i32
 ! CHECK:             %[[VAL_22:.*]] = arith.constant 1 : i32
-! CHECK:             omp.wsloop reduction(@neqv_reduction %[[VAL_7]]#0 -> %[[VAL_23:.*]], @neqv_reduction %[[VAL_9]]#0 -> %[[VAL_24:.*]], @neqv_reduction %[[VAL_11]]#0 -> %[[VAL_25:.*]] : !fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>) {
+! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_18:.*]] : !fir.ref<i32>) reduction(@neqv_reduction %[[VAL_7]]#0 -> %[[VAL_23:.*]], @neqv_reduction %[[VAL_9]]#0 -> %[[VAL_24:.*]], @neqv_reduction %[[VAL_11]]#0 -> %[[VAL_25:.*]] : !fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>) {
 ! CHECK-NEXT:          omp.loop_nest (%[[VAL_26:.*]]) : i32 = (%[[VAL_20]]) to (%[[VAL_21]]) inclusive step (%[[VAL_22]]) {
+! CHECK:                 %[[VAL_19:.*]]:2 = hlfir.declare %[[VAL_18]] {uniq_name = "_QFmultiple_reductionsEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:                 %[[VAL_27:.*]]:2 = hlfir.declare %[[VAL_23]] {uniq_name = "_QFmultiple_reductionsEx"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
 ! CHECK:                 %[[VAL_28:.*]]:2 = hlfir.declare %[[VAL_24]] {uniq_name = "_QFmultiple_reductionsEy"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
 ! CHECK:                 %[[VAL_29:.*]]:2 = hlfir.declare %[[VAL_25]] {uniq_name = "_QFmultiple_reductionsEz"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
diff --git a/flang/test/Lower/OpenMP/wsloop-reduction-logical-or-byref.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-logical-or-byref.f90
index 8f4a6c22c1d74..883064884b637 100644
--- a/flang/test/Lower/OpenMP/wsloop-reduction-logical-or-byref.f90
+++ b/flang/test/Lower/OpenMP/wsloop-reduction-logical-or-byref.f90
@@ -38,13 +38,12 @@
 ! CHECK:           %[[VAL_9:.*]] = fir.convert %[[VAL_8]] : (i1) -> !fir.logical<4>
 ! CHECK:           hlfir.assign %[[VAL_9]] to %[[VAL_4]]#0 : !fir.logical<4>, !fir.ref<!fir.logical<4>>
 ! CHECK:           omp.parallel {
-! CHECK:             %[[VAL_10:.*]] = fir.alloca i32 {bindc_name = "i", pinned, {{.*}}}
-! CHECK:             %[[VAL_11:.*]]:2 = hlfir.declare %[[VAL_10]] {uniq_name = "_QFsimple_reductionEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:             %[[VAL_12:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_13:.*]] = arith.constant 100 : i32
 ! CHECK:             %[[VAL_14:.*]] = arith.constant 1 : i32
-! CHECK:             omp.wsloop reduction(byref @or_reduction %[[VAL_4]]#0 -> %[[VAL_15:.*]] : !fir.ref<!fir.logical<4>>) {
+! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_10:.*]] : !fir.ref<i32>) reduction(byref @or_reduction %[[VAL_4]]#0 -> %[[VAL_15:.*]] : !fir.ref<!fir.logical<4>>) {
 ! CHECK-NEXT:          omp.loop_nest (%[[VAL_16:.*]]) : i32 = (%[[VAL_12]]) to (%[[VAL_13]]) inclusive step (%[[VAL_14]]) {
+! CHECK:                 %[[VAL_11:.*]]:2 = hlfir.declare %[[VAL_10]] {uniq_name = "_QFsimple_reductionEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:                 %[[VAL_17:.*]]:2 = hlfir.declare %[[VAL_15]] {uniq_name = "_QFsimple_reductionEx"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
 ! CHECK:                 fir.store %[[VAL_16]] to %[[VAL_11]]#1 : !fir.ref<i32>
 ! CHECK:                 %[[VAL_18:.*]] = fir.load %[[VAL_17]]#0 : !fir.ref<!fir.logical<4>>
@@ -86,13 +85,12 @@ subroutine simple_reduction(y)
 ! CHECK:           %[[VAL_9:.*]] = fir.convert %[[VAL_8]] : (i1) -> !fir.logical<4>
 ! CHECK:           hlfir.assign %[[VAL_9]] to %[[VAL_4]]#0 : !fir.logical<4>, !fir.ref<!fir.logical<4>>
 ! CHECK:           omp.parallel {
-! CHECK:             %[[VAL_10:.*]] = fir.alloca i32 {bindc_name = "i", pinned, {{.*}}}
-! CHECK:             %[[VAL_11:.*]]:2 = hlfir.declare %[[VAL_10]] {uniq_name = "_QFsimple_reduction_switch_orderEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:             %[[VAL_12:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_13:.*]] = arith.constant 100 : i32
 ! CHECK:             %[[VAL_14:.*]] = arith.constant 1 : i32
-! CHECK:             omp.wsloop reduction(byref @or_reduction %[[VAL_4]]#0 -> %[[VAL_15:.*]] : !fir.ref<!fir.logical<4>>) {
+! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_10:.*]] : !fir.ref<i32>) reduction(byref @or_reduction %[[VAL_4]]#0 -> %[[VAL_15:.*]] : !fir.ref<!fir.logical<4>>) {
 ! CHECK-NEXT:          omp.loop_nest (%[[VAL_16:.*]]) : i32 = (%[[VAL_12]]) to (%[[VAL_13]]) inclusive step (%[[VAL_14]]) {
+! CHECK:                 %[[VAL_11:.*]]:2 = hlfir.declare %[[VAL_10]] {uniq_name = "_QFsimple_reduction_switch_orderEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:                 %[[VAL_17:.*]]:2 = hlfir.declare %[[VAL_15]] {uniq_name = "_QFsimple_reduction_switch_orderEx"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
 ! CHECK:                 fir.store %[[VAL_16]] to %[[VAL_11]]#1 : !fir.ref<i32>
 ! CHECK:                 %[[VAL_18:.*]] = fir.load %[[VAL_11]]#0 : !fir.ref<i32>
@@ -144,13 +142,12 @@ subroutine simple_reduction_switch_order(y)
 ! CHECK:           %[[VAL_17:.*]] = fir.convert %[[VAL_16]] : (i1) -> !fir.logical<4>
 ! CHECK:           hlfir.assign %[[VAL_17]] to %[[VAL_11]]#0 : !fir.logical<4>, !fir.ref<!fir.logical<4>>
 ! CHECK:           omp.parallel {
-! CHECK:             %[[VAL_18:.*]] = fir.alloca i32 {bindc_name = "i", pinned, {{.*}}}
-! CHECK:             %[[VAL_19:.*]]:2 = hlfir.declare %[[VAL_18]] {uniq_name = "_QFmultiple_reductionsEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:             %[[VAL_20:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_21:.*]] = arith.constant 100 : i32
 ! CHECK:             %[[VAL_22:.*]] = arith.constant 1 : i32
-! CHECK:             omp.wsloop reduction(byref @or_reduction %[[VAL_7]]#0 -> %[[VAL_23:.*]], byref @or_reduction %[[VAL_9]]#0 -> %[[VAL_24:.*]], byref @or_reduction %[[VAL_11]]#0 -> %[[VAL_25:.*]] : !fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>) {
+! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_18:.*]] : !fir.ref<i32>) reduction(byref @or_reduction %[[VAL_7]]#0 -> %[[VAL_23:.*]], byref @or_reduction %[[VAL_9]]#0 -> %[[VAL_24:.*]], byref @or_reduction %[[VAL_11]]#0 -> %[[VAL_25:.*]] : !fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>) {
 ! CHECK-NEXT:          omp.loop_nest (%[[VAL_26:.*]]) : i32 = (%[[VAL_20]]) to (%[[VAL_21]]) inclusive step (%[[VAL_22]]) {
+! CHECK:                 %[[VAL_19:.*]]:2 = hlfir.declare %[[VAL_18]] {uniq_name = "_QFmultiple_reductionsEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:                 %[[VAL_27:.*]]:2 = hlfir.declare %[[VAL_23]] {uniq_name = "_QFmultiple_reductionsEx"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
 ! CHECK:                 %[[VAL_28:.*]]:2 = hlfir.declare %[[VAL_24]] {uniq_name = "_QFmultiple_reductionsEy"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
 ! CHECK:                 %[[VAL_29:.*]]:2 = hlfir.declare %[[VAL_25]] {uniq_name = "_QFmultiple_reductionsEz"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
diff --git a/flang/test/Lower/OpenMP/wsloop-reduction-logical-or.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-logical-or.f90
index 9d367797ec216..312c08d17a14d 100644
--- a/flang/test/Lower/OpenMP/wsloop-reduction-logical-or.f90
+++ b/flang/test/Lower/OpenMP/wsloop-reduction-logical-or.f90
@@ -31,13 +31,12 @@
 ! CHECK:           %[[VAL_9:.*]] = fir.convert %[[VAL_8]] : (i1) -> !fir.logical<4>
 ! CHECK:           hlfir.assign %[[VAL_9]] to %[[VAL_4]]#0 : !fir.logical<4>, !fir.ref<!fir.logical<4>>
 ! CHECK:           omp.parallel {
-! CHECK:             %[[VAL_10:.*]] = fir.alloca i32 {bindc_name = "i", pinned, {{.*}}}
-! CHECK:             %[[VAL_11:.*]]:2 = hlfir.declare %[[VAL_10]] {uniq_name = "_QFsimple_reductionEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:             %[[VAL_12:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_13:.*]] = arith.constant 100 : i32
 ! CHECK:             %[[VAL_14:.*]] = arith.constant 1 : i32
-! CHECK:             omp.wsloop reduction(@or_reduction %[[VAL_4]]#0 -> %[[VAL_15:.*]] : !fir.ref<!fir.logical<4>>) {
+! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_10:.*]] : !fir.ref<i32>) reduction(@or_reduction %[[VAL_4]]#0 -> %[[VAL_15:.*]] : !fir.ref<!fir.logical<4>>) {
 ! CHECK-NEXT:          omp.loop_nest (%[[VAL_16:.*]]) : i32 = (%[[VAL_12]]) to (%[[VAL_13]]) inclusive step (%[[VAL_14]]) {
+! CHECK:                 %[[VAL_11:.*]]:2 = hlfir.declare %[[VAL_10]] {uniq_name = "_QFsimple_reductionEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:                 %[[VAL_17:.*]]:2 = hlfir.declare %[[VAL_15]] {uniq_name = "_QFsimple_reductionEx"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
 ! CHECK:                 fir.store %[[VAL_16]] to %[[VAL_11]]#1 : !fir.ref<i32>
 ! CHECK:                 %[[VAL_18:.*]] = fir.load %[[VAL_17]]#0 : !fir.ref<!fir.logical<4>>
@@ -79,13 +78,12 @@ subroutine simple_reduction(y)
 ! CHECK:           %[[VAL_9:.*]] = fir.convert %[[VAL_8]] : (i1) -> !fir.logical<4>
 ! CHECK:           hlfir.assign %[[VAL_9]] to %[[VAL_4]]#0 : !fir.logical<4>, !fir.ref<!fir.logical<4>>
 ! CHECK:           omp.parallel {
-! CHECK:             %[[VAL_10:.*]] = fir.alloca i32 {bindc_name = "i", pinned, {{.*}}}
-! CHECK:             %[[VAL_11:.*]]:2 = hlfir.declare %[[VAL_10]] {uniq_name = "_QFsimple_reduction_switch_orderEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:             %[[VAL_12:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_13:.*]] = arith.constant 100 : i32
 ! CHECK:             %[[VAL_14:.*]] = arith.constant 1 : i32
-! CHECK:             omp.wsloop reduction(@or_reduction %[[VAL_4]]#0 -> %[[VAL_15:.*]] : !fir.ref<!fir.logical<4>>) {
+! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_10:.*]] : !fir.ref<i32>) reduction(@or_reduction %[[VAL_4]]#0 -> %[[VAL_15:.*]] : !fir.ref<!fir.logical<4>>) {
 ! CHECK-NEXT:          omp.loop_nest (%[[VAL_16:.*]]) : i32 = (%[[VAL_12]]) to (%[[VAL_13]]) inclusive step (%[[VAL_14]]) {
+! CHECK:                 %[[VAL_11:.*]]:2 = hlfir.declare %[[VAL_10]] {uniq_name = "_QFsimple_reduction_switch_orderEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:                 %[[VAL_17:.*]]:2 = hlfir.declare %[[VAL_15]] {uniq_name = "_QFsimple_reduction_switch_orderEx"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
 ! CHECK:                 fir.store %[[VAL_16]] to %[[VAL_11]]#1 : !fir.ref<i32>
 ! CHECK:                 %[[VAL_18:.*]] = fir.load %[[VAL_11]]#0 : !fir.ref<i32>
@@ -137,13 +135,12 @@ subroutine simple_reduction_switch_order(y)
 ! CHECK:           %[[VAL_17:.*]] = fir.convert %[[VAL_16]] : (i1) -> !fir.logical<4>
 ! CHECK:           hlfir.assign %[[VAL_17]] to %[[VAL_11]]#0 : !fir.logical<4>, !fir.ref<!fir.logical<4>>
 ! CHECK:           omp.parallel {
-! CHECK:             %[[VAL_18:.*]] = fir.alloca i32 {bindc_name = "i", pinned, {{.*}}}
-! CHECK:             %[[VAL_19:.*]]:2 = hlfir.declare %[[VAL_18]] {uniq_name = "_QFmultiple_reductionsEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:             %[[VAL_20:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_21:.*]] = arith.constant 100 : i32
 ! CHECK:             %[[VAL_22:.*]] = arith.constant 1 : i32
-! CHECK:             omp.wsloop reduction(@or_reduction %[[VAL_7]]#0 -> %[[VAL_23:.*]], @or_reduction %[[VAL_9]]#0 -> %[[VAL_24:.*]], @or_reduction %[[VAL_11]]#0 -> %[[VAL_25:.*]] : !fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>) {
+! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_18:.*]] : !fir.ref<i32>) reduction(@or_reduction %[[VAL_7]]#0 -> %[[VAL_23:.*]], @or_reduction %[[VAL_9]]#0 -> %[[VAL_24:.*]], @or_reduction %[[VAL_11]]#0 -> %[[VAL_25:.*]] : !fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>) {
 ! CHECK-NEXT:          omp.loop_nest (%[[VAL_26:.*]]) : i32 = (%[[VAL_20]]) to (%[[VAL_21]]) inclusive step (%[[VAL_22]]) {
+! CHECK:                 %[[VAL_19:.*]]:2 = hlfir.declare %[[VAL_18]] {uniq_name = "_QFmultiple_reductionsEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:                 %[[VAL_27:.*]]:2 = hlfir.declare %[[VAL_23]] {uniq_name = "_QFmultiple_reductionsEx"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
 ! CHECK:                 %[[VAL_28:.*]]:2 = hlfir.declare %[[VAL_24]] {uniq_name = "_QFmultiple_reductionsEy"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
 ! CHECK:                 %[[VAL_29:.*]]:2 = hlfir.declare %[[VAL_25]] {uniq_name = "_QFmultiple_reductionsEz"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
diff --git a/flang/test/Lower/OpenMP/wsloop-reduction-max-2-byref.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-max-2-byref.f90
index 28c70899e6ccc..0438e19f34391 100644
--- a/flang/test/Lower/OpenMP/wsloop-reduction-max-2-byref.f90
+++ b/flang/test/Lower/OpenMP/wsloop-reduction-max-2-byref.f90
@@ -1,7 +1,7 @@
 ! RUN: bbc -emit-hlfir -fopenmp --force-byref-reduction -o - %s 2>&1 | FileCheck %s
 ! RUN: %flang_fc1 -emit-hlfir -fopenmp -mmlir --force-byref-reduction -o - %s 2>&1 | FileCheck %s
 
-! CHECK: omp.wsloop reduction(byref @max_byref_i32
+! CHECK: omp.wsloop private({{.*}}) reduction(byref @max_byref_i32
 ! CHECK: arith.cmpi sgt
 ! CHECK: arith.select
 
diff --git a/flang/test/Lower/OpenMP/wsloop-reduction-max-2.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-max-2.f90
index abd7ca1ae555d..66c75bbe38f10 100644
--- a/flang/test/Lower/OpenMP/wsloop-reduction-max-2.f90
+++ b/flang/test/Lower/OpenMP/wsloop-reduction-max-2.f90
@@ -1,7 +1,7 @@
 ! RUN: bbc -emit-hlfir -fopenmp -o - %s 2>&1 | FileCheck %s
 ! RUN: %flang_fc1 -emit-hlfir -fopenmp -o - %s 2>&1 | FileCheck %s
 
-! CHECK: omp.wsloop reduction(@max_i32
+! CHECK: omp.wsloop private({{.*}}) reduction(@max_i32
 ! CHECK: arith.cmpi sgt
 ! CHECK: arith.select
 
diff --git a/flang/test/Lower/OpenMP/wsloop-reduction-max-byref.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-max-byref.f90
index 9abff8ccfa3b6..07c18f90480bf 100644
--- a/flang/test/Lower/OpenMP/wsloop-reduction-max-byref.f90
+++ b/flang/test/Lower/OpenMP/wsloop-reduction-max-byref.f90
@@ -45,13 +45,12 @@
 ! CHECK:           %[[VAL_6:.*]] = arith.constant 0 : i32
 ! CHECK:           hlfir.assign %[[VAL_6]] to %[[VAL_4]]#0 : i32, !fir.ref<i32>
 ! CHECK:           omp.parallel {
-! CHECK:             %[[VAL_7:.*]] = fir.alloca i32 {bindc_name = "i", pinned, {{.*}}}
-! CHECK:             %[[VAL_8:.*]]:2 = hlfir.declare %[[VAL_7]] {uniq_name = "_QFreduction_max_intEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:             %[[VAL_9:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_10:.*]] = arith.constant 100 : i32
 ! CHECK:             %[[VAL_11:.*]] = arith.constant 1 : i32
-! CHECK:             omp.wsloop reduction(byref @max_byref_i32 %[[VAL_4]]#0 -> %[[VAL_12:.*]] : !fir.ref<i32>) {
+! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_7:.*]] : !fir.ref<i32>) reduction(byref @max_byref_i32 %[[VAL_4]]#0 -> %[[VAL_12:.*]] : !fir.ref<i32>) {
 ! CHECK-NEXT:          omp.loop_nest (%[[VAL_13:.*]]) : i32 = (%[[VAL_9]]) to (%[[VAL_10]]) inclusive step (%[[VAL_11]]) {
+! CHECK:                 %[[VAL_8:.*]]:2 = hlfir.declare %[[VAL_7]] {uniq_name = "_QFreduction_max_intEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:                 %[[VAL_14:.*]]:2 = hlfir.declare %[[VAL_12]] {uniq_name = "_QFreduction_max_intEx"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:                 fir.store %[[VAL_13]] to %[[VAL_8]]#1 : !fir.ref<i32>
 ! CHECK:                 %[[VAL_15:.*]] = fir.load %[[VAL_8]]#0 : !fir.ref<i32>
@@ -75,13 +74,12 @@
 ! CHECK:           %[[VAL_6:.*]] = arith.constant 0.000000e+00 : f32
 ! CHECK:           hlfir.assign %[[VAL_6]] to %[[VAL_4]]#0 : f32, !fir.ref<f32>
 ! CHECK:           omp.parallel {
-! CHECK:             %[[VAL_7:.*]] = fir.alloca i32 {bindc_name = "i", pinned, {{.*}}}
-! CHECK:             %[[VAL_8:.*]]:2 = hlfir.declare %[[VAL_7]] {uniq_name = "_QFreduction_max_realEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:             %[[VAL_9:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_10:.*]] = arith.constant 100 : i32
 ! CHECK:             %[[VAL_11:.*]] = arith.constant 1 : i32
-! CHECK:             omp.wsloop reduction(byref @max_byref_f32 %[[VAL_4]]#0 -> %[[VAL_12:.*]] : !fir.ref<f32>) {
+! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_7:.*]] : !fir.ref<i32>) reduction(byref @max_byref_f32 %[[VAL_4]]#0 -> %[[VAL_12:.*]] : !fir.ref<f32>) {
 ! CHECK-NEXT:          omp.loop_nest (%[[VAL_13:.*]]) : i32 = (%[[VAL_9]]) to (%[[VAL_10]]) inclusive step (%[[VAL_11]]) {
+! CHECK:                 %[[VAL_8:.*]]:2 = hlfir.declare %[[VAL_7]] {uniq_name = "_QFreduction_max_realEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:                 %[[VAL_14:.*]]:2 = hlfir.declare %[[VAL_12]] {uniq_name = "_QFreduction_max_realEx"} : (!fir.ref<f32>) -> (!fir.ref<f32>, !fir.ref<f32>)
 ! CHECK:                 fir.store %[[VAL_13]] to %[[VAL_8]]#1 : !fir.ref<i32>
 ! CHECK:                 %[[VAL_15:.*]] = fir.load %[[VAL_8]]#0 : !fir.ref<i32>
@@ -95,13 +93,12 @@
 ! CHECK:                 omp.yield
 ! CHECK:             omp.terminator
 ! CHECK:           omp.parallel {
-! CHECK:             %[[VAL_30:.*]] = fir.alloca i32 {bindc_name = "i", pinned, {{.*}}}
-! CHECK:             %[[VAL_31:.*]]:2 = hlfir.declare %[[VAL_30]] {uniq_name = "_QFreduction_max_realEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:             %[[VAL_32:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_33:.*]] = arith.constant 100 : i32
 ! CHECK:             %[[VAL_34:.*]] = arith.constant 1 : i32
-! CHECK:             omp.wsloop reduction(byref @max_byref_f32 %[[VAL_4]]#0 -> %[[VAL_35:.*]] : !fir.ref<f32>) {
+! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_30:.*]] : !fir.ref<i32>) reduction(byref @max_byref_f32 %[[VAL_4]]#0 -> %[[VAL_35:.*]] : !fir.ref<f32>) {
 ! CHECK-NEXT:          omp.loop_nest (%[[VAL_36:.*]]) : i32 = (%[[VAL_32]]) to (%[[VAL_33]]) inclusive step (%[[VAL_34]]) {
+! CHECK:                 %[[VAL_31:.*]]:2 = hlfir.declare %[[VAL_30]] {uniq_name = "_QFreduction_max_realEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:                 %[[VAL_37:.*]]:2 = hlfir.declare %[[VAL_35]] {uniq_name = "_QFreduction_max_realEx"} : (!fir.ref<f32>) -> (!fir.ref<f32>, !fir.ref<f32>)
 ! CHECK:                 fir.store %[[VAL_36]] to %[[VAL_31]]#1 : !fir.ref<i32>
 ! CHECK:                 %[[VAL_38:.*]] = fir.load %[[VAL_31]]#0 : !fir.ref<i32>
diff --git a/flang/test/Lower/OpenMP/wsloop-reduction-max.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-max.f90
index 7237d3f903b74..7bdfa0948c747 100644
--- a/flang/test/Lower/OpenMP/wsloop-reduction-max.f90
+++ b/flang/test/Lower/OpenMP/wsloop-reduction-max.f90
@@ -35,13 +35,12 @@
 ! CHECK:           %[[VAL_6:.*]] = arith.constant 0 : i32
 ! CHECK:           hlfir.assign %[[VAL_6]] to %[[VAL_4]]#0 : i32, !fir.ref<i32>
 ! CHECK:           omp.parallel {
-! CHECK:             %[[VAL_7:.*]] = fir.alloca i32 {bindc_name = "i", pinned, {{.*}}}
-! CHECK:             %[[VAL_8:.*]]:2 = hlfir.declare %[[VAL_7]] {uniq_name = "_QFreduction_max_intEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:             %[[VAL_9:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_10:.*]] = arith.constant 100 : i32
 ! CHECK:             %[[VAL_11:.*]] = arith.constant 1 : i32
-! CHECK:             omp.wsloop reduction(@max_i32 %[[VAL_4]]#0 -> %[[VAL_12:.*]] : !fir.ref<i32>) {
+! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_7:.*]] : !fir.ref<i32>) reduction(@max_i32 %[[VAL_4]]#0 -> %[[VAL_12:.*]] : !fir.ref<i32>) {
 ! CHECK-NEXT:          omp.loop_nest (%[[VAL_13:.*]]) : i32 = (%[[VAL_9]]) to (%[[VAL_10]]) inclusive step (%[[VAL_11]]) {
+! CHECK:                 %[[VAL_8:.*]]:2 = hlfir.declare %[[VAL_7]] {uniq_name = "_QFreduction_max_intEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:                 %[[VAL_14:.*]]:2 = hlfir.declare %[[VAL_12]] {uniq_name = "_QFreduction_max_intEx"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:                 fir.store %[[VAL_13]] to %[[VAL_8]]#1 : !fir.ref<i32>
 ! CHECK:                 %[[VAL_15:.*]] = fir.load %[[VAL_8]]#0 : !fir.ref<i32>
@@ -65,13 +64,12 @@
 ! CHECK:           %[[VAL_6:.*]] = arith.constant 0.000000e+00 : f32
 ! CHECK:           hlfir.assign %[[VAL_6]] to %[[VAL_4]]#0 : f32, !fir.ref<f32>
 ! CHECK:           omp.parallel {
-! CHECK:             %[[VAL_7:.*]] = fir.alloca i32 {bindc_name = "i", pinned, {{.*}}}
-! CHECK:             %[[VAL_8:.*]]:2 = hlfir.declare %[[VAL_7]] {uniq_name = "_QFreduction_max_realEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:             %[[VAL_9:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_10:.*]] = arith.constant 100 : i32
 ! CHECK:             %[[VAL_11:.*]] = arith.constant 1 : i32
-! CHECK:             omp.wsloop reduction(@max_f32 %[[VAL_4]]#0 -> %[[VAL_12:.*]] : !fir.ref<f32>) {
+! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_7:.*]] : !fir.ref<i32>) reduction(@max_f32 %[[VAL_4]]#0 -> %[[VAL_12:.*]] : !fir.ref<f32>) {
 ! CHECK-NEXT:          omp.loop_nest (%[[VAL_13:.*]]) : i32 = (%[[VAL_9]]) to (%[[VAL_10]]) inclusive step (%[[VAL_11]]) {
+! CHECK:                 %[[VAL_8:.*]]:2 = hlfir.declare %[[VAL_7]] {uniq_name = "_QFreduction_max_realEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:                 %[[VAL_14:.*]]:2 = hlfir.declare %[[VAL_12]] {uniq_name = "_QFreduction_max_realEx"} : (!fir.ref<f32>) -> (!fir.ref<f32>, !fir.ref<f32>)
 ! CHECK:                 fir.store %[[VAL_13]] to %[[VAL_8]]#1 : !fir.ref<i32>
 ! CHECK:                 %[[VAL_15:.*]] = fir.load %[[VAL_8]]#0 : !fir.ref<i32>
@@ -85,13 +83,12 @@
 ! CHECK:                 omp.yield
 ! CHECK:             omp.terminator
 ! CHECK:           omp.parallel {
-! CHECK:             %[[VAL_30:.*]] = fir.alloca i32 {bindc_name = "i", pinned, {{.*}}}
-! CHECK:             %[[VAL_31:.*]]:2 = hlfir.declare %[[VAL_30]] {uniq_name = "_QFreduction_max_realEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:             %[[VAL_32:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_33:.*]] = arith.constant 100 : i32
 ! CHECK:             %[[VAL_34:.*]] = arith.constant 1 : i32
-! CHECK:             omp.wsloop reduction(@max_f32 %[[VAL_4]]#0 -> %[[VAL_35:.*]] : !fir.ref<f32>) {
+! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_30:.*]] : !fir.ref<i32>) reduction(@max_f32 %[[VAL_4]]#0 -> %[[VAL_35:.*]] : !fir.ref<f32>) {
 ! CHECK-NEXT:          omp.loop_nest (%[[VAL_36:.*]]) : i32 = (%[[VAL_32]]) to (%[[VAL_33]]) inclusive step (%[[VAL_34]]) {
+! CHECK:                 %[[VAL_31:.*]]:2 = hlfir.declare %[[VAL_30]] {uniq_name = "_QFreduction_max_realEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:                 %[[VAL_37:.*]]:2 = hlfir.declare %[[VAL_35]] {uniq_name = "_QFreduction_max_realEx"} : (!fir.ref<f32>) -> (!fir.ref<f32>, !fir.ref<f32>)
 ! CHECK:                 fir.store %[[VAL_36]] to %[[VAL_31]]#1 : !fir.ref<i32>
 ! CHECK:                 %[[VAL_38:.*]] = fir.load %[[VAL_31]]#0 : !fir.ref<i32>
diff --git a/flang/test/Lower/OpenMP/wsloop-reduction-min-byref.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-min-byref.f90
index a4bfbaa09d2fa..88a455f4b45ac 100644
--- a/flang/test/Lower/OpenMP/wsloop-reduction-min-byref.f90
+++ b/flang/test/Lower/OpenMP/wsloop-reduction-min-byref.f90
@@ -45,13 +45,12 @@
 ! CHECK:           %[[VAL_6:.*]] = arith.constant 0 : i32
 ! CHECK:           hlfir.assign %[[VAL_6]] to %[[VAL_4]]#0 : i32, !fir.ref<i32>
 ! CHECK:           omp.parallel {
-! CHECK:             %[[VAL_7:.*]] = fir.alloca i32 {bindc_name = "i", pinned, {{.*}}}
-! CHECK:             %[[VAL_8:.*]]:2 = hlfir.declare %[[VAL_7]] {uniq_name = "_QFreduction_min_intEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:             %[[VAL_9:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_10:.*]] = arith.constant 100 : i32
 ! CHECK:             %[[VAL_11:.*]] = arith.constant 1 : i32
-! CHECK:             omp.wsloop reduction(byref @min_byref_i32 %[[VAL_4]]#0 -> %[[VAL_12:.*]] : !fir.ref<i32>) {
+! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_7:.*]] : !fir.ref<i32>) reduction(byref @min_byref_i32 %[[VAL_4]]#0 -> %[[VAL_12:.*]] : !fir.ref<i32>) {
 ! CHECK-NEXT:          omp.loop_nest (%[[VAL_13:.*]]) : i32 = (%[[VAL_9]]) to (%[[VAL_10]]) inclusive step (%[[VAL_11]]) {
+! CHECK:                 %[[VAL_8:.*]]:2 = hlfir.declare %[[VAL_7]] {uniq_name = "_QFreduction_min_intEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:                 %[[VAL_14:.*]]:2 = hlfir.declare %[[VAL_12]] {uniq_name = "_QFreduction_min_intEx"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:                 fir.store %[[VAL_13]] to %[[VAL_8]]#1 : !fir.ref<i32>
 ! CHECK:                 %[[VAL_15:.*]] = fir.load %[[VAL_8]]#0 : !fir.ref<i32>
@@ -75,13 +74,12 @@
 ! CHECK:           %[[VAL_6:.*]] = arith.constant 0.000000e+00 : f32
 ! CHECK:           hlfir.assign %[[VAL_6]] to %[[VAL_4]]#0 : f32, !fir.ref<f32>
 ! CHECK:           omp.parallel {
-! CHECK:             %[[VAL_7:.*]] = fir.alloca i32 {bindc_name = "i", pinned, {{.*}}}
-! CHECK:             %[[VAL_8:.*]]:2 = hlfir.declare %[[VAL_7]] {uniq_name = "_QFreduction_min_realEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:             %[[VAL_9:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_10:.*]] = arith.constant 100 : i32
 ! CHECK:             %[[VAL_11:.*]] = arith.constant 1 : i32
-! CHECK:             omp.wsloop reduction(byref @min_byref_f32 %[[VAL_4]]#0 -> %[[VAL_12:.*]] : !fir.ref<f32>) {
+! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_7:.*]] : !fir.ref<i32>) reduction(byref @min_byref_f32 %[[VAL_4]]#0 -> %[[VAL_12:.*]] : !fir.ref<f32>) {
 ! CHECK-NEXT:          omp.loop_nest (%[[VAL_13:.*]]) : i32 = (%[[VAL_9]]) to (%[[VAL_10]]) inclusive step (%[[VAL_11]]) {
+! CHECK:                 %[[VAL_8:.*]]:2 = hlfir.declare %[[VAL_7]] {uniq_name = "_QFreduction_min_realEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:                 %[[VAL_14:.*]]:2 = hlfir.declare %[[VAL_12]] {uniq_name = "_QFreduction_min_realEx"} : (!fir.ref<f32>) -> (!fir.ref<f32>, !fir.ref<f32>)
 ! CHECK:                 fir.store %[[VAL_13]] to %[[VAL_8]]#1 : !fir.ref<i32>
 ! CHECK:                 %[[VAL_15:.*]] = fir.load %[[VAL_8]]#0 : !fir.ref<i32>
@@ -97,13 +95,12 @@
 ! CHECK:             omp.terminator
 ! CHECK:           }
 ! CHECK:           omp.parallel {
-! CHECK:             %[[VAL_30:.*]] = fir.alloca i32 {bindc_name = "i", pinned, {{.*}}}
-! CHECK:             %[[VAL_31:.*]]:2 = hlfir.declare %[[VAL_30]] {uniq_name = "_QFreduction_min_realEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:             %[[VAL_32:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_33:.*]] = arith.constant 100 : i32
 ! CHECK:             %[[VAL_34:.*]] = arith.constant 1 : i32
-! CHECK:             omp.wsloop reduction(byref @min_byref_f32 %[[VAL_4]]#0 -> %[[VAL_35:.*]] : !fir.ref<f32>) {
+! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_30:.*]] : !fir.ref<i32>) reduction(byref @min_byref_f32 %[[VAL_4]]#0 -> %[[VAL_35:.*]] : !fir.ref<f32>) {
 ! CHECK-NEXT:          omp.loop_nest (%[[VAL_36:.*]]) : i32 = (%[[VAL_32]]) to (%[[VAL_33]]) inclusive step (%[[VAL_34]]) {
+! CHECK:                 %[[VAL_31:.*]]:2 = hlfir.declare %[[VAL_30]] {uniq_name = "_QFreduction_min_realEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:                 %[[VAL_37:.*]]:2 = hlfir.declare %[[VAL_35]] {uniq_name = "_QFreduction_min_realEx"} : (!fir.ref<f32>) -> (!fir.ref<f32>, !fir.ref<f32>)
 ! CHECK:                 fir.store %[[VAL_36]] to %[[VAL_31]]#1 : !fir.ref<i32>
 ! CHECK:                 %[[VAL_38:.*]] = fir.load %[[VAL_31]]#0 : !fir.ref<i32>
diff --git a/flang/test/Lower/OpenMP/wsloop-reduction-min.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-min.f90
index ce9e53a17523c..6d4dcf1ab68eb 100644
--- a/flang/test/Lower/OpenMP/wsloop-reduction-min.f90
+++ b/flang/test/Lower/OpenMP/wsloop-reduction-min.f90
@@ -35,13 +35,12 @@
 ! CHECK:           %[[VAL_6:.*]] = arith.constant 0 : i32
 ! CHECK:           hlfir.assign %[[VAL_6]] to %[[VAL_4]]#0 : i32, !fir.ref<i32>
 ! CHECK:           omp.parallel {
-! CHECK:             %[[VAL_7:.*]] = fir.alloca i32 {bindc_name = "i", pinned, {{.*}}}
-! CHECK:             %[[VAL_8:.*]]:2 = hlfir.declare %[[VAL_7]] {uniq_name = "_QFreduction_min_intEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:             %[[VAL_9:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_10:.*]] = arith.constant 100 : i32
 ! CHECK:             %[[VAL_11:.*]] = arith.constant 1 : i32
-! CHECK:             omp.wsloop reduction(@min_i32 %[[VAL_4]]#0 -> %[[VAL_12:.*]] : !fir.ref<i32>) {
+! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_7:.*]] : !fir.ref<i32>) reduction(@min_i32 %[[VAL_4]]#0 -> %[[VAL_12:.*]] : !fir.ref<i32>) {
 ! CHECK-NEXT:          omp.loop_nest (%[[VAL_13:.*]]) : i32 = (%[[VAL_9]]) to (%[[VAL_10]]) inclusive step (%[[VAL_11]]) {
+! CHECK:                 %[[VAL_8:.*]]:2 = hlfir.declare %[[VAL_7]] {uniq_name = "_QFreduction_min_intEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:                 %[[VAL_14:.*]]:2 = hlfir.declare %[[VAL_12]] {uniq_name = "_QFreduction_min_intEx"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:                 fir.store %[[VAL_13]] to %[[VAL_8]]#1 : !fir.ref<i32>
 ! CHECK:                 %[[VAL_15:.*]] = fir.load %[[VAL_8]]#0 : !fir.ref<i32>
@@ -65,13 +64,12 @@
 ! CHECK:           %[[VAL_6:.*]] = arith.constant 0.000000e+00 : f32
 ! CHECK:           hlfir.assign %[[VAL_6]] to %[[VAL_4]]#0 : f32, !fir.ref<f32>
 ! CHECK:           omp.parallel {
-! CHECK:             %[[VAL_7:.*]] = fir.alloca i32 {bindc_name = "i", pinned, {{.*}}}
-! CHECK:             %[[VAL_8:.*]]:2 = hlfir.declare %[[VAL_7]] {uniq_name = "_QFreduction_min_realEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:             %[[VAL_9:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_10:.*]] = arith.constant 100 : i32
 ! CHECK:             %[[VAL_11:.*]] = arith.constant 1 : i32
-! CHECK:             omp.wsloop reduction(@min_f32 %[[VAL_4]]#0 -> %[[VAL_12:.*]] : !fir.ref<f32>) {
+! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_7:.*]] : !fir.ref<i32>) reduction(@min_f32 %[[VAL_4]]#0 -> %[[VAL_12:.*]] : !fir.ref<f32>) {
 ! CHECK-NEXT:          omp.loop_nest (%[[VAL_13:.*]]) : i32 = (%[[VAL_9]]) to (%[[VAL_10]]) inclusive step (%[[VAL_11]]) {
+! CHECK:                 %[[VAL_8:.*]]:2 = hlfir.declare %[[VAL_7]] {uniq_name = "_QFreduction_min_realEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:                 %[[VAL_14:.*]]:2 = hlfir.declare %[[VAL_12]] {uniq_name = "_QFreduction_min_realEx"} : (!fir.ref<f32>) -> (!fir.ref<f32>, !fir.ref<f32>)
 ! CHECK:                 fir.store %[[VAL_13]] to %[[VAL_8]]#1 : !fir.ref<i32>
 ! CHECK:                 %[[VAL_15:.*]] = fir.load %[[VAL_8]]#0 : !fir.ref<i32>
@@ -87,13 +85,12 @@
 ! CHECK:             omp.terminator
 ! CHECK:           }
 ! CHECK:           omp.parallel {
-! CHECK:             %[[VAL_30:.*]] = fir.alloca i32 {bindc_name = "i", pinned, {{.*}}}
-! CHECK:             %[[VAL_31:.*]]:2 = hlfir.declare %[[VAL_30]] {uniq_name = "_QFreduction_min_realEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:             %[[VAL_32:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_33:.*]] = arith.constant 100 : i32
 ! CHECK:             %[[VAL_34:.*]] = arith.constant 1 : i32
-! CHECK:             omp.wsloop reduction(@min_f32 %[[VAL_4]]#0 -> %[[VAL_35:.*]] : !fir.ref<f32>) {
+! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_30:.*]] : !fir.ref<i32>) reduction(@min_f32 %[[VAL_4]]#0 -> %[[VAL_35:.*]] : !fir.ref<f32>) {
 ! CHECK-NEXT:          omp.loop_nest (%[[VAL_36:.*]]) : i32 = (%[[VAL_32]]) to (%[[VAL_33]]) inclusive step (%[[VAL_34]]) {
+! CHECK:                 %[[VAL_31:.*]]:2 = hlfir.declare %[[VAL_30]] {uniq_name = "_QFreduction_min_realEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:                 %[[VAL_37:.*]]:2 = hlfir.declare %[[VAL_35]] {uniq_name = "_QFreduction_min_realEx"} : (!fir.ref<f32>) -> (!fir.ref<f32>, !fir.ref<f32>)
 ! CHECK:                 fir.store %[[VAL_36]] to %[[VAL_31]]#1 : !fir.ref<i32>
 ! CHECK:                 %[[VAL_38:.*]] = fir.load %[[VAL_31]]#0 : !fir.ref<i32>
diff --git a/flang/test/Lower/OpenMP/wsloop-reduction-min2.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-min2.f90
index d83ebb77af3eb..db8e59cb09dfa 100644
--- a/flang/test/Lower/OpenMP/wsloop-reduction-min2.f90
+++ b/flang/test/Lower/OpenMP/wsloop-reduction-min2.f90
@@ -34,13 +34,12 @@ program reduce
 ! CHECK:           %[[VAL_2:.*]] = fir.address_of(@_QFEr) : !fir.ref<i32>
 ! CHECK:           %[[VAL_3:.*]]:2 = hlfir.declare %[[VAL_2]] {uniq_name = "_QFEr"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:           omp.parallel {
-! CHECK:             %[[VAL_4:.*]] = fir.alloca i32 {bindc_name = "i", pinned, {{.*}}}
-! CHECK:             %[[VAL_5:.*]]:2 = hlfir.declare %[[VAL_4]] {uniq_name = "_QFEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:             %[[VAL_6:.*]] = arith.constant 0 : i32
 ! CHECK:             %[[VAL_7:.*]] = arith.constant 10 : i32
 ! CHECK:             %[[VAL_8:.*]] = arith.constant 1 : i32
-! CHECK:             omp.wsloop reduction(@min_i32 %[[VAL_3]]#0 -> %[[VAL_9:.*]] : !fir.ref<i32>) {
+! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_4:.*]] : !fir.ref<i32>) reduction(@min_i32 %[[VAL_3]]#0 -> %[[VAL_9:.*]] : !fir.ref<i32>) {
 ! CHECK-NEXT:          omp.loop_nest (%[[VAL_10:.*]]) : i32 = (%[[VAL_6]]) to (%[[VAL_7]]) inclusive step (%[[VAL_8]]) {
+! CHECK:                 %[[VAL_5:.*]]:2 = hlfir.declare %[[VAL_4]] {uniq_name = "_QFEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:                 %[[VAL_11:.*]]:2 = hlfir.declare %[[VAL_9]] {uniq_name = "_QFEr"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:                 fir.store %[[VAL_10]] to %[[VAL_5]]#1 : !fir.ref<i32>
 ! CHECK:                 %[[VAL_12:.*]] = fir.load %[[VAL_5]]#0 : !fir.ref<i32>
diff --git a/flang/test/Lower/OpenMP/wsloop-reduction-mul-byref.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-mul-byref.f90
index 18554fbb72aee..85df29e83f75d 100644
--- a/flang/test/Lower/OpenMP/wsloop-reduction-mul-byref.f90
+++ b/flang/test/Lower/OpenMP/wsloop-reduction-mul-byref.f90
@@ -88,13 +88,12 @@
 ! CHECK:           %[[VAL_4:.*]] = arith.constant 1 : i32
 ! CHECK:           hlfir.assign %[[VAL_4]] to %[[VAL_3]]#0 : i32, !fir.ref<i32>
 ! CHECK:           omp.parallel {
-! CHECK:             %[[VAL_5:.*]] = fir.alloca i32 {bindc_name = "i", pinned, {{.*}}}
-! CHECK:             %[[VAL_6:.*]]:2 = hlfir.declare %[[VAL_5]] {uniq_name = "_QFsimple_int_reductionEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:             %[[VAL_7:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_8:.*]] = arith.constant 10 : i32
 ! CHECK:             %[[VAL_9:.*]] = arith.constant 1 : i32
-! CHECK:             omp.wsloop reduction(byref @multiply_reduction_byref_i32 %[[VAL_3]]#0 -> %[[VAL_10:.*]] : !fir.ref<i32>) {
+! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_5:.*]] : !fir.ref<i32>) reduction(byref @multiply_reduction_byref_i32 %[[VAL_3]]#0 -> %[[VAL_10:.*]] : !fir.ref<i32>) {
 ! CHECK-NEXT:          omp.loop_nest (%[[VAL_11:.*]]) : i32 = (%[[VAL_7]]) to (%[[VAL_8]]) inclusive step (%[[VAL_9]]) {
+! CHECK:                 %[[VAL_6:.*]]:2 = hlfir.declare %[[VAL_5]] {uniq_name = "_QFsimple_int_reductionEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:                 %[[VAL_12:.*]]:2 = hlfir.declare %[[VAL_10]] {uniq_name = "_QFsimple_int_reductionEx"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:                 fir.store %[[VAL_11]] to %[[VAL_6]]#1 : !fir.ref<i32>
 ! CHECK:                 %[[VAL_13:.*]] = fir.load %[[VAL_12]]#0 : !fir.ref<i32>
@@ -125,13 +124,12 @@ subroutine simple_int_reduction
 ! CHECK:           %[[VAL_4:.*]] = arith.constant 1.000000e+00 : f32
 ! CHECK:           hlfir.assign %[[VAL_4]] to %[[VAL_3]]#0 : f32, !fir.ref<f32>
 ! CHECK:           omp.parallel {
-! CHECK:             %[[VAL_5:.*]] = fir.alloca i32 {bindc_name = "i", pinned, {{.*}}}
-! CHECK:             %[[VAL_6:.*]]:2 = hlfir.declare %[[VAL_5]] {uniq_name = "_QFsimple_real_reductionEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:             %[[VAL_7:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_8:.*]] = arith.constant 10 : i32
 ! CHECK:             %[[VAL_9:.*]] = arith.constant 1 : i32
-! CHECK:             omp.wsloop reduction(byref @multiply_reduction_byref_f32 %[[VAL_3]]#0 -> %[[VAL_10:.*]] : !fir.ref<f32>) {
+! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_5:.*]] : !fir.ref<i32>) reduction(byref @multiply_reduction_byref_f32 %[[VAL_3]]#0 -> %[[VAL_10:.*]] : !fir.ref<f32>) {
 ! CHECK-NEXT:          omp.loop_nest (%[[VAL_11:.*]]) : i32 = (%[[VAL_7]]) to (%[[VAL_8]]) inclusive step (%[[VAL_9]]) {
+! CHECK:                 %[[VAL_6:.*]]:2 = hlfir.declare %[[VAL_5]] {uniq_name = "_QFsimple_real_reductionEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:                 %[[VAL_12:.*]]:2 = hlfir.declare %[[VAL_10]] {uniq_name = "_QFsimple_real_reductionEx"} : (!fir.ref<f32>) -> (!fir.ref<f32>, !fir.ref<f32>)
 ! CHECK:                 fir.store %[[VAL_11]] to %[[VAL_6]]#1 : !fir.ref<i32>
 ! CHECK:                 %[[VAL_13:.*]] = fir.load %[[VAL_12]]#0 : !fir.ref<f32>
@@ -163,13 +161,12 @@ subroutine simple_real_reduction
 ! CHECK:           %[[VAL_4:.*]] = arith.constant 1 : i32
 ! CHECK:           hlfir.assign %[[VAL_4]] to %[[VAL_3]]#0 : i32, !fir.ref<i32>
 ! CHECK:           omp.parallel {
-! CHECK:             %[[VAL_5:.*]] = fir.alloca i32 {bindc_name = "i", pinned, {{.*}}}
-! CHECK:             %[[VAL_6:.*]]:2 = hlfir.declare %[[VAL_5]] {uniq_name = "_QFsimple_int_reduction_switch_orderEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:             %[[VAL_7:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_8:.*]] = arith.constant 10 : i32
 ! CHECK:             %[[VAL_9:.*]] = arith.constant 1 : i32
-! CHECK:             omp.wsloop reduction(byref @multiply_reduction_byref_i32 %[[VAL_3]]#0 -> %[[VAL_10:.*]] : !fir.ref<i32>) {
+! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_5:.*]] : !fir.ref<i32>) reduction(byref @multiply_reduction_byref_i32 %[[VAL_3]]#0 -> %[[VAL_10:.*]] : !fir.ref<i32>) {
 ! CHECK-NEXT:          omp.loop_nest (%[[VAL_11:.*]]) : i32 = (%[[VAL_7]]) to (%[[VAL_8]]) inclusive step (%[[VAL_9]]) {
+! CHECK:                 %[[VAL_6:.*]]:2 = hlfir.declare %[[VAL_5]] {uniq_name = "_QFsimple_int_reduction_switch_orderEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:                 %[[VAL_12:.*]]:2 = hlfir.declare %[[VAL_10]] {uniq_name = "_QFsimple_int_reduction_switch_orderEx"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:                 fir.store %[[VAL_11]] to %[[VAL_6]]#1 : !fir.ref<i32>
 ! CHECK:                 %[[VAL_13:.*]] = fir.load %[[VAL_6]]#0 : !fir.ref<i32>
@@ -200,13 +197,12 @@ subroutine simple_int_reduction_switch_order
 ! CHECK:           %[[VAL_4:.*]] = arith.constant 1.000000e+00 : f32
 ! CHECK:           hlfir.assign %[[VAL_4]] to %[[VAL_3]]#0 : f32, !fir.ref<f32>
 ! CHECK:           omp.parallel {
-! CHECK:             %[[VAL_5:.*]] = fir.alloca i32 {bindc_name = "i", pinned, {{.*}}}
-! CHECK:             %[[VAL_6:.*]]:2 = hlfir.declare %[[VAL_5]] {uniq_name = "_QFsimple_real_reduction_switch_orderEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:             %[[VAL_7:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_8:.*]] = arith.constant 10 : i32
 ! CHECK:             %[[VAL_9:.*]] = arith.constant 1 : i32
-! CHECK:             omp.wsloop reduction(byref @multiply_reduction_byref_f32 %[[VAL_3]]#0 -> %[[VAL_10:.*]] : !fir.ref<f32>) {
+! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_5:.*]] : !fir.ref<i32>) reduction(byref @multiply_reduction_byref_f32 %[[VAL_3]]#0 -> %[[VAL_10:.*]] : !fir.ref<f32>) {
 ! CHECK-NEXT:          omp.loop_nest (%[[VAL_11:.*]]) : i32 = (%[[VAL_7]]) to (%[[VAL_8]]) inclusive step (%[[VAL_9]]) {
+! CHECK:                 %[[VAL_6:.*]]:2 = hlfir.declare %[[VAL_5]] {uniq_name = "_QFsimple_real_reduction_switch_orderEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:                 %[[VAL_12:.*]]:2 = hlfir.declare %[[VAL_10]] {uniq_name = "_QFsimple_real_reduction_switch_orderEx"} : (!fir.ref<f32>) -> (!fir.ref<f32>, !fir.ref<f32>)
 ! CHECK:                 fir.store %[[VAL_11]] to %[[VAL_6]]#1 : !fir.ref<i32>
 ! CHECK:                 %[[VAL_13:.*]] = fir.load %[[VAL_6]]#0 : !fir.ref<i32>
@@ -246,13 +242,12 @@ subroutine simple_real_reduction_switch_order
 ! CHECK:           %[[VAL_10:.*]] = arith.constant 1 : i32
 ! CHECK:           hlfir.assign %[[VAL_10]] to %[[VAL_7]]#0 : i32, !fir.ref<i32>
 ! CHECK:           omp.parallel {
-! CHECK:             %[[VAL_11:.*]] = fir.alloca i32 {bindc_name = "i", pinned, {{.*}}}
-! CHECK:             %[[VAL_12:.*]]:2 = hlfir.declare %[[VAL_11]] {uniq_name = "_QFmultiple_int_reductions_same_typeEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:             %[[VAL_13:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_14:.*]] = arith.constant 10 : i32
 ! CHECK:             %[[VAL_15:.*]] = arith.constant 1 : i32
-! CHECK:             omp.wsloop reduction(byref @multiply_reduction_byref_i32 %[[VAL_3]]#0 -> %[[VAL_16:.*]], byref @multiply_reduction_byref_i32 %[[VAL_5]]#0 -> %[[VAL_17:.*]], byref @multiply_reduction_byref_i32 %[[VAL_7]]#0 -> %[[VAL_18:.*]] : !fir.ref<i32>, !fir.ref<i32>, !fir.ref<i32>) {
+! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_11:.*]] : !fir.ref<i32>) reduction(byref @multiply_reduction_byref_i32 %[[VAL_3]]#0 -> %[[VAL_16:.*]], byref @multiply_reduction_byref_i32 %[[VAL_5]]#0 -> %[[VAL_17:.*]], byref @multiply_reduction_byref_i32 %[[VAL_7]]#0 -> %[[VAL_18:.*]] : !fir.ref<i32>, !fir.ref<i32>, !fir.ref<i32>) {
 ! CHECK-NEXT:          omp.loop_nest (%[[VAL_19:.*]]) : i32 = (%[[VAL_13]]) to (%[[VAL_14]]) inclusive step (%[[VAL_15]]) {
+! CHECK:                 %[[VAL_12:.*]]:2 = hlfir.declare %[[VAL_11]] {uniq_name = "_QFmultiple_int_reductions_same_typeEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:                 %[[VAL_20:.*]]:2 = hlfir.declare %[[VAL_16]] {uniq_name = "_QFmultiple_int_reductions_same_typeEx"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:                 %[[VAL_21:.*]]:2 = hlfir.declare %[[VAL_17]] {uniq_name = "_QFmultiple_int_reductions_same_typeEy"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:                 %[[VAL_22:.*]]:2 = hlfir.declare %[[VAL_18]] {uniq_name = "_QFmultiple_int_reductions_same_typeEz"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
@@ -305,13 +300,12 @@ subroutine multiple_int_reductions_same_type
 ! CHECK:           %[[VAL_10:.*]] = arith.constant 1.000000e+00 : f32
 ! CHECK:           hlfir.assign %[[VAL_10]] to %[[VAL_7]]#0 : f32, !fir.ref<f32>
 ! CHECK:           omp.parallel {
-! CHECK:             %[[VAL_11:.*]] = fir.alloca i32 {bindc_name = "i", pinned, {{.*}}}
-! CHECK:             %[[VAL_12:.*]]:2 = hlfir.declare %[[VAL_11]] {uniq_name = "_QFmultiple_real_reductions_same_typeEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:             %[[VAL_13:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_14:.*]] = arith.constant 10 : i32
 ! CHECK:             %[[VAL_15:.*]] = arith.constant 1 : i32
-! CHECK:             omp.wsloop reduction(byref @multiply_reduction_byref_f32 %[[VAL_3]]#0 -> %[[VAL_16:.*]], byref @multiply_reduction_byref_f32 %[[VAL_5]]#0 -> %[[VAL_17:.*]], byref @multiply_reduction_byref_f32 %[[VAL_7]]#0 -> %[[VAL_18:.*]] : !fir.ref<f32>, !fir.ref<f32>, !fir.ref<f32>) {
+! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_11:.*]] : !fir.ref<i32>) reduction(byref @multiply_reduction_byref_f32 %[[VAL_3]]#0 -> %[[VAL_16:.*]], byref @multiply_reduction_byref_f32 %[[VAL_5]]#0 -> %[[VAL_17:.*]], byref @multiply_reduction_byref_f32 %[[VAL_7]]#0 -> %[[VAL_18:.*]] : !fir.ref<f32>, !fir.ref<f32>, !fir.ref<f32>) {
 ! CHECK-NEXT:          omp.loop_nest (%[[VAL_19:.*]]) : i32 = (%[[VAL_13]]) to (%[[VAL_14]]) inclusive step (%[[VAL_15]]) {
+! CHECK:                 %[[VAL_12:.*]]:2 = hlfir.declare %[[VAL_11]] {uniq_name = "_QFmultiple_real_reductions_same_typeEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:                 %[[VAL_20:.*]]:2 = hlfir.declare %[[VAL_16]] {uniq_name = "_QFmultiple_real_reductions_same_typeEx"} : (!fir.ref<f32>) -> (!fir.ref<f32>, !fir.ref<f32>)
 ! CHECK:                 %[[VAL_21:.*]]:2 = hlfir.declare %[[VAL_17]] {uniq_name = "_QFmultiple_real_reductions_same_typeEy"} : (!fir.ref<f32>) -> (!fir.ref<f32>, !fir.ref<f32>)
 ! CHECK:                 %[[VAL_22:.*]]:2 = hlfir.declare %[[VAL_18]] {uniq_name = "_QFmultiple_real_reductions_same_typeEz"} : (!fir.ref<f32>) -> (!fir.ref<f32>, !fir.ref<f32>)
@@ -371,13 +365,12 @@ subroutine multiple_real_reductions_same_type
 ! CHECK:           %[[VAL_13:.*]] = arith.constant 1.000000e+00 : f64
 ! CHECK:           hlfir.assign %[[VAL_13]] to %[[VAL_3]]#0 : f64, !fir.ref<f64>
 ! CHECK:           omp.parallel {
-! CHECK:             %[[VAL_14:.*]] = fir.alloca i32 {bindc_name = "i", pinned, {{.*}}}
-! CHECK:             %[[VAL_15:.*]]:2 = hlfir.declare %[[VAL_14]] {uniq_name = "_QFmultiple_reductions_different_typeEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:             %[[VAL_16:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_17:.*]] = arith.constant 10 : i32
 ! CHECK:             %[[VAL_18:.*]] = arith.constant 1 : i32
-! CHECK:             omp.wsloop reduction(byref @multiply_reduction_byref_i32 %[[VAL_5]]#0 -> %[[VAL_19:.*]], byref @multiply_reduction_byref_i64 %[[VAL_7]]#0 -> %[[VAL_20:.*]], byref @multiply_reduction_byref_f32 %[[VAL_9]]#0 -> %[[VAL_21:.*]], byref @multiply_reduction_byref_f64 %[[VAL_3]]#0 -> %[[VAL_22:.*]] : !fir.ref<i32>, !fir.ref<i64>, !fir.ref<f32>, !fir.ref<f64>) {
+! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_14:.*]] : !fir.ref<i32>) reduction(byref @multiply_reduction_byref_i32 %[[VAL_5]]#0 -> %[[VAL_19:.*]], byref @multiply_reduction_byref_i64 %[[VAL_7]]#0 -> %[[VAL_20:.*]], byref @multiply_reduction_byref_f32 %[[VAL_9]]#0 -> %[[VAL_21:.*]], byref @multiply_reduction_byref_f64 %[[VAL_3]]#0 -> %[[VAL_22:.*]] : !fir.ref<i32>, !fir.ref<i64>, !fir.ref<f32>, !fir.ref<f64>) {
 ! CHECK-NEXT:          omp.loop_nest (%[[VAL_23:.*]]) : i32 = (%[[VAL_16]]) to (%[[VAL_17]]) inclusive step (%[[VAL_18]]) {
+! CHECK:                 %[[VAL_15:.*]]:2 = hlfir.declare %[[VAL_14]] {uniq_name = "_QFmultiple_reductions_different_typeEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:                 %[[VAL_24:.*]]:2 = hlfir.declare %[[VAL_19]] {uniq_name = "_QFmultiple_reductions_different_typeEx"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:                 %[[VAL_25:.*]]:2 = hlfir.declare %[[VAL_20]] {uniq_name = "_QFmultiple_reductions_different_typeEy"} : (!fir.ref<i64>) -> (!fir.ref<i64>, !fir.ref<i64>)
 ! CHECK:                 %[[VAL_26:.*]]:2 = hlfir.declare %[[VAL_21]] {uniq_name = "_QFmultiple_reductions_different_typeEz"} : (!fir.ref<f32>) -> (!fir.ref<f32>, !fir.ref<f32>)
diff --git a/flang/test/Lower/OpenMP/wsloop-reduction-mul.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-mul.f90
index f5c12ccf61f76..09c44f187f4a2 100644
--- a/flang/test/Lower/OpenMP/wsloop-reduction-mul.f90
+++ b/flang/test/Lower/OpenMP/wsloop-reduction-mul.f90
@@ -55,13 +55,12 @@
 ! CHECK:           %[[VAL_4:.*]] = arith.constant 1 : i32
 ! CHECK:           hlfir.assign %[[VAL_4]] to %[[VAL_3]]#0 : i32, !fir.ref<i32>
 ! CHECK:           omp.parallel {
-! CHECK:             %[[VAL_5:.*]] = fir.alloca i32 {bindc_name = "i", pinned, {{.*}}}
-! CHECK:             %[[VAL_6:.*]]:2 = hlfir.declare %[[VAL_5]] {uniq_name = "_QFsimple_int_reductionEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:             %[[VAL_7:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_8:.*]] = arith.constant 10 : i32
 ! CHECK:             %[[VAL_9:.*]] = arith.constant 1 : i32
-! CHECK:             omp.wsloop reduction(@multiply_reduction_i32 %[[VAL_3]]#0 -> %[[VAL_10:.*]] : !fir.ref<i32>) {
+! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_5:.*]] : !fir.ref<i32>) reduction(@multiply_reduction_i32 %[[VAL_3]]#0 -> %[[VAL_10:.*]] : !fir.ref<i32>) {
 ! CHECK-NEXT:          omp.loop_nest (%[[VAL_11:.*]]) : i32 = (%[[VAL_7]]) to (%[[VAL_8]]) inclusive step (%[[VAL_9]]) {
+! CHECK:                 %[[VAL_6:.*]]:2 = hlfir.declare %[[VAL_5]] {uniq_name = "_QFsimple_int_reductionEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:                 %[[VAL_12:.*]]:2 = hlfir.declare %[[VAL_10]] {uniq_name = "_QFsimple_int_reductionEx"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:                 fir.store %[[VAL_11]] to %[[VAL_6]]#1 : !fir.ref<i32>
 ! CHECK:                 %[[VAL_13:.*]] = fir.load %[[VAL_12]]#0 : !fir.ref<i32>
@@ -92,13 +91,12 @@ subroutine simple_int_reduction
 ! CHECK:           %[[VAL_4:.*]] = arith.constant 1.000000e+00 : f32
 ! CHECK:           hlfir.assign %[[VAL_4]] to %[[VAL_3]]#0 : f32, !fir.ref<f32>
 ! CHECK:           omp.parallel {
-! CHECK:             %[[VAL_5:.*]] = fir.alloca i32 {bindc_name = "i", pinned, {{.*}}}
-! CHECK:             %[[VAL_6:.*]]:2 = hlfir.declare %[[VAL_5]] {uniq_name = "_QFsimple_real_reductionEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:             %[[VAL_7:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_8:.*]] = arith.constant 10 : i32
 ! CHECK:             %[[VAL_9:.*]] = arith.constant 1 : i32
-! CHECK:             omp.wsloop reduction(@multiply_reduction_f32 %[[VAL_3]]#0 -> %[[VAL_10:.*]] : !fir.ref<f32>) {
+! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_5:.*]] : !fir.ref<i32>) reduction(@multiply_reduction_f32 %[[VAL_3]]#0 -> %[[VAL_10:.*]] : !fir.ref<f32>) {
 ! CHECK-NEXT:          omp.loop_nest (%[[VAL_11:.*]]) : i32 = (%[[VAL_7]]) to (%[[VAL_8]]) inclusive step (%[[VAL_9]]) {
+! CHECK:                 %[[VAL_6:.*]]:2 = hlfir.declare %[[VAL_5]] {uniq_name = "_QFsimple_real_reductionEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:                 %[[VAL_12:.*]]:2 = hlfir.declare %[[VAL_10]] {uniq_name = "_QFsimple_real_reductionEx"} : (!fir.ref<f32>) -> (!fir.ref<f32>, !fir.ref<f32>)
 ! CHECK:                 fir.store %[[VAL_11]] to %[[VAL_6]]#1 : !fir.ref<i32>
 ! CHECK:                 %[[VAL_13:.*]] = fir.load %[[VAL_12]]#0 : !fir.ref<f32>
@@ -130,13 +128,12 @@ subroutine simple_real_reduction
 ! CHECK:           %[[VAL_4:.*]] = arith.constant 1 : i32
 ! CHECK:           hlfir.assign %[[VAL_4]] to %[[VAL_3]]#0 : i32, !fir.ref<i32>
 ! CHECK:           omp.parallel {
-! CHECK:             %[[VAL_5:.*]] = fir.alloca i32 {bindc_name = "i", pinned, {{.*}}}
-! CHECK:             %[[VAL_6:.*]]:2 = hlfir.declare %[[VAL_5]] {uniq_name = "_QFsimple_int_reduction_switch_orderEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:             %[[VAL_7:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_8:.*]] = arith.constant 10 : i32
 ! CHECK:             %[[VAL_9:.*]] = arith.constant 1 : i32
-! CHECK:             omp.wsloop reduction(@multiply_reduction_i32 %[[VAL_3]]#0 -> %[[VAL_10:.*]] : !fir.ref<i32>) {
+! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_5:.*]] : !fir.ref<i32>) reduction(@multiply_reduction_i32 %[[VAL_3]]#0 -> %[[VAL_10:.*]] : !fir.ref<i32>) {
 ! CHECK-NEXT:          omp.loop_nest (%[[VAL_11:.*]]) : i32 = (%[[VAL_7]]) to (%[[VAL_8]]) inclusive step (%[[VAL_9]]) {
+! CHECK:                 %[[VAL_6:.*]]:2 = hlfir.declare %[[VAL_5]] {uniq_name = "_QFsimple_int_reduction_switch_orderEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:                 %[[VAL_12:.*]]:2 = hlfir.declare %[[VAL_10]] {uniq_name = "_QFsimple_int_reduction_switch_orderEx"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:                 fir.store %[[VAL_11]] to %[[VAL_6]]#1 : !fir.ref<i32>
 ! CHECK:                 %[[VAL_13:.*]] = fir.load %[[VAL_6]]#0 : !fir.ref<i32>
@@ -167,13 +164,12 @@ subroutine simple_int_reduction_switch_order
 ! CHECK:           %[[VAL_4:.*]] = arith.constant 1.000000e+00 : f32
 ! CHECK:           hlfir.assign %[[VAL_4]] to %[[VAL_3]]#0 : f32, !fir.ref<f32>
 ! CHECK:           omp.parallel {
-! CHECK:             %[[VAL_5:.*]] = fir.alloca i32 {bindc_name = "i", pinned, {{.*}}}
-! CHECK:             %[[VAL_6:.*]]:2 = hlfir.declare %[[VAL_5]] {uniq_name = "_QFsimple_real_reduction_switch_orderEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:             %[[VAL_7:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_8:.*]] = arith.constant 10 : i32
 ! CHECK:             %[[VAL_9:.*]] = arith.constant 1 : i32
-! CHECK:             omp.wsloop reduction(@multiply_reduction_f32 %[[VAL_3]]#0 -> %[[VAL_10:.*]] : !fir.ref<f32>) {
+! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_5:.*]] : !fir.ref<i32>) reduction(@multiply_reduction_f32 %[[VAL_3]]#0 -> %[[VAL_10:.*]] : !fir.ref<f32>) {
 ! CHECK-NEXT:          omp.loop_nest (%[[VAL_11:.*]]) : i32 = (%[[VAL_7]]) to (%[[VAL_8]]) inclusive step (%[[VAL_9]]) {
+! CHECK:                 %[[VAL_6:.*]]:2 = hlfir.declare %[[VAL_5]] {uniq_name = "_QFsimple_real_reduction_switch_orderEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:                 %[[VAL_12:.*]]:2 = hlfir.declare %[[VAL_10]] {uniq_name = "_QFsimple_real_reduction_switch_orderEx"} : (!fir.ref<f32>) -> (!fir.ref<f32>, !fir.ref<f32>)
 ! CHECK:                 fir.store %[[VAL_11]] to %[[VAL_6]]#1 : !fir.ref<i32>
 ! CHECK:                 %[[VAL_13:.*]] = fir.load %[[VAL_6]]#0 : !fir.ref<i32>
@@ -213,13 +209,12 @@ subroutine simple_real_reduction_switch_order
 ! CHECK:           %[[VAL_10:.*]] = arith.constant 1 : i32
 ! CHECK:           hlfir.assign %[[VAL_10]] to %[[VAL_7]]#0 : i32, !fir.ref<i32>
 ! CHECK:           omp.parallel {
-! CHECK:             %[[VAL_11:.*]] = fir.alloca i32 {bindc_name = "i", pinned, {{.*}}}
-! CHECK:             %[[VAL_12:.*]]:2 = hlfir.declare %[[VAL_11]] {uniq_name = "_QFmultiple_int_reductions_same_typeEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:             %[[VAL_13:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_14:.*]] = arith.constant 10 : i32
 ! CHECK:             %[[VAL_15:.*]] = arith.constant 1 : i32
-! CHECK:             omp.wsloop reduction(@multiply_reduction_i32 %[[VAL_3]]#0 -> %[[VAL_16:.*]], @multiply_reduction_i32 %[[VAL_5]]#0 -> %[[VAL_17:.*]], @multiply_reduction_i32 %[[VAL_7]]#0 -> %[[VAL_18:.*]] : !fir.ref<i32>, !fir.ref<i32>, !fir.ref<i32>) {
+! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_11:.*]] : !fir.ref<i32>) reduction(@multiply_reduction_i32 %[[VAL_3]]#0 -> %[[VAL_16:.*]], @multiply_reduction_i32 %[[VAL_5]]#0 -> %[[VAL_17:.*]], @multiply_reduction_i32 %[[VAL_7]]#0 -> %[[VAL_18:.*]] : !fir.ref<i32>, !fir.ref<i32>, !fir.ref<i32>) {
 ! CHECK-NEXT:          omp.loop_nest (%[[VAL_19:.*]]) : i32 = (%[[VAL_13]]) to (%[[VAL_14]]) inclusive step (%[[VAL_15]]) {
+! CHECK:                 %[[VAL_12:.*]]:2 = hlfir.declare %[[VAL_11]] {uniq_name = "_QFmultiple_int_reductions_same_typeEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:                 %[[VAL_20:.*]]:2 = hlfir.declare %[[VAL_16]] {uniq_name = "_QFmultiple_int_reductions_same_typeEx"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:                 %[[VAL_21:.*]]:2 = hlfir.declare %[[VAL_17]] {uniq_name = "_QFmultiple_int_reductions_same_typeEy"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:                 %[[VAL_22:.*]]:2 = hlfir.declare %[[VAL_18]] {uniq_name = "_QFmultiple_int_reductions_same_typeEz"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
@@ -272,13 +267,12 @@ subroutine multiple_int_reductions_same_type
 ! CHECK:           %[[VAL_10:.*]] = arith.constant 1.000000e+00 : f32
 ! CHECK:           hlfir.assign %[[VAL_10]] to %[[VAL_7]]#0 : f32, !fir.ref<f32>
 ! CHECK:           omp.parallel {
-! CHECK:             %[[VAL_11:.*]] = fir.alloca i32 {bindc_name = "i", pinned, {{.*}}}
-! CHECK:             %[[VAL_12:.*]]:2 = hlfir.declare %[[VAL_11]] {uniq_name = "_QFmultiple_real_reductions_same_typeEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:             %[[VAL_13:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_14:.*]] = arith.constant 10 : i32
 ! CHECK:             %[[VAL_15:.*]] = arith.constant 1 : i32
-! CHECK:             omp.wsloop reduction(@multiply_reduction_f32 %[[VAL_3]]#0 -> %[[VAL_16:.*]], @multiply_reduction_f32 %[[VAL_5]]#0 -> %[[VAL_17:.*]], @multiply_reduction_f32 %[[VAL_7]]#0 -> %[[VAL_18:.*]] : !fir.ref<f32>, !fir.ref<f32>, !fir.ref<f32>) {
+! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_11:.*]] : !fir.ref<i32>) reduction(@multiply_reduction_f32 %[[VAL_3]]#0 -> %[[VAL_16:.*]], @multiply_reduction_f32 %[[VAL_5]]#0 -> %[[VAL_17:.*]], @multiply_reduction_f32 %[[VAL_7]]#0 -> %[[VAL_18:.*]] : !fir.ref<f32>, !fir.ref<f32>, !fir.ref<f32>) {
 ! CHECK-NEXT:          omp.loop_nest (%[[VAL_19:.*]]) : i32 = (%[[VAL_13]]) to (%[[VAL_14]]) inclusive step (%[[VAL_15]]) {
+! CHECK:                 %[[VAL_12:.*]]:2 = hlfir.declare %[[VAL_11]] {uniq_name = "_QFmultiple_real_reductions_same_typeEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:                 %[[VAL_20:.*]]:2 = hlfir.declare %[[VAL_16]] {uniq_name = "_QFmultiple_real_reductions_same_typeEx"} : (!fir.ref<f32>) -> (!fir.ref<f32>, !fir.ref<f32>)
 ! CHECK:                 %[[VAL_21:.*]]:2 = hlfir.declare %[[VAL_17]] {uniq_name = "_QFmultiple_real_reductions_same_typeEy"} : (!fir.ref<f32>) -> (!fir.ref<f32>, !fir.ref<f32>)
 ! CHECK:                 %[[VAL_22:.*]]:2 = hlfir.declare %[[VAL_18]] {uniq_name = "_QFmultiple_real_reductions_same_typeEz"} : (!fir.ref<f32>) -> (!fir.ref<f32>, !fir.ref<f32>)
@@ -338,13 +332,12 @@ subroutine multiple_real_reductions_same_type
 ! CHECK:           %[[VAL_13:.*]] = arith.constant 1.000000e+00 : f64
 ! CHECK:           hlfir.assign %[[VAL_13]] to %[[VAL_3]]#0 : f64, !fir.ref<f64>
 ! CHECK:           omp.parallel {
-! CHECK:             %[[VAL_14:.*]] = fir.alloca i32 {bindc_name = "i", pinned, {{.*}}}
-! CHECK:             %[[VAL_15:.*]]:2 = hlfir.declare %[[VAL_14]] {uniq_name = "_QFmultiple_reductions_different_typeEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:             %[[VAL_16:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_17:.*]] = arith.constant 10 : i32
 ! CHECK:             %[[VAL_18:.*]] = arith.constant 1 : i32
-! CHECK:             omp.wsloop reduction(@multiply_reduction_i32 %[[VAL_5]]#0 -> %[[VAL_19:.*]], @multiply_reduction_i64 %[[VAL_7]]#0 -> %[[VAL_20:.*]], @multiply_reduction_f32 %[[VAL_9]]#0 -> %[[VAL_21:.*]], @multiply_reduction_f64 %[[VAL_3]]#0 -> %[[VAL_22:.*]] : !fir.ref<i32>, !fir.ref<i64>, !fir.ref<f32>, !fir.ref<f64>) {
+! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_14:.*]] : !fir.ref<i32>) reduction(@multiply_reduction_i32 %[[VAL_5]]#0 -> %[[VAL_19:.*]], @multiply_reduction_i64 %[[VAL_7]]#0 -> %[[VAL_20:.*]], @multiply_reduction_f32 %[[VAL_9]]#0 -> %[[VAL_21:.*]], @multiply_reduction_f64 %[[VAL_3]]#0 -> %[[VAL_22:.*]] : !fir.ref<i32>, !fir.ref<i64>, !fir.ref<f32>, !fir.ref<f64>) {
 ! CHECK-NEXT:          omp.loop_nest (%[[VAL_23:.*]]) : i32 = (%[[VAL_16]]) to (%[[VAL_17]]) inclusive step (%[[VAL_18]]) {
+! CHECK:                 %[[VAL_15:.*]]:2 = hlfir.declare %[[VAL_14]] {uniq_name = "_QFmultiple_reductions_different_typeEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:                 %[[VAL_24:.*]]:2 = hlfir.declare %[[VAL_19]] {uniq_name = "_QFmultiple_reductions_different_typeEx"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:                 %[[VAL_25:.*]]:2 = hlfir.declare %[[VAL_20]] {uniq_name = "_QFmultiple_reductions_different_typeEy"} : (!fir.ref<i64>) -> (!fir.ref<i64>, !fir.ref<i64>)
 ! CHECK:                 %[[VAL_26:.*]]:2 = hlfir.declare %[[VAL_21]] {uniq_name = "_QFmultiple_reductions_different_typeEz"} : (!fir.ref<f32>) -> (!fir.ref<f32>, !fir.ref<f32>)
diff --git a/flang/test/Lower/OpenMP/wsloop-reduction-multi.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-multi.f90
index 659ba06005670..66229259adf82 100644
--- a/flang/test/Lower/OpenMP/wsloop-reduction-multi.f90
+++ b/flang/test/Lower/OpenMP/wsloop-reduction-multi.f90
@@ -41,7 +41,7 @@
 !CHECK:      %[[Y_DECL:.*]]:2 = hlfir.declare %[[Y_REF]] {uniq_name = "_QFmultiple_reductionEy"} : (!fir.ref<f32>) -> (!fir.ref<f32>, !fir.ref<f32>)
 !CHECK:      %[[Z_REF:.*]] = fir.alloca i32 {bindc_name = "z", uniq_name = "_QFmultiple_reductionEz"}
 !CHECK:      %[[Z_DECL:.*]]:2 = hlfir.declare %[[Z_REF]] {uniq_name = "_QFmultiple_reductionEz"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-!CHECK:      omp.wsloop reduction(
+!CHECK:      omp.wsloop private({{.*}}) reduction(
 !CHECK-SAME: @[[ADD_RED_I32_NAME]] %[[X_DECL]]#0 -> %[[PRV_X:[^,]+]],
 !CHECK-SAME: @[[ADD_RED_F32_NAME]] %[[Y_DECL]]#0 -> %[[PRV_Y:[^,]+]],
 !CHECK-SAME: @[[MIN_RED_I32_NAME]] %[[Z_DECL]]#0 -> %[[PRV_Z:.+]] :
diff --git a/flang/test/Lower/OpenMP/wsloop-reduction-multiple-clauses.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-multiple-clauses.f90
index 5b6ab095b45b6..75773416e4840 100644
--- a/flang/test/Lower/OpenMP/wsloop-reduction-multiple-clauses.f90
+++ b/flang/test/Lower/OpenMP/wsloop-reduction-multiple-clauses.f90
@@ -112,13 +112,12 @@ program main
 ! CHECK:             %[[VAL_11:.*]] = fir.embox %[[VAL_4]]#0(%[[VAL_3]]) : (!fir.ref<!fir.array<3x3xf64>>, !fir.shape<2>) -> !fir.box<!fir.array<3x3xf64>>
 ! CHECK:             %[[VAL_12:.*]] = fir.alloca !fir.box<!fir.array<3x3xf64>>
 ! CHECK:             fir.store %[[VAL_11]] to %[[VAL_12]] : !fir.ref<!fir.box<!fir.array<3x3xf64>>>
-! CHECK:             %[[VAL_13:.*]] = fir.alloca i32 {bindc_name = "i", pinned, {{.*}}}
-! CHECK:             %[[VAL_14:.*]]:2 = hlfir.declare %[[VAL_13]] {uniq_name = "_QFEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:             %[[VAL_15:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_16:.*]] = arith.constant 10 : i32
 ! CHECK:             %[[VAL_17:.*]] = arith.constant 1 : i32
-! CHECK:             omp.wsloop reduction(@add_reduction_f64 %[[VAL_8]]#0 -> %[[VAL_18:.*]], byref @add_reduction_byref_box_3x3xf64 %[[VAL_12]] -> %[[VAL_19:.*]] : !fir.ref<f64>, !fir.ref<!fir.box<!fir.array<3x3xf64>>>) {
+! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_13:.*]] : !fir.ref<i32>) reduction(@add_reduction_f64 %[[VAL_8]]#0 -> %[[VAL_18:.*]], byref @add_reduction_byref_box_3x3xf64 %[[VAL_12]] -> %[[VAL_19:.*]] : !fir.ref<f64>, !fir.ref<!fir.box<!fir.array<3x3xf64>>>) {
 ! CHECK:               omp.loop_nest (%[[VAL_20:.*]]) : i32 = (%[[VAL_15]]) to (%[[VAL_16]]) inclusive step (%[[VAL_17]]) {
+! CHECK:                 %[[VAL_14:.*]]:2 = hlfir.declare %[[VAL_13]] {uniq_name = "_QFEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:                 %[[VAL_21:.*]]:2 = hlfir.declare %[[VAL_18]] {uniq_name = "_QFEscalar"} : (!fir.ref<f64>) -> (!fir.ref<f64>, !fir.ref<f64>)
 ! CHECK:                 %[[VAL_22:.*]]:2 = hlfir.declare %[[VAL_19]] {uniq_name = "_QFEarray"} : (!fir.ref<!fir.box<!fir.array<3x3xf64>>>) -> (!fir.ref<!fir.box<!fir.array<3x3xf64>>>, !fir.ref<!fir.box<!fir.array<3x3xf64>>>)
 ! CHECK:                 fir.store %[[VAL_20]] to %[[VAL_14]]#1 : !fir.ref<i32>
diff --git a/flang/test/Lower/OpenMP/wsloop-reduction-pointer.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-pointer.f90
index 2c126bb8962c2..f706e48b8fda8 100644
--- a/flang/test/Lower/OpenMP/wsloop-reduction-pointer.f90
+++ b/flang/test/Lower/OpenMP/wsloop-reduction-pointer.f90
@@ -87,13 +87,12 @@ program reduce_pointer
 ! CHECK:           %[[VAL_17:.*]] = fir.box_addr %[[VAL_16]] : (!fir.box<!fir.ptr<i32>>) -> !fir.ptr<i32>
 ! CHECK:           hlfir.assign %[[VAL_15]] to %[[VAL_17]] : i32, !fir.ptr<i32>
 ! CHECK:           omp.parallel {
-! CHECK:             %[[VAL_18:.*]] = fir.alloca i32 {bindc_name = "i", pinned, uniq_name = "_QFEi"}
-! CHECK:             %[[VAL_19:.*]]:2 = hlfir.declare %[[VAL_18]] {uniq_name = "_QFEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:             %[[VAL_20:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_21:.*]] = arith.constant 5 : i32
 ! CHECK:             %[[VAL_22:.*]] = arith.constant 1 : i32
-! CHECK:             omp.wsloop reduction(byref @add_reduction_byref_box_ptr_i32 %[[VAL_5]]#0 -> %[[VAL_23:.*]] : !fir.ref<!fir.box<!fir.ptr<i32>>>) {
+! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_18:.*]] : !fir.ref<i32>) reduction(byref @add_reduction_byref_box_ptr_i32 %[[VAL_5]]#0 -> %[[VAL_23:.*]] : !fir.ref<!fir.box<!fir.ptr<i32>>>) {
 ! CHECK:               omp.loop_nest (%[[VAL_24:.*]]) : i32 = (%[[VAL_20]]) to (%[[VAL_21]]) inclusive step (%[[VAL_22]]) {
+! CHECK:                 %[[VAL_19:.*]]:2 = hlfir.declare %[[VAL_18]] {uniq_name = "_QFEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:                 %[[VAL_25:.*]]:2 = hlfir.declare %[[VAL_23]] {fortran_attrs = {{.*}}<pointer>, uniq_name = "_QFEv"} : (!fir.ref<!fir.box<!fir.ptr<i32>>>) -> (!fir.ref<!fir.box<!fir.ptr<i32>>>, !fir.ref<!fir.box<!fir.ptr<i32>>>)
 ! CHECK:                 fir.store %[[VAL_24]] to %[[VAL_19]]#1 : !fir.ref<i32>
 ! CHECK:                 %[[VAL_26:.*]] = fir.load %[[VAL_25]]#0 : !fir.ref<!fir.box<!fir.ptr<i32>>>
diff --git a/flang/test/Lower/OpenMP/wsloop-schedule.f90 b/flang/test/Lower/OpenMP/wsloop-schedule.f90
index ae854a2de0c9d..0ff4ce7c3ede3 100644
--- a/flang/test/Lower/OpenMP/wsloop-schedule.f90
+++ b/flang/test/Lower/OpenMP/wsloop-schedule.f90
@@ -14,7 +14,7 @@ program wsloop_dynamic
 !CHECK:      %[[WS_LB:.*]] = arith.constant 1 : i32
 !CHECK:      %[[WS_UB:.*]] = arith.constant 9 : i32
 !CHECK:      %[[WS_STEP:.*]] = arith.constant 1 : i32
-!CHECK:      omp.wsloop nowait schedule(runtime, simd) {
+!CHECK:      omp.wsloop nowait schedule(runtime, simd) private({{.*}}) {
 !CHECK-NEXT:   omp.loop_nest (%[[I:.*]]) : i32 = (%[[WS_LB]]) to (%[[WS_UB]]) inclusive step (%[[WS_STEP]]) {
 !CHECK:          fir.store %[[I]] to %[[STORE:.*]]#1 : !fir.ref<i32>
 
diff --git a/flang/test/Lower/OpenMP/wsloop-unstructured.f90 b/flang/test/Lower/OpenMP/wsloop-unstructured.f90
index 8c89f863ab877..6174718c08758 100644
--- a/flang/test/Lower/OpenMP/wsloop-unstructured.f90
+++ b/flang/test/Lower/OpenMP/wsloop-unstructured.f90
@@ -29,7 +29,7 @@ end subroutine sub
 ! CHECK-SAME:                      %[[VAL_2:.*]]: !fir.ref<!fir.array<?x?xf32>> {fir.bindc_name = "x"},
 ! CHECK-SAME:                      %[[VAL_3:.*]]: !fir.ref<!fir.array<?x?xf32>> {fir.bindc_name = "y"}) {
 ! [...]
-! CHECK:             omp.wsloop {
+! CHECK:             omp.wsloop private({{.*}}) {
 ! CHECK-NEXT:          omp.loop_nest (%[[VAL_53:.*]], %[[VAL_54:.*]]) : i32 = ({{.*}}) to ({{.*}}) inclusive step ({{.*}}) {
 ! [...]
 ! CHECK:                 cf.br ^bb1
diff --git a/flang/test/Lower/OpenMP/wsloop-variable.f90 b/flang/test/Lower/OpenMP/wsloop-variable.f90
index cc77ce754d97e..50b2b3a21ff1e 100644
--- a/flang/test/Lower/OpenMP/wsloop-variable.f90
+++ b/flang/test/Lower/OpenMP/wsloop-variable.f90
@@ -22,7 +22,7 @@ program wsloop_variable
 !CHECK:      %[[TMP5:.*]] = fir.convert %{{.*}} : (i128) -> i64
 !CHECK:      %[[TMP6:.*]] = fir.convert %[[TMP1]] : (i32) -> i64
 !CHECK:      %[[TMP7:.*]] = fir.convert %{{.*}} : (i32) -> i64
-!CHECK:      omp.wsloop {
+!CHECK:      omp.wsloop private({{.*}}) {
 !CHECK-NEXT:   omp.loop_nest (%[[ARG0:.*]], %[[ARG1:.*]]) : i64 = (%[[TMP2]], %[[TMP5]]) to (%[[TMP3]], %[[TMP6]]) inclusive step (%[[TMP4]], %[[TMP7]]) {
 !CHECK:          %[[ARG0_I16:.*]] = fir.convert %[[ARG0]] : (i64) -> i16
 !CHECK:          fir.store %[[ARG0_I16]] to %[[STORE_IV0:.*]]#1 : !fir.ref<i16>
@@ -48,7 +48,7 @@ program wsloop_variable
 !CHECK:      %[[TMP12:.*]] = arith.constant 1 : i32
 !CHECK:      %[[TMP13:.*]] = fir.convert %{{.*}} : (i8) -> i32
 !CHECK:      %[[TMP14:.*]] = fir.convert %{{.*}} : (i64) -> i32
-!CHECK:      omp.wsloop {
+!CHECK:      omp.wsloop private({{.*}}) {
 !CHECK-NEXT:   omp.loop_nest (%[[ARG0:.*]]) : i32 = (%[[TMP12]]) to (%[[TMP13]]) inclusive step (%[[TMP14]]) {
 !CHECK:          %[[ARG0_I16:.*]] = fir.convert %[[ARG0]] : (i32) -> i16
 !CHECK:          fir.store %[[ARG0_I16]] to %[[STORE3:.*]]#1 : !fir.ref<i16>
@@ -68,7 +68,7 @@ program wsloop_variable
 !CHECK:      %[[TMP17:.*]] = fir.convert %{{.*}} : (i8) -> i64
 !CHECK:      %[[TMP18:.*]] = fir.convert %{{.*}} : (i16) -> i64
 !CHECK:      %[[TMP19:.*]] = fir.convert %{{.*}} : (i32) -> i64
-!CHECK:      omp.wsloop {
+!CHECK:      omp.wsloop private({{.*}}) {
 !CHECK-NEXT:   omp.loop_nest (%[[ARG1:.*]]) : i64 = (%[[TMP17]]) to (%[[TMP18]]) inclusive step (%[[TMP19]])  {
 !CHECK:          %[[ARG1_I128:.*]] = fir.convert %[[ARG1]] : (i64) -> i128
 !CHECK:          fir.store %[[ARG1_I128]] to %[[STORE4:.*]]#1 : !fir.ref<i128>
@@ -123,16 +123,14 @@ subroutine wsloop_variable_sub
   integer(kind=16) :: i16_lb
   real :: x
 
-!CHECK:           %[[VAL_2:.*]] = fir.alloca i16 {bindc_name = "i2", pinned, {{.*}}}
-!CHECK:           %[[VAL_3:.*]]:2 = hlfir.declare %[[VAL_2]] {uniq_name = "_QFwsloop_variable_subEi2"} : (!fir.ref<i16>) -> (!fir.ref<i16>, !fir.ref<i16>)
-
 !CHECK:           %[[VAL_22:.*]] = arith.constant 1 : i32
 !CHECK:           %[[VAL_23:.*]] = fir.load %[[VAL_9]]#0 : !fir.ref<i8>
 !CHECK:           %[[VAL_24:.*]] = fir.load %[[VAL_13]]#0 : !fir.ref<i16>
 !CHECK:           %[[VAL_25:.*]] = fir.convert %[[VAL_23]] : (i8) -> i32
 !CHECK:           %[[VAL_26:.*]] = fir.convert %[[VAL_24]] : (i16) -> i32
-!CHECK:           omp.wsloop {
+!CHECK:           omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_2:.*]] : !fir.ref<i16>) {
 !CHECK-NEXT:        omp.loop_nest (%[[VAL_27:.*]]) : i32 = (%[[VAL_22]]) to (%[[VAL_25]]) inclusive step (%[[VAL_26]]) {
+!CHECK:               %[[VAL_3:.*]]:2 = hlfir.declare %[[VAL_2]] {uniq_name = "_QFwsloop_variable_subEi2"} : (!fir.ref<i16>) -> (!fir.ref<i16>, !fir.ref<i16>)
 !CHECK:               %[[VAL_28:.*]] = fir.convert %[[VAL_27]] : (i32) -> i16
 !CHECK:               fir.store %[[VAL_28]] to %[[VAL_3]]#1 : !fir.ref<i16>
 !CHECK:               %[[VAL_29:.*]] = fir.load %[[VAL_7]]#0 : !fir.ref<i128>
@@ -172,14 +170,13 @@ subroutine wsloop_variable_sub
 !CHECK:           %[[VAL_49:.*]] = arith.constant 5 : i8
 !CHECK:           hlfir.assign %[[VAL_49]] to %[[VAL_19]]#0 : i8, !fir.ref<i8>
 
-!CHECK:           %[[VAL_0:.*]] = fir.alloca i8 {bindc_name = "i1", pinned, {{.*}}}
-!CHECK:           %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0]] {uniq_name = "_QFwsloop_variable_subEi1"} : (!fir.ref<i8>) -> (!fir.ref<i8>, !fir.ref<i8>)
 
 !CHECK:           %[[VAL_50:.*]] = arith.constant 1 : i32
 !CHECK:           %[[VAL_51:.*]] = arith.constant 10 : i32
 !CHECK:           %[[VAL_52:.*]] = arith.constant 1 : i32
-!CHECK:           omp.wsloop {
+!CHECK:           omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_0:.*]] : !fir.ref<i8>) {
 !CHECK-NEXT:        omp.loop_nest (%[[VAL_53:.*]]) : i32 = (%[[VAL_50]]) to (%[[VAL_51]]) inclusive step (%[[VAL_52]]) {
+!CHECK:               %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0]] {uniq_name = "_QFwsloop_variable_subEi1"} : (!fir.ref<i8>) -> (!fir.ref<i8>, !fir.ref<i8>)
 !CHECK:               %[[VAL_54:.*]] = fir.convert %[[VAL_53]] : (i32) -> i8
 !CHECK:               fir.store %[[VAL_54]] to %[[VAL_1]]#1 : !fir.ref<i8>
 !CHECK:               %[[VAL_55:.*]] = fir.load %[[VAL_1]]#0 : !fir.ref<i8>
diff --git a/flang/test/Lower/OpenMP/wsloop.f90 b/flang/test/Lower/OpenMP/wsloop.f90
index 4378233a622ed..44b2f585b3a67 100644
--- a/flang/test/Lower/OpenMP/wsloop.f90
+++ b/flang/test/Lower/OpenMP/wsloop.f90
@@ -7,15 +7,14 @@ subroutine simple_loop
   integer :: i
   ! CHECK:  omp.parallel
   !$OMP PARALLEL
-  ! CHECK:      %[[ALLOCA_IV:.*]] = fir.alloca i32 {{{.*}}, pinned, {{.*}}}
-  ! CHECK:      %[[IV_DECL:.*]]:2 = hlfir.declare %[[ALLOCA_IV]] {uniq_name = "_QFsimple_loopEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
   ! CHECK:      %[[WS_LB:.*]] = arith.constant 1 : i32
   ! CHECK:      %[[WS_UB:.*]] = arith.constant 9 : i32
   ! CHECK:      %[[WS_STEP:.*]] = arith.constant 1 : i32
-  ! CHECK:      omp.wsloop {
+  ! CHECK:      omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[ALLOCA_IV:.*]] : !fir.ref<i32>) {
   ! CHECK-NEXT:   omp.loop_nest (%[[I:.*]]) : i32 = (%[[WS_LB]]) to (%[[WS_UB]]) inclusive step (%[[WS_STEP]]) {
   !$OMP DO
   do i=1, 9
+  ! CHECK:          %[[IV_DECL:.*]]:2 = hlfir.declare %[[ALLOCA_IV]] {uniq_name = "_QFsimple_loopEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
   ! CHECK:          fir.store %[[I]] to %[[IV_DECL:.*]]#1 : !fir.ref<i32>
   ! CHECK:          %[[LOAD_IV:.*]] = fir.load %[[IV_DECL]]#0 : !fir.ref<i32>
   ! CHECK:          fir.call @_FortranAioOutputInteger32({{.*}}, %[[LOAD_IV]]) {{.*}}: (!fir.ref<i8>, i32) -> i1
@@ -32,13 +31,12 @@ subroutine simple_loop_with_step
   integer :: i
   ! CHECK:  omp.parallel
   !$OMP PARALLEL
-  ! CHECK:      %[[ALLOCA_IV:.*]] = fir.alloca i32 {{{.*}}, pinned, {{.*}}}
-  ! CHECK:      %[[IV_DECL:.*]]:2 = hlfir.declare %[[ALLOCA_IV]] {uniq_name = "_QFsimple_loop_with_stepEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
   ! CHECK:      %[[WS_LB:.*]] = arith.constant 1 : i32
   ! CHECK:      %[[WS_UB:.*]] = arith.constant 9 : i32
   ! CHECK:      %[[WS_STEP:.*]] = arith.constant 2 : i32
-  ! CHECK:      omp.wsloop {
+  ! CHECK:      omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[ALLOCA_IV:.*]] : !fir.ref<i32>) {
   ! CHECK-NEXT:   omp.loop_nest (%[[I:.*]]) : i32 = (%[[WS_LB]]) to (%[[WS_UB]]) inclusive step (%[[WS_STEP]]) {
+  ! CHECK:          %[[IV_DECL:.*]]:2 = hlfir.declare %[[ALLOCA_IV]] {uniq_name = "_QFsimple_loop_with_stepEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
   ! CHECK:          fir.store %[[I]] to %[[IV_DECL]]#1 : !fir.ref<i32>
   ! CHECK:          %[[LOAD_IV:.*]] = fir.load %[[IV_DECL]]#0 : !fir.ref<i32>
   !$OMP DO
@@ -57,15 +55,14 @@ subroutine loop_with_schedule_nowait
   integer :: i
   ! CHECK:  omp.parallel
   !$OMP PARALLEL
-  ! CHECK:      %[[ALLOCA_IV:.*]] = fir.alloca i32 {{{.*}}, pinned, {{.*}}}
-  ! CHECK:      %[[IV_DECL:.*]]:2 = hlfir.declare %[[ALLOCA_IV]] {uniq_name = "_QFloop_with_schedule_nowaitEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
   ! CHECK:      %[[WS_LB:.*]] = arith.constant 1 : i32
   ! CHECK:      %[[WS_UB:.*]] = arith.constant 9 : i32
   ! CHECK:      %[[WS_STEP:.*]] = arith.constant 1 : i32
-  ! CHECK:      omp.wsloop nowait schedule(runtime) {
+  ! CHECK:      omp.wsloop nowait schedule(runtime) private(@{{.*}} %{{.*}}#0 -> %[[ALLOCA_IV:.*]] : !fir.ref<i32>) {
   ! CHECK-NEXT:   omp.loop_nest (%[[I:.*]]) : i32 = (%[[WS_LB]]) to (%[[WS_UB]]) inclusive step (%[[WS_STEP]]) {
   !$OMP DO SCHEDULE(runtime)
   do i=1, 9
+  ! CHECK:          %[[IV_DECL:.*]]:2 = hlfir.declare %[[ALLOCA_IV]] {uniq_name = "_QFloop_with_schedule_nowaitEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
   ! CHECK:          fir.store %[[I]] to %[[IV_DECL]]#1 : !fir.ref<i32>
   ! CHECK:          %[[LOAD_IV:.*]] = fir.load %[[IV_DECL]]#0 : !fir.ref<i32>
   ! CHECK:          fir.call @_FortranAioOutputInteger32({{.*}}, %[[LOAD_IV]]) {{.*}}: (!fir.ref<i8>, i32) -> i1
diff --git a/flang/test/Lower/PowerPC/ppc-vec-load-elem-order.f90 b/flang/test/Lower/PowerPC/ppc-vec-load-elem-order.f90
index 214fe423628d6..355fd6c3a742a 100644
--- a/flang/test/Lower/PowerPC/ppc-vec-load-elem-order.f90
+++ b/flang/test/Lower/PowerPC/ppc-vec-load-elem-order.f90
@@ -719,8 +719,8 @@ subroutine vec_xlds_testi64a(arg1, arg2, res)
 ! LLVMIR: %[[arg1:.*]] = load i64, ptr %0, align 8
 ! LLVMIR: %[[addr:.*]] = getelementptr i8, ptr %1, i64 %[[arg1]]
 ! LLVMIR: %[[ld:.*]] = load i64, ptr %[[addr]], align 8
-! LLVMIR: %[[insrt:.*]] = insertelement <2 x i64> undef, i64 %[[ld]], i32 0
-! LLVMIR: %[[shflv:.*]] = shufflevector <2 x i64> %[[insrt]], <2 x i64> undef, <2 x i32> zeroinitializer
+! LLVMIR: %[[insrt:.*]] = insertelement <2 x i64> poison, i64 %[[ld]], i32 0
+! LLVMIR: %[[shflv:.*]] = shufflevector <2 x i64> %[[insrt]], <2 x i64> poison, <2 x i32> zeroinitializer
 ! LLVMIR: store <2 x i64> %[[shflv]], ptr %2, align 16
 end subroutine vec_xlds_testi64a
 
@@ -734,8 +734,8 @@ subroutine vec_xlds_testf64a(arg1, arg2, res)
 ! LLVMIR: %[[arg1:.*]] = load i64, ptr %0, align 8
 ! LLVMIR: %[[addr:.*]] = getelementptr i8, ptr %1, i64 %[[arg1]]
 ! LLVMIR: %[[ld:.*]] = load i64, ptr %[[addr]], align 8
-! LLVMIR: %[[insrt:.*]] = insertelement <2 x i64> undef, i64 %[[ld]], i32 0
-! LLVMIR: %[[shflv:.*]] = shufflevector <2 x i64> %[[insrt]], <2 x i64> undef, <2 x i32> zeroinitializer
+! LLVMIR: %[[insrt:.*]] = insertelement <2 x i64> poison, i64 %[[ld]], i32 0
+! LLVMIR: %[[shflv:.*]] = shufflevector <2 x i64> %[[insrt]], <2 x i64> poison, <2 x i32> zeroinitializer
 ! LLVMIR: %[[bc:.*]] = bitcast <2 x i64> %[[shflv]] to <2 x double>
 ! LLVMIR: store <2 x double> %[[bc]], ptr %2, align 16
 end subroutine vec_xlds_testf64a
diff --git a/flang/test/Lower/PowerPC/ppc-vec-load.f90 b/flang/test/Lower/PowerPC/ppc-vec-load.f90
index a81ed055ce08c..f2c918ecf5bfe 100644
--- a/flang/test/Lower/PowerPC/ppc-vec-load.f90
+++ b/flang/test/Lower/PowerPC/ppc-vec-load.f90
@@ -683,8 +683,8 @@ subroutine vec_xlds_testi64a(arg1, arg2, res)
 ! LLVMIR: %[[arg1:.*]] = load i64, ptr %0, align 8
 ! LLVMIR: %[[addr:.*]] = getelementptr i8, ptr %1, i64 %[[arg1]]
 ! LLVMIR: %[[ld:.*]] = load i64, ptr %[[addr]], align 8
-! LLVMIR: %[[insrt:.*]] = insertelement <2 x i64> undef, i64 %[[ld]], i32 0
-! LLVMIR: %[[shfl:.*]] = shufflevector <2 x i64> %[[insrt]], <2 x i64> undef, <2 x i32> zeroinitializer
+! LLVMIR: %[[insrt:.*]] = insertelement <2 x i64> poison, i64 %[[ld]], i32 0
+! LLVMIR: %[[shfl:.*]] = shufflevector <2 x i64> %[[insrt]], <2 x i64> poison, <2 x i32> zeroinitializer
 ! LLVMIR: store <2 x i64> %[[shfl]], ptr %2, align 16
 end subroutine vec_xlds_testi64a
 
@@ -698,8 +698,8 @@ subroutine vec_xlds_testf64a(arg1, arg2, res)
 ! LLVMIR: %[[arg1:.*]] = load i64, ptr %0, align 8
 ! LLVMIR: %[[addr:.*]] = getelementptr i8, ptr %1, i64 %[[arg1]]
 ! LLVMIR: %[[ld:.*]] = load i64, ptr %[[addr]], align 8
-! LLVMIR: %[[insrt:.*]] = insertelement <2 x i64> undef, i64 %[[ld]], i32 0
-! LLVMIR: %[[shfl:.*]] = shufflevector <2 x i64> %[[insrt]], <2 x i64> undef, <2 x i32> zeroinitializer
+! LLVMIR: %[[insrt:.*]] = insertelement <2 x i64> poison, i64 %[[ld]], i32 0
+! LLVMIR: %[[shfl:.*]] = shufflevector <2 x i64> %[[insrt]], <2 x i64> poison, <2 x i32> zeroinitializer
 ! LLVMIR: %[[bc:.*]] = bitcast <2 x i64> %[[shfl]] to <2 x double>
 ! LLVMIR: store <2 x double> %[[bc]], ptr %2, align 16
 end subroutine vec_xlds_testf64a
diff --git a/flang/test/Lower/PowerPC/ppc-vec-splat-elem-order.f90 b/flang/test/Lower/PowerPC/ppc-vec-splat-elem-order.f90
index d95e9828531cd..50604e1f720f3 100644
--- a/flang/test/Lower/PowerPC/ppc-vec-splat-elem-order.f90
+++ b/flang/test/Lower/PowerPC/ppc-vec-splat-elem-order.f90
@@ -8,8 +8,8 @@ subroutine vec_splat_testf32i64(x)
 
 ! LLVMIR: %[[x:.*]] = load <4 x float>, ptr %{{[0-9]}}, align 16
 ! LLVMIR: %[[ele:.*]] = extractelement <4 x float> %[[x]], i64 3
-! LLVMIR: %[[ins:.*]] = insertelement <4 x float> undef, float %[[ele]], i32 0
-! LLVMIR: %[[y:.*]] = shufflevector <4 x float> %[[ins]], <4 x float> undef, <4 x i32> zeroinitializer
+! LLVMIR: %[[ins:.*]] = insertelement <4 x float> poison, float %[[ele]], i32 0
+! LLVMIR: %[[y:.*]] = shufflevector <4 x float> %[[ins]], <4 x float> poison, <4 x i32> zeroinitializer
 ! LLVMIR: store <4 x float> %[[y]], ptr %{{[0-9]}}, align 16
 end subroutine vec_splat_testf32i64
 
@@ -20,7 +20,7 @@ subroutine vec_splat_testu8i16(x)
 
 ! LLVMIR: %[[x:.*]] = load <16 x i8>, ptr %{{[0-9]}}, align 16
 ! LLVMIR: %[[ele:.*]] = extractelement <16 x i8> %[[x]], i16 15
-! LLVMIR: %[[ins:.*]] = insertelement <16 x i8> undef, i8 %[[ele]], i32 0
-! LLVMIR: %[[y:.*]] = shufflevector <16 x i8> %[[ins]], <16 x i8> undef, <16 x i32> zeroinitializer
+! LLVMIR: %[[ins:.*]] = insertelement <16 x i8> poison, i8 %[[ele]], i32 0
+! LLVMIR: %[[y:.*]] = shufflevector <16 x i8> %[[ins]], <16 x i8> poison, <16 x i32> zeroinitializer
 ! LLVMIR: store <16 x i8> %[[y]], ptr %{{[0-9]}}, align 16
 end subroutine vec_splat_testu8i16
diff --git a/flang/test/Lower/PowerPC/ppc-vec-splat.f90 b/flang/test/Lower/PowerPC/ppc-vec-splat.f90
index 17558926afd5f..f3c1f19d5877d 100644
--- a/flang/test/Lower/PowerPC/ppc-vec-splat.f90
+++ b/flang/test/Lower/PowerPC/ppc-vec-splat.f90
@@ -14,8 +14,8 @@ subroutine vec_splat_testi8i8(x)
 ! LLVMIR: %[[x:.*]] = load <16 x i8>, ptr %{{[0-9]}}, align 16
 ! LLVMIR-LE: %[[ele:.*]] = extractelement <16 x i8> %[[x]], i8 0
 ! LLVMIR-BE: %[[ele:.*]] = extractelement <16 x i8> %[[x]], i8 15
-! LLVMIR: %[[ins:.*]] = insertelement <16 x i8> undef, i8 %[[ele]], i32 0
-! LLVMIR: %[[y:.*]] = shufflevector <16 x i8> %[[ins]], <16 x i8> undef, <16 x i32> zeroinitializer
+! LLVMIR: %[[ins:.*]] = insertelement <16 x i8> poison, i8 %[[ele]], i32 0
+! LLVMIR: %[[y:.*]] = shufflevector <16 x i8> %[[ins]], <16 x i8> poison, <16 x i32> zeroinitializer
 ! LLVMIR: store <16 x i8> %[[y]], ptr %{{[0-9]}}, align 16
 end subroutine vec_splat_testi8i8
 
@@ -27,8 +27,8 @@ subroutine vec_splat_testi8i16(x)
 ! LLVMIR: %[[x:.*]] = load <16 x i8>, ptr %{{[0-9]}}, align 16
 ! LLVMIR-LE: %[[ele:.*]] = extractelement <16 x i8> %[[x]], i16 0
 ! LLVMIR-BE: %[[ele:.*]] = extractelement <16 x i8> %[[x]], i16 15
-! LLVMIR: %[[ins:.*]] = insertelement <16 x i8> undef, i8 %[[ele]], i32 0
-! LLVMIR: %[[y:.*]] = shufflevector <16 x i8> %[[ins]], <16 x i8> undef, <16 x i32> zeroinitializer
+! LLVMIR: %[[ins:.*]] = insertelement <16 x i8> poison, i8 %[[ele]], i32 0
+! LLVMIR: %[[y:.*]] = shufflevector <16 x i8> %[[ins]], <16 x i8> poison, <16 x i32> zeroinitializer
 ! LLVMIR: store <16 x i8> %[[y]], ptr %{{[0-9]}}, align 16
 end subroutine vec_splat_testi8i16
 
@@ -40,8 +40,8 @@ subroutine vec_splat_testi8i32(x)
 ! LLVMIR: %[[x:.*]] = load <16 x i8>, ptr %{{[0-9]}}, align 16
 ! LLVMIR-LE: %[[ele:.*]] = extractelement <16 x i8> %[[x]], i32 0
 ! LLVMIR-BE: %[[ele:.*]] = extractelement <16 x i8> %[[x]], i32 15
-! LLVMIR: %[[ins:.*]] = insertelement <16 x i8> undef, i8 %[[ele]], i32 0
-! LLVMIR: %[[y:.*]] = shufflevector <16 x i8> %[[ins]], <16 x i8> undef, <16 x i32> zeroinitializer
+! LLVMIR: %[[ins:.*]] = insertelement <16 x i8> poison, i8 %[[ele]], i32 0
+! LLVMIR: %[[y:.*]] = shufflevector <16 x i8> %[[ins]], <16 x i8> poison, <16 x i32> zeroinitializer
 ! LLVMIR: store <16 x i8> %[[y]], ptr %{{[0-9]}}, align 16
 end subroutine vec_splat_testi8i32
 
@@ -53,8 +53,8 @@ subroutine vec_splat_testi8i64(x)
 ! LLVMIR: %[[x:.*]] = load <16 x i8>, ptr %{{[0-9]}}, align 16
 ! LLVMIR-LE: %[[ele:.*]] = extractelement <16 x i8> %[[x]], i64 0
 ! LLVMIR-BE: %[[ele:.*]] = extractelement <16 x i8> %[[x]], i64 15
-! LLVMIR: %[[ins:.*]] = insertelement <16 x i8> undef, i8 %[[ele]], i32 0
-! LLVMIR: %[[y:.*]] = shufflevector <16 x i8> %[[ins]], <16 x i8> undef, <16 x i32> zeroinitializer
+! LLVMIR: %[[ins:.*]] = insertelement <16 x i8> poison, i8 %[[ele]], i32 0
+! LLVMIR: %[[y:.*]] = shufflevector <16 x i8> %[[ins]], <16 x i8> poison, <16 x i32> zeroinitializer
 ! LLVMIR: store <16 x i8> %[[y]], ptr %{{[0-9]}}, align 16
 end subroutine vec_splat_testi8i64
 
@@ -66,8 +66,8 @@ subroutine vec_splat_testi16i8(x)
 ! LLVMIR: %[[x:.*]] = load <8 x i16>, ptr %{{[0-9]}}, align 16
 ! LLVMIR-LE: %[[ele:.*]] = extractelement <8 x i16> %[[x]], i8 0
 ! LLVMIR-BE: %[[ele:.*]] = extractelement <8 x i16> %[[x]], i8 7
-! LLVMIR: %[[ins:.*]] = insertelement <8 x i16> undef, i16 %[[ele]], i32 0
-! LLVMIR: %[[y:.*]] = shufflevector <8 x i16> %[[ins]], <8 x i16> undef, <8 x i32> zeroinitializer
+! LLVMIR: %[[ins:.*]] = insertelement <8 x i16> poison, i16 %[[ele]], i32 0
+! LLVMIR: %[[y:.*]] = shufflevector <8 x i16> %[[ins]], <8 x i16> poison, <8 x i32> zeroinitializer
 ! LLVMIR: store <8 x i16> %[[y]], ptr %{{[0-9]}}, align 16
 end subroutine vec_splat_testi16i8
 
@@ -79,8 +79,8 @@ subroutine vec_splat_testi16i16(x)
 ! LLVMIR: %[[x:.*]] = load <8 x i16>, ptr %{{[0-9]}}, align 16
 ! LLVMIR-LE: %[[ele:.*]] = extractelement <8 x i16> %[[x]], i16 0
 ! LLVMIR-BE: %[[ele:.*]] = extractelement <8 x i16> %[[x]], i16 7
-! LLVMIR: %[[ins:.*]] = insertelement <8 x i16> undef, i16 %[[ele]], i32 0
-! LLVMIR: %[[y:.*]] = shufflevector <8 x i16> %[[ins]], <8 x i16> undef, <8 x i32> zeroinitializer
+! LLVMIR: %[[ins:.*]] = insertelement <8 x i16> poison, i16 %[[ele]], i32 0
+! LLVMIR: %[[y:.*]] = shufflevector <8 x i16> %[[ins]], <8 x i16> poison, <8 x i32> zeroinitializer
 ! LLVMIR: store <8 x i16> %[[y]], ptr %{{[0-9]}}, align 16
 end subroutine vec_splat_testi16i16
 
@@ -92,8 +92,8 @@ subroutine vec_splat_testi16i32(x)
 ! LLVMIR: %[[x:.*]] = load <8 x i16>, ptr %{{[0-9]}}, align 16
 ! LLVMIR-LE: %[[ele:.*]] = extractelement <8 x i16> %[[x]], i32 0
 ! LLVMIR-BE: %[[ele:.*]] = extractelement <8 x i16> %[[x]], i32 7
-! LLVMIR: %[[ins:.*]] = insertelement <8 x i16> undef, i16 %[[ele]], i32 0
-! LLVMIR: %[[y:.*]] = shufflevector <8 x i16> %[[ins]], <8 x i16> undef, <8 x i32> zeroinitializer
+! LLVMIR: %[[ins:.*]] = insertelement <8 x i16> poison, i16 %[[ele]], i32 0
+! LLVMIR: %[[y:.*]] = shufflevector <8 x i16> %[[ins]], <8 x i16> poison, <8 x i32> zeroinitializer
 ! LLVMIR: store <8 x i16> %[[y]], ptr %{{[0-9]}}, align 16
 end subroutine vec_splat_testi16i32
 
@@ -105,8 +105,8 @@ subroutine vec_splat_testi16i64(x)
 ! LLVMIR: %[[x:.*]] = load <8 x i16>, ptr %{{[0-9]}}, align 16
 ! LLVMIR-LE: %[[ele:.*]] = extractelement <8 x i16> %[[x]], i64 0
 ! LLVMIR-BE: %[[ele:.*]] = extractelement <8 x i16> %[[x]], i64 7
-! LLVMIR: %[[ins:.*]] = insertelement <8 x i16> undef, i16 %[[ele]], i32 0
-! LLVMIR: %[[y:.*]] = shufflevector <8 x i16> %[[ins]], <8 x i16> undef, <8 x i32> zeroinitializer
+! LLVMIR: %[[ins:.*]] = insertelement <8 x i16> poison, i16 %[[ele]], i32 0
+! LLVMIR: %[[y:.*]] = shufflevector <8 x i16> %[[ins]], <8 x i16> poison, <8 x i32> zeroinitializer
 ! LLVMIR: store <8 x i16> %[[y]], ptr %{{[0-9]}}, align 16
 end subroutine vec_splat_testi16i64
 
@@ -118,8 +118,8 @@ subroutine vec_splat_testi32i8(x)
 ! LLVMIR: %[[x:.*]] = load <4 x i32>, ptr %{{[0-9]}}, align 16
 ! LLVMIR-LE: %[[ele:.*]] = extractelement <4 x i32> %[[x]], i8 0
 ! LLVMIR-BE: %[[ele:.*]] = extractelement <4 x i32> %[[x]], i8 3
-! LLVMIR: %[[ins:.*]] = insertelement <4 x i32> undef, i32 %[[ele]], i32 0
-! LLVMIR: %[[y:.*]] = shufflevector <4 x i32> %[[ins]], <4 x i32> undef, <4 x i32> zeroinitializer
+! LLVMIR: %[[ins:.*]] = insertelement <4 x i32> poison, i32 %[[ele]], i32 0
+! LLVMIR: %[[y:.*]] = shufflevector <4 x i32> %[[ins]], <4 x i32> poison, <4 x i32> zeroinitializer
 ! LLVMIR: store <4 x i32> %[[y]], ptr %{{[0-9]}}, align 16
 end subroutine vec_splat_testi32i8
 
@@ -131,8 +131,8 @@ subroutine vec_splat_testi32i16(x)
 ! LLVMIR: %[[x:.*]] = load <4 x i32>, ptr %{{[0-9]}}, align 16
 ! LLVMIR-LE: %[[ele:.*]] = extractelement <4 x i32> %[[x]], i16 0
 ! LLVMIR-BE: %[[ele:.*]] = extractelement <4 x i32> %[[x]], i16 3
-! LLVMIR: %[[ins:.*]] = insertelement <4 x i32> undef, i32 %[[ele]], i32 0
-! LLVMIR: %[[y:.*]] = shufflevector <4 x i32> %[[ins]], <4 x i32> undef, <4 x i32> zeroinitializer
+! LLVMIR: %[[ins:.*]] = insertelement <4 x i32> poison, i32 %[[ele]], i32 0
+! LLVMIR: %[[y:.*]] = shufflevector <4 x i32> %[[ins]], <4 x i32> poison, <4 x i32> zeroinitializer
 ! LLVMIR: store <4 x i32> %[[y]], ptr %{{[0-9]}}, align 16
 end subroutine vec_splat_testi32i16
 
@@ -144,8 +144,8 @@ subroutine vec_splat_testi32i32(x)
 ! LLVMIR: %[[x:.*]] = load <4 x i32>, ptr %{{[0-9]}}, align 16
 ! LLVMIR-LE: %[[ele:.*]] = extractelement <4 x i32> %[[x]], i32 0
 ! LLVMIR-BE: %[[ele:.*]] = extractelement <4 x i32> %[[x]], i32 3
-! LLVMIR: %[[ins:.*]] = insertelement <4 x i32> undef, i32 %[[ele]], i32 0
-! LLVMIR: %[[y:.*]] = shufflevector <4 x i32> %[[ins]], <4 x i32> undef, <4 x i32> zeroinitializer
+! LLVMIR: %[[ins:.*]] = insertelement <4 x i32> poison, i32 %[[ele]], i32 0
+! LLVMIR: %[[y:.*]] = shufflevector <4 x i32> %[[ins]], <4 x i32> poison, <4 x i32> zeroinitializer
 ! LLVMIR: store <4 x i32> %[[y]], ptr %{{[0-9]}}, align 16
 end subroutine vec_splat_testi32i32
 
@@ -157,8 +157,8 @@ subroutine vec_splat_testi32i64(x)
 ! LLVMIR: %[[x:.*]] = load <4 x i32>, ptr %{{[0-9]}}, align 16
 ! LLVMIR-LE: %[[ele:.*]] = extractelement <4 x i32> %[[x]], i64 0
 ! LLVMIR-BE: %[[ele:.*]] = extractelement <4 x i32> %[[x]], i64 3
-! LLVMIR: %[[ins:.*]] = insertelement <4 x i32> undef, i32 %[[ele]], i32 0
-! LLVMIR: %[[y:.*]] = shufflevector <4 x i32> %[[ins]], <4 x i32> undef, <4 x i32> zeroinitializer
+! LLVMIR: %[[ins:.*]] = insertelement <4 x i32> poison, i32 %[[ele]], i32 0
+! LLVMIR: %[[y:.*]] = shufflevector <4 x i32> %[[ins]], <4 x i32> poison, <4 x i32> zeroinitializer
 ! LLVMIR: store <4 x i32> %[[y]], ptr %{{[0-9]}}, align 16
 end subroutine vec_splat_testi32i64
 
@@ -170,8 +170,8 @@ subroutine vec_splat_testi64i8(x)
 ! LLVMIR: %[[x:.*]] = load <2 x i64>, ptr %{{[0-9]}}, align 16
 ! LLVMIR-LE: %[[ele:.*]] = extractelement <2 x i64> %[[x]], i8 0
 ! LLVMIR-BE: %[[ele:.*]] = extractelement <2 x i64> %[[x]], i8 1
-! LLVMIR: %[[ins:.*]] = insertelement <2 x i64> undef, i64 %[[ele]], i32 0
-! LLVMIR: %[[y:.*]] = shufflevector <2 x i64> %[[ins]], <2 x i64> undef, <2 x i32> zeroinitializer
+! LLVMIR: %[[ins:.*]] = insertelement <2 x i64> poison, i64 %[[ele]], i32 0
+! LLVMIR: %[[y:.*]] = shufflevector <2 x i64> %[[ins]], <2 x i64> poison, <2 x i32> zeroinitializer
 ! LLVMIR: store <2 x i64> %[[y]], ptr %{{[0-9]}}, align 16
 end subroutine vec_splat_testi64i8
 
@@ -183,8 +183,8 @@ subroutine vec_splat_testi64i16(x)
 ! LLVMIR: %[[x:.*]] = load <2 x i64>, ptr %{{[0-9]}}, align 16
 ! LLVMIR-LE: %[[ele:.*]] = extractelement <2 x i64> %[[x]], i16 0
 ! LLVMIR-BE: %[[ele:.*]] = extractelement <2 x i64> %[[x]], i16 1
-! LLVMIR: %[[ins:.*]] = insertelement <2 x i64> undef, i64 %[[ele]], i32 0
-! LLVMIR: %[[y:.*]] = shufflevector <2 x i64> %[[ins]], <2 x i64> undef, <2 x i32> zeroinitializer
+! LLVMIR: %[[ins:.*]] = insertelement <2 x i64> poison, i64 %[[ele]], i32 0
+! LLVMIR: %[[y:.*]] = shufflevector <2 x i64> %[[ins]], <2 x i64> poison, <2 x i32> zeroinitializer
 ! LLVMIR: store <2 x i64> %[[y]], ptr %{{[0-9]}}, align 16
 end subroutine vec_splat_testi64i16
 
@@ -196,8 +196,8 @@ subroutine vec_splat_testi64i32(x)
 ! LLVMIR: %[[x:.*]] = load <2 x i64>, ptr %{{[0-9]}}, align 16
 ! LLVMIR-LE: %[[ele:.*]] = extractelement <2 x i64> %[[x]], i32 0
 ! LLVMIR-BE: %[[ele:.*]] = extractelement <2 x i64> %[[x]], i32 1
-! LLVMIR: %[[ins:.*]] = insertelement <2 x i64> undef, i64 %[[ele]], i32 0
-! LLVMIR: %[[y:.*]] = shufflevector <2 x i64> %[[ins]], <2 x i64> undef, <2 x i32> zeroinitializer
+! LLVMIR: %[[ins:.*]] = insertelement <2 x i64> poison, i64 %[[ele]], i32 0
+! LLVMIR: %[[y:.*]] = shufflevector <2 x i64> %[[ins]], <2 x i64> poison, <2 x i32> zeroinitializer
 ! LLVMIR: store <2 x i64> %[[y]], ptr %{{[0-9]}}, align 16
 end subroutine vec_splat_testi64i32
 
@@ -209,8 +209,8 @@ subroutine vec_splat_testi64i64(x)
 ! LLVMIR: %[[x:.*]] = load <2 x i64>, ptr %{{[0-9]}}, align 16
 ! LLVMIR-LE: %[[ele:.*]] = extractelement <2 x i64> %[[x]], i64 0
 ! LLVMIR-BE: %[[ele:.*]] = extractelement <2 x i64> %[[x]], i64 1
-! LLVMIR: %[[ins:.*]] = insertelement <2 x i64> undef, i64 %[[ele]], i32 0
-! LLVMIR: %[[y:.*]] = shufflevector <2 x i64> %[[ins]], <2 x i64> undef, <2 x i32> zeroinitializer
+! LLVMIR: %[[ins:.*]] = insertelement <2 x i64> poison, i64 %[[ele]], i32 0
+! LLVMIR: %[[y:.*]] = shufflevector <2 x i64> %[[ins]], <2 x i64> poison, <2 x i32> zeroinitializer
 ! LLVMIR: store <2 x i64> %[[y]], ptr %{{[0-9]}}, align 16
 end subroutine vec_splat_testi64i64
 
@@ -222,8 +222,8 @@ subroutine vec_splat_testf32i8(x)
 ! LLVMIR: %[[x:.*]] = load <4 x float>, ptr %{{[0-9]}}, align 16
 ! LLVMIR-LE: %[[ele:.*]] = extractelement <4 x float> %[[x]], i8 0
 ! LLVMIR-BE: %[[ele:.*]] = extractelement <4 x float> %[[x]], i8 3
-! LLVMIR: %[[ins:.*]] = insertelement <4 x float> undef, float %[[ele]], i32 0
-! LLVMIR: %[[y:.*]] = shufflevector <4 x float> %[[ins]], <4 x float> undef, <4 x i32> zeroinitializer
+! LLVMIR: %[[ins:.*]] = insertelement <4 x float> poison, float %[[ele]], i32 0
+! LLVMIR: %[[y:.*]] = shufflevector <4 x float> %[[ins]], <4 x float> poison, <4 x i32> zeroinitializer
 ! LLVMIR: store <4 x float> %[[y]], ptr %{{[0-9]}}, align 16
 end subroutine vec_splat_testf32i8
 
@@ -235,8 +235,8 @@ subroutine vec_splat_testf32i16(x)
 ! LLVMIR: %[[x:.*]] = load <4 x float>, ptr %{{[0-9]}}, align 16
 ! LLVMIR-LE: %[[ele:.*]] = extractelement <4 x float> %[[x]], i16 0
 ! LLVMIR-BE: %[[ele:.*]] = extractelement <4 x float> %[[x]], i16 3
-! LLVMIR: %[[ins:.*]] = insertelement <4 x float> undef, float %[[ele]], i32 0
-! LLVMIR: %[[y:.*]] = shufflevector <4 x float> %[[ins]], <4 x float> undef, <4 x i32> zeroinitializer
+! LLVMIR: %[[ins:.*]] = insertelement <4 x float> poison, float %[[ele]], i32 0
+! LLVMIR: %[[y:.*]] = shufflevector <4 x float> %[[ins]], <4 x float> poison, <4 x i32> zeroinitializer
 ! LLVMIR: store <4 x float> %[[y]], ptr %{{[0-9]}}, align 16
 end subroutine vec_splat_testf32i16
 
@@ -248,8 +248,8 @@ subroutine vec_splat_testf32i32(x)
 ! LLVMIR: %[[x:.*]] = load <4 x float>, ptr %{{[0-9]}}, align 16
 ! LLVMIR-LE: %[[ele:.*]] = extractelement <4 x float> %[[x]], i32 0
 ! LLVMIR-BE: %[[ele:.*]] = extractelement <4 x float> %[[x]], i32 3
-! LLVMIR: %[[ins:.*]] = insertelement <4 x float> undef, float %[[ele]], i32 0
-! LLVMIR: %[[y:.*]] = shufflevector <4 x float> %[[ins]], <4 x float> undef, <4 x i32> zeroinitializer
+! LLVMIR: %[[ins:.*]] = insertelement <4 x float> poison, float %[[ele]], i32 0
+! LLVMIR: %[[y:.*]] = shufflevector <4 x float> %[[ins]], <4 x float> poison, <4 x i32> zeroinitializer
 ! LLVMIR: store <4 x float> %[[y]], ptr %{{[0-9]}}, align 16
 end subroutine vec_splat_testf32i32
 
@@ -261,8 +261,8 @@ subroutine vec_splat_testf32i64(x)
 ! LLVMIR: %[[x:.*]] = load <4 x float>, ptr %{{[0-9]}}, align 16
 ! LLVMIR-LE: %[[ele:.*]] = extractelement <4 x float> %[[x]], i64 0
 ! LLVMIR-BE: %[[ele:.*]] = extractelement <4 x float> %[[x]], i64 3
-! LLVMIR: %[[ins:.*]] = insertelement <4 x float> undef, float %[[ele]], i32 0
-! LLVMIR: %[[y:.*]] = shufflevector <4 x float> %[[ins]], <4 x float> undef, <4 x i32> zeroinitializer
+! LLVMIR: %[[ins:.*]] = insertelement <4 x float> poison, float %[[ele]], i32 0
+! LLVMIR: %[[y:.*]] = shufflevector <4 x float> %[[ins]], <4 x float> poison, <4 x i32> zeroinitializer
 ! LLVMIR: store <4 x float> %[[y]], ptr %{{[0-9]}}, align 16
 end subroutine vec_splat_testf32i64
 
@@ -274,8 +274,8 @@ subroutine vec_splat_testf64i8(x)
 ! LLVMIR: %[[x:.*]] = load <2 x double>, ptr %{{[0-9]}}, align 16
 ! LLVMIR-LE: %[[ele:.*]] = extractelement <2 x double> %[[x]], i8 0
 ! LLVMIR-BE: %[[ele:.*]] = extractelement <2 x double> %[[x]], i8 1
-! LLVMIR: %[[ins:.*]] = insertelement <2 x double> undef, double %[[ele]], i32 0
-! LLVMIR: %[[y:.*]] = shufflevector <2 x double> %[[ins]], <2 x double> undef, <2 x i32> zeroinitializer
+! LLVMIR: %[[ins:.*]] = insertelement <2 x double> poison, double %[[ele]], i32 0
+! LLVMIR: %[[y:.*]] = shufflevector <2 x double> %[[ins]], <2 x double> poison, <2 x i32> zeroinitializer
 ! LLVMIR: store <2 x double> %[[y]], ptr %{{[0-9]}}, align 16
 end subroutine vec_splat_testf64i8
 
@@ -287,8 +287,8 @@ subroutine vec_splat_testf64i16(x)
 ! LLVMIR: %[[x:.*]] = load <2 x double>, ptr %{{[0-9]}}, align 16
 ! LLVMIR-LE: %[[ele:.*]] = extractelement <2 x double> %[[x]], i16 0
 ! LLVMIR-BE: %[[ele:.*]] = extractelement <2 x double> %[[x]], i16 1
-! LLVMIR: %[[ins:.*]] = insertelement <2 x double> undef, double %[[ele]], i32 0
-! LLVMIR: %[[y:.*]] = shufflevector <2 x double> %[[ins]], <2 x double> undef, <2 x i32> zeroinitializer
+! LLVMIR: %[[ins:.*]] = insertelement <2 x double> poison, double %[[ele]], i32 0
+! LLVMIR: %[[y:.*]] = shufflevector <2 x double> %[[ins]], <2 x double> poison, <2 x i32> zeroinitializer
 ! LLVMIR: store <2 x double> %[[y]], ptr %{{[0-9]}}, align 16
 end subroutine vec_splat_testf64i16
 
@@ -300,8 +300,8 @@ subroutine vec_splat_testf64i32(x)
 ! LLVMIR: %[[x:.*]] = load <2 x double>, ptr %{{[0-9]}}, align 16
 ! LLVMIR-LE: %[[ele:.*]] = extractelement <2 x double> %[[x]], i32 0
 ! LLVMIR-BE: %[[ele:.*]] = extractelement <2 x double> %[[x]], i32 1
-! LLVMIR: %[[ins:.*]] = insertelement <2 x double> undef, double %[[ele]], i32 0
-! LLVMIR: %[[y:.*]] = shufflevector <2 x double> %[[ins]], <2 x double> undef, <2 x i32> zeroinitializer
+! LLVMIR: %[[ins:.*]] = insertelement <2 x double> poison, double %[[ele]], i32 0
+! LLVMIR: %[[y:.*]] = shufflevector <2 x double> %[[ins]], <2 x double> poison, <2 x i32> zeroinitializer
 ! LLVMIR: store <2 x double> %[[y]], ptr %{{[0-9]}}, align 16
 end subroutine vec_splat_testf64i32
 
@@ -313,8 +313,8 @@ subroutine vec_splat_testf64i64(x)
 ! LLVMIR: %[[x:.*]] = load <2 x double>, ptr %{{[0-9]}}, align 16
 ! LLVMIR-LE: %[[ele:.*]] = extractelement <2 x double> %[[x]], i64 0
 ! LLVMIR-BE: %[[ele:.*]] = extractelement <2 x double> %[[x]], i64 1
-! LLVMIR: %[[ins:.*]] = insertelement <2 x double> undef, double %[[ele]], i32 0
-! LLVMIR: %[[y:.*]] = shufflevector <2 x double> %[[ins]], <2 x double> undef, <2 x i32> zeroinitializer
+! LLVMIR: %[[ins:.*]] = insertelement <2 x double> poison, double %[[ele]], i32 0
+! LLVMIR: %[[y:.*]] = shufflevector <2 x double> %[[ins]], <2 x double> poison, <2 x i32> zeroinitializer
 ! LLVMIR: store <2 x double> %[[y]], ptr %{{[0-9]}}, align 16
 end subroutine vec_splat_testf64i64
 
@@ -326,8 +326,8 @@ subroutine vec_splat_testu8i8(x)
 ! LLVMIR: %[[x:.*]] = load <16 x i8>, ptr %{{[0-9]}}, align 16
 ! LLVMIR-LE: %[[ele:.*]] = extractelement <16 x i8> %[[x]], i8 0
 ! LLVMIR-BE: %[[ele:.*]] = extractelement <16 x i8> %[[x]], i8 15
-! LLVMIR: %[[ins:.*]] = insertelement <16 x i8> undef, i8 %[[ele]], i32 0
-! LLVMIR: %[[y:.*]] = shufflevector <16 x i8> %[[ins]], <16 x i8> undef, <16 x i32> zeroinitializer
+! LLVMIR: %[[ins:.*]] = insertelement <16 x i8> poison, i8 %[[ele]], i32 0
+! LLVMIR: %[[y:.*]] = shufflevector <16 x i8> %[[ins]], <16 x i8> poison, <16 x i32> zeroinitializer
 ! LLVMIR: store <16 x i8> %[[y]], ptr %{{[0-9]}}, align 16
 end subroutine vec_splat_testu8i8
 
@@ -339,8 +339,8 @@ subroutine vec_splat_testu8i16(x)
 ! LLVMIR: %[[x:.*]] = load <16 x i8>, ptr %{{[0-9]}}, align 16
 ! LLVMIR-LE: %[[ele:.*]] = extractelement <16 x i8> %[[x]], i16 0
 ! LLVMIR-BE: %[[ele:.*]] = extractelement <16 x i8> %[[x]], i16 15
-! LLVMIR: %[[ins:.*]] = insertelement <16 x i8> undef, i8 %[[ele]], i32 0
-! LLVMIR: %[[y:.*]] = shufflevector <16 x i8> %[[ins]], <16 x i8> undef, <16 x i32> zeroinitializer
+! LLVMIR: %[[ins:.*]] = insertelement <16 x i8> poison, i8 %[[ele]], i32 0
+! LLVMIR: %[[y:.*]] = shufflevector <16 x i8> %[[ins]], <16 x i8> poison, <16 x i32> zeroinitializer
 ! LLVMIR: store <16 x i8> %[[y]], ptr %{{[0-9]}}, align 16
 end subroutine vec_splat_testu8i16
 
@@ -352,8 +352,8 @@ subroutine vec_splat_testu8i32(x)
 ! LLVMIR: %[[x:.*]] = load <16 x i8>, ptr %{{[0-9]}}, align 16
 ! LLVMIR-LE: %[[ele:.*]] = extractelement <16 x i8> %[[x]], i32 0
 ! LLVMIR-BE: %[[ele:.*]] = extractelement <16 x i8> %[[x]], i32 15
-! LLVMIR: %[[ins:.*]] = insertelement <16 x i8> undef, i8 %[[ele]], i32 0
-! LLVMIR: %[[y:.*]] = shufflevector <16 x i8> %[[ins]], <16 x i8> undef, <16 x i32> zeroinitializer
+! LLVMIR: %[[ins:.*]] = insertelement <16 x i8> poison, i8 %[[ele]], i32 0
+! LLVMIR: %[[y:.*]] = shufflevector <16 x i8> %[[ins]], <16 x i8> poison, <16 x i32> zeroinitializer
 ! LLVMIR: store <16 x i8> %[[y]], ptr %{{[0-9]}}, align 16
 end subroutine vec_splat_testu8i32
 
@@ -365,8 +365,8 @@ subroutine vec_splat_testu8i64(x)
 ! LLVMIR: %[[x:.*]] = load <16 x i8>, ptr %{{[0-9]}}, align 16
 ! LLVMIR-LE: %[[ele:.*]] = extractelement <16 x i8> %[[x]], i64 0
 ! LLVMIR-BE: %[[ele:.*]] = extractelement <16 x i8> %[[x]], i64 15
-! LLVMIR: %[[ins:.*]] = insertelement <16 x i8> undef, i8 %[[ele]], i32 0
-! LLVMIR: %[[y:.*]] = shufflevector <16 x i8> %[[ins]], <16 x i8> undef, <16 x i32> zeroinitializer
+! LLVMIR: %[[ins:.*]] = insertelement <16 x i8> poison, i8 %[[ele]], i32 0
+! LLVMIR: %[[y:.*]] = shufflevector <16 x i8> %[[ins]], <16 x i8> poison, <16 x i32> zeroinitializer
 ! LLVMIR: store <16 x i8> %[[y]], ptr %{{[0-9]}}, align 16
 end subroutine vec_splat_testu8i64
 
@@ -378,8 +378,8 @@ subroutine vec_splat_testu16i8(x)
 ! LLVMIR: %[[x:.*]] = load <8 x i16>, ptr %{{[0-9]}}, align 16
 ! LLVMIR-LE: %[[ele:.*]] = extractelement <8 x i16> %[[x]], i8 0
 ! LLVMIR-BE: %[[ele:.*]] = extractelement <8 x i16> %[[x]], i8 7
-! LLVMIR: %[[ins:.*]] = insertelement <8 x i16> undef, i16 %[[ele]], i32 0
-! LLVMIR: %[[y:.*]] = shufflevector <8 x i16> %[[ins]], <8 x i16> undef, <8 x i32> zeroinitializer
+! LLVMIR: %[[ins:.*]] = insertelement <8 x i16> poison, i16 %[[ele]], i32 0
+! LLVMIR: %[[y:.*]] = shufflevector <8 x i16> %[[ins]], <8 x i16> poison, <8 x i32> zeroinitializer
 ! LLVMIR: store <8 x i16> %[[y]], ptr %{{[0-9]}}, align 16
 end subroutine vec_splat_testu16i8
 
@@ -391,8 +391,8 @@ subroutine vec_splat_testu16i16(x)
 ! LLVMIR: %[[x:.*]] = load <8 x i16>, ptr %{{[0-9]}}, align 16
 ! LLVMIR-LE: %[[ele:.*]] = extractelement <8 x i16> %[[x]], i16 0
 ! LLVMIR-BE: %[[ele:.*]] = extractelement <8 x i16> %[[x]], i16 7
-! LLVMIR: %[[ins:.*]] = insertelement <8 x i16> undef, i16 %[[ele]], i32 0
-! LLVMIR: %[[y:.*]] = shufflevector <8 x i16> %[[ins]], <8 x i16> undef, <8 x i32> zeroinitializer
+! LLVMIR: %[[ins:.*]] = insertelement <8 x i16> poison, i16 %[[ele]], i32 0
+! LLVMIR: %[[y:.*]] = shufflevector <8 x i16> %[[ins]], <8 x i16> poison, <8 x i32> zeroinitializer
 ! LLVMIR: store <8 x i16> %[[y]], ptr %{{[0-9]}}, align 16
 end subroutine vec_splat_testu16i16
 
@@ -404,8 +404,8 @@ subroutine vec_splat_testu16i32(x)
 ! LLVMIR: %[[x:.*]] = load <8 x i16>, ptr %{{[0-9]}}, align 16
 ! LLVMIR-LE: %[[ele:.*]] = extractelement <8 x i16> %[[x]], i32 0
 ! LLVMIR-BE: %[[ele:.*]] = extractelement <8 x i16> %[[x]], i32 7
-! LLVMIR: %[[ins:.*]] = insertelement <8 x i16> undef, i16 %[[ele]], i32 0
-! LLVMIR: %[[y:.*]] = shufflevector <8 x i16> %[[ins]], <8 x i16> undef, <8 x i32> zeroinitializer
+! LLVMIR: %[[ins:.*]] = insertelement <8 x i16> poison, i16 %[[ele]], i32 0
+! LLVMIR: %[[y:.*]] = shufflevector <8 x i16> %[[ins]], <8 x i16> poison, <8 x i32> zeroinitializer
 ! LLVMIR: store <8 x i16> %[[y]], ptr %{{[0-9]}}, align 16
 end subroutine vec_splat_testu16i32
 
@@ -417,8 +417,8 @@ subroutine vec_splat_testu16i64(x)
 ! LLVMIR: %[[x:.*]] = load <8 x i16>, ptr %{{[0-9]}}, align 16
 ! LLVMIR-LE: %[[ele:.*]] = extractelement <8 x i16> %[[x]], i64 0
 ! LLVMIR-BE: %[[ele:.*]] = extractelement <8 x i16> %[[x]], i64 7
-! LLVMIR: %[[ins:.*]] = insertelement <8 x i16> undef, i16 %[[ele]], i32 0
-! LLVMIR: %[[y:.*]] = shufflevector <8 x i16> %[[ins]], <8 x i16> undef, <8 x i32> zeroinitializer
+! LLVMIR: %[[ins:.*]] = insertelement <8 x i16> poison, i16 %[[ele]], i32 0
+! LLVMIR: %[[y:.*]] = shufflevector <8 x i16> %[[ins]], <8 x i16> poison, <8 x i32> zeroinitializer
 ! LLVMIR: store <8 x i16> %[[y]], ptr %{{[0-9]}}, align 16
 end subroutine vec_splat_testu16i64
 
@@ -430,8 +430,8 @@ subroutine vec_splat_testu32i8(x)
 ! LLVMIR: %[[x:.*]] = load <4 x i32>, ptr %{{[0-9]}}, align 16
 ! LLVMIR-LE: %[[ele:.*]] = extractelement <4 x i32> %[[x]], i8 0
 ! LLVMIR-BE: %[[ele:.*]] = extractelement <4 x i32> %[[x]], i8 3
-! LLVMIR: %[[ins:.*]] = insertelement <4 x i32> undef, i32 %[[ele]], i32 0
-! LLVMIR: %[[y:.*]] = shufflevector <4 x i32> %[[ins]], <4 x i32> undef, <4 x i32> zeroinitializer
+! LLVMIR: %[[ins:.*]] = insertelement <4 x i32> poison, i32 %[[ele]], i32 0
+! LLVMIR: %[[y:.*]] = shufflevector <4 x i32> %[[ins]], <4 x i32> poison, <4 x i32> zeroinitializer
 ! LLVMIR: store <4 x i32> %[[y]], ptr %{{[0-9]}}, align 16
 end subroutine vec_splat_testu32i8
 
@@ -443,8 +443,8 @@ subroutine vec_splat_testu32i16(x)
 ! LLVMIR: %[[x:.*]] = load <4 x i32>, ptr %{{[0-9]}}, align 16
 ! LLVMIR-LE: %[[ele:.*]] = extractelement <4 x i32> %[[x]], i16 0
 ! LLVMIR-BE: %[[ele:.*]] = extractelement <4 x i32> %[[x]], i16 3
-! LLVMIR: %[[ins:.*]] = insertelement <4 x i32> undef, i32 %[[ele]], i32 0
-! LLVMIR: %[[y:.*]] = shufflevector <4 x i32> %[[ins]], <4 x i32> undef, <4 x i32> zeroinitializer
+! LLVMIR: %[[ins:.*]] = insertelement <4 x i32> poison, i32 %[[ele]], i32 0
+! LLVMIR: %[[y:.*]] = shufflevector <4 x i32> %[[ins]], <4 x i32> poison, <4 x i32> zeroinitializer
 ! LLVMIR: store <4 x i32> %[[y]], ptr %{{[0-9]}}, align 16
 end subroutine vec_splat_testu32i16
 
@@ -456,8 +456,8 @@ subroutine vec_splat_testu32i32(x)
 ! LLVMIR: %[[x:.*]] = load <4 x i32>, ptr %{{[0-9]}}, align 16
 ! LLVMIR-LE: %[[ele:.*]] = extractelement <4 x i32> %[[x]], i32 0
 ! LLVMIR-BE: %[[ele:.*]] = extractelement <4 x i32> %[[x]], i32 3
-! LLVMIR: %[[ins:.*]] = insertelement <4 x i32> undef, i32 %[[ele]], i32 0
-! LLVMIR: %[[y:.*]] = shufflevector <4 x i32> %[[ins]], <4 x i32> undef, <4 x i32> zeroinitializer
+! LLVMIR: %[[ins:.*]] = insertelement <4 x i32> poison, i32 %[[ele]], i32 0
+! LLVMIR: %[[y:.*]] = shufflevector <4 x i32> %[[ins]], <4 x i32> poison, <4 x i32> zeroinitializer
 ! LLVMIR: store <4 x i32> %[[y]], ptr %{{[0-9]}}, align 16
 end subroutine vec_splat_testu32i32
 
@@ -469,8 +469,8 @@ subroutine vec_splat_testu32i64(x)
 ! LLVMIR: %[[x:.*]] = load <4 x i32>, ptr %{{[0-9]}}, align 16
 ! LLVMIR-LE: %[[ele:.*]] = extractelement <4 x i32> %[[x]], i64 0
 ! LLVMIR-BE: %[[ele:.*]] = extractelement <4 x i32> %[[x]], i64 3
-! LLVMIR: %[[ins:.*]] = insertelement <4 x i32> undef, i32 %[[ele]], i32 0
-! LLVMIR: %[[y:.*]] = shufflevector <4 x i32> %[[ins]], <4 x i32> undef, <4 x i32> zeroinitializer
+! LLVMIR: %[[ins:.*]] = insertelement <4 x i32> poison, i32 %[[ele]], i32 0
+! LLVMIR: %[[y:.*]] = shufflevector <4 x i32> %[[ins]], <4 x i32> poison, <4 x i32> zeroinitializer
 ! LLVMIR: store <4 x i32> %[[y]], ptr %{{[0-9]}}, align 16
 end subroutine vec_splat_testu32i64
 
@@ -482,8 +482,8 @@ subroutine vec_splat_testu64i8(x)
 ! LLVMIR: %[[x:.*]] = load <2 x i64>, ptr %{{[0-9]}}, align 16
 ! LLVMIR-LE: %[[ele:.*]] = extractelement <2 x i64> %[[x]], i8 0
 ! LLVMIR-BE: %[[ele:.*]] = extractelement <2 x i64> %[[x]], i8 1
-! LLVMIR: %[[ins:.*]] = insertelement <2 x i64> undef, i64 %[[ele]], i32 0
-! LLVMIR: %[[y:.*]] = shufflevector <2 x i64> %[[ins]], <2 x i64> undef, <2 x i32> zeroinitializer
+! LLVMIR: %[[ins:.*]] = insertelement <2 x i64> poison, i64 %[[ele]], i32 0
+! LLVMIR: %[[y:.*]] = shufflevector <2 x i64> %[[ins]], <2 x i64> poison, <2 x i32> zeroinitializer
 ! LLVMIR: store <2 x i64> %[[y]], ptr %{{[0-9]}}, align 16
 end subroutine vec_splat_testu64i8
 
@@ -495,8 +495,8 @@ subroutine vec_splat_testu64i16(x)
 ! LLVMIR: %[[x:.*]] = load <2 x i64>, ptr %{{[0-9]}}, align 16
 ! LLVMIR-LE: %[[ele:.*]] = extractelement <2 x i64> %[[x]], i16 0
 ! LLVMIR-BE: %[[ele:.*]] = extractelement <2 x i64> %[[x]], i16 1
-! LLVMIR: %[[ins:.*]] = insertelement <2 x i64> undef, i64 %[[ele]], i32 0
-! LLVMIR: %[[y:.*]] = shufflevector <2 x i64> %[[ins]], <2 x i64> undef, <2 x i32> zeroinitializer
+! LLVMIR: %[[ins:.*]] = insertelement <2 x i64> poison, i64 %[[ele]], i32 0
+! LLVMIR: %[[y:.*]] = shufflevector <2 x i64> %[[ins]], <2 x i64> poison, <2 x i32> zeroinitializer
 ! LLVMIR: store <2 x i64> %[[y]], ptr %{{[0-9]}}, align 16
 end subroutine vec_splat_testu64i16
 
@@ -508,8 +508,8 @@ subroutine vec_splat_testu64i32(x)
 ! LLVMIR: %[[x:.*]] = load <2 x i64>, ptr %{{[0-9]}}, align 16
 ! LLVMIR-LE: %[[ele:.*]] = extractelement <2 x i64> %[[x]], i32 0
 ! LLVMIR-BE: %[[ele:.*]] = extractelement <2 x i64> %[[x]], i32 1
-! LLVMIR: %[[ins:.*]] = insertelement <2 x i64> undef, i64 %[[ele]], i32 0
-! LLVMIR: %[[y:.*]] = shufflevector <2 x i64> %[[ins]], <2 x i64> undef, <2 x i32> zeroinitializer
+! LLVMIR: %[[ins:.*]] = insertelement <2 x i64> poison, i64 %[[ele]], i32 0
+! LLVMIR: %[[y:.*]] = shufflevector <2 x i64> %[[ins]], <2 x i64> poison, <2 x i32> zeroinitializer
 ! LLVMIR: store <2 x i64> %[[y]], ptr %{{[0-9]}}, align 16
 end subroutine vec_splat_testu64i32
 
@@ -521,8 +521,8 @@ subroutine vec_splat_testu64i64(x)
 ! LLVMIR: %[[x:.*]] = load <2 x i64>, ptr %{{[0-9]}}, align 16
 ! LLVMIR-LE: %[[ele:.*]] = extractelement <2 x i64> %[[x]], i64 0
 ! LLVMIR-BE: %[[ele:.*]] = extractelement <2 x i64> %[[x]], i64 1
-! LLVMIR: %[[ins:.*]] = insertelement <2 x i64> undef, i64 %[[ele]], i32 0
-! LLVMIR: %[[y:.*]] = shufflevector <2 x i64> %[[ins]], <2 x i64> undef, <2 x i32> zeroinitializer
+! LLVMIR: %[[ins:.*]] = insertelement <2 x i64> poison, i64 %[[ele]], i32 0
+! LLVMIR: %[[y:.*]] = shufflevector <2 x i64> %[[ins]], <2 x i64> poison, <2 x i32> zeroinitializer
 ! LLVMIR: store <2 x i64> %[[y]], ptr %{{[0-9]}}, align 16
 end subroutine vec_splat_testu64i64
 
@@ -537,8 +537,8 @@ subroutine vec_splats_testi8(x)
   y = vec_splats(x)
 
 ! LLVMIR: %[[x:.*]] = load i8, ptr %{{[0-9]}}, align 1
-! LLVMIR: %[[ins:.*]] = insertelement <16 x i8> undef, i8 %[[x]], i32 0
-! LLVMIR: %[[y:.*]] = shufflevector <16 x i8> %[[ins]], <16 x i8> undef, <16 x i32> zeroinitializer
+! LLVMIR: %[[ins:.*]] = insertelement <16 x i8> poison, i8 %[[x]], i32 0
+! LLVMIR: %[[y:.*]] = shufflevector <16 x i8> %[[ins]], <16 x i8> poison, <16 x i32> zeroinitializer
 ! LLVMIR: store <16 x i8> %[[y]], ptr %{{[0-9]}}, align 16
 end subroutine vec_splats_testi8
 
@@ -549,8 +549,8 @@ subroutine vec_splats_testi16(x)
   y = vec_splats(x)
 
 ! LLVMIR: %[[x:.*]] = load i16, ptr %{{[0-9]}}, align 2
-! LLVMIR: %[[ins:.*]] = insertelement <8 x i16> undef, i16 %[[x]], i32 0
-! LLVMIR: %[[y:.*]] = shufflevector <8 x i16> %[[ins]], <8 x i16> undef, <8 x i32> zeroinitializer
+! LLVMIR: %[[ins:.*]] = insertelement <8 x i16> poison, i16 %[[x]], i32 0
+! LLVMIR: %[[y:.*]] = shufflevector <8 x i16> %[[ins]], <8 x i16> poison, <8 x i32> zeroinitializer
 ! LLVMIR: store <8 x i16> %[[y]], ptr %{{[0-9]}}, align 16
 end subroutine vec_splats_testi16
 
@@ -561,8 +561,8 @@ subroutine vec_splats_testi32(x)
   y = vec_splats(x)
 
 ! LLVMIR: %[[x:.*]] = load i32, ptr %{{[0-9]}}, align 4
-! LLVMIR: %[[ins:.*]] = insertelement <4 x i32> undef, i32 %[[x]], i32 0
-! LLVMIR: %[[y:.*]] = shufflevector <4 x i32> %[[ins]], <4 x i32> undef, <4 x i32> zeroinitializer
+! LLVMIR: %[[ins:.*]] = insertelement <4 x i32> poison, i32 %[[x]], i32 0
+! LLVMIR: %[[y:.*]] = shufflevector <4 x i32> %[[ins]], <4 x i32> poison, <4 x i32> zeroinitializer
 ! LLVMIR: store <4 x i32> %[[y]], ptr %{{[0-9]}}, align 16
 end subroutine vec_splats_testi32
 
@@ -573,8 +573,8 @@ subroutine vec_splats_testi64(x)
   y = vec_splats(x)
 
 ! LLVMIR: %[[x:.*]] = load i64, ptr %{{[0-9]}}, align 8
-! LLVMIR: %[[ins:.*]] = insertelement <2 x i64> undef, i64 %[[x]], i32 0
-! LLVMIR: %[[y:.*]] = shufflevector <2 x i64> %[[ins]], <2 x i64> undef, <2 x i32> zeroinitializer
+! LLVMIR: %[[ins:.*]] = insertelement <2 x i64> poison, i64 %[[x]], i32 0
+! LLVMIR: %[[y:.*]] = shufflevector <2 x i64> %[[ins]], <2 x i64> poison, <2 x i32> zeroinitializer
 ! LLVMIR: store <2 x i64> %[[y]], ptr %{{[0-9]}}, align 16
 end subroutine vec_splats_testi64
 
@@ -585,8 +585,8 @@ subroutine vec_splats_testf32(x)
   y = vec_splats(x)
 
 ! LLVMIR: %[[x:.*]] = load float, ptr %{{[0-9]}}, align 4
-! LLVMIR: %[[ins:.*]] = insertelement <4 x float> undef, float %[[x]], i32 0
-! LLVMIR: %[[y:.*]] = shufflevector <4 x float> %[[ins]], <4 x float> undef, <4 x i32> zeroinitializer
+! LLVMIR: %[[ins:.*]] = insertelement <4 x float> poison, float %[[x]], i32 0
+! LLVMIR: %[[y:.*]] = shufflevector <4 x float> %[[ins]], <4 x float> poison, <4 x i32> zeroinitializer
 ! LLVMIR: store <4 x float> %[[y]], ptr %{{[0-9]}}, align 16
 end subroutine vec_splats_testf32
 
@@ -597,8 +597,8 @@ subroutine vec_splats_testf64(x)
   y = vec_splats(x)
 
 ! LLVMIR: %[[x:.*]] = load double, ptr %{{[0-9]}}, align 8
-! LLVMIR: %[[ins:.*]] = insertelement <2 x double> undef, double %[[x]], i32 0
-! LLVMIR: %[[y:.*]] = shufflevector <2 x double> %[[ins]], <2 x double> undef, <2 x i32> zeroinitializer
+! LLVMIR: %[[ins:.*]] = insertelement <2 x double> poison, double %[[x]], i32 0
+! LLVMIR: %[[y:.*]] = shufflevector <2 x double> %[[ins]], <2 x double> poison, <2 x i32> zeroinitializer
 ! LLVMIR: store <2 x double> %[[y]], ptr %{{[0-9]}}, align 16
 end subroutine vec_splats_testf64
 
diff --git a/flang/test/Lower/array.f90 b/flang/test/Lower/array.f90
index ec65a3c4071ec..710175739b3a8 100644
--- a/flang/test/Lower/array.f90
+++ b/flang/test/Lower/array.f90
@@ -96,7 +96,7 @@ subroutine s(i,j,k,ii,jj,kk,a1,a2,a3,a4,a5,a6,a7)
   
 end subroutine s
 
-! CHECK-LABEL range
+! CHECK-LABEL: range
 subroutine range()
   ! Compile-time initalized arrays
   integer, dimension(10) :: a0
@@ -114,6 +114,29 @@ subroutine range()
   data c1/6 * (0.0, 0.0)/
 end subroutine range
 
+! CHECK-LABEL: rangeglobal
+subroutine rangeGlobal()
+  integer, dimension(6) :: a0 = (/ 1, 1, 2, 2, 3, 3 /)
+
+end subroutine rangeGlobal
+
+! CHECK-LABEL: hugeglobal
+subroutine hugeGlobal()
+  integer, parameter :: D = 500
+  integer, dimension(D, D) :: a
+
+  a = reshape((/(i, i = 1, D * D)/), shape(a))
+end subroutine hugeGlobal
+
+block data
+  real(selected_real_kind(6)) :: x(5,5)
+  common /block/ x
+  data x(1,1), x(2,1), x(3,1) / 1, 1, 0 /
+  data x(1,2), x(2,2), x(4,2) / 1, 1, 2.4 /
+  data x(1,3), x(2,3), x(4,3) / 1, 1, 2.4 /
+  data x(4,4) / 2.4 /
+end
+
 ! c1 data
 ! CHECK: fir.global internal @_QFrangeEc1(dense<(0.000000e+00,0.000000e+00)> : tensor<3x2xcomplex<f32>>) : !fir.array<2x3xcomplex<f32>>
 
@@ -132,27 +155,6 @@ end subroutine range
 ! c0 array constructor
 ! CHECK: fir.global internal @_QQro.2x3xz4.{{.*}}(dense<{{\[}}[(1.000000e+00,1.500000e+00), (2.000000e+00,2.500000e+00)], [(3.000000e+00,3.500000e+00), (4.000000e+00,4.500000e+00)], [(5.000000e+00,5.500000e+00), (6.000000e+00,6.500000e+00)]]> : tensor<3x2xcomplex<f32>>) constant : !fir.array<2x3xcomplex<f32>>
 
-! CHECK-LABEL rangeGlobal
-subroutine rangeGlobal()
 ! CHECK: fir.global internal @_QFrangeglobal{{.*}}(dense<[1, 1, 2, 2, 3, 3]> : tensor<6xi32>) : !fir.array<6xi32>
-  integer, dimension(6) :: a0 = (/ 1, 1, 2, 2, 3, 3 /)
-
-end subroutine rangeGlobal
-
-! CHECK-LABEL hugeGlobal
-subroutine hugeGlobal()
-  integer, parameter :: D = 500
-  integer, dimension(D, D) :: a
 
 ! CHECK: fir.global internal @_QQro.500x500xi4.{{.*}}(dense<{{.*}}> : tensor<500x500xi32>) constant : !fir.array<500x500xi32>
-  a = reshape((/(i, i = 1, D * D)/), shape(a))
-end subroutine hugeGlobal
-
-block data
-  real(selected_real_kind(6)) :: x(5,5)
-  common /block/ x
-  data x(1,1), x(2,1), x(3,1) / 1, 1, 0 /
-  data x(1,2), x(2,2), x(4,2) / 1, 1, 2.4 /
-  data x(1,3), x(2,3), x(4,3) / 1, 1, 2.4 /
-  data x(4,4) / 2.4 /
-end
diff --git a/flang/test/Lower/assignment.f90 b/flang/test/Lower/assignment.f90
index dde5110bdfa9a..defeec5b7edfe 100644
--- a/flang/test/Lower/assignment.f90
+++ b/flang/test/Lower/assignment.f90
@@ -1,4 +1,4 @@
-! RUN: %flang_fc1 %s -o "-" -emit-fir -cpp -flang-deprecated-no-hlfir | FileCheck %s --check-prefixes=CHECK%if target=x86_64{{.*}} %{,CHECK-X86-64%}
+! RUN: %flang_fc1 %s -o "-" -emit-fir -cpp -flang-deprecated-no-hlfir | FileCheck %s --check-prefixes=CHECK,%if flang-supports-f128-math %{F128%} %else %{F64%}%if target=x86_64-unknown-linux{{.*}} %{,CHECK-X86-64%}
 
 subroutine sub1(a)
   integer :: a
@@ -261,27 +261,29 @@ real function divf(a, b)
 ! CHECK:         return %[[RET]] : complex<f32>
 
 subroutine real_constant()
+  integer, parameter :: rk = merge(16, 8, selected_real_kind(33, 4931)==16)
   real(2) :: a
   real(4) :: b
   real(8) :: c
 #if __x86_64__
   real(10) :: d
 #endif
-  real(16) :: e
+  real(rk) :: e
   a = 2.0_2
   b = 4.0_4
   c = 8.0_8
 #if __x86_64__
   d = 10.0_10
 #endif
-  e = 16.0_16
+  e = 16.0_rk
 end
 
 ! CHECK: %[[A:.*]] = fir.alloca f16
 ! CHECK: %[[B:.*]] = fir.alloca f32
 ! CHECK: %[[C:.*]] = fir.alloca f64
 ! CHECK-X86-64: %[[D:.*]] = fir.alloca f80
-! CHECK: %[[E:.*]] = fir.alloca f128
+! F128: %[[E:.*]] = fir.alloca f128
+! F64: %[[E:.*]] = fir.alloca f64
 ! CHECK: %[[C2:.*]] = arith.constant 2.000000e+00 : f16
 ! CHECK: fir.store %[[C2]] to %[[A]] : !fir.ref<f16>
 ! CHECK: %[[C4:.*]] = arith.constant 4.000000e+00 : f32
@@ -290,8 +292,10 @@ subroutine real_constant()
 ! CHECK: fir.store %[[C8]] to %[[C]] : !fir.ref<f64>
 ! CHECK-X86-64: %[[C10:.*]] = arith.constant 1.000000e+01 : f80
 ! CHECK-X86-64: fir.store %[[C10]] to %[[D]] : !fir.ref<f80>
-! CHECK: %[[C16:.*]] = arith.constant 1.600000e+01 : f128
-! CHECK: fir.store %[[C16]] to %[[E]] : !fir.ref<f128>
+! F128: %[[C16:.*]] = arith.constant 1.600000e+01 : f128
+! F64: %[[C16:.*]] = arith.constant 1.600000e+01 : f64
+! F128: fir.store %[[C16]] to %[[E]] : !fir.ref<f128>
+! F64: fir.store %[[C16]] to %[[E]] : !fir.ref<f64>
 
 subroutine complex_constant()
   complex(4) :: a
diff --git a/flang/test/Lower/basic-function.f90 b/flang/test/Lower/basic-function.f90
index 5f2fabe1b325d..c250a988bf7b3 100644
--- a/flang/test/Lower/basic-function.f90
+++ b/flang/test/Lower/basic-function.f90
@@ -1,4 +1,4 @@
-! RUN: bbc %s -o "-" -emit-fir -hlfir=false | FileCheck %s
+! RUN: bbc -emit-hlfir %s -o - | FileCheck %s --check-prefixes=CHECK%if target=x86_64{{.*}} %{,CHECK-KIND10%}%if flang-supports-f128-math %{,CHECK-KIND16%}
 
 integer(1) function fct1()
 end
@@ -102,15 +102,19 @@ real(8) function rfct4()
 ! CHECK-LABEL: func @_QPrfct4() -> f64
 ! CHECK:         return %{{.*}} : f64
 
-real(10) function rfct5()
+function rfct5()
+  integer, parameter :: kind10 = merge(10, 4, selected_real_kind(p=18).eq.10)
+  real(kind10) :: rfct5
 end
-! CHECK-LABEL: func @_QPrfct5() -> f80
-! CHECK:         return %{{.*}} : f80
+! CHECK-KIND10-LABEL: func @_QPrfct5() -> f80
+! CHECK-KIND10:         return %{{.*}} : f80
 
-real(16) function rfct6()
+function rfct6()
+  integer, parameter :: kind16 = merge(16, 4, selected_real_kind(p=33).eq.16)
+  real(kind16) :: rfct6
 end
-! CHECK-LABEL: func @_QPrfct6() -> f128
-! CHECK:         return %{{.*}} : f128
+! CHECK-KIND16-LABEL: func @_QPrfct6() -> f128
+! CHECK-KIND16:         return %{{.*}} : f128
 
 complex(2) function cplxfct1()
 end
@@ -132,15 +136,19 @@ real(16) function rfct6()
 ! CHECK-LABEL: func @_QPcplxfct4() -> complex<f64>
 ! CHECK:         return %{{.*}} : complex<f64>
 
-complex(10) function cplxfct5()
+function cplxfct5()
+  integer, parameter :: kind10 = merge(10, 4, selected_real_kind(p=18).eq.10)
+  complex(kind10) :: cplxfct5
 end
-! CHECK-LABEL: func @_QPcplxfct5() -> complex<f80>
-! CHECK:         return %{{.*}} : complex<f80>
+! CHECK-KIND10-LABEL: func @_QPcplxfct5() -> complex<f80>
+! CHECK-KIND10:         return %{{.*}} : complex<f80>
 
-complex(16) function cplxfct6()
+function cplxfct6()
+  integer, parameter :: kind16 = merge(16, 4, selected_real_kind(p=33).eq.16)
+  complex(kind16) :: cplxfct6
 end
-! CHECK-LABEL: func @_QPcplxfct6() -> complex<f128>
-! CHECK:         return %{{.*}} : complex<f128>
+! CHECK-KIND16-LABEL: func @_QPcplxfct6() -> complex<f128>
+! CHECK-KIND16:         return %{{.*}} : complex<f128>
 
 function fct_with_character_return(i)
   character(10) :: fct_with_character_return
diff --git a/flang/test/Lower/components.f90 b/flang/test/Lower/components.f90
index 28e836c5d1045..f9297fcfed4ff 100644
--- a/flang/test/Lower/components.f90
+++ b/flang/test/Lower/components.f90
@@ -115,7 +115,7 @@ subroutine issue772(a, x)
   ! CHECK: fir.call @_QPibar()
   ! CHECK-NOT: fir.call @_QPibar()
   print *, a(20)%b(1:ibar():1)
-  ! CHECK return
+  ! CHECK: return
 end subroutine
 
 ! -----------------------------------------------------------------------------
diff --git a/flang/test/Lower/control-flow.f90 b/flang/test/Lower/control-flow.f90
index d605aaf0a9d94..ef66c9e22c8d1 100644
--- a/flang/test/Lower/control-flow.f90
+++ b/flang/test/Lower/control-flow.f90
@@ -3,7 +3,7 @@
 ! RUN: bbc -emit-fir -hlfir=false %s -o - | FileCheck %s
 
 ! check the lowering of a RETURN in the body of a SUBROUTINE
-! CHECK-LABEL one
+! CHECK-LABEL: one
 subroutine one(a,b,c)
   d = 1.0
   if (a .ne. b) then
diff --git a/flang/test/Lower/derived-assignments.f90 b/flang/test/Lower/derived-assignments.f90
index 1048e6199451a..90c360978fd2a 100644
--- a/flang/test/Lower/derived-assignments.f90
+++ b/flang/test/Lower/derived-assignments.f90
@@ -220,17 +220,17 @@ subroutine test_alloc_comp(t1, t2)
 !  end type
 
 !contains
-!  ! cHECK-LABEL: func @_QMcomponent_with_user_def_assignPtest(
-!  ! cHECK-SAME: %[[t1:.*]]: !fir.ref<!fir.type<_QMcomponent_with_user_def_assignTt{a:!fir.type<_QMcomponent_with_user_def_assignTt0{i:i32,j:i32}>,i:i32}>>{{.*}}, %[[t2:.*]]: !fir.ref<!fir.type<_QMcomponent_with_user_def_assignTt{a:!fir.type<_QMcomponent_with_user_def_assignTt0{i:i32,j:i32}>,i:i32}>>{{.*}}) {
+! COM: CHECK-LABEL: func @_QMcomponent_with_user_def_assignPtest(
+! COM: CHECK-SAME: %[[t1:.*]]: !fir.ref<!fir.type<_QMcomponent_with_user_def_assignTt{a:!fir.type<_QMcomponent_with_user_def_assignTt0{i:i32,j:i32}>,i:i32}>>{{.*}}, %[[t2:.*]]: !fir.ref<!fir.type<_QMcomponent_with_user_def_assignTt{a:!fir.type<_QMcomponent_with_user_def_assignTt0{i:i32,j:i32}>,i:i32}>>{{.*}}) {
 !  subroutine test(t1, t2)
 !    type(t) :: t1, t2
-!    ! cHECK: %[[tmpBox:.*]] = fir.alloca !fir.box<!fir.type<_QMcomponent_with_user_def_assignTt{{.*}}>>
-!    ! cHECK: %[[t1Box:.*]] = fir.embox %[[t1]] : (!fir.ref<!fir.type<_QMcomponent_with_user_def_assignTt{{.*}}>>) -> !fir.box<!fir.type<_QMcomponent_with_user_def_assignTt{{.*}}>>
-!    ! cHECK: %[[t2Box:.*]] = fir.embox %[[t2]] : (!fir.ref<!fir.type<_QMcomponent_with_user_def_assignTt{{.*}}>>) -> !fir.box<!fir.type<_QMcomponent_with_user_def_assignTt{{.*}}>>
-!    ! cHECK: fir.store %[[t1Box]] to %[[tmpBox]] : !fir.ref<!fir.box<!fir.type<_QMcomponent_with_user_def_assignTt{{.*}}>>>
-!    ! cHECK: %[[lhs:.*]] = fir.convert %[[tmpBox]] : (!fir.ref<!fir.box<!fir.type<_QMcomponent_with_user_def_assignTt{{.*}}>>>) -> !fir.ref<!fir.box<none>>
-!    ! cHECK: %[[rhs:.*]] = fir.convert %[[t2Box]] : (!fir.box<!fir.type<_QMcomponent_with_user_def_assignTt{{.*}}>>) -> !fir.box<none>
-!    ! cHECK: fir.call @_FortranAAssign(%[[lhs]], %[[rhs]], %{{.*}}, %{{.*}}) {{.*}}: (!fir.ref<!fir.box<none>>, !fir.box<none>, !fir.ref<i8>, i32) -> ()
+! COM: CHECK: %[[tmpBox:.*]] = fir.alloca !fir.box<!fir.type<_QMcomponent_with_user_def_assignTt{{.*}}>>
+! COM: CHECK: %[[t1Box:.*]] = fir.embox %[[t1]] : (!fir.ref<!fir.type<_QMcomponent_with_user_def_assignTt{{.*}}>>) -> !fir.box<!fir.type<_QMcomponent_with_user_def_assignTt{{.*}}>>
+! COM: CHECK: %[[t2Box:.*]] = fir.embox %[[t2]] : (!fir.ref<!fir.type<_QMcomponent_with_user_def_assignTt{{.*}}>>) -> !fir.box<!fir.type<_QMcomponent_with_user_def_assignTt{{.*}}>>
+! COM: CHECK: fir.store %[[t1Box]] to %[[tmpBox]] : !fir.ref<!fir.box<!fir.type<_QMcomponent_with_user_def_assignTt{{.*}}>>>
+! COM: CHECK: %[[lhs:.*]] = fir.convert %[[tmpBox]] : (!fir.ref<!fir.box<!fir.type<_QMcomponent_with_user_def_assignTt{{.*}}>>>) -> !fir.ref<!fir.box<none>>
+! COM: CHECK: %[[rhs:.*]] = fir.convert %[[t2Box]] : (!fir.box<!fir.type<_QMcomponent_with_user_def_assignTt{{.*}}>>) -> !fir.box<none>
+! COM: CHECK: fir.call @_FortranAAssign(%[[lhs]], %[[rhs]], %{{.*}}, %{{.*}}) {{.*}}: (!fir.ref<!fir.box<none>>, !fir.box<none>, !fir.ref<i8>, i32) -> ()
 !    t1 = t2
 !  end subroutine
 !end module
diff --git a/flang/test/Lower/entry-statement-init.f90 b/flang/test/Lower/entry-statement-init.f90
new file mode 100644
index 0000000000000..731ccebef6873
--- /dev/null
+++ b/flang/test/Lower/entry-statement-init.f90
@@ -0,0 +1,26 @@
+! RUN: bbc -emit-hlfir -o - %s | FileCheck %s
+
+! Test initialization and finalizations of dummy arguments in entry statements.
+
+module m
+  type t
+  end type
+contains
+ subroutine test1(x)
+   class(t), intent(out) :: x
+   entry test1_entry()
+ end subroutine
+ subroutine test2(x)
+   class(t), intent(out) :: x
+   entry test2_entry(x)
+ end subroutine
+end module
+! CHECK-LABEL:   func.func @_QMmPtest1_entry(
+! CHECK-NOT: Destroy
+! CHECK-NOT: Initialize
+! CHECK:           return
+
+! CHECK-LABEL:   func.func @_QMmPtest2_entry(
+! CHECK: Destroy
+! CHECK: Initialize
+! CHECK:           return
diff --git a/flang/test/Lower/equivalence-static-init.f90 b/flang/test/Lower/equivalence-static-init.f90
index 4c52a4a7d8448..1a0f53a4d893d 100644
--- a/flang/test/Lower/equivalence-static-init.f90
+++ b/flang/test/Lower/equivalence-static-init.f90
@@ -10,7 +10,7 @@ module module_without_init
 ! CHECK-LABEL: fir.global @_QMmodule_without_initEi : !fir.array<8xi8> {
   ! CHECK: %0 = fir.zero_bits !fir.array<8xi8>
   ! CHECK: fir.has_value %0 : !fir.array<8xi8>
-! CHECK}
+! CHECK: }
 
 
 subroutine test_eqv_init
diff --git a/flang/test/Lower/forall/array-subscripts.f90 b/flang/test/Lower/forall/array-subscripts.f90
index a0ae0500b5fdc..60bac87c7ae54 100644
--- a/flang/test/Lower/forall/array-subscripts.f90
+++ b/flang/test/Lower/forall/array-subscripts.f90
@@ -18,4 +18,4 @@
 ! CHECK: = fir.array_fetch %[[a2]], %{{.*}}, %{{.*}} : (!fir.array<4x4xi32>, index, index) -> i32
 ! CHECK: = fir.array_fetch %[[a3]], %{{.*}}, %{{.*}} : (!fir.array<4x4xi32>, index, index) -> i32
 ! CHECK: = fir.array_update %{{.*}}, %{{.*}}, %{{.*}} : (!fir.array<4x4xi32>, i32, index, index) -> !fir.array<4x4xi32>
-! CHECK : fir.array_merge_store %[[a1]], %[[av]] to %[[a]] : !fir.array<4x4xi32>, !fir.array<4x4xi32>, !fir.ref<!fir.array<4x4xi32>>
+! CHECK: fir.array_merge_store %[[a1]], %[[av]] to %[[a]] : !fir.array<4x4xi32>, !fir.array<4x4xi32>, !fir.ref<!fir.array<4x4xi32>>
diff --git a/flang/test/Lower/fsave-main-program.f90 b/flang/test/Lower/fsave-main-program.f90
index 17fc1b02f5068..e89244c3c7c51 100644
--- a/flang/test/Lower/fsave-main-program.f90
+++ b/flang/test/Lower/fsave-main-program.f90
@@ -1,6 +1,7 @@
 ! Test -fsave-main-program switch.
 ! RUN: %flang_fc1 -emit-hlfir -o - %s | FileCheck --check-prefix=CHECK-DEFAULT %s
 ! RUN: %flang_fc1 -fsave-main-program -emit-hlfir -o - %s | FileCheck --check-prefix=CHECK-SAVE %s
+! RUN: %flang_fc1 -fsave-main-program -fno-save-main-program -emit-hlfir -o - %s | FileCheck --check-prefix=CHECK-DEFAULT %s
 program test
 integer :: i
 call foo(i)
diff --git a/flang/test/Lower/math-lowering/abs.f90 b/flang/test/Lower/math-lowering/abs.f90
index ead603c1d13e8..9d3b8b92cfdd6 100644
--- a/flang/test/Lower/math-lowering/abs.f90
+++ b/flang/test/Lower/math-lowering/abs.f90
@@ -1,9 +1,9 @@
-! RUN: bbc -emit-fir %s -o - --math-runtime=fast | FileCheck --check-prefixes=ALL,FAST %s
-! RUN: %flang_fc1 -emit-fir -mllvm -math-runtime=fast %s -o - | FileCheck --check-prefixes=ALL,FAST %s
-! RUN: bbc -emit-fir %s -o - --math-runtime=relaxed | FileCheck --check-prefixes=ALL,RELAXED %s
-! RUN: %flang_fc1 -emit-fir -mllvm -math-runtime=relaxed %s -o - | FileCheck --check-prefixes=ALL,RELAXED %s
-! RUN: bbc -emit-fir %s -o - --math-runtime=precise | FileCheck --check-prefixes=ALL,PRECISE %s
-! RUN: %flang_fc1 -emit-fir -mllvm -math-runtime=precise %s -o - | FileCheck --check-prefixes=ALL,PRECISE %s
+! RUN: bbc -emit-fir %s -o - --math-runtime=fast | FileCheck --check-prefixes=ALL,FAST-%if flang-supports-f128-math %{F128%} %else %{F64%} %s
+! RUN: %flang_fc1 -emit-fir -mllvm -math-runtime=fast %s -o - | FileCheck --check-prefixes=ALL,FAST,FAST-%if flang-supports-f128-math %{F128%} %else %{F64%} %s
+! RUN: bbc -emit-fir %s -o - --math-runtime=relaxed | FileCheck --check-prefixes=ALL,RELAXED,RELAXED-%if flang-supports-f128-math %{F128%} %else %{F64%} %s
+! RUN: %flang_fc1 -emit-fir -mllvm -math-runtime=relaxed %s -o - | FileCheck --check-prefixes=ALL,RELAXED,RELAXED-%if flang-supports-f128-math %{F128%} %else %{F64%} %s
+! RUN: bbc -emit-fir %s -o - --math-runtime=precise | FileCheck --check-prefixes=ALL,PRECISE,PRECISE-%if flang-supports-f128-math %{F128%} %else %{F64%} %s
+! RUN: %flang_fc1 -emit-fir -mllvm -math-runtime=precise %s -o - | FileCheck --check-prefixes=ALL,PRECISE,PRECISE-%if flang-supports-f128-math %{F128%} %else %{F64%} %s
 
 function test_real4(x)
   real :: x, test_real4
@@ -26,13 +26,17 @@ function test_real8(x)
 ! PRECISE: {{%[A-Za-z0-9._]+}} = fir.call @fabs({{%[A-Za-z0-9._]+}}) {{.*}}: (f64) -> f64
 
 function test_real16(x)
-  real(16) :: x, test_real16
+  integer, parameter :: rk = merge(16, 8, selected_real_kind(33, 4931)==16)
+  real(rk) :: x, test_real16
   test_real16 = abs(x)
 end function
 ! ALL-LABEL: @_QPtest_real16
-! FAST: {{%[A-Za-z0-9._]+}} = math.absf {{%[A-Za-z0-9._]+}} {{.*}}: f128
-! RELAXED: {{%[A-Za-z0-9._]+}} = math.absf {{%[A-Za-z0-9._]+}} {{.*}}: f128
-! PRECISE: {{%[A-Za-z0-9._]+}} = fir.call @llvm.fabs.f128({{%[A-Za-z0-9._]+}}) {{.*}}: (f128) -> f128
+! FAST-F128: {{%[A-Za-z0-9._]+}} = math.absf {{%[A-Za-z0-9._]+}} {{.*}}: f128
+! FAST-F64: {{%[A-Za-z0-9._]+}} = math.absf {{%[A-Za-z0-9._]+}} {{.*}}: f64
+! RELAXED-F128: {{%[A-Za-z0-9._]+}} = math.absf {{%[A-Za-z0-9._]+}} {{.*}}: f128
+! RELAXED-F64: {{%[A-Za-z0-9._]+}} = math.absf {{%[A-Za-z0-9._]+}} {{.*}}: f64
+! PRECISE-F128: {{%[A-Za-z0-9._]+}} = fir.call @llvm.fabs.f128({{%[A-Za-z0-9._]+}}) {{.*}}: (f128) -> f128
+! PRECISE-F64: {{%[A-Za-z0-9._]+}} = fir.call @fabs({{%[A-Za-z0-9._]+}}) {{.*}}: (f64) -> f64
 
 function test_complex4(c)
   complex(4) :: c, test_complex4
@@ -50,6 +54,6 @@ function test_complex8(c)
 
 ! PRECISE-DAG: func.func private @fabsf(f32) -> f32 attributes {fir.bindc_name = "fabsf", fir.runtime}
 ! PRECISE-DAG: func.func private @fabs(f64) -> f64 attributes {fir.bindc_name = "fabs", fir.runtime}
-! PRECISE-DAG: func.func private @llvm.fabs.f128(f128) -> f128 attributes {fir.bindc_name = "llvm.fabs.f128", fir.runtime}
+! PRECISE-F128-DAG: func.func private @llvm.fabs.f128(f128) -> f128 attributes {fir.bindc_name = "llvm.fabs.f128", fir.runtime}
 ! PRECISE-DAG: func.func private @cabsf(complex<f32>) -> f32 attributes {fir.bindc_name = "cabsf", fir.runtime}
 ! PRECISE-DAG: func.func private @cabs(complex<f64>) -> f64 attributes {fir.bindc_name = "cabs", fir.runtime}
diff --git a/flang/test/Lower/math-lowering/aint.f90 b/flang/test/Lower/math-lowering/aint.f90
index e8b17aad675c1..6c7809f8ea1a3 100644
--- a/flang/test/Lower/math-lowering/aint.f90
+++ b/flang/test/Lower/math-lowering/aint.f90
@@ -1,9 +1,10 @@
-! RUN: bbc -emit-fir %s -o - --math-runtime=fast | FileCheck --check-prefixes=ALL %s
-! RUN: %flang_fc1 -emit-fir -mllvm -math-runtime=fast %s -o - | FileCheck --check-prefixes=ALL %s
-! RUN: bbc -emit-fir %s -o - --math-runtime=relaxed | FileCheck --check-prefixes=ALL %s
-! RUN: %flang_fc1 -emit-fir -mllvm -math-runtime=relaxed %s -o - | FileCheck --check-prefixes=ALL %s
-! RUN: bbc -emit-fir %s -o - --math-runtime=precise | FileCheck --check-prefixes=ALL %s
-! RUN: %flang_fc1 -emit-fir -mllvm -math-runtime=precise %s -o - | FileCheck --check-prefixes=ALL %s
+! REQUIRES: x86_64-registered-target
+! RUN: bbc -target x86_64-unknown-linux-gnu -emit-fir %s -o - --math-runtime=fast | FileCheck --check-prefixes=ALL %s
+! RUN: %flang_fc1 -target x86_64-unknown-linux-gnu -emit-fir -mllvm -math-runtime=fast %s -o - | FileCheck --check-prefixes=ALL %s
+! RUN: bbc -target x86_64-unknown-linux-gnu -emit-fir %s -o - --math-runtime=relaxed | FileCheck --check-prefixes=ALL %s
+! RUN: %flang_fc1 -target x86_64-unknown-linux-gnu -emit-fir -mllvm -math-runtime=relaxed %s -o - | FileCheck --check-prefixes=ALL %s
+! RUN: bbc -target x86_64-unknown-linux-gnu -emit-fir %s -o - --math-runtime=precise | FileCheck --check-prefixes=ALL %s
+! RUN: %flang_fc1 -target x86_64-unknown-linux-gnu -emit-fir -mllvm -math-runtime=precise %s -o - | FileCheck --check-prefixes=ALL %s
 
 function test_real4(x)
   real :: x, test_real4
@@ -21,14 +22,6 @@ function test_real8(x)
 ! ALL-LABEL: @_QPtest_real8
 ! ALL: {{%[A-Za-z0-9._]+}} = fir.call @llvm.trunc.f64({{%[A-Za-z0-9._]+}}) {{.*}}: (f64) -> f64
 
-function test_real10(x)
-  real(10) :: x, test_real10
-  test_real10 = aint(x)
-end function
-
-! ALL-LABEL: @_QPtest_real10
-! ALL: {{%[A-Za-z0-9._]+}} = fir.call @llvm.trunc.f80({{%[A-Za-z0-9._]+}}) {{.*}}: (f80) -> f80
-
 ! TODO: wait until fp128 is supported well in llvm.trunc
 !function test_real16(x)
 !  real(16) :: x, test_real16
@@ -37,4 +30,3 @@ function test_real10(x)
 
 ! ALL-DAG: func.func private @llvm.trunc.f32(f32) -> f32 attributes {fir.bindc_name = "llvm.trunc.f32", fir.runtime}
 ! ALL-DAG: func.func private @llvm.trunc.f64(f64) -> f64 attributes {fir.bindc_name = "llvm.trunc.f64", fir.runtime}
-! ALL-DAG: func.func private @llvm.trunc.f80(f80) -> f80 attributes {fir.bindc_name = "llvm.trunc.f80", fir.runtime}
diff --git a/flang/test/Lower/math-lowering/anint.f90 b/flang/test/Lower/math-lowering/anint.f90
index 45dc3ef1d858e..f39009de7e4f9 100644
--- a/flang/test/Lower/math-lowering/anint.f90
+++ b/flang/test/Lower/math-lowering/anint.f90
@@ -1,9 +1,11 @@
-! RUN: bbc -emit-fir %s -o - --math-runtime=fast | FileCheck --check-prefixes=ALL,FAST %s
-! RUN: %flang_fc1 -emit-fir -mllvm -math-runtime=fast %s -o - | FileCheck --check-prefixes=ALL,FAST %s
-! RUN: bbc -emit-fir %s -o - --math-runtime=relaxed | FileCheck --check-prefixes=ALL,RELAXED %s
-! RUN: %flang_fc1 -emit-fir -mllvm -math-runtime=relaxed %s -o - | FileCheck --check-prefixes=ALL,RELAXED %s
-! RUN: bbc -emit-fir %s -o - --math-runtime=precise | FileCheck --check-prefixes=ALL,PRECISE %s
-! RUN: %flang_fc1 -emit-fir -mllvm -math-runtime=precise %s -o - | FileCheck --check-prefixes=ALL,PRECISE %s
+! RUN: %flang_fc1 -emit-hlfir -o - -mllvm -math-runtime=fast %s \
+! RUN: | FileCheck %s --check-prefixes=ALL,FAST%if target=x86_64{{.*}} %{,ALL-KIND10,FAST-KIND10%}
+
+! RUN: %flang_fc1 -emit-hlfir -o - -mllvm -math-runtime=relaxed %s \
+! RUN: | FileCheck %s --check-prefixes=ALL,RELAXED%if target=x86_64{{.*}} %{,ALL-KIND10,RELAXED-KIND10%}
+
+! RUN: %flang_fc1 -emit-hlfir -o - -mllvm -math-runtime=precise %s \
+! RUN: | FileCheck %s --check-prefixes=ALL,PRECISE%if target=x86_64{{.*}} %{,ALL-KIND10,PRECISE-KIND10%}
 
 function test_real4(x)
   real :: x, test_real4
@@ -26,14 +28,15 @@ function test_real8(x)
 ! PRECISE: {{%[A-Za-z0-9._]+}} = fir.call @llvm.round.f64({{%[A-Za-z0-9._]+}}) {{.*}}: (f64) -> f64
 
 function test_real10(x)
-  real(10) :: x, test_real10
+  integer, parameter :: kind10 = merge(10, 4, selected_real_kind(p=18).eq.10)
+  real(kind10) :: x, test_real10
   test_real10 = anint(x)
 end function
 
-! ALL-LABEL: @_QPtest_real10
-! FAST: {{%[A-Za-z0-9._]+}} = llvm.intr.round({{%[A-Za-z0-9._]+}}) : (f80) -> f80
-! RELAXED: {{%[A-Za-z0-9._]+}} = llvm.intr.round({{%[A-Za-z0-9._]+}}) : (f80) -> f80
-! PRECISE: {{%[A-Za-z0-9._]+}} = fir.call @llvm.round.f80({{%[A-Za-z0-9._]+}}) {{.*}}: (f80) -> f80
+! ALL-KIND10-LABEL: @_QPtest_real10
+! FAST-KIND10: {{%[A-Za-z0-9._]+}} = llvm.intr.round({{%[A-Za-z0-9._]+}}) : (f80) -> f80
+! RELAXED-KIND10: {{%[A-Za-z0-9._]+}} = llvm.intr.round({{%[A-Za-z0-9._]+}}) : (f80) -> f80
+! PRECISE-KIND10: {{%[A-Za-z0-9._]+}} = fir.call @llvm.round.f80({{%[A-Za-z0-9._]+}}) {{.*}}: (f80) -> f80
 
 ! TODO: wait until fp128 is supported well in llvm.round
 !function test_real16(x)
@@ -43,4 +46,4 @@ function test_real10(x)
 
 ! PRECISE-DAG: func.func private @llvm.round.f32(f32) -> f32 attributes {fir.bindc_name = "llvm.round.f32", fir.runtime}
 ! PRECISE-DAG: func.func private @llvm.round.f64(f64) -> f64 attributes {fir.bindc_name = "llvm.round.f64", fir.runtime}
-! PRECISE-DAG: func.func private @llvm.round.f80(f80) -> f80 attributes {fir.bindc_name = "llvm.round.f80", fir.runtime}
+! PRECISE-KIND10-DAG: func.func private @llvm.round.f80(f80) -> f80 attributes {fir.bindc_name = "llvm.round.f80", fir.runtime}
diff --git a/flang/test/Lower/math-lowering/sign.f90 b/flang/test/Lower/math-lowering/sign.f90
index fbb47cdebef30..534a6679fbeff 100644
--- a/flang/test/Lower/math-lowering/sign.f90
+++ b/flang/test/Lower/math-lowering/sign.f90
@@ -1,9 +1,11 @@
-! RUN: bbc -emit-fir %s -o - --math-runtime=fast | FileCheck --check-prefixes=ALL,FAST %s
-! RUN: %flang_fc1 -emit-fir -mllvm -math-runtime=fast %s -o - | FileCheck --check-prefixes=ALL,FAST %s
-! RUN: bbc -emit-fir %s -o - --math-runtime=relaxed | FileCheck --check-prefixes=ALL,RELAXED %s
-! RUN: %flang_fc1 -emit-fir -mllvm -math-runtime=relaxed %s -o - | FileCheck --check-prefixes=ALL,RELAXED %s
-! RUN: bbc -emit-fir %s -o - --math-runtime=precise | FileCheck --check-prefixes=ALL,PRECISE %s
-! RUN: %flang_fc1 -emit-fir -mllvm -math-runtime=precise %s -o - | FileCheck --check-prefixes=ALL,PRECISE %s
+! RUN: %flang_fc1 -emit-hlfir -o - -mllvm -math-runtime=fast %s \
+! RUN: | FileCheck %s --check-prefixes=ALL,FAST%if target=x86_64{{.*}} %{,ALL-KIND10,FAST-KIND10%}%if flang-supports-f128-math %{,ALL-KIND16,FAST-KIND16%}
+
+! RUN: %flang_fc1 -emit-hlfir -o - -mllvm -math-runtime=relaxed %s \
+! RUN: | FileCheck %s --check-prefixes=ALL,RELAXED%if target=x86_64{{.*}} %{,ALL-KIND10,RELAXED-KIND10%}%if flang-supports-f128-math %{,ALL-KIND16,RELAXED-KIND16%}
+
+! RUN: %flang_fc1 -emit-hlfir -o - -mllvm -math-runtime=precise %s \
+! RUN: | FileCheck %s --check-prefixes=ALL,PRECISE%if target=x86_64{{.*}} %{,ALL-KIND10,PRECISE-KIND10%}%if flang-supports-f128-math %{,ALL-KIND16,PRECISE-KIND16%}
 
 function test_real4(x, y)
   real :: x, y, test_real4
@@ -26,26 +28,28 @@ function test_real8(x, y)
 ! PRECISE: {{%[A-Za-z0-9._]+}} = fir.call @copysign({{%[A-Za-z0-9._]+}}, {{%[A-Za-z0-9._]+}}) {{.*}}: (f64, f64) -> f64
 
 function test_real10(x, y)
-  real(10) :: x, y, test_real10
+  integer, parameter :: kind10 = merge(10, 4, selected_real_kind(p=18).eq.10)
+  real(kind10) :: x, y, test_real10
   test_real10 = sign(x, y)
 end function
 
-! ALL-LABEL: @_QPtest_real10
-! FAST: {{%[A-Za-z0-9._]+}} = math.copysign {{%[A-Za-z0-9._]+}}, {{%[A-Za-z0-9._]+}} {{.*}}: f80
-! RELAXED: {{%[A-Za-z0-9._]+}} = math.copysign {{%[A-Za-z0-9._]+}}, {{%[A-Za-z0-9._]+}} {{.*}}: f80
-! PRECISE: {{%[A-Za-z0-9._]+}} = fir.call @copysignl({{%[A-Za-z0-9._]+}}, {{%[A-Za-z0-9._]+}}) {{.*}}: (f80, f80) -> f80
+! ALL-KIND10-LABEL: @_QPtest_real10
+! FAST-KIND10: {{%[A-Za-z0-9._]+}} = math.copysign {{%[A-Za-z0-9._]+}}, {{%[A-Za-z0-9._]+}} {{.*}}: f80
+! RELAXED-KIND10: {{%[A-Za-z0-9._]+}} = math.copysign {{%[A-Za-z0-9._]+}}, {{%[A-Za-z0-9._]+}} {{.*}}: f80
+! PRECISE-KIND10: {{%[A-Za-z0-9._]+}} = fir.call @copysignl({{%[A-Za-z0-9._]+}}, {{%[A-Za-z0-9._]+}}) {{.*}}: (f80, f80) -> f80
 
 function test_real16(x, y)
-  real(16) :: x, y, test_real16
+  integer, parameter :: kind16 = merge(16, 4, selected_real_kind(p=33).eq.16)
+  real(kind16) :: x, y, test_real16
   test_real16 = sign(x, y)
 end function
 
-! ALL-LABEL: @_QPtest_real16
-! FAST: {{%[A-Za-z0-9._]+}} = math.copysign {{%[A-Za-z0-9._]+}}, {{%[A-Za-z0-9._]+}} {{.*}}: f128
-! RELAXED: {{%[A-Za-z0-9._]+}} = math.copysign {{%[A-Za-z0-9._]+}}, {{%[A-Za-z0-9._]+}} {{.*}}: f128
-! PRECISE: {{%[A-Za-z0-9._]+}} = fir.call @llvm.copysign.f128({{%[A-Za-z0-9._]+}}, {{%[A-Za-z0-9._]+}}) {{.*}}: (f128, f128) -> f128
+! ALL-KIND16-LABEL: @_QPtest_real16
+! FAST-KIND16: {{%[A-Za-z0-9._]+}} = math.copysign {{%[A-Za-z0-9._]+}}, {{%[A-Za-z0-9._]+}} {{.*}}: f128
+! RELAXED-KIND16: {{%[A-Za-z0-9._]+}} = math.copysign {{%[A-Za-z0-9._]+}}, {{%[A-Za-z0-9._]+}} {{.*}}: f128
+! PRECISE-KIND16: {{%[A-Za-z0-9._]+}} = fir.call @llvm.copysign.f128({{%[A-Za-z0-9._]+}}, {{%[A-Za-z0-9._]+}}) {{.*}}: (f128, f128) -> f128
 
 ! PRECISE-DAG: func.func private @copysignf(f32, f32) -> f32 attributes {fir.bindc_name = "copysignf", fir.runtime}
 ! PRECISE-DAG: func.func private @copysign(f64, f64) -> f64 attributes {fir.bindc_name = "copysign", fir.runtime}
-! PRECISE-DAG: func.func private @copysignl(f80, f80) -> f80 attributes {fir.bindc_name = "copysignl", fir.runtime}
-! PRECISE-DAG: func.func private @llvm.copysign.f128(f128, f128) -> f128 attributes {fir.bindc_name = "llvm.copysign.f128", fir.runtime}
+! PRECISE-KIND10-DAG: func.func private @copysignl(f80, f80) -> f80 attributes {fir.bindc_name = "copysignl", fir.runtime}
+! PRECISE-KIND16-DAG: func.func private @llvm.copysign.f128(f128, f128) -> f128 attributes {fir.bindc_name = "llvm.copysign.f128", fir.runtime}
diff --git a/flang/test/Lower/module_use.f90 b/flang/test/Lower/module_use.f90
index f7c610c9ad22e..92acbfbee0b9e 100644
--- a/flang/test/Lower/module_use.f90
+++ b/flang/test/Lower/module_use.f90
@@ -19,15 +19,15 @@ real function m1use()
 end function
 
 ! TODO: test equivalences once front-end fix in module file is pushed.
-!! CHECK-LABEL func @_QPmodeq1use()
+! COM: CHECK-LABEL: func @_QPmodeq1use()
 !real function modEq1use()
 !  use modEq1
-!  ! CHECK-DAG fir.address_of(@_QMmodeq1Ex1) : !fir.ref<tuple<!fir.array<36xi8>, !fir.array<40xi8>>>
-!  ! CHECK-DAG fir.address_of(@_QMmodeq1Ey1) : !fir.ref<tuple<!fir.array<16xi8>, !fir.array<24xi8>>>
+!  COM: CHECK-DAG: fir.address_of(@_QMmodeq1Ex1) : !fir.ref<tuple<!fir.array<36xi8>, !fir.array<40xi8>>>
+!  COM: CHECK-DAG: fir.address_of(@_QMmodeq1Ey1) : !fir.ref<tuple<!fir.array<16xi8>, !fir.array<24xi8>>>
 !  modEq1use = x2(1) + y1
 !end function
-! CHECK-DAG fir.global @_QMmodeq1Ex1 : tuple<!fir.array<36xi8>, !fir.array<40xi8>>
-! CHECK-DAG fir.global @_QMmodeq1Ey1 : tuple<!fir.array<16xi8>, !fir.array<24xi8>>
+! COM: CHECK-DAG: fir.global @_QMmodeq1Ex1 : tuple<!fir.array<36xi8>, !fir.array<40xi8>>
+! COM: CHECK-DAG: fir.global @_QMmodeq1Ey1 : tuple<!fir.array<16xi8>, !fir.array<24xi8>>
 
 ! CHECK-LABEL: func @_QPmodcommon1use()
 real function modCommon1Use()
diff --git a/flang/test/Lower/pause-statement.f90 b/flang/test/Lower/pause-statement.f90
index e4cba98dc8a79..f4c8f6fbc4385 100644
--- a/flang/test/Lower/pause-statement.f90
+++ b/flang/test/Lower/pause-statement.f90
@@ -1,6 +1,6 @@
 ! RUN: bbc %s -emit-fir --canonicalize -o - | FileCheck %s
 
-! CHECK-LABEL pause_test
+! CHECK-LABEL: pause_test
 subroutine pause_test()
   ! CHECK: fir.call @_Fortran{{.*}}PauseStatement()
   ! CHECK-NEXT: return
diff --git a/flang/test/Lower/polymorphic.f90 b/flang/test/Lower/polymorphic.f90
index 73603d7ee7bee..a1872e225359f 100644
--- a/flang/test/Lower/polymorphic.f90
+++ b/flang/test/Lower/polymorphic.f90
@@ -151,7 +151,7 @@ subroutine call_fct()
 ! CHECK-LABEL: func.func @_QMpolymorphic_testPtest_fct_ret_class() -> !fir.class<!fir.ptr<!fir.type<_QMpolymorphic_testTp1{a:i32,b:i32}>>>
 ! CHECK: return %{{.*}} : !fir.class<!fir.ptr<!fir.type<_QMpolymorphic_testTp1{a:i32,b:i32}>>>
 
-! CHECK-lABEL: func.func @_QMpolymorphic_testPcall_fct()
+! CHECK-LABEL: func.func @_QMpolymorphic_testPcall_fct()
 ! CHECK: %[[RESULT:.*]] = fir.alloca !fir.class<!fir.ptr<!fir.type<_QMpolymorphic_testTp1{a:i32,b:i32}>>> {bindc_name = ".result"}
 ! CHECK: %[[CALL_RES:.*]] = fir.call @_QMpolymorphic_testPtest_fct_ret_class() {{.*}}: () -> !fir.class<!fir.ptr<!fir.type<_QMpolymorphic_testTp1{a:i32,b:i32}>>>
 ! CHECK: fir.save_result %[[CALL_RES]] to %[[RESULT]] : !fir.class<!fir.ptr<!fir.type<_QMpolymorphic_testTp1{a:i32,b:i32}>>>, !fir.ref<!fir.class<!fir.ptr<!fir.type<_QMpolymorphic_testTp1{a:i32,b:i32}>>>>
diff --git a/flang/test/Lower/real-descriptors.f90 b/flang/test/Lower/real-descriptors.f90
index ff7fdc68e7b3a..eb1c4dfae5fd6 100644
--- a/flang/test/Lower/real-descriptors.f90
+++ b/flang/test/Lower/real-descriptors.f90
@@ -1,43 +1,18 @@
-! RUN: bbc %s -o - | tco | FileCheck %s
-
-! CHECK-LABEL: define void @_QQmain()
-program p
-  ! CHECK-DAG:  alloca { ptr, i64, i32, i8, i8, i8, i8 }, align 8
-  ! CHECK-DAG:  alloca { ptr, i64, i32, i8, i8, i8, i8 }, align 8
-  ! CHECK-DAG:  alloca { ptr, i64, i32, i8, i8, i8, i8 }, align 8
-  ! CHECK-DAG:  alloca { ptr, i64, i32, i8, i8, i8, i8 }, align 8
-  ! CHECK-DAG:  alloca { ptr, i64, i32, i8, i8, i8, i8 }, align 8
-  ! CHECK-DAG:  alloca { ptr, i64, i32, i8, i8, i8, i8 }, align 8
-  ! CHECK-DAG:  alloca { ptr, i64, i32, i8, i8, i8, i8 }, align 8
-  ! CHECK-DAG:  alloca { ptr, i64, i32, i8, i8, i8, i8 }, align 8
-  ! CHECK-DAG:  alloca { x86_fp80, x86_fp80 }, i64 1, align 16
-  ! CHECK-DAG:  alloca { fp128, fp128 }, i64 1, align 16
-  ! CHECK-DAG:  alloca { half, half }, i64 1, align 8
-  ! CHECK-DAG:  alloca { bfloat, bfloat }, i64 1, align 8
-  ! CHECK-DAG:  alloca { float, float }, i64 1, align 8
-  ! CHECK-DAG:  alloca { double, double }, i64 1, align 8
-  ! CHECK-DAG:  alloca x86_fp80, i64 1, align 16
-  ! CHECK-DAG:  alloca fp128, i64 1, align 16
-  ! CHECK-DAG:  alloca half, i64 1, align 2
-  ! CHECK-DAG:  alloca bfloat, i64 1, align 2
-  ! CHECK-DAG:  alloca float, i64 1, align 4
-  ! CHECK-DAG:  alloca double, i64 1, align 8
+! RUN: bbc %s -o - | tco | FileCheck %s --check-prefixes=CHECK%if target=x86_64{{.*}} %{,CHECK-KIND10%}%if flang-supports-f128-math %{,CHECK-KIND16%}
 
+! CHECK-LABEL: define void {{.*}}test_reals
+subroutine test_reals(x2, x3, x4, x8, c2, c3, c4, c8)
   character(10) :: in = 'NaN NaN'
 
   real(kind=2)  :: x2
   real(kind=3)  :: x3
   real(kind=4)  :: x4
   real(kind=8)  :: x8
-  real(kind=10) :: x10
-  real(kind=16) :: x16
 
   complex(kind=2)  :: c2
   complex(kind=3)  :: c3
   complex(kind=4)  :: c4
   complex(kind=8)  :: c8
-  complex(kind=10) :: c10
-  complex(kind=16) :: c16
 
   read(in,*) x2
   ! CHECK:      insertvalue { ptr, i64, i32, i8, i8, i8, i8 } { ptr undef, i64 ptrtoint (ptr getelementptr (half, ptr null, i32 1) to i64), i32 {{[0-9]*}}, i8 0, i8 25, i8 0, i8 0 }, ptr %{{[0-9]*}}, 0
@@ -57,17 +32,6 @@ program p
   ! CHECK:      call i1 @_FortranAioOutputReal64(ptr %{{[0-9]*}}, double %{{[0-9]*}})
   print "(z16)", x8
 
-  read(in,*) x10
-  ! CHECK:      insertvalue { ptr, i64, i32, i8, i8, i8, i8 } { ptr undef, i64 ptrtoint (ptr getelementptr (x86_fp80, ptr null, i32 1) to i64), i32 {{[0-9]*}}, i8 0, i8 29, i8 0, i8 0 }, ptr %{{[0-9]*}}, 0
-  ! CHECK:      call i1 @_FortranAioOutputDescriptor(ptr %{{[0-9]*}}, ptr %{{[0-9]*}})
-  print "(z20)", x10
-
-  read(in,*) x16
-  ! CHECK:      insertvalue { ptr, i64, i32, i8, i8, i8, i8 } { ptr undef, i64 ptrtoint (ptr getelementptr (fp128, ptr null, i32 1) to i64), i32 {{[0-9]*}}, i8 0, i8 31, i8 0, i8 0 }, ptr %{{[0-9]*}}, 0
-  ! CHECK:      call i1 @_FortranAioOutputDescriptor(ptr %{{[0-9]*}}, ptr %{{[0-9]*}})
-  print "(z32)", x16
-
-  print*
   read(in,*) c2
   ! CHECK:      insertvalue { ptr, i64, i32, i8, i8, i8, i8 } { ptr undef, i64 ptrtoint (ptr getelementptr ({ half, half }, ptr null, i32 1) to i64), i32 {{[0-9]*}}, i8 0, i8 32, i8 0, i8 0 }, ptr %{{[0-9]*}}, 0
   ! CHECK:      call i1 @_FortranAioOutputDescriptor(ptr %{{[0-9]*}}, ptr %{{[0-9]*}})
@@ -86,13 +50,40 @@ program p
   ! CHECK:      call i1 @_FortranAioOutputComplex64(ptr %{{[0-9]*}}, double %{{[0-9]*}}, double %{{[0-9]*}})
   print "(z16,' ',z16)", c8
 
+end
+
+! CHECK-KIND16-LABEL: test_kind10
+subroutine test_kind10(x10, c10)
+  integer, parameter :: kind10 = merge(10, 4, selected_real_kind(p=18).eq.10)
+  character(10) :: in = 'NaN NaN'
+  real(kind=kind10) :: x10
+  complex(kind=kind10) :: c10
+
+  read(in,*) x10
+  ! CHECK-KIND10:  insertvalue { ptr, i64, i32, i8, i8, i8, i8 } { ptr undef, i64 ptrtoint (ptr getelementptr (x86_fp80, ptr null, i32 1) to i64), i32 {{[0-9]*}}, i8 0, i8 29, i8 0, i8 0 }, ptr %{{[0-9]*}}, 0
+  ! CHECK-KIND10:  call i1 @_FortranAioOutputDescriptor(ptr %{{[0-9]*}}, ptr %{{[0-9]*}})
+  print "(z20)", x10
+
   read(in,*) c10
-  ! CHECK:      insertvalue { ptr, i64, i32, i8, i8, i8, i8 } { ptr undef, i64 ptrtoint (ptr getelementptr ({ x86_fp80, x86_fp80 }, ptr null, i32 1) to i64), i32 {{[0-9]*}}, i8 0, i8 36, i8 0, i8 0 }, ptr %{{[0-9]*}}, 0
-  ! CHECK:      call i1 @_FortranAioOutputDescriptor(ptr %{{[0-9]*}}, ptr %{{[0-9]*}})
+  ! CHECK-KIND10:  insertvalue { ptr, i64, i32, i8, i8, i8, i8 } { ptr undef, i64 ptrtoint (ptr getelementptr ({ x86_fp80, x86_fp80 }, ptr null, i32 1) to i64), i32 {{[0-9]*}}, i8 0, i8 36, i8 0, i8 0 }, ptr %{{[0-9]*}}, 0
+  ! CHECK-KIND10:  call i1 @_FortranAioOutputDescriptor(ptr %{{[0-9]*}}, ptr %{{[0-9]*}})
   print "(z20,' ',z20)", c10
+end subroutine
+
+! CHECK-KIND16-LABEL: test_kind16
+subroutine test_kind16(x16, c16)
+  integer, parameter :: kind16 = merge(16, 4, selected_real_kind(p=33).eq.16)
+  character(10) :: in = 'NaN NaN'
+  real(kind=kind16) :: x16
+  complex(kind=kind16) :: c16
+
+  read(in,*) x16
+  ! CHECK-KIND16:  insertvalue { ptr, i64, i32, i8, i8, i8, i8 } { ptr undef, i64 ptrtoint (ptr getelementptr (fp128, ptr null, i32 1) to i64), i32 {{[0-9]*}}, i8 0, i8 31, i8 0, i8 0 }, ptr %{{[0-9]*}}, 0
+  ! CHECK-KIND16:  call i1 @_FortranAioOutputDescriptor(ptr %{{[0-9]*}}, ptr %{{[0-9]*}})
+  print "(z32)", x16
 
   read(in,*) c16
-  ! CHECK:      insertvalue { ptr, i64, i32, i8, i8, i8, i8 } { ptr undef, i64 ptrtoint (ptr getelementptr ({ fp128, fp128 }, ptr null, i32 1) to i64), i32 {{[0-9]*}}, i8 0, i8 38, i8 0, i8 0 }, ptr %{{[0-9]*}}, 0
-  ! CHECK:      call i1 @_FortranAioOutputDescriptor(ptr %{{[0-9]*}}, ptr %{{[0-9]*}})
+  ! CHECK-KIND16:  insertvalue { ptr, i64, i32, i8, i8, i8, i8 } { ptr undef, i64 ptrtoint (ptr getelementptr ({ fp128, fp128 }, ptr null, i32 1) to i64), i32 {{[0-9]*}}, i8 0, i8 38, i8 0, i8 0 }, ptr %{{[0-9]*}}, 0
+  ! CHECK-KIND16:  call i1 @_FortranAioOutputDescriptor(ptr %{{[0-9]*}}, ptr %{{[0-9]*}})
   print "(z32,' ',z32)", c16
-end
+end subroutine
diff --git a/flang/test/Lower/real-operations-1.f90 b/flang/test/Lower/real-operations-1.f90
index 137d0b5264c25..d155b7dc9570f 100644
--- a/flang/test/Lower/real-operations-1.f90
+++ b/flang/test/Lower/real-operations-1.f90
@@ -1,13 +1,14 @@
-! RUN: bbc -hlfir=false %s -o - | FileCheck %s
-
+! RUN: bbc -emit-hlfir %s -o - | FileCheck %s --check-prefixes=CHECK%if target=x86_64{{.*}} %{,CHECK-KIND10%}%if flang-supports-f128-math %{,CHECK-KIND16%}
 ! Test real add on real kinds.
 
 ! CHECK-LABEL: real2
 REAL(2) FUNCTION real2(x0, x1)
   REAL(2) :: x0
   REAL(2) :: x1
-  ! CHECK-DAG: %[[v1:.+]] = fir.load %arg0 : !fir.ref<f16>
-  ! CHECK-DAG: %[[v2:.+]] = fir.load %arg1 : !fir.ref<f16>
+  ! CHECK: %[[x0:.*]]:2 = hlfir.declare{{.*}}x0"
+  ! CHECK: %[[x1:.*]]:2 = hlfir.declare{{.*}}x1"
+  ! CHECK-DAG: %[[v1:.+]] = fir.load %[[x0]]#0 : !fir.ref<f16>
+  ! CHECK-DAG: %[[v2:.+]] = fir.load %[[x1]]#0 : !fir.ref<f16>
   ! CHECK: %[[v3:.+]] = arith.addf %[[v1]], %[[v2]] {{.*}}: f16
   real2 = x0 + x1
   ! CHECK: return %{{.*}} : f16
@@ -17,8 +18,10 @@ END FUNCTION real2
 REAL(3) FUNCTION real3(x0, x1)
   REAL(3) :: x0
   REAL(3) :: x1
-  ! CHECK-DAG: %[[v1:.+]] = fir.load %arg0 : !fir.ref<bf16>
-  ! CHECK-DAG: %[[v2:.+]] = fir.load %arg1 : !fir.ref<bf16>
+  ! CHECK: %[[x0:.*]]:2 = hlfir.declare{{.*}}x0"
+  ! CHECK: %[[x1:.*]]:2 = hlfir.declare{{.*}}x1"
+  ! CHECK-DAG: %[[v1:.+]] = fir.load %[[x0]]#0 : !fir.ref<bf16>
+  ! CHECK-DAG: %[[v2:.+]] = fir.load %[[x1]]#0 : !fir.ref<bf16>
   ! CHECK: %[[v3:.+]] = arith.addf %[[v1]], %[[v2]] {{.*}}: bf16
   real3 = x0 + x1
   ! CHECK: return %{{.*}} : bf16
@@ -28,8 +31,10 @@ END FUNCTION real3
 REAL(4) FUNCTION real4(x0, x1)
   REAL(4) :: x0
   REAL(4) :: x1
-  ! CHECK-DAG: %[[v1:.+]] = fir.load %arg0 : !fir.ref<f32>
-  ! CHECK-DAG: %[[v2:.+]] = fir.load %arg1 : !fir.ref<f32>
+  ! CHECK: %[[x0:.*]]:2 = hlfir.declare{{.*}}x0"
+  ! CHECK: %[[x1:.*]]:2 = hlfir.declare{{.*}}x1"
+  ! CHECK-DAG: %[[v1:.+]] = fir.load %[[x0]]#0 : !fir.ref<f32>
+  ! CHECK-DAG: %[[v2:.+]] = fir.load %[[x1]]#0 : !fir.ref<f32>
   ! CHECK: %[[v3:.+]] = arith.addf %[[v1]], %[[v2]] {{.*}}: f32
   real4 = x0 + x1
   ! CHECK: return %{{.*}} : f32
@@ -39,8 +44,10 @@ END FUNCTION real4
 REAL FUNCTION defreal(x0, x1)
   REAL :: x0
   REAL :: x1
-  ! CHECK-DAG: %[[v1:.+]] = fir.load %arg0 : !fir.ref<f32>
-  ! CHECK-DAG: %[[v2:.+]] = fir.load %arg1 : !fir.ref<f32>
+  ! CHECK: %[[x0:.*]]:2 = hlfir.declare{{.*}}x0"
+  ! CHECK: %[[x1:.*]]:2 = hlfir.declare{{.*}}x1"
+  ! CHECK-DAG: %[[v1:.+]] = fir.load %[[x0]]#0 : !fir.ref<f32>
+  ! CHECK-DAG: %[[v2:.+]] = fir.load %[[x1]]#0 : !fir.ref<f32>
   ! CHECK: %[[v3:.+]] = arith.addf %[[v1]], %[[v2]] {{.*}}: f32
   defreal = x0 + x1
   ! CHECK: return %{{.*}} : f32
@@ -50,8 +57,10 @@ END FUNCTION defreal
 REAL(8) FUNCTION real8(x0, x1)
   REAL(8) :: x0
   REAL(8) :: x1
-  ! CHECK-DAG: %[[v1:.+]] = fir.load %arg0 : !fir.ref<f64>
-  ! CHECK-DAG: %[[v2:.+]] = fir.load %arg1 : !fir.ref<f64>
+  ! CHECK: %[[x0:.*]]:2 = hlfir.declare{{.*}}x0"
+  ! CHECK: %[[x1:.*]]:2 = hlfir.declare{{.*}}x1"
+  ! CHECK-DAG: %[[v1:.+]] = fir.load %[[x0]]#0 : !fir.ref<f64>
+  ! CHECK-DAG: %[[v2:.+]] = fir.load %[[x1]]#0 : !fir.ref<f64>
   ! CHECK: %[[v3:.+]] = arith.addf %[[v1]], %[[v2]] {{.*}}: f64
   real8 = x0 + x1
   ! CHECK: return %{{.*}} : f64
@@ -61,44 +70,58 @@ END FUNCTION real8
 DOUBLE PRECISION FUNCTION doubleprec(x0, x1)
   DOUBLE PRECISION :: x0
   DOUBLE PRECISION :: x1
-  ! CHECK-DAG: %[[v1:.+]] = fir.load %arg0 : !fir.ref<f64>
-  ! CHECK-DAG: %[[v2:.+]] = fir.load %arg1 : !fir.ref<f64>
+  ! CHECK: %[[x0:.*]]:2 = hlfir.declare{{.*}}x0"
+  ! CHECK: %[[x1:.*]]:2 = hlfir.declare{{.*}}x1"
+  ! CHECK-DAG: %[[v1:.+]] = fir.load %[[x0]]#0 : !fir.ref<f64>
+  ! CHECK-DAG: %[[v2:.+]] = fir.load %[[x1]]#0 : !fir.ref<f64>
   ! CHECK: %[[v3:.+]] = arith.addf %[[v1]], %[[v2]] {{.*}}: f64
   doubleprec = x0 + x1
   ! CHECK: return %{{.*}} : f64
 END FUNCTION doubleprec
 
-! CHECK-LABEL: real10
-REAL(10) FUNCTION real10(x0, x1)
-  REAL(10) :: x0
-  REAL(10) :: x1
-  ! CHECK-DAG: %[[v1:.+]] = fir.load %arg0 : !fir.ref<f80>
-  ! CHECK-DAG: %[[v2:.+]] = fir.load %arg1 : !fir.ref<f80>
-  ! CHECK: %[[v3:.+]] = arith.addf %[[v1]], %[[v2]] {{.*}}: f80
+! CHECK-KIND10-LABEL: real10
+FUNCTION real10(x0, x1)
+  INTEGER, PARAMETER :: kind10 = merge(10, 4, selected_real_kind(p=18).eq.10)
+  REAL(kind10) :: real10
+  REAL(kind10) :: x0
+  REAL(kind10) :: x1
+  ! CHECK-KIND10: %[[x0:.*]]:2 = hlfir.declare{{.*}}x0"
+  ! CHECK-KIND10: %[[x1:.*]]:2 = hlfir.declare{{.*}}x1"
+  ! CHECK-KIND10-DAG: %[[v1:.+]] = fir.load %[[x0]]#0 : !fir.ref<f80>
+  ! CHECK-KIND10-DAG: %[[v2:.+]] = fir.load %[[x1]]#0 : !fir.ref<f80>
+  ! CHECK-KIND10: %[[v3:.+]] = arith.addf %[[v1]], %[[v2]] {{.*}}: f80
   real10 = x0 + x1
-  ! CHECK: return %{{.*}} : f80
+  ! CHECK-KIND10: return %{{.*}} : f80
 END FUNCTION real10
 
-! CHECK-LABEL: real16
-REAL(16) FUNCTION real16(x0, x1)
-  REAL(16) :: x0
-  REAL(16) :: x1
-  ! CHECK-DAG: %[[v1:.+]] = fir.load %arg0 : !fir.ref<f128>
-  ! CHECK-DAG: %[[v2:.+]] = fir.load %arg1 : !fir.ref<f128>
-  ! CHECK: %[[v3:.+]] = arith.addf %[[v1]], %[[v2]] {{.*}}: f128
+! CHECK-KIND16-LABEL: real16(
+FUNCTION real16(x0, x1)
+  INTEGER, PARAMETER :: kind16 = merge(16, 4, selected_real_kind(p=33).eq.16)
+  REAL(kind16) :: real16
+  REAL(kind16) :: x0
+  REAL(kind16) :: x1
+  ! CHECK-KIND16: %[[x0:.*]]:2 = hlfir.declare{{.*}}x0"
+  ! CHECK-KIND16: %[[x1:.*]]:2 = hlfir.declare{{.*}}x1"
+  ! CHECK-KIND16-DAG: %[[v1:.+]] = fir.load %[[x0]]#0 : !fir.ref<f128>
+  ! CHECK-KIND16-DAG: %[[v2:.+]] = fir.load %[[x1]]#0 : !fir.ref<f128>
+  ! CHECK-KIND16: %[[v3:.+]] = arith.addf %[[v1]], %[[v2]] {{.*}}: f128
   real16 = x0 + x1
-  ! CHECK: return %{{.*}} : f128
+  ! CHECK-KIND16: return %{{.*}} : f128
 END FUNCTION real16
 
-! CHECK-LABEL: real16b
-REAL(16) FUNCTION real16b(x0, x1)
-  REAL(16) :: x0
-  REAL(16) :: x1
-  ! CHECK-DAG: %[[v0:.+]] = arith.constant 4.0{{.*}} : f128
-  ! CHECK-DAG: %[[v1:.+]] = fir.load %arg0 : !fir.ref<f128>
-  ! CHECK-DAG: %[[v2:.+]] = fir.load %arg1 : !fir.ref<f128>
-  ! CHECK: %[[v3:.+]] = arith.addf %[[v1]], %[[v2]] {{.*}}: f128
-  ! CHECK: %[[v4:.+]] = arith.subf %[[v3]], %[[v0]] {{.*}}: f128
+! CHECK-KIND16-LABEL: real16b
+FUNCTION real16b(x0, x1)
+  INTEGER, PARAMETER :: kind16 = merge(16, 4, selected_real_kind(p=33).eq.16)
+  REAL(kind16) :: real16b
+  REAL(kind16) :: x0
+  REAL(kind16) :: x1
+  ! CHECK-KIND16: %[[x0:.*]]:2 = hlfir.declare{{.*}}x0"
+  ! CHECK-KIND16: %[[x1:.*]]:2 = hlfir.declare{{.*}}x1"
+  ! CHECK-KIND16-DAG: %[[v1:.+]] = fir.load %[[x0]]#0 : !fir.ref<f128>
+  ! CHECK-KIND16-DAG: %[[v2:.+]] = fir.load %[[x1]]#0 : !fir.ref<f128>
+  ! CHECK-KIND16-DAG: %[[v3:.+]] = arith.addf %[[v1]], %[[v2]] {{.*}}: f128
+  ! CHECK-KIND16-DAG: %[[v0:.+]] = arith.constant 4.0{{.*}} : f128
+  ! CHECK-KIND16: %[[v4:.+]] = arith.subf %[[v3]], %[[v0]] {{.*}}: f128
   real16b = x0 + x1 - 4.0_16
-  ! CHECK: return %{{.*}} : f128
+  ! CHECK-KIND16: return %{{.*}} : f128
 END FUNCTION real16b
diff --git a/flang/test/Lower/unroll.f90 b/flang/test/Lower/unroll.f90
new file mode 100644
index 0000000000000..229755200fd8d
--- /dev/null
+++ b/flang/test/Lower/unroll.f90
@@ -0,0 +1,27 @@
+! RUN: %flang_fc1 -emit-hlfir -o - %s | FileCheck %s
+
+! CHECK: #loop_unroll = #llvm.loop_unroll<disable = false, full = true>
+! CHECK: #loop_annotation = #llvm.loop_annotation<unroll = #loop_unroll>
+
+! CHECK-LABEL: unroll_dir
+subroutine unroll_dir
+  integer :: a(10)
+  !dir$ unroll
+  !CHECK: fir.do_loop {{.*}} attributes {loopAnnotation = #loop_annotation}
+  do i=1,10
+     a(i)=i
+  end do
+end subroutine unroll_dir
+
+
+! CHECK-LABEL: intermediate_directive
+subroutine intermediate_directive
+  integer :: a(10)
+  !dir$ unroll
+  !dir$ unknown
+  !CHECK: fir.do_loop {{.*}} attributes {loopAnnotation = #loop_annotation}
+  do i=1,10
+     a(i)=i
+  end do
+end subroutine intermediate_directive
+
diff --git a/flang/test/Parser/OpenMP/declare-mapper-unparse.f90 b/flang/test/Parser/OpenMP/declare-mapper-unparse.f90
index 5ba147d20955e..407bfd29153fa 100644
--- a/flang/test/Parser/OpenMP/declare-mapper-unparse.f90
+++ b/flang/test/Parser/OpenMP/declare-mapper-unparse.f90
@@ -13,7 +13,7 @@ program main
   !$omp declare mapper(mymapper : ty :: mapped) map(mapped, mapped%x)
 
 !PARSE-TREE:      OpenMPDeclareMapperConstruct
-!PARSE-TREE:        OmpDeclareMapperSpecifier
+!PARSE-TREE:        OmpMapperSpecifier
 !PARSE-TREE:         Name = 'mymapper'
 !PARSE-TREE:         TypeSpec -> DerivedTypeSpec
 !PARSE-TREE:           Name = 'ty'
@@ -28,7 +28,7 @@ program main
   !$omp declare mapper(ty :: mapped) map(mapped, mapped%x)
   
 !PARSE-TREE:      OpenMPDeclareMapperConstruct
-!PARSE-TREE:        OmpDeclareMapperSpecifier
+!PARSE-TREE:        OmpMapperSpecifier
 !PARSE-TREE:         TypeSpec -> DerivedTypeSpec
 !PARSE-TREE:           Name = 'ty'
 !PARSE-TREE:         Name = 'mapped'    
diff --git a/flang/test/Parser/OpenMP/dispatch.f90 b/flang/test/Parser/OpenMP/dispatch.f90
new file mode 100644
index 0000000000000..98cd6090334f3
--- /dev/null
+++ b/flang/test/Parser/OpenMP/dispatch.f90
@@ -0,0 +1,51 @@
+! RUN: %flang_fc1 -fopenmp -fdebug-dump-parse-tree %s | FileCheck %s
+! RUN: %flang_fc1 -fopenmp -fdebug-unparse %s | FileCheck %s --check-prefix="UNPARSE"
+
+integer function func(a, b, c)
+  integer  :: a, b, c
+  func = a + b + c
+end function func
+
+subroutine sub(x)
+  use iso_c_binding
+  integer :: func
+  integer :: r
+  type(c_ptr) :: x
+  integer :: a = 14, b = 7, c = 21
+!UNPARSE: !$OMP DISPATCH DEVICE(3_4) NOWAIT NOCONTEXT(.false._4) NOVARIANTS(.true._4)
+!CHECK: | | ExecutionPartConstruct -> ExecutableConstruct -> OpenMPConstruct -> OpenMPDispatchConstruct
+!CHECK-NEXT: | | | OmpDispatchDirective
+!CHECK: | | | | OmpClauseList -> OmpClause -> Device -> OmpDeviceClause
+!CHECK-NEXT: | | | | | Scalar -> Integer -> Expr = '3_4'
+!CHECK-NEXT: | | | | | | LiteralConstant -> IntLiteralConstant = '3'
+!CHECK-NEXT: | | | | OmpClause -> Nowait
+!CHECK-NEXT: | | | | OmpClause -> Nocontext -> Scalar -> Logical -> Expr = '.false._4'
+!CHECK-NEXT: | | | | | LiteralConstant -> LogicalLiteralConstant
+!CHECK-NEXT: | | | | | | bool = 'false'
+!CHECK-NEXT: | | | | OmpClause -> Novariants -> Scalar -> Logical -> Expr = '.true._4'
+!CHECK-NEXT: | | | | | EQ
+!CHECK-NEXT: | | | | | | Expr = '1_4'
+!CHECK-NEXT: | | | | | | | LiteralConstant -> IntLiteralConstant = '1'
+!CHECK-NEXT: | | | | | | Expr = '1_4'
+!CHECK-NEXT: | | | | | | | LiteralConstant -> IntLiteralConstant = '1'
+!CHECK-NEXT: | | | Block
+ 
+  !$omp dispatch device(3) nowait nocontext(.false.) novariants(1.eq.1)
+  r = func(a, b, c)
+!UNPARSE: !$OMP END DISPATCH
+!CHECK: | | | OmpEndDispatchDirective
+  !$omp end dispatch
+
+!! Test the "no end dispatch" option.
+!UNPARSE: !$OMP DISPATCH  DEVICE(3_4) IS_DEVICE_PTR(x)
+!CHECK: | | ExecutionPartConstruct -> ExecutableConstruct -> OpenMPConstruct -> OpenMPDispatchConstruct
+!CHECK-NEXT: | | | OmpDispatchDirective
+!CHECK: | | | | OmpClause -> IsDevicePtr ->  OmpObjectList -> OmpObject -> Designator -> DataRef -> Name = 'x'  
+  !$omp dispatch device(3) is_device_ptr(x)
+  r = func(a+1, b+2, c+3)
+!CHECK-NOT: | | | OmpEndDispatchDirective
+
+end subroutine sub
+
+
+
diff --git a/flang/test/Parser/OpenMP/in-reduction-clause.f90 b/flang/test/Parser/OpenMP/in-reduction-clause.f90
index 8a0bede62f03f..bb3fadbc5088e 100644
--- a/flang/test/Parser/OpenMP/in-reduction-clause.f90
+++ b/flang/test/Parser/OpenMP/in-reduction-clause.f90
@@ -75,5 +75,5 @@ end subroutine omp_in_reduction_parallel
 !PARSE-TREE-NEXT: OmpLoopDirective -> llvm::omp::Directive = taskloop simd
 !PARSE-TREE-NEXT: OmpClauseList -> OmpClause -> InReduction -> OmpInReductionClause
 !PARSE-TREE-NEXT: OmpReductionIdentifier -> DefinedOperator -> IntrinsicOperator = Add
-!PASRE-TREE-NEXT: OmpObjectList -> OmpObject -> Designator -> DataRef -> Name = 'z'
+!PARSE-TREE-NEXT: OmpObjectList -> OmpObject -> Designator -> DataRef -> Name = 'z'
 
diff --git a/flang/test/Parser/OpenMP/metadirective-dirspec.f90 b/flang/test/Parser/OpenMP/metadirective-dirspec.f90
new file mode 100644
index 0000000000000..73520c41fe77d
--- /dev/null
+++ b/flang/test/Parser/OpenMP/metadirective-dirspec.f90
@@ -0,0 +1,242 @@
+!RUN: %flang_fc1 -fdebug-unparse -fopenmp -fopenmp-version=52 %s | FileCheck --ignore-case --check-prefix="UNPARSE" %s
+!RUN: %flang_fc1 -fdebug-dump-parse-tree -fopenmp -fopenmp-version=52 %s | FileCheck --check-prefix="PARSE-TREE" %s
+
+!Directive specification where directives have arguments
+
+subroutine f00(x)
+  integer :: x(10)
+  !$omp metadirective when(user={condition(.true.)}: &
+  !$omp & allocate(x))
+end
+
+!UNPARSE: SUBROUTINE f00 (x)
+!UNPARSE:  INTEGER x(10_4)
+!UNPARSE: !$OMP METADIRECTIVE  WHEN(USER={CONDITION(.true._4)}: ALLOCATE(x))
+!UNPARSE: END SUBROUTINE
+
+!PARSE-TREE: DeclarationConstruct -> SpecificationConstruct -> OpenMPDeclarativeConstruct -> OmpMetadirectiveDirective
+!PARSE-TREE: | OmpClauseList -> OmpClause -> When -> OmpWhenClause
+!PARSE-TREE: | | Modifier -> OmpContextSelectorSpecification -> OmpTraitSetSelector
+!PARSE-TREE: | | | OmpTraitSetSelectorName -> Value = User
+!PARSE-TREE: | | | OmpTraitSelector
+!PARSE-TREE: | | | | OmpTraitSelectorName -> Value = Condition
+!PARSE-TREE: | | | | Properties
+!PARSE-TREE: | | | | | OmpTraitProperty -> Scalar -> Expr = '.true._4'
+!PARSE-TREE: | | | | | | LiteralConstant -> LogicalLiteralConstant
+!PARSE-TREE: | | | | | | | bool = 'true'
+!PARSE-TREE: | | OmpDirectiveSpecification
+!PARSE-TREE: | | | llvm::omp::Directive = allocate
+!PARSE-TREE: | | | OmpArgument -> OmpLocator -> OmpObject -> Designator -> DataRef -> Name = 'x'
+!PARSE-TREE: | | | OmpClauseList ->
+
+subroutine f01(x)
+  integer :: x
+  !$omp metadirective when(user={condition(.true.)}: &
+  !$omp & critical(x))
+end
+
+!UNPARSE: SUBROUTINE f01 (x)
+!UNPARSE:  INTEGER x
+!UNPARSE: !$OMP METADIRECTIVE  WHEN(USER={CONDITION(.true._4)}: CRITICAL(x))
+!UNPARSE: END SUBROUTINE
+
+!PARSE-TREE: DeclarationConstruct -> SpecificationConstruct -> OpenMPDeclarativeConstruct -> OmpMetadirectiveDirective
+!PARSE-TREE: | OmpClauseList -> OmpClause -> When -> OmpWhenClause
+!PARSE-TREE: | | Modifier -> OmpContextSelectorSpecification -> OmpTraitSetSelector
+!PARSE-TREE: | | | OmpTraitSetSelectorName -> Value = User
+!PARSE-TREE: | | | OmpTraitSelector
+!PARSE-TREE: | | | | OmpTraitSelectorName -> Value = Condition
+!PARSE-TREE: | | | | Properties
+!PARSE-TREE: | | | | | OmpTraitProperty -> Scalar -> Expr = '.true._4'
+!PARSE-TREE: | | | | | | LiteralConstant -> LogicalLiteralConstant
+!PARSE-TREE: | | | | | | | bool = 'true'
+!PARSE-TREE: | | OmpDirectiveSpecification
+!PARSE-TREE: | | | llvm::omp::Directive = critical
+!PARSE-TREE: | | | OmpArgument -> OmpLocator -> OmpObject -> Designator -> DataRef -> Name = 'x'
+!PARSE-TREE: | | | OmpClauseList ->
+
+subroutine f02
+  !$omp metadirective when(user={condition(.true.)}: &
+  !$omp & declare mapper(mymapper : integer :: v) map(tofrom: v))
+end
+
+!UNPARSE: SUBROUTINE f02
+!UNPARSE: !$OMP METADIRECTIVE  WHEN(USER={CONDITION(.true._4)}: DECLARE MAPPER(mymapper:INTEGER:&
+!UNPARSE: !$OMP&:v) MAP(TOFROM: v))
+!UNPARSE: END SUBROUTINE
+
+!PARSE-TREE: OpenMPDeclarativeConstruct -> OmpMetadirectiveDirective
+!PARSE-TREE: | OmpClauseList -> OmpClause -> When -> OmpWhenClause
+!PARSE-TREE: | | Modifier -> OmpContextSelectorSpecification -> OmpTraitSetSelector
+!PARSE-TREE: | | | OmpTraitSetSelectorName -> Value = User
+!PARSE-TREE: | | | OmpTraitSelector
+!PARSE-TREE: | | | | OmpTraitSelectorName -> Value = Condition
+!PARSE-TREE: | | | | Properties
+!PARSE-TREE: | | | | | OmpTraitProperty -> Scalar -> Expr = '.true._4'
+!PARSE-TREE: | | | | | | LiteralConstant -> LogicalLiteralConstant
+!PARSE-TREE: | | | | | | | bool = 'true'
+!PARSE-TREE: | | OmpDirectiveSpecification
+!PARSE-TREE: | | | llvm::omp::Directive = declare mapper
+!PARSE-TREE: | | | OmpArgument -> OmpMapperSpecifier
+!PARSE-TREE: | | | | Name = 'mymapper'
+!PARSE-TREE: | | | | TypeSpec -> IntrinsicTypeSpec -> IntegerTypeSpec ->
+!PARSE-TREE: | | | | Name = 'v'
+!PARSE-TREE: | | | OmpClauseList -> OmpClause -> Map -> OmpMapClause
+!PARSE-TREE: | | | | Modifier -> OmpMapType -> Value = Tofrom
+!PARSE-TREE: | | | | OmpObjectList -> OmpObject -> Designator -> DataRef -> Name = 'v'
+!PARSE-TREE: | | | | bool = 'true'
+!PARSE-TREE: ImplicitPart ->
+
+subroutine f03
+  type :: tt1
+    integer :: x
+  endtype
+  type :: tt2
+    real :: a
+  endtype
+  !$omp metadirective when(user={condition(.true.)}: &
+  !$omp & declare reduction(+ : tt1, tt2 : omp_out = omp_in + omp_out))
+end
+
+!UNPARSE: SUBROUTINE f03
+!UNPARSE:  TYPE :: tt1
+!UNPARSE:   INTEGER :: x
+!UNPARSE:  END TYPE
+!UNPARSE:  TYPE :: tt2
+!UNPARSE:   REAL :: a
+!UNPARSE:  END TYPE
+!UNPARSE: !$OMP METADIRECTIVE  WHEN(USER={CONDITION(.true._4)}: DECLARE REDUCTION(+:tt1,tt2: omp_out=omp_in+omp_out
+!UNPARSE: ))
+!UNPARSE: END SUBROUTINE
+
+!PARSE-TREE: DeclarationConstruct -> SpecificationConstruct -> OpenMPDeclarativeConstruct -> OmpMetadirectiveDirective
+!PARSE-TREE: | OmpClauseList -> OmpClause -> When -> OmpWhenClause
+!PARSE-TREE: | | Modifier -> OmpContextSelectorSpecification -> OmpTraitSetSelector
+!PARSE-TREE: | | | OmpTraitSetSelectorName -> Value = User
+!PARSE-TREE: | | | OmpTraitSelector
+!PARSE-TREE: | | | | OmpTraitSelectorName -> Value = Condition
+!PARSE-TREE: | | | | Properties
+!PARSE-TREE: | | | | | OmpTraitProperty -> Scalar -> Expr = '.true._4'
+!PARSE-TREE: | | | | | | LiteralConstant -> LogicalLiteralConstant
+!PARSE-TREE: | | | | | | | bool = 'true'
+!PARSE-TREE: | | OmpDirectiveSpecification
+!PARSE-TREE: | | | llvm::omp::Directive = declare reduction
+!PARSE-TREE: | | | OmpArgument -> OmpReductionSpecifier
+!PARSE-TREE: | | | | OmpReductionIdentifier -> DefinedOperator -> IntrinsicOperator = Add
+!PARSE-TREE: | | | | OmpTypeNameList -> OmpTypeSpecifier -> TypeSpec -> DerivedTypeSpec
+!PARSE-TREE: | | | | | Name = 'tt1'
+!PARSE-TREE: | | | | OmpTypeSpecifier -> TypeSpec -> DerivedTypeSpec
+!PARSE-TREE: | | | | | Name = 'tt2'
+!PARSE-TREE: | | | | OmpReductionCombiner -> AssignmentStmt = 'omp_out=omp_in+omp_out'
+!PARSE-TREE: | | | | | Variable = 'omp_out'
+!PARSE-TREE: | | | | | | Designator -> DataRef -> Name = 'omp_out'
+!PARSE-TREE: | | | | | Expr = 'omp_in+omp_out'
+!PARSE-TREE: | | | | | | Add
+!PARSE-TREE: | | | | | | | Expr = 'omp_in'
+!PARSE-TREE: | | | | | | | | Designator -> DataRef -> Name = 'omp_in'
+!PARSE-TREE: | | | | | | | Expr = 'omp_out'
+!PARSE-TREE: | | | | | | | | Designator -> DataRef -> Name = 'omp_out'
+!PARSE-TREE: | | | OmpClauseList ->
+
+subroutine f04
+  !$omp metadirective when(user={condition(.true.)}: &
+  !$omp & declare simd(f04))
+end
+
+!UNPARSE: SUBROUTINE f04
+!UNPARSE: !$OMP METADIRECTIVE  WHEN(USER={CONDITION(.true._4)}: DECLARE SIMD(f04))
+!UNPARSE: END SUBROUTINE
+
+!PARSE-TREE: OpenMPDeclarativeConstruct -> OmpMetadirectiveDirective
+!PARSE-TREE: | OmpClauseList -> OmpClause -> When -> OmpWhenClause
+!PARSE-TREE: | | Modifier -> OmpContextSelectorSpecification -> OmpTraitSetSelector
+!PARSE-TREE: | | | OmpTraitSetSelectorName -> Value = User
+!PARSE-TREE: | | | OmpTraitSelector
+!PARSE-TREE: | | | | OmpTraitSelectorName -> Value = Condition
+!PARSE-TREE: | | | | Properties
+!PARSE-TREE: | | | | | OmpTraitProperty -> Scalar -> Expr = '.true._4'
+!PARSE-TREE: | | | | | | LiteralConstant -> LogicalLiteralConstant
+!PARSE-TREE: | | | | | | | bool = 'true'
+!PARSE-TREE: | | OmpDirectiveSpecification
+!PARSE-TREE: | | | llvm::omp::Directive = declare simd
+!PARSE-TREE: | | | OmpArgument -> OmpLocator -> OmpObject -> Designator -> DataRef -> Name = 'f04'
+!PARSE-TREE: | | | OmpClauseList ->
+!PARSE-TREE: ImplicitPart ->
+
+subroutine f05
+  !$omp metadirective when(user={condition(.true.)}: &
+  !$omp & declare target(f05))
+end
+
+!UNPARSE: SUBROUTINE f05
+!UNPARSE: !$OMP METADIRECTIVE  WHEN(USER={CONDITION(.true._4)}: DECLARE TARGET(f05))
+!UNPARSE: END SUBROUTINE
+
+!PARSE-TREE: OpenMPDeclarativeConstruct -> OmpMetadirectiveDirective
+!PARSE-TREE: | OmpClauseList -> OmpClause -> When -> OmpWhenClause
+!PARSE-TREE: | | Modifier -> OmpContextSelectorSpecification -> OmpTraitSetSelector
+!PARSE-TREE: | | | OmpTraitSetSelectorName -> Value = User
+!PARSE-TREE: | | | OmpTraitSelector
+!PARSE-TREE: | | | | OmpTraitSelectorName -> Value = Condition
+!PARSE-TREE: | | | | Properties
+!PARSE-TREE: | | | | | OmpTraitProperty -> Scalar -> Expr = '.true._4'
+!PARSE-TREE: | | | | | | LiteralConstant -> LogicalLiteralConstant
+!PARSE-TREE: | | | | | | | bool = 'true'
+!PARSE-TREE: | | OmpDirectiveSpecification
+!PARSE-TREE: | | | llvm::omp::Directive = declare target
+!PARSE-TREE: | | | OmpArgument -> OmpLocator -> OmpObject -> Designator -> DataRef -> Name = 'f05'
+!PARSE-TREE: | | | OmpClauseList ->
+!PARSE-TREE: ImplicitPart ->
+
+subroutine f06(x, y)
+  integer :: x, y
+  !$omp metadirective when(user={condition(.true.)}: &
+  !$omp & flush(x, y))
+end
+
+!UNPARSE: SUBROUTINE f06 (x, y)
+!UNPARSE:  INTEGER x, y
+!UNPARSE: !$OMP METADIRECTIVE  WHEN(USER={CONDITION(.true._4)}: FLUSH(x, y))
+!UNPARSE: END SUBROUTINE
+
+!PARSE-TREE: DeclarationConstruct -> SpecificationConstruct -> OpenMPDeclarativeConstruct -> OmpMetadirectiveDirective
+!PARSE-TREE: | OmpClauseList -> OmpClause -> When -> OmpWhenClause
+!PARSE-TREE: | | Modifier -> OmpContextSelectorSpecification -> OmpTraitSetSelector
+!PARSE-TREE: | | | OmpTraitSetSelectorName -> Value = User
+!PARSE-TREE: | | | OmpTraitSelector
+!PARSE-TREE: | | | | OmpTraitSelectorName -> Value = Condition
+!PARSE-TREE: | | | | Properties
+!PARSE-TREE: | | | | | OmpTraitProperty -> Scalar -> Expr = '.true._4'
+!PARSE-TREE: | | | | | | LiteralConstant -> LogicalLiteralConstant
+!PARSE-TREE: | | | | | | | bool = 'true'
+!PARSE-TREE: | | OmpDirectiveSpecification
+!PARSE-TREE: | | | llvm::omp::Directive = flush
+!PARSE-TREE: | | | OmpArgument -> OmpLocator -> OmpObject -> Designator -> DataRef -> Name = 'x'
+!PARSE-TREE: | | | OmpArgument -> OmpLocator -> OmpObject -> Designator -> DataRef -> Name = 'y'
+!PARSE-TREE: | | | OmpClauseList ->
+
+subroutine f07
+  integer :: t
+  !$omp metadirective when(user={condition(.true.)}: &
+  !$omp & threadprivate(t))
+end
+
+!UNPARSE: SUBROUTINE f07
+!UNPARSE:  INTEGER t
+!UNPARSE: !$OMP METADIRECTIVE  WHEN(USER={CONDITION(.true._4)}: THREADPRIVATE(t))
+!UNPARSE: END SUBROUTINE
+
+!PARSE-TREE: DeclarationConstruct -> SpecificationConstruct -> OpenMPDeclarativeConstruct -> OmpMetadirectiveDirective
+!PARSE-TREE: | OmpClauseList -> OmpClause -> When -> OmpWhenClause
+!PARSE-TREE: | | Modifier -> OmpContextSelectorSpecification -> OmpTraitSetSelector
+!PARSE-TREE: | | | OmpTraitSetSelectorName -> Value = User
+!PARSE-TREE: | | | OmpTraitSelector
+!PARSE-TREE: | | | | OmpTraitSelectorName -> Value = Condition
+!PARSE-TREE: | | | | Properties
+!PARSE-TREE: | | | | | OmpTraitProperty -> Scalar -> Expr = '.true._4'
+!PARSE-TREE: | | | | | | LiteralConstant -> LogicalLiteralConstant
+!PARSE-TREE: | | | | | | | bool = 'true'
+!PARSE-TREE: | | OmpDirectiveSpecification
+!PARSE-TREE: | | | llvm::omp::Directive = threadprivate
+!PARSE-TREE: | | | OmpArgument -> OmpLocator -> OmpObject -> Designator -> DataRef -> Name = 't'
+!PARSE-TREE: | | | OmpClauseList ->
diff --git a/flang/test/Parser/OpenMP/metadirective-v50.f90 b/flang/test/Parser/OpenMP/metadirective-v50.f90
new file mode 100644
index 0000000000000..d7c3121b8f1b8
--- /dev/null
+++ b/flang/test/Parser/OpenMP/metadirective-v50.f90
@@ -0,0 +1,31 @@
+!RUN: %flang_fc1 -fdebug-unparse -fopenmp -fopenmp-version=50 %s | FileCheck --ignore-case --check-prefix="UNPARSE" %s
+!RUN: %flang_fc1 -fdebug-dump-parse-tree -fopenmp -fopenmp-version=50 %s | FileCheck --check-prefix="PARSE-TREE" %s
+
+subroutine f01
+  continue
+  !$omp metadirective &
+  !$omp & when(user={condition(.true.)}: nothing) &
+  !$omp & default(nothing)
+end
+
+!UNPARSE: SUBROUTINE f01
+!UNPARSE:  CONTINUE
+!UNPARSE: !$OMP METADIRECTIVE  WHEN(USER={CONDITION(.true._4)}: NOTHING) DEFAULT(NOTHING)
+!UNPARSE: END SUBROUTINE
+
+!PARSE-TREE: ExecutionPartConstruct -> ExecutableConstruct -> OpenMPConstruct -> OpenMPStandaloneConstruct -> OmpMetadirectiveDirective
+!PARSE-TREE: | OmpClauseList -> OmpClause -> When -> OmpWhenClause
+!PARSE-TREE: | | Modifier -> OmpContextSelectorSpecification -> OmpTraitSetSelector
+!PARSE-TREE: | | | OmpTraitSetSelectorName -> Value = User
+!PARSE-TREE: | | | OmpTraitSelector
+!PARSE-TREE: | | | | OmpTraitSelectorName -> Value = Condition
+!PARSE-TREE: | | | | Properties
+!PARSE-TREE: | | | | | OmpTraitProperty -> Scalar -> Expr = '.true._4'
+!PARSE-TREE: | | | | | | LiteralConstant -> LogicalLiteralConstant
+!PARSE-TREE: | | | | | | | bool = 'true'
+!PARSE-TREE: | | OmpDirectiveSpecification
+!PARSE-TREE: | | | llvm::omp::Directive = nothing
+!PARSE-TREE: | | | OmpClauseList ->
+!PARSE-TREE: | OmpClause -> Default -> OmpDefaultClause -> OmpDirectiveSpecification
+!PARSE-TREE: | | llvm::omp::Directive = nothing
+!PARSE-TREE: | | OmpClauseList ->
diff --git a/flang/test/Parser/OpenMP/metadirective.f90 b/flang/test/Parser/OpenMP/metadirective.f90
new file mode 100644
index 0000000000000..dce31c2e7db26
--- /dev/null
+++ b/flang/test/Parser/OpenMP/metadirective.f90
@@ -0,0 +1,251 @@
+!RUN: %flang_fc1 -fdebug-unparse -fopenmp -fopenmp-version=52 %s | FileCheck --ignore-case --check-prefix="UNPARSE" %s
+!RUN: %flang_fc1 -fdebug-dump-parse-tree -fopenmp -fopenmp-version=52 %s | FileCheck --check-prefix="PARSE-TREE" %s
+
+subroutine f00
+  continue
+  !$omp metadirective when(construct={target, parallel}: nothing)
+end
+
+!UNPARSE: SUBROUTINE f00
+!UNPARSE:  CONTINUE
+!UNPARSE: !$OMP METADIRECTIVE  WHEN(CONSTRUCT={TARGET, PARALLEL}: NOTHING)
+!UNPARSE: END SUBROUTINE
+
+!PARSE-TREE: ExecutionPartConstruct -> ExecutableConstruct -> OpenMPConstruct -> OpenMPStandaloneConstruct -> OmpMetadirectiveDirective
+!PARSE-TREE: | OmpClauseList -> OmpClause -> When -> OmpWhenClause
+!PARSE-TREE: | | Modifier -> OmpContextSelectorSpecification -> OmpTraitSetSelector
+!PARSE-TREE: | | | OmpTraitSetSelectorName -> Value = Construct
+!PARSE-TREE: | | | OmpTraitSelector
+!PARSE-TREE: | | | | OmpTraitSelectorName -> llvm::omp::Directive = target
+!PARSE-TREE: | | | OmpTraitSelector
+!PARSE-TREE: | | | | OmpTraitSelectorName -> llvm::omp::Directive = parallel
+!PARSE-TREE: | | OmpDirectiveSpecification
+!PARSE-TREE: | | | llvm::omp::Directive = nothing
+!PARSE-TREE: | | | OmpClauseList ->
+
+subroutine f01
+  continue
+  !$omp metadirective when(target_device={kind(host), device_num(1)}: nothing)
+end
+
+!UNPARSE: SUBROUTINE f01
+!UNPARSE:  CONTINUE
+!UNPARSE: !$OMP METADIRECTIVE  WHEN(TARGET_DEVICE={KIND(host), DEVICE_NUM(1_4)}: NOTHING)
+!UNPARSE: END SUBROUTINE
+
+!PARSE-TREE: ExecutionPartConstruct -> ExecutableConstruct -> OpenMPConstruct -> OpenMPStandaloneConstruct -> OmpMetadirectiveDirective
+!PARSE-TREE: | OmpClauseList -> OmpClause -> When -> OmpWhenClause
+!PARSE-TREE: | | Modifier -> OmpContextSelectorSpecification -> OmpTraitSetSelector
+!PARSE-TREE: | | | OmpTraitSetSelectorName -> Value = Target_Device
+!PARSE-TREE: | | | OmpTraitSelector
+!PARSE-TREE: | | | | OmpTraitSelectorName -> Value = Kind
+!PARSE-TREE: | | | | Properties
+!PARSE-TREE: | | | | | OmpTraitProperty -> OmpTraitPropertyName -> string = 'host'
+!PARSE-TREE: | | | OmpTraitSelector
+!PARSE-TREE: | | | | OmpTraitSelectorName -> Value = Device_Num
+!PARSE-TREE: | | | | Properties
+!PARSE-TREE: | | | | | OmpTraitProperty -> Scalar -> Expr = '1_4'
+!PARSE-TREE: | | | | | | LiteralConstant -> IntLiteralConstant = '1'
+!PARSE-TREE: | | OmpDirectiveSpecification
+!PARSE-TREE: | | | llvm::omp::Directive = nothing
+!PARSE-TREE: | | | OmpClauseList ->
+
+subroutine f02
+  continue
+  !$omp metadirective when(target_device={kind(any), device_num(7)}: nothing)
+end
+
+!UNPARSE: SUBROUTINE f02
+!UNPARSE:  CONTINUE
+!UNPARSE: !$OMP METADIRECTIVE  WHEN(TARGET_DEVICE={KIND(any), DEVICE_NUM(7_4)}: NOTHING)
+!UNPARSE: END SUBROUTINE
+
+!PARSE-TREE: ExecutionPartConstruct -> ExecutableConstruct -> OpenMPConstruct -> OpenMPStandaloneConstruct -> OmpMetadirectiveDirective
+!PARSE-TREE: | OmpClauseList -> OmpClause -> When -> OmpWhenClause
+!PARSE-TREE: | | Modifier -> OmpContextSelectorSpecification -> OmpTraitSetSelector
+!PARSE-TREE: | | | OmpTraitSetSelectorName -> Value = Target_Device
+!PARSE-TREE: | | | OmpTraitSelector
+!PARSE-TREE: | | | | OmpTraitSelectorName -> Value = Kind
+!PARSE-TREE: | | | | Properties
+!PARSE-TREE: | | | | | OmpTraitProperty -> OmpTraitPropertyName -> string = 'any'
+!PARSE-TREE: | | | OmpTraitSelector
+!PARSE-TREE: | | | | OmpTraitSelectorName -> Value = Device_Num
+!PARSE-TREE: | | | | Properties
+!PARSE-TREE: | | | | | OmpTraitProperty -> Scalar -> Expr = '7_4'
+!PARSE-TREE: | | | | | | LiteralConstant -> IntLiteralConstant = '7'
+!PARSE-TREE: | | OmpDirectiveSpecification
+!PARSE-TREE: | | | llvm::omp::Directive = nothing
+!PARSE-TREE: | | | OmpClauseList ->
+
+subroutine f03
+  continue
+  !$omp metadirective &
+  !$omp & when(implementation={atomic_default_mem_order(acq_rel)}: nothing)
+end
+
+!UNPARSE: SUBROUTINE f03
+!UNPARSE:  CONTINUE
+!UNPARSE: !$OMP METADIRECTIVE  WHEN(IMPLEMENTATION={ATOMIC_DEFAULT_MEM_ORDER(ACQ_REL)}: &
+!UNPARSE: !$OMP&NOTHING)
+!UNPARSE: END SUBROUTINE
+
+!PARSE-TREE: ExecutionPartConstruct -> ExecutableConstruct -> OpenMPConstruct -> OpenMPStandaloneConstruct -> OmpMetadirectiveDirective
+!PARSE-TREE: | OmpClauseList -> OmpClause -> When -> OmpWhenClause
+!PARSE-TREE: | | Modifier -> OmpContextSelectorSpecification -> OmpTraitSetSelector
+!PARSE-TREE: | | | OmpTraitSetSelectorName -> Value = Implementation
+!PARSE-TREE: | | | OmpTraitSelector
+!PARSE-TREE: | | | | OmpTraitSelectorName -> Value = Atomic_Default_Mem_Order
+!PARSE-TREE: | | | | Properties
+!PARSE-TREE: | | | | | OmpTraitProperty -> OmpClause -> AcqRel
+!PARSE-TREE: | | OmpDirectiveSpecification
+!PARSE-TREE: | | | llvm::omp::Directive = nothing
+!PARSE-TREE: | | | OmpClauseList ->
+
+subroutine f04
+  continue
+  !$omp metadirective &
+  !$omp when(implementation={extension_trait(haha(1), foo(baz, "bar"(1)))}: nothing)
+end
+
+!UNPARSE: SUBROUTINE f04
+!UNPARSE:  CONTINUE
+!UNPARSE: !$OMP METADIRECTIVE  WHEN(IMPLEMENTATION={extension_trait(haha(1_4), foo(baz,bar(1_4&
+!UNPARSE: !$OMP&)))}: NOTHING)
+!UNPARSE: END SUBROUTINE
+
+!PARSE-TREE: ExecutionPartConstruct -> ExecutableConstruct -> OpenMPConstruct -> OpenMPStandaloneConstruct -> OmpMetadirectiveDirective
+!PARSE-TREE: | OmpClauseList -> OmpClause -> When -> OmpWhenClause
+!PARSE-TREE: | | Modifier -> OmpContextSelectorSpecification -> OmpTraitSetSelector
+!PARSE-TREE: | | | OmpTraitSetSelectorName -> Value = Implementation
+!PARSE-TREE: | | | OmpTraitSelector
+!PARSE-TREE: | | | | OmpTraitSelectorName -> string = 'extension_trait'
+!PARSE-TREE: | | | | Properties
+!PARSE-TREE: | | | | | OmpTraitProperty -> OmpTraitPropertyExtension -> Complex
+!PARSE-TREE: | | | | | | OmpTraitPropertyName -> string = 'haha'
+!PARSE-TREE: | | | | | | OmpTraitPropertyExtension -> Scalar -> Expr = '1_4'
+!PARSE-TREE: | | | | | | | LiteralConstant -> IntLiteralConstant = '1'
+!PARSE-TREE: | | | | | OmpTraitProperty -> OmpTraitPropertyExtension
+!PARSE-TREE: | | | | | | OmpTraitPropertyName -> string = 'foo'
+!PARSE-TREE: | | | | | | OmpTraitPropertyExtension -> OmpTraitPropertyName -> string = 'baz'
+!PARSE-TREE: | | | | | | OmpTraitPropertyExtension -> Complex
+!PARSE-TREE: | | | | | | | OmpTraitPropertyName -> string = 'bar'
+!PARSE-TREE: | | | | | | | OmpTraitPropertyExtension -> Scalar -> Expr = '1_4'
+!PARSE-TREE: | | | | | | | | LiteralConstant -> IntLiteralConstant = '1'
+!PARSE-TREE: | | OmpDirectiveSpecification
+!PARSE-TREE: | | | llvm::omp::Directive = nothing
+!PARSE-TREE: | | | OmpClauseList ->
+
+subroutine f05(x)
+  integer :: x
+  continue
+  !$omp metadirective &
+  !$omp & when(user={condition(score(100): .true.)}: &
+  !$omp &    parallel do reduction(+: x)) &
+  !$omp & otherwise(nothing)
+  do i = 1, 10
+  enddo
+end
+
+!UNPARSE: SUBROUTINE f05 (x)
+!UNPARSE:  INTEGER x
+!UNPARSE:  CONTINUE
+!UNPARSE: !$OMP METADIRECTIVE  WHEN(USER={CONDITION(SCORE(100_4): .true._4)}: PARALLEL DO REDUCTION(+&
+!UNPARSE: !$OMP&: x)) OTHERWISE(NOTHING)
+!UNPARSE:  DO i=1_4,10_4
+!UNPARSE:  END DO
+!UNPARSE: END SUBROUTINE
+
+!PARSE-TREE: ExecutionPartConstruct -> ExecutableConstruct -> OpenMPConstruct -> OpenMPStandaloneConstruct -> OmpMetadirectiveDirective
+!PARSE-TREE: | OmpClauseList -> OmpClause -> When -> OmpWhenClause
+!PARSE-TREE: | | Modifier -> OmpContextSelectorSpecification -> OmpTraitSetSelector
+!PARSE-TREE: | | | OmpTraitSetSelectorName -> Value = User
+!PARSE-TREE: | | | OmpTraitSelector
+!PARSE-TREE: | | | | OmpTraitSelectorName -> Value = Condition
+!PARSE-TREE: | | | | Properties
+!PARSE-TREE: | | | | | OmpTraitScore -> Scalar -> Integer -> Expr = '100_4'
+!PARSE-TREE: | | | | | | LiteralConstant -> IntLiteralConstant = '100'
+!PARSE-TREE: | | | | | OmpTraitProperty -> Scalar -> Expr = '.true._4'
+!PARSE-TREE: | | | | | | LiteralConstant -> LogicalLiteralConstant
+!PARSE-TREE: | | | | | | | bool = 'true'
+!PARSE-TREE: | | OmpDirectiveSpecification
+!PARSE-TREE: | | | llvm::omp::Directive = parallel do
+!PARSE-TREE: | | | OmpClauseList -> OmpClause -> Reduction -> OmpReductionClause
+!PARSE-TREE: | | | | Modifier -> OmpReductionIdentifier -> DefinedOperator -> IntrinsicOperator = Add
+!PARSE-TREE: | | | | OmpObjectList -> OmpObject -> Designator -> DataRef -> Name = 'x'
+!PARSE-TREE: | OmpClause -> Otherwise -> OmpOtherwiseClause -> OmpDirectiveSpecification
+!PARSE-TREE: | | llvm::omp::Directive = nothing
+!PARSE-TREE: | | OmpClauseList ->
+
+subroutine f06
+  continue
+  ! Two trait set selectors
+  !$omp metadirective &
+  !$omp & when(implementation={vendor("amd")}, &
+  !$omp &      user={condition(.true.)}: nothing)
+end
+
+!UNPARSE: SUBROUTINE f06
+!UNPARSE:  CONTINUE
+!UNPARSE: !$OMP METADIRECTIVE  WHEN(IMPLEMENTATION={VENDOR(amd)}, USER={CONDITION(.true._4)}: NO&
+!UNPARSE: !$OMP&THING)
+!UNPARSE: END SUBROUTINE
+
+!PARSE-TREE: ExecutionPartConstruct -> ExecutableConstruct -> OpenMPConstruct -> OpenMPStandaloneConstruct -> OmpMetadirectiveDirective
+!PARSE-TREE: | OmpClauseList -> OmpClause -> When -> OmpWhenClause
+!PARSE-TREE: | | Modifier -> OmpContextSelectorSpecification -> OmpTraitSetSelector
+!PARSE-TREE: | | | OmpTraitSetSelectorName -> Value = Implementation
+!PARSE-TREE: | | | OmpTraitSelector
+!PARSE-TREE: | | | | OmpTraitSelectorName -> Value = Vendor
+!PARSE-TREE: | | | | Properties
+!PARSE-TREE: | | | | | OmpTraitProperty -> OmpTraitPropertyName -> string = 'amd'
+!PARSE-TREE: | | OmpTraitSetSelector
+!PARSE-TREE: | | | OmpTraitSetSelectorName -> Value = User
+!PARSE-TREE: | | | OmpTraitSelector
+!PARSE-TREE: | | | | OmpTraitSelectorName -> Value = Condition
+!PARSE-TREE: | | | | Properties
+!PARSE-TREE: | | | | | OmpTraitProperty -> Scalar -> Expr = '.true._4'
+!PARSE-TREE: | | | | | | LiteralConstant -> LogicalLiteralConstant
+!PARSE-TREE: | | | | | | | bool = 'true'
+!PARSE-TREE: | | OmpDirectiveSpecification
+!PARSE-TREE: | | | llvm::omp::Directive = nothing
+!PARSE-TREE: | | | OmpClauseList ->
+
+subroutine f07
+  ! Declarative metadirective
+  !$omp metadirective &
+  !$omp & when(implementation={vendor("amd")}: declare simd) &
+  !$omp & when(user={condition(.true.)}: declare target) &
+  !$omp & otherwise(nothing)
+end
+
+!UNPARSE: SUBROUTINE f07
+!UNPARSE: !$OMP METADIRECTIVE  WHEN(IMPLEMENTATION={VENDOR(amd)}: DECLARE SIMD) WHEN(USE&
+!UNPARSE: !$OMP&R={CONDITION(.true._4)}: DECLARE TARGET) OTHERWISE(NOTHING)
+!UNPARSE: END SUBROUTINE
+
+!PARSE-TREE: OpenMPDeclarativeConstruct -> OmpMetadirectiveDirective
+!PARSE-TREE: | OmpClauseList -> OmpClause -> When -> OmpWhenClause
+!PARSE-TREE: | | Modifier -> OmpContextSelectorSpecification -> OmpTraitSetSelector
+!PARSE-TREE: | | | OmpTraitSetSelectorName -> Value = Implementation
+!PARSE-TREE: | | | OmpTraitSelector
+!PARSE-TREE: | | | | OmpTraitSelectorName -> Value = Vendor
+!PARSE-TREE: | | | | Properties
+!PARSE-TREE: | | | | | OmpTraitProperty -> OmpTraitPropertyName -> string = 'amd'
+!PARSE-TREE: | | OmpDirectiveSpecification
+!PARSE-TREE: | | | llvm::omp::Directive = declare simd
+!PARSE-TREE: | | | OmpClauseList ->
+!PARSE-TREE: | OmpClause -> When -> OmpWhenClause
+!PARSE-TREE: | | Modifier -> OmpContextSelectorSpecification -> OmpTraitSetSelector
+!PARSE-TREE: | | | OmpTraitSetSelectorName -> Value = User
+!PARSE-TREE: | | | OmpTraitSelector
+!PARSE-TREE: | | | | OmpTraitSelectorName -> Value = Condition
+!PARSE-TREE: | | | | Properties
+!PARSE-TREE: | | | | | OmpTraitProperty -> Scalar -> Expr = '.true._4'
+!PARSE-TREE: | | | | | | LiteralConstant -> LogicalLiteralConstant
+!PARSE-TREE: | | | | | | | bool = 'true'
+!PARSE-TREE: | | OmpDirectiveSpecification
+!PARSE-TREE: | | | llvm::omp::Directive = declare target
+!PARSE-TREE: | | | OmpClauseList ->
+!PARSE-TREE: | OmpClause -> Otherwise -> OmpOtherwiseClause -> OmpDirectiveSpecification
+!PARSE-TREE: | | llvm::omp::Directive = nothing
+!PARSE-TREE: | | OmpClauseList ->
\ No newline at end of file
diff --git a/flang/test/Parser/compiler-directives.f90 b/flang/test/Parser/compiler-directives.f90
index 246eaf985251c..f372a9f533a35 100644
--- a/flang/test/Parser/compiler-directives.f90
+++ b/flang/test/Parser/compiler-directives.f90
@@ -35,3 +35,14 @@ subroutine vector_always
   do i=1,10
   enddo
 end subroutine
+
+subroutine unroll
+  !dir$ unroll
+  ! CHECK: !DIR$ UNROLL
+  do i=1,10
+  enddo
+  !dir$ unroll 2
+  ! CHECK: !DIR$ UNROLL 2
+  do i=1,10
+  enddo
+end subroutine
diff --git a/flang/test/Runtime/no-cpp-dep.c b/flang/test/Runtime/no-cpp-dep.c
index 606a5d189f719..4fcf8f9d478d8 100644
--- a/flang/test/Runtime/no-cpp-dep.c
+++ b/flang/test/Runtime/no-cpp-dep.c
@@ -1,12 +1,12 @@
 /*
 This test makes sure that flang's runtime does not depend on the C++ runtime
-library. It tries to link this simple file against libFortranRuntime.a with
+library. It tries to link this simple file against libflang_rt.runtime.a with
 a C compiler.
 
-REQUIRES: c-compiler
+REQUIRES: c-compiler, flang-rt
 
 RUN: %if system-aix %{ export OBJECT_MODE=64 %}
-RUN: %cc -std=c99 %s -I%include %libruntime %libdecimal -lm  \
+RUN: %cc -std=c99 %s -I%include %libruntime -lm  \
 RUN: %if system-aix %{-lpthread %}
 RUN: rm a.out
 */
diff --git a/flang/test/Semantics/OpenMP/cray-pointer-usage.f90 b/flang/test/Semantics/OpenMP/cray-pointer-usage.f90
new file mode 100644
index 0000000000000..6c74462dd2789
--- /dev/null
+++ b/flang/test/Semantics/OpenMP/cray-pointer-usage.f90
@@ -0,0 +1,44 @@
+!RUN: %python %S/../test_errors.py %s %flang -fopenmp
+subroutine test_cray_pointer_usage
+  implicit none
+  integer :: i
+  real(8) :: var(*), pointee(2)
+  pointer(ivar, var)
+  ! ERROR: Cray Pointee 'var' may not appear in LINEAR clause
+  ! ERROR: The list item 'var' specified without the REF 'linear-modifier' must be of INTEGER type
+  ! ERROR: The list item `var` must be a dummy argument
+  !$omp declare simd linear(var)
+
+  pointee = 42.0
+  ivar = loc(pointee)
+
+  !$omp parallel num_threads(2) default(none)
+    ! ERROR: The DEFAULT(NONE) clause requires that the Cray Pointer 'ivar' must be listed in a data-sharing attribute clause
+    print *, var(1)
+  !$omp end parallel
+
+  ! ERROR: Cray Pointee 'var' may not appear in PRIVATE clause, use Cray Pointer 'ivar' instead
+  !$omp parallel num_threads(2) default(none) private(var)
+    print *, var(1)
+  !$omp end parallel
+
+  ! ERROR: Cray Pointee 'var' may not appear in SHARED clause, use Cray Pointer 'ivar' instead
+  !$omp parallel num_threads(2) default(none) shared(var)
+    print *, var(1)
+  !$omp end parallel
+
+  ! ERROR: Cray Pointee 'var' may not appear in LASTPRIVATE clause, use Cray Pointer 'ivar' instead
+  !$omp do lastprivate(var)
+    do i = 1, 10
+      print *, var(1)
+    end do
+  !$omp end do
+
+  !$omp parallel num_threads(2) default(none) firstprivate(ivar)
+    print *, var(1)
+  !$omp end parallel
+
+  !$omp parallel num_threads(2) default(private) shared(ivar)
+    print *, var(1)
+  !$omp end parallel
+end subroutine test_cray_pointer_usage
diff --git a/flang/test/Semantics/OpenMP/dispatch.f90 b/flang/test/Semantics/OpenMP/dispatch.f90
new file mode 100644
index 0000000000000..7dfbeecb2fc1d
--- /dev/null
+++ b/flang/test/Semantics/OpenMP/dispatch.f90
@@ -0,0 +1,24 @@
+! RUN: %python %S/../test_errors.py %s %flang -fopenmp
+
+subroutine sb1
+  integer :: r
+  r = 1
+  !ERROR: The DISPATCH construct does not contain a SUBROUTINE or FUNCTION
+  !$omp dispatch nowait
+  print *,r
+end subroutine
+subroutine sb2
+  integer :: r
+!ERROR: The DISPATCH construct is empty or contains more than one statement
+  !$omp dispatch
+  call foo()
+  r = bar()
+  !$omp end dispatch
+contains
+  subroutine foo
+  end subroutine foo
+  function bar
+    integer :: bar
+    bar = 2
+  end function
+end subroutine
diff --git a/flang/test/Semantics/OpenMP/metadirective-common.f90 b/flang/test/Semantics/OpenMP/metadirective-common.f90
new file mode 100644
index 0000000000000..4988fae9e8edc
--- /dev/null
+++ b/flang/test/Semantics/OpenMP/metadirective-common.f90
@@ -0,0 +1,37 @@
+!RUN: %python %S/../test_errors.py %s %flang -fopenmp -fopenmp-version=52
+
+! Common context selector tests
+
+subroutine f00
+  !$omp metadirective &
+  !$omp & when(implementation={vendor("this")}, &
+!ERROR: Repeated trait set name IMPLEMENTATION in a context specifier
+  !$omp &      implementation={requires(unified_shared_memory)}: nothing)
+end
+
+subroutine f01
+  !$omp metadirective &
+!ERROR: Repeated trait name ISA in a trait set
+  !$omp & when(device={isa("this"), isa("that")}: nothing)
+end
+
+subroutine f02
+  !$omp metadirective &
+!ERROR: SCORE expression must be a non-negative constant integer expression
+  !$omp & when(user={condition(score(-2): .true.)}: nothing)
+end
+
+subroutine f03(x)
+  integer :: x
+  !$omp metadirective &
+!ERROR: SCORE expression must be a non-negative constant integer expression
+  !$omp & when(user={condition(score(x): .true.)}: nothing)
+end
+
+subroutine f04
+  !$omp metadirective &
+!ERROR: Trait property should be a scalar expression
+!ERROR: More invalid properties are present
+  !$omp & when(target_device={device_num("device", "foo"(1))}: nothing)
+end
+
diff --git a/flang/test/Semantics/OpenMP/metadirective-construct.f90 b/flang/test/Semantics/OpenMP/metadirective-construct.f90
new file mode 100644
index 0000000000000..1dd23b1dca67b
--- /dev/null
+++ b/flang/test/Semantics/OpenMP/metadirective-construct.f90
@@ -0,0 +1,33 @@
+!RUN: %python %S/../test_errors.py %s %flang -fopenmp -fopenmp-version=52
+
+! The CONSTRUCT trait set
+
+subroutine f00
+  !$omp metadirective &
+!ERROR: CONDITION is not a valid trait for CONSTRUCT trait set
+  !$omp & when(construct={condition(.true.)}: nothing)
+end
+
+subroutine f01
+  !$omp metadirective &
+!ERROR: Directive-name traits cannot have properties
+  !$omp & when(construct={parallel(nowait), simd}: nothing)
+end
+
+subroutine f02
+  !$omp metadirective &
+!ERROR: SIMD trait requires a clause that is allowed on the DECLARE SIMD directive
+  !$omp & when(construct={simd(nowait)}: nothing)
+end
+
+subroutine f03
+  !$omp metadirective &
+!ERROR: Extension traits are not valid for CONSTRUCT trait set
+  !$omp & when(construct={fred(1)}: nothing)
+end
+
+subroutine f04
+  !$omp metadirective &
+!This is ok
+  !$omp & when(construct={parallel, simd(simdlen(32), notinbranch)}: nothing)
+end
diff --git a/flang/test/Semantics/OpenMP/metadirective-device.f90 b/flang/test/Semantics/OpenMP/metadirective-device.f90
new file mode 100644
index 0000000000000..fb11499066247
--- /dev/null
+++ b/flang/test/Semantics/OpenMP/metadirective-device.f90
@@ -0,0 +1,36 @@
+!RUN: %python %S/../test_errors.py %s %flang -fopenmp -fopenmp-version=52
+
+! The DEVICE and TARGET_DEVICE trait sets
+
+subroutine f00
+  !$omp metadirective &
+!ERROR: DEVICE_NUM is not a valid trait for DEVICE trait set
+  !$omp & when(device={device_num(10)}: nothing)
+end
+
+subroutine f01
+  !$omp metadirective &
+!This is ok: all traits are valid
+  !$omp & when(device={arch("some-arch"), isa("some-isa"), kind("some-kind")}:&
+  !$omp & nothing)
+end
+
+subroutine f02
+  !$omp metadirective &
+!This is ok: all traits are valid
+  !$omp & when(target_device={arch("some-arch"), device_num(10), &
+  !$omp & isa("some-isa"), kind("some-kind"), uid("some-uid")}: nothing)
+end
+
+subroutine f03
+  !$omp metadirective &
+!This is ok: extension traits are allowed
+  !$omp & when(device={some_new_trait}: nothing)
+end
+
+subroutine f04
+  !$omp metadirective &
+!This is ok: extension traits are allowed
+  !$omp & when(target_device={another_new_trait(12, 21)}: nothing)
+end
+
diff --git a/flang/test/Semantics/OpenMP/metadirective-implementation.f90 b/flang/test/Semantics/OpenMP/metadirective-implementation.f90
new file mode 100644
index 0000000000000..7a7642158fc29
--- /dev/null
+++ b/flang/test/Semantics/OpenMP/metadirective-implementation.f90
@@ -0,0 +1,33 @@
+!RUN: %python %S/../test_errors.py %s %flang -fopenmp -fopenmp-version=52
+
+! The IMPLEMENTATION trait set
+
+subroutine f00
+  !$omp metadirective &
+!ERROR: Trait property should be a clause
+  !$omp & when(implementation={atomic_default_mem_order(0)}: nothing)
+end
+
+subroutine f01
+  !$omp metadirective &
+!ERROR: ATOMIC_DEFAULT_MEM_ORDER trait requires a clause from the memory-order clause set
+  !$omp & when(implementation={atomic_default_mem_order(nowait)}: nothing)
+end
+
+subroutine f02
+  !$omp metadirective &
+!ERROR: REQUIRES trait requires a clause from the requirement clause set
+!ERROR: Invalid clause specification for SHARED
+  !$omp & when(implementation={requires(shared)}: nothing)
+end
+
+subroutine f03
+  !$omp metadirective &
+!This is ok
+  !$omp & when(implementation={ &
+  !$omp &         atomic_default_mem_order(relaxed), &
+  !$omp &         extension("foo"), &
+  !$omp &         requires(unified_address),
+  !$omp &         vendor(some_vendor) &
+  !$omp &      }: nothing)
+end
diff --git a/flang/test/Semantics/OpenMP/metadirective-user.f90 b/flang/test/Semantics/OpenMP/metadirective-user.f90
new file mode 100644
index 0000000000000..c4f037d57a9d9
--- /dev/null
+++ b/flang/test/Semantics/OpenMP/metadirective-user.f90
@@ -0,0 +1,29 @@
+!RUN: %python %S/../test_errors.py %s %flang -fopenmp -fopenmp-version=52
+
+! The USER trait set
+
+subroutine f00(x)
+  integer :: x
+  !$omp metadirective &
+!ERROR: CONDITION trait requires a single LOGICAL expression
+  !$omp & when(user={condition(score(2): x)}: nothing)
+end
+
+subroutine f01
+  !$omp metadirective &
+!ERROR: CONDITION trait requires a single expression property
+  !$omp & when(user={condition(.true., .false.)}: nothing)
+end
+
+subroutine f02
+  !$omp metadirective &
+!ERROR: Extension traits are not valid for USER trait set
+  !$omp & when(user={fred}: nothing)
+end
+
+subroutine f03(x)
+  integer :: x
+  !$omp metadirective &
+!This is ok
+  !$omp & when(user={condition(x > 0)}: nothing)
+end
diff --git a/flang/test/Semantics/OpenMP/reduction-derived-component.f90 b/flang/test/Semantics/OpenMP/reduction-derived-component.f90
new file mode 100644
index 0000000000000..87e5f1acda159
--- /dev/null
+++ b/flang/test/Semantics/OpenMP/reduction-derived-component.f90
@@ -0,0 +1,10 @@
+! RUN: %python %S/../test_errors.py %s %flang_fc1 -fopenmp
+subroutine intrinsic_reduction
+  type local
+     integer alpha
+  end type local
+  type(local) a
+!ERROR: A variable that is part of another variable cannot appear on the REDUCTION clause
+!$omp parallel reduction(+:a%alpha)
+!$omp end parallel
+end subroutine intrinsic_reduction
diff --git a/flang/test/Semantics/atomic01.f90 b/flang/test/Semantics/atomic01.f90
index 046692e87c4ad..cf3804b0d605a 100644
--- a/flang/test/Semantics/atomic01.f90
+++ b/flang/test/Semantics/atomic01.f90
@@ -1,14 +1,13 @@
 ! RUN: %python %S/test_errors.py %s %flang_fc1
-! XFAIL: *
 ! This test checks for semantic errors in atomic_add() subroutine based on the
 ! statement specification in section 16.9.20 of the Fortran 2018 standard.
 
 program test_atomic_add
   use iso_fortran_env, only : atomic_int_kind
-  implicit none
+  implicit none(external, type)
 
   integer(kind=atomic_int_kind) atom_object[*], atom_array(2)[*], quantity, array(1), coarray[*], non_coarray
-  integer non_atom_object[*], non_atom, non_scalar(1), status, stat_array(1), coindexed[*]
+  integer non_atom_object[*], non_scalar(1), status, stat_array(1), coindexed[*]
   logical non_integer
 
   !___ standard-conforming calls with required arguments _______
@@ -31,63 +30,80 @@ program test_atomic_add
   !___ non-standard-conforming calls _______
 
   ! atom must be of kind atomic_int_kind
+  ! ERROR: Actual argument for 'atom=' must have kind=atomic_int_kind, but is 'INTEGER(4)'
   call atomic_add(non_atom_object, quantity)
 
   ! atom must be a coarray
+  ! ERROR: 'atom=' argument must be a scalar coarray or coindexed object for intrinsic 'atomic_add'
   call atomic_add(non_coarray, quantity)
 
   ! atom must be a scalar variable
+  ! ERROR: 'atom=' argument must be a scalar coarray or coindexed object for intrinsic 'atomic_add'
   call atomic_add(atom_array, quantity)
 
   ! atom has an unknown keyword argument
+  ! ERROR: unknown keyword argument to intrinsic 'atomic_add'
   call atomic_add(atoms=atom_object, value=quantity)
 
   ! atom has an argument mismatch
+  ! ERROR: Actual argument for 'atom=' must have kind=atomic_int_kind, but is 'INTEGER(4)'
   call atomic_add(atom=non_atom_object, value=quantity)
 
   ! value must be an integer
+  ! ERROR: Actual argument for 'value=' has bad type 'LOGICAL(4)'
   call atomic_add(atom_object, non_integer)
 
   ! value must be an integer scalar
+  ! ERROR: 'value=' argument has unacceptable rank 1
   call atomic_add(atom_object, array)
 
-  ! value must be of kind atomic_int_kind
-  call atomic_add(atom_object, non_atom)
-
   ! value has an unknown keyword argument
+  ! ERROR: unknown keyword argument to intrinsic 'atomic_add'
   call atomic_add(atom_object, values=quantity)
 
   ! value has an argument mismatch
+  ! ERROR: Actual argument for 'value=' has bad type 'LOGICAL(4)'
   call atomic_add(atom_object, value=non_integer)
 
   ! stat must be an integer
+  ! ERROR: Actual argument for 'stat=' has bad type 'LOGICAL(4)'
   call atomic_add(atom_object, quantity, non_integer)
 
   ! stat must be an integer scalar
+  ! ERROR: 'stat=' argument has unacceptable rank 1
   call atomic_add(atom_object, quantity, non_scalar)
 
   ! stat is an intent(out) argument
+  ! ERROR: Actual argument associated with INTENT(OUT) dummy argument 'stat=' is not definable
+  ! ERROR: '8_4' is not a variable or pointer
   call atomic_add(atom_object, quantity, 8)
 
   ! stat has an unknown keyword argument
+  ! ERROR: unknown keyword argument to intrinsic 'atomic_add'
   call atomic_add(atom_object, quantity, statuses=status)
 
   ! stat has an argument mismatch
+  ! ERROR: Actual argument for 'stat=' has bad type 'LOGICAL(4)'
   call atomic_add(atom_object, quantity, stat=non_integer)
 
   ! stat must not be coindexed
+  ! ERROR: 'stat' argument to 'atomic_add' may not be a coindexed object
   call atomic_add(atom_object, quantity, coindexed[1])
 
   ! Too many arguments
+  ! ERROR: too many actual arguments for intrinsic 'atomic_add'
   call atomic_add(atom_object, quantity, status, stat_array(1))
 
   ! Repeated atom keyword
+  ! ERROR: repeated keyword argument to intrinsic 'atomic_add'
   call atomic_add(atom=atom_object, atom=atom_array(1), value=quantity)
 
   ! Repeated value keyword
+  ! ERROR: repeated keyword argument to intrinsic 'atomic_add'
   call atomic_add(atom=atom_object, value=quantity, value=array(1))
 
   ! Repeated stat keyword
+  ! ERROR: repeated keyword argument to intrinsic 'atomic_add'
   call atomic_add(atom=atom_object, value=quantity, stat=status, stat=stat_array(1))
 
 end program test_atomic_add
diff --git a/flang/test/Semantics/atomic02.f90 b/flang/test/Semantics/atomic02.f90
index 10a7c126dbb6d..484239a23ede2 100644
--- a/flang/test/Semantics/atomic02.f90
+++ b/flang/test/Semantics/atomic02.f90
@@ -4,7 +4,7 @@
 
 program test_atomic_and
   use iso_fortran_env, only: atomic_int_kind, atomic_logical_kind
-  implicit none
+  implicit none(external, type)
 
   integer(kind=atomic_int_kind) :: scalar_coarray[*], non_scalar_coarray(10)[*], val, non_coarray
   integer(kind=atomic_int_kind) :: repeated_atom[*], repeated_val, array(10)
diff --git a/flang/test/Semantics/atomic03.f90 b/flang/test/Semantics/atomic03.f90
index 9bb1d1c0df6b1..495df5eb97192 100644
--- a/flang/test/Semantics/atomic03.f90
+++ b/flang/test/Semantics/atomic03.f90
@@ -4,7 +4,7 @@
 
 program test_atomic_cas
   use iso_fortran_env, only: atomic_int_kind, atomic_logical_kind
-  implicit none
+  implicit none(external, type)
 
   integer(kind=atomic_int_kind) :: int_scalar_coarray[*], non_scalar_coarray(10)[*], non_coarray
   integer(kind=atomic_int_kind) :: repeated_atom[*], array(10)
@@ -70,16 +70,16 @@ program test_atomic_cas
 
 ! mismatches where 'atom' has wrong kind
 
-  !ERROR: Actual argument for 'atom=' must have kind=atomic_int_kind or atomic_logical_kind, but is 'INTEGER(4)'
+  !ERROR: Actual argument for 'atom=' must have kind=atomic_int_kind, but is 'INTEGER(4)'
   call atomic_cas(default_kind_coarray, old_int, compare_int, new_int)
 
-  !ERROR: Actual argument for 'atom=' must have kind=atomic_int_kind or atomic_logical_kind, but is 'INTEGER(1)'
+  !ERROR: Actual argument for 'atom=' must have kind=atomic_int_kind, but is 'INTEGER(1)'
   call atomic_cas(kind1_coarray, old_int, compare_int, new_int)
 
-  !ERROR: Actual argument for 'atom=' must have kind=atomic_int_kind or atomic_logical_kind, but is 'LOGICAL(4)'
+  !ERROR: Actual argument for 'atom=' must have kind=atomic_logical_kind, but is 'LOGICAL(4)'
   call atomic_cas(default_kind_logical_coarray, old_logical, compare_logical, new_logical)
 
-  !ERROR: Actual argument for 'atom=' must have kind=atomic_int_kind or atomic_logical_kind, but is 'LOGICAL(1)'
+  !ERROR: Actual argument for 'atom=' must have kind=atomic_logical_kind, but is 'LOGICAL(1)'
   call atomic_cas(kind1_logical_coarray, old_logical, compare_logical, new_logical)
 
 ! mismatch where 'atom' has wrong type
diff --git a/flang/test/Semantics/atomic04.f90 b/flang/test/Semantics/atomic04.f90
index f065bf6404f1a..9df0b56d192a8 100644
--- a/flang/test/Semantics/atomic04.f90
+++ b/flang/test/Semantics/atomic04.f90
@@ -4,7 +4,7 @@
 
 program test_atomic_define
   use iso_fortran_env, only: atomic_int_kind, atomic_logical_kind
-  implicit none
+  implicit none(external, type)
 
   integer(kind=atomic_int_kind) :: scalar_coarray[*], non_scalar_coarray(10)[*], val, non_coarray
   integer(kind=atomic_int_kind) :: repeated_atom[*], repeated_val, array(10)
@@ -64,16 +64,16 @@ program test_atomic_define
   !ERROR: 'atom=' argument must be a scalar coarray or coindexed object for intrinsic 'atomic_define'
   call atomic_define(array, val)
 
-  !ERROR: Actual argument for 'atom=' must have kind=atomic_int_kind or atomic_logical_kind, but is 'INTEGER(4)'
+  !ERROR: Actual argument for 'atom=' must have kind=atomic_int_kind, but is 'INTEGER(4)'
   call atomic_define(default_kind_coarray, val)
 
-  !ERROR: Actual argument for 'atom=' must have kind=atomic_int_kind or atomic_logical_kind, but is 'INTEGER(1)'
+  !ERROR: Actual argument for 'atom=' must have kind=atomic_int_kind, but is 'INTEGER(1)'
   call atomic_define(kind1_coarray, val)
 
-  !ERROR: Actual argument for 'atom=' must have kind=atomic_int_kind or atomic_logical_kind, but is 'LOGICAL(4)'
+  !ERROR: Actual argument for 'atom=' must have kind=atomic_logical_kind, but is 'LOGICAL(4)'
   call atomic_define(default_kind_logical_coarray, val_logical)
 
-  !ERROR: Actual argument for 'atom=' must have kind=atomic_int_kind or atomic_logical_kind, but is 'LOGICAL(1)'
+  !ERROR: Actual argument for 'atom=' must have kind=atomic_logical_kind, but is 'LOGICAL(1)'
   call atomic_define(kind1_logical_coarray, val_logical)
 
   !ERROR: 'value=' argument to 'atomic_define' must have same type as 'atom=', but is 'LOGICAL(8)'
diff --git a/flang/test/Semantics/atomic05.f90 b/flang/test/Semantics/atomic05.f90
index 04c29cdd6046b..98d6b19b1f23d 100644
--- a/flang/test/Semantics/atomic05.f90
+++ b/flang/test/Semantics/atomic05.f90
@@ -4,7 +4,7 @@
 
 program test_atomic_fetch_add
   use iso_fortran_env, only: atomic_int_kind, atomic_logical_kind
-  implicit none
+  implicit none(external, type)
 
   integer(kind=atomic_int_kind) :: scalar_coarray[*], non_scalar_coarray(10)[*], val, old_val, non_coarray
   integer(kind=atomic_int_kind) :: repeated_atom[*], repeated_old, repeated_val, array(10)
diff --git a/flang/test/Semantics/atomic06.f90 b/flang/test/Semantics/atomic06.f90
index e6307d129262e..c6a23dd0077ca 100644
--- a/flang/test/Semantics/atomic06.f90
+++ b/flang/test/Semantics/atomic06.f90
@@ -4,7 +4,7 @@
 
 program test_atomic_fetch_and
   use iso_fortran_env, only: atomic_int_kind, atomic_logical_kind
-  implicit none
+  implicit none(external, type)
 
   integer(kind=atomic_int_kind) :: scalar_coarray[*], non_scalar_coarray(10)[*], val, old_val, non_coarray
   integer(kind=atomic_int_kind) :: repeated_atom[*], repeated_old, repeated_val, array(10)
diff --git a/flang/test/Semantics/atomic07.f90 b/flang/test/Semantics/atomic07.f90
index 0ac7ad152e86b..2bc544b757864 100644
--- a/flang/test/Semantics/atomic07.f90
+++ b/flang/test/Semantics/atomic07.f90
@@ -4,7 +4,7 @@
 
 program test_atomic_fetch_or
   use iso_fortran_env, only: atomic_int_kind
-  implicit none
+  implicit none(external, type)
 
   integer(kind=atomic_int_kind) :: scalar_coarray[*], non_scalar_coarray(10)[*], val, old_val, non_coarray
   integer(kind=atomic_int_kind) :: repeated_atom[*], repeated_old, repeated_val, array(10), val_coarray[*], old_val_coarray[*]
diff --git a/flang/test/Semantics/atomic08.f90 b/flang/test/Semantics/atomic08.f90
index a08512f1c7fe8..f519f9735e00e 100644
--- a/flang/test/Semantics/atomic08.f90
+++ b/flang/test/Semantics/atomic08.f90
@@ -4,7 +4,7 @@
 
 program test_atomic_fetch_xor
   use iso_fortran_env, only: atomic_int_kind, atomic_logical_kind
-  implicit none
+  implicit none(external, type)
 
   integer(kind=atomic_int_kind) :: scalar_coarray[*], non_scalar_coarray(10)[*], val, old_val, non_coarray
   integer(kind=atomic_int_kind) :: repeated_atom[*], repeated_old, repeated_val, array(10)
diff --git a/flang/test/Semantics/atomic09.f90 b/flang/test/Semantics/atomic09.f90
index fc09724d53bc0..e4e062252659a 100644
--- a/flang/test/Semantics/atomic09.f90
+++ b/flang/test/Semantics/atomic09.f90
@@ -4,7 +4,7 @@
 
 program test_atomic_or
   use iso_fortran_env, only: atomic_int_kind, atomic_logical_kind
-  implicit none
+  implicit none(external, type)
 
   integer(kind=atomic_int_kind) :: scalar_coarray[*], non_scalar_coarray(10)[*], val, non_coarray
   integer(kind=atomic_int_kind) :: repeated_atom[*], repeated_val, array(10)
diff --git a/flang/test/Semantics/atomic10.f90 b/flang/test/Semantics/atomic10.f90
index 46fcf537f1810..04efbd6e80fd2 100644
--- a/flang/test/Semantics/atomic10.f90
+++ b/flang/test/Semantics/atomic10.f90
@@ -4,7 +4,7 @@
 
 program test_atomic_ref
   use iso_fortran_env, only: atomic_int_kind, atomic_logical_kind
-  implicit none
+  implicit none(external, type)
 
   integer(kind=atomic_int_kind) :: scalar_coarray[*], non_scalar_coarray(10)[*], val, non_coarray
   integer(kind=atomic_int_kind) :: repeated_atom[*], repeated_val, array(10)
@@ -64,16 +64,16 @@ program test_atomic_ref
   !ERROR: 'atom=' argument must be a scalar coarray or coindexed object for intrinsic 'atomic_ref'
   call atomic_ref(val, array)
 
-  !ERROR: Actual argument for 'atom=' must have kind=atomic_int_kind or atomic_logical_kind, but is 'INTEGER(4)'
+  !ERROR: Actual argument for 'atom=' must have kind=atomic_int_kind, but is 'INTEGER(4)'
   call atomic_ref(val, default_kind_coarray)
 
-  !ERROR: Actual argument for 'atom=' must have kind=atomic_int_kind or atomic_logical_kind, but is 'INTEGER(1)'
+  !ERROR: Actual argument for 'atom=' must have kind=atomic_int_kind, but is 'INTEGER(1)'
   call atomic_ref(val, kind1_coarray)
 
-  !ERROR: Actual argument for 'atom=' must have kind=atomic_int_kind or atomic_logical_kind, but is 'LOGICAL(4)'
+  !ERROR: Actual argument for 'atom=' must have kind=atomic_logical_kind, but is 'LOGICAL(4)'
   call atomic_ref(val_logical, default_kind_logical_coarray)
 
-  !ERROR: Actual argument for 'atom=' must have kind=atomic_int_kind or atomic_logical_kind, but is 'LOGICAL(1)'
+  !ERROR: Actual argument for 'atom=' must have kind=atomic_logical_kind, but is 'LOGICAL(1)'
   call atomic_ref(val_logical, kind1_logical_coarray)
 
   !ERROR: 'value=' argument to 'atomic_ref' must have same type as 'atom=', but is 'LOGICAL(8)'
diff --git a/flang/test/Semantics/atomic11.f90 b/flang/test/Semantics/atomic11.f90
index 1c50825e5541f..d4f951ea02c32 100644
--- a/flang/test/Semantics/atomic11.f90
+++ b/flang/test/Semantics/atomic11.f90
@@ -4,7 +4,7 @@
 
 program test_atomic_xor
   use iso_fortran_env, only: atomic_int_kind, atomic_logical_kind
-  implicit none
+  implicit none(external, type)
 
   integer(kind=atomic_int_kind) :: scalar_coarray[*], non_scalar_coarray(10)[*], val, non_coarray
   integer(kind=atomic_int_kind) :: repeated_atom[*], repeated_val, array(10)
diff --git a/flang/test/Semantics/bind-c06.f90 b/flang/test/Semantics/bind-c06.f90
index 3ad3078c4b4a0..ff78a4743deee 100644
--- a/flang/test/Semantics/bind-c06.f90
+++ b/flang/test/Semantics/bind-c06.f90
@@ -95,4 +95,13 @@ program main
     real :: x(0)
   end type
 
+  interface
+    subroutine badAssumedLen(x,y,z) bind(c)
+      !ERROR: A BIND(C) object must have an interoperable type
+      character(*), pointer :: x
+      !ERROR: A BIND(C) object must have an interoperable type
+      character(*), allocatable :: y
+      character(*) z ! ok
+    end
+  end interface
 end
diff --git a/flang/test/Semantics/bug123534.f90 b/flang/test/Semantics/bug123534.f90
new file mode 100644
index 0000000000000..0a94336732110
--- /dev/null
+++ b/flang/test/Semantics/bug123534.f90
@@ -0,0 +1,33 @@
+! RUN: %python %S/test_modfile.py %s %flang_fc1
+! Simplified regression test for crashreported in
+! https://github.com/llvm/llvm-project/issues/123534.
+module m
+  interface
+    ! f1 returns a pointer to a procedure whose result characteristics
+    ! depend on the value of a dummy argument.
+    function f1()
+      interface
+        function f2(n)
+          integer, intent(in) :: n
+          character(n), pointer :: f2
+        end
+      end interface
+      procedure (f2), pointer :: f1
+    end
+  end interface
+end
+
+!Expect: m.mod
+!module m
+!interface
+!function f1()
+!interface
+!function f2(n)
+!integer(4),intent(in)::n
+!character(n,1),pointer::f2
+!end
+!end interface
+!procedure(f2),pointer::f1
+!end
+!end interface
+!end
diff --git a/flang/test/Semantics/bug123538.f90 b/flang/test/Semantics/bug123538.f90
new file mode 100644
index 0000000000000..2245abe3829e2
--- /dev/null
+++ b/flang/test/Semantics/bug123538.f90
@@ -0,0 +1,7 @@
+!RUN: %python %S/test_errors.py %s %flang_fc1
+procedure(), pointer :: pp => tan
+!ERROR: EXTERNAL attribute was already specified on 'pp'
+!ERROR: POINTER attribute was already specified on 'pp'
+!ERROR: 'pp' was previously initialized
+procedure(real), pointer :: pp => tan
+end
diff --git a/flang/test/Semantics/bug124487.f90 b/flang/test/Semantics/bug124487.f90
new file mode 100644
index 0000000000000..b91757c362367
--- /dev/null
+++ b/flang/test/Semantics/bug124487.f90
@@ -0,0 +1,14 @@
+!RUN: %flang_fc1 -fsyntax-only %s 2>&1 | FileCheck --allow-empty %s
+!CHECK-NOT: error:
+module m
+  interface
+    module subroutine smp(x)
+      character, external :: x
+    end
+  end interface
+end
+submodule (m) sm
+ contains
+  module procedure smp ! crashes here
+  end
+end
diff --git a/flang/test/Semantics/bug124621.f90 b/flang/test/Semantics/bug124621.f90
new file mode 100644
index 0000000000000..1106ed4a25c49
--- /dev/null
+++ b/flang/test/Semantics/bug124621.f90
@@ -0,0 +1,46 @@
+! RUN: %python %S/test_errors.py %s %flang_fc1
+module m
+  type t1
+   contains
+    procedure, pass(from) :: defAsst1
+    generic :: assignment(=) => defAsst1
+  end type
+  type t2
+  end type
+  type t3
+  end type
+  interface assignment(=)
+    module procedure defAsst2
+  end interface
+ contains
+  subroutine defAsst1(to,from)
+    class(*), intent(out) :: to
+    class(t1), intent(in) :: from
+  end
+  subroutine defAsst2(to,from)
+    class(*), intent(out) :: to
+    class(t2), intent(in) :: from
+  end
+end
+
+program test
+  use m
+  type(t1) x1
+  type(t2) x2
+  type(t3) x3
+  j = x1
+  j = x2
+  !ERROR: No intrinsic or user-defined ASSIGNMENT(=) matches operand types INTEGER(4) and TYPE(t3)
+  j = x3
+  x1 = x1
+  x1 = x2
+  !ERROR: No intrinsic or user-defined ASSIGNMENT(=) matches operand types TYPE(t1) and TYPE(t3)
+  x1 = x3
+  x2 = x1
+  x2 = x2
+  !ERROR: No intrinsic or user-defined ASSIGNMENT(=) matches operand types TYPE(t2) and TYPE(t3)
+  x2 = x3
+  x3 = x1
+  x3 = x2
+  x3 = x3
+end
diff --git a/flang/test/Semantics/bug124716.f90 b/flang/test/Semantics/bug124716.f90
new file mode 100644
index 0000000000000..c1487a235721c
--- /dev/null
+++ b/flang/test/Semantics/bug124716.f90
@@ -0,0 +1,36 @@
+! RUN: %python %S/test_modfile.py %s %flang_fc1
+MODULE m1
+  INTERFACE
+    MODULE SUBROUTINE sub1(N, ARR)
+      INTEGER, INTENT(IN) :: N
+      INTEGER, DIMENSION(N) :: ARR
+    END SUBROUTINE
+  END INTERFACE
+END MODULE
+SUBMODULE (m1) m1sub
+ CONTAINS
+  MODULE SUBROUTINE sub1(N, ARR)
+    INTEGER, INTENT(IN) :: N
+    INTEGER, DIMENSION(N) :: ARR
+    PRINT *, "sub1", N, ARR
+  END SUBROUTINE
+END SUBMODULE
+
+!Expect: m1.mod
+!module m1
+!interface
+!module subroutine sub1(n,arr)
+!integer(4),intent(in)::n
+!integer(4)::arr(1_8:int(n,kind=8))
+!end
+!end interface
+!end
+
+!Expect: m1-m1sub.mod
+!submodule(m1) m1sub
+!contains
+!module subroutine sub1(n,arr)
+!integer(4),intent(in)::n
+!integer(4)::arr(1_8:int(n,kind=8))
+!end
+!end
diff --git a/flang/test/Semantics/bug124731.f90 b/flang/test/Semantics/bug124731.f90
new file mode 100644
index 0000000000000..924b41dd1db48
--- /dev/null
+++ b/flang/test/Semantics/bug124731.f90
@@ -0,0 +1,24 @@
+!RUN: %flang_fc1 -fsyntax-only %s  2>&1 | FileCheck %s --allow-empty
+!CHECK-NOT: error:
+module m1
+  interface
+    module subroutine foo
+    end
+  end interface
+  real x
+end
+module m2
+  use m1
+end
+submodule(m1) sm1
+  use m2 ! ok
+ contains
+  module procedure foo
+  end
+end
+submodule(m1) sm2
+ contains
+  subroutine bar
+    use m2 ! ok
+  end
+end
diff --git a/flang/test/Semantics/bug12477.f90 b/flang/test/Semantics/bug12477.f90
new file mode 100644
index 0000000000000..52d079e3b26bc
--- /dev/null
+++ b/flang/test/Semantics/bug12477.f90
@@ -0,0 +1,26 @@
+!RUN: %flang_fc1 -fsyntax-only %s  2>&1 | FileCheck %s --allow-empty
+!CHECK-NOT: error:
+module m
+  type t
+   contains
+    procedure nonelemental
+    generic :: operator(+) => nonelemental
+  end type
+  interface operator(+)
+    procedure elemental
+  end interface
+ contains
+  type(t) elemental function elemental (a, b)
+    class(t), intent(in) :: a, b
+    elemental = t()
+  end
+  type(t) function nonelemental (a, b)
+    class(t), intent(in) :: a, b(:)
+    nonelemental = t()
+  end
+end
+program main
+  use m
+  type(t) x, y(1)
+  x = x + y ! ok
+end
diff --git a/flang/test/Semantics/bug124976.f90 b/flang/test/Semantics/bug124976.f90
new file mode 100644
index 0000000000000..29c21d4ead847
--- /dev/null
+++ b/flang/test/Semantics/bug124976.f90
@@ -0,0 +1,33 @@
+!RUN: %python %S/test_errors.py %s %flang_fc1
+program main
+  type base
+    integer :: x = 1
+  end type
+  type, extends(base) :: child
+    integer :: y = 2
+  end type
+  class(child), allocatable :: c1(:), c2(:,:)
+  class(base), allocatable :: b1(:), b2(:,:)
+  logical var(1)
+  common /blk/ var
+  allocate(c1(2), c2(2,2), b1(2), b2(2,2))
+  !ERROR: Actual argument for 'pad=' has bad type or kind 'CLASS(base)'
+  c2 = reshape(c1, shape(c2), pad=b1)
+  b2 = reshape(b1, shape(b2), pad=c1) ! ok
+  !ERROR: Actual argument for 'to=' has bad type or kind 'CLASS(child)'
+  call move_alloc(b1, c1)
+  call move_alloc(c1, b1) ! ok
+  !ERROR: Actual argument for 'boundary=' has bad type or kind 'CLASS(base)'
+  c1 = eoshift(c1, 1, b1(1))
+  c1 = eoshift(c1, 1, c2(1,1)) ! ok
+  b1 = eoshift(b1, 1, c1(1)) ! ok
+  !ERROR: Actual argument for 'fsource=' has bad type or kind 'CLASS(child)'
+  b1 = merge(b1, c1, var(1))
+  !ERROR: Actual argument for 'fsource=' has bad type or kind 'CLASS(base)'
+  b1 = merge(c1, b1, var(1))
+  b1 = merge(b1, b1, var(1)) ! ok
+  !ERROR: Actual argument for 'vector=' has bad type or kind 'CLASS(base)'
+  c1 = pack(c1, var, b1)
+  c1 = pack(c1, var, c1) ! ok
+  b1 = pack(b1, var, c1) ! ok
+end
diff --git a/flang/test/Semantics/call04.f90 b/flang/test/Semantics/call04.f90
index 9be579fb696c0..3b079aa4fb2b1 100644
--- a/flang/test/Semantics/call04.f90
+++ b/flang/test/Semantics/call04.f90
@@ -56,11 +56,11 @@ subroutine s05(x) ! C846
 subroutine s06(x) ! C847
   use ISO_FORTRAN_ENV, only: lock_type
   !ERROR: An INTENT(OUT) dummy argument may not be, or contain, EVENT_TYPE or LOCK_TYPE
-  type(lock_type), intent(out) :: x
+  type(lock_type), intent(out) :: x[*]
 end subroutine
 
 subroutine s07(x) ! C847
   use ISO_FORTRAN_ENV, only: event_type
   !ERROR: An INTENT(OUT) dummy argument may not be, or contain, EVENT_TYPE or LOCK_TYPE
-  type(event_type), intent(out) :: x
+  type(event_type), intent(out) :: x[*]
 end subroutine
diff --git a/flang/test/Semantics/call08.f90 b/flang/test/Semantics/call08.f90
index f4c690e0c96e0..1b7029102309c 100644
--- a/flang/test/Semantics/call08.f90
+++ b/flang/test/Semantics/call08.f90
@@ -26,6 +26,7 @@ subroutine test(x,c3,c4)
     real :: x(:)[*]
     real, intent(in) :: c3(:)[*]
     real, contiguous, intent(in) :: c4(:)[*]
+    character(2) :: coarr(2)[*] = [ "ab", "cd" ]
     call s01(c1) ! ok
     call s02(c2) ! ok
     call s03(c4) ! ok
@@ -44,5 +45,6 @@ subroutine test(x,c3,c4)
     call s04(c3)
     !ERROR: Actual argument associated with coarray dummy argument 'x=' (not assumed shape or rank) must be simply contiguous
     call s04(x)
+    print *, ichar(coarr(:)(1:1)) ! ok, ensure no bogus contiguity error
   end subroutine
 end module
diff --git a/flang/test/Semantics/call13.f90 b/flang/test/Semantics/call13.f90
index 8b203e4b715d5..3f7fb2efc8f63 100644
--- a/flang/test/Semantics/call13.f90
+++ b/flang/test/Semantics/call13.f90
@@ -22,8 +22,8 @@ subroutine s(assumedRank, coarray, class, classStar, typeStar)
   call implicit10(1, 2, keyword=3)  ! 15.4.2.2(1)
   !ERROR: Assumed rank argument requires an explicit interface
   call implicit11(assumedRank)  ! 15.4.2.2(3)(c)
-  !ERROR: Coarray argument requires an explicit interface
-  call implicit12(coarray)  ! 15.4.2.2(3)(d)
+  call implicit12(coarray)  ! ok
+  call implicit12a(coarray[1]) ! ok
   !ERROR: Parameterized derived type actual argument requires an explicit interface
   call implicit13(pdtx)  ! 15.4.2.2(3)(e)
   call implicit14(class)  ! ok
diff --git a/flang/test/Semantics/call38.f90 b/flang/test/Semantics/call38.f90
index 34aae6b8b1835..b1a35973e35fe 100644
--- a/flang/test/Semantics/call38.f90
+++ b/flang/test/Semantics/call38.f90
@@ -544,3 +544,39 @@ subroutine sub2(arg2)
       character(*) :: arg2(10)
     end subroutine sub2
 end subroutine
+
+subroutine bug123807
+  interface
+    subroutine test(s)
+      character(5), intent(inout) :: s(5)
+    end
+  end interface
+  character(30) :: s30a
+  character(30), allocatable :: s30b
+  character(6) :: s30c(5)
+  character(24) :: s24a
+  character(24), allocatable :: s24b
+  character(4) :: s24c(6)
+  allocate(s30b)
+  allocate(s24b)
+  call test(s30a)
+  call test(s30a(6:))
+  !ERROR: Actual argument has fewer characters remaining in storage sequence (24) than dummy argument 's=' (25)
+  call test(s30a(7:))
+  call test(s30b)
+  call test(s30b(6:))
+  !ERROR: Actual argument has fewer characters remaining in storage sequence (24) than dummy argument 's=' (25)
+  call test(s30b(7:))
+  call test(s30c)
+  call test(s30c(1)(6:))
+  !ERROR: Actual argument has fewer characters remaining in storage sequence (24) than dummy argument 's=' (25)
+  call test(s30c(2))
+  !ERROR: Actual argument has fewer characters remaining in storage sequence (24) than dummy argument 's=' (25)
+  call test(s30c(2)(1:))
+  !ERROR: Actual argument has fewer characters remaining in storage sequence (24) than dummy argument 's=' (25)
+  call test(s24a)
+  !ERROR: Actual argument has fewer characters remaining in storage sequence (24) than dummy argument 's=' (25)
+  call test(s24b)
+  !ERROR: Actual argument array has fewer characters (24) than dummy argument 's=' array (25)
+  call test(s24c)
+end
diff --git a/flang/test/Semantics/complex01.f90 b/flang/test/Semantics/complex01.f90
index c9d408ee0e111..d268eafa68589 100644
--- a/flang/test/Semantics/complex01.f90
+++ b/flang/test/Semantics/complex01.f90
@@ -1,5 +1,5 @@
 ! RUN: %python %S/test_errors.py %s %flang_fc1
-! C718 Each named constant in a complex literal constant shall be of type 
+! C718 Each named constant in a complex literal constant shall be of type
 ! integer or real.
 subroutine s()
   integer :: ivar = 35
@@ -30,4 +30,6 @@ subroutine s()
   complex :: cvar11 = (cconst, 1.0)
   !ERROR: operands must be INTEGER, UNSIGNED, REAL, or BOZ
   complex :: cvar12 = (lconst, 1.0)
+  !ERROR: operands cannot both be BOZ
+  complex :: cvar13 = (z'3f700000', z'00000000')
 end subroutine s
diff --git a/flang/test/Semantics/critical02.f90 b/flang/test/Semantics/critical02.f90
index e1c9bb3e0ff10..692b06b025861 100644
--- a/flang/test/Semantics/critical02.f90
+++ b/flang/test/Semantics/critical02.f90
@@ -82,7 +82,7 @@ end subroutine test8
 
 subroutine test9()
   use iso_fortran_env
-  type(lock_type) :: l
+  type(lock_type), save :: l[*]
 
   critical
     !ERROR: An image control statement is not allowed in a CRITICAL construct
diff --git a/flang/test/Semantics/cuf03.cuf b/flang/test/Semantics/cuf03.cuf
index 93b136ad7d315..fe9dd5b3ecf05 100644
--- a/flang/test/Semantics/cuf03.cuf
+++ b/flang/test/Semantics/cuf03.cuf
@@ -54,6 +54,14 @@ module m
   !ERROR: Object 'um' with ATTRIBUTES(UNIFIED) must be declared in a host subprogram
   real, unified :: um
 
+  type :: t3
+  !ERROR: Component 'r' with ATTRIBUTES(DEVICE) must also be allocatable or pointer
+    real, device :: r
+    real, device, pointer :: rp ! ok
+    real, device, allocatable :: ra ! ok
+    real, device, pointer, contiguous :: rpc ! ok
+  end type
+
  contains
   attributes(device) subroutine devsubr(n,da,rs)
     integer, intent(in) :: n
diff --git a/flang/test/Semantics/cuf09.cuf b/flang/test/Semantics/cuf09.cuf
index fda8938b8ffc0..7d32e0d70ba36 100644
--- a/flang/test/Semantics/cuf09.cuf
+++ b/flang/test/Semantics/cuf09.cuf
@@ -192,3 +192,16 @@ program main
      enddo
   enddo
 end
+
+subroutine host1()
+  integer, device :: a(32)
+  integer :: i, j
+
+  !$cuf kernel do(1) <<<*,32>>>
+  do i = 1, 32
+    a(i) = a(i) * 2.0
+    !ERROR: 'syncthreads' may not be called in device code
+    call syncthreads() ! missing explicit use cudadevice
+    a(i) = a(i) + a(j) - 34.0
+  end do
+end 
diff --git a/flang/test/Semantics/cuf18.cuf b/flang/test/Semantics/cuf18.cuf
new file mode 100644
index 0000000000000..e51e5c9f97e03
--- /dev/null
+++ b/flang/test/Semantics/cuf18.cuf
@@ -0,0 +1,68 @@
+! RUN: %python %S/test_errors.py %s %flang_fc1 -fopenacc
+
+subroutine sub1()
+  real, allocatable, device :: a(:)
+  integer :: i
+
+!ERROR: Unsupported CUDA data transfer
+  a = a + 10 ! Illegal expression according to 3.4.2
+
+  !$cuf kernel do
+  do i = 1, 10
+    a(i) = a(i) + 10 ! ok in cuf kernel do
+  end do
+
+  !$acc parallel loop
+  do i = 1, 10
+    a(i) = a(i) + 10 ! ok in openacc combined construct
+  end do
+
+  !$acc serial loop
+  do i = 1, 10
+    a(i) = a(i) + 10 ! ok in openacc combined construct
+  end do
+
+  !$acc kernels loop
+  do i = 1, 10
+    a(i) = a(i) + 10 ! ok in openacc combined construct
+  end do
+
+  !$acc parallel
+  !$acc loop
+  do i = 1, 10
+    a(i) = a(i) + 10 ! ok in nested openacc construct
+  end do
+  !$acc end parallel
+
+  !$acc kernels
+  !$acc loop
+  do i = 1, 10
+    a(i) = a(i) + 10 ! ok in nested openacc construct
+  end do
+  !$acc end kernels
+
+  !$acc serial
+  !$acc loop
+  do i = 1, 10
+    a(i) = a(i) + 10 ! ok in nested openacc construct
+  end do
+  !$acc end serial
+
+  !$acc loop
+  do i = 1, 10
+    a(i) = a(i) + 10 ! ok acc loop
+  end do
+
+  !$acc data
+
+  do i = 1, 10
+!ERROR: Unsupported CUDA data transfer
+    a(i) = a(i) + 10
+  end do
+
+  !$acc end data
+
+end subroutine
+
+
+
diff --git a/flang/test/Semantics/doconcurrent01.f90 b/flang/test/Semantics/doconcurrent01.f90
index 9bb2b45376835..9d2c9e1ab3115 100644
--- a/flang/test/Semantics/doconcurrent01.f90
+++ b/flang/test/Semantics/doconcurrent01.f90
@@ -97,7 +97,7 @@ end subroutine s3
 
 subroutine s4()
   use iso_fortran_env
-  type(lock_type) :: l
+  type(lock_type), save :: l[*]
 
   do concurrent (i = 1:n)
 !ERROR: An image control statement is not allowed in DO CONCURRENT
diff --git a/flang/test/Semantics/event01b.f90 b/flang/test/Semantics/event01b.f90
index 6a207427f6d4e..0cd8a5bcb1f1f 100644
--- a/flang/test/Semantics/event01b.f90
+++ b/flang/test/Semantics/event01b.f90
@@ -10,8 +10,41 @@ program test_event_post
   implicit none
 
   ! event_type variables must be coarrays
+  !ERROR: Variable 'non_coarray' with EVENT_TYPE or LOCK_TYPE must be a coarray
   type(event_type) non_coarray
 
+  ! event_type potential object components must be nested in coarrays
+  type :: has_event
+    type(event_type) event
+  end type
+  type :: bad1
+    type(has_event) component
+  end type
+  type :: bad2
+    type(has_event), allocatable :: component
+  end type
+  type :: good1
+    type(has_event), pointer :: component
+  end type
+  type :: good2
+    type(has_event), allocatable :: component[:]
+  end type
+  !ERROR: Variable 'non_coarray_component1' with EVENT_TYPE or LOCK_TYPE potential component '%event' must be a coarray
+  type(has_event) non_coarray_component1
+  !ERROR: Variable 'non_coarray_component2' with EVENT_TYPE or LOCK_TYPE potential component '%component%event' must be a coarray
+  type(bad1) non_coarray_component2
+  !ERROR: Variable 'non_coarray_component3' with EVENT_TYPE or LOCK_TYPE potential component '%component%event' must be a coarray
+  type(bad2) non_coarray_component3
+  ! these are okay
+  type(has_event) ok_non_coarray_component1[*]
+  type(has_event), pointer :: ok_non_coarray_component2
+  type(bad1) :: ok_non_coarray_component3[*]
+  type(bad1), pointer :: ok_non_coarray_component4
+  type(bad2) :: ok_non_coarray_component5[*]
+  type(bad2), pointer :: ok_non_coarray_component6
+  type(good1) ok_non_coarray_component7
+  type(good2) ok_non_coarray_component8
+
   type(event_type) concert[*], occurrences(2)[*]
   integer non_event[*], sync_status, co_indexed_integer[*], superfluous_stat, non_scalar(1)
   character(len=128) error_message, co_indexed_character[*], superfluous_errmsg
@@ -25,10 +58,6 @@ program test_event_post
   !ERROR: The event-variable must be of type EVENT_TYPE from module ISO_FORTRAN_ENV
   event post(non_event)
 
-  ! event-variable must be a coarray
-  !ERROR: The event-variable must be a coarray
-  event post(non_coarray)
-
   !ERROR: Must be a scalar value, but is a rank-1 array
   event post(occurrences)
 
diff --git a/flang/test/Semantics/event02b.f90 b/flang/test/Semantics/event02b.f90
index 20ee4047a1fed..94971022878ac 100644
--- a/flang/test/Semantics/event02b.f90
+++ b/flang/test/Semantics/event02b.f90
@@ -10,6 +10,7 @@ program test_event_wait
   implicit none
 
   ! event_type variables must be coarrays
+  !ERROR: Variable 'non_coarray' with EVENT_TYPE or LOCK_TYPE must be a coarray
   type(event_type) non_coarray
 
   type(event_type) concert[*], occurrences(2)[*]
@@ -24,9 +25,6 @@ program test_event_wait
   !ERROR: The event-variable must be of type EVENT_TYPE from module ISO_FORTRAN_ENV
   event wait(non_event)
 
-  !ERROR: The event-variable must be a coarray
-  event wait(non_coarray)
-
   !ERROR: A event-variable in a EVENT WAIT statement may not be a coindexed object
   event wait(concert[1])
 
diff --git a/flang/test/Semantics/event_query.f90 b/flang/test/Semantics/event_query.f90
index 3f38e3dd37877..f648462bc2090 100644
--- a/flang/test/Semantics/event_query.f90
+++ b/flang/test/Semantics/event_query.f90
@@ -1,14 +1,10 @@
 ! RUN: %python %S/test_errors.py %s %flang_fc1
-! XFAIL: *
 ! This test checks for semantic errors in event_query() subroutine based on the
 ! statement specification in section 16.9.72 of the Fortran 2018 standard.
 
 program test_event_query
   use iso_fortran_env, only : event_type
-  implicit none
-
-  ! event_type variables must be coarrays
-  type(event_type) non_coarray
+  implicit none(type,external)
 
   type(event_type) concert[*], occurrences(2)[*]
   integer non_event[*], counter, array(1), coarray[*], sync_status, coindexed[*], non_scalar(1)
@@ -33,70 +29,90 @@ program test_event_query
   !___ non-standard-conforming calls _______
 
   ! event-variable must be event_type
+  ! ERROR: Actual argument for 'event=' has bad type 'INTEGER(4)'
   call event_query(non_event, counter)
 
-  ! event-variable must be a coarray
-  call event_query(non_coarray, counter)
-
   ! event-variable must be a scalar variable
+  ! ERROR: 'event=' argument has unacceptable rank 1
   call event_query(occurrences, counter)
 
   ! event-variable must not be coindexed
+  ! ERROR: EVENT= argument to EVENT_QUERY must not be coindexed
   call event_query(concert[1], counter)
 
   ! event-variable has an unknown keyword argument
+  ! ERROR: unknown keyword argument to intrinsic 'event_query'
   call event_query(events=concert, count=counter)
 
   ! event-variable has an argument mismatch
+  ! ERROR: Actual argument for 'event=' has bad type 'INTEGER(4)'
   call event_query(event=non_event, count=counter)
 
   ! count must be an integer
+  ! ERROR: Actual argument for 'count=' has bad type 'LOGICAL(4)'
   call event_query(concert, non_integer)
 
   ! count must be an integer scalar
+  ! ERROR: 'count=' argument has unacceptable rank 1
   call event_query(concert, non_scalar)
 
   ! count must be have a decimal exponent range
   ! no smaller than that of default integer
+  ! ERROR: COUNT= argument to EVENT_QUERY must be an integer with kind >= 4
   call event_query(concert, non_default)
 
   ! count is an intent(out) argument
+  ! ERROR: Actual argument associated with INTENT(OUT) dummy argument 'count=' is not definable
+  ! ERROR: '4_4' is not a variable or pointer
   call event_query(concert, 4)
 
   ! count has an unknown keyword argument
+  ! ERROR: unknown keyword argument to intrinsic 'event_query'
   call event_query(concert, counts=counter)
 
   ! count has an argument mismatch
+  ! ERROR: Actual argument for 'count=' has bad type 'LOGICAL(4)'
   call event_query(concert, count=non_integer)
 
   ! stat must be an integer
+  ! ERROR: Actual argument for 'stat=' has bad type 'LOGICAL(4)'
   call event_query(concert, counter, non_integer)
 
   ! stat must be an integer scalar
+  ! ERROR: 'stat=' argument has unacceptable rank 1
   call event_query(concert, counter, non_scalar)
 
   ! stat is an intent(out) argument
+  ! ERROR: Actual argument associated with INTENT(OUT) dummy argument 'stat=' is not definable
+  ! ERROR: '8_4' is not a variable or pointer
   call event_query(concert, counter, 8)
 
   ! stat has an unknown keyword argument
+  ! ERROR: unknown keyword argument to intrinsic 'event_query'
   call event_query(concert, counter, status=sync_status)
 
   ! stat has an argument mismatch
+  ! ERROR: Actual argument for 'stat=' has bad type 'LOGICAL(4)'
   call event_query(concert, counter, stat=non_integer)
 
   ! stat must not be coindexed
+  ! ERROR: 'stat' argument to 'event_query' may not be a coindexed object
   call event_query(concert, counter, coindexed[1])
 
   ! Too many arguments
+  ! ERROR: too many actual arguments for intrinsic 'event_query'
   call event_query(concert, counter, sync_status, array(1))
 
   ! Repeated event keyword
+  ! ERROR: repeated keyword argument to intrinsic 'event_query'
   call event_query(event=concert, event=occurrences(1), count=counter)
 
   ! Repeated count keyword
+  ! ERROR: repeated keyword argument to intrinsic 'event_query'
   call event_query(event=concert, count=counter, count=array(1))
 
   ! Repeated stat keyword
+  ! ERROR: repeated keyword argument to intrinsic 'event_query'
   call event_query(event=concert, count=counter, stat=sync_status, stat=array(1))
 
 end program test_event_query
diff --git a/flang/test/Semantics/implicit17.f90 b/flang/test/Semantics/implicit17.f90
new file mode 100644
index 0000000000000..e11123d621501
--- /dev/null
+++ b/flang/test/Semantics/implicit17.f90
@@ -0,0 +1,14 @@
+! RUN: %python %S/test_errors.py %s %flang_fc1 -fimplicit-none-ext
+external x
+integer :: f, i, arr(1) = [0]
+call x
+!ERROR: 'y' is an external procedure without the EXTERNAL attribute in a scope with IMPLICIT NONE(EXTERNAL)
+call y
+!ERROR: 'f' is an external procedure without the EXTERNAL attribute in a scope with IMPLICIT NONE(EXTERNAL)
+i = f()
+block
+  !ERROR: 'z' is an external procedure without the EXTERNAL attribute in a scope with IMPLICIT NONE(EXTERNAL)
+  call z
+end block
+print *, arr(1) ! no error
+end
diff --git a/flang/test/Semantics/kinds01.f90 b/flang/test/Semantics/kinds01.f90
index 5238a90719fe4..82c4d76da3710 100644
--- a/flang/test/Semantics/kinds01.f90
+++ b/flang/test/Semantics/kinds01.f90
@@ -1,3 +1,4 @@
+! REQUIRES: x86_64-registered-target
 ! RUN: %python %S/test_symbols.py %s %flang_fc1
  !DEF: /MainProgram1/jk1 ObjectEntity INTEGER(1)
  integer(kind=1) jk1
diff --git a/flang/test/Semantics/kinds02.f90 b/flang/test/Semantics/kinds02.f90
index 13dbb803de8e5..02b1e6c8c310a 100644
--- a/flang/test/Semantics/kinds02.f90
+++ b/flang/test/Semantics/kinds02.f90
@@ -1,3 +1,4 @@
+! REQUIRES: x86_64-registered-target
 ! RUN: %python %S/test_errors.py %s %flang_fc1
 ! C712 The value of scalar-int-constant-expr shall be nonnegative and 
 ! shall specify a representation method that exists on the processor.
diff --git a/flang/test/Semantics/kinds04_q10.f90 b/flang/test/Semantics/kinds04_q10.f90
index d352daa1cbbf0..3bec7a386585b 100644
--- a/flang/test/Semantics/kinds04_q10.f90
+++ b/flang/test/Semantics/kinds04_q10.f90
@@ -1,14 +1,10 @@
-! RUN: %python %S/test_errors.py %s %flang_fc1 -triple x86_64-unknown-linux-gnu
+! RUN: not %flang_fc1 %s 2>%t.stderr
+! RUN: FileCheck %s --input-file=%t.stderr --check-prefixes=PORTABILITY,ERROR,WARNING%if system-aix %{,AIX_WARNING%}
 ! C716 If both kind-param and exponent-letter appear, exponent-letter
 ! shall be E. (As an extension we also allow an exponent-letter which matches
 ! the kind-param)
 ! C717 The value of kind-param shall specify an approximation method that
 ! exists on the processor.
-!
-! This test is for x86_64, where exponent-letter 'q' is for
-! 10-byte extended precision
-! UNSUPPORTED: system-windows, system-aix
-! REQUIRES: x86-registered-target
 
 subroutine s(var)
   real :: realvar1 = 4.0E6_4
@@ -16,11 +12,13 @@ subroutine s(var)
   real :: realvar3 = 4.0Q6
   !PORTABILITY: Explicit kind parameter together with non-'E' exponent letter is not standard
   real :: realvar4 = 4.0D6_8
-  !PORTABILITY: Explicit kind parameter together with non-'E' exponent letter is not standard
-  real :: realvar5 = 4.0Q6_10
   !WARNING: Explicit kind parameter on real constant disagrees with exponent letter 'q'
+  !AIX_WARNING: underflow on REAL(10) to REAL(4) conversion
+  real :: realvar5 = 4.0Q6_10
+  !PORTABILITY: Explicit kind parameter together with non-'E' exponent letter is not standard
   real :: realvar6 = 4.0Q6_16
   real :: realvar7 = 4.0E6_8
+  !AIX_WARNING: underflow on REAL(10) to REAL(4) conversion
   real :: realvar8 = 4.0E6_10
   real :: realvar9 = 4.0E6_16
   !ERROR: Unsupported REAL(KIND=32)
@@ -31,9 +29,10 @@ subroutine s(var)
   double precision :: doublevar3 = 4.0Q6
   !PORTABILITY: Explicit kind parameter together with non-'E' exponent letter is not standard
   double precision :: doublevar4 = 4.0D6_8
-  !WARNING: Explicit kind parameter on real constant disagrees with exponent letter 'q'
+  !PORTABILITY: Explicit kind parameter together with non-'E' exponent letter is not standard
   double precision :: doublevar5 = 4.0Q6_16
   double precision :: doublevar6 = 4.0E6_8
+  !AIX_WARNING: underflow on REAL(10) to REAL(8) conversion
   double precision :: doublevar7 = 4.0E6_10
   double precision :: doublevar8 = 4.0E6_16
   !ERROR: Unsupported REAL(KIND=32)
diff --git a/flang/test/Semantics/label19.f90 b/flang/test/Semantics/label19.f90
new file mode 100644
index 0000000000000..f8ad05335d070
--- /dev/null
+++ b/flang/test/Semantics/label19.f90
@@ -0,0 +1,19 @@
+! RUN: %python %S/test_errors.py %s %flang_fc1
+program main
+  use, intrinsic:: iso_fortran_env, only: team_type
+  type(team_type) team
+  logical :: p = false
+1 change team(team)
+2 if (p) goto 1 ! ok
+  if (p) goto 2 ! ok
+  if (p) goto 3 ! ok
+  if (p) goto 4 ! ok
+  if (p) goto 5 ! ok
+3 end team
+4 continue
+  if (p) goto 1 ! ok
+  !ERROR: Label '2' is in a construct that prevents its use as a branch target here
+  if (p) goto 2
+  !ERROR: Label '3' is in a construct that prevents its use as a branch target here
+  if (p) goto 3
+5 end
diff --git a/flang/test/Semantics/loop-directives.f90 b/flang/test/Semantics/loop-directives.f90
index 58fb9b8082bc1..e20c0c9d042db 100644
--- a/flang/test/Semantics/loop-directives.f90
+++ b/flang/test/Semantics/loop-directives.f90
@@ -4,11 +4,15 @@
 subroutine empty
   ! WARNING: A DO loop must follow the VECTOR ALWAYS directive
   !dir$ vector always
+  ! WARNING: A DO loop must follow the UNROLL directive
+  !dir$ unroll
 end subroutine empty
 
 subroutine non_do
   ! WARNING: A DO loop must follow the VECTOR ALWAYS directive
   !dir$ vector always
+  ! WARNING: A DO loop must follow the UNROLL directive
+  !dir$ unroll
   a = 1
 end subroutine non_do
 
@@ -16,6 +20,8 @@ subroutine execution_part
   do i=1,10
   ! WARNING: A DO loop must follow the VECTOR ALWAYS directive
   !dir$ vector always
+  ! WARNING: A DO loop must follow the UNROLL directive
+  !dir$ unroll
   end do
 end subroutine execution_part
 
@@ -28,3 +34,13 @@ subroutine test_vector_always_before_acc(a, b, c)
     a(i) = b(i) + c(i)
   enddo
 end subroutine
+
+! OK
+subroutine test_unroll_before_acc(a, b, c)
+  real, dimension(10) :: a,b,c
+  !dir$ unroll
+  !$acc loop
+  do i=1,N
+    a(i) = b(i) + c(i)
+  enddo
+end subroutine
diff --git a/flang/test/Semantics/modfile71.F90 b/flang/test/Semantics/modfile71.F90
new file mode 100644
index 0000000000000..7c3c7f5b48958
--- /dev/null
+++ b/flang/test/Semantics/modfile71.F90
@@ -0,0 +1,121 @@
+!RUN: %flang_fc1 -fsyntax-only -fhermetic-module-files -DSTEP=1 %s
+!RUN: %flang_fc1 -fsyntax-only -DSTEP=2 %s
+!RUN: not %flang_fc1 -fsyntax-only -pedantic %s 2>&1 | FileCheck %s
+
+! Tests that a module captured in a hermetic module file is compatible when
+! USE'd with a module of the same name USE'd directly.
+
+#if STEP == 1
+module modfile71a
+  ! not errors
+  integer, parameter :: good_named_const = 123
+  integer :: good_var = 1
+  type :: good_derived
+    integer component
+  end type
+  procedure(), pointer :: good_proc_ptr
+  generic :: gen => bad_subroutine
+  ! bad, but okay if unused
+  integer, parameter :: unused_bad_named_const = 123
+  integer :: unused_bad_var = 1
+  type :: unused_bad_derived
+    integer component
+  end type
+  procedure(), pointer :: unused_bad_proc_ptr
+  ! errors
+  integer, parameter :: bad_named_const = 123
+  integer :: bad_var = 1
+  type :: bad_derived
+    integer component
+  end type
+  procedure(), pointer :: bad_proc_ptr
+ contains
+  subroutine good_subroutine
+  end
+  subroutine unused_bad_subroutine(x)
+    integer x
+  end
+  subroutine bad_subroutine(x)
+    integer x
+  end
+end
+
+module modfile71b
+  use modfile71a ! capture hermetically
+end
+
+#elif STEP == 2
+module modfile71a
+  ! not errors
+  integer, parameter :: good_named_const = 123
+  integer :: good_var = 1
+  type :: good_derived
+    integer component
+  end type
+  procedure(), pointer :: good_proc_ptr
+  generic :: gen => bad_subroutine
+  ! bad, but okay if unused
+  integer, parameter :: unused_bad_named_const = 666
+  real :: unused_bad_var = 1.
+  type :: unused_bad_derived
+    real component
+  end type
+  real, pointer :: unused_bad_proc_ptr
+  ! errors
+  integer, parameter :: bad_named_const = 666
+  real :: bad_var = 1.
+  type :: bad_derived
+    real component
+  end type
+  real, pointer :: bad_proc_ptr
+ contains
+  subroutine good_subroutine
+  end
+  subroutine unused_bad_subroutine(x)
+    real x
+  end
+  subroutine bad_subroutine(x)
+    real x
+  end
+end
+
+#else
+
+!CHECK: warning: 'bad_derived' is use-associated from 'bad_derived' in two distinct instances of module 'modfile71a'
+!CHECK: warning: 'bad_named_const' is use-associated from 'bad_named_const' in two distinct instances of module 'modfile71a'
+!CHECK: warning: 'bad_proc_ptr' is use-associated from 'bad_proc_ptr' in two distinct instances of module 'modfile71a'
+!CHECK: warning: 'bad_subroutine' is use-associated from 'bad_subroutine' in two distinct instances of module 'modfile71a'
+!CHECK: warning: 'bad_var' is use-associated from 'bad_var' in two distinct instances of module 'modfile71a'
+!CHECK: warning: 'good_derived' is use-associated from 'good_derived' in two distinct instances of module 'modfile71a'
+!CHECK: warning: 'good_named_const' is use-associated from 'good_named_const' in two distinct instances of module 'modfile71a'
+!CHECK: warning: 'good_proc_ptr' is use-associated from 'good_proc_ptr' in two distinct instances of module 'modfile71a'
+!CHECK: warning: 'good_subroutine' is use-associated from 'good_subroutine' in two distinct instances of module 'modfile71a'
+!CHECK: warning: 'good_var' is use-associated from 'good_var' in two distinct instances of module 'modfile71a'
+!CHECK: warning: 'unused_bad_derived' is use-associated from 'unused_bad_derived' in two distinct instances of module 'modfile71a'
+!CHECK: warning: 'unused_bad_named_const' is use-associated from 'unused_bad_named_const' in two distinct instances of module 'modfile71a'
+!CHECK: warning: 'unused_bad_proc_ptr' is use-associated from 'unused_bad_proc_ptr' in two distinct instances of module 'modfile71a'
+!CHECK: warning: 'unused_bad_subroutine' is use-associated from 'unused_bad_subroutine' in two distinct instances of module 'modfile71a'
+!CHECK: warning: 'unused_bad_var' is use-associated from 'unused_bad_var' in two distinct instances of module 'modfile71a'
+!CHECK: error: Reference to 'bad_derived' is ambiguous
+!CHECK: error: Reference to 'bad_named_const' is ambiguous
+!CHECK: error: Reference to 'bad_var' is ambiguous
+!CHECK: error: Reference to 'bad_proc_ptr' is ambiguous
+!CHECK: error: Reference to 'bad_subroutine' is ambiguous
+!CHECK-NOT: error:
+!CHECK-NOT: warning:
+
+program main
+  use modfile71a
+  use modfile71b
+  type(good_derived) goodx
+  type(bad_derived) badx
+  print *, good_named_const
+  good_var = 1
+  good_proc_ptr => null()
+  call good_subroutine
+  print *, bad_named_const
+  print *, bad_var
+  bad_proc_ptr => null()
+  call bad_subroutine(1)
+end
+#endif
diff --git a/flang/test/Semantics/resolve110.f90 b/flang/test/Semantics/resolve110.f90
index 0b9e560e5ed77..398304b4d7672 100644
--- a/flang/test/Semantics/resolve110.f90
+++ b/flang/test/Semantics/resolve110.f90
@@ -1,7 +1,5 @@
 ! RUN: %python %S/test_errors.py %s %flang_fc1
 ! Exercise ways to define and extend non-type-bound generics
-! TODO: crashes compiler (infinite recursion) when build with MSVC
-! XFAIL: system-windows
 
 module m1
   type :: t1; end type
diff --git a/flang/test/Semantics/resolve41.f90 b/flang/test/Semantics/resolve41.f90
index d2a991c0a52dc..4d0b3a3d31e55 100644
--- a/flang/test/Semantics/resolve41.f90
+++ b/flang/test/Semantics/resolve41.f90
@@ -21,7 +21,7 @@ module m
   integer :: ff = 2_f
   !ERROR: REAL(KIND=23) is not a supported type
   real(d/2) :: g
-  !ERROR: REAL*47 is not a supported type
+  !ERROR: REAL(KIND=47) is not a supported type
   real*47 :: h
   !ERROR: COMPLEX*47 is not a supported type
   complex*47 :: i
diff --git a/flang/test/Semantics/self-use.f90 b/flang/test/Semantics/self-use.f90
index 4bc66a24343c6..12433732fc330 100644
--- a/flang/test/Semantics/self-use.f90
+++ b/flang/test/Semantics/self-use.f90
@@ -15,7 +15,6 @@ subroutine modsub
  contains
   module subroutine separate
     !ERROR: Module 'm' cannot USE itself from its own submodule 'submod1'
-    !ERROR: Cannot use-associate 'separate'; it is already declared in this scope
     use m
   end
 end
diff --git a/flang/test/Semantics/sync-stat-list.f90 b/flang/test/Semantics/sync-stat-list.f90
index 85a85f8224534..545733049ca35 100644
--- a/flang/test/Semantics/sync-stat-list.f90
+++ b/flang/test/Semantics/sync-stat-list.f90
@@ -16,7 +16,7 @@ program test_sync_stat_list
   character(len=128) error_message, superfluous_errmsg, coindexed_character[*]
   logical invalid_type
   type(team_type) :: home
-  type(lock_type) :: latch
+  type(lock_type) :: latch[*]
 
   ! valid
   change team (home, stat=sync_status, errmsg=error_message)
diff --git a/flang/test/Transforms/constant-argument-globalisation.fir b/flang/test/Transforms/constant-argument-globalisation.fir
index 5656de0f601e7..02349de40bc0b 100644
--- a/flang/test/Transforms/constant-argument-globalisation.fir
+++ b/flang/test/Transforms/constant-argument-globalisation.fir
@@ -49,8 +49,8 @@ module {
 // DISABLE-LABEL: ; ModuleID =
 // DISABLE-NOT: @_extruded
 // DISABLE:  define void @sub1(
-// DISABLE-SAME: ptr nocapture [[ARG0:%.*]],
-// DISABLE-SAME: ptr nocapture [[ARG1:%.*]])
+// DISABLE-SAME: ptr captures(none) [[ARG0:%.*]],
+// DISABLE-SAME: ptr captures(none) [[ARG1:%.*]])
 // DISABLE-SAME: {
 // DISABLE: [[CONST_R0:%.*]] = alloca double
 // DISABLE: [[CONST_R1:%.*]] = alloca double
diff --git a/flang/test/Transforms/debug-common-block.fir b/flang/test/Transforms/debug-common-block.fir
new file mode 100644
index 0000000000000..481b26369a92c
--- /dev/null
+++ b/flang/test/Transforms/debug-common-block.fir
@@ -0,0 +1,213 @@
+// RUN: fir-opt --add-debug-info --mlir-print-debuginfo %s | FileCheck %s
+
+module attributes {dlti.dl_spec = #dlti.dl_spec<>} {
+  fir.global @__BLNK__ {alignment = 4 : i64} : tuple<i32, !fir.array<8xi8>> {} loc(#loc1)
+  fir.global @a_ {alignment = 4 : i64} : tuple<i32, !fir.array<8xi8>> {} loc(#loc2)
+  func.func @f1() {
+    %c9_i32 = arith.constant 9 : i32
+    %c6_i32 = arith.constant 6 : i32
+    %cst = arith.constant 4.400000e+00 : f32
+    %cst_0 = arith.constant 3.300000e+00 : f32
+    %cst_1 = arith.constant 2.200000e+00 : f32
+    %cst_2 = arith.constant 1.100000e+00 : f32
+    %c4 = arith.constant 4 : index
+    %c0 = arith.constant 0 : index
+    %0 = fir.address_of(@__BLNK__) : !fir.ref<tuple<i32, !fir.array<8xi8>>>
+    %1 = fir.convert %0 : (!fir.ref<tuple<i32, !fir.array<8xi8>>>) -> !fir.ref<!fir.array<?xi8>>
+    %2 = fir.coordinate_of %1, %c0 : (!fir.ref<!fir.array<?xi8>>, index) -> !fir.ref<i8>
+    %3 = fir.convert %2 : (!fir.ref<i8>) -> !fir.ref<f32>
+    %4 = fircg.ext_declare %3 {uniq_name = "_QFf1Ex"} : (!fir.ref<f32>) -> !fir.ref<f32> loc(#loc4)
+    %5 = fir.address_of(@a_) : !fir.ref<tuple<i32, !fir.array<8xi8>>>
+    %6 = fir.convert %5 : (!fir.ref<tuple<i32, !fir.array<8xi8>>>) -> !fir.ref<!fir.array<?xi8>>
+    %7 = fir.coordinate_of %6, %c0 : (!fir.ref<!fir.array<?xi8>>, index) -> !fir.ref<i8>
+    %8 = fir.convert %7 : (!fir.ref<i8>) -> !fir.ref<f32>
+    %9 = fircg.ext_declare %8 {uniq_name = "_QFf1Exa"} : (!fir.ref<f32>) -> !fir.ref<f32> loc(#loc5)
+    %10 = fir.coordinate_of %1, %c4 : (!fir.ref<!fir.array<?xi8>>, index) -> !fir.ref<i8>
+    %11 = fir.convert %10 : (!fir.ref<i8>) -> !fir.ref<f32>
+    %12 = fircg.ext_declare %11 {uniq_name = "_QFf1Ey"} : (!fir.ref<f32>) -> !fir.ref<f32> loc(#loc6)
+    %13 = fir.coordinate_of %6, %c4 : (!fir.ref<!fir.array<?xi8>>, index) -> !fir.ref<i8>
+    %14 = fir.convert %13 : (!fir.ref<i8>) -> !fir.ref<f32>
+    %15 = fircg.ext_declare %14 {uniq_name = "_QFf1Eya"} : (!fir.ref<f32>) -> !fir.ref<f32> loc(#loc7)
+    return
+  } loc(#loc3)
+  func.func @f2() {
+    %c16_i32 = arith.constant 16 : i32
+    %c6_i32 = arith.constant 6 : i32
+    %c8 = arith.constant 8 : index
+    %c4 = arith.constant 4 : index
+    %c0 = arith.constant 0 : index
+    %0 = fir.address_of(@__BLNK__) : !fir.ref<tuple<i32, !fir.array<8xi8>>> loc(#loc19)
+    %1 = fir.convert %0 : (!fir.ref<tuple<i32, !fir.array<8xi8>>>) -> !fir.ref<!fir.array<?xi8>>
+    %2 = fir.coordinate_of %1, %c0 : (!fir.ref<!fir.array<?xi8>>, index) -> !fir.ref<i8>
+    %3 = fir.convert %2 : (!fir.ref<i8>) -> !fir.ref<f32>
+    %4 = fircg.ext_declare %3 {uniq_name = "_QFf2Ex"} : (!fir.ref<f32>) -> !fir.ref<f32> loc(#loc9)
+    %5 = fir.address_of(@a_) : !fir.ref<tuple<i32, !fir.array<8xi8>>>
+    %6 = fir.convert %5 : (!fir.ref<tuple<i32, !fir.array<8xi8>>>) -> !fir.ref<!fir.array<?xi8>>
+    %7 = fir.coordinate_of %6, %c0 : (!fir.ref<!fir.array<?xi8>>, index) -> !fir.ref<i8>
+    %8 = fir.convert %7 : (!fir.ref<i8>) -> !fir.ref<f32>
+    %9 = fircg.ext_declare %8 {uniq_name = "_QFf2Exa"} : (!fir.ref<f32>) -> !fir.ref<f32> loc(#loc10)
+    %10 = fir.coordinate_of %1, %c4 : (!fir.ref<!fir.array<?xi8>>, index) -> !fir.ref<i8>
+    %11 = fir.convert %10 : (!fir.ref<i8>) -> !fir.ref<f32>
+    %12 = fircg.ext_declare %11 {uniq_name = "_QFf2Ey"} : (!fir.ref<f32>) -> !fir.ref<f32> loc(#loc11)
+    %13 = fir.coordinate_of %6, %c4 : (!fir.ref<!fir.array<?xi8>>, index) -> !fir.ref<i8>
+    %14 = fir.convert %13 : (!fir.ref<i8>) -> !fir.ref<f32>
+    %15 = fircg.ext_declare %14 {uniq_name = "_QFf2Eya"} : (!fir.ref<f32>) -> !fir.ref<f32> loc(#loc12)
+    %16 = fir.coordinate_of %1, %c8 : (!fir.ref<!fir.array<?xi8>>, index) -> !fir.ref<i8>
+    %17 = fir.convert %16 : (!fir.ref<i8>) -> !fir.ref<f32>
+    %18 = fircg.ext_declare %17 {uniq_name = "_QFf2Ez"} : (!fir.ref<f32>) -> !fir.ref<f32> loc(#loc13)
+    %19 = fir.coordinate_of %6, %c8 : (!fir.ref<!fir.array<?xi8>>, index) -> !fir.ref<i8>
+    %20 = fir.convert %19 : (!fir.ref<i8>) -> !fir.ref<f32>
+    %21 = fircg.ext_declare %20 {uniq_name = "_QFf2Eza"} : (!fir.ref<f32>) -> !fir.ref<f32> loc(#loc14)
+    return
+  } loc(#loc8)
+  func.func @f3() {
+    %c24_i32 = arith.constant 24 : i32
+    %c23_i32 = arith.constant 23 : i32
+    %c6_i32 = arith.constant 6 : i32
+    %c0 = arith.constant 0 : index
+    %0 = fir.address_of(@__BLNK__) : !fir.ref<tuple<i32, !fir.array<8xi8>>>
+    %1 = fir.convert %0 : (!fir.ref<tuple<i32, !fir.array<8xi8>>>) -> !fir.ref<!fir.array<?xi8>>
+    %2 = fir.coordinate_of %1, %c0 : (!fir.ref<!fir.array<?xi8>>, index) -> !fir.ref<i8>
+    %3 = fir.convert %2 : (!fir.ref<i8>) -> !fir.ref<i32>
+    %4 = fircg.ext_declare %3 {uniq_name = "_QFf3Ex"} : (!fir.ref<i32>) -> !fir.ref<i32> loc(#loc16)
+    %5 = fir.address_of(@a_) : !fir.ref<tuple<i32, !fir.array<8xi8>>>
+    %6 = fir.convert %5 : (!fir.ref<tuple<i32, !fir.array<8xi8>>>) -> !fir.ref<!fir.array<?xi8>>
+    %7 = fir.coordinate_of %6, %c0 : (!fir.ref<!fir.array<?xi8>>, index) -> !fir.ref<i8>
+    %8 = fir.convert %7 : (!fir.ref<i8>) -> !fir.ref<i32>
+    %9 = fircg.ext_declare %8 {uniq_name = "_QFf3Exa"} : (!fir.ref<i32>) -> !fir.ref<i32> loc(#loc17)
+    return
+  } loc(#loc15)
+  func.func @test() {
+    %c34_i32 = arith.constant 34 : i32
+    %c6_i32 = arith.constant 6 : i32
+    %c8 = arith.constant 8 : index
+    %c4 = arith.constant 4 : index
+    %c0 = arith.constant 0 : index
+    %0 = fir.address_of(@__BLNK__) : !fir.ref<tuple<i32, !fir.array<8xi8>>>
+    %1 = fir.convert %0 : (!fir.ref<tuple<i32, !fir.array<8xi8>>>) -> !fir.ref<!fir.array<?xi8>>
+    %2 = fir.coordinate_of %1, %c0 : (!fir.ref<!fir.array<?xi8>>, index) -> !fir.ref<i8>
+    %3 = fir.convert %2 : (!fir.ref<i8>) -> !fir.ref<f32>
+    %4 = fircg.ext_declare %3 {uniq_name = "_QFEv1"} : (!fir.ref<f32>) -> !fir.ref<f32> loc(#loc19)
+    %5 = fir.coordinate_of %1, %c4 : (!fir.ref<!fir.array<?xi8>>, index) -> !fir.ref<i8>
+    %6 = fir.convert %5 : (!fir.ref<i8>) -> !fir.ref<f32>
+    %7 = fircg.ext_declare %6 {uniq_name = "_QFEv2"} : (!fir.ref<f32>) -> !fir.ref<f32> loc(#loc20)
+    %8 = fir.coordinate_of %1, %c8 : (!fir.ref<!fir.array<?xi8>>, index) -> !fir.ref<i8>
+    %9 = fir.convert %8 : (!fir.ref<i8>) -> !fir.ref<f32>
+    %10 = fircg.ext_declare %9 {uniq_name = "_QFEv3"} : (!fir.ref<f32>) -> !fir.ref<f32> loc(#loc21)
+    %11 = fir.address_of(@a_) : !fir.ref<tuple<i32, !fir.array<8xi8>>>
+    %12 = fir.convert %11 : (!fir.ref<tuple<i32, !fir.array<8xi8>>>) -> !fir.ref<!fir.array<?xi8>>
+    %13 = fir.coordinate_of %12, %c0 : (!fir.ref<!fir.array<?xi8>>, index) -> !fir.ref<i8>
+    %14 = fir.convert %13 : (!fir.ref<i8>) -> !fir.ref<f32>
+    %15 = fircg.ext_declare %14 {uniq_name = "_QFEva1"} : (!fir.ref<f32>) -> !fir.ref<f32> loc(#loc22)
+    %16 = fir.coordinate_of %12, %c4 : (!fir.ref<!fir.array<?xi8>>, index) -> !fir.ref<i8>
+    %17 = fir.convert %16 : (!fir.ref<i8>) -> !fir.ref<f32>
+    %18 = fircg.ext_declare %17 {uniq_name = "_QFEva2"} : (!fir.ref<f32>) -> !fir.ref<f32> loc(#loc23)
+    %19 = fir.coordinate_of %12, %c8 : (!fir.ref<!fir.array<?xi8>>, index) -> !fir.ref<i8>
+    %20 = fir.convert %19 : (!fir.ref<i8>) -> !fir.ref<f32>
+    %21 = fircg.ext_declare %20 {uniq_name = "_QFEva3"} : (!fir.ref<f32>) -> !fir.ref<f32> loc(#loc24)
+    return
+  } loc(#loc18)
+}
+
+#loc1 = loc(unknown)
+#loc2 = loc(unknown)
+#loc3 = loc("common.f90":10:1)
+#loc4 = loc("common.f90":12:19)
+#loc5 = loc("common.f90":12:25)
+#loc6 = loc("common.f90":12:22)
+#loc7 = loc("common.f90":12:29)
+#loc8 = loc("common.f90":20:3)
+#loc9 = loc("common.f90":22:3)
+#loc10 = loc("common.f90":22:6)
+#loc11 = loc("common.f90":22:9)
+#loc12 = loc("common.f90":22:13)
+#loc13 = loc("common.f90":22:16)
+#loc14 = loc("common.f90":22:19)
+#loc15 = loc("common.f90":32:18)
+#loc16 = loc("common.f90":35:7)
+#loc17 = loc("common.f90":35:10)
+#loc18 = loc("common.f90":40:1)
+#loc19 = loc("common.f90":43:19)
+#loc20 = loc("common.f90":43:28)
+#loc21 = loc("common.f90":43:22)
+#loc22 = loc("common.f90":43:32)
+#loc23 = loc("common.f90":43:25)
+#loc24 = loc("common.f90":43:36)
+
+
+// CHECK-DAG: #[[XF1:.*]] = #llvm.di_global_variable<scope = #[[CBF1:.+]], name = "x", linkageName = "_QFf1Ex"{{.*}}>
+// CHECK-DAG: #[[YF1:.*]] = #llvm.di_global_variable<scope = #[[CBF1]], name = "y", linkageName = "_QFf1Ey"{{.*}}>
+// CHECK-DAG: #[[EXPXF1:.*]] = #llvm.di_global_variable_expression<var = #[[XF1]]>
+// CHECK-DAG: #[[EXPYF1:.*]] = #llvm.di_global_variable_expression<var = #[[YF1]], expr = <[DW_OP_plus_uconst(4)]>>
+// CHECK-DAG: #[[XAF1:.*]] = #llvm.di_global_variable<scope = #[[CBAF1:.+]], name = "xa", linkageName = "_QFf1Exa"{{.*}}>
+// CHECK-DAG: #[[YAF1:.*]] = #llvm.di_global_variable<scope = #[[CBAF1]], name = "ya", linkageName = "_QFf1Eya"{{.*}}>
+// CHECK-DAG: #[[EXPXAF1:.*]] = #llvm.di_global_variable_expression<var = #[[XAF1]]>
+// CHECK-DAG: #[[EXPYAF1:.*]] = #llvm.di_global_variable_expression<var = #[[YAF1]], expr = <[DW_OP_plus_uconst(4)]>>
+
+// CHECK-DAG: #[[XF2:.*]] = #llvm.di_global_variable<scope = #[[CBF2:.+]], name = "x", linkageName = "_QFf2Ex"{{.*}}>
+// CHECK-DAG: #[[YF2:.*]] = #llvm.di_global_variable<scope = #[[CBF2]], name = "y", linkageName = "_QFf2Ey"{{.*}}>
+// CHECK-DAG: #[[ZF2:.*]] = #llvm.di_global_variable<scope = #[[CBF2]], name = "z", linkageName = "_QFf2Ez"{{.*}}>
+// CHECK-DAG: #[[EXPXF2:.*]] = #llvm.di_global_variable_expression<var = #[[XF2]]>
+// CHECK-DAG: #[[EXPYF2:.*]] = #llvm.di_global_variable_expression<var = #[[YF2]], expr = <[DW_OP_plus_uconst(4)]>>
+// CHECK-DAG: #[[EXPZF2:.*]] = #llvm.di_global_variable_expression<var = #[[ZF2]], expr = <[DW_OP_plus_uconst(8)]>>
+// CHECK-DAG: #[[XAF2:.*]] = #llvm.di_global_variable<scope = #[[CBAF2:.+]], name = "xa", linkageName = "_QFf2Exa"{{.*}}>
+// CHECK-DAG: #[[YAF2:.*]] = #llvm.di_global_variable<scope = #[[CBAF2]], name = "ya", linkageName = "_QFf2Eya"{{.*}}>
+// CHECK-DAG: #[[ZAF2:.*]] = #llvm.di_global_variable<scope = #[[CBAF2]], name = "za", linkageName = "_QFf2Eza"{{.*}}>
+// CHECK-DAG: #[[EXPXAF2:.*]] = #llvm.di_global_variable_expression<var = #[[XAF2]]>
+// CHECK-DAG: #[[EXPYAF2:.*]] = #llvm.di_global_variable_expression<var = #[[YAF2]], expr = <[DW_OP_plus_uconst(4)]>>
+// CHECK-DAG: #[[EXPZAF2:.*]] = #llvm.di_global_variable_expression<var = #[[ZAF2]], expr = <[DW_OP_plus_uconst(8)]>>
+
+// CHECK-DAG: #[[XF3:.*]] = #llvm.di_global_variable<scope = #[[CBF3:.+]], name = "x", linkageName = "_QFf3Ex"{{.*}}>
+// CHECK-DAG: #[[EXPXF3:.*]] = #llvm.di_global_variable_expression<var = #[[XF3]]>
+// CHECK-DAG: #[[XAF3:.*]] = #llvm.di_global_variable<scope = #[[CBAF3:.+]], name = "xa", linkageName = "_QFf3Exa"{{.*}}>
+// CHECK-DAG: #[[EXPXAF3:.*]] = #llvm.di_global_variable_expression<var = #[[XAF3]]>
+
+// CHECK-DAG: #[[V1:.*]] = #llvm.di_global_variable<scope = #[[CBM:.+]], name = "v1", linkageName = "_QFEv1"{{.*}}>
+// CHECK-DAG: #[[EXPV1:.*]] = #llvm.di_global_variable_expression<var = #[[V1]]>
+// CHECK-DAG: #[[V2:.*]] = #llvm.di_global_variable<scope = #[[CBM]], name = "v2", linkageName = "_QFEv2"{{.*}}>
+// CHECK-DAG: #[[EXPV2:.*]] = #llvm.di_global_variable_expression<var = #[[V2]], expr = <[DW_OP_plus_uconst(4)]>>
+// CHECK-DAG: #[[V3:.*]] = #llvm.di_global_variable<scope = #[[CBM]], name = "v3", linkageName = "_QFEv3"{{.*}}>
+// CHECK-DAG: #[[EXPV3:.*]] = #llvm.di_global_variable_expression<var = #[[V3]], expr = <[DW_OP_plus_uconst(8)]>>
+// CHECK-DAG: #[[VA1:.*]] = #llvm.di_global_variable<scope = #[[CBAM:.+]], name = "va1", linkageName = "_QFEva1"{{.*}}>
+// CHECK-DAG: #[[EXPVA1:.*]] = #llvm.di_global_variable_expression<var = #[[VA1]]>
+// CHECK-DAG: #[[VA2:.*]] = #llvm.di_global_variable<scope = #[[CBAM]], name = "va2", linkageName = "_QFEva2"{{.*}}>
+// CHECK-DAG: #[[EXPVA2:.*]] = #llvm.di_global_variable_expression<var = #[[VA2]], expr = <[DW_OP_plus_uconst(4)]>>
+// CHECK-DAG: #[[VA3:.*]] = #llvm.di_global_variable<scope = #[[CBAM]], name = "va3", linkageName = "_QFEva3"{{.*}}>
+// CHECK-DAG: #[[EXPVA3:.*]] = #llvm.di_global_variable_expression<var = #[[VA3]], expr = <[DW_OP_plus_uconst(8)]>>
+
+// CHECK-DAG: #[[F1:.*]] = #llvm.di_subprogram<{{.*}}name = "f1"{{.*}}>
+// CHECK-DAG: #[[F2:.*]] = #llvm.di_subprogram<{{.*}}name = "f2"{{.*}}>
+// CHECK-DAG: #[[F3:.*]] = #llvm.di_subprogram<{{.*}}name = "f3"{{.*}}>
+// CHECK-DAG: #[[TEST:.*]] = #llvm.di_subprogram<{{.*}}name = "test"{{.*}}>
+
+// CHECK-DAG: #[[CBF1]] = #llvm.di_common_block<scope = #[[F1]], name = "__BLNK__"{{.*}}>
+// CHECK-DAG: #[[CBF2]] = #llvm.di_common_block<scope = #[[F2]], name = "__BLNK__"{{.*}}>
+// CHECK-DAG: #[[CBF3]] = #llvm.di_common_block<scope = #[[F3]], name = "__BLNK__"{{.*}}>
+// CHECK-DAG: #[[CBM]] = #llvm.di_common_block<scope = #[[TEST]], name = "__BLNK__"{{.*}}>
+// CHECK-DAG: #[[CBAF1]] = #llvm.di_common_block<scope = #[[F1]], name = "a"{{.*}}>
+// CHECK-DAG: #[[CBAF2]] = #llvm.di_common_block<scope = #[[F2]], name = "a"{{.*}}>
+// CHECK-DAG: #[[CBAF3]] = #llvm.di_common_block<scope = #[[F3]], name = "a"{{.*}}>
+// CHECK-DAG: #[[CBAM]] = #llvm.di_common_block<scope = #[[TEST]], name = "a"{{.*}}>
+
+// CHECK-DAG: fir.global @__BLNK__ {{.*}} loc(#[[LOC1:.*]])
+// CHECK-DAG: fir.global @a_ {{.*}} loc(#[[LOC2:.*]])
+// CHECK-DAG: #[[LOC1]]
+// CHECK-DAG-SAME: #[[EXPXF1]]
+// CHECK-DAG-SAME: #[[EXPYF1]]
+// CHECK-DAG-SAME: #[[EXPXF2]]
+// CHECK-DAG-SAME: #[[EXPYF2]]
+// CHECK-DAG-SAME: #[[EXPZF2]]
+// CHECK-DAG-SAME: #[[EXPXF3]]
+// CHECK-DAG-SAME: #[[EXPV1]]
+// CHECK-DAG-SAME: #[[EXPV2]]
+// CHECK-DAG-SAME: #[[EXPV3]]
+// CHECK-DAG: #[[LOC2]]
+// CHECK-DAG-SAME: #[[EXPXAF1]]
+// CHECK-DAG-SAME: #[[EXPYAF1]]
+// CHECK-DAG-SAME: #[[EXPXAF2]]
+// CHECK-DAG-SAME: #[[EXPYAF2]]
+// CHECK-DAG-SAME: #[[EXPZAF2]]
+// CHECK-DAG-SAME: #[[EXPXAF3]]
+// CHECK-DAG-SAME: #[[EXPVA1]]
+// CHECK-DAG-SAME: #[[EXPVA2]]
+// CHECK-DAG-SAME: #[[EXPVA3]]
diff --git a/flang/test/Transforms/debug-module-1.fir b/flang/test/Transforms/debug-module-1.fir
index 71457d32b1596..ede996f053835 100644
--- a/flang/test/Transforms/debug-module-1.fir
+++ b/flang/test/Transforms/debug-module-1.fir
@@ -30,11 +30,13 @@ module attributes {dlti.dl_spec = #dlti.dl_spec<>} {
 // CHECK-DAG: #[[MOD:.*]] = #llvm.di_module<{{.*}}scope = #[[CU]], name = "helper"{{.*}}>
 // CHECK-DAG: #[[LOC1:.*]] = loc("{{.*}}test.f90":12{{.*}})
 // CHECK-DAG: #[[GLI:.*]] = #llvm.di_global_variable<scope = #[[MOD]], name = "gli", linkageName = "_QMhelperEgli"{{.*}}line = 12, type = #[[I4]], isDefined = true>
+// CHECK-DAG: #[[GLIE:.*]] = #llvm.di_global_variable_expression<var = #[[GLI]]>
 // CHECK-DAG: #[[LOC2:.*]] = loc("{{.*}}test.f90":15{{.*}})
 // CHECK-DAG: #[[GLR:.*]] = #llvm.di_global_variable<scope = #[[MOD]], name = "glr", linkageName = "_QMhelperEglr"{{.*}}line = 15, type = #[[R4]], isDefined = true>
+// CHECK-DAG: #[[GLRE:.*]] = #llvm.di_global_variable_expression<var = #[[GLR]]>
 // CHECK-DAG: #[[LOC3:.*]] = loc("{{.*}}test.f90":20{{.*}})
 // CHECK-DAG: #[[TEST:.*]] = #llvm.di_subprogram<{{.*}}compileUnit = #[[CU]], scope = #[[MOD]], name = "test", linkageName = "_QMhelperPtest"{{.*}}line = 20, scopeLine = 20{{.*}}>
-// CHECK-DAG: loc(fused<#[[GLI]]>[#[[LOC1]]])
-// CHECK-DAG: loc(fused<#[[GLR]]>[#[[LOC2]]])
+// CHECK-DAG: loc(fused<[#[[GLIE]]]>[#[[LOC1]]])
+// CHECK-DAG: loc(fused<[#[[GLRE]]]>[#[[LOC2]]])
 // CHECK-DAG: loc(fused<#[[TEST]]>[#[[LOC3]]])
 
diff --git a/flang/test/Transforms/debug-module-2.fir b/flang/test/Transforms/debug-module-2.fir
index c8d618ce34b26..32a25e303751e 100644
--- a/flang/test/Transforms/debug-module-2.fir
+++ b/flang/test/Transforms/debug-module-2.fir
@@ -20,11 +20,13 @@ module {
 #di_module = #llvm.di_module<file = #di_file, scope = #di_compile_unit, name = "helper", line = 11>
 #di_global_variable = #llvm.di_global_variable<scope = #di_module, name = "gli", linkageName = "_QMhelperEgli", file = #di_file, line = 12, type = #di_basic_type, isDefined = true>
 #di_global_variable1 = #llvm.di_global_variable<scope = #di_module, name = "glr", linkageName = "_QMhelperEglr", file = #di_file, line = 15, type = #di_basic_type1, isDefined = true>
+#di_global_variable_expression = #llvm.di_global_variable_expression<var = #di_global_variable>
+#di_global_variable_expression1 = #llvm.di_global_variable_expression<var = #di_global_variable1>
 
 #loc1 = loc("test.f90":12:11)
 #loc2 = loc("test.f90":15:8)
-#loc3 = loc(fused<#di_global_variable>[#loc1])
-#loc4 = loc(fused<#di_global_variable1>[#loc2])
+#loc3 = loc(fused<[#di_global_variable_expression]>[#loc1])
+#loc4 = loc(fused<[#di_global_variable_expression1]>[#loc2])
 
 
 // CHECK-DAG: #[[GLI:.*]] = #llvm.di_global_variable<{{.*}}name = "gli", linkageName = "_QMhelperEgli"{{.*}}>
diff --git a/flang/test/Transforms/generic-loop-rewriting-todo.mlir b/flang/test/Transforms/generic-loop-rewriting-todo.mlir
index becd6b8dcb5cb..cbde981c4c49d 100644
--- a/flang/test/Transforms/generic-loop-rewriting-todo.mlir
+++ b/flang/test/Transforms/generic-loop-rewriting-todo.mlir
@@ -1,37 +1,31 @@
 // RUN: fir-opt --omp-generic-loop-conversion -verify-diagnostics %s
 
-func.func @_QPtarget_parallel_loop() {
-  omp.target {
-    omp.parallel {
-      %c0 = arith.constant 0 : i32
-      %c10 = arith.constant 10 : i32
-      %c1 = arith.constant 1 : i32
-      // expected-error@below {{not yet implemented: Combined `omp target parallel loop` directive}}
-      omp.loop {
-        omp.loop_nest (%arg3) : i32 = (%c0) to (%c10) inclusive step (%c1) {
-          omp.yield
-        }
+func.func @_QPparallel_loop() {
+  omp.parallel {
+    %c0 = arith.constant 0 : i32
+    %c10 = arith.constant 10 : i32
+    %c1 = arith.constant 1 : i32
+    // expected-error@below {{not yet implemented: Combined `parallel loop` directive}}
+    omp.loop {
+      omp.loop_nest (%arg3) : i32 = (%c0) to (%c10) inclusive step (%c1) {
+        omp.yield
       }
-      omp.terminator
     }
     omp.terminator
   }
   return
 }
 
-func.func @_QPtarget_loop_bind() {
-  omp.target {
-    omp.teams {
-      %c0 = arith.constant 0 : i32
-      %c10 = arith.constant 10 : i32
-      %c1 = arith.constant 1 : i32
-      // expected-error@below {{not yet implemented: Unhandled clause bind in omp.loop operation}}
-      omp.loop bind(thread) {
-        omp.loop_nest (%arg3) : i32 = (%c0) to (%c10) inclusive step (%c1) {
-          omp.yield
-        }
+func.func @_QPloop_bind() {
+  omp.teams {
+    %c0 = arith.constant 0 : i32
+    %c10 = arith.constant 10 : i32
+    %c1 = arith.constant 1 : i32
+    // expected-error@below {{not yet implemented: Unhandled clause bind in omp.loop operation}}
+    omp.loop bind(thread) {
+      omp.loop_nest (%arg3) : i32 = (%c0) to (%c10) inclusive step (%c1) {
+        omp.yield
       }
-      omp.terminator
     }
     omp.terminator
   }
@@ -48,22 +42,18 @@ omp.declare_reduction @add_reduction_i32 : i32 init {
     omp.yield(%0 : i32)
   }
 
-func.func @_QPtarget_loop_order() {
+func.func @_QPloop_order() {
+  omp.teams {
+    %c0 = arith.constant 0 : i32
+    %c10 = arith.constant 10 : i32
+    %c1 = arith.constant 1 : i32
+    %sum = fir.alloca i32 {bindc_name = "i", uniq_name = "_QFtest_orderEi"}
 
-  omp.target {
-    omp.teams {
-      %c0 = arith.constant 0 : i32
-      %c10 = arith.constant 10 : i32
-      %c1 = arith.constant 1 : i32
-      %sum = fir.alloca i32 {bindc_name = "i", uniq_name = "_QFtest_orderEi"}
-
-      // expected-error@below {{not yet implemented: Unhandled clause reduction in omp.loop operation}}
-      omp.loop reduction(@add_reduction_i32 %sum -> %arg2 : !fir.ref<i32>) {
-        omp.loop_nest (%arg3) : i32 = (%c0) to (%c10) inclusive step (%c1) {
-          omp.yield
-        }
+    // expected-error@below {{not yet implemented: Unhandled clause reduction in omp.loop operation}}
+    omp.loop reduction(@add_reduction_i32 %sum -> %arg2 : !fir.ref<i32>) {
+      omp.loop_nest (%arg3) : i32 = (%c0) to (%c10) inclusive step (%c1) {
+        omp.yield
       }
-      omp.terminator
     }
     omp.terminator
   }
diff --git a/flang/test/Transforms/generic-loop-rewriting.mlir b/flang/test/Transforms/generic-loop-rewriting.mlir
index a18ea9853602a..49caf242fe324 100644
--- a/flang/test/Transforms/generic-loop-rewriting.mlir
+++ b/flang/test/Transforms/generic-loop-rewriting.mlir
@@ -1,55 +1,45 @@
 // RUN: fir-opt --omp-generic-loop-conversion %s | FileCheck %s
 
-omp.private {type = private} @_QFtarget_teams_loopEi_private_ref_i32 : !fir.ref<i32> alloc {
-^bb0(%arg0: !fir.ref<i32>):
-  omp.yield(%arg0 : !fir.ref<i32>)
-}
+omp.private {type = private} @_QFteams_loopEi_private_i32 : i32
 
-func.func @_QPtarget_teams_loop() {
+func.func @_QPteams_loop() {
   %i = fir.alloca i32
-  %i_map = omp.map.info var_ptr(%i : !fir.ref<i32>, i32) map_clauses(implicit, exit_release_or_enter_alloc) capture(ByCopy) -> !fir.ref<i32> {name = "i"}
-  omp.target map_entries(%i_map -> %arg0 : !fir.ref<i32>) {
-    omp.teams {
-      %c0 = arith.constant 0 : i32
-      %c10 = arith.constant 10 : i32
-      %c1 = arith.constant 1 : i32
-      omp.loop private(@_QFtarget_teams_loopEi_private_ref_i32 %arg0 -> %arg2 : !fir.ref<i32>) {
-        omp.loop_nest (%arg3) : i32 = (%c0) to (%c10) inclusive step (%c1) {
-          fir.store %arg3 to %arg2 : !fir.ref<i32>
-          omp.yield
-        }
+  omp.teams {
+    %c0 = arith.constant 0 : i32
+    %c10 = arith.constant 10 : i32
+    %c1 = arith.constant 1 : i32
+    omp.loop private(@_QFteams_loopEi_private_i32 %i -> %arg2 : !fir.ref<i32>) {
+      omp.loop_nest (%arg3) : i32 = (%c0) to (%c10) inclusive step (%c1) {
+        fir.store %arg3 to %arg2 : !fir.ref<i32>
+        omp.yield
       }
-      omp.terminator
     }
     omp.terminator
   }
   return
 }
 
-// CHECK-LABEL: func.func @_QPtarget_teams_loop
-// CHECK:         omp.target map_entries(
-// CHECK-SAME:      %{{.*}} -> %[[I_ARG:[^[:space:]]+]] : {{.*}}) {
-// 
-// CHECK:           omp.teams {
+// CHECK-LABEL: func.func @_QPteams_loop
+// CHECK:         %[[I:.*]] = fir.alloca i32
+// CHECK:         omp.teams {
 // 
 // TODO we probably need to move the `loop_nest` bounds ops from the `teams`
 // region to the `parallel` region to avoid making these values `shared`. We can
 // find the backward slices of these bounds that are within the `teams` region
 // and move these slices to the `parallel` op.
 
-// CHECK:             %[[LB:.*]] = arith.constant 0 : i32
-// CHECK:             %[[UB:.*]] = arith.constant 10 : i32
-// CHECK:             %[[STEP:.*]] = arith.constant 1 : i32
+// CHECK:           %[[LB:.*]] = arith.constant 0 : i32
+// CHECK:           %[[UB:.*]] = arith.constant 10 : i32
+// CHECK:           %[[STEP:.*]] = arith.constant 1 : i32
 // 
-// CHECK:             omp.parallel private(@{{.*}} %[[I_ARG]]
-// CHECK-SAME:          -> %[[I_PRIV_ARG:[^[:space:]]+]] : !fir.ref<i32>) {
-// CHECK:               omp.distribute {
-// CHECK:                 omp.wsloop {
+// CHECK:           omp.parallel private(@{{.*}} %[[I]]
+// CHECK-SAME:        -> %[[I_PRIV_ARG:[^[:space:]]+]] : !fir.ref<i32>) {
+// CHECK:             omp.distribute {
+// CHECK:               omp.wsloop {
 // 
-// CHECK:                   omp.loop_nest (%{{.*}}) : i32 = 
-// CHECK-SAME:                (%[[LB]]) to (%[[UB]]) inclusive step (%[[STEP]]) {
-// CHECK:                     fir.store %{{.*}} to %[[I_PRIV_ARG]] : !fir.ref<i32>
-// CHECK:                   }
+// CHECK:                 omp.loop_nest (%{{.*}}) : i32 =
+// CHECK-SAME:              (%[[LB]]) to (%[[UB]]) inclusive step (%[[STEP]]) {
+// CHECK:                   fir.store %{{.*}} to %[[I_PRIV_ARG]] : !fir.ref<i32>
 // CHECK:                 }
 // CHECK:               }
 // CHECK:             }
diff --git a/flang/test/Transforms/loop-versioning.fir b/flang/test/Transforms/loop-versioning.fir
index 2f7c439ed3f4e..cf601fc49b998 100644
--- a/flang/test/Transforms/loop-versioning.fir
+++ b/flang/test/Transforms/loop-versioning.fir
@@ -63,14 +63,14 @@ module attributes {dlti.dl_spec = #dlti.dl_spec<#dlti.dl_entry<f80, dense<128> :
 // CHECK: %{{.*}} = fir.load %[[COORD]] : !fir.ref<f64>
 // CHECK: fir.result %{{.*}}, %{{.*}}
 // CHECK: }
-// CHECK  fir.result %[[LOOP_RES]]#0, %[[LOOP_RES]]#1
+// CHECK: fir.result %[[LOOP_RES]]#0, %[[LOOP_RES]]#1
 // CHECK: } else {
 // CHECK: %[[LOOP_RES2:.*]]:2 = fir.do_loop {{.*}}
 // CHECK: %[[COORD2:.*]] = fir.coordinate_of %[[REBOX]], %{{.*}} : (!fir.box<!fir.array<?xf64>>, i64) -> !fir.ref<f64>
 // CHECK: %{{.*}}= fir.load %[[COORD2]] : !fir.ref<f64>
 // CHECK: fir.result %{{.*}}, %{{.*}}
 // CHECK: }
-// CHECK  fir.result %[[LOOP_RES2]]#0, %[[LOOP_RES2]]#1
+// CHECK: fir.result %[[LOOP_RES2]]#0, %[[LOOP_RES2]]#1
 // CHECK: }
 // CHECK: fir.store %[[IF_RES]]#1 to %{{.*}}
 // CHECK: return
@@ -378,14 +378,14 @@ func.func @sum1dfixed(%arg0: !fir.ref<!fir.array<?xf64>> {fir.bindc_name = "a"},
 // CHECK: %{{.*}} = fir.load %[[COORD]] : !fir.ref<f64>
 // CHECK: fir.result %{{.*}}, %{{.*}}
 // CHECK: }
-// CHECK  fir.result %[[LOOP_RES]]#0, %[[LOOP_RES]]#1
+// CHECK: fir.result %[[LOOP_RES]]#0, %[[LOOP_RES]]#1
 // CHECK: } else {
 // CHECK: %[[LOOP_RES2:.*]]:2 = fir.do_loop {{.*}}
 // CHECK: %[[COORD2:.*]] = fir.coordinate_of %[[ARG0]], %{{.*}} : (!fir.box<!fir.array<?x?xf64>>, i64, i64) -> !fir.ref<f64>
 // CHECK: %{{.*}}= fir.load %[[COORD2]] : !fir.ref<f64>
 // CHECK: fir.result %{{.*}}, %{{.*}}
 // CHECK: }
-// CHECK  fir.result %[[LOOP_RES2]]#0, %[[LOOP_RES2]]#1
+// CHECK: fir.result %[[LOOP_RES2]]#0, %[[LOOP_RES2]]#1
 // CHECK: }
 // CHECK: fir.store %[[IF_RES]]#1 to %{{.*}}
 // CHECK: return
@@ -510,14 +510,14 @@ func.func @sum1dfixed(%arg0: !fir.ref<!fir.array<?xf64>> {fir.bindc_name = "a"},
 // CHECK: %{{.*}} = fir.load %[[COORD]] : !fir.ref<f64>
 // CHECK: fir.result %{{.*}}, %{{.*}}
 // CHECK: }
-// CHECK  fir.result %[[LOOP_RES]]#0, %[[LOOP_RES]]#1
+// CHECK: fir.result %[[LOOP_RES]]#0, %[[LOOP_RES]]#1
 // CHECK: } else {
 // CHECK: %[[LOOP_RES2:.*]]:2 = fir.do_loop {{.*}}
 // CHECK: %[[COORD2:.*]] = fir.coordinate_of %[[ARG0]], %{{.*}} : (!fir.box<!fir.array<?x?x?xf64>>, i64, i64, i64) -> !fir.ref<f64>
 // CHECK: %{{.*}}= fir.load %[[COORD2]] : !fir.ref<f64>
 // CHECK: fir.result %{{.*}}, %{{.*}}
 // CHECK: }
-// CHECK  fir.result %[[LOOP_RES2]]#0, %[[LOOP_RES2]]#1
+// CHECK: fir.result %[[LOOP_RES2]]#0, %[[LOOP_RES2]]#1
 // CHECK: }
 // CHECK: fir.store %[[IF_RES]]#1 to %{{.*}}
 // CHECK: return
diff --git a/flang/test/Transforms/omp-maps-for-privatized-symbols.fir b/flang/test/Transforms/omp-maps-for-privatized-symbols.fir
index d32444aaabf23..10a76126ed054 100644
--- a/flang/test/Transforms/omp-maps-for-privatized-symbols.fir
+++ b/flang/test/Transforms/omp-maps-for-privatized-symbols.fir
@@ -1,12 +1,12 @@
 // RUN: fir-opt --split-input-file --omp-maps-for-privatized-symbols %s | FileCheck %s
 module attributes {omp.is_target_device = false} {
-  omp.private {type = private} @_QFtarget_simpleEsimple_var_private_ref_box_heap_i32 : !fir.ref<!fir.box<!fir.heap<i32>>> alloc {
-  ^bb0(%arg0: !fir.ref<!fir.box<!fir.heap<i32>>>):
-    %0 = fir.alloca !fir.box<!fir.heap<i32>> {bindc_name = "simple_var", pinned, uniq_name = "_QFtarget_simpleEsimple_var"}
-    %1 = fir.load %arg0 : !fir.ref<!fir.box<!fir.heap<i32>>>
-    %5:2 = hlfir.declare %0 {fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "_QFtarget_simpleEsimple_var"} : (!fir.ref<!fir.box<!fir.heap<i32>>>) -> (!fir.ref<!fir.box<!fir.heap<i32>>>, !fir.ref<!fir.box<!fir.heap<i32>>>)
-    omp.yield(%5#0 : !fir.ref<!fir.box<!fir.heap<i32>>>)
+  omp.private {type = private} @_QFtarget_simpleEsimple_var_private_ref_box_heap_i32 : !fir.box<!fir.heap<i32>> init {
+  ^bb0(%arg0: !fir.ref<!fir.box<!fir.heap<i32>>>, %arg1: !fir.ref<!fir.box<!fir.heap<i32>>>):
+    %mold = fir.load %arg0 : !fir.ref<!fir.box<!fir.heap<i32>>>
+    // extract box address, see if it is null, etc
+    omp.yield(%arg1: !fir.ref<!fir.box<!fir.heap<i32>>>)
   }
+
   func.func @_QPtarget_simple() {
     %0 = fir.alloca i32 {bindc_name = "a", uniq_name = "_QFtarget_simpleEa"}
     %1:2 = hlfir.declare %0 {uniq_name = "_QFtarget_simpleEa"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
diff --git a/flang/test/Transforms/simplifyintrinsics.fir b/flang/test/Transforms/simplifyintrinsics.fir
index b2af152e5a913..9a95748c17258 100644
--- a/flang/test/Transforms/simplifyintrinsics.fir
+++ b/flang/test/Transforms/simplifyintrinsics.fir
@@ -428,7 +428,7 @@ module attributes {fir.defaultkind = "a1c4d8i4l4r4", fir.kindmap = "", llvm.targ
 }
 
 // Just check that SOMETHING is being output.
-// CHECK-LABEL @sum_dim() {
+// CHECK-LABEL: @sum_dim() {
 // CHECK: return
 
 
@@ -459,7 +459,7 @@ module attributes {fir.defaultkind = "a1c4d8i4l4r4", fir.kindmap = "", llvm.targ
 }
 
 // Just check that SOMETHING is being output.
-// CHECK-LABEL @sum_1d_unknown() {
+// CHECK-LABEL: @sum_1d_unknown(
 // CHECK: return
 
 // -----
@@ -1260,9 +1260,9 @@ func.func private @_FortranACountDim(!fir.ref<!fir.box<none>>, !fir.box<none>, i
 
 // CHECK-LABEL:   func.func @_QMtestPcount_generate_mask(
 // CHECK-SAME:                                           %[[A:.*]]: !fir.ref<!fir.array<10x10x!fir.logical<4>>> {fir.bindc_name = "mask"}) -> !fir.array<10xi32> {
-// CHECK-NOT        fir.call @_FortranACountDimLogical4_simplified({{.*}})
+// CHECK-NOT:       fir.call @_FortranACountDimLogical4_simplified({{.*}})
 // CHECK:           fir.call @_FortranACountDim({{.*}}) fastmath<contract> : (!fir.ref<!fir.box<none>>, !fir.box<none>, i32, i32, !fir.ref<i8>, i32) -> ()
-// CHECK-NOT        fir.call @_FortranACountDimLogical4_simplified({{.*}})
+// CHECK-NOT:       fir.call @_FortranACountDimLogical4_simplified({{.*}})
 
 // -----
 // Ensure count isn't simplified for unknown dimension arrays
@@ -1285,9 +1285,9 @@ func.func private @_FortranACount(!fir.box<none>, !fir.ref<i8>, i32, i32) -> i64
 
 // CHECK-LABEL:   func.func @_QPmc(
 // CHECK-SAME:                     %[[VAL_0:.*]]: !fir.box<!fir.array<?x?x?x!fir.logical<4>>> {fir.bindc_name = "m"}) -> i32 {
-// CHECK-NOT        fir.call @_FortranACountLogical4_simplified({{.*}})
+// CHECK-NOT:       fir.call @_FortranACountLogical4_simplified({{.*}})
 // CHECK:           %[[RES:.*]] = fir.call @_FortranACount({{.*}}) fastmath<contract> : (!fir.box<none>, !fir.ref<i8>, i32, i32) -> i64
-// CHECK-NOT        fir.call @_FortranACountLogical4_simplified({{.*}})
+// CHECK-NOT:       fir.call @_FortranACountLogical4_simplified({{.*}})
 
 // -----
 // Ensure Any is simplified in correct usage
@@ -1443,9 +1443,9 @@ func.func private @_FortranAAnyDim(!fir.ref<!fir.box<none>>, !fir.box<none>, i32
 
 // CHECK-LABEL:   func.func @_QPtestAny_DimArg(
 // CHECK-SAME:                          %[[ARR:.*]]: !fir.ref<!fir.array<10x10x!fir.logical<4>>> {fir.bindc_name = "a"}) -> !fir.array<10x!fir.logical<4>> {
-// CHECK-NOT        fir.call @_FortranAAnyDimLogical4x1_simplified({{.*}})
+// CHECK-NOT:       fir.call @_FortranAAnyDimLogical4x1_simplified({{.*}})
 // CHECK:           fir.call @_FortranAAnyDim({{.*}}) fastmath<contract> : (!fir.ref<!fir.box<none>>, !fir.box<none>, i32, !fir.ref<i8>, i32) -> ()
-// CHECK-NOT        fir.call @_FortranAAnyDimLogical4x1_simplified({{.*}})
+// CHECK-NOT:       fir.call @_FortranAAnyDimLogical4x1_simplified({{.*}})
 
 // -----
 // Ensure Any is not simplified for unknown dimension arrays
@@ -1468,9 +1468,9 @@ func.func private @_FortranAAny(!fir.box<none>, !fir.ref<i8>, i32, i32) -> i1 at
 
 // CHECK-LABEL:   func.func @_QPtestAny_UnknownDim(
 // CHECK-SAME:                          %[[VAL_0:.*]]: !fir.box<!fir.array<?x!fir.logical<4>>> {fir.bindc_name = "a"}) -> !fir.logical<4> {
-// CHECK-NOT        fir.call @_FortranAAnyLogical4x1_simplified({{.*}})
+// CHECK-NOT:       fir.call @_FortranAAnyLogical4x1_simplified({{.*}})
 // CHECK:           fir.call @_FortranAAny({{.*}}) fastmath<contract> : (!fir.box<none>, !fir.ref<i8>, i32, i32) -> i1
-// CHECK-NOT        fir.call @_FortranAAnyLogical4x1_simplified({{.*}})
+// CHECK-NOT:       fir.call @_FortranAAnyLogical4x1_simplified({{.*}})
 
 // -----
 // Check that multi-rank Any cases are properly simplified
@@ -1682,9 +1682,9 @@ func.func private @_FortranAAllDim(!fir.ref<!fir.box<none>>, !fir.box<none>, i32
 
 // CHECK-LABEL:   func.func @_QPtestAll_DimArg(
 // CHECK-SAME:                          %[[ARR:.*]]: !fir.ref<!fir.array<10x10x!fir.logical<4>>> {fir.bindc_name = "a"}) -> !fir.array<10x!fir.logical<4>> {
-// CHECK-NOT        fir.call @_FortranAAllDimLogical4x1_simplified({{.*}})
+// CHECK-NOT:       fir.call @_FortranAAllDimLogical4x1_simplified({{.*}})
 // CHECK:           fir.call @_FortranAAllDim({{.*}}) fastmath<contract> : (!fir.ref<!fir.box<none>>, !fir.box<none>, i32, !fir.ref<i8>, i32) -> ()
-// CHECK-NOT        fir.call @_FortranAAllDimLogical4x1_simplified({{.*}})
+// CHECK-NOT:       fir.call @_FortranAAllDimLogical4x1_simplified({{.*}})
 
 // -----
 // Check Minloc simplifies correctly for 1D case with 1D mask, I32 input
diff --git a/flang/test/lib/OpenACC/TestOpenACCInterfaces.cpp b/flang/test/lib/OpenACC/TestOpenACCInterfaces.cpp
index a01396748f4c2..5c14809a265e1 100644
--- a/flang/test/lib/OpenACC/TestOpenACCInterfaces.cpp
+++ b/flang/test/lib/OpenACC/TestOpenACCInterfaces.cpp
@@ -28,7 +28,7 @@ struct TestFIROpenACCInterfaces
   void runOnOperation() override {
     mlir::ModuleOp mod = getOperation();
     auto datalayout =
-        fir::support::getOrSetDataLayout(mod, /*allowDefaultLayout=*/true);
+        fir::support::getOrSetMLIRDataLayout(mod, /*allowDefaultLayout=*/true);
     mlir::OpBuilder builder(mod);
     getOperation().walk([&](Operation *op) {
       if (isa<ACC_DATA_ENTRY_OPS>(op)) {
diff --git a/flang/test/lit.cfg.py b/flang/test/lit.cfg.py
index f43234fb125b7..c6266f3976f7c 100644
--- a/flang/test/lit.cfg.py
+++ b/flang/test/lit.cfg.py
@@ -163,17 +163,18 @@
         ToolSubst("%not_todo_abort_cmd", command=FindTool("not"), unresolved="fatal")
     )
 
+if config.flang_include_runtime:
+    config.available_features.add("flang-rt")
+
 # Define some variables to help us test that the flang runtime doesn't depend on
 # the C++ runtime libraries. For this we need a C compiler. If for some reason
 # we don't have one, we can just disable the test.
-if config.cc:
-    libruntime = os.path.join(config.flang_lib_dir, "libFortranRuntime.a")
-    libdecimal = os.path.join(config.flang_lib_dir, "libFortranDecimal.a")
+if config.flang_include_runtime and config.cc:
+    libruntime = os.path.join(config.flang_lib_dir, "libflang_rt.runtime.a")
     include = os.path.join(config.flang_src_dir, "include")
 
     if (
         os.path.isfile(libruntime)
-        and os.path.isfile(libdecimal)
         and os.path.isdir(include)
     ):
         config.available_features.add("c-compiler")
@@ -183,7 +184,6 @@
             )
         )
         tools.append(ToolSubst("%libruntime", command=libruntime, unresolved="fatal"))
-        tools.append(ToolSubst("%libdecimal", command=libdecimal, unresolved="fatal"))
         tools.append(ToolSubst("%include", command=include, unresolved="fatal"))
 
 # Add all the tools and their substitutions (if applicable). Use the search paths provided for
diff --git a/flang/test/lit.site.cfg.py.in b/flang/test/lit.site.cfg.py.in
index d1a0ac763cf8a..697ba3fa79763 100644
--- a/flang/test/lit.site.cfg.py.in
+++ b/flang/test/lit.site.cfg.py.in
@@ -1,6 +1,7 @@
 @LIT_SITE_CFG_IN_HEADER@
 
 import sys
+import lit.util
 
 config.llvm_tools_dir = lit_config.substitute("@LLVM_TOOLS_DIR@")
 config.llvm_shlib_dir = lit_config.substitute(path(r"@SHLIBDIR@"))
@@ -32,6 +33,7 @@ else:
     config.openmp_module_dir = None
 config.flang_runtime_f128_math_lib = "@FLANG_RUNTIME_F128_MATH_LIB@"
 config.have_ldbl_mant_dig_113 = "@HAVE_LDBL_MANT_DIG_113@"
+config.flang_include_runtime = lit.util.pythonize_bool("@FLANG_INCLUDE_RUNTIME@")
 
 import lit.llvm
 lit.llvm.initialize(lit_config, config)
diff --git a/flang/tools/bbc/CMakeLists.txt b/flang/tools/bbc/CMakeLists.txt
index 85aeb85e0c530..f950f03920d3f 100644
--- a/flang/tools/bbc/CMakeLists.txt
+++ b/flang/tools/bbc/CMakeLists.txt
@@ -36,7 +36,7 @@ mlir_target_link_libraries(bbc PRIVATE
   ${extension_libs}
   MLIRAffineToStandard
   MLIRSCFToControlFlow
-  FortranCommon
+  FortranSupport
   FortranParser
   FortranEvaluate
   FortranSemantics
diff --git a/flang/tools/bbc/bbc.cpp b/flang/tools/bbc/bbc.cpp
index dafbcd856389a..3b19a1c2a78d9 100644
--- a/flang/tools/bbc/bbc.cpp
+++ b/flang/tools/bbc/bbc.cpp
@@ -14,11 +14,6 @@
 ///
 //===----------------------------------------------------------------------===//
 
-#include "flang/Common/Fortran-features.h"
-#include "flang/Common/LangOptions.h"
-#include "flang/Common/OpenMP-features.h"
-#include "flang/Common/Version.h"
-#include "flang/Common/default-kinds.h"
 #include "flang/Frontend/CodeGenOptions.h"
 #include "flang/Frontend/TargetOptions.h"
 #include "flang/Lower/Bridge.h"
@@ -42,6 +37,11 @@
 #include "flang/Semantics/runtime-type-info.h"
 #include "flang/Semantics/semantics.h"
 #include "flang/Semantics/unparse-with-symbols.h"
+#include "flang/Support/Fortran-features.h"
+#include "flang/Support/LangOptions.h"
+#include "flang/Support/OpenMP-features.h"
+#include "flang/Support/Version.h"
+#include "flang/Support/default-kinds.h"
 #include "flang/Tools/CrossToolHelpers.h"
 #include "flang/Tools/TargetSetup.h"
 #include "flang/Version.inc"
diff --git a/flang/tools/f18-parse-demo/CMakeLists.txt b/flang/tools/f18-parse-demo/CMakeLists.txt
index df1f1d24a55e5..743a44017e8a5 100644
--- a/flang/tools/f18-parse-demo/CMakeLists.txt
+++ b/flang/tools/f18-parse-demo/CMakeLists.txt
@@ -11,6 +11,6 @@ add_flang_tool(f18-parse-demo
 
 target_link_libraries(f18-parse-demo
   PRIVATE
-  FortranCommon
+  FortranSupport
   FortranParser
   )
diff --git a/flang/tools/f18-parse-demo/f18-parse-demo.cpp b/flang/tools/f18-parse-demo/f18-parse-demo.cpp
index 3025dbd4dca4a..a50c88dc84064 100644
--- a/flang/tools/f18-parse-demo/f18-parse-demo.cpp
+++ b/flang/tools/f18-parse-demo/f18-parse-demo.cpp
@@ -21,8 +21,6 @@
 // scaffolding compiler driver that can test some semantic passes of the
 // F18 compiler under development.
 
-#include "flang/Common/Fortran-features.h"
-#include "flang/Common/default-kinds.h"
 #include "flang/Parser/characters.h"
 #include "flang/Parser/dump-parse-tree.h"
 #include "flang/Parser/message.h"
@@ -31,6 +29,8 @@
 #include "flang/Parser/parsing.h"
 #include "flang/Parser/provenance.h"
 #include "flang/Parser/unparse.h"
+#include "flang/Support/Fortran-features.h"
+#include "flang/Support/default-kinds.h"
 #include "llvm/Support/Errno.h"
 #include "llvm/Support/FileSystem.h"
 #include "llvm/Support/Program.h"
diff --git a/flang/tools/f18/CMakeLists.txt b/flang/tools/f18/CMakeLists.txt
index cc2bc5b8eb5ce..5b5f23b5dc73c 100644
--- a/flang/tools/f18/CMakeLists.txt
+++ b/flang/tools/f18/CMakeLists.txt
@@ -5,7 +5,7 @@ set(LLVM_LINK_COMPONENTS
   )
 
 # Define the list of Fortran module files that need to be compiled
-# to produce an object file for inclusion into the FortranRuntime
+# to produce an object file for inclusion into the flang_rt.runtime
 # library.
 set(MODULES_WITH_IMPLEMENTATION
   "iso_fortran_env_impl"
@@ -105,11 +105,11 @@ if (NOT CMAKE_CROSSCOMPILING)
     endif()
 
     # Some modules have an implementation part that needs to be added to the
-    # FortranRuntime library.
+    # flang_rt.runtime library.
     set(compile_with "-fsyntax-only")
     set(object_output "")
     set(include_in_link FALSE)
-    if(${filename} IN_LIST MODULES_WITH_IMPLEMENTATION)
+    if(${filename} IN_LIST MODULES_WITH_IMPLEMENTATION AND FLANG_INCLUDE_RUNTIME)
       set(object_output "${CMAKE_CURRENT_BINARY_DIR}/${filename}${CMAKE_CXX_OUTPUT_EXTENSION}")
       set(compile_with -c -o ${object_output})
       set(include_in_link TRUE)
@@ -127,14 +127,14 @@ if (NOT CMAKE_CROSSCOMPILING)
     install(FILES ${base}.mod DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}/flang")
 
     # If a module has been compiled into an object file, add the file to
-    # the link line for the FortranRuntime library.
+    # the link line for the flang_rt.runtime library.
     if(include_in_link)
       list(APPEND module_objects ${object_output})
     endif()
   endforeach()
 
   # Set a CACHE variable that is visible to the CMakeLists.txt in runtime/, so that
-  # the compiled Fortran modules can be added to the link line of the FortranRuntime
+  # the compiled Fortran modules can be added to the link line of the flang_rt.runtime
   # library.
   set(FORTRAN_MODULE_OBJECTS ${module_objects} CACHE INTERNAL "" FORCE)
 
diff --git a/flang/tools/tco/CMakeLists.txt b/flang/tools/tco/CMakeLists.txt
index 220f908d2f108..c277cdbf167b6 100644
--- a/flang/tools/tco/CMakeLists.txt
+++ b/flang/tools/tco/CMakeLists.txt
@@ -20,7 +20,7 @@ target_link_libraries(tco PRIVATE
   flangPasses
   FIROpenACCSupport
   FlangOpenMPTransforms
-  FortranCommon
+  FortranSupport
 )
 
 mlir_target_link_libraries(tco PRIVATE
diff --git a/flang/unittests/CMakeLists.txt b/flang/unittests/CMakeLists.txt
index 945067fed4f82..c54ceb3332abf 100644
--- a/flang/unittests/CMakeLists.txt
+++ b/flang/unittests/CMakeLists.txt
@@ -1,6 +1,8 @@
+include(AddFlangOffloadRuntime)
+
 if (FLANG_EXPERIMENTAL_CUDA_RUNTIME)
   # If Fortran runtime is built as CUDA library, the linking
-  # of targets that link FortranRuntime must be done
+  # of targets that link flang_rt.runtime must be done
   # with CUDA_RESOLVE_DEVICE_SYMBOLS.
   # CUDA language must be enabled for CUDA_RESOLVE_DEVICE_SYMBOLS
   # to take effect.
@@ -11,6 +13,11 @@ add_custom_target(FlangUnitTests)
 set_target_properties(FlangUnitTests PROPERTIES FOLDER "Flang/Tests")
 
 function(add_flang_unittest_offload_properties target)
+  # Do not apply runtime properties if not even compiling the runtime.
+  if (NOT FLANG_INCLUDE_RUNTIME)
+    return ()
+  endif ()
+
   # Set CUDA_RESOLVE_DEVICE_SYMBOLS.
   if (FLANG_EXPERIMENTAL_CUDA_RUNTIME)
     set_target_properties(${target}
@@ -75,5 +82,7 @@ add_subdirectory(Optimizer)
 add_subdirectory(Common)
 add_subdirectory(Decimal)
 add_subdirectory(Evaluate)
-add_subdirectory(Runtime)
+if (FLANG_INCLUDE_RUNTIME)
+  add_subdirectory(Runtime)
+endif ()
 add_subdirectory(Frontend)
diff --git a/flang/unittests/Evaluate/CMakeLists.txt b/flang/unittests/Evaluate/CMakeLists.txt
index 52eb385f4763f..2278d61febcb1 100644
--- a/flang/unittests/Evaluate/CMakeLists.txt
+++ b/flang/unittests/Evaluate/CMakeLists.txt
@@ -1,56 +1,42 @@
 set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR})
-add_library(FortranEvaluateTesting
-  testing.cpp
-  fp-testing.cpp
-)
-set_target_properties(FortranEvaluateTesting PROPERTIES FOLDER "Flang/Tests")
-if (LLVM_LINK_LLVM_DYLIB)
-  set(llvm_libs LLVM)
-else()
-  llvm_map_components_to_libnames(llvm_libs Support)
-endif()
-
-target_link_libraries(FortranEvaluateTesting
-  ${llvm_libs})
 
 add_flang_nongtest_unittest(leading-zero-bit-count
-  FortranEvaluateTesting
+  NonGTestTesting
 )
 
 add_flang_nongtest_unittest(bit-population-count
-  FortranEvaluateTesting
+  NonGTestTesting
 )
 
 add_flang_nongtest_unittest(uint128
-  FortranEvaluateTesting
+  NonGTestTesting
 )
 
 add_flang_nongtest_unittest(expression
-  FortranCommon
-  FortranEvaluateTesting
+  FortranSupport
+  NonGTestTesting
   FortranEvaluate
   FortranSemantics
   FortranParser
 )
 
 add_flang_nongtest_unittest(integer
-  FortranEvaluateTesting
+  NonGTestTesting
   FortranEvaluate
   FortranSemantics
 )
 
 add_flang_nongtest_unittest(intrinsics
-  FortranCommon
-  FortranEvaluateTesting
+  FortranSupport
+  NonGTestTesting
   FortranEvaluate
   FortranDecimal
   FortranSemantics
   FortranParser
-  FortranRuntime
 )
 
 add_flang_nongtest_unittest(logical
-  FortranEvaluateTesting
+  NonGTestTesting
   FortranEvaluate
   FortranSemantics
 )
@@ -62,30 +48,32 @@ add_flang_nongtest_unittest(logical
 set(LLVM_REQUIRES_EH ON)
 set(LLVM_REQUIRES_RTTI ON)
 add_flang_nongtest_unittest(real
-  FortranEvaluateTesting
+  NonGTestTesting
   FortranEvaluate
   FortranDecimal
   FortranSemantics
 )
 llvm_update_compile_flags(real.test)
 
-add_flang_nongtest_unittest(reshape
-  FortranEvaluateTesting
-  FortranSemantics
-  FortranEvaluate
-  FortranRuntime
-)
+if (FLANG_INCLUDE_RUNTIME)
+  add_flang_nongtest_unittest(reshape
+    NonGTestTesting
+    FortranSemantics
+    FortranEvaluate
+    flang_rt.runtime
+  )
 
-add_flang_nongtest_unittest(ISO-Fortran-binding
-  FortranEvaluateTesting
-  FortranEvaluate
-  FortranSemantics
-  FortranRuntime
-)
+  add_flang_nongtest_unittest(ISO-Fortran-binding
+    NonGTestTesting
+    FortranEvaluate
+    FortranSemantics
+    flang_rt.runtime
+  )
+endif ()
 
 add_flang_nongtest_unittest(folding
-  FortranCommon
-  FortranEvaluateTesting
+  FortranSupport
+  NonGTestTesting
   FortranEvaluate
   FortranSemantics
 )
diff --git a/flang/unittests/Evaluate/ISO-Fortran-binding.cpp b/flang/unittests/Evaluate/ISO-Fortran-binding.cpp
index 3c98363f90046..2884686b3f26e 100644
--- a/flang/unittests/Evaluate/ISO-Fortran-binding.cpp
+++ b/flang/unittests/Evaluate/ISO-Fortran-binding.cpp
@@ -1,6 +1,6 @@
-#include "testing.h"
-#include "flang/ISO_Fortran_binding_wrapper.h"
+#include "flang/Common/ISO_Fortran_binding_wrapper.h"
 #include "flang/Runtime/descriptor.h"
+#include "flang/Testing/testing.h"
 #include "llvm/Support/raw_ostream.h"
 #include <type_traits>
 
diff --git a/flang/unittests/Evaluate/bit-population-count.cpp b/flang/unittests/Evaluate/bit-population-count.cpp
index 24e721c14f94a..0c9529a92e572 100644
--- a/flang/unittests/Evaluate/bit-population-count.cpp
+++ b/flang/unittests/Evaluate/bit-population-count.cpp
@@ -1,5 +1,5 @@
 #include "flang/Common/bit-population-count.h"
-#include "testing.h"
+#include "flang/Testing/testing.h"
 
 using Fortran::common::BitPopulationCount;
 using Fortran::common::Parity;
diff --git a/flang/unittests/Evaluate/expression.cpp b/flang/unittests/Evaluate/expression.cpp
index f03a6bc2a4e23..d575f36def287 100644
--- a/flang/unittests/Evaluate/expression.cpp
+++ b/flang/unittests/Evaluate/expression.cpp
@@ -1,10 +1,10 @@
 #include "flang/Evaluate/expression.h"
-#include "testing.h"
 #include "flang/Evaluate/fold.h"
 #include "flang/Evaluate/intrinsics.h"
 #include "flang/Evaluate/target.h"
 #include "flang/Evaluate/tools.h"
 #include "flang/Parser/message.h"
+#include "flang/Testing/testing.h"
 #include <cstdio>
 #include <cstdlib>
 #include <string>
diff --git a/flang/unittests/Evaluate/folding.cpp b/flang/unittests/Evaluate/folding.cpp
index 4e8ff9754e4cc..832e55d44316d 100644
--- a/flang/unittests/Evaluate/folding.cpp
+++ b/flang/unittests/Evaluate/folding.cpp
@@ -1,4 +1,3 @@
-#include "testing.h"
 #include "../../lib/Evaluate/host.h"
 #include "flang/Evaluate/call.h"
 #include "flang/Evaluate/expression.h"
@@ -7,6 +6,7 @@
 #include "flang/Evaluate/intrinsics.h"
 #include "flang/Evaluate/target.h"
 #include "flang/Evaluate/tools.h"
+#include "flang/Testing/testing.h"
 #include <tuple>
 
 using namespace Fortran::evaluate;
diff --git a/flang/unittests/Evaluate/fp-testing.h b/flang/unittests/Evaluate/fp-testing.h
deleted file mode 100644
index 9091963a99b32..0000000000000
--- a/flang/unittests/Evaluate/fp-testing.h
+++ /dev/null
@@ -1,27 +0,0 @@
-#ifndef FORTRAN_TEST_EVALUATE_FP_TESTING_H_
-#define FORTRAN_TEST_EVALUATE_FP_TESTING_H_
-
-#include "flang/Common/target-rounding.h"
-#include <fenv.h>
-
-using Fortran::common::RealFlags;
-using Fortran::common::Rounding;
-using Fortran::common::RoundingMode;
-
-class ScopedHostFloatingPointEnvironment {
-public:
-  ScopedHostFloatingPointEnvironment(bool treatSubnormalOperandsAsZero = false,
-      bool flushSubnormalResultsToZero = false);
-  ~ScopedHostFloatingPointEnvironment();
-  void ClearFlags() const;
-  static RealFlags CurrentFlags();
-  static void SetRounding(Rounding rounding);
-
-private:
-  fenv_t originalFenv_;
-#if __x86_64__
-  unsigned int originalMxcsr;
-#endif
-};
-
-#endif // FORTRAN_TEST_EVALUATE_FP_TESTING_H_
diff --git a/flang/unittests/Evaluate/integer.cpp b/flang/unittests/Evaluate/integer.cpp
index 3eabb2434cce6..5ab22cc6443a7 100644
--- a/flang/unittests/Evaluate/integer.cpp
+++ b/flang/unittests/Evaluate/integer.cpp
@@ -1,5 +1,5 @@
 #include "flang/Evaluate/integer.h"
-#include "testing.h"
+#include "flang/Testing/testing.h"
 #include <cstdio>
 #include <string>
 
diff --git a/flang/unittests/Evaluate/intrinsics.cpp b/flang/unittests/Evaluate/intrinsics.cpp
index 86c471d2c6295..cca2f8c30247e 100644
--- a/flang/unittests/Evaluate/intrinsics.cpp
+++ b/flang/unittests/Evaluate/intrinsics.cpp
@@ -1,10 +1,10 @@
 #include "flang/Evaluate/intrinsics.h"
-#include "testing.h"
 #include "flang/Evaluate/common.h"
 #include "flang/Evaluate/expression.h"
 #include "flang/Evaluate/target.h"
 #include "flang/Evaluate/tools.h"
 #include "flang/Parser/provenance.h"
+#include "flang/Testing/testing.h"
 #include "llvm/Support/raw_ostream.h"
 #include <initializer_list>
 #include <map>
diff --git a/flang/unittests/Evaluate/leading-zero-bit-count.cpp b/flang/unittests/Evaluate/leading-zero-bit-count.cpp
index 968946b69f27e..25e2664e76df0 100644
--- a/flang/unittests/Evaluate/leading-zero-bit-count.cpp
+++ b/flang/unittests/Evaluate/leading-zero-bit-count.cpp
@@ -1,5 +1,5 @@
 #include "flang/Common/leading-zero-bit-count.h"
-#include "testing.h"
+#include "flang/Testing/testing.h"
 
 using Fortran::common::LeadingZeroBitCount;
 
diff --git a/flang/unittests/Evaluate/logical.cpp b/flang/unittests/Evaluate/logical.cpp
index d79aff8448712..ba7d0d8d0c0e3 100644
--- a/flang/unittests/Evaluate/logical.cpp
+++ b/flang/unittests/Evaluate/logical.cpp
@@ -1,5 +1,5 @@
-#include "testing.h"
 #include "flang/Evaluate/type.h"
+#include "flang/Testing/testing.h"
 #include <cstdio>
 
 template <int KIND> void testKind() {
diff --git a/flang/unittests/Evaluate/real.cpp b/flang/unittests/Evaluate/real.cpp
index 1bf7fa1016a9e..a28da5c3273ce 100644
--- a/flang/unittests/Evaluate/real.cpp
+++ b/flang/unittests/Evaluate/real.cpp
@@ -1,6 +1,6 @@
-#include "fp-testing.h"
-#include "testing.h"
 #include "flang/Evaluate/type.h"
+#include "flang/Testing/fp-testing.h"
+#include "flang/Testing/testing.h"
 #include "llvm/Support/raw_ostream.h"
 #include <cmath>
 #include <cstdio>
diff --git a/flang/unittests/Evaluate/reshape.cpp b/flang/unittests/Evaluate/reshape.cpp
index 5d138cd3d88d9..16cba15dcef60 100644
--- a/flang/unittests/Evaluate/reshape.cpp
+++ b/flang/unittests/Evaluate/reshape.cpp
@@ -1,6 +1,6 @@
-#include "testing.h"
 #include "flang/Runtime/descriptor.h"
 #include "flang/Runtime/transformational.h"
+#include "flang/Testing/testing.h"
 #include <cinttypes>
 
 using namespace Fortran::common;
diff --git a/flang/unittests/Evaluate/uint128.cpp b/flang/unittests/Evaluate/uint128.cpp
index f05c1c96a7e8b..0b749abe1c080 100644
--- a/flang/unittests/Evaluate/uint128.cpp
+++ b/flang/unittests/Evaluate/uint128.cpp
@@ -1,6 +1,6 @@
 #define AVOID_NATIVE_UINT128_T 1
 #include "flang/Common/uint128.h"
-#include "testing.h"
+#include "flang/Testing/testing.h"
 #include "llvm/Support/raw_ostream.h"
 #include <cinttypes>
 
diff --git a/flang/unittests/Frontend/CMakeLists.txt b/flang/unittests/Frontend/CMakeLists.txt
index 9177997f41f53..16e0949948f26 100644
--- a/flang/unittests/Frontend/CMakeLists.txt
+++ b/flang/unittests/Frontend/CMakeLists.txt
@@ -12,16 +12,20 @@ add_flang_unittest(FlangFrontendTests
 
 target_link_libraries(FlangFrontendTests
   PRIVATE
-  clangBasic
   flangFrontend
   flangFrontendTool
   FortranLower
   FortranParser
   FortranSemantics
-  FortranCommon
+  FortranSupport
   FortranEvaluate
 )
 
+clang_target_link_libraries(FlangFrontendTests
+  PRIVATE
+  clangBasic
+)
+
 mlir_target_link_libraries(FlangFrontendTests
   PRIVATE
   MLIRIR
diff --git a/flang/unittests/Runtime/CMakeLists.txt b/flang/unittests/Runtime/CMakeLists.txt
index 179e439917ff2..f3743be49b015 100644
--- a/flang/unittests/Runtime/CMakeLists.txt
+++ b/flang/unittests/Runtime/CMakeLists.txt
@@ -33,7 +33,7 @@ add_flang_unittest(FlangRuntimeTests
 
 target_link_libraries(FlangRuntimeTests
   PRIVATE
-  FortranRuntime
+  flang_rt.runtime
 )
 
 target_compile_definitions(FlangRuntimeTests PRIVATE NOT_EXE="$<TARGET_FILE:not>")
diff --git a/flang/unittests/Runtime/CUDA/Allocatable.cpp b/flang/unittests/Runtime/CUDA/Allocatable.cpp
index 0f7eb27789316..bdfa8f5cc3213 100644
--- a/flang/unittests/Runtime/CUDA/Allocatable.cpp
+++ b/flang/unittests/Runtime/CUDA/Allocatable.cpp
@@ -9,11 +9,11 @@
 #include "flang/Runtime/allocatable.h"
 #include "gtest/gtest.h"
 #include "../../../runtime/terminator.h"
-#include "flang/Common/Fortran.h"
 #include "flang/Runtime/CUDA/allocator.h"
 #include "flang/Runtime/CUDA/common.h"
 #include "flang/Runtime/CUDA/descriptor.h"
 #include "flang/Runtime/allocator-registry.h"
+#include "flang/Support/Fortran.h"
 
 #include "cuda_runtime.h"
 
diff --git a/flang/unittests/Runtime/CUDA/AllocatorCUF.cpp b/flang/unittests/Runtime/CUDA/AllocatorCUF.cpp
index 7cb25787e7797..5ec122e4c5777 100644
--- a/flang/unittests/Runtime/CUDA/AllocatorCUF.cpp
+++ b/flang/unittests/Runtime/CUDA/AllocatorCUF.cpp
@@ -8,11 +8,11 @@
 
 #include "gtest/gtest.h"
 #include "../../../runtime/terminator.h"
-#include "flang/Common/Fortran.h"
 #include "flang/Runtime/CUDA/allocator.h"
 #include "flang/Runtime/CUDA/descriptor.h"
 #include "flang/Runtime/allocatable.h"
 #include "flang/Runtime/allocator-registry.h"
+#include "flang/Support/Fortran.h"
 
 #include "cuda_runtime.h"
 
diff --git a/flang/unittests/Runtime/CUDA/CMakeLists.txt b/flang/unittests/Runtime/CUDA/CMakeLists.txt
index a7fe604d687bd..860b2664d623b 100644
--- a/flang/unittests/Runtime/CUDA/CMakeLists.txt
+++ b/flang/unittests/Runtime/CUDA/CMakeLists.txt
@@ -15,8 +15,8 @@ endif()
 target_link_libraries(FlangCufRuntimeTests
   PRIVATE
   ${CUDA_RT_TARGET}
-  CufRuntime_cuda_${CUDAToolkit_VERSION_MAJOR}
-  FortranRuntime
+  flang_rt.cuda_${CUDAToolkit_VERSION_MAJOR}
+  flang_rt.runtime
 )
 
 target_include_directories(FlangCufRuntimeTests PRIVATE ${CUDAToolkit_INCLUDE_DIRS})
diff --git a/flang/unittests/Runtime/CUDA/Memory.cpp b/flang/unittests/Runtime/CUDA/Memory.cpp
index 7c8b7aa5a4d78..61c9d1131ee56 100644
--- a/flang/unittests/Runtime/CUDA/Memory.cpp
+++ b/flang/unittests/Runtime/CUDA/Memory.cpp
@@ -10,12 +10,12 @@
 #include "gtest/gtest.h"
 #include "../../../runtime/terminator.h"
 #include "../tools.h"
-#include "flang/Common/Fortran.h"
 #include "flang/Runtime/CUDA/allocator.h"
 #include "flang/Runtime/CUDA/common.h"
 #include "flang/Runtime/CUDA/descriptor.h"
 #include "flang/Runtime/allocatable.h"
 #include "flang/Runtime/allocator-registry.h"
+#include "flang/Support/Fortran.h"
 
 #include "cuda_runtime.h"
 
diff --git a/flang/unittests/Runtime/TemporaryStack.cpp b/flang/unittests/Runtime/TemporaryStack.cpp
index 0a9344969ca6b..4f701e09b2945 100644
--- a/flang/unittests/Runtime/TemporaryStack.cpp
+++ b/flang/unittests/Runtime/TemporaryStack.cpp
@@ -8,7 +8,7 @@
 
 #include "gtest/gtest.h"
 #include "tools.h"
-#include "flang/ISO_Fortran_binding_wrapper.h"
+#include "flang/Common/ISO_Fortran_binding_wrapper.h"
 #include "flang/Runtime/allocatable.h"
 #include "flang/Runtime/cpp-type.h"
 #include "flang/Runtime/descriptor.h"
diff --git a/libc/CMakeLists.txt b/libc/CMakeLists.txt
index e5ac842edf56e..1c4c0cd5aa22b 100644
--- a/libc/CMakeLists.txt
+++ b/libc/CMakeLists.txt
@@ -51,7 +51,8 @@ set(LIBC_KERNEL_HEADERS "/usr/include" CACHE STRING "Path to Linux kernel header
 # Defining a global namespace to enclose all libc functions.
 set(default_namespace "__llvm_libc")
 if(LLVM_VERSION_MAJOR)
-  set(default_namespace "__llvm_libc_${LLVM_VERSION_MAJOR}_${LLVM_VERSION_MINOR}_${LLVM_VERSION_PATCH}_${LLVM_VERSION_SUFFIX}")
+  string(REPLACE "-" "" NS_LLVM_VERSION_SUFFIX ${LLVM_VERSION_SUFFIX})
+  set(default_namespace "__llvm_libc_${LLVM_VERSION_MAJOR}_${LLVM_VERSION_MINOR}_${LLVM_VERSION_PATCH}_${NS_LLVM_VERSION_SUFFIX}")
 endif()
 set(LIBC_NAMESPACE ${default_namespace}
   CACHE STRING "The namespace to use to enclose internal implementations. Must start with '__llvm_libc'."
@@ -262,6 +263,7 @@ if(LIBC_TARGET_OS_IS_GPU)
 endif()
 
 include(LLVMLibCCheckMPFR)
+include(LLVMLibCCheckMPC)
 
 if(LLVM_LIBC_CLANG_TIDY)
   set(LLVM_LIBC_ENABLE_LINTING ON)
diff --git a/libc/cmake/caches/gpu.cmake b/libc/cmake/caches/gpu.cmake
new file mode 100644
index 0000000000000..1867db9ffa12e
--- /dev/null
+++ b/libc/cmake/caches/gpu.cmake
@@ -0,0 +1,4 @@
+set(LLVM_ENABLE_PROJECTS "clang;clang-tools-extra;lld" CACHE STRING "")
+set(LLVM_RUNTIME_TARGETS default;amdgcn-amd-amdhsa;nvptx64-nvidia-cuda CACHE STRING "") 
+set(RUNTIMES_nvptx64-nvidia-cuda_LLVM_ENABLE_RUNTIMES "compiler-rt;libc" CACHE STRING "")
+set(RUNTIMES_amdgcn-amd-amdhsa_LLVM_ENABLE_RUNTIMES "compiler-rt;libc" CACHE STRING "")
diff --git a/libc/cmake/modules/LLVMLibCCheckMPC.cmake b/libc/cmake/modules/LLVMLibCCheckMPC.cmake
new file mode 100644
index 0000000000000..6cc9ee2d9a6f6
--- /dev/null
+++ b/libc/cmake/modules/LLVMLibCCheckMPC.cmake
@@ -0,0 +1,22 @@
+if(LIBC_TESTS_CAN_USE_MPFR)
+  set(LIBC_MPC_INSTALL_PATH "" CACHE PATH "Path to where MPC is installed (e.g. C:/src/install or ~/src/install)")
+
+  if(LIBC_MPC_INSTALL_PATH)
+    set(LIBC_TESTS_CAN_USE_MPC TRUE)
+  elseif(LIBC_TARGET_OS_IS_GPU OR LLVM_LIBC_FULL_BUILD)
+    # In full build mode, the MPC library should be built using our own facilities,
+    # which is currently not possible.
+    set(LIBC_TESTS_CAN_USE_MPC FALSE)
+  else()
+    try_compile(
+      LIBC_TESTS_CAN_USE_MPC
+      ${CMAKE_CURRENT_BINARY_DIR}
+      SOURCES
+        ${LIBC_SOURCE_DIR}/utils/MPCWrapper/check_mpc.cpp
+      COMPILE_DEFINITIONS
+        ${LIBC_COMPILE_OPTIONS_DEFAULT}
+      LINK_LIBRARIES
+        -lmpc -lmpfr -lgmp -latomic
+    )
+  endif()
+endif()
diff --git a/libc/cmake/modules/LLVMLibCCompileOptionRules.cmake b/libc/cmake/modules/LLVMLibCCompileOptionRules.cmake
index 12420db331961..0facb0b9be0c1 100644
--- a/libc/cmake/modules/LLVMLibCCompileOptionRules.cmake
+++ b/libc/cmake/modules/LLVMLibCCompileOptionRules.cmake
@@ -179,8 +179,9 @@ function(_get_common_compile_options output_var flags)
     endif()
     list(APPEND compile_options "-Wconversion")
     list(APPEND compile_options "-Wno-sign-conversion")
-    # Silence this warning because _Complex is a part of C99.
+    list(APPEND compile_options "-Wdeprecated")
     if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
+      # Silence this warning because _Complex is a part of C99.
       list(APPEND compile_options "-fext-numeric-literals")
     else()
       list(APPEND compile_options "-Wno-c99-extensions")
diff --git a/libc/cmake/modules/LLVMLibCFlagRules.cmake b/libc/cmake/modules/LLVMLibCFlagRules.cmake
index e5b0e249c9067..7d5e73c2f1214 100644
--- a/libc/cmake/modules/LLVMLibCFlagRules.cmake
+++ b/libc/cmake/modules/LLVMLibCFlagRules.cmake
@@ -268,23 +268,29 @@ set(EXPLICIT_SIMD_OPT_FLAG "EXPLICIT_SIMD_OPT")
 set(MISC_MATH_BASIC_OPS_OPT_FLAG "MISC_MATH_BASIC_OPS_OPT")
 
 # Skip FMA_OPT flag for targets that don't support fma.
-if(NOT((LIBC_TARGET_ARCHITECTURE_IS_X86_64 AND (LIBC_CPU_FEATURES MATCHES "FMA")) OR
-       LIBC_TARGET_ARCHITECTURE_IS_RISCV64))
-  set(SKIP_FLAG_EXPANSION_FMA_OPT TRUE)
+if(NOT DEFINED SKIP_FLAG_EXPANSION_FMA_OPT)
+  if(NOT((LIBC_TARGET_ARCHITECTURE_IS_X86_64 AND (LIBC_CPU_FEATURES MATCHES "FMA")) OR
+        LIBC_TARGET_ARCHITECTURE_IS_RISCV64))
+    set(SKIP_FLAG_EXPANSION_FMA_OPT TRUE)
+  endif()
 endif()
 
 # Skip EXPLICIT_SIMD_OPT flag for targets that don't support SSE2.
 # Note: one may want to revisit it if they want to control other explicit SIMD
-if(NOT(LIBC_TARGET_ARCHITECTURE_IS_X86_64 AND (LIBC_CPU_FEATURES MATCHES "SSE2")))
-  set(SKIP_FLAG_EXPANSION_EXPLICIT_SIMD_OPT TRUE)
+if(NOT DEFINED SKIP_FLAG_EXPANSION_EXPLICIT_SIMD_OPT)
+  if(NOT(LIBC_TARGET_ARCHITECTURE_IS_X86_64 AND (LIBC_CPU_FEATURES MATCHES "SSE2")))
+    set(SKIP_FLAG_EXPANSION_EXPLICIT_SIMD_OPT TRUE)
+  endif()
 endif()
 
 # Skip ROUND_OPT flag for targets that don't support rounding instructions. On
 # x86, these are SSE4.1 instructions, but we already had code to check for
 # SSE4.2 support.
-if(NOT((LIBC_TARGET_ARCHITECTURE_IS_X86_64 AND (LIBC_CPU_FEATURES MATCHES "SSE4_2")) OR
-       LIBC_TARGET_ARCHITECTURE_IS_AARCH64 OR LIBC_TARGET_OS_IS_GPU))
-  set(SKIP_FLAG_EXPANSION_ROUND_OPT TRUE)
+if(NOT DEFINED SKIP_FLAG_EXPANSION_ROUND_OPT)
+  if(NOT((LIBC_TARGET_ARCHITECTURE_IS_X86_64 AND (LIBC_CPU_FEATURES MATCHES "SSE4_2")) OR
+        LIBC_TARGET_ARCHITECTURE_IS_AARCH64 OR LIBC_TARGET_OS_IS_GPU))
+    set(SKIP_FLAG_EXPANSION_ROUND_OPT TRUE)
+  endif()
 endif()
 
 # Choose whether time_t is 32- or 64-bit, based on target architecture
diff --git a/libc/cmake/modules/LLVMLibCTestRules.cmake b/libc/cmake/modules/LLVMLibCTestRules.cmake
index 10bb9c9487d63..f33db5826537b 100644
--- a/libc/cmake/modules/LLVMLibCTestRules.cmake
+++ b/libc/cmake/modules/LLVMLibCTestRules.cmake
@@ -38,9 +38,8 @@ function(_get_common_test_compile_options output_var c_test flags)
     endif()
     # list(APPEND compile_options "-Wconversion")
     # list(APPEND compile_options "-Wno-sign-conversion")
-    # list(APPEND compile_options "-Wimplicit-fallthrough")
-    # list(APPEND compile_options "-Wwrite-strings")
-    list(APPEND compile_options "-Wextra-semi")
+    list(APPEND compile_options "-Wimplicit-fallthrough")
+    list(APPEND compile_options "-Wwrite-strings")
     # Silence this warning because _Complex is a part of C99.
     if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
       if(NOT c_test)
@@ -51,13 +50,14 @@ function(_get_common_test_compile_options output_var c_test flags)
       list(APPEND compile_options "-Wno-gnu-imaginary-constant")
     endif()
     list(APPEND compile_options "-Wno-pedantic")
-    # if(NOT CMAKE_COMPILER_IS_GNUCXX)
-    #   list(APPEND compile_options "-Wnewline-eof")
-    #   list(APPEND compile_options "-Wnonportable-system-include-path")
-    #   list(APPEND compile_options "-Wstrict-prototypes")
-    #   list(APPEND compile_options "-Wthread-safety")
-    #   list(APPEND compile_options "-Wglobal-constructors")
-    # endif()
+    if(CMAKE_CXX_COMPILER_ID STREQUAL "Clang")
+      list(APPEND compile_options "-Wstrict-prototypes")
+      list(APPEND compile_options "-Wextra-semi")
+      list(APPEND compile_options "-Wnewline-eof")
+      list(APPEND compile_options "-Wnonportable-system-include-path")
+      list(APPEND compile_options "-Wthread-safety")
+      # list(APPEND compile_options "-Wglobal-constructors")
+    endif()
   endif()
   set(${output_var} ${compile_options} PARENT_SCOPE)
 endfunction()
@@ -223,6 +223,8 @@ function(create_libc_unittest fq_target_name)
 
   _get_common_test_compile_options(compile_options "${LIBC_UNITTEST_C_TEST}"
                                    "${LIBC_UNITTEST_FLAGS}")
+  # TODO: Ideally we would have a separate function for link options.
+  set(link_options ${compile_options})
   list(APPEND compile_options ${LIBC_UNITTEST_COMPILE_OPTIONS})
 
   if(SHOW_INTERMEDIATE_OBJECTS)
@@ -277,7 +279,7 @@ function(create_libc_unittest fq_target_name)
   target_include_directories(${fq_build_target_name} SYSTEM PRIVATE ${LIBC_INCLUDE_DIR})
   target_include_directories(${fq_build_target_name} PRIVATE ${LIBC_SOURCE_DIR})
   target_compile_options(${fq_build_target_name} PRIVATE ${compile_options})
-  target_link_options(${fq_build_target_name} PRIVATE ${compile_options})
+  target_link_options(${fq_build_target_name} PRIVATE ${link_options})
 
   if(NOT LIBC_UNITTEST_CXX_STANDARD)
     set(LIBC_UNITTEST_CXX_STANDARD ${CMAKE_CXX_STANDARD})
@@ -558,14 +560,12 @@ function(add_integration_test test_name)
   if(LIBC_TARGET_ARCHITECTURE_IS_AMDGPU)
     target_link_options(${fq_build_target_name} PRIVATE
       ${LIBC_COMPILE_OPTIONS_DEFAULT} ${INTEGRATION_TEST_COMPILE_OPTIONS}
-      -Wno-multi-gpu -mcpu=${LIBC_GPU_TARGET_ARCHITECTURE} -flto
-      "-Wl,-mllvm,-amdgpu-lower-global-ctor-dtor=0" -nostdlib -static
+      -Wno-multi-gpu -mcpu=${LIBC_GPU_TARGET_ARCHITECTURE} -flto -nostdlib -static
       "-Wl,-mllvm,-amdhsa-code-object-version=${LIBC_GPU_CODE_OBJECT_VERSION}")
   elseif(LIBC_TARGET_ARCHITECTURE_IS_NVPTX)
     target_link_options(${fq_build_target_name} PRIVATE
       ${LIBC_COMPILE_OPTIONS_DEFAULT} ${INTEGRATION_TEST_COMPILE_OPTIONS}
       "-Wl,--suppress-stack-size-warning" -Wno-multi-gpu
-      "-Wl,-mllvm,-nvptx-lower-global-ctor-dtor=1"
       "-Wl,-mllvm,-nvptx-emit-init-fini-kernel"
       -march=${LIBC_GPU_TARGET_ARCHITECTURE} -nostdlib -static
       "--cuda-path=${LIBC_CUDA_ROOT}")
diff --git a/libc/config/baremetal/config.json b/libc/config/baremetal/config.json
index 08c581d1c6822..105e417d165f0 100644
--- a/libc/config/baremetal/config.json
+++ b/libc/config/baremetal/config.json
@@ -5,6 +5,9 @@
     }
   },
   "printf": {
+    "LIBC_CONF_PRINTF_DISABLE_FIXED_POINT": {
+      "value": true
+    },
     "LIBC_CONF_PRINTF_DISABLE_FLOAT": {
       "value": true
     },
diff --git a/libc/config/config.json b/libc/config/config.json
index 9a5d5c3c68da6..c38d424229218 100644
--- a/libc/config/config.json
+++ b/libc/config/config.json
@@ -30,6 +30,10 @@
       "value": false,
       "doc": "Use the same mode for double and long double in printf."
     },
+    "LIBC_CONF_PRINTF_FLOAT_TO_STR_USE_FLOAT320": {
+      "value": false,
+      "doc": "Use an alternative printf float implementation based on 320-bit floats"
+    },
     "LIBC_CONF_PRINTF_DISABLE_FIXED_POINT": {
       "value": false,
       "doc": "Disable printing fixed point values in printf and friends."
diff --git a/libc/config/linux/aarch64/entrypoints.txt b/libc/config/linux/aarch64/entrypoints.txt
index 09eb51a3f8fc6..efb5e1c7bd698 100644
--- a/libc/config/linux/aarch64/entrypoints.txt
+++ b/libc/config/linux/aarch64/entrypoints.txt
@@ -32,6 +32,9 @@ set(TARGET_LIBC_ENTRYPOINTS
     libc.src.fcntl.open
     libc.src.fcntl.openat
 
+    # poll.h entrypoints
+    libc.src.poll.poll
+
     # sched.h entrypoints
     libc.src.sched.sched_get_priority_max
     libc.src.sched.sched_get_priority_min
@@ -341,6 +344,7 @@ set(TARGET_LIBC_ENTRYPOINTS
     libc.src.unistd.readlink
     libc.src.unistd.readlinkat
     libc.src.unistd.rmdir
+    libc.src.unistd.setsid
     libc.src.unistd.symlink
     libc.src.unistd.symlinkat
     libc.src.unistd.sysconf
@@ -355,6 +359,7 @@ set(TARGET_LIBC_ENTRYPOINTS
 
     # sys/uio.h entrypoints
     libc.src.sys.uio.writev
+    libc.src.sys.uio.readv
 )
 
 if(LLVM_LIBC_INCLUDE_SCUDO)
diff --git a/libc/config/linux/aarch64/headers.txt b/libc/config/linux/aarch64/headers.txt
index 05f15a0e4e5cb..54c65765abe4a 100644
--- a/libc/config/linux/aarch64/headers.txt
+++ b/libc/config/linux/aarch64/headers.txt
@@ -4,34 +4,37 @@ set(TARGET_PUBLIC_HEADERS
     libc.include.ctype
     libc.include.dlfcn
     libc.include.elf
+    libc.include.endian
     libc.include.errno
     libc.include.features
     libc.include.fenv
     libc.include.float
-    libc.include.stdint
     libc.include.inttypes
     libc.include.limits
     libc.include.link
     libc.include.malloc
     libc.include.math
+    libc.include.poll
     libc.include.pthread
+    libc.include.search
     libc.include.signal
-    libc.include.stdckdint
     libc.include.stdbit
+    libc.include.stdckdint
+    libc.include.stdint
     libc.include.stdio
     libc.include.stdlib
     libc.include.string
     libc.include.strings
-    libc.include.search
     libc.include.sys_mman
     libc.include.sys_socket
     libc.include.sys_syscall
     libc.include.sys_time
+    libc.include.sys_wait
     libc.include.threads
     libc.include.time
+    libc.include.uchar
     libc.include.unistd
     libc.include.wchar
-    libc.include.uchar
 
     libc.include.sys_ioctl
     # Disabled due to epoll_wait syscalls not being available on this platform.
diff --git a/libc/config/linux/arm/entrypoints.txt b/libc/config/linux/arm/entrypoints.txt
index 42ea803baac04..170525c5529a0 100644
--- a/libc/config/linux/arm/entrypoints.txt
+++ b/libc/config/linux/arm/entrypoints.txt
@@ -20,6 +20,9 @@ set(TARGET_LIBC_ENTRYPOINTS
     # errno.h entrypoints
     libc.src.errno.errno
 
+    # poll.h entrypoints
+    libc.src.poll.poll
+
     # string.h entrypoints
     libc.src.string.memccpy
     libc.src.string.memchr
diff --git a/libc/config/linux/riscv/entrypoints.txt b/libc/config/linux/riscv/entrypoints.txt
index 14a05a2f3fbf2..6e67ea559d57b 100644
--- a/libc/config/linux/riscv/entrypoints.txt
+++ b/libc/config/linux/riscv/entrypoints.txt
@@ -32,6 +32,10 @@ set(TARGET_LIBC_ENTRYPOINTS
     libc.src.fcntl.open
     libc.src.fcntl.openat
 
+    # poll.h entrypoints
+    # TODO: https://github.com/llvm/llvm-project/issues/125940
+    # libc.src.poll.poll
+
     # sched.h entrypoints
     libc.src.sched.sched_get_priority_max
     libc.src.sched.sched_get_priority_min
@@ -337,6 +341,7 @@ set(TARGET_LIBC_ENTRYPOINTS
     libc.src.unistd.readlink
     libc.src.unistd.readlinkat
     libc.src.unistd.rmdir
+    libc.src.unistd.setsid
     libc.src.unistd.symlink
     libc.src.unistd.symlinkat
     libc.src.unistd.sysconf
diff --git a/libc/config/linux/x86_64/entrypoints.txt b/libc/config/linux/x86_64/entrypoints.txt
index 366e4d34294d1..0a942516db6c3 100644
--- a/libc/config/linux/x86_64/entrypoints.txt
+++ b/libc/config/linux/x86_64/entrypoints.txt
@@ -32,6 +32,9 @@ set(TARGET_LIBC_ENTRYPOINTS
     libc.src.fcntl.open
     libc.src.fcntl.openat
 
+    # poll.h entrypoints
+    libc.src.poll.poll
+
     # sched.h entrypoints
     libc.src.sched.sched_get_priority_max
     libc.src.sched.sched_get_priority_min
@@ -340,6 +343,7 @@ set(TARGET_LIBC_ENTRYPOINTS
     libc.src.unistd.readlink
     libc.src.unistd.readlinkat
     libc.src.unistd.rmdir
+    libc.src.unistd.setsid
     libc.src.unistd.symlink
     libc.src.unistd.symlinkat
     libc.src.unistd.sysconf
@@ -355,6 +359,7 @@ set(TARGET_LIBC_ENTRYPOINTS
 
     # sys/uio.h entrypoints
     libc.src.sys.uio.writev
+    libc.src.sys.uio.readv
 )
 
 if(LLVM_LIBC_INCLUDE_SCUDO)
@@ -642,6 +647,7 @@ endif()
 if(LIBC_TYPES_HAS_FLOAT16)
   list(APPEND TARGET_LIBM_ENTRYPOINTS
     # math.h C23 _Float16 entrypoints
+    libc.src.math.asinf16
     libc.src.math.canonicalizef16
     libc.src.math.ceilf16
     libc.src.math.copysignf16
diff --git a/libc/config/linux/x86_64/headers.txt b/libc/config/linux/x86_64/headers.txt
index 8750100302ea7..bd2fc3df75964 100644
--- a/libc/config/linux/x86_64/headers.txt
+++ b/libc/config/linux/x86_64/headers.txt
@@ -1,44 +1,38 @@
 set(TARGET_PUBLIC_HEADERS
+    libc.include.arpa_inet
     libc.include.assert
     libc.include.complex
     libc.include.ctype
     libc.include.dirent
     libc.include.dlfcn
     libc.include.elf
+    libc.include.endian
     libc.include.errno
     libc.include.fcntl
     libc.include.features
     libc.include.fenv
     libc.include.float
-    libc.include.stdint
     libc.include.inttypes
     libc.include.limits
     libc.include.link
+    libc.include.locale
     libc.include.malloc
     libc.include.math
+    libc.include.poll
     libc.include.pthread
     libc.include.sched
+    libc.include.search
+    libc.include.setjmp
     libc.include.signal
     libc.include.spawn
-    libc.include.setjmp
-    libc.include.stdckdint
     libc.include.stdbit
+    libc.include.stdckdint
     libc.include.stdfix
+    libc.include.stdint
     libc.include.stdio
     libc.include.stdlib
     libc.include.string
     libc.include.strings
-    libc.include.search
-    libc.include.termios
-    libc.include.threads
-    libc.include.time
-    libc.include.unistd
-    libc.include.wchar
-    libc.include.uchar
-    libc.include.locale
-
-    libc.include.arpa_inet
-
     libc.include.sys_auxv
     libc.include.sys_epoll
     libc.include.sys_ioctl
@@ -56,4 +50,10 @@ set(TARGET_PUBLIC_HEADERS
     libc.include.sys_types
     libc.include.sys_utsname
     libc.include.sys_wait
+    libc.include.termios
+    libc.include.threads
+    libc.include.time
+    libc.include.uchar
+    libc.include.unistd
+    libc.include.wchar
 )
diff --git a/libc/docs/CMakeLists.txt b/libc/docs/CMakeLists.txt
index bb8e3e96e47ca..4ef0e920de683 100644
--- a/libc/docs/CMakeLists.txt
+++ b/libc/docs/CMakeLists.txt
@@ -37,7 +37,9 @@ if (SPHINX_FOUND)
       aio
       arpa/inet
       assert
+      cpio
       ctype
+      endian
       errno
       fenv
       float
@@ -57,6 +59,7 @@ if (SPHINX_FOUND)
       sys/mman
       sys/resource
       sys/stat
+      sys/statvfs
       sys/time
       sys/wait
       termios
diff --git a/libc/docs/configure.rst b/libc/docs/configure.rst
index 3db750b1aed21..940a07754c458 100644
--- a/libc/docs/configure.rst
+++ b/libc/docs/configure.rst
@@ -43,6 +43,7 @@ to learn about the defaults for your platform and target.
     - ``LIBC_CONF_PRINTF_DISABLE_WRITE_INT``: Disable handling of %n in printf format string.
     - ``LIBC_CONF_PRINTF_FLOAT_TO_STR_NO_SPECIALIZE_LD``: Use the same mode for double and long double in printf.
     - ``LIBC_CONF_PRINTF_FLOAT_TO_STR_USE_DYADIC_FLOAT``: Use dyadic float for faster and smaller but less accurate printf doubles.
+    - ``LIBC_CONF_PRINTF_FLOAT_TO_STR_USE_FLOAT320``: Use an alternative printf float implementation based on 320-bit floats
     - ``LIBC_CONF_PRINTF_FLOAT_TO_STR_USE_MEGA_LONG_DOUBLE_TABLE``: Use large table for better printf long double performance.
 * **"pthread" options**
     - ``LIBC_CONF_RAW_MUTEX_DEFAULT_SPIN_COUNT``: Default number of spins before blocking if a mutex is in contention (default to 100).
diff --git a/libc/docs/gpu/building.rst b/libc/docs/gpu/building.rst
index 88643575ae4d9..94d3f1f644e5c 100644
--- a/libc/docs/gpu/building.rst
+++ b/libc/docs/gpu/building.rst
@@ -58,7 +58,8 @@ OpenMP support. We then set ``RUNTIMES_<triple>_LLVM_ENABLE_RUNTIMES`` to enable
 ``libc`` for the GPU targets. The ``LLVM_RUNTIME_TARGETS`` sets the enabled
 targets to build, in this case we want the default target and the GPU targets.
 Note that if ``libc`` were included in ``LLVM_ENABLE_RUNTIMES`` it would build
-targeting the default host environment as well.
+targeting the default host environment as well. Alternatively, you can point
+your build towards the ``libc/cmake/caches/gpu.cmake`` cache file with ``-C``.
 
 Runtimes cross build
 --------------------
diff --git a/libc/docs/gpu/rpc.rst b/libc/docs/gpu/rpc.rst
index 3f312bb9281b3..dde4207df6e49 100644
--- a/libc/docs/gpu/rpc.rst
+++ b/libc/docs/gpu/rpc.rst
@@ -231,8 +231,11 @@ but the following example shows how it can be used by a standard user.
     do {
       auto port = server.try_open(warp_size, /*index=*/0);
       // From libllvmlibc_rpc_server.a in the installation.
-      if (port)
-        handle_libc_opcodes(*port, warp_size);
+      if (!port)
+        continue;
+
+      handle_libc_opcodes(*port, warp_size);
+      port->close();
     } while (cudaStreamQuery(stream) == cudaErrorNotReady);
   }
 
diff --git a/libc/docs/headers/index.rst b/libc/docs/headers/index.rst
index d08552d223252..3dc30ef90a8e4 100644
--- a/libc/docs/headers/index.rst
+++ b/libc/docs/headers/index.rst
@@ -8,7 +8,9 @@ Implementation Status
    arpa/inet
    assert
    complex
+   cpio
    ctype
+   endian
    errno
    fenv
    float
@@ -28,6 +30,7 @@ Implementation Status
    sys/mman
    sys/resource
    sys/stat
+   sys/statvfs
    sys/time
    sys/wait
    termios
diff --git a/libc/docs/headers/math/index.rst b/libc/docs/headers/math/index.rst
index 8548e4a5773bc..3e45e3e618abb 100644
--- a/libc/docs/headers/math/index.rst
+++ b/libc/docs/headers/math/index.rst
@@ -256,7 +256,7 @@ Higher Math Functions
 +-----------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+----------------------------+
 | acospi    |                  |                 |                        |                      |                        | 7.12.4.8               | F.10.1.8                   |
 +-----------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+----------------------------+
-| asin      | |check|          |                 |                        |                      |                        | 7.12.4.2               | F.10.1.2                   |
+| asin      | |check|          |                 |                        | |check|              |                        | 7.12.4.2               | F.10.1.2                   |
 +-----------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+----------------------------+
 | asinh     | |check|          |                 |                        |                      |                        | 7.12.5.2               | F.10.2.2                   |
 +-----------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+----------------------------+
diff --git a/libc/hdr/types/CMakeLists.txt b/libc/hdr/types/CMakeLists.txt
index 3dfa38a020fad..84a2647ba664d 100644
--- a/libc/hdr/types/CMakeLists.txt
+++ b/libc/hdr/types/CMakeLists.txt
@@ -250,15 +250,6 @@ add_proxy_header_library(
     libc.include.locale
 )
 
-add_proxy_header_library(
-  sighandler_t
-  HDRS
-    sighandler_t.h
-  FULL_BUILD_DEPENDS
-    libc.include.llvm-libc-types.__sighandler_t
-    libc.include.signal
-)
-
 add_proxy_header_library(
   stack_t
   HDRS
@@ -342,3 +333,19 @@ add_proxy_header_library(
     libc.include.llvm-libc-types.struct_iovec
     libc.include.sys_uio
 )
+
+add_proxy_header_library(
+  nfds_t
+  HDRS
+    nfds_t.h
+  FULL_BUILD_DEPENDS
+    libc.include.llvm-libc-types.nfds_t
+)
+
+add_proxy_header_library(
+  struct_pollfd
+  HDRS
+    struct_pollfd.h
+  FULL_BUILD_DEPENDS
+    libc.include.llvm-libc-types.struct_pollfd
+)
diff --git a/libc/hdr/types/nfds_t.h b/libc/hdr/types/nfds_t.h
new file mode 100644
index 0000000000000..9143564c2333e
--- /dev/null
+++ b/libc/hdr/types/nfds_t.h
@@ -0,0 +1,23 @@
+//===-- Definition of nfds_t ----------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_HDR_TYPES_NFDS_T_H
+#define LLVM_LIBC_HDR_TYPES_NFDS_T_H
+
+#ifdef LIBC_FULL_BUILD
+
+#include "include/llvm-libc-types/nfds_t.h"
+
+#else // overlay mode
+
+#include <poll.h>
+
+#endif // LLVM_LIBC_FULL_BUILD
+
+#endif // LLVM_LIBC_HDR_TYPES_NFDS_T_H
diff --git a/libc/hdr/types/sighandler_t.h b/libc/hdr/types/sighandler_t.h
deleted file mode 100644
index bc40dd8b4c8f4..0000000000000
--- a/libc/hdr/types/sighandler_t.h
+++ /dev/null
@@ -1,24 +0,0 @@
-//===-- Definition of macros from __sighandler_t.h ------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LLVM_LIBC_HDR_TYPES_SIGHANDLER_T_H
-#define LLVM_LIBC_HDR_TYPES_SIGHANDLER_T_H
-
-#ifdef LIBC_FULL_BUILD
-
-#include "include/llvm-libc-types/__sighandler_t.h"
-
-using sighandler_t = __sighandler_t;
-
-#else // overlay mode
-
-#include <signal.h>
-
-#endif // LLVM_LIBC_FULL_BUILD
-
-#endif // LLVM_LIBC_HDR_TYPES_SIGHANDLER_T_H
diff --git a/libc/hdr/types/struct_pollfd.h b/libc/hdr/types/struct_pollfd.h
new file mode 100644
index 0000000000000..efec6fc80ac10
--- /dev/null
+++ b/libc/hdr/types/struct_pollfd.h
@@ -0,0 +1,23 @@
+//===-- Definition of struct pollfd ---------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_HDR_TYPES_STRUCT_POLLFD_H
+#define LLVM_LIBC_HDR_TYPES_STRUCT_POLLFD_H
+
+#ifdef LIBC_FULL_BUILD
+
+#include "include/llvm-libc-types/struct_pollfd.h"
+
+#else // overlay mode
+
+#include <poll.h>
+
+#endif // LLVM_LIBC_FULL_BUILD
+
+#endif // LLVM_LIBC_HDR_TYPES_STRUCT_POLLFD_H
diff --git a/libc/include/CMakeLists.txt b/libc/include/CMakeLists.txt
index e5ceea360d396..867bd1e5ee20f 100644
--- a/libc/include/CMakeLists.txt
+++ b/libc/include/CMakeLists.txt
@@ -73,6 +73,15 @@ add_header_macro(
     .llvm_libc_common_h
 )
 
+add_header_macro(
+  endian
+  ../libc/include/endian.yaml
+  endian.h
+  DEPENDS
+    .llvm-libc-macros.endian_macros
+    .llvm_libc_common_h
+)
+
 add_header_macro(
   features
   ../libc/include/features.yaml
@@ -275,13 +284,14 @@ add_header_macro(
   signal.h
   DEPENDS
     .llvm-libc-macros.signal_macros
+    .llvm-libc-types.pid_t
     .llvm-libc-types.sig_atomic_t
+    .llvm-libc-types.sighandler_t
+    .llvm-libc-types.siginfo_t
     .llvm-libc-types.sigset_t
+    .llvm-libc-types.stack_t
     .llvm-libc-types.struct_sigaction
     .llvm-libc-types.union_sigval
-    .llvm-libc-types.siginfo_t
-    .llvm-libc-types.stack_t
-    .llvm-libc-types.pid_t
 )
 
 add_header_macro(
@@ -704,6 +714,16 @@ add_header_macro(
     .llvm-libc-types.struct_lconv
 )
 
+add_header_macro(
+  poll
+  ../libc/include/poll.yaml
+  poll.h
+  DEPENDS
+    .llvm-libc-types.struct_pollfd
+    .llvm-libc-types.nfds_t
+    .llvm-libc-macros.poll-macros
+  )
+
 if(NOT LLVM_LIBC_FULL_BUILD)
   # We don't install headers in non-fullbuild mode.
   return()
diff --git a/libc/include/__llvm-libc-common.h b/libc/include/__llvm-libc-common.h
index b5a23c5765f4d..d3f8a4e63268a 100644
--- a/libc/include/__llvm-libc-common.h
+++ b/libc/include/__llvm-libc-common.h
@@ -80,6 +80,9 @@
 #define __NOEXCEPT
 #endif
 
+#undef _Returns_twice
+#define _Returns_twice __attribute__((returns_twice))
+
 #endif // __cplusplus
 
 #endif // LLVM_LIBC_COMMON_H
diff --git a/libc/include/endian.h.def b/libc/include/endian.h.def
new file mode 100644
index 0000000000000..f331f08edc689
--- /dev/null
+++ b/libc/include/endian.h.def
@@ -0,0 +1,17 @@
+//===-- POSIX header endian.h ---------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_ENDIAN_H
+#define LLVM_LIBC_ENDIAN_H
+
+#include "__llvm-libc-common.h"
+#include "llvm-libc-macros/endian-macros.h"
+
+%%public_api()
+
+#endif // LLVM_LIBC_ENDIAN_H
diff --git a/libc/include/endian.yaml b/libc/include/endian.yaml
new file mode 100644
index 0000000000000..6497f897b6393
--- /dev/null
+++ b/libc/include/endian.yaml
@@ -0,0 +1,9 @@
+header: endian.h
+header_template: endian.h.def
+standards:
+  - POSIX
+macros: []
+types: []
+enums: []
+objects: []
+functions: []
diff --git a/libc/include/llvm-libc-macros/CMakeLists.txt b/libc/include/llvm-libc-macros/CMakeLists.txt
index 9d5d9f6544288..ea892a87dbe7a 100644
--- a/libc/include/llvm-libc-macros/CMakeLists.txt
+++ b/libc/include/llvm-libc-macros/CMakeLists.txt
@@ -310,6 +310,12 @@ add_macro_header(
     elf-macros.h
 )
 
+add_macro_header(
+  endian_macros
+  HDR
+    endian-macros.h
+)
+
 add_macro_header(
   locale_macros
   HDR
@@ -321,3 +327,9 @@ add_macro_header(
   HDR
     pthread-macros.h
 )
+
+add_macro_header(
+  poll-macros
+  HDR
+    poll-macros.h
+)
diff --git a/libc/include/llvm-libc-macros/endian-macros.h b/libc/include/llvm-libc-macros/endian-macros.h
new file mode 100644
index 0000000000000..94e1d60f8ff40
--- /dev/null
+++ b/libc/include/llvm-libc-macros/endian-macros.h
@@ -0,0 +1,16 @@
+//===-- Definition of macros from endian.h --------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_MACROS_ENDIAN_MACROS_H
+#define LLVM_LIBC_MACROS_ENDIAN_MACROS_H
+
+#define LITTLE_ENDIAN __ORDER_LITTLE_ENDIAN__
+#define BIG_ENDIAN __ORDER_BIG_ENDIAN__
+#define BYTE_ORDER __BYTE_ORDER__
+
+#endif // LLVM_LIBC_MACROS_ENDIAN_MACROS_H
diff --git a/libc/include/llvm-libc-macros/gpu/signal-macros.h b/libc/include/llvm-libc-macros/gpu/signal-macros.h
index 2d8159240de8b..f0d49ea34fe0e 100644
--- a/libc/include/llvm-libc-macros/gpu/signal-macros.h
+++ b/libc/include/llvm-libc-macros/gpu/signal-macros.h
@@ -16,9 +16,9 @@
 #define SIGSEGV 11
 #define SIGTERM 15
 
-#define SIG_DFL ((__sighandler_t)(0))
-#define SIG_IGN ((__sighandler_t)(1))
-#define SIG_ERR ((__sighandler_t)(-1))
+#define SIG_DFL ((void (*)(int))(0))
+#define SIG_IGN ((void (*)(int))(1))
+#define SIG_ERR ((void (*)(int))(-1))
 
 // Max signal number
 #define NSIG 64
diff --git a/libc/include/llvm-libc-macros/limits-macros.h b/libc/include/llvm-libc-macros/limits-macros.h
index d4aa7ae539e8a..79bbbe401eb02 100644
--- a/libc/include/llvm-libc-macros/limits-macros.h
+++ b/libc/include/llvm-libc-macros/limits-macros.h
@@ -235,4 +235,12 @@
 #define _POSIX_PATH_MAX 256
 #endif
 
+#ifndef _POSIX_ARG_MAX
+#define _POSIX_ARG_MAX 4096
+#endif
+
+#ifndef IOV_MAX
+#define IOV_MAX 1024
+#endif // IOV_MAX
+
 #endif // LLVM_LIBC_MACROS_LIMITS_MACROS_H
diff --git a/libc/include/llvm-libc-macros/linux/CMakeLists.txt b/libc/include/llvm-libc-macros/linux/CMakeLists.txt
index 461b190c02eac..61da55eabd40b 100644
--- a/libc/include/llvm-libc-macros/linux/CMakeLists.txt
+++ b/libc/include/llvm-libc-macros/linux/CMakeLists.txt
@@ -10,6 +10,12 @@ add_header(
     fcntl-macros.h
 )
 
+add_header(
+  poll-macros
+  HDR
+    poll-macros.h
+)
+
 add_header(
   sched_macros
   HDR
diff --git a/libc/include/llvm-libc-macros/linux/poll-macros.h b/libc/include/llvm-libc-macros/linux/poll-macros.h
new file mode 100644
index 0000000000000..d6724c5ee3211
--- /dev/null
+++ b/libc/include/llvm-libc-macros/linux/poll-macros.h
@@ -0,0 +1,65 @@
+//===-- Macros defined in poll.h header file ------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_MACROS_LINUX_POLL_MACROS_H
+#define LLVM_LIBC_MACROS_LINUX_POLL_MACROS_H
+
+// From asm-generic/poll.h, redefined here to avoid redeclaring struct pollfd.
+#ifndef POLLIN
+#define POLLIN 0x0001
+#endif
+
+#ifndef POLLPRI
+#define POLLPRI 0x0002
+#endif
+
+#ifndef POLLOUT
+#define POLLOUT 0x0004
+#endif
+
+#ifndef POLLERR
+#define POLLERR 0x0008
+#endif
+
+#ifndef POLLHUP
+#define POLLHUP 0x0010
+#endif
+
+#ifndef POLLNVAL
+#define POLLNVAL 0x0020
+#endif
+
+#ifndef POLLRDNORM
+#define POLLRDNORM 0x0040
+#endif
+
+#ifndef POLLRDBAND
+#define POLLRDBAND 0x0080
+#endif
+
+#ifndef POLLWRNORM
+#define POLLWRNORM 0x0100
+#endif
+
+#ifndef POLLWRBAND
+#define POLLWRBAND 0x0200
+#endif
+
+#ifndef POLLMSG
+#define POLLMSG 0x0400
+#endif
+
+#ifndef POLLREMOVE
+#define POLLREMOVE 0x1000
+#endif
+
+#ifndef POLLRDHUP
+#define POLLRDHUP 0x2000
+#endif
+
+#endif // LLVM_LIBC_MACROS_LINUX_POLL_MACROS_H
diff --git a/libc/include/llvm-libc-macros/linux/signal-macros.h b/libc/include/llvm-libc-macros/linux/signal-macros.h
index 0b7317ebc9b80..d220241a38206 100644
--- a/libc/include/llvm-libc-macros/linux/signal-macros.h
+++ b/libc/include/llvm-libc-macros/linux/signal-macros.h
@@ -86,9 +86,9 @@
 #error "Signal stack sizes not defined for your platform."
 #endif
 
-#define SIG_DFL ((__sighandler_t)0)
-#define SIG_IGN ((__sighandler_t)1)
-#define SIG_ERR ((__sighandler_t)-1)
+#define SIG_DFL ((void (*)(int))0)
+#define SIG_IGN ((void (*)(int))1)
+#define SIG_ERR ((void (*)(int))(-1))
 
 // SIGCHLD si_codes
 #define CLD_EXITED 1    // child has exited
diff --git a/libc/include/llvm-libc-macros/linux/sys-wait-macros.h b/libc/include/llvm-libc-macros/linux/sys-wait-macros.h
index c101638fdae34..d01cfa71ba390 100644
--- a/libc/include/llvm-libc-macros/linux/sys-wait-macros.h
+++ b/libc/include/llvm-libc-macros/linux/sys-wait-macros.h
@@ -9,36 +9,19 @@
 #ifndef LLVM_LIBC_MACROS_LINUX_SYS_WAIT_MACROS_H
 #define LLVM_LIBC_MACROS_LINUX_SYS_WAIT_MACROS_H
 
-// Wait flags
-#define WNOHANG 1    // Do not block
-#define WUNTRACED 2  // Report is a child has stopped even if untraced
-#define WEXITED 4    // Report dead child
-#define WCONTINUED 8 // Report if a stopped child has been resumed by SIGCONT
-#define WSTOPPED WUNTRACED
-
-// Wait status info macros
-#define __WEXITSTATUS(status) (((status)&0xff00) >> 8)
-#define __WTERMSIG(status) ((status)&0x7f)
-#define __WIFEXITED(status) (__WTERMSIG(status) == 0)
-
-// Macros for constructing status values.
-#define __W_EXITCODE(ret, sig) ((ret) << 8 | (sig))
-#define __W_STOPCODE(sig) ((sig) << 8 | 0x7f)
-#define __W_CONTINUED 0xffff
-#define __WCOREFLAG 0x80
-
-#define WEXITSTATUS(status) __WEXITSTATUS(status)
-#define WTERMSIG(status) __WTERMSIG(status)
-#define WIFEXITED(status) __WIFEXITED(status)
-
-#define WCOREFLAG __WCOREFLAG
-#define W_EXITCODE(ret, sig) __W_EXITCODE(ret, sig)
-#define W_STOPCODE(sig) __W_STOPCODE(sig)
-
-// First argument to waitid:
-#define P_ALL 0
-#define P_PID 1
-#define P_PGID 2
-#define P_PIDFD 3
+#include <linux/wait.h>
+
+#define WCOREDUMP(status) ((status) & WCOREFLAG)
+#define WEXITSTATUS(status) (((status) & 0xff00) >> 8)
+#define WIFCONTINUED(status) ((status) == 0xffff)
+#define WIFEXITED(status) (WTERMSIG(status) == 0)
+#define WIFSIGNALED(status) ((WTERMSIG(status) + 1) >= 2)
+#define WIFSTOPPED(status) (WTERMSIG(status) == 0x7f)
+#define WSTOPSIG(status) WEXITSTATUS(status)
+#define WTERMSIG(status) ((status) & 0x7f)
+
+#define WCOREFLAG 0x80
+#define W_EXITCODE(ret, sig) ((ret) << 8 | (sig))
+#define W_STOPCODE(sig) ((sig) << 8 | 0x7f)
 
 #endif // LLVM_LIBC_MACROS_LINUX_SYS_WAIT_MACROS_H
diff --git a/libc/include/llvm-libc-macros/poll-macros.h b/libc/include/llvm-libc-macros/poll-macros.h
new file mode 100644
index 0000000000000..52b59a978a21f
--- /dev/null
+++ b/libc/include/llvm-libc-macros/poll-macros.h
@@ -0,0 +1,16 @@
+//===-- Macros defined in poll.h header file ------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_MACROS_POLL_MACROS_H
+#define LLVM_LIBC_MACROS_POLL_MACROS_H
+
+#ifdef __linux__
+#include "linux/poll-macros.h"
+#endif
+
+#endif // LLVM_LIBC_MACROS_POLL_MACROS_H
diff --git a/libc/include/llvm-libc-macros/unistd-macros.h b/libc/include/llvm-libc-macros/unistd-macros.h
index 4f27f075fcc61..e09dada04a19d 100644
--- a/libc/include/llvm-libc-macros/unistd-macros.h
+++ b/libc/include/llvm-libc-macros/unistd-macros.h
@@ -5,4 +5,8 @@
 #include "linux/unistd-macros.h"
 #endif
 
+#define STDIN_FILENO 0
+#define STDOUT_FILENO 1
+#define STDERR_FILENO 2
+
 #endif // LLVM_LIBC_MACROS_UNISTD_MACROS_H
diff --git a/libc/include/llvm-libc-types/CMakeLists.txt b/libc/include/llvm-libc-types/CMakeLists.txt
index 6cbaa1ac0b30c..7ed69ab1af6d9 100644
--- a/libc/include/llvm-libc-types/CMakeLists.txt
+++ b/libc/include/llvm-libc-types/CMakeLists.txt
@@ -15,7 +15,6 @@ add_header(__pthread_start_t HDR __pthread_start_t.h)
 add_header(__pthread_tss_dtor_t HDR __pthread_tss_dtor_t.h)
 add_header(__qsortcompare_t HDR __qsortcompare_t.h)
 add_header(__qsortrcompare_t HDR __qsortrcompare_t.h)
-add_header(__sighandler_t HDR __sighandler_t.h)
 add_header(__thread_type HDR __thread_type.h)
 add_header(blkcnt_t HDR blkcnt_t.h)
 add_header(blksize_t HDR blksize_t.h)
@@ -44,6 +43,7 @@ add_header(jmp_buf HDR jmp_buf.h)
 add_header(mbstate_t HDR mbstate_t.h)
 add_header(mode_t HDR mode_t.h)
 add_header(mtx_t HDR mtx_t.h DEPENDS .__futex_word .__mutex_type)
+add_header(nfds_t HDR nfds_t.h)
 add_header(nlink_t HDR nlink_t.h)
 add_header(off_t HDR off_t.h)
 add_header(once_flag HDR once_flag.h DEPENDS .__futex_word)
@@ -65,16 +65,18 @@ if(LIBC_TYPES_TIME_T_IS_32_BIT)
 else()
   add_header(time_t HDR time_t_64.h DEST_HDR time_t.h)
 endif()
+add_header(sighandler_t HDR sighandler_t.h)
 add_header(stack_t HDR stack_t.h DEPENDS .size_t)
 add_header(suseconds_t HDR suseconds_t.h)
+add_header(struct_dirent HDR struct_dirent.h DEPENDS .ino_t .off_t)
+add_header(struct_f_owner_ex HDR struct_f_owner_ex.h DEPENDS .pid_t)
 add_header(struct_flock HDR struct_flock.h DEPENDS .off_t .pid_t)
 add_header(struct_flock64 HDR struct_flock64.h DEPENDS .off64_t .pid_t)
-add_header(struct_f_owner_ex HDR struct_f_owner_ex.h DEPENDS .pid_t)
-add_header(struct_timeval HDR struct_timeval.h DEPENDS .suseconds_t .time_t)
+add_header(struct_pollfd HDR struct_pollfd.h)
 add_header(struct_rlimit HDR struct_rlimit.h DEPENDS .rlim_t)
-add_header(struct_rusage HDR struct_rusage.h DEPENDS .struct_timeval)
-add_header(struct_dirent HDR struct_dirent.h DEPENDS .ino_t .off_t)
 add_header(struct_sched_param HDR struct_sched_param.h)
+add_header(struct_timeval HDR struct_timeval.h DEPENDS .suseconds_t .time_t)
+add_header(struct_rusage HDR struct_rusage.h DEPENDS .struct_timeval)
 add_header(union_sigval HDR union_sigval.h)
 add_header(siginfo_t HDR siginfo_t.h DEPENDS .union_sigval .pid_t .uid_t .clock_t)
 add_header(sig_atomic_t HDR sig_atomic_t.h)
diff --git a/libc/include/llvm-libc-types/__sighandler_t.h b/libc/include/llvm-libc-types/__sighandler_t.h
deleted file mode 100644
index 9c1ac997fc4ee..0000000000000
--- a/libc/include/llvm-libc-types/__sighandler_t.h
+++ /dev/null
@@ -1,14 +0,0 @@
-//===-- Definition of struct __sighandler_t -------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LLVM_LIBC_TYPES___SIGHANDLER_T_H
-#define LLVM_LIBC_TYPES___SIGHANDLER_T_H
-
-typedef void (*__sighandler_t)(int);
-
-#endif // LLVM_LIBC_TYPES___SIGHANDLER_T_H
diff --git a/libc/include/llvm-libc-types/nfds_t.h b/libc/include/llvm-libc-types/nfds_t.h
new file mode 100644
index 0000000000000..c0abccee8a7da
--- /dev/null
+++ b/libc/include/llvm-libc-types/nfds_t.h
@@ -0,0 +1,14 @@
+//===-- Definition of type nfds_t -----------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_TYPES_NFDS_T_H
+#define LLVM_LIBC_TYPES_NFDS_T_H
+
+typedef unsigned int nfds_t;
+
+#endif // LLVM_LIBC_TYPES_NFDS_T_H
diff --git a/libc/include/llvm-libc-types/sighandler_t.h b/libc/include/llvm-libc-types/sighandler_t.h
new file mode 100644
index 0000000000000..f39ab04685200
--- /dev/null
+++ b/libc/include/llvm-libc-types/sighandler_t.h
@@ -0,0 +1,17 @@
+//===-- Definition of sighandler_t ----------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_TYPES_SIGHANDLER_T_H
+#define LLVM_LIBC_TYPES_SIGHANDLER_T_H
+
+#ifdef __linux__
+// For compatibility with glibc.
+typedef void (*sighandler_t)(int);
+#endif
+
+#endif // LLVM_LIBC_TYPES_SIGHANDLER_T_H
diff --git a/libc/include/llvm-libc-types/struct_pollfd.h b/libc/include/llvm-libc-types/struct_pollfd.h
new file mode 100644
index 0000000000000..80abc8e76efc6
--- /dev/null
+++ b/libc/include/llvm-libc-types/struct_pollfd.h
@@ -0,0 +1,18 @@
+//===-- Definition of type struct pollfd ----------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_TYPES_STRUCT_POLLFD_H
+#define LLVM_LIBC_TYPES_STRUCT_POLLFD_H
+
+struct pollfd {
+  int fd;
+  short events;
+  short revents;
+};
+
+#endif // LLVM_LIBC_TYPES_STRUCT_POLLFD_H
diff --git a/libc/include/llvm-libc-types/struct_sigaction.h b/libc/include/llvm-libc-types/struct_sigaction.h
index b4d0c965a4c63..907418b5e0f9a 100644
--- a/libc/include/llvm-libc-types/struct_sigaction.h
+++ b/libc/include/llvm-libc-types/struct_sigaction.h
@@ -25,6 +25,4 @@ struct sigaction {
 #endif
 };
 
-typedef void (*__sighandler_t)(int);
-
 #endif // LLVM_LIBC_TYPES_STRUCT_SIGACTION_H
diff --git a/libc/include/llvm-libc-types/struct_tm.h b/libc/include/llvm-libc-types/struct_tm.h
index 9fef7c5718ea4..2ec74ecac0293 100644
--- a/libc/include/llvm-libc-types/struct_tm.h
+++ b/libc/include/llvm-libc-types/struct_tm.h
@@ -19,6 +19,7 @@ struct tm {
   int tm_wday;  // days since Sunday
   int tm_yday;  // days since January
   int tm_isdst; // Daylight Saving Time flag
+  // TODO: add tm_gmtoff and tm_zone? (posix extensions)
 };
 
 #endif // LLVM_LIBC_TYPES_STRUCT_TM_H
diff --git a/libc/include/locale.h.def b/libc/include/locale.h.def
index 516c6e6275e68..930373eade4d5 100644
--- a/libc/include/locale.h.def
+++ b/libc/include/locale.h.def
@@ -12,8 +12,6 @@
 #include "__llvm-libc-common.h"
 
 #include "llvm-libc-macros/locale-macros.h"
-#include "llvm-libc-types/locale_t.h"
-#include "llvm-libc-types/struct_lconv.h"
 
 %%public_api()
 
diff --git a/libc/include/locale.yaml b/libc/include/locale.yaml
index 9ff53c16398a5..6c71b70e59f0b 100644
--- a/libc/include/locale.yaml
+++ b/libc/include/locale.yaml
@@ -1,5 +1,8 @@
 header: locale.h
 header_template: locale.h.def
+types:
+  - type_name: locale_t
+  - type_name: struct_lconv
 functions:
   - name: localeconv
     standards:
diff --git a/libc/include/math.h.def b/libc/include/math.h.def
index 9822d8bd7ba17..9cfc2f7ea084e 100644
--- a/libc/include/math.h.def
+++ b/libc/include/math.h.def
@@ -12,7 +12,6 @@
 #include "__llvm-libc-common.h"
 #include "llvm-libc-macros/float16-macros.h"
 #include "llvm-libc-macros/math-macros.h"
-#include "llvm-libc-types/float128.h"
 
 
 %%public_api()
diff --git a/libc/include/math.yaml b/libc/include/math.yaml
index 3a660a59d3605..b98bc55f6cc53 100644
--- a/libc/include/math.yaml
+++ b/libc/include/math.yaml
@@ -32,6 +32,13 @@ functions:
     return_type: float
     arguments:
       - type: float
+  - name: asinf16
+    standards:
+      - stdc
+    return_type: _Float16
+    arguments:
+      - type: _Float16
+    guard: LIBC_TYPES_HAS_FLOAT16
   - name: asinhf
     standards:
       - stdc
diff --git a/libc/include/poll.h.def b/libc/include/poll.h.def
new file mode 100644
index 0000000000000..2f0e0376fa7bb
--- /dev/null
+++ b/libc/include/poll.h.def
@@ -0,0 +1,17 @@
+//===-- C standard library header poll.h ----------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_POLL_H
+#define LLVM_LIBC_POLL_H
+
+#include "__llvm-libc-common.h"
+#include "llvm-libc-macros/poll-macros.h"
+
+%%public_api()
+
+#endif // LLVM_LIBC_POLL_H
diff --git a/libc/include/poll.yaml b/libc/include/poll.yaml
new file mode 100644
index 0000000000000..399cc91854cdf
--- /dev/null
+++ b/libc/include/poll.yaml
@@ -0,0 +1,16 @@
+header: poll.h
+header_template: poll.h.def
+macros: []
+types:
+  - type_name: struct_pollfd
+  - type_name: nfds_t
+enums: []
+functions:
+  - name: poll
+    standards:
+      - posix
+    return_type: int
+    arguments:
+      - type: struct pollfd *
+      - type: nfds_t
+      - type: int
diff --git a/libc/include/setjmp.h.def b/libc/include/setjmp.h.def
index cb083b8cd023e..670bc1ac0fe24 100644
--- a/libc/include/setjmp.h.def
+++ b/libc/include/setjmp.h.def
@@ -10,7 +10,6 @@
 #define LLVM_LIBC_SETJMP_H
 
 #include "__llvm-libc-common.h"
-#include "llvm-libc-types/jmp_buf.h"
 
 %%public_api()
 
diff --git a/libc/include/setjmp.yaml b/libc/include/setjmp.yaml
index 2c4f7fb6dfcc7..5fbb9eb2a47e5 100644
--- a/libc/include/setjmp.yaml
+++ b/libc/include/setjmp.yaml
@@ -17,5 +17,7 @@ functions:
     standards:
       - stdc
     return_type: int
+    attributes:
+      - _Returns_twice
     arguments:
       - type: jmp_buf
diff --git a/libc/include/signal.yaml b/libc/include/signal.yaml
index 576e77576ac74..6fdd8c97ccbe2 100644
--- a/libc/include/signal.yaml
+++ b/libc/include/signal.yaml
@@ -3,12 +3,13 @@ header_template: signal.h.def
 macros: []
 types:
   - type_name: pid_t
-  - type_name: stack_t
+  - type_name: sig_atomic_t
+  - type_name: sighandler_t
   - type_name: siginfo_t
-  - type_name: struct_sigaction
   - type_name: sigset_t
+  - type_name: stack_t
+  - type_name: struct_sigaction
   - type_name: union_sigval
-  - type_name: sig_atomic_t
 enums: []
 objects: []
 functions:
@@ -69,10 +70,15 @@ functions:
   - name: signal
     standards:
       - stdc
-    return_type: __sighandler_t
+    # May the Geneva Convention have mercy on my soul... Why this insanity?
+    # Well: signal returns a function pointer to a function with no return
+    # value and which accepts an int. The parameter list appears on the far
+    # right of the declaration. i.e.
+    # void (*signal(int, void (*)(int)))(int);
+    return_type: void (*
     arguments:
       - type: int
-      - type: __sighandler_t
+      - type: void (*)(int)))(int
   - name: sigprocmask
     standards:
       - POSIX
diff --git a/libc/include/spawn.yaml b/libc/include/spawn.yaml
index e725ab9719eda..c763cc76fd094 100644
--- a/libc/include/spawn.yaml
+++ b/libc/include/spawn.yaml
@@ -18,8 +18,8 @@ functions:
       - type: const char *__restrict
       - type: posix_spawn_file_actions_t *
       - type: posix_spawnattr_t *__restrict
-      - type: const char *__restrict *
-      - type: const char *__restrict *
+      - type: char * const * __restrict
+      - type: char * const * __restrict
   - name: posix_spawn_file_actions_addclose
     standards:
       - POSIX
diff --git a/libc/include/stdlib.h.def b/libc/include/stdlib.h.def
index 01b0e1a2395a2..d523f7a53024a 100644
--- a/libc/include/stdlib.h.def
+++ b/libc/include/stdlib.h.def
@@ -10,7 +10,6 @@
 #define LLVM_LIBC_STDLIB_H
 
 #include "__llvm-libc-common.h"
-#include "llvm-libc-types/locale_t.h"
 #include "llvm-libc-macros/stdlib-macros.h"
 
 %%public_api()
diff --git a/libc/include/stdlib.yaml b/libc/include/stdlib.yaml
index 4b68f272613b1..c231a797bd445 100644
--- a/libc/include/stdlib.yaml
+++ b/libc/include/stdlib.yaml
@@ -8,10 +8,11 @@ types:
   - type_name: __qsortrcompare_t
   - type_name: __qsortcompare_t
   - type_name: __bsearchcompare_t
-  - type_name: size_t
-  - type_name: lldiv_t
-  - type_name: ldiv_t
   - type_name: div_t
+  - type_name: ldiv_t
+  - type_name: lldiv_t
+  - type_name: locale_t
+  - type_name: size_t
 enums: []
 objects: []
 functions:
diff --git a/libc/include/string.h.def b/libc/include/string.h.def
index e180f0d2561d3..1bd2687db2bea 100644
--- a/libc/include/string.h.def
+++ b/libc/include/string.h.def
@@ -11,7 +11,6 @@
 
 #include "__llvm-libc-common.h"
 
-#include "llvm-libc-types/locale_t.h"
 #include "llvm-libc-macros/null-macro.h"
 
 %%public_api()
diff --git a/libc/include/string.yaml b/libc/include/string.yaml
index deded309abc2c..9f72b8db6c1eb 100644
--- a/libc/include/string.yaml
+++ b/libc/include/string.yaml
@@ -2,6 +2,7 @@ header: string.h
 header_template: string.h.def
 macros: []
 types:
+  - type_name: locale_t
   - type_name: size_t
 enums: []
 objects: []
diff --git a/libc/include/strings.yaml b/libc/include/strings.yaml
index e672dca6a94dd..b6aa8f6d60b27 100644
--- a/libc/include/strings.yaml
+++ b/libc/include/strings.yaml
@@ -1,7 +1,8 @@
 header: strings.h
 header_template: strings.h.def
 macros: []
-types: []
+types:
+  - type_name: size_t
 enums: []
 objects: []
 functions:
diff --git a/libc/include/sys/resource.yaml b/libc/include/sys/resource.yaml
index 3652d6d490a49..e7645a3b4f0b3 100644
--- a/libc/include/sys/resource.yaml
+++ b/libc/include/sys/resource.yaml
@@ -12,10 +12,12 @@ functions:
       - POSIX
     return_type: int
     arguments:
+      - type: int
       - type: struct rlimit *
   - name: setrlimit
     standards:
       - POSIX
     return_type: int
     arguments:
-      - type: const struct rlimit
+      - type: int
+      - type: const struct rlimit *
diff --git a/libc/include/sys/time.h.def b/libc/include/sys/time.h.def
index b37555e584957..8b7c9ec669b2f 100644
--- a/libc/include/sys/time.h.def
+++ b/libc/include/sys/time.h.def
@@ -11,8 +11,6 @@
 
 #include "__llvm-libc-common.h"
 
-#include "../llvm-libc-types/struct_timeval.h"
-
 #include "../llvm-libc-macros/sys-time-macros.h"
 
 %%public_api()
diff --git a/libc/include/sys/time.yaml b/libc/include/sys/time.yaml
index 687c1f83028d0..ca497bbe92995 100644
--- a/libc/include/sys/time.yaml
+++ b/libc/include/sys/time.yaml
@@ -2,7 +2,8 @@ header: sys/time.h
 header_template: time.h.def
 standards: Linux
 macros: []
-types: []
+types:
+  - type_name: struct_timeval
 enums: []
 functions: []
 objects: []
diff --git a/libc/include/sys/uio.yaml b/libc/include/sys/uio.yaml
index 87c5bdff48245..6d3f336b2b520 100644
--- a/libc/include/sys/uio.yaml
+++ b/libc/include/sys/uio.yaml
@@ -15,3 +15,11 @@ functions:
       - type: int
       - type: const struct iovec *
       - type: int
+  - name: readv
+    standards:
+      - POSIX
+    return_type: ssize_t
+    arguments:
+      - type: int
+      - type: const struct iovec *
+      - type: int
diff --git a/libc/include/unistd.yaml b/libc/include/unistd.yaml
index c1901be446fe5..d04d46bd5c002 100644
--- a/libc/include/unistd.yaml
+++ b/libc/include/unistd.yaml
@@ -275,6 +275,12 @@ functions:
       - type: const void *__restrict
       - type: void *
       - type: ssize_t
+  - name: setsid
+    standards:
+      - POSIX
+    return_type: pid_t
+    arguments:
+      - type: void
   - name: symlink
     standards:
       - POSIX
diff --git a/libc/include/wchar.h.def b/libc/include/wchar.h.def
index d0de1a6762a39..4c25de700d606 100644
--- a/libc/include/wchar.h.def
+++ b/libc/include/wchar.h.def
@@ -11,8 +11,6 @@
 
 #include "__llvm-libc-common.h"
 #include "llvm-libc-macros/wchar-macros.h"
-#include "llvm-libc-types/wint_t.h"
-#include "llvm-libc-types/mbstate_t.h"
 
 %%public_api()
 
diff --git a/libc/src/CMakeLists.txt b/libc/src/CMakeLists.txt
index 41183429f67a7..19a354ceee4b6 100644
--- a/libc/src/CMakeLists.txt
+++ b/libc/src/CMakeLists.txt
@@ -20,6 +20,7 @@ add_subdirectory(unistd)
 if(${LIBC_TARGET_OS} STREQUAL "linux")
   add_subdirectory(dirent)
   add_subdirectory(fcntl)
+  add_subdirectory(poll)
   add_subdirectory(pthread)
   add_subdirectory(sched)
   add_subdirectory(sys)
diff --git a/libc/src/__support/CMakeLists.txt b/libc/src/__support/CMakeLists.txt
index 148484052dcad..17f03a6b6c4a0 100644
--- a/libc/src/__support/CMakeLists.txt
+++ b/libc/src/__support/CMakeLists.txt
@@ -87,7 +87,7 @@ add_header_library(
     endian_internal.h
     macros/properties/architectures.h
     macros/attributes.h
-    macros/properties/cpu_features.h
+    macros/config.h
 )
 
 add_header_library(
@@ -243,12 +243,20 @@ add_header_library(
   HDRS
     complex_type.h
   DEPENDS
-    libc.src.__support.CPP.bit
-    libc.src.__support.FPUtil.fp_bits
     libc.src.__support.macros.properties.types
     libc.src.__support.macros.properties.complex_types
 )
 
+add_header_library(
+  complex_basic_ops
+  HDRS
+    complex_basic_ops.h
+  DEPENDS
+    .complex_type
+    libc.src.__support.CPP.bit
+    libc.src.__support.FPUtil.fp_bits
+)
+
 add_header_library(
   integer_operations
   HDRS
diff --git a/libc/src/__support/CPP/algorithm.h b/libc/src/__support/CPP/algorithm.h
index f5dc9067409eb..7704b3fa81f0c 100644
--- a/libc/src/__support/CPP/algorithm.h
+++ b/libc/src/__support/CPP/algorithm.h
@@ -26,6 +26,8 @@ template <class T> LIBC_INLINE constexpr const T &min(const T &a, const T &b) {
   return (a < b) ? a : b;
 }
 
+template <class T> LIBC_INLINE constexpr T abs(T a) { return a < 0 ? -a : a; }
+
 template <class InputIt, class UnaryPred>
 LIBC_INLINE constexpr InputIt find_if_not(InputIt first, InputIt last,
                                           UnaryPred q) {
diff --git a/libc/src/__support/CPP/span.h b/libc/src/__support/CPP/span.h
index e9e3dbf169ce0..a41c9b744e370 100644
--- a/libc/src/__support/CPP/span.h
+++ b/libc/src/__support/CPP/span.h
@@ -10,7 +10,7 @@
 
 #include <stddef.h> // For size_t
 
-#include "array.h"       // For array
+#include "array.h" // For array
 #include "src/__support/macros/config.h"
 #include "type_traits.h" // For remove_cv_t, enable_if_t, is_same_v, is_const_v
 
@@ -52,6 +52,8 @@ template <typename T> class span {
 
   LIBC_INLINE constexpr span() : span_data(nullptr), span_size(0) {}
 
+  LIBC_INLINE constexpr span(const span &) = default;
+
   LIBC_INLINE constexpr span(pointer first, size_type count)
       : span_data(first), span_size(count) {}
 
diff --git a/libc/src/__support/CPP/type_traits.h b/libc/src/__support/CPP/type_traits.h
index 910cebbb8d059..d48ee23aeae07 100644
--- a/libc/src/__support/CPP/type_traits.h
+++ b/libc/src/__support/CPP/type_traits.h
@@ -26,6 +26,7 @@
 #include "src/__support/CPP/type_traits/is_array.h"
 #include "src/__support/CPP/type_traits/is_base_of.h"
 #include "src/__support/CPP/type_traits/is_class.h"
+#include "src/__support/CPP/type_traits/is_complex.h"
 #include "src/__support/CPP/type_traits/is_const.h"
 #include "src/__support/CPP/type_traits/is_constant_evaluated.h"
 #include "src/__support/CPP/type_traits/is_convertible.h"
diff --git a/libc/src/__support/FPUtil/CMakeLists.txt b/libc/src/__support/FPUtil/CMakeLists.txt
index 522b4afefd48d..bfe0170f09fd9 100644
--- a/libc/src/__support/FPUtil/CMakeLists.txt
+++ b/libc/src/__support/FPUtil/CMakeLists.txt
@@ -110,6 +110,7 @@ add_header_library(
   DEPENDS
     libc.src.__support.common
     libc.src.__support.FPUtil.generic.sqrt
+    libc.src.__support.macros.properties.cpu_features
 )
 
 add_header_library(
@@ -120,6 +121,7 @@ add_header_library(
     libc.src.__support.common
     libc.src.__support.CPP.type_traits
     libc.src.__support.FPUtil.generic.fma
+    libc.src.__support.macros.properties.cpu_features
   FLAGS
     FMA_OPT
 )
@@ -130,6 +132,7 @@ add_header_library(
     multiply_add.h
   DEPENDS
     libc.src.__support.common
+    libc.src.__support.macros.properties.cpu_features
   FLAGS
     FMA_OPT
 )
@@ -150,6 +153,7 @@ add_header_library(
   DEPENDS
     libc.src.__support.common
     libc.src.__support.macros.optimization
+    libc.src.__support.macros.properties.cpu_features
   FLAGS
     ROUND_OPT
 )
@@ -161,6 +165,7 @@ add_header_library(
   DEPENDS
     libc.src.__support.common
     libc.src.__support.number_pair
+    libc.src.__support.macros.properties.cpu_features
     .multiply_add
 )
 
diff --git a/libc/src/__support/FPUtil/double_double.h b/libc/src/__support/FPUtil/double_double.h
index db3c2c8a3d7a6..8c318cfbed363 100644
--- a/libc/src/__support/FPUtil/double_double.h
+++ b/libc/src/__support/FPUtil/double_double.h
@@ -20,41 +20,52 @@ namespace fputil {
 
 #define DEFAULT_DOUBLE_SPLIT 27
 
-using DoubleDouble = LIBC_NAMESPACE::NumberPair<double>;
+template <typename T> struct DefaultSplit;
+template <> struct DefaultSplit<float> {
+  static constexpr size_t VALUE = 12;
+};
+template <> struct DefaultSplit<double> {
+  static constexpr size_t VALUE = DEFAULT_DOUBLE_SPLIT;
+};
+
+using DoubleDouble = NumberPair<double>;
+using FloatFloat = NumberPair<float>;
 
 // The output of Dekker's FastTwoSum algorithm is correct, i.e.:
 //   r.hi + r.lo = a + b exactly
 //   and |r.lo| < eps(r.lo)
 // Assumption: |a| >= |b|, or a = 0.
-template <bool FAST2SUM = true>
-LIBC_INLINE constexpr DoubleDouble exact_add(double a, double b) {
-  DoubleDouble r{0.0, 0.0};
+template <bool FAST2SUM = true, typename T = double>
+LIBC_INLINE constexpr NumberPair<T> exact_add(T a, T b) {
+  NumberPair<T> r{0.0, 0.0};
   if constexpr (FAST2SUM) {
     r.hi = a + b;
-    double t = r.hi - a;
+    T t = r.hi - a;
     r.lo = b - t;
   } else {
     r.hi = a + b;
-    double t1 = r.hi - a;
-    double t2 = r.hi - t1;
-    double t3 = b - t1;
-    double t4 = a - t2;
+    T t1 = r.hi - a;
+    T t2 = r.hi - t1;
+    T t3 = b - t1;
+    T t4 = a - t2;
     r.lo = t3 + t4;
   }
   return r;
 }
 
 // Assumption: |a.hi| >= |b.hi|
-LIBC_INLINE constexpr DoubleDouble add(const DoubleDouble &a,
-                                       const DoubleDouble &b) {
-  DoubleDouble r = exact_add(a.hi, b.hi);
-  double lo = a.lo + b.lo;
+template <typename T>
+LIBC_INLINE constexpr NumberPair<T> add(const NumberPair<T> &a,
+                                        const NumberPair<T> &b) {
+  NumberPair<T> r = exact_add(a.hi, b.hi);
+  T lo = a.lo + b.lo;
   return exact_add(r.hi, r.lo + lo);
 }
 
 // Assumption: |a.hi| >= |b|
-LIBC_INLINE constexpr DoubleDouble add(const DoubleDouble &a, double b) {
-  DoubleDouble r = exact_add<false>(a.hi, b);
+template <typename T>
+LIBC_INLINE constexpr NumberPair<T> add(const NumberPair<T> &a, T b) {
+  NumberPair<T> r = exact_add<false>(a.hi, b);
   return exact_add(r.hi, r.lo + a.lo);
 }
 
@@ -63,12 +74,12 @@ LIBC_INLINE constexpr DoubleDouble add(const DoubleDouble &a, double b) {
 //   Zimmermann, P., "Note on the Veltkamp/Dekker Algorithms with Directed
 //   Roundings," https://inria.hal.science/hal-04480440.
 // Default splitting constant = 2^ceil(prec(double)/2) + 1 = 2^27 + 1.
-template <size_t N = DEFAULT_DOUBLE_SPLIT>
-LIBC_INLINE constexpr DoubleDouble split(double a) {
-  DoubleDouble r{0.0, 0.0};
+template <typename T = double, size_t N = DefaultSplit<T>::VALUE>
+LIBC_INLINE constexpr NumberPair<T> split(T a) {
+  NumberPair<T> r{0.0, 0.0};
   // CN = 2^N.
-  constexpr double CN = static_cast<double>(1 << N);
-  constexpr double C = CN + 1.0;
+  constexpr T CN = static_cast<T>(1 << N);
+  constexpr T C = CN + 1.0;
   double t1 = C * a;
   double t2 = a - t1;
   r.hi = t1 + t2;
@@ -77,16 +88,15 @@ LIBC_INLINE constexpr DoubleDouble split(double a) {
 }
 
 // Helper for non-fma exact mult where the first number is already split.
-template <size_t SPLIT_B = DEFAULT_DOUBLE_SPLIT>
-LIBC_INLINE DoubleDouble exact_mult(const DoubleDouble &as, double a,
-                                    double b) {
-  DoubleDouble bs = split<SPLIT_B>(b);
-  DoubleDouble r{0.0, 0.0};
+template <typename T = double, size_t SPLIT_B = DefaultSplit<T>::VALUE>
+LIBC_INLINE NumberPair<T> exact_mult(const NumberPair<T> &as, T a, T b) {
+  NumberPair<T> bs = split<T, SPLIT_B>(b);
+  NumberPair<T> r{0.0, 0.0};
 
   r.hi = a * b;
-  double t1 = as.hi * bs.hi - r.hi;
-  double t2 = as.hi * bs.lo + t1;
-  double t3 = as.lo * bs.hi + t2;
+  T t1 = as.hi * bs.hi - r.hi;
+  T t2 = as.hi * bs.lo + t1;
+  T t3 = as.lo * bs.hi + t2;
   r.lo = as.lo * bs.lo + t3;
 
   return r;
@@ -99,18 +109,18 @@ LIBC_INLINE DoubleDouble exact_mult(const DoubleDouble &as, double a,
 // Using Theorem 1 in the paper above, without FMA instruction, if we restrict
 // the generated constants to precision <= 51, and splitting it by 2^28 + 1,
 // then a * b = r.hi + r.lo is exact for all rounding modes.
-template <size_t SPLIT_B = 27>
-LIBC_INLINE DoubleDouble exact_mult(double a, double b) {
-  DoubleDouble r{0.0, 0.0};
+template <typename T = double, size_t SPLIT_B = DefaultSplit<T>::VALUE>
+LIBC_INLINE NumberPair<T> exact_mult(T a, T b) {
+  NumberPair<T> r{0.0, 0.0};
 
 #ifdef LIBC_TARGET_CPU_HAS_FMA
   r.hi = a * b;
   r.lo = fputil::multiply_add(a, b, -r.hi);
 #else
   // Dekker's Product.
-  DoubleDouble as = split(a);
+  NumberPair<T> as = split(a);
 
-  r = exact_mult<SPLIT_B>(as, a, b);
+  r = exact_mult<T, SPLIT_B>(as, a, b);
 #endif // LIBC_TARGET_CPU_HAS_FMA
 
   return r;
@@ -125,7 +135,7 @@ LIBC_INLINE DoubleDouble quick_mult(double a, const DoubleDouble &b) {
 template <size_t SPLIT_B = 27>
 LIBC_INLINE DoubleDouble quick_mult(const DoubleDouble &a,
                                     const DoubleDouble &b) {
-  DoubleDouble r = exact_mult<SPLIT_B>(a.hi, b.hi);
+  DoubleDouble r = exact_mult<double, SPLIT_B>(a.hi, b.hi);
   double t1 = multiply_add(a.hi, b.lo, r.lo);
   double t2 = multiply_add(a.lo, b.hi, t1);
   r.lo = t2;
@@ -157,19 +167,20 @@ LIBC_INLINE DoubleDouble multiply_add<DoubleDouble>(const DoubleDouble &a,
 //   rl = q * (ah - bh * rh) + q * (al - bl * rh)
 // as accurate as possible, then the error is bounded by:
 //   |(ah + al) / (bh + bl) - (rh + rl)| < O(bl/bh) * (2^-52 + al/ah + bl/bh)
-LIBC_INLINE DoubleDouble div(const DoubleDouble &a, const DoubleDouble &b) {
-  DoubleDouble r;
-  double q = 1.0 / b.hi;
+template <typename T>
+LIBC_INLINE NumberPair<T> div(const NumberPair<T> &a, const NumberPair<T> &b) {
+  NumberPair<T> r;
+  T q = T(1) / b.hi;
   r.hi = a.hi * q;
 
 #ifdef LIBC_TARGET_CPU_HAS_FMA
-  double e_hi = fputil::multiply_add(b.hi, -r.hi, a.hi);
-  double e_lo = fputil::multiply_add(b.lo, -r.hi, a.lo);
+  T e_hi = fputil::multiply_add(b.hi, -r.hi, a.hi);
+  T e_lo = fputil::multiply_add(b.lo, -r.hi, a.lo);
 #else
-  DoubleDouble b_hi_r_hi = fputil::exact_mult(b.hi, -r.hi);
-  DoubleDouble b_lo_r_hi = fputil::exact_mult(b.lo, -r.hi);
-  double e_hi = (a.hi + b_hi_r_hi.hi) + b_hi_r_hi.lo;
-  double e_lo = (a.lo + b_lo_r_hi.hi) + b_lo_r_hi.lo;
+  NumberPair<T> b_hi_r_hi = fputil::exact_mult(b.hi, -r.hi);
+  NumberPair<T> b_lo_r_hi = fputil::exact_mult(b.lo, -r.hi);
+  T e_hi = (a.hi + b_hi_r_hi.hi) + b_hi_r_hi.lo;
+  T e_lo = (a.lo + b_lo_r_hi.hi) + b_lo_r_hi.lo;
 #endif // LIBC_TARGET_CPU_HAS_FMA
 
   r.lo = q * (e_hi + e_lo);
diff --git a/libc/src/__support/FPUtil/dyadic_float.h b/libc/src/__support/FPUtil/dyadic_float.h
index 289fd01680547..16b690be08612 100644
--- a/libc/src/__support/FPUtil/dyadic_float.h
+++ b/libc/src/__support/FPUtil/dyadic_float.h
@@ -26,6 +26,54 @@
 namespace LIBC_NAMESPACE_DECL {
 namespace fputil {
 
+// Decide whether to round a UInt up, down or not at all at a given bit
+// position, based on the current rounding mode. The assumption is that the
+// caller is going to make the integer `value >> rshift`, and then might need
+// to round it up by 1 depending on the value of the bits shifted off the
+// bottom.
+//
+// `logical_sign` causes the behavior of FE_DOWNWARD and FE_UPWARD to
+// be reversed, which is what you'd want if this is the mantissa of a
+// negative floating-point number.
+//
+// Return value is +1 if the value should be rounded up; -1 if it should be
+// rounded down; 0 if it's exact and needs no rounding.
+template <size_t Bits>
+LIBC_INLINE constexpr int
+rounding_direction(const LIBC_NAMESPACE::UInt<Bits> &value, size_t rshift,
+                   Sign logical_sign) {
+  if (rshift == 0 || (rshift < Bits && (value << (Bits - rshift)) == 0) ||
+      (rshift >= Bits && value == 0))
+    return 0; // exact
+
+  switch (quick_get_round()) {
+  case FE_TONEAREST:
+    if (rshift > 0 && rshift <= Bits && value.get_bit(rshift - 1)) {
+      // We round up, unless the value is an exact halfway case and
+      // the bit that will end up in the units place is 0, in which
+      // case tie-break-to-even says round down.
+      bool round_bit = rshift < Bits ? value.get_bit(rshift) : 0;
+      return round_bit != 0 || (value << (Bits - rshift + 1)) != 0 ? +1 : -1;
+    } else {
+      return -1;
+    }
+  case FE_TOWARDZERO:
+    return -1;
+  case FE_DOWNWARD:
+    return logical_sign.is_neg() &&
+                   (rshift < Bits && (value << (Bits - rshift)) != 0)
+               ? +1
+               : -1;
+  case FE_UPWARD:
+    return logical_sign.is_pos() &&
+                   (rshift < Bits && (value << (Bits - rshift)) != 0)
+               ? +1
+               : -1;
+  default:
+    __builtin_unreachable();
+  }
+}
+
 // A generic class to perform computations of high precision floating points.
 // We store the value in dyadic format, including 3 fields:
 //   sign    : boolean value - false means positive, true means negative
@@ -101,6 +149,27 @@ template <size_t Bits> struct DyadicFloat {
     return exponent + (Bits - 1);
   }
 
+  // Produce a correctly rounded DyadicFloat from a too-large mantissa,
+  // by shifting it down and rounding if necessary.
+  template <size_t MantissaBits>
+  LIBC_INLINE constexpr static DyadicFloat<Bits>
+  round(Sign result_sign, int result_exponent,
+        const LIBC_NAMESPACE::UInt<MantissaBits> &input_mantissa,
+        size_t rshift) {
+    MantissaType result_mantissa(input_mantissa >> rshift);
+    if (rounding_direction(input_mantissa, rshift, result_sign) > 0) {
+      ++result_mantissa;
+      if (result_mantissa == 0) {
+        // Rounding up made the mantissa integer wrap round to 0,
+        // carrying a bit off the top. So we've rounded up to the next
+        // exponent.
+        result_mantissa.set_bit(Bits - 1);
+        ++result_exponent;
+      }
+    }
+    return DyadicFloat(result_sign, result_exponent, result_mantissa);
+  }
+
 #ifdef LIBC_TYPES_HAS_FLOAT16
   template <typename T, bool ShouldSignalExceptions>
   LIBC_INLINE constexpr cpp::enable_if_t<
@@ -374,6 +443,39 @@ template <size_t Bits> struct DyadicFloat {
 
     return new_mant;
   }
+
+  LIBC_INLINE constexpr MantissaType
+  as_mantissa_type_rounded(int *round_dir_out = nullptr) const {
+    int round_dir = 0;
+    MantissaType new_mant;
+    if (mantissa.is_zero()) {
+      new_mant = 0;
+    } else {
+      new_mant = mantissa;
+      if (exponent > 0) {
+        new_mant <<= exponent;
+      } else if (exponent < 0) {
+        size_t shift = -exponent;
+        new_mant >>= shift;
+        round_dir = rounding_direction(mantissa, shift, sign);
+        if (round_dir > 0)
+          ++new_mant;
+      }
+
+      if (sign.is_neg()) {
+        new_mant = (~new_mant) + 1;
+      }
+    }
+
+    if (round_dir_out)
+      *round_dir_out = round_dir;
+
+    return new_mant;
+  }
+
+  LIBC_INLINE constexpr DyadicFloat operator-() const {
+    return DyadicFloat(sign.negate(), exponent, mantissa);
+  }
 };
 
 // Quick add - Add 2 dyadic floats with rounding toward 0 and then normalize the
@@ -433,6 +535,12 @@ LIBC_INLINE constexpr DyadicFloat<Bits> quick_add(DyadicFloat<Bits> a,
   return result.normalize();
 }
 
+template <size_t Bits>
+LIBC_INLINE constexpr DyadicFloat<Bits> quick_sub(DyadicFloat<Bits> a,
+                                                  DyadicFloat<Bits> b) {
+  return quick_add(a, -b);
+}
+
 // Quick Mul - Slightly less accurate but efficient multiplication of 2 dyadic
 // floats with rounding toward 0 and then normalize the output:
 //   result.exponent = a.exponent + b.exponent + Bits,
@@ -464,6 +572,95 @@ LIBC_INLINE constexpr DyadicFloat<Bits> quick_mul(const DyadicFloat<Bits> &a,
   return result;
 }
 
+// Correctly rounded multiplication of 2 dyadic floats, assuming the
+// exponent remains within range.
+template <size_t Bits>
+LIBC_INLINE constexpr DyadicFloat<Bits>
+rounded_mul(const DyadicFloat<Bits> &a, const DyadicFloat<Bits> &b) {
+  using DblMant = LIBC_NAMESPACE::UInt<(2 * Bits)>;
+  Sign result_sign = (a.sign != b.sign) ? Sign::NEG : Sign::POS;
+  int result_exponent = a.exponent + b.exponent + static_cast<int>(Bits);
+  auto product = DblMant(a.mantissa) * DblMant(b.mantissa);
+  // As in quick_mul(), renormalize by 1 bit manually rather than countl_zero
+  if (product.get_bit(2 * Bits - 1) == 0) {
+    product <<= 1;
+    result_exponent -= 1;
+  }
+
+  return DyadicFloat<Bits>::round(result_sign, result_exponent, product, Bits);
+}
+
+// Approximate reciprocal - given a nonzero a, make a good approximation to 1/a.
+// The method is Newton-Raphson iteration, based on quick_mul.
+template <size_t Bits, typename = cpp::enable_if_t<(Bits >= 32)>>
+LIBC_INLINE constexpr DyadicFloat<Bits>
+approx_reciprocal(const DyadicFloat<Bits> &a) {
+  // Given an approximation x to 1/a, a better one is x' = x(2-ax).
+  //
+  // You can derive this by using the Newton-Raphson formula with the function
+  // f(x) = 1/x - a. But another way to see that it works is to say: suppose
+  // that ax = 1-e for some small error e. Then ax' = ax(2-ax) = (1-e)(1+e) =
+  // 1-e^2. So the error in x' is the square of the error in x, i.e. the number
+  // of correct bits in x' is double the number in x.
+
+  // An initial approximation to the reciprocal
+  DyadicFloat<Bits> x(Sign::POS, -32 - a.exponent - int(Bits),
+                      uint64_t(0xFFFFFFFFFFFFFFFF) /
+                          static_cast<uint64_t>(a.mantissa >> (Bits - 32)));
+
+  // The constant 2, which we'll need in every iteration
+  DyadicFloat<Bits> two(Sign::POS, 1, 1);
+
+  // We expect at least 31 correct bits from our 32-bit starting approximation
+  size_t ok_bits = 31;
+
+  // The number of good bits doubles in each iteration, except that rounding
+  // errors introduce a little extra each time. Subtract a bit from our
+  // accuracy assessment to account for that.
+  while (ok_bits < Bits) {
+    x = quick_mul(x, quick_sub(two, quick_mul(a, x)));
+    ok_bits = 2 * ok_bits - 1;
+  }
+
+  return x;
+}
+
+// Correctly rounded division of 2 dyadic floats, assuming the
+// exponent remains within range.
+template <size_t Bits>
+LIBC_INLINE constexpr DyadicFloat<Bits>
+rounded_div(const DyadicFloat<Bits> &af, const DyadicFloat<Bits> &bf) {
+  using DblMant = LIBC_NAMESPACE::UInt<(Bits * 2 + 64)>;
+
+  // Make an approximation to the quotient as a * (1/b). Both the
+  // multiplication and the reciprocal are a bit sloppy, which doesn't
+  // matter, because we're going to correct for that below.
+  auto qf = fputil::quick_mul(af, fputil::approx_reciprocal(bf));
+
+  // Switch to BigInt and stop using quick_add and quick_mul: now
+  // we're working in exact integers so as to get the true remainder.
+  DblMant a = af.mantissa, b = bf.mantissa, q = qf.mantissa;
+  q <<= 2; // leave room for a round bit, even if exponent decreases
+  a <<= af.exponent - bf.exponent - qf.exponent + 2;
+  DblMant qb = q * b;
+  if (qb < a) {
+    DblMant too_small = a - b;
+    while (qb <= too_small) {
+      qb += b;
+      ++q;
+    }
+  } else {
+    while (qb > a) {
+      qb -= b;
+      --q;
+    }
+  }
+
+  DyadicFloat<(Bits * 2)> qbig(qf.sign, qf.exponent - 2, q);
+  return DyadicFloat<Bits>::round(qbig.sign, qbig.exponent + Bits,
+                                  qbig.mantissa, Bits);
+}
+
 // Simple polynomial approximation.
 template <size_t Bits>
 LIBC_INLINE constexpr DyadicFloat<Bits>
diff --git a/libc/src/__support/FPUtil/except_value_utils.h b/libc/src/__support/FPUtil/except_value_utils.h
index f8e4e92d3e1fb..5f767769974d2 100644
--- a/libc/src/__support/FPUtil/except_value_utils.h
+++ b/libc/src/__support/FPUtil/except_value_utils.h
@@ -81,12 +81,19 @@ template <typename T, size_t N> struct ExceptValues {
         StorageType out_bits = values[i].rnd_towardzero_result;
         switch (fputil::quick_get_round()) {
         case FE_UPWARD:
-          out_bits += sign ? values[i].rnd_downward_offset
-                           : values[i].rnd_upward_offset;
+          if (sign)
+            out_bits += values[i].rnd_downward_offset;
+          else
+            out_bits += values[i].rnd_upward_offset;
           break;
         case FE_DOWNWARD:
-          out_bits += sign ? values[i].rnd_upward_offset
-                           : values[i].rnd_downward_offset;
+          // Use conditionals instead of ternary operator to work around gcc's
+          // -Wconversion false positive bug:
+          // https://gcc.gnu.org/bugzilla/show_bug.cgi?id=101537
+          if (sign)
+            out_bits += values[i].rnd_upward_offset;
+          else
+            out_bits += values[i].rnd_downward_offset;
           break;
         case FE_TONEAREST:
           out_bits += values[i].rnd_tonearest_offset;
diff --git a/libc/src/__support/GPU/utils.h b/libc/src/__support/GPU/utils.h
index e138c84c0cb22..323c003f1ff07 100644
--- a/libc/src/__support/GPU/utils.h
+++ b/libc/src/__support/GPU/utils.h
@@ -87,8 +87,9 @@ LIBC_INLINE void sync_threads() { __gpu_sync_threads(); }
 
 LIBC_INLINE void sync_lane(uint64_t lane_mask) { __gpu_sync_lane(lane_mask); }
 
-LIBC_INLINE uint32_t shuffle(uint64_t lane_mask, uint32_t idx, uint32_t x) {
-  return __gpu_shuffle_idx_u32(lane_mask, idx, x);
+LIBC_INLINE uint32_t shuffle(uint64_t lane_mask, uint32_t idx, uint32_t x,
+                             uint32_t width = __gpu_num_lanes()) {
+  return __gpu_shuffle_idx_u32(lane_mask, idx, x, width);
 }
 
 [[noreturn]] LIBC_INLINE void end_program() { __gpu_exit(); }
diff --git a/libc/src/__support/big_int.h b/libc/src/__support/big_int.h
index a95ab4ff8e1ab..f591b41df037b 100644
--- a/libc/src/__support/big_int.h
+++ b/libc/src/__support/big_int.h
@@ -936,6 +936,18 @@ struct BigInt {
   // Return the i-th word of the number.
   LIBC_INLINE constexpr WordType &operator[](size_t i) { return val[i]; }
 
+  // Return the i-th bit of the number.
+  LIBC_INLINE constexpr bool get_bit(size_t i) const {
+    const size_t word_index = i / WORD_SIZE;
+    return 1 & (val[word_index] >> (i % WORD_SIZE));
+  }
+
+  // Set the i-th bit of the number.
+  LIBC_INLINE constexpr void set_bit(size_t i) {
+    const size_t word_index = i / WORD_SIZE;
+    val[word_index] |= WordType(1) << (i % WORD_SIZE);
+  }
+
 private:
   LIBC_INLINE friend constexpr int cmp(const BigInt &lhs, const BigInt &rhs) {
     constexpr auto compare = [](WordType a, WordType b) {
@@ -968,7 +980,7 @@ struct BigInt {
   }
 
   LIBC_INLINE constexpr void decrement() {
-    multiword::add_with_carry(val, cpp::array<WordType, 1>{1});
+    multiword::sub_with_borrow(val, cpp::array<WordType, 1>{1});
   }
 
   LIBC_INLINE constexpr void extend(size_t index, bool is_neg) {
@@ -989,12 +1001,6 @@ struct BigInt {
   LIBC_INLINE constexpr void clear_msb() {
     val.back() &= mask_trailing_ones<WordType, WORD_SIZE - 1>();
   }
-
-  LIBC_INLINE constexpr void set_bit(size_t i) {
-    const size_t word_index = i / WORD_SIZE;
-    val[word_index] |= WordType(1) << (i % WORD_SIZE);
-  }
-
   LIBC_INLINE constexpr static Division divide_unsigned(const BigInt &dividend,
                                                         const BigInt &divider) {
     BigInt remainder = dividend;
diff --git a/libc/src/__support/complex_basic_ops.h b/libc/src/__support/complex_basic_ops.h
new file mode 100644
index 0000000000000..5992ebec0786c
--- /dev/null
+++ b/libc/src/__support/complex_basic_ops.h
@@ -0,0 +1,36 @@
+//===-- complex basic operations --------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC___SUPPORT_COMPLEX_BASIC_OPERATIONS_H
+#define LLVM_LIBC_SRC___SUPPORT_COMPLEX_BASIC_OPERATIONS_H
+
+#include "complex_type.h"
+#include "src/__support/CPP/bit.h"
+#include "src/__support/FPUtil/FPBits.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+template <typename T> LIBC_INLINE constexpr T conjugate(T c) {
+  Complex<make_real_t<T>> c_c = cpp::bit_cast<Complex<make_real_t<T>>>(c);
+  c_c.imag = -c_c.imag;
+  return cpp::bit_cast<T>(c_c);
+}
+
+template <typename T> LIBC_INLINE constexpr T project(T c) {
+  using real_t = make_real_t<T>;
+  Complex<real_t> c_c = cpp::bit_cast<Complex<real_t>>(c);
+  if (fputil::FPBits<real_t>(c_c.real).is_inf() ||
+      fputil::FPBits<real_t>(c_c.imag).is_inf())
+    return cpp::bit_cast<T>(
+        Complex<real_t>{(fputil::FPBits<real_t>::inf(Sign::POS).get_val()),
+                        static_cast<real_t>(c_c.imag > 0 ? 0.0 : -0.0)});
+  return c;
+}
+
+} // namespace LIBC_NAMESPACE_DECL
+#endif // LLVM_LIBC_SRC___SUPPORT_COMPLEX_BASIC_OPERATIONS_H
diff --git a/libc/src/__support/complex_type.h b/libc/src/__support/complex_type.h
index f72ce8a4efd13..93089de61dba8 100644
--- a/libc/src/__support/complex_type.h
+++ b/libc/src/__support/complex_type.h
@@ -9,8 +9,6 @@
 #ifndef LLVM_LIBC_SRC___SUPPORT_COMPLEX_TYPE_H
 #define LLVM_LIBC_SRC___SUPPORT_COMPLEX_TYPE_H
 
-#include "src/__support/CPP/bit.h"
-#include "src/__support/FPUtil/FPBits.h"
 #include "src/__support/macros/config.h"
 #include "src/__support/macros/properties/complex_types.h"
 #include "src/__support/macros/properties/types.h"
@@ -71,24 +69,5 @@ template <> struct make_real<cfloat128> {
 
 template <typename T> using make_real_t = typename make_real<T>::type;
 
-template <typename T> LIBC_INLINE constexpr T conjugate(T c) {
-  Complex<make_real_t<T>> c_c = cpp::bit_cast<Complex<make_real_t<T>>>(c);
-  c_c.imag = -c_c.imag;
-  return cpp::bit_cast<T>(c_c);
-}
-
-template <typename T> LIBC_INLINE constexpr T project(T c) {
-  using real_t = make_real_t<T>;
-  Complex<real_t> c_c = cpp::bit_cast<Complex<real_t>>(c);
-  if (fputil::FPBits<real_t>(c_c.real).is_inf() ||
-      fputil::FPBits<real_t>(c_c.imag).is_inf()) {
-    return cpp::bit_cast<T>(
-        Complex<real_t>{(fputil::FPBits<real_t>::inf(Sign::POS).get_val()),
-                        static_cast<real_t>(c_c.imag > 0 ? 0.0 : -0.0)});
-  } else {
-    return c;
-  }
-}
-
 } // namespace LIBC_NAMESPACE_DECL
 #endif // LLVM_LIBC_SRC___SUPPORT_COMPLEX_TYPE_H
diff --git a/libc/src/__support/integer_to_string.h b/libc/src/__support/integer_to_string.h
index ea620087584cb..13dd83e9ed2b1 100644
--- a/libc/src/__support/integer_to_string.h
+++ b/libc/src/__support/integer_to_string.h
@@ -164,6 +164,170 @@ template <size_t radix> using Custom = details::Fmt<radix>;
 
 } // namespace radix
 
+// Extract the low-order decimal digit from a value of integer type T. The
+// returned value is the digit itself, from 0 to 9. The input value is passed
+// by reference, and modified by dividing by 10, so that iterating this
+// function extracts all the digits of the original number one at a time from
+// low to high.
+template <typename T>
+LIBC_INLINE cpp::enable_if_t<cpp::is_integral_v<T>, uint8_t>
+extract_decimal_digit(T &value) {
+  const uint8_t digit(static_cast<uint8_t>(value % 10));
+  // For built-in integer types, we assume that an adequately fast division is
+  // available. If hardware division isn't implemented, then with a divisor
+  // known at compile time the compiler might be able to generate an optimized
+  // sequence instead.
+  value /= 10;
+  return digit;
+}
+
+// A specialization of extract_decimal_digit for the BigInt type in big_int.h,
+// avoiding the use of general-purpose BigInt division which is very slow.
+template <typename T>
+LIBC_INLINE cpp::enable_if_t<is_big_int_v<T>, uint8_t>
+extract_decimal_digit(T &value) {
+  // There are two essential ways you can turn n into (n/10,n%10). One is
+  // ordinary integer division. The other is a modular-arithmetic approach in
+  // which you first compute n%10 by bit twiddling, then subtract it off to get
+  // a value that is definitely a multiple of 10. Then you divide that by 10 in
+  // two steps: shift right to divide off a factor of 2, and then divide off a
+  // factor of 5 by multiplying by the modular inverse of 5 mod 2^BITS. (That
+  // last step only works if you know there's no remainder, which is why you
+  // had to subtract off the output digit first.)
+  //
+  // Either approach can be made to work in linear time. This code uses the
+  // modular-arithmetic technique, because the other approach either does a lot
+  // of integer divisions (requiring a fast hardware divider), or else uses a
+  // "multiply by an approximation to the reciprocal" technique which depends
+  // on careful error analysis which might go wrong in an untested edge case.
+
+  using Word = typename T::word_type;
+
+  // Find the remainder (value % 10). We do this by breaking up the input
+  // integer into chunks of size WORD_SIZE/2, so that the sum of them doesn't
+  // overflow a Word. Then we sum all the half-words times 6, except the bottom
+  // one, which is added to that sum without scaling.
+  //
+  // Why 6? Because you can imagine that the original number had the form
+  //
+  //   halfwords[0] + K*halfwords[1] + K^2*halfwords[2] + ...
+  //
+  // where K = 2^(WORD_SIZE/2). Since WORD_SIZE is expected to be a multiple of
+  // 8, that makes WORD_SIZE/2 a multiple of 4, so that K is a power of 16. And
+  // all powers of 16 (larger than 1) are congruent to 6 mod 10, by induction:
+  // 16 itself is, and 6^2=36 is also congruent to 6.
+  Word acc_remainder = 0;
+  constexpr Word HALFWORD_BITS = T::WORD_SIZE / 2;
+  constexpr Word HALFWORD_MASK = ((Word(1) << HALFWORD_BITS) - 1);
+  // Sum both halves of all words except the low one.
+  for (size_t i = 1; i < T::WORD_COUNT; i++) {
+    acc_remainder += value.val[i] >> HALFWORD_BITS;
+    acc_remainder += value.val[i] & HALFWORD_MASK;
+  }
+  // Add the high half of the low word. Then we have everything that needs to
+  // be multiplied by 6, so do that.
+  acc_remainder += value.val[0] >> HALFWORD_BITS;
+  acc_remainder *= 6;
+  // Having multiplied it by 6, add the lowest half-word, and then reduce mod
+  // 10 by normal integer division to finish.
+  acc_remainder += value.val[0] & HALFWORD_MASK;
+  uint8_t digit = static_cast<uint8_t>(acc_remainder % 10);
+
+  // Now we have the output digit. Subtract it from the input value, and shift
+  // right to divide by 2.
+  value -= digit;
+  value >>= 1;
+
+  // Now all that's left is to multiply by the inverse of 5 mod 2^BITS. No
+  // matter what the value of BITS, the inverse of 5 has the very convenient
+  // form 0xCCCC...CCCD, with as many C hex digits in the middle as necessary.
+  //
+  // We could construct a second BigInt with all words 0xCCCCCCCCCCCCCCCC,
+  // increment the bottom word, and call a general-purpose multiply function.
+  // But we can do better, by taking advantage of the regularity: we can do
+  // this particular operation in linear time, whereas a general multiplier
+  // would take superlinear time (quadratic in small cases).
+  //
+  // To begin with, instead of computing n*0xCCCC...CCCD, we'll compute
+  // n*0xCCCC...CCCC and then add it to the original n. Then all the words of
+  // the multiplier have the same value 0xCCCCCCCCCCCCCCCC, which I'll just
+  // denote as C. If we also write t = 2^WORD_SIZE, and imagine (as an example)
+  // that the input number has three words x,y,z with x being the low word,
+  // then we're computing
+  //
+  //   (x + y t + z t^2) * (C + C t + C t^2)
+  //
+  // = x C + y C t + z C t^2
+  //       + x C t + y C t^2 + z C t^3
+  //               + x C t^2 + y C t^3 + z C t^4
+  //
+  // but we're working mod t^3, so the high-order terms vanish and this becomes
+  //
+  //   x C + y C t + z C t^2
+  //       + x C t + y C t^2
+  //               + x C t^2
+  //
+  // = x C + (x+y) C t + (x+y+z) C t^2
+  //
+  // So all you have to do is to work from the low word of the integer upwards,
+  // accumulating C times the sum of all the words you've seen so far to get
+  // x*C, (x+y)*C, (x+y+z)*C and so on. In each step you add another product to
+  // the accumulator, and add the accumulator to the corresponding word of the
+  // original number (so that we end up with value*CCCD, not just value*CCCC).
+  //
+  // If you do that literally, then your accumulator has to be three words
+  // wide, because the sum of words can overflow into a second word, and
+  // multiplying by C adds another word. But we can do slightly better by
+  // breaking each product word*C up into a bottom half and a top half. If we
+  // write x*C = xl + xh*t, and similarly for y and z, then our sum becomes
+  //
+  //   (xl + xh t) + (yl + yh t) t + (zl + zh t) t^2
+  //               + (xl + xh t) t + (yl + yh t) t^2
+  //                               + (xl + xh t) t^2
+  //
+  // and if you expand out again, collect terms, and discard t^3 terms, you get
+  //
+  //   (xl)
+  // + (xl + xh + yl) t
+  // + (xl + xh + yl + yh + zl) t^2
+  //
+  // in which each coefficient is the sum of all the low words of the products
+  // up to _and including_ the current word, plus all the high words up to but
+  // _not_ including the current word. So now you only have to retain two words
+  // of sum instead of three.
+  //
+  // We do this entire procedure in a single in-place pass over the input
+  // number, reading each word to make its product with C and then adding the
+  // low word of the accumulator to it.
+  constexpr Word C = Word(-1) / 5 * 4; // calculate 0xCCCC as 4/5 of 0xFFFF
+  Word acc_lo = 0, acc_hi = 0; // accumulator of all the half-products so far
+  Word carry_bit, carry_word = 0;
+
+  for (size_t i = 0; i < T::WORD_COUNT; i++) {
+    // Make the two-word product of C with the current input word.
+    multiword::DoubleWide<Word> product = multiword::mul2(C, value.val[i]);
+
+    // Add the low half of the product to our accumulator, but not yet the high
+    // half.
+    acc_lo = add_with_carry<Word>(acc_lo, product[0], 0, carry_bit);
+    acc_hi += carry_bit;
+
+    // Now the accumulator contains exactly the value we need to add to the
+    // current input word. Add it, plus any carries from lower words, and make
+    // a new word of carry data to propagate into the next iteration.
+    value.val[i] = add_with_carry<Word>(value.val[i], carry_word, 0, carry_bit);
+    carry_word = acc_hi + carry_bit;
+    value.val[i] = add_with_carry<Word>(value.val[i], acc_lo, 0, carry_bit);
+    carry_word += carry_bit;
+
+    // Now add the high half of the current product to our accumulator.
+    acc_lo = add_with_carry<Word>(acc_lo, product[1], 0, carry_bit);
+    acc_hi += carry_bit;
+  }
+
+  return digit;
+}
+
 // See file header for documentation.
 template <typename T, typename Fmt = radix::Dec> class IntegerToString {
   static_assert(cpp::is_integral_v<T> || is_big_int_v<T>);
@@ -229,6 +393,15 @@ template <typename T, typename Fmt = radix::Dec> class IntegerToString {
       }
     }
 
+    LIBC_INLINE static void
+    write_unsigned_number_dec(UNSIGNED_T value,
+                              details::BackwardStringBufferWriter &sink) {
+      while (sink.ok() && value != 0) {
+        const uint8_t digit = extract_decimal_digit(value);
+        sink.push(digit_char(digit));
+      }
+    }
+
     // Returns the absolute value of 'value' as 'UNSIGNED_T'.
     LIBC_INLINE static UNSIGNED_T abs(T value) {
       if (cpp::is_unsigned_v<T> || value >= 0)
@@ -256,7 +429,7 @@ template <typename T, typename Fmt = radix::Dec> class IntegerToString {
     LIBC_INLINE static void write(T value,
                                   details::BackwardStringBufferWriter &sink) {
       if constexpr (Fmt::BASE == 10) {
-        write_unsigned_number(abs(value), sink);
+        write_unsigned_number_dec(abs(value), sink);
       } else {
         write_unsigned_number(static_cast<UNSIGNED_T>(value), sink);
       }
diff --git a/libc/src/__support/macros/optimization.h b/libc/src/__support/macros/optimization.h
index a2634950d431b..253843e5e37aa 100644
--- a/libc/src/__support/macros/optimization.h
+++ b/libc/src/__support/macros/optimization.h
@@ -45,6 +45,7 @@ LIBC_INLINE constexpr bool expects_bool_condition(T value, T expected) {
 #define LIBC_MATH_FAST                                                         \
   (LIBC_MATH_SKIP_ACCURATE_PASS | LIBC_MATH_SMALL_TABLES |                     \
    LIBC_MATH_NO_ERRNO | LIBC_MATH_NO_EXCEPT)
+#define LIBC_MATH_INTERMEDIATE_COMP_IN_FLOAT 0x10
 
 #ifndef LIBC_MATH
 #define LIBC_MATH 0
@@ -58,4 +59,8 @@ LIBC_INLINE constexpr bool expects_bool_condition(T value, T expected) {
 #define LIBC_MATH_HAS_SMALL_TABLES
 #endif
 
+#if (LIBC_MATH & LIBC_MATH_INTERMEDIATE_COMP_IN_FLOAT)
+#define LIBC_MATH_HAS_INTERMEDIATE_COMP_IN_FLOAT
+#endif
+
 #endif // LLVM_LIBC_SRC___SUPPORT_MACROS_OPTIMIZATION_H
diff --git a/libc/src/__support/sign.h b/libc/src/__support/sign.h
index 4a629e4488195..e0de0e0798acb 100644
--- a/libc/src/__support/sign.h
+++ b/libc/src/__support/sign.h
@@ -29,6 +29,8 @@ struct Sign {
   static const Sign POS;
   static const Sign NEG;
 
+  LIBC_INLINE constexpr Sign negate() const { return Sign(!is_negative); }
+
 private:
   LIBC_INLINE constexpr explicit Sign(bool is_negative)
       : is_negative(is_negative) {}
diff --git a/libc/src/complex/generic/CMakeLists.txt b/libc/src/complex/generic/CMakeLists.txt
index 3dae6f8a61495..f1c21e4ed7271 100644
--- a/libc/src/complex/generic/CMakeLists.txt
+++ b/libc/src/complex/generic/CMakeLists.txt
@@ -4,10 +4,8 @@ add_entrypoint_object(
     cproj.cpp
   HDRS
     ../cproj.h
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
   DEPENDS
-    libc.src.__support.complex_type
+    libc.src.__support.complex_basic_ops
 )
 
 add_entrypoint_object(
@@ -16,10 +14,8 @@ add_entrypoint_object(
     cprojf.cpp
   HDRS
     ../cprojf.h
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
   DEPENDS
-    libc.src.__support.complex_type
+    libc.src.__support.complex_basic_ops
 )
 
 add_entrypoint_object(
@@ -28,10 +24,8 @@ add_entrypoint_object(
     cprojl.cpp
   HDRS
     ../cprojl.h
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
   DEPENDS
-    libc.src.__support.complex_type
+    libc.src.__support.complex_basic_ops
 )
 
 add_entrypoint_object(
@@ -40,10 +34,8 @@ add_entrypoint_object(
     cprojf16.cpp
   HDRS
     ../cprojf16.h
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
   DEPENDS
-    libc.src.__support.complex_type
+    libc.src.__support.complex_basic_ops
     libc.src.__support.macros.properties.types
     libc.src.__support.macros.properties.complex_types
 )
@@ -54,10 +46,8 @@ add_entrypoint_object(
     cprojf128.cpp
   HDRS
     ../cprojf128.h
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
   DEPENDS
-    libc.src.__support.complex_type
+    libc.src.__support.complex_basic_ops
     libc.src.__support.macros.properties.types
     libc.src.__support.macros.properties.complex_types
 )
@@ -68,10 +58,8 @@ add_entrypoint_object(
     conj.cpp
   HDRS
     ../conj.h
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
   DEPENDS
-    libc.src.__support.complex_type
+    libc.src.__support.complex_basic_ops
 )
 
 add_entrypoint_object(
@@ -80,10 +68,8 @@ add_entrypoint_object(
     conjf.cpp
   HDRS
     ../conjf.h
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
   DEPENDS
-    libc.src.__support.complex_type
+    libc.src.__support.complex_basic_ops
 )
 
 add_entrypoint_object(
@@ -92,10 +78,8 @@ add_entrypoint_object(
     conjl.cpp
   HDRS
     ../conjl.h
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
   DEPENDS
-    libc.src.__support.complex_type
+    libc.src.__support.complex_basic_ops
 )
 
 add_entrypoint_object(
@@ -104,10 +88,8 @@ add_entrypoint_object(
     conjf16.cpp
   HDRS
     ../conjf16.h
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
   DEPENDS
-    libc.src.__support.complex_type
+    libc.src.__support.complex_basic_ops
     libc.src.__support.macros.properties.types
     libc.src.__support.macros.properties.complex_types
 )
@@ -118,10 +100,8 @@ add_entrypoint_object(
     conjf128.cpp
   HDRS
     ../conjf128.h
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
   DEPENDS
-    libc.src.__support.complex_type
+    libc.src.__support.complex_basic_ops
     libc.src.__support.macros.properties.types
     libc.src.__support.macros.properties.complex_types
 )
@@ -132,8 +112,6 @@ add_entrypoint_object(
     creal.cpp
   HDRS
     ../creal.h
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
   DEPENDS
     libc.src.__support.CPP.bit
     libc.src.__support.complex_type
@@ -145,8 +123,6 @@ add_entrypoint_object(
     crealf.cpp
   HDRS
     ../crealf.h
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
   DEPENDS
     libc.src.__support.CPP.bit
     libc.src.__support.complex_type
@@ -158,8 +134,6 @@ add_entrypoint_object(
     creall.cpp
   HDRS
     ../creall.h
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
   DEPENDS
     libc.src.__support.CPP.bit
     libc.src.__support.complex_type
@@ -171,8 +145,6 @@ add_entrypoint_object(
     crealf16.cpp
   HDRS
     ../crealf16.h
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
   DEPENDS
     libc.src.__support.CPP.bit
     libc.src.__support.complex_type
@@ -186,8 +158,6 @@ add_entrypoint_object(
     crealf128.cpp
   HDRS
     ../crealf128.h
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
   DEPENDS
     libc.src.__support.CPP.bit
     libc.src.__support.complex_type
@@ -201,8 +171,6 @@ add_entrypoint_object(
     cimag.cpp
   HDRS
     ../cimag.h
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
   DEPENDS
     libc.src.__support.CPP.bit
     libc.src.__support.complex_type
@@ -214,8 +182,6 @@ add_entrypoint_object(
     cimagf.cpp
   HDRS
     ../cimagf.h
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
   DEPENDS
     libc.src.__support.CPP.bit
     libc.src.__support.complex_type
@@ -227,8 +193,6 @@ add_entrypoint_object(
     cimagl.cpp
   HDRS
     ../cimagl.h
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
   DEPENDS
     libc.src.__support.CPP.bit
     libc.src.__support.complex_type
@@ -240,8 +204,6 @@ add_entrypoint_object(
     cimagf16.cpp
   HDRS
     ../cimagf16.h
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
   DEPENDS
     libc.src.__support.CPP.bit
     libc.src.__support.complex_type
@@ -255,8 +217,6 @@ add_entrypoint_object(
     cimagf128.cpp
   HDRS
     ../cimagf128.h
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
   DEPENDS
     libc.src.__support.CPP.bit
     libc.src.__support.complex_type
diff --git a/libc/src/complex/generic/conj.cpp b/libc/src/complex/generic/conj.cpp
index cbcd480d6efa5..ec1f5285f6faf 100644
--- a/libc/src/complex/generic/conj.cpp
+++ b/libc/src/complex/generic/conj.cpp
@@ -8,7 +8,7 @@
 
 #include "src/complex/conj.h"
 #include "src/__support/common.h"
-#include "src/__support/complex_type.h"
+#include "src/__support/complex_basic_ops.h"
 
 namespace LIBC_NAMESPACE_DECL {
 
diff --git a/libc/src/complex/generic/conjf.cpp b/libc/src/complex/generic/conjf.cpp
index a1af3d78ebc6a..bf376bc9e87b6 100644
--- a/libc/src/complex/generic/conjf.cpp
+++ b/libc/src/complex/generic/conjf.cpp
@@ -8,7 +8,7 @@
 
 #include "src/complex/conjf.h"
 #include "src/__support/common.h"
-#include "src/__support/complex_type.h"
+#include "src/__support/complex_basic_ops.h"
 
 namespace LIBC_NAMESPACE_DECL {
 
diff --git a/libc/src/complex/generic/conjf128.cpp b/libc/src/complex/generic/conjf128.cpp
index a63809a66e25a..d5faba789fe30 100644
--- a/libc/src/complex/generic/conjf128.cpp
+++ b/libc/src/complex/generic/conjf128.cpp
@@ -8,7 +8,7 @@
 
 #include "src/complex/conjf128.h"
 #include "src/__support/common.h"
-#include "src/__support/complex_type.h"
+#include "src/__support/complex_basic_ops.h"
 
 namespace LIBC_NAMESPACE_DECL {
 
diff --git a/libc/src/complex/generic/conjf16.cpp b/libc/src/complex/generic/conjf16.cpp
index cd1ab67ed1cd9..31e8f9358e5e7 100644
--- a/libc/src/complex/generic/conjf16.cpp
+++ b/libc/src/complex/generic/conjf16.cpp
@@ -8,7 +8,7 @@
 
 #include "src/complex/conjf16.h"
 #include "src/__support/common.h"
-#include "src/__support/complex_type.h"
+#include "src/__support/complex_basic_ops.h"
 
 namespace LIBC_NAMESPACE_DECL {
 
diff --git a/libc/src/complex/generic/conjl.cpp b/libc/src/complex/generic/conjl.cpp
index 8298ede6fa38f..351fa60b1faec 100644
--- a/libc/src/complex/generic/conjl.cpp
+++ b/libc/src/complex/generic/conjl.cpp
@@ -8,7 +8,7 @@
 
 #include "src/complex/conjl.h"
 #include "src/__support/common.h"
-#include "src/__support/complex_type.h"
+#include "src/__support/complex_basic_ops.h"
 
 namespace LIBC_NAMESPACE_DECL {
 
diff --git a/libc/src/complex/generic/cproj.cpp b/libc/src/complex/generic/cproj.cpp
index d5e8c3ff3d9ec..2a9f4578afbbf 100644
--- a/libc/src/complex/generic/cproj.cpp
+++ b/libc/src/complex/generic/cproj.cpp
@@ -8,7 +8,7 @@
 
 #include "src/complex/cproj.h"
 #include "src/__support/common.h"
-#include "src/__support/complex_type.h"
+#include "src/__support/complex_basic_ops.h"
 
 namespace LIBC_NAMESPACE_DECL {
 
diff --git a/libc/src/complex/generic/cprojf.cpp b/libc/src/complex/generic/cprojf.cpp
index d0235f6bfef7e..d4e83dd21ff1c 100644
--- a/libc/src/complex/generic/cprojf.cpp
+++ b/libc/src/complex/generic/cprojf.cpp
@@ -8,7 +8,7 @@
 
 #include "src/complex/cprojf.h"
 #include "src/__support/common.h"
-#include "src/__support/complex_type.h"
+#include "src/__support/complex_basic_ops.h"
 
 namespace LIBC_NAMESPACE_DECL {
 
diff --git a/libc/src/complex/generic/cprojf128.cpp b/libc/src/complex/generic/cprojf128.cpp
index eb2cd08dfc117..72a741d815032 100644
--- a/libc/src/complex/generic/cprojf128.cpp
+++ b/libc/src/complex/generic/cprojf128.cpp
@@ -8,7 +8,7 @@
 
 #include "src/complex/cprojf128.h"
 #include "src/__support/common.h"
-#include "src/__support/complex_type.h"
+#include "src/__support/complex_basic_ops.h"
 
 namespace LIBC_NAMESPACE_DECL {
 
diff --git a/libc/src/complex/generic/cprojf16.cpp b/libc/src/complex/generic/cprojf16.cpp
index 8d2d64a439e02..0928345503bec 100644
--- a/libc/src/complex/generic/cprojf16.cpp
+++ b/libc/src/complex/generic/cprojf16.cpp
@@ -8,7 +8,7 @@
 
 #include "src/complex/cprojf16.h"
 #include "src/__support/common.h"
-#include "src/__support/complex_type.h"
+#include "src/__support/complex_basic_ops.h"
 
 namespace LIBC_NAMESPACE_DECL {
 
diff --git a/libc/src/complex/generic/cprojl.cpp b/libc/src/complex/generic/cprojl.cpp
index 34deeb63b16d0..849a6f72755f4 100644
--- a/libc/src/complex/generic/cprojl.cpp
+++ b/libc/src/complex/generic/cprojl.cpp
@@ -8,7 +8,7 @@
 
 #include "src/complex/cprojl.h"
 #include "src/__support/common.h"
-#include "src/__support/complex_type.h"
+#include "src/__support/complex_basic_ops.h"
 
 namespace LIBC_NAMESPACE_DECL {
 
diff --git a/libc/src/dirent/readdir.h b/libc/src/dirent/readdir.h
index 2dcf5360ebd35..bc46a1bbfbf4a 100644
--- a/libc/src/dirent/readdir.h
+++ b/libc/src/dirent/readdir.h
@@ -9,7 +9,7 @@
 #ifndef LLVM_LIBC_SRC_DIRENT_READDIR_H
 #define LLVM_LIBC_SRC_DIRENT_READDIR_H
 
-#include "src/__support/macros/config.h"
+#include "src/__support/macros/config.h" // LIBC_NAMESPACE_DECL
 #include <dirent.h>
 
 namespace LIBC_NAMESPACE_DECL {
diff --git a/libc/src/math/CMakeLists.txt b/libc/src/math/CMakeLists.txt
index fe5ebd793b40a..82551a4b57f45 100644
--- a/libc/src/math/CMakeLists.txt
+++ b/libc/src/math/CMakeLists.txt
@@ -47,6 +47,8 @@ add_math_entrypoint_object(acoshf)
 
 add_math_entrypoint_object(asin)
 add_math_entrypoint_object(asinf)
+add_math_entrypoint_object(asinf16)
+
 add_math_entrypoint_object(asinh)
 add_math_entrypoint_object(asinhf)
 
diff --git a/libc/src/math/asinf16.h b/libc/src/math/asinf16.h
new file mode 100644
index 0000000000000..f16647ec2a6f9
--- /dev/null
+++ b/libc/src/math/asinf16.h
@@ -0,0 +1,21 @@
+//===-- Implementation header for asinf16 -----------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_MATH_ASINF16_H
+#define LLVM_LIBC_SRC_MATH_ASINF16_H
+
+#include "src/__support/macros/config.h"
+#include "src/__support/macros/properties/types.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+float16 asinf16(float16 x);
+
+} // namespace LIBC_NAMESPACE_DECL
+
+#endif // LLVM_LIBC_SRC_MATH_ASINF16_H
diff --git a/libc/src/math/generic/CMakeLists.txt b/libc/src/math/generic/CMakeLists.txt
index 0e57051807b33..2bda741b453f5 100644
--- a/libc/src/math/generic/CMakeLists.txt
+++ b/libc/src/math/generic/CMakeLists.txt
@@ -6,8 +6,6 @@ add_entrypoint_object(
     canonicalize.cpp
   HDRS
     ../canonicalize.h
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
   DEPENDS
     libc.src.__support.FPUtil.basic_operations
 )
@@ -18,8 +16,6 @@ add_entrypoint_object(
     canonicalizef.cpp
   HDRS
     ../canonicalizef.h
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
   DEPENDS
     libc.src.__support.FPUtil.basic_operations
 )
@@ -30,8 +26,6 @@ add_entrypoint_object(
     canonicalizef16.cpp
   HDRS
     ../canonicalizef16.h
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
   DEPENDS
     libc.src.__support.macros.properties.types
     libc.src.__support.FPUtil.basic_operations
@@ -43,8 +37,6 @@ add_entrypoint_object(
     canonicalizef128.cpp
   HDRS
     ../canonicalizef128.h
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
   DEPENDS
     libc.src.__support.macros.properties.types
     libc.src.__support.FPUtil.basic_operations
@@ -56,8 +48,6 @@ add_entrypoint_object(
     canonicalizel.cpp
   HDRS
     ../canonicalizel.h
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
   DEPENDS
     libc.src.__support.FPUtil.basic_operations
 )
@@ -68,8 +58,6 @@ add_entrypoint_object(
     iscanonical.cpp
   HDRS
     ../iscanonical.h
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
 )
 
 add_entrypoint_object(
@@ -78,8 +66,6 @@ add_entrypoint_object(
     iscanonicalf.cpp
   HDRS
     ../iscanonicalf.h
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
 )
 
 add_entrypoint_object(
@@ -88,8 +74,6 @@ add_entrypoint_object(
     iscanonicall.cpp
   HDRS
     ../iscanonicall.h
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
 )
 
 add_entrypoint_object(
@@ -98,8 +82,6 @@ add_entrypoint_object(
     iscanonicalf16.cpp
   HDRS
     ../iscanonicalf16.h
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
   DEPENDS
     libc.src.__support.macros.properties.types
 )
@@ -110,8 +92,6 @@ add_entrypoint_object(
     iscanonicalf128.cpp
   HDRS
     ../iscanonicalf128.h
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
   DEPENDS
     libc.src.__support.macros.properties.types
 )
@@ -122,8 +102,6 @@ add_entrypoint_object(
     ceil.cpp
   HDRS
     ../ceil.h
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
   DEPENDS
     libc.src.__support.FPUtil.nearest_integer_operations
   FLAGS
@@ -136,8 +114,6 @@ add_entrypoint_object(
     ceilf.cpp
   HDRS
     ../ceilf.h
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
   DEPENDS
     libc.src.__support.FPUtil.nearest_integer_operations
   FLAGS
@@ -150,8 +126,6 @@ add_entrypoint_object(
     ceill.cpp
   HDRS
     ../ceill.h
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
   DEPENDS
     libc.src.__support.FPUtil.nearest_integer_operations
 )
@@ -162,8 +136,6 @@ add_entrypoint_object(
     ceilf16.cpp
   HDRS
     ../ceilf16.h
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
   DEPENDS
     libc.src.__support.FPUtil.cast
     libc.src.__support.FPUtil.nearest_integer_operations
@@ -179,8 +151,6 @@ add_entrypoint_object(
     ceilf128.cpp
   HDRS
     ../ceilf128.h
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
   DEPENDS
     libc.src.__support.macros.properties.types
     libc.src.__support.FPUtil.nearest_integer_operations
@@ -192,8 +162,6 @@ add_entrypoint_object(
     daddl.cpp
   HDRS
     ../daddl.h
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
   DEPENDS
     libc.src.__support.FPUtil.generic.add_sub
 )
@@ -204,8 +172,6 @@ add_entrypoint_object(
     daddf128.cpp
   HDRS
     ../daddf128.h
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
  DEPENDS
     libc.src.__support.macros.properties.types
     libc.src.__support.FPUtil.generic.add_sub
@@ -217,8 +183,6 @@ add_entrypoint_object(
     ddivl.cpp
   HDRS
     ../ddivl.h
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
   DEPENDS
     libc.src.__support.FPUtil.generic.div
 )
@@ -229,8 +193,6 @@ add_entrypoint_object(
     ddivf128.cpp
   HDRS
     ../ddivf128.h
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
   DEPENDS
     libc.src.__support.macros.properties.types
     libc.src.__support.FPUtil.generic.div
@@ -245,8 +207,6 @@ add_entrypoint_object(
   DEPENDS
     libc.src.__support.FPUtil.fma
     libc.src.__support.macros.properties.types
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
 )
 
 add_entrypoint_object(
@@ -257,8 +217,6 @@ add_entrypoint_object(
     ../dfmal.h
   DEPENDS
     libc.src.__support.FPUtil.fma
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
 )
 
 add_entrypoint_object(
@@ -269,8 +227,6 @@ add_entrypoint_object(
     ../dsqrtl.h
   DEPENDS
     libc.src.__support.FPUtil.generic.sqrt
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
 )
 
 add_entrypoint_object(
@@ -282,8 +238,6 @@ add_entrypoint_object(
   DEPENDS
     libc.src.__support.macros.properties.types
     libc.src.__support.FPUtil.generic.sqrt
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
 )
 
 
@@ -296,8 +250,6 @@ add_entrypoint_object(
   DEPENDS
     libc.src.__support.FPUtil.generic.add_sub
     libc.src.__support.macros.properties.types
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
 )
 
 add_entrypoint_object(
@@ -308,8 +260,6 @@ add_entrypoint_object(
     ../dsubl.h
   DEPENDS
     libc.src.__support.FPUtil.generic.add_sub
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
 )
 
 add_header_library(
@@ -392,8 +342,6 @@ add_entrypoint_object(
     libc.src.__support.FPUtil.fenv_impl
     libc.src.__support.FPUtil.fp_bits
     libc.src.__support.macros.optimization
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
 )
 
 add_entrypoint_object(
@@ -413,8 +361,6 @@ add_entrypoint_object(
     libc.src.__support.FPUtil.multiply_add
     libc.src.__support.FPUtil.polyeval
     libc.src.__support.macros.optimization
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
 )
 
 add_entrypoint_object(
@@ -449,8 +395,6 @@ add_entrypoint_object(
     libc.src.__support.FPUtil.fma
     libc.src.__support.FPUtil.multiply_add
     libc.src.__support.macros.optimization
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
 )
 
 add_entrypoint_object(
@@ -468,8 +412,6 @@ add_entrypoint_object(
     libc.src.__support.FPUtil.fp_bits
     libc.src.__support.FPUtil.multiply_add
     libc.src.__support.macros.optimization
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
 )
 
 add_entrypoint_object(
@@ -490,8 +432,6 @@ add_entrypoint_object(
     libc.src.__support.FPUtil.multiply_add
     libc.src.__support.FPUtil.rounding_mode
     libc.src.__support.macros.optimization
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
 )
 
 add_entrypoint_object(
@@ -512,8 +452,6 @@ add_entrypoint_object(
     libc.src.__support.FPUtil.polyeval
     libc.src.__support.FPUtil.rounding_mode
     libc.src.__support.macros.optimization
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
 )
 
 add_entrypoint_object(
@@ -534,7 +472,7 @@ add_entrypoint_object(
     libc.src.__support.macros.optimization
     libc.src.__support.macros.properties.types
   COMPILE_OPTIONS
-    -O3
+    ${libc_opt_high_flag}
 )
 
 add_entrypoint_object(
@@ -555,8 +493,6 @@ add_entrypoint_object(
     libc.src.__support.FPUtil.fp_bits
     libc.src.__support.FPUtil.multiply_add
     libc.src.__support.macros.optimization
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
 )
 
 add_entrypoint_object(
@@ -574,8 +510,6 @@ add_entrypoint_object(
     libc.src.__support.FPUtil.polyeval
     libc.src.__support.common
     libc.src.__support.macros.optimization
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
 )
 
 add_entrypoint_object(
@@ -595,8 +529,6 @@ add_entrypoint_object(
     libc.src.__support.FPUtil.polyeval
     libc.src.__support.FPUtil.rounding_mode
     libc.src.__support.macros.optimization
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
 )
 
 add_entrypoint_object(
@@ -614,8 +546,6 @@ add_entrypoint_object(
     libc.src.__support.FPUtil.fp_bits
     libc.src.__support.FPUtil.multiply_add
     libc.src.__support.macros.optimization
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
 )
 
 add_entrypoint_object(
@@ -635,8 +565,6 @@ add_entrypoint_object(
     libc.src.__support.FPUtil.fp_bits
     libc.src.__support.FPUtil.multiply_add
     libc.src.__support.macros.optimization
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
 )
 
 add_entrypoint_object(
@@ -657,8 +585,6 @@ add_entrypoint_object(
     libc.src.__support.FPUtil.multiply_add
     libc.src.__support.FPUtil.polyeval
     libc.src.__support.macros.optimization
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
 )
 
 add_entrypoint_object(
@@ -696,8 +622,6 @@ add_entrypoint_object(
     libc.src.__support.FPUtil.except_value_utils
     libc.src.__support.FPUtil.multiply_add
     libc.src.__support.macros.optimization
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
 )
 
 add_entrypoint_object(
@@ -708,8 +632,6 @@ add_entrypoint_object(
     ../fabs.h
   DEPENDS
     libc.src.__support.FPUtil.basic_operations
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
   FLAGS
     MISC_MATH_BASIC_OPS_OPT
 )
@@ -722,8 +644,6 @@ add_entrypoint_object(
     ../fabsf.h
   DEPENDS
     libc.src.__support.FPUtil.basic_operations
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
   FLAGS
     MISC_MATH_BASIC_OPS_OPT
 )
@@ -736,8 +656,6 @@ add_entrypoint_object(
     ../fabsl.h
   DEPENDS
     libc.src.__support.FPUtil.basic_operations
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
 )
 
 add_entrypoint_object(
@@ -751,8 +669,6 @@ add_entrypoint_object(
     libc.src.__support.FPUtil.basic_operations
     libc.src.__support.macros.properties.architectures
     libc.src.__support.macros.properties.compiler
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
   FLAGS
     MISC_MATH_BASIC_OPS_OPT
 )
@@ -766,8 +682,6 @@ add_entrypoint_object(
   DEPENDS
     libc.src.__support.macros.properties.types
     libc.src.__support.FPUtil.basic_operations
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
 )
 
 add_entrypoint_object(
@@ -778,8 +692,6 @@ add_entrypoint_object(
     ../fadd.h
   DEPENDS
     libc.src.__support.FPUtil.generic.add_sub
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
 )
 
 add_entrypoint_object(
@@ -790,8 +702,6 @@ add_entrypoint_object(
     ../faddl.h
   DEPENDS
     libc.src.__support.FPUtil.generic.add_sub
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
 )
 
 add_entrypoint_object(
@@ -803,8 +713,6 @@ add_entrypoint_object(
   DEPENDS
     libc.src.__support.FPUtil.generic.add_sub
     libc.src.__support.macros.properties.types
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
 )
 
 add_entrypoint_object(
@@ -813,8 +721,6 @@ add_entrypoint_object(
     trunc.cpp
   HDRS
     ../trunc.h
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
   DEPENDS
     libc.src.__support.FPUtil.nearest_integer_operations
   FLAGS
@@ -827,8 +733,6 @@ add_entrypoint_object(
     truncf.cpp
   HDRS
     ../truncf.h
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
   DEPENDS
     libc.src.__support.FPUtil.nearest_integer_operations
   FLAGS
@@ -841,8 +745,6 @@ add_entrypoint_object(
     truncl.cpp
   HDRS
     ../truncl.h
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
   DEPENDS
     libc.src.__support.FPUtil.nearest_integer_operations
 )
@@ -853,8 +755,6 @@ add_entrypoint_object(
     truncf16.cpp
   HDRS
     ../truncf16.h
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
   DEPENDS
     libc.src.__support.FPUtil.cast
     libc.src.__support.FPUtil.nearest_integer_operations
@@ -870,8 +770,6 @@ add_entrypoint_object(
     truncf128.cpp
   HDRS
     ../truncf128.h
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
   DEPENDS
     libc.src.__support.macros.properties.types
     libc.src.__support.FPUtil.nearest_integer_operations
@@ -883,8 +781,6 @@ add_entrypoint_object(
     floor.cpp
   HDRS
     ../floor.h
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
   DEPENDS
     libc.src.__support.FPUtil.nearest_integer_operations
   FLAGS
@@ -897,8 +793,6 @@ add_entrypoint_object(
     floorf.cpp
   HDRS
     ../floorf.h
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
   DEPENDS
     libc.src.__support.FPUtil.nearest_integer_operations
   FLAGS
@@ -911,8 +805,6 @@ add_entrypoint_object(
     floorl.cpp
   HDRS
     ../floorl.h
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
   DEPENDS
     libc.src.__support.FPUtil.nearest_integer_operations
 )
@@ -923,8 +815,6 @@ add_entrypoint_object(
     floorf16.cpp
   HDRS
     ../floorf16.h
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
   DEPENDS
     libc.src.__support.FPUtil.cast
     libc.src.__support.FPUtil.nearest_integer_operations
@@ -940,8 +830,6 @@ add_entrypoint_object(
     floorf128.cpp
   HDRS
     ../floorf128.h
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
   DEPENDS
     libc.src.__support.macros.properties.types
     libc.src.__support.FPUtil.nearest_integer_operations
@@ -953,8 +841,6 @@ add_entrypoint_object(
     round.cpp
   HDRS
     ../round.h
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
   DEPENDS
     libc.src.__support.FPUtil.nearest_integer_operations
   FLAGS
@@ -967,8 +853,6 @@ add_entrypoint_object(
     roundf.cpp
   HDRS
     ../roundf.h
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
   DEPENDS
     libc.src.__support.FPUtil.nearest_integer_operations
   FLAGS
@@ -981,8 +865,6 @@ add_entrypoint_object(
     roundl.cpp
   HDRS
     ../roundl.h
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
   DEPENDS
     libc.src.__support.FPUtil.nearest_integer_operations
 )
@@ -993,8 +875,6 @@ add_entrypoint_object(
     roundf16.cpp
   HDRS
     ../roundf16.h
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
   DEPENDS
     libc.src.__support.FPUtil.cast
     libc.src.__support.FPUtil.nearest_integer_operations
@@ -1010,8 +890,6 @@ add_entrypoint_object(
     roundf128.cpp
   HDRS
     ../roundf128.h
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
   DEPENDS
     libc.src.__support.macros.properties.types
     libc.src.__support.FPUtil.nearest_integer_operations
@@ -1023,8 +901,6 @@ add_entrypoint_object(
     roundeven.cpp
   HDRS
     ../roundeven.h
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
   DEPENDS
     libc.src.__support.FPUtil.nearest_integer_operations
   FLAGS
@@ -1037,8 +913,6 @@ add_entrypoint_object(
     roundevenf.cpp
   HDRS
     ../roundevenf.h
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
   DEPENDS
     libc.src.__support.FPUtil.nearest_integer_operations
   FLAGS
@@ -1051,8 +925,6 @@ add_entrypoint_object(
     roundevenl.cpp
   HDRS
     ../roundevenl.h
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
   DEPENDS
     libc.src.__support.FPUtil.nearest_integer_operations
 )
@@ -1063,8 +935,6 @@ add_entrypoint_object(
     roundevenf16.cpp
   HDRS
     ../roundevenf16.h
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
   DEPENDS
     libc.src.__support.macros.properties.types
     libc.src.__support.FPUtil.nearest_integer_operations
@@ -1080,8 +950,6 @@ add_entrypoint_object(
     roundevenf128.cpp
   HDRS
     ../roundevenf128.h
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
   DEPENDS
     libc.src.__support.macros.properties.types
     libc.src.__support.FPUtil.nearest_integer_operations
@@ -1093,8 +961,6 @@ add_entrypoint_object(
     lround.cpp
   HDRS
     ../lround.h
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
   DEPENDS
     libc.src.__support.FPUtil.nearest_integer_operations
 )
@@ -1105,8 +971,6 @@ add_entrypoint_object(
     lroundf.cpp
   HDRS
     ../lroundf.h
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
   DEPENDS
     libc.src.__support.FPUtil.nearest_integer_operations
 )
@@ -1117,8 +981,6 @@ add_entrypoint_object(
     lroundl.cpp
   HDRS
     ../lroundl.h
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
   DEPENDS
     libc.src.__support.FPUtil.nearest_integer_operations
 )
@@ -1129,8 +991,6 @@ add_entrypoint_object(
     lroundf16.cpp
   HDRS
     ../lroundf16.h
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
   DEPENDS
     libc.src.__support.macros.properties.types
     libc.src.__support.FPUtil.nearest_integer_operations
@@ -1142,8 +1002,6 @@ add_entrypoint_object(
     lroundf128.cpp
   HDRS
     ../lroundf128.h
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
   DEPENDS
     libc.src.__support.macros.properties.types
     libc.src.__support.FPUtil.nearest_integer_operations
@@ -1155,8 +1013,6 @@ add_entrypoint_object(
     llround.cpp
   HDRS
     ../llround.h
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
   DEPENDS
     libc.src.__support.FPUtil.nearest_integer_operations
 )
@@ -1167,8 +1023,6 @@ add_entrypoint_object(
     llroundf.cpp
   HDRS
     ../llroundf.h
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
   DEPENDS
     libc.src.__support.FPUtil.nearest_integer_operations
 )
@@ -1179,8 +1033,6 @@ add_entrypoint_object(
     llroundl.cpp
   HDRS
     ../llroundl.h
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
   DEPENDS
     libc.src.__support.FPUtil.nearest_integer_operations
 )
@@ -1191,8 +1043,6 @@ add_entrypoint_object(
     llroundf16.cpp
   HDRS
     ../llroundf16.h
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
   DEPENDS
     libc.src.__support.macros.properties.types
     libc.src.__support.FPUtil.nearest_integer_operations
@@ -1204,8 +1054,6 @@ add_entrypoint_object(
     llroundf128.cpp
   HDRS
     ../llroundf128.h
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
   DEPENDS
     libc.src.__support.macros.properties.types
     libc.src.__support.FPUtil.nearest_integer_operations
@@ -1217,8 +1065,6 @@ add_entrypoint_object(
     rint.cpp
   HDRS
     ../rint.h
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
   DEPENDS
     libc.src.__support.FPUtil.nearest_integer_operations
   FLAGS
@@ -1231,8 +1077,6 @@ add_entrypoint_object(
     rintf.cpp
   HDRS
     ../rintf.h
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
   DEPENDS
     libc.src.__support.FPUtil.nearest_integer_operations
   FLAGS
@@ -1245,8 +1089,6 @@ add_entrypoint_object(
     rintl.cpp
   HDRS
     ../rintl.h
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
   DEPENDS
     libc.src.__support.FPUtil.nearest_integer_operations
 )
@@ -1257,8 +1099,6 @@ add_entrypoint_object(
     rintf16.cpp
   HDRS
     ../rintf16.h
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
   DEPENDS
     libc.src.__support.FPUtil.cast
     libc.src.__support.FPUtil.nearest_integer_operations
@@ -1274,8 +1114,6 @@ add_entrypoint_object(
     rintf128.cpp
   HDRS
     ../rintf128.h
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
   DEPENDS
     libc.src.__support.macros.properties.types
     libc.src.__support.FPUtil.nearest_integer_operations
@@ -1287,8 +1125,6 @@ add_entrypoint_object(
     lrint.cpp
   HDRS
     ../lrint.h
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
   DEPENDS
     libc.src.__support.FPUtil.nearest_integer_operations
 )
@@ -1299,8 +1135,6 @@ add_entrypoint_object(
     lrintf.cpp
   HDRS
     ../lrintf.h
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
   DEPENDS
     libc.src.__support.FPUtil.nearest_integer_operations
 )
@@ -1311,8 +1145,6 @@ add_entrypoint_object(
     lrintl.cpp
   HDRS
     ../lrintl.h
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
   DEPENDS
     libc.src.__support.FPUtil.nearest_integer_operations
 )
@@ -1323,8 +1155,6 @@ add_entrypoint_object(
     lrintf16.cpp
   HDRS
     ../lrintf16.h
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
   DEPENDS
     libc.src.__support.macros.properties.types
     libc.src.__support.FPUtil.nearest_integer_operations
@@ -1336,8 +1166,6 @@ add_entrypoint_object(
     lrintf128.cpp
   HDRS
     ../lrintf128.h
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
   DEPENDS
     libc.src.__support.macros.properties.types
     libc.src.__support.FPUtil.nearest_integer_operations
@@ -1349,8 +1177,6 @@ add_entrypoint_object(
     llrint.cpp
   HDRS
     ../llrint.h
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
   DEPENDS
     libc.src.__support.FPUtil.nearest_integer_operations
 )
@@ -1361,8 +1187,6 @@ add_entrypoint_object(
     llrintf.cpp
   HDRS
     ../llrintf.h
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
   DEPENDS
     libc.src.__support.FPUtil.nearest_integer_operations
 )
@@ -1373,8 +1197,6 @@ add_entrypoint_object(
     llrintl.cpp
   HDRS
     ../llrintl.h
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
   DEPENDS
     libc.src.__support.FPUtil.nearest_integer_operations
 )
@@ -1385,8 +1207,6 @@ add_entrypoint_object(
     llrintf16.cpp
   HDRS
     ../llrintf16.h
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
   DEPENDS
     libc.src.__support.macros.properties.types
     libc.src.__support.FPUtil.nearest_integer_operations
@@ -1398,8 +1218,6 @@ add_entrypoint_object(
     llrintf128.cpp
   HDRS
     ../llrintf128.h
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
   DEPENDS
     libc.src.__support.macros.properties.types
     libc.src.__support.FPUtil.nearest_integer_operations
@@ -1413,8 +1231,6 @@ add_entrypoint_object(
     ../nearbyint.h
   DEPENDS
     libc.src.__support.FPUtil.nearest_integer_operations
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
 )
 
 add_entrypoint_object(
@@ -1425,8 +1241,6 @@ add_entrypoint_object(
     ../nearbyintf.h
   DEPENDS
     libc.src.__support.FPUtil.nearest_integer_operations
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
 )
 
 add_entrypoint_object(
@@ -1437,8 +1251,6 @@ add_entrypoint_object(
     ../nearbyintl.h
   DEPENDS
     libc.src.__support.FPUtil.nearest_integer_operations
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
 )
 
 add_entrypoint_object(
@@ -1450,8 +1262,6 @@ add_entrypoint_object(
   DEPENDS
     libc.src.__support.macros.properties.types
     libc.src.__support.FPUtil.nearest_integer_operations
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
 )
 
 add_entrypoint_object(
@@ -1463,8 +1273,6 @@ add_entrypoint_object(
   DEPENDS
     libc.src.__support.macros.properties.types
     libc.src.__support.FPUtil.nearest_integer_operations
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
 )
 
 add_object_library(
@@ -1488,8 +1296,6 @@ add_entrypoint_object(
     libc.src.__support.FPUtil.multiply_add
     libc.src.__support.FPUtil.polyeval
     libc.src.__support.macros.optimization
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
 )
 
 add_entrypoint_object(
@@ -1514,8 +1320,6 @@ add_entrypoint_object(
     libc.src.__support.integer_literals
     libc.src.__support.macros.optimization
     libc.src.errno.errno
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
 )
 
 add_entrypoint_object(
@@ -1535,8 +1339,6 @@ add_entrypoint_object(
     libc.src.__support.FPUtil.rounding_mode
     libc.src.__support.macros.optimization
     libc.src.errno.errno
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
 )
 
 add_entrypoint_object(
@@ -1559,8 +1361,6 @@ add_entrypoint_object(
     libc.src.__support.FPUtil.polyeval
     libc.src.__support.FPUtil.rounding_mode
     libc.src.__support.macros.optimization
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
 )
 
 add_entrypoint_object(
@@ -1585,8 +1385,6 @@ add_entrypoint_object(
     libc.src.__support.integer_literals
     libc.src.__support.macros.optimization
     libc.src.errno.errno
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
 )
 
 add_header_library(
@@ -1615,8 +1413,6 @@ add_entrypoint_object(
     ../exp2f.h
   DEPENDS
     .exp2f_impl
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
 )
 
 add_entrypoint_object(
@@ -1635,8 +1431,6 @@ add_entrypoint_object(
     libc.src.__support.FPUtil.fp_bits
     libc.src.__support.FPUtil.rounding_mode
     libc.src.__support.macros.optimization
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
 )
 
 add_entrypoint_object(
@@ -1657,8 +1451,6 @@ add_entrypoint_object(
     libc.src.__support.FPUtil.rounding_mode
     libc.src.__support.macros.optimization
     libc.src.__support.macros.properties.cpu_features
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
 )
 
 add_entrypoint_object(
@@ -1681,8 +1473,6 @@ add_entrypoint_object(
     libc.src.__support.FPUtil.rounding_mode
     libc.src.__support.macros.optimization
     libc.src.__support.macros.properties.cpu_features
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
 )
 
 add_entrypoint_object(
@@ -1707,8 +1497,6 @@ add_entrypoint_object(
     libc.src.__support.integer_literals
     libc.src.__support.macros.optimization
     libc.src.errno.errno
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
 )
 
 add_header_library(
@@ -1724,8 +1512,6 @@ add_header_library(
     libc.src.__support.macros.optimization
     libc.src.__support.common
     libc.src.errno.errno
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
 )
 
 add_entrypoint_object(
@@ -1736,8 +1522,6 @@ add_entrypoint_object(
     ../exp10f.h
   DEPENDS
     .exp10f_impl
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
 )
 
 add_entrypoint_object(
@@ -1761,8 +1545,6 @@ add_entrypoint_object(
     libc.src.__support.FPUtil.rounding_mode
     libc.src.__support.macros.optimization
     libc.src.__support.macros.properties.cpu_features
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
 )
 
 add_entrypoint_object(
@@ -1782,8 +1564,6 @@ add_entrypoint_object(
     libc.src.__support.FPUtil.polyeval
     libc.src.__support.FPUtil.rounding_mode
     libc.src.__support.macros.optimization
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
 )
 
 add_entrypoint_object(
@@ -1805,8 +1585,6 @@ add_entrypoint_object(
     libc.src.__support.FPUtil.rounding_mode
     libc.src.__support.macros.optimization
     libc.src.__support.macros.properties.cpu_features
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
 )
 
 add_entrypoint_object(
@@ -1831,8 +1609,6 @@ add_entrypoint_object(
     libc.src.__support.integer_literals
     libc.src.__support.macros.optimization
     libc.src.errno.errno
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
 )
 
 add_entrypoint_object(
@@ -1852,8 +1628,6 @@ add_entrypoint_object(
     libc.src.__support.FPUtil.rounding_mode
     libc.src.__support.macros.optimization
     libc.src.errno.errno
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
 )
 
 add_entrypoint_object(
@@ -1874,8 +1648,6 @@ add_entrypoint_object(
     libc.src.__support.FPUtil.polyeval
     libc.src.__support.FPUtil.rounding_mode
     libc.src.__support.macros.optimization
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
 )
 
 add_entrypoint_object(
@@ -1901,8 +1673,6 @@ add_entrypoint_object(
     libc.src.__support.FPUtil.triple_double
     libc.src.__support.macros.optimization
     libc.src.errno.errno
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
 )
 
 add_entrypoint_object(
@@ -1924,8 +1694,6 @@ add_entrypoint_object(
     libc.src.__support.FPUtil.polyeval
     libc.src.__support.FPUtil.sqrt
     libc.src.__support.macros.optimization
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
 )
 
 add_entrypoint_object(
@@ -1936,8 +1704,6 @@ add_entrypoint_object(
     ../copysign.h
   DEPENDS
     libc.src.__support.FPUtil.manipulation_functions
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
   FLAGS
     MISC_MATH_BASIC_OPS_OPT
 )
@@ -1950,8 +1716,6 @@ add_entrypoint_object(
     ../copysignf.h
   DEPENDS
     libc.src.__support.FPUtil.manipulation_functions
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
   FLAGS
     MISC_MATH_BASIC_OPS_OPT
 )
@@ -1964,8 +1728,6 @@ add_entrypoint_object(
     ../copysignl.h
   DEPENDS
     libc.src.__support.FPUtil.manipulation_functions
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
 )
 
 add_entrypoint_object(
@@ -1977,8 +1739,6 @@ add_entrypoint_object(
   DEPENDS
     libc.src.__support.macros.properties.types
     libc.src.__support.FPUtil.manipulation_functions
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
   FLAGS
     MISC_MATH_BASIC_OPS_OPT
 )
@@ -1992,8 +1752,6 @@ add_entrypoint_object(
   DEPENDS
     libc.src.__support.macros.properties.types
     libc.src.__support.FPUtil.manipulation_functions
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
 )
 
 add_entrypoint_object(
@@ -2002,8 +1760,6 @@ add_entrypoint_object(
     frexp.cpp
   HDRS
     ../frexp.h
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
   DEPENDS
     libc.src.__support.FPUtil.manipulation_functions
 )
@@ -2014,8 +1770,6 @@ add_entrypoint_object(
     frexpf.cpp
   HDRS
     ../frexpf.h
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
   DEPENDS
     libc.src.__support.FPUtil.manipulation_functions
 )
@@ -2026,8 +1780,6 @@ add_entrypoint_object(
     frexpl.cpp
   HDRS
     ../frexpl.h
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
   DEPENDS
     libc.src.__support.FPUtil.manipulation_functions
 )
@@ -2038,8 +1790,6 @@ add_entrypoint_object(
     frexpf16.cpp
   HDRS
     ../frexpf16.h
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
   DEPENDS
     libc.src.__support.macros.properties.types
     libc.src.__support.FPUtil.manipulation_functions
@@ -2051,8 +1801,6 @@ add_entrypoint_object(
     frexpf128.cpp
   HDRS
     ../frexpf128.h
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
   DEPENDS
     libc.src.__support.macros.properties.types
     libc.src.__support.FPUtil.manipulation_functions
@@ -2064,8 +1812,6 @@ add_entrypoint_object(
     ilogb.cpp
   HDRS
     ../ilogb.h
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
   DEPENDS
     libc.src.__support.FPUtil.manipulation_functions
 )
@@ -2076,8 +1822,6 @@ add_entrypoint_object(
     ilogbf.cpp
   HDRS
     ../ilogbf.h
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
   DEPENDS
     libc.src.__support.FPUtil.manipulation_functions
 )
@@ -2088,8 +1832,6 @@ add_entrypoint_object(
     ilogbl.cpp
   HDRS
     ../ilogbl.h
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
   DEPENDS
     libc.src.__support.FPUtil.manipulation_functions
 )
@@ -2100,8 +1842,6 @@ add_entrypoint_object(
     ilogbf16.cpp
   HDRS
     ../ilogbf16.h
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
   DEPENDS
     libc.src.__support.macros.properties.types
     libc.src.__support.FPUtil.manipulation_functions
@@ -2113,8 +1853,6 @@ add_entrypoint_object(
     ilogbf128.cpp
   HDRS
     ../ilogbf128.h
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
   DEPENDS
     libc.src.__support.macros.properties.types
     libc.src.__support.FPUtil.manipulation_functions
@@ -2126,8 +1864,6 @@ add_entrypoint_object(
     llogb.cpp
   HDRS
     ../llogb.h
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
   DEPENDS
     libc.src.__support.FPUtil.manipulation_functions
 )
@@ -2138,8 +1874,6 @@ add_entrypoint_object(
     llogbf.cpp
   HDRS
     ../llogbf.h
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
   DEPENDS
     libc.src.__support.FPUtil.manipulation_functions
 )
@@ -2150,8 +1884,6 @@ add_entrypoint_object(
     llogbl.cpp
   HDRS
     ../llogbl.h
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
   DEPENDS
     libc.src.__support.FPUtil.manipulation_functions
 )
@@ -2162,8 +1894,6 @@ add_entrypoint_object(
     llogbf16.cpp
   HDRS
     ../llogbf16.h
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
   DEPENDS
     libc.src.__support.macros.properties.types
     libc.src.__support.FPUtil.manipulation_functions
@@ -2175,8 +1905,6 @@ add_entrypoint_object(
     llogbf128.cpp
   HDRS
     ../llogbf128.h
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
   DEPENDS
     libc.src.__support.macros.properties.types
     libc.src.__support.FPUtil.manipulation_functions
@@ -2188,8 +1916,6 @@ add_entrypoint_object(
     ldexp.cpp
   HDRS
     ../ldexp.h
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
   DEPENDS
     libc.src.__support.FPUtil.manipulation_functions
 )
@@ -2200,8 +1926,6 @@ add_entrypoint_object(
     ldexpf.cpp
   HDRS
     ../ldexpf.h
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
   DEPENDS
     libc.src.__support.FPUtil.manipulation_functions
 )
@@ -2212,8 +1936,6 @@ add_entrypoint_object(
     ldexpl.cpp
   HDRS
     ../ldexpl.h
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
   DEPENDS
     libc.src.__support.FPUtil.manipulation_functions
 )
@@ -2224,8 +1946,6 @@ add_entrypoint_object(
     ldexpf16.cpp
   HDRS
     ../ldexpf16.h
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
   DEPENDS
     libc.src.__support.macros.properties.types
     libc.src.__support.FPUtil.manipulation_functions
@@ -2237,8 +1957,6 @@ add_entrypoint_object(
     ldexpf128.cpp
   HDRS
     ../ldexpf128.h
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
   DEPENDS
     libc.src.__support.macros.properties.types
     libc.src.__support.FPUtil.manipulation_functions
@@ -2282,8 +2000,6 @@ add_entrypoint_object(
     libc.src.__support.FPUtil.polyeval
     libc.src.__support.integer_literals
     libc.src.__support.macros.optimization
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
 )
 
 add_entrypoint_object(
@@ -2299,8 +2015,6 @@ add_entrypoint_object(
     libc.src.__support.FPUtil.fp_bits
     libc.src.__support.FPUtil.fma
     libc.src.__support.FPUtil.polyeval
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
 )
 
 add_entrypoint_object(
@@ -2321,8 +2035,6 @@ add_entrypoint_object(
     libc.src.__support.FPUtil.polyeval
     libc.src.__support.macros.optimization
     libc.src.__support.macros.properties.cpu_features
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
 )
 
 add_entrypoint_object(
@@ -2341,8 +2053,6 @@ add_entrypoint_object(
     libc.src.__support.FPUtil.polyeval
     libc.src.__support.integer_literals
     libc.src.__support.macros.optimization
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
 )
 
 add_entrypoint_object(
@@ -2359,8 +2069,6 @@ add_entrypoint_object(
     libc.src.__support.FPUtil.fma
     libc.src.__support.FPUtil.polyeval
     libc.src.__support.macros.optimization
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
 )
 
 add_entrypoint_object(
@@ -2380,8 +2088,6 @@ add_entrypoint_object(
     libc.src.__support.FPUtil.polyeval
     libc.src.__support.integer_literals
     libc.src.__support.macros.optimization
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
 )
 
 add_entrypoint_object(
@@ -2398,8 +2104,6 @@ add_entrypoint_object(
     libc.src.__support.FPUtil.fma
     libc.src.__support.FPUtil.polyeval
     libc.src.__support.macros.optimization
-    COMPILE_OPTIONS
-    ${libc_opt_high_flag}
 )
 
 add_entrypoint_object(
@@ -2420,8 +2124,6 @@ add_entrypoint_object(
     libc.src.__support.FPUtil.polyeval
     libc.src.__support.macros.optimization
     libc.src.__support.macros.properties.cpu_features
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
 )
 
 add_entrypoint_object(
@@ -2441,8 +2143,6 @@ add_entrypoint_object(
     libc.src.__support.FPUtil.polyeval
     libc.src.__support.integer_literals
     libc.src.__support.macros.optimization
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
 )
 
 add_entrypoint_object(
@@ -2459,8 +2159,6 @@ add_entrypoint_object(
     libc.src.__support.FPUtil.multiply_add
     libc.src.__support.FPUtil.polyeval
     libc.src.__support.macros.optimization
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
 )
 
 add_entrypoint_object(
@@ -2481,8 +2179,6 @@ add_entrypoint_object(
     libc.src.__support.FPUtil.polyeval
     libc.src.__support.macros.optimization
     libc.src.__support.macros.properties.cpu_features
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
 )
 
 add_entrypoint_object(
@@ -2491,8 +2187,6 @@ add_entrypoint_object(
     logb.cpp
   HDRS
     ../logb.h
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
   DEPENDS
     libc.src.__support.FPUtil.manipulation_functions
 )
@@ -2503,8 +2197,6 @@ add_entrypoint_object(
     logbf.cpp
   HDRS
     ../logbf.h
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
   DEPENDS
     libc.src.__support.FPUtil.manipulation_functions
 )
@@ -2515,8 +2207,6 @@ add_entrypoint_object(
     logbl.cpp
   HDRS
     ../logbl.h
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
   DEPENDS
     libc.src.__support.FPUtil.manipulation_functions
 )
@@ -2527,8 +2217,6 @@ add_entrypoint_object(
     logbf16.cpp
   HDRS
     ../logbf16.h
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
   DEPENDS
     libc.src.__support.macros.properties.types
     libc.src.__support.FPUtil.manipulation_functions
@@ -2540,8 +2228,6 @@ add_entrypoint_object(
     logbf128.cpp
   HDRS
     ../logbf128.h
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
   DEPENDS
     libc.src.__support.macros.properties.types
     libc.src.__support.FPUtil.manipulation_functions
@@ -2555,8 +2241,6 @@ add_entrypoint_object(
     ../modf.h
   DEPENDS
     libc.src.__support.FPUtil.manipulation_functions
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
 )
 
 add_entrypoint_object(
@@ -2567,8 +2251,6 @@ add_entrypoint_object(
     ../modff.h
   DEPENDS
     libc.src.__support.FPUtil.manipulation_functions
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
 )
 
 add_entrypoint_object(
@@ -2579,8 +2261,6 @@ add_entrypoint_object(
     ../modfl.h
   DEPENDS
     libc.src.__support.FPUtil.manipulation_functions
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
 )
 
 add_entrypoint_object(
@@ -2592,8 +2272,6 @@ add_entrypoint_object(
   DEPENDS
     libc.src.__support.macros.properties.types
     libc.src.__support.FPUtil.manipulation_functions
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
 )
 
 add_entrypoint_object(
@@ -2605,8 +2283,6 @@ add_entrypoint_object(
   DEPENDS
     libc.src.__support.macros.properties.types
     libc.src.__support.FPUtil.manipulation_functions
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
 )
 
 add_entrypoint_object(
@@ -2617,8 +2293,6 @@ add_entrypoint_object(
     ../fmin.h
   DEPENDS
     libc.src.__support.FPUtil.basic_operations
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
   FLAGS
     MISC_MATH_BASIC_OPS_OPT
 )
@@ -2631,8 +2305,6 @@ add_entrypoint_object(
     ../fminf.h
   DEPENDS
     libc.src.__support.FPUtil.basic_operations
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
   FLAGS
     MISC_MATH_BASIC_OPS_OPT
 )
@@ -2645,8 +2317,6 @@ add_entrypoint_object(
     ../fminl.h
   DEPENDS
     libc.src.__support.FPUtil.basic_operations
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
 )
 
 add_entrypoint_object(
@@ -2658,8 +2328,6 @@ add_entrypoint_object(
   DEPENDS
     libc.src.__support.macros.properties.types
     libc.src.__support.FPUtil.basic_operations
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
 )
 
 add_entrypoint_object(
@@ -2671,8 +2339,6 @@ add_entrypoint_object(
   DEPENDS
     libc.src.__support.macros.properties.types
     libc.src.__support.FPUtil.basic_operations
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
   FLAGS
     MISC_MATH_BASIC_OPS_OPT
 )
@@ -2686,8 +2352,6 @@ add_entrypoint_object(
     ../fmax.h
   DEPENDS
     libc.src.__support.FPUtil.basic_operations
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
   FLAGS
     MISC_MATH_BASIC_OPS_OPT
 )
@@ -2700,8 +2364,6 @@ add_entrypoint_object(
     ../fmaxf.h
   DEPENDS
     libc.src.__support.FPUtil.basic_operations
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
   FLAGS
     MISC_MATH_BASIC_OPS_OPT
 )
@@ -2714,8 +2376,6 @@ add_entrypoint_object(
     ../fmaxl.h
   DEPENDS
     libc.src.__support.FPUtil.basic_operations
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
 )
 
 add_entrypoint_object(
@@ -2727,8 +2387,6 @@ add_entrypoint_object(
   DEPENDS
     libc.src.__support.macros.properties.types
     libc.src.__support.FPUtil.basic_operations
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
 )
 
 add_entrypoint_object(
@@ -2740,8 +2398,6 @@ add_entrypoint_object(
   DEPENDS
     libc.src.__support.macros.properties.types
     libc.src.__support.FPUtil.basic_operations
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
   FLAGS
     MISC_MATH_BASIC_OPS_OPT
 )
@@ -2754,8 +2410,6 @@ add_entrypoint_object(
     ../fmaximum.h
   DEPENDS
     libc.src.__support.FPUtil.basic_operations
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
   FLAGS
     MISC_MATH_BASIC_OPS_OPT
 )
@@ -2768,8 +2422,6 @@ add_entrypoint_object(
     ../fmaximumf.h
   DEPENDS
     libc.src.__support.FPUtil.basic_operations
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
   FLAGS
     MISC_MATH_BASIC_OPS_OPT
 )
@@ -2782,8 +2434,6 @@ add_entrypoint_object(
     ../fmaximuml.h
   DEPENDS
     libc.src.__support.FPUtil.basic_operations
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
 )
 
 add_entrypoint_object(
@@ -2795,8 +2445,6 @@ add_entrypoint_object(
   DEPENDS
     libc.src.__support.macros.properties.types
     libc.src.__support.FPUtil.basic_operations
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
   FLAGS
     MISC_MATH_BASIC_OPS_OPT
 )
@@ -2810,8 +2458,6 @@ add_entrypoint_object(
   DEPENDS
     libc.src.__support.macros.properties.types
     libc.src.__support.FPUtil.basic_operations
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
 )
 
 add_entrypoint_object(
@@ -2822,8 +2468,6 @@ add_entrypoint_object(
     ../fmaximum_num.h
   DEPENDS
     libc.src.__support.FPUtil.basic_operations
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
   FLAGS
     MISC_MATH_BASIC_OPS_OPT
 )
@@ -2836,8 +2480,6 @@ add_entrypoint_object(
     ../fmaximum_numf.h
   DEPENDS
     libc.src.__support.FPUtil.basic_operations
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
   FLAGS
     MISC_MATH_BASIC_OPS_OPT
 )
@@ -2850,8 +2492,6 @@ add_entrypoint_object(
     ../fmaximum_numl.h
   DEPENDS
     libc.src.__support.FPUtil.basic_operations
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
 )
 
 add_entrypoint_object(
@@ -2863,8 +2503,6 @@ add_entrypoint_object(
   DEPENDS
     libc.src.__support.macros.properties.types
     libc.src.__support.FPUtil.basic_operations
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
   FLAGS
     MISC_MATH_BASIC_OPS_OPT
 )
@@ -2878,8 +2516,6 @@ add_entrypoint_object(
   DEPENDS
     libc.src.__support.macros.properties.types
     libc.src.__support.FPUtil.basic_operations
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
 )
 
 add_entrypoint_object(
@@ -2927,8 +2563,6 @@ add_entrypoint_object(
   DEPENDS
     libc.src.__support.macros.properties.types
     libc.src.__support.FPUtil.basic_operations
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
 )
 
 add_entrypoint_object(
@@ -2940,8 +2574,6 @@ add_entrypoint_object(
   DEPENDS
     libc.src.__support.macros.properties.types
     libc.src.__support.FPUtil.basic_operations
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
 )
 
 add_entrypoint_object(
@@ -2989,8 +2621,6 @@ add_entrypoint_object(
   DEPENDS
     libc.src.__support.macros.properties.types
     libc.src.__support.FPUtil.basic_operations
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
 )
 
 add_entrypoint_object(
@@ -3002,8 +2632,6 @@ add_entrypoint_object(
   DEPENDS
     libc.src.__support.macros.properties.types
     libc.src.__support.FPUtil.basic_operations
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
 )
 
 add_entrypoint_object(
@@ -3014,8 +2642,6 @@ add_entrypoint_object(
     ../fminimum.h
   DEPENDS
     libc.src.__support.FPUtil.basic_operations
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
   FLAGS
     MISC_MATH_BASIC_OPS_OPT
 )
@@ -3028,8 +2654,6 @@ add_entrypoint_object(
     ../fminimumf.h
   DEPENDS
     libc.src.__support.FPUtil.basic_operations
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
   FLAGS
     MISC_MATH_BASIC_OPS_OPT
 )
@@ -3042,8 +2666,6 @@ add_entrypoint_object(
     ../fminimuml.h
   DEPENDS
     libc.src.__support.FPUtil.basic_operations
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
 )
 
 add_entrypoint_object(
@@ -3055,8 +2677,6 @@ add_entrypoint_object(
   DEPENDS
     libc.src.__support.macros.properties.types
     libc.src.__support.FPUtil.basic_operations
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
   FLAGS
     MISC_MATH_BASIC_OPS_OPT
 )
@@ -3070,8 +2690,6 @@ add_entrypoint_object(
   DEPENDS
     libc.src.__support.macros.properties.types
     libc.src.__support.FPUtil.basic_operations
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
 )
 
 add_entrypoint_object(
@@ -3082,8 +2700,6 @@ add_entrypoint_object(
     ../fminimum_num.h
   DEPENDS
     libc.src.__support.FPUtil.basic_operations
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
   FLAGS
     MISC_MATH_BASIC_OPS_OPT
 )
@@ -3096,8 +2712,6 @@ add_entrypoint_object(
     ../fminimum_numf.h
   DEPENDS
     libc.src.__support.FPUtil.basic_operations
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
   FLAGS
     MISC_MATH_BASIC_OPS_OPT
 )
@@ -3110,8 +2724,6 @@ add_entrypoint_object(
     ../fminimum_numl.h
   DEPENDS
     libc.src.__support.FPUtil.basic_operations
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
 )
 
 add_entrypoint_object(
@@ -3123,8 +2735,6 @@ add_entrypoint_object(
   DEPENDS
     libc.src.__support.macros.properties.types
     libc.src.__support.FPUtil.basic_operations
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
   FLAGS
     MISC_MATH_BASIC_OPS_OPT
 )
@@ -3138,8 +2748,6 @@ add_entrypoint_object(
   DEPENDS
     libc.src.__support.macros.properties.types
     libc.src.__support.FPUtil.basic_operations
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
 )
 
 add_entrypoint_object(
@@ -3187,8 +2795,6 @@ add_entrypoint_object(
   DEPENDS
     libc.src.__support.macros.properties.types
     libc.src.__support.FPUtil.basic_operations
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
 )
 
 add_entrypoint_object(
@@ -3200,8 +2806,6 @@ add_entrypoint_object(
   DEPENDS
     libc.src.__support.macros.properties.types
     libc.src.__support.FPUtil.basic_operations
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
 )
 
 add_entrypoint_object(
@@ -3249,8 +2853,6 @@ add_entrypoint_object(
   DEPENDS
     libc.src.__support.macros.properties.types
     libc.src.__support.FPUtil.basic_operations
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
 )
 
 add_entrypoint_object(
@@ -3262,8 +2864,6 @@ add_entrypoint_object(
   DEPENDS
     libc.src.__support.macros.properties.types
     libc.src.__support.FPUtil.basic_operations
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
 )
 
 add_entrypoint_object(
@@ -3276,8 +2876,6 @@ add_entrypoint_object(
     libc.hdr.errno_macros
     libc.hdr.fenv_macros
     libc.src.__support.FPUtil.double_double
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
 )
 
 add_entrypoint_object(
@@ -3288,8 +2886,6 @@ add_entrypoint_object(
     ../fmull.h
   DEPENDS
     libc.src.__support.FPUtil.generic.mul
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
 )
 
 add_entrypoint_object(
@@ -3301,8 +2897,6 @@ add_entrypoint_object(
   DEPENDS
     libc.src.__support.macros.properties.types
     libc.src.__support.FPUtil.generic.mul
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
 )
 
 add_entrypoint_object(
@@ -3313,8 +2907,6 @@ add_entrypoint_object(
     ../fsub.h
   DEPENDS
     libc.src.__support.FPUtil.generic.add_sub
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
 )
 
 add_entrypoint_object(
@@ -3325,8 +2917,6 @@ add_entrypoint_object(
     ../fsubl.h
   DEPENDS
     libc.src.__support.FPUtil.generic.add_sub
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
 )
 
 add_entrypoint_object(
@@ -3338,8 +2928,6 @@ add_entrypoint_object(
   DEPENDS
     libc.src.__support.macros.properties.types
     libc.src.__support.FPUtil.generic.add_sub
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
 )
 
 add_entrypoint_object(
@@ -3350,8 +2938,6 @@ add_entrypoint_object(
     ../sqrt.h
   DEPENDS
     libc.src.__support.FPUtil.sqrt
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
 )
 
 
@@ -3363,8 +2949,6 @@ add_entrypoint_object(
     ../sqrtf.h
   DEPENDS
     libc.src.__support.FPUtil.sqrt
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
 )
 
 add_entrypoint_object(
@@ -3375,8 +2959,6 @@ add_entrypoint_object(
     ../sqrtl.h
   DEPENDS
     libc.src.__support.FPUtil.sqrt
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
 )
 
 add_entrypoint_object(
@@ -3387,8 +2969,6 @@ add_entrypoint_object(
     ../sqrtf16.h
   DEPENDS
     libc.src.__support.FPUtil.sqrt
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
 )
 
 add_entrypoint_object(
@@ -3400,8 +2980,6 @@ add_entrypoint_object(
   DEPENDS
     libc.src.__support.macros.properties.types
     libc.src.__support.FPUtil.sqrt
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
   )
 
 add_entrypoint_object(
@@ -3412,8 +2990,6 @@ add_entrypoint_object(
     ../remquof.h
   DEPENDS
     libc.src.__support.FPUtil.division_and_remainder_operations
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
 )
 
 add_entrypoint_object(
@@ -3424,8 +3000,6 @@ add_entrypoint_object(
     ../remquof128.h
   DEPENDS
     libc.src.__support.FPUtil.division_and_remainder_operations
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
 )
 
 add_entrypoint_object(
@@ -3436,8 +3010,6 @@ add_entrypoint_object(
     ../remquo.h
   DEPENDS
     libc.src.__support.FPUtil.division_and_remainder_operations
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
 )
 
 add_entrypoint_object(
@@ -3448,8 +3020,6 @@ add_entrypoint_object(
     ../remquol.h
   DEPENDS
     libc.src.__support.FPUtil.division_and_remainder_operations
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
 )
 
 add_entrypoint_object(
@@ -3461,8 +3031,6 @@ add_entrypoint_object(
   DEPENDS
     libc.src.__support.macros.properties.types
     libc.src.__support.FPUtil.division_and_remainder_operations
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
 )
 
 add_entrypoint_object(
@@ -3473,8 +3041,6 @@ add_entrypoint_object(
     ../remainderf.h
   DEPENDS
     libc.src.__support.FPUtil.division_and_remainder_operations
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
 )
 
 add_entrypoint_object(
@@ -3485,8 +3051,6 @@ add_entrypoint_object(
     ../remainder.h
   DEPENDS
     libc.src.__support.FPUtil.division_and_remainder_operations
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
 )
 
 add_entrypoint_object(
@@ -3497,8 +3061,6 @@ add_entrypoint_object(
     ../remainderl.h
   DEPENDS
     libc.src.__support.FPUtil.division_and_remainder_operations
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
 )
 
 add_entrypoint_object(
@@ -3510,8 +3072,6 @@ add_entrypoint_object(
   DEPENDS
     libc.src.__support.macros.properties.types
     libc.src.__support.FPUtil.division_and_remainder_operations
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
 )
 
 add_entrypoint_object(
@@ -3523,8 +3083,6 @@ add_entrypoint_object(
   DEPENDS
     libc.src.__support.macros.properties.types
     libc.src.__support.FPUtil.division_and_remainder_operations
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
 )
 
 add_entrypoint_object(
@@ -3540,8 +3098,6 @@ add_entrypoint_object(
     libc.src.__support.FPUtil.multiply_add
     libc.src.__support.FPUtil.sqrt
     libc.src.__support.macros.optimization
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
 )
 
 add_entrypoint_object(
@@ -3550,8 +3106,6 @@ add_entrypoint_object(
     fdim.cpp
   HDRS
     ../fdim.h
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
   DEPENDS
     libc.src.__support.FPUtil.basic_operations
 )
@@ -3562,8 +3116,6 @@ add_entrypoint_object(
     fdimf.cpp
   HDRS
     ../fdimf.h
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
   DEPENDS
     libc.src.__support.FPUtil.basic_operations
 )
@@ -3574,8 +3126,6 @@ add_entrypoint_object(
     fdiml.cpp
   HDRS
     ../fdiml.h
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
   DEPENDS
     libc.src.__support.FPUtil.basic_operations
 )
@@ -3586,8 +3136,6 @@ add_entrypoint_object(
     fdimf16.cpp
   HDRS
     ../fdimf16.h
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
   DEPENDS
     libc.src.__support.macros.properties.types
     libc.src.__support.FPUtil.basic_operations
@@ -3599,8 +3147,6 @@ add_entrypoint_object(
     fdimf128.cpp
   HDRS
     ../fdimf128.h
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
   DEPENDS
     libc.src.__support.macros.properties.types
     libc.src.__support.FPUtil.basic_operations
@@ -3612,8 +3158,6 @@ add_entrypoint_object(
     fdiv.cpp
   HDRS
     ../fdiv.h
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
   DEPENDS
     libc.src.__support.FPUtil.generic.div
 )
@@ -3624,8 +3168,6 @@ add_entrypoint_object(
     fdivl.cpp
   HDRS
     ../fdivl.h
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
   DEPENDS
     libc.src.__support.FPUtil.generic.div
 )
@@ -3636,8 +3178,6 @@ add_entrypoint_object(
     fdivf128.cpp
   HDRS
     ../fdivf128.h
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
   DEPENDS
     libc.src.__support.macros.properties.types
     libc.src.__support.FPUtil.generic.div
@@ -3649,8 +3189,6 @@ add_entrypoint_object(
     ffma.cpp
   HDRS
     ../ffma.h
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
   DEPENDS
     libc.src.__support.FPUtil.fma
 )
@@ -3661,8 +3199,6 @@ add_entrypoint_object(
     ffmal.cpp
   HDRS
     ../ffmal.h
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
   DEPENDS
     libc.src.__support.FPUtil.fma
 )
@@ -3673,8 +3209,6 @@ add_entrypoint_object(
     ffmaf128.cpp
   HDRS
     ../ffmaf128.h
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
   DEPENDS
     libc.src.__support.macros.properties.types
     libc.src.__support.FPUtil.fma
@@ -3688,8 +3222,6 @@ add_entrypoint_object(
     ../hypot.h
   DEPENDS
     libc.src.__support.FPUtil.hypot
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
 )
 
 add_entrypoint_object(
@@ -3698,8 +3230,6 @@ add_entrypoint_object(
     issignaling.cpp
   HDRS
     ../issignaling.h
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
 )
 
 add_entrypoint_object(
@@ -3708,8 +3238,6 @@ add_entrypoint_object(
     issignalingf.cpp
   HDRS
     ../issignalingf.h
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
 )
 
 add_entrypoint_object(
@@ -3718,8 +3246,6 @@ add_entrypoint_object(
     issignalingl.cpp
   HDRS
     ../issignalingl.h
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
 )
 
 add_entrypoint_object(
@@ -3728,8 +3254,6 @@ add_entrypoint_object(
     issignalingf16.cpp
   HDRS
     ../issignalingf16.h
-  COMPILE_OPTIONS
-      ${libc_opt_high_flag}
   DEPENDS
     libc.src.__support.macros.properties.types
 )
@@ -3740,8 +3264,6 @@ add_entrypoint_object(
     issignalingf128.cpp
   HDRS
     ../issignalingf128.h
-  COMPILE_OPTIONS
-      ${libc_opt_high_flag}
   DEPENDS
     libc.src.__support.macros.properties.types
 )
@@ -3752,8 +3274,6 @@ add_entrypoint_object(
     isnan.cpp
   HDRS
     ../isnan.h
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
 )
 
 add_entrypoint_object(
@@ -3762,8 +3282,6 @@ add_entrypoint_object(
     isnanf.cpp
   HDRS
     ../isnanf.h
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
 )
 
 add_entrypoint_object(
@@ -3772,8 +3290,6 @@ add_entrypoint_object(
     isnanl.cpp
   HDRS
     ../isnanl.h
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
 )
 
 add_entrypoint_object(
@@ -3785,8 +3301,6 @@ add_entrypoint_object(
   DEPENDS
     libc.src.__support.str_to_float
     libc.src.errno.errno
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
 )
 
 add_entrypoint_object(
@@ -3798,8 +3312,6 @@ add_entrypoint_object(
   DEPENDS
     libc.src.__support.str_to_float
     libc.src.errno.errno
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
 )
 
 add_entrypoint_object(
@@ -3811,8 +3323,6 @@ add_entrypoint_object(
   DEPENDS
     libc.src.__support.str_to_float
     libc.src.errno.errno
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
 )
 
 add_entrypoint_object(
@@ -3824,8 +3334,6 @@ add_entrypoint_object(
   DEPENDS
     libc.src.__support.str_to_float
     libc.src.errno.errno
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
 )
 
 add_entrypoint_object(
@@ -3837,8 +3345,6 @@ add_entrypoint_object(
   DEPENDS
     libc.src.__support.str_to_float
     libc.src.errno.errno
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
 )
 
 add_entrypoint_object(
@@ -3849,8 +3355,6 @@ add_entrypoint_object(
     ../nextafter.h
   DEPENDS
     libc.src.__support.FPUtil.manipulation_functions
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
 )
 
 add_entrypoint_object(
@@ -3861,8 +3365,6 @@ add_entrypoint_object(
     ../nextafterf.h
   DEPENDS
     libc.src.__support.FPUtil.manipulation_functions
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
 )
 
 add_entrypoint_object(
@@ -3873,8 +3375,6 @@ add_entrypoint_object(
     ../nextafterl.h
   DEPENDS
     libc.src.__support.FPUtil.manipulation_functions
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
 )
 
 add_entrypoint_object(
@@ -3886,8 +3386,6 @@ add_entrypoint_object(
   DEPENDS
     libc.src.__support.macros.properties.types
     libc.src.__support.FPUtil.manipulation_functions
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
 )
 
 add_entrypoint_object(
@@ -3899,8 +3397,6 @@ add_entrypoint_object(
   DEPENDS
     libc.src.__support.macros.properties.types
     libc.src.__support.FPUtil.manipulation_functions
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
 )
 
 add_entrypoint_object(
@@ -3911,8 +3407,6 @@ add_entrypoint_object(
     ../nexttoward.h
   DEPENDS
     libc.src.__support.FPUtil.manipulation_functions
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
 )
 
 add_entrypoint_object(
@@ -3923,8 +3417,6 @@ add_entrypoint_object(
     ../nexttowardf.h
   DEPENDS
     libc.src.__support.FPUtil.manipulation_functions
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
 )
 
 add_entrypoint_object(
@@ -3935,8 +3427,6 @@ add_entrypoint_object(
     ../nexttowardl.h
   DEPENDS
     libc.src.__support.FPUtil.manipulation_functions
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
 )
 
 add_entrypoint_object(
@@ -3948,8 +3438,6 @@ add_entrypoint_object(
   DEPENDS
     libc.src.__support.macros.properties.types
     libc.src.__support.FPUtil.manipulation_functions
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
 )
 
 add_entrypoint_object(
@@ -3960,8 +3448,6 @@ add_entrypoint_object(
     ../nextdown.h
   DEPENDS
     libc.src.__support.FPUtil.manipulation_functions
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
 )
 
 add_entrypoint_object(
@@ -3972,8 +3458,6 @@ add_entrypoint_object(
     ../nextdownl.h
   DEPENDS
     libc.src.__support.FPUtil.manipulation_functions
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
 )
 
 add_entrypoint_object(
@@ -3984,8 +3468,6 @@ add_entrypoint_object(
     ../nextdownf.h
   DEPENDS
     libc.src.__support.FPUtil.manipulation_functions
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
 )
 
 add_entrypoint_object(
@@ -3997,8 +3479,6 @@ add_entrypoint_object(
   DEPENDS
     libc.src.__support.macros.properties.types
     libc.src.__support.FPUtil.manipulation_functions
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
 )
 
 add_entrypoint_object(
@@ -4010,8 +3490,6 @@ add_entrypoint_object(
   DEPENDS
     libc.src.__support.macros.properties.types
     libc.src.__support.FPUtil.manipulation_functions
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
 )
 
 add_entrypoint_object(
@@ -4022,8 +3500,6 @@ add_entrypoint_object(
     ../nextup.h
   DEPENDS
     libc.src.__support.FPUtil.manipulation_functions
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
 )
 
 add_entrypoint_object(
@@ -4034,8 +3510,6 @@ add_entrypoint_object(
     ../nextupl.h
   DEPENDS
     libc.src.__support.FPUtil.manipulation_functions
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
 )
 
 add_entrypoint_object(
@@ -4046,8 +3520,6 @@ add_entrypoint_object(
     ../nextupf.h
   DEPENDS
     libc.src.__support.FPUtil.manipulation_functions
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
 )
 
 add_entrypoint_object(
@@ -4059,8 +3531,6 @@ add_entrypoint_object(
   DEPENDS
     libc.src.__support.macros.properties.types
     libc.src.__support.FPUtil.manipulation_functions
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
 )
 
 add_entrypoint_object(
@@ -4072,8 +3542,6 @@ add_entrypoint_object(
   DEPENDS
     libc.src.__support.macros.properties.types
     libc.src.__support.FPUtil.manipulation_functions
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
 )
 
 add_entrypoint_object(
@@ -4084,8 +3552,6 @@ add_entrypoint_object(
     ../fmod.h
   DEPENDS
     libc.src.__support.FPUtil.generic.fmod
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
 )
 
 add_entrypoint_object(
@@ -4096,8 +3562,6 @@ add_entrypoint_object(
     ../fmodf.h
   DEPENDS
     libc.src.__support.FPUtil.generic.fmod
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
 )
 
 add_entrypoint_object(
@@ -4108,8 +3572,6 @@ add_entrypoint_object(
     ../fmodl.h
   DEPENDS
     libc.src.__support.FPUtil.generic.fmod
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
 )
 
 add_entrypoint_object(
@@ -4121,8 +3583,6 @@ add_entrypoint_object(
   DEPENDS
     libc.src.__support.macros.properties.types
     libc.src.__support.FPUtil.generic.fmod
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
 )
 
 add_entrypoint_object(
@@ -4134,8 +3594,6 @@ add_entrypoint_object(
   DEPENDS
     libc.src.__support.macros.properties.types
     libc.src.__support.FPUtil.generic.fmod
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
 )
 
 add_entrypoint_object(
@@ -4146,8 +3604,6 @@ add_entrypoint_object(
     ../fromfp.h
   DEPENDS
     libc.src.__support.FPUtil.nearest_integer_operations
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
 )
 
 add_entrypoint_object(
@@ -4158,8 +3614,6 @@ add_entrypoint_object(
     ../fromfpf.h
   DEPENDS
     libc.src.__support.FPUtil.nearest_integer_operations
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
 )
 
 add_entrypoint_object(
@@ -4170,8 +3624,6 @@ add_entrypoint_object(
     ../fromfpl.h
   DEPENDS
     libc.src.__support.FPUtil.nearest_integer_operations
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
 )
 
 add_entrypoint_object(
@@ -4183,8 +3635,6 @@ add_entrypoint_object(
   DEPENDS
     libc.src.__support.macros.properties.types
     libc.src.__support.FPUtil.nearest_integer_operations
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
 )
 
 add_entrypoint_object(
@@ -4196,8 +3646,6 @@ add_entrypoint_object(
   DEPENDS
     libc.src.__support.macros.properties.types
     libc.src.__support.FPUtil.nearest_integer_operations
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
 )
 
 add_entrypoint_object(
@@ -4208,8 +3656,6 @@ add_entrypoint_object(
     ../fromfpx.h
   DEPENDS
     libc.src.__support.FPUtil.nearest_integer_operations
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
 )
 
 add_entrypoint_object(
@@ -4220,8 +3666,6 @@ add_entrypoint_object(
     ../fromfpxf.h
   DEPENDS
     libc.src.__support.FPUtil.nearest_integer_operations
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
 )
 
 add_entrypoint_object(
@@ -4232,8 +3676,6 @@ add_entrypoint_object(
     ../fromfpxl.h
   DEPENDS
     libc.src.__support.FPUtil.nearest_integer_operations
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
 )
 
 add_entrypoint_object(
@@ -4245,8 +3687,6 @@ add_entrypoint_object(
   DEPENDS
     libc.src.__support.macros.properties.types
     libc.src.__support.FPUtil.nearest_integer_operations
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
 )
 
 add_entrypoint_object(
@@ -4258,8 +3698,6 @@ add_entrypoint_object(
   DEPENDS
     libc.src.__support.macros.properties.types
     libc.src.__support.FPUtil.nearest_integer_operations
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
 )
 
 add_entrypoint_object(
@@ -4270,8 +3708,6 @@ add_entrypoint_object(
     ../ufromfp.h
   DEPENDS
     libc.src.__support.FPUtil.nearest_integer_operations
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
 )
 
 add_entrypoint_object(
@@ -4282,8 +3718,6 @@ add_entrypoint_object(
     ../ufromfpf.h
   DEPENDS
     libc.src.__support.FPUtil.nearest_integer_operations
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
 )
 
 add_entrypoint_object(
@@ -4294,8 +3728,6 @@ add_entrypoint_object(
     ../ufromfpl.h
   DEPENDS
     libc.src.__support.FPUtil.nearest_integer_operations
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
 )
 
 add_entrypoint_object(
@@ -4307,8 +3739,6 @@ add_entrypoint_object(
   DEPENDS
     libc.src.__support.macros.properties.types
     libc.src.__support.FPUtil.nearest_integer_operations
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
 )
 
 add_entrypoint_object(
@@ -4320,8 +3750,6 @@ add_entrypoint_object(
   DEPENDS
     libc.src.__support.macros.properties.types
     libc.src.__support.FPUtil.nearest_integer_operations
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
 )
 
 add_entrypoint_object(
@@ -4332,8 +3760,6 @@ add_entrypoint_object(
     ../ufromfpx.h
   DEPENDS
     libc.src.__support.FPUtil.nearest_integer_operations
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
 )
 
 add_entrypoint_object(
@@ -4344,8 +3770,6 @@ add_entrypoint_object(
     ../ufromfpxf.h
   DEPENDS
     libc.src.__support.FPUtil.nearest_integer_operations
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
 )
 
 add_entrypoint_object(
@@ -4356,8 +3780,6 @@ add_entrypoint_object(
     ../ufromfpxl.h
   DEPENDS
     libc.src.__support.FPUtil.nearest_integer_operations
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
 )
 
 add_entrypoint_object(
@@ -4369,8 +3791,6 @@ add_entrypoint_object(
   DEPENDS
     libc.src.__support.macros.properties.types
     libc.src.__support.FPUtil.nearest_integer_operations
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
 )
 
 add_entrypoint_object(
@@ -4382,8 +3802,6 @@ add_entrypoint_object(
   DEPENDS
     libc.src.__support.macros.properties.types
     libc.src.__support.FPUtil.nearest_integer_operations
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
 )
 
 #TODO: Add errno include to the hyperbolic functions.
@@ -4406,8 +3824,6 @@ add_object_library(
     libc.src.__support.FPUtil.polyeval
     libc.src.__support.common
     libc.src.errno.errno
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
 )
 
 add_entrypoint_object(
@@ -4422,8 +3838,6 @@ add_entrypoint_object(
     libc.src.__support.FPUtil.multiply_add
     libc.src.__support.FPUtil.rounding_mode
     libc.src.__support.macros.optimization
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
 )
 
 add_entrypoint_object(
@@ -4441,8 +3855,6 @@ add_entrypoint_object(
     libc.src.__support.FPUtil.fp_bits
     libc.src.__support.FPUtil.rounding_mode
     libc.src.__support.macros.optimization
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
 )
 
 add_entrypoint_object(
@@ -4456,8 +3868,6 @@ add_entrypoint_object(
     libc.src.__support.FPUtil.fp_bits
     libc.src.__support.FPUtil.rounding_mode
     libc.src.__support.macros.optimization
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
 )
 
 add_entrypoint_object(
@@ -4475,8 +3885,6 @@ add_entrypoint_object(
     libc.src.__support.FPUtil.fp_bits
     libc.src.__support.FPUtil.rounding_mode
     libc.src.__support.macros.optimization
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
 )
 
 add_entrypoint_object(
@@ -4492,8 +3900,6 @@ add_entrypoint_object(
     libc.src.__support.FPUtil.multiply_add
     libc.src.__support.FPUtil.polyeval
     libc.src.__support.macros.optimization
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
 )
 
 add_entrypoint_object(
@@ -4515,8 +3921,6 @@ add_entrypoint_object(
     libc.src.__support.FPUtil.polyeval
     libc.src.__support.FPUtil.rounding_mode
     libc.src.__support.macros.optimization
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
 )
 
 add_entrypoint_object(
@@ -4533,8 +3937,6 @@ add_entrypoint_object(
     libc.src.__support.FPUtil.polyeval
     libc.src.__support.FPUtil.sqrt
     libc.src.__support.macros.optimization
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
 )
 
 add_entrypoint_object(
@@ -4550,8 +3952,6 @@ add_entrypoint_object(
     libc.src.__support.FPUtil.polyeval
     libc.src.__support.FPUtil.sqrt
     libc.src.__support.macros.optimization
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
 )
 
 add_entrypoint_object(
@@ -4564,8 +3964,6 @@ add_entrypoint_object(
     .explogxf
     libc.src.__support.FPUtil.fp_bits
     libc.src.__support.macros.optimization
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
 )
 
 add_object_library(
@@ -4594,8 +3992,25 @@ add_entrypoint_object(
     libc.src.__support.FPUtil.sqrt
     libc.src.__support.macros.optimization
     .inv_trigf_utils
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
+)
+
+add_entrypoint_object(
+  asinf16
+  SRCS
+    asinf16.cpp
+  HDRS
+    ../asinf16.h
+  DEPENDS
+    libc.hdr.errno_macros
+    libc.hdr.fenv_macros
+    libc.src.__support.FPUtil.cast
+    libc.src.__support.FPUtil.fenv_impl
+    libc.src.__support.FPUtil.fp_bits
+    libc.src.__support.FPUtil.multiply_add
+    libc.src.__support.FPUtil.polyeval
+    libc.src.__support.FPUtil.sqrt
+    libc.src.__support.macros.optimization
+    libc.src.__support.macros.properties.types  
 )
 
 add_entrypoint_object(
@@ -4612,8 +4027,6 @@ add_entrypoint_object(
     libc.src.__support.FPUtil.sqrt
     libc.src.__support.macros.optimization
     .inv_trigf_utils
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
 )
 
 add_entrypoint_object(
@@ -4631,8 +4044,6 @@ add_entrypoint_object(
     libc.src.__support.FPUtil.polyeval
     libc.src.__support.FPUtil.rounding_mode
     libc.src.__support.macros.optimization
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
 )
 
 add_entrypoint_object(
@@ -4641,10 +4052,10 @@ add_entrypoint_object(
     atan2f.cpp
   HDRS
     ../atan2f.h
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
+    atan2f_float.h
   DEPENDS
     .inv_trigf_utils
+    libc.src.__support.FPUtil.double_double
     libc.src.__support.FPUtil.fp_bits
     libc.src.__support.FPUtil.multiply_add
     libc.src.__support.FPUtil.nearest_integer
@@ -4659,8 +4070,6 @@ add_entrypoint_object(
     atan2.cpp
   HDRS
     ../atan2.h
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
   DEPENDS
     .inv_trigf_utils
     libc.src.__support.FPUtil.double_double
@@ -4679,8 +4088,6 @@ add_entrypoint_object(
     atan2l.cpp
   HDRS
     ../atan2l.h
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
   DEPENDS
     .atan2
 )
@@ -4694,8 +4101,6 @@ add_entrypoint_object(
   DEPENDS
     libc.hdr.float_macros
     libc.src.__support.FPUtil.manipulation_functions
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
 )
 
 add_entrypoint_object(
@@ -4707,8 +4112,6 @@ add_entrypoint_object(
   DEPENDS
     libc.hdr.float_macros
     libc.src.__support.FPUtil.manipulation_functions
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
 )
 
 add_entrypoint_object(
@@ -4720,8 +4123,6 @@ add_entrypoint_object(
   DEPENDS
     libc.hdr.float_macros
     libc.src.__support.FPUtil.manipulation_functions
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
 )
 
 add_entrypoint_object(
@@ -4734,8 +4135,6 @@ add_entrypoint_object(
     libc.hdr.float_macros
     libc.src.__support.macros.properties.types
     libc.src.__support.FPUtil.manipulation_functions
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
 )
 
 add_entrypoint_object(
@@ -4748,8 +4147,6 @@ add_entrypoint_object(
     libc.hdr.float_macros
     libc.src.__support.macros.properties.types
     libc.src.__support.FPUtil.manipulation_functions
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
 )
 
 add_entrypoint_object(
@@ -4761,8 +4158,6 @@ add_entrypoint_object(
   DEPENDS
     libc.hdr.float_macros
     libc.src.__support.FPUtil.manipulation_functions
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
 )
 
 add_entrypoint_object(
@@ -4774,8 +4169,6 @@ add_entrypoint_object(
   DEPENDS
     libc.hdr.float_macros
     libc.src.__support.FPUtil.manipulation_functions
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
 )
 
 add_entrypoint_object(
@@ -4787,8 +4180,6 @@ add_entrypoint_object(
   DEPENDS
     libc.hdr.float_macros
     libc.src.__support.FPUtil.manipulation_functions
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
 )
 
 add_entrypoint_object(
@@ -4801,8 +4192,6 @@ add_entrypoint_object(
     libc.hdr.float_macros
     libc.src.__support.macros.properties.types
     libc.src.__support.FPUtil.manipulation_functions
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
 )
 
 add_entrypoint_object(
@@ -4815,8 +4204,6 @@ add_entrypoint_object(
     libc.hdr.float_macros
     libc.src.__support.macros.properties.types
     libc.src.__support.FPUtil.manipulation_functions
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
 )
 
 add_entrypoint_object(
@@ -4827,8 +4214,6 @@ add_entrypoint_object(
     ../fmaf.h
   DEPENDS
     libc.src.__support.FPUtil.fma
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
 )
 
 add_entrypoint_object(
@@ -4839,8 +4224,6 @@ add_entrypoint_object(
     ../fma.h
   DEPENDS
     libc.src.__support.FPUtil.fma
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
 )
 
 add_entrypoint_object(
@@ -4851,8 +4234,6 @@ add_entrypoint_object(
     ../totalorder.h
   DEPENDS
     libc.src.__support.FPUtil.basic_operations
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
 )
 
 add_entrypoint_object(
@@ -4863,8 +4244,6 @@ add_entrypoint_object(
     ../totalorderf.h
   DEPENDS
     libc.src.__support.FPUtil.basic_operations
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
 )
 
 add_entrypoint_object(
@@ -4875,8 +4254,6 @@ add_entrypoint_object(
     ../totalorderl.h
   DEPENDS
     libc.src.__support.FPUtil.basic_operations
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
 )
 
 add_entrypoint_object(
@@ -4887,8 +4264,6 @@ add_entrypoint_object(
     ../totalorderf16.h
   DEPENDS
     libc.src.__support.FPUtil.basic_operations
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
 )
 
 add_entrypoint_object(
@@ -4900,8 +4275,6 @@ add_entrypoint_object(
   DEPENDS
     libc.src.__support.FPUtil.basic_operations
     libc.src.__support.macros.properties.types
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
 )
 add_entrypoint_object(
   totalordermag
@@ -4911,8 +4284,6 @@ add_entrypoint_object(
     ../totalordermag.h
   DEPENDS
     libc.src.__support.FPUtil.basic_operations
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
 )
 
 add_entrypoint_object(
@@ -4923,8 +4294,6 @@ add_entrypoint_object(
     ../totalordermagf.h
   DEPENDS
     libc.src.__support.FPUtil.basic_operations
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
 )
 
 add_entrypoint_object(
@@ -4935,8 +4304,6 @@ add_entrypoint_object(
     ../totalordermagl.h
   DEPENDS
     libc.src.__support.FPUtil.basic_operations
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
 )
 
 add_entrypoint_object(
@@ -4947,8 +4314,6 @@ add_entrypoint_object(
     ../totalordermagf16.h
   DEPENDS
     libc.src.__support.FPUtil.basic_operations
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
 )
 
 add_entrypoint_object(
@@ -4960,8 +4325,6 @@ add_entrypoint_object(
   DEPENDS
     libc.src.__support.FPUtil.basic_operations
     libc.src.__support.macros.properties.types
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
 )
 
 add_entrypoint_object(
@@ -4972,8 +4335,6 @@ add_entrypoint_object(
     ../getpayload.h
   DEPENDS
     libc.src.__support.FPUtil.basic_operations
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
 )
 
 add_entrypoint_object(
@@ -4984,8 +4345,6 @@ add_entrypoint_object(
     ../getpayloadf.h
   DEPENDS
     libc.src.__support.FPUtil.basic_operations
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
 )
 
 add_entrypoint_object(
@@ -4996,8 +4355,6 @@ add_entrypoint_object(
     ../getpayloadl.h
   DEPENDS
     libc.src.__support.FPUtil.basic_operations
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
 )
 
 add_entrypoint_object(
@@ -5009,8 +4366,6 @@ add_entrypoint_object(
   DEPENDS
     libc.src.__support.macros.properties.types
     libc.src.__support.FPUtil.basic_operations
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
 )
 
 add_entrypoint_object(
@@ -5022,8 +4377,6 @@ add_entrypoint_object(
   DEPENDS
     libc.src.__support.macros.properties.types
     libc.src.__support.FPUtil.basic_operations
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
 )
 
 add_entrypoint_object(
@@ -5034,8 +4387,6 @@ add_entrypoint_object(
     ../setpayload.h
   DEPENDS
     libc.src.__support.FPUtil.basic_operations
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
 )
 
 add_entrypoint_object(
@@ -5046,8 +4397,6 @@ add_entrypoint_object(
     ../setpayloadf.h
   DEPENDS
     libc.src.__support.FPUtil.basic_operations
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
 )
 
 add_entrypoint_object(
@@ -5058,8 +4407,6 @@ add_entrypoint_object(
     ../setpayloadl.h
   DEPENDS
     libc.src.__support.FPUtil.basic_operations
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
 )
 
 add_entrypoint_object(
@@ -5071,8 +4418,6 @@ add_entrypoint_object(
   DEPENDS
     libc.src.__support.macros.properties.types
     libc.src.__support.FPUtil.basic_operations
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
 )
 
 add_entrypoint_object(
@@ -5084,8 +4429,6 @@ add_entrypoint_object(
   DEPENDS
     libc.src.__support.macros.properties.types
     libc.src.__support.FPUtil.basic_operations
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
 )
 
 add_entrypoint_object(
@@ -5096,8 +4439,6 @@ add_entrypoint_object(
     ../setpayloadsig.h
   DEPENDS
     libc.src.__support.FPUtil.basic_operations
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
 )
 
 add_entrypoint_object(
@@ -5108,8 +4449,6 @@ add_entrypoint_object(
     ../setpayloadsigf.h
   DEPENDS
     libc.src.__support.FPUtil.basic_operations
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
 )
 
 add_entrypoint_object(
@@ -5120,8 +4459,6 @@ add_entrypoint_object(
     ../setpayloadsigl.h
   DEPENDS
     libc.src.__support.FPUtil.basic_operations
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
 )
 
 add_entrypoint_object(
@@ -5133,8 +4470,6 @@ add_entrypoint_object(
   DEPENDS
     libc.src.__support.macros.properties.types
     libc.src.__support.FPUtil.basic_operations
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
 )
 
 add_entrypoint_object(
@@ -5146,8 +4481,6 @@ add_entrypoint_object(
   DEPENDS
     libc.src.__support.macros.properties.types
     libc.src.__support.FPUtil.basic_operations
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
 )
 
 add_entrypoint_object(
@@ -5159,8 +4492,6 @@ add_entrypoint_object(
   DEPENDS
     libc.src.__support.macros.properties.types
     libc.src.__support.FPUtil.generic.add_sub
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
 )
 
 add_entrypoint_object(
@@ -5172,8 +4503,6 @@ add_entrypoint_object(
   DEPENDS
     libc.src.__support.macros.properties.types
     libc.src.__support.FPUtil.generic.add_sub
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
 )
 
 add_entrypoint_object(
@@ -5185,8 +4514,6 @@ add_entrypoint_object(
   DEPENDS
     libc.src.__support.macros.properties.types
     libc.src.__support.FPUtil.generic.add_sub
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
 )
 
 add_entrypoint_object(
@@ -5198,8 +4525,6 @@ add_entrypoint_object(
   DEPENDS
     libc.src.__support.macros.properties.types
     libc.src.__support.FPUtil.generic.add_sub
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
 )
 
 add_entrypoint_object(
@@ -5211,8 +4536,6 @@ add_entrypoint_object(
   DEPENDS
     libc.src.__support.macros.properties.types
     libc.src.__support.FPUtil.generic.add_sub
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
 )
 
 add_entrypoint_object(
@@ -5224,8 +4547,6 @@ add_entrypoint_object(
   DEPENDS
     libc.src.__support.macros.properties.types
     libc.src.__support.FPUtil.generic.add_sub
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
 )
 
 add_entrypoint_object(
@@ -5237,8 +4558,6 @@ add_entrypoint_object(
   DEPENDS
     libc.src.__support.macros.properties.types
     libc.src.__support.FPUtil.generic.add_sub
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
 )
 
 add_entrypoint_object(
@@ -5250,8 +4569,6 @@ add_entrypoint_object(
   DEPENDS
     libc.src.__support.macros.properties.types
     libc.src.__support.FPUtil.generic.add_sub
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
 )
 
 add_entrypoint_object(
@@ -5263,8 +4580,6 @@ add_entrypoint_object(
   DEPENDS
     libc.src.__support.macros.properties.types
     libc.src.__support.FPUtil.generic.div
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
 )
 
 add_entrypoint_object(
@@ -5276,8 +4591,6 @@ add_entrypoint_object(
   DEPENDS
     libc.src.__support.macros.properties.types
     libc.src.__support.FPUtil.generic.div
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
 )
 
 add_entrypoint_object(
@@ -5289,8 +4602,6 @@ add_entrypoint_object(
   DEPENDS
     libc.src.__support.macros.properties.types
     libc.src.__support.FPUtil.generic.div
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
 )
 
 add_entrypoint_object(
@@ -5302,8 +4613,6 @@ add_entrypoint_object(
   DEPENDS
     libc.src.__support.macros.properties.types
     libc.src.__support.FPUtil.generic.div
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
 )
 
 add_entrypoint_object(
@@ -5315,8 +4624,6 @@ add_entrypoint_object(
   DEPENDS
     libc.src.__support.macros.properties.types
     libc.src.__support.FPUtil.fma
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
 )
 
 add_entrypoint_object(
@@ -5328,8 +4635,6 @@ add_entrypoint_object(
   DEPENDS
     libc.src.__support.macros.properties.types
     libc.src.__support.FPUtil.fma
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
 )
 
 add_entrypoint_object(
@@ -5341,8 +4646,6 @@ add_entrypoint_object(
   DEPENDS
     libc.src.__support.macros.properties.types
     libc.src.__support.FPUtil.fma
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
 )
 
 add_entrypoint_object(
@@ -5354,8 +4657,6 @@ add_entrypoint_object(
   DEPENDS
     libc.src.__support.macros.properties.types
     libc.src.__support.FPUtil.fma
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
 )
 
 add_entrypoint_object(
@@ -5367,8 +4668,6 @@ add_entrypoint_object(
   DEPENDS
     libc.src.__support.macros.properties.types
     libc.src.__support.FPUtil.sqrt
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
 )
 
 add_entrypoint_object(
@@ -5380,8 +4679,6 @@ add_entrypoint_object(
   DEPENDS
     libc.src.__support.macros.properties.types
     libc.src.__support.FPUtil.sqrt
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
 )
 
 add_entrypoint_object(
@@ -5393,8 +4690,6 @@ add_entrypoint_object(
   DEPENDS
     libc.src.__support.macros.properties.types
     libc.src.__support.FPUtil.sqrt
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
 )
 
 add_entrypoint_object(
@@ -5406,8 +4701,6 @@ add_entrypoint_object(
   DEPENDS
     libc.src.__support.macros.properties.types
     libc.src.__support.FPUtil.sqrt
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
 )
 
 add_entrypoint_object(
@@ -5418,8 +4711,6 @@ add_entrypoint_object(
     ../fsqrt.h
   DEPENDS
     libc.src.__support.FPUtil.generic.sqrt
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
 )
 
 add_entrypoint_object(
@@ -5430,8 +4721,6 @@ add_entrypoint_object(
     ../fsqrtl.h
   DEPENDS
     libc.src.__support.FPUtil.generic.sqrt
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
 )
 
 add_entrypoint_object(
@@ -5443,8 +4732,6 @@ add_entrypoint_object(
   DEPENDS
     libc.src.__support.macros.properties.types
     libc.src.__support.FPUtil.generic.sqrt
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
 )
 
 add_entrypoint_object(
@@ -5453,8 +4740,6 @@ add_entrypoint_object(
     cbrtf.cpp
   HDRS
     ../cbrtf.h
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
   DEPENDS
     libc.hdr.fenv_macros
     libc.src.__support.FPUtil.fenv_impl
@@ -5469,8 +4754,6 @@ add_entrypoint_object(
     cbrt.cpp
   HDRS
     ../cbrt.h
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
   DEPENDS
     libc.hdr.fenv_macros
     libc.src.__support.FPUtil.double_double
@@ -5491,8 +4774,6 @@ add_entrypoint_object(
     ../dmull.h
   DEPENDS
     libc.src.__support.FPUtil.generic.mul
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
 )
 
 add_entrypoint_object(
@@ -5504,8 +4785,6 @@ add_entrypoint_object(
   DEPENDS
     libc.src.__support.macros.properties.types
     libc.src.__support.FPUtil.generic.mul
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
 )
 
 add_entrypoint_object(
@@ -5517,8 +4796,6 @@ add_entrypoint_object(
   DEPENDS
     libc.src.__support.macros.properties.types
     libc.src.__support.FPUtil.generic.mul
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
 )
 
 add_entrypoint_object(
@@ -5530,8 +4807,6 @@ add_entrypoint_object(
   DEPENDS
     libc.src.__support.macros.properties.types
     libc.src.__support.FPUtil.generic.mul
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
 )
 
 add_entrypoint_object(
@@ -5543,8 +4818,6 @@ add_entrypoint_object(
   DEPENDS
     libc.src.__support.macros.properties.types
     libc.src.__support.FPUtil.generic.mul
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
 )
 
 add_entrypoint_object(
@@ -5556,8 +4829,6 @@ add_entrypoint_object(
   DEPENDS
     libc.src.__support.macros.properties.types
     libc.src.__support.FPUtil.generic.mul
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
 )
 
 add_header_library(
diff --git a/libc/src/math/generic/asinf16.cpp b/libc/src/math/generic/asinf16.cpp
new file mode 100644
index 0000000000000..518c384a61530
--- /dev/null
+++ b/libc/src/math/generic/asinf16.cpp
@@ -0,0 +1,133 @@
+//===-- Half-precision asinf16(x) function --------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception.
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/math/asinf16.h"
+#include "hdr/errno_macros.h"
+#include "hdr/fenv_macros.h"
+#include "src/__support/FPUtil/FEnvImpl.h"
+#include "src/__support/FPUtil/FPBits.h"
+#include "src/__support/FPUtil/PolyEval.h"
+#include "src/__support/FPUtil/cast.h"
+#include "src/__support/FPUtil/multiply_add.h"
+#include "src/__support/FPUtil/sqrt.h"
+#include "src/__support/macros/optimization.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+// Generated by Sollya using the following command:
+// > round(pi/2, D, RN);
+static constexpr float PI_2 = 0x1.921fb54442d18p0f;
+
+LLVM_LIBC_FUNCTION(float16, asinf16, (float16 x)) {
+  using FPBits = fputil::FPBits<float16>;
+  FPBits xbits(x);
+
+  uint16_t x_u = xbits.uintval();
+  uint16_t x_abs = x_u & 0x7fff;
+  float xf = x;
+
+  // |x| > 0x1p0, |x| > 1, or x is NaN.
+  if (LIBC_UNLIKELY(x_abs > 0x3c00)) {
+    // asinf16(NaN) = NaN
+    if (xbits.is_nan()) {
+      if (xbits.is_signaling_nan()) {
+        fputil::raise_except_if_required(FE_INVALID);
+        return FPBits::quiet_nan().get_val();
+      }
+
+      return x;
+    }
+
+    // 1 < |x| <= +/-inf
+    fputil::raise_except_if_required(FE_INVALID);
+    fputil::set_errno_if_required(EDOM);
+
+    return FPBits::quiet_nan().get_val();
+  }
+
+  float xsq = xf * xf;
+
+  // |x| <= 0x1p-1, |x| <= 0.5
+  if (x_abs <= 0x3800) {
+    // asinf16(+/-0) = +/-0
+    if (LIBC_UNLIKELY(x_abs == 0))
+      return x;
+
+    // Exhaustive tests show that,
+    // for |x| <= 0x1.878p-9, when:
+    // x > 0, and rounding upward, or
+    // x < 0, and rounding downward, then,
+    // asin(x) = x * 2^-11 + x
+    // else, in other rounding modes,
+    // asin(x) = x
+    if (LIBC_UNLIKELY(x_abs <= 0x1a1e)) {
+      int rounding = fputil::quick_get_round();
+
+      if ((xbits.is_pos() && rounding == FE_UPWARD) ||
+          (xbits.is_neg() && rounding == FE_DOWNWARD))
+        return fputil::cast<float16>(fputil::multiply_add(xf, 0x1.0p-11f, xf));
+      return x;
+    }
+
+    // Degree-6 minimax odd polynomial of asin(x) generated by Sollya with:
+    // > P = fpminimax(asin(x)/x, [|0, 2, 4, 6, 8|], [|SG...|], [0, 0.5]);
+    float result =
+        fputil::polyeval(xsq, 0x1.000002p0f, 0x1.554c2ap-3f, 0x1.3541ccp-4f,
+                         0x1.43b2d6p-5f, 0x1.a0d73ep-5f);
+    return fputil::cast<float16>(xf * result);
+  }
+
+  // When |x| > 0.5, assume that 0.5 < |x| <= 1,
+  //
+  // Step-by-step range-reduction proof:
+  // 1:  Let y = asin(x), such that, x = sin(y)
+  // 2:  From complimentary angle identity:
+  //       x = sin(y) = cos(pi/2 - y)
+  // 3:  Let z = pi/2 - y, such that x = cos(z)
+  // 4:  From double angle formula; cos(2A) = 1 - sin^2(A):
+  //       z = 2A, z/2 = A
+  //       cos(z) = 1 - 2 * sin^2(z/2)
+  // 5:  Make sin(z/2) subject of the formula:
+  //       sin(z/2) = sqrt((1 - cos(z))/2)
+  // 6:  Recall [3]; x = cos(z). Therefore:
+  //       sin(z/2) = sqrt((1 - x)/2)
+  // 7:  Let u = (1 - x)/2
+  // 8:  Therefore:
+  //       asin(sqrt(u)) = z/2
+  //       2 * asin(sqrt(u)) = z
+  // 9:  Recall [3], z = pi/2 - y. Therefore:
+  //       y = pi/2 - z
+  //       y = pi/2 - 2 * asin(sqrt(u))
+  // 10: Recall [1], y = asin(x). Therefore:
+  //       asin(x) = pi/2 - 2 * asin(sqrt(u))
+  //
+  // WHY?
+  // 11: Recall [7], u = (1 - x)/2
+  // 12: Since 0.5 < x <= 1, therefore:
+  //       0 <= u <= 0.25 and 0 <= sqrt(u) <= 0.5
+  //
+  // Hence, we can reuse the same [0, 0.5] domain polynomial approximation for
+  // Step [10] as `sqrt(u)` is in range.
+
+  // 0x1p-1 < |x| <= 0x1p0, 0.5 < |x| <= 1.0
+  float xf_abs = (xf < 0 ? -xf : xf);
+  float sign = (xbits.uintval() >> 15 == 1 ? -1.0 : 1.0);
+  float u = fputil::multiply_add(-0.5f, xf_abs, 0.5f);
+  float u_sqrt = fputil::sqrt<float>(u);
+
+  // Degree-6 minimax odd polynomial of asin(x) generated by Sollya with:
+  // > P = fpminimax(asin(x)/x, [|0, 2, 4, 6, 8|], [|SG...|], [0, 0.5]);
+  float asin_sqrt_u =
+      u_sqrt * fputil::polyeval(u, 0x1.000002p0f, 0x1.554c2ap-3f,
+                                0x1.3541ccp-4f, 0x1.43b2d6p-5f, 0x1.a0d73ep-5f);
+
+  return fputil::cast<float16>(sign *
+                               fputil::multiply_add(-2.0f, asin_sqrt_u, PI_2));
+}
+
+} // namespace LIBC_NAMESPACE_DECL
diff --git a/libc/src/math/generic/atan2f.cpp b/libc/src/math/generic/atan2f.cpp
index db7639396cdd7..5ac2b29438ea9 100644
--- a/libc/src/math/generic/atan2f.cpp
+++ b/libc/src/math/generic/atan2f.cpp
@@ -17,6 +17,14 @@
 #include "src/__support/macros/config.h"
 #include "src/__support/macros/optimization.h" // LIBC_UNLIKELY
 
+#if defined(LIBC_MATH_HAS_SKIP_ACCURATE_PASS) &&                               \
+    defined(LIBC_MATH_HAS_INTERMEDIATE_COMP_IN_FLOAT)
+
+// We use float-float implementation to reduce size.
+#include "src/math/generic/atan2f_float.h"
+
+#else
+
 namespace LIBC_NAMESPACE_DECL {
 
 namespace {
@@ -324,3 +332,5 @@ LLVM_LIBC_FUNCTION(float, atan2f, (float y, float x)) {
 }
 
 } // namespace LIBC_NAMESPACE_DECL
+
+#endif
diff --git a/libc/src/math/generic/atan2f_float.h b/libc/src/math/generic/atan2f_float.h
new file mode 100644
index 0000000000000..1fd853d735950
--- /dev/null
+++ b/libc/src/math/generic/atan2f_float.h
@@ -0,0 +1,237 @@
+//===-- Single-precision atan2f function ----------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/__support/FPUtil/FPBits.h"
+#include "src/__support/FPUtil/double_double.h"
+#include "src/__support/FPUtil/multiply_add.h"
+#include "src/__support/FPUtil/nearest_integer.h"
+#include "src/__support/FPUtil/rounding_mode.h"
+#include "src/__support/macros/config.h"
+#include "src/__support/macros/optimization.h" // LIBC_UNLIKELY
+#include "src/math/atan2f.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+namespace {
+
+using FloatFloat = fputil::FloatFloat;
+
+// atan(i/64) with i = 0..16, generated by Sollya with:
+// > for i from 0 to 16 do {
+//     a = round(atan(i/16), SG, RN);
+//     b = round(atan(i/16) - a, SG, RN);
+//     print("{", b, ",", a, "},");
+//   };
+constexpr FloatFloat ATAN_I[17] = {
+    {0.0f, 0.0f},
+    {-0x1.1a6042p-30f, 0x1.ff55bcp-5f},
+    {-0x1.54f424p-30f, 0x1.fd5baap-4f},
+    {0x1.79cb6p-28f, 0x1.7b97b4p-3f},
+    {-0x1.b4dfc8p-29f, 0x1.f5b76p-3f},
+    {-0x1.1f0286p-27f, 0x1.362774p-2f},
+    {0x1.e4defp-30f, 0x1.6f6194p-2f},
+    {0x1.e611fep-29f, 0x1.a64eecp-2f},
+    {0x1.586ed4p-28f, 0x1.dac67p-2f},
+    {-0x1.6499e6p-26f, 0x1.0657eap-1f},
+    {0x1.7bdfd6p-26f, 0x1.1e00bap-1f},
+    {-0x1.98e422p-28f, 0x1.345f02p-1f},
+    {0x1.934f7p-28f, 0x1.4978fap-1f},
+    {0x1.c5a6c6p-27f, 0x1.5d5898p-1f},
+    {0x1.5e118cp-27f, 0x1.700a7cp-1f},
+    {-0x1.1d4eb6p-26f, 0x1.819d0cp-1f},
+    {-0x1.777a5cp-26f, 0x1.921fb6p-1f},
+};
+
+// Approximate atan(x) for |x| <= 2^-5.
+// Using degree-3 Taylor polynomial:
+//  P = x - x^3/3
+// Then the absolute error is bounded by:
+//   |atan(x) - P(x)| < |x|^5/5 < 2^(-5*5) / 5 < 2^-27.
+// And the relative error is bounded by:
+//   |(atan(x) - P(x))/atan(x)| < |x|^4 / 4 < 2^-22.
+// For x = x_hi + x_lo, fully expand the polynomial and drop any terms less than
+//   ulp(x_hi^3 / 3) gives us:
+// P(x) ~ x_hi - x_hi^3/3 + x_lo * (1 - x_hi^2)
+FloatFloat atan_eval(const FloatFloat &x) {
+  FloatFloat p;
+  p.hi = x.hi;
+  float x_hi_sq = x.hi * x.hi;
+  // c0 ~ - x_hi^2 / 3
+  float c0 = -0x1.555556p-2f * x_hi_sq;
+  // c1 ~ x_lo * (1 - x_hi^2)
+  float c1 = fputil::multiply_add(x_hi_sq, -x.lo, x.lo);
+  // p.lo ~ - x_hi^3 / 3 + x_lo * (1 - x_hi*2)
+  p.lo = fputil::multiply_add(x.hi, c0, c1);
+  return p;
+}
+
+} // anonymous namespace
+
+// There are several range reduction steps we can take for atan2(y, x) as
+// follow:
+
+// * Range reduction 1: signness
+// atan2(y, x) will return a number between -PI and PI representing the angle
+// forming by the 0x axis and the vector (x, y) on the 0xy-plane.
+// In particular, we have that:
+//   atan2(y, x) = atan( y/x )         if x >= 0 and y >= 0 (I-quadrant)
+//               = pi + atan( y/x )    if x < 0 and y >= 0  (II-quadrant)
+//               = -pi + atan( y/x )   if x < 0 and y < 0   (III-quadrant)
+//               = atan( y/x )         if x >= 0 and y < 0  (IV-quadrant)
+// Since atan function is odd, we can use the formula:
+//   atan(-u) = -atan(u)
+// to adjust the above conditions a bit further:
+//   atan2(y, x) = atan( |y|/|x| )         if x >= 0 and y >= 0 (I-quadrant)
+//               = pi - atan( |y|/|x| )    if x < 0 and y >= 0  (II-quadrant)
+//               = -pi + atan( |y|/|x| )   if x < 0 and y < 0   (III-quadrant)
+//               = -atan( |y|/|x| )        if x >= 0 and y < 0  (IV-quadrant)
+// Which can be simplified to:
+//   atan2(y, x) = sign(y) * atan( |y|/|x| )             if x >= 0
+//               = sign(y) * (pi - atan( |y|/|x| ))      if x < 0
+
+// * Range reduction 2: reciprocal
+// Now that the argument inside atan is positive, we can use the formula:
+//   atan(1/x) = pi/2 - atan(x)
+// to make the argument inside atan <= 1 as follow:
+//   atan2(y, x) = sign(y) * atan( |y|/|x|)            if 0 <= |y| <= x
+//               = sign(y) * (pi/2 - atan( |x|/|y| )   if 0 <= x < |y|
+//               = sign(y) * (pi - atan( |y|/|x| ))    if 0 <= |y| <= -x
+//               = sign(y) * (pi/2 + atan( |x|/|y| ))  if 0 <= -x < |y|
+
+// * Range reduction 3: look up table.
+// After the previous two range reduction steps, we reduce the problem to
+// compute atan(u) with 0 <= u <= 1, or to be precise:
+//   atan( n / d ) where n = min(|x|, |y|) and d = max(|x|, |y|).
+// An accurate polynomial approximation for the whole [0, 1] input range will
+// require a very large degree.  To make it more efficient, we reduce the input
+// range further by finding an integer idx such that:
+//   | n/d - idx/16 | <= 1/32.
+// In particular,
+//   idx := 2^-4 * round(2^4 * n/d)
+// Then for the fast pass, we find a polynomial approximation for:
+//   atan( n/d ) ~ atan( idx/16 ) + (n/d - idx/16) * Q(n/d - idx/16)
+// with Q(x) = x - x^3/3 be the cubic Taylor polynomial of atan(x).
+// It's error in float-float precision is estimated in Sollya to be:
+// > P = x - x^3/3;
+// > dirtyinfnorm(atan(x) - P, [-2^-5, 2^-5]);
+// 0x1.995...p-28.
+
+LLVM_LIBC_FUNCTION(float, atan2f, (float y, float x)) {
+  using FPBits = typename fputil::FPBits<float>;
+  constexpr float IS_NEG[2] = {1.0f, -1.0f};
+  constexpr FloatFloat ZERO = {0.0f, 0.0f};
+  constexpr FloatFloat MZERO = {-0.0f, -0.0f};
+  constexpr FloatFloat PI = {-0x1.777a5cp-24f, 0x1.921fb6p1f};
+  constexpr FloatFloat MPI = {0x1.777a5cp-24f, -0x1.921fb6p1f};
+  constexpr FloatFloat PI_OVER_4 = {-0x1.777a5cp-26f, 0x1.921fb6p-1f};
+  constexpr FloatFloat PI_OVER_2 = {-0x1.777a5cp-25f, 0x1.921fb6p0f};
+  constexpr FloatFloat MPI_OVER_2 = {-0x1.777a5cp-25f, 0x1.921fb6p0f};
+  constexpr FloatFloat THREE_PI_OVER_4 = {-0x1.99bc5cp-28f, 0x1.2d97c8p1f};
+  // Adjustment for constant term:
+  //   CONST_ADJ[x_sign][y_sign][recip]
+  constexpr FloatFloat CONST_ADJ[2][2][2] = {
+      {{ZERO, MPI_OVER_2}, {MZERO, MPI_OVER_2}},
+      {{MPI, PI_OVER_2}, {MPI, PI_OVER_2}}};
+
+  FPBits x_bits(x), y_bits(y);
+  bool x_sign = x_bits.sign().is_neg();
+  bool y_sign = y_bits.sign().is_neg();
+  x_bits = x_bits.abs();
+  y_bits = y_bits.abs();
+  uint32_t x_abs = x_bits.uintval();
+  uint32_t y_abs = y_bits.uintval();
+  bool recip = x_abs < y_abs;
+  uint32_t min_abs = recip ? x_abs : y_abs;
+  uint32_t max_abs = !recip ? x_abs : y_abs;
+  auto min_exp = static_cast<unsigned>(min_abs >> FPBits::FRACTION_LEN);
+  auto max_exp = static_cast<unsigned>(max_abs >> FPBits::FRACTION_LEN);
+
+  float num = FPBits(min_abs).get_val();
+  float den = FPBits(max_abs).get_val();
+
+  // Check for exceptional cases, whether inputs are 0, inf, nan, or close to
+  // overflow, or close to underflow.
+  if (LIBC_UNLIKELY(max_exp > 0xffU - 64U || min_exp < 64U)) {
+    if (x_bits.is_nan() || y_bits.is_nan())
+      return FPBits::quiet_nan().get_val();
+    unsigned x_except = x == 0.0f ? 0 : (FPBits(x_abs).is_inf() ? 2 : 1);
+    unsigned y_except = y == 0.0f ? 0 : (FPBits(y_abs).is_inf() ? 2 : 1);
+
+    // Exceptional cases:
+    //   EXCEPT[y_except][x_except][x_is_neg]
+    // with x_except & y_except:
+    //   0: zero
+    //   1: finite, non-zero
+    //   2: infinity
+    constexpr FloatFloat EXCEPTS[3][3][2] = {
+        {{ZERO, PI}, {ZERO, PI}, {ZERO, PI}},
+        {{PI_OVER_2, PI_OVER_2}, {ZERO, ZERO}, {ZERO, PI}},
+        {{PI_OVER_2, PI_OVER_2},
+         {PI_OVER_2, PI_OVER_2},
+         {PI_OVER_4, THREE_PI_OVER_4}},
+    };
+
+    if ((x_except != 1) || (y_except != 1)) {
+      FloatFloat r = EXCEPTS[y_except][x_except][x_sign];
+      return fputil::multiply_add(IS_NEG[y_sign], r.hi, IS_NEG[y_sign] * r.lo);
+    }
+    bool scale_up = min_exp < 64U;
+    bool scale_down = max_exp > 0xffU - 64U;
+    // At least one input is denormal, multiply both numerator and denominator
+    // by some large enough power of 2 to normalize denormal inputs.
+    if (scale_up) {
+      num *= 0x1.0p32f;
+      if (!scale_down)
+        den *= 0x1.0p32f;
+    } else if (scale_down) {
+      den *= 0x1.0p-32f;
+      num *= 0x1.0p-32f;
+    }
+
+    min_abs = FPBits(num).uintval();
+    max_abs = FPBits(den).uintval();
+    min_exp = static_cast<unsigned>(min_abs >> FPBits::FRACTION_LEN);
+    max_exp = static_cast<unsigned>(max_abs >> FPBits::FRACTION_LEN);
+  }
+
+  float final_sign = IS_NEG[(x_sign != y_sign) != recip];
+  FloatFloat const_term = CONST_ADJ[x_sign][y_sign][recip];
+  unsigned exp_diff = max_exp - min_exp;
+  // We have the following bound for normalized n and d:
+  //   2^(-exp_diff - 1) < n/d < 2^(-exp_diff + 1).
+  if (LIBC_UNLIKELY(exp_diff > 25))
+    return fputil::multiply_add(final_sign, const_term.hi,
+                                final_sign * (const_term.lo + num / den));
+
+  float k = fputil::nearest_integer(16.0f * num / den);
+  unsigned idx = static_cast<unsigned>(k);
+  // k = idx / 16
+  k *= 0x1.0p-4f;
+
+  // Range reduction:
+  // atan(n/d) - atan(k/64) = atan((n/d - k/16) / (1 + (n/d) * (k/16)))
+  //                        = atan((n - d * k/16)) / (d + n * k/16))
+  FloatFloat num_k = fputil::exact_mult(num, k);
+  FloatFloat den_k = fputil::exact_mult(den, k);
+
+  // num_dd = n - d * k
+  FloatFloat num_ff = fputil::exact_add(num - den_k.hi, -den_k.lo);
+  // den_dd = d + n * k
+  FloatFloat den_ff = fputil::exact_add(den, num_k.hi);
+  den_ff.lo += num_k.lo;
+
+  // q = (n - d * k) / (d + n * k)
+  FloatFloat q = fputil::div(num_ff, den_ff);
+  // p ~ atan(q)
+  FloatFloat p = atan_eval(q);
+
+  FloatFloat r = fputil::add(const_term, fputil::add(ATAN_I[idx], p));
+  return final_sign * r.hi;
+}
+
+} // namespace LIBC_NAMESPACE_DECL
diff --git a/libc/src/math/generic/pow.cpp b/libc/src/math/generic/pow.cpp
index 213dbd959039c..a2a0bb698f81a 100644
--- a/libc/src/math/generic/pow.cpp
+++ b/libc/src/math/generic/pow.cpp
@@ -400,7 +400,7 @@ LLVM_LIBC_FUNCTION(double, pow, (double x, double y)) {
 #else
   double c = FPBits(m_x.uintval() & 0x3fff'e000'0000'0000).get_val();
   dx = fputil::multiply_add(RD[idx_x], m_x.get_val() - c, CD[idx_x]); // Exact
-  dx_c0 = fputil::exact_mult<28>(dx, COEFFS[0]);                      // Exact
+  dx_c0 = fputil::exact_mult<double, 28>(dx, COEFFS[0]);              // Exact
 #endif // LIBC_TARGET_CPU_HAS_FMA
 
   double dx2 = dx * dx;
diff --git a/libc/src/math/generic/range_reduction_double_fma.h b/libc/src/math/generic/range_reduction_double_fma.h
index cab031c28baa1..8e0bc3a42462c 100644
--- a/libc/src/math/generic/range_reduction_double_fma.h
+++ b/libc/src/math/generic/range_reduction_double_fma.h
@@ -33,14 +33,14 @@ LIBC_INLINE unsigned LargeRangeReduction::fast(double x, DoubleDouble &u) {
   // 2^62 <= |x_reduced| < 2^(62 + 16) = 2^78
   x_reduced = xbits.get_val();
   // x * c_hi = ph.hi + ph.lo exactly.
-  DoubleDouble ph =
-      fputil::exact_mult<SPLIT>(x_reduced, ONE_TWENTY_EIGHT_OVER_PI[idx][0]);
+  DoubleDouble ph = fputil::exact_mult<double, SPLIT>(
+      x_reduced, ONE_TWENTY_EIGHT_OVER_PI[idx][0]);
   // x * c_mid = pm.hi + pm.lo exactly.
-  DoubleDouble pm =
-      fputil::exact_mult<SPLIT>(x_reduced, ONE_TWENTY_EIGHT_OVER_PI[idx][1]);
+  DoubleDouble pm = fputil::exact_mult<double, SPLIT>(
+      x_reduced, ONE_TWENTY_EIGHT_OVER_PI[idx][1]);
   // x * c_lo = pl.hi + pl.lo exactly.
-  DoubleDouble pl =
-      fputil::exact_mult<SPLIT>(x_reduced, ONE_TWENTY_EIGHT_OVER_PI[idx][2]);
+  DoubleDouble pl = fputil::exact_mult<double, SPLIT>(
+      x_reduced, ONE_TWENTY_EIGHT_OVER_PI[idx][2]);
   // Extract integral parts and fractional parts of (ph.lo + pm.hi).
   double sum_hi = ph.lo + pm.hi;
   double kd = fputil::nearest_integer(sum_hi);
diff --git a/libc/src/math/generic/range_reduction_double_nofma.h b/libc/src/math/generic/range_reduction_double_nofma.h
index 5640732947798..606c3f8185d61 100644
--- a/libc/src/math/generic/range_reduction_double_nofma.h
+++ b/libc/src/math/generic/range_reduction_double_nofma.h
@@ -34,14 +34,14 @@ LIBC_INLINE unsigned LargeRangeReduction::fast(double x, DoubleDouble &u) {
   x_reduced = xbits.get_val();
   // x * c_hi = ph.hi + ph.lo exactly.
   DoubleDouble x_split = fputil::split(x_reduced);
-  DoubleDouble ph = fputil::exact_mult<SPLIT>(x_split, x_reduced,
-                                              ONE_TWENTY_EIGHT_OVER_PI[idx][0]);
+  DoubleDouble ph = fputil::exact_mult<double, SPLIT>(
+      x_split, x_reduced, ONE_TWENTY_EIGHT_OVER_PI[idx][0]);
   // x * c_mid = pm.hi + pm.lo exactly.
-  DoubleDouble pm = fputil::exact_mult<SPLIT>(x_split, x_reduced,
-                                              ONE_TWENTY_EIGHT_OVER_PI[idx][1]);
+  DoubleDouble pm = fputil::exact_mult<double, SPLIT>(
+      x_split, x_reduced, ONE_TWENTY_EIGHT_OVER_PI[idx][1]);
   // x * c_lo = pl.hi + pl.lo exactly.
-  DoubleDouble pl = fputil::exact_mult<SPLIT>(x_split, x_reduced,
-                                              ONE_TWENTY_EIGHT_OVER_PI[idx][2]);
+  DoubleDouble pl = fputil::exact_mult<double, SPLIT>(
+      x_split, x_reduced, ONE_TWENTY_EIGHT_OVER_PI[idx][2]);
   // Extract integral parts and fractional parts of (ph.lo + pm.hi).
   double sum_hi = ph.lo + pm.hi;
   double kd = fputil::nearest_integer(sum_hi);
diff --git a/libc/src/poll/CMakeLists.txt b/libc/src/poll/CMakeLists.txt
new file mode 100644
index 0000000000000..65fbe46bb4612
--- /dev/null
+++ b/libc/src/poll/CMakeLists.txt
@@ -0,0 +1,10 @@
+if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/${LIBC_TARGET_OS})
+  add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/${LIBC_TARGET_OS})
+endif()
+
+add_entrypoint_object(
+  poll
+  ALIAS
+  DEPENDS
+    .${LIBC_TARGET_OS}.poll
+)
diff --git a/libc/src/poll/linux/CMakeLists.txt b/libc/src/poll/linux/CMakeLists.txt
new file mode 100644
index 0000000000000..a9ef596c36c89
--- /dev/null
+++ b/libc/src/poll/linux/CMakeLists.txt
@@ -0,0 +1,14 @@
+add_entrypoint_object(
+  poll
+  SRCS
+    poll.cpp
+  HDRS
+    ../poll.h
+  DEPENDS
+    libc.hdr.types.nfds_t
+    libc.hdr.types.struct_pollfd
+    libc.hdr.types.struct_timespec
+    libc.include.sys_syscall
+    libc.src.__support.OSUtil.osutil
+    libc.src.errno.errno
+)
diff --git a/libc/src/poll/linux/poll.cpp b/libc/src/poll/linux/poll.cpp
new file mode 100644
index 0000000000000..d7c195878ae12
--- /dev/null
+++ b/libc/src/poll/linux/poll.cpp
@@ -0,0 +1,51 @@
+//===-- Implementation of poll --------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/poll/poll.h"
+
+#include "hdr/types/nfds_t.h"
+#include "hdr/types/struct_pollfd.h"
+#include "hdr/types/struct_timespec.h"
+#include "src/__support/OSUtil/syscall.h" // syscall_impl
+#include "src/__support/common.h"
+#include "src/__support/macros/config.h"
+#include "src/errno/libc_errno.h"
+
+#include <sys/syscall.h> // SYS_poll, SYS_ppoll
+
+namespace LIBC_NAMESPACE_DECL {
+
+LLVM_LIBC_FUNCTION(int, poll, (pollfd * fds, nfds_t nfds, int timeout)) {
+  int ret = 0;
+
+#ifdef SYS_poll
+  ret = LIBC_NAMESPACE::syscall_impl<int>(SYS_poll, fds, nfds, timeout);
+#elif defined(SYS_ppoll)
+  timespec ts, *tsp;
+  if (timeout >= 0) {
+    ts.tv_sec = timeout / 1000;
+    ts.tv_nsec = (timeout % 1000) * 1000000;
+    tsp = &ts;
+  } else {
+    tsp = nullptr;
+  }
+  ret =
+      LIBC_NAMESPACE::syscall_impl<int>(SYS_ppoll, fds, nfds, tsp, nullptr, 0);
+#else
+// TODO: https://github.com/llvm/llvm-project/issues/125940
+#error "SYS_ppoll_time64?"
+#endif
+
+  if (ret < 0) {
+    libc_errno = -ret;
+    return -1;
+  }
+  return ret;
+}
+
+} // namespace LIBC_NAMESPACE_DECL
diff --git a/libc/src/poll/poll.h b/libc/src/poll/poll.h
new file mode 100644
index 0000000000000..ac3cc1d36a71d
--- /dev/null
+++ b/libc/src/poll/poll.h
@@ -0,0 +1,22 @@
+//===-- Implementation header for poll ----------------------------*-C++-*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_POLL_POLL_H
+#define LLVM_LIBC_SRC_POLL_POLL_H
+
+#include "hdr/types/nfds_t.h"
+#include "hdr/types/struct_pollfd.h"
+#include "src/__support/macros/config.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+int poll(pollfd *fds, nfds_t nfds, int timeout);
+
+} // namespace LIBC_NAMESPACE_DECL
+
+#endif //  LLVM_LIBC_SRC_POLL_POLL_H
diff --git a/libc/src/setjmp/setjmp_impl.h b/libc/src/setjmp/setjmp_impl.h
index d035409e58195..669f720bda5d3 100644
--- a/libc/src/setjmp/setjmp_impl.h
+++ b/libc/src/setjmp/setjmp_impl.h
@@ -29,7 +29,7 @@ namespace LIBC_NAMESPACE_DECL {
 #ifdef LIBC_COMPILER_IS_GCC
 [[gnu::nothrow]]
 #endif
-int setjmp(jmp_buf buf);
+__attribute__((returns_twice)) int setjmp(jmp_buf buf);
 
 } // namespace LIBC_NAMESPACE_DECL
 
diff --git a/libc/src/signal/linux/CMakeLists.txt b/libc/src/signal/linux/CMakeLists.txt
index f7457d31cf4f8..c0dd61e473881 100644
--- a/libc/src/signal/linux/CMakeLists.txt
+++ b/libc/src/signal/linux/CMakeLists.txt
@@ -127,7 +127,6 @@ add_entrypoint_object(
   DEPENDS
     .sigaction
     libc.hdr.signal_macros
-    libc.hdr.types.sighandler_t
 )
 
 add_entrypoint_object(
diff --git a/libc/src/signal/linux/signal.cpp b/libc/src/signal/linux/signal.cpp
index 1da0ef8c97a20..7c8ea16c6cd2e 100644
--- a/libc/src/signal/linux/signal.cpp
+++ b/libc/src/signal/linux/signal.cpp
@@ -8,14 +8,17 @@
 
 #include "src/signal/signal.h"
 #include "hdr/signal_macros.h"
-#include "hdr/types/sighandler_t.h"
 #include "src/__support/common.h"
 #include "src/__support/macros/config.h"
 #include "src/signal/sigaction.h"
 
 namespace LIBC_NAMESPACE_DECL {
 
-LLVM_LIBC_FUNCTION(sighandler_t, signal, (int signum, sighandler_t handler)) {
+// Our LLVM_LIBC_FUNCTION macro doesn't handle function pointer return types.
+using signal_handler = void (*)(int);
+
+LLVM_LIBC_FUNCTION(signal_handler, signal,
+                   (int signum, signal_handler handler)) {
   struct sigaction action, old;
   action.sa_handler = handler;
   action.sa_flags = SA_RESTART;
diff --git a/libc/src/signal/signal.h b/libc/src/signal/signal.h
index 06e77e11bf0bd..e1f31a8e126c5 100644
--- a/libc/src/signal/signal.h
+++ b/libc/src/signal/signal.h
@@ -9,12 +9,11 @@
 #ifndef LLVM_LIBC_SRC_SIGNAL_SIGNAL_H
 #define LLVM_LIBC_SRC_SIGNAL_SIGNAL_H
 
-#include "hdr/types/sighandler_t.h"
 #include "src/__support/macros/config.h"
 
 namespace LIBC_NAMESPACE_DECL {
 
-sighandler_t signal(int signum, sighandler_t handler);
+void (*signal(int signum, void (*handler)(int)))(int);
 
 } // namespace LIBC_NAMESPACE_DECL
 
diff --git a/libc/src/stdfix/CMakeLists.txt b/libc/src/stdfix/CMakeLists.txt
index 815f739d23efa..37292d85367fe 100644
--- a/libc/src/stdfix/CMakeLists.txt
+++ b/libc/src/stdfix/CMakeLists.txt
@@ -9,8 +9,6 @@ foreach(suffix IN ITEMS hr r lr hk k lk)
       abs${suffix}.h
     SRCS
       abs${suffix}.cpp
-    COMPILE_OPTIONS
-      ${libc_opt_high_flag}
     DEPENDS
       libc.src.__support.fixed_point.fx_bits
   )
@@ -23,8 +21,6 @@ foreach(suffix IN ITEMS uhr ur ulr uhk uk)
       sqrt${suffix}.h
     SRCS
       sqrt${suffix}.cpp
-    COMPILE_OPTIONS
-      ${libc_opt_high_flag}
     DEPENDS
       libc.src.__support.fixed_point.sqrt
   )
@@ -37,8 +33,6 @@ foreach(suffix IN ITEMS hr r lr hk k lk uhr ur ulr uhk uk ulk)
       round${suffix}.h
     SRCS
       round${suffix}.cpp
-    COMPILE_OPTIONS
-      ${libc_opt_high_flag}
     DEPENDS
       libc.src.__support.fixed_point.fx_bits
   )
@@ -61,8 +55,6 @@ add_entrypoint_object(
     uhksqrtus.h
   SRCS
     uhksqrtus.cpp
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
   DEPENDS
     libc.src.__support.fixed_point.sqrt
 )
@@ -73,8 +65,6 @@ add_entrypoint_object(
     uksqrtui.h
   SRCS
     uksqrtui.cpp
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
   DEPENDS
     libc.src.__support.fixed_point.sqrt
 )
@@ -85,8 +75,6 @@ add_entrypoint_object(
     exphk.h
   SRCS
     exphk.cpp
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
   DEPENDS
     libc.src.__support.fixed_point.fx_rep
     libc.src.__support.CPP.bit
@@ -98,8 +86,6 @@ add_entrypoint_object(
     expk.h
   SRCS
     expk.cpp
-  COMPILE_OPTIONS
-    ${libc_opt_high_flag}
   DEPENDS
     libc.src.__support.fixed_point.fx_rep
     libc.src.__support.CPP.bit
diff --git a/libc/src/stdio/printf_core/CMakeLists.txt b/libc/src/stdio/printf_core/CMakeLists.txt
index 9eaffe2f7ed62..ea58067c7070a 100644
--- a/libc/src/stdio/printf_core/CMakeLists.txt
+++ b/libc/src/stdio/printf_core/CMakeLists.txt
@@ -16,6 +16,9 @@ endif()
 if(LIBC_CONF_PRINTF_FLOAT_TO_STR_NO_SPECIALIZE_LD)
   list(APPEND printf_config_copts "-DLIBC_COPT_FLOAT_TO_STR_NO_SPECIALIZE_LD")
 endif()
+if(LIBC_CONF_PRINTF_FLOAT_TO_STR_USE_FLOAT320)
+  list(APPEND printf_config_copts "-DLIBC_COPT_FLOAT_TO_STR_USE_FLOAT320")
+endif()
 if(LIBC_CONF_PRINTF_DISABLE_FIXED_POINT)
   list(APPEND printf_config_copts "-DLIBC_COPT_PRINTF_DISABLE_FIXED_POINT")
 endif()
diff --git a/libc/src/stdio/printf_core/converter_atlas.h b/libc/src/stdio/printf_core/converter_atlas.h
index 18cfe1e717cbe..dfb91b30e80f8 100644
--- a/libc/src/stdio/printf_core/converter_atlas.h
+++ b/libc/src/stdio/printf_core/converter_atlas.h
@@ -26,7 +26,11 @@
 // defines convert_float_decimal
 // defines convert_float_dec_exp
 // defines convert_float_dec_auto
+#ifdef LIBC_COPT_FLOAT_TO_STR_USE_FLOAT320
+#include "src/stdio/printf_core/float_dec_converter_limited.h"
+#else
 #include "src/stdio/printf_core/float_dec_converter.h"
+#endif
 // defines convert_float_hex_exp
 #include "src/stdio/printf_core/float_hex_converter.h"
 #endif // LIBC_COPT_PRINTF_DISABLE_FLOAT
diff --git a/libc/src/stdio/printf_core/float_dec_converter_limited.h b/libc/src/stdio/printf_core/float_dec_converter_limited.h
new file mode 100644
index 0000000000000..95516aa02eaa6
--- /dev/null
+++ b/libc/src/stdio/printf_core/float_dec_converter_limited.h
@@ -0,0 +1,691 @@
+//===-- Decimal Float Converter for printf (320-bit float) ------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file implements an alternative to the Ryū printf algorithm in
+// float_dec_converter.h. Instead of generating output digits 9 at a time on
+// demand, in this implementation, a float is converted to decimal by computing
+// just one power of 10 and multiplying/dividing the entire input by it,
+// generating the whole string of decimal output digits in one go.
+//
+// This avoids the large constant lookup table of Ryū, making it more suitable
+// for low-memory embedded contexts; but it's also faster than the fallback
+// version of Ryū which computes table entries on demand using DyadicFloat,
+// because those must calculate a potentially large power of 10 per 9-digit
+// output block, whereas this computes just one, which does the whole job.
+//
+// The calculation is done in 320-bit DyadicFloat, which provides enough
+// precision to generate 39 correct digits of output from any floating-point
+// size up to and including 128-bit long double, because the rounding errors in
+// computing the largest necessary power of 10 are still smaller than the
+// distance (in the 320-bit float format) between adjacent 39-decimal-digit
+// outputs.
+//
+// No further digits beyond the 39th are generated: if the printf format string
+// asks for more precision than that, the answer is padded with 0s. This is a
+// permitted option in IEEE 754-2019 (section 5.12.2): you're allowed to define
+// a limit H on the number of decimal digits you can generate, and pad with 0s
+// if asked for more than that, subject to the constraint that H must be
+// consistent across all float formats you support (you can't use a smaller H
+// for single precision than double or long double), and must be large enough
+// that even in the largest supported precision the only numbers misrounded are
+// ones extremely close to a rounding boundary. 39 digits is the smallest
+// permitted value for an implementation supporting binary128.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_STDIO_PRINTF_CORE_FLOAT_DEC_CONVERTER_LIMITED_H
+#define LLVM_LIBC_SRC_STDIO_PRINTF_CORE_FLOAT_DEC_CONVERTER_LIMITED_H
+
+#include "src/__support/CPP/algorithm.h"
+#include "src/__support/CPP/string.h"
+#include "src/__support/CPP/string_view.h"
+#include "src/__support/FPUtil/FPBits.h"
+#include "src/__support/FPUtil/dyadic_float.h"
+#include "src/__support/FPUtil/rounding_mode.h"
+#include "src/__support/integer_to_string.h"
+#include "src/__support/libc_assert.h"
+#include "src/__support/macros/config.h"
+#include "src/stdio/printf_core/core_structs.h"
+#include "src/stdio/printf_core/float_inf_nan_converter.h"
+#include "src/stdio/printf_core/writer.h"
+
+namespace LIBC_NAMESPACE_DECL {
+namespace printf_core {
+
+enum class ConversionType { E, F, G };
+using StorageType = fputil::FPBits<long double>::StorageType;
+
+constexpr unsigned MAX_DIGITS = 39;
+constexpr size_t DF_BITS = 320;
+constexpr char DECIMAL_POINT = '.';
+
+struct DigitsInput {
+  // Input mantissa, stored with the explicit leading 1 bit (if any) at the
+  // top. So either it has a value in the range [2^127,2^128) representing a
+  // real number in [1,2), or it has the value 0, representing 0.
+  UInt128 mantissa;
+
+  // Input exponent, as a power of 2 to multiply into mantissa.
+  int exponent;
+
+  // Input sign.
+  Sign sign;
+
+  // Constructor which accepts a mantissa direct from a floating-point format,
+  // and shifts it up to the top of the UInt128 so that a function consuming
+  // this struct afterwards doesn't have to remember which format it came from.
+  DigitsInput(int32_t fraction_len, StorageType mantissa_, int exponent_,
+              Sign sign)
+      : mantissa(UInt128(mantissa_) << (127 - fraction_len)),
+        exponent(exponent_), sign(sign) {
+    if (!(mantissa & (UInt128(1) << 127)) && mantissa != 0) {
+      // Normalize a denormalized input.
+      int shift = cpp::countl_zero(mantissa);
+      mantissa <<= shift;
+      exponent -= shift;
+    }
+  }
+};
+
+struct DigitsOutput {
+  // Output from decimal_digits().
+  //
+  // `digits` is a buffer containing nothing but ASCII digits. Even if the
+  // decimal point needs to appear somewhere in the final output string, it
+  // isn't represented in _this_ string; the client of this object will insert
+  // it in an appropriate place. `ndigits` gives the buffer size.
+  //
+  // `exponent` represents the exponent you would display if the decimal point
+  // comes after the first digit of decimal_digits, e.g. if digits == "1234"
+  // and exponent = 3 then this represents 1.234e3, or just the integer 1234.
+  size_t ndigits;
+  int exponent;
+  char digits[MAX_DIGITS + 1];
+};
+
+// Estimate log10 of a power of 2, by multiplying its exponent by
+// 1292913986/2^32. That is a rounded-down approximation to log10(2), accurate
+// enough that for any binary exponent in the range of float128 it will give
+// the correct value of floor(log10(2^n)).
+LIBC_INLINE int estimate_log10(int exponent_of_2) {
+  return static_cast<int>((exponent_of_2 * 1292913986LL) >> 32);
+}
+
+// Calculate the actual digits of a decimal representation of an FP number.
+//
+// If `e_mode` is true, then `precision` indicates the desired number of output
+// decimal digits. On return, `decimal_digits` will be a string of length
+// exactly `precision` starting with a nonzero digit; `decimal_exponent` will
+// be filled in to indicate the exponent as shown above.
+//
+// If `e_mode` is false, then `precision` indicates the desired number of
+// digits after the decimal point. On return, the last digit in the string
+// `decimal_digits` has a place value of _at least_ 10^-precision. But also, at
+// most `MAX_DIGITS` digits are returned, so the caller may need to pad it at
+// the end with the appropriate number of extra 0s.
+LIBC_INLINE
+DigitsOutput decimal_digits(DigitsInput input, int precision, bool e_mode) {
+  if (input.mantissa == 0) {
+    // Special-case zero, by manually generating the right number of zero
+    // digits and setting an appropriate exponent.
+    DigitsOutput output;
+    if (!e_mode) {
+      // In F mode, it's enough to return an empty string of digits. That's the
+      // same thing we do when given a nonzero number that rounds down to 0.
+      output.ndigits = 0;
+      output.exponent = -precision - 1;
+    } else {
+      // In E mode, generate a string containing the expected number of 0s.
+      __builtin_memset(output.digits, '0', precision);
+      output.ndigits = precision;
+      output.exponent = 0;
+    }
+    return output;
+  }
+
+  // Calculate bounds on log10 of the input value. Its binary exponent bounds
+  // the value between two powers of 2, and we use estimate_log10 to determine
+  // log10 of each of those.
+  //
+  // If a power of 10 falls in the interval between those powers of 2, then
+  // log10_input_min and log10_input_max will differ by 1, and the correct
+  // decimal exponent of the output will be one of those two values. If no
+  // power of 10 is in the interval, then these two values will be equal and
+  // there is only one choice for the decimal exponent.
+  int log10_input_min = estimate_log10(input.exponent - 1);
+  int log10_input_max = estimate_log10(input.exponent);
+
+  // Make a DyadicFloat containing the value 10, to use as the base for
+  // exponentiation.
+  fputil::DyadicFloat<DF_BITS> ten(Sign::POS, 1, 5);
+
+  // Compute the exponent of the lowest-order digit we want as output. In F
+  // mode this depends only on the desired precision. In E mode it's based on
+  // log10_input, which is (an estimate of) the exponent corresponding to the
+  // _high_-order decimal digit of the number.
+  int log10_low_digit = e_mode ? log10_input_min + 1 - precision : -precision;
+
+  // The general plan is to calculate an integer whose decimal representation
+  // is precisely the string of output digits, by doing a DyadicFloat
+  // computation of (input_mantissa / 10^(log10_low_digit)) and then rounding
+  // that to an integer.
+  //
+  // The number of output decimal digits (if the mathematical result of this
+  // operation were computed without overflow) will be one of these:
+  //   (log10_input_min - log10_low_digit + 1)
+  //   (log10_input_max - log10_low_digit + 1)
+  //
+  // In E mode, this means we'll either get the correct number of output digits
+  // immediately, or else one too many (in which case we can correct for that
+  // at the rounding stage). But in F mode, if the number is very large
+  // compared to the number of decimal places the user asked for, we might be
+  // about to generate far too many digits and overflow our float format. In
+  // that case, reset to E mode immediately, to avoid having to detect the
+  // overflow _after_ the multiplication and retry. So if even the smaller
+  // number of possible output digits is too many, we might as well change our
+  // mind right now and switch into E mode.
+  if (log10_input_max - log10_low_digit + 1 > int(MAX_DIGITS)) {
+    precision = MAX_DIGITS;
+    e_mode = true;
+    log10_low_digit = log10_input_min + 1 - precision;
+  }
+
+  // Now actually calculate (input_mantissa / 10^(log10_low_digit)).
+  //
+  // If log10_low_digit < 0, then we calculate 10^(-log10_low_digit) and
+  // multiply by it instead, so that the exponent is non-negative in all cases.
+  // This ensures that the power of 10 is always mathematically speaking an
+  // integer, so that it can be represented exactly in binary (without a
+  // recurring fraction), and when it's small enough to fit in DF_BITS,
+  // fputil::pow_n should return the exact answer, and then
+  // fputil::rounded_{div,mul} will introduce only the unavoidable rounding
+  // error of up to 1/2 ULP.
+  //
+  // Beyond that point, pow_n will be imprecise. But DF_BITS is set high enough
+  // that even for the most difficult cases in 128-bit long double, the extra
+  // precision in the calculation is enough to ensure we still get the right
+  // answer.
+  //
+  // If the output integer doesn't fit in DF_BITS, we set the `overflow` flag.
+
+  // Calculate the power of 10 to divide or multiply by.
+  fputil::DyadicFloat<DF_BITS> power_of_10 =
+      fputil::pow_n(ten, cpp::abs(log10_low_digit));
+
+  // Convert the mantissa into a DyadicFloat, making sure it has the right
+  // sign, so that directed rounding will go in the right direction, if
+  // enabled.
+  fputil::DyadicFloat<DF_BITS> flt_mantissa(
+      input.sign,
+      input.exponent -
+          (cpp::numeric_limits<decltype(input.mantissa)>::digits - 1),
+      input.mantissa);
+
+  // Divide or multiply, depending on whether log10_low_digit was positive
+  // or negative.
+  fputil::DyadicFloat<DF_BITS> flt_quotient =
+      log10_low_digit > 0 ? fputil::rounded_div(flt_mantissa, power_of_10)
+                          : fputil::rounded_mul(flt_mantissa, power_of_10);
+
+  // Convert to an integer.
+  int round_dir;
+  UInt<DF_BITS> integer = flt_quotient.as_mantissa_type_rounded(&round_dir);
+
+  // And take the absolute value.
+  if (flt_quotient.sign.is_neg())
+    integer = -integer;
+
+  // Convert the mantissa integer into a string of decimal digits, and check
+  // to see if it's the right size.
+  const IntegerToString<decltype(integer), radix::Dec> buf{integer};
+  cpp::string_view view = buf.view();
+
+  // Start making the output struct, by copying in the digits from the above
+  // object. At this stage we may also have one digit too many (but that's OK,
+  // there's space for it in the DigitsOutput buffer).
+  DigitsOutput output;
+  output.ndigits = view.size();
+  __builtin_memcpy(output.digits, view.data(), output.ndigits);
+
+  // Set up the output exponent, which is done differently depending on mode.
+  // Also, figure out whether we have one digit too many, and if so, set the
+  // `need_reround` flag and adjust the exponent appropriately.
+  bool need_reround = false;
+  if (e_mode) {
+    // In E mode, the output exponent is the exponent of the first decimal
+    // digit, which we already calculated.
+    output.exponent = log10_input_min;
+
+    // In E mode, we're returning a fixed number of digits, given by
+    // `precision`, so if we have more than that, then we must shorten the
+    // buffer by one digit.
+    //
+    // If this happens, it's because the actual log10 of the input is
+    // log10_input_min + 1. Equivalently, we guessed we'd see something like
+    // X.YZe+NN and instead got WX.YZe+NN. So when we shorten the digit string
+    // by one, we'll also need to increment the output exponent.
+    if (output.ndigits > size_t(precision)) {
+      LIBC_ASSERT(output.ndigits == size_t(precision) + 1);
+      need_reround = true;
+      output.exponent++;
+    }
+  } else {
+    // In F mode, the output exponent is based on the place value of the _last_
+    // digit, so we must recover the exponent of the first digit by adding
+    // the number of digits.
+    //
+    // Because this takes the length of the buffer into account, it sets the
+    // correct decimal exponent even if this digit string is one too long. So
+    // we don't need to adjust the exponent if we reround.
+    output.exponent = int(output.ndigits) - precision - 1;
+
+    // In F mode, the number of returned digits isn't based on `precision`:
+    // it's variable, and we don't mind how many digits we get as long as it
+    // isn't beyond the limit MAX_DIGITS. If it is, we expect that it's only
+    // one digit too long, or else we'd have spotted the problem in advance and
+    // flipped into E mode already.
+    if (output.ndigits > MAX_DIGITS) {
+      LIBC_ASSERT(output.ndigits == MAX_DIGITS + 1);
+      need_reround = true;
+    }
+  }
+
+  if (need_reround) {
+    // If either of the branches above decided that we had one digit too many,
+    // we must now shorten the digit buffer by one. But we can't just truncate:
+    // we need to make sure the remaining n-1 digits are correctly rounded, as
+    // if we'd rounded just once from the original `flt_quotient`.
+    //
+    // In directed rounding modes this can't go wrong. If you had a real number
+    // x, and the first rounding produced floor(x), then the second rounding
+    // wants floor(x/10), and it doesn't matter if you actually compute
+    // floor(floor(x)/10): the result is the same, because each rounding
+    // boundary in the second rounding aligns with one in the first rounding,
+    // which nothing could have crossed. Similarly for rounding away from zero,
+    // with 'floor' replaced with 'ceil' throughout.
+    //
+    // In rounding to nearest, the danger is in the boundary case where the
+    // final digit of the original output is 5. Then if we just rerounded the
+    // digit string to remove the last digit, it would look like an exact
+    // halfway case, and we'd break the tie by choosing the even one of the two
+    // outputs. But if the original value before the first rounding was on one
+    // side or the other of 5, then that supersedes the 'round to even' tie
+    // break. So we need to consult `round_dir` from above, which tells us
+    // which way (if either) the value was adjusted during the first rounding.
+    // Effectively, we treat the last digit as 5+ε or 5-ε.
+    //
+    // To make this work in both directed modes and round-to-nearest mode
+    // without having to look up the rounding direction, a simple rule is: take
+    // account of round_dir if and only if the round digit (the one we're
+    // removing when shortening the buffer) is 5. In directed rounding modes
+    // this makes no difference.
+
+    // Extract the two relevant digits. round_digit is the one we're removing;
+    // new_low_digit is the last one we're keeping, so we need to know if it's
+    // even or odd to handle exact tie cases (when round_dir == 0).
+    --output.ndigits;
+    int round_digit = internal::b36_char_to_int(output.digits[output.ndigits]);
+    int new_low_digit =
+        output.ndigits == 0
+            ? 0
+            : internal::b36_char_to_int(output.digits[output.ndigits - 1]);
+
+    // Make a binary number that we can pass to `fputil::rounding_direction`.
+    // We put new_low_digit at bit 8, and imagine that we're rounding away the
+    // bottom 8 bits. Therefore round_digit must be "just below" bit 8, in the
+    // sense that we set the bottom 8 bits to (256/10 * round_digit) so that
+    // round_digit==5 corresponds to the binary half-way case of 0x80.
+    //
+    // Then we adjust by +1 or -1 based on round_dir if the round digit is 5,
+    // as described above.
+    //
+    // The subexpression `(round_digit * 0x19a) >> 4` is computing the
+    // expression (256/10 * round_digit) mentioned above, accurately enough to
+    // map 5 to exactly 128 but avoiding an integer division (for platforms
+    // where it's slow, e.g. not in hardware).
+    LIBC_NAMESPACE::UInt<64> round_word = (new_low_digit * 256) +
+                                          ((round_digit * 0x19a) >> 4) +
+                                          (round_digit == 5 ? -round_dir : 0);
+
+    // Now we can call the existing binary rounding helper function, which
+    // takes account of the rounding mode.
+    if (fputil::rounding_direction(round_word, 8, flt_quotient.sign) > 0) {
+      // If that returned a positive answer, we must round the number up.
+      //
+      // The number is already in decimal, so we need to increment it one digit
+      // at a time. (A bit painful, but better than going back to the integer
+      // we made it from and doing the decimal conversion all over again.)
+      for (size_t i = output.ndigits; i-- > 0;) {
+        if (output.digits[i] != '9') {
+          output.digits[i] = static_cast<char>(internal::int_to_b36_char(
+              internal::b36_char_to_int(output.digits[i]) + 1));
+          break;
+        } else {
+          output.digits[i] = '0';
+        }
+      }
+    }
+  }
+
+  return output;
+}
+
+LIBC_INLINE int convert_float_inner(Writer *writer,
+                                    const FormatSection &to_conv,
+                                    int32_t fraction_len, int exponent,
+                                    StorageType mantissa, Sign sign,
+                                    ConversionType ctype) {
+  // If to_conv doesn't specify a precision, the precision defaults to 6.
+  unsigned precision = to_conv.precision < 0 ? 6 : to_conv.precision;
+
+  // Decide if we're displaying a sign character, depending on the format flags
+  // and whether the input is negative.
+  char sign_char = 0;
+  if (sign.is_neg())
+    sign_char = '-';
+  else if ((to_conv.flags & FormatFlags::FORCE_SIGN) == FormatFlags::FORCE_SIGN)
+    sign_char = '+'; // FORCE_SIGN has precedence over SPACE_PREFIX
+  else if ((to_conv.flags & FormatFlags::SPACE_PREFIX) ==
+           FormatFlags::SPACE_PREFIX)
+    sign_char = ' ';
+
+  // Prepare the input to decimal_digits().
+  DigitsInput input(fraction_len, mantissa, exponent, sign);
+
+  // Call decimal_digits() in a different way, based on whether the format
+  // character is 'e', 'f', or 'g'. After this loop we expect to have filled
+  // in the following variables:
+
+  // The decimal digits, and the exponent of the topmost one.
+  DigitsOutput output;
+  // The start and end of the digit string we're displaying, as indices into
+  // `output.digits`. The indices may be out of bounds in either direction, in
+  // which case digits beyond the bounds of the buffer should be displayed as
+  // zeroes.
+  //
+  // As usual, the index 'start' is included, and 'limit' is not.
+  int start, limit;
+  // The index of the digit that we display a decimal point immediately after.
+  // Again, represented as an index in `output.digits`, and may be out of
+  // bounds.
+  int pointpos;
+  // Whether we need to display an "e+NNN" exponent suffix at all.
+  bool show_exponent;
+
+  switch (ctype) {
+  case ConversionType::E:
+    // In E mode, we display one digit more than the specified precision
+    // (`%.6e` means six digits _after_ the decimal point, like 1.123456e+00).
+    //
+    // Also, bound the number of digits we request at MAX_DIGITS.
+    output = decimal_digits(input, cpp::min(precision + 1, MAX_DIGITS), true);
+
+    // We display digits from the start of the buffer, and always output
+    // `precision+1` of them (which will append zeroes if the user requested
+    // more than MAX_DIGITS).
+    start = 0;
+    limit = precision + 1;
+
+    // The decimal point is always after the first digit of the buffer.
+    pointpos = start;
+
+    // The exponent is always displayed explicitly.
+    show_exponent = true;
+    break;
+  case ConversionType::F:
+    // In F mode, we provide decimal_digits() with the unmodified input
+    // precision, and let it give us as many digits as we can.
+    output = decimal_digits(input, precision, false);
+
+    // Initialize (start, limit) to display everything from the first nonzero
+    // digit (necessarily at the start of the output buffer) to the digit at
+    // the correct distance after the decimal point.
+    start = 0;
+    limit = 1 + output.exponent + precision;
+
+    // But we must display at least one digit _before_ the decimal point, i.e.
+    // at least precision+1 digits in total. So if we're not already doing
+    // that, we must correct those values.
+    if (limit <= int(precision))
+      start -= precision + 1 - limit;
+
+    // The decimal point appears precisely 'precision' digits before the end of
+    // the digits we output.
+    pointpos = limit - 1 - precision;
+
+    // The exponent is never displayed.
+    show_exponent = false;
+    break;
+  case ConversionType::G:
+    // In G mode, the precision says exactly how many significant digits you
+    // want. (In that respect it's subtly unlike E mode: %.6g means six digits
+    // _including_ the one before the point, whereas %.6e means six digits
+    // _excluding_ that one.)
+    //
+    // Also, a precision of 0 is treated the same as 1.
+    precision = cpp::max(precision, 1u);
+    output = decimal_digits(input, cpp::min(precision, MAX_DIGITS), true);
+
+    // As in E mode, we default to displaying precisely the digits in the
+    // output buffer.
+    start = 0;
+    limit = precision;
+
+    // If we're not in ALTERNATE_FORM mode, trailing zeroes on the mantissa are
+    // removed (although not to the extent of leaving no digits at all - if the
+    // entire output mantissa is all 0 then we keep a single zero digit).
+    if (!(to_conv.flags & FormatFlags::ALTERNATE_FORM)) {
+      // Start by removing trailing zeroes that were outside the buffer
+      // entirely.
+      limit = cpp::min(limit, int(output.ndigits));
+
+      // Then check the digits in the buffer and remove as many as possible.
+      while (limit > 1 && output.digits[limit - 1] == '0')
+        limit--;
+    }
+
+    // Decide whether to display in %e style with an explicit exponent, or %f
+    // style with the decimal point after the units place.
+    //
+    // %e mode is used to avoid an excessive number of leading zeroes after the
+    // decimal point but before the first nonzero digit (specifically, 0.0001
+    // is fine as it is, but 0.00001 prints as 1e-5), and also to avoid adding
+    // trailing zeroes if the last digit in the buffer is still higher than the
+    // units place.
+    //
+    // output.exponent is an int whereas precision is unsigned, so we must
+    // check output.exponent >= 0 before comparing it against precision to
+    // prevent a negative exponent from wrapping round to a large unsigned int.
+    if ((output.exponent >= 0 && output.exponent >= int(precision)) ||
+        output.exponent < -4) {
+      // Display in %e style, so the point goes after the first digit and the
+      // exponent is shown.
+      pointpos = start;
+      show_exponent = true;
+    } else {
+      // Display in %f style, so the point goes at its true mathematical
+      // location and the exponent is not shown.
+      pointpos = output.exponent;
+      show_exponent = false;
+
+      if (output.exponent < 0) {
+        // If the first digit is below the decimal point, add leading zeroes.
+        // (This _decreases_ start, because output.exponent is negative here.)
+        start += output.exponent;
+      } else if (limit <= output.exponent) {
+        // If the last digit is above the decimal point, add trailing zeroes.
+        // (This may involve putting back some zeroes that we trimmed in the
+        // loop above!)
+        limit = output.exponent + 1;
+      }
+    }
+    break;
+  }
+
+  // Find out for sure whether we're displaying the decimal point, so that we
+  // can include it in the calculation of the total string length for padding.
+  //
+  // We never expect pointpos to be _before_ the start of the displayed range
+  // of digits. (If it had been, we'd have added leading zeroes.) But it might
+  // be beyond the end.
+  //
+  // We don't display the point if it appears immediately after the _last_
+  // digit we display, except in ALTERNATE_FORM mode.
+  int last_point_digit =
+      (to_conv.flags & FormatFlags::ALTERNATE_FORM) ? limit - 1 : limit - 2;
+  bool show_point = pointpos <= last_point_digit;
+
+  // Format the exponent suffix (e+NN, e-NN) into a buffer, or leave the buffer
+  // empty if we're not displaying one.
+  char expbuf[16]; // more than enough space for e+NNNN
+  size_t explen = 0;
+  if (show_exponent) {
+    const IntegerToString<decltype(output.exponent),
+                          radix::Dec::WithWidth<2>::WithSign>
+        expcvt{output.exponent};
+    cpp::string_view expview = expcvt.view();
+    expbuf[0] = internal::islower(to_conv.conv_name) ? 'e' : 'E';
+    explen = expview.size() + 1;
+    __builtin_memcpy(expbuf + 1, expview.data(), expview.size());
+  }
+
+  // Now we know enough to work out the length of the unpadded output:
+  //  * whether to write a sign
+  //  * how many mantissa digits to write
+  //  * whether to write a decimal point
+  //  * the length of the trailing exponent string.
+  size_t unpadded_len =
+      (sign_char != 0) + (limit - start) + show_point + explen;
+
+  // Work out how much padding is needed.
+  size_t min_width = to_conv.min_width > 0 ? to_conv.min_width : 0;
+  size_t padding_amount = cpp::max(min_width, unpadded_len) - unpadded_len;
+
+  // Work out what the padding looks like and where it appears.
+  enum class Padding {
+    LeadingSpace,  // spaces at the start of the string
+    Zero,          // zeroes between sign and mantissa
+    TrailingSpace, // spaces at the end of the string
+  } padding = Padding::LeadingSpace;
+  // The '-' flag for left-justification takes priority over the '0' flag
+  if (to_conv.flags & FormatFlags::LEFT_JUSTIFIED)
+    padding = Padding::TrailingSpace;
+  else if (to_conv.flags & FormatFlags::LEADING_ZEROES)
+    padding = Padding::Zero;
+
+  // Finally, write all the output!
+
+  // Leading-space padding, if any
+  if (padding == Padding::LeadingSpace)
+    RET_IF_RESULT_NEGATIVE(writer->write(' ', padding_amount));
+
+  // Sign, if any
+  if (sign_char)
+    RET_IF_RESULT_NEGATIVE(writer->write(sign_char));
+
+  // Zero padding, if any
+  if (padding == Padding::Zero)
+    RET_IF_RESULT_NEGATIVE(writer->write('0', padding_amount));
+
+  // Mantissa digits, maybe with a decimal point
+  for (int pos = start; pos < limit; ++pos) {
+    if (pos >= 0 && pos < int(output.ndigits)) {
+      // Fetch a digit from the buffer
+      RET_IF_RESULT_NEGATIVE(writer->write(output.digits[pos]));
+    } else {
+      // This digit is outside the buffer, so write a zero
+      RET_IF_RESULT_NEGATIVE(writer->write('0'));
+    }
+
+    // Show the decimal point, if this is the digit it comes after
+    if (show_point && pos == pointpos)
+      RET_IF_RESULT_NEGATIVE(writer->write(DECIMAL_POINT));
+  }
+
+  // Exponent
+  RET_IF_RESULT_NEGATIVE(writer->write(cpp::string_view(expbuf, explen)));
+
+  // Trailing-space padding, if any
+  if (padding == Padding::TrailingSpace)
+    RET_IF_RESULT_NEGATIVE(writer->write(' ', padding_amount));
+
+  return WRITE_OK;
+}
+
+template <typename T, cpp::enable_if_t<cpp::is_floating_point_v<T>, int> = 0>
+LIBC_INLINE int
+convert_float_typed(Writer *writer, const FormatSection &to_conv,
+                    fputil::FPBits<T> float_bits, ConversionType ctype) {
+  return convert_float_inner(writer, to_conv, float_bits.FRACTION_LEN,
+                             float_bits.get_explicit_exponent(),
+                             float_bits.get_explicit_mantissa(),
+                             float_bits.sign(), ctype);
+}
+
+LIBC_INLINE int convert_float_outer(Writer *writer,
+                                    const FormatSection &to_conv,
+                                    ConversionType ctype) {
+  if (to_conv.length_modifier == LengthModifier::L) {
+    fputil::FPBits<long double>::StorageType float_raw = to_conv.conv_val_raw;
+    fputil::FPBits<long double> float_bits(float_raw);
+    if (!float_bits.is_inf_or_nan()) {
+      return convert_float_typed<long double>(writer, to_conv, float_bits,
+                                              ctype);
+    }
+  } else {
+    fputil::FPBits<double>::StorageType float_raw =
+        static_cast<fputil::FPBits<double>::StorageType>(to_conv.conv_val_raw);
+    fputil::FPBits<double> float_bits(float_raw);
+    if (!float_bits.is_inf_or_nan()) {
+      return convert_float_typed<double>(writer, to_conv, float_bits, ctype);
+    }
+  }
+
+  return convert_inf_nan(writer, to_conv);
+}
+
+template <typename T, cpp::enable_if_t<cpp::is_floating_point_v<T>, int> = 0>
+LIBC_INLINE int convert_float_decimal_typed(Writer *writer,
+                                            const FormatSection &to_conv,
+                                            fputil::FPBits<T> float_bits) {
+  return convert_float_typed<T>(writer, to_conv, float_bits, ConversionType::F);
+}
+
+template <typename T, cpp::enable_if_t<cpp::is_floating_point_v<T>, int> = 0>
+LIBC_INLINE int convert_float_dec_exp_typed(Writer *writer,
+                                            const FormatSection &to_conv,
+                                            fputil::FPBits<T> float_bits) {
+  return convert_float_typed<T>(writer, to_conv, float_bits, ConversionType::E);
+}
+
+template <typename T, cpp::enable_if_t<cpp::is_floating_point_v<T>, int> = 0>
+LIBC_INLINE int convert_float_dec_auto_typed(Writer *writer,
+                                             const FormatSection &to_conv,
+                                             fputil::FPBits<T> float_bits) {
+  return convert_float_typed<T>(writer, to_conv, float_bits, ConversionType::G);
+}
+
+LIBC_INLINE int convert_float_decimal(Writer *writer,
+                                      const FormatSection &to_conv) {
+  return convert_float_outer(writer, to_conv, ConversionType::F);
+}
+
+LIBC_INLINE int convert_float_dec_exp(Writer *writer,
+                                      const FormatSection &to_conv) {
+  return convert_float_outer(writer, to_conv, ConversionType::E);
+}
+
+LIBC_INLINE int convert_float_dec_auto(Writer *writer,
+                                       const FormatSection &to_conv) {
+  return convert_float_outer(writer, to_conv, ConversionType::G);
+}
+
+} // namespace printf_core
+} // namespace LIBC_NAMESPACE_DECL
+
+#endif // LLVM_LIBC_SRC_STDIO_PRINTF_CORE_FLOAT_DEC_CONVERTER_LIMITED_H
diff --git a/libc/src/string/allocating_string_utils.h b/libc/src/string/allocating_string_utils.h
index b3a8663c2f477..1dece510e7969 100644
--- a/libc/src/string/allocating_string_utils.h
+++ b/libc/src/string/allocating_string_utils.h
@@ -11,7 +11,7 @@
 
 #include "src/__support/CPP/new.h"
 #include "src/__support/CPP/optional.h"
-#include "src/__support/macros/config.h"
+#include "src/__support/macros/config.h" // LIBC_NAMESPACE_DECL
 #include "src/string/memory_utils/inline_memcpy.h"
 #include "src/string/string_utils.h"
 
diff --git a/libc/src/string/memory_utils/aarch64/inline_bcmp.h b/libc/src/string/memory_utils/aarch64/inline_bcmp.h
index 93196415f2662..e41ac202dbaac 100644
--- a/libc/src/string/memory_utils/aarch64/inline_bcmp.h
+++ b/libc/src/string/memory_utils/aarch64/inline_bcmp.h
@@ -9,7 +9,7 @@
 #define LIBC_SRC_STRING_MEMORY_UTILS_AARCH64_INLINE_BCMP_H
 
 #include "src/__support/macros/attributes.h"   // LIBC_INLINE
-#include "src/__support/macros/config.h"
+#include "src/__support/macros/config.h"       // LIBC_NAMESPACE_DECL
 #include "src/__support/macros/optimization.h" // LIBC_UNLIKELY
 #include "src/string/memory_utils/op_aarch64.h"
 #include "src/string/memory_utils/op_generic.h"
diff --git a/libc/src/string/memory_utils/aarch64/inline_memset.h b/libc/src/string/memory_utils/aarch64/inline_memset.h
index a2c05534aa330..efcbfd0705983 100644
--- a/libc/src/string/memory_utils/aarch64/inline_memset.h
+++ b/libc/src/string/memory_utils/aarch64/inline_memset.h
@@ -9,7 +9,7 @@
 #define LIBC_SRC_STRING_MEMORY_UTILS_AARCH64_INLINE_MEMSET_H
 
 #include "src/__support/macros/attributes.h" // LIBC_INLINE
-#include "src/__support/macros/config.h"
+#include "src/__support/macros/config.h"     // LIBC_NAMESPACE_DECL
 #include "src/string/memory_utils/op_aarch64.h"
 #include "src/string/memory_utils/op_generic.h"
 #include "src/string/memory_utils/utils.h" // Ptr, CPtr
diff --git a/libc/src/string/memory_utils/generic/builtin.h b/libc/src/string/memory_utils/generic/builtin.h
index 5670a4e610565..2b9ecc0c046eb 100644
--- a/libc/src/string/memory_utils/generic/builtin.h
+++ b/libc/src/string/memory_utils/generic/builtin.h
@@ -10,7 +10,7 @@
 #define LLVM_LIBC_SRC_STRING_MEMORY_UTILS_GENERIC_BUILTIN_H
 
 #include "src/__support/macros/attributes.h" // LIBC_INLINE
-#include "src/__support/macros/config.h"
+#include "src/__support/macros/config.h"     // LIBC_NAMESPACE_DECL
 #include "src/string/memory_utils/utils.h" // Ptr, CPtr
 
 #include <stddef.h> // size_t
diff --git a/libc/src/string/memory_utils/inline_bcmp.h b/libc/src/string/memory_utils/inline_bcmp.h
index 14cf16ebb6b69..52c738d3a4ce2 100644
--- a/libc/src/string/memory_utils/inline_bcmp.h
+++ b/libc/src/string/memory_utils/inline_bcmp.h
@@ -10,7 +10,8 @@
 #define LLVM_LIBC_SRC_STRING_MEMORY_UTILS_INLINE_BCMP_H
 
 #include "src/__support/common.h"
-#include "src/__support/macros/config.h"
+#include "src/__support/macros/attributes.h" // LIBC_INLINE
+#include "src/__support/macros/config.h"     // LIBC_NAMESPACE_DECL
 #include "src/__support/macros/properties/architectures.h" // LIBC_TARGET_ARCH_IS_
 
 #include <stddef.h> // size_t
diff --git a/libc/src/string/memory_utils/inline_bzero.h b/libc/src/string/memory_utils/inline_bzero.h
index 4a92e8b57a68e..a131b68725de7 100644
--- a/libc/src/string/memory_utils/inline_bzero.h
+++ b/libc/src/string/memory_utils/inline_bzero.h
@@ -10,7 +10,8 @@
 #define LLVM_LIBC_SRC_STRING_MEMORY_UTILS_INLINE_BZERO_H
 
 #include "src/__support/common.h"
-#include "src/__support/macros/config.h"
+#include "src/__support/macros/attributes.h" // LIBC_INLINE
+#include "src/__support/macros/config.h"     // LIBC_NAMESPACE_DECL
 #include "src/string/memory_utils/inline_memset.h"
 
 #include <stddef.h> // size_t
diff --git a/libc/src/string/memory_utils/inline_memmem.h b/libc/src/string/memory_utils/inline_memmem.h
index 15e3d633985d6..1e9649c6e65e3 100644
--- a/libc/src/string/memory_utils/inline_memmem.h
+++ b/libc/src/string/memory_utils/inline_memmem.h
@@ -9,8 +9,8 @@
 #ifndef LLVM_LIBC_SRC_STRING_MEMORY_UTILS_INLINE_MEMMEM_H
 #define LLVM_LIBC_SRC_STRING_MEMORY_UTILS_INLINE_MEMMEM_H
 
-#include "src/__support/macros/attributes.h"
-#include "src/__support/macros/config.h"
+#include "src/__support/macros/attributes.h" // LIBC_INLINE
+#include "src/__support/macros/config.h"     // LIBC_NAMESPACE_DECL
 
 #include <stddef.h>
 
diff --git a/libc/src/string/memory_utils/inline_memmove.h b/libc/src/string/memory_utils/inline_memmove.h
index 85d0159701ece..84671c5c1797e 100644
--- a/libc/src/string/memory_utils/inline_memmove.h
+++ b/libc/src/string/memory_utils/inline_memmove.h
@@ -9,7 +9,8 @@
 #ifndef LLVM_LIBC_SRC_STRING_MEMORY_UTILS_INLINE_MEMMOVE_H
 #define LLVM_LIBC_SRC_STRING_MEMORY_UTILS_INLINE_MEMMOVE_H
 
-#include "src/__support/macros/config.h"
+#include "src/__support/macros/attributes.h" // LIBC_INLINE
+#include "src/__support/macros/config.h"     // LIBC_NAMESPACE_DECL
 #include <stddef.h> // size_t, ptrdiff_t
 
 #if defined(LIBC_TARGET_ARCH_IS_X86)
diff --git a/libc/src/string/memory_utils/inline_strcmp.h b/libc/src/string/memory_utils/inline_strcmp.h
index 281d5b14c6cba..6758e79ae9ca3 100644
--- a/libc/src/string/memory_utils/inline_strcmp.h
+++ b/libc/src/string/memory_utils/inline_strcmp.h
@@ -9,7 +9,8 @@
 #ifndef LLVM_LIBC_SRC_STRING_MEMORY_UTILS_INLINE_STRCMP_H
 #define LLVM_LIBC_SRC_STRING_MEMORY_UTILS_INLINE_STRCMP_H
 
-#include "src/__support/macros/config.h"
+#include "src/__support/macros/attributes.h" // LIBC_INLINE
+#include "src/__support/macros/config.h"     // LIBC_NAMESPACE_DECL
 #include <stddef.h>
 
 namespace LIBC_NAMESPACE_DECL {
diff --git a/libc/src/string/memory_utils/inline_strstr.h b/libc/src/string/memory_utils/inline_strstr.h
index 9c99e920b4b57..5495cc4803ec9 100644
--- a/libc/src/string/memory_utils/inline_strstr.h
+++ b/libc/src/string/memory_utils/inline_strstr.h
@@ -9,7 +9,8 @@
 #ifndef LLVM_LIBC_SRC_STRING_MEMORY_UTILS_INLINE_STRSTR_H
 #define LLVM_LIBC_SRC_STRING_MEMORY_UTILS_INLINE_STRSTR_H
 
-#include "src/__support/macros/config.h"
+#include "src/__support/macros/attributes.h" // LIBC_INLINE
+#include "src/__support/macros/config.h"     // LIBC_NAMESPACE_DECL
 #include "src/string/memory_utils/inline_memmem.h"
 #include "src/string/string_utils.h"
 #include <stddef.h>
diff --git a/libc/src/string/memory_utils/op_aarch64.h b/libc/src/string/memory_utils/op_aarch64.h
index 1090ea2617f09..868c64474c0b4 100644
--- a/libc/src/string/memory_utils/op_aarch64.h
+++ b/libc/src/string/memory_utils/op_aarch64.h
@@ -13,7 +13,8 @@
 #ifndef LLVM_LIBC_SRC_STRING_MEMORY_UTILS_OP_AARCH64_H
 #define LLVM_LIBC_SRC_STRING_MEMORY_UTILS_OP_AARCH64_H
 
-#include "src/__support/macros/config.h"
+#include "src/__support/macros/attributes.h" // LIBC_INLINE
+#include "src/__support/macros/config.h"     // LIBC_NAMESPACE_DECL
 #include "src/__support/macros/properties/architectures.h"
 
 #if defined(LIBC_TARGET_ARCH_IS_AARCH64)
diff --git a/libc/src/string/memory_utils/op_builtin.h b/libc/src/string/memory_utils/op_builtin.h
index d7c1b1f870115..27b621d97f01a 100644
--- a/libc/src/string/memory_utils/op_builtin.h
+++ b/libc/src/string/memory_utils/op_builtin.h
@@ -16,7 +16,8 @@
 #define LLVM_LIBC_SRC_STRING_MEMORY_UTILS_OP_BUILTIN_H
 
 #include "src/__support/CPP/type_traits.h"
-#include "src/__support/macros/config.h"
+#include "src/__support/macros/attributes.h" // LIBC_INLINE
+#include "src/__support/macros/config.h"     // LIBC_NAMESPACE_DECL
 #include "src/string/memory_utils/utils.h"
 
 namespace LIBC_NAMESPACE_DECL {
diff --git a/libc/src/string/memory_utils/op_generic.h b/libc/src/string/memory_utils/op_generic.h
index da20a84dede4d..9349cfdd1d020 100644
--- a/libc/src/string/memory_utils/op_generic.h
+++ b/libc/src/string/memory_utils/op_generic.h
@@ -27,7 +27,8 @@
 #include "src/__support/CPP/type_traits.h"
 #include "src/__support/common.h"
 #include "src/__support/endian_internal.h"
-#include "src/__support/macros/config.h"
+#include "src/__support/macros/attributes.h" // LIBC_INLINE
+#include "src/__support/macros/config.h"     // LIBC_NAMESPACE_DECL
 #include "src/__support/macros/optimization.h"
 #include "src/__support/macros/properties/types.h" // LIBC_TYPES_HAS_INT64
 #include "src/string/memory_utils/op_builtin.h"
diff --git a/libc/src/string/memory_utils/op_riscv.h b/libc/src/string/memory_utils/op_riscv.h
index 2d211de0facfc..4292d7a627a54 100644
--- a/libc/src/string/memory_utils/op_riscv.h
+++ b/libc/src/string/memory_utils/op_riscv.h
@@ -12,7 +12,8 @@
 #ifndef LLVM_LIBC_SRC_STRING_MEMORY_UTILS_OP_RISCV_H
 #define LLVM_LIBC_SRC_STRING_MEMORY_UTILS_OP_RISCV_H
 
-#include "src/__support/macros/config.h"
+#include "src/__support/macros/attributes.h" // LIBC_INLINE
+#include "src/__support/macros/config.h"     // LIBC_NAMESPACE_DECL
 #include "src/__support/macros/properties/architectures.h"
 
 #if defined(LIBC_TARGET_ARCH_IS_ANY_RISCV)
diff --git a/libc/src/string/memory_utils/op_x86.h b/libc/src/string/memory_utils/op_x86.h
index 309610e4ad630..8bd84120c4ffa 100644
--- a/libc/src/string/memory_utils/op_x86.h
+++ b/libc/src/string/memory_utils/op_x86.h
@@ -12,7 +12,8 @@
 #ifndef LLVM_LIBC_SRC_STRING_MEMORY_UTILS_OP_X86_H
 #define LLVM_LIBC_SRC_STRING_MEMORY_UTILS_OP_X86_H
 
-#include "src/__support/macros/config.h"
+#include "src/__support/macros/attributes.h" // LIBC_INLINE
+#include "src/__support/macros/config.h"     // LIBC_NAMESPACE_DECL
 #include "src/__support/macros/properties/architectures.h"
 
 #if defined(LIBC_TARGET_ARCH_IS_X86)
diff --git a/libc/src/string/memory_utils/riscv/inline_bcmp.h b/libc/src/string/memory_utils/riscv/inline_bcmp.h
index 4bdde27cd16be..666ad6382412e 100644
--- a/libc/src/string/memory_utils/riscv/inline_bcmp.h
+++ b/libc/src/string/memory_utils/riscv/inline_bcmp.h
@@ -9,7 +9,7 @@
 #define LIBC_SRC_STRING_MEMORY_UTILS_RISCV_INLINE_BCMP_H
 
 #include "src/__support/macros/attributes.h"               // LIBC_INLINE
-#include "src/__support/macros/config.h"
+#include "src/__support/macros/config.h" // LIBC_NAMESPACE_DECL
 #include "src/__support/macros/properties/architectures.h" // LIBC_TARGET_ARCH_IS_RISCV64
 #include "src/string/memory_utils/generic/aligned_access.h"
 #include "src/string/memory_utils/utils.h" // Ptr, CPtr
diff --git a/libc/src/string/memory_utils/riscv/inline_memcmp.h b/libc/src/string/memory_utils/riscv/inline_memcmp.h
index ca834952d331e..c49e8d0a4ff2f 100644
--- a/libc/src/string/memory_utils/riscv/inline_memcmp.h
+++ b/libc/src/string/memory_utils/riscv/inline_memcmp.h
@@ -9,7 +9,7 @@
 #define LIBC_SRC_STRING_MEMORY_UTILS_RISCV_INLINE_MEMCMP_H
 
 #include "src/__support/macros/attributes.h"               // LIBC_INLINE
-#include "src/__support/macros/config.h"
+#include "src/__support/macros/config.h" // LIBC_NAMESPACE_DECL
 #include "src/__support/macros/properties/architectures.h" // LIBC_TARGET_ARCH_IS_RISCV64
 #include "src/string/memory_utils/generic/aligned_access.h"
 #include "src/string/memory_utils/utils.h" // Ptr, CPtr
diff --git a/libc/src/string/memory_utils/riscv/inline_memcpy.h b/libc/src/string/memory_utils/riscv/inline_memcpy.h
index 8eb87e0f38ce5..e907ae4c079e0 100644
--- a/libc/src/string/memory_utils/riscv/inline_memcpy.h
+++ b/libc/src/string/memory_utils/riscv/inline_memcpy.h
@@ -9,7 +9,7 @@
 #define LIBC_SRC_STRING_MEMORY_UTILS_RISCV_INLINE_MEMCPY_H
 
 #include "src/__support/macros/attributes.h"               // LIBC_INLINE
-#include "src/__support/macros/config.h"
+#include "src/__support/macros/config.h" // LIBC_NAMESPACE_DECL
 #include "src/__support/macros/properties/architectures.h" // LIBC_TARGET_ARCH_IS_RISCV64
 #include "src/string/memory_utils/generic/aligned_access.h"
 #include "src/string/memory_utils/utils.h" // Ptr, CPtr
diff --git a/libc/src/string/memory_utils/riscv/inline_memmove.h b/libc/src/string/memory_utils/riscv/inline_memmove.h
index 28de4c240260d..01a9fa5c4bbdc 100644
--- a/libc/src/string/memory_utils/riscv/inline_memmove.h
+++ b/libc/src/string/memory_utils/riscv/inline_memmove.h
@@ -9,7 +9,7 @@
 #define LLVM_LIBC_SRC_STRING_MEMORY_UTILS_RISCV_INLINE_MEMMOVE_H
 
 #include "src/__support/macros/attributes.h"               // LIBC_INLINE
-#include "src/__support/macros/config.h"
+#include "src/__support/macros/config.h" // LIBC_NAMESPACE_DECL
 #include "src/__support/macros/properties/architectures.h" // LIBC_TARGET_ARCH_IS_RISCV64
 #include "src/string/memory_utils/generic/byte_per_byte.h"
 #include "src/string/memory_utils/utils.h" // Ptr, CPtr
diff --git a/libc/src/string/memory_utils/riscv/inline_memset.h b/libc/src/string/memory_utils/riscv/inline_memset.h
index d6ab523f16414..09a7a765e4cb6 100644
--- a/libc/src/string/memory_utils/riscv/inline_memset.h
+++ b/libc/src/string/memory_utils/riscv/inline_memset.h
@@ -9,7 +9,7 @@
 #define LIBC_SRC_STRING_MEMORY_UTILS_RISCV_INLINE_MEMSET_H
 
 #include "src/__support/macros/attributes.h"               // LIBC_INLINE
-#include "src/__support/macros/config.h"
+#include "src/__support/macros/config.h" // LIBC_NAMESPACE_DECL
 #include "src/__support/macros/properties/architectures.h" // LIBC_TARGET_ARCH_IS_RISCV64
 #include "src/string/memory_utils/generic/aligned_access.h"
 #include "src/string/memory_utils/utils.h" // Ptr, CPtr
diff --git a/libc/src/string/memory_utils/utils.h b/libc/src/string/memory_utils/utils.h
index cae65bddd92f6..5c9bc72208f85 100644
--- a/libc/src/string/memory_utils/utils.h
+++ b/libc/src/string/memory_utils/utils.h
@@ -14,7 +14,7 @@
 #include "src/__support/CPP/type_traits.h"
 #include "src/__support/endian_internal.h"
 #include "src/__support/macros/attributes.h" // LIBC_INLINE
-#include "src/__support/macros/config.h"
+#include "src/__support/macros/config.h"     // LIBC_NAMESPACE_DECL
 #include "src/__support/macros/properties/architectures.h"
 
 #include <stddef.h> // size_t
diff --git a/libc/src/string/memory_utils/x86_64/inline_bcmp.h b/libc/src/string/memory_utils/x86_64/inline_bcmp.h
index cc54c4140ee6e..8be391b428910 100644
--- a/libc/src/string/memory_utils/x86_64/inline_bcmp.h
+++ b/libc/src/string/memory_utils/x86_64/inline_bcmp.h
@@ -9,7 +9,7 @@
 #define LLVM_LIBC_SRC_STRING_MEMORY_UTILS_X86_64_INLINE_BCMP_H
 
 #include "src/__support/macros/attributes.h" // LIBC_INLINE
-#include "src/__support/macros/config.h"
+#include "src/__support/macros/config.h"     // LIBC_NAMESPACE_DECL
 #include "src/string/memory_utils/op_generic.h"
 #include "src/string/memory_utils/op_x86.h"
 #include "src/string/memory_utils/utils.h" // Ptr, CPtr
diff --git a/libc/src/string/memory_utils/x86_64/inline_memset.h b/libc/src/string/memory_utils/x86_64/inline_memset.h
index 9f8e584d2bbb4..35719a89fa600 100644
--- a/libc/src/string/memory_utils/x86_64/inline_memset.h
+++ b/libc/src/string/memory_utils/x86_64/inline_memset.h
@@ -8,8 +8,8 @@
 #ifndef LLVM_LIBC_SRC_STRING_MEMORY_UTILS_X86_64_INLINE_MEMSET_H
 #define LLVM_LIBC_SRC_STRING_MEMORY_UTILS_X86_64_INLINE_MEMSET_H
 
-#include "src/__support/macros/attributes.h" // LIBC_INLINE
-#include "src/__support/macros/config.h"
+#include "src/__support/macros/attributes.h" // LIBC_INLINE, LIBC_INLINE_VAR
+#include "src/__support/macros/config.h"     // LIBC_NAMESPACE_DECL
 #include "src/string/memory_utils/op_generic.h"
 #include "src/string/memory_utils/op_x86.h"
 #include "src/string/memory_utils/utils.h" // Ptr, CPtr
diff --git a/libc/src/sys/uio/CMakeLists.txt b/libc/src/sys/uio/CMakeLists.txt
index 6298f86cd937d..c6a6429790410 100644
--- a/libc/src/sys/uio/CMakeLists.txt
+++ b/libc/src/sys/uio/CMakeLists.txt
@@ -8,3 +8,10 @@ add_entrypoint_object(
   DEPENDS
     .${LIBC_TARGET_OS}.writev
 )
+
+add_entrypoint_object(
+  readv
+  ALIAS
+  DEPENDS
+    .${LIBC_TARGET_OS}.readv
+)
diff --git a/libc/src/sys/uio/linux/CMakeLists.txt b/libc/src/sys/uio/linux/CMakeLists.txt
index 85a7a3ae4d5c2..db4fe68e4ca9c 100644
--- a/libc/src/sys/uio/linux/CMakeLists.txt
+++ b/libc/src/sys/uio/linux/CMakeLists.txt
@@ -5,10 +5,25 @@ add_entrypoint_object(
   HDRS
     ../writev.h
   DEPENDS
+    libc.hdr.types.ssize_t
+    libc.hdr.types.struct_iovec
     libc.include.sys_syscall
     libc.src.__support.OSUtil.osutil
     libc.src.__support.common
     libc.src.errno.errno
+)
+
+add_entrypoint_object(
+  readv
+  SRCS
+    readv.cpp
+  HDRS
+    ../readv.h
+  DEPENDS
     libc.hdr.types.ssize_t
     libc.hdr.types.struct_iovec
+    libc.include.sys_syscall
+    libc.src.__support.OSUtil.osutil
+    libc.src.__support.common
+    libc.src.errno.errno
 )
diff --git a/libc/src/sys/uio/linux/readv.cpp b/libc/src/sys/uio/linux/readv.cpp
new file mode 100644
index 0000000000000..f1393a9749be9
--- /dev/null
+++ b/libc/src/sys/uio/linux/readv.cpp
@@ -0,0 +1,29 @@
+//===-- Implementation file for readv -------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+#include "src/sys/uio/readv.h"
+#include "hdr/types/ssize_t.h"
+#include "hdr/types/struct_iovec.h"
+#include "src/__support/OSUtil/syscall.h"
+#include "src/__support/common.h"
+#include "src/errno/libc_errno.h"
+#include <sys/syscall.h>
+
+namespace LIBC_NAMESPACE_DECL {
+
+LLVM_LIBC_FUNCTION(ssize_t, readv, (int fd, const iovec *iov, int iovcnt)) {
+  long ret = LIBC_NAMESPACE::syscall_impl<long>(SYS_readv, fd, iov, iovcnt);
+  // On failure, return -1 and set errno.
+  if (ret < 0) {
+    libc_errno = static_cast<int>(-ret);
+    return -1;
+  }
+  // On success, return number of bytes read.
+  return static_cast<ssize_t>(ret);
+}
+
+} // namespace LIBC_NAMESPACE_DECL
diff --git a/libc/src/sys/uio/linux/writev.cpp b/libc/src/sys/uio/linux/writev.cpp
index a3bb8986d522e..8992bed95c982 100644
--- a/libc/src/sys/uio/linux/writev.cpp
+++ b/libc/src/sys/uio/linux/writev.cpp
@@ -6,6 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 #include "src/sys/uio/writev.h"
+#include "hdr/types/ssize_t.h"
+#include "hdr/types/struct_iovec.h"
 #include "src/__support/OSUtil/syscall.h"
 #include "src/__support/common.h"
 #include "src/errno/libc_errno.h"
diff --git a/libc/src/sys/uio/readv.h b/libc/src/sys/uio/readv.h
new file mode 100644
index 0000000000000..135b1e6fd1b5a
--- /dev/null
+++ b/libc/src/sys/uio/readv.h
@@ -0,0 +1,22 @@
+//===-- Implementation header for readv -----------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_SYS_UIO_READV_H
+#define LLVM_LIBC_SRC_SYS_UIO_READV_H
+
+#include "hdr/types/ssize_t.h"
+#include "hdr/types/struct_iovec.h"
+#include "src/__support/macros/config.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+ssize_t readv(int fd, const iovec *iov, int iovcnt);
+
+} // namespace LIBC_NAMESPACE_DECL
+
+#endif // LLVM_LIBC_SRC_SYS_UIO_READV_H
diff --git a/libc/src/sys/wait/wait4Impl.h b/libc/src/sys/wait/wait4Impl.h
index 5c2cb3e8e0abf..f2bdeb02f8668 100644
--- a/libc/src/sys/wait/wait4Impl.h
+++ b/libc/src/sys/wait/wait4Impl.h
@@ -65,7 +65,9 @@ LIBC_INLINE ErrorOr<pid_t> wait4impl(pid_t pid, int *wait_status, int options,
       *wait_status = W_STOPCODE(info.si_status);
       break;
     case CLD_CONTINUED:
-      *wait_status = __W_CONTINUED;
+      // Set wait_status to a value that the caller can check via WIFCONTINUED.
+      // glibc has a non-POSIX macro definition __W_CONTINUED for this value.
+      *wait_status = 0xffff;
       break;
     default:
       *wait_status = 0;
diff --git a/libc/src/time/CMakeLists.txt b/libc/src/time/CMakeLists.txt
index ef9bfe57bc4ec..dd28aa67280b7 100644
--- a/libc/src/time/CMakeLists.txt
+++ b/libc/src/time/CMakeLists.txt
@@ -22,6 +22,8 @@ add_object_library(
   DEPENDS
     libc.include.time
     libc.src.__support.CPP.limits
+    libc.src.__support.CPP.string_view
+    libc.src.__support.CPP.optional
     libc.src.errno.errno
     .time_constants
     libc.hdr.types.time_t
diff --git a/libc/src/time/mktime.cpp b/libc/src/time/mktime.cpp
index 3874cad02facb..fc05ff2930434 100644
--- a/libc/src/time/mktime.cpp
+++ b/libc/src/time/mktime.cpp
@@ -14,100 +14,8 @@
 
 namespace LIBC_NAMESPACE_DECL {
 
-// Returns number of years from (1, year).
-static constexpr int64_t get_num_of_leap_years_before(int64_t year) {
-  return (year / 4) - (year / 100) + (year / 400);
-}
-
-// Returns True if year is a leap year.
-static constexpr bool is_leap_year(const int64_t year) {
-  return (((year) % 4) == 0 && (((year) % 100) != 0 || ((year) % 400) == 0));
-}
-
 LLVM_LIBC_FUNCTION(time_t, mktime, (struct tm * tm_out)) {
-  // Unlike most C Library functions, mktime doesn't just die on bad input.
-  // TODO(rtenneti); Handle leap seconds.
-  int64_t tm_year_from_base = tm_out->tm_year + time_constants::TIME_YEAR_BASE;
-
-  // 32-bit end-of-the-world is 03:14:07 UTC on 19 January 2038.
-  if (sizeof(time_t) == 4 &&
-      tm_year_from_base >= time_constants::END_OF32_BIT_EPOCH_YEAR) {
-    if (tm_year_from_base > time_constants::END_OF32_BIT_EPOCH_YEAR)
-      return time_utils::out_of_range();
-    if (tm_out->tm_mon > 0)
-      return time_utils::out_of_range();
-    if (tm_out->tm_mday > 19)
-      return time_utils::out_of_range();
-    else if (tm_out->tm_mday == 19) {
-      if (tm_out->tm_hour > 3)
-        return time_utils::out_of_range();
-      else if (tm_out->tm_hour == 3) {
-        if (tm_out->tm_min > 14)
-          return time_utils::out_of_range();
-        else if (tm_out->tm_min == 14) {
-          if (tm_out->tm_sec > 7)
-            return time_utils::out_of_range();
-        }
-      }
-    }
-  }
-
-  // Years are ints.  A 32-bit year will fit into a 64-bit time_t.
-  // A 64-bit year will not.
-  static_assert(
-      sizeof(int) == 4,
-      "ILP64 is unimplemented. This implementation requires 32-bit integers.");
-
-  // Calculate number of months and years from tm_mon.
-  int64_t month = tm_out->tm_mon;
-  if (month < 0 || month >= time_constants::MONTHS_PER_YEAR - 1) {
-    int64_t years = month / 12;
-    month %= 12;
-    if (month < 0) {
-      years--;
-      month += 12;
-    }
-    tm_year_from_base += years;
-  }
-  bool tm_year_is_leap = is_leap_year(tm_year_from_base);
-
-  // Calculate total number of days based on the month and the day (tm_mday).
-  int64_t total_days = tm_out->tm_mday - 1;
-  for (int64_t i = 0; i < month; ++i)
-    total_days += time_constants::NON_LEAP_YEAR_DAYS_IN_MONTH[i];
-  // Add one day if it is a leap year and the month is after February.
-  if (tm_year_is_leap && month > 1)
-    total_days++;
-
-  // Calculate total numbers of days based on the year.
-  total_days += (tm_year_from_base - time_constants::EPOCH_YEAR) *
-                time_constants::DAYS_PER_NON_LEAP_YEAR;
-  if (tm_year_from_base >= time_constants::EPOCH_YEAR) {
-    total_days += get_num_of_leap_years_before(tm_year_from_base - 1) -
-                  get_num_of_leap_years_before(time_constants::EPOCH_YEAR);
-  } else if (tm_year_from_base >= 1) {
-    total_days -= get_num_of_leap_years_before(time_constants::EPOCH_YEAR) -
-                  get_num_of_leap_years_before(tm_year_from_base - 1);
-  } else {
-    // Calculate number of leap years until 0th year.
-    total_days -= get_num_of_leap_years_before(time_constants::EPOCH_YEAR) -
-                  get_num_of_leap_years_before(0);
-    if (tm_year_from_base <= 0) {
-      total_days -= 1; // Subtract 1 for 0th year.
-      // Calculate number of leap years until -1 year
-      if (tm_year_from_base < 0) {
-        total_days -= get_num_of_leap_years_before(-tm_year_from_base) -
-                      get_num_of_leap_years_before(1);
-      }
-    }
-  }
-
-  // TODO: https://github.com/llvm/llvm-project/issues/121962
-  // Need to handle timezone and update of tm_isdst.
-  int64_t seconds = tm_out->tm_sec +
-                    tm_out->tm_min * time_constants::SECONDS_PER_MIN +
-                    tm_out->tm_hour * time_constants::SECONDS_PER_HOUR +
-                    total_days * time_constants::SECONDS_PER_DAY;
+  int64_t seconds = time_utils::mktime_internal(tm_out);
 
   // Update the tm structure's year, month, day, etc. from seconds.
   if (time_utils::update_from_seconds(seconds, tm_out) < 0)
diff --git a/libc/src/time/time_constants.h b/libc/src/time/time_constants.h
index 3e25f741745ab..bcf19ff5f193e 100644
--- a/libc/src/time/time_constants.h
+++ b/libc/src/time/time_constants.h
@@ -18,7 +18,7 @@ namespace LIBC_NAMESPACE_DECL {
 namespace time_constants {
 
 enum Month : int {
-  JANUARY,
+  JANUARY = 0,
   FEBRUARY,
   MARCH,
   APRIL,
@@ -32,14 +32,28 @@ enum Month : int {
   DECEMBER
 };
 
+enum WeekDay : int {
+  SUNDAY = 0,
+  MONDAY,
+  TUESDAY,
+  WEDNESDAY,
+  THURSDAY,
+  FRIDAY,
+  SATURDAY
+};
+
 constexpr int SECONDS_PER_MIN = 60;
 constexpr int MINUTES_PER_HOUR = 60;
 constexpr int HOURS_PER_DAY = 24;
 constexpr int DAYS_PER_WEEK = 7;
+constexpr int WEEKS_PER_YEAR = 52;
 constexpr int MONTHS_PER_YEAR = 12;
 constexpr int DAYS_PER_NON_LEAP_YEAR = 365;
 constexpr int DAYS_PER_LEAP_YEAR = 366;
 
+constexpr int LAST_DAY_OF_NON_LEAP_YEAR = DAYS_PER_NON_LEAP_YEAR - 1;
+constexpr int LAST_DAY_OF_LEAP_YEAR = DAYS_PER_LEAP_YEAR - 1;
+
 constexpr int SECONDS_PER_HOUR = SECONDS_PER_MIN * MINUTES_PER_HOUR;
 constexpr int SECONDS_PER_DAY = SECONDS_PER_HOUR * HOURS_PER_DAY;
 constexpr int NUMBER_OF_SECONDS_IN_LEAP_YEAR =
@@ -49,6 +63,8 @@ constexpr int TIME_YEAR_BASE = 1900;
 constexpr int EPOCH_YEAR = 1970;
 constexpr int EPOCH_WEEK_DAY = 4;
 
+constexpr int ISO_FIRST_DAY_OF_YEAR = 3; // the 4th day of the year, 0-indexed.
+
 // For asctime the behavior is undefined if struct tm's tm_wday or tm_mon are
 // not within the normal ranges as defined in <time.h>, or if struct tm's
 // tm_year exceeds {INT_MAX}-1990, or if the below asctime_internal algorithm
diff --git a/libc/src/time/time_utils.cpp b/libc/src/time/time_utils.cpp
index abc93b8cb961e..3ccb2dd934967 100644
--- a/libc/src/time/time_utils.cpp
+++ b/libc/src/time/time_utils.cpp
@@ -12,9 +12,103 @@
 #include "src/__support/macros/config.h"
 #include "src/time/time_constants.h"
 
+#include <stdint.h>
+
 namespace LIBC_NAMESPACE_DECL {
 namespace time_utils {
 
+// TODO: clean this up in a followup patch
+int64_t mktime_internal(const tm *tm_out) {
+  // Unlike most C Library functions, mktime doesn't just die on bad input.
+  // TODO(rtenneti); Handle leap seconds.
+  int64_t tm_year_from_base = tm_out->tm_year + time_constants::TIME_YEAR_BASE;
+
+  // 32-bit end-of-the-world is 03:14:07 UTC on 19 January 2038.
+  if (sizeof(time_t) == 4 &&
+      tm_year_from_base >= time_constants::END_OF32_BIT_EPOCH_YEAR) {
+    if (tm_year_from_base > time_constants::END_OF32_BIT_EPOCH_YEAR)
+      return time_utils::out_of_range();
+    if (tm_out->tm_mon > 0)
+      return time_utils::out_of_range();
+    if (tm_out->tm_mday > 19)
+      return time_utils::out_of_range();
+    else if (tm_out->tm_mday == 19) {
+      if (tm_out->tm_hour > 3)
+        return time_utils::out_of_range();
+      else if (tm_out->tm_hour == 3) {
+        if (tm_out->tm_min > 14)
+          return time_utils::out_of_range();
+        else if (tm_out->tm_min == 14) {
+          if (tm_out->tm_sec > 7)
+            return time_utils::out_of_range();
+        }
+      }
+    }
+  }
+
+  // Years are ints.  A 32-bit year will fit into a 64-bit time_t.
+  // A 64-bit year will not.
+  static_assert(
+      sizeof(int) == 4,
+      "ILP64 is unimplemented. This implementation requires 32-bit integers.");
+
+  // Calculate number of months and years from tm_mon.
+  int64_t month = tm_out->tm_mon;
+  if (month < 0 || month >= time_constants::MONTHS_PER_YEAR - 1) {
+    int64_t years = month / 12;
+    month %= 12;
+    if (month < 0) {
+      years--;
+      month += 12;
+    }
+    tm_year_from_base += years;
+  }
+  bool tm_year_is_leap = time_utils::is_leap_year(tm_year_from_base);
+
+  // Calculate total number of days based on the month and the day (tm_mday).
+  int64_t total_days = tm_out->tm_mday - 1;
+  for (int64_t i = 0; i < month; ++i)
+    total_days += time_constants::NON_LEAP_YEAR_DAYS_IN_MONTH[i];
+  // Add one day if it is a leap year and the month is after February.
+  if (tm_year_is_leap && month > 1)
+    total_days++;
+
+  // Calculate total numbers of days based on the year.
+  total_days += (tm_year_from_base - time_constants::EPOCH_YEAR) *
+                time_constants::DAYS_PER_NON_LEAP_YEAR;
+  if (tm_year_from_base >= time_constants::EPOCH_YEAR) {
+    total_days +=
+        time_utils::get_num_of_leap_years_before(tm_year_from_base - 1) -
+        time_utils::get_num_of_leap_years_before(time_constants::EPOCH_YEAR);
+  } else if (tm_year_from_base >= 1) {
+    total_days -=
+        time_utils::get_num_of_leap_years_before(time_constants::EPOCH_YEAR) -
+        time_utils::get_num_of_leap_years_before(tm_year_from_base - 1);
+  } else {
+    // Calculate number of leap years until 0th year.
+    total_days -=
+        time_utils::get_num_of_leap_years_before(time_constants::EPOCH_YEAR) -
+        time_utils::get_num_of_leap_years_before(0);
+    if (tm_year_from_base <= 0) {
+      total_days -= 1; // Subtract 1 for 0th year.
+      // Calculate number of leap years until -1 year
+      if (tm_year_from_base < 0) {
+        total_days -=
+            time_utils::get_num_of_leap_years_before(-tm_year_from_base) -
+            time_utils::get_num_of_leap_years_before(1);
+      }
+    }
+  }
+
+  // TODO: https://github.com/llvm/llvm-project/issues/121962
+  // Need to handle timezone and update of tm_isdst.
+  int64_t seconds = tm_out->tm_sec +
+                    tm_out->tm_min * time_constants::SECONDS_PER_MIN +
+                    tm_out->tm_hour * time_constants::SECONDS_PER_HOUR +
+                    total_days * time_constants::SECONDS_PER_DAY;
+  return seconds;
+}
+
 static int64_t computeRemainingYears(int64_t daysPerYears,
                                      int64_t quotientYears,
                                      int64_t *remainingDays) {
@@ -42,7 +136,7 @@ static int64_t computeRemainingYears(int64_t daysPerYears,
 //
 // Compute the number of months from the remaining days. Finally, adjust years
 // to be 1900 and months to be from January.
-int64_t update_from_seconds(int64_t total_seconds, struct tm *tm) {
+int64_t update_from_seconds(int64_t total_seconds, tm *tm) {
   // Days in month starting from March in the year 2000.
   static const char daysInMonth[] = {31 /* Mar */, 30, 31, 30, 31, 31,
                                      30,           31, 30, 31, 31, 29};
diff --git a/libc/src/time/time_utils.h b/libc/src/time/time_utils.h
index 5e0a692d4db04..30cdcbc170639 100644
--- a/libc/src/time/time_utils.h
+++ b/libc/src/time/time_utils.h
@@ -12,6 +12,8 @@
 #include "hdr/types/size_t.h"
 #include "hdr/types/struct_tm.h"
 #include "hdr/types/time_t.h"
+#include "src/__support/CPP/optional.h"
+#include "src/__support/CPP/string_view.h"
 #include "src/__support/common.h"
 #include "src/__support/macros/config.h"
 #include "src/errno/libc_errno.h"
@@ -22,9 +24,13 @@
 namespace LIBC_NAMESPACE_DECL {
 namespace time_utils {
 
+// calculates the seconds from the epoch for tm_in. Does not update the struct,
+// you must call update_from_seconds for that.
+int64_t mktime_internal(const tm *tm_out);
+
 // Update the "tm" structure's year, month, etc. members from seconds.
 // "total_seconds" is the number of seconds since January 1st, 1970.
-extern int64_t update_from_seconds(int64_t total_seconds, struct tm *tm);
+int64_t update_from_seconds(int64_t total_seconds, tm *tm);
 
 // TODO(michaelrj): move these functions to use ErrorOr instead of setting
 // errno. They always accompany a specific return value so we only need the one
@@ -43,7 +49,7 @@ LIBC_INLINE time_t out_of_range() {
 
 LIBC_INLINE void invalid_value() { libc_errno = EINVAL; }
 
-LIBC_INLINE char *asctime(const struct tm *timeptr, char *buffer,
+LIBC_INLINE char *asctime(const tm *timeptr, char *buffer,
                           size_t bufferLength) {
   if (timeptr == nullptr || buffer == nullptr) {
     invalid_value();
@@ -61,6 +67,7 @@ LIBC_INLINE char *asctime(const struct tm *timeptr, char *buffer,
   }
 
   // TODO(michaelr): move this to use the strftime machinery
+  // equivalent to strftime(buffer, bufferLength, "%a %b %T %Y\n", timeptr)
   int written_size = __builtin_snprintf(
       buffer, bufferLength, "%.3s %.3s%3d %.2d:%.2d:%.2d %d\n",
       time_constants::WEEK_DAY_NAMES[timeptr->tm_wday].data(),
@@ -76,7 +83,7 @@ LIBC_INLINE char *asctime(const struct tm *timeptr, char *buffer,
   return buffer;
 }
 
-LIBC_INLINE struct tm *gmtime_internal(const time_t *timer, struct tm *result) {
+LIBC_INLINE tm *gmtime_internal(const time_t *timer, tm *result) {
   int64_t seconds = *timer;
   // Update the tm structure's year, month, day, etc. from seconds.
   if (update_from_seconds(seconds, result) < 0) {
@@ -89,11 +96,252 @@ LIBC_INLINE struct tm *gmtime_internal(const time_t *timer, struct tm *result) {
 
 // TODO: localtime is not yet implemented and a temporary solution is to
 //       use gmtime, https://github.com/llvm/llvm-project/issues/107597
-LIBC_INLINE struct tm *localtime(const time_t *t_ptr) {
-  static struct tm result;
+LIBC_INLINE tm *localtime(const time_t *t_ptr) {
+  static tm result;
   return time_utils::gmtime_internal(t_ptr, &result);
 }
 
+// Returns number of years from (1, year).
+LIBC_INLINE constexpr int64_t get_num_of_leap_years_before(int64_t year) {
+  return (year / 4) - (year / 100) + (year / 400);
+}
+
+// Returns True if year is a leap year.
+LIBC_INLINE constexpr bool is_leap_year(const int64_t year) {
+  return (((year) % 4) == 0 && (((year) % 100) != 0 || ((year) % 400) == 0));
+}
+
+LIBC_INLINE constexpr int get_days_in_year(const int year) {
+  return is_leap_year(year) ? time_constants::DAYS_PER_LEAP_YEAR
+                            : time_constants::DAYS_PER_NON_LEAP_YEAR;
+}
+
+// This is a helper class that takes a struct tm and lets you inspect its
+// values. Where relevant, results are bounds checked and returned as optionals.
+// This class does not, however, do data normalization except where necessary.
+// It will faithfully return a date of 9999-99-99, even though that makes no
+// sense.
+class TMReader final {
+  const tm *timeptr;
+
+  template <size_t N>
+  LIBC_INLINE constexpr cpp::optional<cpp::string_view>
+  bounds_check(const cpp::array<cpp::string_view, N> &arr, int index) const {
+    if (index >= 0 && index < static_cast<int>(arr.size()))
+      return arr[index];
+    return cpp::nullopt;
+  }
+
+public:
+  LIBC_INLINE constexpr explicit TMReader(const tm *tmptr) : timeptr(tmptr) {}
+
+  // Strings
+  LIBC_INLINE constexpr cpp::optional<cpp::string_view>
+  get_weekday_short_name() const {
+    return bounds_check(time_constants::WEEK_DAY_NAMES, timeptr->tm_wday);
+  }
+
+  LIBC_INLINE constexpr cpp::optional<cpp::string_view>
+  get_weekday_full_name() const {
+    return bounds_check(time_constants::WEEK_DAY_FULL_NAMES, timeptr->tm_wday);
+  }
+
+  LIBC_INLINE constexpr cpp::optional<cpp::string_view>
+  get_month_short_name() const {
+    return bounds_check(time_constants::MONTH_NAMES, timeptr->tm_mon);
+  }
+
+  LIBC_INLINE constexpr cpp::optional<cpp::string_view>
+  get_month_full_name() const {
+    return bounds_check(time_constants::MONTH_FULL_NAMES, timeptr->tm_mon);
+  }
+
+  LIBC_INLINE constexpr cpp::string_view get_am_pm() const {
+    if (timeptr->tm_hour < 12)
+      return "AM";
+    return "PM";
+  }
+
+  LIBC_INLINE constexpr cpp::string_view get_timezone_name() const {
+    // TODO: timezone support
+    return "UTC";
+  }
+
+  // Numbers
+  LIBC_INLINE constexpr int get_sec() const { return timeptr->tm_sec; }
+  LIBC_INLINE constexpr int get_min() const { return timeptr->tm_min; }
+  LIBC_INLINE constexpr int get_hour() const { return timeptr->tm_hour; }
+  LIBC_INLINE constexpr int get_mday() const { return timeptr->tm_mday; }
+  LIBC_INLINE constexpr int get_mon() const { return timeptr->tm_mon; }
+  LIBC_INLINE constexpr int get_yday() const { return timeptr->tm_yday; }
+  LIBC_INLINE constexpr int get_wday() const { return timeptr->tm_wday; }
+  LIBC_INLINE constexpr int get_isdst() const { return timeptr->tm_isdst; }
+
+  // returns the year, counting from 1900
+  LIBC_INLINE constexpr int get_year_raw() const { return timeptr->tm_year; }
+  // returns the year, counting from 0
+  LIBC_INLINE constexpr int get_year() const {
+    return timeptr->tm_year + time_constants::TIME_YEAR_BASE;
+  }
+
+  LIBC_INLINE constexpr int is_leap_year() const {
+    return time_utils::is_leap_year(get_year());
+  }
+
+  LIBC_INLINE constexpr int get_iso_wday() const {
+    using time_constants::DAYS_PER_WEEK;
+    using time_constants::MONDAY;
+    // ISO uses a week that starts on Monday, but struct tm starts its week on
+    // Sunday. This function normalizes the weekday so that it always returns a
+    // value 0-6
+    const int NORMALIZED_WDAY = timeptr->tm_wday % DAYS_PER_WEEK;
+    return (NORMALIZED_WDAY + (DAYS_PER_WEEK - MONDAY)) % DAYS_PER_WEEK;
+  }
+
+  // returns the week of the current year, with weeks starting on start_day.
+  LIBC_INLINE constexpr int get_week(time_constants::WeekDay start_day) const {
+    using time_constants::DAYS_PER_WEEK;
+    // The most recent start_day. The rest of the days into the current week
+    // don't count, so ignore them.
+    // Also add 7 to handle start_day > tm_wday
+    const int start_of_cur_week =
+        timeptr->tm_yday -
+        ((timeptr->tm_wday + DAYS_PER_WEEK - start_day) % DAYS_PER_WEEK);
+
+    // The original formula is ceil((start_of_cur_week + 1) / DAYS_PER_WEEK)
+    // That becomes (start_of_cur_week + 1 + DAYS_PER_WEEK - 1) / DAYS_PER_WEEK)
+    // Which simplifies to (start_of_cur_week + DAYS_PER_WEEK) / DAYS_PER_WEEK
+    const int ceil_weeks_since_start =
+        (start_of_cur_week + DAYS_PER_WEEK) / DAYS_PER_WEEK;
+
+    return ceil_weeks_since_start;
+  }
+
+  LIBC_INLINE constexpr int get_iso_week() const {
+    using time_constants::DAYS_PER_WEEK;
+    using time_constants::ISO_FIRST_DAY_OF_YEAR;
+    using time_constants::MONDAY;
+    using time_constants::WeekDay;
+    using time_constants::WEEKS_PER_YEAR;
+
+    constexpr WeekDay START_DAY = MONDAY;
+
+    // The most recent start_day. The rest of the days into the current week
+    // don't count, so ignore them.
+    // Also add 7 to handle start_day > tm_wday
+    const int start_of_cur_week =
+        timeptr->tm_yday -
+        ((timeptr->tm_wday + DAYS_PER_WEEK - START_DAY) % DAYS_PER_WEEK);
+
+    // if the week starts in the previous year, and also if the 4th of this year
+    // is not in this week.
+    if (start_of_cur_week < -3) {
+      const int days_into_prev_year =
+          get_days_in_year(get_year() - 1) + start_of_cur_week;
+      // Each year has at least 52 weeks, but a year's last week will be 53 if
+      // its first week starts in the previous year and its last week ends
+      // in the next year. We know get_year() - 1 must extend into get_year(),
+      // so here we check if it also extended into get_year() - 2 and add 1 week
+      // if it does.
+      return WEEKS_PER_YEAR +
+             ((days_into_prev_year % DAYS_PER_WEEK) > ISO_FIRST_DAY_OF_YEAR);
+    }
+
+    // subtract 1 to account for yday being 0 indexed
+    const int days_until_end_of_year =
+        get_days_in_year(get_year()) - start_of_cur_week - 1;
+
+    // if there are less than 3 days from the start of this week to the end of
+    // the year, then there must be 4 days in this week in the next year, which
+    // means that this week is the first week of that year.
+    if (days_until_end_of_year < 3)
+      return 1;
+
+    // else just calculate the current week like normal.
+    const int ceil_weeks_since_start =
+        (start_of_cur_week + DAYS_PER_WEEK) / DAYS_PER_WEEK;
+
+    // add 1 if this year's first week starts in the previous year.
+    const int WEEK_STARTS_IN_PREV_YEAR =
+        ((start_of_cur_week + time_constants::DAYS_PER_WEEK) %
+         time_constants::DAYS_PER_WEEK) > time_constants::ISO_FIRST_DAY_OF_YEAR;
+    return ceil_weeks_since_start + WEEK_STARTS_IN_PREV_YEAR;
+  }
+
+  LIBC_INLINE constexpr int get_iso_year() const {
+    const int BASE_YEAR = get_year();
+    // The ISO year is the same as a standard year for all dates after the start
+    // of the first week and before the last week. Since the first ISO week of a
+    // year starts on the 4th, anything after that is in this year.
+    if (timeptr->tm_yday >= time_constants::ISO_FIRST_DAY_OF_YEAR &&
+        timeptr->tm_yday < time_constants::DAYS_PER_NON_LEAP_YEAR -
+                               time_constants::DAYS_PER_WEEK)
+      return BASE_YEAR;
+
+    const int ISO_WDAY = get_iso_wday();
+    // The first week of the ISO year is defined as the week containing the
+    // 4th day of January.
+
+    // first week
+    if (timeptr->tm_yday < time_constants::ISO_FIRST_DAY_OF_YEAR) {
+      /*
+      If jan 4 is in this week, then we're in BASE_YEAR, else we're in the
+      previous year. The formula's been rearranged so here's the derivation:
+
+              +--------+-- days until jan 4
+              |        |
+       wday + (4 - yday) < 7
+       |               |
+       +---------------+-- weekday of jan 4
+
+       rearranged to get all the constants on one side:
+
+       wday - yday < 7 - 4
+      */
+      const int IS_CUR_YEAR = (ISO_WDAY - timeptr->tm_yday <
+                               time_constants::DAYS_PER_WEEK -
+                                   time_constants::ISO_FIRST_DAY_OF_YEAR);
+      return BASE_YEAR - !IS_CUR_YEAR;
+    }
+
+    // last week
+    const int DAYS_LEFT_IN_YEAR =
+        get_days_in_year(get_year()) - timeptr->tm_yday;
+    /*
+    Similar to above, we're checking if jan 4 (of next year) is in this week. If
+    it is, this is in the next year. Note that this also handles the case of
+    yday > days in year gracefully.
+
+           +------------------+-- days until jan 4 (of next year)
+           |                  |
+    wday + (4 + remaining days) < 7
+    |                         |
+    +-------------------------+-- weekday of jan 4
+
+    rearranging we get:
+
+    wday + remaining days < 7 - 4
+    */
+    const int IS_NEXT_YEAR =
+        (ISO_WDAY + DAYS_LEFT_IN_YEAR <
+         time_constants::DAYS_PER_WEEK - time_constants::ISO_FIRST_DAY_OF_YEAR);
+    return BASE_YEAR + IS_NEXT_YEAR;
+  }
+
+  LIBC_INLINE constexpr time_t get_epoch() const {
+    return mktime_internal(timeptr);
+  }
+
+  // returns the timezone offset in microwave time:
+  // return (hours * 100) + minutes;
+  // This means that a shift of -4:30 is returned as -430, simplifying
+  // conversion.
+  LIBC_INLINE constexpr int get_timezone_offset() const {
+    // TODO: timezone support
+    return 0;
+  }
+};
+
 } // namespace time_utils
 } // namespace LIBC_NAMESPACE_DECL
 
diff --git a/libc/src/unistd/CMakeLists.txt b/libc/src/unistd/CMakeLists.txt
index 6bdea0c7693bd..fb563ec4ecfd9 100644
--- a/libc/src/unistd/CMakeLists.txt
+++ b/libc/src/unistd/CMakeLists.txt
@@ -231,6 +231,13 @@ add_entrypoint_object(
     .${LIBC_TARGET_OS}.rmdir
 )
 
+add_entrypoint_object(
+  setsid
+  ALIAS
+  DEPENDS
+    .${LIBC_TARGET_OS}.setsid
+)
+
 add_entrypoint_object(
   symlink
   ALIAS
diff --git a/libc/src/unistd/linux/CMakeLists.txt b/libc/src/unistd/linux/CMakeLists.txt
index 2bb17f56f7b32..afdc595d0b26f 100644
--- a/libc/src/unistd/linux/CMakeLists.txt
+++ b/libc/src/unistd/linux/CMakeLists.txt
@@ -459,6 +459,18 @@ add_entrypoint_object(
     libc.src.errno.errno
 )
 
+add_entrypoint_object(
+  setsid
+  SRCS
+    setsid.cpp
+  HDRS
+    ../setsid.h
+  DEPENDS
+    libc.hdr.types.pid_t
+    libc.include.sys_syscall
+    libc.src.__support.OSUtil.osutil
+)
+
 add_entrypoint_object(
   symlink
   SRCS
diff --git a/libc/src/unistd/linux/setsid.cpp b/libc/src/unistd/linux/setsid.cpp
new file mode 100644
index 0000000000000..df4629bb326cc
--- /dev/null
+++ b/libc/src/unistd/linux/setsid.cpp
@@ -0,0 +1,24 @@
+//===-- Linux implementation of setsid-------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/unistd/setsid.h"
+
+#include "hdr/types/pid_t.h"
+#include "src/__support/OSUtil/syscall.h" // For internal syscall function.
+#include "src/__support/common.h"
+#include "src/__support/macros/config.h"
+
+#include <sys/syscall.h> // For syscall numbers.
+
+namespace LIBC_NAMESPACE_DECL {
+
+LLVM_LIBC_FUNCTION(pid_t, setsid, ()) {
+  return LIBC_NAMESPACE::syscall_impl<pid_t>(SYS_setsid);
+}
+
+} // namespace LIBC_NAMESPACE_DECL
diff --git a/libc/src/unistd/setsid.h b/libc/src/unistd/setsid.h
new file mode 100644
index 0000000000000..44dd7dd2ce816
--- /dev/null
+++ b/libc/src/unistd/setsid.h
@@ -0,0 +1,21 @@
+//===-- Implementation header for setsid ------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_UNISTD_GETPID_H
+#define LLVM_LIBC_SRC_UNISTD_GETPID_H
+
+#include "hdr/types/pid_t.h"
+#include "src/__support/macros/config.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+pid_t setsid();
+
+} // namespace LIBC_NAMESPACE_DECL
+
+#endif // LLVM_LIBC_SRC_UNISTD_GETPID_H
diff --git a/libc/test/UnitTest/FPExceptMatcher.cpp b/libc/test/UnitTest/FPExceptMatcher.cpp
index 119a06985b8f1..d66066023984e 100644
--- a/libc/test/UnitTest/FPExceptMatcher.cpp
+++ b/libc/test/UnitTest/FPExceptMatcher.cpp
@@ -37,7 +37,7 @@ static void sigfpeHandler(int sig) {
 }
 
 FPExceptMatcher::FPExceptMatcher(FunctionCaller *func) {
-  sighandler_t oldSIGFPEHandler = signal(SIGFPE, &sigfpeHandler);
+  auto *oldSIGFPEHandler = signal(SIGFPE, &sigfpeHandler);
 
   caughtExcept = false;
   fenv_t oldEnv;
diff --git a/libc/test/UnitTest/FPMatcher.h b/libc/test/UnitTest/FPMatcher.h
index b8e240bf328ce..53e0c16f22101 100644
--- a/libc/test/UnitTest/FPMatcher.h
+++ b/libc/test/UnitTest/FPMatcher.h
@@ -11,7 +11,6 @@
 
 #include "src/__support/CPP/array.h"
 #include "src/__support/CPP/type_traits.h"
-#include "src/__support/CPP/type_traits/is_complex.h"
 #include "src/__support/FPUtil/FEnvImpl.h"
 #include "src/__support/FPUtil/FPBits.h"
 #include "src/__support/FPUtil/fpbits_str.h"
diff --git a/libc/test/integration/src/__support/GPU/CMakeLists.txt b/libc/test/integration/src/__support/GPU/CMakeLists.txt
index 7811e0da45ddc..68bbc3849bc7e 100644
--- a/libc/test/integration/src/__support/GPU/CMakeLists.txt
+++ b/libc/test/integration/src/__support/GPU/CMakeLists.txt
@@ -9,3 +9,12 @@ add_integration_test(
   LOADER_ARGS
     --threads 64
 )
+
+add_integration_test(
+  shuffle_test
+  SUITE libc-support-gpu-tests
+  SRCS
+    shuffle.cpp
+  LOADER_ARGS
+    --threads 64
+)
diff --git a/libc/test/integration/src/__support/GPU/shuffle.cpp b/libc/test/integration/src/__support/GPU/shuffle.cpp
new file mode 100644
index 0000000000000..c346a2eb3f0c2
--- /dev/null
+++ b/libc/test/integration/src/__support/GPU/shuffle.cpp
@@ -0,0 +1,33 @@
+//===-- Test for the shuffle operations on the GPU ------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/__support/CPP/bit.h"
+#include "src/__support/GPU/utils.h"
+#include "test/IntegrationTest/test.h"
+
+using namespace LIBC_NAMESPACE;
+
+// Test to make sure the shuffle instruction works by doing a simple broadcast.
+// Each iteration reduces the width, so it will broadcast to a subset we check.
+static void test_shuffle() {
+  uint64_t mask = gpu::get_lane_mask();
+  EXPECT_EQ(cpp::popcount(mask), gpu::get_lane_size());
+
+  uint32_t x = gpu::get_lane_id();
+  for (uint32_t width = gpu::get_lane_size(); width > 0; width /= 2)
+    EXPECT_EQ(gpu::shuffle(mask, 0, x, width), (x / width) * width);
+}
+
+TEST_MAIN(int argc, char **argv, char **envp) {
+  if (gpu::get_thread_id() >= gpu::get_lane_size())
+    return 0;
+
+  test_shuffle();
+
+  return 0;
+}
diff --git a/libc/test/src/CMakeLists.txt b/libc/test/src/CMakeLists.txt
index 22ec43588f744..b7c145788c0cd 100644
--- a/libc/test/src/CMakeLists.txt
+++ b/libc/test/src/CMakeLists.txt
@@ -1,12 +1,21 @@
 function(add_fp_unittest name)
   cmake_parse_arguments(
     "MATH_UNITTEST"
-    "NEED_MPFR;UNIT_TEST_ONLY;HERMETIC_TEST_ONLY" # Optional arguments
+    "NEED_MPFR;NEED_MPC;UNIT_TEST_ONLY;HERMETIC_TEST_ONLY" # Optional arguments
     "" # Single value arguments
     "LINK_LIBRARIES;DEPENDS" # Multi-value arguments
     ${ARGN}
   )
 
+  if(MATH_UNITTEST_NEED_MPC)
+    set(MATH_UNITTEST_NEED_MPFR TRUE)
+    if(NOT LIBC_TESTS_CAN_USE_MPC)
+      message(VERBOSE "Complex test ${name} will be skipped as MPC library is not available.")
+      return()
+    endif()
+    list(APPEND MATH_UNITTEST_LINK_LIBRARIES libcMPCWrapper)
+  endif()
+
   if(MATH_UNITTEST_NEED_MPFR)
     if(NOT LIBC_TESTS_CAN_USE_MPFR)
       message(VERBOSE "Math test ${name} will be skipped as MPFR library is not available.")
@@ -68,6 +77,7 @@ add_subdirectory(inttypes)
 
 if(${LIBC_TARGET_OS} STREQUAL "linux")
   add_subdirectory(fcntl)
+  add_subdirectory(poll)
   add_subdirectory(sched)
   add_subdirectory(sys)
   add_subdirectory(termios)
diff --git a/libc/test/src/__support/CMakeLists.txt b/libc/test/src/__support/CMakeLists.txt
index aeb8edf305d05..8d175e857fcd1 100644
--- a/libc/test/src/__support/CMakeLists.txt
+++ b/libc/test/src/__support/CMakeLists.txt
@@ -234,7 +234,7 @@ add_libc_test(
     libc.src.stdlib.srand
     libc.src.string.memset
   COMPILE_OPTIONS
-    -O3
+    ${libc_opt_high_flag}
   UNIT_TEST_ONLY
     # Aligned Allocation is not supported in hermetic builds.
 )
diff --git a/libc/test/src/__support/integer_to_string_test.cpp b/libc/test/src/__support/integer_to_string_test.cpp
index e644751b56c93..6ec2dc6eba521 100644
--- a/libc/test/src/__support/integer_to_string_test.cpp
+++ b/libc/test/src/__support/integer_to_string_test.cpp
@@ -16,6 +16,7 @@
 
 #include "test/UnitTest/Test.h"
 
+using LIBC_NAMESPACE::BigInt;
 using LIBC_NAMESPACE::IntegerToString;
 using LIBC_NAMESPACE::cpp::span;
 using LIBC_NAMESPACE::cpp::string_view;
@@ -297,6 +298,107 @@ TEST(LlvmLibcIntegerToStringTest, Sign) {
   EXPECT(DEC, 1, "+1");
 }
 
+TEST(LlvmLibcIntegerToStringTest, BigInt_Base_10) {
+  uint64_t int256_max_w64[4] = {
+      0xFFFFFFFFFFFFFFFF,
+      0xFFFFFFFFFFFFFFFF,
+      0xFFFFFFFFFFFFFFFF,
+      0x7FFFFFFFFFFFFFFF,
+  };
+  uint64_t int256_min_w64[4] = {
+      0,
+      0,
+      0,
+      0x8000000000000000,
+  };
+  uint32_t int256_max_w32[8] = {
+      0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+      0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0x7FFFFFFF,
+  };
+  uint32_t int256_min_w32[8] = {
+      0, 0, 0, 0, 0, 0, 0, 0x80000000,
+  };
+  uint16_t int256_max_w16[16] = {
+      0xFFFF, 0xFFFF, 0xFFFF, 0xFFFF, 0xFFFF, 0xFFFF, 0xFFFF, 0xFFFF,
+      0xFFFF, 0xFFFF, 0xFFFF, 0xFFFF, 0xFFFF, 0xFFFF, 0xFFFF, 0x7FFF,
+  };
+  uint16_t int256_min_w16[16] = {
+      0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0x8000,
+  };
+
+  using unsigned_type_w64 = IntegerToString<BigInt<256, false, uint64_t>, Dec>;
+  EXPECT(unsigned_type_w64, 0, "0");
+  EXPECT(unsigned_type_w64, 1, "1");
+  EXPECT(unsigned_type_w64, -1,
+         "115792089237316195423570985008687907853269984665640564039457584007913"
+         "129639935");
+  EXPECT(unsigned_type_w64, int256_max_w64,
+         "578960446186580977117854925043439539266349923328202820197287920039565"
+         "64819967");
+  EXPECT(unsigned_type_w64, int256_min_w64,
+         "578960446186580977117854925043439539266349923328202820197287920039565"
+         "64819968");
+
+  using unsigned_type_w32 = IntegerToString<BigInt<256, false, uint32_t>, Dec>;
+  EXPECT(unsigned_type_w32, 0, "0");
+  EXPECT(unsigned_type_w32, 1, "1");
+  EXPECT(unsigned_type_w32, -1,
+         "115792089237316195423570985008687907853269984665640564039457584007913"
+         "129639935");
+  EXPECT(unsigned_type_w32, int256_max_w32,
+         "578960446186580977117854925043439539266349923328202820197287920039565"
+         "64819967");
+  EXPECT(unsigned_type_w32, int256_min_w32,
+         "578960446186580977117854925043439539266349923328202820197287920039565"
+         "64819968");
+
+  using unsigned_type_w16 = IntegerToString<BigInt<256, false, uint16_t>, Dec>;
+  EXPECT(unsigned_type_w16, 0, "0");
+  EXPECT(unsigned_type_w16, 1, "1");
+  EXPECT(unsigned_type_w16, -1,
+         "115792089237316195423570985008687907853269984665640564039457584007913"
+         "129639935");
+  EXPECT(unsigned_type_w16, int256_max_w16,
+         "578960446186580977117854925043439539266349923328202820197287920039565"
+         "64819967");
+  EXPECT(unsigned_type_w16, int256_min_w16,
+         "578960446186580977117854925043439539266349923328202820197287920039565"
+         "64819968");
+
+  using signed_type_w64 = IntegerToString<BigInt<256, true, uint64_t>, Dec>;
+  EXPECT(signed_type_w64, 0, "0");
+  EXPECT(signed_type_w64, 1, "1");
+  EXPECT(signed_type_w64, -1, "-1");
+  EXPECT(signed_type_w64, int256_max_w64,
+         "578960446186580977117854925043439539266349923328202820197287920039565"
+         "64819967");
+  EXPECT(signed_type_w64, int256_min_w64,
+         "-57896044618658097711785492504343953926634992332820282019728792003956"
+         "564819968");
+
+  using signed_type_w32 = IntegerToString<BigInt<256, true, uint32_t>, Dec>;
+  EXPECT(signed_type_w32, 0, "0");
+  EXPECT(signed_type_w32, 1, "1");
+  EXPECT(signed_type_w32, -1, "-1");
+  EXPECT(signed_type_w32, int256_max_w32,
+         "578960446186580977117854925043439539266349923328202820197287920039565"
+         "64819967");
+  EXPECT(signed_type_w32, int256_min_w32,
+         "-57896044618658097711785492504343953926634992332820282019728792003956"
+         "564819968");
+
+  using signed_type_w16 = IntegerToString<BigInt<256, true, uint16_t>, Dec>;
+  EXPECT(signed_type_w16, 0, "0");
+  EXPECT(signed_type_w16, 1, "1");
+  EXPECT(signed_type_w16, -1, "-1");
+  EXPECT(signed_type_w16, int256_max_w16,
+         "578960446186580977117854925043439539266349923328202820197287920039565"
+         "64819967");
+  EXPECT(signed_type_w16, int256_min_w16,
+         "-57896044618658097711785492504343953926634992332820282019728792003956"
+         "564819968");
+}
+
 TEST(LlvmLibcIntegerToStringTest, BufferOverrun) {
   { // Writing '0' in an empty buffer requiring zero digits : works
     const auto view =
diff --git a/libc/test/src/compiler/stack_chk_guard_test.cpp b/libc/test/src/compiler/stack_chk_guard_test.cpp
index 4ec8398c9fc95..301031ff47bd5 100644
--- a/libc/test/src/compiler/stack_chk_guard_test.cpp
+++ b/libc/test/src/compiler/stack_chk_guard_test.cpp
@@ -7,24 +7,9 @@
 //===----------------------------------------------------------------------===//
 
 #include "hdr/signal_macros.h"
-#include "src/__support/macros/sanitizer.h"
 #include "src/compiler/__stack_chk_fail.h"
-#include "src/string/memset.h"
 #include "test/UnitTest/Test.h"
 
 TEST(LlvmLibcStackChkFail, Death) {
   EXPECT_DEATH([] { __stack_chk_fail(); }, WITH_SIGNAL(SIGABRT));
 }
-
-// Disable the test when asan is enabled so that it doesn't immediately fail
-// after the memset, but before the stack canary is re-checked.
-#ifndef LIBC_HAS_ADDRESS_SANITIZER
-TEST(LlvmLibcStackChkFail, Smash) {
-  EXPECT_DEATH(
-      [] {
-        int arr[20];
-        LIBC_NAMESPACE::memset(arr, 0xAA, 2001);
-      },
-      WITH_SIGNAL(SIGABRT));
-}
-#endif // LIBC_HAS_ADDRESS_SANITIZER
diff --git a/libc/test/src/complex/CMakeLists.txt b/libc/test/src/complex/CMakeLists.txt
index d6b62e4686a22..efd1ede63eca5 100644
--- a/libc/test/src/complex/CMakeLists.txt
+++ b/libc/test/src/complex/CMakeLists.txt
@@ -1,6 +1,6 @@
 add_custom_target(libc-complex-unittests)
 
-add_libc_test(
+add_fp_unittest(
   conj_test
   SUITE
     libc-complex-unittests
@@ -8,11 +8,9 @@ add_libc_test(
     conj_test.cpp
   DEPENDS
     libc.src.complex.conj
-  LINK_LIBRARIES
-      LibcFPTestHelpers
 )
 
-add_libc_test(
+add_fp_unittest(
   conjf_test
   SUITE
     libc-complex-unittests
@@ -20,11 +18,9 @@ add_libc_test(
     conjf_test.cpp
   DEPENDS
     libc.src.complex.conjf
-  LINK_LIBRARIES
-    LibcFPTestHelpers
 )
 
-add_libc_test(
+add_fp_unittest(
   conjl_test
   SUITE
     libc-complex-unittests
@@ -32,11 +28,9 @@ add_libc_test(
     conjl_test.cpp
   DEPENDS
     libc.src.complex.conjl
-  LINK_LIBRARIES
-    LibcFPTestHelpers
 )
 
-add_libc_test(
+add_fp_unittest(
   conjf16_test
   SUITE
     libc-complex-unittests
@@ -44,11 +38,9 @@ add_libc_test(
     conjf16_test.cpp
   DEPENDS
     libc.src.complex.conjf16
-  LINK_LIBRARIES
-    LibcFPTestHelpers
 )
 
-add_libc_test(
+add_fp_unittest(
   conjf128_test
   SUITE
     libc-complex-unittests
@@ -56,11 +48,9 @@ add_libc_test(
     conjf128_test.cpp
   DEPENDS
     libc.src.complex.conjf128
-  LINK_LIBRARIES
-    LibcFPTestHelpers
 )
 
-add_libc_test(
+add_fp_unittest(
   cproj_test
   SUITE
     libc-complex-unittests
@@ -68,23 +58,20 @@ add_libc_test(
     cproj_test.cpp
   DEPENDS
     libc.src.complex.cproj
-  LINK_LIBRARIES
-      LibcFPTestHelpers
 )
 
-add_libc_test(
+add_fp_unittest(
   cprojf_test
+  NEED_MPC
   SUITE
     libc-complex-unittests
   SRCS
     cprojf_test.cpp
   DEPENDS
     libc.src.complex.cprojf
-  LINK_LIBRARIES
-    LibcFPTestHelpers
 )
 
-add_libc_test(
+add_fp_unittest(
   cprojl_test
   SUITE
     libc-complex-unittests
@@ -92,11 +79,9 @@ add_libc_test(
     cprojl_test.cpp
   DEPENDS
     libc.src.complex.cprojl
-  LINK_LIBRARIES
-    LibcFPTestHelpers
 )
 
-add_libc_test(
+add_fp_unittest(
   cprojf16_test
   SUITE
     libc-complex-unittests
@@ -104,11 +89,9 @@ add_libc_test(
     cprojf16_test.cpp
   DEPENDS
     libc.src.complex.cprojf16
-  LINK_LIBRARIES
-    LibcFPTestHelpers
 )
 
-add_libc_test(
+add_fp_unittest(
   cprojf128_test
   SUITE
     libc-complex-unittests
@@ -116,11 +99,9 @@ add_libc_test(
     cprojf128_test.cpp
   DEPENDS
     libc.src.complex.cprojf128
-  LINK_LIBRARIES
-    LibcFPTestHelpers
 )
 
-add_libc_test(
+add_fp_unittest(
   creal_test
   SUITE
     libc-complex-unittests
@@ -128,11 +109,9 @@ add_libc_test(
     creal_test.cpp
   DEPENDS
     libc.src.complex.creal
-  LINK_LIBRARIES
-      LibcFPTestHelpers
 )
 
-add_libc_test(
+add_fp_unittest(
   crealf_test
   SUITE
     libc-complex-unittests
@@ -140,11 +119,9 @@ add_libc_test(
     crealf_test.cpp
   DEPENDS
     libc.src.complex.crealf
-  LINK_LIBRARIES
-    LibcFPTestHelpers
 )
 
-add_libc_test(
+add_fp_unittest(
   creall_test
   SUITE
     libc-complex-unittests
@@ -152,11 +129,9 @@ add_libc_test(
     creall_test.cpp
   DEPENDS
     libc.src.complex.creall
-  LINK_LIBRARIES
-    LibcFPTestHelpers
 )
 
-add_libc_test(
+add_fp_unittest(
   crealf16_test
   SUITE
     libc-complex-unittests
@@ -164,11 +139,9 @@ add_libc_test(
     crealf16_test.cpp
   DEPENDS
     libc.src.complex.crealf16
-  LINK_LIBRARIES
-    LibcFPTestHelpers
 )
 
-add_libc_test(
+add_fp_unittest(
   crealf128_test
   SUITE
     libc-complex-unittests
@@ -176,11 +149,9 @@ add_libc_test(
     crealf128_test.cpp
   DEPENDS
     libc.src.complex.crealf128
-  LINK_LIBRARIES
-    LibcFPTestHelpers
 )
 
-add_libc_test(
+add_fp_unittest(
   cimag_test
   SUITE
     libc-complex-unittests
@@ -188,11 +159,9 @@ add_libc_test(
     cimag_test.cpp
   DEPENDS
     libc.src.complex.cimag
-  LINK_LIBRARIES
-    LibcFPTestHelpers
 )
 
-add_libc_test(
+add_fp_unittest(
   cimagf_test
   SUITE
     libc-complex-unittests
@@ -200,11 +169,9 @@ add_libc_test(
     cimagf_test.cpp
   DEPENDS
     libc.src.complex.cimagf
-  LINK_LIBRARIES
-    LibcFPTestHelpers
 )
 
-add_libc_test(
+add_fp_unittest(
   cimagl_test
   SUITE
     libc-complex-unittests
@@ -212,11 +179,9 @@ add_libc_test(
     cimagl_test.cpp
   DEPENDS
     libc.src.complex.cimagl
-  LINK_LIBRARIES
-    LibcFPTestHelpers
 )
 
-add_libc_test(
+add_fp_unittest(
   cimagf16_test
   SUITE
     libc-complex-unittests
@@ -224,11 +189,9 @@ add_libc_test(
     cimagf16_test.cpp
   DEPENDS
     libc.src.complex.cimagf16
-  LINK_LIBRARIES
-    LibcFPTestHelpers
 )
 
-add_libc_test(
+add_fp_unittest(
   cimagf128_test
   SUITE
     libc-complex-unittests
@@ -236,6 +199,4 @@ add_libc_test(
     cimagf128_test.cpp
   DEPENDS
     libc.src.complex.cimagf128
-  LINK_LIBRARIES
-    LibcFPTestHelpers
 )
diff --git a/libc/test/src/complex/cprojf_test.cpp b/libc/test/src/complex/cprojf_test.cpp
index 7123ed4e28d4b..4635aa17ad02d 100644
--- a/libc/test/src/complex/cprojf_test.cpp
+++ b/libc/test/src/complex/cprojf_test.cpp
@@ -10,4 +10,16 @@
 
 #include "src/complex/cprojf.h"
 
+#include "utils/MPCWrapper/MPCUtils.h"
+
+using LlvmLibcCprojTestMPC = LIBC_NAMESPACE::testing::FPTest<float>;
+
+namespace mpc = LIBC_NAMESPACE::testing::mpc;
+
+TEST_F(LlvmLibcCprojTestMPC, MPCRND) {
+  _Complex float test = 5.0 + 10.0i;
+  EXPECT_MPC_MATCH_ALL_ROUNDING(mpc::Operation::Cproj, test,
+                                LIBC_NAMESPACE::cprojf(test), 0.5);
+}
+
 LIST_CPROJ_TESTS(_Complex float, float, LIBC_NAMESPACE::cprojf)
diff --git a/libc/test/src/math/CMakeLists.txt b/libc/test/src/math/CMakeLists.txt
index bbcdf2363c1e2..6a3dd8c9deff0 100644
--- a/libc/test/src/math/CMakeLists.txt
+++ b/libc/test/src/math/CMakeLists.txt
@@ -1597,7 +1597,7 @@ add_fp_unittest(
     libc.src.math.sqrtf
     libc.src.__support.FPUtil.generic.sqrt
   COMPILE_OPTIONS
-    -O3
+    ${libc_opt_high_flag}
 )
 
 add_fp_unittest(
@@ -1613,7 +1613,7 @@ add_fp_unittest(
     libc.src.math.sqrt
     libc.src.__support.FPUtil.generic.sqrt
   COMPILE_OPTIONS
-    -O3
+    ${libc_opt_high_flag}
 )
 
 add_fp_unittest(
@@ -1629,7 +1629,7 @@ add_fp_unittest(
     libc.src.math.sqrtl
     libc.src.__support.FPUtil.generic.sqrt
   COMPILE_OPTIONS
-    -O3
+    ${libc_opt_high_flag}
 )
 
 add_fp_unittest(
@@ -2186,6 +2186,17 @@ add_fp_unittest(
     libc.src.__support.FPUtil.fp_bits
 )
 
+add_fp_unittest(
+  asinf16_test
+  NEED_MPFR
+  SUITE
+    libc-math-unittests
+  SRCS
+    asinf16_test.cpp
+  DEPENDS
+    libc.src.math.asinf16  
+)
+
 add_fp_unittest(
   acosf_test
   NEED_MPFR
diff --git a/libc/test/src/math/asinf16_test.cpp b/libc/test/src/math/asinf16_test.cpp
new file mode 100644
index 0000000000000..9593cad16ac77
--- /dev/null
+++ b/libc/test/src/math/asinf16_test.cpp
@@ -0,0 +1,42 @@
+//===-- Exhaustive test for asinf16 ---------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/math/asinf16.h"
+#include "test/UnitTest/FPMatcher.h"
+#include "test/UnitTest/Test.h"
+#include "utils/MPFRWrapper/MPFRUtils.h"
+
+using LlvmLibcAsinf16Test = LIBC_NAMESPACE::testing::FPTest<float16>;
+
+namespace mpfr = LIBC_NAMESPACE::testing::mpfr;
+
+// Range: [0, Inf]
+static constexpr uint16_t POS_START = 0x0000U;
+static constexpr uint16_t POS_STOP = 0x7c00U;
+
+// Range: [-Inf, 0]
+static constexpr uint16_t NEG_START = 0x8000U;
+static constexpr uint16_t NEG_STOP = 0xfc00U;
+
+TEST_F(LlvmLibcAsinf16Test, PositiveRange) {
+  for (uint16_t v = POS_START; v <= POS_STOP; ++v) {
+    float16 x = FPBits(v).get_val();
+
+    EXPECT_MPFR_MATCH_ALL_ROUNDING(mpfr::Operation::Asin, x,
+                                   LIBC_NAMESPACE::asinf16(x), 0.5);
+  }
+}
+
+TEST_F(LlvmLibcAsinf16Test, NegativeRange) {
+  for (uint16_t v = NEG_START; v <= NEG_STOP; ++v) {
+    float16 x = FPBits(v).get_val();
+
+    EXPECT_MPFR_MATCH_ALL_ROUNDING(mpfr::Operation::Asin, x,
+                                   LIBC_NAMESPACE::asinf16(x), 0.5);
+  }
+}
diff --git a/libc/test/src/math/exhaustive/CMakeLists.txt b/libc/test/src/math/exhaustive/CMakeLists.txt
index 423c3b7a8bfd1..b1927dbc19a3b 100644
--- a/libc/test/src/math/exhaustive/CMakeLists.txt
+++ b/libc/test/src/math/exhaustive/CMakeLists.txt
@@ -305,7 +305,7 @@ add_fp_unittest(
   SRCS
     hypotf_test.cpp
   COMPILE_OPTIONS
-    -O3
+    ${libc_opt_high_flag}
   DEPENDS
     .exhaustive_test
     libc.src.math.hypotf
diff --git a/libc/test/src/math/smoke/CMakeLists.txt b/libc/test/src/math/smoke/CMakeLists.txt
index e4501eb75fa48..14447728fb18a 100644
--- a/libc/test/src/math/smoke/CMakeLists.txt
+++ b/libc/test/src/math/smoke/CMakeLists.txt
@@ -45,6 +45,7 @@ add_fp_unittest(
   DEPENDS
     libc.src.errno.errno
     libc.src.math.cospif16
+    libc.src.__support.FPUtil.cast
 )
 
 add_fp_unittest(
@@ -69,6 +70,7 @@ add_fp_unittest(
   DEPENDS
     libc.src.errno.errno
     libc.src.math.sinf16
+    libc.src.__support.FPUtil.cast
 )
 
 add_fp_unittest(
@@ -1251,6 +1253,7 @@ add_fp_unittest(
     libc.hdr.fenv_macros
     libc.src.errno.errno
     libc.src.math.exp2m1f16
+    libc.src.__support.FPUtil.cast
 )
 
 add_fp_unittest(
@@ -2990,7 +2993,7 @@ add_fp_unittest(
   DEPENDS
     libc.src.__support.FPUtil.generic.sqrt
   COMPILE_OPTIONS
-    -O3
+    ${libc_opt_high_flag}
 )
 
 add_fp_unittest(
@@ -3004,7 +3007,7 @@ add_fp_unittest(
   DEPENDS
     libc.src.__support.FPUtil.generic.sqrt
   COMPILE_OPTIONS
-    -O3
+    ${libc_opt_high_flag}
 )
 
 add_fp_unittest(
@@ -3018,7 +3021,7 @@ add_fp_unittest(
   DEPENDS
     libc.src.__support.FPUtil.generic.sqrt
   COMPILE_OPTIONS
-    -O3
+    ${libc_opt_high_flag}
 )
 
 add_fp_unittest(
@@ -3035,7 +3038,7 @@ add_fp_unittest(
     libc.src.math.sqrtf128
     libc.src.__support.FPUtil.generic.sqrt
   COMPILE_OPTIONS
-    -O3
+    ${libc_opt_high_flag}
 )
 
 add_fp_unittest(
@@ -3954,6 +3957,17 @@ add_fp_unittest(
     libc.src.__support.FPUtil.fp_bits
 )
 
+add_fp_unittest(
+  asinf16_test
+  SUITE
+    libc-math-smoke-tests
+  SRCS
+    asinf16_test.cpp
+  DEPENDS
+    libc.src.errno.errno
+    libc.src.math.asinf16  
+)
+
 add_fp_unittest(
   acosf_test
   SUITE
diff --git a/libc/test/src/math/smoke/asinf16_test.cpp b/libc/test/src/math/smoke/asinf16_test.cpp
new file mode 100644
index 0000000000000..9f675b08319c0
--- /dev/null
+++ b/libc/test/src/math/smoke/asinf16_test.cpp
@@ -0,0 +1,42 @@
+//===-- Unittests for asinf16 ---------------------------------------------===//
+//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception.
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/errno/libc_errno.h"
+#include "src/math/asinf16.h"
+#include "test/UnitTest/FPMatcher.h"
+#include "test/UnitTest/Test.h"
+
+using LlvmLibcAsinf16Test = LIBC_NAMESPACE::testing::FPTest<float16>;
+
+TEST_F(LlvmLibcAsinf16Test, SpecialNumbers) {
+  LIBC_NAMESPACE::libc_errno = 0;
+  EXPECT_FP_EQ(aNaN, LIBC_NAMESPACE::asinf16(aNaN));
+  EXPECT_MATH_ERRNO(0);
+
+  EXPECT_FP_EQ_WITH_EXCEPTION(aNaN, LIBC_NAMESPACE::asinf16(sNaN), FE_INVALID);
+  EXPECT_MATH_ERRNO(0);
+
+  EXPECT_FP_EQ(zero, LIBC_NAMESPACE::asinf16(zero));
+  EXPECT_MATH_ERRNO(0);
+
+  EXPECT_FP_EQ(neg_zero, LIBC_NAMESPACE::asinf16(neg_zero));
+  EXPECT_MATH_ERRNO(0);
+
+  EXPECT_FP_EQ(aNaN, LIBC_NAMESPACE::asinf16(inf));
+  EXPECT_MATH_ERRNO(EDOM);
+
+  EXPECT_FP_EQ(aNaN, LIBC_NAMESPACE::asinf16(neg_inf));
+  EXPECT_MATH_ERRNO(EDOM);
+
+  EXPECT_FP_EQ(aNaN, LIBC_NAMESPACE::asinf16(2.0f));
+  EXPECT_MATH_ERRNO(EDOM);
+
+  EXPECT_FP_EQ(aNaN, LIBC_NAMESPACE::asinf16(-2.0f));
+  EXPECT_MATH_ERRNO(EDOM);
+}
diff --git a/libc/test/src/math/smoke/cospif16_test.cpp b/libc/test/src/math/smoke/cospif16_test.cpp
index f6d7483393191..135267ab2ae6f 100644
--- a/libc/test/src/math/smoke/cospif16_test.cpp
+++ b/libc/test/src/math/smoke/cospif16_test.cpp
@@ -6,6 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
+#include "src/__support/FPUtil/cast.h"
 #include "src/errno/libc_errno.h"
 #include "src/math/cospif16.h"
 #include "test/UnitTest/FPMatcher.h"
@@ -19,10 +20,10 @@ TEST_F(LlvmLibcCospif16Test, SpecialNumbers) {
   EXPECT_FP_EQ(aNaN, LIBC_NAMESPACE::cospif16(aNaN));
   EXPECT_MATH_ERRNO(0);
 
-  EXPECT_FP_EQ(1.0f, LIBC_NAMESPACE::cospif16(zero));
+  EXPECT_FP_EQ(FPBits::one().get_val(), LIBC_NAMESPACE::cospif16(zero));
   EXPECT_MATH_ERRNO(0);
 
-  EXPECT_FP_EQ(1.0f, LIBC_NAMESPACE::cospif16(neg_zero));
+  EXPECT_FP_EQ(FPBits::one().get_val(), LIBC_NAMESPACE::cospif16(neg_zero));
   EXPECT_MATH_ERRNO(0);
 
   EXPECT_FP_EQ(aNaN, LIBC_NAMESPACE::cospif16(inf));
@@ -33,12 +34,24 @@ TEST_F(LlvmLibcCospif16Test, SpecialNumbers) {
 }
 
 TEST_F(LlvmLibcCospif16Test, Integers) {
-  EXPECT_FP_EQ(1.0f, LIBC_NAMESPACE::cospif16(-0x420));
-  EXPECT_FP_EQ(1.0f, LIBC_NAMESPACE::cospif16(-0x1.4p+14));
-  EXPECT_FP_EQ(-1.0f, LIBC_NAMESPACE::cospif16(0x421));
-  EXPECT_FP_EQ(-1.0f, LIBC_NAMESPACE::cospif16(0x333));
-  EXPECT_FP_EQ(zero, LIBC_NAMESPACE::cospif16(-0x1.28p4));
-  EXPECT_FP_EQ(zero, LIBC_NAMESPACE::cospif16(-0x1.ffcp9));
-  EXPECT_FP_EQ(zero, LIBC_NAMESPACE::cospif16(0x1.01p7));
-  EXPECT_FP_EQ(zero, LIBC_NAMESPACE::cospif16(0x1.f6cp9));
+  EXPECT_FP_EQ(FPBits::one().get_val(),
+               LIBC_NAMESPACE::cospif16(
+                   LIBC_NAMESPACE::fputil::cast<float16>(-0x420.0p0)));
+  EXPECT_FP_EQ(FPBits::one().get_val(),
+               LIBC_NAMESPACE::cospif16(
+                   LIBC_NAMESPACE::fputil::cast<float16>(-0x1.4p+14)));
+  EXPECT_FP_EQ(FPBits::one(Sign::NEG).get_val(),
+               LIBC_NAMESPACE::cospif16(
+                   LIBC_NAMESPACE::fputil::cast<float16>(0x421.0p0)));
+  EXPECT_FP_EQ(FPBits::one(Sign::NEG).get_val(),
+               LIBC_NAMESPACE::cospif16(
+                   LIBC_NAMESPACE::fputil::cast<float16>(0x333.0p0)));
+  EXPECT_FP_EQ(zero, LIBC_NAMESPACE::cospif16(
+                         LIBC_NAMESPACE::fputil::cast<float16>(-0x1.28p4)));
+  EXPECT_FP_EQ(zero, LIBC_NAMESPACE::cospif16(
+                         LIBC_NAMESPACE::fputil::cast<float16>(-0x1.ffcp9)));
+  EXPECT_FP_EQ(zero, LIBC_NAMESPACE::cospif16(
+                         LIBC_NAMESPACE::fputil::cast<float16>(0x1.01p7)));
+  EXPECT_FP_EQ(zero, LIBC_NAMESPACE::cospif16(
+                         LIBC_NAMESPACE::fputil::cast<float16>(0x1.f6cp9)));
 }
diff --git a/libc/test/src/math/smoke/exp2m1f16_test.cpp b/libc/test/src/math/smoke/exp2m1f16_test.cpp
index 8be86973f3213..f423196a70360 100644
--- a/libc/test/src/math/smoke/exp2m1f16_test.cpp
+++ b/libc/test/src/math/smoke/exp2m1f16_test.cpp
@@ -7,6 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "hdr/fenv_macros.h"
+#include "src/__support/FPUtil/cast.h"
 #include "src/errno/libc_errno.h"
 #include "src/math/exp2m1f16.h"
 #include "test/UnitTest/FPMatcher.h"
@@ -44,7 +45,7 @@ TEST_F(LlvmLibcExp2m1f16Test, Overflow) {
                               FE_OVERFLOW | FE_INEXACT);
   EXPECT_MATH_ERRNO(ERANGE);
 
-  float16 x = 16.0;
+  float16 x = LIBC_NAMESPACE::fputil::cast<float16>(16.0);
 
   EXPECT_FP_EQ_WITH_EXCEPTION_ROUNDING_NEAREST(
       inf, LIBC_NAMESPACE::exp2m1f16(x), FE_OVERFLOW | FE_INEXACT);
@@ -69,9 +70,11 @@ TEST_F(LlvmLibcExp2m1f16Test, ResultNearNegOne) {
   EXPECT_FP_EQ_WITH_EXCEPTION(-1.0, LIBC_NAMESPACE::exp2m1f16(neg_max_normal),
                               FE_INEXACT);
 
-  EXPECT_FP_EQ_ALL_ROUNDING(-0x1.ffcp-1, LIBC_NAMESPACE::exp2m1f16(-11.0));
+  EXPECT_FP_EQ_ALL_ROUNDING(
+      -0x1.ffcp-1,
+      LIBC_NAMESPACE::exp2m1f16(LIBC_NAMESPACE::fputil::cast<float16>(-11.0)));
 
-  float16 x = -12.0;
+  float16 x = LIBC_NAMESPACE::fputil::cast<float16>(-12.0);
 
   EXPECT_FP_EQ_WITH_EXCEPTION_ROUNDING_NEAREST(
       -1.0, LIBC_NAMESPACE::exp2m1f16(x), FE_INEXACT);
diff --git a/libc/test/src/math/smoke/sinpif16_test.cpp b/libc/test/src/math/smoke/sinpif16_test.cpp
index 0bcd38a60d849..a79fd5281ee68 100644
--- a/libc/test/src/math/smoke/sinpif16_test.cpp
+++ b/libc/test/src/math/smoke/sinpif16_test.cpp
@@ -6,6 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
+#include "src/__support/FPUtil/cast.h"
 #include "src/errno/libc_errno.h"
 #include "src/math/sinpif16.h"
 #include "test/UnitTest/FPMatcher.h"
@@ -33,10 +34,18 @@ TEST_F(LlvmLibcSinpif16Test, SpecialNumbers) {
 }
 
 TEST_F(LlvmLibcSinpif16Test, Integers) {
-  EXPECT_FP_EQ(neg_zero, LIBC_NAMESPACE::sinpif16(-0x420));
-  EXPECT_FP_EQ(neg_zero, LIBC_NAMESPACE::sinpif16(-0x1p+10));
-  EXPECT_FP_EQ(neg_zero, LIBC_NAMESPACE::sinpif16(-0x1.4p+14));
-  EXPECT_FP_EQ(zero, LIBC_NAMESPACE::sinpif16(0x420));
-  EXPECT_FP_EQ(zero, LIBC_NAMESPACE::sinpif16(0x1.cp+15));
-  EXPECT_FP_EQ(zero, LIBC_NAMESPACE::sinpif16(0x1.cp+7));
+  EXPECT_FP_EQ(neg_zero,
+               LIBC_NAMESPACE::sinpif16(
+                   LIBC_NAMESPACE::fputil::cast<float16>(-0x420.0p0)));
+  EXPECT_FP_EQ(neg_zero, LIBC_NAMESPACE::sinpif16(
+                             LIBC_NAMESPACE::fputil::cast<float16>(-0x1p+10)));
+  EXPECT_FP_EQ(neg_zero,
+               LIBC_NAMESPACE::sinpif16(
+                   LIBC_NAMESPACE::fputil::cast<float16>(-0x1.4p+14)));
+  EXPECT_FP_EQ(zero, LIBC_NAMESPACE::sinpif16(
+                         LIBC_NAMESPACE::fputil::cast<float16>(0x420.0p0)));
+  EXPECT_FP_EQ(zero, LIBC_NAMESPACE::sinpif16(
+                         LIBC_NAMESPACE::fputil::cast<float16>(0x1.cp+15)));
+  EXPECT_FP_EQ(zero, LIBC_NAMESPACE::sinpif16(
+                         LIBC_NAMESPACE::fputil::cast<float16>(0x1.cp+7)));
 }
diff --git a/libc/test/src/poll/CMakeLists.txt b/libc/test/src/poll/CMakeLists.txt
new file mode 100644
index 0000000000000..c4af14168b906
--- /dev/null
+++ b/libc/test/src/poll/CMakeLists.txt
@@ -0,0 +1,14 @@
+add_custom_target(libc_poll_unittests)
+
+add_libc_unittest(
+  poll_test
+  SUITE
+    libc_poll_unittests
+  SRCS
+    poll_test.cpp
+  DEPENDS
+    libc.hdr.limits_macros
+    libc.src.errno.errno
+    libc.src.poll.poll
+    libc.test.UnitTest.ErrnoSetterMatcher
+)
diff --git a/libc/test/src/poll/poll_test.cpp b/libc/test/src/poll/poll_test.cpp
new file mode 100644
index 0000000000000..30f5e41c61ecf
--- /dev/null
+++ b/libc/test/src/poll/poll_test.cpp
@@ -0,0 +1,25 @@
+//===-- Unittests for poll ------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "hdr/limits_macros.h" // UINT_MAX
+#include "src/errno/libc_errno.h"
+#include "src/poll/poll.h"
+#include "test/UnitTest/Test.h"
+
+TEST(LlvmLibcPollTest, SmokeTest) {
+  LIBC_NAMESPACE::libc_errno = 0;
+  int ret = LIBC_NAMESPACE::poll(nullptr, 0, 0);
+  ASSERT_ERRNO_SUCCESS();
+  ASSERT_EQ(0, ret);
+}
+TEST(LlvmLibcPollTest, SmokeFailureTest) {
+  LIBC_NAMESPACE::libc_errno = 0;
+  int ret = LIBC_NAMESPACE::poll(nullptr, UINT_MAX, 0);
+  ASSERT_ERRNO_EQ(EINVAL);
+  ASSERT_EQ(-1, ret);
+}
diff --git a/libc/test/src/setjmp/CMakeLists.txt b/libc/test/src/setjmp/CMakeLists.txt
index 049df89ba39a6..392230784bd99 100644
--- a/libc/test/src/setjmp/CMakeLists.txt
+++ b/libc/test/src/setjmp/CMakeLists.txt
@@ -11,6 +11,8 @@ add_libc_unittest(
     libc_setjmp_unittests
   SRCS
     setjmp_test.cpp
+  CXX_STANDARD
+    20
   DEPENDS
     libc.src.setjmp.longjmp
     libc.src.setjmp.setjmp
diff --git a/libc/test/src/setjmp/setjmp_test.cpp b/libc/test/src/setjmp/setjmp_test.cpp
index 9e5f74a1734b3..27113cd6e0631 100644
--- a/libc/test/src/setjmp/setjmp_test.cpp
+++ b/libc/test/src/setjmp/setjmp_test.cpp
@@ -27,7 +27,7 @@ TEST(LlvmLibcSetJmpTest, SetAndJumpBack) {
   // The first time setjmp is called, it should return 0.
   // Subsequent calls will return the value passed to jump_back below.
   if (LIBC_NAMESPACE::setjmp(buf) <= MAX_LOOP) {
-    ++n;
+    n = n + 1;
     jump_back(buf, n);
   }
   ASSERT_EQ(longjmp_called, n);
diff --git a/libc/test/src/signal/CMakeLists.txt b/libc/test/src/signal/CMakeLists.txt
index a27f5b8f1000e..f86ce2ae96857 100644
--- a/libc/test/src/signal/CMakeLists.txt
+++ b/libc/test/src/signal/CMakeLists.txt
@@ -74,7 +74,6 @@ add_libc_unittest(
   SRCS
     signal_test.cpp
   DEPENDS
-    libc.hdr.types.sighandler_t
     libc.src.errno.errno
     libc.src.signal.raise
     libc.src.signal.signal
diff --git a/libc/test/src/signal/signal_test.cpp b/libc/test/src/signal/signal_test.cpp
index 4b57311eee2d8..bac9c3b8b68bb 100644
--- a/libc/test/src/signal/signal_test.cpp
+++ b/libc/test/src/signal/signal_test.cpp
@@ -13,14 +13,12 @@
 #include "test/UnitTest/ErrnoSetterMatcher.h"
 #include "test/UnitTest/Test.h"
 
-#include "hdr/types/sighandler_t.h"
-
 using LIBC_NAMESPACE::testing::ErrnoSetterMatcher::Fails;
 using LIBC_NAMESPACE::testing::ErrnoSetterMatcher::Succeeds;
 
 TEST(LlvmLibcSignal, Invalid) {
   LIBC_NAMESPACE::libc_errno = 0;
-  sighandler_t valid = +[](int) {};
+  auto *valid = +[](int) {};
   EXPECT_THAT((void *)LIBC_NAMESPACE::signal(0, valid),
               Fails(EINVAL, (void *)SIG_ERR));
   EXPECT_THAT((void *)LIBC_NAMESPACE::signal(65, valid),
diff --git a/libc/test/src/stdbit/stdc_bit_ceil_uc_test.cpp b/libc/test/src/stdbit/stdc_bit_ceil_uc_test.cpp
index 1ef87b0d44de6..6915859b7c669 100644
--- a/libc/test/src/stdbit/stdc_bit_ceil_uc_test.cpp
+++ b/libc/test/src/stdbit/stdc_bit_ceil_uc_test.cpp
@@ -17,18 +17,21 @@ TEST(LlvmLibcStdcBitceilUcTest, Zero) {
 
 TEST(LlvmLibcStdcBitceilUcTest, Ones) {
   for (unsigned i = 0U; i != UCHAR_WIDTH; ++i)
-    EXPECT_EQ(LIBC_NAMESPACE::stdc_bit_ceil_uc(1U << i),
-              static_cast<unsigned char>(1U << i));
+    EXPECT_EQ(
+        LIBC_NAMESPACE::stdc_bit_ceil_uc(static_cast<unsigned char>(1U << i)),
+        static_cast<unsigned char>(1U << i));
 }
 
 TEST(LlvmLibcStdcBitceilUcTest, OneLessThanPowsTwo) {
   for (unsigned i = 2U; i != UCHAR_WIDTH; ++i)
-    EXPECT_EQ(LIBC_NAMESPACE::stdc_bit_ceil_uc((1U << i) - 1),
+    EXPECT_EQ(LIBC_NAMESPACE::stdc_bit_ceil_uc(
+                  static_cast<unsigned char>((1U << i) - 1)),
               static_cast<unsigned char>(1U << i));
 }
 
 TEST(LlvmLibcStdcBitceilUcTest, OneMoreThanPowsTwo) {
   for (unsigned i = 1U; i != UCHAR_WIDTH - 1; ++i)
-    EXPECT_EQ(LIBC_NAMESPACE::stdc_bit_ceil_uc((1U << i) + 1),
+    EXPECT_EQ(LIBC_NAMESPACE::stdc_bit_ceil_uc(
+                  static_cast<unsigned char>((1U << i) + 1)),
               static_cast<unsigned char>(1U << (i + 1)));
 }
diff --git a/libc/test/src/stdbit/stdc_bit_ceil_us_test.cpp b/libc/test/src/stdbit/stdc_bit_ceil_us_test.cpp
index 56873c51828f1..9a8b46f250f48 100644
--- a/libc/test/src/stdbit/stdc_bit_ceil_us_test.cpp
+++ b/libc/test/src/stdbit/stdc_bit_ceil_us_test.cpp
@@ -17,18 +17,21 @@ TEST(LlvmLibcStdcBitceilUsTest, Zero) {
 
 TEST(LlvmLibcStdcBitceilUsTest, Ones) {
   for (unsigned i = 0U; i != USHRT_WIDTH; ++i)
-    EXPECT_EQ(LIBC_NAMESPACE::stdc_bit_ceil_us(1U << i),
-              static_cast<unsigned short>(1U << i));
+    EXPECT_EQ(
+        LIBC_NAMESPACE::stdc_bit_ceil_us(static_cast<unsigned short>(1U << i)),
+        static_cast<unsigned short>(1U << i));
 }
 
 TEST(LlvmLibcStdcBitceilUsTest, OneLessThanPowsTwo) {
   for (unsigned i = 2U; i != USHRT_WIDTH; ++i)
-    EXPECT_EQ(LIBC_NAMESPACE::stdc_bit_ceil_us((1U << i) - 1),
+    EXPECT_EQ(LIBC_NAMESPACE::stdc_bit_ceil_us(
+                  static_cast<unsigned short>((1U << i) - 1)),
               static_cast<unsigned short>(1U << i));
 }
 
 TEST(LlvmLibcStdcBitceilUsTest, OneMoreThanPowsTwo) {
   for (unsigned i = 1U; i != USHRT_WIDTH - 1; ++i)
-    EXPECT_EQ(LIBC_NAMESPACE::stdc_bit_ceil_us((1U << i) + 1),
+    EXPECT_EQ(LIBC_NAMESPACE::stdc_bit_ceil_us(
+                  static_cast<unsigned short>((1U << i) + 1)),
               static_cast<unsigned short>(1U << (i + 1)));
 }
diff --git a/libc/test/src/stdbit/stdc_first_leading_one_uc_test.cpp b/libc/test/src/stdbit/stdc_first_leading_one_uc_test.cpp
index b8c8db587098e..2ab8397015288 100644
--- a/libc/test/src/stdbit/stdc_first_leading_one_uc_test.cpp
+++ b/libc/test/src/stdbit/stdc_first_leading_one_uc_test.cpp
@@ -16,6 +16,7 @@ TEST(LlvmLibcStdcFirstLeadingOneUcTest, Zero) {
 
 TEST(LlvmLibcStdcFirstLeadingOneUcTest, OneHot) {
   for (unsigned i = 0U; i != UCHAR_WIDTH; ++i)
-    EXPECT_EQ(LIBC_NAMESPACE::stdc_first_leading_one_uc(1U << i),
+    EXPECT_EQ(LIBC_NAMESPACE::stdc_first_leading_one_uc(
+                  static_cast<unsigned char>(1U << i)),
               UCHAR_WIDTH - i);
 }
diff --git a/libc/test/src/stdbit/stdc_first_leading_one_us_test.cpp b/libc/test/src/stdbit/stdc_first_leading_one_us_test.cpp
index e9488335d9b00..de81275205424 100644
--- a/libc/test/src/stdbit/stdc_first_leading_one_us_test.cpp
+++ b/libc/test/src/stdbit/stdc_first_leading_one_us_test.cpp
@@ -16,6 +16,7 @@ TEST(LlvmLibcStdcFirstLeadingOneUsTest, Zero) {
 
 TEST(LlvmLibcStdcFirstLeadingOneUsTest, OneHot) {
   for (unsigned i = 0U; i != USHRT_WIDTH; ++i)
-    EXPECT_EQ(LIBC_NAMESPACE::stdc_first_leading_one_us(1U << i),
+    EXPECT_EQ(LIBC_NAMESPACE::stdc_first_leading_one_us(
+                  static_cast<unsigned short>(1U << i)),
               USHRT_WIDTH - i);
 }
diff --git a/libc/test/src/stdbit/stdc_first_leading_zero_uc_test.cpp b/libc/test/src/stdbit/stdc_first_leading_zero_uc_test.cpp
index ac7e8c7d9e64b..a19d0ab83a9bd 100644
--- a/libc/test/src/stdbit/stdc_first_leading_zero_uc_test.cpp
+++ b/libc/test/src/stdbit/stdc_first_leading_zero_uc_test.cpp
@@ -16,6 +16,7 @@ TEST(LlvmLibcStdcFirstLeadingZeroUcTest, ALL) {
 
 TEST(LlvmLibcStdcFirstLeadingZeroUcTest, ZeroHot) {
   for (unsigned i = 0U; i != UCHAR_WIDTH; ++i)
-    EXPECT_EQ(LIBC_NAMESPACE::stdc_first_leading_zero_uc(~(1U << i)),
+    EXPECT_EQ(LIBC_NAMESPACE::stdc_first_leading_zero_uc(
+                  static_cast<unsigned char>(~(1U << i))),
               UCHAR_WIDTH - i);
 }
diff --git a/libc/test/src/stdbit/stdc_first_leading_zero_us_test.cpp b/libc/test/src/stdbit/stdc_first_leading_zero_us_test.cpp
index 37f8612675a7a..2971267f82a6f 100644
--- a/libc/test/src/stdbit/stdc_first_leading_zero_us_test.cpp
+++ b/libc/test/src/stdbit/stdc_first_leading_zero_us_test.cpp
@@ -16,6 +16,7 @@ TEST(LlvmLibcStdcFirstLeadingZeroUsTest, ALL) {
 
 TEST(LlvmLibcStdcFirstLeadingZeroUsTest, ZeroHot) {
   for (unsigned i = 0U; i != USHRT_WIDTH; ++i)
-    EXPECT_EQ(LIBC_NAMESPACE::stdc_first_leading_zero_us(~(1U << i)),
+    EXPECT_EQ(LIBC_NAMESPACE::stdc_first_leading_zero_us(
+                  static_cast<unsigned short>(~(1U << i))),
               USHRT_WIDTH - i);
 }
diff --git a/libc/test/src/stdbit/stdc_first_trailing_one_uc_test.cpp b/libc/test/src/stdbit/stdc_first_trailing_one_uc_test.cpp
index ed2b4921cdada..5ca4cfca1b479 100644
--- a/libc/test/src/stdbit/stdc_first_trailing_one_uc_test.cpp
+++ b/libc/test/src/stdbit/stdc_first_trailing_one_uc_test.cpp
@@ -16,5 +16,7 @@ TEST(LlvmLibcStdcFirstTrailingOneUcTest, ALL) {
 
 TEST(LlvmLibcStdcFirstTrailingOneUcTest, OneHot) {
   for (unsigned i = 0U; i != UCHAR_WIDTH; ++i)
-    EXPECT_EQ(LIBC_NAMESPACE::stdc_first_trailing_one_uc(1U << i), i + 1);
+    EXPECT_EQ(LIBC_NAMESPACE::stdc_first_trailing_one_uc(
+                  static_cast<unsigned char>(1U << i)),
+              i + 1);
 }
diff --git a/libc/test/src/stdbit/stdc_first_trailing_one_us_test.cpp b/libc/test/src/stdbit/stdc_first_trailing_one_us_test.cpp
index 60021552310be..46c69acfb7f07 100644
--- a/libc/test/src/stdbit/stdc_first_trailing_one_us_test.cpp
+++ b/libc/test/src/stdbit/stdc_first_trailing_one_us_test.cpp
@@ -16,5 +16,7 @@ TEST(LlvmLibcStdcFirstTrailingOneUsTest, ALL) {
 
 TEST(LlvmLibcStdcFirstTrailingOneUsTest, OneHot) {
   for (unsigned i = 0U; i != USHRT_WIDTH; ++i)
-    EXPECT_EQ(LIBC_NAMESPACE::stdc_first_trailing_one_us(1U << i), i + 1);
+    EXPECT_EQ(LIBC_NAMESPACE::stdc_first_trailing_one_us(
+                  static_cast<unsigned short>(1U << i)),
+              i + 1);
 }
diff --git a/libc/test/src/stdbit/stdc_first_trailing_zero_uc_test.cpp b/libc/test/src/stdbit/stdc_first_trailing_zero_uc_test.cpp
index 2b17aa6536e66..9535ad9ffa3a2 100644
--- a/libc/test/src/stdbit/stdc_first_trailing_zero_uc_test.cpp
+++ b/libc/test/src/stdbit/stdc_first_trailing_zero_uc_test.cpp
@@ -16,5 +16,7 @@ TEST(LlvmLibcStdcFirstTrailingZeroUcTest, ALL) {
 
 TEST(LlvmLibcStdcFirstTrailingZeroUcTest, ZeroHot) {
   for (unsigned i = 0U; i != UCHAR_WIDTH; ++i)
-    EXPECT_EQ(LIBC_NAMESPACE::stdc_first_trailing_zero_uc(~(1U << i)), i + 1);
+    EXPECT_EQ(LIBC_NAMESPACE::stdc_first_trailing_zero_uc(
+                  static_cast<unsigned char>(~(1U << i))),
+              i + 1);
 }
diff --git a/libc/test/src/stdbit/stdc_first_trailing_zero_us_test.cpp b/libc/test/src/stdbit/stdc_first_trailing_zero_us_test.cpp
index e370379300e4a..e0dc34fd89996 100644
--- a/libc/test/src/stdbit/stdc_first_trailing_zero_us_test.cpp
+++ b/libc/test/src/stdbit/stdc_first_trailing_zero_us_test.cpp
@@ -16,5 +16,7 @@ TEST(LlvmLibcStdcFirstTrailingZeroUsTest, ALL) {
 
 TEST(LlvmLibcStdcFirstTrailingZeroUsTest, ZeroHot) {
   for (unsigned i = 0U; i != USHRT_WIDTH; ++i)
-    EXPECT_EQ(LIBC_NAMESPACE::stdc_first_trailing_zero_us(~(1U << i)), i + 1);
+    EXPECT_EQ(LIBC_NAMESPACE::stdc_first_trailing_zero_us(
+                  static_cast<unsigned short>(~(1U << i))),
+              i + 1);
 }
diff --git a/libc/test/src/stdbit/stdc_has_single_bit_uc_test.cpp b/libc/test/src/stdbit/stdc_has_single_bit_uc_test.cpp
index 1bc189cf0b665..9dd2bdc12479d 100644
--- a/libc/test/src/stdbit/stdc_has_single_bit_uc_test.cpp
+++ b/libc/test/src/stdbit/stdc_has_single_bit_uc_test.cpp
@@ -16,5 +16,7 @@ TEST(LlvmLibcStdcHasSingleBitUcTest, Zero) {
 
 TEST(LlvmLibcStdcHasSingleBitUcTest, OneHot) {
   for (unsigned i = 0U; i != UCHAR_WIDTH; ++i)
-    EXPECT_EQ(LIBC_NAMESPACE::stdc_has_single_bit_uc(1U << i), true);
+    EXPECT_EQ(LIBC_NAMESPACE::stdc_has_single_bit_uc(
+                  static_cast<unsigned char>(1U << i)),
+              true);
 }
diff --git a/libc/test/src/stdbit/stdc_has_single_bit_us_test.cpp b/libc/test/src/stdbit/stdc_has_single_bit_us_test.cpp
index a038f6fac0123..3ff0b83751ebf 100644
--- a/libc/test/src/stdbit/stdc_has_single_bit_us_test.cpp
+++ b/libc/test/src/stdbit/stdc_has_single_bit_us_test.cpp
@@ -16,5 +16,7 @@ TEST(LlvmLibcStdcHasSingleBitUsTest, Zero) {
 
 TEST(LlvmLibcStdcHasSingleBitUsTest, OneHot) {
   for (unsigned i = 0U; i != USHRT_WIDTH; ++i)
-    EXPECT_EQ(LIBC_NAMESPACE::stdc_has_single_bit_us(1U << i), true);
+    EXPECT_EQ(LIBC_NAMESPACE::stdc_has_single_bit_us(
+                  static_cast<unsigned short>(1U << i)),
+              true);
 }
diff --git a/libc/test/src/stdbit/stdc_leading_ones_uc_test.cpp b/libc/test/src/stdbit/stdc_leading_ones_uc_test.cpp
index 5d32d92e327a3..4ba240fdafad4 100644
--- a/libc/test/src/stdbit/stdc_leading_ones_uc_test.cpp
+++ b/libc/test/src/stdbit/stdc_leading_ones_uc_test.cpp
@@ -17,6 +17,7 @@ TEST(LlvmLibcStdcLeadingOnesUcTest, All) {
 
 TEST(LlvmLibcStdcLeadingOnesUcTest, ZeroHot) {
   for (unsigned i = 0U; i != UCHAR_WIDTH; ++i)
-    EXPECT_EQ(LIBC_NAMESPACE::stdc_leading_ones_uc(~(1U << i)),
+    EXPECT_EQ(LIBC_NAMESPACE::stdc_leading_ones_uc(
+                  static_cast<unsigned char>(~(1U << i))),
               UCHAR_WIDTH - i - 1U);
 }
diff --git a/libc/test/src/stdbit/stdc_leading_ones_us_test.cpp b/libc/test/src/stdbit/stdc_leading_ones_us_test.cpp
index 91a125370ec15..0f93eed9e10b8 100644
--- a/libc/test/src/stdbit/stdc_leading_ones_us_test.cpp
+++ b/libc/test/src/stdbit/stdc_leading_ones_us_test.cpp
@@ -17,6 +17,7 @@ TEST(LlvmLibcStdcLeadingOnesUsTest, All) {
 
 TEST(LlvmLibcStdcLeadingOnesUsTest, ZeroHot) {
   for (unsigned i = 0U; i != USHRT_WIDTH; ++i)
-    EXPECT_EQ(LIBC_NAMESPACE::stdc_leading_ones_us(~(1U << i)),
+    EXPECT_EQ(LIBC_NAMESPACE::stdc_leading_ones_us(
+                  static_cast<unsigned short>(~(1U << i))),
               USHRT_WIDTH - i - 1U);
 }
diff --git a/libc/test/src/stdbit/stdc_leading_zeros_uc_test.cpp b/libc/test/src/stdbit/stdc_leading_zeros_uc_test.cpp
index 3d555072927ac..42f78c296fe09 100644
--- a/libc/test/src/stdbit/stdc_leading_zeros_uc_test.cpp
+++ b/libc/test/src/stdbit/stdc_leading_zeros_uc_test.cpp
@@ -17,6 +17,7 @@ TEST(LlvmLibcStdcLeadingZerosUcTest, Zero) {
 
 TEST(LlvmLibcStdcLeadingZerosUcTest, OneHot) {
   for (unsigned i = 0U; i != UCHAR_WIDTH; ++i)
-    EXPECT_EQ(LIBC_NAMESPACE::stdc_leading_zeros_uc(1U << i),
+    EXPECT_EQ(LIBC_NAMESPACE::stdc_leading_zeros_uc(
+                  static_cast<unsigned char>(1U << i)),
               UCHAR_WIDTH - i - 1U);
 }
diff --git a/libc/test/src/stdbit/stdc_leading_zeros_us_test.cpp b/libc/test/src/stdbit/stdc_leading_zeros_us_test.cpp
index afb418a24ad54..967ceb13ff1d7 100644
--- a/libc/test/src/stdbit/stdc_leading_zeros_us_test.cpp
+++ b/libc/test/src/stdbit/stdc_leading_zeros_us_test.cpp
@@ -17,6 +17,7 @@ TEST(LlvmLibcStdcLeadingZerosUsTest, Zero) {
 
 TEST(LlvmLibcStdcLeadingZerosUsTest, OneHot) {
   for (unsigned i = 0U; i != USHRT_WIDTH; ++i)
-    EXPECT_EQ(LIBC_NAMESPACE::stdc_leading_zeros_us(1U << i),
+    EXPECT_EQ(LIBC_NAMESPACE::stdc_leading_zeros_us(
+                  static_cast<unsigned short>(1U << i)),
               USHRT_WIDTH - i - 1U);
 }
diff --git a/libc/test/src/stdbit/stdc_trailing_ones_uc_test.cpp b/libc/test/src/stdbit/stdc_trailing_ones_uc_test.cpp
index 79d4e5b8b8032..0036408513388 100644
--- a/libc/test/src/stdbit/stdc_trailing_ones_uc_test.cpp
+++ b/libc/test/src/stdbit/stdc_trailing_ones_uc_test.cpp
@@ -17,5 +17,7 @@ TEST(LlvmLibcStdcTrailingOnesUcTest, ALL) {
 
 TEST(LlvmLibcStdcTrailingOnesUcTest, ZeroHot) {
   for (unsigned i = 0U; i != UCHAR_WIDTH; ++i)
-    EXPECT_EQ(LIBC_NAMESPACE::stdc_trailing_ones_uc(~(1U << i)), i);
+    EXPECT_EQ(LIBC_NAMESPACE::stdc_trailing_ones_uc(
+                  static_cast<unsigned char>(~(1U << i))),
+              i);
 }
diff --git a/libc/test/src/stdbit/stdc_trailing_ones_us_test.cpp b/libc/test/src/stdbit/stdc_trailing_ones_us_test.cpp
index 7ab15743ed1e0..5ebacc829c543 100644
--- a/libc/test/src/stdbit/stdc_trailing_ones_us_test.cpp
+++ b/libc/test/src/stdbit/stdc_trailing_ones_us_test.cpp
@@ -17,5 +17,7 @@ TEST(LlvmLibcStdcTrailingOnesUsTest, ALL) {
 
 TEST(LlvmLibcStdcTrailingOnesUsTest, ZeroHot) {
   for (unsigned i = 0U; i != USHRT_WIDTH; ++i)
-    EXPECT_EQ(LIBC_NAMESPACE::stdc_trailing_ones_us(~(1U << i)), i);
+    EXPECT_EQ(LIBC_NAMESPACE::stdc_trailing_ones_us(
+                  static_cast<unsigned short>(~(1U << i))),
+              i);
 }
diff --git a/libc/test/src/stdbit/stdc_trailing_zeros_uc_test.cpp b/libc/test/src/stdbit/stdc_trailing_zeros_uc_test.cpp
index c02b518865d9f..129ab38c45ea8 100644
--- a/libc/test/src/stdbit/stdc_trailing_zeros_uc_test.cpp
+++ b/libc/test/src/stdbit/stdc_trailing_zeros_uc_test.cpp
@@ -17,5 +17,7 @@ TEST(LlvmLibcStdcTrailingZerosUcTest, Zero) {
 
 TEST(LlvmLibcStdcTrailingZerosUcTest, OneHot) {
   for (unsigned i = 0U; i != UCHAR_WIDTH; ++i)
-    EXPECT_EQ(LIBC_NAMESPACE::stdc_trailing_zeros_uc(1U << i), i);
+    EXPECT_EQ(LIBC_NAMESPACE::stdc_trailing_zeros_uc(
+                  static_cast<unsigned char>(1U << i)),
+              i);
 }
diff --git a/libc/test/src/stdbit/stdc_trailing_zeros_us_test.cpp b/libc/test/src/stdbit/stdc_trailing_zeros_us_test.cpp
index a9f8327dfd914..e1171f24ccfda 100644
--- a/libc/test/src/stdbit/stdc_trailing_zeros_us_test.cpp
+++ b/libc/test/src/stdbit/stdc_trailing_zeros_us_test.cpp
@@ -17,5 +17,7 @@ TEST(LlvmLibcStdcTrailingZerosUsTest, Zero) {
 
 TEST(LlvmLibcStdcTrailingZerosUsTest, OneHot) {
   for (unsigned i = 0U; i != USHRT_WIDTH; ++i)
-    EXPECT_EQ(LIBC_NAMESPACE::stdc_trailing_zeros_us(1U << i), i);
+    EXPECT_EQ(LIBC_NAMESPACE::stdc_trailing_zeros_us(
+                  static_cast<unsigned short>(1U << i)),
+              i);
 }
diff --git a/libc/test/src/stdfix/CMakeLists.txt b/libc/test/src/stdfix/CMakeLists.txt
index 90d20438edb4b..e4d4fc5b52558 100644
--- a/libc/test/src/stdfix/CMakeLists.txt
+++ b/libc/test/src/stdfix/CMakeLists.txt
@@ -14,7 +14,7 @@ foreach(suffix IN ITEMS hr r lr hk k lk)
     SRCS
       abs${suffix}_test.cpp
     COMPILE_OPTIONS
-      -O3
+      ${libc_opt_high_flag}
     DEPENDS
       libc.src.stdfix.abs${suffix}
       libc.src.__support.fixed_point.fx_bits
@@ -31,7 +31,7 @@ foreach(suffix IN ITEMS uhr ur ulr uhk uk)
     SRCS
       sqrt${suffix}_test.cpp
     COMPILE_OPTIONS
-      -O3
+      ${libc_opt_high_flag}
     DEPENDS
       libc.src.stdfix.sqrt${suffix}
       libc.src.__support.CPP.bit
@@ -52,7 +52,7 @@ foreach(suffix IN ITEMS hr r lr hk k lk uhr ur ulr uhk uk ulk)
     SRCS
       round${suffix}_test.cpp
     COMPILE_OPTIONS
-      -O3
+      ${libc_opt_high_flag}
     DEPENDS
       libc.src.stdfix.round${suffix}
       libc.src.__support.fixed_point.fx_bits
@@ -67,7 +67,7 @@ foreach(suffix IN ITEMS hr r lr hk k lk uhr ur ulr uhk uk ulk)
     SRCS
       ${suffix}bits_test.cpp
     COMPILE_OPTIONS
-      -O3
+      ${libc_opt_high_flag}
     DEPENDS
       libc.src.stdfix.${suffix}bits
       libc.src.__support.CPP.bit
@@ -84,7 +84,7 @@ add_libc_test(
   SRCS
     uhksqrtus_test.cpp
   COMPILE_OPTIONS
-    -O3
+    ${libc_opt_high_flag}
   DEPENDS
     libc.src.stdfix.uhksqrtus
     libc.src.__support.CPP.bit
@@ -103,7 +103,7 @@ add_libc_test(
   SRCS
     uksqrtui_test.cpp
   COMPILE_OPTIONS
-    -O3
+    ${libc_opt_high_flag}
   DEPENDS
     libc.src.stdfix.uksqrtui
     libc.src.__support.CPP.bit
@@ -122,7 +122,7 @@ add_libc_test(
   SRCS
     exphk_test.cpp
   COMPILE_OPTIONS
-    -O3
+    ${libc_opt_high_flag}
   DEPENDS
     libc.src.stdfix.exphk
     libc.src.math.exp
@@ -140,7 +140,7 @@ add_libc_test(
   SRCS
     expk_test.cpp
   COMPILE_OPTIONS
-    -O3
+    ${libc_opt_high_flag}
   DEPENDS
     libc.src.stdfix.expk
     libc.src.math.exp
diff --git a/libc/test/src/stdio/CMakeLists.txt b/libc/test/src/stdio/CMakeLists.txt
index e17f8d8c101a9..01904a30504ed 100644
--- a/libc/test/src/stdio/CMakeLists.txt
+++ b/libc/test/src/stdio/CMakeLists.txt
@@ -116,7 +116,8 @@ add_libc_test(
 if(LIBC_CONF_PRINTF_DISABLE_FLOAT)
   list(APPEND sprintf_test_copts "-DLIBC_COPT_PRINTF_DISABLE_FLOAT")
 endif()
-if(LIBC_CONF_PRINTF_FLOAT_TO_STR_USE_DYADIC_FLOAT)
+if(LIBC_CONF_PRINTF_FLOAT_TO_STR_USE_DYADIC_FLOAT OR
+    LIBC_CONF_PRINTF_FLOAT_TO_STR_USE_FLOAT320)
   list(APPEND sprintf_test_copts "-DLIBC_COPT_FLOAT_TO_STR_REDUCED_PRECISION")
 endif()
 if(LIBC_CONF_PRINTF_DISABLE_INDEX_MODE)
diff --git a/libc/test/src/stdio/fileop_test.cpp b/libc/test/src/stdio/fileop_test.cpp
index 98ead6edd38b4..c662a9fbd0860 100644
--- a/libc/test/src/stdio/fileop_test.cpp
+++ b/libc/test/src/stdio/fileop_test.cpp
@@ -31,7 +31,7 @@ TEST(LlvmLibcFILETest, SimpleFileOperations) {
   constexpr char FILENAME[] = "testdata/simple_operations.test";
   ::FILE *file = LIBC_NAMESPACE::fopen(FILENAME, "w");
   ASSERT_FALSE(file == nullptr);
-  ASSERT_EQ(LIBC_NAMESPACE::fileno(file), 3);
+  ASSERT_GE(LIBC_NAMESPACE::fileno(file), 0);
   constexpr char CONTENT[] = "1234567890987654321";
   ASSERT_EQ(sizeof(CONTENT) - 1,
             LIBC_NAMESPACE::fwrite(CONTENT, 1, sizeof(CONTENT) - 1, file));
diff --git a/libc/test/src/stdio/sprintf_test.cpp b/libc/test/src/stdio/sprintf_test.cpp
index e8303ff3416b2..f6af6ad3e364b 100644
--- a/libc/test/src/stdio/sprintf_test.cpp
+++ b/libc/test/src/stdio/sprintf_test.cpp
@@ -1861,6 +1861,113 @@ TEST(LlvmLibcSPrintfTest, FloatDecimalLongDoubleConv) {
       "179817332113");
 #endif
 #endif // LIBC_TYPES_LONG_DOUBLE_IS_X86_FLOAT80
+
+#if defined(LIBC_TYPES_LONG_DOUBLE_IS_FLOAT128)
+  // Some exceptionally difficult cases for 39-digit precision. (That's the
+  // number of digits supported by the float320 algorithm, and should still be
+  // correct under other algorithms. So these are still enabled even under
+  // LIBC_COPT_FLOAT_TO_STR_REDUCED_PRECISION.)
+  //
+  // These were found by number-theoretic search to be the worst cases in terms
+  // of being extremely close to the rounding boundary between two possible
+  // decimal outputs. For example, the first of these cases has a true value
+  // beginning with
+  //
+  // 2.245786964418815522831613614422112838795000000000000000000
+  //                                          0000000000000000010767969...
+  //
+  // so you need to compute a _long_ way past the 39th digit to find out
+  // whether to round the ...8795 up to 880 or not!
+  //
+  // The first half of these cases all rounded up; the second half all rounded
+  // down. You can see that in both sections the final decimal digit is
+  // sometimes odd and sometimes even, ruling out the possibility that we're
+  // getting these right by mistakenly assuming them to be _exactly_ on the
+  // boundary.
+
+  written = LIBC_NAMESPACE::sprintf(buff, "%#.39Lg",
+                                    0x1.bde5716bba8d70255b4be10e0a0ap-3388L);
+  ASSERT_STREQ_LEN(written, buff,
+                   "2.24578696441881552283161361442211283880e-1020");
+  written = LIBC_NAMESPACE::sprintf(buff, "%#.39Lg",
+                                    0x1.64b78defc8712684490980d80808p-3391L);
+  ASSERT_STREQ_LEN(written, buff,
+                   "2.24578696441881552283161361442211283880e-1021");
+  written = LIBC_NAMESPACE::sprintf(buff, "%#.39Lg",
+                                    0x1.c0eed9d1ea4b6f215accb15cb42cp-4714L);
+  ASSERT_STREQ_LEN(written, buff,
+                   "1.54362575487963943466308346767014523161e-1419");
+  written = LIBC_NAMESPACE::sprintf(buff, "%#.39Lg",
+                                    0x1.a393eaafc3dbabfcd30442cef525p-12600L);
+  ASSERT_STREQ_LEN(written, buff,
+                   "1.72435694193924008931441874634575361189e-3793");
+  written = LIBC_NAMESPACE::sprintf(buff, "%#.39Lg",
+                                    0x1.b7a5248baf85133c9b7bf3241f75p+11050L);
+  ASSERT_STREQ_LEN(written, buff,
+                   "4.13346579244549095104252956440178208514e+3326");
+  written = LIBC_NAMESPACE::sprintf(buff, "%#.39Lg",
+                                    0x1.81bf6d977f99ff7bd9debdd52815p+1359L);
+  ASSERT_STREQ_LEN(written, buff,
+                   "1.89595297593127274811683259716608232064e+409");
+  written = LIBC_NAMESPACE::sprintf(buff, "%#.39Lg",
+                                    0x1.fcb9cfd65f068b758d30ba19a494p-4451L);
+  ASSERT_STREQ_LEN(written, buff,
+                   "2.59258570015527007681686041122929728653e-1340");
+  written = LIBC_NAMESPACE::sprintf(buff, "%#.39Lg",
+                                    0x1.7d8b5be0c744e89829e48b933b6fp-4448L);
+  ASSERT_STREQ_LEN(written, buff,
+                   "1.55555142009316204609011624673757837192e-1339");
+  written = LIBC_NAMESPACE::sprintf(buff, "%#.39Lg",
+                                    0x1.dbb1f01aef7b93c37ca00217888cp-12205L);
+  ASSERT_STREQ_LEN(written, buff,
+                   "1.57758077908296078543773740016012372216e-3674");
+  written = LIBC_NAMESPACE::sprintf(buff, "%#.39Lg",
+                                    0x1.fb037d72fdf1a8aa8bdfc2586fe3p+2580L);
+  ASSERT_STREQ_LEN(written, buff,
+                   "8.99846610004600287527755301065037553046e+776");
+
+  written = LIBC_NAMESPACE::sprintf(buff, "%#.39Lg",
+                                    0x1.9af8fe5febf751a795292e30335dp+2052L);
+  ASSERT_STREQ_LEN(written, buff,
+                   "8.30087814106622071390265113980150545241e+617");
+  written = LIBC_NAMESPACE::sprintf(buff, "%#.39Lg",
+                                    0x1.48c731e6565f748610edbe8cf5e4p+2049L);
+  ASSERT_STREQ_LEN(written, buff,
+                   "8.30087814106622071390265113980150545241e+616");
+  written = LIBC_NAMESPACE::sprintf(buff, "%#.39Lg",
+                                    0x1.8e78bc76e98998b7bbf6c8f80f2ap-5671L);
+  ASSERT_STREQ_LEN(written, buff,
+                   "1.12473970318904704063044350553302771341e-1707");
+  written = LIBC_NAMESPACE::sprintf(buff, "%#.39Lg",
+                                    0x1.59a832c16a7797aec70196ddf9dbp-16181L);
+  ASSERT_STREQ_LEN(written, buff,
+                   "1.45896738321434823250358135932839533040e-4871");
+  written = LIBC_NAMESPACE::sprintf(buff, "%#.39Lg",
+                                    0x1.14868f0121f946256c01457e617cp-16184L);
+  ASSERT_STREQ_LEN(written, buff,
+                   "1.45896738321434823250358135932839533040e-4872");
+  written = LIBC_NAMESPACE::sprintf(buff, "%#.39Lg",
+                                    0x1.0ca8c4b525b1128506cdc668df43p+11017L);
+  ASSERT_STREQ_LEN(written, buff,
+                   "2.94051951004764266903705588743096762647e+3316");
+  written = LIBC_NAMESPACE::sprintf(buff, "%#.39Lg",
+                                    0x1.fcd40bb49b18aa19d66a3c5572dep+5547L);
+  ASSERT_STREQ_LEN(written, buff,
+                   "1.29335350323078956384272678475060580129e+1670");
+  written = LIBC_NAMESPACE::sprintf(buff, "%#.39Lg",
+                                    0x1.0e5e51510a653e744d6b84efed86p-13613L);
+  ASSERT_STREQ_LEN(written, buff,
+                   "1.26585813611514592337442314391432904094e-4098");
+  written = LIBC_NAMESPACE::sprintf(buff, "%#.39Lg",
+                                    0x1.ff41f8aa97e676e95b1a6a7751fap+16366L);
+  ASSERT_STREQ_LEN(written, buff,
+                   "9.06377119787295161934827646572467920486e+4926");
+  written = LIBC_NAMESPACE::sprintf(buff, "%#.39Lg",
+                                    0x1.4770bf66ccafd7d80ac9bb3dded7p-1843L);
+  ASSERT_STREQ_LEN(written, buff,
+                   "2.03521509642091239545213340619368190283e-555");
+
+#endif // LIBC_TYPES_LONG_DOUBLE_IS_FLOAT128
 }
 
 TEST(LlvmLibcSPrintfTest, FloatExponentConv) {
diff --git a/libc/test/src/stdlib/CMakeLists.txt b/libc/test/src/stdlib/CMakeLists.txt
index aba76833be9d4..e6c8a629c71fa 100644
--- a/libc/test/src/stdlib/CMakeLists.txt
+++ b/libc/test/src/stdlib/CMakeLists.txt
@@ -168,7 +168,8 @@ add_libc_test(
     .strtol_test_support
 )
 
-if(LIBC_CONF_PRINTF_FLOAT_TO_STR_USE_DYADIC_FLOAT)
+if(LIBC_CONF_PRINTF_FLOAT_TO_STR_USE_DYADIC_FLOAT OR
+    LIBC_CONF_PRINTF_FLOAT_TO_STR_USE_FLOAT320)
   list(APPEND strfrom_test_copts "-DLIBC_COPT_FLOAT_TO_STR_REDUCED_PRECISION")
 endif()
 
diff --git a/libc/test/src/sys/uio/CMakeLists.txt b/libc/test/src/sys/uio/CMakeLists.txt
index 45f8d14c16179..7ba02be5d1cc4 100644
--- a/libc/test/src/sys/uio/CMakeLists.txt
+++ b/libc/test/src/sys/uio/CMakeLists.txt
@@ -1,15 +1,36 @@
 add_custom_target(libc_sys_uio_unittests)
+
 add_libc_unittest(
   writev_test
   SUITE
-  libc_sys_uio_unittests
+    libc_sys_uio_unittests
   SRCS
     writev_test.cpp
   DEPENDS
-    libc.src.errno.errno
+    libc.hdr.types.struct_iovec
     libc.src.__support.common
+    libc.src.errno.errno
+    libc.src.fcntl.open
     libc.src.sys.uio.writev
     libc.src.unistd.close
+    libc.src.unistd.unlink
+    libc.test.UnitTest.ErrnoSetterMatcher
+)
+
+add_libc_unittest(
+  readv_test
+  SUITE
+    libc_sys_uio_unittests
+  SRCS
+    readv_test.cpp
+  DEPENDS
+    libc.hdr.types.struct_iovec
+    libc.src.__support.common
+    libc.src.errno.errno
     libc.src.fcntl.open
+    libc.src.sys.uio.readv
+    libc.src.unistd.close
+    libc.src.unistd.unlink
+    libc.src.unistd.write
     libc.test.UnitTest.ErrnoSetterMatcher
 )
diff --git a/libc/test/src/sys/uio/readv_test.cpp b/libc/test/src/sys/uio/readv_test.cpp
new file mode 100644
index 0000000000000..7852b8a4b136d
--- /dev/null
+++ b/libc/test/src/sys/uio/readv_test.cpp
@@ -0,0 +1,43 @@
+//===-- Unittests for readv -----------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "hdr/types/struct_iovec.h"
+#include "src/fcntl/open.h"
+#include "src/sys/uio/readv.h"
+#include "src/unistd/close.h"
+#include "src/unistd/unlink.h"
+#include "src/unistd/write.h"
+#include "test/UnitTest/ErrnoSetterMatcher.h"
+#include "test/UnitTest/Test.h"
+
+using namespace LIBC_NAMESPACE::testing::ErrnoSetterMatcher;
+
+TEST(LlvmLibcSysUioReadvTest, SmokeTest) {
+  const char *filename = "./LlvmLibcSysUioReadvTest";
+  int fd = LIBC_NAMESPACE::open(filename, O_WRONLY | O_CREAT, 0644);
+  ASSERT_THAT(fd, returns(GT(0)).with_errno(EQ(0)));
+  const char data[] = "Hello, World!\n";
+  ASSERT_THAT(LIBC_NAMESPACE::write(fd, data, sizeof(data)),
+              returns(EQ(sizeof(data))).with_errno(EQ(0)));
+  ASSERT_THAT(LIBC_NAMESPACE::close(fd), Succeeds());
+
+  fd = LIBC_NAMESPACE::open(filename, O_RDONLY);
+  ASSERT_THAT(fd, returns(GT(0)).with_errno(EQ(0)));
+  char buf0[2];
+  char buf1[3];
+  struct iovec iov[2];
+  iov[0].iov_base = buf0;
+  iov[0].iov_len = 1;
+  iov[1].iov_base = buf1;
+  iov[1].iov_len = 2;
+  ASSERT_THAT(LIBC_NAMESPACE::readv(fd, iov, 2),
+              returns(EQ(3)).with_errno(EQ(0)));
+  ASSERT_THAT(LIBC_NAMESPACE::close(fd), Succeeds());
+  ASSERT_THAT(LIBC_NAMESPACE::unlink(filename),
+              returns(EQ(0)).with_errno(EQ(0)));
+}
diff --git a/libc/test/src/sys/uio/writev_test.cpp b/libc/test/src/sys/uio/writev_test.cpp
index a9a314813182d..2c1c445cc7868 100644
--- a/libc/test/src/sys/uio/writev_test.cpp
+++ b/libc/test/src/sys/uio/writev_test.cpp
@@ -6,16 +6,19 @@
 //
 //===----------------------------------------------------------------------===//
 
+#include "hdr/types/struct_iovec.h"
 #include "src/fcntl/open.h"
 #include "src/sys/uio/writev.h"
 #include "src/unistd/close.h"
+#include "src/unistd/unlink.h"
 #include "test/UnitTest/ErrnoSetterMatcher.h"
 #include "test/UnitTest/Test.h"
 
 using namespace LIBC_NAMESPACE::testing::ErrnoSetterMatcher;
 
 TEST(LlvmLibcSysUioWritevTest, SmokeTest) {
-  int fd = LIBC_NAMESPACE::open("/dev/null", O_WRONLY);
+  const char *filename = "./LlvmLibcSysUioWritevTest";
+  int fd = LIBC_NAMESPACE::open(filename, O_WRONLY | O_CREAT, 0644);
   ASSERT_THAT(fd, returns(GT(0)).with_errno(EQ(0)));
   const char *data = "Hello, World!\n";
   struct iovec iov[2];
@@ -26,4 +29,6 @@ TEST(LlvmLibcSysUioWritevTest, SmokeTest) {
   ASSERT_THAT(LIBC_NAMESPACE::writev(fd, iov, 2),
               returns(EQ(15)).with_errno(EQ(0)));
   ASSERT_THAT(LIBC_NAMESPACE::close(fd), Succeeds());
+  ASSERT_THAT(LIBC_NAMESPACE::unlink(filename),
+              returns(EQ(0)).with_errno(EQ(0)));
 }
diff --git a/libc/test/src/unistd/CMakeLists.txt b/libc/test/src/unistd/CMakeLists.txt
index b01cce931a1eb..665cb367ba4fd 100644
--- a/libc/test/src/unistd/CMakeLists.txt
+++ b/libc/test/src/unistd/CMakeLists.txt
@@ -287,6 +287,16 @@ add_libc_unittest(
     libc.src.__support.CPP.string_view
 )
 
+add_libc_unittest(
+  setsid_test
+  SUITE
+    libc_unistd_unittests
+  SRCS
+    setsid_test.cpp
+  DEPENDS
+    libc.src.unistd.setsid
+)
+
 add_libc_unittest(
   symlink_test
   SUITE
diff --git a/libc/test/src/unistd/setsid_test.cpp b/libc/test/src/unistd/setsid_test.cpp
new file mode 100644
index 0000000000000..466faa16ea1eb
--- /dev/null
+++ b/libc/test/src/unistd/setsid_test.cpp
@@ -0,0 +1,15 @@
+//===-- Unittests for setsid ----------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/unistd/setsid.h"
+#include "test/UnitTest/Test.h"
+
+TEST(LlvmLibcGetPidTest, SmokeTest) {
+  // setsid always succeeds. So, we just call it as a smoke test.
+  LIBC_NAMESPACE::setsid();
+}
diff --git a/libc/utils/CMakeLists.txt b/libc/utils/CMakeLists.txt
index a33c13a045a8a..b85136f2d420f 100644
--- a/libc/utils/CMakeLists.txt
+++ b/libc/utils/CMakeLists.txt
@@ -2,4 +2,5 @@ add_subdirectory(hdrgen)
 
 if(LLVM_INCLUDE_TESTS)
   add_subdirectory(MPFRWrapper)
+  add_subdirectory(MPCWrapper)
 endif()
diff --git a/libc/utils/MPCWrapper/CMakeLists.txt b/libc/utils/MPCWrapper/CMakeLists.txt
new file mode 100644
index 0000000000000..38e635c553935
--- /dev/null
+++ b/libc/utils/MPCWrapper/CMakeLists.txt
@@ -0,0 +1,32 @@
+if(LIBC_TESTS_CAN_USE_MPC)
+  add_library(libcMPCWrapper STATIC
+    MPCUtils.cpp
+    MPCUtils.h
+  )
+  _get_common_test_compile_options(compile_options "" "")
+  list(REMOVE_ITEM compile_options "-ffreestanding")
+  target_compile_options(libcMPCWrapper PRIVATE -O3 ${compile_options})
+  add_dependencies(
+    libcMPCWrapper
+    libcMPCommon
+    libc.src.__support.CPP.array
+    libc.src.__support.CPP.string
+    libc.src.__support.CPP.stringstream
+    libc.src.__support.CPP.type_traits
+    libc.src.__support.FPUtil.fp_bits
+    libc.src.__support.complex_type
+    LibcTest.unit
+  )
+  if(EXISTS ${LLVM_LIBC_MPFR_INSTALL_PATH})
+    target_include_directories(libcMPCWrapper PUBLIC ${LLVM_LIBC_MPFR_INSTALL_PATH}/include)
+    target_link_directories(libcMPCWrapper PUBLIC ${LLVM_LIBC_MPFR_INSTALL_PATH}/lib)
+  endif()
+  if(EXISTS ${LIBC_MPC_INSTALL_PATH})
+    target_include_directories(libcMPCWrapper PUBLIC ${LIBC_MPC_INSTALL_PATH}/include)
+    target_link_directories(libcMPCWrapper PUBLIC ${LIBC_MPC_INSTALL_PATH}/lib)
+  endif()
+  target_include_directories(libcMPCWrapper PUBLIC ${LIBC_SOURCE_DIR})
+  target_link_libraries(libcMPCWrapper PUBLIC libcMPCommon LibcFPTestHelpers.unit LibcTest.unit mpc)
+elseif(NOT LIBC_TARGET_OS_IS_GPU AND NOT LLVM_LIBC_FULL_BUILD)
+  message(WARNING "Math tests using MPC will be skipped.")
+endif()
diff --git a/libc/utils/MPCWrapper/MPCUtils.cpp b/libc/utils/MPCWrapper/MPCUtils.cpp
new file mode 100644
index 0000000000000..4d5f685a823b0
--- /dev/null
+++ b/libc/utils/MPCWrapper/MPCUtils.cpp
@@ -0,0 +1,344 @@
+//===-- Utils which wrap MPC ----------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "MPCUtils.h"
+
+#include "src/__support/CPP/array.h"
+#include "src/__support/CPP/stringstream.h"
+#include "utils/MPFRWrapper/MPCommon.h"
+
+#include <stdint.h>
+
+#include "mpc.h"
+
+template <typename T> using FPBits = LIBC_NAMESPACE::fputil::FPBits<T>;
+
+namespace LIBC_NAMESPACE_DECL {
+namespace testing {
+namespace mpc {
+
+static inline cpp::string str(RoundingMode mode) {
+  switch (mode) {
+  case RoundingMode::Upward:
+    return "MPFR_RNDU";
+  case RoundingMode::Downward:
+    return "MPFR_RNDD";
+  case RoundingMode::TowardZero:
+    return "MPFR_RNDZ";
+  case RoundingMode::Nearest:
+    return "MPFR_RNDN";
+  }
+}
+
+class MPCNumber {
+private:
+  unsigned int precision;
+  mpc_t value;
+  mpc_rnd_t mpc_rounding;
+
+public:
+  explicit MPCNumber(unsigned int p) : precision(p), mpc_rounding(MPC_RNDNN) {
+    mpc_init2(value, precision);
+  }
+
+  MPCNumber() : precision(256), mpc_rounding(MPC_RNDNN) {
+    mpc_init2(value, 256);
+  }
+
+  MPCNumber(unsigned int p, mpc_rnd_t rnd) : precision(p), mpc_rounding(rnd) {
+    mpc_init2(value, precision);
+  }
+
+  template <typename XType,
+            cpp::enable_if_t<cpp::is_same_v<_Complex float, XType>, bool> = 0>
+  MPCNumber(XType x,
+            unsigned int precision = mpfr::ExtraPrecision<float>::VALUE,
+            RoundingMode rnd = RoundingMode::Nearest)
+      : precision(precision),
+        mpc_rounding(MPC_RND(mpfr::get_mpfr_rounding_mode(rnd),
+                             mpfr::get_mpfr_rounding_mode(rnd))) {
+    mpc_init2(value, precision);
+    Complex<float> x_c = cpp::bit_cast<Complex<float>>(x);
+    mpfr_t real, imag;
+    mpfr_init2(real, precision);
+    mpfr_init2(imag, precision);
+    mpfr_set_flt(real, x_c.real, mpfr::get_mpfr_rounding_mode(rnd));
+    mpfr_set_flt(imag, x_c.imag, mpfr::get_mpfr_rounding_mode(rnd));
+    mpc_set_fr_fr(value, real, imag, mpc_rounding);
+    mpfr_clear(real);
+    mpfr_clear(imag);
+  }
+
+  template <typename XType,
+            cpp::enable_if_t<cpp::is_same_v<_Complex double, XType>, bool> = 0>
+  MPCNumber(XType x,
+            unsigned int precision = mpfr::ExtraPrecision<double>::VALUE,
+            RoundingMode rnd = RoundingMode::Nearest)
+      : precision(precision),
+        mpc_rounding(MPC_RND(mpfr::get_mpfr_rounding_mode(rnd),
+                             mpfr::get_mpfr_rounding_mode(rnd))) {
+    mpc_init2(value, precision);
+    Complex<double> x_c = cpp::bit_cast<Complex<double>>(x);
+    mpc_set_d_d(value, x_c.real, x_c.imag, mpc_rounding);
+  }
+
+  MPCNumber(const MPCNumber &other)
+      : precision(other.precision), mpc_rounding(other.mpc_rounding) {
+    mpc_init2(value, precision);
+    mpc_set(value, other.value, mpc_rounding);
+  }
+
+  ~MPCNumber() { mpc_clear(value); }
+
+  MPCNumber &operator=(const MPCNumber &rhs) {
+    precision = rhs.precision;
+    mpc_rounding = rhs.mpc_rounding;
+    mpc_init2(value, precision);
+    mpc_set(value, rhs.value, mpc_rounding);
+    return *this;
+  }
+
+  void setValue(mpc_t val) const { mpc_set(val, value, mpc_rounding); }
+
+  mpc_t &getValue() { return value; }
+
+  MPCNumber carg() const {
+    mpfr_t res;
+    MPCNumber result(precision, mpc_rounding);
+
+    mpfr_init2(res, precision);
+
+    mpc_arg(res, value, MPC_RND_RE(mpc_rounding));
+    mpc_set_fr(result.value, res, mpc_rounding);
+
+    mpfr_clear(res);
+
+    return result;
+  }
+
+  MPCNumber cproj() const {
+    MPCNumber result(precision, mpc_rounding);
+    mpc_proj(result.value, value, mpc_rounding);
+    return result;
+  }
+};
+
+namespace internal {
+
+template <typename InputType>
+cpp::enable_if_t<cpp::is_complex_v<InputType>, MPCNumber>
+unary_operation(Operation op, InputType input, unsigned int precision,
+                RoundingMode rounding) {
+  MPCNumber mpcInput(input, precision, rounding);
+  switch (op) {
+  case Operation::Carg:
+    return mpcInput.carg();
+  case Operation::Cproj:
+    return mpcInput.cproj();
+  default:
+    __builtin_unreachable();
+  }
+}
+
+template <typename InputType, typename OutputType>
+bool compare_unary_operation_single_output_same_type(Operation op,
+                                                     InputType input,
+                                                     OutputType libc_result,
+                                                     double ulp_tolerance,
+                                                     RoundingMode rounding) {
+
+  unsigned int precision =
+      mpfr::get_precision<make_real_t<InputType>>(ulp_tolerance);
+
+  MPCNumber mpc_result;
+  mpc_result = unary_operation(op, input, precision, rounding);
+
+  mpc_t mpc_result_val;
+  mpc_init2(mpc_result_val, precision);
+  mpc_result.setValue(mpc_result_val);
+
+  mpfr_t real, imag;
+  mpfr_init2(real, precision);
+  mpfr_init2(imag, precision);
+  mpc_real(real, mpc_result_val, mpfr::get_mpfr_rounding_mode(rounding));
+  mpc_imag(imag, mpc_result_val, mpfr::get_mpfr_rounding_mode(rounding));
+
+  mpfr::MPFRNumber mpfr_real(real, precision, rounding);
+  mpfr::MPFRNumber mpfr_imag(imag, precision, rounding);
+
+  double ulp_real = mpfr_real.ulp(
+      (cpp::bit_cast<Complex<make_real_t<InputType>>>(libc_result)).real);
+  double ulp_imag = mpfr_imag.ulp(
+      (cpp::bit_cast<Complex<make_real_t<InputType>>>(libc_result)).imag);
+  mpc_clear(mpc_result_val);
+  mpfr_clear(real);
+  mpfr_clear(imag);
+  return (ulp_real <= ulp_tolerance) && (ulp_imag <= ulp_tolerance);
+}
+
+template bool compare_unary_operation_single_output_same_type(
+    Operation, _Complex float, _Complex float, double, RoundingMode);
+template bool compare_unary_operation_single_output_same_type(
+    Operation, _Complex double, _Complex double, double, RoundingMode);
+
+template <typename InputType, typename OutputType>
+bool compare_unary_operation_single_output_different_type(
+    Operation op, InputType input, OutputType libc_result, double ulp_tolerance,
+    RoundingMode rounding) {
+
+  unsigned int precision =
+      mpfr::get_precision<make_real_t<InputType>>(ulp_tolerance);
+
+  MPCNumber mpc_result;
+  mpc_result = unary_operation(op, input, precision, rounding);
+
+  mpc_t mpc_result_val;
+  mpc_init2(mpc_result_val, precision);
+  mpc_result.setValue(mpc_result_val);
+
+  mpfr_t real;
+  mpfr_init2(real, precision);
+  mpc_real(real, mpc_result_val, mpfr::get_mpfr_rounding_mode(rounding));
+
+  mpfr::MPFRNumber mpfr_real(real, precision, rounding);
+
+  double ulp_real = mpfr_real.ulp(libc_result);
+  mpc_clear(mpc_result_val);
+  mpfr_clear(real);
+  return (ulp_real <= ulp_tolerance);
+}
+
+template bool compare_unary_operation_single_output_different_type(
+    Operation, _Complex float, float, double, RoundingMode);
+template bool compare_unary_operation_single_output_different_type(
+    Operation, _Complex double, double, double, RoundingMode);
+
+template <typename InputType, typename OutputType>
+void explain_unary_operation_single_output_different_type_error(
+    Operation op, InputType input, OutputType libc_result, double ulp_tolerance,
+    RoundingMode rounding) {
+
+  unsigned int precision =
+      mpfr::get_precision<make_real_t<InputType>>(ulp_tolerance);
+
+  MPCNumber mpc_result;
+  mpc_result = unary_operation(op, input, precision, rounding);
+
+  mpc_t mpc_result_val;
+  mpc_init2(mpc_result_val, precision);
+  mpc_result.setValue(mpc_result_val);
+
+  mpfr_t real;
+  mpfr_init2(real, precision);
+  mpc_real(real, mpc_result_val, mpfr::get_mpfr_rounding_mode(rounding));
+
+  mpfr::MPFRNumber mpfr_result(real, precision, rounding);
+  mpfr::MPFRNumber mpfrLibcResult(libc_result, precision, rounding);
+  mpfr::MPFRNumber mpfrInputReal(
+      cpp::bit_cast<Complex<make_real_t<InputType>>>(input).real, precision,
+      rounding);
+  mpfr::MPFRNumber mpfrInputImag(
+      cpp::bit_cast<Complex<make_real_t<InputType>>>(input).imag, precision,
+      rounding);
+
+  cpp::array<char, 2048> msg_buf;
+  cpp::StringStream msg(msg_buf);
+  msg << "Match value not within tolerance value of MPFR result:\n"
+      << "  Input: " << mpfrInputReal.str() << " + " << mpfrInputImag.str()
+      << "i\n"
+      << "  Rounding mode: " << str(rounding) << '\n'
+      << "    Libc: " << mpfrLibcResult.str() << '\n'
+      << "    MPC: " << mpfr_result.str() << '\n'
+      << '\n'
+      << "  ULP error: " << mpfr_result.ulp_as_mpfr_number(libc_result).str()
+      << '\n';
+  tlog << msg.str();
+  mpc_clear(mpc_result_val);
+  mpfr_clear(real);
+}
+
+template void explain_unary_operation_single_output_different_type_error(
+    Operation, _Complex float, float, double, RoundingMode);
+template void explain_unary_operation_single_output_different_type_error(
+    Operation, _Complex double, double, double, RoundingMode);
+
+template <typename InputType, typename OutputType>
+void explain_unary_operation_single_output_same_type_error(
+    Operation op, InputType input, OutputType libc_result, double ulp_tolerance,
+    RoundingMode rounding) {
+
+  unsigned int precision =
+      mpfr::get_precision<make_real_t<InputType>>(ulp_tolerance);
+
+  MPCNumber mpc_result;
+  mpc_result = unary_operation(op, input, precision, rounding);
+
+  mpc_t mpc_result_val;
+  mpc_init2(mpc_result_val, precision);
+  mpc_result.setValue(mpc_result_val);
+
+  mpfr_t real, imag;
+  mpfr_init2(real, precision);
+  mpfr_init2(imag, precision);
+  mpc_real(real, mpc_result_val, mpfr::get_mpfr_rounding_mode(rounding));
+  mpc_imag(imag, mpc_result_val, mpfr::get_mpfr_rounding_mode(rounding));
+
+  mpfr::MPFRNumber mpfr_real(real, precision, rounding);
+  mpfr::MPFRNumber mpfr_imag(imag, precision, rounding);
+  mpfr::MPFRNumber mpfrLibcResultReal(
+      cpp::bit_cast<Complex<make_real_t<InputType>>>(libc_result).real,
+      precision, rounding);
+  mpfr::MPFRNumber mpfrLibcResultImag(
+      cpp::bit_cast<Complex<make_real_t<InputType>>>(libc_result).imag,
+      precision, rounding);
+  mpfr::MPFRNumber mpfrInputReal(
+      cpp::bit_cast<Complex<make_real_t<InputType>>>(input).real, precision,
+      rounding);
+  mpfr::MPFRNumber mpfrInputImag(
+      cpp::bit_cast<Complex<make_real_t<InputType>>>(input).imag, precision,
+      rounding);
+
+  cpp::array<char, 2048> msg_buf;
+  cpp::StringStream msg(msg_buf);
+  msg << "Match value not within tolerance value of MPFR result:\n"
+      << "  Input: " << mpfrInputReal.str() << " + " << mpfrInputImag.str()
+      << "i\n"
+      << "  Rounding mode: " << str(rounding) << " , " << str(rounding) << '\n'
+      << "    Libc: " << mpfrLibcResultReal.str() << " + "
+      << mpfrLibcResultImag.str() << "i\n"
+      << "    MPC: " << mpfr_real.str() << " + " << mpfr_imag.str() << "i\n"
+      << '\n'
+      << "  ULP error: "
+      << mpfr_real
+             .ulp_as_mpfr_number(
+                 cpp::bit_cast<Complex<make_real_t<InputType>>>(libc_result)
+                     .real)
+             .str()
+      << " , "
+      << mpfr_imag
+             .ulp_as_mpfr_number(
+                 cpp::bit_cast<Complex<make_real_t<InputType>>>(libc_result)
+                     .imag)
+             .str()
+      << '\n';
+  tlog << msg.str();
+  mpc_clear(mpc_result_val);
+  mpfr_clear(real);
+  mpfr_clear(imag);
+}
+
+template void explain_unary_operation_single_output_same_type_error(
+    Operation, _Complex float, _Complex float, double, RoundingMode);
+template void explain_unary_operation_single_output_same_type_error(
+    Operation, _Complex double, _Complex double, double, RoundingMode);
+
+} // namespace internal
+
+} // namespace mpc
+} // namespace testing
+} // namespace LIBC_NAMESPACE_DECL
diff --git a/libc/utils/MPCWrapper/MPCUtils.h b/libc/utils/MPCWrapper/MPCUtils.h
new file mode 100644
index 0000000000000..d141b4d986920
--- /dev/null
+++ b/libc/utils/MPCWrapper/MPCUtils.h
@@ -0,0 +1,270 @@
+//===-- MPCUtils.h ----------------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_UTILS_MPCWRAPPER_MPCUTILS_H
+#define LLVM_LIBC_UTILS_MPCWRAPPER_MPCUTILS_H
+
+#include "src/__support/CPP/type_traits.h"
+#include "src/__support/complex_type.h"
+#include "src/__support/macros/config.h"
+#include "src/__support/macros/properties/complex_types.h"
+#include "src/__support/macros/properties/types.h"
+#include "test/UnitTest/RoundingModeUtils.h"
+#include "test/UnitTest/Test.h"
+
+#include <stdint.h>
+
+namespace LIBC_NAMESPACE_DECL {
+namespace testing {
+namespace mpc {
+
+enum class Operation {
+  // Operations which take a single complex floating point number as input
+  // and produce a single floating point number as output which has the same
+  // floating point type as the real/imaginary part of the input.
+  BeginUnaryOperationsSingleOutputDifferentOutputType,
+  Carg,
+  Cabs,
+  EndUnaryOperationsSingleOutputDifferentOutputType,
+
+  // Operations which take a single complex floating point number as input
+  // and produce a single complex floating point number of the same kind
+  // as output.
+  BeginUnaryOperationsSingleOutputSameOutputType,
+  Cproj,
+  Csqrt,
+  Clog,
+  Cexp,
+  Csinh,
+  Ccosh,
+  Ctanh,
+  Casinh,
+  Cacosh,
+  Catanh,
+  Csin,
+  Ccos,
+  Ctan,
+  Casin,
+  Cacos,
+  Catan,
+  EndUnaryOperationsSingleOutputSameOutputType,
+
+  // Operations which take two complex floating point numbers as input
+  // and produce a single complex floating point number of the same kind
+  // as output.
+  BeginBinaryOperationsSingleOutput,
+  Cpow,
+  EndBinaryOperationsSingleOutput,
+};
+
+using LIBC_NAMESPACE::fputil::testing::RoundingMode;
+
+template <typename T> struct BinaryInput {
+  static_assert(LIBC_NAMESPACE::cpp::is_complex_v<T>,
+                "Template parameter of BinaryInput must be a complex floating "
+                "point type.");
+
+  using Type = T;
+  T x, y;
+};
+
+namespace internal {
+
+template <typename InputType, typename OutputType>
+bool compare_unary_operation_single_output_same_type(Operation op,
+                                                     InputType input,
+                                                     OutputType libc_output,
+                                                     double ulp_tolerance,
+                                                     RoundingMode rounding);
+
+template <typename InputType, typename OutputType>
+bool compare_unary_operation_single_output_different_type(
+    Operation op, InputType input, OutputType libc_output, double ulp_tolerance,
+    RoundingMode rounding);
+
+template <typename InputType, typename OutputType>
+bool compare_binary_operation_one_output(Operation op,
+                                         const BinaryInput<InputType> &input,
+                                         OutputType libc_output,
+                                         double ulp_tolerance,
+                                         RoundingMode rounding);
+
+template <typename InputType, typename OutputType>
+void explain_unary_operation_single_output_same_type_error(
+    Operation op, InputType input, OutputType match_value, double ulp_tolerance,
+    RoundingMode rounding);
+
+template <typename InputType, typename OutputType>
+void explain_unary_operation_single_output_different_type_error(
+    Operation op, InputType input, OutputType match_value, double ulp_tolerance,
+    RoundingMode rounding);
+
+template <typename InputType, typename OutputType>
+void explain_binary_operation_one_output_error(
+    Operation op, const BinaryInput<InputType> &input, OutputType match_value,
+    double ulp_tolerance, RoundingMode rounding);
+
+template <Operation op, typename InputType, typename OutputType>
+class MPCMatcher : public testing::Matcher<OutputType> {
+private:
+  InputType input;
+  OutputType match_value;
+  double ulp_tolerance;
+  RoundingMode rounding;
+
+public:
+  MPCMatcher(InputType testInput, double ulp_tolerance, RoundingMode rounding)
+      : input(testInput), ulp_tolerance(ulp_tolerance), rounding(rounding) {}
+
+  bool match(OutputType libcResult) {
+    match_value = libcResult;
+    return match(input, match_value);
+  }
+
+  void explainError() override { // NOLINT
+    explain_error(input, match_value);
+  }
+
+private:
+  template <typename InType, typename OutType>
+  bool match(InType in, OutType out) {
+    if (cpp::is_same_v<InType, OutType>) {
+      return compare_unary_operation_single_output_same_type(
+          op, in, out, ulp_tolerance, rounding);
+    } else {
+      return compare_unary_operation_single_output_different_type(
+          op, in, out, ulp_tolerance, rounding);
+    }
+  }
+
+  template <typename T, typename U>
+  bool match(const BinaryInput<T> &in, U out) {
+    return compare_binary_operation_one_output(op, in, out, ulp_tolerance,
+                                               rounding);
+  }
+
+  template <typename InType, typename OutType>
+  void explain_error(InType in, OutType out) {
+    if (cpp::is_same_v<InType, OutType>) {
+      explain_unary_operation_single_output_same_type_error(
+          op, in, out, ulp_tolerance, rounding);
+    } else {
+      explain_unary_operation_single_output_different_type_error(
+          op, in, out, ulp_tolerance, rounding);
+    }
+  }
+
+  template <typename T, typename U>
+  void explain_error(const BinaryInput<T> &in, U out) {
+    explain_binary_operation_one_output_error(op, in, out, ulp_tolerance,
+                                              rounding);
+  }
+};
+
+} // namespace internal
+
+// Return true if the input and ouput types for the operation op are valid
+// types.
+template <Operation op, typename InputType, typename OutputType>
+constexpr bool is_valid_operation() {
+  return (Operation::BeginBinaryOperationsSingleOutput < op &&
+          op < Operation::EndBinaryOperationsSingleOutput &&
+          cpp::is_complex_type_same<InputType, OutputType> &&
+          cpp::is_complex_v<InputType>) ||
+         (Operation::BeginUnaryOperationsSingleOutputSameOutputType < op &&
+          op < Operation::EndUnaryOperationsSingleOutputSameOutputType &&
+          cpp::is_complex_type_same<InputType, OutputType> &&
+          cpp::is_complex_v<InputType>) ||
+         (Operation::BeginUnaryOperationsSingleOutputDifferentOutputType < op &&
+          op < Operation::EndUnaryOperationsSingleOutputDifferentOutputType &&
+          cpp::is_same_v<make_real_t<InputType>, OutputType> &&
+          cpp::is_complex_v<InputType>);
+}
+
+template <Operation op, typename InputType, typename OutputType>
+cpp::enable_if_t<is_valid_operation<op, InputType, OutputType>(),
+                 internal::MPCMatcher<op, InputType, OutputType>>
+get_mpc_matcher(InputType input, [[maybe_unused]] OutputType output,
+                double ulp_tolerance, RoundingMode rounding) {
+  return internal::MPCMatcher<op, InputType, OutputType>(input, ulp_tolerance,
+                                                         rounding);
+}
+
+} // namespace mpc
+} // namespace testing
+} // namespace LIBC_NAMESPACE_DECL
+
+#define EXPECT_MPC_MATCH_DEFAULT(op, input, match_value, ulp_tolerance)        \
+  EXPECT_THAT(match_value,                                                     \
+              LIBC_NAMESPACE::testing::mpc::get_mpc_matcher<op>(               \
+                  input, match_value, ulp_tolerance,                           \
+                  LIBC_NAMESPACE::fputil::testing::RoundingMode::Nearest))
+
+#define EXPECT_MPC_MATCH_ROUNDING(op, input, match_value, ulp_tolerance,       \
+                                  rounding)                                    \
+  EXPECT_THAT(match_value, LIBC_NAMESPACE::testing::mpc::get_mpc_matcher<op>(  \
+                               input, match_value, ulp_tolerance, rounding))
+
+#define EXPECT_MPC_MATCH_ALL_ROUNDING_HELPER(op, input, match_value,           \
+                                             ulp_tolerance, rounding)          \
+  {                                                                            \
+    MPCRND::ForceRoundingMode __r(rounding);                                   \
+    if (__r.success) {                                                         \
+      EXPECT_MPC_MATCH_ROUNDING(op, input, match_value, ulp_tolerance,         \
+                                rounding);                                     \
+    }                                                                          \
+  }
+
+#define EXPECT_MPC_MATCH_ALL_ROUNDING(op, input, match_value, ulp_tolerance)   \
+  {                                                                            \
+    namespace MPCRND = LIBC_NAMESPACE::fputil::testing;                        \
+    for (int i = 0; i < 4; i++) {                                              \
+      MPCRND::RoundingMode r_mode = static_cast<MPCRND::RoundingMode>(i);      \
+      EXPECT_MPC_MATCH_ALL_ROUNDING_HELPER(op, input, match_value,             \
+                                           ulp_tolerance, r_mode);             \
+    }                                                                          \
+  }
+
+#define TEST_MPC_MATCH_ROUNDING(op, input, match_value, ulp_tolerance,         \
+                                rounding)                                      \
+  LIBC_NAMESPACE::testing::mpc::get_mpc_matcher<op>(input, match_value,        \
+                                                    ulp_tolerance, rounding)   \
+      .match(match_value)
+
+#define ASSERT_MPC_MATCH_DEFAULT(op, input, match_value, ulp_tolerance)        \
+  ASSERT_THAT(match_value,                                                     \
+              LIBC_NAMESPACE::testing::mpc::get_mpc_matcher<op>(               \
+                  input, match_value, ulp_tolerance,                           \
+                  LIBC_NAMESPACE::fputil::testing::RoundingMode::Nearest))
+
+#define ASSERT_MPC_MATCH_ROUNDING(op, input, match_value, ulp_tolerance,       \
+                                  rounding)                                    \
+  ASSERT_THAT(match_value, LIBC_NAMESPACE::testing::mpc::get_mpc_matcher<op>(  \
+                               input, match_value, ulp_tolerance, rounding))
+
+#define ASSERT_MPC_MATCH_ALL_ROUNDING_HELPER(op, input, match_value,           \
+                                             ulp_tolerance, rounding)          \
+  {                                                                            \
+    MPCRND::ForceRoundingMode __r(rounding);                                   \
+    if (__r.success) {                                                         \
+      ASSERT_MPC_MATCH_ROUNDING(op, input, match_value, ulp_tolerance,         \
+                                rounding);                                     \
+    }                                                                          \
+  }
+
+#define ASSERT_MPC_MATCH_ALL_ROUNDING(op, input, match_value, ulp_tolerance)   \
+  {                                                                            \
+    namespace MPCRND = LIBC_NAMESPACE::fputil::testing;                        \
+    for (int i = 0; i < 4; i++) {                                              \
+      MPCRND::RoundingMode r_mode = static_cast<MPCRND::RoundingMode>(i);      \
+      ASSERT_MPC_MATCH_ALL_ROUNDING_HELPER(op, input, match_value,             \
+                                           ulp_tolerance, r_mode);             \
+    }                                                                          \
+  }
+
+#endif // LLVM_LIBC_UTILS_MPCWRAPPER_MPCUTILS_H
diff --git a/libc/utils/MPCWrapper/check_mpc.cpp b/libc/utils/MPCWrapper/check_mpc.cpp
new file mode 100644
index 0000000000000..f1f0505252773
--- /dev/null
+++ b/libc/utils/MPCWrapper/check_mpc.cpp
@@ -0,0 +1,8 @@
+#include <mpc.h>
+
+int main() {
+  mpc_t x;
+  mpc_init2(x, 256);
+  mpc_clear(x);
+  return 0;
+}
diff --git a/libc/utils/MPFRWrapper/CMakeLists.txt b/libc/utils/MPFRWrapper/CMakeLists.txt
index 0101c9f399082..9ff7fa109ff97 100644
--- a/libc/utils/MPFRWrapper/CMakeLists.txt
+++ b/libc/utils/MPFRWrapper/CMakeLists.txt
@@ -1,20 +1,45 @@
+if(LIBC_TESTS_CAN_USE_MPFR OR LIBC_TESTS_CAN_USE_MPC)
+  add_library(libcMPCommon STATIC
+    MPCommon.cpp
+    MPCommon.h
+    mpfr_inc.h
+  )
+  _get_common_test_compile_options(compile_options "" "")
+  # mpfr/gmp headers do not work with -ffreestanding flag.
+  list(REMOVE_ITEM compile_options "-ffreestanding")
+  target_compile_options(libcMPCommon PRIVATE -O3 ${compile_options})
+  add_dependencies(
+    libcMPCommon
+    libc.src.__support.CPP.string
+    libc.src.__support.CPP.string_view
+    libc.src.__support.CPP.type_traits
+    libc.src.__support.FPUtil.cast
+    libc.src.__support.FPUtil.fp_bits
+  )
+  if(EXISTS ${LLVM_LIBC_MPFR_INSTALL_PATH})
+    target_include_directories(libcMPCommon PUBLIC ${LLVM_LIBC_MPFR_INSTALL_PATH}/include)
+    target_link_directories(libcMPCommon PUBLIC ${LLVM_LIBC_MPFR_INSTALL_PATH}/lib)
+  endif()
+  target_include_directories(libcMPCommon PUBLIC ${LIBC_SOURCE_DIR})
+  target_link_libraries(libcMPCommon PUBLIC LibcFPTestHelpers.unit mpfr gmp)
+elseif(NOT LIBC_TARGET_OS_IS_GPU AND NOT LLVM_LIBC_FULL_BUILD)
+  message(WARNING "Math tests using MPFR will be skipped.")
+endif()
+
 if(LIBC_TESTS_CAN_USE_MPFR)
   add_library(libcMPFRWrapper STATIC
     MPFRUtils.cpp
     MPFRUtils.h
-    mpfr_inc.h
   )
   _get_common_test_compile_options(compile_options "" "")
   # mpfr/gmp headers do not work with -ffreestanding flag.
   list(REMOVE_ITEM compile_options "-ffreestanding")
-  target_compile_options(libcMPFRWrapper PRIVATE -O3 ${compile_options})
+  target_compile_options(libcMPFRWrapper PRIVATE ${libc_opt_high_flag} ${compile_options})
   add_dependencies(
     libcMPFRWrapper
+    libcMPCommon
     libc.src.__support.CPP.array
     libc.src.__support.CPP.stringstream
-    libc.src.__support.CPP.string_view
-    libc.src.__support.CPP.type_traits
-    libc.src.__support.FPUtil.cast
     libc.src.__support.FPUtil.fp_bits
     libc.src.__support.FPUtil.fpbits_str
     LibcTest.unit
@@ -24,7 +49,7 @@ if(LIBC_TESTS_CAN_USE_MPFR)
     target_link_directories(libcMPFRWrapper PUBLIC ${LLVM_LIBC_MPFR_INSTALL_PATH}/lib)
   endif()
   target_include_directories(libcMPFRWrapper PUBLIC ${LIBC_SOURCE_DIR})
-  target_link_libraries(libcMPFRWrapper PUBLIC LibcFPTestHelpers.unit LibcTest.unit mpfr gmp)
+  target_link_libraries(libcMPFRWrapper PUBLIC libcMPCommon LibcFPTestHelpers.unit LibcTest.unit)
 elseif(NOT LIBC_TARGET_OS_IS_GPU AND NOT LLVM_LIBC_FULL_BUILD)
   message(WARNING "Math tests using MPFR will be skipped.")
 endif()
diff --git a/libc/utils/MPFRWrapper/MPCommon.cpp b/libc/utils/MPFRWrapper/MPCommon.cpp
new file mode 100644
index 0000000000000..8f104c908f036
--- /dev/null
+++ b/libc/utils/MPFRWrapper/MPCommon.cpp
@@ -0,0 +1,561 @@
+//===-- Utils used by both MPCWrapper and MPFRWrapper----------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "MPCommon.h"
+
+#include "src/__support/CPP/string_view.h"
+#include "src/__support/FPUtil/cast.h"
+#include "src/__support/macros/config.h"
+#include "src/__support/macros/properties/types.h"
+
+namespace LIBC_NAMESPACE_DECL {
+namespace testing {
+namespace mpfr {
+
+MPFRNumber::MPFRNumber() : mpfr_precision(256), mpfr_rounding(MPFR_RNDN) {
+  mpfr_init2(value, mpfr_precision);
+}
+
+MPFRNumber::MPFRNumber(const MPFRNumber &other)
+    : mpfr_precision(other.mpfr_precision), mpfr_rounding(other.mpfr_rounding) {
+  mpfr_init2(value, mpfr_precision);
+  mpfr_set(value, other.value, mpfr_rounding);
+}
+
+MPFRNumber::MPFRNumber(const MPFRNumber &other, unsigned int precision)
+    : mpfr_precision(precision), mpfr_rounding(other.mpfr_rounding) {
+  mpfr_init2(value, mpfr_precision);
+  mpfr_set(value, other.value, mpfr_rounding);
+}
+
+MPFRNumber::MPFRNumber(const mpfr_t x, unsigned int precision,
+                       RoundingMode rounding)
+    : mpfr_precision(precision),
+      mpfr_rounding(get_mpfr_rounding_mode(rounding)) {
+  mpfr_init2(value, mpfr_precision);
+  mpfr_set(value, x, mpfr_rounding);
+}
+
+MPFRNumber::~MPFRNumber() { mpfr_clear(value); }
+
+MPFRNumber &MPFRNumber::operator=(const MPFRNumber &rhs) {
+  mpfr_precision = rhs.mpfr_precision;
+  mpfr_rounding = rhs.mpfr_rounding;
+  mpfr_set(value, rhs.value, mpfr_rounding);
+  return *this;
+}
+
+bool MPFRNumber::is_nan() const { return mpfr_nan_p(value); }
+
+MPFRNumber MPFRNumber::abs() const {
+  MPFRNumber result(*this);
+  mpfr_abs(result.value, value, mpfr_rounding);
+  return result;
+}
+
+MPFRNumber MPFRNumber::acos() const {
+  MPFRNumber result(*this);
+  mpfr_acos(result.value, value, mpfr_rounding);
+  return result;
+}
+
+MPFRNumber MPFRNumber::acosh() const {
+  MPFRNumber result(*this);
+  mpfr_acosh(result.value, value, mpfr_rounding);
+  return result;
+}
+
+MPFRNumber MPFRNumber::add(const MPFRNumber &b) const {
+  MPFRNumber result(*this);
+  mpfr_add(result.value, value, b.value, mpfr_rounding);
+  return result;
+}
+
+MPFRNumber MPFRNumber::asin() const {
+  MPFRNumber result(*this);
+  mpfr_asin(result.value, value, mpfr_rounding);
+  return result;
+}
+
+MPFRNumber MPFRNumber::asinh() const {
+  MPFRNumber result(*this);
+  mpfr_asinh(result.value, value, mpfr_rounding);
+  return result;
+}
+
+MPFRNumber MPFRNumber::atan() const {
+  MPFRNumber result(*this);
+  mpfr_atan(result.value, value, mpfr_rounding);
+  return result;
+}
+
+MPFRNumber MPFRNumber::atan2(const MPFRNumber &b) {
+  MPFRNumber result(*this);
+  mpfr_atan2(result.value, value, b.value, mpfr_rounding);
+  return result;
+}
+
+MPFRNumber MPFRNumber::atanh() const {
+  MPFRNumber result(*this);
+  mpfr_atanh(result.value, value, mpfr_rounding);
+  return result;
+}
+
+MPFRNumber MPFRNumber::cbrt() const {
+  MPFRNumber result(*this);
+  mpfr_cbrt(result.value, value, mpfr_rounding);
+  return result;
+}
+
+MPFRNumber MPFRNumber::ceil() const {
+  MPFRNumber result(*this);
+  mpfr_ceil(result.value, value);
+  return result;
+}
+
+MPFRNumber MPFRNumber::cos() const {
+  MPFRNumber result(*this);
+  mpfr_cos(result.value, value, mpfr_rounding);
+  return result;
+}
+
+MPFRNumber MPFRNumber::cosh() const {
+  MPFRNumber result(*this);
+  mpfr_cosh(result.value, value, mpfr_rounding);
+  return result;
+}
+
+MPFRNumber MPFRNumber::cospi() const {
+  MPFRNumber result(*this);
+
+#if MPFR_VERSION_MAJOR > 4 ||                                                  \
+    (MPFR_VERSION_MAJOR == 4 && MPFR_VERSION_MINOR >= 2)
+  mpfr_cospi(result.value, value, mpfr_rounding);
+  return result;
+#else
+  if (mpfr_integer_p(value)) {
+    mpz_t integer;
+    mpz_init(integer);
+    mpfr_get_z(integer, value, mpfr_rounding);
+
+    int d = mpz_tstbit(integer, 0);
+    mpfr_set_si(result.value, d ? -1 : 1, mpfr_rounding);
+    mpz_clear(integer);
+    return result;
+  }
+
+  MPFRNumber value_pi(0.0, 1280);
+  mpfr_const_pi(value_pi.value, MPFR_RNDN);
+  mpfr_mul(value_pi.value, value_pi.value, value, MPFR_RNDN);
+  mpfr_cos(result.value, value_pi.value, mpfr_rounding);
+
+  return result;
+#endif
+}
+
+MPFRNumber MPFRNumber::erf() const {
+  MPFRNumber result(*this);
+  mpfr_erf(result.value, value, mpfr_rounding);
+  return result;
+}
+
+MPFRNumber MPFRNumber::exp() const {
+  MPFRNumber result(*this);
+  mpfr_exp(result.value, value, mpfr_rounding);
+  return result;
+}
+
+MPFRNumber MPFRNumber::exp2() const {
+  MPFRNumber result(*this);
+  mpfr_exp2(result.value, value, mpfr_rounding);
+  return result;
+}
+
+MPFRNumber MPFRNumber::exp2m1() const {
+  // TODO: Only use mpfr_exp2m1 once CI and buildbots get MPFR >= 4.2.0.
+#if MPFR_VERSION_MAJOR > 4 ||                                                  \
+    (MPFR_VERSION_MAJOR == 4 && MPFR_VERSION_MINOR >= 2)
+  MPFRNumber result(*this);
+  mpfr_exp2m1(result.value, value, mpfr_rounding);
+  return result;
+#else
+  unsigned int prec = mpfr_precision * 3;
+  MPFRNumber result(*this, prec);
+
+  float f = mpfr_get_flt(abs().value, mpfr_rounding);
+  if (f > 0.5f && f < 0x1.0p30f) {
+    mpfr_exp2(result.value, value, mpfr_rounding);
+    mpfr_sub_ui(result.value, result.value, 1, mpfr_rounding);
+    return result;
+  }
+
+  MPFRNumber ln2(2.0f, prec);
+  // log(2)
+  mpfr_log(ln2.value, ln2.value, mpfr_rounding);
+  // x * log(2)
+  mpfr_mul(result.value, value, ln2.value, mpfr_rounding);
+  // e^(x * log(2)) - 1
+  int ex = mpfr_expm1(result.value, result.value, mpfr_rounding);
+  mpfr_subnormalize(result.value, ex, mpfr_rounding);
+  return result;
+#endif
+}
+
+MPFRNumber MPFRNumber::exp10() const {
+  MPFRNumber result(*this);
+  mpfr_exp10(result.value, value, mpfr_rounding);
+  return result;
+}
+
+MPFRNumber MPFRNumber::exp10m1() const {
+  // TODO: Only use mpfr_exp10m1 once CI and buildbots get MPFR >= 4.2.0.
+#if MPFR_VERSION_MAJOR > 4 ||                                                  \
+    (MPFR_VERSION_MAJOR == 4 && MPFR_VERSION_MINOR >= 2)
+  MPFRNumber result(*this);
+  mpfr_exp10m1(result.value, value, mpfr_rounding);
+  return result;
+#else
+  unsigned int prec = mpfr_precision * 3;
+  MPFRNumber result(*this, prec);
+
+  MPFRNumber ln10(10.0f, prec);
+  // log(10)
+  mpfr_log(ln10.value, ln10.value, mpfr_rounding);
+  // x * log(10)
+  mpfr_mul(result.value, value, ln10.value, mpfr_rounding);
+  // e^(x * log(10)) - 1
+  int ex = mpfr_expm1(result.value, result.value, mpfr_rounding);
+  mpfr_subnormalize(result.value, ex, mpfr_rounding);
+  return result;
+#endif
+}
+
+MPFRNumber MPFRNumber::expm1() const {
+  MPFRNumber result(*this);
+  mpfr_expm1(result.value, value, mpfr_rounding);
+  return result;
+}
+
+MPFRNumber MPFRNumber::div(const MPFRNumber &b) const {
+  MPFRNumber result(*this);
+  mpfr_div(result.value, value, b.value, mpfr_rounding);
+  return result;
+}
+
+MPFRNumber MPFRNumber::floor() const {
+  MPFRNumber result(*this);
+  mpfr_floor(result.value, value);
+  return result;
+}
+
+MPFRNumber MPFRNumber::fmod(const MPFRNumber &b) {
+  MPFRNumber result(*this);
+  mpfr_fmod(result.value, value, b.value, mpfr_rounding);
+  return result;
+}
+
+MPFRNumber MPFRNumber::frexp(int &exp) {
+  MPFRNumber result(*this);
+  mpfr_exp_t resultExp;
+  mpfr_frexp(&resultExp, result.value, value, mpfr_rounding);
+  exp = resultExp;
+  return result;
+}
+
+MPFRNumber MPFRNumber::hypot(const MPFRNumber &b) {
+  MPFRNumber result(*this);
+  mpfr_hypot(result.value, value, b.value, mpfr_rounding);
+  return result;
+}
+
+MPFRNumber MPFRNumber::log() const {
+  MPFRNumber result(*this);
+  mpfr_log(result.value, value, mpfr_rounding);
+  return result;
+}
+
+MPFRNumber MPFRNumber::log2() const {
+  MPFRNumber result(*this);
+  mpfr_log2(result.value, value, mpfr_rounding);
+  return result;
+}
+
+MPFRNumber MPFRNumber::log10() const {
+  MPFRNumber result(*this);
+  mpfr_log10(result.value, value, mpfr_rounding);
+  return result;
+}
+
+MPFRNumber MPFRNumber::log1p() const {
+  MPFRNumber result(*this);
+  mpfr_log1p(result.value, value, mpfr_rounding);
+  return result;
+}
+
+MPFRNumber MPFRNumber::pow(const MPFRNumber &b) {
+  MPFRNumber result(*this);
+  mpfr_pow(result.value, value, b.value, mpfr_rounding);
+  return result;
+}
+
+MPFRNumber MPFRNumber::remquo(const MPFRNumber &divisor, int &quotient) {
+  MPFRNumber remainder(*this);
+  long q;
+  mpfr_remquo(remainder.value, &q, value, divisor.value, mpfr_rounding);
+  quotient = q;
+  return remainder;
+}
+
+MPFRNumber MPFRNumber::round() const {
+  MPFRNumber result(*this);
+  mpfr_round(result.value, value);
+  return result;
+}
+
+MPFRNumber MPFRNumber::roundeven() const {
+  MPFRNumber result(*this);
+#if MPFR_VERSION_MAJOR >= 4
+  mpfr_roundeven(result.value, value);
+#else
+  mpfr_rint(result.value, value, MPFR_RNDN);
+#endif
+  return result;
+}
+
+bool MPFRNumber::round_to_long(long &result) const {
+  // We first calculate the rounded value. This way, when converting
+  // to long using mpfr_get_si, the rounding direction of MPFR_RNDN
+  // (or any other rounding mode), does not have an influence.
+  MPFRNumber roundedValue = round();
+  mpfr_clear_erangeflag();
+  result = mpfr_get_si(roundedValue.value, MPFR_RNDN);
+  return mpfr_erangeflag_p();
+}
+
+bool MPFRNumber::round_to_long(mpfr_rnd_t rnd, long &result) const {
+  MPFRNumber rint_result(*this);
+  mpfr_rint(rint_result.value, value, rnd);
+  return rint_result.round_to_long(result);
+}
+
+MPFRNumber MPFRNumber::rint(mpfr_rnd_t rnd) const {
+  MPFRNumber result(*this);
+  mpfr_rint(result.value, value, rnd);
+  return result;
+}
+
+MPFRNumber MPFRNumber::mod_2pi() const {
+  MPFRNumber result(0.0, 1280);
+  MPFRNumber _2pi(0.0, 1280);
+  mpfr_const_pi(_2pi.value, MPFR_RNDN);
+  mpfr_mul_si(_2pi.value, _2pi.value, 2, MPFR_RNDN);
+  mpfr_fmod(result.value, value, _2pi.value, MPFR_RNDN);
+  return result;
+}
+
+MPFRNumber MPFRNumber::mod_pi_over_2() const {
+  MPFRNumber result(0.0, 1280);
+  MPFRNumber pi_over_2(0.0, 1280);
+  mpfr_const_pi(pi_over_2.value, MPFR_RNDN);
+  mpfr_mul_d(pi_over_2.value, pi_over_2.value, 0.5, MPFR_RNDN);
+  mpfr_fmod(result.value, value, pi_over_2.value, MPFR_RNDN);
+  return result;
+}
+
+MPFRNumber MPFRNumber::mod_pi_over_4() const {
+  MPFRNumber result(0.0, 1280);
+  MPFRNumber pi_over_4(0.0, 1280);
+  mpfr_const_pi(pi_over_4.value, MPFR_RNDN);
+  mpfr_mul_d(pi_over_4.value, pi_over_4.value, 0.25, MPFR_RNDN);
+  mpfr_fmod(result.value, value, pi_over_4.value, MPFR_RNDN);
+  return result;
+}
+
+MPFRNumber MPFRNumber::sin() const {
+  MPFRNumber result(*this);
+  mpfr_sin(result.value, value, mpfr_rounding);
+  return result;
+}
+
+MPFRNumber MPFRNumber::sinpi() const {
+  MPFRNumber result(*this);
+
+#if MPFR_VERSION_MAJOR > 4 ||                                                  \
+    (MPFR_VERSION_MAJOR == 4 && MPFR_VERSION_MINOR >= 2)
+
+  mpfr_sinpi(result.value, value, mpfr_rounding);
+  return result;
+#else
+  if (mpfr_integer_p(value)) {
+    mpfr_set_si(result.value, 0, mpfr_rounding);
+    return result;
+  }
+
+  MPFRNumber value_mul_two(*this);
+  mpfr_mul_si(value_mul_two.value, value, 2, MPFR_RNDN);
+
+  if (mpfr_integer_p(value_mul_two.value)) {
+    auto d = mpfr_get_si(value, MPFR_RNDD);
+    mpfr_set_si(result.value, (d & 1) ? -1 : 1, mpfr_rounding);
+    return result;
+  }
+
+  MPFRNumber value_pi(0.0, 1280);
+  mpfr_const_pi(value_pi.value, MPFR_RNDN);
+  mpfr_mul(value_pi.value, value_pi.value, value, MPFR_RNDN);
+  mpfr_sin(result.value, value_pi.value, mpfr_rounding);
+  return result;
+#endif
+}
+
+MPFRNumber MPFRNumber::sinh() const {
+  MPFRNumber result(*this);
+  mpfr_sinh(result.value, value, mpfr_rounding);
+  return result;
+}
+
+MPFRNumber MPFRNumber::sqrt() const {
+  MPFRNumber result(*this);
+  mpfr_sqrt(result.value, value, mpfr_rounding);
+  return result;
+}
+
+MPFRNumber MPFRNumber::sub(const MPFRNumber &b) const {
+  MPFRNumber result(*this);
+  mpfr_sub(result.value, value, b.value, mpfr_rounding);
+  return result;
+}
+
+MPFRNumber MPFRNumber::tan() const {
+  MPFRNumber result(*this);
+  mpfr_tan(result.value, value, mpfr_rounding);
+  return result;
+}
+
+MPFRNumber MPFRNumber::tanh() const {
+  MPFRNumber result(*this);
+  mpfr_tanh(result.value, value, mpfr_rounding);
+  return result;
+}
+
+MPFRNumber MPFRNumber::tanpi() const {
+  MPFRNumber result(*this);
+
+#if MPFR_VERSION_MAJOR > 4 ||                                                  \
+    (MPFR_VERSION_MAJOR == 4 && MPFR_VERSION_MINOR >= 2)
+
+  mpfr_tanpi(result.value, value, mpfr_rounding);
+  return result;
+#else
+  MPFRNumber value_ret_exact(*this);
+  MPFRNumber value_one(*this);
+  mpfr_set_si(value_one.value, 1, MPFR_RNDN);
+  mpfr_fmod(value_ret_exact.value, value, value_one.value, mpfr_rounding);
+  mpfr_mul_si(value_ret_exact.value, value_ret_exact.value, 4, MPFR_RNDN);
+
+  if (mpfr_integer_p(value_ret_exact.value)) {
+    int mod = mpfr_get_si(value_ret_exact.value, MPFR_RNDN);
+    mod = (mod < 0 ? -1 * mod : mod);
+
+    switch (mod) {
+    case 0:
+      mpfr_set_si(result.value, 0, mpfr_rounding);
+      break;
+    case 1:
+      mpfr_set_si(result.value, (mpfr_signbit(value) ? -1 : 1), mpfr_rounding);
+      break;
+    case 2: {
+      auto d = mpfr_get_si(value, MPFR_RNDZ);
+      d += mpfr_sgn(value) > 0 ? 0 : 1;
+      mpfr_set_inf(result.value, (d & 1) ? -1 : 1);
+      break;
+    }
+    case 3:
+      mpfr_set_si(result.value, (mpfr_signbit(value) ? 1 : -1), mpfr_rounding);
+      break;
+    }
+
+    return result;
+  }
+
+  MPFRNumber value_pi(0.0, 1280);
+  mpfr_const_pi(value_pi.value, MPFR_RNDN);
+  mpfr_mul(value_pi.value, value_pi.value, value, MPFR_RNDN);
+  mpfr_tan(result.value, value_pi.value, mpfr_rounding);
+  return result;
+#endif
+}
+
+MPFRNumber MPFRNumber::trunc() const {
+  MPFRNumber result(*this);
+  mpfr_trunc(result.value, value);
+  return result;
+}
+
+MPFRNumber MPFRNumber::fma(const MPFRNumber &b, const MPFRNumber &c) {
+  MPFRNumber result(*this);
+  mpfr_fma(result.value, value, b.value, c.value, mpfr_rounding);
+  return result;
+}
+
+MPFRNumber MPFRNumber::mul(const MPFRNumber &b) {
+  MPFRNumber result(*this);
+  mpfr_mul(result.value, value, b.value, mpfr_rounding);
+  return result;
+}
+
+cpp::string MPFRNumber::str() const {
+  // 200 bytes should be more than sufficient to hold a 100-digit number
+  // plus additional bytes for the decimal point, '-' sign etc.
+  constexpr size_t printBufSize = 200;
+  char buffer[printBufSize];
+  mpfr_snprintf(buffer, printBufSize, "%100.50Rf", value);
+  cpp::string_view view(buffer);
+  // Trim whitespaces
+  const char whitespace = ' ';
+  while (!view.empty() && view.front() == whitespace)
+    view.remove_prefix(1);
+  while (!view.empty() && view.back() == whitespace)
+    view.remove_suffix(1);
+  return cpp::string(view.data());
+}
+
+void MPFRNumber::dump(const char *msg) const {
+  mpfr_printf("%s%.128g\n", msg, value);
+}
+
+template <> float MPFRNumber::as<float>() const {
+  return mpfr_get_flt(value, mpfr_rounding);
+}
+
+template <> double MPFRNumber::as<double>() const {
+  return mpfr_get_d(value, mpfr_rounding);
+}
+
+template <> long double MPFRNumber::as<long double>() const {
+  return mpfr_get_ld(value, mpfr_rounding);
+}
+
+#ifdef LIBC_TYPES_HAS_FLOAT16
+template <> float16 MPFRNumber::as<float16>() const {
+  // TODO: Either prove that this cast won't cause double-rounding errors, or
+  // find a better way to get a float16.
+  return fputil::cast<float16>(mpfr_get_d(value, mpfr_rounding));
+}
+#endif
+
+#ifdef LIBC_TYPES_FLOAT128_IS_NOT_LONG_DOUBLE
+template <> float128 MPFRNumber::as<float128>() const {
+  return mpfr_get_float128(value, mpfr_rounding);
+}
+
+#endif // LIBC_TYPES_FLOAT128_IS_NOT_LONG_DOUBLE
+
+} // namespace mpfr
+} // namespace testing
+} // namespace LIBC_NAMESPACE_DECL
diff --git a/libc/utils/MPFRWrapper/MPCommon.h b/libc/utils/MPFRWrapper/MPCommon.h
new file mode 100644
index 0000000000000..eaa512e30bc86
--- /dev/null
+++ b/libc/utils/MPFRWrapper/MPCommon.h
@@ -0,0 +1,342 @@
+//===-- MPCommon.h ----------------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_UTILS_MPFRWRAPPER_MPCOMMON_H
+#define LLVM_LIBC_UTILS_MPFRWRAPPER_MPCOMMON_H
+
+#include "src/__support/CPP/string.h"
+#include "src/__support/CPP/type_traits.h"
+#include "src/__support/FPUtil/FPBits.h"
+#include "src/__support/macros/config.h"
+#include "test/UnitTest/RoundingModeUtils.h"
+
+#include <stdint.h>
+
+#include "mpfr_inc.h"
+
+#ifdef LIBC_TYPES_FLOAT128_IS_NOT_LONG_DOUBLE
+extern "C" {
+int mpfr_set_float128(mpfr_ptr, float128, mpfr_rnd_t);
+float128 mpfr_get_float128(mpfr_srcptr, mpfr_rnd_t);
+}
+#endif
+
+namespace LIBC_NAMESPACE_DECL {
+namespace testing {
+namespace mpfr {
+
+template <typename T> using FPBits = LIBC_NAMESPACE::fputil::FPBits<T>;
+using LIBC_NAMESPACE::fputil::testing::RoundingMode;
+
+// A precision value which allows sufficiently large additional
+// precision compared to the floating point precision.
+template <typename T> struct ExtraPrecision;
+
+#ifdef LIBC_TYPES_HAS_FLOAT16
+template <> struct ExtraPrecision<float16> {
+  static constexpr unsigned int VALUE = 128;
+};
+#endif
+
+template <> struct ExtraPrecision<float> {
+  static constexpr unsigned int VALUE = 128;
+};
+
+template <> struct ExtraPrecision<double> {
+  static constexpr unsigned int VALUE = 256;
+};
+
+template <> struct ExtraPrecision<long double> {
+#ifdef LIBC_TYPES_LONG_DOUBLE_IS_FLOAT128
+  static constexpr unsigned int VALUE = 512;
+#else
+  static constexpr unsigned int VALUE = 256;
+#endif
+};
+
+#if defined(LIBC_TYPES_FLOAT128_IS_NOT_LONG_DOUBLE)
+template <> struct ExtraPrecision<float128> {
+  static constexpr unsigned int VALUE = 512;
+};
+#endif // LIBC_TYPES_FLOAT128_IS_NOT_LONG_DOUBLE
+
+// If the ulp tolerance is less than or equal to 0.5, we would check that the
+// result is rounded correctly with respect to the rounding mode by using the
+// same precision as the inputs.
+template <typename T>
+static inline unsigned int get_precision(double ulp_tolerance) {
+  if (ulp_tolerance <= 0.5) {
+    return LIBC_NAMESPACE::fputil::FPBits<T>::FRACTION_LEN + 1;
+  } else {
+    return ExtraPrecision<T>::VALUE;
+  }
+}
+
+static inline mpfr_rnd_t get_mpfr_rounding_mode(RoundingMode mode) {
+  switch (mode) {
+  case RoundingMode::Upward:
+    return MPFR_RNDU;
+    break;
+  case RoundingMode::Downward:
+    return MPFR_RNDD;
+    break;
+  case RoundingMode::TowardZero:
+    return MPFR_RNDZ;
+    break;
+  case RoundingMode::Nearest:
+    return MPFR_RNDN;
+    break;
+  }
+  __builtin_unreachable();
+}
+
+class MPFRNumber {
+  unsigned int mpfr_precision;
+  mpfr_rnd_t mpfr_rounding;
+  mpfr_t value;
+
+public:
+  MPFRNumber();
+  // We use explicit EnableIf specializations to disallow implicit
+  // conversions. Implicit conversions can potentially lead to loss of
+  // precision. We exceptionally allow implicit conversions from float16
+  // to float, as the MPFR API does not support float16, thus requiring
+  // conversion to a higher-precision format.
+  template <typename XType,
+            cpp::enable_if_t<cpp::is_same_v<float, XType>
+#ifdef LIBC_TYPES_HAS_FLOAT16
+                                 || cpp::is_same_v<float16, XType>
+#endif
+                             ,
+                             int> = 0>
+  explicit MPFRNumber(XType x,
+                      unsigned int precision = ExtraPrecision<XType>::VALUE,
+                      RoundingMode rounding = RoundingMode::Nearest)
+      : mpfr_precision(precision),
+        mpfr_rounding(get_mpfr_rounding_mode(rounding)) {
+    mpfr_init2(value, mpfr_precision);
+    mpfr_set_flt(value, x, mpfr_rounding);
+  }
+
+  template <typename XType,
+            cpp::enable_if_t<cpp::is_same_v<double, XType>, int> = 0>
+  explicit MPFRNumber(XType x,
+                      unsigned int precision = ExtraPrecision<XType>::VALUE,
+                      RoundingMode rounding = RoundingMode::Nearest)
+      : mpfr_precision(precision),
+        mpfr_rounding(get_mpfr_rounding_mode(rounding)) {
+    mpfr_init2(value, mpfr_precision);
+    mpfr_set_d(value, x, mpfr_rounding);
+  }
+
+  template <typename XType,
+            cpp::enable_if_t<cpp::is_same_v<long double, XType>, int> = 0>
+  explicit MPFRNumber(XType x,
+                      unsigned int precision = ExtraPrecision<XType>::VALUE,
+                      RoundingMode rounding = RoundingMode::Nearest)
+      : mpfr_precision(precision),
+        mpfr_rounding(get_mpfr_rounding_mode(rounding)) {
+    mpfr_init2(value, mpfr_precision);
+    mpfr_set_ld(value, x, mpfr_rounding);
+  }
+
+#ifdef LIBC_TYPES_FLOAT128_IS_NOT_LONG_DOUBLE
+  template <typename XType,
+            cpp::enable_if_t<cpp::is_same_v<float128, XType>, int> = 0>
+  explicit MPFRNumber(XType x,
+                      unsigned int precision = ExtraPrecision<XType>::VALUE,
+                      RoundingMode rounding = RoundingMode::Nearest)
+      : mpfr_precision(precision),
+        mpfr_rounding(get_mpfr_rounding_mode(rounding)) {
+    mpfr_init2(value, mpfr_precision);
+    mpfr_set_float128(value, x, mpfr_rounding);
+  }
+#endif // LIBC_TYPES_FLOAT128_IS_NOT_LONG_DOUBLE
+
+  template <typename XType,
+            cpp::enable_if_t<cpp::is_integral_v<XType>, int> = 0>
+  explicit MPFRNumber(XType x,
+                      unsigned int precision = ExtraPrecision<float>::VALUE,
+                      RoundingMode rounding = RoundingMode::Nearest)
+      : mpfr_precision(precision),
+        mpfr_rounding(get_mpfr_rounding_mode(rounding)) {
+    mpfr_init2(value, mpfr_precision);
+    mpfr_set_sj(value, x, mpfr_rounding);
+  }
+
+  MPFRNumber(const MPFRNumber &other);
+  MPFRNumber(const MPFRNumber &other, unsigned int precision);
+  MPFRNumber(const mpfr_t x, unsigned int precision, RoundingMode rounding);
+
+  ~MPFRNumber();
+
+  MPFRNumber &operator=(const MPFRNumber &rhs);
+
+  bool is_nan() const;
+  MPFRNumber abs() const;
+  MPFRNumber acos() const;
+  MPFRNumber acosh() const;
+  MPFRNumber add(const MPFRNumber &b) const;
+  MPFRNumber asin() const;
+  MPFRNumber asinh() const;
+  MPFRNumber atan() const;
+  MPFRNumber atan2(const MPFRNumber &b);
+  MPFRNumber atanh() const;
+  MPFRNumber cbrt() const;
+  MPFRNumber ceil() const;
+  MPFRNumber cos() const;
+  MPFRNumber cosh() const;
+  MPFRNumber cospi() const;
+  MPFRNumber erf() const;
+  MPFRNumber exp() const;
+  MPFRNumber exp2() const;
+  MPFRNumber exp2m1() const;
+  MPFRNumber exp10() const;
+  MPFRNumber exp10m1() const;
+  MPFRNumber expm1() const;
+  MPFRNumber div(const MPFRNumber &b) const;
+  MPFRNumber floor() const;
+  MPFRNumber fmod(const MPFRNumber &b);
+  MPFRNumber frexp(int &exp);
+  MPFRNumber hypot(const MPFRNumber &b);
+  MPFRNumber log() const;
+  MPFRNumber log2() const;
+  MPFRNumber log10() const;
+  MPFRNumber log1p() const;
+  MPFRNumber pow(const MPFRNumber &b);
+  MPFRNumber remquo(const MPFRNumber &divisor, int &quotient);
+  MPFRNumber round() const;
+  MPFRNumber roundeven() const;
+  bool round_to_long(long &result) const;
+  bool round_to_long(mpfr_rnd_t rnd, long &result) const;
+  MPFRNumber rint(mpfr_rnd_t rnd) const;
+  MPFRNumber mod_2pi() const;
+  MPFRNumber mod_pi_over_2() const;
+  MPFRNumber mod_pi_over_4() const;
+  MPFRNumber sin() const;
+  MPFRNumber sinpi() const;
+  MPFRNumber sinh() const;
+  MPFRNumber sqrt() const;
+  MPFRNumber sub(const MPFRNumber &b) const;
+  MPFRNumber tan() const;
+  MPFRNumber tanh() const;
+  MPFRNumber tanpi() const;
+  MPFRNumber trunc() const;
+  MPFRNumber fma(const MPFRNumber &b, const MPFRNumber &c);
+  MPFRNumber mul(const MPFRNumber &b);
+  cpp::string str() const;
+
+  template <typename T> T as() const;
+  void dump(const char *msg) const;
+
+  // Return the ULP (units-in-the-last-place) difference between the
+  // stored MPFR and a floating point number.
+  //
+  // We define ULP difference as follows:
+  //   If exponents of this value and the |input| are same, then:
+  //     ULP(this_value, input) = abs(this_value - input) / eps(input)
+  //   else:
+  //     max = max(abs(this_value), abs(input))
+  //     min = min(abs(this_value), abs(input))
+  //     maxExponent = exponent(max)
+  //     ULP(this_value, input) = (max - 2^maxExponent) / eps(max) +
+  //                              (2^maxExponent - min) / eps(min)
+  //
+  // Remarks:
+  // 1. A ULP of 0.0 will imply that the value is correctly rounded.
+  // 2. We expect that this value and the value to be compared (the [input]
+  //    argument) are reasonable close, and we will provide an upper bound
+  //    of ULP value for testing.  Morever, most of the fractional parts of
+  //    ULP value do not matter much, so using double as the return type
+  //    should be good enough.
+  // 3. For close enough values (values which don't diff in their exponent by
+  //    not more than 1), a ULP difference of N indicates a bit distance
+  //    of N between this number and [input].
+  // 4. A values of +0.0 and -0.0 are treated as equal.
+  template <typename T>
+  cpp::enable_if_t<cpp::is_floating_point_v<T>, MPFRNumber>
+  ulp_as_mpfr_number(T input) {
+    T thisAsT = as<T>();
+    if (thisAsT == input)
+      return MPFRNumber(0.0);
+
+    if (is_nan()) {
+      if (FPBits<T>(input).is_nan())
+        return MPFRNumber(0.0);
+      return MPFRNumber(FPBits<T>::inf().get_val());
+    }
+
+    int thisExponent = FPBits<T>(thisAsT).get_exponent();
+    int inputExponent = FPBits<T>(input).get_exponent();
+    // Adjust the exponents for denormal numbers.
+    if (FPBits<T>(thisAsT).is_subnormal())
+      ++thisExponent;
+    if (FPBits<T>(input).is_subnormal())
+      ++inputExponent;
+
+    if (thisAsT * input < 0 || thisExponent == inputExponent) {
+      MPFRNumber inputMPFR(input);
+      mpfr_sub(inputMPFR.value, value, inputMPFR.value, MPFR_RNDN);
+      mpfr_abs(inputMPFR.value, inputMPFR.value, MPFR_RNDN);
+      mpfr_mul_2si(inputMPFR.value, inputMPFR.value,
+                   -thisExponent + FPBits<T>::FRACTION_LEN, MPFR_RNDN);
+      return inputMPFR;
+    }
+
+    // If the control reaches here, it means that this number and input are
+    // of the same sign but different exponent. In such a case, ULP error is
+    // calculated as sum of two parts.
+    thisAsT = FPBits<T>(thisAsT).abs().get_val();
+    input = FPBits<T>(input).abs().get_val();
+    T min = thisAsT > input ? input : thisAsT;
+    T max = thisAsT > input ? thisAsT : input;
+    int minExponent = FPBits<T>(min).get_exponent();
+    int maxExponent = FPBits<T>(max).get_exponent();
+    // Adjust the exponents for denormal numbers.
+    if (FPBits<T>(min).is_subnormal())
+      ++minExponent;
+    if (FPBits<T>(max).is_subnormal())
+      ++maxExponent;
+
+    MPFRNumber minMPFR(min);
+    MPFRNumber maxMPFR(max);
+
+    MPFRNumber pivot(uint32_t(1));
+    mpfr_mul_2si(pivot.value, pivot.value, maxExponent, MPFR_RNDN);
+
+    mpfr_sub(minMPFR.value, pivot.value, minMPFR.value, MPFR_RNDN);
+    mpfr_mul_2si(minMPFR.value, minMPFR.value,
+                 -minExponent + FPBits<T>::FRACTION_LEN, MPFR_RNDN);
+
+    mpfr_sub(maxMPFR.value, maxMPFR.value, pivot.value, MPFR_RNDN);
+    mpfr_mul_2si(maxMPFR.value, maxMPFR.value,
+                 -maxExponent + FPBits<T>::FRACTION_LEN, MPFR_RNDN);
+
+    mpfr_add(minMPFR.value, minMPFR.value, maxMPFR.value, MPFR_RNDN);
+    return minMPFR;
+  }
+
+  template <typename T>
+  cpp::enable_if_t<cpp::is_floating_point_v<T>, cpp::string>
+  ulp_as_string(T input) {
+    MPFRNumber num = ulp_as_mpfr_number(input);
+    return num.str();
+  }
+
+  template <typename T>
+  cpp::enable_if_t<cpp::is_floating_point_v<T>, double> ulp(T input) {
+    MPFRNumber num = ulp_as_mpfr_number(input);
+    return num.as<double>();
+  }
+};
+
+} // namespace mpfr
+} // namespace testing
+} // namespace LIBC_NAMESPACE_DECL
+
+#endif // LLVM_LIBC_UTILS_MPFRWRAPPER_MPCOMMON_H
diff --git a/libc/utils/MPFRWrapper/MPFRUtils.cpp b/libc/utils/MPFRWrapper/MPFRUtils.cpp
index 0dac497bb779a..775a3d1a31964 100644
--- a/libc/utils/MPFRWrapper/MPFRUtils.cpp
+++ b/libc/utils/MPFRWrapper/MPFRUtils.cpp
@@ -7,806 +7,17 @@
 //===----------------------------------------------------------------------===//
 
 #include "MPFRUtils.h"
+#include "MPCommon.h"
 
 #include "src/__support/CPP/array.h"
-#include "src/__support/CPP/string.h"
-#include "src/__support/CPP/string_view.h"
 #include "src/__support/CPP/stringstream.h"
-#include "src/__support/FPUtil/FPBits.h"
-#include "src/__support/FPUtil/cast.h"
 #include "src/__support/FPUtil/fpbits_str.h"
 #include "src/__support/macros/config.h"
 #include "src/__support/macros/properties/types.h"
 
-#include <stdint.h>
-
-#include "mpfr_inc.h"
-
-#ifdef LIBC_TYPES_FLOAT128_IS_NOT_LONG_DOUBLE
-extern "C" {
-int mpfr_set_float128(mpfr_ptr, float128, mpfr_rnd_t);
-float128 mpfr_get_float128(mpfr_srcptr, mpfr_rnd_t);
-}
-#endif
-
-template <typename T> using FPBits = LIBC_NAMESPACE::fputil::FPBits<T>;
-
 namespace LIBC_NAMESPACE_DECL {
 namespace testing {
 namespace mpfr {
-
-// A precision value which allows sufficiently large additional
-// precision compared to the floating point precision.
-template <typename T> struct ExtraPrecision;
-
-#ifdef LIBC_TYPES_HAS_FLOAT16
-template <> struct ExtraPrecision<float16> {
-  static constexpr unsigned int VALUE = 128;
-};
-#endif
-
-template <> struct ExtraPrecision<float> {
-  static constexpr unsigned int VALUE = 128;
-};
-
-template <> struct ExtraPrecision<double> {
-  static constexpr unsigned int VALUE = 256;
-};
-
-template <> struct ExtraPrecision<long double> {
-#ifdef LIBC_TYPES_LONG_DOUBLE_IS_FLOAT128
-  static constexpr unsigned int VALUE = 512;
-#else
-  static constexpr unsigned int VALUE = 256;
-#endif
-};
-
-#if defined(LIBC_TYPES_FLOAT128_IS_NOT_LONG_DOUBLE)
-template <> struct ExtraPrecision<float128> {
-  static constexpr unsigned int VALUE = 512;
-};
-#endif // LIBC_TYPES_FLOAT128_IS_NOT_LONG_DOUBLE
-
-// If the ulp tolerance is less than or equal to 0.5, we would check that the
-// result is rounded correctly with respect to the rounding mode by using the
-// same precision as the inputs.
-template <typename T>
-static inline unsigned int get_precision(double ulp_tolerance) {
-  if (ulp_tolerance <= 0.5) {
-    return LIBC_NAMESPACE::fputil::FPBits<T>::FRACTION_LEN + 1;
-  } else {
-    return ExtraPrecision<T>::VALUE;
-  }
-}
-
-static inline mpfr_rnd_t get_mpfr_rounding_mode(RoundingMode mode) {
-  switch (mode) {
-  case RoundingMode::Upward:
-    return MPFR_RNDU;
-    break;
-  case RoundingMode::Downward:
-    return MPFR_RNDD;
-    break;
-  case RoundingMode::TowardZero:
-    return MPFR_RNDZ;
-    break;
-  case RoundingMode::Nearest:
-    return MPFR_RNDN;
-    break;
-  }
-  __builtin_unreachable();
-}
-
-class MPFRNumber {
-  unsigned int mpfr_precision;
-  mpfr_rnd_t mpfr_rounding;
-
-  mpfr_t value;
-
-public:
-  MPFRNumber() : mpfr_precision(256), mpfr_rounding(MPFR_RNDN) {
-    mpfr_init2(value, mpfr_precision);
-  }
-
-  // We use explicit EnableIf specializations to disallow implicit
-  // conversions. Implicit conversions can potentially lead to loss of
-  // precision. We exceptionally allow implicit conversions from float16
-  // to float, as the MPFR API does not support float16, thus requiring
-  // conversion to a higher-precision format.
-  template <typename XType,
-            cpp::enable_if_t<cpp::is_same_v<float, XType>
-#ifdef LIBC_TYPES_HAS_FLOAT16
-                                 || cpp::is_same_v<float16, XType>
-#endif
-                             ,
-                             int> = 0>
-  explicit MPFRNumber(XType x,
-                      unsigned int precision = ExtraPrecision<XType>::VALUE,
-                      RoundingMode rounding = RoundingMode::Nearest)
-      : mpfr_precision(precision),
-        mpfr_rounding(get_mpfr_rounding_mode(rounding)) {
-    mpfr_init2(value, mpfr_precision);
-    mpfr_set_flt(value, x, mpfr_rounding);
-  }
-
-  template <typename XType,
-            cpp::enable_if_t<cpp::is_same_v<double, XType>, int> = 0>
-  explicit MPFRNumber(XType x,
-                      unsigned int precision = ExtraPrecision<XType>::VALUE,
-                      RoundingMode rounding = RoundingMode::Nearest)
-      : mpfr_precision(precision),
-        mpfr_rounding(get_mpfr_rounding_mode(rounding)) {
-    mpfr_init2(value, mpfr_precision);
-    mpfr_set_d(value, x, mpfr_rounding);
-  }
-
-  template <typename XType,
-            cpp::enable_if_t<cpp::is_same_v<long double, XType>, int> = 0>
-  explicit MPFRNumber(XType x,
-                      unsigned int precision = ExtraPrecision<XType>::VALUE,
-                      RoundingMode rounding = RoundingMode::Nearest)
-      : mpfr_precision(precision),
-        mpfr_rounding(get_mpfr_rounding_mode(rounding)) {
-    mpfr_init2(value, mpfr_precision);
-    mpfr_set_ld(value, x, mpfr_rounding);
-  }
-
-#ifdef LIBC_TYPES_FLOAT128_IS_NOT_LONG_DOUBLE
-  template <typename XType,
-            cpp::enable_if_t<cpp::is_same_v<float128, XType>, int> = 0>
-  explicit MPFRNumber(XType x,
-                      unsigned int precision = ExtraPrecision<XType>::VALUE,
-                      RoundingMode rounding = RoundingMode::Nearest)
-      : mpfr_precision(precision),
-        mpfr_rounding(get_mpfr_rounding_mode(rounding)) {
-    mpfr_init2(value, mpfr_precision);
-    mpfr_set_float128(value, x, mpfr_rounding);
-  }
-#endif // LIBC_TYPES_FLOAT128_IS_NOT_LONG_DOUBLE
-
-  template <typename XType,
-            cpp::enable_if_t<cpp::is_integral_v<XType>, int> = 0>
-  explicit MPFRNumber(XType x,
-                      unsigned int precision = ExtraPrecision<float>::VALUE,
-                      RoundingMode rounding = RoundingMode::Nearest)
-      : mpfr_precision(precision),
-        mpfr_rounding(get_mpfr_rounding_mode(rounding)) {
-    mpfr_init2(value, mpfr_precision);
-    mpfr_set_sj(value, x, mpfr_rounding);
-  }
-
-  MPFRNumber(const MPFRNumber &other)
-      : mpfr_precision(other.mpfr_precision),
-        mpfr_rounding(other.mpfr_rounding) {
-    mpfr_init2(value, mpfr_precision);
-    mpfr_set(value, other.value, mpfr_rounding);
-  }
-
-  MPFRNumber(const MPFRNumber &other, unsigned int precision)
-      : mpfr_precision(precision), mpfr_rounding(other.mpfr_rounding) {
-    mpfr_init2(value, mpfr_precision);
-    mpfr_set(value, other.value, mpfr_rounding);
-  }
-
-  ~MPFRNumber() { mpfr_clear(value); }
-
-  MPFRNumber &operator=(const MPFRNumber &rhs) {
-    mpfr_precision = rhs.mpfr_precision;
-    mpfr_rounding = rhs.mpfr_rounding;
-    mpfr_set(value, rhs.value, mpfr_rounding);
-    return *this;
-  }
-
-  bool is_nan() const { return mpfr_nan_p(value); }
-
-  MPFRNumber abs() const {
-    MPFRNumber result(*this);
-    mpfr_abs(result.value, value, mpfr_rounding);
-    return result;
-  }
-
-  MPFRNumber acos() const {
-    MPFRNumber result(*this);
-    mpfr_acos(result.value, value, mpfr_rounding);
-    return result;
-  }
-
-  MPFRNumber acosh() const {
-    MPFRNumber result(*this);
-    mpfr_acosh(result.value, value, mpfr_rounding);
-    return result;
-  }
-
-  MPFRNumber add(const MPFRNumber &b) const {
-    MPFRNumber result(*this);
-    mpfr_add(result.value, value, b.value, mpfr_rounding);
-    return result;
-  }
-
-  MPFRNumber asin() const {
-    MPFRNumber result(*this);
-    mpfr_asin(result.value, value, mpfr_rounding);
-    return result;
-  }
-
-  MPFRNumber asinh() const {
-    MPFRNumber result(*this);
-    mpfr_asinh(result.value, value, mpfr_rounding);
-    return result;
-  }
-
-  MPFRNumber atan() const {
-    MPFRNumber result(*this);
-    mpfr_atan(result.value, value, mpfr_rounding);
-    return result;
-  }
-
-  MPFRNumber atan2(const MPFRNumber &b) {
-    MPFRNumber result(*this);
-    mpfr_atan2(result.value, value, b.value, mpfr_rounding);
-    return result;
-  }
-
-  MPFRNumber atanh() const {
-    MPFRNumber result(*this);
-    mpfr_atanh(result.value, value, mpfr_rounding);
-    return result;
-  }
-
-  MPFRNumber cbrt() const {
-    MPFRNumber result(*this);
-    mpfr_cbrt(result.value, value, mpfr_rounding);
-    return result;
-  }
-
-  MPFRNumber ceil() const {
-    MPFRNumber result(*this);
-    mpfr_ceil(result.value, value);
-    return result;
-  }
-
-  MPFRNumber cos() const {
-    MPFRNumber result(*this);
-    mpfr_cos(result.value, value, mpfr_rounding);
-    return result;
-  }
-
-  MPFRNumber cosh() const {
-    MPFRNumber result(*this);
-    mpfr_cosh(result.value, value, mpfr_rounding);
-    return result;
-  }
-
-  MPFRNumber cospi() const {
-    MPFRNumber result(*this);
-
-#if MPFR_VERSION_MAJOR > 4 ||                                                  \
-    (MPFR_VERSION_MAJOR == 4 && MPFR_VERSION_MINOR >= 2)
-    mpfr_cospi(result.value, value, mpfr_rounding);
-    return result;
-#else
-    if (mpfr_integer_p(value)) {
-      mpz_t integer;
-      mpz_init(integer);
-      mpfr_get_z(integer, value, mpfr_rounding);
-
-      int d = mpz_tstbit(integer, 0);
-      mpfr_set_si(result.value, d ? -1 : 1, mpfr_rounding);
-      mpz_clear(integer);
-      return result;
-    }
-
-    MPFRNumber value_pi(0.0, 1280);
-    mpfr_const_pi(value_pi.value, MPFR_RNDN);
-    mpfr_mul(value_pi.value, value_pi.value, value, MPFR_RNDN);
-    mpfr_cos(result.value, value_pi.value, mpfr_rounding);
-
-    return result;
-#endif
-  }
-
-  MPFRNumber erf() const {
-    MPFRNumber result(*this);
-    mpfr_erf(result.value, value, mpfr_rounding);
-    return result;
-  }
-
-  MPFRNumber exp() const {
-    MPFRNumber result(*this);
-    mpfr_exp(result.value, value, mpfr_rounding);
-    return result;
-  }
-
-  MPFRNumber exp2() const {
-    MPFRNumber result(*this);
-    mpfr_exp2(result.value, value, mpfr_rounding);
-    return result;
-  }
-
-  MPFRNumber exp2m1() const {
-    // TODO: Only use mpfr_exp2m1 once CI and buildbots get MPFR >= 4.2.0.
-#if MPFR_VERSION_MAJOR > 4 ||                                                  \
-    (MPFR_VERSION_MAJOR == 4 && MPFR_VERSION_MINOR >= 2)
-    MPFRNumber result(*this);
-    mpfr_exp2m1(result.value, value, mpfr_rounding);
-    return result;
-#else
-    unsigned int prec = mpfr_precision * 3;
-    MPFRNumber result(*this, prec);
-
-    float f = mpfr_get_flt(abs().value, mpfr_rounding);
-    if (f > 0.5f && f < 0x1.0p30f) {
-      mpfr_exp2(result.value, value, mpfr_rounding);
-      mpfr_sub_ui(result.value, result.value, 1, mpfr_rounding);
-      return result;
-    }
-
-    MPFRNumber ln2(2.0f, prec);
-    // log(2)
-    mpfr_log(ln2.value, ln2.value, mpfr_rounding);
-    // x * log(2)
-    mpfr_mul(result.value, value, ln2.value, mpfr_rounding);
-    // e^(x * log(2)) - 1
-    int ex = mpfr_expm1(result.value, result.value, mpfr_rounding);
-    mpfr_subnormalize(result.value, ex, mpfr_rounding);
-    return result;
-#endif
-  }
-
-  MPFRNumber exp10() const {
-    MPFRNumber result(*this);
-    mpfr_exp10(result.value, value, mpfr_rounding);
-    return result;
-  }
-
-  MPFRNumber exp10m1() const {
-    // TODO: Only use mpfr_exp10m1 once CI and buildbots get MPFR >= 4.2.0.
-#if MPFR_VERSION_MAJOR > 4 ||                                                  \
-    (MPFR_VERSION_MAJOR == 4 && MPFR_VERSION_MINOR >= 2)
-    MPFRNumber result(*this);
-    mpfr_exp10m1(result.value, value, mpfr_rounding);
-    return result;
-#else
-    unsigned int prec = mpfr_precision * 3;
-    MPFRNumber result(*this, prec);
-
-    MPFRNumber ln10(10.0f, prec);
-    // log(10)
-    mpfr_log(ln10.value, ln10.value, mpfr_rounding);
-    // x * log(10)
-    mpfr_mul(result.value, value, ln10.value, mpfr_rounding);
-    // e^(x * log(10)) - 1
-    int ex = mpfr_expm1(result.value, result.value, mpfr_rounding);
-    mpfr_subnormalize(result.value, ex, mpfr_rounding);
-    return result;
-#endif
-  }
-
-  MPFRNumber expm1() const {
-    MPFRNumber result(*this);
-    mpfr_expm1(result.value, value, mpfr_rounding);
-    return result;
-  }
-
-  MPFRNumber div(const MPFRNumber &b) const {
-    MPFRNumber result(*this);
-    mpfr_div(result.value, value, b.value, mpfr_rounding);
-    return result;
-  }
-
-  MPFRNumber floor() const {
-    MPFRNumber result(*this);
-    mpfr_floor(result.value, value);
-    return result;
-  }
-
-  MPFRNumber fmod(const MPFRNumber &b) {
-    MPFRNumber result(*this);
-    mpfr_fmod(result.value, value, b.value, mpfr_rounding);
-    return result;
-  }
-
-  MPFRNumber frexp(int &exp) {
-    MPFRNumber result(*this);
-    mpfr_exp_t resultExp;
-    mpfr_frexp(&resultExp, result.value, value, mpfr_rounding);
-    exp = resultExp;
-    return result;
-  }
-
-  MPFRNumber hypot(const MPFRNumber &b) {
-    MPFRNumber result(*this);
-    mpfr_hypot(result.value, value, b.value, mpfr_rounding);
-    return result;
-  }
-
-  MPFRNumber log() const {
-    MPFRNumber result(*this);
-    mpfr_log(result.value, value, mpfr_rounding);
-    return result;
-  }
-
-  MPFRNumber log2() const {
-    MPFRNumber result(*this);
-    mpfr_log2(result.value, value, mpfr_rounding);
-    return result;
-  }
-
-  MPFRNumber log10() const {
-    MPFRNumber result(*this);
-    mpfr_log10(result.value, value, mpfr_rounding);
-    return result;
-  }
-
-  MPFRNumber log1p() const {
-    MPFRNumber result(*this);
-    mpfr_log1p(result.value, value, mpfr_rounding);
-    return result;
-  }
-
-  MPFRNumber pow(const MPFRNumber &b) {
-    MPFRNumber result(*this);
-    mpfr_pow(result.value, value, b.value, mpfr_rounding);
-    return result;
-  }
-
-  MPFRNumber remquo(const MPFRNumber &divisor, int &quotient) {
-    MPFRNumber remainder(*this);
-    long q;
-    mpfr_remquo(remainder.value, &q, value, divisor.value, mpfr_rounding);
-    quotient = q;
-    return remainder;
-  }
-
-  MPFRNumber round() const {
-    MPFRNumber result(*this);
-    mpfr_round(result.value, value);
-    return result;
-  }
-
-  MPFRNumber roundeven() const {
-    MPFRNumber result(*this);
-#if MPFR_VERSION_MAJOR >= 4
-    mpfr_roundeven(result.value, value);
-#else
-    mpfr_rint(result.value, value, MPFR_RNDN);
-#endif
-    return result;
-  }
-
-  bool round_to_long(long &result) const {
-    // We first calculate the rounded value. This way, when converting
-    // to long using mpfr_get_si, the rounding direction of MPFR_RNDN
-    // (or any other rounding mode), does not have an influence.
-    MPFRNumber roundedValue = round();
-    mpfr_clear_erangeflag();
-    result = mpfr_get_si(roundedValue.value, MPFR_RNDN);
-    return mpfr_erangeflag_p();
-  }
-
-  bool round_to_long(mpfr_rnd_t rnd, long &result) const {
-    MPFRNumber rint_result(*this);
-    mpfr_rint(rint_result.value, value, rnd);
-    return rint_result.round_to_long(result);
-  }
-
-  MPFRNumber rint(mpfr_rnd_t rnd) const {
-    MPFRNumber result(*this);
-    mpfr_rint(result.value, value, rnd);
-    return result;
-  }
-
-  MPFRNumber mod_2pi() const {
-    MPFRNumber result(0.0, 1280);
-    MPFRNumber _2pi(0.0, 1280);
-    mpfr_const_pi(_2pi.value, MPFR_RNDN);
-    mpfr_mul_si(_2pi.value, _2pi.value, 2, MPFR_RNDN);
-    mpfr_fmod(result.value, value, _2pi.value, MPFR_RNDN);
-    return result;
-  }
-
-  MPFRNumber mod_pi_over_2() const {
-    MPFRNumber result(0.0, 1280);
-    MPFRNumber pi_over_2(0.0, 1280);
-    mpfr_const_pi(pi_over_2.value, MPFR_RNDN);
-    mpfr_mul_d(pi_over_2.value, pi_over_2.value, 0.5, MPFR_RNDN);
-    mpfr_fmod(result.value, value, pi_over_2.value, MPFR_RNDN);
-    return result;
-  }
-
-  MPFRNumber mod_pi_over_4() const {
-    MPFRNumber result(0.0, 1280);
-    MPFRNumber pi_over_4(0.0, 1280);
-    mpfr_const_pi(pi_over_4.value, MPFR_RNDN);
-    mpfr_mul_d(pi_over_4.value, pi_over_4.value, 0.25, MPFR_RNDN);
-    mpfr_fmod(result.value, value, pi_over_4.value, MPFR_RNDN);
-    return result;
-  }
-
-  MPFRNumber sin() const {
-    MPFRNumber result(*this);
-    mpfr_sin(result.value, value, mpfr_rounding);
-    return result;
-  }
-
-  MPFRNumber sinpi() const {
-    MPFRNumber result(*this);
-
-#if MPFR_VERSION_MAJOR > 4 ||                                                  \
-    (MPFR_VERSION_MAJOR == 4 && MPFR_VERSION_MINOR >= 2)
-
-    mpfr_sinpi(result.value, value, mpfr_rounding);
-    return result;
-#else
-    if (mpfr_integer_p(value)) {
-      mpfr_set_si(result.value, 0, mpfr_rounding);
-      return result;
-    }
-
-    MPFRNumber value_mul_two(*this);
-    mpfr_mul_si(value_mul_two.value, value, 2, MPFR_RNDN);
-
-    if (mpfr_integer_p(value_mul_two.value)) {
-      auto d = mpfr_get_si(value, MPFR_RNDD);
-      mpfr_set_si(result.value, (d & 1) ? -1 : 1, mpfr_rounding);
-      return result;
-    }
-
-    MPFRNumber value_pi(0.0, 1280);
-    mpfr_const_pi(value_pi.value, MPFR_RNDN);
-    mpfr_mul(value_pi.value, value_pi.value, value, MPFR_RNDN);
-    mpfr_sin(result.value, value_pi.value, mpfr_rounding);
-    return result;
-#endif
-  }
-
-  MPFRNumber sinh() const {
-    MPFRNumber result(*this);
-    mpfr_sinh(result.value, value, mpfr_rounding);
-    return result;
-  }
-
-  MPFRNumber sqrt() const {
-    MPFRNumber result(*this);
-    mpfr_sqrt(result.value, value, mpfr_rounding);
-    return result;
-  }
-
-  MPFRNumber sub(const MPFRNumber &b) const {
-    MPFRNumber result(*this);
-    mpfr_sub(result.value, value, b.value, mpfr_rounding);
-    return result;
-  }
-
-  MPFRNumber tan() const {
-    MPFRNumber result(*this);
-    mpfr_tan(result.value, value, mpfr_rounding);
-    return result;
-  }
-
-  MPFRNumber tanh() const {
-    MPFRNumber result(*this);
-    mpfr_tanh(result.value, value, mpfr_rounding);
-    return result;
-  }
-
-  MPFRNumber tanpi() const {
-    MPFRNumber result(*this);
-
-#if MPFR_VERSION_MAJOR > 4 ||                                                  \
-    (MPFR_VERSION_MAJOR == 4 && MPFR_VERSION_MINOR >= 2)
-
-    mpfr_tanpi(result.value, value, mpfr_rounding);
-    return result;
-#else
-    MPFRNumber value_ret_exact(*this);
-    MPFRNumber value_one(*this);
-    mpfr_set_si(value_one.value, 1, MPFR_RNDN);
-    mpfr_fmod(value_ret_exact.value, value, value_one.value, mpfr_rounding);
-    mpfr_mul_si(value_ret_exact.value, value_ret_exact.value, 4, MPFR_RNDN);
-
-    if (mpfr_integer_p(value_ret_exact.value)) {
-      int mod = mpfr_get_si(value_ret_exact.value, MPFR_RNDN);
-      mod = (mod < 0 ? -1 * mod : mod);
-
-      switch (mod) {
-      case 0:
-        mpfr_set_si(result.value, 0, mpfr_rounding);
-        break;
-      case 1:
-        mpfr_set_si(result.value, (mpfr_signbit(value) ? -1 : 1),
-                    mpfr_rounding);
-        break;
-      case 2: {
-        auto d = mpfr_get_si(value, MPFR_RNDZ);
-        d += mpfr_sgn(value) > 0 ? 0 : 1;
-        mpfr_set_inf(result.value, (d & 1) ? -1 : 1);
-        break;
-      }
-      case 3:
-        mpfr_set_si(result.value, (mpfr_signbit(value) ? 1 : -1),
-                    mpfr_rounding);
-        break;
-      }
-
-      return result;
-    }
-
-    MPFRNumber value_pi(0.0, 1280);
-    mpfr_const_pi(value_pi.value, MPFR_RNDN);
-    mpfr_mul(value_pi.value, value_pi.value, value, MPFR_RNDN);
-    mpfr_tan(result.value, value_pi.value, mpfr_rounding);
-    return result;
-#endif
-  }
-
-  MPFRNumber trunc() const {
-    MPFRNumber result(*this);
-    mpfr_trunc(result.value, value);
-    return result;
-  }
-
-  MPFRNumber fma(const MPFRNumber &b, const MPFRNumber &c) {
-    MPFRNumber result(*this);
-    mpfr_fma(result.value, value, b.value, c.value, mpfr_rounding);
-    return result;
-  }
-
-  MPFRNumber mul(const MPFRNumber &b) {
-    MPFRNumber result(*this);
-    mpfr_mul(result.value, value, b.value, mpfr_rounding);
-    return result;
-  }
-
-  cpp::string str() const {
-    // 200 bytes should be more than sufficient to hold a 100-digit number
-    // plus additional bytes for the decimal point, '-' sign etc.
-    constexpr size_t printBufSize = 200;
-    char buffer[printBufSize];
-    mpfr_snprintf(buffer, printBufSize, "%100.50Rf", value);
-    cpp::string_view view(buffer);
-    // Trim whitespaces
-    const char whitespace = ' ';
-    while (!view.empty() && view.front() == whitespace)
-      view.remove_prefix(1);
-    while (!view.empty() && view.back() == whitespace)
-      view.remove_suffix(1);
-    return cpp::string(view.data());
-  }
-
-  // These functions are useful for debugging.
-  template <typename T> T as() const;
-
-  void dump(const char *msg) const { mpfr_printf("%s%.128g\n", msg, value); }
-
-  // Return the ULP (units-in-the-last-place) difference between the
-  // stored MPFR and a floating point number.
-  //
-  // We define ULP difference as follows:
-  //   If exponents of this value and the |input| are same, then:
-  //     ULP(this_value, input) = abs(this_value - input) / eps(input)
-  //   else:
-  //     max = max(abs(this_value), abs(input))
-  //     min = min(abs(this_value), abs(input))
-  //     maxExponent = exponent(max)
-  //     ULP(this_value, input) = (max - 2^maxExponent) / eps(max) +
-  //                              (2^maxExponent - min) / eps(min)
-  //
-  // Remarks:
-  // 1. A ULP of 0.0 will imply that the value is correctly rounded.
-  // 2. We expect that this value and the value to be compared (the [input]
-  //    argument) are reasonable close, and we will provide an upper bound
-  //    of ULP value for testing.  Morever, most of the fractional parts of
-  //    ULP value do not matter much, so using double as the return type
-  //    should be good enough.
-  // 3. For close enough values (values which don't diff in their exponent by
-  //    not more than 1), a ULP difference of N indicates a bit distance
-  //    of N between this number and [input].
-  // 4. A values of +0.0 and -0.0 are treated as equal.
-  template <typename T>
-  cpp::enable_if_t<cpp::is_floating_point_v<T>, MPFRNumber>
-  ulp_as_mpfr_number(T input) {
-    T thisAsT = as<T>();
-    if (thisAsT == input)
-      return MPFRNumber(0.0);
-
-    if (is_nan()) {
-      if (FPBits<T>(input).is_nan())
-        return MPFRNumber(0.0);
-      return MPFRNumber(FPBits<T>::inf().get_val());
-    }
-
-    int thisExponent = FPBits<T>(thisAsT).get_exponent();
-    int inputExponent = FPBits<T>(input).get_exponent();
-    // Adjust the exponents for denormal numbers.
-    if (FPBits<T>(thisAsT).is_subnormal())
-      ++thisExponent;
-    if (FPBits<T>(input).is_subnormal())
-      ++inputExponent;
-
-    if (thisAsT * input < 0 || thisExponent == inputExponent) {
-      MPFRNumber inputMPFR(input);
-      mpfr_sub(inputMPFR.value, value, inputMPFR.value, MPFR_RNDN);
-      mpfr_abs(inputMPFR.value, inputMPFR.value, MPFR_RNDN);
-      mpfr_mul_2si(inputMPFR.value, inputMPFR.value,
-                   -thisExponent + FPBits<T>::FRACTION_LEN, MPFR_RNDN);
-      return inputMPFR;
-    }
-
-    // If the control reaches here, it means that this number and input are
-    // of the same sign but different exponent. In such a case, ULP error is
-    // calculated as sum of two parts.
-    thisAsT = FPBits<T>(thisAsT).abs().get_val();
-    input = FPBits<T>(input).abs().get_val();
-    T min = thisAsT > input ? input : thisAsT;
-    T max = thisAsT > input ? thisAsT : input;
-    int minExponent = FPBits<T>(min).get_exponent();
-    int maxExponent = FPBits<T>(max).get_exponent();
-    // Adjust the exponents for denormal numbers.
-    if (FPBits<T>(min).is_subnormal())
-      ++minExponent;
-    if (FPBits<T>(max).is_subnormal())
-      ++maxExponent;
-
-    MPFRNumber minMPFR(min);
-    MPFRNumber maxMPFR(max);
-
-    MPFRNumber pivot(uint32_t(1));
-    mpfr_mul_2si(pivot.value, pivot.value, maxExponent, MPFR_RNDN);
-
-    mpfr_sub(minMPFR.value, pivot.value, minMPFR.value, MPFR_RNDN);
-    mpfr_mul_2si(minMPFR.value, minMPFR.value,
-                 -minExponent + FPBits<T>::FRACTION_LEN, MPFR_RNDN);
-
-    mpfr_sub(maxMPFR.value, maxMPFR.value, pivot.value, MPFR_RNDN);
-    mpfr_mul_2si(maxMPFR.value, maxMPFR.value,
-                 -maxExponent + FPBits<T>::FRACTION_LEN, MPFR_RNDN);
-
-    mpfr_add(minMPFR.value, minMPFR.value, maxMPFR.value, MPFR_RNDN);
-    return minMPFR;
-  }
-
-  template <typename T>
-  cpp::enable_if_t<cpp::is_floating_point_v<T>, cpp::string>
-  ulp_as_string(T input) {
-    MPFRNumber num = ulp_as_mpfr_number(input);
-    return num.str();
-  }
-
-  template <typename T>
-  cpp::enable_if_t<cpp::is_floating_point_v<T>, double> ulp(T input) {
-    MPFRNumber num = ulp_as_mpfr_number(input);
-    return num.as<double>();
-  }
-};
-
-template <> float MPFRNumber::as<float>() const {
-  return mpfr_get_flt(value, mpfr_rounding);
-}
-
-template <> double MPFRNumber::as<double>() const {
-  return mpfr_get_d(value, mpfr_rounding);
-}
-
-template <> long double MPFRNumber::as<long double>() const {
-  return mpfr_get_ld(value, mpfr_rounding);
-}
-
-#ifdef LIBC_TYPES_HAS_FLOAT16
-template <> float16 MPFRNumber::as<float16>() const {
-  // TODO: Either prove that this cast won't cause double-rounding errors, or
-  // find a better way to get a float16.
-  return fputil::cast<float16>(mpfr_get_d(value, mpfr_rounding));
-}
-#endif
-
-#ifdef LIBC_TYPES_FLOAT128_IS_NOT_LONG_DOUBLE
-template <> float128 MPFRNumber::as<float128>() const {
-  return mpfr_get_float128(value, mpfr_rounding);
-}
-
-#endif // LIBC_TYPES_FLOAT128_IS_NOT_LONG_DOUBLE
-
 namespace internal {
 
 template <typename InputType>
diff --git a/libc/utils/MPFRWrapper/MPFRUtils.h b/libc/utils/MPFRWrapper/MPFRUtils.h
index c7a57819f68b7..bc65f87c6b5ab 100644
--- a/libc/utils/MPFRWrapper/MPFRUtils.h
+++ b/libc/utils/MPFRWrapper/MPFRUtils.h
@@ -21,7 +21,7 @@ namespace testing {
 namespace mpfr {
 
 enum class Operation : int {
-  // Operations with take a single floating point number as input
+  // Operations which take a single floating point number as input
   // and produce a single floating point number as output. The input
   // and output floating point numbers are of the same kind.
   BeginUnaryOperationsSingleOutput,
@@ -87,10 +87,10 @@ enum class Operation : int {
   EndBinaryOperationsSingleOutput,
 
   // Operations which take two floating point numbers of the same type as
-  // input and produce two outputs. The first output is a floating nubmer of
-  // the same type as the inputs. The second output is af type 'int'.
+  // input and produce two outputs. The first output is a floating point number
+  // of the same type as the inputs. The second output is of type 'int'.
   BeginBinaryOperationsTwoOutputs,
-  RemQuo, // The first output, the floating point output, is the remainder.
+  RemQuo, // The first output(floating point) is the remainder.
   EndBinaryOperationsTwoOutputs,
 
   // Operations which take three floating point nubmers of the same type as
diff --git a/libc/utils/docgen/cpio.yaml b/libc/utils/docgen/cpio.yaml
new file mode 100644
index 0000000000000..b31c03778fba5
--- /dev/null
+++ b/libc/utils/docgen/cpio.yaml
@@ -0,0 +1,44 @@
+macros:
+  C_IRUSR:
+    in-latest-posix: ''
+  C_IWUSR:
+    in-latest-posix: ''
+  C_IXUSR:
+    in-latest-posix: ''
+  C_IRGRP:
+    in-latest-posix: ''
+  C_IWGRP:
+    in-latest-posix: ''
+  C_IXGRP:
+    in-latest-posix: ''
+  C_IROTH:
+    in-latest-posix: ''
+  C_IWOTH:
+    in-latest-posix: ''
+  C_IXOTH:
+    in-latest-posix: ''
+  C_ISUID:
+    in-latest-posix: ''
+  C_ISGID:
+    in-latest-posix: ''
+  C_ISVTX:
+    in-latest-posix: ''
+  C_ISDIR:
+    in-latest-posix: ''
+  C_ISFIFO:
+    in-latest-posix: ''
+  C_ISREG:
+    in-latest-posix: ''
+  C_ISBLK:
+    in-latest-posix: ''
+  C_ISCHR:
+    in-latest-posix: ''
+  C_ISCTG:
+    in-latest-posix: ''
+  C_ISLNK:
+    in-latest-posix: ''
+  C_ISSOCK:
+    in-latest-posix: ''
+  MAGIC:
+    in-latest-posix: ''
+
diff --git a/libc/utils/docgen/docgen.py b/libc/utils/docgen/docgen.py
index 09db284ef9282..5a57987b3c51e 100755
--- a/libc/utils/docgen/docgen.py
+++ b/libc/utils/docgen/docgen.py
@@ -57,6 +57,7 @@ def check_api(header: Header, api: Dict):
         "c-definition",
         "in-latest-posix",
         "removed-in-posix-2008",
+        "removed-in-posix-2024",
     ]
 
     # Validate macros
@@ -140,6 +141,10 @@ def print_functions_rst(header: Header, functions: Dict):
             print(
                 f"    - `removed in POSIX.1-2008 <https://pubs.opengroup.org/onlinepubs/007904875/functions/{name}.html>`__"
             )
+        elif "removed-in-posix-2024" in functions[name]:
+            print(
+                f"    - `removed in POSIX.1-2024 <https://pubs.opengroup.org/onlinepubs/9699919799.2018edition/functions/{name}.html>`__"
+            )
         else:
             print("    -")
 
diff --git a/libc/utils/docgen/endian.yaml b/libc/utils/docgen/endian.yaml
new file mode 100644
index 0000000000000..f100a11260667
--- /dev/null
+++ b/libc/utils/docgen/endian.yaml
@@ -0,0 +1,32 @@
+macros:
+  LITTLE_ENDIAN:
+    in-latest-posix: ''
+  BIG_ENDIAN:
+    in-latest-posix: ''
+  BYTE_ORDER:
+    in-latest-posix: ''
+  be16toh:
+    in-latest-posix: ''
+  be32toh:
+    in-latest-posix: ''
+  be64toh:
+    in-latest-posix: ''
+  htobe16:
+    in-latest-posix: ''
+  htobe32:
+    in-latest-posix: ''
+  htobe64:
+    in-latest-posix: ''
+  htole16:
+    in-latest-posix: ''
+  htole32:
+    in-latest-posix: ''
+  htole64:
+    in-latest-posix: ''
+  le16toh:
+    in-latest-posix: ''
+  le32toh:
+    in-latest-posix: ''
+  le64toh:
+    in-latest-posix: ''
+
diff --git a/libc/utils/docgen/sys/statvfs.yaml b/libc/utils/docgen/sys/statvfs.yaml
new file mode 100644
index 0000000000000..b13c3cb7203e5
--- /dev/null
+++ b/libc/utils/docgen/sys/statvfs.yaml
@@ -0,0 +1,11 @@
+functions:
+  fstatvfs:
+    in-latest-posix: ''
+  statvfs:
+    in-latest-posix: ''
+
+macros:
+  ST_RDONLY:
+    in-latest-posix: ''
+  ST_NOSUID:
+    in-latest-posix: ''
diff --git a/libc/utils/docgen/sys/time.yaml b/libc/utils/docgen/sys/time.yaml
index 1e3909adafea3..b7678f403cdcf 100644
--- a/libc/utils/docgen/sys/time.yaml
+++ b/libc/utils/docgen/sys/time.yaml
@@ -3,3 +3,9 @@ functions:
     in-latest-posix: ''
   utimes:
     in-latest-posix: ''
+  getitimer:
+    removed-in-posix-2024: ''
+  gettimeofday:
+    removed-in-posix-2024: ''
+  setitimer:
+    removed-in-posix-2024: ''
diff --git a/libc/utils/gpu/server/CMakeLists.txt b/libc/utils/gpu/server/CMakeLists.txt
index a109d603318b6..ae8a0d902f459 100644
--- a/libc/utils/gpu/server/CMakeLists.txt
+++ b/libc/utils/gpu/server/CMakeLists.txt
@@ -11,6 +11,7 @@ target_include_directories(llvmlibc_rpc_server PUBLIC ${CMAKE_CURRENT_SOURCE_DIR
 
 # Ignore unsupported clang attributes if we're using GCC.
 target_compile_options(llvmlibc_rpc_server PUBLIC
+                       $<$<CXX_COMPILER_ID:Clang>:-Wno-c99-extensions>
                        $<$<CXX_COMPILER_ID:GNU>:-Wno-attributes>)
 target_compile_definitions(llvmlibc_rpc_server PUBLIC
                            LIBC_COPT_USE_C_ASSERT
diff --git a/libclc/CMakeLists.txt b/libclc/CMakeLists.txt
index 2c2c7f16e2944..2978fadc2c29f 100644
--- a/libclc/CMakeLists.txt
+++ b/libclc/CMakeLists.txt
@@ -32,10 +32,6 @@ set_property(DIRECTORY APPEND PROPERTY CMAKE_CONFIGURE_DEPENDS
   spirv64/lib/SOURCES;
   # CLC internal libraries
   clc/lib/generic/SOURCES;
-  clc/lib/clspv/SOURCES;
-  clc/lib/clspv64/SOURCES;
-  clc/lib/spirv/SOURCES;
-  clc/lib/spirv64/SOURCES;
 )
 
 set( LIBCLC_MIN_LLVM 3.9.0 )
@@ -114,7 +110,7 @@ endforeach()
 if( TARGET llvm-spirv )
   get_host_tool_path( llvm-spirv LLVM_SPIRV llvm-spirv_exe llvm-spirv_target )
 else()
-  find_program( LLVM_SPIRV llvm-spirv PATHS ${LLVM_TOOLS_BINARY_DIR} NO_DEFAULT_PATH )
+  find_program( LLVM_SPIRV llvm-spirv HINTS ${LLVM_TOOLS_BINARY_DIR} )
   set( llvm-spirv_exe "${LLVM_SPIRV}" )
   set( llvm-spirv_target )
 endif()
@@ -266,30 +262,38 @@ foreach( t ${LIBCLC_TARGETS_TO_BUILD} )
   list( GET TRIPLE 1 VENDOR )
   list( GET TRIPLE 2 OS )
 
-  set( dirs )
+  set( opencl_dirs )
 
   if ( NOT ${ARCH} STREQUAL spirv AND NOT ${ARCH} STREQUAL spirv64 AND
        NOT ${ARCH} STREQUAL clspv AND NOT ${ARCH} STREQUAL clspv64)
-    LIST( APPEND dirs generic )
+    LIST( APPEND opencl_dirs generic )
   endif()
 
   if( ${ARCH} STREQUAL r600 OR ${ARCH} STREQUAL amdgcn )
-    list( APPEND dirs amdgpu )
+    list( APPEND opencl_dirs amdgpu )
   endif()
 
-  # nvptx is special
+  # Some targets' directories alias others
   if( ${ARCH} STREQUAL nvptx OR ${ARCH} STREQUAL nvptx64 )
     set( DARCH ptx )
+  elseif( ${ARCH} STREQUAL clspv OR ${ARCH} STREQUAL clspv64 )
+    set( DARCH clspv )
+  elseif( ${ARCH} STREQUAL spirv OR ${ARCH} STREQUAL spirv64 )
+    set( DARCH spirv )
+  elseif( ${ARCH} STREQUAL amdgcn-mesa3d )
+    set( DARCH amdgcn-amdhsa )
   else()
     set( DARCH ${ARCH} )
   endif()
 
   set( clc_lib_files )
+  set( clc_dirs ${dirs} generic )
+
   libclc_configure_lib_source(
     clc_lib_files
     CLC_INTERNAL
     LIB_ROOT_DIR clc
-    DIRS ${dirs} ${DARCH} ${DARCH}-${OS} ${DARCH}-${VENDOR}-${OS}
+    DIRS ${clc_dirs} ${DARCH} ${DARCH}-${OS} ${DARCH}-${VENDOR}-${OS}
   )
 
   set( opencl_lib_files )
@@ -306,7 +310,7 @@ foreach( t ${LIBCLC_TARGETS_TO_BUILD} )
 
   libclc_configure_lib_source(
     opencl_lib_files
-    DIRS ${dirs} ${DARCH} ${DARCH}-${OS} ${DARCH}-${VENDOR}-${OS}
+    DIRS ${opencl_dirs} ${DARCH} ${DARCH}-${OS} ${DARCH}-${VENDOR}-${OS}
   )
 
   foreach( d ${${t}_devices} )
@@ -347,7 +351,6 @@ foreach( t ${LIBCLC_TARGETS_TO_BUILD} )
     string( TOUPPER "CLC_${MACRO_ARCH}" CLC_TARGET_DEFINE )
 
     list( APPEND build_flags
-      -D__CLC_INTERNAL
       -D${CLC_TARGET_DEFINE}
       # All libclc builtin libraries see CLC headers
       -I${CMAKE_CURRENT_SOURCE_DIR}/clc/include
@@ -359,12 +362,14 @@ foreach( t ${LIBCLC_TARGETS_TO_BUILD} )
       list( APPEND build_flags -mcpu=${cpu} )
     endif()
 
+    set( clc_build_flags ${build_flags} -DCLC_INTERNAL )
+
     add_libclc_builtin_set(
       CLC_INTERNAL
       ARCH ${ARCH}
       ARCH_SUFFIX clc-${arch_suffix}
       TRIPLE ${clang_triple}
-      COMPILE_FLAGS ${build_flags}
+      COMPILE_FLAGS ${clc_build_flags}
       OPT_FLAGS ${opt_flags}
       LIB_FILES ${clc_lib_files}
     )
diff --git a/libclc/amdgcn-mesa3d b/libclc/amdgcn-mesa3d
deleted file mode 120000
index 400782833efe6..0000000000000
--- a/libclc/amdgcn-mesa3d
+++ /dev/null
@@ -1 +0,0 @@
-amdgcn-amdhsa
\ No newline at end of file
diff --git a/libclc/amdgcn/lib/SOURCES b/libclc/amdgcn/lib/SOURCES
index b235457f9ab7c..4ea66385fe50e 100644
--- a/libclc/amdgcn/lib/SOURCES
+++ b/libclc/amdgcn/lib/SOURCES
@@ -1,5 +1,4 @@
 cl_khr_int64_extended_atomics/minmax_helpers.ll
-integer/popcount.cl
 math/fmax.cl
 math/fmin.cl
 math/ldexp.cl
diff --git a/libclc/amdgcn/lib/integer/popcount.cl b/libclc/amdgcn/lib/integer/popcount.cl
deleted file mode 100644
index 3b493fbd146f0..0000000000000
--- a/libclc/amdgcn/lib/integer/popcount.cl
+++ /dev/null
@@ -1,6 +0,0 @@
-#include <clc/clc.h>
-#include <clc/utils.h>
-#include <integer/popcount.h>
-
-#define __CLC_BODY "popcount.inc"
-#include <clc/integer/gentype.inc>
diff --git a/libclc/amdgcn/lib/integer/popcount.inc b/libclc/amdgcn/lib/integer/popcount.inc
deleted file mode 100644
index 402ddb768c6a6..0000000000000
--- a/libclc/amdgcn/lib/integer/popcount.inc
+++ /dev/null
@@ -1,17 +0,0 @@
-_CLC_OVERLOAD _CLC_DEF __CLC_GENTYPE popcount(__CLC_GENTYPE x) {
-/* LLVM-4+ implements i16 ops for VI+ ASICs. However, ctpop implementation
- * is missing until r326535. Therefore we have to convert sub i32 types to uint
- * as a workaround. */
-#if __clang_major__ < 7 && __clang_major__ > 3 && __CLC_GENSIZE < 32
-	/* Prevent sign extension on uint conversion */
-	const __CLC_U_GENTYPE y = __CLC_XCONCAT(as_, __CLC_U_GENTYPE)(x);
-	/* Convert to uintX */
-	const __CLC_XCONCAT(uint, __CLC_VECSIZE) z = __CLC_XCONCAT(convert_uint, __CLC_VECSIZE)(y);
-	/* Call popcount on uintX type */
-	const __CLC_XCONCAT(uint, __CLC_VECSIZE) res = __clc_native_popcount(z);
-	/* Convert the result back to gentype. */
-	return __CLC_XCONCAT(convert_, __CLC_GENTYPE)(res);
-#else
-	return __clc_native_popcount(x);
-#endif
-}
diff --git a/libclc/clc/include/clc/common/clc_sign.h b/libclc/clc/include/clc/common/clc_sign.h
new file mode 100644
index 0000000000000..9e0984db7bb3b
--- /dev/null
+++ b/libclc/clc/include/clc/common/clc_sign.h
@@ -0,0 +1,12 @@
+#ifndef __CLC_COMMON_CLC_SIGN_H__
+#define __CLC_COMMON_CLC_SIGN_H__
+
+#define __CLC_FUNCTION __clc_sign
+#define __CLC_BODY <clc/math/unary_decl.inc>
+
+#include <clc/math/gentype.inc>
+
+#undef __CLC_BODY
+#undef __CLC_FUNCTION
+
+#endif // __CLC_COMMON_CLC_SIGN_H__
diff --git a/libclc/clc/include/clc/integer/clc_abs.h b/libclc/clc/include/clc/integer/clc_abs.h
index 31c62d311a006..59bd807b96060 100644
--- a/libclc/clc/include/clc/integer/clc_abs.h
+++ b/libclc/clc/include/clc/integer/clc_abs.h
@@ -1,14 +1,7 @@
 #ifndef __CLC_INTEGER_CLC_ABS_H__
 #define __CLC_INTEGER_CLC_ABS_H__
 
-#if defined(CLC_CLSPV) || defined(CLC_SPIRV)
-// clspv and spir-v targets provide their own OpenCL-compatible abs
-#define __clc_abs abs
-#else
-
 #define __CLC_BODY <clc/integer/clc_abs.inc>
 #include <clc/integer/gentype.inc>
 
-#endif
-
 #endif // __CLC_INTEGER_CLC_ABS_H__
diff --git a/libclc/clc/include/clc/integer/clc_abs_diff.h b/libclc/clc/include/clc/integer/clc_abs_diff.h
index 9c33fcff23b79..021a9b6bc45a0 100644
--- a/libclc/clc/include/clc/integer/clc_abs_diff.h
+++ b/libclc/clc/include/clc/integer/clc_abs_diff.h
@@ -1,14 +1,7 @@
 #ifndef __CLC_INTEGER_CLC_ABS_DIFF_H__
 #define __CLC_INTEGER_CLC_ABS_DIFF_H__
 
-#if defined(CLC_CLSPV) || defined(CLC_SPIRV)
-// clspv and spir-v targets provide their own OpenCL-compatible abs_diff
-#define __clc_abs_diff abs_diff
-#else
-
 #define __CLC_BODY <clc/integer/clc_abs_diff.inc>
 #include <clc/integer/gentype.inc>
 
-#endif
-
 #endif // __CLC_INTEGER_CLC_ABS_DIFF_H__
diff --git a/libclc/clc/include/clc/integer/clc_add_sat.h b/libclc/clc/include/clc/integer/clc_add_sat.h
new file mode 100644
index 0000000000000..8db19627f18e3
--- /dev/null
+++ b/libclc/clc/include/clc/integer/clc_add_sat.h
@@ -0,0 +1,12 @@
+#ifndef __CLC_INTEGER_CLC_ADD_SAT_H__
+#define __CLC_INTEGER_CLC_ADD_SAT_H__
+
+#define __CLC_FUNCTION __clc_add_sat
+#define __CLC_BODY <clc/shared/binary_decl.inc>
+
+#include <clc/integer/gentype.inc>
+
+#undef __CLC_BODY
+#undef __CLC_FUNCTION
+
+#endif // __CLC_INTEGER_CLC_ADD_SAT_H__
diff --git a/libclc/clc/include/clc/integer/clc_clz.h b/libclc/clc/include/clc/integer/clc_clz.h
new file mode 100644
index 0000000000000..a642391e46960
--- /dev/null
+++ b/libclc/clc/include/clc/integer/clc_clz.h
@@ -0,0 +1,12 @@
+#ifndef __CLC_INTEGER_CLC_CLZ_H__
+#define __CLC_INTEGER_CLC_CLZ_H__
+
+#define __CLC_FUNCTION __clc_clz
+#define __CLC_BODY <clc/shared/unary_decl.inc>
+
+#include <clc/integer/gentype.inc>
+
+#undef __CLC_BODY
+#undef __CLC_FUNCTION
+
+#endif // __CLC_INTEGER_CLC_CLZ_H__
diff --git a/libclc/clc/include/clc/integer/clc_hadd.h b/libclc/clc/include/clc/integer/clc_hadd.h
new file mode 100644
index 0000000000000..6693afbb8b803
--- /dev/null
+++ b/libclc/clc/include/clc/integer/clc_hadd.h
@@ -0,0 +1,12 @@
+#ifndef __CLC_INTEGER_CLC_HADD_H__
+#define __CLC_INTEGER_CLC_HADD_H__
+
+#define __CLC_FUNCTION __clc_hadd
+#define __CLC_BODY <clc/shared/binary_decl.inc>
+
+#include <clc/integer/gentype.inc>
+
+#undef __CLC_BODY
+#undef __CLC_FUNCTION
+
+#endif // __CLC_INTEGER_CLC_HADD_H__
diff --git a/libclc/clc/include/clc/integer/clc_mad24.h b/libclc/clc/include/clc/integer/clc_mad24.h
new file mode 100644
index 0000000000000..13d39d04d1a2a
--- /dev/null
+++ b/libclc/clc/include/clc/integer/clc_mad24.h
@@ -0,0 +1,12 @@
+#ifndef __CLC_INTEGER_CLC_MAD24_H__
+#define __CLC_INTEGER_CLC_MAD24_H__
+
+#define __CLC_FUNCTION __clc_mad24
+#define __CLC_BODY <clc/shared/ternary_decl.inc>
+
+#include <clc/integer/gentype24.inc>
+
+#undef __CLC_BODY
+#undef __CLC_FUNCTION
+
+#endif // __CLC_INTEGER_CLC_MAD24_H__
diff --git a/libclc/clc/include/clc/integer/clc_mad_hi.h b/libclc/clc/include/clc/integer/clc_mad_hi.h
new file mode 100644
index 0000000000000..24a590df6027a
--- /dev/null
+++ b/libclc/clc/include/clc/integer/clc_mad_hi.h
@@ -0,0 +1,8 @@
+#ifndef __CLC_INTEGER_CLC_MAD_HI_H__
+#define __CLC_INTEGER_CLC_MAD_HI_H__
+
+#include <clc/integer/clc_mul_hi.h>
+
+#define __clc_mad_hi(a, b, c) (__clc_mul_hi((a), (b)) + (c))
+
+#endif // __CLC_INTEGER_CLC_MAD_HI_H__
diff --git a/libclc/clc/include/clc/integer/clc_mad_sat.h b/libclc/clc/include/clc/integer/clc_mad_sat.h
new file mode 100644
index 0000000000000..c474d067b0f62
--- /dev/null
+++ b/libclc/clc/include/clc/integer/clc_mad_sat.h
@@ -0,0 +1,12 @@
+#ifndef __CLC_INTEGER_CLC_MAD_SAT_H__
+#define __CLC_INTEGER_CLC_MAD_SAT_H__
+
+#define __CLC_FUNCTION __clc_mad_sat
+#define __CLC_BODY <clc/shared/ternary_decl.inc>
+
+#include <clc/integer/gentype.inc>
+
+#undef __CLC_BODY
+#undef __CLC_FUNCTION
+
+#endif // __CLC_INTEGER_CLC_MAD_SAT_H__
diff --git a/libclc/clc/include/clc/integer/clc_mul24.h b/libclc/clc/include/clc/integer/clc_mul24.h
new file mode 100644
index 0000000000000..acab4e9da9eaa
--- /dev/null
+++ b/libclc/clc/include/clc/integer/clc_mul24.h
@@ -0,0 +1,12 @@
+#ifndef __CLC_INTEGER_CLC_MUL24_H__
+#define __CLC_INTEGER_CLC_MUL24_H__
+
+#define __CLC_FUNCTION __clc_mul24
+#define __CLC_BODY <clc/shared/binary_decl.inc>
+
+#include <clc/integer/gentype24.inc>
+
+#undef __CLC_BODY
+#undef __CLC_FUNCTION
+
+#endif // __CLC_INTEGER_CLC_MUL24_H__
diff --git a/libclc/clc/include/clc/integer/clc_mul_hi.h b/libclc/clc/include/clc/integer/clc_mul_hi.h
new file mode 100644
index 0000000000000..e395794066f96
--- /dev/null
+++ b/libclc/clc/include/clc/integer/clc_mul_hi.h
@@ -0,0 +1,12 @@
+#ifndef __CLC_INTEGER_CLC_MUL_HI_H__
+#define __CLC_INTEGER_CLC_MUL_HI_H__
+
+#define __CLC_FUNCTION __clc_mul_hi
+#define __CLC_BODY <clc/shared/binary_decl.inc>
+
+#include <clc/integer/gentype.inc>
+
+#undef __CLC_BODY
+#undef __CLC_FUNCTION
+
+#endif // __CLC_INTEGER_CLC_MUL_HI_H__
diff --git a/libclc/clc/include/clc/integer/clc_popcount.h b/libclc/clc/include/clc/integer/clc_popcount.h
new file mode 100644
index 0000000000000..50f6d173c70b4
--- /dev/null
+++ b/libclc/clc/include/clc/integer/clc_popcount.h
@@ -0,0 +1,12 @@
+#ifndef __CLC_INTEGER_CLC_POPCOUNT_H__
+#define __CLC_INTEGER_CLC_POPCOUNT_H__
+
+#define __CLC_FUNCTION __clc_popcount
+#define __CLC_BODY <clc/shared/unary_decl.inc>
+
+#include <clc/integer/gentype.inc>
+
+#undef __CLC_INTRINSIC
+#undef __CLC_FUNCTION
+
+#endif // __CLC_INTEGER_CLC_POPCOUNT_H__
diff --git a/libclc/clc/include/clc/integer/clc_rhadd.h b/libclc/clc/include/clc/integer/clc_rhadd.h
new file mode 100644
index 0000000000000..9f8d1f367ac05
--- /dev/null
+++ b/libclc/clc/include/clc/integer/clc_rhadd.h
@@ -0,0 +1,12 @@
+#ifndef __CLC_INTEGER_CLC_RHADD_H__
+#define __CLC_INTEGER_CLC_RHADD_H__
+
+#define __CLC_FUNCTION __clc_rhadd
+#define __CLC_BODY <clc/shared/binary_decl.inc>
+
+#include <clc/integer/gentype.inc>
+
+#undef __CLC_BODY
+#undef __CLC_FUNCTION
+
+#endif // __CLC_INTEGER_CLC_RHADD_H__
diff --git a/libclc/clc/include/clc/integer/clc_rotate.h b/libclc/clc/include/clc/integer/clc_rotate.h
new file mode 100644
index 0000000000000..21c945a9ae1bd
--- /dev/null
+++ b/libclc/clc/include/clc/integer/clc_rotate.h
@@ -0,0 +1,12 @@
+#ifndef __CLC_INTEGER_CLC_ROTATE_H__
+#define __CLC_INTEGER_CLC_ROTATE_H__
+
+#define __CLC_FUNCTION __clc_rotate
+#define __CLC_BODY <clc/shared/binary_decl.inc>
+
+#include <clc/integer/gentype.inc>
+
+#undef __CLC_BODY
+#undef __CLC_FUNCTION
+
+#endif // __CLC_INTEGER_CLC_ROTATE_H__
diff --git a/libclc/clc/include/clc/integer/clc_sub_sat.h b/libclc/clc/include/clc/integer/clc_sub_sat.h
new file mode 100644
index 0000000000000..95a8fd8873e99
--- /dev/null
+++ b/libclc/clc/include/clc/integer/clc_sub_sat.h
@@ -0,0 +1,12 @@
+#ifndef __CLC_INTEGER_CLC_SUB_SAT_H__
+#define __CLC_INTEGER_CLC_SUB_SAT_H__
+
+#define __CLC_FUNCTION __clc_sub_sat
+#define __CLC_BODY <clc/shared/binary_decl.inc>
+
+#include <clc/integer/gentype.inc>
+
+#undef __CLC_BODY
+#undef __CLC_FUNCTION
+
+#endif // __CLC_INTEGER_CLC_SUB_SAT_H__
diff --git a/libclc/clc/include/clc/integer/clc_upsample.h b/libclc/clc/include/clc/integer/clc_upsample.h
new file mode 100644
index 0000000000000..262e33af95488
--- /dev/null
+++ b/libclc/clc/include/clc/integer/clc_upsample.h
@@ -0,0 +1,32 @@
+#ifndef __CLC_INTEGER_CLC_UPSAMPLE_H__
+#define __CLC_INTEGER_CLC_UPSAMPLE_H__
+
+#include <clc/clcfunc.h>
+#include <clc/clctypes.h>
+
+#define __CLC_UPSAMPLE_DECL(BGENTYPE, GENTYPE, UGENTYPE)                       \
+  _CLC_OVERLOAD _CLC_DECL BGENTYPE __clc_upsample(GENTYPE hi, UGENTYPE lo);
+
+#define __CLC_UPSAMPLE_VEC(BGENTYPE, GENTYPE, UGENTYPE)                        \
+  __CLC_UPSAMPLE_DECL(BGENTYPE, GENTYPE, UGENTYPE)                             \
+  __CLC_UPSAMPLE_DECL(BGENTYPE##2, GENTYPE##2, UGENTYPE##2)                    \
+  __CLC_UPSAMPLE_DECL(BGENTYPE##3, GENTYPE##3, UGENTYPE##3)                    \
+  __CLC_UPSAMPLE_DECL(BGENTYPE##4, GENTYPE##4, UGENTYPE##4)                    \
+  __CLC_UPSAMPLE_DECL(BGENTYPE##8, GENTYPE##8, UGENTYPE##8)                    \
+  __CLC_UPSAMPLE_DECL(BGENTYPE##16, GENTYPE##16, UGENTYPE##16)
+
+#define __CLC_UPSAMPLE_TYPES()                                                 \
+  __CLC_UPSAMPLE_VEC(short, char, uchar)                                       \
+  __CLC_UPSAMPLE_VEC(ushort, uchar, uchar)                                     \
+  __CLC_UPSAMPLE_VEC(int, short, ushort)                                       \
+  __CLC_UPSAMPLE_VEC(uint, ushort, ushort)                                     \
+  __CLC_UPSAMPLE_VEC(long, int, uint)                                          \
+  __CLC_UPSAMPLE_VEC(ulong, uint, uint)
+
+__CLC_UPSAMPLE_TYPES()
+
+#undef __CLC_UPSAMPLE_TYPES
+#undef __CLC_UPSAMPLE_DECL
+#undef __CLC_UPSAMPLE_VEC
+
+#endif // __CLC_INTEGER_CLC_UPSAMPLE_H__
diff --git a/libclc/clc/include/clc/integer/definitions.h b/libclc/clc/include/clc/integer/definitions.h
new file mode 100644
index 0000000000000..1b51c11a93f80
--- /dev/null
+++ b/libclc/clc/include/clc/integer/definitions.h
@@ -0,0 +1,24 @@
+#ifndef __CLC_INTEGER_DEFINITIONS_H__
+#define __CLC_INTEGER_DEFINITIONS_H__
+
+#define CHAR_BIT 8
+#define INT_MAX 2147483647
+#define INT_MIN (-2147483647 - 1)
+#define LONG_MAX 0x7fffffffffffffffL
+#define LONG_MIN (-0x7fffffffffffffffL - 1)
+#define CHAR_MAX SCHAR_MAX
+#define CHAR_MIN SCHAR_MIN
+#define SCHAR_MAX 127
+#define SCHAR_MIN (-127 - 1)
+#define SHRT_MAX 32767
+#define SHRT_MIN (-32767 - 1)
+#define UCHAR_MAX 255
+#define UCHAR_MIN 0
+#define USHRT_MAX 65535
+#define USHRT_MIN 0
+#define UINT_MAX 0xffffffff
+#define UINT_MIN 0
+#define ULONG_MAX 0xffffffffffffffffUL
+#define ULONG_MIN 0UL
+
+#endif // __CLC_INTEGER_DEFINITIONS_H__
diff --git a/libclc/clc/include/clc/integer/gentype24.inc b/libclc/clc/include/clc/integer/gentype24.inc
new file mode 100644
index 0000000000000..ad34c3818b1d6
--- /dev/null
+++ b/libclc/clc/include/clc/integer/gentype24.inc
@@ -0,0 +1,137 @@
+#include <clc/clcfunc.h>
+#include <clc/clctypes.h>
+
+#define __CLC_GENSIZE 32
+#undef __CLC_SCALAR_GENTYPE
+#define __CLC_SCALAR_GENTYPE int
+
+#define __CLC_GENTYPE int
+#define __CLC_U_GENTYPE uint
+#define __CLC_S_GENTYPE int
+#define __CLC_SCALAR 1
+#define __CLC_VECSIZE
+#include __CLC_BODY
+#undef __CLC_VECSIZE
+#undef __CLC_SCALAR
+#undef __CLC_GENTYPE
+#undef __CLC_U_GENTYPE
+#undef __CLC_S_GENTYPE
+
+#define __CLC_GENTYPE int2
+#define __CLC_U_GENTYPE uint2
+#define __CLC_S_GENTYPE int2
+#define __CLC_VECSIZE 2
+#include __CLC_BODY
+#undef __CLC_VECSIZE
+#undef __CLC_GENTYPE
+#undef __CLC_U_GENTYPE
+#undef __CLC_S_GENTYPE
+
+#define __CLC_GENTYPE int3
+#define __CLC_U_GENTYPE uint3
+#define __CLC_S_GENTYPE int3
+#define __CLC_VECSIZE 3
+#include __CLC_BODY
+#undef __CLC_VECSIZE
+#undef __CLC_GENTYPE
+#undef __CLC_U_GENTYPE
+#undef __CLC_S_GENTYPE
+
+#define __CLC_GENTYPE int4
+#define __CLC_U_GENTYPE uint4
+#define __CLC_S_GENTYPE int4
+#define __CLC_VECSIZE 4
+#include __CLC_BODY
+#undef __CLC_VECSIZE
+#undef __CLC_GENTYPE
+#undef __CLC_U_GENTYPE
+#undef __CLC_S_GENTYPE
+
+#define __CLC_GENTYPE int8
+#define __CLC_U_GENTYPE uint8
+#define __CLC_S_GENTYPE int8
+#define __CLC_VECSIZE 8
+#include __CLC_BODY
+#undef __CLC_VECSIZE
+#undef __CLC_GENTYPE
+#undef __CLC_U_GENTYPE
+#undef __CLC_S_GENTYPE
+
+#define __CLC_GENTYPE int16
+#define __CLC_U_GENTYPE uint16
+#define __CLC_S_GENTYPE int16
+#define __CLC_VECSIZE 16
+#include __CLC_BODY
+#undef __CLC_VECSIZE
+#undef __CLC_GENTYPE
+#undef __CLC_U_GENTYPE
+#undef __CLC_S_GENTYPE
+
+#undef __CLC_SCALAR_GENTYPE
+#define __CLC_SCALAR_GENTYPE uint
+
+#define __CLC_GENTYPE uint
+#define __CLC_U_GENTYPE uint
+#define __CLC_S_GENTYPE int
+#define __CLC_SCALAR 1
+#define __CLC_VECSIZE
+#include __CLC_BODY
+#undef __CLC_VECSIZE
+#undef __CLC_SCALAR
+#undef __CLC_GENTYPE
+#undef __CLC_U_GENTYPE
+#undef __CLC_S_GENTYPE
+
+#define __CLC_GENTYPE uint2
+#define __CLC_U_GENTYPE uint2
+#define __CLC_S_GENTYPE int2
+#define __CLC_VECSIZE 2
+#include __CLC_BODY
+#undef __CLC_VECSIZE
+#undef __CLC_GENTYPE
+#undef __CLC_U_GENTYPE
+#undef __CLC_S_GENTYPE
+
+#define __CLC_GENTYPE uint3
+#define __CLC_U_GENTYPE uint3
+#define __CLC_S_GENTYPE int3
+#define __CLC_VECSIZE 3
+#include __CLC_BODY
+#undef __CLC_VECSIZE
+#undef __CLC_GENTYPE
+#undef __CLC_U_GENTYPE
+#undef __CLC_S_GENTYPE
+
+#define __CLC_GENTYPE uint4
+#define __CLC_U_GENTYPE uint4
+#define __CLC_S_GENTYPE int4
+#define __CLC_VECSIZE 4
+#include __CLC_BODY
+#undef __CLC_VECSIZE
+#undef __CLC_GENTYPE
+#undef __CLC_U_GENTYPE
+#undef __CLC_S_GENTYPE
+
+#define __CLC_GENTYPE uint8
+#define __CLC_U_GENTYPE uint8
+#define __CLC_S_GENTYPE int8
+#define __CLC_VECSIZE 8
+#include __CLC_BODY
+#undef __CLC_VECSIZE
+#undef __CLC_GENTYPE
+#undef __CLC_U_GENTYPE
+#undef __CLC_S_GENTYPE
+
+#define __CLC_GENTYPE uint16
+#define __CLC_U_GENTYPE uint16
+#define __CLC_S_GENTYPE int16
+#define __CLC_VECSIZE 16
+#include __CLC_BODY
+#undef __CLC_VECSIZE
+#undef __CLC_GENTYPE
+#undef __CLC_U_GENTYPE
+#undef __CLC_S_GENTYPE
+
+#undef __CLC_GENSIZE
+#undef __CLC_SCALAR_GENTYPE
+#undef __CLC_BODY
diff --git a/libclc/clc/include/clc/math/clc_copysign.h b/libclc/clc/include/clc/math/clc_copysign.h
new file mode 100644
index 0000000000000..2b08acf73795f
--- /dev/null
+++ b/libclc/clc/include/clc/math/clc_copysign.h
@@ -0,0 +1,12 @@
+#ifndef __CLC_MATH_CLC_COPYSIGN_H__
+#define __CLC_MATH_CLC_COPYSIGN_H__
+
+#define __CLC_BODY <clc/shared/binary_decl.inc>
+#define __CLC_FUNCTION __clc_copysign
+
+#include <clc/math/gentype.inc>
+
+#undef __CLC_BODY
+#undef __CLC_FUNCTION
+
+#endif // __CLC_MATH_CLC_COPYSIGN_H__
diff --git a/libclc/clc/include/clc/math/clc_mad.h b/libclc/clc/include/clc/math/clc_mad.h
index 3eb718e87f370..5427c0215688e 100644
--- a/libclc/clc/include/clc/math/clc_mad.h
+++ b/libclc/clc/include/clc/math/clc_mad.h
@@ -1,7 +1,7 @@
 #ifndef __CLC_MATH_CLC_MAD_H__
 #define __CLC_MATH_CLC_MAD_H__
 
-#define __CLC_BODY <clc/math/ternary_decl.inc>
+#define __CLC_BODY <clc/shared/ternary_decl.inc>
 #define __CLC_FUNCTION __clc_mad
 
 #include <clc/math/gentype.inc>
diff --git a/libclc/clc/include/clc/math/clc_subnormal_config.h b/libclc/clc/include/clc/math/clc_subnormal_config.h
new file mode 100644
index 0000000000000..eaab5aaee985b
--- /dev/null
+++ b/libclc/clc/include/clc/math/clc_subnormal_config.h
@@ -0,0 +1,32 @@
+/*
+ * Copyright (c) 2015 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+#ifndef __CLC_MATH_CLC_SUBNORMAL_CONFIG_H__
+#define __CLC_MATH_CLC_SUBNORMAL_CONFIG_H__
+
+#include <clc/clcfunc.h>
+
+_CLC_DECL bool __clc_subnormals_disabled();
+_CLC_DECL bool __clc_fp16_subnormals_supported();
+_CLC_DECL bool __clc_fp32_subnormals_supported();
+_CLC_DECL bool __clc_fp64_subnormals_supported();
+
+#endif // __CLC_MATH_CLC_SUBNORMAL_CONFIG_H__
diff --git a/libclc/clc/include/clc/math/gentype.inc b/libclc/clc/include/clc/math/gentype.inc
index 87719f2d9bc0e..3c80f1c6172ad 100644
--- a/libclc/clc/include/clc/math/gentype.inc
+++ b/libclc/clc/include/clc/math/gentype.inc
@@ -3,55 +3,69 @@
 
 #define __CLC_SCALAR_GENTYPE float
 #define __CLC_FPSIZE 32
+#define __CLC_FP_LIT(x) x##F
 
 #define __CLC_GENTYPE float
 #define __CLC_INTN int
+#define __CLC_BIT_INTN int
 #define __CLC_SCALAR
 #include __CLC_BODY
 #undef __CLC_GENTYPE
+#undef __CLC_BIT_INTN
 #undef __CLC_INTN
 #undef __CLC_SCALAR
 
 #define __CLC_GENTYPE float2
 #define __CLC_INTN int2
+#define __CLC_BIT_INTN int2
 #define __CLC_VECSIZE 2
 #include __CLC_BODY
 #undef __CLC_VECSIZE
 #undef __CLC_GENTYPE
+#undef __CLC_BIT_INTN
 #undef __CLC_INTN
 
 #define __CLC_GENTYPE float3
 #define __CLC_INTN int3
+#define __CLC_BIT_INTN int3
 #define __CLC_VECSIZE 3
 #include __CLC_BODY
 #undef __CLC_VECSIZE
 #undef __CLC_GENTYPE
+#undef __CLC_BIT_INTN
 #undef __CLC_INTN
 
 #define __CLC_GENTYPE float4
 #define __CLC_INTN int4
+#define __CLC_BIT_INTN int4
 #define __CLC_VECSIZE 4
 #include __CLC_BODY
 #undef __CLC_VECSIZE
 #undef __CLC_GENTYPE
+#undef __CLC_BIT_INTN
 #undef __CLC_INTN
 
 #define __CLC_GENTYPE float8
 #define __CLC_INTN int8
+#define __CLC_BIT_INTN int8
 #define __CLC_VECSIZE 8
 #include __CLC_BODY
 #undef __CLC_VECSIZE
 #undef __CLC_GENTYPE
+#undef __CLC_BIT_INTN
 #undef __CLC_INTN
 
 #define __CLC_GENTYPE float16
 #define __CLC_INTN int16
+#define __CLC_BIT_INTN int16
 #define __CLC_VECSIZE 16
 #include __CLC_BODY
 #undef __CLC_VECSIZE
 #undef __CLC_GENTYPE
+#undef __CLC_BIT_INTN
 #undef __CLC_INTN
 
+#undef __CLC_FP_LIT
 #undef __CLC_FPSIZE
 #undef __CLC_SCALAR_GENTYPE
 
@@ -61,55 +75,69 @@
 
 #define __CLC_SCALAR_GENTYPE double
 #define __CLC_FPSIZE 64
+#define __CLC_FP_LIT(x) (x)
 
 #define __CLC_SCALAR
 #define __CLC_GENTYPE double
 #define __CLC_INTN int
+#define __CLC_BIT_INTN long
 #include __CLC_BODY
 #undef __CLC_GENTYPE
+#undef __CLC_BIT_INTN
 #undef __CLC_INTN
 #undef __CLC_SCALAR
 
 #define __CLC_GENTYPE double2
 #define __CLC_INTN int2
+#define __CLC_BIT_INTN long2
 #define __CLC_VECSIZE 2
 #include __CLC_BODY
 #undef __CLC_VECSIZE
 #undef __CLC_GENTYPE
+#undef __CLC_BIT_INTN
 #undef __CLC_INTN
 
 #define __CLC_GENTYPE double3
 #define __CLC_INTN int3
+#define __CLC_BIT_INTN long3
 #define __CLC_VECSIZE 3
 #include __CLC_BODY
 #undef __CLC_VECSIZE
 #undef __CLC_GENTYPE
+#undef __CLC_BIT_INTN
 #undef __CLC_INTN
 
 #define __CLC_GENTYPE double4
 #define __CLC_INTN int4
+#define __CLC_BIT_INTN long4
 #define __CLC_VECSIZE 4
 #include __CLC_BODY
 #undef __CLC_VECSIZE
 #undef __CLC_GENTYPE
+#undef __CLC_BIT_INTN
 #undef __CLC_INTN
 
 #define __CLC_GENTYPE double8
 #define __CLC_INTN int8
+#define __CLC_BIT_INTN long8
 #define __CLC_VECSIZE 8
 #include __CLC_BODY
 #undef __CLC_VECSIZE
 #undef __CLC_GENTYPE
+#undef __CLC_BIT_INTN
 #undef __CLC_INTN
 
 #define __CLC_GENTYPE double16
 #define __CLC_INTN int16
+#define __CLC_BIT_INTN long16
 #define __CLC_VECSIZE 16
 #include __CLC_BODY
 #undef __CLC_VECSIZE
 #undef __CLC_GENTYPE
+#undef __CLC_BIT_INTN
 #undef __CLC_INTN
 
+#undef __CLC_FP_LIT
 #undef __CLC_FPSIZE
 #undef __CLC_SCALAR_GENTYPE
 #endif
@@ -121,55 +149,69 @@
 
 #define __CLC_SCALAR_GENTYPE half
 #define __CLC_FPSIZE 16
+#define __CLC_FP_LIT(x) x##H
 
 #define __CLC_SCALAR
 #define __CLC_GENTYPE half
 #define __CLC_INTN int
+#define __CLC_BIT_INTN short
 #include __CLC_BODY
 #undef __CLC_GENTYPE
+#undef __CLC_BIT_INTN
 #undef __CLC_INTN
 #undef __CLC_SCALAR
 
 #define __CLC_GENTYPE half2
 #define __CLC_INTN int2
+#define __CLC_BIT_INTN short2
 #define __CLC_VECSIZE 2
 #include __CLC_BODY
 #undef __CLC_VECSIZE
 #undef __CLC_GENTYPE
+#undef __CLC_BIT_INTN
 #undef __CLC_INTN
 
 #define __CLC_GENTYPE half3
 #define __CLC_INTN int3
+#define __CLC_BIT_INTN short3
 #define __CLC_VECSIZE 3
 #include __CLC_BODY
 #undef __CLC_VECSIZE
 #undef __CLC_GENTYPE
+#undef __CLC_BIT_INTN
 #undef __CLC_INTN
 
 #define __CLC_GENTYPE half4
 #define __CLC_INTN int4
+#define __CLC_BIT_INTN short4
 #define __CLC_VECSIZE 4
 #include __CLC_BODY
 #undef __CLC_VECSIZE
 #undef __CLC_GENTYPE
+#undef __CLC_BIT_INTN
 #undef __CLC_INTN
 
 #define __CLC_GENTYPE half8
 #define __CLC_INTN int8
+#define __CLC_BIT_INTN short8
 #define __CLC_VECSIZE 8
 #include __CLC_BODY
 #undef __CLC_VECSIZE
 #undef __CLC_GENTYPE
+#undef __CLC_BIT_INTN
 #undef __CLC_INTN
 
 #define __CLC_GENTYPE half16
 #define __CLC_INTN int16
+#define __CLC_BIT_INTN short16
 #define __CLC_VECSIZE 16
 #include __CLC_BODY
 #undef __CLC_VECSIZE
 #undef __CLC_GENTYPE
+#undef __CLC_BIT_INTN
 #undef __CLC_INTN
 
+#undef __CLC_FP_LIT
 #undef __CLC_FPSIZE
 #undef __CLC_SCALAR_GENTYPE
 #endif
diff --git a/libclc/clc/include/clc/math/math.h b/libclc/clc/include/clc/math/math.h
new file mode 100644
index 0000000000000..ed37af237bf82
--- /dev/null
+++ b/libclc/clc/include/clc/math/math.h
@@ -0,0 +1,117 @@
+/*
+ * Copyright (c) 2014 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+
+#ifndef __CLC_MATH_MATH_H__
+#define __CLC_MATH_MATH_H__
+
+#include <clc/clc_as_type.h>
+#include <clc/clcfunc.h>
+#include <clc/math/clc_subnormal_config.h>
+
+#define SNAN 0x001
+#define QNAN 0x002
+#define NINF 0x004
+#define NNOR 0x008
+#define NSUB 0x010
+#define NZER 0x020
+#define PZER 0x040
+#define PSUB 0x080
+#define PNOR 0x100
+#define PINF 0x200
+
+#if (defined __AMDGCN__ || defined __R600__) && !defined __HAS_FMAF__
+#define HAVE_HW_FMA32() (0)
+#elif defined(CLC_SPIRV)
+bool __attribute__((noinline)) __clc_runtime_has_hw_fma32(void);
+#define HAVE_HW_FMA32() __clc_runtime_has_hw_fma32()
+#else
+#define HAVE_HW_FMA32() (1)
+#endif
+
+#define HAVE_BITALIGN() (0)
+#define HAVE_FAST_FMA32() (0)
+
+#define MATH_DIVIDE(X, Y) ((X) / (Y))
+#define MATH_RECIP(X) (1.0f / (X))
+#define MATH_SQRT(X) sqrt(X)
+
+#define SIGNBIT_SP32 0x80000000
+#define EXSIGNBIT_SP32 0x7fffffff
+#define EXPBITS_SP32 0x7f800000
+#define MANTBITS_SP32 0x007fffff
+#define ONEEXPBITS_SP32 0x3f800000
+#define TWOEXPBITS_SP32 0x40000000
+#define HALFEXPBITS_SP32 0x3f000000
+#define IMPBIT_SP32 0x00800000
+#define QNANBITPATT_SP32 0x7fc00000
+#define INDEFBITPATT_SP32 0xffc00000
+#define PINFBITPATT_SP32 0x7f800000
+#define NINFBITPATT_SP32 0xff800000
+#define EXPBIAS_SP32 127
+#define EXPSHIFTBITS_SP32 23
+#define BIASEDEMIN_SP32 1
+#define EMIN_SP32 -126
+#define BIASEDEMAX_SP32 254
+#define EMAX_SP32 127
+#define LAMBDA_SP32 1.0e30
+#define MANTLENGTH_SP32 24
+#define BASEDIGITS_SP32 7
+
+_CLC_OVERLOAD _CLC_INLINE float __clc_flush_denormal_if_not_supported(float x) {
+  int ix = __clc_as_int(x);
+  if (!__clc_fp32_subnormals_supported() && ((ix & EXPBITS_SP32) == 0) &&
+      ((ix & MANTBITS_SP32) != 0)) {
+    ix &= SIGNBIT_SP32;
+    x = __clc_as_float(ix);
+  }
+  return x;
+}
+
+#ifdef cl_khr_fp64
+
+#define SIGNBIT_DP64 0x8000000000000000L
+#define EXSIGNBIT_DP64 0x7fffffffffffffffL
+#define EXPBITS_DP64 0x7ff0000000000000L
+#define MANTBITS_DP64 0x000fffffffffffffL
+#define ONEEXPBITS_DP64 0x3ff0000000000000L
+#define TWOEXPBITS_DP64 0x4000000000000000L
+#define HALFEXPBITS_DP64 0x3fe0000000000000L
+#define IMPBIT_DP64 0x0010000000000000L
+#define QNANBITPATT_DP64 0x7ff8000000000000L
+#define INDEFBITPATT_DP64 0xfff8000000000000L
+#define PINFBITPATT_DP64 0x7ff0000000000000L
+#define NINFBITPATT_DP64 0xfff0000000000000L
+#define EXPBIAS_DP64 1023
+#define EXPSHIFTBITS_DP64 52
+#define BIASEDEMIN_DP64 1
+#define EMIN_DP64 -1022
+#define BIASEDEMAX_DP64 2046 /* 0x7fe */
+#define EMAX_DP64 1023       /* 0x3ff */
+#define LAMBDA_DP64 1.0e300
+#define MANTLENGTH_DP64 53
+#define BASEDIGITS_DP64 15
+
+#endif // cl_khr_fp64
+
+#define ALIGNED(x) __attribute__((aligned(x)))
+
+#endif // __CLC_MATH_MATH_H__
diff --git a/libclc/generic/lib/math/tables.h b/libclc/clc/include/clc/math/tables.h
similarity index 81%
rename from libclc/generic/lib/math/tables.h
rename to libclc/clc/include/clc/math/tables.h
index ea5221e1f72b0..71a332144c474 100644
--- a/libclc/generic/lib/math/tables.h
+++ b/libclc/clc/include/clc/math/tables.h
@@ -20,32 +20,30 @@
  * THE SOFTWARE.
  */
 
+#ifndef __CLC_MATH_TABLES_H__
+#define __CLC_MATH_TABLES_H__
+
 #include <clc/clctypes.h>
 
 #define TABLE_SPACE __constant
 
 #define TABLE_MANGLE(NAME) __clc_##NAME
 
-#define DECLARE_TABLE(TYPE,NAME,LENGTH) \
-    TABLE_SPACE TYPE NAME [ LENGTH ]
+#define DECLARE_TABLE(TYPE, NAME, LENGTH) TABLE_SPACE TYPE NAME[LENGTH]
 
-#define TABLE_FUNCTION(TYPE,TABLE,NAME) \
-    TYPE TABLE_MANGLE(NAME)(size_t idx) { \
-        return TABLE[idx]; \
-    }
+#define TABLE_FUNCTION(TYPE, TABLE, NAME)                                      \
+  TYPE TABLE_MANGLE(NAME)(size_t idx) { return TABLE[idx]; }
 
-#define TABLE_FUNCTION_DECL(TYPE, NAME) \
-    TYPE TABLE_MANGLE(NAME)(size_t idx);
+#define TABLE_FUNCTION_DECL(TYPE, NAME) TYPE TABLE_MANGLE(NAME)(size_t idx);
 
-#define USE_TABLE(NAME, IDX) \
-    TABLE_MANGLE(NAME)(IDX)
+#define USE_TABLE(NAME, IDX) TABLE_MANGLE(NAME)(IDX)
 
 TABLE_FUNCTION_DECL(float2, loge_tbl);
 TABLE_FUNCTION_DECL(float, log_inv_tbl);
 TABLE_FUNCTION_DECL(float2, log_inv_tbl_ep);
 TABLE_FUNCTION_DECL(float2, log2_tbl);
 TABLE_FUNCTION_DECL(float2, log10_tbl);
-TABLE_FUNCTION_DECL(uint4,  pibits_tbl);
+TABLE_FUNCTION_DECL(uint4, pibits_tbl);
 TABLE_FUNCTION_DECL(float2, sinhcosh_tbl);
 TABLE_FUNCTION_DECL(float2, cbrt_tbl);
 TABLE_FUNCTION_DECL(float, exp_tbl);
@@ -67,3 +65,5 @@ TABLE_FUNCTION_DECL(double2, powlog_tbl);
 TABLE_FUNCTION_DECL(double2, log_f_inv_tbl);
 
 #endif // cl_khr_fp64
+
+#endif // __CLC_MATH_TABLES_H__
diff --git a/libclc/clc/include/clc/relational/clc_all.h b/libclc/clc/include/clc/relational/clc_all.h
index 7be3d132dd53d..2ffced19ba0e5 100644
--- a/libclc/clc/include/clc/relational/clc_all.h
+++ b/libclc/clc/include/clc/relational/clc_all.h
@@ -1,11 +1,6 @@
 #ifndef __CLC_RELATIONAL_CLC_ALL_H__
 #define __CLC_RELATIONAL_CLC_ALL_H__
 
-#if defined(CLC_CLSPV) || defined(CLC_SPIRV)
-// clspv and spir-v targets provide their own OpenCL-compatible all
-#define __clc_all all
-#else
-
 #include <clc/clcfunc.h>
 #include <clc/clctypes.h>
 
@@ -27,6 +22,4 @@ _CLC_VECTOR_ALL_DECL(long)
 #undef _CLC_ALL_DECL
 #undef _CLC_VECTOR_ALL_DECL
 
-#endif
-
 #endif // __CLC_RELATIONAL_CLC_ALL_H__
diff --git a/libclc/clc/include/clc/relational/clc_any.h b/libclc/clc/include/clc/relational/clc_any.h
index 27dbffeb2eecd..2f554334d9bac 100644
--- a/libclc/clc/include/clc/relational/clc_any.h
+++ b/libclc/clc/include/clc/relational/clc_any.h
@@ -1,11 +1,6 @@
 #ifndef __CLC_RELATIONAL_CLC_ANY_H__
 #define __CLC_RELATIONAL_CLC_ANY_H__
 
-#if defined(CLC_CLSPV) || defined(CLC_SPIRV)
-// clspv and spir-v targets provide their own OpenCL-compatible any
-#define __clc_any any
-#else
-
 #include <clc/clcfunc.h>
 #include <clc/clctypes.h>
 
@@ -27,6 +22,4 @@ _CLC_VECTOR_ANY_DECL(long)
 #undef _CLC_ANY_DECL
 #undef _CLC_VECTOR_ANY_DECL
 
-#endif
-
 #endif // __CLC_RELATIONAL_CLC_ANY_H__
diff --git a/libclc/clc/include/clc/relational/clc_isequal.h b/libclc/clc/include/clc/relational/clc_isequal.h
index 0f31fb9530a14..84bf0974dbbf5 100644
--- a/libclc/clc/include/clc/relational/clc_isequal.h
+++ b/libclc/clc/include/clc/relational/clc_isequal.h
@@ -1,11 +1,6 @@
 #ifndef __CLC_RELATIONAL_CLC_ISEQUAL_H__
 #define __CLC_RELATIONAL_CLC_ISEQUAL_H__
 
-#if defined(CLC_CLSPV) || defined(CLC_SPIRV)
-// clspv and spir-v targets provide their own OpenCL-compatible isequal
-#define __clc_isequal isequal
-#else
-
 #include <clc/clcfunc.h>
 #include <clc/clctypes.h>
 
@@ -37,6 +32,4 @@ _CLC_VECTOR_ISEQUAL_DECL(half, short)
 #undef _CLC_ISEQUAL_DECL
 #undef _CLC_VECTOR_ISEQUAL_DECL
 
-#endif
-
 #endif //  __CLC_RELATIONAL_CLC_ISEQUAL_H__
diff --git a/libclc/clc/include/clc/relational/clc_isfinite.h b/libclc/clc/include/clc/relational/clc_isfinite.h
index 3ed276e07a2f1..82bcc6ec2da27 100644
--- a/libclc/clc/include/clc/relational/clc_isfinite.h
+++ b/libclc/clc/include/clc/relational/clc_isfinite.h
@@ -1,11 +1,6 @@
 #ifndef __CLC_RELATIONAL_CLC_ISFINITE_H__
 #define __CLC_RELATIONAL_CLC_ISFINITE_H__
 
-#if defined(CLC_CLSPV) || defined(CLC_SPIRV)
-// clspv and spir-v targets provide their own OpenCL-compatible isfinite
-#define __clc_isfinite isfinite
-#else
-
 #define __CLC_FUNCTION __clc_isfinite
 #define __CLC_BODY <clc/relational/unary_decl.inc>
 
@@ -14,6 +9,4 @@
 #undef __CLC_BODY
 #undef __CLC_FUNCTION
 
-#endif
-
 #endif // __CLC_RELATIONAL_CLC_ISFINITE_H__
diff --git a/libclc/clc/include/clc/relational/clc_isgreater.h b/libclc/clc/include/clc/relational/clc_isgreater.h
index b51d59aeb5499..31961e4c51679 100644
--- a/libclc/clc/include/clc/relational/clc_isgreater.h
+++ b/libclc/clc/include/clc/relational/clc_isgreater.h
@@ -1,11 +1,6 @@
 #ifndef __CLC_RELATIONAL_CLC_ISGREATER_H__
 #define __CLC_RELATIONAL_CLC_ISGREATER_H__
 
-#if defined(CLC_CLSPV) || defined(CLC_SPIRV)
-// clspv and spir-v targets provide their own OpenCL-compatible isgreater
-#define __clc_isgreater isgreater
-#else
-
 #define __CLC_FUNCTION __clc_isgreater
 #define __CLC_BODY <clc/relational/binary_decl.inc>
 
@@ -14,6 +9,4 @@
 #undef __CLC_BODY
 #undef __CLC_FUNCTION
 
-#endif
-
 #endif // __CLC_RELATIONAL_CLC_ISGREATER_H__
diff --git a/libclc/clc/include/clc/relational/clc_isgreaterequal.h b/libclc/clc/include/clc/relational/clc_isgreaterequal.h
index b7ffce151847f..0e072fad09655 100644
--- a/libclc/clc/include/clc/relational/clc_isgreaterequal.h
+++ b/libclc/clc/include/clc/relational/clc_isgreaterequal.h
@@ -1,11 +1,6 @@
 #ifndef __CLC_RELATIONAL_CLC_ISGREATEREQUAL_H__
 #define __CLC_RELATIONAL_CLC_ISGREATEREQUAL_H__
 
-#if defined(CLC_CLSPV) || defined(CLC_SPIRV)
-// clspv and spir-v targets provide their own OpenCL-compatible isgreaterequal
-#define __clc_isgreaterequal isgreaterequal
-#else
-
 #define __CLC_FUNCTION __clc_isgreaterequal
 #define __CLC_BODY <clc/relational/binary_decl.inc>
 
@@ -14,6 +9,4 @@
 #undef __CLC_BODY
 #undef __CLC_FUNCTION
 
-#endif
-
 #endif // __CLC_RELATIONAL_CLC_ISGREATEREQUAL_H__
diff --git a/libclc/clc/include/clc/relational/clc_isinf.h b/libclc/clc/include/clc/relational/clc_isinf.h
index 3f60bec5654a2..b666953d4a8e6 100644
--- a/libclc/clc/include/clc/relational/clc_isinf.h
+++ b/libclc/clc/include/clc/relational/clc_isinf.h
@@ -1,11 +1,6 @@
 #ifndef __CLC_RELATIONAL_CLC_ISINF_H__
 #define __CLC_RELATIONAL_CLC_ISINF_H__
 
-#if defined(CLC_CLSPV) || defined(CLC_SPIRV)
-// clspv and spir-v targets provide their own OpenCL-compatible isinf
-#define __clc_isinf isinf
-#else
-
 #include <clc/clcfunc.h>
 #include <clc/clctypes.h>
 
@@ -37,6 +32,4 @@ _CLC_VECTOR_ISINF_DECL(short, half)
 #undef _CLC_ISINF_DECL
 #undef _CLC_VECTOR_ISINF_DECL
 
-#endif
-
 #endif // __CLC_RELATIONAL_CLC_ISINF_H__
diff --git a/libclc/clc/include/clc/relational/clc_isless.h b/libclc/clc/include/clc/relational/clc_isless.h
index c6950aa61ad90..482fddfe4f8af 100644
--- a/libclc/clc/include/clc/relational/clc_isless.h
+++ b/libclc/clc/include/clc/relational/clc_isless.h
@@ -1,11 +1,6 @@
 #ifndef __CLC_RELATIONAL_CLC_ISLESS_H__
 #define __CLC_RELATIONAL_CLC_ISLESS_H__
 
-#if defined(CLC_CLSPV) || defined(CLC_SPIRV)
-// clspv and spir-v targets provide their own OpenCL-compatible isless
-#define __clc_isless isless
-#else
-
 #define __CLC_FUNCTION __clc_isless
 #define __CLC_BODY <clc/relational/binary_decl.inc>
 
@@ -14,6 +9,4 @@
 #undef __CLC_BODY
 #undef __CLC_FUNCTION
 
-#endif
-
 #endif // __CLC_RELATIONAL_CLC_ISLESS_H__
diff --git a/libclc/clc/include/clc/relational/clc_islessequal.h b/libclc/clc/include/clc/relational/clc_islessequal.h
index 7efac163e106a..520f3d9c6ffd6 100644
--- a/libclc/clc/include/clc/relational/clc_islessequal.h
+++ b/libclc/clc/include/clc/relational/clc_islessequal.h
@@ -1,11 +1,6 @@
 #ifndef __CLC_RELATIONAL_CLC_ISLESSEQUAL_H__
 #define __CLC_RELATIONAL_CLC_ISLESSEQUAL_H__
 
-#if defined(CLC_CLSPV) || defined(CLC_SPIRV)
-// clspv and spir-v targets provide their own OpenCL-compatible islessequal
-#define __clc_islessequal islessequal
-#else
-
 #define __CLC_FUNCTION __clc_islessequal
 #define __CLC_BODY <clc/relational/binary_decl.inc>
 
@@ -14,6 +9,4 @@
 #undef __CLC_BODY
 #undef __CLC_FUNCTION
 
-#endif
-
 #endif // __CLC_RELATIONAL_CLC_ISLESSEQUAL_H__
diff --git a/libclc/clc/include/clc/relational/clc_islessgreater.h b/libclc/clc/include/clc/relational/clc_islessgreater.h
index df3c5e513c86c..e90eadbbca5e5 100644
--- a/libclc/clc/include/clc/relational/clc_islessgreater.h
+++ b/libclc/clc/include/clc/relational/clc_islessgreater.h
@@ -1,11 +1,6 @@
 #ifndef __CLC_RELATIONAL_CLC_ISLESSGREATER_H__
 #define __CLC_RELATIONAL_CLC_ISLESSGREATER_H__
 
-#if defined(CLC_CLSPV) || defined(CLC_SPIRV)
-// clspv and spir-v targets provide their own OpenCL-compatible islessgreater
-#define __clc_islessgreater islessgreater
-#else
-
 #define __CLC_FUNCTION __clc_islessgreater
 #define __CLC_BODY <clc/relational/binary_decl.inc>
 
@@ -14,6 +9,4 @@
 #undef __CLC_BODY
 #undef __CLC_FUNCTION
 
-#endif
-
 #endif // __CLC_RELATIONAL_CLC_ISLESSGREATER_H__
diff --git a/libclc/clc/include/clc/relational/clc_isnormal.h b/libclc/clc/include/clc/relational/clc_isnormal.h
index 48ee6b83a5711..269abf0037411 100644
--- a/libclc/clc/include/clc/relational/clc_isnormal.h
+++ b/libclc/clc/include/clc/relational/clc_isnormal.h
@@ -1,11 +1,6 @@
 #ifndef __CLC_RELATIONAL_CLC_ISNORMAL_H__
 #define __CLC_RELATIONAL_CLC_ISNORMAL_H__
 
-#if defined(CLC_CLSPV) || defined(CLC_SPIRV)
-// clspv and spir-v targets provide their own OpenCL-compatible isnormal
-#define __clc_isnormal isnormal
-#else
-
 #define __CLC_FUNCTION __clc_isnormal
 #define __CLC_BODY <clc/relational/unary_decl.inc>
 
@@ -14,6 +9,4 @@
 #undef __CLC_BODY
 #undef __CLC_FUNCTION
 
-#endif
-
 #endif // __CLC_RELATIONAL_CLC_ISNORMAL_H__
diff --git a/libclc/clc/include/clc/relational/clc_isnotequal.h b/libclc/clc/include/clc/relational/clc_isnotequal.h
index 55c1bd91b2dd5..598657658ec58 100644
--- a/libclc/clc/include/clc/relational/clc_isnotequal.h
+++ b/libclc/clc/include/clc/relational/clc_isnotequal.h
@@ -1,11 +1,6 @@
 #ifndef __CLC_RELATIONAL_CLC_ISNOTEQUAL_H__
 #define __CLC_RELATIONAL_CLC_ISNOTEQUAL_H__
 
-#if defined(CLC_CLSPV) || defined(CLC_SPIRV)
-// clspv and spir-v targets provide their own OpenCL-compatible isnotequal
-#define __clc_isnotequal isnotequal
-#else
-
 #define __CLC_FUNCTION __clc_isnotequal
 #define __CLC_BODY <clc/relational/binary_decl.inc>
 
@@ -14,6 +9,4 @@
 #undef __CLC_BODY
 #undef __CLC_FUNCTION
 
-#endif
-
 #endif // __CLC_RELATIONAL_CLC_ISNOTEQUAL_H__
diff --git a/libclc/clc/include/clc/relational/clc_isordered.h b/libclc/clc/include/clc/relational/clc_isordered.h
index 5ce2bfe334027..f4363d3d8a832 100644
--- a/libclc/clc/include/clc/relational/clc_isordered.h
+++ b/libclc/clc/include/clc/relational/clc_isordered.h
@@ -1,11 +1,6 @@
 #ifndef __CLC_RELATIONAL_CLC_ISORDERED_H__
 #define __CLC_RELATIONAL_CLC_ISORDERED_H__
 
-#if defined(CLC_CLSPV) || defined(CLC_SPIRV)
-// clspv and spir-v targets provide their own OpenCL-compatible isordered
-#define __clc_isordered isordered
-#else
-
 #define __CLC_FUNCTION __clc_isordered
 #define __CLC_BODY <clc/relational/binary_decl.inc>
 
@@ -14,6 +9,4 @@
 #undef __CLC_BODY
 #undef __CLC_FUNCTION
 
-#endif
-
 #endif // __CLC_RELATIONAL_CLC_ISORDERED_H__
diff --git a/libclc/clc/include/clc/relational/clc_isunordered.h b/libclc/clc/include/clc/relational/clc_isunordered.h
index 305d2b4e9131f..e7f01826d5cc9 100644
--- a/libclc/clc/include/clc/relational/clc_isunordered.h
+++ b/libclc/clc/include/clc/relational/clc_isunordered.h
@@ -1,11 +1,6 @@
 #ifndef __CLC_RELATIONAL_CLC_ISUNORDERED_H__
 #define __CLC_RELATIONAL_CLC_ISUNORDERED_H__
 
-#if defined(CLC_CLSPV) || defined(CLC_SPIRV)
-// clspv and spir-v targets provide their own OpenCL-compatible isunordered
-#define __clc_isunordered isunordered
-#else
-
 #define __CLC_FUNCTION __clc_isunordered
 #define __CLC_BODY <clc/relational/binary_decl.inc>
 
@@ -14,6 +9,4 @@
 #undef __CLC_BODY
 #undef __CLC_FUNCTION
 
-#endif
-
 #endif // __CLC_RELATIONAL_CLC_ISUNORDERED_H__
diff --git a/libclc/clc/include/clc/relational/clc_signbit.h b/libclc/clc/include/clc/relational/clc_signbit.h
index 45a7112c9eb96..55561dd834871 100644
--- a/libclc/clc/include/clc/relational/clc_signbit.h
+++ b/libclc/clc/include/clc/relational/clc_signbit.h
@@ -1,11 +1,6 @@
 #ifndef __CLC_RELATIONAL_CLC_SIGNBIT_H__
 #define __CLC_RELATIONAL_CLC_SIGNBIT_H__
 
-#if defined(CLC_CLSPV) || defined(CLC_SPIRV)
-// clspv and spir-v targets provide their own OpenCL-compatible signbit
-#define __clc_signbit signbit
-#else
-
 #define __CLC_FUNCTION __clc_signbit
 #define __CLC_BODY <clc/relational/unary_decl.inc>
 
@@ -14,6 +9,4 @@
 #undef __CLC_BODY
 #undef __CLC_FUNCTION
 
-#endif
-
 #endif // __CLC_RELATIONAL_CLC_SIGNBIT_H__
diff --git a/libclc/clc/include/clc/relational/relational.h b/libclc/clc/include/clc/relational/relational.h
index 54241b6493c8e..2210d64bdac7b 100644
--- a/libclc/clc/include/clc/relational/relational.h
+++ b/libclc/clc/include/clc/relational/relational.h
@@ -6,140 +6,55 @@
  * when the result is true.
  */
 
-#define _CLC_DEFINE_RELATIONAL_UNARY_SCALAR(RET_TYPE, FUNCTION, BUILTIN_NAME,  \
-                                            ARG_TYPE)                          \
+#define _CLC_DEFINE_SIMPLE_RELATIONAL_BINARY(RET_TYPE, RET_TYPE_VEC, FUNCTION, \
+                                             ARG1_TYPE, ARG2_TYPE)             \
+  _CLC_DEF _CLC_OVERLOAD RET_TYPE FUNCTION(ARG1_TYPE x, ARG2_TYPE y) {         \
+    return _CLC_RELATIONAL_OP(x, y);                                           \
+  }                                                                            \
+  _CLC_DEF _CLC_OVERLOAD RET_TYPE_VEC##2 FUNCTION(ARG1_TYPE##2 x,              \
+                                                  ARG2_TYPE##2 y) {            \
+    return _CLC_RELATIONAL_OP(x, y);                                           \
+  }                                                                            \
+  _CLC_DEF _CLC_OVERLOAD RET_TYPE_VEC##3 FUNCTION(ARG1_TYPE##3 x,              \
+                                                  ARG2_TYPE##3 y) {            \
+    return _CLC_RELATIONAL_OP(x, y);                                           \
+  }                                                                            \
+  _CLC_DEF _CLC_OVERLOAD RET_TYPE_VEC##4 FUNCTION(ARG1_TYPE##4 x,              \
+                                                  ARG2_TYPE##4 y) {            \
+    return _CLC_RELATIONAL_OP(x, y);                                           \
+  }                                                                            \
+  _CLC_DEF _CLC_OVERLOAD RET_TYPE_VEC##8 FUNCTION(ARG1_TYPE##8 x,              \
+                                                  ARG2_TYPE##8 y) {            \
+    return _CLC_RELATIONAL_OP(x, y);                                           \
+  }                                                                            \
+  _CLC_DEF _CLC_OVERLOAD RET_TYPE_VEC##16 FUNCTION(ARG1_TYPE##16 x,            \
+                                                   ARG2_TYPE##16 y) {          \
+    return _CLC_RELATIONAL_OP(x, y);                                           \
+  }
+
+#define fcNan (__FPCLASS_SNAN | __FPCLASS_QNAN)
+#define fcInf (__FPCLASS_POSINF | __FPCLASS_NEGINF)
+#define fcNormal (__FPCLASS_POSNORMAL | __FPCLASS_NEGNORMAL)
+#define fcPosFinite                                                            \
+  (__FPCLASS_POSNORMAL | __FPCLASS_POSSUBNORMAL | __FPCLASS_POSZERO)
+#define fcNegFinite                                                            \
+  (__FPCLASS_NEGNORMAL | __FPCLASS_NEGSUBNORMAL | __FPCLASS_NEGZERO)
+#define fcFinite (fcPosFinite | fcNegFinite)
+
+#define _CLC_DEFINE_ISFPCLASS_VEC(RET_TYPE, FUNCTION, MASK, ARG_TYPE)          \
   _CLC_DEF _CLC_OVERLOAD RET_TYPE FUNCTION(ARG_TYPE x) {                       \
-    return BUILTIN_NAME(x);                                                    \
+    return (RET_TYPE)(__builtin_isfpclass(x, (MASK)) != (RET_TYPE)0);          \
   }
 
-#define _CLC_DEFINE_RELATIONAL_UNARY_VEC2(RET_TYPE, FUNCTION, ARG_TYPE)        \
+#define _CLC_DEFINE_ISFPCLASS(RET_TYPE, VEC_RET_TYPE, FUNCTION, MASK,          \
+                              ARG_TYPE)                                        \
   _CLC_DEF _CLC_OVERLOAD RET_TYPE FUNCTION(ARG_TYPE x) {                       \
-    return (RET_TYPE)((RET_TYPE){FUNCTION(x.lo), FUNCTION(x.hi)} !=            \
-                      (RET_TYPE)0);                                            \
-  }
-
-#define _CLC_DEFINE_RELATIONAL_UNARY_VEC3(RET_TYPE, FUNCTION, ARG_TYPE)        \
-  _CLC_DEF _CLC_OVERLOAD RET_TYPE FUNCTION(ARG_TYPE x) {                       \
-    return (RET_TYPE)((RET_TYPE){FUNCTION(x.s0), FUNCTION(x.s1),               \
-                                 FUNCTION(x.s2)} != (RET_TYPE)0);              \
-  }
-
-#define _CLC_DEFINE_RELATIONAL_UNARY_VEC4(RET_TYPE, FUNCTION, ARG_TYPE)        \
-  _CLC_DEF _CLC_OVERLOAD RET_TYPE FUNCTION(ARG_TYPE x) {                       \
-    return (RET_TYPE)((RET_TYPE){FUNCTION(x.s0), FUNCTION(x.s1),               \
-                                 FUNCTION(x.s2),                               \
-                                 FUNCTION(x.s3)} != (RET_TYPE)0);              \
-  }
-
-#define _CLC_DEFINE_RELATIONAL_UNARY_VEC8(RET_TYPE, FUNCTION, ARG_TYPE)        \
-  _CLC_DEF _CLC_OVERLOAD RET_TYPE FUNCTION(ARG_TYPE x) {                       \
-    return (                                                                   \
-        RET_TYPE)((RET_TYPE){FUNCTION(x.s0), FUNCTION(x.s1), FUNCTION(x.s2),   \
-                             FUNCTION(x.s3), FUNCTION(x.s4), FUNCTION(x.s5),   \
-                             FUNCTION(x.s6), FUNCTION(x.s7)} != (RET_TYPE)0);  \
-  }
-
-#define _CLC_DEFINE_RELATIONAL_UNARY_VEC16(RET_TYPE, FUNCTION, ARG_TYPE)       \
-  _CLC_DEF _CLC_OVERLOAD RET_TYPE FUNCTION(ARG_TYPE x) {                       \
-    return (                                                                   \
-        RET_TYPE)((RET_TYPE){FUNCTION(x.s0), FUNCTION(x.s1), FUNCTION(x.s2),   \
-                             FUNCTION(x.s3), FUNCTION(x.s4), FUNCTION(x.s5),   \
-                             FUNCTION(x.s6), FUNCTION(x.s7), FUNCTION(x.s8),   \
-                             FUNCTION(x.s9), FUNCTION(x.sa), FUNCTION(x.sb),   \
-                             FUNCTION(x.sc), FUNCTION(x.sd), FUNCTION(x.se),   \
-                             FUNCTION(x.sf)} != (RET_TYPE)0);                  \
-  }
-
-#define _CLC_DEFINE_RELATIONAL_UNARY_VEC_ALL(RET_TYPE, FUNCTION, ARG_TYPE)     \
-  _CLC_DEFINE_RELATIONAL_UNARY_VEC2(RET_TYPE##2, FUNCTION, ARG_TYPE##2)        \
-  _CLC_DEFINE_RELATIONAL_UNARY_VEC3(RET_TYPE##3, FUNCTION, ARG_TYPE##3)        \
-  _CLC_DEFINE_RELATIONAL_UNARY_VEC4(RET_TYPE##4, FUNCTION, ARG_TYPE##4)        \
-  _CLC_DEFINE_RELATIONAL_UNARY_VEC8(RET_TYPE##8, FUNCTION, ARG_TYPE##8)        \
-  _CLC_DEFINE_RELATIONAL_UNARY_VEC16(RET_TYPE##16, FUNCTION, ARG_TYPE##16)
-
-#define _CLC_DEFINE_RELATIONAL_UNARY(RET_TYPE, FUNCTION, BUILTIN_FUNCTION,     \
-                                     ARG_TYPE)                                 \
-  _CLC_DEFINE_RELATIONAL_UNARY_SCALAR(RET_TYPE, FUNCTION, BUILTIN_FUNCTION,    \
-                                      ARG_TYPE)                                \
-  _CLC_DEFINE_RELATIONAL_UNARY_VEC_ALL(RET_TYPE, FUNCTION, ARG_TYPE)
-
-#define _CLC_DEFINE_RELATIONAL_BINARY_SCALAR(RET_TYPE, FUNCTION, BUILTIN_NAME, \
-                                             ARG0_TYPE, ARG1_TYPE)             \
-  _CLC_DEF _CLC_OVERLOAD RET_TYPE FUNCTION(ARG0_TYPE x, ARG1_TYPE y) {         \
-    return BUILTIN_NAME(x, y);                                                 \
-  }
-
-#define _CLC_DEFINE_RELATIONAL_BINARY_VEC(RET_TYPE, FUNCTION, ARG0_TYPE,       \
-                                          ARG1_TYPE)                           \
-  _CLC_DEF _CLC_OVERLOAD RET_TYPE FUNCTION(ARG0_TYPE x, ARG1_TYPE y) {         \
-    return (RET_TYPE)((RET_TYPE){FUNCTION(x.lo, y.lo),                         \
-                                 FUNCTION(x.hi, y.hi)} != (RET_TYPE)0);        \
-  }
-
-#define _CLC_DEFINE_RELATIONAL_BINARY_VEC2(RET_TYPE, FUNCTION, ARG0_TYPE,      \
-                                           ARG1_TYPE)                          \
-  _CLC_DEF _CLC_OVERLOAD RET_TYPE FUNCTION(ARG0_TYPE x, ARG1_TYPE y) {         \
-    return (RET_TYPE)((RET_TYPE){FUNCTION(x.lo, y.lo),                         \
-                                 FUNCTION(x.hi, y.hi)} != (RET_TYPE)0);        \
-  }
-
-#define _CLC_DEFINE_RELATIONAL_BINARY_VEC3(RET_TYPE, FUNCTION, ARG0_TYPE,      \
-                                           ARG1_TYPE)                          \
-  _CLC_DEF _CLC_OVERLOAD RET_TYPE FUNCTION(ARG0_TYPE x, ARG1_TYPE y) {         \
-    return (RET_TYPE)((RET_TYPE){FUNCTION(x.s0, y.s0), FUNCTION(x.s1, y.s1),   \
-                                 FUNCTION(x.s2, y.s2)} != (RET_TYPE)0);        \
-  }
-
-#define _CLC_DEFINE_RELATIONAL_BINARY_VEC4(RET_TYPE, FUNCTION, ARG0_TYPE,      \
-                                           ARG1_TYPE)                          \
-  _CLC_DEF _CLC_OVERLOAD RET_TYPE FUNCTION(ARG0_TYPE x, ARG1_TYPE y) {         \
-    return (RET_TYPE)((RET_TYPE){FUNCTION(x.s0, y.s0), FUNCTION(x.s1, y.s1),   \
-                                 FUNCTION(x.s2, y.s2),                         \
-                                 FUNCTION(x.s3, y.s3)} != (RET_TYPE)0);        \
-  }
-
-#define _CLC_DEFINE_RELATIONAL_BINARY_VEC8(RET_TYPE, FUNCTION, ARG0_TYPE,      \
-                                           ARG1_TYPE)                          \
-  _CLC_DEF _CLC_OVERLOAD RET_TYPE FUNCTION(ARG0_TYPE x, ARG1_TYPE y) {         \
-    return (RET_TYPE)((RET_TYPE){FUNCTION(x.s0, y.s0), FUNCTION(x.s1, y.s1),   \
-                                 FUNCTION(x.s2, y.s2), FUNCTION(x.s3, y.s3),   \
-                                 FUNCTION(x.s4, y.s4), FUNCTION(x.s5, y.s5),   \
-                                 FUNCTION(x.s6, y.s6),                         \
-                                 FUNCTION(x.s7, y.s7)} != (RET_TYPE)0);        \
-  }
-
-#define _CLC_DEFINE_RELATIONAL_BINARY_VEC16(RET_TYPE, FUNCTION, ARG0_TYPE,     \
-                                            ARG1_TYPE)                         \
-  _CLC_DEF _CLC_OVERLOAD RET_TYPE FUNCTION(ARG0_TYPE x, ARG1_TYPE y) {         \
-    return (RET_TYPE)((RET_TYPE){FUNCTION(x.s0, y.s0), FUNCTION(x.s1, y.s1),   \
-                                 FUNCTION(x.s2, y.s2), FUNCTION(x.s3, y.s3),   \
-                                 FUNCTION(x.s4, y.s4), FUNCTION(x.s5, y.s5),   \
-                                 FUNCTION(x.s6, y.s6), FUNCTION(x.s7, y.s7),   \
-                                 FUNCTION(x.s8, y.s8), FUNCTION(x.s9, y.s9),   \
-                                 FUNCTION(x.sa, y.sa), FUNCTION(x.sb, y.sb),   \
-                                 FUNCTION(x.sc, y.sc), FUNCTION(x.sd, y.sd),   \
-                                 FUNCTION(x.se, y.se),                         \
-                                 FUNCTION(x.sf, y.sf)} != (RET_TYPE)0);        \
-  }
-
-#define _CLC_DEFINE_RELATIONAL_BINARY_VEC_ALL(RET_TYPE, FUNCTION, ARG0_TYPE,   \
-                                              ARG1_TYPE)                       \
-  _CLC_DEFINE_RELATIONAL_BINARY_VEC2(RET_TYPE##2, FUNCTION, ARG0_TYPE##2,      \
-                                     ARG1_TYPE##2)                             \
-  _CLC_DEFINE_RELATIONAL_BINARY_VEC3(RET_TYPE##3, FUNCTION, ARG0_TYPE##3,      \
-                                     ARG1_TYPE##3)                             \
-  _CLC_DEFINE_RELATIONAL_BINARY_VEC4(RET_TYPE##4, FUNCTION, ARG0_TYPE##4,      \
-                                     ARG1_TYPE##4)                             \
-  _CLC_DEFINE_RELATIONAL_BINARY_VEC8(RET_TYPE##8, FUNCTION, ARG0_TYPE##8,      \
-                                     ARG1_TYPE##8)                             \
-  _CLC_DEFINE_RELATIONAL_BINARY_VEC16(RET_TYPE##16, FUNCTION, ARG0_TYPE##16,   \
-                                      ARG1_TYPE##16)
-
-#define _CLC_DEFINE_RELATIONAL_BINARY(RET_TYPE, FUNCTION, BUILTIN_FUNCTION,    \
-                                      ARG0_TYPE, ARG1_TYPE)                    \
-  _CLC_DEFINE_RELATIONAL_BINARY_SCALAR(RET_TYPE, FUNCTION, BUILTIN_FUNCTION,   \
-                                       ARG0_TYPE, ARG1_TYPE)                   \
-  _CLC_DEFINE_RELATIONAL_BINARY_VEC_ALL(RET_TYPE, FUNCTION, ARG0_TYPE,         \
-                                        ARG1_TYPE)
+    return __builtin_isfpclass(x, (MASK));                                     \
+  }                                                                            \
+  _CLC_DEFINE_ISFPCLASS_VEC(VEC_RET_TYPE##2, FUNCTION, MASK, ARG_TYPE##2)      \
+  _CLC_DEFINE_ISFPCLASS_VEC(VEC_RET_TYPE##3, FUNCTION, MASK, ARG_TYPE##3)      \
+  _CLC_DEFINE_ISFPCLASS_VEC(VEC_RET_TYPE##4, FUNCTION, MASK, ARG_TYPE##4)      \
+  _CLC_DEFINE_ISFPCLASS_VEC(VEC_RET_TYPE##8, FUNCTION, MASK, ARG_TYPE##8)      \
+  _CLC_DEFINE_ISFPCLASS_VEC(VEC_RET_TYPE##16, FUNCTION, MASK, ARG_TYPE##16)
 
 #endif // __CLC_RELATIONAL_RELATIONAL_H__
diff --git a/libclc/clc/include/clc/shared/binary_def.inc b/libclc/clc/include/clc/shared/binary_def.inc
new file mode 100644
index 0000000000000..e6ef867f12c6b
--- /dev/null
+++ b/libclc/clc/include/clc/shared/binary_def.inc
@@ -0,0 +1,10 @@
+#include <clc/utils.h>
+
+#ifndef __CLC_FUNCTION
+#define __CLC_FUNCTION(x) __CLC_CONCAT(__clc_, x)
+#endif
+
+_CLC_OVERLOAD _CLC_DEF __CLC_GENTYPE FUNCTION(__CLC_GENTYPE a,
+                                              __CLC_GENTYPE b) {
+  return __CLC_FUNCTION(FUNCTION)(a, b);
+}
diff --git a/libclc/clc/include/clc/shared/clc_max.h b/libclc/clc/include/clc/shared/clc_max.h
index 388f001a27782..9bfa05552a399 100644
--- a/libclc/clc/include/clc/shared/clc_max.h
+++ b/libclc/clc/include/clc/shared/clc_max.h
@@ -1,17 +1,10 @@
 #ifndef __CLC_SHARED_CLC_MAX_H__
 #define __CLC_SHARED_CLC_MAX_H__
 
-#if defined(CLC_CLSPV) || defined(CLC_SPIRV)
-// clspv and spir-v targets provide their own OpenCL-compatible max
-#define __clc_max max
-#else
-
 #define __CLC_BODY <clc/shared/clc_max.inc>
 #include <clc/integer/gentype.inc>
 
 #define __CLC_BODY <clc/shared/clc_max.inc>
 #include <clc/math/gentype.inc>
 
-#endif
-
 #endif // __CLC_SHARED_CLC_MAX_H__
diff --git a/libclc/clc/include/clc/shared/clc_min.h b/libclc/clc/include/clc/shared/clc_min.h
index c8d920e1b4eb8..a10193885328f 100644
--- a/libclc/clc/include/clc/shared/clc_min.h
+++ b/libclc/clc/include/clc/shared/clc_min.h
@@ -1,17 +1,10 @@
 #ifndef __CLC_SHARED_CLC_MIN_H__
 #define __CLC_SHARED_CLC_MIN_H__
 
-#if defined(CLC_CLSPV) || defined(CLC_SPIRV)
-// clspv and spir-v targets provide their own OpenCL-compatible min
-#define __clc_min min
-#else
-
 #define __CLC_BODY <clc/shared/clc_min.inc>
 #include <clc/integer/gentype.inc>
 
 #define __CLC_BODY <clc/shared/clc_min.inc>
 #include <clc/math/gentype.inc>
 
-#endif
-
 #endif // __CLC_SHARED_CLC_MIN_H__
diff --git a/libclc/clc/include/clc/math/ternary_decl.inc b/libclc/clc/include/clc/shared/ternary_decl.inc
similarity index 100%
rename from libclc/clc/include/clc/math/ternary_decl.inc
rename to libclc/clc/include/clc/shared/ternary_decl.inc
diff --git a/libclc/clc/include/clc/shared/ternary_def.inc b/libclc/clc/include/clc/shared/ternary_def.inc
new file mode 100644
index 0000000000000..59528d825a171
--- /dev/null
+++ b/libclc/clc/include/clc/shared/ternary_def.inc
@@ -0,0 +1,10 @@
+#include <clc/utils.h>
+
+#ifndef __CLC_FUNCTION
+#define __CLC_FUNCTION(x) __CLC_CONCAT(__clc_, x)
+#endif
+
+_CLC_OVERLOAD _CLC_DEF __CLC_GENTYPE FUNCTION(__CLC_GENTYPE a, __CLC_GENTYPE b,
+                                              __CLC_GENTYPE c) {
+  return __CLC_FUNCTION(FUNCTION)(a, b, c);
+}
diff --git a/libclc/clc/include/clc/shared/unary_decl.inc b/libclc/clc/include/clc/shared/unary_decl.inc
new file mode 100644
index 0000000000000..9858d908da09f
--- /dev/null
+++ b/libclc/clc/include/clc/shared/unary_decl.inc
@@ -0,0 +1 @@
+_CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE __CLC_FUNCTION(__CLC_GENTYPE x);
diff --git a/libclc/clc/include/clc/shared/unary_def.inc b/libclc/clc/include/clc/shared/unary_def.inc
new file mode 100644
index 0000000000000..e0f34ad817e8b
--- /dev/null
+++ b/libclc/clc/include/clc/shared/unary_def.inc
@@ -0,0 +1,9 @@
+#include <clc/utils.h>
+
+#ifndef __CLC_FUNCTION
+#define __CLC_FUNCTION(x) __CLC_CONCAT(__clc_, x)
+#endif
+
+_CLC_OVERLOAD _CLC_DEF __CLC_GENTYPE FUNCTION(__CLC_GENTYPE a) {
+  return __CLC_FUNCTION(FUNCTION)(a);
+}
diff --git a/libclc/clc/lib/clspv/SOURCES b/libclc/clc/lib/clspv/SOURCES
deleted file mode 100644
index 81f90a24d00d6..0000000000000
--- a/libclc/clc/lib/clspv/SOURCES
+++ /dev/null
@@ -1,9 +0,0 @@
-../generic/math/clc_ceil.cl
-../generic/math/clc_fabs.cl
-../generic/math/clc_floor.cl
-../generic/math/clc_mad.cl
-../generic/math/clc_nextafter.cl
-../generic/math/clc_rint.cl
-../generic/math/clc_trunc.cl
-../generic/relational/clc_select.cl
-../generic/shared/clc_clamp.cl
diff --git a/libclc/clc/lib/clspv64 b/libclc/clc/lib/clspv64
deleted file mode 120000
index ea01ba94bc636..0000000000000
--- a/libclc/clc/lib/clspv64
+++ /dev/null
@@ -1 +0,0 @@
-clspv
\ No newline at end of file
diff --git a/libclc/clc/lib/generic/SOURCES b/libclc/clc/lib/generic/SOURCES
index 59dad8e860689..b0eaf84c41438 100644
--- a/libclc/clc/lib/generic/SOURCES
+++ b/libclc/clc/lib/generic/SOURCES
@@ -1,10 +1,24 @@
 common/clc_degrees.cl
 common/clc_radians.cl
+common/clc_sign.cl
 common/clc_smoothstep.cl
 geometric/clc_dot.cl
 integer/clc_abs.cl
 integer/clc_abs_diff.cl
+integer/clc_add_sat.cl
+integer/clc_clz.cl
+integer/clc_hadd.cl
+integer/clc_mad24.cl
+integer/clc_mad_sat.cl
+integer/clc_mul24.cl
+integer/clc_mul_hi.cl
+integer/clc_popcount.cl
+integer/clc_rhadd.cl
+integer/clc_rotate.cl
+integer/clc_sub_sat.cl
+integer/clc_upsample.cl
 math/clc_ceil.cl
+math/clc_copysign.cl
 math/clc_fabs.cl
 math/clc_floor.cl
 math/clc_mad.cl
diff --git a/libclc/clc/lib/generic/common/clc_sign.cl b/libclc/clc/lib/generic/common/clc_sign.cl
new file mode 100644
index 0000000000000..3c0384d71a65d
--- /dev/null
+++ b/libclc/clc/lib/generic/common/clc_sign.cl
@@ -0,0 +1,6 @@
+#include <clc/math/clc_copysign.h>
+#include <clc/relational/clc_isnan.h>
+#include <clc/relational/clc_select.h>
+
+#define __CLC_BODY <clc_sign.inc>
+#include <clc/math/gentype.inc>
diff --git a/libclc/clc/lib/generic/common/clc_sign.inc b/libclc/clc/lib/generic/common/clc_sign.inc
new file mode 100644
index 0000000000000..85fa330e58e9e
--- /dev/null
+++ b/libclc/clc/lib/generic/common/clc_sign.inc
@@ -0,0 +1,7 @@
+_CLC_DEF _CLC_OVERLOAD __CLC_GENTYPE __clc_sign(__CLC_GENTYPE x) {
+  __CLC_BIT_INTN ret_zero = __clc_isnan(x) || x == __CLC_FP_LIT(0.0);
+  __CLC_GENTYPE ret_val =
+      __clc_select((__CLC_GENTYPE)__CLC_FP_LIT(1.0),
+                   (__CLC_GENTYPE)__CLC_FP_LIT(0.0), ret_zero);
+  return __clc_copysign(ret_val, x);
+}
diff --git a/libclc/clc/lib/generic/integer/clc_add_sat.cl b/libclc/clc/lib/generic/integer/clc_add_sat.cl
new file mode 100644
index 0000000000000..344c105720e08
--- /dev/null
+++ b/libclc/clc/lib/generic/integer/clc_add_sat.cl
@@ -0,0 +1,7 @@
+#include <clc/internal/clc.h>
+
+#define FUNCTION __clc_add_sat
+#define __CLC_FUNCTION(x) __builtin_elementwise_add_sat
+#define __CLC_BODY <clc/shared/binary_def.inc>
+
+#include <clc/integer/gentype.inc>
diff --git a/libclc/clc/lib/generic/integer/clc_clz.cl b/libclc/clc/lib/generic/integer/clc_clz.cl
new file mode 100644
index 0000000000000..592b65f262bd6
--- /dev/null
+++ b/libclc/clc/lib/generic/integer/clc_clz.cl
@@ -0,0 +1,44 @@
+#include <clc/clcmacro.h>
+#include <clc/integer/clc_clz.h>
+#include <clc/internal/clc.h>
+
+_CLC_OVERLOAD _CLC_DEF char __clc_clz(char x) {
+  return __clc_clz((ushort)(uchar)x) - 8;
+}
+
+_CLC_OVERLOAD _CLC_DEF uchar __clc_clz(uchar x) {
+  return __clc_clz((ushort)x) - 8;
+}
+
+_CLC_OVERLOAD _CLC_DEF short __clc_clz(short x) {
+  return x ? __builtin_clzs(x) : 16;
+}
+
+_CLC_OVERLOAD _CLC_DEF ushort __clc_clz(ushort x) {
+  return x ? __builtin_clzs(x) : 16;
+}
+
+_CLC_OVERLOAD _CLC_DEF int __clc_clz(int x) {
+  return x ? __builtin_clz(x) : 32;
+}
+
+_CLC_OVERLOAD _CLC_DEF uint __clc_clz(uint x) {
+  return x ? __builtin_clz(x) : 32;
+}
+
+_CLC_OVERLOAD _CLC_DEF long __clc_clz(long x) {
+  return x ? __builtin_clzl(x) : 64;
+}
+
+_CLC_OVERLOAD _CLC_DEF ulong __clc_clz(ulong x) {
+  return x ? __builtin_clzl(x) : 64;
+}
+
+_CLC_UNARY_VECTORIZE(_CLC_OVERLOAD _CLC_DEF, char, __clc_clz, char)
+_CLC_UNARY_VECTORIZE(_CLC_OVERLOAD _CLC_DEF, uchar, __clc_clz, uchar)
+_CLC_UNARY_VECTORIZE(_CLC_OVERLOAD _CLC_DEF, short, __clc_clz, short)
+_CLC_UNARY_VECTORIZE(_CLC_OVERLOAD _CLC_DEF, ushort, __clc_clz, ushort)
+_CLC_UNARY_VECTORIZE(_CLC_OVERLOAD _CLC_DEF, int, __clc_clz, int)
+_CLC_UNARY_VECTORIZE(_CLC_OVERLOAD _CLC_DEF, uint, __clc_clz, uint)
+_CLC_UNARY_VECTORIZE(_CLC_OVERLOAD _CLC_DEF, long, __clc_clz, long)
+_CLC_UNARY_VECTORIZE(_CLC_OVERLOAD _CLC_DEF, ulong, __clc_clz, ulong)
diff --git a/libclc/clc/lib/generic/integer/clc_hadd.cl b/libclc/clc/lib/generic/integer/clc_hadd.cl
new file mode 100644
index 0000000000000..8e91d41a843aa
--- /dev/null
+++ b/libclc/clc/lib/generic/integer/clc_hadd.cl
@@ -0,0 +1,4 @@
+#include <clc/internal/clc.h>
+
+#define __CLC_BODY <clc_hadd.inc>
+#include <clc/integer/gentype.inc>
diff --git a/libclc/clc/lib/generic/integer/clc_hadd.inc b/libclc/clc/lib/generic/integer/clc_hadd.inc
new file mode 100644
index 0000000000000..14d921599446b
--- /dev/null
+++ b/libclc/clc/lib/generic/integer/clc_hadd.inc
@@ -0,0 +1,8 @@
+// hadd = (x+y)>>1
+// This can be simplified to x>>1 + y>>1 + (1 if both x and y have the 1s bit
+// set) This saves us having to do any checks for overflow in the addition sum
+_CLC_OVERLOAD _CLC_DEF __CLC_GENTYPE __clc_hadd(__CLC_GENTYPE x,
+                                                __CLC_GENTYPE y) {
+  return (x >> (__CLC_GENTYPE)1) + (y >> (__CLC_GENTYPE)1) +
+         (x & y & (__CLC_GENTYPE)1);
+}
diff --git a/libclc/clc/lib/generic/integer/clc_mad24.cl b/libclc/clc/lib/generic/integer/clc_mad24.cl
new file mode 100644
index 0000000000000..86c319cff6d24
--- /dev/null
+++ b/libclc/clc/lib/generic/integer/clc_mad24.cl
@@ -0,0 +1,5 @@
+#include <clc/internal/clc.h>
+#include <clc/integer/clc_mul24.h>
+
+#define __CLC_BODY <clc_mad24.inc>
+#include <clc/integer/gentype24.inc>
diff --git a/libclc/clc/lib/generic/integer/clc_mad24.inc b/libclc/clc/lib/generic/integer/clc_mad24.inc
new file mode 100644
index 0000000000000..61c8587d4f86f
--- /dev/null
+++ b/libclc/clc/lib/generic/integer/clc_mad24.inc
@@ -0,0 +1,5 @@
+_CLC_OVERLOAD _CLC_DEF __CLC_GENTYPE __clc_mad24(__CLC_GENTYPE x,
+                                                 __CLC_GENTYPE y,
+                                                 __CLC_GENTYPE z) {
+  return __clc_mul24(x, y) + z;
+}
diff --git a/libclc/clc/lib/generic/integer/clc_mad_sat.cl b/libclc/clc/lib/generic/integer/clc_mad_sat.cl
new file mode 100644
index 0000000000000..4e559dba2b2f5
--- /dev/null
+++ b/libclc/clc/lib/generic/integer/clc_mad_sat.cl
@@ -0,0 +1,119 @@
+#include <clc/clcmacro.h>
+#include <clc/integer/clc_add_sat.h>
+#include <clc/integer/clc_mad24.h>
+#include <clc/integer/clc_mul_hi.h>
+#include <clc/integer/clc_upsample.h>
+#include <clc/integer/definitions.h>
+#include <clc/internal/clc.h>
+#include <clc/relational/clc_select.h>
+#include <clc/shared/clc_clamp.h>
+
+#define __CLC_CONVERT_TY(X, TY) __builtin_convertvector(X, TY)
+
+// Macro for defining mad_sat variants for char/uchar/short/ushort
+// FIXME: Once using __clc_convert_ty, can easily unify scalar and vector defs
+#define __CLC_DEFINE_SIMPLE_MAD_SAT(TYPE, UP_TYPE, LIT_PREFIX)                 \
+  _CLC_OVERLOAD _CLC_DEF TYPE __clc_mad_sat(TYPE x, TYPE y, TYPE z) {          \
+    return __clc_clamp(                                                        \
+        (UP_TYPE)__clc_mad24((UP_TYPE)x, (UP_TYPE)y, (UP_TYPE)z),              \
+        (UP_TYPE)LIT_PREFIX##_MIN, (UP_TYPE)LIT_PREFIX##_MAX);                 \
+  }
+
+#define __CLC_DEFINE_SIMPLE_MAD_SAT_VEC(TYPE, UP_TYPE, LIT_PREFIX)             \
+  _CLC_OVERLOAD _CLC_DEF TYPE __clc_mad_sat(TYPE x, TYPE y, TYPE z) {          \
+    UP_TYPE upscaled_mad = __clc_mad24(__CLC_CONVERT_TY(x, UP_TYPE),           \
+                                       __CLC_CONVERT_TY(y, UP_TYPE),           \
+                                       __CLC_CONVERT_TY(z, UP_TYPE));          \
+    UP_TYPE clamped_mad = __clc_clamp(upscaled_mad, (UP_TYPE)LIT_PREFIX##_MIN, \
+                                      (UP_TYPE)LIT_PREFIX##_MAX);              \
+    return __CLC_CONVERT_TY(clamped_mad, TYPE);                                \
+  }
+
+#define __CLC_DEFINE_SIMPLE_MAD_SAT_ALL_TYS(TYPE, UP_TYPE, LIT_PREFIX)         \
+  __CLC_DEFINE_SIMPLE_MAD_SAT(TYPE, UP_TYPE, LIT_PREFIX)                       \
+  __CLC_DEFINE_SIMPLE_MAD_SAT_VEC(TYPE##2, UP_TYPE##2, LIT_PREFIX)             \
+  __CLC_DEFINE_SIMPLE_MAD_SAT_VEC(TYPE##3, UP_TYPE##3, LIT_PREFIX)             \
+  __CLC_DEFINE_SIMPLE_MAD_SAT_VEC(TYPE##4, UP_TYPE##4, LIT_PREFIX)             \
+  __CLC_DEFINE_SIMPLE_MAD_SAT_VEC(TYPE##8, UP_TYPE##8, LIT_PREFIX)             \
+  __CLC_DEFINE_SIMPLE_MAD_SAT_VEC(TYPE##16, UP_TYPE##16, LIT_PREFIX)
+
+__CLC_DEFINE_SIMPLE_MAD_SAT_ALL_TYS(char, int, CHAR)
+__CLC_DEFINE_SIMPLE_MAD_SAT_ALL_TYS(uchar, uint, UCHAR)
+__CLC_DEFINE_SIMPLE_MAD_SAT_ALL_TYS(short, int, SHRT)
+__CLC_DEFINE_SIMPLE_MAD_SAT_ALL_TYS(ushort, uint, USHRT)
+
+// Macro for defining mad_sat variants for uint/ulong
+#define __CLC_DEFINE_UINTLONG_MAD_SAT(UTYPE, STYPE, ULIT_PREFIX)               \
+  _CLC_OVERLOAD _CLC_DEF UTYPE __clc_mad_sat(UTYPE x, UTYPE y, UTYPE z) {      \
+    STYPE has_mul_hi = __clc_mul_hi(x, y) != (UTYPE)0;                         \
+    return __clc_select(__clc_add_sat(x * y, z), (UTYPE)ULIT_PREFIX##_MAX,     \
+                        has_mul_hi);                                           \
+  }
+
+#define __CLC_DEFINE_UINTLONG_MAD_SAT_ALL_TYS(UTY, STY, ULIT_PREFIX)           \
+  __CLC_DEFINE_UINTLONG_MAD_SAT(UTY, STY, ULIT_PREFIX)                         \
+  __CLC_DEFINE_UINTLONG_MAD_SAT(UTY##2, STY##2, ULIT_PREFIX)                   \
+  __CLC_DEFINE_UINTLONG_MAD_SAT(UTY##3, STY##3, ULIT_PREFIX)                   \
+  __CLC_DEFINE_UINTLONG_MAD_SAT(UTY##4, STY##4, ULIT_PREFIX)                   \
+  __CLC_DEFINE_UINTLONG_MAD_SAT(UTY##8, STY##8, ULIT_PREFIX)                   \
+  __CLC_DEFINE_UINTLONG_MAD_SAT(UTY##16, STY##16, ULIT_PREFIX)
+
+__CLC_DEFINE_UINTLONG_MAD_SAT_ALL_TYS(uint, int, UINT)
+__CLC_DEFINE_UINTLONG_MAD_SAT_ALL_TYS(ulong, long, ULONG)
+
+// Macro for defining mad_sat variants for int
+#define __CLC_DEFINE_SINT_MAD_SAT(INTTY, UINTTY, SLONGTY)                      \
+  _CLC_OVERLOAD _CLC_DEF INTTY __clc_mad_sat(INTTY x, INTTY y, INTTY z) {      \
+    INTTY mhi = __clc_mul_hi(x, y);                                            \
+    UINTTY mlo = __clc_as_##UINTTY(x * y);                                     \
+    SLONGTY m = __clc_upsample(mhi, mlo);                                      \
+    m += __CLC_CONVERT_TY(z, SLONGTY);                                         \
+    m = __clc_clamp(m, (SLONGTY)INT_MIN, (SLONGTY)INT_MAX);                    \
+    return __CLC_CONVERT_TY(m, INTTY);                                         \
+  }
+
+// FIXME: Once using __clc_convert_ty, can easily unify scalar and vector defs
+#define __CLC_DEFINE_SINT_MAD_SAT_ALL_TYS(INTTY, UINTTY, SLONGTY)              \
+  _CLC_OVERLOAD _CLC_DEF INTTY __clc_mad_sat(INTTY x, INTTY y, INTTY z) {      \
+    INTTY mhi = __clc_mul_hi(x, y);                                            \
+    UINTTY mlo = __clc_as_##UINTTY(x * y);                                     \
+    SLONGTY m = __clc_upsample(mhi, mlo);                                      \
+    m += z;                                                                    \
+    return __clc_clamp(m, (SLONGTY)INT_MIN, (SLONGTY)INT_MAX);                 \
+  }                                                                            \
+  __CLC_DEFINE_SINT_MAD_SAT(INTTY##2, UINTTY##2, SLONGTY##2)                   \
+  __CLC_DEFINE_SINT_MAD_SAT(INTTY##3, UINTTY##3, SLONGTY##3)                   \
+  __CLC_DEFINE_SINT_MAD_SAT(INTTY##4, UINTTY##4, SLONGTY##4)                   \
+  __CLC_DEFINE_SINT_MAD_SAT(INTTY##8, UINTTY##8, SLONGTY##8)                   \
+  __CLC_DEFINE_SINT_MAD_SAT(INTTY##16, UINTTY##16, SLONGTY##16)
+
+__CLC_DEFINE_SINT_MAD_SAT_ALL_TYS(int, uint, long)
+
+// Macro for defining mad_sat variants for long
+#define __CLC_DEFINE_SLONG_MAD_SAT(SLONGTY, ULONGTY)                           \
+  _CLC_OVERLOAD _CLC_DEF SLONGTY __clc_mad_sat(SLONGTY x, SLONGTY y,           \
+                                               SLONGTY z) {                    \
+    SLONGTY hi = __clc_mul_hi(x, y);                                           \
+    ULONGTY ulo = __clc_as_##ULONGTY(x * y);                                   \
+    SLONGTY max1 = (x < 0) == (y < 0) && hi != 0;                              \
+    SLONGTY max2 = hi == 0 && ulo >= LONG_MAX &&                               \
+                   (z > 0 || (ulo + __clc_as_##ULONGTY(z)) > LONG_MAX);        \
+    SLONGTY min1 = (((x < 0) != (y < 0)) && hi != -1);                         \
+    SLONGTY min2 =                                                             \
+        hi == -1 && ulo <= ((ULONGTY)LONG_MAX + 1UL) &&                        \
+        (z < 0 || __clc_as_##ULONGTY(z) < ((ULONGTY)LONG_MAX - ulo));          \
+    SLONGTY ret = __clc_as_##SLONGTY(ulo + __clc_as_##ULONGTY(z));             \
+    ret = __clc_select(ret, (SLONGTY)LONG_MAX, (SLONGTY)(max1 || max2));       \
+    ret = __clc_select(ret, (SLONGTY)LONG_MIN, (SLONGTY)(min1 || min2));       \
+    return ret;                                                                \
+  }
+
+#define __CLC_DEFINE_SLONG_MAD_SAT_ALL_TYS(SLONGTY, ULONGTY)                   \
+  __CLC_DEFINE_SLONG_MAD_SAT(SLONGTY, ULONGTY)                                 \
+  __CLC_DEFINE_SLONG_MAD_SAT(SLONGTY##2, ULONGTY##2)                           \
+  __CLC_DEFINE_SLONG_MAD_SAT(SLONGTY##3, ULONGTY##3)                           \
+  __CLC_DEFINE_SLONG_MAD_SAT(SLONGTY##4, ULONGTY##4)                           \
+  __CLC_DEFINE_SLONG_MAD_SAT(SLONGTY##8, ULONGTY##8)                           \
+  __CLC_DEFINE_SLONG_MAD_SAT(SLONGTY##16, ULONGTY##16)
+
+__CLC_DEFINE_SLONG_MAD_SAT_ALL_TYS(long, ulong)
diff --git a/libclc/clc/lib/generic/integer/clc_mul24.cl b/libclc/clc/lib/generic/integer/clc_mul24.cl
new file mode 100644
index 0000000000000..6513a896a8b1d
--- /dev/null
+++ b/libclc/clc/lib/generic/integer/clc_mul24.cl
@@ -0,0 +1,4 @@
+#include <clc/internal/clc.h>
+
+#define __CLC_BODY <clc_mul24.inc>
+#include <clc/integer/gentype24.inc>
diff --git a/libclc/clc/lib/generic/integer/clc_mul24.inc b/libclc/clc/lib/generic/integer/clc_mul24.inc
new file mode 100644
index 0000000000000..d7e8091c98a31
--- /dev/null
+++ b/libclc/clc/lib/generic/integer/clc_mul24.inc
@@ -0,0 +1,11 @@
+// We need to use shifts here in order to mantain the sign bit for signed
+// integers.  The compiler should optimize this to (x & 0x00FFFFFF) for
+// unsigned integers.
+#define CONVERT_TO_24BIT(x) (((x) << 8) >> 8)
+
+_CLC_OVERLOAD _CLC_DEF __CLC_GENTYPE __clc_mul24(__CLC_GENTYPE x,
+                                                 __CLC_GENTYPE y) {
+  return CONVERT_TO_24BIT(x) * CONVERT_TO_24BIT(y);
+}
+
+#undef CONVERT_TO_24BIT
diff --git a/libclc/clc/lib/generic/integer/clc_mul_hi.cl b/libclc/clc/lib/generic/integer/clc_mul_hi.cl
new file mode 100644
index 0000000000000..cf4acc5429cb4
--- /dev/null
+++ b/libclc/clc/lib/generic/integer/clc_mul_hi.cl
@@ -0,0 +1,132 @@
+#include <clc/integer/clc_hadd.h>
+#include <clc/integer/definitions.h>
+#include <clc/internal/clc.h>
+
+// TODO: Replace with __clc_convert_<type> when available
+#define __CLC_CONVERT_TY(X, TY) __builtin_convertvector(X, TY)
+
+#define __CLC_MUL_HI_VEC_IMPL(BGENTYPE, GENTYPE, GENSIZE)                      \
+  _CLC_OVERLOAD _CLC_DEF GENTYPE __clc_mul_hi(GENTYPE x, GENTYPE y) {          \
+    BGENTYPE large_x = __CLC_CONVERT_TY(x, BGENTYPE);                          \
+    BGENTYPE large_y = __CLC_CONVERT_TY(y, BGENTYPE);                          \
+    BGENTYPE large_mul_hi = (large_x * large_y) >> (BGENTYPE)GENSIZE;          \
+    return __CLC_CONVERT_TY(large_mul_hi, GENTYPE);                            \
+  }
+
+// For all types EXCEPT long, which is implemented separately
+#define __CLC_MUL_HI_IMPL(BGENTYPE, GENTYPE, GENSIZE)                          \
+  _CLC_OVERLOAD _CLC_DEF GENTYPE __clc_mul_hi(GENTYPE x, GENTYPE y) {          \
+    return (GENTYPE)(((BGENTYPE)x * (BGENTYPE)y) >> GENSIZE);                  \
+  }
+
+#define __CLC_MUL_HI_DEC_IMPL(BTYPE, TYPE, BITS)                               \
+  __CLC_MUL_HI_IMPL(BTYPE, TYPE, BITS)                                         \
+  __CLC_MUL_HI_VEC_IMPL(BTYPE##2, TYPE##2, BITS)                               \
+  __CLC_MUL_HI_VEC_IMPL(BTYPE##3, TYPE##3, BITS)                               \
+  __CLC_MUL_HI_VEC_IMPL(BTYPE##4, TYPE##4, BITS)                               \
+  __CLC_MUL_HI_VEC_IMPL(BTYPE##8, TYPE##8, BITS)                               \
+  __CLC_MUL_HI_VEC_IMPL(BTYPE##16, TYPE##16, BITS)
+
+_CLC_OVERLOAD _CLC_DEF long __clc_mul_hi(long x, long y) {
+  long f, o, i;
+  ulong l;
+
+  // Move the high/low halves of x/y into the lower 32-bits of variables so
+  // that we can multiply them without worrying about overflow.
+  long x_hi = x >> 32;
+  long x_lo = x & UINT_MAX;
+  long y_hi = y >> 32;
+  long y_lo = y & UINT_MAX;
+
+  // Multiply all of the components according to FOIL method
+  f = x_hi * y_hi;
+  o = x_hi * y_lo;
+  i = x_lo * y_hi;
+  l = x_lo * y_lo;
+
+  // Now add the components back together in the following steps:
+  // F: doesn't need to be modified
+  // O/I: Need to be added together.
+  // L: Shift right by 32-bits, then add into the sum of O and I
+  // Once O/I/L are summed up, then shift the sum by 32-bits and add to F.
+  //
+  // We use hadd to give us a bit of extra precision for the intermediate sums
+  // but as a result, we shift by 31 bits instead of 32
+  return (long)(f + (__clc_hadd(o, (i + (long)((ulong)l >> 32))) >> 31));
+}
+
+_CLC_OVERLOAD _CLC_DEF ulong __clc_mul_hi(ulong x, ulong y) {
+  ulong f, o, i;
+  ulong l;
+
+  // Move the high/low halves of x/y into the lower 32-bits of variables so
+  // that we can multiply them without worrying about overflow.
+  ulong x_hi = x >> 32;
+  ulong x_lo = x & UINT_MAX;
+  ulong y_hi = y >> 32;
+  ulong y_lo = y & UINT_MAX;
+
+  // Multiply all of the components according to FOIL method
+  f = x_hi * y_hi;
+  o = x_hi * y_lo;
+  i = x_lo * y_hi;
+  l = x_lo * y_lo;
+
+  // Now add the components back together, taking care to respect the fact that:
+  // F: doesn't need to be modified
+  // O/I: Need to be added together.
+  // L: Shift right by 32-bits, then add into the sum of O and I
+  // Once O/I/L are summed up, then shift the sum by 32-bits and add to F.
+  //
+  // We use hadd to give us a bit of extra precision for the intermediate sums
+  // but as a result, we shift by 31 bits instead of 32
+  return (f + (__clc_hadd(o, (i + (l >> 32))) >> 31));
+}
+
+// Vector-based mul_hi implementation for logn/ulong. See comments in the scalar
+// versions for more detail.
+#define __CLC_MUL_HI_LONG_VEC_IMPL(TY, UTY)                                    \
+  _CLC_OVERLOAD _CLC_DEF TY __clc_mul_hi(TY x, TY y) {                         \
+    TY f, o, i;                                                                \
+    UTY l;                                                                     \
+                                                                               \
+    TY x_hi = x >> 32;                                                         \
+    TY x_lo = x & UINT_MAX;                                                    \
+    TY y_hi = y >> 32;                                                         \
+    TY y_lo = y & UINT_MAX;                                                    \
+                                                                               \
+    f = x_hi * y_hi;                                                           \
+    o = x_hi * y_lo;                                                           \
+    i = x_lo * y_hi;                                                           \
+    l = __CLC_CONVERT_TY(x_lo * y_lo, UTY);                                    \
+    i += __CLC_CONVERT_TY(l >> (UTY)32, TY);                                   \
+                                                                               \
+    return f + (__clc_hadd(o, i) >> (TY)31);                                   \
+  }
+
+#define __CLC_MUL_HI_LONG_IMPL(BTYPE, UBTYPE)                                  \
+  __CLC_MUL_HI_LONG_VEC_IMPL(BTYPE##2, UBTYPE##2)                              \
+  __CLC_MUL_HI_LONG_VEC_IMPL(BTYPE##3, UBTYPE##3)                              \
+  __CLC_MUL_HI_LONG_VEC_IMPL(BTYPE##4, UBTYPE##4)                              \
+  __CLC_MUL_HI_LONG_VEC_IMPL(BTYPE##8, UBTYPE##8)                              \
+  __CLC_MUL_HI_LONG_VEC_IMPL(BTYPE##16, UBTYPE##16)
+
+#define __CLC_MUL_HI_TYPES()                                                   \
+  __CLC_MUL_HI_DEC_IMPL(short, char, 8)                                        \
+  __CLC_MUL_HI_DEC_IMPL(ushort, uchar, 8)                                      \
+  __CLC_MUL_HI_DEC_IMPL(int, short, 16)                                        \
+  __CLC_MUL_HI_DEC_IMPL(uint, ushort, 16)                                      \
+  __CLC_MUL_HI_DEC_IMPL(long, int, 32)                                         \
+  __CLC_MUL_HI_DEC_IMPL(ulong, uint, 32)                                       \
+  __CLC_MUL_HI_LONG_IMPL(long, ulong)                                          \
+  __CLC_MUL_HI_LONG_IMPL(ulong, ulong)
+
+__CLC_MUL_HI_TYPES()
+
+#undef __CLC_MUL_HI_TYPES
+#undef __CLC_MUL_HI_LONG_IMPL
+#undef __CLC_MUL_HI_LONG_VEC_IMPL
+#undef __CLC_MUL_HI_DEC_IMPL
+#undef __CLC_MUL_HI_IMPL
+#undef __CLC_MUL_HI_VEC_IMPL
+#undef __CLC_CONVERT_TY
diff --git a/libclc/clc/lib/generic/integer/clc_popcount.cl b/libclc/clc/lib/generic/integer/clc_popcount.cl
new file mode 100644
index 0000000000000..12e851c15d795
--- /dev/null
+++ b/libclc/clc/lib/generic/integer/clc_popcount.cl
@@ -0,0 +1,7 @@
+#include <clc/internal/clc.h>
+
+#define FUNCTION __clc_popcount
+#define __CLC_FUNCTION(x) __builtin_elementwise_popcount
+#define __CLC_BODY <clc/shared/unary_def.inc>
+
+#include <clc/integer/gentype.inc>
diff --git a/libclc/clc/lib/generic/integer/clc_rhadd.cl b/libclc/clc/lib/generic/integer/clc_rhadd.cl
new file mode 100644
index 0000000000000..00bd2f0ac8058
--- /dev/null
+++ b/libclc/clc/lib/generic/integer/clc_rhadd.cl
@@ -0,0 +1,4 @@
+#include <clc/internal/clc.h>
+
+#define __CLC_BODY <clc_rhadd.inc>
+#include <clc/integer/gentype.inc>
diff --git a/libclc/clc/lib/generic/integer/clc_rhadd.inc b/libclc/clc/lib/generic/integer/clc_rhadd.inc
new file mode 100644
index 0000000000000..d363c42061ffe
--- /dev/null
+++ b/libclc/clc/lib/generic/integer/clc_rhadd.inc
@@ -0,0 +1,8 @@
+// rhadd = (x+y+1)>>1
+// This can be simplified to x>>1 + y>>1 + (1 if either x or y have the 1s bit
+// set) This saves us having to do any checks for overflow in the addition sums
+_CLC_OVERLOAD _CLC_DEF __CLC_GENTYPE __clc_rhadd(__CLC_GENTYPE x,
+                                                 __CLC_GENTYPE y) {
+  return (x >> (__CLC_GENTYPE)1) + (y >> (__CLC_GENTYPE)1) +
+         ((x & (__CLC_GENTYPE)1) | (y & (__CLC_GENTYPE)1));
+}
diff --git a/libclc/clc/lib/generic/integer/clc_rotate.cl b/libclc/clc/lib/generic/integer/clc_rotate.cl
new file mode 100644
index 0000000000000..7546862fe401e
--- /dev/null
+++ b/libclc/clc/lib/generic/integer/clc_rotate.cl
@@ -0,0 +1,5 @@
+#include <clc/internal/clc.h>
+#include <clc/utils.h>
+
+#define __CLC_BODY <clc_rotate.inc>
+#include <clc/integer/gentype.inc>
diff --git a/libclc/clc/lib/generic/integer/clc_rotate.inc b/libclc/clc/lib/generic/integer/clc_rotate.inc
new file mode 100644
index 0000000000000..f144553eabd52
--- /dev/null
+++ b/libclc/clc/lib/generic/integer/clc_rotate.inc
@@ -0,0 +1,22 @@
+#define __CLC_AS_GENTYPE(x) __CLC_XCONCAT(__clc_as_, __CLC_GENTYPE)(x)
+#define __CLC_AS_U_GENTYPE(x) __CLC_XCONCAT(__clc_as_, __CLC_U_GENTYPE)(x)
+
+// The rotate(A, B) builtin left-shifts corresponding to the usual OpenCL shift
+// modulo rules. These rules state that A is left-shifted by the log2(N) least
+// significant bits in B when viewed as an unsigned integer value. Thus we don't
+// have to worry about signed shift amounts, and can perform the computation in
+// unsigned types.
+_CLC_OVERLOAD _CLC_DEF __CLC_GENTYPE __clc_rotate(__CLC_GENTYPE x,
+                                                  __CLC_GENTYPE n) {
+  __CLC_U_GENTYPE x_as_u = __CLC_AS_U_GENTYPE(x);
+  __CLC_U_GENTYPE mask = (__CLC_U_GENTYPE)(__CLC_GENSIZE - 1);
+
+  __CLC_U_GENTYPE lshift_amt = __CLC_AS_U_GENTYPE(n) & mask;
+
+  __CLC_U_GENTYPE rshift_amt =
+      (((__CLC_U_GENTYPE)__CLC_GENSIZE - lshift_amt) & mask);
+
+  __CLC_U_GENTYPE result = (x_as_u << lshift_amt) | (x_as_u >> rshift_amt);
+
+  return __CLC_AS_GENTYPE(result);
+}
diff --git a/libclc/clc/lib/generic/integer/clc_sub_sat.cl b/libclc/clc/lib/generic/integer/clc_sub_sat.cl
new file mode 100644
index 0000000000000..184713531fdbe
--- /dev/null
+++ b/libclc/clc/lib/generic/integer/clc_sub_sat.cl
@@ -0,0 +1,7 @@
+#include <clc/internal/clc.h>
+
+#define FUNCTION __clc_sub_sat
+#define __CLC_FUNCTION(x) __builtin_elementwise_sub_sat
+#define __CLC_BODY <clc/shared/binary_def.inc>
+
+#include <clc/integer/gentype.inc>
diff --git a/libclc/clc/lib/generic/integer/clc_upsample.cl b/libclc/clc/lib/generic/integer/clc_upsample.cl
new file mode 100644
index 0000000000000..d53ef7240bfc2
--- /dev/null
+++ b/libclc/clc/lib/generic/integer/clc_upsample.cl
@@ -0,0 +1,35 @@
+#include <clc/internal/clc.h>
+
+// TODO: Replace with __clc_convert_<type> when available
+#define __CLC_CONVERT_TY(X, TY) __builtin_convertvector(X, TY)
+
+#define __CLC_UPSAMPLE_VEC_IMPL(BGENTYPE, GENTYPE, UGENTYPE, GENSIZE)          \
+  _CLC_OVERLOAD _CLC_DEF BGENTYPE __clc_upsample(GENTYPE hi, UGENTYPE lo) {    \
+    BGENTYPE large_hi = __CLC_CONVERT_TY(hi, BGENTYPE);                        \
+    BGENTYPE large_lo = __CLC_CONVERT_TY(lo, BGENTYPE);                        \
+    return (large_hi << (BGENTYPE)GENSIZE) | large_lo;                         \
+  }
+
+#define __CLC_UPSAMPLE_IMPL(BGENTYPE, GENTYPE, UGENTYPE, GENSIZE)              \
+  _CLC_OVERLOAD _CLC_DEF BGENTYPE __clc_upsample(GENTYPE hi, UGENTYPE lo) {    \
+    return ((BGENTYPE)hi << GENSIZE) | lo;                                     \
+  }                                                                            \
+  __CLC_UPSAMPLE_VEC_IMPL(BGENTYPE##2, GENTYPE##2, UGENTYPE##2, GENSIZE)       \
+  __CLC_UPSAMPLE_VEC_IMPL(BGENTYPE##3, GENTYPE##3, UGENTYPE##3, GENSIZE)       \
+  __CLC_UPSAMPLE_VEC_IMPL(BGENTYPE##4, GENTYPE##4, UGENTYPE##4, GENSIZE)       \
+  __CLC_UPSAMPLE_VEC_IMPL(BGENTYPE##8, GENTYPE##8, UGENTYPE##8, GENSIZE)       \
+  __CLC_UPSAMPLE_VEC_IMPL(BGENTYPE##16, GENTYPE##16, UGENTYPE##16, GENSIZE)
+
+#define __CLC_UPSAMPLE_TYPES()                                                 \
+  __CLC_UPSAMPLE_IMPL(short, char, uchar, 8)                                   \
+  __CLC_UPSAMPLE_IMPL(ushort, uchar, uchar, 8)                                 \
+  __CLC_UPSAMPLE_IMPL(int, short, ushort, 16)                                  \
+  __CLC_UPSAMPLE_IMPL(uint, ushort, ushort, 16)                                \
+  __CLC_UPSAMPLE_IMPL(long, int, uint, 32)                                     \
+  __CLC_UPSAMPLE_IMPL(ulong, uint, uint, 32)
+
+__CLC_UPSAMPLE_TYPES()
+
+#undef __CLC_UPSAMPLE_TYPES
+#undef __CLC_UPSAMPLE_IMPL
+#undef __CLC_CONVERT_TY
diff --git a/libclc/clc/lib/generic/math/clc_copysign.cl b/libclc/clc/lib/generic/math/clc_copysign.cl
new file mode 100644
index 0000000000000..e225ec80692fb
--- /dev/null
+++ b/libclc/clc/lib/generic/math/clc_copysign.cl
@@ -0,0 +1,27 @@
+#include <clc/clcmacro.h>
+#include <clc/internal/clc.h>
+
+_CLC_DEFINE_BINARY_BUILTIN_NO_SCALARIZE(float, __clc_copysign,
+                                        __builtin_elementwise_copysign, float,
+                                        float)
+
+#ifdef cl_khr_fp64
+
+#pragma OPENCL EXTENSION cl_khr_fp64 : enable
+
+_CLC_DEFINE_BINARY_BUILTIN_NO_SCALARIZE(double, __clc_copysign,
+                                        __builtin_elementwise_copysign, double,
+                                        double)
+
+#endif
+
+#ifdef cl_khr_fp16
+
+#pragma OPENCL EXTENSION cl_khr_fp16 : enable
+
+_CLC_DEFINE_BINARY_BUILTIN_NO_SCALARIZE(half, __clc_copysign,
+                                        __builtin_elementwise_copysign, half,
+                                        half)
+
+#endif
+
diff --git a/libclc/clc/lib/generic/relational/clc_all.cl b/libclc/clc/lib/generic/relational/clc_all.cl
index e371126d144f0..5193f0f743179 100644
--- a/libclc/clc/lib/generic/relational/clc_all.cl
+++ b/libclc/clc/lib/generic/relational/clc_all.cl
@@ -1,28 +1,21 @@
 #include <clc/internal/clc.h>
 
 #define _CLC_ALL(v) (((v) >> ((sizeof(v) * 8) - 1)) & 0x1)
-#define _CLC_ALL2(v) (_CLC_ALL((v).s0) & _CLC_ALL((v).s1))
-#define _CLC_ALL3(v) (_CLC_ALL2((v)) & _CLC_ALL((v).s2))
-#define _CLC_ALL4(v) (_CLC_ALL3((v)) & _CLC_ALL((v).s3))
-#define _CLC_ALL8(v)                                                           \
-  (_CLC_ALL4((v)) & _CLC_ALL((v).s4) & _CLC_ALL((v).s5) & _CLC_ALL((v).s6) &   \
-   _CLC_ALL((v).s7))
-#define _CLC_ALL16(v)                                                          \
-  (_CLC_ALL8((v)) & _CLC_ALL((v).s8) & _CLC_ALL((v).s9) & _CLC_ALL((v).sA) &   \
-   _CLC_ALL((v).sB) & _CLC_ALL((v).sC) & _CLC_ALL((v).sD) & _CLC_ALL((v).sE) & \
-   _CLC_ALL((v).sf))
 
-#define ALL_ID(TYPE) _CLC_OVERLOAD _CLC_DEF int __clc_all(TYPE v)
+#define _CLC_ALL_VEC(TYPE)                                                     \
+  _CLC_OVERLOAD _CLC_DEF int __clc_all(TYPE v) {                               \
+    return _CLC_ALL(__builtin_reduce_and(v));                                  \
+  }
 
-#define ALL_VECTORIZE(TYPE)                                                    \
-  ALL_ID(TYPE) { return _CLC_ALL(v); }                                         \
-  ALL_ID(TYPE##2) { return _CLC_ALL2(v); }                                     \
-  ALL_ID(TYPE##3) { return _CLC_ALL3(v); }                                     \
-  ALL_ID(TYPE##4) { return _CLC_ALL4(v); }                                     \
-  ALL_ID(TYPE##8) { return _CLC_ALL8(v); }                                     \
-  ALL_ID(TYPE##16) { return _CLC_ALL16(v); }
+#define _CLC_DEFINE_ALL(TYPE)                                                  \
+  _CLC_OVERLOAD _CLC_DEF int __clc_all(TYPE v) { return _CLC_ALL(v); }         \
+  _CLC_ALL_VEC(TYPE##2)                                                        \
+  _CLC_ALL_VEC(TYPE##3)                                                        \
+  _CLC_ALL_VEC(TYPE##4)                                                        \
+  _CLC_ALL_VEC(TYPE##8)                                                        \
+  _CLC_ALL_VEC(TYPE##16)
 
-ALL_VECTORIZE(char)
-ALL_VECTORIZE(short)
-ALL_VECTORIZE(int)
-ALL_VECTORIZE(long)
+_CLC_DEFINE_ALL(char)
+_CLC_DEFINE_ALL(short)
+_CLC_DEFINE_ALL(int)
+_CLC_DEFINE_ALL(long)
diff --git a/libclc/clc/lib/generic/relational/clc_any.cl b/libclc/clc/lib/generic/relational/clc_any.cl
index e69f2113c94f5..e86bafaac1993 100644
--- a/libclc/clc/lib/generic/relational/clc_any.cl
+++ b/libclc/clc/lib/generic/relational/clc_any.cl
@@ -1,28 +1,21 @@
 #include <clc/internal/clc.h>
 
 #define _CLC_ANY(v) (((v) >> ((sizeof(v) * 8) - 1)) & 0x1)
-#define _CLC_ANY2(v) (_CLC_ANY((v).s0) | _CLC_ANY((v).s1))
-#define _CLC_ANY3(v) (_CLC_ANY2((v)) | _CLC_ANY((v).s2))
-#define _CLC_ANY4(v) (_CLC_ANY3((v)) | _CLC_ANY((v).s3))
-#define _CLC_ANY8(v)                                                           \
-  (_CLC_ANY4((v)) | _CLC_ANY((v).s4) | _CLC_ANY((v).s5) | _CLC_ANY((v).s6) |   \
-   _CLC_ANY((v).s7))
-#define _CLC_ANY16(v)                                                          \
-  (_CLC_ANY8((v)) | _CLC_ANY((v).s8) | _CLC_ANY((v).s9) | _CLC_ANY((v).sA) |   \
-   _CLC_ANY((v).sB) | _CLC_ANY((v).sC) | _CLC_ANY((v).sD) | _CLC_ANY((v).sE) | \
-   _CLC_ANY((v).sf))
 
-#define ANY_ID(TYPE) _CLC_OVERLOAD _CLC_DEF int __clc_any(TYPE v)
+#define _CLC_ANY_VEC(TYPE)                                                     \
+  _CLC_OVERLOAD _CLC_DEF int __clc_any(TYPE v) {                               \
+    return _CLC_ANY(__builtin_reduce_or(v));                                   \
+  }
 
-#define ANY_VECTORIZE(TYPE)                                                    \
-  ANY_ID(TYPE) { return _CLC_ANY(v); }                                         \
-  ANY_ID(TYPE##2) { return _CLC_ANY2(v); }                                     \
-  ANY_ID(TYPE##3) { return _CLC_ANY3(v); }                                     \
-  ANY_ID(TYPE##4) { return _CLC_ANY4(v); }                                     \
-  ANY_ID(TYPE##8) { return _CLC_ANY8(v); }                                     \
-  ANY_ID(TYPE##16) { return _CLC_ANY16(v); }
+#define _CLC_DEFINE_ANY(TYPE)                                                  \
+  _CLC_OVERLOAD _CLC_DEF int __clc_any(TYPE v) { return _CLC_ANY(v); }         \
+  _CLC_ANY_VEC(TYPE##2)                                                        \
+  _CLC_ANY_VEC(TYPE##3)                                                        \
+  _CLC_ANY_VEC(TYPE##4)                                                        \
+  _CLC_ANY_VEC(TYPE##8)                                                        \
+  _CLC_ANY_VEC(TYPE##16)
 
-ANY_VECTORIZE(char)
-ANY_VECTORIZE(short)
-ANY_VECTORIZE(int)
-ANY_VECTORIZE(long)
+_CLC_DEFINE_ANY(char)
+_CLC_DEFINE_ANY(short)
+_CLC_DEFINE_ANY(int)
+_CLC_DEFINE_ANY(long)
diff --git a/libclc/clc/lib/generic/relational/clc_isequal.cl b/libclc/clc/lib/generic/relational/clc_isequal.cl
index 7664df7767cb3..053a237289fd6 100644
--- a/libclc/clc/lib/generic/relational/clc_isequal.cl
+++ b/libclc/clc/lib/generic/relational/clc_isequal.cl
@@ -1,44 +1,28 @@
 #include <clc/internal/clc.h>
+#include <clc/relational/relational.h>
 
-#define _CLC_DEFINE_ISEQUAL(RET_TYPE, FUNCTION, ARG1_TYPE, ARG2_TYPE)          \
-  _CLC_DEF _CLC_OVERLOAD RET_TYPE FUNCTION(ARG1_TYPE x, ARG2_TYPE y) {         \
-    return (x == y);                                                           \
-  }
+#define _CLC_RELATIONAL_OP(X, Y) (X) == (Y)
 
-_CLC_DEFINE_ISEQUAL(int, __clc_isequal, float, float)
-_CLC_DEFINE_ISEQUAL(int2, __clc_isequal, float2, float2)
-_CLC_DEFINE_ISEQUAL(int3, __clc_isequal, float3, float3)
-_CLC_DEFINE_ISEQUAL(int4, __clc_isequal, float4, float4)
-_CLC_DEFINE_ISEQUAL(int8, __clc_isequal, float8, float8)
-_CLC_DEFINE_ISEQUAL(int16, __clc_isequal, float16, float16)
+_CLC_DEFINE_SIMPLE_RELATIONAL_BINARY(int, int, __clc_isequal, float, float)
 
 #ifdef cl_khr_fp64
 
 #pragma OPENCL EXTENSION cl_khr_fp64 : enable
 
-// The scalar version of __clc_isequal(double) returns an int, but the vector
-// versions return long.
-_CLC_DEFINE_ISEQUAL(int, __clc_isequal, double, double)
-_CLC_DEFINE_ISEQUAL(long2, __clc_isequal, double2, double2)
-_CLC_DEFINE_ISEQUAL(long3, __clc_isequal, double3, double3)
-_CLC_DEFINE_ISEQUAL(long4, __clc_isequal, double4, double4)
-_CLC_DEFINE_ISEQUAL(long8, __clc_isequal, double8, double8)
-_CLC_DEFINE_ISEQUAL(long16, __clc_isequal, double16, double16)
+// The scalar version of __clc_isequal(double, double) returns an int, but the
+// vector versions return long.
+_CLC_DEFINE_SIMPLE_RELATIONAL_BINARY(int, long, __clc_isequal, double, double)
 
 #endif
+
 #ifdef cl_khr_fp16
 
 #pragma OPENCL EXTENSION cl_khr_fp16 : enable
 
-// The scalar version of __clc_isequal(half) returns an int, but the vector
-// versions return short.
-_CLC_DEFINE_ISEQUAL(int, __clc_isequal, half, half)
-_CLC_DEFINE_ISEQUAL(short2, __clc_isequal, half2, half2)
-_CLC_DEFINE_ISEQUAL(short3, __clc_isequal, half3, half3)
-_CLC_DEFINE_ISEQUAL(short4, __clc_isequal, half4, half4)
-_CLC_DEFINE_ISEQUAL(short8, __clc_isequal, half8, half8)
-_CLC_DEFINE_ISEQUAL(short16, __clc_isequal, half16, half16)
+// The scalar version of __clc_isequal(half, half) returns an int, but the
+// vector versions return short.
+_CLC_DEFINE_SIMPLE_RELATIONAL_BINARY(int, short, __clc_isequal, half, half)
 
 #endif
 
-#undef _CLC_DEFINE_ISEQUAL
+#undef _CLC_RELATIONAL_OP
diff --git a/libclc/clc/lib/generic/relational/clc_isfinite.cl b/libclc/clc/lib/generic/relational/clc_isfinite.cl
index c3def5dc5f0d5..2d28f6f4cccf7 100644
--- a/libclc/clc/lib/generic/relational/clc_isfinite.cl
+++ b/libclc/clc/lib/generic/relational/clc_isfinite.cl
@@ -1,7 +1,7 @@
 #include <clc/internal/clc.h>
 #include <clc/relational/relational.h>
 
-_CLC_DEFINE_RELATIONAL_UNARY(int, __clc_isfinite, __builtin_isfinite, float)
+_CLC_DEFINE_ISFPCLASS(int, int, __clc_isfinite, fcFinite, float)
 
 #ifdef cl_khr_fp64
 
@@ -9,23 +9,16 @@ _CLC_DEFINE_RELATIONAL_UNARY(int, __clc_isfinite, __builtin_isfinite, float)
 
 // The scalar version of __clc_isfinite(double) returns an int, but the vector
 // versions return long.
-_CLC_DEF _CLC_OVERLOAD int __clc_isfinite(double x) {
-  return __builtin_isfinite(x);
-}
-
-_CLC_DEFINE_RELATIONAL_UNARY_VEC_ALL(long, __clc_isfinite, double)
+_CLC_DEFINE_ISFPCLASS(int, long, __clc_isfinite, fcFinite, double)
 
 #endif
+
 #ifdef cl_khr_fp16
 
 #pragma OPENCL EXTENSION cl_khr_fp16 : enable
 
 // The scalar version of __clc_isfinite(half) returns an int, but the vector
 // versions return short.
-_CLC_DEF _CLC_OVERLOAD int __clc_isfinite(half x) {
-  return __builtin_isfinite(x);
-}
-
-_CLC_DEFINE_RELATIONAL_UNARY_VEC_ALL(short, __clc_isfinite, half)
+_CLC_DEFINE_ISFPCLASS(int, short, __clc_isfinite, fcFinite, half)
 
 #endif
diff --git a/libclc/clc/lib/generic/relational/clc_isgreater.cl b/libclc/clc/lib/generic/relational/clc_isgreater.cl
index 39fb6b07fb185..ec14fa9a2ec08 100644
--- a/libclc/clc/lib/generic/relational/clc_isgreater.cl
+++ b/libclc/clc/lib/generic/relational/clc_isgreater.cl
@@ -1,12 +1,9 @@
 #include <clc/internal/clc.h>
 #include <clc/relational/relational.h>
 
-// Note: It would be nice to use __builtin_isgreater with vector inputs, but it
-// seems to only take scalar values as input, which will produce incorrect
-// output for vector input types.
+#define _CLC_RELATIONAL_OP(X, Y) (X) > (Y)
 
-_CLC_DEFINE_RELATIONAL_BINARY(int, __clc_isgreater, __builtin_isgreater, float,
-                              float)
+_CLC_DEFINE_SIMPLE_RELATIONAL_BINARY(int, int, __clc_isgreater, float, float)
 
 #ifdef cl_khr_fp64
 
@@ -14,12 +11,7 @@ _CLC_DEFINE_RELATIONAL_BINARY(int, __clc_isgreater, __builtin_isgreater, float,
 
 // The scalar version of __clc_isgreater(double, double) returns an int, but the
 // vector versions return long.
-
-_CLC_DEF _CLC_OVERLOAD int __clc_isgreater(double x, double y) {
-  return __builtin_isgreater(x, y);
-}
-
-_CLC_DEFINE_RELATIONAL_BINARY_VEC_ALL(long, __clc_isgreater, double, double)
+_CLC_DEFINE_SIMPLE_RELATIONAL_BINARY(int, long, __clc_isgreater, double, double)
 
 #endif
 
@@ -29,11 +21,8 @@ _CLC_DEFINE_RELATIONAL_BINARY_VEC_ALL(long, __clc_isgreater, double, double)
 
 // The scalar version of __clc_isgreater(half, half) returns an int, but the
 // vector versions return short.
-
-_CLC_DEF _CLC_OVERLOAD int __clc_isgreater(half x, half y) {
-  return __builtin_isgreater(x, y);
-}
-
-_CLC_DEFINE_RELATIONAL_BINARY_VEC_ALL(short, __clc_isgreater, half, half)
+_CLC_DEFINE_SIMPLE_RELATIONAL_BINARY(int, short, __clc_isgreater, half, half)
 
 #endif
+
+#undef _CLC_RELATIONAL_OP
diff --git a/libclc/clc/lib/generic/relational/clc_isgreaterequal.cl b/libclc/clc/lib/generic/relational/clc_isgreaterequal.cl
index ccf7c881a5549..e96f2325cbad4 100644
--- a/libclc/clc/lib/generic/relational/clc_isgreaterequal.cl
+++ b/libclc/clc/lib/generic/relational/clc_isgreaterequal.cl
@@ -1,12 +1,10 @@
 #include <clc/internal/clc.h>
 #include <clc/relational/relational.h>
 
-// Note: It would be nice to use __builtin_isgreaterequal with vector inputs,
-// but it seems to only take scalar values as input, which will produce
-// incorrect output for vector input types.
+#define _CLC_RELATIONAL_OP(X, Y) (X) >= (Y)
 
-_CLC_DEFINE_RELATIONAL_BINARY(int, __clc_isgreaterequal,
-                              __builtin_isgreaterequal, float, float)
+_CLC_DEFINE_SIMPLE_RELATIONAL_BINARY(int, int, __clc_isgreaterequal, float,
+                                     float)
 
 #ifdef cl_khr_fp64
 
@@ -14,26 +12,20 @@ _CLC_DEFINE_RELATIONAL_BINARY(int, __clc_isgreaterequal,
 
 // The scalar version of __clc_isgreaterequal(double, double) returns an int,
 // but the vector versions return long.
-
-_CLC_DEF _CLC_OVERLOAD int __clc_isgreaterequal(double x, double y) {
-  return __builtin_isgreaterequal(x, y);
-}
-
-_CLC_DEFINE_RELATIONAL_BINARY_VEC_ALL(long, __clc_isgreaterequal, double,
-                                      double)
+_CLC_DEFINE_SIMPLE_RELATIONAL_BINARY(int, long, __clc_isgreaterequal, double,
+                                     double)
 
 #endif
+
 #ifdef cl_khr_fp16
 
 #pragma OPENCL EXTENSION cl_khr_fp16 : enable
 
-// The scalar version of __clc_isgreaterequal(half, half) returns an int, but
+// The scalar version of __clc_isgreaterequal(half, hafl) returns an int, but
 // the vector versions return short.
-
-_CLC_DEF _CLC_OVERLOAD int __clc_isgreaterequal(half x, half y) {
-  return __builtin_isgreaterequal(x, y);
-}
-
-_CLC_DEFINE_RELATIONAL_BINARY_VEC_ALL(short, __clc_isgreaterequal, half, half)
+_CLC_DEFINE_SIMPLE_RELATIONAL_BINARY(int, short, __clc_isgreaterequal, half,
+                                     half)
 
 #endif
+
+#undef _CLC_RELATIONAL_OP
diff --git a/libclc/clc/lib/generic/relational/clc_isinf.cl b/libclc/clc/lib/generic/relational/clc_isinf.cl
index afe29122f36a3..799c017c22bbc 100644
--- a/libclc/clc/lib/generic/relational/clc_isinf.cl
+++ b/libclc/clc/lib/generic/relational/clc_isinf.cl
@@ -1,7 +1,7 @@
 #include <clc/internal/clc.h>
 #include <clc/relational/relational.h>
 
-_CLC_DEFINE_RELATIONAL_UNARY(int, __clc_isinf, __builtin_isinf, float)
+_CLC_DEFINE_ISFPCLASS(int, int, __clc_isinf, fcInf, float)
 
 #ifdef cl_khr_fp64
 
@@ -9,9 +9,8 @@ _CLC_DEFINE_RELATIONAL_UNARY(int, __clc_isinf, __builtin_isinf, float)
 
 // The scalar version of __clc_isinf(double) returns an int, but the vector
 // versions return long.
-_CLC_DEF _CLC_OVERLOAD int __clc_isinf(double x) { return __builtin_isinf(x); }
+_CLC_DEFINE_ISFPCLASS(int, long, __clc_isinf, fcInf, double)
 
-_CLC_DEFINE_RELATIONAL_UNARY_VEC_ALL(long, __clc_isinf, double)
 #endif
 
 #ifdef cl_khr_fp16
@@ -20,7 +19,6 @@ _CLC_DEFINE_RELATIONAL_UNARY_VEC_ALL(long, __clc_isinf, double)
 
 // The scalar version of __clc_isinf(half) returns an int, but the vector
 // versions return short.
-_CLC_DEF _CLC_OVERLOAD int __clc_isinf(half x) { return __builtin_isinf(x); }
+_CLC_DEFINE_ISFPCLASS(int, short, __clc_isinf, fcInf, half)
 
-_CLC_DEFINE_RELATIONAL_UNARY_VEC_ALL(short, __clc_isinf, half)
 #endif
diff --git a/libclc/clc/lib/generic/relational/clc_isless.cl b/libclc/clc/lib/generic/relational/clc_isless.cl
index 1204a5057d864..0ce001d31d696 100644
--- a/libclc/clc/lib/generic/relational/clc_isless.cl
+++ b/libclc/clc/lib/generic/relational/clc_isless.cl
@@ -1,37 +1,28 @@
 #include <clc/internal/clc.h>
 #include <clc/relational/relational.h>
 
-// Note: It would be nice to use __builtin_isless with vector inputs, but it
-// seems to only take scalar values as input, which will produce incorrect
-// output for vector input types.
+#define _CLC_RELATIONAL_OP(X, Y) (X) < (Y)
 
-_CLC_DEFINE_RELATIONAL_BINARY(int, __clc_isless, __builtin_isless, float, float)
+_CLC_DEFINE_SIMPLE_RELATIONAL_BINARY(int, int, __clc_isless, float, float)
 
 #ifdef cl_khr_fp64
 
 #pragma OPENCL EXTENSION cl_khr_fp64 : enable
 
-// The scalar version of __clc_isless(double, double) returns an int, but the
-// vector versions return long.
-
-_CLC_DEF _CLC_OVERLOAD int __clc_isless(double x, double y) {
-  return __builtin_isless(x, y);
-}
-
-_CLC_DEFINE_RELATIONAL_BINARY_VEC_ALL(long, __clc_isless, double, double)
+// The scalar version of __clc_isless(double, double) returns an int, but
+// the vector versions return long.
+_CLC_DEFINE_SIMPLE_RELATIONAL_BINARY(int, long, __clc_isless, double, double)
 
 #endif
+
 #ifdef cl_khr_fp16
 
 #pragma OPENCL EXTENSION cl_khr_fp16 : enable
 
-// The scalar version of __clc_isless(half, half) returns an int, but the vector
-// versions return short.
-
-_CLC_DEF _CLC_OVERLOAD int __clc_isless(half x, half y) {
-  return __builtin_isless(x, y);
-}
-
-_CLC_DEFINE_RELATIONAL_BINARY_VEC_ALL(short, __clc_isless, half, half)
+// The scalar version of __clc_isless(half, half) returns an int, but the
+// vector versions return short.
+_CLC_DEFINE_SIMPLE_RELATIONAL_BINARY(int, short, __clc_isless, half, half)
 
 #endif
+
+#undef _CLC_RELATIONAL_OP
diff --git a/libclc/clc/lib/generic/relational/clc_islessequal.cl b/libclc/clc/lib/generic/relational/clc_islessequal.cl
index 6fde763263e2b..2d1d6d199fdab 100644
--- a/libclc/clc/lib/generic/relational/clc_islessequal.cl
+++ b/libclc/clc/lib/generic/relational/clc_islessequal.cl
@@ -1,12 +1,9 @@
 #include <clc/internal/clc.h>
 #include <clc/relational/relational.h>
 
-// Note: It would be nice to use __builtin_islessequal with vector inputs, but
-// it seems to only take scalar values as input, which will produce incorrect
-// output for vector input types.
+#define _CLC_RELATIONAL_OP(X, Y) (X) <= (Y)
 
-_CLC_DEFINE_RELATIONAL_BINARY(int, __clc_islessequal, __builtin_islessequal,
-                              float, float)
+_CLC_DEFINE_SIMPLE_RELATIONAL_BINARY(int, int, __clc_islessequal, float, float)
 
 #ifdef cl_khr_fp64
 
@@ -14,12 +11,8 @@ _CLC_DEFINE_RELATIONAL_BINARY(int, __clc_islessequal, __builtin_islessequal,
 
 // The scalar version of __clc_islessequal(double, double) returns an int, but
 // the vector versions return long.
-
-_CLC_DEF _CLC_OVERLOAD int __clc_islessequal(double x, double y) {
-  return __builtin_islessequal(x, y);
-}
-
-_CLC_DEFINE_RELATIONAL_BINARY_VEC_ALL(long, __clc_islessequal, double, double)
+_CLC_DEFINE_SIMPLE_RELATIONAL_BINARY(int, long, __clc_islessequal, double,
+                                     double)
 
 #endif
 
@@ -29,11 +22,8 @@ _CLC_DEFINE_RELATIONAL_BINARY_VEC_ALL(long, __clc_islessequal, double, double)
 
 // The scalar version of __clc_islessequal(half, half) returns an int, but the
 // vector versions return short.
-
-_CLC_DEF _CLC_OVERLOAD int __clc_islessequal(half x, half y) {
-  return __builtin_islessequal(x, y);
-}
-
-_CLC_DEFINE_RELATIONAL_BINARY_VEC_ALL(short, __clc_islessequal, half, half)
+_CLC_DEFINE_SIMPLE_RELATIONAL_BINARY(int, short, __clc_islessequal, half, half)
 
 #endif
+
+#undef _CLC_RELATIONAL_OP
diff --git a/libclc/clc/lib/generic/relational/clc_islessgreater.cl b/libclc/clc/lib/generic/relational/clc_islessgreater.cl
index 5106c9f460e2c..3ca3c37731d15 100644
--- a/libclc/clc/lib/generic/relational/clc_islessgreater.cl
+++ b/libclc/clc/lib/generic/relational/clc_islessgreater.cl
@@ -1,12 +1,10 @@
 #include <clc/internal/clc.h>
 #include <clc/relational/relational.h>
 
-// Note: It would be nice to use __builtin_islessgreater with vector inputs, but
-// it seems to only take scalar values as input, which will produce incorrect
-// output for vector input types.
+#define _CLC_RELATIONAL_OP(X, Y) ((X) < (Y)) || ((X) > (Y))
 
-_CLC_DEFINE_RELATIONAL_BINARY(int, __clc_islessgreater, __builtin_islessgreater,
-                              float, float)
+_CLC_DEFINE_SIMPLE_RELATIONAL_BINARY(int, int, __clc_islessgreater, float,
+                                     float)
 
 #ifdef cl_khr_fp64
 
@@ -14,25 +12,20 @@ _CLC_DEFINE_RELATIONAL_BINARY(int, __clc_islessgreater, __builtin_islessgreater,
 
 // The scalar version of __clc_islessgreater(double, double) returns an int, but
 // the vector versions return long.
-
-_CLC_DEF _CLC_OVERLOAD int __clc_islessgreater(double x, double y) {
-  return __builtin_islessgreater(x, y);
-}
-
-_CLC_DEFINE_RELATIONAL_BINARY_VEC_ALL(long, __clc_islessgreater, double, double)
+_CLC_DEFINE_SIMPLE_RELATIONAL_BINARY(int, long, __clc_islessgreater, double,
+                                     double)
 
 #endif
+
 #ifdef cl_khr_fp16
 
 #pragma OPENCL EXTENSION cl_khr_fp16 : enable
 
 // The scalar version of __clc_islessgreater(half, half) returns an int, but the
 // vector versions return short.
-
-_CLC_DEF _CLC_OVERLOAD int __clc_islessgreater(half x, half y) {
-  return __builtin_islessgreater(x, y);
-}
-
-_CLC_DEFINE_RELATIONAL_BINARY_VEC_ALL(short, __clc_islessgreater, half, half)
+_CLC_DEFINE_SIMPLE_RELATIONAL_BINARY(int, short, __clc_islessgreater, half,
+                                     half)
 
 #endif
+
+#undef _CLC_RELATIONAL_OP
diff --git a/libclc/clc/lib/generic/relational/clc_isnan.cl b/libclc/clc/lib/generic/relational/clc_isnan.cl
index fb30cd5419214..6a0672dfa87a8 100644
--- a/libclc/clc/lib/generic/relational/clc_isnan.cl
+++ b/libclc/clc/lib/generic/relational/clc_isnan.cl
@@ -1,17 +1,15 @@
 #include <clc/internal/clc.h>
 #include <clc/relational/relational.h>
 
-_CLC_DEFINE_RELATIONAL_UNARY(int, __clc_isnan, __builtin_isnan, float)
+_CLC_DEFINE_ISFPCLASS(int, int, __clc_isnan, fcNan, float)
 
 #ifdef cl_khr_fp64
 
 #pragma OPENCL EXTENSION cl_khr_fp64 : enable
 
 // The scalar version of __clc_isnan(double) returns an int, but the vector
-// versions return long.
-_CLC_DEF _CLC_OVERLOAD int __clc_isnan(double x) { return __builtin_isnan(x); }
-
-_CLC_DEFINE_RELATIONAL_UNARY_VEC_ALL(long, __clc_isnan, double)
+// versions return a long.
+_CLC_DEFINE_ISFPCLASS(int, long, __clc_isnan, fcNan, double)
 
 #endif
 
@@ -20,9 +18,7 @@ _CLC_DEFINE_RELATIONAL_UNARY_VEC_ALL(long, __clc_isnan, double)
 #pragma OPENCL EXTENSION cl_khr_fp16 : enable
 
 // The scalar version of __clc_isnan(half) returns an int, but the vector
-// versions return short.
-_CLC_DEF _CLC_OVERLOAD int __clc_isnan(half x) { return __builtin_isnan(x); }
-
-_CLC_DEFINE_RELATIONAL_UNARY_VEC_ALL(short, __clc_isnan, half)
+// versions return a short.
+_CLC_DEFINE_ISFPCLASS(int, short, __clc_isnan, fcNan, half)
 
 #endif
diff --git a/libclc/clc/lib/generic/relational/clc_isnormal.cl b/libclc/clc/lib/generic/relational/clc_isnormal.cl
index e0da8cc0756f4..1e31b04e2a6fd 100644
--- a/libclc/clc/lib/generic/relational/clc_isnormal.cl
+++ b/libclc/clc/lib/generic/relational/clc_isnormal.cl
@@ -1,7 +1,7 @@
 #include <clc/internal/clc.h>
 #include <clc/relational/relational.h>
 
-_CLC_DEFINE_RELATIONAL_UNARY(int, __clc_isnormal, __builtin_isnormal, float)
+_CLC_DEFINE_ISFPCLASS(int, int, __clc_isnormal, fcNormal, float)
 
 #ifdef cl_khr_fp64
 
@@ -9,23 +9,16 @@ _CLC_DEFINE_RELATIONAL_UNARY(int, __clc_isnormal, __builtin_isnormal, float)
 
 // The scalar version of __clc_isnormal(double) returns an int, but the vector
 // versions return long.
-_CLC_DEF _CLC_OVERLOAD int __clc_isnormal(double x) {
-  return __builtin_isnormal(x);
-}
-
-_CLC_DEFINE_RELATIONAL_UNARY_VEC_ALL(long, __clc_isnormal, double)
+_CLC_DEFINE_ISFPCLASS(int, long, __clc_isnormal, fcNormal, double)
 
 #endif
+
 #ifdef cl_khr_fp16
 
 #pragma OPENCL EXTENSION cl_khr_fp16 : enable
 
 // The scalar version of __clc_isnormal(half) returns an int, but the vector
 // versions return short.
-_CLC_DEF _CLC_OVERLOAD int __clc_isnormal(half x) {
-  return __builtin_isnormal(x);
-}
-
-_CLC_DEFINE_RELATIONAL_UNARY_VEC_ALL(short, __clc_isnormal, half)
+_CLC_DEFINE_ISFPCLASS(int, short, __clc_isnormal, fcNormal, half)
 
 #endif
diff --git a/libclc/clc/lib/generic/relational/clc_isnotequal.cl b/libclc/clc/lib/generic/relational/clc_isnotequal.cl
index 9f90713b2da50..d1ee4deab25c8 100644
--- a/libclc/clc/lib/generic/relational/clc_isnotequal.cl
+++ b/libclc/clc/lib/generic/relational/clc_isnotequal.cl
@@ -1,33 +1,28 @@
 #include <clc/internal/clc.h>
 #include <clc/relational/relational.h>
 
-#define _CLC_DEFINE_ISNOTEQUAL(RET_TYPE, FUNCTION, ARG1_TYPE, ARG2_TYPE)       \
-  _CLC_DEF _CLC_OVERLOAD RET_TYPE FUNCTION(ARG1_TYPE x, ARG2_TYPE y) {         \
-    return (x != y);                                                           \
-  }
+#define _CLC_RELATIONAL_OP(X, Y) (X) != (Y)
 
-_CLC_DEFINE_ISNOTEQUAL(int, __clc_isnotequal, float, float)
-_CLC_DEFINE_RELATIONAL_BINARY_VEC_ALL(int, __clc_isnotequal, float, float)
+_CLC_DEFINE_SIMPLE_RELATIONAL_BINARY(int, int, __clc_isnotequal, float, float)
 
 #ifdef cl_khr_fp64
+
 #pragma OPENCL EXTENSION cl_khr_fp64 : enable
 
 // The scalar version of __clc_isnotequal(double, double) returns an int, but
 // the vector versions return long.
-
-_CLC_DEFINE_ISNOTEQUAL(int, __clc_isnotequal, double, double)
-_CLC_DEFINE_RELATIONAL_BINARY_VEC_ALL(long, __clc_isnotequal, double, double)
+_CLC_DEFINE_SIMPLE_RELATIONAL_BINARY(int, long, __clc_isnotequal, double, double)
 
 #endif
+
 #ifdef cl_khr_fp16
+
 #pragma OPENCL EXTENSION cl_khr_fp16 : enable
 
 // The scalar version of __clc_isnotequal(half, half) returns an int, but the
 // vector versions return short.
-
-_CLC_DEFINE_ISNOTEQUAL(int, __clc_isnotequal, half, half)
-_CLC_DEFINE_RELATIONAL_BINARY_VEC_ALL(short, __clc_isnotequal, half, half)
+_CLC_DEFINE_SIMPLE_RELATIONAL_BINARY(int, short, __clc_isnotequal, half, half)
 
 #endif
 
-#undef _CLC_DEFINE_ISNOTEQUAL
+#undef _CLC_RELATIONAL_OP
diff --git a/libclc/clc/lib/generic/relational/clc_isordered.cl b/libclc/clc/lib/generic/relational/clc_isordered.cl
index 6183d1ddf918f..73cd96a0a56ed 100644
--- a/libclc/clc/lib/generic/relational/clc_isordered.cl
+++ b/libclc/clc/lib/generic/relational/clc_isordered.cl
@@ -2,33 +2,29 @@
 #include <clc/relational/clc_isequal.h>
 #include <clc/relational/relational.h>
 
-#define _CLC_DEFINE_ISORDERED(RET_TYPE, FUNCTION, ARG1_TYPE, ARG2_TYPE)        \
-  _CLC_DEF _CLC_OVERLOAD RET_TYPE FUNCTION(ARG1_TYPE x, ARG2_TYPE y) {         \
-    return __clc_isequal(x, x) && __clc_isequal(y, y);                         \
-  }
+#define _CLC_RELATIONAL_OP(X, Y)                                               \
+  __clc_isequal((X), (X)) && __clc_isequal((Y), (Y))
 
-_CLC_DEFINE_ISORDERED(int, __clc_isordered, float, float)
-_CLC_DEFINE_RELATIONAL_BINARY_VEC_ALL(int, __clc_isordered, float, float)
+_CLC_DEFINE_SIMPLE_RELATIONAL_BINARY(int, int, __clc_isordered, float, float)
 
 #ifdef cl_khr_fp64
+
 #pragma OPENCL EXTENSION cl_khr_fp64 : enable
 
 // The scalar version of __clc_isordered(double, double) returns an int, but the
 // vector versions return long.
-
-_CLC_DEFINE_ISORDERED(int, __clc_isordered, double, double)
-_CLC_DEFINE_RELATIONAL_BINARY_VEC_ALL(long, __clc_isordered, double, double)
+_CLC_DEFINE_SIMPLE_RELATIONAL_BINARY(int, long, __clc_isordered, double, double)
 
 #endif
+
 #ifdef cl_khr_fp16
+
 #pragma OPENCL EXTENSION cl_khr_fp16 : enable
 
 // The scalar version of __clc_isordered(half, half) returns an int, but the
 // vector versions return short.
-
-_CLC_DEFINE_ISORDERED(int, __clc_isordered, half, half)
-_CLC_DEFINE_RELATIONAL_BINARY_VEC_ALL(short, __clc_isordered, half, half)
+_CLC_DEFINE_SIMPLE_RELATIONAL_BINARY(int, short, __clc_isordered, half, half)
 
 #endif
 
-#undef _CLC_DEFINE_ISORDERED
+#undef _CLC_RELATIONAL_OP
diff --git a/libclc/clc/lib/generic/relational/clc_isunordered.cl b/libclc/clc/lib/generic/relational/clc_isunordered.cl
index dbbec031a65e5..fefda8e567517 100644
--- a/libclc/clc/lib/generic/relational/clc_isunordered.cl
+++ b/libclc/clc/lib/generic/relational/clc_isunordered.cl
@@ -1,12 +1,11 @@
 #include <clc/internal/clc.h>
+#include <clc/relational/clc_isequal.h>
 #include <clc/relational/relational.h>
 
-// Note: It would be nice to use __builtin_isunordered with vector inputs, but
-// it seems to only take scalar values as input, which will produce incorrect
-// output for vector input types.
+#define _CLC_RELATIONAL_OP(X, Y)                                               \
+  !__clc_isequal((X), (X)) || !__clc_isequal((Y), (Y))
 
-_CLC_DEFINE_RELATIONAL_BINARY(int, __clc_isunordered, __builtin_isunordered,
-                              float, float)
+_CLC_DEFINE_SIMPLE_RELATIONAL_BINARY(int, int, __clc_isunordered, float, float)
 
 #ifdef cl_khr_fp64
 
@@ -14,25 +13,18 @@ _CLC_DEFINE_RELATIONAL_BINARY(int, __clc_isunordered, __builtin_isunordered,
 
 // The scalar version of __clc_isunordered(double, double) returns an int, but
 // the vector versions return long.
-
-_CLC_DEF _CLC_OVERLOAD int __clc_isunordered(double x, double y) {
-  return __builtin_isunordered(x, y);
-}
-
-_CLC_DEFINE_RELATIONAL_BINARY_VEC_ALL(long, __clc_isunordered, double, double)
+_CLC_DEFINE_SIMPLE_RELATIONAL_BINARY(int, long, __clc_isunordered, double, double)
 
 #endif
+
 #ifdef cl_khr_fp16
 
 #pragma OPENCL EXTENSION cl_khr_fp16 : enable
 
 // The scalar version of __clc_isunordered(half, half) returns an int, but the
 // vector versions return short.
-
-_CLC_DEF _CLC_OVERLOAD int __clc_isunordered(half x, half y) {
-  return __builtin_isunordered(x, y);
-}
-
-_CLC_DEFINE_RELATIONAL_BINARY_VEC_ALL(short, __clc_isunordered, half, half)
+_CLC_DEFINE_SIMPLE_RELATIONAL_BINARY(int, short, __clc_isunordered, half, half)
 
 #endif
+
+#undef _CLC_RELATIONAL_OP
diff --git a/libclc/clc/lib/generic/relational/clc_signbit.cl b/libclc/clc/lib/generic/relational/clc_signbit.cl
index b1b294379e5a8..67043c42d0ebb 100644
--- a/libclc/clc/lib/generic/relational/clc_signbit.cl
+++ b/libclc/clc/lib/generic/relational/clc_signbit.cl
@@ -1,7 +1,56 @@
 #include <clc/internal/clc.h>
 #include <clc/relational/relational.h>
 
-_CLC_DEFINE_RELATIONAL_UNARY(int, __clc_signbit, __builtin_signbitf, float)
+#define _CLC_DEFINE_RELATIONAL_UNARY_VEC2(RET_TYPE, FUNCTION, ARG_TYPE)        \
+  _CLC_DEF _CLC_OVERLOAD RET_TYPE FUNCTION(ARG_TYPE x) {                       \
+    return (RET_TYPE)((RET_TYPE){FUNCTION(x.lo), FUNCTION(x.hi)} !=            \
+                      (RET_TYPE)0);                                            \
+  }
+
+#define _CLC_DEFINE_RELATIONAL_UNARY_VEC3(RET_TYPE, FUNCTION, ARG_TYPE)        \
+  _CLC_DEF _CLC_OVERLOAD RET_TYPE FUNCTION(ARG_TYPE x) {                       \
+    return (RET_TYPE)((RET_TYPE){FUNCTION(x.s0), FUNCTION(x.s1),               \
+                                 FUNCTION(x.s2)} != (RET_TYPE)0);              \
+  }
+
+#define _CLC_DEFINE_RELATIONAL_UNARY_VEC4(RET_TYPE, FUNCTION, ARG_TYPE)        \
+  _CLC_DEF _CLC_OVERLOAD RET_TYPE FUNCTION(ARG_TYPE x) {                       \
+    return (RET_TYPE)((RET_TYPE){FUNCTION(x.s0), FUNCTION(x.s1),               \
+                                 FUNCTION(x.s2),                               \
+                                 FUNCTION(x.s3)} != (RET_TYPE)0);              \
+  }
+
+#define _CLC_DEFINE_RELATIONAL_UNARY_VEC8(RET_TYPE, FUNCTION, ARG_TYPE)        \
+  _CLC_DEF _CLC_OVERLOAD RET_TYPE FUNCTION(ARG_TYPE x) {                       \
+    return (                                                                   \
+        RET_TYPE)((RET_TYPE){FUNCTION(x.s0), FUNCTION(x.s1), FUNCTION(x.s2),   \
+                             FUNCTION(x.s3), FUNCTION(x.s4), FUNCTION(x.s5),   \
+                             FUNCTION(x.s6), FUNCTION(x.s7)} != (RET_TYPE)0);  \
+  }
+
+#define _CLC_DEFINE_RELATIONAL_UNARY_VEC16(RET_TYPE, FUNCTION, ARG_TYPE)       \
+  _CLC_DEF _CLC_OVERLOAD RET_TYPE FUNCTION(ARG_TYPE x) {                       \
+    return (                                                                   \
+        RET_TYPE)((RET_TYPE){FUNCTION(x.s0), FUNCTION(x.s1), FUNCTION(x.s2),   \
+                             FUNCTION(x.s3), FUNCTION(x.s4), FUNCTION(x.s5),   \
+                             FUNCTION(x.s6), FUNCTION(x.s7), FUNCTION(x.s8),   \
+                             FUNCTION(x.s9), FUNCTION(x.sa), FUNCTION(x.sb),   \
+                             FUNCTION(x.sc), FUNCTION(x.sd), FUNCTION(x.se),   \
+                             FUNCTION(x.sf)} != (RET_TYPE)0);                  \
+  }
+
+#define _CLC_DEFINE_RELATIONAL_UNARY_VEC_ALL(RET_TYPE, FUNCTION, ARG_TYPE)     \
+  _CLC_DEFINE_RELATIONAL_UNARY_VEC2(RET_TYPE##2, FUNCTION, ARG_TYPE##2)        \
+  _CLC_DEFINE_RELATIONAL_UNARY_VEC3(RET_TYPE##3, FUNCTION, ARG_TYPE##3)        \
+  _CLC_DEFINE_RELATIONAL_UNARY_VEC4(RET_TYPE##4, FUNCTION, ARG_TYPE##4)        \
+  _CLC_DEFINE_RELATIONAL_UNARY_VEC8(RET_TYPE##8, FUNCTION, ARG_TYPE##8)        \
+  _CLC_DEFINE_RELATIONAL_UNARY_VEC16(RET_TYPE##16, FUNCTION, ARG_TYPE##16)
+
+_CLC_DEF _CLC_OVERLOAD int __clc_signbit(float x) {
+  return __builtin_signbitf(x);
+}
+
+_CLC_DEFINE_RELATIONAL_UNARY_VEC_ALL(int, __clc_signbit, float)
 
 #ifdef cl_khr_fp64
 
diff --git a/libclc/clc/lib/spirv/SOURCES b/libclc/clc/lib/spirv/SOURCES
deleted file mode 100644
index 813b1e3d69937..0000000000000
--- a/libclc/clc/lib/spirv/SOURCES
+++ /dev/null
@@ -1,13 +0,0 @@
-../generic/common/clc_degrees.cl
-../generic/common/clc_radians.cl
-../generic/common/clc_smoothstep.cl
-../generic/geometric/clc_dot.cl
-../generic/math/clc_ceil.cl
-../generic/math/clc_fabs.cl
-../generic/math/clc_floor.cl
-../generic/math/clc_mad.cl
-../generic/math/clc_nextafter.cl
-../generic/math/clc_rint.cl
-../generic/math/clc_trunc.cl
-../generic/relational/clc_select.cl
-../generic/shared/clc_clamp.cl
diff --git a/libclc/clc/lib/spirv64/SOURCES b/libclc/clc/lib/spirv64/SOURCES
deleted file mode 100644
index 813b1e3d69937..0000000000000
--- a/libclc/clc/lib/spirv64/SOURCES
+++ /dev/null
@@ -1,13 +0,0 @@
-../generic/common/clc_degrees.cl
-../generic/common/clc_radians.cl
-../generic/common/clc_smoothstep.cl
-../generic/geometric/clc_dot.cl
-../generic/math/clc_ceil.cl
-../generic/math/clc_fabs.cl
-../generic/math/clc_floor.cl
-../generic/math/clc_mad.cl
-../generic/math/clc_nextafter.cl
-../generic/math/clc_rint.cl
-../generic/math/clc_trunc.cl
-../generic/relational/clc_select.cl
-../generic/shared/clc_clamp.cl
diff --git a/libclc/clspv/lib/math/fma.cl b/libclc/clspv/lib/math/fma.cl
index e6251db4e92db..73c6e158601d9 100644
--- a/libclc/clspv/lib/math/fma.cl
+++ b/libclc/clspv/lib/math/fma.cl
@@ -24,9 +24,9 @@
 // (__clc_sw_fma), but avoids the use of ulong in favor of uint2. The logic has
 // been updated as appropriate.
 
-#include "../../../generic/lib/math/math.h"
 #include <clc/clc.h>
 #include <clc/clcmacro.h>
+#include <clc/math/math.h>
 
 struct fp {
   uint2 mantissa;
diff --git a/libclc/clspv/lib/subnormal_config.cl b/libclc/clspv/lib/subnormal_config.cl
index 167fe1b1a1bcf..3ab5a6394b933 100644
--- a/libclc/clspv/lib/subnormal_config.cl
+++ b/libclc/clspv/lib/subnormal_config.cl
@@ -21,8 +21,7 @@
  */
 
 #include <clc/clc.h>
-
-#include "config.h"
+#include <clc/math/clc_subnormal_config.h>
 
 _CLC_DEF bool __clc_fp16_subnormals_supported() { return false; }
 
diff --git a/libclc/clspv64 b/libclc/clspv64
deleted file mode 120000
index ea01ba94bc636..0000000000000
--- a/libclc/clspv64
+++ /dev/null
@@ -1 +0,0 @@
-clspv
\ No newline at end of file
diff --git a/libclc/generic/include/clc/float/definitions.h b/libclc/generic/include/clc/float/definitions.h
index 62501230c92de..be3d0130f3e61 100644
--- a/libclc/generic/include/clc/float/definitions.h
+++ b/libclc/generic/include/clc/float/definitions.h
@@ -31,9 +31,7 @@
 #define M_SQRT2_F       0x1.6a09e6p+0f
 #define M_SQRT1_2_F     0x1.6a09e6p-1f
 
-#ifdef __CLC_INTERNAL
-#define M_LOG210_F      0x1.a934f0p+1f
-#endif
+#define M_LOG210_F 0x1.a934f0p+1f
 
 #ifdef cl_khr_fp64
 
diff --git a/libclc/generic/include/clc/integer/clz.h b/libclc/generic/include/clc/integer/clz.h
index f7cdbf78ec060..5479279eb84a2 100644
--- a/libclc/generic/include/clc/integer/clz.h
+++ b/libclc/generic/include/clc/integer/clz.h
@@ -1,2 +1,7 @@
-#define __CLC_BODY <clc/integer/clz.inc>
+#define __CLC_FUNCTION clz
+#define __CLC_BODY <clc/shared/unary_decl.inc>
+
 #include <clc/integer/gentype.inc>
+
+#undef __CLC_BODY
+#undef __CLC_FUNCTION
diff --git a/libclc/generic/include/clc/integer/clz.inc b/libclc/generic/include/clc/integer/clz.inc
deleted file mode 100644
index 45826d10c9faf..0000000000000
--- a/libclc/generic/include/clc/integer/clz.inc
+++ /dev/null
@@ -1 +0,0 @@
-_CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE clz(__CLC_GENTYPE x);
diff --git a/libclc/generic/include/clc/integer/definitions.h b/libclc/generic/include/clc/integer/definitions.h
deleted file mode 100644
index 0079c30123db8..0000000000000
--- a/libclc/generic/include/clc/integer/definitions.h
+++ /dev/null
@@ -1,15 +0,0 @@
-#define CHAR_BIT 8
-#define INT_MAX 2147483647
-#define INT_MIN (-2147483647 - 1)
-#define LONG_MAX  0x7fffffffffffffffL
-#define LONG_MIN (-0x7fffffffffffffffL - 1)
-#define CHAR_MAX SCHAR_MAX
-#define CHAR_MIN SCHAR_MIN
-#define SCHAR_MAX 127
-#define SCHAR_MIN (-127 - 1)
-#define SHRT_MAX 32767
-#define SHRT_MIN (-32767 - 1)
-#define UCHAR_MAX 255
-#define USHRT_MAX 65535
-#define UINT_MAX 0xffffffff
-#define ULONG_MAX 0xffffffffffffffffUL
diff --git a/libclc/generic/include/clc/integer/hadd.h b/libclc/generic/include/clc/integer/hadd.h
index 37304e26cc2d6..0d73944ac11ba 100644
--- a/libclc/generic/include/clc/integer/hadd.h
+++ b/libclc/generic/include/clc/integer/hadd.h
@@ -1,2 +1,7 @@
-#define __CLC_BODY <clc/integer/hadd.inc>
+#define __CLC_FUNCTION hadd
+#define __CLC_BODY <clc/shared/binary_decl.inc>
+
 #include <clc/integer/gentype.inc>
+
+#undef __CLC_BODY
+#undef __CLC_FUNCTION
diff --git a/libclc/generic/include/clc/integer/hadd.inc b/libclc/generic/include/clc/integer/hadd.inc
deleted file mode 100644
index f698989cef202..0000000000000
--- a/libclc/generic/include/clc/integer/hadd.inc
+++ /dev/null
@@ -1 +0,0 @@
-_CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE hadd(__CLC_GENTYPE x, __CLC_GENTYPE y);
diff --git a/libclc/generic/include/clc/integer/mad24.h b/libclc/generic/include/clc/integer/mad24.h
index 0c120faac2b15..f7292f79f821f 100644
--- a/libclc/generic/include/clc/integer/mad24.h
+++ b/libclc/generic/include/clc/integer/mad24.h
@@ -1,3 +1,6 @@
-#define __CLC_BODY <clc/integer/mad24.inc>
-#include <clc/integer/integer-gentype.inc>
-#undef __CLC_BODY
+#define __CLC_FUNCTION mad24
+#define __CLC_BODY <clc/shared/ternary_decl.inc>
+
+#include <clc/integer/gentype24.inc>
+
+#undef __CLC_FUNCTION
diff --git a/libclc/generic/include/clc/integer/mad24.inc b/libclc/generic/include/clc/integer/mad24.inc
deleted file mode 100644
index 81fe0c2a89266..0000000000000
--- a/libclc/generic/include/clc/integer/mad24.inc
+++ /dev/null
@@ -1 +0,0 @@
-_CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE mad24(__CLC_GENTYPE x, __CLC_GENTYPE y, __CLC_GENTYPE z);
diff --git a/libclc/generic/include/clc/integer/mad_hi.h b/libclc/generic/include/clc/integer/mad_hi.h
index 863ce92d9f2d8..9f9de75df7613 100644
--- a/libclc/generic/include/clc/integer/mad_hi.h
+++ b/libclc/generic/include/clc/integer/mad_hi.h
@@ -1 +1,7 @@
-#define mad_hi(a, b, c) (mul_hi((a),(b))+(c))
+#define __CLC_FUNCTION mad_hi
+#define __CLC_BODY <clc/shared/ternary_decl.inc>
+
+#include <clc/integer/gentype.inc>
+
+#undef __CLC_FUNCTION
+#undef __CLC_BODY
diff --git a/libclc/generic/include/clc/integer/mul24.h b/libclc/generic/include/clc/integer/mul24.h
index 4f97098d70f0f..27e0f20040cb9 100644
--- a/libclc/generic/include/clc/integer/mul24.h
+++ b/libclc/generic/include/clc/integer/mul24.h
@@ -1,3 +1,7 @@
-#define __CLC_BODY <clc/integer/mul24.inc>
-#include <clc/integer/integer-gentype.inc>
+#define __CLC_FUNCTION mul24
+#define __CLC_BODY <clc/shared/binary_decl.inc>
+
+#include <clc/integer/gentype24.inc>
+
 #undef __CLC_BODY
+#undef __CLC_FUNCTION
diff --git a/libclc/generic/include/clc/integer/mul24.inc b/libclc/generic/include/clc/integer/mul24.inc
deleted file mode 100644
index 8cbf7c10ac447..0000000000000
--- a/libclc/generic/include/clc/integer/mul24.inc
+++ /dev/null
@@ -1 +0,0 @@
-_CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE mul24(__CLC_GENTYPE x, __CLC_GENTYPE y);
diff --git a/libclc/generic/include/clc/integer/mul_hi.h b/libclc/generic/include/clc/integer/mul_hi.h
index 27b95d83442f9..3607a80f43967 100644
--- a/libclc/generic/include/clc/integer/mul_hi.h
+++ b/libclc/generic/include/clc/integer/mul_hi.h
@@ -1,2 +1,7 @@
-#define __CLC_BODY <clc/integer/mul_hi.inc>
+#define __CLC_FUNCTION mul_hi
+#define __CLC_BODY <clc/shared/binary_decl.inc>
+
 #include <clc/integer/gentype.inc>
+
+#undef __CLC_BODY
+#undef __CLC_FUNCTION
diff --git a/libclc/generic/include/clc/integer/mul_hi.inc b/libclc/generic/include/clc/integer/mul_hi.inc
deleted file mode 100644
index ce9e5c0b2c18c..0000000000000
--- a/libclc/generic/include/clc/integer/mul_hi.inc
+++ /dev/null
@@ -1 +0,0 @@
-_CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE mul_hi(__CLC_GENTYPE x, __CLC_GENTYPE y);
diff --git a/libclc/generic/include/clc/integer/popcount.h b/libclc/generic/include/clc/integer/popcount.h
index 23335f45b6fe3..a3ce8ebb1df1f 100644
--- a/libclc/generic/include/clc/integer/popcount.h
+++ b/libclc/generic/include/clc/integer/popcount.h
@@ -1,5 +1,7 @@
 #define __CLC_FUNCTION popcount
-#define __CLC_BODY <clc/integer/unary.inc>
+#define __CLC_BODY <clc/shared/unary_decl.inc>
+
 #include <clc/integer/gentype.inc>
-#undef __CLC_FUNCTION
+
 #undef __CLC_BODY
+#undef __CLC_FUNCTION
diff --git a/libclc/generic/include/clc/integer/rhadd.h b/libclc/generic/include/clc/integer/rhadd.h
index 69b43faeebd24..a777d4b66f2f8 100644
--- a/libclc/generic/include/clc/integer/rhadd.h
+++ b/libclc/generic/include/clc/integer/rhadd.h
@@ -1,2 +1,7 @@
-#define __CLC_BODY <clc/integer/rhadd.inc>
+#define __CLC_FUNCTION rhadd
+#define __CLC_BODY <clc/shared/binary_decl.inc>
+
 #include <clc/integer/gentype.inc>
+
+#undef __CLC_BODY
+#undef __CLC_FUNCTION
diff --git a/libclc/generic/include/clc/integer/rhadd.inc b/libclc/generic/include/clc/integer/rhadd.inc
deleted file mode 100644
index 88ccaf09fd5ef..0000000000000
--- a/libclc/generic/include/clc/integer/rhadd.inc
+++ /dev/null
@@ -1 +0,0 @@
-_CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE rhadd(__CLC_GENTYPE x, __CLC_GENTYPE y);
diff --git a/libclc/generic/include/clc/integer/upsample.h b/libclc/generic/include/clc/integer/upsample.h
index 0b36b692a2c8d..37038f6ad9034 100644
--- a/libclc/generic/include/clc/integer/upsample.h
+++ b/libclc/generic/include/clc/integer/upsample.h
@@ -1,25 +1,24 @@
-#define __CLC_UPSAMPLE_DECL(BGENTYPE, GENTYPE, UGENTYPE) \
-    _CLC_OVERLOAD _CLC_DECL BGENTYPE upsample(GENTYPE hi, UGENTYPE lo);
+#define __CLC_UPSAMPLE_DECL(BGENTYPE, GENTYPE, UGENTYPE)                       \
+  _CLC_OVERLOAD _CLC_DECL BGENTYPE upsample(GENTYPE hi, UGENTYPE lo);
 
-#define __CLC_UPSAMPLE_VEC(BGENTYPE, GENTYPE, UGENTYPE) \
-    __CLC_UPSAMPLE_DECL(BGENTYPE, GENTYPE, UGENTYPE) \
-    __CLC_UPSAMPLE_DECL(BGENTYPE##2, GENTYPE##2, UGENTYPE##2) \
-    __CLC_UPSAMPLE_DECL(BGENTYPE##3, GENTYPE##3, UGENTYPE##3) \
-    __CLC_UPSAMPLE_DECL(BGENTYPE##4, GENTYPE##4, UGENTYPE##4) \
-    __CLC_UPSAMPLE_DECL(BGENTYPE##8, GENTYPE##8, UGENTYPE##8) \
-    __CLC_UPSAMPLE_DECL(BGENTYPE##16, GENTYPE##16, UGENTYPE##16) \
+#define __CLC_UPSAMPLE_VEC(BGENTYPE, GENTYPE, UGENTYPE)                        \
+  __CLC_UPSAMPLE_DECL(BGENTYPE, GENTYPE, UGENTYPE)                             \
+  __CLC_UPSAMPLE_DECL(BGENTYPE##2, GENTYPE##2, UGENTYPE##2)                    \
+  __CLC_UPSAMPLE_DECL(BGENTYPE##3, GENTYPE##3, UGENTYPE##3)                    \
+  __CLC_UPSAMPLE_DECL(BGENTYPE##4, GENTYPE##4, UGENTYPE##4)                    \
+  __CLC_UPSAMPLE_DECL(BGENTYPE##8, GENTYPE##8, UGENTYPE##8)                    \
+  __CLC_UPSAMPLE_DECL(BGENTYPE##16, GENTYPE##16, UGENTYPE##16)
 
-#define __CLC_UPSAMPLE_TYPES() \
-    __CLC_UPSAMPLE_VEC(short, char, uchar) \
-    __CLC_UPSAMPLE_VEC(ushort, uchar, uchar) \
-    __CLC_UPSAMPLE_VEC(int, short, ushort) \
-    __CLC_UPSAMPLE_VEC(uint, ushort, ushort) \
-    __CLC_UPSAMPLE_VEC(long, int, uint) \
-    __CLC_UPSAMPLE_VEC(ulong, uint, uint) \
+#define __CLC_UPSAMPLE_TYPES()                                                 \
+  __CLC_UPSAMPLE_VEC(short, char, uchar)                                       \
+  __CLC_UPSAMPLE_VEC(ushort, uchar, uchar)                                     \
+  __CLC_UPSAMPLE_VEC(int, short, ushort)                                       \
+  __CLC_UPSAMPLE_VEC(uint, ushort, ushort)                                     \
+  __CLC_UPSAMPLE_VEC(long, int, uint)                                          \
+  __CLC_UPSAMPLE_VEC(ulong, uint, uint)
 
 __CLC_UPSAMPLE_TYPES()
 
 #undef __CLC_UPSAMPLE_TYPES
 #undef __CLC_UPSAMPLE_DECL
 #undef __CLC_UPSAMPLE_VEC
-
diff --git a/libclc/generic/include/clc/math/fma.h b/libclc/generic/include/clc/math/fma.h
index c7c23eb2055f6..bedfc5414da00 100644
--- a/libclc/generic/include/clc/math/fma.h
+++ b/libclc/generic/include/clc/math/fma.h
@@ -1,4 +1,4 @@
-#define __CLC_BODY <clc/math/ternary_decl.inc>
+#define __CLC_BODY <clc/shared/ternary_decl.inc>
 #define __CLC_FUNCTION fma
 
 #include <clc/math/gentype.inc>
diff --git a/libclc/generic/include/clc/math/mad.h b/libclc/generic/include/clc/math/mad.h
index 99b6484193012..6f3f1f6201807 100644
--- a/libclc/generic/include/clc/math/mad.h
+++ b/libclc/generic/include/clc/math/mad.h
@@ -1,4 +1,4 @@
-#define __CLC_BODY <clc/math/ternary_decl.inc>
+#define __CLC_BODY <clc/shared/ternary_decl.inc>
 #define __CLC_FUNCTION mad
 
 #include <clc/math/gentype.inc>
diff --git a/libclc/generic/include/config.h b/libclc/generic/include/config.h
deleted file mode 100644
index 7aa5967f4eb68..0000000000000
--- a/libclc/generic/include/config.h
+++ /dev/null
@@ -1,28 +0,0 @@
-/*
- * Copyright (c) 2015 Advanced Micro Devices, Inc.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a copy
- * of this software and associated documentation files (the "Software"), to deal
- * in the Software without restriction, including without limitation the rights
- * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
- * copies of the Software, and to permit persons to whom the Software is
- * furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
- * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
- * THE SOFTWARE.
- */
-
-#include <clc/clcfunc.h>
-
-_CLC_DECL bool __clc_subnormals_disabled();
-_CLC_DECL bool __clc_fp16_subnormals_supported();
-_CLC_DECL bool __clc_fp32_subnormals_supported();
-_CLC_DECL bool __clc_fp64_subnormals_supported();
diff --git a/libclc/generic/include/integer/popcount.h b/libclc/generic/include/integer/popcount.h
deleted file mode 100644
index 00c753753bb4e..0000000000000
--- a/libclc/generic/include/integer/popcount.h
+++ /dev/null
@@ -1,3 +0,0 @@
-#define __CLC_FUNCTION __clc_native_popcount
-#define __CLC_INTRINSIC "llvm.ctpop"
-#include <integer/unary_intrin.inc>
diff --git a/libclc/generic/include/integer/unary_intrin.inc b/libclc/generic/include/integer/unary_intrin.inc
deleted file mode 100644
index ee9862a4c5b3a..0000000000000
--- a/libclc/generic/include/integer/unary_intrin.inc
+++ /dev/null
@@ -1,20 +0,0 @@
-#define __CLC_INTRINSIC_DEF(SCALAR_TYPE, BIT_SIZE) \
-_CLC_OVERLOAD SCALAR_TYPE __CLC_FUNCTION(SCALAR_TYPE x) __asm(__CLC_INTRINSIC ".i" BIT_SIZE); \
-_CLC_OVERLOAD SCALAR_TYPE##2 __CLC_FUNCTION(SCALAR_TYPE##2 x) __asm(__CLC_INTRINSIC ".v2i" BIT_SIZE); \
-_CLC_OVERLOAD SCALAR_TYPE##3 __CLC_FUNCTION(SCALAR_TYPE##3 x) __asm(__CLC_INTRINSIC ".v3i" BIT_SIZE); \
-_CLC_OVERLOAD SCALAR_TYPE##4 __CLC_FUNCTION(SCALAR_TYPE##4 x) __asm(__CLC_INTRINSIC ".v4i" BIT_SIZE); \
-_CLC_OVERLOAD SCALAR_TYPE##8 __CLC_FUNCTION(SCALAR_TYPE##8 x) __asm(__CLC_INTRINSIC ".v8i" BIT_SIZE); \
-_CLC_OVERLOAD SCALAR_TYPE##16 __CLC_FUNCTION(SCALAR_TYPE##16 x) __asm(__CLC_INTRINSIC ".v16i" BIT_SIZE);
-
-__CLC_INTRINSIC_DEF(char, "8")
-__CLC_INTRINSIC_DEF(uchar, "8")
-__CLC_INTRINSIC_DEF(short, "16")
-__CLC_INTRINSIC_DEF(ushort, "16")
-__CLC_INTRINSIC_DEF(int, "32")
-__CLC_INTRINSIC_DEF(uint, "32")
-__CLC_INTRINSIC_DEF(long, "64")
-__CLC_INTRINSIC_DEF(ulong, "64")
-
-#undef __CLC_FUNCTION
-#undef __CLC_INTRINSIC
-#undef __CLC_INTRINSIC_DEF
diff --git a/libclc/generic/include/math/clc_fma.h b/libclc/generic/include/math/clc_fma.h
index 490cb9bcefc86..598df66cf72e9 100644
--- a/libclc/generic/include/math/clc_fma.h
+++ b/libclc/generic/include/math/clc_fma.h
@@ -4,7 +4,7 @@
 
 #define __FLOAT_ONLY
 #define __CLC_FUNCTION __clc_sw_fma
-#define __CLC_BODY <clc/math/ternary_decl.inc>
+#define __CLC_BODY <clc/shared/ternary_decl.inc>
 #include <clc/math/gentype.inc>
 #undef __CLC_BODY
 #undef __CLC_FUNCTION
diff --git a/libclc/generic/include/math/clc_ldexp.h b/libclc/generic/include/math/clc_ldexp.h
index 454b7ed3dcee5..eb83f16240185 100644
--- a/libclc/generic/include/math/clc_ldexp.h
+++ b/libclc/generic/include/math/clc_ldexp.h
@@ -1,3 +1,5 @@
+#include <clc/clcfunc.h>
+
 _CLC_DEF _CLC_OVERLOAD float __clc_ldexp(float, int);
 
 #ifdef cl_khr_fp64
diff --git a/libclc/generic/lib/SOURCES b/libclc/generic/lib/SOURCES
index 217e3bca48b7a..a62c87902a6a7 100644
--- a/libclc/generic/lib/SOURCES
+++ b/libclc/generic/lib/SOURCES
@@ -68,6 +68,7 @@ integer/add_sat.cl
 integer/clz.cl
 integer/hadd.cl
 integer/mad24.cl
+integer/mad_hi.cl
 integer/mad_sat.cl
 integer/mul24.cl
 integer/mul_hi.cl
diff --git a/libclc/generic/lib/common/sign.cl b/libclc/generic/lib/common/sign.cl
index ad8f7405e0cb3..2f04d93b3e47e 100644
--- a/libclc/generic/lib/common/sign.cl
+++ b/libclc/generic/lib/common/sign.cl
@@ -1,37 +1,8 @@
 #include <clc/clc.h>
 #include <clc/clcmacro.h>
+#include <clc/common/clc_sign.h>
 
-#define SIGN(TYPE, F) \
-_CLC_DEF _CLC_OVERLOAD TYPE sign(TYPE x) { \
-  if (isnan(x)) { \
-    return 0.0F;   \
-  }               \
-  if (x > 0.0F) { \
-    return 1.0F;  \
-  }               \
-  if (x < 0.0F) { \
-    return -1.0F; \
-  }               \
-  return x; /* -0.0 or +0.0 */  \
-}
+#define FUNCTION sign
+#define __CLC_BODY <clc/shared/unary_def.inc>
 
-SIGN(float, f)
-_CLC_UNARY_VECTORIZE(_CLC_OVERLOAD _CLC_DEF, float, sign, float)
-
-#ifdef cl_khr_fp64
-
-#pragma OPENCL EXTENSION cl_khr_fp64 : enable
-
-SIGN(double, )
-_CLC_UNARY_VECTORIZE(_CLC_OVERLOAD _CLC_DEF, double, sign, double)
-
-#endif
-
-#ifdef cl_khr_fp16
-
-#pragma OPENCL EXTENSION cl_khr_fp16 : enable
-
-SIGN(half,)
-_CLC_UNARY_VECTORIZE(_CLC_OVERLOAD _CLC_DEF, half, sign, half)
-
-#endif
+#include <clc/math/gentype.inc>
diff --git a/libclc/generic/lib/integer/add_sat.cl b/libclc/generic/lib/integer/add_sat.cl
index 11a4a331fbd02..895eeeba43826 100644
--- a/libclc/generic/lib/integer/add_sat.cl
+++ b/libclc/generic/lib/integer/add_sat.cl
@@ -1,73 +1,7 @@
 #include <clc/clc.h>
-#include <clc/clcmacro.h>
+#include <clc/integer/clc_add_sat.h>
 
-// From add_sat.ll
-_CLC_DECL char   __clc_add_sat_s8(char, char);
-_CLC_DECL uchar  __clc_add_sat_u8(uchar, uchar);
-_CLC_DECL short  __clc_add_sat_s16(short, short);
-_CLC_DECL ushort __clc_add_sat_u16(ushort, ushort);
-_CLC_DECL int    __clc_add_sat_s32(int, int);
-_CLC_DECL uint   __clc_add_sat_u32(uint, uint);
-_CLC_DECL long   __clc_add_sat_s64(long, long);
-_CLC_DECL ulong  __clc_add_sat_u64(ulong, ulong);
+#define FUNCTION add_sat
+#define __CLC_BODY <clc/shared/binary_def.inc>
 
-_CLC_OVERLOAD _CLC_DEF char add_sat(char x, char y) {
-  short r = x + y;
-  return convert_char_sat(r);
-}
-
-_CLC_OVERLOAD _CLC_DEF uchar add_sat(uchar x, uchar y) {
-  ushort r = x + y;
-  return convert_uchar_sat(r);
-}
-
-_CLC_OVERLOAD _CLC_DEF short add_sat(short x, short y) {
-  int r = x + y;
-  return convert_short_sat(r);
-}
-
-_CLC_OVERLOAD _CLC_DEF ushort add_sat(ushort x, ushort y) {
-  uint r = x + y;
-  return convert_ushort_sat(r);
-}
-
-_CLC_OVERLOAD _CLC_DEF int add_sat(int x, int y) {
-  int r;
-  if (__builtin_sadd_overflow(x, y, &r))
-    // The oveflow can only occur if both are pos or both are neg,
-    // thus we only need to check one operand
-    return x > 0 ? INT_MAX : INT_MIN;
-  return r;
-}
-
-_CLC_OVERLOAD _CLC_DEF uint add_sat(uint x, uint y) {
-  uint r;
-  if (__builtin_uadd_overflow(x, y, &r))
-	return UINT_MAX;
-  return r;
-}
-
-_CLC_OVERLOAD _CLC_DEF long add_sat(long x, long y) {
-  long r;
-  if (__builtin_saddl_overflow(x, y, &r))
-    // The oveflow can only occur if both are pos or both are neg,
-    // thus we only need to check one operand
-    return x > 0 ? LONG_MAX : LONG_MIN;
-  return r;
-}
-
-_CLC_OVERLOAD _CLC_DEF ulong add_sat(ulong x, ulong y) {
-  ulong r;
-  if (__builtin_uaddl_overflow(x, y, &r))
-	return ULONG_MAX;
-  return r;
-}
-
-_CLC_BINARY_VECTORIZE(_CLC_OVERLOAD _CLC_DEF, char, add_sat, char, char)
-_CLC_BINARY_VECTORIZE(_CLC_OVERLOAD _CLC_DEF, uchar, add_sat, uchar, uchar)
-_CLC_BINARY_VECTORIZE(_CLC_OVERLOAD _CLC_DEF, short, add_sat, short, short)
-_CLC_BINARY_VECTORIZE(_CLC_OVERLOAD _CLC_DEF, ushort, add_sat, ushort, ushort)
-_CLC_BINARY_VECTORIZE(_CLC_OVERLOAD _CLC_DEF, int, add_sat, int, int)
-_CLC_BINARY_VECTORIZE(_CLC_OVERLOAD _CLC_DEF, uint, add_sat, uint, uint)
-_CLC_BINARY_VECTORIZE(_CLC_OVERLOAD _CLC_DEF, long, add_sat, long, long)
-_CLC_BINARY_VECTORIZE(_CLC_OVERLOAD _CLC_DEF, ulong, add_sat, ulong, ulong)
+#include <clc/integer/gentype.inc>
diff --git a/libclc/generic/lib/integer/clz.cl b/libclc/generic/lib/integer/clz.cl
index 904d027d37613..c2d423174b36c 100644
--- a/libclc/generic/lib/integer/clz.cl
+++ b/libclc/generic/lib/integer/clz.cl
@@ -1,43 +1,7 @@
 #include <clc/clc.h>
-#include <clc/clcmacro.h>
+#include <clc/integer/clc_clz.h>
 
-_CLC_OVERLOAD _CLC_DEF char clz(char x) {
-  return clz((ushort)(uchar)x) - 8;
-}
+#define FUNCTION clz
+#define __CLC_BODY <clc/shared/unary_def.inc>
 
-_CLC_OVERLOAD _CLC_DEF uchar clz(uchar x) {
-  return clz((ushort)x) - 8;
-}
-
-_CLC_OVERLOAD _CLC_DEF short clz(short x) {
-  return x ? __builtin_clzs(x) : 16;
-}
-
-_CLC_OVERLOAD _CLC_DEF ushort clz(ushort x) {
-  return x ? __builtin_clzs(x) : 16;
-}
-
-_CLC_OVERLOAD _CLC_DEF int clz(int x) {
-  return x ? __builtin_clz(x) : 32;
-}
-
-_CLC_OVERLOAD _CLC_DEF uint clz(uint x) {
-  return x ? __builtin_clz(x) : 32;
-}
-
-_CLC_OVERLOAD _CLC_DEF long clz(long x) {
-  return x ? __builtin_clzl(x) : 64;
-}
-
-_CLC_OVERLOAD _CLC_DEF ulong clz(ulong x) {
-  return x ? __builtin_clzl(x) : 64;
-}
-
-_CLC_UNARY_VECTORIZE(_CLC_OVERLOAD _CLC_DEF, char, clz, char)
-_CLC_UNARY_VECTORIZE(_CLC_OVERLOAD _CLC_DEF, uchar, clz, uchar)
-_CLC_UNARY_VECTORIZE(_CLC_OVERLOAD _CLC_DEF, short, clz, short)
-_CLC_UNARY_VECTORIZE(_CLC_OVERLOAD _CLC_DEF, ushort, clz, ushort)
-_CLC_UNARY_VECTORIZE(_CLC_OVERLOAD _CLC_DEF, int, clz, int)
-_CLC_UNARY_VECTORIZE(_CLC_OVERLOAD _CLC_DEF, uint, clz, uint)
-_CLC_UNARY_VECTORIZE(_CLC_OVERLOAD _CLC_DEF, long, clz, long)
-_CLC_UNARY_VECTORIZE(_CLC_OVERLOAD _CLC_DEF, ulong, clz, ulong)
+#include <clc/integer/gentype.inc>
diff --git a/libclc/generic/lib/integer/hadd.cl b/libclc/generic/lib/integer/hadd.cl
index 749026e5a8ad8..d8461f1d5398b 100644
--- a/libclc/generic/lib/integer/hadd.cl
+++ b/libclc/generic/lib/integer/hadd.cl
@@ -1,4 +1,7 @@
 #include <clc/clc.h>
+#include <clc/integer/clc_hadd.h>
+
+#define FUNCTION hadd
+#define __CLC_BODY <clc/shared/binary_def.inc>
 
-#define __CLC_BODY <hadd.inc>
 #include <clc/integer/gentype.inc>
diff --git a/libclc/generic/lib/integer/hadd.inc b/libclc/generic/lib/integer/hadd.inc
deleted file mode 100644
index ea59d9bd7db5f..0000000000000
--- a/libclc/generic/lib/integer/hadd.inc
+++ /dev/null
@@ -1,6 +0,0 @@
-//hadd = (x+y)>>1
-//This can be simplified to x>>1 + y>>1 + (1 if both x and y have the 1s bit set)
-//This saves us having to do any checks for overflow in the addition sum
-_CLC_OVERLOAD _CLC_DEF __CLC_GENTYPE hadd(__CLC_GENTYPE x, __CLC_GENTYPE y) {
-    return (x>>(__CLC_GENTYPE)1)+(y>>(__CLC_GENTYPE)1)+(x&y&(__CLC_GENTYPE)1);
-}
diff --git a/libclc/generic/lib/integer/mad24.cl b/libclc/generic/lib/integer/mad24.cl
index e29e99f28b56f..24935b54f24cd 100644
--- a/libclc/generic/lib/integer/mad24.cl
+++ b/libclc/generic/lib/integer/mad24.cl
@@ -1,4 +1,7 @@
 #include <clc/clc.h>
+#include <clc/integer/clc_mad24.h>
 
-#define __CLC_BODY <mad24.inc>
-#include <clc/integer/integer-gentype.inc>
+#define FUNCTION mad24
+#define __CLC_BODY <clc/shared/ternary_def.inc>
+
+#include <clc/integer/gentype24.inc>
diff --git a/libclc/generic/lib/integer/mad24.inc b/libclc/generic/lib/integer/mad24.inc
deleted file mode 100644
index 902b0aafe4c87..0000000000000
--- a/libclc/generic/lib/integer/mad24.inc
+++ /dev/null
@@ -1,3 +0,0 @@
-_CLC_OVERLOAD _CLC_DEF __CLC_GENTYPE mad24(__CLC_GENTYPE x, __CLC_GENTYPE y, __CLC_GENTYPE z){
-  return mul24(x, y) + z;
-}
diff --git a/libclc/generic/lib/integer/mad_hi.cl b/libclc/generic/lib/integer/mad_hi.cl
new file mode 100644
index 0000000000000..7915faa966ec4
--- /dev/null
+++ b/libclc/generic/lib/integer/mad_hi.cl
@@ -0,0 +1,7 @@
+#include <clc/clc.h>
+#include <clc/integer/clc_mad_hi.h>
+
+#define FUNCTION mad_hi
+#define __CLC_BODY <clc/shared/ternary_def.inc>
+
+#include <clc/integer/gentype.inc>
diff --git a/libclc/generic/lib/integer/mad_sat.cl b/libclc/generic/lib/integer/mad_sat.cl
index 2372eaacd6da8..e9c44364deee0 100644
--- a/libclc/generic/lib/integer/mad_sat.cl
+++ b/libclc/generic/lib/integer/mad_sat.cl
@@ -1,72 +1,7 @@
 #include <clc/clc.h>
-#include <clc/clcmacro.h>
+#include <clc/integer/clc_mad_sat.h>
 
-_CLC_OVERLOAD _CLC_DEF char mad_sat(char x, char y, char z) {
-  return clamp((short)mad24((short)x, (short)y, (short)z), (short)CHAR_MIN, (short) CHAR_MAX);
-}
+#define FUNCTION mad_sat
+#define __CLC_BODY <clc/shared/ternary_def.inc>
 
-_CLC_OVERLOAD _CLC_DEF uchar mad_sat(uchar x, uchar y, uchar z) {
-  return clamp((ushort)mad24((ushort)x, (ushort)y, (ushort)z), (ushort)0, (ushort) UCHAR_MAX);
-}
-
-_CLC_OVERLOAD _CLC_DEF short mad_sat(short x, short y, short z) {
-  return clamp((int)mad24((int)x, (int)y, (int)z), (int)SHRT_MIN, (int) SHRT_MAX);
-}
-
-_CLC_OVERLOAD _CLC_DEF ushort mad_sat(ushort x, ushort y, ushort z) {
-  return clamp((uint)mad24((uint)x, (uint)y, (uint)z), (uint)0, (uint) USHRT_MAX);
-}
-
-_CLC_OVERLOAD _CLC_DEF int mad_sat(int x, int y, int z) {
-  int mhi = mul_hi(x, y);
-  uint mlo = x * y;
-  long m = upsample(mhi, mlo);
-  m += z;
-  if (m > INT_MAX)
-    return INT_MAX;
-  if (m < INT_MIN)
-    return INT_MIN;
-  return m;
-}
-
-_CLC_OVERLOAD _CLC_DEF uint mad_sat(uint x, uint y, uint z) {
-  if (mul_hi(x, y) != 0)
-    return UINT_MAX;
-  return add_sat(x * y, z);
-}
-
-_CLC_OVERLOAD _CLC_DEF long mad_sat(long x, long y, long z) {
-  long hi = mul_hi(x, y);
-  ulong ulo = x * y;
-  long  slo = x * y;
-  /* Big overflow of more than 2 bits, add can't fix this */
-  if (((x < 0) == (y < 0)) && hi != 0)
-    return LONG_MAX;
-  /* Low overflow in mul and z not neg enough to correct it */
-  if (hi == 0 && ulo >= LONG_MAX && (z > 0 || (ulo + z) > LONG_MAX))
-    return LONG_MAX;
-  /* Big overflow of more than 2 bits, add can't fix this */
-  if (((x < 0) != (y < 0)) && hi != -1)
-    return LONG_MIN;
-  /* Low overflow in mul and z not pos enough to correct it */
-  if (hi == -1 && ulo <= ((ulong)LONG_MAX + 1UL) && (z < 0 || z < (LONG_MAX - ulo)))
-    return LONG_MIN;
-  /* We have checked all conditions, any overflow in addition returns
-   * the correct value */
-  return ulo + z;
-}
-
-_CLC_OVERLOAD _CLC_DEF ulong mad_sat(ulong x, ulong y, ulong z) {
-  if (mul_hi(x, y) != 0)
-    return ULONG_MAX;
-  return add_sat(x * y, z);
-}
-
-_CLC_TERNARY_VECTORIZE(_CLC_OVERLOAD _CLC_DEF, char, mad_sat, char, char, char)
-_CLC_TERNARY_VECTORIZE(_CLC_OVERLOAD _CLC_DEF, uchar, mad_sat, uchar, uchar, uchar)
-_CLC_TERNARY_VECTORIZE(_CLC_OVERLOAD _CLC_DEF, short, mad_sat, short, short, short)
-_CLC_TERNARY_VECTORIZE(_CLC_OVERLOAD _CLC_DEF, ushort, mad_sat, ushort, ushort, ushort)
-_CLC_TERNARY_VECTORIZE(_CLC_OVERLOAD _CLC_DEF, int, mad_sat, int, int, int)
-_CLC_TERNARY_VECTORIZE(_CLC_OVERLOAD _CLC_DEF, uint, mad_sat, uint, uint, uint)
-_CLC_TERNARY_VECTORIZE(_CLC_OVERLOAD _CLC_DEF, long, mad_sat, long, long, long)
-_CLC_TERNARY_VECTORIZE(_CLC_OVERLOAD _CLC_DEF, ulong, mad_sat, ulong, ulong, ulong)
+#include <clc/integer/gentype.inc>
diff --git a/libclc/generic/lib/integer/mul24.cl b/libclc/generic/lib/integer/mul24.cl
index 8aedca64b8590..b8cec2c62c337 100644
--- a/libclc/generic/lib/integer/mul24.cl
+++ b/libclc/generic/lib/integer/mul24.cl
@@ -1,4 +1,7 @@
 #include <clc/clc.h>
+#include <clc/integer/clc_mul24.h>
 
-#define __CLC_BODY <mul24.inc>
-#include <clc/integer/integer-gentype.inc>
+#define FUNCTION mul24
+#define __CLC_BODY <clc/shared/binary_def.inc>
+
+#include <clc/integer/gentype24.inc>
diff --git a/libclc/generic/lib/integer/mul24.inc b/libclc/generic/lib/integer/mul24.inc
deleted file mode 100644
index 95a2f1d6f31ba..0000000000000
--- a/libclc/generic/lib/integer/mul24.inc
+++ /dev/null
@@ -1,11 +0,0 @@
-
-// We need to use shifts here in order to mantain the sign bit for signed
-// integers.  The compiler should optimize this to (x & 0x00FFFFFF) for
-// unsigned integers.
-#define CONVERT_TO_24BIT(x) (((x) << 8) >> 8)
-
-_CLC_OVERLOAD _CLC_DEF __CLC_GENTYPE mul24(__CLC_GENTYPE x, __CLC_GENTYPE y){
-  return CONVERT_TO_24BIT(x) * CONVERT_TO_24BIT(y);
-}
-
-#undef CONVERT_TO_24BIT
diff --git a/libclc/generic/lib/integer/mul_hi.cl b/libclc/generic/lib/integer/mul_hi.cl
index 174d893afb14f..a19ef7ecda2b9 100644
--- a/libclc/generic/lib/integer/mul_hi.cl
+++ b/libclc/generic/lib/integer/mul_hi.cl
@@ -1,109 +1,7 @@
 #include <clc/clc.h>
+#include <clc/integer/clc_mul_hi.h>
 
-//For all types EXCEPT long, which is implemented separately
-#define __CLC_MUL_HI_IMPL(BGENTYPE, GENTYPE, GENSIZE) \
-    _CLC_OVERLOAD _CLC_DEF GENTYPE mul_hi(GENTYPE x, GENTYPE y){ \
-        return (GENTYPE)(((BGENTYPE)x * (BGENTYPE)y) >> GENSIZE); \
-    } \
+#define FUNCTION mul_hi
+#define __CLC_BODY <clc/shared/binary_def.inc>
 
-//FOIL-based long mul_hi
-//
-// Summary: Treat mul_hi(long x, long y) as:
-// (a+b) * (c+d) where a and c are the high-order parts of x and y respectively
-// and b and d are the low-order parts of x and y.
-// Thinking back to algebra, we use FOIL to do the work.
-
-_CLC_OVERLOAD _CLC_DEF long mul_hi(long x, long y){
-    long f, o, i;
-    ulong l;
-
-    //Move the high/low halves of x/y into the lower 32-bits of variables so
-    //that we can multiply them without worrying about overflow.
-    long x_hi = x >> 32;
-    long x_lo = x & UINT_MAX;
-    long y_hi = y >> 32;
-    long y_lo = y & UINT_MAX;
-
-    //Multiply all of the components according to FOIL method
-    f = x_hi * y_hi;
-    o = x_hi * y_lo;
-    i = x_lo * y_hi;
-    l = x_lo * y_lo;
-
-    //Now add the components back together in the following steps:
-    //F: doesn't need to be modified
-    //O/I: Need to be added together.
-    //L: Shift right by 32-bits, then add into the sum of O and I
-    //Once O/I/L are summed up, then shift the sum by 32-bits and add to F.
-    //
-    //We use hadd to give us a bit of extra precision for the intermediate sums
-    //but as a result, we shift by 31 bits instead of 32
-    return (long)(f + (hadd(o, (i + (long)((ulong)l>>32))) >> 31));
-}
-
-_CLC_OVERLOAD _CLC_DEF ulong mul_hi(ulong x, ulong y){
-    ulong f, o, i;
-    ulong l;
-
-    //Move the high/low halves of x/y into the lower 32-bits of variables so
-    //that we can multiply them without worrying about overflow.
-    ulong x_hi = x >> 32;
-    ulong x_lo = x & UINT_MAX;
-    ulong y_hi = y >> 32;
-    ulong y_lo = y & UINT_MAX;
-
-    //Multiply all of the components according to FOIL method
-    f = x_hi * y_hi;
-    o = x_hi * y_lo;
-    i = x_lo * y_hi;
-    l = x_lo * y_lo;
-
-    //Now add the components back together, taking care to respect the fact that:
-    //F: doesn't need to be modified
-    //O/I: Need to be added together.
-    //L: Shift right by 32-bits, then add into the sum of O and I
-    //Once O/I/L are summed up, then shift the sum by 32-bits and add to F.
-    //
-    //We use hadd to give us a bit of extra precision for the intermediate sums
-    //but as a result, we shift by 31 bits instead of 32
-    return (f + (hadd(o, (i + (l>>32))) >> 31));
-}
-
-#define __CLC_MUL_HI_VEC(GENTYPE) \
-    _CLC_OVERLOAD _CLC_DEF GENTYPE##2 mul_hi(GENTYPE##2 x, GENTYPE##2 y){ \
-        return (GENTYPE##2){mul_hi(x.s0, y.s0), mul_hi(x.s1, y.s1)}; \
-    } \
-    _CLC_OVERLOAD _CLC_DEF GENTYPE##3 mul_hi(GENTYPE##3 x, GENTYPE##3 y){ \
-        return (GENTYPE##3){mul_hi(x.s0, y.s0), mul_hi(x.s1, y.s1), mul_hi(x.s2, y.s2)}; \
-    } \
-    _CLC_OVERLOAD _CLC_DEF GENTYPE##4 mul_hi(GENTYPE##4 x, GENTYPE##4 y){ \
-        return (GENTYPE##4){mul_hi(x.lo, y.lo), mul_hi(x.hi, y.hi)}; \
-    } \
-    _CLC_OVERLOAD _CLC_DEF GENTYPE##8 mul_hi(GENTYPE##8 x, GENTYPE##8 y){ \
-        return (GENTYPE##8){mul_hi(x.lo, y.lo), mul_hi(x.hi, y.hi)}; \
-    } \
-    _CLC_OVERLOAD _CLC_DEF GENTYPE##16 mul_hi(GENTYPE##16 x, GENTYPE##16 y){ \
-        return (GENTYPE##16){mul_hi(x.lo, y.lo), mul_hi(x.hi, y.hi)}; \
-    } \
-
-#define __CLC_MUL_HI_DEC_IMPL(BTYPE, TYPE, BITS) \
-    __CLC_MUL_HI_IMPL(BTYPE, TYPE, BITS) \
-    __CLC_MUL_HI_VEC(TYPE)
-
-#define __CLC_MUL_HI_TYPES() \
-    __CLC_MUL_HI_DEC_IMPL(short, char, 8) \
-    __CLC_MUL_HI_DEC_IMPL(ushort, uchar, 8) \
-    __CLC_MUL_HI_DEC_IMPL(int, short, 16) \
-    __CLC_MUL_HI_DEC_IMPL(uint, ushort, 16) \
-    __CLC_MUL_HI_DEC_IMPL(long, int, 32) \
-    __CLC_MUL_HI_DEC_IMPL(ulong, uint, 32) \
-    __CLC_MUL_HI_VEC(long) \
-    __CLC_MUL_HI_VEC(ulong)
-
-__CLC_MUL_HI_TYPES()
-
-#undef __CLC_MUL_HI_TYPES
-#undef __CLC_MUL_HI_DEC_IMPL
-#undef __CLC_MUL_HI_IMPL
-#undef __CLC_MUL_HI_VEC
-#undef __CLC_B32
+#include <clc/integer/gentype.inc>
diff --git a/libclc/generic/lib/integer/popcount.cl b/libclc/generic/lib/integer/popcount.cl
index ca83b1afaf9da..4e8a6ebab6596 100644
--- a/libclc/generic/lib/integer/popcount.cl
+++ b/libclc/generic/lib/integer/popcount.cl
@@ -1,8 +1,7 @@
 #include <clc/clc.h>
-#include <integer/popcount.h>
+#include <clc/integer/clc_popcount.h>
 
-#define __CLC_FUNC popcount
-#define __CLC_IMPL_FUNC __clc_native_popcount
+#define FUNCTION popcount
+#define __CLC_BODY <clc/shared/unary_def.inc>
 
-#define __CLC_BODY "../clc_unary.inc"
 #include <clc/integer/gentype.inc>
diff --git a/libclc/generic/lib/integer/rhadd.cl b/libclc/generic/lib/integer/rhadd.cl
index c985870f7c7a2..b1cb492cad8b8 100644
--- a/libclc/generic/lib/integer/rhadd.cl
+++ b/libclc/generic/lib/integer/rhadd.cl
@@ -1,4 +1,7 @@
 #include <clc/clc.h>
+#include <clc/integer/clc_rhadd.h>
+
+#define FUNCTION rhadd
+#define __CLC_BODY <clc/shared/binary_def.inc>
 
-#define __CLC_BODY <rhadd.inc>
 #include <clc/integer/gentype.inc>
diff --git a/libclc/generic/lib/integer/rhadd.inc b/libclc/generic/lib/integer/rhadd.inc
deleted file mode 100644
index 3d6076874808e..0000000000000
--- a/libclc/generic/lib/integer/rhadd.inc
+++ /dev/null
@@ -1,6 +0,0 @@
-//rhadd = (x+y+1)>>1
-//This can be simplified to x>>1 + y>>1 + (1 if either x or y have the 1s bit set)
-//This saves us having to do any checks for overflow in the addition sums
-_CLC_OVERLOAD _CLC_DEF __CLC_GENTYPE rhadd(__CLC_GENTYPE x, __CLC_GENTYPE y) {
-    return (x>>(__CLC_GENTYPE)1)+(y>>(__CLC_GENTYPE)1)+((x&(__CLC_GENTYPE)1)|(y&(__CLC_GENTYPE)1));
-}
diff --git a/libclc/generic/lib/integer/rotate.cl b/libclc/generic/lib/integer/rotate.cl
index 27ce515c72933..1e72af30f33f2 100644
--- a/libclc/generic/lib/integer/rotate.cl
+++ b/libclc/generic/lib/integer/rotate.cl
@@ -1,4 +1,7 @@
 #include <clc/clc.h>
+#include <clc/integer/clc_rotate.h>
+
+#define FUNCTION rotate
+#define __CLC_BODY <clc/shared/binary_def.inc>
 
-#define __CLC_BODY <rotate.inc>
 #include <clc/integer/gentype.inc>
diff --git a/libclc/generic/lib/integer/rotate.inc b/libclc/generic/lib/integer/rotate.inc
deleted file mode 100644
index 33bb0a85241d2..0000000000000
--- a/libclc/generic/lib/integer/rotate.inc
+++ /dev/null
@@ -1,42 +0,0 @@
-/**
- * Not necessarily optimal... but it produces correct results (at least for int)
- * If we're lucky, LLVM will recognize the pattern and produce rotate
- * instructions:
- * http://llvm.1065342.n5.nabble.com/rotate-td47679.html
- * 
- * Eventually, someone should feel free to implement an llvm-specific version
- */
-
-_CLC_OVERLOAD _CLC_DEF __CLC_GENTYPE rotate(__CLC_GENTYPE x, __CLC_GENTYPE n){
-    //Try to avoid extra work if someone's spinning the value through multiple
-    //full rotations
-    n = n % (__CLC_GENTYPE)__CLC_GENSIZE;
-
-#ifdef __CLC_SCALAR
-    if (n > 0){
-        return (x << n) | (((__CLC_U_GENTYPE)x) >> (__CLC_GENSIZE - n));
-    } else if (n == 0){
-        return x;
-    } else {
-        return ( (((__CLC_U_GENTYPE)x) >> -n) | (x << (__CLC_GENSIZE + n)) );
-    }
-#else
-    //XXX: There's a lot of __builtin_astype calls to cast everything to
-    //     unsigned ... This should be improved so that if __CLC_GENTYPE==__CLC_U_GENTYPE, no
-    //     casts are required.
-    
-    __CLC_U_GENTYPE x_1 = __builtin_astype(x, __CLC_U_GENTYPE);
-
-    //XXX: Is (__CLC_U_GENTYPE >> S__CLC_GENTYPE) | (__CLC_U_GENTYPE << S__CLC_GENTYPE) legal?
-    //     If so, then combine the amt and shifts into a single set of statements
-    
-    __CLC_U_GENTYPE amt;
-    amt = (n < (__CLC_GENTYPE)0 ? __builtin_astype((__CLC_GENTYPE)0-n, __CLC_U_GENTYPE) : (__CLC_U_GENTYPE)0);
-    x_1 = (x_1 >> amt) | (x_1 << ((__CLC_U_GENTYPE)__CLC_GENSIZE - amt));
-
-    amt = (n < (__CLC_GENTYPE)0 ? (__CLC_U_GENTYPE)0 : __builtin_astype(n, __CLC_U_GENTYPE));
-    x_1 = (x_1 << amt) | (x_1 >> ((__CLC_U_GENTYPE)__CLC_GENSIZE - amt));
-
-    return __builtin_astype(x_1, __CLC_GENTYPE);
-#endif
-}
diff --git a/libclc/generic/lib/integer/sub_sat.cl b/libclc/generic/lib/integer/sub_sat.cl
index e6beef787c036..e153738169b94 100644
--- a/libclc/generic/lib/integer/sub_sat.cl
+++ b/libclc/generic/lib/integer/sub_sat.cl
@@ -1,61 +1,7 @@
 #include <clc/clc.h>
-#include <clc/clcmacro.h>
+#include <clc/integer/clc_sub_sat.h>
 
-_CLC_OVERLOAD _CLC_DEF char sub_sat(char x, char y) {
-  short r = x - y;
-  return convert_char_sat(r);
-}
+#define FUNCTION sub_sat
+#define __CLC_BODY <clc/shared/binary_def.inc>
 
-_CLC_OVERLOAD _CLC_DEF uchar sub_sat(uchar x, uchar y) {
-  short r = x - y;
-  return convert_uchar_sat(r);
-}
-
-_CLC_OVERLOAD _CLC_DEF short sub_sat(short x, short y) {
-  int r = x - y;
-  return convert_short_sat(r);
-}
-
-_CLC_OVERLOAD _CLC_DEF ushort sub_sat(ushort x, ushort y) {
-  int r = x - y;
-  return convert_ushort_sat(r);
-}
-
-_CLC_OVERLOAD _CLC_DEF int sub_sat(int x, int y) {
-  int r;
-  if (__builtin_ssub_overflow(x, y, &r))
-    // The oveflow can only occur in the direction of the first operand
-    return x > 0 ? INT_MAX : INT_MIN;
-  return r;
-}
-
-_CLC_OVERLOAD _CLC_DEF uint sub_sat(uint x, uint y) {
-  uint r;
-  if (__builtin_usub_overflow(x, y, &r))
-	return 0;
-  return r;
-}
-
-_CLC_OVERLOAD _CLC_DEF long sub_sat(long x, long y) {
-  long r;
-  if (__builtin_ssubl_overflow(x, y, &r))
-    // The oveflow can only occur in the direction of the first operand
-    return x > 0 ? LONG_MAX : LONG_MIN;
-  return r;
-}
-
-_CLC_OVERLOAD _CLC_DEF ulong sub_sat(ulong x, ulong y) {
-  ulong r;
-  if (__builtin_usubl_overflow(x, y, &r))
-	return 0;
-  return r;
-}
-
-_CLC_BINARY_VECTORIZE(_CLC_OVERLOAD _CLC_DEF, char, sub_sat, char, char)
-_CLC_BINARY_VECTORIZE(_CLC_OVERLOAD _CLC_DEF, uchar, sub_sat, uchar, uchar)
-_CLC_BINARY_VECTORIZE(_CLC_OVERLOAD _CLC_DEF, short, sub_sat, short, short)
-_CLC_BINARY_VECTORIZE(_CLC_OVERLOAD _CLC_DEF, ushort, sub_sat, ushort, ushort)
-_CLC_BINARY_VECTORIZE(_CLC_OVERLOAD _CLC_DEF, int, sub_sat, int, int)
-_CLC_BINARY_VECTORIZE(_CLC_OVERLOAD _CLC_DEF, uint, sub_sat, uint, uint)
-_CLC_BINARY_VECTORIZE(_CLC_OVERLOAD _CLC_DEF, long, sub_sat, long, long)
-_CLC_BINARY_VECTORIZE(_CLC_OVERLOAD _CLC_DEF, ulong, sub_sat, ulong, ulong)
+#include <clc/integer/gentype.inc>
diff --git a/libclc/generic/lib/integer/upsample.cl b/libclc/generic/lib/integer/upsample.cl
index da77315f8f934..984a731e3b4d1 100644
--- a/libclc/generic/lib/integer/upsample.cl
+++ b/libclc/generic/lib/integer/upsample.cl
@@ -1,32 +1,34 @@
 #include <clc/clc.h>
+#include <clc/integer/clc_upsample.h>
 
-#define __CLC_UPSAMPLE_IMPL(BGENTYPE, GENTYPE, UGENTYPE, GENSIZE) \
-    _CLC_OVERLOAD _CLC_DEF BGENTYPE upsample(GENTYPE hi, UGENTYPE lo){ \
-        return ((BGENTYPE)hi << GENSIZE) | lo; \
-    } \
-    _CLC_OVERLOAD _CLC_DEF BGENTYPE##2 upsample(GENTYPE##2 hi, UGENTYPE##2 lo){ \
-        return (BGENTYPE##2){upsample(hi.s0, lo.s0), upsample(hi.s1, lo.s1)}; \
-    } \
-    _CLC_OVERLOAD _CLC_DEF BGENTYPE##3 upsample(GENTYPE##3 hi, UGENTYPE##3 lo){ \
-        return (BGENTYPE##3){upsample(hi.s0, lo.s0), upsample(hi.s1, lo.s1), upsample(hi.s2, lo.s2)}; \
-    } \
-    _CLC_OVERLOAD _CLC_DEF BGENTYPE##4 upsample(GENTYPE##4 hi, UGENTYPE##4 lo){ \
-        return (BGENTYPE##4){upsample(hi.lo, lo.lo), upsample(hi.hi, lo.hi)}; \
-    } \
-    _CLC_OVERLOAD _CLC_DEF BGENTYPE##8 upsample(GENTYPE##8 hi, UGENTYPE##8 lo){ \
-        return (BGENTYPE##8){upsample(hi.lo, lo.lo), upsample(hi.hi, lo.hi)}; \
-    } \
-    _CLC_OVERLOAD _CLC_DEF BGENTYPE##16 upsample(GENTYPE##16 hi, UGENTYPE##16 lo){ \
-        return (BGENTYPE##16){upsample(hi.lo, lo.lo), upsample(hi.hi, lo.hi)}; \
-    } \
+#define __CLC_UPSAMPLE_IMPL(BGENTYPE, GENTYPE, UGENTYPE)                       \
+  _CLC_OVERLOAD _CLC_DEF BGENTYPE upsample(GENTYPE hi, UGENTYPE lo) {          \
+    return __clc_upsample(hi, lo);                                             \
+  }                                                                            \
+  _CLC_OVERLOAD _CLC_DEF BGENTYPE##2 upsample(GENTYPE##2 hi, UGENTYPE##2 lo) { \
+    return __clc_upsample(hi, lo);                                             \
+  }                                                                            \
+  _CLC_OVERLOAD _CLC_DEF BGENTYPE##3 upsample(GENTYPE##3 hi, UGENTYPE##3 lo) { \
+    return __clc_upsample(hi, lo);                                             \
+  }                                                                            \
+  _CLC_OVERLOAD _CLC_DEF BGENTYPE##4 upsample(GENTYPE##4 hi, UGENTYPE##4 lo) { \
+    return __clc_upsample(hi, lo);                                             \
+  }                                                                            \
+  _CLC_OVERLOAD _CLC_DEF BGENTYPE##8 upsample(GENTYPE##8 hi, UGENTYPE##8 lo) { \
+    return __clc_upsample(hi, lo);                                             \
+  }                                                                            \
+  _CLC_OVERLOAD _CLC_DEF BGENTYPE##16 upsample(GENTYPE##16 hi,                 \
+                                               UGENTYPE##16 lo) {              \
+    return __clc_upsample(hi, lo);                                             \
+  }
 
-#define __CLC_UPSAMPLE_TYPES() \
-    __CLC_UPSAMPLE_IMPL(short, char, uchar, 8) \
-    __CLC_UPSAMPLE_IMPL(ushort, uchar, uchar, 8) \
-    __CLC_UPSAMPLE_IMPL(int, short, ushort, 16) \
-    __CLC_UPSAMPLE_IMPL(uint, ushort, ushort, 16) \
-    __CLC_UPSAMPLE_IMPL(long, int, uint, 32) \
-    __CLC_UPSAMPLE_IMPL(ulong, uint, uint, 32) \
+#define __CLC_UPSAMPLE_TYPES()                                                 \
+  __CLC_UPSAMPLE_IMPL(short, char, uchar)                                      \
+  __CLC_UPSAMPLE_IMPL(ushort, uchar, uchar)                                    \
+  __CLC_UPSAMPLE_IMPL(int, short, ushort)                                      \
+  __CLC_UPSAMPLE_IMPL(uint, ushort, ushort)                                    \
+  __CLC_UPSAMPLE_IMPL(long, int, uint)                                         \
+  __CLC_UPSAMPLE_IMPL(ulong, uint, uint)
 
 __CLC_UPSAMPLE_TYPES()
 
diff --git a/libclc/generic/lib/math/acos.cl b/libclc/generic/lib/math/acos.cl
index d71d10024b180..273ddbd4b1ee5 100644
--- a/libclc/generic/lib/math/acos.cl
+++ b/libclc/generic/lib/math/acos.cl
@@ -19,10 +19,10 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
+
 #include <clc/clc.h>
 #include <clc/clcmacro.h>
-
-#include "math.h"
+#include <clc/math/math.h>
 
 _CLC_OVERLOAD _CLC_DEF float acos(float x) {
     // Computes arccos(x).
diff --git a/libclc/generic/lib/math/acosh.cl b/libclc/generic/lib/math/acosh.cl
index 977c2e929b34c..41671e94ea78c 100644
--- a/libclc/generic/lib/math/acosh.cl
+++ b/libclc/generic/lib/math/acosh.cl
@@ -20,11 +20,10 @@
  * THE SOFTWARE.
  */
 
+#include "ep_log.h"
 #include <clc/clc.h>
 #include <clc/clcmacro.h>
-
-#include "ep_log.h"
-#include "math.h"
+#include <clc/math/math.h>
 
 _CLC_OVERLOAD _CLC_DEF  float acosh(float x) {
     uint ux = as_uint(x);
diff --git a/libclc/generic/lib/math/acospi.cl b/libclc/generic/lib/math/acospi.cl
index 5aa8a083df4e9..d640ff24e5386 100644
--- a/libclc/generic/lib/math/acospi.cl
+++ b/libclc/generic/lib/math/acospi.cl
@@ -22,8 +22,7 @@
 
 #include <clc/clc.h>
 #include <clc/clcmacro.h>
-
-#include "math.h"
+#include <clc/math/math.h>
 
 _CLC_OVERLOAD _CLC_DEF float acospi(float x) {
     // Computes arccos(x).
diff --git a/libclc/generic/lib/math/asin.cl b/libclc/generic/lib/math/asin.cl
index 443dec830eb2f..ae60b88e9037c 100644
--- a/libclc/generic/lib/math/asin.cl
+++ b/libclc/generic/lib/math/asin.cl
@@ -22,8 +22,7 @@
 
 #include <clc/clc.h>
 #include <clc/clcmacro.h>
-
-#include "math.h"
+#include <clc/math/math.h>
 
 _CLC_OVERLOAD _CLC_DEF float asin(float x) {
     // Computes arcsin(x).
diff --git a/libclc/generic/lib/math/asinh.cl b/libclc/generic/lib/math/asinh.cl
index 686d9f7a95e5d..1a60fae8bdf9b 100644
--- a/libclc/generic/lib/math/asinh.cl
+++ b/libclc/generic/lib/math/asinh.cl
@@ -20,11 +20,10 @@
  * THE SOFTWARE.
  */
 
+#include "ep_log.h"
 #include <clc/clc.h>
 #include <clc/clcmacro.h>
-
-#include "math.h"
-#include "ep_log.h"
+#include <clc/math/math.h>
 
 _CLC_OVERLOAD _CLC_DEF float asinh(float x) {
     uint ux = as_uint(x);
diff --git a/libclc/generic/lib/math/asinpi.cl b/libclc/generic/lib/math/asinpi.cl
index 18dc53000b034..64b15aff6792c 100644
--- a/libclc/generic/lib/math/asinpi.cl
+++ b/libclc/generic/lib/math/asinpi.cl
@@ -22,8 +22,7 @@
 
 #include <clc/clc.h>
 #include <clc/clcmacro.h>
-
-#include "math.h"
+#include <clc/math/math.h>
 
 _CLC_OVERLOAD _CLC_DEF float asinpi(float x) {
     // Computes arcsin(x).
diff --git a/libclc/generic/lib/math/atan.cl b/libclc/generic/lib/math/atan.cl
index b6b067f2bf4ff..95c247304f91a 100644
--- a/libclc/generic/lib/math/atan.cl
+++ b/libclc/generic/lib/math/atan.cl
@@ -22,8 +22,7 @@
 
 #include <clc/clc.h>
 #include <clc/clcmacro.h>
-
-#include "math.h"
+#include <clc/math/math.h>
 
 _CLC_OVERLOAD _CLC_DEF float atan(float x)
 {
diff --git a/libclc/generic/lib/math/atan2.cl b/libclc/generic/lib/math/atan2.cl
index 32419937a856b..067e43e445ed1 100644
--- a/libclc/generic/lib/math/atan2.cl
+++ b/libclc/generic/lib/math/atan2.cl
@@ -22,9 +22,8 @@
 
 #include <clc/clc.h>
 #include <clc/clcmacro.h>
-
-#include "math.h"
-#include "tables.h"
+#include <clc/math/math.h>
+#include <clc/math/tables.h>
 
 _CLC_OVERLOAD _CLC_DEF float atan2(float y, float x)
 {
diff --git a/libclc/generic/lib/math/atan2pi.cl b/libclc/generic/lib/math/atan2pi.cl
index e631918f7539e..d67cf90df63f8 100644
--- a/libclc/generic/lib/math/atan2pi.cl
+++ b/libclc/generic/lib/math/atan2pi.cl
@@ -22,9 +22,8 @@
 
 #include <clc/clc.h>
 #include <clc/clcmacro.h>
-
-#include "math.h"
-#include "tables.h"
+#include <clc/math/math.h>
+#include <clc/math/tables.h>
 
 _CLC_OVERLOAD _CLC_DEF  float atan2pi(float y, float x) {
     const float pi = 0x1.921fb6p+1f;
diff --git a/libclc/generic/lib/math/atanh.cl b/libclc/generic/lib/math/atanh.cl
index 10bad190cc0dc..81ae1a2ca801c 100644
--- a/libclc/generic/lib/math/atanh.cl
+++ b/libclc/generic/lib/math/atanh.cl
@@ -22,8 +22,7 @@
 
 #include <clc/clc.h>
 #include <clc/clcmacro.h>
-
-#include "math.h"
+#include <clc/math/math.h>
 
 _CLC_OVERLOAD _CLC_DEF float atanh(float x) {
     uint ux = as_uint(x);
diff --git a/libclc/generic/lib/math/atanpi.cl b/libclc/generic/lib/math/atanpi.cl
index 8522acf349933..8c7bac4bb4bb2 100644
--- a/libclc/generic/lib/math/atanpi.cl
+++ b/libclc/generic/lib/math/atanpi.cl
@@ -22,8 +22,7 @@
 
 #include <clc/clc.h>
 #include <clc/clcmacro.h>
-
-#include "math.h"
+#include <clc/math/math.h>
 
 _CLC_OVERLOAD _CLC_DEF float atanpi(float x) {
     const float pi = 3.1415926535897932f;
diff --git a/libclc/generic/lib/math/cbrt.cl b/libclc/generic/lib/math/cbrt.cl
index 49f2ba8a411cb..a2bf152f35f94 100644
--- a/libclc/generic/lib/math/cbrt.cl
+++ b/libclc/generic/lib/math/cbrt.cl
@@ -22,9 +22,8 @@
 
 #include <clc/clc.h>
 #include <clc/clcmacro.h>
-
-#include "math.h"
-#include "tables.h"
+#include <clc/math/math.h>
+#include <clc/math/tables.h>
 
 _CLC_OVERLOAD _CLC_DEF float cbrt(float x) {
 
diff --git a/libclc/generic/lib/math/clc_exp10.cl b/libclc/generic/lib/math/clc_exp10.cl
index 572aa396942b7..0eb53d013a85a 100644
--- a/libclc/generic/lib/math/clc_exp10.cl
+++ b/libclc/generic/lib/math/clc_exp10.cl
@@ -23,12 +23,11 @@
 #include <clc/clc.h>
 #include <clc/clcmacro.h>
 #include <clc/math/clc_mad.h>
+#include <clc/math/clc_subnormal_config.h>
+#include <clc/math/math.h>
+#include <clc/math/tables.h>
 #include <clc/relational/clc_isnan.h>
 
-#include "config.h"
-#include "math.h"
-#include "tables.h"
-
 //    Algorithm:
 //
 //    e^x = 2^(x/ln(2)) = 2^(x*(64/ln(2))/64)
diff --git a/libclc/generic/lib/math/clc_fma.cl b/libclc/generic/lib/math/clc_fma.cl
index 15de4c8032a93..33f5072425d98 100644
--- a/libclc/generic/lib/math/clc_fma.cl
+++ b/libclc/generic/lib/math/clc_fma.cl
@@ -23,13 +23,13 @@
 #include <clc/clc.h>
 #include <clc/clcmacro.h>
 #include <clc/integer/clc_abs.h>
+#include <clc/integer/clc_clz.h>
+#include <clc/math/clc_subnormal_config.h>
+#include <clc/math/math.h>
 #include <clc/relational/clc_isinf.h>
 #include <clc/relational/clc_isnan.h>
 #include <clc/shared/clc_max.h>
 
-#include "config.h"
-#include "math.h"
-
 struct fp {
   ulong mantissa;
   int exponent;
@@ -119,7 +119,7 @@ _CLC_DEF _CLC_OVERLOAD float __clc_sw_fma(float a, float b, float c) {
   }
 
   // detect overflow/underflow
-  int overflow_bits = 3 - clz(st_fma.mantissa);
+  int overflow_bits = 3 - __clc_clz(st_fma.mantissa);
 
   // adjust exponent
   st_fma.exponent += overflow_bits;
diff --git a/libclc/generic/lib/math/clc_fmod.cl b/libclc/generic/lib/math/clc_fmod.cl
index 5d101373178dd..a4a2ab791df68 100644
--- a/libclc/generic/lib/math/clc_fmod.cl
+++ b/libclc/generic/lib/math/clc_fmod.cl
@@ -22,166 +22,164 @@
 
 #include <clc/clc.h>
 #include <clc/clcmacro.h>
+#include <clc/integer/clc_clz.h>
 #include <clc/math/clc_floor.h>
+#include <clc/math/clc_subnormal_config.h>
 #include <clc/math/clc_trunc.h>
+#include <clc/math/math.h>
 #include <clc/shared/clc_max.h>
-
 #include <math/clc_remainder.h>
-#include "config.h"
-#include "math.h"
-
-_CLC_DEF _CLC_OVERLOAD float __clc_fmod(float x, float y)
-{
-    int ux = as_int(x);
-    int ax = ux & EXSIGNBIT_SP32;
-    float xa = as_float(ax);
-    int sx = ux ^ ax;
-    int ex = ax >> EXPSHIFTBITS_SP32;
-
-    int uy = as_int(y);
-    int ay = uy & EXSIGNBIT_SP32;
-    float ya = as_float(ay);
-    int ey = ay >> EXPSHIFTBITS_SP32;
-
-    float xr = as_float(0x3f800000 | (ax & 0x007fffff));
-    float yr = as_float(0x3f800000 | (ay & 0x007fffff));
-    int c;
-    int k = ex - ey;
-
-    while (k > 0) {
-        c = xr >= yr;
-        xr -= c ? yr : 0.0f;
-        xr += xr;
-        --k;
-    }
 
+_CLC_DEF _CLC_OVERLOAD float __clc_fmod(float x, float y) {
+  int ux = as_int(x);
+  int ax = ux & EXSIGNBIT_SP32;
+  float xa = as_float(ax);
+  int sx = ux ^ ax;
+  int ex = ax >> EXPSHIFTBITS_SP32;
+
+  int uy = as_int(y);
+  int ay = uy & EXSIGNBIT_SP32;
+  float ya = as_float(ay);
+  int ey = ay >> EXPSHIFTBITS_SP32;
+
+  float xr = as_float(0x3f800000 | (ax & 0x007fffff));
+  float yr = as_float(0x3f800000 | (ay & 0x007fffff));
+  int c;
+  int k = ex - ey;
+
+  while (k > 0) {
     c = xr >= yr;
     xr -= c ? yr : 0.0f;
+    xr += xr;
+    --k;
+  }
 
-    int lt = ex < ey;
-
-    xr = lt ? xa : xr;
-    yr = lt ? ya : yr;
+  c = xr >= yr;
+  xr -= c ? yr : 0.0f;
 
+  int lt = ex < ey;
 
-    float s = as_float(ey << EXPSHIFTBITS_SP32);
-    xr *= lt ? 1.0f : s;
+  xr = lt ? xa : xr;
+  yr = lt ? ya : yr;
 
-    c = ax == ay;
-    xr = c ? 0.0f : xr;
+  float s = as_float(ey << EXPSHIFTBITS_SP32);
+  xr *= lt ? 1.0f : s;
 
-    xr = as_float(sx ^ as_int(xr));
+  c = ax == ay;
+  xr = c ? 0.0f : xr;
 
-    c = ax > PINFBITPATT_SP32 | ay > PINFBITPATT_SP32 | ax == PINFBITPATT_SP32 | ay == 0;
-    xr = c ? as_float(QNANBITPATT_SP32) : xr;
+  xr = as_float(sx ^ as_int(xr));
 
-    return xr;
+  c = ax > PINFBITPATT_SP32 | ay > PINFBITPATT_SP32 | ax == PINFBITPATT_SP32 |
+      ay == 0;
+  xr = c ? as_float(QNANBITPATT_SP32) : xr;
 
+  return xr;
 }
 _CLC_BINARY_VECTORIZE(_CLC_DEF _CLC_OVERLOAD, float, __clc_fmod, float, float);
 
 #ifdef cl_khr_fp64
-_CLC_DEF _CLC_OVERLOAD double __clc_fmod(double x, double y)
-{
-    ulong ux = as_ulong(x);
-    ulong ax = ux & ~SIGNBIT_DP64;
-    ulong xsgn = ux ^ ax;
-    double dx = as_double(ax);
-    int xexp = convert_int(ax >> EXPSHIFTBITS_DP64);
-    int xexp1 = 11 - (int) clz(ax & MANTBITS_DP64);
-    xexp1 = xexp < 1 ? xexp1 : xexp;
-
-    ulong uy = as_ulong(y);
-    ulong ay = uy & ~SIGNBIT_DP64;
-    double dy = as_double(ay);
-    int yexp = convert_int(ay >> EXPSHIFTBITS_DP64);
-    int yexp1 = 11 - (int) clz(ay & MANTBITS_DP64);
-    yexp1 = yexp < 1 ? yexp1 : yexp;
-
-    // First assume |x| > |y|
-
-    // Set ntimes to the number of times we need to do a
-    // partial remainder. If the exponent of x is an exact multiple
-    // of 53 larger than the exponent of y, and the mantissa of x is
-    // less than the mantissa of y, ntimes will be one too large
-    // but it doesn't matter - it just means that we'll go round
-    // the loop below one extra time.
-    int ntimes = __clc_max(0, (xexp1 - yexp1) / 53);
-    double w =  ldexp(dy, ntimes * 53);
-    w = ntimes == 0 ? dy : w;
-    double scale = ntimes == 0 ? 1.0 : 0x1.0p-53;
-
-    // Each time round the loop we compute a partial remainder.
-    // This is done by subtracting a large multiple of w
-    // from x each time, where w is a scaled up version of y.
-    // The subtraction must be performed exactly in quad
-    // precision, though the result at each stage can
-    // fit exactly in a double precision number.
-    int i;
-    double t, v, p, pp;
-
-    for (i = 0; i < ntimes; i++) {
-        // Compute integral multiplier
-        t = __clc_trunc(dx / w);
-
-        // Compute w * t in quad precision
-        p = w * t;
-        pp = fma(w, t, -p);
-
-        // Subtract w * t from dx
-        v = dx - p;
-        dx = v + (((dx - v) - p) - pp);
-
-        // If t was one too large, dx will be negative. Add back one w.
-        dx += dx < 0.0 ? w : 0.0;
-
-        // Scale w down by 2^(-53) for the next iteration
-        w *= scale;
-    }
-
-    // One more time
-    // Variable todd says whether the integer t is odd or not
-    t = __clc_floor(dx / w);
-    long lt = (long)t;
-    int todd = lt & 1;
-
+_CLC_DEF _CLC_OVERLOAD double __clc_fmod(double x, double y) {
+  ulong ux = as_ulong(x);
+  ulong ax = ux & ~SIGNBIT_DP64;
+  ulong xsgn = ux ^ ax;
+  double dx = as_double(ax);
+  int xexp = convert_int(ax >> EXPSHIFTBITS_DP64);
+  int xexp1 = 11 - (int)__clc_clz(ax & MANTBITS_DP64);
+  xexp1 = xexp < 1 ? xexp1 : xexp;
+
+  ulong uy = as_ulong(y);
+  ulong ay = uy & ~SIGNBIT_DP64;
+  double dy = as_double(ay);
+  int yexp = convert_int(ay >> EXPSHIFTBITS_DP64);
+  int yexp1 = 11 - (int)__clc_clz(ay & MANTBITS_DP64);
+  yexp1 = yexp < 1 ? yexp1 : yexp;
+
+  // First assume |x| > |y|
+
+  // Set ntimes to the number of times we need to do a
+  // partial remainder. If the exponent of x is an exact multiple
+  // of 53 larger than the exponent of y, and the mantissa of x is
+  // less than the mantissa of y, ntimes will be one too large
+  // but it doesn't matter - it just means that we'll go round
+  // the loop below one extra time.
+  int ntimes = __clc_max(0, (xexp1 - yexp1) / 53);
+  double w = ldexp(dy, ntimes * 53);
+  w = ntimes == 0 ? dy : w;
+  double scale = ntimes == 0 ? 1.0 : 0x1.0p-53;
+
+  // Each time round the loop we compute a partial remainder.
+  // This is done by subtracting a large multiple of w
+  // from x each time, where w is a scaled up version of y.
+  // The subtraction must be performed exactly in quad
+  // precision, though the result at each stage can
+  // fit exactly in a double precision number.
+  int i;
+  double t, v, p, pp;
+
+  for (i = 0; i < ntimes; i++) {
+    // Compute integral multiplier
+    t = __clc_trunc(dx / w);
+
+    // Compute w * t in quad precision
     p = w * t;
     pp = fma(w, t, -p);
+
+    // Subtract w * t from dx
     v = dx - p;
     dx = v + (((dx - v) - p) - pp);
-    i = dx < 0.0;
-    todd ^= i;
-    dx += i ? w : 0.0;
 
-    // At this point, dx lies in the range [0,dy)
-    double ret = as_double(xsgn ^ as_ulong(dx));
-    dx = as_double(ax);
+    // If t was one too large, dx will be negative. Add back one w.
+    dx += dx < 0.0 ? w : 0.0;
+
+    // Scale w down by 2^(-53) for the next iteration
+    w *= scale;
+  }
+
+  // One more time
+  // Variable todd says whether the integer t is odd or not
+  t = __clc_floor(dx / w);
+  long lt = (long)t;
+  int todd = lt & 1;
+
+  p = w * t;
+  pp = fma(w, t, -p);
+  v = dx - p;
+  dx = v + (((dx - v) - p) - pp);
+  i = dx < 0.0;
+  todd ^= i;
+  dx += i ? w : 0.0;
+
+  // At this point, dx lies in the range [0,dy)
+  double ret = as_double(xsgn ^ as_ulong(dx));
+  dx = as_double(ax);
 
-    // Now handle |x| == |y|
-    int c = dx == dy;
-    t = as_double(xsgn);
-    ret = c ? t : ret;
+  // Now handle |x| == |y|
+  int c = dx == dy;
+  t = as_double(xsgn);
+  ret = c ? t : ret;
 
-    // Next, handle |x| < |y|
-    c = dx < dy;
-    ret = c ? x : ret;
+  // Next, handle |x| < |y|
+  c = dx < dy;
+  ret = c ? x : ret;
 
-    // We don't need anything special for |x| == 0
+  // We don't need anything special for |x| == 0
 
-    // |y| is 0
-    c = dy == 0.0;
-    ret = c ? as_double(QNANBITPATT_DP64) : ret;
+  // |y| is 0
+  c = dy == 0.0;
+  ret = c ? as_double(QNANBITPATT_DP64) : ret;
 
-    // y is +-Inf, NaN
-    c = yexp > BIASEDEMAX_DP64;
-    t = y == y ? x : y;
-    ret = c ? t : ret;
+  // y is +-Inf, NaN
+  c = yexp > BIASEDEMAX_DP64;
+  t = y == y ? x : y;
+  ret = c ? t : ret;
 
-    // x is +=Inf, NaN
-    c = xexp > BIASEDEMAX_DP64;
-    ret = c ? as_double(QNANBITPATT_DP64) : ret;
+  // x is +=Inf, NaN
+  c = xexp > BIASEDEMAX_DP64;
+  ret = c ? as_double(QNANBITPATT_DP64) : ret;
 
-    return ret;
+  return ret;
 }
-_CLC_BINARY_VECTORIZE(_CLC_DEF _CLC_OVERLOAD, double, __clc_fmod, double, double);
+_CLC_BINARY_VECTORIZE(_CLC_DEF _CLC_OVERLOAD, double, __clc_fmod, double,
+                      double);
 #endif
diff --git a/libclc/generic/lib/math/clc_hypot.cl b/libclc/generic/lib/math/clc_hypot.cl
index d889969d6d8c2..fd2e87b4a1ed8 100644
--- a/libclc/generic/lib/math/clc_hypot.cl
+++ b/libclc/generic/lib/math/clc_hypot.cl
@@ -24,13 +24,12 @@
 #include <clc/clcmacro.h>
 #include <clc/integer/clc_abs.h>
 #include <clc/math/clc_mad.h>
+#include <clc/math/clc_subnormal_config.h>
+#include <clc/math/math.h>
 #include <clc/relational/clc_isnan.h>
 #include <clc/shared/clc_clamp.h>
 #include <math/clc_hypot.h>
 
-#include "config.h"
-#include "math.h"
-
 // Returns sqrt(x*x + y*y) with no overflow or underflow unless the result
 // warrants it
 _CLC_DEF _CLC_OVERLOAD float __clc_hypot(float x, float y) {
diff --git a/libclc/generic/lib/math/clc_ldexp.cl b/libclc/generic/lib/math/clc_ldexp.cl
index 6d3721588df81..a03f05d217756 100644
--- a/libclc/generic/lib/math/clc_ldexp.cl
+++ b/libclc/generic/lib/math/clc_ldexp.cl
@@ -20,10 +20,11 @@
  * THE SOFTWARE.
  */
 
-#include "config.h"
-#include "math.h"
 #include <clc/clc.h>
 #include <clc/clcmacro.h>
+#include <clc/integer/clc_add_sat.h>
+#include <clc/math/clc_subnormal_config.h>
+#include <clc/math/math.h>
 #include <clc/relational/clc_isinf.h>
 #include <clc/relational/clc_isnan.h>
 #include <clc/shared/clc_clamp.h>
@@ -37,7 +38,7 @@ _CLC_DEF _CLC_OVERLOAD float __clc_ldexp(float x, int n) {
     int e = (i >> 23) & 0xff;
     int m = i & 0x007fffff;
     int s = i & 0x80000000;
-    int v = add_sat(e, n);
+    int v = __clc_add_sat(e, n);
     v = __clc_clamp(v, 0, 0xff);
     int mr = e == 0 | v == 0 | v == 0xff ? 0 : m;
     int c = e == 0xff;
diff --git a/libclc/generic/lib/math/clc_pow.cl b/libclc/generic/lib/math/clc_pow.cl
index 4abfaf1c10df4..5dcd392c0f7ed 100644
--- a/libclc/generic/lib/math/clc_pow.cl
+++ b/libclc/generic/lib/math/clc_pow.cl
@@ -24,10 +24,9 @@
 #include <clc/clcmacro.h>
 #include <clc/math/clc_fabs.h>
 #include <clc/math/clc_mad.h>
-
-#include "config.h"
-#include "math.h"
-#include "tables.h"
+#include <clc/math/clc_subnormal_config.h>
+#include <clc/math/math.h>
+#include <clc/math/tables.h>
 
 /*
  compute pow using log and exp
diff --git a/libclc/generic/lib/math/clc_pown.cl b/libclc/generic/lib/math/clc_pown.cl
index c020892664602..a0f968c238e99 100644
--- a/libclc/generic/lib/math/clc_pown.cl
+++ b/libclc/generic/lib/math/clc_pown.cl
@@ -24,10 +24,9 @@
 #include <clc/clcmacro.h>
 #include <clc/math/clc_fabs.h>
 #include <clc/math/clc_mad.h>
-
-#include "config.h"
-#include "math.h"
-#include "tables.h"
+#include <clc/math/clc_subnormal_config.h>
+#include <clc/math/math.h>
+#include <clc/math/tables.h>
 
 // compute pow using log and exp
 // x^y = exp(y * log(x))
diff --git a/libclc/generic/lib/math/clc_powr.cl b/libclc/generic/lib/math/clc_powr.cl
index 9516be34456b8..7e1a6f2a02e7a 100644
--- a/libclc/generic/lib/math/clc_powr.cl
+++ b/libclc/generic/lib/math/clc_powr.cl
@@ -24,10 +24,9 @@
 #include <clc/clcmacro.h>
 #include <clc/math/clc_fabs.h>
 #include <clc/math/clc_mad.h>
-
-#include "config.h"
-#include "math.h"
-#include "tables.h"
+#include <clc/math/clc_subnormal_config.h>
+#include <clc/math/math.h>
+#include <clc/math/tables.h>
 
 // compute pow using log and exp
 // x^y = exp(y * log(x))
diff --git a/libclc/generic/lib/math/clc_remainder.cl b/libclc/generic/lib/math/clc_remainder.cl
index 8a0ce8816fcb3..31d17d5aaf6b6 100644
--- a/libclc/generic/lib/math/clc_remainder.cl
+++ b/libclc/generic/lib/math/clc_remainder.cl
@@ -22,200 +22,200 @@
 
 #include <clc/clc.h>
 #include <clc/clcmacro.h>
+#include <clc/integer/clc_clz.h>
 #include <clc/math/clc_floor.h>
+#include <clc/math/clc_subnormal_config.h>
 #include <clc/math/clc_trunc.h>
+#include <clc/math/math.h>
 #include <clc/shared/clc_max.h>
-
 #include <math/clc_remainder.h>
-#include "config.h"
-#include "math.h"
-
-_CLC_DEF _CLC_OVERLOAD float __clc_remainder(float x, float y)
-{
-    int ux = as_int(x);
-    int ax = ux & EXSIGNBIT_SP32;
-    float xa = as_float(ax);
-    int sx = ux ^ ax;
-    int ex = ax >> EXPSHIFTBITS_SP32;
-
-    int uy = as_int(y);
-    int ay = uy & EXSIGNBIT_SP32;
-    float ya = as_float(ay);
-    int ey = ay >> EXPSHIFTBITS_SP32;
-
-    float xr = as_float(0x3f800000 | (ax & 0x007fffff));
-    float yr = as_float(0x3f800000 | (ay & 0x007fffff));
-    int c;
-    int k = ex - ey;
-
-    uint q = 0;
-
-    while (k > 0) {
-        c = xr >= yr;
-        q = (q << 1) | c;
-        xr -= c ? yr : 0.0f;
-        xr += xr;
-	--k;
-    }
-
-    c = xr > yr;
+
+_CLC_DEF _CLC_OVERLOAD float __clc_remainder(float x, float y) {
+  int ux = as_int(x);
+  int ax = ux & EXSIGNBIT_SP32;
+  float xa = as_float(ax);
+  int sx = ux ^ ax;
+  int ex = ax >> EXPSHIFTBITS_SP32;
+
+  int uy = as_int(y);
+  int ay = uy & EXSIGNBIT_SP32;
+  float ya = as_float(ay);
+  int ey = ay >> EXPSHIFTBITS_SP32;
+
+  float xr = as_float(0x3f800000 | (ax & 0x007fffff));
+  float yr = as_float(0x3f800000 | (ay & 0x007fffff));
+  int c;
+  int k = ex - ey;
+
+  uint q = 0;
+
+  while (k > 0) {
+    c = xr >= yr;
     q = (q << 1) | c;
     xr -= c ? yr : 0.0f;
+    xr += xr;
+    --k;
+  }
 
-    int lt = ex < ey;
+  c = xr > yr;
+  q = (q << 1) | c;
+  xr -= c ? yr : 0.0f;
 
-    q = lt ? 0 : q;
-    xr = lt ? xa : xr;
-    yr = lt ? ya : yr;
+  int lt = ex < ey;
 
-    c = (yr < 2.0f * xr) | ((yr == 2.0f * xr) & ((q & 0x1) == 0x1));
-    xr -= c ? yr : 0.0f;
-    q += c;
+  q = lt ? 0 : q;
+  xr = lt ? xa : xr;
+  yr = lt ? ya : yr;
 
-    float s = as_float(ey << EXPSHIFTBITS_SP32);
-    xr *= lt ? 1.0f : s;
+  c = (yr < 2.0f * xr) | ((yr == 2.0f * xr) & ((q & 0x1) == 0x1));
+  xr -= c ? yr : 0.0f;
+  q += c;
 
-    c = ax == ay;
-    xr = c ? 0.0f : xr;
+  float s = as_float(ey << EXPSHIFTBITS_SP32);
+  xr *= lt ? 1.0f : s;
 
-    xr = as_float(sx ^ as_int(xr));
+  c = ax == ay;
+  xr = c ? 0.0f : xr;
 
-    c = ax > PINFBITPATT_SP32 | ay > PINFBITPATT_SP32 | ax == PINFBITPATT_SP32 | ay == 0;
-    xr = c ? as_float(QNANBITPATT_SP32) : xr;
+  xr = as_float(sx ^ as_int(xr));
 
-    return xr;
+  c = ax > PINFBITPATT_SP32 | ay > PINFBITPATT_SP32 | ax == PINFBITPATT_SP32 |
+      ay == 0;
+  xr = c ? as_float(QNANBITPATT_SP32) : xr;
 
+  return xr;
 }
-_CLC_BINARY_VECTORIZE(_CLC_DEF _CLC_OVERLOAD, float, __clc_remainder, float, float);
+_CLC_BINARY_VECTORIZE(_CLC_DEF _CLC_OVERLOAD, float, __clc_remainder, float,
+                      float);
 
 #ifdef cl_khr_fp64
-_CLC_DEF _CLC_OVERLOAD double __clc_remainder(double x, double y)
-{
-    ulong ux = as_ulong(x);
-    ulong ax = ux & ~SIGNBIT_DP64;
-    ulong xsgn = ux ^ ax;
-    double dx = as_double(ax);
-    int xexp = convert_int(ax >> EXPSHIFTBITS_DP64);
-    int xexp1 = 11 - (int) clz(ax & MANTBITS_DP64);
-    xexp1 = xexp < 1 ? xexp1 : xexp;
-
-    ulong uy = as_ulong(y);
-    ulong ay = uy & ~SIGNBIT_DP64;
-    double dy = as_double(ay);
-    int yexp = convert_int(ay >> EXPSHIFTBITS_DP64);
-    int yexp1 = 11 - (int) clz(ay & MANTBITS_DP64);
-    yexp1 = yexp < 1 ? yexp1 : yexp;
-
-    int qsgn = ((ux ^ uy) & SIGNBIT_DP64) == 0UL ? 1 : -1;
-
-    // First assume |x| > |y|
-
-    // Set ntimes to the number of times we need to do a
-    // partial remainder. If the exponent of x is an exact multiple
-    // of 53 larger than the exponent of y, and the mantissa of x is
-    // less than the mantissa of y, ntimes will be one too large
-    // but it doesn't matter - it just means that we'll go round
-    // the loop below one extra time.
-    int ntimes = __clc_max(0, (xexp1 - yexp1) / 53);
-    double w =  ldexp(dy, ntimes * 53);
-    w = ntimes == 0 ? dy : w;
-    double scale = ntimes == 0 ? 1.0 : 0x1.0p-53;
-
-    // Each time round the loop we compute a partial remainder.
-    // This is done by subtracting a large multiple of w
-    // from x each time, where w is a scaled up version of y.
-    // The subtraction must be performed exactly in quad
-    // precision, though the result at each stage can
-    // fit exactly in a double precision number.
-    int i;
-    double t, v, p, pp;
-
-    for (i = 0; i < ntimes; i++) {
-        // Compute integral multiplier
-        t = __clc_trunc(dx / w);
-
-        // Compute w * t in quad precision
-        p = w * t;
-        pp = fma(w, t, -p);
-
-        // Subtract w * t from dx
-        v = dx - p;
-        dx = v + (((dx - v) - p) - pp);
-
-        // If t was one too large, dx will be negative. Add back one w.
-        dx += dx < 0.0 ? w : 0.0;
-
-        // Scale w down by 2^(-53) for the next iteration
-        w *= scale;
-    }
-
-    // One more time
-    // Variable todd says whether the integer t is odd or not
-    t = __clc_floor(dx / w);
-    long lt = (long)t;
-    int todd = lt & 1;
-
+_CLC_DEF _CLC_OVERLOAD double __clc_remainder(double x, double y) {
+  ulong ux = as_ulong(x);
+  ulong ax = ux & ~SIGNBIT_DP64;
+  ulong xsgn = ux ^ ax;
+  double dx = as_double(ax);
+  int xexp = convert_int(ax >> EXPSHIFTBITS_DP64);
+  int xexp1 = 11 - (int)__clc_clz(ax & MANTBITS_DP64);
+  xexp1 = xexp < 1 ? xexp1 : xexp;
+
+  ulong uy = as_ulong(y);
+  ulong ay = uy & ~SIGNBIT_DP64;
+  double dy = as_double(ay);
+  int yexp = convert_int(ay >> EXPSHIFTBITS_DP64);
+  int yexp1 = 11 - (int)__clc_clz(ay & MANTBITS_DP64);
+  yexp1 = yexp < 1 ? yexp1 : yexp;
+
+  int qsgn = ((ux ^ uy) & SIGNBIT_DP64) == 0UL ? 1 : -1;
+
+  // First assume |x| > |y|
+
+  // Set ntimes to the number of times we need to do a
+  // partial remainder. If the exponent of x is an exact multiple
+  // of 53 larger than the exponent of y, and the mantissa of x is
+  // less than the mantissa of y, ntimes will be one too large
+  // but it doesn't matter - it just means that we'll go round
+  // the loop below one extra time.
+  int ntimes = __clc_max(0, (xexp1 - yexp1) / 53);
+  double w = ldexp(dy, ntimes * 53);
+  w = ntimes == 0 ? dy : w;
+  double scale = ntimes == 0 ? 1.0 : 0x1.0p-53;
+
+  // Each time round the loop we compute a partial remainder.
+  // This is done by subtracting a large multiple of w
+  // from x each time, where w is a scaled up version of y.
+  // The subtraction must be performed exactly in quad
+  // precision, though the result at each stage can
+  // fit exactly in a double precision number.
+  int i;
+  double t, v, p, pp;
+
+  for (i = 0; i < ntimes; i++) {
+    // Compute integral multiplier
+    t = __clc_trunc(dx / w);
+
+    // Compute w * t in quad precision
     p = w * t;
     pp = fma(w, t, -p);
+
+    // Subtract w * t from dx
     v = dx - p;
     dx = v + (((dx - v) - p) - pp);
-    i = dx < 0.0;
-    todd ^= i;
-    dx += i ? w : 0.0;
 
-    // At this point, dx lies in the range [0,dy)
+    // If t was one too large, dx will be negative. Add back one w.
+    dx += dx < 0.0 ? w : 0.0;
+
+    // Scale w down by 2^(-53) for the next iteration
+    w *= scale;
+  }
+
+  // One more time
+  // Variable todd says whether the integer t is odd or not
+  t = __clc_floor(dx / w);
+  long lt = (long)t;
+  int todd = lt & 1;
+
+  p = w * t;
+  pp = fma(w, t, -p);
+  v = dx - p;
+  dx = v + (((dx - v) - p) - pp);
+  i = dx < 0.0;
+  todd ^= i;
+  dx += i ? w : 0.0;
+
+  // At this point, dx lies in the range [0,dy)
 
-    // For the fmod function, we're done apart from setting the correct sign.
-    //
-    // For the remainder function, we need to adjust dx
-    // so that it lies in the range (-y/2, y/2] by carefully
-    // subtracting w (== dy == y) if necessary. The rigmarole
-    // with todd is to get the correct sign of the result
-    // when x/y lies exactly half way between two integers,
-    // when we need to choose the even integer.
+  // For the fmod function, we're done apart from setting the correct sign.
+  //
+  // For the remainder function, we need to adjust dx
+  // so that it lies in the range (-y/2, y/2] by carefully
+  // subtracting w (== dy == y) if necessary. The rigmarole
+  // with todd is to get the correct sign of the result
+  // when x/y lies exactly half way between two integers,
+  // when we need to choose the even integer.
 
-    int al = (2.0*dx > w) | (todd & (2.0*dx == w));
-    double dxl = dx - (al ? w : 0.0);
+  int al = (2.0 * dx > w) | (todd & (2.0 * dx == w));
+  double dxl = dx - (al ? w : 0.0);
 
-    int ag = (dx > 0.5*w) | (todd & (dx == 0.5*w));
-    double dxg = dx - (ag ? w : 0.0);
+  int ag = (dx > 0.5 * w) | (todd & (dx == 0.5 * w));
+  double dxg = dx - (ag ? w : 0.0);
 
-    dx = dy < 0x1.0p+1022 ? dxl : dxg;
+  dx = dy < 0x1.0p+1022 ? dxl : dxg;
 
-    double ret = as_double(xsgn ^ as_ulong(dx));
-    dx = as_double(ax);
+  double ret = as_double(xsgn ^ as_ulong(dx));
+  dx = as_double(ax);
 
-    // Now handle |x| == |y|
-    int c = dx == dy;
-    t = as_double(xsgn);
-    ret = c ? t : ret;
+  // Now handle |x| == |y|
+  int c = dx == dy;
+  t = as_double(xsgn);
+  ret = c ? t : ret;
 
-    // Next, handle |x| < |y|
-    c = dx < dy;
-    ret = c ? x : ret;
+  // Next, handle |x| < |y|
+  c = dx < dy;
+  ret = c ? x : ret;
 
-    c &= (yexp < 1023 & 2.0*dx > dy) | (dx > 0.5*dy);
-    // we could use a conversion here instead since qsgn = +-1
-    p = qsgn == 1 ? -1.0 : 1.0;
-    t = fma(y, p, x);
-    ret = c ? t : ret;
+  c &= (yexp<1023 & 2.0 * dx> dy) | (dx > 0.5 * dy);
+  // we could use a conversion here instead since qsgn = +-1
+  p = qsgn == 1 ? -1.0 : 1.0;
+  t = fma(y, p, x);
+  ret = c ? t : ret;
 
-    // We don't need anything special for |x| == 0
+  // We don't need anything special for |x| == 0
 
-    // |y| is 0
-    c = dy == 0.0;
-    ret = c ? as_double(QNANBITPATT_DP64) : ret;
+  // |y| is 0
+  c = dy == 0.0;
+  ret = c ? as_double(QNANBITPATT_DP64) : ret;
 
-    // y is +-Inf, NaN
-    c = yexp > BIASEDEMAX_DP64;
-    t = y == y ? x : y;
-    ret = c ? t : ret;
+  // y is +-Inf, NaN
+  c = yexp > BIASEDEMAX_DP64;
+  t = y == y ? x : y;
+  ret = c ? t : ret;
 
-    // x is +=Inf, NaN
-    c = xexp > BIASEDEMAX_DP64;
-    ret = c ? as_double(QNANBITPATT_DP64) : ret;
+  // x is +=Inf, NaN
+  c = xexp > BIASEDEMAX_DP64;
+  ret = c ? as_double(QNANBITPATT_DP64) : ret;
 
-    return ret;
+  return ret;
 }
-_CLC_BINARY_VECTORIZE(_CLC_DEF _CLC_OVERLOAD, double, __clc_remainder, double, double);
+_CLC_BINARY_VECTORIZE(_CLC_DEF _CLC_OVERLOAD, double, __clc_remainder, double,
+                      double);
 #endif
diff --git a/libclc/generic/lib/math/clc_remquo.cl b/libclc/generic/lib/math/clc_remquo.cl
index 8d2e5f9a74bfe..af3e7a2b07500 100644
--- a/libclc/generic/lib/math/clc_remquo.cl
+++ b/libclc/generic/lib/math/clc_remquo.cl
@@ -22,14 +22,14 @@
 
 #include <clc/clc.h>
 #include <clc/clcmacro.h>
+#include <clc/integer/clc_clz.h>
 #include <clc/math/clc_floor.h>
+#include <clc/math/clc_subnormal_config.h>
 #include <clc/math/clc_trunc.h>
+#include <clc/math/math.h>
 #include <clc/shared/clc_max.h>
 #include <math/clc_remainder.h>
 
-#include "config.h"
-#include "math.h"
-
 _CLC_DEF _CLC_OVERLOAD float __clc_remquo(float x, float y,
                                           __private int *quo) {
   x = __clc_flush_denormal_if_not_supported(x);
@@ -135,14 +135,14 @@ _CLC_DEF _CLC_OVERLOAD double __clc_remquo(double x, double y,
   ulong xsgn = ux ^ ax;
   double dx = as_double(ax);
   int xexp = convert_int(ax >> EXPSHIFTBITS_DP64);
-  int xexp1 = 11 - (int)clz(ax & MANTBITS_DP64);
+  int xexp1 = 11 - (int)__clc_clz(ax & MANTBITS_DP64);
   xexp1 = xexp < 1 ? xexp1 : xexp;
 
   ulong uy = as_ulong(y);
   ulong ay = uy & ~SIGNBIT_DP64;
   double dy = as_double(ay);
   int yexp = convert_int(ay >> EXPSHIFTBITS_DP64);
-  int yexp1 = 11 - (int)clz(ay & MANTBITS_DP64);
+  int yexp1 = 11 - (int)__clc_clz(ay & MANTBITS_DP64);
   yexp1 = yexp < 1 ? yexp1 : yexp;
 
   int qsgn = ((ux ^ uy) & SIGNBIT_DP64) == 0UL ? 1 : -1;
diff --git a/libclc/generic/lib/math/clc_rootn.cl b/libclc/generic/lib/math/clc_rootn.cl
index 70ae02ac2370c..42b983784c14d 100644
--- a/libclc/generic/lib/math/clc_rootn.cl
+++ b/libclc/generic/lib/math/clc_rootn.cl
@@ -24,10 +24,9 @@
 #include <clc/clcmacro.h>
 #include <clc/math/clc_fabs.h>
 #include <clc/math/clc_mad.h>
-
-#include "config.h"
-#include "math.h"
-#include "tables.h"
+#include <clc/math/clc_subnormal_config.h>
+#include <clc/math/math.h>
+#include <clc/math/tables.h>
 
 // compute pow using log and exp
 // x^y = exp(y * log(x))
diff --git a/libclc/generic/lib/math/clc_tan.cl b/libclc/generic/lib/math/clc_tan.cl
index 4daaee59b10d2..c1af01fbb7ec7 100644
--- a/libclc/generic/lib/math/clc_tan.cl
+++ b/libclc/generic/lib/math/clc_tan.cl
@@ -19,16 +19,15 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
+#include "sincos_helpers.h"
 #include <clc/clc.h>
 #include <clc/clcmacro.h>
 #include <clc/math/clc_fabs.h>
+#include <clc/math/math.h>
+#include <clc/math/tables.h>
 #include <clc/relational/clc_isinf.h>
 #include <clc/relational/clc_isnan.h>
 
-#include "math.h"
-#include "sincos_helpers.h"
-#include "tables.h"
-
 _CLC_DEF _CLC_OVERLOAD float __clc_tan(float x) {
   int ix = as_int(x);
   int ax = ix & 0x7fffffff;
diff --git a/libclc/generic/lib/math/clc_tanpi.cl b/libclc/generic/lib/math/clc_tanpi.cl
index 65d1984593706..90f36eae63996 100644
--- a/libclc/generic/lib/math/clc_tanpi.cl
+++ b/libclc/generic/lib/math/clc_tanpi.cl
@@ -19,12 +19,12 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
-#include <clc/clc.h>
-#include <clc/clcmacro.h>
 
-#include "math.h"
 #include "sincos_helpers.h"
-#include "tables.h"
+#include <clc/clc.h>
+#include <clc/clcmacro.h>
+#include <clc/math/math.h>
+#include <clc/math/tables.h>
 
 _CLC_DEF _CLC_OVERLOAD float __clc_tanpi(float x)
 {
diff --git a/libclc/generic/lib/math/copysign.cl b/libclc/generic/lib/math/copysign.cl
index 08045bebf88a1..59cd4f6520305 100644
--- a/libclc/generic/lib/math/copysign.cl
+++ b/libclc/generic/lib/math/copysign.cl
@@ -1,27 +1,7 @@
 #include <clc/clc.h>
-#include <clc/clcmacro.h>
+#include <clc/math/clc_copysign.h>
 
-_CLC_DEFINE_BINARY_BUILTIN(float, copysign, __builtin_copysignf, float, float)
+#define FUNCTION copysign
+#define __CLC_BODY <clc/shared/binary_def.inc>
 
-#ifdef cl_khr_fp64
-
-#pragma OPENCL EXTENSION cl_khr_fp64 : enable
-
-_CLC_DEFINE_BINARY_BUILTIN(double, copysign, __builtin_copysign, double, double)
-
-#endif
-
-#ifdef cl_khr_fp16
-
-#pragma OPENCL EXTENSION cl_khr_fp16 : enable
-
-_CLC_DEF _CLC_OVERLOAD half copysign(half x, half y)
-{
-   ushort sign_x = as_ushort(x) & 0x8000u;
-   ushort unsigned_y = as_ushort(y) & 0x7ffffu;
-
-   return as_half((ushort)(sign_x | unsigned_y));
-}
-_CLC_BINARY_VECTORIZE(_CLC_OVERLOAD _CLC_DEF, half, copysign, half, half)
-
-#endif
+#include <clc/math/gentype.inc>
diff --git a/libclc/generic/lib/math/cos.cl b/libclc/generic/lib/math/cos.cl
index 2945c90d6fe0f..e13210cd40864 100644
--- a/libclc/generic/lib/math/cos.cl
+++ b/libclc/generic/lib/math/cos.cl
@@ -20,11 +20,10 @@
  * THE SOFTWARE.
  */
 
+#include "sincos_helpers.h"
 #include <clc/clc.h>
 #include <clc/clcmacro.h>
-
-#include "math.h"
-#include "sincos_helpers.h"
+#include <clc/math/math.h>
 
 _CLC_OVERLOAD _CLC_DEF float cos(float x)
 {
diff --git a/libclc/generic/lib/math/cosh.cl b/libclc/generic/lib/math/cosh.cl
index 02b03a761aa19..5b5400178e28e 100644
--- a/libclc/generic/lib/math/cosh.cl
+++ b/libclc/generic/lib/math/cosh.cl
@@ -22,9 +22,8 @@
 
 #include <clc/clc.h>
 #include <clc/clcmacro.h>
-
-#include "math.h"
-#include "tables.h"
+#include <clc/math/math.h>
+#include <clc/math/tables.h>
 
 _CLC_OVERLOAD _CLC_DEF float cosh(float x) {
 
diff --git a/libclc/generic/lib/math/cospi.cl b/libclc/generic/lib/math/cospi.cl
index 98e989267c917..a3bfb8f20eee4 100644
--- a/libclc/generic/lib/math/cospi.cl
+++ b/libclc/generic/lib/math/cospi.cl
@@ -22,8 +22,8 @@
 
 #include <clc/clc.h>
 #include <clc/clcmacro.h>
+#include <clc/math/math.h>
 
-#include "math.h"
 #include "sincos_helpers.h"
 #include "sincospiF_piby4.h"
 #ifdef cl_khr_fp64
diff --git a/libclc/generic/lib/math/ep_log.cl b/libclc/generic/lib/math/ep_log.cl
index 3c2c62c3d305b..90c9fa426fec1 100644
--- a/libclc/generic/lib/math/ep_log.cl
+++ b/libclc/generic/lib/math/ep_log.cl
@@ -22,10 +22,10 @@
 
 #ifdef cl_khr_fp64
 
-#include <clc/clc.h>
 #include "ep_log.h"
-#include "math.h"
-#include "tables.h"
+#include <clc/clc.h>
+#include <clc/math/math.h>
+#include <clc/math/tables.h>
 
 #pragma OPENCL EXTENSION cl_khr_fp64 : enable
 
diff --git a/libclc/generic/lib/math/ep_log.h b/libclc/generic/lib/math/ep_log.h
index 7f99ac60b72fb..414e6231f7fd6 100644
--- a/libclc/generic/lib/math/ep_log.h
+++ b/libclc/generic/lib/math/ep_log.h
@@ -20,6 +20,8 @@
  * THE SOFTWARE.
  */
 
+#include <clc/clcfunc.h>
+
 #ifdef cl_khr_fp64
 
 #pragma OPENCL EXTENSION cl_khr_fp64 : enable
diff --git a/libclc/generic/lib/math/erf.cl b/libclc/generic/lib/math/erf.cl
index ae8b6ab784558..bc305fe414a32 100644
--- a/libclc/generic/lib/math/erf.cl
+++ b/libclc/generic/lib/math/erf.cl
@@ -22,8 +22,7 @@
 
 #include <clc/clc.h>
 #include <clc/clcmacro.h>
-
-#include "math.h"
+#include <clc/math/math.h>
 
 /*
  * ====================================================
diff --git a/libclc/generic/lib/math/erfc.cl b/libclc/generic/lib/math/erfc.cl
index c4d34ea85e98b..39d8d826280bc 100644
--- a/libclc/generic/lib/math/erfc.cl
+++ b/libclc/generic/lib/math/erfc.cl
@@ -22,8 +22,7 @@
 
 #include <clc/clc.h>
 #include <clc/clcmacro.h>
-
-#include "math.h"
+#include <clc/math/math.h>
 
 /*
  * ====================================================
diff --git a/libclc/generic/lib/math/exp.cl b/libclc/generic/lib/math/exp.cl
index 95dc0db44df76..f71ed07c76b36 100644
--- a/libclc/generic/lib/math/exp.cl
+++ b/libclc/generic/lib/math/exp.cl
@@ -22,8 +22,7 @@
 
 #include <clc/clc.h>
 #include <clc/clcmacro.h>
-
-#include "math.h"
+#include <clc/math/math.h>
 
 _CLC_OVERLOAD _CLC_DEF float exp(float x) {
 
diff --git a/libclc/generic/lib/math/exp2.cl b/libclc/generic/lib/math/exp2.cl
index 8d718311f896f..13c4789797da9 100644
--- a/libclc/generic/lib/math/exp2.cl
+++ b/libclc/generic/lib/math/exp2.cl
@@ -22,8 +22,7 @@
 
 #include <clc/clc.h>
 #include <clc/clcmacro.h>
-
-#include "math.h"
+#include <clc/math/math.h>
 
 _CLC_OVERLOAD _CLC_DEF float exp2(float x) {
 
diff --git a/libclc/generic/lib/math/exp_helper.cl b/libclc/generic/lib/math/exp_helper.cl
index 046f306466bca..ea58d49815393 100644
--- a/libclc/generic/lib/math/exp_helper.cl
+++ b/libclc/generic/lib/math/exp_helper.cl
@@ -21,9 +21,8 @@
  */
 
 #include <clc/clc.h>
-
-#include "math.h"
-#include "tables.h"
+#include <clc/math/math.h>
+#include <clc/math/tables.h>
 
 #ifdef cl_khr_fp64
 
diff --git a/libclc/generic/lib/math/expm1.cl b/libclc/generic/lib/math/expm1.cl
index d50a88e0aaae8..5e2141ae791f5 100644
--- a/libclc/generic/lib/math/expm1.cl
+++ b/libclc/generic/lib/math/expm1.cl
@@ -1,8 +1,7 @@
 #include <clc/clc.h>
 #include <clc/clcmacro.h>
-
-#include "math.h"
-#include "tables.h"
+#include <clc/math/math.h>
+#include <clc/math/tables.h>
 
 /* Refer to the exp routine for the underlying algorithm */
 
diff --git a/libclc/generic/lib/math/fdim.cl b/libclc/generic/lib/math/fdim.cl
index 6899cb7549bc8..4980d6266a518 100644
--- a/libclc/generic/lib/math/fdim.cl
+++ b/libclc/generic/lib/math/fdim.cl
@@ -1,6 +1,5 @@
 #include <clc/clc.h>
-
-#include "math.h"
+#include <clc/math/math.h>
 
 #define __CLC_BODY <fdim.inc>
 #include <clc/math/gentype.inc>
diff --git a/libclc/generic/lib/math/fma.cl b/libclc/generic/lib/math/fma.cl
index 9ad81be696d95..00d5857fb897b 100644
--- a/libclc/generic/lib/math/fma.cl
+++ b/libclc/generic/lib/math/fma.cl
@@ -1,6 +1,6 @@
 #include <clc/clc.h>
+#include <clc/math/math.h>
 
-#include "math.h"
 #include "math/clc_fma.h"
 
 #define __CLC_BODY <fma.inc>
diff --git a/libclc/generic/lib/math/ilogb.cl b/libclc/generic/lib/math/ilogb.cl
index f16b4404fbebe..c9a5014a975db 100644
--- a/libclc/generic/lib/math/ilogb.cl
+++ b/libclc/generic/lib/math/ilogb.cl
@@ -21,9 +21,9 @@
  * THE SOFTWARE.
  */
 
-#include "math.h"
 #include <clc/clc.h>
 #include <clc/clcmacro.h>
+#include <clc/math/math.h>
 
 _CLC_OVERLOAD _CLC_DEF int ilogb(float x) {
     uint ux = as_uint(x);
diff --git a/libclc/generic/lib/math/ldexp.cl b/libclc/generic/lib/math/ldexp.cl
index a999c63902ce8..72708f74b5cc1 100644
--- a/libclc/generic/lib/math/ldexp.cl
+++ b/libclc/generic/lib/math/ldexp.cl
@@ -20,11 +20,11 @@
  * THE SOFTWARE.
  */
 
-#include "config.h"
-#include "math.h"
 #include "math/clc_ldexp.h"
 #include <clc/clc.h>
 #include <clc/clcmacro.h>
+#include <clc/math/clc_subnormal_config.h>
+#include <clc/math/math.h>
 
 _CLC_DEFINE_BINARY_BUILTIN(float, ldexp, __clc_ldexp, float, int)
 
diff --git a/libclc/generic/lib/math/lgamma_r.cl b/libclc/generic/lib/math/lgamma_r.cl
index bd68a76068ec5..cef836ae9747e 100644
--- a/libclc/generic/lib/math/lgamma_r.cl
+++ b/libclc/generic/lib/math/lgamma_r.cl
@@ -23,8 +23,7 @@
 
 #include <clc/clc.h>
 #include <clc/clcmacro.h>
-
-#include "math.h"
+#include <clc/math/math.h>
 
 /*
  * ====================================================
diff --git a/libclc/generic/lib/math/log10.cl b/libclc/generic/lib/math/log10.cl
index 3abb14a28d8ad..38e36d7992bce 100644
--- a/libclc/generic/lib/math/log10.cl
+++ b/libclc/generic/lib/math/log10.cl
@@ -20,7 +20,7 @@
  * THE SOFTWARE.
  */
 
-#include "tables.h"
+#include <clc/math/tables.h>
 #include <clc/clc.h>
 #include <clc/clcmacro.h>
 
diff --git a/libclc/generic/lib/math/log1p.cl b/libclc/generic/lib/math/log1p.cl
index 7fef79ca7b8f3..da4a921e98659 100644
--- a/libclc/generic/lib/math/log1p.cl
+++ b/libclc/generic/lib/math/log1p.cl
@@ -22,9 +22,8 @@
 
 #include <clc/clc.h>
 #include <clc/clcmacro.h>
-
-#include "math.h"
-#include "tables.h"
+#include <clc/math/math.h>
+#include <clc/math/tables.h>
 
 _CLC_OVERLOAD _CLC_DEF float log1p(float x)
 {
diff --git a/libclc/generic/lib/math/log2.cl b/libclc/generic/lib/math/log2.cl
index a6f9692881bb8..c3bcf64a01ab0 100644
--- a/libclc/generic/lib/math/log2.cl
+++ b/libclc/generic/lib/math/log2.cl
@@ -20,7 +20,7 @@
  * THE SOFTWARE.
  */
 
-#include "tables.h"
+#include <clc/math/tables.h>
 #include <clc/clc.h>
 #include <clc/clcmacro.h>
 
diff --git a/libclc/generic/lib/math/log_base.h b/libclc/generic/lib/math/log_base.h
index b8110ca1779a2..bd0169c03236f 100644
--- a/libclc/generic/lib/math/log_base.h
+++ b/libclc/generic/lib/math/log_base.h
@@ -20,7 +20,7 @@
  * THE SOFTWARE.
  */
 
-#include "math.h"
+#include <clc/math/math.h>
 
 /*
    Algorithm:
diff --git a/libclc/generic/lib/math/logb.cl b/libclc/generic/lib/math/logb.cl
index d91972f780ba9..ae3813225265f 100644
--- a/libclc/generic/lib/math/logb.cl
+++ b/libclc/generic/lib/math/logb.cl
@@ -1,6 +1,6 @@
-#include "math.h"
 #include <clc/clc.h>
 #include <clc/clcmacro.h>
+#include <clc/math/math.h>
 
 _CLC_OVERLOAD _CLC_DEF float logb(float x) {
     int ax = as_int(x) & EXSIGNBIT_SP32;
diff --git a/libclc/generic/lib/math/math.h b/libclc/generic/lib/math/math.h
deleted file mode 100644
index d5ef0871e5201..0000000000000
--- a/libclc/generic/lib/math/math.h
+++ /dev/null
@@ -1,117 +0,0 @@
-/*
- * Copyright (c) 2014 Advanced Micro Devices, Inc.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a copy
- * of this software and associated documentation files (the "Software"), to deal
- * in the Software without restriction, including without limitation the rights
- * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
- * copies of the Software, and to permit persons to whom the Software is
- * furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
- * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
- * THE SOFTWARE.
- */
-
-#ifndef __CLC_MATH_H_
-#define __CLC_MATH_H_
-
-#include "clc/clcfunc.h"
-#include "clc/as_type.h"
-#include "config.h"
-
-#define SNAN 0x001
-#define QNAN 0x002
-#define NINF 0x004
-#define NNOR 0x008
-#define NSUB 0x010
-#define NZER 0x020
-#define PZER 0x040
-#define PSUB 0x080
-#define PNOR 0x100
-#define PINF 0x200
-
-#if (defined __AMDGCN__ || defined __R600__) && !defined __HAS_FMAF__
-#define HAVE_HW_FMA32() (0)
-#elif defined(CLC_SPIRV)
-bool __attribute__((noinline)) __clc_runtime_has_hw_fma32(void);
-#define HAVE_HW_FMA32() __clc_runtime_has_hw_fma32()
-#else
-#define HAVE_HW_FMA32() (1)
-#endif
-
-#define HAVE_BITALIGN() (0)
-#define HAVE_FAST_FMA32() (0)
-
-#define MATH_DIVIDE(X, Y) ((X) / (Y))
-#define MATH_RECIP(X) (1.0f / (X))
-#define MATH_SQRT(X) sqrt(X)
-
-#define SIGNBIT_SP32      0x80000000
-#define EXSIGNBIT_SP32    0x7fffffff
-#define EXPBITS_SP32      0x7f800000
-#define MANTBITS_SP32     0x007fffff
-#define ONEEXPBITS_SP32   0x3f800000
-#define TWOEXPBITS_SP32   0x40000000
-#define HALFEXPBITS_SP32  0x3f000000
-#define IMPBIT_SP32       0x00800000
-#define QNANBITPATT_SP32  0x7fc00000
-#define INDEFBITPATT_SP32 0xffc00000
-#define PINFBITPATT_SP32  0x7f800000
-#define NINFBITPATT_SP32  0xff800000
-#define EXPBIAS_SP32      127
-#define EXPSHIFTBITS_SP32 23
-#define BIASEDEMIN_SP32   1
-#define EMIN_SP32         -126
-#define BIASEDEMAX_SP32   254
-#define EMAX_SP32         127
-#define LAMBDA_SP32       1.0e30
-#define MANTLENGTH_SP32   24
-#define BASEDIGITS_SP32   7
-
-_CLC_OVERLOAD _CLC_INLINE float __clc_flush_denormal_if_not_supported(float x)
-{
-	int ix = as_int(x);
-	if (!__clc_fp32_subnormals_supported() &&
-		((ix & EXPBITS_SP32) == 0) && ((ix & MANTBITS_SP32) != 0)) {
-		ix &= SIGNBIT_SP32;
-		x = as_float(ix);
-	}
-	return x;
-}
-
-#ifdef cl_khr_fp64
-
-#define SIGNBIT_DP64      0x8000000000000000L
-#define EXSIGNBIT_DP64    0x7fffffffffffffffL
-#define EXPBITS_DP64      0x7ff0000000000000L
-#define MANTBITS_DP64     0x000fffffffffffffL
-#define ONEEXPBITS_DP64   0x3ff0000000000000L
-#define TWOEXPBITS_DP64   0x4000000000000000L
-#define HALFEXPBITS_DP64  0x3fe0000000000000L
-#define IMPBIT_DP64       0x0010000000000000L
-#define QNANBITPATT_DP64  0x7ff8000000000000L
-#define INDEFBITPATT_DP64 0xfff8000000000000L
-#define PINFBITPATT_DP64  0x7ff0000000000000L
-#define NINFBITPATT_DP64  0xfff0000000000000L
-#define EXPBIAS_DP64      1023
-#define EXPSHIFTBITS_DP64 52
-#define BIASEDEMIN_DP64   1
-#define EMIN_DP64         -1022
-#define BIASEDEMAX_DP64   2046 /* 0x7fe */
-#define EMAX_DP64         1023 /* 0x3ff */
-#define LAMBDA_DP64       1.0e300
-#define MANTLENGTH_DP64   53
-#define BASEDIGITS_DP64   15
-
-#endif // cl_khr_fp64
-
-#define ALIGNED(x)	__attribute__((aligned(x)))
-#endif // __CLC_MATH_H_
diff --git a/libclc/generic/lib/math/modf.cl b/libclc/generic/lib/math/modf.cl
index a462cab2e802b..5098a41d079c5 100644
--- a/libclc/generic/lib/math/modf.cl
+++ b/libclc/generic/lib/math/modf.cl
@@ -21,8 +21,7 @@
  */
 
 #include <clc/clc.h>
-
-#include "math.h"
+#include <clc/math/math.h>
 
 #define __CLC_BODY <modf.inc>
 #include <clc/math/gentype.inc>
diff --git a/libclc/generic/lib/math/sin.cl b/libclc/generic/lib/math/sin.cl
index c271e67828066..9b3d7a9580b14 100644
--- a/libclc/generic/lib/math/sin.cl
+++ b/libclc/generic/lib/math/sin.cl
@@ -20,11 +20,10 @@
  * THE SOFTWARE.
  */
 
+#include "sincos_helpers.h"
 #include <clc/clc.h>
 #include <clc/clcmacro.h>
-
-#include "math.h"
-#include "sincos_helpers.h"
+#include <clc/math/math.h>
 
 _CLC_OVERLOAD _CLC_DEF float sin(float x)
 {
diff --git a/libclc/generic/lib/math/sincos_helpers.cl b/libclc/generic/lib/math/sincos_helpers.cl
index e291e81ed980d..22f2bf61bf27d 100644
--- a/libclc/generic/lib/math/sincos_helpers.cl
+++ b/libclc/generic/lib/math/sincos_helpers.cl
@@ -20,15 +20,16 @@
  * THE SOFTWARE.
  */
 
+#include "sincos_helpers.h"
 #include <clc/clc.h>
+#include <clc/integer/clc_clz.h>
+#include <clc/integer/clc_mul_hi.h>
 #include <clc/math/clc_mad.h>
 #include <clc/math/clc_trunc.h>
+#include <clc/math/math.h>
+#include <clc/math/tables.h>
 #include <clc/shared/clc_max.h>
 
-#include "math.h"
-#include "sincos_helpers.h"
-#include "tables.h"
-
 #define bitalign(hi, lo, shift) ((hi) << (32 - (shift))) | ((lo) >> (shift));
 
 #define bytealign(src0, src1, src2)                                            \
@@ -180,11 +181,11 @@ _CLC_DEF int __clc_argReductionSmallS(float *r, float *rr, float x) {
 
 #define FULL_MUL(A, B, HI, LO)                                                 \
   LO = A * B;                                                                  \
-  HI = mul_hi(A, B)
+  HI = __clc_mul_hi(A, B)
 
 #define FULL_MAD(A, B, C, HI, LO)                                              \
   LO = ((A) * (B) + (C));                                                      \
-  HI = mul_hi(A, B);                                                           \
+  HI = __clc_mul_hi(A, B);                                                     \
   HI += LO < C
 
 _CLC_DEF int __clc_argReductionLargeS(float *r, float *rr, float x) {
@@ -278,7 +279,7 @@ _CLC_DEF int __clc_argReductionLargeS(float *r, float *rr, float x) {
   p5 = p5 ^ flip;
 
   // Find exponent and shift away leading zeroes and hidden bit
-  xe = clz(p7) + 1;
+  xe = __clc_clz(p7) + 1;
   shift = 32 - xe;
   p7 = bitalign(p7, p6, shift);
   p6 = bitalign(p6, p5, shift);
@@ -291,7 +292,7 @@ _CLC_DEF int __clc_argReductionLargeS(float *r, float *rr, float x) {
 
   // Get 24 more bits of fraction in another float, there are not long strings
   // of zeroes here
-  int xxe = clz(p7) + 1;
+  int xxe = __clc_clz(p7) + 1;
   p7 = bitalign(p7, p6, 32 - xxe);
   float q0 = as_float(sign | ((127 - (xe + 23 + xxe)) << 23) | (p7 >> 9));
 
diff --git a/libclc/generic/lib/math/sincos_helpers.h b/libclc/generic/lib/math/sincos_helpers.h
index e307abc48b2df..6dbca73aa2a2e 100644
--- a/libclc/generic/lib/math/sincos_helpers.h
+++ b/libclc/generic/lib/math/sincos_helpers.h
@@ -20,7 +20,8 @@
  * THE SOFTWARE.
  */
 
-#include "clc/clcfunc.h"
+#include <clc/clcfunc.h>
+#include <clc/clctypes.h>
 
 _CLC_DECL float __clc_sinf_piby4(float x, float y);
 _CLC_DECL float __clc_cosf_piby4(float x, float y);
@@ -31,8 +32,10 @@ _CLC_DECL int __clc_argReductionS(float *r, float *rr, float x);
 
 #pragma OPENCL EXTENSION cl_khr_fp64 : enable
 
-_CLC_DECL void __clc_remainder_piby2_medium(double x, double *r, double *rr, int *regn);
-_CLC_DECL void __clc_remainder_piby2_large(double x, double *r, double *rr, int *regn);
+_CLC_DECL void __clc_remainder_piby2_medium(double x, double *r, double *rr,
+                                            int *regn);
+_CLC_DECL void __clc_remainder_piby2_large(double x, double *r, double *rr,
+                                           int *regn);
 _CLC_DECL double2 __clc_sincos_piby4(double x, double xx);
 
 #endif
diff --git a/libclc/generic/lib/math/sinh.cl b/libclc/generic/lib/math/sinh.cl
index 23500c1f49b7a..41a8bffb3a69c 100644
--- a/libclc/generic/lib/math/sinh.cl
+++ b/libclc/generic/lib/math/sinh.cl
@@ -22,9 +22,8 @@
 
 #include <clc/clc.h>
 #include <clc/clcmacro.h>
-
-#include "math.h"
-#include "tables.h"
+#include <clc/math/math.h>
+#include <clc/math/tables.h>
 
 _CLC_OVERLOAD _CLC_DEF float sinh(float x)
 {
diff --git a/libclc/generic/lib/math/sinpi.cl b/libclc/generic/lib/math/sinpi.cl
index 01b340b855c44..81cad9cc17800 100644
--- a/libclc/generic/lib/math/sinpi.cl
+++ b/libclc/generic/lib/math/sinpi.cl
@@ -22,8 +22,8 @@
 
 #include <clc/clc.h>
 #include <clc/clcmacro.h>
+#include <clc/math/math.h>
 
-#include "math.h"
 #include "sincospiF_piby4.h"
 #ifdef cl_khr_fp64
 #include "sincosD_piby4.h"
diff --git a/libclc/generic/lib/math/tables.cl b/libclc/generic/lib/math/tables.cl
index 596487c89e568..8eefbffb234f1 100644
--- a/libclc/generic/lib/math/tables.cl
+++ b/libclc/generic/lib/math/tables.cl
@@ -22,7 +22,7 @@
 
 #include <clc/clc.h>
 
-#include "tables.h"
+#include <clc/math/tables.h>
 
 DECLARE_TABLE(float2, LOGE_TBL, 129) = {
     (float2)(0x0.000000p+0f, 0x0.000000p+0f),
diff --git a/libclc/generic/lib/math/tanh.cl b/libclc/generic/lib/math/tanh.cl
index d9509c57b0507..80eb33fab903e 100644
--- a/libclc/generic/lib/math/tanh.cl
+++ b/libclc/generic/lib/math/tanh.cl
@@ -22,8 +22,7 @@
 
 #include <clc/clc.h>
 #include <clc/clcmacro.h>
-
-#include "math.h"
+#include <clc/math/math.h>
 
 _CLC_OVERLOAD _CLC_DEF float tanh(float x)
 {
diff --git a/libclc/generic/lib/math/tgamma.cl b/libclc/generic/lib/math/tgamma.cl
index 314ffda4ec9d5..76614b575b632 100644
--- a/libclc/generic/lib/math/tgamma.cl
+++ b/libclc/generic/lib/math/tgamma.cl
@@ -23,8 +23,7 @@
 
 #include <clc/clc.h>
 #include <clc/clcmacro.h>
-
-#include "math.h"
+#include <clc/math/math.h>
 
 _CLC_OVERLOAD _CLC_DEF float tgamma(float x) {
     const float pi = 3.1415926535897932384626433832795f;
diff --git a/libclc/generic/lib/subnormal_config.cl b/libclc/generic/lib/subnormal_config.cl
index 4bcecfd82e18a..7bb25cedc1a43 100644
--- a/libclc/generic/lib/subnormal_config.cl
+++ b/libclc/generic/lib/subnormal_config.cl
@@ -21,8 +21,7 @@
  */
 
 #include <clc/clc.h>
-
-#include "config.h"
+#include <clc/math/clc_subnormal_config.h>
 
 _CLC_DEF bool __clc_fp16_subnormals_supported() {
   return false;
diff --git a/libclc/r600/lib/math/fmax.cl b/libclc/r600/lib/math/fmax.cl
index a43530fc7507e..7c258e666b66e 100644
--- a/libclc/r600/lib/math/fmax.cl
+++ b/libclc/r600/lib/math/fmax.cl
@@ -1,7 +1,6 @@
 #include <clc/clc.h>
 #include <clc/clcmacro.h>
-
-#include "../../../generic/lib/math/math.h"
+#include <clc/math/math.h>
 
 _CLC_DEF _CLC_OVERLOAD float fmax(float x, float y)
 {
diff --git a/libclc/r600/lib/math/fmin.cl b/libclc/r600/lib/math/fmin.cl
index a43655da13845..052980dd11156 100644
--- a/libclc/r600/lib/math/fmin.cl
+++ b/libclc/r600/lib/math/fmin.cl
@@ -1,7 +1,6 @@
 #include <clc/clc.h>
 #include <clc/clcmacro.h>
-
-#include "../../../generic/lib/math/math.h"
+#include <clc/math/math.h>
 
 _CLC_DEF _CLC_OVERLOAD float fmin(float x, float y)
 {
diff --git a/libclc/spirv/lib/subnormal_config.cl b/libclc/spirv/lib/subnormal_config.cl
index 167fe1b1a1bcf..3ab5a6394b933 100644
--- a/libclc/spirv/lib/subnormal_config.cl
+++ b/libclc/spirv/lib/subnormal_config.cl
@@ -21,8 +21,7 @@
  */
 
 #include <clc/clc.h>
-
-#include "config.h"
+#include <clc/math/clc_subnormal_config.h>
 
 _CLC_DEF bool __clc_fp16_subnormals_supported() { return false; }
 
diff --git a/libclc/spirv64/lib/SOURCES b/libclc/spirv64/lib/SOURCES
deleted file mode 100644
index 854cba614c8bf..0000000000000
--- a/libclc/spirv64/lib/SOURCES
+++ /dev/null
@@ -1,90 +0,0 @@
-subnormal_config.cl
-../../generic/lib/async/async_work_group_strided_copy.cl
-../../generic/lib/async/wait_group_events.cl
-../../generic/lib/common/degrees.cl
-../../generic/lib/common/mix.cl
-../../generic/lib/common/radians.cl
-../../generic/lib/common/sign.cl
-../../generic/lib/common/smoothstep.cl
-../../generic/lib/common/step.cl
-../../generic/lib/geometric/cross.cl
-../../generic/lib/geometric/distance.cl
-../../generic/lib/geometric/dot.cl
-../../generic/lib/geometric/fast_distance.cl
-../../generic/lib/geometric/fast_length.cl
-../../generic/lib/geometric/fast_normalize.cl
-../../generic/lib/geometric/length.cl
-../../generic/lib/geometric/normalize.cl
-../../generic/lib/integer/rotate.cl
-../../generic/lib/integer/mad_sat.cl
-../../generic/lib/math/acos.cl
-../../generic/lib/math/acosh.cl
-../../generic/lib/math/acospi.cl
-../../generic/lib/math/asin.cl
-../../generic/lib/math/asinh.cl
-../../generic/lib/math/asinpi.cl
-../../generic/lib/math/atan.cl
-../../generic/lib/math/atan2.cl
-../../generic/lib/math/atan2pi.cl
-../../generic/lib/math/atanh.cl
-../../generic/lib/math/atanpi.cl
-../../generic/lib/math/cbrt.cl
-../../generic/lib/math/cos.cl
-../../generic/lib/math/cosh.cl
-../../generic/lib/math/cospi.cl
-../../generic/lib/math/ep_log.cl
-../../generic/lib/math/erf.cl
-../../generic/lib/math/erfc.cl
-../../generic/lib/math/exp.cl
-../../generic/lib/math/exp_helper.cl
-../../generic/lib/math/expm1.cl
-../../generic/lib/math/exp2.cl
-../../generic/lib/math/clc_exp10.cl
-../../generic/lib/math/exp10.cl
-../../generic/lib/math/clc_fma.cl
-math/fma.cl
-../../generic/lib/math/clc_fmod.cl
-../../generic/lib/math/fmod.cl
-../../generic/lib/math/fract.cl
-../../generic/lib/math/frexp.cl
-../../generic/lib/math/half_rsqrt.cl
-../../generic/lib/math/half_sqrt.cl
-../../generic/lib/math/clc_hypot.cl
-../../generic/lib/math/hypot.cl
-../../generic/lib/math/ilogb.cl
-../../generic/lib/math/clc_ldexp.cl
-../../generic/lib/math/ldexp.cl
-../../generic/lib/math/lgamma.cl
-../../generic/lib/math/lgamma_r.cl
-../../generic/lib/math/log.cl
-../../generic/lib/math/log10.cl
-../../generic/lib/math/log1p.cl
-../../generic/lib/math/log2.cl
-../../generic/lib/math/logb.cl
-../../generic/lib/math/modf.cl
-../../generic/lib/math/tables.cl
-../../generic/lib/math/clc_pow.cl
-../../generic/lib/math/pow.cl
-../../generic/lib/math/clc_pown.cl
-../../generic/lib/math/pown.cl
-../../generic/lib/math/clc_powr.cl
-../../generic/lib/math/powr.cl
-../../generic/lib/math/clc_remainder.cl
-../../generic/lib/math/remainder.cl
-../../generic/lib/math/clc_remquo.cl
-../../generic/lib/math/remquo.cl
-../../generic/lib/math/clc_rootn.cl
-../../generic/lib/math/rootn.cl
-../../generic/lib/math/sin.cl
-../../generic/lib/math/sincos.cl
-../../generic/lib/math/sincos_helpers.cl
-../../generic/lib/math/sinh.cl
-../../generic/lib/math/sinpi.cl
-../../generic/lib/math/clc_tan.cl
-../../generic/lib/math/tan.cl
-../../generic/lib/math/tanh.cl
-../../generic/lib/math/clc_tanpi.cl
-../../generic/lib/math/tanpi.cl
-../../generic/lib/math/tgamma.cl
-../../generic/lib/shared/vload.cl
-../../generic/lib/shared/vstore.cl
diff --git a/libclc/spirv64/lib/math/fma.cl b/libclc/spirv64/lib/math/fma.cl
deleted file mode 100644
index 79142425e52d2..0000000000000
--- a/libclc/spirv64/lib/math/fma.cl
+++ /dev/null
@@ -1,11 +0,0 @@
-#include <clc/clc.h>
-#include <math/clc_fma.h>
-
-#define __CLC_BODY <fma.inc>
-#define __FLOAT_ONLY
-#include <clc/math/gentype.inc>
-
-bool __clc_runtime_has_hw_fma32()
-{
-    return false;
-}
diff --git a/libclc/spirv64/lib/math/fma.inc b/libclc/spirv64/lib/math/fma.inc
deleted file mode 100644
index 0f12c565758ff..0000000000000
--- a/libclc/spirv64/lib/math/fma.inc
+++ /dev/null
@@ -1,3 +0,0 @@
-_CLC_DEF _CLC_OVERLOAD __CLC_GENTYPE fma(__CLC_GENTYPE a, __CLC_GENTYPE b, __CLC_GENTYPE c) {
-	return __clc_sw_fma(a, b, c);
-}
diff --git a/libclc/spirv64/lib/subnormal_config.cl b/libclc/spirv64/lib/subnormal_config.cl
deleted file mode 100644
index 167fe1b1a1bcf..0000000000000
--- a/libclc/spirv64/lib/subnormal_config.cl
+++ /dev/null
@@ -1,31 +0,0 @@
-/*
- * Copyright (c) 2015 Advanced Micro Devices, Inc.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a copy
- * of this software and associated documentation files (the "Software"), to deal
- * in the Software without restriction, including without limitation the rights
- * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
- * copies of the Software, and to permit persons to whom the Software is
- * furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
- * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
- * THE SOFTWARE.
- */
-
-#include <clc/clc.h>
-
-#include "config.h"
-
-_CLC_DEF bool __clc_fp16_subnormals_supported() { return false; }
-
-_CLC_DEF bool __clc_fp32_subnormals_supported() { return false; }
-
-_CLC_DEF bool __clc_fp64_subnormals_supported() { return false; }
diff --git a/libcxx/docs/FeatureTestMacroTable.rst b/libcxx/docs/FeatureTestMacroTable.rst
index cfb0e5cfb129c..ccaa784ccb088 100644
--- a/libcxx/docs/FeatureTestMacroTable.rst
+++ b/libcxx/docs/FeatureTestMacroTable.rst
@@ -330,6 +330,10 @@ Status
     ---------------------------------------------------------- -----------------
     ``__cpp_lib_expected``                                     ``202211L``
     ---------------------------------------------------------- -----------------
+    ``__cpp_lib_flat_map``                                     ``202207L``
+    ---------------------------------------------------------- -----------------
+    ``__cpp_lib_flat_set``                                     *unimplemented*
+    ---------------------------------------------------------- -----------------
     ``__cpp_lib_format_ranges``                                ``202207L``
     ---------------------------------------------------------- -----------------
     ``__cpp_lib_formatters``                                   *unimplemented*
diff --git a/libcxx/docs/ReleaseNotes.rst b/libcxx/docs/ReleaseNotes.rst
index 47042c4af1649..c4170fa4b1275 100644
--- a/libcxx/docs/ReleaseNotes.rst
+++ b/libcxx/docs/ReleaseNotes.rst
@@ -1,9 +1,10 @@
-.. include:: ReleaseNotes/20.rst
+.. include:: ReleaseNotes/21.rst
 
 .. Make sure to reference the non-live release notes in a toctree to avoid Sphinx errors.
 .. toctree::
     :hidden:
 
+    ReleaseNotes/20
     ReleaseNotes/19
 
 .. The release notes are in versioned files, but we make sure to keep publishing
diff --git a/libcxx/docs/ReleaseNotes/20.rst b/libcxx/docs/ReleaseNotes/20.rst
index 2736061544c53..57ab0c167544b 100644
--- a/libcxx/docs/ReleaseNotes/20.rst
+++ b/libcxx/docs/ReleaseNotes/20.rst
@@ -35,6 +35,8 @@ see the `releases page <https://llvm.org/releases/>`_.
 What's New in Libc++ 20.0.0?
 ==============================
 
+The main focus of the libc++ team has been to implement new C++20, C++23, and C++26 features.
+
 Implemented Papers
 ------------------
 
@@ -44,7 +46,7 @@ Implemented Papers
 - P2985R0: A type trait for detecting virtual base classes (`Github <https://github.com/llvm/llvm-project/issues/105432>`__)
 - ``std::jthread`` and ``<stop_token>`` are not guarded behind ``-fexperimental-library`` anymore
 - P2674R1: A trait for implicit lifetime types (`Github <https://github.com/llvm/llvm-project/issues/105259>`__)
-- P0429R9: A Standard ``flat_map`` is partially implemented and ``flat_map`` is provided (`Github <https://github.com/llvm/llvm-project/issues/105190>`__)
+- P0429R9: A Standard ``flat_map`` (`Github <https://github.com/llvm/llvm-project/issues/105190>`__)
 
 Improvements and New Features
 -----------------------------
@@ -111,6 +113,13 @@ Improvements and New Features
   std::errc::not_a_directory``, or use ``err.default_error_condition()`` to map to an ``error_condition``, and then test
   its ``value()`` and ``category()``.
 
+- ``std::stable_sort`` uses radix sort for integral types now, which can improve the performance up to 10 times, depending
+  on type of sorted elements and the initial state of the sorted array.
+
+- Reduced the amount of debug information generated for internal typedefs. This reduces the size of debug builds.
+
+- Added :ref:`hardening mode <hardening>` support for ``forward_list`` and ``bitset``.
+
 Deprecations and Removals
 -------------------------
 
@@ -129,10 +138,10 @@ Deprecations and Removals
   supported as an extension anymore, please migrate any code that uses e.g. ``std::vector<const T>`` to be
   standards conforming.
 
-- Non-conforming member typedefs ``base``, ``iterator``, ``const_iterator``, and ``const_reference`` of ``std::bitset``,
-  and member typedef ``base`` of ``std::forward_list`` and ``std::list`` are removed. Previously, these member typedefs
-  (except ``const_reference``) were private but could cause ambiguity in name lookup. Code that expects such ambiguity
-  will possibly not compile in LLVM 20.
+- Non-conforming member typedefs ``base``, ``iterator``, ``const_iterator``, ``size_type``, ``difference_type``, and
+  ``const_reference`` of ``std::bitset``, and member typedef ``base`` of ``std::forward_list`` and ``std::list`` are
+  removed. Previously, these member typedefs (except ``const_reference``) were private but could cause ambiguity in name
+  lookup. Code that expects such ambiguity will possibly not compile in LLVM 20.
 
 - The function ``__libcpp_verbose_abort()`` is now ``noexcept``, to match ``std::terminate()``. (The combination of
   ``noexcept`` and ``[[noreturn]]`` has special significance for function effects analysis.) For backwards compatibility,
@@ -196,8 +205,3 @@ ABI Affecting Changes
 
 - The localization support base API has been reimplemented, leading to different functions being exported from the
   libc++ built library on Windows and Windows-like platforms.
-
-Build System Changes
---------------------
-
-- TODO
diff --git a/libcxx/docs/ReleaseNotes/21.rst b/libcxx/docs/ReleaseNotes/21.rst
new file mode 100644
index 0000000000000..2439360797023
--- /dev/null
+++ b/libcxx/docs/ReleaseNotes/21.rst
@@ -0,0 +1,72 @@
+===========================================
+Libc++ 21.0.0 (In-Progress) Release Notes
+===========================================
+
+.. contents::
+   :local:
+   :depth: 2
+
+Written by the `Libc++ Team <https://libcxx.llvm.org>`_
+
+.. warning::
+
+   These are in-progress notes for the upcoming libc++ 20.0.0 release.
+   Release notes for previous releases can be found on
+   `the Download Page <https://releases.llvm.org/download.html>`_.
+
+Introduction
+============
+
+This document contains the release notes for the libc++ C++ Standard Library,
+part of the LLVM Compiler Infrastructure, release 20.0.0. Here we describe the
+status of libc++ in some detail, including major improvements from the previous
+release and new feature work. For the general LLVM release notes, see `the LLVM
+documentation <https://llvm.org/docs/ReleaseNotes.html>`_. All LLVM releases may
+be downloaded from the `LLVM releases web site <https://llvm.org/releases/>`_.
+
+For more information about libc++, please see the `Libc++ Web Site
+<https://libcxx.llvm.org>`_ or the `LLVM Web Site <https://llvm.org>`_.
+
+Note that if you are reading this file from a Git checkout or the
+main Libc++ web page, this document applies to the *next* release, not
+the current one. To see the release notes for a specific release, please
+see the `releases page <https://llvm.org/releases/>`_.
+
+What's New in Libc++ 21.0.0?
+==============================
+
+Implemented Papers
+------------------
+
+- N4258: Cleaning-up noexcept in the Library (`Github <https://github.com/llvm/llvm-project/issues/99937>`__)
+
+Improvements and New Features
+-----------------------------
+
+- The ``std::ranges::{copy, copy_n, copy_backward}`` algorithms have been optimized for ``std::vector<bool>::iterator``\s,
+  resulting in a performance improvement of up to 2000x.
+
+- Updated formatting library to Unicode 16.0.0.
+
+Deprecations and Removals
+-------------------------
+
+Upcoming Deprecations and Removals
+----------------------------------
+
+LLVM 22
+~~~~~~~
+
+- TODO
+
+
+ABI Affecting Changes
+---------------------
+
+- TODO
+
+
+Build System Changes
+--------------------
+
+- TODO
diff --git a/libcxx/docs/Status/Cxx17Papers.csv b/libcxx/docs/Status/Cxx17Papers.csv
index fbcac452adb8b..24fc7f718c360 100644
--- a/libcxx/docs/Status/Cxx17Papers.csv
+++ b/libcxx/docs/Status/Cxx17Papers.csv
@@ -3,7 +3,7 @@
 "`N4089 <https://wg21.link/N4089>`__","Safe conversions in ``unique_ptr<T[]>``\ .","2014-11 (Urbana)","|Complete|","5",""
 "`N4169 <https://wg21.link/N4169>`__","A proposal to add invoke function template","2014-11 (Urbana)","|Complete|","3.7",""
 "`N4190 <https://wg21.link/N4190>`__","Removing auto_ptr, random_shuffle(), And Old <functional> Stuff.","2014-11 (Urbana)","|Complete|","15",""
-"`N4258 <https://wg21.link/N4258>`__","Cleaning-up noexcept in the Library.","2014-11 (Urbana)","|In Progress|","3.7",""
+"`N4258 <https://wg21.link/N4258>`__","Cleaning-up noexcept in the Library.","2014-11 (Urbana)","|Complete|","21",""
 "`N4259 <https://wg21.link/N4259>`__","Wording for std::uncaught_exceptions","2014-11 (Urbana)","|Complete|","3.7","``std::uncaught_exception`` is deprecated since LLVM 20"
 "`N4277 <https://wg21.link/N4277>`__","TriviallyCopyable ``reference_wrapper``\ .","2014-11 (Urbana)","|Complete|","3.2",""
 "`N4279 <https://wg21.link/N4279>`__","Improved insertion interface for unique-key maps.","2014-11 (Urbana)","|Complete|","3.7",""
diff --git a/libcxx/docs/Status/Cxx23Papers.csv b/libcxx/docs/Status/Cxx23Papers.csv
index 24398574064e6..264c5417a5c28 100644
--- a/libcxx/docs/Status/Cxx23Papers.csv
+++ b/libcxx/docs/Status/Cxx23Papers.csv
@@ -52,7 +52,7 @@
 "`P2443R1 <https://wg21.link/P2443R1>`__","``views::chunk_by``","2022-02 (Virtual)","|Complete|","18",""
 "","","","","",""
 "`P0009R18 <https://wg21.link/P0009R18>`__","mdspan: A Non-Owning Multidimensional Array Reference","2022-07 (Virtual)","|Complete|","18",""
-"`P0429R9 <https://wg21.link/P0429R9>`__","A Standard ``flat_map``","2022-07 (Virtual)","|In Progress|","",""
+"`P0429R9 <https://wg21.link/P0429R9>`__","A Standard ``flat_map``","2022-07 (Virtual)","|Complete|","20",""
 "`P1169R4 <https://wg21.link/P1169R4>`__","``static operator()``","2022-07 (Virtual)","|Complete|","16",""
 "`P1222R4 <https://wg21.link/P1222R4>`__","A Standard ``flat_set``","2022-07 (Virtual)","","",""
 "`P1223R5 <https://wg21.link/P1223R5>`__","``ranges::find_last()``, ``ranges::find_last_if()``, and ``ranges::find_last_if_not()``","2022-07 (Virtual)","|Complete|","19",""
diff --git a/libcxx/docs/Status/FormatPaper.csv b/libcxx/docs/Status/FormatPaper.csv
index 343fa62f13565..6ddae9e2a1518 100644
--- a/libcxx/docs/Status/FormatPaper.csv
+++ b/libcxx/docs/Status/FormatPaper.csv
@@ -2,8 +2,8 @@ Section,Description,Dependencies,Assignee,Status,First released version
 `P1361 <https://wg21.link/P1361>`__ `P2372 <https://wg21.link/P2372>`__,"Formatting chrono"
 `[time.syn] <https://wg21.link/time.syn>`_,"Formatter ``chrono::duration<Rep, Period>``",,Mark de Wever,|Complete|,16
 `[time.syn] <https://wg21.link/time.syn>`_,"Formatter ``chrono::sys_time<Duration>``",,Mark de Wever,|Complete|,17
-`[time.syn] <https://wg21.link/time.syn>`_,"Formatter ``chrono::utc_time<Duration>``",A ``<chrono>`` implementation,Mark de Wever,|Complete|,20
-`[time.syn] <https://wg21.link/time.syn>`_,"Formatter ``chrono::tai_time<Duration>``",A ``<chrono>`` implementation,Mark de Wever,,,
+`[time.syn] <https://wg21.link/time.syn>`_,"Formatter ``chrono::utc_time<Duration>``",,Mark de Wever,|Complete|,20
+`[time.syn] <https://wg21.link/time.syn>`_,"Formatter ``chrono::tai_time<Duration>``",,Mark de Wever,|Complete|,21
 `[time.syn] <https://wg21.link/time.syn>`_,"Formatter ``chrono::gps_time<Duration>``",A ``<chrono>`` implementation,Mark de Wever,,,
 `[time.syn] <https://wg21.link/time.syn>`_,"Formatter ``chrono::file_time<Duration>``",,Mark de Wever,|Complete|,17
 `[time.syn] <https://wg21.link/time.syn>`_,"Formatter ``chrono::local_time<Duration>``",,Mark de Wever,|Complete|,17
diff --git a/libcxx/docs/TestingLibcxx.rst b/libcxx/docs/TestingLibcxx.rst
index e98b96bfb478f..4da7f3e85d291 100644
--- a/libcxx/docs/TestingLibcxx.rst
+++ b/libcxx/docs/TestingLibcxx.rst
@@ -435,6 +435,38 @@ writing tests easier. See `libc++-specific Lit Directives`_ for more information
        extension.)
 
 
+C++ Standard version tests
+~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Historically libc++ tests used to filter the tests for C++ Standard versions
+with lit directives like:
+
+.. code-block:: cpp
+
+   // UNSUPPORTED: c++03, c++11, c++14, c++17, c++20, c++23
+
+With C++ Standards released every 3 years, this solution is not scalable.
+Instead use:
+
+.. code-block:: cpp
+
+   // UNSUPPORTED: std-at-least-c++26
+
+There is no corresponding ``std-at-most-c++23``. This could be useful when
+tests are only valid for a small set of standard versions. For example, a
+deprecation test is only valid when the feature is deprecated until it is
+removed from the Standard. These tests should be written like:
+
+.. code-block:: cpp
+
+   // REQUIRES: c++17 || c++20 || c++23
+
+.. note::
+
+   There are a lot of tests with the first style, these can remain as they are.
+   The new style is only intended to be used for new tests.
+
+
 Benchmarks
 ==========
 
diff --git a/libcxx/include/CMakeLists.txt b/libcxx/include/CMakeLists.txt
index 78d3192542b5a..255a0474c0f6b 100644
--- a/libcxx/include/CMakeLists.txt
+++ b/libcxx/include/CMakeLists.txt
@@ -270,6 +270,7 @@ set(files
   __chrono/steady_clock.h
   __chrono/sys_info.h
   __chrono/system_clock.h
+  __chrono/tai_clock.h
   __chrono/time_point.h
   __chrono/time_zone.h
   __chrono/time_zone_link.h
@@ -362,8 +363,11 @@ set(files
   __filesystem/space_info.h
   __filesystem/u8path.h
   __flat_map/flat_map.h
+  __flat_map/flat_multimap.h
   __flat_map/key_value_iterator.h
+  __flat_map/sorted_equivalent.h
   __flat_map/sorted_unique.h
+  __flat_map/utils.h
   __format/buffer.h
   __format/concepts.h
   __format/container_adaptor.h
@@ -553,12 +557,14 @@ set(files
   __memory/compressed_pair.h
   __memory/concepts.h
   __memory/construct_at.h
+  __memory/destroy.h
   __memory/destruct_n.h
   __memory/inout_ptr.h
   __memory/noexcept_move_assign_container.h
   __memory/out_ptr.h
   __memory/pointer_traits.h
   __memory/ranges_construct_at.h
+  __memory/ranges_destroy.h
   __memory/ranges_uninitialized_algorithms.h
   __memory/raw_storage_iterator.h
   __memory/shared_count.h
diff --git a/libcxx/include/__algorithm/copy.h b/libcxx/include/__algorithm/copy.h
index 962aa90059d57..7454c874a4d93 100644
--- a/libcxx/include/__algorithm/copy.h
+++ b/libcxx/include/__algorithm/copy.h
@@ -13,8 +13,10 @@
 #include <__algorithm/for_each_segment.h>
 #include <__algorithm/min.h>
 #include <__config>
+#include <__fwd/bit_reference.h>
 #include <__iterator/iterator_traits.h>
 #include <__iterator/segmented_iterator.h>
+#include <__memory/pointer_traits.h>
 #include <__type_traits/common_type.h>
 #include <__type_traits/enable_if.h>
 #include <__utility/move.h>
@@ -29,9 +31,129 @@ _LIBCPP_PUSH_MACROS
 
 _LIBCPP_BEGIN_NAMESPACE_STD
 
+template <class _InputIterator, class _OutputIterator>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _OutputIterator
+copy(_InputIterator __first, _InputIterator __last, _OutputIterator __result);
+
 template <class _InIter, class _Sent, class _OutIter>
 inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 pair<_InIter, _OutIter> __copy(_InIter, _Sent, _OutIter);
 
+template <class _Cp, bool _IsConst>
+_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI __bit_iterator<_Cp, false> __copy_aligned(
+    __bit_iterator<_Cp, _IsConst> __first, __bit_iterator<_Cp, _IsConst> __last, __bit_iterator<_Cp, false> __result) {
+  using _In             = __bit_iterator<_Cp, _IsConst>;
+  using difference_type = typename _In::difference_type;
+  using __storage_type  = typename _In::__storage_type;
+
+  const int __bits_per_word = _In::__bits_per_word;
+  difference_type __n       = __last - __first;
+  if (__n > 0) {
+    // do first word
+    if (__first.__ctz_ != 0) {
+      unsigned __clz       = __bits_per_word - __first.__ctz_;
+      difference_type __dn = std::min(static_cast<difference_type>(__clz), __n);
+      __n -= __dn;
+      __storage_type __m = (~__storage_type(0) << __first.__ctz_) & (~__storage_type(0) >> (__clz - __dn));
+      __storage_type __b = *__first.__seg_ & __m;
+      *__result.__seg_ &= ~__m;
+      *__result.__seg_ |= __b;
+      __result.__seg_ += (__dn + __result.__ctz_) / __bits_per_word;
+      __result.__ctz_ = static_cast<unsigned>((__dn + __result.__ctz_) % __bits_per_word);
+      ++__first.__seg_;
+      // __first.__ctz_ = 0;
+    }
+    // __first.__ctz_ == 0;
+    // do middle words
+    __storage_type __nw = __n / __bits_per_word;
+    std::copy(std::__to_address(__first.__seg_),
+              std::__to_address(__first.__seg_ + __nw),
+              std::__to_address(__result.__seg_));
+    __n -= __nw * __bits_per_word;
+    __result.__seg_ += __nw;
+    // do last word
+    if (__n > 0) {
+      __first.__seg_ += __nw;
+      __storage_type __m = ~__storage_type(0) >> (__bits_per_word - __n);
+      __storage_type __b = *__first.__seg_ & __m;
+      *__result.__seg_ &= ~__m;
+      *__result.__seg_ |= __b;
+      __result.__ctz_ = static_cast<unsigned>(__n);
+    }
+  }
+  return __result;
+}
+
+template <class _Cp, bool _IsConst>
+_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI __bit_iterator<_Cp, false> __copy_unaligned(
+    __bit_iterator<_Cp, _IsConst> __first, __bit_iterator<_Cp, _IsConst> __last, __bit_iterator<_Cp, false> __result) {
+  using _In             = __bit_iterator<_Cp, _IsConst>;
+  using difference_type = typename _In::difference_type;
+  using __storage_type  = typename _In::__storage_type;
+
+  const int __bits_per_word = _In::__bits_per_word;
+  difference_type __n       = __last - __first;
+  if (__n > 0) {
+    // do first word
+    if (__first.__ctz_ != 0) {
+      unsigned __clz_f     = __bits_per_word - __first.__ctz_;
+      difference_type __dn = std::min(static_cast<difference_type>(__clz_f), __n);
+      __n -= __dn;
+      __storage_type __m   = (~__storage_type(0) << __first.__ctz_) & (~__storage_type(0) >> (__clz_f - __dn));
+      __storage_type __b   = *__first.__seg_ & __m;
+      unsigned __clz_r     = __bits_per_word - __result.__ctz_;
+      __storage_type __ddn = std::min<__storage_type>(__dn, __clz_r);
+      __m                  = (~__storage_type(0) << __result.__ctz_) & (~__storage_type(0) >> (__clz_r - __ddn));
+      *__result.__seg_ &= ~__m;
+      if (__result.__ctz_ > __first.__ctz_)
+        *__result.__seg_ |= __b << (__result.__ctz_ - __first.__ctz_);
+      else
+        *__result.__seg_ |= __b >> (__first.__ctz_ - __result.__ctz_);
+      __result.__seg_ += (__ddn + __result.__ctz_) / __bits_per_word;
+      __result.__ctz_ = static_cast<unsigned>((__ddn + __result.__ctz_) % __bits_per_word);
+      __dn -= __ddn;
+      if (__dn > 0) {
+        __m = ~__storage_type(0) >> (__bits_per_word - __dn);
+        *__result.__seg_ &= ~__m;
+        *__result.__seg_ |= __b >> (__first.__ctz_ + __ddn);
+        __result.__ctz_ = static_cast<unsigned>(__dn);
+      }
+      ++__first.__seg_;
+      // __first.__ctz_ = 0;
+    }
+    // __first.__ctz_ == 0;
+    // do middle words
+    unsigned __clz_r   = __bits_per_word - __result.__ctz_;
+    __storage_type __m = ~__storage_type(0) << __result.__ctz_;
+    for (; __n >= __bits_per_word; __n -= __bits_per_word, ++__first.__seg_) {
+      __storage_type __b = *__first.__seg_;
+      *__result.__seg_ &= ~__m;
+      *__result.__seg_ |= __b << __result.__ctz_;
+      ++__result.__seg_;
+      *__result.__seg_ &= __m;
+      *__result.__seg_ |= __b >> __clz_r;
+    }
+    // do last word
+    if (__n > 0) {
+      __m                 = ~__storage_type(0) >> (__bits_per_word - __n);
+      __storage_type __b  = *__first.__seg_ & __m;
+      __storage_type __dn = std::min(__n, static_cast<difference_type>(__clz_r));
+      __m                 = (~__storage_type(0) << __result.__ctz_) & (~__storage_type(0) >> (__clz_r - __dn));
+      *__result.__seg_ &= ~__m;
+      *__result.__seg_ |= __b << __result.__ctz_;
+      __result.__seg_ += (__dn + __result.__ctz_) / __bits_per_word;
+      __result.__ctz_ = static_cast<unsigned>((__dn + __result.__ctz_) % __bits_per_word);
+      __n -= __dn;
+      if (__n > 0) {
+        __m = ~__storage_type(0) >> (__bits_per_word - __n);
+        *__result.__seg_ &= ~__m;
+        *__result.__seg_ |= __b >> __dn;
+        __result.__ctz_ = static_cast<unsigned>(__n);
+      }
+    }
+  }
+  return __result;
+}
+
 struct __copy_impl {
   template <class _InIter, class _Sent, class _OutIter>
   _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 pair<_InIter, _OutIter>
@@ -95,6 +217,16 @@ struct __copy_impl {
     }
   }
 
+  template <class _Cp, bool _IsConst>
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 pair<__bit_iterator<_Cp, _IsConst>, __bit_iterator<_Cp, false> >
+  operator()(__bit_iterator<_Cp, _IsConst> __first,
+             __bit_iterator<_Cp, _IsConst> __last,
+             __bit_iterator<_Cp, false> __result) const {
+    if (__first.__ctz_ == __result.__ctz_)
+      return std::make_pair(__last, std::__copy_aligned(__first, __last, __result));
+    return std::make_pair(__last, std::__copy_unaligned(__first, __last, __result));
+  }
+
   // At this point, the iterators have been unwrapped so any `contiguous_iterator` has been unwrapped to a pointer.
   template <class _In, class _Out, __enable_if_t<__can_lower_copy_assignment_to_memmove<_In, _Out>::value, int> = 0>
   _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 pair<_In*, _Out*>
@@ -110,7 +242,7 @@ __copy(_InIter __first, _Sent __last, _OutIter __result) {
 }
 
 template <class _InputIterator, class _OutputIterator>
-inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _OutputIterator
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _OutputIterator
 copy(_InputIterator __first, _InputIterator __last, _OutputIterator __result) {
   return std::__copy(__first, __last, __result).second;
 }
diff --git a/libcxx/include/__algorithm/copy_backward.h b/libcxx/include/__algorithm/copy_backward.h
index 48a768f577f55..02ffc14361e6a 100644
--- a/libcxx/include/__algorithm/copy_backward.h
+++ b/libcxx/include/__algorithm/copy_backward.h
@@ -10,11 +10,14 @@
 #define _LIBCPP___ALGORITHM_COPY_BACKWARD_H
 
 #include <__algorithm/copy_move_common.h>
+#include <__algorithm/copy_n.h>
 #include <__algorithm/iterator_operations.h>
 #include <__algorithm/min.h>
 #include <__config>
+#include <__fwd/bit_reference.h>
 #include <__iterator/iterator_traits.h>
 #include <__iterator/segmented_iterator.h>
+#include <__memory/pointer_traits.h>
 #include <__type_traits/common_type.h>
 #include <__type_traits/enable_if.h>
 #include <__type_traits/is_constructible.h>
@@ -34,6 +37,124 @@ template <class _AlgPolicy, class _InIter, class _Sent, class _OutIter>
 _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 pair<_InIter, _OutIter>
 __copy_backward(_InIter __first, _Sent __last, _OutIter __result);
 
+template <class _Cp, bool _IsConst>
+_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI __bit_iterator<_Cp, false> __copy_backward_aligned(
+    __bit_iterator<_Cp, _IsConst> __first, __bit_iterator<_Cp, _IsConst> __last, __bit_iterator<_Cp, false> __result) {
+  using _In             = __bit_iterator<_Cp, _IsConst>;
+  using difference_type = typename _In::difference_type;
+  using __storage_type  = typename _In::__storage_type;
+
+  const int __bits_per_word = _In::__bits_per_word;
+  difference_type __n       = __last - __first;
+  if (__n > 0) {
+    // do first word
+    if (__last.__ctz_ != 0) {
+      difference_type __dn = std::min(static_cast<difference_type>(__last.__ctz_), __n);
+      __n -= __dn;
+      unsigned __clz     = __bits_per_word - __last.__ctz_;
+      __storage_type __m = (~__storage_type(0) << (__last.__ctz_ - __dn)) & (~__storage_type(0) >> __clz);
+      __storage_type __b = *__last.__seg_ & __m;
+      *__result.__seg_ &= ~__m;
+      *__result.__seg_ |= __b;
+      __result.__ctz_ = static_cast<unsigned>(((-__dn & (__bits_per_word - 1)) + __result.__ctz_) % __bits_per_word);
+      // __last.__ctz_ = 0
+    }
+    // __last.__ctz_ == 0 || __n == 0
+    // __result.__ctz_ == 0 || __n == 0
+    // do middle words
+    __storage_type __nw = __n / __bits_per_word;
+    __result.__seg_ -= __nw;
+    __last.__seg_ -= __nw;
+    std::copy_n(std::__to_address(__last.__seg_), __nw, std::__to_address(__result.__seg_));
+    __n -= __nw * __bits_per_word;
+    // do last word
+    if (__n > 0) {
+      __storage_type __m = ~__storage_type(0) << (__bits_per_word - __n);
+      __storage_type __b = *--__last.__seg_ & __m;
+      *--__result.__seg_ &= ~__m;
+      *__result.__seg_ |= __b;
+      __result.__ctz_ = static_cast<unsigned>(-__n & (__bits_per_word - 1));
+    }
+  }
+  return __result;
+}
+
+template <class _Cp, bool _IsConst>
+_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI __bit_iterator<_Cp, false> __copy_backward_unaligned(
+    __bit_iterator<_Cp, _IsConst> __first, __bit_iterator<_Cp, _IsConst> __last, __bit_iterator<_Cp, false> __result) {
+  using _In             = __bit_iterator<_Cp, _IsConst>;
+  using difference_type = typename _In::difference_type;
+  using __storage_type  = typename _In::__storage_type;
+
+  const int __bits_per_word = _In::__bits_per_word;
+  difference_type __n       = __last - __first;
+  if (__n > 0) {
+    // do first word
+    if (__last.__ctz_ != 0) {
+      difference_type __dn = std::min(static_cast<difference_type>(__last.__ctz_), __n);
+      __n -= __dn;
+      unsigned __clz_l     = __bits_per_word - __last.__ctz_;
+      __storage_type __m   = (~__storage_type(0) << (__last.__ctz_ - __dn)) & (~__storage_type(0) >> __clz_l);
+      __storage_type __b   = *__last.__seg_ & __m;
+      unsigned __clz_r     = __bits_per_word - __result.__ctz_;
+      __storage_type __ddn = std::min(__dn, static_cast<difference_type>(__result.__ctz_));
+      if (__ddn > 0) {
+        __m = (~__storage_type(0) << (__result.__ctz_ - __ddn)) & (~__storage_type(0) >> __clz_r);
+        *__result.__seg_ &= ~__m;
+        if (__result.__ctz_ > __last.__ctz_)
+          *__result.__seg_ |= __b << (__result.__ctz_ - __last.__ctz_);
+        else
+          *__result.__seg_ |= __b >> (__last.__ctz_ - __result.__ctz_);
+        __result.__ctz_ = static_cast<unsigned>(((-__ddn & (__bits_per_word - 1)) + __result.__ctz_) % __bits_per_word);
+        __dn -= __ddn;
+      }
+      if (__dn > 0) {
+        // __result.__ctz_ == 0
+        --__result.__seg_;
+        __result.__ctz_ = static_cast<unsigned>(-__dn & (__bits_per_word - 1));
+        __m             = ~__storage_type(0) << __result.__ctz_;
+        *__result.__seg_ &= ~__m;
+        __last.__ctz_ -= __dn + __ddn;
+        *__result.__seg_ |= __b << (__result.__ctz_ - __last.__ctz_);
+      }
+      // __last.__ctz_ = 0
+    }
+    // __last.__ctz_ == 0 || __n == 0
+    // __result.__ctz_ != 0 || __n == 0
+    // do middle words
+    unsigned __clz_r   = __bits_per_word - __result.__ctz_;
+    __storage_type __m = ~__storage_type(0) >> __clz_r;
+    for (; __n >= __bits_per_word; __n -= __bits_per_word) {
+      __storage_type __b = *--__last.__seg_;
+      *__result.__seg_ &= ~__m;
+      *__result.__seg_ |= __b >> __clz_r;
+      *--__result.__seg_ &= __m;
+      *__result.__seg_ |= __b << __result.__ctz_;
+    }
+    // do last word
+    if (__n > 0) {
+      __m                 = ~__storage_type(0) << (__bits_per_word - __n);
+      __storage_type __b  = *--__last.__seg_ & __m;
+      __clz_r             = __bits_per_word - __result.__ctz_;
+      __storage_type __dn = std::min(__n, static_cast<difference_type>(__result.__ctz_));
+      __m                 = (~__storage_type(0) << (__result.__ctz_ - __dn)) & (~__storage_type(0) >> __clz_r);
+      *__result.__seg_ &= ~__m;
+      *__result.__seg_ |= __b >> (__bits_per_word - __result.__ctz_);
+      __result.__ctz_ = static_cast<unsigned>(((-__dn & (__bits_per_word - 1)) + __result.__ctz_) % __bits_per_word);
+      __n -= __dn;
+      if (__n > 0) {
+        // __result.__ctz_ == 0
+        --__result.__seg_;
+        __result.__ctz_ = static_cast<unsigned>(-__n & (__bits_per_word - 1));
+        __m             = ~__storage_type(0) << __result.__ctz_;
+        *__result.__seg_ &= ~__m;
+        *__result.__seg_ |= __b << (__result.__ctz_ - (__bits_per_word - __n - __dn));
+      }
+    }
+  }
+  return __result;
+}
+
 template <class _AlgPolicy>
 struct __copy_backward_impl {
   template <class _InIter, class _Sent, class _OutIter>
@@ -107,6 +228,16 @@ struct __copy_backward_impl {
     }
   }
 
+  template <class _Cp, bool _IsConst>
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 pair<__bit_iterator<_Cp, _IsConst>, __bit_iterator<_Cp, false> >
+  operator()(__bit_iterator<_Cp, _IsConst> __first,
+             __bit_iterator<_Cp, _IsConst> __last,
+             __bit_iterator<_Cp, false> __result) {
+    if (__last.__ctz_ == __result.__ctz_)
+      return std::make_pair(__last, std::__copy_backward_aligned(__first, __last, __result));
+    return std::make_pair(__last, std::__copy_backward_unaligned(__first, __last, __result));
+  }
+
   // At this point, the iterators have been unwrapped so any `contiguous_iterator` has been unwrapped to a pointer.
   template <class _In, class _Out, __enable_if_t<__can_lower_copy_assignment_to_memmove<_In, _Out>::value, int> = 0>
   _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 pair<_In*, _Out*>
diff --git a/libcxx/include/__algorithm/fill_n.h b/libcxx/include/__algorithm/fill_n.h
index a7e01c45b9222..0da78e1f38c4c 100644
--- a/libcxx/include/__algorithm/fill_n.h
+++ b/libcxx/include/__algorithm/fill_n.h
@@ -41,11 +41,7 @@ __fill_n_bool(__bit_iterator<_Cp, false> __first, typename __size_difference_typ
   if (__first.__ctz_ != 0) {
     __storage_type __clz_f = static_cast<__storage_type>(__bits_per_word - __first.__ctz_);
     __storage_type __dn    = std::min(__clz_f, __n);
-    __storage_type __m     = (~__storage_type(0) << __first.__ctz_) & (~__storage_type(0) >> (__clz_f - __dn));
-    if (_FillVal)
-      *__first.__seg_ |= __m;
-    else
-      *__first.__seg_ &= ~__m;
+    std::__fill_masked_range(std::__to_address(__first.__seg_), __clz_f - __dn, __first.__ctz_, _FillVal);
     __n -= __dn;
     ++__first.__seg_;
   }
@@ -56,11 +52,7 @@ __fill_n_bool(__bit_iterator<_Cp, false> __first, typename __size_difference_typ
   // do last partial word
   if (__n > 0) {
     __first.__seg_ += __nw;
-    __storage_type __m = ~__storage_type(0) >> (__bits_per_word - __n);
-    if (_FillVal)
-      *__first.__seg_ |= __m;
-    else
-      *__first.__seg_ &= ~__m;
+    std::__fill_masked_range(std::__to_address(__first.__seg_), __bits_per_word - __n, 0u, _FillVal);
   }
 }
 
diff --git a/libcxx/include/__algorithm/radix_sort.h b/libcxx/include/__algorithm/radix_sort.h
index de6927995e74c..f6d9fb1ad7ca9 100644
--- a/libcxx/include/__algorithm/radix_sort.h
+++ b/libcxx/include/__algorithm/radix_sort.h
@@ -33,6 +33,7 @@
 #include <__bit/countl.h>
 #include <__config>
 #include <__functional/identity.h>
+#include <__iterator/access.h>
 #include <__iterator/distance.h>
 #include <__iterator/iterator_traits.h>
 #include <__iterator/move_iterator.h>
@@ -67,7 +68,7 @@ _LIBCPP_BEGIN_NAMESPACE_STD
 #if _LIBCPP_STD_VER >= 14
 
 template <class _InputIterator, class _OutputIterator>
-_LIBCPP_HIDE_FROM_ABI pair<_OutputIterator, __iter_value_type<_InputIterator>>
+_LIBCPP_HIDE_FROM_ABI constexpr pair<_OutputIterator, __iter_value_type<_InputIterator>>
 __partial_sum_max(_InputIterator __first, _InputIterator __last, _OutputIterator __result) {
   if (__first == __last)
     return {__result, 0};
@@ -109,7 +110,7 @@ struct __counting_sort_traits {
 };
 
 template <class _Radix, class _Integer>
-_LIBCPP_HIDE_FROM_ABI auto __nth_radix(size_t __radix_number, _Radix __radix, _Integer __n) {
+_LIBCPP_HIDE_FROM_ABI constexpr auto __nth_radix(size_t __radix_number, _Radix __radix, _Integer __n) {
   static_assert(is_unsigned<_Integer>::value);
   using __traits = __counting_sort_traits<_Integer, _Radix>;
 
@@ -117,7 +118,7 @@ _LIBCPP_HIDE_FROM_ABI auto __nth_radix(size_t __radix_number, _Radix __radix, _I
 }
 
 template <class _ForwardIterator, class _Map, class _RandomAccessIterator>
-_LIBCPP_HIDE_FROM_ABI void
+_LIBCPP_HIDE_FROM_ABI constexpr void
 __collect(_ForwardIterator __first, _ForwardIterator __last, _Map __map, _RandomAccessIterator __counters) {
   using __value_type = __iter_value_type<_ForwardIterator>;
   using __traits     = __counting_sort_traits<__value_type, _Map>;
@@ -129,7 +130,7 @@ __collect(_ForwardIterator __first, _ForwardIterator __last, _Map __map, _Random
 }
 
 template <class _ForwardIterator, class _RandomAccessIterator1, class _Map, class _RandomAccessIterator2>
-_LIBCPP_HIDE_FROM_ABI void
+_LIBCPP_HIDE_FROM_ABI constexpr void
 __dispose(_ForwardIterator __first,
           _ForwardIterator __last,
           _RandomAccessIterator1 __result,
@@ -147,7 +148,7 @@ template <class _ForwardIterator,
           class _RandomAccessIterator1,
           class _RandomAccessIterator2,
           size_t... _Radices>
-_LIBCPP_HIDE_FROM_ABI bool __collect_impl(
+_LIBCPP_HIDE_FROM_ABI constexpr bool __collect_impl(
     _ForwardIterator __first,
     _ForwardIterator __last,
     _Map __map,
@@ -177,7 +178,7 @@ _LIBCPP_HIDE_FROM_ABI bool __collect_impl(
 }
 
 template <class _ForwardIterator, class _Map, class _Radix, class _RandomAccessIterator1, class _RandomAccessIterator2>
-_LIBCPP_HIDE_FROM_ABI bool
+_LIBCPP_HIDE_FROM_ABI constexpr bool
 __collect(_ForwardIterator __first,
           _ForwardIterator __last,
           _Map __map,
@@ -191,7 +192,7 @@ __collect(_ForwardIterator __first,
 }
 
 template <class _BidirectionalIterator, class _RandomAccessIterator1, class _Map, class _RandomAccessIterator2>
-_LIBCPP_HIDE_FROM_ABI void __dispose_backward(
+_LIBCPP_HIDE_FROM_ABI constexpr void __dispose_backward(
     _BidirectionalIterator __first,
     _BidirectionalIterator __last,
     _RandomAccessIterator1 __result,
@@ -206,7 +207,7 @@ _LIBCPP_HIDE_FROM_ABI void __dispose_backward(
 }
 
 template <class _ForwardIterator, class _RandomAccessIterator, class _Map>
-_LIBCPP_HIDE_FROM_ABI _RandomAccessIterator
+_LIBCPP_HIDE_FROM_ABI constexpr _RandomAccessIterator
 __counting_sort_impl(_ForwardIterator __first, _ForwardIterator __last, _RandomAccessIterator __result, _Map __map) {
   using __value_type = __iter_value_type<_ForwardIterator>;
   using __traits     = __counting_sort_traits<__value_type, _Map>;
@@ -225,7 +226,7 @@ template <class _RandomAccessIterator1,
           class _Radix,
           enable_if_t< __radix_sort_traits<__iter_value_type<_RandomAccessIterator1>, _Map, _Radix>::__radix_count == 1,
                        int> = 0>
-_LIBCPP_HIDE_FROM_ABI void __radix_sort_impl(
+_LIBCPP_HIDE_FROM_ABI constexpr void __radix_sort_impl(
     _RandomAccessIterator1 __first,
     _RandomAccessIterator1 __last,
     _RandomAccessIterator2 __buffer,
@@ -245,7 +246,7 @@ template <
     class _Radix,
     enable_if_t< __radix_sort_traits<__iter_value_type<_RandomAccessIterator1>, _Map, _Radix>::__radix_count % 2 == 0,
                  int> = 0 >
-_LIBCPP_HIDE_FROM_ABI void __radix_sort_impl(
+_LIBCPP_HIDE_FROM_ABI constexpr void __radix_sort_impl(
     _RandomAccessIterator1 __first,
     _RandomAccessIterator1 __last,
     _RandomAccessIterator2 __buffer_begin,
@@ -307,7 +308,7 @@ struct __low_byte_fn {
 };
 
 template <class _RandomAccessIterator1, class _RandomAccessIterator2, class _Map, class _Radix>
-_LIBCPP_HIDE_FROM_ABI void
+_LIBCPP_HIDE_FROM_ABI constexpr void
 __radix_sort(_RandomAccessIterator1 __first,
              _RandomAccessIterator1 __last,
              _RandomAccessIterator2 __buffer,
@@ -318,7 +319,7 @@ __radix_sort(_RandomAccessIterator1 __first,
 }
 
 template <class _RandomAccessIterator1, class _RandomAccessIterator2>
-_LIBCPP_HIDE_FROM_ABI void
+_LIBCPP_HIDE_FROM_ABI constexpr void
 __radix_sort(_RandomAccessIterator1 __first, _RandomAccessIterator1 __last, _RandomAccessIterator2 __buffer) {
   std::__radix_sort(__first, __last, __buffer, __identity{}, __low_byte_fn{});
 }
diff --git a/libcxx/include/__algorithm/stable_sort.h b/libcxx/include/__algorithm/stable_sort.h
index 3cfbcf08d2c5c..49d3cca2526ea 100644
--- a/libcxx/include/__algorithm/stable_sort.h
+++ b/libcxx/include/__algorithm/stable_sort.h
@@ -25,6 +25,7 @@
 #include <__memory/unique_temporary_buffer.h>
 #include <__type_traits/desugars_to.h>
 #include <__type_traits/enable_if.h>
+#include <__type_traits/is_constant_evaluated.h>
 #include <__type_traits/is_integral.h>
 #include <__type_traits/is_same.h>
 #include <__type_traits/is_trivially_assignable.h>
@@ -253,6 +254,12 @@ _LIBCPP_CONSTEXPR_SINCE_CXX26 void __stable_sort(
   if constexpr (__allowed_radix_sort) {
     if (__len <= __buff_size && __len >= static_cast<difference_type>(__radix_sort_min_bound<value_type>()) &&
         __len <= static_cast<difference_type>(__radix_sort_max_bound<value_type>())) {
+      if (__libcpp_is_constant_evaluated()) {
+        for (auto* __p = __buff; __p < __buff + __buff_size; ++__p) {
+          std::__construct_at(__p);
+        }
+      }
+
       std::__radix_sort(__first, __last, __buff);
       return;
     }
diff --git a/libcxx/include/__bit_reference b/libcxx/include/__bit_reference
index 67abb023122ed..aad470394732c 100644
--- a/libcxx/include/__bit_reference
+++ b/libcxx/include/__bit_reference
@@ -10,8 +10,11 @@
 #ifndef _LIBCPP___BIT_REFERENCE
 #define _LIBCPP___BIT_REFERENCE
 
+#include <__algorithm/copy.h>
+#include <__algorithm/copy_backward.h>
 #include <__algorithm/copy_n.h>
 #include <__algorithm/min.h>
+#include <__assert>
 #include <__bit/countr.h>
 #include <__compare/ordering.h>
 #include <__config>
@@ -22,9 +25,13 @@
 #include <__memory/construct_at.h>
 #include <__memory/pointer_traits.h>
 #include <__type_traits/conditional.h>
+#include <__type_traits/enable_if.h>
 #include <__type_traits/is_constant_evaluated.h>
+#include <__type_traits/is_unsigned.h>
 #include <__type_traits/void_t.h>
+#include <__utility/pair.h>
 #include <__utility/swap.h>
+#include <climits>
 
 #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
 #  pragma GCC system_header
@@ -55,6 +62,26 @@ struct __size_difference_type_traits<_Cp, __void_t<typename _Cp::difference_type
   using size_type       = typename _Cp::size_type;
 };
 
+// This function is designed to operate correctly even for smaller integral types like `uint8_t`, `uint16_t`,
+// or `unsigned short`. Casting back to _StorageType is crucial to prevent undefined behavior that can arise
+// from integral promotions.
+// See https://github.com/llvm/llvm-project/pull/122410.
+template <class _StoragePointer>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 void
+__fill_masked_range(_StoragePointer __word, unsigned __clz, unsigned __ctz, bool __fill_val) {
+  static_assert(is_unsigned<typename pointer_traits<_StoragePointer>::element_type>::value,
+                "__fill_masked_range must be called with unsigned type");
+  using _StorageType = typename pointer_traits<_StoragePointer>::element_type;
+  _LIBCPP_ASSERT_VALID_INPUT_RANGE(
+      __ctz + __clz < sizeof(_StorageType) * CHAR_BIT, "__fill_masked_range called with invalid range");
+  _StorageType __m = static_cast<_StorageType>(static_cast<_StorageType>(~static_cast<_StorageType>(0)) >> __clz) &
+                     static_cast<_StorageType>(static_cast<_StorageType>(~static_cast<_StorageType>(0)) << __ctz);
+  if (__fill_val)
+    *__word |= __m;
+  else
+    *__word &= static_cast<_StorageType>(~__m);
+}
+
 template <class _Cp, bool = __has_storage_type<_Cp>::value>
 class __bit_reference {
   using __storage_type _LIBCPP_NODEBUG    = typename _Cp::__storage_type;
@@ -183,258 +210,6 @@ private:
         __mask_(__m) {}
 };
 
-// copy
-
-template <class _Cp, bool _IsConst>
-_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI __bit_iterator<_Cp, false> __copy_aligned(
-    __bit_iterator<_Cp, _IsConst> __first, __bit_iterator<_Cp, _IsConst> __last, __bit_iterator<_Cp, false> __result) {
-  using _In             = __bit_iterator<_Cp, _IsConst>;
-  using difference_type = typename _In::difference_type;
-  using __storage_type  = typename _In::__storage_type;
-
-  const int __bits_per_word = _In::__bits_per_word;
-  difference_type __n       = __last - __first;
-  if (__n > 0) {
-    // do first word
-    if (__first.__ctz_ != 0) {
-      unsigned __clz       = __bits_per_word - __first.__ctz_;
-      difference_type __dn = std::min(static_cast<difference_type>(__clz), __n);
-      __n -= __dn;
-      __storage_type __m = (~__storage_type(0) << __first.__ctz_) & (~__storage_type(0) >> (__clz - __dn));
-      __storage_type __b = *__first.__seg_ & __m;
-      *__result.__seg_ &= ~__m;
-      *__result.__seg_ |= __b;
-      __result.__seg_ += (__dn + __result.__ctz_) / __bits_per_word;
-      __result.__ctz_ = static_cast<unsigned>((__dn + __result.__ctz_) % __bits_per_word);
-      ++__first.__seg_;
-      // __first.__ctz_ = 0;
-    }
-    // __first.__ctz_ == 0;
-    // do middle words
-    __storage_type __nw = __n / __bits_per_word;
-    std::copy_n(std::__to_address(__first.__seg_), __nw, std::__to_address(__result.__seg_));
-    __n -= __nw * __bits_per_word;
-    __result.__seg_ += __nw;
-    // do last word
-    if (__n > 0) {
-      __first.__seg_ += __nw;
-      __storage_type __m = ~__storage_type(0) >> (__bits_per_word - __n);
-      __storage_type __b = *__first.__seg_ & __m;
-      *__result.__seg_ &= ~__m;
-      *__result.__seg_ |= __b;
-      __result.__ctz_ = static_cast<unsigned>(__n);
-    }
-  }
-  return __result;
-}
-
-template <class _Cp, bool _IsConst>
-_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI __bit_iterator<_Cp, false> __copy_unaligned(
-    __bit_iterator<_Cp, _IsConst> __first, __bit_iterator<_Cp, _IsConst> __last, __bit_iterator<_Cp, false> __result) {
-  using _In             = __bit_iterator<_Cp, _IsConst>;
-  using difference_type = typename _In::difference_type;
-  using __storage_type  = typename _In::__storage_type;
-
-  const int __bits_per_word = _In::__bits_per_word;
-  difference_type __n       = __last - __first;
-  if (__n > 0) {
-    // do first word
-    if (__first.__ctz_ != 0) {
-      unsigned __clz_f     = __bits_per_word - __first.__ctz_;
-      difference_type __dn = std::min(static_cast<difference_type>(__clz_f), __n);
-      __n -= __dn;
-      __storage_type __m   = (~__storage_type(0) << __first.__ctz_) & (~__storage_type(0) >> (__clz_f - __dn));
-      __storage_type __b   = *__first.__seg_ & __m;
-      unsigned __clz_r     = __bits_per_word - __result.__ctz_;
-      __storage_type __ddn = std::min<__storage_type>(__dn, __clz_r);
-      __m                  = (~__storage_type(0) << __result.__ctz_) & (~__storage_type(0) >> (__clz_r - __ddn));
-      *__result.__seg_ &= ~__m;
-      if (__result.__ctz_ > __first.__ctz_)
-        *__result.__seg_ |= __b << (__result.__ctz_ - __first.__ctz_);
-      else
-        *__result.__seg_ |= __b >> (__first.__ctz_ - __result.__ctz_);
-      __result.__seg_ += (__ddn + __result.__ctz_) / __bits_per_word;
-      __result.__ctz_ = static_cast<unsigned>((__ddn + __result.__ctz_) % __bits_per_word);
-      __dn -= __ddn;
-      if (__dn > 0) {
-        __m = ~__storage_type(0) >> (__bits_per_word - __dn);
-        *__result.__seg_ &= ~__m;
-        *__result.__seg_ |= __b >> (__first.__ctz_ + __ddn);
-        __result.__ctz_ = static_cast<unsigned>(__dn);
-      }
-      ++__first.__seg_;
-      // __first.__ctz_ = 0;
-    }
-    // __first.__ctz_ == 0;
-    // do middle words
-    unsigned __clz_r   = __bits_per_word - __result.__ctz_;
-    __storage_type __m = ~__storage_type(0) << __result.__ctz_;
-    for (; __n >= __bits_per_word; __n -= __bits_per_word, ++__first.__seg_) {
-      __storage_type __b = *__first.__seg_;
-      *__result.__seg_ &= ~__m;
-      *__result.__seg_ |= __b << __result.__ctz_;
-      ++__result.__seg_;
-      *__result.__seg_ &= __m;
-      *__result.__seg_ |= __b >> __clz_r;
-    }
-    // do last word
-    if (__n > 0) {
-      __m                 = ~__storage_type(0) >> (__bits_per_word - __n);
-      __storage_type __b  = *__first.__seg_ & __m;
-      __storage_type __dn = std::min(__n, static_cast<difference_type>(__clz_r));
-      __m                 = (~__storage_type(0) << __result.__ctz_) & (~__storage_type(0) >> (__clz_r - __dn));
-      *__result.__seg_ &= ~__m;
-      *__result.__seg_ |= __b << __result.__ctz_;
-      __result.__seg_ += (__dn + __result.__ctz_) / __bits_per_word;
-      __result.__ctz_ = static_cast<unsigned>((__dn + __result.__ctz_) % __bits_per_word);
-      __n -= __dn;
-      if (__n > 0) {
-        __m = ~__storage_type(0) >> (__bits_per_word - __n);
-        *__result.__seg_ &= ~__m;
-        *__result.__seg_ |= __b >> __dn;
-        __result.__ctz_ = static_cast<unsigned>(__n);
-      }
-    }
-  }
-  return __result;
-}
-
-template <class _Cp, bool _IsConst>
-inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 __bit_iterator<_Cp, false>
-copy(__bit_iterator<_Cp, _IsConst> __first, __bit_iterator<_Cp, _IsConst> __last, __bit_iterator<_Cp, false> __result) {
-  if (__first.__ctz_ == __result.__ctz_)
-    return std::__copy_aligned(__first, __last, __result);
-  return std::__copy_unaligned(__first, __last, __result);
-}
-
-// copy_backward
-
-template <class _Cp, bool _IsConst>
-_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI __bit_iterator<_Cp, false> __copy_backward_aligned(
-    __bit_iterator<_Cp, _IsConst> __first, __bit_iterator<_Cp, _IsConst> __last, __bit_iterator<_Cp, false> __result) {
-  using _In             = __bit_iterator<_Cp, _IsConst>;
-  using difference_type = typename _In::difference_type;
-  using __storage_type  = typename _In::__storage_type;
-
-  const int __bits_per_word = _In::__bits_per_word;
-  difference_type __n       = __last - __first;
-  if (__n > 0) {
-    // do first word
-    if (__last.__ctz_ != 0) {
-      difference_type __dn = std::min(static_cast<difference_type>(__last.__ctz_), __n);
-      __n -= __dn;
-      unsigned __clz     = __bits_per_word - __last.__ctz_;
-      __storage_type __m = (~__storage_type(0) << (__last.__ctz_ - __dn)) & (~__storage_type(0) >> __clz);
-      __storage_type __b = *__last.__seg_ & __m;
-      *__result.__seg_ &= ~__m;
-      *__result.__seg_ |= __b;
-      __result.__ctz_ = static_cast<unsigned>(((-__dn & (__bits_per_word - 1)) + __result.__ctz_) % __bits_per_word);
-      // __last.__ctz_ = 0
-    }
-    // __last.__ctz_ == 0 || __n == 0
-    // __result.__ctz_ == 0 || __n == 0
-    // do middle words
-    __storage_type __nw = __n / __bits_per_word;
-    __result.__seg_ -= __nw;
-    __last.__seg_ -= __nw;
-    std::copy_n(std::__to_address(__last.__seg_), __nw, std::__to_address(__result.__seg_));
-    __n -= __nw * __bits_per_word;
-    // do last word
-    if (__n > 0) {
-      __storage_type __m = ~__storage_type(0) << (__bits_per_word - __n);
-      __storage_type __b = *--__last.__seg_ & __m;
-      *--__result.__seg_ &= ~__m;
-      *__result.__seg_ |= __b;
-      __result.__ctz_ = static_cast<unsigned>(-__n & (__bits_per_word - 1));
-    }
-  }
-  return __result;
-}
-
-template <class _Cp, bool _IsConst>
-_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI __bit_iterator<_Cp, false> __copy_backward_unaligned(
-    __bit_iterator<_Cp, _IsConst> __first, __bit_iterator<_Cp, _IsConst> __last, __bit_iterator<_Cp, false> __result) {
-  using _In             = __bit_iterator<_Cp, _IsConst>;
-  using difference_type = typename _In::difference_type;
-  using __storage_type  = typename _In::__storage_type;
-
-  const int __bits_per_word = _In::__bits_per_word;
-  difference_type __n       = __last - __first;
-  if (__n > 0) {
-    // do first word
-    if (__last.__ctz_ != 0) {
-      difference_type __dn = std::min(static_cast<difference_type>(__last.__ctz_), __n);
-      __n -= __dn;
-      unsigned __clz_l     = __bits_per_word - __last.__ctz_;
-      __storage_type __m   = (~__storage_type(0) << (__last.__ctz_ - __dn)) & (~__storage_type(0) >> __clz_l);
-      __storage_type __b   = *__last.__seg_ & __m;
-      unsigned __clz_r     = __bits_per_word - __result.__ctz_;
-      __storage_type __ddn = std::min(__dn, static_cast<difference_type>(__result.__ctz_));
-      if (__ddn > 0) {
-        __m = (~__storage_type(0) << (__result.__ctz_ - __ddn)) & (~__storage_type(0) >> __clz_r);
-        *__result.__seg_ &= ~__m;
-        if (__result.__ctz_ > __last.__ctz_)
-          *__result.__seg_ |= __b << (__result.__ctz_ - __last.__ctz_);
-        else
-          *__result.__seg_ |= __b >> (__last.__ctz_ - __result.__ctz_);
-        __result.__ctz_ = static_cast<unsigned>(((-__ddn & (__bits_per_word - 1)) + __result.__ctz_) % __bits_per_word);
-        __dn -= __ddn;
-      }
-      if (__dn > 0) {
-        // __result.__ctz_ == 0
-        --__result.__seg_;
-        __result.__ctz_ = static_cast<unsigned>(-__dn & (__bits_per_word - 1));
-        __m             = ~__storage_type(0) << __result.__ctz_;
-        *__result.__seg_ &= ~__m;
-        __last.__ctz_ -= __dn + __ddn;
-        *__result.__seg_ |= __b << (__result.__ctz_ - __last.__ctz_);
-      }
-      // __last.__ctz_ = 0
-    }
-    // __last.__ctz_ == 0 || __n == 0
-    // __result.__ctz_ != 0 || __n == 0
-    // do middle words
-    unsigned __clz_r   = __bits_per_word - __result.__ctz_;
-    __storage_type __m = ~__storage_type(0) >> __clz_r;
-    for (; __n >= __bits_per_word; __n -= __bits_per_word) {
-      __storage_type __b = *--__last.__seg_;
-      *__result.__seg_ &= ~__m;
-      *__result.__seg_ |= __b >> __clz_r;
-      *--__result.__seg_ &= __m;
-      *__result.__seg_ |= __b << __result.__ctz_;
-    }
-    // do last word
-    if (__n > 0) {
-      __m                 = ~__storage_type(0) << (__bits_per_word - __n);
-      __storage_type __b  = *--__last.__seg_ & __m;
-      __clz_r             = __bits_per_word - __result.__ctz_;
-      __storage_type __dn = std::min(__n, static_cast<difference_type>(__result.__ctz_));
-      __m                 = (~__storage_type(0) << (__result.__ctz_ - __dn)) & (~__storage_type(0) >> __clz_r);
-      *__result.__seg_ &= ~__m;
-      *__result.__seg_ |= __b >> (__bits_per_word - __result.__ctz_);
-      __result.__ctz_ = static_cast<unsigned>(((-__dn & (__bits_per_word - 1)) + __result.__ctz_) % __bits_per_word);
-      __n -= __dn;
-      if (__n > 0) {
-        // __result.__ctz_ == 0
-        --__result.__seg_;
-        __result.__ctz_ = static_cast<unsigned>(-__n & (__bits_per_word - 1));
-        __m             = ~__storage_type(0) << __result.__ctz_;
-        *__result.__seg_ &= ~__m;
-        *__result.__seg_ |= __b << (__result.__ctz_ - (__bits_per_word - __n - __dn));
-      }
-    }
-  }
-  return __result;
-}
-
-template <class _Cp, bool _IsConst>
-inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 __bit_iterator<_Cp, false> copy_backward(
-    __bit_iterator<_Cp, _IsConst> __first, __bit_iterator<_Cp, _IsConst> __last, __bit_iterator<_Cp, false> __result) {
-  if (__last.__ctz_ == __result.__ctz_)
-    return std::__copy_backward_aligned(__first, __last, __result);
-  return std::__copy_backward_unaligned(__first, __last, __result);
-}
-
 // move
 
 template <class _Cp, bool _IsConst>
@@ -989,17 +764,17 @@ private:
   _LIBCPP_CONSTEXPR_SINCE_CXX20 friend __bit_iterator<_Dp, false> __copy_unaligned(
       __bit_iterator<_Dp, _IC> __first, __bit_iterator<_Dp, _IC> __last, __bit_iterator<_Dp, false> __result);
   template <class _Dp, bool _IC>
-  _LIBCPP_CONSTEXPR_SINCE_CXX20 friend __bit_iterator<_Dp, false>
-  copy(__bit_iterator<_Dp, _IC> __first, __bit_iterator<_Dp, _IC> __last, __bit_iterator<_Dp, false> __result);
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 friend pair<__bit_iterator<_Dp, _IC>, __bit_iterator<_Dp, false> >
+  __copy_impl::operator()(
+      __bit_iterator<_Dp, _IC> __first, __bit_iterator<_Dp, _IC> __last, __bit_iterator<_Dp, false> __result) const;
   template <class _Dp, bool _IC>
   _LIBCPP_CONSTEXPR_SINCE_CXX20 friend __bit_iterator<_Dp, false> __copy_backward_aligned(
       __bit_iterator<_Dp, _IC> __first, __bit_iterator<_Dp, _IC> __last, __bit_iterator<_Dp, false> __result);
   template <class _Dp, bool _IC>
   _LIBCPP_CONSTEXPR_SINCE_CXX20 friend __bit_iterator<_Dp, false> __copy_backward_unaligned(
       __bit_iterator<_Dp, _IC> __first, __bit_iterator<_Dp, _IC> __last, __bit_iterator<_Dp, false> __result);
-  template <class _Dp, bool _IC>
-  _LIBCPP_CONSTEXPR_SINCE_CXX20 friend __bit_iterator<_Dp, false>
-  copy_backward(__bit_iterator<_Dp, _IC> __first, __bit_iterator<_Dp, _IC> __last, __bit_iterator<_Dp, false> __result);
+  template <class _AlgPolicy>
+  friend struct __copy_backward_impl;
   template <class _Cl, class _Cr>
   friend __bit_iterator<_Cr, false>
       __swap_ranges_aligned(__bit_iterator<_Cl, false>, __bit_iterator<_Cl, false>, __bit_iterator<_Cr, false>);
diff --git a/libcxx/include/__chrono/convert_to_tm.h b/libcxx/include/__chrono/convert_to_tm.h
index 7d06a38d87f26..d9ccf693160d2 100644
--- a/libcxx/include/__chrono/convert_to_tm.h
+++ b/libcxx/include/__chrono/convert_to_tm.h
@@ -23,6 +23,7 @@
 #include <__chrono/statically_widen.h>
 #include <__chrono/sys_info.h>
 #include <__chrono/system_clock.h>
+#include <__chrono/tai_clock.h>
 #include <__chrono/time_point.h>
 #include <__chrono/utc_clock.h>
 #include <__chrono/weekday.h>
@@ -35,6 +36,7 @@
 #include <__config>
 #include <__format/format_error.h>
 #include <__memory/addressof.h>
+#include <__type_traits/common_type.h>
 #include <__type_traits/is_convertible.h>
 #include <__type_traits/is_specialization.h>
 #include <cstdint>
@@ -112,6 +114,16 @@ _LIBCPP_HIDE_FROM_ABI _Tm __convert_to_tm(chrono::utc_time<_Duration> __tp) {
   return __result;
 }
 
+template <class _Tm, class _Duration>
+_LIBCPP_HIDE_FROM_ABI _Tm __convert_to_tm(chrono::tai_time<_Duration> __tp) {
+  using _Rp = common_type_t<_Duration, chrono::seconds>;
+  // The time between the TAI epoch (1958-01-01) and UNIX epoch (1970-01-01).
+  // This avoids leap second conversion when going from TAI to UTC.
+  // (It also avoids issues when the date is before the UTC epoch.)
+  constexpr chrono::seconds __offset{4383 * 24 * 60 * 60};
+  return std::__convert_to_tm<_Tm>(chrono::sys_time<_Rp>{__tp.time_since_epoch() - __offset});
+}
+
 #    endif // _LIBCPP_HAS_EXPERIMENTAL_TZDB
 #  endif   // _LIBCPP_HAS_TIME_ZONE_DATABASE && _LIBCPP_HAS_FILESYSTEM && _LIBCPP_HAS_LOCALIZATION
 
@@ -131,6 +143,8 @@ _LIBCPP_HIDE_FROM_ABI _Tm __convert_to_tm(const _ChronoT& __value) {
 #    if _LIBCPP_HAS_EXPERIMENTAL_TZDB
     else if constexpr (same_as<typename _ChronoT::clock, chrono::utc_clock>)
       return std::__convert_to_tm<_Tm>(__value);
+    else if constexpr (same_as<typename _ChronoT::clock, chrono::tai_clock>)
+      return std::__convert_to_tm<_Tm>(__value);
 #    endif // _LIBCPP_HAS_EXPERIMENTAL_TZDB
 #  endif   // _LIBCPP_HAS_TIME_ZONE_DATABASE && _LIBCPP_HAS_FILESYSTEM && _LIBCPP_HAS_LOCALIZATION
     else if constexpr (same_as<typename _ChronoT::clock, chrono::file_clock>)
diff --git a/libcxx/include/__chrono/formatter.h b/libcxx/include/__chrono/formatter.h
index d17acd274e4cd..13bcb717291f6 100644
--- a/libcxx/include/__chrono/formatter.h
+++ b/libcxx/include/__chrono/formatter.h
@@ -31,6 +31,7 @@
 #  include <__chrono/statically_widen.h>
 #  include <__chrono/sys_info.h>
 #  include <__chrono/system_clock.h>
+#  include <__chrono/tai_clock.h>
 #  include <__chrono/time_point.h>
 #  include <__chrono/utc_clock.h>
 #  include <__chrono/weekday.h>
@@ -232,9 +233,11 @@ _LIBCPP_HIDE_FROM_ABI __time_zone __convert_to_time_zone([[maybe_unused]] const
   if constexpr (same_as<_Tp, chrono::sys_info>)
     return {__value.abbrev, __value.offset};
 #      if _LIBCPP_HAS_TIME_ZONE_DATABASE && _LIBCPP_HAS_FILESYSTEM
+  else if constexpr (__is_time_point<_Tp> && requires { requires same_as<typename _Tp::clock, chrono::tai_clock>; })
+    return {"TAI", chrono::seconds{0}};
   else if constexpr (__is_specialization_v<_Tp, chrono::zoned_time>)
     return __formatter::__convert_to_time_zone(__value.get_info());
-#      endif
+#      endif // _LIBCPP_HAS_TIME_ZONE_DATABASE && _LIBCPP_HAS_FILESYSTEM
   else
 #    endif // _LIBCPP_HAS_EXPERIMENTAL_TZDB
     return {"UTC", chrono::seconds{0}};
@@ -734,6 +737,17 @@ struct _LIBCPP_TEMPLATE_VIS formatter<chrono::utc_time<_Duration>, _CharT> : pub
   }
 };
 
+template <class _Duration, __fmt_char_type _CharT>
+struct _LIBCPP_TEMPLATE_VIS formatter<chrono::tai_time<_Duration>, _CharT> : public __formatter_chrono<_CharT> {
+public:
+  using _Base _LIBCPP_NODEBUG = __formatter_chrono<_CharT>;
+
+  template <class _ParseContext>
+  _LIBCPP_HIDE_FROM_ABI constexpr typename _ParseContext::iterator parse(_ParseContext& __ctx) {
+    return _Base::__parse(__ctx, __format_spec::__fields_chrono, __format_spec::__flags::__clock);
+  }
+};
+
 #      endif // _LIBCPP_HAS_EXPERIMENTAL_TZDB
 #    endif   // _LIBCPP_HAS_TIME_ZONE_DATABASE && _LIBCPP_HAS_FILESYSTEM
 
diff --git a/libcxx/include/__chrono/ostream.h b/libcxx/include/__chrono/ostream.h
index ed9ad8e346ba9..b8cd6a4680662 100644
--- a/libcxx/include/__chrono/ostream.h
+++ b/libcxx/include/__chrono/ostream.h
@@ -26,6 +26,7 @@
 #  include <__chrono/statically_widen.h>
 #  include <__chrono/sys_info.h>
 #  include <__chrono/system_clock.h>
+#  include <__chrono/tai_clock.h>
 #  include <__chrono/utc_clock.h>
 #  include <__chrono/weekday.h>
 #  include <__chrono/year.h>
@@ -71,6 +72,12 @@ operator<<(basic_ostream<_CharT, _Traits>& __os, const utc_time<_Duration>& __tp
   return __os << std::format(__os.getloc(), _LIBCPP_STATICALLY_WIDEN(_CharT, "{:L%F %T}"), __tp);
 }
 
+template <class _CharT, class _Traits, class _Duration>
+_LIBCPP_HIDE_FROM_ABI basic_ostream<_CharT, _Traits>&
+operator<<(basic_ostream<_CharT, _Traits>& __os, const tai_time<_Duration>& __tp) {
+  return __os << std::format(__os.getloc(), _LIBCPP_STATICALLY_WIDEN(_CharT, "{:L%F %T}"), __tp);
+}
+
 #      endif // _LIBCPP_HAS_EXPERIMENTAL_TZDB
 #    endif   // _LIBCPP_HAS_TIME_ZONE_DATABASE && _LIBCPP_HAS_FILESYSTEM
 
diff --git a/libcxx/include/__chrono/tai_clock.h b/libcxx/include/__chrono/tai_clock.h
new file mode 100644
index 0000000000000..14c8b70a94af5
--- /dev/null
+++ b/libcxx/include/__chrono/tai_clock.h
@@ -0,0 +1,108 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___CHRONO_TAI_CLOCK_H
+#define _LIBCPP___CHRONO_TAI_CLOCK_H
+
+#include <version>
+// Enable the contents of the header only when libc++ was built with experimental features enabled.
+#if _LIBCPP_HAS_EXPERIMENTAL_TZDB
+
+#  include <__assert>
+#  include <__chrono/duration.h>
+#  include <__chrono/time_point.h>
+#  include <__chrono/utc_clock.h>
+#  include <__config>
+#  include <__type_traits/common_type.h>
+
+#  if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#    pragma GCC system_header
+#  endif
+
+_LIBCPP_PUSH_MACROS
+#  include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#  if _LIBCPP_STD_VER >= 20 && _LIBCPP_HAS_TIME_ZONE_DATABASE && _LIBCPP_HAS_FILESYSTEM && _LIBCPP_HAS_LOCALIZATION
+
+namespace chrono {
+
+class tai_clock;
+
+template <class _Duration>
+using tai_time    = time_point<tai_clock, _Duration>;
+using tai_seconds = tai_time<seconds>;
+
+// [time.clock.tai.overview]/1
+//    The clock tai_clock measures seconds since 1958-01-01 00:00:00 and is
+//    offset 10s ahead of UTC at this date. That is, 1958-01-01 00:00:00 TAI is
+//    equivalent to 1957-12-31 23:59:50 UTC. Leap seconds are not inserted into
+//    TAI. Therefore every time a leap second is inserted into UTC, UTC shifts
+//    another second with respect to TAI. For example by 2000-01-01 there had
+//    been 22 positive and 0 negative leap seconds inserted so 2000-01-01
+//    00:00:00 UTC is equivalent to 2000-01-01 00:00:32 TAI (22s plus the
+//    initial 10s offset).
+//
+// Note this does not specify what the UTC offset before 1958-01-01 00:00:00
+// TAI is, nor does it follow the "real" TAI clock between 1958-01-01 and the
+// start of the UTC epoch. So while the member functions are fully specified in
+// the standard, they do not technically follow the "real-world" TAI clock with
+// 100% accuracy.
+//
+// https://koka-lang.github.io/koka/doc/std_time_utc.html contains more
+// information and references.
+class tai_clock {
+public:
+  using rep                       = utc_clock::rep;
+  using period                    = utc_clock::period;
+  using duration                  = chrono::duration<rep, period>;
+  using time_point                = chrono::time_point<tai_clock>;
+  static constexpr bool is_steady = false; // The utc_clock is not steady.
+
+  // The static difference between UTC and TAI time.
+  static constexpr chrono::seconds __offset{378691210};
+
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI static time_point now() { return from_utc(utc_clock::now()); }
+
+  template <class _Duration>
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI static utc_time<common_type_t<_Duration, seconds>>
+  to_utc(const tai_time<_Duration>& __time) noexcept {
+    using _Rp                    = common_type_t<_Duration, seconds>;
+    _Duration __time_since_epoch = __time.time_since_epoch();
+    _LIBCPP_ASSERT_ARGUMENT_WITHIN_DOMAIN(__time_since_epoch >= utc_time<_Rp>::min().time_since_epoch() + __offset,
+                                          "the TAI to UTC conversion would underflow");
+
+    return utc_time<_Rp>{__time_since_epoch - __offset};
+  }
+
+  template <class _Duration>
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI static tai_time<common_type_t<_Duration, seconds>>
+  from_utc(const utc_time<_Duration>& __time) noexcept {
+    using _Rp                    = common_type_t<_Duration, seconds>;
+    _Duration __time_since_epoch = __time.time_since_epoch();
+    _LIBCPP_ASSERT_ARGUMENT_WITHIN_DOMAIN(__time_since_epoch <= utc_time<_Rp>::max().time_since_epoch() - __offset,
+                                          "the UTC to TAI conversion would overflow");
+
+    return tai_time<_Rp>{__time_since_epoch + __offset};
+  }
+};
+
+} // namespace chrono
+
+#  endif // _LIBCPP_STD_VER >= 20 && _LIBCPP_HAS_TIME_ZONE_DATABASE && _LIBCPP_HAS_FILESYSTEM &&
+         // _LIBCPP_HAS_LOCALIZATION
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP_HAS_EXPERIMENTAL_TZDB
+
+#endif // _LIBCPP___CHRONO_TAI_CLOCK_H
diff --git a/libcxx/include/__config b/libcxx/include/__config
index 30b2d147dc563..1c6dd8f36c32f 100644
--- a/libcxx/include/__config
+++ b/libcxx/include/__config
@@ -28,7 +28,7 @@
 // _LIBCPP_VERSION represents the version of libc++, which matches the version of LLVM.
 // Given a LLVM release LLVM XX.YY.ZZ (e.g. LLVM 17.0.1 == 17.00.01), _LIBCPP_VERSION is
 // defined to XXYYZZ.
-#  define _LIBCPP_VERSION 200000
+#  define _LIBCPP_VERSION 210000
 
 #  define _LIBCPP_CONCAT_IMPL(_X, _Y) _X##_Y
 #  define _LIBCPP_CONCAT(_X, _Y) _LIBCPP_CONCAT_IMPL(_X, _Y)
diff --git a/libcxx/include/__configuration/abi.h b/libcxx/include/__configuration/abi.h
index c6ef6fdcdf96e..5e40e308c8926 100644
--- a/libcxx/include/__configuration/abi.h
+++ b/libcxx/include/__configuration/abi.h
@@ -57,8 +57,6 @@
 // because it changes the mangling of the virtual function located in the vtable, which
 // changes how it gets signed.
 #  define _LIBCPP_ABI_BAD_FUNCTION_CALL_GOOD_WHAT_MESSAGE
-// Enable optimized version of __do_get_(un)signed which avoids redundant copies.
-#  define _LIBCPP_ABI_OPTIMIZED_LOCALE_NUM_GET
 // Give reverse_iterator<T> one data member of type T, not two.
 // Also, in C++17 and later, don't derive iterator types from std::iterator.
 #  define _LIBCPP_ABI_NO_ITERATOR_BASES
diff --git a/libcxx/include/__configuration/compiler.h b/libcxx/include/__configuration/compiler.h
index 80ece22bb50bd..cf459a0619b23 100644
--- a/libcxx/include/__configuration/compiler.h
+++ b/libcxx/include/__configuration/compiler.h
@@ -33,8 +33,8 @@
 // Warn if a compiler version is used that is not supported anymore
 // LLVM RELEASE Update the minimum compiler versions
 #  if defined(_LIBCPP_CLANG_VER)
-#    if _LIBCPP_CLANG_VER < 1700
-#      warning "Libc++ only supports Clang 17 and later"
+#    if _LIBCPP_CLANG_VER < 1800
+#      warning "Libc++ only supports Clang 18 and later"
 #    endif
 #  elif defined(_LIBCPP_APPLE_CLANG_VER)
 #    if _LIBCPP_APPLE_CLANG_VER < 1500
diff --git a/libcxx/include/__cxx03/__memory/uninitialized_algorithms.h b/libcxx/include/__cxx03/__memory/uninitialized_algorithms.h
index d595c8c6cf49e..7a7cc64f08ab3 100644
--- a/libcxx/include/__cxx03/__memory/uninitialized_algorithms.h
+++ b/libcxx/include/__cxx03/__memory/uninitialized_algorithms.h
@@ -642,7 +642,8 @@ __uninitialized_allocator_relocate(_Alloc& __alloc, _Tp* __first, _Tp* __last, _
     __guard.__complete();
     std::__allocator_destroy(__alloc, __first, __last);
   } else {
-    __builtin_memcpy(const_cast<__remove_const_t<_Tp>*>(__result), __first, sizeof(_Tp) * (__last - __first));
+    // Casting to void* to suppress clang complaining that this is technically UB.
+    __builtin_memcpy(static_cast<void*>(__result), __first, sizeof(_Tp) * (__last - __first));
   }
 }
 
diff --git a/libcxx/include/__flat_map/flat_map.h b/libcxx/include/__flat_map/flat_map.h
index ab53b7a285ca4..a0594ed9dc411 100644
--- a/libcxx/include/__flat_map/flat_map.h
+++ b/libcxx/include/__flat_map/flat_map.h
@@ -29,9 +29,11 @@
 #include <__cstddef/ptrdiff_t.h>
 #include <__flat_map/key_value_iterator.h>
 #include <__flat_map/sorted_unique.h>
+#include <__flat_map/utils.h>
 #include <__functional/invoke.h>
 #include <__functional/is_transparent.h>
 #include <__functional/operations.h>
+#include <__fwd/vector.h>
 #include <__iterator/concepts.h>
 #include <__iterator/distance.h>
 #include <__iterator/iterator_traits.h>
@@ -131,7 +133,7 @@ class flat_map {
   _LIBCPP_HIDE_FROM_ABI static constexpr bool __allocator_ctor_constraint =
       _And<uses_allocator<key_container_type, _Allocator>, uses_allocator<mapped_container_type, _Allocator>>::value;
 
-  _LIBCPP_HIDE_FROM_ABI static constexpr bool __is_compare_transparent = __is_transparent_v<_Compare, _Compare>;
+  _LIBCPP_HIDE_FROM_ABI static constexpr bool __is_compare_transparent = __is_transparent_v<_Compare>;
 
 public:
   // [flat.map.cons], construct/copy/destroy
@@ -153,7 +155,7 @@ class flat_map {
 #  if _LIBCPP_HAS_EXCEPTIONS
   } catch (...) {
     __other.clear();
-    // gcc does not like the `throw` keyword in a conditional noexcept function
+    // gcc does not like the `throw` keyword in a conditionally noexcept function
     if constexpr (!(is_nothrow_move_constructible_v<_KeyContainer> &&
                     is_nothrow_move_constructible_v<_MappedContainer> && is_nothrow_move_constructible_v<_Compare>)) {
       throw;
@@ -518,16 +520,16 @@ class flat_map {
     return emplace_hint(__hint, std::move(__x));
   }
 
-  template <class _Pp>
-    requires is_constructible_v<pair<key_type, mapped_type>, _Pp>
-  _LIBCPP_HIDE_FROM_ABI pair<iterator, bool> insert(_Pp&& __x) {
-    return emplace(std::forward<_Pp>(__x));
+  template <class _PairLike>
+    requires is_constructible_v<pair<key_type, mapped_type>, _PairLike>
+  _LIBCPP_HIDE_FROM_ABI pair<iterator, bool> insert(_PairLike&& __x) {
+    return emplace(std::forward<_PairLike>(__x));
   }
 
-  template <class _Pp>
-    requires is_constructible_v<pair<key_type, mapped_type>, _Pp>
-  _LIBCPP_HIDE_FROM_ABI iterator insert(const_iterator __hint, _Pp&& __x) {
-    return emplace_hint(__hint, std::forward<_Pp>(__x));
+  template <class _PairLike>
+    requires is_constructible_v<pair<key_type, mapped_type>, _PairLike>
+  _LIBCPP_HIDE_FROM_ABI iterator insert(const_iterator __hint, _PairLike&& __x) {
+    return emplace_hint(__hint, std::forward<_PairLike>(__x));
   }
 
   template <class _InputIterator>
@@ -860,22 +862,10 @@ class flat_map {
     __containers_.values.erase(__containers_.values.begin() + __dist, __containers_.values.end());
   }
 
-  template <class _InputIterator, class _Sentinel>
-  _LIBCPP_HIDE_FROM_ABI size_type __append(_InputIterator __first, _Sentinel __last) {
-    size_type __num_of_appended = 0;
-    for (; __first != __last; ++__first) {
-      value_type __kv = *__first;
-      __containers_.keys.insert(__containers_.keys.end(), std::move(__kv.first));
-      __containers_.values.insert(__containers_.values.end(), std::move(__kv.second));
-      ++__num_of_appended;
-    }
-    return __num_of_appended;
-  }
-
   template <bool _WasSorted, class _InputIterator, class _Sentinel>
   _LIBCPP_HIDE_FROM_ABI void __append_sort_merge_unique(_InputIterator __first, _Sentinel __last) {
     auto __on_failure        = std::__make_exception_guard([&]() noexcept { clear() /* noexcept */; });
-    size_t __num_of_appended = __append(std::move(__first), std::move(__last));
+    size_t __num_of_appended = __flat_map_utils::__append(*this, std::move(__first), std::move(__last));
     if (__num_of_appended != 0) {
       auto __zv                  = ranges::views::zip(__containers_.keys, __containers_.values);
       auto __append_start_offset = __containers_.keys.size() - __num_of_appended;
@@ -963,7 +953,8 @@ class flat_map {
 
     if (__key_it == __containers_.keys.end() || __compare_(__key, *__key_it)) {
       return pair<iterator, bool>(
-          __try_emplace_exact_hint(
+          __flat_map_utils::__emplace_exact_pos(
+              *this,
               std::move(__key_it),
               std::move(__mapped_it),
               std::forward<_KeyArg>(__key),
@@ -989,10 +980,13 @@ class flat_map {
   _LIBCPP_HIDE_FROM_ABI pair<iterator, bool> __try_emplace_hint(const_iterator __hint, _Kp&& __key, _Args&&... __args) {
     if (__is_hint_correct(__hint, __key)) {
       if (__hint == cend() || __compare_(__key, __hint->first)) {
-        return {
-            __try_emplace_exact_hint(
-                __hint.__key_iter_, __hint.__mapped_iter_, std::forward<_Kp>(__key), std::forward<_Args>(__args)...),
-            true};
+        return {__flat_map_utils::__emplace_exact_pos(
+                    *this,
+                    __hint.__key_iter_,
+                    __hint.__mapped_iter_,
+                    std::forward<_Kp>(__key),
+                    std::forward<_Args>(__args)...),
+                true};
       } else {
         // key equals
         auto __dist = __hint - cbegin();
@@ -1003,49 +997,6 @@ class flat_map {
     }
   }
 
-  template <class _IterK, class _IterM, class _KeyArg, class... _MArgs>
-  _LIBCPP_HIDE_FROM_ABI iterator
-  __try_emplace_exact_hint(_IterK&& __it_key, _IterM&& __it_mapped, _KeyArg&& __key, _MArgs&&... __mapped_args) {
-    auto __on_key_failed = std::__make_exception_guard([&]() noexcept {
-      if constexpr (__container_traits<_KeyContainer>::__emplacement_has_strong_exception_safety_guarantee) {
-        // Nothing to roll back!
-      } else {
-        // we need to clear both because we don't know the state of our keys anymore
-        clear() /* noexcept */;
-      }
-    });
-    auto __key_it        = __containers_.keys.emplace(__it_key, std::forward<_KeyArg>(__key));
-    __on_key_failed.__complete();
-
-    auto __on_value_failed = std::__make_exception_guard([&]() noexcept {
-      if constexpr (!__container_traits<_MappedContainer>::__emplacement_has_strong_exception_safety_guarantee) {
-        // we need to clear both because we don't know the state of our values anymore
-        clear() /* noexcept */;
-      } else {
-        // In this case, we know the values are just like before we attempted emplacement,
-        // and we also know that the keys have been emplaced successfully. Just roll back the keys.
-#  if _LIBCPP_HAS_EXCEPTIONS
-        try {
-#  endif // _LIBCPP_HAS_EXCEPTIONS
-          __containers_.keys.erase(__key_it);
-#  if _LIBCPP_HAS_EXCEPTIONS
-        } catch (...) {
-          // Now things are funky for real. We're failing to rollback the keys.
-          // Just give up and clear the whole thing.
-          //
-          // Also, swallow the exception that happened during the rollback and let the
-          // original value-emplacement exception propagate normally.
-          clear() /* noexcept */;
-        }
-#  endif // _LIBCPP_HAS_EXCEPTIONS
-      }
-    });
-    auto __mapped_it = __containers_.values.emplace(__it_mapped, std::forward<_MArgs>(__mapped_args)...);
-    __on_value_failed.__complete();
-
-    return iterator(std::move(__key_it), std::move(__mapped_it));
-  }
-
   template <class _Kp, class _Mapped>
   _LIBCPP_HIDE_FROM_ABI pair<iterator, bool> __insert_or_assign(_Kp&& __key, _Mapped&& __mapped) {
     auto __r = try_emplace(std::forward<_Kp>(__key), std::forward<_Mapped>(__mapped));
@@ -1087,8 +1038,10 @@ class flat_map {
   friend typename flat_map<_Key2, _Tp2, _Compare2, _KeyContainer2, _MappedContainer2>::size_type
   erase_if(flat_map<_Key2, _Tp2, _Compare2, _KeyContainer2, _MappedContainer2>&, _Predicate);
 
+  friend __flat_map_utils;
+
   containers __containers_;
-  [[no_unique_address]] key_compare __compare_;
+  _LIBCPP_NO_UNIQUE_ADDRESS key_compare __compare_;
 
   struct __key_equiv {
     _LIBCPP_HIDE_FROM_ABI __key_equiv(key_compare __c) : __comp_(__c) {}
@@ -1187,22 +1140,20 @@ template <ranges::input_range _Range,
           class _Compare   = less<__range_key_type<_Range>>,
           class _Allocator = allocator<byte>,
           class            = __enable_if_t<!__is_allocator<_Compare>::value && __is_allocator<_Allocator>::value>>
-flat_map(from_range_t, _Range&&, _Compare = _Compare(), _Allocator = _Allocator())
-    -> flat_map<
-        __range_key_type<_Range>,
-        __range_mapped_type<_Range>,
-        _Compare,
-        vector<__range_key_type<_Range>, __allocator_traits_rebind_t<_Allocator, __range_key_type<_Range>>>,
-        vector<__range_mapped_type<_Range>, __allocator_traits_rebind_t<_Allocator, __range_mapped_type<_Range>>>>;
+flat_map(from_range_t, _Range&&, _Compare = _Compare(), _Allocator = _Allocator()) -> flat_map<
+    __range_key_type<_Range>,
+    __range_mapped_type<_Range>,
+    _Compare,
+    vector<__range_key_type<_Range>, __allocator_traits_rebind_t<_Allocator, __range_key_type<_Range>>>,
+    vector<__range_mapped_type<_Range>, __allocator_traits_rebind_t<_Allocator, __range_mapped_type<_Range>>>>;
 
 template <ranges::input_range _Range, class _Allocator, class = __enable_if_t<__is_allocator<_Allocator>::value>>
-flat_map(from_range_t, _Range&&, _Allocator)
-    -> flat_map<
-        __range_key_type<_Range>,
-        __range_mapped_type<_Range>,
-        less<__range_key_type<_Range>>,
-        vector<__range_key_type<_Range>, __allocator_traits_rebind_t<_Allocator, __range_key_type<_Range>>>,
-        vector<__range_mapped_type<_Range>, __allocator_traits_rebind_t<_Allocator, __range_mapped_type<_Range>>>>;
+flat_map(from_range_t, _Range&&, _Allocator) -> flat_map<
+    __range_key_type<_Range>,
+    __range_mapped_type<_Range>,
+    less<__range_key_type<_Range>>,
+    vector<__range_key_type<_Range>, __allocator_traits_rebind_t<_Allocator, __range_key_type<_Range>>>,
+    vector<__range_mapped_type<_Range>, __allocator_traits_rebind_t<_Allocator, __range_mapped_type<_Range>>>>;
 
 template <class _Key, class _Tp, class _Compare = less<_Key>>
   requires(!__is_allocator<_Compare>::value)
diff --git a/libcxx/include/__flat_map/flat_multimap.h b/libcxx/include/__flat_map/flat_multimap.h
new file mode 100644
index 0000000000000..ea77fb5d79bd2
--- /dev/null
+++ b/libcxx/include/__flat_map/flat_multimap.h
@@ -0,0 +1,1010 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___FLAT_MAP_FLAT_MULTIMAP_H
+#define _LIBCPP___FLAT_MAP_FLAT_MULTIMAP_H
+
+#include <__algorithm/lexicographical_compare_three_way.h>
+#include <__algorithm/min.h>
+#include <__algorithm/ranges_equal.h>
+#include <__algorithm/ranges_equal_range.h>
+#include <__algorithm/ranges_inplace_merge.h>
+#include <__algorithm/ranges_is_sorted.h>
+#include <__algorithm/ranges_lower_bound.h>
+#include <__algorithm/ranges_partition_point.h>
+#include <__algorithm/ranges_sort.h>
+#include <__algorithm/ranges_unique.h>
+#include <__algorithm/ranges_upper_bound.h>
+#include <__algorithm/remove_if.h>
+#include <__assert>
+#include <__compare/synth_three_way.h>
+#include <__concepts/convertible_to.h>
+#include <__concepts/swappable.h>
+#include <__config>
+#include <__cstddef/byte.h>
+#include <__cstddef/ptrdiff_t.h>
+#include <__flat_map/key_value_iterator.h>
+#include <__flat_map/sorted_equivalent.h>
+#include <__flat_map/utils.h>
+#include <__functional/invoke.h>
+#include <__functional/is_transparent.h>
+#include <__functional/operations.h>
+#include <__fwd/vector.h>
+#include <__iterator/concepts.h>
+#include <__iterator/distance.h>
+#include <__iterator/iterator_traits.h>
+#include <__iterator/ranges_iterator_traits.h>
+#include <__iterator/reverse_iterator.h>
+#include <__memory/allocator_traits.h>
+#include <__memory/uses_allocator.h>
+#include <__memory/uses_allocator_construction.h>
+#include <__ranges/access.h>
+#include <__ranges/concepts.h>
+#include <__ranges/container_compatible_range.h>
+#include <__ranges/drop_view.h>
+#include <__ranges/from_range.h>
+#include <__ranges/ref_view.h>
+#include <__ranges/size.h>
+#include <__ranges/subrange.h>
+#include <__ranges/zip_view.h>
+#include <__type_traits/conjunction.h>
+#include <__type_traits/container_traits.h>
+#include <__type_traits/invoke.h>
+#include <__type_traits/is_allocator.h>
+#include <__type_traits/is_nothrow_constructible.h>
+#include <__type_traits/is_same.h>
+#include <__type_traits/maybe_const.h>
+#include <__utility/exception_guard.h>
+#include <__utility/move.h>
+#include <__utility/pair.h>
+#include <__utility/scope_guard.h>
+#include <__vector/vector.h>
+#include <initializer_list>
+#include <stdexcept>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+#if _LIBCPP_STD_VER >= 23
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _Key,
+          class _Tp,
+          class _Compare         = less<_Key>,
+          class _KeyContainer    = vector<_Key>,
+          class _MappedContainer = vector<_Tp>>
+class flat_multimap {
+  template <class, class, class, class, class>
+  friend class flat_multimap;
+
+  static_assert(is_same_v<_Key, typename _KeyContainer::value_type>);
+  static_assert(is_same_v<_Tp, typename _MappedContainer::value_type>);
+  static_assert(!is_same_v<_KeyContainer, std::vector<bool>>, "vector<bool> is not a sequence container");
+  static_assert(!is_same_v<_MappedContainer, std::vector<bool>>, "vector<bool> is not a sequence container");
+
+  template <bool _Const>
+  using __iterator _LIBCPP_NODEBUG = __key_value_iterator<flat_multimap, _KeyContainer, _MappedContainer, _Const>;
+
+public:
+  // types
+  using key_type               = _Key;
+  using mapped_type            = _Tp;
+  using value_type             = pair<key_type, mapped_type>;
+  using key_compare            = __type_identity_t<_Compare>;
+  using reference              = pair<const key_type&, mapped_type&>;
+  using const_reference        = pair<const key_type&, const mapped_type&>;
+  using size_type              = size_t;
+  using difference_type        = ptrdiff_t;
+  using iterator               = __iterator<false>; // see [container.requirements]
+  using const_iterator         = __iterator<true>;  // see [container.requirements]
+  using reverse_iterator       = std::reverse_iterator<iterator>;
+  using const_reverse_iterator = std::reverse_iterator<const_iterator>;
+  using key_container_type     = _KeyContainer;
+  using mapped_container_type  = _MappedContainer;
+
+  class value_compare {
+  private:
+    key_compare __comp_;
+    _LIBCPP_HIDE_FROM_ABI value_compare(key_compare __c) : __comp_(__c) {}
+    friend flat_multimap;
+
+  public:
+    _LIBCPP_HIDE_FROM_ABI bool operator()(const_reference __x, const_reference __y) const {
+      return __comp_(__x.first, __y.first);
+    }
+  };
+
+  struct containers {
+    key_container_type keys;
+    mapped_container_type values;
+  };
+
+private:
+  template <class _Allocator>
+  _LIBCPP_HIDE_FROM_ABI static constexpr bool __allocator_ctor_constraint =
+      _And<uses_allocator<key_container_type, _Allocator>, uses_allocator<mapped_container_type, _Allocator>>::value;
+
+  _LIBCPP_HIDE_FROM_ABI static constexpr bool __is_compare_transparent = __is_transparent_v<_Compare>;
+
+public:
+  // [flat.map.cons], construct/copy/destroy
+  _LIBCPP_HIDE_FROM_ABI flat_multimap() noexcept(
+      is_nothrow_default_constructible_v<_KeyContainer> && is_nothrow_default_constructible_v<_MappedContainer> &&
+      is_nothrow_default_constructible_v<_Compare>)
+      : __containers_(), __compare_() {}
+
+  _LIBCPP_HIDE_FROM_ABI flat_multimap(const flat_multimap&) = default;
+
+  // The copy/move constructors are not specified in the spec, which means they should be defaulted.
+  // However, the move constructor can potentially leave a moved-from object in an inconsistent
+  // state if an exception is thrown.
+  _LIBCPP_HIDE_FROM_ABI flat_multimap(flat_multimap&& __other) noexcept(
+      is_nothrow_move_constructible_v<_KeyContainer> && is_nothrow_move_constructible_v<_MappedContainer> &&
+      is_nothrow_move_constructible_v<_Compare>)
+#  if _LIBCPP_HAS_EXCEPTIONS
+      try
+#  endif // _LIBCPP_HAS_EXCEPTIONS
+      : __containers_(std::move(__other.__containers_)), __compare_(std::move(__other.__compare_)) {
+    __other.clear();
+#  if _LIBCPP_HAS_EXCEPTIONS
+  } catch (...) {
+    __other.clear();
+    // gcc does not like the `throw` keyword in a conditionally noexcept function
+    if constexpr (!(is_nothrow_move_constructible_v<_KeyContainer> &&
+                    is_nothrow_move_constructible_v<_MappedContainer> && is_nothrow_move_constructible_v<_Compare>)) {
+      throw;
+    }
+#  endif // _LIBCPP_HAS_EXCEPTIONS
+  }
+
+  template <class _Allocator>
+    requires __allocator_ctor_constraint<_Allocator>
+  _LIBCPP_HIDE_FROM_ABI flat_multimap(const flat_multimap& __other, const _Allocator& __alloc)
+      : flat_multimap(__ctor_uses_allocator_tag{},
+                      __alloc,
+                      __other.__containers_.keys,
+                      __other.__containers_.values,
+                      __other.__compare_) {}
+
+  template <class _Allocator>
+    requires __allocator_ctor_constraint<_Allocator>
+  _LIBCPP_HIDE_FROM_ABI flat_multimap(flat_multimap&& __other, const _Allocator& __alloc)
+#  if _LIBCPP_HAS_EXCEPTIONS
+      try
+#  endif // _LIBCPP_HAS_EXCEPTIONS
+      : flat_multimap(__ctor_uses_allocator_tag{},
+                      __alloc,
+                      std::move(__other.__containers_.keys),
+                      std::move(__other.__containers_.values),
+                      std::move(__other.__compare_)) {
+    __other.clear();
+#  if _LIBCPP_HAS_EXCEPTIONS
+  } catch (...) {
+    __other.clear();
+    throw;
+#  endif // _LIBCPP_HAS_EXCEPTIONS
+  }
+
+  _LIBCPP_HIDE_FROM_ABI flat_multimap(
+      key_container_type __key_cont, mapped_container_type __mapped_cont, const key_compare& __comp = key_compare())
+      : __containers_{.keys = std::move(__key_cont), .values = std::move(__mapped_cont)}, __compare_(__comp) {
+    _LIBCPP_ASSERT_VALID_INPUT_RANGE(__containers_.keys.size() == __containers_.values.size(),
+                                     "flat_multimap keys and mapped containers have different size");
+    __sort();
+  }
+
+  template <class _Allocator>
+    requires __allocator_ctor_constraint<_Allocator>
+  _LIBCPP_HIDE_FROM_ABI flat_multimap(
+      const key_container_type& __key_cont, const mapped_container_type& __mapped_cont, const _Allocator& __alloc)
+      : flat_multimap(__ctor_uses_allocator_tag{}, __alloc, __key_cont, __mapped_cont) {
+    _LIBCPP_ASSERT_VALID_INPUT_RANGE(__containers_.keys.size() == __containers_.values.size(),
+                                     "flat_multimap keys and mapped containers have different size");
+    __sort();
+  }
+
+  template <class _Allocator>
+    requires __allocator_ctor_constraint<_Allocator>
+  _LIBCPP_HIDE_FROM_ABI
+  flat_multimap(const key_container_type& __key_cont,
+                const mapped_container_type& __mapped_cont,
+                const key_compare& __comp,
+                const _Allocator& __alloc)
+      : flat_multimap(__ctor_uses_allocator_tag{}, __alloc, __key_cont, __mapped_cont, __comp) {
+    _LIBCPP_ASSERT_VALID_INPUT_RANGE(__containers_.keys.size() == __containers_.values.size(),
+                                     "flat_multimap keys and mapped containers have different size");
+    __sort();
+  }
+
+  _LIBCPP_HIDE_FROM_ABI
+  flat_multimap(sorted_equivalent_t,
+                key_container_type __key_cont,
+                mapped_container_type __mapped_cont,
+                const key_compare& __comp = key_compare())
+      : __containers_{.keys = std::move(__key_cont), .values = std::move(__mapped_cont)}, __compare_(__comp) {
+    _LIBCPP_ASSERT_VALID_INPUT_RANGE(__containers_.keys.size() == __containers_.values.size(),
+                                     "flat_multimap keys and mapped containers have different size");
+    _LIBCPP_ASSERT_SEMANTIC_REQUIREMENT(__is_sorted(__containers_.keys), "Key container is not sorted");
+  }
+
+  template <class _Allocator>
+    requires __allocator_ctor_constraint<_Allocator>
+  _LIBCPP_HIDE_FROM_ABI
+  flat_multimap(sorted_equivalent_t,
+                const key_container_type& __key_cont,
+                const mapped_container_type& __mapped_cont,
+                const _Allocator& __alloc)
+      : flat_multimap(__ctor_uses_allocator_tag{}, __alloc, __key_cont, __mapped_cont) {
+    _LIBCPP_ASSERT_VALID_INPUT_RANGE(__containers_.keys.size() == __containers_.values.size(),
+                                     "flat_multimap keys and mapped containers have different size");
+    _LIBCPP_ASSERT_SEMANTIC_REQUIREMENT(__is_sorted(__containers_.keys), "Key container is not sorted");
+  }
+
+  template <class _Allocator>
+    requires __allocator_ctor_constraint<_Allocator>
+  _LIBCPP_HIDE_FROM_ABI
+  flat_multimap(sorted_equivalent_t,
+                const key_container_type& __key_cont,
+                const mapped_container_type& __mapped_cont,
+                const key_compare& __comp,
+                const _Allocator& __alloc)
+      : flat_multimap(__ctor_uses_allocator_tag{}, __alloc, __key_cont, __mapped_cont, __comp) {
+    _LIBCPP_ASSERT_VALID_INPUT_RANGE(__containers_.keys.size() == __containers_.values.size(),
+                                     "flat_multimap keys and mapped containers have different size");
+    _LIBCPP_ASSERT_SEMANTIC_REQUIREMENT(__is_sorted(__containers_.keys), "Key container is not sorted");
+  }
+
+  _LIBCPP_HIDE_FROM_ABI explicit flat_multimap(const key_compare& __comp) : __containers_(), __compare_(__comp) {}
+
+  template <class _Allocator>
+    requires __allocator_ctor_constraint<_Allocator>
+  _LIBCPP_HIDE_FROM_ABI flat_multimap(const key_compare& __comp, const _Allocator& __alloc)
+      : flat_multimap(__ctor_uses_allocator_empty_tag{}, __alloc, __comp) {}
+
+  template <class _Allocator>
+    requires __allocator_ctor_constraint<_Allocator>
+  _LIBCPP_HIDE_FROM_ABI explicit flat_multimap(const _Allocator& __alloc)
+      : flat_multimap(__ctor_uses_allocator_empty_tag{}, __alloc) {}
+
+  template <class _InputIterator>
+    requires __has_input_iterator_category<_InputIterator>::value
+  _LIBCPP_HIDE_FROM_ABI
+  flat_multimap(_InputIterator __first, _InputIterator __last, const key_compare& __comp = key_compare())
+      : __containers_(), __compare_(__comp) {
+    insert(__first, __last);
+  }
+
+  template <class _InputIterator, class _Allocator>
+    requires(__has_input_iterator_category<_InputIterator>::value && __allocator_ctor_constraint<_Allocator>)
+  _LIBCPP_HIDE_FROM_ABI
+  flat_multimap(_InputIterator __first, _InputIterator __last, const key_compare& __comp, const _Allocator& __alloc)
+      : flat_multimap(__ctor_uses_allocator_empty_tag{}, __alloc, __comp) {
+    insert(__first, __last);
+  }
+
+  template <class _InputIterator, class _Allocator>
+    requires(__has_input_iterator_category<_InputIterator>::value && __allocator_ctor_constraint<_Allocator>)
+  _LIBCPP_HIDE_FROM_ABI flat_multimap(_InputIterator __first, _InputIterator __last, const _Allocator& __alloc)
+      : flat_multimap(__ctor_uses_allocator_empty_tag{}, __alloc) {
+    insert(__first, __last);
+  }
+
+  template <_ContainerCompatibleRange<value_type> _Range>
+  _LIBCPP_HIDE_FROM_ABI flat_multimap(from_range_t __fr, _Range&& __rg)
+      : flat_multimap(__fr, std::forward<_Range>(__rg), key_compare()) {}
+
+  template <_ContainerCompatibleRange<value_type> _Range, class _Allocator>
+    requires __allocator_ctor_constraint<_Allocator>
+  _LIBCPP_HIDE_FROM_ABI flat_multimap(from_range_t, _Range&& __rg, const _Allocator& __alloc)
+      : flat_multimap(__ctor_uses_allocator_empty_tag{}, __alloc) {
+    insert_range(std::forward<_Range>(__rg));
+  }
+
+  template <_ContainerCompatibleRange<value_type> _Range>
+  _LIBCPP_HIDE_FROM_ABI flat_multimap(from_range_t, _Range&& __rg, const key_compare& __comp) : flat_multimap(__comp) {
+    insert_range(std::forward<_Range>(__rg));
+  }
+
+  template <_ContainerCompatibleRange<value_type> _Range, class _Allocator>
+    requires __allocator_ctor_constraint<_Allocator>
+  _LIBCPP_HIDE_FROM_ABI flat_multimap(from_range_t, _Range&& __rg, const key_compare& __comp, const _Allocator& __alloc)
+      : flat_multimap(__ctor_uses_allocator_empty_tag{}, __alloc, __comp) {
+    insert_range(std::forward<_Range>(__rg));
+  }
+
+  template <class _InputIterator>
+    requires __has_input_iterator_category<_InputIterator>::value
+  _LIBCPP_HIDE_FROM_ABI flat_multimap(
+      sorted_equivalent_t, _InputIterator __first, _InputIterator __last, const key_compare& __comp = key_compare())
+      : __containers_(), __compare_(__comp) {
+    insert(sorted_equivalent, __first, __last);
+  }
+  template <class _InputIterator, class _Allocator>
+    requires(__has_input_iterator_category<_InputIterator>::value && __allocator_ctor_constraint<_Allocator>)
+  _LIBCPP_HIDE_FROM_ABI
+  flat_multimap(sorted_equivalent_t,
+                _InputIterator __first,
+                _InputIterator __last,
+                const key_compare& __comp,
+                const _Allocator& __alloc)
+      : flat_multimap(__ctor_uses_allocator_empty_tag{}, __alloc, __comp) {
+    insert(sorted_equivalent, __first, __last);
+  }
+
+  template <class _InputIterator, class _Allocator>
+    requires(__has_input_iterator_category<_InputIterator>::value && __allocator_ctor_constraint<_Allocator>)
+  _LIBCPP_HIDE_FROM_ABI
+  flat_multimap(sorted_equivalent_t, _InputIterator __first, _InputIterator __last, const _Allocator& __alloc)
+      : flat_multimap(__ctor_uses_allocator_empty_tag{}, __alloc) {
+    insert(sorted_equivalent, __first, __last);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI flat_multimap(initializer_list<value_type> __il, const key_compare& __comp = key_compare())
+      : flat_multimap(__il.begin(), __il.end(), __comp) {}
+
+  template <class _Allocator>
+    requires __allocator_ctor_constraint<_Allocator>
+  _LIBCPP_HIDE_FROM_ABI
+  flat_multimap(initializer_list<value_type> __il, const key_compare& __comp, const _Allocator& __alloc)
+      : flat_multimap(__il.begin(), __il.end(), __comp, __alloc) {}
+
+  template <class _Allocator>
+    requires __allocator_ctor_constraint<_Allocator>
+  _LIBCPP_HIDE_FROM_ABI flat_multimap(initializer_list<value_type> __il, const _Allocator& __alloc)
+      : flat_multimap(__il.begin(), __il.end(), __alloc) {}
+
+  _LIBCPP_HIDE_FROM_ABI
+  flat_multimap(sorted_equivalent_t, initializer_list<value_type> __il, const key_compare& __comp = key_compare())
+      : flat_multimap(sorted_equivalent, __il.begin(), __il.end(), __comp) {}
+
+  template <class _Allocator>
+    requires __allocator_ctor_constraint<_Allocator>
+  _LIBCPP_HIDE_FROM_ABI flat_multimap(
+      sorted_equivalent_t, initializer_list<value_type> __il, const key_compare& __comp, const _Allocator& __alloc)
+      : flat_multimap(sorted_equivalent, __il.begin(), __il.end(), __comp, __alloc) {}
+
+  template <class _Allocator>
+    requires __allocator_ctor_constraint<_Allocator>
+  _LIBCPP_HIDE_FROM_ABI flat_multimap(sorted_equivalent_t, initializer_list<value_type> __il, const _Allocator& __alloc)
+      : flat_multimap(sorted_equivalent, __il.begin(), __il.end(), __alloc) {}
+
+  _LIBCPP_HIDE_FROM_ABI flat_multimap& operator=(initializer_list<value_type> __il) {
+    clear();
+    insert(__il);
+    return *this;
+  }
+
+  // copy/move assignment are not specified in the spec (defaulted)
+  // but move assignment can potentially leave moved from object in an inconsistent
+  // state if an exception is thrown
+  _LIBCPP_HIDE_FROM_ABI flat_multimap& operator=(const flat_multimap&) = default;
+
+  _LIBCPP_HIDE_FROM_ABI flat_multimap& operator=(flat_multimap&& __other) noexcept(
+      is_nothrow_move_assignable_v<_KeyContainer> && is_nothrow_move_assignable_v<_MappedContainer> &&
+      is_nothrow_move_assignable_v<_Compare>) {
+    auto __clear_other_guard = std::__make_scope_guard([&]() noexcept { __other.clear() /* noexcept */; });
+    auto __clear_self_guard  = std::__make_exception_guard([&]() noexcept { clear() /* noexcept */; });
+    __containers_            = std::move(__other.__containers_);
+    __compare_               = std::move(__other.__compare_);
+    __clear_self_guard.__complete();
+    return *this;
+  }
+
+  // iterators
+  _LIBCPP_HIDE_FROM_ABI iterator begin() noexcept {
+    return iterator(__containers_.keys.begin(), __containers_.values.begin());
+  }
+
+  _LIBCPP_HIDE_FROM_ABI const_iterator begin() const noexcept {
+    return const_iterator(__containers_.keys.begin(), __containers_.values.begin());
+  }
+
+  _LIBCPP_HIDE_FROM_ABI iterator end() noexcept {
+    return iterator(__containers_.keys.end(), __containers_.values.end());
+  }
+
+  _LIBCPP_HIDE_FROM_ABI const_iterator end() const noexcept {
+    return const_iterator(__containers_.keys.end(), __containers_.values.end());
+  }
+
+  _LIBCPP_HIDE_FROM_ABI reverse_iterator rbegin() noexcept { return reverse_iterator(end()); }
+  _LIBCPP_HIDE_FROM_ABI const_reverse_iterator rbegin() const noexcept { return const_reverse_iterator(end()); }
+  _LIBCPP_HIDE_FROM_ABI reverse_iterator rend() noexcept { return reverse_iterator(begin()); }
+  _LIBCPP_HIDE_FROM_ABI const_reverse_iterator rend() const noexcept { return const_reverse_iterator(begin()); }
+
+  _LIBCPP_HIDE_FROM_ABI const_iterator cbegin() const noexcept { return begin(); }
+  _LIBCPP_HIDE_FROM_ABI const_iterator cend() const noexcept { return end(); }
+  _LIBCPP_HIDE_FROM_ABI const_reverse_iterator crbegin() const noexcept { return const_reverse_iterator(end()); }
+  _LIBCPP_HIDE_FROM_ABI const_reverse_iterator crend() const noexcept { return const_reverse_iterator(begin()); }
+
+  // [flat.map.capacity], capacity
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI bool empty() const noexcept { return __containers_.keys.empty(); }
+
+  _LIBCPP_HIDE_FROM_ABI size_type size() const noexcept { return __containers_.keys.size(); }
+
+  _LIBCPP_HIDE_FROM_ABI size_type max_size() const noexcept {
+    return std::min<size_type>(__containers_.keys.max_size(), __containers_.values.max_size());
+  }
+
+  // [flat.map.modifiers], modifiers
+  template <class... _Args>
+    requires is_constructible_v<pair<key_type, mapped_type>, _Args...> && is_move_constructible_v<key_type> &&
+             is_move_constructible_v<mapped_type>
+  _LIBCPP_HIDE_FROM_ABI iterator emplace(_Args&&... __args) {
+    std::pair<key_type, mapped_type> __pair(std::forward<_Args>(__args)...);
+    auto __key_it    = ranges::upper_bound(__containers_.keys, __pair.first, __compare_);
+    auto __mapped_it = __corresponding_mapped_it(*this, __key_it);
+
+    return __flat_map_utils::__emplace_exact_pos(
+        *this, std::move(__key_it), std::move(__mapped_it), std::move(__pair.first), std::move(__pair.second));
+  }
+
+  template <class... _Args>
+    requires is_constructible_v<pair<key_type, mapped_type>, _Args...>
+  _LIBCPP_HIDE_FROM_ABI iterator emplace_hint(const_iterator __hint, _Args&&... __args) {
+    std::pair<key_type, mapped_type> __pair(std::forward<_Args>(__args)...);
+
+    auto __prev_larger  = __hint != cbegin() && __compare_(__pair.first, (__hint - 1)->first);
+    auto __next_smaller = __hint != cend() && __compare_(__hint->first, __pair.first);
+
+    auto __hint_distance = __hint.__key_iter_ - __containers_.keys.cbegin();
+    auto __key_iter      = __containers_.keys.begin() + __hint_distance;
+    auto __mapped_iter   = __containers_.values.begin() + __hint_distance;
+
+    if (!__prev_larger && !__next_smaller) [[likely]] {
+      // hint correct, just use exact hint iterators
+    } else if (__prev_larger && !__next_smaller) {
+      // the hint position is more to the right than the key should have been.
+      // we want to emplace the element to a position as right as possible
+      // e.g. Insert new element "2" in the following range
+      // 1, 1, 2, 2, 2, 3, 4, 6
+      //                   ^
+      //                   |
+      //                  hint
+      // We want to insert "2" after the last existing "2"
+      __key_iter    = ranges::upper_bound(__containers_.keys.begin(), __key_iter, __pair.first, __compare_);
+      __mapped_iter = __corresponding_mapped_it(*this, __key_iter);
+    } else {
+      _LIBCPP_ASSERT_INTERNAL(!__prev_larger && __next_smaller, "this means that the multimap is not sorted");
+
+      // the hint position is more to the left than the key should have been.
+      // we want to emplace the element to a position as left as possible
+      //  1, 1, 2, 2, 2, 3, 4, 6
+      //  ^
+      //  |
+      // hint
+      // We want to insert "2" before the first existing "2"
+      __key_iter    = ranges::lower_bound(__key_iter, __containers_.keys.end(), __pair.first, __compare_);
+      __mapped_iter = __corresponding_mapped_it(*this, __key_iter);
+    }
+    return __flat_map_utils::__emplace_exact_pos(
+        *this, __key_iter, __mapped_iter, std::move(__pair.first), std::move(__pair.second));
+  }
+
+  _LIBCPP_HIDE_FROM_ABI iterator insert(const value_type& __x) { return emplace(__x); }
+
+  _LIBCPP_HIDE_FROM_ABI iterator insert(value_type&& __x) { return emplace(std::move(__x)); }
+
+  _LIBCPP_HIDE_FROM_ABI iterator insert(const_iterator __hint, const value_type& __x) {
+    return emplace_hint(__hint, __x);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI iterator insert(const_iterator __hint, value_type&& __x) {
+    return emplace_hint(__hint, std::move(__x));
+  }
+
+  template <class _PairLike>
+    requires is_constructible_v<pair<key_type, mapped_type>, _PairLike>
+  _LIBCPP_HIDE_FROM_ABI iterator insert(_PairLike&& __x) {
+    return emplace(std::forward<_PairLike>(__x));
+  }
+
+  template <class _PairLike>
+    requires is_constructible_v<pair<key_type, mapped_type>, _PairLike>
+  _LIBCPP_HIDE_FROM_ABI iterator insert(const_iterator __hint, _PairLike&& __x) {
+    return emplace_hint(__hint, std::forward<_PairLike>(__x));
+  }
+
+  template <class _InputIterator>
+    requires __has_input_iterator_category<_InputIterator>::value
+  _LIBCPP_HIDE_FROM_ABI void insert(_InputIterator __first, _InputIterator __last) {
+    if constexpr (sized_sentinel_for<_InputIterator, _InputIterator>) {
+      __reserve(__last - __first);
+    }
+    __append_sort_merge</*WasSorted = */ false>(std::move(__first), std::move(__last));
+  }
+
+  template <class _InputIterator>
+    requires __has_input_iterator_category<_InputIterator>::value
+  _LIBCPP_HIDE_FROM_ABI void insert(sorted_equivalent_t, _InputIterator __first, _InputIterator __last) {
+    if constexpr (sized_sentinel_for<_InputIterator, _InputIterator>) {
+      __reserve(__last - __first);
+    }
+
+    __append_sort_merge</*WasSorted = */ true>(std::move(__first), std::move(__last));
+  }
+
+  template <_ContainerCompatibleRange<value_type> _Range>
+  _LIBCPP_HIDE_FROM_ABI void insert_range(_Range&& __range) {
+    if constexpr (ranges::sized_range<_Range>) {
+      __reserve(ranges::size(__range));
+    }
+
+    __append_sort_merge</*WasSorted = */ false>(ranges::begin(__range), ranges::end(__range));
+  }
+
+  _LIBCPP_HIDE_FROM_ABI void insert(initializer_list<value_type> __il) { insert(__il.begin(), __il.end()); }
+
+  _LIBCPP_HIDE_FROM_ABI void insert(sorted_equivalent_t, initializer_list<value_type> __il) {
+    insert(sorted_equivalent, __il.begin(), __il.end());
+  }
+
+  _LIBCPP_HIDE_FROM_ABI containers extract() && {
+    auto __guard = std::__make_scope_guard([&]() noexcept { clear() /* noexcept */; });
+    auto __ret   = std::move(__containers_);
+    return __ret;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI void replace(key_container_type&& __key_cont, mapped_container_type&& __mapped_cont) {
+    _LIBCPP_ASSERT_VALID_INPUT_RANGE(
+        __key_cont.size() == __mapped_cont.size(), "flat_multimap keys and mapped containers have different size");
+
+    _LIBCPP_ASSERT_SEMANTIC_REQUIREMENT(__is_sorted(__key_cont), "Key container is not sorted");
+    auto __guard         = std::__make_exception_guard([&]() noexcept { clear() /* noexcept */; });
+    __containers_.keys   = std::move(__key_cont);
+    __containers_.values = std::move(__mapped_cont);
+    __guard.__complete();
+  }
+
+  _LIBCPP_HIDE_FROM_ABI iterator erase(iterator __position) {
+    return __erase(__position.__key_iter_, __position.__mapped_iter_);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI iterator erase(const_iterator __position) {
+    return __erase(__position.__key_iter_, __position.__mapped_iter_);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI size_type erase(const key_type& __x) {
+    auto [__first, __last] = equal_range(__x);
+    auto __res             = __last - __first;
+    erase(__first, __last);
+    return __res;
+  }
+
+  template <class _Kp>
+    requires(__is_compare_transparent && !is_convertible_v<_Kp &&, iterator> &&
+             !is_convertible_v<_Kp &&, const_iterator>)
+  _LIBCPP_HIDE_FROM_ABI size_type erase(_Kp&& __x) {
+    auto [__first, __last] = equal_range(__x);
+    auto __res             = __last - __first;
+    erase(__first, __last);
+    return __res;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI iterator erase(const_iterator __first, const_iterator __last) {
+    auto __on_failure = std::__make_exception_guard([&]() noexcept { clear() /* noexcept */; });
+    auto __key_it     = __containers_.keys.erase(__first.__key_iter_, __last.__key_iter_);
+    auto __mapped_it  = __containers_.values.erase(__first.__mapped_iter_, __last.__mapped_iter_);
+    __on_failure.__complete();
+    return iterator(std::move(__key_it), std::move(__mapped_it));
+  }
+
+  _LIBCPP_HIDE_FROM_ABI void swap(flat_multimap& __y) noexcept {
+    // warning: The spec has unconditional noexcept, which means that
+    // if any of the following functions throw an exception,
+    // std::terminate will be called
+    ranges::swap(__compare_, __y.__compare_);
+    ranges::swap(__containers_.keys, __y.__containers_.keys);
+    ranges::swap(__containers_.values, __y.__containers_.values);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI void clear() noexcept {
+    __containers_.keys.clear();
+    __containers_.values.clear();
+  }
+
+  // observers
+  _LIBCPP_HIDE_FROM_ABI key_compare key_comp() const { return __compare_; }
+  _LIBCPP_HIDE_FROM_ABI value_compare value_comp() const { return value_compare(__compare_); }
+
+  _LIBCPP_HIDE_FROM_ABI const key_container_type& keys() const noexcept { return __containers_.keys; }
+  _LIBCPP_HIDE_FROM_ABI const mapped_container_type& values() const noexcept { return __containers_.values; }
+
+  // map operations
+  _LIBCPP_HIDE_FROM_ABI iterator find(const key_type& __x) { return __find_impl(*this, __x); }
+
+  _LIBCPP_HIDE_FROM_ABI const_iterator find(const key_type& __x) const { return __find_impl(*this, __x); }
+
+  template <class _Kp>
+    requires __is_compare_transparent
+  _LIBCPP_HIDE_FROM_ABI iterator find(const _Kp& __x) {
+    return __find_impl(*this, __x);
+  }
+
+  template <class _Kp>
+    requires __is_compare_transparent
+  _LIBCPP_HIDE_FROM_ABI const_iterator find(const _Kp& __x) const {
+    return __find_impl(*this, __x);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI size_type count(const key_type& __x) const {
+    auto [__first, __last] = equal_range(__x);
+    return __last - __first;
+  }
+
+  template <class _Kp>
+    requires __is_compare_transparent
+  _LIBCPP_HIDE_FROM_ABI size_type count(const _Kp& __x) const {
+    auto [__first, __last] = equal_range(__x);
+    return __last - __first;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI bool contains(const key_type& __x) const { return find(__x) != end(); }
+
+  template <class _Kp>
+    requires __is_compare_transparent
+  _LIBCPP_HIDE_FROM_ABI bool contains(const _Kp& __x) const {
+    return find(__x) != end();
+  }
+
+  _LIBCPP_HIDE_FROM_ABI iterator lower_bound(const key_type& __x) { return __lower_bound<iterator>(*this, __x); }
+
+  _LIBCPP_HIDE_FROM_ABI const_iterator lower_bound(const key_type& __x) const {
+    return __lower_bound<const_iterator>(*this, __x);
+  }
+
+  template <class _Kp>
+    requires __is_compare_transparent
+  _LIBCPP_HIDE_FROM_ABI iterator lower_bound(const _Kp& __x) {
+    return __lower_bound<iterator>(*this, __x);
+  }
+
+  template <class _Kp>
+    requires __is_compare_transparent
+  _LIBCPP_HIDE_FROM_ABI const_iterator lower_bound(const _Kp& __x) const {
+    return __lower_bound<const_iterator>(*this, __x);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI iterator upper_bound(const key_type& __x) { return __upper_bound<iterator>(*this, __x); }
+
+  _LIBCPP_HIDE_FROM_ABI const_iterator upper_bound(const key_type& __x) const {
+    return __upper_bound<const_iterator>(*this, __x);
+  }
+
+  template <class _Kp>
+    requires __is_compare_transparent
+  _LIBCPP_HIDE_FROM_ABI iterator upper_bound(const _Kp& __x) {
+    return __upper_bound<iterator>(*this, __x);
+  }
+
+  template <class _Kp>
+    requires __is_compare_transparent
+  _LIBCPP_HIDE_FROM_ABI const_iterator upper_bound(const _Kp& __x) const {
+    return __upper_bound<const_iterator>(*this, __x);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI pair<iterator, iterator> equal_range(const key_type& __x) {
+    return __equal_range_impl(*this, __x);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI pair<const_iterator, const_iterator> equal_range(const key_type& __x) const {
+    return __equal_range_impl(*this, __x);
+  }
+
+  template <class _Kp>
+    requires __is_compare_transparent
+  _LIBCPP_HIDE_FROM_ABI pair<iterator, iterator> equal_range(const _Kp& __x) {
+    return __equal_range_impl(*this, __x);
+  }
+  template <class _Kp>
+    requires __is_compare_transparent
+  _LIBCPP_HIDE_FROM_ABI pair<const_iterator, const_iterator> equal_range(const _Kp& __x) const {
+    return __equal_range_impl(*this, __x);
+  }
+
+  friend _LIBCPP_HIDE_FROM_ABI bool operator==(const flat_multimap& __x, const flat_multimap& __y) {
+    return ranges::equal(__x, __y);
+  }
+
+  friend _LIBCPP_HIDE_FROM_ABI auto operator<=>(const flat_multimap& __x, const flat_multimap& __y) {
+    return std::lexicographical_compare_three_way(
+        __x.begin(), __x.end(), __y.begin(), __y.end(), std::__synth_three_way);
+  }
+
+  friend _LIBCPP_HIDE_FROM_ABI void swap(flat_multimap& __x, flat_multimap& __y) noexcept { __x.swap(__y); }
+
+private:
+  struct __ctor_uses_allocator_tag {
+    explicit _LIBCPP_HIDE_FROM_ABI __ctor_uses_allocator_tag() = default;
+  };
+  struct __ctor_uses_allocator_empty_tag {
+    explicit _LIBCPP_HIDE_FROM_ABI __ctor_uses_allocator_empty_tag() = default;
+  };
+
+  template <class _Allocator, class _KeyCont, class _MappedCont, class... _CompArg>
+    requires __allocator_ctor_constraint<_Allocator>
+  _LIBCPP_HIDE_FROM_ABI
+  flat_multimap(__ctor_uses_allocator_tag,
+                const _Allocator& __alloc,
+                _KeyCont&& __key_cont,
+                _MappedCont&& __mapped_cont,
+                _CompArg&&... __comp)
+      : __containers_{.keys = std::make_obj_using_allocator<key_container_type>(
+                          __alloc, std::forward<_KeyCont>(__key_cont)),
+                      .values = std::make_obj_using_allocator<mapped_container_type>(
+                          __alloc, std::forward<_MappedCont>(__mapped_cont))},
+        __compare_(std::forward<_CompArg>(__comp)...) {}
+
+  template <class _Allocator, class... _CompArg>
+    requires __allocator_ctor_constraint<_Allocator>
+  _LIBCPP_HIDE_FROM_ABI flat_multimap(__ctor_uses_allocator_empty_tag, const _Allocator& __alloc, _CompArg&&... __comp)
+      : __containers_{.keys   = std::make_obj_using_allocator<key_container_type>(__alloc),
+                      .values = std::make_obj_using_allocator<mapped_container_type>(__alloc)},
+        __compare_(std::forward<_CompArg>(__comp)...) {}
+
+  _LIBCPP_HIDE_FROM_ABI bool __is_sorted(auto&& __key_container) const {
+    return ranges::is_sorted(__key_container, __compare_);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI void __sort() {
+    auto __zv = ranges::views::zip(__containers_.keys, __containers_.values);
+    ranges::sort(__zv, __compare_, [](const auto& __p) -> decltype(auto) { return std::get<0>(__p); });
+  }
+
+  template <class _Self, class _KeyIter>
+  _LIBCPP_HIDE_FROM_ABI static auto __corresponding_mapped_it(_Self&& __self, _KeyIter&& __key_iter) {
+    return __self.__containers_.values.begin() +
+           static_cast<ranges::range_difference_t<mapped_container_type>>(
+               ranges::distance(__self.__containers_.keys.begin(), __key_iter));
+  }
+
+  template <bool _WasSorted, class _InputIterator, class _Sentinel>
+  _LIBCPP_HIDE_FROM_ABI void __append_sort_merge(_InputIterator __first, _Sentinel __last) {
+    auto __on_failure     = std::__make_exception_guard([&]() noexcept { clear() /* noexcept */; });
+    size_t __num_appended = __flat_map_utils::__append(*this, std::move(__first), std::move(__last));
+    if (__num_appended != 0) {
+      auto __zv                  = ranges::views::zip(__containers_.keys, __containers_.values);
+      auto __append_start_offset = __containers_.keys.size() - __num_appended;
+      auto __end                 = __zv.end();
+      auto __compare_key         = [this](const auto& __p1, const auto& __p2) {
+        return __compare_(std::get<0>(__p1), std::get<0>(__p2));
+      };
+      if constexpr (!_WasSorted) {
+        ranges::sort(__zv.begin() + __append_start_offset, __end, __compare_key);
+      } else {
+        _LIBCPP_ASSERT_SEMANTIC_REQUIREMENT(
+            __is_sorted(__containers_.keys | ranges::views::drop(__append_start_offset)),
+            "Key container is not sorted");
+      }
+      ranges::inplace_merge(__zv.begin(), __zv.begin() + __append_start_offset, __end, __compare_key);
+    }
+    __on_failure.__complete();
+  }
+
+  template <class _Self, class _Kp>
+  _LIBCPP_HIDE_FROM_ABI static auto __find_impl(_Self&& __self, const _Kp& __key) {
+    auto __it   = __self.lower_bound(__key);
+    auto __last = __self.end();
+    if (__it == __last || __self.__compare_(__key, __it->first)) {
+      return __last;
+    }
+    return __it;
+  }
+
+  template <class _Self, class _Kp>
+  _LIBCPP_HIDE_FROM_ABI static auto __equal_range_impl(_Self&& __self, const _Kp& __key) {
+    auto [__key_first, __key_last] = ranges::equal_range(__self.__containers_.keys, __key, __self.__compare_);
+
+    using __iterator_type = ranges::iterator_t<decltype(__self)>;
+    return std::make_pair(__iterator_type(__key_first, __corresponding_mapped_it(__self, __key_first)),
+                          __iterator_type(__key_last, __corresponding_mapped_it(__self, __key_last)));
+  }
+
+  template <class _Res, class _Self, class _Kp>
+  _LIBCPP_HIDE_FROM_ABI static _Res __lower_bound(_Self&& __self, _Kp& __x) {
+    auto __key_iter    = ranges::lower_bound(__self.__containers_.keys, __x, __self.__compare_);
+    auto __mapped_iter = __corresponding_mapped_it(__self, __key_iter);
+    return _Res(std::move(__key_iter), std::move(__mapped_iter));
+  }
+
+  template <class _Res, class _Self, class _Kp>
+  _LIBCPP_HIDE_FROM_ABI static _Res __upper_bound(_Self&& __self, _Kp& __x) {
+    auto __key_iter    = ranges::upper_bound(__self.__containers_.keys, __x, __self.__compare_);
+    auto __mapped_iter = __corresponding_mapped_it(__self, __key_iter);
+    return _Res(std::move(__key_iter), std::move(__mapped_iter));
+  }
+
+  _LIBCPP_HIDE_FROM_ABI void __reserve(size_t __size) {
+    if constexpr (requires { __containers_.keys.reserve(__size); }) {
+      __containers_.keys.reserve(__size);
+    }
+
+    if constexpr (requires { __containers_.values.reserve(__size); }) {
+      __containers_.values.reserve(__size);
+    }
+  }
+
+  template <class _KIter, class _MIter>
+  _LIBCPP_HIDE_FROM_ABI iterator __erase(_KIter __key_iter_to_remove, _MIter __mapped_iter_to_remove) {
+    auto __on_failure  = std::__make_exception_guard([&]() noexcept { clear() /* noexcept */; });
+    auto __key_iter    = __containers_.keys.erase(__key_iter_to_remove);
+    auto __mapped_iter = __containers_.values.erase(__mapped_iter_to_remove);
+    __on_failure.__complete();
+    return iterator(std::move(__key_iter), std::move(__mapped_iter));
+  }
+
+  template <class _Key2, class _Tp2, class _Compare2, class _KeyContainer2, class _MappedContainer2, class _Predicate>
+  friend typename flat_multimap<_Key2, _Tp2, _Compare2, _KeyContainer2, _MappedContainer2>::size_type
+  erase_if(flat_multimap<_Key2, _Tp2, _Compare2, _KeyContainer2, _MappedContainer2>&, _Predicate);
+
+  friend __flat_map_utils;
+
+  containers __containers_;
+  _LIBCPP_NO_UNIQUE_ADDRESS key_compare __compare_;
+
+  struct __key_equiv {
+    _LIBCPP_HIDE_FROM_ABI __key_equiv(key_compare __c) : __comp_(__c) {}
+    _LIBCPP_HIDE_FROM_ABI bool operator()(const_reference __x, const_reference __y) const {
+      return !__comp_(std::get<0>(__x), std::get<0>(__y)) && !__comp_(std::get<0>(__y), std::get<0>(__x));
+    }
+    key_compare __comp_;
+  };
+};
+
+template <class _KeyContainer, class _MappedContainer, class _Compare = less<typename _KeyContainer::value_type>>
+  requires(!__is_allocator<_Compare>::value && !__is_allocator<_KeyContainer>::value &&
+           !__is_allocator<_MappedContainer>::value &&
+           is_invocable_v<const _Compare&,
+                          const typename _KeyContainer::value_type&,
+                          const typename _KeyContainer::value_type&>)
+flat_multimap(_KeyContainer, _MappedContainer, _Compare = _Compare())
+    -> flat_multimap<typename _KeyContainer::value_type,
+                     typename _MappedContainer::value_type,
+                     _Compare,
+                     _KeyContainer,
+                     _MappedContainer>;
+
+template <class _KeyContainer, class _MappedContainer, class _Allocator>
+  requires(uses_allocator_v<_KeyContainer, _Allocator> && uses_allocator_v<_MappedContainer, _Allocator> &&
+           !__is_allocator<_KeyContainer>::value && !__is_allocator<_MappedContainer>::value)
+flat_multimap(_KeyContainer, _MappedContainer, _Allocator)
+    -> flat_multimap<typename _KeyContainer::value_type,
+                     typename _MappedContainer::value_type,
+                     less<typename _KeyContainer::value_type>,
+                     _KeyContainer,
+                     _MappedContainer>;
+
+template <class _KeyContainer, class _MappedContainer, class _Compare, class _Allocator>
+  requires(!__is_allocator<_Compare>::value && !__is_allocator<_KeyContainer>::value &&
+           !__is_allocator<_MappedContainer>::value && uses_allocator_v<_KeyContainer, _Allocator> &&
+           uses_allocator_v<_MappedContainer, _Allocator> &&
+           is_invocable_v<const _Compare&,
+                          const typename _KeyContainer::value_type&,
+                          const typename _KeyContainer::value_type&>)
+flat_multimap(_KeyContainer, _MappedContainer, _Compare, _Allocator)
+    -> flat_multimap<typename _KeyContainer::value_type,
+                     typename _MappedContainer::value_type,
+                     _Compare,
+                     _KeyContainer,
+                     _MappedContainer>;
+
+template <class _KeyContainer, class _MappedContainer, class _Compare = less<typename _KeyContainer::value_type>>
+  requires(!__is_allocator<_Compare>::value && !__is_allocator<_KeyContainer>::value &&
+           !__is_allocator<_MappedContainer>::value &&
+           is_invocable_v<const _Compare&,
+                          const typename _KeyContainer::value_type&,
+                          const typename _KeyContainer::value_type&>)
+flat_multimap(sorted_equivalent_t, _KeyContainer, _MappedContainer, _Compare = _Compare())
+    -> flat_multimap<typename _KeyContainer::value_type,
+                     typename _MappedContainer::value_type,
+                     _Compare,
+                     _KeyContainer,
+                     _MappedContainer>;
+
+template <class _KeyContainer, class _MappedContainer, class _Allocator>
+  requires(uses_allocator_v<_KeyContainer, _Allocator> && uses_allocator_v<_MappedContainer, _Allocator> &&
+           !__is_allocator<_KeyContainer>::value && !__is_allocator<_MappedContainer>::value)
+flat_multimap(sorted_equivalent_t, _KeyContainer, _MappedContainer, _Allocator)
+    -> flat_multimap<typename _KeyContainer::value_type,
+                     typename _MappedContainer::value_type,
+                     less<typename _KeyContainer::value_type>,
+                     _KeyContainer,
+                     _MappedContainer>;
+
+template <class _KeyContainer, class _MappedContainer, class _Compare, class _Allocator>
+  requires(!__is_allocator<_Compare>::value && !__is_allocator<_KeyContainer>::value &&
+           !__is_allocator<_MappedContainer>::value && uses_allocator_v<_KeyContainer, _Allocator> &&
+           uses_allocator_v<_MappedContainer, _Allocator> &&
+           is_invocable_v<const _Compare&,
+                          const typename _KeyContainer::value_type&,
+                          const typename _KeyContainer::value_type&>)
+flat_multimap(sorted_equivalent_t, _KeyContainer, _MappedContainer, _Compare, _Allocator)
+    -> flat_multimap<typename _KeyContainer::value_type,
+                     typename _MappedContainer::value_type,
+                     _Compare,
+                     _KeyContainer,
+                     _MappedContainer>;
+
+template <class _InputIterator, class _Compare = less<__iter_key_type<_InputIterator>>>
+  requires(__has_input_iterator_category<_InputIterator>::value && !__is_allocator<_Compare>::value)
+flat_multimap(_InputIterator, _InputIterator, _Compare = _Compare())
+    -> flat_multimap<__iter_key_type<_InputIterator>, __iter_mapped_type<_InputIterator>, _Compare>;
+
+template <class _InputIterator, class _Compare = less<__iter_key_type<_InputIterator>>>
+  requires(__has_input_iterator_category<_InputIterator>::value && !__is_allocator<_Compare>::value)
+flat_multimap(sorted_equivalent_t, _InputIterator, _InputIterator, _Compare = _Compare())
+    -> flat_multimap<__iter_key_type<_InputIterator>, __iter_mapped_type<_InputIterator>, _Compare>;
+
+template <ranges::input_range _Range,
+          class _Compare   = less<__range_key_type<_Range>>,
+          class _Allocator = allocator<byte>,
+          class            = __enable_if_t<!__is_allocator<_Compare>::value && __is_allocator<_Allocator>::value>>
+flat_multimap(from_range_t, _Range&&, _Compare = _Compare(), _Allocator = _Allocator()) -> flat_multimap<
+    __range_key_type<_Range>,
+    __range_mapped_type<_Range>,
+    _Compare,
+    vector<__range_key_type<_Range>, __allocator_traits_rebind_t<_Allocator, __range_key_type<_Range>>>,
+    vector<__range_mapped_type<_Range>, __allocator_traits_rebind_t<_Allocator, __range_mapped_type<_Range>>>>;
+
+template <ranges::input_range _Range, class _Allocator, class = __enable_if_t<__is_allocator<_Allocator>::value>>
+flat_multimap(from_range_t, _Range&&, _Allocator) -> flat_multimap<
+    __range_key_type<_Range>,
+    __range_mapped_type<_Range>,
+    less<__range_key_type<_Range>>,
+    vector<__range_key_type<_Range>, __allocator_traits_rebind_t<_Allocator, __range_key_type<_Range>>>,
+    vector<__range_mapped_type<_Range>, __allocator_traits_rebind_t<_Allocator, __range_mapped_type<_Range>>>>;
+
+template <class _Key, class _Tp, class _Compare = less<_Key>>
+  requires(!__is_allocator<_Compare>::value)
+flat_multimap(initializer_list<pair<_Key, _Tp>>, _Compare = _Compare()) -> flat_multimap<_Key, _Tp, _Compare>;
+
+template <class _Key, class _Tp, class _Compare = less<_Key>>
+  requires(!__is_allocator<_Compare>::value)
+flat_multimap(sorted_equivalent_t, initializer_list<pair<_Key, _Tp>>, _Compare = _Compare())
+    -> flat_multimap<_Key, _Tp, _Compare>;
+
+template <class _Key, class _Tp, class _Compare, class _KeyContainer, class _MappedContainer, class _Allocator>
+struct uses_allocator<flat_multimap<_Key, _Tp, _Compare, _KeyContainer, _MappedContainer>, _Allocator>
+    : bool_constant<uses_allocator_v<_KeyContainer, _Allocator> && uses_allocator_v<_MappedContainer, _Allocator>> {};
+
+template <class _Key, class _Tp, class _Compare, class _KeyContainer, class _MappedContainer, class _Predicate>
+_LIBCPP_HIDE_FROM_ABI typename flat_multimap<_Key, _Tp, _Compare, _KeyContainer, _MappedContainer>::size_type
+erase_if(flat_multimap<_Key, _Tp, _Compare, _KeyContainer, _MappedContainer>& __flat_multimap, _Predicate __pred) {
+  auto __zv     = ranges::views::zip(__flat_multimap.__containers_.keys, __flat_multimap.__containers_.values);
+  auto __first  = __zv.begin();
+  auto __last   = __zv.end();
+  auto __guard  = std::__make_exception_guard([&] { __flat_multimap.clear(); });
+  auto __it     = std::remove_if(__first, __last, [&](auto&& __zipped) -> bool {
+    using _Ref = typename flat_multimap<_Key, _Tp, _Compare, _KeyContainer, _MappedContainer>::const_reference;
+    return __pred(_Ref(std::get<0>(__zipped), std::get<1>(__zipped)));
+  });
+  auto __res    = __last - __it;
+  auto __offset = __it - __first;
+
+  const auto __erase_container = [&](auto& __cont) { __cont.erase(__cont.begin() + __offset, __cont.end()); };
+
+  __erase_container(__flat_multimap.__containers_.keys);
+  __erase_container(__flat_multimap.__containers_.values);
+
+  __guard.__complete();
+  return __res;
+}
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP_STD_VER >= 23
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___FLAT_MAP_FLAT_MULTIMAP_H
diff --git a/libcxx/include/__flat_map/sorted_equivalent.h b/libcxx/include/__flat_map/sorted_equivalent.h
new file mode 100644
index 0000000000000..1db935cc6ee75
--- /dev/null
+++ b/libcxx/include/__flat_map/sorted_equivalent.h
@@ -0,0 +1,31 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+#ifndef _LIBCPP___FLAT_MAP_SORTED_EQUIVALENT_H
+#define _LIBCPP___FLAT_MAP_SORTED_EQUIVALENT_H
+
+#include <__config>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+#if _LIBCPP_STD_VER >= 23
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+struct sorted_equivalent_t {
+  explicit sorted_equivalent_t() = default;
+};
+inline constexpr sorted_equivalent_t sorted_equivalent{};
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP_STD_VER >= 23
+
+#endif // _LIBCPP___FLAT_MAP_SORTED_EQUIVALENT_H
diff --git a/libcxx/include/__flat_map/utils.h b/libcxx/include/__flat_map/utils.h
new file mode 100644
index 0000000000000..acb7dca7ffe96
--- /dev/null
+++ b/libcxx/include/__flat_map/utils.h
@@ -0,0 +1,103 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___FLAT_MAP_UTILS_H
+#define _LIBCPP___FLAT_MAP_UTILS_H
+
+#include <__config>
+#include <__type_traits/container_traits.h>
+#include <__utility/exception_guard.h>
+#include <__utility/forward.h>
+#include <__utility/move.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+#if _LIBCPP_STD_VER >= 23
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+// These utilities are defined in a class instead of a namespace so that this class can be befriended more easily.
+struct __flat_map_utils {
+  // Emplace a {key: value} into a flat_{multi}map, at the exact position that
+  // __it_key and __it_mapped point to, assuming that the key is not already present in the map.
+  // When an exception is thrown during the emplacement, the function will try its best to
+  // roll back the changes it made to the map. If it cannot roll back the changes, it will
+  // clear the map.
+  template <class _Map, class _IterK, class _IterM, class _KeyArg, class... _MArgs>
+  _LIBCPP_HIDE_FROM_ABI static typename _Map::iterator __emplace_exact_pos(
+      _Map& __map, _IterK&& __it_key, _IterM&& __it_mapped, _KeyArg&& __key, _MArgs&&... __mapped_args) {
+    auto __on_key_failed = std::__make_exception_guard([&]() noexcept {
+      using _KeyContainer = typename _Map::key_container_type;
+      if constexpr (__container_traits<_KeyContainer>::__emplacement_has_strong_exception_safety_guarantee) {
+        // Nothing to roll back!
+      } else {
+        // we need to clear both because we don't know the state of our keys anymore
+        __map.clear() /* noexcept */;
+      }
+    });
+    auto __key_it        = __map.__containers_.keys.emplace(__it_key, std::forward<_KeyArg>(__key));
+    __on_key_failed.__complete();
+
+    auto __on_value_failed = std::__make_exception_guard([&]() noexcept {
+      using _MappedContainer = typename _Map::mapped_container_type;
+      if constexpr (!__container_traits<_MappedContainer>::__emplacement_has_strong_exception_safety_guarantee) {
+        // we need to clear both because we don't know the state of our values anymore
+        __map.clear() /* noexcept */;
+      } else {
+        // In this case, we know the values are just like before we attempted emplacement,
+        // and we also know that the keys have been emplaced successfully. Just roll back the keys.
+#  if _LIBCPP_HAS_EXCEPTIONS
+        try {
+#  endif // _LIBCPP_HAS_EXCEPTIONS
+          __map.__containers_.keys.erase(__key_it);
+#  if _LIBCPP_HAS_EXCEPTIONS
+        } catch (...) {
+          // Now things are funky for real. We're failing to rollback the keys.
+          // Just give up and clear the whole thing.
+          //
+          // Also, swallow the exception that happened during the rollback and let the
+          // original value-emplacement exception propagate normally.
+          __map.clear() /* noexcept */;
+        }
+#  endif // _LIBCPP_HAS_EXCEPTIONS
+      }
+    });
+    auto __mapped_it = __map.__containers_.values.emplace(__it_mapped, std::forward<_MArgs>(__mapped_args)...);
+    __on_value_failed.__complete();
+
+    return typename _Map::iterator(std::move(__key_it), std::move(__mapped_it));
+  }
+
+  // TODO: We could optimize this, see
+  // https://github.com/llvm/llvm-project/issues/108624
+  template <class _Map, class _InputIterator, class _Sentinel>
+  _LIBCPP_HIDE_FROM_ABI static typename _Map::size_type
+  __append(_Map& __map, _InputIterator __first, _Sentinel __last) {
+    typename _Map::size_type __num_appended = 0;
+    for (; __first != __last; ++__first) {
+      typename _Map::value_type __kv = *__first;
+      __map.__containers_.keys.insert(__map.__containers_.keys.end(), std::move(__kv.first));
+      __map.__containers_.values.insert(__map.__containers_.values.end(), std::move(__kv.second));
+      ++__num_appended;
+    }
+    return __num_appended;
+  }
+};
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP_STD_VER >= 23
+
+_LIBCPP_POP_MACROS
+
+#endif // #define _LIBCPP___FLAT_MAP_UTILS_H
diff --git a/libcxx/include/__format/buffer.h b/libcxx/include/__format/buffer.h
index 9509f19e16724..c88b7f3222010 100644
--- a/libcxx/include/__format/buffer.h
+++ b/libcxx/include/__format/buffer.h
@@ -14,7 +14,7 @@
 #include <__algorithm/fill_n.h>
 #include <__algorithm/max.h>
 #include <__algorithm/min.h>
-#include <__algorithm/ranges_copy_n.h>
+#include <__algorithm/ranges_copy.h>
 #include <__algorithm/transform.h>
 #include <__algorithm/unwrap_iter.h>
 #include <__concepts/same_as.h>
@@ -29,14 +29,16 @@
 #include <__iterator/wrap_iter.h>
 #include <__memory/addressof.h>
 #include <__memory/allocate_at_least.h>
+#include <__memory/allocator.h>
 #include <__memory/allocator_traits.h>
 #include <__memory/construct_at.h>
-#include <__memory/ranges_construct_at.h>
+#include <__memory/destroy.h>
 #include <__memory/uninitialized_algorithms.h>
 #include <__type_traits/add_pointer.h>
 #include <__type_traits/conditional.h>
 #include <__utility/exception_guard.h>
 #include <__utility/move.h>
+#include <stdexcept>
 #include <string_view>
 
 #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
@@ -52,24 +54,147 @@ _LIBCPP_BEGIN_NAMESPACE_STD
 
 namespace __format {
 
+// A helper to limit the total size of code units written.
+class _LIBCPP_HIDE_FROM_ABI __max_output_size {
+public:
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI explicit __max_output_size(size_t __max_size) : __max_size_{__max_size} {}
+
+  // This function adjusts the size of a (bulk) write operations. It ensures the
+  // number of code units written by a __output_buffer never exceeds
+  // __max_size_ code units.
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI size_t __write_request(size_t __code_units) {
+    size_t __result =
+        __code_units_written_ < __max_size_ ? std::min(__code_units, __max_size_ - __code_units_written_) : 0;
+    __code_units_written_ += __code_units;
+    return __result;
+  }
+
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI size_t __code_units_written() const noexcept { return __code_units_written_; }
+
+private:
+  size_t __max_size_;
+  // The code units that would have been written if there was no limit.
+  // format_to_n returns this value.
+  size_t __code_units_written_{0};
+};
+
 /// A "buffer" that handles writing to the proper iterator.
 ///
 /// This helper is used together with the @ref back_insert_iterator to offer
 /// type-erasure for the formatting functions. This reduces the number to
 /// template instantiations.
+///
+/// The design is the following:
+/// - There is an external object that connects the buffer to the output.
+/// - This buffer object:
+///   - inherits publicly from this class.
+///   - has a static or dynamic buffer.
+///   - has a static member function to make space in its buffer write
+///     operations. This can be done by increasing the size of the internal
+///     buffer or by writing the contents of the buffer to the output iterator.
+///
+///     This member function is a constructor argument, so its name is not
+///     fixed. The code uses the name __prepare_write.
+/// - The number of output code units can be limited by a __max_output_size
+///   object. This is used in format_to_n This object:
+///   - Contains the maximum number of code units to be written.
+///   - Contains the number of code units that are requested to be written.
+///     This number is returned to the user of format_to_n.
+///   - The write functions call the object's __request_write member function.
+///     This function:
+///     - Updates the number of code units that are requested to be written.
+///     - Returns the number of code units that can be written without
+///       exceeding the maximum number of code units to be written.
+///
+/// Documentation for the buffer usage members:
+/// - __ptr_
+///   The start of the buffer.
+/// - __capacity_
+///   The number of code units that can be written. This means
+///   [__ptr_, __ptr_ + __capacity_) is a valid range to write to.
+/// - __size_
+///   The number of code units written in the buffer. The next code unit will
+///   be written at __ptr_ + __size_. This __size_ may NOT contain the total
+///   number of code units written by the __output_buffer. Whether or not it
+///   does depends on the sub-class used. Typically the total number of code
+///   units written is not interesting. It is interesting for format_to_n which
+///   has its own way to track this number.
+///
+/// Documentation for the modifying buffer operations:
+/// The subclasses have a function with the following signature:
+///
+///   static void __prepare_write(
+///     __output_buffer<_CharT>& __buffer, size_t __code_units);
+///
+/// This function is called when a write function writes more code units than
+/// the buffer's available space. When an __max_output_size object is provided
+/// the number of code units is the number of code units returned from
+/// __max_output_size::__request_write function.
+///
+/// - The __buffer contains *this. Since the class containing this function
+///   inherits from __output_buffer it's safe to cast it to the subclass being
+///   used.
+/// - The __code_units is the number of code units the caller will write + 1.
+///   - This value does not take the available space of the buffer into account.
+///   - The push_back function is more efficient when writing before resizing,
+///     this means the buffer should always have room for one code unit. Hence
+///     the + 1 is the size.
+/// - When the function returns there is room for at least one additional code
+///   unit. There is no requirement there is room for __code_units code units:
+///   - The class has some "bulk" operations. For example, __copy which copies
+///     the contents of a basic_string_view to the output. If the sub-class has
+///     a fixed size buffer the size of the basic_string_view may be larger
+///     than the buffer. In that case it's impossible to honor the requested
+///     size.
+///   - When the buffer has room for at least one code unit the function may be
+///     a no-op.
+/// - When the function makes space for more code units it uses one for these
+///   functions to signal the change:
+///   - __buffer_flushed()
+///     - This function is typically used for a fixed sized buffer.
+///     - The current contents of [__ptr_, __ptr_ + __size_) have been
+///       processed.
+///     - __ptr_ remains unchanged.
+///     - __capacity_ remains unchanged.
+///     - __size_ will be set to 0.
+///   - __buffer_moved(_CharT* __ptr, size_t __capacity)
+///     - This function is typically used for a dynamic sized buffer. There the
+///       location of the buffer changes due to reallocations.
+///     - __ptr_ will be set to __ptr. (This value may be the old value of
+///       __ptr_).
+///     - __capacity_ will be set to __capacity. (This value may be the old
+///       value of __capacity_).
+///     - __size_ remains unchanged,
+///     - The range [__ptr, __ptr + __size_) contains the original data of the
+///       range [__ptr_, __ptr_ + __size_).
+///
+/// The push_back function expects a valid buffer and a capacity of at least 1.
+/// This means:
+/// - The class is constructed with a valid buffer,
+/// - __buffer_moved is called with a valid buffer is used before the first
+///   write operation,
+/// - no write function is ever called, or
+/// - the class is constructed with a __max_output_size object with __max_size 0.
+///
+/// The latter option allows formatted_size to use the output buffer without
+/// ever writing anything to the buffer.
 template <__fmt_char_type _CharT>
 class _LIBCPP_TEMPLATE_VIS __output_buffer {
 public:
-  using value_type = _CharT;
+  using value_type _LIBCPP_NODEBUG           = _CharT;
+  using __prepare_write_type _LIBCPP_NODEBUG = void (*)(__output_buffer<_CharT>&, size_t);
 
-  template <class _Tp>
-  _LIBCPP_HIDE_FROM_ABI explicit __output_buffer(_CharT* __ptr, size_t __capacity, _Tp* __obj)
-      : __ptr_(__ptr),
-        __capacity_(__capacity),
-        __flush_([](_CharT* __p, size_t __n, void* __o) { static_cast<_Tp*>(__o)->__flush(__p, __n); }),
-        __obj_(__obj) {}
+  [[nodiscard]]
+  _LIBCPP_HIDE_FROM_ABI explicit __output_buffer(_CharT* __ptr, size_t __capacity, __prepare_write_type __function)
+      : __output_buffer{__ptr, __capacity, __function, nullptr} {}
 
-  _LIBCPP_HIDE_FROM_ABI void __reset(_CharT* __ptr, size_t __capacity) {
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI explicit __output_buffer(
+      _CharT* __ptr, size_t __capacity, __prepare_write_type __function, __max_output_size* __max_output_size)
+      : __ptr_(__ptr), __capacity_(__capacity), __prepare_write_(__function), __max_output_size_(__max_output_size) {}
+
+  _LIBCPP_HIDE_FROM_ABI void __buffer_flushed() { __size_ = 0; }
+
+  _LIBCPP_HIDE_FROM_ABI void __buffer_moved(_CharT* __ptr, size_t __capacity) {
     __ptr_      = __ptr;
     __capacity_ = __capacity;
   }
@@ -78,12 +203,18 @@ class _LIBCPP_TEMPLATE_VIS __output_buffer {
 
   // Used in std::back_insert_iterator.
   _LIBCPP_HIDE_FROM_ABI void push_back(_CharT __c) {
+    if (__max_output_size_ && __max_output_size_->__write_request(1) == 0)
+      return;
+
+    _LIBCPP_ASSERT_INTERNAL(
+        __ptr_ && __size_ < __capacity_ && __available() >= 1, "attempted to write outside the buffer");
+
     __ptr_[__size_++] = __c;
 
     // Profiling showed flushing after adding is more efficient than flushing
     // when entering the function.
     if (__size_ == __capacity_)
-      __flush();
+      __prepare_write(0);
   }
 
   /// Copies the input __str to the buffer.
@@ -104,25 +235,20 @@ class _LIBCPP_TEMPLATE_VIS __output_buffer {
     // upper case. For integral these strings are short.
     // TODO FMT Look at the improvements above.
     size_t __n = __str.size();
-
-    __flush_on_overflow(__n);
-    if (__n < __capacity_) { //  push_back requires the buffer to have room for at least one character (so use <).
-      std::copy_n(__str.data(), __n, std::addressof(__ptr_[__size_]));
-      __size_ += __n;
-      return;
+    if (__max_output_size_) {
+      __n = __max_output_size_->__write_request(__n);
+      if (__n == 0)
+        return;
     }
 
-    // The output doesn't fit in the internal buffer.
-    // Copy the data in "__capacity_" sized chunks.
-    _LIBCPP_ASSERT_INTERNAL(__size_ == 0, "the buffer should be flushed by __flush_on_overflow");
     const _InCharT* __first = __str.data();
     do {
-      size_t __chunk = std::min(__n, __capacity_);
+      __prepare_write(__n);
+      size_t __chunk = std::min(__n, __available());
       std::copy_n(__first, __chunk, std::addressof(__ptr_[__size_]));
-      __size_ = __chunk;
+      __size_ += __chunk;
       __first += __chunk;
       __n -= __chunk;
-      __flush();
     } while (__n);
   }
 
@@ -136,121 +262,59 @@ class _LIBCPP_TEMPLATE_VIS __output_buffer {
     _LIBCPP_ASSERT_INTERNAL(__first <= __last, "not a valid range");
 
     size_t __n = static_cast<size_t>(__last - __first);
-    __flush_on_overflow(__n);
-    if (__n < __capacity_) { //  push_back requires the buffer to have room for at least one character (so use <).
-      std::transform(__first, __last, std::addressof(__ptr_[__size_]), std::move(__operation));
-      __size_ += __n;
-      return;
+    if (__max_output_size_) {
+      __n = __max_output_size_->__write_request(__n);
+      if (__n == 0)
+        return;
     }
 
-    // The output doesn't fit in the internal buffer.
-    // Transform the data in "__capacity_" sized chunks.
-    _LIBCPP_ASSERT_INTERNAL(__size_ == 0, "the buffer should be flushed by __flush_on_overflow");
     do {
-      size_t __chunk = std::min(__n, __capacity_);
+      __prepare_write(__n);
+      size_t __chunk = std::min(__n, __available());
       std::transform(__first, __first + __chunk, std::addressof(__ptr_[__size_]), __operation);
-      __size_ = __chunk;
+      __size_ += __chunk;
       __first += __chunk;
       __n -= __chunk;
-      __flush();
     } while (__n);
   }
 
   /// A \c fill_n wrapper.
   _LIBCPP_HIDE_FROM_ABI void __fill(size_t __n, _CharT __value) {
-    __flush_on_overflow(__n);
-    if (__n < __capacity_) { //  push_back requires the buffer to have room for at least one character (so use <).
-      std::fill_n(std::addressof(__ptr_[__size_]), __n, __value);
-      __size_ += __n;
-      return;
+    if (__max_output_size_) {
+      __n = __max_output_size_->__write_request(__n);
+      if (__n == 0)
+        return;
     }
 
-    // The output doesn't fit in the internal buffer.
-    // Fill the buffer in "__capacity_" sized chunks.
-    _LIBCPP_ASSERT_INTERNAL(__size_ == 0, "the buffer should be flushed by __flush_on_overflow");
     do {
-      size_t __chunk = std::min(__n, __capacity_);
+      __prepare_write(__n);
+      size_t __chunk = std::min(__n, __available());
       std::fill_n(std::addressof(__ptr_[__size_]), __chunk, __value);
-      __size_ = __chunk;
+      __size_ += __chunk;
       __n -= __chunk;
-      __flush();
     } while (__n);
   }
 
-  _LIBCPP_HIDE_FROM_ABI void __flush() {
-    __flush_(__ptr_, __size_, __obj_);
-    __size_ = 0;
-  }
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI size_t __capacity() const { return __capacity_; }
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI size_t __size() const { return __size_; }
 
 private:
   _CharT* __ptr_;
   size_t __capacity_;
   size_t __size_{0};
-  void (*__flush_)(_CharT*, size_t, void*);
-  void* __obj_;
-
-  /// Flushes the buffer when the output operation would overflow the buffer.
-  ///
-  /// A simple approach for the overflow detection would be something along the
-  /// lines:
-  /// \code
-  /// // The internal buffer is large enough.
-  /// if (__n <= __capacity_) {
-  ///   // Flush when we really would overflow.
-  ///   if (__size_ + __n >= __capacity_)
-  ///     __flush();
-  ///   ...
-  /// }
-  /// \endcode
-  ///
-  /// This approach works for all cases but one:
-  /// A __format_to_n_buffer_base where \ref __enable_direct_output is true.
-  /// In that case the \ref __capacity_ of the buffer changes during the first
-  /// \ref __flush. During that operation the output buffer switches from its
-  /// __writer_ to its __storage_. The \ref __capacity_ of the former depends
-  /// on the value of n, of the latter is a fixed size. For example:
-  /// - a format_to_n call with a 10'000 char buffer,
-  /// - the buffer is filled with 9'500 chars,
-  /// - adding 1'000 elements would overflow the buffer so the buffer gets
-  ///   changed and the \ref __capacity_ decreases from 10'000 to
-  ///   __buffer_size (256 at the time of writing).
-  ///
-  /// This means that the \ref __flush for this class may need to copy a part of
-  /// the internal buffer to the proper output. In this example there will be
-  /// 500 characters that need this copy operation.
-  ///
-  /// Note it would be more efficient to write 500 chars directly and then swap
-  /// the buffers. This would make the code more complex and \ref format_to_n is
-  /// not the most common use case. Therefore the optimization isn't done.
-  _LIBCPP_HIDE_FROM_ABI void __flush_on_overflow(size_t __n) {
-    if (__size_ + __n >= __capacity_)
-      __flush();
-  }
-};
-
-/// A storage using an internal buffer.
-///
-/// This storage is used when writing a single element to the output iterator
-/// is expensive.
-template <__fmt_char_type _CharT>
-class _LIBCPP_TEMPLATE_VIS __internal_storage {
-public:
-  _LIBCPP_HIDE_FROM_ABI _CharT* __begin() { return __buffer_; }
+  void (*__prepare_write_)(__output_buffer<_CharT>&, size_t);
+  __max_output_size* __max_output_size_;
 
-  static constexpr size_t __buffer_size = 256 / sizeof(_CharT);
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI size_t __available() const { return __capacity_ - __size_; }
 
-private:
-  _CharT __buffer_[__buffer_size];
+  _LIBCPP_HIDE_FROM_ABI void __prepare_write(size_t __code_units) {
+    // Always have space for one additional code unit. This is a precondition of the push_back function.
+    __code_units += 1;
+    if (__available() < __code_units)
+      __prepare_write_(*this, __code_units + 1);
+  }
 };
 
-/// A storage writing directly to the storage.
-///
-/// This requires the storage to be a contiguous buffer of \a _CharT.
-/// Since the output is directly written to the underlying storage this class
-/// is just an empty class.
-template <__fmt_char_type _CharT>
-class _LIBCPP_TEMPLATE_VIS __direct_storage {};
-
 template <class _OutIt, class _CharT>
 concept __enable_direct_output =
     __fmt_char_type<_CharT> &&
@@ -259,40 +323,6 @@ concept __enable_direct_output =
      // `#ifdef`.
      || same_as<_OutIt, __wrap_iter<_CharT*>>);
 
-/// Write policy for directly writing to the underlying output.
-template <class _OutIt, __fmt_char_type _CharT>
-class _LIBCPP_TEMPLATE_VIS __writer_direct {
-public:
-  _LIBCPP_HIDE_FROM_ABI explicit __writer_direct(_OutIt __out_it) : __out_it_(__out_it) {}
-
-  _LIBCPP_HIDE_FROM_ABI _OutIt __out_it() { return __out_it_; }
-
-  _LIBCPP_HIDE_FROM_ABI void __flush(_CharT*, size_t __n) {
-    // _OutIt can be a __wrap_iter<CharT*>. Therefore the original iterator
-    // is adjusted.
-    __out_it_ += __n;
-  }
-
-private:
-  _OutIt __out_it_;
-};
-
-/// Write policy for copying the buffer to the output.
-template <class _OutIt, __fmt_char_type _CharT>
-class _LIBCPP_TEMPLATE_VIS __writer_iterator {
-public:
-  _LIBCPP_HIDE_FROM_ABI explicit __writer_iterator(_OutIt __out_it) : __out_it_{std::move(__out_it)} {}
-
-  _LIBCPP_HIDE_FROM_ABI _OutIt __out_it() && { return std::move(__out_it_); }
-
-  _LIBCPP_HIDE_FROM_ABI void __flush(_CharT* __ptr, size_t __n) {
-    __out_it_ = std::ranges::copy_n(__ptr, __n, std::move(__out_it_)).out;
-  }
-
-private:
-  _OutIt __out_it_;
-};
-
 /// Concept to see whether a \a _Container is insertable.
 ///
 /// The concept is used to validate whether multiple calls to a
@@ -310,196 +340,220 @@ concept __insertable =
 /// Extract the container type of a \ref back_insert_iterator.
 template <class _It>
 struct _LIBCPP_TEMPLATE_VIS __back_insert_iterator_container {
-  using type = void;
+  using type _LIBCPP_NODEBUG = void;
 };
 
 template <__insertable _Container>
 struct _LIBCPP_TEMPLATE_VIS __back_insert_iterator_container<back_insert_iterator<_Container>> {
-  using type = _Container;
+  using type _LIBCPP_NODEBUG = _Container;
 };
 
-/// Write policy for inserting the buffer in a container.
-template <class _Container>
-class _LIBCPP_TEMPLATE_VIS __writer_container {
+// A dynamically growing buffer.
+template <__fmt_char_type _CharT>
+class _LIBCPP_TEMPLATE_VIS __allocating_buffer : public __output_buffer<_CharT> {
 public:
-  using _CharT _LIBCPP_NODEBUG = typename _Container::value_type;
+  __allocating_buffer(const __allocating_buffer&)            = delete;
+  __allocating_buffer& operator=(const __allocating_buffer&) = delete;
 
-  _LIBCPP_HIDE_FROM_ABI explicit __writer_container(back_insert_iterator<_Container> __out_it)
-      : __container_{__out_it.__get_container()} {}
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI __allocating_buffer() : __allocating_buffer{nullptr} {}
 
-  _LIBCPP_HIDE_FROM_ABI auto __out_it() { return std::back_inserter(*__container_); }
+  [[nodiscard]]
+  _LIBCPP_HIDE_FROM_ABI explicit __allocating_buffer(__max_output_size* __max_output_size)
+      : __output_buffer<_CharT>{__small_buffer_, __buffer_size_, __prepare_write, __max_output_size} {}
 
-  _LIBCPP_HIDE_FROM_ABI void __flush(_CharT* __ptr, size_t __n) {
-    __container_->insert(__container_->end(), __ptr, __ptr + __n);
+  _LIBCPP_HIDE_FROM_ABI ~__allocating_buffer() {
+    if (__ptr_ != __small_buffer_)
+      _Alloc{}.deallocate(__ptr_, this->__capacity());
   }
 
-private:
-  _Container* __container_;
-};
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI basic_string_view<_CharT> __view() { return {__ptr_, this->__size()}; }
 
-/// Selects the type of the writer used for the output iterator.
-template <class _OutIt, class _CharT>
-class _LIBCPP_TEMPLATE_VIS __writer_selector {
-  using _Container _LIBCPP_NODEBUG = typename __back_insert_iterator_container<_OutIt>::type;
+private:
+  using _Alloc _LIBCPP_NODEBUG = allocator<_CharT>;
 
-public:
-  using type =
-      conditional_t<!same_as<_Container, void>,
-                    __writer_container<_Container>,
-                    conditional_t<__enable_direct_output<_OutIt, _CharT>,
-                                  __writer_direct<_OutIt, _CharT>,
-                                  __writer_iterator<_OutIt, _CharT>>>;
-};
+  // Since allocating is expensive the class has a small internal buffer. When
+  // its capacity is exceeded a dynamic buffer will be allocated.
+  static constexpr size_t __buffer_size_ = 256;
+  _CharT __small_buffer_[__buffer_size_];
 
-/// The generic formatting buffer.
-template <class _OutIt, __fmt_char_type _CharT>
-  requires(output_iterator<_OutIt, const _CharT&>)
-class _LIBCPP_TEMPLATE_VIS __format_buffer {
-  using _Storage _LIBCPP_NODEBUG =
-      conditional_t<__enable_direct_output<_OutIt, _CharT>, __direct_storage<_CharT>, __internal_storage<_CharT>>;
+  _CharT* __ptr_{__small_buffer_};
 
-public:
-  _LIBCPP_HIDE_FROM_ABI explicit __format_buffer(_OutIt __out_it)
-    requires(same_as<_Storage, __internal_storage<_CharT>>)
-      : __output_(__storage_.__begin(), __storage_.__buffer_size, this), __writer_(std::move(__out_it)) {}
+  _LIBCPP_HIDE_FROM_ABI void __grow_buffer(size_t __capacity) {
+    if (__capacity < __buffer_size_)
+      return;
 
-  _LIBCPP_HIDE_FROM_ABI explicit __format_buffer(_OutIt __out_it)
-    requires(same_as<_Storage, __direct_storage<_CharT>>)
-      : __output_(std::__unwrap_iter(__out_it), size_t(-1), this), __writer_(std::move(__out_it)) {}
+    _LIBCPP_ASSERT_INTERNAL(__capacity > this->__capacity(), "the buffer must grow");
 
-  _LIBCPP_HIDE_FROM_ABI auto __make_output_iterator() { return __output_.__make_output_iterator(); }
+    // _CharT is an implicit lifetime type so can be used without explicit
+    // construction or destruction.
+    _Alloc __alloc;
+    auto __result = std::__allocate_at_least(__alloc, __capacity);
+    std::copy_n(__ptr_, this->__size(), __result.ptr);
+    if (__ptr_ != __small_buffer_)
+      __alloc.deallocate(__ptr_, this->__capacity());
 
-  _LIBCPP_HIDE_FROM_ABI void __flush(_CharT* __ptr, size_t __n) { __writer_.__flush(__ptr, __n); }
+    __ptr_ = __result.ptr;
+    this->__buffer_moved(__ptr_, __result.count);
+  }
 
-  _LIBCPP_HIDE_FROM_ABI _OutIt __out_it() && {
-    __output_.__flush();
-    return std::move(__writer_).__out_it();
+  _LIBCPP_HIDE_FROM_ABI void __prepare_write(size_t __size_hint) {
+    __grow_buffer(std::max<size_t>(this->__capacity() + __size_hint, this->__capacity() * 1.6));
   }
 
-private:
-  _LIBCPP_NO_UNIQUE_ADDRESS _Storage __storage_;
-  __output_buffer<_CharT> __output_;
-  typename __writer_selector<_OutIt, _CharT>::type __writer_;
+  _LIBCPP_HIDE_FROM_ABI static void __prepare_write(__output_buffer<_CharT>& __buffer, size_t __size_hint) {
+    static_cast<__allocating_buffer<_CharT>&>(__buffer).__prepare_write(__size_hint);
+  }
 };
 
-/// A buffer that counts the number of insertions.
-///
-/// Since \ref formatted_size only needs to know the size, the output itself is
-/// discarded.
-template <__fmt_char_type _CharT>
-class _LIBCPP_TEMPLATE_VIS __formatted_size_buffer {
+// A buffer that directly writes to the underlying buffer.
+template <class _OutIt, __fmt_char_type _CharT>
+class _LIBCPP_TEMPLATE_VIS __direct_iterator_buffer : public __output_buffer<_CharT> {
 public:
-  _LIBCPP_HIDE_FROM_ABI auto __make_output_iterator() { return __output_.__make_output_iterator(); }
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI explicit __direct_iterator_buffer(_OutIt __out_it)
+      : __direct_iterator_buffer{__out_it, nullptr} {}
 
-  _LIBCPP_HIDE_FROM_ABI void __flush(const _CharT*, size_t __n) { __size_ += __n; }
+  [[nodiscard]]
+  _LIBCPP_HIDE_FROM_ABI explicit __direct_iterator_buffer(_OutIt __out_it, __max_output_size* __max_output_size)
+      : __output_buffer<_CharT>{std::__unwrap_iter(__out_it), __buffer_size, __prepare_write, __max_output_size},
+        __out_it_(__out_it) {}
 
-  _LIBCPP_HIDE_FROM_ABI size_t __result() && {
-    __output_.__flush();
-    return __size_;
-  }
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI _OutIt __out_it() && { return __out_it_ + this->__size(); }
 
 private:
-  __internal_storage<_CharT> __storage_;
-  __output_buffer<_CharT> __output_{__storage_.__begin(), __storage_.__buffer_size, this};
-  size_t __size_{0};
-};
+  // The function format_to expects a buffer large enough for the output. The
+  // function format_to_n has its own helper class that restricts the number of
+  // write options. So this function class can pretend to have an infinite
+  // buffer.
+  static constexpr size_t __buffer_size = -1;
+
+  _OutIt __out_it_;
 
-/// The base of a buffer that counts and limits the number of insertions.
-template <class _OutIt, __fmt_char_type _CharT, bool>
-  requires(output_iterator<_OutIt, const _CharT&>)
-struct _LIBCPP_TEMPLATE_VIS __format_to_n_buffer_base {
-  using _Size _LIBCPP_NODEBUG = iter_difference_t<_OutIt>;
+  _LIBCPP_HIDE_FROM_ABI static void
+  __prepare_write([[maybe_unused]] __output_buffer<_CharT>& __buffer, [[maybe_unused]] size_t __size_hint) {
+    std::__throw_length_error("__direct_iterator_buffer");
+  }
+};
 
+// A buffer that writes its output to the end of a container.
+template <class _OutIt, __fmt_char_type _CharT>
+class _LIBCPP_TEMPLATE_VIS __container_inserter_buffer : public __output_buffer<_CharT> {
 public:
-  _LIBCPP_HIDE_FROM_ABI explicit __format_to_n_buffer_base(_OutIt __out_it, _Size __max_size)
-      : __writer_(std::move(__out_it)), __max_size_(std::max(_Size(0), __max_size)) {}
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI explicit __container_inserter_buffer(_OutIt __out_it)
+      : __container_inserter_buffer{__out_it, nullptr} {}
 
-  _LIBCPP_HIDE_FROM_ABI void __flush(_CharT* __ptr, size_t __n) {
-    if (_Size(__size_) <= __max_size_)
-      __writer_.__flush(__ptr, std::min(_Size(__n), __max_size_ - __size_));
-    __size_ += __n;
+  [[nodiscard]]
+  _LIBCPP_HIDE_FROM_ABI explicit __container_inserter_buffer(_OutIt __out_it, __max_output_size* __max_output_size)
+      : __output_buffer<_CharT>{__small_buffer_, __buffer_size, __prepare_write, __max_output_size},
+        __container_{__out_it.__get_container()} {}
+
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI auto __out_it() && {
+    __container_->insert(__container_->end(), __small_buffer_, __small_buffer_ + this->__size());
+    return std::back_inserter(*__container_);
   }
 
-protected:
-  __internal_storage<_CharT> __storage_;
-  __output_buffer<_CharT> __output_{__storage_.__begin(), __storage_.__buffer_size, this};
-  typename __writer_selector<_OutIt, _CharT>::type __writer_;
+private:
+  typename __back_insert_iterator_container<_OutIt>::type* __container_;
+
+  // This class uses a fixed size buffer and appends the elements in
+  // __buffer_size chunks. An alternative would be to use an allocating buffer
+  // and append the output in a single write operation. Benchmarking showed no
+  // performance difference.
+  static constexpr size_t __buffer_size = 256;
+  _CharT __small_buffer_[__buffer_size];
+
+  _LIBCPP_HIDE_FROM_ABI void __prepare_write() {
+    __container_->insert(__container_->end(), __small_buffer_, __small_buffer_ + this->__size());
+    this->__buffer_flushed();
+  }
 
-  _Size __max_size_;
-  _Size __size_{0};
+  _LIBCPP_HIDE_FROM_ABI static void
+  __prepare_write(__output_buffer<_CharT>& __buffer, [[maybe_unused]] size_t __size_hint) {
+    static_cast<__container_inserter_buffer<_OutIt, _CharT>&>(__buffer).__prepare_write();
+  }
 };
 
-/// The base of a buffer that counts and limits the number of insertions.
-///
-/// This version is used when \c __enable_direct_output<_OutIt, _CharT> == true.
-///
-/// This class limits the size available to the direct writer so it will not
-/// exceed the maximum number of code units.
+// A buffer that writes to an iterator.
+//
+// Unlike the __container_inserter_buffer this class' performance does benefit
+// from allocating and then inserting.
 template <class _OutIt, __fmt_char_type _CharT>
-  requires(output_iterator<_OutIt, const _CharT&>)
-class _LIBCPP_TEMPLATE_VIS __format_to_n_buffer_base<_OutIt, _CharT, true> {
-  using _Size _LIBCPP_NODEBUG = iter_difference_t<_OutIt>;
-
+class _LIBCPP_TEMPLATE_VIS __iterator_buffer : public __allocating_buffer<_CharT> {
 public:
-  _LIBCPP_HIDE_FROM_ABI explicit __format_to_n_buffer_base(_OutIt __out_it, _Size __max_size)
-      : __output_(std::__unwrap_iter(__out_it), __max_size, this),
-        __writer_(std::move(__out_it)),
-        __max_size_(__max_size) {
-    if (__max_size <= 0) [[unlikely]]
-      __output_.__reset(__storage_.__begin(), __storage_.__buffer_size);
-  }
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI explicit __iterator_buffer(_OutIt __out_it)
+      : __allocating_buffer<_CharT>{}, __out_it_{std::move(__out_it)} {}
 
-  _LIBCPP_HIDE_FROM_ABI void __flush(_CharT* __ptr, size_t __n) {
-    // A __flush to the direct writer happens in the following occasions:
-    // - The format function has written the maximum number of allowed code
-    //   units. At this point it's no longer valid to write to this writer. So
-    //   switch to the internal storage. This internal storage doesn't need to
-    //   be written anywhere so the __flush for that storage writes no output.
-    // - Like above, but the next "mass write" operation would overflow the
-    //   buffer. In that case the buffer is pre-emptively switched. The still
-    //   valid code units will be written separately.
-    // - The format_to_n function is finished. In this case there's no need to
-    //   switch the buffer, but for simplicity the buffers are still switched.
-    // When the __max_size <= 0 the constructor already switched the buffers.
-    if (__size_ == 0 && __ptr != __storage_.__begin()) {
-      __writer_.__flush(__ptr, __n);
-      __output_.__reset(__storage_.__begin(), __storage_.__buffer_size);
-    } else if (__size_ < __max_size_) {
-      // Copies a part of the internal buffer to the output up to n characters.
-      // See __output_buffer<_CharT>::__flush_on_overflow for more information.
-      _Size __s = std::min(_Size(__n), __max_size_ - __size_);
-      std::copy_n(__ptr, __s, __writer_.__out_it());
-      __writer_.__flush(__ptr, __s);
-    }
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI explicit __iterator_buffer(_OutIt __out_it, __max_output_size* __max_output_size)
+      : __allocating_buffer<_CharT>{__max_output_size}, __out_it_{std::move(__out_it)} {}
 
-    __size_ += __n;
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI auto __out_it() && {
+    return std::ranges::copy(this->__view(), std::move(__out_it_)).out;
   }
 
-protected:
-  __internal_storage<_CharT> __storage_;
-  __output_buffer<_CharT> __output_;
-  __writer_direct<_OutIt, _CharT> __writer_;
+private:
+  _OutIt __out_it_;
+};
+
+// Selects the type of the buffer used for the output iterator.
+template <class _OutIt, __fmt_char_type _CharT>
+class _LIBCPP_TEMPLATE_VIS __buffer_selector {
+  using _Container _LIBCPP_NODEBUG = __back_insert_iterator_container<_OutIt>::type;
 
-  _Size __max_size_;
-  _Size __size_{0};
+public:
+  using type _LIBCPP_NODEBUG =
+      conditional_t<!same_as<_Container, void>,
+                    __container_inserter_buffer<_OutIt, _CharT>,
+                    conditional_t<__enable_direct_output<_OutIt, _CharT>,
+                                  __direct_iterator_buffer<_OutIt, _CharT>,
+                                  __iterator_buffer<_OutIt, _CharT>>>;
 };
 
-/// The buffer that counts and limits the number of insertions.
+// A buffer that counts and limits the number of insertions.
 template <class _OutIt, __fmt_char_type _CharT>
-  requires(output_iterator<_OutIt, const _CharT&>)
-struct _LIBCPP_TEMPLATE_VIS __format_to_n_buffer final
-    : public __format_to_n_buffer_base< _OutIt, _CharT, __enable_direct_output<_OutIt, _CharT>> {
-  using _Base _LIBCPP_NODEBUG = __format_to_n_buffer_base<_OutIt, _CharT, __enable_direct_output<_OutIt, _CharT>>;
-  using _Size _LIBCPP_NODEBUG = iter_difference_t<_OutIt>;
+class _LIBCPP_TEMPLATE_VIS __format_to_n_buffer : private __buffer_selector<_OutIt, _CharT>::type {
+public:
+  using _Base _LIBCPP_NODEBUG = __buffer_selector<_OutIt, _CharT>::type;
+
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI __format_to_n_buffer(_OutIt __out_it, iter_difference_t<_OutIt> __n)
+      : _Base{std::move(__out_it), std::addressof(__max_output_size_)},
+        __max_output_size_{__n < 0 ? size_t{0} : static_cast<size_t>(__n)} {}
+
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI auto __make_output_iterator() { return _Base::__make_output_iterator(); }
+
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI format_to_n_result<_OutIt> __result() && {
+    return {static_cast<_Base&&>(*this).__out_it(),
+            static_cast<iter_difference_t<_OutIt>>(__max_output_size_.__code_units_written())};
+  }
+
+private:
+  __max_output_size __max_output_size_;
+};
 
+// A buffer that counts the number of insertions.
+//
+// Since formatted_size only needs to know the size, the output itself is
+// discarded.
+template <__fmt_char_type _CharT>
+class _LIBCPP_TEMPLATE_VIS __formatted_size_buffer : private __output_buffer<_CharT> {
 public:
-  _LIBCPP_HIDE_FROM_ABI explicit __format_to_n_buffer(_OutIt __out_it, _Size __max_size)
-      : _Base(std::move(__out_it), __max_size) {}
-  _LIBCPP_HIDE_FROM_ABI auto __make_output_iterator() { return this->__output_.__make_output_iterator(); }
+  using _Base _LIBCPP_NODEBUG = __output_buffer<_CharT>;
+
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI __formatted_size_buffer()
+      : _Base{nullptr, 0, __prepare_write, std::addressof(__max_output_size_)} {}
+
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI auto __make_output_iterator() { return _Base::__make_output_iterator(); }
+
+  // This function does not need to be r-value qualified, however this is
+  // consistent with similar objects.
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI size_t __result() && { return __max_output_size_.__code_units_written(); }
+
+private:
+  __max_output_size __max_output_size_{0};
 
-  _LIBCPP_HIDE_FROM_ABI format_to_n_result<_OutIt> __result() && {
-    this->__output_.__flush();
-    return {std::move(this->__writer_).__out_it(), this->__size_};
+  _LIBCPP_HIDE_FROM_ABI static void
+  __prepare_write([[maybe_unused]] __output_buffer<_CharT>& __buffer, [[maybe_unused]] size_t __size_hint) {
+    // Note this function does not satisfy the requirement of giving a 1 code unit buffer.
+    _LIBCPP_ASSERT_INTERNAL(
+        false, "Since __max_output_size_.__max_size_ == 0 there should never be call to this function.");
   }
 };
 
@@ -526,11 +580,11 @@ class _LIBCPP_TEMPLATE_VIS __retarget_buffer {
   using _Alloc _LIBCPP_NODEBUG = allocator<_CharT>;
 
 public:
-  using value_type = _CharT;
+  using value_type _LIBCPP_NODEBUG = _CharT;
 
   struct __iterator {
-    using difference_type = ptrdiff_t;
-    using value_type      = _CharT;
+    using difference_type _LIBCPP_NODEBUG = ptrdiff_t;
+    using value_type _LIBCPP_NODEBUG      = _CharT;
 
     _LIBCPP_HIDE_FROM_ABI constexpr explicit __iterator(__retarget_buffer& __buffer)
         : __buffer_(std::addressof(__buffer)) {}
@@ -566,7 +620,7 @@ class _LIBCPP_TEMPLATE_VIS __retarget_buffer {
   }
 
   _LIBCPP_HIDE_FROM_ABI ~__retarget_buffer() {
-    ranges::destroy_n(__ptr_, __size_);
+    std::destroy_n(__ptr_, __size_);
     allocator_traits<_Alloc>::deallocate(__alloc_, __ptr_, __capacity_);
   }
 
@@ -631,7 +685,7 @@ class _LIBCPP_TEMPLATE_VIS __retarget_buffer {
     // guard is optimized away so there is no runtime overhead.
     std::uninitialized_move_n(__ptr_, __size_, __result.ptr);
     __guard.__complete();
-    ranges::destroy_n(__ptr_, __size_);
+    std::destroy_n(__ptr_, __size_);
     allocator_traits<_Alloc>::deallocate(__alloc_, __ptr_, __capacity_);
 
     __ptr_      = __result.ptr;
diff --git a/libcxx/include/__format/escaped_output_table.h b/libcxx/include/__format/escaped_output_table.h
index 7a0b35239861e..1401b4637d839 100644
--- a/libcxx/include/__format/escaped_output_table.h
+++ b/libcxx/include/__format/escaped_output_table.h
@@ -109,7 +109,7 @@ namespace __escaped_output_table {
 /// - bits [14, 31] The lower bound code point of the range. The upper bound of
 ///   the range is lower bound + size. Note the code expects code units the fit
 ///   into 18 bits, instead of the 21 bits needed for the full Unicode range.
-_LIBCPP_HIDE_FROM_ABI inline constexpr uint32_t __entries[711] = {
+_LIBCPP_HIDE_FROM_ABI inline constexpr uint32_t __entries[735] = {
     0x00000020 /* 00000000 - 00000020 [   33] */,
     0x001fc021 /* 0000007f - 000000a0 [   34] */,
     0x002b4000 /* 000000ad - 000000ad [    1] */,
@@ -136,7 +136,7 @@ _LIBCPP_HIDE_FROM_ABI inline constexpr uint32_t __entries[711] = {
     0x02170001 /* 0000085c - 0000085d [    2] */,
     0x0217c000 /* 0000085f - 0000085f [    1] */,
     0x021ac004 /* 0000086b - 0000086f [    5] */,
-    0x0223c008 /* 0000088f - 00000897 [    9] */,
+    0x0223c007 /* 0000088f - 00000896 [    8] */,
     0x02388000 /* 000008e2 - 000008e2 [    1] */,
     0x02610000 /* 00000984 - 00000984 [    1] */,
     0x02634001 /* 0000098d - 0000098e [    2] */,
@@ -331,12 +331,11 @@ _LIBCPP_HIDE_FROM_ABI inline constexpr uint32_t __entries[711] = {
     0x06a68005 /* 00001a9a - 00001a9f [    6] */,
     0x06ab8001 /* 00001aae - 00001aaf [    2] */,
     0x06b3c030 /* 00001acf - 00001aff [   49] */,
-    0x06d34002 /* 00001b4d - 00001b4f [    3] */,
-    0x06dfc000 /* 00001b7f - 00001b7f [    1] */,
+    0x06d34000 /* 00001b4d - 00001b4d [    1] */,
     0x06fd0007 /* 00001bf4 - 00001bfb [    8] */,
     0x070e0002 /* 00001c38 - 00001c3a [    3] */,
     0x07128002 /* 00001c4a - 00001c4c [    3] */,
-    0x07224006 /* 00001c89 - 00001c8f [    7] */,
+    0x0722c004 /* 00001c8b - 00001c8f [    5] */,
     0x072ec001 /* 00001cbb - 00001cbc [    2] */,
     0x07320007 /* 00001cc8 - 00001ccf [    8] */,
     0x073ec004 /* 00001cfb - 00001cff [    5] */,
@@ -364,7 +363,7 @@ _LIBCPP_HIDE_FROM_ABI inline constexpr uint32_t __entries[711] = {
     0x0830400e /* 000020c1 - 000020cf [   15] */,
     0x083c400e /* 000020f1 - 000020ff [   15] */,
     0x08630003 /* 0000218c - 0000218f [    4] */,
-    0x0909c018 /* 00002427 - 0000243f [   25] */,
+    0x090a8015 /* 0000242a - 0000243f [   22] */,
     0x0912c014 /* 0000244b - 0000245f [   21] */,
     0x0add0001 /* 00002b74 - 00002b75 [    2] */,
     0x0ae58000 /* 00002b96 - 00002b96 [    1] */,
@@ -393,16 +392,16 @@ _LIBCPP_HIDE_FROM_ABI inline constexpr uint32_t __entries[711] = {
     0x0c400004 /* 00003100 - 00003104 [    5] */,
     0x0c4c0000 /* 00003130 - 00003130 [    1] */,
     0x0c63c000 /* 0000318f - 0000318f [    1] */,
-    0x0c79000a /* 000031e4 - 000031ee [   11] */,
+    0x0c798008 /* 000031e6 - 000031ee [    9] */,
     0x0c87c000 /* 0000321f - 0000321f [    1] */,
     0x29234002 /* 0000a48d - 0000a48f [    3] */,
     0x2931c008 /* 0000a4c7 - 0000a4cf [    9] */,
     0x298b0013 /* 0000a62c - 0000a63f [   20] */,
     0x29be0007 /* 0000a6f8 - 0000a6ff [    8] */,
-    0x29f2c004 /* 0000a7cb - 0000a7cf [    5] */,
+    0x29f38001 /* 0000a7ce - 0000a7cf [    2] */,
     0x29f48000 /* 0000a7d2 - 0000a7d2 [    1] */,
     0x29f50000 /* 0000a7d4 - 0000a7d4 [    1] */,
-    0x29f68017 /* 0000a7da - 0000a7f1 [   24] */,
+    0x29f74014 /* 0000a7dd - 0000a7f1 [   21] */,
     0x2a0b4002 /* 0000a82d - 0000a82f [    3] */,
     0x2a0e8005 /* 0000a83a - 0000a83f [    6] */,
     0x2a1e0007 /* 0000a878 - 0000a87f [    8] */,
@@ -491,7 +490,8 @@ _LIBCPP_HIDE_FROM_ABI inline constexpr uint32_t __entries[711] = {
     0x41688000 /* 000105a2 - 000105a2 [    1] */,
     0x416c8000 /* 000105b2 - 000105b2 [    1] */,
     0x416e8000 /* 000105ba - 000105ba [    1] */,
-    0x416f4042 /* 000105bd - 000105ff [   67] */,
+    0x416f4002 /* 000105bd - 000105bf [    3] */,
+    0x417d000b /* 000105f4 - 000105ff [   12] */,
     0x41cdc008 /* 00010737 - 0001073f [    9] */,
     0x41d58009 /* 00010756 - 0001075f [   10] */,
     0x41da0017 /* 00010768 - 0001077f [   24] */,
@@ -534,11 +534,15 @@ _LIBCPP_HIDE_FROM_ABI inline constexpr uint32_t __entries[711] = {
     0x432cc00c /* 00010cb3 - 00010cbf [   13] */,
     0x433cc006 /* 00010cf3 - 00010cf9 [    7] */,
     0x434a0007 /* 00010d28 - 00010d2f [    8] */,
-    0x434e8125 /* 00010d3a - 00010e5f [  294] */,
+    0x434e8005 /* 00010d3a - 00010d3f [    6] */,
+    0x43598002 /* 00010d66 - 00010d68 [    3] */,
+    0x43618007 /* 00010d86 - 00010d8d [    8] */,
+    0x436400cf /* 00010d90 - 00010e5f [  208] */,
     0x439fc000 /* 00010e7f - 00010e7f [    1] */,
     0x43aa8000 /* 00010eaa - 00010eaa [    1] */,
     0x43ab8001 /* 00010eae - 00010eaf [    2] */,
-    0x43ac804a /* 00010eb2 - 00010efc [   75] */,
+    0x43ac800f /* 00010eb2 - 00010ec1 [   16] */,
+    0x43b14036 /* 00010ec5 - 00010efb [   55] */,
     0x43ca0007 /* 00010f28 - 00010f2f [    8] */,
     0x43d68015 /* 00010f5a - 00010f6f [   22] */,
     0x43e28025 /* 00010f8a - 00010faf [   38] */,
@@ -578,7 +582,18 @@ _LIBCPP_HIDE_FROM_ABI inline constexpr uint32_t __entries[711] = {
     0x44d60004 /* 00011358 - 0001135c [    5] */,
     0x44d90001 /* 00011364 - 00011365 [    2] */,
     0x44db4002 /* 0001136d - 0001136f [    3] */,
-    0x44dd408a /* 00011375 - 000113ff [  139] */,
+    0x44dd400a /* 00011375 - 0001137f [   11] */,
+    0x44e28000 /* 0001138a - 0001138a [    1] */,
+    0x44e30001 /* 0001138c - 0001138d [    2] */,
+    0x44e3c000 /* 0001138f - 0001138f [    1] */,
+    0x44ed8000 /* 000113b6 - 000113b6 [    1] */,
+    0x44f04000 /* 000113c1 - 000113c1 [    1] */,
+    0x44f0c001 /* 000113c3 - 000113c4 [    2] */,
+    0x44f18000 /* 000113c6 - 000113c6 [    1] */,
+    0x44f2c000 /* 000113cb - 000113cb [    1] */,
+    0x44f58000 /* 000113d6 - 000113d6 [    1] */,
+    0x44f64007 /* 000113d9 - 000113e0 [    8] */,
+    0x44f8c01c /* 000113e3 - 000113ff [   29] */,
     0x45170000 /* 0001145c - 0001145c [    1] */,
     0x4518801d /* 00011462 - 0001147f [   30] */,
     0x45320007 /* 000114c8 - 000114cf [    8] */,
@@ -589,7 +604,8 @@ _LIBCPP_HIDE_FROM_ABI inline constexpr uint32_t __entries[711] = {
     0x45968005 /* 0001165a - 0001165f [    6] */,
     0x459b4012 /* 0001166d - 0001167f [   19] */,
     0x45ae8005 /* 000116ba - 000116bf [    6] */,
-    0x45b28035 /* 000116ca - 000116ff [   54] */,
+    0x45b28005 /* 000116ca - 000116cf [    6] */,
+    0x45b9001b /* 000116e4 - 000116ff [   28] */,
     0x45c6c001 /* 0001171b - 0001171c [    2] */,
     0x45cb0003 /* 0001172c - 0001172f [    4] */,
     0x45d1c0b8 /* 00011747 - 000117ff [  185] */,
@@ -609,7 +625,9 @@ _LIBCPP_HIDE_FROM_ABI inline constexpr uint32_t __entries[711] = {
     0x46920007 /* 00011a48 - 00011a4f [    8] */,
     0x46a8c00c /* 00011aa3 - 00011aaf [   13] */,
     0x46be4006 /* 00011af9 - 00011aff [    7] */,
-    0x46c280f5 /* 00011b0a - 00011bff [  246] */,
+    0x46c280b5 /* 00011b0a - 00011bbf [  182] */,
+    0x46f8800d /* 00011be2 - 00011bef [   14] */,
+    0x46fe8005 /* 00011bfa - 00011bff [    6] */,
     0x47024000 /* 00011c09 - 00011c09 [    1] */,
     0x470dc000 /* 00011c37 - 00011c37 [    1] */,
     0x47118009 /* 00011c46 - 00011c4f [   10] */,
@@ -633,7 +651,7 @@ _LIBCPP_HIDE_FROM_ABI inline constexpr uint32_t __entries[711] = {
     0x47be4006 /* 00011ef9 - 00011eff [    7] */,
     0x47c44000 /* 00011f11 - 00011f11 [    1] */,
     0x47cec002 /* 00011f3b - 00011f3d [    3] */,
-    0x47d68055 /* 00011f5a - 00011faf [   86] */,
+    0x47d6c054 /* 00011f5b - 00011faf [   85] */,
     0x47ec400e /* 00011fb1 - 00011fbf [   15] */,
     0x47fc800c /* 00011ff2 - 00011ffe [   13] */,
     0x48e68065 /* 0001239a - 000123ff [  102] */,
@@ -642,8 +660,10 @@ _LIBCPP_HIDE_FROM_ABI inline constexpr uint32_t __entries[711] = {
     0x49510a4b /* 00012544 - 00012f8f [ 2636] */,
     0x4bfcc00c /* 00012ff3 - 00012fff [   13] */,
     0x4d0c000f /* 00013430 - 0001343f [   16] */,
-    0x4d158fa9 /* 00013456 - 000143ff [ 4010] */,
-    0x5191e1b8 /* 00014647 - 000167ff [ 8633] */,
+    0x4d158009 /* 00013456 - 0001345f [   10] */,
+    0x50fec004 /* 000143fb - 000143ff [    5] */,
+    0x5191dab8 /* 00014647 - 000160ff [ 6841] */,
+    0x584e86c5 /* 0001613a - 000167ff [ 1734] */,
     0x5a8e4006 /* 00016a39 - 00016a3f [    7] */,
     0x5a97c000 /* 00016a5f - 00016a5f [    1] */,
     0x5a9a8003 /* 00016a6a - 00016a6d [    4] */,
@@ -655,7 +675,8 @@ _LIBCPP_HIDE_FROM_ABI inline constexpr uint32_t __entries[711] = {
     0x5ad68000 /* 00016b5a - 00016b5a [    1] */,
     0x5ad88000 /* 00016b62 - 00016b62 [    1] */,
     0x5ade0004 /* 00016b78 - 00016b7c [    5] */,
-    0x5ae402af /* 00016b90 - 00016e3f [  688] */,
+    0x5ae401af /* 00016b90 - 00016d3f [  432] */,
+    0x5b5e80c5 /* 00016d7a - 00016e3f [  198] */,
     0x5ba6c064 /* 00016e9b - 00016eff [  101] */,
     0x5bd2c003 /* 00016f4b - 00016f4e [    4] */,
     0x5be20006 /* 00016f88 - 00016f8e [    7] */,
@@ -663,7 +684,7 @@ _LIBCPP_HIDE_FROM_ABI inline constexpr uint32_t __entries[711] = {
     0x5bf9400a /* 00016fe5 - 00016fef [   11] */,
     0x5bfc800d /* 00016ff2 - 00016fff [   14] */,
     0x61fe0007 /* 000187f8 - 000187ff [    8] */,
-    0x63358029 /* 00018cd6 - 00018cff [   42] */,
+    0x63358028 /* 00018cd6 - 00018cfe [   41] */,
     0x634262e6 /* 00018d09 - 0001afef [ 8935] */,
     0x6bfd0000 /* 0001aff4 - 0001aff4 [    1] */,
     0x6bff0000 /* 0001affc - 0001affc [    1] */,
@@ -678,7 +699,9 @@ _LIBCPP_HIDE_FROM_ABI inline constexpr uint32_t __entries[711] = {
     0x6f1f4002 /* 0001bc7d - 0001bc7f [    3] */,
     0x6f224006 /* 0001bc89 - 0001bc8f [    7] */,
     0x6f268001 /* 0001bc9a - 0001bc9b [    2] */,
-    0x6f28125f /* 0001bca0 - 0001ceff [ 4704] */,
+    0x6f280f5f /* 0001bca0 - 0001cbff [ 3936] */,
+    0x733e8005 /* 0001ccfa - 0001ccff [    6] */,
+    0x73ad004b /* 0001ceb4 - 0001ceff [   76] */,
     0x73cb8001 /* 0001cf2e - 0001cf2f [    2] */,
     0x73d1c008 /* 0001cf47 - 0001cf4f [    9] */,
     0x73f1003b /* 0001cfc4 - 0001cfff [   60] */,
@@ -730,7 +753,9 @@ _LIBCPP_HIDE_FROM_ABI inline constexpr uint32_t __entries[711] = {
     0x78abc010 /* 0001e2af - 0001e2bf [   17] */,
     0x78be8004 /* 0001e2fa - 0001e2fe [    5] */,
     0x78c001cf /* 0001e300 - 0001e4cf [  464] */,
-    0x793e82e5 /* 0001e4fa - 0001e7df [  742] */,
+    0x793e80d5 /* 0001e4fa - 0001e5cf [  214] */,
+    0x797ec003 /* 0001e5fb - 0001e5fe [    4] */,
+    0x798001df /* 0001e600 - 0001e7df [  480] */,
     0x79f9c000 /* 0001e7e7 - 0001e7e7 [    1] */,
     0x79fb0000 /* 0001e7ec - 0001e7ec [    1] */,
     0x79fbc000 /* 0001e7ef - 0001e7ef [    1] */,
@@ -800,18 +825,17 @@ _LIBCPP_HIDE_FROM_ABI inline constexpr uint32_t __entries[711] = {
     0x7e168005 /* 0001f85a - 0001f85f [    6] */,
     0x7e220007 /* 0001f888 - 0001f88f [    8] */,
     0x7e2b8001 /* 0001f8ae - 0001f8af [    2] */,
-    0x7e2c804d /* 0001f8b2 - 0001f8ff [   78] */,
+    0x7e2f0003 /* 0001f8bc - 0001f8bf [    4] */,
+    0x7e30803d /* 0001f8c2 - 0001f8ff [   62] */,
     0x7e95000b /* 0001fa54 - 0001fa5f [   12] */,
     0x7e9b8001 /* 0001fa6e - 0001fa6f [    2] */,
     0x7e9f4002 /* 0001fa7d - 0001fa7f [    3] */,
-    0x7ea24006 /* 0001fa89 - 0001fa8f [    7] */,
-    0x7eaf8000 /* 0001fabe - 0001fabe [    1] */,
-    0x7eb18007 /* 0001fac6 - 0001facd [    8] */,
-    0x7eb70003 /* 0001fadc - 0001fadf [    4] */,
-    0x7eba4006 /* 0001fae9 - 0001faef [    7] */,
+    0x7ea28004 /* 0001fa8a - 0001fa8e [    5] */,
+    0x7eb1c006 /* 0001fac7 - 0001facd [    7] */,
+    0x7eb74001 /* 0001fadd - 0001fade [    2] */,
+    0x7eba8005 /* 0001faea - 0001faef [    6] */,
     0x7ebe4006 /* 0001faf9 - 0001faff [    7] */,
     0x7ee4c000 /* 0001fb93 - 0001fb93 [    1] */,
-    0x7ef2c024 /* 0001fbcb - 0001fbef [   37] */,
     0x7efe8405 /* 0001fbfa - 0001ffff [ 1030] */,
     0xa9b8001f /* 0002a6e0 - 0002a6ff [   32] */,
     0xadce8005 /* 0002b73a - 0002b73f [    6] */,
diff --git a/libcxx/include/__format/extended_grapheme_cluster_table.h b/libcxx/include/__format/extended_grapheme_cluster_table.h
index 7653a9e03b815..f76e018df7ae1 100644
--- a/libcxx/include/__format/extended_grapheme_cluster_table.h
+++ b/libcxx/include/__format/extended_grapheme_cluster_table.h
@@ -125,7 +125,7 @@ enum class __property : uint8_t {
 /// following benchmark.
 /// libcxx/benchmarks/std_format_spec_string_unicode.bench.cpp
 // clang-format off
-_LIBCPP_HIDE_FROM_ABI inline constexpr uint32_t __entries[1496] = {
+_LIBCPP_HIDE_FROM_ABI inline constexpr uint32_t __entries[1501] = {
     0x00000091,
     0x00005005,
     0x00005811,
@@ -164,7 +164,7 @@ _LIBCPP_HIDE_FROM_ABI inline constexpr uint32_t __entries[1496] = {
     0x00414842,
     0x0042c822,
     0x00448018,
-    0x0044c072,
+    0x0044b882,
     0x00465172,
     0x00471008,
     0x004719f2,
@@ -246,14 +246,12 @@ _LIBCPP_HIDE_FROM_ABI inline constexpr uint32_t __entries[1496] = {
     0x0064101a,
     0x0065e002,
     0x0065f00a,
-    0x0065f802,
-    0x0066001a,
+    0x0065f812,
+    0x0066080a,
     0x00661002,
     0x0066181a,
-    0x00663002,
-    0x0066381a,
-    0x0066501a,
-    0x00666012,
+    0x00663022,
+    0x00665032,
     0x0066a812,
     0x00671012,
     0x0067980a,
@@ -318,10 +316,8 @@ _LIBCPP_HIDE_FROM_ABI inline constexpr uint32_t __entries[1496] = {
     0x008b047c,
     0x008d457b,
     0x009ae822,
-    0x00b89022,
-    0x00b8a80a,
-    0x00b99012,
-    0x00b9a00a,
+    0x00b89032,
+    0x00b99022,
     0x00ba9012,
     0x00bb9012,
     0x00bda012,
@@ -361,29 +357,23 @@ _LIBCPP_HIDE_FROM_ABI inline constexpr uint32_t __entries[1496] = {
     0x00d581e2,
     0x00d80032,
     0x00d8200a,
-    0x00d9a062,
-    0x00d9d80a,
-    0x00d9e002,
-    0x00d9e84a,
-    0x00da1002,
-    0x00da181a,
+    0x00d9a092,
+    0x00d9f03a,
+    0x00da1022,
     0x00db5882,
     0x00dc0012,
     0x00dc100a,
     0x00dd080a,
     0x00dd1032,
     0x00dd301a,
-    0x00dd4012,
-    0x00dd500a,
-    0x00dd5822,
+    0x00dd4052,
     0x00df3002,
     0x00df380a,
     0x00df4012,
     0x00df502a,
     0x00df6802,
     0x00df700a,
-    0x00df7822,
-    0x00df901a,
+    0x00df7842,
     0x00e1207a,
     0x00e16072,
     0x00e1a01a,
@@ -475,7 +465,8 @@ _LIBCPP_HIDE_FROM_ABI inline constexpr uint32_t __entries[1496] = {
     0x0547f802,
     0x05493072,
     0x054a38a2,
-    0x054a901a,
+    0x054a900a,
+    0x054a9802,
     0x054b01c4,
     0x054c0022,
     0x054c180a,
@@ -484,7 +475,8 @@ _LIBCPP_HIDE_FROM_ABI inline constexpr uint32_t __entries[1496] = {
     0x054db032,
     0x054dd01a,
     0x054de012,
-    0x054df02a,
+    0x054df01a,
+    0x054e0002,
     0x054f2802,
     0x05514852,
     0x0551781a,
@@ -1328,8 +1320,9 @@ _LIBCPP_HIDE_FROM_ABI inline constexpr uint32_t __entries[1496] = {
     0x0851f802,
     0x08572812,
     0x08692032,
+    0x086b4842,
     0x08755812,
-    0x0877e822,
+    0x0877e032,
     0x087a30a2,
     0x087c1032,
     0x0880000a,
@@ -1357,7 +1350,8 @@ _LIBCPP_HIDE_FROM_ABI inline constexpr uint32_t __entries[1496] = {
     0x088c100a,
     0x088d982a,
     0x088db082,
-    0x088df81a,
+    0x088df80a,
+    0x088e0002,
     0x088e1018,
     0x088e4832,
     0x088e700a,
@@ -1365,9 +1359,7 @@ _LIBCPP_HIDE_FROM_ABI inline constexpr uint32_t __entries[1496] = {
     0x0891602a,
     0x08917822,
     0x0891901a,
-    0x0891a002,
-    0x0891a80a,
-    0x0891b012,
+    0x0891a032,
     0x0891f002,
     0x08920802,
     0x0896f802,
@@ -1381,11 +1373,24 @@ _LIBCPP_HIDE_FROM_ABI inline constexpr uint32_t __entries[1496] = {
     0x089a0002,
     0x089a083a,
     0x089a381a,
-    0x089a582a,
+    0x089a581a,
+    0x089a6802,
     0x089ab802,
     0x089b101a,
     0x089b3062,
     0x089b8042,
+    0x089dc002,
+    0x089dc81a,
+    0x089dd852,
+    0x089e1002,
+    0x089e2802,
+    0x089e3822,
+    0x089e500a,
+    0x089e601a,
+    0x089e7022,
+    0x089e8808,
+    0x089e9002,
+    0x089f0812,
     0x08a1a82a,
     0x08a1c072,
     0x08a2001a,
@@ -1422,10 +1427,10 @@ _LIBCPP_HIDE_FROM_ABI inline constexpr uint32_t __entries[1496] = {
     0x08b5600a,
     0x08b56802,
     0x08b5701a,
-    0x08b58052,
-    0x08b5b00a,
-    0x08b5b802,
-    0x08b8e822,
+    0x08b58072,
+    0x08b8e802,
+    0x08b8f00a,
+    0x08b8f802,
     0x08b91032,
     0x08b9300a,
     0x08b93842,
@@ -1436,9 +1441,7 @@ _LIBCPP_HIDE_FROM_ABI inline constexpr uint32_t __entries[1496] = {
     0x08c98002,
     0x08c9884a,
     0x08c9b81a,
-    0x08c9d812,
-    0x08c9e80a,
-    0x08c9f002,
+    0x08c9d832,
     0x08c9f808,
     0x08ca000a,
     0x08ca0808,
@@ -1495,28 +1498,29 @@ _LIBCPP_HIDE_FROM_ABI inline constexpr uint32_t __entries[1496] = {
     0x08f9a01a,
     0x08f9b042,
     0x08f9f01a,
-    0x08fa0002,
-    0x08fa080a,
-    0x08fa1002,
+    0x08fa0022,
+    0x08fad002,
     0x09a180f1,
     0x09a20002,
     0x09a238e2,
+    0x0b08f0b2,
+    0x0b09502a,
+    0x0b096822,
     0x0b578042,
     0x0b598062,
+    0x0b6b180c,
+    0x0b6b383c,
     0x0b7a7802,
     0x0b7a8b6a,
     0x0b7c7832,
     0x0b7f2002,
-    0x0b7f801a,
+    0x0b7f8012,
     0x0de4e812,
     0x0de50031,
     0x0e7802d2,
     0x0e798162,
-    0x0e8b2802,
-    0x0e8b300a,
-    0x0e8b3822,
-    0x0e8b680a,
-    0x0e8b7042,
+    0x0e8b2842,
+    0x0e8b6852,
     0x0e8b9871,
     0x0e8bd872,
     0x0e8c2862,
@@ -1538,6 +1542,7 @@ _LIBCPP_HIDE_FROM_ABI inline constexpr uint32_t __entries[1496] = {
     0x0f157002,
     0x0f176032,
     0x0f276032,
+    0x0f2f7012,
     0x0f468062,
     0x0f4a2062,
     0x0f8007f3,
diff --git a/libcxx/include/__format/format_functions.h b/libcxx/include/__format/format_functions.h
index b920be5acbe86..5feaf7e5a064a 100644
--- a/libcxx/include/__format/format_functions.h
+++ b/libcxx/include/__format/format_functions.h
@@ -31,7 +31,6 @@
 #include <__format/formatter_pointer.h>
 #include <__format/formatter_string.h>
 #include <__format/parser_std_format_spec.h>
-#include <__iterator/back_insert_iterator.h>
 #include <__iterator/concepts.h>
 #include <__iterator/incrementable_traits.h>
 #include <__iterator/iterator_traits.h> // iter_value_t
@@ -411,7 +410,7 @@ _LIBCPP_HIDE_FROM_ABI _OutIt __vformat_to(_OutIt __out_it,
     return std::__format::__vformat_to(
         basic_format_parse_context{__fmt, __args.__size()}, std::__format_context_create(std::move(__out_it), __args));
   else {
-    __format::__format_buffer<_OutIt, _CharT> __buffer{std::move(__out_it)};
+    typename __format::__buffer_selector<_OutIt, _CharT>::type __buffer{std::move(__out_it)};
     std::__format::__vformat_to(basic_format_parse_context{__fmt, __args.__size()},
                                 std::__format_context_create(__buffer.__make_output_iterator(), __args));
     return std::move(__buffer).__out_it();
@@ -452,9 +451,9 @@ format_to(_OutIt __out_it, wformat_string<_Args...> __fmt, _Args&&... __args) {
 // fires too eagerly, see http://llvm.org/PR61563.
 template <class = void>
 [[nodiscard]] _LIBCPP_ALWAYS_INLINE inline _LIBCPP_HIDE_FROM_ABI string vformat(string_view __fmt, format_args __args) {
-  string __res;
-  std::vformat_to(std::back_inserter(__res), __fmt, __args);
-  return __res;
+  __format::__allocating_buffer<char> __buffer;
+  std::vformat_to(__buffer.__make_output_iterator(), __fmt, __args);
+  return string{__buffer.__view()};
 }
 
 #  if _LIBCPP_HAS_WIDE_CHARACTERS
@@ -463,9 +462,9 @@ template <class = void>
 template <class = void>
 [[nodiscard]] _LIBCPP_ALWAYS_INLINE inline _LIBCPP_HIDE_FROM_ABI wstring
 vformat(wstring_view __fmt, wformat_args __args) {
-  wstring __res;
-  std::vformat_to(std::back_inserter(__res), __fmt, __args);
-  return __res;
+  __format::__allocating_buffer<wchar_t> __buffer;
+  std::vformat_to(__buffer.__make_output_iterator(), __fmt, __args);
+  return wstring{__buffer.__view()};
 }
 #  endif
 
@@ -544,7 +543,7 @@ _LIBCPP_HIDE_FROM_ABI _OutIt __vformat_to(
     return std::__format::__vformat_to(basic_format_parse_context{__fmt, __args.__size()},
                                        std::__format_context_create(std::move(__out_it), __args, std::move(__loc)));
   else {
-    __format::__format_buffer<_OutIt, _CharT> __buffer{std::move(__out_it)};
+    typename __format::__buffer_selector<_OutIt, _CharT>::type __buffer{std::move(__out_it)};
     std::__format::__vformat_to(
         basic_format_parse_context{__fmt, __args.__size()},
         std::__format_context_create(__buffer.__make_output_iterator(), __args, std::move(__loc)));
@@ -585,9 +584,9 @@ format_to(_OutIt __out_it, locale __loc, wformat_string<_Args...> __fmt, _Args&&
 template <class = void>
 [[nodiscard]] _LIBCPP_ALWAYS_INLINE inline _LIBCPP_HIDE_FROM_ABI string
 vformat(locale __loc, string_view __fmt, format_args __args) {
-  string __res;
-  std::vformat_to(std::back_inserter(__res), std::move(__loc), __fmt, __args);
-  return __res;
+  __format::__allocating_buffer<char> __buffer;
+  std::vformat_to(__buffer.__make_output_iterator(), std::move(__loc), __fmt, __args);
+  return string{__buffer.__view()};
 }
 
 #    if _LIBCPP_HAS_WIDE_CHARACTERS
@@ -596,9 +595,9 @@ vformat(locale __loc, string_view __fmt, format_args __args) {
 template <class = void>
 [[nodiscard]] _LIBCPP_ALWAYS_INLINE inline _LIBCPP_HIDE_FROM_ABI wstring
 vformat(locale __loc, wstring_view __fmt, wformat_args __args) {
-  wstring __res;
-  std::vformat_to(std::back_inserter(__res), std::move(__loc), __fmt, __args);
-  return __res;
+  __format::__allocating_buffer<wchar_t> __buffer;
+  std::vformat_to(__buffer.__make_output_iterator(), std::move(__loc), __fmt, __args);
+  return wstring{__buffer.__view()};
 }
 #    endif
 
diff --git a/libcxx/include/__format/indic_conjunct_break_table.h b/libcxx/include/__format/indic_conjunct_break_table.h
index df6cfe6a02f34..f48ea625908e9 100644
--- a/libcxx/include/__format/indic_conjunct_break_table.h
+++ b/libcxx/include/__format/indic_conjunct_break_table.h
@@ -107,10 +107,9 @@ enum class __property : uint8_t {
 /// following benchmark.
 /// libcxx/benchmarks/std_format_spec_string_unicode.bench.cpp
 // clang-format off
-_LIBCPP_HIDE_FROM_ABI inline constexpr uint32_t __entries[201] = {
-    0x00180139,
-    0x001a807d,
-    0x00241811,
+_LIBCPP_HIDE_FROM_ABI inline constexpr uint32_t __entries[403] = {
+    0x001801bd,
+    0x00241819,
     0x002c88b1,
     0x002df801,
     0x002e0805,
@@ -125,6 +124,7 @@ _LIBCPP_HIDE_FROM_ABI inline constexpr uint32_t __entries[201] = {
     0x0037500d,
     0x00388801,
     0x00398069,
+    0x003d3029,
     0x003f5821,
     0x003fe801,
     0x0040b00d,
@@ -132,87 +132,174 @@ _LIBCPP_HIDE_FROM_ABI inline constexpr uint32_t __entries[201] = {
     0x00412809,
     0x00414811,
     0x0042c809,
-    0x0044c01d,
+    0x0044b821,
     0x0046505d,
-    0x00471871,
+    0x0047187d,
     0x0048a890,
+    0x0049d001,
     0x0049e001,
+    0x004a081d,
     0x004a6802,
-    0x004a880d,
+    0x004a8819,
     0x004ac01c,
+    0x004b1005,
     0x004bc01c,
+    0x004c0801,
     0x004ca84c,
     0x004d5018,
     0x004d9000,
     0x004db00c,
     0x004de001,
+    0x004df001,
+    0x004e080d,
     0x004e6802,
+    0x004eb801,
     0x004ee004,
     0x004ef800,
+    0x004f1005,
     0x004f8004,
     0x004ff001,
+    0x00500805,
     0x0051e001,
+    0x00520805,
+    0x00523805,
+    0x00525809,
+    0x00528801,
+    0x00538005,
+    0x0053a801,
+    0x00540805,
     0x0054a84c,
     0x00555018,
     0x00559004,
     0x0055a810,
     0x0055e001,
+    0x00560811,
+    0x00563805,
     0x00566802,
+    0x00571005,
     0x0057c800,
+    0x0057d015,
+    0x00580801,
     0x0058a84c,
     0x00595018,
     0x00599004,
     0x0059a810,
     0x0059e001,
+    0x0059f005,
+    0x005a080d,
     0x005a6802,
+    0x005aa809,
     0x005ae004,
     0x005af800,
+    0x005b1005,
     0x005b8800,
+    0x005c1001,
+    0x005df001,
+    0x005e0001,
+    0x005e6801,
+    0x005eb801,
+    0x00600001,
+    0x00602001,
     0x0060a84c,
     0x0061503c,
     0x0061e001,
+    0x0061f009,
+    0x00623009,
+    0x00625009,
     0x00626802,
     0x0062a805,
     0x0062c008,
+    0x00631005,
+    0x00640801,
     0x0065e001,
+    0x0065f805,
+    0x00661001,
+    0x00663009,
+    0x0066500d,
+    0x0066a805,
+    0x00671005,
+    0x00680005,
     0x0068a894,
     0x0069d805,
+    0x0069f001,
+    0x006a080d,
     0x006a6802,
-    0x0071c009,
-    0x0072400d,
-    0x0075c009,
-    0x0076400d,
+    0x006ab801,
+    0x006b1005,
+    0x006c0801,
+    0x006e5001,
+    0x006e7801,
+    0x006e9009,
+    0x006eb001,
+    0x006ef801,
+    0x00718801,
+    0x0071a019,
+    0x0072381d,
+    0x00758801,
+    0x0075a021,
+    0x00764019,
     0x0078c005,
     0x0079a801,
     0x0079b801,
     0x0079c801,
-    0x007b8805,
-    0x007ba001,
-    0x007bd00d,
-    0x007c0001,
-    0x007c1009,
+    0x007b8835,
+    0x007c0011,
     0x007c3005,
+    0x007c6829,
+    0x007cc88d,
     0x007e3001,
-    0x0081b801,
+    0x0081680d,
+    0x00819015,
     0x0081c805,
+    0x0081e805,
+    0x0082c005,
+    0x0082f009,
+    0x0083880d,
+    0x00841001,
+    0x00842805,
     0x00846801,
+    0x0084e801,
     0x009ae809,
-    0x00b8a001,
-    0x00be9001,
+    0x00b8900d,
+    0x00b99009,
+    0x00ba9005,
+    0x00bb9005,
+    0x00bda005,
+    0x00bdb819,
+    0x00be3001,
+    0x00be4829,
     0x00bee801,
+    0x00c05809,
+    0x00c07801,
+    0x00c42805,
     0x00c54801,
+    0x00c90009,
+    0x00c93805,
+    0x00c99001,
     0x00c9c809,
     0x00d0b805,
+    0x00d0d801,
+    0x00d2b001,
+    0x00d2c019,
     0x00d30001,
-    0x00d3a81d,
+    0x00d31001,
+    0x00d3281d,
+    0x00d39825,
     0x00d3f801,
-    0x00d58035,
-    0x00d5f83d,
-    0x00d9a001,
+    0x00d58079,
+    0x00d8000d,
+    0x00d9a025,
+    0x00da1009,
     0x00db5821,
-    0x00dd5801,
+    0x00dc0005,
+    0x00dd100d,
+    0x00dd4015,
     0x00df3001,
-    0x00e1b801,
+    0x00df4005,
+    0x00df6801,
+    0x00df7811,
+    0x00e1601d,
+    0x00e1b005,
     0x00e68009,
     0x00e6a031,
     0x00e71019,
@@ -221,82 +308,193 @@ _LIBCPP_HIDE_FROM_ABI inline constexpr uint32_t __entries[201] = {
     0x00e7c005,
     0x00ee00fd,
     0x01006801,
-    0x01068031,
-    0x01070801,
-    0x0107282d,
+    0x01068081,
     0x01677809,
     0x016bf801,
     0x016f007d,
     0x01815015,
     0x0184c805,
-    0x05337801,
+    0x0533780d,
     0x0533a025,
     0x0534f005,
     0x05378005,
+    0x05401001,
+    0x05403001,
+    0x05405801,
+    0x05412805,
     0x05416001,
+    0x05462005,
     0x05470045,
-    0x05495809,
+    0x0547f801,
+    0x0549301d,
+    0x054a3829,
+    0x054a9801,
+    0x054c0009,
     0x054d9801,
+    0x054db00d,
+    0x054de005,
+    0x054e0001,
+    0x054f2801,
+    0x05514815,
+    0x05518805,
+    0x0551a805,
+    0x05521801,
+    0x05526001,
+    0x0553e001,
     0x05558001,
     0x05559009,
     0x0555b805,
     0x0555f005,
     0x05560801,
+    0x05576005,
     0x0557b001,
+    0x055f2801,
+    0x055f4001,
     0x055f6801,
     0x07d8f001,
+    0x07f0003d,
     0x07f1003d,
+    0x07fcf005,
     0x080fe801,
     0x08170001,
     0x081bb011,
-    0x08506801,
-    0x08507801,
+    0x08500809,
+    0x08502805,
+    0x0850600d,
     0x0851c009,
     0x0851f801,
     0x08572805,
     0x0869200d,
+    0x086b4811,
     0x08755805,
-    0x0877e809,
+    0x0877e00d,
     0x087a3029,
     0x087c100d,
+    0x08800801,
+    0x0881c039,
     0x08838001,
-    0x0883f801,
-    0x0885d001,
+    0x08839805,
+    0x0883f809,
+    0x0885980d,
+    0x0885c805,
+    0x08861001,
     0x08880009,
-    0x08899805,
+    0x08893811,
+    0x0889681d,
     0x088b9801,
-    0x088e5001,
-    0x0891b001,
-    0x08974805,
+    0x088c0005,
+    0x088db021,
+    0x088e0001,
+    0x088e480d,
+    0x088e7801,
+    0x08917809,
+    0x0891a00d,
+    0x0891f001,
+    0x08920801,
+    0x0896f801,
+    0x0897181d,
+    0x08980005,
     0x0899d805,
+    0x0899f001,
+    0x089a0001,
+    0x089a6801,
+    0x089ab801,
     0x089b3019,
     0x089b8011,
+    0x089dc001,
+    0x089dd815,
+    0x089e1001,
+    0x089e2801,
+    0x089e3809,
+    0x089e7009,
+    0x089e9001,
+    0x089f0805,
+    0x08a1c01d,
+    0x08a21009,
     0x08a23001,
     0x08a2f001,
-    0x08a61801,
-    0x08ae0001,
-    0x08b5b801,
-    0x08b95801,
-    0x08c1d001,
-    0x08c9f001,
+    0x08a58001,
+    0x08a59815,
+    0x08a5d001,
+    0x08a5e801,
+    0x08a5f805,
+    0x08a61005,
+    0x08ad7801,
+    0x08ad900d,
+    0x08ade005,
+    0x08adf805,
+    0x08aee005,
+    0x08b1981d,
+    0x08b1e801,
+    0x08b1f805,
+    0x08b55801,
+    0x08b56801,
+    0x08b5801d,
+    0x08b8e801,
+    0x08b8f801,
+    0x08b9100d,
+    0x08b93811,
+    0x08c17821,
+    0x08c1c805,
+    0x08c98001,
+    0x08c9d80d,
     0x08ca1801,
-    0x08d1a001,
+    0x08cea00d,
+    0x08ced005,
+    0x08cf0001,
+    0x08d00825,
+    0x08d19815,
+    0x08d1d80d,
     0x08d23801,
-    0x08d4c801,
-    0x08ea1001,
-    0x08ea2005,
+    0x08d28815,
+    0x08d2c809,
+    0x08d45031,
+    0x08d4c005,
+    0x08e18019,
+    0x08e1c015,
+    0x08e1f801,
+    0x08e49055,
+    0x08e55019,
+    0x08e59005,
+    0x08e5a805,
+    0x08e98815,
+    0x08e9d001,
+    0x08e9e005,
+    0x08e9f819,
+    0x08ea3801,
+    0x08ec8005,
+    0x08eca801,
     0x08ecb801,
-    0x08fa1001,
+    0x08f79805,
+    0x08f80005,
+    0x08f9b011,
+    0x08fa0009,
+    0x08fad001,
+    0x09a20001,
+    0x09a23839,
+    0x0b08f02d,
+    0x0b096809,
     0x0b578011,
     0x0b598019,
-    0x0de4f001,
-    0x0e8b2801,
-    0x0e8b3809,
-    0x0e8b7011,
+    0x0b7a7801,
+    0x0b7c780d,
+    0x0b7f2001,
+    0x0b7f8005,
+    0x0de4e805,
+    0x0e7800b5,
+    0x0e798059,
+    0x0e8b2811,
+    0x0e8b6815,
     0x0e8bd81d,
     0x0e8c2819,
     0x0e8d500d,
     0x0e921009,
+    0x0ed000d9,
+    0x0ed1d8c5,
+    0x0ed3a801,
+    0x0ed42001,
+    0x0ed4d811,
+    0x0ed50839,
     0x0f000019,
     0x0f004041,
     0x0f00d819,
@@ -307,8 +505,12 @@ _LIBCPP_HIDE_FROM_ABI inline constexpr uint32_t __entries[201] = {
     0x0f157001,
     0x0f17600d,
     0x0f27600d,
+    0x0f2f7005,
     0x0f468019,
-    0x0f4a2019};
+    0x0f4a2019,
+    0x0f9fd811,
+    0x7001017d,
+    0x700803bd};
 // clang-format on
 
 /// Returns the indic conjuct break property of a code point.
diff --git a/libcxx/include/__format/width_estimation_table.h b/libcxx/include/__format/width_estimation_table.h
index 5b4b3950c6a1d..0ea0b4f413a74 100644
--- a/libcxx/include/__format/width_estimation_table.h
+++ b/libcxx/include/__format/width_estimation_table.h
@@ -119,7 +119,7 @@ namespace __width_estimation_table {
 /// - bits [0, 13] The size of the range, allowing 16384 elements.
 /// - bits [14, 31] The lower bound code point of the range. The upper bound of
 ///   the range is lower bound + size.
-_LIBCPP_HIDE_FROM_ABI inline constexpr uint32_t __entries[107] = {
+_LIBCPP_HIDE_FROM_ABI inline constexpr uint32_t __entries[110] = {
     0x0440005f /* 00001100 - 0000115f [   96] */, //
     0x08c68001 /* 0000231a - 0000231b [    2] */, //
     0x08ca4001 /* 00002329 - 0000232a [    2] */, //
@@ -128,8 +128,10 @@ _LIBCPP_HIDE_FROM_ABI inline constexpr uint32_t __entries[107] = {
     0x08fcc000 /* 000023f3 - 000023f3 [    1] */, //
     0x097f4001 /* 000025fd - 000025fe [    2] */, //
     0x09850001 /* 00002614 - 00002615 [    2] */, //
+    0x098c0007 /* 00002630 - 00002637 [    8] */, //
     0x0992000b /* 00002648 - 00002653 [   12] */, //
     0x099fc000 /* 0000267f - 0000267f [    1] */, //
+    0x09a28005 /* 0000268a - 0000268f [    6] */, //
     0x09a4c000 /* 00002693 - 00002693 [    1] */, //
     0x09a84000 /* 000026a1 - 000026a1 [    1] */, //
     0x09aa8001 /* 000026aa - 000026ab [    2] */, //
@@ -163,7 +165,7 @@ _LIBCPP_HIDE_FROM_ABI inline constexpr uint32_t __entries[107] = {
     0x0c264066 /* 00003099 - 000030ff [  103] */, //
     0x0c41402a /* 00003105 - 0000312f [   43] */, //
     0x0c4c405d /* 00003131 - 0000318e [   94] */, //
-    0x0c640053 /* 00003190 - 000031e3 [   84] */, //
+    0x0c640055 /* 00003190 - 000031e5 [   86] */, //
     0x0c7bc02f /* 000031ef - 0000321e [   48] */, //
     0x0c880027 /* 00003220 - 00003247 [   40] */, //
     0x0c943fff /* 00003250 - 0000724f [16384] */, //
@@ -182,7 +184,7 @@ _LIBCPP_HIDE_FROM_ABI inline constexpr uint32_t __entries[107] = {
     0x5bfc0001 /* 00016ff0 - 00016ff1 [    2] */, //
     0x5c0017f7 /* 00017000 - 000187f7 [ 6136] */, //
     0x620004d5 /* 00018800 - 00018cd5 [ 1238] */, //
-    0x63400008 /* 00018d00 - 00018d08 [    9] */, //
+    0x633fc009 /* 00018cff - 00018d08 [   10] */, //
     0x6bfc0003 /* 0001aff0 - 0001aff3 [    4] */, //
     0x6bfd4006 /* 0001aff5 - 0001affb [    7] */, //
     0x6bff4001 /* 0001affd - 0001affe [    2] */, //
@@ -192,6 +194,8 @@ _LIBCPP_HIDE_FROM_ABI inline constexpr uint32_t __entries[107] = {
     0x6c554000 /* 0001b155 - 0001b155 [    1] */, //
     0x6c590003 /* 0001b164 - 0001b167 [    4] */, //
     0x6c5c018b /* 0001b170 - 0001b2fb [  396] */, //
+    0x74c00056 /* 0001d300 - 0001d356 [   87] */, //
+    0x74d80016 /* 0001d360 - 0001d376 [   23] */, //
     0x7c010000 /* 0001f004 - 0001f004 [    1] */, //
     0x7c33c000 /* 0001f0cf - 0001f0cf [    1] */, //
     0x7c638000 /* 0001f18e - 0001f18e [    1] */, //
@@ -213,11 +217,10 @@ _LIBCPP_HIDE_FROM_ABI inline constexpr uint32_t __entries[107] = {
     0x7dfc0000 /* 0001f7f0 - 0001f7f0 [    1] */, //
     0x7e4000ff /* 0001f900 - 0001f9ff [  256] */, //
     0x7e9c000c /* 0001fa70 - 0001fa7c [   13] */, //
-    0x7ea00008 /* 0001fa80 - 0001fa88 [    9] */, //
-    0x7ea4002d /* 0001fa90 - 0001fabd [   46] */, //
-    0x7eafc006 /* 0001fabf - 0001fac5 [    7] */, //
-    0x7eb3800d /* 0001face - 0001fadb [   14] */, //
-    0x7eb80008 /* 0001fae0 - 0001fae8 [    9] */, //
+    0x7ea00009 /* 0001fa80 - 0001fa89 [   10] */, //
+    0x7ea3c037 /* 0001fa8f - 0001fac6 [   56] */, //
+    0x7eb3800e /* 0001face - 0001fadc [   15] */, //
+    0x7eb7c00a /* 0001fadf - 0001fae9 [   11] */, //
     0x7ebc0008 /* 0001faf0 - 0001faf8 [    9] */, //
     0x80003fff /* 00020000 - 00023fff [16384] */, //
     0x90003fff /* 00024000 - 00027fff [16384] */, //
diff --git a/libcxx/include/__functional/is_transparent.h b/libcxx/include/__functional/is_transparent.h
index b2d62f2e3ead8..567df1a662f54 100644
--- a/libcxx/include/__functional/is_transparent.h
+++ b/libcxx/include/__functional/is_transparent.h
@@ -21,11 +21,11 @@ _LIBCPP_BEGIN_NAMESPACE_STD
 
 #if _LIBCPP_STD_VER >= 14
 
-template <class _Tp, class, class = void>
+template <class _Tp, class _Key = void, class = void>
 inline const bool __is_transparent_v = false;
 
-template <class _Tp, class _Up>
-inline const bool __is_transparent_v<_Tp, _Up, __void_t<typename _Tp::is_transparent> > = true;
+template <class _Tp, class _Key>
+inline const bool __is_transparent_v<_Tp, _Key, __void_t<typename _Tp::is_transparent> > = true;
 
 #endif
 
diff --git a/libcxx/include/__fwd/bit_reference.h b/libcxx/include/__fwd/bit_reference.h
index 30462b6ce4c92..d65f043e89ad6 100644
--- a/libcxx/include/__fwd/bit_reference.h
+++ b/libcxx/include/__fwd/bit_reference.h
@@ -10,6 +10,9 @@
 #define _LIBCPP___FWD_BIT_REFERENCE_H
 
 #include <__config>
+#include <__memory/pointer_traits.h>
+#include <__type_traits/enable_if.h>
+#include <__type_traits/is_unsigned.h>
 
 #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
 #  pragma GCC system_header
@@ -23,6 +26,10 @@ class __bit_iterator;
 template <class, class = void>
 struct __size_difference_type_traits;
 
+template <class _StoragePointer>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 void
+__fill_masked_range(_StoragePointer __word, unsigned __ctz, unsigned __clz, bool __fill_val);
+
 _LIBCPP_END_NAMESPACE_STD
 
 #endif // _LIBCPP___FWD_BIT_REFERENCE_H
diff --git a/libcxx/include/__hash_table b/libcxx/include/__hash_table
index 9a82ec51daee7..d7b312f8774fc 100644
--- a/libcxx/include/__hash_table
+++ b/libcxx/include/__hash_table
@@ -770,9 +770,10 @@ public:
 
   _LIBCPP_HIDE_FROM_ABI __hash_table& operator=(const __hash_table& __u);
   _LIBCPP_HIDE_FROM_ABI __hash_table& operator=(__hash_table&& __u)
-      _NOEXCEPT_(__node_traits::propagate_on_container_move_assignment::value&&
-                     is_nothrow_move_assignable<__node_allocator>::value&& is_nothrow_move_assignable<hasher>::value&&
-                         is_nothrow_move_assignable<key_equal>::value);
+      _NOEXCEPT_(is_nothrow_move_assignable<hasher>::value&& is_nothrow_move_assignable<key_equal>::value &&
+                 ((__node_traits::propagate_on_container_move_assignment::value &&
+                   is_nothrow_move_assignable<__node_allocator>::value) ||
+                  allocator_traits<__node_allocator>::is_always_equal::value));
   template <class _InputIterator>
   _LIBCPP_HIDE_FROM_ABI void __assign_unique(_InputIterator __first, _InputIterator __last);
   template <class _InputIterator>
@@ -1238,10 +1239,11 @@ void __hash_table<_Tp, _Hash, _Equal, _Alloc>::__move_assign(__hash_table& __u,
 }
 
 template <class _Tp, class _Hash, class _Equal, class _Alloc>
-inline __hash_table<_Tp, _Hash, _Equal, _Alloc>&
-__hash_table<_Tp, _Hash, _Equal, _Alloc>::operator=(__hash_table&& __u) _NOEXCEPT_(
-    __node_traits::propagate_on_container_move_assignment::value&& is_nothrow_move_assignable<__node_allocator>::value&&
-        is_nothrow_move_assignable<hasher>::value&& is_nothrow_move_assignable<key_equal>::value) {
+inline __hash_table<_Tp, _Hash, _Equal, _Alloc>& __hash_table<_Tp, _Hash, _Equal, _Alloc>::operator=(__hash_table&& __u)
+    _NOEXCEPT_(is_nothrow_move_assignable<hasher>::value&& is_nothrow_move_assignable<key_equal>::value &&
+               ((__node_traits::propagate_on_container_move_assignment::value &&
+                 is_nothrow_move_assignable<__node_allocator>::value) ||
+                allocator_traits<__node_allocator>::is_always_equal::value)) {
   __move_assign(__u, integral_constant<bool, __node_traits::propagate_on_container_move_assignment::value>());
   return *this;
 }
diff --git a/libcxx/include/__locale b/libcxx/include/__locale
index e10eb62fb844b..dfe79d5e506f1 100644
--- a/libcxx/include/__locale
+++ b/libcxx/include/__locale
@@ -60,8 +60,9 @@ public:
 
   static const category // values assigned here are for exposition only
       none    = 0,
-      collate = LC_COLLATE_MASK, ctype = LC_CTYPE_MASK, monetary = LC_MONETARY_MASK, numeric = LC_NUMERIC_MASK,
-      time = LC_TIME_MASK, messages = LC_MESSAGES_MASK, all = collate | ctype | monetary | numeric | time | messages;
+      collate = _LIBCPP_COLLATE_MASK, ctype = _LIBCPP_CTYPE_MASK, monetary = _LIBCPP_MONETARY_MASK,
+      numeric = _LIBCPP_NUMERIC_MASK, time = _LIBCPP_TIME_MASK, messages = _LIBCPP_MESSAGES_MASK,
+      all = collate | ctype | monetary | numeric | time | messages;
 
   // construct/copy/destroy:
   locale() _NOEXCEPT;
diff --git a/libcxx/include/__locale_dir/locale_base_api.h b/libcxx/include/__locale_dir/locale_base_api.h
index b112a4aef7765..bbee9f49867fd 100644
--- a/libcxx/include/__locale_dir/locale_base_api.h
+++ b/libcxx/include/__locale_dir/locale_base_api.h
@@ -29,11 +29,22 @@
 // -----------------
 // namespace __locale {
 //  using __locale_t = implementation-defined;
+//  using __lconv_t  = implementation-defined;
 //  __locale_t  __newlocale(int, const char*, __locale_t);
 //  void        __freelocale(__locale_t);
-//  lconv*      __localeconv(__locale_t&);
+//  char*       __setlocale(int, const char*);
+//  __lconv_t*  __localeconv(__locale_t&);
 // }
 //
+// #define _LIBCPP_COLLATE_MASK   /* implementation-defined */
+// #define _LIBCPP_CTYPE_MASK     /* implementation-defined */
+// #define _LIBCPP_MONETARY_MASK  /* implementation-defined */
+// #define _LIBCPP_NUMERIC_MASK   /* implementation-defined */
+// #define _LIBCPP_TIME_MASK      /* implementation-defined */
+// #define _LIBCPP_MESSAGES_MASK  /* implementation-defined */
+// #define _LIBCPP_ALL_MASK       /* implementation-defined */
+// #define _LIBCPP_LC_ALL         /* implementation-defined */
+//
 // Strtonum functions
 // ------------------
 // namespace __locale {
@@ -133,14 +144,28 @@ namespace __locale {
 // Locale management
 //
 using __locale_t _LIBCPP_NODEBUG = locale_t;
+using __lconv_t _LIBCPP_NODEBUG  = lconv;
 
 inline _LIBCPP_HIDE_FROM_ABI __locale_t __newlocale(int __category_mask, const char* __name, __locale_t __loc) {
   return newlocale(__category_mask, __name, __loc);
 }
 
+inline _LIBCPP_HIDE_FROM_ABI char* __setlocale(int __category, char const* __locale) {
+  return ::setlocale(__category, __locale);
+}
+
 inline _LIBCPP_HIDE_FROM_ABI void __freelocale(__locale_t __loc) { freelocale(__loc); }
 
-inline _LIBCPP_HIDE_FROM_ABI lconv* __localeconv(__locale_t& __loc) { return __libcpp_localeconv_l(__loc); }
+inline _LIBCPP_HIDE_FROM_ABI __lconv_t* __localeconv(__locale_t& __loc) { return __libcpp_localeconv_l(__loc); }
+
+#  define _LIBCPP_COLLATE_MASK LC_COLLATE_MASK
+#  define _LIBCPP_CTYPE_MASK LC_CTYPE_MASK
+#  define _LIBCPP_MONETARY_MASK LC_MONETARY_MASK
+#  define _LIBCPP_NUMERIC_MASK LC_NUMERIC_MASK
+#  define _LIBCPP_TIME_MASK LC_TIME_MASK
+#  define _LIBCPP_MESSAGES_MASK LC_MESSAGES_MASK
+#  define _LIBCPP_ALL_MASK LC_ALL_MASK
+#  define _LIBCPP_LC_ALL LC_ALL
 
 //
 // Strtonum functions
diff --git a/libcxx/include/__locale_dir/support/bsd_like.h b/libcxx/include/__locale_dir/support/bsd_like.h
index b3933c71c6b26..c0080b13a08cf 100644
--- a/libcxx/include/__locale_dir/support/bsd_like.h
+++ b/libcxx/include/__locale_dir/support/bsd_like.h
@@ -36,7 +36,17 @@ namespace __locale {
 //
 // Locale management
 //
+#define _LIBCPP_COLLATE_MASK LC_COLLATE_MASK
+#define _LIBCPP_CTYPE_MASK LC_CTYPE_MASK
+#define _LIBCPP_MONETARY_MASK LC_MONETARY_MASK
+#define _LIBCPP_NUMERIC_MASK LC_NUMERIC_MASK
+#define _LIBCPP_TIME_MASK LC_TIME_MASK
+#define _LIBCPP_MESSAGES_MASK LC_MESSAGES_MASK
+#define _LIBCPP_ALL_MASK LC_ALL_MASK
+#define _LIBCPP_LC_ALL LC_ALL
+
 using __locale_t = ::locale_t;
+using __lconv_t  = std::lconv;
 
 inline _LIBCPP_HIDE_FROM_ABI __locale_t __newlocale(int __category_mask, const char* __locale, __locale_t __base) {
   return ::newlocale(__category_mask, __locale, __base);
@@ -44,7 +54,11 @@ inline _LIBCPP_HIDE_FROM_ABI __locale_t __newlocale(int __category_mask, const c
 
 inline _LIBCPP_HIDE_FROM_ABI void __freelocale(__locale_t __loc) { ::freelocale(__loc); }
 
-inline _LIBCPP_HIDE_FROM_ABI lconv* __localeconv(__locale_t& __loc) { return ::localeconv_l(__loc); }
+inline _LIBCPP_HIDE_FROM_ABI char* __setlocale(int __category, char const* __locale) {
+  return ::setlocale(__category, __locale);
+}
+
+inline _LIBCPP_HIDE_FROM_ABI __lconv_t* __localeconv(__locale_t& __loc) { return ::localeconv_l(__loc); }
 
 //
 // Strtonum functions
diff --git a/libcxx/include/__locale_dir/support/fuchsia.h b/libcxx/include/__locale_dir/support/fuchsia.h
index 4a54896c8e268..237f48562d6e0 100644
--- a/libcxx/include/__locale_dir/support/fuchsia.h
+++ b/libcxx/include/__locale_dir/support/fuchsia.h
@@ -40,7 +40,17 @@ struct __locale_guard {
 //
 // Locale management
 //
+#define _LIBCPP_COLLATE_MASK LC_COLLATE_MASK
+#define _LIBCPP_CTYPE_MASK LC_CTYPE_MASK
+#define _LIBCPP_MONETARY_MASK LC_MONETARY_MASK
+#define _LIBCPP_NUMERIC_MASK LC_NUMERIC_MASK
+#define _LIBCPP_TIME_MASK LC_TIME_MASK
+#define _LIBCPP_MESSAGES_MASK LC_MESSAGES_MASK
+#define _LIBCPP_ALL_MASK LC_ALL_MASK
+#define _LIBCPP_LC_ALL LC_ALL
+
 using __locale_t = locale_t;
+using __lconv_t  = std::lconv;
 
 inline _LIBCPP_HIDE_FROM_ABI __locale_t __newlocale(int __category_mask, const char* __name, __locale_t __loc) {
   return ::newlocale(__category_mask, __name, __loc);
@@ -48,7 +58,11 @@ inline _LIBCPP_HIDE_FROM_ABI __locale_t __newlocale(int __category_mask, const c
 
 inline _LIBCPP_HIDE_FROM_ABI void __freelocale(__locale_t __loc) { ::freelocale(__loc); }
 
-inline _LIBCPP_HIDE_FROM_ABI lconv* __localeconv(__locale_t& __loc) {
+inline _LIBCPP_HIDE_FROM_ABI char* __setlocale(int __category, char const* __locale) {
+  return ::setlocale(__category, __locale);
+}
+
+inline _LIBCPP_HIDE_FROM_ABI __lconv_t* __localeconv(__locale_t& __loc) {
   __locale_guard __current(__loc);
   return std::localeconv();
 }
diff --git a/libcxx/include/__locale_dir/support/windows.h b/libcxx/include/__locale_dir/support/windows.h
index eca0e17d94c85..f0f76c527264a 100644
--- a/libcxx/include/__locale_dir/support/windows.h
+++ b/libcxx/include/__locale_dir/support/windows.h
@@ -26,22 +26,14 @@
 #  pragma GCC system_header
 #endif
 
-#define _CATMASK(n) ((1 << (n)) >> 1)
-#define LC_COLLATE_MASK _CATMASK(LC_COLLATE)
-#define LC_CTYPE_MASK _CATMASK(LC_CTYPE)
-#define LC_MONETARY_MASK _CATMASK(LC_MONETARY)
-#define LC_NUMERIC_MASK _CATMASK(LC_NUMERIC)
-#define LC_TIME_MASK _CATMASK(LC_TIME)
-#define LC_MESSAGES_MASK _CATMASK(6)
-#define LC_ALL_MASK                                                                                                    \
-  (LC_COLLATE_MASK | LC_CTYPE_MASK | LC_MESSAGES_MASK | LC_MONETARY_MASK | LC_NUMERIC_MASK | LC_TIME_MASK)
-
 _LIBCPP_BEGIN_NAMESPACE_STD
 namespace __locale {
 
+using __lconv_t = std::lconv;
+
 class __lconv_storage {
 public:
-  __lconv_storage(const lconv* __lc_input) {
+  __lconv_storage(const __lconv_t* __lc_input) {
     __lc_ = *__lc_input;
 
     __decimal_point_     = __lc_input->decimal_point;
@@ -67,10 +59,10 @@ class __lconv_storage {
     __lc_.negative_sign     = const_cast<char*>(__negative_sign_.c_str());
   }
 
-  std::lconv* __get() { return &__lc_; }
+  __lconv_t* __get() { return &__lc_; }
 
 private:
-  std::lconv __lc_;
+  __lconv_t __lc_;
   std::string __decimal_point_;
   std::string __thousands_sep_;
   std::string __grouping_;
@@ -86,6 +78,18 @@ class __lconv_storage {
 //
 // Locale management
 //
+#define _CATMASK(n) ((1 << (n)) >> 1)
+#define _LIBCPP_COLLATE_MASK _CATMASK(LC_COLLATE)
+#define _LIBCPP_CTYPE_MASK _CATMASK(LC_CTYPE)
+#define _LIBCPP_MONETARY_MASK _CATMASK(LC_MONETARY)
+#define _LIBCPP_NUMERIC_MASK _CATMASK(LC_NUMERIC)
+#define _LIBCPP_TIME_MASK _CATMASK(LC_TIME)
+#define _LIBCPP_MESSAGES_MASK _CATMASK(6)
+#define _LIBCPP_ALL_MASK                                                                                               \
+  (_LIBCPP_COLLATE_MASK | _LIBCPP_CTYPE_MASK | _LIBCPP_MESSAGES_MASK | _LIBCPP_MONETARY_MASK | _LIBCPP_NUMERIC_MASK |  \
+   _LIBCPP_TIME_MASK)
+#define _LIBCPP_LC_ALL LC_ALL
+
 class __locale_t {
 public:
   __locale_t() : __locale_(nullptr), __locale_str_(nullptr), __lc_(nullptr) {}
@@ -137,7 +141,7 @@ class __locale_t {
 
   operator ::_locale_t() const { return __locale_; }
 
-  std::lconv* __store_lconv(const std::lconv* __input_lc) {
+  __lconv_t* __store_lconv(const __lconv_t* __input_lc) {
     delete __lc_;
     __lc_ = new __lconv_storage(__input_lc);
     return __lc_->__get();
@@ -151,7 +155,13 @@ class __locale_t {
 
 _LIBCPP_EXPORTED_FROM_ABI __locale_t __newlocale(int __mask, const char* __locale, __locale_t __base);
 inline _LIBCPP_HIDE_FROM_ABI void __freelocale(__locale_t __loc) { ::_free_locale(__loc); }
-_LIBCPP_EXPORTED_FROM_ABI lconv* __localeconv(__locale_t& __loc);
+inline _LIBCPP_HIDE_FROM_ABI char* __setlocale(int __category, const char* __locale) {
+  char* __new_locale = ::setlocale(__category, __locale);
+  if (__new_locale == nullptr)
+    std::__throw_bad_alloc();
+  return __new_locale;
+}
+_LIBCPP_EXPORTED_FROM_ABI __lconv_t* __localeconv(__locale_t& __loc);
 
 //
 // Strtonum functions
@@ -205,7 +215,7 @@ inline _LIBCPP_HIDE_FROM_ABI size_t __strxfrm(char* __dest, const char* __src, s
   return ::_strxfrm_l(__dest, __src, __n, __loc);
 }
 
-#ifndef _LIBCPP_HAS_NO_WIDE_CHARACTERS
+#if _LIBCPP_HAS_WIDE_CHARACTERS
 inline _LIBCPP_HIDE_FROM_ABI int __iswctype(wint_t __c, wctype_t __type, __locale_t __loc) {
   return ::_iswctype_l(__c, __type, __loc);
 }
@@ -230,7 +240,7 @@ inline _LIBCPP_HIDE_FROM_ABI int __wcscoll(const wchar_t* __ws1, const wchar_t*
 inline _LIBCPP_HIDE_FROM_ABI size_t __wcsxfrm(wchar_t* __dest, const wchar_t* __src, size_t __n, __locale_t __loc) {
   return ::_wcsxfrm_l(__dest, __src, __n, __loc);
 }
-#endif // !_LIBCPP_HAS_NO_WIDE_CHARACTERS
+#endif // _LIBCPP_HAS_WIDE_CHARACTERS
 
 #if defined(__MINGW32__) && __MSVCRT_VERSION__ < 0x0800
 _LIBCPP_EXPORTED_FROM_ABI size_t __strftime(char*, size_t, const char*, const struct tm*, __locale_t);
@@ -292,7 +302,7 @@ struct __locale_guard {
     // Setting the locale can be expensive even when the locale given is
     // already the current locale, so do an explicit check to see if the
     // current locale is already the one we want.
-    const char* __lc = __setlocale(nullptr);
+    const char* __lc = __locale::__setlocale(LC_ALL, nullptr);
     // If every category is the same, the locale string will simply be the
     // locale name, otherwise it will be a semicolon-separated string listing
     // each category.  In the second case, we know at least one category won't
@@ -301,7 +311,7 @@ struct __locale_guard {
       __locale_all = _strdup(__lc);
       if (__locale_all == nullptr)
         __throw_bad_alloc();
-      __setlocale(__l.__get_locale());
+      __locale::__setlocale(LC_ALL, __l.__get_locale());
     }
   }
   _LIBCPP_HIDE_FROM_ABI ~__locale_guard() {
@@ -310,17 +320,11 @@ struct __locale_guard {
     // for the different categories in the same format as returned by
     // setlocale(LC_ALL, nullptr).
     if (__locale_all != nullptr) {
-      __setlocale(__locale_all);
+      __locale::__setlocale(LC_ALL, __locale_all);
       free(__locale_all);
     }
     _configthreadlocale(__status);
   }
-  _LIBCPP_HIDE_FROM_ABI static const char* __setlocale(const char* __locale) {
-    const char* __new_locale = setlocale(LC_ALL, __locale);
-    if (__new_locale == nullptr)
-      __throw_bad_alloc();
-    return __new_locale;
-  }
   int __status;
   char* __locale_all = nullptr;
 };
diff --git a/libcxx/include/__memory/construct_at.h b/libcxx/include/__memory/construct_at.h
index 1f129d17970b1..21337e766b2d0 100644
--- a/libcxx/include/__memory/construct_at.h
+++ b/libcxx/include/__memory/construct_at.h
@@ -57,9 +57,6 @@ _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _Tp* __construct_at(_Tp* __l
 // The internal functions are available regardless of the language version (with the exception of the `__destroy_at`
 // taking an array).
 
-template <class _ForwardIterator>
-_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _ForwardIterator __destroy(_ForwardIterator, _ForwardIterator);
-
 template <class _Tp, __enable_if_t<!is_array<_Tp>::value, int> = 0>
 _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void __destroy_at(_Tp* __loc) {
   _LIBCPP_ASSERT_NON_NULL(__loc != nullptr, "null pointer given to destroy_at");
@@ -70,28 +67,12 @@ _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void __destroy_at(_Tp* __loc
 template <class _Tp, __enable_if_t<is_array<_Tp>::value, int> = 0>
 _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void __destroy_at(_Tp* __loc) {
   _LIBCPP_ASSERT_NON_NULL(__loc != nullptr, "null pointer given to destroy_at");
-  std::__destroy(std::begin(*__loc), std::end(*__loc));
+  auto const __end = std::end(*__loc);
+  for (auto __it = std::begin(*__loc); __it != __end; ++__it)
+    std::__destroy_at(__it);
 }
 #endif
 
-template <class _ForwardIterator>
-_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _ForwardIterator
-__destroy(_ForwardIterator __first, _ForwardIterator __last) {
-  for (; __first != __last; ++__first)
-    std::__destroy_at(std::addressof(*__first));
-  return __first;
-}
-
-template <class _BidirectionalIterator>
-_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _BidirectionalIterator
-__reverse_destroy(_BidirectionalIterator __first, _BidirectionalIterator __last) {
-  while (__last != __first) {
-    --__last;
-    std::__destroy_at(std::addressof(*__last));
-  }
-  return __last;
-}
-
 #if _LIBCPP_STD_VER >= 17
 
 template <class _Tp, enable_if_t<!is_array_v<_Tp>, int> = 0>
@@ -106,18 +87,6 @@ _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void destroy_at(_Tp* __loc)
 }
 #  endif
 
-template <class _ForwardIterator>
-_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void destroy(_ForwardIterator __first, _ForwardIterator __last) {
-  (void)std::__destroy(std::move(__first), std::move(__last));
-}
-
-template <class _ForwardIterator, class _Size>
-_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _ForwardIterator destroy_n(_ForwardIterator __first, _Size __n) {
-  for (; __n > 0; (void)++__first, --__n)
-    std::__destroy_at(std::addressof(*__first));
-  return __first;
-}
-
 #endif // _LIBCPP_STD_VER >= 17
 
 _LIBCPP_END_NAMESPACE_STD
diff --git a/libcxx/include/__memory/destroy.h b/libcxx/include/__memory/destroy.h
new file mode 100644
index 0000000000000..69a252ba1331d
--- /dev/null
+++ b/libcxx/include/__memory/destroy.h
@@ -0,0 +1,71 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___MEMORY_DESTROY_H
+#define _LIBCPP___MEMORY_DESTROY_H
+
+#include <__config>
+#include <__memory/addressof.h>
+#include <__memory/allocator_traits.h>
+#include <__memory/construct_at.h>
+#include <__utility/move.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _ForwardIterator>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _ForwardIterator
+__destroy(_ForwardIterator __first, _ForwardIterator __last) {
+  for (; __first != __last; ++__first)
+    std::__destroy_at(std::addressof(*__first));
+  return __first;
+}
+
+template <class _BidirectionalIterator>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _BidirectionalIterator
+__reverse_destroy(_BidirectionalIterator __first, _BidirectionalIterator __last) {
+  while (__last != __first) {
+    --__last;
+    std::__destroy_at(std::addressof(*__last));
+  }
+  return __last;
+}
+
+// Destroy all elements in [__first, __last) from left to right using allocator destruction.
+template <class _Alloc, class _Iter, class _Sent>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void
+__allocator_destroy(_Alloc& __alloc, _Iter __first, _Sent __last) {
+  for (; __first != __last; ++__first)
+    allocator_traits<_Alloc>::destroy(__alloc, std::addressof(*__first));
+}
+
+#if _LIBCPP_STD_VER >= 17
+template <class _ForwardIterator>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void destroy(_ForwardIterator __first, _ForwardIterator __last) {
+  (void)std::__destroy(std::move(__first), std::move(__last));
+}
+
+template <class _ForwardIterator, class _Size>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _ForwardIterator destroy_n(_ForwardIterator __first, _Size __n) {
+  for (; __n > 0; (void)++__first, --__n)
+    std::__destroy_at(std::addressof(*__first));
+  return __first;
+}
+#endif
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___MEMORY_DESTROY_H
diff --git a/libcxx/include/__memory/ranges_construct_at.h b/libcxx/include/__memory/ranges_construct_at.h
index 35ed796510003..b8a94678c10c8 100644
--- a/libcxx/include/__memory/ranges_construct_at.h
+++ b/libcxx/include/__memory/ranges_construct_at.h
@@ -61,41 +61,6 @@ inline namespace __cpo {
 inline constexpr auto destroy_at = __destroy_at{};
 } // namespace __cpo
 
-// destroy
-
-struct __destroy {
-  template <__nothrow_input_iterator _InputIterator, __nothrow_sentinel_for<_InputIterator> _Sentinel>
-    requires destructible<iter_value_t<_InputIterator>>
-  _LIBCPP_HIDE_FROM_ABI constexpr _InputIterator operator()(_InputIterator __first, _Sentinel __last) const noexcept {
-    return std::__destroy(std::move(__first), std::move(__last));
-  }
-
-  template <__nothrow_input_range _InputRange>
-    requires destructible<range_value_t<_InputRange>>
-  _LIBCPP_HIDE_FROM_ABI constexpr borrowed_iterator_t<_InputRange> operator()(_InputRange&& __range) const noexcept {
-    return (*this)(ranges::begin(__range), ranges::end(__range));
-  }
-};
-
-inline namespace __cpo {
-inline constexpr auto destroy = __destroy{};
-} // namespace __cpo
-
-// destroy_n
-
-struct __destroy_n {
-  template <__nothrow_input_iterator _InputIterator>
-    requires destructible<iter_value_t<_InputIterator>>
-  _LIBCPP_HIDE_FROM_ABI constexpr _InputIterator
-  operator()(_InputIterator __first, iter_difference_t<_InputIterator> __n) const noexcept {
-    return std::destroy_n(std::move(__first), __n);
-  }
-};
-
-inline namespace __cpo {
-inline constexpr auto destroy_n = __destroy_n{};
-} // namespace __cpo
-
 } // namespace ranges
 
 #endif // _LIBCPP_STD_VER >= 20
diff --git a/libcxx/include/__memory/ranges_destroy.h b/libcxx/include/__memory/ranges_destroy.h
new file mode 100644
index 0000000000000..f134f1422e82c
--- /dev/null
+++ b/libcxx/include/__memory/ranges_destroy.h
@@ -0,0 +1,79 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___MEMORY_RANGES_DESTROY_H
+#define _LIBCPP___MEMORY_RANGES_DESTROY_H
+
+#include <__concepts/destructible.h>
+#include <__config>
+#include <__iterator/incrementable_traits.h>
+#include <__iterator/iterator_traits.h>
+#include <__memory/concepts.h>
+#include <__memory/destroy.h>
+#include <__ranges/access.h>
+#include <__ranges/concepts.h>
+#include <__ranges/dangling.h>
+#include <__utility/move.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER >= 20
+namespace ranges {
+
+// destroy
+
+struct __destroy {
+  template <__nothrow_input_iterator _InputIterator, __nothrow_sentinel_for<_InputIterator> _Sentinel>
+    requires destructible<iter_value_t<_InputIterator>>
+  _LIBCPP_HIDE_FROM_ABI constexpr _InputIterator operator()(_InputIterator __first, _Sentinel __last) const noexcept {
+    return std::__destroy(std::move(__first), std::move(__last));
+  }
+
+  template <__nothrow_input_range _InputRange>
+    requires destructible<range_value_t<_InputRange>>
+  _LIBCPP_HIDE_FROM_ABI constexpr borrowed_iterator_t<_InputRange> operator()(_InputRange&& __range) const noexcept {
+    return (*this)(ranges::begin(__range), ranges::end(__range));
+  }
+};
+
+inline namespace __cpo {
+inline constexpr auto destroy = __destroy{};
+} // namespace __cpo
+
+// destroy_n
+
+struct __destroy_n {
+  template <__nothrow_input_iterator _InputIterator>
+    requires destructible<iter_value_t<_InputIterator>>
+  _LIBCPP_HIDE_FROM_ABI constexpr _InputIterator
+  operator()(_InputIterator __first, iter_difference_t<_InputIterator> __n) const noexcept {
+    return std::destroy_n(std::move(__first), __n);
+  }
+};
+
+inline namespace __cpo {
+inline constexpr auto destroy_n = __destroy_n{};
+} // namespace __cpo
+
+} // namespace ranges
+
+#endif // _LIBCPP_STD_VER >= 20
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___MEMORY_RANGES_DESTROY_H
diff --git a/libcxx/include/__memory/shared_ptr.h b/libcxx/include/__memory/shared_ptr.h
index 06b1fc488cf51..d513fa692c1f9 100644
--- a/libcxx/include/__memory/shared_ptr.h
+++ b/libcxx/include/__memory/shared_ptr.h
@@ -29,6 +29,7 @@
 #include <__memory/auto_ptr.h>
 #include <__memory/compressed_pair.h>
 #include <__memory/construct_at.h>
+#include <__memory/destroy.h>
 #include <__memory/pointer_traits.h>
 #include <__memory/shared_count.h>
 #include <__memory/uninitialized_algorithms.h>
diff --git a/libcxx/include/__memory/uninitialized_algorithms.h b/libcxx/include/__memory/uninitialized_algorithms.h
index a02a88399a7a7..d7f210d12a212 100644
--- a/libcxx/include/__memory/uninitialized_algorithms.h
+++ b/libcxx/include/__memory/uninitialized_algorithms.h
@@ -21,6 +21,7 @@
 #include <__memory/addressof.h>
 #include <__memory/allocator_traits.h>
 #include <__memory/construct_at.h>
+#include <__memory/destroy.h>
 #include <__memory/pointer_traits.h>
 #include <__type_traits/enable_if.h>
 #include <__type_traits/extent.h>
@@ -511,14 +512,6 @@ __uninitialized_allocator_value_construct_n_multidimensional(_Alloc& __alloc, _B
 
 #endif // _LIBCPP_STD_VER >= 17
 
-// Destroy all elements in [__first, __last) from left to right using allocator destruction.
-template <class _Alloc, class _Iter, class _Sent>
-_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void
-__allocator_destroy(_Alloc& __alloc, _Iter __first, _Sent __last) {
-  for (; __first != __last; ++__first)
-    allocator_traits<_Alloc>::destroy(__alloc, std::__to_address(__first));
-}
-
 template <class _Alloc, class _Iter>
 class _AllocatorDestroyRangeReverse {
 public:
diff --git a/libcxx/include/__new/allocate.h b/libcxx/include/__new/allocate.h
index a64663c09fa35..738fa62af4d61 100644
--- a/libcxx/include/__new/allocate.h
+++ b/libcxx/include/__new/allocate.h
@@ -50,7 +50,8 @@ _LIBCPP_HIDE_FROM_ABI void __libcpp_operator_delete(_Args... __args) _NOEXCEPT {
 }
 
 template <class _Tp>
-inline _LIBCPP_HIDE_FROM_ABI _Tp* __libcpp_allocate(__element_count __n, size_t __align = _LIBCPP_ALIGNOF(_Tp)) {
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_NO_CFI _Tp*
+__libcpp_allocate(__element_count __n, size_t __align = _LIBCPP_ALIGNOF(_Tp)) {
   size_t __size = static_cast<size_t>(__n) * sizeof(_Tp);
 #if _LIBCPP_HAS_ALIGNED_ALLOCATION
   if (__is_overaligned_for_new(__align)) {
diff --git a/libcxx/include/__pstl/backends/libdispatch.h b/libcxx/include/__pstl/backends/libdispatch.h
index 4c63c4c844207..a640a40352f5c 100644
--- a/libcxx/include/__pstl/backends/libdispatch.h
+++ b/libcxx/include/__pstl/backends/libdispatch.h
@@ -22,6 +22,7 @@
 #include <__iterator/move_iterator.h>
 #include <__memory/allocator.h>
 #include <__memory/construct_at.h>
+#include <__memory/destroy.h>
 #include <__memory/unique_ptr.h>
 #include <__new/exceptions.h>
 #include <__numeric/reduce.h>
diff --git a/libcxx/include/__string/constexpr_c_functions.h b/libcxx/include/__string/constexpr_c_functions.h
index 0bc128b68b579..fbe7e10d440ce 100644
--- a/libcxx/include/__string/constexpr_c_functions.h
+++ b/libcxx/include/__string/constexpr_c_functions.h
@@ -204,23 +204,26 @@ _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _Tp& __assign_trivially_copy
   return __dest;
 }
 
-template <class _Tp, class _Up, __enable_if_t<__is_always_bitcastable<_Up, _Tp>::value, int> = 0>
+template <class _Tp, class _Up>
 _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 _Tp*
 __constexpr_memmove(_Tp* __dest, _Up* __src, __element_count __n) {
+  static_assert(__is_always_bitcastable<_Up, _Tp>::value);
   size_t __count = static_cast<size_t>(__n);
   if (__libcpp_is_constant_evaluated()) {
 #ifdef _LIBCPP_COMPILER_CLANG_BASED
-    if (is_same<__remove_cv_t<_Tp>, __remove_cv_t<_Up> >::value) {
+    if _LIBCPP_CONSTEXPR (is_same<__remove_cv_t<_Tp>, __remove_cv_t<_Up> >::value) {
       ::__builtin_memmove(__dest, __src, __count * sizeof(_Tp));
       return __dest;
-    }
+    } else
 #endif
-    if (std::__is_pointer_in_range(__src, __src + __count, __dest)) {
-      for (; __count > 0; --__count)
-        std::__assign_trivially_copyable(__dest[__count - 1], __src[__count - 1]);
-    } else {
-      for (size_t __i = 0; __i != __count; ++__i)
-        std::__assign_trivially_copyable(__dest[__i], __src[__i]);
+    {
+      if (std::__is_pointer_in_range(__src, __src + __count, __dest)) {
+        for (; __count > 0; --__count)
+          std::__assign_trivially_copyable(__dest[__count - 1], __src[__count - 1]);
+      } else {
+        for (size_t __i = 0; __i != __count; ++__i)
+          std::__assign_trivially_copyable(__dest[__i], __src[__i]);
+      }
     }
   } else if (__count > 0) {
     ::__builtin_memmove(__dest, __src, (__count - 1) * sizeof(_Tp) + __datasizeof_v<_Tp>);
diff --git a/libcxx/include/__tree b/libcxx/include/__tree
index acad6c33f8782..c627641d5d86f 100644
--- a/libcxx/include/__tree
+++ b/libcxx/include/__tree
@@ -987,9 +987,12 @@ public:
   _LIBCPP_HIDE_FROM_ABI __tree(__tree&& __t) _NOEXCEPT_(
       is_nothrow_move_constructible<__node_allocator>::value&& is_nothrow_move_constructible<value_compare>::value);
   _LIBCPP_HIDE_FROM_ABI __tree(__tree&& __t, const allocator_type& __a);
-  _LIBCPP_HIDE_FROM_ABI __tree& operator=(__tree&& __t) _NOEXCEPT_(
-      __node_traits::propagate_on_container_move_assignment::value&& is_nothrow_move_assignable<value_compare>::value&&
-          is_nothrow_move_assignable<__node_allocator>::value);
+  _LIBCPP_HIDE_FROM_ABI __tree& operator=(__tree&& __t)
+      _NOEXCEPT_(is_nothrow_move_assignable<value_compare>::value &&
+                 ((__node_traits::propagate_on_container_move_assignment::value &&
+                   is_nothrow_move_assignable<__node_allocator>::value) ||
+                  allocator_traits<__node_allocator>::is_always_equal::value));
+
   _LIBCPP_HIDE_FROM_ABI ~__tree();
 
   _LIBCPP_HIDE_FROM_ABI iterator begin() _NOEXCEPT { return iterator(__begin_node()); }
@@ -1520,11 +1523,11 @@ void __tree<_Tp, _Compare, _Allocator>::__move_assign(__tree& __t, false_type) {
 }
 
 template <class _Tp, class _Compare, class _Allocator>
-__tree<_Tp, _Compare, _Allocator>& __tree<_Tp, _Compare, _Allocator>::operator=(__tree&& __t) _NOEXCEPT_(
-    __node_traits::propagate_on_container_move_assignment::value&& is_nothrow_move_assignable<value_compare>::value&&
-        is_nothrow_move_assignable<__node_allocator>::value)
-
-{
+__tree<_Tp, _Compare, _Allocator>& __tree<_Tp, _Compare, _Allocator>::operator=(__tree&& __t)
+    _NOEXCEPT_(is_nothrow_move_assignable<value_compare>::value &&
+               ((__node_traits::propagate_on_container_move_assignment::value &&
+                 is_nothrow_move_assignable<__node_allocator>::value) ||
+                allocator_traits<__node_allocator>::is_always_equal::value)) {
   __move_assign(__t, integral_constant<bool, __node_traits::propagate_on_container_move_assignment::value>());
   return *this;
 }
diff --git a/libcxx/include/__type_traits/is_trivially_relocatable.h b/libcxx/include/__type_traits/is_trivially_relocatable.h
index c0871731cc001..9b0e240de55f4 100644
--- a/libcxx/include/__type_traits/is_trivially_relocatable.h
+++ b/libcxx/include/__type_traits/is_trivially_relocatable.h
@@ -11,7 +11,6 @@
 
 #include <__config>
 #include <__type_traits/enable_if.h>
-#include <__type_traits/integral_constant.h>
 #include <__type_traits/is_same.h>
 #include <__type_traits/is_trivially_copyable.h>
 
@@ -23,8 +22,11 @@ _LIBCPP_BEGIN_NAMESPACE_STD
 
 // A type is trivially relocatable if a move construct + destroy of the original object is equivalent to
 // `memcpy(dst, src, sizeof(T))`.
-
-#if __has_builtin(__is_trivially_relocatable)
+//
+// Note that we don't use the __is_trivially_relocatable Clang builtin right now because it does not
+// implement the semantics of any current or future trivial relocation proposal and it can lead to
+// incorrect optimizations on some platforms (Windows) and supported compilers (AppleClang).
+#if __has_builtin(__is_trivially_relocatable) && 0
 template <class _Tp, class = void>
 struct __libcpp_is_trivially_relocatable : integral_constant<bool, __is_trivially_relocatable(_Tp)> {};
 #else
diff --git a/libcxx/include/__type_traits/promote.h b/libcxx/include/__type_traits/promote.h
index 0f545bc507398..b449a749004ab 100644
--- a/libcxx/include/__type_traits/promote.h
+++ b/libcxx/include/__type_traits/promote.h
@@ -13,20 +13,12 @@
 #include <__type_traits/integral_constant.h>
 #include <__type_traits/is_arithmetic.h>
 
-#if defined(_LIBCPP_CLANG_VER) && _LIBCPP_CLANG_VER == 1700
-#  include <__type_traits/is_same.h>
-#  include <__utility/declval.h>
-#endif
-
 #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
 #  pragma GCC system_header
 #endif
 
 _LIBCPP_BEGIN_NAMESPACE_STD
 
-// TODO(LLVM-20): Remove this workaround
-#if !defined(_LIBCPP_CLANG_VER) || _LIBCPP_CLANG_VER != 1700
-
 template <class... _Args>
 class __promote {
   static_assert((is_arithmetic<_Args>::value && ...));
@@ -39,10 +31,10 @@ class __promote {
   static double __test(unsigned long);
   static double __test(long long);
   static double __test(unsigned long long);
-#  if _LIBCPP_HAS_INT128
+#if _LIBCPP_HAS_INT128
   static double __test(__int128_t);
   static double __test(__uint128_t);
-#  endif
+#endif
   static double __test(double);
   static long double __test(long double);
 
@@ -50,79 +42,6 @@ class __promote {
   using type = decltype((__test(_Args()) + ...));
 };
 
-#else
-
-template <class _Tp>
-struct __numeric_type {
-  static void __test(...);
-  static float __test(float);
-  static double __test(char);
-  static double __test(int);
-  static double __test(unsigned);
-  static double __test(long);
-  static double __test(unsigned long);
-  static double __test(long long);
-  static double __test(unsigned long long);
-#  if _LIBCPP_HAS_INT128
-  static double __test(__int128_t);
-  static double __test(__uint128_t);
-#  endif
-  static double __test(double);
-  static long double __test(long double);
-
-  typedef decltype(__test(std::declval<_Tp>())) type;
-  static const bool value = _IsNotSame<type, void>::value;
-};
-
-template <>
-struct __numeric_type<void> {
-  static const bool value = true;
-};
-
-template <class _A1,
-          class _A2 = void,
-          class _A3 = void,
-          bool      = __numeric_type<_A1>::value && __numeric_type<_A2>::value && __numeric_type<_A3>::value>
-class __promote_imp {
-public:
-  static const bool value = false;
-};
-
-template <class _A1, class _A2, class _A3>
-class __promote_imp<_A1, _A2, _A3, true> {
-private:
-  typedef typename __promote_imp<_A1>::type __type1;
-  typedef typename __promote_imp<_A2>::type __type2;
-  typedef typename __promote_imp<_A3>::type __type3;
-
-public:
-  typedef decltype(__type1() + __type2() + __type3()) type;
-  static const bool value = true;
-};
-
-template <class _A1, class _A2>
-class __promote_imp<_A1, _A2, void, true> {
-private:
-  typedef typename __promote_imp<_A1>::type __type1;
-  typedef typename __promote_imp<_A2>::type __type2;
-
-public:
-  typedef decltype(__type1() + __type2()) type;
-  static const bool value = true;
-};
-
-template <class _A1>
-class __promote_imp<_A1, void, void, true> {
-public:
-  typedef typename __numeric_type<_A1>::type type;
-  static const bool value = true;
-};
-
-template <class _A1, class _A2 = void, class _A3 = void>
-class __promote : public __promote_imp<_A1, _A2, _A3> {};
-
-#endif // !defined(_LIBCPP_CLANG_VER) || _LIBCPP_CLANG_VER >= 1700
-
 _LIBCPP_END_NAMESPACE_STD
 
 #endif // _LIBCPP___TYPE_TRAITS_PROMOTE_H
diff --git a/libcxx/include/__type_traits/result_of.h b/libcxx/include/__type_traits/result_of.h
index 217ca70b4cd20..e6adec7f9c978 100644
--- a/libcxx/include/__type_traits/result_of.h
+++ b/libcxx/include/__type_traits/result_of.h
@@ -22,10 +22,15 @@ _LIBCPP_BEGIN_NAMESPACE_STD
 
 #if _LIBCPP_STD_VER <= 17 || defined(_LIBCPP_ENABLE_CXX20_REMOVED_TYPE_TRAITS)
 template <class _Callable>
-struct _LIBCPP_DEPRECATED_IN_CXX17 result_of;
+struct _LIBCPP_DEPRECATED_IN_CXX17 _LIBCPP_NO_SPECIALIZATIONS result_of;
 
+_LIBCPP_DIAGNOSTIC_PUSH
+#if __has_warning("-Winvalid-specialization")
+_LIBCPP_CLANG_DIAGNOSTIC_IGNORED("-Winvalid-specialization")
+#endif
 template <class _Fp, class... _Args>
 struct _LIBCPP_TEMPLATE_VIS result_of<_Fp(_Args...)> : __invoke_result<_Fp, _Args...> {};
+_LIBCPP_DIAGNOSTIC_POP
 
 #  if _LIBCPP_STD_VER >= 14
 template <class _Tp>
diff --git a/libcxx/include/__vector/vector.h b/libcxx/include/__vector/vector.h
index 66cb622e20963..244506065db61 100644
--- a/libcxx/include/__vector/vector.h
+++ b/libcxx/include/__vector/vector.h
@@ -84,9 +84,6 @@ _LIBCPP_BEGIN_NAMESPACE_STD
 
 template <class _Tp, class _Allocator /* = allocator<_Tp> */>
 class _LIBCPP_TEMPLATE_VIS vector {
-private:
-  typedef allocator<_Tp> __default_allocator_type;
-
 public:
   //
   // Types
@@ -783,14 +780,18 @@ class _LIBCPP_TEMPLATE_VIS vector {
 
   _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void __move_assign_alloc(vector&, false_type) _NOEXCEPT {}
 
-  static _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI pointer __add_alignment_assumption(pointer __p) _NOEXCEPT {
-#ifndef _LIBCPP_CXX03_LANG
-    if constexpr (is_pointer<pointer>::value) {
-      if (!__libcpp_is_constant_evaluated()) {
-        return static_cast<pointer>(__builtin_assume_aligned(__p, alignof(decltype(*__p))));
-      }
+  template <class _Ptr = pointer, __enable_if_t<is_pointer<_Ptr>::value, int> = 0>
+  static _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI _LIBCPP_NO_CFI pointer
+  __add_alignment_assumption(_Ptr __p) _NOEXCEPT {
+    if (!__libcpp_is_constant_evaluated()) {
+      return static_cast<pointer>(__builtin_assume_aligned(__p, _LIBCPP_ALIGNOF(decltype(*__p))));
     }
-#endif
+    return __p;
+  }
+
+  template <class _Ptr = pointer, __enable_if_t<!is_pointer<_Ptr>::value, int> = 0>
+  static _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI _LIBCPP_NO_CFI pointer
+  __add_alignment_assumption(_Ptr __p) _NOEXCEPT {
     return __p;
   }
 };
diff --git a/libcxx/include/__vector/vector_bool.h b/libcxx/include/__vector/vector_bool.h
index 4f1c442ce0be8..35f0f745e201c 100644
--- a/libcxx/include/__vector/vector_bool.h
+++ b/libcxx/include/__vector/vector_bool.h
@@ -10,6 +10,7 @@
 #define _LIBCPP___VECTOR_VECTOR_BOOL_H
 
 #include <__algorithm/copy.h>
+#include <__algorithm/copy_backward.h>
 #include <__algorithm/fill_n.h>
 #include <__algorithm/iterator_operations.h>
 #include <__algorithm/max.h>
@@ -532,10 +533,8 @@ template <class _Allocator>
 _LIBCPP_CONSTEXPR_SINCE_CXX20 typename vector<bool, _Allocator>::size_type
 vector<bool, _Allocator>::max_size() const _NOEXCEPT {
   size_type __amax = __storage_traits::max_size(__alloc_);
-  size_type __nmax = numeric_limits<size_type>::max() / 2; // end() >= begin(), always
-  if (__nmax / __bits_per_word <= __amax)
-    return __nmax;
-  return __internal_cap_to_external(__amax);
+  size_type __nmax = numeric_limits<difference_type>::max();
+  return __nmax / __bits_per_word <= __amax ? __nmax : __internal_cap_to_external(__amax);
 }
 
 //  Precondition:  __new_size > capacity()
@@ -552,36 +551,29 @@ vector<bool, _Allocator>::__recommend(size_type __new_size) const {
 }
 
 //  Default constructs __n objects starting at __end_
-//  Precondition:  __n > 0
 //  Precondition:  size() + __n <= capacity()
 //  Postcondition:  size() == size() + __n
 template <class _Allocator>
 inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void
 vector<bool, _Allocator>::__construct_at_end(size_type __n, bool __x) {
-  size_type __old_size = this->__size_;
+  _LIBCPP_ASSERT_INTERNAL(
+      capacity() >= size() + __n, "vector<bool>::__construct_at_end called with insufficient capacity");
+  std::fill_n(end(), __n, __x);
   this->__size_ += __n;
-  if (__old_size == 0 || ((__old_size - 1) / __bits_per_word) != ((this->__size_ - 1) / __bits_per_word)) {
-    if (this->__size_ <= __bits_per_word)
-      this->__begin_[0] = __storage_type(0);
-    else
-      this->__begin_[(this->__size_ - 1) / __bits_per_word] = __storage_type(0);
-  }
-  std::fill_n(__make_iter(__old_size), __n, __x);
+  if (end().__ctz_ != 0) // Ensure uninitialized leading bits in the last word are set to zero
+    std::fill_n(end(), __bits_per_word - end().__ctz_, 0);
 }
 
 template <class _Allocator>
 template <class _InputIterator, class _Sentinel>
 _LIBCPP_CONSTEXPR_SINCE_CXX20 void
 vector<bool, _Allocator>::__construct_at_end(_InputIterator __first, _Sentinel __last, size_type __n) {
-  size_type __old_size = this->__size_;
+  _LIBCPP_ASSERT_INTERNAL(
+      capacity() >= size() + __n, "vector<bool>::__construct_at_end called with insufficient capacity");
+  std::__copy(std::move(__first), std::move(__last), end());
   this->__size_ += __n;
-  if (__old_size == 0 || ((__old_size - 1) / __bits_per_word) != ((this->__size_ - 1) / __bits_per_word)) {
-    if (this->__size_ <= __bits_per_word)
-      this->__begin_[0] = __storage_type(0);
-    else
-      this->__begin_[(this->__size_ - 1) / __bits_per_word] = __storage_type(0);
-  }
-  std::__copy(std::move(__first), std::move(__last), __make_iter(__old_size));
+  if (end().__ctz_ != 0) // Ensure uninitialized leading bits in the last word are set to zero
+    std::fill_n(end(), __bits_per_word - end().__ctz_, 0);
 }
 
 template <class _Allocator>
diff --git a/libcxx/include/bitset b/libcxx/include/bitset
index 10576eb80bf2e..a20842985b3d5 100644
--- a/libcxx/include/bitset
+++ b/libcxx/include/bitset
@@ -129,6 +129,8 @@ template <size_t N> struct hash<std::bitset<N>>;
 #if __cplusplus < 201103L && defined(_LIBCPP_USE_FROZEN_CXX03_HEADERS)
 #  include <__cxx03/bitset>
 #else
+#  include <__algorithm/copy.h>
+#  include <__algorithm/copy_backward.h>
 #  include <__algorithm/count.h>
 #  include <__algorithm/fill.h>
 #  include <__algorithm/fill_n.h>
diff --git a/libcxx/include/chrono b/libcxx/include/chrono
index 10695eea649fb..b39f060bf6b08 100644
--- a/libcxx/include/chrono
+++ b/libcxx/include/chrono
@@ -335,6 +335,34 @@ struct leap_second_info {                               // C++20
 template<class Duration>                                // C++20
   leap_second_info get_leap_second_info(const utc_time<Duration>& ut);
 
+
+// [time.clock.tai], class tai_clock
+class tai_clock {                                      // C++20
+public:
+    using rep                       = a signed arithmetic type;
+    using period                    = ratio<unspecified, unspecified>;
+    using duration                  = chrono::duration<rep, period>;
+    using time_point                = chrono::time_point<tai_clock>;
+    static constexpr bool is_steady = unspecified;
+
+    static time_point now();
+
+    template<class Duration>
+      static utc_time<common_type_t<Duration, seconds>>
+        to_utc(const tai_time<Duration>& t);
+    template<class Duration>
+      static tai_time<common_type_t<Duration, seconds>>
+        from_utc(const utc_time<Duration>& t);
+};
+
+template<class Duration>
+using tai_time  = time_point<tai_clock, Duration>;      // C++20
+using tai_seconds = tai_time<seconds>;                  // C++20
+
+template<class charT, class traits, class Duration>     // C++20
+  basic_ostream<charT, traits>&
+    operator<<(basic_ostream<charT, traits>& os, const tai_time<Duration>& t);
+
 class file_clock                                        // C++20
 {
 public:
@@ -894,16 +922,18 @@ strong_ordering operator<=>(const time_zone_link& x, const time_zone_link& y);
 }  // chrono
 
 namespace std {
+  template<class Rep, class Period, class charT>
+    struct formatter<chrono::duration<Rep, Period>, charT>;                       // C++20
   template<class Duration, class charT>
     struct formatter<chrono::sys_time<Duration>, charT>;                          // C++20
   template<class Duration, class charT>
     struct formatter<chrono::utc_time<Duration>, charT>;                          // C++20
   template<class Duration, class charT>
-    struct formatter<chrono::filetime<Duration>, charT>;                          // C++20
+    struct formatter<chrono::tai_time<Duration>, charT>;                          // C++20
+  template<class Duration, class charT>
+    struct formatter<chrono::file_time<Duration>, charT>;                         // C++20
   template<class Duration, class charT>
     struct formatter<chrono::local_time<Duration>, charT>;                        // C++20
-  template<class Rep, class Period, class charT>
-    struct formatter<chrono::duration<Rep, Period>, charT>;                       // C++20
   template<class charT> struct formatter<chrono::day, charT>;                     // C++20
   template<class charT> struct formatter<chrono::month, charT>;                   // C++20
   template<class charT> struct formatter<chrono::year, charT>;                    // C++20
@@ -1014,6 +1044,7 @@ constexpr chrono::year                                  operator ""y(unsigned lo
 
 #    if _LIBCPP_HAS_TIME_ZONE_DATABASE && _LIBCPP_HAS_FILESYSTEM && _LIBCPP_HAS_LOCALIZATION
 #      include <__chrono/leap_second.h>
+#      include <__chrono/tai_clock.h>
 #      include <__chrono/time_zone.h>
 #      include <__chrono/time_zone_link.h>
 #      include <__chrono/tzdb.h>
diff --git a/libcxx/include/deque b/libcxx/include/deque
index df3094cff7f89..95200b4801d7f 100644
--- a/libcxx/include/deque
+++ b/libcxx/include/deque
@@ -59,9 +59,9 @@ public:
 
     deque& operator=(const deque& c);
     deque& operator=(deque&& c)
-        noexcept(
-             allocator_type::propagate_on_container_move_assignment::value &&
-             is_nothrow_move_assignable<allocator_type>::value);
+        noexcept((__alloc_traits::propagate_on_container_move_assignment::value &&
+                  is_nothrow_move_assignable<allocator_type>::value) ||
+                 allocator_traits<allocator_type>::is_always_equal::value);
     deque& operator=(initializer_list<value_type> il);
 
     template <class InputIterator>
@@ -674,9 +674,10 @@ public:
 
   _LIBCPP_HIDE_FROM_ABI deque(deque&& __c) noexcept(is_nothrow_move_constructible<allocator_type>::value);
   _LIBCPP_HIDE_FROM_ABI deque(deque&& __c, const __type_identity_t<allocator_type>& __a);
-  _LIBCPP_HIDE_FROM_ABI deque&
-  operator=(deque&& __c) noexcept(__alloc_traits::propagate_on_container_move_assignment::value &&
-                                  is_nothrow_move_assignable<allocator_type>::value);
+  _LIBCPP_HIDE_FROM_ABI deque& operator=(deque&& __c) noexcept(
+      (__alloc_traits::propagate_on_container_move_assignment::value &&
+       is_nothrow_move_assignable<allocator_type>::value) ||
+      allocator_traits<allocator_type>::is_always_equal::value);
 
   _LIBCPP_HIDE_FROM_ABI void assign(initializer_list<value_type> __il) { assign(__il.begin(), __il.end()); }
 #  endif // _LIBCPP_CXX03_LANG
@@ -1379,8 +1380,9 @@ inline deque<_Tp, _Allocator>::deque(deque&& __c, const __type_identity_t<alloca
 
 template <class _Tp, class _Allocator>
 inline deque<_Tp, _Allocator>& deque<_Tp, _Allocator>::operator=(deque&& __c) noexcept(
-    __alloc_traits::propagate_on_container_move_assignment::value &&
-    is_nothrow_move_assignable<allocator_type>::value) {
+    (__alloc_traits::propagate_on_container_move_assignment::value &&
+     is_nothrow_move_assignable<allocator_type>::value) ||
+    allocator_traits<allocator_type>::is_always_equal::value) {
   __move_assign(__c, integral_constant<bool, __alloc_traits::propagate_on_container_move_assignment::value>());
   return *this;
 }
diff --git a/libcxx/include/flat_map b/libcxx/include/flat_map
index dbe5d8ee8f8c3..2552450081734 100644
--- a/libcxx/include/flat_map
+++ b/libcxx/include/flat_map
@@ -35,6 +35,25 @@ namespace std {
            class Predicate>
     typename flat_map<Key, T, Compare, KeyContainer, MappedContainer>::size_type
       erase_if(flat_map<Key, T, Compare, KeyContainer, MappedContainer>& c, Predicate pred);
+
+  // [flat.multimap], class template flat_multimap
+  template<class Key, class T, class Compare = less<Key>,
+           class KeyContainer = vector<Key>, class MappedContainer = vector<T>>
+    class flat_multimap;
+
+  struct sorted_equivalent_t { explicit sorted_equivalent_t() = default; };
+  inline constexpr sorted_equivalent_t sorted_equivalent{};
+
+  template<class Key, class T, class Compare, class KeyContainer, class MappedContainer,
+           class Allocator>
+    struct uses_allocator<flat_multimap<Key, T, Compare, KeyContainer, MappedContainer>,
+                          Allocator>;
+
+  // [flat.multimap.erasure], erasure for flat_multimap
+  template<class Key, class T, class Compare, class KeyContainer, class MappedContainer,
+           class Predicate>
+    typename flat_multimap<Key, T, Compare, KeyContainer, MappedContainer>::size_type
+      erase_if(flat_multimap<Key, T, Compare, KeyContainer, MappedContainer>& c, Predicate pred);
 */
 
 #if __cplusplus < 201103L && defined(_LIBCPP_USE_FROZEN_CXX03_HEADERS)
@@ -44,6 +63,8 @@ namespace std {
 
 #  if _LIBCPP_STD_VER >= 23
 #    include <__flat_map/flat_map.h>
+#    include <__flat_map/flat_multimap.h>
+#    include <__flat_map/sorted_equivalent.h>
 #    include <__flat_map/sorted_unique.h>
 #  endif
 
diff --git a/libcxx/include/forward_list b/libcxx/include/forward_list
index f3b9617ab2e04..4b6ca8ea8587c 100644
--- a/libcxx/include/forward_list
+++ b/libcxx/include/forward_list
@@ -58,9 +58,9 @@ public:
 
     forward_list& operator=(const forward_list& x);
     forward_list& operator=(forward_list&& x)
-        noexcept(
-             allocator_type::propagate_on_container_move_assignment::value &&
-             is_nothrow_move_assignable<allocator_type>::value);
+        noexcept((__node_traits::propagate_on_container_move_assignment::value &&
+                  is_nothrow_move_assignable<allocator_type>::value) ||
+                 allocator_traits<allocator_type>::is_always_equal::value);
     forward_list& operator=(initializer_list<value_type> il);
 
     template <class InputIterator>
@@ -717,8 +717,9 @@ public:
   _LIBCPP_HIDE_FROM_ABI forward_list(initializer_list<value_type> __il, const allocator_type& __a);
 
   _LIBCPP_HIDE_FROM_ABI forward_list& operator=(forward_list&& __x) noexcept(
-      __node_traits::propagate_on_container_move_assignment::value &&
-      is_nothrow_move_assignable<allocator_type>::value);
+      (__node_traits::propagate_on_container_move_assignment::value &&
+       is_nothrow_move_assignable<allocator_type>::value) ||
+      allocator_traits<allocator_type>::is_always_equal::value);
 
   _LIBCPP_HIDE_FROM_ABI forward_list& operator=(initializer_list<value_type> __il);
 
@@ -1009,8 +1010,10 @@ void forward_list<_Tp, _Alloc>::__move_assign(forward_list& __x, false_type) {
 }
 
 template <class _Tp, class _Alloc>
-inline forward_list<_Tp, _Alloc>& forward_list<_Tp, _Alloc>::operator=(forward_list&& __x) _NOEXCEPT_(
-    __node_traits::propagate_on_container_move_assignment::value&& is_nothrow_move_assignable<allocator_type>::value) {
+inline forward_list<_Tp, _Alloc>& forward_list<_Tp, _Alloc>::operator=(forward_list&& __x) noexcept(
+    (__node_traits::propagate_on_container_move_assignment::value &&
+     is_nothrow_move_assignable<allocator_type>::value) ||
+    allocator_traits<allocator_type>::is_always_equal::value) {
   __move_assign(__x, integral_constant<bool, __node_traits::propagate_on_container_move_assignment::value>());
   return *this;
 }
diff --git a/libcxx/include/fstream b/libcxx/include/fstream
index f0e9425e0a53d..de5c07035dba9 100644
--- a/libcxx/include/fstream
+++ b/libcxx/include/fstream
@@ -216,12 +216,6 @@ typedef basic_fstream<wchar_t> wfstream;
 _LIBCPP_PUSH_MACROS
 #  include <__undef_macros>
 
-#  if !defined(_LIBCPP_MSVCRT) && !defined(_NEWLIB_VERSION)
-#    define _LIBCPP_HAS_OFF_T_FUNCTIONS 1
-#  else
-#    define _LIBCPP_HAS_OFF_T_FUNCTIONS 0
-#  endif
-
 #  if _LIBCPP_HAS_FILESYSTEM && _LIBCPP_HAS_LOCALIZATION
 
 _LIBCPP_BEGIN_NAMESPACE_STD
@@ -362,6 +356,9 @@ private:
   bool __read_mode();
   void __write_mode();
 
+  _LIBCPP_HIDE_FROM_ABI static int __fseek(FILE* __file, pos_type __offset, int __whence);
+  _LIBCPP_HIDE_FROM_ABI static pos_type __ftell(FILE* __file);
+
   _LIBCPP_EXPORTED_FROM_ABI friend FILE* __get_ostream_file(ostream&);
 
   // There are multiple (__)open function, they use different C-API open
@@ -936,31 +933,42 @@ basic_filebuf<_CharT, _Traits>::seekoff(off_type __off, ios_base::seekdir __way,
   default:
     return pos_type(off_type(-1));
   }
-#    if !_LIBCPP_HAS_OFF_T_FUNCTIONS
-  if (fseek(__file_, __width > 0 ? __width * __off : 0, __whence))
+  if (__fseek(__file_, __width > 0 ? __width * __off : 0, __whence))
     return pos_type(off_type(-1));
-  pos_type __r = ftell(__file_);
-#    else
-  if (::fseeko(__file_, __width > 0 ? __width * __off : 0, __whence))
-    return pos_type(off_type(-1));
-  pos_type __r = ftello(__file_);
-#    endif
+  pos_type __r = __ftell(__file_);
   __r.state(__st_);
   return __r;
 }
 
+template <class _CharT, class _Traits>
+int basic_filebuf<_CharT, _Traits>::__fseek(FILE* __file, pos_type __offset, int __whence) {
+#    if defined(_LIBCPP_MSVCRT_LIKE)
+  return _fseeki64(__file, __offset, __whence);
+#    elif defined(_NEWLIB_VERSION)
+  return fseek(__file, __offset, __whence);
+#    else
+  return ::fseeko(__file, __offset, __whence);
+#    endif
+}
+
+template <class _CharT, class _Traits>
+typename basic_filebuf<_CharT, _Traits>::pos_type basic_filebuf<_CharT, _Traits>::__ftell(FILE* __file) {
+#    if defined(_LIBCPP_MSVCRT_LIKE)
+  return _ftelli64(__file);
+#    elif defined(_NEWLIB_VERSION)
+  return ftell(__file);
+#    else
+  return ftello(__file);
+#    endif
+}
+
 template <class _CharT, class _Traits>
 typename basic_filebuf<_CharT, _Traits>::pos_type
 basic_filebuf<_CharT, _Traits>::seekpos(pos_type __sp, ios_base::openmode) {
   if (__file_ == nullptr || sync())
     return pos_type(off_type(-1));
-#    if !_LIBCPP_HAS_OFF_T_FUNCTIONS
-  if (fseek(__file_, __sp, SEEK_SET))
+  if (__fseek(__file_, __sp, SEEK_SET))
     return pos_type(off_type(-1));
-#    else
-  if (::fseeko(__file_, __sp, SEEK_SET))
-    return pos_type(off_type(-1));
-#    endif
   __st_ = __sp.state();
   return __sp;
 }
@@ -1007,13 +1015,8 @@ int basic_filebuf<_CharT, _Traits>::sync() {
         }
       }
     }
-#    if !_LIBCPP_HAS_OFF_T_FUNCTIONS
-    if (fseek(__file_, -__c, SEEK_CUR))
+    if (__fseek(__file_, -__c, SEEK_CUR))
       return -1;
-#    else
-    if (::fseeko(__file_, -__c, SEEK_CUR))
-      return -1;
-#    endif
     if (__update_st)
       __st_ = __state;
     __extbufnext_ = __extbufend_ = __extbuf_;
diff --git a/libcxx/include/list b/libcxx/include/list
index 5e2fd40d6ee9d..3fcf796ebc03d 100644
--- a/libcxx/include/list
+++ b/libcxx/include/list
@@ -60,9 +60,9 @@ public:
 
     list& operator=(const list& x);
     list& operator=(list&& x)
-        noexcept(
-             allocator_type::propagate_on_container_move_assignment::value &&
-             is_nothrow_move_assignable<allocator_type>::value);
+        noexcept((__node_alloc_traits::propagate_on_container_move_assignment::value &&
+                  is_nothrow_move_assignable<__node_allocator>::value) ||
+                 allocator_traits<allocator_type>::is_always_equal::value);
     list& operator=(initializer_list<value_type>);
     template <class Iter>
         void assign(Iter first, Iter last);
@@ -728,9 +728,10 @@ public:
 
   _LIBCPP_HIDE_FROM_ABI list(list&& __c) _NOEXCEPT_(is_nothrow_move_constructible<__node_allocator>::value);
   _LIBCPP_HIDE_FROM_ABI list(list&& __c, const __type_identity_t<allocator_type>& __a);
-  _LIBCPP_HIDE_FROM_ABI list& operator=(list&& __c)
-      _NOEXCEPT_(__node_alloc_traits::propagate_on_container_move_assignment::value&&
-                     is_nothrow_move_assignable<__node_allocator>::value);
+  _LIBCPP_HIDE_FROM_ABI list& operator=(list&& __c) noexcept(
+      (__node_alloc_traits::propagate_on_container_move_assignment::value &&
+       is_nothrow_move_assignable<__node_allocator>::value) ||
+      allocator_traits<allocator_type>::is_always_equal::value);
 
   _LIBCPP_HIDE_FROM_ABI list& operator=(initializer_list<value_type> __il) {
     assign(__il.begin(), __il.end());
@@ -1067,8 +1068,9 @@ inline list<_Tp, _Alloc>::list(list&& __c, const __type_identity_t<allocator_typ
 
 template <class _Tp, class _Alloc>
 inline list<_Tp, _Alloc>& list<_Tp, _Alloc>::operator=(list&& __c) noexcept(
-    __node_alloc_traits::propagate_on_container_move_assignment::value &&
-    is_nothrow_move_assignable<__node_allocator>::value) {
+    (__node_alloc_traits::propagate_on_container_move_assignment::value &&
+     is_nothrow_move_assignable<__node_allocator>::value) ||
+    allocator_traits<allocator_type>::is_always_equal::value) {
   __move_assign(__c, integral_constant<bool, __node_alloc_traits::propagate_on_container_move_assignment::value>());
   return *this;
 }
diff --git a/libcxx/include/locale b/libcxx/include/locale
index 5af674d19414b..be0f31cece671 100644
--- a/libcxx/include/locale
+++ b/libcxx/include/locale
@@ -400,8 +400,9 @@ struct __num_get : protected __num_get_base {
       unsigned*& __g_end,
       unsigned& __dc,
       _CharT* __atoms);
-#    ifndef _LIBCPP_ABI_OPTIMIZED_LOCALE_NUM_GET
-  static string __stage2_int_prep(ios_base& __iob, _CharT* __atoms, _CharT& __thousands_sep);
+
+  [[__deprecated__("This exists only for ABI compatibility")]] static string
+  __stage2_int_prep(ios_base& __iob, _CharT* __atoms, _CharT& __thousands_sep);
   static int __stage2_int_loop(
       _CharT __ct,
       int __base,
@@ -414,55 +415,32 @@ struct __num_get : protected __num_get_base {
       unsigned*& __g_end,
       _CharT* __atoms);
 
-#    else
-  static string __stage2_int_prep(ios_base& __iob, _CharT& __thousands_sep) {
+  _LIBCPP_HIDE_FROM_ABI static string __stage2_int_prep(ios_base& __iob, _CharT& __thousands_sep) {
     locale __loc                 = __iob.getloc();
     const numpunct<_CharT>& __np = use_facet<numpunct<_CharT> >(__loc);
     __thousands_sep              = __np.thousands_sep();
     return __np.grouping();
   }
 
-  const _CharT* __do_widen(ios_base& __iob, _CharT* __atoms) const { return __do_widen_p(__iob, __atoms); }
-
-  static int __stage2_int_loop(
-      _CharT __ct,
-      int __base,
-      char* __a,
-      char*& __a_end,
-      unsigned& __dc,
-      _CharT __thousands_sep,
-      const string& __grouping,
-      unsigned* __g,
-      unsigned*& __g_end,
-      const _CharT* __atoms);
+  _LIBCPP_HIDE_FROM_ABI const _CharT* __do_widen(ios_base& __iob, _CharT* __atoms) const {
+    return __do_widen_p(__iob, __atoms);
+  }
 
 private:
   template <typename _Tp>
-  const _Tp* __do_widen_p(ios_base& __iob, _Tp* __atoms) const {
+  _LIBCPP_HIDE_FROM_ABI const _Tp* __do_widen_p(ios_base& __iob, _Tp* __atoms) const {
     locale __loc = __iob.getloc();
     use_facet<ctype<_Tp> >(__loc).widen(__src, __src + __int_chr_cnt, __atoms);
     return __atoms;
   }
 
-  const char* __do_widen_p(ios_base& __iob, char* __atoms) const {
+  _LIBCPP_HIDE_FROM_ABI const char* __do_widen_p(ios_base& __iob, char* __atoms) const {
     (void)__iob;
     (void)__atoms;
     return __src;
   }
-#    endif
 };
 
-#    ifndef _LIBCPP_ABI_OPTIMIZED_LOCALE_NUM_GET
-template <class _CharT>
-string __num_get<_CharT>::__stage2_int_prep(ios_base& __iob, _CharT* __atoms, _CharT& __thousands_sep) {
-  locale __loc = __iob.getloc();
-  std::use_facet<ctype<_CharT> >(__loc).widen(__src, __src + __int_chr_cnt, __atoms);
-  const numpunct<_CharT>& __np = std::use_facet<numpunct<_CharT> >(__loc);
-  __thousands_sep              = __np.thousands_sep();
-  return __np.grouping();
-}
-#    endif
-
 template <class _CharT>
 string __num_get<_CharT>::__stage2_float_prep(
     ios_base& __iob, _CharT* __atoms, _CharT& __decimal_point, _CharT& __thousands_sep) {
@@ -475,18 +453,17 @@ string __num_get<_CharT>::__stage2_float_prep(
 }
 
 template <class _CharT>
-int
-#    ifndef _LIBCPP_ABI_OPTIMIZED_LOCALE_NUM_GET
-__num_get<_CharT>::__stage2_int_loop(_CharT __ct, int __base, char* __a, char*& __a_end,
-                  unsigned& __dc, _CharT __thousands_sep, const string& __grouping,
-                  unsigned* __g, unsigned*& __g_end, _CharT* __atoms)
-#    else
-__num_get<_CharT>::__stage2_int_loop(_CharT __ct, int __base, char* __a, char*& __a_end,
-                  unsigned& __dc, _CharT __thousands_sep, const string& __grouping,
-                  unsigned* __g, unsigned*& __g_end, const _CharT* __atoms)
-
-#    endif
-{
+int __num_get<_CharT>::__stage2_int_loop(
+    _CharT __ct,
+    int __base,
+    char* __a,
+    char*& __a_end,
+    unsigned& __dc,
+    _CharT __thousands_sep,
+    const string& __grouping,
+    unsigned* __g,
+    unsigned*& __g_end,
+    _CharT* __atoms) {
   if (__a_end == __a && (__ct == __atoms[24] || __ct == __atoms[25])) {
     *__a_end++ = __ct == __atoms[24] ? '+' : '-';
     __dc       = 0;
@@ -856,14 +833,9 @@ _InputIterator num_get<_CharT, _InputIterator>::__do_get_signed(
   // Stage 2
   char_type __thousands_sep;
   const int __atoms_size = __num_get_base::__int_chr_cnt;
-#    ifdef _LIBCPP_ABI_OPTIMIZED_LOCALE_NUM_GET
   char_type __atoms1[__atoms_size];
   const char_type* __atoms = this->__do_widen(__iob, __atoms1);
   string __grouping        = this->__stage2_int_prep(__iob, __thousands_sep);
-#    else
-  char_type __atoms[__atoms_size];
-  string __grouping = this->__stage2_int_prep(__iob, __atoms, __thousands_sep);
-#    endif
   string __buf;
   __buf.resize(__buf.capacity());
   char* __a     = &__buf[0];
@@ -879,7 +851,17 @@ _InputIterator num_get<_CharT, _InputIterator>::__do_get_signed(
       __a     = &__buf[0];
       __a_end = __a + __tmp;
     }
-    if (this->__stage2_int_loop(*__b, __base, __a, __a_end, __dc, __thousands_sep, __grouping, __g, __g_end, __atoms))
+    if (this->__stage2_int_loop(
+            *__b,
+            __base,
+            __a,
+            __a_end,
+            __dc,
+            __thousands_sep,
+            __grouping,
+            __g,
+            __g_end,
+            const_cast<char_type*>(__atoms)))
       break;
   }
   if (__grouping.size() != 0 && __g_end - __g < __num_get_base::__num_get_buf_sz)
@@ -905,14 +887,9 @@ _InputIterator num_get<_CharT, _InputIterator>::__do_get_unsigned(
   // Stage 2
   char_type __thousands_sep;
   const int __atoms_size = __num_get_base::__int_chr_cnt;
-#    ifdef _LIBCPP_ABI_OPTIMIZED_LOCALE_NUM_GET
   char_type __atoms1[__atoms_size];
   const char_type* __atoms = this->__do_widen(__iob, __atoms1);
   string __grouping        = this->__stage2_int_prep(__iob, __thousands_sep);
-#    else
-  char_type __atoms[__atoms_size];
-  string __grouping = this->__stage2_int_prep(__iob, __atoms, __thousands_sep);
-#    endif
   string __buf;
   __buf.resize(__buf.capacity());
   char* __a     = &__buf[0];
@@ -928,7 +905,17 @@ _InputIterator num_get<_CharT, _InputIterator>::__do_get_unsigned(
       __a     = &__buf[0];
       __a_end = __a + __tmp;
     }
-    if (this->__stage2_int_loop(*__b, __base, __a, __a_end, __dc, __thousands_sep, __grouping, __g, __g_end, __atoms))
+    if (this->__stage2_int_loop(
+            *__b,
+            __base,
+            __a,
+            __a_end,
+            __dc,
+            __thousands_sep,
+            __grouping,
+            __g,
+            __g_end,
+            const_cast<char_type*>(__atoms)))
       break;
   }
   if (__grouping.size() != 0 && __g_end - __g < __num_get_base::__num_get_buf_sz)
diff --git a/libcxx/include/memory b/libcxx/include/memory
index fc62606ea0fd3..6661bc4f43756 100644
--- a/libcxx/include/memory
+++ b/libcxx/include/memory
@@ -958,12 +958,14 @@ template<class Pointer = void, class Smart, class... Args>
 
 #  if _LIBCPP_STD_VER >= 17
 #    include <__memory/construct_at.h>
+#    include <__memory/destroy.h>
 #  endif
 
 #  if _LIBCPP_STD_VER >= 20
 #    include <__memory/assume_aligned.h>
 #    include <__memory/concepts.h>
 #    include <__memory/ranges_construct_at.h>
+#    include <__memory/ranges_destroy.h>
 #    include <__memory/ranges_uninitialized_algorithms.h>
 #    include <__memory/uses_allocator_construction.h>
 #  endif
diff --git a/libcxx/include/module.modulemap b/libcxx/include/module.modulemap
index 85b88ca137f85..b0720703bd0de 100644
--- a/libcxx/include/module.modulemap
+++ b/libcxx/include/module.modulemap
@@ -967,6 +967,10 @@ module std [system] {
       header "__chrono/system_clock.h"
       export std.chrono.time_point
     }
+    module tai_clock {
+      header "__chrono/tai_clock.h"
+      export std.chrono.time_point
+    }
     module time_point                 { header "__chrono/time_point.h" }
     module time_zone_link             { header "__chrono/time_zone_link.h" }
     module time_zone                  { header "__chrono/time_zone.h" }
@@ -1244,16 +1248,30 @@ module std [system] {
   }
 
   module flat_map {
-    module flat_map                       { header "__flat_map/flat_map.h" }
+    module flat_map                       {
+      header "__flat_map/flat_map.h"
+      export std.vector.vector
+      export std.vector.fwd
+    }
+    module flat_multimap                  {
+      header "__flat_map/flat_multimap.h"
+      export std.vector.vector
+      export std.vector.fwd
+    }
     module key_value_iterator             { header "__flat_map/key_value_iterator.h" }
+    module sorted_equivalent              { header "__flat_map/sorted_equivalent.h" }
     module sorted_unique                  { header "__flat_map/sorted_unique.h" }
+    module utils                          { header "__flat_map/utils.h" }
 
     header "flat_map"
     export *
   }
 
   module format {
-    module buffer                             { header "__format/buffer.h" }
+    module buffer                             {
+      header "__format/buffer.h"
+      export std.iterator.back_insert_iterator
+    }
     module concepts                           { header "__format/concepts.h" }
     module container_adaptor                  { header "__format/container_adaptor.h" }
     module enable_insertable                  { header "__format/enable_insertable.h" }
@@ -1549,6 +1567,7 @@ module std [system] {
     module compressed_pair                    { header "__memory/compressed_pair.h" }
     module concepts                           { header "__memory/concepts.h" }
     module construct_at                       { header "__memory/construct_at.h" }
+    module destroy                            { header "__memory/destroy.h" }
     module destruct_n                         { header "__memory/destruct_n.h" }
     module fwd                                { header "__fwd/memory.h" }
     module inout_ptr                          { header "__memory/inout_ptr.h" }
@@ -1556,6 +1575,7 @@ module std [system] {
     module out_ptr                            { header "__memory/out_ptr.h" }
     module pointer_traits                     { header "__memory/pointer_traits.h" }
     module ranges_construct_at                { header "__memory/ranges_construct_at.h" }
+    module ranges_destroy                     { header "__memory/ranges_destroy.h" }
     module ranges_uninitialized_algorithms {
       header "__memory/ranges_uninitialized_algorithms.h"
       export std.algorithm.in_out_result
diff --git a/libcxx/include/string b/libcxx/include/string
index fdd8085106dcc..b280f5f458459 100644
--- a/libcxx/include/string
+++ b/libcxx/include/string
@@ -763,22 +763,19 @@ struct __padding<0> {};
 
 template <class _CharT, class _Traits, class _Allocator>
 class basic_string {
-private:
-  using __default_allocator_type _LIBCPP_NODEBUG = allocator<_CharT>;
-
 public:
-  typedef basic_string __self;
-  typedef basic_string_view<_CharT, _Traits> __self_view;
-  typedef _Traits traits_type;
-  typedef _CharT value_type;
-  typedef _Allocator allocator_type;
-  typedef allocator_traits<allocator_type> __alloc_traits;
-  typedef typename __alloc_traits::size_type size_type;
-  typedef typename __alloc_traits::difference_type difference_type;
-  typedef value_type& reference;
-  typedef const value_type& const_reference;
-  typedef typename __alloc_traits::pointer pointer;
-  typedef typename __alloc_traits::const_pointer const_pointer;
+  using __self _LIBCPP_NODEBUG         = basic_string;
+  using __self_view _LIBCPP_NODEBUG    = basic_string_view<_CharT, _Traits>;
+  using traits_type                    = _Traits;
+  using value_type                     = _CharT;
+  using allocator_type                 = _Allocator;
+  using __alloc_traits _LIBCPP_NODEBUG = allocator_traits<allocator_type>;
+  using size_type                      = typename __alloc_traits::size_type;
+  using difference_type                = typename __alloc_traits::difference_type;
+  using reference                      = value_type&;
+  using const_reference                = const value_type&;
+  using pointer                        = typename __alloc_traits::pointer;
+  using const_pointer                  = typename __alloc_traits::const_pointer;
 
   // A basic_string contains the following members which may be trivially relocatable:
   // - pointer: is currently assumed to be trivially relocatable, but is still checked in case that changes
@@ -841,14 +838,14 @@ public:
   // Users might provide custom allocators, and prior to C++20 we have no existing way to detect whether the allocator's
   // pointer type is contiguous (though it has to be by the Standard). Using the wrapper type ensures the iterator is
   // considered contiguous.
-  typedef __bounded_iter<__wrap_iter<pointer> > iterator;
-  typedef __bounded_iter<__wrap_iter<const_pointer> > const_iterator;
+  using iterator       = __bounded_iter<__wrap_iter<pointer> >;
+  using const_iterator = __bounded_iter<__wrap_iter<const_pointer> >;
 #  else
-  typedef __wrap_iter<pointer> iterator;
-  typedef __wrap_iter<const_pointer> const_iterator;
+  using iterator       = __wrap_iter<pointer>;
+  using const_iterator = __wrap_iter<const_pointer>;
 #  endif
-  typedef std::reverse_iterator<iterator> reverse_iterator;
-  typedef std::reverse_iterator<const_iterator> const_reverse_iterator;
+  using reverse_iterator       = std::reverse_iterator<iterator>;
+  using const_reverse_iterator = std::reverse_iterator<const_iterator>;
 
 private:
   static_assert(CHAR_BIT == 8, "This implementation assumes that one byte contains 8 bits");
@@ -1716,6 +1713,9 @@ public:
       _NOEXCEPT_(!__alloc_traits::propagate_on_container_swap::value || __is_nothrow_swappable_v<allocator_type>);
 #  endif
 
+  // [string.ops]
+  // ------------
+
   _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 const value_type* c_str() const _NOEXCEPT { return data(); }
   _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 const value_type* data() const _NOEXCEPT {
     return std::__to_address(__get_pointer());
@@ -1730,87 +1730,208 @@ public:
     return __alloc_;
   }
 
+  // find
+
   _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 size_type
-  find(const basic_string& __str, size_type __pos = 0) const _NOEXCEPT;
+  find(const basic_string& __str, size_type __pos = 0) const _NOEXCEPT {
+    return std::__str_find<value_type, size_type, traits_type, npos>(data(), size(), __str.data(), __pos, __str.size());
+  }
 
   template <class _Tp, __enable_if_t<__can_be_converted_to_string_view<_CharT, _Traits, _Tp>::value, int> = 0>
   _LIBCPP_METHOD_TEMPLATE_IMPLICIT_INSTANTIATION_VIS _LIBCPP_CONSTEXPR_SINCE_CXX20 size_type
-  find(const _Tp& __t, size_type __pos = 0) const _NOEXCEPT;
+  find(const _Tp& __t, size_type __pos = 0) const _NOEXCEPT {
+    __self_view __sv = __t;
+    return std::__str_find<value_type, size_type, traits_type, npos>(data(), size(), __sv.data(), __pos, __sv.size());
+  }
+
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 size_type find(const value_type* __s, size_type __pos, size_type __n) const _NOEXCEPT {
+    _LIBCPP_ASSERT_NON_NULL(__n == 0 || __s != nullptr, "string::find(): received nullptr");
+    return std::__str_find<value_type, size_type, traits_type, npos>(data(), size(), __s, __pos, __n);
+  }
 
-  _LIBCPP_CONSTEXPR_SINCE_CXX20 size_type find(const value_type* __s, size_type __pos, size_type __n) const _NOEXCEPT;
   _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 size_type
-  find(const value_type* __s, size_type __pos = 0) const _NOEXCEPT;
-  _LIBCPP_CONSTEXPR_SINCE_CXX20 size_type find(value_type __c, size_type __pos = 0) const _NOEXCEPT;
+  find(const value_type* __s, size_type __pos = 0) const _NOEXCEPT {
+    _LIBCPP_ASSERT_NON_NULL(__s != nullptr, "string::find(): received nullptr");
+    return std::__str_find<value_type, size_type, traits_type, npos>(
+        data(), size(), __s, __pos, traits_type::length(__s));
+  }
+
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 size_type find(value_type __c, size_type __pos = 0) const _NOEXCEPT {
+    return std::__str_find<value_type, size_type, traits_type, npos>(data(), size(), __c, __pos);
+  }
+
+  // rfind
 
   _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 size_type
-  rfind(const basic_string& __str, size_type __pos = npos) const _NOEXCEPT;
+  rfind(const basic_string& __str, size_type __pos = npos) const _NOEXCEPT {
+    return std::__str_rfind<value_type, size_type, traits_type, npos>(
+        data(), size(), __str.data(), __pos, __str.size());
+  }
 
   template <class _Tp, __enable_if_t<__can_be_converted_to_string_view<_CharT, _Traits, _Tp>::value, int> = 0>
   _LIBCPP_METHOD_TEMPLATE_IMPLICIT_INSTANTIATION_VIS _LIBCPP_CONSTEXPR_SINCE_CXX20 size_type
-  rfind(const _Tp& __t, size_type __pos = npos) const _NOEXCEPT;
+  rfind(const _Tp& __t, size_type __pos = npos) const _NOEXCEPT {
+    __self_view __sv = __t;
+    return std::__str_rfind<value_type, size_type, traits_type, npos>(data(), size(), __sv.data(), __pos, __sv.size());
+  }
+
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 size_type rfind(const value_type* __s, size_type __pos, size_type __n) const _NOEXCEPT {
+    _LIBCPP_ASSERT_NON_NULL(__n == 0 || __s != nullptr, "string::rfind(): received nullptr");
+    return std::__str_rfind<value_type, size_type, traits_type, npos>(data(), size(), __s, __pos, __n);
+  }
 
-  _LIBCPP_CONSTEXPR_SINCE_CXX20 size_type rfind(const value_type* __s, size_type __pos, size_type __n) const _NOEXCEPT;
   _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 size_type
-  rfind(const value_type* __s, size_type __pos = npos) const _NOEXCEPT;
-  _LIBCPP_CONSTEXPR_SINCE_CXX20 size_type rfind(value_type __c, size_type __pos = npos) const _NOEXCEPT;
+  rfind(const value_type* __s, size_type __pos = npos) const _NOEXCEPT {
+    _LIBCPP_ASSERT_NON_NULL(__s != nullptr, "string::rfind(): received nullptr");
+    return std::__str_rfind<value_type, size_type, traits_type, npos>(
+        data(), size(), __s, __pos, traits_type::length(__s));
+  }
+
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 size_type rfind(value_type __c, size_type __pos = npos) const _NOEXCEPT {
+    return std::__str_rfind<value_type, size_type, traits_type, npos>(data(), size(), __c, __pos);
+  }
+
+  // find_first_of
 
   _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 size_type
-  find_first_of(const basic_string& __str, size_type __pos = 0) const _NOEXCEPT;
+  find_first_of(const basic_string& __str, size_type __pos = 0) const _NOEXCEPT {
+    return std::__str_find_first_of<value_type, size_type, traits_type, npos>(
+        data(), size(), __str.data(), __pos, __str.size());
+  }
 
   template <class _Tp, __enable_if_t<__can_be_converted_to_string_view<_CharT, _Traits, _Tp>::value, int> = 0>
   _LIBCPP_METHOD_TEMPLATE_IMPLICIT_INSTANTIATION_VIS _LIBCPP_CONSTEXPR_SINCE_CXX20 size_type
-  find_first_of(const _Tp& __t, size_type __pos = 0) const _NOEXCEPT;
+  find_first_of(const _Tp& __t, size_type __pos = 0) const _NOEXCEPT {
+    __self_view __sv = __t;
+    return std::__str_find_first_of<value_type, size_type, traits_type, npos>(
+        data(), size(), __sv.data(), __pos, __sv.size());
+  }
 
   _LIBCPP_CONSTEXPR_SINCE_CXX20 size_type
-  find_first_of(const value_type* __s, size_type __pos, size_type __n) const _NOEXCEPT;
+  find_first_of(const value_type* __s, size_type __pos, size_type __n) const _NOEXCEPT {
+    _LIBCPP_ASSERT_NON_NULL(__n == 0 || __s != nullptr, "string::find_first_of(): received nullptr");
+    return std::__str_find_first_of<value_type, size_type, traits_type, npos>(data(), size(), __s, __pos, __n);
+  }
+
   _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 size_type
-  find_first_of(const value_type* __s, size_type __pos = 0) const _NOEXCEPT;
+  find_first_of(const value_type* __s, size_type __pos = 0) const _NOEXCEPT {
+    _LIBCPP_ASSERT_NON_NULL(__s != nullptr, "string::find_first_of(): received nullptr");
+    return std::__str_find_first_of<value_type, size_type, traits_type, npos>(
+        data(), size(), __s, __pos, traits_type::length(__s));
+  }
+
   _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 size_type
-  find_first_of(value_type __c, size_type __pos = 0) const _NOEXCEPT;
+  find_first_of(value_type __c, size_type __pos = 0) const _NOEXCEPT {
+    return find(__c, __pos);
+  }
+
+  // find_last_of
 
   _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 size_type
-  find_last_of(const basic_string& __str, size_type __pos = npos) const _NOEXCEPT;
+  find_last_of(const basic_string& __str, size_type __pos = npos) const _NOEXCEPT {
+    return std::__str_find_last_of<value_type, size_type, traits_type, npos>(
+        data(), size(), __str.data(), __pos, __str.size());
+  }
 
   template <class _Tp, __enable_if_t<__can_be_converted_to_string_view<_CharT, _Traits, _Tp>::value, int> = 0>
   _LIBCPP_METHOD_TEMPLATE_IMPLICIT_INSTANTIATION_VIS _LIBCPP_CONSTEXPR_SINCE_CXX20 size_type
-  find_last_of(const _Tp& __t, size_type __pos = npos) const _NOEXCEPT;
+  find_last_of(const _Tp& __t, size_type __pos = npos) const _NOEXCEPT {
+    __self_view __sv = __t;
+    return std::__str_find_last_of<value_type, size_type, traits_type, npos>(
+        data(), size(), __sv.data(), __pos, __sv.size());
+  }
 
   _LIBCPP_CONSTEXPR_SINCE_CXX20 size_type
-  find_last_of(const value_type* __s, size_type __pos, size_type __n) const _NOEXCEPT;
+  find_last_of(const value_type* __s, size_type __pos, size_type __n) const _NOEXCEPT {
+    _LIBCPP_ASSERT_NON_NULL(__n == 0 || __s != nullptr, "string::find_last_of(): received nullptr");
+    return std::__str_find_last_of<value_type, size_type, traits_type, npos>(data(), size(), __s, __pos, __n);
+  }
+
   _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 size_type
-  find_last_of(const value_type* __s, size_type __pos = npos) const _NOEXCEPT;
+  find_last_of(const value_type* __s, size_type __pos = npos) const _NOEXCEPT {
+    _LIBCPP_ASSERT_NON_NULL(__s != nullptr, "string::find_last_of(): received nullptr");
+    return std::__str_find_last_of<value_type, size_type, traits_type, npos>(
+        data(), size(), __s, __pos, traits_type::length(__s));
+  }
+
   _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 size_type
-  find_last_of(value_type __c, size_type __pos = npos) const _NOEXCEPT;
+  find_last_of(value_type __c, size_type __pos = npos) const _NOEXCEPT {
+    return rfind(__c, __pos);
+  }
+
+  // find_first_not_of
 
   _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 size_type
-  find_first_not_of(const basic_string& __str, size_type __pos = 0) const _NOEXCEPT;
+  find_first_not_of(const basic_string& __str, size_type __pos = 0) const _NOEXCEPT {
+    return std::__str_find_first_not_of<value_type, size_type, traits_type, npos>(
+        data(), size(), __str.data(), __pos, __str.size());
+  }
 
   template <class _Tp, __enable_if_t<__can_be_converted_to_string_view<_CharT, _Traits, _Tp>::value, int> = 0>
   _LIBCPP_METHOD_TEMPLATE_IMPLICIT_INSTANTIATION_VIS _LIBCPP_CONSTEXPR_SINCE_CXX20 size_type
-  find_first_not_of(const _Tp& __t, size_type __pos = 0) const _NOEXCEPT;
+  find_first_not_of(const _Tp& __t, size_type __pos = 0) const _NOEXCEPT {
+    __self_view __sv = __t;
+    return std::__str_find_first_not_of<value_type, size_type, traits_type, npos>(
+        data(), size(), __sv.data(), __pos, __sv.size());
+  }
 
   _LIBCPP_CONSTEXPR_SINCE_CXX20 size_type
-  find_first_not_of(const value_type* __s, size_type __pos, size_type __n) const _NOEXCEPT;
+  find_first_not_of(const value_type* __s, size_type __pos, size_type __n) const _NOEXCEPT {
+    _LIBCPP_ASSERT_NON_NULL(__n == 0 || __s != nullptr, "string::find_first_not_of(): received nullptr");
+    return std::__str_find_first_not_of<value_type, size_type, traits_type, npos>(data(), size(), __s, __pos, __n);
+  }
+
   _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 size_type
-  find_first_not_of(const value_type* __s, size_type __pos = 0) const _NOEXCEPT;
+  find_first_not_of(const value_type* __s, size_type __pos = 0) const _NOEXCEPT {
+    _LIBCPP_ASSERT_NON_NULL(__s != nullptr, "string::find_first_not_of(): received nullptr");
+    return std::__str_find_first_not_of<value_type, size_type, traits_type, npos>(
+        data(), size(), __s, __pos, traits_type::length(__s));
+  }
+
   _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 size_type
-  find_first_not_of(value_type __c, size_type __pos = 0) const _NOEXCEPT;
+  find_first_not_of(value_type __c, size_type __pos = 0) const _NOEXCEPT {
+    return std::__str_find_first_not_of<value_type, size_type, traits_type, npos>(data(), size(), __c, __pos);
+  }
+
+  // find_last_not_of
 
   _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 size_type
-  find_last_not_of(const basic_string& __str, size_type __pos = npos) const _NOEXCEPT;
+  find_last_not_of(const basic_string& __str, size_type __pos = npos) const _NOEXCEPT {
+    return std::__str_find_last_not_of<value_type, size_type, traits_type, npos>(
+        data(), size(), __str.data(), __pos, __str.size());
+  }
 
   template <class _Tp, __enable_if_t<__can_be_converted_to_string_view<_CharT, _Traits, _Tp>::value, int> = 0>
   _LIBCPP_METHOD_TEMPLATE_IMPLICIT_INSTANTIATION_VIS _LIBCPP_CONSTEXPR_SINCE_CXX20 size_type
-  find_last_not_of(const _Tp& __t, size_type __pos = npos) const _NOEXCEPT;
+  find_last_not_of(const _Tp& __t, size_type __pos = npos) const _NOEXCEPT {
+    __self_view __sv = __t;
+    return std::__str_find_last_not_of<value_type, size_type, traits_type, npos>(
+        data(), size(), __sv.data(), __pos, __sv.size());
+  }
 
   _LIBCPP_CONSTEXPR_SINCE_CXX20 size_type
-  find_last_not_of(const value_type* __s, size_type __pos, size_type __n) const _NOEXCEPT;
+  find_last_not_of(const value_type* __s, size_type __pos, size_type __n) const _NOEXCEPT {
+    _LIBCPP_ASSERT_NON_NULL(__n == 0 || __s != nullptr, "string::find_last_not_of(): received nullptr");
+    return std::__str_find_last_not_of<value_type, size_type, traits_type, npos>(data(), size(), __s, __pos, __n);
+  }
+
   _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 size_type
-  find_last_not_of(const value_type* __s, size_type __pos = npos) const _NOEXCEPT;
+  find_last_not_of(const value_type* __s, size_type __pos = npos) const _NOEXCEPT {
+    _LIBCPP_ASSERT_NON_NULL(__s != nullptr, "string::find_last_not_of(): received nullptr");
+    return std::__str_find_last_not_of<value_type, size_type, traits_type, npos>(
+        data(), size(), __s, __pos, traits_type::length(__s));
+  }
+
   _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 size_type
-  find_last_not_of(value_type __c, size_type __pos = npos) const _NOEXCEPT;
+  find_last_not_of(value_type __c, size_type __pos = npos) const _NOEXCEPT {
+    return std::__str_find_last_not_of<value_type, size_type, traits_type, npos>(data(), size(), __c, __pos);
+  }
 
-  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 int compare(const basic_string& __str) const _NOEXCEPT;
+  // compare
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 int compare(const basic_string& __str) const _NOEXCEPT {
+    return compare(__self_view(__str));
+  }
 
   template <class _Tp, __enable_if_t<__can_be_converted_to_string_view<_CharT, _Traits, _Tp>::value, int> = 0>
   _LIBCPP_METHOD_TEMPLATE_IMPLICIT_INSTANTIATION_VIS _LIBCPP_CONSTEXPR_SINCE_CXX20 int
@@ -1818,25 +1939,46 @@ public:
 
   template <class _Tp, __enable_if_t<__can_be_converted_to_string_view<_CharT, _Traits, _Tp>::value, int> = 0>
   _LIBCPP_METHOD_TEMPLATE_IMPLICIT_INSTANTIATION_VIS _LIBCPP_CONSTEXPR_SINCE_CXX20 int
-  compare(size_type __pos1, size_type __n1, const _Tp& __t) const;
+  compare(size_type __pos1, size_type __n1, const _Tp& __t) const {
+    __self_view __sv = __t;
+    return compare(__pos1, __n1, __sv.data(), __sv.size());
+  }
 
   _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 int
-  compare(size_type __pos1, size_type __n1, const basic_string& __str) const;
+  compare(size_type __pos1, size_type __n1, const basic_string& __str) const {
+    return compare(__pos1, __n1, __str.data(), __str.size());
+  }
+
   _LIBCPP_CONSTEXPR_SINCE_CXX20 int
-  compare(size_type __pos1, size_type __n1, const basic_string& __str, size_type __pos2, size_type __n2 = npos) const;
+  compare(size_type __pos1, size_type __n1, const basic_string& __str, size_type __pos2, size_type __n2 = npos) const {
+    return compare(__pos1, __n1, __self_view(__str), __pos2, __n2);
+  }
 
   template <class _Tp,
             __enable_if_t<__can_be_converted_to_string_view<_CharT, _Traits, _Tp>::value &&
                               !__is_same_uncvref<_Tp, basic_string>::value,
                           int> = 0>
   inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 int
-  compare(size_type __pos1, size_type __n1, const _Tp& __t, size_type __pos2, size_type __n2 = npos) const;
+  compare(size_type __pos1, size_type __n1, const _Tp& __t, size_type __pos2, size_type __n2 = npos) const {
+    __self_view __sv = __t;
+    return __self_view(*this).substr(__pos1, __n1).compare(__sv.substr(__pos2, __n2));
+  }
+
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 int compare(const value_type* __s) const _NOEXCEPT {
+    _LIBCPP_ASSERT_NON_NULL(__s != nullptr, "string::compare(): received nullptr");
+    return compare(0, npos, __s, traits_type::length(__s));
+  }
+
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 int compare(size_type __pos1, size_type __n1, const value_type* __s) const {
+    _LIBCPP_ASSERT_NON_NULL(__s != nullptr, "string::compare(): received nullptr");
+    return compare(__pos1, __n1, __s, traits_type::length(__s));
+  }
 
-  _LIBCPP_CONSTEXPR_SINCE_CXX20 int compare(const value_type* __s) const _NOEXCEPT;
-  _LIBCPP_CONSTEXPR_SINCE_CXX20 int compare(size_type __pos1, size_type __n1, const value_type* __s) const;
   _LIBCPP_CONSTEXPR_SINCE_CXX20 int
   compare(size_type __pos1, size_type __n1, const value_type* __s, size_type __n2) const;
 
+  // starts_with
+
 #  if _LIBCPP_STD_VER >= 20
   constexpr _LIBCPP_HIDE_FROM_ABI bool starts_with(__self_view __sv) const noexcept {
     return __self_view(typename __self_view::__assume_valid(), data(), size()).starts_with(__sv);
@@ -1850,6 +1992,8 @@ public:
     return starts_with(__self_view(__s));
   }
 
+  // ends_with
+
   constexpr _LIBCPP_HIDE_FROM_ABI bool ends_with(__self_view __sv) const noexcept {
     return __self_view(typename __self_view::__assume_valid(), data(), size()).ends_with(__sv);
   }
@@ -1863,6 +2007,8 @@ public:
   }
 #  endif
 
+  // contains
+
 #  if _LIBCPP_STD_VER >= 23
   constexpr _LIBCPP_HIDE_FROM_ABI bool contains(__self_view __sv) const noexcept {
     return __self_view(typename __self_view::__assume_valid(), data(), size()).contains(__sv);
@@ -1879,11 +2025,7 @@ public:
 
   _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 bool __invariants() const;
 
-  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void __clear_and_shrink() _NOEXCEPT;
-
 private:
-  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void __shrink_or_extend(size_type __target_capacity);
-
   _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_STRING_INTERNAL_MEMORY_ACCESS bool
   __is_long() const _NOEXCEPT {
     if (__libcpp_is_constant_evaluated() && __builtin_constant_p(__rep_.__l.__is_long_)) {
@@ -2079,6 +2221,21 @@ private:
 #  endif
   }
 
+  // Disable ASan annotations and enable them again when going out of scope. It is assumed that the string is in a valid
+  // state at that point, so `size()` can be called safely.
+  struct [[__nodiscard__]] __annotation_guard {
+    __annotation_guard(const __annotation_guard&)            = delete;
+    __annotation_guard& operator=(const __annotation_guard&) = delete;
+
+    _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 __annotation_guard(basic_string& __str) : __str_(__str) {
+      __str_.__annotate_delete();
+    }
+
+    _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 ~__annotation_guard() { __str_.__annotate_new(__str_.size()); }
+
+    basic_string& __str_;
+  };
+
   template <size_type __a>
   static _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 size_type __align_it(size_type __s) _NOEXCEPT {
     return (__s + (__a - 1)) & ~(__a - 1);
@@ -2176,7 +2333,11 @@ private:
       __alloc_ = __str.__alloc_;
     else {
       if (!__str.__is_long()) {
-        __clear_and_shrink();
+        if (__is_long()) {
+          __annotate_delete();
+          __alloc_traits::deallocate(__alloc_, __get_long_pointer(), capacity() + 1);
+          __rep_ = __rep();
+        }
         __alloc_ = __str.__alloc_;
       } else {
         __annotate_delete();
@@ -3357,7 +3518,16 @@ _LIBCPP_CONSTEXPR_SINCE_CXX20 void basic_string<_CharT, _Traits, _Allocator>::re
   if (__requested_capacity <= capacity())
     return;
 
-  __shrink_or_extend(__recommend(__requested_capacity));
+  __annotation_guard __g(*this);
+  auto __allocation = std::__allocate_at_least(__alloc_, __recommend(__requested_capacity) + 1);
+  auto __size       = size();
+  __begin_lifetime(__allocation.ptr, __allocation.count);
+  traits_type::copy(std::__to_address(__allocation.ptr), data(), __size + 1);
+  if (__is_long())
+    __alloc_traits::deallocate(__alloc_, __get_long_pointer(), __get_long_cap());
+  __set_long_cap(__allocation.count);
+  __set_long_size(__size);
+  __set_long_pointer(__allocation.ptr);
 }
 
 template <class _CharT, class _Traits, class _Allocator>
@@ -3366,69 +3536,46 @@ inline _LIBCPP_CONSTEXPR_SINCE_CXX20 void basic_string<_CharT, _Traits, _Allocat
   if (__target_capacity == capacity())
     return;
 
-  __shrink_or_extend(__target_capacity);
-}
+  _LIBCPP_ASSERT_INTERNAL(__is_long(), "Trying to shrink small string");
 
-template <class _CharT, class _Traits, class _Allocator>
-inline _LIBCPP_CONSTEXPR_SINCE_CXX20 void
-basic_string<_CharT, _Traits, _Allocator>::__shrink_or_extend(size_type __target_capacity) {
-  __annotate_delete();
-  auto __guard    = std::__make_scope_guard(__annotate_new_size(*this));
-  size_type __cap = capacity();
-  size_type __sz  = size();
-
-  pointer __new_data, __p;
-  bool __was_long, __now_long;
+  // We're a long string and we're shrinking into the small buffer.
   if (__fits_in_sso(__target_capacity)) {
-    __was_long = true;
-    __now_long = false;
-    __new_data = __get_short_pointer();
-    __p        = __get_long_pointer();
-  } else {
-    if (__target_capacity > __cap) {
-      // Extend
-      // - called from reserve should propagate the exception thrown.
-      auto __allocation = std::__allocate_at_least(__alloc_, __target_capacity + 1);
-      __new_data        = __allocation.ptr;
-      __target_capacity = __allocation.count - 1;
-    } else {
-      // Shrink
-      // - called from shrink_to_fit should not throw.
-      // - called from reserve may throw but is not required to.
+    __annotation_guard __g(*this);
+    auto __ptr  = __get_long_pointer();
+    auto __size = __get_long_size();
+    auto __cap  = __get_long_cap();
+    traits_type::copy(std::__to_address(__get_short_pointer()), std::__to_address(__ptr), __size + 1);
+    __set_short_size(__size);
+    __alloc_traits::deallocate(__alloc_, __ptr, __cap);
+    return;
+  }
+
 #  if _LIBCPP_HAS_EXCEPTIONS
-      try {
+  try {
 #  endif // _LIBCPP_HAS_EXCEPTIONS
-        auto __allocation = std::__allocate_at_least(__alloc_, __target_capacity + 1);
-
-        // The Standard mandates shrink_to_fit() does not increase the capacity.
-        // With equal capacity keep the existing buffer. This avoids extra work
-        // due to swapping the elements.
-        if (__allocation.count - 1 > capacity()) {
-          __alloc_traits::deallocate(__alloc_, __allocation.ptr, __allocation.count);
-          return;
-        }
-        __new_data        = __allocation.ptr;
-        __target_capacity = __allocation.count - 1;
+    __annotation_guard __g(*this);
+    auto __size       = size();
+    auto __allocation = std::__allocate_at_least(__alloc_, __target_capacity + 1);
+
+    // The Standard mandates shrink_to_fit() does not increase the capacity.
+    // With equal capacity keep the existing buffer. This avoids extra work
+    // due to swapping the elements.
+    if (__allocation.count - 1 > capacity()) {
+      __alloc_traits::deallocate(__alloc_, __allocation.ptr, __allocation.count);
+      return;
+    }
+
+    __begin_lifetime(__allocation.ptr, __allocation.count);
+    auto __ptr = __get_long_pointer();
+    traits_type::copy(std::__to_address(__allocation.ptr), std::__to_address(__ptr), __size + 1);
+    __alloc_traits::deallocate(__alloc_, __ptr, __get_long_cap());
+    __set_long_cap(__allocation.count);
+    __set_long_pointer(__allocation.ptr);
 #  if _LIBCPP_HAS_EXCEPTIONS
-      } catch (...) {
-        return;
-      }
+  } catch (...) {
+    return;
+  }
 #  endif // _LIBCPP_HAS_EXCEPTIONS
-    }
-    __begin_lifetime(__new_data, __target_capacity + 1);
-    __now_long = true;
-    __was_long = __is_long();
-    __p        = __get_pointer();
-  }
-  traits_type::copy(std::__to_address(__new_data), std::__to_address(__p), size() + 1);
-  if (__was_long)
-    __alloc_traits::deallocate(__alloc_, __p, __cap + 1);
-  if (__now_long) {
-    __set_long_cap(__target_capacity + 1);
-    __set_long_size(__sz);
-    __set_long_pointer(__new_data);
-  } else
-    __set_short_size(__sz);
 }
 
 template <class _CharT, class _Traits, class _Allocator>
@@ -3482,243 +3629,6 @@ inline _LIBCPP_CONSTEXPR_SINCE_CXX20 void basic_string<_CharT, _Traits, _Allocat
     __str.__annotate_new(__str.__get_short_size());
 }
 
-// find
-
-template <class _CharT, class _Traits, class _Allocator>
-_LIBCPP_CONSTEXPR_SINCE_CXX20 typename basic_string<_CharT, _Traits, _Allocator>::size_type
-basic_string<_CharT, _Traits, _Allocator>::find(const value_type* __s, size_type __pos, size_type __n) const _NOEXCEPT {
-  _LIBCPP_ASSERT_NON_NULL(__n == 0 || __s != nullptr, "string::find(): received nullptr");
-  return std::__str_find<value_type, size_type, traits_type, npos>(data(), size(), __s, __pos, __n);
-}
-
-template <class _CharT, class _Traits, class _Allocator>
-inline _LIBCPP_CONSTEXPR_SINCE_CXX20 typename basic_string<_CharT, _Traits, _Allocator>::size_type
-basic_string<_CharT, _Traits, _Allocator>::find(const basic_string& __str, size_type __pos) const _NOEXCEPT {
-  return std::__str_find<value_type, size_type, traits_type, npos>(data(), size(), __str.data(), __pos, __str.size());
-}
-
-template <class _CharT, class _Traits, class _Allocator>
-template <class _Tp, __enable_if_t<__can_be_converted_to_string_view<_CharT, _Traits, _Tp>::value, int> >
-_LIBCPP_CONSTEXPR_SINCE_CXX20 typename basic_string<_CharT, _Traits, _Allocator>::size_type
-basic_string<_CharT, _Traits, _Allocator>::find(const _Tp& __t, size_type __pos) const _NOEXCEPT {
-  __self_view __sv = __t;
-  return std::__str_find<value_type, size_type, traits_type, npos>(data(), size(), __sv.data(), __pos, __sv.size());
-}
-
-template <class _CharT, class _Traits, class _Allocator>
-inline _LIBCPP_CONSTEXPR_SINCE_CXX20 typename basic_string<_CharT, _Traits, _Allocator>::size_type
-basic_string<_CharT, _Traits, _Allocator>::find(const value_type* __s, size_type __pos) const _NOEXCEPT {
-  _LIBCPP_ASSERT_NON_NULL(__s != nullptr, "string::find(): received nullptr");
-  return std::__str_find<value_type, size_type, traits_type, npos>(
-      data(), size(), __s, __pos, traits_type::length(__s));
-}
-
-template <class _CharT, class _Traits, class _Allocator>
-_LIBCPP_CONSTEXPR_SINCE_CXX20 typename basic_string<_CharT, _Traits, _Allocator>::size_type
-basic_string<_CharT, _Traits, _Allocator>::find(value_type __c, size_type __pos) const _NOEXCEPT {
-  return std::__str_find<value_type, size_type, traits_type, npos>(data(), size(), __c, __pos);
-}
-
-// rfind
-
-template <class _CharT, class _Traits, class _Allocator>
-_LIBCPP_CONSTEXPR_SINCE_CXX20 typename basic_string<_CharT, _Traits, _Allocator>::size_type
-basic_string<_CharT, _Traits, _Allocator>::rfind(
-    const value_type* __s, size_type __pos, size_type __n) const _NOEXCEPT {
-  _LIBCPP_ASSERT_NON_NULL(__n == 0 || __s != nullptr, "string::rfind(): received nullptr");
-  return std::__str_rfind<value_type, size_type, traits_type, npos>(data(), size(), __s, __pos, __n);
-}
-
-template <class _CharT, class _Traits, class _Allocator>
-inline _LIBCPP_CONSTEXPR_SINCE_CXX20 typename basic_string<_CharT, _Traits, _Allocator>::size_type
-basic_string<_CharT, _Traits, _Allocator>::rfind(const basic_string& __str, size_type __pos) const _NOEXCEPT {
-  return std::__str_rfind<value_type, size_type, traits_type, npos>(data(), size(), __str.data(), __pos, __str.size());
-}
-
-template <class _CharT, class _Traits, class _Allocator>
-template <class _Tp, __enable_if_t<__can_be_converted_to_string_view<_CharT, _Traits, _Tp>::value, int> >
-_LIBCPP_CONSTEXPR_SINCE_CXX20 typename basic_string<_CharT, _Traits, _Allocator>::size_type
-basic_string<_CharT, _Traits, _Allocator>::rfind(const _Tp& __t, size_type __pos) const _NOEXCEPT {
-  __self_view __sv = __t;
-  return std::__str_rfind<value_type, size_type, traits_type, npos>(data(), size(), __sv.data(), __pos, __sv.size());
-}
-
-template <class _CharT, class _Traits, class _Allocator>
-inline _LIBCPP_CONSTEXPR_SINCE_CXX20 typename basic_string<_CharT, _Traits, _Allocator>::size_type
-basic_string<_CharT, _Traits, _Allocator>::rfind(const value_type* __s, size_type __pos) const _NOEXCEPT {
-  _LIBCPP_ASSERT_NON_NULL(__s != nullptr, "string::rfind(): received nullptr");
-  return std::__str_rfind<value_type, size_type, traits_type, npos>(
-      data(), size(), __s, __pos, traits_type::length(__s));
-}
-
-template <class _CharT, class _Traits, class _Allocator>
-_LIBCPP_CONSTEXPR_SINCE_CXX20 typename basic_string<_CharT, _Traits, _Allocator>::size_type
-basic_string<_CharT, _Traits, _Allocator>::rfind(value_type __c, size_type __pos) const _NOEXCEPT {
-  return std::__str_rfind<value_type, size_type, traits_type, npos>(data(), size(), __c, __pos);
-}
-
-// find_first_of
-
-template <class _CharT, class _Traits, class _Allocator>
-_LIBCPP_CONSTEXPR_SINCE_CXX20 typename basic_string<_CharT, _Traits, _Allocator>::size_type
-basic_string<_CharT, _Traits, _Allocator>::find_first_of(
-    const value_type* __s, size_type __pos, size_type __n) const _NOEXCEPT {
-  _LIBCPP_ASSERT_NON_NULL(__n == 0 || __s != nullptr, "string::find_first_of(): received nullptr");
-  return std::__str_find_first_of<value_type, size_type, traits_type, npos>(data(), size(), __s, __pos, __n);
-}
-
-template <class _CharT, class _Traits, class _Allocator>
-inline _LIBCPP_CONSTEXPR_SINCE_CXX20 typename basic_string<_CharT, _Traits, _Allocator>::size_type
-basic_string<_CharT, _Traits, _Allocator>::find_first_of(const basic_string& __str, size_type __pos) const _NOEXCEPT {
-  return std::__str_find_first_of<value_type, size_type, traits_type, npos>(
-      data(), size(), __str.data(), __pos, __str.size());
-}
-
-template <class _CharT, class _Traits, class _Allocator>
-template <class _Tp, __enable_if_t<__can_be_converted_to_string_view<_CharT, _Traits, _Tp>::value, int> >
-_LIBCPP_CONSTEXPR_SINCE_CXX20 typename basic_string<_CharT, _Traits, _Allocator>::size_type
-basic_string<_CharT, _Traits, _Allocator>::find_first_of(const _Tp& __t, size_type __pos) const _NOEXCEPT {
-  __self_view __sv = __t;
-  return std::__str_find_first_of<value_type, size_type, traits_type, npos>(
-      data(), size(), __sv.data(), __pos, __sv.size());
-}
-
-template <class _CharT, class _Traits, class _Allocator>
-inline _LIBCPP_CONSTEXPR_SINCE_CXX20 typename basic_string<_CharT, _Traits, _Allocator>::size_type
-basic_string<_CharT, _Traits, _Allocator>::find_first_of(const value_type* __s, size_type __pos) const _NOEXCEPT {
-  _LIBCPP_ASSERT_NON_NULL(__s != nullptr, "string::find_first_of(): received nullptr");
-  return std::__str_find_first_of<value_type, size_type, traits_type, npos>(
-      data(), size(), __s, __pos, traits_type::length(__s));
-}
-
-template <class _CharT, class _Traits, class _Allocator>
-inline _LIBCPP_CONSTEXPR_SINCE_CXX20 typename basic_string<_CharT, _Traits, _Allocator>::size_type
-basic_string<_CharT, _Traits, _Allocator>::find_first_of(value_type __c, size_type __pos) const _NOEXCEPT {
-  return find(__c, __pos);
-}
-
-// find_last_of
-
-template <class _CharT, class _Traits, class _Allocator>
-inline _LIBCPP_CONSTEXPR_SINCE_CXX20 typename basic_string<_CharT, _Traits, _Allocator>::size_type
-basic_string<_CharT, _Traits, _Allocator>::find_last_of(
-    const value_type* __s, size_type __pos, size_type __n) const _NOEXCEPT {
-  _LIBCPP_ASSERT_NON_NULL(__n == 0 || __s != nullptr, "string::find_last_of(): received nullptr");
-  return std::__str_find_last_of<value_type, size_type, traits_type, npos>(data(), size(), __s, __pos, __n);
-}
-
-template <class _CharT, class _Traits, class _Allocator>
-inline _LIBCPP_CONSTEXPR_SINCE_CXX20 typename basic_string<_CharT, _Traits, _Allocator>::size_type
-basic_string<_CharT, _Traits, _Allocator>::find_last_of(const basic_string& __str, size_type __pos) const _NOEXCEPT {
-  return std::__str_find_last_of<value_type, size_type, traits_type, npos>(
-      data(), size(), __str.data(), __pos, __str.size());
-}
-
-template <class _CharT, class _Traits, class _Allocator>
-template <class _Tp, __enable_if_t<__can_be_converted_to_string_view<_CharT, _Traits, _Tp>::value, int> >
-_LIBCPP_CONSTEXPR_SINCE_CXX20 typename basic_string<_CharT, _Traits, _Allocator>::size_type
-basic_string<_CharT, _Traits, _Allocator>::find_last_of(const _Tp& __t, size_type __pos) const _NOEXCEPT {
-  __self_view __sv = __t;
-  return std::__str_find_last_of<value_type, size_type, traits_type, npos>(
-      data(), size(), __sv.data(), __pos, __sv.size());
-}
-
-template <class _CharT, class _Traits, class _Allocator>
-inline _LIBCPP_CONSTEXPR_SINCE_CXX20 typename basic_string<_CharT, _Traits, _Allocator>::size_type
-basic_string<_CharT, _Traits, _Allocator>::find_last_of(const value_type* __s, size_type __pos) const _NOEXCEPT {
-  _LIBCPP_ASSERT_NON_NULL(__s != nullptr, "string::find_last_of(): received nullptr");
-  return std::__str_find_last_of<value_type, size_type, traits_type, npos>(
-      data(), size(), __s, __pos, traits_type::length(__s));
-}
-
-template <class _CharT, class _Traits, class _Allocator>
-inline _LIBCPP_CONSTEXPR_SINCE_CXX20 typename basic_string<_CharT, _Traits, _Allocator>::size_type
-basic_string<_CharT, _Traits, _Allocator>::find_last_of(value_type __c, size_type __pos) const _NOEXCEPT {
-  return rfind(__c, __pos);
-}
-
-// find_first_not_of
-
-template <class _CharT, class _Traits, class _Allocator>
-_LIBCPP_CONSTEXPR_SINCE_CXX20 typename basic_string<_CharT, _Traits, _Allocator>::size_type
-basic_string<_CharT, _Traits, _Allocator>::find_first_not_of(
-    const value_type* __s, size_type __pos, size_type __n) const _NOEXCEPT {
-  _LIBCPP_ASSERT_NON_NULL(__n == 0 || __s != nullptr, "string::find_first_not_of(): received nullptr");
-  return std::__str_find_first_not_of<value_type, size_type, traits_type, npos>(data(), size(), __s, __pos, __n);
-}
-
-template <class _CharT, class _Traits, class _Allocator>
-inline _LIBCPP_CONSTEXPR_SINCE_CXX20 typename basic_string<_CharT, _Traits, _Allocator>::size_type
-basic_string<_CharT, _Traits, _Allocator>::find_first_not_of(
-    const basic_string& __str, size_type __pos) const _NOEXCEPT {
-  return std::__str_find_first_not_of<value_type, size_type, traits_type, npos>(
-      data(), size(), __str.data(), __pos, __str.size());
-}
-
-template <class _CharT, class _Traits, class _Allocator>
-template <class _Tp, __enable_if_t<__can_be_converted_to_string_view<_CharT, _Traits, _Tp>::value, int> >
-_LIBCPP_CONSTEXPR_SINCE_CXX20 typename basic_string<_CharT, _Traits, _Allocator>::size_type
-basic_string<_CharT, _Traits, _Allocator>::find_first_not_of(const _Tp& __t, size_type __pos) const _NOEXCEPT {
-  __self_view __sv = __t;
-  return std::__str_find_first_not_of<value_type, size_type, traits_type, npos>(
-      data(), size(), __sv.data(), __pos, __sv.size());
-}
-
-template <class _CharT, class _Traits, class _Allocator>
-inline _LIBCPP_CONSTEXPR_SINCE_CXX20 typename basic_string<_CharT, _Traits, _Allocator>::size_type
-basic_string<_CharT, _Traits, _Allocator>::find_first_not_of(const value_type* __s, size_type __pos) const _NOEXCEPT {
-  _LIBCPP_ASSERT_NON_NULL(__s != nullptr, "string::find_first_not_of(): received nullptr");
-  return std::__str_find_first_not_of<value_type, size_type, traits_type, npos>(
-      data(), size(), __s, __pos, traits_type::length(__s));
-}
-
-template <class _CharT, class _Traits, class _Allocator>
-inline _LIBCPP_CONSTEXPR_SINCE_CXX20 typename basic_string<_CharT, _Traits, _Allocator>::size_type
-basic_string<_CharT, _Traits, _Allocator>::find_first_not_of(value_type __c, size_type __pos) const _NOEXCEPT {
-  return std::__str_find_first_not_of<value_type, size_type, traits_type, npos>(data(), size(), __c, __pos);
-}
-
-// find_last_not_of
-
-template <class _CharT, class _Traits, class _Allocator>
-_LIBCPP_CONSTEXPR_SINCE_CXX20 typename basic_string<_CharT, _Traits, _Allocator>::size_type
-basic_string<_CharT, _Traits, _Allocator>::find_last_not_of(
-    const value_type* __s, size_type __pos, size_type __n) const _NOEXCEPT {
-  _LIBCPP_ASSERT_NON_NULL(__n == 0 || __s != nullptr, "string::find_last_not_of(): received nullptr");
-  return std::__str_find_last_not_of<value_type, size_type, traits_type, npos>(data(), size(), __s, __pos, __n);
-}
-
-template <class _CharT, class _Traits, class _Allocator>
-inline _LIBCPP_CONSTEXPR_SINCE_CXX20 typename basic_string<_CharT, _Traits, _Allocator>::size_type
-basic_string<_CharT, _Traits, _Allocator>::find_last_not_of(
-    const basic_string& __str, size_type __pos) const _NOEXCEPT {
-  return std::__str_find_last_not_of<value_type, size_type, traits_type, npos>(
-      data(), size(), __str.data(), __pos, __str.size());
-}
-
-template <class _CharT, class _Traits, class _Allocator>
-template <class _Tp, __enable_if_t<__can_be_converted_to_string_view<_CharT, _Traits, _Tp>::value, int> >
-_LIBCPP_CONSTEXPR_SINCE_CXX20 typename basic_string<_CharT, _Traits, _Allocator>::size_type
-basic_string<_CharT, _Traits, _Allocator>::find_last_not_of(const _Tp& __t, size_type __pos) const _NOEXCEPT {
-  __self_view __sv = __t;
-  return std::__str_find_last_not_of<value_type, size_type, traits_type, npos>(
-      data(), size(), __sv.data(), __pos, __sv.size());
-}
-
-template <class _CharT, class _Traits, class _Allocator>
-inline _LIBCPP_CONSTEXPR_SINCE_CXX20 typename basic_string<_CharT, _Traits, _Allocator>::size_type
-basic_string<_CharT, _Traits, _Allocator>::find_last_not_of(const value_type* __s, size_type __pos) const _NOEXCEPT {
-  _LIBCPP_ASSERT_NON_NULL(__s != nullptr, "string::find_last_not_of(): received nullptr");
-  return std::__str_find_last_not_of<value_type, size_type, traits_type, npos>(
-      data(), size(), __s, __pos, traits_type::length(__s));
-}
-
-template <class _CharT, class _Traits, class _Allocator>
-inline _LIBCPP_CONSTEXPR_SINCE_CXX20 typename basic_string<_CharT, _Traits, _Allocator>::size_type
-basic_string<_CharT, _Traits, _Allocator>::find_last_not_of(value_type __c, size_type __pos) const _NOEXCEPT {
-  return std::__str_find_last_not_of<value_type, size_type, traits_type, npos>(data(), size(), __c, __pos);
-}
-
 // compare
 
 template <class _CharT, class _Traits, class _Allocator>
@@ -3737,12 +3647,6 @@ _LIBCPP_CONSTEXPR_SINCE_CXX20 int basic_string<_CharT, _Traits, _Allocator>::com
   return 0;
 }
 
-template <class _CharT, class _Traits, class _Allocator>
-inline _LIBCPP_CONSTEXPR_SINCE_CXX20 int
-basic_string<_CharT, _Traits, _Allocator>::compare(const basic_string& __str) const _NOEXCEPT {
-  return compare(__self_view(__str));
-}
-
 template <class _CharT, class _Traits, class _Allocator>
 inline _LIBCPP_CONSTEXPR_SINCE_CXX20 int basic_string<_CharT, _Traits, _Allocator>::compare(
     size_type __pos1, size_type __n1, const value_type* __s, size_type __n2) const {
@@ -3761,51 +3665,6 @@ inline _LIBCPP_CONSTEXPR_SINCE_CXX20 int basic_string<_CharT, _Traits, _Allocato
   return __r;
 }
 
-template <class _CharT, class _Traits, class _Allocator>
-template <class _Tp, __enable_if_t<__can_be_converted_to_string_view<_CharT, _Traits, _Tp>::value, int> >
-_LIBCPP_CONSTEXPR_SINCE_CXX20 int
-basic_string<_CharT, _Traits, _Allocator>::compare(size_type __pos1, size_type __n1, const _Tp& __t) const {
-  __self_view __sv = __t;
-  return compare(__pos1, __n1, __sv.data(), __sv.size());
-}
-
-template <class _CharT, class _Traits, class _Allocator>
-inline _LIBCPP_CONSTEXPR_SINCE_CXX20 int
-basic_string<_CharT, _Traits, _Allocator>::compare(size_type __pos1, size_type __n1, const basic_string& __str) const {
-  return compare(__pos1, __n1, __str.data(), __str.size());
-}
-
-template <class _CharT, class _Traits, class _Allocator>
-template <class _Tp,
-          __enable_if_t<__can_be_converted_to_string_view<_CharT, _Traits, _Tp>::value &&
-                            !__is_same_uncvref<_Tp, basic_string<_CharT, _Traits, _Allocator> >::value,
-                        int> >
-_LIBCPP_CONSTEXPR_SINCE_CXX20 int basic_string<_CharT, _Traits, _Allocator>::compare(
-    size_type __pos1, size_type __n1, const _Tp& __t, size_type __pos2, size_type __n2) const {
-  __self_view __sv = __t;
-  return __self_view(*this).substr(__pos1, __n1).compare(__sv.substr(__pos2, __n2));
-}
-
-template <class _CharT, class _Traits, class _Allocator>
-_LIBCPP_CONSTEXPR_SINCE_CXX20 int basic_string<_CharT, _Traits, _Allocator>::compare(
-    size_type __pos1, size_type __n1, const basic_string& __str, size_type __pos2, size_type __n2) const {
-  return compare(__pos1, __n1, __self_view(__str), __pos2, __n2);
-}
-
-template <class _CharT, class _Traits, class _Allocator>
-_LIBCPP_CONSTEXPR_SINCE_CXX20 int
-basic_string<_CharT, _Traits, _Allocator>::compare(const value_type* __s) const _NOEXCEPT {
-  _LIBCPP_ASSERT_NON_NULL(__s != nullptr, "string::compare(): received nullptr");
-  return compare(0, npos, __s, traits_type::length(__s));
-}
-
-template <class _CharT, class _Traits, class _Allocator>
-_LIBCPP_CONSTEXPR_SINCE_CXX20 int
-basic_string<_CharT, _Traits, _Allocator>::compare(size_type __pos1, size_type __n1, const value_type* __s) const {
-  _LIBCPP_ASSERT_NON_NULL(__s != nullptr, "string::compare(): received nullptr");
-  return compare(__pos1, __n1, __s, traits_type::length(__s));
-}
-
 // __invariants
 
 template <class _CharT, class _Traits, class _Allocator>
@@ -3821,18 +3680,6 @@ inline _LIBCPP_CONSTEXPR_SINCE_CXX20 bool basic_string<_CharT, _Traits, _Allocat
   return true;
 }
 
-// __clear_and_shrink
-
-template <class _CharT, class _Traits, class _Allocator>
-inline _LIBCPP_CONSTEXPR_SINCE_CXX20 void basic_string<_CharT, _Traits, _Allocator>::__clear_and_shrink() _NOEXCEPT {
-  clear();
-  if (__is_long()) {
-    __annotate_delete();
-    __alloc_traits::deallocate(__alloc_, __get_long_pointer(), capacity() + 1);
-    __rep_ = __rep();
-  }
-}
-
 // operator==
 
 template <class _CharT, class _Traits, class _Allocator>
diff --git a/libcxx/include/version b/libcxx/include/version
index 57d6ec629d27c..29a71ed574e56 100644
--- a/libcxx/include/version
+++ b/libcxx/include/version
@@ -101,6 +101,8 @@ __cpp_lib_execution                                     201902L <execution>
                                                         201603L // C++17
 __cpp_lib_expected                                      202211L <expected>
 __cpp_lib_filesystem                                    201703L <filesystem>
+__cpp_lib_flat_map                                      202207L <flat_map>
+__cpp_lib_flat_set                                      202207L <flat_set>
 __cpp_lib_format                                        202110L <format>
 __cpp_lib_format_path                                   202403L <filesystem>
 __cpp_lib_format_ranges                                 202207L <format>
@@ -480,6 +482,8 @@ __cpp_lib_void_t                                        201411L <type_traits>
 # define __cpp_lib_constexpr_typeinfo                   202106L
 # define __cpp_lib_containers_ranges                    202202L
 # define __cpp_lib_expected                             202211L
+# define __cpp_lib_flat_map                             202207L
+// # define __cpp_lib_flat_set                             202207L
 # define __cpp_lib_format_ranges                        202207L
 // # define __cpp_lib_formatters                           202302L
 # define __cpp_lib_forward_like                         202207L
diff --git a/libcxx/lib/abi/i686-linux-android21.libcxxabi.v1.stable.exceptions.nonew.abilist b/libcxx/lib/abi/i686-linux-android21.libcxxabi.v1.stable.exceptions.nonew.abilist
index 50f87bc05ec91..b38cedb012e8c 100644
--- a/libcxx/lib/abi/i686-linux-android21.libcxxabi.v1.stable.exceptions.nonew.abilist
+++ b/libcxx/lib/abi/i686-linux-android21.libcxxabi.v1.stable.exceptions.nonew.abilist
@@ -1221,7 +1221,7 @@
 {'is_defined': True, 'name': '_ZNSt6__ndk131__arrive_barrier_algorithm_baseEPNS_24__barrier_algorithm_baseEh', 'type': 'FUNC'}
 {'is_defined': True, 'name': '_ZNSt6__ndk132__destroy_barrier_algorithm_baseEPNS_24__barrier_algorithm_baseE', 'type': 'FUNC'}
 {'is_defined': True, 'name': '_ZNSt6__ndk134__construct_barrier_algorithm_baseERi', 'type': 'FUNC'}
-{'is_defined': True, 'name': '_ZNSt6__ndk13cinE', 'size': 88, 'type': 'OBJECT'}
+{'is_defined': True, 'name': '_ZNSt6__ndk13cinE', 'size': 148, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '_ZNSt6__ndk13pmr15memory_resourceD0Ev', 'type': 'FUNC'}
 {'is_defined': True, 'name': '_ZNSt6__ndk13pmr15memory_resourceD1Ev', 'type': 'FUNC'}
 {'is_defined': True, 'name': '_ZNSt6__ndk13pmr15memory_resourceD2Ev', 'type': 'FUNC'}
@@ -1289,9 +1289,9 @@
 {'is_defined': True, 'name': '_ZNSt6__ndk14__fs10filesystem8__removeERKNS1_4pathEPNS_10error_codeE', 'type': 'FUNC'}
 {'is_defined': True, 'name': '_ZNSt6__ndk14__fs10filesystem8__renameERKNS1_4pathES4_PNS_10error_codeE', 'type': 'FUNC'}
 {'is_defined': True, 'name': '_ZNSt6__ndk14__fs10filesystem8__statusERKNS1_4pathEPNS_10error_codeE', 'type': 'FUNC'}
-{'is_defined': True, 'name': '_ZNSt6__ndk14cerrE', 'size': 84, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '_ZNSt6__ndk14clogE', 'size': 84, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '_ZNSt6__ndk14coutE', 'size': 84, 'type': 'OBJECT'}
+{'is_defined': True, 'name': '_ZNSt6__ndk14cerrE', 'size': 136, 'type': 'OBJECT'}
+{'is_defined': True, 'name': '_ZNSt6__ndk14clogE', 'size': 136, 'type': 'OBJECT'}
+{'is_defined': True, 'name': '_ZNSt6__ndk14coutE', 'size': 136, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '_ZNSt6__ndk14stodERKNS_12basic_stringIcNS_11char_traitsIcEENS_9allocatorIcEEEEPj', 'type': 'FUNC'}
 {'is_defined': True, 'name': '_ZNSt6__ndk14stodERKNS_12basic_stringIwNS_11char_traitsIwEENS_9allocatorIwEEEEPj', 'type': 'FUNC'}
 {'is_defined': True, 'name': '_ZNSt6__ndk14stofERKNS_12basic_stringIcNS_11char_traitsIcEENS_9allocatorIcEEEEPj', 'type': 'FUNC'}
@@ -1300,7 +1300,7 @@
 {'is_defined': True, 'name': '_ZNSt6__ndk14stoiERKNS_12basic_stringIwNS_11char_traitsIwEENS_9allocatorIwEEEEPji', 'type': 'FUNC'}
 {'is_defined': True, 'name': '_ZNSt6__ndk14stolERKNS_12basic_stringIcNS_11char_traitsIcEENS_9allocatorIcEEEEPji', 'type': 'FUNC'}
 {'is_defined': True, 'name': '_ZNSt6__ndk14stolERKNS_12basic_stringIwNS_11char_traitsIwEENS_9allocatorIwEEEEPji', 'type': 'FUNC'}
-{'is_defined': True, 'name': '_ZNSt6__ndk14wcinE', 'size': 88, 'type': 'OBJECT'}
+{'is_defined': True, 'name': '_ZNSt6__ndk14wcinE', 'size': 148, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '_ZNSt6__ndk15alignEjjRPvRj', 'type': 'FUNC'}
 {'is_defined': True, 'name': '_ZNSt6__ndk15ctypeIcE10table_sizeE', 'size': 4, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '_ZNSt6__ndk15ctypeIcE13classic_tableEv', 'type': 'FUNC'}
@@ -1325,9 +1325,9 @@
 {'is_defined': True, 'name': '_ZNSt6__ndk15stollERKNS_12basic_stringIwNS_11char_traitsIwEENS_9allocatorIwEEEEPji', 'type': 'FUNC'}
 {'is_defined': True, 'name': '_ZNSt6__ndk15stoulERKNS_12basic_stringIcNS_11char_traitsIcEENS_9allocatorIcEEEEPji', 'type': 'FUNC'}
 {'is_defined': True, 'name': '_ZNSt6__ndk15stoulERKNS_12basic_stringIwNS_11char_traitsIwEENS_9allocatorIwEEEEPji', 'type': 'FUNC'}
-{'is_defined': True, 'name': '_ZNSt6__ndk15wcerrE', 'size': 84, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '_ZNSt6__ndk15wclogE', 'size': 84, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '_ZNSt6__ndk15wcoutE', 'size': 84, 'type': 'OBJECT'}
+{'is_defined': True, 'name': '_ZNSt6__ndk15wcerrE', 'size': 136, 'type': 'OBJECT'}
+{'is_defined': True, 'name': '_ZNSt6__ndk15wclogE', 'size': 136, 'type': 'OBJECT'}
+{'is_defined': True, 'name': '_ZNSt6__ndk15wcoutE', 'size': 136, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '_ZNSt6__ndk16__clocEv', 'type': 'FUNC'}
 {'is_defined': True, 'name': '_ZNSt6__ndk16__itoa8__u32toaEjPc', 'type': 'FUNC'}
 {'is_defined': True, 'name': '_ZNSt6__ndk16__itoa8__u64toaEyPc', 'type': 'FUNC'}
@@ -1729,10 +1729,8 @@
 {'is_defined': True, 'name': '_ZTINSt6__ndk120__time_get_c_storageIwEE', 'size': 8, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '_ZTINSt6__ndk13pmr15memory_resourceE', 'size': 8, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '_ZTINSt6__ndk13pmr25monotonic_buffer_resourceE', 'size': 12, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '_ZTINSt6__ndk13pmr26__null_memory_resource_impE', 'size': 12, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '_ZTINSt6__ndk13pmr26synchronized_pool_resourceE', 'size': 12, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '_ZTINSt6__ndk13pmr28unsynchronized_pool_resourceE', 'size': 12, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '_ZTINSt6__ndk13pmr32__new_delete_memory_resource_impE', 'size': 12, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '_ZTINSt6__ndk14__fs10filesystem16filesystem_errorE', 'size': 12, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '_ZTINSt6__ndk15ctypeIcEE', 'size': 32, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '_ZTINSt6__ndk15ctypeIwEE', 'size': 32, 'type': 'OBJECT'}
@@ -1966,10 +1964,8 @@
 {'is_defined': True, 'name': '_ZTSNSt6__ndk120__time_get_c_storageIwEE', 'size': 37, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '_ZTSNSt6__ndk13pmr15memory_resourceE', 'size': 33, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '_ZTSNSt6__ndk13pmr25monotonic_buffer_resourceE', 'size': 43, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '_ZTSNSt6__ndk13pmr26__null_memory_resource_impE', 'size': 44, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '_ZTSNSt6__ndk13pmr26synchronized_pool_resourceE', 'size': 44, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '_ZTSNSt6__ndk13pmr28unsynchronized_pool_resourceE', 'size': 46, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '_ZTSNSt6__ndk13pmr32__new_delete_memory_resource_impE', 'size': 50, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '_ZTSNSt6__ndk14__fs10filesystem16filesystem_errorE', 'size': 47, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '_ZTSNSt6__ndk15ctypeIcEE', 'size': 21, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '_ZTSNSt6__ndk15ctypeIwEE', 'size': 21, 'type': 'OBJECT'}
@@ -2195,10 +2191,8 @@
 {'is_defined': True, 'name': '_ZTVNSt6__ndk120__codecvt_utf8_utf16IwEE', 'size': 48, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '_ZTVNSt6__ndk13pmr15memory_resourceE', 'size': 28, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '_ZTVNSt6__ndk13pmr25monotonic_buffer_resourceE', 'size': 28, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '_ZTVNSt6__ndk13pmr26__null_memory_resource_impE', 'size': 28, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '_ZTVNSt6__ndk13pmr26synchronized_pool_resourceE', 'size': 28, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '_ZTVNSt6__ndk13pmr28unsynchronized_pool_resourceE', 'size': 28, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '_ZTVNSt6__ndk13pmr32__new_delete_memory_resource_impE', 'size': 28, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '_ZTVNSt6__ndk14__fs10filesystem16filesystem_errorE', 'size': 20, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '_ZTVNSt6__ndk15ctypeIcEE', 'size': 52, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '_ZTVNSt6__ndk15ctypeIwEE', 'size': 68, 'type': 'OBJECT'}
diff --git a/libcxx/lib/abi/powerpc-ibm-aix.libcxxabi.v1.stable.exceptions.nonew.abilist b/libcxx/lib/abi/powerpc-ibm-aix.libcxxabi.v1.stable.exceptions.nonew.abilist
index 7ab386411a957..9f299d65e63f8 100644
--- a/libcxx/lib/abi/powerpc-ibm-aix.libcxxabi.v1.stable.exceptions.nonew.abilist
+++ b/libcxx/lib/abi/powerpc-ibm-aix.libcxxabi.v1.stable.exceptions.nonew.abilist
@@ -1839,8 +1839,6 @@
 {'import_export': 'wEXP', 'is_defined': True, 'name': '_ZTINSt3__119basic_ostringstreamIcNS_11char_traitsIcEENS_9allocatorIcEEEE', 'storage_mapping_class': 'RW', 'type': 'OBJECT'}
 {'import_export': 'wEXP', 'is_defined': True, 'name': '_ZTINSt3__120__time_get_c_storageIcEE', 'storage_mapping_class': 'RW', 'type': 'OBJECT'}
 {'import_export': 'wEXP', 'is_defined': True, 'name': '_ZTINSt3__120__time_get_c_storageIwEE', 'storage_mapping_class': 'RW', 'type': 'OBJECT'}
-{'import_export': 'wEXP', 'is_defined': True, 'name': '_ZTINSt3__13pmr26__null_memory_resource_impE', 'storage_mapping_class': 'RW', 'type': 'OBJECT'}
-{'import_export': 'wEXP', 'is_defined': True, 'name': '_ZTINSt3__13pmr32__new_delete_memory_resource_impE', 'storage_mapping_class': 'RW', 'type': 'OBJECT'}
 {'import_export': 'wEXP', 'is_defined': True, 'name': '_ZTINSt3__17collateIcEE', 'storage_mapping_class': 'RW', 'type': 'OBJECT'}
 {'import_export': 'wEXP', 'is_defined': True, 'name': '_ZTINSt3__17collateIwEE', 'storage_mapping_class': 'RW', 'type': 'OBJECT'}
 {'import_export': 'wEXP', 'is_defined': True, 'name': '_ZTINSt3__17num_getIcNS_19istreambuf_iteratorIcNS_11char_traitsIcEEEEEE', 'storage_mapping_class': 'RW', 'type': 'OBJECT'}
@@ -1914,8 +1912,6 @@
 {'import_export': 'wEXP', 'is_defined': True, 'name': '_ZTSNSt3__119basic_ostringstreamIcNS_11char_traitsIcEENS_9allocatorIcEEEE', 'storage_mapping_class': 'RO', 'type': 'OBJECT'}
 {'import_export': 'wEXP', 'is_defined': True, 'name': '_ZTSNSt3__120__time_get_c_storageIcEE', 'storage_mapping_class': 'RO', 'type': 'OBJECT'}
 {'import_export': 'wEXP', 'is_defined': True, 'name': '_ZTSNSt3__120__time_get_c_storageIwEE', 'storage_mapping_class': 'RO', 'type': 'OBJECT'}
-{'import_export': 'wEXP', 'is_defined': True, 'name': '_ZTSNSt3__13pmr26__null_memory_resource_impE', 'storage_mapping_class': 'RO', 'type': 'OBJECT'}
-{'import_export': 'wEXP', 'is_defined': True, 'name': '_ZTSNSt3__13pmr32__new_delete_memory_resource_impE', 'storage_mapping_class': 'RO', 'type': 'OBJECT'}
 {'import_export': 'wEXP', 'is_defined': True, 'name': '_ZTSNSt3__17collateIcEE', 'storage_mapping_class': 'RO', 'type': 'OBJECT'}
 {'import_export': 'wEXP', 'is_defined': True, 'name': '_ZTSNSt3__17collateIwEE', 'storage_mapping_class': 'RO', 'type': 'OBJECT'}
 {'import_export': 'wEXP', 'is_defined': True, 'name': '_ZTSNSt3__17num_getIcNS_19istreambuf_iteratorIcNS_11char_traitsIcEEEEEE', 'storage_mapping_class': 'RO', 'type': 'OBJECT'}
@@ -1983,8 +1979,6 @@
 {'import_export': 'wEXP', 'is_defined': True, 'name': '_ZTVNSt3__118basic_stringstreamIcNS_11char_traitsIcEENS_9allocatorIcEEEE', 'storage_mapping_class': 'RW', 'type': 'OBJECT'}
 {'import_export': 'wEXP', 'is_defined': True, 'name': '_ZTVNSt3__119basic_istringstreamIcNS_11char_traitsIcEENS_9allocatorIcEEEE', 'storage_mapping_class': 'RW', 'type': 'OBJECT'}
 {'import_export': 'wEXP', 'is_defined': True, 'name': '_ZTVNSt3__119basic_ostringstreamIcNS_11char_traitsIcEENS_9allocatorIcEEEE', 'storage_mapping_class': 'RW', 'type': 'OBJECT'}
-{'import_export': 'wEXP', 'is_defined': True, 'name': '_ZTVNSt3__13pmr26__null_memory_resource_impE', 'storage_mapping_class': 'RW', 'type': 'OBJECT'}
-{'import_export': 'wEXP', 'is_defined': True, 'name': '_ZTVNSt3__13pmr32__new_delete_memory_resource_impE', 'storage_mapping_class': 'RW', 'type': 'OBJECT'}
 {'import_export': 'wEXP', 'is_defined': True, 'name': '_ZTVNSt3__17collateIcEE', 'storage_mapping_class': 'RW', 'type': 'OBJECT'}
 {'import_export': 'wEXP', 'is_defined': True, 'name': '_ZTVNSt3__17collateIwEE', 'storage_mapping_class': 'RW', 'type': 'OBJECT'}
 {'import_export': 'wEXP', 'is_defined': True, 'name': '_ZTVNSt3__17num_getIcNS_19istreambuf_iteratorIcNS_11char_traitsIcEEEEEE', 'storage_mapping_class': 'RW', 'type': 'OBJECT'}
diff --git a/libcxx/lib/abi/powerpc64-ibm-aix.libcxxabi.v1.stable.exceptions.nonew.abilist b/libcxx/lib/abi/powerpc64-ibm-aix.libcxxabi.v1.stable.exceptions.nonew.abilist
index dd1833f139b2b..3a071b9ff52d9 100644
--- a/libcxx/lib/abi/powerpc64-ibm-aix.libcxxabi.v1.stable.exceptions.nonew.abilist
+++ b/libcxx/lib/abi/powerpc64-ibm-aix.libcxxabi.v1.stable.exceptions.nonew.abilist
@@ -1839,8 +1839,6 @@
 {'import_export': 'wEXP', 'is_defined': True, 'name': '_ZTINSt3__119basic_ostringstreamIcNS_11char_traitsIcEENS_9allocatorIcEEEE', 'storage_mapping_class': 'RW', 'type': 'OBJECT'}
 {'import_export': 'wEXP', 'is_defined': True, 'name': '_ZTINSt3__120__time_get_c_storageIcEE', 'storage_mapping_class': 'RW', 'type': 'OBJECT'}
 {'import_export': 'wEXP', 'is_defined': True, 'name': '_ZTINSt3__120__time_get_c_storageIwEE', 'storage_mapping_class': 'RW', 'type': 'OBJECT'}
-{'import_export': 'wEXP', 'is_defined': True, 'name': '_ZTINSt3__13pmr26__null_memory_resource_impE', 'storage_mapping_class': 'RW', 'type': 'OBJECT'}
-{'import_export': 'wEXP', 'is_defined': True, 'name': '_ZTINSt3__13pmr32__new_delete_memory_resource_impE', 'storage_mapping_class': 'RW', 'type': 'OBJECT'}
 {'import_export': 'wEXP', 'is_defined': True, 'name': '_ZTINSt3__17collateIcEE', 'storage_mapping_class': 'RW', 'type': 'OBJECT'}
 {'import_export': 'wEXP', 'is_defined': True, 'name': '_ZTINSt3__17collateIwEE', 'storage_mapping_class': 'RW', 'type': 'OBJECT'}
 {'import_export': 'wEXP', 'is_defined': True, 'name': '_ZTINSt3__17num_getIcNS_19istreambuf_iteratorIcNS_11char_traitsIcEEEEEE', 'storage_mapping_class': 'RW', 'type': 'OBJECT'}
@@ -1914,8 +1912,6 @@
 {'import_export': 'wEXP', 'is_defined': True, 'name': '_ZTSNSt3__119basic_ostringstreamIcNS_11char_traitsIcEENS_9allocatorIcEEEE', 'storage_mapping_class': 'RO', 'type': 'OBJECT'}
 {'import_export': 'wEXP', 'is_defined': True, 'name': '_ZTSNSt3__120__time_get_c_storageIcEE', 'storage_mapping_class': 'RO', 'type': 'OBJECT'}
 {'import_export': 'wEXP', 'is_defined': True, 'name': '_ZTSNSt3__120__time_get_c_storageIwEE', 'storage_mapping_class': 'RO', 'type': 'OBJECT'}
-{'import_export': 'wEXP', 'is_defined': True, 'name': '_ZTSNSt3__13pmr26__null_memory_resource_impE', 'storage_mapping_class': 'RO', 'type': 'OBJECT'}
-{'import_export': 'wEXP', 'is_defined': True, 'name': '_ZTSNSt3__13pmr32__new_delete_memory_resource_impE', 'storage_mapping_class': 'RO', 'type': 'OBJECT'}
 {'import_export': 'wEXP', 'is_defined': True, 'name': '_ZTSNSt3__17collateIcEE', 'storage_mapping_class': 'RO', 'type': 'OBJECT'}
 {'import_export': 'wEXP', 'is_defined': True, 'name': '_ZTSNSt3__17collateIwEE', 'storage_mapping_class': 'RO', 'type': 'OBJECT'}
 {'import_export': 'wEXP', 'is_defined': True, 'name': '_ZTSNSt3__17num_getIcNS_19istreambuf_iteratorIcNS_11char_traitsIcEEEEEE', 'storage_mapping_class': 'RO', 'type': 'OBJECT'}
@@ -1983,8 +1979,6 @@
 {'import_export': 'wEXP', 'is_defined': True, 'name': '_ZTVNSt3__118basic_stringstreamIcNS_11char_traitsIcEENS_9allocatorIcEEEE', 'storage_mapping_class': 'RW', 'type': 'OBJECT'}
 {'import_export': 'wEXP', 'is_defined': True, 'name': '_ZTVNSt3__119basic_istringstreamIcNS_11char_traitsIcEENS_9allocatorIcEEEE', 'storage_mapping_class': 'RW', 'type': 'OBJECT'}
 {'import_export': 'wEXP', 'is_defined': True, 'name': '_ZTVNSt3__119basic_ostringstreamIcNS_11char_traitsIcEENS_9allocatorIcEEEE', 'storage_mapping_class': 'RW', 'type': 'OBJECT'}
-{'import_export': 'wEXP', 'is_defined': True, 'name': '_ZTVNSt3__13pmr26__null_memory_resource_impE', 'storage_mapping_class': 'RW', 'type': 'OBJECT'}
-{'import_export': 'wEXP', 'is_defined': True, 'name': '_ZTVNSt3__13pmr32__new_delete_memory_resource_impE', 'storage_mapping_class': 'RW', 'type': 'OBJECT'}
 {'import_export': 'wEXP', 'is_defined': True, 'name': '_ZTVNSt3__17collateIcEE', 'storage_mapping_class': 'RW', 'type': 'OBJECT'}
 {'import_export': 'wEXP', 'is_defined': True, 'name': '_ZTVNSt3__17collateIwEE', 'storage_mapping_class': 'RW', 'type': 'OBJECT'}
 {'import_export': 'wEXP', 'is_defined': True, 'name': '_ZTVNSt3__17num_getIcNS_19istreambuf_iteratorIcNS_11char_traitsIcEEEEEE', 'storage_mapping_class': 'RW', 'type': 'OBJECT'}
diff --git a/libcxx/lib/abi/x86_64-apple-darwin.libcxxabi.v1.stable.exceptions.nonew.abilist b/libcxx/lib/abi/x86_64-apple-darwin.libcxxabi.v1.stable.exceptions.nonew.abilist
index 473f2c4be6b67..39e1aaefffbc6 100644
--- a/libcxx/lib/abi/x86_64-apple-darwin.libcxxabi.v1.stable.exceptions.nonew.abilist
+++ b/libcxx/lib/abi/x86_64-apple-darwin.libcxxabi.v1.stable.exceptions.nonew.abilist
@@ -2089,10 +2089,8 @@
 {'is_defined': True, 'name': '__ZTINSt3__120__time_get_c_storageIwEE', 'size': 0, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '__ZTINSt3__13pmr15memory_resourceE', 'size': 0, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '__ZTINSt3__13pmr25monotonic_buffer_resourceE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTINSt3__13pmr26__null_memory_resource_impE', 'size': 0, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '__ZTINSt3__13pmr26synchronized_pool_resourceE', 'size': 0, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '__ZTINSt3__13pmr28unsynchronized_pool_resourceE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTINSt3__13pmr32__new_delete_memory_resource_impE', 'size': 0, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '__ZTINSt3__14__fs10filesystem16filesystem_errorE', 'size': 0, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '__ZTINSt3__15ctypeIcEE', 'size': 0, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '__ZTINSt3__15ctypeIwEE', 'size': 0, 'type': 'OBJECT'}
@@ -2314,10 +2312,8 @@
 {'is_defined': True, 'name': '__ZTSNSt3__120__codecvt_utf8_utf16IwEE', 'size': 0, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '__ZTSNSt3__13pmr15memory_resourceE', 'size': 0, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '__ZTSNSt3__13pmr25monotonic_buffer_resourceE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTSNSt3__13pmr26__null_memory_resource_impE', 'size': 0, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '__ZTSNSt3__13pmr26synchronized_pool_resourceE', 'size': 0, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '__ZTSNSt3__13pmr28unsynchronized_pool_resourceE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTSNSt3__13pmr32__new_delete_memory_resource_impE', 'size': 0, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '__ZTSNSt3__14__fs10filesystem16filesystem_errorE', 'size': 0, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '__ZTSNSt3__15ctypeIcEE', 'size': 0, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '__ZTSNSt3__15ctypeIwEE', 'size': 0, 'type': 'OBJECT'}
diff --git a/libcxx/lib/abi/x86_64-linux-android21.libcxxabi.v1.stable.exceptions.nonew.abilist b/libcxx/lib/abi/x86_64-linux-android21.libcxxabi.v1.stable.exceptions.nonew.abilist
index 975d7ef20676b..5c8329c1f6650 100644
--- a/libcxx/lib/abi/x86_64-linux-android21.libcxxabi.v1.stable.exceptions.nonew.abilist
+++ b/libcxx/lib/abi/x86_64-linux-android21.libcxxabi.v1.stable.exceptions.nonew.abilist
@@ -1221,7 +1221,7 @@
 {'is_defined': True, 'name': '_ZNSt6__ndk131__arrive_barrier_algorithm_baseEPNS_24__barrier_algorithm_baseEh', 'type': 'FUNC'}
 {'is_defined': True, 'name': '_ZNSt6__ndk132__destroy_barrier_algorithm_baseEPNS_24__barrier_algorithm_baseE', 'type': 'FUNC'}
 {'is_defined': True, 'name': '_ZNSt6__ndk134__construct_barrier_algorithm_baseERl', 'type': 'FUNC'}
-{'is_defined': True, 'name': '_ZNSt6__ndk13cinE', 'size': 168, 'type': 'OBJECT'}
+{'is_defined': True, 'name': '_ZNSt6__ndk13cinE', 'size': 280, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '_ZNSt6__ndk13pmr15memory_resourceD0Ev', 'type': 'FUNC'}
 {'is_defined': True, 'name': '_ZNSt6__ndk13pmr15memory_resourceD1Ev', 'type': 'FUNC'}
 {'is_defined': True, 'name': '_ZNSt6__ndk13pmr15memory_resourceD2Ev', 'type': 'FUNC'}
@@ -1289,9 +1289,9 @@
 {'is_defined': True, 'name': '_ZNSt6__ndk14__fs10filesystem8__removeERKNS1_4pathEPNS_10error_codeE', 'type': 'FUNC'}
 {'is_defined': True, 'name': '_ZNSt6__ndk14__fs10filesystem8__renameERKNS1_4pathES4_PNS_10error_codeE', 'type': 'FUNC'}
 {'is_defined': True, 'name': '_ZNSt6__ndk14__fs10filesystem8__statusERKNS1_4pathEPNS_10error_codeE', 'type': 'FUNC'}
-{'is_defined': True, 'name': '_ZNSt6__ndk14cerrE', 'size': 160, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '_ZNSt6__ndk14clogE', 'size': 160, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '_ZNSt6__ndk14coutE', 'size': 160, 'type': 'OBJECT'}
+{'is_defined': True, 'name': '_ZNSt6__ndk14cerrE', 'size': 264, 'type': 'OBJECT'}
+{'is_defined': True, 'name': '_ZNSt6__ndk14clogE', 'size': 264, 'type': 'OBJECT'}
+{'is_defined': True, 'name': '_ZNSt6__ndk14coutE', 'size': 264, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '_ZNSt6__ndk14stodERKNS_12basic_stringIcNS_11char_traitsIcEENS_9allocatorIcEEEEPm', 'type': 'FUNC'}
 {'is_defined': True, 'name': '_ZNSt6__ndk14stodERKNS_12basic_stringIwNS_11char_traitsIwEENS_9allocatorIwEEEEPm', 'type': 'FUNC'}
 {'is_defined': True, 'name': '_ZNSt6__ndk14stofERKNS_12basic_stringIcNS_11char_traitsIcEENS_9allocatorIcEEEEPm', 'type': 'FUNC'}
@@ -1300,7 +1300,7 @@
 {'is_defined': True, 'name': '_ZNSt6__ndk14stoiERKNS_12basic_stringIwNS_11char_traitsIwEENS_9allocatorIwEEEEPmi', 'type': 'FUNC'}
 {'is_defined': True, 'name': '_ZNSt6__ndk14stolERKNS_12basic_stringIcNS_11char_traitsIcEENS_9allocatorIcEEEEPmi', 'type': 'FUNC'}
 {'is_defined': True, 'name': '_ZNSt6__ndk14stolERKNS_12basic_stringIwNS_11char_traitsIwEENS_9allocatorIwEEEEPmi', 'type': 'FUNC'}
-{'is_defined': True, 'name': '_ZNSt6__ndk14wcinE', 'size': 168, 'type': 'OBJECT'}
+{'is_defined': True, 'name': '_ZNSt6__ndk14wcinE', 'size': 280, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '_ZNSt6__ndk15alignEmmRPvRm', 'type': 'FUNC'}
 {'is_defined': True, 'name': '_ZNSt6__ndk15ctypeIcE10table_sizeE', 'size': 8, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '_ZNSt6__ndk15ctypeIcE13classic_tableEv', 'type': 'FUNC'}
@@ -1325,9 +1325,9 @@
 {'is_defined': True, 'name': '_ZNSt6__ndk15stollERKNS_12basic_stringIwNS_11char_traitsIwEENS_9allocatorIwEEEEPmi', 'type': 'FUNC'}
 {'is_defined': True, 'name': '_ZNSt6__ndk15stoulERKNS_12basic_stringIcNS_11char_traitsIcEENS_9allocatorIcEEEEPmi', 'type': 'FUNC'}
 {'is_defined': True, 'name': '_ZNSt6__ndk15stoulERKNS_12basic_stringIwNS_11char_traitsIwEENS_9allocatorIwEEEEPmi', 'type': 'FUNC'}
-{'is_defined': True, 'name': '_ZNSt6__ndk15wcerrE', 'size': 160, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '_ZNSt6__ndk15wclogE', 'size': 160, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '_ZNSt6__ndk15wcoutE', 'size': 160, 'type': 'OBJECT'}
+{'is_defined': True, 'name': '_ZNSt6__ndk15wcerrE', 'size': 264, 'type': 'OBJECT'}
+{'is_defined': True, 'name': '_ZNSt6__ndk15wclogE', 'size': 264, 'type': 'OBJECT'}
+{'is_defined': True, 'name': '_ZNSt6__ndk15wcoutE', 'size': 264, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '_ZNSt6__ndk16__clocEv', 'type': 'FUNC'}
 {'is_defined': True, 'name': '_ZNSt6__ndk16__itoa8__u32toaEjPc', 'type': 'FUNC'}
 {'is_defined': True, 'name': '_ZNSt6__ndk16__itoa8__u64toaEmPc', 'type': 'FUNC'}
@@ -1729,10 +1729,8 @@
 {'is_defined': True, 'name': '_ZTINSt6__ndk120__time_get_c_storageIwEE', 'size': 16, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '_ZTINSt6__ndk13pmr15memory_resourceE', 'size': 16, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '_ZTINSt6__ndk13pmr25monotonic_buffer_resourceE', 'size': 24, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '_ZTINSt6__ndk13pmr26__null_memory_resource_impE', 'size': 24, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '_ZTINSt6__ndk13pmr26synchronized_pool_resourceE', 'size': 24, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '_ZTINSt6__ndk13pmr28unsynchronized_pool_resourceE', 'size': 24, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '_ZTINSt6__ndk13pmr32__new_delete_memory_resource_impE', 'size': 24, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '_ZTINSt6__ndk14__fs10filesystem16filesystem_errorE', 'size': 24, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '_ZTINSt6__ndk15ctypeIcEE', 'size': 56, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '_ZTINSt6__ndk15ctypeIwEE', 'size': 56, 'type': 'OBJECT'}
@@ -1963,10 +1961,8 @@
 {'is_defined': True, 'name': '_ZTSNSt6__ndk120__time_get_c_storageIwEE', 'size': 37, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '_ZTSNSt6__ndk13pmr15memory_resourceE', 'size': 33, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '_ZTSNSt6__ndk13pmr25monotonic_buffer_resourceE', 'size': 43, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '_ZTSNSt6__ndk13pmr26__null_memory_resource_impE', 'size': 44, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '_ZTSNSt6__ndk13pmr26synchronized_pool_resourceE', 'size': 44, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '_ZTSNSt6__ndk13pmr28unsynchronized_pool_resourceE', 'size': 46, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '_ZTSNSt6__ndk13pmr32__new_delete_memory_resource_impE', 'size': 50, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '_ZTSNSt6__ndk14__fs10filesystem16filesystem_errorE', 'size': 47, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '_ZTSNSt6__ndk15ctypeIcEE', 'size': 21, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '_ZTSNSt6__ndk15ctypeIwEE', 'size': 21, 'type': 'OBJECT'}
@@ -2189,10 +2185,8 @@
 {'is_defined': True, 'name': '_ZTVNSt6__ndk120__codecvt_utf8_utf16IwEE', 'size': 96, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '_ZTVNSt6__ndk13pmr15memory_resourceE', 'size': 56, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '_ZTVNSt6__ndk13pmr25monotonic_buffer_resourceE', 'size': 56, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '_ZTVNSt6__ndk13pmr26__null_memory_resource_impE', 'size': 56, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '_ZTVNSt6__ndk13pmr26synchronized_pool_resourceE', 'size': 56, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '_ZTVNSt6__ndk13pmr28unsynchronized_pool_resourceE', 'size': 56, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '_ZTVNSt6__ndk13pmr32__new_delete_memory_resource_impE', 'size': 56, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '_ZTVNSt6__ndk14__fs10filesystem16filesystem_errorE', 'size': 40, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '_ZTVNSt6__ndk15ctypeIcEE', 'size': 104, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '_ZTVNSt6__ndk15ctypeIwEE', 'size': 136, 'type': 'OBJECT'}
diff --git a/libcxx/lib/abi/x86_64-unknown-freebsd.libcxxabi.v1.stable.exceptions.nonew.abilist b/libcxx/lib/abi/x86_64-unknown-freebsd.libcxxabi.v1.stable.exceptions.nonew.abilist
index d52678ebabfb1..3b2f99add3906 100644
--- a/libcxx/lib/abi/x86_64-unknown-freebsd.libcxxabi.v1.stable.exceptions.nonew.abilist
+++ b/libcxx/lib/abi/x86_64-unknown-freebsd.libcxxabi.v1.stable.exceptions.nonew.abilist
@@ -1236,7 +1236,7 @@
 {'is_defined': True, 'name': '_ZNSt3__131__arrive_barrier_algorithm_baseEPNS_24__barrier_algorithm_baseEh', 'type': 'FUNC'}
 {'is_defined': True, 'name': '_ZNSt3__132__destroy_barrier_algorithm_baseEPNS_24__barrier_algorithm_baseE', 'type': 'FUNC'}
 {'is_defined': True, 'name': '_ZNSt3__134__construct_barrier_algorithm_baseERl', 'type': 'FUNC'}
-{'is_defined': True, 'name': '_ZNSt3__13cinE', 'size': 168, 'type': 'OBJECT'}
+{'is_defined': True, 'name': '_ZNSt3__13cinE', 'size': 280, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '_ZNSt3__13pmr15memory_resourceD0Ev', 'type': 'FUNC'}
 {'is_defined': True, 'name': '_ZNSt3__13pmr15memory_resourceD1Ev', 'type': 'FUNC'}
 {'is_defined': True, 'name': '_ZNSt3__13pmr15memory_resourceD2Ev', 'type': 'FUNC'}
@@ -1304,9 +1304,9 @@
 {'is_defined': True, 'name': '_ZNSt3__14__fs10filesystem8__removeERKNS1_4pathEPNS_10error_codeE', 'type': 'FUNC'}
 {'is_defined': True, 'name': '_ZNSt3__14__fs10filesystem8__renameERKNS1_4pathES4_PNS_10error_codeE', 'type': 'FUNC'}
 {'is_defined': True, 'name': '_ZNSt3__14__fs10filesystem8__statusERKNS1_4pathEPNS_10error_codeE', 'type': 'FUNC'}
-{'is_defined': True, 'name': '_ZNSt3__14cerrE', 'size': 160, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '_ZNSt3__14clogE', 'size': 160, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '_ZNSt3__14coutE', 'size': 160, 'type': 'OBJECT'}
+{'is_defined': True, 'name': '_ZNSt3__14cerrE', 'size': 264, 'type': 'OBJECT'}
+{'is_defined': True, 'name': '_ZNSt3__14clogE', 'size': 264, 'type': 'OBJECT'}
+{'is_defined': True, 'name': '_ZNSt3__14coutE', 'size': 264, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '_ZNSt3__14stodERKNS_12basic_stringIcNS_11char_traitsIcEENS_9allocatorIcEEEEPm', 'type': 'FUNC'}
 {'is_defined': True, 'name': '_ZNSt3__14stodERKNS_12basic_stringIwNS_11char_traitsIwEENS_9allocatorIwEEEEPm', 'type': 'FUNC'}
 {'is_defined': True, 'name': '_ZNSt3__14stofERKNS_12basic_stringIcNS_11char_traitsIcEENS_9allocatorIcEEEEPm', 'type': 'FUNC'}
@@ -1315,7 +1315,7 @@
 {'is_defined': True, 'name': '_ZNSt3__14stoiERKNS_12basic_stringIwNS_11char_traitsIwEENS_9allocatorIwEEEEPmi', 'type': 'FUNC'}
 {'is_defined': True, 'name': '_ZNSt3__14stolERKNS_12basic_stringIcNS_11char_traitsIcEENS_9allocatorIcEEEEPmi', 'type': 'FUNC'}
 {'is_defined': True, 'name': '_ZNSt3__14stolERKNS_12basic_stringIwNS_11char_traitsIwEENS_9allocatorIwEEEEPmi', 'type': 'FUNC'}
-{'is_defined': True, 'name': '_ZNSt3__14wcinE', 'size': 168, 'type': 'OBJECT'}
+{'is_defined': True, 'name': '_ZNSt3__14wcinE', 'size': 280, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '_ZNSt3__15alignEmmRPvRm', 'type': 'FUNC'}
 {'is_defined': True, 'name': '_ZNSt3__15ctypeIcE10table_sizeE', 'size': 8, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '_ZNSt3__15ctypeIcE13classic_tableEv', 'type': 'FUNC'}
@@ -1340,9 +1340,9 @@
 {'is_defined': True, 'name': '_ZNSt3__15stollERKNS_12basic_stringIwNS_11char_traitsIwEENS_9allocatorIwEEEEPmi', 'type': 'FUNC'}
 {'is_defined': True, 'name': '_ZNSt3__15stoulERKNS_12basic_stringIcNS_11char_traitsIcEENS_9allocatorIcEEEEPmi', 'type': 'FUNC'}
 {'is_defined': True, 'name': '_ZNSt3__15stoulERKNS_12basic_stringIwNS_11char_traitsIwEENS_9allocatorIwEEEEPmi', 'type': 'FUNC'}
-{'is_defined': True, 'name': '_ZNSt3__15wcerrE', 'size': 160, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '_ZNSt3__15wclogE', 'size': 160, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '_ZNSt3__15wcoutE', 'size': 160, 'type': 'OBJECT'}
+{'is_defined': True, 'name': '_ZNSt3__15wcerrE', 'size': 264, 'type': 'OBJECT'}
+{'is_defined': True, 'name': '_ZNSt3__15wclogE', 'size': 264, 'type': 'OBJECT'}
+{'is_defined': True, 'name': '_ZNSt3__15wcoutE', 'size': 264, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '_ZNSt3__16__itoa8__u32toaEjPc', 'type': 'FUNC'}
 {'is_defined': True, 'name': '_ZNSt3__16__itoa8__u64toaEmPc', 'type': 'FUNC'}
 {'is_defined': True, 'name': '_ZNSt3__16__sortIRNS_6__lessIaaEEPaEEvT0_S5_T_', 'type': 'FUNC'}
@@ -1703,10 +1703,8 @@
 {'is_defined': True, 'name': '_ZTINSt3__120__time_get_c_storageIwEE', 'size': 16, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '_ZTINSt3__13pmr15memory_resourceE', 'size': 16, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '_ZTINSt3__13pmr25monotonic_buffer_resourceE', 'size': 24, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '_ZTINSt3__13pmr26__null_memory_resource_impE', 'size': 24, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '_ZTINSt3__13pmr26synchronized_pool_resourceE', 'size': 24, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '_ZTINSt3__13pmr28unsynchronized_pool_resourceE', 'size': 24, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '_ZTINSt3__13pmr32__new_delete_memory_resource_impE', 'size': 24, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '_ZTINSt3__14__fs10filesystem16filesystem_errorE', 'size': 24, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '_ZTINSt3__15ctypeIcEE', 'size': 56, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '_ZTINSt3__15ctypeIwEE', 'size': 56, 'type': 'OBJECT'}
@@ -1838,10 +1836,8 @@
 {'is_defined': True, 'name': '_ZTSNSt3__120__time_get_c_storageIwEE', 'size': 34, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '_ZTSNSt3__13pmr15memory_resourceE', 'size': 30, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '_ZTSNSt3__13pmr25monotonic_buffer_resourceE', 'size': 40, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '_ZTSNSt3__13pmr26__null_memory_resource_impE', 'size': 41, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '_ZTSNSt3__13pmr26synchronized_pool_resourceE', 'size': 41, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '_ZTSNSt3__13pmr28unsynchronized_pool_resourceE', 'size': 43, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '_ZTSNSt3__13pmr32__new_delete_memory_resource_impE', 'size': 47, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '_ZTSNSt3__14__fs10filesystem16filesystem_errorE', 'size': 44, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '_ZTSNSt3__15ctypeIcEE', 'size': 18, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '_ZTSNSt3__15ctypeIwEE', 'size': 18, 'type': 'OBJECT'}
@@ -1970,10 +1966,8 @@
 {'is_defined': True, 'name': '_ZTVNSt3__120__codecvt_utf8_utf16IwEE', 'size': 96, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '_ZTVNSt3__13pmr15memory_resourceE', 'size': 56, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '_ZTVNSt3__13pmr25monotonic_buffer_resourceE', 'size': 56, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '_ZTVNSt3__13pmr26__null_memory_resource_impE', 'size': 56, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '_ZTVNSt3__13pmr26synchronized_pool_resourceE', 'size': 56, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '_ZTVNSt3__13pmr28unsynchronized_pool_resourceE', 'size': 56, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '_ZTVNSt3__13pmr32__new_delete_memory_resource_impE', 'size': 56, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '_ZTVNSt3__14__fs10filesystem16filesystem_errorE', 'size': 40, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '_ZTVNSt3__15ctypeIcEE', 'size': 104, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '_ZTVNSt3__15ctypeIwEE', 'size': 136, 'type': 'OBJECT'}
diff --git a/libcxx/lib/abi/x86_64-unknown-linux-gnu.libcxxabi.v1.stable.exceptions.nonew.abilist b/libcxx/lib/abi/x86_64-unknown-linux-gnu.libcxxabi.v1.stable.exceptions.nonew.abilist
index 66a634585e919..3aa9e50a7bc3d 100644
--- a/libcxx/lib/abi/x86_64-unknown-linux-gnu.libcxxabi.v1.stable.exceptions.nonew.abilist
+++ b/libcxx/lib/abi/x86_64-unknown-linux-gnu.libcxxabi.v1.stable.exceptions.nonew.abilist
@@ -1234,7 +1234,7 @@
 {'is_defined': True, 'name': '_ZNSt3__131__arrive_barrier_algorithm_baseEPNS_24__barrier_algorithm_baseEh', 'type': 'FUNC'}
 {'is_defined': True, 'name': '_ZNSt3__132__destroy_barrier_algorithm_baseEPNS_24__barrier_algorithm_baseE', 'type': 'FUNC'}
 {'is_defined': True, 'name': '_ZNSt3__134__construct_barrier_algorithm_baseERl', 'type': 'FUNC'}
-{'is_defined': True, 'name': '_ZNSt3__13cinE', 'size': 168, 'type': 'OBJECT'}
+{'is_defined': True, 'name': '_ZNSt3__13cinE', 'size': 280, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '_ZNSt3__13pmr15memory_resourceD0Ev', 'type': 'FUNC'}
 {'is_defined': True, 'name': '_ZNSt3__13pmr15memory_resourceD1Ev', 'type': 'FUNC'}
 {'is_defined': True, 'name': '_ZNSt3__13pmr15memory_resourceD2Ev', 'type': 'FUNC'}
@@ -1302,9 +1302,9 @@
 {'is_defined': True, 'name': '_ZNSt3__14__fs10filesystem8__removeERKNS1_4pathEPNS_10error_codeE', 'type': 'FUNC'}
 {'is_defined': True, 'name': '_ZNSt3__14__fs10filesystem8__renameERKNS1_4pathES4_PNS_10error_codeE', 'type': 'FUNC'}
 {'is_defined': True, 'name': '_ZNSt3__14__fs10filesystem8__statusERKNS1_4pathEPNS_10error_codeE', 'type': 'FUNC'}
-{'is_defined': True, 'name': '_ZNSt3__14cerrE', 'size': 160, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '_ZNSt3__14clogE', 'size': 160, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '_ZNSt3__14coutE', 'size': 160, 'type': 'OBJECT'}
+{'is_defined': True, 'name': '_ZNSt3__14cerrE', 'size': 264, 'type': 'OBJECT'}
+{'is_defined': True, 'name': '_ZNSt3__14clogE', 'size': 264, 'type': 'OBJECT'}
+{'is_defined': True, 'name': '_ZNSt3__14coutE', 'size': 264, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '_ZNSt3__14stodERKNS_12basic_stringIcNS_11char_traitsIcEENS_9allocatorIcEEEEPm', 'type': 'FUNC'}
 {'is_defined': True, 'name': '_ZNSt3__14stodERKNS_12basic_stringIwNS_11char_traitsIwEENS_9allocatorIwEEEEPm', 'type': 'FUNC'}
 {'is_defined': True, 'name': '_ZNSt3__14stofERKNS_12basic_stringIcNS_11char_traitsIcEENS_9allocatorIcEEEEPm', 'type': 'FUNC'}
@@ -1313,7 +1313,7 @@
 {'is_defined': True, 'name': '_ZNSt3__14stoiERKNS_12basic_stringIwNS_11char_traitsIwEENS_9allocatorIwEEEEPmi', 'type': 'FUNC'}
 {'is_defined': True, 'name': '_ZNSt3__14stolERKNS_12basic_stringIcNS_11char_traitsIcEENS_9allocatorIcEEEEPmi', 'type': 'FUNC'}
 {'is_defined': True, 'name': '_ZNSt3__14stolERKNS_12basic_stringIwNS_11char_traitsIwEENS_9allocatorIwEEEEPmi', 'type': 'FUNC'}
-{'is_defined': True, 'name': '_ZNSt3__14wcinE', 'size': 168, 'type': 'OBJECT'}
+{'is_defined': True, 'name': '_ZNSt3__14wcinE', 'size': 280, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '_ZNSt3__15alignEmmRPvRm', 'type': 'FUNC'}
 {'is_defined': True, 'name': '_ZNSt3__15ctypeIcE10table_sizeE', 'size': 8, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '_ZNSt3__15ctypeIcE13classic_tableEv', 'type': 'FUNC'}
@@ -1340,9 +1340,9 @@
 {'is_defined': True, 'name': '_ZNSt3__15stollERKNS_12basic_stringIwNS_11char_traitsIwEENS_9allocatorIwEEEEPmi', 'type': 'FUNC'}
 {'is_defined': True, 'name': '_ZNSt3__15stoulERKNS_12basic_stringIcNS_11char_traitsIcEENS_9allocatorIcEEEEPmi', 'type': 'FUNC'}
 {'is_defined': True, 'name': '_ZNSt3__15stoulERKNS_12basic_stringIwNS_11char_traitsIwEENS_9allocatorIwEEEEPmi', 'type': 'FUNC'}
-{'is_defined': True, 'name': '_ZNSt3__15wcerrE', 'size': 160, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '_ZNSt3__15wclogE', 'size': 160, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '_ZNSt3__15wcoutE', 'size': 160, 'type': 'OBJECT'}
+{'is_defined': True, 'name': '_ZNSt3__15wcerrE', 'size': 264, 'type': 'OBJECT'}
+{'is_defined': True, 'name': '_ZNSt3__15wclogE', 'size': 264, 'type': 'OBJECT'}
+{'is_defined': True, 'name': '_ZNSt3__15wcoutE', 'size': 264, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '_ZNSt3__16__clocEv', 'type': 'FUNC'}
 {'is_defined': True, 'name': '_ZNSt3__16__itoa8__u32toaEjPc', 'type': 'FUNC'}
 {'is_defined': True, 'name': '_ZNSt3__16__itoa8__u64toaEmPc', 'type': 'FUNC'}
@@ -1704,10 +1704,8 @@
 {'is_defined': True, 'name': '_ZTINSt3__120__time_get_c_storageIwEE', 'size': 16, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '_ZTINSt3__13pmr15memory_resourceE', 'size': 16, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '_ZTINSt3__13pmr25monotonic_buffer_resourceE', 'size': 24, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '_ZTINSt3__13pmr26__null_memory_resource_impE', 'size': 24, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '_ZTINSt3__13pmr26synchronized_pool_resourceE', 'size': 24, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '_ZTINSt3__13pmr28unsynchronized_pool_resourceE', 'size': 24, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '_ZTINSt3__13pmr32__new_delete_memory_resource_impE', 'size': 24, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '_ZTINSt3__14__fs10filesystem16filesystem_errorE', 'size': 24, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '_ZTINSt3__15ctypeIcEE', 'size': 56, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '_ZTINSt3__15ctypeIwEE', 'size': 56, 'type': 'OBJECT'}
@@ -1839,10 +1837,8 @@
 {'is_defined': True, 'name': '_ZTSNSt3__120__time_get_c_storageIwEE', 'size': 34, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '_ZTSNSt3__13pmr15memory_resourceE', 'size': 30, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '_ZTSNSt3__13pmr25monotonic_buffer_resourceE', 'size': 40, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '_ZTSNSt3__13pmr26__null_memory_resource_impE', 'size': 41, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '_ZTSNSt3__13pmr26synchronized_pool_resourceE', 'size': 41, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '_ZTSNSt3__13pmr28unsynchronized_pool_resourceE', 'size': 43, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '_ZTSNSt3__13pmr32__new_delete_memory_resource_impE', 'size': 47, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '_ZTSNSt3__14__fs10filesystem16filesystem_errorE', 'size': 44, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '_ZTSNSt3__15ctypeIcEE', 'size': 18, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '_ZTSNSt3__15ctypeIwEE', 'size': 18, 'type': 'OBJECT'}
@@ -1971,10 +1967,8 @@
 {'is_defined': True, 'name': '_ZTVNSt3__120__codecvt_utf8_utf16IwEE', 'size': 96, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '_ZTVNSt3__13pmr15memory_resourceE', 'size': 56, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '_ZTVNSt3__13pmr25monotonic_buffer_resourceE', 'size': 56, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '_ZTVNSt3__13pmr26__null_memory_resource_impE', 'size': 56, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '_ZTVNSt3__13pmr26synchronized_pool_resourceE', 'size': 56, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '_ZTVNSt3__13pmr28unsynchronized_pool_resourceE', 'size': 56, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '_ZTVNSt3__13pmr32__new_delete_memory_resource_impE', 'size': 56, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '_ZTVNSt3__14__fs10filesystem16filesystem_errorE', 'size': 40, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '_ZTVNSt3__15ctypeIcEE', 'size': 104, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '_ZTVNSt3__15ctypeIwEE', 'size': 136, 'type': 'OBJECT'}
diff --git a/libcxx/lib/abi/x86_64-unknown-linux-gnu.libcxxabi.v1.stable.noexceptions.nonew.abilist b/libcxx/lib/abi/x86_64-unknown-linux-gnu.libcxxabi.v1.stable.noexceptions.nonew.abilist
index 1c247963684cb..74681c2205a34 100644
--- a/libcxx/lib/abi/x86_64-unknown-linux-gnu.libcxxabi.v1.stable.noexceptions.nonew.abilist
+++ b/libcxx/lib/abi/x86_64-unknown-linux-gnu.libcxxabi.v1.stable.noexceptions.nonew.abilist
@@ -1205,7 +1205,7 @@
 {'is_defined': True, 'name': '_ZNSt3__131__arrive_barrier_algorithm_baseEPNS_24__barrier_algorithm_baseEh', 'type': 'FUNC'}
 {'is_defined': True, 'name': '_ZNSt3__132__destroy_barrier_algorithm_baseEPNS_24__barrier_algorithm_baseE', 'type': 'FUNC'}
 {'is_defined': True, 'name': '_ZNSt3__134__construct_barrier_algorithm_baseERl', 'type': 'FUNC'}
-{'is_defined': True, 'name': '_ZNSt3__13cinE', 'size': 168, 'type': 'OBJECT'}
+{'is_defined': True, 'name': '_ZNSt3__13cinE', 'size': 280, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '_ZNSt3__13pmr15memory_resourceD0Ev', 'type': 'FUNC'}
 {'is_defined': True, 'name': '_ZNSt3__13pmr15memory_resourceD1Ev', 'type': 'FUNC'}
 {'is_defined': True, 'name': '_ZNSt3__13pmr15memory_resourceD2Ev', 'type': 'FUNC'}
@@ -1273,9 +1273,9 @@
 {'is_defined': True, 'name': '_ZNSt3__14__fs10filesystem8__removeERKNS1_4pathEPNS_10error_codeE', 'type': 'FUNC'}
 {'is_defined': True, 'name': '_ZNSt3__14__fs10filesystem8__renameERKNS1_4pathES4_PNS_10error_codeE', 'type': 'FUNC'}
 {'is_defined': True, 'name': '_ZNSt3__14__fs10filesystem8__statusERKNS1_4pathEPNS_10error_codeE', 'type': 'FUNC'}
-{'is_defined': True, 'name': '_ZNSt3__14cerrE', 'size': 160, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '_ZNSt3__14clogE', 'size': 160, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '_ZNSt3__14coutE', 'size': 160, 'type': 'OBJECT'}
+{'is_defined': True, 'name': '_ZNSt3__14cerrE', 'size': 264, 'type': 'OBJECT'}
+{'is_defined': True, 'name': '_ZNSt3__14clogE', 'size': 264, 'type': 'OBJECT'}
+{'is_defined': True, 'name': '_ZNSt3__14coutE', 'size': 264, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '_ZNSt3__14stodERKNS_12basic_stringIcNS_11char_traitsIcEENS_9allocatorIcEEEEPm', 'type': 'FUNC'}
 {'is_defined': True, 'name': '_ZNSt3__14stodERKNS_12basic_stringIwNS_11char_traitsIwEENS_9allocatorIwEEEEPm', 'type': 'FUNC'}
 {'is_defined': True, 'name': '_ZNSt3__14stofERKNS_12basic_stringIcNS_11char_traitsIcEENS_9allocatorIcEEEEPm', 'type': 'FUNC'}
@@ -1284,7 +1284,7 @@
 {'is_defined': True, 'name': '_ZNSt3__14stoiERKNS_12basic_stringIwNS_11char_traitsIwEENS_9allocatorIwEEEEPmi', 'type': 'FUNC'}
 {'is_defined': True, 'name': '_ZNSt3__14stolERKNS_12basic_stringIcNS_11char_traitsIcEENS_9allocatorIcEEEEPmi', 'type': 'FUNC'}
 {'is_defined': True, 'name': '_ZNSt3__14stolERKNS_12basic_stringIwNS_11char_traitsIwEENS_9allocatorIwEEEEPmi', 'type': 'FUNC'}
-{'is_defined': True, 'name': '_ZNSt3__14wcinE', 'size': 168, 'type': 'OBJECT'}
+{'is_defined': True, 'name': '_ZNSt3__14wcinE', 'size': 280, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '_ZNSt3__15alignEmmRPvRm', 'type': 'FUNC'}
 {'is_defined': True, 'name': '_ZNSt3__15ctypeIcE10table_sizeE', 'size': 8, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '_ZNSt3__15ctypeIcE13classic_tableEv', 'type': 'FUNC'}
@@ -1311,9 +1311,9 @@
 {'is_defined': True, 'name': '_ZNSt3__15stollERKNS_12basic_stringIwNS_11char_traitsIwEENS_9allocatorIwEEEEPmi', 'type': 'FUNC'}
 {'is_defined': True, 'name': '_ZNSt3__15stoulERKNS_12basic_stringIcNS_11char_traitsIcEENS_9allocatorIcEEEEPmi', 'type': 'FUNC'}
 {'is_defined': True, 'name': '_ZNSt3__15stoulERKNS_12basic_stringIwNS_11char_traitsIwEENS_9allocatorIwEEEEPmi', 'type': 'FUNC'}
-{'is_defined': True, 'name': '_ZNSt3__15wcerrE', 'size': 160, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '_ZNSt3__15wclogE', 'size': 160, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '_ZNSt3__15wcoutE', 'size': 160, 'type': 'OBJECT'}
+{'is_defined': True, 'name': '_ZNSt3__15wcerrE', 'size': 264, 'type': 'OBJECT'}
+{'is_defined': True, 'name': '_ZNSt3__15wclogE', 'size': 264, 'type': 'OBJECT'}
+{'is_defined': True, 'name': '_ZNSt3__15wcoutE', 'size': 264, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '_ZNSt3__16__clocEv', 'type': 'FUNC'}
 {'is_defined': True, 'name': '_ZNSt3__16__itoa8__u32toaEjPc', 'type': 'FUNC'}
 {'is_defined': True, 'name': '_ZNSt3__16__itoa8__u64toaEmPc', 'type': 'FUNC'}
@@ -1675,10 +1675,8 @@
 {'is_defined': True, 'name': '_ZTINSt3__120__time_get_c_storageIwEE', 'size': 16, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '_ZTINSt3__13pmr15memory_resourceE', 'size': 16, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '_ZTINSt3__13pmr25monotonic_buffer_resourceE', 'size': 24, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '_ZTINSt3__13pmr26__null_memory_resource_impE', 'size': 24, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '_ZTINSt3__13pmr26synchronized_pool_resourceE', 'size': 24, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '_ZTINSt3__13pmr28unsynchronized_pool_resourceE', 'size': 24, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '_ZTINSt3__13pmr32__new_delete_memory_resource_impE', 'size': 24, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '_ZTINSt3__14__fs10filesystem16filesystem_errorE', 'size': 24, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '_ZTINSt3__15ctypeIcEE', 'size': 56, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '_ZTINSt3__15ctypeIwEE', 'size': 56, 'type': 'OBJECT'}
@@ -1810,10 +1808,8 @@
 {'is_defined': True, 'name': '_ZTSNSt3__120__time_get_c_storageIwEE', 'size': 34, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '_ZTSNSt3__13pmr15memory_resourceE', 'size': 30, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '_ZTSNSt3__13pmr25monotonic_buffer_resourceE', 'size': 40, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '_ZTSNSt3__13pmr26__null_memory_resource_impE', 'size': 41, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '_ZTSNSt3__13pmr26synchronized_pool_resourceE', 'size': 41, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '_ZTSNSt3__13pmr28unsynchronized_pool_resourceE', 'size': 43, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '_ZTSNSt3__13pmr32__new_delete_memory_resource_impE', 'size': 47, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '_ZTSNSt3__14__fs10filesystem16filesystem_errorE', 'size': 44, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '_ZTSNSt3__15ctypeIcEE', 'size': 18, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '_ZTSNSt3__15ctypeIwEE', 'size': 18, 'type': 'OBJECT'}
@@ -1942,10 +1938,8 @@
 {'is_defined': True, 'name': '_ZTVNSt3__120__codecvt_utf8_utf16IwEE', 'size': 96, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '_ZTVNSt3__13pmr15memory_resourceE', 'size': 56, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '_ZTVNSt3__13pmr25monotonic_buffer_resourceE', 'size': 56, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '_ZTVNSt3__13pmr26__null_memory_resource_impE', 'size': 56, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '_ZTVNSt3__13pmr26synchronized_pool_resourceE', 'size': 56, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '_ZTVNSt3__13pmr28unsynchronized_pool_resourceE', 'size': 56, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '_ZTVNSt3__13pmr32__new_delete_memory_resource_impE', 'size': 56, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '_ZTVNSt3__14__fs10filesystem16filesystem_errorE', 'size': 40, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '_ZTVNSt3__15ctypeIcEE', 'size': 104, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '_ZTVNSt3__15ctypeIwEE', 'size': 136, 'type': 'OBJECT'}
diff --git a/libcxx/modules/std/chrono.inc b/libcxx/modules/std/chrono.inc
index 98f14f716c207..43e8da36e9044 100644
--- a/libcxx/modules/std/chrono.inc
+++ b/libcxx/modules/std/chrono.inc
@@ -97,13 +97,13 @@ export namespace std {
 
     using std::chrono::get_leap_second_info;
 
-#    if 0
     // [time.clock.tai], class tai_clock
     using std::chrono::tai_clock;
 
     using std::chrono::tai_seconds;
     using std::chrono::tai_time;
 
+#    if 0
     // [time.clock.gps], class gps_clock
     using std::chrono::gps_clock;
 
diff --git a/libcxx/modules/std/flat_map.inc b/libcxx/modules/std/flat_map.inc
index 6a86229bceaba..e9521749dc4a8 100644
--- a/libcxx/modules/std/flat_map.inc
+++ b/libcxx/modules/std/flat_map.inc
@@ -20,8 +20,6 @@ export namespace std {
   // [flat.map.erasure], erasure for flat_map
   using std::erase_if;
 
-#endif // _LIBCPP_STD_VER >= 23
-#if 0
   // [flat.multimap], class template flat_multimap
   using std::flat_multimap;
 
@@ -29,5 +27,5 @@ export namespace std {
   using std::sorted_equivalent_t;
 
   // [flat.multimap.erasure], erasure for flat_multimap
-#endif
+#endif // _LIBCPP_STD_VER >= 23
 } // namespace std
diff --git a/libcxx/src/experimental/time_zone.cpp b/libcxx/src/experimental/time_zone.cpp
index 764a89ab513c8..289164ab12036 100644
--- a/libcxx/src/experimental/time_zone.cpp
+++ b/libcxx/src/experimental/time_zone.cpp
@@ -199,7 +199,7 @@ __format(const __tz::__continuation& __continuation, const string& __letters, se
                    // active at the end. This should be determined separately.
                    return chrono::seconds{0};
                  else
-                   static_assert(sizeof(_Tp) == 0); // TODO TZDB static_assert(false); after droping clang-16 support
+                   static_assert(false);
 
                  std::__libcpp_unreachable();
                },
@@ -225,7 +225,7 @@ __format(const __tz::__continuation& __continuation, const string& __letters, se
         else if constexpr (same_as<_Tp, __tz::__constrained_weekday>)
           return __value(__year, __month);
         else
-          static_assert(sizeof(_Tp) == 0); // TODO TZDB static_assert(false); after droping clang-16 support
+          static_assert(false);
 
         std::__libcpp_unreachable();
       },
@@ -668,7 +668,7 @@ __first_rule(seconds __stdoff, const vector<__tz::__rule>& __rules) {
                __continuation_end,
                __continuation.__stdoff + __save,
                chrono::duration_cast<minutes>(__save),
-               __continuation.__format},
+               chrono::__format(__continuation, __continuation.__format, __save)},
       true};
 }
 
@@ -688,7 +688,7 @@ __get_sys_info(sys_seconds __time,
         else if constexpr (same_as<_Tp, __tz::__save>)
           return chrono::__get_sys_info_basic(__time, __continuation_begin, __continuation, __value.__time);
         else
-          static_assert(sizeof(_Tp) == 0); // TODO TZDB static_assert(false); after droping clang-16 support
+          static_assert(false);
 
         std::__libcpp_unreachable();
       },
diff --git a/libcxx/src/include/overridable_function.h b/libcxx/src/include/overridable_function.h
index 7372e347831bb..6c70f6242ddd6 100644
--- a/libcxx/src/include/overridable_function.h
+++ b/libcxx/src/include/overridable_function.h
@@ -29,81 +29,106 @@
 // This is a low-level utility which does not work on all platforms, since it needs
 // to make assumptions about the object file format in use. Furthermore, it requires
 // the "base definition" of the function (the one we want to check whether it has been
-// overridden) to be defined using the _LIBCPP_OVERRIDABLE_FUNCTION macro.
+// overridden) to be annotated with the _LIBCPP_MAKE_OVERRIDABLE_FUNCTION_DETECTABLE macro.
 //
 // This currently works with Mach-O files (used on Darwin) and with ELF files (used on Linux
 // and others). On platforms where we know how to implement this detection, the macro
 // _LIBCPP_CAN_DETECT_OVERRIDDEN_FUNCTION is defined to 1, and it is defined to 0 on
-// other platforms. The _LIBCPP_OVERRIDABLE_FUNCTION macro expands to regular function
-// definition on unsupported platforms so that it can be used to decorate functions
-// regardless of whether detection is actually supported.
+// other platforms. The _LIBCPP_MAKE_OVERRIDABLE_FUNCTION_DETECTABLE macro is defined to
+// nothing on unsupported platforms so that it can be used to decorate functions regardless
+// of whether detection is actually supported.
 //
 // How does this work?
 // -------------------
 //
 // Let's say we want to check whether a weak function `f` has been overridden by the user.
-// The general mechanism works by defining a symbol `f_impl__` and a weak alias `f` via the
-// _LIBCPP_OVERRIDABLE_FUNCTION macro.
+// The general mechanism works by placing `f`'s definition (in the libc++ built library)
+// inside a special section, which we do using the `__section__` attribute via the
+// _LIBCPP_MAKE_OVERRIDABLE_FUNCTION_DETECTABLE macro.
 //
 // Then, when comes the time to check whether the function has been overridden, we take
-// the address of the function `f` and we check whether it is different from `f_impl__`.
-// If so it means the function was overriden by the user.
+// the address of the function and we check whether it falls inside the special function
+// we created. This can be done by finding pointers to the start and the end of the section
+// (which is done differently for ELF and Mach-O), and then checking whether `f` falls
+// within those bounds. If it falls within those bounds, then `f` is still inside the
+// special section and so it is the version we defined in the libc++ built library, i.e.
+// it was not overridden. Otherwise, it was overridden by the user because it falls
+// outside of the section.
 //
 // Important note
 // --------------
 //
-// This mechanism should never be used outside of the libc++ built library. Functions defined
-// with this macro must be defined at global scope.
+// This mechanism should never be used outside of the libc++ built library. In particular,
+// attempting to use this within the libc++ headers will not work at all because we don't
+// want to be defining special sections inside user's executables which use our headers.
 //
 
 #if defined(_LIBCPP_OBJECT_FORMAT_MACHO)
 
-_LIBCPP_BEGIN_NAMESPACE_STD
-
-template <auto _Func>
-_LIBCPP_HIDE_FROM_ABI constexpr bool __is_function_overridden();
+#  define _LIBCPP_CAN_DETECT_OVERRIDDEN_FUNCTION 1
+#  define _LIBCPP_MAKE_OVERRIDABLE_FUNCTION_DETECTABLE                                                                 \
+    __attribute__((__section__("__TEXT,__lcxx_override,regular,pure_instructions")))
 
+_LIBCPP_BEGIN_NAMESPACE_STD
+template <class _Ret, class... _Args>
+_LIBCPP_HIDE_FROM_ABI bool __is_function_overridden(_Ret (*__fptr)(_Args...)) noexcept {
+  // Declare two dummy bytes and give them these special `__asm` values. These values are
+  // defined by the linker, which means that referring to `&__lcxx_override_start` will
+  // effectively refer to the address where the section starts (and same for the end).
+  extern char __lcxx_override_start __asm("section$start$__TEXT$__lcxx_override");
+  extern char __lcxx_override_end __asm("section$end$__TEXT$__lcxx_override");
+
+  // Now get a uintptr_t out of these locations, and out of the function pointer.
+  uintptr_t __start = reinterpret_cast<uintptr_t>(&__lcxx_override_start);
+  uintptr_t __end   = reinterpret_cast<uintptr_t>(&__lcxx_override_end);
+  uintptr_t __ptr   = reinterpret_cast<uintptr_t>(__fptr);
+
+#  if __has_feature(ptrauth_calls)
+  // We must pass a void* to ptrauth_strip since it only accepts a pointer type. Also, in particular,
+  // we must NOT pass a function pointer, otherwise we will strip the function pointer, and then attempt
+  // to authenticate and re-sign it when casting it to a uintptr_t again, which will fail because we just
+  // stripped the function pointer. See rdar://122927845.
+  __ptr = reinterpret_cast<uintptr_t>(ptrauth_strip(reinterpret_cast<void*>(__ptr), ptrauth_key_function_pointer));
+#  endif
+
+  // Finally, the function was overridden if it falls outside of the section's bounds.
+  return __ptr < __start || __ptr > __end;
+}
 _LIBCPP_END_NAMESPACE_STD
 
-#  define _LIBCPP_CAN_DETECT_OVERRIDDEN_FUNCTION 1
-#  define _LIBCPP_OVERRIDABLE_FUNCTION(symbol, type, name, arglist)                                                    \
-    static __attribute__((used)) type symbol##_impl__ arglist __asm__("_" _LIBCPP_TOSTRING(symbol));                   \
-    __asm__(".globl _" _LIBCPP_TOSTRING(symbol));                                                                      \
-    __asm__(".weak_definition _" _LIBCPP_TOSTRING(symbol));                                                            \
-    extern __typeof(symbol##_impl__) name __attribute__((weak_import));                                                \
-    _LIBCPP_BEGIN_NAMESPACE_STD                                                                                        \
-    template <>                                                                                                        \
-    inline bool __is_function_overridden<static_cast<type(*) arglist>(name)>() {                                       \
-      return static_cast<type(*) arglist>(name) != symbol##_impl__;                                                    \
-    }                                                                                                                  \
-    _LIBCPP_END_NAMESPACE_STD                                                                                          \
-    static type symbol##_impl__ arglist
-
-#elif defined(_LIBCPP_OBJECT_FORMAT_ELF)
+// The NVPTX linker cannot create '__start/__stop' sections.
+#elif defined(_LIBCPP_OBJECT_FORMAT_ELF) && !defined(__NVPTX__)
 
-_LIBCPP_BEGIN_NAMESPACE_STD
+#  define _LIBCPP_CAN_DETECT_OVERRIDDEN_FUNCTION 1
+#  define _LIBCPP_MAKE_OVERRIDABLE_FUNCTION_DETECTABLE __attribute__((__section__("__lcxx_override")))
 
-template <auto _Func>
-_LIBCPP_HIDE_FROM_ABI constexpr bool __is_function_overridden();
+// This is very similar to what we do for Mach-O above. The ELF linker will implicitly define
+// variables with those names corresponding to the start and the end of the section.
+//
+// See https://stackoverflow.com/questions/16552710/how-do-you-get-the-start-and-end-addresses-of-a-custom-elf-section
+extern char __start___lcxx_override;
+extern char __stop___lcxx_override;
 
+_LIBCPP_BEGIN_NAMESPACE_STD
+template <class _Ret, class... _Args>
+_LIBCPP_HIDE_FROM_ABI bool __is_function_overridden(_Ret (*__fptr)(_Args...)) noexcept {
+  uintptr_t __start = reinterpret_cast<uintptr_t>(&__start___lcxx_override);
+  uintptr_t __end   = reinterpret_cast<uintptr_t>(&__stop___lcxx_override);
+  uintptr_t __ptr   = reinterpret_cast<uintptr_t>(__fptr);
+
+#  if __has_feature(ptrauth_calls)
+  // We must pass a void* to ptrauth_strip since it only accepts a pointer type. See full explanation above.
+  __ptr = reinterpret_cast<uintptr_t>(ptrauth_strip(reinterpret_cast<void*>(__ptr), ptrauth_key_function_pointer));
+#  endif
+
+  return __ptr < __start || __ptr > __end;
+}
 _LIBCPP_END_NAMESPACE_STD
 
-#  define _LIBCPP_CAN_DETECT_OVERRIDDEN_FUNCTION 1
-#  define _LIBCPP_OVERRIDABLE_FUNCTION(symbol, type, name, arglist)                                                    \
-    static type symbol##_impl__ arglist __asm__(_LIBCPP_TOSTRING(symbol##_impl__));                                    \
-    [[gnu::weak, gnu::alias(_LIBCPP_TOSTRING(symbol##_impl__))]] type name arglist;                                    \
-    _LIBCPP_BEGIN_NAMESPACE_STD                                                                                        \
-    template <>                                                                                                        \
-    inline bool __is_function_overridden<static_cast<type(*) arglist>(name)>() {                                       \
-      return static_cast<type(*) arglist>(name) != symbol##_impl__;                                                    \
-    }                                                                                                                  \
-    _LIBCPP_END_NAMESPACE_STD                                                                                          \
-    static type symbol##_impl__ arglist
-
 #else
 
 #  define _LIBCPP_CAN_DETECT_OVERRIDDEN_FUNCTION 0
-#  define _LIBCPP_OVERRIDABLE_FUNCTION(symbol, type, name, arglist) _LIBCPP_WEAK type name arglist
+#  define _LIBCPP_MAKE_OVERRIDABLE_FUNCTION_DETECTABLE /* nothing */
 
 #endif
 
diff --git a/libcxx/src/iostream.cpp b/libcxx/src/iostream.cpp
index 6db02d5603794..d91f9f0c04826 100644
--- a/libcxx/src/iostream.cpp
+++ b/libcxx/src/iostream.cpp
@@ -7,90 +7,58 @@
 //===----------------------------------------------------------------------===//
 
 #include "std_stream.h"
-#include <__locale>
-#include <new>
-#include <string>
 
-#define _str(s) #s
-#define str(s) _str(s)
-#define _LIBCPP_ABI_NAMESPACE_STR str(_LIBCPP_ABI_NAMESPACE)
+#include <__memory/construct_at.h>
+#include <__ostream/basic_ostream.h>
+#include <istream>
 
-_LIBCPP_BEGIN_NAMESPACE_STD
-
-alignas(istream) _LIBCPP_EXPORTED_FROM_ABI char cin[sizeof(istream)]
-#if defined(_LIBCPP_ABI_MICROSOFT) && defined(__clang__)
-    __asm__("?cin@" _LIBCPP_ABI_NAMESPACE_STR "@std@@3V?$basic_istream@DU?$char_traits@D@" _LIBCPP_ABI_NAMESPACE_STR
-            "@std@@@12@A")
-#endif
-        ;
-alignas(__stdinbuf<char>) static char __cin[sizeof(__stdinbuf<char>)];
-static mbstate_t mb_cin;
-
-#if _LIBCPP_HAS_WIDE_CHARACTERS
-alignas(wistream) _LIBCPP_EXPORTED_FROM_ABI char wcin[sizeof(wistream)]
-#  if defined(_LIBCPP_ABI_MICROSOFT) && defined(__clang__)
-    __asm__("?wcin@" _LIBCPP_ABI_NAMESPACE_STR "@std@@3V?$basic_istream@_WU?$char_traits@_W@" _LIBCPP_ABI_NAMESPACE_STR
-            "@std@@@12@A")
-#  endif
-        ;
-alignas(__stdinbuf<wchar_t>) static char __wcin[sizeof(__stdinbuf<wchar_t>)];
-static mbstate_t mb_wcin;
-#endif // _LIBCPP_HAS_WIDE_CHARACTERS
+#define ABI_NAMESPACE_STR _LIBCPP_TOSTRING(_LIBCPP_ABI_NAMESPACE)
 
-alignas(ostream) _LIBCPP_EXPORTED_FROM_ABI char cout[sizeof(ostream)]
-#if defined(_LIBCPP_ABI_MICROSOFT) && defined(__clang__)
-    __asm__("?cout@" _LIBCPP_ABI_NAMESPACE_STR "@std@@3V?$basic_ostream@DU?$char_traits@D@" _LIBCPP_ABI_NAMESPACE_STR
-            "@std@@@12@A")
-#endif
-        ;
-alignas(__stdoutbuf<char>) static char __cout[sizeof(__stdoutbuf<char>)];
-static mbstate_t mb_cout;
+_LIBCPP_BEGIN_NAMESPACE_STD
 
-#if _LIBCPP_HAS_WIDE_CHARACTERS
-alignas(wostream) _LIBCPP_EXPORTED_FROM_ABI char wcout[sizeof(wostream)]
-#  if defined(_LIBCPP_ABI_MICROSOFT) && defined(__clang__)
-    __asm__("?wcout@" _LIBCPP_ABI_NAMESPACE_STR "@std@@3V?$basic_ostream@_WU?$char_traits@_W@" _LIBCPP_ABI_NAMESPACE_STR
-            "@std@@@12@A")
-#  endif
-        ;
-alignas(__stdoutbuf<wchar_t>) static char __wcout[sizeof(__stdoutbuf<wchar_t>)];
-static mbstate_t mb_wcout;
-#endif // _LIBCPP_HAS_WIDE_CHARACTERS
+template <class StreamT, class BufferT>
+union stream_data {
+  stream_data() {}
+  ~stream_data() {}
+  struct {
+    // The stream has to be the first element, since that's referenced by the stream declarations in <iostream>
+    StreamT stream;
+    BufferT buffer;
+    mbstate_t mb;
+  };
+
+  void init(FILE* stdstream) {
+    mb = {};
+    std::construct_at(&buffer, stdstream, &mb);
+    std::construct_at(&stream, &buffer);
+  }
+};
 
-alignas(ostream) _LIBCPP_EXPORTED_FROM_ABI char cerr[sizeof(ostream)]
-#if defined(_LIBCPP_ABI_MICROSOFT) && defined(__clang__)
-    __asm__("?cerr@" _LIBCPP_ABI_NAMESPACE_STR "@std@@3V?$basic_ostream@DU?$char_traits@D@" _LIBCPP_ABI_NAMESPACE_STR
-            "@std@@@12@A")
+#define CHAR_MANGLING_char "D"
+#define CHAR_MANGLING_wchar_t "_W"
+#define CHAR_MANGLING(CharT) CHAR_MANGLING_##CharT
+
+#ifdef _LIBCPP_ABI_MICROSOFT
+#  define STREAM(StreamT, BufferT, CharT, var)                                                                         \
+    stream_data<StreamT<CharT>, BufferT<CharT>> var __asm__(                                                           \
+        "?" #var "@" ABI_NAMESPACE_STR "@std@@3V?$" #StreamT                                                           \
+        "@" CHAR_MANGLING(CharT) "U?$char_traits@" CHAR_MANGLING(CharT) "@" ABI_NAMESPACE_STR "@std@@@12@A")
+#else
+#  define STREAM(StreamT, BufferT, CharT, var) stream_data<StreamT<CharT>, BufferT<CharT>> var
 #endif
-        ;
-alignas(__stdoutbuf<char>) static char __cerr[sizeof(__stdoutbuf<char>)];
-static mbstate_t mb_cerr;
-
-#if _LIBCPP_HAS_WIDE_CHARACTERS
-alignas(wostream) _LIBCPP_EXPORTED_FROM_ABI char wcerr[sizeof(wostream)]
-#  if defined(_LIBCPP_ABI_MICROSOFT) && defined(__clang__)
-    __asm__("?wcerr@" _LIBCPP_ABI_NAMESPACE_STR "@std@@3V?$basic_ostream@_WU?$char_traits@_W@" _LIBCPP_ABI_NAMESPACE_STR
-            "@std@@@12@A")
-#  endif
-        ;
-alignas(__stdoutbuf<wchar_t>) static char __wcerr[sizeof(__stdoutbuf<wchar_t>)];
-static mbstate_t mb_wcerr;
-#endif // _LIBCPP_HAS_WIDE_CHARACTERS
 
-alignas(ostream) _LIBCPP_EXPORTED_FROM_ABI char clog[sizeof(ostream)]
-#if defined(_LIBCPP_ABI_MICROSOFT) && defined(__clang__)
-    __asm__("?clog@" _LIBCPP_ABI_NAMESPACE_STR "@std@@3V?$basic_ostream@DU?$char_traits@D@" _LIBCPP_ABI_NAMESPACE_STR
-            "@std@@@12@A")
-#endif
-        ;
+// These definitions and the declarations in <iostream> technically cause ODR violations, since they have different
+// types (stream_data and {i,o}stream respectively). This means that <iostream> should never be included in this TU.
 
+_LIBCPP_EXPORTED_FROM_ABI STREAM(basic_istream, __stdinbuf, char, cin);
+_LIBCPP_EXPORTED_FROM_ABI STREAM(basic_ostream, __stdoutbuf, char, cout);
+_LIBCPP_EXPORTED_FROM_ABI STREAM(basic_ostream, __stdoutbuf, char, cerr);
+_LIBCPP_EXPORTED_FROM_ABI STREAM(basic_ostream, __stdoutbuf, char, clog);
 #if _LIBCPP_HAS_WIDE_CHARACTERS
-alignas(wostream) _LIBCPP_EXPORTED_FROM_ABI char wclog[sizeof(wostream)]
-#  if defined(_LIBCPP_ABI_MICROSOFT) && defined(__clang__)
-    __asm__("?wclog@" _LIBCPP_ABI_NAMESPACE_STR "@std@@3V?$basic_ostream@_WU?$char_traits@_W@" _LIBCPP_ABI_NAMESPACE_STR
-            "@std@@@12@A")
-#  endif
-        ;
+_LIBCPP_EXPORTED_FROM_ABI STREAM(basic_istream, __stdinbuf, wchar_t, wcin);
+_LIBCPP_EXPORTED_FROM_ABI STREAM(basic_ostream, __stdoutbuf, wchar_t, wcout);
+_LIBCPP_EXPORTED_FROM_ABI STREAM(basic_ostream, __stdoutbuf, wchar_t, wcerr);
+_LIBCPP_EXPORTED_FROM_ABI STREAM(basic_ostream, __stdoutbuf, wchar_t, wclog);
 #endif // _LIBCPP_HAS_WIDE_CHARACTERS
 
 // Pretend we're inside a system header so the compiler doesn't flag the use of the init_priority
@@ -103,7 +71,7 @@ alignas(wostream) _LIBCPP_EXPORTED_FROM_ABI char wclog[sizeof(wostream)]
 static void force_locale_initialization() {
 #if defined(_LIBCPP_MSVCRT_LIKE)
   static bool once = []() {
-    auto loc = __locale::__newlocale(LC_ALL_MASK, "C", 0);
+    auto loc = __locale::__newlocale(_LIBCPP_ALL_MASK, "C", 0);
     {
       __locale::__locale_guard g(loc); // forces initialization of locale TLS
       ((void)g);
@@ -124,37 +92,34 @@ class DoIOSInit {
 DoIOSInit::DoIOSInit() {
   force_locale_initialization();
 
-  istream* cin_ptr  = ::new (cin) istream(::new (__cin) __stdinbuf<char>(stdin, &mb_cin));
-  ostream* cout_ptr = ::new (cout) ostream(::new (__cout) __stdoutbuf<char>(stdout, &mb_cout));
-  ostream* cerr_ptr = ::new (cerr) ostream(::new (__cerr) __stdoutbuf<char>(stderr, &mb_cerr));
-  ::new (clog) ostream(cerr_ptr->rdbuf());
-  cin_ptr->tie(cout_ptr);
-  std::unitbuf(*cerr_ptr);
-  cerr_ptr->tie(cout_ptr);
+  cin.init(stdin);
+  cout.init(stdout);
+  cerr.init(stderr);
+  clog.init(stderr);
+
+  cin.stream.tie(&cout.stream);
+  std::unitbuf(cerr.stream);
+  cerr.stream.tie(&cout.stream);
 
 #if _LIBCPP_HAS_WIDE_CHARACTERS
-  wistream* wcin_ptr  = ::new (wcin) wistream(::new (__wcin) __stdinbuf<wchar_t>(stdin, &mb_wcin));
-  wostream* wcout_ptr = ::new (wcout) wostream(::new (__wcout) __stdoutbuf<wchar_t>(stdout, &mb_wcout));
-  wostream* wcerr_ptr = ::new (wcerr) wostream(::new (__wcerr) __stdoutbuf<wchar_t>(stderr, &mb_wcerr));
-  ::new (wclog) wostream(wcerr_ptr->rdbuf());
-
-  wcin_ptr->tie(wcout_ptr);
-  std::unitbuf(*wcerr_ptr);
-  wcerr_ptr->tie(wcout_ptr);
+  wcin.init(stdin);
+  wcout.init(stdout);
+  wcerr.init(stderr);
+  wclog.init(stderr);
+
+  wcin.stream.tie(&wcout.stream);
+  std::unitbuf(wcerr.stream);
+  wcerr.stream.tie(&wcout.stream);
 #endif
 }
 
 DoIOSInit::~DoIOSInit() {
-  ostream* cout_ptr = reinterpret_cast<ostream*>(cout);
-  cout_ptr->flush();
-  ostream* clog_ptr = reinterpret_cast<ostream*>(clog);
-  clog_ptr->flush();
+  cout.stream.flush();
+  clog.stream.flush();
 
 #if _LIBCPP_HAS_WIDE_CHARACTERS
-  wostream* wcout_ptr = reinterpret_cast<wostream*>(wcout);
-  wcout_ptr->flush();
-  wostream* wclog_ptr = reinterpret_cast<wostream*>(wclog);
-  wclog_ptr->flush();
+  wcout.stream.flush();
+  wclog.stream.flush();
 #endif
 }
 
diff --git a/libcxx/src/locale.cpp b/libcxx/src/locale.cpp
index fb67a729cd0f2..b5908bf8be725 100644
--- a/libcxx/src/locale.cpp
+++ b/libcxx/src/locale.cpp
@@ -51,7 +51,7 @@ _LIBCPP_PUSH_MACROS
 _LIBCPP_BEGIN_NAMESPACE_STD
 
 struct __libcpp_unique_locale {
-  __libcpp_unique_locale(const char* nm) : __loc_(__locale::__newlocale(LC_ALL_MASK, nm, 0)) {}
+  __libcpp_unique_locale(const char* nm) : __loc_(__locale::__newlocale(_LIBCPP_ALL_MASK, nm, 0)) {}
 
   ~__libcpp_unique_locale() {
     if (__loc_)
@@ -74,7 +74,7 @@ __locale::__locale_t __cloc() {
   // In theory this could create a race condition. In practice
   // the race condition is non-fatal since it will just create
   // a little resource leak. Better approach would be appreciated.
-  static __locale::__locale_t result = __locale::__newlocale(LC_ALL_MASK, "C", 0);
+  static __locale::__locale_t result = __locale::__newlocale(_LIBCPP_ALL_MASK, "C", 0);
   return result;
 }
 #endif // __cloc_defined
@@ -570,7 +570,7 @@ locale locale::global(const locale& loc) {
   locale r  = g;
   g         = loc;
   if (g.name() != "*")
-    setlocale(LC_ALL, g.name().c_str());
+    __locale::__setlocale(_LIBCPP_LC_ALL, g.name().c_str());
   return r;
 }
 
@@ -600,7 +600,7 @@ long locale::id::__get() {
 // template <> class collate_byname<char>
 
 collate_byname<char>::collate_byname(const char* n, size_t refs)
-    : collate<char>(refs), __l_(__locale::__newlocale(LC_ALL_MASK, n, 0)) {
+    : collate<char>(refs), __l_(__locale::__newlocale(_LIBCPP_ALL_MASK, n, 0)) {
   if (__l_ == 0)
     __throw_runtime_error(
         ("collate_byname<char>::collate_byname"
@@ -610,7 +610,7 @@ collate_byname<char>::collate_byname(const char* n, size_t refs)
 }
 
 collate_byname<char>::collate_byname(const string& name, size_t refs)
-    : collate<char>(refs), __l_(__locale::__newlocale(LC_ALL_MASK, name.c_str(), 0)) {
+    : collate<char>(refs), __l_(__locale::__newlocale(_LIBCPP_ALL_MASK, name.c_str(), 0)) {
   if (__l_ == 0)
     __throw_runtime_error(
         ("collate_byname<char>::collate_byname"
@@ -644,7 +644,7 @@ collate_byname<char>::string_type collate_byname<char>::do_transform(const char_
 
 #if _LIBCPP_HAS_WIDE_CHARACTERS
 collate_byname<wchar_t>::collate_byname(const char* n, size_t refs)
-    : collate<wchar_t>(refs), __l_(__locale::__newlocale(LC_ALL_MASK, n, 0)) {
+    : collate<wchar_t>(refs), __l_(__locale::__newlocale(_LIBCPP_ALL_MASK, n, 0)) {
   if (__l_ == 0)
     __throw_runtime_error(
         ("collate_byname<wchar_t>::collate_byname(size_t refs)"
@@ -654,7 +654,7 @@ collate_byname<wchar_t>::collate_byname(const char* n, size_t refs)
 }
 
 collate_byname<wchar_t>::collate_byname(const string& name, size_t refs)
-    : collate<wchar_t>(refs), __l_(__locale::__newlocale(LC_ALL_MASK, name.c_str(), 0)) {
+    : collate<wchar_t>(refs), __l_(__locale::__newlocale(_LIBCPP_ALL_MASK, name.c_str(), 0)) {
   if (__l_ == 0)
     __throw_runtime_error(
         ("collate_byname<wchar_t>::collate_byname(size_t refs)"
@@ -1047,7 +1047,7 @@ const unsigned short* ctype<char>::__classic_upper_table() _NOEXCEPT {
 // template <> class ctype_byname<char>
 
 ctype_byname<char>::ctype_byname(const char* name, size_t refs)
-    : ctype<char>(0, false, refs), __l_(__locale::__newlocale(LC_ALL_MASK, name, 0)) {
+    : ctype<char>(0, false, refs), __l_(__locale::__newlocale(_LIBCPP_ALL_MASK, name, 0)) {
   if (__l_ == 0)
     __throw_runtime_error(
         ("ctype_byname<char>::ctype_byname"
@@ -1057,7 +1057,7 @@ ctype_byname<char>::ctype_byname(const char* name, size_t refs)
 }
 
 ctype_byname<char>::ctype_byname(const string& name, size_t refs)
-    : ctype<char>(0, false, refs), __l_(__locale::__newlocale(LC_ALL_MASK, name.c_str(), 0)) {
+    : ctype<char>(0, false, refs), __l_(__locale::__newlocale(_LIBCPP_ALL_MASK, name.c_str(), 0)) {
   if (__l_ == 0)
     __throw_runtime_error(
         ("ctype_byname<char>::ctype_byname"
@@ -1092,7 +1092,7 @@ const char* ctype_byname<char>::do_tolower(char_type* low, const char_type* high
 
 #if _LIBCPP_HAS_WIDE_CHARACTERS
 ctype_byname<wchar_t>::ctype_byname(const char* name, size_t refs)
-    : ctype<wchar_t>(refs), __l_(__locale::__newlocale(LC_ALL_MASK, name, 0)) {
+    : ctype<wchar_t>(refs), __l_(__locale::__newlocale(_LIBCPP_ALL_MASK, name, 0)) {
   if (__l_ == 0)
     __throw_runtime_error(
         ("ctype_byname<wchar_t>::ctype_byname"
@@ -1102,7 +1102,7 @@ ctype_byname<wchar_t>::ctype_byname(const char* name, size_t refs)
 }
 
 ctype_byname<wchar_t>::ctype_byname(const string& name, size_t refs)
-    : ctype<wchar_t>(refs), __l_(__locale::__newlocale(LC_ALL_MASK, name.c_str(), 0)) {
+    : ctype<wchar_t>(refs), __l_(__locale::__newlocale(_LIBCPP_ALL_MASK, name.c_str(), 0)) {
   if (__l_ == 0)
     __throw_runtime_error(
         ("ctype_byname<wchar_t>::ctype_byname"
@@ -1342,7 +1342,7 @@ constinit locale::id codecvt<wchar_t, char, mbstate_t>::id;
 codecvt<wchar_t, char, mbstate_t>::codecvt(size_t refs) : locale::facet(refs), __l_(_LIBCPP_GET_C_LOCALE) {}
 
 codecvt<wchar_t, char, mbstate_t>::codecvt(const char* nm, size_t refs)
-    : locale::facet(refs), __l_(__locale::__newlocale(LC_ALL_MASK, nm, 0)) {
+    : locale::facet(refs), __l_(__locale::__newlocale(_LIBCPP_ALL_MASK, nm, 0)) {
   if (__l_ == 0)
     __throw_runtime_error(
         ("codecvt_byname<wchar_t, char, mbstate_t>::codecvt_byname"
@@ -4067,7 +4067,7 @@ void numpunct_byname<char>::__init(const char* nm) {
            string(nm))
               .c_str());
 
-    lconv* lc = __locale::__localeconv(loc.get());
+    __locale::__lconv_t* lc = __locale::__localeconv(loc.get());
     if (!checked_string_to_char_convert(__decimal_point_, lc->decimal_point, loc.get()))
       __decimal_point_ = base::do_decimal_point();
     if (!checked_string_to_char_convert(__thousands_sep_, lc->thousands_sep, loc.get()))
@@ -4098,7 +4098,7 @@ void numpunct_byname<wchar_t>::__init(const char* nm) {
            string(nm))
               .c_str());
 
-    lconv* lc = __locale::__localeconv(loc.get());
+    __locale::__lconv_t* lc = __locale::__localeconv(loc.get());
     checked_string_to_wchar_convert(__decimal_point_, lc->decimal_point, loc.get());
     checked_string_to_wchar_convert(__thousands_sep_, lc->thousands_sep, loc.get());
     __grouping_ = lc->grouping;
@@ -4442,12 +4442,12 @@ const wstring& __time_get_c_storage<wchar_t>::__r() const {
 
 // time_get_byname
 
-__time_get::__time_get(const char* nm) : __loc_(__locale::__newlocale(LC_ALL_MASK, nm, 0)) {
+__time_get::__time_get(const char* nm) : __loc_(__locale::__newlocale(_LIBCPP_ALL_MASK, nm, 0)) {
   if (__loc_ == 0)
     __throw_runtime_error(("time_get_byname failed to construct for " + string(nm)).c_str());
 }
 
-__time_get::__time_get(const string& nm) : __loc_(__locale::__newlocale(LC_ALL_MASK, nm.c_str(), 0)) {
+__time_get::__time_get(const string& nm) : __loc_(__locale::__newlocale(_LIBCPP_ALL_MASK, nm.c_str(), 0)) {
   if (__loc_ == 0)
     __throw_runtime_error(("time_get_byname failed to construct for " + nm).c_str());
 }
@@ -5027,12 +5027,12 @@ time_base::dateorder __time_get_storage<wchar_t>::__do_date_order() const {
 
 // time_put
 
-__time_put::__time_put(const char* nm) : __loc_(__locale::__newlocale(LC_ALL_MASK, nm, 0)) {
+__time_put::__time_put(const char* nm) : __loc_(__locale::__newlocale(_LIBCPP_ALL_MASK, nm, 0)) {
   if (__loc_ == 0)
     __throw_runtime_error(("time_put_byname failed to construct for " + string(nm)).c_str());
 }
 
-__time_put::__time_put(const string& nm) : __loc_(__locale::__newlocale(LC_ALL_MASK, nm.c_str(), 0)) {
+__time_put::__time_put(const string& nm) : __loc_(__locale::__newlocale(_LIBCPP_ALL_MASK, nm.c_str(), 0)) {
   if (__loc_ == 0)
     __throw_runtime_error(("time_put_byname failed to construct for " + nm).c_str());
 }
@@ -5433,7 +5433,7 @@ void moneypunct_byname<char, false>::init(const char* nm) {
   if (!loc)
     __throw_runtime_error(("moneypunct_byname failed to construct for " + string(nm)).c_str());
 
-  lconv* lc = __locale::__localeconv(loc.get());
+  __locale::__lconv_t* lc = __locale::__localeconv(loc.get());
   if (!checked_string_to_char_convert(__decimal_point_, lc->mon_decimal_point, loc.get()))
     __decimal_point_ = base::do_decimal_point();
   if (!checked_string_to_char_convert(__thousands_sep_, lc->mon_thousands_sep, loc.get()))
@@ -5468,7 +5468,7 @@ void moneypunct_byname<char, true>::init(const char* nm) {
   if (!loc)
     __throw_runtime_error(("moneypunct_byname failed to construct for " + string(nm)).c_str());
 
-  lconv* lc = __locale::__localeconv(loc.get());
+  __locale::__lconv_t* lc = __locale::__localeconv(loc.get());
   if (!checked_string_to_char_convert(__decimal_point_, lc->mon_decimal_point, loc.get()))
     __decimal_point_ = base::do_decimal_point();
   if (!checked_string_to_char_convert(__thousands_sep_, lc->mon_thousands_sep, loc.get()))
@@ -5523,7 +5523,7 @@ void moneypunct_byname<wchar_t, false>::init(const char* nm) {
   __libcpp_unique_locale loc(nm);
   if (!loc)
     __throw_runtime_error(("moneypunct_byname failed to construct for " + string(nm)).c_str());
-  lconv* lc = __locale::__localeconv(loc.get());
+  __locale::__lconv_t* lc = __locale::__localeconv(loc.get());
   if (!checked_string_to_wchar_convert(__decimal_point_, lc->mon_decimal_point, loc.get()))
     __decimal_point_ = base::do_decimal_point();
   if (!checked_string_to_wchar_convert(__thousands_sep_, lc->mon_thousands_sep, loc.get()))
@@ -5578,7 +5578,7 @@ void moneypunct_byname<wchar_t, true>::init(const char* nm) {
   if (!loc)
     __throw_runtime_error(("moneypunct_byname failed to construct for " + string(nm)).c_str());
 
-  lconv* lc = __locale::__localeconv(loc.get());
+  __locale::__lconv_t* lc = __locale::__localeconv(loc.get());
   if (!checked_string_to_wchar_convert(__decimal_point_, lc->mon_decimal_point, loc.get()))
     __decimal_point_ = base::do_decimal_point();
   if (!checked_string_to_wchar_convert(__thousands_sep_, lc->mon_thousands_sep, loc.get()))
@@ -5650,6 +5650,16 @@ void moneypunct_byname<wchar_t, true>::init(const char* nm) {
 
 void __do_nothing(void*) {}
 
+// Legacy ABI __num_get functions - the new ones are _LIBCPP_HIDE_FROM_ABI
+template <class _CharT>
+string __num_get<_CharT>::__stage2_int_prep(ios_base& __iob, _CharT* __atoms, _CharT& __thousands_sep) {
+  locale __loc = __iob.getloc();
+  std::use_facet<ctype<_CharT> >(__loc).widen(__src, __src + __int_chr_cnt, __atoms);
+  const numpunct<_CharT>& __np = std::use_facet<numpunct<_CharT> >(__loc);
+  __thousands_sep              = __np.thousands_sep();
+  return __np.grouping();
+}
+
 template class _LIBCPP_CLASS_TEMPLATE_INSTANTIATION_VIS collate<char>;
 _LIBCPP_IF_WIDE_CHARACTERS(template class _LIBCPP_CLASS_TEMPLATE_INSTANTIATION_VIS collate<wchar_t>;)
 
diff --git a/libcxx/src/memory_resource.cpp b/libcxx/src/memory_resource.cpp
index e1a9e1a8fac49..ec9565f731bfa 100644
--- a/libcxx/src/memory_resource.cpp
+++ b/libcxx/src/memory_resource.cpp
@@ -38,7 +38,7 @@ static bool is_aligned_to(void* ptr, size_t align) {
 }
 #endif
 
-class _LIBCPP_EXPORTED_FROM_ABI __new_delete_memory_resource_imp : public memory_resource {
+class _LIBCPP_HIDDEN __new_delete_memory_resource_imp : public memory_resource {
   void* do_allocate(size_t bytes, size_t align) override {
 #if _LIBCPP_HAS_ALIGNED_ALLOCATION
     return std::__libcpp_allocate<std::byte>(__element_count(bytes), align);
@@ -63,7 +63,7 @@ class _LIBCPP_EXPORTED_FROM_ABI __new_delete_memory_resource_imp : public memory
 
 // null_memory_resource()
 
-class _LIBCPP_EXPORTED_FROM_ABI __null_memory_resource_imp : public memory_resource {
+class _LIBCPP_HIDDEN __null_memory_resource_imp : public memory_resource {
   void* do_allocate(size_t, size_t) override { __throw_bad_alloc(); }
   void do_deallocate(void*, size_t, size_t) override {}
   bool do_is_equal(const memory_resource& other) const noexcept override { return &other == this; }
diff --git a/libcxx/src/new.cpp b/libcxx/src/new.cpp
index b14b52248df33..e010fe4c4f191 100644
--- a/libcxx/src/new.cpp
+++ b/libcxx/src/new.cpp
@@ -43,7 +43,7 @@ static void* operator_new_impl(std::size_t size) {
   return p;
 }
 
-_LIBCPP_OVERRIDABLE_FUNCTION(_Znwm, void*, operator new, (std::size_t size)) _THROW_BAD_ALLOC {
+_LIBCPP_MAKE_OVERRIDABLE_FUNCTION_DETECTABLE _LIBCPP_WEAK void* operator new(std::size_t size) _THROW_BAD_ALLOC {
   void* p = operator_new_impl(size);
   if (p == nullptr)
     __throw_bad_alloc_shim();
@@ -54,7 +54,7 @@ _LIBCPP_WEAK void* operator new(size_t size, const std::nothrow_t&) noexcept {
 #  if !_LIBCPP_HAS_EXCEPTIONS
 #    if _LIBCPP_CAN_DETECT_OVERRIDDEN_FUNCTION
   _LIBCPP_ASSERT_SHIM(
-      !std::__is_function_overridden<static_cast<void* (*)(std::size_t)>(&operator new)>(),
+      !std::__is_function_overridden(static_cast<void* (*)(std::size_t)>(&operator new)),
       "libc++ was configured with exceptions disabled and `operator new(size_t)` has been overridden, "
       "but `operator new(size_t, nothrow_t)` has not been overridden. This is problematic because "
       "`operator new(size_t, nothrow_t)` must call `operator new(size_t)`, which will terminate in case "
@@ -74,7 +74,7 @@ _LIBCPP_WEAK void* operator new(size_t size, const std::nothrow_t&) noexcept {
 #  endif
 }
 
-_LIBCPP_OVERRIDABLE_FUNCTION(_Znam, void*, operator new[], (size_t size)) _THROW_BAD_ALLOC {
+_LIBCPP_MAKE_OVERRIDABLE_FUNCTION_DETECTABLE _LIBCPP_WEAK void* operator new[](size_t size) _THROW_BAD_ALLOC {
   return ::operator new(size);
 }
 
@@ -82,7 +82,7 @@ _LIBCPP_WEAK void* operator new[](size_t size, const std::nothrow_t&) noexcept {
 #  if !_LIBCPP_HAS_EXCEPTIONS
 #    if _LIBCPP_CAN_DETECT_OVERRIDDEN_FUNCTION
   _LIBCPP_ASSERT_SHIM(
-      !std::__is_function_overridden<static_cast<void* (*)(std::size_t)>(&operator new[])>(),
+      !std::__is_function_overridden(static_cast<void* (*)(std::size_t)>(&operator new[])),
       "libc++ was configured with exceptions disabled and `operator new[](size_t)` has been overridden, "
       "but `operator new[](size_t, nothrow_t)` has not been overridden. This is problematic because "
       "`operator new[](size_t, nothrow_t)` must call `operator new[](size_t)`, which will terminate in case "
@@ -136,8 +136,8 @@ static void* operator_new_aligned_impl(std::size_t size, std::align_val_t alignm
   return p;
 }
 
-_LIBCPP_OVERRIDABLE_FUNCTION(_ZnwmSt11align_val_t, void*, operator new, (std::size_t size, std::align_val_t alignment))
-_THROW_BAD_ALLOC {
+_LIBCPP_MAKE_OVERRIDABLE_FUNCTION_DETECTABLE _LIBCPP_WEAK void*
+operator new(std::size_t size, std::align_val_t alignment) _THROW_BAD_ALLOC {
   void* p = operator_new_aligned_impl(size, alignment);
   if (p == nullptr)
     __throw_bad_alloc_shim();
@@ -148,7 +148,7 @@ _LIBCPP_WEAK void* operator new(size_t size, std::align_val_t alignment, const s
 #    if !_LIBCPP_HAS_EXCEPTIONS
 #      if _LIBCPP_CAN_DETECT_OVERRIDDEN_FUNCTION
   _LIBCPP_ASSERT_SHIM(
-      !std::__is_function_overridden<static_cast<void* (*)(std::size_t, std::align_val_t)>(&operator new)>(),
+      !std::__is_function_overridden(static_cast<void* (*)(std::size_t, std::align_val_t)>(&operator new)),
       "libc++ was configured with exceptions disabled and `operator new(size_t, align_val_t)` has been overridden, "
       "but `operator new(size_t, align_val_t, nothrow_t)` has not been overridden. This is problematic because "
       "`operator new(size_t, align_val_t, nothrow_t)` must call `operator new(size_t, align_val_t)`, which will "
@@ -168,14 +168,16 @@ _LIBCPP_WEAK void* operator new(size_t size, std::align_val_t alignment, const s
 #    endif
 }
 
-_LIBCPP_OVERRIDABLE_FUNCTION(_ZnamSt11align_val_t, void*, operator new[], (size_t size, std::align_val_t alignment))
-_THROW_BAD_ALLOC { return ::operator new(size, alignment); }
+_LIBCPP_MAKE_OVERRIDABLE_FUNCTION_DETECTABLE _LIBCPP_WEAK void*
+operator new[](size_t size, std::align_val_t alignment) _THROW_BAD_ALLOC {
+  return ::operator new(size, alignment);
+}
 
 _LIBCPP_WEAK void* operator new[](size_t size, std::align_val_t alignment, const std::nothrow_t&) noexcept {
 #    if !_LIBCPP_HAS_EXCEPTIONS
 #      if _LIBCPP_CAN_DETECT_OVERRIDDEN_FUNCTION
   _LIBCPP_ASSERT_SHIM(
-      !std::__is_function_overridden<static_cast<void* (*)(std::size_t, std::align_val_t)>(&operator new[])>(),
+      !std::__is_function_overridden(static_cast<void* (*)(std::size_t, std::align_val_t)>(&operator new[])),
       "libc++ was configured with exceptions disabled and `operator new[](size_t, align_val_t)` has been overridden, "
       "but `operator new[](size_t, align_val_t, nothrow_t)` has not been overridden. This is problematic because "
       "`operator new[](size_t, align_val_t, nothrow_t)` must call `operator new[](size_t, align_val_t)`, which will "
diff --git a/libcxx/src/support/win32/locale_win32.cpp b/libcxx/src/support/win32/locale_win32.cpp
index ec2dd7f36ec70..24402e818d95d 100644
--- a/libcxx/src/support/win32/locale_win32.cpp
+++ b/libcxx/src/support/win32/locale_win32.cpp
@@ -26,7 +26,7 @@ __locale_t __newlocale(int /*mask*/, const char* locale, __locale_t /*base*/) {
   return {::_create_locale(LC_ALL, locale), locale};
 }
 
-lconv* __localeconv(__locale_t& loc) {
+__lconv_t* __localeconv(__locale_t& loc) {
   __locale_guard __current(loc);
   lconv* lc = std::localeconv();
   if (!lc)
diff --git a/libcxx/src/system_error.cpp b/libcxx/src/system_error.cpp
index d5ec73084f638..164fb72621c17 100644
--- a/libcxx/src/system_error.cpp
+++ b/libcxx/src/system_error.cpp
@@ -21,10 +21,6 @@
 
 #include "include/config_elast.h"
 
-#if defined(__ANDROID__)
-#  include <android/api-level.h>
-#endif
-
 #if defined(_LIBCPP_WIN32API)
 #  include <windows.h>
 #  include <winerror.h>
diff --git a/libcxx/test/benchmarks/GenerateInput.h b/libcxx/test/benchmarks/GenerateInput.h
index 6d5c5167e91ed..c87fd69162e9d 100644
--- a/libcxx/test/benchmarks/GenerateInput.h
+++ b/libcxx/test/benchmarks/GenerateInput.h
@@ -11,6 +11,7 @@
 
 #include <algorithm>
 #include <climits>
+#include <concepts>
 #include <cstddef>
 #include <random>
 #include <string>
@@ -171,4 +172,36 @@ inline std::vector<const char*> getRandomCStringInputs(std::size_t N) {
   return cinputs;
 }
 
+template <class T>
+struct Generate {
+  // When the contents don't matter
+  static T arbitrary();
+
+  // Prefer a cheap-to-construct element if possible
+  static T cheap();
+
+  // Prefer an expensive-to-construct element if possible
+  static T expensive();
+};
+
+template <class T>
+  requires std::integral<T>
+struct Generate<T> {
+  static T arbitrary() { return 42; }
+  static T cheap() { return 42; }
+  static T expensive() { return 42; }
+  static T random() { return getRandomInteger<T>(std::numeric_limits<T>::min(), std::numeric_limits<T>::max()); }
+};
+
+template <>
+struct Generate<std::string> {
+  static std::string arbitrary() { return "hello world"; }
+  static std::string cheap() { return "small"; }
+  static std::string expensive() { return std::string(256, 'x'); }
+  static std::string random() {
+    auto length = getRandomInteger<std::size_t>(1, 1024);
+    return getRandomString(length);
+  }
+};
+
 #endif // BENCHMARK_GENERATE_INPUT_H
diff --git a/libcxx/test/benchmarks/Utilities.h b/libcxx/test/benchmarks/Utilities.h
deleted file mode 100644
index fed16ba51f995..0000000000000
--- a/libcxx/test/benchmarks/Utilities.h
+++ /dev/null
@@ -1,37 +0,0 @@
-// -*- C++ -*-
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef BENCHMARK_UTILITIES_H
-#define BENCHMARK_UTILITIES_H
-
-#include <cassert>
-#include <type_traits>
-
-#include "benchmark/benchmark.h"
-
-namespace UtilitiesInternal {
-template <class Container>
-auto HaveDataImpl(int) -> decltype((std::declval<Container&>().data(), std::true_type{}));
-template <class Container>
-auto HaveDataImpl(long) -> std::false_type;
-template <class T>
-using HasData = decltype(HaveDataImpl<T>(0));
-} // namespace UtilitiesInternal
-
-template <class Container, std::enable_if_t<UtilitiesInternal::HasData<Container>::value>* = nullptr>
-void DoNotOptimizeData(Container& c) {
-  benchmark::DoNotOptimize(c.data());
-}
-
-template <class Container, std::enable_if_t<!UtilitiesInternal::HasData<Container>::value>* = nullptr>
-void DoNotOptimizeData(Container& c) {
-  benchmark::DoNotOptimize(&c);
-}
-
-#endif // BENCHMARK_UTILITIES_H
diff --git a/libcxx/test/benchmarks/algorithms/algorithms.partition_point.bench.cpp b/libcxx/test/benchmarks/algorithms/algorithms.partition_point.bench.cpp
index 0777acbafb5cc..e0bd7e36f78ad 100644
--- a/libcxx/test/benchmarks/algorithms/algorithms.partition_point.bench.cpp
+++ b/libcxx/test/benchmarks/algorithms/algorithms.partition_point.bench.cpp
@@ -6,7 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-// UNSUPPORTED: c++03, c++11, c++14
+// UNSUPPORTED: c++03, c++11, c++14, c++17
 
 #include <algorithm>
 #include <array>
diff --git a/libcxx/test/benchmarks/algorithms/copy.bench.cpp b/libcxx/test/benchmarks/algorithms/copy.bench.cpp
new file mode 100644
index 0000000000000..b6f0f15eb7703
--- /dev/null
+++ b/libcxx/test/benchmarks/algorithms/copy.bench.cpp
@@ -0,0 +1,89 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
+
+#include <algorithm>
+#include <benchmark/benchmark.h>
+#include <vector>
+
+static void bm_ranges_copy_vb(benchmark::State& state, bool aligned) {
+  auto n = state.range();
+  std::vector<bool> in(n, true);
+  std::vector<bool> out(aligned ? n : n + 8);
+  benchmark::DoNotOptimize(&in);
+  auto dst = aligned ? out.begin() : out.begin() + 4;
+  for (auto _ : state) {
+    benchmark::DoNotOptimize(std::ranges::copy(in, dst));
+    benchmark::DoNotOptimize(&out);
+  }
+}
+
+static void bm_ranges_copy_n_vb(benchmark::State& state, bool aligned) {
+  auto n = state.range();
+  std::vector<bool> in(n, true);
+  std::vector<bool> out(aligned ? n : n + 8);
+  benchmark::DoNotOptimize(&in);
+  auto src = in.begin();
+  auto dst = aligned ? out.begin() : out.begin() + 4;
+  for (auto _ : state) {
+    benchmark::DoNotOptimize(std::ranges::copy_n(src, n, dst));
+    benchmark::DoNotOptimize(&out);
+  }
+}
+
+static void bm_copy_vb(benchmark::State& state, bool aligned) {
+  auto n = state.range();
+  std::vector<bool> in(n, true);
+  std::vector<bool> out(aligned ? n : n + 8);
+  benchmark::DoNotOptimize(&in);
+  auto beg = in.begin();
+  auto end = in.end();
+  auto dst = aligned ? out.begin() : out.begin() + 4;
+  for (auto _ : state) {
+    benchmark::DoNotOptimize(std::copy(beg, end, dst));
+    benchmark::DoNotOptimize(&out);
+  }
+}
+
+static void bm_copy_n_vb(benchmark::State& state, bool aligned) {
+  auto n = state.range();
+  std::vector<bool> in(n, true);
+  std::vector<bool> out(aligned ? n : n + 8);
+  benchmark::DoNotOptimize(&in);
+  auto src = in.begin();
+  auto dst = aligned ? out.begin() : out.begin() + 4;
+  for (auto _ : state) {
+    benchmark::DoNotOptimize(std::copy_n(src, n, dst));
+    benchmark::DoNotOptimize(&out);
+  }
+}
+
+static void bm_ranges_copy_vb_aligned(benchmark::State& state) { bm_ranges_copy_vb(state, true); }
+static void bm_ranges_copy_vb_unaligned(benchmark::State& state) { bm_ranges_copy_vb(state, false); }
+static void bm_ranges_copy_n_vb_aligned(benchmark::State& state) { bm_ranges_copy_n_vb(state, true); }
+static void bm_ranges_copy_n_vb_unaligned(benchmark::State& state) { bm_ranges_copy_n_vb(state, false); }
+
+static void bm_copy_vb_aligned(benchmark::State& state) { bm_copy_vb(state, true); }
+static void bm_copy_vb_unaligned(benchmark::State& state) { bm_copy_vb(state, false); }
+static void bm_copy_n_vb_aligned(benchmark::State& state) { bm_copy_n_vb(state, true); }
+static void bm_copy_n_vb_unaligned(benchmark::State& state) { bm_copy_n_vb(state, false); }
+
+// Test std::ranges::copy for vector<bool>::iterator
+BENCHMARK(bm_ranges_copy_vb_aligned)->Range(8, 1 << 16)->DenseRange(102400, 204800, 4096);
+BENCHMARK(bm_ranges_copy_n_vb_aligned)->Range(8, 1 << 20);
+BENCHMARK(bm_ranges_copy_vb_unaligned)->Range(8, 1 << 20);
+BENCHMARK(bm_ranges_copy_n_vb_unaligned)->Range(8, 1 << 20);
+
+// Test std::copy for vector<bool>::iterator
+BENCHMARK(bm_copy_vb_aligned)->Range(8, 1 << 20);
+BENCHMARK(bm_copy_n_vb_aligned)->Range(8, 1 << 20);
+BENCHMARK(bm_copy_vb_unaligned)->Range(8, 1 << 20);
+BENCHMARK(bm_copy_n_vb_unaligned)->Range(8, 1 << 20);
+
+BENCHMARK_MAIN();
diff --git a/libcxx/test/benchmarks/algorithms/copy_backward.bench.cpp b/libcxx/test/benchmarks/algorithms/copy_backward.bench.cpp
new file mode 100644
index 0000000000000..c943d9a874b49
--- /dev/null
+++ b/libcxx/test/benchmarks/algorithms/copy_backward.bench.cpp
@@ -0,0 +1,55 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
+
+#include <algorithm>
+#include <benchmark/benchmark.h>
+#include <vector>
+
+static void bm_ranges_copy_backward_vb(benchmark::State& state, bool aligned) {
+  auto n = state.range();
+  std::vector<bool> in(n, true);
+  std::vector<bool> out(aligned ? n : n + 8);
+  benchmark::DoNotOptimize(&in);
+  auto dst = aligned ? out.end() : out.end() - 4;
+  for (auto _ : state) {
+    benchmark::DoNotOptimize(std::ranges::copy_backward(in, dst));
+    benchmark::DoNotOptimize(&out);
+  }
+}
+
+static void bm_copy_backward_vb(benchmark::State& state, bool aligned) {
+  auto n = state.range();
+  std::vector<bool> in(n, true);
+  std::vector<bool> out(aligned ? n : n + 8);
+  benchmark::DoNotOptimize(&in);
+  auto beg = in.begin();
+  auto end = in.end();
+  auto dst = aligned ? out.end() : out.end() - 4;
+  for (auto _ : state) {
+    benchmark::DoNotOptimize(std::copy_backward(beg, end, dst));
+    benchmark::DoNotOptimize(&out);
+  }
+}
+
+static void bm_ranges_copy_backward_vb_aligned(benchmark::State& state) { bm_ranges_copy_backward_vb(state, true); }
+static void bm_ranges_copy_backward_vb_unaligned(benchmark::State& state) { bm_ranges_copy_backward_vb(state, false); }
+
+static void bm_copy_backward_vb_aligned(benchmark::State& state) { bm_copy_backward_vb(state, true); }
+static void bm_copy_backward_vb_unaligned(benchmark::State& state) { bm_copy_backward_vb(state, false); }
+
+// Test std::ranges::copy_backward for vector<bool>::iterator
+BENCHMARK(bm_ranges_copy_backward_vb_aligned)->Range(8, 1 << 16)->DenseRange(102400, 204800, 4096);
+BENCHMARK(bm_ranges_copy_backward_vb_unaligned)->Range(8, 1 << 20);
+
+// Test std::copy_backward for vector<bool>::iterator
+BENCHMARK(bm_copy_backward_vb_aligned)->Range(8, 1 << 20);
+BENCHMARK(bm_copy_backward_vb_unaligned)->Range(8, 1 << 20);
+
+BENCHMARK_MAIN();
diff --git a/libcxx/test/benchmarks/algorithms/lower_bound.bench.cpp b/libcxx/test/benchmarks/algorithms/lower_bound.bench.cpp
index d9d57969df67a..31fb3597241fc 100644
--- a/libcxx/test/benchmarks/algorithms/lower_bound.bench.cpp
+++ b/libcxx/test/benchmarks/algorithms/lower_bound.bench.cpp
@@ -6,7 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-// UNSUPPORTED: c++03, c++11, c++14
+// UNSUPPORTED: c++03, c++11, c++14, c++17
 
 #include <algorithm>
 #include <numeric>
diff --git a/libcxx/test/benchmarks/algorithms/make_heap.bench.cpp b/libcxx/test/benchmarks/algorithms/make_heap.bench.cpp
index b7320e17c3e50..64d559620c512 100644
--- a/libcxx/test/benchmarks/algorithms/make_heap.bench.cpp
+++ b/libcxx/test/benchmarks/algorithms/make_heap.bench.cpp
@@ -6,7 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-// UNSUPPORTED: c++03, c++11, c++14
+// UNSUPPORTED: c++03, c++11, c++14, c++17
 
 #include <algorithm>
 
diff --git a/libcxx/test/benchmarks/algorithms/make_heap_then_sort_heap.bench.cpp b/libcxx/test/benchmarks/algorithms/make_heap_then_sort_heap.bench.cpp
index 5991d2846aee4..c6dc136be3ac4 100644
--- a/libcxx/test/benchmarks/algorithms/make_heap_then_sort_heap.bench.cpp
+++ b/libcxx/test/benchmarks/algorithms/make_heap_then_sort_heap.bench.cpp
@@ -6,7 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-// UNSUPPORTED: c++03, c++11, c++14
+// UNSUPPORTED: c++03, c++11, c++14, c++17
 
 #include <algorithm>
 
diff --git a/libcxx/test/benchmarks/algorithms/pop_heap.bench.cpp b/libcxx/test/benchmarks/algorithms/pop_heap.bench.cpp
index 5fef52284239d..e4b96a0ae48c7 100644
--- a/libcxx/test/benchmarks/algorithms/pop_heap.bench.cpp
+++ b/libcxx/test/benchmarks/algorithms/pop_heap.bench.cpp
@@ -6,7 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-// UNSUPPORTED: c++03, c++11, c++14
+// UNSUPPORTED: c++03, c++11, c++14, c++17
 
 #include <algorithm>
 
diff --git a/libcxx/test/benchmarks/algorithms/pstl.stable_sort.bench.cpp b/libcxx/test/benchmarks/algorithms/pstl.stable_sort.bench.cpp
index 10254ac12cf56..a385185ec7fe5 100644
--- a/libcxx/test/benchmarks/algorithms/pstl.stable_sort.bench.cpp
+++ b/libcxx/test/benchmarks/algorithms/pstl.stable_sort.bench.cpp
@@ -6,7 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-// UNSUPPORTED: c++03, c++11, c++14
+// UNSUPPORTED: c++03, c++11, c++14, c++17
 // UNSUPPORTED: libcpp-has-no-incomplete-pstl
 
 #include <algorithm>
diff --git a/libcxx/test/benchmarks/algorithms/push_heap.bench.cpp b/libcxx/test/benchmarks/algorithms/push_heap.bench.cpp
index 89d8122bd1dbe..7dfa0285348bd 100644
--- a/libcxx/test/benchmarks/algorithms/push_heap.bench.cpp
+++ b/libcxx/test/benchmarks/algorithms/push_heap.bench.cpp
@@ -6,7 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-// UNSUPPORTED: c++03, c++11, c++14
+// UNSUPPORTED: c++03, c++11, c++14, c++17
 
 #include <algorithm>
 
diff --git a/libcxx/test/benchmarks/algorithms/reverse.bench.cpp b/libcxx/test/benchmarks/algorithms/reverse.bench.cpp
new file mode 100644
index 0000000000000..2d8dd819ac24c
--- /dev/null
+++ b/libcxx/test/benchmarks/algorithms/reverse.bench.cpp
@@ -0,0 +1,48 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
+
+#include <algorithm>
+#include <iterator>
+#include <string>
+#include <vector>
+
+#include <benchmark/benchmark.h>
+#include "../GenerateInput.h"
+
+template <class T>
+static void bm_reverse(benchmark::State& state) {
+  std::size_t const n = state.range();
+  std::vector<T> vec;
+  std::generate_n(std::back_inserter(vec), n, [] { return Generate<T>::cheap(); });
+  for (auto _ : state) {
+    std::reverse(vec.begin(), vec.end());
+    benchmark::DoNotOptimize(vec);
+  }
+}
+BENCHMARK(bm_reverse<int>)->Name("std::reverse(vector<int>)")->DenseRange(1, 8)->Range(16, 1 << 20);
+BENCHMARK(bm_reverse<std::string>)->Name("std::reverse(vector<string>)")->DenseRange(1, 8)->Range(16, 1 << 20);
+
+template <class T>
+static void bm_ranges_reverse(benchmark::State& state) {
+  std::size_t const n = state.range();
+  std::vector<T> vec;
+  std::generate_n(std::back_inserter(vec), n, [] { return Generate<T>::cheap(); });
+  for (auto _ : state) {
+    std::ranges::reverse(vec.begin(), vec.end());
+    benchmark::DoNotOptimize(vec);
+  }
+}
+BENCHMARK(bm_ranges_reverse<int>)->Name("ranges::reverse(vector<int>)")->DenseRange(1, 8)->Range(16, 1 << 20);
+BENCHMARK(bm_ranges_reverse<std::string>)
+    ->Name("ranges::reverse(vector<string>)")
+    ->DenseRange(1, 8)
+    ->Range(16, 1 << 20);
+
+BENCHMARK_MAIN();
diff --git a/libcxx/test/benchmarks/algorithms/set_intersection.bench.cpp b/libcxx/test/benchmarks/algorithms/set_intersection.bench.cpp
index 9bde4bb29dc22..40292179781ee 100644
--- a/libcxx/test/benchmarks/algorithms/set_intersection.bench.cpp
+++ b/libcxx/test/benchmarks/algorithms/set_intersection.bench.cpp
@@ -6,7 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-// UNSUPPORTED: c++03, c++11, c++14
+// UNSUPPORTED: c++03, c++11, c++14, c++17
 
 #include <algorithm>
 #include <cstdlib>
diff --git a/libcxx/test/benchmarks/algorithms/sort.bench.cpp b/libcxx/test/benchmarks/algorithms/sort.bench.cpp
index 899272e34795f..7f3ce6ff7a07e 100644
--- a/libcxx/test/benchmarks/algorithms/sort.bench.cpp
+++ b/libcxx/test/benchmarks/algorithms/sort.bench.cpp
@@ -6,7 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-// UNSUPPORTED: c++03, c++11, c++14
+// UNSUPPORTED: c++03, c++11, c++14, c++17
 
 #include <algorithm>
 
diff --git a/libcxx/test/benchmarks/algorithms/sort_heap.bench.cpp b/libcxx/test/benchmarks/algorithms/sort_heap.bench.cpp
index ee4b6bfc7387b..1ce9f1a6df9af 100644
--- a/libcxx/test/benchmarks/algorithms/sort_heap.bench.cpp
+++ b/libcxx/test/benchmarks/algorithms/sort_heap.bench.cpp
@@ -6,7 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-// UNSUPPORTED: c++03, c++11, c++14
+// UNSUPPORTED: c++03, c++11, c++14, c++17
 
 #include <algorithm>
 
diff --git a/libcxx/test/benchmarks/algorithms/stable_sort.bench.cpp b/libcxx/test/benchmarks/algorithms/stable_sort.bench.cpp
index c68f73838c319..26e8de935f5c5 100644
--- a/libcxx/test/benchmarks/algorithms/stable_sort.bench.cpp
+++ b/libcxx/test/benchmarks/algorithms/stable_sort.bench.cpp
@@ -6,7 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-// UNSUPPORTED: c++03, c++11, c++14
+// UNSUPPORTED: c++03, c++11, c++14, c++17
 
 #include <algorithm>
 
diff --git a/libcxx/test/benchmarks/containers/ContainerBenchmarks.h b/libcxx/test/benchmarks/containers/ContainerBenchmarks.h
deleted file mode 100644
index 5fc8981619672..0000000000000
--- a/libcxx/test/benchmarks/containers/ContainerBenchmarks.h
+++ /dev/null
@@ -1,332 +0,0 @@
-// -*- C++ -*-
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef BENCHMARK_CONTAINER_BENCHMARKS_H
-#define BENCHMARK_CONTAINER_BENCHMARKS_H
-
-#include <cassert>
-#include <iterator>
-#include <utility>
-
-#include "benchmark/benchmark.h"
-#include "../Utilities.h"
-#include "test_iterators.h"
-
-namespace ContainerBenchmarks {
-
-template <class Container>
-void BM_ConstructSize(benchmark::State& st, Container) {
-  auto size = st.range(0);
-  for (auto _ : st) {
-    Container c(size);
-    DoNotOptimizeData(c);
-  }
-}
-
-template <class Container>
-void BM_CopyConstruct(benchmark::State& st, Container) {
-  auto size = st.range(0);
-  Container c(size);
-  for (auto _ : st) {
-    auto v = c;
-    DoNotOptimizeData(v);
-  }
-}
-
-template <class Container>
-void BM_Assignment(benchmark::State& st, Container) {
-  auto size = st.range(0);
-  Container c1;
-  Container c2(size);
-  for (auto _ : st) {
-    c1 = c2;
-    DoNotOptimizeData(c1);
-    DoNotOptimizeData(c2);
-  }
-}
-
-template <std::size_t... sz, typename Container, typename GenInputs>
-void BM_AssignInputIterIter(benchmark::State& st, Container c, GenInputs gen) {
-  auto v = gen(1, sz...);
-  c.resize(st.range(0), v[0]);
-  auto in = gen(st.range(1), sz...);
-  benchmark::DoNotOptimize(&in);
-  benchmark::DoNotOptimize(&c);
-  for (auto _ : st) {
-    c.assign(cpp17_input_iterator(in.begin()), cpp17_input_iterator(in.end()));
-    benchmark::ClobberMemory();
-  }
-}
-
-template <class Container>
-void BM_ConstructSizeValue(benchmark::State& st, Container, typename Container::value_type const& val) {
-  const auto size = st.range(0);
-  for (auto _ : st) {
-    Container c(size, val);
-    DoNotOptimizeData(c);
-  }
-}
-
-template <class Container, class GenInputs>
-void BM_ConstructIterIter(benchmark::State& st, Container, GenInputs gen) {
-  auto in          = gen(st.range(0));
-  const auto begin = in.begin();
-  const auto end   = in.end();
-  benchmark::DoNotOptimize(&in);
-  while (st.KeepRunning()) {
-    Container c(begin, end);
-    DoNotOptimizeData(c);
-  }
-}
-
-template <class Container, class GenInputs>
-void BM_ConstructFromRange(benchmark::State& st, Container, GenInputs gen) {
-  auto in = gen(st.range(0));
-  benchmark::DoNotOptimize(&in);
-  while (st.KeepRunning()) {
-    Container c(std::from_range, in);
-    DoNotOptimizeData(c);
-  }
-}
-
-template <class Container>
-void BM_Pushback_no_grow(benchmark::State& state, Container c) {
-  int count = state.range(0);
-  c.reserve(count);
-  while (state.KeepRunningBatch(count)) {
-    c.clear();
-    for (int i = 0; i != count; ++i) {
-      c.push_back(i);
-    }
-    benchmark::DoNotOptimize(c.data());
-  }
-}
-
-template <class Container, class GenInputs>
-void BM_InsertValue(benchmark::State& st, Container c, GenInputs gen) {
-  auto in        = gen(st.range(0));
-  const auto end = in.end();
-  while (st.KeepRunning()) {
-    c.clear();
-    for (auto it = in.begin(); it != end; ++it) {
-      benchmark::DoNotOptimize(&(*c.insert(*it).first));
-    }
-    benchmark::ClobberMemory();
-  }
-}
-
-template <class Container, class GenInputs>
-void BM_InsertValueRehash(benchmark::State& st, Container c, GenInputs gen) {
-  auto in        = gen(st.range(0));
-  const auto end = in.end();
-  while (st.KeepRunning()) {
-    c.clear();
-    c.rehash(16);
-    for (auto it = in.begin(); it != end; ++it) {
-      benchmark::DoNotOptimize(&(*c.insert(*it).first));
-    }
-    benchmark::ClobberMemory();
-  }
-}
-
-template <class Container, class GenInputs>
-void BM_Insert_InputIterIter_NoRealloc(benchmark::State& st, Container c, GenInputs gen) {
-  auto in = gen(st.range(0));
-  DoNotOptimizeData(in);
-  const auto size = c.size();
-  const auto beg  = cpp17_input_iterator(in.begin());
-  const auto end  = cpp17_input_iterator(in.end());
-  c.reserve(size + in.size()); // force no reallocation
-  for (auto _ : st) {
-    benchmark::DoNotOptimize(&(*c.insert(c.begin(), beg, end)));
-    st.PauseTiming();
-    c.erase(c.begin() + size, c.end()); // avoid the container to grow indefinitely
-    st.ResumeTiming();
-    DoNotOptimizeData(c);
-    benchmark::ClobberMemory();
-  }
-}
-
-template <class Container, class GenInputs>
-void BM_Insert_InputIterIter_Realloc_HalfFilled(benchmark::State& st, Container, GenInputs gen) {
-  const auto size = st.range(0);
-  Container a     = gen(size);
-  Container in    = gen(size + 10);
-  DoNotOptimizeData(a);
-  DoNotOptimizeData(in);
-  const auto beg = cpp17_input_iterator(in.begin());
-  const auto end = cpp17_input_iterator(in.end());
-  for (auto _ : st) {
-    st.PauseTiming();
-    Container c;
-    c.reserve(size * 2); // Reallocation with half-filled container
-    c = a;
-    st.ResumeTiming();
-    benchmark::DoNotOptimize(&(*c.insert(c.begin(), beg, end)));
-    DoNotOptimizeData(c);
-    benchmark::ClobberMemory();
-  }
-}
-
-template <class Container, class GenInputs>
-void BM_Insert_InputIterIter_Realloc_NearFull(benchmark::State& st, Container, GenInputs gen) {
-  const auto size = st.range(0);
-  Container a     = gen(size);
-  Container in    = gen(10);
-  DoNotOptimizeData(a);
-  DoNotOptimizeData(in);
-  const auto beg = cpp17_input_iterator(in.begin());
-  const auto end = cpp17_input_iterator(in.end());
-  for (auto _ : st) {
-    st.PauseTiming();
-    Container c;
-    c.reserve(size + 5); // Reallocation almost-full container
-    c = a;
-    st.ResumeTiming();
-    benchmark::DoNotOptimize(&(*c.insert(c.begin(), beg, end)));
-    DoNotOptimizeData(c);
-    benchmark::ClobberMemory();
-  }
-}
-
-template <class Container, class GenInputs>
-void BM_InsertDuplicate(benchmark::State& st, Container c, GenInputs gen) {
-  auto in        = gen(st.range(0));
-  const auto end = in.end();
-  c.insert(in.begin(), in.end());
-  benchmark::DoNotOptimize(&c);
-  benchmark::DoNotOptimize(&in);
-  while (st.KeepRunning()) {
-    for (auto it = in.begin(); it != end; ++it) {
-      benchmark::DoNotOptimize(&(*c.insert(*it).first));
-    }
-    benchmark::ClobberMemory();
-  }
-}
-
-template <class Container, class GenInputs>
-void BM_EmplaceDuplicate(benchmark::State& st, Container c, GenInputs gen) {
-  auto in        = gen(st.range(0));
-  const auto end = in.end();
-  c.insert(in.begin(), in.end());
-  benchmark::DoNotOptimize(&c);
-  benchmark::DoNotOptimize(&in);
-  while (st.KeepRunning()) {
-    for (auto it = in.begin(); it != end; ++it) {
-      benchmark::DoNotOptimize(&(*c.emplace(*it).first));
-    }
-    benchmark::ClobberMemory();
-  }
-}
-
-template <class Container, class GenInputs>
-void BM_erase_iter_in_middle(benchmark::State& st, Container, GenInputs gen) {
-  auto in = gen(st.range(0));
-  Container c(in.begin(), in.end());
-  assert(c.size() > 2);
-  for (auto _ : st) {
-    auto mid    = std::next(c.begin(), c.size() / 2);
-    auto tmp    = *mid;
-    auto result = c.erase(mid); // erase an element in the middle
-    benchmark::DoNotOptimize(result);
-    c.push_back(std::move(tmp)); // and then push it back at the end to avoid needing a new container
-  }
-}
-
-template <class Container, class GenInputs>
-void BM_erase_iter_at_start(benchmark::State& st, Container, GenInputs gen) {
-  auto in = gen(st.range(0));
-  Container c(in.begin(), in.end());
-  assert(c.size() > 2);
-  for (auto _ : st) {
-    auto it     = c.begin();
-    auto tmp    = *it;
-    auto result = c.erase(it); // erase the first element
-    benchmark::DoNotOptimize(result);
-    c.push_back(std::move(tmp)); // and then push it back at the end to avoid needing a new container
-  }
-}
-
-template <class Container, class GenInputs>
-void BM_Find(benchmark::State& st, Container c, GenInputs gen) {
-  auto in = gen(st.range(0));
-  c.insert(in.begin(), in.end());
-  benchmark::DoNotOptimize(&(*c.begin()));
-  const auto end = in.data() + in.size();
-  while (st.KeepRunning()) {
-    for (auto it = in.data(); it != end; ++it) {
-      benchmark::DoNotOptimize(&(*c.find(*it)));
-    }
-    benchmark::ClobberMemory();
-  }
-}
-
-template <class Container, class GenInputs>
-void BM_FindRehash(benchmark::State& st, Container c, GenInputs gen) {
-  c.rehash(8);
-  auto in = gen(st.range(0));
-  c.insert(in.begin(), in.end());
-  benchmark::DoNotOptimize(&(*c.begin()));
-  const auto end = in.data() + in.size();
-  while (st.KeepRunning()) {
-    for (auto it = in.data(); it != end; ++it) {
-      benchmark::DoNotOptimize(&(*c.find(*it)));
-    }
-    benchmark::ClobberMemory();
-  }
-}
-
-template <class Container, class GenInputs>
-void BM_Rehash(benchmark::State& st, Container c, GenInputs gen) {
-  auto in = gen(st.range(0));
-  c.max_load_factor(3.0);
-  c.insert(in.begin(), in.end());
-  benchmark::DoNotOptimize(c);
-  const auto bucket_count = c.bucket_count();
-  while (st.KeepRunning()) {
-    c.rehash(bucket_count + 1);
-    c.rehash(bucket_count);
-    benchmark::ClobberMemory();
-  }
-}
-
-template <class Container, class GenInputs>
-void BM_Compare_same_container(benchmark::State& st, Container, GenInputs gen) {
-  auto in = gen(st.range(0));
-  Container c1(in.begin(), in.end());
-  Container c2 = c1;
-
-  benchmark::DoNotOptimize(&(*c1.begin()));
-  benchmark::DoNotOptimize(&(*c2.begin()));
-  while (st.KeepRunning()) {
-    bool res = c1 == c2;
-    benchmark::DoNotOptimize(&res);
-    benchmark::ClobberMemory();
-  }
-}
-
-template <class Container, class GenInputs>
-void BM_Compare_different_containers(benchmark::State& st, Container, GenInputs gen) {
-  auto in1 = gen(st.range(0));
-  auto in2 = gen(st.range(0));
-  Container c1(in1.begin(), in1.end());
-  Container c2(in2.begin(), in2.end());
-
-  benchmark::DoNotOptimize(&(*c1.begin()));
-  benchmark::DoNotOptimize(&(*c2.begin()));
-  while (st.KeepRunning()) {
-    bool res = c1 == c2;
-    benchmark::DoNotOptimize(&res);
-    benchmark::ClobberMemory();
-  }
-}
-
-} // namespace ContainerBenchmarks
-
-#endif // BENCHMARK_CONTAINER_BENCHMARKS_H
diff --git a/libcxx/test/benchmarks/containers/associative/associative_container_benchmarks.h b/libcxx/test/benchmarks/containers/associative/associative_container_benchmarks.h
new file mode 100644
index 0000000000000..fb4455c4aa9da
--- /dev/null
+++ b/libcxx/test/benchmarks/containers/associative/associative_container_benchmarks.h
@@ -0,0 +1,512 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef TEST_BENCHMARKS_CONTAINERS_ASSOCIATIVE_CONTAINER_BENCHMARKS_H
+#define TEST_BENCHMARKS_CONTAINERS_ASSOCIATIVE_CONTAINER_BENCHMARKS_H
+
+#include <algorithm>
+#include <iterator>
+#include <random>
+#include <string>
+#include <type_traits>
+#include <utility>
+#include <vector>
+
+#include "benchmark/benchmark.h"
+#include "../../GenerateInput.h"
+
+namespace support {
+
+template <class Container>
+struct adapt_operations {
+  // using ValueType = ...;
+  // using KeyType   = ...;
+  // static ValueType value_from_key(KeyType const& k);
+  // static KeyType key_from_value(ValueType const& value);
+
+  // using InsertionResult = ...;
+  // static Container::iterator get_iterator(InsertionResult const&);
+};
+
+template <class Container>
+void associative_container_benchmarks(std::string container) {
+  using Key   = typename Container::key_type;
+  using Value = typename Container::value_type;
+
+  auto generate_unique_keys = [=](std::size_t n) {
+    std::set<Key> keys;
+    while (keys.size() < n) {
+      Key k = Generate<Key>::random();
+      keys.insert(k);
+    }
+    return std::vector<Key>(keys.begin(), keys.end());
+  };
+
+  auto make_value_types = [](std::vector<Key> const& keys) {
+    std::vector<Value> kv;
+    for (Key const& k : keys)
+      kv.push_back(adapt_operations<Container>::value_from_key(k));
+    return kv;
+  };
+
+  auto get_key = [](Value const& v) { return adapt_operations<Container>::key_from_value(v); };
+
+  auto bench = [&](std::string operation, auto f) {
+    benchmark::RegisterBenchmark(container + "::" + operation, f)->Arg(32)->Arg(1024)->Arg(8192);
+  };
+
+  static constexpr bool is_multi_key_container =
+      !std::is_same_v<typename adapt_operations<Container>::InsertionResult,
+                      std::pair<typename Container::iterator, bool>>;
+
+  static constexpr bool is_ordered_container = requires(Container c, Key k) { c.lower_bound(k); };
+
+  // These benchmarks are structured to perform the operation being benchmarked
+  // a small number of times at each iteration, in order to offset the cost of
+  // PauseTiming() and ResumeTiming().
+  static constexpr std::size_t BatchSize = 32;
+
+  struct alignas(Container) ScratchSpace {
+    char storage[sizeof(Container)];
+  };
+
+  /////////////////////////
+  // Constructors
+  /////////////////////////
+  bench("ctor(const&)", [=](auto& st) {
+    const std::size_t size = st.range(0);
+    std::vector<Value> in  = make_value_types(generate_unique_keys(size));
+    Container src(in.begin(), in.end());
+    ScratchSpace c[BatchSize];
+
+    while (st.KeepRunningBatch(BatchSize)) {
+      for (std::size_t i = 0; i != BatchSize; ++i) {
+        new (c + i) Container(src);
+        benchmark::DoNotOptimize(c + i);
+        benchmark::ClobberMemory();
+      }
+
+      st.PauseTiming();
+      for (std::size_t i = 0; i != BatchSize; ++i) {
+        reinterpret_cast<Container*>(c + i)->~Container();
+      }
+      st.ResumeTiming();
+    }
+  });
+
+  bench("ctor(iterator, iterator) (unsorted sequence)", [=](auto& st) {
+    const std::size_t size = st.range(0);
+    std::mt19937 randomness;
+    std::vector<Key> keys = generate_unique_keys(size);
+    std::shuffle(keys.begin(), keys.end(), randomness);
+    std::vector<Value> in = make_value_types(keys);
+    ScratchSpace c[BatchSize];
+
+    while (st.KeepRunningBatch(BatchSize)) {
+      for (std::size_t i = 0; i != BatchSize; ++i) {
+        new (c + i) Container(in.begin(), in.end());
+        benchmark::DoNotOptimize(c + i);
+        benchmark::ClobberMemory();
+      }
+
+      st.PauseTiming();
+      for (std::size_t i = 0; i != BatchSize; ++i) {
+        reinterpret_cast<Container*>(c + i)->~Container();
+      }
+      st.ResumeTiming();
+    }
+  });
+
+  bench("ctor(iterator, iterator) (sorted sequence)", [=](auto& st) {
+    const std::size_t size = st.range(0);
+    std::vector<Key> keys  = generate_unique_keys(size);
+    std::sort(keys.begin(), keys.end());
+    std::vector<Value> in = make_value_types(keys);
+    ScratchSpace c[BatchSize];
+
+    while (st.KeepRunningBatch(BatchSize)) {
+      for (std::size_t i = 0; i != BatchSize; ++i) {
+        new (c + i) Container(in.begin(), in.end());
+        benchmark::DoNotOptimize(c + i);
+        benchmark::ClobberMemory();
+      }
+
+      st.PauseTiming();
+      for (std::size_t i = 0; i != BatchSize; ++i) {
+        reinterpret_cast<Container*>(c + i)->~Container();
+      }
+      st.ResumeTiming();
+    }
+  });
+
+  /////////////////////////
+  // Assignment
+  /////////////////////////
+  bench("operator=(const&)", [=](auto& st) {
+    const std::size_t size = st.range(0);
+    std::vector<Value> in  = make_value_types(generate_unique_keys(size));
+    Container src(in.begin(), in.end());
+    Container c[BatchSize];
+
+    while (st.KeepRunningBatch(BatchSize)) {
+      for (std::size_t i = 0; i != BatchSize; ++i) {
+        c[i] = src;
+        benchmark::DoNotOptimize(c[i]);
+        benchmark::ClobberMemory();
+      }
+
+      st.PauseTiming();
+      for (std::size_t i = 0; i != BatchSize; ++i) {
+        c[i].clear();
+      }
+      st.ResumeTiming();
+    }
+  });
+
+  /////////////////////////
+  // Insertion
+  /////////////////////////
+  bench("insert(value) (already present)", [=](auto& st) {
+    const std::size_t size = st.range(0);
+    std::vector<Value> in  = make_value_types(generate_unique_keys(size));
+    Value to_insert        = in[in.size() / 2]; // pick any existing value
+    std::vector<Container> c(BatchSize, Container(in.begin(), in.end()));
+    typename Container::iterator inserted[BatchSize];
+
+    while (st.KeepRunningBatch(BatchSize)) {
+      for (std::size_t i = 0; i != BatchSize; ++i) {
+        inserted[i] = adapt_operations<Container>::get_iterator(c[i].insert(to_insert));
+        benchmark::DoNotOptimize(inserted[i]);
+        benchmark::DoNotOptimize(c[i]);
+        benchmark::ClobberMemory();
+      }
+
+      if constexpr (is_multi_key_container) {
+        st.PauseTiming();
+        for (std::size_t i = 0; i != BatchSize; ++i) {
+          c[i].erase(inserted[i]);
+        }
+        st.ResumeTiming();
+      }
+    }
+  });
+
+  bench("insert(value) (new value)", [=](auto& st) {
+    const std::size_t size = st.range(0);
+    std::vector<Value> in  = make_value_types(generate_unique_keys(size + 1));
+    Value to_insert        = in.back();
+    in.pop_back();
+    std::vector<Container> c(BatchSize, Container(in.begin(), in.end()));
+
+    while (st.KeepRunningBatch(BatchSize)) {
+      for (std::size_t i = 0; i != BatchSize; ++i) {
+        auto result = c[i].insert(to_insert);
+        benchmark::DoNotOptimize(result);
+        benchmark::DoNotOptimize(c[i]);
+        benchmark::ClobberMemory();
+      }
+
+      st.PauseTiming();
+      for (std::size_t i = 0; i != BatchSize; ++i) {
+        c[i].erase(get_key(to_insert));
+      }
+      st.ResumeTiming();
+    }
+  });
+
+  // The insert(hint, ...) methods are only relevant for ordered containers, and we lack
+  // a good way to compute a hint for unordered ones.
+  if constexpr (is_ordered_container) {
+    bench("insert(hint, value) (good hint)", [=](auto& st) {
+      const std::size_t size = st.range(0);
+      std::vector<Value> in  = make_value_types(generate_unique_keys(size + 1));
+      Value to_insert        = in.back();
+      in.pop_back();
+
+      std::vector<Container> c(BatchSize, Container(in.begin(), in.end()));
+      typename Container::iterator hints[BatchSize];
+      for (std::size_t i = 0; i != BatchSize; ++i) {
+        hints[i] = c[i].lower_bound(get_key(to_insert));
+      }
+
+      while (st.KeepRunningBatch(BatchSize)) {
+        for (std::size_t i = 0; i != BatchSize; ++i) {
+          auto result = c[i].insert(hints[i], to_insert);
+          benchmark::DoNotOptimize(result);
+          benchmark::DoNotOptimize(c[i]);
+          benchmark::ClobberMemory();
+        }
+
+        st.PauseTiming();
+        for (std::size_t i = 0; i != BatchSize; ++i) {
+          c[i].erase(get_key(to_insert));
+          hints[i] = c[i].lower_bound(get_key(to_insert)); // refresh hints in case of invalidation
+        }
+        st.ResumeTiming();
+      }
+    });
+
+    bench("insert(hint, value) (bad hint)", [=](auto& st) {
+      const std::size_t size = st.range(0);
+      std::vector<Value> in  = make_value_types(generate_unique_keys(size + 1));
+      Value to_insert        = in.back();
+      in.pop_back();
+      std::vector<Container> c(BatchSize, Container(in.begin(), in.end()));
+
+      while (st.KeepRunningBatch(BatchSize)) {
+        for (std::size_t i = 0; i != BatchSize; ++i) {
+          auto result = c[i].insert(c[i].begin(), to_insert);
+          benchmark::DoNotOptimize(result);
+          benchmark::DoNotOptimize(c[i]);
+          benchmark::ClobberMemory();
+        }
+
+        st.PauseTiming();
+        for (std::size_t i = 0; i != BatchSize; ++i) {
+          c[i].erase(get_key(to_insert));
+        }
+        st.ResumeTiming();
+      }
+    });
+  }
+
+  bench("insert(iterator, iterator) (all new keys)", [=](auto& st) {
+    const std::size_t size = st.range(0);
+    std::vector<Value> in  = make_value_types(generate_unique_keys(size + (size / 10)));
+
+    // Populate a container with a small number of elements, that's what containers will start with.
+    std::vector<Value> small;
+    for (std::size_t i = 0; i != (size / 10); ++i) {
+      small.push_back(in.back());
+      in.pop_back();
+    }
+    Container c(small.begin(), small.end());
+
+    for ([[maybe_unused]] auto _ : st) {
+      c.insert(in.begin(), in.end());
+      benchmark::DoNotOptimize(c);
+      benchmark::ClobberMemory();
+
+      st.PauseTiming();
+      c = Container(small.begin(), small.end());
+      st.ResumeTiming();
+    }
+  });
+
+  bench("insert(iterator, iterator) (half new keys)", [=](auto& st) {
+    const std::size_t size = st.range(0);
+    std::vector<Value> in  = make_value_types(generate_unique_keys(size));
+
+    // Populate a container that already contains half the elements we'll try inserting,
+    // that's what our container will start with.
+    std::vector<Value> small;
+    for (std::size_t i = 0; i != size / 2; ++i) {
+      small.push_back(in.at(i * 2));
+    }
+    Container c(small.begin(), small.end());
+
+    for ([[maybe_unused]] auto _ : st) {
+      c.insert(in.begin(), in.end());
+      benchmark::DoNotOptimize(c);
+      benchmark::ClobberMemory();
+
+      st.PauseTiming();
+      c = Container(small.begin(), small.end());
+      st.ResumeTiming();
+    }
+  });
+
+  /////////////////////////
+  // Erasure
+  /////////////////////////
+  bench("erase(key) (existent)", [=](auto& st) {
+    const std::size_t size = st.range(0);
+    std::vector<Value> in  = make_value_types(generate_unique_keys(size));
+    Value element          = in[in.size() / 2]; // pick any element
+    std::vector<Container> c(BatchSize, Container(in.begin(), in.end()));
+
+    while (st.KeepRunningBatch(BatchSize)) {
+      for (std::size_t i = 0; i != BatchSize; ++i) {
+        auto result = c[i].erase(get_key(element));
+        benchmark::DoNotOptimize(result);
+        benchmark::DoNotOptimize(c[i]);
+        benchmark::ClobberMemory();
+      }
+
+      st.PauseTiming();
+      for (std::size_t i = 0; i != BatchSize; ++i) {
+        c[i].insert(element);
+      }
+      st.ResumeTiming();
+    }
+  });
+
+  bench("erase(key) (non-existent)", [=](auto& st) {
+    const std::size_t size = st.range(0);
+    std::vector<Value> in  = make_value_types(generate_unique_keys(size + BatchSize));
+    std::vector<Key> keys;
+    for (std::size_t i = 0; i != BatchSize; ++i) {
+      keys.push_back(get_key(in.back()));
+      in.pop_back();
+    }
+    Container c(in.begin(), in.end());
+
+    while (st.KeepRunningBatch(BatchSize)) {
+      for (std::size_t i = 0; i != BatchSize; ++i) {
+        auto result = c.erase(keys[i]);
+        benchmark::DoNotOptimize(result);
+        benchmark::DoNotOptimize(c);
+        benchmark::ClobberMemory();
+      }
+
+      // no cleanup required because we erased a non-existent element
+    }
+  });
+
+  bench("erase(iterator)", [=](auto& st) {
+    const std::size_t size = st.range(0);
+    std::vector<Value> in  = make_value_types(generate_unique_keys(size));
+    Value element          = in[in.size() / 2]; // pick any element
+
+    std::vector<Container> c;
+    std::vector<typename Container::iterator> iterators;
+    for (std::size_t i = 0; i != BatchSize; ++i) {
+      c.push_back(Container(in.begin(), in.end()));
+      iterators.push_back(c[i].find(get_key(element)));
+    }
+
+    while (st.KeepRunningBatch(BatchSize)) {
+      for (std::size_t i = 0; i != BatchSize; ++i) {
+        auto result = c[i].erase(iterators[i]);
+        benchmark::DoNotOptimize(result);
+        benchmark::DoNotOptimize(c[i]);
+        benchmark::ClobberMemory();
+      }
+
+      st.PauseTiming();
+      for (std::size_t i = 0; i != BatchSize; ++i) {
+        iterators[i] = adapt_operations<Container>::get_iterator(c[i].insert(element));
+      }
+      st.ResumeTiming();
+    }
+  });
+
+  bench("erase(iterator, iterator) (erase half the container)", [=](auto& st) {
+    const std::size_t size = st.range(0);
+    std::vector<Value> in  = make_value_types(generate_unique_keys(size));
+    Container c(in.begin(), in.end());
+
+    auto first = std::next(c.begin(), c.size() / 4);
+    auto last  = std::next(c.begin(), 3 * (c.size() / 4));
+    for ([[maybe_unused]] auto _ : st) {
+      auto result = c.erase(first, last);
+      benchmark::DoNotOptimize(result);
+      benchmark::DoNotOptimize(c);
+      benchmark::ClobberMemory();
+
+      st.PauseTiming();
+      c     = Container(in.begin(), in.end());
+      first = std::next(c.begin(), c.size() / 4);
+      last  = std::next(c.begin(), 3 * (c.size() / 4));
+      st.ResumeTiming();
+    }
+  });
+
+  bench("clear()", [=](auto& st) {
+    const std::size_t size = st.range(0);
+    std::vector<Value> in  = make_value_types(generate_unique_keys(size));
+    Container c(in.begin(), in.end());
+
+    for ([[maybe_unused]] auto _ : st) {
+      c.clear();
+      benchmark::DoNotOptimize(c);
+      benchmark::ClobberMemory();
+
+      st.PauseTiming();
+      c = Container(in.begin(), in.end());
+      st.ResumeTiming();
+    }
+  });
+
+  /////////////////////////
+  // Query
+  /////////////////////////
+  auto with_existent_key = [=](auto func) {
+    return [=](auto& st) {
+      const std::size_t size = st.range(0);
+      std::vector<Value> in  = make_value_types(generate_unique_keys(size));
+      // Pick any `BatchSize` number of elements
+      std::vector<Key> keys;
+      for (std::size_t i = 0; i < in.size(); i += (in.size() / BatchSize)) {
+        keys.push_back(get_key(in.at(i)));
+      }
+      Container c(in.begin(), in.end());
+
+      while (st.KeepRunningBatch(BatchSize)) {
+        for (std::size_t i = 0; i != BatchSize; ++i) {
+          auto result = func(c, keys[i]);
+          benchmark::DoNotOptimize(c);
+          benchmark::DoNotOptimize(result);
+          benchmark::ClobberMemory();
+        }
+      }
+    };
+  };
+
+  auto with_nonexistent_key = [=](auto func) {
+    return [=](auto& st) {
+      const std::size_t size = st.range(0);
+      std::vector<Value> in  = make_value_types(generate_unique_keys(size + BatchSize));
+      std::vector<Key> keys;
+      for (std::size_t i = 0; i != BatchSize; ++i) {
+        keys.push_back(get_key(in.back()));
+        in.pop_back();
+      }
+      Container c(in.begin(), in.end());
+
+      while (st.KeepRunningBatch(BatchSize)) {
+        for (std::size_t i = 0; i != BatchSize; ++i) {
+          auto result = func(c, keys[i]);
+          benchmark::DoNotOptimize(c);
+          benchmark::DoNotOptimize(result);
+          benchmark::ClobberMemory();
+        }
+      }
+    };
+  };
+
+  auto find = [](Container const& c, Key const& key) { return c.find(key); };
+  bench("find(key) (existent)", with_existent_key(find));
+  bench("find(key) (non-existent)", with_nonexistent_key(find));
+
+  auto count = [](Container const& c, Key const& key) { return c.count(key); };
+  bench("count(key) (existent)", with_existent_key(count));
+  bench("count(key) (non-existent)", with_nonexistent_key(count));
+
+  auto contains = [](Container const& c, Key const& key) { return c.contains(key); };
+  bench("contains(key) (existent)", with_existent_key(contains));
+  bench("contains(key) (non-existent)", with_nonexistent_key(contains));
+
+  if constexpr (is_ordered_container) {
+    auto lower_bound = [](Container const& c, Key const& key) { return c.lower_bound(key); };
+    bench("lower_bound(key) (existent)", with_existent_key(lower_bound));
+    bench("lower_bound(key) (non-existent)", with_nonexistent_key(lower_bound));
+
+    auto upper_bound = [](Container const& c, Key const& key) { return c.upper_bound(key); };
+    bench("upper_bound(key) (existent)", with_existent_key(upper_bound));
+    bench("upper_bound(key) (non-existent)", with_nonexistent_key(upper_bound));
+
+    auto equal_range = [](Container const& c, Key const& key) { return c.equal_range(key); };
+    bench("equal_range(key) (existent)", with_existent_key(equal_range));
+    bench("equal_range(key) (non-existent)", with_nonexistent_key(equal_range));
+  }
+}
+
+} // namespace support
+
+#endif // TEST_BENCHMARKS_CONTAINERS_ASSOCIATIVE_CONTAINER_BENCHMARKS_H
diff --git a/libcxx/test/benchmarks/containers/associative/flat_map.bench.cpp b/libcxx/test/benchmarks/containers/associative/flat_map.bench.cpp
new file mode 100644
index 0000000000000..82902d50f31e6
--- /dev/null
+++ b/libcxx/test/benchmarks/containers/associative/flat_map.bench.cpp
@@ -0,0 +1,36 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// REQUIRES: std-at-least-c++26
+
+#include <flat_map>
+#include <utility>
+
+#include "associative_container_benchmarks.h"
+#include "../../GenerateInput.h"
+#include "benchmark/benchmark.h"
+
+template <class K, class V>
+struct support::adapt_operations<std::flat_map<K, V>> {
+  using ValueType = typename std::flat_map<K, V>::value_type;
+  using KeyType   = typename std::flat_map<K, V>::key_type;
+  static ValueType value_from_key(KeyType const& k) { return {k, Generate<V>::arbitrary()}; }
+  static KeyType key_from_value(ValueType const& value) { return value.first; }
+
+  using InsertionResult = std::pair<typename std::flat_map<K, V>::iterator, bool>;
+  static auto get_iterator(InsertionResult const& result) { return result.first; }
+};
+
+int main(int argc, char** argv) {
+  support::associative_container_benchmarks<std::flat_map<int, int>>("std::flat_map<int, int>");
+
+  benchmark::Initialize(&argc, argv);
+  benchmark::RunSpecifiedBenchmarks();
+  benchmark::Shutdown();
+  return 0;
+}
diff --git a/libcxx/test/benchmarks/containers/associative/flat_multimap.bench.cpp b/libcxx/test/benchmarks/containers/associative/flat_multimap.bench.cpp
new file mode 100644
index 0000000000000..f752f79b3b454
--- /dev/null
+++ b/libcxx/test/benchmarks/containers/associative/flat_multimap.bench.cpp
@@ -0,0 +1,35 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// REQUIRES: std-at-least-c++26
+
+#include <flat_map>
+
+#include "associative_container_benchmarks.h"
+#include "../../GenerateInput.h"
+#include "benchmark/benchmark.h"
+
+template <class K, class V>
+struct support::adapt_operations<std::flat_multimap<K, V>> {
+  using ValueType = typename std::flat_multimap<K, V>::value_type;
+  using KeyType   = typename std::flat_multimap<K, V>::key_type;
+  static ValueType value_from_key(KeyType const& k) { return {k, Generate<V>::arbitrary()}; }
+  static KeyType key_from_value(ValueType const& value) { return value.first; }
+
+  using InsertionResult = typename std::flat_multimap<K, V>::iterator;
+  static auto get_iterator(InsertionResult const& result) { return result; }
+};
+
+int main(int argc, char** argv) {
+  support::associative_container_benchmarks<std::flat_multimap<int, int>>("std::flat_multimap<int, int>");
+
+  benchmark::Initialize(&argc, argv);
+  benchmark::RunSpecifiedBenchmarks();
+  benchmark::Shutdown();
+  return 0;
+}
diff --git a/libcxx/test/benchmarks/containers/associative/map.bench.cpp b/libcxx/test/benchmarks/containers/associative/map.bench.cpp
new file mode 100644
index 0000000000000..cee669ae0a667
--- /dev/null
+++ b/libcxx/test/benchmarks/containers/associative/map.bench.cpp
@@ -0,0 +1,37 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17
+
+#include <map>
+#include <string>
+#include <utility>
+
+#include "associative_container_benchmarks.h"
+#include "../../GenerateInput.h"
+#include "benchmark/benchmark.h"
+
+template <class K, class V>
+struct support::adapt_operations<std::map<K, V>> {
+  using ValueType = typename std::map<K, V>::value_type;
+  using KeyType   = typename std::map<K, V>::key_type;
+  static ValueType value_from_key(KeyType const& k) { return {k, Generate<V>::arbitrary()}; }
+  static KeyType key_from_value(ValueType const& value) { return value.first; }
+
+  using InsertionResult = std::pair<typename std::map<K, V>::iterator, bool>;
+  static auto get_iterator(InsertionResult const& result) { return result.first; }
+};
+
+int main(int argc, char** argv) {
+  support::associative_container_benchmarks<std::map<int, int>>("std::map<int, int>");
+
+  benchmark::Initialize(&argc, argv);
+  benchmark::RunSpecifiedBenchmarks();
+  benchmark::Shutdown();
+  return 0;
+}
diff --git a/libcxx/test/benchmarks/containers/associative/multimap.bench.cpp b/libcxx/test/benchmarks/containers/associative/multimap.bench.cpp
new file mode 100644
index 0000000000000..6ae93f06aa363
--- /dev/null
+++ b/libcxx/test/benchmarks/containers/associative/multimap.bench.cpp
@@ -0,0 +1,36 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17
+
+#include <map>
+#include <string>
+
+#include "associative_container_benchmarks.h"
+#include "../../GenerateInput.h"
+#include "benchmark/benchmark.h"
+
+template <class K, class V>
+struct support::adapt_operations<std::multimap<K, V>> {
+  using ValueType = typename std::multimap<K, V>::value_type;
+  using KeyType   = typename std::multimap<K, V>::key_type;
+  static ValueType value_from_key(KeyType const& k) { return {k, Generate<V>::arbitrary()}; }
+  static KeyType key_from_value(ValueType const& value) { return value.first; }
+
+  using InsertionResult = typename std::multimap<K, V>::iterator;
+  static auto get_iterator(InsertionResult const& result) { return result; }
+};
+
+int main(int argc, char** argv) {
+  support::associative_container_benchmarks<std::multimap<int, int>>("std::multimap<int, int>");
+
+  benchmark::Initialize(&argc, argv);
+  benchmark::RunSpecifiedBenchmarks();
+  benchmark::Shutdown();
+  return 0;
+}
diff --git a/libcxx/test/benchmarks/containers/associative/multiset.bench.cpp b/libcxx/test/benchmarks/containers/associative/multiset.bench.cpp
new file mode 100644
index 0000000000000..894f159a52e4a
--- /dev/null
+++ b/libcxx/test/benchmarks/containers/associative/multiset.bench.cpp
@@ -0,0 +1,34 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17
+
+#include <set>
+
+#include "associative_container_benchmarks.h"
+#include "benchmark/benchmark.h"
+
+template <class K>
+struct support::adapt_operations<std::multiset<K>> {
+  using ValueType = typename std::multiset<K>::value_type;
+  using KeyType   = typename std::multiset<K>::key_type;
+  static ValueType value_from_key(KeyType const& k) { return k; }
+  static KeyType key_from_value(ValueType const& value) { return value; }
+
+  using InsertionResult = typename std::multiset<K>::iterator;
+  static auto get_iterator(InsertionResult const& result) { return result; }
+};
+
+int main(int argc, char** argv) {
+  support::associative_container_benchmarks<std::multiset<int>>("std::multiset<int>");
+
+  benchmark::Initialize(&argc, argv);
+  benchmark::RunSpecifiedBenchmarks();
+  benchmark::Shutdown();
+  return 0;
+}
diff --git a/libcxx/test/benchmarks/containers/associative/set.bench.cpp b/libcxx/test/benchmarks/containers/associative/set.bench.cpp
new file mode 100644
index 0000000000000..6b7b142c792ba
--- /dev/null
+++ b/libcxx/test/benchmarks/containers/associative/set.bench.cpp
@@ -0,0 +1,35 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17
+
+#include <set>
+#include <utility>
+
+#include "associative_container_benchmarks.h"
+#include "benchmark/benchmark.h"
+
+template <class K>
+struct support::adapt_operations<std::set<K>> {
+  using ValueType = typename std::set<K>::value_type;
+  using KeyType   = typename std::set<K>::key_type;
+  static ValueType value_from_key(KeyType const& k) { return k; }
+  static KeyType key_from_value(ValueType const& value) { return value; }
+
+  using InsertionResult = std::pair<typename std::set<K>::iterator, bool>;
+  static auto get_iterator(InsertionResult const& result) { return result.first; }
+};
+
+int main(int argc, char** argv) {
+  support::associative_container_benchmarks<std::set<int>>("std::set<int>");
+
+  benchmark::Initialize(&argc, argv);
+  benchmark::RunSpecifiedBenchmarks();
+  benchmark::Shutdown();
+  return 0;
+}
diff --git a/libcxx/test/benchmarks/containers/associative/unordered_map.bench.cpp b/libcxx/test/benchmarks/containers/associative/unordered_map.bench.cpp
new file mode 100644
index 0000000000000..57adec2d214d4
--- /dev/null
+++ b/libcxx/test/benchmarks/containers/associative/unordered_map.bench.cpp
@@ -0,0 +1,36 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17
+
+#include <unordered_map>
+#include <utility>
+
+#include "associative_container_benchmarks.h"
+#include "../../GenerateInput.h"
+#include "benchmark/benchmark.h"
+
+template <class K, class V>
+struct support::adapt_operations<std::unordered_map<K, V>> {
+  using ValueType = typename std::unordered_map<K, V>::value_type;
+  using KeyType   = typename std::unordered_map<K, V>::key_type;
+  static ValueType value_from_key(KeyType const& k) { return {k, Generate<V>::arbitrary()}; }
+  static KeyType key_from_value(ValueType const& value) { return value.first; }
+
+  using InsertionResult = std::pair<typename std::unordered_map<K, V>::iterator, bool>;
+  static auto get_iterator(InsertionResult const& result) { return result.first; }
+};
+
+int main(int argc, char** argv) {
+  support::associative_container_benchmarks<std::unordered_map<int, int>>("std::unordered_map<int, int>");
+
+  benchmark::Initialize(&argc, argv);
+  benchmark::RunSpecifiedBenchmarks();
+  benchmark::Shutdown();
+  return 0;
+}
diff --git a/libcxx/test/benchmarks/containers/associative/unordered_multimap.bench.cpp b/libcxx/test/benchmarks/containers/associative/unordered_multimap.bench.cpp
new file mode 100644
index 0000000000000..8738ca4bf9f0c
--- /dev/null
+++ b/libcxx/test/benchmarks/containers/associative/unordered_multimap.bench.cpp
@@ -0,0 +1,35 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17
+
+#include <unordered_map>
+
+#include "associative_container_benchmarks.h"
+#include "../../GenerateInput.h"
+#include "benchmark/benchmark.h"
+
+template <class K, class V>
+struct support::adapt_operations<std::unordered_multimap<K, V>> {
+  using ValueType = typename std::unordered_multimap<K, V>::value_type;
+  using KeyType   = typename std::unordered_multimap<K, V>::key_type;
+  static ValueType value_from_key(KeyType const& k) { return {k, Generate<V>::arbitrary()}; }
+  static KeyType key_from_value(ValueType const& value) { return value.first; }
+
+  using InsertionResult = typename std::unordered_multimap<K, V>::iterator;
+  static auto get_iterator(InsertionResult const& result) { return result; }
+};
+
+int main(int argc, char** argv) {
+  support::associative_container_benchmarks<std::unordered_multimap<int, int>>("std::unordered_multimap<int, int>");
+
+  benchmark::Initialize(&argc, argv);
+  benchmark::RunSpecifiedBenchmarks();
+  benchmark::Shutdown();
+  return 0;
+}
diff --git a/libcxx/test/benchmarks/containers/associative/unordered_multiset.bench.cpp b/libcxx/test/benchmarks/containers/associative/unordered_multiset.bench.cpp
new file mode 100644
index 0000000000000..4888b01bfeba0
--- /dev/null
+++ b/libcxx/test/benchmarks/containers/associative/unordered_multiset.bench.cpp
@@ -0,0 +1,34 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17
+
+#include <unordered_set>
+
+#include "associative_container_benchmarks.h"
+#include "benchmark/benchmark.h"
+
+template <class K>
+struct support::adapt_operations<std::unordered_multiset<K>> {
+  using ValueType = typename std::unordered_multiset<K>::value_type;
+  using KeyType   = typename std::unordered_multiset<K>::key_type;
+  static ValueType value_from_key(KeyType const& k) { return k; }
+  static KeyType key_from_value(ValueType const& value) { return value; }
+
+  using InsertionResult = typename std::unordered_multiset<K>::iterator;
+  static auto get_iterator(InsertionResult const& result) { return result; }
+};
+
+int main(int argc, char** argv) {
+  support::associative_container_benchmarks<std::unordered_multiset<int>>("std::unordered_multiset<int>");
+
+  benchmark::Initialize(&argc, argv);
+  benchmark::RunSpecifiedBenchmarks();
+  benchmark::Shutdown();
+  return 0;
+}
diff --git a/libcxx/test/benchmarks/containers/associative/unordered_set.bench.cpp b/libcxx/test/benchmarks/containers/associative/unordered_set.bench.cpp
new file mode 100644
index 0000000000000..56420bdaadfbf
--- /dev/null
+++ b/libcxx/test/benchmarks/containers/associative/unordered_set.bench.cpp
@@ -0,0 +1,35 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17
+
+#include <unordered_set>
+#include <utility>
+
+#include "associative_container_benchmarks.h"
+#include "benchmark/benchmark.h"
+
+template <class K>
+struct support::adapt_operations<std::unordered_set<K>> {
+  using ValueType = typename std::unordered_set<K>::value_type;
+  using KeyType   = typename std::unordered_set<K>::key_type;
+  static ValueType value_from_key(KeyType const& k) { return k; }
+  static KeyType key_from_value(ValueType const& value) { return value; }
+
+  using InsertionResult = std::pair<typename std::unordered_set<K>::iterator, bool>;
+  static auto get_iterator(InsertionResult const& result) { return result.first; }
+};
+
+int main(int argc, char** argv) {
+  support::associative_container_benchmarks<std::unordered_set<int>>("std::unordered_set<int>");
+
+  benchmark::Initialize(&argc, argv);
+  benchmark::RunSpecifiedBenchmarks();
+  benchmark::Shutdown();
+  return 0;
+}
diff --git a/libcxx/test/benchmarks/containers/deque.bench.cpp b/libcxx/test/benchmarks/containers/deque.bench.cpp
deleted file mode 100644
index 7ff1093a9391c..0000000000000
--- a/libcxx/test/benchmarks/containers/deque.bench.cpp
+++ /dev/null
@@ -1,55 +0,0 @@
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
-
-#include <deque>
-#include <string>
-
-#include "benchmark/benchmark.h"
-
-#include "ContainerBenchmarks.h"
-#include "../GenerateInput.h"
-
-using namespace ContainerBenchmarks;
-
-constexpr std::size_t TestNumInputs = 1024;
-
-BENCHMARK_CAPTURE(BM_ConstructSize, deque_byte, std::deque<unsigned char>{})->Arg(5140480);
-
-BENCHMARK_CAPTURE(BM_ConstructSizeValue, deque_byte, std::deque<unsigned char>{}, 0)->Arg(5140480);
-
-BENCHMARK_CAPTURE(BM_ConstructIterIter, deque_char, std::deque<char>{}, getRandomIntegerInputs<char>)
-    ->Arg(TestNumInputs);
-
-BENCHMARK_CAPTURE(BM_ConstructIterIter, deque_size_t, std::deque<size_t>{}, getRandomIntegerInputs<size_t>)
-    ->Arg(TestNumInputs);
-
-BENCHMARK_CAPTURE(BM_ConstructIterIter, deque_string, std::deque<std::string>{}, getRandomStringInputs)
-    ->Arg(TestNumInputs);
-
-BENCHMARK_CAPTURE(BM_ConstructFromRange, deque_char, std::deque<char>{}, getRandomIntegerInputs<char>)
-    ->Arg(TestNumInputs);
-
-BENCHMARK_CAPTURE(BM_ConstructFromRange, deque_size_t, std::deque<size_t>{}, getRandomIntegerInputs<size_t>)
-    ->Arg(TestNumInputs);
-
-BENCHMARK_CAPTURE(BM_ConstructFromRange, deque_string, std::deque<std::string>{}, getRandomStringInputs)
-    ->Arg(TestNumInputs);
-
-BENCHMARK_CAPTURE(BM_erase_iter_in_middle, deque_int, std::deque<int>{}, getRandomIntegerInputs<int>)
-    ->Range(TestNumInputs, TestNumInputs * 10);
-BENCHMARK_CAPTURE(BM_erase_iter_in_middle, deque_string, std::deque<std::string>{}, getRandomStringInputs)
-    ->Range(TestNumInputs, TestNumInputs * 10);
-
-BENCHMARK_CAPTURE(BM_erase_iter_at_start, deque_int, std::deque<int>{}, getRandomIntegerInputs<int>)
-    ->Range(TestNumInputs, TestNumInputs * 10);
-BENCHMARK_CAPTURE(BM_erase_iter_at_start, deque_string, std::deque<std::string>{}, getRandomStringInputs)
-    ->Range(TestNumInputs, TestNumInputs * 10);
-
-BENCHMARK_MAIN();
diff --git a/libcxx/test/benchmarks/containers/map.bench.cpp b/libcxx/test/benchmarks/containers/map.bench.cpp
deleted file mode 100644
index e37c7d8d55163..0000000000000
--- a/libcxx/test/benchmarks/containers/map.bench.cpp
+++ /dev/null
@@ -1,949 +0,0 @@
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-// UNSUPPORTED: c++03, c++11, c++14
-
-#include <algorithm>
-#include <cstdint>
-#include <map>
-#include <random>
-#include <vector>
-
-#include "../CartesianBenchmarks.h"
-#include "benchmark/benchmark.h"
-#include "test_macros.h"
-
-// When VALIDATE is defined the benchmark will run to validate the benchmarks.
-// The time taken by several operations depend on whether or not an element
-// exists. To avoid errors in the benchmark these operations have a validation
-// mode to test the benchmark. Since they are not meant to be benchmarked the
-// number of sizes tested is limited to 1.
-// #define VALIDATE
-
-namespace {
-
-enum class Mode { Hit, Miss };
-
-struct AllModes : EnumValuesAsTuple<AllModes, Mode, 2> {
-  static constexpr const char* Names[] = {"ExistingElement", "NewElement"};
-};
-
-// The positions of the hints to pick:
-// - Begin picks the first item. The item cannot be put before this element.
-// - Thrid picks the third item. This is just an element with a valid entry
-//   before and after it.
-// - Correct contains the correct hint.
-// - End contains a hint to the end of the map.
-enum class Hint { Begin, Third, Correct, End };
-struct AllHints : EnumValuesAsTuple<AllHints, Hint, 4> {
-  static constexpr const char* Names[] = {"Begin", "Third", "Correct", "End"};
-};
-
-enum class Order { Sorted, Random };
-struct AllOrders : EnumValuesAsTuple<AllOrders, Order, 2> {
-  static constexpr const char* Names[] = {"Sorted", "Random"};
-};
-
-struct TestSets {
-  std::vector<uint64_t> Keys;
-  std::vector<std::map<uint64_t, int64_t> > Maps;
-  std::vector<std::vector<typename std::map<uint64_t, int64_t>::const_iterator> > Hints;
-};
-
-enum class Shuffle { None, Keys, Hints };
-
-TestSets makeTestingSets(size_t MapSize, Mode mode, Shuffle shuffle, size_t max_maps) {
-  /*
-   * The shuffle does not retain the random number generator to use the same
-   * set of random numbers for every iteration.
-   */
-  TestSets R;
-
-  int MapCount = std::min(max_maps, 1000000 / MapSize);
-
-  for (uint64_t I = 0; I < MapSize; ++I) {
-    R.Keys.push_back(mode == Mode::Hit ? 2 * I + 2 : 2 * I + 1);
-  }
-  if (shuffle == Shuffle::Keys)
-    std::shuffle(R.Keys.begin(), R.Keys.end(), std::mt19937());
-
-  for (int M = 0; M < MapCount; ++M) {
-    auto& map   = R.Maps.emplace_back();
-    auto& hints = R.Hints.emplace_back();
-    for (uint64_t I = 0; I < MapSize; ++I) {
-      hints.push_back(map.insert(std::make_pair(2 * I + 2, 0)).first);
-    }
-    if (shuffle == Shuffle::Hints)
-      std::shuffle(hints.begin(), hints.end(), std::mt19937());
-  }
-
-  return R;
-}
-
-struct Base {
-  size_t MapSize;
-  Base(size_t T) : MapSize(T) {}
-
-  std::string baseName() const { return "_MapSize=" + std::to_string(MapSize); }
-};
-
-//*******************************************************************|
-//                       Member functions                            |
-//*******************************************************************|
-
-struct ConstructorDefault {
-  void run(benchmark::State& State) const {
-    for (auto _ : State) {
-      benchmark::DoNotOptimize(std::map<uint64_t, int64_t>());
-    }
-  }
-
-  std::string name() const { return "BM_ConstructorDefault"; }
-};
-
-struct ConstructorIterator : Base {
-  using Base::Base;
-
-  void run(benchmark::State& State) const {
-    auto Data = makeTestingSets(MapSize, Mode::Hit, Shuffle::None, 1);
-    auto& Map = Data.Maps.front();
-    while (State.KeepRunningBatch(MapSize)) {
-#ifndef VALIDATE
-      benchmark::DoNotOptimize(std::map<uint64_t, int64_t>(Map.begin(), Map.end()));
-#else
-      std::map<uint64_t, int64_t> M{Map.begin(), Map.end()};
-      if (M != Map)
-        State.SkipWithError("Map copy not identical");
-#endif
-    }
-  }
-
-  std::string name() const { return "BM_ConstructorIterator" + baseName(); }
-};
-
-struct ConstructorCopy : Base {
-  using Base::Base;
-
-  void run(benchmark::State& State) const {
-    auto Data = makeTestingSets(MapSize, Mode::Hit, Shuffle::None, 1);
-    auto& Map = Data.Maps.front();
-    while (State.KeepRunningBatch(MapSize)) {
-#ifndef VALIDATE
-      std::map<uint64_t, int64_t> M(Map);
-      benchmark::DoNotOptimize(M);
-#else
-      std::map<uint64_t, int64_t> M(Map);
-      if (M != Map)
-        State.SkipWithError("Map copy not identical");
-#endif
-    }
-  }
-
-  std::string name() const { return "BM_ConstructorCopy" + baseName(); }
-};
-
-struct ConstructorMove : Base {
-  using Base::Base;
-
-  void run(benchmark::State& State) const {
-    auto Data = makeTestingSets(MapSize, Mode::Hit, Shuffle::None, 1000);
-    while (State.KeepRunningBatch(MapSize * Data.Maps.size())) {
-      for (auto& Map : Data.Maps) {
-        std::map<uint64_t, int64_t> M(std::move(Map));
-        benchmark::DoNotOptimize(M);
-      }
-      State.PauseTiming();
-      Data = makeTestingSets(MapSize, Mode::Hit, Shuffle::None, 1000);
-      State.ResumeTiming();
-    }
-  }
-
-  std::string name() const { return "BM_ConstructorMove" + baseName(); }
-};
-
-//*******************************************************************|
-//                           Capacity                                |
-//*******************************************************************|
-
-struct Empty : Base {
-  using Base::Base;
-
-  void run(benchmark::State& State) const {
-    auto Data = makeTestingSets(MapSize, Mode::Hit, Shuffle::None, 1);
-    auto& Map = Data.Maps.front();
-    for (auto _ : State) {
-#ifndef VALIDATE
-      benchmark::DoNotOptimize(Map.empty());
-#else
-      if (Map.empty())
-        State.SkipWithError("Map contains an invalid number of elements.");
-#endif
-    }
-  }
-
-  std::string name() const { return "BM_Empty" + baseName(); }
-};
-
-struct Size : Base {
-  using Base::Base;
-
-  void run(benchmark::State& State) const {
-    auto Data = makeTestingSets(MapSize, Mode::Hit, Shuffle::None, 1);
-    auto& Map = Data.Maps.front();
-    for (auto _ : State) {
-#ifndef VALIDATE
-      benchmark::DoNotOptimize(Map.size());
-#else
-      if (Map.size() != MapSize)
-        State.SkipWithError("Map contains an invalid number of elements.");
-#endif
-    }
-  }
-
-  std::string name() const { return "BM_Size" + baseName(); }
-};
-
-//*******************************************************************|
-//                           Modifiers                               |
-//*******************************************************************|
-
-struct Clear : Base {
-  using Base::Base;
-
-  void run(benchmark::State& State) const {
-    auto Data = makeTestingSets(MapSize, Mode::Hit, Shuffle::None, 1000);
-    while (State.KeepRunningBatch(MapSize * Data.Maps.size())) {
-      for (auto& Map : Data.Maps) {
-        Map.clear();
-        benchmark::DoNotOptimize(Map);
-      }
-      State.PauseTiming();
-      Data = makeTestingSets(MapSize, Mode::Hit, Shuffle::None, 1000);
-      State.ResumeTiming();
-    }
-  }
-
-  std::string name() const { return "BM_Clear" + baseName(); }
-};
-
-template <class Mode, class Order>
-struct Insert : Base {
-  using Base::Base;
-
-  void run(benchmark::State& State) const {
-    auto Data = makeTestingSets(MapSize, Mode(), Order::value == ::Order::Random ? Shuffle::Keys : Shuffle::None, 1000);
-    while (State.KeepRunningBatch(MapSize * Data.Maps.size())) {
-      for (auto& Map : Data.Maps) {
-        for (auto K : Data.Keys) {
-#ifndef VALIDATE
-          benchmark::DoNotOptimize(Map.insert(std::make_pair(K, 1)));
-#else
-          bool Inserted = Map.insert(std::make_pair(K, 1)).second;
-          if (Mode() == ::Mode::Hit) {
-            if (Inserted)
-              State.SkipWithError("Inserted a duplicate element");
-          } else {
-            if (!Inserted)
-              State.SkipWithError("Failed to insert e new element");
-          }
-#endif
-        }
-      }
-
-      State.PauseTiming();
-      Data = makeTestingSets(MapSize, Mode(), Order::value == ::Order::Random ? Shuffle::Keys : Shuffle::None, 1000);
-      State.ResumeTiming();
-    }
-  }
-
-  std::string name() const { return "BM_Insert" + baseName() + Mode::name() + Order::name(); }
-};
-
-template <class Mode, class Hint>
-struct InsertHint : Base {
-  using Base::Base;
-
-  template < ::Hint hint>
-  typename std::enable_if<hint == ::Hint::Correct>::type run(benchmark::State& State) const {
-    auto Data = makeTestingSets(MapSize, Mode(), Shuffle::None, 1000);
-    while (State.KeepRunningBatch(MapSize * Data.Maps.size())) {
-      for (size_t I = 0; I < Data.Maps.size(); ++I) {
-        auto& Map = Data.Maps[I];
-        auto H    = Data.Hints[I].begin();
-        for (auto K : Data.Keys) {
-#ifndef VALIDATE
-          benchmark::DoNotOptimize(Map.insert(*H, std::make_pair(K, 1)));
-#else
-          auto Inserted = Map.insert(*H, std::make_pair(K, 1));
-          if (Mode() == ::Mode::Hit) {
-            if (Inserted != *H)
-              State.SkipWithError("Inserted a duplicate element");
-          } else {
-            if (++Inserted != *H)
-              State.SkipWithError("Failed to insert a new element");
-          }
-#endif
-          ++H;
-        }
-      }
-
-      State.PauseTiming();
-      Data = makeTestingSets(MapSize, Mode(), Shuffle::None, 1000);
-      State.ResumeTiming();
-    }
-  }
-
-  template < ::Hint hint>
-  typename std::enable_if<hint != ::Hint::Correct>::type run(benchmark::State& State) const {
-    auto Data = makeTestingSets(MapSize, Mode(), Shuffle::None, 1000);
-    while (State.KeepRunningBatch(MapSize * Data.Maps.size())) {
-      for (size_t I = 0; I < Data.Maps.size(); ++I) {
-        auto& Map  = Data.Maps[I];
-        auto Third = *(Data.Hints[I].begin() + 2);
-        for (auto K : Data.Keys) {
-          auto Itor = hint == ::Hint::Begin ? Map.begin() : hint == ::Hint::Third ? Third : Map.end();
-#ifndef VALIDATE
-          benchmark::DoNotOptimize(Map.insert(Itor, std::make_pair(K, 1)));
-#else
-          size_t Size = Map.size();
-          Map.insert(Itor, std::make_pair(K, 1));
-          if (Mode() == ::Mode::Hit) {
-            if (Size != Map.size())
-              State.SkipWithError("Inserted a duplicate element");
-          } else {
-            if (Size + 1 != Map.size())
-              State.SkipWithError("Failed to insert a new element");
-          }
-#endif
-        }
-      }
-
-      State.PauseTiming();
-      Data = makeTestingSets(MapSize, Mode(), Shuffle::None, 1000);
-      State.ResumeTiming();
-    }
-  }
-
-  void run(benchmark::State& State) const {
-    static constexpr auto h = Hint();
-    run<h>(State);
-  }
-
-  std::string name() const { return "BM_InsertHint" + baseName() + Mode::name() + Hint::name(); }
-};
-
-template <class Mode, class Order>
-struct InsertAssign : Base {
-  using Base::Base;
-
-  void run(benchmark::State& State) const {
-    auto Data = makeTestingSets(MapSize, Mode(), Order::value == ::Order::Random ? Shuffle::Keys : Shuffle::None, 1000);
-    while (State.KeepRunningBatch(MapSize * Data.Maps.size())) {
-      for (auto& Map : Data.Maps) {
-        for (auto K : Data.Keys) {
-#ifndef VALIDATE
-          benchmark::DoNotOptimize(Map.insert_or_assign(K, 1));
-#else
-          bool Inserted = Map.insert_or_assign(K, 1).second;
-          if (Mode() == ::Mode::Hit) {
-            if (Inserted)
-              State.SkipWithError("Inserted a duplicate element");
-          } else {
-            if (!Inserted)
-              State.SkipWithError("Failed to insert e new element");
-          }
-#endif
-        }
-      }
-
-      State.PauseTiming();
-      Data = makeTestingSets(MapSize, Mode(), Order::value == ::Order::Random ? Shuffle::Keys : Shuffle::None, 1000);
-      State.ResumeTiming();
-    }
-  }
-
-  std::string name() const { return "BM_InsertAssign" + baseName() + Mode::name() + Order::name(); }
-};
-
-template <class Mode, class Hint>
-struct InsertAssignHint : Base {
-  using Base::Base;
-
-  template < ::Hint hint>
-  typename std::enable_if<hint == ::Hint::Correct>::type run(benchmark::State& State) const {
-    auto Data = makeTestingSets(MapSize, Mode(), Shuffle::None, 1000);
-    while (State.KeepRunningBatch(MapSize * Data.Maps.size())) {
-      for (size_t I = 0; I < Data.Maps.size(); ++I) {
-        auto& Map = Data.Maps[I];
-        auto H    = Data.Hints[I].begin();
-        for (auto K : Data.Keys) {
-#ifndef VALIDATE
-          benchmark::DoNotOptimize(Map.insert_or_assign(*H, K, 1));
-#else
-          auto Inserted = Map.insert_or_assign(*H, K, 1);
-          if (Mode() == ::Mode::Hit) {
-            if (Inserted != *H)
-              State.SkipWithError("Inserted a duplicate element");
-          } else {
-            if (++Inserted != *H)
-              State.SkipWithError("Failed to insert a new element");
-          }
-#endif
-          ++H;
-        }
-      }
-
-      State.PauseTiming();
-      Data = makeTestingSets(MapSize, Mode(), Shuffle::None, 1000);
-      State.ResumeTiming();
-    }
-  }
-
-  template < ::Hint hint>
-  typename std::enable_if<hint != ::Hint::Correct>::type run(benchmark::State& State) const {
-    auto Data = makeTestingSets(MapSize, Mode(), Shuffle::None, 1000);
-    while (State.KeepRunningBatch(MapSize * Data.Maps.size())) {
-      for (size_t I = 0; I < Data.Maps.size(); ++I) {
-        auto& Map  = Data.Maps[I];
-        auto Third = *(Data.Hints[I].begin() + 2);
-        for (auto K : Data.Keys) {
-          auto Itor = hint == ::Hint::Begin ? Map.begin() : hint == ::Hint::Third ? Third : Map.end();
-#ifndef VALIDATE
-          benchmark::DoNotOptimize(Map.insert_or_assign(Itor, K, 1));
-#else
-          size_t Size = Map.size();
-          Map.insert_or_assign(Itor, K, 1);
-          if (Mode() == ::Mode::Hit) {
-            if (Size != Map.size())
-              State.SkipWithError("Inserted a duplicate element");
-          } else {
-            if (Size + 1 != Map.size())
-              State.SkipWithError("Failed to insert a new element");
-          }
-#endif
-        }
-      }
-
-      State.PauseTiming();
-      Data = makeTestingSets(MapSize, Mode(), Shuffle::None, 1000);
-      State.ResumeTiming();
-    }
-  }
-
-  void run(benchmark::State& State) const {
-    static constexpr auto h = Hint();
-    run<h>(State);
-  }
-
-  std::string name() const { return "BM_InsertAssignHint" + baseName() + Mode::name() + Hint::name(); }
-};
-
-template <class Mode, class Order>
-struct Emplace : Base {
-  using Base::Base;
-
-  void run(benchmark::State& State) const {
-    auto Data = makeTestingSets(MapSize, Mode(), Order::value == ::Order::Random ? Shuffle::Keys : Shuffle::None, 1000);
-    while (State.KeepRunningBatch(MapSize * Data.Maps.size())) {
-      for (auto& Map : Data.Maps) {
-        for (auto K : Data.Keys) {
-#ifndef VALIDATE
-          benchmark::DoNotOptimize(Map.emplace(K, 1));
-#else
-          bool Inserted = Map.emplace(K, 1).second;
-          if (Mode() == ::Mode::Hit) {
-            if (Inserted)
-              State.SkipWithError("Emplaced a duplicate element");
-          } else {
-            if (!Inserted)
-              State.SkipWithError("Failed to emplace a new element");
-          }
-#endif
-        }
-      }
-
-      State.PauseTiming();
-      Data = makeTestingSets(MapSize, Mode(), Order::value == ::Order::Random ? Shuffle::Keys : Shuffle::None, 1000);
-      State.ResumeTiming();
-    }
-  }
-
-  std::string name() const { return "BM_Emplace" + baseName() + Mode::name() + Order::name(); }
-};
-
-template <class Mode, class Hint>
-struct EmplaceHint : Base {
-  using Base::Base;
-
-  template < ::Hint hint>
-  typename std::enable_if<hint == ::Hint::Correct>::type run(benchmark::State& State) const {
-    auto Data = makeTestingSets(MapSize, Mode(), Shuffle::None, 1000);
-    while (State.KeepRunningBatch(MapSize * Data.Maps.size())) {
-      for (size_t I = 0; I < Data.Maps.size(); ++I) {
-        auto& Map = Data.Maps[I];
-        auto H    = Data.Hints[I].begin();
-        for (auto K : Data.Keys) {
-#ifndef VALIDATE
-          benchmark::DoNotOptimize(Map.emplace_hint(*H, K, 1));
-#else
-          auto Inserted = Map.emplace_hint(*H, K, 1);
-          if (Mode() == ::Mode::Hit) {
-            if (Inserted != *H)
-              State.SkipWithError("Emplaced a duplicate element");
-          } else {
-            if (++Inserted != *H)
-              State.SkipWithError("Failed to emplace a new element");
-          }
-#endif
-          ++H;
-        }
-      }
-
-      State.PauseTiming();
-      Data = makeTestingSets(MapSize, Mode(), Shuffle::None, 1000);
-      State.ResumeTiming();
-    }
-  }
-
-  template < ::Hint hint>
-  typename std::enable_if<hint != ::Hint::Correct>::type run(benchmark::State& State) const {
-    auto Data = makeTestingSets(MapSize, Mode(), Shuffle::None, 1000);
-    while (State.KeepRunningBatch(MapSize * Data.Maps.size())) {
-      for (size_t I = 0; I < Data.Maps.size(); ++I) {
-        auto& Map  = Data.Maps[I];
-        auto Third = *(Data.Hints[I].begin() + 2);
-        for (auto K : Data.Keys) {
-          auto Itor = hint == ::Hint::Begin ? Map.begin() : hint == ::Hint::Third ? Third : Map.end();
-#ifndef VALIDATE
-          benchmark::DoNotOptimize(Map.emplace_hint(Itor, K, 1));
-#else
-          size_t Size = Map.size();
-          Map.emplace_hint(Itor, K, 1);
-          if (Mode() == ::Mode::Hit) {
-            if (Size != Map.size())
-              State.SkipWithError("Emplaced a duplicate element");
-          } else {
-            if (Size + 1 != Map.size())
-              State.SkipWithError("Failed to emplace a new element");
-          }
-#endif
-        }
-      }
-
-      State.PauseTiming();
-      Data = makeTestingSets(MapSize, Mode(), Shuffle::None, 1000);
-      State.ResumeTiming();
-    }
-  }
-
-  void run(benchmark::State& State) const {
-    static constexpr auto h = Hint();
-    run<h>(State);
-  }
-
-  std::string name() const { return "BM_EmplaceHint" + baseName() + Mode::name() + Hint::name(); }
-};
-
-template <class Mode, class Order>
-struct TryEmplace : Base {
-  using Base::Base;
-
-  void run(benchmark::State& State) const {
-    auto Data = makeTestingSets(MapSize, Mode(), Order::value == ::Order::Random ? Shuffle::Keys : Shuffle::None, 1000);
-    while (State.KeepRunningBatch(MapSize * Data.Maps.size())) {
-      for (auto& Map : Data.Maps) {
-        for (auto K : Data.Keys) {
-#ifndef VALIDATE
-          benchmark::DoNotOptimize(Map.try_emplace(K, 1));
-#else
-          bool Inserted = Map.try_emplace(K, 1).second;
-          if (Mode() == ::Mode::Hit) {
-            if (Inserted)
-              State.SkipWithError("Emplaced a duplicate element");
-          } else {
-            if (!Inserted)
-              State.SkipWithError("Failed to emplace a new element");
-          }
-#endif
-        }
-      }
-
-      State.PauseTiming();
-      Data = makeTestingSets(MapSize, Mode(), Order::value == ::Order::Random ? Shuffle::Keys : Shuffle::None, 1000);
-      State.ResumeTiming();
-    }
-  }
-
-  std::string name() const { return "BM_TryEmplace" + baseName() + Mode::name() + Order::name(); }
-};
-
-template <class Mode, class Hint>
-struct TryEmplaceHint : Base {
-  using Base::Base;
-
-  template < ::Hint hint>
-  typename std::enable_if<hint == ::Hint::Correct>::type run(benchmark::State& State) const {
-    auto Data = makeTestingSets(MapSize, Mode(), Shuffle::None, 1000);
-    while (State.KeepRunningBatch(MapSize * Data.Maps.size())) {
-      for (size_t I = 0; I < Data.Maps.size(); ++I) {
-        auto& Map = Data.Maps[I];
-        auto H    = Data.Hints[I].begin();
-        for (auto K : Data.Keys) {
-#ifndef VALIDATE
-          benchmark::DoNotOptimize(Map.try_emplace(*H, K, 1));
-#else
-          auto Inserted = Map.try_emplace(*H, K, 1);
-          if (Mode() == ::Mode::Hit) {
-            if (Inserted != *H)
-              State.SkipWithError("Emplaced a duplicate element");
-          } else {
-            if (++Inserted != *H)
-              State.SkipWithError("Failed to emplace a new element");
-          }
-#endif
-          ++H;
-        }
-      }
-
-      State.PauseTiming();
-      Data = makeTestingSets(MapSize, Mode(), Shuffle::None, 1000);
-      State.ResumeTiming();
-    }
-  }
-
-  template < ::Hint hint>
-  typename std::enable_if<hint != ::Hint::Correct>::type run(benchmark::State& State) const {
-    auto Data = makeTestingSets(MapSize, Mode(), Shuffle::None, 1000);
-    while (State.KeepRunningBatch(MapSize * Data.Maps.size())) {
-      for (size_t I = 0; I < Data.Maps.size(); ++I) {
-        auto& Map  = Data.Maps[I];
-        auto Third = *(Data.Hints[I].begin() + 2);
-        for (auto K : Data.Keys) {
-          auto Itor = hint == ::Hint::Begin ? Map.begin() : hint == ::Hint::Third ? Third : Map.end();
-#ifndef VALIDATE
-          benchmark::DoNotOptimize(Map.try_emplace(Itor, K, 1));
-#else
-          size_t Size = Map.size();
-          Map.try_emplace(Itor, K, 1);
-          if (Mode() == ::Mode::Hit) {
-            if (Size != Map.size())
-              State.SkipWithError("Emplaced a duplicate element");
-          } else {
-            if (Size + 1 != Map.size())
-              State.SkipWithError("Failed to emplace a new element");
-          }
-#endif
-        }
-      }
-
-      State.PauseTiming();
-      Data = makeTestingSets(MapSize, Mode(), Shuffle::None, 1000);
-      State.ResumeTiming();
-    }
-  }
-
-  void run(benchmark::State& State) const {
-    static constexpr auto h = Hint();
-    run<h>(State);
-  }
-
-  std::string name() const { return "BM_TryEmplaceHint" + baseName() + Mode::name() + Hint::name(); }
-};
-
-template <class Mode, class Order>
-struct Erase : Base {
-  using Base::Base;
-
-  void run(benchmark::State& State) const {
-    auto Data = makeTestingSets(MapSize, Mode(), Order::value == ::Order::Random ? Shuffle::Keys : Shuffle::None, 1000);
-    while (State.KeepRunningBatch(MapSize * Data.Maps.size())) {
-      for (auto& Map : Data.Maps) {
-        for (auto K : Data.Keys) {
-#ifndef VALIDATE
-          benchmark::DoNotOptimize(Map.erase(K));
-#else
-          size_t I = Map.erase(K);
-          if (Mode() == ::Mode::Hit) {
-            if (I == 0)
-              State.SkipWithError("Did not find the existing element");
-          } else {
-            if (I == 1)
-              State.SkipWithError("Did find the non-existing element");
-          }
-#endif
-        }
-      }
-
-      State.PauseTiming();
-      Data = makeTestingSets(MapSize, Mode(), Order::value == ::Order::Random ? Shuffle::Keys : Shuffle::None, 1000);
-      State.ResumeTiming();
-    }
-  }
-
-  std::string name() const { return "BM_Erase" + baseName() + Mode::name() + Order::name(); }
-};
-
-template <class Order>
-struct EraseIterator : Base {
-  using Base::Base;
-
-  void run(benchmark::State& State) const {
-    auto Data =
-        makeTestingSets(MapSize, Mode::Hit, Order::value == ::Order::Random ? Shuffle::Hints : Shuffle::None, 1000);
-    while (State.KeepRunningBatch(MapSize * Data.Maps.size())) {
-      for (size_t I = 0; I < Data.Maps.size(); ++I) {
-        auto& Map = Data.Maps[I];
-        for (auto H : Data.Hints[I]) {
-          benchmark::DoNotOptimize(Map.erase(H));
-        }
-#ifdef VALIDATE
-        if (!Map.empty())
-          State.SkipWithError("Did not erase the entire map");
-#endif
-      }
-
-      State.PauseTiming();
-      Data =
-          makeTestingSets(MapSize, Mode::Hit, Order::value == ::Order::Random ? Shuffle::Hints : Shuffle::None, 1000);
-      State.ResumeTiming();
-    }
-  }
-
-  std::string name() const { return "BM_EraseIterator" + baseName() + Order::name(); }
-};
-
-struct EraseRange : Base {
-  using Base::Base;
-
-  void run(benchmark::State& State) const {
-    auto Data = makeTestingSets(MapSize, Mode::Hit, Shuffle::None, 1000);
-    while (State.KeepRunningBatch(MapSize * Data.Maps.size())) {
-      for (auto& Map : Data.Maps) {
-#ifndef VALIDATE
-        benchmark::DoNotOptimize(Map.erase(Map.begin(), Map.end()));
-#else
-        Map.erase(Map.begin(), Map.end());
-        if (!Map.empty())
-          State.SkipWithError("Did not erase the entire map");
-#endif
-      }
-
-      State.PauseTiming();
-      Data = makeTestingSets(MapSize, Mode::Hit, Shuffle::None, 1000);
-      State.ResumeTiming();
-    }
-  }
-
-  std::string name() const { return "BM_EraseRange" + baseName(); }
-};
-
-//*******************************************************************|
-//                            Lookup                                 |
-//*******************************************************************|
-
-template <class Mode, class Order>
-struct Count : Base {
-  using Base::Base;
-
-  void run(benchmark::State& State) const {
-    auto Data = makeTestingSets(MapSize, Mode(), Order::value == ::Order::Random ? Shuffle::Keys : Shuffle::None, 1);
-    auto& Map = Data.Maps.front();
-    while (State.KeepRunningBatch(MapSize)) {
-      for (auto K : Data.Keys) {
-#ifndef VALIDATE
-        benchmark::DoNotOptimize(Map.count(K));
-#else
-        size_t I = Map.count(K);
-        if (Mode() == ::Mode::Hit) {
-          if (I == 0)
-            State.SkipWithError("Did not find the existing element");
-        } else {
-          if (I == 1)
-            State.SkipWithError("Did find the non-existing element");
-        }
-#endif
-      }
-    }
-  }
-
-  std::string name() const { return "BM_Count" + baseName() + Mode::name() + Order::name(); }
-};
-
-template <class Mode, class Order>
-struct Find : Base {
-  using Base::Base;
-
-  void run(benchmark::State& State) const {
-    auto Data = makeTestingSets(MapSize, Mode(), Order::value == ::Order::Random ? Shuffle::Keys : Shuffle::None, 1);
-    auto& Map = Data.Maps.front();
-    while (State.KeepRunningBatch(MapSize)) {
-      for (auto K : Data.Keys) {
-#ifndef VALIDATE
-        benchmark::DoNotOptimize(Map.find(K));
-#else
-        auto Itor = Map.find(K);
-        if (Mode() == ::Mode::Hit) {
-          if (Itor == Map.end())
-            State.SkipWithError("Did not find the existing element");
-        } else {
-          if (Itor != Map.end())
-            State.SkipWithError("Did find the non-existing element");
-        }
-#endif
-      }
-    }
-  }
-
-  std::string name() const { return "BM_Find" + baseName() + Mode::name() + Order::name(); }
-};
-
-template <class Mode, class Order>
-struct EqualRange : Base {
-  using Base::Base;
-
-  void run(benchmark::State& State) const {
-    auto Data = makeTestingSets(MapSize, Mode(), Order::value == ::Order::Random ? Shuffle::Keys : Shuffle::None, 1);
-    auto& Map = Data.Maps.front();
-    while (State.KeepRunningBatch(MapSize)) {
-      for (auto K : Data.Keys) {
-#ifndef VALIDATE
-        benchmark::DoNotOptimize(Map.equal_range(K));
-#else
-        auto Range = Map.equal_range(K);
-        if (Mode() == ::Mode::Hit) {
-          // Adjust validation for the last element.
-          auto Key = K;
-          if (Range.second == Map.end() && K == 2 * MapSize) {
-            --Range.second;
-            Key -= 2;
-          }
-          if (Range.first == Map.end() || Range.first->first != K || Range.second == Map.end() ||
-              Range.second->first - 2 != Key)
-            State.SkipWithError("Did not find the existing element");
-        } else {
-          if (Range.first == Map.end() || Range.first->first - 1 != K || Range.second == Map.end() ||
-              Range.second->first - 1 != K)
-            State.SkipWithError("Did find the non-existing element");
-        }
-#endif
-      }
-    }
-  }
-
-  std::string name() const { return "BM_EqualRange" + baseName() + Mode::name() + Order::name(); }
-};
-
-template <class Mode, class Order>
-struct LowerBound : Base {
-  using Base::Base;
-
-  void run(benchmark::State& State) const {
-    auto Data = makeTestingSets(MapSize, Mode(), Order::value == ::Order::Random ? Shuffle::Keys : Shuffle::None, 1);
-    auto& Map = Data.Maps.front();
-    while (State.KeepRunningBatch(MapSize)) {
-      for (auto K : Data.Keys) {
-#ifndef VALIDATE
-        benchmark::DoNotOptimize(Map.lower_bound(K));
-#else
-        auto Itor = Map.lower_bound(K);
-        if (Mode() == ::Mode::Hit) {
-          if (Itor == Map.end() || Itor->first != K)
-            State.SkipWithError("Did not find the existing element");
-        } else {
-          if (Itor == Map.end() || Itor->first - 1 != K)
-            State.SkipWithError("Did find the non-existing element");
-        }
-#endif
-      }
-    }
-  }
-
-  std::string name() const { return "BM_LowerBound" + baseName() + Mode::name() + Order::name(); }
-};
-
-template <class Mode, class Order>
-struct UpperBound : Base {
-  using Base::Base;
-
-  void run(benchmark::State& State) const {
-    auto Data = makeTestingSets(MapSize, Mode(), Order::value == ::Order::Random ? Shuffle::Keys : Shuffle::None, 1);
-    auto& Map = Data.Maps.front();
-    while (State.KeepRunningBatch(MapSize)) {
-      for (auto K : Data.Keys) {
-#ifndef VALIDATE
-        benchmark::DoNotOptimize(Map.upper_bound(K));
-#else
-        std::map<uint64_t, int64_t>::iterator Itor = Map.upper_bound(K);
-        if (Mode() == ::Mode::Hit) {
-          // Adjust validation for the last element.
-          auto Key = K;
-          if (Itor == Map.end() && K == 2 * MapSize) {
-            --Itor;
-            Key -= 2;
-          }
-          if (Itor == Map.end() || Itor->first - 2 != Key)
-            State.SkipWithError("Did not find the existing element");
-        } else {
-          if (Itor == Map.end() || Itor->first - 1 != K)
-            State.SkipWithError("Did find the non-existing element");
-        }
-#endif
-      }
-    }
-  }
-
-  std::string name() const { return "BM_UpperBound" + baseName() + Mode::name() + Order::name(); }
-};
-
-} // namespace
-
-int main(int argc, char** argv) {
-  benchmark::Initialize(&argc, argv);
-  if (benchmark::ReportUnrecognizedArguments(argc, argv))
-    return 1;
-
-#ifdef VALIDATE
-  const std::vector<size_t> MapSize{10};
-#else
-  const std::vector<size_t> MapSize{10, 100, 1000, 10000, 100000, 1000000};
-#endif
-
-  // Member functions
-  makeCartesianProductBenchmark<ConstructorDefault>();
-  makeCartesianProductBenchmark<ConstructorIterator>(MapSize);
-  makeCartesianProductBenchmark<ConstructorCopy>(MapSize);
-  makeCartesianProductBenchmark<ConstructorMove>(MapSize);
-
-  // Capacity
-  makeCartesianProductBenchmark<Empty>(MapSize);
-  makeCartesianProductBenchmark<Size>(MapSize);
-
-  // Modifiers
-  makeCartesianProductBenchmark<Clear>(MapSize);
-  makeCartesianProductBenchmark<Insert, AllModes, AllOrders>(MapSize);
-  makeCartesianProductBenchmark<InsertHint, AllModes, AllHints>(MapSize);
-  makeCartesianProductBenchmark<InsertAssign, AllModes, AllOrders>(MapSize);
-  makeCartesianProductBenchmark<InsertAssignHint, AllModes, AllHints>(MapSize);
-
-  makeCartesianProductBenchmark<Emplace, AllModes, AllOrders>(MapSize);
-  makeCartesianProductBenchmark<EmplaceHint, AllModes, AllHints>(MapSize);
-  makeCartesianProductBenchmark<TryEmplace, AllModes, AllOrders>(MapSize);
-  makeCartesianProductBenchmark<TryEmplaceHint, AllModes, AllHints>(MapSize);
-  makeCartesianProductBenchmark<Erase, AllModes, AllOrders>(MapSize);
-  makeCartesianProductBenchmark<EraseIterator, AllOrders>(MapSize);
-  makeCartesianProductBenchmark<EraseRange>(MapSize);
-
-  // Lookup
-  makeCartesianProductBenchmark<Count, AllModes, AllOrders>(MapSize);
-  makeCartesianProductBenchmark<Find, AllModes, AllOrders>(MapSize);
-  makeCartesianProductBenchmark<EqualRange, AllModes, AllOrders>(MapSize);
-  makeCartesianProductBenchmark<LowerBound, AllModes, AllOrders>(MapSize);
-  makeCartesianProductBenchmark<UpperBound, AllModes, AllOrders>(MapSize);
-
-  benchmark::RunSpecifiedBenchmarks();
-}
diff --git a/libcxx/test/benchmarks/containers/ordered_set.bench.cpp b/libcxx/test/benchmarks/containers/ordered_set.bench.cpp
deleted file mode 100644
index cb68902c6dcc8..0000000000000
--- a/libcxx/test/benchmarks/containers/ordered_set.bench.cpp
+++ /dev/null
@@ -1,232 +0,0 @@
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-// UNSUPPORTED: c++03, c++11, c++14
-
-#include <algorithm>
-#include <cstdint>
-#include <memory>
-#include <numeric>
-#include <random>
-#include <set>
-#include <string>
-#include <vector>
-
-#include "../CartesianBenchmarks.h"
-#include "benchmark/benchmark.h"
-#include "test_macros.h"
-
-namespace {
-
-enum class HitType { Hit, Miss };
-
-struct AllHitTypes : EnumValuesAsTuple<AllHitTypes, HitType, 2> {
-  static constexpr const char* Names[] = {"Hit", "Miss"};
-};
-
-enum class AccessPattern { Ordered, Random };
-
-struct AllAccessPattern : EnumValuesAsTuple<AllAccessPattern, AccessPattern, 2> {
-  static constexpr const char* Names[] = {"Ordered", "Random"};
-};
-
-void sortKeysBy(std::vector<uint64_t>& Keys, AccessPattern AP) {
-  if (AP == AccessPattern::Random) {
-    std::random_device R;
-    std::mt19937 M(R());
-    std::shuffle(std::begin(Keys), std::end(Keys), M);
-  }
-}
-
-struct TestSets {
-  std::vector<std::set<uint64_t> > Sets;
-  std::vector<uint64_t> Keys;
-};
-
-TestSets makeTestingSets(size_t TableSize, size_t NumTables, HitType Hit, AccessPattern Access) {
-  TestSets R;
-  R.Sets.resize(1);
-
-  for (uint64_t I = 0; I < TableSize; ++I) {
-    R.Sets[0].insert(2 * I);
-    R.Keys.push_back(Hit == HitType::Hit ? 2 * I : 2 * I + 1);
-  }
-  R.Sets.resize(NumTables, R.Sets[0]);
-  sortKeysBy(R.Keys, Access);
-
-  return R;
-}
-
-struct Base {
-  size_t TableSize;
-  size_t NumTables;
-  Base(size_t T, size_t N) : TableSize(T), NumTables(N) {}
-
-  bool skip() const {
-    size_t Total = TableSize * NumTables;
-    return Total < 100 || Total > 1000000;
-  }
-
-  std::string baseName() const {
-    return "_TableSize" + std::to_string(TableSize) + "_NumTables" + std::to_string(NumTables);
-  }
-};
-
-template <class Access>
-struct Create : Base {
-  using Base::Base;
-
-  void run(benchmark::State& State) const {
-    std::vector<uint64_t> Keys(TableSize);
-    std::iota(Keys.begin(), Keys.end(), uint64_t{0});
-    sortKeysBy(Keys, Access());
-
-    while (State.KeepRunningBatch(TableSize * NumTables)) {
-      std::vector<std::set<uint64_t>> Sets(NumTables);
-      for (auto K : Keys) {
-        for (auto& Set : Sets) {
-          benchmark::DoNotOptimize(Set.insert(K));
-        }
-      }
-    }
-  }
-
-  std::string name() const { return "BM_Create" + Access::name() + baseName(); }
-};
-
-template <class Hit, class Access>
-struct Find : Base {
-  using Base::Base;
-
-  void run(benchmark::State& State) const {
-    auto Data = makeTestingSets(TableSize, NumTables, Hit(), Access());
-
-    while (State.KeepRunningBatch(TableSize * NumTables)) {
-      for (auto K : Data.Keys) {
-        for (auto& Set : Data.Sets) {
-          benchmark::DoNotOptimize(Set.find(K));
-        }
-      }
-    }
-  }
-
-  std::string name() const { return "BM_Find" + Hit::name() + Access::name() + baseName(); }
-};
-
-template <class Hit, class Access>
-struct FindNeEnd : Base {
-  using Base::Base;
-
-  void run(benchmark::State& State) const {
-    auto Data = makeTestingSets(TableSize, NumTables, Hit(), Access());
-
-    while (State.KeepRunningBatch(TableSize * NumTables)) {
-      for (auto K : Data.Keys) {
-        for (auto& Set : Data.Sets) {
-          benchmark::DoNotOptimize(Set.find(K) != Set.end());
-        }
-      }
-    }
-  }
-
-  std::string name() const { return "BM_FindNeEnd" + Hit::name() + Access::name() + baseName(); }
-};
-
-template <class Access>
-struct InsertHit : Base {
-  using Base::Base;
-
-  void run(benchmark::State& State) const {
-    auto Data = makeTestingSets(TableSize, NumTables, HitType::Hit, Access());
-
-    while (State.KeepRunningBatch(TableSize * NumTables)) {
-      for (auto K : Data.Keys) {
-        for (auto& Set : Data.Sets) {
-          benchmark::DoNotOptimize(Set.insert(K));
-        }
-      }
-    }
-  }
-
-  std::string name() const { return "BM_InsertHit" + Access::name() + baseName(); }
-};
-
-template <class Access>
-struct InsertMissAndErase : Base {
-  using Base::Base;
-
-  void run(benchmark::State& State) const {
-    auto Data = makeTestingSets(TableSize, NumTables, HitType::Miss, Access());
-
-    while (State.KeepRunningBatch(TableSize * NumTables)) {
-      for (auto K : Data.Keys) {
-        for (auto& Set : Data.Sets) {
-          benchmark::DoNotOptimize(Set.erase(Set.insert(K).first));
-        }
-      }
-    }
-  }
-
-  std::string name() const { return "BM_InsertMissAndErase" + Access::name() + baseName(); }
-};
-
-struct IterateRangeFor : Base {
-  using Base::Base;
-
-  void run(benchmark::State& State) const {
-    auto Data = makeTestingSets(TableSize, NumTables, HitType::Miss, AccessPattern::Ordered);
-
-    while (State.KeepRunningBatch(TableSize * NumTables)) {
-      for (auto& Set : Data.Sets) {
-        for (auto& V : Set) {
-          benchmark::DoNotOptimize(const_cast<std::set<uint64_t>::reference>(V));
-        }
-      }
-    }
-  }
-
-  std::string name() const { return "BM_IterateRangeFor" + baseName(); }
-};
-
-struct IterateBeginEnd : Base {
-  using Base::Base;
-
-  void run(benchmark::State& State) const {
-    auto Data = makeTestingSets(TableSize, NumTables, HitType::Miss, AccessPattern::Ordered);
-
-    while (State.KeepRunningBatch(TableSize * NumTables)) {
-      for (auto& Set : Data.Sets) {
-        for (auto it = Set.begin(); it != Set.end(); ++it) {
-          benchmark::DoNotOptimize(const_cast<std::set<uint64_t>::reference>(*it));
-        }
-      }
-    }
-  }
-
-  std::string name() const { return "BM_IterateBeginEnd" + baseName(); }
-};
-
-} // namespace
-
-int main(int argc, char** argv) {
-  benchmark::Initialize(&argc, argv);
-  if (benchmark::ReportUnrecognizedArguments(argc, argv))
-    return 1;
-
-  const std::vector<size_t> TableSize{1, 10, 100, 1000, 10000, 100000, 1000000};
-  const std::vector<size_t> NumTables{1, 10, 100, 1000, 10000, 100000, 1000000};
-
-  makeCartesianProductBenchmark<Create, AllAccessPattern>(TableSize, NumTables);
-  makeCartesianProductBenchmark<Find, AllHitTypes, AllAccessPattern>(TableSize, NumTables);
-  makeCartesianProductBenchmark<FindNeEnd, AllHitTypes, AllAccessPattern>(TableSize, NumTables);
-  makeCartesianProductBenchmark<InsertHit, AllAccessPattern>(TableSize, NumTables);
-  makeCartesianProductBenchmark<InsertMissAndErase, AllAccessPattern>(TableSize, NumTables);
-  makeCartesianProductBenchmark<IterateRangeFor>(TableSize, NumTables);
-  makeCartesianProductBenchmark<IterateBeginEnd>(TableSize, NumTables);
-  benchmark::RunSpecifiedBenchmarks();
-}
diff --git a/libcxx/test/benchmarks/containers/sequence/deque.bench.cpp b/libcxx/test/benchmarks/containers/sequence/deque.bench.cpp
new file mode 100644
index 0000000000000..e37c9fef4ac23
--- /dev/null
+++ b/libcxx/test/benchmarks/containers/sequence/deque.bench.cpp
@@ -0,0 +1,25 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17
+
+#include <deque>
+#include <string>
+
+#include "sequence_container_benchmarks.h"
+#include "benchmark/benchmark.h"
+
+int main(int argc, char** argv) {
+  support::sequence_container_benchmarks<std::deque<int>>("std::deque<int>");
+  support::sequence_container_benchmarks<std::deque<std::string>>("std::deque<std::string>");
+
+  benchmark::Initialize(&argc, argv);
+  benchmark::RunSpecifiedBenchmarks();
+  benchmark::Shutdown();
+  return 0;
+}
diff --git a/libcxx/test/benchmarks/containers/sequence/list.bench.cpp b/libcxx/test/benchmarks/containers/sequence/list.bench.cpp
new file mode 100644
index 0000000000000..e40aae6cf9fa5
--- /dev/null
+++ b/libcxx/test/benchmarks/containers/sequence/list.bench.cpp
@@ -0,0 +1,25 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17
+
+#include <list>
+#include <string>
+
+#include "sequence_container_benchmarks.h"
+#include "benchmark/benchmark.h"
+
+int main(int argc, char** argv) {
+  support::sequence_container_benchmarks<std::list<int>>("std::list<int>");
+  support::sequence_container_benchmarks<std::list<std::string>>("std::list<std::string>");
+
+  benchmark::Initialize(&argc, argv);
+  benchmark::RunSpecifiedBenchmarks();
+  benchmark::Shutdown();
+  return 0;
+}
diff --git a/libcxx/test/benchmarks/containers/sequence/sequence_container_benchmarks.h b/libcxx/test/benchmarks/containers/sequence/sequence_container_benchmarks.h
new file mode 100644
index 0000000000000..dcd251d6997dd
--- /dev/null
+++ b/libcxx/test/benchmarks/containers/sequence/sequence_container_benchmarks.h
@@ -0,0 +1,455 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef TEST_BENCHMARKS_CONTAINERS_SEQUENCE_SEQUENCE_CONTAINER_BENCHMARKS_H
+#define TEST_BENCHMARKS_CONTAINERS_SEQUENCE_SEQUENCE_CONTAINER_BENCHMARKS_H
+
+#include <algorithm>
+#include <cassert>
+#include <cstddef>
+#include <iterator>
+#include <ranges> // for std::from_range
+#include <string>
+#include <type_traits>
+#include <vector>
+
+#include "benchmark/benchmark.h"
+#include "test_iterators.h"
+#include "../../GenerateInput.h"
+
+namespace support {
+
+template <class Container>
+void DoNotOptimizeData(Container& c) {
+  if constexpr (requires { c.data(); }) {
+    benchmark::DoNotOptimize(c.data());
+  } else {
+    benchmark::DoNotOptimize(&c);
+  }
+}
+
+template <class Container>
+void sequence_container_benchmarks(std::string container) {
+  using ValueType = typename Container::value_type;
+
+  using Generator     = ValueType (*)();
+  Generator cheap     = [] { return Generate<ValueType>::cheap(); };
+  Generator expensive = [] { return Generate<ValueType>::expensive(); };
+  auto tostr          = [&](Generator gen) -> std::string {
+    return gen == cheap ? " (cheap elements)" : " (expensive elements)";
+  };
+  std::vector<Generator> generators;
+  generators.push_back(cheap);
+  if constexpr (!std::is_integral_v<ValueType>) {
+    generators.push_back(expensive);
+  }
+
+  // Some of these benchmarks are structured to perform the operation being benchmarked
+  // a small number of times at each iteration, in order to offset the cost of
+  // PauseTiming() and ResumeTiming().
+  static constexpr std::size_t BatchSize = 32;
+
+  auto bench = [&](std::string operation, auto f) {
+    benchmark::RegisterBenchmark(container + "::" + operation, f)->Arg(32)->Arg(1024)->Arg(8192);
+  };
+
+  /////////////////////////
+  // Constructors
+  /////////////////////////
+  if constexpr (std::is_constructible_v<Container, std::size_t>) {
+    // not all containers provide this constructor
+    bench("ctor(size)", [](auto& st) {
+      auto const size = st.range(0);
+
+      for ([[maybe_unused]] auto _ : st) {
+        Container c(size); // we assume the destructor doesn't dominate the benchmark
+        DoNotOptimizeData(c);
+      }
+    });
+  }
+
+  for (auto gen : generators)
+    bench("ctor(size, value_type)" + tostr(gen), [gen](auto& st) {
+      auto const size = st.range(0);
+      ValueType value = gen();
+      benchmark::DoNotOptimize(value);
+
+      for ([[maybe_unused]] auto _ : st) {
+        Container c(size, value); // we assume the destructor doesn't dominate the benchmark
+        DoNotOptimizeData(c);
+      }
+    });
+
+  for (auto gen : generators)
+    bench("ctor(Iterator, Iterator)" + tostr(gen), [gen](auto& st) {
+      auto const size = st.range(0);
+      std::vector<ValueType> in;
+      std::generate_n(std::back_inserter(in), size, gen);
+      const auto begin = in.begin();
+      const auto end   = in.end();
+      benchmark::DoNotOptimize(in);
+
+      for ([[maybe_unused]] auto _ : st) {
+        Container c(begin, end); // we assume the destructor doesn't dominate the benchmark
+        DoNotOptimizeData(c);
+      }
+    });
+
+#if defined(__cpp_lib_containers_ranges) && __cpp_lib_containers_ranges >= 202202L
+  for (auto gen : generators)
+    bench("ctor(Range)" + tostr(gen), [gen](auto& st) {
+      auto const size = st.range(0);
+      std::vector<ValueType> in;
+      std::generate_n(std::back_inserter(in), size, gen);
+      benchmark::DoNotOptimize(in);
+
+      for ([[maybe_unused]] auto _ : st) {
+        Container c(std::from_range, in); // we assume the destructor doesn't dominate the benchmark
+        DoNotOptimizeData(c);
+      }
+    });
+#endif
+
+  for (auto gen : generators)
+    bench("ctor(const&)" + tostr(gen), [gen](auto& st) {
+      auto const size = st.range(0);
+      Container in;
+      std::generate_n(std::back_inserter(in), size, gen);
+      DoNotOptimizeData(in);
+
+      for ([[maybe_unused]] auto _ : st) {
+        Container c(in); // we assume the destructor doesn't dominate the benchmark
+        DoNotOptimizeData(c);
+        DoNotOptimizeData(in);
+      }
+    });
+
+  /////////////////////////
+  // Assignment
+  /////////////////////////
+  for (auto gen : generators)
+    bench("operator=(const&)" + tostr(gen), [gen](auto& st) {
+      auto const size = st.range(0);
+      Container in1, in2;
+      std::generate_n(std::back_inserter(in1), size, gen);
+      std::generate_n(std::back_inserter(in2), size, gen);
+      DoNotOptimizeData(in1);
+      DoNotOptimizeData(in2);
+
+      // Assign from one of two containers in succession to avoid
+      // hitting a self-assignment corner-case
+      Container c(in1);
+      bool toggle = false;
+      for ([[maybe_unused]] auto _ : st) {
+        c      = toggle ? in1 : in2;
+        toggle = !toggle;
+        DoNotOptimizeData(c);
+        DoNotOptimizeData(in1);
+        DoNotOptimizeData(in2);
+      }
+    });
+
+  // Benchmark Container::assign(input-iter, input-iter) when the container already contains
+  // the same number of elements that we're assigning. The intent is to check whether the
+  // implementation basically creates a new container from scratch or manages to reuse the
+  // pre-existing storage.
+  for (auto gen : generators)
+    bench("assign(input-iter, input-iter) (full container)" + tostr(gen), [gen](auto& st) {
+      auto const size = st.range(0);
+      std::vector<ValueType> in1, in2;
+      std::generate_n(std::back_inserter(in1), size, gen);
+      std::generate_n(std::back_inserter(in2), size, gen);
+      DoNotOptimizeData(in1);
+      DoNotOptimizeData(in2);
+
+      Container c(in1.begin(), in1.end());
+      bool toggle = false;
+      for ([[maybe_unused]] auto _ : st) {
+        std::vector<ValueType>& in = toggle ? in1 : in2;
+        auto first                 = in.data();
+        auto last                  = in.data() + in.size();
+        c.assign(cpp17_input_iterator(first), cpp17_input_iterator(last));
+        toggle = !toggle;
+        DoNotOptimizeData(c);
+      }
+    });
+
+  /////////////////////////
+  // Insertion
+  /////////////////////////
+  for (auto gen : generators)
+    bench("insert(begin)" + tostr(gen), [gen](auto& st) {
+      auto const size = st.range(0);
+      std::vector<ValueType> in;
+      std::generate_n(std::back_inserter(in), size, gen);
+      DoNotOptimizeData(in);
+
+      Container c(in.begin(), in.end());
+      DoNotOptimizeData(c);
+
+      ValueType value = gen();
+      benchmark::DoNotOptimize(value);
+
+      for ([[maybe_unused]] auto _ : st) {
+        c.insert(c.begin(), value);
+        DoNotOptimizeData(c);
+
+        c.erase(std::prev(c.end())); // avoid growing indefinitely
+      }
+    });
+
+  if constexpr (std::random_access_iterator<typename Container::iterator>) {
+    for (auto gen : generators)
+      bench("insert(middle)" + tostr(gen), [gen](auto& st) {
+        auto const size = st.range(0);
+        std::vector<ValueType> in;
+        std::generate_n(std::back_inserter(in), size, gen);
+        DoNotOptimizeData(in);
+
+        Container c(in.begin(), in.end());
+        DoNotOptimizeData(c);
+
+        ValueType value = gen();
+        benchmark::DoNotOptimize(value);
+
+        for ([[maybe_unused]] auto _ : st) {
+          auto mid = c.begin() + (size / 2); // requires random-access iterators in order to make sense
+          c.insert(mid, value);
+          DoNotOptimizeData(c);
+
+          c.erase(c.end() - 1); // avoid growing indefinitely
+        }
+      });
+  }
+
+  if constexpr (requires(Container c) { c.reserve(0); }) {
+    // Insert at the start of a vector in a scenario where the vector already
+    // has enough capacity to hold all the elements we are inserting.
+    for (auto gen : generators)
+      bench("insert(begin, input-iter, input-iter) (no realloc)" + tostr(gen), [gen](auto& st) {
+        auto const size = st.range(0);
+        std::vector<ValueType> in;
+        std::generate_n(std::back_inserter(in), size, gen);
+        DoNotOptimizeData(in);
+        auto first = in.data();
+        auto last  = in.data() + in.size();
+
+        const int small = 100; // arbitrary
+        Container c;
+        c.reserve(size + small); // ensure no reallocation
+        std::generate_n(std::back_inserter(c), small, gen);
+
+        for ([[maybe_unused]] auto _ : st) {
+          c.insert(c.begin(), cpp17_input_iterator(first), cpp17_input_iterator(last));
+          DoNotOptimizeData(c);
+
+          st.PauseTiming();
+          c.erase(c.begin() + small, c.end()); // avoid growing indefinitely
+          st.ResumeTiming();
+        }
+      });
+
+    // Insert at the start of a vector in a scenario where the vector already
+    // has almost enough capacity to hold all the elements we are inserting,
+    // but does need to reallocate.
+    for (auto gen : generators)
+      bench("insert(begin, input-iter, input-iter) (half filled)" + tostr(gen), [gen](auto& st) {
+        auto const size = st.range(0);
+        std::vector<ValueType> in;
+        std::generate_n(std::back_inserter(in), size, gen);
+        DoNotOptimizeData(in);
+        auto first = in.data();
+        auto last  = in.data() + in.size();
+
+        const int overflow = size / 10; // 10% of elements won't fit in the vector when we insert
+        Container c;
+        for ([[maybe_unused]] auto _ : st) {
+          st.PauseTiming();
+          c = Container();
+          c.reserve(size);
+          std::generate_n(std::back_inserter(c), overflow, gen);
+          st.ResumeTiming();
+
+          c.insert(c.begin(), cpp17_input_iterator(first), cpp17_input_iterator(last));
+          DoNotOptimizeData(c);
+        }
+      });
+
+    // Insert at the start of a vector in a scenario where the vector can fit a few
+    // more elements, but needs to reallocate almost immediately to fit the remaining
+    // elements.
+    for (auto gen : generators)
+      bench("insert(begin, input-iter, input-iter) (near full)" + tostr(gen), [gen](auto& st) {
+        auto const size = st.range(0);
+        std::vector<ValueType> in;
+        std::generate_n(std::back_inserter(in), size, gen);
+        DoNotOptimizeData(in);
+        auto first = in.data();
+        auto last  = in.data() + in.size();
+
+        auto const overflow = 9 * (size / 10); // 90% of elements won't fit in the vector when we insert
+        Container c;
+        for ([[maybe_unused]] auto _ : st) {
+          st.PauseTiming();
+          c = Container();
+          c.reserve(size);
+          std::generate_n(std::back_inserter(c), overflow, gen);
+          st.ResumeTiming();
+
+          c.insert(c.begin(), cpp17_input_iterator(first), cpp17_input_iterator(last));
+          DoNotOptimizeData(c);
+        }
+      });
+  }
+
+  /////////////////////////
+  // Variations of push_back
+  /////////////////////////
+  static constexpr bool has_push_back = requires(Container c, ValueType v) { c.push_back(v); };
+  static constexpr bool has_capacity  = requires(Container c) { c.capacity(); };
+  static constexpr bool has_reserve   = requires(Container c) { c.reserve(0); };
+  if constexpr (has_push_back) {
+    if constexpr (has_capacity) {
+      // For containers where we can observe capacity(), push_back a single element
+      // without reserving to ensure the container needs to grow
+      for (auto gen : generators)
+        bench("push_back() (growing)" + tostr(gen), [gen](auto& st) {
+          auto const size = st.range(0);
+          std::vector<ValueType> in;
+          std::generate_n(std::back_inserter(in), size, gen);
+          DoNotOptimizeData(in);
+
+          auto at_capacity = [](Container c) {
+            while (c.size() < c.capacity())
+              c.push_back(c.back());
+            return c;
+          };
+
+          std::vector<Container> c(BatchSize, at_capacity(Container(in.begin(), in.end())));
+          std::vector<Container> const original = c;
+
+          while (st.KeepRunningBatch(BatchSize)) {
+            for (std::size_t i = 0; i != BatchSize; ++i) {
+              c[i].push_back(in[i]);
+              DoNotOptimizeData(c[i]);
+            }
+
+            st.PauseTiming();
+            for (std::size_t i = 0; i != BatchSize; ++i) {
+              c[i] = at_capacity(Container(in.begin(), in.end()));
+              assert(c[i].size() == c[i].capacity());
+            }
+            st.ResumeTiming();
+          }
+        });
+    }
+
+    // For containers where we can reserve, push_back a single element after reserving to
+    // ensure the container doesn't grow
+    if constexpr (has_reserve) {
+      for (auto gen : generators)
+        bench("push_back() (with reserve)" + tostr(gen), [gen](auto& st) {
+          auto const size = st.range(0);
+          std::vector<ValueType> in;
+          std::generate_n(std::back_inserter(in), size, gen);
+          DoNotOptimizeData(in);
+
+          Container c(in.begin(), in.end());
+          // Ensure the container has enough capacity
+          c.reserve(c.size() + BatchSize);
+          DoNotOptimizeData(c);
+
+          while (st.KeepRunningBatch(BatchSize)) {
+            for (std::size_t i = 0; i != BatchSize; ++i) {
+              c.push_back(in[i]);
+            }
+            DoNotOptimizeData(c);
+
+            st.PauseTiming();
+            c.erase(c.end() - BatchSize, c.end());
+            st.ResumeTiming();
+          }
+        });
+    }
+
+    // push_back many elements: this is amortized constant for std::vector but not all containers
+    for (auto gen : generators)
+      bench("push_back() (many elements)" + tostr(gen), [gen](auto& st) {
+        auto const size = st.range(0);
+        std::vector<ValueType> in;
+        std::generate_n(std::back_inserter(in), size, gen);
+        DoNotOptimizeData(in);
+
+        Container c;
+        DoNotOptimizeData(c);
+        while (st.KeepRunningBatch(size)) {
+          for (int i = 0; i != size; ++i) {
+            c.push_back(in[i]);
+          }
+          DoNotOptimizeData(c);
+
+          st.PauseTiming();
+          c.clear();
+          st.ResumeTiming();
+        }
+      });
+  }
+
+  /////////////////////////
+  // Erasure
+  /////////////////////////
+  for (auto gen : generators)
+    bench("erase(begin)" + tostr(gen), [gen](auto& st) {
+      auto const size = st.range(0);
+      std::vector<ValueType> in;
+      std::generate_n(std::back_inserter(in), size, gen);
+      DoNotOptimizeData(in);
+
+      Container c(in.begin(), in.end());
+      DoNotOptimizeData(c);
+
+      ValueType value = gen();
+      benchmark::DoNotOptimize(value);
+
+      for ([[maybe_unused]] auto _ : st) {
+        c.erase(c.begin());
+        DoNotOptimizeData(c);
+
+        c.insert(c.end(), value); // re-insert an element at the end to avoid needing a new container
+      }
+    });
+
+  if constexpr (std::random_access_iterator<typename Container::iterator>) {
+    for (auto gen : generators)
+      bench("erase(middle)" + tostr(gen), [gen](auto& st) {
+        auto const size = st.range(0);
+        std::vector<ValueType> in;
+        std::generate_n(std::back_inserter(in), size, gen);
+        DoNotOptimizeData(in);
+
+        Container c(in.begin(), in.end());
+        DoNotOptimizeData(c);
+
+        ValueType value = gen();
+        benchmark::DoNotOptimize(value);
+
+        for ([[maybe_unused]] auto _ : st) {
+          auto mid = c.begin() + (size / 2);
+          c.erase(mid);
+          DoNotOptimizeData(c);
+
+          c.insert(c.end(), value); // re-insert an element at the end to avoid needing a new container
+        }
+      });
+  }
+}
+
+} // namespace support
+
+#endif // TEST_BENCHMARKS_CONTAINERS_SEQUENCE_SEQUENCE_CONTAINER_BENCHMARKS_H
diff --git a/libcxx/test/benchmarks/containers/sequence/vector.bench.cpp b/libcxx/test/benchmarks/containers/sequence/vector.bench.cpp
new file mode 100644
index 0000000000000..599db1d90fa9a
--- /dev/null
+++ b/libcxx/test/benchmarks/containers/sequence/vector.bench.cpp
@@ -0,0 +1,25 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17
+
+#include <string>
+#include <vector>
+
+#include "sequence_container_benchmarks.h"
+#include "benchmark/benchmark.h"
+
+int main(int argc, char** argv) {
+  support::sequence_container_benchmarks<std::vector<int>>("std::vector<int>");
+  support::sequence_container_benchmarks<std::vector<std::string>>("std::vector<std::string>");
+
+  benchmark::Initialize(&argc, argv);
+  benchmark::RunSpecifiedBenchmarks();
+  benchmark::Shutdown();
+  return 0;
+}
diff --git a/libcxx/test/benchmarks/containers/string.bench.cpp b/libcxx/test/benchmarks/containers/string.bench.cpp
index 0b62c87acf7a2..aeff6ad6f6333 100644
--- a/libcxx/test/benchmarks/containers/string.bench.cpp
+++ b/libcxx/test/benchmarks/containers/string.bench.cpp
@@ -6,7 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-// UNSUPPORTED: c++03, c++11, c++14
+// UNSUPPORTED: c++03, c++11, c++14, c++17
 
 #include <cstdint>
 #include <cstdlib>
diff --git a/libcxx/test/benchmarks/containers/unordered_set_operations.bench.cpp b/libcxx/test/benchmarks/containers/unordered_set_operations.bench.cpp
deleted file mode 100644
index a8448ef5a0cfb..0000000000000
--- a/libcxx/test/benchmarks/containers/unordered_set_operations.bench.cpp
+++ /dev/null
@@ -1,245 +0,0 @@
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
-
-#include <cstdint>
-#include <cstdlib>
-#include <cstring>
-#include <functional>
-#include <unordered_set>
-#include <vector>
-
-#include "benchmark/benchmark.h"
-
-#include "ContainerBenchmarks.h"
-#include "../GenerateInput.h"
-#include "test_macros.h"
-
-using namespace ContainerBenchmarks;
-
-constexpr std::size_t TestNumInputs = 1024;
-
-// The purpose of this hash function is to NOT be implemented as the identity function,
-// which is how std::hash is implemented for smaller integral types.
-struct NonIdentityScalarHash : std::hash<unsigned long long> {};
-
-// The sole purpose of this comparator is to be used in BM_Rehash, where
-// we need something slow enough to be easily noticable in benchmark results.
-// The default implementation of operator== for strings seems to be a little
-// too fast for that specific benchmark to reliably show a noticeable
-// improvement, but unoptimized bytewise comparison fits just right.
-// Early return is there just for convenience, since we only compare strings
-// of equal length in BM_Rehash.
-struct SlowStringEq {
-  SlowStringEq() = default;
-  inline TEST_ALWAYS_INLINE bool operator()(const std::string& lhs, const std::string& rhs) const {
-    if (lhs.size() != rhs.size())
-      return false;
-
-    bool eq = true;
-    for (size_t i = 0; i < lhs.size(); ++i) {
-      eq &= lhs[i] == rhs[i];
-    }
-    return eq;
-  }
-};
-
-//----------------------------------------------------------------------------//
-//                       BM_InsertValue
-// ---------------------------------------------------------------------------//
-
-// Sorted Ascending //
-BENCHMARK_CAPTURE(
-    BM_InsertValue, unordered_set_uint32, std::unordered_set<uint32_t>{}, getRandomIntegerInputs<uint32_t>)
-    ->Arg(TestNumInputs);
-
-BENCHMARK_CAPTURE(
-    BM_InsertValue, unordered_set_uint32_sorted, std::unordered_set<uint32_t>{}, getSortedIntegerInputs<uint32_t>)
-    ->Arg(TestNumInputs);
-
-// Top Bytes //
-BENCHMARK_CAPTURE(BM_InsertValue,
-                  unordered_set_top_bits_uint32,
-                  std::unordered_set<uint32_t>{},
-                  getSortedTopBitsIntegerInputs<uint32_t>)
-    ->Arg(TestNumInputs);
-
-BENCHMARK_CAPTURE(BM_InsertValueRehash,
-                  unordered_set_top_bits_uint32,
-                  std::unordered_set<uint32_t, NonIdentityScalarHash>{},
-                  getSortedTopBitsIntegerInputs<uint32_t>)
-    ->Arg(TestNumInputs);
-
-// String //
-BENCHMARK_CAPTURE(BM_InsertValue, unordered_set_string, std::unordered_set<std::string>{}, getRandomStringInputs)
-    ->Arg(TestNumInputs);
-
-BENCHMARK_CAPTURE(BM_InsertValueRehash, unordered_set_string, std::unordered_set<std::string>{}, getRandomStringInputs)
-    ->Arg(TestNumInputs);
-
-// Prefixed String //
-BENCHMARK_CAPTURE(
-    BM_InsertValue, unordered_set_prefixed_string, std::unordered_set<std::string>{}, getPrefixedRandomStringInputs)
-    ->Arg(TestNumInputs);
-
-BENCHMARK_CAPTURE(BM_InsertValueRehash,
-                  unordered_set_prefixed_string,
-                  std::unordered_set<std::string>{},
-                  getPrefixedRandomStringInputs)
-    ->Arg(TestNumInputs);
-
-//----------------------------------------------------------------------------//
-//                         BM_Find
-// ---------------------------------------------------------------------------//
-
-// Random //
-BENCHMARK_CAPTURE(
-    BM_Find, unordered_set_random_uint64, std::unordered_set<uint64_t>{}, getRandomIntegerInputs<uint64_t>)
-    ->Arg(TestNumInputs);
-
-BENCHMARK_CAPTURE(BM_FindRehash,
-                  unordered_set_random_uint64,
-                  std::unordered_set<uint64_t, NonIdentityScalarHash>{},
-                  getRandomIntegerInputs<uint64_t>)
-    ->Arg(TestNumInputs);
-
-// Sorted //
-BENCHMARK_CAPTURE(
-    BM_Find, unordered_set_sorted_uint64, std::unordered_set<uint64_t>{}, getSortedIntegerInputs<uint64_t>)
-    ->Arg(TestNumInputs);
-
-BENCHMARK_CAPTURE(BM_FindRehash,
-                  unordered_set_sorted_uint64,
-                  std::unordered_set<uint64_t, NonIdentityScalarHash>{},
-                  getSortedIntegerInputs<uint64_t>)
-    ->Arg(TestNumInputs);
-
-// Sorted //
-#ifndef TEST_HAS_NO_INT128
-BENCHMARK_CAPTURE(BM_Find,
-                  unordered_set_sorted_uint128,
-                  std::unordered_set<__uint128_t>{},
-                  getSortedTopBitsIntegerInputs<__uint128_t>)
-    ->Arg(TestNumInputs);
-
-BENCHMARK_CAPTURE(BM_FindRehash,
-                  unordered_set_sorted_uint128,
-                  std::unordered_set<__uint128_t>{},
-                  getSortedTopBitsIntegerInputs<__uint128_t>)
-    ->Arg(TestNumInputs);
-#endif
-
-// Sorted //
-BENCHMARK_CAPTURE(
-    BM_Find, unordered_set_sorted_uint32, std::unordered_set<uint32_t>{}, getSortedIntegerInputs<uint32_t>)
-    ->Arg(TestNumInputs);
-
-BENCHMARK_CAPTURE(BM_FindRehash,
-                  unordered_set_sorted_uint32,
-                  std::unordered_set<uint32_t, NonIdentityScalarHash>{},
-                  getSortedIntegerInputs<uint32_t>)
-    ->Arg(TestNumInputs);
-
-// Sorted Ascending //
-BENCHMARK_CAPTURE(
-    BM_Find, unordered_set_sorted_large_uint64, std::unordered_set<uint64_t>{}, getSortedLargeIntegerInputs<uint64_t>)
-    ->Arg(TestNumInputs);
-
-BENCHMARK_CAPTURE(BM_FindRehash,
-                  unordered_set_sorted_large_uint64,
-                  std::unordered_set<uint64_t, NonIdentityScalarHash>{},
-                  getSortedLargeIntegerInputs<uint64_t>)
-    ->Arg(TestNumInputs);
-
-// Top Bits //
-BENCHMARK_CAPTURE(
-    BM_Find, unordered_set_top_bits_uint64, std::unordered_set<uint64_t>{}, getSortedTopBitsIntegerInputs<uint64_t>)
-    ->Arg(TestNumInputs);
-
-BENCHMARK_CAPTURE(BM_FindRehash,
-                  unordered_set_top_bits_uint64,
-                  std::unordered_set<uint64_t, NonIdentityScalarHash>{},
-                  getSortedTopBitsIntegerInputs<uint64_t>)
-    ->Arg(TestNumInputs);
-
-// String //
-BENCHMARK_CAPTURE(BM_Find, unordered_set_string, std::unordered_set<std::string>{}, getRandomStringInputs)
-    ->Arg(TestNumInputs);
-
-BENCHMARK_CAPTURE(BM_FindRehash, unordered_set_string, std::unordered_set<std::string>{}, getRandomStringInputs)
-    ->Arg(TestNumInputs);
-
-// Prefixed String //
-BENCHMARK_CAPTURE(
-    BM_Find, unordered_set_prefixed_string, std::unordered_set<std::string>{}, getPrefixedRandomStringInputs)
-    ->Arg(TestNumInputs);
-
-BENCHMARK_CAPTURE(
-    BM_FindRehash, unordered_set_prefixed_string, std::unordered_set<std::string>{}, getPrefixedRandomStringInputs)
-    ->Arg(TestNumInputs);
-
-//----------------------------------------------------------------------------//
-//                         BM_Rehash
-// ---------------------------------------------------------------------------//
-
-BENCHMARK_CAPTURE(BM_Rehash,
-                  unordered_set_string_arg,
-                  std::unordered_set<std::string, std::hash<std::string>, SlowStringEq>{},
-                  getRandomStringInputs)
-    ->Arg(TestNumInputs);
-
-BENCHMARK_CAPTURE(BM_Rehash, unordered_set_int_arg, std::unordered_set<int>{}, getRandomIntegerInputs<int>)
-    ->Arg(TestNumInputs);
-
-//----------------------------------------------------------------------------//
-//                         BM_Compare
-// ---------------------------------------------------------------------------//
-
-BENCHMARK_CAPTURE(
-    BM_Compare_same_container, unordered_set_string, std::unordered_set<std::string>{}, getRandomStringInputs)
-    ->Arg(TestNumInputs);
-
-BENCHMARK_CAPTURE(BM_Compare_same_container, unordered_set_int, std::unordered_set<int>{}, getRandomIntegerInputs<int>)
-    ->Arg(TestNumInputs);
-
-BENCHMARK_CAPTURE(
-    BM_Compare_different_containers, unordered_set_string, std::unordered_set<std::string>{}, getRandomStringInputs)
-    ->Arg(TestNumInputs);
-
-BENCHMARK_CAPTURE(
-    BM_Compare_different_containers, unordered_set_int, std::unordered_set<int>{}, getRandomIntegerInputs<int>)
-    ->Arg(TestNumInputs);
-
-///////////////////////////////////////////////////////////////////////////////
-BENCHMARK_CAPTURE(BM_InsertDuplicate, unordered_set_int, std::unordered_set<int>{}, getRandomIntegerInputs<int>)
-    ->Arg(TestNumInputs);
-BENCHMARK_CAPTURE(BM_InsertDuplicate, unordered_set_string, std::unordered_set<std::string>{}, getRandomStringInputs)
-    ->Arg(TestNumInputs);
-
-BENCHMARK_CAPTURE(BM_EmplaceDuplicate, unordered_set_int, std::unordered_set<int>{}, getRandomIntegerInputs<int>)
-    ->Arg(TestNumInputs);
-BENCHMARK_CAPTURE(BM_EmplaceDuplicate, unordered_set_string, std::unordered_set<std::string>{}, getRandomStringInputs)
-    ->Arg(TestNumInputs);
-
-BENCHMARK_CAPTURE(
-    BM_InsertDuplicate, unordered_set_int_insert_arg, std::unordered_set<int>{}, getRandomIntegerInputs<int>)
-    ->Arg(TestNumInputs);
-BENCHMARK_CAPTURE(
-    BM_InsertDuplicate, unordered_set_string_insert_arg, std::unordered_set<std::string>{}, getRandomStringInputs)
-    ->Arg(TestNumInputs);
-
-BENCHMARK_CAPTURE(
-    BM_EmplaceDuplicate, unordered_set_int_insert_arg, std::unordered_set<int>{}, getRandomIntegerInputs<unsigned>)
-    ->Arg(TestNumInputs);
-
-BENCHMARK_CAPTURE(
-    BM_EmplaceDuplicate, unordered_set_string_arg, std::unordered_set<std::string>{}, getRandomCStringInputs)
-    ->Arg(TestNumInputs);
-
-BENCHMARK_MAIN();
diff --git a/libcxx/test/benchmarks/containers/vector_operations.bench.cpp b/libcxx/test/benchmarks/containers/vector_operations.bench.cpp
deleted file mode 100644
index 1cd754ca7e780..0000000000000
--- a/libcxx/test/benchmarks/containers/vector_operations.bench.cpp
+++ /dev/null
@@ -1,108 +0,0 @@
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
-
-#include <cstdint>
-#include <cstdlib>
-#include <cstring>
-#include <deque>
-#include <functional>
-#include <memory>
-#include <string>
-#include <vector>
-
-#include "benchmark/benchmark.h"
-#include "ContainerBenchmarks.h"
-#include "../GenerateInput.h"
-
-using namespace ContainerBenchmarks;
-
-constexpr std::size_t TestNumInputs = 1024;
-
-BENCHMARK_CAPTURE(BM_ConstructSize, vector_byte, std::vector<unsigned char>{})->Arg(5140480);
-
-BENCHMARK_CAPTURE(BM_CopyConstruct, vector_int, std::vector<int>{})->Arg(5140480);
-
-BENCHMARK_CAPTURE(BM_Assignment, vector_int, std::vector<int>{})->Arg(5140480);
-
-BENCHMARK_CAPTURE(BM_ConstructSizeValue, vector_byte, std::vector<unsigned char>{}, 0)->Arg(5140480);
-
-BENCHMARK_CAPTURE(BM_ConstructIterIter, vector_char, std::vector<char>{}, getRandomIntegerInputs<char>)
-    ->Arg(TestNumInputs);
-
-BENCHMARK_CAPTURE(BM_ConstructIterIter, vector_size_t, std::vector<size_t>{}, getRandomIntegerInputs<size_t>)
-    ->Arg(TestNumInputs);
-
-BENCHMARK_CAPTURE(BM_ConstructIterIter, vector_string, std::vector<std::string>{}, getRandomStringInputs)
-    ->Arg(TestNumInputs);
-
-BENCHMARK_CAPTURE(BM_ConstructFromRange, vector_char, std::vector<char>{}, getRandomIntegerInputs<char>)
-    ->Arg(TestNumInputs);
-
-BENCHMARK_CAPTURE(BM_ConstructFromRange, vector_size_t, std::vector<size_t>{}, getRandomIntegerInputs<size_t>)
-    ->Arg(TestNumInputs);
-
-BENCHMARK_CAPTURE(BM_ConstructFromRange, vector_string, std::vector<std::string>{}, getRandomStringInputs)
-    ->Arg(TestNumInputs);
-
-BENCHMARK_CAPTURE(BM_Pushback_no_grow, vector_int, std::vector<int>{})->Arg(TestNumInputs);
-
-BENCHMARK_CAPTURE(BM_erase_iter_in_middle, vector_int, std::vector<int>{}, getRandomIntegerInputs<int>)
-    ->Range(TestNumInputs, TestNumInputs * 10);
-BENCHMARK_CAPTURE(BM_erase_iter_in_middle, vector_string, std::vector<std::string>{}, getRandomStringInputs)
-    ->Range(TestNumInputs, TestNumInputs * 10);
-
-BENCHMARK_CAPTURE(BM_erase_iter_at_start, vector_int, std::vector<int>{}, getRandomIntegerInputs<int>)
-    ->Range(TestNumInputs, TestNumInputs * 10);
-BENCHMARK_CAPTURE(BM_erase_iter_at_start, vector_string, std::vector<std::string>{}, getRandomStringInputs)
-    ->Range(TestNumInputs, TestNumInputs * 10);
-
-template <class T>
-void bm_grow(benchmark::State& state) {
-  for (auto _ : state) {
-    std::vector<T> vec;
-    benchmark::DoNotOptimize(vec);
-    for (size_t i = 0; i != 2048; ++i)
-      vec.emplace_back();
-    benchmark::DoNotOptimize(vec);
-  }
-}
-BENCHMARK(bm_grow<int>);
-BENCHMARK(bm_grow<std::string>);
-BENCHMARK(bm_grow<std::unique_ptr<int>>);
-BENCHMARK(bm_grow<std::deque<int>>);
-
-BENCHMARK_CAPTURE(BM_AssignInputIterIter, vector_int, std::vector<int>{}, getRandomIntegerInputs<int>)
-    ->Args({TestNumInputs, TestNumInputs});
-
-BENCHMARK_CAPTURE(
-    BM_AssignInputIterIter<32>, vector_string, std::vector<std::string>{}, getRandomStringInputsWithLength)
-    ->Args({TestNumInputs, TestNumInputs});
-
-BENCHMARK_CAPTURE(BM_AssignInputIterIter<100>,
-                  vector_vector_int,
-                  std::vector<std::vector<int>>{},
-                  getRandomIntegerInputsWithLength<int>)
-    ->Args({TestNumInputs, TestNumInputs});
-
-BENCHMARK_CAPTURE(BM_Insert_InputIterIter_NoRealloc, vector_int, std::vector<int>(100, 1), getRandomIntegerInputs<int>)
-    ->Arg(514048);
-BENCHMARK_CAPTURE(
-    BM_Insert_InputIterIter_Realloc_HalfFilled, vector_int, std::vector<int>{}, getRandomIntegerInputs<int>)
-    ->Arg(514048);
-BENCHMARK_CAPTURE(BM_Insert_InputIterIter_Realloc_NearFull, vector_int, std::vector<int>{}, getRandomIntegerInputs<int>)
-    ->Arg(514048);
-BENCHMARK_CAPTURE(
-    BM_Insert_InputIterIter_Realloc_HalfFilled, vector_string, std::vector<std::string>{}, getSSORandomStringInputs)
-    ->Arg(514048);
-BENCHMARK_CAPTURE(
-    BM_Insert_InputIterIter_Realloc_NearFull, vector_string, std::vector<std::string>{}, getSSORandomStringInputs)
-    ->Arg(514048);
-
-BENCHMARK_MAIN();
diff --git a/libcxx/test/benchmarks/filesystem.bench.cpp b/libcxx/test/benchmarks/filesystem.bench.cpp
index 83a87c86d3de0..dc6b0ac537f7e 100644
--- a/libcxx/test/benchmarks/filesystem.bench.cpp
+++ b/libcxx/test/benchmarks/filesystem.bench.cpp
@@ -6,7 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-// UNSUPPORTED: c++03, c++11, c++14
+// UNSUPPORTED: c++03, c++11, c++14, c++17
 
 #include <filesystem>
 
diff --git a/libcxx/test/benchmarks/hash.bench.cpp b/libcxx/test/benchmarks/hash.bench.cpp
index 1e1a0f36ec116..ca958765dc210 100644
--- a/libcxx/test/benchmarks/hash.bench.cpp
+++ b/libcxx/test/benchmarks/hash.bench.cpp
@@ -6,7 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-// UNSUPPORTED: c++03
+// UNSUPPORTED: c++03, c++11, c++14, c++17
 
 #include <cstdint>
 #include <cstddef>
diff --git a/libcxx/test/benchmarks/variant_visit_1.bench.cpp b/libcxx/test/benchmarks/variant_visit_1.bench.cpp
index 42b22aabaee04..f1b702530bed3 100644
--- a/libcxx/test/benchmarks/variant_visit_1.bench.cpp
+++ b/libcxx/test/benchmarks/variant_visit_1.bench.cpp
@@ -6,7 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-// UNSUPPORTED: c++03, c++11, c++14
+// UNSUPPORTED: c++03, c++11, c++14, c++17
 
 #include "benchmark/benchmark.h"
 
diff --git a/libcxx/test/benchmarks/variant_visit_2.bench.cpp b/libcxx/test/benchmarks/variant_visit_2.bench.cpp
index 328048cabc443..7dd8d02b358be 100644
--- a/libcxx/test/benchmarks/variant_visit_2.bench.cpp
+++ b/libcxx/test/benchmarks/variant_visit_2.bench.cpp
@@ -6,7 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-// UNSUPPORTED: c++03, c++11, c++14
+// UNSUPPORTED: c++03, c++11, c++14, c++17
 
 #include "benchmark/benchmark.h"
 
diff --git a/libcxx/test/benchmarks/variant_visit_3.bench.cpp b/libcxx/test/benchmarks/variant_visit_3.bench.cpp
index 40f8c1b5fa262..0fe42b0d8e009 100644
--- a/libcxx/test/benchmarks/variant_visit_3.bench.cpp
+++ b/libcxx/test/benchmarks/variant_visit_3.bench.cpp
@@ -6,7 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-// UNSUPPORTED: c++03, c++11, c++14
+// UNSUPPORTED: c++03, c++11, c++14, c++17
 
 #include "benchmark/benchmark.h"
 
diff --git a/libcxx/test/configs/nvptx-libc++-shared.cfg.in b/libcxx/test/configs/nvptx-libc++-shared.cfg.in
index 9a3ca9c8da950..e07ed35da4d5a 100644
--- a/libcxx/test/configs/nvptx-libc++-shared.cfg.in
+++ b/libcxx/test/configs/nvptx-libc++-shared.cfg.in
@@ -10,8 +10,6 @@ config.substitutions.append(('%{link_flags}',
    '-nostdlib++ -startfiles -stdlib '
    '-L %{lib-dir} -lc++ -lc++abi '
    '-Wl,--suppress-stack-size-warning '
-   '-Wl,-mllvm,-nvptx-lower-global-ctor-dtor=1 '
-   '-Wl,-mllvm,-nvptx-emit-init-fini-kernel'
 ))
 config.substitutions.append(('%{exec}',
     '%{executor} --no-parallelism'
diff --git a/libcxx/test/libcxx/atomics/diagnose_invalid_memory_order.verify.cpp b/libcxx/test/libcxx/atomics/diagnose_invalid_memory_order.verify.cpp
index 2790916edaf69..1b0b945f33700 100644
--- a/libcxx/test/libcxx/atomics/diagnose_invalid_memory_order.verify.cpp
+++ b/libcxx/test/libcxx/atomics/diagnose_invalid_memory_order.verify.cpp
@@ -6,11 +6,6 @@
 //
 //===----------------------------------------------------------------------===//
 
-// This test fails with Clang <18 because diagnose_if doesn't emit all of the
-// diagnostics when -fdelayed-template-parsing is enabled, like it is in MSVC
-// mode.
-// XFAIL: msvc && clang-17
-
 // REQUIRES: diagnose-if-support
 
 // <atomic>
diff --git a/libcxx/test/libcxx/clang_tidy.gen.py b/libcxx/test/libcxx/clang_tidy.gen.py
index 06f277e901d33..f1135749febe4 100644
--- a/libcxx/test/libcxx/clang_tidy.gen.py
+++ b/libcxx/test/libcxx/clang_tidy.gen.py
@@ -26,9 +26,6 @@
 // The GCC compiler flags are not always compatible with clang-tidy.
 // UNSUPPORTED: gcc
 
-// Clang 17 has false positives.
-// UNSUPPORTED: clang-17
-
 {lit_header_restrictions.get(header, '')}
 {lit_header_undeprecations.get(header, '')}
 
diff --git a/libcxx/test/libcxx/containers/associative/map/at.abort.pass.cpp b/libcxx/test/libcxx/containers/associative/map/at.abort.pass.cpp
index 36aa644abb596..d68ee5f528599 100644
--- a/libcxx/test/libcxx/containers/associative/map/at.abort.pass.cpp
+++ b/libcxx/test/libcxx/containers/associative/map/at.abort.pass.cpp
@@ -23,14 +23,11 @@
 
 #include "test_macros.h"
 
-
-void exit_success(int) {
-    std::_Exit(EXIT_SUCCESS);
-}
+void exit_success(int) { std::_Exit(EXIT_SUCCESS); }
 
 int main(int, char**) {
-    std::signal(SIGABRT, exit_success);
-    std::map<int, int> map;
-    map.at(1);
-    return EXIT_FAILURE;
+  std::signal(SIGABRT, exit_success);
+  std::map<int, int> map;
+  map.at(1);
+  return EXIT_FAILURE;
 }
diff --git a/libcxx/test/libcxx/containers/associative/map/at.const.abort.pass.cpp b/libcxx/test/libcxx/containers/associative/map/at.const.abort.pass.cpp
index f3dcca888f2b0..bbc8c7c4d726a 100644
--- a/libcxx/test/libcxx/containers/associative/map/at.const.abort.pass.cpp
+++ b/libcxx/test/libcxx/containers/associative/map/at.const.abort.pass.cpp
@@ -23,14 +23,11 @@
 
 #include "test_macros.h"
 
-
-void exit_success(int) {
-    std::_Exit(EXIT_SUCCESS);
-}
+void exit_success(int) { std::_Exit(EXIT_SUCCESS); }
 
 int main(int, char**) {
-    std::signal(SIGABRT, exit_success);
-    std::map<int, int> const map;
-    map.at(1);
-    return EXIT_FAILURE;
+  std::signal(SIGABRT, exit_success);
+  std::map<int, int> const map;
+  map.at(1);
+  return EXIT_FAILURE;
 }
diff --git a/libcxx/test/libcxx/containers/associative/map/find.modules.compile.pass.mm b/libcxx/test/libcxx/containers/associative/map/find.modules.compile.pass.mm
index 8b2f0057bcbc1..82b1c494c9566 100644
--- a/libcxx/test/libcxx/containers/associative/map/find.modules.compile.pass.mm
+++ b/libcxx/test/libcxx/containers/associative/map/find.modules.compile.pass.mm
@@ -13,6 +13,4 @@
 
 #include <map>
 
-void f(std::map<int, int> const& map, int key) {
-    (void)map.find(key);
-}
+void f(std::map<int, int> const& map, int key) { (void)map.find(key); }
diff --git a/libcxx/test/libcxx/containers/associative/non_const_comparator.incomplete.verify.cpp b/libcxx/test/libcxx/containers/associative/non_const_comparator.incomplete.verify.cpp
index 3f7086f7e37f3..bae78ba4b78c6 100644
--- a/libcxx/test/libcxx/containers/associative/non_const_comparator.incomplete.verify.cpp
+++ b/libcxx/test/libcxx/containers/associative/non_const_comparator.incomplete.verify.cpp
@@ -17,31 +17,33 @@
 #include <map>
 #include <functional>
 
-template <template <typename ...> class Container>
+template <template <typename...> class Container>
 void test_set() {
-  struct KeyBase { };
-  struct KeyDerived;  // derives from KeyBase, but incomplete at this point
+  struct KeyBase {};
+  struct KeyDerived; // derives from KeyBase, but incomplete at this point
 
   // Name the type but don't instantiate it.
   using C = Container<KeyDerived*, std::less<KeyBase*>>;
 
   // Instantiate it but don't ODR use any members.
-  typename C::value_type dummy; (void)dummy;
+  typename C::value_type dummy;
+  (void)dummy;
 
   // Complete the types.
-  struct KeyDerived : KeyBase { };
+  struct KeyDerived : KeyBase {};
 
   C c; // ODR use it, which should be OK
 }
 
-template <template <typename ...> class Container>
+template <template <typename...> class Container>
 void test_map() {
-  struct Value { };
-  struct KeyBase { };
+  struct Value {};
+  struct KeyBase {};
   struct KeyDerived;
   using C = Container<KeyDerived*, Value, std::less<KeyBase*>>;
-  typename C::value_type dummy; (void)dummy;
-  struct KeyDerived : KeyBase { };
+  typename C::value_type dummy;
+  (void)dummy;
+  struct KeyDerived : KeyBase {};
   C c;
 }
 
diff --git a/libcxx/test/libcxx/containers/associative/tree_balance_after_insert.pass.cpp b/libcxx/test/libcxx/containers/associative/tree_balance_after_insert.pass.cpp
index a99f4fff1300e..ccd84af44c3a5 100644
--- a/libcxx/test/libcxx/containers/associative/tree_balance_after_insert.pass.cpp
+++ b/libcxx/test/libcxx/containers/associative/tree_balance_after_insert.pass.cpp
@@ -20,1021 +20,315 @@
 
 #include "test_macros.h"
 
-struct Node
-{
-    Node* __left_;
-    Node* __right_;
-    Node* __parent_;
-    bool __is_black_;
+struct Node {
+  Node* __left_;
+  Node* __right_;
+  Node* __parent_;
+  bool __is_black_;
 
-    Node* __parent_unsafe() const { return __parent_; }
-    void __set_parent(Node* x) { __parent_ = x;}
+  Node* __parent_unsafe() const { return __parent_; }
+  void __set_parent(Node* x) { __parent_ = x; }
 
-    Node() : __left_(), __right_(), __parent_(), __is_black_() {}
+  Node() : __left_(), __right_(), __parent_(), __is_black_() {}
 };
 
-void
-test1()
-{
-    {
-        Node root;
-        Node a;
-        Node b;
-        Node c;
-        Node d;
-
-        root.__left_ = &c;
-
-        c.__parent_ = &root;
-        c.__left_ = &b;
-        c.__right_ = &d;
-        c.__is_black_ = true;
-
-        b.__parent_ = &c;
-        b.__left_ = &a;
-        b.__right_ = 0;
-        b.__is_black_ = false;
-
-        d.__parent_ = &c;
-        d.__left_ = 0;
-        d.__right_ = 0;
-        d.__is_black_ = false;
-
-        a.__parent_ = &b;
-        a.__left_ = 0;
-        a.__right_ = 0;
-        a.__is_black_ = false;
-
-        std::__tree_balance_after_insert(root.__left_, &a);
-
-        assert(std::__tree_invariant(root.__left_));
-
-        assert(root.__left_ == &c);
-
-        assert(c.__parent_ == &root);
-        assert(c.__left_ == &b);
-        assert(c.__right_ == &d);
-        assert(c.__is_black_ == true);
-
-        assert(b.__parent_ == &c);
-        assert(b.__left_ == &a);
-        assert(b.__right_ == 0);
-        assert(b.__is_black_ == true);
-
-        assert(d.__parent_ == &c);
-        assert(d.__left_ == 0);
-        assert(d.__right_ == 0);
-        assert(d.__is_black_ == true);
-
-        assert(a.__parent_ == &b);
-        assert(a.__left_ == 0);
-        assert(a.__right_ == 0);
-        assert(a.__is_black_ == false);
-    }
-    {
-        Node root;
-        Node a;
-        Node b;
-        Node c;
-        Node d;
-
-        root.__left_ = &c;
-
-        c.__parent_ = &root;
-        c.__left_ = &b;
-        c.__right_ = &d;
-        c.__is_black_ = true;
-
-        b.__parent_ = &c;
-        b.__left_ = 0;
-        b.__right_ = &a;
-        b.__is_black_ = false;
-
-        d.__parent_ = &c;
-        d.__left_ = 0;
-        d.__right_ = 0;
-        d.__is_black_ = false;
-
-        a.__parent_ = &b;
-        a.__left_ = 0;
-        a.__right_ = 0;
-        a.__is_black_ = false;
-
-        std::__tree_balance_after_insert(root.__left_, &a);
-
-        assert(std::__tree_invariant(root.__left_));
-
-        assert(root.__left_ == &c);
-
-        assert(c.__parent_ == &root);
-        assert(c.__left_ == &b);
-        assert(c.__right_ == &d);
-        assert(c.__is_black_ == true);
-
-        assert(b.__parent_ == &c);
-        assert(b.__left_ == 0);
-        assert(b.__right_ == &a);
-        assert(b.__is_black_ == true);
-
-        assert(d.__parent_ == &c);
-        assert(d.__left_ == 0);
-        assert(d.__right_ == 0);
-        assert(d.__is_black_ == true);
-
-        assert(a.__parent_ == &b);
-        assert(a.__left_ == 0);
-        assert(a.__right_ == 0);
-        assert(a.__is_black_ == false);
-    }
-    {
-        Node root;
-        Node a;
-        Node b;
-        Node c;
-        Node d;
-
-        root.__left_ = &c;
-
-        c.__parent_ = &root;
-        c.__left_ = &b;
-        c.__right_ = &d;
-        c.__is_black_ = true;
-
-        b.__parent_ = &c;
-        b.__left_ = 0;
-        b.__right_ = 0;
-        b.__is_black_ = false;
-
-        d.__parent_ = &c;
-        d.__left_ = &a;
-        d.__right_ = 0;
-        d.__is_black_ = false;
-
-        a.__parent_ = &d;
-        a.__left_ = 0;
-        a.__right_ = 0;
-        a.__is_black_ = false;
-
-        std::__tree_balance_after_insert(root.__left_, &a);
-
-        assert(std::__tree_invariant(root.__left_));
-
-        assert(root.__left_ == &c);
-
-        assert(c.__parent_ == &root);
-        assert(c.__left_ == &b);
-        assert(c.__right_ == &d);
-        assert(c.__is_black_ == true);
-
-        assert(b.__parent_ == &c);
-        assert(b.__left_ == 0);
-        assert(b.__right_ == 0);
-        assert(b.__is_black_ == true);
-
-        assert(d.__parent_ == &c);
-        assert(d.__left_ == &a);
-        assert(d.__right_ == 0);
-        assert(d.__is_black_ == true);
-
-        assert(a.__parent_ == &d);
-        assert(a.__left_ == 0);
-        assert(a.__right_ == 0);
-        assert(a.__is_black_ == false);
-    }
-    {
-        Node root;
-        Node a;
-        Node b;
-        Node c;
-        Node d;
-
-        root.__left_ = &c;
-
-        c.__parent_ = &root;
-        c.__left_ = &b;
-        c.__right_ = &d;
-        c.__is_black_ = true;
-
-        b.__parent_ = &c;
-        b.__left_ = 0;
-        b.__right_ = 0;
-        b.__is_black_ = false;
-
-        d.__parent_ = &c;
-        d.__left_ = 0;
-        d.__right_ = &a;
-        d.__is_black_ = false;
-
-        a.__parent_ = &d;
-        a.__left_ = 0;
-        a.__right_ = 0;
-        a.__is_black_ = false;
-
-        std::__tree_balance_after_insert(root.__left_, &a);
-
-        assert(std::__tree_invariant(root.__left_));
-
-        assert(root.__left_ == &c);
-
-        assert(c.__parent_ == &root);
-        assert(c.__left_ == &b);
-        assert(c.__right_ == &d);
-        assert(c.__is_black_ == true);
-
-        assert(b.__parent_ == &c);
-        assert(b.__left_ == 0);
-        assert(b.__right_ == 0);
-        assert(b.__is_black_ == true);
-
-        assert(d.__parent_ == &c);
-        assert(d.__left_ == 0);
-        assert(d.__right_ == &a);
-        assert(d.__is_black_ == true);
-
-        assert(a.__parent_ == &d);
-        assert(a.__left_ == 0);
-        assert(a.__right_ == 0);
-        assert(a.__is_black_ == false);
-    }
-    {
-        Node root;
-        Node a;
-        Node b;
-        Node c;
-        Node d;
-        Node e;
-        Node f;
-        Node g;
-        Node h;
-        Node i;
-
-        root.__left_ = &c;
-
-        c.__parent_ = &root;
-        c.__left_ = &b;
-        c.__right_ = &d;
-        c.__is_black_ = true;
-
-        b.__parent_ = &c;
-        b.__left_ = &a;
-        b.__right_ = &g;
-        b.__is_black_ = false;
-
-        d.__parent_ = &c;
-        d.__left_ = &h;
-        d.__right_ = &i;
-        d.__is_black_ = false;
-
-        a.__parent_ = &b;
-        a.__left_ = &e;
-        a.__right_ = &f;
-        a.__is_black_ = false;
-
-        e.__parent_ = &a;
-        e.__is_black_ = true;
-
-        f.__parent_ = &a;
-        f.__is_black_ = true;
-
-        g.__parent_ = &b;
-        g.__is_black_ = true;
-
-        h.__parent_ = &d;
-        h.__is_black_ = true;
-
-        i.__parent_ = &d;
-        i.__is_black_ = true;
-
-        std::__tree_balance_after_insert(root.__left_, &a);
-
-        assert(std::__tree_invariant(root.__left_));
-
-        assert(root.__left_ == &c);
-
-        assert(c.__parent_ == &root);
-        assert(c.__left_ == &b);
-        assert(c.__right_ == &d);
-        assert(c.__is_black_ == true);
-
-        assert(b.__parent_ == &c);
-        assert(b.__left_ == &a);
-        assert(b.__right_ == &g);
-        assert(b.__is_black_ == true);
-
-        assert(d.__parent_ == &c);
-        assert(d.__left_ == &h);
-        assert(d.__right_ == &i);
-        assert(d.__is_black_ == true);
-
-        assert(a.__parent_ == &b);
-        assert(a.__left_ == &e);
-        assert(a.__right_ == &f);
-        assert(a.__is_black_ == false);
-    }
-    {
-        Node root;
-        Node a;
-        Node b;
-        Node c;
-        Node d;
-        Node e;
-        Node f;
-        Node g;
-        Node h;
-        Node i;
-
-        root.__left_ = &c;
-
-        c.__parent_ = &root;
-        c.__left_ = &b;
-        c.__right_ = &d;
-        c.__is_black_ = true;
-
-        b.__parent_ = &c;
-        b.__left_ = &g;
-        b.__right_ = &a;
-        b.__is_black_ = false;
-
-        d.__parent_ = &c;
-        d.__left_ = &h;
-        d.__right_ = &i;
-        d.__is_black_ = false;
-
-        a.__parent_ = &b;
-        a.__left_ = &e;
-        a.__right_ = &f;
-        a.__is_black_ = false;
-
-        e.__parent_ = &a;
-        e.__is_black_ = true;
-
-        f.__parent_ = &a;
-        f.__is_black_ = true;
-
-        g.__parent_ = &b;
-        g.__is_black_ = true;
-
-        h.__parent_ = &d;
-        h.__is_black_ = true;
-
-        i.__parent_ = &d;
-        i.__is_black_ = true;
-
-        std::__tree_balance_after_insert(root.__left_, &a);
-
-        assert(std::__tree_invariant(root.__left_));
-
-        assert(root.__left_ == &c);
-
-        assert(c.__parent_ == &root);
-        assert(c.__left_ == &b);
-        assert(c.__right_ == &d);
-        assert(c.__is_black_ == true);
-
-        assert(b.__parent_ == &c);
-        assert(b.__left_ == &g);
-        assert(b.__right_ == &a);
-        assert(b.__is_black_ == true);
-
-        assert(d.__parent_ == &c);
-        assert(d.__left_ == &h);
-        assert(d.__right_ == &i);
-        assert(d.__is_black_ == true);
-
-        assert(a.__parent_ == &b);
-        assert(a.__left_ == &e);
-        assert(a.__right_ == &f);
-        assert(a.__is_black_ == false);
-    }
-    {
-        Node root;
-        Node a;
-        Node b;
-        Node c;
-        Node d;
-        Node e;
-        Node f;
-        Node g;
-        Node h;
-        Node i;
-
-        root.__left_ = &c;
-
-        c.__parent_ = &root;
-        c.__left_ = &b;
-        c.__right_ = &d;
-        c.__is_black_ = true;
-
-        b.__parent_ = &c;
-        b.__left_ = &g;
-        b.__right_ = &h;
-        b.__is_black_ = false;
-
-        d.__parent_ = &c;
-        d.__left_ = &a;
-        d.__right_ = &i;
-        d.__is_black_ = false;
-
-        a.__parent_ = &d;
-        a.__left_ = &e;
-        a.__right_ = &f;
-        a.__is_black_ = false;
-
-        e.__parent_ = &a;
-        e.__is_black_ = true;
-
-        f.__parent_ = &a;
-        f.__is_black_ = true;
-
-        g.__parent_ = &b;
-        g.__is_black_ = true;
-
-        h.__parent_ = &b;
-        h.__is_black_ = true;
-
-        i.__parent_ = &d;
-        i.__is_black_ = true;
-
-        std::__tree_balance_after_insert(root.__left_, &a);
-
-        assert(std::__tree_invariant(root.__left_));
-
-        assert(root.__left_ == &c);
-
-        assert(c.__parent_ == &root);
-        assert(c.__left_ == &b);
-        assert(c.__right_ == &d);
-        assert(c.__is_black_ == true);
-
-        assert(b.__parent_ == &c);
-        assert(b.__left_ == &g);
-        assert(b.__right_ == &h);
-        assert(b.__is_black_ == true);
-
-        assert(d.__parent_ == &c);
-        assert(d.__left_ == &a);
-        assert(d.__right_ == &i);
-        assert(d.__is_black_ == true);
-
-        assert(a.__parent_ == &d);
-        assert(a.__left_ == &e);
-        assert(a.__right_ == &f);
-        assert(a.__is_black_ == false);
-    }
-    {
-        Node root;
-        Node a;
-        Node b;
-        Node c;
-        Node d;
-        Node e;
-        Node f;
-        Node g;
-        Node h;
-        Node i;
-
-        root.__left_ = &c;
-
-        c.__parent_ = &root;
-        c.__left_ = &b;
-        c.__right_ = &d;
-        c.__is_black_ = true;
-
-        b.__parent_ = &c;
-        b.__left_ = &g;
-        b.__right_ = &h;
-        b.__is_black_ = false;
-
-        d.__parent_ = &c;
-        d.__left_ = &i;
-        d.__right_ = &a;
-        d.__is_black_ = false;
-
-        a.__parent_ = &d;
-        a.__left_ = &e;
-        a.__right_ = &f;
-        a.__is_black_ = false;
-
-        e.__parent_ = &a;
-        e.__is_black_ = true;
+void test1() {
+  {
+    Node root;
+    Node a;
+    Node b;
+    Node c;
+    Node d;
 
-        f.__parent_ = &a;
-        f.__is_black_ = true;
-
-        g.__parent_ = &b;
-        g.__is_black_ = true;
-
-        h.__parent_ = &b;
-        h.__is_black_ = true;
-
-        i.__parent_ = &d;
-        i.__is_black_ = true;
+    root.__left_ = &c;
 
-        std::__tree_balance_after_insert(root.__left_, &a);
-
-        assert(std::__tree_invariant(root.__left_));
+    c.__parent_   = &root;
+    c.__left_     = &b;
+    c.__right_    = &d;
+    c.__is_black_ = true;
 
-        assert(root.__left_ == &c);
-
-        assert(c.__parent_ == &root);
-        assert(c.__left_ == &b);
-        assert(c.__right_ == &d);
-        assert(c.__is_black_ == true);
-
-        assert(b.__parent_ == &c);
-        assert(b.__left_ == &g);
-        assert(b.__right_ == &h);
-        assert(b.__is_black_ == true);
+    b.__parent_   = &c;
+    b.__left_     = &a;
+    b.__right_    = 0;
+    b.__is_black_ = false;
 
-        assert(d.__parent_ == &c);
-        assert(d.__left_ == &i);
-        assert(d.__right_ == &a);
-        assert(d.__is_black_ == true);
-
-        assert(a.__parent_ == &d);
-        assert(a.__left_ == &e);
-        assert(a.__right_ == &f);
-        assert(a.__is_black_ == false);
-    }
-}
+    d.__parent_   = &c;
+    d.__left_     = 0;
+    d.__right_    = 0;
+    d.__is_black_ = false;
 
-void
-test2()
-{
-    {
-        Node root;
-        Node a;
-        Node b;
-        Node c;
-
-        root.__left_ = &c;
-
-        c.__parent_ = &root;
-        c.__left_ = &a;
-        c.__right_ = 0;
-        c.__is_black_ = true;
-
-        a.__parent_ = &c;
-        a.__left_ = 0;
-        a.__right_ = &b;
-        a.__is_black_ = false;
-
-        b.__parent_ = &a;
-        b.__left_ = 0;
-        b.__right_ = 0;
-        b.__is_black_ = false;
-
-        std::__tree_balance_after_insert(root.__left_, &b);
-
-        assert(std::__tree_invariant(root.__left_));
-
-        assert(root.__left_ == &b);
-
-        assert(c.__parent_ == &b);
-        assert(c.__left_ == 0);
-        assert(c.__right_ == 0);
-        assert(c.__is_black_ == false);
-
-        assert(a.__parent_ == &b);
-        assert(a.__left_ == 0);
-        assert(a.__right_ == 0);
-        assert(a.__is_black_ == false);
-
-        assert(b.__parent_ == &root);
-        assert(b.__left_ == &a);
-        assert(b.__right_ == &c);
-        assert(b.__is_black_ == true);
-    }
-    {
-        Node root;
-        Node a;
-        Node b;
-        Node c;
-
-        root.__left_ = &a;
-
-        a.__parent_ = &root;
-        a.__left_ = 0;
-        a.__right_ = &c;
-        a.__is_black_ = true;
-
-        c.__parent_ = &a;
-        c.__left_ = &b;
-        c.__right_ = 0;
-        c.__is_black_ = false;
-
-        b.__parent_ = &c;
-        b.__left_ = 0;
-        b.__right_ = 0;
-        b.__is_black_ = false;
-
-        std::__tree_balance_after_insert(root.__left_, &b);
-
-        assert(std::__tree_invariant(root.__left_));
-
-        assert(root.__left_ == &b);
-
-        assert(a.__parent_ == &b);
-        assert(a.__left_ == 0);
-        assert(a.__right_ == 0);
-        assert(a.__is_black_ == false);
-
-        assert(c.__parent_ == &b);
-        assert(c.__left_ == 0);
-        assert(c.__right_ == 0);
-        assert(c.__is_black_ == false);
-
-        assert(b.__parent_ == &root);
-        assert(b.__left_ == &a);
-        assert(b.__right_ == &c);
-        assert(b.__is_black_ == true);
-    }
-    {
-        Node root;
-        Node a;
-        Node b;
-        Node c;
-        Node d;
-        Node e;
-        Node f;
-        Node g;
-
-        root.__left_ = &c;
-
-        c.__parent_ = &root;
-        c.__left_ = &a;
-        c.__right_ = &g;
-        c.__is_black_ = true;
+    a.__parent_   = &b;
+    a.__left_     = 0;
+    a.__right_    = 0;
+    a.__is_black_ = false;
 
-        a.__parent_ = &c;
-        a.__left_ = &d;
-        a.__right_ = &b;
-        a.__is_black_ = false;
-
-        b.__parent_ = &a;
-        b.__left_ = &e;
-        b.__right_ = &f;
-        b.__is_black_ = false;
-
-        d.__parent_ = &a;
-        d.__is_black_ = true;
-
-        e.__parent_ = &b;
-        e.__is_black_ = true;
-
-        f.__parent_ = &b;
-        f.__is_black_ = true;
-
-        g.__parent_ = &c;
-        g.__is_black_ = true;
-
-        std::__tree_balance_after_insert(root.__left_, &b);
-
-        assert(std::__tree_invariant(root.__left_));
-
-        assert(root.__left_ == &b);
-
-        assert(c.__parent_ == &b);
-        assert(c.__left_ == &f);
-        assert(c.__right_ == &g);
-        assert(c.__is_black_ == false);
-
-        assert(a.__parent_ == &b);
-        assert(a.__left_ == &d);
-        assert(a.__right_ == &e);
-        assert(a.__is_black_ == false);
-
-        assert(b.__parent_ == &root);
-        assert(b.__left_ == &a);
-        assert(b.__right_ == &c);
-        assert(b.__is_black_ == true);
+    std::__tree_balance_after_insert(root.__left_, &a);
 
-        assert(d.__parent_ == &a);
-        assert(d.__is_black_ == true);
+    assert(std::__tree_invariant(root.__left_));
 
-        assert(e.__parent_ == &a);
-        assert(e.__is_black_ == true);
+    assert(root.__left_ == &c);
 
-        assert(f.__parent_ == &c);
-        assert(f.__is_black_ == true);
+    assert(c.__parent_ == &root);
+    assert(c.__left_ == &b);
+    assert(c.__right_ == &d);
+    assert(c.__is_black_ == true);
 
-        assert(g.__parent_ == &c);
-        assert(g.__is_black_ == true);
-    }
-    {
-        Node root;
-        Node a;
-        Node b;
-        Node c;
-        Node d;
-        Node e;
-        Node f;
-        Node g;
+    assert(b.__parent_ == &c);
+    assert(b.__left_ == &a);
+    assert(b.__right_ == 0);
+    assert(b.__is_black_ == true);
 
-        root.__left_ = &a;
-
-        a.__parent_ = &root;
-        a.__left_ = &d;
-        a.__right_ = &c;
-        a.__is_black_ = true;
-
-        c.__parent_ = &a;
-        c.__left_ = &b;
-        c.__right_ = &g;
-        c.__is_black_ = false;
+    assert(d.__parent_ == &c);
+    assert(d.__left_ == 0);
+    assert(d.__right_ == 0);
+    assert(d.__is_black_ == true);
 
-        b.__parent_ = &c;
-        b.__left_ = &e;
-        b.__right_ = &f;
-        b.__is_black_ = false;
+    assert(a.__parent_ == &b);
+    assert(a.__left_ == 0);
+    assert(a.__right_ == 0);
+    assert(a.__is_black_ == false);
+  }
+  {
+    Node root;
+    Node a;
+    Node b;
+    Node c;
+    Node d;
 
-        d.__parent_ = &a;
-        d.__is_black_ = true;
+    root.__left_ = &c;
 
-        e.__parent_ = &b;
-        e.__is_black_ = true;
+    c.__parent_   = &root;
+    c.__left_     = &b;
+    c.__right_    = &d;
+    c.__is_black_ = true;
 
-        f.__parent_ = &b;
-        f.__is_black_ = true;
+    b.__parent_   = &c;
+    b.__left_     = 0;
+    b.__right_    = &a;
+    b.__is_black_ = false;
 
-        g.__parent_ = &c;
-        g.__is_black_ = true;
+    d.__parent_   = &c;
+    d.__left_     = 0;
+    d.__right_    = 0;
+    d.__is_black_ = false;
 
-        std::__tree_balance_after_insert(root.__left_, &b);
+    a.__parent_   = &b;
+    a.__left_     = 0;
+    a.__right_    = 0;
+    a.__is_black_ = false;
 
-        assert(std::__tree_invariant(root.__left_));
+    std::__tree_balance_after_insert(root.__left_, &a);
 
-        assert(root.__left_ == &b);
+    assert(std::__tree_invariant(root.__left_));
 
-        assert(c.__parent_ == &b);
-        assert(c.__left_ == &f);
-        assert(c.__right_ == &g);
-        assert(c.__is_black_ == false);
+    assert(root.__left_ == &c);
 
-        assert(a.__parent_ == &b);
-        assert(a.__left_ == &d);
-        assert(a.__right_ == &e);
-        assert(a.__is_black_ == false);
+    assert(c.__parent_ == &root);
+    assert(c.__left_ == &b);
+    assert(c.__right_ == &d);
+    assert(c.__is_black_ == true);
 
-        assert(b.__parent_ == &root);
-        assert(b.__left_ == &a);
-        assert(b.__right_ == &c);
-        assert(b.__is_black_ == true);
+    assert(b.__parent_ == &c);
+    assert(b.__left_ == 0);
+    assert(b.__right_ == &a);
+    assert(b.__is_black_ == true);
 
-        assert(d.__parent_ == &a);
-        assert(d.__is_black_ == true);
+    assert(d.__parent_ == &c);
+    assert(d.__left_ == 0);
+    assert(d.__right_ == 0);
+    assert(d.__is_black_ == true);
 
-        assert(e.__parent_ == &a);
-        assert(e.__is_black_ == true);
+    assert(a.__parent_ == &b);
+    assert(a.__left_ == 0);
+    assert(a.__right_ == 0);
+    assert(a.__is_black_ == false);
+  }
+  {
+    Node root;
+    Node a;
+    Node b;
+    Node c;
+    Node d;
 
-        assert(f.__parent_ == &c);
-        assert(f.__is_black_ == true);
+    root.__left_ = &c;
 
-        assert(g.__parent_ == &c);
-        assert(g.__is_black_ == true);
-    }
-}
+    c.__parent_   = &root;
+    c.__left_     = &b;
+    c.__right_    = &d;
+    c.__is_black_ = true;
 
-void
-test3()
-{
-    {
-        Node root;
-        Node a;
-        Node b;
-        Node c;
-
-        root.__left_ = &c;
-
-        c.__parent_ = &root;
-        c.__left_ = &b;
-        c.__right_ = 0;
-        c.__is_black_ = true;
-
-        b.__parent_ = &c;
-        b.__left_ = &a;
-        b.__right_ = 0;
-        b.__is_black_ = false;
-
-        a.__parent_ = &b;
-        a.__left_ = 0;
-        a.__right_ = 0;
-        a.__is_black_ = false;
-
-        std::__tree_balance_after_insert(root.__left_, &a);
-
-        assert(std::__tree_invariant(root.__left_));
-
-        assert(root.__left_ == &b);
-
-        assert(c.__parent_ == &b);
-        assert(c.__left_ == 0);
-        assert(c.__right_ == 0);
-        assert(c.__is_black_ == false);
-
-        assert(a.__parent_ == &b);
-        assert(a.__left_ == 0);
-        assert(a.__right_ == 0);
-        assert(a.__is_black_ == false);
-
-        assert(b.__parent_ == &root);
-        assert(b.__left_ == &a);
-        assert(b.__right_ == &c);
-        assert(b.__is_black_ == true);
-    }
-    {
-        Node root;
-        Node a;
-        Node b;
-        Node c;
-
-        root.__left_ = &a;
-
-        a.__parent_ = &root;
-        a.__left_ = 0;
-        a.__right_ = &b;
-        a.__is_black_ = true;
-
-        b.__parent_ = &a;
-        b.__left_ = 0;
-        b.__right_ = &c;
-        b.__is_black_ = false;
-
-        c.__parent_ = &b;
-        c.__left_ = 0;
-        c.__right_ = 0;
-        c.__is_black_ = false;
-
-        std::__tree_balance_after_insert(root.__left_, &c);
-
-        assert(std::__tree_invariant(root.__left_));
-
-        assert(root.__left_ == &b);
-
-        assert(a.__parent_ == &b);
-        assert(a.__left_ == 0);
-        assert(a.__right_ == 0);
-        assert(a.__is_black_ == false);
-
-        assert(c.__parent_ == &b);
-        assert(c.__left_ == 0);
-        assert(c.__right_ == 0);
-        assert(c.__is_black_ == false);
-
-        assert(b.__parent_ == &root);
-        assert(b.__left_ == &a);
-        assert(b.__right_ == &c);
-        assert(b.__is_black_ == true);
-    }
-    {
-        Node root;
-        Node a;
-        Node b;
-        Node c;
-        Node d;
-        Node e;
-        Node f;
-        Node g;
-
-        root.__left_ = &c;
-
-        c.__parent_ = &root;
-        c.__left_ = &b;
-        c.__right_ = &g;
-        c.__is_black_ = true;
+    b.__parent_   = &c;
+    b.__left_     = 0;
+    b.__right_    = 0;
+    b.__is_black_ = false;
 
-        b.__parent_ = &c;
-        b.__left_ = &a;
-        b.__right_ = &f;
-        b.__is_black_ = false;
-
-        a.__parent_ = &b;
-        a.__left_ = &d;
-        a.__right_ = &e;
-        a.__is_black_ = false;
-
-        d.__parent_ = &a;
-        d.__is_black_ = true;
-
-        e.__parent_ = &a;
-        e.__is_black_ = true;
-
-        f.__parent_ = &b;
-        f.__is_black_ = true;
-
-        g.__parent_ = &c;
-        g.__is_black_ = true;
-
-        std::__tree_balance_after_insert(root.__left_, &a);
-
-        assert(std::__tree_invariant(root.__left_));
-
-        assert(root.__left_ == &b);
-
-        assert(c.__parent_ == &b);
-        assert(c.__left_ == &f);
-        assert(c.__right_ == &g);
-        assert(c.__is_black_ == false);
-
-        assert(a.__parent_ == &b);
-        assert(a.__left_ == &d);
-        assert(a.__right_ == &e);
-        assert(a.__is_black_ == false);
-
-        assert(b.__parent_ == &root);
-        assert(b.__left_ == &a);
-        assert(b.__right_ == &c);
-        assert(b.__is_black_ == true);
+    d.__parent_   = &c;
+    d.__left_     = &a;
+    d.__right_    = 0;
+    d.__is_black_ = false;
 
-        assert(d.__parent_ == &a);
-        assert(d.__is_black_ == true);
+    a.__parent_   = &d;
+    a.__left_     = 0;
+    a.__right_    = 0;
+    a.__is_black_ = false;
 
-        assert(e.__parent_ == &a);
-        assert(e.__is_black_ == true);
+    std::__tree_balance_after_insert(root.__left_, &a);
 
-        assert(f.__parent_ == &c);
-        assert(f.__is_black_ == true);
+    assert(std::__tree_invariant(root.__left_));
 
-        assert(g.__parent_ == &c);
-        assert(g.__is_black_ == true);
-    }
-    {
-        Node root;
-        Node a;
-        Node b;
-        Node c;
-        Node d;
-        Node e;
-        Node f;
-        Node g;
+    assert(root.__left_ == &c);
 
-        root.__left_ = &a;
-
-        a.__parent_ = &root;
-        a.__left_ = &d;
-        a.__right_ = &b;
-        a.__is_black_ = true;
-
-        b.__parent_ = &a;
-        b.__left_ = &e;
-        b.__right_ = &c;
-        b.__is_black_ = false;
+    assert(c.__parent_ == &root);
+    assert(c.__left_ == &b);
+    assert(c.__right_ == &d);
+    assert(c.__is_black_ == true);
 
-        c.__parent_ = &b;
-        c.__left_ = &f;
-        c.__right_ = &g;
-        c.__is_black_ = false;
+    assert(b.__parent_ == &c);
+    assert(b.__left_ == 0);
+    assert(b.__right_ == 0);
+    assert(b.__is_black_ == true);
 
-        d.__parent_ = &a;
-        d.__is_black_ = true;
+    assert(d.__parent_ == &c);
+    assert(d.__left_ == &a);
+    assert(d.__right_ == 0);
+    assert(d.__is_black_ == true);
 
-        e.__parent_ = &b;
-        e.__is_black_ = true;
+    assert(a.__parent_ == &d);
+    assert(a.__left_ == 0);
+    assert(a.__right_ == 0);
+    assert(a.__is_black_ == false);
+  }
+  {
+    Node root;
+    Node a;
+    Node b;
+    Node c;
+    Node d;
 
-        f.__parent_ = &c;
-        f.__is_black_ = true;
+    root.__left_ = &c;
 
-        g.__parent_ = &c;
-        g.__is_black_ = true;
+    c.__parent_   = &root;
+    c.__left_     = &b;
+    c.__right_    = &d;
+    c.__is_black_ = true;
 
-        std::__tree_balance_after_insert(root.__left_, &c);
+    b.__parent_   = &c;
+    b.__left_     = 0;
+    b.__right_    = 0;
+    b.__is_black_ = false;
 
-        assert(std::__tree_invariant(root.__left_));
+    d.__parent_   = &c;
+    d.__left_     = 0;
+    d.__right_    = &a;
+    d.__is_black_ = false;
 
-        assert(root.__left_ == &b);
+    a.__parent_   = &d;
+    a.__left_     = 0;
+    a.__right_    = 0;
+    a.__is_black_ = false;
 
-        assert(c.__parent_ == &b);
-        assert(c.__left_ == &f);
-        assert(c.__right_ == &g);
-        assert(c.__is_black_ == false);
+    std::__tree_balance_after_insert(root.__left_, &a);
 
-        assert(a.__parent_ == &b);
-        assert(a.__left_ == &d);
-        assert(a.__right_ == &e);
-        assert(a.__is_black_ == false);
+    assert(std::__tree_invariant(root.__left_));
 
-        assert(b.__parent_ == &root);
-        assert(b.__left_ == &a);
-        assert(b.__right_ == &c);
-        assert(b.__is_black_ == true);
+    assert(root.__left_ == &c);
 
-        assert(d.__parent_ == &a);
-        assert(d.__is_black_ == true);
+    assert(c.__parent_ == &root);
+    assert(c.__left_ == &b);
+    assert(c.__right_ == &d);
+    assert(c.__is_black_ == true);
 
-        assert(e.__parent_ == &a);
-        assert(e.__is_black_ == true);
+    assert(b.__parent_ == &c);
+    assert(b.__left_ == 0);
+    assert(b.__right_ == 0);
+    assert(b.__is_black_ == true);
 
-        assert(f.__parent_ == &c);
-        assert(f.__is_black_ == true);
+    assert(d.__parent_ == &c);
+    assert(d.__left_ == 0);
+    assert(d.__right_ == &a);
+    assert(d.__is_black_ == true);
 
-        assert(g.__parent_ == &c);
-        assert(g.__is_black_ == true);
-    }
-}
+    assert(a.__parent_ == &d);
+    assert(a.__left_ == 0);
+    assert(a.__right_ == 0);
+    assert(a.__is_black_ == false);
+  }
+  {
+    Node root;
+    Node a;
+    Node b;
+    Node c;
+    Node d;
+    Node e;
+    Node f;
+    Node g;
+    Node h;
+    Node i;
+
+    root.__left_ = &c;
+
+    c.__parent_   = &root;
+    c.__left_     = &b;
+    c.__right_    = &d;
+    c.__is_black_ = true;
+
+    b.__parent_   = &c;
+    b.__left_     = &a;
+    b.__right_    = &g;
+    b.__is_black_ = false;
+
+    d.__parent_   = &c;
+    d.__left_     = &h;
+    d.__right_    = &i;
+    d.__is_black_ = false;
+
+    a.__parent_   = &b;
+    a.__left_     = &e;
+    a.__right_    = &f;
+    a.__is_black_ = false;
+
+    e.__parent_   = &a;
+    e.__is_black_ = true;
+
+    f.__parent_   = &a;
+    f.__is_black_ = true;
+
+    g.__parent_   = &b;
+    g.__is_black_ = true;
+
+    h.__parent_   = &d;
+    h.__is_black_ = true;
+
+    i.__parent_   = &d;
+    i.__is_black_ = true;
+
+    std::__tree_balance_after_insert(root.__left_, &a);
+
+    assert(std::__tree_invariant(root.__left_));
+
+    assert(root.__left_ == &c);
 
-void
-test4()
-{
+    assert(c.__parent_ == &root);
+    assert(c.__left_ == &b);
+    assert(c.__right_ == &d);
+    assert(c.__is_black_ == true);
+
+    assert(b.__parent_ == &c);
+    assert(b.__left_ == &a);
+    assert(b.__right_ == &g);
+    assert(b.__is_black_ == true);
+
+    assert(d.__parent_ == &c);
+    assert(d.__left_ == &h);
+    assert(d.__right_ == &i);
+    assert(d.__is_black_ == true);
+
+    assert(a.__parent_ == &b);
+    assert(a.__left_ == &e);
+    assert(a.__right_ == &f);
+    assert(a.__is_black_ == false);
+  }
+  {
     Node root;
     Node a;
     Node b;
@@ -1044,57 +338,257 @@ test4()
     Node f;
     Node g;
     Node h;
+    Node i;
 
-    root.__left_ = &a;
-    a.__parent_ = &root;
+    root.__left_ = &c;
+
+    c.__parent_   = &root;
+    c.__left_     = &b;
+    c.__right_    = &d;
+    c.__is_black_ = true;
+
+    b.__parent_   = &c;
+    b.__left_     = &g;
+    b.__right_    = &a;
+    b.__is_black_ = false;
+
+    d.__parent_   = &c;
+    d.__left_     = &h;
+    d.__right_    = &i;
+    d.__is_black_ = false;
+
+    a.__parent_   = &b;
+    a.__left_     = &e;
+    a.__right_    = &f;
+    a.__is_black_ = false;
+
+    e.__parent_   = &a;
+    e.__is_black_ = true;
+
+    f.__parent_   = &a;
+    f.__is_black_ = true;
+
+    g.__parent_   = &b;
+    g.__is_black_ = true;
+
+    h.__parent_   = &d;
+    h.__is_black_ = true;
+
+    i.__parent_   = &d;
+    i.__is_black_ = true;
 
     std::__tree_balance_after_insert(root.__left_, &a);
 
     assert(std::__tree_invariant(root.__left_));
 
-    assert(root.__parent_ == 0);
-    assert(root.__left_ == &a);
-    assert(root.__right_ == 0);
-    assert(root.__is_black_ == false);
+    assert(root.__left_ == &c);
 
-    assert(a.__parent_ == &root);
-    assert(a.__left_ == 0);
-    assert(a.__right_ == 0);
-    assert(a.__is_black_ == true);
+    assert(c.__parent_ == &root);
+    assert(c.__left_ == &b);
+    assert(c.__right_ == &d);
+    assert(c.__is_black_ == true);
 
-    a.__right_ = &b;
-    b.__parent_ = &a;
+    assert(b.__parent_ == &c);
+    assert(b.__left_ == &g);
+    assert(b.__right_ == &a);
+    assert(b.__is_black_ == true);
 
-    std::__tree_balance_after_insert(root.__left_, &b);
+    assert(d.__parent_ == &c);
+    assert(d.__left_ == &h);
+    assert(d.__right_ == &i);
+    assert(d.__is_black_ == true);
+
+    assert(a.__parent_ == &b);
+    assert(a.__left_ == &e);
+    assert(a.__right_ == &f);
+    assert(a.__is_black_ == false);
+  }
+  {
+    Node root;
+    Node a;
+    Node b;
+    Node c;
+    Node d;
+    Node e;
+    Node f;
+    Node g;
+    Node h;
+    Node i;
+
+    root.__left_ = &c;
+
+    c.__parent_   = &root;
+    c.__left_     = &b;
+    c.__right_    = &d;
+    c.__is_black_ = true;
+
+    b.__parent_   = &c;
+    b.__left_     = &g;
+    b.__right_    = &h;
+    b.__is_black_ = false;
+
+    d.__parent_   = &c;
+    d.__left_     = &a;
+    d.__right_    = &i;
+    d.__is_black_ = false;
+
+    a.__parent_   = &d;
+    a.__left_     = &e;
+    a.__right_    = &f;
+    a.__is_black_ = false;
+
+    e.__parent_   = &a;
+    e.__is_black_ = true;
+
+    f.__parent_   = &a;
+    f.__is_black_ = true;
+
+    g.__parent_   = &b;
+    g.__is_black_ = true;
+
+    h.__parent_   = &b;
+    h.__is_black_ = true;
+
+    i.__parent_   = &d;
+    i.__is_black_ = true;
+
+    std::__tree_balance_after_insert(root.__left_, &a);
 
     assert(std::__tree_invariant(root.__left_));
 
-    assert(root.__parent_ == 0);
-    assert(root.__left_ == &a);
-    assert(root.__right_ == 0);
-    assert(root.__is_black_ == false);
+    assert(root.__left_ == &c);
 
-    assert(a.__parent_ == &root);
-    assert(a.__left_ == 0);
-    assert(a.__right_ == &b);
-    assert(a.__is_black_ == true);
+    assert(c.__parent_ == &root);
+    assert(c.__left_ == &b);
+    assert(c.__right_ == &d);
+    assert(c.__is_black_ == true);
 
-    assert(b.__parent_ == &a);
-    assert(b.__left_ == 0);
-    assert(b.__right_ == 0);
-    assert(b.__is_black_ == false);
+    assert(b.__parent_ == &c);
+    assert(b.__left_ == &g);
+    assert(b.__right_ == &h);
+    assert(b.__is_black_ == true);
 
-    b.__right_ = &c;
-    c.__parent_ = &b;
+    assert(d.__parent_ == &c);
+    assert(d.__left_ == &a);
+    assert(d.__right_ == &i);
+    assert(d.__is_black_ == true);
 
-    std::__tree_balance_after_insert(root.__left_, &c);
+    assert(a.__parent_ == &d);
+    assert(a.__left_ == &e);
+    assert(a.__right_ == &f);
+    assert(a.__is_black_ == false);
+  }
+  {
+    Node root;
+    Node a;
+    Node b;
+    Node c;
+    Node d;
+    Node e;
+    Node f;
+    Node g;
+    Node h;
+    Node i;
+
+    root.__left_ = &c;
+
+    c.__parent_   = &root;
+    c.__left_     = &b;
+    c.__right_    = &d;
+    c.__is_black_ = true;
+
+    b.__parent_   = &c;
+    b.__left_     = &g;
+    b.__right_    = &h;
+    b.__is_black_ = false;
+
+    d.__parent_   = &c;
+    d.__left_     = &i;
+    d.__right_    = &a;
+    d.__is_black_ = false;
+
+    a.__parent_   = &d;
+    a.__left_     = &e;
+    a.__right_    = &f;
+    a.__is_black_ = false;
+
+    e.__parent_   = &a;
+    e.__is_black_ = true;
+
+    f.__parent_   = &a;
+    f.__is_black_ = true;
+
+    g.__parent_   = &b;
+    g.__is_black_ = true;
+
+    h.__parent_   = &b;
+    h.__is_black_ = true;
+
+    i.__parent_   = &d;
+    i.__is_black_ = true;
+
+    std::__tree_balance_after_insert(root.__left_, &a);
+
+    assert(std::__tree_invariant(root.__left_));
+
+    assert(root.__left_ == &c);
+
+    assert(c.__parent_ == &root);
+    assert(c.__left_ == &b);
+    assert(c.__right_ == &d);
+    assert(c.__is_black_ == true);
+
+    assert(b.__parent_ == &c);
+    assert(b.__left_ == &g);
+    assert(b.__right_ == &h);
+    assert(b.__is_black_ == true);
+
+    assert(d.__parent_ == &c);
+    assert(d.__left_ == &i);
+    assert(d.__right_ == &a);
+    assert(d.__is_black_ == true);
+
+    assert(a.__parent_ == &d);
+    assert(a.__left_ == &e);
+    assert(a.__right_ == &f);
+    assert(a.__is_black_ == false);
+  }
+}
+
+void test2() {
+  {
+    Node root;
+    Node a;
+    Node b;
+    Node c;
+
+    root.__left_ = &c;
+
+    c.__parent_   = &root;
+    c.__left_     = &a;
+    c.__right_    = 0;
+    c.__is_black_ = true;
+
+    a.__parent_   = &c;
+    a.__left_     = 0;
+    a.__right_    = &b;
+    a.__is_black_ = false;
+
+    b.__parent_   = &a;
+    b.__left_     = 0;
+    b.__right_    = 0;
+    b.__is_black_ = false;
+
+    std::__tree_balance_after_insert(root.__left_, &b);
 
     assert(std::__tree_invariant(root.__left_));
 
-    assert(root.__parent_ == 0);
     assert(root.__left_ == &b);
-    assert(root.__right_ == 0);
-    assert(root.__is_black_ == false);
+
+    assert(c.__parent_ == &b);
+    assert(c.__left_ == 0);
+    assert(c.__right_ == 0);
+    assert(c.__is_black_ == false);
 
     assert(a.__parent_ == &b);
     assert(a.__left_ == 0);
@@ -1105,226 +599,359 @@ test4()
     assert(b.__left_ == &a);
     assert(b.__right_ == &c);
     assert(b.__is_black_ == true);
+  }
+  {
+    Node root;
+    Node a;
+    Node b;
+    Node c;
 
-    assert(c.__parent_ == &b);
-    assert(c.__left_ == 0);
-    assert(c.__right_ == 0);
-    assert(c.__is_black_ == false);
+    root.__left_ = &a;
+
+    a.__parent_   = &root;
+    a.__left_     = 0;
+    a.__right_    = &c;
+    a.__is_black_ = true;
 
-    c.__right_ = &d;
-    d.__parent_ = &c;
+    c.__parent_   = &a;
+    c.__left_     = &b;
+    c.__right_    = 0;
+    c.__is_black_ = false;
 
-    std::__tree_balance_after_insert(root.__left_, &d);
+    b.__parent_   = &c;
+    b.__left_     = 0;
+    b.__right_    = 0;
+    b.__is_black_ = false;
+
+    std::__tree_balance_after_insert(root.__left_, &b);
 
     assert(std::__tree_invariant(root.__left_));
 
-    assert(root.__parent_ == 0);
     assert(root.__left_ == &b);
-    assert(root.__right_ == 0);
-    assert(root.__is_black_ == false);
 
     assert(a.__parent_ == &b);
     assert(a.__left_ == 0);
     assert(a.__right_ == 0);
-    assert(a.__is_black_ == true);
+    assert(a.__is_black_ == false);
+
+    assert(c.__parent_ == &b);
+    assert(c.__left_ == 0);
+    assert(c.__right_ == 0);
+    assert(c.__is_black_ == false);
 
     assert(b.__parent_ == &root);
     assert(b.__left_ == &a);
     assert(b.__right_ == &c);
     assert(b.__is_black_ == true);
+  }
+  {
+    Node root;
+    Node a;
+    Node b;
+    Node c;
+    Node d;
+    Node e;
+    Node f;
+    Node g;
 
-    assert(c.__parent_ == &b);
-    assert(c.__left_ == 0);
-    assert(c.__right_ == &d);
-    assert(c.__is_black_ == true);
+    root.__left_ = &c;
 
-    assert(d.__parent_ == &c);
-    assert(d.__left_ == 0);
-    assert(d.__right_ == 0);
-    assert(d.__is_black_ == false);
+    c.__parent_   = &root;
+    c.__left_     = &a;
+    c.__right_    = &g;
+    c.__is_black_ = true;
 
-    d.__right_ = &e;
-    e.__parent_ = &d;
+    a.__parent_   = &c;
+    a.__left_     = &d;
+    a.__right_    = &b;
+    a.__is_black_ = false;
 
-    std::__tree_balance_after_insert(root.__left_, &e);
+    b.__parent_   = &a;
+    b.__left_     = &e;
+    b.__right_    = &f;
+    b.__is_black_ = false;
+
+    d.__parent_   = &a;
+    d.__is_black_ = true;
+
+    e.__parent_   = &b;
+    e.__is_black_ = true;
+
+    f.__parent_   = &b;
+    f.__is_black_ = true;
+
+    g.__parent_   = &c;
+    g.__is_black_ = true;
+
+    std::__tree_balance_after_insert(root.__left_, &b);
 
     assert(std::__tree_invariant(root.__left_));
 
-    assert(root.__parent_ == 0);
     assert(root.__left_ == &b);
-    assert(root.__right_ == 0);
-    assert(root.__is_black_ == false);
+
+    assert(c.__parent_ == &b);
+    assert(c.__left_ == &f);
+    assert(c.__right_ == &g);
+    assert(c.__is_black_ == false);
+
+    assert(a.__parent_ == &b);
+    assert(a.__left_ == &d);
+    assert(a.__right_ == &e);
+    assert(a.__is_black_ == false);
 
     assert(b.__parent_ == &root);
     assert(b.__left_ == &a);
-    assert(b.__right_ == &d);
+    assert(b.__right_ == &c);
     assert(b.__is_black_ == true);
 
-    assert(a.__parent_ == &b);
-    assert(a.__left_ == 0);
-    assert(a.__right_ == 0);
-    assert(a.__is_black_ == true);
-
-    assert(d.__parent_ == &b);
-    assert(d.__left_ == &c);
-    assert(d.__right_ == &e);
+    assert(d.__parent_ == &a);
     assert(d.__is_black_ == true);
 
-    assert(c.__parent_ == &d);
-    assert(c.__left_ == 0);
-    assert(c.__right_ == 0);
-    assert(c.__is_black_ == false);
+    assert(e.__parent_ == &a);
+    assert(e.__is_black_ == true);
 
-    assert(e.__parent_ == &d);
-    assert(e.__left_ == 0);
-    assert(e.__right_ == 0);
-    assert(e.__is_black_ == false);
+    assert(f.__parent_ == &c);
+    assert(f.__is_black_ == true);
 
-    e.__right_ = &f;
-    f.__parent_ = &e;
+    assert(g.__parent_ == &c);
+    assert(g.__is_black_ == true);
+  }
+  {
+    Node root;
+    Node a;
+    Node b;
+    Node c;
+    Node d;
+    Node e;
+    Node f;
+    Node g;
 
-    std::__tree_balance_after_insert(root.__left_, &f);
+    root.__left_ = &a;
+
+    a.__parent_   = &root;
+    a.__left_     = &d;
+    a.__right_    = &c;
+    a.__is_black_ = true;
+
+    c.__parent_   = &a;
+    c.__left_     = &b;
+    c.__right_    = &g;
+    c.__is_black_ = false;
+
+    b.__parent_   = &c;
+    b.__left_     = &e;
+    b.__right_    = &f;
+    b.__is_black_ = false;
+
+    d.__parent_   = &a;
+    d.__is_black_ = true;
+
+    e.__parent_   = &b;
+    e.__is_black_ = true;
+
+    f.__parent_   = &b;
+    f.__is_black_ = true;
+
+    g.__parent_   = &c;
+    g.__is_black_ = true;
+
+    std::__tree_balance_after_insert(root.__left_, &b);
 
     assert(std::__tree_invariant(root.__left_));
 
-    assert(root.__parent_ == 0);
     assert(root.__left_ == &b);
-    assert(root.__right_ == 0);
-    assert(root.__is_black_ == false);
+
+    assert(c.__parent_ == &b);
+    assert(c.__left_ == &f);
+    assert(c.__right_ == &g);
+    assert(c.__is_black_ == false);
+
+    assert(a.__parent_ == &b);
+    assert(a.__left_ == &d);
+    assert(a.__right_ == &e);
+    assert(a.__is_black_ == false);
 
     assert(b.__parent_ == &root);
     assert(b.__left_ == &a);
-    assert(b.__right_ == &d);
+    assert(b.__right_ == &c);
     assert(b.__is_black_ == true);
 
-    assert(a.__parent_ == &b);
-    assert(a.__left_ == 0);
-    assert(a.__right_ == 0);
-    assert(a.__is_black_ == true);
+    assert(d.__parent_ == &a);
+    assert(d.__is_black_ == true);
 
-    assert(d.__parent_ == &b);
-    assert(d.__left_ == &c);
-    assert(d.__right_ == &e);
-    assert(d.__is_black_ == false);
+    assert(e.__parent_ == &a);
+    assert(e.__is_black_ == true);
 
-    assert(c.__parent_ == &d);
-    assert(c.__left_ == 0);
-    assert(c.__right_ == 0);
-    assert(c.__is_black_ == true);
+    assert(f.__parent_ == &c);
+    assert(f.__is_black_ == true);
 
-    assert(e.__parent_ == &d);
-    assert(e.__left_ == 0);
-    assert(e.__right_ == &f);
-    assert(e.__is_black_ == true);
+    assert(g.__parent_ == &c);
+    assert(g.__is_black_ == true);
+  }
+}
 
-    assert(f.__parent_ == &e);
-    assert(f.__left_ == 0);
-    assert(f.__right_ == 0);
-    assert(f.__is_black_ == false);
+void test3() {
+  {
+    Node root;
+    Node a;
+    Node b;
+    Node c;
+
+    root.__left_ = &c;
+
+    c.__parent_   = &root;
+    c.__left_     = &b;
+    c.__right_    = 0;
+    c.__is_black_ = true;
 
-    f.__right_ = &g;
-    g.__parent_ = &f;
+    b.__parent_   = &c;
+    b.__left_     = &a;
+    b.__right_    = 0;
+    b.__is_black_ = false;
 
-    std::__tree_balance_after_insert(root.__left_, &g);
+    a.__parent_   = &b;
+    a.__left_     = 0;
+    a.__right_    = 0;
+    a.__is_black_ = false;
+
+    std::__tree_balance_after_insert(root.__left_, &a);
 
     assert(std::__tree_invariant(root.__left_));
 
-    assert(root.__parent_ == 0);
     assert(root.__left_ == &b);
-    assert(root.__right_ == 0);
-    assert(root.__is_black_ == false);
+
+    assert(c.__parent_ == &b);
+    assert(c.__left_ == 0);
+    assert(c.__right_ == 0);
+    assert(c.__is_black_ == false);
+
+    assert(a.__parent_ == &b);
+    assert(a.__left_ == 0);
+    assert(a.__right_ == 0);
+    assert(a.__is_black_ == false);
 
     assert(b.__parent_ == &root);
     assert(b.__left_ == &a);
-    assert(b.__right_ == &d);
+    assert(b.__right_ == &c);
     assert(b.__is_black_ == true);
+  }
+  {
+    Node root;
+    Node a;
+    Node b;
+    Node c;
+
+    root.__left_ = &a;
+
+    a.__parent_   = &root;
+    a.__left_     = 0;
+    a.__right_    = &b;
+    a.__is_black_ = true;
+
+    b.__parent_   = &a;
+    b.__left_     = 0;
+    b.__right_    = &c;
+    b.__is_black_ = false;
+
+    c.__parent_   = &b;
+    c.__left_     = 0;
+    c.__right_    = 0;
+    c.__is_black_ = false;
+
+    std::__tree_balance_after_insert(root.__left_, &c);
+
+    assert(std::__tree_invariant(root.__left_));
+
+    assert(root.__left_ == &b);
 
     assert(a.__parent_ == &b);
     assert(a.__left_ == 0);
     assert(a.__right_ == 0);
-    assert(a.__is_black_ == true);
-
-    assert(d.__parent_ == &b);
-    assert(d.__left_ == &c);
-    assert(d.__right_ == &f);
-    assert(d.__is_black_ == false);
+    assert(a.__is_black_ == false);
 
-    assert(c.__parent_ == &d);
+    assert(c.__parent_ == &b);
     assert(c.__left_ == 0);
     assert(c.__right_ == 0);
-    assert(c.__is_black_ == true);
+    assert(c.__is_black_ == false);
 
-    assert(f.__parent_ == &d);
-    assert(f.__left_ == &e);
-    assert(f.__right_ == &g);
-    assert(f.__is_black_ == true);
+    assert(b.__parent_ == &root);
+    assert(b.__left_ == &a);
+    assert(b.__right_ == &c);
+    assert(b.__is_black_ == true);
+  }
+  {
+    Node root;
+    Node a;
+    Node b;
+    Node c;
+    Node d;
+    Node e;
+    Node f;
+    Node g;
 
-    assert(e.__parent_ == &f);
-    assert(e.__left_ == 0);
-    assert(e.__right_ == 0);
-    assert(e.__is_black_ == false);
+    root.__left_ = &c;
 
-    assert(g.__parent_ == &f);
-    assert(g.__left_ == 0);
-    assert(g.__right_ == 0);
-    assert(g.__is_black_ == false);
+    c.__parent_   = &root;
+    c.__left_     = &b;
+    c.__right_    = &g;
+    c.__is_black_ = true;
 
-    g.__right_ = &h;
-    h.__parent_ = &g;
+    b.__parent_   = &c;
+    b.__left_     = &a;
+    b.__right_    = &f;
+    b.__is_black_ = false;
 
-    std::__tree_balance_after_insert(root.__left_, &h);
+    a.__parent_   = &b;
+    a.__left_     = &d;
+    a.__right_    = &e;
+    a.__is_black_ = false;
 
-    assert(std::__tree_invariant(root.__left_));
+    d.__parent_   = &a;
+    d.__is_black_ = true;
 
-    assert(root.__parent_ == 0);
-    assert(root.__left_ == &d);
-    assert(root.__right_ == 0);
-    assert(root.__is_black_ == false);
+    e.__parent_   = &a;
+    e.__is_black_ = true;
 
-    assert(d.__parent_ == &root);
-    assert(d.__left_ == &b);
-    assert(d.__right_ == &f);
-    assert(d.__is_black_ == true);
+    f.__parent_   = &b;
+    f.__is_black_ = true;
 
-    assert(b.__parent_ == &d);
-    assert(b.__left_ == &a);
-    assert(b.__right_ == &c);
-    assert(b.__is_black_ == false);
+    g.__parent_   = &c;
+    g.__is_black_ = true;
 
-    assert(a.__parent_ == &b);
-    assert(a.__left_ == 0);
-    assert(a.__right_ == 0);
-    assert(a.__is_black_ == true);
+    std::__tree_balance_after_insert(root.__left_, &a);
+
+    assert(std::__tree_invariant(root.__left_));
+
+    assert(root.__left_ == &b);
 
     assert(c.__parent_ == &b);
-    assert(c.__left_ == 0);
-    assert(c.__right_ == 0);
-    assert(c.__is_black_ == true);
+    assert(c.__left_ == &f);
+    assert(c.__right_ == &g);
+    assert(c.__is_black_ == false);
 
-    assert(f.__parent_ == &d);
-    assert(f.__left_ == &e);
-    assert(f.__right_ == &g);
-    assert(f.__is_black_ == false);
+    assert(a.__parent_ == &b);
+    assert(a.__left_ == &d);
+    assert(a.__right_ == &e);
+    assert(a.__is_black_ == false);
 
-    assert(e.__parent_ == &f);
-    assert(e.__left_ == 0);
-    assert(e.__right_ == 0);
-    assert(e.__is_black_ == true);
+    assert(b.__parent_ == &root);
+    assert(b.__left_ == &a);
+    assert(b.__right_ == &c);
+    assert(b.__is_black_ == true);
 
-    assert(g.__parent_ == &f);
-    assert(g.__left_ == 0);
-    assert(g.__right_ == &h);
-    assert(g.__is_black_ == true);
+    assert(d.__parent_ == &a);
+    assert(d.__is_black_ == true);
 
-    assert(h.__parent_ == &g);
-    assert(h.__left_ == 0);
-    assert(h.__right_ == 0);
-    assert(h.__is_black_ == false);
-}
+    assert(e.__parent_ == &a);
+    assert(e.__is_black_ == true);
 
-void
-test5()
-{
+    assert(f.__parent_ == &c);
+    assert(f.__is_black_ == true);
+
+    assert(g.__parent_ == &c);
+    assert(g.__is_black_ == true);
+  }
+  {
     Node root;
     Node a;
     Node b;
@@ -1333,292 +960,653 @@ test5()
     Node e;
     Node f;
     Node g;
-    Node h;
 
-    root.__left_ = &h;
-    h.__parent_ = &root;
+    root.__left_ = &a;
 
-    std::__tree_balance_after_insert(root.__left_, &h);
+    a.__parent_   = &root;
+    a.__left_     = &d;
+    a.__right_    = &b;
+    a.__is_black_ = true;
 
-    assert(std::__tree_invariant(root.__left_));
+    b.__parent_   = &a;
+    b.__left_     = &e;
+    b.__right_    = &c;
+    b.__is_black_ = false;
+
+    c.__parent_   = &b;
+    c.__left_     = &f;
+    c.__right_    = &g;
+    c.__is_black_ = false;
+
+    d.__parent_   = &a;
+    d.__is_black_ = true;
 
-    assert(root.__parent_ == 0);
-    assert(root.__left_ == &h);
-    assert(root.__right_ == 0);
-    assert(root.__is_black_ == false);
+    e.__parent_   = &b;
+    e.__is_black_ = true;
 
-    assert(h.__parent_ == &root);
-    assert(h.__left_ == 0);
-    assert(h.__right_ == 0);
-    assert(h.__is_black_ == true);
+    f.__parent_   = &c;
+    f.__is_black_ = true;
 
-    h.__left_ = &g;
-    g.__parent_ = &h;
+    g.__parent_   = &c;
+    g.__is_black_ = true;
 
-    std::__tree_balance_after_insert(root.__left_, &g);
+    std::__tree_balance_after_insert(root.__left_, &c);
 
     assert(std::__tree_invariant(root.__left_));
 
-    assert(root.__parent_ == 0);
-    assert(root.__left_ == &h);
-    assert(root.__right_ == 0);
-    assert(root.__is_black_ == false);
+    assert(root.__left_ == &b);
 
-    assert(h.__parent_ == &root);
-    assert(h.__left_ == &g);
-    assert(h.__right_ == 0);
-    assert(h.__is_black_ == true);
+    assert(c.__parent_ == &b);
+    assert(c.__left_ == &f);
+    assert(c.__right_ == &g);
+    assert(c.__is_black_ == false);
 
-    assert(g.__parent_ == &h);
-    assert(g.__left_ == 0);
-    assert(g.__right_ == 0);
-    assert(g.__is_black_ == false);
+    assert(a.__parent_ == &b);
+    assert(a.__left_ == &d);
+    assert(a.__right_ == &e);
+    assert(a.__is_black_ == false);
 
-    g.__left_ = &f;
-    f.__parent_ = &g;
+    assert(b.__parent_ == &root);
+    assert(b.__left_ == &a);
+    assert(b.__right_ == &c);
+    assert(b.__is_black_ == true);
 
-    std::__tree_balance_after_insert(root.__left_, &f);
+    assert(d.__parent_ == &a);
+    assert(d.__is_black_ == true);
 
-    assert(std::__tree_invariant(root.__left_));
+    assert(e.__parent_ == &a);
+    assert(e.__is_black_ == true);
 
-    assert(root.__parent_ == 0);
-    assert(root.__left_ == &g);
-    assert(root.__right_ == 0);
-    assert(root.__is_black_ == false);
+    assert(f.__parent_ == &c);
+    assert(f.__is_black_ == true);
 
-    assert(g.__parent_ == &root);
-    assert(g.__left_ == &f);
-    assert(g.__right_ == &h);
+    assert(g.__parent_ == &c);
     assert(g.__is_black_ == true);
+  }
+}
 
-    assert(f.__parent_ == &g);
-    assert(f.__left_ == 0);
-    assert(f.__right_ == 0);
-    assert(f.__is_black_ == false);
+void test4() {
+  Node root;
+  Node a;
+  Node b;
+  Node c;
+  Node d;
+  Node e;
+  Node f;
+  Node g;
+  Node h;
 
-    assert(h.__parent_ == &g);
-    assert(h.__left_ == 0);
-    assert(h.__right_ == 0);
-    assert(h.__is_black_ == false);
+  root.__left_ = &a;
+  a.__parent_  = &root;
 
-    f.__left_ = &e;
-    e.__parent_ = &f;
+  std::__tree_balance_after_insert(root.__left_, &a);
 
-    std::__tree_balance_after_insert(root.__left_, &e);
+  assert(std::__tree_invariant(root.__left_));
 
-    assert(std::__tree_invariant(root.__left_));
+  assert(root.__parent_ == 0);
+  assert(root.__left_ == &a);
+  assert(root.__right_ == 0);
+  assert(root.__is_black_ == false);
 
-    assert(root.__parent_ == 0);
-    assert(root.__left_ == &g);
-    assert(root.__right_ == 0);
-    assert(root.__is_black_ == false);
+  assert(a.__parent_ == &root);
+  assert(a.__left_ == 0);
+  assert(a.__right_ == 0);
+  assert(a.__is_black_ == true);
 
-    assert(g.__parent_ == &root);
-    assert(g.__left_ == &f);
-    assert(g.__right_ == &h);
-    assert(g.__is_black_ == true);
+  a.__right_  = &b;
+  b.__parent_ = &a;
 
-    assert(f.__parent_ == &g);
-    assert(f.__left_ == &e);
-    assert(f.__right_ == 0);
-    assert(f.__is_black_ == true);
+  std::__tree_balance_after_insert(root.__left_, &b);
 
-    assert(e.__parent_ == &f);
-    assert(e.__left_ == 0);
-    assert(e.__right_ == 0);
-    assert(e.__is_black_ == false);
+  assert(std::__tree_invariant(root.__left_));
 
-    assert(h.__parent_ == &g);
-    assert(h.__left_ == 0);
-    assert(h.__right_ == 0);
-    assert(h.__is_black_ == true);
+  assert(root.__parent_ == 0);
+  assert(root.__left_ == &a);
+  assert(root.__right_ == 0);
+  assert(root.__is_black_ == false);
 
-    e.__left_ = &d;
-    d.__parent_ = &e;
+  assert(a.__parent_ == &root);
+  assert(a.__left_ == 0);
+  assert(a.__right_ == &b);
+  assert(a.__is_black_ == true);
 
-    std::__tree_balance_after_insert(root.__left_, &d);
+  assert(b.__parent_ == &a);
+  assert(b.__left_ == 0);
+  assert(b.__right_ == 0);
+  assert(b.__is_black_ == false);
 
-    assert(std::__tree_invariant(root.__left_));
+  b.__right_  = &c;
+  c.__parent_ = &b;
 
-    assert(root.__parent_ == 0);
-    assert(root.__left_ == &g);
-    assert(root.__right_ == 0);
-    assert(root.__is_black_ == false);
+  std::__tree_balance_after_insert(root.__left_, &c);
 
-    assert(g.__parent_ == &root);
-    assert(g.__left_ == &e);
-    assert(g.__right_ == &h);
-    assert(g.__is_black_ == true);
+  assert(std::__tree_invariant(root.__left_));
 
-    assert(e.__parent_ == &g);
-    assert(e.__left_ == &d);
-    assert(e.__right_ == &f);
-    assert(e.__is_black_ == true);
+  assert(root.__parent_ == 0);
+  assert(root.__left_ == &b);
+  assert(root.__right_ == 0);
+  assert(root.__is_black_ == false);
 
-    assert(d.__parent_ == &e);
-    assert(d.__left_ == 0);
-    assert(d.__right_ == 0);
-    assert(d.__is_black_ == false);
+  assert(a.__parent_ == &b);
+  assert(a.__left_ == 0);
+  assert(a.__right_ == 0);
+  assert(a.__is_black_ == false);
 
-    assert(f.__parent_ == &e);
-    assert(f.__left_ == 0);
-    assert(f.__right_ == 0);
-    assert(f.__is_black_ == false);
+  assert(b.__parent_ == &root);
+  assert(b.__left_ == &a);
+  assert(b.__right_ == &c);
+  assert(b.__is_black_ == true);
 
-    assert(h.__parent_ == &g);
-    assert(h.__left_ == 0);
-    assert(h.__right_ == 0);
-    assert(h.__is_black_ == true);
+  assert(c.__parent_ == &b);
+  assert(c.__left_ == 0);
+  assert(c.__right_ == 0);
+  assert(c.__is_black_ == false);
 
-    d.__left_ = &c;
-    c.__parent_ = &d;
+  c.__right_  = &d;
+  d.__parent_ = &c;
 
-    std::__tree_balance_after_insert(root.__left_, &c);
+  std::__tree_balance_after_insert(root.__left_, &d);
 
-    assert(std::__tree_invariant(root.__left_));
+  assert(std::__tree_invariant(root.__left_));
 
-    assert(root.__parent_ == 0);
-    assert(root.__left_ == &g);
-    assert(root.__right_ == 0);
-    assert(root.__is_black_ == false);
+  assert(root.__parent_ == 0);
+  assert(root.__left_ == &b);
+  assert(root.__right_ == 0);
+  assert(root.__is_black_ == false);
 
-    assert(g.__parent_ == &root);
-    assert(g.__left_ == &e);
-    assert(g.__right_ == &h);
-    assert(g.__is_black_ == true);
+  assert(a.__parent_ == &b);
+  assert(a.__left_ == 0);
+  assert(a.__right_ == 0);
+  assert(a.__is_black_ == true);
 
-    assert(e.__parent_ == &g);
-    assert(e.__left_ == &d);
-    assert(e.__right_ == &f);
-    assert(e.__is_black_ == false);
+  assert(b.__parent_ == &root);
+  assert(b.__left_ == &a);
+  assert(b.__right_ == &c);
+  assert(b.__is_black_ == true);
 
-    assert(d.__parent_ == &e);
-    assert(d.__left_ == &c);
-    assert(d.__right_ == 0);
-    assert(d.__is_black_ == true);
+  assert(c.__parent_ == &b);
+  assert(c.__left_ == 0);
+  assert(c.__right_ == &d);
+  assert(c.__is_black_ == true);
 
-    assert(c.__parent_ == &d);
-    assert(c.__left_ == 0);
-    assert(c.__right_ == 0);
-    assert(c.__is_black_ == false);
+  assert(d.__parent_ == &c);
+  assert(d.__left_ == 0);
+  assert(d.__right_ == 0);
+  assert(d.__is_black_ == false);
 
-    assert(f.__parent_ == &e);
-    assert(f.__left_ == 0);
-    assert(f.__right_ == 0);
-    assert(f.__is_black_ == true);
+  d.__right_  = &e;
+  e.__parent_ = &d;
 
-    assert(h.__parent_ == &g);
-    assert(h.__left_ == 0);
-    assert(h.__right_ == 0);
-    assert(h.__is_black_ == true);
+  std::__tree_balance_after_insert(root.__left_, &e);
 
-    c.__left_ = &b;
-    b.__parent_ = &c;
+  assert(std::__tree_invariant(root.__left_));
 
-    std::__tree_balance_after_insert(root.__left_, &b);
+  assert(root.__parent_ == 0);
+  assert(root.__left_ == &b);
+  assert(root.__right_ == 0);
+  assert(root.__is_black_ == false);
 
-    assert(std::__tree_invariant(root.__left_));
+  assert(b.__parent_ == &root);
+  assert(b.__left_ == &a);
+  assert(b.__right_ == &d);
+  assert(b.__is_black_ == true);
 
-    assert(root.__parent_ == 0);
-    assert(root.__left_ == &g);
-    assert(root.__right_ == 0);
-    assert(root.__is_black_ == false);
+  assert(a.__parent_ == &b);
+  assert(a.__left_ == 0);
+  assert(a.__right_ == 0);
+  assert(a.__is_black_ == true);
 
-    assert(g.__parent_ == &root);
-    assert(g.__left_ == &e);
-    assert(g.__right_ == &h);
-    assert(g.__is_black_ == true);
+  assert(d.__parent_ == &b);
+  assert(d.__left_ == &c);
+  assert(d.__right_ == &e);
+  assert(d.__is_black_ == true);
+
+  assert(c.__parent_ == &d);
+  assert(c.__left_ == 0);
+  assert(c.__right_ == 0);
+  assert(c.__is_black_ == false);
+
+  assert(e.__parent_ == &d);
+  assert(e.__left_ == 0);
+  assert(e.__right_ == 0);
+  assert(e.__is_black_ == false);
+
+  e.__right_  = &f;
+  f.__parent_ = &e;
 
-    assert(e.__parent_ == &g);
-    assert(e.__left_ == &c);
-    assert(e.__right_ == &f);
-    assert(e.__is_black_ == false);
+  std::__tree_balance_after_insert(root.__left_, &f);
+
+  assert(std::__tree_invariant(root.__left_));
 
-    assert(c.__parent_ == &e);
-    assert(c.__left_ == &b);
-    assert(c.__right_ == &d);
-    assert(c.__is_black_ == true);
+  assert(root.__parent_ == 0);
+  assert(root.__left_ == &b);
+  assert(root.__right_ == 0);
+  assert(root.__is_black_ == false);
+
+  assert(b.__parent_ == &root);
+  assert(b.__left_ == &a);
+  assert(b.__right_ == &d);
+  assert(b.__is_black_ == true);
+
+  assert(a.__parent_ == &b);
+  assert(a.__left_ == 0);
+  assert(a.__right_ == 0);
+  assert(a.__is_black_ == true);
+
+  assert(d.__parent_ == &b);
+  assert(d.__left_ == &c);
+  assert(d.__right_ == &e);
+  assert(d.__is_black_ == false);
+
+  assert(c.__parent_ == &d);
+  assert(c.__left_ == 0);
+  assert(c.__right_ == 0);
+  assert(c.__is_black_ == true);
+
+  assert(e.__parent_ == &d);
+  assert(e.__left_ == 0);
+  assert(e.__right_ == &f);
+  assert(e.__is_black_ == true);
+
+  assert(f.__parent_ == &e);
+  assert(f.__left_ == 0);
+  assert(f.__right_ == 0);
+  assert(f.__is_black_ == false);
+
+  f.__right_  = &g;
+  g.__parent_ = &f;
+
+  std::__tree_balance_after_insert(root.__left_, &g);
+
+  assert(std::__tree_invariant(root.__left_));
+
+  assert(root.__parent_ == 0);
+  assert(root.__left_ == &b);
+  assert(root.__right_ == 0);
+  assert(root.__is_black_ == false);
+
+  assert(b.__parent_ == &root);
+  assert(b.__left_ == &a);
+  assert(b.__right_ == &d);
+  assert(b.__is_black_ == true);
+
+  assert(a.__parent_ == &b);
+  assert(a.__left_ == 0);
+  assert(a.__right_ == 0);
+  assert(a.__is_black_ == true);
+
+  assert(d.__parent_ == &b);
+  assert(d.__left_ == &c);
+  assert(d.__right_ == &f);
+  assert(d.__is_black_ == false);
+
+  assert(c.__parent_ == &d);
+  assert(c.__left_ == 0);
+  assert(c.__right_ == 0);
+  assert(c.__is_black_ == true);
+
+  assert(f.__parent_ == &d);
+  assert(f.__left_ == &e);
+  assert(f.__right_ == &g);
+  assert(f.__is_black_ == true);
+
+  assert(e.__parent_ == &f);
+  assert(e.__left_ == 0);
+  assert(e.__right_ == 0);
+  assert(e.__is_black_ == false);
+
+  assert(g.__parent_ == &f);
+  assert(g.__left_ == 0);
+  assert(g.__right_ == 0);
+  assert(g.__is_black_ == false);
+
+  g.__right_  = &h;
+  h.__parent_ = &g;
+
+  std::__tree_balance_after_insert(root.__left_, &h);
+
+  assert(std::__tree_invariant(root.__left_));
+
+  assert(root.__parent_ == 0);
+  assert(root.__left_ == &d);
+  assert(root.__right_ == 0);
+  assert(root.__is_black_ == false);
+
+  assert(d.__parent_ == &root);
+  assert(d.__left_ == &b);
+  assert(d.__right_ == &f);
+  assert(d.__is_black_ == true);
+
+  assert(b.__parent_ == &d);
+  assert(b.__left_ == &a);
+  assert(b.__right_ == &c);
+  assert(b.__is_black_ == false);
+
+  assert(a.__parent_ == &b);
+  assert(a.__left_ == 0);
+  assert(a.__right_ == 0);
+  assert(a.__is_black_ == true);
+
+  assert(c.__parent_ == &b);
+  assert(c.__left_ == 0);
+  assert(c.__right_ == 0);
+  assert(c.__is_black_ == true);
+
+  assert(f.__parent_ == &d);
+  assert(f.__left_ == &e);
+  assert(f.__right_ == &g);
+  assert(f.__is_black_ == false);
+
+  assert(e.__parent_ == &f);
+  assert(e.__left_ == 0);
+  assert(e.__right_ == 0);
+  assert(e.__is_black_ == true);
+
+  assert(g.__parent_ == &f);
+  assert(g.__left_ == 0);
+  assert(g.__right_ == &h);
+  assert(g.__is_black_ == true);
+
+  assert(h.__parent_ == &g);
+  assert(h.__left_ == 0);
+  assert(h.__right_ == 0);
+  assert(h.__is_black_ == false);
+}
 
-    assert(b.__parent_ == &c);
-    assert(b.__left_ == 0);
-    assert(b.__right_ == 0);
-    assert(b.__is_black_ == false);
+void test5() {
+  Node root;
+  Node a;
+  Node b;
+  Node c;
+  Node d;
+  Node e;
+  Node f;
+  Node g;
+  Node h;
 
-    assert(d.__parent_ == &c);
-    assert(d.__left_ == 0);
-    assert(d.__right_ == 0);
-    assert(d.__is_black_ == false);
+  root.__left_ = &h;
+  h.__parent_  = &root;
 
-    assert(f.__parent_ == &e);
-    assert(f.__left_ == 0);
-    assert(f.__right_ == 0);
-    assert(f.__is_black_ == true);
+  std::__tree_balance_after_insert(root.__left_, &h);
 
-    assert(h.__parent_ == &g);
-    assert(h.__left_ == 0);
-    assert(h.__right_ == 0);
-    assert(h.__is_black_ == true);
+  assert(std::__tree_invariant(root.__left_));
 
-    b.__left_ = &a;
-    a.__parent_ = &b;
+  assert(root.__parent_ == 0);
+  assert(root.__left_ == &h);
+  assert(root.__right_ == 0);
+  assert(root.__is_black_ == false);
 
-    std::__tree_balance_after_insert(root.__left_, &a);
+  assert(h.__parent_ == &root);
+  assert(h.__left_ == 0);
+  assert(h.__right_ == 0);
+  assert(h.__is_black_ == true);
 
-    assert(std::__tree_invariant(root.__left_));
+  h.__left_   = &g;
+  g.__parent_ = &h;
 
-    assert(root.__parent_ == 0);
-    assert(root.__left_ == &e);
-    assert(root.__right_ == 0);
-    assert(root.__is_black_ == false);
+  std::__tree_balance_after_insert(root.__left_, &g);
 
-    assert(e.__parent_ == &root);
-    assert(e.__left_ == &c);
-    assert(e.__right_ == &g);
-    assert(e.__is_black_ == true);
+  assert(std::__tree_invariant(root.__left_));
 
-    assert(c.__parent_ == &e);
-    assert(c.__left_ == &b);
-    assert(c.__right_ == &d);
-    assert(c.__is_black_ == false);
+  assert(root.__parent_ == 0);
+  assert(root.__left_ == &h);
+  assert(root.__right_ == 0);
+  assert(root.__is_black_ == false);
 
-    assert(b.__parent_ == &c);
-    assert(b.__left_ == &a);
-    assert(b.__right_ == 0);
-    assert(b.__is_black_ == true);
+  assert(h.__parent_ == &root);
+  assert(h.__left_ == &g);
+  assert(h.__right_ == 0);
+  assert(h.__is_black_ == true);
 
-    assert(a.__parent_ == &b);
-    assert(a.__left_ == 0);
-    assert(a.__right_ == 0);
-    assert(a.__is_black_ == false);
+  assert(g.__parent_ == &h);
+  assert(g.__left_ == 0);
+  assert(g.__right_ == 0);
+  assert(g.__is_black_ == false);
 
-    assert(d.__parent_ == &c);
-    assert(d.__left_ == 0);
-    assert(d.__right_ == 0);
-    assert(d.__is_black_ == true);
+  g.__left_   = &f;
+  f.__parent_ = &g;
 
-    assert(g.__parent_ == &e);
-    assert(g.__left_ == &f);
-    assert(g.__right_ == &h);
-    assert(g.__is_black_ == false);
+  std::__tree_balance_after_insert(root.__left_, &f);
 
-    assert(f.__parent_ == &g);
-    assert(f.__left_ == 0);
-    assert(f.__right_ == 0);
-    assert(f.__is_black_ == true);
+  assert(std::__tree_invariant(root.__left_));
+
+  assert(root.__parent_ == 0);
+  assert(root.__left_ == &g);
+  assert(root.__right_ == 0);
+  assert(root.__is_black_ == false);
+
+  assert(g.__parent_ == &root);
+  assert(g.__left_ == &f);
+  assert(g.__right_ == &h);
+  assert(g.__is_black_ == true);
+
+  assert(f.__parent_ == &g);
+  assert(f.__left_ == 0);
+  assert(f.__right_ == 0);
+  assert(f.__is_black_ == false);
+
+  assert(h.__parent_ == &g);
+  assert(h.__left_ == 0);
+  assert(h.__right_ == 0);
+  assert(h.__is_black_ == false);
+
+  f.__left_   = &e;
+  e.__parent_ = &f;
+
+  std::__tree_balance_after_insert(root.__left_, &e);
+
+  assert(std::__tree_invariant(root.__left_));
+
+  assert(root.__parent_ == 0);
+  assert(root.__left_ == &g);
+  assert(root.__right_ == 0);
+  assert(root.__is_black_ == false);
+
+  assert(g.__parent_ == &root);
+  assert(g.__left_ == &f);
+  assert(g.__right_ == &h);
+  assert(g.__is_black_ == true);
+
+  assert(f.__parent_ == &g);
+  assert(f.__left_ == &e);
+  assert(f.__right_ == 0);
+  assert(f.__is_black_ == true);
+
+  assert(e.__parent_ == &f);
+  assert(e.__left_ == 0);
+  assert(e.__right_ == 0);
+  assert(e.__is_black_ == false);
+
+  assert(h.__parent_ == &g);
+  assert(h.__left_ == 0);
+  assert(h.__right_ == 0);
+  assert(h.__is_black_ == true);
+
+  e.__left_   = &d;
+  d.__parent_ = &e;
+
+  std::__tree_balance_after_insert(root.__left_, &d);
+
+  assert(std::__tree_invariant(root.__left_));
+
+  assert(root.__parent_ == 0);
+  assert(root.__left_ == &g);
+  assert(root.__right_ == 0);
+  assert(root.__is_black_ == false);
+
+  assert(g.__parent_ == &root);
+  assert(g.__left_ == &e);
+  assert(g.__right_ == &h);
+  assert(g.__is_black_ == true);
+
+  assert(e.__parent_ == &g);
+  assert(e.__left_ == &d);
+  assert(e.__right_ == &f);
+  assert(e.__is_black_ == true);
+
+  assert(d.__parent_ == &e);
+  assert(d.__left_ == 0);
+  assert(d.__right_ == 0);
+  assert(d.__is_black_ == false);
+
+  assert(f.__parent_ == &e);
+  assert(f.__left_ == 0);
+  assert(f.__right_ == 0);
+  assert(f.__is_black_ == false);
+
+  assert(h.__parent_ == &g);
+  assert(h.__left_ == 0);
+  assert(h.__right_ == 0);
+  assert(h.__is_black_ == true);
+
+  d.__left_   = &c;
+  c.__parent_ = &d;
+
+  std::__tree_balance_after_insert(root.__left_, &c);
+
+  assert(std::__tree_invariant(root.__left_));
 
-    assert(h.__parent_ == &g);
-    assert(h.__left_ == 0);
-    assert(h.__right_ == 0);
-    assert(h.__is_black_ == true);
+  assert(root.__parent_ == 0);
+  assert(root.__left_ == &g);
+  assert(root.__right_ == 0);
+  assert(root.__is_black_ == false);
+
+  assert(g.__parent_ == &root);
+  assert(g.__left_ == &e);
+  assert(g.__right_ == &h);
+  assert(g.__is_black_ == true);
+
+  assert(e.__parent_ == &g);
+  assert(e.__left_ == &d);
+  assert(e.__right_ == &f);
+  assert(e.__is_black_ == false);
+
+  assert(d.__parent_ == &e);
+  assert(d.__left_ == &c);
+  assert(d.__right_ == 0);
+  assert(d.__is_black_ == true);
+
+  assert(c.__parent_ == &d);
+  assert(c.__left_ == 0);
+  assert(c.__right_ == 0);
+  assert(c.__is_black_ == false);
+
+  assert(f.__parent_ == &e);
+  assert(f.__left_ == 0);
+  assert(f.__right_ == 0);
+  assert(f.__is_black_ == true);
+
+  assert(h.__parent_ == &g);
+  assert(h.__left_ == 0);
+  assert(h.__right_ == 0);
+  assert(h.__is_black_ == true);
+
+  c.__left_   = &b;
+  b.__parent_ = &c;
+
+  std::__tree_balance_after_insert(root.__left_, &b);
+
+  assert(std::__tree_invariant(root.__left_));
+
+  assert(root.__parent_ == 0);
+  assert(root.__left_ == &g);
+  assert(root.__right_ == 0);
+  assert(root.__is_black_ == false);
+
+  assert(g.__parent_ == &root);
+  assert(g.__left_ == &e);
+  assert(g.__right_ == &h);
+  assert(g.__is_black_ == true);
+
+  assert(e.__parent_ == &g);
+  assert(e.__left_ == &c);
+  assert(e.__right_ == &f);
+  assert(e.__is_black_ == false);
+
+  assert(c.__parent_ == &e);
+  assert(c.__left_ == &b);
+  assert(c.__right_ == &d);
+  assert(c.__is_black_ == true);
+
+  assert(b.__parent_ == &c);
+  assert(b.__left_ == 0);
+  assert(b.__right_ == 0);
+  assert(b.__is_black_ == false);
+
+  assert(d.__parent_ == &c);
+  assert(d.__left_ == 0);
+  assert(d.__right_ == 0);
+  assert(d.__is_black_ == false);
+
+  assert(f.__parent_ == &e);
+  assert(f.__left_ == 0);
+  assert(f.__right_ == 0);
+  assert(f.__is_black_ == true);
+
+  assert(h.__parent_ == &g);
+  assert(h.__left_ == 0);
+  assert(h.__right_ == 0);
+  assert(h.__is_black_ == true);
+
+  b.__left_   = &a;
+  a.__parent_ = &b;
+
+  std::__tree_balance_after_insert(root.__left_, &a);
+
+  assert(std::__tree_invariant(root.__left_));
+
+  assert(root.__parent_ == 0);
+  assert(root.__left_ == &e);
+  assert(root.__right_ == 0);
+  assert(root.__is_black_ == false);
+
+  assert(e.__parent_ == &root);
+  assert(e.__left_ == &c);
+  assert(e.__right_ == &g);
+  assert(e.__is_black_ == true);
+
+  assert(c.__parent_ == &e);
+  assert(c.__left_ == &b);
+  assert(c.__right_ == &d);
+  assert(c.__is_black_ == false);
+
+  assert(b.__parent_ == &c);
+  assert(b.__left_ == &a);
+  assert(b.__right_ == 0);
+  assert(b.__is_black_ == true);
+
+  assert(a.__parent_ == &b);
+  assert(a.__left_ == 0);
+  assert(a.__right_ == 0);
+  assert(a.__is_black_ == false);
+
+  assert(d.__parent_ == &c);
+  assert(d.__left_ == 0);
+  assert(d.__right_ == 0);
+  assert(d.__is_black_ == true);
+
+  assert(g.__parent_ == &e);
+  assert(g.__left_ == &f);
+  assert(g.__right_ == &h);
+  assert(g.__is_black_ == false);
+
+  assert(f.__parent_ == &g);
+  assert(f.__left_ == 0);
+  assert(f.__right_ == 0);
+  assert(f.__is_black_ == true);
+
+  assert(h.__parent_ == &g);
+  assert(h.__left_ == 0);
+  assert(h.__right_ == 0);
+  assert(h.__is_black_ == true);
 }
 
-int main(int, char**)
-{
-    test1();
-    test2();
-    test3();
-    test4();
-    test5();
+int main(int, char**) {
+  test1();
+  test2();
+  test3();
+  test4();
+  test5();
 
   return 0;
 }
diff --git a/libcxx/test/libcxx/containers/associative/tree_key_value_traits.pass.cpp b/libcxx/test/libcxx/containers/associative/tree_key_value_traits.pass.cpp
index bebdcaaedf103..e3a5a6f634138 100644
--- a/libcxx/test/libcxx/containers/associative/tree_key_value_traits.pass.cpp
+++ b/libcxx/test/libcxx/containers/associative/tree_key_value_traits.pass.cpp
@@ -47,10 +47,8 @@ void testKeyValueTrait() {
     static_assert((std::is_same<Traits::key_type, int>::value), "");
     static_assert((std::is_same<Traits::mapped_type, int>::value), "");
     static_assert((std::is_same<Traits::__node_value_type, Tp>::value), "");
-    static_assert((std::is_same<Traits::__container_value_type,
-                               std::pair<const int, int> >::value), "");
-    static_assert((std::is_same<Traits::__map_value_type,
-                               std::pair<const int, int> >::value), "");
+    static_assert((std::is_same<Traits::__container_value_type, std::pair<const int, int> >::value), "");
+    static_assert((std::is_same<Traits::__map_value_type, std::pair<const int, int> >::value), "");
     static_assert(Traits::__is_map == true, "");
   }
 }
diff --git a/libcxx/test/libcxx/containers/associative/tree_left_rotate.pass.cpp b/libcxx/test/libcxx/containers/associative/tree_left_rotate.pass.cpp
index f8073d6233c9b..d97a1c89f1f70 100644
--- a/libcxx/test/libcxx/containers/associative/tree_left_rotate.pass.cpp
+++ b/libcxx/test/libcxx/containers/associative/tree_left_rotate.pass.cpp
@@ -20,87 +20,81 @@
 
 #include "test_macros.h"
 
-struct Node
-{
-    Node* __left_;
-    Node* __right_;
-    Node* __parent_;
+struct Node {
+  Node* __left_;
+  Node* __right_;
+  Node* __parent_;
 
-    Node* __parent_unsafe() const { return __parent_; }
-    void __set_parent(Node* x) { __parent_ = x;}
+  Node* __parent_unsafe() const { return __parent_; }
+  void __set_parent(Node* x) { __parent_ = x; }
 
-    Node() : __left_(), __right_(), __parent_() {}
+  Node() : __left_(), __right_(), __parent_() {}
 };
 
-void
-test1()
-{
-    Node root;
-    Node x;
-    Node y;
-    root.__left_ = &x;
-    x.__left_ = 0;
-    x.__right_ = &y;
-    x.__parent_ = &root;
-    y.__left_ = 0;
-    y.__right_ = 0;
-    y.__parent_ = &x;
-    std::__tree_left_rotate(&x);
-    assert(root.__parent_ == 0);
-    assert(root.__left_ == &y);
-    assert(root.__right_ == 0);
-    assert(y.__parent_ == &root);
-    assert(y.__left_ == &x);
-    assert(y.__right_ == 0);
-    assert(x.__parent_ == &y);
-    assert(x.__left_ == 0);
-    assert(x.__right_ == 0);
+void test1() {
+  Node root;
+  Node x;
+  Node y;
+  root.__left_ = &x;
+  x.__left_    = 0;
+  x.__right_   = &y;
+  x.__parent_  = &root;
+  y.__left_    = 0;
+  y.__right_   = 0;
+  y.__parent_  = &x;
+  std::__tree_left_rotate(&x);
+  assert(root.__parent_ == 0);
+  assert(root.__left_ == &y);
+  assert(root.__right_ == 0);
+  assert(y.__parent_ == &root);
+  assert(y.__left_ == &x);
+  assert(y.__right_ == 0);
+  assert(x.__parent_ == &y);
+  assert(x.__left_ == 0);
+  assert(x.__right_ == 0);
 }
 
-void
-test2()
-{
-    Node root;
-    Node x;
-    Node y;
-    Node a;
-    Node b;
-    Node c;
-    root.__left_ = &x;
-    x.__left_ = &a;
-    x.__right_ = &y;
-    x.__parent_ = &root;
-    y.__left_ = &b;
-    y.__right_ = &c;
-    y.__parent_ = &x;
-    a.__parent_ = &x;
-    b.__parent_ = &y;
-    c.__parent_ = &y;
-    std::__tree_left_rotate(&x);
-    assert(root.__parent_ == 0);
-    assert(root.__left_ == &y);
-    assert(root.__right_ == 0);
-    assert(y.__parent_ == &root);
-    assert(y.__left_ == &x);
-    assert(y.__right_ == &c);
-    assert(x.__parent_ == &y);
-    assert(x.__left_ == &a);
-    assert(x.__right_ == &b);
-    assert(a.__parent_ == &x);
-    assert(a.__left_ == 0);
-    assert(a.__right_ == 0);
-    assert(b.__parent_ == &x);
-    assert(b.__left_ == 0);
-    assert(b.__right_ == 0);
-    assert(c.__parent_ == &y);
-    assert(c.__left_ == 0);
-    assert(c.__right_ == 0);
+void test2() {
+  Node root;
+  Node x;
+  Node y;
+  Node a;
+  Node b;
+  Node c;
+  root.__left_ = &x;
+  x.__left_    = &a;
+  x.__right_   = &y;
+  x.__parent_  = &root;
+  y.__left_    = &b;
+  y.__right_   = &c;
+  y.__parent_  = &x;
+  a.__parent_  = &x;
+  b.__parent_  = &y;
+  c.__parent_  = &y;
+  std::__tree_left_rotate(&x);
+  assert(root.__parent_ == 0);
+  assert(root.__left_ == &y);
+  assert(root.__right_ == 0);
+  assert(y.__parent_ == &root);
+  assert(y.__left_ == &x);
+  assert(y.__right_ == &c);
+  assert(x.__parent_ == &y);
+  assert(x.__left_ == &a);
+  assert(x.__right_ == &b);
+  assert(a.__parent_ == &x);
+  assert(a.__left_ == 0);
+  assert(a.__right_ == 0);
+  assert(b.__parent_ == &x);
+  assert(b.__left_ == 0);
+  assert(b.__right_ == 0);
+  assert(c.__parent_ == &y);
+  assert(c.__left_ == 0);
+  assert(c.__right_ == 0);
 }
 
-int main(int, char**)
-{
-    test1();
-    test2();
+int main(int, char**) {
+  test1();
+  test2();
 
   return 0;
 }
diff --git a/libcxx/test/libcxx/containers/associative/tree_remove.pass.cpp b/libcxx/test/libcxx/containers/associative/tree_remove.pass.cpp
index 0b2d86722912e..e543c3360a685 100644
--- a/libcxx/test/libcxx/containers/associative/tree_remove.pass.cpp
+++ b/libcxx/test/libcxx/containers/associative/tree_remove.pass.cpp
@@ -20,1317 +20,911 @@
 
 #include "test_macros.h"
 
-struct Node
-{
-    Node* __left_;
-    Node* __right_;
-    Node* __parent_;
-    bool __is_black_;
+struct Node {
+  Node* __left_;
+  Node* __right_;
+  Node* __parent_;
+  bool __is_black_;
 
-    Node* __parent_unsafe() const { return __parent_; }
-    void __set_parent(Node* x) { __parent_ = x;}
+  Node* __parent_unsafe() const { return __parent_; }
+  void __set_parent(Node* x) { __parent_ = x; }
 
-    Node() : __left_(), __right_(), __parent_(), __is_black_() {}
+  Node() : __left_(), __right_(), __parent_(), __is_black_() {}
 };
 
-void
-test1()
-{
-    {
-        // Left
-        // Case 1 -> Case 2 -> x is red turned to black
-        Node root;
-        Node b;
-        Node c;
-        Node d;
-        Node e;
-        Node y;
-
-        root.__left_ = &b;
-
-        b.__parent_ = &root;
-        b.__left_ = &y;
-        b.__right_ = &d;
-        b.__is_black_ = true;
-
-        y.__parent_ = &b;
-        y.__left_ = 0;
-        y.__right_ = 0;
-        y.__is_black_ = true;
-
-        d.__parent_ = &b;
-        d.__left_ = &c;
-        d.__right_ = &e;
-        d.__is_black_ = false;
-
-        c.__parent_ = &d;
-        c.__left_ = 0;
-        c.__right_ = 0;
-        c.__is_black_ = true;
-
-        e.__parent_ = &d;
-        e.__left_ = 0;
-        e.__right_ = 0;
-        e.__is_black_ = true;
-
-        std::__tree_remove(root.__left_, &y);
-        assert(std::__tree_invariant(root.__left_));
-
-        assert(root.__parent_ == 0);
-        assert(root.__left_ == &d);
-        assert(root.__right_ == 0);
-        assert(root.__is_black_ == false);
-
-        assert(d.__parent_ == &root);
-        assert(d.__left_ == &b);
-        assert(d.__right_ == &e);
-        assert(d.__is_black_ == true);
-
-        assert(b.__parent_ == &d);
-        assert(b.__left_ == 0);
-        assert(b.__right_ == &c);
-        assert(b.__is_black_ == true);
-
-        assert(c.__parent_ == &b);
-        assert(c.__left_ == 0);
-        assert(c.__right_ == 0);
-        assert(c.__is_black_ == false);
-
-        assert(e.__parent_ == &d);
-        assert(e.__left_ == 0);
-        assert(e.__right_ == 0);
-        assert(e.__is_black_ == true);
-    }
-    {
-        // Right
-        // Case 1 -> Case 2 -> x is red turned to black
-        Node root;
-        Node b;
-        Node c;
-        Node d;
-        Node e;
-        Node y;
-
-        root.__left_ = &b;
-
-        b.__parent_ = &root;
-        b.__right_ = &y;
-        b.__left_ = &d;
-        b.__is_black_ = true;
-
-        y.__parent_ = &b;
-        y.__right_ = 0;
-        y.__left_ = 0;
-        y.__is_black_ = true;
-
-        d.__parent_ = &b;
-        d.__right_ = &c;
-        d.__left_ = &e;
-        d.__is_black_ = false;
-
-        c.__parent_ = &d;
-        c.__right_ = 0;
-        c.__left_ = 0;
-        c.__is_black_ = true;
-
-        e.__parent_ = &d;
-        e.__right_ = 0;
-        e.__left_ = 0;
-        e.__is_black_ = true;
-
-        std::__tree_remove(root.__left_, &y);
-        assert(std::__tree_invariant(root.__left_));
-
-        assert(root.__parent_ == 0);
-        assert(root.__left_ == &d);
-        assert(root.__right_ == 0);
-        assert(root.__is_black_ == false);
-
-        assert(d.__parent_ == &root);
-        assert(d.__right_ == &b);
-        assert(d.__left_ == &e);
-        assert(d.__is_black_ == true);
-
-        assert(b.__parent_ == &d);
-        assert(b.__right_ == 0);
-        assert(b.__left_ == &c);
-        assert(b.__is_black_ == true);
-
-        assert(c.__parent_ == &b);
-        assert(c.__right_ == 0);
-        assert(c.__left_ == 0);
-        assert(c.__is_black_ == false);
-
-        assert(e.__parent_ == &d);
-        assert(e.__right_ == 0);
-        assert(e.__left_ == 0);
-        assert(e.__is_black_ == true);
-    }
-    {
-        // Left
-        // Case 1 -> Case 3 -> Case 4
-        Node root;
-        Node b;
-        Node c;
-        Node d;
-        Node e;
-        Node f;
-        Node y;
-
-        root.__left_ = &b;
-
-        b.__parent_ = &root;
-        b.__left_ = &y;
-        b.__right_ = &d;
-        b.__is_black_ = true;
-
-        y.__parent_ = &b;
-        y.__left_ = 0;
-        y.__right_ = 0;
-        y.__is_black_ = true;
-
-        d.__parent_ = &b;
-        d.__left_ = &c;
-        d.__right_ = &e;
-        d.__is_black_ = false;
-
-        c.__parent_ = &d;
-        c.__left_ = &f;
-        c.__right_ = 0;
-        c.__is_black_ = true;
-
-        e.__parent_ = &d;
-        e.__left_ = 0;
-        e.__right_ = 0;
-        e.__is_black_ = true;
-
-        f.__parent_ = &c;
-        f.__left_ = 0;
-        f.__right_ = 0;
-        f.__is_black_ = false;
-
-        std::__tree_remove(root.__left_, &y);
-        assert(std::__tree_invariant(root.__left_));
-
-        assert(root.__parent_ == 0);
-        assert(root.__left_ == &d);
-        assert(root.__right_ == 0);
-        assert(root.__is_black_ == false);
-
-        assert(d.__parent_ == &root);
-        assert(d.__left_ == &f);
-        assert(d.__right_ == &e);
-        assert(d.__is_black_ == true);
-
-        assert(f.__parent_ == &d);
-        assert(f.__left_ == &b);
-        assert(f.__right_ == &c);
-        assert(f.__is_black_ == false);
-
-        assert(b.__parent_ == &f);
-        assert(b.__left_ == 0);
-        assert(b.__right_ == 0);
-        assert(b.__is_black_ == true);
-
-        assert(c.__parent_ == &f);
-        assert(c.__left_ == 0);
-        assert(c.__right_ == 0);
-        assert(c.__is_black_ == true);
-
-        assert(e.__parent_ == &d);
-        assert(e.__left_ == 0);
-        assert(e.__right_ == 0);
-        assert(e.__is_black_ == true);
-    }
-    {
-        // Right
-        // Case 1 -> Case 3 -> Case 4
-        Node root;
-        Node b;
-        Node c;
-        Node d;
-        Node e;
-        Node f;
-        Node y;
-
-        root.__left_ = &b;
-
-        b.__parent_ = &root;
-        b.__right_ = &y;
-        b.__left_ = &d;
-        b.__is_black_ = true;
-
-        y.__parent_ = &b;
-        y.__right_ = 0;
-        y.__left_ = 0;
-        y.__is_black_ = true;
-
-        d.__parent_ = &b;
-        d.__right_ = &c;
-        d.__left_ = &e;
-        d.__is_black_ = false;
-
-        c.__parent_ = &d;
-        c.__right_ = &f;
-        c.__left_ = 0;
-        c.__is_black_ = true;
-
-        e.__parent_ = &d;
-        e.__right_ = 0;
-        e.__left_ = 0;
-        e.__is_black_ = true;
-
-        f.__parent_ = &c;
-        f.__right_ = 0;
-        f.__left_ = 0;
-        f.__is_black_ = false;
-
-        std::__tree_remove(root.__left_, &y);
-        assert(std::__tree_invariant(root.__left_));
-
-        assert(root.__parent_ == 0);
-        assert(root.__left_ == &d);
-        assert(root.__right_ == 0);
-        assert(root.__is_black_ == false);
-
-        assert(d.__parent_ == &root);
-        assert(d.__right_ == &f);
-        assert(d.__left_ == &e);
-        assert(d.__is_black_ == true);
-
-        assert(f.__parent_ == &d);
-        assert(f.__right_ == &b);
-        assert(f.__left_ == &c);
-        assert(f.__is_black_ == false);
-
-        assert(b.__parent_ == &f);
-        assert(b.__right_ == 0);
-        assert(b.__left_ == 0);
-        assert(b.__is_black_ == true);
-
-        assert(c.__parent_ == &f);
-        assert(c.__right_ == 0);
-        assert(c.__left_ == 0);
-        assert(c.__is_black_ == true);
-
-        assert(e.__parent_ == &d);
-        assert(e.__right_ == 0);
-        assert(e.__left_ == 0);
-        assert(e.__is_black_ == true);
-    }
-}
+void test1() {
+  {
+    // Left
+    // Case 1 -> Case 2 -> x is red turned to black
+    Node root;
+    Node b;
+    Node c;
+    Node d;
+    Node e;
+    Node y;
 
-void
-test2()
-{
-    {
-        Node root;
-        Node a;
-        Node b;
-        Node c;
-
-        root.__left_ = &b;
-
-        b.__parent_ = &root;
-        b.__left_ = &a;
-        b.__right_ = &c;
-        b.__is_black_ = true;
+    root.__left_ = &b;
 
-        a.__parent_ = &b;
-        a.__left_ = 0;
-        a.__right_ = 0;
-        a.__is_black_ = true;
+    b.__parent_   = &root;
+    b.__left_     = &y;
+    b.__right_    = &d;
+    b.__is_black_ = true;
 
-        c.__parent_ = &b;
-        c.__left_ = 0;
-        c.__right_ = 0;
-        c.__is_black_ = true;
+    y.__parent_   = &b;
+    y.__left_     = 0;
+    y.__right_    = 0;
+    y.__is_black_ = true;
 
-        std::__tree_remove(root.__left_, &a);
+    d.__parent_   = &b;
+    d.__left_     = &c;
+    d.__right_    = &e;
+    d.__is_black_ = false;
 
-        assert(std::__tree_invariant(root.__left_));
+    c.__parent_   = &d;
+    c.__left_     = 0;
+    c.__right_    = 0;
+    c.__is_black_ = true;
 
-        assert(root.__parent_ == 0);
-        assert(root.__left_ == &b);
-        assert(root.__right_ == 0);
-        assert(root.__is_black_ == false);
-
-        assert(b.__parent_ == &root);
-        assert(b.__left_ == 0);
-        assert(b.__right_ == &c);
-        assert(b.__is_black_ == true);
-
-        assert(c.__parent_ == &b);
-        assert(c.__left_ == 0);
-        assert(c.__right_ == 0);
-        assert(c.__is_black_ == false);
-
-        std::__tree_remove(root.__left_, &b);
-
-        assert(std::__tree_invariant(root.__left_));
-
-        assert(root.__parent_ == 0);
-        assert(root.__left_ == &c);
-        assert(root.__right_ == 0);
-        assert(root.__is_black_ == false);
-
-        assert(c.__parent_ == &root);
-        assert(c.__left_ == 0);
-        assert(c.__right_ == 0);
-        assert(c.__is_black_ == true);
-
-        std::__tree_remove(root.__left_, &c);
+    e.__parent_   = &d;
+    e.__left_     = 0;
+    e.__right_    = 0;
+    e.__is_black_ = true;
 
-        assert(std::__tree_invariant(root.__left_));
-
-        assert(root.__parent_ == 0);
-        assert(root.__left_ == 0);
-        assert(root.__right_ == 0);
-        assert(root.__is_black_ == false);
-    }
-    {
-        Node root;
-        Node a;
-        Node b;
-        Node c;
-
-        root.__left_ = &b;
-
-        b.__parent_ = &root;
-        b.__left_ = &a;
-        b.__right_ = &c;
-        b.__is_black_ = true;
+    std::__tree_remove(root.__left_, &y);
+    assert(std::__tree_invariant(root.__left_));
 
-        a.__parent_ = &b;
-        a.__left_ = 0;
-        a.__right_ = 0;
-        a.__is_black_ = false;
+    assert(root.__parent_ == 0);
+    assert(root.__left_ == &d);
+    assert(root.__right_ == 0);
+    assert(root.__is_black_ == false);
 
-        c.__parent_ = &b;
-        c.__left_ = 0;
-        c.__right_ = 0;
-        c.__is_black_ = false;
+    assert(d.__parent_ == &root);
+    assert(d.__left_ == &b);
+    assert(d.__right_ == &e);
+    assert(d.__is_black_ == true);
 
-        std::__tree_remove(root.__left_, &a);
+    assert(b.__parent_ == &d);
+    assert(b.__left_ == 0);
+    assert(b.__right_ == &c);
+    assert(b.__is_black_ == true);
 
-        assert(std::__tree_invariant(root.__left_));
+    assert(c.__parent_ == &b);
+    assert(c.__left_ == 0);
+    assert(c.__right_ == 0);
+    assert(c.__is_black_ == false);
 
-        assert(root.__parent_ == 0);
-        assert(root.__left_ == &b);
-        assert(root.__right_ == 0);
-        assert(root.__is_black_ == false);
-
-        assert(b.__parent_ == &root);
-        assert(b.__left_ == 0);
-        assert(b.__right_ == &c);
-        assert(b.__is_black_ == true);
-
-        assert(c.__parent_ == &b);
-        assert(c.__left_ == 0);
-        assert(c.__right_ == 0);
-        assert(c.__is_black_ == false);
-
-        std::__tree_remove(root.__left_, &b);
-
-        assert(std::__tree_invariant(root.__left_));
-
-        assert(root.__parent_ == 0);
-        assert(root.__left_ == &c);
-        assert(root.__right_ == 0);
-        assert(root.__is_black_ == false);
-
-        assert(c.__parent_ == &root);
-        assert(c.__left_ == 0);
-        assert(c.__right_ == 0);
-        assert(c.__is_black_ == true);
+    assert(e.__parent_ == &d);
+    assert(e.__left_ == 0);
+    assert(e.__right_ == 0);
+    assert(e.__is_black_ == true);
+  }
+  {
+    // Right
+    // Case 1 -> Case 2 -> x is red turned to black
+    Node root;
+    Node b;
+    Node c;
+    Node d;
+    Node e;
+    Node y;
 
-        std::__tree_remove(root.__left_, &c);
+    root.__left_ = &b;
 
-        assert(std::__tree_invariant(root.__left_));
+    b.__parent_   = &root;
+    b.__right_    = &y;
+    b.__left_     = &d;
+    b.__is_black_ = true;
 
-        assert(root.__parent_ == 0);
-        assert(root.__left_ == 0);
-        assert(root.__right_ == 0);
-        assert(root.__is_black_ == false);
-    }
-    {
-        Node root;
-        Node a;
-        Node b;
-        Node c;
-
-        root.__left_ = &b;
-
-        b.__parent_ = &root;
-        b.__left_ = &a;
-        b.__right_ = &c;
-        b.__is_black_ = true;
+    y.__parent_   = &b;
+    y.__right_    = 0;
+    y.__left_     = 0;
+    y.__is_black_ = true;
 
-        a.__parent_ = &b;
-        a.__left_ = 0;
-        a.__right_ = 0;
-        a.__is_black_ = true;
-
-        c.__parent_ = &b;
-        c.__left_ = 0;
-        c.__right_ = 0;
-        c.__is_black_ = true;
+    d.__parent_   = &b;
+    d.__right_    = &c;
+    d.__left_     = &e;
+    d.__is_black_ = false;
 
-        std::__tree_remove(root.__left_, &a);
+    c.__parent_   = &d;
+    c.__right_    = 0;
+    c.__left_     = 0;
+    c.__is_black_ = true;
 
-        assert(std::__tree_invariant(root.__left_));
+    e.__parent_   = &d;
+    e.__right_    = 0;
+    e.__left_     = 0;
+    e.__is_black_ = true;
 
-        assert(root.__parent_ == 0);
-        assert(root.__left_ == &b);
-        assert(root.__right_ == 0);
-        assert(root.__is_black_ == false);
-
-        assert(b.__parent_ == &root);
-        assert(b.__left_ == 0);
-        assert(b.__right_ == &c);
-        assert(b.__is_black_ == true);
-
-        assert(c.__parent_ == &b);
-        assert(c.__left_ == 0);
-        assert(c.__right_ == 0);
-        assert(c.__is_black_ == false);
-
-        std::__tree_remove(root.__left_, &c);
-
-        assert(std::__tree_invariant(root.__left_));
-
-        assert(root.__parent_ == 0);
-        assert(root.__left_ == &b);
-        assert(root.__right_ == 0);
-        assert(root.__is_black_ == false);
-
-        assert(b.__parent_ == &root);
-        assert(b.__left_ == 0);
-        assert(b.__right_ == 0);
-        assert(b.__is_black_ == true);
+    std::__tree_remove(root.__left_, &y);
+    assert(std::__tree_invariant(root.__left_));
 
-        std::__tree_remove(root.__left_, &b);
+    assert(root.__parent_ == 0);
+    assert(root.__left_ == &d);
+    assert(root.__right_ == 0);
+    assert(root.__is_black_ == false);
 
-        assert(std::__tree_invariant(root.__left_));
+    assert(d.__parent_ == &root);
+    assert(d.__right_ == &b);
+    assert(d.__left_ == &e);
+    assert(d.__is_black_ == true);
 
-        assert(root.__parent_ == 0);
-        assert(root.__left_ == 0);
-        assert(root.__right_ == 0);
-        assert(root.__is_black_ == false);
-    }
-    {
-        Node root;
-        Node a;
-        Node b;
-        Node c;
-
-        root.__left_ = &b;
-
-        b.__parent_ = &root;
-        b.__left_ = &a;
-        b.__right_ = &c;
-        b.__is_black_ = true;
+    assert(b.__parent_ == &d);
+    assert(b.__right_ == 0);
+    assert(b.__left_ == &c);
+    assert(b.__is_black_ == true);
 
-        a.__parent_ = &b;
-        a.__left_ = 0;
-        a.__right_ = 0;
-        a.__is_black_ = false;
-
-        c.__parent_ = &b;
-        c.__left_ = 0;
-        c.__right_ = 0;
-        c.__is_black_ = false;
+    assert(c.__parent_ == &b);
+    assert(c.__right_ == 0);
+    assert(c.__left_ == 0);
+    assert(c.__is_black_ == false);
 
-        std::__tree_remove(root.__left_, &a);
+    assert(e.__parent_ == &d);
+    assert(e.__right_ == 0);
+    assert(e.__left_ == 0);
+    assert(e.__is_black_ == true);
+  }
+  {
+    // Left
+    // Case 1 -> Case 3 -> Case 4
+    Node root;
+    Node b;
+    Node c;
+    Node d;
+    Node e;
+    Node f;
+    Node y;
 
-        assert(std::__tree_invariant(root.__left_));
+    root.__left_ = &b;
 
-        assert(root.__parent_ == 0);
-        assert(root.__left_ == &b);
-        assert(root.__right_ == 0);
-        assert(root.__is_black_ == false);
-
-        assert(b.__parent_ == &root);
-        assert(b.__left_ == 0);
-        assert(b.__right_ == &c);
-        assert(b.__is_black_ == true);
-
-        assert(c.__parent_ == &b);
-        assert(c.__left_ == 0);
-        assert(c.__right_ == 0);
-        assert(c.__is_black_ == false);
-
-        std::__tree_remove(root.__left_, &c);
-
-        assert(std::__tree_invariant(root.__left_));
-
-        assert(root.__parent_ == 0);
-        assert(root.__left_ == &b);
-        assert(root.__right_ == 0);
-        assert(root.__is_black_ == false);
-
-        assert(b.__parent_ == &root);
-        assert(b.__left_ == 0);
-        assert(b.__right_ == 0);
-        assert(b.__is_black_ == true);
+    b.__parent_   = &root;
+    b.__left_     = &y;
+    b.__right_    = &d;
+    b.__is_black_ = true;
 
-        std::__tree_remove(root.__left_, &b);
+    y.__parent_   = &b;
+    y.__left_     = 0;
+    y.__right_    = 0;
+    y.__is_black_ = true;
 
-        assert(std::__tree_invariant(root.__left_));
+    d.__parent_   = &b;
+    d.__left_     = &c;
+    d.__right_    = &e;
+    d.__is_black_ = false;
 
-        assert(root.__parent_ == 0);
-        assert(root.__left_ == 0);
-        assert(root.__right_ == 0);
-        assert(root.__is_black_ == false);
-    }
-    {
-        Node root;
-        Node a;
-        Node b;
-        Node c;
-
-        root.__left_ = &b;
-
-        b.__parent_ = &root;
-        b.__left_ = &a;
-        b.__right_ = &c;
-        b.__is_black_ = true;
+    c.__parent_   = &d;
+    c.__left_     = &f;
+    c.__right_    = 0;
+    c.__is_black_ = true;
 
-        a.__parent_ = &b;
-        a.__left_ = 0;
-        a.__right_ = 0;
-        a.__is_black_ = true;
-
-        c.__parent_ = &b;
-        c.__left_ = 0;
-        c.__right_ = 0;
-        c.__is_black_ = true;
+    e.__parent_   = &d;
+    e.__left_     = 0;
+    e.__right_    = 0;
+    e.__is_black_ = true;
 
-        std::__tree_remove(root.__left_, &b);
+    f.__parent_   = &c;
+    f.__left_     = 0;
+    f.__right_    = 0;
+    f.__is_black_ = false;
 
-        assert(std::__tree_invariant(root.__left_));
+    std::__tree_remove(root.__left_, &y);
+    assert(std::__tree_invariant(root.__left_));
 
-        assert(root.__parent_ == 0);
-        assert(root.__left_ == &c);
-        assert(root.__right_ == 0);
-        assert(root.__is_black_ == false);
-
-        assert(a.__parent_ == &c);
-        assert(a.__left_ == 0);
-        assert(a.__right_ == 0);
-        assert(a.__is_black_ == false);
-
-        assert(c.__parent_ == &root);
-        assert(c.__left_ == &a);
-        assert(c.__right_ == 0);
-        assert(c.__is_black_ == true);
-
-        std::__tree_remove(root.__left_, &a);
-
-        assert(std::__tree_invariant(root.__left_));
-
-        assert(root.__parent_ == 0);
-        assert(root.__left_ == &c);
-        assert(root.__right_ == 0);
-        assert(root.__is_black_ == false);
-
-        assert(c.__parent_ == &root);
-        assert(c.__left_ == 0);
-        assert(c.__right_ == 0);
-        assert(c.__is_black_ == true);
+    assert(root.__parent_ == 0);
+    assert(root.__left_ == &d);
+    assert(root.__right_ == 0);
+    assert(root.__is_black_ == false);
 
-        std::__tree_remove(root.__left_, &c);
+    assert(d.__parent_ == &root);
+    assert(d.__left_ == &f);
+    assert(d.__right_ == &e);
+    assert(d.__is_black_ == true);
 
-        assert(std::__tree_invariant(root.__left_));
+    assert(f.__parent_ == &d);
+    assert(f.__left_ == &b);
+    assert(f.__right_ == &c);
+    assert(f.__is_black_ == false);
 
-        assert(root.__parent_ == 0);
-        assert(root.__left_ == 0);
-        assert(root.__right_ == 0);
-        assert(root.__is_black_ == false);
-    }
-    {
-        Node root;
-        Node a;
-        Node b;
-        Node c;
-
-        root.__left_ = &b;
-
-        b.__parent_ = &root;
-        b.__left_ = &a;
-        b.__right_ = &c;
-        b.__is_black_ = true;
+    assert(b.__parent_ == &f);
+    assert(b.__left_ == 0);
+    assert(b.__right_ == 0);
+    assert(b.__is_black_ == true);
 
-        a.__parent_ = &b;
-        a.__left_ = 0;
-        a.__right_ = 0;
-        a.__is_black_ = false;
-
-        c.__parent_ = &b;
-        c.__left_ = 0;
-        c.__right_ = 0;
-        c.__is_black_ = false;
+    assert(c.__parent_ == &f);
+    assert(c.__left_ == 0);
+    assert(c.__right_ == 0);
+    assert(c.__is_black_ == true);
+
+    assert(e.__parent_ == &d);
+    assert(e.__left_ == 0);
+    assert(e.__right_ == 0);
+    assert(e.__is_black_ == true);
+  }
+  {
+    // Right
+    // Case 1 -> Case 3 -> Case 4
+    Node root;
+    Node b;
+    Node c;
+    Node d;
+    Node e;
+    Node f;
+    Node y;
+
+    root.__left_ = &b;
 
-        std::__tree_remove(root.__left_, &b);
+    b.__parent_   = &root;
+    b.__right_    = &y;
+    b.__left_     = &d;
+    b.__is_black_ = true;
 
-        assert(std::__tree_invariant(root.__left_));
+    y.__parent_   = &b;
+    y.__right_    = 0;
+    y.__left_     = 0;
+    y.__is_black_ = true;
 
-        assert(root.__parent_ == 0);
-        assert(root.__left_ == &c);
-        assert(root.__right_ == 0);
-        assert(root.__is_black_ == false);
-
-        assert(a.__parent_ == &c);
-        assert(a.__left_ == 0);
-        assert(a.__right_ == 0);
-        assert(a.__is_black_ == false);
-
-        assert(c.__parent_ == &root);
-        assert(c.__left_ == &a);
-        assert(c.__right_ == 0);
-        assert(c.__is_black_ == true);
-
-        std::__tree_remove(root.__left_, &a);
-
-        assert(std::__tree_invariant(root.__left_));
-
-        assert(root.__parent_ == 0);
-        assert(root.__left_ == &c);
-        assert(root.__right_ == 0);
-        assert(root.__is_black_ == false);
-
-        assert(c.__parent_ == &root);
-        assert(c.__left_ == 0);
-        assert(c.__right_ == 0);
-        assert(c.__is_black_ == true);
+    d.__parent_   = &b;
+    d.__right_    = &c;
+    d.__left_     = &e;
+    d.__is_black_ = false;
 
-        std::__tree_remove(root.__left_, &c);
+    c.__parent_   = &d;
+    c.__right_    = &f;
+    c.__left_     = 0;
+    c.__is_black_ = true;
 
-        assert(std::__tree_invariant(root.__left_));
+    e.__parent_   = &d;
+    e.__right_    = 0;
+    e.__left_     = 0;
+    e.__is_black_ = true;
 
-        assert(root.__parent_ == 0);
-        assert(root.__left_ == 0);
-        assert(root.__right_ == 0);
-        assert(root.__is_black_ == false);
-    }
-    {
-        Node root;
-        Node a;
-        Node b;
-        Node c;
-
-        root.__left_ = &b;
-
-        b.__parent_ = &root;
-        b.__left_ = &a;
-        b.__right_ = &c;
-        b.__is_black_ = true;
+    f.__parent_   = &c;
+    f.__right_    = 0;
+    f.__left_     = 0;
+    f.__is_black_ = false;
 
-        a.__parent_ = &b;
-        a.__left_ = 0;
-        a.__right_ = 0;
-        a.__is_black_ = true;
-
-        c.__parent_ = &b;
-        c.__left_ = 0;
-        c.__right_ = 0;
-        c.__is_black_ = true;
+    std::__tree_remove(root.__left_, &y);
+    assert(std::__tree_invariant(root.__left_));
 
-        std::__tree_remove(root.__left_, &b);
+    assert(root.__parent_ == 0);
+    assert(root.__left_ == &d);
+    assert(root.__right_ == 0);
+    assert(root.__is_black_ == false);
+
+    assert(d.__parent_ == &root);
+    assert(d.__right_ == &f);
+    assert(d.__left_ == &e);
+    assert(d.__is_black_ == true);
 
-        assert(std::__tree_invariant(root.__left_));
+    assert(f.__parent_ == &d);
+    assert(f.__right_ == &b);
+    assert(f.__left_ == &c);
+    assert(f.__is_black_ == false);
+
+    assert(b.__parent_ == &f);
+    assert(b.__right_ == 0);
+    assert(b.__left_ == 0);
+    assert(b.__is_black_ == true);
+
+    assert(c.__parent_ == &f);
+    assert(c.__right_ == 0);
+    assert(c.__left_ == 0);
+    assert(c.__is_black_ == true);
+
+    assert(e.__parent_ == &d);
+    assert(e.__right_ == 0);
+    assert(e.__left_ == 0);
+    assert(e.__is_black_ == true);
+  }
+}
+
+void test2() {
+  {
+    Node root;
+    Node a;
+    Node b;
+    Node c;
+
+    root.__left_ = &b;
+
+    b.__parent_   = &root;
+    b.__left_     = &a;
+    b.__right_    = &c;
+    b.__is_black_ = true;
+
+    a.__parent_   = &b;
+    a.__left_     = 0;
+    a.__right_    = 0;
+    a.__is_black_ = true;
+
+    c.__parent_   = &b;
+    c.__left_     = 0;
+    c.__right_    = 0;
+    c.__is_black_ = true;
+
+    std::__tree_remove(root.__left_, &a);
+
+    assert(std::__tree_invariant(root.__left_));
+
+    assert(root.__parent_ == 0);
+    assert(root.__left_ == &b);
+    assert(root.__right_ == 0);
+    assert(root.__is_black_ == false);
+
+    assert(b.__parent_ == &root);
+    assert(b.__left_ == 0);
+    assert(b.__right_ == &c);
+    assert(b.__is_black_ == true);
+
+    assert(c.__parent_ == &b);
+    assert(c.__left_ == 0);
+    assert(c.__right_ == 0);
+    assert(c.__is_black_ == false);
+
+    std::__tree_remove(root.__left_, &b);
+
+    assert(std::__tree_invariant(root.__left_));
+
+    assert(root.__parent_ == 0);
+    assert(root.__left_ == &c);
+    assert(root.__right_ == 0);
+    assert(root.__is_black_ == false);
+
+    assert(c.__parent_ == &root);
+    assert(c.__left_ == 0);
+    assert(c.__right_ == 0);
+    assert(c.__is_black_ == true);
+
+    std::__tree_remove(root.__left_, &c);
+
+    assert(std::__tree_invariant(root.__left_));
+
+    assert(root.__parent_ == 0);
+    assert(root.__left_ == 0);
+    assert(root.__right_ == 0);
+    assert(root.__is_black_ == false);
+  }
+  {
+    Node root;
+    Node a;
+    Node b;
+    Node c;
+
+    root.__left_ = &b;
+
+    b.__parent_   = &root;
+    b.__left_     = &a;
+    b.__right_    = &c;
+    b.__is_black_ = true;
+
+    a.__parent_   = &b;
+    a.__left_     = 0;
+    a.__right_    = 0;
+    a.__is_black_ = false;
+
+    c.__parent_   = &b;
+    c.__left_     = 0;
+    c.__right_    = 0;
+    c.__is_black_ = false;
+
+    std::__tree_remove(root.__left_, &a);
+
+    assert(std::__tree_invariant(root.__left_));
+
+    assert(root.__parent_ == 0);
+    assert(root.__left_ == &b);
+    assert(root.__right_ == 0);
+    assert(root.__is_black_ == false);
+
+    assert(b.__parent_ == &root);
+    assert(b.__left_ == 0);
+    assert(b.__right_ == &c);
+    assert(b.__is_black_ == true);
+
+    assert(c.__parent_ == &b);
+    assert(c.__left_ == 0);
+    assert(c.__right_ == 0);
+    assert(c.__is_black_ == false);
+
+    std::__tree_remove(root.__left_, &b);
+
+    assert(std::__tree_invariant(root.__left_));
+
+    assert(root.__parent_ == 0);
+    assert(root.__left_ == &c);
+    assert(root.__right_ == 0);
+    assert(root.__is_black_ == false);
 
-        assert(root.__parent_ == 0);
-        assert(root.__left_ == &c);
-        assert(root.__right_ == 0);
-        assert(root.__is_black_ == false);
-
-        assert(a.__parent_ == &c);
-        assert(a.__left_ == 0);
-        assert(a.__right_ == 0);
-        assert(a.__is_black_ == false);
-
-        assert(c.__parent_ == &root);
-        assert(c.__left_ == &a);
-        assert(c.__right_ == 0);
-        assert(c.__is_black_ == true);
-
-        std::__tree_remove(root.__left_, &c);
-
-        assert(std::__tree_invariant(root.__left_));
-
-        assert(root.__parent_ == 0);
-        assert(root.__left_ == &a);
-        assert(root.__right_ == 0);
-        assert(root.__is_black_ == false);
-
-        assert(a.__parent_ == &root);
-        assert(a.__left_ == 0);
-        assert(a.__right_ == 0);
-        assert(a.__is_black_ == true);
+    assert(c.__parent_ == &root);
+    assert(c.__left_ == 0);
+    assert(c.__right_ == 0);
+    assert(c.__is_black_ == true);
 
-        std::__tree_remove(root.__left_, &a);
+    std::__tree_remove(root.__left_, &c);
 
-        assert(std::__tree_invariant(root.__left_));
+    assert(std::__tree_invariant(root.__left_));
 
-        assert(root.__parent_ == 0);
-        assert(root.__left_ == 0);
-        assert(root.__right_ == 0);
-        assert(root.__is_black_ == false);
-    }
-    {
-        Node root;
-        Node a;
-        Node b;
-        Node c;
-
-        root.__left_ = &b;
-
-        b.__parent_ = &root;
-        b.__left_ = &a;
-        b.__right_ = &c;
-        b.__is_black_ = true;
+    assert(root.__parent_ == 0);
+    assert(root.__left_ == 0);
+    assert(root.__right_ == 0);
+    assert(root.__is_black_ == false);
+  }
+  {
+    Node root;
+    Node a;
+    Node b;
+    Node c;
 
-        a.__parent_ = &b;
-        a.__left_ = 0;
-        a.__right_ = 0;
-        a.__is_black_ = false;
-
-        c.__parent_ = &b;
-        c.__left_ = 0;
-        c.__right_ = 0;
-        c.__is_black_ = false;
+    root.__left_ = &b;
 
-        std::__tree_remove(root.__left_, &b);
+    b.__parent_   = &root;
+    b.__left_     = &a;
+    b.__right_    = &c;
+    b.__is_black_ = true;
 
-        assert(std::__tree_invariant(root.__left_));
+    a.__parent_   = &b;
+    a.__left_     = 0;
+    a.__right_    = 0;
+    a.__is_black_ = true;
 
-        assert(root.__parent_ == 0);
-        assert(root.__left_ == &c);
-        assert(root.__right_ == 0);
-        assert(root.__is_black_ == false);
-
-        assert(a.__parent_ == &c);
-        assert(a.__left_ == 0);
-        assert(a.__right_ == 0);
-        assert(a.__is_black_ == false);
-
-        assert(c.__parent_ == &root);
-        assert(c.__left_ == &a);
-        assert(c.__right_ == 0);
-        assert(c.__is_black_ == true);
-
-        std::__tree_remove(root.__left_, &c);
-
-        assert(std::__tree_invariant(root.__left_));
-
-        assert(root.__parent_ == 0);
-        assert(root.__left_ == &a);
-        assert(root.__right_ == 0);
-        assert(root.__is_black_ == false);
-
-        assert(a.__parent_ == &root);
-        assert(a.__left_ == 0);
-        assert(a.__right_ == 0);
-        assert(a.__is_black_ == true);
+    c.__parent_   = &b;
+    c.__left_     = 0;
+    c.__right_    = 0;
+    c.__is_black_ = true;
 
-        std::__tree_remove(root.__left_, &a);
+    std::__tree_remove(root.__left_, &a);
 
-        assert(std::__tree_invariant(root.__left_));
+    assert(std::__tree_invariant(root.__left_));
 
-        assert(root.__parent_ == 0);
-        assert(root.__left_ == 0);
-        assert(root.__right_ == 0);
-        assert(root.__is_black_ == false);
-    }
-    {
-        Node root;
-        Node a;
-        Node b;
-        Node c;
-
-        root.__left_ = &b;
-
-        b.__parent_ = &root;
-        b.__left_ = &a;
-        b.__right_ = &c;
-        b.__is_black_ = true;
+    assert(root.__parent_ == 0);
+    assert(root.__left_ == &b);
+    assert(root.__right_ == 0);
+    assert(root.__is_black_ == false);
 
-        a.__parent_ = &b;
-        a.__left_ = 0;
-        a.__right_ = 0;
-        a.__is_black_ = true;
-
-        c.__parent_ = &b;
-        c.__left_ = 0;
-        c.__right_ = 0;
-        c.__is_black_ = true;
+    assert(b.__parent_ == &root);
+    assert(b.__left_ == 0);
+    assert(b.__right_ == &c);
+    assert(b.__is_black_ == true);
 
-        std::__tree_remove(root.__left_, &c);
+    assert(c.__parent_ == &b);
+    assert(c.__left_ == 0);
+    assert(c.__right_ == 0);
+    assert(c.__is_black_ == false);
 
-        assert(std::__tree_invariant(root.__left_));
+    std::__tree_remove(root.__left_, &c);
 
-        assert(root.__parent_ == 0);
-        assert(root.__left_ == &b);
-        assert(root.__right_ == 0);
-        assert(root.__is_black_ == false);
-
-        assert(a.__parent_ == &b);
-        assert(a.__left_ == 0);
-        assert(a.__right_ == 0);
-        assert(a.__is_black_ == false);
-
-        assert(b.__parent_ == &root);
-        assert(b.__left_ == &a);
-        assert(b.__right_ == 0);
-        assert(b.__is_black_ == true);
-
-        std::__tree_remove(root.__left_, &b);
-
-        assert(std::__tree_invariant(root.__left_));
-
-        assert(root.__parent_ == 0);
-        assert(root.__left_ == &a);
-        assert(root.__right_ == 0);
-        assert(root.__is_black_ == false);
-
-        assert(a.__parent_ == &root);
-        assert(a.__left_ == 0);
-        assert(a.__right_ == 0);
-        assert(a.__is_black_ == true);
+    assert(std::__tree_invariant(root.__left_));
 
-        std::__tree_remove(root.__left_, &a);
+    assert(root.__parent_ == 0);
+    assert(root.__left_ == &b);
+    assert(root.__right_ == 0);
+    assert(root.__is_black_ == false);
 
-        assert(std::__tree_invariant(root.__left_));
+    assert(b.__parent_ == &root);
+    assert(b.__left_ == 0);
+    assert(b.__right_ == 0);
+    assert(b.__is_black_ == true);
 
-        assert(root.__parent_ == 0);
-        assert(root.__left_ == 0);
-        assert(root.__right_ == 0);
-        assert(root.__is_black_ == false);
-    }
-    {
-        Node root;
-        Node a;
-        Node b;
-        Node c;
-
-        root.__left_ = &b;
-
-        b.__parent_ = &root;
-        b.__left_ = &a;
-        b.__right_ = &c;
-        b.__is_black_ = true;
+    std::__tree_remove(root.__left_, &b);
 
-        a.__parent_ = &b;
-        a.__left_ = 0;
-        a.__right_ = 0;
-        a.__is_black_ = false;
-
-        c.__parent_ = &b;
-        c.__left_ = 0;
-        c.__right_ = 0;
-        c.__is_black_ = false;
+    assert(std::__tree_invariant(root.__left_));
 
-        std::__tree_remove(root.__left_, &c);
+    assert(root.__parent_ == 0);
+    assert(root.__left_ == 0);
+    assert(root.__right_ == 0);
+    assert(root.__is_black_ == false);
+  }
+  {
+    Node root;
+    Node a;
+    Node b;
+    Node c;
 
-        assert(std::__tree_invariant(root.__left_));
+    root.__left_ = &b;
 
-        assert(root.__parent_ == 0);
-        assert(root.__left_ == &b);
-        assert(root.__right_ == 0);
-        assert(root.__is_black_ == false);
-
-        assert(a.__parent_ == &b);
-        assert(a.__left_ == 0);
-        assert(a.__right_ == 0);
-        assert(a.__is_black_ == false);
-
-        assert(b.__parent_ == &root);
-        assert(b.__left_ == &a);
-        assert(b.__right_ == 0);
-        assert(b.__is_black_ == true);
-
-        std::__tree_remove(root.__left_, &b);
-
-        assert(std::__tree_invariant(root.__left_));
-
-        assert(root.__parent_ == 0);
-        assert(root.__left_ == &a);
-        assert(root.__right_ == 0);
-        assert(root.__is_black_ == false);
-
-        assert(a.__parent_ == &root);
-        assert(a.__left_ == 0);
-        assert(a.__right_ == 0);
-        assert(a.__is_black_ == true);
+    b.__parent_   = &root;
+    b.__left_     = &a;
+    b.__right_    = &c;
+    b.__is_black_ = true;
 
-        std::__tree_remove(root.__left_, &a);
+    a.__parent_   = &b;
+    a.__left_     = 0;
+    a.__right_    = 0;
+    a.__is_black_ = false;
 
-        assert(std::__tree_invariant(root.__left_));
+    c.__parent_   = &b;
+    c.__left_     = 0;
+    c.__right_    = 0;
+    c.__is_black_ = false;
 
-        assert(root.__parent_ == 0);
-        assert(root.__left_ == 0);
-        assert(root.__right_ == 0);
-        assert(root.__is_black_ == false);
-    }
-    {
-        Node root;
-        Node a;
-        Node b;
-        Node c;
-
-        root.__left_ = &b;
-
-        b.__parent_ = &root;
-        b.__left_ = &a;
-        b.__right_ = &c;
-        b.__is_black_ = true;
+    std::__tree_remove(root.__left_, &a);
 
-        a.__parent_ = &b;
-        a.__left_ = 0;
-        a.__right_ = 0;
-        a.__is_black_ = true;
-
-        c.__parent_ = &b;
-        c.__left_ = 0;
-        c.__right_ = 0;
-        c.__is_black_ = true;
+    assert(std::__tree_invariant(root.__left_));
 
-        std::__tree_remove(root.__left_, &c);
+    assert(root.__parent_ == 0);
+    assert(root.__left_ == &b);
+    assert(root.__right_ == 0);
+    assert(root.__is_black_ == false);
 
-        assert(std::__tree_invariant(root.__left_));
+    assert(b.__parent_ == &root);
+    assert(b.__left_ == 0);
+    assert(b.__right_ == &c);
+    assert(b.__is_black_ == true);
 
-        assert(root.__parent_ == 0);
-        assert(root.__left_ == &b);
-        assert(root.__right_ == 0);
-        assert(root.__is_black_ == false);
-
-        assert(a.__parent_ == &b);
-        assert(a.__left_ == 0);
-        assert(a.__right_ == 0);
-        assert(a.__is_black_ == false);
+    assert(c.__parent_ == &b);
+    assert(c.__left_ == 0);
+    assert(c.__right_ == 0);
+    assert(c.__is_black_ == false);
 
-        assert(b.__parent_ == &root);
-        assert(b.__left_ == &a);
-        assert(b.__right_ == 0);
-        assert(b.__is_black_ == true);
+    std::__tree_remove(root.__left_, &c);
 
-        std::__tree_remove(root.__left_, &a);
+    assert(std::__tree_invariant(root.__left_));
 
-        assert(std::__tree_invariant(root.__left_));
+    assert(root.__parent_ == 0);
+    assert(root.__left_ == &b);
+    assert(root.__right_ == 0);
+    assert(root.__is_black_ == false);
 
-        assert(root.__parent_ == 0);
-        assert(root.__left_ == &b);
-        assert(root.__right_ == 0);
-        assert(root.__is_black_ == false);
+    assert(b.__parent_ == &root);
+    assert(b.__left_ == 0);
+    assert(b.__right_ == 0);
+    assert(b.__is_black_ == true);
 
-        assert(b.__parent_ == &root);
-        assert(b.__left_ == 0);
-        assert(b.__right_ == 0);
-        assert(b.__is_black_ == true);
+    std::__tree_remove(root.__left_, &b);
 
-        std::__tree_remove(root.__left_, &b);
+    assert(std::__tree_invariant(root.__left_));
 
-        assert(std::__tree_invariant(root.__left_));
+    assert(root.__parent_ == 0);
+    assert(root.__left_ == 0);
+    assert(root.__right_ == 0);
+    assert(root.__is_black_ == false);
+  }
+  {
+    Node root;
+    Node a;
+    Node b;
+    Node c;
 
-        assert(root.__parent_ == 0);
-        assert(root.__left_ == 0);
-        assert(root.__right_ == 0);
-        assert(root.__is_black_ == false);
-    }
-    {
-        Node root;
-        Node a;
-        Node b;
-        Node c;
+    root.__left_ = &b;
 
-        root.__left_ = &b;
+    b.__parent_   = &root;
+    b.__left_     = &a;
+    b.__right_    = &c;
+    b.__is_black_ = true;
 
-        b.__parent_ = &root;
-        b.__left_ = &a;
-        b.__right_ = &c;
-        b.__is_black_ = true;
+    a.__parent_   = &b;
+    a.__left_     = 0;
+    a.__right_    = 0;
+    a.__is_black_ = true;
 
-        a.__parent_ = &b;
-        a.__left_ = 0;
-        a.__right_ = 0;
-        a.__is_black_ = false;
-
-        c.__parent_ = &b;
-        c.__left_ = 0;
-        c.__right_ = 0;
-        c.__is_black_ = false;
+    c.__parent_   = &b;
+    c.__left_     = 0;
+    c.__right_    = 0;
+    c.__is_black_ = true;
 
-        std::__tree_remove(root.__left_, &c);
+    std::__tree_remove(root.__left_, &b);
 
-        assert(std::__tree_invariant(root.__left_));
+    assert(std::__tree_invariant(root.__left_));
 
-        assert(root.__parent_ == 0);
-        assert(root.__left_ == &b);
-        assert(root.__right_ == 0);
-        assert(root.__is_black_ == false);
-
-        assert(a.__parent_ == &b);
-        assert(a.__left_ == 0);
-        assert(a.__right_ == 0);
-        assert(a.__is_black_ == false);
+    assert(root.__parent_ == 0);
+    assert(root.__left_ == &c);
+    assert(root.__right_ == 0);
+    assert(root.__is_black_ == false);
 
-        assert(b.__parent_ == &root);
-        assert(b.__left_ == &a);
-        assert(b.__right_ == 0);
-        assert(b.__is_black_ == true);
+    assert(a.__parent_ == &c);
+    assert(a.__left_ == 0);
+    assert(a.__right_ == 0);
+    assert(a.__is_black_ == false);
 
-        std::__tree_remove(root.__left_, &a);
+    assert(c.__parent_ == &root);
+    assert(c.__left_ == &a);
+    assert(c.__right_ == 0);
+    assert(c.__is_black_ == true);
 
-        assert(std::__tree_invariant(root.__left_));
+    std::__tree_remove(root.__left_, &a);
 
-        assert(root.__parent_ == 0);
-        assert(root.__left_ == &b);
-        assert(root.__right_ == 0);
-        assert(root.__is_black_ == false);
+    assert(std::__tree_invariant(root.__left_));
 
-        assert(b.__parent_ == &root);
-        assert(b.__left_ == 0);
-        assert(b.__right_ == 0);
-        assert(b.__is_black_ == true);
+    assert(root.__parent_ == 0);
+    assert(root.__left_ == &c);
+    assert(root.__right_ == 0);
+    assert(root.__is_black_ == false);
 
-        std::__tree_remove(root.__left_, &b);
+    assert(c.__parent_ == &root);
+    assert(c.__left_ == 0);
+    assert(c.__right_ == 0);
+    assert(c.__is_black_ == true);
 
-        assert(std::__tree_invariant(root.__left_));
+    std::__tree_remove(root.__left_, &c);
 
-        assert(root.__parent_ == 0);
-        assert(root.__left_ == 0);
-        assert(root.__right_ == 0);
-        assert(root.__is_black_ == false);
-    }
-}
+    assert(std::__tree_invariant(root.__left_));
 
-void
-test3()
-{
+    assert(root.__parent_ == 0);
+    assert(root.__left_ == 0);
+    assert(root.__right_ == 0);
+    assert(root.__is_black_ == false);
+  }
+  {
     Node root;
     Node a;
     Node b;
     Node c;
-    Node d;
-    Node e;
-    Node f;
-    Node g;
-    Node h;
 
-    root.__left_ = &e;
+    root.__left_ = &b;
 
-    e.__parent_ = &root;
-    e.__left_ = &c;
-    e.__right_ = &g;
-    e.__is_black_ = true;
+    b.__parent_   = &root;
+    b.__left_     = &a;
+    b.__right_    = &c;
+    b.__is_black_ = true;
 
-    c.__parent_ = &e;
-    c.__left_ = &b;
-    c.__right_ = &d;
+    a.__parent_   = &b;
+    a.__left_     = 0;
+    a.__right_    = 0;
+    a.__is_black_ = false;
+
+    c.__parent_   = &b;
+    c.__left_     = 0;
+    c.__right_    = 0;
     c.__is_black_ = false;
 
-    g.__parent_ = &e;
-    g.__left_ = &f;
-    g.__right_ = &h;
-    g.__is_black_ = false;
+    std::__tree_remove(root.__left_, &b);
+
+    assert(std::__tree_invariant(root.__left_));
+
+    assert(root.__parent_ == 0);
+    assert(root.__left_ == &c);
+    assert(root.__right_ == 0);
+    assert(root.__is_black_ == false);
+
+    assert(a.__parent_ == &c);
+    assert(a.__left_ == 0);
+    assert(a.__right_ == 0);
+    assert(a.__is_black_ == false);
+
+    assert(c.__parent_ == &root);
+    assert(c.__left_ == &a);
+    assert(c.__right_ == 0);
+    assert(c.__is_black_ == true);
+
+    std::__tree_remove(root.__left_, &a);
+
+    assert(std::__tree_invariant(root.__left_));
+
+    assert(root.__parent_ == 0);
+    assert(root.__left_ == &c);
+    assert(root.__right_ == 0);
+    assert(root.__is_black_ == false);
+
+    assert(c.__parent_ == &root);
+    assert(c.__left_ == 0);
+    assert(c.__right_ == 0);
+    assert(c.__is_black_ == true);
+
+    std::__tree_remove(root.__left_, &c);
+
+    assert(std::__tree_invariant(root.__left_));
+
+    assert(root.__parent_ == 0);
+    assert(root.__left_ == 0);
+    assert(root.__right_ == 0);
+    assert(root.__is_black_ == false);
+  }
+  {
+    Node root;
+    Node a;
+    Node b;
+    Node c;
+
+    root.__left_ = &b;
 
-    b.__parent_ = &c;
-    b.__left_ = &a;
-    b.__right_ = 0;
+    b.__parent_   = &root;
+    b.__left_     = &a;
+    b.__right_    = &c;
     b.__is_black_ = true;
 
-    d.__parent_ = &c;
-    d.__left_ = 0;
-    d.__right_ = 0;
-    d.__is_black_ = true;
+    a.__parent_   = &b;
+    a.__left_     = 0;
+    a.__right_    = 0;
+    a.__is_black_ = true;
+
+    c.__parent_   = &b;
+    c.__left_     = 0;
+    c.__right_    = 0;
+    c.__is_black_ = true;
+
+    std::__tree_remove(root.__left_, &b);
+
+    assert(std::__tree_invariant(root.__left_));
+
+    assert(root.__parent_ == 0);
+    assert(root.__left_ == &c);
+    assert(root.__right_ == 0);
+    assert(root.__is_black_ == false);
+
+    assert(a.__parent_ == &c);
+    assert(a.__left_ == 0);
+    assert(a.__right_ == 0);
+    assert(a.__is_black_ == false);
+
+    assert(c.__parent_ == &root);
+    assert(c.__left_ == &a);
+    assert(c.__right_ == 0);
+    assert(c.__is_black_ == true);
+
+    std::__tree_remove(root.__left_, &c);
+
+    assert(std::__tree_invariant(root.__left_));
+
+    assert(root.__parent_ == 0);
+    assert(root.__left_ == &a);
+    assert(root.__right_ == 0);
+    assert(root.__is_black_ == false);
+
+    assert(a.__parent_ == &root);
+    assert(a.__left_ == 0);
+    assert(a.__right_ == 0);
+    assert(a.__is_black_ == true);
+
+    std::__tree_remove(root.__left_, &a);
+
+    assert(std::__tree_invariant(root.__left_));
+
+    assert(root.__parent_ == 0);
+    assert(root.__left_ == 0);
+    assert(root.__right_ == 0);
+    assert(root.__is_black_ == false);
+  }
+  {
+    Node root;
+    Node a;
+    Node b;
+    Node c;
 
-    f.__parent_ = &g;
-    f.__left_ = 0;
-    f.__right_ = 0;
-    f.__is_black_ = true;
+    root.__left_ = &b;
 
-    h.__parent_ = &g;
-    h.__left_ = 0;
-    h.__right_ = 0;
-    h.__is_black_ = true;
+    b.__parent_   = &root;
+    b.__left_     = &a;
+    b.__right_    = &c;
+    b.__is_black_ = true;
 
-    a.__parent_ = &b;
-    a.__left_ = 0;
-    a.__right_ = 0;
+    a.__parent_   = &b;
+    a.__left_     = 0;
+    a.__right_    = 0;
     a.__is_black_ = false;
 
-    assert(std::__tree_invariant(root.__left_));
+    c.__parent_   = &b;
+    c.__left_     = 0;
+    c.__right_    = 0;
+    c.__is_black_ = false;
 
-    std::__tree_remove(root.__left_, &h);
+    std::__tree_remove(root.__left_, &b);
 
     assert(std::__tree_invariant(root.__left_));
 
     assert(root.__parent_ == 0);
-    assert(root.__left_ == &e);
+    assert(root.__left_ == &c);
     assert(root.__right_ == 0);
     assert(root.__is_black_ == false);
 
-    assert(e.__parent_ == &root);
-    assert(e.__left_ == &c);
-    assert(e.__right_ == &g);
-    assert(e.__is_black_ == true);
-
-    assert(c.__parent_ == &e);
-    assert(c.__left_ == &b);
-    assert(c.__right_ == &d);
-    assert(c.__is_black_ == false);
-
-    assert(g.__parent_ == &e);
-    assert(g.__left_ == &f);
-    assert(g.__right_ == 0);
-    assert(g.__is_black_ == true);
-
-    assert(b.__parent_ == &c);
-    assert(b.__left_ == &a);
-    assert(b.__right_ == 0);
-    assert(b.__is_black_ == true);
-
-    assert(a.__parent_ == &b);
+    assert(a.__parent_ == &c);
     assert(a.__left_ == 0);
     assert(a.__right_ == 0);
     assert(a.__is_black_ == false);
 
-    assert(d.__parent_ == &c);
-    assert(d.__left_ == 0);
-    assert(d.__right_ == 0);
-    assert(d.__is_black_ == true);
-
-    assert(f.__parent_ == &g);
-    assert(f.__left_ == 0);
-    assert(f.__right_ == 0);
-    assert(f.__is_black_ == false);
+    assert(c.__parent_ == &root);
+    assert(c.__left_ == &a);
+    assert(c.__right_ == 0);
+    assert(c.__is_black_ == true);
 
-    std::__tree_remove(root.__left_, &g);
+    std::__tree_remove(root.__left_, &c);
 
     assert(std::__tree_invariant(root.__left_));
 
     assert(root.__parent_ == 0);
-    assert(root.__left_ == &e);
+    assert(root.__left_ == &a);
     assert(root.__right_ == 0);
     assert(root.__is_black_ == false);
 
-    assert(e.__parent_ == &root);
-    assert(e.__left_ == &c);
-    assert(e.__right_ == &f);
-    assert(e.__is_black_ == true);
-
-    assert(c.__parent_ == &e);
-    assert(c.__left_ == &b);
-    assert(c.__right_ == &d);
-    assert(c.__is_black_ == false);
-
-    assert(b.__parent_ == &c);
-    assert(b.__left_ == &a);
-    assert(b.__right_ == 0);
-    assert(b.__is_black_ == true);
-
-    assert(a.__parent_ == &b);
+    assert(a.__parent_ == &root);
     assert(a.__left_ == 0);
     assert(a.__right_ == 0);
-    assert(a.__is_black_ == false);
-
-    assert(d.__parent_ == &c);
-    assert(d.__left_ == 0);
-    assert(d.__right_ == 0);
-    assert(d.__is_black_ == true);
-
-    assert(f.__parent_ == &e);
-    assert(f.__left_ == 0);
-    assert(f.__right_ == 0);
-    assert(f.__is_black_ == true);
+    assert(a.__is_black_ == true);
 
-    std::__tree_remove(root.__left_, &f);
+    std::__tree_remove(root.__left_, &a);
 
     assert(std::__tree_invariant(root.__left_));
 
     assert(root.__parent_ == 0);
-    assert(root.__left_ == &c);
+    assert(root.__left_ == 0);
     assert(root.__right_ == 0);
     assert(root.__is_black_ == false);
+  }
+  {
+    Node root;
+    Node a;
+    Node b;
+    Node c;
 
-    assert(c.__parent_ == &root);
-    assert(c.__left_ == &b);
-    assert(c.__right_ == &e);
-    assert(c.__is_black_ == true);
-
-    assert(b.__parent_ == &c);
-    assert(b.__left_ == &a);
-    assert(b.__right_ == 0);
-    assert(b.__is_black_ == true);
+    root.__left_ = &b;
 
-    assert(e.__parent_ == &c);
-    assert(e.__left_ == &d);
-    assert(e.__right_ == 0);
-    assert(e.__is_black_ == true);
+    b.__parent_   = &root;
+    b.__left_     = &a;
+    b.__right_    = &c;
+    b.__is_black_ = true;
 
-    assert(a.__parent_ == &b);
-    assert(a.__left_ == 0);
-    assert(a.__right_ == 0);
-    assert(a.__is_black_ == false);
+    a.__parent_   = &b;
+    a.__left_     = 0;
+    a.__right_    = 0;
+    a.__is_black_ = true;
 
-    assert(d.__parent_ == &e);
-    assert(d.__left_ == 0);
-    assert(d.__right_ == 0);
-    assert(d.__is_black_ == false);
+    c.__parent_   = &b;
+    c.__left_     = 0;
+    c.__right_    = 0;
+    c.__is_black_ = true;
 
-    std::__tree_remove(root.__left_, &e);
+    std::__tree_remove(root.__left_, &c);
 
     assert(std::__tree_invariant(root.__left_));
 
     assert(root.__parent_ == 0);
-    assert(root.__left_ == &c);
+    assert(root.__left_ == &b);
     assert(root.__right_ == 0);
     assert(root.__is_black_ == false);
 
-    assert(c.__parent_ == &root);
-    assert(c.__left_ == &b);
-    assert(c.__right_ == &d);
-    assert(c.__is_black_ == true);
-
-    assert(b.__parent_ == &c);
-    assert(b.__left_ == &a);
-    assert(b.__right_ == 0);
-    assert(b.__is_black_ == true);
-
     assert(a.__parent_ == &b);
     assert(a.__left_ == 0);
     assert(a.__right_ == 0);
     assert(a.__is_black_ == false);
 
-    assert(d.__parent_ == &c);
-    assert(d.__left_ == 0);
-    assert(d.__right_ == 0);
-    assert(d.__is_black_ == true);
+    assert(b.__parent_ == &root);
+    assert(b.__left_ == &a);
+    assert(b.__right_ == 0);
+    assert(b.__is_black_ == true);
 
-    std::__tree_remove(root.__left_, &d);
+    std::__tree_remove(root.__left_, &b);
 
     assert(std::__tree_invariant(root.__left_));
 
     assert(root.__parent_ == 0);
-    assert(root.__left_ == &b);
+    assert(root.__left_ == &a);
     assert(root.__right_ == 0);
     assert(root.__is_black_ == false);
 
-    assert(b.__parent_ == &root);
-    assert(b.__left_ == &a);
-    assert(b.__right_ == &c);
-    assert(b.__is_black_ == true);
-
-    assert(a.__parent_ == &b);
+    assert(a.__parent_ == &root);
     assert(a.__left_ == 0);
     assert(a.__right_ == 0);
     assert(a.__is_black_ == true);
 
-    assert(c.__parent_ == &b);
-    assert(c.__left_ == 0);
-    assert(c.__right_ == 0);
-    assert(c.__is_black_ == true);
+    std::__tree_remove(root.__left_, &a);
+
+    assert(std::__tree_invariant(root.__left_));
+
+    assert(root.__parent_ == 0);
+    assert(root.__left_ == 0);
+    assert(root.__right_ == 0);
+    assert(root.__is_black_ == false);
+  }
+  {
+    Node root;
+    Node a;
+    Node b;
+    Node c;
+
+    root.__left_ = &b;
+
+    b.__parent_   = &root;
+    b.__left_     = &a;
+    b.__right_    = &c;
+    b.__is_black_ = true;
+
+    a.__parent_   = &b;
+    a.__left_     = 0;
+    a.__right_    = 0;
+    a.__is_black_ = false;
+
+    c.__parent_   = &b;
+    c.__left_     = 0;
+    c.__right_    = 0;
+    c.__is_black_ = false;
 
     std::__tree_remove(root.__left_, &c);
 
@@ -1341,16 +935,16 @@ test3()
     assert(root.__right_ == 0);
     assert(root.__is_black_ == false);
 
-    assert(b.__parent_ == &root);
-    assert(b.__left_ == &a);
-    assert(b.__right_ == 0);
-    assert(b.__is_black_ == true);
-
     assert(a.__parent_ == &b);
     assert(a.__left_ == 0);
     assert(a.__right_ == 0);
     assert(a.__is_black_ == false);
 
+    assert(b.__parent_ == &root);
+    assert(b.__left_ == &a);
+    assert(b.__right_ == 0);
+    assert(b.__is_black_ == true);
+
     std::__tree_remove(root.__left_, &b);
 
     assert(std::__tree_invariant(root.__left_));
@@ -1373,284 +967,680 @@ test3()
     assert(root.__left_ == 0);
     assert(root.__right_ == 0);
     assert(root.__is_black_ == false);
-}
-
-void
-test4()
-{
+  }
+  {
     Node root;
     Node a;
     Node b;
     Node c;
-    Node d;
-    Node e;
-    Node f;
-    Node g;
-    Node h;
-
-    root.__left_ = &d;
 
-    d.__parent_ = &root;
-    d.__left_ = &b;
-    d.__right_ = &f;
-    d.__is_black_ = true;
+    root.__left_ = &b;
 
-    b.__parent_ = &d;
-    b.__left_ = &a;
-    b.__right_ = &c;
-    b.__is_black_ = false;
-
-    f.__parent_ = &d;
-    f.__left_ = &e;
-    f.__right_ = &g;
-    f.__is_black_ = false;
+    b.__parent_   = &root;
+    b.__left_     = &a;
+    b.__right_    = &c;
+    b.__is_black_ = true;
 
-    a.__parent_ = &b;
-    a.__left_ = 0;
-    a.__right_ = 0;
+    a.__parent_   = &b;
+    a.__left_     = 0;
+    a.__right_    = 0;
     a.__is_black_ = true;
 
-    c.__parent_ = &b;
-    c.__left_ = 0;
-    c.__right_ = 0;
+    c.__parent_   = &b;
+    c.__left_     = 0;
+    c.__right_    = 0;
     c.__is_black_ = true;
 
-    e.__parent_ = &f;
-    e.__left_ = 0;
-    e.__right_ = 0;
-    e.__is_black_ = true;
+    std::__tree_remove(root.__left_, &c);
 
-    g.__parent_ = &f;
-    g.__left_ = 0;
-    g.__right_ = &h;
-    g.__is_black_ = true;
+    assert(std::__tree_invariant(root.__left_));
 
-    h.__parent_ = &g;
-    h.__left_ = 0;
-    h.__right_ = 0;
-    h.__is_black_ = false;
+    assert(root.__parent_ == 0);
+    assert(root.__left_ == &b);
+    assert(root.__right_ == 0);
+    assert(root.__is_black_ == false);
 
-    assert(std::__tree_invariant(root.__left_));
+    assert(a.__parent_ == &b);
+    assert(a.__left_ == 0);
+    assert(a.__right_ == 0);
+    assert(a.__is_black_ == false);
+
+    assert(b.__parent_ == &root);
+    assert(b.__left_ == &a);
+    assert(b.__right_ == 0);
+    assert(b.__is_black_ == true);
 
     std::__tree_remove(root.__left_, &a);
 
     assert(std::__tree_invariant(root.__left_));
 
     assert(root.__parent_ == 0);
-    assert(root.__left_ == &d);
+    assert(root.__left_ == &b);
     assert(root.__right_ == 0);
     assert(root.__is_black_ == false);
 
-    assert(d.__parent_ == &root);
-    assert(d.__left_ == &b);
-    assert(d.__right_ == &f);
-    assert(d.__is_black_ == true);
-
-    assert(b.__parent_ == &d);
+    assert(b.__parent_ == &root);
     assert(b.__left_ == 0);
-    assert(b.__right_ == &c);
+    assert(b.__right_ == 0);
     assert(b.__is_black_ == true);
 
-    assert(f.__parent_ == &d);
-    assert(f.__left_ == &e);
-    assert(f.__right_ == &g);
-    assert(f.__is_black_ == false);
+    std::__tree_remove(root.__left_, &b);
 
-    assert(c.__parent_ == &b);
-    assert(c.__left_ == 0);
-    assert(c.__right_ == 0);
-    assert(c.__is_black_ == false);
+    assert(std::__tree_invariant(root.__left_));
 
-    assert(e.__parent_ == &f);
-    assert(e.__left_ == 0);
-    assert(e.__right_ == 0);
-    assert(e.__is_black_ == true);
+    assert(root.__parent_ == 0);
+    assert(root.__left_ == 0);
+    assert(root.__right_ == 0);
+    assert(root.__is_black_ == false);
+  }
+  {
+    Node root;
+    Node a;
+    Node b;
+    Node c;
 
-    assert(g.__parent_ == &f);
-    assert(g.__left_ == 0);
-    assert(g.__right_ == &h);
-    assert(g.__is_black_ == true);
+    root.__left_ = &b;
 
-    assert(h.__parent_ == &g);
-    assert(h.__left_ == 0);
-    assert(h.__right_ == 0);
-    assert(h.__is_black_ == false);
+    b.__parent_   = &root;
+    b.__left_     = &a;
+    b.__right_    = &c;
+    b.__is_black_ = true;
 
-    std::__tree_remove(root.__left_, &b);
+    a.__parent_   = &b;
+    a.__left_     = 0;
+    a.__right_    = 0;
+    a.__is_black_ = false;
+
+    c.__parent_   = &b;
+    c.__left_     = 0;
+    c.__right_    = 0;
+    c.__is_black_ = false;
+
+    std::__tree_remove(root.__left_, &c);
 
     assert(std::__tree_invariant(root.__left_));
 
     assert(root.__parent_ == 0);
-    assert(root.__left_ == &d);
+    assert(root.__left_ == &b);
     assert(root.__right_ == 0);
     assert(root.__is_black_ == false);
 
-    assert(d.__parent_ == &root);
-    assert(d.__left_ == &c);
-    assert(d.__right_ == &f);
-    assert(d.__is_black_ == true);
+    assert(a.__parent_ == &b);
+    assert(a.__left_ == 0);
+    assert(a.__right_ == 0);
+    assert(a.__is_black_ == false);
 
-    assert(c.__parent_ == &d);
-    assert(c.__left_ == 0);
-    assert(c.__right_ == 0);
-    assert(c.__is_black_ == true);
+    assert(b.__parent_ == &root);
+    assert(b.__left_ == &a);
+    assert(b.__right_ == 0);
+    assert(b.__is_black_ == true);
 
-    assert(f.__parent_ == &d);
-    assert(f.__left_ == &e);
-    assert(f.__right_ == &g);
-    assert(f.__is_black_ == false);
+    std::__tree_remove(root.__left_, &a);
 
-    assert(e.__parent_ == &f);
-    assert(e.__left_ == 0);
-    assert(e.__right_ == 0);
-    assert(e.__is_black_ == true);
+    assert(std::__tree_invariant(root.__left_));
 
-    assert(g.__parent_ == &f);
-    assert(g.__left_ == 0);
-    assert(g.__right_ == &h);
-    assert(g.__is_black_ == true);
+    assert(root.__parent_ == 0);
+    assert(root.__left_ == &b);
+    assert(root.__right_ == 0);
+    assert(root.__is_black_ == false);
 
-    assert(h.__parent_ == &g);
-    assert(h.__left_ == 0);
-    assert(h.__right_ == 0);
-    assert(h.__is_black_ == false);
+    assert(b.__parent_ == &root);
+    assert(b.__left_ == 0);
+    assert(b.__right_ == 0);
+    assert(b.__is_black_ == true);
 
-    std::__tree_remove(root.__left_, &c);
+    std::__tree_remove(root.__left_, &b);
 
     assert(std::__tree_invariant(root.__left_));
 
     assert(root.__parent_ == 0);
-    assert(root.__left_ == &f);
+    assert(root.__left_ == 0);
     assert(root.__right_ == 0);
     assert(root.__is_black_ == false);
+  }
+}
 
-    assert(f.__parent_ == &root);
-    assert(f.__left_ == &d);
-    assert(f.__right_ == &g);
-    assert(f.__is_black_ == true);
+void test3() {
+  Node root;
+  Node a;
+  Node b;
+  Node c;
+  Node d;
+  Node e;
+  Node f;
+  Node g;
+  Node h;
+
+  root.__left_ = &e;
+
+  e.__parent_   = &root;
+  e.__left_     = &c;
+  e.__right_    = &g;
+  e.__is_black_ = true;
+
+  c.__parent_   = &e;
+  c.__left_     = &b;
+  c.__right_    = &d;
+  c.__is_black_ = false;
+
+  g.__parent_   = &e;
+  g.__left_     = &f;
+  g.__right_    = &h;
+  g.__is_black_ = false;
+
+  b.__parent_   = &c;
+  b.__left_     = &a;
+  b.__right_    = 0;
+  b.__is_black_ = true;
+
+  d.__parent_   = &c;
+  d.__left_     = 0;
+  d.__right_    = 0;
+  d.__is_black_ = true;
+
+  f.__parent_   = &g;
+  f.__left_     = 0;
+  f.__right_    = 0;
+  f.__is_black_ = true;
+
+  h.__parent_   = &g;
+  h.__left_     = 0;
+  h.__right_    = 0;
+  h.__is_black_ = true;
+
+  a.__parent_   = &b;
+  a.__left_     = 0;
+  a.__right_    = 0;
+  a.__is_black_ = false;
+
+  assert(std::__tree_invariant(root.__left_));
+
+  std::__tree_remove(root.__left_, &h);
+
+  assert(std::__tree_invariant(root.__left_));
+
+  assert(root.__parent_ == 0);
+  assert(root.__left_ == &e);
+  assert(root.__right_ == 0);
+  assert(root.__is_black_ == false);
+
+  assert(e.__parent_ == &root);
+  assert(e.__left_ == &c);
+  assert(e.__right_ == &g);
+  assert(e.__is_black_ == true);
+
+  assert(c.__parent_ == &e);
+  assert(c.__left_ == &b);
+  assert(c.__right_ == &d);
+  assert(c.__is_black_ == false);
+
+  assert(g.__parent_ == &e);
+  assert(g.__left_ == &f);
+  assert(g.__right_ == 0);
+  assert(g.__is_black_ == true);
+
+  assert(b.__parent_ == &c);
+  assert(b.__left_ == &a);
+  assert(b.__right_ == 0);
+  assert(b.__is_black_ == true);
+
+  assert(a.__parent_ == &b);
+  assert(a.__left_ == 0);
+  assert(a.__right_ == 0);
+  assert(a.__is_black_ == false);
+
+  assert(d.__parent_ == &c);
+  assert(d.__left_ == 0);
+  assert(d.__right_ == 0);
+  assert(d.__is_black_ == true);
+
+  assert(f.__parent_ == &g);
+  assert(f.__left_ == 0);
+  assert(f.__right_ == 0);
+  assert(f.__is_black_ == false);
+
+  std::__tree_remove(root.__left_, &g);
+
+  assert(std::__tree_invariant(root.__left_));
+
+  assert(root.__parent_ == 0);
+  assert(root.__left_ == &e);
+  assert(root.__right_ == 0);
+  assert(root.__is_black_ == false);
+
+  assert(e.__parent_ == &root);
+  assert(e.__left_ == &c);
+  assert(e.__right_ == &f);
+  assert(e.__is_black_ == true);
+
+  assert(c.__parent_ == &e);
+  assert(c.__left_ == &b);
+  assert(c.__right_ == &d);
+  assert(c.__is_black_ == false);
+
+  assert(b.__parent_ == &c);
+  assert(b.__left_ == &a);
+  assert(b.__right_ == 0);
+  assert(b.__is_black_ == true);
+
+  assert(a.__parent_ == &b);
+  assert(a.__left_ == 0);
+  assert(a.__right_ == 0);
+  assert(a.__is_black_ == false);
+
+  assert(d.__parent_ == &c);
+  assert(d.__left_ == 0);
+  assert(d.__right_ == 0);
+  assert(d.__is_black_ == true);
+
+  assert(f.__parent_ == &e);
+  assert(f.__left_ == 0);
+  assert(f.__right_ == 0);
+  assert(f.__is_black_ == true);
+
+  std::__tree_remove(root.__left_, &f);
+
+  assert(std::__tree_invariant(root.__left_));
+
+  assert(root.__parent_ == 0);
+  assert(root.__left_ == &c);
+  assert(root.__right_ == 0);
+  assert(root.__is_black_ == false);
+
+  assert(c.__parent_ == &root);
+  assert(c.__left_ == &b);
+  assert(c.__right_ == &e);
+  assert(c.__is_black_ == true);
+
+  assert(b.__parent_ == &c);
+  assert(b.__left_ == &a);
+  assert(b.__right_ == 0);
+  assert(b.__is_black_ == true);
+
+  assert(e.__parent_ == &c);
+  assert(e.__left_ == &d);
+  assert(e.__right_ == 0);
+  assert(e.__is_black_ == true);
+
+  assert(a.__parent_ == &b);
+  assert(a.__left_ == 0);
+  assert(a.__right_ == 0);
+  assert(a.__is_black_ == false);
+
+  assert(d.__parent_ == &e);
+  assert(d.__left_ == 0);
+  assert(d.__right_ == 0);
+  assert(d.__is_black_ == false);
+
+  std::__tree_remove(root.__left_, &e);
+
+  assert(std::__tree_invariant(root.__left_));
+
+  assert(root.__parent_ == 0);
+  assert(root.__left_ == &c);
+  assert(root.__right_ == 0);
+  assert(root.__is_black_ == false);
+
+  assert(c.__parent_ == &root);
+  assert(c.__left_ == &b);
+  assert(c.__right_ == &d);
+  assert(c.__is_black_ == true);
+
+  assert(b.__parent_ == &c);
+  assert(b.__left_ == &a);
+  assert(b.__right_ == 0);
+  assert(b.__is_black_ == true);
+
+  assert(a.__parent_ == &b);
+  assert(a.__left_ == 0);
+  assert(a.__right_ == 0);
+  assert(a.__is_black_ == false);
 
-    assert(d.__parent_ == &f);
-    assert(d.__left_ == 0);
-    assert(d.__right_ == &e);
-    assert(d.__is_black_ == true);
+  assert(d.__parent_ == &c);
+  assert(d.__left_ == 0);
+  assert(d.__right_ == 0);
+  assert(d.__is_black_ == true);
 
-    assert(g.__parent_ == &f);
-    assert(g.__left_ == 0);
-    assert(g.__right_ == &h);
-    assert(g.__is_black_ == true);
+  std::__tree_remove(root.__left_, &d);
+
+  assert(std::__tree_invariant(root.__left_));
+
+  assert(root.__parent_ == 0);
+  assert(root.__left_ == &b);
+  assert(root.__right_ == 0);
+  assert(root.__is_black_ == false);
+
+  assert(b.__parent_ == &root);
+  assert(b.__left_ == &a);
+  assert(b.__right_ == &c);
+  assert(b.__is_black_ == true);
+
+  assert(a.__parent_ == &b);
+  assert(a.__left_ == 0);
+  assert(a.__right_ == 0);
+  assert(a.__is_black_ == true);
 
-    assert(e.__parent_ == &d);
-    assert(e.__left_ == 0);
-    assert(e.__right_ == 0);
-    assert(e.__is_black_ == false);
+  assert(c.__parent_ == &b);
+  assert(c.__left_ == 0);
+  assert(c.__right_ == 0);
+  assert(c.__is_black_ == true);
 
-    assert(h.__parent_ == &g);
-    assert(h.__left_ == 0);
-    assert(h.__right_ == 0);
-    assert(h.__is_black_ == false);
+  std::__tree_remove(root.__left_, &c);
 
-    std::__tree_remove(root.__left_, &d);
+  assert(std::__tree_invariant(root.__left_));
 
-    assert(std::__tree_invariant(root.__left_));
+  assert(root.__parent_ == 0);
+  assert(root.__left_ == &b);
+  assert(root.__right_ == 0);
+  assert(root.__is_black_ == false);
 
-    assert(root.__parent_ == 0);
-    assert(root.__left_ == &f);
-    assert(root.__right_ == 0);
-    assert(root.__is_black_ == false);
+  assert(b.__parent_ == &root);
+  assert(b.__left_ == &a);
+  assert(b.__right_ == 0);
+  assert(b.__is_black_ == true);
 
-    assert(f.__parent_ == &root);
-    assert(f.__left_ == &e);
-    assert(f.__right_ == &g);
-    assert(f.__is_black_ == true);
+  assert(a.__parent_ == &b);
+  assert(a.__left_ == 0);
+  assert(a.__right_ == 0);
+  assert(a.__is_black_ == false);
 
-    assert(e.__parent_ == &f);
-    assert(e.__left_ == 0);
-    assert(e.__right_ == 0);
-    assert(e.__is_black_ == true);
+  std::__tree_remove(root.__left_, &b);
 
-    assert(g.__parent_ == &f);
-    assert(g.__left_ == 0);
-    assert(g.__right_ == &h);
-    assert(g.__is_black_ == true);
+  assert(std::__tree_invariant(root.__left_));
 
-    assert(h.__parent_ == &g);
-    assert(h.__left_ == 0);
-    assert(h.__right_ == 0);
-    assert(h.__is_black_ == false);
+  assert(root.__parent_ == 0);
+  assert(root.__left_ == &a);
+  assert(root.__right_ == 0);
+  assert(root.__is_black_ == false);
 
-    std::__tree_remove(root.__left_, &e);
+  assert(a.__parent_ == &root);
+  assert(a.__left_ == 0);
+  assert(a.__right_ == 0);
+  assert(a.__is_black_ == true);
 
-    assert(std::__tree_invariant(root.__left_));
+  std::__tree_remove(root.__left_, &a);
 
-    assert(root.__parent_ == 0);
-    assert(root.__left_ == &g);
-    assert(root.__right_ == 0);
-    assert(root.__is_black_ == false);
+  assert(std::__tree_invariant(root.__left_));
 
-    assert(g.__parent_ == &root);
-    assert(g.__left_ == &f);
-    assert(g.__right_ == &h);
-    assert(g.__is_black_ == true);
+  assert(root.__parent_ == 0);
+  assert(root.__left_ == 0);
+  assert(root.__right_ == 0);
+  assert(root.__is_black_ == false);
+}
 
-    assert(f.__parent_ == &g);
-    assert(f.__left_ == 0);
-    assert(f.__right_ == 0);
-    assert(f.__is_black_ == true);
+void test4() {
+  Node root;
+  Node a;
+  Node b;
+  Node c;
+  Node d;
+  Node e;
+  Node f;
+  Node g;
+  Node h;
+
+  root.__left_ = &d;
+
+  d.__parent_   = &root;
+  d.__left_     = &b;
+  d.__right_    = &f;
+  d.__is_black_ = true;
+
+  b.__parent_   = &d;
+  b.__left_     = &a;
+  b.__right_    = &c;
+  b.__is_black_ = false;
+
+  f.__parent_   = &d;
+  f.__left_     = &e;
+  f.__right_    = &g;
+  f.__is_black_ = false;
+
+  a.__parent_   = &b;
+  a.__left_     = 0;
+  a.__right_    = 0;
+  a.__is_black_ = true;
+
+  c.__parent_   = &b;
+  c.__left_     = 0;
+  c.__right_    = 0;
+  c.__is_black_ = true;
+
+  e.__parent_   = &f;
+  e.__left_     = 0;
+  e.__right_    = 0;
+  e.__is_black_ = true;
+
+  g.__parent_   = &f;
+  g.__left_     = 0;
+  g.__right_    = &h;
+  g.__is_black_ = true;
+
+  h.__parent_   = &g;
+  h.__left_     = 0;
+  h.__right_    = 0;
+  h.__is_black_ = false;
+
+  assert(std::__tree_invariant(root.__left_));
+
+  std::__tree_remove(root.__left_, &a);
+
+  assert(std::__tree_invariant(root.__left_));
+
+  assert(root.__parent_ == 0);
+  assert(root.__left_ == &d);
+  assert(root.__right_ == 0);
+  assert(root.__is_black_ == false);
+
+  assert(d.__parent_ == &root);
+  assert(d.__left_ == &b);
+  assert(d.__right_ == &f);
+  assert(d.__is_black_ == true);
+
+  assert(b.__parent_ == &d);
+  assert(b.__left_ == 0);
+  assert(b.__right_ == &c);
+  assert(b.__is_black_ == true);
+
+  assert(f.__parent_ == &d);
+  assert(f.__left_ == &e);
+  assert(f.__right_ == &g);
+  assert(f.__is_black_ == false);
+
+  assert(c.__parent_ == &b);
+  assert(c.__left_ == 0);
+  assert(c.__right_ == 0);
+  assert(c.__is_black_ == false);
+
+  assert(e.__parent_ == &f);
+  assert(e.__left_ == 0);
+  assert(e.__right_ == 0);
+  assert(e.__is_black_ == true);
+
+  assert(g.__parent_ == &f);
+  assert(g.__left_ == 0);
+  assert(g.__right_ == &h);
+  assert(g.__is_black_ == true);
+
+  assert(h.__parent_ == &g);
+  assert(h.__left_ == 0);
+  assert(h.__right_ == 0);
+  assert(h.__is_black_ == false);
+
+  std::__tree_remove(root.__left_, &b);
+
+  assert(std::__tree_invariant(root.__left_));
+
+  assert(root.__parent_ == 0);
+  assert(root.__left_ == &d);
+  assert(root.__right_ == 0);
+  assert(root.__is_black_ == false);
+
+  assert(d.__parent_ == &root);
+  assert(d.__left_ == &c);
+  assert(d.__right_ == &f);
+  assert(d.__is_black_ == true);
+
+  assert(c.__parent_ == &d);
+  assert(c.__left_ == 0);
+  assert(c.__right_ == 0);
+  assert(c.__is_black_ == true);
+
+  assert(f.__parent_ == &d);
+  assert(f.__left_ == &e);
+  assert(f.__right_ == &g);
+  assert(f.__is_black_ == false);
+
+  assert(e.__parent_ == &f);
+  assert(e.__left_ == 0);
+  assert(e.__right_ == 0);
+  assert(e.__is_black_ == true);
+
+  assert(g.__parent_ == &f);
+  assert(g.__left_ == 0);
+  assert(g.__right_ == &h);
+  assert(g.__is_black_ == true);
+
+  assert(h.__parent_ == &g);
+  assert(h.__left_ == 0);
+  assert(h.__right_ == 0);
+  assert(h.__is_black_ == false);
+
+  std::__tree_remove(root.__left_, &c);
+
+  assert(std::__tree_invariant(root.__left_));
+
+  assert(root.__parent_ == 0);
+  assert(root.__left_ == &f);
+  assert(root.__right_ == 0);
+  assert(root.__is_black_ == false);
+
+  assert(f.__parent_ == &root);
+  assert(f.__left_ == &d);
+  assert(f.__right_ == &g);
+  assert(f.__is_black_ == true);
+
+  assert(d.__parent_ == &f);
+  assert(d.__left_ == 0);
+  assert(d.__right_ == &e);
+  assert(d.__is_black_ == true);
+
+  assert(g.__parent_ == &f);
+  assert(g.__left_ == 0);
+  assert(g.__right_ == &h);
+  assert(g.__is_black_ == true);
+
+  assert(e.__parent_ == &d);
+  assert(e.__left_ == 0);
+  assert(e.__right_ == 0);
+  assert(e.__is_black_ == false);
+
+  assert(h.__parent_ == &g);
+  assert(h.__left_ == 0);
+  assert(h.__right_ == 0);
+  assert(h.__is_black_ == false);
+
+  std::__tree_remove(root.__left_, &d);
+
+  assert(std::__tree_invariant(root.__left_));
+
+  assert(root.__parent_ == 0);
+  assert(root.__left_ == &f);
+  assert(root.__right_ == 0);
+  assert(root.__is_black_ == false);
+
+  assert(f.__parent_ == &root);
+  assert(f.__left_ == &e);
+  assert(f.__right_ == &g);
+  assert(f.__is_black_ == true);
+
+  assert(e.__parent_ == &f);
+  assert(e.__left_ == 0);
+  assert(e.__right_ == 0);
+  assert(e.__is_black_ == true);
+
+  assert(g.__parent_ == &f);
+  assert(g.__left_ == 0);
+  assert(g.__right_ == &h);
+  assert(g.__is_black_ == true);
 
-    assert(h.__parent_ == &g);
-    assert(h.__left_ == 0);
-    assert(h.__right_ == 0);
-    assert(h.__is_black_ == true);
+  assert(h.__parent_ == &g);
+  assert(h.__left_ == 0);
+  assert(h.__right_ == 0);
+  assert(h.__is_black_ == false);
 
-    std::__tree_remove(root.__left_, &f);
+  std::__tree_remove(root.__left_, &e);
+
+  assert(std::__tree_invariant(root.__left_));
+
+  assert(root.__parent_ == 0);
+  assert(root.__left_ == &g);
+  assert(root.__right_ == 0);
+  assert(root.__is_black_ == false);
+
+  assert(g.__parent_ == &root);
+  assert(g.__left_ == &f);
+  assert(g.__right_ == &h);
+  assert(g.__is_black_ == true);
+
+  assert(f.__parent_ == &g);
+  assert(f.__left_ == 0);
+  assert(f.__right_ == 0);
+  assert(f.__is_black_ == true);
 
-    assert(std::__tree_invariant(root.__left_));
+  assert(h.__parent_ == &g);
+  assert(h.__left_ == 0);
+  assert(h.__right_ == 0);
+  assert(h.__is_black_ == true);
 
-    assert(root.__parent_ == 0);
-    assert(root.__left_ == &g);
-    assert(root.__right_ == 0);
-    assert(root.__is_black_ == false);
+  std::__tree_remove(root.__left_, &f);
 
-    assert(g.__parent_ == &root);
-    assert(g.__left_ == 0);
-    assert(g.__right_ == &h);
-    assert(g.__is_black_ == true);
+  assert(std::__tree_invariant(root.__left_));
 
-    assert(h.__parent_ == &g);
-    assert(h.__left_ == 0);
-    assert(h.__right_ == 0);
-    assert(h.__is_black_ == false);
+  assert(root.__parent_ == 0);
+  assert(root.__left_ == &g);
+  assert(root.__right_ == 0);
+  assert(root.__is_black_ == false);
 
-    std::__tree_remove(root.__left_, &g);
+  assert(g.__parent_ == &root);
+  assert(g.__left_ == 0);
+  assert(g.__right_ == &h);
+  assert(g.__is_black_ == true);
 
-    assert(std::__tree_invariant(root.__left_));
+  assert(h.__parent_ == &g);
+  assert(h.__left_ == 0);
+  assert(h.__right_ == 0);
+  assert(h.__is_black_ == false);
 
-    assert(root.__parent_ == 0);
-    assert(root.__left_ == &h);
-    assert(root.__right_ == 0);
-    assert(root.__is_black_ == false);
+  std::__tree_remove(root.__left_, &g);
 
-    assert(h.__parent_ == &root);
-    assert(h.__left_ == 0);
-    assert(h.__right_ == 0);
-    assert(h.__is_black_ == true);
+  assert(std::__tree_invariant(root.__left_));
 
-    std::__tree_remove(root.__left_, &h);
+  assert(root.__parent_ == 0);
+  assert(root.__left_ == &h);
+  assert(root.__right_ == 0);
+  assert(root.__is_black_ == false);
 
-    assert(std::__tree_invariant(root.__left_));
+  assert(h.__parent_ == &root);
+  assert(h.__left_ == 0);
+  assert(h.__right_ == 0);
+  assert(h.__is_black_ == true);
 
-    assert(root.__parent_ == 0);
-    assert(root.__left_ == 0);
-    assert(root.__right_ == 0);
-    assert(root.__is_black_ == false);
+  std::__tree_remove(root.__left_, &h);
+
+  assert(std::__tree_invariant(root.__left_));
+
+  assert(root.__parent_ == 0);
+  assert(root.__left_ == 0);
+  assert(root.__right_ == 0);
+  assert(root.__is_black_ == false);
 }
 
-int main(int, char**)
-{
-    test1();
-    test2();
-    test3();
-    test4();
+int main(int, char**) {
+  test1();
+  test2();
+  test3();
+  test4();
 
   return 0;
 }
diff --git a/libcxx/test/libcxx/containers/associative/tree_right_rotate.pass.cpp b/libcxx/test/libcxx/containers/associative/tree_right_rotate.pass.cpp
index 16ca81fddc7dc..b86446f5be101 100644
--- a/libcxx/test/libcxx/containers/associative/tree_right_rotate.pass.cpp
+++ b/libcxx/test/libcxx/containers/associative/tree_right_rotate.pass.cpp
@@ -20,87 +20,81 @@
 
 #include "test_macros.h"
 
-struct Node
-{
-    Node* __left_;
-    Node* __right_;
-    Node* __parent_;
+struct Node {
+  Node* __left_;
+  Node* __right_;
+  Node* __parent_;
 
-    Node* __parent_unsafe() const { return __parent_; }
-    void __set_parent(Node* x) { __parent_ = x;}
+  Node* __parent_unsafe() const { return __parent_; }
+  void __set_parent(Node* x) { __parent_ = x; }
 
-    Node() : __left_(), __right_(), __parent_() {}
+  Node() : __left_(), __right_(), __parent_() {}
 };
 
-void
-test1()
-{
-    Node root;
-    Node x;
-    Node y;
-    root.__left_ = &x;
-    x.__left_ = &y;
-    x.__right_ = 0;
-    x.__parent_ = &root;
-    y.__left_ = 0;
-    y.__right_ = 0;
-    y.__parent_ = &x;
-    std::__tree_right_rotate(&x);
-    assert(root.__parent_ == 0);
-    assert(root.__left_ == &y);
-    assert(root.__right_ == 0);
-    assert(y.__parent_ == &root);
-    assert(y.__left_ == 0);
-    assert(y.__right_ == &x);
-    assert(x.__parent_ == &y);
-    assert(x.__left_ == 0);
-    assert(x.__right_ == 0);
+void test1() {
+  Node root;
+  Node x;
+  Node y;
+  root.__left_ = &x;
+  x.__left_    = &y;
+  x.__right_   = 0;
+  x.__parent_  = &root;
+  y.__left_    = 0;
+  y.__right_   = 0;
+  y.__parent_  = &x;
+  std::__tree_right_rotate(&x);
+  assert(root.__parent_ == 0);
+  assert(root.__left_ == &y);
+  assert(root.__right_ == 0);
+  assert(y.__parent_ == &root);
+  assert(y.__left_ == 0);
+  assert(y.__right_ == &x);
+  assert(x.__parent_ == &y);
+  assert(x.__left_ == 0);
+  assert(x.__right_ == 0);
 }
 
-void
-test2()
-{
-    Node root;
-    Node x;
-    Node y;
-    Node a;
-    Node b;
-    Node c;
-    root.__left_ = &x;
-    x.__left_ = &y;
-    x.__right_ = &c;
-    x.__parent_ = &root;
-    y.__left_ = &a;
-    y.__right_ = &b;
-    y.__parent_ = &x;
-    a.__parent_ = &y;
-    b.__parent_ = &y;
-    c.__parent_ = &x;
-    std::__tree_right_rotate(&x);
-    assert(root.__parent_ == 0);
-    assert(root.__left_ == &y);
-    assert(root.__right_ == 0);
-    assert(y.__parent_ == &root);
-    assert(y.__left_ == &a);
-    assert(y.__right_ == &x);
-    assert(x.__parent_ == &y);
-    assert(x.__left_ == &b);
-    assert(x.__right_ == &c);
-    assert(a.__parent_ == &y);
-    assert(a.__left_ == 0);
-    assert(a.__right_ == 0);
-    assert(b.__parent_ == &x);
-    assert(b.__left_ == 0);
-    assert(b.__right_ == 0);
-    assert(c.__parent_ == &x);
-    assert(c.__left_ == 0);
-    assert(c.__right_ == 0);
+void test2() {
+  Node root;
+  Node x;
+  Node y;
+  Node a;
+  Node b;
+  Node c;
+  root.__left_ = &x;
+  x.__left_    = &y;
+  x.__right_   = &c;
+  x.__parent_  = &root;
+  y.__left_    = &a;
+  y.__right_   = &b;
+  y.__parent_  = &x;
+  a.__parent_  = &y;
+  b.__parent_  = &y;
+  c.__parent_  = &x;
+  std::__tree_right_rotate(&x);
+  assert(root.__parent_ == 0);
+  assert(root.__left_ == &y);
+  assert(root.__right_ == 0);
+  assert(y.__parent_ == &root);
+  assert(y.__left_ == &a);
+  assert(y.__right_ == &x);
+  assert(x.__parent_ == &y);
+  assert(x.__left_ == &b);
+  assert(x.__right_ == &c);
+  assert(a.__parent_ == &y);
+  assert(a.__left_ == 0);
+  assert(a.__right_ == 0);
+  assert(b.__parent_ == &x);
+  assert(b.__left_ == 0);
+  assert(b.__right_ == 0);
+  assert(c.__parent_ == &x);
+  assert(c.__left_ == 0);
+  assert(c.__right_ == 0);
 }
 
-int main(int, char**)
-{
-    test1();
-    test2();
+int main(int, char**) {
+  test1();
+  test2();
 
   return 0;
 }
diff --git a/libcxx/test/libcxx/containers/containers.adaptors/flat.map/assert.input_range.pass.cpp b/libcxx/test/libcxx/containers/container.adaptors/flat.map/assert.input_range.pass.cpp
similarity index 100%
rename from libcxx/test/libcxx/containers/containers.adaptors/flat.map/assert.input_range.pass.cpp
rename to libcxx/test/libcxx/containers/container.adaptors/flat.map/assert.input_range.pass.cpp
diff --git a/libcxx/test/libcxx/containers/containers.adaptors/flat.map/assert.sorted_unique.pass.cpp b/libcxx/test/libcxx/containers/container.adaptors/flat.map/assert.sorted_unique.pass.cpp
similarity index 100%
rename from libcxx/test/libcxx/containers/containers.adaptors/flat.map/assert.sorted_unique.pass.cpp
rename to libcxx/test/libcxx/containers/container.adaptors/flat.map/assert.sorted_unique.pass.cpp
diff --git a/libcxx/test/libcxx/containers/container.adaptors/flat.multimap/assert.input_range.pass.cpp b/libcxx/test/libcxx/containers/container.adaptors/flat.multimap/assert.input_range.pass.cpp
new file mode 100644
index 0000000000000..504f36fcd00b8
--- /dev/null
+++ b/libcxx/test/libcxx/containers/container.adaptors/flat.multimap/assert.input_range.pass.cpp
@@ -0,0 +1,66 @@
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// REQUIRES: has-unix-headers
+// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
+// UNSUPPORTED: libcpp-hardening-mode=none
+// XFAIL: libcpp-hardening-mode=debug && availability-verbose_abort-missing
+
+// <flat_map>
+
+// flat_multimap(key_container_type , mapped_container_type , const key_compare& __comp = key_compare())
+// flat_multimap(const key_container_type& , const mapped_container_type& , const _Allocator& )
+// flat_multimap(const key_container_type& , const mapped_container_type& , const key_compare&, const _Allocator& )
+// void replace(key_container_type&& , mapped_container_type&&)
+//
+
+#include <flat_map>
+#include <functional>
+#include <memory>
+#include <vector>
+
+#include "check_assertion.h"
+
+int main(int, char**) {
+  using M = std::flat_multimap<int, int>;
+
+  TEST_LIBCPP_ASSERT_FAILURE(
+      ([] { M m({1, 2, 3}, {4}); }()), "flat_multimap keys and mapped containers have different size");
+
+  TEST_LIBCPP_ASSERT_FAILURE(([] { M m({1, 2, 3}, {4}, std::less<int>{}); }()),
+                             "flat_multimap keys and mapped containers have different size");
+
+  TEST_LIBCPP_ASSERT_FAILURE(
+      ([] {
+        const std::vector keys{1, 2, 3};
+        const std::vector values{4};
+        const std::allocator<int> alloc{};
+        M m(keys, values, alloc);
+      }()),
+      "flat_multimap keys and mapped containers have different size");
+
+  TEST_LIBCPP_ASSERT_FAILURE(
+      ([] {
+        const std::vector keys{1, 2, 3};
+        const std::vector values{4};
+        const std::less<int> key_compare{};
+        const std::allocator<int> alloc{};
+        M m(keys, values, key_compare, alloc);
+      }()),
+      "flat_multimap keys and mapped containers have different size");
+
+  TEST_LIBCPP_ASSERT_FAILURE(
+      ([] {
+        std::vector keys{1, 2, 3};
+        std::vector values{4};
+        M m;
+        m.replace(std::move(keys), std::move(values));
+      }()),
+      "flat_multimap keys and mapped containers have different size");
+
+  return 0;
+}
diff --git a/libcxx/test/libcxx/containers/container.adaptors/flat.multimap/assert.sorted_equivalent.pass.cpp b/libcxx/test/libcxx/containers/container.adaptors/flat.multimap/assert.sorted_equivalent.pass.cpp
new file mode 100644
index 0000000000000..6b8ad3c7ac9aa
--- /dev/null
+++ b/libcxx/test/libcxx/containers/container.adaptors/flat.multimap/assert.sorted_equivalent.pass.cpp
@@ -0,0 +1,225 @@
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// REQUIRES: has-unix-headers
+// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
+// UNSUPPORTED: libcpp-hardening-mode=none
+// REQUIRES: libcpp-hardening-mode=debug
+// XFAIL: libcpp-hardening-mode=debug && availability-verbose_abort-missing
+
+// <flat_map>
+
+// flat_multimap(key_container_type , mapped_container_type , const key_compare& __comp = key_compare())
+// flat_multimap(const key_container_type& , const mapped_container_type& , const _Allocator& )
+// flat_multimap(const key_container_type& , const mapped_container_type& , const key_compare&, const _Allocator& )
+// void replace(key_container_type&& , mapped_container_type&&)
+//
+
+#include <flat_map>
+#include <functional>
+#include <initializer_list>
+#include <memory>
+#include <utility>
+#include <vector>
+
+#include "check_assertion.h"
+
+int main(int, char**) {
+  using M = std::flat_multimap<int, int>;
+
+  TEST_LIBCPP_ASSERT_FAILURE(
+      ([] { M m(std::sorted_equivalent, {2, 2, 1}, {4, 5, 6}); }()), "Key container is not sorted");
+
+  TEST_LIBCPP_ASSERT_FAILURE(
+      ([] { M m(std::sorted_equivalent, {4, 2, 3}, {4, 5, 6}); }()), "Key container is not sorted");
+
+  TEST_LIBCPP_ASSERT_FAILURE(
+      ([] { M m(std::sorted_equivalent, {2, 2, 1}, {4, 5, 6}, std::less<int>{}); }()), "Key container is not sorted");
+
+  TEST_LIBCPP_ASSERT_FAILURE(
+      ([] { M m(std::sorted_equivalent, {4, 2, 3}, {4, 5, 6}, std::less<int>{}); }()), "Key container is not sorted");
+
+  TEST_LIBCPP_ASSERT_FAILURE(
+      ([] {
+        const std::vector keys{2, 2, 1};
+        const std::vector values{4, 5, 6};
+        const std::allocator<int> alloc{};
+        M m(std::sorted_equivalent, keys, values, alloc);
+      }()),
+      "Key container is not sorted");
+
+  TEST_LIBCPP_ASSERT_FAILURE(
+      ([] {
+        const std::vector keys{4, 2, 3};
+        const std::vector values{4, 5, 6};
+        const std::allocator<int> alloc{};
+        M m(std::sorted_equivalent, keys, values, alloc);
+      }()),
+      "Key container is not sorted");
+
+  TEST_LIBCPP_ASSERT_FAILURE(
+      ([] {
+        const std::vector keys{2, 2, 1};
+        const std::vector values{4, 5, 6};
+        const std::allocator<int> alloc{};
+        const std::less<int> comp{};
+        M m(std::sorted_equivalent, keys, values, comp, alloc);
+      }()),
+      "Key container is not sorted");
+
+  TEST_LIBCPP_ASSERT_FAILURE(
+      ([] {
+        const std::vector keys{4, 2, 3};
+        const std::vector values{4, 5, 6};
+        const std::allocator<int> alloc{};
+        const std::less<int> comp{};
+        M m(std::sorted_equivalent, keys, values, comp, alloc);
+      }()),
+      "Key container is not sorted");
+
+  TEST_LIBCPP_ASSERT_FAILURE(
+      ([] {
+        const std::vector<std::pair<int, int>> v{{2, 4}, {2, 5}, {1, 6}};
+        const std::less<int> comp{};
+        M m(std::sorted_equivalent, v.begin(), v.end(), comp);
+      }()),
+      "Key container is not sorted");
+
+  TEST_LIBCPP_ASSERT_FAILURE(
+      ([] {
+        const std::vector<std::pair<int, int>> v{{4, 4}, {2, 5}, {3, 6}};
+        const std::less<int> comp{};
+        M m(std::sorted_equivalent, v.begin(), v.end(), comp);
+      }()),
+      "Key container is not sorted");
+
+  TEST_LIBCPP_ASSERT_FAILURE(
+      ([] {
+        const std::vector<std::pair<int, int>> v{{2, 4}, {2, 5}, {1, 6}};
+        const std::less<int> comp{};
+        const std::allocator<int> alloc{};
+        M m(std::sorted_equivalent, v.begin(), v.end(), comp, alloc);
+      }()),
+      "Key container is not sorted");
+
+  TEST_LIBCPP_ASSERT_FAILURE(
+      ([] {
+        const std::vector<std::pair<int, int>> v{{4, 4}, {2, 5}, {3, 6}};
+        const std::less<int> comp{};
+        const std::allocator<int> alloc{};
+        M m(std::sorted_equivalent, v.begin(), v.end(), comp, alloc);
+      }()),
+      "Key container is not sorted");
+
+  TEST_LIBCPP_ASSERT_FAILURE(
+      ([] {
+        const std::vector<std::pair<int, int>> v{{2, 4}, {2, 5}, {1, 6}};
+        const std::allocator<int> alloc{};
+        M m(std::sorted_equivalent, v.begin(), v.end(), alloc);
+      }()),
+      "Key container is not sorted");
+
+  TEST_LIBCPP_ASSERT_FAILURE(
+      ([] {
+        const std::vector<std::pair<int, int>> v{{4, 4}, {2, 5}, {3, 6}};
+        const std::allocator<int> alloc{};
+        M m(std::sorted_equivalent, v.begin(), v.end(), alloc);
+      }()),
+      "Key container is not sorted");
+
+  TEST_LIBCPP_ASSERT_FAILURE(
+      ([] {
+        std::initializer_list<std::pair<int, int>> v{{2, 4}, {2, 5}, {1, 6}};
+        const std::less<int> comp{};
+        M m(std::sorted_equivalent, v, comp);
+      }()),
+      "Key container is not sorted");
+
+  TEST_LIBCPP_ASSERT_FAILURE(
+      ([] {
+        std::initializer_list<std::pair<int, int>> v{{4, 4}, {2, 5}, {3, 6}};
+        const std::less<int> comp{};
+        M m(std::sorted_equivalent, v, comp);
+      }()),
+      "Key container is not sorted");
+
+  TEST_LIBCPP_ASSERT_FAILURE(
+      ([] {
+        std::initializer_list<std::pair<int, int>> v{{2, 4}, {2, 5}, {1, 6}};
+        const std::less<int> comp{};
+        const std::allocator<int> alloc{};
+        M m(std::sorted_equivalent, v, comp, alloc);
+      }()),
+      "Key container is not sorted");
+
+  TEST_LIBCPP_ASSERT_FAILURE(
+      ([] {
+        std::initializer_list<std::pair<int, int>> v{{4, 4}, {2, 5}, {3, 6}};
+        const std::less<int> comp{};
+        const std::allocator<int> alloc{};
+        M m(std::sorted_equivalent, v, comp, alloc);
+      }()),
+      "Key container is not sorted");
+
+  TEST_LIBCPP_ASSERT_FAILURE(
+      ([] {
+        std::initializer_list<std::pair<int, int>> v{{2, 4}, {2, 5}, {1, 6}};
+        const std::allocator<int> alloc{};
+        M m(std::sorted_equivalent, v, alloc);
+      }()),
+      "Key container is not sorted");
+
+  TEST_LIBCPP_ASSERT_FAILURE(
+      ([] {
+        std::initializer_list<std::pair<int, int>> v{{4, 4}, {2, 5}, {3, 6}};
+        const std::allocator<int> alloc{};
+        M m(std::sorted_equivalent, v, alloc);
+      }()),
+      "Key container is not sorted");
+
+  TEST_LIBCPP_ASSERT_FAILURE(
+      ([] {
+        const std::vector<std::pair<int, int>> v{{2, 4}, {2, 5}, {1, 6}};
+        M m;
+        m.insert(std::sorted_equivalent, v.begin(), v.end());
+      }()),
+      "Key container is not sorted");
+
+  TEST_LIBCPP_ASSERT_FAILURE(
+      ([] {
+        const std::vector<std::pair<int, int>> v{{4, 4}, {2, 5}, {3, 6}};
+        M m;
+        m.insert(std::sorted_equivalent, v.begin(), v.end());
+      }()),
+      "Key container is not sorted");
+
+  TEST_LIBCPP_ASSERT_FAILURE(
+      ([] {
+        std::initializer_list<std::pair<int, int>> v{{2, 4}, {2, 5}, {1, 6}};
+        M m;
+        m.insert(std::sorted_equivalent, v);
+      }()),
+      "Key container is not sorted");
+
+  TEST_LIBCPP_ASSERT_FAILURE(
+      ([] {
+        std::initializer_list<std::pair<int, int>> v{{4, 4}, {2, 5}, {3, 6}};
+        M m;
+        m.insert(std::sorted_equivalent, v);
+      }()),
+      "Key container is not sorted");
+
+  TEST_LIBCPP_ASSERT_FAILURE(
+      ([] {
+        std::vector keys{2, 1, 3};
+        std::vector values{4, 5, 6};
+        M m;
+        m.replace(std::move(keys), std::move(values));
+      }()),
+      "Key container is not sorted");
+  return 0;
+}
diff --git a/libcxx/test/libcxx/containers/gnu_cxx/hash_map_name_lookup.pass.cpp b/libcxx/test/libcxx/containers/gnu_cxx/hash_map_name_lookup.pass.cpp
index 50d35f3dd8b80..7e87330ef855d 100644
--- a/libcxx/test/libcxx/containers/gnu_cxx/hash_map_name_lookup.pass.cpp
+++ b/libcxx/test/libcxx/containers/gnu_cxx/hash_map_name_lookup.pass.cpp
@@ -22,7 +22,6 @@ struct unique_ptr;
 
 #include "test_macros.h"
 
-
 namespace __gnu_cxx {
 template class hash_map<int, int>;
 }
diff --git a/libcxx/test/libcxx/containers/sequences/array/triviality.pass.cpp b/libcxx/test/libcxx/containers/sequences/array/triviality.pass.cpp
index d124e84771657..8b0f5a391eb32 100644
--- a/libcxx/test/libcxx/containers/sequences/array/triviality.pass.cpp
+++ b/libcxx/test/libcxx/containers/sequences/array/triviality.pass.cpp
@@ -13,42 +13,39 @@
 #include <array>
 #include <type_traits>
 
-
-struct Empty { };
+struct Empty {};
 
 struct TrivialCopy {
-    int i;
-    double j;
+  int i;
+  double j;
 };
 
 struct NonTrivialCopy {
-    NonTrivialCopy(NonTrivialCopy const&) { }
-    NonTrivialCopy& operator=(NonTrivialCopy const&) { return *this; }
+  NonTrivialCopy(NonTrivialCopy const&) {}
+  NonTrivialCopy& operator=(NonTrivialCopy const&) { return *this; }
 };
 
 template <typename T>
-void check_trivially_copyable()
-{
-    static_assert(std::is_trivially_copyable<std::array<T, 0> >::value, "");
-    static_assert(std::is_trivially_copyable<std::array<T, 1> >::value, "");
-    static_assert(std::is_trivially_copyable<std::array<T, 2> >::value, "");
-    static_assert(std::is_trivially_copyable<std::array<T, 3> >::value, "");
+void check_trivially_copyable() {
+  static_assert(std::is_trivially_copyable<std::array<T, 0> >::value, "");
+  static_assert(std::is_trivially_copyable<std::array<T, 1> >::value, "");
+  static_assert(std::is_trivially_copyable<std::array<T, 2> >::value, "");
+  static_assert(std::is_trivially_copyable<std::array<T, 3> >::value, "");
 }
 
-int main(int, char**)
-{
-    check_trivially_copyable<int>();
-    check_trivially_copyable<long>();
-    check_trivially_copyable<double>();
-    check_trivially_copyable<long double>();
-    check_trivially_copyable<Empty>();
-    check_trivially_copyable<TrivialCopy>();
-
-    // Check that std::array<T, 0> is still trivially copyable when T is not
-    static_assert( std::is_trivially_copyable<std::array<NonTrivialCopy, 0> >::value, "");
-    static_assert(!std::is_trivially_copyable<std::array<NonTrivialCopy, 1> >::value, "");
-    static_assert(!std::is_trivially_copyable<std::array<NonTrivialCopy, 2> >::value, "");
-    static_assert(!std::is_trivially_copyable<std::array<NonTrivialCopy, 3> >::value, "");
-
-    return 0;
+int main(int, char**) {
+  check_trivially_copyable<int>();
+  check_trivially_copyable<long>();
+  check_trivially_copyable<double>();
+  check_trivially_copyable<long double>();
+  check_trivially_copyable<Empty>();
+  check_trivially_copyable<TrivialCopy>();
+
+  // Check that std::array<T, 0> is still trivially copyable when T is not
+  static_assert(std::is_trivially_copyable<std::array<NonTrivialCopy, 0> >::value, "");
+  static_assert(!std::is_trivially_copyable<std::array<NonTrivialCopy, 1> >::value, "");
+  static_assert(!std::is_trivially_copyable<std::array<NonTrivialCopy, 2> >::value, "");
+  static_assert(!std::is_trivially_copyable<std::array<NonTrivialCopy, 3> >::value, "");
+
+  return 0;
 }
diff --git a/libcxx/test/libcxx/containers/sequences/deque/asan.pass.cpp b/libcxx/test/libcxx/containers/sequences/deque/asan.pass.cpp
index e8091acefc368..46ca62dda7b20 100644
--- a/libcxx/test/libcxx/containers/sequences/deque/asan.pass.cpp
+++ b/libcxx/test/libcxx/containers/sequences/deque/asan.pass.cpp
@@ -23,49 +23,46 @@
 
 extern "C" void __sanitizer_set_death_callback(void (*callback)(void));
 
-void do_exit() {
-  exit(0);
-}
+void do_exit() { exit(0); }
 
-int main(int, char**)
-{
-    {
-        typedef cpp17_input_iterator<int*> MyInputIter;
-        // Should not trigger ASan.
-        std::deque<int> v;
-        int i[] = {42};
-        v.insert(v.begin(), MyInputIter(i), MyInputIter(i + 1));
-        assert(v[0] == 42);
-        assert(is_double_ended_contiguous_container_asan_correct(v));
-    }
-    {
-        typedef int T;
-        typedef std::deque<T, min_allocator<T> > C;
-        const T t[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9};
-        C c(std::begin(t), std::end(t));
-        assert(is_double_ended_contiguous_container_asan_correct(c));
-    }
-    {
-        typedef char T;
-        typedef std::deque<T, safe_allocator<T> > C;
-        const T t[] = {'0', '1', '2', '3', '4', '5', '6', '7', '8', '9'};
-        C c(std::begin(t), std::end(t));
-        c.pop_front();
-        assert(is_double_ended_contiguous_container_asan_correct(c));
-    }
-    __sanitizer_set_death_callback(do_exit);
-    {
-        typedef int T;
-        typedef std::deque<T> C;
-        const T t[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9};
-        C c(std::begin(t), std::end(t));
-        assert(is_double_ended_contiguous_container_asan_correct(c));
-        T* ptr = &c[0];
-        for(size_t i = 0; i < (8 + sizeof(T) - 1)/sizeof(T); ++i)
-          c.pop_front();
-        *ptr = 1;
-        volatile T foo = c[c.size()]; // should trigger ASAN. Use volatile to prevent being optimized away.
-        assert(false);          // if we got here, ASAN didn't trigger
-        ((void)foo);
-    }
+int main(int, char**) {
+  {
+    typedef cpp17_input_iterator<int*> MyInputIter;
+    // Should not trigger ASan.
+    std::deque<int> v;
+    int i[] = {42};
+    v.insert(v.begin(), MyInputIter(i), MyInputIter(i + 1));
+    assert(v[0] == 42);
+    assert(is_double_ended_contiguous_container_asan_correct(v));
+  }
+  {
+    typedef int T;
+    typedef std::deque<T, min_allocator<T> > C;
+    const T t[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9};
+    C c(std::begin(t), std::end(t));
+    assert(is_double_ended_contiguous_container_asan_correct(c));
+  }
+  {
+    typedef char T;
+    typedef std::deque<T, safe_allocator<T> > C;
+    const T t[] = {'0', '1', '2', '3', '4', '5', '6', '7', '8', '9'};
+    C c(std::begin(t), std::end(t));
+    c.pop_front();
+    assert(is_double_ended_contiguous_container_asan_correct(c));
+  }
+  __sanitizer_set_death_callback(do_exit);
+  {
+    typedef int T;
+    typedef std::deque<T> C;
+    const T t[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9};
+    C c(std::begin(t), std::end(t));
+    assert(is_double_ended_contiguous_container_asan_correct(c));
+    T* ptr = &c[0];
+    for (size_t i = 0; i < (8 + sizeof(T) - 1) / sizeof(T); ++i)
+      c.pop_front();
+    *ptr           = 1;
+    volatile T foo = c[c.size()]; // should trigger ASAN. Use volatile to prevent being optimized away.
+    assert(false);                // if we got here, ASAN didn't trigger
+    ((void)foo);
+  }
 }
diff --git a/libcxx/test/libcxx/containers/sequences/deque/assert.pop_back.empty.pass.cpp b/libcxx/test/libcxx/containers/sequences/deque/assert.pop_back.empty.pass.cpp
index 19cd41f4cf3a2..6bdb117485609 100644
--- a/libcxx/test/libcxx/containers/sequences/deque/assert.pop_back.empty.pass.cpp
+++ b/libcxx/test/libcxx/containers/sequences/deque/assert.pop_back.empty.pass.cpp
@@ -20,10 +20,10 @@
 #include "check_assertion.h"
 
 int main(int, char**) {
-    std::deque<int> q;
-    q.push_back(0);
-    q.pop_back();
-    TEST_LIBCPP_ASSERT_FAILURE(q.pop_back(), "deque::pop_back called on an empty deque");
+  std::deque<int> q;
+  q.push_back(0);
+  q.pop_back();
+  TEST_LIBCPP_ASSERT_FAILURE(q.pop_back(), "deque::pop_back called on an empty deque");
 
-    return 0;
+  return 0;
 }
diff --git a/libcxx/test/libcxx/containers/sequences/deque/incomplete.pass.cpp b/libcxx/test/libcxx/containers/sequences/deque/incomplete.pass.cpp
index 004a7c8b16006..ebf5869078331 100644
--- a/libcxx/test/libcxx/containers/sequences/deque/incomplete.pass.cpp
+++ b/libcxx/test/libcxx/containers/sequences/deque/incomplete.pass.cpp
@@ -24,11 +24,10 @@ struct A {
   std::deque<A>::reverse_iterator it2;
 };
 
-int main(int, char**)
-{
+int main(int, char**) {
   A a;
   assert(a.d.size() == 0);
-  a.it = a.d.begin();
+  a.it  = a.d.begin();
   a.it2 = a.d.rend();
 
   return 0;
diff --git a/libcxx/test/libcxx/containers/sequences/deque/segmented_iterator.compile.pass.cpp b/libcxx/test/libcxx/containers/sequences/deque/segmented_iterator.compile.pass.cpp
index 27100f9b45e3e..7ae67d8d0252f 100644
--- a/libcxx/test/libcxx/containers/sequences/deque/segmented_iterator.compile.pass.cpp
+++ b/libcxx/test/libcxx/containers/sequences/deque/segmented_iterator.compile.pass.cpp
@@ -8,6 +8,5 @@
 
 #include <deque>
 
-
 using DequeIterator = typename std::deque<int>::iterator;
 static_assert(std::__is_segmented_iterator<DequeIterator>::value, "");
diff --git a/libcxx/test/libcxx/containers/sequences/deque/spare_block_handling.pass.cpp b/libcxx/test/libcxx/containers/sequences/deque/spare_block_handling.pass.cpp
index e9e003432301e..4fa4982de88fa 100644
--- a/libcxx/test/libcxx/containers/sequences/deque/spare_block_handling.pass.cpp
+++ b/libcxx/test/libcxx/containers/sequences/deque/spare_block_handling.pass.cpp
@@ -33,12 +33,16 @@ struct ContainerAdaptor : public Adaptor {
 
 template <class Deque>
 static void print(const Deque& d) {
-  std::printf("%zu : __front_spare() == %zu"
-              " : __back_spare() == %zu"
-              " : __capacity() == %zu"
-              " : bytes allocated == %zu\n",
-              d.size(), d.__front_spare(), d.__back_spare(), d.__capacity(),
-              malloc_allocator_base::outstanding_bytes);
+  std::printf(
+      "%zu : __front_spare() == %zu"
+      " : __back_spare() == %zu"
+      " : __capacity() == %zu"
+      " : bytes allocated == %zu\n",
+      d.size(),
+      d.__front_spare(),
+      d.__back_spare(),
+      d.__capacity(),
+      malloc_allocator_base::outstanding_bytes);
 }
 
 template <class T>
@@ -48,7 +52,7 @@ template <class T>
 using BlockSize = std::__deque_block_size<T, std::ptrdiff_t>;
 
 struct LargeT {
-  LargeT() = default;
+  LargeT()       = default;
   char buff[256] = {};
 };
 static_assert(BlockSize<LargeT>::value == 16, "");
@@ -57,15 +61,15 @@ const auto& AllocBytes = malloc_allocator_base::outstanding_bytes;
 
 template <class Deque>
 struct PrintOnFailure {
-   explicit PrintOnFailure(Deque const& deque) : deque_(&deque) {}
-   void operator()() const { print(*deque_); }
+  explicit PrintOnFailure(Deque const& deque) : deque_(&deque) {}
+  void operator()() const { print(*deque_); }
+
 private:
   const Deque* deque_;
 
   PrintOnFailure(PrintOnFailure const&) = delete;
 };
 
-
 static void push_back() {
   const auto BS = BlockSize<LargeT>::value;
   std::unique_ptr<Deque<LargeT>> dp(new Deque<LargeT>);
@@ -194,11 +198,11 @@ static void push_front() {
 }
 
 static void std_queue() {
-  using D = Deque<LargeT>;
+  using D     = Deque<LargeT>;
   using Queue = std::queue<LargeT, D>;
   ContainerAdaptor<Queue> CA;
   const D& d = CA.GetContainer();
-  Queue &q = CA;
+  Queue& q   = CA;
   PrintOnFailure<Deque<LargeT>> on_fail(d);
 
   while (d.__block_count() < 4)
diff --git a/libcxx/test/libcxx/containers/sequences/list/list.cons/debug.copy.pass.cpp b/libcxx/test/libcxx/containers/sequences/list/list.cons/debug.copy.pass.cpp
index 8277a1b71038f..144a69691b4d3 100644
--- a/libcxx/test/libcxx/containers/sequences/list/list.cons/debug.copy.pass.cpp
+++ b/libcxx/test/libcxx/containers/sequences/list/list.cons/debug.copy.pass.cpp
@@ -18,11 +18,13 @@
 #include "check_assertion.h"
 
 int main(int, char**) {
-    std::list<int> l1;
-    l1.push_back(1); l1.push_back(2); l1.push_back(3);
-    std::list<int>::iterator i = l1.begin();
-    std::list<int> l2 = l1;
-    TEST_LIBCPP_ASSERT_FAILURE(l2.erase(i), "list::erase(iterator) called with an iterator not referring to this list");
+  std::list<int> l1;
+  l1.push_back(1);
+  l1.push_back(2);
+  l1.push_back(3);
+  std::list<int>::iterator i = l1.begin();
+  std::list<int> l2          = l1;
+  TEST_LIBCPP_ASSERT_FAILURE(l2.erase(i), "list::erase(iterator) called with an iterator not referring to this list");
 
-    return 0;
+  return 0;
 }
diff --git a/libcxx/test/libcxx/containers/sequences/list/list.modifiers/assert.erase_iter.end.pass.cpp b/libcxx/test/libcxx/containers/sequences/list/list.modifiers/assert.erase_iter.end.pass.cpp
index 531dd8de399de..6441d533cfa29 100644
--- a/libcxx/test/libcxx/containers/sequences/list/list.modifiers/assert.erase_iter.end.pass.cpp
+++ b/libcxx/test/libcxx/containers/sequences/list/list.modifiers/assert.erase_iter.end.pass.cpp
@@ -20,10 +20,10 @@
 #include "check_assertion.h"
 
 int main(int, char**) {
-    int a1[] = {1, 2, 3};
-    std::list<int> l1(a1, a1+3);
-    std::list<int>::const_iterator i = l1.end();
-    TEST_LIBCPP_ASSERT_FAILURE(l1.erase(i), "list::erase(iterator) called with a non-dereferenceable iterator");
+  int a1[] = {1, 2, 3};
+  std::list<int> l1(a1, a1 + 3);
+  std::list<int>::const_iterator i = l1.end();
+  TEST_LIBCPP_ASSERT_FAILURE(l1.erase(i), "list::erase(iterator) called with a non-dereferenceable iterator");
 
-    return 0;
+  return 0;
 }
diff --git a/libcxx/test/libcxx/containers/sequences/list/list.modifiers/assert.pop_back.empty.pass.cpp b/libcxx/test/libcxx/containers/sequences/list/list.modifiers/assert.pop_back.empty.pass.cpp
index d7c6999162270..c344264852995 100644
--- a/libcxx/test/libcxx/containers/sequences/list/list.modifiers/assert.pop_back.empty.pass.cpp
+++ b/libcxx/test/libcxx/containers/sequences/list/list.modifiers/assert.pop_back.empty.pass.cpp
@@ -21,15 +21,15 @@
 #include "check_assertion.h"
 
 int main(int, char**) {
-    int a[] = {1, 2, 3};
-    std::list<int> c(a, a+3);
-    c.pop_back();
-    assert(c == std::list<int>(a, a+2));
-    c.pop_back();
-    assert(c == std::list<int>(a, a+1));
-    c.pop_back();
-    assert(c.empty());
-    TEST_LIBCPP_ASSERT_FAILURE(c.pop_back(), "list::pop_back() called on an empty list");
+  int a[] = {1, 2, 3};
+  std::list<int> c(a, a + 3);
+  c.pop_back();
+  assert(c == std::list<int>(a, a + 2));
+  c.pop_back();
+  assert(c == std::list<int>(a, a + 1));
+  c.pop_back();
+  assert(c.empty());
+  TEST_LIBCPP_ASSERT_FAILURE(c.pop_back(), "list::pop_back() called on an empty list");
 
-    return 0;
+  return 0;
 }
diff --git a/libcxx/test/libcxx/containers/sequences/list/list.modifiers/debug.emplace.pass.cpp b/libcxx/test/libcxx/containers/sequences/list/list.modifiers/debug.emplace.pass.cpp
index d0f82ed5055ee..cd2f1eeea976a 100644
--- a/libcxx/test/libcxx/containers/sequences/list/list.modifiers/debug.emplace.pass.cpp
+++ b/libcxx/test/libcxx/containers/sequences/list/list.modifiers/debug.emplace.pass.cpp
@@ -25,10 +25,10 @@ struct A {
 };
 
 int main(int, char**) {
-    std::list<A> c1;
-    std::list<A> c2;
-    TEST_LIBCPP_ASSERT_FAILURE(c1.emplace(c2.cbegin(), 2, 3.5),
-                               "list::emplace(iterator, args...) called with an iterator not referring to this list");
+  std::list<A> c1;
+  std::list<A> c2;
+  TEST_LIBCPP_ASSERT_FAILURE(c1.emplace(c2.cbegin(), 2, 3.5),
+                             "list::emplace(iterator, args...) called with an iterator not referring to this list");
 
-    return 0;
+  return 0;
 }
diff --git a/libcxx/test/libcxx/containers/sequences/list/list.modifiers/debug.erase.iter.pass.cpp b/libcxx/test/libcxx/containers/sequences/list/list.modifiers/debug.erase.iter.pass.cpp
index d85b42a6c73e1..fb61673f4b7a4 100644
--- a/libcxx/test/libcxx/containers/sequences/list/list.modifiers/debug.erase.iter.pass.cpp
+++ b/libcxx/test/libcxx/containers/sequences/list/list.modifiers/debug.erase.iter.pass.cpp
@@ -18,11 +18,11 @@
 #include "check_assertion.h"
 
 int main(int, char**) {
-    int a1[] = {1, 2, 3};
-    std::list<int> l1(a1, a1+3);
-    std::list<int> l2(a1, a1+3);
-    std::list<int>::const_iterator i = l2.begin();
-    TEST_LIBCPP_ASSERT_FAILURE(l1.erase(i), "list::erase(iterator) called with an iterator not referring to this list");
+  int a1[] = {1, 2, 3};
+  std::list<int> l1(a1, a1 + 3);
+  std::list<int> l2(a1, a1 + 3);
+  std::list<int>::const_iterator i = l2.begin();
+  TEST_LIBCPP_ASSERT_FAILURE(l1.erase(i), "list::erase(iterator) called with an iterator not referring to this list");
 
-    return 0;
+  return 0;
 }
diff --git a/libcxx/test/libcxx/containers/sequences/list/list.modifiers/debug.erase.iter_iter.pass.cpp b/libcxx/test/libcxx/containers/sequences/list/list.modifiers/debug.erase.iter_iter.pass.cpp
index a7c4ea492c797..9d4198a120f99 100644
--- a/libcxx/test/libcxx/containers/sequences/list/list.modifiers/debug.erase.iter_iter.pass.cpp
+++ b/libcxx/test/libcxx/containers/sequences/list/list.modifiers/debug.erase.iter_iter.pass.cpp
@@ -18,40 +18,40 @@
 #include "check_assertion.h"
 
 int main(int, char**) {
-    // First iterator from another container
-    {
-        int a1[] = {1, 2, 3};
-        std::list<int> l1(a1, a1+3);
-        std::list<int> l2(a1, a1+3);
-        TEST_LIBCPP_ASSERT_FAILURE(l1.erase(l2.cbegin(), std::next(l1.cbegin())),
-                                "list::erase(iterator, iterator) called with an iterator not referring to this list");
-    }
-
-    // Second iterator from another container
-    {
-        int a1[] = {1, 2, 3};
-        std::list<int> l1(a1, a1+3);
-        std::list<int> l2(a1, a1+3);
-        TEST_LIBCPP_ASSERT_FAILURE(l1.erase(l1.cbegin(), std::next(l2.cbegin())),
-                                "list::erase(iterator, iterator) called with an iterator not referring to this list");
-    }
-
-    // Both iterators from another container
-    {
-        int a1[] = {1, 2, 3};
-        std::list<int> l1(a1, a1+3);
-        std::list<int> l2(a1, a1+3);
-        TEST_LIBCPP_ASSERT_FAILURE(l1.erase(l2.cbegin(), std::next(l2.cbegin())),
-                                "list::erase(iterator, iterator) called with an iterator not referring to this list");
-    }
-
-    // With an invalid range
-    {
-        int a1[] = {1, 2, 3};
-        std::list<int> l1(a1, a1+3);
-        TEST_LIBCPP_ASSERT_FAILURE(l1.erase(std::next(l1.cbegin()), l1.cbegin()),
-                                "Attempted to increment a non-incrementable list::const_iterator");
-    }
-
-    return 0;
+  // First iterator from another container
+  {
+    int a1[] = {1, 2, 3};
+    std::list<int> l1(a1, a1 + 3);
+    std::list<int> l2(a1, a1 + 3);
+    TEST_LIBCPP_ASSERT_FAILURE(l1.erase(l2.cbegin(), std::next(l1.cbegin())),
+                               "list::erase(iterator, iterator) called with an iterator not referring to this list");
+  }
+
+  // Second iterator from another container
+  {
+    int a1[] = {1, 2, 3};
+    std::list<int> l1(a1, a1 + 3);
+    std::list<int> l2(a1, a1 + 3);
+    TEST_LIBCPP_ASSERT_FAILURE(l1.erase(l1.cbegin(), std::next(l2.cbegin())),
+                               "list::erase(iterator, iterator) called with an iterator not referring to this list");
+  }
+
+  // Both iterators from another container
+  {
+    int a1[] = {1, 2, 3};
+    std::list<int> l1(a1, a1 + 3);
+    std::list<int> l2(a1, a1 + 3);
+    TEST_LIBCPP_ASSERT_FAILURE(l1.erase(l2.cbegin(), std::next(l2.cbegin())),
+                               "list::erase(iterator, iterator) called with an iterator not referring to this list");
+  }
+
+  // With an invalid range
+  {
+    int a1[] = {1, 2, 3};
+    std::list<int> l1(a1, a1 + 3);
+    TEST_LIBCPP_ASSERT_FAILURE(l1.erase(std::next(l1.cbegin()), l1.cbegin()),
+                               "Attempted to increment a non-incrementable list::const_iterator");
+  }
+
+  return 0;
 }
diff --git a/libcxx/test/libcxx/containers/sequences/list/list.modifiers/debug.insert.iter_iter_iter.pass.cpp b/libcxx/test/libcxx/containers/sequences/list/list.modifiers/debug.insert.iter_iter_iter.pass.cpp
index 124f6c4b63065..13326a1d97ce5 100644
--- a/libcxx/test/libcxx/containers/sequences/list/list.modifiers/debug.insert.iter_iter_iter.pass.cpp
+++ b/libcxx/test/libcxx/containers/sequences/list/list.modifiers/debug.insert.iter_iter_iter.pass.cpp
@@ -19,11 +19,11 @@
 #include "check_assertion.h"
 
 int main(int, char**) {
-    std::list<int> v(100);
-    std::list<int> v2(100);
-    int a[] = {1, 2, 3, 4, 5};
-    TEST_LIBCPP_ASSERT_FAILURE(v.insert(v2.cbegin(), a, a + 5),
-                                "list::insert(iterator, range) called with an iterator not referring to this list");
+  std::list<int> v(100);
+  std::list<int> v2(100);
+  int a[] = {1, 2, 3, 4, 5};
+  TEST_LIBCPP_ASSERT_FAILURE(v.insert(v2.cbegin(), a, a + 5),
+                             "list::insert(iterator, range) called with an iterator not referring to this list");
 
-    return 0;
+  return 0;
 }
diff --git a/libcxx/test/libcxx/containers/sequences/list/list.modifiers/debug.insert.iter_rvalue.pass.cpp b/libcxx/test/libcxx/containers/sequences/list/list.modifiers/debug.insert.iter_rvalue.pass.cpp
index fd4e8b82cb412..f4bce9a50a050 100644
--- a/libcxx/test/libcxx/containers/sequences/list/list.modifiers/debug.insert.iter_rvalue.pass.cpp
+++ b/libcxx/test/libcxx/containers/sequences/list/list.modifiers/debug.insert.iter_rvalue.pass.cpp
@@ -18,10 +18,10 @@
 #include "check_assertion.h"
 
 int main(int, char**) {
-    std::list<int> v1(3);
-    std::list<int> v2(3);
-    TEST_LIBCPP_ASSERT_FAILURE(v1.insert(v2.begin(), 4),
-                               "list::insert(iterator, x) called with an iterator not referring to this list");
+  std::list<int> v1(3);
+  std::list<int> v2(3);
+  TEST_LIBCPP_ASSERT_FAILURE(
+      v1.insert(v2.begin(), 4), "list::insert(iterator, x) called with an iterator not referring to this list");
 
-    return 0;
+  return 0;
 }
diff --git a/libcxx/test/libcxx/containers/sequences/list/list.modifiers/debug.insert.iter_size_value.pass.cpp b/libcxx/test/libcxx/containers/sequences/list/list.modifiers/debug.insert.iter_size_value.pass.cpp
index ee442a2b513db..46bad61f9c981 100644
--- a/libcxx/test/libcxx/containers/sequences/list/list.modifiers/debug.insert.iter_size_value.pass.cpp
+++ b/libcxx/test/libcxx/containers/sequences/list/list.modifiers/debug.insert.iter_size_value.pass.cpp
@@ -18,10 +18,10 @@
 #include "check_assertion.h"
 
 int main(int, char**) {
-    std::list<int> c1(100);
-    std::list<int> c2;
-    TEST_LIBCPP_ASSERT_FAILURE(c1.insert(c2.cbegin(), 5, 1),
-                               "list::insert(iterator, n, x) called with an iterator not referring to this list");
+  std::list<int> c1(100);
+  std::list<int> c2;
+  TEST_LIBCPP_ASSERT_FAILURE(
+      c1.insert(c2.cbegin(), 5, 1), "list::insert(iterator, n, x) called with an iterator not referring to this list");
 
-    return 0;
+  return 0;
 }
diff --git a/libcxx/test/libcxx/containers/sequences/list/list.modifiers/debug.insert.iter_value.pass.cpp b/libcxx/test/libcxx/containers/sequences/list/list.modifiers/debug.insert.iter_value.pass.cpp
index b61d66ecaaa62..16fa8efd07b6b 100644
--- a/libcxx/test/libcxx/containers/sequences/list/list.modifiers/debug.insert.iter_value.pass.cpp
+++ b/libcxx/test/libcxx/containers/sequences/list/list.modifiers/debug.insert.iter_value.pass.cpp
@@ -18,11 +18,11 @@
 #include "check_assertion.h"
 
 int main(int, char**) {
-    std::list<int> v1(3);
-    std::list<int> v2(3);
-    int i = 4;
-    TEST_LIBCPP_ASSERT_FAILURE(v1.insert(v2.begin(), i),
-                               "list::insert(iterator, x) called with an iterator not referring to this list");
+  std::list<int> v1(3);
+  std::list<int> v2(3);
+  int i = 4;
+  TEST_LIBCPP_ASSERT_FAILURE(
+      v1.insert(v2.begin(), i), "list::insert(iterator, x) called with an iterator not referring to this list");
 
-    return 0;
+  return 0;
 }
diff --git a/libcxx/test/libcxx/containers/sequences/list/list.ops/debug.splice.pos_list.pass.cpp b/libcxx/test/libcxx/containers/sequences/list/list.ops/debug.splice.pos_list.pass.cpp
index 81890ea309bdd..be759d06036c3 100644
--- a/libcxx/test/libcxx/containers/sequences/list/list.ops/debug.splice.pos_list.pass.cpp
+++ b/libcxx/test/libcxx/containers/sequences/list/list.ops/debug.splice.pos_list.pass.cpp
@@ -20,7 +20,7 @@
 int main(int, char**) {
   std::list<int> v1(3);
   std::list<int> v2(3);
-  TEST_LIBCPP_ASSERT_FAILURE(v1.splice(v2.begin(), v2),
-                              "list::splice(iterator, list) called with an iterator not referring to this list");
+  TEST_LIBCPP_ASSERT_FAILURE(
+      v1.splice(v2.begin(), v2), "list::splice(iterator, list) called with an iterator not referring to this list");
   return 0;
 }
diff --git a/libcxx/test/libcxx/containers/sequences/list/list.ops/debug.splice.pos_list_iter.pass.cpp b/libcxx/test/libcxx/containers/sequences/list/list.ops/debug.splice.pos_list_iter.pass.cpp
index 05aa7aae8f43b..1786a6c64edcb 100644
--- a/libcxx/test/libcxx/containers/sequences/list/list.ops/debug.splice.pos_list_iter.pass.cpp
+++ b/libcxx/test/libcxx/containers/sequences/list/list.ops/debug.splice.pos_list_iter.pass.cpp
@@ -18,11 +18,11 @@
 #include "check_assertion.h"
 
 int main(int, char**) {
-    std::list<int> v1(3);
-    std::list<int> v2(3);
-    TEST_LIBCPP_ASSERT_FAILURE(
-        v1.splice(v1.begin(), v2, v1.begin()),
-        "list::splice(iterator, list, iterator) called with the second iterator not referring to the list argument");
+  std::list<int> v1(3);
+  std::list<int> v2(3);
+  TEST_LIBCPP_ASSERT_FAILURE(
+      v1.splice(v1.begin(), v2, v1.begin()),
+      "list::splice(iterator, list, iterator) called with the second iterator not referring to the list argument");
 
   return 0;
 }
diff --git a/libcxx/test/libcxx/containers/sequences/vector.bool/trivial_for_purposes_of_call.pass.cpp b/libcxx/test/libcxx/containers/sequences/vector.bool/trivial_for_purposes_of_call.pass.cpp
index d29b0f1a95abd..4d37b83ba9cd6 100644
--- a/libcxx/test/libcxx/containers/sequences/vector.bool/trivial_for_purposes_of_call.pass.cpp
+++ b/libcxx/test/libcxx/containers/sequences/vector.bool/trivial_for_purposes_of_call.pass.cpp
@@ -31,12 +31,12 @@
 #include <vector>
 
 template <class T>
-using IsTrivialForCall = std::integral_constant<bool,
-  std::is_trivially_copy_constructible<T>::value &&
-  std::is_trivially_move_constructible<T>::value &&
-  std::is_trivially_destructible<T>::value
-  // Ignore the all-deleted case, it shouldn't occur here.
-  >;
+using IsTrivialForCall = std::integral_constant<
+    bool,
+    std::is_trivially_copy_constructible<T>::value && std::is_trivially_move_constructible<T>::value &&
+        std::is_trivially_destructible<T>::value
+    // Ignore the all-deleted case, it shouldn't occur here.
+    >;
 
 void test_const_iterator() {
   using It = std::vector<bool>::const_iterator;
diff --git a/libcxx/test/libcxx/containers/sequences/vector/asan.pass.cpp b/libcxx/test/libcxx/containers/sequences/vector/asan.pass.cpp
index 614323b1ffd7b..03d2b3e6ce9b9 100644
--- a/libcxx/test/libcxx/containers/sequences/vector/asan.pass.cpp
+++ b/libcxx/test/libcxx/containers/sequences/vector/asan.pass.cpp
@@ -23,12 +23,9 @@
 
 extern "C" void __sanitizer_set_death_callback(void (*callback)(void));
 
-void do_exit() {
-  exit(0);
-}
+void do_exit() { exit(0); }
 
-int main(int, char**)
-{
+int main(int, char**) {
 #if TEST_STD_VER >= 11
   {
     typedef int T;
diff --git a/libcxx/test/libcxx/containers/sequences/vector/asan_throw.pass.cpp b/libcxx/test/libcxx/containers/sequences/vector/asan_throw.pass.cpp
index 6a267e546f927..dcfa8029cfc0d 100644
--- a/libcxx/test/libcxx/containers/sequences/vector/asan_throw.pass.cpp
+++ b/libcxx/test/libcxx/containers/sequences/vector/asan_throw.pass.cpp
@@ -17,10 +17,10 @@
 
 class X {
 public:
-  X(const X &x) { Init(x.a); }
+  X(const X& x) { Init(x.a); }
   X(char arg) { Init(arg); }
   X() { Init(42); }
-  X &operator=(const X &x) {
+  X& operator=(const X& x) {
     Init(x.a);
     return *this;
   }
@@ -40,19 +40,17 @@ class X {
 
 class ThrowOnCopy {
 public:
-    ThrowOnCopy() : should_throw(false) {}
-    explicit ThrowOnCopy(bool xshould_throw) : should_throw(xshould_throw) {}
-
-    ThrowOnCopy(ThrowOnCopy const & other)
-        : should_throw(other.should_throw)
-    {
-        if (should_throw) {
-            throw 0;
-        }
+  ThrowOnCopy() : should_throw(false) {}
+  explicit ThrowOnCopy(bool xshould_throw) : should_throw(xshould_throw) {}
+
+  ThrowOnCopy(ThrowOnCopy const& other) : should_throw(other.should_throw) {
+    if (should_throw) {
+      throw 0;
     }
-    ThrowOnCopy& operator=(ThrowOnCopy const&) = default;
+  }
+  ThrowOnCopy& operator=(ThrowOnCopy const&) = default;
 
-    bool should_throw;
+  bool should_throw;
 };
 
 void test_push_back() {
@@ -175,7 +173,6 @@ void test_insert_n() {
   assert(0);
 }
 
-
 void test_insert_n2() {
   std::vector<ThrowOnCopy> v(10);
   v.reserve(100);
diff --git a/libcxx/test/libcxx/containers/sequences/vector/asan_turning_off.pass.cpp b/libcxx/test/libcxx/containers/sequences/vector/asan_turning_off.pass.cpp
index e2760271d60a5..1ad60d0544d41 100644
--- a/libcxx/test/libcxx/containers/sequences/vector/asan_turning_off.pass.cpp
+++ b/libcxx/test/libcxx/containers/sequences/vector/asan_turning_off.pass.cpp
@@ -25,7 +25,7 @@ struct reuse_allocator {
   static size_t const N = 100;
   reuse_allocator() {
     for (size_t i = 0; i < N; ++i)
-      __buffers[i] = malloc(8*1024);
+      __buffers[i] = malloc(8 * 1024);
   }
   ~reuse_allocator() {
     for (size_t i = 0; i < N; ++i)
@@ -46,8 +46,8 @@ struct user_allocator {
   user_allocator() = default;
   template <class U>
   user_allocator(user_allocator<U>) {}
-  friend bool operator==(user_allocator, user_allocator) {return true;}
-  friend bool operator!=(user_allocator x, user_allocator y) {return !(x == y);}
+  friend bool operator==(user_allocator, user_allocator) { return true; }
+  friend bool operator!=(user_allocator x, user_allocator y) { return !(x == y); }
 
   T* allocate(size_t) { return (T*)reuse_buffers.alloc(); }
   void deallocate(T*, size_t) noexcept {}
diff --git a/libcxx/test/libcxx/containers/sequences/vector/assert.pop_back.empty.pass.cpp b/libcxx/test/libcxx/containers/sequences/vector/assert.pop_back.empty.pass.cpp
index ad80be69897ed..6734f25b9db10 100644
--- a/libcxx/test/libcxx/containers/sequences/vector/assert.pop_back.empty.pass.cpp
+++ b/libcxx/test/libcxx/containers/sequences/vector/assert.pop_back.empty.pass.cpp
@@ -20,10 +20,10 @@
 #include "check_assertion.h"
 
 int main(int, char**) {
-    std::vector<int> v;
-    v.push_back(0);
-    v.pop_back();
-    TEST_LIBCPP_ASSERT_FAILURE(v.pop_back(), "vector::pop_back called on an empty vector");
+  std::vector<int> v;
+  v.push_back(0);
+  v.pop_back();
+  TEST_LIBCPP_ASSERT_FAILURE(v.pop_back(), "vector::pop_back called on an empty vector");
 
-    return 0;
+  return 0;
 }
diff --git a/libcxx/test/libcxx/containers/sequences/vector/erase.modules.compile.pass.mm b/libcxx/test/libcxx/containers/sequences/vector/erase.modules.compile.pass.mm
index 0e21a2e168b2c..d27067399eda8 100644
--- a/libcxx/test/libcxx/containers/sequences/vector/erase.modules.compile.pass.mm
+++ b/libcxx/test/libcxx/containers/sequences/vector/erase.modules.compile.pass.mm
@@ -13,6 +13,4 @@
 
 #include <vector>
 
-void f(std::vector<int> vec, std::vector<int>::iterator it) {
-    (void)vec.erase(it);
-}
+void f(std::vector<int> vec, std::vector<int>::iterator it) { (void)vec.erase(it); }
diff --git a/libcxx/test/libcxx/containers/sequences/vector/exception_safety_exceptions_disabled.pass.cpp b/libcxx/test/libcxx/containers/sequences/vector/exception_safety_exceptions_disabled.pass.cpp
index df562474c185a..66fdf37556bce 100644
--- a/libcxx/test/libcxx/containers/sequences/vector/exception_safety_exceptions_disabled.pass.cpp
+++ b/libcxx/test/libcxx/containers/sequences/vector/exception_safety_exceptions_disabled.pass.cpp
@@ -22,7 +22,7 @@
 #include "test_macros.h"
 
 #ifndef TEST_HAS_NO_EXCEPTIONS
-#error exceptions should be disabled.
+#  error exceptions should be disabled.
 #endif
 
 bool allow_moves = false;
diff --git a/libcxx/test/libcxx/containers/sequences/vector/invalid_allocator.verify.cpp b/libcxx/test/libcxx/containers/sequences/vector/invalid_allocator.verify.cpp
index c1d72bcd555ce..e2de7730f2599 100644
--- a/libcxx/test/libcxx/containers/sequences/vector/invalid_allocator.verify.cpp
+++ b/libcxx/test/libcxx/containers/sequences/vector/invalid_allocator.verify.cpp
@@ -11,9 +11,9 @@
 #include <vector>
 
 class FooAllocator {
- public:
+public:
   using value_type = int;
-  FooAllocator() = default;
+  FooAllocator()   = default;
 
   int* allocate(int num_objects);
 
@@ -24,5 +24,6 @@ class FooAllocator {
 };
 
 void func() {
-  std::vector<int, FooAllocator> v; //expected-error-re@*:* {{static assertion failed {{.*}}This allocator has to implement rebind}}
+  std::vector<int, FooAllocator>
+      v; //expected-error-re@*:* {{static assertion failed {{.*}}This allocator has to implement rebind}}
 }
diff --git a/libcxx/test/libcxx/containers/sequences/vector/robust_against_adl.pass.cpp b/libcxx/test/libcxx/containers/sequences/vector/robust_against_adl.pass.cpp
index 9c780ae98d1e8..c6868ce4cb867 100644
--- a/libcxx/test/libcxx/containers/sequences/vector/robust_against_adl.pass.cpp
+++ b/libcxx/test/libcxx/containers/sequences/vector/robust_against_adl.pass.cpp
@@ -17,35 +17,38 @@
 #include "test_macros.h"
 
 struct Incomplete;
-template<class T> struct Holder { T t; };
+template <class T>
+struct Holder {
+  T t;
+};
 
-template<class T, class AdlTrap = Holder<Incomplete>>
+template <class T, class AdlTrap = Holder<Incomplete>>
 struct MyAlloc {
-    using value_type = T;
-    MyAlloc() = default;
-    template<class U> MyAlloc(const MyAlloc<U>&) {}
-    T *allocate(std::size_t n) { return std::allocator<T>().allocate(n); }
-    void deallocate(T *p, std::size_t n) { return std::allocator<T>().deallocate(p, n); }
+  using value_type = T;
+  MyAlloc()        = default;
+  template <class U>
+  MyAlloc(const MyAlloc<U>&) {}
+  T* allocate(std::size_t n) { return std::allocator<T>().allocate(n); }
+  void deallocate(T* p, std::size_t n) { return std::allocator<T>().deallocate(p, n); }
 };
 
-int main(int, char**)
-{
-    std::vector<bool, MyAlloc<bool>> vb;
-    // std::fill_n triggers ADL because __bit_iterator has the container type as a template argument
-    // std::vector<bool, MyAlloc<bool>> wb(100);
-
-    std::vector<int, MyAlloc<int>> v;
-    std::vector<int, MyAlloc<int>> w(100);
-    v.push_back(1);
-    v.insert(v.end(), 2);
-    v.insert(v.end(), w.begin(), w.end());
-    v.pop_back();
-    v.erase(v.begin());
-    v.erase(v.begin(), v.end());
+int main(int, char**) {
+  std::vector<bool, MyAlloc<bool>> vb;
+  // std::fill_n triggers ADL because __bit_iterator has the container type as a template argument
+  // std::vector<bool, MyAlloc<bool>> wb(100);
+
+  std::vector<int, MyAlloc<int>> v;
+  std::vector<int, MyAlloc<int>> w(100);
+  v.push_back(1);
+  v.insert(v.end(), 2);
+  v.insert(v.end(), w.begin(), w.end());
+  v.pop_back();
+  v.erase(v.begin());
+  v.erase(v.begin(), v.end());
 #if TEST_STD_VER >= 14
-    // TODO: vector::swap is not robust against ADL because we compare allocators, and that
-    //       triggers ADL when looking up operator==.
-    // v.swap(w);
+  // TODO: vector::swap is not robust against ADL because we compare allocators, and that
+  //       triggers ADL when looking up operator==.
+  // v.swap(w);
 #endif
-    return 0;
+  return 0;
 }
diff --git a/libcxx/test/libcxx/containers/unord/key_value_traits.pass.cpp b/libcxx/test/libcxx/containers/unord/key_value_traits.pass.cpp
index 4b389fa759f89..e00a028489a72 100644
--- a/libcxx/test/libcxx/containers/unord/key_value_traits.pass.cpp
+++ b/libcxx/test/libcxx/containers/unord/key_value_traits.pass.cpp
@@ -47,10 +47,8 @@ void testKeyValueTrait() {
     static_assert((std::is_same<Traits::key_type, int>::value), "");
     static_assert((std::is_same<Traits::mapped_type, int>::value), "");
     static_assert((std::is_same<Traits::__node_value_type, Tp>::value), "");
-    static_assert((std::is_same<Traits::__container_value_type,
-                               std::pair<const int, int> >::value), "");
-    static_assert((std::is_same<Traits::__map_value_type,
-                               std::pair<const int, int> >::value), "");
+    static_assert((std::is_same<Traits::__container_value_type, std::pair<const int, int> >::value), "");
+    static_assert((std::is_same<Traits::__map_value_type, std::pair<const int, int> >::value), "");
     static_assert(Traits::__is_map == true, "");
   }
 }
diff --git a/libcxx/test/libcxx/containers/unord/next_pow2.pass.cpp b/libcxx/test/libcxx/containers/unord/next_pow2.pass.cpp
index 8a5d448dbf922..53a1dd2d3fa76 100644
--- a/libcxx/test/libcxx/containers/unord/next_pow2.pass.cpp
+++ b/libcxx/test/libcxx/containers/unord/next_pow2.pass.cpp
@@ -27,65 +27,52 @@
 
 #include "test_macros.h"
 
+bool is_power_of_two(unsigned long n) { return __builtin_popcount(n) == 1; }
 
-bool
-is_power_of_two(unsigned long n)
-{
-    return __builtin_popcount(n) == 1;
+void test_next_pow2_val(std::size_t n) {
+  std::size_t npow2 = std::__next_hash_pow2(n);
+  assert(is_power_of_two(npow2) && npow2 > n);
 }
 
-void test_next_pow2_val(std::size_t n)
-{
-        std::size_t npow2 = std::__next_hash_pow2(n);
-        assert(is_power_of_two(npow2) && npow2 > n);
-}
-
-void
-test_next_pow2()
-{
-    assert(!is_power_of_two(0));
-    assert(is_power_of_two(1));
-    assert(is_power_of_two(2));
-    assert(!is_power_of_two(3));
-
-    assert(std::__next_hash_pow2(0) == 0);
-    assert(std::__next_hash_pow2(1) == 1);
-
-    for (std::size_t n = 2; n < (sizeof(std::size_t) * 8 - 1); ++n)
-    {
-        std::size_t pow2 = 1ULL << n;
-        assert(std::__next_hash_pow2(pow2) == pow2);
-    }
-
-    test_next_pow2_val(3);
-    test_next_pow2_val(7);
-    test_next_pow2_val(9);
-    test_next_pow2_val(15);
-    test_next_pow2_val(127);
-    test_next_pow2_val(129);
+void test_next_pow2() {
+  assert(!is_power_of_two(0));
+  assert(is_power_of_two(1));
+  assert(is_power_of_two(2));
+  assert(!is_power_of_two(3));
+
+  assert(std::__next_hash_pow2(0) == 0);
+  assert(std::__next_hash_pow2(1) == 1);
+
+  for (std::size_t n = 2; n < (sizeof(std::size_t) * 8 - 1); ++n) {
+    std::size_t pow2 = 1ULL << n;
+    assert(std::__next_hash_pow2(pow2) == pow2);
+  }
+
+  test_next_pow2_val(3);
+  test_next_pow2_val(7);
+  test_next_pow2_val(9);
+  test_next_pow2_val(15);
+  test_next_pow2_val(127);
+  test_next_pow2_val(129);
 }
 
 // Note: this is only really useful when run with -fsanitize=undefined.
-void
-fuzz_unordered_map_reserve(unsigned num_inserts,
-                           unsigned num_reserve1,
-                           unsigned num_reserve2)
-{
-    std::unordered_map<std::uint64_t, unsigned long> m;
-    m.reserve(num_reserve1);
-    for (unsigned I = 0; I < num_inserts; ++I) m[I] = 0;
-    m.reserve(num_reserve2);
-    assert(m.bucket_count() >= num_reserve2);
+void fuzz_unordered_map_reserve(unsigned num_inserts, unsigned num_reserve1, unsigned num_reserve2) {
+  std::unordered_map<std::uint64_t, unsigned long> m;
+  m.reserve(num_reserve1);
+  for (unsigned I = 0; I < num_inserts; ++I)
+    m[I] = 0;
+  m.reserve(num_reserve2);
+  assert(m.bucket_count() >= num_reserve2);
 }
 
-int main(int, char**)
-{
-    test_next_pow2();
+int main(int, char**) {
+  test_next_pow2();
 
-    for (unsigned num_inserts = 0; num_inserts <= 64; ++num_inserts)
-        for (unsigned num_reserve1 = 1; num_reserve1 <= 64; ++num_reserve1)
-            for (unsigned num_reserve2 = 1; num_reserve2 <= 64; ++num_reserve2)
-                fuzz_unordered_map_reserve(num_inserts, num_reserve1, num_reserve2);
+  for (unsigned num_inserts = 0; num_inserts <= 64; ++num_inserts)
+    for (unsigned num_reserve1 = 1; num_reserve1 <= 64; ++num_reserve1)
+      for (unsigned num_reserve2 = 1; num_reserve2 <= 64; ++num_reserve2)
+        fuzz_unordered_map_reserve(num_inserts, num_reserve1, num_reserve2);
 
-    return 0;
+  return 0;
 }
diff --git a/libcxx/test/libcxx/containers/unord/next_prime.pass.cpp b/libcxx/test/libcxx/containers/unord/next_prime.pass.cpp
index 17a29033856c9..44b0eb96e2b9b 100644
--- a/libcxx/test/libcxx/containers/unord/next_prime.pass.cpp
+++ b/libcxx/test/libcxx/containers/unord/next_prime.pass.cpp
@@ -24,34 +24,28 @@
 
 #include "test_macros.h"
 
-bool
-is_prime(std::size_t n)
-{
-    switch (n)
-    {
-    case 0:
-    case 1:
-        return false;
-    }
-    for (std::size_t i = 2; i*i <= n; ++i)
-    {
-        if (n % i == 0)
-            return false;
-    }
-    return true;
+bool is_prime(std::size_t n) {
+  switch (n) {
+  case 0:
+  case 1:
+    return false;
+  }
+  for (std::size_t i = 2; i * i <= n; ++i) {
+    if (n % i == 0)
+      return false;
+  }
+  return true;
 }
 
-int main(int, char**)
-{
-    assert(std::__next_prime(0) == 0);
-    for (std::size_t n = 1; n <= 100000; ++n)
-    {
-        std::size_t p = std::__next_prime(n);
-        assert(p >= n);
-        for (std::size_t i = n; i < p; ++i)
-            assert(!is_prime(i));
-        assert(is_prime(p));
-    }
+int main(int, char**) {
+  assert(std::__next_prime(0) == 0);
+  for (std::size_t n = 1; n <= 100000; ++n) {
+    std::size_t p = std::__next_prime(n);
+    assert(p >= n);
+    for (std::size_t i = n; i < p; ++i)
+      assert(!is_prime(i));
+    assert(is_prime(p));
+  }
 
   return 0;
 }
diff --git a/libcxx/test/libcxx/containers/unord/non_const_comparator.incomplete.verify.cpp b/libcxx/test/libcxx/containers/unord/non_const_comparator.incomplete.verify.cpp
index 9fd4f698b5f96..c370cd65085a1 100644
--- a/libcxx/test/libcxx/containers/unord/non_const_comparator.incomplete.verify.cpp
+++ b/libcxx/test/libcxx/containers/unord/non_const_comparator.incomplete.verify.cpp
@@ -19,32 +19,33 @@
 
 #include "test_macros.h"
 
-
-template <template <typename ...> class Container>
+template <template <typename...> class Container>
 void test_set() {
-  struct KeyBase { };
-  struct KeyDerived;  // derives from KeyBase, but incomplete at this point
+  struct KeyBase {};
+  struct KeyDerived; // derives from KeyBase, but incomplete at this point
 
   // Name the type but don't instantiate it.
   using C = Container<KeyDerived*, std::hash<KeyBase*>, std::equal_to<KeyBase*>>;
 
   // Instantiate it but don't ODR use any members.
-  typename C::value_type dummy; (void)dummy;
+  typename C::value_type dummy;
+  (void)dummy;
 
   // Complete the types.
-  struct KeyDerived : KeyBase { };
+  struct KeyDerived : KeyBase {};
 
   C c; // ODR use it, which should be OK
 }
 
-template <template <typename ...> class Container>
+template <template <typename...> class Container>
 void test_map() {
-  struct Value { };
-  struct KeyBase { };
+  struct Value {};
+  struct KeyBase {};
   struct KeyDerived;
   using C = Container<KeyDerived*, Value, std::hash<KeyBase*>, std::equal_to<KeyBase*>>;
-  typename C::value_type dummy; (void)dummy;
-  struct KeyDerived : KeyBase { };
+  typename C::value_type dummy;
+  (void)dummy;
+  struct KeyDerived : KeyBase {};
   C c;
 }
 
diff --git a/libcxx/test/libcxx/containers/unord/unord.map/assert.bucket.pass.cpp b/libcxx/test/libcxx/containers/unord/unord.map/assert.bucket.pass.cpp
index 8d23673dfc50f..26621bce52187 100644
--- a/libcxx/test/libcxx/containers/unord/unord.map/assert.bucket.pass.cpp
+++ b/libcxx/test/libcxx/containers/unord/unord.map/assert.bucket.pass.cpp
@@ -21,9 +21,9 @@
 #include "check_assertion.h"
 
 int main(int, char**) {
-    typedef std::unordered_map<int, std::string> C;
-    C c;
-    TEST_LIBCPP_ASSERT_FAILURE(c.bucket(3), "unordered container::bucket(key) called when bucket_count() == 0");
+  typedef std::unordered_map<int, std::string> C;
+  C c;
+  TEST_LIBCPP_ASSERT_FAILURE(c.bucket(3), "unordered container::bucket(key) called when bucket_count() == 0");
 
-    return 0;
+  return 0;
 }
diff --git a/libcxx/test/libcxx/containers/unord/unord.map/assert.bucket_size.pass.cpp b/libcxx/test/libcxx/containers/unord/unord.map/assert.bucket_size.pass.cpp
index 0a0bb888182e9..ce6534f95f8dc 100644
--- a/libcxx/test/libcxx/containers/unord/unord.map/assert.bucket_size.pass.cpp
+++ b/libcxx/test/libcxx/containers/unord/unord.map/assert.bucket_size.pass.cpp
@@ -25,9 +25,9 @@
 #include "check_assertion.h"
 
 int main(int, char**) {
-    typedef std::unordered_map<int, std::string> C;
-    C c;
-    TEST_LIBCPP_ASSERT_FAILURE(c.bucket_size(3), "unordered container::bucket_size(n) called with n >= bucket_count()");
+  typedef std::unordered_map<int, std::string> C;
+  C c;
+  TEST_LIBCPP_ASSERT_FAILURE(c.bucket_size(3), "unordered container::bucket_size(n) called with n >= bucket_count()");
 
-    return 0;
+  return 0;
 }
diff --git a/libcxx/test/libcxx/containers/unord/unord.map/assert.max_load_factor.pass.cpp b/libcxx/test/libcxx/containers/unord/unord.map/assert.max_load_factor.pass.cpp
index 452a3ae8c3818..117758c5385e0 100644
--- a/libcxx/test/libcxx/containers/unord/unord.map/assert.max_load_factor.pass.cpp
+++ b/libcxx/test/libcxx/containers/unord/unord.map/assert.max_load_factor.pass.cpp
@@ -26,9 +26,9 @@
 #include "check_assertion.h"
 
 int main(int, char**) {
-    typedef std::unordered_map<int, std::string> C;
-    C c;
-    TEST_LIBCPP_ASSERT_FAILURE(c.max_load_factor(0), "unordered container::max_load_factor(lf) called with lf <= 0");
+  typedef std::unordered_map<int, std::string> C;
+  C c;
+  TEST_LIBCPP_ASSERT_FAILURE(c.max_load_factor(0), "unordered container::max_load_factor(lf) called with lf <= 0");
 
-    return 0;
+  return 0;
 }
diff --git a/libcxx/test/libcxx/containers/unord/unord.map/at.abort.pass.cpp b/libcxx/test/libcxx/containers/unord/unord.map/at.abort.pass.cpp
index 01fcdf8a8910e..bd6da6c4d69c9 100644
--- a/libcxx/test/libcxx/containers/unord/unord.map/at.abort.pass.cpp
+++ b/libcxx/test/libcxx/containers/unord/unord.map/at.abort.pass.cpp
@@ -24,10 +24,9 @@
 
 #include "test_macros.h"
 
-
 int main(int, char**) {
-    std::signal(SIGABRT, [](int) { std::_Exit(EXIT_SUCCESS); });
-    std::unordered_map<int, int> map;
-    map.at(1);
-    return EXIT_FAILURE;
+  std::signal(SIGABRT, [](int) { std::_Exit(EXIT_SUCCESS); });
+  std::unordered_map<int, int> map;
+  map.at(1);
+  return EXIT_FAILURE;
 }
diff --git a/libcxx/test/libcxx/containers/unord/unord.map/at.const.abort.pass.cpp b/libcxx/test/libcxx/containers/unord/unord.map/at.const.abort.pass.cpp
index 9e3ba1189bcd1..badd243dc1264 100644
--- a/libcxx/test/libcxx/containers/unord/unord.map/at.const.abort.pass.cpp
+++ b/libcxx/test/libcxx/containers/unord/unord.map/at.const.abort.pass.cpp
@@ -24,10 +24,9 @@
 
 #include "test_macros.h"
 
-
 int main(int, char**) {
-    std::signal(SIGABRT, [](int) { std::_Exit(EXIT_SUCCESS); });
-    std::unordered_map<int, int> const map;
-    map.at(1);
-    return EXIT_FAILURE;
+  std::signal(SIGABRT, [](int) { std::_Exit(EXIT_SUCCESS); });
+  std::unordered_map<int, int> const map;
+  map.at(1);
+  return EXIT_FAILURE;
 }
diff --git a/libcxx/test/libcxx/containers/unord/unord.map/debug.insert.hint_const_lvalue.pass.cpp b/libcxx/test/libcxx/containers/unord/unord.map/debug.insert.hint_const_lvalue.pass.cpp
index 9826941477e70..927a9e8813325 100644
--- a/libcxx/test/libcxx/containers/unord/unord.map/debug.insert.hint_const_lvalue.pass.cpp
+++ b/libcxx/test/libcxx/containers/unord/unord.map/debug.insert.hint_const_lvalue.pass.cpp
@@ -19,21 +19,23 @@
 #include "test_macros.h"
 
 int main(int, char**) {
-    typedef std::unordered_map<double, int> C;
-    typedef C::value_type P;
-    C c;
-    C c2;
-    C::const_iterator e = c2.end();
-    P v(3.5, 3);
+  typedef std::unordered_map<double, int> C;
+  typedef C::value_type P;
+  C c;
+  C c2;
+  C::const_iterator e = c2.end();
+  P v(3.5, 3);
 #if TEST_STD_VER < 11
-    TEST_LIBCPP_ASSERT_FAILURE(
-        c.insert(e, v),
-        "unordered_map::insert(const_iterator, const value_type&) called with an iterator not referring to this unordered_map");
+  TEST_LIBCPP_ASSERT_FAILURE(
+      c.insert(e, v),
+      "unordered_map::insert(const_iterator, const value_type&) called with an iterator not "
+      "referring to this unordered_map");
 #else
-    TEST_LIBCPP_ASSERT_FAILURE(
-        c.insert(e, v),
-        "unordered_map::insert(const_iterator, value_type&&) called with an iterator not referring to this unordered_map");
+  TEST_LIBCPP_ASSERT_FAILURE(
+      c.insert(e, v),
+      "unordered_map::insert(const_iterator, value_type&&) called with an iterator not "
+      "referring to this unordered_map");
 #endif
 
-    return 0;
+  return 0;
 }
diff --git a/libcxx/test/libcxx/containers/unord/unord.map/debug.insert.hint_rvalue.pass.cpp b/libcxx/test/libcxx/containers/unord/unord.map/debug.insert.hint_rvalue.pass.cpp
index e4adb91b8e7f9..e8a34c3c60851 100644
--- a/libcxx/test/libcxx/containers/unord/unord.map/debug.insert.hint_rvalue.pass.cpp
+++ b/libcxx/test/libcxx/containers/unord/unord.map/debug.insert.hint_rvalue.pass.cpp
@@ -21,14 +21,15 @@
 #include "test_macros.h"
 
 int main(int, char**) {
-    typedef std::unordered_map<double, int> C;
-    typedef C::value_type P;
-    C c;
-    C c2;
-    C::const_iterator e = c2.end();
-    TEST_LIBCPP_ASSERT_FAILURE(
-        c.insert(e, P(3.5, 3)),
-        "unordered_map::insert(const_iterator, const value_type&) called with an iterator not referring to this unordered_map");
+  typedef std::unordered_map<double, int> C;
+  typedef C::value_type P;
+  C c;
+  C c2;
+  C::const_iterator e = c2.end();
+  TEST_LIBCPP_ASSERT_FAILURE(
+      c.insert(e, P(3.5, 3)),
+      "unordered_map::insert(const_iterator, const value_type&) called with an iterator not "
+      "referring to this unordered_map");
 
-    return 0;
+  return 0;
 }
diff --git a/libcxx/test/libcxx/containers/unord/unord.map/debug.swap.pass.cpp b/libcxx/test/libcxx/containers/unord/unord.map/debug.swap.pass.cpp
index c9c100979ff08..cda7f971da1e5 100644
--- a/libcxx/test/libcxx/containers/unord/unord.map/debug.swap.pass.cpp
+++ b/libcxx/test/libcxx/containers/unord/unord.map/debug.swap.pass.cpp
@@ -22,17 +22,17 @@
 #include "check_assertion.h"
 
 int main(int, char**) {
-    typedef std::pair<int, int> P;
-    P a1[] = {P(1, 1), P(3, 3), P(7, 7), P(9, 9), P(10, 10)};
-    P a2[] = {P(0, 0), P(2, 2), P(4, 4), P(5, 5), P(6, 6), P(8, 8), P(11, 11)};
-    std::unordered_map<int, int> c1(a1, a1+sizeof(a1)/sizeof(a1[0]));
-    std::unordered_map<int, int> c2(a2, a2+sizeof(a2)/sizeof(a2[0]));
-    std::unordered_map<int, int>::iterator i1 = c1.begin();
-    std::unordered_map<int, int>::iterator i2 = c2.begin();
-    swap(c1, c2);
-    c1.erase(i2);
-    TEST_LIBCPP_ASSERT_FAILURE(
-        c1.erase(i1), "unordered container erase(iterator) called with an iterator not referring to this container");
-
-    return 0;
+  typedef std::pair<int, int> P;
+  P a1[] = {P(1, 1), P(3, 3), P(7, 7), P(9, 9), P(10, 10)};
+  P a2[] = {P(0, 0), P(2, 2), P(4, 4), P(5, 5), P(6, 6), P(8, 8), P(11, 11)};
+  std::unordered_map<int, int> c1(a1, a1 + sizeof(a1) / sizeof(a1[0]));
+  std::unordered_map<int, int> c2(a2, a2 + sizeof(a2) / sizeof(a2[0]));
+  std::unordered_map<int, int>::iterator i1 = c1.begin();
+  std::unordered_map<int, int>::iterator i2 = c2.begin();
+  swap(c1, c2);
+  c1.erase(i2);
+  TEST_LIBCPP_ASSERT_FAILURE(
+      c1.erase(i1), "unordered container erase(iterator) called with an iterator not referring to this container");
+
+  return 0;
 }
diff --git a/libcxx/test/libcxx/containers/unord/unord.map/unord.map.modifiers/debug.erase.iter.pass.cpp b/libcxx/test/libcxx/containers/unord/unord.map/unord.map.modifiers/debug.erase.iter.pass.cpp
index 777b00ec2d425..ef5a53e3de470 100644
--- a/libcxx/test/libcxx/containers/unord/unord.map/unord.map.modifiers/debug.erase.iter.pass.cpp
+++ b/libcxx/test/libcxx/containers/unord/unord.map/unord.map.modifiers/debug.erase.iter.pass.cpp
@@ -18,26 +18,26 @@
 #include "check_assertion.h"
 
 int main(int, char**) {
-    // With end()
-    {
-        typedef std::pair<int, int> P;
-        P a1[] = {P(1, 1), P(2, 2), P(3, 3)};
-        std::unordered_map<int, int> l1(a1, a1+3);
-        std::unordered_map<int, int>::const_iterator i = l1.end();
-        TEST_LIBCPP_ASSERT_FAILURE(l1.erase(i),
-                                "unordered container erase(iterator) called with a non-dereferenceable iterator");
-    }
-
-    // With iterator from another container
-    {
-        typedef std::pair<int, int> P;
-        P a1[] = {P(1, 1), P(2, 2), P(3, 3)};
-        std::unordered_map<int, int> l1(a1, a1+3);
-        std::unordered_map<int, int> l2(a1, a1+3);
-        std::unordered_map<int, int>::const_iterator i = l2.begin();
-        TEST_LIBCPP_ASSERT_FAILURE(
-            l1.erase(i), "unordered container erase(iterator) called with an iterator not referring to this container");
-    }
-
-    return 0;
+  // With end()
+  {
+    typedef std::pair<int, int> P;
+    P a1[] = {P(1, 1), P(2, 2), P(3, 3)};
+    std::unordered_map<int, int> l1(a1, a1 + 3);
+    std::unordered_map<int, int>::const_iterator i = l1.end();
+    TEST_LIBCPP_ASSERT_FAILURE(
+        l1.erase(i), "unordered container erase(iterator) called with a non-dereferenceable iterator");
+  }
+
+  // With iterator from another container
+  {
+    typedef std::pair<int, int> P;
+    P a1[] = {P(1, 1), P(2, 2), P(3, 3)};
+    std::unordered_map<int, int> l1(a1, a1 + 3);
+    std::unordered_map<int, int> l2(a1, a1 + 3);
+    std::unordered_map<int, int>::const_iterator i = l2.begin();
+    TEST_LIBCPP_ASSERT_FAILURE(
+        l1.erase(i), "unordered container erase(iterator) called with an iterator not referring to this container");
+  }
+
+  return 0;
 }
diff --git a/libcxx/test/libcxx/containers/unord/unord.map/unord.map.modifiers/debug.erase.iter_iter.pass.cpp b/libcxx/test/libcxx/containers/unord/unord.map/unord.map.modifiers/debug.erase.iter_iter.pass.cpp
index b99e4451a434a..1f364bf2ec192 100644
--- a/libcxx/test/libcxx/containers/unord/unord.map/unord.map.modifiers/debug.erase.iter_iter.pass.cpp
+++ b/libcxx/test/libcxx/containers/unord/unord.map/unord.map.modifiers/debug.erase.iter_iter.pass.cpp
@@ -18,47 +18,47 @@
 #include "check_assertion.h"
 
 int main(int, char**) {
-    // First iterator from a different container
-    {
-        typedef std::pair<int, int> P;
-        P a1[] = {P(1, 1), P(2, 2), P(3, 3)};
-        std::unordered_map<int, int> l1(a1, a1+3);
-        std::unordered_map<int, int> l2(a1, a1+3);
-        TEST_LIBCPP_ASSERT_FAILURE(
-            l1.erase(l2.cbegin(), std::next(l1.cbegin())),
-            "unordered container::erase(iterator, iterator) called with an iterator not referring to this container");
-    }
+  // First iterator from a different container
+  {
+    typedef std::pair<int, int> P;
+    P a1[] = {P(1, 1), P(2, 2), P(3, 3)};
+    std::unordered_map<int, int> l1(a1, a1 + 3);
+    std::unordered_map<int, int> l2(a1, a1 + 3);
+    TEST_LIBCPP_ASSERT_FAILURE(
+        l1.erase(l2.cbegin(), std::next(l1.cbegin())),
+        "unordered container::erase(iterator, iterator) called with an iterator not referring to this container");
+  }
 
-    // Second iterator from a different container
-    {
-        typedef std::pair<int, int> P;
-        P a1[] = {P(1, 1), P(2, 2), P(3, 3)};
-        std::unordered_map<int, int> l1(a1, a1+3);
-        std::unordered_map<int, int> l2(a1, a1+3);
-        TEST_LIBCPP_ASSERT_FAILURE(
-            l1.erase(l1.cbegin(), std::next(l2.cbegin())),
-            "unordered container::erase(iterator, iterator) called with an iterator not referring to this container");
-    }
+  // Second iterator from a different container
+  {
+    typedef std::pair<int, int> P;
+    P a1[] = {P(1, 1), P(2, 2), P(3, 3)};
+    std::unordered_map<int, int> l1(a1, a1 + 3);
+    std::unordered_map<int, int> l2(a1, a1 + 3);
+    TEST_LIBCPP_ASSERT_FAILURE(
+        l1.erase(l1.cbegin(), std::next(l2.cbegin())),
+        "unordered container::erase(iterator, iterator) called with an iterator not referring to this container");
+  }
 
-    // Both iterators from a different container
-    {
-        typedef std::pair<int, int> P;
-        P a1[] = {P(1, 1), P(2, 2), P(3, 3)};
-        std::unordered_map<int, int> l1(a1, a1+3);
-        std::unordered_map<int, int> l2(a1, a1+3);
-        TEST_LIBCPP_ASSERT_FAILURE(
-            l1.erase(l2.cbegin(), std::next(l2.cbegin())),
-            "unordered container::erase(iterator, iterator) called with an iterator not referring to this container");
-    }
+  // Both iterators from a different container
+  {
+    typedef std::pair<int, int> P;
+    P a1[] = {P(1, 1), P(2, 2), P(3, 3)};
+    std::unordered_map<int, int> l1(a1, a1 + 3);
+    std::unordered_map<int, int> l2(a1, a1 + 3);
+    TEST_LIBCPP_ASSERT_FAILURE(
+        l1.erase(l2.cbegin(), std::next(l2.cbegin())),
+        "unordered container::erase(iterator, iterator) called with an iterator not referring to this container");
+  }
 
-    // With iterators that don't form a valid range
-    {
-        typedef std::pair<int, int> P;
-        P a1[] = {P(1, 1), P(2, 2), P(3, 3)};
-        std::unordered_map<int, int> l1(a1, a1+3);
-        TEST_LIBCPP_ASSERT_FAILURE(l1.erase(std::next(l1.cbegin()), l1.cbegin()),
-                                "Attempted to increment a non-incrementable unordered container const_iterator");
-    }
+  // With iterators that don't form a valid range
+  {
+    typedef std::pair<int, int> P;
+    P a1[] = {P(1, 1), P(2, 2), P(3, 3)};
+    std::unordered_map<int, int> l1(a1, a1 + 3);
+    TEST_LIBCPP_ASSERT_FAILURE(l1.erase(std::next(l1.cbegin()), l1.cbegin()),
+                               "Attempted to increment a non-incrementable unordered container const_iterator");
+  }
 
-    return 0;
+  return 0;
 }
diff --git a/libcxx/test/libcxx/containers/unord/unord.multimap/assert.bucket.pass.cpp b/libcxx/test/libcxx/containers/unord/unord.multimap/assert.bucket.pass.cpp
index ec3892f444cd6..61c1651fdd956 100644
--- a/libcxx/test/libcxx/containers/unord/unord.multimap/assert.bucket.pass.cpp
+++ b/libcxx/test/libcxx/containers/unord/unord.multimap/assert.bucket.pass.cpp
@@ -25,9 +25,9 @@
 #include "check_assertion.h"
 
 int main(int, char**) {
-    typedef std::unordered_multimap<int, std::string> C;
-    C c;
-    TEST_LIBCPP_ASSERT_FAILURE(c.bucket(3), "unordered container::bucket(key) called when bucket_count() == 0");
+  typedef std::unordered_multimap<int, std::string> C;
+  C c;
+  TEST_LIBCPP_ASSERT_FAILURE(c.bucket(3), "unordered container::bucket(key) called when bucket_count() == 0");
 
-    return 0;
+  return 0;
 }
diff --git a/libcxx/test/libcxx/containers/unord/unord.multimap/assert.bucket_size.pass.cpp b/libcxx/test/libcxx/containers/unord/unord.multimap/assert.bucket_size.pass.cpp
index ea5904b94019e..8f2efdb6e56e6 100644
--- a/libcxx/test/libcxx/containers/unord/unord.multimap/assert.bucket_size.pass.cpp
+++ b/libcxx/test/libcxx/containers/unord/unord.multimap/assert.bucket_size.pass.cpp
@@ -25,9 +25,9 @@
 #include "check_assertion.h"
 
 int main(int, char**) {
-    typedef std::unordered_multimap<int, std::string> C;
-    C c;
-    TEST_LIBCPP_ASSERT_FAILURE(c.bucket_size(3), "unordered container::bucket_size(n) called with n >= bucket_count()");
+  typedef std::unordered_multimap<int, std::string> C;
+  C c;
+  TEST_LIBCPP_ASSERT_FAILURE(c.bucket_size(3), "unordered container::bucket_size(n) called with n >= bucket_count()");
 
-    return 0;
+  return 0;
 }
diff --git a/libcxx/test/libcxx/containers/unord/unord.multimap/assert.max_load_factor.pass.cpp b/libcxx/test/libcxx/containers/unord/unord.multimap/assert.max_load_factor.pass.cpp
index 65f7c91d293d6..dda5fe8b632bd 100644
--- a/libcxx/test/libcxx/containers/unord/unord.multimap/assert.max_load_factor.pass.cpp
+++ b/libcxx/test/libcxx/containers/unord/unord.multimap/assert.max_load_factor.pass.cpp
@@ -26,9 +26,9 @@
 #include "check_assertion.h"
 
 int main(int, char**) {
-    typedef std::unordered_multimap<int, std::string> C;
-    C c;
-    TEST_LIBCPP_ASSERT_FAILURE(c.max_load_factor(0), "unordered container::max_load_factor(lf) called with lf <= 0");
+  typedef std::unordered_multimap<int, std::string> C;
+  C c;
+  TEST_LIBCPP_ASSERT_FAILURE(c.max_load_factor(0), "unordered container::max_load_factor(lf) called with lf <= 0");
 
-    return 0;
+  return 0;
 }
diff --git a/libcxx/test/libcxx/containers/unord/unord.multimap/debug.insert.hint_const_lvalue.pass.cpp b/libcxx/test/libcxx/containers/unord/unord.multimap/debug.insert.hint_const_lvalue.pass.cpp
index e84c8f245ea35..017431077b714 100644
--- a/libcxx/test/libcxx/containers/unord/unord.multimap/debug.insert.hint_const_lvalue.pass.cpp
+++ b/libcxx/test/libcxx/containers/unord/unord.multimap/debug.insert.hint_const_lvalue.pass.cpp
@@ -18,15 +18,16 @@
 #include "check_assertion.h"
 
 int main(int, char**) {
-    typedef std::unordered_multimap<double, int> C;
-    typedef C::value_type P;
-    C c;
-    C c2;
-    C::const_iterator e = c2.end();
-    P v(3.5, 3);
-    TEST_LIBCPP_ASSERT_FAILURE(
-        c.insert(e, v),
-        "unordered container::emplace_hint(const_iterator, args...) called with an iterator not referring to this unordered container");
+  typedef std::unordered_multimap<double, int> C;
+  typedef C::value_type P;
+  C c;
+  C c2;
+  C::const_iterator e = c2.end();
+  P v(3.5, 3);
+  TEST_LIBCPP_ASSERT_FAILURE(
+      c.insert(e, v),
+      "unordered container::emplace_hint(const_iterator, args...) called with an iterator not "
+      "referring to this unordered container");
 
-    return 0;
+  return 0;
 }
diff --git a/libcxx/test/libcxx/containers/unord/unord.multimap/debug.insert.hint_rvalue.pass.cpp b/libcxx/test/libcxx/containers/unord/unord.multimap/debug.insert.hint_rvalue.pass.cpp
index 43663d0dd47b5..3713d78e347fa 100644
--- a/libcxx/test/libcxx/containers/unord/unord.multimap/debug.insert.hint_rvalue.pass.cpp
+++ b/libcxx/test/libcxx/containers/unord/unord.multimap/debug.insert.hint_rvalue.pass.cpp
@@ -20,14 +20,15 @@
 #include "check_assertion.h"
 
 int main(int, char**) {
-    typedef std::unordered_multimap<double, int> C;
-    typedef C::value_type P;
-    C c;
-    C c2;
-    C::const_iterator e = c2.end();
-    TEST_LIBCPP_ASSERT_FAILURE(
-        c.insert(e, P(3.5, 3)),
-        "unordered container::emplace_hint(const_iterator, args...) called with an iterator not referring to this unordered container");
+  typedef std::unordered_multimap<double, int> C;
+  typedef C::value_type P;
+  C c;
+  C c2;
+  C::const_iterator e = c2.end();
+  TEST_LIBCPP_ASSERT_FAILURE(
+      c.insert(e, P(3.5, 3)),
+      "unordered container::emplace_hint(const_iterator, args...) called with an iterator not "
+      "referring to this unordered container");
 
-    return 0;
+  return 0;
 }
diff --git a/libcxx/test/libcxx/containers/unord/unord.multimap/debug.swap.pass.cpp b/libcxx/test/libcxx/containers/unord/unord.multimap/debug.swap.pass.cpp
index fe4663f8ee295..d467182c8fae9 100644
--- a/libcxx/test/libcxx/containers/unord/unord.multimap/debug.swap.pass.cpp
+++ b/libcxx/test/libcxx/containers/unord/unord.multimap/debug.swap.pass.cpp
@@ -22,17 +22,17 @@
 #include "check_assertion.h"
 
 int main(int, char**) {
-    typedef std::pair<int, int> P;
-    P a1[] = {P(1, 1), P(3, 3), P(7, 7), P(9, 9), P(10, 10)};
-    P a2[] = {P(0, 0), P(2, 2), P(4, 4), P(5, 5), P(6, 6), P(8, 8), P(11, 11)};
-    std::unordered_multimap<int, int> c1(a1, a1+sizeof(a1)/sizeof(a1[0]));
-    std::unordered_multimap<int, int> c2(a2, a2+sizeof(a2)/sizeof(a2[0]));
-    std::unordered_multimap<int, int>::iterator i1 = c1.begin();
-    std::unordered_multimap<int, int>::iterator i2 = c2.begin();
-    swap(c1, c2);
-    c1.erase(i2);
-    TEST_LIBCPP_ASSERT_FAILURE(
-        c1.erase(i1), "unordered container erase(iterator) called with an iterator not referring to this container");
-
-    return 0;
+  typedef std::pair<int, int> P;
+  P a1[] = {P(1, 1), P(3, 3), P(7, 7), P(9, 9), P(10, 10)};
+  P a2[] = {P(0, 0), P(2, 2), P(4, 4), P(5, 5), P(6, 6), P(8, 8), P(11, 11)};
+  std::unordered_multimap<int, int> c1(a1, a1 + sizeof(a1) / sizeof(a1[0]));
+  std::unordered_multimap<int, int> c2(a2, a2 + sizeof(a2) / sizeof(a2[0]));
+  std::unordered_multimap<int, int>::iterator i1 = c1.begin();
+  std::unordered_multimap<int, int>::iterator i2 = c2.begin();
+  swap(c1, c2);
+  c1.erase(i2);
+  TEST_LIBCPP_ASSERT_FAILURE(
+      c1.erase(i1), "unordered container erase(iterator) called with an iterator not referring to this container");
+
+  return 0;
 }
diff --git a/libcxx/test/libcxx/containers/unord/unord.multimap/unord.multimap.modifiers/debug.erase.iter.pass.cpp b/libcxx/test/libcxx/containers/unord/unord.multimap/unord.multimap.modifiers/debug.erase.iter.pass.cpp
index f114a25b49673..0a430edf8b7b6 100644
--- a/libcxx/test/libcxx/containers/unord/unord.multimap/unord.multimap.modifiers/debug.erase.iter.pass.cpp
+++ b/libcxx/test/libcxx/containers/unord/unord.multimap/unord.multimap.modifiers/debug.erase.iter.pass.cpp
@@ -18,26 +18,26 @@
 #include "check_assertion.h"
 
 int main(int, char**) {
-    // With end()
-    {
-        typedef std::pair<int, int> P;
-        P a1[] = {P(1, 1), P(2, 2), P(3, 3)};
-        std::unordered_multimap<int, int> l1(a1, a1+3);
-        std::unordered_multimap<int, int>::const_iterator i = l1.end();
-        TEST_LIBCPP_ASSERT_FAILURE(l1.erase(i),
-                                "unordered container erase(iterator) called with a non-dereferenceable iterator");
-    }
-
-    // With iterator from another container
-    {
-        typedef std::pair<int, int> P;
-        P a1[] = {P(1, 1), P(2, 2), P(3, 3)};
-        std::unordered_multimap<int, int> l1(a1, a1+3);
-        std::unordered_multimap<int, int> l2(a1, a1+3);
-        std::unordered_multimap<int, int>::const_iterator i = l2.begin();
-        TEST_LIBCPP_ASSERT_FAILURE(
-            l1.erase(i), "unordered container erase(iterator) called with an iterator not referring to this container");
-    }
-
-    return 0;
+  // With end()
+  {
+    typedef std::pair<int, int> P;
+    P a1[] = {P(1, 1), P(2, 2), P(3, 3)};
+    std::unordered_multimap<int, int> l1(a1, a1 + 3);
+    std::unordered_multimap<int, int>::const_iterator i = l1.end();
+    TEST_LIBCPP_ASSERT_FAILURE(
+        l1.erase(i), "unordered container erase(iterator) called with a non-dereferenceable iterator");
+  }
+
+  // With iterator from another container
+  {
+    typedef std::pair<int, int> P;
+    P a1[] = {P(1, 1), P(2, 2), P(3, 3)};
+    std::unordered_multimap<int, int> l1(a1, a1 + 3);
+    std::unordered_multimap<int, int> l2(a1, a1 + 3);
+    std::unordered_multimap<int, int>::const_iterator i = l2.begin();
+    TEST_LIBCPP_ASSERT_FAILURE(
+        l1.erase(i), "unordered container erase(iterator) called with an iterator not referring to this container");
+  }
+
+  return 0;
 }
diff --git a/libcxx/test/libcxx/containers/unord/unord.multimap/unord.multimap.modifiers/debug.erase.iter_iter.pass.cpp b/libcxx/test/libcxx/containers/unord/unord.multimap/unord.multimap.modifiers/debug.erase.iter_iter.pass.cpp
index 9c63ac6a651b7..8c1382c5bbae4 100644
--- a/libcxx/test/libcxx/containers/unord/unord.multimap/unord.multimap.modifiers/debug.erase.iter_iter.pass.cpp
+++ b/libcxx/test/libcxx/containers/unord/unord.multimap/unord.multimap.modifiers/debug.erase.iter_iter.pass.cpp
@@ -18,47 +18,47 @@
 #include "check_assertion.h"
 
 int main(int, char**) {
-    // With first iterator from another container
-    {
-        typedef std::pair<int, int> P;
-        P a1[] = {P(1, 1), P(2, 2), P(3, 3)};
-        std::unordered_multimap<int, int> l1(a1, a1+3);
-        std::unordered_multimap<int, int> l2(a1, a1+3);
-        TEST_LIBCPP_ASSERT_FAILURE(
-            l1.erase(l2.cbegin(), std::next(l1.cbegin())),
-            "unordered container::erase(iterator, iterator) called with an iterator not referring to this container");
-    }
+  // With first iterator from another container
+  {
+    typedef std::pair<int, int> P;
+    P a1[] = {P(1, 1), P(2, 2), P(3, 3)};
+    std::unordered_multimap<int, int> l1(a1, a1 + 3);
+    std::unordered_multimap<int, int> l2(a1, a1 + 3);
+    TEST_LIBCPP_ASSERT_FAILURE(
+        l1.erase(l2.cbegin(), std::next(l1.cbegin())),
+        "unordered container::erase(iterator, iterator) called with an iterator not referring to this container");
+  }
 
-    // With second iterator from another container
-    {
-        typedef std::pair<int, int> P;
-        P a1[] = {P(1, 1), P(2, 2), P(3, 3)};
-        std::unordered_multimap<int, int> l1(a1, a1+3);
-        std::unordered_multimap<int, int> l2(a1, a1+3);
-        TEST_LIBCPP_ASSERT_FAILURE(
-            l1.erase(l1.cbegin(), std::next(l2.cbegin())),
-            "unordered container::erase(iterator, iterator) called with an iterator not referring to this container");
-    }
+  // With second iterator from another container
+  {
+    typedef std::pair<int, int> P;
+    P a1[] = {P(1, 1), P(2, 2), P(3, 3)};
+    std::unordered_multimap<int, int> l1(a1, a1 + 3);
+    std::unordered_multimap<int, int> l2(a1, a1 + 3);
+    TEST_LIBCPP_ASSERT_FAILURE(
+        l1.erase(l1.cbegin(), std::next(l2.cbegin())),
+        "unordered container::erase(iterator, iterator) called with an iterator not referring to this container");
+  }
 
-    // With both iterators from another container
-    {
-        typedef std::pair<int, int> P;
-        P a1[] = {P(1, 1), P(2, 2), P(3, 3)};
-        std::unordered_multimap<int, int> l1(a1, a1+3);
-        std::unordered_multimap<int, int> l2(a1, a1+3);
-        TEST_LIBCPP_ASSERT_FAILURE(
-            l1.erase(l2.cbegin(), std::next(l2.cbegin())),
-            "unordered container::erase(iterator, iterator) called with an iterator not referring to this container");
-    }
+  // With both iterators from another container
+  {
+    typedef std::pair<int, int> P;
+    P a1[] = {P(1, 1), P(2, 2), P(3, 3)};
+    std::unordered_multimap<int, int> l1(a1, a1 + 3);
+    std::unordered_multimap<int, int> l2(a1, a1 + 3);
+    TEST_LIBCPP_ASSERT_FAILURE(
+        l1.erase(l2.cbegin(), std::next(l2.cbegin())),
+        "unordered container::erase(iterator, iterator) called with an iterator not referring to this container");
+  }
 
-    // With an invalid range
-    {
-        typedef std::pair<int, int> P;
-        P a1[] = {P(1, 1), P(2, 2), P(3, 3)};
-        std::unordered_multimap<int, int> l1(a1, a1+3);
-        TEST_LIBCPP_ASSERT_FAILURE(l1.erase(std::next(l1.cbegin()), l1.cbegin()),
-                                "Attempted to increment a non-incrementable unordered container const_iterator");
-    }
+  // With an invalid range
+  {
+    typedef std::pair<int, int> P;
+    P a1[] = {P(1, 1), P(2, 2), P(3, 3)};
+    std::unordered_multimap<int, int> l1(a1, a1 + 3);
+    TEST_LIBCPP_ASSERT_FAILURE(l1.erase(std::next(l1.cbegin()), l1.cbegin()),
+                               "Attempted to increment a non-incrementable unordered container const_iterator");
+  }
 
-    return 0;
+  return 0;
 }
diff --git a/libcxx/test/libcxx/containers/unord/unord.multiset/assert.bucket.pass.cpp b/libcxx/test/libcxx/containers/unord/unord.multiset/assert.bucket.pass.cpp
index 49fdb723b4230..eebf11c27bb17 100644
--- a/libcxx/test/libcxx/containers/unord/unord.multiset/assert.bucket.pass.cpp
+++ b/libcxx/test/libcxx/containers/unord/unord.multiset/assert.bucket.pass.cpp
@@ -24,9 +24,9 @@
 #include "check_assertion.h"
 
 int main(int, char**) {
-    typedef std::unordered_multiset<int> C;
-    C c;
-    TEST_LIBCPP_ASSERT_FAILURE(c.bucket(3), "unordered container::bucket(key) called when bucket_count() == 0");
+  typedef std::unordered_multiset<int> C;
+  C c;
+  TEST_LIBCPP_ASSERT_FAILURE(c.bucket(3), "unordered container::bucket(key) called when bucket_count() == 0");
 
-    return 0;
+  return 0;
 }
diff --git a/libcxx/test/libcxx/containers/unord/unord.multiset/assert.bucket_size.pass.cpp b/libcxx/test/libcxx/containers/unord/unord.multiset/assert.bucket_size.pass.cpp
index 5709fd1e30d9a..1c107ca11193c 100644
--- a/libcxx/test/libcxx/containers/unord/unord.multiset/assert.bucket_size.pass.cpp
+++ b/libcxx/test/libcxx/containers/unord/unord.multiset/assert.bucket_size.pass.cpp
@@ -24,9 +24,9 @@
 #include "check_assertion.h"
 
 int main(int, char**) {
-    typedef std::unordered_multiset<int> C;
-    C c;
-    TEST_LIBCPP_ASSERT_FAILURE(c.bucket_size(3), "unordered container::bucket_size(n) called with n >= bucket_count()");
+  typedef std::unordered_multiset<int> C;
+  C c;
+  TEST_LIBCPP_ASSERT_FAILURE(c.bucket_size(3), "unordered container::bucket_size(n) called with n >= bucket_count()");
 
-    return 0;
+  return 0;
 }
diff --git a/libcxx/test/libcxx/containers/unord/unord.multiset/assert.max_load_factor.pass.cpp b/libcxx/test/libcxx/containers/unord/unord.multiset/assert.max_load_factor.pass.cpp
index c54bb833e7240..a38849482cee9 100644
--- a/libcxx/test/libcxx/containers/unord/unord.multiset/assert.max_load_factor.pass.cpp
+++ b/libcxx/test/libcxx/containers/unord/unord.multiset/assert.max_load_factor.pass.cpp
@@ -25,9 +25,9 @@
 #include "check_assertion.h"
 
 int main(int, char**) {
-    typedef std::unordered_multiset<int> C;
-    C c;
-    TEST_LIBCPP_ASSERT_FAILURE(c.max_load_factor(0), "unordered container::max_load_factor(lf) called with lf <= 0");
+  typedef std::unordered_multiset<int> C;
+  C c;
+  TEST_LIBCPP_ASSERT_FAILURE(c.max_load_factor(0), "unordered container::max_load_factor(lf) called with lf <= 0");
 
-    return 0;
+  return 0;
 }
diff --git a/libcxx/test/libcxx/containers/unord/unord.multiset/debug.erase.iter.pass.cpp b/libcxx/test/libcxx/containers/unord/unord.multiset/debug.erase.iter.pass.cpp
index 16bab55067bae..41818b814cf98 100644
--- a/libcxx/test/libcxx/containers/unord/unord.multiset/debug.erase.iter.pass.cpp
+++ b/libcxx/test/libcxx/containers/unord/unord.multiset/debug.erase.iter.pass.cpp
@@ -18,24 +18,24 @@
 #include "check_assertion.h"
 
 int main(int, char**) {
-    // With end()
-    {
-        int a1[] = {1, 2, 3};
-        std::unordered_multiset<int> l1(a1, a1+3);
-        std::unordered_multiset<int>::const_iterator i = l1.end();
-        TEST_LIBCPP_ASSERT_FAILURE(l1.erase(i),
-                                "unordered container erase(iterator) called with a non-dereferenceable iterator");
-    }
-
-    // With iterator from another container
-    {
-        int a1[] = {1, 2, 3};
-        std::unordered_multiset<int> l1(a1, a1+3);
-        std::unordered_multiset<int> l2(a1, a1+3);
-        std::unordered_multiset<int>::const_iterator i = l2.begin();
-        TEST_LIBCPP_ASSERT_FAILURE(
-            l1.erase(i), "unordered container erase(iterator) called with an iterator not referring to this container");
-    }
-
-    return 0;
+  // With end()
+  {
+    int a1[] = {1, 2, 3};
+    std::unordered_multiset<int> l1(a1, a1 + 3);
+    std::unordered_multiset<int>::const_iterator i = l1.end();
+    TEST_LIBCPP_ASSERT_FAILURE(
+        l1.erase(i), "unordered container erase(iterator) called with a non-dereferenceable iterator");
+  }
+
+  // With iterator from another container
+  {
+    int a1[] = {1, 2, 3};
+    std::unordered_multiset<int> l1(a1, a1 + 3);
+    std::unordered_multiset<int> l2(a1, a1 + 3);
+    std::unordered_multiset<int>::const_iterator i = l2.begin();
+    TEST_LIBCPP_ASSERT_FAILURE(
+        l1.erase(i), "unordered container erase(iterator) called with an iterator not referring to this container");
+  }
+
+  return 0;
 }
diff --git a/libcxx/test/libcxx/containers/unord/unord.multiset/debug.erase.iter_iter.pass.cpp b/libcxx/test/libcxx/containers/unord/unord.multiset/debug.erase.iter_iter.pass.cpp
index 30b9d80752de8..a85a61c17575c 100644
--- a/libcxx/test/libcxx/containers/unord/unord.multiset/debug.erase.iter_iter.pass.cpp
+++ b/libcxx/test/libcxx/containers/unord/unord.multiset/debug.erase.iter_iter.pass.cpp
@@ -18,43 +18,43 @@
 #include "check_assertion.h"
 
 int main(int, char**) {
-    // With first iterator from another container
-    {
-        int a1[] = {1, 2, 3};
-        std::unordered_multiset<int> l1(a1, a1+3);
-        std::unordered_multiset<int> l2(a1, a1+3);
-        TEST_LIBCPP_ASSERT_FAILURE(
-            l1.erase(l2.cbegin(), std::next(l1.cbegin())),
-            "unordered container::erase(iterator, iterator) called with an iterator not referring to this container");
-    }
-
-    // With second iterator from another container
-    {
-        int a1[] = {1, 2, 3};
-        std::unordered_multiset<int> l1(a1, a1+3);
-        std::unordered_multiset<int> l2(a1, a1+3);
-        TEST_LIBCPP_ASSERT_FAILURE(
-            l1.erase(l1.cbegin(), std::next(l2.cbegin())),
-            "unordered container::erase(iterator, iterator) called with an iterator not referring to this container");
-    }
-
-    // With both iterators from another container
-    {
-        int a1[] = {1, 2, 3};
-        std::unordered_multiset<int> l1(a1, a1+3);
-        std::unordered_multiset<int> l2(a1, a1+3);
-        TEST_LIBCPP_ASSERT_FAILURE(
-            l1.erase(l2.cbegin(), std::next(l2.cbegin())),
-            "unordered container::erase(iterator, iterator) called with an iterator not referring to this container");
-    }
-
-    // With an invalid range
-    {
-        int a1[] = {1, 2, 3};
-        std::unordered_multiset<int> l1(a1, a1+3);
-        TEST_LIBCPP_ASSERT_FAILURE(l1.erase(std::next(l1.cbegin()), l1.cbegin()),
-                                "Attempted to increment a non-incrementable unordered container const_iterator");
-    }
-
-    return 0;
+  // With first iterator from another container
+  {
+    int a1[] = {1, 2, 3};
+    std::unordered_multiset<int> l1(a1, a1 + 3);
+    std::unordered_multiset<int> l2(a1, a1 + 3);
+    TEST_LIBCPP_ASSERT_FAILURE(
+        l1.erase(l2.cbegin(), std::next(l1.cbegin())),
+        "unordered container::erase(iterator, iterator) called with an iterator not referring to this container");
+  }
+
+  // With second iterator from another container
+  {
+    int a1[] = {1, 2, 3};
+    std::unordered_multiset<int> l1(a1, a1 + 3);
+    std::unordered_multiset<int> l2(a1, a1 + 3);
+    TEST_LIBCPP_ASSERT_FAILURE(
+        l1.erase(l1.cbegin(), std::next(l2.cbegin())),
+        "unordered container::erase(iterator, iterator) called with an iterator not referring to this container");
+  }
+
+  // With both iterators from another container
+  {
+    int a1[] = {1, 2, 3};
+    std::unordered_multiset<int> l1(a1, a1 + 3);
+    std::unordered_multiset<int> l2(a1, a1 + 3);
+    TEST_LIBCPP_ASSERT_FAILURE(
+        l1.erase(l2.cbegin(), std::next(l2.cbegin())),
+        "unordered container::erase(iterator, iterator) called with an iterator not referring to this container");
+  }
+
+  // With an invalid range
+  {
+    int a1[] = {1, 2, 3};
+    std::unordered_multiset<int> l1(a1, a1 + 3);
+    TEST_LIBCPP_ASSERT_FAILURE(l1.erase(std::next(l1.cbegin()), l1.cbegin()),
+                               "Attempted to increment a non-incrementable unordered container const_iterator");
+  }
+
+  return 0;
 }
diff --git a/libcxx/test/libcxx/containers/unord/unord.multiset/debug.insert.hint_const_lvalue.pass.cpp b/libcxx/test/libcxx/containers/unord/unord.multiset/debug.insert.hint_const_lvalue.pass.cpp
index 71ce908af04b0..2274dd3e710ee 100644
--- a/libcxx/test/libcxx/containers/unord/unord.multiset/debug.insert.hint_const_lvalue.pass.cpp
+++ b/libcxx/test/libcxx/containers/unord/unord.multiset/debug.insert.hint_const_lvalue.pass.cpp
@@ -18,15 +18,16 @@
 #include "check_assertion.h"
 
 int main(int, char**) {
-    typedef std::unordered_multiset<double> C;
-    typedef C::value_type P;
-    C c;
-    C c2;
-    C::const_iterator e = c2.end();
-    P v(3.5);
-    TEST_LIBCPP_ASSERT_FAILURE(
-        c.insert(e, v),
-        "unordered container::emplace_hint(const_iterator, args...) called with an iterator not referring to this unordered container");
+  typedef std::unordered_multiset<double> C;
+  typedef C::value_type P;
+  C c;
+  C c2;
+  C::const_iterator e = c2.end();
+  P v(3.5);
+  TEST_LIBCPP_ASSERT_FAILURE(
+      c.insert(e, v),
+      "unordered container::emplace_hint(const_iterator, args...) called with an iterator not "
+      "referring to this unordered container");
 
-    return 0;
+  return 0;
 }
diff --git a/libcxx/test/libcxx/containers/unord/unord.multiset/debug.swap.pass.cpp b/libcxx/test/libcxx/containers/unord/unord.multiset/debug.swap.pass.cpp
index b94107da04e97..aa6b56f83fcc9 100644
--- a/libcxx/test/libcxx/containers/unord/unord.multiset/debug.swap.pass.cpp
+++ b/libcxx/test/libcxx/containers/unord/unord.multiset/debug.swap.pass.cpp
@@ -22,16 +22,16 @@
 #include "check_assertion.h"
 
 int main(int, char**) {
-    int a1[] = {1, 3, 7, 9, 10};
-    int a2[] = {0, 2, 4, 5, 6, 8, 11};
-    std::unordered_multiset<int> c1(a1, a1+sizeof(a1)/sizeof(a1[0]));
-    std::unordered_multiset<int> c2(a2, a2+sizeof(a2)/sizeof(a2[0]));
-    std::unordered_multiset<int>::iterator i1 = c1.begin();
-    std::unordered_multiset<int>::iterator i2 = c2.begin();
-    swap(c1, c2);
-    c1.erase(i2);
-    TEST_LIBCPP_ASSERT_FAILURE(
-        c1.erase(i1), "unordered container erase(iterator) called with an iterator not referring to this container");
-
-    return 0;
+  int a1[] = {1, 3, 7, 9, 10};
+  int a2[] = {0, 2, 4, 5, 6, 8, 11};
+  std::unordered_multiset<int> c1(a1, a1 + sizeof(a1) / sizeof(a1[0]));
+  std::unordered_multiset<int> c2(a2, a2 + sizeof(a2) / sizeof(a2[0]));
+  std::unordered_multiset<int>::iterator i1 = c1.begin();
+  std::unordered_multiset<int>::iterator i2 = c2.begin();
+  swap(c1, c2);
+  c1.erase(i2);
+  TEST_LIBCPP_ASSERT_FAILURE(
+      c1.erase(i1), "unordered container erase(iterator) called with an iterator not referring to this container");
+
+  return 0;
 }
diff --git a/libcxx/test/libcxx/containers/unord/unord.set/assert.bucket.pass.cpp b/libcxx/test/libcxx/containers/unord/unord.set/assert.bucket.pass.cpp
index c0bfb00efc68a..6504a88a2e1f6 100644
--- a/libcxx/test/libcxx/containers/unord/unord.set/assert.bucket.pass.cpp
+++ b/libcxx/test/libcxx/containers/unord/unord.set/assert.bucket.pass.cpp
@@ -24,9 +24,9 @@
 #include "check_assertion.h"
 
 int main(int, char**) {
-    typedef std::unordered_set<int> C;
-    C c;
-    TEST_LIBCPP_ASSERT_FAILURE(c.bucket(3), "unordered container::bucket(key) called when bucket_count() == 0");
+  typedef std::unordered_set<int> C;
+  C c;
+  TEST_LIBCPP_ASSERT_FAILURE(c.bucket(3), "unordered container::bucket(key) called when bucket_count() == 0");
 
-    return 0;
+  return 0;
 }
diff --git a/libcxx/test/libcxx/containers/unord/unord.set/assert.bucket_size.pass.cpp b/libcxx/test/libcxx/containers/unord/unord.set/assert.bucket_size.pass.cpp
index 3e25b448cb872..18440548d76c2 100644
--- a/libcxx/test/libcxx/containers/unord/unord.set/assert.bucket_size.pass.cpp
+++ b/libcxx/test/libcxx/containers/unord/unord.set/assert.bucket_size.pass.cpp
@@ -24,9 +24,9 @@
 #include "check_assertion.h"
 
 int main(int, char**) {
-    typedef std::unordered_set<int> C;
-    C c;
-    TEST_LIBCPP_ASSERT_FAILURE(c.bucket_size(3), "unordered container::bucket_size(n) called with n >= bucket_count()");
+  typedef std::unordered_set<int> C;
+  C c;
+  TEST_LIBCPP_ASSERT_FAILURE(c.bucket_size(3), "unordered container::bucket_size(n) called with n >= bucket_count()");
 
-    return 0;
+  return 0;
 }
diff --git a/libcxx/test/libcxx/containers/unord/unord.set/assert.max_load_factor.pass.cpp b/libcxx/test/libcxx/containers/unord/unord.set/assert.max_load_factor.pass.cpp
index ec1f1f3bd5ce1..80e89411d99c3 100644
--- a/libcxx/test/libcxx/containers/unord/unord.set/assert.max_load_factor.pass.cpp
+++ b/libcxx/test/libcxx/containers/unord/unord.set/assert.max_load_factor.pass.cpp
@@ -25,10 +25,9 @@
 #include "check_assertion.h"
 
 int main(int, char**) {
-    typedef std::unordered_set<int> C;
-    C c;
-    TEST_LIBCPP_ASSERT_FAILURE(c.max_load_factor(-0.5f),
-                               "unordered container::max_load_factor(lf) called with lf <= 0");
+  typedef std::unordered_set<int> C;
+  C c;
+  TEST_LIBCPP_ASSERT_FAILURE(c.max_load_factor(-0.5f), "unordered container::max_load_factor(lf) called with lf <= 0");
 
-    return 0;
+  return 0;
 }
diff --git a/libcxx/test/libcxx/containers/unord/unord.set/debug.erase.iter.pass.cpp b/libcxx/test/libcxx/containers/unord/unord.set/debug.erase.iter.pass.cpp
index 580829c1b6fc3..8dbcb0576cd85 100644
--- a/libcxx/test/libcxx/containers/unord/unord.set/debug.erase.iter.pass.cpp
+++ b/libcxx/test/libcxx/containers/unord/unord.set/debug.erase.iter.pass.cpp
@@ -18,24 +18,24 @@
 #include "check_assertion.h"
 
 int main(int, char**) {
-    // With end()
-    {
-        int a1[] = {1, 2, 3};
-        std::unordered_set<int> l1(a1, a1+3);
-        std::unordered_set<int>::const_iterator i = l1.end();
-        TEST_LIBCPP_ASSERT_FAILURE(l1.erase(i),
-                                "unordered container erase(iterator) called with a non-dereferenceable iterator");
-    }
-
-    // With iterator from another container
-    {
-        int a1[] = {1, 2, 3};
-        std::unordered_set<int> l1(a1, a1+3);
-        std::unordered_set<int> l2(a1, a1+3);
-        std::unordered_set<int>::const_iterator i = l2.begin();
-        TEST_LIBCPP_ASSERT_FAILURE(
-            l1.erase(i), "unordered container erase(iterator) called with an iterator not referring to this container");
-    }
-
-    return 0;
+  // With end()
+  {
+    int a1[] = {1, 2, 3};
+    std::unordered_set<int> l1(a1, a1 + 3);
+    std::unordered_set<int>::const_iterator i = l1.end();
+    TEST_LIBCPP_ASSERT_FAILURE(
+        l1.erase(i), "unordered container erase(iterator) called with a non-dereferenceable iterator");
+  }
+
+  // With iterator from another container
+  {
+    int a1[] = {1, 2, 3};
+    std::unordered_set<int> l1(a1, a1 + 3);
+    std::unordered_set<int> l2(a1, a1 + 3);
+    std::unordered_set<int>::const_iterator i = l2.begin();
+    TEST_LIBCPP_ASSERT_FAILURE(
+        l1.erase(i), "unordered container erase(iterator) called with an iterator not referring to this container");
+  }
+
+  return 0;
 }
diff --git a/libcxx/test/libcxx/containers/unord/unord.set/debug.erase.iter_iter.pass.cpp b/libcxx/test/libcxx/containers/unord/unord.set/debug.erase.iter_iter.pass.cpp
index 5b5b71ab31af2..c714b70ced328 100644
--- a/libcxx/test/libcxx/containers/unord/unord.set/debug.erase.iter_iter.pass.cpp
+++ b/libcxx/test/libcxx/containers/unord/unord.set/debug.erase.iter_iter.pass.cpp
@@ -18,43 +18,43 @@
 #include "check_assertion.h"
 
 int main(int, char**) {
-    // With first iterator from another container
-    {
-        int a1[] = {1, 2, 3};
-        std::unordered_set<int> l1(a1, a1+3);
-        std::unordered_set<int> l2(a1, a1+3);
-        TEST_LIBCPP_ASSERT_FAILURE(
-            l1.erase(l2.cbegin(), std::next(l1.cbegin())),
-            "unordered container::erase(iterator, iterator) called with an iterator not referring to this container");
-    }
-
-    // With second iterator from another container
-    {
-        int a1[] = {1, 2, 3};
-        std::unordered_set<int> l1(a1, a1+3);
-        std::unordered_set<int> l2(a1, a1+3);
-        TEST_LIBCPP_ASSERT_FAILURE(
-            l1.erase(l1.cbegin(), std::next(l2.cbegin())),
-            "unordered container::erase(iterator, iterator) called with an iterator not referring to this container");
-    }
-
-    // With both iterators from another container
-    {
-        int a1[] = {1, 2, 3};
-        std::unordered_set<int> l1(a1, a1+3);
-        std::unordered_set<int> l2(a1, a1+3);
-        TEST_LIBCPP_ASSERT_FAILURE(
-            l1.erase(l2.cbegin(), std::next(l2.cbegin())),
-            "unordered container::erase(iterator, iterator) called with an iterator not referring to this container");
-    }
-
-    // With an invalid range
-    {
-        int a1[] = {1, 2, 3};
-        std::unordered_set<int> l1(a1, a1+3);
-        TEST_LIBCPP_ASSERT_FAILURE(l1.erase(std::next(l1.cbegin()), l1.cbegin()),
-                                "Attempted to increment a non-incrementable unordered container const_iterator");
-    }
-
-    return 0;
+  // With first iterator from another container
+  {
+    int a1[] = {1, 2, 3};
+    std::unordered_set<int> l1(a1, a1 + 3);
+    std::unordered_set<int> l2(a1, a1 + 3);
+    TEST_LIBCPP_ASSERT_FAILURE(
+        l1.erase(l2.cbegin(), std::next(l1.cbegin())),
+        "unordered container::erase(iterator, iterator) called with an iterator not referring to this container");
+  }
+
+  // With second iterator from another container
+  {
+    int a1[] = {1, 2, 3};
+    std::unordered_set<int> l1(a1, a1 + 3);
+    std::unordered_set<int> l2(a1, a1 + 3);
+    TEST_LIBCPP_ASSERT_FAILURE(
+        l1.erase(l1.cbegin(), std::next(l2.cbegin())),
+        "unordered container::erase(iterator, iterator) called with an iterator not referring to this container");
+  }
+
+  // With both iterators from another container
+  {
+    int a1[] = {1, 2, 3};
+    std::unordered_set<int> l1(a1, a1 + 3);
+    std::unordered_set<int> l2(a1, a1 + 3);
+    TEST_LIBCPP_ASSERT_FAILURE(
+        l1.erase(l2.cbegin(), std::next(l2.cbegin())),
+        "unordered container::erase(iterator, iterator) called with an iterator not referring to this container");
+  }
+
+  // With an invalid range
+  {
+    int a1[] = {1, 2, 3};
+    std::unordered_set<int> l1(a1, a1 + 3);
+    TEST_LIBCPP_ASSERT_FAILURE(l1.erase(std::next(l1.cbegin()), l1.cbegin()),
+                               "Attempted to increment a non-incrementable unordered container const_iterator");
+  }
+
+  return 0;
 }
diff --git a/libcxx/test/libcxx/containers/unord/unord.set/debug.insert.hint_const_lvalue.pass.cpp b/libcxx/test/libcxx/containers/unord/unord.set/debug.insert.hint_const_lvalue.pass.cpp
index 8a95111632247..2d4e0e7350831 100644
--- a/libcxx/test/libcxx/containers/unord/unord.set/debug.insert.hint_const_lvalue.pass.cpp
+++ b/libcxx/test/libcxx/containers/unord/unord.set/debug.insert.hint_const_lvalue.pass.cpp
@@ -18,15 +18,16 @@
 #include "check_assertion.h"
 
 int main(int, char**) {
-    typedef std::unordered_set<double> C;
-    typedef C::value_type P;
-    C c;
-    C c2;
-    C::const_iterator e = c2.end();
-    P v(3.5);
-    TEST_LIBCPP_ASSERT_FAILURE(
-        c.insert(e, v),
-        "unordered_set::insert(const_iterator, const value_type&) called with an iterator not referring to this unordered_set");
+  typedef std::unordered_set<double> C;
+  typedef C::value_type P;
+  C c;
+  C c2;
+  C::const_iterator e = c2.end();
+  P v(3.5);
+  TEST_LIBCPP_ASSERT_FAILURE(
+      c.insert(e, v),
+      "unordered_set::insert(const_iterator, const value_type&) called with an iterator not "
+      "referring to this unordered_set");
 
-    return 0;
+  return 0;
 }
diff --git a/libcxx/test/libcxx/containers/unord/unord.set/debug.swap.pass.cpp b/libcxx/test/libcxx/containers/unord/unord.set/debug.swap.pass.cpp
index 12d7fb71977a8..8f362f318f2e3 100644
--- a/libcxx/test/libcxx/containers/unord/unord.set/debug.swap.pass.cpp
+++ b/libcxx/test/libcxx/containers/unord/unord.set/debug.swap.pass.cpp
@@ -22,16 +22,16 @@
 #include "check_assertion.h"
 
 int main(int, char**) {
-    int a1[] = {1, 3, 7, 9, 10};
-    int a2[] = {0, 2, 4, 5, 6, 8, 11};
-    std::unordered_set<int> c1(a1, a1+sizeof(a1)/sizeof(a1[0]));
-    std::unordered_set<int> c2(a2, a2+sizeof(a2)/sizeof(a2[0]));
-    std::unordered_set<int>::iterator i1 = c1.begin();
-    std::unordered_set<int>::iterator i2 = c2.begin();
-    swap(c1, c2);
-    c1.erase(i2);
-    TEST_LIBCPP_ASSERT_FAILURE(
-        c1.erase(i1), "unordered container erase(iterator) called with an iterator not referring to this container");
-
-    return 0;
+  int a1[] = {1, 3, 7, 9, 10};
+  int a2[] = {0, 2, 4, 5, 6, 8, 11};
+  std::unordered_set<int> c1(a1, a1 + sizeof(a1) / sizeof(a1[0]));
+  std::unordered_set<int> c2(a2, a2 + sizeof(a2) / sizeof(a2[0]));
+  std::unordered_set<int>::iterator i1 = c1.begin();
+  std::unordered_set<int>::iterator i2 = c2.begin();
+  swap(c1, c2);
+  c1.erase(i2);
+  TEST_LIBCPP_ASSERT_FAILURE(
+      c1.erase(i1), "unordered container erase(iterator) called with an iterator not referring to this container");
+
+  return 0;
 }
diff --git a/libcxx/test/libcxx/containers/unord/unord.set/missing_hash_specialization.verify.cpp b/libcxx/test/libcxx/containers/unord/unord.set/missing_hash_specialization.verify.cpp
index f6d93c7e6ca57..c66496acc6d07 100644
--- a/libcxx/test/libcxx/containers/unord/unord.set/missing_hash_specialization.verify.cpp
+++ b/libcxx/test/libcxx/containers/unord/unord.set/missing_hash_specialization.verify.cpp
@@ -24,26 +24,26 @@
 using VT = std::pair<int, int>;
 
 struct BadHashNoCopy {
-  BadHashNoCopy() = default;
+  BadHashNoCopy()                     = default;
   BadHashNoCopy(BadHashNoCopy const&) = delete;
 
   template <class T>
-  std::size_t operator()(T const&) const { return 0; }
-};
-
-struct BadHashNoCall {
-
+  std::size_t operator()(T const&) const {
+    return 0;
+  }
 };
 
+struct BadHashNoCall {};
 
 struct GoodHashNoDefault {
   explicit GoodHashNoDefault(void*) {}
   template <class T>
-  std::size_t operator()(T const&) const { return 0; }
+  std::size_t operator()(T const&) const {
+    return 0;
+  }
 };
 
 int main(int, char**) {
-
   {
     using Set = std::unordered_set<VT>;
     Set s; // expected-error@__hash_table:* {{the specified hash does not meet the Hash requirements}}
@@ -63,7 +63,7 @@ int main(int, char**) {
   }
   {
     using Set = std::unordered_set<int, GoodHashNoDefault>;
-    Set s(/*bucketcount*/42, GoodHashNoDefault(nullptr));
+    Set s(/*bucketcount*/ 42, GoodHashNoDefault(nullptr));
   }
 
   return 0;
diff --git a/libcxx/test/libcxx/containers/views/mdspan/extents/assert.ctor_from_array.pass.cpp b/libcxx/test/libcxx/containers/views/mdspan/extents/assert.ctor_from_array.pass.cpp
index 7fe78a788bf87..90cb0c84a063b 100644
--- a/libcxx/test/libcxx/containers/views/mdspan/extents/assert.ctor_from_array.pass.cpp
+++ b/libcxx/test/libcxx/containers/views/mdspan/extents/assert.ctor_from_array.pass.cpp
@@ -43,11 +43,8 @@ int main(int, char**) {
   }
   // mismatch of static extent
   {
-    TEST_LIBCPP_ASSERT_FAILURE(
-        ([] {
-          std::extents<int, D, 5> e1(std::array{1000, 3});
-        }()),
-        "extents construction: mismatch of provided arguments with static extents.");
+    TEST_LIBCPP_ASSERT_FAILURE(([] { std::extents<int, D, 5> e1(std::array{1000, 3}); }()),
+                               "extents construction: mismatch of provided arguments with static extents.");
   }
   // value out of range
   {
diff --git a/libcxx/test/libcxx/containers/views/views.span/span.cons/assert.iter_sent.pass.cpp b/libcxx/test/libcxx/containers/views/views.span/span.cons/assert.iter_sent.pass.cpp
index fa2a98e33b8f0..64e3cc8750cd1 100644
--- a/libcxx/test/libcxx/containers/views/views.span/span.cons/assert.iter_sent.pass.cpp
+++ b/libcxx/test/libcxx/containers/views/views.span/span.cons/assert.iter_sent.pass.cpp
@@ -28,21 +28,31 @@
 #include "check_assertion.h"
 
 int main(int, char**) {
-    {
-        std::array<int, 3> array{0, 1, 2};
-
-        auto invalid_range = [&] { std::span<int> const s(array.end(), array.begin()); (void)s; };
-        TEST_LIBCPP_ASSERT_FAILURE(invalid_range(), "invalid range in span's constructor (iterator, sentinel)");
-    }
-    {
-        std::array<int, 3> array{0, 1, 2};
-
-        auto invalid_range = [&] { std::span<int, 3> const s(array.end(), array.begin()); (void)s; };
-        TEST_LIBCPP_ASSERT_FAILURE(invalid_range(), "invalid range in span's constructor (iterator, sentinel)");
-
-        auto invalid_size = [&] { std::span<int, 3> const s(array.begin(), array.begin() + 2); (void)s; };
-        TEST_LIBCPP_ASSERT_FAILURE(invalid_size(), "invalid range in span's constructor (iterator, sentinel): last - first != extent");
-    }
-
-    return 0;
+  {
+    std::array<int, 3> array{0, 1, 2};
+
+    auto invalid_range = [&] {
+      std::span<int> const s(array.end(), array.begin());
+      (void)s;
+    };
+    TEST_LIBCPP_ASSERT_FAILURE(invalid_range(), "invalid range in span's constructor (iterator, sentinel)");
+  }
+  {
+    std::array<int, 3> array{0, 1, 2};
+
+    auto invalid_range = [&] {
+      std::span<int, 3> const s(array.end(), array.begin());
+      (void)s;
+    };
+    TEST_LIBCPP_ASSERT_FAILURE(invalid_range(), "invalid range in span's constructor (iterator, sentinel)");
+
+    auto invalid_size = [&] {
+      std::span<int, 3> const s(array.begin(), array.begin() + 2);
+      (void)s;
+    };
+    TEST_LIBCPP_ASSERT_FAILURE(
+        invalid_size(), "invalid range in span's constructor (iterator, sentinel): last - first != extent");
+  }
+
+  return 0;
 }
diff --git a/libcxx/test/libcxx/containers/views/views.span/span.cons/assert.other_span.pass.cpp b/libcxx/test/libcxx/containers/views/views.span/span.cons/assert.other_span.pass.cpp
index c288c2acd6b59..9a4e65b0af903 100644
--- a/libcxx/test/libcxx/containers/views/views.span/span.cons/assert.other_span.pass.cpp
+++ b/libcxx/test/libcxx/containers/views/views.span/span.cons/assert.other_span.pass.cpp
@@ -23,11 +23,14 @@
 #include "check_assertion.h"
 
 int main(int, char**) {
-    std::array<int, 3> array{0, 1, 2};
-    std::span<int> other(array.data(), 3);
+  std::array<int, 3> array{0, 1, 2};
+  std::span<int> other(array.data(), 3);
 
-    auto invalid_source = [&] { std::span<int, 2> const s(other); (void)s; };
-    TEST_LIBCPP_ASSERT_FAILURE(invalid_source(), "size mismatch in span's constructor (other span)");
+  auto invalid_source = [&] {
+    std::span<int, 2> const s(other);
+    (void)s;
+  };
+  TEST_LIBCPP_ASSERT_FAILURE(invalid_source(), "size mismatch in span's constructor (other span)");
 
-    return 0;
+  return 0;
 }
diff --git a/libcxx/test/libcxx/containers/views/views.span/span.cons/assert.range.pass.cpp b/libcxx/test/libcxx/containers/views/views.span/span.cons/assert.range.pass.cpp
index 1f9af3e03009a..5e36e9a0418ae 100644
--- a/libcxx/test/libcxx/containers/views/views.span/span.cons/assert.range.pass.cpp
+++ b/libcxx/test/libcxx/containers/views/views.span/span.cons/assert.range.pass.cpp
@@ -23,10 +23,16 @@
 #include "check_assertion.h"
 
 int main(int, char**) {
-    std::vector<int> vec{0, 1, 2}; // must use std::vector instead of std::array, because std::span has a special constructor from std::array
+  std::vector<int> vec{
+      0,
+      1,
+      2}; // must use std::vector instead of std::array, because std::span has a special constructor from std::array
 
-    auto invalid_size = [&] { std::span<int, 2> const s(vec); (void)s; };
-    TEST_LIBCPP_ASSERT_FAILURE(invalid_size(), "size mismatch in span's constructor (range)");
+  auto invalid_size = [&] {
+    std::span<int, 2> const s(vec);
+    (void)s;
+  };
+  TEST_LIBCPP_ASSERT_FAILURE(invalid_size(), "size mismatch in span's constructor (range)");
 
-    return 0;
+  return 0;
 }
diff --git a/libcxx/test/libcxx/containers/views/views.span/span.elem/assert.back.pass.cpp b/libcxx/test/libcxx/containers/views/views.span/span.elem/assert.back.pass.cpp
index ea98fe81ee2f8..7e656b7f0b6e9 100644
--- a/libcxx/test/libcxx/containers/views/views.span/span.elem/assert.back.pass.cpp
+++ b/libcxx/test/libcxx/containers/views/views.span/span.elem/assert.back.pass.cpp
@@ -23,17 +23,17 @@
 #include "check_assertion.h"
 
 int main(int, char**) {
-    {
-        std::array<int, 3> array{0, 1, 2};
-        std::span<int> const s(array.data(), 0);
-        TEST_LIBCPP_ASSERT_FAILURE(s.back(), "span<T>::back() on empty span");
-    }
-
-    {
-        std::array<int, 3> array{0, 1, 2};
-        std::span<int, 0> const s(array.data(), 0);
-        TEST_LIBCPP_ASSERT_FAILURE(s.back(), "span<T, N>::back() on empty span");
-    }
-
-    return 0;
+  {
+    std::array<int, 3> array{0, 1, 2};
+    std::span<int> const s(array.data(), 0);
+    TEST_LIBCPP_ASSERT_FAILURE(s.back(), "span<T>::back() on empty span");
+  }
+
+  {
+    std::array<int, 3> array{0, 1, 2};
+    std::span<int, 0> const s(array.data(), 0);
+    TEST_LIBCPP_ASSERT_FAILURE(s.back(), "span<T, N>::back() on empty span");
+  }
+
+  return 0;
 }
diff --git a/libcxx/test/libcxx/containers/views/views.span/span.elem/assert.front.pass.cpp b/libcxx/test/libcxx/containers/views/views.span/span.elem/assert.front.pass.cpp
index 2660ca1f90c14..0068aad8cc346 100644
--- a/libcxx/test/libcxx/containers/views/views.span/span.elem/assert.front.pass.cpp
+++ b/libcxx/test/libcxx/containers/views/views.span/span.elem/assert.front.pass.cpp
@@ -23,17 +23,17 @@
 #include "check_assertion.h"
 
 int main(int, char**) {
-    {
-        std::array<int, 3> array{0, 1, 2};
-        std::span<int> const s(array.data(), 0);
-        TEST_LIBCPP_ASSERT_FAILURE(s.front(), "span<T>::front() on empty span");
-    }
-
-    {
-        std::array<int, 3> array{0, 1, 2};
-        std::span<int, 0> const s(array.data(), 0);
-        TEST_LIBCPP_ASSERT_FAILURE(s.front(), "span<T, N>::front() on empty span");
-    }
-
-    return 0;
+  {
+    std::array<int, 3> array{0, 1, 2};
+    std::span<int> const s(array.data(), 0);
+    TEST_LIBCPP_ASSERT_FAILURE(s.front(), "span<T>::front() on empty span");
+  }
+
+  {
+    std::array<int, 3> array{0, 1, 2};
+    std::span<int, 0> const s(array.data(), 0);
+    TEST_LIBCPP_ASSERT_FAILURE(s.front(), "span<T, N>::front() on empty span");
+  }
+
+  return 0;
 }
diff --git a/libcxx/test/libcxx/containers/views/views.span/span.elem/assert.op_idx.pass.cpp b/libcxx/test/libcxx/containers/views/views.span/span.elem/assert.op_idx.pass.cpp
index e7d7958844181..501067f740788 100644
--- a/libcxx/test/libcxx/containers/views/views.span/span.elem/assert.op_idx.pass.cpp
+++ b/libcxx/test/libcxx/containers/views/views.span/span.elem/assert.op_idx.pass.cpp
@@ -23,17 +23,17 @@
 #include "check_assertion.h"
 
 int main(int, char**) {
-    {
-        std::array<int, 3> array{0, 1, 2};
-        std::span<int> const s(array.data(), array.size());
-        TEST_LIBCPP_ASSERT_FAILURE(s[3], "span<T>::operator[](index): index out of range");
-    }
-
-    {
-        std::array<int, 3> array{0, 1, 2};
-        std::span<int, 3> const s(array.data(), array.size());
-        TEST_LIBCPP_ASSERT_FAILURE(s[3], "span<T, N>::operator[](index): index out of range");
-    }
-
-    return 0;
+  {
+    std::array<int, 3> array{0, 1, 2};
+    std::span<int> const s(array.data(), array.size());
+    TEST_LIBCPP_ASSERT_FAILURE(s[3], "span<T>::operator[](index): index out of range");
+  }
+
+  {
+    std::array<int, 3> array{0, 1, 2};
+    std::span<int, 3> const s(array.data(), array.size());
+    TEST_LIBCPP_ASSERT_FAILURE(s[3], "span<T, N>::operator[](index): index out of range");
+  }
+
+  return 0;
 }
diff --git a/libcxx/test/libcxx/containers/views/views.span/span.sub/assert.first.pass.cpp b/libcxx/test/libcxx/containers/views/views.span/span.sub/assert.first.pass.cpp
index eb6b442d64849..f241f2a5aadf2 100644
--- a/libcxx/test/libcxx/containers/views/views.span/span.sub/assert.first.pass.cpp
+++ b/libcxx/test/libcxx/containers/views/views.span/span.sub/assert.first.pass.cpp
@@ -23,18 +23,18 @@
 #include "check_assertion.h"
 
 int main(int, char**) {
-    {
-        std::array<int, 3> array{0, 1, 2};
-        std::span<int> const s(array.data(), array.size());
-        TEST_LIBCPP_ASSERT_FAILURE(s.first(4), "span<T>::first(count): count out of range");
-        TEST_LIBCPP_ASSERT_FAILURE(s.first<4>(), "span<T>::first<Count>(): Count out of range");
-    }
-    {
-        std::array<int, 3> array{0, 1, 2};
-        std::span<int, 3> const s(array.data(), array.size());
-        TEST_LIBCPP_ASSERT_FAILURE(s.first(4), "span<T, N>::first(count): count out of range");
-        // s.first<4>() caught at compile-time (tested elsewhere)
-    }
+  {
+    std::array<int, 3> array{0, 1, 2};
+    std::span<int> const s(array.data(), array.size());
+    TEST_LIBCPP_ASSERT_FAILURE(s.first(4), "span<T>::first(count): count out of range");
+    TEST_LIBCPP_ASSERT_FAILURE(s.first<4>(), "span<T>::first<Count>(): Count out of range");
+  }
+  {
+    std::array<int, 3> array{0, 1, 2};
+    std::span<int, 3> const s(array.data(), array.size());
+    TEST_LIBCPP_ASSERT_FAILURE(s.first(4), "span<T, N>::first(count): count out of range");
+    // s.first<4>() caught at compile-time (tested elsewhere)
+  }
 
-    return 0;
+  return 0;
 }
diff --git a/libcxx/test/libcxx/containers/views/views.span/span.sub/assert.last.pass.cpp b/libcxx/test/libcxx/containers/views/views.span/span.sub/assert.last.pass.cpp
index 141735db02a60..032df689b1b88 100644
--- a/libcxx/test/libcxx/containers/views/views.span/span.sub/assert.last.pass.cpp
+++ b/libcxx/test/libcxx/containers/views/views.span/span.sub/assert.last.pass.cpp
@@ -23,18 +23,18 @@
 #include "check_assertion.h"
 
 int main(int, char**) {
-    {
-        std::array<int, 3> array{0, 1, 2};
-        std::span<int> const s(array.data(), array.size());
-        TEST_LIBCPP_ASSERT_FAILURE(s.last(4), "span<T>::last(count): count out of range");
-        TEST_LIBCPP_ASSERT_FAILURE(s.last<4>(), "span<T>::last<Count>(): Count out of range");
-    }
-    {
-        std::array<int, 3> array{0, 1, 2};
-        std::span<int, 3> const s(array.data(), array.size());
-        TEST_LIBCPP_ASSERT_FAILURE(s.last(4), "span<T, N>::last(count): count out of range");
-        // s.last<4>() caught at compile-time (tested elsewhere)
-    }
+  {
+    std::array<int, 3> array{0, 1, 2};
+    std::span<int> const s(array.data(), array.size());
+    TEST_LIBCPP_ASSERT_FAILURE(s.last(4), "span<T>::last(count): count out of range");
+    TEST_LIBCPP_ASSERT_FAILURE(s.last<4>(), "span<T>::last<Count>(): Count out of range");
+  }
+  {
+    std::array<int, 3> array{0, 1, 2};
+    std::span<int, 3> const s(array.data(), array.size());
+    TEST_LIBCPP_ASSERT_FAILURE(s.last(4), "span<T, N>::last(count): count out of range");
+    // s.last<4>() caught at compile-time (tested elsewhere)
+  }
 
-    return 0;
+  return 0;
 }
diff --git a/libcxx/test/libcxx/containers/views/views.span/span.sub/assert.subspan.pass.cpp b/libcxx/test/libcxx/containers/views/views.span/span.sub/assert.subspan.pass.cpp
index 7ed228e96a483..5dd0fa0530189 100644
--- a/libcxx/test/libcxx/containers/views/views.span/span.sub/assert.subspan.pass.cpp
+++ b/libcxx/test/libcxx/containers/views/views.span/span.sub/assert.subspan.pass.cpp
@@ -32,30 +32,30 @@
 #include "check_assertion.h"
 
 int main(int, char**) {
-    {
-        std::array<int, 3> array{0, 1, 2};
-        std::span<int> const s(array.data(), array.size());
-        TEST_LIBCPP_ASSERT_FAILURE(s.subspan(4), "span<T>::subspan(offset, count): offset out of range");
-        TEST_LIBCPP_ASSERT_FAILURE(s.subspan<4>(), "span<T>::subspan<Offset, Count>(): Offset out of range");
-
-        TEST_LIBCPP_ASSERT_FAILURE(s.subspan(0, 4), "span<T>::subspan(offset, count): offset + count out of range");
-        TEST_LIBCPP_ASSERT_FAILURE((s.subspan<0, 4>()), "span<T>::subspan<Offset, Count>(): Offset + Count out of range");
-
-        TEST_LIBCPP_ASSERT_FAILURE(s.subspan(1, 3), "span<T>::subspan(offset, count): offset + count out of range");
-        TEST_LIBCPP_ASSERT_FAILURE((s.subspan<1, 3>()), "span<T>::subspan<Offset, Count>(): Offset + Count out of range");
-    }
-    {
-        std::array<int, 3> array{0, 1, 2};
-        std::span<int, 3> const s(array.data(), array.size());
-        TEST_LIBCPP_ASSERT_FAILURE(s.subspan(4), "span<T, N>::subspan(offset, count): offset out of range");
-        // s.subspan<4>() caught at compile-time (tested in libcxx/test/std/containers/views/views.span/span.sub/subspan.verify.cpp)
-
-        TEST_LIBCPP_ASSERT_FAILURE(s.subspan(0, 4), "span<T, N>::subspan(offset, count): offset + count out of range");
-        // s.subspan<0, 4>() caught at compile-time (tested in libcxx/test/std/containers/views/views.span/span.sub/subspan.verify.cpp)
-
-        TEST_LIBCPP_ASSERT_FAILURE(s.subspan(1, 3), "span<T, N>::subspan(offset, count): offset + count out of range");
-        // s.subspan<1, 3>() caught at compile-time (tested in libcxx/test/std/containers/views/views.span/span.sub/subspan.verify.cpp)
-    }
-
-    return 0;
+  {
+    std::array<int, 3> array{0, 1, 2};
+    std::span<int> const s(array.data(), array.size());
+    TEST_LIBCPP_ASSERT_FAILURE(s.subspan(4), "span<T>::subspan(offset, count): offset out of range");
+    TEST_LIBCPP_ASSERT_FAILURE(s.subspan<4>(), "span<T>::subspan<Offset, Count>(): Offset out of range");
+
+    TEST_LIBCPP_ASSERT_FAILURE(s.subspan(0, 4), "span<T>::subspan(offset, count): offset + count out of range");
+    TEST_LIBCPP_ASSERT_FAILURE((s.subspan<0, 4>()), "span<T>::subspan<Offset, Count>(): Offset + Count out of range");
+
+    TEST_LIBCPP_ASSERT_FAILURE(s.subspan(1, 3), "span<T>::subspan(offset, count): offset + count out of range");
+    TEST_LIBCPP_ASSERT_FAILURE((s.subspan<1, 3>()), "span<T>::subspan<Offset, Count>(): Offset + Count out of range");
+  }
+  {
+    std::array<int, 3> array{0, 1, 2};
+    std::span<int, 3> const s(array.data(), array.size());
+    TEST_LIBCPP_ASSERT_FAILURE(s.subspan(4), "span<T, N>::subspan(offset, count): offset out of range");
+    // s.subspan<4>() caught at compile-time (tested in libcxx/test/std/containers/views/views.span/span.sub/subspan.verify.cpp)
+
+    TEST_LIBCPP_ASSERT_FAILURE(s.subspan(0, 4), "span<T, N>::subspan(offset, count): offset + count out of range");
+    // s.subspan<0, 4>() caught at compile-time (tested in libcxx/test/std/containers/views/views.span/span.sub/subspan.verify.cpp)
+
+    TEST_LIBCPP_ASSERT_FAILURE(s.subspan(1, 3), "span<T, N>::subspan(offset, count): offset + count out of range");
+    // s.subspan<1, 3>() caught at compile-time (tested in libcxx/test/std/containers/views/views.span/span.sub/subspan.verify.cpp)
+  }
+
+  return 0;
 }
diff --git a/libcxx/test/libcxx/diagnostics/chrono.nodiscard.verify.cpp b/libcxx/test/libcxx/diagnostics/chrono.nodiscard.verify.cpp
index 644c5b598c018..bb40e0cfc4e1b 100644
--- a/libcxx/test/libcxx/diagnostics/chrono.nodiscard.verify.cpp
+++ b/libcxx/test/libcxx/diagnostics/chrono.nodiscard.verify.cpp
@@ -102,4 +102,15 @@ void test(std::chrono::time_zone tz, std::chrono::time_zone_link link, std::chro
     zt.get_sys_time();   // expected-warning {{ignoring return value of function declared with 'nodiscard' attribute}}
     zt.get_info();       // expected-warning {{ignoring return value of function declared with 'nodiscard' attribute}}
   }
+
+  { // [time.clock.tai]
+    // expected-warning@+1 {{ignoring return value of function declared with 'nodiscard' attribute}}
+    std::chrono::tai_clock::now();
+
+    // expected-warning@+1 {{ignoring return value of function declared with 'nodiscard' attribute}}
+    std::chrono::tai_clock::to_utc(std::chrono::tai_seconds{});
+
+    // expected-warning@+1 {{ignoring return value of function declared with 'nodiscard' attribute}}
+    std::chrono::tai_clock::from_utc(std::chrono::utc_seconds{});
+  }
 }
diff --git a/libcxx/test/libcxx/feature_test_macro/ftm_metadata.sh.py b/libcxx/test/libcxx/feature_test_macro/ftm_metadata.sh.py
index 541447a5916c0..4f23773f9a0a5 100644
--- a/libcxx/test/libcxx/feature_test_macro/ftm_metadata.sh.py
+++ b/libcxx/test/libcxx/feature_test_macro/ftm_metadata.sh.py
@@ -11,7 +11,7 @@
 import sys
 
 sys.path.append(sys.argv[1])
-from generate_feature_test_macro_components import FeatureTestMacros
+from generate_feature_test_macro_components import FeatureTestMacros, Metadata
 
 
 def test(output, expected):
@@ -19,38 +19,29 @@ def test(output, expected):
 
 
 ftm = FeatureTestMacros(sys.argv[2])
+
 test(
     ftm.ftm_metadata,
     {
-        "__cpp_lib_any": {
-            "headers": ["any"],
-            "test_suite_guard": None,
-            "libcxx_guard": None,
-        },
-        "__cpp_lib_barrier": {
-            "headers": ["barrier"],
-            "test_suite_guard": "!defined(_LIBCPP_VERSION) || (_LIBCPP_HAS_THREADS && _LIBCPP_AVAILABILITY_HAS_SYNC)",
-            "libcxx_guard": "_LIBCPP_HAS_THREADS && _LIBCPP_AVAILABILITY_HAS_SYNC",
-        },
-        "__cpp_lib_format": {
-            "headers": ["format"],
-            "test_suite_guard": None,
-            "libcxx_guard": None,
-        },
-        "__cpp_lib_parallel_algorithm": {
-            "headers": ["algorithm", "numeric"],
-            "test_suite_guard": None,
-            "libcxx_guard": None,
-        },
-        "__cpp_lib_variant": {
-            "headers": ["variant"],
-            "test_suite_guard": None,
-            "libcxx_guard": None,
-        },
-        "__cpp_lib_missing_FTM_in_older_standard": {
-            "headers": [],
-            "test_suite_guard": None,
-            "libcxx_guard": None,
-        },
+        "__cpp_lib_any": Metadata(
+            headers=["any"], test_suite_guard=None, libcxx_guard=None
+        ),
+        "__cpp_lib_barrier": Metadata(
+            headers=["barrier"],
+            test_suite_guard="!defined(_LIBCPP_VERSION) || (_LIBCPP_HAS_THREADS && _LIBCPP_AVAILABILITY_HAS_SYNC)",
+            libcxx_guard="_LIBCPP_HAS_THREADS && _LIBCPP_AVAILABILITY_HAS_SYNC",
+        ),
+        "__cpp_lib_format": Metadata(
+            headers=["format"], test_suite_guard=None, libcxx_guard=None
+        ),
+        "__cpp_lib_parallel_algorithm": Metadata(
+            headers=["algorithm", "numeric"], test_suite_guard=None, libcxx_guard=None
+        ),
+        "__cpp_lib_variant": Metadata(
+            headers=["variant"], test_suite_guard=None, libcxx_guard=None
+        ),
+        "__cpp_lib_missing_FTM_in_older_standard": Metadata(
+            headers=[], test_suite_guard=None, libcxx_guard=None
+        ),
     },
 )
diff --git a/libcxx/test/libcxx/feature_test_macro/version_header_implementation.sh.py b/libcxx/test/libcxx/feature_test_macro/version_header_implementation.sh.py
index b9e087c210767..2771a2f7d8abf 100644
--- a/libcxx/test/libcxx/feature_test_macro/version_header_implementation.sh.py
+++ b/libcxx/test/libcxx/feature_test_macro/version_header_implementation.sh.py
@@ -16,7 +16,7 @@
 del sys.argv[1:3]
 
 sys.path.append(UTILS)
-from generate_feature_test_macro_components import FeatureTestMacros
+from generate_feature_test_macro_components import FeatureTestMacros, VersionHeader
 
 class Test(unittest.TestCase):
     def setUp(self):
@@ -27,114 +27,114 @@ def test_implementation(self):
         expected = {
             "17": [
                 {
-                    "__cpp_lib_any": {
-                        "value": "201606L",
-                        "implemented": True,
-                        "need_undef": False,
-                        "condition": None,
-                    },
+                    "__cpp_lib_any": VersionHeader(
+                        value="201606L",
+                        implemented=True,
+                        need_undef=False,
+                        condition=None,
+                    ),
                 },
                 {
-                    "__cpp_lib_parallel_algorithm": {
-                        "value": "201603L",
-                        "implemented": True,
-                        "need_undef": False,
-                        "condition": None,
-                    },
+                    "__cpp_lib_parallel_algorithm": VersionHeader(
+                        value="201603L",
+                        implemented=True,
+                        need_undef=False,
+                        condition=None,
+                    ),
                 },
                 {
-                    "__cpp_lib_variant": {
-                        "value": "202102L",
-                        "implemented": True,
-                        "need_undef": False,
-                        "condition": None,
-                    },
+                    "__cpp_lib_variant": VersionHeader(
+                        value="202102L",
+                        implemented=True,
+                        need_undef=False,
+                        condition=None,
+                    ),
                 },
                 {
-                    "__cpp_lib_missing_FTM_in_older_standard": {
-                        "value": "2017L",
-                        "implemented": False,
-                        "need_undef": False,
-                        "condition": None,
-                    },
+                    "__cpp_lib_missing_FTM_in_older_standard" : VersionHeader(
+                        value = "2017L",
+                        implemented = False,
+                        need_undef = False,
+                        condition = None,
+                    ),
                 },
             ],
             "20": [
                 {
-                    "__cpp_lib_barrier": {
-                        "value": "201907L",
-                        "implemented": True,
-                        "need_undef": False,
-                        "condition": "_LIBCPP_HAS_THREADS && _LIBCPP_AVAILABILITY_HAS_SYNC",
-                    },
+                    "__cpp_lib_barrier": VersionHeader(
+                        value="201907L",
+                        implemented=True,
+                        need_undef=False,
+                        condition="_LIBCPP_HAS_THREADS && _LIBCPP_AVAILABILITY_HAS_SYNC",
+                    ),
                 },
                 {
-                    "__cpp_lib_format": {
-                        "value": "202110L",
-                        "implemented": False,
-                        "need_undef": False,
-                        "condition": None,
-                    },
+                    "__cpp_lib_format": VersionHeader(
+                        value="202110L",
+                        implemented=False,
+                        need_undef=False,
+                        condition=None,
+                    ),
                 },
                 {
-                    "__cpp_lib_variant": {
-                        "value": "202106L",
-                        "implemented": False,
-                        "need_undef": False,
-                        "condition": None,
-                    },
+                    "__cpp_lib_variant": VersionHeader(
+                        value="202106L",
+                        implemented=False,
+                        need_undef=False,
+                        condition=None,
+                    ),
                 },
                 {
-                    "__cpp_lib_missing_FTM_in_older_standard": {
-                        "value": "2020L",
-                        "implemented": False,
-                        "need_undef": False,
-                        "condition": None,
-                    },
+                    "__cpp_lib_missing_FTM_in_older_standard" : VersionHeader(
+                        value = "2020L",
+                        implemented = False,
+                        need_undef = False,
+                        condition = None,
+                    ),
                 },
             ],
             "23": [
                 {
-                    "__cpp_lib_format": {
-                        "value": "202207L",
-                        "implemented": False,
-                        "need_undef": False,
-                        "condition": None,
-                    },
+                    "__cpp_lib_format": VersionHeader(
+                        value="202207L",
+                        implemented=False,
+                        need_undef=False,
+                        condition=None,
+                    ),
                 },
             ],
             "26": [
                 {
-                    "__cpp_lib_barrier": {
-                        "value": "299900L",
-                        "implemented": True,
-                        "need_undef": True,
-                        "condition": "_LIBCPP_HAS_THREADS && _LIBCPP_AVAILABILITY_HAS_SYNC",
-                    },
+                    "__cpp_lib_barrier": VersionHeader(
+                        value="299900L",
+                        implemented=True,
+                        need_undef=True,
+                        condition="_LIBCPP_HAS_THREADS && _LIBCPP_AVAILABILITY_HAS_SYNC",
+                    ),
                 },
                 {
-                    "__cpp_lib_format": {
-                        "value": "202311L",
-                        "implemented": False,
-                        "need_undef": False,
-                        "condition": None,
-                    },
+                    "__cpp_lib_format": VersionHeader(
+                        value="202311L",
+                        implemented=False,
+                        need_undef=False,
+                        condition=None,
+                    ),
                 },
                 {
-                    "__cpp_lib_variant": {
-                        "value": "202306L",
-                        "implemented": False,
-                        "need_undef": False,
-                        "condition": None,
-                    },
+                    "__cpp_lib_variant": VersionHeader(
+                        value="202306L",
+                        implemented=False,
+                        need_undef=False,
+                        condition=None,
+                    ),
                 },
                 {
-                    "__cpp_lib_missing_FTM_in_older_standard": {
-                        "value": "2026L",
-                        "implemented": False,
-                        "need_undef": False,
-                        "condition": None,
-                    },
+                    "__cpp_lib_missing_FTM_in_older_standard" : VersionHeader(
+                        value = "2026L",
+                        implemented = False,
+                        need_undef = False,
+                        condition = None,
+                    ),
                 },
             ],
         }
diff --git a/libcxx/test/libcxx/fuzzing/random.pass.cpp b/libcxx/test/libcxx/fuzzing/random.pass.cpp
index af80fb8ba18d9..cb074bd60fdc8 100644
--- a/libcxx/test/libcxx/fuzzing/random.pass.cpp
+++ b/libcxx/test/libcxx/fuzzing/random.pass.cpp
@@ -8,7 +8,7 @@
 
 // This test fails because Clang no longer enables -fdelayed-template-parsing
 // by default on Windows with C++20 (#69431).
-// XFAIL: msvc && (clang-18 || clang-19 || clang-20)
+// XFAIL: msvc && (clang-18 || clang-19 || clang-20 || clang-21)
 
 // UNSUPPORTED: c++03, c++11
 
diff --git a/libcxx/test/libcxx/gdb/gdb_pretty_printer_test.sh.cpp b/libcxx/test/libcxx/gdb/gdb_pretty_printer_test.sh.cpp
index ff951d94db0a4..6509bb58140ab 100644
--- a/libcxx/test/libcxx/gdb/gdb_pretty_printer_test.sh.cpp
+++ b/libcxx/test/libcxx/gdb/gdb_pretty_printer_test.sh.cpp
@@ -12,7 +12,7 @@
 // UNSUPPORTED: c++03
 
 // TODO: Investigate these failures which break the CI.
-// UNSUPPORTED: clang-17, clang-18, clang-19
+// UNSUPPORTED: clang-18, clang-19, clang-20
 
 // The Android libc++ tests are run on a non-Android host, connected to an
 // Android device over adb. gdb needs special support to make this work (e.g.
diff --git a/libcxx/test/libcxx/ranges/no_specializations.verify.cpp b/libcxx/test/libcxx/ranges/no_specializations.verify.cpp
index 69d458a920558..489e3a6a73744 100644
--- a/libcxx/test/libcxx/ranges/no_specializations.verify.cpp
+++ b/libcxx/test/libcxx/ranges/no_specializations.verify.cpp
@@ -13,7 +13,9 @@
 
 #include <ranges>
 
-#if !__has_warning("-Winvalid-specialization")
+#include "test_macros.h"
+
+#if !__has_warning("-Winvalid-specialization") || TEST_STD_VER <= 20
 // expected-no-diagnostics
 #else
 struct S {};
diff --git a/libcxx/test/libcxx/ranges/range.adaptors/range.lazy.split/no_unique_address.compile.pass.cpp b/libcxx/test/libcxx/ranges/range.adaptors/range.lazy.split/no_unique_address.compile.pass.cpp
index a0bfb7c4a246b..4a975f472b828 100644
--- a/libcxx/test/libcxx/ranges/range.adaptors/range.lazy.split/no_unique_address.compile.pass.cpp
+++ b/libcxx/test/libcxx/ranges/range.adaptors/range.lazy.split/no_unique_address.compile.pass.cpp
@@ -7,7 +7,6 @@
 //===----------------------------------------------------------------------===//
 
 // UNSUPPORTED: c++03, c++11, c++14, c++17
-// XFAIL: msvc && clang-17
 
 // class lazy_split_view {
 //   _LIBCPP_NO_UNIQUE_ADDRESS _View __base_ = _View();
diff --git a/libcxx/test/libcxx/ranges/range.adaptors/range.split/no_unique_address.compile.pass.cpp b/libcxx/test/libcxx/ranges/range.adaptors/range.split/no_unique_address.compile.pass.cpp
index 694cf1fd0d0e4..7950827dcc868 100644
--- a/libcxx/test/libcxx/ranges/range.adaptors/range.split/no_unique_address.compile.pass.cpp
+++ b/libcxx/test/libcxx/ranges/range.adaptors/range.split/no_unique_address.compile.pass.cpp
@@ -7,7 +7,6 @@
 //===----------------------------------------------------------------------===//
 
 // UNSUPPORTED: c++03, c++11, c++14, c++17
-// XFAIL: msvc && clang-17
 
 // class split_view {
 //   _LIBCPP_NO_UNIQUE_ADDRESS _View __base_ = _View();
diff --git a/libcxx/test/libcxx/ranges/range.factories/range.istream.view/no_unique_address.compile.pass.cpp b/libcxx/test/libcxx/ranges/range.factories/range.istream.view/no_unique_address.compile.pass.cpp
index a77c4e4d1bcdb..56d973d411408 100644
--- a/libcxx/test/libcxx/ranges/range.factories/range.istream.view/no_unique_address.compile.pass.cpp
+++ b/libcxx/test/libcxx/ranges/range.factories/range.istream.view/no_unique_address.compile.pass.cpp
@@ -8,7 +8,6 @@
 
 // UNSUPPORTED: no-localization
 // UNSUPPORTED: c++03, c++11, c++14, c++17
-// XFAIL: msvc && clang-17
 
 // Test the libc++ extension that the value stored in `std::ranges::istream_view` has been marked
 // as _LIBCPP_NO_UNIQUE_ADDRESS
@@ -21,4 +20,3 @@ struct Empty {
 };
 
 static_assert(sizeof(std::ranges::istream_view<Empty>) == sizeof(void*));
-
diff --git a/libcxx/test/libcxx/strings/basic.string/string.modifiers/clear_and_shrink.pass.cpp b/libcxx/test/libcxx/strings/basic.string/string.modifiers/clear_and_shrink.pass.cpp
deleted file mode 100644
index 8f4c9102fde87..0000000000000
--- a/libcxx/test/libcxx/strings/basic.string/string.modifiers/clear_and_shrink.pass.cpp
+++ /dev/null
@@ -1,44 +0,0 @@
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-// <string>
-
-// Call __clear_and_shrink() and ensure string invariants hold
-
-#include <string>
-#include <cassert>
-
-#include "test_macros.h"
-
-TEST_CONSTEXPR_CXX20 bool test() {
-  std::string l = "Long string so that allocation definitely, for sure, absolutely happens. Probably.";
-  std::string s = "short";
-
-  assert(l.__invariants());
-  assert(s.__invariants());
-
-  s.__clear_and_shrink();
-  assert(s.__invariants());
-  assert(s.size() == 0);
-
-  std::string::size_type cap = l.capacity();
-  l.__clear_and_shrink();
-  assert(l.__invariants());
-  assert(l.size() == 0);
-  assert(l.capacity() < cap);
-
-  return true;
-}
-
-int main(int, char**) {
-  test();
-#if TEST_STD_VER > 17
-  static_assert(test());
-#endif
-  return 0;
-}
diff --git a/libcxx/test/libcxx/time/time.clock/time.clock.tai/time.clock.tai.members/assert.from_utc.pass.cpp b/libcxx/test/libcxx/time/time.clock/time.clock.tai/time.clock.tai.members/assert.from_utc.pass.cpp
new file mode 100644
index 0000000000000..f04595ce34ceb
--- /dev/null
+++ b/libcxx/test/libcxx/time/time.clock/time.clock.tai/time.clock.tai.members/assert.from_utc.pass.cpp
@@ -0,0 +1,73 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// REQUIRES: std-at-least-c++20
+// UNSUPPORTED: no-filesystem, no-localization, no-tzdb
+
+// XFAIL: libcpp-has-no-experimental-tzdb
+// XFAIL: availability-tzdb-missing
+
+// REQUIRES: libcpp-hardening-mode={{extensive|debug}}
+// REQUIRES: has-unix-headers
+// XFAIL: libcpp-hardening-mode=debug && availability-verbose_abort-missing
+
+// <chrono>
+//
+// class tai_clock;
+
+// static tai_time<common_type_t<_Duration, seconds>>
+// from_utc(const utc_time<_Duration>& t) noexcept;
+
+#include <chrono>
+
+#include "check_assertion.h"
+
+// The function is specified as
+//   tai_time<common_type_t<Duration, seconds>>{t.time_since_epoch()} + 378691210s
+// When t == t.max() there will be a signed integral overflow (other values too).
+int main(int, char**) {
+  using namespace std::literals::chrono_literals;
+  constexpr std::chrono::seconds offset{378691210};
+
+  (void)std::chrono::tai_clock::from_utc(std::chrono::utc_time<std::chrono::nanoseconds>::max() - offset);
+  TEST_LIBCPP_ASSERT_FAILURE(
+      std::chrono::tai_clock::from_utc(std::chrono::utc_time<std::chrono::nanoseconds>::max() - offset + 1ns),
+      "the UTC to TAI conversion would overflow");
+
+  (void)std::chrono::tai_clock::from_utc(std::chrono::utc_time<std::chrono::microseconds>::max() - offset);
+  TEST_LIBCPP_ASSERT_FAILURE(
+      std::chrono::tai_clock::from_utc(std::chrono::utc_time<std::chrono::microseconds>::max() - offset + 1us),
+      "the UTC to TAI conversion would overflow");
+
+  (void)std::chrono::tai_clock::from_utc(std::chrono::utc_time<std::chrono::milliseconds>::max() - offset);
+  TEST_LIBCPP_ASSERT_FAILURE(
+      std::chrono::tai_clock::from_utc(std::chrono::utc_time<std::chrono::milliseconds>::max() - offset + 1ms),
+      "the UTC to TAI conversion would overflow");
+
+  (void)std::chrono::tai_clock::from_utc(std::chrono::utc_seconds::max() - offset);
+  TEST_LIBCPP_ASSERT_FAILURE(std::chrono::tai_clock::from_utc(std::chrono::utc_seconds::max() - offset + 1s),
+                             "the UTC to TAI conversion would overflow");
+
+  // The conversion uses `common_type_t<Duration, seconds>` so types "larger"
+  // than seconds are converted to seconds. Types "larger" than seconds are
+  // stored in "smaller" intergral and the overflow can never occur.
+
+  // Validate the types can never overflow on all current (and future) supported platforms.
+  static_assert(std::chrono::utc_time<std::chrono::days>::max() <= std::chrono::utc_seconds::max() - offset);
+  static_assert(std::chrono::utc_time<std::chrono::weeks>::max() <= std::chrono::utc_seconds::max() - offset);
+  static_assert(std::chrono::utc_time<std::chrono::months>::max() <= std::chrono::utc_seconds::max() - offset);
+  static_assert(std::chrono::utc_time<std::chrono::years>::max() <= std::chrono::utc_seconds::max() - offset);
+
+  // Validate the run-time conversion works.
+  (void)std::chrono::tai_clock::from_utc(std::chrono::utc_time<std::chrono::days>::max());
+  (void)std::chrono::tai_clock::from_utc(std::chrono::utc_time<std::chrono::weeks>::max());
+  (void)std::chrono::tai_clock::from_utc(std::chrono::utc_time<std::chrono::months>::max());
+  (void)std::chrono::tai_clock::from_utc(std::chrono::utc_time<std::chrono::years>::max());
+
+  return 0;
+}
diff --git a/libcxx/test/libcxx/time/time.clock/time.clock.tai/time.clock.tai.members/assert.to_utc.pass.cpp b/libcxx/test/libcxx/time/time.clock/time.clock.tai/time.clock.tai.members/assert.to_utc.pass.cpp
new file mode 100644
index 0000000000000..624c6c4bce4ca
--- /dev/null
+++ b/libcxx/test/libcxx/time/time.clock/time.clock.tai/time.clock.tai.members/assert.to_utc.pass.cpp
@@ -0,0 +1,73 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// REQUIRES: std-at-least-c++20
+// UNSUPPORTED: no-filesystem, no-localization, no-tzdb
+
+// XFAIL: libcpp-has-no-experimental-tzdb
+// XFAIL: availability-tzdb-missing
+
+// REQUIRES: libcpp-hardening-mode={{extensive|debug}}
+// REQUIRES: has-unix-headers
+// XFAIL: libcpp-hardening-mode=debug && availability-verbose_abort-missing
+
+// <chrono>
+//
+// class tai_clock;
+
+// static utc_time<common_type_t<_Duration, seconds>>
+// to_utc(const tai_time<_Duration>& t) noexcept;
+
+#include <chrono>
+
+#include "check_assertion.h"
+
+// The function is specified as
+//   utc_time<common_type_t<Duration, seconds>>{t.time_since_epoch()} - 378691210s
+// When t == t.min() there will be a signed integral underlow (other values too).
+int main(int, char**) {
+  using namespace std::literals::chrono_literals;
+  constexpr std::chrono::seconds offset{378691210};
+
+  (void)std::chrono::tai_clock::to_utc(std::chrono::tai_time<std::chrono::nanoseconds>::min() + offset);
+  TEST_LIBCPP_ASSERT_FAILURE(
+      std::chrono::tai_clock::to_utc(std::chrono::tai_time<std::chrono::nanoseconds>::min() + offset - 1ns),
+      "the TAI to UTC conversion would underflow");
+
+  (void)std::chrono::tai_clock::to_utc(std::chrono::tai_time<std::chrono::microseconds>::min() + offset);
+  TEST_LIBCPP_ASSERT_FAILURE(
+      std::chrono::tai_clock::to_utc(std::chrono::tai_time<std::chrono::microseconds>::min() + offset - 1us),
+      "the TAI to UTC conversion would underflow");
+
+  (void)std::chrono::tai_clock::to_utc(std::chrono::tai_time<std::chrono::milliseconds>::min() + offset);
+  TEST_LIBCPP_ASSERT_FAILURE(
+      std::chrono::tai_clock::to_utc(std::chrono::tai_time<std::chrono::milliseconds>::min() + offset - 1ms),
+      "the TAI to UTC conversion would underflow");
+
+  (void)std::chrono::tai_clock::to_utc(std::chrono::tai_seconds::min() + offset);
+  TEST_LIBCPP_ASSERT_FAILURE(std::chrono::tai_clock::to_utc(std::chrono::tai_seconds::min() + offset - 1s),
+                             "the TAI to UTC conversion would underflow");
+
+  // The conversion uses `common_type_t<Duration, seconds>` so types "larger"
+  // than seconds are converted to seconds. Types "larger" than seconds are
+  // stored in "smaller" intergral and the underflow can never occur.
+
+  // Validate the types can never underflow on all current (and future) supported platforms.
+  static_assert(std::chrono::tai_time<std::chrono::days>::min() >= std::chrono::tai_seconds::min() + offset);
+  static_assert(std::chrono::tai_time<std::chrono::weeks>::min() >= std::chrono::tai_seconds::min() + offset);
+  static_assert(std::chrono::tai_time<std::chrono::months>::min() >= std::chrono::tai_seconds::min() + offset);
+  static_assert(std::chrono::tai_time<std::chrono::years>::min() >= std::chrono::tai_seconds::min() + offset);
+
+  // Validate the run-time conversion works.
+  (void)std::chrono::tai_clock::to_utc(std::chrono::tai_time<std::chrono::days>::min());
+  (void)std::chrono::tai_clock::to_utc(std::chrono::tai_time<std::chrono::weeks>::min());
+  (void)std::chrono::tai_clock::to_utc(std::chrono::tai_time<std::chrono::months>::min());
+  (void)std::chrono::tai_clock::to_utc(std::chrono::tai_time<std::chrono::years>::min());
+
+  return 0;
+}
diff --git a/libcxx/test/libcxx/time/time.zone/time.zone.timezone/time.zone.members/get_info.sys_time.pass.cpp b/libcxx/test/libcxx/time/time.zone/time.zone.timezone/time.zone.members/get_info.sys_time.pass.cpp
index 7f08c64d5e0e7..afd1273421f39 100644
--- a/libcxx/test/libcxx/time/time.zone/time.zone.timezone/time.zone.members/get_info.sys_time.pass.cpp
+++ b/libcxx/test/libcxx/time/time.zone/time.zone.timezone/time.zone.members/get_info.sys_time.pass.cpp
@@ -157,7 +157,6 @@ static void test_abbrev(std::string_view input, std::string_view expected) {
   TEST_LIBCPP_REQUIRE(result == expected, TEST_WRITE_CONCATENATED("\nExpected ", expected, "\nActual ", result, '\n'));
 }
 
-// This format is valid, however is not used in the tzdata.zi.
 static void percentage_z_format() {
   test_abbrev(
       R"(
@@ -188,6 +187,12 @@ Z Format 0:45 F %z)",
 R F 1999 max - Jan 5 0 -1 foo
 Z Format 0:45 F %z)",
       "-0015");
+
+  test_abbrev(
+      R"(
+Z Format -1:2:20 - LMT 1912 Ja 1 1u
+-1 - %z)",
+      "-01");
 }
 
 int main(int, const char**) {
diff --git a/libcxx/test/libcxx/type_traits/is_trivially_relocatable.compile.pass.cpp b/libcxx/test/libcxx/type_traits/is_trivially_relocatable.compile.pass.cpp
index 674df1d021905..213d06d314a07 100644
--- a/libcxx/test/libcxx/type_traits/is_trivially_relocatable.compile.pass.cpp
+++ b/libcxx/test/libcxx/type_traits/is_trivially_relocatable.compile.pass.cpp
@@ -54,11 +54,17 @@ struct MoveOnlyTriviallyCopyable {
   MoveOnlyTriviallyCopyable(MoveOnlyTriviallyCopyable&&)                 = default;
   MoveOnlyTriviallyCopyable& operator=(MoveOnlyTriviallyCopyable&&)      = default;
 };
-#ifndef _MSC_VER
 static_assert(std::__libcpp_is_trivially_relocatable<MoveOnlyTriviallyCopyable>::value, "");
-#else
-static_assert(!std::__libcpp_is_trivially_relocatable<MoveOnlyTriviallyCopyable>::value, "");
-#endif
+
+struct NonTrivialMoveConstructor {
+  NonTrivialMoveConstructor(NonTrivialMoveConstructor&&);
+};
+static_assert(!std::__libcpp_is_trivially_relocatable<NonTrivialMoveConstructor>::value, "");
+
+struct NonTrivialDestructor {
+  ~NonTrivialDestructor() {}
+};
+static_assert(!std::__libcpp_is_trivially_relocatable<NonTrivialDestructor>::value, "");
 
 // library-internal types
 // ----------------------
diff --git a/libcxx/test/libcxx/type_traits/no_specializations.verify.cpp b/libcxx/test/libcxx/type_traits/no_specializations.verify.cpp
index e6d960667e8c0..807d01e381b49 100644
--- a/libcxx/test/libcxx/type_traits/no_specializations.verify.cpp
+++ b/libcxx/test/libcxx/type_traits/no_specializations.verify.cpp
@@ -36,15 +36,22 @@ SPECIALIZE_TRAIT(make_unsigned);        // expected-error {{cannot be specialize
 SPECIALIZE_TRAIT(remove_all_extents);   // expected-error {{cannot be specialized}}
 SPECIALIZE_TRAIT(remove_const);         // expected-error {{cannot be specialized}}
 SPECIALIZE_TRAIT(remove_cv);            // expected-error {{cannot be specialized}}
-SPECIALIZE_TRAIT(remove_cvref);         // expected-error {{cannot be specialized}}
 SPECIALIZE_TRAIT(remove_extent);        // expected-error {{cannot be specialized}}
 SPECIALIZE_TRAIT(remove_pointer);       // expected-error {{cannot be specialized}}
 SPECIALIZE_TRAIT(remove_reference);     // expected-error {{cannot be specialized}}
 SPECIALIZE_TRAIT(remove_volatile);      // expected-error {{cannot be specialized}}
-SPECIALIZE_TRAIT(type_identity);        // expected-error {{cannot be specialized}}
 SPECIALIZE_TRAIT(underlying_type);      // expected-error {{cannot be specialized}}
-SPECIALIZE_TRAIT(unwrap_reference);     // expected-error {{cannot be specialized}}
-SPECIALIZE_TRAIT(unwrap_ref_decay);     // expected-error {{cannot be specialized}}
+
+#  if TEST_STD_VER <= 17
+SPECIALIZE_TRAIT(result_of); // expected-error {{cannot be specialized}}
+#  endif
+
+#  if TEST_STD_VER >= 20
+SPECIALIZE_TRAIT(remove_cvref);     // expected-error {{cannot be specialized}}
+SPECIALIZE_TRAIT(type_identity);    // expected-error {{cannot be specialized}}
+SPECIALIZE_TRAIT(unwrap_reference); // expected-error {{cannot be specialized}}
+SPECIALIZE_TRAIT(unwrap_ref_decay); // expected-error {{cannot be specialized}}
+#  endif
 
 #  undef SPECIALIZE_TRAIT
 #  define SPECIALIZE_UTT(Trait)                                                                                        \
@@ -96,7 +103,6 @@ SPECIALIZE_UTT(is_move_assignable);                 // expected-error 2 {{cannot
 SPECIALIZE_UTT(is_move_constructible);              // expected-error 2 {{cannot be specialized}}
 SPECIALIZE_BTT(is_nothrow_assignable);              // expected-error 2 {{cannot be specialized}}
 SPECIALIZE_UTT(is_nothrow_constructible);           // expected-error 2 {{cannot be specialized}}
-SPECIALIZE_BTT(is_nothrow_convertible);             // expected-error 2 {{cannot be specialized}}
 SPECIALIZE_UTT(is_nothrow_copy_assignable);         // expected-error 2 {{cannot be specialized}}
 SPECIALIZE_UTT(is_nothrow_copy_constructible);      // expected-error 2 {{cannot be specialized}}
 SPECIALIZE_UTT(is_nothrow_default_constructible);   // expected-error 2 {{cannot be specialized}}
@@ -130,7 +136,6 @@ SPECIALIZE_UTT(is_trivially_default_constructible); // expected-error 2 {{cannot
 SPECIALIZE_UTT(is_trivially_destructible);          // expected-error 2 {{cannot be specialized}}
 SPECIALIZE_UTT(is_trivially_move_assignable);       // expected-error 2 {{cannot be specialized}}
 SPECIALIZE_UTT(is_trivially_move_constructible);    // expected-error 2 {{cannot be specialized}}
-SPECIALIZE_UTT(is_unbounded_array);                 // expected-error 2 {{cannot be specialized}}
 SPECIALIZE_UTT(is_union);                           // expected-error 2 {{cannot be specialized}}
 SPECIALIZE_UTT(is_unsigned);                        // expected-error 2 {{cannot be specialized}}
 SPECIALIZE_UTT(is_void);                            // expected-error 2 {{cannot be specialized}}
@@ -140,11 +145,12 @@ SPECIALIZE_UTT(rank);                               // expected-error 2 {{cannot
 
 #  if TEST_STD_VER <= 17
 SPECIALIZE_UTT(is_literal_type); // expected-error 2 {{cannot be specialized}}
-SPECIALIZE_UTT(result_of);       // expected-error 2 {{cannot be specialized}}
 #  endif
 
 #  if TEST_STD_VER >= 20
-SPECIALIZE_UTT(is_bounded_array); // expected-error 2 {{cannot be specialized}}
+SPECIALIZE_UTT(is_bounded_array);       // expected-error 2 {{cannot be specialized}}
+SPECIALIZE_BTT(is_nothrow_convertible); // expected-error 2 {{cannot be specialized}}
+SPECIALIZE_UTT(is_unbounded_array);     // expected-error 2 {{cannot be specialized}}
 #  endif
 
 #  if TEST_STD_VER >= 23
@@ -171,6 +177,8 @@ struct std::conditional<true, S, S>; // expected-error {{cannot be specialized}}
 template <>
 struct std::enable_if<true, S>; // expected-error {{cannot be specialized}}
 
+#if TEST_STD_VER >= 20
 template <>
 struct std::integral_constant<S, {}>; // expected-error {{cannot be specialized}}
 #endif
+#endif
diff --git a/libcxx/test/libcxx/utilities/format/format.string/format.string.std/escaped_output.pass.cpp b/libcxx/test/libcxx/utilities/format/format.string/format.string.std/escaped_output.pass.cpp
index 430495e1aba8e..d24db360173b1 100644
--- a/libcxx/test/libcxx/utilities/format/format.string/format.string.std/escaped_output.pass.cpp
+++ b/libcxx/test/libcxx/utilities/format/format.string/format.string.std/escaped_output.pass.cpp
@@ -48,7 +48,7 @@ inline constexpr int Cc = 65;
 inline constexpr int Cf = 170;
 inline constexpr int Cs = 2'048;
 inline constexpr int Co = 137'468;
-inline constexpr int Cn = 824'718;
+inline constexpr int Cn = 819'533;
 inline constexpr int C  = Cc + Cf + Cs + Co + Cn;
 
 // This is the final part of the Unicode properties table:
diff --git a/libcxx/test/libcxx/utilities/format/format.string/format.string.std/extended_grapheme_cluster.h b/libcxx/test/libcxx/utilities/format/format.string/format.string.std/extended_grapheme_cluster.h
index eb7500a828ccf..9664622ab4e40 100644
--- a/libcxx/test/libcxx/utilities/format/format.string/format.string.std/extended_grapheme_cluster.h
+++ b/libcxx/test/libcxx/utilities/format/format.string/format.string.std/extended_grapheme_cluster.h
@@ -82,7 +82,7 @@ struct data {
 };
 
 /// The data for UTF-8.
-std::array<data<char>, 1187> data_utf8 = {{
+std::array<data<char>, 1093> data_utf8 = {{
      {"\U00000020\U00000020", {32, 32}, {1, 2}},
      {"\U00000020\U00000308\U00000020", {32, 32}, {3, 4}},
      {"\U00000020\U0000000d", {32, 13}, {1, 2}},
@@ -91,8 +91,8 @@ std::array<data<char>, 1187> data_utf8 = {{
      {"\U00000020\U00000308\U0000000a", {32, 10}, {3, 4}},
      {"\U00000020\U00000001", {32, 1}, {1, 2}},
      {"\U00000020\U00000308\U00000001", {32, 1}, {3, 4}},
-     {"\U00000020\U0000034f", {32}, {3}},
-     {"\U00000020\U00000308\U0000034f", {32}, {5}},
+     {"\U00000020\U0000200c", {32}, {4}},
+     {"\U00000020\U00000308\U0000200c", {32}, {6}},
      {"\U00000020\U0001f1e6", {32, 127462}, {1, 5}},
      {"\U00000020\U00000308\U0001f1e6", {32, 127462}, {3, 7}},
      {"\U00000020\U00000600", {32, 1536}, {1, 3}},
@@ -109,8 +109,6 @@ std::array<data<char>, 1187> data_utf8 = {{
      {"\U00000020\U00000308\U0000ac00", {32, 44032}, {3, 6}},
      {"\U00000020\U0000ac01", {32, 44033}, {1, 4}},
      {"\U00000020\U00000308\U0000ac01", {32, 44033}, {3, 6}},
-     {"\U00000020\U00000900", {32}, {4}},
-     {"\U00000020\U00000308\U00000900", {32}, {6}},
      {"\U00000020\U00000903", {32}, {4}},
      {"\U00000020\U00000308\U00000903", {32}, {6}},
      {"\U00000020\U00000904", {32, 2308}, {1, 4}},
@@ -123,8 +121,8 @@ std::array<data<char>, 1187> data_utf8 = {{
      {"\U00000020\U00000308\U0000231a", {32, 8986}, {3, 6}},
      {"\U00000020\U00000300", {32}, {3}},
      {"\U00000020\U00000308\U00000300", {32}, {5}},
-     {"\U00000020\U0000093c", {32}, {4}},
-     {"\U00000020\U00000308\U0000093c", {32}, {6}},
+     {"\U00000020\U00000900", {32}, {4}},
+     {"\U00000020\U00000308\U00000900", {32}, {6}},
      {"\U00000020\U0000094d", {32}, {4}},
      {"\U00000020\U00000308\U0000094d", {32}, {6}},
      {"\U00000020\U0000200d", {32}, {4}},
@@ -139,8 +137,8 @@ std::array<data<char>, 1187> data_utf8 = {{
      {"\U0000000d\U00000308\U0000000a", {13, 776, 10}, {1, 3, 4}},
      {"\U0000000d\U00000001", {13, 1}, {1, 2}},
      {"\U0000000d\U00000308\U00000001", {13, 776, 1}, {1, 3, 4}},
-     {"\U0000000d\U0000034f", {13, 847}, {1, 3}},
-     {"\U0000000d\U00000308\U0000034f", {13, 776}, {1, 5}},
+     {"\U0000000d\U0000200c", {13, 8204}, {1, 4}},
+     {"\U0000000d\U00000308\U0000200c", {13, 776}, {1, 6}},
      {"\U0000000d\U0001f1e6", {13, 127462}, {1, 5}},
      {"\U0000000d\U00000308\U0001f1e6", {13, 776, 127462}, {1, 3, 7}},
      {"\U0000000d\U00000600", {13, 1536}, {1, 3}},
@@ -157,8 +155,6 @@ std::array<data<char>, 1187> data_utf8 = {{
      {"\U0000000d\U00000308\U0000ac00", {13, 776, 44032}, {1, 3, 6}},
      {"\U0000000d\U0000ac01", {13, 44033}, {1, 4}},
      {"\U0000000d\U00000308\U0000ac01", {13, 776, 44033}, {1, 3, 6}},
-     {"\U0000000d\U00000900", {13, 2304}, {1, 4}},
-     {"\U0000000d\U00000308\U00000900", {13, 776}, {1, 6}},
      {"\U0000000d\U00000903", {13, 2307}, {1, 4}},
      {"\U0000000d\U00000308\U00000903", {13, 776}, {1, 6}},
      {"\U0000000d\U00000904", {13, 2308}, {1, 4}},
@@ -171,8 +167,8 @@ std::array<data<char>, 1187> data_utf8 = {{
      {"\U0000000d\U00000308\U0000231a", {13, 776, 8986}, {1, 3, 6}},
      {"\U0000000d\U00000300", {13, 768}, {1, 3}},
      {"\U0000000d\U00000308\U00000300", {13, 776}, {1, 5}},
-     {"\U0000000d\U0000093c", {13, 2364}, {1, 4}},
-     {"\U0000000d\U00000308\U0000093c", {13, 776}, {1, 6}},
+     {"\U0000000d\U00000900", {13, 2304}, {1, 4}},
+     {"\U0000000d\U00000308\U00000900", {13, 776}, {1, 6}},
      {"\U0000000d\U0000094d", {13, 2381}, {1, 4}},
      {"\U0000000d\U00000308\U0000094d", {13, 776}, {1, 6}},
      {"\U0000000d\U0000200d", {13, 8205}, {1, 4}},
@@ -187,8 +183,8 @@ std::array<data<char>, 1187> data_utf8 = {{
      {"\U0000000a\U00000308\U0000000a", {10, 776, 10}, {1, 3, 4}},
      {"\U0000000a\U00000001", {10, 1}, {1, 2}},
      {"\U0000000a\U00000308\U00000001", {10, 776, 1}, {1, 3, 4}},
-     {"\U0000000a\U0000034f", {10, 847}, {1, 3}},
-     {"\U0000000a\U00000308\U0000034f", {10, 776}, {1, 5}},
+     {"\U0000000a\U0000200c", {10, 8204}, {1, 4}},
+     {"\U0000000a\U00000308\U0000200c", {10, 776}, {1, 6}},
      {"\U0000000a\U0001f1e6", {10, 127462}, {1, 5}},
      {"\U0000000a\U00000308\U0001f1e6", {10, 776, 127462}, {1, 3, 7}},
      {"\U0000000a\U00000600", {10, 1536}, {1, 3}},
@@ -205,8 +201,6 @@ std::array<data<char>, 1187> data_utf8 = {{
      {"\U0000000a\U00000308\U0000ac00", {10, 776, 44032}, {1, 3, 6}},
      {"\U0000000a\U0000ac01", {10, 44033}, {1, 4}},
      {"\U0000000a\U00000308\U0000ac01", {10, 776, 44033}, {1, 3, 6}},
-     {"\U0000000a\U00000900", {10, 2304}, {1, 4}},
-     {"\U0000000a\U00000308\U00000900", {10, 776}, {1, 6}},
      {"\U0000000a\U00000903", {10, 2307}, {1, 4}},
      {"\U0000000a\U00000308\U00000903", {10, 776}, {1, 6}},
      {"\U0000000a\U00000904", {10, 2308}, {1, 4}},
@@ -219,8 +213,8 @@ std::array<data<char>, 1187> data_utf8 = {{
      {"\U0000000a\U00000308\U0000231a", {10, 776, 8986}, {1, 3, 6}},
      {"\U0000000a\U00000300", {10, 768}, {1, 3}},
      {"\U0000000a\U00000308\U00000300", {10, 776}, {1, 5}},
-     {"\U0000000a\U0000093c", {10, 2364}, {1, 4}},
-     {"\U0000000a\U00000308\U0000093c", {10, 776}, {1, 6}},
+     {"\U0000000a\U00000900", {10, 2304}, {1, 4}},
+     {"\U0000000a\U00000308\U00000900", {10, 776}, {1, 6}},
      {"\U0000000a\U0000094d", {10, 2381}, {1, 4}},
      {"\U0000000a\U00000308\U0000094d", {10, 776}, {1, 6}},
      {"\U0000000a\U0000200d", {10, 8205}, {1, 4}},
@@ -235,8 +229,8 @@ std::array<data<char>, 1187> data_utf8 = {{
      {"\U00000001\U00000308\U0000000a", {1, 776, 10}, {1, 3, 4}},
      {"\U00000001\U00000001", {1, 1}, {1, 2}},
      {"\U00000001\U00000308\U00000001", {1, 776, 1}, {1, 3, 4}},
-     {"\U00000001\U0000034f", {1, 847}, {1, 3}},
-     {"\U00000001\U00000308\U0000034f", {1, 776}, {1, 5}},
+     {"\U00000001\U0000200c", {1, 8204}, {1, 4}},
+     {"\U00000001\U00000308\U0000200c", {1, 776}, {1, 6}},
      {"\U00000001\U0001f1e6", {1, 127462}, {1, 5}},
      {"\U00000001\U00000308\U0001f1e6", {1, 776, 127462}, {1, 3, 7}},
      {"\U00000001\U00000600", {1, 1536}, {1, 3}},
@@ -253,8 +247,6 @@ std::array<data<char>, 1187> data_utf8 = {{
      {"\U00000001\U00000308\U0000ac00", {1, 776, 44032}, {1, 3, 6}},
      {"\U00000001\U0000ac01", {1, 44033}, {1, 4}},
      {"\U00000001\U00000308\U0000ac01", {1, 776, 44033}, {1, 3, 6}},
-     {"\U00000001\U00000900", {1, 2304}, {1, 4}},
-     {"\U00000001\U00000308\U00000900", {1, 776}, {1, 6}},
      {"\U00000001\U00000903", {1, 2307}, {1, 4}},
      {"\U00000001\U00000308\U00000903", {1, 776}, {1, 6}},
      {"\U00000001\U00000904", {1, 2308}, {1, 4}},
@@ -267,62 +259,60 @@ std::array<data<char>, 1187> data_utf8 = {{
      {"\U00000001\U00000308\U0000231a", {1, 776, 8986}, {1, 3, 6}},
      {"\U00000001\U00000300", {1, 768}, {1, 3}},
      {"\U00000001\U00000308\U00000300", {1, 776}, {1, 5}},
-     {"\U00000001\U0000093c", {1, 2364}, {1, 4}},
-     {"\U00000001\U00000308\U0000093c", {1, 776}, {1, 6}},
+     {"\U00000001\U00000900", {1, 2304}, {1, 4}},
+     {"\U00000001\U00000308\U00000900", {1, 776}, {1, 6}},
      {"\U00000001\U0000094d", {1, 2381}, {1, 4}},
      {"\U00000001\U00000308\U0000094d", {1, 776}, {1, 6}},
      {"\U00000001\U0000200d", {1, 8205}, {1, 4}},
      {"\U00000001\U00000308\U0000200d", {1, 776}, {1, 6}},
      {"\U00000001\U00000378", {1, 888}, {1, 3}},
      {"\U00000001\U00000308\U00000378", {1, 776, 888}, {1, 3, 5}},
-     {"\U0000034f\U00000020", {847, 32}, {2, 3}},
-     {"\U0000034f\U00000308\U00000020", {847, 32}, {4, 5}},
-     {"\U0000034f\U0000000d", {847, 13}, {2, 3}},
-     {"\U0000034f\U00000308\U0000000d", {847, 13}, {4, 5}},
-     {"\U0000034f\U0000000a", {847, 10}, {2, 3}},
-     {"\U0000034f\U00000308\U0000000a", {847, 10}, {4, 5}},
-     {"\U0000034f\U00000001", {847, 1}, {2, 3}},
-     {"\U0000034f\U00000308\U00000001", {847, 1}, {4, 5}},
-     {"\U0000034f\U0000034f", {847}, {4}},
-     {"\U0000034f\U00000308\U0000034f", {847}, {6}},
-     {"\U0000034f\U0001f1e6", {847, 127462}, {2, 6}},
-     {"\U0000034f\U00000308\U0001f1e6", {847, 127462}, {4, 8}},
-     {"\U0000034f\U00000600", {847, 1536}, {2, 4}},
-     {"\U0000034f\U00000308\U00000600", {847, 1536}, {4, 6}},
-     {"\U0000034f\U00000a03", {847}, {5}},
-     {"\U0000034f\U00000308\U00000a03", {847}, {7}},
-     {"\U0000034f\U00001100", {847, 4352}, {2, 5}},
-     {"\U0000034f\U00000308\U00001100", {847, 4352}, {4, 7}},
-     {"\U0000034f\U00001160", {847, 4448}, {2, 5}},
-     {"\U0000034f\U00000308\U00001160", {847, 4448}, {4, 7}},
-     {"\U0000034f\U000011a8", {847, 4520}, {2, 5}},
-     {"\U0000034f\U00000308\U000011a8", {847, 4520}, {4, 7}},
-     {"\U0000034f\U0000ac00", {847, 44032}, {2, 5}},
-     {"\U0000034f\U00000308\U0000ac00", {847, 44032}, {4, 7}},
-     {"\U0000034f\U0000ac01", {847, 44033}, {2, 5}},
-     {"\U0000034f\U00000308\U0000ac01", {847, 44033}, {4, 7}},
-     {"\U0000034f\U00000900", {847}, {5}},
-     {"\U0000034f\U00000308\U00000900", {847}, {7}},
-     {"\U0000034f\U00000903", {847}, {5}},
-     {"\U0000034f\U00000308\U00000903", {847}, {7}},
-     {"\U0000034f\U00000904", {847, 2308}, {2, 5}},
-     {"\U0000034f\U00000308\U00000904", {847, 2308}, {4, 7}},
-     {"\U0000034f\U00000d4e", {847, 3406}, {2, 5}},
-     {"\U0000034f\U00000308\U00000d4e", {847, 3406}, {4, 7}},
-     {"\U0000034f\U00000915", {847, 2325}, {2, 5}},
-     {"\U0000034f\U00000308\U00000915", {847, 2325}, {4, 7}},
-     {"\U0000034f\U0000231a", {847, 8986}, {2, 5}},
-     {"\U0000034f\U00000308\U0000231a", {847, 8986}, {4, 7}},
-     {"\U0000034f\U00000300", {847}, {4}},
-     {"\U0000034f\U00000308\U00000300", {847}, {6}},
-     {"\U0000034f\U0000093c", {847}, {5}},
-     {"\U0000034f\U00000308\U0000093c", {847}, {7}},
-     {"\U0000034f\U0000094d", {847}, {5}},
-     {"\U0000034f\U00000308\U0000094d", {847}, {7}},
-     {"\U0000034f\U0000200d", {847}, {5}},
-     {"\U0000034f\U00000308\U0000200d", {847}, {7}},
-     {"\U0000034f\U00000378", {847, 888}, {2, 4}},
-     {"\U0000034f\U00000308\U00000378", {847, 888}, {4, 6}},
+     {"\U0000200c\U00000020", {8204, 32}, {3, 4}},
+     {"\U0000200c\U00000308\U00000020", {8204, 32}, {5, 6}},
+     {"\U0000200c\U0000000d", {8204, 13}, {3, 4}},
+     {"\U0000200c\U00000308\U0000000d", {8204, 13}, {5, 6}},
+     {"\U0000200c\U0000000a", {8204, 10}, {3, 4}},
+     {"\U0000200c\U00000308\U0000000a", {8204, 10}, {5, 6}},
+     {"\U0000200c\U00000001", {8204, 1}, {3, 4}},
+     {"\U0000200c\U00000308\U00000001", {8204, 1}, {5, 6}},
+     {"\U0000200c\U0000200c", {8204}, {6}},
+     {"\U0000200c\U00000308\U0000200c", {8204}, {8}},
+     {"\U0000200c\U0001f1e6", {8204, 127462}, {3, 7}},
+     {"\U0000200c\U00000308\U0001f1e6", {8204, 127462}, {5, 9}},
+     {"\U0000200c\U00000600", {8204, 1536}, {3, 5}},
+     {"\U0000200c\U00000308\U00000600", {8204, 1536}, {5, 7}},
+     {"\U0000200c\U00000a03", {8204}, {6}},
+     {"\U0000200c\U00000308\U00000a03", {8204}, {8}},
+     {"\U0000200c\U00001100", {8204, 4352}, {3, 6}},
+     {"\U0000200c\U00000308\U00001100", {8204, 4352}, {5, 8}},
+     {"\U0000200c\U00001160", {8204, 4448}, {3, 6}},
+     {"\U0000200c\U00000308\U00001160", {8204, 4448}, {5, 8}},
+     {"\U0000200c\U000011a8", {8204, 4520}, {3, 6}},
+     {"\U0000200c\U00000308\U000011a8", {8204, 4520}, {5, 8}},
+     {"\U0000200c\U0000ac00", {8204, 44032}, {3, 6}},
+     {"\U0000200c\U00000308\U0000ac00", {8204, 44032}, {5, 8}},
+     {"\U0000200c\U0000ac01", {8204, 44033}, {3, 6}},
+     {"\U0000200c\U00000308\U0000ac01", {8204, 44033}, {5, 8}},
+     {"\U0000200c\U00000903", {8204}, {6}},
+     {"\U0000200c\U00000308\U00000903", {8204}, {8}},
+     {"\U0000200c\U00000904", {8204, 2308}, {3, 6}},
+     {"\U0000200c\U00000308\U00000904", {8204, 2308}, {5, 8}},
+     {"\U0000200c\U00000d4e", {8204, 3406}, {3, 6}},
+     {"\U0000200c\U00000308\U00000d4e", {8204, 3406}, {5, 8}},
+     {"\U0000200c\U00000915", {8204, 2325}, {3, 6}},
+     {"\U0000200c\U00000308\U00000915", {8204, 2325}, {5, 8}},
+     {"\U0000200c\U0000231a", {8204, 8986}, {3, 6}},
+     {"\U0000200c\U00000308\U0000231a", {8204, 8986}, {5, 8}},
+     {"\U0000200c\U00000300", {8204}, {5}},
+     {"\U0000200c\U00000308\U00000300", {8204}, {7}},
+     {"\U0000200c\U00000900", {8204}, {6}},
+     {"\U0000200c\U00000308\U00000900", {8204}, {8}},
+     {"\U0000200c\U0000094d", {8204}, {6}},
+     {"\U0000200c\U00000308\U0000094d", {8204}, {8}},
+     {"\U0000200c\U0000200d", {8204}, {6}},
+     {"\U0000200c\U00000308\U0000200d", {8204}, {8}},
+     {"\U0000200c\U00000378", {8204, 888}, {3, 5}},
+     {"\U0000200c\U00000308\U00000378", {8204, 888}, {5, 7}},
      {"\U0001f1e6\U00000020", {127462, 32}, {4, 5}},
      {"\U0001f1e6\U00000308\U00000020", {127462, 32}, {6, 7}},
      {"\U0001f1e6\U0000000d", {127462, 13}, {4, 5}},
@@ -331,8 +321,8 @@ std::array<data<char>, 1187> data_utf8 = {{
      {"\U0001f1e6\U00000308\U0000000a", {127462, 10}, {6, 7}},
      {"\U0001f1e6\U00000001", {127462, 1}, {4, 5}},
      {"\U0001f1e6\U00000308\U00000001", {127462, 1}, {6, 7}},
-     {"\U0001f1e6\U0000034f", {127462}, {6}},
-     {"\U0001f1e6\U00000308\U0000034f", {127462}, {8}},
+     {"\U0001f1e6\U0000200c", {127462}, {7}},
+     {"\U0001f1e6\U00000308\U0000200c", {127462}, {9}},
      {"\U0001f1e6\U0001f1e6", {127462}, {8}},
      {"\U0001f1e6\U00000308\U0001f1e6", {127462, 127462}, {6, 10}},
      {"\U0001f1e6\U00000600", {127462, 1536}, {4, 6}},
@@ -349,8 +339,6 @@ std::array<data<char>, 1187> data_utf8 = {{
      {"\U0001f1e6\U00000308\U0000ac00", {127462, 44032}, {6, 9}},
      {"\U0001f1e6\U0000ac01", {127462, 44033}, {4, 7}},
      {"\U0001f1e6\U00000308\U0000ac01", {127462, 44033}, {6, 9}},
-     {"\U0001f1e6\U00000900", {127462}, {7}},
-     {"\U0001f1e6\U00000308\U00000900", {127462}, {9}},
      {"\U0001f1e6\U00000903", {127462}, {7}},
      {"\U0001f1e6\U00000308\U00000903", {127462}, {9}},
      {"\U0001f1e6\U00000904", {127462, 2308}, {4, 7}},
@@ -363,8 +351,8 @@ std::array<data<char>, 1187> data_utf8 = {{
      {"\U0001f1e6\U00000308\U0000231a", {127462, 8986}, {6, 9}},
      {"\U0001f1e6\U00000300", {127462}, {6}},
      {"\U0001f1e6\U00000308\U00000300", {127462}, {8}},
-     {"\U0001f1e6\U0000093c", {127462}, {7}},
-     {"\U0001f1e6\U00000308\U0000093c", {127462}, {9}},
+     {"\U0001f1e6\U00000900", {127462}, {7}},
+     {"\U0001f1e6\U00000308\U00000900", {127462}, {9}},
      {"\U0001f1e6\U0000094d", {127462}, {7}},
      {"\U0001f1e6\U00000308\U0000094d", {127462}, {9}},
      {"\U0001f1e6\U0000200d", {127462}, {7}},
@@ -379,8 +367,8 @@ std::array<data<char>, 1187> data_utf8 = {{
      {"\U00000600\U00000308\U0000000a", {1536, 10}, {4, 5}},
      {"\U00000600\U00000001", {1536, 1}, {2, 3}},
      {"\U00000600\U00000308\U00000001", {1536, 1}, {4, 5}},
-     {"\U00000600\U0000034f", {1536}, {4}},
-     {"\U00000600\U00000308\U0000034f", {1536}, {6}},
+     {"\U00000600\U0000200c", {1536}, {5}},
+     {"\U00000600\U00000308\U0000200c", {1536}, {7}},
      {"\U00000600\U0001f1e6", {1536}, {6}},
      {"\U00000600\U00000308\U0001f1e6", {1536, 127462}, {4, 8}},
      {"\U00000600\U00000600", {1536}, {4}},
@@ -397,8 +385,6 @@ std::array<data<char>, 1187> data_utf8 = {{
      {"\U00000600\U00000308\U0000ac00", {1536, 44032}, {4, 7}},
      {"\U00000600\U0000ac01", {1536}, {5}},
      {"\U00000600\U00000308\U0000ac01", {1536, 44033}, {4, 7}},
-     {"\U00000600\U00000900", {1536}, {5}},
-     {"\U00000600\U00000308\U00000900", {1536}, {7}},
      {"\U00000600\U00000903", {1536}, {5}},
      {"\U00000600\U00000308\U00000903", {1536}, {7}},
      {"\U00000600\U00000904", {1536}, {5}},
@@ -411,8 +397,8 @@ std::array<data<char>, 1187> data_utf8 = {{
      {"\U00000600\U00000308\U0000231a", {1536, 8986}, {4, 7}},
      {"\U00000600\U00000300", {1536}, {4}},
      {"\U00000600\U00000308\U00000300", {1536}, {6}},
-     {"\U00000600\U0000093c", {1536}, {5}},
-     {"\U00000600\U00000308\U0000093c", {1536}, {7}},
+     {"\U00000600\U00000900", {1536}, {5}},
+     {"\U00000600\U00000308\U00000900", {1536}, {7}},
      {"\U00000600\U0000094d", {1536}, {5}},
      {"\U00000600\U00000308\U0000094d", {1536}, {7}},
      {"\U00000600\U0000200d", {1536}, {5}},
@@ -427,8 +413,8 @@ std::array<data<char>, 1187> data_utf8 = {{
      {"\U00000a03\U00000308\U0000000a", {2563, 10}, {5, 6}},
      {"\U00000a03\U00000001", {2563, 1}, {3, 4}},
      {"\U00000a03\U00000308\U00000001", {2563, 1}, {5, 6}},
-     {"\U00000a03\U0000034f", {2563}, {5}},
-     {"\U00000a03\U00000308\U0000034f", {2563}, {7}},
+     {"\U00000a03\U0000200c", {2563}, {6}},
+     {"\U00000a03\U00000308\U0000200c", {2563}, {8}},
      {"\U00000a03\U0001f1e6", {2563, 127462}, {3, 7}},
      {"\U00000a03\U00000308\U0001f1e6", {2563, 127462}, {5, 9}},
      {"\U00000a03\U00000600", {2563, 1536}, {3, 5}},
@@ -445,8 +431,6 @@ std::array<data<char>, 1187> data_utf8 = {{
      {"\U00000a03\U00000308\U0000ac00", {2563, 44032}, {5, 8}},
      {"\U00000a03\U0000ac01", {2563, 44033}, {3, 6}},
      {"\U00000a03\U00000308\U0000ac01", {2563, 44033}, {5, 8}},
-     {"\U00000a03\U00000900", {2563}, {6}},
-     {"\U00000a03\U00000308\U00000900", {2563}, {8}},
      {"\U00000a03\U00000903", {2563}, {6}},
      {"\U00000a03\U00000308\U00000903", {2563}, {8}},
      {"\U00000a03\U00000904", {2563, 2308}, {3, 6}},
@@ -459,8 +443,8 @@ std::array<data<char>, 1187> data_utf8 = {{
      {"\U00000a03\U00000308\U0000231a", {2563, 8986}, {5, 8}},
      {"\U00000a03\U00000300", {2563}, {5}},
      {"\U00000a03\U00000308\U00000300", {2563}, {7}},
-     {"\U00000a03\U0000093c", {2563}, {6}},
-     {"\U00000a03\U00000308\U0000093c", {2563}, {8}},
+     {"\U00000a03\U00000900", {2563}, {6}},
+     {"\U00000a03\U00000308\U00000900", {2563}, {8}},
      {"\U00000a03\U0000094d", {2563}, {6}},
      {"\U00000a03\U00000308\U0000094d", {2563}, {8}},
      {"\U00000a03\U0000200d", {2563}, {6}},
@@ -475,8 +459,8 @@ std::array<data<char>, 1187> data_utf8 = {{
      {"\U00001100\U00000308\U0000000a", {4352, 10}, {5, 6}},
      {"\U00001100\U00000001", {4352, 1}, {3, 4}},
      {"\U00001100\U00000308\U00000001", {4352, 1}, {5, 6}},
-     {"\U00001100\U0000034f", {4352}, {5}},
-     {"\U00001100\U00000308\U0000034f", {4352}, {7}},
+     {"\U00001100\U0000200c", {4352}, {6}},
+     {"\U00001100\U00000308\U0000200c", {4352}, {8}},
      {"\U00001100\U0001f1e6", {4352, 127462}, {3, 7}},
      {"\U00001100\U00000308\U0001f1e6", {4352, 127462}, {5, 9}},
      {"\U00001100\U00000600", {4352, 1536}, {3, 5}},
@@ -493,8 +477,6 @@ std::array<data<char>, 1187> data_utf8 = {{
      {"\U00001100\U00000308\U0000ac00", {4352, 44032}, {5, 8}},
      {"\U00001100\U0000ac01", {4352}, {6}},
      {"\U00001100\U00000308\U0000ac01", {4352, 44033}, {5, 8}},
-     {"\U00001100\U00000900", {4352}, {6}},
-     {"\U00001100\U00000308\U00000900", {4352}, {8}},
      {"\U00001100\U00000903", {4352}, {6}},
      {"\U00001100\U00000308\U00000903", {4352}, {8}},
      {"\U00001100\U00000904", {4352, 2308}, {3, 6}},
@@ -507,8 +489,8 @@ std::array<data<char>, 1187> data_utf8 = {{
      {"\U00001100\U00000308\U0000231a", {4352, 8986}, {5, 8}},
      {"\U00001100\U00000300", {4352}, {5}},
      {"\U00001100\U00000308\U00000300", {4352}, {7}},
-     {"\U00001100\U0000093c", {4352}, {6}},
-     {"\U00001100\U00000308\U0000093c", {4352}, {8}},
+     {"\U00001100\U00000900", {4352}, {6}},
+     {"\U00001100\U00000308\U00000900", {4352}, {8}},
      {"\U00001100\U0000094d", {4352}, {6}},
      {"\U00001100\U00000308\U0000094d", {4352}, {8}},
      {"\U00001100\U0000200d", {4352}, {6}},
@@ -523,8 +505,8 @@ std::array<data<char>, 1187> data_utf8 = {{
      {"\U00001160\U00000308\U0000000a", {4448, 10}, {5, 6}},
      {"\U00001160\U00000001", {4448, 1}, {3, 4}},
      {"\U00001160\U00000308\U00000001", {4448, 1}, {5, 6}},
-     {"\U00001160\U0000034f", {4448}, {5}},
-     {"\U00001160\U00000308\U0000034f", {4448}, {7}},
+     {"\U00001160\U0000200c", {4448}, {6}},
+     {"\U00001160\U00000308\U0000200c", {4448}, {8}},
      {"\U00001160\U0001f1e6", {4448, 127462}, {3, 7}},
      {"\U00001160\U00000308\U0001f1e6", {4448, 127462}, {5, 9}},
      {"\U00001160\U00000600", {4448, 1536}, {3, 5}},
@@ -541,8 +523,6 @@ std::array<data<char>, 1187> data_utf8 = {{
      {"\U00001160\U00000308\U0000ac00", {4448, 44032}, {5, 8}},
      {"\U00001160\U0000ac01", {4448, 44033}, {3, 6}},
      {"\U00001160\U00000308\U0000ac01", {4448, 44033}, {5, 8}},
-     {"\U00001160\U00000900", {4448}, {6}},
-     {"\U00001160\U00000308\U00000900", {4448}, {8}},
      {"\U00001160\U00000903", {4448}, {6}},
      {"\U00001160\U00000308\U00000903", {4448}, {8}},
      {"\U00001160\U00000904", {4448, 2308}, {3, 6}},
@@ -555,8 +535,8 @@ std::array<data<char>, 1187> data_utf8 = {{
      {"\U00001160\U00000308\U0000231a", {4448, 8986}, {5, 8}},
      {"\U00001160\U00000300", {4448}, {5}},
      {"\U00001160\U00000308\U00000300", {4448}, {7}},
-     {"\U00001160\U0000093c", {4448}, {6}},
-     {"\U00001160\U00000308\U0000093c", {4448}, {8}},
+     {"\U00001160\U00000900", {4448}, {6}},
+     {"\U00001160\U00000308\U00000900", {4448}, {8}},
      {"\U00001160\U0000094d", {4448}, {6}},
      {"\U00001160\U00000308\U0000094d", {4448}, {8}},
      {"\U00001160\U0000200d", {4448}, {6}},
@@ -571,8 +551,8 @@ std::array<data<char>, 1187> data_utf8 = {{
      {"\U000011a8\U00000308\U0000000a", {4520, 10}, {5, 6}},
      {"\U000011a8\U00000001", {4520, 1}, {3, 4}},
      {"\U000011a8\U00000308\U00000001", {4520, 1}, {5, 6}},
-     {"\U000011a8\U0000034f", {4520}, {5}},
-     {"\U000011a8\U00000308\U0000034f", {4520}, {7}},
+     {"\U000011a8\U0000200c", {4520}, {6}},
+     {"\U000011a8\U00000308\U0000200c", {4520}, {8}},
      {"\U000011a8\U0001f1e6", {4520, 127462}, {3, 7}},
      {"\U000011a8\U00000308\U0001f1e6", {4520, 127462}, {5, 9}},
      {"\U000011a8\U00000600", {4520, 1536}, {3, 5}},
@@ -589,8 +569,6 @@ std::array<data<char>, 1187> data_utf8 = {{
      {"\U000011a8\U00000308\U0000ac00", {4520, 44032}, {5, 8}},
      {"\U000011a8\U0000ac01", {4520, 44033}, {3, 6}},
      {"\U000011a8\U00000308\U0000ac01", {4520, 44033}, {5, 8}},
-     {"\U000011a8\U00000900", {4520}, {6}},
-     {"\U000011a8\U00000308\U00000900", {4520}, {8}},
      {"\U000011a8\U00000903", {4520}, {6}},
      {"\U000011a8\U00000308\U00000903", {4520}, {8}},
      {"\U000011a8\U00000904", {4520, 2308}, {3, 6}},
@@ -603,8 +581,8 @@ std::array<data<char>, 1187> data_utf8 = {{
      {"\U000011a8\U00000308\U0000231a", {4520, 8986}, {5, 8}},
      {"\U000011a8\U00000300", {4520}, {5}},
      {"\U000011a8\U00000308\U00000300", {4520}, {7}},
-     {"\U000011a8\U0000093c", {4520}, {6}},
-     {"\U000011a8\U00000308\U0000093c", {4520}, {8}},
+     {"\U000011a8\U00000900", {4520}, {6}},
+     {"\U000011a8\U00000308\U00000900", {4520}, {8}},
      {"\U000011a8\U0000094d", {4520}, {6}},
      {"\U000011a8\U00000308\U0000094d", {4520}, {8}},
      {"\U000011a8\U0000200d", {4520}, {6}},
@@ -619,8 +597,8 @@ std::array<data<char>, 1187> data_utf8 = {{
      {"\U0000ac00\U00000308\U0000000a", {44032, 10}, {5, 6}},
      {"\U0000ac00\U00000001", {44032, 1}, {3, 4}},
      {"\U0000ac00\U00000308\U00000001", {44032, 1}, {5, 6}},
-     {"\U0000ac00\U0000034f", {44032}, {5}},
-     {"\U0000ac00\U00000308\U0000034f", {44032}, {7}},
+     {"\U0000ac00\U0000200c", {44032}, {6}},
+     {"\U0000ac00\U00000308\U0000200c", {44032}, {8}},
      {"\U0000ac00\U0001f1e6", {44032, 127462}, {3, 7}},
      {"\U0000ac00\U00000308\U0001f1e6", {44032, 127462}, {5, 9}},
      {"\U0000ac00\U00000600", {44032, 1536}, {3, 5}},
@@ -637,8 +615,6 @@ std::array<data<char>, 1187> data_utf8 = {{
      {"\U0000ac00\U00000308\U0000ac00", {44032, 44032}, {5, 8}},
      {"\U0000ac00\U0000ac01", {44032, 44033}, {3, 6}},
      {"\U0000ac00\U00000308\U0000ac01", {44032, 44033}, {5, 8}},
-     {"\U0000ac00\U00000900", {44032}, {6}},
-     {"\U0000ac00\U00000308\U00000900", {44032}, {8}},
      {"\U0000ac00\U00000903", {44032}, {6}},
      {"\U0000ac00\U00000308\U00000903", {44032}, {8}},
      {"\U0000ac00\U00000904", {44032, 2308}, {3, 6}},
@@ -651,8 +627,8 @@ std::array<data<char>, 1187> data_utf8 = {{
      {"\U0000ac00\U00000308\U0000231a", {44032, 8986}, {5, 8}},
      {"\U0000ac00\U00000300", {44032}, {5}},
      {"\U0000ac00\U00000308\U00000300", {44032}, {7}},
-     {"\U0000ac00\U0000093c", {44032}, {6}},
-     {"\U0000ac00\U00000308\U0000093c", {44032}, {8}},
+     {"\U0000ac00\U00000900", {44032}, {6}},
+     {"\U0000ac00\U00000308\U00000900", {44032}, {8}},
      {"\U0000ac00\U0000094d", {44032}, {6}},
      {"\U0000ac00\U00000308\U0000094d", {44032}, {8}},
      {"\U0000ac00\U0000200d", {44032}, {6}},
@@ -667,8 +643,8 @@ std::array<data<char>, 1187> data_utf8 = {{
      {"\U0000ac01\U00000308\U0000000a", {44033, 10}, {5, 6}},
      {"\U0000ac01\U00000001", {44033, 1}, {3, 4}},
      {"\U0000ac01\U00000308\U00000001", {44033, 1}, {5, 6}},
-     {"\U0000ac01\U0000034f", {44033}, {5}},
-     {"\U0000ac01\U00000308\U0000034f", {44033}, {7}},
+     {"\U0000ac01\U0000200c", {44033}, {6}},
+     {"\U0000ac01\U00000308\U0000200c", {44033}, {8}},
      {"\U0000ac01\U0001f1e6", {44033, 127462}, {3, 7}},
      {"\U0000ac01\U00000308\U0001f1e6", {44033, 127462}, {5, 9}},
      {"\U0000ac01\U00000600", {44033, 1536}, {3, 5}},
@@ -685,8 +661,6 @@ std::array<data<char>, 1187> data_utf8 = {{
      {"\U0000ac01\U00000308\U0000ac00", {44033, 44032}, {5, 8}},
      {"\U0000ac01\U0000ac01", {44033, 44033}, {3, 6}},
      {"\U0000ac01\U00000308\U0000ac01", {44033, 44033}, {5, 8}},
-     {"\U0000ac01\U00000900", {44033}, {6}},
-     {"\U0000ac01\U00000308\U00000900", {44033}, {8}},
      {"\U0000ac01\U00000903", {44033}, {6}},
      {"\U0000ac01\U00000308\U00000903", {44033}, {8}},
      {"\U0000ac01\U00000904", {44033, 2308}, {3, 6}},
@@ -699,62 +673,14 @@ std::array<data<char>, 1187> data_utf8 = {{
      {"\U0000ac01\U00000308\U0000231a", {44033, 8986}, {5, 8}},
      {"\U0000ac01\U00000300", {44033}, {5}},
      {"\U0000ac01\U00000308\U00000300", {44033}, {7}},
-     {"\U0000ac01\U0000093c", {44033}, {6}},
-     {"\U0000ac01\U00000308\U0000093c", {44033}, {8}},
+     {"\U0000ac01\U00000900", {44033}, {6}},
+     {"\U0000ac01\U00000308\U00000900", {44033}, {8}},
      {"\U0000ac01\U0000094d", {44033}, {6}},
      {"\U0000ac01\U00000308\U0000094d", {44033}, {8}},
      {"\U0000ac01\U0000200d", {44033}, {6}},
      {"\U0000ac01\U00000308\U0000200d", {44033}, {8}},
      {"\U0000ac01\U00000378", {44033, 888}, {3, 5}},
      {"\U0000ac01\U00000308\U00000378", {44033, 888}, {5, 7}},
-     {"\U00000900\U00000020", {2304, 32}, {3, 4}},
-     {"\U00000900\U00000308\U00000020", {2304, 32}, {5, 6}},
-     {"\U00000900\U0000000d", {2304, 13}, {3, 4}},
-     {"\U00000900\U00000308\U0000000d", {2304, 13}, {5, 6}},
-     {"\U00000900\U0000000a", {2304, 10}, {3, 4}},
-     {"\U00000900\U00000308\U0000000a", {2304, 10}, {5, 6}},
-     {"\U00000900\U00000001", {2304, 1}, {3, 4}},
-     {"\U00000900\U00000308\U00000001", {2304, 1}, {5, 6}},
-     {"\U00000900\U0000034f", {2304}, {5}},
-     {"\U00000900\U00000308\U0000034f", {2304}, {7}},
-     {"\U00000900\U0001f1e6", {2304, 127462}, {3, 7}},
-     {"\U00000900\U00000308\U0001f1e6", {2304, 127462}, {5, 9}},
-     {"\U00000900\U00000600", {2304, 1536}, {3, 5}},
-     {"\U00000900\U00000308\U00000600", {2304, 1536}, {5, 7}},
-     {"\U00000900\U00000a03", {2304}, {6}},
-     {"\U00000900\U00000308\U00000a03", {2304}, {8}},
-     {"\U00000900\U00001100", {2304, 4352}, {3, 6}},
-     {"\U00000900\U00000308\U00001100", {2304, 4352}, {5, 8}},
-     {"\U00000900\U00001160", {2304, 4448}, {3, 6}},
-     {"\U00000900\U00000308\U00001160", {2304, 4448}, {5, 8}},
-     {"\U00000900\U000011a8", {2304, 4520}, {3, 6}},
-     {"\U00000900\U00000308\U000011a8", {2304, 4520}, {5, 8}},
-     {"\U00000900\U0000ac00", {2304, 44032}, {3, 6}},
-     {"\U00000900\U00000308\U0000ac00", {2304, 44032}, {5, 8}},
-     {"\U00000900\U0000ac01", {2304, 44033}, {3, 6}},
-     {"\U00000900\U00000308\U0000ac01", {2304, 44033}, {5, 8}},
-     {"\U00000900\U00000900", {2304}, {6}},
-     {"\U00000900\U00000308\U00000900", {2304}, {8}},
-     {"\U00000900\U00000903", {2304}, {6}},
-     {"\U00000900\U00000308\U00000903", {2304}, {8}},
-     {"\U00000900\U00000904", {2304, 2308}, {3, 6}},
-     {"\U00000900\U00000308\U00000904", {2304, 2308}, {5, 8}},
-     {"\U00000900\U00000d4e", {2304, 3406}, {3, 6}},
-     {"\U00000900\U00000308\U00000d4e", {2304, 3406}, {5, 8}},
-     {"\U00000900\U00000915", {2304, 2325}, {3, 6}},
-     {"\U00000900\U00000308\U00000915", {2304, 2325}, {5, 8}},
-     {"\U00000900\U0000231a", {2304, 8986}, {3, 6}},
-     {"\U00000900\U00000308\U0000231a", {2304, 8986}, {5, 8}},
-     {"\U00000900\U00000300", {2304}, {5}},
-     {"\U00000900\U00000308\U00000300", {2304}, {7}},
-     {"\U00000900\U0000093c", {2304}, {6}},
-     {"\U00000900\U00000308\U0000093c", {2304}, {8}},
-     {"\U00000900\U0000094d", {2304}, {6}},
-     {"\U00000900\U00000308\U0000094d", {2304}, {8}},
-     {"\U00000900\U0000200d", {2304}, {6}},
-     {"\U00000900\U00000308\U0000200d", {2304}, {8}},
-     {"\U00000900\U00000378", {2304, 888}, {3, 5}},
-     {"\U00000900\U00000308\U00000378", {2304, 888}, {5, 7}},
      {"\U00000903\U00000020", {2307, 32}, {3, 4}},
      {"\U00000903\U00000308\U00000020", {2307, 32}, {5, 6}},
      {"\U00000903\U0000000d", {2307, 13}, {3, 4}},
@@ -763,8 +689,8 @@ std::array<data<char>, 1187> data_utf8 = {{
      {"\U00000903\U00000308\U0000000a", {2307, 10}, {5, 6}},
      {"\U00000903\U00000001", {2307, 1}, {3, 4}},
      {"\U00000903\U00000308\U00000001", {2307, 1}, {5, 6}},
-     {"\U00000903\U0000034f", {2307}, {5}},
-     {"\U00000903\U00000308\U0000034f", {2307}, {7}},
+     {"\U00000903\U0000200c", {2307}, {6}},
+     {"\U00000903\U00000308\U0000200c", {2307}, {8}},
      {"\U00000903\U0001f1e6", {2307, 127462}, {3, 7}},
      {"\U00000903\U00000308\U0001f1e6", {2307, 127462}, {5, 9}},
      {"\U00000903\U00000600", {2307, 1536}, {3, 5}},
@@ -781,8 +707,6 @@ std::array<data<char>, 1187> data_utf8 = {{
      {"\U00000903\U00000308\U0000ac00", {2307, 44032}, {5, 8}},
      {"\U00000903\U0000ac01", {2307, 44033}, {3, 6}},
      {"\U00000903\U00000308\U0000ac01", {2307, 44033}, {5, 8}},
-     {"\U00000903\U00000900", {2307}, {6}},
-     {"\U00000903\U00000308\U00000900", {2307}, {8}},
      {"\U00000903\U00000903", {2307}, {6}},
      {"\U00000903\U00000308\U00000903", {2307}, {8}},
      {"\U00000903\U00000904", {2307, 2308}, {3, 6}},
@@ -795,8 +719,8 @@ std::array<data<char>, 1187> data_utf8 = {{
      {"\U00000903\U00000308\U0000231a", {2307, 8986}, {5, 8}},
      {"\U00000903\U00000300", {2307}, {5}},
      {"\U00000903\U00000308\U00000300", {2307}, {7}},
-     {"\U00000903\U0000093c", {2307}, {6}},
-     {"\U00000903\U00000308\U0000093c", {2307}, {8}},
+     {"\U00000903\U00000900", {2307}, {6}},
+     {"\U00000903\U00000308\U00000900", {2307}, {8}},
      {"\U00000903\U0000094d", {2307}, {6}},
      {"\U00000903\U00000308\U0000094d", {2307}, {8}},
      {"\U00000903\U0000200d", {2307}, {6}},
@@ -811,8 +735,8 @@ std::array<data<char>, 1187> data_utf8 = {{
      {"\U00000904\U00000308\U0000000a", {2308, 10}, {5, 6}},
      {"\U00000904\U00000001", {2308, 1}, {3, 4}},
      {"\U00000904\U00000308\U00000001", {2308, 1}, {5, 6}},
-     {"\U00000904\U0000034f", {2308}, {5}},
-     {"\U00000904\U00000308\U0000034f", {2308}, {7}},
+     {"\U00000904\U0000200c", {2308}, {6}},
+     {"\U00000904\U00000308\U0000200c", {2308}, {8}},
      {"\U00000904\U0001f1e6", {2308, 127462}, {3, 7}},
      {"\U00000904\U00000308\U0001f1e6", {2308, 127462}, {5, 9}},
      {"\U00000904\U00000600", {2308, 1536}, {3, 5}},
@@ -829,8 +753,6 @@ std::array<data<char>, 1187> data_utf8 = {{
      {"\U00000904\U00000308\U0000ac00", {2308, 44032}, {5, 8}},
      {"\U00000904\U0000ac01", {2308, 44033}, {3, 6}},
      {"\U00000904\U00000308\U0000ac01", {2308, 44033}, {5, 8}},
-     {"\U00000904\U00000900", {2308}, {6}},
-     {"\U00000904\U00000308\U00000900", {2308}, {8}},
      {"\U00000904\U00000903", {2308}, {6}},
      {"\U00000904\U00000308\U00000903", {2308}, {8}},
      {"\U00000904\U00000904", {2308, 2308}, {3, 6}},
@@ -843,8 +765,8 @@ std::array<data<char>, 1187> data_utf8 = {{
      {"\U00000904\U00000308\U0000231a", {2308, 8986}, {5, 8}},
      {"\U00000904\U00000300", {2308}, {5}},
      {"\U00000904\U00000308\U00000300", {2308}, {7}},
-     {"\U00000904\U0000093c", {2308}, {6}},
-     {"\U00000904\U00000308\U0000093c", {2308}, {8}},
+     {"\U00000904\U00000900", {2308}, {6}},
+     {"\U00000904\U00000308\U00000900", {2308}, {8}},
      {"\U00000904\U0000094d", {2308}, {6}},
      {"\U00000904\U00000308\U0000094d", {2308}, {8}},
      {"\U00000904\U0000200d", {2308}, {6}},
@@ -859,8 +781,8 @@ std::array<data<char>, 1187> data_utf8 = {{
      {"\U00000d4e\U00000308\U0000000a", {3406, 10}, {5, 6}},
      {"\U00000d4e\U00000001", {3406, 1}, {3, 4}},
      {"\U00000d4e\U00000308\U00000001", {3406, 1}, {5, 6}},
-     {"\U00000d4e\U0000034f", {3406}, {5}},
-     {"\U00000d4e\U00000308\U0000034f", {3406}, {7}},
+     {"\U00000d4e\U0000200c", {3406}, {6}},
+     {"\U00000d4e\U00000308\U0000200c", {3406}, {8}},
      {"\U00000d4e\U0001f1e6", {3406}, {7}},
      {"\U00000d4e\U00000308\U0001f1e6", {3406, 127462}, {5, 9}},
      {"\U00000d4e\U00000600", {3406}, {5}},
@@ -877,8 +799,6 @@ std::array<data<char>, 1187> data_utf8 = {{
      {"\U00000d4e\U00000308\U0000ac00", {3406, 44032}, {5, 8}},
      {"\U00000d4e\U0000ac01", {3406}, {6}},
      {"\U00000d4e\U00000308\U0000ac01", {3406, 44033}, {5, 8}},
-     {"\U00000d4e\U00000900", {3406}, {6}},
-     {"\U00000d4e\U00000308\U00000900", {3406}, {8}},
      {"\U00000d4e\U00000903", {3406}, {6}},
      {"\U00000d4e\U00000308\U00000903", {3406}, {8}},
      {"\U00000d4e\U00000904", {3406}, {6}},
@@ -891,8 +811,8 @@ std::array<data<char>, 1187> data_utf8 = {{
      {"\U00000d4e\U00000308\U0000231a", {3406, 8986}, {5, 8}},
      {"\U00000d4e\U00000300", {3406}, {5}},
      {"\U00000d4e\U00000308\U00000300", {3406}, {7}},
-     {"\U00000d4e\U0000093c", {3406}, {6}},
-     {"\U00000d4e\U00000308\U0000093c", {3406}, {8}},
+     {"\U00000d4e\U00000900", {3406}, {6}},
+     {"\U00000d4e\U00000308\U00000900", {3406}, {8}},
      {"\U00000d4e\U0000094d", {3406}, {6}},
      {"\U00000d4e\U00000308\U0000094d", {3406}, {8}},
      {"\U00000d4e\U0000200d", {3406}, {6}},
@@ -907,8 +827,8 @@ std::array<data<char>, 1187> data_utf8 = {{
      {"\U00000915\U00000308\U0000000a", {2325, 10}, {5, 6}},
      {"\U00000915\U00000001", {2325, 1}, {3, 4}},
      {"\U00000915\U00000308\U00000001", {2325, 1}, {5, 6}},
-     {"\U00000915\U0000034f", {2325}, {5}},
-     {"\U00000915\U00000308\U0000034f", {2325}, {7}},
+     {"\U00000915\U0000200c", {2325}, {6}},
+     {"\U00000915\U00000308\U0000200c", {2325}, {8}},
      {"\U00000915\U0001f1e6", {2325, 127462}, {3, 7}},
      {"\U00000915\U00000308\U0001f1e6", {2325, 127462}, {5, 9}},
      {"\U00000915\U00000600", {2325, 1536}, {3, 5}},
@@ -925,8 +845,6 @@ std::array<data<char>, 1187> data_utf8 = {{
      {"\U00000915\U00000308\U0000ac00", {2325, 44032}, {5, 8}},
      {"\U00000915\U0000ac01", {2325, 44033}, {3, 6}},
      {"\U00000915\U00000308\U0000ac01", {2325, 44033}, {5, 8}},
-     {"\U00000915\U00000900", {2325}, {6}},
-     {"\U00000915\U00000308\U00000900", {2325}, {8}},
      {"\U00000915\U00000903", {2325}, {6}},
      {"\U00000915\U00000308\U00000903", {2325}, {8}},
      {"\U00000915\U00000904", {2325, 2308}, {3, 6}},
@@ -939,8 +857,8 @@ std::array<data<char>, 1187> data_utf8 = {{
      {"\U00000915\U00000308\U0000231a", {2325, 8986}, {5, 8}},
      {"\U00000915\U00000300", {2325}, {5}},
      {"\U00000915\U00000308\U00000300", {2325}, {7}},
-     {"\U00000915\U0000093c", {2325}, {6}},
-     {"\U00000915\U00000308\U0000093c", {2325}, {8}},
+     {"\U00000915\U00000900", {2325}, {6}},
+     {"\U00000915\U00000308\U00000900", {2325}, {8}},
      {"\U00000915\U0000094d", {2325}, {6}},
      {"\U00000915\U00000308\U0000094d", {2325}, {8}},
      {"\U00000915\U0000200d", {2325}, {6}},
@@ -955,8 +873,8 @@ std::array<data<char>, 1187> data_utf8 = {{
      {"\U0000231a\U00000308\U0000000a", {8986, 10}, {5, 6}},
      {"\U0000231a\U00000001", {8986, 1}, {3, 4}},
      {"\U0000231a\U00000308\U00000001", {8986, 1}, {5, 6}},
-     {"\U0000231a\U0000034f", {8986}, {5}},
-     {"\U0000231a\U00000308\U0000034f", {8986}, {7}},
+     {"\U0000231a\U0000200c", {8986}, {6}},
+     {"\U0000231a\U00000308\U0000200c", {8986}, {8}},
      {"\U0000231a\U0001f1e6", {8986, 127462}, {3, 7}},
      {"\U0000231a\U00000308\U0001f1e6", {8986, 127462}, {5, 9}},
      {"\U0000231a\U00000600", {8986, 1536}, {3, 5}},
@@ -973,8 +891,6 @@ std::array<data<char>, 1187> data_utf8 = {{
      {"\U0000231a\U00000308\U0000ac00", {8986, 44032}, {5, 8}},
      {"\U0000231a\U0000ac01", {8986, 44033}, {3, 6}},
      {"\U0000231a\U00000308\U0000ac01", {8986, 44033}, {5, 8}},
-     {"\U0000231a\U00000900", {8986}, {6}},
-     {"\U0000231a\U00000308\U00000900", {8986}, {8}},
      {"\U0000231a\U00000903", {8986}, {6}},
      {"\U0000231a\U00000308\U00000903", {8986}, {8}},
      {"\U0000231a\U00000904", {8986, 2308}, {3, 6}},
@@ -987,8 +903,8 @@ std::array<data<char>, 1187> data_utf8 = {{
      {"\U0000231a\U00000308\U0000231a", {8986, 8986}, {5, 8}},
      {"\U0000231a\U00000300", {8986}, {5}},
      {"\U0000231a\U00000308\U00000300", {8986}, {7}},
-     {"\U0000231a\U0000093c", {8986}, {6}},
-     {"\U0000231a\U00000308\U0000093c", {8986}, {8}},
+     {"\U0000231a\U00000900", {8986}, {6}},
+     {"\U0000231a\U00000308\U00000900", {8986}, {8}},
      {"\U0000231a\U0000094d", {8986}, {6}},
      {"\U0000231a\U00000308\U0000094d", {8986}, {8}},
      {"\U0000231a\U0000200d", {8986}, {6}},
@@ -1003,8 +919,8 @@ std::array<data<char>, 1187> data_utf8 = {{
      {"\U00000300\U00000308\U0000000a", {768, 10}, {4, 5}},
      {"\U00000300\U00000001", {768, 1}, {2, 3}},
      {"\U00000300\U00000308\U00000001", {768, 1}, {4, 5}},
-     {"\U00000300\U0000034f", {768}, {4}},
-     {"\U00000300\U00000308\U0000034f", {768}, {6}},
+     {"\U00000300\U0000200c", {768}, {5}},
+     {"\U00000300\U00000308\U0000200c", {768}, {7}},
      {"\U00000300\U0001f1e6", {768, 127462}, {2, 6}},
      {"\U00000300\U00000308\U0001f1e6", {768, 127462}, {4, 8}},
      {"\U00000300\U00000600", {768, 1536}, {2, 4}},
@@ -1021,8 +937,6 @@ std::array<data<char>, 1187> data_utf8 = {{
      {"\U00000300\U00000308\U0000ac00", {768, 44032}, {4, 7}},
      {"\U00000300\U0000ac01", {768, 44033}, {2, 5}},
      {"\U00000300\U00000308\U0000ac01", {768, 44033}, {4, 7}},
-     {"\U00000300\U00000900", {768}, {5}},
-     {"\U00000300\U00000308\U00000900", {768}, {7}},
      {"\U00000300\U00000903", {768}, {5}},
      {"\U00000300\U00000308\U00000903", {768}, {7}},
      {"\U00000300\U00000904", {768, 2308}, {2, 5}},
@@ -1035,62 +949,60 @@ std::array<data<char>, 1187> data_utf8 = {{
      {"\U00000300\U00000308\U0000231a", {768, 8986}, {4, 7}},
      {"\U00000300\U00000300", {768}, {4}},
      {"\U00000300\U00000308\U00000300", {768}, {6}},
-     {"\U00000300\U0000093c", {768}, {5}},
-     {"\U00000300\U00000308\U0000093c", {768}, {7}},
+     {"\U00000300\U00000900", {768}, {5}},
+     {"\U00000300\U00000308\U00000900", {768}, {7}},
      {"\U00000300\U0000094d", {768}, {5}},
      {"\U00000300\U00000308\U0000094d", {768}, {7}},
      {"\U00000300\U0000200d", {768}, {5}},
      {"\U00000300\U00000308\U0000200d", {768}, {7}},
      {"\U00000300\U00000378", {768, 888}, {2, 4}},
      {"\U00000300\U00000308\U00000378", {768, 888}, {4, 6}},
-     {"\U0000093c\U00000020", {2364, 32}, {3, 4}},
-     {"\U0000093c\U00000308\U00000020", {2364, 32}, {5, 6}},
-     {"\U0000093c\U0000000d", {2364, 13}, {3, 4}},
-     {"\U0000093c\U00000308\U0000000d", {2364, 13}, {5, 6}},
-     {"\U0000093c\U0000000a", {2364, 10}, {3, 4}},
-     {"\U0000093c\U00000308\U0000000a", {2364, 10}, {5, 6}},
-     {"\U0000093c\U00000001", {2364, 1}, {3, 4}},
-     {"\U0000093c\U00000308\U00000001", {2364, 1}, {5, 6}},
-     {"\U0000093c\U0000034f", {2364}, {5}},
-     {"\U0000093c\U00000308\U0000034f", {2364}, {7}},
-     {"\U0000093c\U0001f1e6", {2364, 127462}, {3, 7}},
-     {"\U0000093c\U00000308\U0001f1e6", {2364, 127462}, {5, 9}},
-     {"\U0000093c\U00000600", {2364, 1536}, {3, 5}},
-     {"\U0000093c\U00000308\U00000600", {2364, 1536}, {5, 7}},
-     {"\U0000093c\U00000a03", {2364}, {6}},
-     {"\U0000093c\U00000308\U00000a03", {2364}, {8}},
-     {"\U0000093c\U00001100", {2364, 4352}, {3, 6}},
-     {"\U0000093c\U00000308\U00001100", {2364, 4352}, {5, 8}},
-     {"\U0000093c\U00001160", {2364, 4448}, {3, 6}},
-     {"\U0000093c\U00000308\U00001160", {2364, 4448}, {5, 8}},
-     {"\U0000093c\U000011a8", {2364, 4520}, {3, 6}},
-     {"\U0000093c\U00000308\U000011a8", {2364, 4520}, {5, 8}},
-     {"\U0000093c\U0000ac00", {2364, 44032}, {3, 6}},
-     {"\U0000093c\U00000308\U0000ac00", {2364, 44032}, {5, 8}},
-     {"\U0000093c\U0000ac01", {2364, 44033}, {3, 6}},
-     {"\U0000093c\U00000308\U0000ac01", {2364, 44033}, {5, 8}},
-     {"\U0000093c\U00000900", {2364}, {6}},
-     {"\U0000093c\U00000308\U00000900", {2364}, {8}},
-     {"\U0000093c\U00000903", {2364}, {6}},
-     {"\U0000093c\U00000308\U00000903", {2364}, {8}},
-     {"\U0000093c\U00000904", {2364, 2308}, {3, 6}},
-     {"\U0000093c\U00000308\U00000904", {2364, 2308}, {5, 8}},
-     {"\U0000093c\U00000d4e", {2364, 3406}, {3, 6}},
-     {"\U0000093c\U00000308\U00000d4e", {2364, 3406}, {5, 8}},
-     {"\U0000093c\U00000915", {2364, 2325}, {3, 6}},
-     {"\U0000093c\U00000308\U00000915", {2364, 2325}, {5, 8}},
-     {"\U0000093c\U0000231a", {2364, 8986}, {3, 6}},
-     {"\U0000093c\U00000308\U0000231a", {2364, 8986}, {5, 8}},
-     {"\U0000093c\U00000300", {2364}, {5}},
-     {"\U0000093c\U00000308\U00000300", {2364}, {7}},
-     {"\U0000093c\U0000093c", {2364}, {6}},
-     {"\U0000093c\U00000308\U0000093c", {2364}, {8}},
-     {"\U0000093c\U0000094d", {2364}, {6}},
-     {"\U0000093c\U00000308\U0000094d", {2364}, {8}},
-     {"\U0000093c\U0000200d", {2364}, {6}},
-     {"\U0000093c\U00000308\U0000200d", {2364}, {8}},
-     {"\U0000093c\U00000378", {2364, 888}, {3, 5}},
-     {"\U0000093c\U00000308\U00000378", {2364, 888}, {5, 7}},
+     {"\U00000900\U00000020", {2304, 32}, {3, 4}},
+     {"\U00000900\U00000308\U00000020", {2304, 32}, {5, 6}},
+     {"\U00000900\U0000000d", {2304, 13}, {3, 4}},
+     {"\U00000900\U00000308\U0000000d", {2304, 13}, {5, 6}},
+     {"\U00000900\U0000000a", {2304, 10}, {3, 4}},
+     {"\U00000900\U00000308\U0000000a", {2304, 10}, {5, 6}},
+     {"\U00000900\U00000001", {2304, 1}, {3, 4}},
+     {"\U00000900\U00000308\U00000001", {2304, 1}, {5, 6}},
+     {"\U00000900\U0000200c", {2304}, {6}},
+     {"\U00000900\U00000308\U0000200c", {2304}, {8}},
+     {"\U00000900\U0001f1e6", {2304, 127462}, {3, 7}},
+     {"\U00000900\U00000308\U0001f1e6", {2304, 127462}, {5, 9}},
+     {"\U00000900\U00000600", {2304, 1536}, {3, 5}},
+     {"\U00000900\U00000308\U00000600", {2304, 1536}, {5, 7}},
+     {"\U00000900\U00000a03", {2304}, {6}},
+     {"\U00000900\U00000308\U00000a03", {2304}, {8}},
+     {"\U00000900\U00001100", {2304, 4352}, {3, 6}},
+     {"\U00000900\U00000308\U00001100", {2304, 4352}, {5, 8}},
+     {"\U00000900\U00001160", {2304, 4448}, {3, 6}},
+     {"\U00000900\U00000308\U00001160", {2304, 4448}, {5, 8}},
+     {"\U00000900\U000011a8", {2304, 4520}, {3, 6}},
+     {"\U00000900\U00000308\U000011a8", {2304, 4520}, {5, 8}},
+     {"\U00000900\U0000ac00", {2304, 44032}, {3, 6}},
+     {"\U00000900\U00000308\U0000ac00", {2304, 44032}, {5, 8}},
+     {"\U00000900\U0000ac01", {2304, 44033}, {3, 6}},
+     {"\U00000900\U00000308\U0000ac01", {2304, 44033}, {5, 8}},
+     {"\U00000900\U00000903", {2304}, {6}},
+     {"\U00000900\U00000308\U00000903", {2304}, {8}},
+     {"\U00000900\U00000904", {2304, 2308}, {3, 6}},
+     {"\U00000900\U00000308\U00000904", {2304, 2308}, {5, 8}},
+     {"\U00000900\U00000d4e", {2304, 3406}, {3, 6}},
+     {"\U00000900\U00000308\U00000d4e", {2304, 3406}, {5, 8}},
+     {"\U00000900\U00000915", {2304, 2325}, {3, 6}},
+     {"\U00000900\U00000308\U00000915", {2304, 2325}, {5, 8}},
+     {"\U00000900\U0000231a", {2304, 8986}, {3, 6}},
+     {"\U00000900\U00000308\U0000231a", {2304, 8986}, {5, 8}},
+     {"\U00000900\U00000300", {2304}, {5}},
+     {"\U00000900\U00000308\U00000300", {2304}, {7}},
+     {"\U00000900\U00000900", {2304}, {6}},
+     {"\U00000900\U00000308\U00000900", {2304}, {8}},
+     {"\U00000900\U0000094d", {2304}, {6}},
+     {"\U00000900\U00000308\U0000094d", {2304}, {8}},
+     {"\U00000900\U0000200d", {2304}, {6}},
+     {"\U00000900\U00000308\U0000200d", {2304}, {8}},
+     {"\U00000900\U00000378", {2304, 888}, {3, 5}},
+     {"\U00000900\U00000308\U00000378", {2304, 888}, {5, 7}},
      {"\U0000094d\U00000020", {2381, 32}, {3, 4}},
      {"\U0000094d\U00000308\U00000020", {2381, 32}, {5, 6}},
      {"\U0000094d\U0000000d", {2381, 13}, {3, 4}},
@@ -1099,8 +1011,8 @@ std::array<data<char>, 1187> data_utf8 = {{
      {"\U0000094d\U00000308\U0000000a", {2381, 10}, {5, 6}},
      {"\U0000094d\U00000001", {2381, 1}, {3, 4}},
      {"\U0000094d\U00000308\U00000001", {2381, 1}, {5, 6}},
-     {"\U0000094d\U0000034f", {2381}, {5}},
-     {"\U0000094d\U00000308\U0000034f", {2381}, {7}},
+     {"\U0000094d\U0000200c", {2381}, {6}},
+     {"\U0000094d\U00000308\U0000200c", {2381}, {8}},
      {"\U0000094d\U0001f1e6", {2381, 127462}, {3, 7}},
      {"\U0000094d\U00000308\U0001f1e6", {2381, 127462}, {5, 9}},
      {"\U0000094d\U00000600", {2381, 1536}, {3, 5}},
@@ -1117,8 +1029,6 @@ std::array<data<char>, 1187> data_utf8 = {{
      {"\U0000094d\U00000308\U0000ac00", {2381, 44032}, {5, 8}},
      {"\U0000094d\U0000ac01", {2381, 44033}, {3, 6}},
      {"\U0000094d\U00000308\U0000ac01", {2381, 44033}, {5, 8}},
-     {"\U0000094d\U00000900", {2381}, {6}},
-     {"\U0000094d\U00000308\U00000900", {2381}, {8}},
      {"\U0000094d\U00000903", {2381}, {6}},
      {"\U0000094d\U00000308\U00000903", {2381}, {8}},
      {"\U0000094d\U00000904", {2381, 2308}, {3, 6}},
@@ -1131,8 +1041,8 @@ std::array<data<char>, 1187> data_utf8 = {{
      {"\U0000094d\U00000308\U0000231a", {2381, 8986}, {5, 8}},
      {"\U0000094d\U00000300", {2381}, {5}},
      {"\U0000094d\U00000308\U00000300", {2381}, {7}},
-     {"\U0000094d\U0000093c", {2381}, {6}},
-     {"\U0000094d\U00000308\U0000093c", {2381}, {8}},
+     {"\U0000094d\U00000900", {2381}, {6}},
+     {"\U0000094d\U00000308\U00000900", {2381}, {8}},
      {"\U0000094d\U0000094d", {2381}, {6}},
      {"\U0000094d\U00000308\U0000094d", {2381}, {8}},
      {"\U0000094d\U0000200d", {2381}, {6}},
@@ -1147,8 +1057,8 @@ std::array<data<char>, 1187> data_utf8 = {{
      {"\U0000200d\U00000308\U0000000a", {8205, 10}, {5, 6}},
      {"\U0000200d\U00000001", {8205, 1}, {3, 4}},
      {"\U0000200d\U00000308\U00000001", {8205, 1}, {5, 6}},
-     {"\U0000200d\U0000034f", {8205}, {5}},
-     {"\U0000200d\U00000308\U0000034f", {8205}, {7}},
+     {"\U0000200d\U0000200c", {8205}, {6}},
+     {"\U0000200d\U00000308\U0000200c", {8205}, {8}},
      {"\U0000200d\U0001f1e6", {8205, 127462}, {3, 7}},
      {"\U0000200d\U00000308\U0001f1e6", {8205, 127462}, {5, 9}},
      {"\U0000200d\U00000600", {8205, 1536}, {3, 5}},
@@ -1165,8 +1075,6 @@ std::array<data<char>, 1187> data_utf8 = {{
      {"\U0000200d\U00000308\U0000ac00", {8205, 44032}, {5, 8}},
      {"\U0000200d\U0000ac01", {8205, 44033}, {3, 6}},
      {"\U0000200d\U00000308\U0000ac01", {8205, 44033}, {5, 8}},
-     {"\U0000200d\U00000900", {8205}, {6}},
-     {"\U0000200d\U00000308\U00000900", {8205}, {8}},
      {"\U0000200d\U00000903", {8205}, {6}},
      {"\U0000200d\U00000308\U00000903", {8205}, {8}},
      {"\U0000200d\U00000904", {8205, 2308}, {3, 6}},
@@ -1179,8 +1087,8 @@ std::array<data<char>, 1187> data_utf8 = {{
      {"\U0000200d\U00000308\U0000231a", {8205, 8986}, {5, 8}},
      {"\U0000200d\U00000300", {8205}, {5}},
      {"\U0000200d\U00000308\U00000300", {8205}, {7}},
-     {"\U0000200d\U0000093c", {8205}, {6}},
-     {"\U0000200d\U00000308\U0000093c", {8205}, {8}},
+     {"\U0000200d\U00000900", {8205}, {6}},
+     {"\U0000200d\U00000308\U00000900", {8205}, {8}},
      {"\U0000200d\U0000094d", {8205}, {6}},
      {"\U0000200d\U00000308\U0000094d", {8205}, {8}},
      {"\U0000200d\U0000200d", {8205}, {6}},
@@ -1195,8 +1103,8 @@ std::array<data<char>, 1187> data_utf8 = {{
      {"\U00000378\U00000308\U0000000a", {888, 10}, {4, 5}},
      {"\U00000378\U00000001", {888, 1}, {2, 3}},
      {"\U00000378\U00000308\U00000001", {888, 1}, {4, 5}},
-     {"\U00000378\U0000034f", {888}, {4}},
-     {"\U00000378\U00000308\U0000034f", {888}, {6}},
+     {"\U00000378\U0000200c", {888}, {5}},
+     {"\U00000378\U00000308\U0000200c", {888}, {7}},
      {"\U00000378\U0001f1e6", {888, 127462}, {2, 6}},
      {"\U00000378\U00000308\U0001f1e6", {888, 127462}, {4, 8}},
      {"\U00000378\U00000600", {888, 1536}, {2, 4}},
@@ -1213,8 +1121,6 @@ std::array<data<char>, 1187> data_utf8 = {{
      {"\U00000378\U00000308\U0000ac00", {888, 44032}, {4, 7}},
      {"\U00000378\U0000ac01", {888, 44033}, {2, 5}},
      {"\U00000378\U00000308\U0000ac01", {888, 44033}, {4, 7}},
-     {"\U00000378\U00000900", {888}, {5}},
-     {"\U00000378\U00000308\U00000900", {888}, {7}},
      {"\U00000378\U00000903", {888}, {5}},
      {"\U00000378\U00000308\U00000903", {888}, {7}},
      {"\U00000378\U00000904", {888, 2308}, {2, 5}},
@@ -1227,8 +1133,8 @@ std::array<data<char>, 1187> data_utf8 = {{
      {"\U00000378\U00000308\U0000231a", {888, 8986}, {4, 7}},
      {"\U00000378\U00000300", {888}, {4}},
      {"\U00000378\U00000308\U00000300", {888}, {6}},
-     {"\U00000378\U0000093c", {888}, {5}},
-     {"\U00000378\U00000308\U0000093c", {888}, {7}},
+     {"\U00000378\U00000900", {888}, {5}},
+     {"\U00000378\U00000308\U00000900", {888}, {7}},
      {"\U00000378\U0000094d", {888}, {5}},
      {"\U00000378\U00000308\U0000094d", {888}, {7}},
      {"\U00000378\U0000200d", {888}, {5}},
@@ -1277,7 +1183,7 @@ std::array<data<char>, 1187> data_utf8 = {{
 /// since the size of the code units differ the breaks can contain different
 /// values.
 #ifndef TEST_HAS_NO_WIDE_CHARACTERS
-std::array<data<wchar_t>, 1187> data_utf16 = {{
+std::array<data<wchar_t>, 1093> data_utf16 = {{
      {L"\U00000020\U00000020", {32, 32}, {1, 2}},
      {L"\U00000020\U00000308\U00000020", {32, 32}, {2, 3}},
      {L"\U00000020\U0000000d", {32, 13}, {1, 2}},
@@ -1286,8 +1192,8 @@ std::array<data<wchar_t>, 1187> data_utf16 = {{
      {L"\U00000020\U00000308\U0000000a", {32, 10}, {2, 3}},
      {L"\U00000020\U00000001", {32, 1}, {1, 2}},
      {L"\U00000020\U00000308\U00000001", {32, 1}, {2, 3}},
-     {L"\U00000020\U0000034f", {32}, {2}},
-     {L"\U00000020\U00000308\U0000034f", {32}, {3}},
+     {L"\U00000020\U0000200c", {32}, {2}},
+     {L"\U00000020\U00000308\U0000200c", {32}, {3}},
      {L"\U00000020\U0001f1e6", {32, 127462}, {1, 3}},
      {L"\U00000020\U00000308\U0001f1e6", {32, 127462}, {2, 4}},
      {L"\U00000020\U00000600", {32, 1536}, {1, 2}},
@@ -1304,8 +1210,6 @@ std::array<data<wchar_t>, 1187> data_utf16 = {{
      {L"\U00000020\U00000308\U0000ac00", {32, 44032}, {2, 3}},
      {L"\U00000020\U0000ac01", {32, 44033}, {1, 2}},
      {L"\U00000020\U00000308\U0000ac01", {32, 44033}, {2, 3}},
-     {L"\U00000020\U00000900", {32}, {2}},
-     {L"\U00000020\U00000308\U00000900", {32}, {3}},
      {L"\U00000020\U00000903", {32}, {2}},
      {L"\U00000020\U00000308\U00000903", {32}, {3}},
      {L"\U00000020\U00000904", {32, 2308}, {1, 2}},
@@ -1318,8 +1222,8 @@ std::array<data<wchar_t>, 1187> data_utf16 = {{
      {L"\U00000020\U00000308\U0000231a", {32, 8986}, {2, 3}},
      {L"\U00000020\U00000300", {32}, {2}},
      {L"\U00000020\U00000308\U00000300", {32}, {3}},
-     {L"\U00000020\U0000093c", {32}, {2}},
-     {L"\U00000020\U00000308\U0000093c", {32}, {3}},
+     {L"\U00000020\U00000900", {32}, {2}},
+     {L"\U00000020\U00000308\U00000900", {32}, {3}},
      {L"\U00000020\U0000094d", {32}, {2}},
      {L"\U00000020\U00000308\U0000094d", {32}, {3}},
      {L"\U00000020\U0000200d", {32}, {2}},
@@ -1334,8 +1238,8 @@ std::array<data<wchar_t>, 1187> data_utf16 = {{
      {L"\U0000000d\U00000308\U0000000a", {13, 776, 10}, {1, 2, 3}},
      {L"\U0000000d\U00000001", {13, 1}, {1, 2}},
      {L"\U0000000d\U00000308\U00000001", {13, 776, 1}, {1, 2, 3}},
-     {L"\U0000000d\U0000034f", {13, 847}, {1, 2}},
-     {L"\U0000000d\U00000308\U0000034f", {13, 776}, {1, 3}},
+     {L"\U0000000d\U0000200c", {13, 8204}, {1, 2}},
+     {L"\U0000000d\U00000308\U0000200c", {13, 776}, {1, 3}},
      {L"\U0000000d\U0001f1e6", {13, 127462}, {1, 3}},
      {L"\U0000000d\U00000308\U0001f1e6", {13, 776, 127462}, {1, 2, 4}},
      {L"\U0000000d\U00000600", {13, 1536}, {1, 2}},
@@ -1352,8 +1256,6 @@ std::array<data<wchar_t>, 1187> data_utf16 = {{
      {L"\U0000000d\U00000308\U0000ac00", {13, 776, 44032}, {1, 2, 3}},
      {L"\U0000000d\U0000ac01", {13, 44033}, {1, 2}},
      {L"\U0000000d\U00000308\U0000ac01", {13, 776, 44033}, {1, 2, 3}},
-     {L"\U0000000d\U00000900", {13, 2304}, {1, 2}},
-     {L"\U0000000d\U00000308\U00000900", {13, 776}, {1, 3}},
      {L"\U0000000d\U00000903", {13, 2307}, {1, 2}},
      {L"\U0000000d\U00000308\U00000903", {13, 776}, {1, 3}},
      {L"\U0000000d\U00000904", {13, 2308}, {1, 2}},
@@ -1366,8 +1268,8 @@ std::array<data<wchar_t>, 1187> data_utf16 = {{
      {L"\U0000000d\U00000308\U0000231a", {13, 776, 8986}, {1, 2, 3}},
      {L"\U0000000d\U00000300", {13, 768}, {1, 2}},
      {L"\U0000000d\U00000308\U00000300", {13, 776}, {1, 3}},
-     {L"\U0000000d\U0000093c", {13, 2364}, {1, 2}},
-     {L"\U0000000d\U00000308\U0000093c", {13, 776}, {1, 3}},
+     {L"\U0000000d\U00000900", {13, 2304}, {1, 2}},
+     {L"\U0000000d\U00000308\U00000900", {13, 776}, {1, 3}},
      {L"\U0000000d\U0000094d", {13, 2381}, {1, 2}},
      {L"\U0000000d\U00000308\U0000094d", {13, 776}, {1, 3}},
      {L"\U0000000d\U0000200d", {13, 8205}, {1, 2}},
@@ -1382,8 +1284,8 @@ std::array<data<wchar_t>, 1187> data_utf16 = {{
      {L"\U0000000a\U00000308\U0000000a", {10, 776, 10}, {1, 2, 3}},
      {L"\U0000000a\U00000001", {10, 1}, {1, 2}},
      {L"\U0000000a\U00000308\U00000001", {10, 776, 1}, {1, 2, 3}},
-     {L"\U0000000a\U0000034f", {10, 847}, {1, 2}},
-     {L"\U0000000a\U00000308\U0000034f", {10, 776}, {1, 3}},
+     {L"\U0000000a\U0000200c", {10, 8204}, {1, 2}},
+     {L"\U0000000a\U00000308\U0000200c", {10, 776}, {1, 3}},
      {L"\U0000000a\U0001f1e6", {10, 127462}, {1, 3}},
      {L"\U0000000a\U00000308\U0001f1e6", {10, 776, 127462}, {1, 2, 4}},
      {L"\U0000000a\U00000600", {10, 1536}, {1, 2}},
@@ -1400,8 +1302,6 @@ std::array<data<wchar_t>, 1187> data_utf16 = {{
      {L"\U0000000a\U00000308\U0000ac00", {10, 776, 44032}, {1, 2, 3}},
      {L"\U0000000a\U0000ac01", {10, 44033}, {1, 2}},
      {L"\U0000000a\U00000308\U0000ac01", {10, 776, 44033}, {1, 2, 3}},
-     {L"\U0000000a\U00000900", {10, 2304}, {1, 2}},
-     {L"\U0000000a\U00000308\U00000900", {10, 776}, {1, 3}},
      {L"\U0000000a\U00000903", {10, 2307}, {1, 2}},
      {L"\U0000000a\U00000308\U00000903", {10, 776}, {1, 3}},
      {L"\U0000000a\U00000904", {10, 2308}, {1, 2}},
@@ -1414,8 +1314,8 @@ std::array<data<wchar_t>, 1187> data_utf16 = {{
      {L"\U0000000a\U00000308\U0000231a", {10, 776, 8986}, {1, 2, 3}},
      {L"\U0000000a\U00000300", {10, 768}, {1, 2}},
      {L"\U0000000a\U00000308\U00000300", {10, 776}, {1, 3}},
-     {L"\U0000000a\U0000093c", {10, 2364}, {1, 2}},
-     {L"\U0000000a\U00000308\U0000093c", {10, 776}, {1, 3}},
+     {L"\U0000000a\U00000900", {10, 2304}, {1, 2}},
+     {L"\U0000000a\U00000308\U00000900", {10, 776}, {1, 3}},
      {L"\U0000000a\U0000094d", {10, 2381}, {1, 2}},
      {L"\U0000000a\U00000308\U0000094d", {10, 776}, {1, 3}},
      {L"\U0000000a\U0000200d", {10, 8205}, {1, 2}},
@@ -1430,8 +1330,8 @@ std::array<data<wchar_t>, 1187> data_utf16 = {{
      {L"\U00000001\U00000308\U0000000a", {1, 776, 10}, {1, 2, 3}},
      {L"\U00000001\U00000001", {1, 1}, {1, 2}},
      {L"\U00000001\U00000308\U00000001", {1, 776, 1}, {1, 2, 3}},
-     {L"\U00000001\U0000034f", {1, 847}, {1, 2}},
-     {L"\U00000001\U00000308\U0000034f", {1, 776}, {1, 3}},
+     {L"\U00000001\U0000200c", {1, 8204}, {1, 2}},
+     {L"\U00000001\U00000308\U0000200c", {1, 776}, {1, 3}},
      {L"\U00000001\U0001f1e6", {1, 127462}, {1, 3}},
      {L"\U00000001\U00000308\U0001f1e6", {1, 776, 127462}, {1, 2, 4}},
      {L"\U00000001\U00000600", {1, 1536}, {1, 2}},
@@ -1448,8 +1348,6 @@ std::array<data<wchar_t>, 1187> data_utf16 = {{
      {L"\U00000001\U00000308\U0000ac00", {1, 776, 44032}, {1, 2, 3}},
      {L"\U00000001\U0000ac01", {1, 44033}, {1, 2}},
      {L"\U00000001\U00000308\U0000ac01", {1, 776, 44033}, {1, 2, 3}},
-     {L"\U00000001\U00000900", {1, 2304}, {1, 2}},
-     {L"\U00000001\U00000308\U00000900", {1, 776}, {1, 3}},
      {L"\U00000001\U00000903", {1, 2307}, {1, 2}},
      {L"\U00000001\U00000308\U00000903", {1, 776}, {1, 3}},
      {L"\U00000001\U00000904", {1, 2308}, {1, 2}},
@@ -1462,62 +1360,60 @@ std::array<data<wchar_t>, 1187> data_utf16 = {{
      {L"\U00000001\U00000308\U0000231a", {1, 776, 8986}, {1, 2, 3}},
      {L"\U00000001\U00000300", {1, 768}, {1, 2}},
      {L"\U00000001\U00000308\U00000300", {1, 776}, {1, 3}},
-     {L"\U00000001\U0000093c", {1, 2364}, {1, 2}},
-     {L"\U00000001\U00000308\U0000093c", {1, 776}, {1, 3}},
+     {L"\U00000001\U00000900", {1, 2304}, {1, 2}},
+     {L"\U00000001\U00000308\U00000900", {1, 776}, {1, 3}},
      {L"\U00000001\U0000094d", {1, 2381}, {1, 2}},
      {L"\U00000001\U00000308\U0000094d", {1, 776}, {1, 3}},
      {L"\U00000001\U0000200d", {1, 8205}, {1, 2}},
      {L"\U00000001\U00000308\U0000200d", {1, 776}, {1, 3}},
      {L"\U00000001\U00000378", {1, 888}, {1, 2}},
      {L"\U00000001\U00000308\U00000378", {1, 776, 888}, {1, 2, 3}},
-     {L"\U0000034f\U00000020", {847, 32}, {1, 2}},
-     {L"\U0000034f\U00000308\U00000020", {847, 32}, {2, 3}},
-     {L"\U0000034f\U0000000d", {847, 13}, {1, 2}},
-     {L"\U0000034f\U00000308\U0000000d", {847, 13}, {2, 3}},
-     {L"\U0000034f\U0000000a", {847, 10}, {1, 2}},
-     {L"\U0000034f\U00000308\U0000000a", {847, 10}, {2, 3}},
-     {L"\U0000034f\U00000001", {847, 1}, {1, 2}},
-     {L"\U0000034f\U00000308\U00000001", {847, 1}, {2, 3}},
-     {L"\U0000034f\U0000034f", {847}, {2}},
-     {L"\U0000034f\U00000308\U0000034f", {847}, {3}},
-     {L"\U0000034f\U0001f1e6", {847, 127462}, {1, 3}},
-     {L"\U0000034f\U00000308\U0001f1e6", {847, 127462}, {2, 4}},
-     {L"\U0000034f\U00000600", {847, 1536}, {1, 2}},
-     {L"\U0000034f\U00000308\U00000600", {847, 1536}, {2, 3}},
-     {L"\U0000034f\U00000a03", {847}, {2}},
-     {L"\U0000034f\U00000308\U00000a03", {847}, {3}},
-     {L"\U0000034f\U00001100", {847, 4352}, {1, 2}},
-     {L"\U0000034f\U00000308\U00001100", {847, 4352}, {2, 3}},
-     {L"\U0000034f\U00001160", {847, 4448}, {1, 2}},
-     {L"\U0000034f\U00000308\U00001160", {847, 4448}, {2, 3}},
-     {L"\U0000034f\U000011a8", {847, 4520}, {1, 2}},
-     {L"\U0000034f\U00000308\U000011a8", {847, 4520}, {2, 3}},
-     {L"\U0000034f\U0000ac00", {847, 44032}, {1, 2}},
-     {L"\U0000034f\U00000308\U0000ac00", {847, 44032}, {2, 3}},
-     {L"\U0000034f\U0000ac01", {847, 44033}, {1, 2}},
-     {L"\U0000034f\U00000308\U0000ac01", {847, 44033}, {2, 3}},
-     {L"\U0000034f\U00000900", {847}, {2}},
-     {L"\U0000034f\U00000308\U00000900", {847}, {3}},
-     {L"\U0000034f\U00000903", {847}, {2}},
-     {L"\U0000034f\U00000308\U00000903", {847}, {3}},
-     {L"\U0000034f\U00000904", {847, 2308}, {1, 2}},
-     {L"\U0000034f\U00000308\U00000904", {847, 2308}, {2, 3}},
-     {L"\U0000034f\U00000d4e", {847, 3406}, {1, 2}},
-     {L"\U0000034f\U00000308\U00000d4e", {847, 3406}, {2, 3}},
-     {L"\U0000034f\U00000915", {847, 2325}, {1, 2}},
-     {L"\U0000034f\U00000308\U00000915", {847, 2325}, {2, 3}},
-     {L"\U0000034f\U0000231a", {847, 8986}, {1, 2}},
-     {L"\U0000034f\U00000308\U0000231a", {847, 8986}, {2, 3}},
-     {L"\U0000034f\U00000300", {847}, {2}},
-     {L"\U0000034f\U00000308\U00000300", {847}, {3}},
-     {L"\U0000034f\U0000093c", {847}, {2}},
-     {L"\U0000034f\U00000308\U0000093c", {847}, {3}},
-     {L"\U0000034f\U0000094d", {847}, {2}},
-     {L"\U0000034f\U00000308\U0000094d", {847}, {3}},
-     {L"\U0000034f\U0000200d", {847}, {2}},
-     {L"\U0000034f\U00000308\U0000200d", {847}, {3}},
-     {L"\U0000034f\U00000378", {847, 888}, {1, 2}},
-     {L"\U0000034f\U00000308\U00000378", {847, 888}, {2, 3}},
+     {L"\U0000200c\U00000020", {8204, 32}, {1, 2}},
+     {L"\U0000200c\U00000308\U00000020", {8204, 32}, {2, 3}},
+     {L"\U0000200c\U0000000d", {8204, 13}, {1, 2}},
+     {L"\U0000200c\U00000308\U0000000d", {8204, 13}, {2, 3}},
+     {L"\U0000200c\U0000000a", {8204, 10}, {1, 2}},
+     {L"\U0000200c\U00000308\U0000000a", {8204, 10}, {2, 3}},
+     {L"\U0000200c\U00000001", {8204, 1}, {1, 2}},
+     {L"\U0000200c\U00000308\U00000001", {8204, 1}, {2, 3}},
+     {L"\U0000200c\U0000200c", {8204}, {2}},
+     {L"\U0000200c\U00000308\U0000200c", {8204}, {3}},
+     {L"\U0000200c\U0001f1e6", {8204, 127462}, {1, 3}},
+     {L"\U0000200c\U00000308\U0001f1e6", {8204, 127462}, {2, 4}},
+     {L"\U0000200c\U00000600", {8204, 1536}, {1, 2}},
+     {L"\U0000200c\U00000308\U00000600", {8204, 1536}, {2, 3}},
+     {L"\U0000200c\U00000a03", {8204}, {2}},
+     {L"\U0000200c\U00000308\U00000a03", {8204}, {3}},
+     {L"\U0000200c\U00001100", {8204, 4352}, {1, 2}},
+     {L"\U0000200c\U00000308\U00001100", {8204, 4352}, {2, 3}},
+     {L"\U0000200c\U00001160", {8204, 4448}, {1, 2}},
+     {L"\U0000200c\U00000308\U00001160", {8204, 4448}, {2, 3}},
+     {L"\U0000200c\U000011a8", {8204, 4520}, {1, 2}},
+     {L"\U0000200c\U00000308\U000011a8", {8204, 4520}, {2, 3}},
+     {L"\U0000200c\U0000ac00", {8204, 44032}, {1, 2}},
+     {L"\U0000200c\U00000308\U0000ac00", {8204, 44032}, {2, 3}},
+     {L"\U0000200c\U0000ac01", {8204, 44033}, {1, 2}},
+     {L"\U0000200c\U00000308\U0000ac01", {8204, 44033}, {2, 3}},
+     {L"\U0000200c\U00000903", {8204}, {2}},
+     {L"\U0000200c\U00000308\U00000903", {8204}, {3}},
+     {L"\U0000200c\U00000904", {8204, 2308}, {1, 2}},
+     {L"\U0000200c\U00000308\U00000904", {8204, 2308}, {2, 3}},
+     {L"\U0000200c\U00000d4e", {8204, 3406}, {1, 2}},
+     {L"\U0000200c\U00000308\U00000d4e", {8204, 3406}, {2, 3}},
+     {L"\U0000200c\U00000915", {8204, 2325}, {1, 2}},
+     {L"\U0000200c\U00000308\U00000915", {8204, 2325}, {2, 3}},
+     {L"\U0000200c\U0000231a", {8204, 8986}, {1, 2}},
+     {L"\U0000200c\U00000308\U0000231a", {8204, 8986}, {2, 3}},
+     {L"\U0000200c\U00000300", {8204}, {2}},
+     {L"\U0000200c\U00000308\U00000300", {8204}, {3}},
+     {L"\U0000200c\U00000900", {8204}, {2}},
+     {L"\U0000200c\U00000308\U00000900", {8204}, {3}},
+     {L"\U0000200c\U0000094d", {8204}, {2}},
+     {L"\U0000200c\U00000308\U0000094d", {8204}, {3}},
+     {L"\U0000200c\U0000200d", {8204}, {2}},
+     {L"\U0000200c\U00000308\U0000200d", {8204}, {3}},
+     {L"\U0000200c\U00000378", {8204, 888}, {1, 2}},
+     {L"\U0000200c\U00000308\U00000378", {8204, 888}, {2, 3}},
      {L"\U0001f1e6\U00000020", {127462, 32}, {2, 3}},
      {L"\U0001f1e6\U00000308\U00000020", {127462, 32}, {3, 4}},
      {L"\U0001f1e6\U0000000d", {127462, 13}, {2, 3}},
@@ -1526,8 +1422,8 @@ std::array<data<wchar_t>, 1187> data_utf16 = {{
      {L"\U0001f1e6\U00000308\U0000000a", {127462, 10}, {3, 4}},
      {L"\U0001f1e6\U00000001", {127462, 1}, {2, 3}},
      {L"\U0001f1e6\U00000308\U00000001", {127462, 1}, {3, 4}},
-     {L"\U0001f1e6\U0000034f", {127462}, {3}},
-     {L"\U0001f1e6\U00000308\U0000034f", {127462}, {4}},
+     {L"\U0001f1e6\U0000200c", {127462}, {3}},
+     {L"\U0001f1e6\U00000308\U0000200c", {127462}, {4}},
      {L"\U0001f1e6\U0001f1e6", {127462}, {4}},
      {L"\U0001f1e6\U00000308\U0001f1e6", {127462, 127462}, {3, 5}},
      {L"\U0001f1e6\U00000600", {127462, 1536}, {2, 3}},
@@ -1544,8 +1440,6 @@ std::array<data<wchar_t>, 1187> data_utf16 = {{
      {L"\U0001f1e6\U00000308\U0000ac00", {127462, 44032}, {3, 4}},
      {L"\U0001f1e6\U0000ac01", {127462, 44033}, {2, 3}},
      {L"\U0001f1e6\U00000308\U0000ac01", {127462, 44033}, {3, 4}},
-     {L"\U0001f1e6\U00000900", {127462}, {3}},
-     {L"\U0001f1e6\U00000308\U00000900", {127462}, {4}},
      {L"\U0001f1e6\U00000903", {127462}, {3}},
      {L"\U0001f1e6\U00000308\U00000903", {127462}, {4}},
      {L"\U0001f1e6\U00000904", {127462, 2308}, {2, 3}},
@@ -1558,8 +1452,8 @@ std::array<data<wchar_t>, 1187> data_utf16 = {{
      {L"\U0001f1e6\U00000308\U0000231a", {127462, 8986}, {3, 4}},
      {L"\U0001f1e6\U00000300", {127462}, {3}},
      {L"\U0001f1e6\U00000308\U00000300", {127462}, {4}},
-     {L"\U0001f1e6\U0000093c", {127462}, {3}},
-     {L"\U0001f1e6\U00000308\U0000093c", {127462}, {4}},
+     {L"\U0001f1e6\U00000900", {127462}, {3}},
+     {L"\U0001f1e6\U00000308\U00000900", {127462}, {4}},
      {L"\U0001f1e6\U0000094d", {127462}, {3}},
      {L"\U0001f1e6\U00000308\U0000094d", {127462}, {4}},
      {L"\U0001f1e6\U0000200d", {127462}, {3}},
@@ -1574,8 +1468,8 @@ std::array<data<wchar_t>, 1187> data_utf16 = {{
      {L"\U00000600\U00000308\U0000000a", {1536, 10}, {2, 3}},
      {L"\U00000600\U00000001", {1536, 1}, {1, 2}},
      {L"\U00000600\U00000308\U00000001", {1536, 1}, {2, 3}},
-     {L"\U00000600\U0000034f", {1536}, {2}},
-     {L"\U00000600\U00000308\U0000034f", {1536}, {3}},
+     {L"\U00000600\U0000200c", {1536}, {2}},
+     {L"\U00000600\U00000308\U0000200c", {1536}, {3}},
      {L"\U00000600\U0001f1e6", {1536}, {3}},
      {L"\U00000600\U00000308\U0001f1e6", {1536, 127462}, {2, 4}},
      {L"\U00000600\U00000600", {1536}, {2}},
@@ -1592,8 +1486,6 @@ std::array<data<wchar_t>, 1187> data_utf16 = {{
      {L"\U00000600\U00000308\U0000ac00", {1536, 44032}, {2, 3}},
      {L"\U00000600\U0000ac01", {1536}, {2}},
      {L"\U00000600\U00000308\U0000ac01", {1536, 44033}, {2, 3}},
-     {L"\U00000600\U00000900", {1536}, {2}},
-     {L"\U00000600\U00000308\U00000900", {1536}, {3}},
      {L"\U00000600\U00000903", {1536}, {2}},
      {L"\U00000600\U00000308\U00000903", {1536}, {3}},
      {L"\U00000600\U00000904", {1536}, {2}},
@@ -1606,8 +1498,8 @@ std::array<data<wchar_t>, 1187> data_utf16 = {{
      {L"\U00000600\U00000308\U0000231a", {1536, 8986}, {2, 3}},
      {L"\U00000600\U00000300", {1536}, {2}},
      {L"\U00000600\U00000308\U00000300", {1536}, {3}},
-     {L"\U00000600\U0000093c", {1536}, {2}},
-     {L"\U00000600\U00000308\U0000093c", {1536}, {3}},
+     {L"\U00000600\U00000900", {1536}, {2}},
+     {L"\U00000600\U00000308\U00000900", {1536}, {3}},
      {L"\U00000600\U0000094d", {1536}, {2}},
      {L"\U00000600\U00000308\U0000094d", {1536}, {3}},
      {L"\U00000600\U0000200d", {1536}, {2}},
@@ -1622,8 +1514,8 @@ std::array<data<wchar_t>, 1187> data_utf16 = {{
      {L"\U00000a03\U00000308\U0000000a", {2563, 10}, {2, 3}},
      {L"\U00000a03\U00000001", {2563, 1}, {1, 2}},
      {L"\U00000a03\U00000308\U00000001", {2563, 1}, {2, 3}},
-     {L"\U00000a03\U0000034f", {2563}, {2}},
-     {L"\U00000a03\U00000308\U0000034f", {2563}, {3}},
+     {L"\U00000a03\U0000200c", {2563}, {2}},
+     {L"\U00000a03\U00000308\U0000200c", {2563}, {3}},
      {L"\U00000a03\U0001f1e6", {2563, 127462}, {1, 3}},
      {L"\U00000a03\U00000308\U0001f1e6", {2563, 127462}, {2, 4}},
      {L"\U00000a03\U00000600", {2563, 1536}, {1, 2}},
@@ -1640,8 +1532,6 @@ std::array<data<wchar_t>, 1187> data_utf16 = {{
      {L"\U00000a03\U00000308\U0000ac00", {2563, 44032}, {2, 3}},
      {L"\U00000a03\U0000ac01", {2563, 44033}, {1, 2}},
      {L"\U00000a03\U00000308\U0000ac01", {2563, 44033}, {2, 3}},
-     {L"\U00000a03\U00000900", {2563}, {2}},
-     {L"\U00000a03\U00000308\U00000900", {2563}, {3}},
      {L"\U00000a03\U00000903", {2563}, {2}},
      {L"\U00000a03\U00000308\U00000903", {2563}, {3}},
      {L"\U00000a03\U00000904", {2563, 2308}, {1, 2}},
@@ -1654,8 +1544,8 @@ std::array<data<wchar_t>, 1187> data_utf16 = {{
      {L"\U00000a03\U00000308\U0000231a", {2563, 8986}, {2, 3}},
      {L"\U00000a03\U00000300", {2563}, {2}},
      {L"\U00000a03\U00000308\U00000300", {2563}, {3}},
-     {L"\U00000a03\U0000093c", {2563}, {2}},
-     {L"\U00000a03\U00000308\U0000093c", {2563}, {3}},
+     {L"\U00000a03\U00000900", {2563}, {2}},
+     {L"\U00000a03\U00000308\U00000900", {2563}, {3}},
      {L"\U00000a03\U0000094d", {2563}, {2}},
      {L"\U00000a03\U00000308\U0000094d", {2563}, {3}},
      {L"\U00000a03\U0000200d", {2563}, {2}},
@@ -1670,8 +1560,8 @@ std::array<data<wchar_t>, 1187> data_utf16 = {{
      {L"\U00001100\U00000308\U0000000a", {4352, 10}, {2, 3}},
      {L"\U00001100\U00000001", {4352, 1}, {1, 2}},
      {L"\U00001100\U00000308\U00000001", {4352, 1}, {2, 3}},
-     {L"\U00001100\U0000034f", {4352}, {2}},
-     {L"\U00001100\U00000308\U0000034f", {4352}, {3}},
+     {L"\U00001100\U0000200c", {4352}, {2}},
+     {L"\U00001100\U00000308\U0000200c", {4352}, {3}},
      {L"\U00001100\U0001f1e6", {4352, 127462}, {1, 3}},
      {L"\U00001100\U00000308\U0001f1e6", {4352, 127462}, {2, 4}},
      {L"\U00001100\U00000600", {4352, 1536}, {1, 2}},
@@ -1688,8 +1578,6 @@ std::array<data<wchar_t>, 1187> data_utf16 = {{
      {L"\U00001100\U00000308\U0000ac00", {4352, 44032}, {2, 3}},
      {L"\U00001100\U0000ac01", {4352}, {2}},
      {L"\U00001100\U00000308\U0000ac01", {4352, 44033}, {2, 3}},
-     {L"\U00001100\U00000900", {4352}, {2}},
-     {L"\U00001100\U00000308\U00000900", {4352}, {3}},
      {L"\U00001100\U00000903", {4352}, {2}},
      {L"\U00001100\U00000308\U00000903", {4352}, {3}},
      {L"\U00001100\U00000904", {4352, 2308}, {1, 2}},
@@ -1702,8 +1590,8 @@ std::array<data<wchar_t>, 1187> data_utf16 = {{
      {L"\U00001100\U00000308\U0000231a", {4352, 8986}, {2, 3}},
      {L"\U00001100\U00000300", {4352}, {2}},
      {L"\U00001100\U00000308\U00000300", {4352}, {3}},
-     {L"\U00001100\U0000093c", {4352}, {2}},
-     {L"\U00001100\U00000308\U0000093c", {4352}, {3}},
+     {L"\U00001100\U00000900", {4352}, {2}},
+     {L"\U00001100\U00000308\U00000900", {4352}, {3}},
      {L"\U00001100\U0000094d", {4352}, {2}},
      {L"\U00001100\U00000308\U0000094d", {4352}, {3}},
      {L"\U00001100\U0000200d", {4352}, {2}},
@@ -1718,8 +1606,8 @@ std::array<data<wchar_t>, 1187> data_utf16 = {{
      {L"\U00001160\U00000308\U0000000a", {4448, 10}, {2, 3}},
      {L"\U00001160\U00000001", {4448, 1}, {1, 2}},
      {L"\U00001160\U00000308\U00000001", {4448, 1}, {2, 3}},
-     {L"\U00001160\U0000034f", {4448}, {2}},
-     {L"\U00001160\U00000308\U0000034f", {4448}, {3}},
+     {L"\U00001160\U0000200c", {4448}, {2}},
+     {L"\U00001160\U00000308\U0000200c", {4448}, {3}},
      {L"\U00001160\U0001f1e6", {4448, 127462}, {1, 3}},
      {L"\U00001160\U00000308\U0001f1e6", {4448, 127462}, {2, 4}},
      {L"\U00001160\U00000600", {4448, 1536}, {1, 2}},
@@ -1736,8 +1624,6 @@ std::array<data<wchar_t>, 1187> data_utf16 = {{
      {L"\U00001160\U00000308\U0000ac00", {4448, 44032}, {2, 3}},
      {L"\U00001160\U0000ac01", {4448, 44033}, {1, 2}},
      {L"\U00001160\U00000308\U0000ac01", {4448, 44033}, {2, 3}},
-     {L"\U00001160\U00000900", {4448}, {2}},
-     {L"\U00001160\U00000308\U00000900", {4448}, {3}},
      {L"\U00001160\U00000903", {4448}, {2}},
      {L"\U00001160\U00000308\U00000903", {4448}, {3}},
      {L"\U00001160\U00000904", {4448, 2308}, {1, 2}},
@@ -1750,8 +1636,8 @@ std::array<data<wchar_t>, 1187> data_utf16 = {{
      {L"\U00001160\U00000308\U0000231a", {4448, 8986}, {2, 3}},
      {L"\U00001160\U00000300", {4448}, {2}},
      {L"\U00001160\U00000308\U00000300", {4448}, {3}},
-     {L"\U00001160\U0000093c", {4448}, {2}},
-     {L"\U00001160\U00000308\U0000093c", {4448}, {3}},
+     {L"\U00001160\U00000900", {4448}, {2}},
+     {L"\U00001160\U00000308\U00000900", {4448}, {3}},
      {L"\U00001160\U0000094d", {4448}, {2}},
      {L"\U00001160\U00000308\U0000094d", {4448}, {3}},
      {L"\U00001160\U0000200d", {4448}, {2}},
@@ -1766,8 +1652,8 @@ std::array<data<wchar_t>, 1187> data_utf16 = {{
      {L"\U000011a8\U00000308\U0000000a", {4520, 10}, {2, 3}},
      {L"\U000011a8\U00000001", {4520, 1}, {1, 2}},
      {L"\U000011a8\U00000308\U00000001", {4520, 1}, {2, 3}},
-     {L"\U000011a8\U0000034f", {4520}, {2}},
-     {L"\U000011a8\U00000308\U0000034f", {4520}, {3}},
+     {L"\U000011a8\U0000200c", {4520}, {2}},
+     {L"\U000011a8\U00000308\U0000200c", {4520}, {3}},
      {L"\U000011a8\U0001f1e6", {4520, 127462}, {1, 3}},
      {L"\U000011a8\U00000308\U0001f1e6", {4520, 127462}, {2, 4}},
      {L"\U000011a8\U00000600", {4520, 1536}, {1, 2}},
@@ -1784,8 +1670,6 @@ std::array<data<wchar_t>, 1187> data_utf16 = {{
      {L"\U000011a8\U00000308\U0000ac00", {4520, 44032}, {2, 3}},
      {L"\U000011a8\U0000ac01", {4520, 44033}, {1, 2}},
      {L"\U000011a8\U00000308\U0000ac01", {4520, 44033}, {2, 3}},
-     {L"\U000011a8\U00000900", {4520}, {2}},
-     {L"\U000011a8\U00000308\U00000900", {4520}, {3}},
      {L"\U000011a8\U00000903", {4520}, {2}},
      {L"\U000011a8\U00000308\U00000903", {4520}, {3}},
      {L"\U000011a8\U00000904", {4520, 2308}, {1, 2}},
@@ -1798,8 +1682,8 @@ std::array<data<wchar_t>, 1187> data_utf16 = {{
      {L"\U000011a8\U00000308\U0000231a", {4520, 8986}, {2, 3}},
      {L"\U000011a8\U00000300", {4520}, {2}},
      {L"\U000011a8\U00000308\U00000300", {4520}, {3}},
-     {L"\U000011a8\U0000093c", {4520}, {2}},
-     {L"\U000011a8\U00000308\U0000093c", {4520}, {3}},
+     {L"\U000011a8\U00000900", {4520}, {2}},
+     {L"\U000011a8\U00000308\U00000900", {4520}, {3}},
      {L"\U000011a8\U0000094d", {4520}, {2}},
      {L"\U000011a8\U00000308\U0000094d", {4520}, {3}},
      {L"\U000011a8\U0000200d", {4520}, {2}},
@@ -1814,8 +1698,8 @@ std::array<data<wchar_t>, 1187> data_utf16 = {{
      {L"\U0000ac00\U00000308\U0000000a", {44032, 10}, {2, 3}},
      {L"\U0000ac00\U00000001", {44032, 1}, {1, 2}},
      {L"\U0000ac00\U00000308\U00000001", {44032, 1}, {2, 3}},
-     {L"\U0000ac00\U0000034f", {44032}, {2}},
-     {L"\U0000ac00\U00000308\U0000034f", {44032}, {3}},
+     {L"\U0000ac00\U0000200c", {44032}, {2}},
+     {L"\U0000ac00\U00000308\U0000200c", {44032}, {3}},
      {L"\U0000ac00\U0001f1e6", {44032, 127462}, {1, 3}},
      {L"\U0000ac00\U00000308\U0001f1e6", {44032, 127462}, {2, 4}},
      {L"\U0000ac00\U00000600", {44032, 1536}, {1, 2}},
@@ -1832,8 +1716,6 @@ std::array<data<wchar_t>, 1187> data_utf16 = {{
      {L"\U0000ac00\U00000308\U0000ac00", {44032, 44032}, {2, 3}},
      {L"\U0000ac00\U0000ac01", {44032, 44033}, {1, 2}},
      {L"\U0000ac00\U00000308\U0000ac01", {44032, 44033}, {2, 3}},
-     {L"\U0000ac00\U00000900", {44032}, {2}},
-     {L"\U0000ac00\U00000308\U00000900", {44032}, {3}},
      {L"\U0000ac00\U00000903", {44032}, {2}},
      {L"\U0000ac00\U00000308\U00000903", {44032}, {3}},
      {L"\U0000ac00\U00000904", {44032, 2308}, {1, 2}},
@@ -1846,8 +1728,8 @@ std::array<data<wchar_t>, 1187> data_utf16 = {{
      {L"\U0000ac00\U00000308\U0000231a", {44032, 8986}, {2, 3}},
      {L"\U0000ac00\U00000300", {44032}, {2}},
      {L"\U0000ac00\U00000308\U00000300", {44032}, {3}},
-     {L"\U0000ac00\U0000093c", {44032}, {2}},
-     {L"\U0000ac00\U00000308\U0000093c", {44032}, {3}},
+     {L"\U0000ac00\U00000900", {44032}, {2}},
+     {L"\U0000ac00\U00000308\U00000900", {44032}, {3}},
      {L"\U0000ac00\U0000094d", {44032}, {2}},
      {L"\U0000ac00\U00000308\U0000094d", {44032}, {3}},
      {L"\U0000ac00\U0000200d", {44032}, {2}},
@@ -1862,8 +1744,8 @@ std::array<data<wchar_t>, 1187> data_utf16 = {{
      {L"\U0000ac01\U00000308\U0000000a", {44033, 10}, {2, 3}},
      {L"\U0000ac01\U00000001", {44033, 1}, {1, 2}},
      {L"\U0000ac01\U00000308\U00000001", {44033, 1}, {2, 3}},
-     {L"\U0000ac01\U0000034f", {44033}, {2}},
-     {L"\U0000ac01\U00000308\U0000034f", {44033}, {3}},
+     {L"\U0000ac01\U0000200c", {44033}, {2}},
+     {L"\U0000ac01\U00000308\U0000200c", {44033}, {3}},
      {L"\U0000ac01\U0001f1e6", {44033, 127462}, {1, 3}},
      {L"\U0000ac01\U00000308\U0001f1e6", {44033, 127462}, {2, 4}},
      {L"\U0000ac01\U00000600", {44033, 1536}, {1, 2}},
@@ -1880,8 +1762,6 @@ std::array<data<wchar_t>, 1187> data_utf16 = {{
      {L"\U0000ac01\U00000308\U0000ac00", {44033, 44032}, {2, 3}},
      {L"\U0000ac01\U0000ac01", {44033, 44033}, {1, 2}},
      {L"\U0000ac01\U00000308\U0000ac01", {44033, 44033}, {2, 3}},
-     {L"\U0000ac01\U00000900", {44033}, {2}},
-     {L"\U0000ac01\U00000308\U00000900", {44033}, {3}},
      {L"\U0000ac01\U00000903", {44033}, {2}},
      {L"\U0000ac01\U00000308\U00000903", {44033}, {3}},
      {L"\U0000ac01\U00000904", {44033, 2308}, {1, 2}},
@@ -1894,62 +1774,14 @@ std::array<data<wchar_t>, 1187> data_utf16 = {{
      {L"\U0000ac01\U00000308\U0000231a", {44033, 8986}, {2, 3}},
      {L"\U0000ac01\U00000300", {44033}, {2}},
      {L"\U0000ac01\U00000308\U00000300", {44033}, {3}},
-     {L"\U0000ac01\U0000093c", {44033}, {2}},
-     {L"\U0000ac01\U00000308\U0000093c", {44033}, {3}},
+     {L"\U0000ac01\U00000900", {44033}, {2}},
+     {L"\U0000ac01\U00000308\U00000900", {44033}, {3}},
      {L"\U0000ac01\U0000094d", {44033}, {2}},
      {L"\U0000ac01\U00000308\U0000094d", {44033}, {3}},
      {L"\U0000ac01\U0000200d", {44033}, {2}},
      {L"\U0000ac01\U00000308\U0000200d", {44033}, {3}},
      {L"\U0000ac01\U00000378", {44033, 888}, {1, 2}},
      {L"\U0000ac01\U00000308\U00000378", {44033, 888}, {2, 3}},
-     {L"\U00000900\U00000020", {2304, 32}, {1, 2}},
-     {L"\U00000900\U00000308\U00000020", {2304, 32}, {2, 3}},
-     {L"\U00000900\U0000000d", {2304, 13}, {1, 2}},
-     {L"\U00000900\U00000308\U0000000d", {2304, 13}, {2, 3}},
-     {L"\U00000900\U0000000a", {2304, 10}, {1, 2}},
-     {L"\U00000900\U00000308\U0000000a", {2304, 10}, {2, 3}},
-     {L"\U00000900\U00000001", {2304, 1}, {1, 2}},
-     {L"\U00000900\U00000308\U00000001", {2304, 1}, {2, 3}},
-     {L"\U00000900\U0000034f", {2304}, {2}},
-     {L"\U00000900\U00000308\U0000034f", {2304}, {3}},
-     {L"\U00000900\U0001f1e6", {2304, 127462}, {1, 3}},
-     {L"\U00000900\U00000308\U0001f1e6", {2304, 127462}, {2, 4}},
-     {L"\U00000900\U00000600", {2304, 1536}, {1, 2}},
-     {L"\U00000900\U00000308\U00000600", {2304, 1536}, {2, 3}},
-     {L"\U00000900\U00000a03", {2304}, {2}},
-     {L"\U00000900\U00000308\U00000a03", {2304}, {3}},
-     {L"\U00000900\U00001100", {2304, 4352}, {1, 2}},
-     {L"\U00000900\U00000308\U00001100", {2304, 4352}, {2, 3}},
-     {L"\U00000900\U00001160", {2304, 4448}, {1, 2}},
-     {L"\U00000900\U00000308\U00001160", {2304, 4448}, {2, 3}},
-     {L"\U00000900\U000011a8", {2304, 4520}, {1, 2}},
-     {L"\U00000900\U00000308\U000011a8", {2304, 4520}, {2, 3}},
-     {L"\U00000900\U0000ac00", {2304, 44032}, {1, 2}},
-     {L"\U00000900\U00000308\U0000ac00", {2304, 44032}, {2, 3}},
-     {L"\U00000900\U0000ac01", {2304, 44033}, {1, 2}},
-     {L"\U00000900\U00000308\U0000ac01", {2304, 44033}, {2, 3}},
-     {L"\U00000900\U00000900", {2304}, {2}},
-     {L"\U00000900\U00000308\U00000900", {2304}, {3}},
-     {L"\U00000900\U00000903", {2304}, {2}},
-     {L"\U00000900\U00000308\U00000903", {2304}, {3}},
-     {L"\U00000900\U00000904", {2304, 2308}, {1, 2}},
-     {L"\U00000900\U00000308\U00000904", {2304, 2308}, {2, 3}},
-     {L"\U00000900\U00000d4e", {2304, 3406}, {1, 2}},
-     {L"\U00000900\U00000308\U00000d4e", {2304, 3406}, {2, 3}},
-     {L"\U00000900\U00000915", {2304, 2325}, {1, 2}},
-     {L"\U00000900\U00000308\U00000915", {2304, 2325}, {2, 3}},
-     {L"\U00000900\U0000231a", {2304, 8986}, {1, 2}},
-     {L"\U00000900\U00000308\U0000231a", {2304, 8986}, {2, 3}},
-     {L"\U00000900\U00000300", {2304}, {2}},
-     {L"\U00000900\U00000308\U00000300", {2304}, {3}},
-     {L"\U00000900\U0000093c", {2304}, {2}},
-     {L"\U00000900\U00000308\U0000093c", {2304}, {3}},
-     {L"\U00000900\U0000094d", {2304}, {2}},
-     {L"\U00000900\U00000308\U0000094d", {2304}, {3}},
-     {L"\U00000900\U0000200d", {2304}, {2}},
-     {L"\U00000900\U00000308\U0000200d", {2304}, {3}},
-     {L"\U00000900\U00000378", {2304, 888}, {1, 2}},
-     {L"\U00000900\U00000308\U00000378", {2304, 888}, {2, 3}},
      {L"\U00000903\U00000020", {2307, 32}, {1, 2}},
      {L"\U00000903\U00000308\U00000020", {2307, 32}, {2, 3}},
      {L"\U00000903\U0000000d", {2307, 13}, {1, 2}},
@@ -1958,8 +1790,8 @@ std::array<data<wchar_t>, 1187> data_utf16 = {{
      {L"\U00000903\U00000308\U0000000a", {2307, 10}, {2, 3}},
      {L"\U00000903\U00000001", {2307, 1}, {1, 2}},
      {L"\U00000903\U00000308\U00000001", {2307, 1}, {2, 3}},
-     {L"\U00000903\U0000034f", {2307}, {2}},
-     {L"\U00000903\U00000308\U0000034f", {2307}, {3}},
+     {L"\U00000903\U0000200c", {2307}, {2}},
+     {L"\U00000903\U00000308\U0000200c", {2307}, {3}},
      {L"\U00000903\U0001f1e6", {2307, 127462}, {1, 3}},
      {L"\U00000903\U00000308\U0001f1e6", {2307, 127462}, {2, 4}},
      {L"\U00000903\U00000600", {2307, 1536}, {1, 2}},
@@ -1976,8 +1808,6 @@ std::array<data<wchar_t>, 1187> data_utf16 = {{
      {L"\U00000903\U00000308\U0000ac00", {2307, 44032}, {2, 3}},
      {L"\U00000903\U0000ac01", {2307, 44033}, {1, 2}},
      {L"\U00000903\U00000308\U0000ac01", {2307, 44033}, {2, 3}},
-     {L"\U00000903\U00000900", {2307}, {2}},
-     {L"\U00000903\U00000308\U00000900", {2307}, {3}},
      {L"\U00000903\U00000903", {2307}, {2}},
      {L"\U00000903\U00000308\U00000903", {2307}, {3}},
      {L"\U00000903\U00000904", {2307, 2308}, {1, 2}},
@@ -1990,8 +1820,8 @@ std::array<data<wchar_t>, 1187> data_utf16 = {{
      {L"\U00000903\U00000308\U0000231a", {2307, 8986}, {2, 3}},
      {L"\U00000903\U00000300", {2307}, {2}},
      {L"\U00000903\U00000308\U00000300", {2307}, {3}},
-     {L"\U00000903\U0000093c", {2307}, {2}},
-     {L"\U00000903\U00000308\U0000093c", {2307}, {3}},
+     {L"\U00000903\U00000900", {2307}, {2}},
+     {L"\U00000903\U00000308\U00000900", {2307}, {3}},
      {L"\U00000903\U0000094d", {2307}, {2}},
      {L"\U00000903\U00000308\U0000094d", {2307}, {3}},
      {L"\U00000903\U0000200d", {2307}, {2}},
@@ -2006,8 +1836,8 @@ std::array<data<wchar_t>, 1187> data_utf16 = {{
      {L"\U00000904\U00000308\U0000000a", {2308, 10}, {2, 3}},
      {L"\U00000904\U00000001", {2308, 1}, {1, 2}},
      {L"\U00000904\U00000308\U00000001", {2308, 1}, {2, 3}},
-     {L"\U00000904\U0000034f", {2308}, {2}},
-     {L"\U00000904\U00000308\U0000034f", {2308}, {3}},
+     {L"\U00000904\U0000200c", {2308}, {2}},
+     {L"\U00000904\U00000308\U0000200c", {2308}, {3}},
      {L"\U00000904\U0001f1e6", {2308, 127462}, {1, 3}},
      {L"\U00000904\U00000308\U0001f1e6", {2308, 127462}, {2, 4}},
      {L"\U00000904\U00000600", {2308, 1536}, {1, 2}},
@@ -2024,8 +1854,6 @@ std::array<data<wchar_t>, 1187> data_utf16 = {{
      {L"\U00000904\U00000308\U0000ac00", {2308, 44032}, {2, 3}},
      {L"\U00000904\U0000ac01", {2308, 44033}, {1, 2}},
      {L"\U00000904\U00000308\U0000ac01", {2308, 44033}, {2, 3}},
-     {L"\U00000904\U00000900", {2308}, {2}},
-     {L"\U00000904\U00000308\U00000900", {2308}, {3}},
      {L"\U00000904\U00000903", {2308}, {2}},
      {L"\U00000904\U00000308\U00000903", {2308}, {3}},
      {L"\U00000904\U00000904", {2308, 2308}, {1, 2}},
@@ -2038,8 +1866,8 @@ std::array<data<wchar_t>, 1187> data_utf16 = {{
      {L"\U00000904\U00000308\U0000231a", {2308, 8986}, {2, 3}},
      {L"\U00000904\U00000300", {2308}, {2}},
      {L"\U00000904\U00000308\U00000300", {2308}, {3}},
-     {L"\U00000904\U0000093c", {2308}, {2}},
-     {L"\U00000904\U00000308\U0000093c", {2308}, {3}},
+     {L"\U00000904\U00000900", {2308}, {2}},
+     {L"\U00000904\U00000308\U00000900", {2308}, {3}},
      {L"\U00000904\U0000094d", {2308}, {2}},
      {L"\U00000904\U00000308\U0000094d", {2308}, {3}},
      {L"\U00000904\U0000200d", {2308}, {2}},
@@ -2054,8 +1882,8 @@ std::array<data<wchar_t>, 1187> data_utf16 = {{
      {L"\U00000d4e\U00000308\U0000000a", {3406, 10}, {2, 3}},
      {L"\U00000d4e\U00000001", {3406, 1}, {1, 2}},
      {L"\U00000d4e\U00000308\U00000001", {3406, 1}, {2, 3}},
-     {L"\U00000d4e\U0000034f", {3406}, {2}},
-     {L"\U00000d4e\U00000308\U0000034f", {3406}, {3}},
+     {L"\U00000d4e\U0000200c", {3406}, {2}},
+     {L"\U00000d4e\U00000308\U0000200c", {3406}, {3}},
      {L"\U00000d4e\U0001f1e6", {3406}, {3}},
      {L"\U00000d4e\U00000308\U0001f1e6", {3406, 127462}, {2, 4}},
      {L"\U00000d4e\U00000600", {3406}, {2}},
@@ -2072,8 +1900,6 @@ std::array<data<wchar_t>, 1187> data_utf16 = {{
      {L"\U00000d4e\U00000308\U0000ac00", {3406, 44032}, {2, 3}},
      {L"\U00000d4e\U0000ac01", {3406}, {2}},
      {L"\U00000d4e\U00000308\U0000ac01", {3406, 44033}, {2, 3}},
-     {L"\U00000d4e\U00000900", {3406}, {2}},
-     {L"\U00000d4e\U00000308\U00000900", {3406}, {3}},
      {L"\U00000d4e\U00000903", {3406}, {2}},
      {L"\U00000d4e\U00000308\U00000903", {3406}, {3}},
      {L"\U00000d4e\U00000904", {3406}, {2}},
@@ -2086,8 +1912,8 @@ std::array<data<wchar_t>, 1187> data_utf16 = {{
      {L"\U00000d4e\U00000308\U0000231a", {3406, 8986}, {2, 3}},
      {L"\U00000d4e\U00000300", {3406}, {2}},
      {L"\U00000d4e\U00000308\U00000300", {3406}, {3}},
-     {L"\U00000d4e\U0000093c", {3406}, {2}},
-     {L"\U00000d4e\U00000308\U0000093c", {3406}, {3}},
+     {L"\U00000d4e\U00000900", {3406}, {2}},
+     {L"\U00000d4e\U00000308\U00000900", {3406}, {3}},
      {L"\U00000d4e\U0000094d", {3406}, {2}},
      {L"\U00000d4e\U00000308\U0000094d", {3406}, {3}},
      {L"\U00000d4e\U0000200d", {3406}, {2}},
@@ -2102,8 +1928,8 @@ std::array<data<wchar_t>, 1187> data_utf16 = {{
      {L"\U00000915\U00000308\U0000000a", {2325, 10}, {2, 3}},
      {L"\U00000915\U00000001", {2325, 1}, {1, 2}},
      {L"\U00000915\U00000308\U00000001", {2325, 1}, {2, 3}},
-     {L"\U00000915\U0000034f", {2325}, {2}},
-     {L"\U00000915\U00000308\U0000034f", {2325}, {3}},
+     {L"\U00000915\U0000200c", {2325}, {2}},
+     {L"\U00000915\U00000308\U0000200c", {2325}, {3}},
      {L"\U00000915\U0001f1e6", {2325, 127462}, {1, 3}},
      {L"\U00000915\U00000308\U0001f1e6", {2325, 127462}, {2, 4}},
      {L"\U00000915\U00000600", {2325, 1536}, {1, 2}},
@@ -2120,8 +1946,6 @@ std::array<data<wchar_t>, 1187> data_utf16 = {{
      {L"\U00000915\U00000308\U0000ac00", {2325, 44032}, {2, 3}},
      {L"\U00000915\U0000ac01", {2325, 44033}, {1, 2}},
      {L"\U00000915\U00000308\U0000ac01", {2325, 44033}, {2, 3}},
-     {L"\U00000915\U00000900", {2325}, {2}},
-     {L"\U00000915\U00000308\U00000900", {2325}, {3}},
      {L"\U00000915\U00000903", {2325}, {2}},
      {L"\U00000915\U00000308\U00000903", {2325}, {3}},
      {L"\U00000915\U00000904", {2325, 2308}, {1, 2}},
@@ -2134,8 +1958,8 @@ std::array<data<wchar_t>, 1187> data_utf16 = {{
      {L"\U00000915\U00000308\U0000231a", {2325, 8986}, {2, 3}},
      {L"\U00000915\U00000300", {2325}, {2}},
      {L"\U00000915\U00000308\U00000300", {2325}, {3}},
-     {L"\U00000915\U0000093c", {2325}, {2}},
-     {L"\U00000915\U00000308\U0000093c", {2325}, {3}},
+     {L"\U00000915\U00000900", {2325}, {2}},
+     {L"\U00000915\U00000308\U00000900", {2325}, {3}},
      {L"\U00000915\U0000094d", {2325}, {2}},
      {L"\U00000915\U00000308\U0000094d", {2325}, {3}},
      {L"\U00000915\U0000200d", {2325}, {2}},
@@ -2150,8 +1974,8 @@ std::array<data<wchar_t>, 1187> data_utf16 = {{
      {L"\U0000231a\U00000308\U0000000a", {8986, 10}, {2, 3}},
      {L"\U0000231a\U00000001", {8986, 1}, {1, 2}},
      {L"\U0000231a\U00000308\U00000001", {8986, 1}, {2, 3}},
-     {L"\U0000231a\U0000034f", {8986}, {2}},
-     {L"\U0000231a\U00000308\U0000034f", {8986}, {3}},
+     {L"\U0000231a\U0000200c", {8986}, {2}},
+     {L"\U0000231a\U00000308\U0000200c", {8986}, {3}},
      {L"\U0000231a\U0001f1e6", {8986, 127462}, {1, 3}},
      {L"\U0000231a\U00000308\U0001f1e6", {8986, 127462}, {2, 4}},
      {L"\U0000231a\U00000600", {8986, 1536}, {1, 2}},
@@ -2168,8 +1992,6 @@ std::array<data<wchar_t>, 1187> data_utf16 = {{
      {L"\U0000231a\U00000308\U0000ac00", {8986, 44032}, {2, 3}},
      {L"\U0000231a\U0000ac01", {8986, 44033}, {1, 2}},
      {L"\U0000231a\U00000308\U0000ac01", {8986, 44033}, {2, 3}},
-     {L"\U0000231a\U00000900", {8986}, {2}},
-     {L"\U0000231a\U00000308\U00000900", {8986}, {3}},
      {L"\U0000231a\U00000903", {8986}, {2}},
      {L"\U0000231a\U00000308\U00000903", {8986}, {3}},
      {L"\U0000231a\U00000904", {8986, 2308}, {1, 2}},
@@ -2182,8 +2004,8 @@ std::array<data<wchar_t>, 1187> data_utf16 = {{
      {L"\U0000231a\U00000308\U0000231a", {8986, 8986}, {2, 3}},
      {L"\U0000231a\U00000300", {8986}, {2}},
      {L"\U0000231a\U00000308\U00000300", {8986}, {3}},
-     {L"\U0000231a\U0000093c", {8986}, {2}},
-     {L"\U0000231a\U00000308\U0000093c", {8986}, {3}},
+     {L"\U0000231a\U00000900", {8986}, {2}},
+     {L"\U0000231a\U00000308\U00000900", {8986}, {3}},
      {L"\U0000231a\U0000094d", {8986}, {2}},
      {L"\U0000231a\U00000308\U0000094d", {8986}, {3}},
      {L"\U0000231a\U0000200d", {8986}, {2}},
@@ -2198,8 +2020,8 @@ std::array<data<wchar_t>, 1187> data_utf16 = {{
      {L"\U00000300\U00000308\U0000000a", {768, 10}, {2, 3}},
      {L"\U00000300\U00000001", {768, 1}, {1, 2}},
      {L"\U00000300\U00000308\U00000001", {768, 1}, {2, 3}},
-     {L"\U00000300\U0000034f", {768}, {2}},
-     {L"\U00000300\U00000308\U0000034f", {768}, {3}},
+     {L"\U00000300\U0000200c", {768}, {2}},
+     {L"\U00000300\U00000308\U0000200c", {768}, {3}},
      {L"\U00000300\U0001f1e6", {768, 127462}, {1, 3}},
      {L"\U00000300\U00000308\U0001f1e6", {768, 127462}, {2, 4}},
      {L"\U00000300\U00000600", {768, 1536}, {1, 2}},
@@ -2216,8 +2038,6 @@ std::array<data<wchar_t>, 1187> data_utf16 = {{
      {L"\U00000300\U00000308\U0000ac00", {768, 44032}, {2, 3}},
      {L"\U00000300\U0000ac01", {768, 44033}, {1, 2}},
      {L"\U00000300\U00000308\U0000ac01", {768, 44033}, {2, 3}},
-     {L"\U00000300\U00000900", {768}, {2}},
-     {L"\U00000300\U00000308\U00000900", {768}, {3}},
      {L"\U00000300\U00000903", {768}, {2}},
      {L"\U00000300\U00000308\U00000903", {768}, {3}},
      {L"\U00000300\U00000904", {768, 2308}, {1, 2}},
@@ -2230,62 +2050,60 @@ std::array<data<wchar_t>, 1187> data_utf16 = {{
      {L"\U00000300\U00000308\U0000231a", {768, 8986}, {2, 3}},
      {L"\U00000300\U00000300", {768}, {2}},
      {L"\U00000300\U00000308\U00000300", {768}, {3}},
-     {L"\U00000300\U0000093c", {768}, {2}},
-     {L"\U00000300\U00000308\U0000093c", {768}, {3}},
+     {L"\U00000300\U00000900", {768}, {2}},
+     {L"\U00000300\U00000308\U00000900", {768}, {3}},
      {L"\U00000300\U0000094d", {768}, {2}},
      {L"\U00000300\U00000308\U0000094d", {768}, {3}},
      {L"\U00000300\U0000200d", {768}, {2}},
      {L"\U00000300\U00000308\U0000200d", {768}, {3}},
      {L"\U00000300\U00000378", {768, 888}, {1, 2}},
      {L"\U00000300\U00000308\U00000378", {768, 888}, {2, 3}},
-     {L"\U0000093c\U00000020", {2364, 32}, {1, 2}},
-     {L"\U0000093c\U00000308\U00000020", {2364, 32}, {2, 3}},
-     {L"\U0000093c\U0000000d", {2364, 13}, {1, 2}},
-     {L"\U0000093c\U00000308\U0000000d", {2364, 13}, {2, 3}},
-     {L"\U0000093c\U0000000a", {2364, 10}, {1, 2}},
-     {L"\U0000093c\U00000308\U0000000a", {2364, 10}, {2, 3}},
-     {L"\U0000093c\U00000001", {2364, 1}, {1, 2}},
-     {L"\U0000093c\U00000308\U00000001", {2364, 1}, {2, 3}},
-     {L"\U0000093c\U0000034f", {2364}, {2}},
-     {L"\U0000093c\U00000308\U0000034f", {2364}, {3}},
-     {L"\U0000093c\U0001f1e6", {2364, 127462}, {1, 3}},
-     {L"\U0000093c\U00000308\U0001f1e6", {2364, 127462}, {2, 4}},
-     {L"\U0000093c\U00000600", {2364, 1536}, {1, 2}},
-     {L"\U0000093c\U00000308\U00000600", {2364, 1536}, {2, 3}},
-     {L"\U0000093c\U00000a03", {2364}, {2}},
-     {L"\U0000093c\U00000308\U00000a03", {2364}, {3}},
-     {L"\U0000093c\U00001100", {2364, 4352}, {1, 2}},
-     {L"\U0000093c\U00000308\U00001100", {2364, 4352}, {2, 3}},
-     {L"\U0000093c\U00001160", {2364, 4448}, {1, 2}},
-     {L"\U0000093c\U00000308\U00001160", {2364, 4448}, {2, 3}},
-     {L"\U0000093c\U000011a8", {2364, 4520}, {1, 2}},
-     {L"\U0000093c\U00000308\U000011a8", {2364, 4520}, {2, 3}},
-     {L"\U0000093c\U0000ac00", {2364, 44032}, {1, 2}},
-     {L"\U0000093c\U00000308\U0000ac00", {2364, 44032}, {2, 3}},
-     {L"\U0000093c\U0000ac01", {2364, 44033}, {1, 2}},
-     {L"\U0000093c\U00000308\U0000ac01", {2364, 44033}, {2, 3}},
-     {L"\U0000093c\U00000900", {2364}, {2}},
-     {L"\U0000093c\U00000308\U00000900", {2364}, {3}},
-     {L"\U0000093c\U00000903", {2364}, {2}},
-     {L"\U0000093c\U00000308\U00000903", {2364}, {3}},
-     {L"\U0000093c\U00000904", {2364, 2308}, {1, 2}},
-     {L"\U0000093c\U00000308\U00000904", {2364, 2308}, {2, 3}},
-     {L"\U0000093c\U00000d4e", {2364, 3406}, {1, 2}},
-     {L"\U0000093c\U00000308\U00000d4e", {2364, 3406}, {2, 3}},
-     {L"\U0000093c\U00000915", {2364, 2325}, {1, 2}},
-     {L"\U0000093c\U00000308\U00000915", {2364, 2325}, {2, 3}},
-     {L"\U0000093c\U0000231a", {2364, 8986}, {1, 2}},
-     {L"\U0000093c\U00000308\U0000231a", {2364, 8986}, {2, 3}},
-     {L"\U0000093c\U00000300", {2364}, {2}},
-     {L"\U0000093c\U00000308\U00000300", {2364}, {3}},
-     {L"\U0000093c\U0000093c", {2364}, {2}},
-     {L"\U0000093c\U00000308\U0000093c", {2364}, {3}},
-     {L"\U0000093c\U0000094d", {2364}, {2}},
-     {L"\U0000093c\U00000308\U0000094d", {2364}, {3}},
-     {L"\U0000093c\U0000200d", {2364}, {2}},
-     {L"\U0000093c\U00000308\U0000200d", {2364}, {3}},
-     {L"\U0000093c\U00000378", {2364, 888}, {1, 2}},
-     {L"\U0000093c\U00000308\U00000378", {2364, 888}, {2, 3}},
+     {L"\U00000900\U00000020", {2304, 32}, {1, 2}},
+     {L"\U00000900\U00000308\U00000020", {2304, 32}, {2, 3}},
+     {L"\U00000900\U0000000d", {2304, 13}, {1, 2}},
+     {L"\U00000900\U00000308\U0000000d", {2304, 13}, {2, 3}},
+     {L"\U00000900\U0000000a", {2304, 10}, {1, 2}},
+     {L"\U00000900\U00000308\U0000000a", {2304, 10}, {2, 3}},
+     {L"\U00000900\U00000001", {2304, 1}, {1, 2}},
+     {L"\U00000900\U00000308\U00000001", {2304, 1}, {2, 3}},
+     {L"\U00000900\U0000200c", {2304}, {2}},
+     {L"\U00000900\U00000308\U0000200c", {2304}, {3}},
+     {L"\U00000900\U0001f1e6", {2304, 127462}, {1, 3}},
+     {L"\U00000900\U00000308\U0001f1e6", {2304, 127462}, {2, 4}},
+     {L"\U00000900\U00000600", {2304, 1536}, {1, 2}},
+     {L"\U00000900\U00000308\U00000600", {2304, 1536}, {2, 3}},
+     {L"\U00000900\U00000a03", {2304}, {2}},
+     {L"\U00000900\U00000308\U00000a03", {2304}, {3}},
+     {L"\U00000900\U00001100", {2304, 4352}, {1, 2}},
+     {L"\U00000900\U00000308\U00001100", {2304, 4352}, {2, 3}},
+     {L"\U00000900\U00001160", {2304, 4448}, {1, 2}},
+     {L"\U00000900\U00000308\U00001160", {2304, 4448}, {2, 3}},
+     {L"\U00000900\U000011a8", {2304, 4520}, {1, 2}},
+     {L"\U00000900\U00000308\U000011a8", {2304, 4520}, {2, 3}},
+     {L"\U00000900\U0000ac00", {2304, 44032}, {1, 2}},
+     {L"\U00000900\U00000308\U0000ac00", {2304, 44032}, {2, 3}},
+     {L"\U00000900\U0000ac01", {2304, 44033}, {1, 2}},
+     {L"\U00000900\U00000308\U0000ac01", {2304, 44033}, {2, 3}},
+     {L"\U00000900\U00000903", {2304}, {2}},
+     {L"\U00000900\U00000308\U00000903", {2304}, {3}},
+     {L"\U00000900\U00000904", {2304, 2308}, {1, 2}},
+     {L"\U00000900\U00000308\U00000904", {2304, 2308}, {2, 3}},
+     {L"\U00000900\U00000d4e", {2304, 3406}, {1, 2}},
+     {L"\U00000900\U00000308\U00000d4e", {2304, 3406}, {2, 3}},
+     {L"\U00000900\U00000915", {2304, 2325}, {1, 2}},
+     {L"\U00000900\U00000308\U00000915", {2304, 2325}, {2, 3}},
+     {L"\U00000900\U0000231a", {2304, 8986}, {1, 2}},
+     {L"\U00000900\U00000308\U0000231a", {2304, 8986}, {2, 3}},
+     {L"\U00000900\U00000300", {2304}, {2}},
+     {L"\U00000900\U00000308\U00000300", {2304}, {3}},
+     {L"\U00000900\U00000900", {2304}, {2}},
+     {L"\U00000900\U00000308\U00000900", {2304}, {3}},
+     {L"\U00000900\U0000094d", {2304}, {2}},
+     {L"\U00000900\U00000308\U0000094d", {2304}, {3}},
+     {L"\U00000900\U0000200d", {2304}, {2}},
+     {L"\U00000900\U00000308\U0000200d", {2304}, {3}},
+     {L"\U00000900\U00000378", {2304, 888}, {1, 2}},
+     {L"\U00000900\U00000308\U00000378", {2304, 888}, {2, 3}},
      {L"\U0000094d\U00000020", {2381, 32}, {1, 2}},
      {L"\U0000094d\U00000308\U00000020", {2381, 32}, {2, 3}},
      {L"\U0000094d\U0000000d", {2381, 13}, {1, 2}},
@@ -2294,8 +2112,8 @@ std::array<data<wchar_t>, 1187> data_utf16 = {{
      {L"\U0000094d\U00000308\U0000000a", {2381, 10}, {2, 3}},
      {L"\U0000094d\U00000001", {2381, 1}, {1, 2}},
      {L"\U0000094d\U00000308\U00000001", {2381, 1}, {2, 3}},
-     {L"\U0000094d\U0000034f", {2381}, {2}},
-     {L"\U0000094d\U00000308\U0000034f", {2381}, {3}},
+     {L"\U0000094d\U0000200c", {2381}, {2}},
+     {L"\U0000094d\U00000308\U0000200c", {2381}, {3}},
      {L"\U0000094d\U0001f1e6", {2381, 127462}, {1, 3}},
      {L"\U0000094d\U00000308\U0001f1e6", {2381, 127462}, {2, 4}},
      {L"\U0000094d\U00000600", {2381, 1536}, {1, 2}},
@@ -2312,8 +2130,6 @@ std::array<data<wchar_t>, 1187> data_utf16 = {{
      {L"\U0000094d\U00000308\U0000ac00", {2381, 44032}, {2, 3}},
      {L"\U0000094d\U0000ac01", {2381, 44033}, {1, 2}},
      {L"\U0000094d\U00000308\U0000ac01", {2381, 44033}, {2, 3}},
-     {L"\U0000094d\U00000900", {2381}, {2}},
-     {L"\U0000094d\U00000308\U00000900", {2381}, {3}},
      {L"\U0000094d\U00000903", {2381}, {2}},
      {L"\U0000094d\U00000308\U00000903", {2381}, {3}},
      {L"\U0000094d\U00000904", {2381, 2308}, {1, 2}},
@@ -2326,8 +2142,8 @@ std::array<data<wchar_t>, 1187> data_utf16 = {{
      {L"\U0000094d\U00000308\U0000231a", {2381, 8986}, {2, 3}},
      {L"\U0000094d\U00000300", {2381}, {2}},
      {L"\U0000094d\U00000308\U00000300", {2381}, {3}},
-     {L"\U0000094d\U0000093c", {2381}, {2}},
-     {L"\U0000094d\U00000308\U0000093c", {2381}, {3}},
+     {L"\U0000094d\U00000900", {2381}, {2}},
+     {L"\U0000094d\U00000308\U00000900", {2381}, {3}},
      {L"\U0000094d\U0000094d", {2381}, {2}},
      {L"\U0000094d\U00000308\U0000094d", {2381}, {3}},
      {L"\U0000094d\U0000200d", {2381}, {2}},
@@ -2342,8 +2158,8 @@ std::array<data<wchar_t>, 1187> data_utf16 = {{
      {L"\U0000200d\U00000308\U0000000a", {8205, 10}, {2, 3}},
      {L"\U0000200d\U00000001", {8205, 1}, {1, 2}},
      {L"\U0000200d\U00000308\U00000001", {8205, 1}, {2, 3}},
-     {L"\U0000200d\U0000034f", {8205}, {2}},
-     {L"\U0000200d\U00000308\U0000034f", {8205}, {3}},
+     {L"\U0000200d\U0000200c", {8205}, {2}},
+     {L"\U0000200d\U00000308\U0000200c", {8205}, {3}},
      {L"\U0000200d\U0001f1e6", {8205, 127462}, {1, 3}},
      {L"\U0000200d\U00000308\U0001f1e6", {8205, 127462}, {2, 4}},
      {L"\U0000200d\U00000600", {8205, 1536}, {1, 2}},
@@ -2360,8 +2176,6 @@ std::array<data<wchar_t>, 1187> data_utf16 = {{
      {L"\U0000200d\U00000308\U0000ac00", {8205, 44032}, {2, 3}},
      {L"\U0000200d\U0000ac01", {8205, 44033}, {1, 2}},
      {L"\U0000200d\U00000308\U0000ac01", {8205, 44033}, {2, 3}},
-     {L"\U0000200d\U00000900", {8205}, {2}},
-     {L"\U0000200d\U00000308\U00000900", {8205}, {3}},
      {L"\U0000200d\U00000903", {8205}, {2}},
      {L"\U0000200d\U00000308\U00000903", {8205}, {3}},
      {L"\U0000200d\U00000904", {8205, 2308}, {1, 2}},
@@ -2374,8 +2188,8 @@ std::array<data<wchar_t>, 1187> data_utf16 = {{
      {L"\U0000200d\U00000308\U0000231a", {8205, 8986}, {2, 3}},
      {L"\U0000200d\U00000300", {8205}, {2}},
      {L"\U0000200d\U00000308\U00000300", {8205}, {3}},
-     {L"\U0000200d\U0000093c", {8205}, {2}},
-     {L"\U0000200d\U00000308\U0000093c", {8205}, {3}},
+     {L"\U0000200d\U00000900", {8205}, {2}},
+     {L"\U0000200d\U00000308\U00000900", {8205}, {3}},
      {L"\U0000200d\U0000094d", {8205}, {2}},
      {L"\U0000200d\U00000308\U0000094d", {8205}, {3}},
      {L"\U0000200d\U0000200d", {8205}, {2}},
@@ -2390,8 +2204,8 @@ std::array<data<wchar_t>, 1187> data_utf16 = {{
      {L"\U00000378\U00000308\U0000000a", {888, 10}, {2, 3}},
      {L"\U00000378\U00000001", {888, 1}, {1, 2}},
      {L"\U00000378\U00000308\U00000001", {888, 1}, {2, 3}},
-     {L"\U00000378\U0000034f", {888}, {2}},
-     {L"\U00000378\U00000308\U0000034f", {888}, {3}},
+     {L"\U00000378\U0000200c", {888}, {2}},
+     {L"\U00000378\U00000308\U0000200c", {888}, {3}},
      {L"\U00000378\U0001f1e6", {888, 127462}, {1, 3}},
      {L"\U00000378\U00000308\U0001f1e6", {888, 127462}, {2, 4}},
      {L"\U00000378\U00000600", {888, 1536}, {1, 2}},
@@ -2408,8 +2222,6 @@ std::array<data<wchar_t>, 1187> data_utf16 = {{
      {L"\U00000378\U00000308\U0000ac00", {888, 44032}, {2, 3}},
      {L"\U00000378\U0000ac01", {888, 44033}, {1, 2}},
      {L"\U00000378\U00000308\U0000ac01", {888, 44033}, {2, 3}},
-     {L"\U00000378\U00000900", {888}, {2}},
-     {L"\U00000378\U00000308\U00000900", {888}, {3}},
      {L"\U00000378\U00000903", {888}, {2}},
      {L"\U00000378\U00000308\U00000903", {888}, {3}},
      {L"\U00000378\U00000904", {888, 2308}, {1, 2}},
@@ -2422,8 +2234,8 @@ std::array<data<wchar_t>, 1187> data_utf16 = {{
      {L"\U00000378\U00000308\U0000231a", {888, 8986}, {2, 3}},
      {L"\U00000378\U00000300", {888}, {2}},
      {L"\U00000378\U00000308\U00000300", {888}, {3}},
-     {L"\U00000378\U0000093c", {888}, {2}},
-     {L"\U00000378\U00000308\U0000093c", {888}, {3}},
+     {L"\U00000378\U00000900", {888}, {2}},
+     {L"\U00000378\U00000308\U00000900", {888}, {3}},
      {L"\U00000378\U0000094d", {888}, {2}},
      {L"\U00000378\U00000308\U0000094d", {888}, {3}},
      {L"\U00000378\U0000200d", {888}, {2}},
@@ -2471,7 +2283,7 @@ std::array<data<wchar_t>, 1187> data_utf16 = {{
 /// Note that most of the data for the UTF-16 and UTF-32 are identical. However
 /// since the size of the code units differ the breaks can contain different
 /// values.
-std::array<data<wchar_t>, 1187> data_utf32 = {{
+std::array<data<wchar_t>, 1093> data_utf32 = {{
      {L"\U00000020\U00000020", {32, 32}, {1, 2}},
      {L"\U00000020\U00000308\U00000020", {32, 32}, {2, 3}},
      {L"\U00000020\U0000000d", {32, 13}, {1, 2}},
@@ -2480,8 +2292,8 @@ std::array<data<wchar_t>, 1187> data_utf32 = {{
      {L"\U00000020\U00000308\U0000000a", {32, 10}, {2, 3}},
      {L"\U00000020\U00000001", {32, 1}, {1, 2}},
      {L"\U00000020\U00000308\U00000001", {32, 1}, {2, 3}},
-     {L"\U00000020\U0000034f", {32}, {2}},
-     {L"\U00000020\U00000308\U0000034f", {32}, {3}},
+     {L"\U00000020\U0000200c", {32}, {2}},
+     {L"\U00000020\U00000308\U0000200c", {32}, {3}},
      {L"\U00000020\U0001f1e6", {32, 127462}, {1, 2}},
      {L"\U00000020\U00000308\U0001f1e6", {32, 127462}, {2, 3}},
      {L"\U00000020\U00000600", {32, 1536}, {1, 2}},
@@ -2498,8 +2310,6 @@ std::array<data<wchar_t>, 1187> data_utf32 = {{
      {L"\U00000020\U00000308\U0000ac00", {32, 44032}, {2, 3}},
      {L"\U00000020\U0000ac01", {32, 44033}, {1, 2}},
      {L"\U00000020\U00000308\U0000ac01", {32, 44033}, {2, 3}},
-     {L"\U00000020\U00000900", {32}, {2}},
-     {L"\U00000020\U00000308\U00000900", {32}, {3}},
      {L"\U00000020\U00000903", {32}, {2}},
      {L"\U00000020\U00000308\U00000903", {32}, {3}},
      {L"\U00000020\U00000904", {32, 2308}, {1, 2}},
@@ -2512,8 +2322,8 @@ std::array<data<wchar_t>, 1187> data_utf32 = {{
      {L"\U00000020\U00000308\U0000231a", {32, 8986}, {2, 3}},
      {L"\U00000020\U00000300", {32}, {2}},
      {L"\U00000020\U00000308\U00000300", {32}, {3}},
-     {L"\U00000020\U0000093c", {32}, {2}},
-     {L"\U00000020\U00000308\U0000093c", {32}, {3}},
+     {L"\U00000020\U00000900", {32}, {2}},
+     {L"\U00000020\U00000308\U00000900", {32}, {3}},
      {L"\U00000020\U0000094d", {32}, {2}},
      {L"\U00000020\U00000308\U0000094d", {32}, {3}},
      {L"\U00000020\U0000200d", {32}, {2}},
@@ -2528,8 +2338,8 @@ std::array<data<wchar_t>, 1187> data_utf32 = {{
      {L"\U0000000d\U00000308\U0000000a", {13, 776, 10}, {1, 2, 3}},
      {L"\U0000000d\U00000001", {13, 1}, {1, 2}},
      {L"\U0000000d\U00000308\U00000001", {13, 776, 1}, {1, 2, 3}},
-     {L"\U0000000d\U0000034f", {13, 847}, {1, 2}},
-     {L"\U0000000d\U00000308\U0000034f", {13, 776}, {1, 3}},
+     {L"\U0000000d\U0000200c", {13, 8204}, {1, 2}},
+     {L"\U0000000d\U00000308\U0000200c", {13, 776}, {1, 3}},
      {L"\U0000000d\U0001f1e6", {13, 127462}, {1, 2}},
      {L"\U0000000d\U00000308\U0001f1e6", {13, 776, 127462}, {1, 2, 3}},
      {L"\U0000000d\U00000600", {13, 1536}, {1, 2}},
@@ -2546,8 +2356,6 @@ std::array<data<wchar_t>, 1187> data_utf32 = {{
      {L"\U0000000d\U00000308\U0000ac00", {13, 776, 44032}, {1, 2, 3}},
      {L"\U0000000d\U0000ac01", {13, 44033}, {1, 2}},
      {L"\U0000000d\U00000308\U0000ac01", {13, 776, 44033}, {1, 2, 3}},
-     {L"\U0000000d\U00000900", {13, 2304}, {1, 2}},
-     {L"\U0000000d\U00000308\U00000900", {13, 776}, {1, 3}},
      {L"\U0000000d\U00000903", {13, 2307}, {1, 2}},
      {L"\U0000000d\U00000308\U00000903", {13, 776}, {1, 3}},
      {L"\U0000000d\U00000904", {13, 2308}, {1, 2}},
@@ -2560,8 +2368,8 @@ std::array<data<wchar_t>, 1187> data_utf32 = {{
      {L"\U0000000d\U00000308\U0000231a", {13, 776, 8986}, {1, 2, 3}},
      {L"\U0000000d\U00000300", {13, 768}, {1, 2}},
      {L"\U0000000d\U00000308\U00000300", {13, 776}, {1, 3}},
-     {L"\U0000000d\U0000093c", {13, 2364}, {1, 2}},
-     {L"\U0000000d\U00000308\U0000093c", {13, 776}, {1, 3}},
+     {L"\U0000000d\U00000900", {13, 2304}, {1, 2}},
+     {L"\U0000000d\U00000308\U00000900", {13, 776}, {1, 3}},
      {L"\U0000000d\U0000094d", {13, 2381}, {1, 2}},
      {L"\U0000000d\U00000308\U0000094d", {13, 776}, {1, 3}},
      {L"\U0000000d\U0000200d", {13, 8205}, {1, 2}},
@@ -2576,8 +2384,8 @@ std::array<data<wchar_t>, 1187> data_utf32 = {{
      {L"\U0000000a\U00000308\U0000000a", {10, 776, 10}, {1, 2, 3}},
      {L"\U0000000a\U00000001", {10, 1}, {1, 2}},
      {L"\U0000000a\U00000308\U00000001", {10, 776, 1}, {1, 2, 3}},
-     {L"\U0000000a\U0000034f", {10, 847}, {1, 2}},
-     {L"\U0000000a\U00000308\U0000034f", {10, 776}, {1, 3}},
+     {L"\U0000000a\U0000200c", {10, 8204}, {1, 2}},
+     {L"\U0000000a\U00000308\U0000200c", {10, 776}, {1, 3}},
      {L"\U0000000a\U0001f1e6", {10, 127462}, {1, 2}},
      {L"\U0000000a\U00000308\U0001f1e6", {10, 776, 127462}, {1, 2, 3}},
      {L"\U0000000a\U00000600", {10, 1536}, {1, 2}},
@@ -2594,8 +2402,6 @@ std::array<data<wchar_t>, 1187> data_utf32 = {{
      {L"\U0000000a\U00000308\U0000ac00", {10, 776, 44032}, {1, 2, 3}},
      {L"\U0000000a\U0000ac01", {10, 44033}, {1, 2}},
      {L"\U0000000a\U00000308\U0000ac01", {10, 776, 44033}, {1, 2, 3}},
-     {L"\U0000000a\U00000900", {10, 2304}, {1, 2}},
-     {L"\U0000000a\U00000308\U00000900", {10, 776}, {1, 3}},
      {L"\U0000000a\U00000903", {10, 2307}, {1, 2}},
      {L"\U0000000a\U00000308\U00000903", {10, 776}, {1, 3}},
      {L"\U0000000a\U00000904", {10, 2308}, {1, 2}},
@@ -2608,8 +2414,8 @@ std::array<data<wchar_t>, 1187> data_utf32 = {{
      {L"\U0000000a\U00000308\U0000231a", {10, 776, 8986}, {1, 2, 3}},
      {L"\U0000000a\U00000300", {10, 768}, {1, 2}},
      {L"\U0000000a\U00000308\U00000300", {10, 776}, {1, 3}},
-     {L"\U0000000a\U0000093c", {10, 2364}, {1, 2}},
-     {L"\U0000000a\U00000308\U0000093c", {10, 776}, {1, 3}},
+     {L"\U0000000a\U00000900", {10, 2304}, {1, 2}},
+     {L"\U0000000a\U00000308\U00000900", {10, 776}, {1, 3}},
      {L"\U0000000a\U0000094d", {10, 2381}, {1, 2}},
      {L"\U0000000a\U00000308\U0000094d", {10, 776}, {1, 3}},
      {L"\U0000000a\U0000200d", {10, 8205}, {1, 2}},
@@ -2624,8 +2430,8 @@ std::array<data<wchar_t>, 1187> data_utf32 = {{
      {L"\U00000001\U00000308\U0000000a", {1, 776, 10}, {1, 2, 3}},
      {L"\U00000001\U00000001", {1, 1}, {1, 2}},
      {L"\U00000001\U00000308\U00000001", {1, 776, 1}, {1, 2, 3}},
-     {L"\U00000001\U0000034f", {1, 847}, {1, 2}},
-     {L"\U00000001\U00000308\U0000034f", {1, 776}, {1, 3}},
+     {L"\U00000001\U0000200c", {1, 8204}, {1, 2}},
+     {L"\U00000001\U00000308\U0000200c", {1, 776}, {1, 3}},
      {L"\U00000001\U0001f1e6", {1, 127462}, {1, 2}},
      {L"\U00000001\U00000308\U0001f1e6", {1, 776, 127462}, {1, 2, 3}},
      {L"\U00000001\U00000600", {1, 1536}, {1, 2}},
@@ -2642,8 +2448,6 @@ std::array<data<wchar_t>, 1187> data_utf32 = {{
      {L"\U00000001\U00000308\U0000ac00", {1, 776, 44032}, {1, 2, 3}},
      {L"\U00000001\U0000ac01", {1, 44033}, {1, 2}},
      {L"\U00000001\U00000308\U0000ac01", {1, 776, 44033}, {1, 2, 3}},
-     {L"\U00000001\U00000900", {1, 2304}, {1, 2}},
-     {L"\U00000001\U00000308\U00000900", {1, 776}, {1, 3}},
      {L"\U00000001\U00000903", {1, 2307}, {1, 2}},
      {L"\U00000001\U00000308\U00000903", {1, 776}, {1, 3}},
      {L"\U00000001\U00000904", {1, 2308}, {1, 2}},
@@ -2656,62 +2460,60 @@ std::array<data<wchar_t>, 1187> data_utf32 = {{
      {L"\U00000001\U00000308\U0000231a", {1, 776, 8986}, {1, 2, 3}},
      {L"\U00000001\U00000300", {1, 768}, {1, 2}},
      {L"\U00000001\U00000308\U00000300", {1, 776}, {1, 3}},
-     {L"\U00000001\U0000093c", {1, 2364}, {1, 2}},
-     {L"\U00000001\U00000308\U0000093c", {1, 776}, {1, 3}},
+     {L"\U00000001\U00000900", {1, 2304}, {1, 2}},
+     {L"\U00000001\U00000308\U00000900", {1, 776}, {1, 3}},
      {L"\U00000001\U0000094d", {1, 2381}, {1, 2}},
      {L"\U00000001\U00000308\U0000094d", {1, 776}, {1, 3}},
      {L"\U00000001\U0000200d", {1, 8205}, {1, 2}},
      {L"\U00000001\U00000308\U0000200d", {1, 776}, {1, 3}},
      {L"\U00000001\U00000378", {1, 888}, {1, 2}},
      {L"\U00000001\U00000308\U00000378", {1, 776, 888}, {1, 2, 3}},
-     {L"\U0000034f\U00000020", {847, 32}, {1, 2}},
-     {L"\U0000034f\U00000308\U00000020", {847, 32}, {2, 3}},
-     {L"\U0000034f\U0000000d", {847, 13}, {1, 2}},
-     {L"\U0000034f\U00000308\U0000000d", {847, 13}, {2, 3}},
-     {L"\U0000034f\U0000000a", {847, 10}, {1, 2}},
-     {L"\U0000034f\U00000308\U0000000a", {847, 10}, {2, 3}},
-     {L"\U0000034f\U00000001", {847, 1}, {1, 2}},
-     {L"\U0000034f\U00000308\U00000001", {847, 1}, {2, 3}},
-     {L"\U0000034f\U0000034f", {847}, {2}},
-     {L"\U0000034f\U00000308\U0000034f", {847}, {3}},
-     {L"\U0000034f\U0001f1e6", {847, 127462}, {1, 2}},
-     {L"\U0000034f\U00000308\U0001f1e6", {847, 127462}, {2, 3}},
-     {L"\U0000034f\U00000600", {847, 1536}, {1, 2}},
-     {L"\U0000034f\U00000308\U00000600", {847, 1536}, {2, 3}},
-     {L"\U0000034f\U00000a03", {847}, {2}},
-     {L"\U0000034f\U00000308\U00000a03", {847}, {3}},
-     {L"\U0000034f\U00001100", {847, 4352}, {1, 2}},
-     {L"\U0000034f\U00000308\U00001100", {847, 4352}, {2, 3}},
-     {L"\U0000034f\U00001160", {847, 4448}, {1, 2}},
-     {L"\U0000034f\U00000308\U00001160", {847, 4448}, {2, 3}},
-     {L"\U0000034f\U000011a8", {847, 4520}, {1, 2}},
-     {L"\U0000034f\U00000308\U000011a8", {847, 4520}, {2, 3}},
-     {L"\U0000034f\U0000ac00", {847, 44032}, {1, 2}},
-     {L"\U0000034f\U00000308\U0000ac00", {847, 44032}, {2, 3}},
-     {L"\U0000034f\U0000ac01", {847, 44033}, {1, 2}},
-     {L"\U0000034f\U00000308\U0000ac01", {847, 44033}, {2, 3}},
-     {L"\U0000034f\U00000900", {847}, {2}},
-     {L"\U0000034f\U00000308\U00000900", {847}, {3}},
-     {L"\U0000034f\U00000903", {847}, {2}},
-     {L"\U0000034f\U00000308\U00000903", {847}, {3}},
-     {L"\U0000034f\U00000904", {847, 2308}, {1, 2}},
-     {L"\U0000034f\U00000308\U00000904", {847, 2308}, {2, 3}},
-     {L"\U0000034f\U00000d4e", {847, 3406}, {1, 2}},
-     {L"\U0000034f\U00000308\U00000d4e", {847, 3406}, {2, 3}},
-     {L"\U0000034f\U00000915", {847, 2325}, {1, 2}},
-     {L"\U0000034f\U00000308\U00000915", {847, 2325}, {2, 3}},
-     {L"\U0000034f\U0000231a", {847, 8986}, {1, 2}},
-     {L"\U0000034f\U00000308\U0000231a", {847, 8986}, {2, 3}},
-     {L"\U0000034f\U00000300", {847}, {2}},
-     {L"\U0000034f\U00000308\U00000300", {847}, {3}},
-     {L"\U0000034f\U0000093c", {847}, {2}},
-     {L"\U0000034f\U00000308\U0000093c", {847}, {3}},
-     {L"\U0000034f\U0000094d", {847}, {2}},
-     {L"\U0000034f\U00000308\U0000094d", {847}, {3}},
-     {L"\U0000034f\U0000200d", {847}, {2}},
-     {L"\U0000034f\U00000308\U0000200d", {847}, {3}},
-     {L"\U0000034f\U00000378", {847, 888}, {1, 2}},
-     {L"\U0000034f\U00000308\U00000378", {847, 888}, {2, 3}},
+     {L"\U0000200c\U00000020", {8204, 32}, {1, 2}},
+     {L"\U0000200c\U00000308\U00000020", {8204, 32}, {2, 3}},
+     {L"\U0000200c\U0000000d", {8204, 13}, {1, 2}},
+     {L"\U0000200c\U00000308\U0000000d", {8204, 13}, {2, 3}},
+     {L"\U0000200c\U0000000a", {8204, 10}, {1, 2}},
+     {L"\U0000200c\U00000308\U0000000a", {8204, 10}, {2, 3}},
+     {L"\U0000200c\U00000001", {8204, 1}, {1, 2}},
+     {L"\U0000200c\U00000308\U00000001", {8204, 1}, {2, 3}},
+     {L"\U0000200c\U0000200c", {8204}, {2}},
+     {L"\U0000200c\U00000308\U0000200c", {8204}, {3}},
+     {L"\U0000200c\U0001f1e6", {8204, 127462}, {1, 2}},
+     {L"\U0000200c\U00000308\U0001f1e6", {8204, 127462}, {2, 3}},
+     {L"\U0000200c\U00000600", {8204, 1536}, {1, 2}},
+     {L"\U0000200c\U00000308\U00000600", {8204, 1536}, {2, 3}},
+     {L"\U0000200c\U00000a03", {8204}, {2}},
+     {L"\U0000200c\U00000308\U00000a03", {8204}, {3}},
+     {L"\U0000200c\U00001100", {8204, 4352}, {1, 2}},
+     {L"\U0000200c\U00000308\U00001100", {8204, 4352}, {2, 3}},
+     {L"\U0000200c\U00001160", {8204, 4448}, {1, 2}},
+     {L"\U0000200c\U00000308\U00001160", {8204, 4448}, {2, 3}},
+     {L"\U0000200c\U000011a8", {8204, 4520}, {1, 2}},
+     {L"\U0000200c\U00000308\U000011a8", {8204, 4520}, {2, 3}},
+     {L"\U0000200c\U0000ac00", {8204, 44032}, {1, 2}},
+     {L"\U0000200c\U00000308\U0000ac00", {8204, 44032}, {2, 3}},
+     {L"\U0000200c\U0000ac01", {8204, 44033}, {1, 2}},
+     {L"\U0000200c\U00000308\U0000ac01", {8204, 44033}, {2, 3}},
+     {L"\U0000200c\U00000903", {8204}, {2}},
+     {L"\U0000200c\U00000308\U00000903", {8204}, {3}},
+     {L"\U0000200c\U00000904", {8204, 2308}, {1, 2}},
+     {L"\U0000200c\U00000308\U00000904", {8204, 2308}, {2, 3}},
+     {L"\U0000200c\U00000d4e", {8204, 3406}, {1, 2}},
+     {L"\U0000200c\U00000308\U00000d4e", {8204, 3406}, {2, 3}},
+     {L"\U0000200c\U00000915", {8204, 2325}, {1, 2}},
+     {L"\U0000200c\U00000308\U00000915", {8204, 2325}, {2, 3}},
+     {L"\U0000200c\U0000231a", {8204, 8986}, {1, 2}},
+     {L"\U0000200c\U00000308\U0000231a", {8204, 8986}, {2, 3}},
+     {L"\U0000200c\U00000300", {8204}, {2}},
+     {L"\U0000200c\U00000308\U00000300", {8204}, {3}},
+     {L"\U0000200c\U00000900", {8204}, {2}},
+     {L"\U0000200c\U00000308\U00000900", {8204}, {3}},
+     {L"\U0000200c\U0000094d", {8204}, {2}},
+     {L"\U0000200c\U00000308\U0000094d", {8204}, {3}},
+     {L"\U0000200c\U0000200d", {8204}, {2}},
+     {L"\U0000200c\U00000308\U0000200d", {8204}, {3}},
+     {L"\U0000200c\U00000378", {8204, 888}, {1, 2}},
+     {L"\U0000200c\U00000308\U00000378", {8204, 888}, {2, 3}},
      {L"\U0001f1e6\U00000020", {127462, 32}, {1, 2}},
      {L"\U0001f1e6\U00000308\U00000020", {127462, 32}, {2, 3}},
      {L"\U0001f1e6\U0000000d", {127462, 13}, {1, 2}},
@@ -2720,8 +2522,8 @@ std::array<data<wchar_t>, 1187> data_utf32 = {{
      {L"\U0001f1e6\U00000308\U0000000a", {127462, 10}, {2, 3}},
      {L"\U0001f1e6\U00000001", {127462, 1}, {1, 2}},
      {L"\U0001f1e6\U00000308\U00000001", {127462, 1}, {2, 3}},
-     {L"\U0001f1e6\U0000034f", {127462}, {2}},
-     {L"\U0001f1e6\U00000308\U0000034f", {127462}, {3}},
+     {L"\U0001f1e6\U0000200c", {127462}, {2}},
+     {L"\U0001f1e6\U00000308\U0000200c", {127462}, {3}},
      {L"\U0001f1e6\U0001f1e6", {127462}, {2}},
      {L"\U0001f1e6\U00000308\U0001f1e6", {127462, 127462}, {2, 3}},
      {L"\U0001f1e6\U00000600", {127462, 1536}, {1, 2}},
@@ -2738,8 +2540,6 @@ std::array<data<wchar_t>, 1187> data_utf32 = {{
      {L"\U0001f1e6\U00000308\U0000ac00", {127462, 44032}, {2, 3}},
      {L"\U0001f1e6\U0000ac01", {127462, 44033}, {1, 2}},
      {L"\U0001f1e6\U00000308\U0000ac01", {127462, 44033}, {2, 3}},
-     {L"\U0001f1e6\U00000900", {127462}, {2}},
-     {L"\U0001f1e6\U00000308\U00000900", {127462}, {3}},
      {L"\U0001f1e6\U00000903", {127462}, {2}},
      {L"\U0001f1e6\U00000308\U00000903", {127462}, {3}},
      {L"\U0001f1e6\U00000904", {127462, 2308}, {1, 2}},
@@ -2752,8 +2552,8 @@ std::array<data<wchar_t>, 1187> data_utf32 = {{
      {L"\U0001f1e6\U00000308\U0000231a", {127462, 8986}, {2, 3}},
      {L"\U0001f1e6\U00000300", {127462}, {2}},
      {L"\U0001f1e6\U00000308\U00000300", {127462}, {3}},
-     {L"\U0001f1e6\U0000093c", {127462}, {2}},
-     {L"\U0001f1e6\U00000308\U0000093c", {127462}, {3}},
+     {L"\U0001f1e6\U00000900", {127462}, {2}},
+     {L"\U0001f1e6\U00000308\U00000900", {127462}, {3}},
      {L"\U0001f1e6\U0000094d", {127462}, {2}},
      {L"\U0001f1e6\U00000308\U0000094d", {127462}, {3}},
      {L"\U0001f1e6\U0000200d", {127462}, {2}},
@@ -2768,8 +2568,8 @@ std::array<data<wchar_t>, 1187> data_utf32 = {{
      {L"\U00000600\U00000308\U0000000a", {1536, 10}, {2, 3}},
      {L"\U00000600\U00000001", {1536, 1}, {1, 2}},
      {L"\U00000600\U00000308\U00000001", {1536, 1}, {2, 3}},
-     {L"\U00000600\U0000034f", {1536}, {2}},
-     {L"\U00000600\U00000308\U0000034f", {1536}, {3}},
+     {L"\U00000600\U0000200c", {1536}, {2}},
+     {L"\U00000600\U00000308\U0000200c", {1536}, {3}},
      {L"\U00000600\U0001f1e6", {1536}, {2}},
      {L"\U00000600\U00000308\U0001f1e6", {1536, 127462}, {2, 3}},
      {L"\U00000600\U00000600", {1536}, {2}},
@@ -2786,8 +2586,6 @@ std::array<data<wchar_t>, 1187> data_utf32 = {{
      {L"\U00000600\U00000308\U0000ac00", {1536, 44032}, {2, 3}},
      {L"\U00000600\U0000ac01", {1536}, {2}},
      {L"\U00000600\U00000308\U0000ac01", {1536, 44033}, {2, 3}},
-     {L"\U00000600\U00000900", {1536}, {2}},
-     {L"\U00000600\U00000308\U00000900", {1536}, {3}},
      {L"\U00000600\U00000903", {1536}, {2}},
      {L"\U00000600\U00000308\U00000903", {1536}, {3}},
      {L"\U00000600\U00000904", {1536}, {2}},
@@ -2800,8 +2598,8 @@ std::array<data<wchar_t>, 1187> data_utf32 = {{
      {L"\U00000600\U00000308\U0000231a", {1536, 8986}, {2, 3}},
      {L"\U00000600\U00000300", {1536}, {2}},
      {L"\U00000600\U00000308\U00000300", {1536}, {3}},
-     {L"\U00000600\U0000093c", {1536}, {2}},
-     {L"\U00000600\U00000308\U0000093c", {1536}, {3}},
+     {L"\U00000600\U00000900", {1536}, {2}},
+     {L"\U00000600\U00000308\U00000900", {1536}, {3}},
      {L"\U00000600\U0000094d", {1536}, {2}},
      {L"\U00000600\U00000308\U0000094d", {1536}, {3}},
      {L"\U00000600\U0000200d", {1536}, {2}},
@@ -2816,8 +2614,8 @@ std::array<data<wchar_t>, 1187> data_utf32 = {{
      {L"\U00000a03\U00000308\U0000000a", {2563, 10}, {2, 3}},
      {L"\U00000a03\U00000001", {2563, 1}, {1, 2}},
      {L"\U00000a03\U00000308\U00000001", {2563, 1}, {2, 3}},
-     {L"\U00000a03\U0000034f", {2563}, {2}},
-     {L"\U00000a03\U00000308\U0000034f", {2563}, {3}},
+     {L"\U00000a03\U0000200c", {2563}, {2}},
+     {L"\U00000a03\U00000308\U0000200c", {2563}, {3}},
      {L"\U00000a03\U0001f1e6", {2563, 127462}, {1, 2}},
      {L"\U00000a03\U00000308\U0001f1e6", {2563, 127462}, {2, 3}},
      {L"\U00000a03\U00000600", {2563, 1536}, {1, 2}},
@@ -2834,8 +2632,6 @@ std::array<data<wchar_t>, 1187> data_utf32 = {{
      {L"\U00000a03\U00000308\U0000ac00", {2563, 44032}, {2, 3}},
      {L"\U00000a03\U0000ac01", {2563, 44033}, {1, 2}},
      {L"\U00000a03\U00000308\U0000ac01", {2563, 44033}, {2, 3}},
-     {L"\U00000a03\U00000900", {2563}, {2}},
-     {L"\U00000a03\U00000308\U00000900", {2563}, {3}},
      {L"\U00000a03\U00000903", {2563}, {2}},
      {L"\U00000a03\U00000308\U00000903", {2563}, {3}},
      {L"\U00000a03\U00000904", {2563, 2308}, {1, 2}},
@@ -2848,8 +2644,8 @@ std::array<data<wchar_t>, 1187> data_utf32 = {{
      {L"\U00000a03\U00000308\U0000231a", {2563, 8986}, {2, 3}},
      {L"\U00000a03\U00000300", {2563}, {2}},
      {L"\U00000a03\U00000308\U00000300", {2563}, {3}},
-     {L"\U00000a03\U0000093c", {2563}, {2}},
-     {L"\U00000a03\U00000308\U0000093c", {2563}, {3}},
+     {L"\U00000a03\U00000900", {2563}, {2}},
+     {L"\U00000a03\U00000308\U00000900", {2563}, {3}},
      {L"\U00000a03\U0000094d", {2563}, {2}},
      {L"\U00000a03\U00000308\U0000094d", {2563}, {3}},
      {L"\U00000a03\U0000200d", {2563}, {2}},
@@ -2864,8 +2660,8 @@ std::array<data<wchar_t>, 1187> data_utf32 = {{
      {L"\U00001100\U00000308\U0000000a", {4352, 10}, {2, 3}},
      {L"\U00001100\U00000001", {4352, 1}, {1, 2}},
      {L"\U00001100\U00000308\U00000001", {4352, 1}, {2, 3}},
-     {L"\U00001100\U0000034f", {4352}, {2}},
-     {L"\U00001100\U00000308\U0000034f", {4352}, {3}},
+     {L"\U00001100\U0000200c", {4352}, {2}},
+     {L"\U00001100\U00000308\U0000200c", {4352}, {3}},
      {L"\U00001100\U0001f1e6", {4352, 127462}, {1, 2}},
      {L"\U00001100\U00000308\U0001f1e6", {4352, 127462}, {2, 3}},
      {L"\U00001100\U00000600", {4352, 1536}, {1, 2}},
@@ -2882,8 +2678,6 @@ std::array<data<wchar_t>, 1187> data_utf32 = {{
      {L"\U00001100\U00000308\U0000ac00", {4352, 44032}, {2, 3}},
      {L"\U00001100\U0000ac01", {4352}, {2}},
      {L"\U00001100\U00000308\U0000ac01", {4352, 44033}, {2, 3}},
-     {L"\U00001100\U00000900", {4352}, {2}},
-     {L"\U00001100\U00000308\U00000900", {4352}, {3}},
      {L"\U00001100\U00000903", {4352}, {2}},
      {L"\U00001100\U00000308\U00000903", {4352}, {3}},
      {L"\U00001100\U00000904", {4352, 2308}, {1, 2}},
@@ -2896,8 +2690,8 @@ std::array<data<wchar_t>, 1187> data_utf32 = {{
      {L"\U00001100\U00000308\U0000231a", {4352, 8986}, {2, 3}},
      {L"\U00001100\U00000300", {4352}, {2}},
      {L"\U00001100\U00000308\U00000300", {4352}, {3}},
-     {L"\U00001100\U0000093c", {4352}, {2}},
-     {L"\U00001100\U00000308\U0000093c", {4352}, {3}},
+     {L"\U00001100\U00000900", {4352}, {2}},
+     {L"\U00001100\U00000308\U00000900", {4352}, {3}},
      {L"\U00001100\U0000094d", {4352}, {2}},
      {L"\U00001100\U00000308\U0000094d", {4352}, {3}},
      {L"\U00001100\U0000200d", {4352}, {2}},
@@ -2912,8 +2706,8 @@ std::array<data<wchar_t>, 1187> data_utf32 = {{
      {L"\U00001160\U00000308\U0000000a", {4448, 10}, {2, 3}},
      {L"\U00001160\U00000001", {4448, 1}, {1, 2}},
      {L"\U00001160\U00000308\U00000001", {4448, 1}, {2, 3}},
-     {L"\U00001160\U0000034f", {4448}, {2}},
-     {L"\U00001160\U00000308\U0000034f", {4448}, {3}},
+     {L"\U00001160\U0000200c", {4448}, {2}},
+     {L"\U00001160\U00000308\U0000200c", {4448}, {3}},
      {L"\U00001160\U0001f1e6", {4448, 127462}, {1, 2}},
      {L"\U00001160\U00000308\U0001f1e6", {4448, 127462}, {2, 3}},
      {L"\U00001160\U00000600", {4448, 1536}, {1, 2}},
@@ -2930,8 +2724,6 @@ std::array<data<wchar_t>, 1187> data_utf32 = {{
      {L"\U00001160\U00000308\U0000ac00", {4448, 44032}, {2, 3}},
      {L"\U00001160\U0000ac01", {4448, 44033}, {1, 2}},
      {L"\U00001160\U00000308\U0000ac01", {4448, 44033}, {2, 3}},
-     {L"\U00001160\U00000900", {4448}, {2}},
-     {L"\U00001160\U00000308\U00000900", {4448}, {3}},
      {L"\U00001160\U00000903", {4448}, {2}},
      {L"\U00001160\U00000308\U00000903", {4448}, {3}},
      {L"\U00001160\U00000904", {4448, 2308}, {1, 2}},
@@ -2944,8 +2736,8 @@ std::array<data<wchar_t>, 1187> data_utf32 = {{
      {L"\U00001160\U00000308\U0000231a", {4448, 8986}, {2, 3}},
      {L"\U00001160\U00000300", {4448}, {2}},
      {L"\U00001160\U00000308\U00000300", {4448}, {3}},
-     {L"\U00001160\U0000093c", {4448}, {2}},
-     {L"\U00001160\U00000308\U0000093c", {4448}, {3}},
+     {L"\U00001160\U00000900", {4448}, {2}},
+     {L"\U00001160\U00000308\U00000900", {4448}, {3}},
      {L"\U00001160\U0000094d", {4448}, {2}},
      {L"\U00001160\U00000308\U0000094d", {4448}, {3}},
      {L"\U00001160\U0000200d", {4448}, {2}},
@@ -2960,8 +2752,8 @@ std::array<data<wchar_t>, 1187> data_utf32 = {{
      {L"\U000011a8\U00000308\U0000000a", {4520, 10}, {2, 3}},
      {L"\U000011a8\U00000001", {4520, 1}, {1, 2}},
      {L"\U000011a8\U00000308\U00000001", {4520, 1}, {2, 3}},
-     {L"\U000011a8\U0000034f", {4520}, {2}},
-     {L"\U000011a8\U00000308\U0000034f", {4520}, {3}},
+     {L"\U000011a8\U0000200c", {4520}, {2}},
+     {L"\U000011a8\U00000308\U0000200c", {4520}, {3}},
      {L"\U000011a8\U0001f1e6", {4520, 127462}, {1, 2}},
      {L"\U000011a8\U00000308\U0001f1e6", {4520, 127462}, {2, 3}},
      {L"\U000011a8\U00000600", {4520, 1536}, {1, 2}},
@@ -2978,8 +2770,6 @@ std::array<data<wchar_t>, 1187> data_utf32 = {{
      {L"\U000011a8\U00000308\U0000ac00", {4520, 44032}, {2, 3}},
      {L"\U000011a8\U0000ac01", {4520, 44033}, {1, 2}},
      {L"\U000011a8\U00000308\U0000ac01", {4520, 44033}, {2, 3}},
-     {L"\U000011a8\U00000900", {4520}, {2}},
-     {L"\U000011a8\U00000308\U00000900", {4520}, {3}},
      {L"\U000011a8\U00000903", {4520}, {2}},
      {L"\U000011a8\U00000308\U00000903", {4520}, {3}},
      {L"\U000011a8\U00000904", {4520, 2308}, {1, 2}},
@@ -2992,8 +2782,8 @@ std::array<data<wchar_t>, 1187> data_utf32 = {{
      {L"\U000011a8\U00000308\U0000231a", {4520, 8986}, {2, 3}},
      {L"\U000011a8\U00000300", {4520}, {2}},
      {L"\U000011a8\U00000308\U00000300", {4520}, {3}},
-     {L"\U000011a8\U0000093c", {4520}, {2}},
-     {L"\U000011a8\U00000308\U0000093c", {4520}, {3}},
+     {L"\U000011a8\U00000900", {4520}, {2}},
+     {L"\U000011a8\U00000308\U00000900", {4520}, {3}},
      {L"\U000011a8\U0000094d", {4520}, {2}},
      {L"\U000011a8\U00000308\U0000094d", {4520}, {3}},
      {L"\U000011a8\U0000200d", {4520}, {2}},
@@ -3008,8 +2798,8 @@ std::array<data<wchar_t>, 1187> data_utf32 = {{
      {L"\U0000ac00\U00000308\U0000000a", {44032, 10}, {2, 3}},
      {L"\U0000ac00\U00000001", {44032, 1}, {1, 2}},
      {L"\U0000ac00\U00000308\U00000001", {44032, 1}, {2, 3}},
-     {L"\U0000ac00\U0000034f", {44032}, {2}},
-     {L"\U0000ac00\U00000308\U0000034f", {44032}, {3}},
+     {L"\U0000ac00\U0000200c", {44032}, {2}},
+     {L"\U0000ac00\U00000308\U0000200c", {44032}, {3}},
      {L"\U0000ac00\U0001f1e6", {44032, 127462}, {1, 2}},
      {L"\U0000ac00\U00000308\U0001f1e6", {44032, 127462}, {2, 3}},
      {L"\U0000ac00\U00000600", {44032, 1536}, {1, 2}},
@@ -3026,8 +2816,6 @@ std::array<data<wchar_t>, 1187> data_utf32 = {{
      {L"\U0000ac00\U00000308\U0000ac00", {44032, 44032}, {2, 3}},
      {L"\U0000ac00\U0000ac01", {44032, 44033}, {1, 2}},
      {L"\U0000ac00\U00000308\U0000ac01", {44032, 44033}, {2, 3}},
-     {L"\U0000ac00\U00000900", {44032}, {2}},
-     {L"\U0000ac00\U00000308\U00000900", {44032}, {3}},
      {L"\U0000ac00\U00000903", {44032}, {2}},
      {L"\U0000ac00\U00000308\U00000903", {44032}, {3}},
      {L"\U0000ac00\U00000904", {44032, 2308}, {1, 2}},
@@ -3040,8 +2828,8 @@ std::array<data<wchar_t>, 1187> data_utf32 = {{
      {L"\U0000ac00\U00000308\U0000231a", {44032, 8986}, {2, 3}},
      {L"\U0000ac00\U00000300", {44032}, {2}},
      {L"\U0000ac00\U00000308\U00000300", {44032}, {3}},
-     {L"\U0000ac00\U0000093c", {44032}, {2}},
-     {L"\U0000ac00\U00000308\U0000093c", {44032}, {3}},
+     {L"\U0000ac00\U00000900", {44032}, {2}},
+     {L"\U0000ac00\U00000308\U00000900", {44032}, {3}},
      {L"\U0000ac00\U0000094d", {44032}, {2}},
      {L"\U0000ac00\U00000308\U0000094d", {44032}, {3}},
      {L"\U0000ac00\U0000200d", {44032}, {2}},
@@ -3056,8 +2844,8 @@ std::array<data<wchar_t>, 1187> data_utf32 = {{
      {L"\U0000ac01\U00000308\U0000000a", {44033, 10}, {2, 3}},
      {L"\U0000ac01\U00000001", {44033, 1}, {1, 2}},
      {L"\U0000ac01\U00000308\U00000001", {44033, 1}, {2, 3}},
-     {L"\U0000ac01\U0000034f", {44033}, {2}},
-     {L"\U0000ac01\U00000308\U0000034f", {44033}, {3}},
+     {L"\U0000ac01\U0000200c", {44033}, {2}},
+     {L"\U0000ac01\U00000308\U0000200c", {44033}, {3}},
      {L"\U0000ac01\U0001f1e6", {44033, 127462}, {1, 2}},
      {L"\U0000ac01\U00000308\U0001f1e6", {44033, 127462}, {2, 3}},
      {L"\U0000ac01\U00000600", {44033, 1536}, {1, 2}},
@@ -3074,8 +2862,6 @@ std::array<data<wchar_t>, 1187> data_utf32 = {{
      {L"\U0000ac01\U00000308\U0000ac00", {44033, 44032}, {2, 3}},
      {L"\U0000ac01\U0000ac01", {44033, 44033}, {1, 2}},
      {L"\U0000ac01\U00000308\U0000ac01", {44033, 44033}, {2, 3}},
-     {L"\U0000ac01\U00000900", {44033}, {2}},
-     {L"\U0000ac01\U00000308\U00000900", {44033}, {3}},
      {L"\U0000ac01\U00000903", {44033}, {2}},
      {L"\U0000ac01\U00000308\U00000903", {44033}, {3}},
      {L"\U0000ac01\U00000904", {44033, 2308}, {1, 2}},
@@ -3088,62 +2874,14 @@ std::array<data<wchar_t>, 1187> data_utf32 = {{
      {L"\U0000ac01\U00000308\U0000231a", {44033, 8986}, {2, 3}},
      {L"\U0000ac01\U00000300", {44033}, {2}},
      {L"\U0000ac01\U00000308\U00000300", {44033}, {3}},
-     {L"\U0000ac01\U0000093c", {44033}, {2}},
-     {L"\U0000ac01\U00000308\U0000093c", {44033}, {3}},
+     {L"\U0000ac01\U00000900", {44033}, {2}},
+     {L"\U0000ac01\U00000308\U00000900", {44033}, {3}},
      {L"\U0000ac01\U0000094d", {44033}, {2}},
      {L"\U0000ac01\U00000308\U0000094d", {44033}, {3}},
      {L"\U0000ac01\U0000200d", {44033}, {2}},
      {L"\U0000ac01\U00000308\U0000200d", {44033}, {3}},
      {L"\U0000ac01\U00000378", {44033, 888}, {1, 2}},
      {L"\U0000ac01\U00000308\U00000378", {44033, 888}, {2, 3}},
-     {L"\U00000900\U00000020", {2304, 32}, {1, 2}},
-     {L"\U00000900\U00000308\U00000020", {2304, 32}, {2, 3}},
-     {L"\U00000900\U0000000d", {2304, 13}, {1, 2}},
-     {L"\U00000900\U00000308\U0000000d", {2304, 13}, {2, 3}},
-     {L"\U00000900\U0000000a", {2304, 10}, {1, 2}},
-     {L"\U00000900\U00000308\U0000000a", {2304, 10}, {2, 3}},
-     {L"\U00000900\U00000001", {2304, 1}, {1, 2}},
-     {L"\U00000900\U00000308\U00000001", {2304, 1}, {2, 3}},
-     {L"\U00000900\U0000034f", {2304}, {2}},
-     {L"\U00000900\U00000308\U0000034f", {2304}, {3}},
-     {L"\U00000900\U0001f1e6", {2304, 127462}, {1, 2}},
-     {L"\U00000900\U00000308\U0001f1e6", {2304, 127462}, {2, 3}},
-     {L"\U00000900\U00000600", {2304, 1536}, {1, 2}},
-     {L"\U00000900\U00000308\U00000600", {2304, 1536}, {2, 3}},
-     {L"\U00000900\U00000a03", {2304}, {2}},
-     {L"\U00000900\U00000308\U00000a03", {2304}, {3}},
-     {L"\U00000900\U00001100", {2304, 4352}, {1, 2}},
-     {L"\U00000900\U00000308\U00001100", {2304, 4352}, {2, 3}},
-     {L"\U00000900\U00001160", {2304, 4448}, {1, 2}},
-     {L"\U00000900\U00000308\U00001160", {2304, 4448}, {2, 3}},
-     {L"\U00000900\U000011a8", {2304, 4520}, {1, 2}},
-     {L"\U00000900\U00000308\U000011a8", {2304, 4520}, {2, 3}},
-     {L"\U00000900\U0000ac00", {2304, 44032}, {1, 2}},
-     {L"\U00000900\U00000308\U0000ac00", {2304, 44032}, {2, 3}},
-     {L"\U00000900\U0000ac01", {2304, 44033}, {1, 2}},
-     {L"\U00000900\U00000308\U0000ac01", {2304, 44033}, {2, 3}},
-     {L"\U00000900\U00000900", {2304}, {2}},
-     {L"\U00000900\U00000308\U00000900", {2304}, {3}},
-     {L"\U00000900\U00000903", {2304}, {2}},
-     {L"\U00000900\U00000308\U00000903", {2304}, {3}},
-     {L"\U00000900\U00000904", {2304, 2308}, {1, 2}},
-     {L"\U00000900\U00000308\U00000904", {2304, 2308}, {2, 3}},
-     {L"\U00000900\U00000d4e", {2304, 3406}, {1, 2}},
-     {L"\U00000900\U00000308\U00000d4e", {2304, 3406}, {2, 3}},
-     {L"\U00000900\U00000915", {2304, 2325}, {1, 2}},
-     {L"\U00000900\U00000308\U00000915", {2304, 2325}, {2, 3}},
-     {L"\U00000900\U0000231a", {2304, 8986}, {1, 2}},
-     {L"\U00000900\U00000308\U0000231a", {2304, 8986}, {2, 3}},
-     {L"\U00000900\U00000300", {2304}, {2}},
-     {L"\U00000900\U00000308\U00000300", {2304}, {3}},
-     {L"\U00000900\U0000093c", {2304}, {2}},
-     {L"\U00000900\U00000308\U0000093c", {2304}, {3}},
-     {L"\U00000900\U0000094d", {2304}, {2}},
-     {L"\U00000900\U00000308\U0000094d", {2304}, {3}},
-     {L"\U00000900\U0000200d", {2304}, {2}},
-     {L"\U00000900\U00000308\U0000200d", {2304}, {3}},
-     {L"\U00000900\U00000378", {2304, 888}, {1, 2}},
-     {L"\U00000900\U00000308\U00000378", {2304, 888}, {2, 3}},
      {L"\U00000903\U00000020", {2307, 32}, {1, 2}},
      {L"\U00000903\U00000308\U00000020", {2307, 32}, {2, 3}},
      {L"\U00000903\U0000000d", {2307, 13}, {1, 2}},
@@ -3152,8 +2890,8 @@ std::array<data<wchar_t>, 1187> data_utf32 = {{
      {L"\U00000903\U00000308\U0000000a", {2307, 10}, {2, 3}},
      {L"\U00000903\U00000001", {2307, 1}, {1, 2}},
      {L"\U00000903\U00000308\U00000001", {2307, 1}, {2, 3}},
-     {L"\U00000903\U0000034f", {2307}, {2}},
-     {L"\U00000903\U00000308\U0000034f", {2307}, {3}},
+     {L"\U00000903\U0000200c", {2307}, {2}},
+     {L"\U00000903\U00000308\U0000200c", {2307}, {3}},
      {L"\U00000903\U0001f1e6", {2307, 127462}, {1, 2}},
      {L"\U00000903\U00000308\U0001f1e6", {2307, 127462}, {2, 3}},
      {L"\U00000903\U00000600", {2307, 1536}, {1, 2}},
@@ -3170,8 +2908,6 @@ std::array<data<wchar_t>, 1187> data_utf32 = {{
      {L"\U00000903\U00000308\U0000ac00", {2307, 44032}, {2, 3}},
      {L"\U00000903\U0000ac01", {2307, 44033}, {1, 2}},
      {L"\U00000903\U00000308\U0000ac01", {2307, 44033}, {2, 3}},
-     {L"\U00000903\U00000900", {2307}, {2}},
-     {L"\U00000903\U00000308\U00000900", {2307}, {3}},
      {L"\U00000903\U00000903", {2307}, {2}},
      {L"\U00000903\U00000308\U00000903", {2307}, {3}},
      {L"\U00000903\U00000904", {2307, 2308}, {1, 2}},
@@ -3184,8 +2920,8 @@ std::array<data<wchar_t>, 1187> data_utf32 = {{
      {L"\U00000903\U00000308\U0000231a", {2307, 8986}, {2, 3}},
      {L"\U00000903\U00000300", {2307}, {2}},
      {L"\U00000903\U00000308\U00000300", {2307}, {3}},
-     {L"\U00000903\U0000093c", {2307}, {2}},
-     {L"\U00000903\U00000308\U0000093c", {2307}, {3}},
+     {L"\U00000903\U00000900", {2307}, {2}},
+     {L"\U00000903\U00000308\U00000900", {2307}, {3}},
      {L"\U00000903\U0000094d", {2307}, {2}},
      {L"\U00000903\U00000308\U0000094d", {2307}, {3}},
      {L"\U00000903\U0000200d", {2307}, {2}},
@@ -3200,8 +2936,8 @@ std::array<data<wchar_t>, 1187> data_utf32 = {{
      {L"\U00000904\U00000308\U0000000a", {2308, 10}, {2, 3}},
      {L"\U00000904\U00000001", {2308, 1}, {1, 2}},
      {L"\U00000904\U00000308\U00000001", {2308, 1}, {2, 3}},
-     {L"\U00000904\U0000034f", {2308}, {2}},
-     {L"\U00000904\U00000308\U0000034f", {2308}, {3}},
+     {L"\U00000904\U0000200c", {2308}, {2}},
+     {L"\U00000904\U00000308\U0000200c", {2308}, {3}},
      {L"\U00000904\U0001f1e6", {2308, 127462}, {1, 2}},
      {L"\U00000904\U00000308\U0001f1e6", {2308, 127462}, {2, 3}},
      {L"\U00000904\U00000600", {2308, 1536}, {1, 2}},
@@ -3218,8 +2954,6 @@ std::array<data<wchar_t>, 1187> data_utf32 = {{
      {L"\U00000904\U00000308\U0000ac00", {2308, 44032}, {2, 3}},
      {L"\U00000904\U0000ac01", {2308, 44033}, {1, 2}},
      {L"\U00000904\U00000308\U0000ac01", {2308, 44033}, {2, 3}},
-     {L"\U00000904\U00000900", {2308}, {2}},
-     {L"\U00000904\U00000308\U00000900", {2308}, {3}},
      {L"\U00000904\U00000903", {2308}, {2}},
      {L"\U00000904\U00000308\U00000903", {2308}, {3}},
      {L"\U00000904\U00000904", {2308, 2308}, {1, 2}},
@@ -3232,8 +2966,8 @@ std::array<data<wchar_t>, 1187> data_utf32 = {{
      {L"\U00000904\U00000308\U0000231a", {2308, 8986}, {2, 3}},
      {L"\U00000904\U00000300", {2308}, {2}},
      {L"\U00000904\U00000308\U00000300", {2308}, {3}},
-     {L"\U00000904\U0000093c", {2308}, {2}},
-     {L"\U00000904\U00000308\U0000093c", {2308}, {3}},
+     {L"\U00000904\U00000900", {2308}, {2}},
+     {L"\U00000904\U00000308\U00000900", {2308}, {3}},
      {L"\U00000904\U0000094d", {2308}, {2}},
      {L"\U00000904\U00000308\U0000094d", {2308}, {3}},
      {L"\U00000904\U0000200d", {2308}, {2}},
@@ -3248,8 +2982,8 @@ std::array<data<wchar_t>, 1187> data_utf32 = {{
      {L"\U00000d4e\U00000308\U0000000a", {3406, 10}, {2, 3}},
      {L"\U00000d4e\U00000001", {3406, 1}, {1, 2}},
      {L"\U00000d4e\U00000308\U00000001", {3406, 1}, {2, 3}},
-     {L"\U00000d4e\U0000034f", {3406}, {2}},
-     {L"\U00000d4e\U00000308\U0000034f", {3406}, {3}},
+     {L"\U00000d4e\U0000200c", {3406}, {2}},
+     {L"\U00000d4e\U00000308\U0000200c", {3406}, {3}},
      {L"\U00000d4e\U0001f1e6", {3406}, {2}},
      {L"\U00000d4e\U00000308\U0001f1e6", {3406, 127462}, {2, 3}},
      {L"\U00000d4e\U00000600", {3406}, {2}},
@@ -3266,8 +3000,6 @@ std::array<data<wchar_t>, 1187> data_utf32 = {{
      {L"\U00000d4e\U00000308\U0000ac00", {3406, 44032}, {2, 3}},
      {L"\U00000d4e\U0000ac01", {3406}, {2}},
      {L"\U00000d4e\U00000308\U0000ac01", {3406, 44033}, {2, 3}},
-     {L"\U00000d4e\U00000900", {3406}, {2}},
-     {L"\U00000d4e\U00000308\U00000900", {3406}, {3}},
      {L"\U00000d4e\U00000903", {3406}, {2}},
      {L"\U00000d4e\U00000308\U00000903", {3406}, {3}},
      {L"\U00000d4e\U00000904", {3406}, {2}},
@@ -3280,8 +3012,8 @@ std::array<data<wchar_t>, 1187> data_utf32 = {{
      {L"\U00000d4e\U00000308\U0000231a", {3406, 8986}, {2, 3}},
      {L"\U00000d4e\U00000300", {3406}, {2}},
      {L"\U00000d4e\U00000308\U00000300", {3406}, {3}},
-     {L"\U00000d4e\U0000093c", {3406}, {2}},
-     {L"\U00000d4e\U00000308\U0000093c", {3406}, {3}},
+     {L"\U00000d4e\U00000900", {3406}, {2}},
+     {L"\U00000d4e\U00000308\U00000900", {3406}, {3}},
      {L"\U00000d4e\U0000094d", {3406}, {2}},
      {L"\U00000d4e\U00000308\U0000094d", {3406}, {3}},
      {L"\U00000d4e\U0000200d", {3406}, {2}},
@@ -3296,8 +3028,8 @@ std::array<data<wchar_t>, 1187> data_utf32 = {{
      {L"\U00000915\U00000308\U0000000a", {2325, 10}, {2, 3}},
      {L"\U00000915\U00000001", {2325, 1}, {1, 2}},
      {L"\U00000915\U00000308\U00000001", {2325, 1}, {2, 3}},
-     {L"\U00000915\U0000034f", {2325}, {2}},
-     {L"\U00000915\U00000308\U0000034f", {2325}, {3}},
+     {L"\U00000915\U0000200c", {2325}, {2}},
+     {L"\U00000915\U00000308\U0000200c", {2325}, {3}},
      {L"\U00000915\U0001f1e6", {2325, 127462}, {1, 2}},
      {L"\U00000915\U00000308\U0001f1e6", {2325, 127462}, {2, 3}},
      {L"\U00000915\U00000600", {2325, 1536}, {1, 2}},
@@ -3314,8 +3046,6 @@ std::array<data<wchar_t>, 1187> data_utf32 = {{
      {L"\U00000915\U00000308\U0000ac00", {2325, 44032}, {2, 3}},
      {L"\U00000915\U0000ac01", {2325, 44033}, {1, 2}},
      {L"\U00000915\U00000308\U0000ac01", {2325, 44033}, {2, 3}},
-     {L"\U00000915\U00000900", {2325}, {2}},
-     {L"\U00000915\U00000308\U00000900", {2325}, {3}},
      {L"\U00000915\U00000903", {2325}, {2}},
      {L"\U00000915\U00000308\U00000903", {2325}, {3}},
      {L"\U00000915\U00000904", {2325, 2308}, {1, 2}},
@@ -3328,8 +3058,8 @@ std::array<data<wchar_t>, 1187> data_utf32 = {{
      {L"\U00000915\U00000308\U0000231a", {2325, 8986}, {2, 3}},
      {L"\U00000915\U00000300", {2325}, {2}},
      {L"\U00000915\U00000308\U00000300", {2325}, {3}},
-     {L"\U00000915\U0000093c", {2325}, {2}},
-     {L"\U00000915\U00000308\U0000093c", {2325}, {3}},
+     {L"\U00000915\U00000900", {2325}, {2}},
+     {L"\U00000915\U00000308\U00000900", {2325}, {3}},
      {L"\U00000915\U0000094d", {2325}, {2}},
      {L"\U00000915\U00000308\U0000094d", {2325}, {3}},
      {L"\U00000915\U0000200d", {2325}, {2}},
@@ -3344,8 +3074,8 @@ std::array<data<wchar_t>, 1187> data_utf32 = {{
      {L"\U0000231a\U00000308\U0000000a", {8986, 10}, {2, 3}},
      {L"\U0000231a\U00000001", {8986, 1}, {1, 2}},
      {L"\U0000231a\U00000308\U00000001", {8986, 1}, {2, 3}},
-     {L"\U0000231a\U0000034f", {8986}, {2}},
-     {L"\U0000231a\U00000308\U0000034f", {8986}, {3}},
+     {L"\U0000231a\U0000200c", {8986}, {2}},
+     {L"\U0000231a\U00000308\U0000200c", {8986}, {3}},
      {L"\U0000231a\U0001f1e6", {8986, 127462}, {1, 2}},
      {L"\U0000231a\U00000308\U0001f1e6", {8986, 127462}, {2, 3}},
      {L"\U0000231a\U00000600", {8986, 1536}, {1, 2}},
@@ -3362,8 +3092,6 @@ std::array<data<wchar_t>, 1187> data_utf32 = {{
      {L"\U0000231a\U00000308\U0000ac00", {8986, 44032}, {2, 3}},
      {L"\U0000231a\U0000ac01", {8986, 44033}, {1, 2}},
      {L"\U0000231a\U00000308\U0000ac01", {8986, 44033}, {2, 3}},
-     {L"\U0000231a\U00000900", {8986}, {2}},
-     {L"\U0000231a\U00000308\U00000900", {8986}, {3}},
      {L"\U0000231a\U00000903", {8986}, {2}},
      {L"\U0000231a\U00000308\U00000903", {8986}, {3}},
      {L"\U0000231a\U00000904", {8986, 2308}, {1, 2}},
@@ -3376,8 +3104,8 @@ std::array<data<wchar_t>, 1187> data_utf32 = {{
      {L"\U0000231a\U00000308\U0000231a", {8986, 8986}, {2, 3}},
      {L"\U0000231a\U00000300", {8986}, {2}},
      {L"\U0000231a\U00000308\U00000300", {8986}, {3}},
-     {L"\U0000231a\U0000093c", {8986}, {2}},
-     {L"\U0000231a\U00000308\U0000093c", {8986}, {3}},
+     {L"\U0000231a\U00000900", {8986}, {2}},
+     {L"\U0000231a\U00000308\U00000900", {8986}, {3}},
      {L"\U0000231a\U0000094d", {8986}, {2}},
      {L"\U0000231a\U00000308\U0000094d", {8986}, {3}},
      {L"\U0000231a\U0000200d", {8986}, {2}},
@@ -3392,8 +3120,8 @@ std::array<data<wchar_t>, 1187> data_utf32 = {{
      {L"\U00000300\U00000308\U0000000a", {768, 10}, {2, 3}},
      {L"\U00000300\U00000001", {768, 1}, {1, 2}},
      {L"\U00000300\U00000308\U00000001", {768, 1}, {2, 3}},
-     {L"\U00000300\U0000034f", {768}, {2}},
-     {L"\U00000300\U00000308\U0000034f", {768}, {3}},
+     {L"\U00000300\U0000200c", {768}, {2}},
+     {L"\U00000300\U00000308\U0000200c", {768}, {3}},
      {L"\U00000300\U0001f1e6", {768, 127462}, {1, 2}},
      {L"\U00000300\U00000308\U0001f1e6", {768, 127462}, {2, 3}},
      {L"\U00000300\U00000600", {768, 1536}, {1, 2}},
@@ -3410,8 +3138,6 @@ std::array<data<wchar_t>, 1187> data_utf32 = {{
      {L"\U00000300\U00000308\U0000ac00", {768, 44032}, {2, 3}},
      {L"\U00000300\U0000ac01", {768, 44033}, {1, 2}},
      {L"\U00000300\U00000308\U0000ac01", {768, 44033}, {2, 3}},
-     {L"\U00000300\U00000900", {768}, {2}},
-     {L"\U00000300\U00000308\U00000900", {768}, {3}},
      {L"\U00000300\U00000903", {768}, {2}},
      {L"\U00000300\U00000308\U00000903", {768}, {3}},
      {L"\U00000300\U00000904", {768, 2308}, {1, 2}},
@@ -3424,62 +3150,60 @@ std::array<data<wchar_t>, 1187> data_utf32 = {{
      {L"\U00000300\U00000308\U0000231a", {768, 8986}, {2, 3}},
      {L"\U00000300\U00000300", {768}, {2}},
      {L"\U00000300\U00000308\U00000300", {768}, {3}},
-     {L"\U00000300\U0000093c", {768}, {2}},
-     {L"\U00000300\U00000308\U0000093c", {768}, {3}},
+     {L"\U00000300\U00000900", {768}, {2}},
+     {L"\U00000300\U00000308\U00000900", {768}, {3}},
      {L"\U00000300\U0000094d", {768}, {2}},
      {L"\U00000300\U00000308\U0000094d", {768}, {3}},
      {L"\U00000300\U0000200d", {768}, {2}},
      {L"\U00000300\U00000308\U0000200d", {768}, {3}},
      {L"\U00000300\U00000378", {768, 888}, {1, 2}},
      {L"\U00000300\U00000308\U00000378", {768, 888}, {2, 3}},
-     {L"\U0000093c\U00000020", {2364, 32}, {1, 2}},
-     {L"\U0000093c\U00000308\U00000020", {2364, 32}, {2, 3}},
-     {L"\U0000093c\U0000000d", {2364, 13}, {1, 2}},
-     {L"\U0000093c\U00000308\U0000000d", {2364, 13}, {2, 3}},
-     {L"\U0000093c\U0000000a", {2364, 10}, {1, 2}},
-     {L"\U0000093c\U00000308\U0000000a", {2364, 10}, {2, 3}},
-     {L"\U0000093c\U00000001", {2364, 1}, {1, 2}},
-     {L"\U0000093c\U00000308\U00000001", {2364, 1}, {2, 3}},
-     {L"\U0000093c\U0000034f", {2364}, {2}},
-     {L"\U0000093c\U00000308\U0000034f", {2364}, {3}},
-     {L"\U0000093c\U0001f1e6", {2364, 127462}, {1, 2}},
-     {L"\U0000093c\U00000308\U0001f1e6", {2364, 127462}, {2, 3}},
-     {L"\U0000093c\U00000600", {2364, 1536}, {1, 2}},
-     {L"\U0000093c\U00000308\U00000600", {2364, 1536}, {2, 3}},
-     {L"\U0000093c\U00000a03", {2364}, {2}},
-     {L"\U0000093c\U00000308\U00000a03", {2364}, {3}},
-     {L"\U0000093c\U00001100", {2364, 4352}, {1, 2}},
-     {L"\U0000093c\U00000308\U00001100", {2364, 4352}, {2, 3}},
-     {L"\U0000093c\U00001160", {2364, 4448}, {1, 2}},
-     {L"\U0000093c\U00000308\U00001160", {2364, 4448}, {2, 3}},
-     {L"\U0000093c\U000011a8", {2364, 4520}, {1, 2}},
-     {L"\U0000093c\U00000308\U000011a8", {2364, 4520}, {2, 3}},
-     {L"\U0000093c\U0000ac00", {2364, 44032}, {1, 2}},
-     {L"\U0000093c\U00000308\U0000ac00", {2364, 44032}, {2, 3}},
-     {L"\U0000093c\U0000ac01", {2364, 44033}, {1, 2}},
-     {L"\U0000093c\U00000308\U0000ac01", {2364, 44033}, {2, 3}},
-     {L"\U0000093c\U00000900", {2364}, {2}},
-     {L"\U0000093c\U00000308\U00000900", {2364}, {3}},
-     {L"\U0000093c\U00000903", {2364}, {2}},
-     {L"\U0000093c\U00000308\U00000903", {2364}, {3}},
-     {L"\U0000093c\U00000904", {2364, 2308}, {1, 2}},
-     {L"\U0000093c\U00000308\U00000904", {2364, 2308}, {2, 3}},
-     {L"\U0000093c\U00000d4e", {2364, 3406}, {1, 2}},
-     {L"\U0000093c\U00000308\U00000d4e", {2364, 3406}, {2, 3}},
-     {L"\U0000093c\U00000915", {2364, 2325}, {1, 2}},
-     {L"\U0000093c\U00000308\U00000915", {2364, 2325}, {2, 3}},
-     {L"\U0000093c\U0000231a", {2364, 8986}, {1, 2}},
-     {L"\U0000093c\U00000308\U0000231a", {2364, 8986}, {2, 3}},
-     {L"\U0000093c\U00000300", {2364}, {2}},
-     {L"\U0000093c\U00000308\U00000300", {2364}, {3}},
-     {L"\U0000093c\U0000093c", {2364}, {2}},
-     {L"\U0000093c\U00000308\U0000093c", {2364}, {3}},
-     {L"\U0000093c\U0000094d", {2364}, {2}},
-     {L"\U0000093c\U00000308\U0000094d", {2364}, {3}},
-     {L"\U0000093c\U0000200d", {2364}, {2}},
-     {L"\U0000093c\U00000308\U0000200d", {2364}, {3}},
-     {L"\U0000093c\U00000378", {2364, 888}, {1, 2}},
-     {L"\U0000093c\U00000308\U00000378", {2364, 888}, {2, 3}},
+     {L"\U00000900\U00000020", {2304, 32}, {1, 2}},
+     {L"\U00000900\U00000308\U00000020", {2304, 32}, {2, 3}},
+     {L"\U00000900\U0000000d", {2304, 13}, {1, 2}},
+     {L"\U00000900\U00000308\U0000000d", {2304, 13}, {2, 3}},
+     {L"\U00000900\U0000000a", {2304, 10}, {1, 2}},
+     {L"\U00000900\U00000308\U0000000a", {2304, 10}, {2, 3}},
+     {L"\U00000900\U00000001", {2304, 1}, {1, 2}},
+     {L"\U00000900\U00000308\U00000001", {2304, 1}, {2, 3}},
+     {L"\U00000900\U0000200c", {2304}, {2}},
+     {L"\U00000900\U00000308\U0000200c", {2304}, {3}},
+     {L"\U00000900\U0001f1e6", {2304, 127462}, {1, 2}},
+     {L"\U00000900\U00000308\U0001f1e6", {2304, 127462}, {2, 3}},
+     {L"\U00000900\U00000600", {2304, 1536}, {1, 2}},
+     {L"\U00000900\U00000308\U00000600", {2304, 1536}, {2, 3}},
+     {L"\U00000900\U00000a03", {2304}, {2}},
+     {L"\U00000900\U00000308\U00000a03", {2304}, {3}},
+     {L"\U00000900\U00001100", {2304, 4352}, {1, 2}},
+     {L"\U00000900\U00000308\U00001100", {2304, 4352}, {2, 3}},
+     {L"\U00000900\U00001160", {2304, 4448}, {1, 2}},
+     {L"\U00000900\U00000308\U00001160", {2304, 4448}, {2, 3}},
+     {L"\U00000900\U000011a8", {2304, 4520}, {1, 2}},
+     {L"\U00000900\U00000308\U000011a8", {2304, 4520}, {2, 3}},
+     {L"\U00000900\U0000ac00", {2304, 44032}, {1, 2}},
+     {L"\U00000900\U00000308\U0000ac00", {2304, 44032}, {2, 3}},
+     {L"\U00000900\U0000ac01", {2304, 44033}, {1, 2}},
+     {L"\U00000900\U00000308\U0000ac01", {2304, 44033}, {2, 3}},
+     {L"\U00000900\U00000903", {2304}, {2}},
+     {L"\U00000900\U00000308\U00000903", {2304}, {3}},
+     {L"\U00000900\U00000904", {2304, 2308}, {1, 2}},
+     {L"\U00000900\U00000308\U00000904", {2304, 2308}, {2, 3}},
+     {L"\U00000900\U00000d4e", {2304, 3406}, {1, 2}},
+     {L"\U00000900\U00000308\U00000d4e", {2304, 3406}, {2, 3}},
+     {L"\U00000900\U00000915", {2304, 2325}, {1, 2}},
+     {L"\U00000900\U00000308\U00000915", {2304, 2325}, {2, 3}},
+     {L"\U00000900\U0000231a", {2304, 8986}, {1, 2}},
+     {L"\U00000900\U00000308\U0000231a", {2304, 8986}, {2, 3}},
+     {L"\U00000900\U00000300", {2304}, {2}},
+     {L"\U00000900\U00000308\U00000300", {2304}, {3}},
+     {L"\U00000900\U00000900", {2304}, {2}},
+     {L"\U00000900\U00000308\U00000900", {2304}, {3}},
+     {L"\U00000900\U0000094d", {2304}, {2}},
+     {L"\U00000900\U00000308\U0000094d", {2304}, {3}},
+     {L"\U00000900\U0000200d", {2304}, {2}},
+     {L"\U00000900\U00000308\U0000200d", {2304}, {3}},
+     {L"\U00000900\U00000378", {2304, 888}, {1, 2}},
+     {L"\U00000900\U00000308\U00000378", {2304, 888}, {2, 3}},
      {L"\U0000094d\U00000020", {2381, 32}, {1, 2}},
      {L"\U0000094d\U00000308\U00000020", {2381, 32}, {2, 3}},
      {L"\U0000094d\U0000000d", {2381, 13}, {1, 2}},
@@ -3488,8 +3212,8 @@ std::array<data<wchar_t>, 1187> data_utf32 = {{
      {L"\U0000094d\U00000308\U0000000a", {2381, 10}, {2, 3}},
      {L"\U0000094d\U00000001", {2381, 1}, {1, 2}},
      {L"\U0000094d\U00000308\U00000001", {2381, 1}, {2, 3}},
-     {L"\U0000094d\U0000034f", {2381}, {2}},
-     {L"\U0000094d\U00000308\U0000034f", {2381}, {3}},
+     {L"\U0000094d\U0000200c", {2381}, {2}},
+     {L"\U0000094d\U00000308\U0000200c", {2381}, {3}},
      {L"\U0000094d\U0001f1e6", {2381, 127462}, {1, 2}},
      {L"\U0000094d\U00000308\U0001f1e6", {2381, 127462}, {2, 3}},
      {L"\U0000094d\U00000600", {2381, 1536}, {1, 2}},
@@ -3506,8 +3230,6 @@ std::array<data<wchar_t>, 1187> data_utf32 = {{
      {L"\U0000094d\U00000308\U0000ac00", {2381, 44032}, {2, 3}},
      {L"\U0000094d\U0000ac01", {2381, 44033}, {1, 2}},
      {L"\U0000094d\U00000308\U0000ac01", {2381, 44033}, {2, 3}},
-     {L"\U0000094d\U00000900", {2381}, {2}},
-     {L"\U0000094d\U00000308\U00000900", {2381}, {3}},
      {L"\U0000094d\U00000903", {2381}, {2}},
      {L"\U0000094d\U00000308\U00000903", {2381}, {3}},
      {L"\U0000094d\U00000904", {2381, 2308}, {1, 2}},
@@ -3520,8 +3242,8 @@ std::array<data<wchar_t>, 1187> data_utf32 = {{
      {L"\U0000094d\U00000308\U0000231a", {2381, 8986}, {2, 3}},
      {L"\U0000094d\U00000300", {2381}, {2}},
      {L"\U0000094d\U00000308\U00000300", {2381}, {3}},
-     {L"\U0000094d\U0000093c", {2381}, {2}},
-     {L"\U0000094d\U00000308\U0000093c", {2381}, {3}},
+     {L"\U0000094d\U00000900", {2381}, {2}},
+     {L"\U0000094d\U00000308\U00000900", {2381}, {3}},
      {L"\U0000094d\U0000094d", {2381}, {2}},
      {L"\U0000094d\U00000308\U0000094d", {2381}, {3}},
      {L"\U0000094d\U0000200d", {2381}, {2}},
@@ -3536,8 +3258,8 @@ std::array<data<wchar_t>, 1187> data_utf32 = {{
      {L"\U0000200d\U00000308\U0000000a", {8205, 10}, {2, 3}},
      {L"\U0000200d\U00000001", {8205, 1}, {1, 2}},
      {L"\U0000200d\U00000308\U00000001", {8205, 1}, {2, 3}},
-     {L"\U0000200d\U0000034f", {8205}, {2}},
-     {L"\U0000200d\U00000308\U0000034f", {8205}, {3}},
+     {L"\U0000200d\U0000200c", {8205}, {2}},
+     {L"\U0000200d\U00000308\U0000200c", {8205}, {3}},
      {L"\U0000200d\U0001f1e6", {8205, 127462}, {1, 2}},
      {L"\U0000200d\U00000308\U0001f1e6", {8205, 127462}, {2, 3}},
      {L"\U0000200d\U00000600", {8205, 1536}, {1, 2}},
@@ -3554,8 +3276,6 @@ std::array<data<wchar_t>, 1187> data_utf32 = {{
      {L"\U0000200d\U00000308\U0000ac00", {8205, 44032}, {2, 3}},
      {L"\U0000200d\U0000ac01", {8205, 44033}, {1, 2}},
      {L"\U0000200d\U00000308\U0000ac01", {8205, 44033}, {2, 3}},
-     {L"\U0000200d\U00000900", {8205}, {2}},
-     {L"\U0000200d\U00000308\U00000900", {8205}, {3}},
      {L"\U0000200d\U00000903", {8205}, {2}},
      {L"\U0000200d\U00000308\U00000903", {8205}, {3}},
      {L"\U0000200d\U00000904", {8205, 2308}, {1, 2}},
@@ -3568,8 +3288,8 @@ std::array<data<wchar_t>, 1187> data_utf32 = {{
      {L"\U0000200d\U00000308\U0000231a", {8205, 8986}, {2, 3}},
      {L"\U0000200d\U00000300", {8205}, {2}},
      {L"\U0000200d\U00000308\U00000300", {8205}, {3}},
-     {L"\U0000200d\U0000093c", {8205}, {2}},
-     {L"\U0000200d\U00000308\U0000093c", {8205}, {3}},
+     {L"\U0000200d\U00000900", {8205}, {2}},
+     {L"\U0000200d\U00000308\U00000900", {8205}, {3}},
      {L"\U0000200d\U0000094d", {8205}, {2}},
      {L"\U0000200d\U00000308\U0000094d", {8205}, {3}},
      {L"\U0000200d\U0000200d", {8205}, {2}},
@@ -3584,8 +3304,8 @@ std::array<data<wchar_t>, 1187> data_utf32 = {{
      {L"\U00000378\U00000308\U0000000a", {888, 10}, {2, 3}},
      {L"\U00000378\U00000001", {888, 1}, {1, 2}},
      {L"\U00000378\U00000308\U00000001", {888, 1}, {2, 3}},
-     {L"\U00000378\U0000034f", {888}, {2}},
-     {L"\U00000378\U00000308\U0000034f", {888}, {3}},
+     {L"\U00000378\U0000200c", {888}, {2}},
+     {L"\U00000378\U00000308\U0000200c", {888}, {3}},
      {L"\U00000378\U0001f1e6", {888, 127462}, {1, 2}},
      {L"\U00000378\U00000308\U0001f1e6", {888, 127462}, {2, 3}},
      {L"\U00000378\U00000600", {888, 1536}, {1, 2}},
@@ -3602,8 +3322,6 @@ std::array<data<wchar_t>, 1187> data_utf32 = {{
      {L"\U00000378\U00000308\U0000ac00", {888, 44032}, {2, 3}},
      {L"\U00000378\U0000ac01", {888, 44033}, {1, 2}},
      {L"\U00000378\U00000308\U0000ac01", {888, 44033}, {2, 3}},
-     {L"\U00000378\U00000900", {888}, {2}},
-     {L"\U00000378\U00000308\U00000900", {888}, {3}},
      {L"\U00000378\U00000903", {888}, {2}},
      {L"\U00000378\U00000308\U00000903", {888}, {3}},
      {L"\U00000378\U00000904", {888, 2308}, {1, 2}},
@@ -3616,8 +3334,8 @@ std::array<data<wchar_t>, 1187> data_utf32 = {{
      {L"\U00000378\U00000308\U0000231a", {888, 8986}, {2, 3}},
      {L"\U00000378\U00000300", {888}, {2}},
      {L"\U00000378\U00000308\U00000300", {888}, {3}},
-     {L"\U00000378\U0000093c", {888}, {2}},
-     {L"\U00000378\U00000308\U0000093c", {888}, {3}},
+     {L"\U00000378\U00000900", {888}, {2}},
+     {L"\U00000378\U00000308\U00000900", {888}, {3}},
      {L"\U00000378\U0000094d", {888}, {2}},
      {L"\U00000378\U00000308\U0000094d", {888}, {3}},
      {L"\U00000378\U0000200d", {888}, {2}},
diff --git a/libcxx/test/libcxx/utilities/format/format.string/format.string.std/extended_grapheme_cluster.pass.cpp b/libcxx/test/libcxx/utilities/format/format.string/format.string.std/extended_grapheme_cluster.pass.cpp
index dd1f4b607ddfc..90f7cb2c6ee09 100644
--- a/libcxx/test/libcxx/utilities/format/format.string/format.string.std/extended_grapheme_cluster.pass.cpp
+++ b/libcxx/test/libcxx/utilities/format/format.string/format.string.std/extended_grapheme_cluster.pass.cpp
@@ -40,15 +40,15 @@ constexpr int count_entries(cluster::__property property) {
       });
 }
 
-static_assert(count_entries(cluster::__property::__Prepend) == 27);
+static_assert(count_entries(cluster::__property::__Prepend) == 28);
 static_assert(count_entries(cluster::__property::__CR) == 1);
 static_assert(count_entries(cluster::__property::__LF) == 1);
 static_assert(count_entries(cluster::__property::__Control) == 3893);
-static_assert(count_entries(cluster::__property::__Extend) == 2130);
+static_assert(count_entries(cluster::__property::__Extend) == 2198);
 static_assert(count_entries(cluster::__property::__Regional_Indicator) == 26);
-static_assert(count_entries(cluster::__property::__SpacingMark) == 395);
+static_assert(count_entries(cluster::__property::__SpacingMark) == 378);
 static_assert(count_entries(cluster::__property::__L) == 125);
-static_assert(count_entries(cluster::__property::__V) == 95);
+static_assert(count_entries(cluster::__property::__V) == 100);
 static_assert(count_entries(cluster::__property::__T) == 137);
 static_assert(count_entries(cluster::__property::__LV) == 399);
 static_assert(count_entries(cluster::__property::__LVT) == 10773);
@@ -68,7 +68,7 @@ constexpr int count_entries(inCB::__property property) {
 
 static_assert(count_entries(inCB::__property::__Linker) == 6);
 static_assert(count_entries(inCB::__property::__Consonant) == 240);
-static_assert(count_entries(inCB::__property::__Extend) == 884);
+static_assert(count_entries(inCB::__property::__Extend) == 2192);
 
 } // namespace
 
diff --git a/libcxx/test/std/algorithms/alg.modifying.operations/alg.copy/copy.pass.cpp b/libcxx/test/std/algorithms/alg.modifying.operations/alg.copy/copy.pass.cpp
index b5f0a32b986a0..1ca397c92a334 100644
--- a/libcxx/test/std/algorithms/alg.modifying.operations/alg.copy/copy.pass.cpp
+++ b/libcxx/test/std/algorithms/alg.modifying.operations/alg.copy/copy.pass.cpp
@@ -14,6 +14,7 @@
 
 #include <algorithm>
 #include <cassert>
+#include <vector>
 
 #include "test_macros.h"
 #include "test_iterators.h"
@@ -59,6 +60,26 @@ struct TestInIters {
   }
 };
 
+TEST_CONSTEXPR_CXX20 bool test_vector_bool(std::size_t N) {
+  std::vector<bool> in(N, false);
+  for (std::size_t i = 0; i < N; i += 2)
+    in[i] = true;
+
+  { // Test copy with aligned bytes
+    std::vector<bool> out(N);
+    std::copy(in.begin(), in.end(), out.begin());
+    assert(in == out);
+  }
+  { // Test copy with unaligned bytes
+    std::vector<bool> out(N + 8);
+    std::copy(in.begin(), in.end(), out.begin() + 4);
+    for (std::size_t i = 0; i < N; ++i)
+      assert(out[i + 4] == in[i]);
+  }
+
+  return true;
+}
+
 TEST_CONSTEXPR_CXX20 bool test() {
   types::for_each(types::cpp17_input_iterator_list<int*>(), TestInIters());
 
@@ -78,13 +99,23 @@ TEST_CONSTEXPR_CXX20 bool test() {
     assert(std::equal(a, a + 10, expected));
   }
 
+  { // Test vector<bool>::iterator optimization
+    assert(test_vector_bool(8));
+    assert(test_vector_bool(19));
+    assert(test_vector_bool(32));
+    assert(test_vector_bool(49));
+    assert(test_vector_bool(64));
+    assert(test_vector_bool(199));
+    assert(test_vector_bool(256));
+  }
+
   return true;
 }
 
 int main(int, char**) {
   test();
 
-#if TEST_STD_VER > 17
+#if TEST_STD_VER >= 20
   static_assert(test());
 #endif
 
diff --git a/libcxx/test/std/algorithms/alg.modifying.operations/alg.copy/copy_backward.pass.cpp b/libcxx/test/std/algorithms/alg.modifying.operations/alg.copy/copy_backward.pass.cpp
index 928903de1ade2..445c7718e1111 100644
--- a/libcxx/test/std/algorithms/alg.modifying.operations/alg.copy/copy_backward.pass.cpp
+++ b/libcxx/test/std/algorithms/alg.modifying.operations/alg.copy/copy_backward.pass.cpp
@@ -15,6 +15,7 @@
 
 #include <algorithm>
 #include <cassert>
+#include <vector>
 
 #include "test_macros.h"
 #include "test_iterators.h"
@@ -36,47 +37,63 @@ class Derived : public PaddedBase {
 };
 
 template <class InIter, class OutIter>
-TEST_CONSTEXPR_CXX20 void
-test_copy_backward()
-{
+TEST_CONSTEXPR_CXX20 void test_copy_backward() {
   {
     const unsigned N = 1000;
-    int ia[N] = {};
+    int ia[N]        = {};
     for (unsigned i = 0; i < N; ++i)
-        ia[i] = i;
+      ia[i] = i;
     int ib[N] = {0};
 
-    OutIter r = std::copy_backward(InIter(ia), InIter(ia+N), OutIter(ib+N));
+    OutIter r = std::copy_backward(InIter(ia), InIter(ia + N), OutIter(ib + N));
     assert(base(r) == ib);
     for (unsigned i = 0; i < N; ++i)
-        assert(ia[i] == ib[i]);
+      assert(ia[i] == ib[i]);
   }
 }
 
-TEST_CONSTEXPR_CXX20 bool
-test()
-{
-    test_copy_backward<bidirectional_iterator<const int*>, bidirectional_iterator<int*> >();
-    test_copy_backward<bidirectional_iterator<const int*>, random_access_iterator<int*> >();
-    test_copy_backward<bidirectional_iterator<const int*>, int*>();
+TEST_CONSTEXPR_CXX20 bool test_vector_bool(std::size_t N) {
+  std::vector<bool> in(N, false);
+  for (std::size_t i = 0; i < N; i += 2)
+    in[i] = true;
 
-    test_copy_backward<random_access_iterator<const int*>, bidirectional_iterator<int*> >();
-    test_copy_backward<random_access_iterator<const int*>, random_access_iterator<int*> >();
-    test_copy_backward<random_access_iterator<const int*>, int*>();
+  { // Test copy_backward with aligned bytes
+    std::vector<bool> out(N);
+    std::copy_backward(in.begin(), in.end(), out.end());
+    assert(in == out);
+  }
+  { // Test copy_backward with unaligned bytes
+    std::vector<bool> out(N + 8);
+    std::copy_backward(in.begin(), in.end(), out.end() - 4);
+    for (std::size_t i = 0; i < N; ++i)
+      assert(out[i + 4] == in[i]);
+  }
+
+  return true;
+};
 
-    test_copy_backward<const int*, bidirectional_iterator<int*> >();
-    test_copy_backward<const int*, random_access_iterator<int*> >();
-    test_copy_backward<const int*, int*>();
+TEST_CONSTEXPR_CXX20 bool test() {
+  test_copy_backward<bidirectional_iterator<const int*>, bidirectional_iterator<int*> >();
+  test_copy_backward<bidirectional_iterator<const int*>, random_access_iterator<int*> >();
+  test_copy_backward<bidirectional_iterator<const int*>, int*>();
+
+  test_copy_backward<random_access_iterator<const int*>, bidirectional_iterator<int*> >();
+  test_copy_backward<random_access_iterator<const int*>, random_access_iterator<int*> >();
+  test_copy_backward<random_access_iterator<const int*>, int*>();
+
+  test_copy_backward<const int*, bidirectional_iterator<int*> >();
+  test_copy_backward<const int*, random_access_iterator<int*> >();
+  test_copy_backward<const int*, int*>();
 
 #if TEST_STD_VER > 17
-    test_copy_backward<contiguous_iterator<const int*>, bidirectional_iterator<int*>>();
-    test_copy_backward<contiguous_iterator<const int*>, random_access_iterator<int*>>();
-    test_copy_backward<contiguous_iterator<const int*>, int*>();
-
-    test_copy_backward<bidirectional_iterator<const int*>, contiguous_iterator<int*>>();
-    test_copy_backward<random_access_iterator<const int*>, contiguous_iterator<int*>>();
-    test_copy_backward<contiguous_iterator<const int*>, contiguous_iterator<int*>>();
-    test_copy_backward<const int*, contiguous_iterator<int*>>();
+  test_copy_backward<contiguous_iterator<const int*>, bidirectional_iterator<int*>>();
+  test_copy_backward<contiguous_iterator<const int*>, random_access_iterator<int*>>();
+  test_copy_backward<contiguous_iterator<const int*>, int*>();
+
+  test_copy_backward<bidirectional_iterator<const int*>, contiguous_iterator<int*>>();
+  test_copy_backward<random_access_iterator<const int*>, contiguous_iterator<int*>>();
+  test_copy_backward<contiguous_iterator<const int*>, contiguous_iterator<int*>>();
+  test_copy_backward<const int*, contiguous_iterator<int*>>();
 #endif
 
   { // Make sure that padding bits aren't copied
@@ -96,15 +113,24 @@ test()
     assert(std::equal(a, a + 10, expected));
   }
 
-    return true;
+  { // Test vector<bool>::iterator optimization
+    assert(test_vector_bool(8));
+    assert(test_vector_bool(19));
+    assert(test_vector_bool(32));
+    assert(test_vector_bool(49));
+    assert(test_vector_bool(64));
+    assert(test_vector_bool(199));
+    assert(test_vector_bool(256));
+  }
+
+  return true;
 }
 
-int main(int, char**)
-{
-    test();
+int main(int, char**) {
+  test();
 
 #if TEST_STD_VER > 17
-    static_assert(test());
+  static_assert(test());
 #endif
 
   return 0;
diff --git a/libcxx/test/std/algorithms/alg.modifying.operations/alg.copy/copy_n.pass.cpp b/libcxx/test/std/algorithms/alg.modifying.operations/alg.copy/copy_n.pass.cpp
index b0acc1060101c..889e71f4eceb9 100644
--- a/libcxx/test/std/algorithms/alg.modifying.operations/alg.copy/copy_n.pass.cpp
+++ b/libcxx/test/std/algorithms/alg.modifying.operations/alg.copy/copy_n.pass.cpp
@@ -14,6 +14,7 @@
 
 #include <algorithm>
 #include <cassert>
+#include <vector>
 
 #include "test_macros.h"
 #include "test_iterators.h"
@@ -37,20 +38,18 @@ class Derived : public PaddedBase {
 };
 
 template <class InIter, class OutIter>
-TEST_CONSTEXPR_CXX20 void
-test_copy_n()
-{
+TEST_CONSTEXPR_CXX20 void test_copy_n() {
   {
     const unsigned N = 1000;
-    int ia[N] = {};
+    int ia[N]        = {};
     for (unsigned i = 0; i < N; ++i)
-        ia[i] = i;
+      ia[i] = i;
     int ib[N] = {0};
 
-    OutIter r = std::copy_n(InIter(ia), UDI(N/2), OutIter(ib));
-    assert(base(r) == ib+N/2);
-    for (unsigned i = 0; i < N/2; ++i)
-        assert(ia[i] == ib[i]);
+    OutIter r = std::copy_n(InIter(ia), UDI(N / 2), OutIter(ib));
+    assert(base(r) == ib + N / 2);
+    for (unsigned i = 0; i < N / 2; ++i)
+      assert(ia[i] == ib[i]);
   }
 
   { // Make sure that padding bits aren't copied
@@ -70,53 +69,80 @@ test_copy_n()
   }
 }
 
-TEST_CONSTEXPR_CXX20 bool
-test()
-{
-    test_copy_n<cpp17_input_iterator<const int*>, cpp17_output_iterator<int*> >();
-    test_copy_n<cpp17_input_iterator<const int*>, cpp17_input_iterator<int*> >();
-    test_copy_n<cpp17_input_iterator<const int*>, forward_iterator<int*> >();
-    test_copy_n<cpp17_input_iterator<const int*>, bidirectional_iterator<int*> >();
-    test_copy_n<cpp17_input_iterator<const int*>, random_access_iterator<int*> >();
-    test_copy_n<cpp17_input_iterator<const int*>, int*>();
-
-    test_copy_n<forward_iterator<const int*>, cpp17_output_iterator<int*> >();
-    test_copy_n<forward_iterator<const int*>, cpp17_input_iterator<int*> >();
-    test_copy_n<forward_iterator<const int*>, forward_iterator<int*> >();
-    test_copy_n<forward_iterator<const int*>, bidirectional_iterator<int*> >();
-    test_copy_n<forward_iterator<const int*>, random_access_iterator<int*> >();
-    test_copy_n<forward_iterator<const int*>, int*>();
-
-    test_copy_n<bidirectional_iterator<const int*>, cpp17_output_iterator<int*> >();
-    test_copy_n<bidirectional_iterator<const int*>, cpp17_input_iterator<int*> >();
-    test_copy_n<bidirectional_iterator<const int*>, forward_iterator<int*> >();
-    test_copy_n<bidirectional_iterator<const int*>, bidirectional_iterator<int*> >();
-    test_copy_n<bidirectional_iterator<const int*>, random_access_iterator<int*> >();
-    test_copy_n<bidirectional_iterator<const int*>, int*>();
-
-    test_copy_n<random_access_iterator<const int*>, cpp17_output_iterator<int*> >();
-    test_copy_n<random_access_iterator<const int*>, cpp17_input_iterator<int*> >();
-    test_copy_n<random_access_iterator<const int*>, forward_iterator<int*> >();
-    test_copy_n<random_access_iterator<const int*>, bidirectional_iterator<int*> >();
-    test_copy_n<random_access_iterator<const int*>, random_access_iterator<int*> >();
-    test_copy_n<random_access_iterator<const int*>, int*>();
-
-    test_copy_n<const int*, cpp17_output_iterator<int*> >();
-    test_copy_n<const int*, cpp17_input_iterator<int*> >();
-    test_copy_n<const int*, forward_iterator<int*> >();
-    test_copy_n<const int*, bidirectional_iterator<int*> >();
-    test_copy_n<const int*, random_access_iterator<int*> >();
-    test_copy_n<const int*, int*>();
+TEST_CONSTEXPR_CXX20 bool test_vector_bool(std::size_t N) {
+  std::vector<bool> in(N, false);
+  for (std::size_t i = 0; i < N; i += 2)
+    in[i] = true;
+
+  { // Test copy with aligned bytes
+    std::vector<bool> out(N);
+    std::copy_n(in.begin(), N, out.begin());
+    assert(in == out);
+  }
+  { // Test copy with unaligned bytes
+    std::vector<bool> out(N + 8);
+    std::copy_n(in.begin(), N, out.begin() + 4);
+    for (std::size_t i = 0; i < N; ++i)
+      assert(out[i + 4] == in[i]);
+  }
+
+  return true;
+}
+
+TEST_CONSTEXPR_CXX20 bool test() {
+  test_copy_n<cpp17_input_iterator<const int*>, cpp17_output_iterator<int*> >();
+  test_copy_n<cpp17_input_iterator<const int*>, cpp17_input_iterator<int*> >();
+  test_copy_n<cpp17_input_iterator<const int*>, forward_iterator<int*> >();
+  test_copy_n<cpp17_input_iterator<const int*>, bidirectional_iterator<int*> >();
+  test_copy_n<cpp17_input_iterator<const int*>, random_access_iterator<int*> >();
+  test_copy_n<cpp17_input_iterator<const int*>, int*>();
+
+  test_copy_n<forward_iterator<const int*>, cpp17_output_iterator<int*> >();
+  test_copy_n<forward_iterator<const int*>, cpp17_input_iterator<int*> >();
+  test_copy_n<forward_iterator<const int*>, forward_iterator<int*> >();
+  test_copy_n<forward_iterator<const int*>, bidirectional_iterator<int*> >();
+  test_copy_n<forward_iterator<const int*>, random_access_iterator<int*> >();
+  test_copy_n<forward_iterator<const int*>, int*>();
+
+  test_copy_n<bidirectional_iterator<const int*>, cpp17_output_iterator<int*> >();
+  test_copy_n<bidirectional_iterator<const int*>, cpp17_input_iterator<int*> >();
+  test_copy_n<bidirectional_iterator<const int*>, forward_iterator<int*> >();
+  test_copy_n<bidirectional_iterator<const int*>, bidirectional_iterator<int*> >();
+  test_copy_n<bidirectional_iterator<const int*>, random_access_iterator<int*> >();
+  test_copy_n<bidirectional_iterator<const int*>, int*>();
+
+  test_copy_n<random_access_iterator<const int*>, cpp17_output_iterator<int*> >();
+  test_copy_n<random_access_iterator<const int*>, cpp17_input_iterator<int*> >();
+  test_copy_n<random_access_iterator<const int*>, forward_iterator<int*> >();
+  test_copy_n<random_access_iterator<const int*>, bidirectional_iterator<int*> >();
+  test_copy_n<random_access_iterator<const int*>, random_access_iterator<int*> >();
+  test_copy_n<random_access_iterator<const int*>, int*>();
+
+  test_copy_n<const int*, cpp17_output_iterator<int*> >();
+  test_copy_n<const int*, cpp17_input_iterator<int*> >();
+  test_copy_n<const int*, forward_iterator<int*> >();
+  test_copy_n<const int*, bidirectional_iterator<int*> >();
+  test_copy_n<const int*, random_access_iterator<int*> >();
+  test_copy_n<const int*, int*>();
+
+  { // Test vector<bool>::iterator optimization
+    assert(test_vector_bool(8));
+    assert(test_vector_bool(19));
+    assert(test_vector_bool(32));
+    assert(test_vector_bool(49));
+    assert(test_vector_bool(64));
+    assert(test_vector_bool(199));
+    assert(test_vector_bool(256));
+  }
 
   return true;
 }
 
-int main(int, char**)
-{
-    test();
+int main(int, char**) {
+  test();
 
 #if TEST_STD_VER > 17
-    static_assert(test());
+  static_assert(test());
 #endif
 
   return 0;
diff --git a/libcxx/test/std/algorithms/alg.modifying.operations/alg.copy/ranges.copy.pass.cpp b/libcxx/test/std/algorithms/alg.modifying.operations/alg.copy/ranges.copy.pass.cpp
index 2507e594fe944..68356c80ba7f6 100644
--- a/libcxx/test/std/algorithms/alg.modifying.operations/alg.copy/ranges.copy.pass.cpp
+++ b/libcxx/test/std/algorithms/alg.modifying.operations/alg.copy/ranges.copy.pass.cpp
@@ -26,6 +26,7 @@
 
 #include "almost_satisfies_types.h"
 #include "test_iterators.h"
+#include "test_macros.h"
 #include "type_algorithms.h"
 
 template <class In, class Out = In, class Sent = sentinel_wrapper<In>>
@@ -99,6 +100,28 @@ constexpr void test_iterators() {
 }
 // clang-format on
 
+#if TEST_STD_VER >= 23
+constexpr bool test_vector_bool(std::size_t N) {
+  std::vector<bool> in(N, false);
+  for (std::size_t i = 0; i < N; i += 2)
+    in[i] = true;
+
+  { // Test copy with aligned bytes
+    std::vector<bool> out(N);
+    std::ranges::copy(in, out.begin());
+    assert(in == out);
+  }
+  { // Test copy with unaligned bytes
+    std::vector<bool> out(N + 8);
+    std::ranges::copy(in, out.begin() + 4);
+    for (std::size_t i = 0; i < N; ++i)
+      assert(out[i + 4] == in[i]);
+  }
+
+  return true;
+}
+#endif
+
 constexpr bool test() {
   types::for_each(types::forward_iterator_list<int*>{}, []<class Out>() {
     test_iterators<cpp20_input_iterator<int*>, Out, sentinel_wrapper<cpp20_input_iterator<int*>>>();
@@ -204,6 +227,18 @@ constexpr bool test() {
     }
   }
 
+#if TEST_STD_VER >= 23
+  { // Test vector<bool>::iterator optimization
+    assert(test_vector_bool(8));
+    assert(test_vector_bool(19));
+    assert(test_vector_bool(32));
+    assert(test_vector_bool(49));
+    assert(test_vector_bool(64));
+    assert(test_vector_bool(199));
+    assert(test_vector_bool(256));
+  }
+#endif
+
   return true;
 }
 
diff --git a/libcxx/test/std/algorithms/alg.modifying.operations/alg.copy/ranges.copy_backward.pass.cpp b/libcxx/test/std/algorithms/alg.modifying.operations/alg.copy/ranges.copy_backward.pass.cpp
index 343447446ab2d..a7fa3db23e6ba 100644
--- a/libcxx/test/std/algorithms/alg.modifying.operations/alg.copy/ranges.copy_backward.pass.cpp
+++ b/libcxx/test/std/algorithms/alg.modifying.operations/alg.copy/ranges.copy_backward.pass.cpp
@@ -29,6 +29,7 @@
 
 #include "almost_satisfies_types.h"
 #include "test_iterators.h"
+#include "test_macros.h"
 
 template <class In, class Out = In, class Sent = sentinel_wrapper<In>>
 concept HasCopyBackwardIt = requires(In in, Sent sent, Out out) { std::ranges::copy_backward(in, sent, out); };
@@ -61,16 +62,16 @@ template <class In, class Out, class Sent>
 constexpr void test_iterators() {
   { // simple test
     {
-      std::array in {1, 2, 3, 4};
+      std::array in{1, 2, 3, 4};
       std::array<int, 4> out;
       std::same_as<std::ranges::in_out_result<In, Out>> auto ret =
-        std::ranges::copy_backward(In(in.data()), Sent(In(in.data() + in.size())), Out(out.data() + out.size()));
+          std::ranges::copy_backward(In(in.data()), Sent(In(in.data() + in.size())), Out(out.data() + out.size()));
       assert(in == out);
       assert(base(ret.in) == in.data() + in.size());
       assert(base(ret.out) == out.data());
     }
     {
-      std::array in {1, 2, 3, 4};
+      std::array in{1, 2, 3, 4};
       std::array<int, 4> out;
       auto range = std::ranges::subrange(In(in.data()), Sent(In(in.data() + in.size())));
       std::same_as<std::ranges::in_out_result<In, Out>> auto ret =
@@ -94,7 +95,7 @@ constexpr void test_iterators() {
       std::array<int, 0> in;
       std::array<int, 0> out;
       auto range = std::ranges::subrange(In(in.data()), Sent(In(in.data() + in.size())));
-      auto ret = std::ranges::copy_backward(range, Out(out.data()));
+      auto ret   = std::ranges::copy_backward(range, Out(out.data()));
       assert(base(ret.in) == in.data() + in.size());
       assert(base(ret.out) == out.data());
     }
@@ -104,16 +105,16 @@ constexpr void test_iterators() {
 template <class InContainer, class OutContainer, class In, class Out, class Sent = In>
 constexpr void test_containers() {
   {
-    InContainer in {1, 2, 3, 4};
+    InContainer in{1, 2, 3, 4};
     OutContainer out(4);
     std::same_as<std::ranges::in_out_result<In, Out>> auto ret =
-      std::ranges::copy_backward(In(in.begin()), Sent(In(in.end())), Out(out.end()));
+        std::ranges::copy_backward(In(in.begin()), Sent(In(in.end())), Out(out.end()));
     assert(std::ranges::equal(in, out));
     assert(base(ret.in) == in.end());
     assert(base(ret.out) == out.begin());
   }
   {
-    InContainer in {1, 2, 3, 4};
+    InContainer in{1, 2, 3, 4};
     OutContainer out(4);
     auto range = std::ranges::subrange(In(in.begin()), Sent(In(in.end())));
     std::same_as<std::ranges::in_out_result<In, Out>> auto ret = std::ranges::copy_backward(range, Out(out.end()));
@@ -125,13 +126,12 @@ constexpr void test_containers() {
 
 template <class Iter, class Sent>
 constexpr void test_join_view() {
-  auto to_subranges = std::views::transform([](auto& vec) {
-          return std::ranges::subrange(Iter(vec.begin()), Sent(Iter(vec.end())));
-        });
+  auto to_subranges =
+      std::views::transform([](auto& vec) { return std::ranges::subrange(Iter(vec.begin()), Sent(Iter(vec.end()))); });
 
   { // segmented -> contiguous
     std::vector<std::vector<int>> vectors = {};
-    auto range = vectors | to_subranges;
+    auto range                            = vectors | to_subranges;
     std::vector<std::ranges::subrange<Iter, Sent>> subrange_vector(range.begin(), range.end());
     std::array<int, 0> arr;
 
@@ -140,7 +140,7 @@ constexpr void test_join_view() {
   }
   { // segmented -> contiguous
     std::vector<std::vector<int>> vectors = {{1, 2, 3, 4}, {5, 6, 7, 8}, {9, 10}, {}};
-    auto range = vectors | to_subranges;
+    auto range                            = vectors | to_subranges;
     std::vector<std::ranges::subrange<Iter, Sent>> subrange_vector(range.begin(), range.end());
     std::array<int, 10> arr;
 
@@ -149,7 +149,7 @@ constexpr void test_join_view() {
   }
   { // contiguous -> segmented
     std::vector<std::vector<int>> vectors = {{0, 0, 0, 0}, {0, 0}, {0, 0, 0, 0}, {}};
-    auto range = vectors | to_subranges;
+    auto range                            = vectors | to_subranges;
     std::vector<std::ranges::subrange<Iter, Sent>> subrange_vector(range.begin(), range.end());
     std::array arr = {1, 2, 3, 4, 5, 6, 7, 8, 9, 10};
 
@@ -158,10 +158,10 @@ constexpr void test_join_view() {
   }
   { // segmented -> segmented
     std::vector<std::vector<int>> vectors = {{1, 2, 3, 4}, {5, 6, 7, 8}, {9, 10}, {}};
-    auto range1 = vectors | to_subranges;
+    auto range1                           = vectors | to_subranges;
     std::vector<std::ranges::subrange<Iter, Sent>> subrange_vector(range1.begin(), range1.end());
     std::vector<std::vector<int>> to_vectors = {{0, 0, 0, 0}, {0, 0, 0, 0}, {}, {0, 0}};
-    auto range2 = to_vectors | to_subranges;
+    auto range2                              = to_vectors | to_subranges;
     std::vector<std::ranges::subrange<Iter, Sent>> to_subrange_vector(range2.begin(), range2.end());
 
     std::ranges::copy_backward(subrange_vector | std::views::join, (to_subrange_vector | std::views::join).end());
@@ -224,6 +224,30 @@ constexpr void test_proxy_in_iterators() {
   test_sentinels<ProxyIterator, Out>();
 }
 
+#if TEST_STD_VER >= 23
+
+constexpr bool test_vector_bool(std::size_t N) {
+  std::vector<bool> in(N, false);
+  for (std::size_t i = 0; i < N; i += 2)
+    in[i] = true;
+
+  { // Test copy_backward with aligned bytes
+    std::vector<bool> out(N);
+    std::ranges::copy_backward(in, out.end());
+    assert(in == out);
+  }
+  { // Test copy_backward with unaligned bytes
+    std::vector<bool> out(N + 8);
+    std::ranges::copy_backward(in, out.end() - 4);
+    for (std::size_t i = 0; i < N; ++i)
+      assert(out[i + 4] == in[i]);
+  }
+
+  return true;
+};
+
+#endif
+
 constexpr bool test() {
   test_in_iterators<bidirectional_iterator>();
   test_in_iterators<random_access_iterator>();
@@ -237,13 +261,13 @@ constexpr bool test() {
   { // check that ranges::dangling is returned
     std::array<int, 4> out;
     std::same_as<std::ranges::in_out_result<std::ranges::dangling, int*>> auto ret =
-      std::ranges::copy_backward(std::array {1, 2, 3, 4}, out.data() + out.size());
+        std::ranges::copy_backward(std::array{1, 2, 3, 4}, out.data() + out.size());
     assert(ret.out == out.data());
     assert((out == std::array{1, 2, 3, 4}));
   }
 
   { // check that an iterator is returned with a borrowing range
-    std::array in {1, 2, 3, 4};
+    std::array in{1, 2, 3, 4};
     std::array<int, 4> out;
     std::same_as<std::ranges::in_out_result<std::array<int, 4>::iterator, int*>> auto ret =
         std::ranges::copy_backward(std::views::all(in), out.data() + out.size());
@@ -254,8 +278,8 @@ constexpr bool test() {
 
   { // check that every element is copied exactly once
     struct CopyOnce {
-      bool copied = false;
-      constexpr CopyOnce() = default;
+      bool copied                               = false;
+      constexpr CopyOnce()                      = default;
       constexpr CopyOnce(const CopyOnce& other) = delete;
       constexpr CopyOnce& operator=(const CopyOnce& other) {
         assert(!other.copied);
@@ -264,16 +288,16 @@ constexpr bool test() {
       }
     };
     {
-      std::array<CopyOnce, 4> in {};
-      std::array<CopyOnce, 4> out {};
+      std::array<CopyOnce, 4> in{};
+      std::array<CopyOnce, 4> out{};
       auto ret = std::ranges::copy_backward(in.begin(), in.end(), out.end());
       assert(ret.in == in.end());
       assert(ret.out == out.begin());
       assert(std::all_of(out.begin(), out.end(), [](const auto& e) { return e.copied; }));
     }
     {
-      std::array<CopyOnce, 4> in {};
-      std::array<CopyOnce, 4> out {};
+      std::array<CopyOnce, 4> in{};
+      std::array<CopyOnce, 4> out{};
       auto ret = std::ranges::copy_backward(in, out.end());
       assert(ret.in == in.end());
       assert(ret.out == out.begin());
@@ -284,8 +308,8 @@ constexpr bool test() {
   { // check that the range is copied backwards
     struct OnlyBackwardsCopyable {
       OnlyBackwardsCopyable* next = nullptr;
-      bool canCopy = false;
-      OnlyBackwardsCopyable() = default;
+      bool canCopy                = false;
+      OnlyBackwardsCopyable()     = default;
       constexpr OnlyBackwardsCopyable& operator=(const OnlyBackwardsCopyable&) {
         assert(canCopy);
         if (next != nullptr)
@@ -294,12 +318,12 @@ constexpr bool test() {
       }
     };
     {
-      std::array<OnlyBackwardsCopyable, 3> in {};
-      std::array<OnlyBackwardsCopyable, 3> out {};
-      out[1].next = &out[0];
-      out[2].next = &out[1];
+      std::array<OnlyBackwardsCopyable, 3> in{};
+      std::array<OnlyBackwardsCopyable, 3> out{};
+      out[1].next    = &out[0];
+      out[2].next    = &out[1];
       out[2].canCopy = true;
-      auto ret = std::ranges::copy_backward(in, out.end());
+      auto ret       = std::ranges::copy_backward(in, out.end());
       assert(ret.in == in.end());
       assert(ret.out == out.begin());
       assert(out[0].canCopy);
@@ -307,12 +331,12 @@ constexpr bool test() {
       assert(out[2].canCopy);
     }
     {
-      std::array<OnlyBackwardsCopyable, 3> in {};
-      std::array<OnlyBackwardsCopyable, 3> out {};
-      out[1].next = &out[0];
-      out[2].next = &out[1];
+      std::array<OnlyBackwardsCopyable, 3> in{};
+      std::array<OnlyBackwardsCopyable, 3> out{};
+      out[1].next    = &out[0];
+      out[2].next    = &out[1];
       out[2].canCopy = true;
-      auto ret = std::ranges::copy_backward(in.begin(), in.end(), out.end());
+      auto ret       = std::ranges::copy_backward(in.begin(), in.end(), out.end());
       assert(ret.in == in.end());
       assert(ret.out == out.begin());
       assert(out[0].canCopy);
@@ -321,6 +345,18 @@ constexpr bool test() {
     }
   }
 
+#if TEST_STD_VER >= 23
+  { // Test vector<bool>::iterator optimization
+    assert(test_vector_bool(8));
+    assert(test_vector_bool(19));
+    assert(test_vector_bool(32));
+    assert(test_vector_bool(49));
+    assert(test_vector_bool(64));
+    assert(test_vector_bool(199));
+    assert(test_vector_bool(256));
+  }
+#endif
+
   return true;
 }
 
diff --git a/libcxx/test/std/algorithms/alg.modifying.operations/alg.copy/ranges.copy_n.pass.cpp b/libcxx/test/std/algorithms/alg.modifying.operations/alg.copy/ranges.copy_n.pass.cpp
index d2a2b7c488830..c7031f63a02f6 100644
--- a/libcxx/test/std/algorithms/alg.modifying.operations/alg.copy/ranges.copy_n.pass.cpp
+++ b/libcxx/test/std/algorithms/alg.modifying.operations/alg.copy/ranges.copy_n.pass.cpp
@@ -19,8 +19,10 @@
 #include <array>
 #include <cassert>
 #include <ranges>
+#include <vector>
 
 #include "almost_satisfies_types.h"
+#include "test_macros.h"
 #include "test_iterators.h"
 
 template <class In, class Out = In, class Count = std::size_t>
@@ -41,10 +43,10 @@ static_assert(std::is_same_v<std::ranges::copy_result<int, long>, std::ranges::i
 template <class In, class Out, class Sent = In>
 constexpr void test_iterators() {
   { // simple test
-    std::array in {1, 2, 3, 4};
+    std::array in{1, 2, 3, 4};
     std::array<int, 4> out;
     std::same_as<std::ranges::in_out_result<In, Out>> auto ret =
-      std::ranges::copy_n(In(in.data()), in.size(), Out(out.data()));
+        std::ranges::copy_n(In(in.data()), in.size(), Out(out.data()));
     assert(in == out);
     assert(base(ret.in) == in.data() + in.size());
     assert(base(ret.out) == out.data() + out.size());
@@ -70,13 +72,37 @@ constexpr void test_in_iterators() {
 
 template <class Out>
 constexpr void test_proxy_in_iterators() {
-  test_iterators<ProxyIterator<cpp20_input_iterator<int*>>, Out, sentinel_wrapper<ProxyIterator<cpp20_input_iterator<int*>>>>();
+  test_iterators<ProxyIterator<cpp20_input_iterator<int*>>,
+                 Out,
+                 sentinel_wrapper<ProxyIterator<cpp20_input_iterator<int*>>>>();
   test_iterators<ProxyIterator<forward_iterator<int*>>, Out>();
   test_iterators<ProxyIterator<bidirectional_iterator<int*>>, Out>();
   test_iterators<ProxyIterator<random_access_iterator<int*>>, Out>();
   test_iterators<ProxyIterator<contiguous_iterator<int*>>, Out>();
 }
 
+#if TEST_STD_VER >= 23
+constexpr bool test_vector_bool(std::size_t N) {
+  std::vector<bool> in(N, false);
+  for (std::size_t i = 0; i < N; i += 2)
+    in[i] = true;
+
+  { // Test copy with aligned bytes
+    std::vector<bool> out(N);
+    std::ranges::copy_n(in.begin(), N, out.begin());
+    assert(in == out);
+  }
+  { // Test copy with unaligned bytes
+    std::vector<bool> out(N + 8);
+    std::ranges::copy_n(in.begin(), N, out.begin() + 4);
+    for (std::size_t i = 0; i < N; ++i)
+      assert(out[i + 4] == in[i]);
+  }
+
+  return true;
+};
+#endif
+
 constexpr bool test() {
   test_in_iterators<cpp20_input_iterator<int*>>();
   test_in_iterators<forward_iterator<int*>>();
@@ -92,8 +118,8 @@ constexpr bool test() {
 
   { // check that every element is copied exactly once
     struct CopyOnce {
-      bool copied = false;
-      constexpr CopyOnce() = default;
+      bool copied                               = false;
+      constexpr CopyOnce()                      = default;
       constexpr CopyOnce(const CopyOnce& other) = delete;
       constexpr CopyOnce& operator=(const CopyOnce& other) {
         assert(!other.copied);
@@ -101,14 +127,26 @@ constexpr bool test() {
         return *this;
       }
     };
-    std::array<CopyOnce, 4> in {};
-    std::array<CopyOnce, 4> out {};
+    std::array<CopyOnce, 4> in{};
+    std::array<CopyOnce, 4> out{};
     auto ret = std::ranges::copy_n(in.begin(), in.size(), out.begin());
     assert(ret.in == in.end());
     assert(ret.out == out.end());
     assert(std::all_of(out.begin(), out.end(), [](const auto& e) { return e.copied; }));
   }
 
+#if TEST_STD_VER >= 23
+  { // Test vector<bool>::iterator optimization
+    assert(test_vector_bool(8));
+    assert(test_vector_bool(19));
+    assert(test_vector_bool(32));
+    assert(test_vector_bool(49));
+    assert(test_vector_bool(64));
+    assert(test_vector_bool(199));
+    assert(test_vector_bool(256));
+  }
+#endif
+
   return true;
 }
 
diff --git a/libcxx/test/std/algorithms/alg.modifying.operations/alg.fill/fill.pass.cpp b/libcxx/test/std/algorithms/alg.modifying.operations/alg.fill/fill.pass.cpp
index 619dc7242a366..7656be73c14c6 100644
--- a/libcxx/test/std/algorithms/alg.modifying.operations/alg.fill/fill.pass.cpp
+++ b/libcxx/test/std/algorithms/alg.modifying.operations/alg.fill/fill.pass.cpp
@@ -19,6 +19,7 @@
 #include <cstddef>
 #include <vector>
 
+#include "sized_allocator.h"
 #include "test_macros.h"
 #include "test_iterators.h"
 
@@ -46,6 +47,39 @@ struct Test {
   }
 };
 
+// Make sure std::fill behaves properly with std::vector<bool> iterators with custom size types.
+// See https://github.com/llvm/llvm-project/pull/122410.
+TEST_CONSTEXPR_CXX20 void test_bititer_with_custom_sized_types() {
+  {
+    using Alloc = sized_allocator<bool, std::uint8_t, std::int8_t>;
+    std::vector<bool, Alloc> in(100, false, Alloc(1));
+    std::vector<bool, Alloc> expected(100, true, Alloc(1));
+    std::fill(in.begin(), in.end(), true);
+    assert(in == expected);
+  }
+  {
+    using Alloc = sized_allocator<bool, std::uint16_t, std::int16_t>;
+    std::vector<bool, Alloc> in(200, false, Alloc(1));
+    std::vector<bool, Alloc> expected(200, true, Alloc(1));
+    std::fill(in.begin(), in.end(), true);
+    assert(in == expected);
+  }
+  {
+    using Alloc = sized_allocator<bool, std::uint32_t, std::int32_t>;
+    std::vector<bool, Alloc> in(200, false, Alloc(1));
+    std::vector<bool, Alloc> expected(200, true, Alloc(1));
+    std::fill(in.begin(), in.end(), true);
+    assert(in == expected);
+  }
+  {
+    using Alloc = sized_allocator<bool, std::uint64_t, std::int64_t>;
+    std::vector<bool, Alloc> in(200, false, Alloc(1));
+    std::vector<bool, Alloc> expected(200, true, Alloc(1));
+    std::fill(in.begin(), in.end(), true);
+    assert(in == expected);
+  }
+}
+
 TEST_CONSTEXPR_CXX20 bool test() {
   types::for_each(types::forward_iterator_list<char*>(), Test<char>());
   types::for_each(types::forward_iterator_list<int*>(), Test<int>());
@@ -93,6 +127,9 @@ TEST_CONSTEXPR_CXX20 bool test() {
       assert(in == expected);
     }
   }
+
+  test_bititer_with_custom_sized_types();
+
   return true;
 }
 
diff --git a/libcxx/test/std/algorithms/alg.modifying.operations/alg.fill/fill_n.pass.cpp b/libcxx/test/std/algorithms/alg.modifying.operations/alg.fill/fill_n.pass.cpp
index 7d6770de702bf..3b67101a8b29e 100644
--- a/libcxx/test/std/algorithms/alg.modifying.operations/alg.fill/fill_n.pass.cpp
+++ b/libcxx/test/std/algorithms/alg.modifying.operations/alg.fill/fill_n.pass.cpp
@@ -15,159 +15,177 @@
 
 #include <algorithm>
 #include <cassert>
+#include <vector>
 
+#include "sized_allocator.h"
 #include "test_macros.h"
 #include "test_iterators.h"
 #include "user_defined_integral.h"
 
 #if TEST_STD_VER > 17
 TEST_CONSTEXPR bool test_constexpr() {
-    const std::size_t N = 5;
-    int ib[] = {0, 0, 0, 0, 0, 0}; // one bigger than N
-
-    auto it = std::fill_n(std::begin(ib), N, 5);
-    return it == (std::begin(ib) + N)
-        && std::all_of(std::begin(ib), it, [](int a) {return a == 5; })
-        && *it == 0 // don't overwrite the last value in the output array
-        ;
-    }
+  const std::size_t N = 5;
+  int ib[]            = {0, 0, 0, 0, 0, 0}; // one bigger than N
+
+  auto it = std::fill_n(std::begin(ib), N, 5);
+  return it == (std::begin(ib) + N) && std::all_of(std::begin(ib), it, [](int a) { return a == 5; }) &&
+       *it == 0 // don't overwrite the last value in the output array
+      ;
+}
 #endif
 
 typedef UserDefinedIntegral<unsigned> UDI;
 
 template <class Iter>
-void
-test_char()
-{
-    char a[4] = {};
-    Iter it = std::fill_n(Iter(a), UDI(4), char(1));
-    assert(base(it) == a + 4);
-    assert(a[0] == 1);
-    assert(a[1] == 1);
-    assert(a[2] == 1);
-    assert(a[3] == 1);
+void test_char() {
+  char a[4] = {};
+  Iter it   = std::fill_n(Iter(a), UDI(4), char(1));
+  assert(base(it) == a + 4);
+  assert(a[0] == 1);
+  assert(a[1] == 1);
+  assert(a[2] == 1);
+  assert(a[3] == 1);
 }
 
 template <class Iter>
-void
-test_int()
-{
-    int a[4] = {};
-    Iter it = std::fill_n(Iter(a), UDI(4), 1);
-    assert(base(it) == a + 4);
-    assert(a[0] == 1);
-    assert(a[1] == 1);
-    assert(a[2] == 1);
-    assert(a[3] == 1);
+void test_int() {
+  int a[4] = {};
+  Iter it  = std::fill_n(Iter(a), UDI(4), 1);
+  assert(base(it) == a + 4);
+  assert(a[0] == 1);
+  assert(a[1] == 1);
+  assert(a[2] == 1);
+  assert(a[3] == 1);
 }
 
-void
-test_int_array()
-{
-    int a[4] = {};
-    assert(std::fill_n(a, UDI(4), static_cast<char>(1)) == a + 4);
-    assert(a[0] == 1);
-    assert(a[1] == 1);
-    assert(a[2] == 1);
-    assert(a[3] == 1);
+void test_int_array() {
+  int a[4] = {};
+  assert(std::fill_n(a, UDI(4), static_cast<char>(1)) == a + 4);
+  assert(a[0] == 1);
+  assert(a[1] == 1);
+  assert(a[2] == 1);
+  assert(a[3] == 1);
 }
 
 struct source {
-    source() : i(0) { }
+  source() : i(0) {}
 
-    operator int() const { return i++; }
-    mutable int i;
+  operator int() const { return i++; }
+  mutable int i;
 };
 
-void
-test_int_array_struct_source()
-{
-    int a[4] = {};
-    assert(std::fill_n(a, UDI(4), source()) == a + 4);
-    assert(a[0] == 0);
-    assert(a[1] == 1);
-    assert(a[2] == 2);
-    assert(a[3] == 3);
+void test_int_array_struct_source() {
+  int a[4] = {};
+  assert(std::fill_n(a, UDI(4), source()) == a + 4);
+  assert(a[0] == 0);
+  assert(a[1] == 1);
+  assert(a[2] == 2);
+  assert(a[3] == 3);
 }
 
 struct test1 {
-    test1() : c(0) { }
-    test1(char xc) : c(xc + 1) { }
-    char c;
+  test1() : c(0) {}
+  test1(char xc) : c(xc + 1) {}
+  char c;
 };
 
-void
-test_struct_array()
-{
-    test1 test1a[4] = {};
-    assert(std::fill_n(test1a, UDI(4), static_cast<char>(10)) == test1a + 4);
-    assert(test1a[0].c == 11);
-    assert(test1a[1].c == 11);
-    assert(test1a[2].c == 11);
-    assert(test1a[3].c == 11);
+void test_struct_array() {
+  test1 test1a[4] = {};
+  assert(std::fill_n(test1a, UDI(4), static_cast<char>(10)) == test1a + 4);
+  assert(test1a[0].c == 11);
+  assert(test1a[1].c == 11);
+  assert(test1a[2].c == 11);
+  assert(test1a[3].c == 11);
 }
 
-class A
-{
-    char a_;
+class A {
+  char a_;
+
 public:
-    A() {}
-    explicit A(char a) : a_(a) {}
-    operator unsigned char() const {return 'b';}
+  A() {}
+  explicit A(char a) : a_(a) {}
+  operator unsigned char() const { return 'b'; }
 
-    friend bool operator==(const A& x, const A& y)
-        {return x.a_ == y.a_;}
+  friend bool operator==(const A& x, const A& y) { return x.a_ == y.a_; }
 };
 
-void
-test5()
-{
-    A a[3];
-    assert(std::fill_n(&a[0], UDI(3), A('a')) == a+3);
-    assert(a[0] == A('a'));
-    assert(a[1] == A('a'));
-    assert(a[2] == A('a'));
+void test5() {
+  A a[3];
+  assert(std::fill_n(&a[0], UDI(3), A('a')) == a + 3);
+  assert(a[0] == A('a'));
+  assert(a[1] == A('a'));
+  assert(a[2] == A('a'));
 }
 
-struct Storage
-{
-  union
-  {
+struct Storage {
+  union {
     unsigned char a;
     unsigned char b;
   };
 };
 
-void test6()
-{
+void test6() {
   Storage foo[5];
   std::fill_n(&foo[0], UDI(5), Storage());
 }
 
+// Make sure std::fill_n behaves properly with std::vector<bool> iterators with custom size types.
+// See https://github.com/llvm/llvm-project/pull/122410.
+TEST_CONSTEXPR_CXX20 void test_bititer_with_custom_sized_types() {
+  {
+    using Alloc = sized_allocator<bool, std::uint8_t, std::int8_t>;
+    std::vector<bool, Alloc> in(100, false, Alloc(1));
+    std::vector<bool, Alloc> expected(100, true, Alloc(1));
+    std::fill_n(in.begin(), in.size(), true);
+    assert(in == expected);
+  }
+  {
+    using Alloc = sized_allocator<bool, std::uint16_t, std::int16_t>;
+    std::vector<bool, Alloc> in(200, false, Alloc(1));
+    std::vector<bool, Alloc> expected(200, true, Alloc(1));
+    std::fill_n(in.begin(), in.size(), true);
+    assert(in == expected);
+  }
+  {
+    using Alloc = sized_allocator<bool, std::uint32_t, std::int32_t>;
+    std::vector<bool, Alloc> in(200, false, Alloc(1));
+    std::vector<bool, Alloc> expected(200, true, Alloc(1));
+    std::fill_n(in.begin(), in.size(), true);
+    assert(in == expected);
+  }
+  {
+    using Alloc = sized_allocator<bool, std::uint64_t, std::int64_t>;
+    std::vector<bool, Alloc> in(200, false, Alloc(1));
+    std::vector<bool, Alloc> expected(200, true, Alloc(1));
+    std::fill_n(in.begin(), in.size(), true);
+    assert(in == expected);
+  }
+}
+
+int main(int, char**) {
+  test_char<cpp17_output_iterator<char*> >();
+  test_char<forward_iterator<char*> >();
+  test_char<bidirectional_iterator<char*> >();
+  test_char<random_access_iterator<char*> >();
+  test_char<char*>();
 
-int main(int, char**)
-{
-    test_char<cpp17_output_iterator<char*> >();
-    test_char<forward_iterator<char*> >();
-    test_char<bidirectional_iterator<char*> >();
-    test_char<random_access_iterator<char*> >();
-    test_char<char*>();
+  test_int<cpp17_output_iterator<int*> >();
+  test_int<forward_iterator<int*> >();
+  test_int<bidirectional_iterator<int*> >();
+  test_int<random_access_iterator<int*> >();
+  test_int<int*>();
 
-    test_int<cpp17_output_iterator<int*> >();
-    test_int<forward_iterator<int*> >();
-    test_int<bidirectional_iterator<int*> >();
-    test_int<random_access_iterator<int*> >();
-    test_int<int*>();
+  test_int_array();
+  test_int_array_struct_source();
+  test_struct_array();
 
-    test_int_array();
-    test_int_array_struct_source();
-    test_struct_array();
+  test5();
+  test6();
 
-    test5();
-    test6();
+  test_bititer_with_custom_sized_types();
 
 #if TEST_STD_VER > 17
-    static_assert(test_constexpr());
+  static_assert(test_constexpr());
 #endif
 
   return 0;
diff --git a/libcxx/test/std/algorithms/alg.modifying.operations/alg.fill/ranges.fill.pass.cpp b/libcxx/test/std/algorithms/alg.modifying.operations/alg.fill/ranges.fill.pass.cpp
index 5dc375e0e8dc0..30dfdd5486f5b 100644
--- a/libcxx/test/std/algorithms/alg.modifying.operations/alg.fill/ranges.fill.pass.cpp
+++ b/libcxx/test/std/algorithms/alg.modifying.operations/alg.fill/ranges.fill.pass.cpp
@@ -20,9 +20,12 @@
 #include <cassert>
 #include <ranges>
 #include <string>
+#include <vector>
 
+#include "sized_allocator.h"
 #include "almost_satisfies_types.h"
 #include "test_iterators.h"
+#include "test_macros.h"
 
 template <class Iter, class Sent = sentinel_wrapper<Iter>>
 concept HasFillIt = requires(Iter iter, Sent sent) { std::ranges::fill(iter, sent, int{}); };
@@ -53,7 +56,7 @@ constexpr void test_iterators() {
     }
     {
       int a[3];
-      auto range = std::ranges::subrange(It(a), Sent(It(a + 3)));
+      auto range                = std::ranges::subrange(It(a), Sent(It(a + 3)));
       std::same_as<It> auto ret = std::ranges::fill(range, 1);
       assert(std::all_of(a, a + 3, [](int i) { return i == 1; }));
       assert(base(ret) == a + 3);
@@ -69,12 +72,51 @@ constexpr void test_iterators() {
     {
       std::array<int, 0> a;
       auto range = std::ranges::subrange(It(a.data()), Sent(It(a.data())));
-      auto ret = std::ranges::fill(range, 1);
+      auto ret   = std::ranges::fill(range, 1);
       assert(base(ret) == a.data());
     }
   }
 }
 
+// Make sure std::ranges::fill behaves properly with std::vector<bool> iterators with custom
+// size types. See https://github.com/llvm/llvm-project/pull/122410.
+//
+// The `ranges::{fill, fill_n}` algorithms require `vector<bool, Alloc>::iterator` to satisfy
+// the `std::indirectly_writable` concept when used with `vector<bool, Alloc>`, which is only
+// satisfied since C++23.
+#if TEST_STD_VER >= 23
+TEST_CONSTEXPR_CXX20 void test_bititer_with_custom_sized_types() {
+  {
+    using Alloc = sized_allocator<bool, std::uint8_t, std::int8_t>;
+    std::vector<bool, Alloc> in(100, false, Alloc(1));
+    std::vector<bool, Alloc> expected(100, true, Alloc(1));
+    std::ranges::fill(in, true);
+    assert(in == expected);
+  }
+  {
+    using Alloc = sized_allocator<bool, std::uint16_t, std::int16_t>;
+    std::vector<bool, Alloc> in(200, false, Alloc(1));
+    std::vector<bool, Alloc> expected(200, true, Alloc(1));
+    std::ranges::fill(in, true);
+    assert(in == expected);
+  }
+  {
+    using Alloc = sized_allocator<bool, std::uint32_t, std::int32_t>;
+    std::vector<bool, Alloc> in(200, false, Alloc(1));
+    std::vector<bool, Alloc> expected(200, true, Alloc(1));
+    std::ranges::fill(in, true);
+    assert(in == expected);
+  }
+  {
+    using Alloc = sized_allocator<bool, std::uint64_t, std::int64_t>;
+    std::vector<bool, Alloc> in(200, false, Alloc(1));
+    std::vector<bool, Alloc> expected(200, true, Alloc(1));
+    std::ranges::fill(in, true);
+    assert(in == expected);
+  }
+}
+#endif
+
 constexpr bool test() {
   test_iterators<cpp17_output_iterator<int*>, sentinel_wrapper<cpp17_output_iterator<int*>>>();
   test_iterators<cpp20_output_iterator<int*>, sentinel_wrapper<cpp20_output_iterator<int*>>>();
@@ -94,19 +136,19 @@ constexpr bool test() {
     };
     {
       S a[5];
-      std::ranges::fill(a, a + 5, S {true});
+      std::ranges::fill(a, a + 5, S{true});
       assert(std::all_of(a, a + 5, [](S& s) { return s.copied; }));
     }
     {
       S a[5];
-      std::ranges::fill(a, S {true});
+      std::ranges::fill(a, S{true});
       assert(std::all_of(a, a + 5, [](S& s) { return s.copied; }));
     }
   }
 
   { // check that std::ranges::dangling is returned
     [[maybe_unused]] std::same_as<std::ranges::dangling> decltype(auto) ret =
-        std::ranges::fill(std::array<int, 10> {}, 1);
+        std::ranges::fill(std::array<int, 10>{}, 1);
   }
 
   { // check that std::ranges::dangling isn't returned with a borrowing range
@@ -131,6 +173,10 @@ constexpr bool test() {
     }
   }
 
+#if TEST_STD_VER >= 23
+  test_bititer_with_custom_sized_types();
+#endif
+
   return true;
 }
 
diff --git a/libcxx/test/std/algorithms/alg.modifying.operations/alg.fill/ranges.fill_n.pass.cpp b/libcxx/test/std/algorithms/alg.modifying.operations/alg.fill/ranges.fill_n.pass.cpp
index 10ff385d47428..ae70e7155f67f 100644
--- a/libcxx/test/std/algorithms/alg.modifying.operations/alg.fill/ranges.fill_n.pass.cpp
+++ b/libcxx/test/std/algorithms/alg.modifying.operations/alg.fill/ranges.fill_n.pass.cpp
@@ -18,9 +18,12 @@
 #include <cassert>
 #include <ranges>
 #include <string>
+#include <vector>
 
+#include "sized_allocator.h"
 #include "almost_satisfies_types.h"
 #include "test_iterators.h"
+#include "test_macros.h"
 
 template <class Iter>
 concept HasFillN = requires(Iter iter) { std::ranges::fill_n(iter, int{}, int{}); };
@@ -48,6 +51,45 @@ constexpr void test_iterators() {
   }
 }
 
+// Make sure std::ranges::fill_n behaves properly with std::vector<bool> iterators with custom
+// size types. See https://github.com/llvm/llvm-project/pull/122410.
+//
+// The `ranges::{fill, fill_n}` algorithms require `vector<bool, Alloc>::iterator` to satisfy
+// the `std::indirectly_writable` concept when used with `vector<bool, Alloc>`, which is only
+// satisfied since C++23.
+#if TEST_STD_VER >= 23
+TEST_CONSTEXPR_CXX20 void test_bititer_with_custom_sized_types() {
+  {
+    using Alloc = sized_allocator<bool, std::uint8_t, std::int8_t>;
+    std::vector<bool, Alloc> in(100, false, Alloc(1));
+    std::vector<bool, Alloc> expected(100, true, Alloc(1));
+    std::ranges::fill_n(std::ranges::begin(in), in.size(), true);
+    assert(in == expected);
+  }
+  {
+    using Alloc = sized_allocator<bool, std::uint16_t, std::int16_t>;
+    std::vector<bool, Alloc> in(200, false, Alloc(1));
+    std::vector<bool, Alloc> expected(200, true, Alloc(1));
+    std::ranges::fill_n(std::ranges::begin(in), in.size(), true);
+    assert(in == expected);
+  }
+  {
+    using Alloc = sized_allocator<bool, std::uint32_t, std::int32_t>;
+    std::vector<bool, Alloc> in(200, false, Alloc(1));
+    std::vector<bool, Alloc> expected(200, true, Alloc(1));
+    std::ranges::fill_n(std::ranges::begin(in), in.size(), true);
+    assert(in == expected);
+  }
+  {
+    using Alloc = sized_allocator<bool, std::uint64_t, std::int64_t>;
+    std::vector<bool, Alloc> in(200, false, Alloc(1));
+    std::vector<bool, Alloc> expected(200, true, Alloc(1));
+    std::ranges::fill_n(std::ranges::begin(in), in.size(), true);
+    assert(in == expected);
+  }
+}
+#endif
+
 constexpr bool test() {
   test_iterators<cpp17_output_iterator<int*>, sentinel_wrapper<cpp17_output_iterator<int*>>>();
   test_iterators<cpp20_output_iterator<int*>, sentinel_wrapper<cpp20_output_iterator<int*>>>();
@@ -68,7 +110,7 @@ constexpr bool test() {
     };
 
     S a[5];
-    std::ranges::fill_n(a, 5, S {});
+    std::ranges::fill_n(a, 5, S{});
     assert(std::all_of(a, a + 5, [](S& s) { return s.copied; }));
   }
 
@@ -79,6 +121,10 @@ constexpr bool test() {
     assert(std::all_of(a.begin(), a.end(), [](auto& s) { return s == "long long string so no SSO"; }));
   }
 
+#if TEST_STD_VER >= 23
+  test_bititer_with_custom_sized_types();
+#endif
+
   return true;
 }
 
diff --git a/libcxx/test/std/algorithms/alg.sorting/alg.sort/stable.sort/stable_sort.pass.cpp b/libcxx/test/std/algorithms/alg.sorting/alg.sort/stable.sort/stable_sort.pass.cpp
index b3b458808c44a..4ee1d795a23b2 100644
--- a/libcxx/test/std/algorithms/alg.sorting/alg.sort/stable.sort/stable_sort.pass.cpp
+++ b/libcxx/test/std/algorithms/alg.sorting/alg.sort/stable.sort/stable_sort.pass.cpp
@@ -199,6 +199,8 @@ int main(int, char**) {
 #if TEST_STD_VER >= 26
   static_assert(test<int>());
   static_assert(test<float>());
+  // test constexprness of radix sort branch
+  static_assert(test<char>());
 #endif
   return 0;
 }
diff --git a/libcxx/test/std/atomics/atomics.types.generic/atomics.types.float/fetch_add.pass.cpp b/libcxx/test/std/atomics/atomics.types.generic/atomics.types.float/fetch_add.pass.cpp
index c7a797171e0a7..b162c2da4f337 100644
--- a/libcxx/test/std/atomics/atomics.types.generic/atomics.types.float/fetch_add.pass.cpp
+++ b/libcxx/test/std/atomics/atomics.types.generic/atomics.types.float/fetch_add.pass.cpp
@@ -10,7 +10,7 @@
 
 // Older versions of clang have a bug with atomic builtins affecting double and long double.
 // Fixed by 5fdd0948.
-// XFAIL: target=powerpc-ibm-{{.*}} && (clang-17 || clang-18)
+// XFAIL: target=powerpc-ibm-{{.*}} && clang-18
 
 // https://github.com/llvm/llvm-project/issues/72893
 // XFAIL: target={{x86_64-.*}} && tsan
diff --git a/libcxx/test/std/atomics/atomics.types.generic/atomics.types.float/fetch_sub.pass.cpp b/libcxx/test/std/atomics/atomics.types.generic/atomics.types.float/fetch_sub.pass.cpp
index 00d43a61acc69..8784037aa5e82 100644
--- a/libcxx/test/std/atomics/atomics.types.generic/atomics.types.float/fetch_sub.pass.cpp
+++ b/libcxx/test/std/atomics/atomics.types.generic/atomics.types.float/fetch_sub.pass.cpp
@@ -10,7 +10,7 @@
 
 // Older versions of clang have a bug with atomic builtins affecting double and long double.
 // Fixed by 5fdd0948.
-// XFAIL: target=powerpc-ibm-{{.*}} && (clang-17 || clang-18)
+// XFAIL: target=powerpc-ibm-{{.*}} && clang-18
 
 // https://github.com/llvm/llvm-project/issues/72893
 // XFAIL: target={{x86_64-.*}} && tsan
diff --git a/libcxx/test/std/atomics/atomics.types.generic/atomics.types.float/operator.minus_equals.pass.cpp b/libcxx/test/std/atomics/atomics.types.generic/atomics.types.float/operator.minus_equals.pass.cpp
index 5dddb7c5472e1..e0e079436075f 100644
--- a/libcxx/test/std/atomics/atomics.types.generic/atomics.types.float/operator.minus_equals.pass.cpp
+++ b/libcxx/test/std/atomics/atomics.types.generic/atomics.types.float/operator.minus_equals.pass.cpp
@@ -10,7 +10,7 @@
 
 // Older versions of clang have a bug with atomic builtins affecting double and long double.
 // Fixed by 5fdd0948.
-// XFAIL: target=powerpc-ibm-{{.*}} && (clang-17 || clang-18)
+// XFAIL: target=powerpc-ibm-{{.*}} && clang-18
 
 // floating-point-type operator-=(floating-point-type) volatile noexcept;
 // floating-point-type operator-=(floating-point-type) noexcept;
diff --git a/libcxx/test/std/atomics/atomics.types.generic/atomics.types.float/operator.plus_equals.pass.cpp b/libcxx/test/std/atomics/atomics.types.generic/atomics.types.float/operator.plus_equals.pass.cpp
index cf7b494a3a800..7e2c10106e9ab 100644
--- a/libcxx/test/std/atomics/atomics.types.generic/atomics.types.float/operator.plus_equals.pass.cpp
+++ b/libcxx/test/std/atomics/atomics.types.generic/atomics.types.float/operator.plus_equals.pass.cpp
@@ -10,7 +10,7 @@
 
 // Older versions of clang have a bug with atomic builtins affecting double and long double.
 // Fixed by 5fdd0948.
-// XFAIL: target=powerpc-ibm-{{.*}} && (clang-17 || clang-18)
+// XFAIL: target=powerpc-ibm-{{.*}} && clang-18
 
 // floating-point-type operator+=(floating-point-type) volatile noexcept;
 // floating-point-type operator+=(floating-point-type) noexcept;
diff --git a/libcxx/test/std/containers/Copyable.h b/libcxx/test/std/containers/Copyable.h
index 4aa5a97afe6b0..984cd6ac00d79 100644
--- a/libcxx/test/std/containers/Copyable.h
+++ b/libcxx/test/std/containers/Copyable.h
@@ -9,8 +9,7 @@
 #ifndef COPYABLE_H
 #define COPYABLE_H
 
-class Copyable
-{
+class Copyable {
 public:
 };
 
diff --git a/libcxx/test/std/containers/Emplaceable.h b/libcxx/test/std/containers/Emplaceable.h
index 1a4e14505fb21..afc4e2e38b0eb 100644
--- a/libcxx/test/std/containers/Emplaceable.h
+++ b/libcxx/test/std/containers/Emplaceable.h
@@ -14,30 +14,32 @@
 
 #if TEST_STD_VER >= 11
 
-class Emplaceable
-{
-    Emplaceable(const Emplaceable&);
-    Emplaceable& operator=(const Emplaceable&);
+class Emplaceable {
+  Emplaceable(const Emplaceable&);
+  Emplaceable& operator=(const Emplaceable&);
+
+  int int_;
+  double double_;
 
-    int int_;
-    double double_;
 public:
-    Emplaceable() : int_(0), double_(0) {}
-    Emplaceable(int i, double d) : int_(i), double_(d) {}
-    Emplaceable(Emplaceable&& x)
-        : int_(x.int_), double_(x.double_)
-            {x.int_ = 0; x.double_ = 0;}
-    Emplaceable& operator=(Emplaceable&& x)
-        {int_ = x.int_; x.int_ = 0;
-         double_ = x.double_; x.double_ = 0;
-         return *this;}
-
-    bool operator==(const Emplaceable& x) const
-        {return int_ == x.int_ && double_ == x.double_;}
-    bool operator<(const Emplaceable& x) const
-        {return int_ < x.int_ || (int_ == x.int_ && double_ < x.double_);}
-
-    int get() const {return int_;}
+  Emplaceable() : int_(0), double_(0) {}
+  Emplaceable(int i, double d) : int_(i), double_(d) {}
+  Emplaceable(Emplaceable&& x) : int_(x.int_), double_(x.double_) {
+    x.int_    = 0;
+    x.double_ = 0;
+  }
+  Emplaceable& operator=(Emplaceable&& x) {
+    int_      = x.int_;
+    x.int_    = 0;
+    double_   = x.double_;
+    x.double_ = 0;
+    return *this;
+  }
+
+  bool operator==(const Emplaceable& x) const { return int_ == x.int_ && double_ == x.double_; }
+  bool operator<(const Emplaceable& x) const { return int_ < x.int_ || (int_ == x.int_ && double_ < x.double_); }
+
+  int get() const { return int_; }
 };
 
 template <>
@@ -45,7 +47,7 @@ struct std::hash<Emplaceable> {
   typedef Emplaceable argument_type;
   typedef std::size_t result_type;
 
-  std::size_t operator()(const Emplaceable& x) const { return x.get(); }
+  std::size_t operator()(const Emplaceable& x) const { return static_cast<std::size_t>(x.get()); }
 };
 
 #endif // TEST_STD_VER >= 11
diff --git a/libcxx/test/std/containers/NotConstructible.h b/libcxx/test/std/containers/NotConstructible.h
index 213aa1442d5c2..8c74be44de21b 100644
--- a/libcxx/test/std/containers/NotConstructible.h
+++ b/libcxx/test/std/containers/NotConstructible.h
@@ -11,17 +11,14 @@
 
 #include <functional>
 
-class NotConstructible
-{
-    NotConstructible(const NotConstructible&);
-    NotConstructible& operator=(const NotConstructible&);
+class NotConstructible {
+  NotConstructible(const NotConstructible&);
+  NotConstructible& operator=(const NotConstructible&);
+
 public:
 };
 
-inline
-bool
-operator==(const NotConstructible&, const NotConstructible&)
-{return true;}
+inline bool operator==(const NotConstructible&, const NotConstructible&) { return true; }
 
 template <>
 struct std::hash<NotConstructible> {
diff --git a/libcxx/test/std/containers/associative/from_range_associative_containers.h b/libcxx/test/std/containers/associative/from_range_associative_containers.h
index d227c41f6d84a..cb3646d738968 100644
--- a/libcxx/test/std/containers/associative/from_range_associative_containers.h
+++ b/libcxx/test/std/containers/associative/from_range_associative_containers.h
@@ -25,11 +25,13 @@
 #include "test_macros.h"
 
 template <class Container, class Range>
-concept HasFromRangeCtr = requires (Range&& range) {
+concept HasFromRangeCtr = requires(Range&& range) {
   Container(std::from_range, std::forward<Range>(range));
   Container(std::from_range, std::forward<Range>(range), std::less<typename Container::key_type>());
-  Container(std::from_range, std::forward<Range>(range), std::less<typename Container::key_type>(),
-      std::allocator<typename Container::value_type>());
+  Container(std::from_range,
+            std::forward<Range>(range),
+            std::less<typename Container::key_type>(),
+            std::allocator<typename Container::value_type>());
   Container(std::from_range, std::forward<Range>(range), std::allocator<typename Container::value_type>());
 };
 
@@ -50,7 +52,7 @@ constexpr bool test_map_constraints() {
   return true;
 }
 
-template <template <class ...> class Container,
+template <template <class...> class Container,
           class K,
           class V,
           class Iter,
@@ -100,13 +102,7 @@ void test_associative_map_with_input(std::vector<ValueType>&& input) {
   }
 }
 
-template <template <class ...> class Container,
-          class K,
-          class V,
-          class Iter,
-          class Sent,
-          class Comp,
-          class Alloc>
+template <template <class...> class Container, class K, class V, class Iter, class Sent, class Comp, class Alloc>
 void test_associative_map() {
   auto test_with_input = &test_associative_map_with_input<Container, K, V, Iter, Sent, Comp, Alloc>;
 
@@ -118,7 +114,7 @@ void test_associative_map() {
   test_with_input({{1, 2}});
 }
 
-template <template <class ...> class Container>
+template <template <class...> class Container>
 void test_associative_map_move_only() {
   std::pair<const int, MoveOnly> input[5];
   std::ranges::subrange in(std::move_iterator{input}, std::move_iterator{input + 5});
@@ -126,17 +122,15 @@ void test_associative_map_move_only() {
   [[maybe_unused]] Container<int, MoveOnly> c(std::from_range, in);
 }
 
-template <template <class ...> class Container>
+template <template <class...> class Container>
 void test_map_exception_safety_throwing_copy() {
 #if !defined(TEST_HAS_NO_EXCEPTIONS)
   using K = int;
   using V = ThrowingCopy<3>;
 
-  V::throwing_enabled = false;
-  std::pair<const K, V> in[5] = {
-    {1, {}}, {2, {}}, {3, {}}, {4, {}}, {5, {}}
-  };
-  V::throwing_enabled = true;
+  V::throwing_enabled         = false;
+  std::pair<const K, V> in[5] = {{1, {}}, {2, {}}, {3, {}}, {4, {}}, {5, {}}};
+  V::throwing_enabled         = true;
   V::reset();
 
   try {
@@ -150,13 +144,11 @@ void test_map_exception_safety_throwing_copy() {
 #endif
 }
 
-template <template <class ...> class Container, class K, class V>
+template <template <class...> class Container, class K, class V>
 void test_map_exception_safety_throwing_allocator() {
 #if !defined(TEST_HAS_NO_EXCEPTIONS)
   using ValueType = std::pair<const K, V>;
-  ValueType in[] = {
-    ValueType{K{1}, V{1}}
-  };
+  ValueType in[]  = {ValueType{K{1}, V{1}}};
 
   try {
     ThrowingAllocator<ValueType> alloc;
@@ -172,11 +164,13 @@ void test_map_exception_safety_throwing_allocator() {
 }
 
 template <class Container, class Range>
-concept SetHasFromRangeCtr = requires (Range&& range) {
+concept SetHasFromRangeCtr = requires(Range&& range) {
   Container(std::from_range, std::forward<Range>(range));
   Container(std::from_range, std::forward<Range>(range), std::less<typename Container::value_type>());
-  Container(std::from_range, std::forward<Range>(range), std::less<typename Container::value_type>(),
-      std::allocator<typename Container::value_type>());
+  Container(std::from_range,
+            std::forward<Range>(range),
+            std::less<typename Container::value_type>(),
+            std::allocator<typename Container::value_type>());
   Container(std::from_range, std::forward<Range>(range), std::allocator<typename Container::value_type>());
 };
 
@@ -194,12 +188,7 @@ constexpr bool test_set_constraints() {
   return true;
 }
 
-template <template <class ...> class Container,
-          class T,
-          class Iter,
-          class Sent,
-          class Comp,
-          class Alloc>
+template <template <class...> class Container, class T, class Iter, class Sent, class Comp, class Alloc>
 void test_associative_set_with_input(std::vector<T>&& input) {
   { // (range)
     std::ranges::subrange in(Iter(input.data()), Sent(Iter(input.data() + input.size())));
@@ -242,12 +231,7 @@ void test_associative_set_with_input(std::vector<T>&& input) {
   }
 }
 
-template <template <class ...> class Container,
-          class T,
-          class Iter,
-          class Sent,
-          class Comp,
-          class Alloc>
+template <template <class...> class Container, class T, class Iter, class Sent, class Comp, class Alloc>
 void test_associative_set() {
   auto test_with_input = &test_associative_set_with_input<Container, T, Iter, Sent, Comp, Alloc>;
 
@@ -259,7 +243,7 @@ void test_associative_set() {
   test_with_input({5});
 }
 
-template <template <class ...> class Container>
+template <template <class...> class Container>
 void test_associative_set_move_only() {
   MoveOnly input[5];
   std::ranges::subrange in(std::move_iterator{input}, std::move_iterator{input + 5});
@@ -267,7 +251,7 @@ void test_associative_set_move_only() {
   [[maybe_unused]] Container<MoveOnly> c(std::from_range, in);
 }
 
-template <template <class ...> class Container>
+template <template <class...> class Container>
 void test_set_exception_safety_throwing_copy() {
 #if !defined(TEST_HAS_NO_EXCEPTIONS)
   using T = ThrowingCopy<3>;
@@ -285,7 +269,7 @@ void test_set_exception_safety_throwing_copy() {
 #endif
 }
 
-template <template <class ...> class Container, class T>
+template <template <class...> class Container, class T>
 void test_set_exception_safety_throwing_allocator() {
 #if !defined(TEST_HAS_NO_EXCEPTIONS)
   T in[] = {1, 2};
diff --git a/libcxx/test/std/containers/associative/iterator_types.pass.cpp b/libcxx/test/std/containers/associative/iterator_types.pass.cpp
index 1b8556fb558aa..2800549905f1b 100644
--- a/libcxx/test/std/containers/associative/iterator_types.pass.cpp
+++ b/libcxx/test/std/containers/associative/iterator_types.pass.cpp
@@ -14,7 +14,6 @@
 #include "min_allocator.h"
 #include "test_allocator.h"
 
-
 template <class Map, class ValueTp, class PtrT, class CPtrT>
 void testMap() {
   typedef typename Map::difference_type Diff;
@@ -34,11 +33,9 @@ void testMap() {
   }
 }
 
-
 template <class Set, class ValueTp, class CPtrT>
 void testSet() {
-  static_assert((std::is_same<typename Set::iterator,
-                             typename Set::const_iterator>::value), "");
+  static_assert((std::is_same<typename Set::iterator, typename Set::const_iterator>::value), "");
   typedef typename Set::difference_type Diff;
   {
     typedef typename Set::iterator It;
@@ -46,7 +43,6 @@ void testSet() {
     static_assert((std::is_same<typename It::reference, ValueTp const&>::value), "");
     static_assert((std::is_same<typename It::pointer, CPtrT>::value), "");
     static_assert((std::is_same<typename It::difference_type, Diff>::value), "");
-
   }
 }
 
diff --git a/libcxx/test/std/containers/associative/map/PR28469_undefined_behavior_segfault.sh.cpp b/libcxx/test/std/containers/associative/map/PR28469_undefined_behavior_segfault.sh.cpp
index 584b829e36aa1..a72ad8187e7e9 100644
--- a/libcxx/test/std/containers/associative/map/PR28469_undefined_behavior_segfault.sh.cpp
+++ b/libcxx/test/std/containers/associative/map/PR28469_undefined_behavior_segfault.sh.cpp
@@ -22,13 +22,13 @@
 void dummy() {}
 
 struct F {
-    std::map<int, std::function<void()> > m;
-    F() { m[42] = &dummy; }
+  std::map<int, std::function<void()> > m;
+  F() { m[42] = &dummy; }
 };
 
 int main(int, char**) {
-    F f;
-    f = F();
+  F f;
+  f = F();
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/associative/map/allocator_mismatch.verify.cpp b/libcxx/test/std/containers/associative/map/allocator_mismatch.verify.cpp
index 1a72c6327977f..48bdaf2390ff7 100644
--- a/libcxx/test/std/containers/associative/map/allocator_mismatch.verify.cpp
+++ b/libcxx/test/std/containers/associative/map/allocator_mismatch.verify.cpp
@@ -12,4 +12,4 @@
 #include <map>
 
 std::map<int, int, std::less<int>, std::allocator<long> > m;
-  // expected-error-re@*:* {{static assertion failed{{.*}}Allocator::value_type must be same type as value_type}}
+// expected-error-re@*:* {{static assertion failed{{.*}}Allocator::value_type must be same type as value_type}}
diff --git a/libcxx/test/std/containers/associative/map/compare.pass.cpp b/libcxx/test/std/containers/associative/map/compare.pass.cpp
index 31221b234af5e..cb7a64b0409fc 100644
--- a/libcxx/test/std/containers/associative/map/compare.pass.cpp
+++ b/libcxx/test/std/containers/associative/map/compare.pass.cpp
@@ -22,34 +22,34 @@
 #include "test_macros.h"
 
 struct Key {
-  template <typename T> Key(const T&) {}
-  bool operator< (const Key&) const { return false; }
+  template <typename T>
+  Key(const T&) {}
+  bool operator<(const Key&) const { return false; }
 };
 
-int main(int, char**)
-{
-    typedef std::map<Key, int> MapT;
-    typedef MapT::iterator Iter;
-    typedef std::pair<Iter, bool> IterBool;
-    {
-        MapT m_empty;
-        MapT m_contains;
-        m_contains[Key(0)] = 42;
-
-        Iter it = m_empty.find(Key(0));
-        assert(it == m_empty.end());
-        it = m_contains.find(Key(0));
-        assert(it != m_contains.end());
-    }
-    {
-        MapT map;
-        IterBool result = map.insert(std::make_pair(Key(0), 42));
-        assert(result.second);
-        assert(result.first->second == 42);
-        IterBool result2 = map.insert(std::make_pair(Key(0), 43));
-        assert(!result2.second);
-        assert(map[Key(0)] == 42);
-    }
+int main(int, char**) {
+  typedef std::map<Key, int> MapT;
+  typedef MapT::iterator Iter;
+  typedef std::pair<Iter, bool> IterBool;
+  {
+    MapT m_empty;
+    MapT m_contains;
+    m_contains[Key(0)] = 42;
+
+    Iter it = m_empty.find(Key(0));
+    assert(it == m_empty.end());
+    it = m_contains.find(Key(0));
+    assert(it != m_contains.end());
+  }
+  {
+    MapT map;
+    IterBool result = map.insert(std::make_pair(Key(0), 42));
+    assert(result.second);
+    assert(result.first->second == 42);
+    IterBool result2 = map.insert(std::make_pair(Key(0), 43));
+    assert(!result2.second);
+    assert(map[Key(0)] == 42);
+  }
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/associative/map/get_allocator.pass.cpp b/libcxx/test/std/containers/associative/map/get_allocator.pass.cpp
index b214e44b41ba5..2dbd662d3b7ac 100644
--- a/libcxx/test/std/containers/associative/map/get_allocator.pass.cpp
+++ b/libcxx/test/std/containers/associative/map/get_allocator.pass.cpp
@@ -20,18 +20,17 @@
 #include "test_macros.h"
 
 int main(int, char**) {
-    typedef std::pair<const int, std::string> ValueType;
-    {
-        std::allocator<ValueType> alloc;
-        const std::map<int, std::string> m(alloc);
-        assert(m.get_allocator() == alloc);
-    }
-    {
-        other_allocator<ValueType> alloc(1);
-        const std::map<int, std::string, std::less<int>,
-                       other_allocator<ValueType> > m(alloc);
-        assert(m.get_allocator() == alloc);
-    }
+  typedef std::pair<const int, std::string> ValueType;
+  {
+    std::allocator<ValueType> alloc;
+    const std::map<int, std::string> m(alloc);
+    assert(m.get_allocator() == alloc);
+  }
+  {
+    other_allocator<ValueType> alloc(1);
+    const std::map<int, std::string, std::less<int>, other_allocator<ValueType> > m(alloc);
+    assert(m.get_allocator() == alloc);
+  }
 
-    return 0;
+  return 0;
 }
diff --git a/libcxx/test/std/containers/associative/map/incomplete_type.pass.cpp b/libcxx/test/std/containers/associative/map/incomplete_type.pass.cpp
index c663f8b618c3a..99fc612698094 100644
--- a/libcxx/test/std/containers/associative/map/incomplete_type.pass.cpp
+++ b/libcxx/test/std/containers/associative/map/incomplete_type.pass.cpp
@@ -16,17 +16,17 @@
 #include "test_macros.h"
 
 struct A {
-    typedef std::map<A, A> Map;
-    int data;
-    Map m;
-    Map::iterator it;
-    Map::const_iterator cit;
+  typedef std::map<A, A> Map;
+  int data;
+  Map m;
+  Map::iterator it;
+  Map::const_iterator cit;
 };
 
 inline bool operator==(A const& L, A const& R) { return &L == &R; }
-inline bool operator<(A const& L, A const& R)  { return L.data < R.data; }
+inline bool operator<(A const& L, A const& R) { return L.data < R.data; }
 int main(int, char**) {
-    A a;
+  A a;
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/associative/map/iterator_concept_conformance.compile.pass.cpp b/libcxx/test/std/containers/associative/map/iterator_concept_conformance.compile.pass.cpp
index 4676b2eb9561b..13cf025579497 100644
--- a/libcxx/test/std/containers/associative/map/iterator_concept_conformance.compile.pass.cpp
+++ b/libcxx/test/std/containers/associative/map/iterator_concept_conformance.compile.pass.cpp
@@ -14,11 +14,11 @@
 
 #include <iterator>
 
-using iterator = std::map<int, int>::iterator;
-using const_iterator = std::map<int, int>::const_iterator;
-using reverse_iterator = std::map<int, int>::reverse_iterator;
+using iterator               = std::map<int, int>::iterator;
+using const_iterator         = std::map<int, int>::const_iterator;
+using reverse_iterator       = std::map<int, int>::reverse_iterator;
 using const_reverse_iterator = std::map<int, int>::const_reverse_iterator;
-using value_type = std::pair<const int, int>;
+using value_type             = std::pair<const int, int>;
 
 static_assert(std::bidirectional_iterator<iterator>);
 static_assert(!std::random_access_iterator<iterator>);
@@ -31,9 +31,9 @@ static_assert(!std::sized_sentinel_for<iterator, iterator>);
 static_assert(!std::sized_sentinel_for<iterator, const_iterator>);
 static_assert(!std::sized_sentinel_for<iterator, reverse_iterator>);
 static_assert(!std::sized_sentinel_for<iterator, const_reverse_iterator>);
-static_assert( std::indirectly_movable<iterator, std::pair<int, int>*>);
+static_assert(std::indirectly_movable<iterator, std::pair<int, int>*>);
 static_assert(!std::indirectly_movable_storable<iterator, std::pair<int, int>*>);
-static_assert( std::indirectly_copyable<iterator, std::pair<int, int>*>);
+static_assert(std::indirectly_copyable<iterator, std::pair<int, int>*>);
 static_assert(!std::indirectly_copyable_storable<iterator, std::pair<int, int>*>);
 static_assert(!std::indirectly_swappable<iterator, iterator>);
 
diff --git a/libcxx/test/std/containers/associative/map/map.access/at.pass.cpp b/libcxx/test/std/containers/associative/map/map.access/at.pass.cpp
index c13c6a64acb20..cabc9f7eb1066 100644
--- a/libcxx/test/std/containers/associative/map/map.access/at.pass.cpp
+++ b/libcxx/test/std/containers/associative/map/map.access/at.pass.cpp
@@ -20,145 +20,128 @@
 #include "min_allocator.h"
 #include "test_macros.h"
 
-int main(int, char**)
-{
-    {
-        typedef std::pair<const int, double> V;
-        V ar[] =
-        {
-            V(1, 1.5),
-            V(2, 2.5),
-            V(3, 3.5),
-            V(4, 4.5),
-            V(5, 5.5),
-            V(7, 7.5),
-            V(8, 8.5),
-        };
-        std::map<int, double> m(ar, ar+sizeof(ar)/sizeof(ar[0]));
-        assert(m.size() == 7);
-        assert(m.at(1) == 1.5);
-        m.at(1) = -1.5;
-        assert(m.at(1) == -1.5);
-        assert(m.at(2) == 2.5);
-        assert(m.at(3) == 3.5);
-        assert(m.at(4) == 4.5);
-        assert(m.at(5) == 5.5);
+int main(int, char**) {
+  {
+    typedef std::pair<const int, double> V;
+    V ar[] = {
+        V(1, 1.5),
+        V(2, 2.5),
+        V(3, 3.5),
+        V(4, 4.5),
+        V(5, 5.5),
+        V(7, 7.5),
+        V(8, 8.5),
+    };
+    std::map<int, double> m(ar, ar + sizeof(ar) / sizeof(ar[0]));
+    assert(m.size() == 7);
+    assert(m.at(1) == 1.5);
+    m.at(1) = -1.5;
+    assert(m.at(1) == -1.5);
+    assert(m.at(2) == 2.5);
+    assert(m.at(3) == 3.5);
+    assert(m.at(4) == 4.5);
+    assert(m.at(5) == 5.5);
 #ifndef TEST_HAS_NO_EXCEPTIONS
-        try
-        {
-            TEST_IGNORE_NODISCARD m.at(6);
-            assert(false);
-        }
-        catch (std::out_of_range&)
-        {
-        }
-#endif
-        assert(m.at(7) == 7.5);
-        assert(m.at(8) == 8.5);
-        assert(m.size() == 7);
+    try {
+      TEST_IGNORE_NODISCARD m.at(6);
+      assert(false);
+    } catch (std::out_of_range&) {
     }
-    {
-        typedef std::pair<const int, double> V;
-        V ar[] =
-        {
-            V(1, 1.5),
-            V(2, 2.5),
-            V(3, 3.5),
-            V(4, 4.5),
-            V(5, 5.5),
-            V(7, 7.5),
-            V(8, 8.5),
-        };
-        const std::map<int, double> m(ar, ar+sizeof(ar)/sizeof(ar[0]));
-        assert(m.size() == 7);
-        assert(m.at(1) == 1.5);
-        assert(m.at(2) == 2.5);
-        assert(m.at(3) == 3.5);
-        assert(m.at(4) == 4.5);
-        assert(m.at(5) == 5.5);
-#ifndef TEST_HAS_NO_EXCEPTIONS
-        try
-        {
-            TEST_IGNORE_NODISCARD m.at(6);
-            assert(false);
-        }
-        catch (std::out_of_range&)
-        {
-        }
 #endif
-        assert(m.at(7) == 7.5);
-        assert(m.at(8) == 8.5);
-        assert(m.size() == 7);
-    }
-#if TEST_STD_VER >= 11
-    {
-        typedef std::pair<const int, double> V;
-        V ar[] =
-        {
-            V(1, 1.5),
-            V(2, 2.5),
-            V(3, 3.5),
-            V(4, 4.5),
-            V(5, 5.5),
-            V(7, 7.5),
-            V(8, 8.5),
-        };
-        std::map<int, double, std::less<int>, min_allocator<V>> m(ar, ar+sizeof(ar)/sizeof(ar[0]));
-        assert(m.size() == 7);
-        assert(m.at(1) == 1.5);
-        m.at(1) = -1.5;
-        assert(m.at(1) == -1.5);
-        assert(m.at(2) == 2.5);
-        assert(m.at(3) == 3.5);
-        assert(m.at(4) == 4.5);
-        assert(m.at(5) == 5.5);
+    assert(m.at(7) == 7.5);
+    assert(m.at(8) == 8.5);
+    assert(m.size() == 7);
+  }
+  {
+    typedef std::pair<const int, double> V;
+    V ar[] = {
+        V(1, 1.5),
+        V(2, 2.5),
+        V(3, 3.5),
+        V(4, 4.5),
+        V(5, 5.5),
+        V(7, 7.5),
+        V(8, 8.5),
+    };
+    const std::map<int, double> m(ar, ar + sizeof(ar) / sizeof(ar[0]));
+    assert(m.size() == 7);
+    assert(m.at(1) == 1.5);
+    assert(m.at(2) == 2.5);
+    assert(m.at(3) == 3.5);
+    assert(m.at(4) == 4.5);
+    assert(m.at(5) == 5.5);
 #ifndef TEST_HAS_NO_EXCEPTIONS
-        try
-        {
-            TEST_IGNORE_NODISCARD m.at(6);
-            assert(false);
-        }
-        catch (std::out_of_range&)
-        {
-        }
-#endif
-        assert(m.at(7) == 7.5);
-        assert(m.at(8) == 8.5);
-        assert(m.size() == 7);
+    try {
+      TEST_IGNORE_NODISCARD m.at(6);
+      assert(false);
+    } catch (std::out_of_range&) {
     }
-    {
-        typedef std::pair<const int, double> V;
-        V ar[] =
-        {
-            V(1, 1.5),
-            V(2, 2.5),
-            V(3, 3.5),
-            V(4, 4.5),
-            V(5, 5.5),
-            V(7, 7.5),
-            V(8, 8.5),
-        };
-        const std::map<int, double, std::less<int>, min_allocator<V>> m(ar, ar+sizeof(ar)/sizeof(ar[0]));
-        assert(m.size() == 7);
-        assert(m.at(1) == 1.5);
-        assert(m.at(2) == 2.5);
-        assert(m.at(3) == 3.5);
-        assert(m.at(4) == 4.5);
-        assert(m.at(5) == 5.5);
-#ifndef TEST_HAS_NO_EXCEPTIONS
-        try
-        {
-            TEST_IGNORE_NODISCARD m.at(6);
-            assert(false);
-        }
-        catch (std::out_of_range&)
-        {
-        }
 #endif
-        assert(m.at(7) == 7.5);
-        assert(m.at(8) == 8.5);
-        assert(m.size() == 7);
+    assert(m.at(7) == 7.5);
+    assert(m.at(8) == 8.5);
+    assert(m.size() == 7);
+  }
+#if TEST_STD_VER >= 11
+  {
+    typedef std::pair<const int, double> V;
+    V ar[] = {
+        V(1, 1.5),
+        V(2, 2.5),
+        V(3, 3.5),
+        V(4, 4.5),
+        V(5, 5.5),
+        V(7, 7.5),
+        V(8, 8.5),
+    };
+    std::map<int, double, std::less<int>, min_allocator<V>> m(ar, ar + sizeof(ar) / sizeof(ar[0]));
+    assert(m.size() == 7);
+    assert(m.at(1) == 1.5);
+    m.at(1) = -1.5;
+    assert(m.at(1) == -1.5);
+    assert(m.at(2) == 2.5);
+    assert(m.at(3) == 3.5);
+    assert(m.at(4) == 4.5);
+    assert(m.at(5) == 5.5);
+#  ifndef TEST_HAS_NO_EXCEPTIONS
+    try {
+      TEST_IGNORE_NODISCARD m.at(6);
+      assert(false);
+    } catch (std::out_of_range&) {
+    }
+#  endif
+    assert(m.at(7) == 7.5);
+    assert(m.at(8) == 8.5);
+    assert(m.size() == 7);
+  }
+  {
+    typedef std::pair<const int, double> V;
+    V ar[] = {
+        V(1, 1.5),
+        V(2, 2.5),
+        V(3, 3.5),
+        V(4, 4.5),
+        V(5, 5.5),
+        V(7, 7.5),
+        V(8, 8.5),
+    };
+    const std::map<int, double, std::less<int>, min_allocator<V>> m(ar, ar + sizeof(ar) / sizeof(ar[0]));
+    assert(m.size() == 7);
+    assert(m.at(1) == 1.5);
+    assert(m.at(2) == 2.5);
+    assert(m.at(3) == 3.5);
+    assert(m.at(4) == 4.5);
+    assert(m.at(5) == 5.5);
+#  ifndef TEST_HAS_NO_EXCEPTIONS
+    try {
+      TEST_IGNORE_NODISCARD m.at(6);
+      assert(false);
+    } catch (std::out_of_range&) {
     }
+#  endif
+    assert(m.at(7) == 7.5);
+    assert(m.at(8) == 8.5);
+    assert(m.size() == 7);
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/associative/map/map.access/empty.pass.cpp b/libcxx/test/std/containers/associative/map/map.access/empty.pass.cpp
index a4cd019ee4df3..21118eca69693 100644
--- a/libcxx/test/std/containers/associative/map/map.access/empty.pass.cpp
+++ b/libcxx/test/std/containers/associative/map/map.access/empty.pass.cpp
@@ -18,9 +18,8 @@
 #include "test_macros.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
+int main(int, char**) {
+  {
     typedef std::map<int, double> M;
     M m;
     assert(m.empty());
@@ -28,9 +27,9 @@ int main(int, char**)
     assert(!m.empty());
     m.clear();
     assert(m.empty());
-    }
+  }
 #if TEST_STD_VER >= 11
-    {
+  {
     typedef std::map<int, double, std::less<int>, min_allocator<std::pair<const int, double>>> M;
     M m;
     assert(m.empty());
@@ -38,7 +37,7 @@ int main(int, char**)
     assert(!m.empty());
     m.clear();
     assert(m.empty());
-    }
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/associative/map/map.access/empty.verify.cpp b/libcxx/test/std/containers/associative/map/map.access/empty.verify.cpp
index 27ca7c66ccebf..f44411df0bb00 100644
--- a/libcxx/test/std/containers/associative/map/map.access/empty.verify.cpp
+++ b/libcxx/test/std/containers/associative/map/map.access/empty.verify.cpp
@@ -17,6 +17,6 @@
 #include <map>
 
 void f() {
-    std::map<int, int> c;
-    c.empty(); // expected-warning {{ignoring return value of function declared with 'nodiscard' attribute}}
+  std::map<int, int> c;
+  c.empty(); // expected-warning {{ignoring return value of function declared with 'nodiscard' attribute}}
 }
diff --git a/libcxx/test/std/containers/associative/map/map.access/index_key.pass.cpp b/libcxx/test/std/containers/associative/map/map.access/index_key.pass.cpp
index eaa8e1d6d3848..58610c69ed3dc 100644
--- a/libcxx/test/std/containers/associative/map/map.access/index_key.pass.cpp
+++ b/libcxx/test/std/containers/associative/map/map.access/index_key.pass.cpp
@@ -20,15 +20,13 @@
 #include "min_allocator.h"
 #include "private_constructor.h"
 #if TEST_STD_VER >= 11
-#include "container_test_types.h"
+#  include "container_test_types.h"
 #endif
 
-int main(int, char**)
-{
-    {
+int main(int, char**) {
+  {
     typedef std::pair<const int, double> V;
-    V ar[] =
-    {
+    V ar[] = {
         V(1, 1.5),
         V(2, 2.5),
         V(3, 3.5),
@@ -37,7 +35,7 @@ int main(int, char**)
         V(7, 7.5),
         V(8, 8.5),
     };
-    std::map<int, double> m(ar, ar+sizeof(ar)/sizeof(ar[0]));
+    std::map<int, double> m(ar, ar + sizeof(ar) / sizeof(ar[0]));
     assert(m.size() == 7);
     assert(m[1] == 1.5);
     assert(m.size() == 7);
@@ -49,12 +47,11 @@ int main(int, char**)
     m[6] = 6.5;
     assert(m[6] == 6.5);
     assert(m.size() == 8);
-    }
+  }
 #if TEST_STD_VER >= 11
-    {
+  {
     typedef std::pair<const int, double> V;
-    V ar[] =
-    {
+    V ar[] = {
         V(1, 1.5),
         V(2, 2.5),
         V(3, 3.5),
@@ -63,12 +60,12 @@ int main(int, char**)
         V(7, 7.5),
         V(8, 8.5),
     };
-    std::map<int, double, std::less<int>, min_allocator<V>> m(ar, ar+sizeof(ar)/sizeof(ar[0]));
+    std::map<int, double, std::less<int>, min_allocator<V>> m(ar, ar + sizeof(ar) / sizeof(ar[0]));
     assert(m.size() == 7);
     assert(m[1] == 1.5);
     assert(m.size() == 7);
     const int i = 1;
-    m[i] = -1.5;
+    m[i]        = -1.5;
     assert(m[1] == -1.5);
     assert(m.size() == 7);
     assert(m[6] == 0);
@@ -76,46 +73,45 @@ int main(int, char**)
     m[6] = 6.5;
     assert(m[6] == 6.5);
     assert(m.size() == 8);
+  }
+  {
+    // Use "container_test_types.h" to check what arguments get passed
+    // to the allocator for operator[]
+    using Container         = TCT::map<>;
+    using Key               = Container::key_type;
+    using MappedType        = Container::mapped_type;
+    ConstructController* cc = getConstructController();
+    cc->reset();
+    {
+      Container c;
+      const Key k(1);
+      cc->expect<std::piecewise_construct_t const&, std::tuple<Key const&>&&, std::tuple<>&&>();
+      MappedType& mref = c[k];
+      assert(!cc->unchecked());
+      {
+        DisableAllocationGuard g;
+        MappedType& mref2 = c[k];
+        assert(&mref == &mref2);
+      }
     }
     {
-        // Use "container_test_types.h" to check what arguments get passed
-        // to the allocator for operator[]
-        using Container = TCT::map<>;
-        using Key = Container::key_type;
-        using MappedType = Container::mapped_type;
-        ConstructController* cc = getConstructController();
-        cc->reset();
-        {
-            Container c;
-            const Key k(1);
-            cc->expect<std::piecewise_construct_t const&, std::tuple<Key const&>&&, std::tuple<>&&>();
-            MappedType& mref = c[k];
-            assert(!cc->unchecked());
-            {
-                DisableAllocationGuard g;
-                MappedType& mref2 = c[k];
-                assert(&mref == &mref2);
-            }
-        }
-        {
-            Container c;
-            Key k(1);
-            cc->expect<std::piecewise_construct_t const&, std::tuple<Key const&>&&, std::tuple<>&&>();
-            MappedType& mref = c[k];
-            assert(!cc->unchecked());
-            {
-                DisableAllocationGuard g;
-                MappedType& mref2 = c[k];
-                assert(&mref == &mref2);
-            }
-        }
+      Container c;
+      Key k(1);
+      cc->expect<std::piecewise_construct_t const&, std::tuple<Key const&>&&, std::tuple<>&&>();
+      MappedType& mref = c[k];
+      assert(!cc->unchecked());
+      {
+        DisableAllocationGuard g;
+        MappedType& mref2 = c[k];
+        assert(&mref == &mref2);
+      }
     }
+  }
 #endif
 #if TEST_STD_VER > 11
-    {
+  {
     typedef std::pair<const int, double> V;
-    V ar[] =
-    {
+    V ar[] = {
         V(1, 1.5),
         V(2, 2.5),
         V(3, 3.5),
@@ -124,7 +120,7 @@ int main(int, char**)
         V(7, 7.5),
         V(8, 8.5),
     };
-    std::map<int, double, std::less<>> m(ar, ar+sizeof(ar)/sizeof(ar[0]));
+    std::map<int, double, std::less<>> m(ar, ar + sizeof(ar) / sizeof(ar[0]));
 
     assert(m.size() == 7);
     assert(m[1] == 1.5);
@@ -137,7 +133,7 @@ int main(int, char**)
     m[6] = 6.5;
     assert(m[6] == 6.5);
     assert(m.size() == 8);
-    }
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/associative/map/map.access/index_rv_key.pass.cpp b/libcxx/test/std/containers/associative/map/map.access/index_rv_key.pass.cpp
index 5e92e6d1b2b0c..d5d01b96a6988 100644
--- a/libcxx/test/std/containers/associative/map/map.access/index_rv_key.pass.cpp
+++ b/libcxx/test/std/containers/associative/map/map.access/index_rv_key.pass.cpp
@@ -23,9 +23,8 @@
 #include "min_allocator.h"
 #include "container_test_types.h"
 
-int main(int, char**)
-{
-    {
+int main(int, char**) {
+  {
     std::map<MoveOnly, double> m;
     assert(m.size() == 0);
     assert(m[1] == 0.0);
@@ -38,8 +37,8 @@ int main(int, char**)
     m[6] = 6.5;
     assert(m[6] == 6.5);
     assert(m.size() == 2);
-    }
-    {
+  }
+  {
     typedef std::pair<const MoveOnly, double> V;
     std::map<MoveOnly, double, std::less<MoveOnly>, min_allocator<V>> m;
     assert(m.size() == 0);
@@ -53,29 +52,29 @@ int main(int, char**)
     m[6] = 6.5;
     assert(m[6] == 6.5);
     assert(m.size() == 2);
-    }
+  }
+  {
+    // Use "container_test_types.h" to check what arguments get passed
+    // to the allocator for operator[]
+    using Container         = TCT::map<>;
+    using Key               = Container::key_type;
+    using MappedType        = Container::mapped_type;
+    ConstructController* cc = getConstructController();
+    cc->reset();
     {
-        // Use "container_test_types.h" to check what arguments get passed
-        // to the allocator for operator[]
-        using Container = TCT::map<>;
-        using Key = Container::key_type;
-        using MappedType = Container::mapped_type;
-        ConstructController* cc = getConstructController();
-        cc->reset();
-        {
-            Container c;
-            Key k(1);
-            cc->expect<std::piecewise_construct_t const&, std::tuple<Key &&>&&, std::tuple<>&&>();
-            MappedType& mref = c[std::move(k)];
-            assert(!cc->unchecked());
-            {
-                Key k2(1);
-                DisableAllocationGuard g;
-                MappedType& mref2 = c[std::move(k2)];
-                assert(&mref == &mref2);
-            }
-        }
+      Container c;
+      Key k(1);
+      cc->expect<std::piecewise_construct_t const&, std::tuple<Key&&>&&, std::tuple<>&&>();
+      MappedType& mref = c[std::move(k)];
+      assert(!cc->unchecked());
+      {
+        Key k2(1);
+        DisableAllocationGuard g;
+        MappedType& mref2 = c[std::move(k2)];
+        assert(&mref == &mref2);
+      }
     }
+  }
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/associative/map/map.access/index_tuple.pass.cpp b/libcxx/test/std/containers/associative/map/map.access/index_tuple.pass.cpp
index cf75e1ebb570f..3f33cfd699151 100644
--- a/libcxx/test/std/containers/associative/map/map.access/index_tuple.pass.cpp
+++ b/libcxx/test/std/containers/associative/map/map.access/index_tuple.pass.cpp
@@ -20,8 +20,8 @@
 #include <tuple>
 
 int main(int, char**) {
-    std::map<std::tuple<int, int>, std::size_t> m;
-    m[std::make_tuple(2, 3)] = 7;
+  std::map<std::tuple<int, int>, std::size_t> m;
+  m[std::make_tuple(2, 3)] = 7;
 
-    return 0;
+  return 0;
 }
diff --git a/libcxx/test/std/containers/associative/map/map.access/iterator.pass.cpp b/libcxx/test/std/containers/associative/map/map.access/iterator.pass.cpp
index 4589f93fda1e1..bbad44c8edb30 100644
--- a/libcxx/test/std/containers/associative/map/map.access/iterator.pass.cpp
+++ b/libcxx/test/std/containers/associative/map/map.access/iterator.pass.cpp
@@ -32,234 +32,129 @@
 #include "test_macros.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef std::pair<const int, double> V;
-        V ar[] =
-        {
-            V(1, 1),
-            V(1, 1.5),
-            V(1, 2),
-            V(2, 1),
-            V(2, 1.5),
-            V(2, 2),
-            V(3, 1),
-            V(3, 1.5),
-            V(3, 2),
-            V(4, 1),
-            V(4, 1.5),
-            V(4, 2),
-            V(5, 1),
-            V(5, 1.5),
-            V(5, 2),
-            V(6, 1),
-            V(6, 1.5),
-            V(6, 2),
-            V(7, 1),
-            V(7, 1.5),
-            V(7, 2),
-            V(8, 1),
-            V(8, 1.5),
-            V(8, 2)
-        };
-        std::map<int, double> m(ar, ar+sizeof(ar)/sizeof(ar[0]));
-        assert(static_cast<std::size_t>(std::distance(m.begin(), m.end())) == m.size());
-        assert(static_cast<std::size_t>(std::distance(m.rbegin(), m.rend())) == m.size());
-        std::map<int, double>::iterator i;
-        i = m.begin();
-        std::map<int, double>::const_iterator k = i;
-        assert(i == k);
-        for (int j = 1; static_cast<std::size_t>(j) <= m.size(); ++j, ++i)
-        {
-            assert(i->first == j);
-            assert(i->second == 1);
-            i->second = 2.5;
-            assert(i->second == 2.5);
-        }
-        assert(i == m.end());
-        for (int j = m.size(); j >= 1; --j)
-        {
-            --i;
-            assert(i->first == j);
-            assert(i->second == 2.5);
-            i->second = 1;
-            assert(i->second == 1);
-        }
-        assert(i == m.begin());
+int main(int, char**) {
+  {
+    typedef std::pair<const int, double> V;
+    V ar[] = {V(1, 1),   V(1, 1.5), V(1, 2),   V(2, 1),   V(2, 1.5), V(2, 2),   V(3, 1),   V(3, 1.5),
+              V(3, 2),   V(4, 1),   V(4, 1.5), V(4, 2),   V(5, 1),   V(5, 1.5), V(5, 2),   V(6, 1),
+              V(6, 1.5), V(6, 2),   V(7, 1),   V(7, 1.5), V(7, 2),   V(8, 1),   V(8, 1.5), V(8, 2)};
+    std::map<int, double> m(ar, ar + sizeof(ar) / sizeof(ar[0]));
+    assert(static_cast<std::size_t>(std::distance(m.begin(), m.end())) == m.size());
+    assert(static_cast<std::size_t>(std::distance(m.rbegin(), m.rend())) == m.size());
+    std::map<int, double>::iterator i;
+    i                                       = m.begin();
+    std::map<int, double>::const_iterator k = i;
+    assert(i == k);
+    for (int j = 1; static_cast<std::size_t>(j) <= m.size(); ++j, ++i) {
+      assert(i->first == j);
+      assert(i->second == 1);
+      i->second = 2.5;
+      assert(i->second == 2.5);
     }
-    {
-        typedef std::pair<const int, double> V;
-        V ar[] =
-        {
-            V(1, 1),
-            V(1, 1.5),
-            V(1, 2),
-            V(2, 1),
-            V(2, 1.5),
-            V(2, 2),
-            V(3, 1),
-            V(3, 1.5),
-            V(3, 2),
-            V(4, 1),
-            V(4, 1.5),
-            V(4, 2),
-            V(5, 1),
-            V(5, 1.5),
-            V(5, 2),
-            V(6, 1),
-            V(6, 1.5),
-            V(6, 2),
-            V(7, 1),
-            V(7, 1.5),
-            V(7, 2),
-            V(8, 1),
-            V(8, 1.5),
-            V(8, 2)
-        };
-        const std::map<int, double> m(ar, ar+sizeof(ar)/sizeof(ar[0]));
-        assert(static_cast<std::size_t>(std::distance(m.begin(), m.end())) == m.size());
-        assert(static_cast<std::size_t>(std::distance(m.cbegin(), m.cend())) == m.size());
-        assert(static_cast<std::size_t>(std::distance(m.rbegin(), m.rend())) == m.size());
-        assert(static_cast<std::size_t>(std::distance(m.crbegin(), m.crend())) == m.size());
-        std::map<int, double>::const_iterator i;
-        i = m.begin();
-        for (int j = 1; static_cast<std::size_t>(j) <= m.size(); ++j, ++i)
-        {
-            assert(i->first == j);
-            assert(i->second == 1);
-        }
-        assert(i == m.end());
-        for (int j = m.size(); j >= 1; --j)
-        {
-            --i;
-            assert(i->first == j);
-            assert(i->second == 1);
-        }
-        assert(i == m.begin());
+    assert(i == m.end());
+    for (int j = m.size(); j >= 1; --j) {
+      --i;
+      assert(i->first == j);
+      assert(i->second == 2.5);
+      i->second = 1;
+      assert(i->second == 1);
     }
+    assert(i == m.begin());
+  }
+  {
+    typedef std::pair<const int, double> V;
+    V ar[] = {V(1, 1),   V(1, 1.5), V(1, 2),   V(2, 1),   V(2, 1.5), V(2, 2),   V(3, 1),   V(3, 1.5),
+              V(3, 2),   V(4, 1),   V(4, 1.5), V(4, 2),   V(5, 1),   V(5, 1.5), V(5, 2),   V(6, 1),
+              V(6, 1.5), V(6, 2),   V(7, 1),   V(7, 1.5), V(7, 2),   V(8, 1),   V(8, 1.5), V(8, 2)};
+    const std::map<int, double> m(ar, ar + sizeof(ar) / sizeof(ar[0]));
+    assert(static_cast<std::size_t>(std::distance(m.begin(), m.end())) == m.size());
+    assert(static_cast<std::size_t>(std::distance(m.cbegin(), m.cend())) == m.size());
+    assert(static_cast<std::size_t>(std::distance(m.rbegin(), m.rend())) == m.size());
+    assert(static_cast<std::size_t>(std::distance(m.crbegin(), m.crend())) == m.size());
+    std::map<int, double>::const_iterator i;
+    i = m.begin();
+    for (int j = 1; static_cast<std::size_t>(j) <= m.size(); ++j, ++i) {
+      assert(i->first == j);
+      assert(i->second == 1);
+    }
+    assert(i == m.end());
+    for (int j = m.size(); j >= 1; --j) {
+      --i;
+      assert(i->first == j);
+      assert(i->second == 1);
+    }
+    assert(i == m.begin());
+  }
 #if TEST_STD_VER >= 11
-    {
-        typedef std::pair<const int, double> V;
-        V ar[] =
-        {
-            V(1, 1),
-            V(1, 1.5),
-            V(1, 2),
-            V(2, 1),
-            V(2, 1.5),
-            V(2, 2),
-            V(3, 1),
-            V(3, 1.5),
-            V(3, 2),
-            V(4, 1),
-            V(4, 1.5),
-            V(4, 2),
-            V(5, 1),
-            V(5, 1.5),
-            V(5, 2),
-            V(6, 1),
-            V(6, 1.5),
-            V(6, 2),
-            V(7, 1),
-            V(7, 1.5),
-            V(7, 2),
-            V(8, 1),
-            V(8, 1.5),
-            V(8, 2)
-        };
-        std::map<int, double, std::less<int>, min_allocator<V>> m(ar, ar+sizeof(ar)/sizeof(ar[0]));
-        assert(static_cast<std::size_t>(std::distance(m.begin(), m.end())) == m.size());
-        assert(static_cast<std::size_t>(std::distance(m.rbegin(), m.rend())) == m.size());
-        std::map<int, double, std::less<int>, min_allocator<V>>::iterator i;
-        i = m.begin();
-        std::map<int, double, std::less<int>, min_allocator<V>>::const_iterator k = i;
-        assert(i == k);
-        for (int j = 1; static_cast<std::size_t>(j) <= m.size(); ++j, ++i)
-        {
-            assert(i->first == j);
-            assert(i->second == 1);
-            i->second = 2.5;
-            assert(i->second == 2.5);
-        }
-        assert(i == m.end());
-        for (int j = m.size(); j >= 1; --j)
-        {
-            --i;
-            assert(i->first == j);
-            assert(i->second == 2.5);
-            i->second = 1;
-            assert(i->second == 1);
-        }
-        assert(i == m.begin());
+  {
+    typedef std::pair<const int, double> V;
+    V ar[] = {V(1, 1),   V(1, 1.5), V(1, 2),   V(2, 1),   V(2, 1.5), V(2, 2),   V(3, 1),   V(3, 1.5),
+              V(3, 2),   V(4, 1),   V(4, 1.5), V(4, 2),   V(5, 1),   V(5, 1.5), V(5, 2),   V(6, 1),
+              V(6, 1.5), V(6, 2),   V(7, 1),   V(7, 1.5), V(7, 2),   V(8, 1),   V(8, 1.5), V(8, 2)};
+    std::map<int, double, std::less<int>, min_allocator<V>> m(ar, ar + sizeof(ar) / sizeof(ar[0]));
+    assert(static_cast<std::size_t>(std::distance(m.begin(), m.end())) == m.size());
+    assert(static_cast<std::size_t>(std::distance(m.rbegin(), m.rend())) == m.size());
+    std::map<int, double, std::less<int>, min_allocator<V>>::iterator i;
+    i                                                                         = m.begin();
+    std::map<int, double, std::less<int>, min_allocator<V>>::const_iterator k = i;
+    assert(i == k);
+    for (int j = 1; static_cast<std::size_t>(j) <= m.size(); ++j, ++i) {
+      assert(i->first == j);
+      assert(i->second == 1);
+      i->second = 2.5;
+      assert(i->second == 2.5);
+    }
+    assert(i == m.end());
+    for (int j = m.size(); j >= 1; --j) {
+      --i;
+      assert(i->first == j);
+      assert(i->second == 2.5);
+      i->second = 1;
+      assert(i->second == 1);
     }
-    {
-        typedef std::pair<const int, double> V;
-        V ar[] =
-        {
-            V(1, 1),
-            V(1, 1.5),
-            V(1, 2),
-            V(2, 1),
-            V(2, 1.5),
-            V(2, 2),
-            V(3, 1),
-            V(3, 1.5),
-            V(3, 2),
-            V(4, 1),
-            V(4, 1.5),
-            V(4, 2),
-            V(5, 1),
-            V(5, 1.5),
-            V(5, 2),
-            V(6, 1),
-            V(6, 1.5),
-            V(6, 2),
-            V(7, 1),
-            V(7, 1.5),
-            V(7, 2),
-            V(8, 1),
-            V(8, 1.5),
-            V(8, 2)
-        };
-        const std::map<int, double, std::less<int>, min_allocator<V>> m(ar, ar+sizeof(ar)/sizeof(ar[0]));
-        assert(static_cast<std::size_t>(std::distance(m.begin(), m.end())) == m.size());
-        assert(static_cast<std::size_t>(std::distance(m.cbegin(), m.cend())) == m.size());
-        assert(static_cast<std::size_t>(std::distance(m.rbegin(), m.rend())) == m.size());
-        assert(static_cast<std::size_t>(std::distance(m.crbegin(), m.crend())) == m.size());
-        std::map<int, double, std::less<int>, min_allocator<V>>::const_iterator i;
-        i = m.begin();
-        for (int j = 1; static_cast<std::size_t>(j) <= m.size(); ++j, ++i)
-        {
-            assert(i->first == j);
-            assert(i->second == 1);
-        }
-        assert(i == m.end());
-        for (int j = m.size(); j >= 1; --j)
-        {
-            --i;
-            assert(i->first == j);
-            assert(i->second == 1);
-        }
-        assert(i == m.begin());
+    assert(i == m.begin());
+  }
+  {
+    typedef std::pair<const int, double> V;
+    V ar[] = {V(1, 1),   V(1, 1.5), V(1, 2),   V(2, 1),   V(2, 1.5), V(2, 2),   V(3, 1),   V(3, 1.5),
+              V(3, 2),   V(4, 1),   V(4, 1.5), V(4, 2),   V(5, 1),   V(5, 1.5), V(5, 2),   V(6, 1),
+              V(6, 1.5), V(6, 2),   V(7, 1),   V(7, 1.5), V(7, 2),   V(8, 1),   V(8, 1.5), V(8, 2)};
+    const std::map<int, double, std::less<int>, min_allocator<V>> m(ar, ar + sizeof(ar) / sizeof(ar[0]));
+    assert(static_cast<std::size_t>(std::distance(m.begin(), m.end())) == m.size());
+    assert(static_cast<std::size_t>(std::distance(m.cbegin(), m.cend())) == m.size());
+    assert(static_cast<std::size_t>(std::distance(m.rbegin(), m.rend())) == m.size());
+    assert(static_cast<std::size_t>(std::distance(m.crbegin(), m.crend())) == m.size());
+    std::map<int, double, std::less<int>, min_allocator<V>>::const_iterator i;
+    i = m.begin();
+    for (int j = 1; static_cast<std::size_t>(j) <= m.size(); ++j, ++i) {
+      assert(i->first == j);
+      assert(i->second == 1);
     }
+    assert(i == m.end());
+    for (int j = m.size(); j >= 1; --j) {
+      --i;
+      assert(i->first == j);
+      assert(i->second == 1);
+    }
+    assert(i == m.begin());
+  }
 #endif
 #if TEST_STD_VER > 11
-    { // N3644 testing
-        typedef std::map<int, double> C;
-        C::iterator ii1{}, ii2{};
-        C::iterator ii4 = ii1;
-        C::const_iterator cii{};
-        assert ( ii1 == ii2 );
-        assert ( ii1 == ii4 );
+  { // N3644 testing
+    typedef std::map<int, double> C;
+    C::iterator ii1{}, ii2{};
+    C::iterator ii4 = ii1;
+    C::const_iterator cii{};
+    assert(ii1 == ii2);
+    assert(ii1 == ii4);
 
-        assert (!(ii1 != ii2 ));
+    assert(!(ii1 != ii2));
 
-        assert ( (ii1 == cii ));
-        assert ( (cii == ii1 ));
-        assert (!(ii1 != cii ));
-        assert (!(cii != ii1 ));
-    }
+    assert((ii1 == cii));
+    assert((cii == ii1));
+    assert(!(ii1 != cii));
+    assert(!(cii != ii1));
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/associative/map/map.access/max_size.pass.cpp b/libcxx/test/std/containers/associative/map/map.access/max_size.pass.cpp
index e10f20359bb18..1a5d35c2579d8 100644
--- a/libcxx/test/std/containers/associative/map/map.access/max_size.pass.cpp
+++ b/libcxx/test/std/containers/associative/map/map.access/max_size.pass.cpp
@@ -20,8 +20,7 @@
 #include "test_allocator.h"
 #include "test_macros.h"
 
-int main(int, char**)
-{
+int main(int, char**) {
   typedef std::pair<const int, int> KV;
   {
     typedef limited_allocator<KV, 10> A;
@@ -33,16 +32,14 @@ int main(int, char**)
   {
     typedef limited_allocator<KV, (std::size_t)-1> A;
     typedef std::map<int, int, std::less<int>, A> C;
-    const C::size_type max_dist =
-        static_cast<C::size_type>(std::numeric_limits<C::difference_type>::max());
+    const C::size_type max_dist = static_cast<C::size_type>(std::numeric_limits<C::difference_type>::max());
     C c;
     assert(c.max_size() <= max_dist);
     LIBCPP_ASSERT(c.max_size() == max_dist);
   }
   {
     typedef std::map<char, int> C;
-    const C::size_type max_dist =
-        static_cast<C::size_type>(std::numeric_limits<C::difference_type>::max());
+    const C::size_type max_dist = static_cast<C::size_type>(std::numeric_limits<C::difference_type>::max());
     C c;
     assert(c.max_size() <= max_dist);
     assert(c.max_size() <= alloc_max_size(c.get_allocator()));
diff --git a/libcxx/test/std/containers/associative/map/map.access/size.pass.cpp b/libcxx/test/std/containers/associative/map/map.access/size.pass.cpp
index 837f720ec84d4..23914f6b8ed16 100644
--- a/libcxx/test/std/containers/associative/map/map.access/size.pass.cpp
+++ b/libcxx/test/std/containers/associative/map/map.access/size.pass.cpp
@@ -18,9 +18,8 @@
 #include "test_macros.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
+int main(int, char**) {
+  {
     typedef std::map<int, double> M;
     M m;
     assert(m.size() == 0);
@@ -36,9 +35,9 @@ int main(int, char**)
     assert(m.size() == 1);
     m.erase(m.begin());
     assert(m.size() == 0);
-    }
+  }
 #if TEST_STD_VER >= 11
-    {
+  {
     typedef std::map<int, double, std::less<int>, min_allocator<std::pair<const int, double>>> M;
     M m;
     assert(m.size() == 0);
@@ -54,7 +53,7 @@ int main(int, char**)
     assert(m.size() == 1);
     m.erase(m.begin());
     assert(m.size() == 0);
-    }
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/associative/map/map.cons/alloc.pass.cpp b/libcxx/test/std/containers/associative/map/map.cons/alloc.pass.cpp
index 45e867109f8ef..52199204d7914 100644
--- a/libcxx/test/std/containers/associative/map/map.cons/alloc.pass.cpp
+++ b/libcxx/test/std/containers/associative/map/map.cons/alloc.pass.cpp
@@ -19,33 +19,32 @@
 #include "test_allocator.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
+int main(int, char**) {
+  {
     typedef std::less<int> C;
     typedef test_allocator<std::pair<const int, double> > A;
     std::map<int, double, C, A> m(A(5));
     assert(m.empty());
     assert(m.begin() == m.end());
     assert(m.get_allocator() == A(5));
-    }
+  }
 #if TEST_STD_VER >= 11
-    {
+  {
     typedef std::less<int> C;
     typedef min_allocator<std::pair<const int, double> > A;
     std::map<int, double, C, A> m(A{});
     assert(m.empty());
     assert(m.begin() == m.end());
     assert(m.get_allocator() == A());
-    }
-    {
+  }
+  {
     typedef std::less<int> C;
     typedef explicit_allocator<std::pair<const int, double> > A;
     std::map<int, double, C, A> m(A{});
     assert(m.empty());
     assert(m.begin() == m.end());
     assert(m.get_allocator() == A());
-    }
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/associative/map/map.cons/assign_initializer_list.pass.cpp b/libcxx/test/std/containers/associative/map/map.cons/assign_initializer_list.pass.cpp
index 2a557aa488d10..f237c9b56deee 100644
--- a/libcxx/test/std/containers/associative/map/map.cons/assign_initializer_list.pass.cpp
+++ b/libcxx/test/std/containers/associative/map/map.cons/assign_initializer_list.pass.cpp
@@ -24,55 +24,30 @@
 void test_basic() {
   {
     typedef std::pair<const int, double> V;
-    std::map<int, double> m =
-                            {
-                                {20, 1},
-                            };
-    m =
-                            {
-                                {1, 1},
-                                {1, 1.5},
-                                {1, 2},
-                                {2, 1},
-                                {2, 1.5},
-                                {2, 2},
-                                {3, 1},
-                                {3, 1.5},
-                                {3, 2}
-                            };
+    std::map<int, double> m = {
+        {20, 1},
+    };
+    m = {{1, 1}, {1, 1.5}, {1, 2}, {2, 1}, {2, 1.5}, {2, 2}, {3, 1}, {3, 1.5}, {3, 2}};
     assert(m.size() == 3);
     assert(std::distance(m.begin(), m.end()) == 3);
     assert(*m.begin() == V(1, 1));
     assert(*std::next(m.begin()) == V(2, 1));
     assert(*std::next(m.begin(), 2) == V(3, 1));
-    }
-    {
+  }
+  {
     typedef std::pair<const int, double> V;
-    std::map<int, double, std::less<int>, min_allocator<V>> m =
-                            {
-                                {20, 1},
-                            };
-    m =
-                            {
-                                {1, 1},
-                                {1, 1.5},
-                                {1, 2},
-                                {2, 1},
-                                {2, 1.5},
-                                {2, 2},
-                                {3, 1},
-                                {3, 1.5},
-                                {3, 2}
-                            };
+    std::map<int, double, std::less<int>, min_allocator<V>> m = {
+        {20, 1},
+    };
+    m = {{1, 1}, {1, 1.5}, {1, 2}, {2, 1}, {2, 1.5}, {2, 2}, {3, 1}, {3, 1.5}, {3, 2}};
     assert(m.size() == 3);
     assert(std::distance(m.begin(), m.end()) == 3);
     assert(*m.begin() == V(1, 1));
     assert(*std::next(m.begin()) == V(2, 1));
     assert(*std::next(m.begin(), 2) == V(3, 1));
-    }
+  }
 }
 
-
 void duplicate_keys_test() {
   test_allocator_statistics alloc_stats;
   typedef std::map<int, int, std::less<int>, test_allocator<std::pair<const int, int> > > Map;
@@ -88,8 +63,7 @@ void duplicate_keys_test() {
   LIBCPP_ASSERT(alloc_stats.alloc_count == 0);
 }
 
-int main(int, char**)
-{
+int main(int, char**) {
   test_basic();
   duplicate_keys_test();
 
diff --git a/libcxx/test/std/containers/associative/map/map.cons/compare.pass.cpp b/libcxx/test/std/containers/associative/map/map.cons/compare.pass.cpp
index 04702f319771e..bd27a2f4ddeb9 100644
--- a/libcxx/test/std/containers/associative/map/map.cons/compare.pass.cpp
+++ b/libcxx/test/std/containers/associative/map/map.cons/compare.pass.cpp
@@ -19,23 +19,22 @@
 #include "../../../test_compare.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
+int main(int, char**) {
+  {
     typedef test_less<int> C;
     const std::map<int, double, C> m(C(3));
     assert(m.empty());
     assert(m.begin() == m.end());
     assert(m.key_comp() == C(3));
-    }
+  }
 #if TEST_STD_VER >= 11
-    {
+  {
     typedef test_less<int> C;
     const std::map<int, double, C, min_allocator<std::pair<const int, double>>> m(C(3));
     assert(m.empty());
     assert(m.begin() == m.end());
     assert(m.key_comp() == C(3));
-    }
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/associative/map/map.cons/compare_alloc.pass.cpp b/libcxx/test/std/containers/associative/map/map.cons/compare_alloc.pass.cpp
index 9d425e7218d64..20f6a08db6a36 100644
--- a/libcxx/test/std/containers/associative/map/map.cons/compare_alloc.pass.cpp
+++ b/libcxx/test/std/containers/associative/map/map.cons/compare_alloc.pass.cpp
@@ -20,9 +20,8 @@
 #include "test_allocator.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
+int main(int, char**) {
+  {
     typedef test_less<int> C;
     typedef test_allocator<std::pair<const int, double> > A;
     std::map<int, double, C, A> m(C(4), A(5));
@@ -30,9 +29,9 @@ int main(int, char**)
     assert(m.begin() == m.end());
     assert(m.key_comp() == C(4));
     assert(m.get_allocator() == A(5));
-    }
+  }
 #if TEST_STD_VER >= 11
-    {
+  {
     typedef test_less<int> C;
     typedef min_allocator<std::pair<const int, double> > A;
     std::map<int, double, C, A> m(C(4), A());
@@ -40,8 +39,8 @@ int main(int, char**)
     assert(m.begin() == m.end());
     assert(m.key_comp() == C(4));
     assert(m.get_allocator() == A());
-    }
-    {
+  }
+  {
     typedef test_less<int> C;
     typedef explicit_allocator<std::pair<const int, double> > A;
     std::map<int, double, C, A> m(C(4), A{});
@@ -49,7 +48,7 @@ int main(int, char**)
     assert(m.begin() == m.end());
     assert(m.key_comp() == C(4));
     assert(m.get_allocator() == A{});
-    }
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/associative/map/map.cons/compare_copy_constructible.compile.fail.cpp b/libcxx/test/std/containers/associative/map/map.cons/compare_copy_constructible.compile.fail.cpp
index 3c714de40af93..c81cb880e03aa 100644
--- a/libcxx/test/std/containers/associative/map/map.cons/compare_copy_constructible.compile.fail.cpp
+++ b/libcxx/test/std/containers/associative/map/map.cons/compare_copy_constructible.compile.fail.cpp
@@ -15,16 +15,16 @@
 
 template <class T>
 struct Comp {
-    bool operator () (const T& lhs, const T& rhs) const { return lhs < rhs; }
+  bool operator()(const T& lhs, const T& rhs) const { return lhs < rhs; }
 
-    Comp () {}
-private:
-    Comp (const Comp &); // declared but not defined
-    };
+  Comp() {}
 
+private:
+  Comp(const Comp&); // declared but not defined
+};
 
 int main(int, char**) {
-    std::map<int, int, Comp<int> > m;
+  std::map<int, int, Comp<int> > m;
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/associative/map/map.cons/copy.pass.cpp b/libcxx/test/std/containers/associative/map/map.cons/copy.pass.cpp
index 5899dec8b790e..e7a69aff0b036 100644
--- a/libcxx/test/std/containers/associative/map/map.cons/copy.pass.cpp
+++ b/libcxx/test/std/containers/associative/map/map.cons/copy.pass.cpp
@@ -20,111 +20,107 @@
 #include "test_allocator.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef std::pair<const int, double> V;
-        V ar[] =
-        {
-            V(1, 1),
-            V(1, 1.5),
-            V(1, 2),
-            V(2, 1),
-            V(2, 1.5),
-            V(2, 2),
-            V(3, 1),
-            V(3, 1.5),
-            V(3, 2),
-        };
-        typedef test_less<int> C;
-        typedef test_allocator<V> A;
-        std::map<int, double, C, A> mo(ar, ar+sizeof(ar)/sizeof(ar[0]), C(5), A(7));
-        std::map<int, double, C, A> m = mo;
-        assert(m.get_allocator() == A(7));
-        assert(m.key_comp() == C(5));
-        assert(m.size() == 3);
-        assert(std::distance(m.begin(), m.end()) == 3);
-        assert(*m.begin() == V(1, 1));
-        assert(*std::next(m.begin()) == V(2, 1));
-        assert(*std::next(m.begin(), 2) == V(3, 1));
+int main(int, char**) {
+  {
+    typedef std::pair<const int, double> V;
+    V ar[] = {
+        V(1, 1),
+        V(1, 1.5),
+        V(1, 2),
+        V(2, 1),
+        V(2, 1.5),
+        V(2, 2),
+        V(3, 1),
+        V(3, 1.5),
+        V(3, 2),
+    };
+    typedef test_less<int> C;
+    typedef test_allocator<V> A;
+    std::map<int, double, C, A> mo(ar, ar + sizeof(ar) / sizeof(ar[0]), C(5), A(7));
+    std::map<int, double, C, A> m = mo;
+    assert(m.get_allocator() == A(7));
+    assert(m.key_comp() == C(5));
+    assert(m.size() == 3);
+    assert(std::distance(m.begin(), m.end()) == 3);
+    assert(*m.begin() == V(1, 1));
+    assert(*std::next(m.begin()) == V(2, 1));
+    assert(*std::next(m.begin(), 2) == V(3, 1));
 
-        assert(mo.get_allocator() == A(7));
-        assert(mo.key_comp() == C(5));
-        assert(mo.size() == 3);
-        assert(std::distance(mo.begin(), mo.end()) == 3);
-        assert(*mo.begin() == V(1, 1));
-        assert(*std::next(mo.begin()) == V(2, 1));
-        assert(*std::next(mo.begin(), 2) == V(3, 1));
-    }
+    assert(mo.get_allocator() == A(7));
+    assert(mo.key_comp() == C(5));
+    assert(mo.size() == 3);
+    assert(std::distance(mo.begin(), mo.end()) == 3);
+    assert(*mo.begin() == V(1, 1));
+    assert(*std::next(mo.begin()) == V(2, 1));
+    assert(*std::next(mo.begin(), 2) == V(3, 1));
+  }
 #if TEST_STD_VER >= 11
-    {
-        typedef std::pair<const int, double> V;
-        V ar[] =
-        {
-            V(1, 1),
-            V(1, 1.5),
-            V(1, 2),
-            V(2, 1),
-            V(2, 1.5),
-            V(2, 2),
-            V(3, 1),
-            V(3, 1.5),
-            V(3, 2),
-        };
-        typedef test_less<int> C;
-        typedef other_allocator<V> A;
-        std::map<int, double, C, A> mo(ar, ar+sizeof(ar)/sizeof(ar[0]), C(5), A(7));
-        std::map<int, double, C, A> m = mo;
-        assert(m.get_allocator() == A(-2));
-        assert(m.key_comp() == C(5));
-        assert(m.size() == 3);
-        assert(std::distance(m.begin(), m.end()) == 3);
-        assert(*m.begin() == V(1, 1));
-        assert(*std::next(m.begin()) == V(2, 1));
-        assert(*std::next(m.begin(), 2) == V(3, 1));
+  {
+    typedef std::pair<const int, double> V;
+    V ar[] = {
+        V(1, 1),
+        V(1, 1.5),
+        V(1, 2),
+        V(2, 1),
+        V(2, 1.5),
+        V(2, 2),
+        V(3, 1),
+        V(3, 1.5),
+        V(3, 2),
+    };
+    typedef test_less<int> C;
+    typedef other_allocator<V> A;
+    std::map<int, double, C, A> mo(ar, ar + sizeof(ar) / sizeof(ar[0]), C(5), A(7));
+    std::map<int, double, C, A> m = mo;
+    assert(m.get_allocator() == A(-2));
+    assert(m.key_comp() == C(5));
+    assert(m.size() == 3);
+    assert(std::distance(m.begin(), m.end()) == 3);
+    assert(*m.begin() == V(1, 1));
+    assert(*std::next(m.begin()) == V(2, 1));
+    assert(*std::next(m.begin(), 2) == V(3, 1));
 
-        assert(mo.get_allocator() == A(7));
-        assert(mo.key_comp() == C(5));
-        assert(mo.size() == 3);
-        assert(std::distance(mo.begin(), mo.end()) == 3);
-        assert(*mo.begin() == V(1, 1));
-        assert(*std::next(mo.begin()) == V(2, 1));
-        assert(*std::next(mo.begin(), 2) == V(3, 1));
-    }
-    {
-        typedef std::pair<const int, double> V;
-        V ar[] =
-        {
-            V(1, 1),
-            V(1, 1.5),
-            V(1, 2),
-            V(2, 1),
-            V(2, 1.5),
-            V(2, 2),
-            V(3, 1),
-            V(3, 1.5),
-            V(3, 2),
-        };
-        typedef test_less<int> C;
-        typedef min_allocator<V> A;
-        std::map<int, double, C, A> mo(ar, ar+sizeof(ar)/sizeof(ar[0]), C(5), A());
-        std::map<int, double, C, A> m = mo;
-        assert(m.get_allocator() == A());
-        assert(m.key_comp() == C(5));
-        assert(m.size() == 3);
-        assert(std::distance(m.begin(), m.end()) == 3);
-        assert(*m.begin() == V(1, 1));
-        assert(*std::next(m.begin()) == V(2, 1));
-        assert(*std::next(m.begin(), 2) == V(3, 1));
+    assert(mo.get_allocator() == A(7));
+    assert(mo.key_comp() == C(5));
+    assert(mo.size() == 3);
+    assert(std::distance(mo.begin(), mo.end()) == 3);
+    assert(*mo.begin() == V(1, 1));
+    assert(*std::next(mo.begin()) == V(2, 1));
+    assert(*std::next(mo.begin(), 2) == V(3, 1));
+  }
+  {
+    typedef std::pair<const int, double> V;
+    V ar[] = {
+        V(1, 1),
+        V(1, 1.5),
+        V(1, 2),
+        V(2, 1),
+        V(2, 1.5),
+        V(2, 2),
+        V(3, 1),
+        V(3, 1.5),
+        V(3, 2),
+    };
+    typedef test_less<int> C;
+    typedef min_allocator<V> A;
+    std::map<int, double, C, A> mo(ar, ar + sizeof(ar) / sizeof(ar[0]), C(5), A());
+    std::map<int, double, C, A> m = mo;
+    assert(m.get_allocator() == A());
+    assert(m.key_comp() == C(5));
+    assert(m.size() == 3);
+    assert(std::distance(m.begin(), m.end()) == 3);
+    assert(*m.begin() == V(1, 1));
+    assert(*std::next(m.begin()) == V(2, 1));
+    assert(*std::next(m.begin(), 2) == V(3, 1));
 
-        assert(mo.get_allocator() == A());
-        assert(mo.key_comp() == C(5));
-        assert(mo.size() == 3);
-        assert(std::distance(mo.begin(), mo.end()) == 3);
-        assert(*mo.begin() == V(1, 1));
-        assert(*std::next(mo.begin()) == V(2, 1));
-        assert(*std::next(mo.begin(), 2) == V(3, 1));
-    }
+    assert(mo.get_allocator() == A());
+    assert(mo.key_comp() == C(5));
+    assert(mo.size() == 3);
+    assert(std::distance(mo.begin(), mo.end()) == 3);
+    assert(*mo.begin() == V(1, 1));
+    assert(*std::next(mo.begin()) == V(2, 1));
+    assert(*std::next(mo.begin(), 2) == V(3, 1));
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/associative/map/map.cons/copy_alloc.pass.cpp b/libcxx/test/std/containers/associative/map/map.cons/copy_alloc.pass.cpp
index 72402b1e987b8..aaf1be867b748 100644
--- a/libcxx/test/std/containers/associative/map/map.cons/copy_alloc.pass.cpp
+++ b/libcxx/test/std/containers/associative/map/map.cons/copy_alloc.pass.cpp
@@ -20,12 +20,10 @@
 #include "test_allocator.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
+int main(int, char**) {
+  {
     typedef std::pair<const int, double> V;
-    V ar[] =
-    {
+    V ar[] = {
         V(1, 1),
         V(1, 1.5),
         V(1, 2),
@@ -38,7 +36,7 @@ int main(int, char**)
     };
     typedef test_less<int> C;
     typedef test_allocator<V> A;
-    std::map<int, double, C, A> mo(ar, ar+sizeof(ar)/sizeof(ar[0]), C(5), A(7));
+    std::map<int, double, C, A> mo(ar, ar + sizeof(ar) / sizeof(ar[0]), C(5), A(7));
     std::map<int, double, C, A> m(mo, A(3));
     assert(m.get_allocator() == A(3));
     assert(m.key_comp() == C(5));
@@ -55,12 +53,11 @@ int main(int, char**)
     assert(*mo.begin() == V(1, 1));
     assert(*std::next(mo.begin()) == V(2, 1));
     assert(*std::next(mo.begin(), 2) == V(3, 1));
-    }
+  }
 #if TEST_STD_VER >= 11
-    {
+  {
     typedef std::pair<const int, double> V;
-    V ar[] =
-    {
+    V ar[] = {
         V(1, 1),
         V(1, 1.5),
         V(1, 2),
@@ -73,7 +70,7 @@ int main(int, char**)
     };
     typedef test_less<int> C;
     typedef min_allocator<V> A;
-    std::map<int, double, C, A> mo(ar, ar+sizeof(ar)/sizeof(ar[0]), C(5), A());
+    std::map<int, double, C, A> mo(ar, ar + sizeof(ar) / sizeof(ar[0]), C(5), A());
     std::map<int, double, C, A> m(mo, A());
     assert(m.get_allocator() == A());
     assert(m.key_comp() == C(5));
@@ -90,11 +87,10 @@ int main(int, char**)
     assert(*mo.begin() == V(1, 1));
     assert(*std::next(mo.begin()) == V(2, 1));
     assert(*std::next(mo.begin(), 2) == V(3, 1));
-    }
-    {
+  }
+  {
     typedef std::pair<const int, double> V;
-    V ar[] =
-    {
+    V ar[] = {
         V(1, 1),
         V(1, 1.5),
         V(1, 2),
@@ -107,7 +103,7 @@ int main(int, char**)
     };
     typedef test_less<int> C;
     typedef explicit_allocator<V> A;
-    std::map<int, double, C, A> mo(ar, ar+sizeof(ar)/sizeof(ar[0]), C(5), A{});
+    std::map<int, double, C, A> mo(ar, ar + sizeof(ar) / sizeof(ar[0]), C(5), A{});
     std::map<int, double, C, A> m(mo, A{});
     assert(m.get_allocator() == A());
     assert(m.key_comp() == C(5));
@@ -124,7 +120,7 @@ int main(int, char**)
     assert(*mo.begin() == V(1, 1));
     assert(*std::next(mo.begin()) == V(2, 1));
     assert(*std::next(mo.begin(), 2) == V(3, 1));
-    }
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/associative/map/map.cons/copy_assign.pass.cpp b/libcxx/test/std/containers/associative/map/map.cons/copy_assign.pass.cpp
index 91e5572bf1213..07c8bab2f201d 100644
--- a/libcxx/test/std/containers/associative/map/map.cons/copy_assign.pass.cpp
+++ b/libcxx/test/std/containers/associative/map/map.cons/copy_assign.pass.cpp
@@ -31,317 +31,266 @@ std::vector<int> ca_deallocs;
 template <class T>
 class counting_allocatorT {
 public:
-    typedef T value_type;
-    int foo{0};
-    counting_allocatorT(int f) noexcept : foo(f) {}
+  typedef T value_type;
+  int foo{0};
+  counting_allocatorT(int f) noexcept : foo(f) {}
 
-    using propagate_on_container_copy_assignment = std::true_type;
-    template <class U> counting_allocatorT(const counting_allocatorT<U>& other) noexcept {foo = other.foo;}
-    template <class U> bool operator==(const counting_allocatorT<U>& other) const noexcept { return foo == other.foo; }
-    template <class U> bool operator!=(const counting_allocatorT<U>& other) const noexcept { return foo != other.foo; }
+  using propagate_on_container_copy_assignment = std::true_type;
+  template <class U>
+  counting_allocatorT(const counting_allocatorT<U>& other) noexcept {
+    foo = other.foo;
+  }
+  template <class U>
+  bool operator==(const counting_allocatorT<U>& other) const noexcept {
+    return foo == other.foo;
+  }
+  template <class U>
+  bool operator!=(const counting_allocatorT<U>& other) const noexcept {
+    return foo != other.foo;
+  }
 
-    T* allocate(std::size_t n) const {
-        ca_allocs.push_back(foo);
-        void * const pv = ::malloc(n * sizeof(T));
-        return static_cast<T *>(pv);
-    }
-    void deallocate(T* p, std::size_t) const noexcept {
-        ca_deallocs.push_back(foo);
-        free(p);
-    }
+  T* allocate(std::size_t n) const {
+    ca_allocs.push_back(foo);
+    void* const pv = ::malloc(n * sizeof(T));
+    return static_cast<T*>(pv);
+  }
+  void deallocate(T* p, std::size_t) const noexcept {
+    ca_deallocs.push_back(foo);
+    free(p);
+  }
 };
 
 template <class T>
 class counting_allocatorF {
 public:
-    typedef T value_type;
-    int foo{0};
-    counting_allocatorF(int f) noexcept : foo(f) {}
+  typedef T value_type;
+  int foo{0};
+  counting_allocatorF(int f) noexcept : foo(f) {}
 
-    using propagate_on_container_copy_assignment = std::false_type;
-    template <class U> counting_allocatorF(const counting_allocatorF<U>& other) noexcept {foo = other.foo;}
-    template <class U> bool operator==(const counting_allocatorF<U>& other) const noexcept { return foo == other.foo; }
-    template <class U> bool operator!=(const counting_allocatorF<U>& other) const noexcept { return foo != other.foo; }
+  using propagate_on_container_copy_assignment = std::false_type;
+  template <class U>
+  counting_allocatorF(const counting_allocatorF<U>& other) noexcept {
+    foo = other.foo;
+  }
+  template <class U>
+  bool operator==(const counting_allocatorF<U>& other) const noexcept {
+    return foo == other.foo;
+  }
+  template <class U>
+  bool operator!=(const counting_allocatorF<U>& other) const noexcept {
+    return foo != other.foo;
+  }
 
-    T* allocate(std::size_t n) const {
-        ca_allocs.push_back(foo);
-        void * const pv = ::malloc(n * sizeof(T));
-        return static_cast<T *>(pv);
-    }
-    void deallocate(T* p, std::size_t) const noexcept {
-        ca_deallocs.push_back(foo);
-        free(p);
-    }
+  T* allocate(std::size_t n) const {
+    ca_allocs.push_back(foo);
+    void* const pv = ::malloc(n * sizeof(T));
+    return static_cast<T*>(pv);
+  }
+  void deallocate(T* p, std::size_t) const noexcept {
+    ca_deallocs.push_back(foo);
+    free(p);
+  }
 };
 
 bool balanced_allocs() {
-    std::vector<int> temp1, temp2;
+  std::vector<int> temp1, temp2;
 
-    std::printf("Allocations = %zu, deallocations = %zu\n", ca_allocs.size(),
-                ca_deallocs.size());
-    if (ca_allocs.size() != ca_deallocs.size())
-        return false;
+  std::printf("Allocations = %zu, deallocations = %zu\n", ca_allocs.size(), ca_deallocs.size());
+  if (ca_allocs.size() != ca_deallocs.size())
+    return false;
 
-    temp1 = ca_allocs;
-    std::sort(temp1.begin(), temp1.end());
-    temp2.clear();
-    std::unique_copy(temp1.begin(), temp1.end(), std::back_inserter<std::vector<int>>(temp2));
-    std::printf("There were %zu different allocators\n", temp2.size());
+  temp1 = ca_allocs;
+  std::sort(temp1.begin(), temp1.end());
+  temp2.clear();
+  std::unique_copy(temp1.begin(), temp1.end(), std::back_inserter<std::vector<int>>(temp2));
+  std::printf("There were %zu different allocators\n", temp2.size());
 
-    for (std::vector<int>::const_iterator it = temp2.begin(); it != temp2.end(); ++it ) {
-        std::ptrdiff_t const allocs = std::count(ca_allocs.begin(), ca_allocs.end(), *it);
-        std::ptrdiff_t const deallocs = std::count(ca_deallocs.begin(), ca_deallocs.end(), *it);
-        std::printf("%d: %td vs %td\n", *it, allocs, deallocs);
-        if (allocs != deallocs)
-            return false;
-    }
+  for (std::vector<int>::const_iterator it = temp2.begin(); it != temp2.end(); ++it) {
+    std::ptrdiff_t const allocs   = std::count(ca_allocs.begin(), ca_allocs.end(), *it);
+    std::ptrdiff_t const deallocs = std::count(ca_deallocs.begin(), ca_deallocs.end(), *it);
+    std::printf("%d: %td vs %td\n", *it, allocs, deallocs);
+    if (allocs != deallocs)
+      return false;
+  }
 
-    temp1 = ca_allocs;
-    std::sort(temp1.begin(), temp1.end());
-    temp2.clear();
-    std::unique_copy(temp1.begin(), temp1.end(), std::back_inserter<std::vector<int>>(temp2));
-    std::printf("There were %zu different (de)allocators\n", temp2.size());
+  temp1 = ca_allocs;
+  std::sort(temp1.begin(), temp1.end());
+  temp2.clear();
+  std::unique_copy(temp1.begin(), temp1.end(), std::back_inserter<std::vector<int>>(temp2));
+  std::printf("There were %zu different (de)allocators\n", temp2.size());
 
-    for (std::vector<int>::const_iterator it = ca_deallocs.begin(); it != ca_deallocs.end(); ++it ) {
-        std::ptrdiff_t const allocs = std::count(ca_allocs.begin(), ca_allocs.end(), *it);
-        std::ptrdiff_t const deallocs = std::count(ca_deallocs.begin(), ca_deallocs.end(), *it);
-        std::printf("%d: %td vs %td\n", *it, allocs, deallocs);
-        if (allocs != deallocs)
-            return false;
-    }
+  for (std::vector<int>::const_iterator it = ca_deallocs.begin(); it != ca_deallocs.end(); ++it) {
+    std::ptrdiff_t const allocs   = std::count(ca_allocs.begin(), ca_allocs.end(), *it);
+    std::ptrdiff_t const deallocs = std::count(ca_deallocs.begin(), ca_deallocs.end(), *it);
+    std::printf("%d: %td vs %td\n", *it, allocs, deallocs);
+    if (allocs != deallocs)
+      return false;
+  }
 
-    return true;
+  return true;
 }
 #endif
 
-int main(int, char**)
-{
-    {
-        typedef std::pair<const int, double> V;
-        V ar[] =
-        {
-            V(1, 1),
-            V(1, 1.5),
-            V(1, 2),
-            V(2, 1),
-            V(2, 1.5),
-            V(2, 2),
-            V(3, 1),
-            V(3, 1.5),
-            V(3, 2)
-        };
-        typedef test_less<int> C;
-        typedef test_allocator<V> A;
-        std::map<int, double, C, A> mo(ar, ar+sizeof(ar)/sizeof(ar[0]), C(5), A(2));
-        std::map<int, double, C, A> m(ar, ar+sizeof(ar)/sizeof(ar[0])/2, C(3), A(7));
-        m = mo;
-        assert(m.get_allocator() == A(7));
-        assert(m.key_comp() == C(5));
-        assert(m.size() == 3);
-        assert(std::distance(m.begin(), m.end()) == 3);
-        assert(*m.begin() == V(1, 1));
-        assert(*std::next(m.begin()) == V(2, 1));
-        assert(*std::next(m.begin(), 2) == V(3, 1));
+int main(int, char**) {
+  {
+    typedef std::pair<const int, double> V;
+    V ar[] = {V(1, 1), V(1, 1.5), V(1, 2), V(2, 1), V(2, 1.5), V(2, 2), V(3, 1), V(3, 1.5), V(3, 2)};
+    typedef test_less<int> C;
+    typedef test_allocator<V> A;
+    std::map<int, double, C, A> mo(ar, ar + sizeof(ar) / sizeof(ar[0]), C(5), A(2));
+    std::map<int, double, C, A> m(ar, ar + sizeof(ar) / sizeof(ar[0]) / 2, C(3), A(7));
+    m = mo;
+    assert(m.get_allocator() == A(7));
+    assert(m.key_comp() == C(5));
+    assert(m.size() == 3);
+    assert(std::distance(m.begin(), m.end()) == 3);
+    assert(*m.begin() == V(1, 1));
+    assert(*std::next(m.begin()) == V(2, 1));
+    assert(*std::next(m.begin(), 2) == V(3, 1));
 
-        assert(mo.get_allocator() == A(2));
-        assert(mo.key_comp() == C(5));
-        assert(mo.size() == 3);
-        assert(std::distance(mo.begin(), mo.end()) == 3);
-        assert(*mo.begin() == V(1, 1));
-        assert(*std::next(mo.begin()) == V(2, 1));
-        assert(*std::next(mo.begin(), 2) == V(3, 1));
-    }
-    {
-        typedef std::pair<const int, double> V;
-        const V ar[] =
-        {
-            V(1, 1),
-            V(2, 1),
-            V(3, 1),
-        };
-        std::map<int, double> m(ar, ar+sizeof(ar)/sizeof(ar[0]));
-        std::map<int, double> *p = &m;
-        m = *p;
+    assert(mo.get_allocator() == A(2));
+    assert(mo.key_comp() == C(5));
+    assert(mo.size() == 3);
+    assert(std::distance(mo.begin(), mo.end()) == 3);
+    assert(*mo.begin() == V(1, 1));
+    assert(*std::next(mo.begin()) == V(2, 1));
+    assert(*std::next(mo.begin(), 2) == V(3, 1));
+  }
+  {
+    typedef std::pair<const int, double> V;
+    const V ar[] = {
+        V(1, 1),
+        V(2, 1),
+        V(3, 1),
+    };
+    std::map<int, double> m(ar, ar + sizeof(ar) / sizeof(ar[0]));
+    std::map<int, double>* p = &m;
+    m                        = *p;
 
-        assert(m.size() == 3);
-        assert(std::equal(m.begin(), m.end(), ar));
-    }
-    {
-        typedef std::pair<const int, double> V;
-        V ar[] =
-        {
-            V(1, 1),
-            V(1, 1.5),
-            V(1, 2),
-            V(2, 1),
-            V(2, 1.5),
-            V(2, 2),
-            V(3, 1),
-            V(3, 1.5),
-            V(3, 2)
-        };
-        typedef test_less<int> C;
-        typedef other_allocator<V> A;
-        std::map<int, double, C, A> mo(ar, ar+sizeof(ar)/sizeof(ar[0]), C(5), A(2));
-        std::map<int, double, C, A> m(ar, ar+sizeof(ar)/sizeof(ar[0])/2, C(3), A(7));
-        m = mo;
-        assert(m.get_allocator() == A(2));
-        assert(m.key_comp() == C(5));
-        assert(m.size() == 3);
-        assert(std::distance(m.begin(), m.end()) == 3);
-        assert(*m.begin() == V(1, 1));
-        assert(*std::next(m.begin()) == V(2, 1));
-        assert(*std::next(m.begin(), 2) == V(3, 1));
+    assert(m.size() == 3);
+    assert(std::equal(m.begin(), m.end(), ar));
+  }
+  {
+    typedef std::pair<const int, double> V;
+    V ar[] = {V(1, 1), V(1, 1.5), V(1, 2), V(2, 1), V(2, 1.5), V(2, 2), V(3, 1), V(3, 1.5), V(3, 2)};
+    typedef test_less<int> C;
+    typedef other_allocator<V> A;
+    std::map<int, double, C, A> mo(ar, ar + sizeof(ar) / sizeof(ar[0]), C(5), A(2));
+    std::map<int, double, C, A> m(ar, ar + sizeof(ar) / sizeof(ar[0]) / 2, C(3), A(7));
+    m = mo;
+    assert(m.get_allocator() == A(2));
+    assert(m.key_comp() == C(5));
+    assert(m.size() == 3);
+    assert(std::distance(m.begin(), m.end()) == 3);
+    assert(*m.begin() == V(1, 1));
+    assert(*std::next(m.begin()) == V(2, 1));
+    assert(*std::next(m.begin(), 2) == V(3, 1));
 
-        assert(mo.get_allocator() == A(2));
-        assert(mo.key_comp() == C(5));
-        assert(mo.size() == 3);
-        assert(std::distance(mo.begin(), mo.end()) == 3);
-        assert(*mo.begin() == V(1, 1));
-        assert(*std::next(mo.begin()) == V(2, 1));
-        assert(*std::next(mo.begin(), 2) == V(3, 1));
-    }
+    assert(mo.get_allocator() == A(2));
+    assert(mo.key_comp() == C(5));
+    assert(mo.size() == 3);
+    assert(std::distance(mo.begin(), mo.end()) == 3);
+    assert(*mo.begin() == V(1, 1));
+    assert(*std::next(mo.begin()) == V(2, 1));
+    assert(*std::next(mo.begin(), 2) == V(3, 1));
+  }
 #if TEST_STD_VER >= 11
-    {
-        typedef std::pair<const int, double> V;
-        V ar[] =
-        {
-            V(1, 1),
-            V(1, 1.5),
-            V(1, 2),
-            V(2, 1),
-            V(2, 1.5),
-            V(2, 2),
-            V(3, 1),
-            V(3, 1.5),
-            V(3, 2)
-        };
-        typedef test_less<int> C;
-        typedef min_allocator<V> A;
-        std::map<int, double, C, A> mo(ar, ar+sizeof(ar)/sizeof(ar[0]), C(5), A());
-        std::map<int, double, C, A> m(ar, ar+sizeof(ar)/sizeof(ar[0])/2, C(3), A());
-        m = mo;
-        assert(m.get_allocator() == A());
-        assert(m.key_comp() == C(5));
-        assert(m.size() == 3);
-        assert(std::distance(m.begin(), m.end()) == 3);
-        assert(*m.begin() == V(1, 1));
-        assert(*std::next(m.begin()) == V(2, 1));
-        assert(*std::next(m.begin(), 2) == V(3, 1));
+  {
+    typedef std::pair<const int, double> V;
+    V ar[] = {V(1, 1), V(1, 1.5), V(1, 2), V(2, 1), V(2, 1.5), V(2, 2), V(3, 1), V(3, 1.5), V(3, 2)};
+    typedef test_less<int> C;
+    typedef min_allocator<V> A;
+    std::map<int, double, C, A> mo(ar, ar + sizeof(ar) / sizeof(ar[0]), C(5), A());
+    std::map<int, double, C, A> m(ar, ar + sizeof(ar) / sizeof(ar[0]) / 2, C(3), A());
+    m = mo;
+    assert(m.get_allocator() == A());
+    assert(m.key_comp() == C(5));
+    assert(m.size() == 3);
+    assert(std::distance(m.begin(), m.end()) == 3);
+    assert(*m.begin() == V(1, 1));
+    assert(*std::next(m.begin()) == V(2, 1));
+    assert(*std::next(m.begin(), 2) == V(3, 1));
 
-        assert(mo.get_allocator() == A());
-        assert(mo.key_comp() == C(5));
-        assert(mo.size() == 3);
-        assert(std::distance(mo.begin(), mo.end()) == 3);
-        assert(*mo.begin() == V(1, 1));
-        assert(*std::next(mo.begin()) == V(2, 1));
-        assert(*std::next(mo.begin(), 2) == V(3, 1));
-    }
-    {
-        typedef std::pair<const int, double> V;
-        V ar[] =
-        {
-            V(1, 1),
-            V(1, 1.5),
-            V(1, 2),
-            V(2, 1),
-            V(2, 1.5),
-            V(2, 2),
-            V(3, 1),
-            V(3, 1.5),
-            V(3, 2)
-        };
-        typedef test_less<int> C;
-        typedef min_allocator<V> A;
-        std::map<int, double, C, A> mo(ar, ar+sizeof(ar)/sizeof(ar[0]), C(5), A());
-        std::map<int, double, C, A> m(ar, ar+sizeof(ar)/sizeof(ar[0])/2, C(3), A());
-        m = mo;
-        assert(m.get_allocator() == A());
-        assert(m.key_comp() == C(5));
-        assert(m.size() == 3);
-        assert(std::distance(m.begin(), m.end()) == 3);
-        assert(*m.begin() == V(1, 1));
-        assert(*std::next(m.begin()) == V(2, 1));
-        assert(*std::next(m.begin(), 2) == V(3, 1));
+    assert(mo.get_allocator() == A());
+    assert(mo.key_comp() == C(5));
+    assert(mo.size() == 3);
+    assert(std::distance(mo.begin(), mo.end()) == 3);
+    assert(*mo.begin() == V(1, 1));
+    assert(*std::next(mo.begin()) == V(2, 1));
+    assert(*std::next(mo.begin(), 2) == V(3, 1));
+  }
+  {
+    typedef std::pair<const int, double> V;
+    V ar[] = {V(1, 1), V(1, 1.5), V(1, 2), V(2, 1), V(2, 1.5), V(2, 2), V(3, 1), V(3, 1.5), V(3, 2)};
+    typedef test_less<int> C;
+    typedef min_allocator<V> A;
+    std::map<int, double, C, A> mo(ar, ar + sizeof(ar) / sizeof(ar[0]), C(5), A());
+    std::map<int, double, C, A> m(ar, ar + sizeof(ar) / sizeof(ar[0]) / 2, C(3), A());
+    m = mo;
+    assert(m.get_allocator() == A());
+    assert(m.key_comp() == C(5));
+    assert(m.size() == 3);
+    assert(std::distance(m.begin(), m.end()) == 3);
+    assert(*m.begin() == V(1, 1));
+    assert(*std::next(m.begin()) == V(2, 1));
+    assert(*std::next(m.begin(), 2) == V(3, 1));
 
-        assert(mo.get_allocator() == A());
-        assert(mo.key_comp() == C(5));
-        assert(mo.size() == 3);
-        assert(std::distance(mo.begin(), mo.end()) == 3);
-        assert(*mo.begin() == V(1, 1));
-        assert(*std::next(mo.begin()) == V(2, 1));
-        assert(*std::next(mo.begin(), 2) == V(3, 1));
-    }
+    assert(mo.get_allocator() == A());
+    assert(mo.key_comp() == C(5));
+    assert(mo.size() == 3);
+    assert(std::distance(mo.begin(), mo.end()) == 3);
+    assert(*mo.begin() == V(1, 1));
+    assert(*std::next(mo.begin()) == V(2, 1));
+    assert(*std::next(mo.begin(), 2) == V(3, 1));
+  }
 
-    assert(balanced_allocs());
-    {
-        typedef std::pair<const int, double> V;
-        V ar[] =
-        {
-            V(1, 1),
-            V(1, 1.5),
-            V(1, 2),
-            V(2, 1),
-            V(2, 1.5),
-            V(2, 2),
-            V(3, 1),
-            V(3, 1.5),
-            V(3, 2)
-        };
-        typedef test_less<int> C;
-        typedef counting_allocatorT<V> A;
-        std::map<int, double, C, A> mo(ar, ar+sizeof(ar)/sizeof(ar[0]), C(5), A(1));
-        std::map<int, double, C, A> m(ar, ar+sizeof(ar)/sizeof(ar[0])/2, C(3), A(2));
-        m = mo;
-        assert(m.key_comp() == C(5));
-        assert(m.size() == 3);
-        assert(std::distance(m.begin(), m.end()) == 3);
-        assert(*m.begin() == V(1, 1));
-        assert(*std::next(m.begin()) == V(2, 1));
-        assert(*std::next(m.begin(), 2) == V(3, 1));
+  assert(balanced_allocs());
+  {
+    typedef std::pair<const int, double> V;
+    V ar[] = {V(1, 1), V(1, 1.5), V(1, 2), V(2, 1), V(2, 1.5), V(2, 2), V(3, 1), V(3, 1.5), V(3, 2)};
+    typedef test_less<int> C;
+    typedef counting_allocatorT<V> A;
+    std::map<int, double, C, A> mo(ar, ar + sizeof(ar) / sizeof(ar[0]), C(5), A(1));
+    std::map<int, double, C, A> m(ar, ar + sizeof(ar) / sizeof(ar[0]) / 2, C(3), A(2));
+    m = mo;
+    assert(m.key_comp() == C(5));
+    assert(m.size() == 3);
+    assert(std::distance(m.begin(), m.end()) == 3);
+    assert(*m.begin() == V(1, 1));
+    assert(*std::next(m.begin()) == V(2, 1));
+    assert(*std::next(m.begin(), 2) == V(3, 1));
 
-        assert(mo.key_comp() == C(5));
-        assert(mo.size() == 3);
-        assert(std::distance(mo.begin(), mo.end()) == 3);
-        assert(*mo.begin() == V(1, 1));
-        assert(*std::next(mo.begin()) == V(2, 1));
-        assert(*std::next(mo.begin(), 2) == V(3, 1));
-    }
-    assert(balanced_allocs());
-    {
-        typedef std::pair<const int, double> V;
-        V ar[] =
-        {
-            V(1, 1),
-            V(1, 1.5),
-            V(1, 2),
-            V(2, 1),
-            V(2, 1.5),
-            V(2, 2),
-            V(3, 1),
-            V(3, 1.5),
-            V(3, 2)
-        };
-        typedef test_less<int> C;
-        typedef counting_allocatorF<V> A;
-        std::map<int, double, C, A> mo(ar, ar+sizeof(ar)/sizeof(ar[0]), C(5), A(100));
-        std::map<int, double, C, A> m(ar, ar+sizeof(ar)/sizeof(ar[0])/2, C(3), A(200));
-        m = mo;
-        assert(m.key_comp() == C(5));
-        assert(m.size() == 3);
-        assert(std::distance(m.begin(), m.end()) == 3);
-        assert(*m.begin() == V(1, 1));
-        assert(*std::next(m.begin()) == V(2, 1));
-        assert(*std::next(m.begin(), 2) == V(3, 1));
+    assert(mo.key_comp() == C(5));
+    assert(mo.size() == 3);
+    assert(std::distance(mo.begin(), mo.end()) == 3);
+    assert(*mo.begin() == V(1, 1));
+    assert(*std::next(mo.begin()) == V(2, 1));
+    assert(*std::next(mo.begin(), 2) == V(3, 1));
+  }
+  assert(balanced_allocs());
+  {
+    typedef std::pair<const int, double> V;
+    V ar[] = {V(1, 1), V(1, 1.5), V(1, 2), V(2, 1), V(2, 1.5), V(2, 2), V(3, 1), V(3, 1.5), V(3, 2)};
+    typedef test_less<int> C;
+    typedef counting_allocatorF<V> A;
+    std::map<int, double, C, A> mo(ar, ar + sizeof(ar) / sizeof(ar[0]), C(5), A(100));
+    std::map<int, double, C, A> m(ar, ar + sizeof(ar) / sizeof(ar[0]) / 2, C(3), A(200));
+    m = mo;
+    assert(m.key_comp() == C(5));
+    assert(m.size() == 3);
+    assert(std::distance(m.begin(), m.end()) == 3);
+    assert(*m.begin() == V(1, 1));
+    assert(*std::next(m.begin()) == V(2, 1));
+    assert(*std::next(m.begin(), 2) == V(3, 1));
 
-        assert(mo.key_comp() == C(5));
-        assert(mo.size() == 3);
-        assert(std::distance(mo.begin(), mo.end()) == 3);
-        assert(*mo.begin() == V(1, 1));
-        assert(*std::next(mo.begin()) == V(2, 1));
-        assert(*std::next(mo.begin(), 2) == V(3, 1));
-    }
-    assert(balanced_allocs());
+    assert(mo.key_comp() == C(5));
+    assert(mo.size() == 3);
+    assert(std::distance(mo.begin(), mo.end()) == 3);
+    assert(*mo.begin() == V(1, 1));
+    assert(*std::next(mo.begin()) == V(2, 1));
+    assert(*std::next(mo.begin(), 2) == V(3, 1));
+  }
+  assert(balanced_allocs());
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/associative/map/map.cons/deduct.pass.cpp b/libcxx/test/std/containers/associative/map/map.cons/deduct.pass.cpp
index d68e6af2681a5..c91fe4b23566a 100644
--- a/libcxx/test/std/containers/associative/map/map.cons/deduct.pass.cpp
+++ b/libcxx/test/std/containers/associative/map/map.cons/deduct.pass.cpp
@@ -45,114 +45,114 @@
 #include "deduction_guides_sfinae_checks.h"
 #include "test_allocator.h"
 
-using P = std::pair<int, long>;
+using P  = std::pair<int, long>;
 using PC = std::pair<const int, long>;
 
-int main(int, char**)
-{
-    {
-    const P arr[] = { {1,1L}, {2,2L}, {1,1L}, {INT_MAX,1L}, {3,1L} };
+int main(int, char**) {
+  {
+    const P arr[] = {{1, 1L}, {2, 2L}, {1, 1L}, {INT_MAX, 1L}, {3, 1L}};
     std::map m(std::begin(arr), std::end(arr));
 
     ASSERT_SAME_TYPE(decltype(m), std::map<int, long>);
-    const PC expected_m[] = { {1,1L}, {2,2L}, {3,1L}, {INT_MAX,1L} };
+    const PC expected_m[] = {{1, 1L}, {2, 2L}, {3, 1L}, {INT_MAX, 1L}};
     assert(std::equal(m.begin(), m.end(), std::begin(expected_m), std::end(expected_m)));
-    }
+  }
 
-    {
-    const P arr[] = { {1,1L}, {2,2L}, {1,1L}, {INT_MAX,1L}, {3,1L} };
+  {
+    const P arr[] = {{1, 1L}, {2, 2L}, {1, 1L}, {INT_MAX, 1L}, {3, 1L}};
     std::map m(std::begin(arr), std::end(arr), std::greater<int>());
 
     ASSERT_SAME_TYPE(decltype(m), std::map<int, long, std::greater<int>>);
-    const PC expected_m[] = { {INT_MAX,1L}, {3,1L}, {2,2L}, {1,1L} };
+    const PC expected_m[] = {{INT_MAX, 1L}, {3, 1L}, {2, 2L}, {1, 1L}};
     assert(std::equal(m.begin(), m.end(), std::begin(expected_m), std::end(expected_m)));
-    }
+  }
 
-    {
-    const P arr[] = { {1,1L}, {2,2L}, {1,1L}, {INT_MAX,1L}, {3,1L} };
+  {
+    const P arr[] = {{1, 1L}, {2, 2L}, {1, 1L}, {INT_MAX, 1L}, {3, 1L}};
     std::map m(std::begin(arr), std::end(arr), std::greater<int>(), test_allocator<PC>(0, 42));
 
     ASSERT_SAME_TYPE(decltype(m), std::map<int, long, std::greater<int>, test_allocator<PC>>);
-    const PC expected_m[] = { {INT_MAX,1L}, {3,1L}, {2,2L}, {1,1L} };
+    const PC expected_m[] = {{INT_MAX, 1L}, {3, 1L}, {2, 2L}, {1, 1L}};
     assert(std::equal(m.begin(), m.end(), std::begin(expected_m), std::end(expected_m)));
     assert(m.get_allocator().get_id() == 42);
-    }
+  }
 
-    {
+  {
     std::map<int, long> source;
     std::map m(source);
     ASSERT_SAME_TYPE(decltype(m), decltype(source));
     assert(m.size() == 0);
-    }
+  }
 
-    {
+  {
     std::map<int, long> source;
-    std::map m{source};  // braces instead of parens
+    std::map m{source}; // braces instead of parens
     ASSERT_SAME_TYPE(decltype(m), decltype(source));
     assert(m.size() == 0);
-    }
+  }
 
-    {
+  {
     std::map<int, long> source;
     std::map m(source, std::map<int, long>::allocator_type());
     ASSERT_SAME_TYPE(decltype(m), decltype(source));
     assert(m.size() == 0);
-    }
+  }
 
-    {
-    std::map m{ P{1,1L}, P{2,2L}, P{1,1L}, P{INT_MAX,1L}, P{3,1L} };
+  {
+    std::map m{P{1, 1L}, P{2, 2L}, P{1, 1L}, P{INT_MAX, 1L}, P{3, 1L}};
 
     ASSERT_SAME_TYPE(decltype(m), std::map<int, long>);
-    const PC expected_m[] = { {1,1L}, {2,2L}, {3,1L}, {INT_MAX,1L} };
+    const PC expected_m[] = {{1, 1L}, {2, 2L}, {3, 1L}, {INT_MAX, 1L}};
     assert(std::equal(m.begin(), m.end(), std::begin(expected_m), std::end(expected_m)));
-    }
+  }
 
-    {
-    std::map m({ P{1,1L}, P{2,2L}, P{1,1L}, P{INT_MAX,1L}, P{3,1L} }, std::greater<int>());
+  {
+    std::map m({P{1, 1L}, P{2, 2L}, P{1, 1L}, P{INT_MAX, 1L}, P{3, 1L}}, std::greater<int>());
 
     ASSERT_SAME_TYPE(decltype(m), std::map<int, long, std::greater<int>>);
-    const PC expected_m[] = { {INT_MAX,1L}, {3,1L}, {2,2L}, {1,1L} };
+    const PC expected_m[] = {{INT_MAX, 1L}, {3, 1L}, {2, 2L}, {1, 1L}};
     assert(std::equal(m.begin(), m.end(), std::begin(expected_m), std::end(expected_m)));
-    }
+  }
 
-    {
-    std::map m({ P{1,1L}, P{2,2L}, P{1,1L}, P{INT_MAX,1L}, P{3,1L} }, std::greater<int>(), test_allocator<PC>(0, 43));
+  {
+    std::map m(
+        {P{1, 1L}, P{2, 2L}, P{1, 1L}, P{INT_MAX, 1L}, P{3, 1L}}, std::greater<int>(), test_allocator<PC>(0, 43));
 
     ASSERT_SAME_TYPE(decltype(m), std::map<int, long, std::greater<int>, test_allocator<PC>>);
-    const PC expected_m[] = { {INT_MAX,1L}, {3,1L}, {2,2L}, {1,1L} };
+    const PC expected_m[] = {{INT_MAX, 1L}, {3, 1L}, {2, 2L}, {1, 1L}};
     assert(std::equal(m.begin(), m.end(), std::begin(expected_m), std::end(expected_m)));
     assert(m.get_allocator().get_id() == 43);
-    }
+  }
 
-    {
-    const P arr[] = { {1,1L}, {2,2L}, {1,1L}, {INT_MAX,1L}, {3,1L} };
+  {
+    const P arr[] = {{1, 1L}, {2, 2L}, {1, 1L}, {INT_MAX, 1L}, {3, 1L}};
     std::map m(std::begin(arr), std::end(arr), test_allocator<PC>(0, 44));
 
     ASSERT_SAME_TYPE(decltype(m), std::map<int, long, std::less<int>, test_allocator<PC>>);
-    const PC expected_m[] = { {1,1L}, {2,2L}, {3,1L}, {INT_MAX,1L} };
+    const PC expected_m[] = {{1, 1L}, {2, 2L}, {3, 1L}, {INT_MAX, 1L}};
     assert(std::equal(m.begin(), m.end(), std::begin(expected_m), std::end(expected_m)));
     assert(m.get_allocator().get_id() == 44);
-    }
+  }
 
-    {
-    std::map m({ P{1,1L}, P{2,2L}, P{1,1L}, P{INT_MAX,1L}, P{3,1L} }, test_allocator<PC>(0, 45));
+  {
+    std::map m({P{1, 1L}, P{2, 2L}, P{1, 1L}, P{INT_MAX, 1L}, P{3, 1L}}, test_allocator<PC>(0, 45));
 
     ASSERT_SAME_TYPE(decltype(m), std::map<int, long, std::less<int>, test_allocator<PC>>);
-    const PC expected_m[] = { {1,1L}, {2,2L}, {3,1L}, {INT_MAX,1L} };
+    const PC expected_m[] = {{1, 1L}, {2, 2L}, {3, 1L}, {INT_MAX, 1L}};
     assert(std::equal(m.begin(), m.end(), std::begin(expected_m), std::end(expected_m)));
     assert(m.get_allocator().get_id() == 45);
-    }
+  }
 
-    {
+  {
     // Examples from LWG3025
     std::map m{std::pair{1, 1}, {2, 2}, {3, 3}};
     ASSERT_SAME_TYPE(decltype(m), std::map<int, int>);
 
     std::map m2{m.begin(), m.end()};
     ASSERT_SAME_TYPE(decltype(m2), std::map<int, int>);
-    }
+  }
 
-    {
+  {
     // Examples from LWG3531
     std::map m1{{std::pair{1, 2}, {3, 4}}, std::less<int>()};
     ASSERT_SAME_TYPE(decltype(m1), std::map<int, int>);
@@ -160,38 +160,38 @@ int main(int, char**)
     using value_type = std::pair<const int, int>;
     std::map m2{{value_type{1, 2}, {3, 4}}, std::less<int>()};
     ASSERT_SAME_TYPE(decltype(m2), std::map<int, int>);
-    }
+  }
 
 #if TEST_STD_VER >= 23
-    {
-      using Range = std::array<P, 0>;
-      using Comp = std::greater<int>;
-      using DefaultComp = std::less<int>;
-      using Alloc = test_allocator<PC>;
-
-      { // (from_range, range)
-        std::map c(std::from_range, Range());
-        static_assert(std::is_same_v<decltype(c), std::map<int, long>>);
-      }
-
-      { // (from_range, range, comp)
-        std::map c(std::from_range, Range(), Comp());
-        static_assert(std::is_same_v<decltype(c), std::map<int, long, Comp>>);
-      }
-
-      { // (from_range, range, comp, alloc)
-        std::map c(std::from_range, Range(), Comp(), Alloc());
-        static_assert(std::is_same_v<decltype(c), std::map<int, long, Comp, Alloc>>);
-      }
-
-      { // (from_range, range, alloc)
-        std::map c(std::from_range, Range(), Alloc());
-        static_assert(std::is_same_v<decltype(c), std::map<int, long, DefaultComp, Alloc>>);
-      }
+  {
+    using Range       = std::array<P, 0>;
+    using Comp        = std::greater<int>;
+    using DefaultComp = std::less<int>;
+    using Alloc       = test_allocator<PC>;
+
+    { // (from_range, range)
+      std::map c(std::from_range, Range());
+      static_assert(std::is_same_v<decltype(c), std::map<int, long>>);
+    }
+
+    { // (from_range, range, comp)
+      std::map c(std::from_range, Range(), Comp());
+      static_assert(std::is_same_v<decltype(c), std::map<int, long, Comp>>);
+    }
+
+    { // (from_range, range, comp, alloc)
+      std::map c(std::from_range, Range(), Comp(), Alloc());
+      static_assert(std::is_same_v<decltype(c), std::map<int, long, Comp, Alloc>>);
+    }
+
+    { // (from_range, range, alloc)
+      std::map c(std::from_range, Range(), Alloc());
+      static_assert(std::is_same_v<decltype(c), std::map<int, long, DefaultComp, Alloc>>);
     }
+  }
 #endif
 
-    AssociativeContainerDeductionGuidesSfinaeAway<std::map, std::map<int, long>>();
+  AssociativeContainerDeductionGuidesSfinaeAway<std::map, std::map<int, long>>();
 
-    return 0;
+  return 0;
 }
diff --git a/libcxx/test/std/containers/associative/map/map.cons/deduct.verify.cpp b/libcxx/test/std/containers/associative/map/map.cons/deduct.verify.cpp
index b314e6fba6964..c8c0e1e538db2 100644
--- a/libcxx/test/std/containers/associative/map/map.cons/deduct.verify.cpp
+++ b/libcxx/test/std/containers/associative/map/map.cons/deduct.verify.cpp
@@ -32,74 +32,75 @@
 #include <type_traits>
 
 struct NotAnAllocator {
-    friend bool operator<(NotAnAllocator, NotAnAllocator) { return false; }
+  friend bool operator<(NotAnAllocator, NotAnAllocator) { return false; }
 };
 
-using P = std::pair<int, long>;
+using P  = std::pair<int, long>;
 using PC = std::pair<const int, long>;
 
-int main(int, char**)
-{
-    {
-        // cannot deduce Key and T from nothing
-        std::map m; // expected-error-re {{no viable constructor or deduction guide for deduction of template arguments of '{{(std::)?}}map'}}
-    }
-    {
-        // cannot deduce Key and T from just (Compare)
-        std::map m(std::less<int>{});
-            // expected-error-re@-1{{no viable constructor or deduction guide for deduction of template arguments of '{{(std::)?}}map'}}
-    }
-    {
-        // cannot deduce Key and T from just (Compare, Allocator)
-        std::map m(std::less<int>{}, std::allocator<PC>{});
-            // expected-error-re@-1{{no viable constructor or deduction guide for deduction of template arguments of '{{(std::)?}}map'}}
-    }
-    {
-        // cannot deduce Key and T from just (Allocator)
-        std::map m(std::allocator<PC>{});
-            // expected-error-re@-1{{no viable constructor or deduction guide for deduction of template arguments of '{{(std::)?}}map'}}
-    }
-    {
-        // refuse to rebind the allocator if Allocator::value_type is not exactly what we expect
-        const P arr[] = { {1,1L}, {2,2L}, {3,3L} };
-        std::map m(arr, arr + 3, std::allocator<P>());
-            // expected-error-re@map:*{{static assertion failed{{( due to requirement '.*')?}}{{.*}}Allocator::value_type must be same type as value_type}}
-    }
-    {
-        // cannot convert from some arbitrary unrelated type
-        NotAnAllocator a;
-        std::map m(a); // expected-error-re{{no viable constructor or deduction guide for deduction of template arguments of '{{(std::)?}}map'}}
-    }
-    {
-        // cannot deduce that the inner braced things should be std::pair and not something else
-        std::map m{ {1,1L}, {2,2L}, {3,3L} };
-            // expected-error-re@-1{{no viable constructor or deduction guide for deduction of template arguments of '{{(std::)?}}map'}}
-    }
-    {
-        // cannot deduce that the inner braced things should be std::pair and not something else
-        std::map m({ {1,1L}, {2,2L}, {3,3L} }, std::less<int>());
-            // expected-error-re@-1{{no viable constructor or deduction guide for deduction of template arguments of '{{(std::)?}}map'}}
-    }
-    {
-        // cannot deduce that the inner braced things should be std::pair and not something else
-        std::map m({ {1,1L}, {2,2L}, {3,3L} }, std::less<int>(), std::allocator<PC>());
-            // expected-error-re@-1{{no viable constructor or deduction guide for deduction of template arguments of '{{(std::)?}}map'}}
-    }
-    {
-        // cannot deduce that the inner braced things should be std::pair and not something else
-        std::map m({ {1,1L}, {2,2L}, {3,3L} }, std::allocator<PC>());
-            // expected-error-re@-1{{no viable constructor or deduction guide for deduction of template arguments of '{{(std::)?}}map'}}
-    }
-    {
-        // since we have parens, not braces, this deliberately does not find the initializer_list constructor
-        std::map m(P{1,1L});
-            // expected-error-re@-1{{no viable constructor or deduction guide for deduction of template arguments of '{{(std::)?}}map'}}
-    }
-    {
-        // since we have parens, not braces, this deliberately does not find the initializer_list constructor
-        std::map m(PC{1,1L});
-            // expected-error-re@-1{{no viable constructor or deduction guide for deduction of template arguments of '{{(std::)?}}map'}}
-    }
+int main(int, char**) {
+  {
+    // cannot deduce Key and T from nothing
+    std::map m;
+    // expected-error-re@-1{{no viable constructor or deduction guide for deduction of template arguments of '{{(std::)?}}map'}}
+  }
+  {
+    // cannot deduce Key and T from just (Compare)
+    std::map m(std::less<int>{});
+    // expected-error-re@-1{{no viable constructor or deduction guide for deduction of template arguments of '{{(std::)?}}map'}}
+  }
+  {
+    // cannot deduce Key and T from just (Compare, Allocator)
+    std::map m(std::less<int>{}, std::allocator<PC>{});
+    // expected-error-re@-1{{no viable constructor or deduction guide for deduction of template arguments of '{{(std::)?}}map'}}
+  }
+  {
+    // cannot deduce Key and T from just (Allocator)
+    std::map m(std::allocator<PC>{});
+    // expected-error-re@-1{{no viable constructor or deduction guide for deduction of template arguments of '{{(std::)?}}map'}}
+  }
+  {
+    // refuse to rebind the allocator if Allocator::value_type is not exactly what we expect
+    const P arr[] = {{1, 1L}, {2, 2L}, {3, 3L}};
+    std::map m(arr, arr + 3, std::allocator<P>());
+    // expected-error-re@map:*{{static assertion failed{{( due to requirement '.*')?}}{{.*}}Allocator::value_type must be same type as value_type}}
+  }
+  {
+    // cannot convert from some arbitrary unrelated type
+    NotAnAllocator a;
+    std::map m(a);
+    // expected-error-re@-1{{no viable constructor or deduction guide for deduction of template arguments of '{{(std::)?}}map'}}
+  }
+  {
+    // cannot deduce that the inner braced things should be std::pair and not something else
+    std::map m{{1, 1L}, {2, 2L}, {3, 3L}};
+    // expected-error-re@-1{{no viable constructor or deduction guide for deduction of template arguments of '{{(std::)?}}map'}}
+  }
+  {
+    // cannot deduce that the inner braced things should be std::pair and not something else
+    std::map m({{1, 1L}, {2, 2L}, {3, 3L}}, std::less<int>());
+    // expected-error-re@-1{{no viable constructor or deduction guide for deduction of template arguments of '{{(std::)?}}map'}}
+  }
+  {
+    // cannot deduce that the inner braced things should be std::pair and not something else
+    std::map m({{1, 1L}, {2, 2L}, {3, 3L}}, std::less<int>(), std::allocator<PC>());
+    // expected-error-re@-1{{no viable constructor or deduction guide for deduction of template arguments of '{{(std::)?}}map'}}
+  }
+  {
+    // cannot deduce that the inner braced things should be std::pair and not something else
+    std::map m({{1, 1L}, {2, 2L}, {3, 3L}}, std::allocator<PC>());
+    // expected-error-re@-1{{no viable constructor or deduction guide for deduction of template arguments of '{{(std::)?}}map'}}
+  }
+  {
+    // since we have parens, not braces, this deliberately does not find the initializer_list constructor
+    std::map m(P{1, 1L});
+    // expected-error-re@-1{{no viable constructor or deduction guide for deduction of template arguments of '{{(std::)?}}map'}}
+  }
+  {
+    // since we have parens, not braces, this deliberately does not find the initializer_list constructor
+    std::map m(PC{1, 1L});
+    // expected-error-re@-1{{no viable constructor or deduction guide for deduction of template arguments of '{{(std::)?}}map'}}
+  }
 
-    return 0;
+  return 0;
 }
diff --git a/libcxx/test/std/containers/associative/map/map.cons/deduct_const.pass.cpp b/libcxx/test/std/containers/associative/map/map.cons/deduct_const.pass.cpp
index 5b36d1e27f744..dcc2d4510bcb6 100644
--- a/libcxx/test/std/containers/associative/map/map.cons/deduct_const.pass.cpp
+++ b/libcxx/test/std/containers/associative/map/map.cons/deduct_const.pass.cpp
@@ -35,73 +35,73 @@
 
 #include "test_allocator.h"
 
-using P = std::pair<int, long>;
-using PC = std::pair<const int, long>;
+using P   = std::pair<int, long>;
+using PC  = std::pair<const int, long>;
 using PCC = std::pair<const int, const long>;
 
-int main(int, char**)
-{
-    {
-    const PCC arr[] = { {1,1L}, {2,2L}, {1,1L}, {INT_MAX,1L}, {3,1L} };
+int main(int, char**) {
+  {
+    const PCC arr[] = {{1, 1L}, {2, 2L}, {1, 1L}, {INT_MAX, 1L}, {3, 1L}};
     std::map m(std::begin(arr), std::end(arr));
 
     ASSERT_SAME_TYPE(decltype(m), std::map<int, const long>);
-    const PCC expected_m[] = { {1,1L}, {2,2L}, {3,1L}, {INT_MAX,1L} };
+    const PCC expected_m[] = {{1, 1L}, {2, 2L}, {3, 1L}, {INT_MAX, 1L}};
     assert(std::equal(m.begin(), m.end(), std::begin(expected_m), std::end(expected_m)));
-    }
+  }
 
-    {
-    const PCC arr[] = { {1,1L}, {2,2L}, {1,1L}, {INT_MAX,1L}, {3,1L} };
+  {
+    const PCC arr[] = {{1, 1L}, {2, 2L}, {1, 1L}, {INT_MAX, 1L}, {3, 1L}};
     std::map m(std::begin(arr), std::end(arr), std::greater<int>());
 
     ASSERT_SAME_TYPE(decltype(m), std::map<int, const long, std::greater<int>>);
-    const PCC expected_m[] = { {INT_MAX,1L}, {3,1L}, {2,2L}, {1, 1L} };
+    const PCC expected_m[] = {{INT_MAX, 1L}, {3, 1L}, {2, 2L}, {1, 1L}};
     assert(std::equal(m.begin(), m.end(), std::begin(expected_m), std::end(expected_m)));
-    }
+  }
 
-    {
-    const PCC arr[] = { {1,1L}, {2,2L}, {1,1L}, {INT_MAX,1L}, {3,1L} };
+  {
+    const PCC arr[] = {{1, 1L}, {2, 2L}, {1, 1L}, {INT_MAX, 1L}, {3, 1L}};
     std::map m(std::begin(arr), std::end(arr), std::greater<int>(), test_allocator<PCC>(0, 42));
 
     ASSERT_SAME_TYPE(decltype(m), std::map<int, const long, std::greater<int>, test_allocator<PCC>>);
-    const PCC expected_m[] = { {INT_MAX,1L}, {3,1L}, {2,2L}, {1, 1L} };
+    const PCC expected_m[] = {{INT_MAX, 1L}, {3, 1L}, {2, 2L}, {1, 1L}};
     assert(std::equal(m.begin(), m.end(), std::begin(expected_m), std::end(expected_m)));
     assert(m.get_allocator().get_id() == 42);
-    }
+  }
 
-    {
-    std::map m{ PC{1,1L}, PC{2,2L}, PC{1,1L}, PC{INT_MAX,1L}, PC{3,1L} };
+  {
+    std::map m{PC{1, 1L}, PC{2, 2L}, PC{1, 1L}, PC{INT_MAX, 1L}, PC{3, 1L}};
 
     ASSERT_SAME_TYPE(decltype(m), std::map<int, long>);
-    const PC expected_m[] = { {1, 1L}, {2,2L}, {3,1L}, {INT_MAX,1L} };
+    const PC expected_m[] = {{1, 1L}, {2, 2L}, {3, 1L}, {INT_MAX, 1L}};
     assert(std::equal(m.begin(), m.end(), std::begin(expected_m), std::end(expected_m)));
-    }
+  }
 
-    {
-    std::map m({ PC{1,1L}, PC{2,2L}, PC{1,1L}, PC{INT_MAX,1L}, PC{3,1L} }, std::greater<int>());
+  {
+    std::map m({PC{1, 1L}, PC{2, 2L}, PC{1, 1L}, PC{INT_MAX, 1L}, PC{3, 1L}}, std::greater<int>());
 
     ASSERT_SAME_TYPE(decltype(m), std::map<int, long, std::greater<int>>);
-    const PC expected_m[] = { {INT_MAX,1L}, {3,1L}, {2,2L}, {1, 1L} };
+    const PC expected_m[] = {{INT_MAX, 1L}, {3, 1L}, {2, 2L}, {1, 1L}};
     assert(std::equal(m.begin(), m.end(), std::begin(expected_m), std::end(expected_m)));
-    }
+  }
 
-    {
-    std::map m({ PC{1,1L}, PC{2,2L}, PC{1,1L}, PC{INT_MAX,1L}, PC{3,1L} }, std::greater<int>(), test_allocator<PC>(0, 43));
+  {
+    std::map m(
+        {PC{1, 1L}, PC{2, 2L}, PC{1, 1L}, PC{INT_MAX, 1L}, PC{3, 1L}}, std::greater<int>(), test_allocator<PC>(0, 43));
 
     ASSERT_SAME_TYPE(decltype(m), std::map<int, long, std::greater<int>, test_allocator<PC>>);
-    const PC expected_m[] = { {INT_MAX,1L}, {3,1L}, {2,2L}, {1, 1L} };
+    const PC expected_m[] = {{INT_MAX, 1L}, {3, 1L}, {2, 2L}, {1, 1L}};
     assert(std::equal(m.begin(), m.end(), std::begin(expected_m), std::end(expected_m)));
     assert(m.get_allocator().get_id() == 43);
-    }
+  }
 
-    {
-    std::map m({ PC{1,1L}, PC{2,2L}, PC{1,1L}, PC{INT_MAX,1L}, PC{3,1L} }, test_allocator<PC>(0, 45));
+  {
+    std::map m({PC{1, 1L}, PC{2, 2L}, PC{1, 1L}, PC{INT_MAX, 1L}, PC{3, 1L}}, test_allocator<PC>(0, 45));
 
     ASSERT_SAME_TYPE(decltype(m), std::map<int, long, std::less<int>, test_allocator<PC>>);
-    const PC expected_m[] = { {1, 1L}, {2,2L}, {3,1L}, {INT_MAX,1L} };
+    const PC expected_m[] = {{1, 1L}, {2, 2L}, {3, 1L}, {INT_MAX, 1L}};
     assert(std::equal(m.begin(), m.end(), std::begin(expected_m), std::end(expected_m)));
     assert(m.get_allocator().get_id() == 45);
-    }
+  }
 
-    return 0;
+  return 0;
 }
diff --git a/libcxx/test/std/containers/associative/map/map.cons/default.pass.cpp b/libcxx/test/std/containers/associative/map/map.cons/default.pass.cpp
index 55a99d20a53ce..1b48370387a8f 100644
--- a/libcxx/test/std/containers/associative/map/map.cons/default.pass.cpp
+++ b/libcxx/test/std/containers/associative/map/map.cons/default.pass.cpp
@@ -18,38 +18,37 @@
 #include "test_macros.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
+int main(int, char**) {
+  {
     std::map<int, double> m;
     assert(m.empty());
     assert(m.begin() == m.end());
-    }
+  }
 #if TEST_STD_VER >= 11
-    {
+  {
     std::map<int, double, std::less<int>, min_allocator<std::pair<const int, double>>> m;
     assert(m.empty());
     assert(m.begin() == m.end());
-    }
-    {
+  }
+  {
     typedef explicit_allocator<std::pair<const int, double>> A;
-        {
-        std::map<int, double, std::less<int>, A> m;
-        assert(m.empty());
-        assert(m.begin() == m.end());
-        }
-        {
-        A a;
-        std::map<int, double, std::less<int>, A> m(a);
-        assert(m.empty());
-        assert(m.begin() == m.end());
-        }
+    {
+      std::map<int, double, std::less<int>, A> m;
+      assert(m.empty());
+      assert(m.begin() == m.end());
     }
     {
+      A a;
+      std::map<int, double, std::less<int>, A> m(a);
+      assert(m.empty());
+      assert(m.begin() == m.end());
+    }
+  }
+  {
     std::map<int, double> m = {};
     assert(m.empty());
     assert(m.begin() == m.end());
-    }
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/associative/map/map.cons/default_noexcept.pass.cpp b/libcxx/test/std/containers/associative/map/map.cons/default_noexcept.pass.cpp
index 1139d9fc87ff6..1e43a893d66c1 100644
--- a/libcxx/test/std/containers/associative/map/map.cons/default_noexcept.pass.cpp
+++ b/libcxx/test/std/containers/associative/map/map.cons/default_noexcept.pass.cpp
@@ -26,34 +26,32 @@
 #include "test_allocator.h"
 
 template <class T>
-struct some_comp
-{
-    typedef T value_type;
-    some_comp();
-    bool operator()(const T&, const T&) const { return false; }
+struct some_comp {
+  typedef T value_type;
+  some_comp();
+  bool operator()(const T&, const T&) const { return false; }
 };
 
-int main(int, char**)
-{
-    typedef std::pair<const MoveOnly, MoveOnly> V;
+int main(int, char**) {
+  typedef std::pair<const MoveOnly, MoveOnly> V;
 #if defined(_LIBCPP_VERSION)
-    {
-        typedef std::map<MoveOnly, MoveOnly> C;
-        static_assert(std::is_nothrow_default_constructible<C>::value, "");
-    }
-    {
-        typedef std::map<MoveOnly, MoveOnly, std::less<MoveOnly>, test_allocator<V>> C;
-        static_assert(std::is_nothrow_default_constructible<C>::value, "");
-    }
+  {
+    typedef std::map<MoveOnly, MoveOnly> C;
+    static_assert(std::is_nothrow_default_constructible<C>::value, "");
+  }
+  {
+    typedef std::map<MoveOnly, MoveOnly, std::less<MoveOnly>, test_allocator<V>> C;
+    static_assert(std::is_nothrow_default_constructible<C>::value, "");
+  }
 #endif // _LIBCPP_VERSION
-    {
-        typedef std::map<MoveOnly, MoveOnly, std::less<MoveOnly>, other_allocator<V>> C;
-        static_assert(!std::is_nothrow_default_constructible<C>::value, "");
-    }
-    {
-        typedef std::map<MoveOnly, MoveOnly, some_comp<MoveOnly>> C;
-        static_assert(!std::is_nothrow_default_constructible<C>::value, "");
-    }
+  {
+    typedef std::map<MoveOnly, MoveOnly, std::less<MoveOnly>, other_allocator<V>> C;
+    static_assert(!std::is_nothrow_default_constructible<C>::value, "");
+  }
+  {
+    typedef std::map<MoveOnly, MoveOnly, some_comp<MoveOnly>> C;
+    static_assert(!std::is_nothrow_default_constructible<C>::value, "");
+  }
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/associative/map/map.cons/dtor_noexcept.pass.cpp b/libcxx/test/std/containers/associative/map/map.cons/dtor_noexcept.pass.cpp
index 7d28e188f819d..8497b942ad8eb 100644
--- a/libcxx/test/std/containers/associative/map/map.cons/dtor_noexcept.pass.cpp
+++ b/libcxx/test/std/containers/associative/map/map.cons/dtor_noexcept.pass.cpp
@@ -20,33 +20,31 @@
 #include "test_allocator.h"
 
 template <class T>
-struct some_comp
-{
-    typedef T value_type;
-    ~some_comp() noexcept(false);
-    bool operator()(const T&, const T&) const noexcept { return false; }
+struct some_comp {
+  typedef T value_type;
+  ~some_comp() noexcept(false);
+  bool operator()(const T&, const T&) const noexcept { return false; }
 };
 
-int main(int, char**)
-{
-    typedef std::pair<const MoveOnly, MoveOnly> V;
-    {
-        typedef std::map<MoveOnly, MoveOnly> C;
-        static_assert(std::is_nothrow_destructible<C>::value, "");
-    }
-    {
-        typedef std::map<MoveOnly, MoveOnly, std::less<MoveOnly>, test_allocator<V>> C;
-        static_assert(std::is_nothrow_destructible<C>::value, "");
-    }
-    {
-        typedef std::map<MoveOnly, MoveOnly, std::less<MoveOnly>, other_allocator<V>> C;
-        static_assert(std::is_nothrow_destructible<C>::value, "");
-    }
+int main(int, char**) {
+  typedef std::pair<const MoveOnly, MoveOnly> V;
+  {
+    typedef std::map<MoveOnly, MoveOnly> C;
+    static_assert(std::is_nothrow_destructible<C>::value, "");
+  }
+  {
+    typedef std::map<MoveOnly, MoveOnly, std::less<MoveOnly>, test_allocator<V>> C;
+    static_assert(std::is_nothrow_destructible<C>::value, "");
+  }
+  {
+    typedef std::map<MoveOnly, MoveOnly, std::less<MoveOnly>, other_allocator<V>> C;
+    static_assert(std::is_nothrow_destructible<C>::value, "");
+  }
 #if defined(_LIBCPP_VERSION)
-    {
-        typedef std::map<MoveOnly, MoveOnly, some_comp<MoveOnly>> C;
-        static_assert(!std::is_nothrow_destructible<C>::value, "");
-    }
+  {
+    typedef std::map<MoveOnly, MoveOnly, some_comp<MoveOnly>> C;
+    static_assert(!std::is_nothrow_destructible<C>::value, "");
+  }
 #endif // _LIBCPP_VERSION
 
   return 0;
diff --git a/libcxx/test/std/containers/associative/map/map.cons/from_range.pass.cpp b/libcxx/test/std/containers/associative/map/map.cons/from_range.pass.cpp
index f375dbfcf47b2..dc06266064a8d 100644
--- a/libcxx/test/std/containers/associative/map/map.cons/from_range.pass.cpp
+++ b/libcxx/test/std/containers/associative/map/map.cons/from_range.pass.cpp
@@ -24,13 +24,9 @@
 void test_duplicates() {
   using T = std::pair<const int, char>;
 
-  std::array input = {
-    T{1, 'a'}, T{2, 'a'}, T{3, 'a'}, T{3, 'b'}, T{3, 'c'}, T{2, 'b'}, T{4, 'a'}
-  };
-  std::array expected = {
-    T{1, 'a'}, T{2, 'a'}, T{3, 'a'}, T{4, 'a'}
-  };
-  auto c = std::map<int, char>(std::from_range, input);
+  std::array input    = {T{1, 'a'}, T{2, 'a'}, T{3, 'a'}, T{3, 'b'}, T{3, 'c'}, T{2, 'b'}, T{4, 'a'}};
+  std::array expected = {T{1, 'a'}, T{2, 'a'}, T{3, 'a'}, T{4, 'a'}};
+  auto c              = std::map<int, char>(std::from_range, input);
   assert(std::ranges::is_permutation(expected, c));
 }
 
diff --git a/libcxx/test/std/containers/associative/map/map.cons/initializer_list.pass.cpp b/libcxx/test/std/containers/associative/map/map.cons/initializer_list.pass.cpp
index 7eebf49c38943..b18dc5cf754a0 100644
--- a/libcxx/test/std/containers/associative/map/map.cons/initializer_list.pass.cpp
+++ b/libcxx/test/std/containers/associative/map/map.cons/initializer_list.pass.cpp
@@ -20,48 +20,26 @@
 #include "test_macros.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
+int main(int, char**) {
+  {
     typedef std::pair<const int, double> V;
-    std::map<int, double> m =
-                            {
-                                {1, 1},
-                                {1, 1.5},
-                                {1, 2},
-                                {2, 1},
-                                {2, 1.5},
-                                {2, 2},
-                                {3, 1},
-                                {3, 1.5},
-                                {3, 2}
-                            };
+    std::map<int, double> m = {{1, 1}, {1, 1.5}, {1, 2}, {2, 1}, {2, 1.5}, {2, 2}, {3, 1}, {3, 1.5}, {3, 2}};
     assert(m.size() == 3);
     assert(std::distance(m.begin(), m.end()) == 3);
     assert(*m.begin() == V(1, 1));
     assert(*std::next(m.begin()) == V(2, 1));
     assert(*std::next(m.begin(), 2) == V(3, 1));
-    }
-    {
+  }
+  {
     typedef std::pair<const int, double> V;
-    std::map<int, double, std::less<int>, min_allocator<V>> m =
-                            {
-                                {1, 1},
-                                {1, 1.5},
-                                {1, 2},
-                                {2, 1},
-                                {2, 1.5},
-                                {2, 2},
-                                {3, 1},
-                                {3, 1.5},
-                                {3, 2}
-                            };
+    std::map<int, double, std::less<int>, min_allocator<V>> m = {
+        {1, 1}, {1, 1.5}, {1, 2}, {2, 1}, {2, 1.5}, {2, 2}, {3, 1}, {3, 1.5}, {3, 2}};
     assert(m.size() == 3);
     assert(std::distance(m.begin(), m.end()) == 3);
     assert(*m.begin() == V(1, 1));
     assert(*std::next(m.begin()) == V(2, 1));
     assert(*std::next(m.begin(), 2) == V(3, 1));
-    }
+  }
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/associative/map/map.cons/initializer_list_compare.pass.cpp b/libcxx/test/std/containers/associative/map/map.cons/initializer_list_compare.pass.cpp
index 441c96d813e7b..301384362bfe7 100644
--- a/libcxx/test/std/containers/associative/map/map.cons/initializer_list_compare.pass.cpp
+++ b/libcxx/test/std/containers/associative/map/map.cons/initializer_list_compare.pass.cpp
@@ -20,50 +20,30 @@
 #include "../../../test_compare.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
+int main(int, char**) {
+  {
     typedef std::pair<const int, double> V;
     typedef test_less<int> C;
-    std::map<int, double, C> m({
-                                {1, 1},
-                                {1, 1.5},
-                                {1, 2},
-                                {2, 1},
-                                {2, 1.5},
-                                {2, 2},
-                                {3, 1},
-                                {3, 1.5},
-                                {3, 2}
-                               }, C(3));
+    std::map<int, double, C> m({{1, 1}, {1, 1.5}, {1, 2}, {2, 1}, {2, 1.5}, {2, 2}, {3, 1}, {3, 1.5}, {3, 2}}, C(3));
     assert(m.size() == 3);
     assert(std::distance(m.begin(), m.end()) == 3);
     assert(*m.begin() == V(1, 1));
     assert(*std::next(m.begin()) == V(2, 1));
     assert(*std::next(m.begin(), 2) == V(3, 1));
     assert(m.key_comp() == C(3));
-    }
-    {
+  }
+  {
     typedef std::pair<const int, double> V;
     typedef test_less<int> C;
-    std::map<int, double, C, min_allocator<std::pair<const int, double>>> m({
-                                {1, 1},
-                                {1, 1.5},
-                                {1, 2},
-                                {2, 1},
-                                {2, 1.5},
-                                {2, 2},
-                                {3, 1},
-                                {3, 1.5},
-                                {3, 2}
-                               }, C(3));
+    std::map<int, double, C, min_allocator<std::pair<const int, double>>> m(
+        {{1, 1}, {1, 1.5}, {1, 2}, {2, 1}, {2, 1.5}, {2, 2}, {3, 1}, {3, 1.5}, {3, 2}}, C(3));
     assert(m.size() == 3);
     assert(std::distance(m.begin(), m.end()) == 3);
     assert(*m.begin() == V(1, 1));
     assert(*std::next(m.begin()) == V(2, 1));
     assert(*std::next(m.begin(), 2) == V(3, 1));
     assert(m.key_comp() == C(3));
-    }
+  }
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/associative/map/map.cons/initializer_list_compare_alloc.pass.cpp b/libcxx/test/std/containers/associative/map/map.cons/initializer_list_compare_alloc.pass.cpp
index 95ae86e09ab7e..da138ebd9ae61 100644
--- a/libcxx/test/std/containers/associative/map/map.cons/initializer_list_compare_alloc.pass.cpp
+++ b/libcxx/test/std/containers/associative/map/map.cons/initializer_list_compare_alloc.pass.cpp
@@ -21,23 +21,13 @@
 #include "test_allocator.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
+int main(int, char**) {
+  {
     typedef std::pair<const int, double> V;
     typedef test_less<int> C;
     typedef test_allocator<std::pair<const int, double> > A;
-    std::map<int, double, C, A> m({
-                                   {1, 1},
-                                   {1, 1.5},
-                                   {1, 2},
-                                   {2, 1},
-                                   {2, 1.5},
-                                   {2, 2},
-                                   {3, 1},
-                                   {3, 1.5},
-                                   {3, 2}
-                                  }, C(3), A(6));
+    std::map<int, double, C, A> m(
+        {{1, 1}, {1, 1.5}, {1, 2}, {2, 1}, {2, 1.5}, {2, 2}, {3, 1}, {3, 1.5}, {3, 2}}, C(3), A(6));
     assert(m.size() == 3);
     assert(std::distance(m.begin(), m.end()) == 3);
     assert(*m.begin() == V(1, 1));
@@ -45,22 +35,13 @@ int main(int, char**)
     assert(*std::next(m.begin(), 2) == V(3, 1));
     assert(m.key_comp() == C(3));
     assert(m.get_allocator() == A(6));
-    }
-    {
+  }
+  {
     typedef std::pair<const int, double> V;
     typedef test_less<int> C;
     typedef min_allocator<std::pair<const int, double> > A;
-    std::map<int, double, C, A> m({
-                                   {1, 1},
-                                   {1, 1.5},
-                                   {1, 2},
-                                   {2, 1},
-                                   {2, 1.5},
-                                   {2, 2},
-                                   {3, 1},
-                                   {3, 1.5},
-                                   {3, 2}
-                                  }, C(3), A());
+    std::map<int, double, C, A> m(
+        {{1, 1}, {1, 1.5}, {1, 2}, {2, 1}, {2, 1.5}, {2, 2}, {3, 1}, {3, 1.5}, {3, 2}}, C(3), A());
     assert(m.size() == 3);
     assert(std::distance(m.begin(), m.end()) == 3);
     assert(*m.begin() == V(1, 1));
@@ -68,23 +49,14 @@ int main(int, char**)
     assert(*std::next(m.begin(), 2) == V(3, 1));
     assert(m.key_comp() == C(3));
     assert(m.get_allocator() == A());
-    }
-    {
+  }
+  {
     typedef std::pair<const int, double> V;
     typedef min_allocator<V> A;
     typedef test_less<int> C;
     typedef std::map<int, double, C, A> M;
     A a;
-    M m ({ {1, 1},
-           {1, 1.5},
-           {1, 2},
-           {2, 1},
-           {2, 1.5},
-           {2, 2},
-           {3, 1},
-           {3, 1.5},
-           {3, 2}
-          }, a);
+    M m({{1, 1}, {1, 1.5}, {1, 2}, {2, 1}, {2, 1.5}, {2, 2}, {3, 1}, {3, 1.5}, {3, 2}}, a);
 
     assert(m.size() == 3);
     assert(std::distance(m.begin(), m.end()) == 3);
@@ -92,23 +64,14 @@ int main(int, char**)
     assert(*std::next(m.begin()) == V(2, 1));
     assert(*std::next(m.begin(), 2) == V(3, 1));
     assert(m.get_allocator() == a);
-    }
-    {
+  }
+  {
     typedef std::pair<const int, double> V;
     typedef explicit_allocator<V> A;
     typedef test_less<int> C;
     A a;
-    std::map<int, double, C, A> m({
-                                   {1, 1},
-                                   {1, 1.5},
-                                   {1, 2},
-                                   {2, 1},
-                                   {2, 1.5},
-                                   {2, 2},
-                                   {3, 1},
-                                   {3, 1.5},
-                                   {3, 2}
-                                  }, C(3), a);
+    std::map<int, double, C, A> m(
+        {{1, 1}, {1, 1.5}, {1, 2}, {2, 1}, {2, 1.5}, {2, 2}, {3, 1}, {3, 1.5}, {3, 2}}, C(3), a);
     assert(m.size() == 3);
     assert(std::distance(m.begin(), m.end()) == 3);
     assert(*m.begin() == V(1, 1));
@@ -116,7 +79,7 @@ int main(int, char**)
     assert(*std::next(m.begin(), 2) == V(3, 1));
     assert(m.key_comp() == C(3));
     assert(m.get_allocator() == a);
-    }
+  }
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/associative/map/map.cons/iter_iter.pass.cpp b/libcxx/test/std/containers/associative/map/map.cons/iter_iter.pass.cpp
index c3032e72fe5da..c8ed1413c3084 100644
--- a/libcxx/test/std/containers/associative/map/map.cons/iter_iter.pass.cpp
+++ b/libcxx/test/std/containers/associative/map/map.cons/iter_iter.pass.cpp
@@ -19,12 +19,10 @@
 #include "test_macros.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
+int main(int, char**) {
+  {
     typedef std::pair<const int, double> V;
-    V ar[] =
-    {
+    V ar[] = {
         V(1, 1),
         V(1, 1.5),
         V(1, 2),
@@ -35,18 +33,17 @@ int main(int, char**)
         V(3, 1.5),
         V(3, 2),
     };
-    std::map<int, double> m(ar, ar+sizeof(ar)/sizeof(ar[0]));
+    std::map<int, double> m(ar, ar + sizeof(ar) / sizeof(ar[0]));
     assert(m.size() == 3);
     assert(std::distance(m.begin(), m.end()) == 3);
     assert(*m.begin() == V(1, 1));
     assert(*std::next(m.begin()) == V(2, 1));
     assert(*std::next(m.begin(), 2) == V(3, 1));
-    }
+  }
 #if TEST_STD_VER >= 11
-    {
+  {
     typedef std::pair<const int, double> V;
-    V ar[] =
-    {
+    V ar[] = {
         V(1, 1),
         V(1, 1.5),
         V(1, 2),
@@ -57,13 +54,14 @@ int main(int, char**)
         V(3, 1.5),
         V(3, 2),
     };
-    std::map<int, double, std::less<int>, min_allocator<std::pair<const int, double>>> m(ar, ar+sizeof(ar)/sizeof(ar[0]));
+    std::map<int, double, std::less<int>, min_allocator<std::pair<const int, double>>> m(
+        ar, ar + sizeof(ar) / sizeof(ar[0]));
     assert(m.size() == 3);
     assert(std::distance(m.begin(), m.end()) == 3);
     assert(*m.begin() == V(1, 1));
     assert(*std::next(m.begin()) == V(2, 1));
     assert(*std::next(m.begin(), 2) == V(3, 1));
-    }
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/associative/map/map.cons/iter_iter_comp.pass.cpp b/libcxx/test/std/containers/associative/map/map.cons/iter_iter_comp.pass.cpp
index 7649ede9a3fb0..f8ecee04a528a 100644
--- a/libcxx/test/std/containers/associative/map/map.cons/iter_iter_comp.pass.cpp
+++ b/libcxx/test/std/containers/associative/map/map.cons/iter_iter_comp.pass.cpp
@@ -20,12 +20,10 @@
 #include "../../../test_compare.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
+int main(int, char**) {
+  {
     typedef std::pair<const int, double> V;
-    V ar[] =
-    {
+    V ar[] = {
         V(1, 1),
         V(1, 1.5),
         V(1, 2),
@@ -37,19 +35,18 @@ int main(int, char**)
         V(3, 2),
     };
     typedef test_less<int> C;
-    std::map<int, double, C> m(ar, ar+sizeof(ar)/sizeof(ar[0]), C(5));
+    std::map<int, double, C> m(ar, ar + sizeof(ar) / sizeof(ar[0]), C(5));
     assert(m.key_comp() == C(5));
     assert(m.size() == 3);
     assert(std::distance(m.begin(), m.end()) == 3);
     assert(*m.begin() == V(1, 1));
     assert(*std::next(m.begin()) == V(2, 1));
     assert(*std::next(m.begin(), 2) == V(3, 1));
-    }
+  }
 #if TEST_STD_VER >= 11
-    {
+  {
     typedef std::pair<const int, double> V;
-    V ar[] =
-    {
+    V ar[] = {
         V(1, 1),
         V(1, 1.5),
         V(1, 2),
@@ -61,14 +58,14 @@ int main(int, char**)
         V(3, 2),
     };
     typedef test_less<int> C;
-    std::map<int, double, C, min_allocator<std::pair<const int, double>>> m(ar, ar+sizeof(ar)/sizeof(ar[0]), C(5));
+    std::map<int, double, C, min_allocator<std::pair<const int, double>>> m(ar, ar + sizeof(ar) / sizeof(ar[0]), C(5));
     assert(m.key_comp() == C(5));
     assert(m.size() == 3);
     assert(std::distance(m.begin(), m.end()) == 3);
     assert(*m.begin() == V(1, 1));
     assert(*std::next(m.begin()) == V(2, 1));
     assert(*std::next(m.begin(), 2) == V(3, 1));
-    }
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/associative/map/map.cons/iter_iter_comp_alloc.pass.cpp b/libcxx/test/std/containers/associative/map/map.cons/iter_iter_comp_alloc.pass.cpp
index e8714346951da..862b07bb11731 100644
--- a/libcxx/test/std/containers/associative/map/map.cons/iter_iter_comp_alloc.pass.cpp
+++ b/libcxx/test/std/containers/associative/map/map.cons/iter_iter_comp_alloc.pass.cpp
@@ -22,12 +22,10 @@
 #include "test_allocator.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
+int main(int, char**) {
+  {
     typedef std::pair<const int, double> V;
-    V ar[] =
-    {
+    V ar[] = {
         V(1, 1),
         V(1, 1.5),
         V(1, 2),
@@ -40,7 +38,7 @@ int main(int, char**)
     };
     typedef test_less<int> C;
     typedef test_allocator<V> A;
-    std::map<int, double, C, A> m(ar, ar+sizeof(ar)/sizeof(ar[0]), C(5), A(7));
+    std::map<int, double, C, A> m(ar, ar + sizeof(ar) / sizeof(ar[0]), C(5), A(7));
     assert(m.get_allocator() == A(7));
     assert(m.key_comp() == C(5));
     assert(m.size() == 3);
@@ -48,12 +46,11 @@ int main(int, char**)
     assert(*m.begin() == V(1, 1));
     assert(*std::next(m.begin()) == V(2, 1));
     assert(*std::next(m.begin(), 2) == V(3, 1));
-    }
+  }
 #if TEST_STD_VER >= 11
-    {
+  {
     typedef std::pair<const int, double> V;
-    V ar[] =
-    {
+    V ar[] = {
         V(1, 1),
         V(1, 1.5),
         V(1, 2),
@@ -66,7 +63,7 @@ int main(int, char**)
     };
     typedef test_less<int> C;
     typedef min_allocator<V> A;
-    std::map<int, double, C, A> m(ar, ar+sizeof(ar)/sizeof(ar[0]), C(5), A());
+    std::map<int, double, C, A> m(ar, ar + sizeof(ar) / sizeof(ar[0]), C(5), A());
     assert(m.get_allocator() == A());
     assert(m.key_comp() == C(5));
     assert(m.size() == 3);
@@ -74,12 +71,11 @@ int main(int, char**)
     assert(*m.begin() == V(1, 1));
     assert(*std::next(m.begin()) == V(2, 1));
     assert(*std::next(m.begin(), 2) == V(3, 1));
-    }
-#if TEST_STD_VER > 11
-    {
+  }
+#  if TEST_STD_VER > 11
+  {
     typedef std::pair<const int, double> V;
-    V ar[] =
-    {
+    V ar[] = {
         V(1, 1),
         V(1, 1.5),
         V(1, 2),
@@ -91,33 +87,33 @@ int main(int, char**)
         V(3, 2),
     };
     {
-    typedef min_allocator<V> A;
-    typedef test_less<int> C;
-    A a;
-    std::map<int, double, C, A> m(ar, ar+sizeof(ar)/sizeof(ar[0]), a );
+      typedef min_allocator<V> A;
+      typedef test_less<int> C;
+      A a;
+      std::map<int, double, C, A> m(ar, ar + sizeof(ar) / sizeof(ar[0]), a);
 
-    assert(m.size() == 3);
-    assert(std::distance(m.begin(), m.end()) == 3);
-    assert(*m.begin() == V(1, 1));
-    assert(*std::next(m.begin()) == V(2, 1));
-    assert(*std::next(m.begin(), 2) == V(3, 1));
-    assert(m.get_allocator() == a);
+      assert(m.size() == 3);
+      assert(std::distance(m.begin(), m.end()) == 3);
+      assert(*m.begin() == V(1, 1));
+      assert(*std::next(m.begin()) == V(2, 1));
+      assert(*std::next(m.begin(), 2) == V(3, 1));
+      assert(m.get_allocator() == a);
     }
     {
-    typedef explicit_allocator<V> A;
-    typedef test_less<int> C;
-    A a;
-    std::map<int, double, C, A> m(ar, ar+sizeof(ar)/sizeof(ar[0]), a );
+      typedef explicit_allocator<V> A;
+      typedef test_less<int> C;
+      A a;
+      std::map<int, double, C, A> m(ar, ar + sizeof(ar) / sizeof(ar[0]), a);
 
-    assert(m.size() == 3);
-    assert(std::distance(m.begin(), m.end()) == 3);
-    assert(*m.begin() == V(1, 1));
-    assert(*std::next(m.begin()) == V(2, 1));
-    assert(*std::next(m.begin(), 2) == V(3, 1));
-    assert(m.get_allocator() == a);
+      assert(m.size() == 3);
+      assert(std::distance(m.begin(), m.end()) == 3);
+      assert(*m.begin() == V(1, 1));
+      assert(*std::next(m.begin()) == V(2, 1));
+      assert(*std::next(m.begin(), 2) == V(3, 1));
+      assert(m.get_allocator() == a);
     }
-    }
-#endif
+  }
+#  endif
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/associative/map/map.cons/move.pass.cpp b/libcxx/test/std/containers/associative/map/map.cons/move.pass.cpp
index 7a883bed0d6e8..d33ef527c91de 100644
--- a/libcxx/test/std/containers/associative/map/map.cons/move.pass.cpp
+++ b/libcxx/test/std/containers/associative/map/map.cons/move.pass.cpp
@@ -22,101 +22,98 @@
 #include "test_allocator.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    typedef std::pair<const int, double> V;
-    {
-        typedef test_less<int> C;
-        typedef test_allocator<V> A;
-        std::map<int, double, C, A> mo(C(5), A(7));
-        std::map<int, double, C, A> m = std::move(mo);
-        assert(m.get_allocator() == A(7));
-        assert(m.key_comp() == C(5));
-        assert(m.size() == 0);
-        assert(std::distance(m.begin(), m.end()) == 0);
+int main(int, char**) {
+  typedef std::pair<const int, double> V;
+  {
+    typedef test_less<int> C;
+    typedef test_allocator<V> A;
+    std::map<int, double, C, A> mo(C(5), A(7));
+    std::map<int, double, C, A> m = std::move(mo);
+    assert(m.get_allocator() == A(7));
+    assert(m.key_comp() == C(5));
+    assert(m.size() == 0);
+    assert(std::distance(m.begin(), m.end()) == 0);
 
-        assert(mo.get_allocator() == A(7));
-        assert(mo.get_allocator().get_id() == test_alloc_base::moved_value);
-        assert(mo.key_comp() == C(5));
-        assert(mo.size() == 0);
-        assert(std::distance(mo.begin(), mo.end()) == 0);
-    }
-    {
-        V ar[] =
-        {
-            V(1, 1),
-            V(1, 1.5),
-            V(1, 2),
-            V(2, 1),
-            V(2, 1.5),
-            V(2, 2),
-            V(3, 1),
-            V(3, 1.5),
-            V(3, 2),
-        };
-        typedef test_less<int> C;
-        typedef test_allocator<V> A;
-        std::map<int, double, C, A> mo(ar, ar+sizeof(ar)/sizeof(ar[0]), C(5), A(7));
-        std::map<int, double, C, A> m = std::move(mo);
-        assert(m.get_allocator() == A(7));
-        assert(m.key_comp() == C(5));
-        assert(m.size() == 3);
-        assert(std::distance(m.begin(), m.end()) == 3);
-        assert(*m.begin() == V(1, 1));
-        assert(*std::next(m.begin()) == V(2, 1));
-        assert(*std::next(m.begin(), 2) == V(3, 1));
+    assert(mo.get_allocator() == A(7));
+    assert(mo.get_allocator().get_id() == test_alloc_base::moved_value);
+    assert(mo.key_comp() == C(5));
+    assert(mo.size() == 0);
+    assert(std::distance(mo.begin(), mo.end()) == 0);
+  }
+  {
+    V ar[] = {
+        V(1, 1),
+        V(1, 1.5),
+        V(1, 2),
+        V(2, 1),
+        V(2, 1.5),
+        V(2, 2),
+        V(3, 1),
+        V(3, 1.5),
+        V(3, 2),
+    };
+    typedef test_less<int> C;
+    typedef test_allocator<V> A;
+    std::map<int, double, C, A> mo(ar, ar + sizeof(ar) / sizeof(ar[0]), C(5), A(7));
+    std::map<int, double, C, A> m = std::move(mo);
+    assert(m.get_allocator() == A(7));
+    assert(m.key_comp() == C(5));
+    assert(m.size() == 3);
+    assert(std::distance(m.begin(), m.end()) == 3);
+    assert(*m.begin() == V(1, 1));
+    assert(*std::next(m.begin()) == V(2, 1));
+    assert(*std::next(m.begin(), 2) == V(3, 1));
 
-        assert(mo.get_allocator() == A(7));
-        assert(mo.get_allocator().get_id() == test_alloc_base::moved_value);
-        assert(mo.key_comp() == C(5));
-        assert(mo.size() == 0);
-        assert(std::distance(mo.begin(), mo.end()) == 0);
-    }
-    {
-        typedef test_less<int> C;
-        typedef min_allocator<V> A;
-        std::map<int, double, C, A> mo(C(5), A());
-        std::map<int, double, C, A> m = std::move(mo);
-        assert(m.get_allocator() == A());
-        assert(m.key_comp() == C(5));
-        assert(m.size() == 0);
-        assert(std::distance(m.begin(), m.end()) == 0);
+    assert(mo.get_allocator() == A(7));
+    assert(mo.get_allocator().get_id() == test_alloc_base::moved_value);
+    assert(mo.key_comp() == C(5));
+    assert(mo.size() == 0);
+    assert(std::distance(mo.begin(), mo.end()) == 0);
+  }
+  {
+    typedef test_less<int> C;
+    typedef min_allocator<V> A;
+    std::map<int, double, C, A> mo(C(5), A());
+    std::map<int, double, C, A> m = std::move(mo);
+    assert(m.get_allocator() == A());
+    assert(m.key_comp() == C(5));
+    assert(m.size() == 0);
+    assert(std::distance(m.begin(), m.end()) == 0);
 
-        assert(mo.get_allocator() == A());
-        assert(mo.key_comp() == C(5));
-        assert(mo.size() == 0);
-        assert(std::distance(mo.begin(), mo.end()) == 0);
-    }
-    {
-        V ar[] =
-        {
-            V(1, 1),
-            V(1, 1.5),
-            V(1, 2),
-            V(2, 1),
-            V(2, 1.5),
-            V(2, 2),
-            V(3, 1),
-            V(3, 1.5),
-            V(3, 2),
-        };
-        typedef test_less<int> C;
-        typedef min_allocator<V> A;
-        std::map<int, double, C, A> mo(ar, ar+sizeof(ar)/sizeof(ar[0]), C(5), A());
-        std::map<int, double, C, A> m = std::move(mo);
-        assert(m.get_allocator() == A());
-        assert(m.key_comp() == C(5));
-        assert(m.size() == 3);
-        assert(std::distance(m.begin(), m.end()) == 3);
-        assert(*m.begin() == V(1, 1));
-        assert(*std::next(m.begin()) == V(2, 1));
-        assert(*std::next(m.begin(), 2) == V(3, 1));
+    assert(mo.get_allocator() == A());
+    assert(mo.key_comp() == C(5));
+    assert(mo.size() == 0);
+    assert(std::distance(mo.begin(), mo.end()) == 0);
+  }
+  {
+    V ar[] = {
+        V(1, 1),
+        V(1, 1.5),
+        V(1, 2),
+        V(2, 1),
+        V(2, 1.5),
+        V(2, 2),
+        V(3, 1),
+        V(3, 1.5),
+        V(3, 2),
+    };
+    typedef test_less<int> C;
+    typedef min_allocator<V> A;
+    std::map<int, double, C, A> mo(ar, ar + sizeof(ar) / sizeof(ar[0]), C(5), A());
+    std::map<int, double, C, A> m = std::move(mo);
+    assert(m.get_allocator() == A());
+    assert(m.key_comp() == C(5));
+    assert(m.size() == 3);
+    assert(std::distance(m.begin(), m.end()) == 3);
+    assert(*m.begin() == V(1, 1));
+    assert(*std::next(m.begin()) == V(2, 1));
+    assert(*std::next(m.begin(), 2) == V(3, 1));
 
-        assert(mo.get_allocator() == A());
-        assert(mo.key_comp() == C(5));
-        assert(mo.size() == 0);
-        assert(std::distance(mo.begin(), mo.end()) == 0);
-    }
+    assert(mo.get_allocator() == A());
+    assert(mo.key_comp() == C(5));
+    assert(mo.size() == 0);
+    assert(std::distance(mo.begin(), mo.end()) == 0);
+  }
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/associative/map/map.cons/move_alloc.pass.cpp b/libcxx/test/std/containers/associative/map/map.cons/move_alloc.pass.cpp
index 63630164b2e64..886a4fa714d89 100644
--- a/libcxx/test/std/containers/associative/map/map.cons/move_alloc.pass.cpp
+++ b/libcxx/test/std/containers/associative/map/map.cons/move_alloc.pass.cpp
@@ -25,253 +25,131 @@
 #include "min_allocator.h"
 #include "Counter.h"
 
-int main(int, char**)
-{
+int main(int, char**) {
+  {
+    typedef std::pair<MoveOnly, MoveOnly> V;
+    typedef std::pair<const MoveOnly, MoveOnly> VC;
+    typedef test_less<MoveOnly> C;
+    typedef test_allocator<VC> A;
+    typedef std::map<MoveOnly, MoveOnly, C, A> M;
+    typedef std::move_iterator<V*> I;
+    V a1[] = {V(1, 1), V(1, 2), V(1, 3), V(2, 1), V(2, 2), V(2, 3), V(3, 1), V(3, 2), V(3, 3)};
+    M m1(I(a1), I(a1 + sizeof(a1) / sizeof(a1[0])), C(5), A(7));
+    V a2[] = {V(1, 1), V(1, 2), V(1, 3), V(2, 1), V(2, 2), V(2, 3), V(3, 1), V(3, 2), V(3, 3)};
+    M m2(I(a2), I(a2 + sizeof(a2) / sizeof(a2[0])), C(5), A(7));
+    M m3(std::move(m1), A(7));
+    assert(m3 == m2);
+    assert(m3.get_allocator() == A(7));
+    assert(m3.key_comp() == C(5));
+    LIBCPP_ASSERT(m1.empty());
+  }
+  {
+    typedef std::pair<MoveOnly, MoveOnly> V;
+    typedef std::pair<const MoveOnly, MoveOnly> VC;
+    typedef test_less<MoveOnly> C;
+    typedef test_allocator<VC> A;
+    typedef std::map<MoveOnly, MoveOnly, C, A> M;
+    typedef std::move_iterator<V*> I;
+    V a1[] = {V(1, 1), V(1, 2), V(1, 3), V(2, 1), V(2, 2), V(2, 3), V(3, 1), V(3, 2), V(3, 3)};
+    M m1(I(a1), I(a1 + sizeof(a1) / sizeof(a1[0])), C(5), A(7));
+    V a2[] = {V(1, 1), V(1, 2), V(1, 3), V(2, 1), V(2, 2), V(2, 3), V(3, 1), V(3, 2), V(3, 3)};
+    M m2(I(a2), I(a2 + sizeof(a2) / sizeof(a2[0])), C(5), A(7));
+    M m3(std::move(m1), A(5));
+    assert(m3 == m2);
+    assert(m3.get_allocator() == A(5));
+    assert(m3.key_comp() == C(5));
+    LIBCPP_ASSERT(m1.empty());
+  }
+  {
+    typedef std::pair<MoveOnly, MoveOnly> V;
+    typedef std::pair<const MoveOnly, MoveOnly> VC;
+    typedef test_less<MoveOnly> C;
+    typedef other_allocator<VC> A;
+    typedef std::map<MoveOnly, MoveOnly, C, A> M;
+    typedef std::move_iterator<V*> I;
+    V a1[] = {V(1, 1), V(1, 2), V(1, 3), V(2, 1), V(2, 2), V(2, 3), V(3, 1), V(3, 2), V(3, 3)};
+    M m1(I(a1), I(a1 + sizeof(a1) / sizeof(a1[0])), C(5), A(7));
+    V a2[] = {V(1, 1), V(1, 2), V(1, 3), V(2, 1), V(2, 2), V(2, 3), V(3, 1), V(3, 2), V(3, 3)};
+    M m2(I(a2), I(a2 + sizeof(a2) / sizeof(a2[0])), C(5), A(7));
+    M m3(std::move(m1), A(5));
+    assert(m3 == m2);
+    assert(m3.get_allocator() == A(5));
+    assert(m3.key_comp() == C(5));
+    LIBCPP_ASSERT(m1.empty());
+  }
+  {
+    typedef Counter<int> T;
+    typedef std::pair<int, T> V;
+    typedef std::pair<const int, T> VC;
+    typedef test_allocator<VC> A;
+    typedef std::less<int> C;
+    typedef std::map<const int, T, C, A> M;
+    typedef V* I;
+    Counter_base::gConstructed = 0;
     {
-        typedef std::pair<MoveOnly, MoveOnly> V;
-        typedef std::pair<const MoveOnly, MoveOnly> VC;
-        typedef test_less<MoveOnly> C;
-        typedef test_allocator<VC> A;
-        typedef std::map<MoveOnly, MoveOnly, C, A> M;
-        typedef std::move_iterator<V*> I;
-        V a1[] =
-        {
-            V(1, 1),
-            V(1, 2),
-            V(1, 3),
-            V(2, 1),
-            V(2, 2),
-            V(2, 3),
-            V(3, 1),
-            V(3, 2),
-            V(3, 3)
-        };
-        M m1(I(a1), I(a1+sizeof(a1)/sizeof(a1[0])), C(5), A(7));
-        V a2[] =
-        {
-            V(1, 1),
-            V(1, 2),
-            V(1, 3),
-            V(2, 1),
-            V(2, 2),
-            V(2, 3),
-            V(3, 1),
-            V(3, 2),
-            V(3, 3)
-        };
-        M m2(I(a2), I(a2+sizeof(a2)/sizeof(a2[0])), C(5), A(7));
-        M m3(std::move(m1), A(7));
-        assert(m3 == m2);
-        assert(m3.get_allocator() == A(7));
-        assert(m3.key_comp() == C(5));
-        LIBCPP_ASSERT(m1.empty());
-    }
-    {
-        typedef std::pair<MoveOnly, MoveOnly> V;
-        typedef std::pair<const MoveOnly, MoveOnly> VC;
-        typedef test_less<MoveOnly> C;
-        typedef test_allocator<VC> A;
-        typedef std::map<MoveOnly, MoveOnly, C, A> M;
-        typedef std::move_iterator<V*> I;
-        V a1[] =
-        {
-            V(1, 1),
-            V(1, 2),
-            V(1, 3),
-            V(2, 1),
-            V(2, 2),
-            V(2, 3),
-            V(3, 1),
-            V(3, 2),
-            V(3, 3)
-        };
-        M m1(I(a1), I(a1+sizeof(a1)/sizeof(a1[0])), C(5), A(7));
-        V a2[] =
-        {
-            V(1, 1),
-            V(1, 2),
-            V(1, 3),
-            V(2, 1),
-            V(2, 2),
-            V(2, 3),
-            V(3, 1),
-            V(3, 2),
-            V(3, 3)
-        };
-        M m2(I(a2), I(a2+sizeof(a2)/sizeof(a2[0])), C(5), A(7));
-        M m3(std::move(m1), A(5));
-        assert(m3 == m2);
-        assert(m3.get_allocator() == A(5));
-        assert(m3.key_comp() == C(5));
-        LIBCPP_ASSERT(m1.empty());
-    }
-    {
-        typedef std::pair<MoveOnly, MoveOnly> V;
-        typedef std::pair<const MoveOnly, MoveOnly> VC;
-        typedef test_less<MoveOnly> C;
-        typedef other_allocator<VC> A;
-        typedef std::map<MoveOnly, MoveOnly, C, A> M;
-        typedef std::move_iterator<V*> I;
-        V a1[] =
-        {
-            V(1, 1),
-            V(1, 2),
-            V(1, 3),
-            V(2, 1),
-            V(2, 2),
-            V(2, 3),
-            V(3, 1),
-            V(3, 2),
-            V(3, 3)
-        };
-        M m1(I(a1), I(a1+sizeof(a1)/sizeof(a1[0])), C(5), A(7));
-        V a2[] =
-        {
-            V(1, 1),
-            V(1, 2),
-            V(1, 3),
-            V(2, 1),
-            V(2, 2),
-            V(2, 3),
-            V(3, 1),
-            V(3, 2),
-            V(3, 3)
-        };
-        M m2(I(a2), I(a2+sizeof(a2)/sizeof(a2[0])), C(5), A(7));
-        M m3(std::move(m1), A(5));
-        assert(m3 == m2);
-        assert(m3.get_allocator() == A(5));
-        assert(m3.key_comp() == C(5));
-        LIBCPP_ASSERT(m1.empty());
-    }
-    {
-        typedef Counter<int> T;
-        typedef std::pair<int, T> V;
-        typedef std::pair<const int, T> VC;
-        typedef test_allocator<VC> A;
-        typedef std::less<int> C;
-        typedef std::map<const int, T, C, A> M;
-        typedef V* I;
-        Counter_base::gConstructed = 0;
-        {
-            V a1[] =
-            {
-                V(1, 1),
-                V(1, 2),
-                V(1, 3),
-                V(2, 1),
-                V(2, 2),
-                V(2, 3),
-                V(3, 1),
-                V(3, 2),
-                V(3, 3)
-            };
-            const std::size_t num = sizeof(a1)/sizeof(a1[0]);
-            assert(Counter_base::gConstructed == num);
+      V a1[]                = {V(1, 1), V(1, 2), V(1, 3), V(2, 1), V(2, 2), V(2, 3), V(3, 1), V(3, 2), V(3, 3)};
+      const std::size_t num = sizeof(a1) / sizeof(a1[0]);
+      assert(Counter_base::gConstructed == num);
 
-            M m1(I(a1), I(a1+num), C(), A());
-            assert(Counter_base::gConstructed == num+3);
+      M m1(I(a1), I(a1 + num), C(), A());
+      assert(Counter_base::gConstructed == num + 3);
 
-            M m2(m1);
-            assert(m2 == m1);
-            assert(Counter_base::gConstructed == num+6);
+      M m2(m1);
+      assert(m2 == m1);
+      assert(Counter_base::gConstructed == num + 6);
 
-            M m3(std::move(m1), A());
-            assert(m3 == m2);
-            LIBCPP_ASSERT(m1.empty());
-            assert(Counter_base::gConstructed >= (int)(num+6));
-            assert(Counter_base::gConstructed <= (int)(num+6+m1.size()));
+      M m3(std::move(m1), A());
+      assert(m3 == m2);
+      LIBCPP_ASSERT(m1.empty());
+      assert(Counter_base::gConstructed >= (int)(num + 6));
+      assert(Counter_base::gConstructed <= (int)(num + 6 + m1.size()));
 
-            {
-            M m4(std::move(m2), A(5));
-            assert(Counter_base::gConstructed >= (int)(num+6));
-            assert(Counter_base::gConstructed <= (int)(num+6+m1.size()+m2.size()));
-            assert(m4 == m3);
-            LIBCPP_ASSERT(m2.empty());
-            }
-            assert(Counter_base::gConstructed >= (int)(num+3));
-            assert(Counter_base::gConstructed <= (int)(num+3+m1.size()+m2.size()));
-        }
-        assert(Counter_base::gConstructed == 0);
-    }
-    {
-        typedef std::pair<MoveOnly, MoveOnly> V;
-        typedef std::pair<const MoveOnly, MoveOnly> VC;
-        typedef test_less<MoveOnly> C;
-        typedef min_allocator<VC> A;
-        typedef std::map<MoveOnly, MoveOnly, C, A> M;
-        typedef std::move_iterator<V*> I;
-        V a1[] =
-        {
-            V(1, 1),
-            V(1, 2),
-            V(1, 3),
-            V(2, 1),
-            V(2, 2),
-            V(2, 3),
-            V(3, 1),
-            V(3, 2),
-            V(3, 3)
-        };
-        M m1(I(a1), I(a1+sizeof(a1)/sizeof(a1[0])), C(5), A());
-        V a2[] =
-        {
-            V(1, 1),
-            V(1, 2),
-            V(1, 3),
-            V(2, 1),
-            V(2, 2),
-            V(2, 3),
-            V(3, 1),
-            V(3, 2),
-            V(3, 3)
-        };
-        M m2(I(a2), I(a2+sizeof(a2)/sizeof(a2[0])), C(5), A());
-        M m3(std::move(m1), A());
-        assert(m3 == m2);
-        assert(m3.get_allocator() == A());
-        assert(m3.key_comp() == C(5));
-        LIBCPP_ASSERT(m1.empty());
-    }
-    {
-        typedef std::pair<MoveOnly, MoveOnly> V;
-        typedef std::pair<const MoveOnly, MoveOnly> VC;
-        typedef test_less<MoveOnly> C;
-        typedef explicit_allocator<VC> A;
-        typedef std::map<MoveOnly, MoveOnly, C, A> M;
-        typedef std::move_iterator<V*> I;
-        V a1[] =
-        {
-            V(1, 1),
-            V(1, 2),
-            V(1, 3),
-            V(2, 1),
-            V(2, 2),
-            V(2, 3),
-            V(3, 1),
-            V(3, 2),
-            V(3, 3)
-        };
-        M m1(I(a1), I(a1+sizeof(a1)/sizeof(a1[0])), C(5), A{});
-        V a2[] =
-        {
-            V(1, 1),
-            V(1, 2),
-            V(1, 3),
-            V(2, 1),
-            V(2, 2),
-            V(2, 3),
-            V(3, 1),
-            V(3, 2),
-            V(3, 3)
-        };
-        M m2(I(a2), I(a2+sizeof(a2)/sizeof(a2[0])), C(5), A{});
-        M m3(std::move(m1), A{});
-        assert(m3 == m2);
-        assert(m3.get_allocator() == A{});
-        assert(m3.key_comp() == C(5));
-        LIBCPP_ASSERT(m1.empty());
+      {
+        M m4(std::move(m2), A(5));
+        assert(Counter_base::gConstructed >= (int)(num + 6));
+        assert(Counter_base::gConstructed <= (int)(num + 6 + m1.size() + m2.size()));
+        assert(m4 == m3);
+        LIBCPP_ASSERT(m2.empty());
+      }
+      assert(Counter_base::gConstructed >= (int)(num + 3));
+      assert(Counter_base::gConstructed <= (int)(num + 3 + m1.size() + m2.size()));
     }
+    assert(Counter_base::gConstructed == 0);
+  }
+  {
+    typedef std::pair<MoveOnly, MoveOnly> V;
+    typedef std::pair<const MoveOnly, MoveOnly> VC;
+    typedef test_less<MoveOnly> C;
+    typedef min_allocator<VC> A;
+    typedef std::map<MoveOnly, MoveOnly, C, A> M;
+    typedef std::move_iterator<V*> I;
+    V a1[] = {V(1, 1), V(1, 2), V(1, 3), V(2, 1), V(2, 2), V(2, 3), V(3, 1), V(3, 2), V(3, 3)};
+    M m1(I(a1), I(a1 + sizeof(a1) / sizeof(a1[0])), C(5), A());
+    V a2[] = {V(1, 1), V(1, 2), V(1, 3), V(2, 1), V(2, 2), V(2, 3), V(3, 1), V(3, 2), V(3, 3)};
+    M m2(I(a2), I(a2 + sizeof(a2) / sizeof(a2[0])), C(5), A());
+    M m3(std::move(m1), A());
+    assert(m3 == m2);
+    assert(m3.get_allocator() == A());
+    assert(m3.key_comp() == C(5));
+    LIBCPP_ASSERT(m1.empty());
+  }
+  {
+    typedef std::pair<MoveOnly, MoveOnly> V;
+    typedef std::pair<const MoveOnly, MoveOnly> VC;
+    typedef test_less<MoveOnly> C;
+    typedef explicit_allocator<VC> A;
+    typedef std::map<MoveOnly, MoveOnly, C, A> M;
+    typedef std::move_iterator<V*> I;
+    V a1[] = {V(1, 1), V(1, 2), V(1, 3), V(2, 1), V(2, 2), V(2, 3), V(3, 1), V(3, 2), V(3, 3)};
+    M m1(I(a1), I(a1 + sizeof(a1) / sizeof(a1[0])), C(5), A{});
+    V a2[] = {V(1, 1), V(1, 2), V(1, 3), V(2, 1), V(2, 2), V(2, 3), V(3, 1), V(3, 2), V(3, 3)};
+    M m2(I(a2), I(a2 + sizeof(a2) / sizeof(a2[0])), C(5), A{});
+    M m3(std::move(m1), A{});
+    assert(m3 == m2);
+    assert(m3.get_allocator() == A{});
+    assert(m3.key_comp() == C(5));
+    LIBCPP_ASSERT(m1.empty());
+  }
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/associative/map/map.cons/move_assign.pass.cpp b/libcxx/test/std/containers/associative/map/map.cons/move_assign.pass.cpp
index c19fba5169b10..94f991976d6ad 100644
--- a/libcxx/test/std/containers/associative/map/map.cons/move_assign.pass.cpp
+++ b/libcxx/test/std/containers/associative/map/map.cons/move_assign.pass.cpp
@@ -23,168 +23,79 @@
 #include "test_allocator.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef std::pair<MoveOnly, MoveOnly> V;
-        typedef std::pair<const MoveOnly, MoveOnly> VC;
-        typedef test_less<MoveOnly> C;
-        typedef test_allocator<VC> A;
-        typedef std::map<MoveOnly, MoveOnly, C, A> M;
-        typedef std::move_iterator<V*> I;
-        V a1[] =
-        {
-            V(1, 1),
-            V(1, 2),
-            V(1, 3),
-            V(2, 1),
-            V(2, 2),
-            V(2, 3),
-            V(3, 1),
-            V(3, 2),
-            V(3, 3)
-        };
-        M m1(I(a1), I(a1+sizeof(a1)/sizeof(a1[0])), C(5), A(7));
-        V a2[] =
-        {
-            V(1, 1),
-            V(1, 2),
-            V(1, 3),
-            V(2, 1),
-            V(2, 2),
-            V(2, 3),
-            V(3, 1),
-            V(3, 2),
-            V(3, 3)
-        };
-        M m2(I(a2), I(a2+sizeof(a2)/sizeof(a2[0])), C(5), A(7));
-        M m3(C(3), A(7));
-        m3 = std::move(m1);
-        assert(m3 == m2);
-        assert(m3.get_allocator() == A(7));
-        assert(m3.key_comp() == C(5));
-        assert(m1.empty());
-    }
-    {
-        typedef std::pair<MoveOnly, MoveOnly> V;
-        typedef std::pair<const MoveOnly, MoveOnly> VC;
-        typedef test_less<MoveOnly> C;
-        typedef test_allocator<VC> A;
-        typedef std::map<MoveOnly, MoveOnly, C, A> M;
-        typedef std::move_iterator<V*> I;
-        V a1[] =
-        {
-            V(1, 1),
-            V(1, 2),
-            V(1, 3),
-            V(2, 1),
-            V(2, 2),
-            V(2, 3),
-            V(3, 1),
-            V(3, 2),
-            V(3, 3)
-        };
-        M m1(I(a1), I(a1+sizeof(a1)/sizeof(a1[0])), C(5), A(7));
-        V a2[] =
-        {
-            V(1, 1),
-            V(1, 2),
-            V(1, 3),
-            V(2, 1),
-            V(2, 2),
-            V(2, 3),
-            V(3, 1),
-            V(3, 2),
-            V(3, 3)
-        };
-        M m2(I(a2), I(a2+sizeof(a2)/sizeof(a2[0])), C(5), A(7));
-        M m3(C(3), A(5));
-        m3 = std::move(m1);
-        assert(m3 == m2);
-        assert(m3.get_allocator() == A(5));
-        assert(m3.key_comp() == C(5));
-        LIBCPP_ASSERT(m1.empty());
-    }
-    {
-        typedef std::pair<MoveOnly, MoveOnly> V;
-        typedef std::pair<const MoveOnly, MoveOnly> VC;
-        typedef test_less<MoveOnly> C;
-        typedef other_allocator<VC> A;
-        typedef std::map<MoveOnly, MoveOnly, C, A> M;
-        typedef std::move_iterator<V*> I;
-        V a1[] =
-        {
-            V(1, 1),
-            V(1, 2),
-            V(1, 3),
-            V(2, 1),
-            V(2, 2),
-            V(2, 3),
-            V(3, 1),
-            V(3, 2),
-            V(3, 3)
-        };
-        M m1(I(a1), I(a1+sizeof(a1)/sizeof(a1[0])), C(5), A(7));
-        V a2[] =
-        {
-            V(1, 1),
-            V(1, 2),
-            V(1, 3),
-            V(2, 1),
-            V(2, 2),
-            V(2, 3),
-            V(3, 1),
-            V(3, 2),
-            V(3, 3)
-        };
-        M m2(I(a2), I(a2+sizeof(a2)/sizeof(a2[0])), C(5), A(7));
-        M m3(C(3), A(5));
-        m3 = std::move(m1);
-        assert(m3 == m2);
-        assert(m3.get_allocator() == A(7));
-        assert(m3.key_comp() == C(5));
-        assert(m1.empty());
-    }
-    {
-        typedef std::pair<MoveOnly, MoveOnly> V;
-        typedef std::pair<const MoveOnly, MoveOnly> VC;
-        typedef test_less<MoveOnly> C;
-        typedef min_allocator<VC> A;
-        typedef std::map<MoveOnly, MoveOnly, C, A> M;
-        typedef std::move_iterator<V*> I;
-        V a1[] =
-        {
-            V(1, 1),
-            V(1, 2),
-            V(1, 3),
-            V(2, 1),
-            V(2, 2),
-            V(2, 3),
-            V(3, 1),
-            V(3, 2),
-            V(3, 3)
-        };
-        M m1(I(a1), I(a1+sizeof(a1)/sizeof(a1[0])), C(5), A());
-        V a2[] =
-        {
-            V(1, 1),
-            V(1, 2),
-            V(1, 3),
-            V(2, 1),
-            V(2, 2),
-            V(2, 3),
-            V(3, 1),
-            V(3, 2),
-            V(3, 3)
-        };
-        M m2(I(a2), I(a2+sizeof(a2)/sizeof(a2[0])), C(5), A());
-        M m3(C(3), A());
-        m3 = std::move(m1);
-        assert(m3 == m2);
-        assert(m3.get_allocator() == A());
-        assert(m3.key_comp() == C(5));
-        assert(m1.empty());
-    }
+int main(int, char**) {
+  {
+    typedef std::pair<MoveOnly, MoveOnly> V;
+    typedef std::pair<const MoveOnly, MoveOnly> VC;
+    typedef test_less<MoveOnly> C;
+    typedef test_allocator<VC> A;
+    typedef std::map<MoveOnly, MoveOnly, C, A> M;
+    typedef std::move_iterator<V*> I;
+    V a1[] = {V(1, 1), V(1, 2), V(1, 3), V(2, 1), V(2, 2), V(2, 3), V(3, 1), V(3, 2), V(3, 3)};
+    M m1(I(a1), I(a1 + sizeof(a1) / sizeof(a1[0])), C(5), A(7));
+    V a2[] = {V(1, 1), V(1, 2), V(1, 3), V(2, 1), V(2, 2), V(2, 3), V(3, 1), V(3, 2), V(3, 3)};
+    M m2(I(a2), I(a2 + sizeof(a2) / sizeof(a2[0])), C(5), A(7));
+    M m3(C(3), A(7));
+    m3 = std::move(m1);
+    assert(m3 == m2);
+    assert(m3.get_allocator() == A(7));
+    assert(m3.key_comp() == C(5));
+    assert(m1.empty());
+  }
+  {
+    typedef std::pair<MoveOnly, MoveOnly> V;
+    typedef std::pair<const MoveOnly, MoveOnly> VC;
+    typedef test_less<MoveOnly> C;
+    typedef test_allocator<VC> A;
+    typedef std::map<MoveOnly, MoveOnly, C, A> M;
+    typedef std::move_iterator<V*> I;
+    V a1[] = {V(1, 1), V(1, 2), V(1, 3), V(2, 1), V(2, 2), V(2, 3), V(3, 1), V(3, 2), V(3, 3)};
+    M m1(I(a1), I(a1 + sizeof(a1) / sizeof(a1[0])), C(5), A(7));
+    V a2[] = {V(1, 1), V(1, 2), V(1, 3), V(2, 1), V(2, 2), V(2, 3), V(3, 1), V(3, 2), V(3, 3)};
+    M m2(I(a2), I(a2 + sizeof(a2) / sizeof(a2[0])), C(5), A(7));
+    M m3(C(3), A(5));
+    m3 = std::move(m1);
+    assert(m3 == m2);
+    assert(m3.get_allocator() == A(5));
+    assert(m3.key_comp() == C(5));
+    LIBCPP_ASSERT(m1.empty());
+  }
+  {
+    typedef std::pair<MoveOnly, MoveOnly> V;
+    typedef std::pair<const MoveOnly, MoveOnly> VC;
+    typedef test_less<MoveOnly> C;
+    typedef other_allocator<VC> A;
+    typedef std::map<MoveOnly, MoveOnly, C, A> M;
+    typedef std::move_iterator<V*> I;
+    V a1[] = {V(1, 1), V(1, 2), V(1, 3), V(2, 1), V(2, 2), V(2, 3), V(3, 1), V(3, 2), V(3, 3)};
+    M m1(I(a1), I(a1 + sizeof(a1) / sizeof(a1[0])), C(5), A(7));
+    V a2[] = {V(1, 1), V(1, 2), V(1, 3), V(2, 1), V(2, 2), V(2, 3), V(3, 1), V(3, 2), V(3, 3)};
+    M m2(I(a2), I(a2 + sizeof(a2) / sizeof(a2[0])), C(5), A(7));
+    M m3(C(3), A(5));
+    m3 = std::move(m1);
+    assert(m3 == m2);
+    assert(m3.get_allocator() == A(7));
+    assert(m3.key_comp() == C(5));
+    assert(m1.empty());
+  }
+  {
+    typedef std::pair<MoveOnly, MoveOnly> V;
+    typedef std::pair<const MoveOnly, MoveOnly> VC;
+    typedef test_less<MoveOnly> C;
+    typedef min_allocator<VC> A;
+    typedef std::map<MoveOnly, MoveOnly, C, A> M;
+    typedef std::move_iterator<V*> I;
+    V a1[] = {V(1, 1), V(1, 2), V(1, 3), V(2, 1), V(2, 2), V(2, 3), V(3, 1), V(3, 2), V(3, 3)};
+    M m1(I(a1), I(a1 + sizeof(a1) / sizeof(a1[0])), C(5), A());
+    V a2[] = {V(1, 1), V(1, 2), V(1, 3), V(2, 1), V(2, 2), V(2, 3), V(3, 1), V(3, 2), V(3, 3)};
+    M m2(I(a2), I(a2 + sizeof(a2) / sizeof(a2[0])), C(5), A());
+    M m3(C(3), A());
+    m3 = std::move(m1);
+    assert(m3 == m2);
+    assert(m3.get_allocator() == A());
+    assert(m3.key_comp() == C(5));
+    assert(m1.empty());
+  }
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/associative/map/map.cons/move_assign_noexcept.compile.pass.cpp b/libcxx/test/std/containers/associative/map/map.cons/move_assign_noexcept.compile.pass.cpp
new file mode 100644
index 0000000000000..a4c8ef1c5b429
--- /dev/null
+++ b/libcxx/test/std/containers/associative/map/map.cons/move_assign_noexcept.compile.pass.cpp
@@ -0,0 +1,61 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// <map>
+
+// map& operator=(map&& c)
+//     noexcept(
+//          allocator_type::propagate_on_container_move_assignment::value &&
+//          is_nothrow_move_assignable<allocator_type>::value &&
+//          is_nothrow_move_assignable<key_compare>::value);
+
+// This tests a conforming extension
+
+// UNSUPPORTED: c++03
+
+#include <map>
+
+#include "test_macros.h"
+#include "MoveOnly.h"
+#include "test_allocator.h"
+
+template <class T>
+struct some_comp {
+  using value_type = T;
+  some_comp& operator=(const some_comp&);
+  bool operator()(const T&, const T&) const { return false; }
+};
+
+template <class T>
+struct always_equal_alloc {
+  using value_type = T;
+  always_equal_alloc(const always_equal_alloc&);
+  void allocate(std::size_t);
+};
+
+template <class T>
+struct not_always_equal_alloc {
+  int i;
+  using value_type = T;
+  not_always_equal_alloc(const not_always_equal_alloc&);
+  void allocate(std::size_t);
+};
+
+template <template <class> class Alloc>
+using multimap_alloc = std::map<MoveOnly, MoveOnly, std::less<MoveOnly>, Alloc<std::pair<const MoveOnly, MoveOnly>>>;
+
+static_assert(std::is_nothrow_move_assignable<multimap_alloc<std::allocator>>::value, "");
+static_assert(!std::is_nothrow_move_assignable<multimap_alloc<test_allocator>>::value, "");
+#if TEST_STD_VER >= 17
+static_assert(std::is_nothrow_move_assignable<multimap_alloc<always_equal_alloc>>::value, "");
+#endif
+static_assert(!std::is_nothrow_move_assignable<multimap_alloc<not_always_equal_alloc>>::value, "");
+#if defined(_LIBCPP_VERSION)
+static_assert(std::is_nothrow_move_assignable<multimap_alloc<other_allocator>>::value, "");
+#endif // _LIBCPP_VERSION
+static_assert(!std::is_nothrow_move_assignable<std::map<int, int, some_comp<int>>>::value, "");
diff --git a/libcxx/test/std/containers/associative/map/map.cons/move_assign_noexcept.pass.cpp b/libcxx/test/std/containers/associative/map/map.cons/move_assign_noexcept.pass.cpp
deleted file mode 100644
index 82a5dbac12610..0000000000000
--- a/libcxx/test/std/containers/associative/map/map.cons/move_assign_noexcept.pass.cpp
+++ /dev/null
@@ -1,59 +0,0 @@
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-// <map>
-
-// map& operator=(map&& c)
-//     noexcept(
-//          allocator_type::propagate_on_container_move_assignment::value &&
-//          is_nothrow_move_assignable<allocator_type>::value &&
-//          is_nothrow_move_assignable<key_compare>::value);
-
-// This tests a conforming extension
-
-// UNSUPPORTED: c++03
-
-#include <map>
-#include <cassert>
-
-#include "test_macros.h"
-#include "MoveOnly.h"
-#include "test_allocator.h"
-
-template <class T>
-struct some_comp
-{
-    typedef T value_type;
-    some_comp& operator=(const some_comp&);
-    bool operator()(const T&, const T&) const { return false; }
-};
-
-int main(int, char**)
-{
-    typedef std::pair<const MoveOnly, MoveOnly> V;
-    {
-        typedef std::map<MoveOnly, MoveOnly> C;
-        static_assert(std::is_nothrow_move_assignable<C>::value, "");
-    }
-    {
-        typedef std::map<MoveOnly, MoveOnly, std::less<MoveOnly>, test_allocator<V>> C;
-        static_assert(!std::is_nothrow_move_assignable<C>::value, "");
-    }
-#if defined(_LIBCPP_VERSION)
-    {
-        typedef std::map<MoveOnly, MoveOnly, std::less<MoveOnly>, other_allocator<V>> C;
-        static_assert(std::is_nothrow_move_assignable<C>::value, "");
-    }
-#endif // _LIBCPP_VERSION
-    {
-        typedef std::map<MoveOnly, MoveOnly, some_comp<MoveOnly>> C;
-        static_assert(!std::is_nothrow_move_assignable<C>::value, "");
-    }
-
-  return 0;
-}
diff --git a/libcxx/test/std/containers/associative/map/map.cons/move_noexcept.pass.cpp b/libcxx/test/std/containers/associative/map/map.cons/move_noexcept.pass.cpp
index 5454d1c750c78..06316ccf0901a 100644
--- a/libcxx/test/std/containers/associative/map/map.cons/move_noexcept.pass.cpp
+++ b/libcxx/test/std/containers/associative/map/map.cons/move_noexcept.pass.cpp
@@ -24,34 +24,32 @@
 #include "test_allocator.h"
 
 template <class T>
-struct some_comp
-{
-    typedef T value_type;
-    some_comp(const some_comp&);
-    bool operator()(const T&, const T&) const { return false; }
+struct some_comp {
+  typedef T value_type;
+  some_comp(const some_comp&);
+  bool operator()(const T&, const T&) const { return false; }
 };
 
-int main(int, char**)
-{
+int main(int, char**) {
 #if defined(_LIBCPP_VERSION)
-    typedef std::pair<const MoveOnly, MoveOnly> V;
-    {
-        typedef std::map<MoveOnly, MoveOnly> C;
-        static_assert(std::is_nothrow_move_constructible<C>::value, "");
-    }
-    {
-        typedef std::map<MoveOnly, MoveOnly, std::less<MoveOnly>, test_allocator<V>> C;
-        static_assert(std::is_nothrow_move_constructible<C>::value, "");
-    }
-    {
-        typedef std::map<MoveOnly, MoveOnly, std::less<MoveOnly>, other_allocator<V>> C;
-        static_assert(std::is_nothrow_move_constructible<C>::value, "");
-    }
+  typedef std::pair<const MoveOnly, MoveOnly> V;
+  {
+    typedef std::map<MoveOnly, MoveOnly> C;
+    static_assert(std::is_nothrow_move_constructible<C>::value, "");
+  }
+  {
+    typedef std::map<MoveOnly, MoveOnly, std::less<MoveOnly>, test_allocator<V>> C;
+    static_assert(std::is_nothrow_move_constructible<C>::value, "");
+  }
+  {
+    typedef std::map<MoveOnly, MoveOnly, std::less<MoveOnly>, other_allocator<V>> C;
+    static_assert(std::is_nothrow_move_constructible<C>::value, "");
+  }
 #endif // _LIBCPP_VERSION
-    {
-        typedef std::map<MoveOnly, MoveOnly, some_comp<MoveOnly>> C;
-        static_assert(!std::is_nothrow_move_constructible<C>::value, "");
-    }
+  {
+    typedef std::map<MoveOnly, MoveOnly, some_comp<MoveOnly>> C;
+    static_assert(!std::is_nothrow_move_constructible<C>::value, "");
+  }
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/associative/map/map.erasure/erase_if.pass.cpp b/libcxx/test/std/containers/associative/map/map.erasure/erase_if.pass.cpp
index 6dfbbfea848af..ccfd800cafde0 100644
--- a/libcxx/test/std/containers/associative/map/map.erasure/erase_if.pass.cpp
+++ b/libcxx/test/std/containers/associative/map/map.erasure/erase_if.pass.cpp
@@ -21,12 +21,11 @@
 
 using Init = std::initializer_list<int>;
 template <typename M>
-M make (Init vals)
-{
-    M ret;
-    for (int v : vals)
-        ret[static_cast<typename M::key_type>(v)] = static_cast<typename M::mapped_type>(v + 10);
-    return ret;
+M make(Init vals) {
+  M ret;
+  for (int v : vals)
+    ret[static_cast<typename M::key_type>(v)] = static_cast<typename M::mapped_type>(v + 10);
+  return ret;
 }
 
 template <typename M, typename Pred>
@@ -38,41 +37,39 @@ void test0(Init vals, Pred p, Init expected, std::size_t expected_erased_count)
 }
 
 template <typename S>
-void test()
-{
-    auto is1 = [](auto v) { return v.first == 1;};
-    auto is2 = [](auto v) { return v.first == 2;};
-    auto is3 = [](auto v) { return v.first == 3;};
-    auto is4 = [](auto v) { return v.first == 4;};
-    auto True  = [](auto) { return true; };
-    auto False = [](auto) { return false; };
+void test() {
+  auto is1   = [](auto v) { return v.first == 1; };
+  auto is2   = [](auto v) { return v.first == 2; };
+  auto is3   = [](auto v) { return v.first == 3; };
+  auto is4   = [](auto v) { return v.first == 4; };
+  auto True  = [](auto) { return true; };
+  auto False = [](auto) { return false; };
 
-    test0<S>({}, is1, {}, 0);
+  test0<S>({}, is1, {}, 0);
 
-    test0<S>({1}, is1, {}, 1);
-    test0<S>({1}, is2, {1}, 0);
+  test0<S>({1}, is1, {}, 1);
+  test0<S>({1}, is2, {1}, 0);
 
-    test0<S>({1, 2}, is1, {2}, 1);
-    test0<S>({1, 2}, is2, {1}, 1);
-    test0<S>({1, 2}, is3, {1, 2}, 0);
+  test0<S>({1, 2}, is1, {2}, 1);
+  test0<S>({1, 2}, is2, {1}, 1);
+  test0<S>({1, 2}, is3, {1, 2}, 0);
 
-    test0<S>({1, 2, 3}, is1, {2, 3}, 1);
-    test0<S>({1, 2, 3}, is2, {1, 3}, 1);
-    test0<S>({1, 2, 3}, is3, {1, 2}, 1);
-    test0<S>({1, 2, 3}, is4, {1, 2, 3}, 0);
+  test0<S>({1, 2, 3}, is1, {2, 3}, 1);
+  test0<S>({1, 2, 3}, is2, {1, 3}, 1);
+  test0<S>({1, 2, 3}, is3, {1, 2}, 1);
+  test0<S>({1, 2, 3}, is4, {1, 2, 3}, 0);
 
-    test0<S>({1, 2, 3}, True, {}, 3);
-    test0<S>({1, 2, 3}, False, {1, 2, 3}, 0);
+  test0<S>({1, 2, 3}, True, {}, 3);
+  test0<S>({1, 2, 3}, False, {1, 2, 3}, 0);
 }
 
-int main(int, char**)
-{
-    test<std::map<int, int>>();
-    test<std::map<int, int, std::less<int>, min_allocator<std::pair<const int, int>>>> ();
-    test<std::map<int, int, std::less<int>, test_allocator<std::pair<const int, int>>>> ();
+int main(int, char**) {
+  test<std::map<int, int>>();
+  test<std::map<int, int, std::less<int>, min_allocator<std::pair<const int, int>>>>();
+  test<std::map<int, int, std::less<int>, test_allocator<std::pair<const int, int>>>>();
 
-    test<std::map<long, short>>();
-    test<std::map<short, double>>();
+  test<std::map<long, short>>();
+  test<std::map<short, double>>();
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/associative/map/map.modifiers/clear.pass.cpp b/libcxx/test/std/containers/associative/map/map.modifiers/clear.pass.cpp
index 5c6d00e9f3a8a..938f22eb880a6 100644
--- a/libcxx/test/std/containers/associative/map/map.modifiers/clear.pass.cpp
+++ b/libcxx/test/std/containers/associative/map/map.modifiers/clear.pass.cpp
@@ -18,49 +18,46 @@
 #include "test_macros.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef std::map<int, double> M;
-        typedef std::pair<int, double> P;
-        P ar[] =
-        {
-            P(1, 1.5),
-            P(2, 2.5),
-            P(3, 3.5),
-            P(4, 4.5),
-            P(5, 5.5),
-            P(6, 6.5),
-            P(7, 7.5),
-            P(8, 8.5),
-        };
-        M m(ar, ar + sizeof(ar)/sizeof(ar[0]));
-        assert(m.size() == 8);
-        ASSERT_NOEXCEPT(m.clear());
-        m.clear();
-        assert(m.size() == 0);
-    }
+int main(int, char**) {
+  {
+    typedef std::map<int, double> M;
+    typedef std::pair<int, double> P;
+    P ar[] = {
+        P(1, 1.5),
+        P(2, 2.5),
+        P(3, 3.5),
+        P(4, 4.5),
+        P(5, 5.5),
+        P(6, 6.5),
+        P(7, 7.5),
+        P(8, 8.5),
+    };
+    M m(ar, ar + sizeof(ar) / sizeof(ar[0]));
+    assert(m.size() == 8);
+    ASSERT_NOEXCEPT(m.clear());
+    m.clear();
+    assert(m.size() == 0);
+  }
 #if TEST_STD_VER >= 11
-    {
-        typedef std::map<int, double, std::less<int>, min_allocator<std::pair<const int, double>>> M;
-        typedef std::pair<int, double> P;
-        P ar[] =
-        {
-            P(1, 1.5),
-            P(2, 2.5),
-            P(3, 3.5),
-            P(4, 4.5),
-            P(5, 5.5),
-            P(6, 6.5),
-            P(7, 7.5),
-            P(8, 8.5),
-        };
-        M m(ar, ar + sizeof(ar)/sizeof(ar[0]));
-        assert(m.size() == 8);
-        ASSERT_NOEXCEPT(m.clear());
-        m.clear();
-        assert(m.size() == 0);
-    }
+  {
+    typedef std::map<int, double, std::less<int>, min_allocator<std::pair<const int, double>>> M;
+    typedef std::pair<int, double> P;
+    P ar[] = {
+        P(1, 1.5),
+        P(2, 2.5),
+        P(3, 3.5),
+        P(4, 4.5),
+        P(5, 5.5),
+        P(6, 6.5),
+        P(7, 7.5),
+        P(8, 8.5),
+    };
+    M m(ar, ar + sizeof(ar) / sizeof(ar[0]));
+    assert(m.size() == 8);
+    ASSERT_NOEXCEPT(m.clear());
+    m.clear();
+    assert(m.size() == 0);
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/associative/map/map.modifiers/emplace.pass.cpp b/libcxx/test/std/containers/associative/map/map.modifiers/emplace.pass.cpp
index f57269c3c5698..36b073c2b551e 100644
--- a/libcxx/test/std/containers/associative/map/map.modifiers/emplace.pass.cpp
+++ b/libcxx/test/std/containers/associative/map/map.modifiers/emplace.pass.cpp
@@ -24,142 +24,131 @@
 #include "DefaultOnly.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef std::map<int, DefaultOnly> M;
-        typedef std::pair<M::iterator, bool> R;
-        M m;
-        assert(DefaultOnly::count == 0);
-        R r = m.emplace();
-        assert(r.second);
-        assert(r.first == m.begin());
-        assert(m.size() == 1);
-        assert(m.begin()->first == 0);
-        assert(m.begin()->second == DefaultOnly());
-        assert(DefaultOnly::count == 1);
-        r = m.emplace(std::piecewise_construct, std::forward_as_tuple(1),
-                                                std::forward_as_tuple());
-        assert(r.second);
-        assert(r.first == std::next(m.begin()));
-        assert(m.size() == 2);
-        assert(std::next(m.begin())->first == 1);
-        assert(std::next(m.begin())->second == DefaultOnly());
-        assert(DefaultOnly::count == 2);
-        r = m.emplace(std::piecewise_construct, std::forward_as_tuple(1),
-                                                std::forward_as_tuple());
-        assert(!r.second);
-        assert(r.first == std::next(m.begin()));
-        assert(m.size() == 2);
-        assert(std::next(m.begin())->first == 1);
-        assert(std::next(m.begin())->second == DefaultOnly());
-        assert(DefaultOnly::count == 2);
-    }
+int main(int, char**) {
+  {
+    typedef std::map<int, DefaultOnly> M;
+    typedef std::pair<M::iterator, bool> R;
+    M m;
     assert(DefaultOnly::count == 0);
-    {
-        typedef std::map<int, Emplaceable> M;
-        typedef std::pair<M::iterator, bool> R;
-        M m;
-        R r = m.emplace(std::piecewise_construct, std::forward_as_tuple(2),
-                                                  std::forward_as_tuple());
-        assert(r.second);
-        assert(r.first == m.begin());
-        assert(m.size() == 1);
-        assert(m.begin()->first == 2);
-        assert(m.begin()->second == Emplaceable());
-        r = m.emplace(std::piecewise_construct, std::forward_as_tuple(1),
-                                                std::forward_as_tuple(2, 3.5));
-        assert(r.second);
-        assert(r.first == m.begin());
-        assert(m.size() == 2);
-        assert(m.begin()->first == 1);
-        assert(m.begin()->second == Emplaceable(2, 3.5));
-        r = m.emplace(std::piecewise_construct, std::forward_as_tuple(1),
-                                                std::forward_as_tuple(2, 3.5));
-        assert(!r.second);
-        assert(r.first == m.begin());
-        assert(m.size() == 2);
-        assert(m.begin()->first == 1);
-        assert(m.begin()->second == Emplaceable(2, 3.5));
-    }
-    {
-        typedef std::map<int, double> M;
-        typedef std::pair<M::iterator, bool> R;
-        M m;
-        R r = m.emplace(M::value_type(2, 3.5));
-        assert(r.second);
-        assert(r.first == m.begin());
-        assert(m.size() == 1);
-        assert(m.begin()->first == 2);
-        assert(m.begin()->second == 3.5);
-    }
-    {
-        typedef std::map<int, DefaultOnly, std::less<int>, min_allocator<std::pair<const int, DefaultOnly>>> M;
-        typedef std::pair<M::iterator, bool> R;
-        M m;
-        assert(DefaultOnly::count == 0);
-        R r = m.emplace();
-        assert(r.second);
-        assert(r.first == m.begin());
-        assert(m.size() == 1);
-        assert(m.begin()->first == 0);
-        assert(m.begin()->second == DefaultOnly());
-        assert(DefaultOnly::count == 1);
-        r = m.emplace(std::piecewise_construct, std::forward_as_tuple(1),
-                                                std::forward_as_tuple());
-        assert(r.second);
-        assert(r.first == std::next(m.begin()));
-        assert(m.size() == 2);
-        assert(std::next(m.begin())->first == 1);
-        assert(std::next(m.begin())->second == DefaultOnly());
-        assert(DefaultOnly::count == 2);
-        r = m.emplace(std::piecewise_construct, std::forward_as_tuple(1),
-                                                std::forward_as_tuple());
-        assert(!r.second);
-        assert(r.first == std::next(m.begin()));
-        assert(m.size() == 2);
-        assert(std::next(m.begin())->first == 1);
-        assert(std::next(m.begin())->second == DefaultOnly());
-        assert(DefaultOnly::count == 2);
-    }
+    R r = m.emplace();
+    assert(r.second);
+    assert(r.first == m.begin());
+    assert(m.size() == 1);
+    assert(m.begin()->first == 0);
+    assert(m.begin()->second == DefaultOnly());
+    assert(DefaultOnly::count == 1);
+    r = m.emplace(std::piecewise_construct, std::forward_as_tuple(1), std::forward_as_tuple());
+    assert(r.second);
+    assert(r.first == std::next(m.begin()));
+    assert(m.size() == 2);
+    assert(std::next(m.begin())->first == 1);
+    assert(std::next(m.begin())->second == DefaultOnly());
+    assert(DefaultOnly::count == 2);
+    r = m.emplace(std::piecewise_construct, std::forward_as_tuple(1), std::forward_as_tuple());
+    assert(!r.second);
+    assert(r.first == std::next(m.begin()));
+    assert(m.size() == 2);
+    assert(std::next(m.begin())->first == 1);
+    assert(std::next(m.begin())->second == DefaultOnly());
+    assert(DefaultOnly::count == 2);
+  }
+  assert(DefaultOnly::count == 0);
+  {
+    typedef std::map<int, Emplaceable> M;
+    typedef std::pair<M::iterator, bool> R;
+    M m;
+    R r = m.emplace(std::piecewise_construct, std::forward_as_tuple(2), std::forward_as_tuple());
+    assert(r.second);
+    assert(r.first == m.begin());
+    assert(m.size() == 1);
+    assert(m.begin()->first == 2);
+    assert(m.begin()->second == Emplaceable());
+    r = m.emplace(std::piecewise_construct, std::forward_as_tuple(1), std::forward_as_tuple(2, 3.5));
+    assert(r.second);
+    assert(r.first == m.begin());
+    assert(m.size() == 2);
+    assert(m.begin()->first == 1);
+    assert(m.begin()->second == Emplaceable(2, 3.5));
+    r = m.emplace(std::piecewise_construct, std::forward_as_tuple(1), std::forward_as_tuple(2, 3.5));
+    assert(!r.second);
+    assert(r.first == m.begin());
+    assert(m.size() == 2);
+    assert(m.begin()->first == 1);
+    assert(m.begin()->second == Emplaceable(2, 3.5));
+  }
+  {
+    typedef std::map<int, double> M;
+    typedef std::pair<M::iterator, bool> R;
+    M m;
+    R r = m.emplace(M::value_type(2, 3.5));
+    assert(r.second);
+    assert(r.first == m.begin());
+    assert(m.size() == 1);
+    assert(m.begin()->first == 2);
+    assert(m.begin()->second == 3.5);
+  }
+  {
+    typedef std::map<int, DefaultOnly, std::less<int>, min_allocator<std::pair<const int, DefaultOnly>>> M;
+    typedef std::pair<M::iterator, bool> R;
+    M m;
     assert(DefaultOnly::count == 0);
-    {
-        typedef std::map<int, Emplaceable, std::less<int>, min_allocator<std::pair<const int, Emplaceable>>> M;
-        typedef std::pair<M::iterator, bool> R;
-        M m;
-        R r = m.emplace(std::piecewise_construct, std::forward_as_tuple(2),
-                                                  std::forward_as_tuple());
-        assert(r.second);
-        assert(r.first == m.begin());
-        assert(m.size() == 1);
-        assert(m.begin()->first == 2);
-        assert(m.begin()->second == Emplaceable());
-        r = m.emplace(std::piecewise_construct, std::forward_as_tuple(1),
-                                                std::forward_as_tuple(2, 3.5));
-        assert(r.second);
-        assert(r.first == m.begin());
-        assert(m.size() == 2);
-        assert(m.begin()->first == 1);
-        assert(m.begin()->second == Emplaceable(2, 3.5));
-        r = m.emplace(std::piecewise_construct, std::forward_as_tuple(1),
-                                                std::forward_as_tuple(2, 3.5));
-        assert(!r.second);
-        assert(r.first == m.begin());
-        assert(m.size() == 2);
-        assert(m.begin()->first == 1);
-        assert(m.begin()->second == Emplaceable(2, 3.5));
-    }
-    {
-        typedef std::map<int, double, std::less<int>, min_allocator<std::pair<const int, double>>> M;
-        typedef std::pair<M::iterator, bool> R;
-        M m;
-        R r = m.emplace(M::value_type(2, 3.5));
-        assert(r.second);
-        assert(r.first == m.begin());
-        assert(m.size() == 1);
-        assert(m.begin()->first == 2);
-        assert(m.begin()->second == 3.5);
-    }
+    R r = m.emplace();
+    assert(r.second);
+    assert(r.first == m.begin());
+    assert(m.size() == 1);
+    assert(m.begin()->first == 0);
+    assert(m.begin()->second == DefaultOnly());
+    assert(DefaultOnly::count == 1);
+    r = m.emplace(std::piecewise_construct, std::forward_as_tuple(1), std::forward_as_tuple());
+    assert(r.second);
+    assert(r.first == std::next(m.begin()));
+    assert(m.size() == 2);
+    assert(std::next(m.begin())->first == 1);
+    assert(std::next(m.begin())->second == DefaultOnly());
+    assert(DefaultOnly::count == 2);
+    r = m.emplace(std::piecewise_construct, std::forward_as_tuple(1), std::forward_as_tuple());
+    assert(!r.second);
+    assert(r.first == std::next(m.begin()));
+    assert(m.size() == 2);
+    assert(std::next(m.begin())->first == 1);
+    assert(std::next(m.begin())->second == DefaultOnly());
+    assert(DefaultOnly::count == 2);
+  }
+  assert(DefaultOnly::count == 0);
+  {
+    typedef std::map<int, Emplaceable, std::less<int>, min_allocator<std::pair<const int, Emplaceable>>> M;
+    typedef std::pair<M::iterator, bool> R;
+    M m;
+    R r = m.emplace(std::piecewise_construct, std::forward_as_tuple(2), std::forward_as_tuple());
+    assert(r.second);
+    assert(r.first == m.begin());
+    assert(m.size() == 1);
+    assert(m.begin()->first == 2);
+    assert(m.begin()->second == Emplaceable());
+    r = m.emplace(std::piecewise_construct, std::forward_as_tuple(1), std::forward_as_tuple(2, 3.5));
+    assert(r.second);
+    assert(r.first == m.begin());
+    assert(m.size() == 2);
+    assert(m.begin()->first == 1);
+    assert(m.begin()->second == Emplaceable(2, 3.5));
+    r = m.emplace(std::piecewise_construct, std::forward_as_tuple(1), std::forward_as_tuple(2, 3.5));
+    assert(!r.second);
+    assert(r.first == m.begin());
+    assert(m.size() == 2);
+    assert(m.begin()->first == 1);
+    assert(m.begin()->second == Emplaceable(2, 3.5));
+  }
+  {
+    typedef std::map<int, double, std::less<int>, min_allocator<std::pair<const int, double>>> M;
+    typedef std::pair<M::iterator, bool> R;
+    M m;
+    R r = m.emplace(M::value_type(2, 3.5));
+    assert(r.second);
+    assert(r.first == m.begin());
+    assert(m.size() == 1);
+    assert(m.begin()->first == 2);
+    assert(m.begin()->second == 3.5);
+  }
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/associative/map/map.modifiers/emplace_hint.pass.cpp b/libcxx/test/std/containers/associative/map/map.modifiers/emplace_hint.pass.cpp
index 85353fc43d754..56d1c5c2b90ac 100644
--- a/libcxx/test/std/containers/associative/map/map.modifiers/emplace_hint.pass.cpp
+++ b/libcxx/test/std/containers/associative/map/map.modifiers/emplace_hint.pass.cpp
@@ -23,138 +23,117 @@
 #include "DefaultOnly.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef std::map<int, DefaultOnly> M;
-        typedef M::iterator R;
-        M m;
-        assert(DefaultOnly::count == 0);
-        R r = m.emplace_hint(m.end());
-        assert(r == m.begin());
-        assert(m.size() == 1);
-        assert(m.begin()->first == 0);
-        assert(m.begin()->second == DefaultOnly());
-        assert(DefaultOnly::count == 1);
-        r = m.emplace_hint(m.end(), std::piecewise_construct,
-                                       std::forward_as_tuple(1),
-                                       std::forward_as_tuple());
-        assert(r == std::next(m.begin()));
-        assert(m.size() == 2);
-        assert(std::next(m.begin())->first == 1);
-        assert(std::next(m.begin())->second == DefaultOnly());
-        assert(DefaultOnly::count == 2);
-        r = m.emplace_hint(m.end(), std::piecewise_construct,
-                                       std::forward_as_tuple(1),
-                                       std::forward_as_tuple());
-        assert(r == std::next(m.begin()));
-        assert(m.size() == 2);
-        assert(std::next(m.begin())->first == 1);
-        assert(std::next(m.begin())->second == DefaultOnly());
-        assert(DefaultOnly::count == 2);
-    }
+int main(int, char**) {
+  {
+    typedef std::map<int, DefaultOnly> M;
+    typedef M::iterator R;
+    M m;
     assert(DefaultOnly::count == 0);
-    {
-        typedef std::map<int, Emplaceable> M;
-        typedef M::iterator R;
-        M m;
-        R r = m.emplace_hint(m.end(), std::piecewise_construct,
-                                       std::forward_as_tuple(2),
-                                       std::forward_as_tuple());
-        assert(r == m.begin());
-        assert(m.size() == 1);
-        assert(m.begin()->first == 2);
-        assert(m.begin()->second == Emplaceable());
-        r = m.emplace_hint(m.end(), std::piecewise_construct,
-                                    std::forward_as_tuple(1),
-                                    std::forward_as_tuple(2, 3.5));
-        assert(r == m.begin());
-        assert(m.size() == 2);
-        assert(m.begin()->first == 1);
-        assert(m.begin()->second == Emplaceable(2, 3.5));
-        r = m.emplace_hint(m.end(), std::piecewise_construct,
-                                    std::forward_as_tuple(1),
-                                    std::forward_as_tuple(2, 3.5));
-        assert(r == m.begin());
-        assert(m.size() == 2);
-        assert(m.begin()->first == 1);
-        assert(m.begin()->second == Emplaceable(2, 3.5));
-    }
-    {
-        typedef std::map<int, double> M;
-        typedef M::iterator R;
-        M m;
-        R r = m.emplace_hint(m.end(), M::value_type(2, 3.5));
-        assert(r == m.begin());
-        assert(m.size() == 1);
-        assert(m.begin()->first == 2);
-        assert(m.begin()->second == 3.5);
-    }
-    {
-        typedef std::map<int, DefaultOnly, std::less<int>, min_allocator<std::pair<const int, DefaultOnly>>> M;
-        typedef M::iterator R;
-        M m;
-        assert(DefaultOnly::count == 0);
-        R r = m.emplace_hint(m.end());
-        assert(r == m.begin());
-        assert(m.size() == 1);
-        assert(m.begin()->first == 0);
-        assert(m.begin()->second == DefaultOnly());
-        assert(DefaultOnly::count == 1);
-        r = m.emplace_hint(m.end(), std::piecewise_construct,
-                                       std::forward_as_tuple(1),
-                                       std::forward_as_tuple());
-        assert(r == std::next(m.begin()));
-        assert(m.size() == 2);
-        assert(std::next(m.begin())->first == 1);
-        assert(std::next(m.begin())->second == DefaultOnly());
-        assert(DefaultOnly::count == 2);
-        r = m.emplace_hint(m.end(), std::piecewise_construct,
-                                       std::forward_as_tuple(1),
-                                       std::forward_as_tuple());
-        assert(r == std::next(m.begin()));
-        assert(m.size() == 2);
-        assert(std::next(m.begin())->first == 1);
-        assert(std::next(m.begin())->second == DefaultOnly());
-        assert(DefaultOnly::count == 2);
-    }
+    R r = m.emplace_hint(m.end());
+    assert(r == m.begin());
+    assert(m.size() == 1);
+    assert(m.begin()->first == 0);
+    assert(m.begin()->second == DefaultOnly());
+    assert(DefaultOnly::count == 1);
+    r = m.emplace_hint(m.end(), std::piecewise_construct, std::forward_as_tuple(1), std::forward_as_tuple());
+    assert(r == std::next(m.begin()));
+    assert(m.size() == 2);
+    assert(std::next(m.begin())->first == 1);
+    assert(std::next(m.begin())->second == DefaultOnly());
+    assert(DefaultOnly::count == 2);
+    r = m.emplace_hint(m.end(), std::piecewise_construct, std::forward_as_tuple(1), std::forward_as_tuple());
+    assert(r == std::next(m.begin()));
+    assert(m.size() == 2);
+    assert(std::next(m.begin())->first == 1);
+    assert(std::next(m.begin())->second == DefaultOnly());
+    assert(DefaultOnly::count == 2);
+  }
+  assert(DefaultOnly::count == 0);
+  {
+    typedef std::map<int, Emplaceable> M;
+    typedef M::iterator R;
+    M m;
+    R r = m.emplace_hint(m.end(), std::piecewise_construct, std::forward_as_tuple(2), std::forward_as_tuple());
+    assert(r == m.begin());
+    assert(m.size() == 1);
+    assert(m.begin()->first == 2);
+    assert(m.begin()->second == Emplaceable());
+    r = m.emplace_hint(m.end(), std::piecewise_construct, std::forward_as_tuple(1), std::forward_as_tuple(2, 3.5));
+    assert(r == m.begin());
+    assert(m.size() == 2);
+    assert(m.begin()->first == 1);
+    assert(m.begin()->second == Emplaceable(2, 3.5));
+    r = m.emplace_hint(m.end(), std::piecewise_construct, std::forward_as_tuple(1), std::forward_as_tuple(2, 3.5));
+    assert(r == m.begin());
+    assert(m.size() == 2);
+    assert(m.begin()->first == 1);
+    assert(m.begin()->second == Emplaceable(2, 3.5));
+  }
+  {
+    typedef std::map<int, double> M;
+    typedef M::iterator R;
+    M m;
+    R r = m.emplace_hint(m.end(), M::value_type(2, 3.5));
+    assert(r == m.begin());
+    assert(m.size() == 1);
+    assert(m.begin()->first == 2);
+    assert(m.begin()->second == 3.5);
+  }
+  {
+    typedef std::map<int, DefaultOnly, std::less<int>, min_allocator<std::pair<const int, DefaultOnly>>> M;
+    typedef M::iterator R;
+    M m;
     assert(DefaultOnly::count == 0);
-    {
-        typedef std::map<int, Emplaceable, std::less<int>, min_allocator<std::pair<const int, Emplaceable>>> M;
-        typedef M::iterator R;
-        M m;
-        R r = m.emplace_hint(m.end(), std::piecewise_construct,
-                                       std::forward_as_tuple(2),
-                                       std::forward_as_tuple());
-        assert(r == m.begin());
-        assert(m.size() == 1);
-        assert(m.begin()->first == 2);
-        assert(m.begin()->second == Emplaceable());
-        r = m.emplace_hint(m.end(), std::piecewise_construct,
-                                    std::forward_as_tuple(1),
-                                    std::forward_as_tuple(2, 3.5));
-        assert(r == m.begin());
-        assert(m.size() == 2);
-        assert(m.begin()->first == 1);
-        assert(m.begin()->second == Emplaceable(2, 3.5));
-        r = m.emplace_hint(m.end(), std::piecewise_construct,
-                                    std::forward_as_tuple(1),
-                                    std::forward_as_tuple(2, 3.5));
-        assert(r == m.begin());
-        assert(m.size() == 2);
-        assert(m.begin()->first == 1);
-        assert(m.begin()->second == Emplaceable(2, 3.5));
-    }
-    {
-        typedef std::map<int, double, std::less<int>, min_allocator<std::pair<const int, double>>> M;
-        typedef M::iterator R;
-        M m;
-        R r = m.emplace_hint(m.end(), M::value_type(2, 3.5));
-        assert(r == m.begin());
-        assert(m.size() == 1);
-        assert(m.begin()->first == 2);
-        assert(m.begin()->second == 3.5);
-    }
+    R r = m.emplace_hint(m.end());
+    assert(r == m.begin());
+    assert(m.size() == 1);
+    assert(m.begin()->first == 0);
+    assert(m.begin()->second == DefaultOnly());
+    assert(DefaultOnly::count == 1);
+    r = m.emplace_hint(m.end(), std::piecewise_construct, std::forward_as_tuple(1), std::forward_as_tuple());
+    assert(r == std::next(m.begin()));
+    assert(m.size() == 2);
+    assert(std::next(m.begin())->first == 1);
+    assert(std::next(m.begin())->second == DefaultOnly());
+    assert(DefaultOnly::count == 2);
+    r = m.emplace_hint(m.end(), std::piecewise_construct, std::forward_as_tuple(1), std::forward_as_tuple());
+    assert(r == std::next(m.begin()));
+    assert(m.size() == 2);
+    assert(std::next(m.begin())->first == 1);
+    assert(std::next(m.begin())->second == DefaultOnly());
+    assert(DefaultOnly::count == 2);
+  }
+  assert(DefaultOnly::count == 0);
+  {
+    typedef std::map<int, Emplaceable, std::less<int>, min_allocator<std::pair<const int, Emplaceable>>> M;
+    typedef M::iterator R;
+    M m;
+    R r = m.emplace_hint(m.end(), std::piecewise_construct, std::forward_as_tuple(2), std::forward_as_tuple());
+    assert(r == m.begin());
+    assert(m.size() == 1);
+    assert(m.begin()->first == 2);
+    assert(m.begin()->second == Emplaceable());
+    r = m.emplace_hint(m.end(), std::piecewise_construct, std::forward_as_tuple(1), std::forward_as_tuple(2, 3.5));
+    assert(r == m.begin());
+    assert(m.size() == 2);
+    assert(m.begin()->first == 1);
+    assert(m.begin()->second == Emplaceable(2, 3.5));
+    r = m.emplace_hint(m.end(), std::piecewise_construct, std::forward_as_tuple(1), std::forward_as_tuple(2, 3.5));
+    assert(r == m.begin());
+    assert(m.size() == 2);
+    assert(m.begin()->first == 1);
+    assert(m.begin()->second == Emplaceable(2, 3.5));
+  }
+  {
+    typedef std::map<int, double, std::less<int>, min_allocator<std::pair<const int, double>>> M;
+    typedef M::iterator R;
+    M m;
+    R r = m.emplace_hint(m.end(), M::value_type(2, 3.5));
+    assert(r == m.begin());
+    assert(m.size() == 1);
+    assert(m.begin()->first == 2);
+    assert(m.begin()->second == 3.5);
+  }
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/associative/map/map.modifiers/erase_iter.pass.cpp b/libcxx/test/std/containers/associative/map/map.modifiers/erase_iter.pass.cpp
index 4486ca65d306e..9a0cf9bd21cee 100644
--- a/libcxx/test/std/containers/associative/map/map.modifiers/erase_iter.pass.cpp
+++ b/libcxx/test/std/containers/associative/map/map.modifiers/erase_iter.pass.cpp
@@ -18,243 +18,239 @@
 #include "test_macros.h"
 #include "min_allocator.h"
 
-struct TemplateConstructor
-{
-    template<typename T>
-    TemplateConstructor (const T&) {}
+struct TemplateConstructor {
+  template <typename T>
+  TemplateConstructor(const T&) {}
 };
 
 bool operator<(const TemplateConstructor&, const TemplateConstructor&) { return false; }
 
-int main(int, char**)
-{
-    {
-        typedef std::map<int, double> M;
-        typedef std::pair<int, double> P;
-        typedef M::iterator I;
-        P ar[] =
-        {
-            P(1, 1.5),
-            P(2, 2.5),
-            P(3, 3.5),
-            P(4, 4.5),
-            P(5, 5.5),
-            P(6, 6.5),
-            P(7, 7.5),
-            P(8, 8.5),
-        };
-        M m(ar, ar + sizeof(ar)/sizeof(ar[0]));
-        assert(m.size() == 8);
-        I i = m.erase(std::next(m.cbegin(), 3));
-        assert(m.size() == 7);
-        assert(i == std::next(m.begin(), 3));
-        assert(m.begin()->first == 1);
-        assert(m.begin()->second == 1.5);
-        assert(std::next(m.begin())->first == 2);
-        assert(std::next(m.begin())->second == 2.5);
-        assert(std::next(m.begin(), 2)->first == 3);
-        assert(std::next(m.begin(), 2)->second == 3.5);
-        assert(std::next(m.begin(), 3)->first == 5);
-        assert(std::next(m.begin(), 3)->second == 5.5);
-        assert(std::next(m.begin(), 4)->first == 6);
-        assert(std::next(m.begin(), 4)->second == 6.5);
-        assert(std::next(m.begin(), 5)->first == 7);
-        assert(std::next(m.begin(), 5)->second == 7.5);
-        assert(std::next(m.begin(), 6)->first == 8);
-        assert(std::next(m.begin(), 6)->second == 8.5);
+int main(int, char**) {
+  {
+    typedef std::map<int, double> M;
+    typedef std::pair<int, double> P;
+    typedef M::iterator I;
+    P ar[] = {
+        P(1, 1.5),
+        P(2, 2.5),
+        P(3, 3.5),
+        P(4, 4.5),
+        P(5, 5.5),
+        P(6, 6.5),
+        P(7, 7.5),
+        P(8, 8.5),
+    };
+    M m(ar, ar + sizeof(ar) / sizeof(ar[0]));
+    assert(m.size() == 8);
+    I i = m.erase(std::next(m.cbegin(), 3));
+    assert(m.size() == 7);
+    assert(i == std::next(m.begin(), 3));
+    assert(m.begin()->first == 1);
+    assert(m.begin()->second == 1.5);
+    assert(std::next(m.begin())->first == 2);
+    assert(std::next(m.begin())->second == 2.5);
+    assert(std::next(m.begin(), 2)->first == 3);
+    assert(std::next(m.begin(), 2)->second == 3.5);
+    assert(std::next(m.begin(), 3)->first == 5);
+    assert(std::next(m.begin(), 3)->second == 5.5);
+    assert(std::next(m.begin(), 4)->first == 6);
+    assert(std::next(m.begin(), 4)->second == 6.5);
+    assert(std::next(m.begin(), 5)->first == 7);
+    assert(std::next(m.begin(), 5)->second == 7.5);
+    assert(std::next(m.begin(), 6)->first == 8);
+    assert(std::next(m.begin(), 6)->second == 8.5);
 
-        i = m.erase(std::next(m.cbegin(), 0));
-        assert(m.size() == 6);
-        assert(i == m.begin());
-        assert(m.begin()->first == 2);
-        assert(m.begin()->second == 2.5);
-        assert(std::next(m.begin())->first == 3);
-        assert(std::next(m.begin())->second == 3.5);
-        assert(std::next(m.begin(), 2)->first == 5);
-        assert(std::next(m.begin(), 2)->second == 5.5);
-        assert(std::next(m.begin(), 3)->first == 6);
-        assert(std::next(m.begin(), 3)->second == 6.5);
-        assert(std::next(m.begin(), 4)->first == 7);
-        assert(std::next(m.begin(), 4)->second == 7.5);
-        assert(std::next(m.begin(), 5)->first == 8);
-        assert(std::next(m.begin(), 5)->second == 8.5);
+    i = m.erase(std::next(m.cbegin(), 0));
+    assert(m.size() == 6);
+    assert(i == m.begin());
+    assert(m.begin()->first == 2);
+    assert(m.begin()->second == 2.5);
+    assert(std::next(m.begin())->first == 3);
+    assert(std::next(m.begin())->second == 3.5);
+    assert(std::next(m.begin(), 2)->first == 5);
+    assert(std::next(m.begin(), 2)->second == 5.5);
+    assert(std::next(m.begin(), 3)->first == 6);
+    assert(std::next(m.begin(), 3)->second == 6.5);
+    assert(std::next(m.begin(), 4)->first == 7);
+    assert(std::next(m.begin(), 4)->second == 7.5);
+    assert(std::next(m.begin(), 5)->first == 8);
+    assert(std::next(m.begin(), 5)->second == 8.5);
 
-        i = m.erase(std::next(m.cbegin(), 5));
-        assert(m.size() == 5);
-        assert(i == m.end());
-        assert(m.begin()->first == 2);
-        assert(m.begin()->second == 2.5);
-        assert(std::next(m.begin())->first == 3);
-        assert(std::next(m.begin())->second == 3.5);
-        assert(std::next(m.begin(), 2)->first == 5);
-        assert(std::next(m.begin(), 2)->second == 5.5);
-        assert(std::next(m.begin(), 3)->first == 6);
-        assert(std::next(m.begin(), 3)->second == 6.5);
-        assert(std::next(m.begin(), 4)->first == 7);
-        assert(std::next(m.begin(), 4)->second == 7.5);
+    i = m.erase(std::next(m.cbegin(), 5));
+    assert(m.size() == 5);
+    assert(i == m.end());
+    assert(m.begin()->first == 2);
+    assert(m.begin()->second == 2.5);
+    assert(std::next(m.begin())->first == 3);
+    assert(std::next(m.begin())->second == 3.5);
+    assert(std::next(m.begin(), 2)->first == 5);
+    assert(std::next(m.begin(), 2)->second == 5.5);
+    assert(std::next(m.begin(), 3)->first == 6);
+    assert(std::next(m.begin(), 3)->second == 6.5);
+    assert(std::next(m.begin(), 4)->first == 7);
+    assert(std::next(m.begin(), 4)->second == 7.5);
 
-        i = m.erase(std::next(m.cbegin(), 1));
-        assert(m.size() == 4);
-        assert(i == std::next(m.begin()));
-        assert(m.begin()->first == 2);
-        assert(m.begin()->second == 2.5);
-        assert(std::next(m.begin())->first == 5);
-        assert(std::next(m.begin())->second == 5.5);
-        assert(std::next(m.begin(), 2)->first == 6);
-        assert(std::next(m.begin(), 2)->second == 6.5);
-        assert(std::next(m.begin(), 3)->first == 7);
-        assert(std::next(m.begin(), 3)->second == 7.5);
+    i = m.erase(std::next(m.cbegin(), 1));
+    assert(m.size() == 4);
+    assert(i == std::next(m.begin()));
+    assert(m.begin()->first == 2);
+    assert(m.begin()->second == 2.5);
+    assert(std::next(m.begin())->first == 5);
+    assert(std::next(m.begin())->second == 5.5);
+    assert(std::next(m.begin(), 2)->first == 6);
+    assert(std::next(m.begin(), 2)->second == 6.5);
+    assert(std::next(m.begin(), 3)->first == 7);
+    assert(std::next(m.begin(), 3)->second == 7.5);
 
-        i = m.erase(std::next(m.cbegin(), 2));
-        assert(m.size() == 3);
-        assert(i == std::next(m.begin(), 2));
-        assert(m.begin()->first == 2);
-        assert(m.begin()->second == 2.5);
-        assert(std::next(m.begin())->first == 5);
-        assert(std::next(m.begin())->second == 5.5);
-        assert(std::next(m.begin(), 2)->first == 7);
-        assert(std::next(m.begin(), 2)->second == 7.5);
+    i = m.erase(std::next(m.cbegin(), 2));
+    assert(m.size() == 3);
+    assert(i == std::next(m.begin(), 2));
+    assert(m.begin()->first == 2);
+    assert(m.begin()->second == 2.5);
+    assert(std::next(m.begin())->first == 5);
+    assert(std::next(m.begin())->second == 5.5);
+    assert(std::next(m.begin(), 2)->first == 7);
+    assert(std::next(m.begin(), 2)->second == 7.5);
 
-        i = m.erase(std::next(m.cbegin(), 2));
-        assert(m.size() == 2);
-        assert(i == std::next(m.begin(), 2));
-        assert(m.begin()->first == 2);
-        assert(m.begin()->second == 2.5);
-        assert(std::next(m.begin())->first == 5);
-        assert(std::next(m.begin())->second == 5.5);
+    i = m.erase(std::next(m.cbegin(), 2));
+    assert(m.size() == 2);
+    assert(i == std::next(m.begin(), 2));
+    assert(m.begin()->first == 2);
+    assert(m.begin()->second == 2.5);
+    assert(std::next(m.begin())->first == 5);
+    assert(std::next(m.begin())->second == 5.5);
 
-        i = m.erase(std::next(m.cbegin(), 0));
-        assert(m.size() == 1);
-        assert(i == std::next(m.begin(), 0));
-        assert(m.begin()->first == 5);
-        assert(m.begin()->second == 5.5);
+    i = m.erase(std::next(m.cbegin(), 0));
+    assert(m.size() == 1);
+    assert(i == std::next(m.begin(), 0));
+    assert(m.begin()->first == 5);
+    assert(m.begin()->second == 5.5);
 
-        i = m.erase(m.cbegin());
-        assert(m.size() == 0);
-        assert(i == m.begin());
-        assert(i == m.end());
-    }
+    i = m.erase(m.cbegin());
+    assert(m.size() == 0);
+    assert(i == m.begin());
+    assert(i == m.end());
+  }
 #if TEST_STD_VER >= 11
-    {
-        typedef std::map<int, double, std::less<int>, min_allocator<std::pair<const int, double>>> M;
-        typedef std::pair<int, double> P;
-        typedef M::iterator I;
-        P ar[] =
-        {
-            P(1, 1.5),
-            P(2, 2.5),
-            P(3, 3.5),
-            P(4, 4.5),
-            P(5, 5.5),
-            P(6, 6.5),
-            P(7, 7.5),
-            P(8, 8.5),
-        };
-        M m(ar, ar + sizeof(ar)/sizeof(ar[0]));
-        assert(m.size() == 8);
-        I i = m.erase(std::next(m.cbegin(), 3));
-        assert(m.size() == 7);
-        assert(i == std::next(m.begin(), 3));
-        assert(m.begin()->first == 1);
-        assert(m.begin()->second == 1.5);
-        assert(std::next(m.begin())->first == 2);
-        assert(std::next(m.begin())->second == 2.5);
-        assert(std::next(m.begin(), 2)->first == 3);
-        assert(std::next(m.begin(), 2)->second == 3.5);
-        assert(std::next(m.begin(), 3)->first == 5);
-        assert(std::next(m.begin(), 3)->second == 5.5);
-        assert(std::next(m.begin(), 4)->first == 6);
-        assert(std::next(m.begin(), 4)->second == 6.5);
-        assert(std::next(m.begin(), 5)->first == 7);
-        assert(std::next(m.begin(), 5)->second == 7.5);
-        assert(std::next(m.begin(), 6)->first == 8);
-        assert(std::next(m.begin(), 6)->second == 8.5);
+  {
+    typedef std::map<int, double, std::less<int>, min_allocator<std::pair<const int, double>>> M;
+    typedef std::pair<int, double> P;
+    typedef M::iterator I;
+    P ar[] = {
+        P(1, 1.5),
+        P(2, 2.5),
+        P(3, 3.5),
+        P(4, 4.5),
+        P(5, 5.5),
+        P(6, 6.5),
+        P(7, 7.5),
+        P(8, 8.5),
+    };
+    M m(ar, ar + sizeof(ar) / sizeof(ar[0]));
+    assert(m.size() == 8);
+    I i = m.erase(std::next(m.cbegin(), 3));
+    assert(m.size() == 7);
+    assert(i == std::next(m.begin(), 3));
+    assert(m.begin()->first == 1);
+    assert(m.begin()->second == 1.5);
+    assert(std::next(m.begin())->first == 2);
+    assert(std::next(m.begin())->second == 2.5);
+    assert(std::next(m.begin(), 2)->first == 3);
+    assert(std::next(m.begin(), 2)->second == 3.5);
+    assert(std::next(m.begin(), 3)->first == 5);
+    assert(std::next(m.begin(), 3)->second == 5.5);
+    assert(std::next(m.begin(), 4)->first == 6);
+    assert(std::next(m.begin(), 4)->second == 6.5);
+    assert(std::next(m.begin(), 5)->first == 7);
+    assert(std::next(m.begin(), 5)->second == 7.5);
+    assert(std::next(m.begin(), 6)->first == 8);
+    assert(std::next(m.begin(), 6)->second == 8.5);
 
-        i = m.erase(std::next(m.cbegin(), 0));
-        assert(m.size() == 6);
-        assert(i == m.begin());
-        assert(m.begin()->first == 2);
-        assert(m.begin()->second == 2.5);
-        assert(std::next(m.begin())->first == 3);
-        assert(std::next(m.begin())->second == 3.5);
-        assert(std::next(m.begin(), 2)->first == 5);
-        assert(std::next(m.begin(), 2)->second == 5.5);
-        assert(std::next(m.begin(), 3)->first == 6);
-        assert(std::next(m.begin(), 3)->second == 6.5);
-        assert(std::next(m.begin(), 4)->first == 7);
-        assert(std::next(m.begin(), 4)->second == 7.5);
-        assert(std::next(m.begin(), 5)->first == 8);
-        assert(std::next(m.begin(), 5)->second == 8.5);
+    i = m.erase(std::next(m.cbegin(), 0));
+    assert(m.size() == 6);
+    assert(i == m.begin());
+    assert(m.begin()->first == 2);
+    assert(m.begin()->second == 2.5);
+    assert(std::next(m.begin())->first == 3);
+    assert(std::next(m.begin())->second == 3.5);
+    assert(std::next(m.begin(), 2)->first == 5);
+    assert(std::next(m.begin(), 2)->second == 5.5);
+    assert(std::next(m.begin(), 3)->first == 6);
+    assert(std::next(m.begin(), 3)->second == 6.5);
+    assert(std::next(m.begin(), 4)->first == 7);
+    assert(std::next(m.begin(), 4)->second == 7.5);
+    assert(std::next(m.begin(), 5)->first == 8);
+    assert(std::next(m.begin(), 5)->second == 8.5);
 
-        i = m.erase(std::next(m.cbegin(), 5));
-        assert(m.size() == 5);
-        assert(i == m.end());
-        assert(m.begin()->first == 2);
-        assert(m.begin()->second == 2.5);
-        assert(std::next(m.begin())->first == 3);
-        assert(std::next(m.begin())->second == 3.5);
-        assert(std::next(m.begin(), 2)->first == 5);
-        assert(std::next(m.begin(), 2)->second == 5.5);
-        assert(std::next(m.begin(), 3)->first == 6);
-        assert(std::next(m.begin(), 3)->second == 6.5);
-        assert(std::next(m.begin(), 4)->first == 7);
-        assert(std::next(m.begin(), 4)->second == 7.5);
+    i = m.erase(std::next(m.cbegin(), 5));
+    assert(m.size() == 5);
+    assert(i == m.end());
+    assert(m.begin()->first == 2);
+    assert(m.begin()->second == 2.5);
+    assert(std::next(m.begin())->first == 3);
+    assert(std::next(m.begin())->second == 3.5);
+    assert(std::next(m.begin(), 2)->first == 5);
+    assert(std::next(m.begin(), 2)->second == 5.5);
+    assert(std::next(m.begin(), 3)->first == 6);
+    assert(std::next(m.begin(), 3)->second == 6.5);
+    assert(std::next(m.begin(), 4)->first == 7);
+    assert(std::next(m.begin(), 4)->second == 7.5);
 
-        i = m.erase(std::next(m.cbegin(), 1));
-        assert(m.size() == 4);
-        assert(i == std::next(m.begin()));
-        assert(m.begin()->first == 2);
-        assert(m.begin()->second == 2.5);
-        assert(std::next(m.begin())->first == 5);
-        assert(std::next(m.begin())->second == 5.5);
-        assert(std::next(m.begin(), 2)->first == 6);
-        assert(std::next(m.begin(), 2)->second == 6.5);
-        assert(std::next(m.begin(), 3)->first == 7);
-        assert(std::next(m.begin(), 3)->second == 7.5);
+    i = m.erase(std::next(m.cbegin(), 1));
+    assert(m.size() == 4);
+    assert(i == std::next(m.begin()));
+    assert(m.begin()->first == 2);
+    assert(m.begin()->second == 2.5);
+    assert(std::next(m.begin())->first == 5);
+    assert(std::next(m.begin())->second == 5.5);
+    assert(std::next(m.begin(), 2)->first == 6);
+    assert(std::next(m.begin(), 2)->second == 6.5);
+    assert(std::next(m.begin(), 3)->first == 7);
+    assert(std::next(m.begin(), 3)->second == 7.5);
 
-        i = m.erase(std::next(m.cbegin(), 2));
-        assert(m.size() == 3);
-        assert(i == std::next(m.begin(), 2));
-        assert(m.begin()->first == 2);
-        assert(m.begin()->second == 2.5);
-        assert(std::next(m.begin())->first == 5);
-        assert(std::next(m.begin())->second == 5.5);
-        assert(std::next(m.begin(), 2)->first == 7);
-        assert(std::next(m.begin(), 2)->second == 7.5);
+    i = m.erase(std::next(m.cbegin(), 2));
+    assert(m.size() == 3);
+    assert(i == std::next(m.begin(), 2));
+    assert(m.begin()->first == 2);
+    assert(m.begin()->second == 2.5);
+    assert(std::next(m.begin())->first == 5);
+    assert(std::next(m.begin())->second == 5.5);
+    assert(std::next(m.begin(), 2)->first == 7);
+    assert(std::next(m.begin(), 2)->second == 7.5);
 
-        i = m.erase(std::next(m.cbegin(), 2));
-        assert(m.size() == 2);
-        assert(i == std::next(m.begin(), 2));
-        assert(m.begin()->first == 2);
-        assert(m.begin()->second == 2.5);
-        assert(std::next(m.begin())->first == 5);
-        assert(std::next(m.begin())->second == 5.5);
+    i = m.erase(std::next(m.cbegin(), 2));
+    assert(m.size() == 2);
+    assert(i == std::next(m.begin(), 2));
+    assert(m.begin()->first == 2);
+    assert(m.begin()->second == 2.5);
+    assert(std::next(m.begin())->first == 5);
+    assert(std::next(m.begin())->second == 5.5);
 
-        i = m.erase(std::next(m.cbegin(), 0));
-        assert(m.size() == 1);
-        assert(i == std::next(m.begin(), 0));
-        assert(m.begin()->first == 5);
-        assert(m.begin()->second == 5.5);
+    i = m.erase(std::next(m.cbegin(), 0));
+    assert(m.size() == 1);
+    assert(i == std::next(m.begin(), 0));
+    assert(m.begin()->first == 5);
+    assert(m.begin()->second == 5.5);
 
-        i = m.erase(m.cbegin());
-        assert(m.size() == 0);
-        assert(i == m.begin());
-        assert(i == m.end());
-    }
+    i = m.erase(m.cbegin());
+    assert(m.size() == 0);
+    assert(i == m.begin());
+    assert(i == m.end());
+  }
 #endif
 #if TEST_STD_VER >= 14
-    {
+  {
     //  This is LWG #2059
-        typedef TemplateConstructor T;
-        typedef std::map<T, int> C;
-        typedef C::iterator I;
+    typedef TemplateConstructor T;
+    typedef std::map<T, int> C;
+    typedef C::iterator I;
 
-        C c;
-        T a{0};
-        I it = c.find(a);
-        if (it != c.end())
-            c.erase(it);
-    }
+    C c;
+    T a{0};
+    I it = c.find(a);
+    if (it != c.end())
+      c.erase(it);
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/associative/map/map.modifiers/erase_iter_iter.pass.cpp b/libcxx/test/std/containers/associative/map/map.modifiers/erase_iter_iter.pass.cpp
index 0189b1cfa1652..671f114951fc4 100644
--- a/libcxx/test/std/containers/associative/map/map.modifiers/erase_iter_iter.pass.cpp
+++ b/libcxx/test/std/containers/associative/map/map.modifiers/erase_iter_iter.pass.cpp
@@ -18,141 +18,138 @@
 #include "test_macros.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef std::map<int, double> M;
-        typedef std::pair<int, double> P;
-        typedef M::iterator I;
-        P ar[] =
-        {
-            P(1, 1.5),
-            P(2, 2.5),
-            P(3, 3.5),
-            P(4, 4.5),
-            P(5, 5.5),
-            P(6, 6.5),
-            P(7, 7.5),
-            P(8, 8.5),
-        };
-        M m(ar, ar + sizeof(ar)/sizeof(ar[0]));
-        assert(m.size() == 8);
-        I i = m.erase(m.cbegin(), m.cbegin());
-        assert(m.size() == 8);
-        assert(i == m.begin());
-        assert(m.begin()->first == 1);
-        assert(m.begin()->second == 1.5);
-        assert(std::next(m.begin())->first == 2);
-        assert(std::next(m.begin())->second == 2.5);
-        assert(std::next(m.begin(), 2)->first == 3);
-        assert(std::next(m.begin(), 2)->second == 3.5);
-        assert(std::next(m.begin(), 3)->first == 4);
-        assert(std::next(m.begin(), 3)->second == 4.5);
-        assert(std::next(m.begin(), 4)->first == 5);
-        assert(std::next(m.begin(), 4)->second == 5.5);
-        assert(std::next(m.begin(), 5)->first == 6);
-        assert(std::next(m.begin(), 5)->second == 6.5);
-        assert(std::next(m.begin(), 6)->first == 7);
-        assert(std::next(m.begin(), 6)->second == 7.5);
-        assert(std::next(m.begin(), 7)->first == 8);
-        assert(std::next(m.begin(), 7)->second == 8.5);
+int main(int, char**) {
+  {
+    typedef std::map<int, double> M;
+    typedef std::pair<int, double> P;
+    typedef M::iterator I;
+    P ar[] = {
+        P(1, 1.5),
+        P(2, 2.5),
+        P(3, 3.5),
+        P(4, 4.5),
+        P(5, 5.5),
+        P(6, 6.5),
+        P(7, 7.5),
+        P(8, 8.5),
+    };
+    M m(ar, ar + sizeof(ar) / sizeof(ar[0]));
+    assert(m.size() == 8);
+    I i = m.erase(m.cbegin(), m.cbegin());
+    assert(m.size() == 8);
+    assert(i == m.begin());
+    assert(m.begin()->first == 1);
+    assert(m.begin()->second == 1.5);
+    assert(std::next(m.begin())->first == 2);
+    assert(std::next(m.begin())->second == 2.5);
+    assert(std::next(m.begin(), 2)->first == 3);
+    assert(std::next(m.begin(), 2)->second == 3.5);
+    assert(std::next(m.begin(), 3)->first == 4);
+    assert(std::next(m.begin(), 3)->second == 4.5);
+    assert(std::next(m.begin(), 4)->first == 5);
+    assert(std::next(m.begin(), 4)->second == 5.5);
+    assert(std::next(m.begin(), 5)->first == 6);
+    assert(std::next(m.begin(), 5)->second == 6.5);
+    assert(std::next(m.begin(), 6)->first == 7);
+    assert(std::next(m.begin(), 6)->second == 7.5);
+    assert(std::next(m.begin(), 7)->first == 8);
+    assert(std::next(m.begin(), 7)->second == 8.5);
 
-        i = m.erase(m.cbegin(), std::next(m.cbegin(), 2));
-        assert(m.size() == 6);
-        assert(i == m.begin());
-        assert(std::next(m.begin(), 0)->first == 3);
-        assert(std::next(m.begin(), 0)->second == 3.5);
-        assert(std::next(m.begin(), 1)->first == 4);
-        assert(std::next(m.begin(), 1)->second == 4.5);
-        assert(std::next(m.begin(), 2)->first == 5);
-        assert(std::next(m.begin(), 2)->second == 5.5);
-        assert(std::next(m.begin(), 3)->first == 6);
-        assert(std::next(m.begin(), 3)->second == 6.5);
-        assert(std::next(m.begin(), 4)->first == 7);
-        assert(std::next(m.begin(), 4)->second == 7.5);
-        assert(std::next(m.begin(), 5)->first == 8);
-        assert(std::next(m.begin(), 5)->second == 8.5);
+    i = m.erase(m.cbegin(), std::next(m.cbegin(), 2));
+    assert(m.size() == 6);
+    assert(i == m.begin());
+    assert(std::next(m.begin(), 0)->first == 3);
+    assert(std::next(m.begin(), 0)->second == 3.5);
+    assert(std::next(m.begin(), 1)->first == 4);
+    assert(std::next(m.begin(), 1)->second == 4.5);
+    assert(std::next(m.begin(), 2)->first == 5);
+    assert(std::next(m.begin(), 2)->second == 5.5);
+    assert(std::next(m.begin(), 3)->first == 6);
+    assert(std::next(m.begin(), 3)->second == 6.5);
+    assert(std::next(m.begin(), 4)->first == 7);
+    assert(std::next(m.begin(), 4)->second == 7.5);
+    assert(std::next(m.begin(), 5)->first == 8);
+    assert(std::next(m.begin(), 5)->second == 8.5);
 
-        i = m.erase(std::next(m.cbegin(), 2), std::next(m.cbegin(), 6));
-        assert(m.size() == 2);
-        assert(i == std::next(m.begin(), 2));
-        assert(std::next(m.begin(), 0)->first == 3);
-        assert(std::next(m.begin(), 0)->second == 3.5);
-        assert(std::next(m.begin(), 1)->first == 4);
-        assert(std::next(m.begin(), 1)->second == 4.5);
+    i = m.erase(std::next(m.cbegin(), 2), std::next(m.cbegin(), 6));
+    assert(m.size() == 2);
+    assert(i == std::next(m.begin(), 2));
+    assert(std::next(m.begin(), 0)->first == 3);
+    assert(std::next(m.begin(), 0)->second == 3.5);
+    assert(std::next(m.begin(), 1)->first == 4);
+    assert(std::next(m.begin(), 1)->second == 4.5);
 
-        i = m.erase(m.cbegin(), m.cend());
-        assert(m.size() == 0);
-        assert(i == m.begin());
-        assert(i == m.end());
-    }
+    i = m.erase(m.cbegin(), m.cend());
+    assert(m.size() == 0);
+    assert(i == m.begin());
+    assert(i == m.end());
+  }
 #if TEST_STD_VER >= 11
-    {
-        typedef std::map<int, double, std::less<int>, min_allocator<std::pair<const int, double>>> M;
-        typedef std::pair<int, double> P;
-        typedef M::iterator I;
-        P ar[] =
-        {
-            P(1, 1.5),
-            P(2, 2.5),
-            P(3, 3.5),
-            P(4, 4.5),
-            P(5, 5.5),
-            P(6, 6.5),
-            P(7, 7.5),
-            P(8, 8.5),
-        };
-        M m(ar, ar + sizeof(ar)/sizeof(ar[0]));
-        assert(m.size() == 8);
-        I i = m.erase(m.cbegin(), m.cbegin());
-        assert(m.size() == 8);
-        assert(i == m.begin());
-        assert(m.begin()->first == 1);
-        assert(m.begin()->second == 1.5);
-        assert(std::next(m.begin())->first == 2);
-        assert(std::next(m.begin())->second == 2.5);
-        assert(std::next(m.begin(), 2)->first == 3);
-        assert(std::next(m.begin(), 2)->second == 3.5);
-        assert(std::next(m.begin(), 3)->first == 4);
-        assert(std::next(m.begin(), 3)->second == 4.5);
-        assert(std::next(m.begin(), 4)->first == 5);
-        assert(std::next(m.begin(), 4)->second == 5.5);
-        assert(std::next(m.begin(), 5)->first == 6);
-        assert(std::next(m.begin(), 5)->second == 6.5);
-        assert(std::next(m.begin(), 6)->first == 7);
-        assert(std::next(m.begin(), 6)->second == 7.5);
-        assert(std::next(m.begin(), 7)->first == 8);
-        assert(std::next(m.begin(), 7)->second == 8.5);
+  {
+    typedef std::map<int, double, std::less<int>, min_allocator<std::pair<const int, double>>> M;
+    typedef std::pair<int, double> P;
+    typedef M::iterator I;
+    P ar[] = {
+        P(1, 1.5),
+        P(2, 2.5),
+        P(3, 3.5),
+        P(4, 4.5),
+        P(5, 5.5),
+        P(6, 6.5),
+        P(7, 7.5),
+        P(8, 8.5),
+    };
+    M m(ar, ar + sizeof(ar) / sizeof(ar[0]));
+    assert(m.size() == 8);
+    I i = m.erase(m.cbegin(), m.cbegin());
+    assert(m.size() == 8);
+    assert(i == m.begin());
+    assert(m.begin()->first == 1);
+    assert(m.begin()->second == 1.5);
+    assert(std::next(m.begin())->first == 2);
+    assert(std::next(m.begin())->second == 2.5);
+    assert(std::next(m.begin(), 2)->first == 3);
+    assert(std::next(m.begin(), 2)->second == 3.5);
+    assert(std::next(m.begin(), 3)->first == 4);
+    assert(std::next(m.begin(), 3)->second == 4.5);
+    assert(std::next(m.begin(), 4)->first == 5);
+    assert(std::next(m.begin(), 4)->second == 5.5);
+    assert(std::next(m.begin(), 5)->first == 6);
+    assert(std::next(m.begin(), 5)->second == 6.5);
+    assert(std::next(m.begin(), 6)->first == 7);
+    assert(std::next(m.begin(), 6)->second == 7.5);
+    assert(std::next(m.begin(), 7)->first == 8);
+    assert(std::next(m.begin(), 7)->second == 8.5);
 
-        i = m.erase(m.cbegin(), std::next(m.cbegin(), 2));
-        assert(m.size() == 6);
-        assert(i == m.begin());
-        assert(std::next(m.begin(), 0)->first == 3);
-        assert(std::next(m.begin(), 0)->second == 3.5);
-        assert(std::next(m.begin(), 1)->first == 4);
-        assert(std::next(m.begin(), 1)->second == 4.5);
-        assert(std::next(m.begin(), 2)->first == 5);
-        assert(std::next(m.begin(), 2)->second == 5.5);
-        assert(std::next(m.begin(), 3)->first == 6);
-        assert(std::next(m.begin(), 3)->second == 6.5);
-        assert(std::next(m.begin(), 4)->first == 7);
-        assert(std::next(m.begin(), 4)->second == 7.5);
-        assert(std::next(m.begin(), 5)->first == 8);
-        assert(std::next(m.begin(), 5)->second == 8.5);
+    i = m.erase(m.cbegin(), std::next(m.cbegin(), 2));
+    assert(m.size() == 6);
+    assert(i == m.begin());
+    assert(std::next(m.begin(), 0)->first == 3);
+    assert(std::next(m.begin(), 0)->second == 3.5);
+    assert(std::next(m.begin(), 1)->first == 4);
+    assert(std::next(m.begin(), 1)->second == 4.5);
+    assert(std::next(m.begin(), 2)->first == 5);
+    assert(std::next(m.begin(), 2)->second == 5.5);
+    assert(std::next(m.begin(), 3)->first == 6);
+    assert(std::next(m.begin(), 3)->second == 6.5);
+    assert(std::next(m.begin(), 4)->first == 7);
+    assert(std::next(m.begin(), 4)->second == 7.5);
+    assert(std::next(m.begin(), 5)->first == 8);
+    assert(std::next(m.begin(), 5)->second == 8.5);
 
-        i = m.erase(std::next(m.cbegin(), 2), std::next(m.cbegin(), 6));
-        assert(m.size() == 2);
-        assert(i == std::next(m.begin(), 2));
-        assert(std::next(m.begin(), 0)->first == 3);
-        assert(std::next(m.begin(), 0)->second == 3.5);
-        assert(std::next(m.begin(), 1)->first == 4);
-        assert(std::next(m.begin(), 1)->second == 4.5);
+    i = m.erase(std::next(m.cbegin(), 2), std::next(m.cbegin(), 6));
+    assert(m.size() == 2);
+    assert(i == std::next(m.begin(), 2));
+    assert(std::next(m.begin(), 0)->first == 3);
+    assert(std::next(m.begin(), 0)->second == 3.5);
+    assert(std::next(m.begin(), 1)->first == 4);
+    assert(std::next(m.begin(), 1)->second == 4.5);
 
-        i = m.erase(m.cbegin(), m.cend());
-        assert(m.size() == 0);
-        assert(i == m.begin());
-        assert(i == m.end());
-    }
+    i = m.erase(m.cbegin(), m.cend());
+    assert(m.size() == 0);
+    assert(i == m.begin());
+    assert(i == m.end());
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/associative/map/map.modifiers/erase_key.pass.cpp b/libcxx/test/std/containers/associative/map/map.modifiers/erase_key.pass.cpp
index 8a2bc9c37128c..f276467d97029 100644
--- a/libcxx/test/std/containers/associative/map/map.modifiers/erase_key.pass.cpp
+++ b/libcxx/test/std/containers/associative/map/map.modifiers/erase_key.pass.cpp
@@ -18,259 +18,256 @@
 #include "test_macros.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef std::map<int, double> M;
-        typedef std::pair<int, double> P;
-        typedef M::size_type R;
-        P ar[] =
-        {
-            P(1, 1.5),
-            P(2, 2.5),
-            P(3, 3.5),
-            P(4, 4.5),
-            P(5, 5.5),
-            P(6, 6.5),
-            P(7, 7.5),
-            P(8, 8.5),
-        };
-        M m(ar, ar + sizeof(ar)/sizeof(ar[0]));
-        assert(m.size() == 8);
-        R s = m.erase(9);
-        assert(s == 0);
-        assert(m.size() == 8);
-        assert(m.begin()->first == 1);
-        assert(m.begin()->second == 1.5);
-        assert(std::next(m.begin())->first == 2);
-        assert(std::next(m.begin())->second == 2.5);
-        assert(std::next(m.begin(), 2)->first == 3);
-        assert(std::next(m.begin(), 2)->second == 3.5);
-        assert(std::next(m.begin(), 3)->first == 4);
-        assert(std::next(m.begin(), 3)->second == 4.5);
-        assert(std::next(m.begin(), 4)->first == 5);
-        assert(std::next(m.begin(), 4)->second == 5.5);
-        assert(std::next(m.begin(), 5)->first == 6);
-        assert(std::next(m.begin(), 5)->second == 6.5);
-        assert(std::next(m.begin(), 6)->first == 7);
-        assert(std::next(m.begin(), 6)->second == 7.5);
-        assert(std::next(m.begin(), 7)->first == 8);
-        assert(std::next(m.begin(), 7)->second == 8.5);
+int main(int, char**) {
+  {
+    typedef std::map<int, double> M;
+    typedef std::pair<int, double> P;
+    typedef M::size_type R;
+    P ar[] = {
+        P(1, 1.5),
+        P(2, 2.5),
+        P(3, 3.5),
+        P(4, 4.5),
+        P(5, 5.5),
+        P(6, 6.5),
+        P(7, 7.5),
+        P(8, 8.5),
+    };
+    M m(ar, ar + sizeof(ar) / sizeof(ar[0]));
+    assert(m.size() == 8);
+    R s = m.erase(9);
+    assert(s == 0);
+    assert(m.size() == 8);
+    assert(m.begin()->first == 1);
+    assert(m.begin()->second == 1.5);
+    assert(std::next(m.begin())->first == 2);
+    assert(std::next(m.begin())->second == 2.5);
+    assert(std::next(m.begin(), 2)->first == 3);
+    assert(std::next(m.begin(), 2)->second == 3.5);
+    assert(std::next(m.begin(), 3)->first == 4);
+    assert(std::next(m.begin(), 3)->second == 4.5);
+    assert(std::next(m.begin(), 4)->first == 5);
+    assert(std::next(m.begin(), 4)->second == 5.5);
+    assert(std::next(m.begin(), 5)->first == 6);
+    assert(std::next(m.begin(), 5)->second == 6.5);
+    assert(std::next(m.begin(), 6)->first == 7);
+    assert(std::next(m.begin(), 6)->second == 7.5);
+    assert(std::next(m.begin(), 7)->first == 8);
+    assert(std::next(m.begin(), 7)->second == 8.5);
 
-        s = m.erase(4);
-        assert(m.size() == 7);
-        assert(s == 1);
-        assert(m.begin()->first == 1);
-        assert(m.begin()->second == 1.5);
-        assert(std::next(m.begin())->first == 2);
-        assert(std::next(m.begin())->second == 2.5);
-        assert(std::next(m.begin(), 2)->first == 3);
-        assert(std::next(m.begin(), 2)->second == 3.5);
-        assert(std::next(m.begin(), 3)->first == 5);
-        assert(std::next(m.begin(), 3)->second == 5.5);
-        assert(std::next(m.begin(), 4)->first == 6);
-        assert(std::next(m.begin(), 4)->second == 6.5);
-        assert(std::next(m.begin(), 5)->first == 7);
-        assert(std::next(m.begin(), 5)->second == 7.5);
-        assert(std::next(m.begin(), 6)->first == 8);
-        assert(std::next(m.begin(), 6)->second == 8.5);
+    s = m.erase(4);
+    assert(m.size() == 7);
+    assert(s == 1);
+    assert(m.begin()->first == 1);
+    assert(m.begin()->second == 1.5);
+    assert(std::next(m.begin())->first == 2);
+    assert(std::next(m.begin())->second == 2.5);
+    assert(std::next(m.begin(), 2)->first == 3);
+    assert(std::next(m.begin(), 2)->second == 3.5);
+    assert(std::next(m.begin(), 3)->first == 5);
+    assert(std::next(m.begin(), 3)->second == 5.5);
+    assert(std::next(m.begin(), 4)->first == 6);
+    assert(std::next(m.begin(), 4)->second == 6.5);
+    assert(std::next(m.begin(), 5)->first == 7);
+    assert(std::next(m.begin(), 5)->second == 7.5);
+    assert(std::next(m.begin(), 6)->first == 8);
+    assert(std::next(m.begin(), 6)->second == 8.5);
 
-        s = m.erase(1);
-        assert(m.size() == 6);
-        assert(s == 1);
-        assert(m.begin()->first == 2);
-        assert(m.begin()->second == 2.5);
-        assert(std::next(m.begin())->first == 3);
-        assert(std::next(m.begin())->second == 3.5);
-        assert(std::next(m.begin(), 2)->first == 5);
-        assert(std::next(m.begin(), 2)->second == 5.5);
-        assert(std::next(m.begin(), 3)->first == 6);
-        assert(std::next(m.begin(), 3)->second == 6.5);
-        assert(std::next(m.begin(), 4)->first == 7);
-        assert(std::next(m.begin(), 4)->second == 7.5);
-        assert(std::next(m.begin(), 5)->first == 8);
-        assert(std::next(m.begin(), 5)->second == 8.5);
+    s = m.erase(1);
+    assert(m.size() == 6);
+    assert(s == 1);
+    assert(m.begin()->first == 2);
+    assert(m.begin()->second == 2.5);
+    assert(std::next(m.begin())->first == 3);
+    assert(std::next(m.begin())->second == 3.5);
+    assert(std::next(m.begin(), 2)->first == 5);
+    assert(std::next(m.begin(), 2)->second == 5.5);
+    assert(std::next(m.begin(), 3)->first == 6);
+    assert(std::next(m.begin(), 3)->second == 6.5);
+    assert(std::next(m.begin(), 4)->first == 7);
+    assert(std::next(m.begin(), 4)->second == 7.5);
+    assert(std::next(m.begin(), 5)->first == 8);
+    assert(std::next(m.begin(), 5)->second == 8.5);
 
-        s = m.erase(8);
-        assert(m.size() == 5);
-        assert(s == 1);
-        assert(m.begin()->first == 2);
-        assert(m.begin()->second == 2.5);
-        assert(std::next(m.begin())->first == 3);
-        assert(std::next(m.begin())->second == 3.5);
-        assert(std::next(m.begin(), 2)->first == 5);
-        assert(std::next(m.begin(), 2)->second == 5.5);
-        assert(std::next(m.begin(), 3)->first == 6);
-        assert(std::next(m.begin(), 3)->second == 6.5);
-        assert(std::next(m.begin(), 4)->first == 7);
-        assert(std::next(m.begin(), 4)->second == 7.5);
+    s = m.erase(8);
+    assert(m.size() == 5);
+    assert(s == 1);
+    assert(m.begin()->first == 2);
+    assert(m.begin()->second == 2.5);
+    assert(std::next(m.begin())->first == 3);
+    assert(std::next(m.begin())->second == 3.5);
+    assert(std::next(m.begin(), 2)->first == 5);
+    assert(std::next(m.begin(), 2)->second == 5.5);
+    assert(std::next(m.begin(), 3)->first == 6);
+    assert(std::next(m.begin(), 3)->second == 6.5);
+    assert(std::next(m.begin(), 4)->first == 7);
+    assert(std::next(m.begin(), 4)->second == 7.5);
 
-        s = m.erase(3);
-        assert(m.size() == 4);
-        assert(s == 1);
-        assert(m.begin()->first == 2);
-        assert(m.begin()->second == 2.5);
-        assert(std::next(m.begin())->first == 5);
-        assert(std::next(m.begin())->second == 5.5);
-        assert(std::next(m.begin(), 2)->first == 6);
-        assert(std::next(m.begin(), 2)->second == 6.5);
-        assert(std::next(m.begin(), 3)->first == 7);
-        assert(std::next(m.begin(), 3)->second == 7.5);
+    s = m.erase(3);
+    assert(m.size() == 4);
+    assert(s == 1);
+    assert(m.begin()->first == 2);
+    assert(m.begin()->second == 2.5);
+    assert(std::next(m.begin())->first == 5);
+    assert(std::next(m.begin())->second == 5.5);
+    assert(std::next(m.begin(), 2)->first == 6);
+    assert(std::next(m.begin(), 2)->second == 6.5);
+    assert(std::next(m.begin(), 3)->first == 7);
+    assert(std::next(m.begin(), 3)->second == 7.5);
 
-        s = m.erase(6);
-        assert(m.size() == 3);
-        assert(s == 1);
-        assert(m.begin()->first == 2);
-        assert(m.begin()->second == 2.5);
-        assert(std::next(m.begin())->first == 5);
-        assert(std::next(m.begin())->second == 5.5);
-        assert(std::next(m.begin(), 2)->first == 7);
-        assert(std::next(m.begin(), 2)->second == 7.5);
+    s = m.erase(6);
+    assert(m.size() == 3);
+    assert(s == 1);
+    assert(m.begin()->first == 2);
+    assert(m.begin()->second == 2.5);
+    assert(std::next(m.begin())->first == 5);
+    assert(std::next(m.begin())->second == 5.5);
+    assert(std::next(m.begin(), 2)->first == 7);
+    assert(std::next(m.begin(), 2)->second == 7.5);
 
-        s = m.erase(7);
-        assert(m.size() == 2);
-        assert(s == 1);
-        assert(m.begin()->first == 2);
-        assert(m.begin()->second == 2.5);
-        assert(std::next(m.begin())->first == 5);
-        assert(std::next(m.begin())->second == 5.5);
+    s = m.erase(7);
+    assert(m.size() == 2);
+    assert(s == 1);
+    assert(m.begin()->first == 2);
+    assert(m.begin()->second == 2.5);
+    assert(std::next(m.begin())->first == 5);
+    assert(std::next(m.begin())->second == 5.5);
 
-        s = m.erase(2);
-        assert(m.size() == 1);
-        assert(s == 1);
-        assert(m.begin()->first == 5);
-        assert(m.begin()->second == 5.5);
+    s = m.erase(2);
+    assert(m.size() == 1);
+    assert(s == 1);
+    assert(m.begin()->first == 5);
+    assert(m.begin()->second == 5.5);
 
-        s = m.erase(5);
-        assert(m.size() == 0);
-        assert(s == 1);
-    }
+    s = m.erase(5);
+    assert(m.size() == 0);
+    assert(s == 1);
+  }
 #if TEST_STD_VER >= 11
-    {
-        typedef std::map<int, double, std::less<int>, min_allocator<std::pair<const int, double>>> M;
-        typedef std::pair<int, double> P;
-        typedef M::size_type R;
-        P ar[] =
-        {
-            P(1, 1.5),
-            P(2, 2.5),
-            P(3, 3.5),
-            P(4, 4.5),
-            P(5, 5.5),
-            P(6, 6.5),
-            P(7, 7.5),
-            P(8, 8.5),
-        };
-        M m(ar, ar + sizeof(ar)/sizeof(ar[0]));
-        assert(m.size() == 8);
-        R s = m.erase(9);
-        assert(s == 0);
-        assert(m.size() == 8);
-        assert(m.begin()->first == 1);
-        assert(m.begin()->second == 1.5);
-        assert(std::next(m.begin())->first == 2);
-        assert(std::next(m.begin())->second == 2.5);
-        assert(std::next(m.begin(), 2)->first == 3);
-        assert(std::next(m.begin(), 2)->second == 3.5);
-        assert(std::next(m.begin(), 3)->first == 4);
-        assert(std::next(m.begin(), 3)->second == 4.5);
-        assert(std::next(m.begin(), 4)->first == 5);
-        assert(std::next(m.begin(), 4)->second == 5.5);
-        assert(std::next(m.begin(), 5)->first == 6);
-        assert(std::next(m.begin(), 5)->second == 6.5);
-        assert(std::next(m.begin(), 6)->first == 7);
-        assert(std::next(m.begin(), 6)->second == 7.5);
-        assert(std::next(m.begin(), 7)->first == 8);
-        assert(std::next(m.begin(), 7)->second == 8.5);
+  {
+    typedef std::map<int, double, std::less<int>, min_allocator<std::pair<const int, double>>> M;
+    typedef std::pair<int, double> P;
+    typedef M::size_type R;
+    P ar[] = {
+        P(1, 1.5),
+        P(2, 2.5),
+        P(3, 3.5),
+        P(4, 4.5),
+        P(5, 5.5),
+        P(6, 6.5),
+        P(7, 7.5),
+        P(8, 8.5),
+    };
+    M m(ar, ar + sizeof(ar) / sizeof(ar[0]));
+    assert(m.size() == 8);
+    R s = m.erase(9);
+    assert(s == 0);
+    assert(m.size() == 8);
+    assert(m.begin()->first == 1);
+    assert(m.begin()->second == 1.5);
+    assert(std::next(m.begin())->first == 2);
+    assert(std::next(m.begin())->second == 2.5);
+    assert(std::next(m.begin(), 2)->first == 3);
+    assert(std::next(m.begin(), 2)->second == 3.5);
+    assert(std::next(m.begin(), 3)->first == 4);
+    assert(std::next(m.begin(), 3)->second == 4.5);
+    assert(std::next(m.begin(), 4)->first == 5);
+    assert(std::next(m.begin(), 4)->second == 5.5);
+    assert(std::next(m.begin(), 5)->first == 6);
+    assert(std::next(m.begin(), 5)->second == 6.5);
+    assert(std::next(m.begin(), 6)->first == 7);
+    assert(std::next(m.begin(), 6)->second == 7.5);
+    assert(std::next(m.begin(), 7)->first == 8);
+    assert(std::next(m.begin(), 7)->second == 8.5);
 
-        s = m.erase(4);
-        assert(m.size() == 7);
-        assert(s == 1);
-        assert(m.begin()->first == 1);
-        assert(m.begin()->second == 1.5);
-        assert(std::next(m.begin())->first == 2);
-        assert(std::next(m.begin())->second == 2.5);
-        assert(std::next(m.begin(), 2)->first == 3);
-        assert(std::next(m.begin(), 2)->second == 3.5);
-        assert(std::next(m.begin(), 3)->first == 5);
-        assert(std::next(m.begin(), 3)->second == 5.5);
-        assert(std::next(m.begin(), 4)->first == 6);
-        assert(std::next(m.begin(), 4)->second == 6.5);
-        assert(std::next(m.begin(), 5)->first == 7);
-        assert(std::next(m.begin(), 5)->second == 7.5);
-        assert(std::next(m.begin(), 6)->first == 8);
-        assert(std::next(m.begin(), 6)->second == 8.5);
+    s = m.erase(4);
+    assert(m.size() == 7);
+    assert(s == 1);
+    assert(m.begin()->first == 1);
+    assert(m.begin()->second == 1.5);
+    assert(std::next(m.begin())->first == 2);
+    assert(std::next(m.begin())->second == 2.5);
+    assert(std::next(m.begin(), 2)->first == 3);
+    assert(std::next(m.begin(), 2)->second == 3.5);
+    assert(std::next(m.begin(), 3)->first == 5);
+    assert(std::next(m.begin(), 3)->second == 5.5);
+    assert(std::next(m.begin(), 4)->first == 6);
+    assert(std::next(m.begin(), 4)->second == 6.5);
+    assert(std::next(m.begin(), 5)->first == 7);
+    assert(std::next(m.begin(), 5)->second == 7.5);
+    assert(std::next(m.begin(), 6)->first == 8);
+    assert(std::next(m.begin(), 6)->second == 8.5);
 
-        s = m.erase(1);
-        assert(m.size() == 6);
-        assert(s == 1);
-        assert(m.begin()->first == 2);
-        assert(m.begin()->second == 2.5);
-        assert(std::next(m.begin())->first == 3);
-        assert(std::next(m.begin())->second == 3.5);
-        assert(std::next(m.begin(), 2)->first == 5);
-        assert(std::next(m.begin(), 2)->second == 5.5);
-        assert(std::next(m.begin(), 3)->first == 6);
-        assert(std::next(m.begin(), 3)->second == 6.5);
-        assert(std::next(m.begin(), 4)->first == 7);
-        assert(std::next(m.begin(), 4)->second == 7.5);
-        assert(std::next(m.begin(), 5)->first == 8);
-        assert(std::next(m.begin(), 5)->second == 8.5);
+    s = m.erase(1);
+    assert(m.size() == 6);
+    assert(s == 1);
+    assert(m.begin()->first == 2);
+    assert(m.begin()->second == 2.5);
+    assert(std::next(m.begin())->first == 3);
+    assert(std::next(m.begin())->second == 3.5);
+    assert(std::next(m.begin(), 2)->first == 5);
+    assert(std::next(m.begin(), 2)->second == 5.5);
+    assert(std::next(m.begin(), 3)->first == 6);
+    assert(std::next(m.begin(), 3)->second == 6.5);
+    assert(std::next(m.begin(), 4)->first == 7);
+    assert(std::next(m.begin(), 4)->second == 7.5);
+    assert(std::next(m.begin(), 5)->first == 8);
+    assert(std::next(m.begin(), 5)->second == 8.5);
 
-        s = m.erase(8);
-        assert(m.size() == 5);
-        assert(s == 1);
-        assert(m.begin()->first == 2);
-        assert(m.begin()->second == 2.5);
-        assert(std::next(m.begin())->first == 3);
-        assert(std::next(m.begin())->second == 3.5);
-        assert(std::next(m.begin(), 2)->first == 5);
-        assert(std::next(m.begin(), 2)->second == 5.5);
-        assert(std::next(m.begin(), 3)->first == 6);
-        assert(std::next(m.begin(), 3)->second == 6.5);
-        assert(std::next(m.begin(), 4)->first == 7);
-        assert(std::next(m.begin(), 4)->second == 7.5);
+    s = m.erase(8);
+    assert(m.size() == 5);
+    assert(s == 1);
+    assert(m.begin()->first == 2);
+    assert(m.begin()->second == 2.5);
+    assert(std::next(m.begin())->first == 3);
+    assert(std::next(m.begin())->second == 3.5);
+    assert(std::next(m.begin(), 2)->first == 5);
+    assert(std::next(m.begin(), 2)->second == 5.5);
+    assert(std::next(m.begin(), 3)->first == 6);
+    assert(std::next(m.begin(), 3)->second == 6.5);
+    assert(std::next(m.begin(), 4)->first == 7);
+    assert(std::next(m.begin(), 4)->second == 7.5);
 
-        s = m.erase(3);
-        assert(m.size() == 4);
-        assert(s == 1);
-        assert(m.begin()->first == 2);
-        assert(m.begin()->second == 2.5);
-        assert(std::next(m.begin())->first == 5);
-        assert(std::next(m.begin())->second == 5.5);
-        assert(std::next(m.begin(), 2)->first == 6);
-        assert(std::next(m.begin(), 2)->second == 6.5);
-        assert(std::next(m.begin(), 3)->first == 7);
-        assert(std::next(m.begin(), 3)->second == 7.5);
+    s = m.erase(3);
+    assert(m.size() == 4);
+    assert(s == 1);
+    assert(m.begin()->first == 2);
+    assert(m.begin()->second == 2.5);
+    assert(std::next(m.begin())->first == 5);
+    assert(std::next(m.begin())->second == 5.5);
+    assert(std::next(m.begin(), 2)->first == 6);
+    assert(std::next(m.begin(), 2)->second == 6.5);
+    assert(std::next(m.begin(), 3)->first == 7);
+    assert(std::next(m.begin(), 3)->second == 7.5);
 
-        s = m.erase(6);
-        assert(m.size() == 3);
-        assert(s == 1);
-        assert(m.begin()->first == 2);
-        assert(m.begin()->second == 2.5);
-        assert(std::next(m.begin())->first == 5);
-        assert(std::next(m.begin())->second == 5.5);
-        assert(std::next(m.begin(), 2)->first == 7);
-        assert(std::next(m.begin(), 2)->second == 7.5);
+    s = m.erase(6);
+    assert(m.size() == 3);
+    assert(s == 1);
+    assert(m.begin()->first == 2);
+    assert(m.begin()->second == 2.5);
+    assert(std::next(m.begin())->first == 5);
+    assert(std::next(m.begin())->second == 5.5);
+    assert(std::next(m.begin(), 2)->first == 7);
+    assert(std::next(m.begin(), 2)->second == 7.5);
 
-        s = m.erase(7);
-        assert(m.size() == 2);
-        assert(s == 1);
-        assert(m.begin()->first == 2);
-        assert(m.begin()->second == 2.5);
-        assert(std::next(m.begin())->first == 5);
-        assert(std::next(m.begin())->second == 5.5);
+    s = m.erase(7);
+    assert(m.size() == 2);
+    assert(s == 1);
+    assert(m.begin()->first == 2);
+    assert(m.begin()->second == 2.5);
+    assert(std::next(m.begin())->first == 5);
+    assert(std::next(m.begin())->second == 5.5);
 
-        s = m.erase(2);
-        assert(m.size() == 1);
-        assert(s == 1);
-        assert(m.begin()->first == 5);
-        assert(m.begin()->second == 5.5);
+    s = m.erase(2);
+    assert(m.size() == 1);
+    assert(s == 1);
+    assert(m.begin()->first == 5);
+    assert(m.begin()->second == 5.5);
 
-        s = m.erase(5);
-        assert(m.size() == 0);
-        assert(s == 1);
-    }
+    s = m.erase(5);
+    assert(m.size() == 0);
+    assert(s == 1);
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/associative/map/map.modifiers/extract_iterator.pass.cpp b/libcxx/test/std/containers/associative/map/map.modifiers/extract_iterator.pass.cpp
index 6cdfac774a2c3..db360cb8fe701 100644
--- a/libcxx/test/std/containers/associative/map/map.modifiers/extract_iterator.pass.cpp
+++ b/libcxx/test/std/containers/associative/map/map.modifiers/extract_iterator.pass.cpp
@@ -20,50 +20,44 @@
 #include "Counter.h"
 
 template <class Container>
-void test(Container& c)
-{
-    std::size_t sz = c.size();
-
-    auto some_key = c.cbegin()->first;
-
-    for (auto first = c.cbegin(); first != c.cend();)
-    {
-        auto key_value = first->first;
-        typename Container::node_type t = c.extract(first++);
-        --sz;
-        assert(t.key() == key_value);
-        t.key() = some_key;
-        assert(t.key() == some_key);
-        assert(t.get_allocator() == c.get_allocator());
-        assert(sz == c.size());
-    }
-
-    assert(c.size() == 0);
+void test(Container& c) {
+  std::size_t sz = c.size();
+
+  auto some_key = c.cbegin()->first;
+
+  for (auto first = c.cbegin(); first != c.cend();) {
+    auto key_value                  = first->first;
+    typename Container::node_type t = c.extract(first++);
+    --sz;
+    assert(t.key() == key_value);
+    t.key() = some_key;
+    assert(t.key() == some_key);
+    assert(t.get_allocator() == c.get_allocator());
+    assert(sz == c.size());
+  }
+
+  assert(c.size() == 0);
 }
 
-int main(int, char**)
-{
-    {
-        using map_type = std::map<int, int>;
-        map_type m = {{1,1}, {2,2}, {3,3}, {4,4}, {5,5}, {6,6}};
-        test(m);
-    }
-
-    {
-        std::map<Counter<int>, Counter<int>> m =
-            {{1,1}, {2,2}, {3,3}, {4,4}, {5,5}, {6,6}};
-        assert(Counter_base::gConstructed == 12);
-        test(m);
-        assert(Counter_base::gConstructed == 0);
-    }
-
-    {
-        using min_alloc_map =
-            std::map<int, int, std::less<int>,
-                     min_allocator<std::pair<const int, int>>>;
-        min_alloc_map m = {{1, 1}, {2, 2}, {3, 3}, {4, 4}, {5, 5}, {6, 6}};
-        test(m);
-    }
+int main(int, char**) {
+  {
+    using map_type = std::map<int, int>;
+    map_type m     = {{1, 1}, {2, 2}, {3, 3}, {4, 4}, {5, 5}, {6, 6}};
+    test(m);
+  }
+
+  {
+    std::map<Counter<int>, Counter<int>> m = {{1, 1}, {2, 2}, {3, 3}, {4, 4}, {5, 5}, {6, 6}};
+    assert(Counter_base::gConstructed == 12);
+    test(m);
+    assert(Counter_base::gConstructed == 0);
+  }
+
+  {
+    using min_alloc_map = std::map<int, int, std::less<int>, min_allocator<std::pair<const int, int>>>;
+    min_alloc_map m     = {{1, 1}, {2, 2}, {3, 3}, {4, 4}, {5, 5}, {6, 6}};
+    test(m);
+  }
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/associative/map/map.modifiers/extract_key.pass.cpp b/libcxx/test/std/containers/associative/map/map.modifiers/extract_key.pass.cpp
index c0ffa343497e2..7ad9558c90c07 100644
--- a/libcxx/test/std/containers/associative/map/map.modifiers/extract_key.pass.cpp
+++ b/libcxx/test/std/containers/associative/map/map.modifiers/extract_key.pass.cpp
@@ -20,59 +20,52 @@
 #include "Counter.h"
 
 template <class Container, class KeyTypeIter>
-void test(Container& c, KeyTypeIter first, KeyTypeIter last)
-{
-    std::size_t sz = c.size();
-    assert((std::size_t)std::distance(first, last) == sz);
+void test(Container& c, KeyTypeIter first, KeyTypeIter last) {
+  std::size_t sz = c.size();
+  assert((std::size_t)std::distance(first, last) == sz);
 
-    for (KeyTypeIter copy = first; copy != last; ++copy)
-    {
-        typename Container::node_type t = c.extract(*copy);
-        assert(!t.empty());
-        --sz;
-        assert(t.key() == *copy);
-        t.key() = *first; // We should be able to mutate key.
-        assert(t.key() == *first);
-        assert(t.get_allocator() == c.get_allocator());
-        assert(sz == c.size());
-    }
+  for (KeyTypeIter copy = first; copy != last; ++copy) {
+    typename Container::node_type t = c.extract(*copy);
+    assert(!t.empty());
+    --sz;
+    assert(t.key() == *copy);
+    t.key() = *first; // We should be able to mutate key.
+    assert(t.key() == *first);
+    assert(t.get_allocator() == c.get_allocator());
+    assert(sz == c.size());
+  }
 
-    assert(c.size() == 0);
+  assert(c.size() == 0);
 
-    for (KeyTypeIter copy = first; copy != last; ++copy)
-    {
-        typename Container::node_type t = c.extract(*copy);
-        assert(t.empty());
-    }
+  for (KeyTypeIter copy = first; copy != last; ++copy) {
+    typename Container::node_type t = c.extract(*copy);
+    assert(t.empty());
+  }
 }
 
-int main(int, char**)
-{
-    {
-        std::map<int, int> m = {{1,1}, {2,2}, {3,3}, {4,4}, {5,5}, {6,6}};
-        int keys[] = {1, 2, 3, 4, 5, 6};
-        test(m, std::begin(keys), std::end(keys));
-    }
+int main(int, char**) {
+  {
+    std::map<int, int> m = {{1, 1}, {2, 2}, {3, 3}, {4, 4}, {5, 5}, {6, 6}};
+    int keys[]           = {1, 2, 3, 4, 5, 6};
+    test(m, std::begin(keys), std::end(keys));
+  }
 
+  {
+    std::map<Counter<int>, Counter<int>> m = {{1, 1}, {2, 2}, {3, 3}, {4, 4}, {5, 5}, {6, 6}};
     {
-        std::map<Counter<int>, Counter<int>> m =
-            {{1,1}, {2,2}, {3,3}, {4,4}, {5,5}, {6,6}};
-        {
-            Counter<int> keys[] = {1, 2, 3, 4, 5, 6};
-            assert(Counter_base::gConstructed == 12+6);
-            test(m, std::begin(keys), std::end(keys));
-        }
-        assert(Counter_base::gConstructed == 0);
+      Counter<int> keys[] = {1, 2, 3, 4, 5, 6};
+      assert(Counter_base::gConstructed == 12 + 6);
+      test(m, std::begin(keys), std::end(keys));
     }
+    assert(Counter_base::gConstructed == 0);
+  }
 
-    {
-        using min_alloc_map =
-            std::map<int, int, std::less<int>,
-                     min_allocator<std::pair<const int, int>>>;
-        min_alloc_map m = {{1, 1}, {2, 2}, {3, 3}, {4, 4}, {5, 5}, {6, 6}};
-        int keys[] = {1, 2, 3, 4, 5, 6};
-        test(m, std::begin(keys), std::end(keys));
-    }
+  {
+    using min_alloc_map = std::map<int, int, std::less<int>, min_allocator<std::pair<const int, int>>>;
+    min_alloc_map m     = {{1, 1}, {2, 2}, {3, 3}, {4, 4}, {5, 5}, {6, 6}};
+    int keys[]          = {1, 2, 3, 4, 5, 6};
+    test(m, std::begin(keys), std::end(keys));
+  }
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/associative/map/map.modifiers/insert_and_emplace_allocator_requirements.pass.cpp b/libcxx/test/std/containers/associative/map/map.modifiers/insert_and_emplace_allocator_requirements.pass.cpp
index fb1251c7fb58c..5dfd415e4382e 100644
--- a/libcxx/test/std/containers/associative/map/map.modifiers/insert_and_emplace_allocator_requirements.pass.cpp
+++ b/libcxx/test/std/containers/associative/map/map.modifiers/insert_and_emplace_allocator_requirements.pass.cpp
@@ -22,8 +22,7 @@
 #include "container_test_types.h"
 #include "../../../map_allocator_requirement_test_templates.h"
 
-int main(int, char**)
-{
+int main(int, char**) {
   testMapInsert<TCT::map<> >();
   testMapInsertHint<TCT::map<> >();
   testMapEmplace<TCT::map<> >();
diff --git a/libcxx/test/std/containers/associative/map/map.modifiers/insert_cv.pass.cpp b/libcxx/test/std/containers/associative/map/map.modifiers/insert_cv.pass.cpp
index a2436440f1947..e845461cfbf31 100644
--- a/libcxx/test/std/containers/associative/map/map.modifiers/insert_cv.pass.cpp
+++ b/libcxx/test/std/containers/associative/map/map.modifiers/insert_cv.pass.cpp
@@ -19,54 +19,52 @@
 #include "min_allocator.h"
 
 template <class Container>
-void do_insert_cv_test()
-{
-    typedef Container M;
-    typedef std::pair<typename M::iterator, bool> R;
-    typedef typename M::value_type VT;
-    M m;
+void do_insert_cv_test() {
+  typedef Container M;
+  typedef std::pair<typename M::iterator, bool> R;
+  typedef typename M::value_type VT;
+  M m;
 
-    const VT v1(2, 2.5);
-    R r = m.insert(v1);
-    assert(r.second);
-    assert(r.first == m.begin());
-    assert(m.size() == 1);
-    assert(r.first->first == 2);
-    assert(r.first->second == 2.5);
+  const VT v1(2, 2.5);
+  R r = m.insert(v1);
+  assert(r.second);
+  assert(r.first == m.begin());
+  assert(m.size() == 1);
+  assert(r.first->first == 2);
+  assert(r.first->second == 2.5);
 
-    const VT v2(1, 1.5);
-    r = m.insert(v2);
-    assert(r.second);
-    assert(r.first == m.begin());
-    assert(m.size() == 2);
-    assert(r.first->first == 1);
-    assert(r.first->second == 1.5);
+  const VT v2(1, 1.5);
+  r = m.insert(v2);
+  assert(r.second);
+  assert(r.first == m.begin());
+  assert(m.size() == 2);
+  assert(r.first->first == 1);
+  assert(r.first->second == 1.5);
 
-    const VT v3(3, 3.5);
-    r = m.insert(v3);
-    assert(r.second);
-    assert(r.first == std::prev(m.end()));
-    assert(m.size() == 3);
-    assert(r.first->first == 3);
-    assert(r.first->second == 3.5);
+  const VT v3(3, 3.5);
+  r = m.insert(v3);
+  assert(r.second);
+  assert(r.first == std::prev(m.end()));
+  assert(m.size() == 3);
+  assert(r.first->first == 3);
+  assert(r.first->second == 3.5);
 
-    const VT v4(3, 4.5);
-    r = m.insert(v4);
-    assert(!r.second);
-    assert(r.first == std::prev(m.end()));
-    assert(m.size() == 3);
-    assert(r.first->first == 3);
-    assert(r.first->second == 3.5);
+  const VT v4(3, 4.5);
+  r = m.insert(v4);
+  assert(!r.second);
+  assert(r.first == std::prev(m.end()));
+  assert(m.size() == 3);
+  assert(r.first->first == 3);
+  assert(r.first->second == 3.5);
 }
 
-int main(int, char**)
-{
-    do_insert_cv_test<std::map<int, double> >();
+int main(int, char**) {
+  do_insert_cv_test<std::map<int, double> >();
 #if TEST_STD_VER >= 11
-    {
-        typedef std::map<int, double, std::less<int>, min_allocator<std::pair<const int, double>>> M;
-        do_insert_cv_test<M>();
-    }
+  {
+    typedef std::map<int, double, std::less<int>, min_allocator<std::pair<const int, double>>> M;
+    do_insert_cv_test<M>();
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/associative/map/map.modifiers/insert_initializer_list.pass.cpp b/libcxx/test/std/containers/associative/map/map.modifiers/insert_initializer_list.pass.cpp
index 313e461560fb9..1f6e5c6371a11 100644
--- a/libcxx/test/std/containers/associative/map/map.modifiers/insert_initializer_list.pass.cpp
+++ b/libcxx/test/std/containers/associative/map/map.modifiers/insert_initializer_list.pass.cpp
@@ -20,52 +20,35 @@
 #include "test_macros.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
+int main(int, char**) {
+  {
     typedef std::pair<const int, double> V;
-    std::map<int, double> m =
-                            {
-                                {1, 1},
-                                {1, 1.5},
-                                {1, 2},
-                                {3, 1},
-                                {3, 1.5},
-                                {3, 2}
-                            };
+    std::map<int, double> m = {{1, 1}, {1, 1.5}, {1, 2}, {3, 1}, {3, 1.5}, {3, 2}};
     m.insert({
-                 {2, 1},
-                 {2, 1.5},
-                 {2, 2},
-             });
+        {2, 1},
+        {2, 1.5},
+        {2, 2},
+    });
     assert(m.size() == 3);
     assert(std::distance(m.begin(), m.end()) == 3);
     assert(*m.begin() == V(1, 1));
     assert(*std::next(m.begin()) == V(2, 1));
     assert(*std::next(m.begin(), 2) == V(3, 1));
-    }
-    {
+  }
+  {
     typedef std::pair<const int, double> V;
-    std::map<int, double, std::less<int>, min_allocator<V>> m =
-                            {
-                                {1, 1},
-                                {1, 1.5},
-                                {1, 2},
-                                {3, 1},
-                                {3, 1.5},
-                                {3, 2}
-                            };
+    std::map<int, double, std::less<int>, min_allocator<V>> m = {{1, 1}, {1, 1.5}, {1, 2}, {3, 1}, {3, 1.5}, {3, 2}};
     m.insert({
-                 {2, 1},
-                 {2, 1.5},
-                 {2, 2},
-             });
+        {2, 1},
+        {2, 1.5},
+        {2, 2},
+    });
     assert(m.size() == 3);
     assert(std::distance(m.begin(), m.end()) == 3);
     assert(*m.begin() == V(1, 1));
     assert(*std::next(m.begin()) == V(2, 1));
     assert(*std::next(m.begin(), 2) == V(3, 1));
-    }
+  }
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/associative/map/map.modifiers/insert_iter_cv.pass.cpp b/libcxx/test/std/containers/associative/map/map.modifiers/insert_iter_cv.pass.cpp
index 6260fd9d93e9b..5256ca2c8d451 100644
--- a/libcxx/test/std/containers/associative/map/map.modifiers/insert_iter_cv.pass.cpp
+++ b/libcxx/test/std/containers/associative/map/map.modifiers/insert_iter_cv.pass.cpp
@@ -19,50 +19,48 @@
 #include "min_allocator.h"
 
 template <class Container>
-void do_insert_iter_cv_test()
-{
-    typedef Container M;
-    typedef typename M::iterator R;
-    typedef typename M::value_type VT;
+void do_insert_iter_cv_test() {
+  typedef Container M;
+  typedef typename M::iterator R;
+  typedef typename M::value_type VT;
 
-    M m;
-    const VT v1(2, 2.5);
-    R r = m.insert(m.end(), v1);
-    assert(r == m.begin());
-    assert(m.size() == 1);
-    assert(r->first == 2);
-    assert(r->second == 2.5);
+  M m;
+  const VT v1(2, 2.5);
+  R r = m.insert(m.end(), v1);
+  assert(r == m.begin());
+  assert(m.size() == 1);
+  assert(r->first == 2);
+  assert(r->second == 2.5);
 
-    const VT v2(1, 1.5);
-    r = m.insert(m.end(), v2);
-    assert(r == m.begin());
-    assert(m.size() == 2);
-    assert(r->first == 1);
-    assert(r->second == 1.5);
+  const VT v2(1, 1.5);
+  r = m.insert(m.end(), v2);
+  assert(r == m.begin());
+  assert(m.size() == 2);
+  assert(r->first == 1);
+  assert(r->second == 1.5);
 
-    const VT v3(3, 3.5);
-    r = m.insert(m.end(), v3);
-    assert(r == std::prev(m.end()));
-    assert(m.size() == 3);
-    assert(r->first == 3);
-    assert(r->second == 3.5);
+  const VT v3(3, 3.5);
+  r = m.insert(m.end(), v3);
+  assert(r == std::prev(m.end()));
+  assert(m.size() == 3);
+  assert(r->first == 3);
+  assert(r->second == 3.5);
 
-    const VT v4(3, 4.5);
-    r = m.insert(m.end(), v4);
-    assert(r == std::prev(m.end()));
-    assert(m.size() == 3);
-    assert(r->first == 3);
-    assert(r->second == 3.5);
+  const VT v4(3, 4.5);
+  r = m.insert(m.end(), v4);
+  assert(r == std::prev(m.end()));
+  assert(m.size() == 3);
+  assert(r->first == 3);
+  assert(r->second == 3.5);
 }
 
-int main(int, char**)
-{
-    do_insert_iter_cv_test<std::map<int, double> >();
+int main(int, char**) {
+  do_insert_iter_cv_test<std::map<int, double> >();
 #if TEST_STD_VER >= 11
-    {
-        typedef std::map<int, double, std::less<int>, min_allocator<std::pair<const int, double>>> M;
-        do_insert_iter_cv_test<M>();
-    }
+  {
+    typedef std::map<int, double, std::less<int>, min_allocator<std::pair<const int, double>>> M;
+    do_insert_iter_cv_test<M>();
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/associative/map/map.modifiers/insert_iter_iter.pass.cpp b/libcxx/test/std/containers/associative/map/map.modifiers/insert_iter_iter.pass.cpp
index b2b2ad155dd9e..0baff65f61938 100644
--- a/libcxx/test/std/containers/associative/map/map.modifiers/insert_iter_iter.pass.cpp
+++ b/libcxx/test/std/containers/associative/map/map.modifiers/insert_iter_iter.pass.cpp
@@ -20,59 +20,56 @@
 #include "test_iterators.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef std::map<int, double> M;
-        typedef std::pair<int, double> P;
-        P ar[] =
-        {
-            P(1, 1),
-            P(1, 1.5),
-            P(1, 2),
-            P(2, 1),
-            P(2, 1.5),
-            P(2, 2),
-            P(3, 1),
-            P(3, 1.5),
-            P(3, 2),
-        };
-        M m;
-        m.insert(cpp17_input_iterator<P*>(ar), cpp17_input_iterator<P*>(ar + sizeof(ar)/sizeof(ar[0])));
-        assert(m.size() == 3);
-        assert(m.begin()->first == 1);
-        assert(m.begin()->second == 1);
-        assert(std::next(m.begin())->first == 2);
-        assert(std::next(m.begin())->second == 1);
-        assert(std::next(m.begin(), 2)->first == 3);
-        assert(std::next(m.begin(), 2)->second == 1);
-    }
+int main(int, char**) {
+  {
+    typedef std::map<int, double> M;
+    typedef std::pair<int, double> P;
+    P ar[] = {
+        P(1, 1),
+        P(1, 1.5),
+        P(1, 2),
+        P(2, 1),
+        P(2, 1.5),
+        P(2, 2),
+        P(3, 1),
+        P(3, 1.5),
+        P(3, 2),
+    };
+    M m;
+    m.insert(cpp17_input_iterator<P*>(ar), cpp17_input_iterator<P*>(ar + sizeof(ar) / sizeof(ar[0])));
+    assert(m.size() == 3);
+    assert(m.begin()->first == 1);
+    assert(m.begin()->second == 1);
+    assert(std::next(m.begin())->first == 2);
+    assert(std::next(m.begin())->second == 1);
+    assert(std::next(m.begin(), 2)->first == 3);
+    assert(std::next(m.begin(), 2)->second == 1);
+  }
 #if TEST_STD_VER >= 11
-    {
-        typedef std::map<int, double, std::less<int>, min_allocator<std::pair<const int, double>>> M;
-        typedef std::pair<int, double> P;
-        P ar[] =
-        {
-            P(1, 1),
-            P(1, 1.5),
-            P(1, 2),
-            P(2, 1),
-            P(2, 1.5),
-            P(2, 2),
-            P(3, 1),
-            P(3, 1.5),
-            P(3, 2),
-        };
-        M m;
-        m.insert(cpp17_input_iterator<P*>(ar), cpp17_input_iterator<P*>(ar + sizeof(ar)/sizeof(ar[0])));
-        assert(m.size() == 3);
-        assert(m.begin()->first == 1);
-        assert(m.begin()->second == 1);
-        assert(std::next(m.begin())->first == 2);
-        assert(std::next(m.begin())->second == 1);
-        assert(std::next(m.begin(), 2)->first == 3);
-        assert(std::next(m.begin(), 2)->second == 1);
-    }
+  {
+    typedef std::map<int, double, std::less<int>, min_allocator<std::pair<const int, double>>> M;
+    typedef std::pair<int, double> P;
+    P ar[] = {
+        P(1, 1),
+        P(1, 1.5),
+        P(1, 2),
+        P(2, 1),
+        P(2, 1.5),
+        P(2, 2),
+        P(3, 1),
+        P(3, 1.5),
+        P(3, 2),
+    };
+    M m;
+    m.insert(cpp17_input_iterator<P*>(ar), cpp17_input_iterator<P*>(ar + sizeof(ar) / sizeof(ar[0])));
+    assert(m.size() == 3);
+    assert(m.begin()->first == 1);
+    assert(m.begin()->second == 1);
+    assert(std::next(m.begin())->first == 2);
+    assert(std::next(m.begin())->second == 1);
+    assert(std::next(m.begin(), 2)->first == 3);
+    assert(std::next(m.begin(), 2)->second == 1);
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/associative/map/map.modifiers/insert_iter_rv.pass.cpp b/libcxx/test/std/containers/associative/map/map.modifiers/insert_iter_rv.pass.cpp
index 443271d5b4739..5d72bae39bad1 100644
--- a/libcxx/test/std/containers/associative/map/map.modifiers/insert_iter_rv.pass.cpp
+++ b/libcxx/test/std/containers/associative/map/map.modifiers/insert_iter_rv.pass.cpp
@@ -23,77 +23,74 @@
 #include "test_macros.h"
 
 template <class Container, class Pair>
-void do_insert_iter_rv_test()
-{
-    typedef Container M;
-    typedef Pair P;
-    typedef typename M::iterator R;
+void do_insert_iter_rv_test() {
+  typedef Container M;
+  typedef Pair P;
+  typedef typename M::iterator R;
+  M m;
+  R r = m.insert(m.end(), P(2, 2));
+  assert(r == m.begin());
+  assert(m.size() == 1);
+  assert(r->first == 2);
+  assert(r->second == 2);
+
+  r = m.insert(m.end(), P(1, 1));
+  assert(r == m.begin());
+  assert(m.size() == 2);
+  assert(r->first == 1);
+  assert(r->second == 1);
+
+  r = m.insert(m.end(), P(3, 3));
+  assert(r == std::prev(m.end()));
+  assert(m.size() == 3);
+  assert(r->first == 3);
+  assert(r->second == 3);
+
+  r = m.insert(m.end(), P(3, 4));
+  assert(r == std::prev(m.end()));
+  assert(m.size() == 3);
+  assert(r->first == 3);
+  assert(r->second == 3);
+}
+int main(int, char**) {
+  do_insert_iter_rv_test<std::map<int, MoveOnly>, std::pair<int, MoveOnly>>();
+  do_insert_iter_rv_test<std::map<int, MoveOnly>, std::pair<const int, MoveOnly>>();
+
+  {
+    typedef std::map<int, MoveOnly, std::less<int>, min_allocator<std::pair<const int, MoveOnly>>> M;
+    typedef std::pair<int, MoveOnly> P;
+    typedef std::pair<const int, MoveOnly> CP;
+    do_insert_iter_rv_test<M, P>();
+    do_insert_iter_rv_test<M, CP>();
+  }
+  {
+    typedef std::map<int, MoveOnly> M;
+    typedef M::iterator R;
     M m;
-    R r = m.insert(m.end(), P(2, 2));
+    R r = m.insert(m.end(), {2, MoveOnly(2)});
     assert(r == m.begin());
     assert(m.size() == 1);
     assert(r->first == 2);
     assert(r->second == 2);
 
-    r = m.insert(m.end(), P(1, 1));
+    r = m.insert(m.end(), {1, MoveOnly(1)});
     assert(r == m.begin());
     assert(m.size() == 2);
     assert(r->first == 1);
     assert(r->second == 1);
 
-    r = m.insert(m.end(), P(3, 3));
+    r = m.insert(m.end(), {3, MoveOnly(3)});
     assert(r == std::prev(m.end()));
     assert(m.size() == 3);
     assert(r->first == 3);
     assert(r->second == 3);
 
-    r = m.insert(m.end(), P(3, 4));
+    r = m.insert(m.end(), {3, MoveOnly(3)});
     assert(r == std::prev(m.end()));
     assert(m.size() == 3);
     assert(r->first == 3);
     assert(r->second == 3);
-}
-int main(int, char**)
-{
-    do_insert_iter_rv_test<std::map<int, MoveOnly>, std::pair<int, MoveOnly>>();
-    do_insert_iter_rv_test<std::map<int, MoveOnly>, std::pair<const int, MoveOnly>>();
-
-    {
-        typedef std::map<int, MoveOnly, std::less<int>, min_allocator<std::pair<const int, MoveOnly>>> M;
-        typedef std::pair<int, MoveOnly> P;
-        typedef std::pair<const int, MoveOnly> CP;
-        do_insert_iter_rv_test<M, P>();
-        do_insert_iter_rv_test<M, CP>();
-    }
-    {
-        typedef std::map<int, MoveOnly> M;
-        typedef M::iterator R;
-        M m;
-        R r = m.insert(m.end(), {2, MoveOnly(2)});
-        assert(r == m.begin());
-        assert(m.size() == 1);
-        assert(r->first == 2);
-        assert(r->second == 2);
-
-        r = m.insert(m.end(), {1, MoveOnly(1)});
-        assert(r == m.begin());
-        assert(m.size() == 2);
-        assert(r->first == 1);
-        assert(r->second == 1);
-
-        r = m.insert(m.end(), {3, MoveOnly(3)});
-        assert(r == std::prev(m.end()));
-        assert(m.size() == 3);
-        assert(r->first == 3);
-        assert(r->second == 3);
-
-        r = m.insert(m.end(), {3, MoveOnly(3)});
-        assert(r == std::prev(m.end()));
-        assert(m.size() == 3);
-        assert(r->first == 3);
-        assert(r->second == 3);
-    }
-
+  }
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/associative/map/map.modifiers/insert_node_type.pass.cpp b/libcxx/test/std/containers/associative/map/map.modifiers/insert_node_type.pass.cpp
index 5d910bc3368be..1875c3635d9d2 100644
--- a/libcxx/test/std/containers/associative/map/map.modifiers/insert_node_type.pass.cpp
+++ b/libcxx/test/std/containers/associative/map/map.modifiers/insert_node_type.pass.cpp
@@ -43,70 +43,64 @@ void verify_insert_return_type(T&& t) {
 
 template <class Container>
 typename Container::node_type
-node_factory(typename Container::key_type const& key,
-             typename Container::mapped_type const& mapped)
-{
-    static Container c;
-    auto p = c.insert({key, mapped});
-    assert(p.second);
-    return c.extract(p.first);
+node_factory(typename Container::key_type const& key, typename Container::mapped_type const& mapped) {
+  static Container c;
+  auto p = c.insert({key, mapped});
+  assert(p.second);
+  return c.extract(p.first);
 }
 
 template <class Container>
-void test(Container& c)
-{
-    auto* nf = &node_factory<Container>;
-
-    for (int i = 0; i != 10; ++i)
-    {
-        typename Container::node_type node = nf(i, i + 1);
-        assert(!node.empty());
-        typename Container::insert_return_type irt = c.insert(std::move(node));
-        assert(node.empty());
-        assert(irt.inserted);
-        assert(irt.node.empty());
-        assert(irt.position->first == i && irt.position->second == i + 1);
-        verify_insert_return_type<Container>(irt);
-    }
-
-    assert(c.size() == 10);
-
-    { // Insert empty node.
-        typename Container::node_type def;
-        auto irt = c.insert(std::move(def));
-        assert(def.empty());
-        assert(!irt.inserted);
-        assert(irt.node.empty());
-        assert(irt.position == c.end());
-        verify_insert_return_type<Container>(irt);
-    }
-
-    { // Insert duplicate node.
-        typename Container::node_type dupl = nf(0, 42);
-        auto irt = c.insert(std::move(dupl));
-        assert(dupl.empty());
-        assert(!irt.inserted);
-        assert(!irt.node.empty());
-        assert(irt.position == c.find(0));
-        assert(irt.node.key() == 0 && irt.node.mapped() == 42);
-        verify_insert_return_type<Container>(irt);
-    }
-
-    assert(c.size() == 10);
-
-    for (int i = 0; i != 10; ++i)
-    {
-        assert(c.count(i) == 1);
-        assert(c[i] == i + 1);
-    }
+void test(Container& c) {
+  auto* nf = &node_factory<Container>;
+
+  for (int i = 0; i != 10; ++i) {
+    typename Container::node_type node = nf(i, i + 1);
+    assert(!node.empty());
+    typename Container::insert_return_type irt = c.insert(std::move(node));
+    assert(node.empty());
+    assert(irt.inserted);
+    assert(irt.node.empty());
+    assert(irt.position->first == i && irt.position->second == i + 1);
+    verify_insert_return_type<Container>(irt);
+  }
+
+  assert(c.size() == 10);
+
+  { // Insert empty node.
+    typename Container::node_type def;
+    auto irt = c.insert(std::move(def));
+    assert(def.empty());
+    assert(!irt.inserted);
+    assert(irt.node.empty());
+    assert(irt.position == c.end());
+    verify_insert_return_type<Container>(irt);
+  }
+
+  { // Insert duplicate node.
+    typename Container::node_type dupl = nf(0, 42);
+    auto irt                           = c.insert(std::move(dupl));
+    assert(dupl.empty());
+    assert(!irt.inserted);
+    assert(!irt.node.empty());
+    assert(irt.position == c.find(0));
+    assert(irt.node.key() == 0 && irt.node.mapped() == 42);
+    verify_insert_return_type<Container>(irt);
+  }
+
+  assert(c.size() == 10);
+
+  for (int i = 0; i != 10; ++i) {
+    assert(c.count(i) == 1);
+    assert(c[i] == i + 1);
+  }
 }
 
-int main(int, char**)
-{
-    std::map<int, int> m;
-    test(m);
-    std::map<int, int, std::less<int>, min_allocator<std::pair<const int, int>>> m2;
-    test(m2);
+int main(int, char**) {
+  std::map<int, int> m;
+  test(m);
+  std::map<int, int, std::less<int>, min_allocator<std::pair<const int, int>>> m2;
+  test(m2);
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/associative/map/map.modifiers/insert_node_type_hint.pass.cpp b/libcxx/test/std/containers/associative/map/map.modifiers/insert_node_type_hint.pass.cpp
index 3ef005480e8c2..f7db47173bc3b 100644
--- a/libcxx/test/std/containers/associative/map/map.modifiers/insert_node_type_hint.pass.cpp
+++ b/libcxx/test/std/containers/associative/map/map.modifiers/insert_node_type_hint.pass.cpp
@@ -20,47 +20,41 @@
 
 template <class Container>
 typename Container::node_type
-node_factory(typename Container::key_type const& key,
-             typename Container::mapped_type const& mapped)
-{
-    static Container c;
-    auto p = c.insert({key, mapped});
-    assert(p.second);
-    return c.extract(p.first);
+node_factory(typename Container::key_type const& key, typename Container::mapped_type const& mapped) {
+  static Container c;
+  auto p = c.insert({key, mapped});
+  assert(p.second);
+  return c.extract(p.first);
 }
 
 template <class Container>
-void test(Container& c)
-{
-    auto* nf = &node_factory<Container>;
-
-    for (int i = 0; i != 10; ++i)
-    {
-        typename Container::node_type node = nf(i, i + 1);
-        assert(!node.empty());
-        std::size_t prev = c.size();
-        auto it = c.insert(c.end(), std::move(node));
-        assert(node.empty());
-        assert(prev + 1 == c.size());
-        assert(it->first == i);
-        assert(it->second == i + 1);
-    }
-
-    assert(c.size() == 10);
-
-    for (int i = 0; i != 10; ++i)
-    {
-        assert(c.count(i) == 1);
-        assert(c[i] == i + 1);
-    }
+void test(Container& c) {
+  auto* nf = &node_factory<Container>;
+
+  for (int i = 0; i != 10; ++i) {
+    typename Container::node_type node = nf(i, i + 1);
+    assert(!node.empty());
+    std::size_t prev = c.size();
+    auto it          = c.insert(c.end(), std::move(node));
+    assert(node.empty());
+    assert(prev + 1 == c.size());
+    assert(it->first == i);
+    assert(it->second == i + 1);
+  }
+
+  assert(c.size() == 10);
+
+  for (int i = 0; i != 10; ++i) {
+    assert(c.count(i) == 1);
+    assert(c[i] == i + 1);
+  }
 }
 
-int main(int, char**)
-{
-    std::map<int, int> m;
-    test(m);
-    std::map<int, int, std::less<int>, min_allocator<std::pair<const int, int>>> m2;
-    test(m2);
+int main(int, char**) {
+  std::map<int, int> m;
+  test(m);
+  std::map<int, int, std::less<int>, min_allocator<std::pair<const int, int>>> m2;
+  test(m2);
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/associative/map/map.modifiers/insert_or_assign.pass.cpp b/libcxx/test/std/containers/associative/map/map.modifiers/insert_or_assign.pass.cpp
index b865888eaf795..8a129b0295180 100644
--- a/libcxx/test/std/containers/associative/map/map.modifiers/insert_or_assign.pass.cpp
+++ b/libcxx/test/std/containers/associative/map/map.modifiers/insert_or_assign.pass.cpp
@@ -28,262 +28,260 @@
 
 #include "test_macros.h"
 
-class Moveable
-{
-    Moveable(const Moveable&);
-    Moveable& operator=(const Moveable&);
+class Moveable {
+  Moveable(const Moveable&);
+  Moveable& operator=(const Moveable&);
+
+  int int_;
+  double double_;
 
-    int int_;
-    double double_;
 public:
-    Moveable() : int_(0), double_(0) {}
-    Moveable(int i, double d) : int_(i), double_(d) {}
-    Moveable(Moveable&& x)
-        : int_(x.int_), double_(x.double_)
-            {x.int_ = -1; x.double_ = -1;}
-    Moveable& operator=(Moveable&& x)
-        {int_ = x.int_; x.int_ = -1;
-         double_ = x.double_; x.double_ = -1;
-         return *this;
-        }
-
-    bool operator==(const Moveable& x) const
-        {return int_ == x.int_ && double_ == x.double_;}
-    bool operator<(const Moveable& x) const
-        {return int_ < x.int_ || (int_ == x.int_ && double_ < x.double_);}
-
-    int get() const {return int_;}
-    bool moved() const {return int_ == -1;}
+  Moveable() : int_(0), double_(0) {}
+  Moveable(int i, double d) : int_(i), double_(d) {}
+  Moveable(Moveable&& x) : int_(x.int_), double_(x.double_) {
+    x.int_    = -1;
+    x.double_ = -1;
+  }
+  Moveable& operator=(Moveable&& x) {
+    int_      = x.int_;
+    x.int_    = -1;
+    double_   = x.double_;
+    x.double_ = -1;
+    return *this;
+  }
+
+  bool operator==(const Moveable& x) const { return int_ == x.int_ && double_ == x.double_; }
+  bool operator<(const Moveable& x) const { return int_ < x.int_ || (int_ == x.int_ && double_ < x.double_); }
+
+  int get() const { return int_; }
+  bool moved() const { return int_ == -1; }
 };
 
-
-int main(int, char**)
-{
-    { // pair<iterator, bool> insert_or_assign(const key_type& k, M&& obj);
-        typedef std::map<int, Moveable> M;
-        typedef std::pair<M::iterator, bool> R;
-        M m;
-        R r;
-        for ( int i = 0; i < 20; i += 2 )
-            m.emplace ( i, Moveable(i, (double) i));
-        assert(m.size() == 10);
-
-        for (int i=0; i < 20; i += 2)
-        {
-            Moveable mv(i+1, i+1);
-            r = m.insert_or_assign(i, std::move(mv));
-            assert(m.size() == 10);
-            assert(!r.second);                    // was not inserted
-            assert(mv.moved());                   // was moved from
-            assert(r.first->first == i);          // key
-            assert(r.first->second.get() == i+1); // value
-        }
-
-        Moveable mv1(5, 5.0);
-        r = m.insert_or_assign(-1, std::move(mv1));
-        assert(m.size() == 11);
-        assert(r.second);                    // was inserted
-        assert(mv1.moved());                 // was moved from
-        assert(r.first->first        == -1); // key
-        assert(r.first->second.get() == 5);  // value
-
-        Moveable mv2(9, 9.0);
-        r = m.insert_or_assign(3, std::move(mv2));
-        assert(m.size() == 12);
-        assert(r.second);                   // was inserted
-        assert(mv2.moved());                // was moved from
-        assert(r.first->first        == 3); // key
-        assert(r.first->second.get() == 9); // value
-
-        Moveable mv3(-1, 5.0);
-        r = m.insert_or_assign(117, std::move(mv3));
-        assert(m.size() == 13);
-        assert(r.second);                     // was inserted
-        assert(mv3.moved());                  // was moved from
-        assert(r.first->first        == 117); // key
-        assert(r.first->second.get() == -1);  // value
-    }
-    { // pair<iterator, bool> insert_or_assign(key_type&& k, M&& obj);
-        typedef std::map<Moveable, Moveable> M;
-        typedef std::pair<M::iterator, bool> R;
-        M m;
-        R r;
-        for ( int i = 0; i < 20; i += 2 )
-            m.emplace ( Moveable(i, (double) i), Moveable(i+1, (double) i+1));
-        assert(m.size() == 10);
-
-        Moveable mvkey1(2, 2.0);
-        Moveable mv1(4, 4.0);
-        r = m.insert_or_assign(std::move(mvkey1), std::move(mv1));
-        assert(m.size() == 10);
-        assert(!r.second);                  // was not inserted
-        assert(!mvkey1.moved());            // was not moved from
-        assert(mv1.moved());                // was moved from
-        assert(r.first->first == mvkey1);   // key
-        assert(r.first->second.get() == 4); // value
-
-        Moveable mvkey2(3, 3.0);
-        Moveable mv2(5, 5.0);
-        r = m.try_emplace(std::move(mvkey2), std::move(mv2));
-        assert(m.size() == 11);
-        assert(r.second);                   // was inserted
-        assert(mv2.moved());                // was moved from
-        assert(mvkey2.moved());             // was moved from
-        assert(r.first->first.get()  == 3); // key
-        assert(r.first->second.get() == 5); // value
-    }
-    { // iterator insert_or_assign(const_iterator hint, const key_type& k, M&& obj);
-        typedef std::map<int, Moveable> M;
-        M m;
-        M::iterator r;
-        for ( int i = 0; i < 20; i += 2 )
-            m.emplace ( i, Moveable(i, (double) i));
-        assert(m.size() == 10);
-        M::const_iterator it = m.find(2);
-
-        Moveable mv1(3, 3.0);
-        r = m.insert_or_assign(it, 2, std::move(mv1));
-        assert(m.size() == 10);
-        assert(mv1.moved());           // was moved from
-        assert(r->first        == 2);  // key
-        assert(r->second.get() == 3);  // value
-
-        Moveable mv2(5, 5.0);
-        r = m.insert_or_assign(it, 3, std::move(mv2));
-        assert(m.size() == 11);
-        assert(mv2.moved());           // was moved from
-        assert(r->first        == 3);  // key
-        assert(r->second.get() == 5);  // value
-
-        // wrong hint: begin()
-        Moveable mv3(7, 7.0);
-        r = m.insert_or_assign(m.begin(), 4, std::move(mv3));
-        assert(m.size() == 11);
-        assert(mv3.moved());           // was moved from
-        assert(r->first        == 4);  // key
-        assert(r->second.get() == 7);  // value
-
-        Moveable mv4(9, 9.0);
-        r = m.insert_or_assign(m.begin(), 5, std::move(mv4));
-        assert(m.size() == 12);
-        assert(mv4.moved());           // was moved from
-        assert(r->first        == 5);  // key
-        assert(r->second.get() == 9);  // value
-
-        // wrong hint: end()
-        Moveable mv5(11, 11.0);
-        r = m.insert_or_assign(m.end(), 6, std::move(mv5));
-        assert(m.size() == 12);
-        assert(mv5.moved());           // was moved from
-        assert(r->first        == 6);  // key
-        assert(r->second.get() == 11); // value
-
-        Moveable mv6(13, 13.0);
-        r = m.insert_or_assign(m.end(), 7, std::move(mv6));
-        assert(m.size() == 13);
-        assert(mv6.moved());           // was moved from
-        assert(r->first        == 7);  // key
-        assert(r->second.get() == 13); // value
-
-        // wrong hint: third element
-        Moveable mv7(15, 15.0);
-        r = m.insert_or_assign(std::next(m.begin(), 2), 8, std::move(mv7));
-        assert(m.size() == 13);
-        assert(mv7.moved());           // was moved from
-        assert(r->first        == 8);  // key
-        assert(r->second.get() == 15); // value
-
-        Moveable mv8(17, 17.0);
-        r = m.insert_or_assign(std::next(m.begin(), 2), 9, std::move(mv8));
-        assert(m.size() == 14);
-        assert(mv8.moved());           // was moved from
-        assert(r->first        == 9);  // key
-        assert(r->second.get() == 17); // value
-    }
-    { // iterator insert_or_assign(const_iterator hint, key_type&& k, M&& obj);
-        typedef std::map<Moveable, Moveable> M;
-        M m;
-        M::iterator r;
-        for ( int i = 0; i < 20; i += 2 )
-            m.emplace ( Moveable(i, (double) i), Moveable(i+1, (double) i+1));
-        assert(m.size() == 10);
-        M::const_iterator it = std::next(m.cbegin());
-
-        Moveable mvkey1(2, 2.0);
-        Moveable mv1(4, 4.0);
-        r = m.insert_or_assign(it, std::move(mvkey1), std::move(mv1));
-        assert(m.size() == 10);
-        assert(mv1.moved());          // was moved from
-        assert(!mvkey1.moved());      // was not moved from
-        assert(r->first == mvkey1);   // key
-        assert(r->second.get() == 4); // value
-
-        Moveable mvkey2(3, 3.0);
-        Moveable mv2(5, 5.0);
-        r = m.insert_or_assign(it, std::move(mvkey2), std::move(mv2));
-        assert(m.size() == 11);
-        assert(mv2.moved());           // was moved from
-        assert(mvkey2.moved());        // was moved from
-        assert(r->first.get()  == 3);  // key
-        assert(r->second.get() == 5);  // value
-
-        // wrong hint: begin()
-        Moveable mvkey3(6, 6.0);
-        Moveable mv3(8, 8.0);
-        r = m.insert_or_assign(m.begin(), std::move(mvkey3), std::move(mv3));
-        assert(m.size() == 11);
-        assert(mv3.moved());           // was moved from
-        assert(!mvkey3.moved());       // was not moved from
-        assert(r->first == mvkey3);    // key
-        assert(r->second.get() == 8);  // value
-
-        Moveable mvkey4(7, 7.0);
-        Moveable mv4(9, 9.0);
-        r = m.insert_or_assign(m.begin(), std::move(mvkey4), std::move(mv4));
-        assert(m.size() == 12);
-        assert(mv4.moved());           // was moved from
-        assert(mvkey4.moved());        // was moved from
-        assert(r->first.get()  == 7);  // key
-        assert(r->second.get() == 9);  // value
-
-        // wrong hint: end()
-        Moveable mvkey5(8, 8.0);
-        Moveable mv5(10, 10.0);
-        r = m.insert_or_assign(m.end(), std::move(mvkey5), std::move(mv5));
-        assert(m.size() == 12);
-        assert(mv5.moved());           // was moved from
-        assert(!mvkey5.moved());       // was not moved from
-        assert(r->first == mvkey5);    // key
-        assert(r->second.get() == 10); // value
-
-        Moveable mvkey6(9, 9.0);
-        Moveable mv6(11, 11.0);
-        r = m.insert_or_assign(m.end(), std::move(mvkey6), std::move(mv6));
-        assert(m.size() == 13);
-        assert(mv6.moved());           // was moved from
-        assert(mvkey6.moved());        // was moved from
-        assert(r->first.get()  == 9);  // key
-        assert(r->second.get() == 11); // value
-
-        // wrong hint: third element
-        Moveable mvkey7(10, 10.0);
-        Moveable mv7(12, 12.0);
-        r = m.insert_or_assign(std::next(m.begin(), 2), std::move(mvkey7), std::move(mv7));
-        assert(m.size() == 13);
-        assert(mv7.moved());           // was moved from
-        assert(!mvkey7.moved());       // was not moved from
-        assert(r->first == mvkey7);    // key
-        assert(r->second.get() == 12); // value
-
-        Moveable mvkey8(11, 11.0);
-        Moveable mv8(13, 13.0);
-        r = m.insert_or_assign(std::next(m.begin(), 2), std::move(mvkey8), std::move(mv8));
-        assert(m.size() == 14);
-        assert(mv8.moved());           // was moved from
-        assert(mvkey8.moved());        // was moved from
-        assert(r->first.get()  == 11); // key
-        assert(r->second.get() == 13); // value
+int main(int, char**) {
+  { // pair<iterator, bool> insert_or_assign(const key_type& k, M&& obj);
+    typedef std::map<int, Moveable> M;
+    typedef std::pair<M::iterator, bool> R;
+    M m;
+    R r;
+    for (int i = 0; i < 20; i += 2)
+      m.emplace(i, Moveable(i, (double)i));
+    assert(m.size() == 10);
+
+    for (int i = 0; i < 20; i += 2) {
+      Moveable mv(i + 1, i + 1);
+      r = m.insert_or_assign(i, std::move(mv));
+      assert(m.size() == 10);
+      assert(!r.second);                      // was not inserted
+      assert(mv.moved());                     // was moved from
+      assert(r.first->first == i);            // key
+      assert(r.first->second.get() == i + 1); // value
     }
 
+    Moveable mv1(5, 5.0);
+    r = m.insert_or_assign(-1, std::move(mv1));
+    assert(m.size() == 11);
+    assert(r.second);                   // was inserted
+    assert(mv1.moved());                // was moved from
+    assert(r.first->first == -1);       // key
+    assert(r.first->second.get() == 5); // value
+
+    Moveable mv2(9, 9.0);
+    r = m.insert_or_assign(3, std::move(mv2));
+    assert(m.size() == 12);
+    assert(r.second);                   // was inserted
+    assert(mv2.moved());                // was moved from
+    assert(r.first->first == 3);        // key
+    assert(r.first->second.get() == 9); // value
+
+    Moveable mv3(-1, 5.0);
+    r = m.insert_or_assign(117, std::move(mv3));
+    assert(m.size() == 13);
+    assert(r.second);                    // was inserted
+    assert(mv3.moved());                 // was moved from
+    assert(r.first->first == 117);       // key
+    assert(r.first->second.get() == -1); // value
+  }
+  { // pair<iterator, bool> insert_or_assign(key_type&& k, M&& obj);
+    typedef std::map<Moveable, Moveable> M;
+    typedef std::pair<M::iterator, bool> R;
+    M m;
+    R r;
+    for (int i = 0; i < 20; i += 2)
+      m.emplace(Moveable(i, (double)i), Moveable(i + 1, (double)i + 1));
+    assert(m.size() == 10);
+
+    Moveable mvkey1(2, 2.0);
+    Moveable mv1(4, 4.0);
+    r = m.insert_or_assign(std::move(mvkey1), std::move(mv1));
+    assert(m.size() == 10);
+    assert(!r.second);                  // was not inserted
+    assert(!mvkey1.moved());            // was not moved from
+    assert(mv1.moved());                // was moved from
+    assert(r.first->first == mvkey1);   // key
+    assert(r.first->second.get() == 4); // value
+
+    Moveable mvkey2(3, 3.0);
+    Moveable mv2(5, 5.0);
+    r = m.try_emplace(std::move(mvkey2), std::move(mv2));
+    assert(m.size() == 11);
+    assert(r.second);                   // was inserted
+    assert(mv2.moved());                // was moved from
+    assert(mvkey2.moved());             // was moved from
+    assert(r.first->first.get() == 3);  // key
+    assert(r.first->second.get() == 5); // value
+  }
+  { // iterator insert_or_assign(const_iterator hint, const key_type& k, M&& obj);
+    typedef std::map<int, Moveable> M;
+    M m;
+    M::iterator r;
+    for (int i = 0; i < 20; i += 2)
+      m.emplace(i, Moveable(i, (double)i));
+    assert(m.size() == 10);
+    M::const_iterator it = m.find(2);
+
+    Moveable mv1(3, 3.0);
+    r = m.insert_or_assign(it, 2, std::move(mv1));
+    assert(m.size() == 10);
+    assert(mv1.moved());          // was moved from
+    assert(r->first == 2);        // key
+    assert(r->second.get() == 3); // value
+
+    Moveable mv2(5, 5.0);
+    r = m.insert_or_assign(it, 3, std::move(mv2));
+    assert(m.size() == 11);
+    assert(mv2.moved());          // was moved from
+    assert(r->first == 3);        // key
+    assert(r->second.get() == 5); // value
+
+    // wrong hint: begin()
+    Moveable mv3(7, 7.0);
+    r = m.insert_or_assign(m.begin(), 4, std::move(mv3));
+    assert(m.size() == 11);
+    assert(mv3.moved());          // was moved from
+    assert(r->first == 4);        // key
+    assert(r->second.get() == 7); // value
+
+    Moveable mv4(9, 9.0);
+    r = m.insert_or_assign(m.begin(), 5, std::move(mv4));
+    assert(m.size() == 12);
+    assert(mv4.moved());          // was moved from
+    assert(r->first == 5);        // key
+    assert(r->second.get() == 9); // value
+
+    // wrong hint: end()
+    Moveable mv5(11, 11.0);
+    r = m.insert_or_assign(m.end(), 6, std::move(mv5));
+    assert(m.size() == 12);
+    assert(mv5.moved());           // was moved from
+    assert(r->first == 6);         // key
+    assert(r->second.get() == 11); // value
+
+    Moveable mv6(13, 13.0);
+    r = m.insert_or_assign(m.end(), 7, std::move(mv6));
+    assert(m.size() == 13);
+    assert(mv6.moved());           // was moved from
+    assert(r->first == 7);         // key
+    assert(r->second.get() == 13); // value
+
+    // wrong hint: third element
+    Moveable mv7(15, 15.0);
+    r = m.insert_or_assign(std::next(m.begin(), 2), 8, std::move(mv7));
+    assert(m.size() == 13);
+    assert(mv7.moved());           // was moved from
+    assert(r->first == 8);         // key
+    assert(r->second.get() == 15); // value
+
+    Moveable mv8(17, 17.0);
+    r = m.insert_or_assign(std::next(m.begin(), 2), 9, std::move(mv8));
+    assert(m.size() == 14);
+    assert(mv8.moved());           // was moved from
+    assert(r->first == 9);         // key
+    assert(r->second.get() == 17); // value
+  }
+  { // iterator insert_or_assign(const_iterator hint, key_type&& k, M&& obj);
+    typedef std::map<Moveable, Moveable> M;
+    M m;
+    M::iterator r;
+    for (int i = 0; i < 20; i += 2)
+      m.emplace(Moveable(i, (double)i), Moveable(i + 1, (double)i + 1));
+    assert(m.size() == 10);
+    M::const_iterator it = std::next(m.cbegin());
+
+    Moveable mvkey1(2, 2.0);
+    Moveable mv1(4, 4.0);
+    r = m.insert_or_assign(it, std::move(mvkey1), std::move(mv1));
+    assert(m.size() == 10);
+    assert(mv1.moved());          // was moved from
+    assert(!mvkey1.moved());      // was not moved from
+    assert(r->first == mvkey1);   // key
+    assert(r->second.get() == 4); // value
+
+    Moveable mvkey2(3, 3.0);
+    Moveable mv2(5, 5.0);
+    r = m.insert_or_assign(it, std::move(mvkey2), std::move(mv2));
+    assert(m.size() == 11);
+    assert(mv2.moved());          // was moved from
+    assert(mvkey2.moved());       // was moved from
+    assert(r->first.get() == 3);  // key
+    assert(r->second.get() == 5); // value
+
+    // wrong hint: begin()
+    Moveable mvkey3(6, 6.0);
+    Moveable mv3(8, 8.0);
+    r = m.insert_or_assign(m.begin(), std::move(mvkey3), std::move(mv3));
+    assert(m.size() == 11);
+    assert(mv3.moved());          // was moved from
+    assert(!mvkey3.moved());      // was not moved from
+    assert(r->first == mvkey3);   // key
+    assert(r->second.get() == 8); // value
+
+    Moveable mvkey4(7, 7.0);
+    Moveable mv4(9, 9.0);
+    r = m.insert_or_assign(m.begin(), std::move(mvkey4), std::move(mv4));
+    assert(m.size() == 12);
+    assert(mv4.moved());          // was moved from
+    assert(mvkey4.moved());       // was moved from
+    assert(r->first.get() == 7);  // key
+    assert(r->second.get() == 9); // value
+
+    // wrong hint: end()
+    Moveable mvkey5(8, 8.0);
+    Moveable mv5(10, 10.0);
+    r = m.insert_or_assign(m.end(), std::move(mvkey5), std::move(mv5));
+    assert(m.size() == 12);
+    assert(mv5.moved());           // was moved from
+    assert(!mvkey5.moved());       // was not moved from
+    assert(r->first == mvkey5);    // key
+    assert(r->second.get() == 10); // value
+
+    Moveable mvkey6(9, 9.0);
+    Moveable mv6(11, 11.0);
+    r = m.insert_or_assign(m.end(), std::move(mvkey6), std::move(mv6));
+    assert(m.size() == 13);
+    assert(mv6.moved());           // was moved from
+    assert(mvkey6.moved());        // was moved from
+    assert(r->first.get() == 9);   // key
+    assert(r->second.get() == 11); // value
+
+    // wrong hint: third element
+    Moveable mvkey7(10, 10.0);
+    Moveable mv7(12, 12.0);
+    r = m.insert_or_assign(std::next(m.begin(), 2), std::move(mvkey7), std::move(mv7));
+    assert(m.size() == 13);
+    assert(mv7.moved());           // was moved from
+    assert(!mvkey7.moved());       // was not moved from
+    assert(r->first == mvkey7);    // key
+    assert(r->second.get() == 12); // value
+
+    Moveable mvkey8(11, 11.0);
+    Moveable mv8(13, 13.0);
+    r = m.insert_or_assign(std::next(m.begin(), 2), std::move(mvkey8), std::move(mv8));
+    assert(m.size() == 14);
+    assert(mv8.moved());           // was moved from
+    assert(mvkey8.moved());        // was moved from
+    assert(r->first.get() == 11);  // key
+    assert(r->second.get() == 13); // value
+  }
+
   return 0;
 }
diff --git a/libcxx/test/std/containers/associative/map/map.modifiers/insert_range.pass.cpp b/libcxx/test/std/containers/associative/map/map.modifiers/insert_range.pass.cpp
index 0d4b1c960a958..95d8897cea282 100644
--- a/libcxx/test/std/containers/associative/map/map.modifiers/insert_range.pass.cpp
+++ b/libcxx/test/std/containers/associative/map/map.modifiers/insert_range.pass.cpp
@@ -23,7 +23,7 @@
 int main(int, char**) {
   // Note: we want to use a pair with non-const elements for input (an assignable type is a lot more convenient) but
   // have to use the exact `value_type` of the map (that is, `pair<const K, V>`) for the allocator.
-  using Pair = std::pair<int, char>;
+  using Pair      = std::pair<int, char>;
   using ConstPair = std::pair<const int, char>;
   for_all_iterators_and_allocators<ConstPair, const Pair*>([]<class Iter, class Sent, class Alloc>() {
     test_map_set_insert_range<std::map<int, char, test_less<int>, Alloc>, Pair, Iter, Sent>();
diff --git a/libcxx/test/std/containers/associative/map/map.modifiers/insert_rv.pass.cpp b/libcxx/test/std/containers/associative/map/map.modifiers/insert_rv.pass.cpp
index afb5bba86fd3e..345bfde531cf9 100644
--- a/libcxx/test/std/containers/associative/map/map.modifiers/insert_rv.pass.cpp
+++ b/libcxx/test/std/containers/associative/map/map.modifiers/insert_rv.pass.cpp
@@ -24,85 +24,83 @@
 #include "test_macros.h"
 
 template <class Container, class Pair>
-void do_insert_rv_test()
-{
-    typedef Container M;
-    typedef Pair P;
-    typedef std::pair<typename M::iterator, bool> R;
+void do_insert_rv_test() {
+  typedef Container M;
+  typedef Pair P;
+  typedef std::pair<typename M::iterator, bool> R;
+  M m;
+  R r = m.insert(P(2, 2));
+  assert(r.second);
+  assert(r.first == m.begin());
+  assert(m.size() == 1);
+  assert(r.first->first == 2);
+  assert(r.first->second == 2);
+
+  r = m.insert(P(1, 1));
+  assert(r.second);
+  assert(r.first == m.begin());
+  assert(m.size() == 2);
+  assert(r.first->first == 1);
+  assert(r.first->second == 1);
+
+  r = m.insert(P(3, 3));
+  assert(r.second);
+  assert(r.first == std::prev(m.end()));
+  assert(m.size() == 3);
+  assert(r.first->first == 3);
+  assert(r.first->second == 3);
+
+  r = m.insert(P(3, 3));
+  assert(!r.second);
+  assert(r.first == std::prev(m.end()));
+  assert(m.size() == 3);
+  assert(r.first->first == 3);
+  assert(r.first->second == 3);
+}
+
+int main(int, char**) {
+  do_insert_rv_test<std::map<int, MoveOnly>, std::pair<int, MoveOnly>>();
+  do_insert_rv_test<std::map<int, MoveOnly>, std::pair<const int, MoveOnly>>();
+
+  {
+    typedef std::map<int, MoveOnly, std::less<int>, min_allocator<std::pair<const int, MoveOnly>>> M;
+    typedef std::pair<int, MoveOnly> P;
+    typedef std::pair<const int, MoveOnly> CP;
+    do_insert_rv_test<M, P>();
+    do_insert_rv_test<M, CP>();
+  }
+  {
+    typedef std::map<int, MoveOnly> M;
+    typedef std::pair<M::iterator, bool> R;
     M m;
-    R r = m.insert(P(2, 2));
+    R r = m.insert({2, MoveOnly(2)});
     assert(r.second);
     assert(r.first == m.begin());
     assert(m.size() == 1);
     assert(r.first->first == 2);
     assert(r.first->second == 2);
 
-    r = m.insert(P(1, 1));
+    r = m.insert({1, MoveOnly(1)});
     assert(r.second);
     assert(r.first == m.begin());
     assert(m.size() == 2);
     assert(r.first->first == 1);
     assert(r.first->second == 1);
 
-    r = m.insert(P(3, 3));
+    r = m.insert({3, MoveOnly(3)});
     assert(r.second);
     assert(r.first == std::prev(m.end()));
     assert(m.size() == 3);
     assert(r.first->first == 3);
     assert(r.first->second == 3);
 
-    r = m.insert(P(3, 3));
+    r = m.insert({3, MoveOnly(3)});
     assert(!r.second);
     assert(r.first == std::prev(m.end()));
     assert(m.size() == 3);
     assert(r.first->first == 3);
     assert(r.first->second == 3);
-}
-
-int main(int, char**)
-{
-    do_insert_rv_test<std::map<int, MoveOnly>, std::pair<int, MoveOnly>>();
-    do_insert_rv_test<std::map<int, MoveOnly>, std::pair<const int, MoveOnly>>();
-
-    {
-        typedef std::map<int, MoveOnly, std::less<int>, min_allocator<std::pair<const int, MoveOnly>>> M;
-        typedef std::pair<int, MoveOnly> P;
-        typedef std::pair<const int, MoveOnly> CP;
-        do_insert_rv_test<M, P>();
-        do_insert_rv_test<M, CP>();
-    }
-    {
-        typedef std::map<int, MoveOnly> M;
-        typedef std::pair<M::iterator, bool> R;
-        M m;
-        R r = m.insert({2, MoveOnly(2)});
-        assert(r.second);
-        assert(r.first == m.begin());
-        assert(m.size() == 1);
-        assert(r.first->first == 2);
-        assert(r.first->second == 2);
-
-        r = m.insert({1, MoveOnly(1)});
-        assert(r.second);
-        assert(r.first == m.begin());
-        assert(m.size() == 2);
-        assert(r.first->first == 1);
-        assert(r.first->second == 1);
-
-        r = m.insert({3, MoveOnly(3)});
-        assert(r.second);
-        assert(r.first == std::prev(m.end()));
-        assert(m.size() == 3);
-        assert(r.first->first == 3);
-        assert(r.first->second == 3);
-
-        r = m.insert({3, MoveOnly(3)});
-        assert(!r.second);
-        assert(r.first == std::prev(m.end()));
-        assert(m.size() == 3);
-        assert(r.first->first == 3);
-        assert(r.first->second == 3);
-    }
+  }
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/associative/map/map.modifiers/merge.pass.cpp b/libcxx/test/std/containers/associative/map/map.modifiers/merge.pass.cpp
index d760658d54f5f..3613acfa18c1d 100644
--- a/libcxx/test/std/containers/associative/map/map.modifiers/merge.pass.cpp
+++ b/libcxx/test/std/containers/associative/map/map.modifiers/merge.pass.cpp
@@ -27,124 +27,113 @@
 #include "Counter.h"
 
 template <class Map>
-bool map_equal(const Map& map, Map other)
-{
-    return map == other;
+bool map_equal(const Map& map, Map other) {
+  return map == other;
 }
 
 #ifndef TEST_HAS_NO_EXCEPTIONS
-struct throw_comparator
-{
-    bool& should_throw_;
+struct throw_comparator {
+  bool& should_throw_;
 
-    throw_comparator(bool& should_throw) : should_throw_(should_throw) {}
+  throw_comparator(bool& should_throw) : should_throw_(should_throw) {}
 
-    template <class T>
-    bool operator()(const T& lhs, const T& rhs) const
-    {
-        if (should_throw_)
-            throw 0;
-        return lhs < rhs;
-    }
+  template <class T>
+  bool operator()(const T& lhs, const T& rhs) const {
+    if (should_throw_)
+      throw 0;
+    return lhs < rhs;
+  }
 };
 #endif
 
-int main(int, char**)
-{
-    {
-        std::map<int, int> src{{1, 0}, {3, 0}, {5, 0}};
-        std::map<int, int> dst{{2, 0}, {4, 0}, {5, 0}};
-        dst.merge(src);
-        assert(map_equal(src, {{5,0}}));
-        assert(map_equal(dst, {{1, 0}, {2, 0}, {3, 0}, {4, 0}, {5, 0}}));
-    }
+int main(int, char**) {
+  {
+    std::map<int, int> src{{1, 0}, {3, 0}, {5, 0}};
+    std::map<int, int> dst{{2, 0}, {4, 0}, {5, 0}};
+    dst.merge(src);
+    assert(map_equal(src, {{5, 0}}));
+    assert(map_equal(dst, {{1, 0}, {2, 0}, {3, 0}, {4, 0}, {5, 0}}));
+  }
 
 #ifndef TEST_HAS_NO_EXCEPTIONS
-    {
-        bool do_throw = false;
-        typedef std::map<Counter<int>, int, throw_comparator> map_type;
-        map_type src({{1, 0}, {3, 0}, {5, 0}}, throw_comparator(do_throw));
-        map_type dst({{2, 0}, {4, 0}, {5, 0}}, throw_comparator(do_throw));
-
-        assert(Counter_base::gConstructed == 6);
-
-        do_throw = true;
-        try
-        {
-            dst.merge(src);
-        }
-        catch (int)
-        {
-            do_throw = false;
-        }
-        assert(!do_throw);
-        assert(map_equal(src, map_type({{1, 0}, {3, 0}, {5, 0}}, throw_comparator(do_throw))));
-        assert(map_equal(dst, map_type({{2, 0}, {4, 0}, {5, 0}}, throw_comparator(do_throw))));
+  {
+    bool do_throw = false;
+    typedef std::map<Counter<int>, int, throw_comparator> map_type;
+    map_type src({{1, 0}, {3, 0}, {5, 0}}, throw_comparator(do_throw));
+    map_type dst({{2, 0}, {4, 0}, {5, 0}}, throw_comparator(do_throw));
+
+    assert(Counter_base::gConstructed == 6);
+
+    do_throw = true;
+    try {
+      dst.merge(src);
+    } catch (int) {
+      do_throw = false;
     }
+    assert(!do_throw);
+    assert(map_equal(src, map_type({{1, 0}, {3, 0}, {5, 0}}, throw_comparator(do_throw))));
+    assert(map_equal(dst, map_type({{2, 0}, {4, 0}, {5, 0}}, throw_comparator(do_throw))));
+  }
 #endif
-    assert(Counter_base::gConstructed == 0);
-    struct comparator
-    {
-        comparator() = default;
+  assert(Counter_base::gConstructed == 0);
+  struct comparator {
+    comparator() = default;
 
-        bool operator()(const Counter<int>& lhs, const Counter<int>& rhs) const
-        {
-            return lhs < rhs;
-        }
-    };
-    {
-        typedef std::map<Counter<int>, int, std::less<Counter<int>>> first_map_type;
-        typedef std::map<Counter<int>, int, comparator> second_map_type;
-        typedef std::multimap<Counter<int>, int, comparator> third_map_type;
+    bool operator()(const Counter<int>& lhs, const Counter<int>& rhs) const { return lhs < rhs; }
+  };
+  {
+    typedef std::map<Counter<int>, int, std::less<Counter<int>>> first_map_type;
+    typedef std::map<Counter<int>, int, comparator> second_map_type;
+    typedef std::multimap<Counter<int>, int, comparator> third_map_type;
 
-        {
-            first_map_type first{{1, 0}, {2, 0}, {3, 0}};
-            second_map_type second{{2, 0}, {3, 0}, {4, 0}};
-            third_map_type third{{1, 0}, {3, 0}};
+    {
+      first_map_type first{{1, 0}, {2, 0}, {3, 0}};
+      second_map_type second{{2, 0}, {3, 0}, {4, 0}};
+      third_map_type third{{1, 0}, {3, 0}};
 
-            assert(Counter_base::gConstructed == 8);
+      assert(Counter_base::gConstructed == 8);
 
-            first.merge(second);
-            first.merge(third);
+      first.merge(second);
+      first.merge(third);
 
-            assert(map_equal(first, {{1, 0}, {2, 0}, {3, 0}, {4, 0}}));
-            assert(map_equal(second, {{2, 0}, {3, 0}}));
-            assert(map_equal(third, {{1, 0}, {3, 0}}));
+      assert(map_equal(first, {{1, 0}, {2, 0}, {3, 0}, {4, 0}}));
+      assert(map_equal(second, {{2, 0}, {3, 0}}));
+      assert(map_equal(third, {{1, 0}, {3, 0}}));
 
-            assert(Counter_base::gConstructed == 8);
-        }
-        assert(Counter_base::gConstructed == 0);
-        {
-            first_map_type first{{1, 0}, {2, 0}, {3, 0}};
-            second_map_type second{{2, 0}, {3, 0}, {4, 0}};
-            third_map_type third{{1, 0}, {3, 0}};
+      assert(Counter_base::gConstructed == 8);
+    }
+    assert(Counter_base::gConstructed == 0);
+    {
+      first_map_type first{{1, 0}, {2, 0}, {3, 0}};
+      second_map_type second{{2, 0}, {3, 0}, {4, 0}};
+      third_map_type third{{1, 0}, {3, 0}};
 
-            assert(Counter_base::gConstructed == 8);
+      assert(Counter_base::gConstructed == 8);
 
-            first.merge(std::move(second));
-            first.merge(std::move(third));
+      first.merge(std::move(second));
+      first.merge(std::move(third));
 
-            assert(map_equal(first, {{1, 0}, {2, 0}, {3, 0}, {4, 0}}));
-            assert(map_equal(second, {{2, 0}, {3, 0}}));
-            assert(map_equal(third, {{1, 0}, {3, 0}}));
+      assert(map_equal(first, {{1, 0}, {2, 0}, {3, 0}, {4, 0}}));
+      assert(map_equal(second, {{2, 0}, {3, 0}}));
+      assert(map_equal(third, {{1, 0}, {3, 0}}));
 
-            assert(Counter_base::gConstructed == 8);
-        }
-        assert(Counter_base::gConstructed == 0);
+      assert(Counter_base::gConstructed == 8);
     }
     assert(Counter_base::gConstructed == 0);
+  }
+  assert(Counter_base::gConstructed == 0);
+  {
+    std::map<int, int> first;
+    {
+      std::map<int, int> second;
+      first.merge(second);
+      first.merge(std::move(second));
+    }
     {
-        std::map<int, int> first;
-        {
-            std::map<int, int> second;
-            first.merge(second);
-            first.merge(std::move(second));
-        }
-        {
-            std::multimap<int, int> second;
-            first.merge(second);
-            first.merge(std::move(second));
-        }
+      std::multimap<int, int> second;
+      first.merge(second);
+      first.merge(std::move(second));
     }
-    return 0;
+  }
+  return 0;
 }
diff --git a/libcxx/test/std/containers/associative/map/map.modifiers/try.emplace.pass.cpp b/libcxx/test/std/containers/associative/map/map.modifiers/try.emplace.pass.cpp
index 3f7a83c2b5418..a087704fb4daf 100644
--- a/libcxx/test/std/containers/associative/map/map.modifiers/try.emplace.pass.cpp
+++ b/libcxx/test/std/containers/associative/map/map.modifiers/try.emplace.pass.cpp
@@ -28,159 +28,156 @@
 
 #include "test_macros.h"
 
-class Moveable
-{
-    Moveable(const Moveable&);
-    Moveable& operator=(const Moveable&);
+class Moveable {
+  Moveable(const Moveable&);
+  Moveable& operator=(const Moveable&);
+
+  int int_;
+  double double_;
 
-    int int_;
-    double double_;
 public:
-    Moveable() : int_(0), double_(0) {}
-    Moveable(int i, double d) : int_(i), double_(d) {}
-    Moveable(Moveable&& x)
-        : int_(x.int_), double_(x.double_)
-            {x.int_ = -1; x.double_ = -1;}
-    Moveable& operator=(Moveable&& x)
-        {int_ = x.int_; x.int_ = -1;
-         double_ = x.double_; x.double_ = -1;
-         return *this;
-        }
-
-    bool operator==(const Moveable& x) const
-        {return int_ == x.int_ && double_ == x.double_;}
-    bool operator<(const Moveable& x) const
-        {return int_ < x.int_ || (int_ == x.int_ && double_ < x.double_);}
-
-    int get() const {return int_;}
-    bool moved() const {return int_ == -1;}
+  Moveable() : int_(0), double_(0) {}
+  Moveable(int i, double d) : int_(i), double_(d) {}
+  Moveable(Moveable&& x) : int_(x.int_), double_(x.double_) {
+    x.int_    = -1;
+    x.double_ = -1;
+  }
+  Moveable& operator=(Moveable&& x) {
+    int_      = x.int_;
+    x.int_    = -1;
+    double_   = x.double_;
+    x.double_ = -1;
+    return *this;
+  }
+
+  bool operator==(const Moveable& x) const { return int_ == x.int_ && double_ == x.double_; }
+  bool operator<(const Moveable& x) const { return int_ < x.int_ || (int_ == x.int_ && double_ < x.double_); }
+
+  int get() const { return int_; }
+  bool moved() const { return int_ == -1; }
 };
 
-
-int main(int, char**)
-{
-    { // pair<iterator, bool> try_emplace(const key_type& k, Args&&... args);
-        typedef std::map<int, Moveable> M;
-        typedef std::pair<M::iterator, bool> R;
-        M m;
-        R r;
-        for (int i = 0; i < 20; i += 2)
-            m.emplace (i, Moveable(i, (double) i));
-        assert(m.size() == 10);
-
-        Moveable mv1(3, 3.0);
-        for (int i=0; i < 20; i += 2)
-        {
-            r = m.try_emplace(i, std::move(mv1));
-            assert(m.size() == 10);
-            assert(!r.second);              // was not inserted
-            assert(!mv1.moved());           // was not moved from
-            assert(r.first->first == i);    // key
-        }
-
-        r = m.try_emplace(-1, std::move(mv1));
-        assert(m.size() == 11);
-        assert(r.second);                   // was inserted
-        assert(mv1.moved());                // was moved from
-        assert(r.first->first == -1);       // key
-        assert(r.first->second.get() == 3); // value
-
-        Moveable mv2(5, 3.0);
-        r = m.try_emplace(5, std::move(mv2));
-        assert(m.size() == 12);
-        assert(r.second);                   // was inserted
-        assert(mv2.moved());                // was moved from
-        assert(r.first->first == 5);        // key
-        assert(r.first->second.get() == 5); // value
-
-        Moveable mv3(-1, 3.0);
-        r = m.try_emplace(117, std::move(mv2));
-        assert(m.size() == 13);
-        assert(r.second);                    // was inserted
-        assert(mv2.moved());                 // was moved from
-        assert(r.first->first == 117);       // key
-        assert(r.first->second.get() == -1); // value
+int main(int, char**) {
+  { // pair<iterator, bool> try_emplace(const key_type& k, Args&&... args);
+    typedef std::map<int, Moveable> M;
+    typedef std::pair<M::iterator, bool> R;
+    M m;
+    R r;
+    for (int i = 0; i < 20; i += 2)
+      m.emplace(i, Moveable(i, (double)i));
+    assert(m.size() == 10);
+
+    Moveable mv1(3, 3.0);
+    for (int i = 0; i < 20; i += 2) {
+      r = m.try_emplace(i, std::move(mv1));
+      assert(m.size() == 10);
+      assert(!r.second);           // was not inserted
+      assert(!mv1.moved());        // was not moved from
+      assert(r.first->first == i); // key
     }
 
-    {  // pair<iterator, bool> try_emplace(key_type&& k, Args&&... args);
-        typedef std::map<Moveable, Moveable> M;
-        typedef std::pair<M::iterator, bool> R;
-        M m;
-        R r;
-        for ( int i = 0; i < 20; i += 2 )
-            m.emplace ( Moveable(i, (double) i), Moveable(i+1, (double) i+1));
-        assert(m.size() == 10);
-
-        Moveable mvkey1(2, 2.0);
-        Moveable mv1(4, 4.0);
-        r = m.try_emplace(std::move(mvkey1), std::move(mv1));
-        assert(m.size() == 10);
-        assert(!r.second);                 // was not inserted
-        assert(!mv1.moved());              // was not moved from
-        assert(!mvkey1.moved());           // was not moved from
-        assert(r.first->first == mvkey1);  // key
-
-        Moveable mvkey2(3, 3.0);
-        r = m.try_emplace(std::move(mvkey2), std::move(mv1));
-        assert(m.size() == 11);
-        assert(r.second);                   // was inserted
-        assert(mv1.moved());                // was moved from
-        assert(mvkey2.moved());             // was moved from
-        assert(r.first->first.get()  == 3); // key
-        assert(r.first->second.get() == 4); // value
+    r = m.try_emplace(-1, std::move(mv1));
+    assert(m.size() == 11);
+    assert(r.second);                   // was inserted
+    assert(mv1.moved());                // was moved from
+    assert(r.first->first == -1);       // key
+    assert(r.first->second.get() == 3); // value
+
+    Moveable mv2(5, 3.0);
+    r = m.try_emplace(5, std::move(mv2));
+    assert(m.size() == 12);
+    assert(r.second);                   // was inserted
+    assert(mv2.moved());                // was moved from
+    assert(r.first->first == 5);        // key
+    assert(r.first->second.get() == 5); // value
+
+    Moveable mv3(-1, 3.0);
+    r = m.try_emplace(117, std::move(mv2));
+    assert(m.size() == 13);
+    assert(r.second);                    // was inserted
+    assert(mv2.moved());                 // was moved from
+    assert(r.first->first == 117);       // key
+    assert(r.first->second.get() == -1); // value
+  }
+
+  { // pair<iterator, bool> try_emplace(key_type&& k, Args&&... args);
+    typedef std::map<Moveable, Moveable> M;
+    typedef std::pair<M::iterator, bool> R;
+    M m;
+    R r;
+    for (int i = 0; i < 20; i += 2)
+      m.emplace(Moveable(i, (double)i), Moveable(i + 1, (double)i + 1));
+    assert(m.size() == 10);
+
+    Moveable mvkey1(2, 2.0);
+    Moveable mv1(4, 4.0);
+    r = m.try_emplace(std::move(mvkey1), std::move(mv1));
+    assert(m.size() == 10);
+    assert(!r.second);                // was not inserted
+    assert(!mv1.moved());             // was not moved from
+    assert(!mvkey1.moved());          // was not moved from
+    assert(r.first->first == mvkey1); // key
+
+    Moveable mvkey2(3, 3.0);
+    r = m.try_emplace(std::move(mvkey2), std::move(mv1));
+    assert(m.size() == 11);
+    assert(r.second);                   // was inserted
+    assert(mv1.moved());                // was moved from
+    assert(mvkey2.moved());             // was moved from
+    assert(r.first->first.get() == 3);  // key
+    assert(r.first->second.get() == 4); // value
+  }
+
+  { // iterator try_emplace(const_iterator hint, const key_type& k, Args&&... args);
+    typedef std::map<int, Moveable> M;
+    M m;
+    M::iterator r;
+    for (int i = 0; i < 20; i += 2)
+      m.try_emplace(i, Moveable(i, (double)i));
+    assert(m.size() == 10);
+    M::const_iterator it = m.find(2);
+
+    Moveable mv1(3, 3.0);
+    for (int i = 0; i < 20; i += 2) {
+      r = m.try_emplace(it, i, std::move(mv1));
+      assert(m.size() == 10);
+      assert(!mv1.moved());         // was not moved from
+      assert(r->first == i);        // key
+      assert(r->second.get() == i); // value
     }
 
-    {  // iterator try_emplace(const_iterator hint, const key_type& k, Args&&... args);
-        typedef std::map<int, Moveable> M;
-        M m;
-        M::iterator r;
-        for ( int i = 0; i < 20; i += 2 )
-            m.try_emplace ( i, Moveable(i, (double) i));
-        assert(m.size() == 10);
-        M::const_iterator it = m.find(2);
-
-        Moveable mv1(3, 3.0);
-        for (int i=0; i < 20; i += 2)
-        {
-            r = m.try_emplace(it, i, std::move(mv1));
-            assert(m.size() == 10);
-            assert(!mv1.moved());         // was not moved from
-            assert(r->first == i);        // key
-            assert(r->second.get() == i); // value
-        }
-
-        r = m.try_emplace(it, 3, std::move(mv1));
-        assert(m.size() == 11);
-        assert(mv1.moved());          // was moved from
-        assert(r->first == 3);        // key
-        assert(r->second.get() == 3); // value
-    }
-
-    {  // iterator try_emplace(const_iterator hint, key_type&& k, Args&&... args);
-        typedef std::map<Moveable, Moveable> M;
-        M m;
-        M::iterator r;
-        for ( int i = 0; i < 20; i += 2 )
-            m.emplace ( Moveable(i, (double) i), Moveable(i+1, (double) i+1));
-        assert(m.size() == 10);
-        M::const_iterator it = std::next(m.cbegin());
-
-        Moveable mvkey1(2, 2.0);
-        Moveable mv1(4, 4.0);
-        r = m.try_emplace(it, std::move(mvkey1), std::move(mv1));
-        assert(m.size() == 10);
-        assert(!mv1.moved());        // was not moved from
-        assert(!mvkey1.moved());     // was not moved from
-        assert(r->first == mvkey1);  // key
-
-        Moveable mvkey2(3, 3.0);
-        r = m.try_emplace(it, std::move(mvkey2), std::move(mv1));
-        assert(m.size() == 11);
-        assert(mv1.moved());          // was moved from
-        assert(mvkey2.moved());       // was moved from
-        assert(r->first.get()  == 3); // key
-        assert(r->second.get() == 4); // value
-    }
+    r = m.try_emplace(it, 3, std::move(mv1));
+    assert(m.size() == 11);
+    assert(mv1.moved());          // was moved from
+    assert(r->first == 3);        // key
+    assert(r->second.get() == 3); // value
+  }
+
+  { // iterator try_emplace(const_iterator hint, key_type&& k, Args&&... args);
+    typedef std::map<Moveable, Moveable> M;
+    M m;
+    M::iterator r;
+    for (int i = 0; i < 20; i += 2)
+      m.emplace(Moveable(i, (double)i), Moveable(i + 1, (double)i + 1));
+    assert(m.size() == 10);
+    M::const_iterator it = std::next(m.cbegin());
+
+    Moveable mvkey1(2, 2.0);
+    Moveable mv1(4, 4.0);
+    r = m.try_emplace(it, std::move(mvkey1), std::move(mv1));
+    assert(m.size() == 10);
+    assert(!mv1.moved());       // was not moved from
+    assert(!mvkey1.moved());    // was not moved from
+    assert(r->first == mvkey1); // key
+
+    Moveable mvkey2(3, 3.0);
+    r = m.try_emplace(it, std::move(mvkey2), std::move(mv1));
+    assert(m.size() == 11);
+    assert(mv1.moved());          // was moved from
+    assert(mvkey2.moved());       // was moved from
+    assert(r->first.get() == 3);  // key
+    assert(r->second.get() == 4); // value
+  }
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/associative/map/map.nonmember/op_compare.pass.cpp b/libcxx/test/std/containers/associative/map/map.nonmember/op_compare.pass.cpp
index 21d7669b42631..9e376f6a1862b 100644
--- a/libcxx/test/std/containers/associative/map/map.nonmember/op_compare.pass.cpp
+++ b/libcxx/test/std/containers/associative/map/map.nonmember/op_compare.pass.cpp
@@ -39,43 +39,43 @@
 #include "test_comparisons.h"
 
 int main(int, char**) {
-    typedef std::map<int, std::string> map_type;
-    typedef map_type::value_type value_type;
-    {
-        map_type m1, m2;
-        m1.insert(value_type(1, "abc"));
-        m2.insert(value_type(2, "abc"));
-        const map_type& cm1 = m1, cm2 = m2;
-        assert(testComparisons(cm1, cm2, false, true));
-    }
-    {
-        map_type m1, m2;
-        m1.insert(value_type(1, "abc"));
-        m2.insert(value_type(1, "abc"));
-        const map_type& cm1 = m1, cm2 = m2;
-        assert(testComparisons(cm1, cm2, true, false));
-    }
-    {
-        map_type m1, m2;
-        m1.insert(value_type(1, "ab"));
-        m2.insert(value_type(1, "abc"));
-        const map_type& cm1 = m1, cm2 = m2;
-        assert(testComparisons(cm1, cm2, false, true));
-    }
-    {
-        map_type m1, m2;
-        m1.insert(value_type(1, "abc"));
-        m2.insert(value_type(1, "bcd"));
-        const map_type& cm1 = m1, cm2 = m2;
-        assert(testComparisons(cm1, cm2, false, true));
-    }
-    {
-        map_type m1, m2;
-        m1.insert(value_type(1, "abc"));
-        m2.insert(value_type(1, "abc"));
-        m2.insert(value_type(2, "abc"));
-        const map_type& cm1 = m1, cm2 = m2;
-        assert(testComparisons(cm1, cm2, false, true));
-    }
-    return 0;
+  typedef std::map<int, std::string> map_type;
+  typedef map_type::value_type value_type;
+  {
+    map_type m1, m2;
+    m1.insert(value_type(1, "abc"));
+    m2.insert(value_type(2, "abc"));
+    const map_type &cm1 = m1, cm2 = m2;
+    assert(testComparisons(cm1, cm2, false, true));
+  }
+  {
+    map_type m1, m2;
+    m1.insert(value_type(1, "abc"));
+    m2.insert(value_type(1, "abc"));
+    const map_type &cm1 = m1, cm2 = m2;
+    assert(testComparisons(cm1, cm2, true, false));
+  }
+  {
+    map_type m1, m2;
+    m1.insert(value_type(1, "ab"));
+    m2.insert(value_type(1, "abc"));
+    const map_type &cm1 = m1, cm2 = m2;
+    assert(testComparisons(cm1, cm2, false, true));
+  }
+  {
+    map_type m1, m2;
+    m1.insert(value_type(1, "abc"));
+    m2.insert(value_type(1, "bcd"));
+    const map_type &cm1 = m1, cm2 = m2;
+    assert(testComparisons(cm1, cm2, false, true));
+  }
+  {
+    map_type m1, m2;
+    m1.insert(value_type(1, "abc"));
+    m2.insert(value_type(1, "abc"));
+    m2.insert(value_type(2, "abc"));
+    const map_type &cm1 = m1, cm2 = m2;
+    assert(testComparisons(cm1, cm2, false, true));
+  }
+  return 0;
 }
diff --git a/libcxx/test/std/containers/associative/map/map.observers/key_comp.pass.cpp b/libcxx/test/std/containers/associative/map/map.observers/key_comp.pass.cpp
index f44e78fd283ed..f0757ab47ee48 100644
--- a/libcxx/test/std/containers/associative/map/map.observers/key_comp.pass.cpp
+++ b/libcxx/test/std/containers/associative/map/map.observers/key_comp.pass.cpp
@@ -15,16 +15,16 @@
 #include <string>
 
 int main(int, char**) {
-    typedef std::map<int, std::string> map_type;
+  typedef std::map<int, std::string> map_type;
 
-    map_type m;
-    std::pair<map_type::iterator, bool> p1 = m.insert(map_type::value_type(1, "abc"));
-    std::pair<map_type::iterator, bool> p2 = m.insert(map_type::value_type(2, "abc"));
+  map_type m;
+  std::pair<map_type::iterator, bool> p1 = m.insert(map_type::value_type(1, "abc"));
+  std::pair<map_type::iterator, bool> p2 = m.insert(map_type::value_type(2, "abc"));
 
-    const map_type& cm = m;
+  const map_type& cm = m;
 
-    assert(cm.key_comp()(p1.first->first, p2.first->first));
-    assert(!cm.key_comp()(p2.first->first, p1.first->first));
+  assert(cm.key_comp()(p1.first->first, p2.first->first));
+  assert(!cm.key_comp()(p2.first->first, p1.first->first));
 
-    return 0;
+  return 0;
 }
diff --git a/libcxx/test/std/containers/associative/map/map.observers/value_comp.pass.cpp b/libcxx/test/std/containers/associative/map/map.observers/value_comp.pass.cpp
index 02b9659649833..edd1cd3c5c271 100644
--- a/libcxx/test/std/containers/associative/map/map.observers/value_comp.pass.cpp
+++ b/libcxx/test/std/containers/associative/map/map.observers/value_comp.pass.cpp
@@ -15,16 +15,16 @@
 #include <string>
 
 int main(int, char**) {
-    typedef std::map<int, std::string> map_type;
+  typedef std::map<int, std::string> map_type;
 
-    map_type m;
-    std::pair<map_type::iterator, bool> p1 = m.insert(map_type::value_type(1, "abc"));
-    std::pair<map_type::iterator, bool> p2 = m.insert(map_type::value_type(2, "abc"));
+  map_type m;
+  std::pair<map_type::iterator, bool> p1 = m.insert(map_type::value_type(1, "abc"));
+  std::pair<map_type::iterator, bool> p2 = m.insert(map_type::value_type(2, "abc"));
 
-    const map_type& cm = m;
+  const map_type& cm = m;
 
-    assert(cm.value_comp()(*p1.first, *p2.first));
-    assert(!cm.value_comp()(*p2.first, *p1.first));
+  assert(cm.value_comp()(*p1.first, *p2.first));
+  assert(!cm.value_comp()(*p2.first, *p1.first));
 
-    return 0;
+  return 0;
 }
diff --git a/libcxx/test/std/containers/associative/map/map.ops/contains.pass.cpp b/libcxx/test/std/containers/associative/map/map.ops/contains.pass.cpp
index 4221dc119f66e..7191d4c5e835d 100644
--- a/libcxx/test/std/containers/associative/map/map.ops/contains.pass.cpp
+++ b/libcxx/test/std/containers/associative/map/map.ops/contains.pass.cpp
@@ -17,45 +17,44 @@
 
 template <typename T, typename P, typename B, typename... Pairs>
 void test(B bad, Pairs... args) {
-    T map;
-    P pairs[] = {args...};
+  T map;
+  P pairs[] = {args...};
 
-    for (auto& p : pairs) map.insert(p);
-    for (auto& p : pairs) assert(map.contains(p.first));
+  for (auto& p : pairs)
+    map.insert(p);
+  for (auto& p : pairs)
+    assert(map.contains(p.first));
 
-    assert(!map.contains(bad));
+  assert(!map.contains(bad));
 }
 
-struct E { int a = 1; double b = 1; char c = 1; };
-
-int main(int, char**)
-{
-    {
-        test<std::map<char, int>, std::pair<char, int> >(
-            'e', std::make_pair('a', 10), std::make_pair('b', 11),
-            std::make_pair('c', 12), std::make_pair('d', 13));
-
-        test<std::map<char, char>, std::pair<char, char> >(
-            'e', std::make_pair('a', 'a'), std::make_pair('b', 'a'),
-            std::make_pair('c', 'a'), std::make_pair('d', 'b'));
-
-        test<std::map<int, E>, std::pair<int, E> >(
-            -1, std::make_pair(1, E{}), std::make_pair(2, E{}),
-            std::make_pair(3, E{}), std::make_pair(4, E{}));
-    }
-    {
-        test<std::multimap<char, int>, std::pair<char, int> >(
-            'e', std::make_pair('a', 10), std::make_pair('b', 11),
-            std::make_pair('c', 12), std::make_pair('d', 13));
-
-        test<std::multimap<char, char>, std::pair<char, char> >(
-            'e', std::make_pair('a', 'a'), std::make_pair('b', 'a'),
-            std::make_pair('c', 'a'), std::make_pair('d', 'b'));
-
-        test<std::multimap<int, E>, std::pair<int, E> >(
-            -1, std::make_pair(1, E{}), std::make_pair(2, E{}),
-            std::make_pair(3, E{}), std::make_pair(4, E{}));
-    }
-
-    return 0;
+struct E {
+  int a    = 1;
+  double b = 1;
+  char c   = 1;
+};
+
+int main(int, char**) {
+  {
+    test<std::map<char, int>, std::pair<char, int> >(
+        'e', std::make_pair('a', 10), std::make_pair('b', 11), std::make_pair('c', 12), std::make_pair('d', 13));
+
+    test<std::map<char, char>, std::pair<char, char> >(
+        'e', std::make_pair('a', 'a'), std::make_pair('b', 'a'), std::make_pair('c', 'a'), std::make_pair('d', 'b'));
+
+    test<std::map<int, E>, std::pair<int, E> >(
+        -1, std::make_pair(1, E{}), std::make_pair(2, E{}), std::make_pair(3, E{}), std::make_pair(4, E{}));
+  }
+  {
+    test<std::multimap<char, int>, std::pair<char, int> >(
+        'e', std::make_pair('a', 10), std::make_pair('b', 11), std::make_pair('c', 12), std::make_pair('d', 13));
+
+    test<std::multimap<char, char>, std::pair<char, char> >(
+        'e', std::make_pair('a', 'a'), std::make_pair('b', 'a'), std::make_pair('c', 'a'), std::make_pair('d', 'b'));
+
+    test<std::multimap<int, E>, std::pair<int, E> >(
+        -1, std::make_pair(1, E{}), std::make_pair(2, E{}), std::make_pair(3, E{}), std::make_pair(4, E{}));
+  }
+
+  return 0;
 }
diff --git a/libcxx/test/std/containers/associative/map/map.ops/contains_transparent.pass.cpp b/libcxx/test/std/containers/associative/map/map.ops/contains_transparent.pass.cpp
index c5543a6eee3d3..f80f1a2714bfd 100644
--- a/libcxx/test/std/containers/associative/map/map.ops/contains_transparent.pass.cpp
+++ b/libcxx/test/std/containers/associative/map/map.ops/contains_transparent.pass.cpp
@@ -18,18 +18,11 @@
 struct Comp {
   using is_transparent = void;
 
-  bool operator()(const std::pair<int, int>& lhs,
-                  const std::pair<int, int>& rhs) const {
-    return lhs < rhs;
-  }
-
-  bool operator()(const std::pair<int, int>& lhs, int rhs) const {
-    return lhs.first < rhs;
-  }
-
-  bool operator()(int lhs, const std::pair<int, int>& rhs) const {
-    return lhs < rhs.first;
-  }
+  bool operator()(const std::pair<int, int>& lhs, const std::pair<int, int>& rhs) const { return lhs < rhs; }
+
+  bool operator()(const std::pair<int, int>& lhs, int rhs) const { return lhs.first < rhs; }
+
+  bool operator()(int lhs, const std::pair<int, int>& rhs) const { return lhs < rhs.first; }
 };
 
 template <typename Container>
diff --git a/libcxx/test/std/containers/associative/map/map.ops/count0.pass.cpp b/libcxx/test/std/containers/associative/map/map.ops/count0.pass.cpp
index aa9952337bee5..c7ba765178969 100644
--- a/libcxx/test/std/containers/associative/map/map.ops/count0.pass.cpp
+++ b/libcxx/test/std/containers/associative/map/map.ops/count0.pass.cpp
@@ -18,23 +18,21 @@
 // equal_range shall not participate in overload resolution unless the
 // qualified-id Compare::is_transparent is valid and denotes a type
 
-
 #include <map>
 #include <cassert>
 
 #include "test_macros.h"
 #include "is_transparent.h"
 
-int main(int, char**)
-{
-    {
+int main(int, char**) {
+  {
     typedef std::map<int, double, transparent_less> M;
     assert(M().count(C2Int{5}) == 0);
-    }
-    {
+  }
+  {
     typedef std::map<int, double, transparent_less_not_referenceable> M;
     assert(M().count(C2Int{5}) == 0);
-    }
+  }
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/associative/map/map.ops/count1.compile.fail.cpp b/libcxx/test/std/containers/associative/map/map.ops/count1.compile.fail.cpp
index 41463d682926b..0c9d51edccd34 100644
--- a/libcxx/test/std/containers/associative/map/map.ops/count1.compile.fail.cpp
+++ b/libcxx/test/std/containers/associative/map/map.ops/count1.compile.fail.cpp
@@ -18,18 +18,16 @@
 // equal_range shall not participate in overload resolution unless the
 // qualified-id Compare::is_transparent is valid and denotes a type
 
-
 #include <map>
 #include <cassert>
 
 #include "test_macros.h"
 #include "is_transparent.h"
 
-int main(int, char**)
-{
-    {
+int main(int, char**) {
+  {
     typedef std::map<int, double, transparent_less_no_type> M;
 
     TEST_IGNORE_NODISCARD M().count(C2Int{5});
-    }
+  }
 }
diff --git a/libcxx/test/std/containers/associative/map/map.ops/count2.compile.fail.cpp b/libcxx/test/std/containers/associative/map/map.ops/count2.compile.fail.cpp
index 2e27429c33c8c..0cc81b3339930 100644
--- a/libcxx/test/std/containers/associative/map/map.ops/count2.compile.fail.cpp
+++ b/libcxx/test/std/containers/associative/map/map.ops/count2.compile.fail.cpp
@@ -18,18 +18,16 @@
 // equal_range shall not participate in overload resolution unless the
 // qualified-id Compare::is_transparent is valid and denotes a type
 
-
 #include <map>
 #include <cassert>
 
 #include "test_macros.h"
 #include "is_transparent.h"
 
-int main(int, char**)
-{
-    {
+int main(int, char**) {
+  {
     typedef std::map<int, double, transparent_less_private> M;
 
     TEST_IGNORE_NODISCARD M().count(C2Int{5});
-    }
+  }
 }
diff --git a/libcxx/test/std/containers/associative/map/map.ops/count3.compile.fail.cpp b/libcxx/test/std/containers/associative/map/map.ops/count3.compile.fail.cpp
index 9f60b2d6e0d08..1c6a4cd3db095 100644
--- a/libcxx/test/std/containers/associative/map/map.ops/count3.compile.fail.cpp
+++ b/libcxx/test/std/containers/associative/map/map.ops/count3.compile.fail.cpp
@@ -18,18 +18,16 @@
 // equal_range shall not participate in overload resolution unless the
 // qualified-id Compare::is_transparent is valid and denotes a type
 
-
 #include <map>
 #include <cassert>
 
 #include "test_macros.h"
 #include "is_transparent.h"
 
-int main(int, char**)
-{
-    {
+int main(int, char**) {
+  {
     typedef std::map<int, double, transparent_less_not_a_type> M;
 
     TEST_IGNORE_NODISCARD M().count(C2Int{5});
-    }
+  }
 }
diff --git a/libcxx/test/std/containers/associative/map/map.ops/count_transparent.pass.cpp b/libcxx/test/std/containers/associative/map/map.ops/count_transparent.pass.cpp
index 89686c807ed5d..fc097c1dc672c 100644
--- a/libcxx/test/std/containers/associative/map/map.ops/count_transparent.pass.cpp
+++ b/libcxx/test/std/containers/associative/map/map.ops/count_transparent.pass.cpp
@@ -22,23 +22,15 @@
 struct Comp {
   using is_transparent = void;
 
-  bool operator()(const std::pair<int, int> &lhs,
-                  const std::pair<int, int> &rhs) const {
-    return lhs < rhs;
-  }
-
-  bool operator()(const std::pair<int, int> &lhs, int rhs) const {
-    return lhs.first < rhs;
-  }
-
-  bool operator()(int lhs, const std::pair<int, int> &rhs) const {
-    return lhs < rhs.first;
-  }
+  bool operator()(const std::pair<int, int>& lhs, const std::pair<int, int>& rhs) const { return lhs < rhs; }
+
+  bool operator()(const std::pair<int, int>& lhs, int rhs) const { return lhs.first < rhs; }
+
+  bool operator()(int lhs, const std::pair<int, int>& rhs) const { return lhs < rhs.first; }
 };
 
 int main(int, char**) {
-  std::map<std::pair<int, int>, int, Comp> s{
-      {{2, 1}, 1}, {{1, 2}, 2}, {{1, 3}, 3}, {{1, 4}, 4}, {{2, 2}, 5}};
+  std::map<std::pair<int, int>, int, Comp> s{{{2, 1}, 1}, {{1, 2}, 2}, {{1, 3}, 3}, {{1, 4}, 4}, {{2, 2}, 5}};
 
   auto cnt = s.count(1);
   assert(cnt == 3);
diff --git a/libcxx/test/std/containers/associative/map/map.ops/equal_range.pass.cpp b/libcxx/test/std/containers/associative/map/map.ops/equal_range.pass.cpp
index 00d0866f0237e..883cd379802e6 100644
--- a/libcxx/test/std/containers/associative/map/map.ops/equal_range.pass.cpp
+++ b/libcxx/test/std/containers/associative/map/map.ops/equal_range.pass.cpp
@@ -21,300 +21,249 @@
 #include "private_constructor.h"
 #include "is_transparent.h"
 
-int main(int, char**)
-{
-    {
+int main(int, char**) {
+  {
     typedef std::pair<const int, double> V;
     typedef std::map<int, double> M;
     {
-        typedef std::pair<M::iterator, M::iterator> R;
-        V ar[] =
-        {
-            V(5, 5),
-            V(7, 6),
-            V(9, 7),
-            V(11, 8),
-            V(13, 9),
-            V(15, 10),
-            V(17, 11),
-            V(19, 12)
-        };
-        M m(ar, ar+sizeof(ar)/sizeof(ar[0]));
-        R r = m.equal_range(5);
-        assert(r.first == std::next(m.begin(), 0));
-        assert(r.second == std::next(m.begin(), 1));
-        r = m.equal_range(7);
-        assert(r.first == std::next(m.begin(), 1));
-        assert(r.second == std::next(m.begin(), 2));
-        r = m.equal_range(9);
-        assert(r.first == std::next(m.begin(), 2));
-        assert(r.second == std::next(m.begin(), 3));
-        r = m.equal_range(11);
-        assert(r.first == std::next(m.begin(), 3));
-        assert(r.second == std::next(m.begin(), 4));
-        r = m.equal_range(13);
-        assert(r.first == std::next(m.begin(), 4));
-        assert(r.second == std::next(m.begin(), 5));
-        r = m.equal_range(15);
-        assert(r.first == std::next(m.begin(), 5));
-        assert(r.second == std::next(m.begin(), 6));
-        r = m.equal_range(17);
-        assert(r.first == std::next(m.begin(), 6));
-        assert(r.second == std::next(m.begin(), 7));
-        r = m.equal_range(19);
-        assert(r.first == std::next(m.begin(), 7));
-        assert(r.second == std::next(m.begin(), 8));
-        r = m.equal_range(4);
-        assert(r.first == std::next(m.begin(), 0));
-        assert(r.second == std::next(m.begin(), 0));
-        r = m.equal_range(6);
-        assert(r.first == std::next(m.begin(), 1));
-        assert(r.second == std::next(m.begin(), 1));
-        r = m.equal_range(8);
-        assert(r.first == std::next(m.begin(), 2));
-        assert(r.second == std::next(m.begin(), 2));
-        r = m.equal_range(10);
-        assert(r.first == std::next(m.begin(), 3));
-        assert(r.second == std::next(m.begin(), 3));
-        r = m.equal_range(12);
-        assert(r.first == std::next(m.begin(), 4));
-        assert(r.second == std::next(m.begin(), 4));
-        r = m.equal_range(14);
-        assert(r.first == std::next(m.begin(), 5));
-        assert(r.second == std::next(m.begin(), 5));
-        r = m.equal_range(16);
-        assert(r.first == std::next(m.begin(), 6));
-        assert(r.second == std::next(m.begin(), 6));
-        r = m.equal_range(18);
-        assert(r.first == std::next(m.begin(), 7));
-        assert(r.second == std::next(m.begin(), 7));
-        r = m.equal_range(20);
-        assert(r.first == std::next(m.begin(), 8));
-        assert(r.second == std::next(m.begin(), 8));
+      typedef std::pair<M::iterator, M::iterator> R;
+      V ar[] = {V(5, 5), V(7, 6), V(9, 7), V(11, 8), V(13, 9), V(15, 10), V(17, 11), V(19, 12)};
+      M m(ar, ar + sizeof(ar) / sizeof(ar[0]));
+      R r = m.equal_range(5);
+      assert(r.first == std::next(m.begin(), 0));
+      assert(r.second == std::next(m.begin(), 1));
+      r = m.equal_range(7);
+      assert(r.first == std::next(m.begin(), 1));
+      assert(r.second == std::next(m.begin(), 2));
+      r = m.equal_range(9);
+      assert(r.first == std::next(m.begin(), 2));
+      assert(r.second == std::next(m.begin(), 3));
+      r = m.equal_range(11);
+      assert(r.first == std::next(m.begin(), 3));
+      assert(r.second == std::next(m.begin(), 4));
+      r = m.equal_range(13);
+      assert(r.first == std::next(m.begin(), 4));
+      assert(r.second == std::next(m.begin(), 5));
+      r = m.equal_range(15);
+      assert(r.first == std::next(m.begin(), 5));
+      assert(r.second == std::next(m.begin(), 6));
+      r = m.equal_range(17);
+      assert(r.first == std::next(m.begin(), 6));
+      assert(r.second == std::next(m.begin(), 7));
+      r = m.equal_range(19);
+      assert(r.first == std::next(m.begin(), 7));
+      assert(r.second == std::next(m.begin(), 8));
+      r = m.equal_range(4);
+      assert(r.first == std::next(m.begin(), 0));
+      assert(r.second == std::next(m.begin(), 0));
+      r = m.equal_range(6);
+      assert(r.first == std::next(m.begin(), 1));
+      assert(r.second == std::next(m.begin(), 1));
+      r = m.equal_range(8);
+      assert(r.first == std::next(m.begin(), 2));
+      assert(r.second == std::next(m.begin(), 2));
+      r = m.equal_range(10);
+      assert(r.first == std::next(m.begin(), 3));
+      assert(r.second == std::next(m.begin(), 3));
+      r = m.equal_range(12);
+      assert(r.first == std::next(m.begin(), 4));
+      assert(r.second == std::next(m.begin(), 4));
+      r = m.equal_range(14);
+      assert(r.first == std::next(m.begin(), 5));
+      assert(r.second == std::next(m.begin(), 5));
+      r = m.equal_range(16);
+      assert(r.first == std::next(m.begin(), 6));
+      assert(r.second == std::next(m.begin(), 6));
+      r = m.equal_range(18);
+      assert(r.first == std::next(m.begin(), 7));
+      assert(r.second == std::next(m.begin(), 7));
+      r = m.equal_range(20);
+      assert(r.first == std::next(m.begin(), 8));
+      assert(r.second == std::next(m.begin(), 8));
     }
     {
-        typedef std::pair<M::const_iterator, M::const_iterator> R;
-        V ar[] =
-        {
-            V(5, 5),
-            V(7, 6),
-            V(9, 7),
-            V(11, 8),
-            V(13, 9),
-            V(15, 10),
-            V(17, 11),
-            V(19, 12)
-        };
-        const M m(ar, ar+sizeof(ar)/sizeof(ar[0]));
-        R r = m.equal_range(5);
-        assert(r.first == std::next(m.begin(), 0));
-        assert(r.second == std::next(m.begin(), 1));
-        r = m.equal_range(7);
-        assert(r.first == std::next(m.begin(), 1));
-        assert(r.second == std::next(m.begin(), 2));
-        r = m.equal_range(9);
-        assert(r.first == std::next(m.begin(), 2));
-        assert(r.second == std::next(m.begin(), 3));
-        r = m.equal_range(11);
-        assert(r.first == std::next(m.begin(), 3));
-        assert(r.second == std::next(m.begin(), 4));
-        r = m.equal_range(13);
-        assert(r.first == std::next(m.begin(), 4));
-        assert(r.second == std::next(m.begin(), 5));
-        r = m.equal_range(15);
-        assert(r.first == std::next(m.begin(), 5));
-        assert(r.second == std::next(m.begin(), 6));
-        r = m.equal_range(17);
-        assert(r.first == std::next(m.begin(), 6));
-        assert(r.second == std::next(m.begin(), 7));
-        r = m.equal_range(19);
-        assert(r.first == std::next(m.begin(), 7));
-        assert(r.second == std::next(m.begin(), 8));
-        r = m.equal_range(4);
-        assert(r.first == std::next(m.begin(), 0));
-        assert(r.second == std::next(m.begin(), 0));
-        r = m.equal_range(6);
-        assert(r.first == std::next(m.begin(), 1));
-        assert(r.second == std::next(m.begin(), 1));
-        r = m.equal_range(8);
-        assert(r.first == std::next(m.begin(), 2));
-        assert(r.second == std::next(m.begin(), 2));
-        r = m.equal_range(10);
-        assert(r.first == std::next(m.begin(), 3));
-        assert(r.second == std::next(m.begin(), 3));
-        r = m.equal_range(12);
-        assert(r.first == std::next(m.begin(), 4));
-        assert(r.second == std::next(m.begin(), 4));
-        r = m.equal_range(14);
-        assert(r.first == std::next(m.begin(), 5));
-        assert(r.second == std::next(m.begin(), 5));
-        r = m.equal_range(16);
-        assert(r.first == std::next(m.begin(), 6));
-        assert(r.second == std::next(m.begin(), 6));
-        r = m.equal_range(18);
-        assert(r.first == std::next(m.begin(), 7));
-        assert(r.second == std::next(m.begin(), 7));
-        r = m.equal_range(20);
-        assert(r.first == std::next(m.begin(), 8));
-        assert(r.second == std::next(m.begin(), 8));
-    }
+      typedef std::pair<M::const_iterator, M::const_iterator> R;
+      V ar[] = {V(5, 5), V(7, 6), V(9, 7), V(11, 8), V(13, 9), V(15, 10), V(17, 11), V(19, 12)};
+      const M m(ar, ar + sizeof(ar) / sizeof(ar[0]));
+      R r = m.equal_range(5);
+      assert(r.first == std::next(m.begin(), 0));
+      assert(r.second == std::next(m.begin(), 1));
+      r = m.equal_range(7);
+      assert(r.first == std::next(m.begin(), 1));
+      assert(r.second == std::next(m.begin(), 2));
+      r = m.equal_range(9);
+      assert(r.first == std::next(m.begin(), 2));
+      assert(r.second == std::next(m.begin(), 3));
+      r = m.equal_range(11);
+      assert(r.first == std::next(m.begin(), 3));
+      assert(r.second == std::next(m.begin(), 4));
+      r = m.equal_range(13);
+      assert(r.first == std::next(m.begin(), 4));
+      assert(r.second == std::next(m.begin(), 5));
+      r = m.equal_range(15);
+      assert(r.first == std::next(m.begin(), 5));
+      assert(r.second == std::next(m.begin(), 6));
+      r = m.equal_range(17);
+      assert(r.first == std::next(m.begin(), 6));
+      assert(r.second == std::next(m.begin(), 7));
+      r = m.equal_range(19);
+      assert(r.first == std::next(m.begin(), 7));
+      assert(r.second == std::next(m.begin(), 8));
+      r = m.equal_range(4);
+      assert(r.first == std::next(m.begin(), 0));
+      assert(r.second == std::next(m.begin(), 0));
+      r = m.equal_range(6);
+      assert(r.first == std::next(m.begin(), 1));
+      assert(r.second == std::next(m.begin(), 1));
+      r = m.equal_range(8);
+      assert(r.first == std::next(m.begin(), 2));
+      assert(r.second == std::next(m.begin(), 2));
+      r = m.equal_range(10);
+      assert(r.first == std::next(m.begin(), 3));
+      assert(r.second == std::next(m.begin(), 3));
+      r = m.equal_range(12);
+      assert(r.first == std::next(m.begin(), 4));
+      assert(r.second == std::next(m.begin(), 4));
+      r = m.equal_range(14);
+      assert(r.first == std::next(m.begin(), 5));
+      assert(r.second == std::next(m.begin(), 5));
+      r = m.equal_range(16);
+      assert(r.first == std::next(m.begin(), 6));
+      assert(r.second == std::next(m.begin(), 6));
+      r = m.equal_range(18);
+      assert(r.first == std::next(m.begin(), 7));
+      assert(r.second == std::next(m.begin(), 7));
+      r = m.equal_range(20);
+      assert(r.first == std::next(m.begin(), 8));
+      assert(r.second == std::next(m.begin(), 8));
     }
+  }
 #if TEST_STD_VER >= 11
-    {
+  {
     typedef std::pair<const int, double> V;
     typedef std::map<int, double, std::less<int>, min_allocator<V>> M;
     {
-        typedef std::pair<M::iterator, M::iterator> R;
-        V ar[] =
-        {
-            V(5, 5),
-            V(7, 6),
-            V(9, 7),
-            V(11, 8),
-            V(13, 9),
-            V(15, 10),
-            V(17, 11),
-            V(19, 12)
-        };
-        M m(ar, ar+sizeof(ar)/sizeof(ar[0]));
-        R r = m.equal_range(5);
-        assert(r.first == std::next(m.begin(), 0));
-        assert(r.second == std::next(m.begin(), 1));
-        r = m.equal_range(7);
-        assert(r.first == std::next(m.begin(), 1));
-        assert(r.second == std::next(m.begin(), 2));
-        r = m.equal_range(9);
-        assert(r.first == std::next(m.begin(), 2));
-        assert(r.second == std::next(m.begin(), 3));
-        r = m.equal_range(11);
-        assert(r.first == std::next(m.begin(), 3));
-        assert(r.second == std::next(m.begin(), 4));
-        r = m.equal_range(13);
-        assert(r.first == std::next(m.begin(), 4));
-        assert(r.second == std::next(m.begin(), 5));
-        r = m.equal_range(15);
-        assert(r.first == std::next(m.begin(), 5));
-        assert(r.second == std::next(m.begin(), 6));
-        r = m.equal_range(17);
-        assert(r.first == std::next(m.begin(), 6));
-        assert(r.second == std::next(m.begin(), 7));
-        r = m.equal_range(19);
-        assert(r.first == std::next(m.begin(), 7));
-        assert(r.second == std::next(m.begin(), 8));
-        r = m.equal_range(4);
-        assert(r.first == std::next(m.begin(), 0));
-        assert(r.second == std::next(m.begin(), 0));
-        r = m.equal_range(6);
-        assert(r.first == std::next(m.begin(), 1));
-        assert(r.second == std::next(m.begin(), 1));
-        r = m.equal_range(8);
-        assert(r.first == std::next(m.begin(), 2));
-        assert(r.second == std::next(m.begin(), 2));
-        r = m.equal_range(10);
-        assert(r.first == std::next(m.begin(), 3));
-        assert(r.second == std::next(m.begin(), 3));
-        r = m.equal_range(12);
-        assert(r.first == std::next(m.begin(), 4));
-        assert(r.second == std::next(m.begin(), 4));
-        r = m.equal_range(14);
-        assert(r.first == std::next(m.begin(), 5));
-        assert(r.second == std::next(m.begin(), 5));
-        r = m.equal_range(16);
-        assert(r.first == std::next(m.begin(), 6));
-        assert(r.second == std::next(m.begin(), 6));
-        r = m.equal_range(18);
-        assert(r.first == std::next(m.begin(), 7));
-        assert(r.second == std::next(m.begin(), 7));
-        r = m.equal_range(20);
-        assert(r.first == std::next(m.begin(), 8));
-        assert(r.second == std::next(m.begin(), 8));
+      typedef std::pair<M::iterator, M::iterator> R;
+      V ar[] = {V(5, 5), V(7, 6), V(9, 7), V(11, 8), V(13, 9), V(15, 10), V(17, 11), V(19, 12)};
+      M m(ar, ar + sizeof(ar) / sizeof(ar[0]));
+      R r = m.equal_range(5);
+      assert(r.first == std::next(m.begin(), 0));
+      assert(r.second == std::next(m.begin(), 1));
+      r = m.equal_range(7);
+      assert(r.first == std::next(m.begin(), 1));
+      assert(r.second == std::next(m.begin(), 2));
+      r = m.equal_range(9);
+      assert(r.first == std::next(m.begin(), 2));
+      assert(r.second == std::next(m.begin(), 3));
+      r = m.equal_range(11);
+      assert(r.first == std::next(m.begin(), 3));
+      assert(r.second == std::next(m.begin(), 4));
+      r = m.equal_range(13);
+      assert(r.first == std::next(m.begin(), 4));
+      assert(r.second == std::next(m.begin(), 5));
+      r = m.equal_range(15);
+      assert(r.first == std::next(m.begin(), 5));
+      assert(r.second == std::next(m.begin(), 6));
+      r = m.equal_range(17);
+      assert(r.first == std::next(m.begin(), 6));
+      assert(r.second == std::next(m.begin(), 7));
+      r = m.equal_range(19);
+      assert(r.first == std::next(m.begin(), 7));
+      assert(r.second == std::next(m.begin(), 8));
+      r = m.equal_range(4);
+      assert(r.first == std::next(m.begin(), 0));
+      assert(r.second == std::next(m.begin(), 0));
+      r = m.equal_range(6);
+      assert(r.first == std::next(m.begin(), 1));
+      assert(r.second == std::next(m.begin(), 1));
+      r = m.equal_range(8);
+      assert(r.first == std::next(m.begin(), 2));
+      assert(r.second == std::next(m.begin(), 2));
+      r = m.equal_range(10);
+      assert(r.first == std::next(m.begin(), 3));
+      assert(r.second == std::next(m.begin(), 3));
+      r = m.equal_range(12);
+      assert(r.first == std::next(m.begin(), 4));
+      assert(r.second == std::next(m.begin(), 4));
+      r = m.equal_range(14);
+      assert(r.first == std::next(m.begin(), 5));
+      assert(r.second == std::next(m.begin(), 5));
+      r = m.equal_range(16);
+      assert(r.first == std::next(m.begin(), 6));
+      assert(r.second == std::next(m.begin(), 6));
+      r = m.equal_range(18);
+      assert(r.first == std::next(m.begin(), 7));
+      assert(r.second == std::next(m.begin(), 7));
+      r = m.equal_range(20);
+      assert(r.first == std::next(m.begin(), 8));
+      assert(r.second == std::next(m.begin(), 8));
     }
     {
-        typedef std::pair<M::const_iterator, M::const_iterator> R;
-        V ar[] =
-        {
-            V(5, 5),
-            V(7, 6),
-            V(9, 7),
-            V(11, 8),
-            V(13, 9),
-            V(15, 10),
-            V(17, 11),
-            V(19, 12)
-        };
-        const M m(ar, ar+sizeof(ar)/sizeof(ar[0]));
-        R r = m.equal_range(5);
-        assert(r.first == std::next(m.begin(), 0));
-        assert(r.second == std::next(m.begin(), 1));
-        r = m.equal_range(7);
-        assert(r.first == std::next(m.begin(), 1));
-        assert(r.second == std::next(m.begin(), 2));
-        r = m.equal_range(9);
-        assert(r.first == std::next(m.begin(), 2));
-        assert(r.second == std::next(m.begin(), 3));
-        r = m.equal_range(11);
-        assert(r.first == std::next(m.begin(), 3));
-        assert(r.second == std::next(m.begin(), 4));
-        r = m.equal_range(13);
-        assert(r.first == std::next(m.begin(), 4));
-        assert(r.second == std::next(m.begin(), 5));
-        r = m.equal_range(15);
-        assert(r.first == std::next(m.begin(), 5));
-        assert(r.second == std::next(m.begin(), 6));
-        r = m.equal_range(17);
-        assert(r.first == std::next(m.begin(), 6));
-        assert(r.second == std::next(m.begin(), 7));
-        r = m.equal_range(19);
-        assert(r.first == std::next(m.begin(), 7));
-        assert(r.second == std::next(m.begin(), 8));
-        r = m.equal_range(4);
-        assert(r.first == std::next(m.begin(), 0));
-        assert(r.second == std::next(m.begin(), 0));
-        r = m.equal_range(6);
-        assert(r.first == std::next(m.begin(), 1));
-        assert(r.second == std::next(m.begin(), 1));
-        r = m.equal_range(8);
-        assert(r.first == std::next(m.begin(), 2));
-        assert(r.second == std::next(m.begin(), 2));
-        r = m.equal_range(10);
-        assert(r.first == std::next(m.begin(), 3));
-        assert(r.second == std::next(m.begin(), 3));
-        r = m.equal_range(12);
-        assert(r.first == std::next(m.begin(), 4));
-        assert(r.second == std::next(m.begin(), 4));
-        r = m.equal_range(14);
-        assert(r.first == std::next(m.begin(), 5));
-        assert(r.second == std::next(m.begin(), 5));
-        r = m.equal_range(16);
-        assert(r.first == std::next(m.begin(), 6));
-        assert(r.second == std::next(m.begin(), 6));
-        r = m.equal_range(18);
-        assert(r.first == std::next(m.begin(), 7));
-        assert(r.second == std::next(m.begin(), 7));
-        r = m.equal_range(20);
-        assert(r.first == std::next(m.begin(), 8));
-        assert(r.second == std::next(m.begin(), 8));
-    }
+      typedef std::pair<M::const_iterator, M::const_iterator> R;
+      V ar[] = {V(5, 5), V(7, 6), V(9, 7), V(11, 8), V(13, 9), V(15, 10), V(17, 11), V(19, 12)};
+      const M m(ar, ar + sizeof(ar) / sizeof(ar[0]));
+      R r = m.equal_range(5);
+      assert(r.first == std::next(m.begin(), 0));
+      assert(r.second == std::next(m.begin(), 1));
+      r = m.equal_range(7);
+      assert(r.first == std::next(m.begin(), 1));
+      assert(r.second == std::next(m.begin(), 2));
+      r = m.equal_range(9);
+      assert(r.first == std::next(m.begin(), 2));
+      assert(r.second == std::next(m.begin(), 3));
+      r = m.equal_range(11);
+      assert(r.first == std::next(m.begin(), 3));
+      assert(r.second == std::next(m.begin(), 4));
+      r = m.equal_range(13);
+      assert(r.first == std::next(m.begin(), 4));
+      assert(r.second == std::next(m.begin(), 5));
+      r = m.equal_range(15);
+      assert(r.first == std::next(m.begin(), 5));
+      assert(r.second == std::next(m.begin(), 6));
+      r = m.equal_range(17);
+      assert(r.first == std::next(m.begin(), 6));
+      assert(r.second == std::next(m.begin(), 7));
+      r = m.equal_range(19);
+      assert(r.first == std::next(m.begin(), 7));
+      assert(r.second == std::next(m.begin(), 8));
+      r = m.equal_range(4);
+      assert(r.first == std::next(m.begin(), 0));
+      assert(r.second == std::next(m.begin(), 0));
+      r = m.equal_range(6);
+      assert(r.first == std::next(m.begin(), 1));
+      assert(r.second == std::next(m.begin(), 1));
+      r = m.equal_range(8);
+      assert(r.first == std::next(m.begin(), 2));
+      assert(r.second == std::next(m.begin(), 2));
+      r = m.equal_range(10);
+      assert(r.first == std::next(m.begin(), 3));
+      assert(r.second == std::next(m.begin(), 3));
+      r = m.equal_range(12);
+      assert(r.first == std::next(m.begin(), 4));
+      assert(r.second == std::next(m.begin(), 4));
+      r = m.equal_range(14);
+      assert(r.first == std::next(m.begin(), 5));
+      assert(r.second == std::next(m.begin(), 5));
+      r = m.equal_range(16);
+      assert(r.first == std::next(m.begin(), 6));
+      assert(r.second == std::next(m.begin(), 6));
+      r = m.equal_range(18);
+      assert(r.first == std::next(m.begin(), 7));
+      assert(r.second == std::next(m.begin(), 7));
+      r = m.equal_range(20);
+      assert(r.first == std::next(m.begin(), 8));
+      assert(r.second == std::next(m.begin(), 8));
     }
+  }
 #endif
 #if TEST_STD_VER > 11
-    {
+  {
     typedef std::pair<const int, double> V;
     typedef std::map<int, double, std::less<>> M;
     typedef std::pair<M::iterator, M::iterator> R;
 
-    V ar[] =
-    {
-        V(5, 5),
-        V(7, 6),
-        V(9, 7),
-        V(11, 8),
-        V(13, 9),
-        V(15, 10),
-        V(17, 11),
-        V(19, 12)
-    };
-    M m(ar, ar+sizeof(ar)/sizeof(ar[0]));
+    V ar[] = {V(5, 5), V(7, 6), V(9, 7), V(11, 8), V(13, 9), V(15, 10), V(17, 11), V(19, 12)};
+    M m(ar, ar + sizeof(ar) / sizeof(ar[0]));
     R r = m.equal_range(5);
     assert(r.first == std::next(m.begin(), 0));
     assert(r.second == std::next(m.begin(), 1));
@@ -418,21 +367,21 @@ int main(int, char**)
     r = m.equal_range(C2Int(20));
     assert(r.first == std::next(m.begin(), 8));
     assert(r.second == std::next(m.begin(), 8));
-    }
-    {
+  }
+  {
     typedef PrivateConstructor PC;
     typedef std::map<PC, double, std::less<>> M;
     typedef std::pair<M::iterator, M::iterator> R;
 
     M m;
-    m [ PC::make(5)  ] = 5;
-    m [ PC::make(7)  ] = 6;
-    m [ PC::make(9)  ] = 7;
-    m [ PC::make(11) ] = 8;
-    m [ PC::make(13) ] = 9;
-    m [ PC::make(15) ] = 10;
-    m [ PC::make(17) ] = 11;
-    m [ PC::make(19) ] = 12;
+    m[PC::make(5)]  = 5;
+    m[PC::make(7)]  = 6;
+    m[PC::make(9)]  = 7;
+    m[PC::make(11)] = 8;
+    m[PC::make(13)] = 9;
+    m[PC::make(15)] = 10;
+    m[PC::make(17)] = 11;
+    m[PC::make(19)] = 12;
 
     R r = m.equal_range(5);
     assert(r.first == std::next(m.begin(), 0));
@@ -485,7 +434,7 @@ int main(int, char**)
     r = m.equal_range(20);
     assert(r.first == std::next(m.begin(), 8));
     assert(r.second == std::next(m.begin(), 8));
-    }
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/associative/map/map.ops/equal_range0.pass.cpp b/libcxx/test/std/containers/associative/map/map.ops/equal_range0.pass.cpp
index 6beb9b2246e29..75724bdb387ce 100644
--- a/libcxx/test/std/containers/associative/map/map.ops/equal_range0.pass.cpp
+++ b/libcxx/test/std/containers/associative/map/map.ops/equal_range0.pass.cpp
@@ -19,29 +19,27 @@
 // equal_range shall not participate in overload resolution unless the
 // qualified-id Compare::is_transparent is valid and denotes a type
 
-
 #include <map>
 #include <cassert>
 
 #include "test_macros.h"
 #include "is_transparent.h"
 
-int main(int, char**)
-{
-    {
+int main(int, char**) {
+  {
     typedef std::map<int, double, transparent_less> M;
     typedef std::pair<typename M::iterator, typename M::iterator> P;
     M example;
     P result = example.equal_range(C2Int{5});
     assert(result.first == result.second);
-    }
-    {
+  }
+  {
     typedef std::map<int, double, transparent_less_not_referenceable> M;
     typedef std::pair<typename M::iterator, typename M::iterator> P;
     M example;
     P result = example.equal_range(C2Int{5});
     assert(result.first == result.second);
-    }
+  }
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/associative/map/map.ops/equal_range1.compile.fail.cpp b/libcxx/test/std/containers/associative/map/map.ops/equal_range1.compile.fail.cpp
index 826d510ad71ea..d846a58899174 100644
--- a/libcxx/test/std/containers/associative/map/map.ops/equal_range1.compile.fail.cpp
+++ b/libcxx/test/std/containers/associative/map/map.ops/equal_range1.compile.fail.cpp
@@ -19,18 +19,16 @@
 // equal_range shall not participate in overload resolution unless the
 // qualified-id Compare::is_transparent is valid and denotes a type
 
-
 #include <map>
 #include <cassert>
 
 #include "test_macros.h"
 #include "is_transparent.h"
 
-int main(int, char**)
-{
-    {
+int main(int, char**) {
+  {
     typedef std::map<int, double, transparent_less_no_type> M;
 
     TEST_IGNORE_NODISCARD M().equal_range(C2Int{5});
-    }
+  }
 }
diff --git a/libcxx/test/std/containers/associative/map/map.ops/equal_range2.compile.fail.cpp b/libcxx/test/std/containers/associative/map/map.ops/equal_range2.compile.fail.cpp
index 50616cfe8b1c6..e341b3cb0ee45 100644
--- a/libcxx/test/std/containers/associative/map/map.ops/equal_range2.compile.fail.cpp
+++ b/libcxx/test/std/containers/associative/map/map.ops/equal_range2.compile.fail.cpp
@@ -19,18 +19,16 @@
 // equal_range shall not participate in overload resolution unless the
 // qualified-id Compare::is_transparent is valid and denotes a type
 
-
 #include <map>
 #include <cassert>
 
 #include "test_macros.h"
 #include "is_transparent.h"
 
-int main(int, char**)
-{
-    {
+int main(int, char**) {
+  {
     typedef std::map<int, double, transparent_less_private> M;
 
     TEST_IGNORE_NODISCARD M().equal_range(C2Int{5});
-    }
+  }
 }
diff --git a/libcxx/test/std/containers/associative/map/map.ops/equal_range3.compile.fail.cpp b/libcxx/test/std/containers/associative/map/map.ops/equal_range3.compile.fail.cpp
index b4b58d33dedbb..f99fc888ff52a 100644
--- a/libcxx/test/std/containers/associative/map/map.ops/equal_range3.compile.fail.cpp
+++ b/libcxx/test/std/containers/associative/map/map.ops/equal_range3.compile.fail.cpp
@@ -19,18 +19,16 @@
 // equal_range shall not participate in overload resolution unless the
 // qualified-id Compare::is_transparent is valid and denotes a type
 
-
 #include <map>
 #include <cassert>
 
 #include "test_macros.h"
 #include "is_transparent.h"
 
-int main(int, char**)
-{
-    {
+int main(int, char**) {
+  {
     typedef std::map<int, double, transparent_less_not_a_type> M;
 
     TEST_IGNORE_NODISCARD M().equal_range(C2Int{5});
-    }
+  }
 }
diff --git a/libcxx/test/std/containers/associative/map/map.ops/equal_range_transparent.pass.cpp b/libcxx/test/std/containers/associative/map/map.ops/equal_range_transparent.pass.cpp
index 10330a134b054..2b651468887ea 100644
--- a/libcxx/test/std/containers/associative/map/map.ops/equal_range_transparent.pass.cpp
+++ b/libcxx/test/std/containers/associative/map/map.ops/equal_range_transparent.pass.cpp
@@ -25,25 +25,17 @@
 struct Comp {
   using is_transparent = void;
 
-  bool operator()(const std::pair<int, int> &lhs,
-                  const std::pair<int, int> &rhs) const {
-    return lhs < rhs;
-  }
+  bool operator()(const std::pair<int, int>& lhs, const std::pair<int, int>& rhs) const { return lhs < rhs; }
 
-  bool operator()(const std::pair<int, int> &lhs, int rhs) const {
-    return lhs.first < rhs;
-  }
+  bool operator()(const std::pair<int, int>& lhs, int rhs) const { return lhs.first < rhs; }
 
-  bool operator()(int lhs, const std::pair<int, int> &rhs) const {
-    return lhs < rhs.first;
-  }
+  bool operator()(int lhs, const std::pair<int, int>& rhs) const { return lhs < rhs.first; }
 };
 
 int main(int, char**) {
-  std::map<std::pair<int, int>, int, Comp> s{
-      {{2, 1}, 1}, {{1, 2}, 2}, {{1, 3}, 3}, {{1, 4}, 4}, {{2, 2}, 5}};
+  std::map<std::pair<int, int>, int, Comp> s{{{2, 1}, 1}, {{1, 2}, 2}, {{1, 3}, 3}, {{1, 4}, 4}, {{2, 2}, 5}};
 
-  auto er = s.equal_range(1);
+  auto er   = s.equal_range(1);
   long nels = 0;
 
   for (auto it = er.first; it != er.second; it++) {
diff --git a/libcxx/test/std/containers/associative/map/map.ops/find.pass.cpp b/libcxx/test/std/containers/associative/map/map.ops/find.pass.cpp
index 4724162c5e35a..534d78128407d 100644
--- a/libcxx/test/std/containers/associative/map/map.ops/find.pass.cpp
+++ b/libcxx/test/std/containers/associative/map/map.ops/find.pass.cpp
@@ -21,168 +21,117 @@
 #include "private_constructor.h"
 #include "is_transparent.h"
 
-int main(int, char**)
-{
-    {
+int main(int, char**) {
+  {
     typedef std::pair<const int, double> V;
     typedef std::map<int, double> M;
     {
-        typedef M::iterator R;
-        V ar[] =
-        {
-            V(5, 5),
-            V(6, 6),
-            V(7, 7),
-            V(8, 8),
-            V(9, 9),
-            V(10, 10),
-            V(11, 11),
-            V(12, 12)
-        };
-        M m(ar, ar+sizeof(ar)/sizeof(ar[0]));
-        R r = m.find(5);
-        assert(r == m.begin());
-        r = m.find(6);
-        assert(r == std::next(m.begin()));
-        r = m.find(7);
-        assert(r == std::next(m.begin(), 2));
-        r = m.find(8);
-        assert(r == std::next(m.begin(), 3));
-        r = m.find(9);
-        assert(r == std::next(m.begin(), 4));
-        r = m.find(10);
-        assert(r == std::next(m.begin(), 5));
-        r = m.find(11);
-        assert(r == std::next(m.begin(), 6));
-        r = m.find(12);
-        assert(r == std::next(m.begin(), 7));
-        r = m.find(4);
-        assert(r == std::next(m.begin(), 8));
+      typedef M::iterator R;
+      V ar[] = {V(5, 5), V(6, 6), V(7, 7), V(8, 8), V(9, 9), V(10, 10), V(11, 11), V(12, 12)};
+      M m(ar, ar + sizeof(ar) / sizeof(ar[0]));
+      R r = m.find(5);
+      assert(r == m.begin());
+      r = m.find(6);
+      assert(r == std::next(m.begin()));
+      r = m.find(7);
+      assert(r == std::next(m.begin(), 2));
+      r = m.find(8);
+      assert(r == std::next(m.begin(), 3));
+      r = m.find(9);
+      assert(r == std::next(m.begin(), 4));
+      r = m.find(10);
+      assert(r == std::next(m.begin(), 5));
+      r = m.find(11);
+      assert(r == std::next(m.begin(), 6));
+      r = m.find(12);
+      assert(r == std::next(m.begin(), 7));
+      r = m.find(4);
+      assert(r == std::next(m.begin(), 8));
     }
     {
-        typedef M::const_iterator R;
-        V ar[] =
-        {
-            V(5, 5),
-            V(6, 6),
-            V(7, 7),
-            V(8, 8),
-            V(9, 9),
-            V(10, 10),
-            V(11, 11),
-            V(12, 12)
-        };
-        const M m(ar, ar+sizeof(ar)/sizeof(ar[0]));
-        R r = m.find(5);
-        assert(r == m.begin());
-        r = m.find(6);
-        assert(r == std::next(m.begin()));
-        r = m.find(7);
-        assert(r == std::next(m.begin(), 2));
-        r = m.find(8);
-        assert(r == std::next(m.begin(), 3));
-        r = m.find(9);
-        assert(r == std::next(m.begin(), 4));
-        r = m.find(10);
-        assert(r == std::next(m.begin(), 5));
-        r = m.find(11);
-        assert(r == std::next(m.begin(), 6));
-        r = m.find(12);
-        assert(r == std::next(m.begin(), 7));
-        r = m.find(4);
-        assert(r == std::next(m.begin(), 8));
-    }
+      typedef M::const_iterator R;
+      V ar[] = {V(5, 5), V(6, 6), V(7, 7), V(8, 8), V(9, 9), V(10, 10), V(11, 11), V(12, 12)};
+      const M m(ar, ar + sizeof(ar) / sizeof(ar[0]));
+      R r = m.find(5);
+      assert(r == m.begin());
+      r = m.find(6);
+      assert(r == std::next(m.begin()));
+      r = m.find(7);
+      assert(r == std::next(m.begin(), 2));
+      r = m.find(8);
+      assert(r == std::next(m.begin(), 3));
+      r = m.find(9);
+      assert(r == std::next(m.begin(), 4));
+      r = m.find(10);
+      assert(r == std::next(m.begin(), 5));
+      r = m.find(11);
+      assert(r == std::next(m.begin(), 6));
+      r = m.find(12);
+      assert(r == std::next(m.begin(), 7));
+      r = m.find(4);
+      assert(r == std::next(m.begin(), 8));
     }
+  }
 #if TEST_STD_VER >= 11
-    {
+  {
     typedef std::pair<const int, double> V;
     typedef std::map<int, double, std::less<int>, min_allocator<V>> M;
     {
-        typedef M::iterator R;
-        V ar[] =
-        {
-            V(5, 5),
-            V(6, 6),
-            V(7, 7),
-            V(8, 8),
-            V(9, 9),
-            V(10, 10),
-            V(11, 11),
-            V(12, 12)
-        };
-        M m(ar, ar+sizeof(ar)/sizeof(ar[0]));
-        R r = m.find(5);
-        assert(r == m.begin());
-        r = m.find(6);
-        assert(r == std::next(m.begin()));
-        r = m.find(7);
-        assert(r == std::next(m.begin(), 2));
-        r = m.find(8);
-        assert(r == std::next(m.begin(), 3));
-        r = m.find(9);
-        assert(r == std::next(m.begin(), 4));
-        r = m.find(10);
-        assert(r == std::next(m.begin(), 5));
-        r = m.find(11);
-        assert(r == std::next(m.begin(), 6));
-        r = m.find(12);
-        assert(r == std::next(m.begin(), 7));
-        r = m.find(4);
-        assert(r == std::next(m.begin(), 8));
+      typedef M::iterator R;
+      V ar[] = {V(5, 5), V(6, 6), V(7, 7), V(8, 8), V(9, 9), V(10, 10), V(11, 11), V(12, 12)};
+      M m(ar, ar + sizeof(ar) / sizeof(ar[0]));
+      R r = m.find(5);
+      assert(r == m.begin());
+      r = m.find(6);
+      assert(r == std::next(m.begin()));
+      r = m.find(7);
+      assert(r == std::next(m.begin(), 2));
+      r = m.find(8);
+      assert(r == std::next(m.begin(), 3));
+      r = m.find(9);
+      assert(r == std::next(m.begin(), 4));
+      r = m.find(10);
+      assert(r == std::next(m.begin(), 5));
+      r = m.find(11);
+      assert(r == std::next(m.begin(), 6));
+      r = m.find(12);
+      assert(r == std::next(m.begin(), 7));
+      r = m.find(4);
+      assert(r == std::next(m.begin(), 8));
     }
     {
-        typedef M::const_iterator R;
-        V ar[] =
-        {
-            V(5, 5),
-            V(6, 6),
-            V(7, 7),
-            V(8, 8),
-            V(9, 9),
-            V(10, 10),
-            V(11, 11),
-            V(12, 12)
-        };
-        const M m(ar, ar+sizeof(ar)/sizeof(ar[0]));
-        R r = m.find(5);
-        assert(r == m.begin());
-        r = m.find(6);
-        assert(r == std::next(m.begin()));
-        r = m.find(7);
-        assert(r == std::next(m.begin(), 2));
-        r = m.find(8);
-        assert(r == std::next(m.begin(), 3));
-        r = m.find(9);
-        assert(r == std::next(m.begin(), 4));
-        r = m.find(10);
-        assert(r == std::next(m.begin(), 5));
-        r = m.find(11);
-        assert(r == std::next(m.begin(), 6));
-        r = m.find(12);
-        assert(r == std::next(m.begin(), 7));
-        r = m.find(4);
-        assert(r == std::next(m.begin(), 8));
-    }
+      typedef M::const_iterator R;
+      V ar[] = {V(5, 5), V(6, 6), V(7, 7), V(8, 8), V(9, 9), V(10, 10), V(11, 11), V(12, 12)};
+      const M m(ar, ar + sizeof(ar) / sizeof(ar[0]));
+      R r = m.find(5);
+      assert(r == m.begin());
+      r = m.find(6);
+      assert(r == std::next(m.begin()));
+      r = m.find(7);
+      assert(r == std::next(m.begin(), 2));
+      r = m.find(8);
+      assert(r == std::next(m.begin(), 3));
+      r = m.find(9);
+      assert(r == std::next(m.begin(), 4));
+      r = m.find(10);
+      assert(r == std::next(m.begin(), 5));
+      r = m.find(11);
+      assert(r == std::next(m.begin(), 6));
+      r = m.find(12);
+      assert(r == std::next(m.begin(), 7));
+      r = m.find(4);
+      assert(r == std::next(m.begin(), 8));
     }
+  }
 #endif
 #if TEST_STD_VER > 11
-    {
+  {
     typedef std::pair<const int, double> V;
     typedef std::map<int, double, std::less<>> M;
     typedef M::iterator R;
 
-    V ar[] =
-    {
-        V(5, 5),
-        V(6, 6),
-        V(7, 7),
-        V(8, 8),
-        V(9, 9),
-        V(10, 10),
-        V(11, 11),
-        V(12, 12)
-    };
-    M m(ar, ar+sizeof(ar)/sizeof(ar[0]));
+    V ar[] = {V(5, 5), V(6, 6), V(7, 7), V(8, 8), V(9, 9), V(10, 10), V(11, 11), V(12, 12)};
+    M m(ar, ar + sizeof(ar) / sizeof(ar[0]));
     R r = m.find(5);
     assert(r == m.begin());
     r = m.find(6);
@@ -220,22 +169,22 @@ int main(int, char**)
     assert(r == std::next(m.begin(), 7));
     r = m.find(C2Int(4));
     assert(r == std::next(m.begin(), 8));
-    }
+  }
 
-    {
+  {
     typedef PrivateConstructor PC;
     typedef std::map<PC, double, std::less<>> M;
     typedef M::iterator R;
 
     M m;
-    m [ PC::make(5)  ] = 5;
-    m [ PC::make(6)  ] = 6;
-    m [ PC::make(7)  ] = 7;
-    m [ PC::make(8)  ] = 8;
-    m [ PC::make(9)  ] = 9;
-    m [ PC::make(10) ] = 10;
-    m [ PC::make(11) ] = 11;
-    m [ PC::make(12) ] = 12;
+    m[PC::make(5)]  = 5;
+    m[PC::make(6)]  = 6;
+    m[PC::make(7)]  = 7;
+    m[PC::make(8)]  = 8;
+    m[PC::make(9)]  = 9;
+    m[PC::make(10)] = 10;
+    m[PC::make(11)] = 11;
+    m[PC::make(12)] = 12;
 
     R r = m.find(5);
     assert(r == m.begin());
@@ -255,7 +204,7 @@ int main(int, char**)
     assert(r == std::next(m.begin(), 7));
     r = m.find(4);
     assert(r == std::next(m.begin(), 8));
-    }
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/associative/map/map.ops/find0.pass.cpp b/libcxx/test/std/containers/associative/map/map.ops/find0.pass.cpp
index 6017e9b7e8491..9825d6c5879b1 100644
--- a/libcxx/test/std/containers/associative/map/map.ops/find0.pass.cpp
+++ b/libcxx/test/std/containers/associative/map/map.ops/find0.pass.cpp
@@ -19,25 +19,23 @@
 // equal_range shall not participate in overload resolution unless the
 // qualified-id Compare::is_transparent is valid and denotes a type
 
-
 #include <map>
 #include <cassert>
 
 #include "test_macros.h"
 #include "is_transparent.h"
 
-int main(int, char**)
-{
-    {
+int main(int, char**) {
+  {
     typedef std::map<int, double, transparent_less> M;
     M example;
     assert(example.find(C2Int{5}) == example.end());
-    }
-    {
+  }
+  {
     typedef std::map<int, double, transparent_less_not_referenceable> M;
     M example;
     assert(example.find(C2Int{5}) == example.end());
-    }
+  }
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/associative/map/map.ops/find1.compile.fail.cpp b/libcxx/test/std/containers/associative/map/map.ops/find1.compile.fail.cpp
index 358983d846eb6..b67b3b33b38cc 100644
--- a/libcxx/test/std/containers/associative/map/map.ops/find1.compile.fail.cpp
+++ b/libcxx/test/std/containers/associative/map/map.ops/find1.compile.fail.cpp
@@ -19,18 +19,16 @@
 // equal_range shall not participate in overload resolution unless the
 // qualified-id Compare::is_transparent is valid and denotes a type
 
-
 #include <map>
 #include <cassert>
 
 #include "test_macros.h"
 #include "is_transparent.h"
 
-int main(int, char**)
-{
-    {
+int main(int, char**) {
+  {
     typedef std::map<int, double, transparent_less_no_type> M;
 
     TEST_IGNORE_NODISCARD M().find(C2Int{5});
-    }
+  }
 }
diff --git a/libcxx/test/std/containers/associative/map/map.ops/find2.compile.fail.cpp b/libcxx/test/std/containers/associative/map/map.ops/find2.compile.fail.cpp
index 130f2bcd5c696..49ded6659a501 100644
--- a/libcxx/test/std/containers/associative/map/map.ops/find2.compile.fail.cpp
+++ b/libcxx/test/std/containers/associative/map/map.ops/find2.compile.fail.cpp
@@ -19,18 +19,16 @@
 // equal_range shall not participate in overload resolution unless the
 // qualified-id Compare::is_transparent is valid and denotes a type
 
-
 #include <map>
 #include <cassert>
 
 #include "test_macros.h"
 #include "is_transparent.h"
 
-int main(int, char**)
-{
-    {
+int main(int, char**) {
+  {
     typedef std::map<int, double, transparent_less_private> M;
 
     TEST_IGNORE_NODISCARD M().find(C2Int{5});
-    }
+  }
 }
diff --git a/libcxx/test/std/containers/associative/map/map.ops/find3.compile.fail.cpp b/libcxx/test/std/containers/associative/map/map.ops/find3.compile.fail.cpp
index 6340de32cd8e9..ba978dde21a76 100644
--- a/libcxx/test/std/containers/associative/map/map.ops/find3.compile.fail.cpp
+++ b/libcxx/test/std/containers/associative/map/map.ops/find3.compile.fail.cpp
@@ -19,18 +19,16 @@
 // equal_range shall not participate in overload resolution unless the
 // qualified-id Compare::is_transparent is valid and denotes a type
 
-
 #include <map>
 #include <cassert>
 
 #include "test_macros.h"
 #include "is_transparent.h"
 
-int main(int, char**)
-{
-    {
+int main(int, char**) {
+  {
     typedef std::map<int, double, transparent_less_not_a_type> M;
 
     TEST_IGNORE_NODISCARD M().find(C2Int{5});
-    }
+  }
 }
diff --git a/libcxx/test/std/containers/associative/map/map.ops/lower_bound.pass.cpp b/libcxx/test/std/containers/associative/map/map.ops/lower_bound.pass.cpp
index 777b98215e59f..755ad36a3ba4d 100644
--- a/libcxx/test/std/containers/associative/map/map.ops/lower_bound.pass.cpp
+++ b/libcxx/test/std/containers/associative/map/map.ops/lower_bound.pass.cpp
@@ -21,232 +21,181 @@
 #include "private_constructor.h"
 #include "is_transparent.h"
 
-int main(int, char**)
-{
-    {
+int main(int, char**) {
+  {
     typedef std::pair<const int, double> V;
     typedef std::map<int, double> M;
     {
-        typedef M::iterator R;
-        V ar[] =
-        {
-            V(5, 5),
-            V(7, 6),
-            V(9, 7),
-            V(11, 8),
-            V(13, 9),
-            V(15, 10),
-            V(17, 11),
-            V(19, 12)
-        };
-        M m(ar, ar+sizeof(ar)/sizeof(ar[0]));
-        R r = m.lower_bound(5);
-        assert(r == m.begin());
-        r = m.lower_bound(7);
-        assert(r == std::next(m.begin()));
-        r = m.lower_bound(9);
-        assert(r == std::next(m.begin(), 2));
-        r = m.lower_bound(11);
-        assert(r == std::next(m.begin(), 3));
-        r = m.lower_bound(13);
-        assert(r == std::next(m.begin(), 4));
-        r = m.lower_bound(15);
-        assert(r == std::next(m.begin(), 5));
-        r = m.lower_bound(17);
-        assert(r == std::next(m.begin(), 6));
-        r = m.lower_bound(19);
-        assert(r == std::next(m.begin(), 7));
-        r = m.lower_bound(4);
-        assert(r == std::next(m.begin(), 0));
-        r = m.lower_bound(6);
-        assert(r == std::next(m.begin(), 1));
-        r = m.lower_bound(8);
-        assert(r == std::next(m.begin(), 2));
-        r = m.lower_bound(10);
-        assert(r == std::next(m.begin(), 3));
-        r = m.lower_bound(12);
-        assert(r == std::next(m.begin(), 4));
-        r = m.lower_bound(14);
-        assert(r == std::next(m.begin(), 5));
-        r = m.lower_bound(16);
-        assert(r == std::next(m.begin(), 6));
-        r = m.lower_bound(18);
-        assert(r == std::next(m.begin(), 7));
-        r = m.lower_bound(20);
-        assert(r == std::next(m.begin(), 8));
+      typedef M::iterator R;
+      V ar[] = {V(5, 5), V(7, 6), V(9, 7), V(11, 8), V(13, 9), V(15, 10), V(17, 11), V(19, 12)};
+      M m(ar, ar + sizeof(ar) / sizeof(ar[0]));
+      R r = m.lower_bound(5);
+      assert(r == m.begin());
+      r = m.lower_bound(7);
+      assert(r == std::next(m.begin()));
+      r = m.lower_bound(9);
+      assert(r == std::next(m.begin(), 2));
+      r = m.lower_bound(11);
+      assert(r == std::next(m.begin(), 3));
+      r = m.lower_bound(13);
+      assert(r == std::next(m.begin(), 4));
+      r = m.lower_bound(15);
+      assert(r == std::next(m.begin(), 5));
+      r = m.lower_bound(17);
+      assert(r == std::next(m.begin(), 6));
+      r = m.lower_bound(19);
+      assert(r == std::next(m.begin(), 7));
+      r = m.lower_bound(4);
+      assert(r == std::next(m.begin(), 0));
+      r = m.lower_bound(6);
+      assert(r == std::next(m.begin(), 1));
+      r = m.lower_bound(8);
+      assert(r == std::next(m.begin(), 2));
+      r = m.lower_bound(10);
+      assert(r == std::next(m.begin(), 3));
+      r = m.lower_bound(12);
+      assert(r == std::next(m.begin(), 4));
+      r = m.lower_bound(14);
+      assert(r == std::next(m.begin(), 5));
+      r = m.lower_bound(16);
+      assert(r == std::next(m.begin(), 6));
+      r = m.lower_bound(18);
+      assert(r == std::next(m.begin(), 7));
+      r = m.lower_bound(20);
+      assert(r == std::next(m.begin(), 8));
     }
     {
-        typedef M::const_iterator R;
-        V ar[] =
-        {
-            V(5, 5),
-            V(7, 6),
-            V(9, 7),
-            V(11, 8),
-            V(13, 9),
-            V(15, 10),
-            V(17, 11),
-            V(19, 12)
-        };
-        const M m(ar, ar+sizeof(ar)/sizeof(ar[0]));
-        R r = m.lower_bound(5);
-        assert(r == m.begin());
-        r = m.lower_bound(7);
-        assert(r == std::next(m.begin()));
-        r = m.lower_bound(9);
-        assert(r == std::next(m.begin(), 2));
-        r = m.lower_bound(11);
-        assert(r == std::next(m.begin(), 3));
-        r = m.lower_bound(13);
-        assert(r == std::next(m.begin(), 4));
-        r = m.lower_bound(15);
-        assert(r == std::next(m.begin(), 5));
-        r = m.lower_bound(17);
-        assert(r == std::next(m.begin(), 6));
-        r = m.lower_bound(19);
-        assert(r == std::next(m.begin(), 7));
-        r = m.lower_bound(4);
-        assert(r == std::next(m.begin(), 0));
-        r = m.lower_bound(6);
-        assert(r == std::next(m.begin(), 1));
-        r = m.lower_bound(8);
-        assert(r == std::next(m.begin(), 2));
-        r = m.lower_bound(10);
-        assert(r == std::next(m.begin(), 3));
-        r = m.lower_bound(12);
-        assert(r == std::next(m.begin(), 4));
-        r = m.lower_bound(14);
-        assert(r == std::next(m.begin(), 5));
-        r = m.lower_bound(16);
-        assert(r == std::next(m.begin(), 6));
-        r = m.lower_bound(18);
-        assert(r == std::next(m.begin(), 7));
-        r = m.lower_bound(20);
-        assert(r == std::next(m.begin(), 8));
-    }
+      typedef M::const_iterator R;
+      V ar[] = {V(5, 5), V(7, 6), V(9, 7), V(11, 8), V(13, 9), V(15, 10), V(17, 11), V(19, 12)};
+      const M m(ar, ar + sizeof(ar) / sizeof(ar[0]));
+      R r = m.lower_bound(5);
+      assert(r == m.begin());
+      r = m.lower_bound(7);
+      assert(r == std::next(m.begin()));
+      r = m.lower_bound(9);
+      assert(r == std::next(m.begin(), 2));
+      r = m.lower_bound(11);
+      assert(r == std::next(m.begin(), 3));
+      r = m.lower_bound(13);
+      assert(r == std::next(m.begin(), 4));
+      r = m.lower_bound(15);
+      assert(r == std::next(m.begin(), 5));
+      r = m.lower_bound(17);
+      assert(r == std::next(m.begin(), 6));
+      r = m.lower_bound(19);
+      assert(r == std::next(m.begin(), 7));
+      r = m.lower_bound(4);
+      assert(r == std::next(m.begin(), 0));
+      r = m.lower_bound(6);
+      assert(r == std::next(m.begin(), 1));
+      r = m.lower_bound(8);
+      assert(r == std::next(m.begin(), 2));
+      r = m.lower_bound(10);
+      assert(r == std::next(m.begin(), 3));
+      r = m.lower_bound(12);
+      assert(r == std::next(m.begin(), 4));
+      r = m.lower_bound(14);
+      assert(r == std::next(m.begin(), 5));
+      r = m.lower_bound(16);
+      assert(r == std::next(m.begin(), 6));
+      r = m.lower_bound(18);
+      assert(r == std::next(m.begin(), 7));
+      r = m.lower_bound(20);
+      assert(r == std::next(m.begin(), 8));
     }
+  }
 #if TEST_STD_VER >= 11
-    {
+  {
     typedef std::pair<const int, double> V;
     typedef std::map<int, double, std::less<int>, min_allocator<V>> M;
     {
-        typedef M::iterator R;
-        V ar[] =
-        {
-            V(5, 5),
-            V(7, 6),
-            V(9, 7),
-            V(11, 8),
-            V(13, 9),
-            V(15, 10),
-            V(17, 11),
-            V(19, 12)
-        };
-        M m(ar, ar+sizeof(ar)/sizeof(ar[0]));
-        R r = m.lower_bound(5);
-        assert(r == m.begin());
-        r = m.lower_bound(7);
-        assert(r == std::next(m.begin()));
-        r = m.lower_bound(9);
-        assert(r == std::next(m.begin(), 2));
-        r = m.lower_bound(11);
-        assert(r == std::next(m.begin(), 3));
-        r = m.lower_bound(13);
-        assert(r == std::next(m.begin(), 4));
-        r = m.lower_bound(15);
-        assert(r == std::next(m.begin(), 5));
-        r = m.lower_bound(17);
-        assert(r == std::next(m.begin(), 6));
-        r = m.lower_bound(19);
-        assert(r == std::next(m.begin(), 7));
-        r = m.lower_bound(4);
-        assert(r == std::next(m.begin(), 0));
-        r = m.lower_bound(6);
-        assert(r == std::next(m.begin(), 1));
-        r = m.lower_bound(8);
-        assert(r == std::next(m.begin(), 2));
-        r = m.lower_bound(10);
-        assert(r == std::next(m.begin(), 3));
-        r = m.lower_bound(12);
-        assert(r == std::next(m.begin(), 4));
-        r = m.lower_bound(14);
-        assert(r == std::next(m.begin(), 5));
-        r = m.lower_bound(16);
-        assert(r == std::next(m.begin(), 6));
-        r = m.lower_bound(18);
-        assert(r == std::next(m.begin(), 7));
-        r = m.lower_bound(20);
-        assert(r == std::next(m.begin(), 8));
+      typedef M::iterator R;
+      V ar[] = {V(5, 5), V(7, 6), V(9, 7), V(11, 8), V(13, 9), V(15, 10), V(17, 11), V(19, 12)};
+      M m(ar, ar + sizeof(ar) / sizeof(ar[0]));
+      R r = m.lower_bound(5);
+      assert(r == m.begin());
+      r = m.lower_bound(7);
+      assert(r == std::next(m.begin()));
+      r = m.lower_bound(9);
+      assert(r == std::next(m.begin(), 2));
+      r = m.lower_bound(11);
+      assert(r == std::next(m.begin(), 3));
+      r = m.lower_bound(13);
+      assert(r == std::next(m.begin(), 4));
+      r = m.lower_bound(15);
+      assert(r == std::next(m.begin(), 5));
+      r = m.lower_bound(17);
+      assert(r == std::next(m.begin(), 6));
+      r = m.lower_bound(19);
+      assert(r == std::next(m.begin(), 7));
+      r = m.lower_bound(4);
+      assert(r == std::next(m.begin(), 0));
+      r = m.lower_bound(6);
+      assert(r == std::next(m.begin(), 1));
+      r = m.lower_bound(8);
+      assert(r == std::next(m.begin(), 2));
+      r = m.lower_bound(10);
+      assert(r == std::next(m.begin(), 3));
+      r = m.lower_bound(12);
+      assert(r == std::next(m.begin(), 4));
+      r = m.lower_bound(14);
+      assert(r == std::next(m.begin(), 5));
+      r = m.lower_bound(16);
+      assert(r == std::next(m.begin(), 6));
+      r = m.lower_bound(18);
+      assert(r == std::next(m.begin(), 7));
+      r = m.lower_bound(20);
+      assert(r == std::next(m.begin(), 8));
     }
     {
-        typedef M::const_iterator R;
-        V ar[] =
-        {
-            V(5, 5),
-            V(7, 6),
-            V(9, 7),
-            V(11, 8),
-            V(13, 9),
-            V(15, 10),
-            V(17, 11),
-            V(19, 12)
-        };
-        const M m(ar, ar+sizeof(ar)/sizeof(ar[0]));
-        R r = m.lower_bound(5);
-        assert(r == m.begin());
-        r = m.lower_bound(7);
-        assert(r == std::next(m.begin()));
-        r = m.lower_bound(9);
-        assert(r == std::next(m.begin(), 2));
-        r = m.lower_bound(11);
-        assert(r == std::next(m.begin(), 3));
-        r = m.lower_bound(13);
-        assert(r == std::next(m.begin(), 4));
-        r = m.lower_bound(15);
-        assert(r == std::next(m.begin(), 5));
-        r = m.lower_bound(17);
-        assert(r == std::next(m.begin(), 6));
-        r = m.lower_bound(19);
-        assert(r == std::next(m.begin(), 7));
-        r = m.lower_bound(4);
-        assert(r == std::next(m.begin(), 0));
-        r = m.lower_bound(6);
-        assert(r == std::next(m.begin(), 1));
-        r = m.lower_bound(8);
-        assert(r == std::next(m.begin(), 2));
-        r = m.lower_bound(10);
-        assert(r == std::next(m.begin(), 3));
-        r = m.lower_bound(12);
-        assert(r == std::next(m.begin(), 4));
-        r = m.lower_bound(14);
-        assert(r == std::next(m.begin(), 5));
-        r = m.lower_bound(16);
-        assert(r == std::next(m.begin(), 6));
-        r = m.lower_bound(18);
-        assert(r == std::next(m.begin(), 7));
-        r = m.lower_bound(20);
-        assert(r == std::next(m.begin(), 8));
-    }
+      typedef M::const_iterator R;
+      V ar[] = {V(5, 5), V(7, 6), V(9, 7), V(11, 8), V(13, 9), V(15, 10), V(17, 11), V(19, 12)};
+      const M m(ar, ar + sizeof(ar) / sizeof(ar[0]));
+      R r = m.lower_bound(5);
+      assert(r == m.begin());
+      r = m.lower_bound(7);
+      assert(r == std::next(m.begin()));
+      r = m.lower_bound(9);
+      assert(r == std::next(m.begin(), 2));
+      r = m.lower_bound(11);
+      assert(r == std::next(m.begin(), 3));
+      r = m.lower_bound(13);
+      assert(r == std::next(m.begin(), 4));
+      r = m.lower_bound(15);
+      assert(r == std::next(m.begin(), 5));
+      r = m.lower_bound(17);
+      assert(r == std::next(m.begin(), 6));
+      r = m.lower_bound(19);
+      assert(r == std::next(m.begin(), 7));
+      r = m.lower_bound(4);
+      assert(r == std::next(m.begin(), 0));
+      r = m.lower_bound(6);
+      assert(r == std::next(m.begin(), 1));
+      r = m.lower_bound(8);
+      assert(r == std::next(m.begin(), 2));
+      r = m.lower_bound(10);
+      assert(r == std::next(m.begin(), 3));
+      r = m.lower_bound(12);
+      assert(r == std::next(m.begin(), 4));
+      r = m.lower_bound(14);
+      assert(r == std::next(m.begin(), 5));
+      r = m.lower_bound(16);
+      assert(r == std::next(m.begin(), 6));
+      r = m.lower_bound(18);
+      assert(r == std::next(m.begin(), 7));
+      r = m.lower_bound(20);
+      assert(r == std::next(m.begin(), 8));
     }
+  }
 #endif
 #if TEST_STD_VER > 11
-    {
+  {
     typedef std::pair<const int, double> V;
-    typedef std::map<int, double, std::less <>> M;
+    typedef std::map<int, double, std::less<>> M;
     typedef M::iterator R;
 
-    V ar[] =
-    {
-        V(5, 5),
-        V(7, 6),
-        V(9, 7),
-        V(11, 8),
-        V(13, 9),
-        V(15, 10),
-        V(17, 11),
-        V(19, 12)
-    };
-    M m(ar, ar+sizeof(ar)/sizeof(ar[0]));
+    V ar[] = {V(5, 5), V(7, 6), V(9, 7), V(11, 8), V(13, 9), V(15, 10), V(17, 11), V(19, 12)};
+    M m(ar, ar + sizeof(ar) / sizeof(ar[0]));
     R r = m.lower_bound(5);
     assert(r == m.begin());
     r = m.lower_bound(7);
@@ -316,22 +265,22 @@ int main(int, char**)
     assert(r == std::next(m.begin(), 7));
     r = m.lower_bound(C2Int(20));
     assert(r == std::next(m.begin(), 8));
-    }
+  }
 
-    {
+  {
     typedef PrivateConstructor PC;
     typedef std::map<PC, double, std::less<>> M;
     typedef M::iterator R;
 
     M m;
-    m [ PC::make(5)  ] = 5;
-    m [ PC::make(7)  ] = 6;
-    m [ PC::make(9)  ] = 7;
-    m [ PC::make(11) ] = 8;
-    m [ PC::make(13) ] = 9;
-    m [ PC::make(15) ] = 10;
-    m [ PC::make(17) ] = 11;
-    m [ PC::make(19) ] = 12;
+    m[PC::make(5)]  = 5;
+    m[PC::make(7)]  = 6;
+    m[PC::make(9)]  = 7;
+    m[PC::make(11)] = 8;
+    m[PC::make(13)] = 9;
+    m[PC::make(15)] = 10;
+    m[PC::make(17)] = 11;
+    m[PC::make(19)] = 12;
 
     R r = m.lower_bound(5);
     assert(r == m.begin());
@@ -367,7 +316,7 @@ int main(int, char**)
     assert(r == std::next(m.begin(), 7));
     r = m.lower_bound(20);
     assert(r == std::next(m.begin(), 8));
-    }
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/associative/map/map.ops/lower_bound0.pass.cpp b/libcxx/test/std/containers/associative/map/map.ops/lower_bound0.pass.cpp
index 78a42904d8c62..fe7fe381a86eb 100644
--- a/libcxx/test/std/containers/associative/map/map.ops/lower_bound0.pass.cpp
+++ b/libcxx/test/std/containers/associative/map/map.ops/lower_bound0.pass.cpp
@@ -19,25 +19,23 @@
 // equal_range shall not participate in overload resolution unless the
 // qualified-id Compare::is_transparent is valid and denotes a type
 
-
 #include <map>
 #include <cassert>
 
 #include "test_macros.h"
 #include "is_transparent.h"
 
-int main(int, char**)
-{
-    {
+int main(int, char**) {
+  {
     typedef std::map<int, double, transparent_less> M;
     M example;
     assert(example.lower_bound(C2Int{5}) == example.end());
-    }
-    {
+  }
+  {
     typedef std::map<int, double, transparent_less_not_referenceable> M;
     M example;
     assert(example.lower_bound(C2Int{5}) == example.end());
-    }
+  }
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/associative/map/map.ops/lower_bound1.compile.fail.cpp b/libcxx/test/std/containers/associative/map/map.ops/lower_bound1.compile.fail.cpp
index 91ef0befab5b8..e07177b7a6e86 100644
--- a/libcxx/test/std/containers/associative/map/map.ops/lower_bound1.compile.fail.cpp
+++ b/libcxx/test/std/containers/associative/map/map.ops/lower_bound1.compile.fail.cpp
@@ -19,18 +19,16 @@
 // equal_range shall not participate in overload resolution unless the
 // qualified-id Compare::is_transparent is valid and denotes a type
 
-
 #include <map>
 #include <cassert>
 
 #include "test_macros.h"
 #include "is_transparent.h"
 
-int main(int, char**)
-{
-    {
+int main(int, char**) {
+  {
     typedef std::map<int, double, transparent_less_no_type> M;
 
     TEST_IGNORE_NODISCARD M().lower_bound(C2Int{5});
-    }
+  }
 }
diff --git a/libcxx/test/std/containers/associative/map/map.ops/lower_bound2.compile.fail.cpp b/libcxx/test/std/containers/associative/map/map.ops/lower_bound2.compile.fail.cpp
index 7ab8094b098d7..72dc1a5eb2b37 100644
--- a/libcxx/test/std/containers/associative/map/map.ops/lower_bound2.compile.fail.cpp
+++ b/libcxx/test/std/containers/associative/map/map.ops/lower_bound2.compile.fail.cpp
@@ -19,18 +19,16 @@
 // equal_range shall not participate in overload resolution unless the
 // qualified-id Compare::is_transparent is valid and denotes a type
 
-
 #include <map>
 #include <cassert>
 
 #include "test_macros.h"
 #include "is_transparent.h"
 
-int main(int, char**)
-{
-    {
+int main(int, char**) {
+  {
     typedef std::map<int, double, transparent_less_private> M;
 
     TEST_IGNORE_NODISCARD M().lower_bound(C2Int{5});
-    }
+  }
 }
diff --git a/libcxx/test/std/containers/associative/map/map.ops/lower_bound3.compile.fail.cpp b/libcxx/test/std/containers/associative/map/map.ops/lower_bound3.compile.fail.cpp
index 09f7ac4c88001..10da1aad25f29 100644
--- a/libcxx/test/std/containers/associative/map/map.ops/lower_bound3.compile.fail.cpp
+++ b/libcxx/test/std/containers/associative/map/map.ops/lower_bound3.compile.fail.cpp
@@ -25,11 +25,10 @@
 #include "test_macros.h"
 #include "is_transparent.h"
 
-int main(int, char**)
-{
-    {
+int main(int, char**) {
+  {
     typedef std::map<int, double, transparent_less_not_a_type> M;
 
     TEST_IGNORE_NODISCARD M().lower_bound(C2Int{5});
-    }
+  }
 }
diff --git a/libcxx/test/std/containers/associative/map/map.ops/upper_bound.pass.cpp b/libcxx/test/std/containers/associative/map/map.ops/upper_bound.pass.cpp
index 3bacd76ed684b..bd967e3dfb742 100644
--- a/libcxx/test/std/containers/associative/map/map.ops/upper_bound.pass.cpp
+++ b/libcxx/test/std/containers/associative/map/map.ops/upper_bound.pass.cpp
@@ -20,231 +20,180 @@
 #include "min_allocator.h"
 #include "private_constructor.h"
 
-int main(int, char**)
-{
-    {
+int main(int, char**) {
+  {
     typedef std::pair<const int, double> V;
     typedef std::map<int, double> M;
     {
-        typedef M::iterator R;
-        V ar[] =
-        {
-            V(5, 5),
-            V(7, 6),
-            V(9, 7),
-            V(11, 8),
-            V(13, 9),
-            V(15, 10),
-            V(17, 11),
-            V(19, 12)
-        };
-        M m(ar, ar+sizeof(ar)/sizeof(ar[0]));
-        R r = m.upper_bound(5);
-        assert(r == std::next(m.begin(), 1));
-        r = m.upper_bound(7);
-        assert(r == std::next(m.begin(), 2));
-        r = m.upper_bound(9);
-        assert(r == std::next(m.begin(), 3));
-        r = m.upper_bound(11);
-        assert(r == std::next(m.begin(), 4));
-        r = m.upper_bound(13);
-        assert(r == std::next(m.begin(), 5));
-        r = m.upper_bound(15);
-        assert(r == std::next(m.begin(), 6));
-        r = m.upper_bound(17);
-        assert(r == std::next(m.begin(), 7));
-        r = m.upper_bound(19);
-        assert(r == std::next(m.begin(), 8));
-        r = m.upper_bound(4);
-        assert(r == std::next(m.begin(), 0));
-        r = m.upper_bound(6);
-        assert(r == std::next(m.begin(), 1));
-        r = m.upper_bound(8);
-        assert(r == std::next(m.begin(), 2));
-        r = m.upper_bound(10);
-        assert(r == std::next(m.begin(), 3));
-        r = m.upper_bound(12);
-        assert(r == std::next(m.begin(), 4));
-        r = m.upper_bound(14);
-        assert(r == std::next(m.begin(), 5));
-        r = m.upper_bound(16);
-        assert(r == std::next(m.begin(), 6));
-        r = m.upper_bound(18);
-        assert(r == std::next(m.begin(), 7));
-        r = m.upper_bound(20);
-        assert(r == std::next(m.begin(), 8));
+      typedef M::iterator R;
+      V ar[] = {V(5, 5), V(7, 6), V(9, 7), V(11, 8), V(13, 9), V(15, 10), V(17, 11), V(19, 12)};
+      M m(ar, ar + sizeof(ar) / sizeof(ar[0]));
+      R r = m.upper_bound(5);
+      assert(r == std::next(m.begin(), 1));
+      r = m.upper_bound(7);
+      assert(r == std::next(m.begin(), 2));
+      r = m.upper_bound(9);
+      assert(r == std::next(m.begin(), 3));
+      r = m.upper_bound(11);
+      assert(r == std::next(m.begin(), 4));
+      r = m.upper_bound(13);
+      assert(r == std::next(m.begin(), 5));
+      r = m.upper_bound(15);
+      assert(r == std::next(m.begin(), 6));
+      r = m.upper_bound(17);
+      assert(r == std::next(m.begin(), 7));
+      r = m.upper_bound(19);
+      assert(r == std::next(m.begin(), 8));
+      r = m.upper_bound(4);
+      assert(r == std::next(m.begin(), 0));
+      r = m.upper_bound(6);
+      assert(r == std::next(m.begin(), 1));
+      r = m.upper_bound(8);
+      assert(r == std::next(m.begin(), 2));
+      r = m.upper_bound(10);
+      assert(r == std::next(m.begin(), 3));
+      r = m.upper_bound(12);
+      assert(r == std::next(m.begin(), 4));
+      r = m.upper_bound(14);
+      assert(r == std::next(m.begin(), 5));
+      r = m.upper_bound(16);
+      assert(r == std::next(m.begin(), 6));
+      r = m.upper_bound(18);
+      assert(r == std::next(m.begin(), 7));
+      r = m.upper_bound(20);
+      assert(r == std::next(m.begin(), 8));
     }
     {
-        typedef M::const_iterator R;
-        V ar[] =
-        {
-            V(5, 5),
-            V(7, 6),
-            V(9, 7),
-            V(11, 8),
-            V(13, 9),
-            V(15, 10),
-            V(17, 11),
-            V(19, 12)
-        };
-        const M m(ar, ar+sizeof(ar)/sizeof(ar[0]));
-        R r = m.upper_bound(5);
-        assert(r == std::next(m.begin(), 1));
-        r = m.upper_bound(7);
-        assert(r == std::next(m.begin(), 2));
-        r = m.upper_bound(9);
-        assert(r == std::next(m.begin(), 3));
-        r = m.upper_bound(11);
-        assert(r == std::next(m.begin(), 4));
-        r = m.upper_bound(13);
-        assert(r == std::next(m.begin(), 5));
-        r = m.upper_bound(15);
-        assert(r == std::next(m.begin(), 6));
-        r = m.upper_bound(17);
-        assert(r == std::next(m.begin(), 7));
-        r = m.upper_bound(19);
-        assert(r == std::next(m.begin(), 8));
-        r = m.upper_bound(4);
-        assert(r == std::next(m.begin(), 0));
-        r = m.upper_bound(6);
-        assert(r == std::next(m.begin(), 1));
-        r = m.upper_bound(8);
-        assert(r == std::next(m.begin(), 2));
-        r = m.upper_bound(10);
-        assert(r == std::next(m.begin(), 3));
-        r = m.upper_bound(12);
-        assert(r == std::next(m.begin(), 4));
-        r = m.upper_bound(14);
-        assert(r == std::next(m.begin(), 5));
-        r = m.upper_bound(16);
-        assert(r == std::next(m.begin(), 6));
-        r = m.upper_bound(18);
-        assert(r == std::next(m.begin(), 7));
-        r = m.upper_bound(20);
-        assert(r == std::next(m.begin(), 8));
-    }
+      typedef M::const_iterator R;
+      V ar[] = {V(5, 5), V(7, 6), V(9, 7), V(11, 8), V(13, 9), V(15, 10), V(17, 11), V(19, 12)};
+      const M m(ar, ar + sizeof(ar) / sizeof(ar[0]));
+      R r = m.upper_bound(5);
+      assert(r == std::next(m.begin(), 1));
+      r = m.upper_bound(7);
+      assert(r == std::next(m.begin(), 2));
+      r = m.upper_bound(9);
+      assert(r == std::next(m.begin(), 3));
+      r = m.upper_bound(11);
+      assert(r == std::next(m.begin(), 4));
+      r = m.upper_bound(13);
+      assert(r == std::next(m.begin(), 5));
+      r = m.upper_bound(15);
+      assert(r == std::next(m.begin(), 6));
+      r = m.upper_bound(17);
+      assert(r == std::next(m.begin(), 7));
+      r = m.upper_bound(19);
+      assert(r == std::next(m.begin(), 8));
+      r = m.upper_bound(4);
+      assert(r == std::next(m.begin(), 0));
+      r = m.upper_bound(6);
+      assert(r == std::next(m.begin(), 1));
+      r = m.upper_bound(8);
+      assert(r == std::next(m.begin(), 2));
+      r = m.upper_bound(10);
+      assert(r == std::next(m.begin(), 3));
+      r = m.upper_bound(12);
+      assert(r == std::next(m.begin(), 4));
+      r = m.upper_bound(14);
+      assert(r == std::next(m.begin(), 5));
+      r = m.upper_bound(16);
+      assert(r == std::next(m.begin(), 6));
+      r = m.upper_bound(18);
+      assert(r == std::next(m.begin(), 7));
+      r = m.upper_bound(20);
+      assert(r == std::next(m.begin(), 8));
     }
+  }
 #if TEST_STD_VER >= 11
-    {
+  {
     typedef std::pair<const int, double> V;
     typedef std::map<int, double, std::less<int>, min_allocator<V>> M;
     {
-        typedef M::iterator R;
-        V ar[] =
-        {
-            V(5, 5),
-            V(7, 6),
-            V(9, 7),
-            V(11, 8),
-            V(13, 9),
-            V(15, 10),
-            V(17, 11),
-            V(19, 12)
-        };
-        M m(ar, ar+sizeof(ar)/sizeof(ar[0]));
-        R r = m.upper_bound(5);
-        assert(r == std::next(m.begin(), 1));
-        r = m.upper_bound(7);
-        assert(r == std::next(m.begin(), 2));
-        r = m.upper_bound(9);
-        assert(r == std::next(m.begin(), 3));
-        r = m.upper_bound(11);
-        assert(r == std::next(m.begin(), 4));
-        r = m.upper_bound(13);
-        assert(r == std::next(m.begin(), 5));
-        r = m.upper_bound(15);
-        assert(r == std::next(m.begin(), 6));
-        r = m.upper_bound(17);
-        assert(r == std::next(m.begin(), 7));
-        r = m.upper_bound(19);
-        assert(r == std::next(m.begin(), 8));
-        r = m.upper_bound(4);
-        assert(r == std::next(m.begin(), 0));
-        r = m.upper_bound(6);
-        assert(r == std::next(m.begin(), 1));
-        r = m.upper_bound(8);
-        assert(r == std::next(m.begin(), 2));
-        r = m.upper_bound(10);
-        assert(r == std::next(m.begin(), 3));
-        r = m.upper_bound(12);
-        assert(r == std::next(m.begin(), 4));
-        r = m.upper_bound(14);
-        assert(r == std::next(m.begin(), 5));
-        r = m.upper_bound(16);
-        assert(r == std::next(m.begin(), 6));
-        r = m.upper_bound(18);
-        assert(r == std::next(m.begin(), 7));
-        r = m.upper_bound(20);
-        assert(r == std::next(m.begin(), 8));
+      typedef M::iterator R;
+      V ar[] = {V(5, 5), V(7, 6), V(9, 7), V(11, 8), V(13, 9), V(15, 10), V(17, 11), V(19, 12)};
+      M m(ar, ar + sizeof(ar) / sizeof(ar[0]));
+      R r = m.upper_bound(5);
+      assert(r == std::next(m.begin(), 1));
+      r = m.upper_bound(7);
+      assert(r == std::next(m.begin(), 2));
+      r = m.upper_bound(9);
+      assert(r == std::next(m.begin(), 3));
+      r = m.upper_bound(11);
+      assert(r == std::next(m.begin(), 4));
+      r = m.upper_bound(13);
+      assert(r == std::next(m.begin(), 5));
+      r = m.upper_bound(15);
+      assert(r == std::next(m.begin(), 6));
+      r = m.upper_bound(17);
+      assert(r == std::next(m.begin(), 7));
+      r = m.upper_bound(19);
+      assert(r == std::next(m.begin(), 8));
+      r = m.upper_bound(4);
+      assert(r == std::next(m.begin(), 0));
+      r = m.upper_bound(6);
+      assert(r == std::next(m.begin(), 1));
+      r = m.upper_bound(8);
+      assert(r == std::next(m.begin(), 2));
+      r = m.upper_bound(10);
+      assert(r == std::next(m.begin(), 3));
+      r = m.upper_bound(12);
+      assert(r == std::next(m.begin(), 4));
+      r = m.upper_bound(14);
+      assert(r == std::next(m.begin(), 5));
+      r = m.upper_bound(16);
+      assert(r == std::next(m.begin(), 6));
+      r = m.upper_bound(18);
+      assert(r == std::next(m.begin(), 7));
+      r = m.upper_bound(20);
+      assert(r == std::next(m.begin(), 8));
     }
     {
-        typedef M::const_iterator R;
-        V ar[] =
-        {
-            V(5, 5),
-            V(7, 6),
-            V(9, 7),
-            V(11, 8),
-            V(13, 9),
-            V(15, 10),
-            V(17, 11),
-            V(19, 12)
-        };
-        const M m(ar, ar+sizeof(ar)/sizeof(ar[0]));
-        R r = m.upper_bound(5);
-        assert(r == std::next(m.begin(), 1));
-        r = m.upper_bound(7);
-        assert(r == std::next(m.begin(), 2));
-        r = m.upper_bound(9);
-        assert(r == std::next(m.begin(), 3));
-        r = m.upper_bound(11);
-        assert(r == std::next(m.begin(), 4));
-        r = m.upper_bound(13);
-        assert(r == std::next(m.begin(), 5));
-        r = m.upper_bound(15);
-        assert(r == std::next(m.begin(), 6));
-        r = m.upper_bound(17);
-        assert(r == std::next(m.begin(), 7));
-        r = m.upper_bound(19);
-        assert(r == std::next(m.begin(), 8));
-        r = m.upper_bound(4);
-        assert(r == std::next(m.begin(), 0));
-        r = m.upper_bound(6);
-        assert(r == std::next(m.begin(), 1));
-        r = m.upper_bound(8);
-        assert(r == std::next(m.begin(), 2));
-        r = m.upper_bound(10);
-        assert(r == std::next(m.begin(), 3));
-        r = m.upper_bound(12);
-        assert(r == std::next(m.begin(), 4));
-        r = m.upper_bound(14);
-        assert(r == std::next(m.begin(), 5));
-        r = m.upper_bound(16);
-        assert(r == std::next(m.begin(), 6));
-        r = m.upper_bound(18);
-        assert(r == std::next(m.begin(), 7));
-        r = m.upper_bound(20);
-        assert(r == std::next(m.begin(), 8));
-    }
+      typedef M::const_iterator R;
+      V ar[] = {V(5, 5), V(7, 6), V(9, 7), V(11, 8), V(13, 9), V(15, 10), V(17, 11), V(19, 12)};
+      const M m(ar, ar + sizeof(ar) / sizeof(ar[0]));
+      R r = m.upper_bound(5);
+      assert(r == std::next(m.begin(), 1));
+      r = m.upper_bound(7);
+      assert(r == std::next(m.begin(), 2));
+      r = m.upper_bound(9);
+      assert(r == std::next(m.begin(), 3));
+      r = m.upper_bound(11);
+      assert(r == std::next(m.begin(), 4));
+      r = m.upper_bound(13);
+      assert(r == std::next(m.begin(), 5));
+      r = m.upper_bound(15);
+      assert(r == std::next(m.begin(), 6));
+      r = m.upper_bound(17);
+      assert(r == std::next(m.begin(), 7));
+      r = m.upper_bound(19);
+      assert(r == std::next(m.begin(), 8));
+      r = m.upper_bound(4);
+      assert(r == std::next(m.begin(), 0));
+      r = m.upper_bound(6);
+      assert(r == std::next(m.begin(), 1));
+      r = m.upper_bound(8);
+      assert(r == std::next(m.begin(), 2));
+      r = m.upper_bound(10);
+      assert(r == std::next(m.begin(), 3));
+      r = m.upper_bound(12);
+      assert(r == std::next(m.begin(), 4));
+      r = m.upper_bound(14);
+      assert(r == std::next(m.begin(), 5));
+      r = m.upper_bound(16);
+      assert(r == std::next(m.begin(), 6));
+      r = m.upper_bound(18);
+      assert(r == std::next(m.begin(), 7));
+      r = m.upper_bound(20);
+      assert(r == std::next(m.begin(), 8));
     }
+  }
 #endif
 #if TEST_STD_VER > 11
-    {
+  {
     typedef std::pair<const int, double> V;
     typedef std::map<int, double, std::less<>> M;
     typedef M::iterator R;
-    V ar[] =
-    {
-        V(5, 5),
-        V(7, 6),
-        V(9, 7),
-        V(11, 8),
-        V(13, 9),
-        V(15, 10),
-        V(17, 11),
-        V(19, 12)
-    };
-    M m(ar, ar+sizeof(ar)/sizeof(ar[0]));
+    V ar[] = {V(5, 5), V(7, 6), V(9, 7), V(11, 8), V(13, 9), V(15, 10), V(17, 11), V(19, 12)};
+    M m(ar, ar + sizeof(ar) / sizeof(ar[0]));
     R r = m.upper_bound(5);
     assert(r == std::next(m.begin(), 1));
     r = m.upper_bound(7);
@@ -279,22 +228,22 @@ int main(int, char**)
     assert(r == std::next(m.begin(), 7));
     r = m.upper_bound(20);
     assert(r == std::next(m.begin(), 8));
-    }
+  }
 
-    {
+  {
     typedef PrivateConstructor PC;
     typedef std::map<PC, double, std::less<>> M;
     typedef M::iterator R;
 
     M m;
-    m [ PC::make(5)  ] = 5;
-    m [ PC::make(7)  ] = 6;
-    m [ PC::make(9)  ] = 7;
-    m [ PC::make(11) ] = 8;
-    m [ PC::make(13) ] = 9;
-    m [ PC::make(15) ] = 10;
-    m [ PC::make(17) ] = 11;
-    m [ PC::make(19) ] = 12;
+    m[PC::make(5)]  = 5;
+    m[PC::make(7)]  = 6;
+    m[PC::make(9)]  = 7;
+    m[PC::make(11)] = 8;
+    m[PC::make(13)] = 9;
+    m[PC::make(15)] = 10;
+    m[PC::make(17)] = 11;
+    m[PC::make(19)] = 12;
 
     R r = m.upper_bound(5);
     assert(r == std::next(m.begin(), 1));
@@ -330,7 +279,7 @@ int main(int, char**)
     assert(r == std::next(m.begin(), 7));
     r = m.upper_bound(20);
     assert(r == std::next(m.begin(), 8));
-    }
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/associative/map/map.ops/upper_bound0.pass.cpp b/libcxx/test/std/containers/associative/map/map.ops/upper_bound0.pass.cpp
index 370c7af65cfc8..525aa673ea74b 100644
--- a/libcxx/test/std/containers/associative/map/map.ops/upper_bound0.pass.cpp
+++ b/libcxx/test/std/containers/associative/map/map.ops/upper_bound0.pass.cpp
@@ -19,25 +19,23 @@
 // equal_range shall not participate in overload resolution unless the
 // qualified-id Compare::is_transparent is valid and denotes a type
 
-
 #include <map>
 #include <cassert>
 
 #include "test_macros.h"
 #include "is_transparent.h"
 
-int main(int, char**)
-{
-    {
+int main(int, char**) {
+  {
     typedef std::map<int, double, transparent_less> M;
     M example;
     assert(example.upper_bound(C2Int{5}) == example.end());
-    }
-    {
+  }
+  {
     typedef std::map<int, double, transparent_less_not_referenceable> M;
     M example;
     assert(example.upper_bound(C2Int{5}) == example.end());
-    }
+  }
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/associative/map/map.ops/upper_bound1.compile.fail.cpp b/libcxx/test/std/containers/associative/map/map.ops/upper_bound1.compile.fail.cpp
index 92003b60e849a..14d69988f7016 100644
--- a/libcxx/test/std/containers/associative/map/map.ops/upper_bound1.compile.fail.cpp
+++ b/libcxx/test/std/containers/associative/map/map.ops/upper_bound1.compile.fail.cpp
@@ -19,18 +19,16 @@
 // equal_range shall not participate in overload resolution unless the
 // qualified-id Compare::is_transparent is valid and denotes a type
 
-
 #include <map>
 #include <cassert>
 
 #include "test_macros.h"
 #include "is_transparent.h"
 
-int main(int, char**)
-{
-    {
+int main(int, char**) {
+  {
     typedef std::map<int, double, transparent_less_no_type> M;
 
     TEST_IGNORE_NODISCARD M().upper_bound(C2Int{5});
-    }
+  }
 }
diff --git a/libcxx/test/std/containers/associative/map/map.ops/upper_bound2.compile.fail.cpp b/libcxx/test/std/containers/associative/map/map.ops/upper_bound2.compile.fail.cpp
index e02ef4d054119..a66536a9d1f9b 100644
--- a/libcxx/test/std/containers/associative/map/map.ops/upper_bound2.compile.fail.cpp
+++ b/libcxx/test/std/containers/associative/map/map.ops/upper_bound2.compile.fail.cpp
@@ -19,18 +19,16 @@
 // equal_range shall not participate in overload resolution unless the
 // qualified-id Compare::is_transparent is valid and denotes a type
 
-
 #include <map>
 #include <cassert>
 
 #include "test_macros.h"
 #include "is_transparent.h"
 
-int main(int, char**)
-{
-    {
+int main(int, char**) {
+  {
     typedef std::map<int, double, transparent_less_private> M;
 
     TEST_IGNORE_NODISCARD M().upper_bound(C2Int{5});
-    }
+  }
 }
diff --git a/libcxx/test/std/containers/associative/map/map.ops/upper_bound3.compile.fail.cpp b/libcxx/test/std/containers/associative/map/map.ops/upper_bound3.compile.fail.cpp
index 24aa859054749..ae6eab1249284 100644
--- a/libcxx/test/std/containers/associative/map/map.ops/upper_bound3.compile.fail.cpp
+++ b/libcxx/test/std/containers/associative/map/map.ops/upper_bound3.compile.fail.cpp
@@ -19,18 +19,16 @@
 // equal_range shall not participate in overload resolution unless the
 // qualified-id Compare::is_transparent is valid and denotes a type
 
-
 #include <map>
 #include <cassert>
 
 #include "test_macros.h"
 #include "is_transparent.h"
 
-int main(int, char**)
-{
-    {
+int main(int, char**) {
+  {
     typedef std::map<int, double, transparent_less_not_a_type> M;
 
     TEST_IGNORE_NODISCARD M().upper_bound(C2Int{5});
-    }
+  }
 }
diff --git a/libcxx/test/std/containers/associative/map/map.special/member_swap.pass.cpp b/libcxx/test/std/containers/associative/map/map.special/member_swap.pass.cpp
index ddce2225e0c63..ffb74f62b8877 100644
--- a/libcxx/test/std/containers/associative/map/map.special/member_swap.pass.cpp
+++ b/libcxx/test/std/containers/associative/map/map.special/member_swap.pass.cpp
@@ -18,160 +18,95 @@
 #include "test_macros.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    typedef std::pair<const int, double> V;
-    {
+int main(int, char**) {
+  typedef std::pair<const int, double> V;
+  {
     typedef std::map<int, double> M;
     {
-        M m1;
-        M m2;
-        M m1_save = m1;
-        M m2_save = m2;
-        m1.swap(m2);
-        assert(m1 == m2_save);
-        assert(m2 == m1_save);
+      M m1;
+      M m2;
+      M m1_save = m1;
+      M m2_save = m2;
+      m1.swap(m2);
+      assert(m1 == m2_save);
+      assert(m2 == m1_save);
     }
     {
-        V ar2[] =
-        {
-            V(5, 5),
-            V(6, 6),
-            V(7, 7),
-            V(8, 8),
-            V(9, 9),
-            V(10, 10),
-            V(11, 11),
-            V(12, 12)
-        };
-        M m1;
-        M m2(ar2, ar2+sizeof(ar2)/sizeof(ar2[0]));
-        M m1_save = m1;
-        M m2_save = m2;
-        m1.swap(m2);
-        assert(m1 == m2_save);
-        assert(m2 == m1_save);
+      V ar2[] = {V(5, 5), V(6, 6), V(7, 7), V(8, 8), V(9, 9), V(10, 10), V(11, 11), V(12, 12)};
+      M m1;
+      M m2(ar2, ar2 + sizeof(ar2) / sizeof(ar2[0]));
+      M m1_save = m1;
+      M m2_save = m2;
+      m1.swap(m2);
+      assert(m1 == m2_save);
+      assert(m2 == m1_save);
     }
     {
-        V ar1[] =
-        {
-            V(1, 1),
-            V(2, 2),
-            V(3, 3),
-            V(4, 4)
-        };
-        M m1(ar1, ar1+sizeof(ar1)/sizeof(ar1[0]));
-        M m2;
-        M m1_save = m1;
-        M m2_save = m2;
-        m1.swap(m2);
-        assert(m1 == m2_save);
-        assert(m2 == m1_save);
+      V ar1[] = {V(1, 1), V(2, 2), V(3, 3), V(4, 4)};
+      M m1(ar1, ar1 + sizeof(ar1) / sizeof(ar1[0]));
+      M m2;
+      M m1_save = m1;
+      M m2_save = m2;
+      m1.swap(m2);
+      assert(m1 == m2_save);
+      assert(m2 == m1_save);
     }
     {
-        V ar1[] =
-        {
-            V(1, 1),
-            V(2, 2),
-            V(3, 3),
-            V(4, 4)
-        };
-        V ar2[] =
-        {
-            V(5, 5),
-            V(6, 6),
-            V(7, 7),
-            V(8, 8),
-            V(9, 9),
-            V(10, 10),
-            V(11, 11),
-            V(12, 12)
-        };
-        M m1(ar1, ar1+sizeof(ar1)/sizeof(ar1[0]));
-        M m2(ar2, ar2+sizeof(ar2)/sizeof(ar2[0]));
-        M m1_save = m1;
-        M m2_save = m2;
-        m1.swap(m2);
-        assert(m1 == m2_save);
-        assert(m2 == m1_save);
-    }
+      V ar1[] = {V(1, 1), V(2, 2), V(3, 3), V(4, 4)};
+      V ar2[] = {V(5, 5), V(6, 6), V(7, 7), V(8, 8), V(9, 9), V(10, 10), V(11, 11), V(12, 12)};
+      M m1(ar1, ar1 + sizeof(ar1) / sizeof(ar1[0]));
+      M m2(ar2, ar2 + sizeof(ar2) / sizeof(ar2[0]));
+      M m1_save = m1;
+      M m2_save = m2;
+      m1.swap(m2);
+      assert(m1 == m2_save);
+      assert(m2 == m1_save);
     }
+  }
 #if TEST_STD_VER >= 11
-    {
+  {
     typedef std::map<int, double, std::less<int>, min_allocator<V>> M;
     {
-        M m1;
-        M m2;
-        M m1_save = m1;
-        M m2_save = m2;
-        m1.swap(m2);
-        assert(m1 == m2_save);
-        assert(m2 == m1_save);
+      M m1;
+      M m2;
+      M m1_save = m1;
+      M m2_save = m2;
+      m1.swap(m2);
+      assert(m1 == m2_save);
+      assert(m2 == m1_save);
     }
     {
-        V ar2[] =
-        {
-            V(5, 5),
-            V(6, 6),
-            V(7, 7),
-            V(8, 8),
-            V(9, 9),
-            V(10, 10),
-            V(11, 11),
-            V(12, 12)
-        };
-        M m1;
-        M m2(ar2, ar2+sizeof(ar2)/sizeof(ar2[0]));
-        M m1_save = m1;
-        M m2_save = m2;
-        m1.swap(m2);
-        assert(m1 == m2_save);
-        assert(m2 == m1_save);
+      V ar2[] = {V(5, 5), V(6, 6), V(7, 7), V(8, 8), V(9, 9), V(10, 10), V(11, 11), V(12, 12)};
+      M m1;
+      M m2(ar2, ar2 + sizeof(ar2) / sizeof(ar2[0]));
+      M m1_save = m1;
+      M m2_save = m2;
+      m1.swap(m2);
+      assert(m1 == m2_save);
+      assert(m2 == m1_save);
     }
     {
-        V ar1[] =
-        {
-            V(1, 1),
-            V(2, 2),
-            V(3, 3),
-            V(4, 4)
-        };
-        M m1(ar1, ar1+sizeof(ar1)/sizeof(ar1[0]));
-        M m2;
-        M m1_save = m1;
-        M m2_save = m2;
-        m1.swap(m2);
-        assert(m1 == m2_save);
-        assert(m2 == m1_save);
+      V ar1[] = {V(1, 1), V(2, 2), V(3, 3), V(4, 4)};
+      M m1(ar1, ar1 + sizeof(ar1) / sizeof(ar1[0]));
+      M m2;
+      M m1_save = m1;
+      M m2_save = m2;
+      m1.swap(m2);
+      assert(m1 == m2_save);
+      assert(m2 == m1_save);
     }
     {
-        V ar1[] =
-        {
-            V(1, 1),
-            V(2, 2),
-            V(3, 3),
-            V(4, 4)
-        };
-        V ar2[] =
-        {
-            V(5, 5),
-            V(6, 6),
-            V(7, 7),
-            V(8, 8),
-            V(9, 9),
-            V(10, 10),
-            V(11, 11),
-            V(12, 12)
-        };
-        M m1(ar1, ar1+sizeof(ar1)/sizeof(ar1[0]));
-        M m2(ar2, ar2+sizeof(ar2)/sizeof(ar2[0]));
-        M m1_save = m1;
-        M m2_save = m2;
-        m1.swap(m2);
-        assert(m1 == m2_save);
-        assert(m2 == m1_save);
-    }
+      V ar1[] = {V(1, 1), V(2, 2), V(3, 3), V(4, 4)};
+      V ar2[] = {V(5, 5), V(6, 6), V(7, 7), V(8, 8), V(9, 9), V(10, 10), V(11, 11), V(12, 12)};
+      M m1(ar1, ar1 + sizeof(ar1) / sizeof(ar1[0]));
+      M m2(ar2, ar2 + sizeof(ar2) / sizeof(ar2[0]));
+      M m1_save = m1;
+      M m2_save = m2;
+      m1.swap(m2);
+      assert(m1 == m2_save);
+      assert(m2 == m1_save);
     }
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/associative/map/map.special/non_member_swap.pass.cpp b/libcxx/test/std/containers/associative/map/map.special/non_member_swap.pass.cpp
index 18d991c15e4ac..ba390acd128f5 100644
--- a/libcxx/test/std/containers/associative/map/map.special/non_member_swap.pass.cpp
+++ b/libcxx/test/std/containers/associative/map/map.special/non_member_swap.pass.cpp
@@ -21,262 +21,149 @@
 #include "../../../test_compare.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    typedef std::pair<const int, double> V;
-    {
+int main(int, char**) {
+  typedef std::pair<const int, double> V;
+  {
     typedef std::map<int, double> M;
     {
-        M m1;
-        M m2;
-        M m1_save = m1;
-        M m2_save = m2;
-        swap(m1, m2);
-        assert(m1 == m2_save);
-        assert(m2 == m1_save);
-    }
-    {
-        V ar2[] =
-        {
-            V(5, 5),
-            V(6, 6),
-            V(7, 7),
-            V(8, 8),
-            V(9, 9),
-            V(10, 10),
-            V(11, 11),
-            V(12, 12)
-        };
-        M m1;
-        M m2(ar2, ar2+sizeof(ar2)/sizeof(ar2[0]));
-        M m1_save = m1;
-        M m2_save = m2;
-        swap(m1, m2);
-        assert(m1 == m2_save);
-        assert(m2 == m1_save);
-    }
-    {
-        V ar1[] =
-        {
-            V(1, 1),
-            V(2, 2),
-            V(3, 3),
-            V(4, 4)
-        };
-        M m1(ar1, ar1+sizeof(ar1)/sizeof(ar1[0]));
-        M m2;
-        M m1_save = m1;
-        M m2_save = m2;
-        swap(m1, m2);
-        assert(m1 == m2_save);
-        assert(m2 == m1_save);
+      M m1;
+      M m2;
+      M m1_save = m1;
+      M m2_save = m2;
+      swap(m1, m2);
+      assert(m1 == m2_save);
+      assert(m2 == m1_save);
     }
     {
-        V ar1[] =
-        {
-            V(1, 1),
-            V(2, 2),
-            V(3, 3),
-            V(4, 4)
-        };
-        V ar2[] =
-        {
-            V(5, 5),
-            V(6, 6),
-            V(7, 7),
-            V(8, 8),
-            V(9, 9),
-            V(10, 10),
-            V(11, 11),
-            V(12, 12)
-        };
-        M m1(ar1, ar1+sizeof(ar1)/sizeof(ar1[0]));
-        M m2(ar2, ar2+sizeof(ar2)/sizeof(ar2[0]));
-        M m1_save = m1;
-        M m2_save = m2;
-        swap(m1, m2);
-        assert(m1 == m2_save);
-        assert(m2 == m1_save);
-    }
+      V ar2[] = {V(5, 5), V(6, 6), V(7, 7), V(8, 8), V(9, 9), V(10, 10), V(11, 11), V(12, 12)};
+      M m1;
+      M m2(ar2, ar2 + sizeof(ar2) / sizeof(ar2[0]));
+      M m1_save = m1;
+      M m2_save = m2;
+      swap(m1, m2);
+      assert(m1 == m2_save);
+      assert(m2 == m1_save);
     }
     {
-        typedef test_allocator<V> A;
-        typedef test_less<int> C;
-        typedef std::map<int, double, C, A> M;
-        V ar1[] =
-        {
-            V(1, 1),
-            V(2, 2),
-            V(3, 3),
-            V(4, 4)
-        };
-        V ar2[] =
-        {
-            V(5, 5),
-            V(6, 6),
-            V(7, 7),
-            V(8, 8),
-            V(9, 9),
-            V(10, 10),
-            V(11, 11),
-            V(12, 12)
-        };
-        M m1(ar1, ar1+sizeof(ar1)/sizeof(ar1[0]), C(1), A(1, 1));
-        M m2(ar2, ar2+sizeof(ar2)/sizeof(ar2[0]), C(2), A(1, 2));
-        M m1_save = m1;
-        M m2_save = m2;
-        swap(m1, m2);
-        assert(m1 == m2_save);
-        assert(m2 == m1_save);
-        assert(m1.key_comp() == C(2));
-        assert(m1.get_allocator().get_id() == 1); // not swapped
-        assert(m2.key_comp() == C(1));
-        assert(m2.get_allocator().get_id() == 2);
+      V ar1[] = {V(1, 1), V(2, 2), V(3, 3), V(4, 4)};
+      M m1(ar1, ar1 + sizeof(ar1) / sizeof(ar1[0]));
+      M m2;
+      M m1_save = m1;
+      M m2_save = m2;
+      swap(m1, m2);
+      assert(m1 == m2_save);
+      assert(m2 == m1_save);
     }
     {
-        typedef other_allocator<V> A;
-        typedef test_less<int> C;
-        typedef std::map<int, double, C, A> M;
-        V ar1[] =
-        {
-            V(1, 1),
-            V(2, 2),
-            V(3, 3),
-            V(4, 4)
-        };
-        V ar2[] =
-        {
-            V(5, 5),
-            V(6, 6),
-            V(7, 7),
-            V(8, 8),
-            V(9, 9),
-            V(10, 10),
-            V(11, 11),
-            V(12, 12)
-        };
-        M m1(ar1, ar1+sizeof(ar1)/sizeof(ar1[0]), C(1), A(1));
-        M m2(ar2, ar2+sizeof(ar2)/sizeof(ar2[0]), C(2), A(2));
-        M m1_save = m1;
-        M m2_save = m2;
-        swap(m1, m2);
-        assert(m1 == m2_save);
-        assert(m2 == m1_save);
-        assert(m1.key_comp() == C(2));
-        assert(m1.get_allocator() == A(2));
-        assert(m2.key_comp() == C(1));
-        assert(m2.get_allocator() == A(1));
+      V ar1[] = {V(1, 1), V(2, 2), V(3, 3), V(4, 4)};
+      V ar2[] = {V(5, 5), V(6, 6), V(7, 7), V(8, 8), V(9, 9), V(10, 10), V(11, 11), V(12, 12)};
+      M m1(ar1, ar1 + sizeof(ar1) / sizeof(ar1[0]));
+      M m2(ar2, ar2 + sizeof(ar2) / sizeof(ar2[0]));
+      M m1_save = m1;
+      M m2_save = m2;
+      swap(m1, m2);
+      assert(m1 == m2_save);
+      assert(m2 == m1_save);
     }
+  }
+  {
+    typedef test_allocator<V> A;
+    typedef test_less<int> C;
+    typedef std::map<int, double, C, A> M;
+    V ar1[] = {V(1, 1), V(2, 2), V(3, 3), V(4, 4)};
+    V ar2[] = {V(5, 5), V(6, 6), V(7, 7), V(8, 8), V(9, 9), V(10, 10), V(11, 11), V(12, 12)};
+    M m1(ar1, ar1 + sizeof(ar1) / sizeof(ar1[0]), C(1), A(1, 1));
+    M m2(ar2, ar2 + sizeof(ar2) / sizeof(ar2[0]), C(2), A(1, 2));
+    M m1_save = m1;
+    M m2_save = m2;
+    swap(m1, m2);
+    assert(m1 == m2_save);
+    assert(m2 == m1_save);
+    assert(m1.key_comp() == C(2));
+    assert(m1.get_allocator().get_id() == 1); // not swapped
+    assert(m2.key_comp() == C(1));
+    assert(m2.get_allocator().get_id() == 2);
+  }
+  {
+    typedef other_allocator<V> A;
+    typedef test_less<int> C;
+    typedef std::map<int, double, C, A> M;
+    V ar1[] = {V(1, 1), V(2, 2), V(3, 3), V(4, 4)};
+    V ar2[] = {V(5, 5), V(6, 6), V(7, 7), V(8, 8), V(9, 9), V(10, 10), V(11, 11), V(12, 12)};
+    M m1(ar1, ar1 + sizeof(ar1) / sizeof(ar1[0]), C(1), A(1));
+    M m2(ar2, ar2 + sizeof(ar2) / sizeof(ar2[0]), C(2), A(2));
+    M m1_save = m1;
+    M m2_save = m2;
+    swap(m1, m2);
+    assert(m1 == m2_save);
+    assert(m2 == m1_save);
+    assert(m1.key_comp() == C(2));
+    assert(m1.get_allocator() == A(2));
+    assert(m2.key_comp() == C(1));
+    assert(m2.get_allocator() == A(1));
+  }
 #if TEST_STD_VER >= 11
-    {
+  {
     typedef std::map<int, double, std::less<int>, min_allocator<V>> M;
     {
-        M m1;
-        M m2;
-        M m1_save = m1;
-        M m2_save = m2;
-        swap(m1, m2);
-        assert(m1 == m2_save);
-        assert(m2 == m1_save);
+      M m1;
+      M m2;
+      M m1_save = m1;
+      M m2_save = m2;
+      swap(m1, m2);
+      assert(m1 == m2_save);
+      assert(m2 == m1_save);
     }
     {
-        V ar2[] =
-        {
-            V(5, 5),
-            V(6, 6),
-            V(7, 7),
-            V(8, 8),
-            V(9, 9),
-            V(10, 10),
-            V(11, 11),
-            V(12, 12)
-        };
-        M m1;
-        M m2(ar2, ar2+sizeof(ar2)/sizeof(ar2[0]));
-        M m1_save = m1;
-        M m2_save = m2;
-        swap(m1, m2);
-        assert(m1 == m2_save);
-        assert(m2 == m1_save);
+      V ar2[] = {V(5, 5), V(6, 6), V(7, 7), V(8, 8), V(9, 9), V(10, 10), V(11, 11), V(12, 12)};
+      M m1;
+      M m2(ar2, ar2 + sizeof(ar2) / sizeof(ar2[0]));
+      M m1_save = m1;
+      M m2_save = m2;
+      swap(m1, m2);
+      assert(m1 == m2_save);
+      assert(m2 == m1_save);
     }
     {
-        V ar1[] =
-        {
-            V(1, 1),
-            V(2, 2),
-            V(3, 3),
-            V(4, 4)
-        };
-        M m1(ar1, ar1+sizeof(ar1)/sizeof(ar1[0]));
-        M m2;
-        M m1_save = m1;
-        M m2_save = m2;
-        swap(m1, m2);
-        assert(m1 == m2_save);
-        assert(m2 == m1_save);
-    }
-    {
-        V ar1[] =
-        {
-            V(1, 1),
-            V(2, 2),
-            V(3, 3),
-            V(4, 4)
-        };
-        V ar2[] =
-        {
-            V(5, 5),
-            V(6, 6),
-            V(7, 7),
-            V(8, 8),
-            V(9, 9),
-            V(10, 10),
-            V(11, 11),
-            V(12, 12)
-        };
-        M m1(ar1, ar1+sizeof(ar1)/sizeof(ar1[0]));
-        M m2(ar2, ar2+sizeof(ar2)/sizeof(ar2[0]));
-        M m1_save = m1;
-        M m2_save = m2;
-        swap(m1, m2);
-        assert(m1 == m2_save);
-        assert(m2 == m1_save);
-    }
+      V ar1[] = {V(1, 1), V(2, 2), V(3, 3), V(4, 4)};
+      M m1(ar1, ar1 + sizeof(ar1) / sizeof(ar1[0]));
+      M m2;
+      M m1_save = m1;
+      M m2_save = m2;
+      swap(m1, m2);
+      assert(m1 == m2_save);
+      assert(m2 == m1_save);
     }
     {
-        typedef min_allocator<V> A;
-        typedef test_less<int> C;
-        typedef std::map<int, double, C, A> M;
-        V ar1[] =
-        {
-            V(1, 1),
-            V(2, 2),
-            V(3, 3),
-            V(4, 4)
-        };
-        V ar2[] =
-        {
-            V(5, 5),
-            V(6, 6),
-            V(7, 7),
-            V(8, 8),
-            V(9, 9),
-            V(10, 10),
-            V(11, 11),
-            V(12, 12)
-        };
-        M m1(ar1, ar1+sizeof(ar1)/sizeof(ar1[0]), C(1), A());
-        M m2(ar2, ar2+sizeof(ar2)/sizeof(ar2[0]), C(2), A());
-        M m1_save = m1;
-        M m2_save = m2;
-        swap(m1, m2);
-        assert(m1 == m2_save);
-        assert(m2 == m1_save);
-        assert(m1.key_comp() == C(2));
-        assert(m1.get_allocator() == A());
-        assert(m2.key_comp() == C(1));
-        assert(m2.get_allocator() == A());
+      V ar1[] = {V(1, 1), V(2, 2), V(3, 3), V(4, 4)};
+      V ar2[] = {V(5, 5), V(6, 6), V(7, 7), V(8, 8), V(9, 9), V(10, 10), V(11, 11), V(12, 12)};
+      M m1(ar1, ar1 + sizeof(ar1) / sizeof(ar1[0]));
+      M m2(ar2, ar2 + sizeof(ar2) / sizeof(ar2[0]));
+      M m1_save = m1;
+      M m2_save = m2;
+      swap(m1, m2);
+      assert(m1 == m2_save);
+      assert(m2 == m1_save);
     }
+  }
+  {
+    typedef min_allocator<V> A;
+    typedef test_less<int> C;
+    typedef std::map<int, double, C, A> M;
+    V ar1[] = {V(1, 1), V(2, 2), V(3, 3), V(4, 4)};
+    V ar2[] = {V(5, 5), V(6, 6), V(7, 7), V(8, 8), V(9, 9), V(10, 10), V(11, 11), V(12, 12)};
+    M m1(ar1, ar1 + sizeof(ar1) / sizeof(ar1[0]), C(1), A());
+    M m2(ar2, ar2 + sizeof(ar2) / sizeof(ar2[0]), C(2), A());
+    M m1_save = m1;
+    M m2_save = m2;
+    swap(m1, m2);
+    assert(m1 == m2_save);
+    assert(m2 == m1_save);
+    assert(m1.key_comp() == C(2));
+    assert(m1.get_allocator() == A());
+    assert(m2.key_comp() == C(1));
+    assert(m2.get_allocator() == A());
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/associative/map/map.special/swap_noexcept.pass.cpp b/libcxx/test/std/containers/associative/map/map.special/swap_noexcept.pass.cpp
index 96f96f548a357..118158fd45ec3 100644
--- a/libcxx/test/std/containers/associative/map/map.special/swap_noexcept.pass.cpp
+++ b/libcxx/test/std/containers/associative/map/map.special/swap_noexcept.pass.cpp
@@ -29,23 +29,21 @@
 #include "test_allocator.h"
 
 template <class T>
-struct some_comp
-{
-    typedef T value_type;
+struct some_comp {
+  typedef T value_type;
 
-    some_comp() {}
-    some_comp(const some_comp&) {}
-    bool operator()(const T&, const T&) const { return false; }
+  some_comp() {}
+  some_comp(const some_comp&) {}
+  bool operator()(const T&, const T&) const { return false; }
 };
 
 template <class T>
-struct some_comp2
-{
-    typedef T value_type;
+struct some_comp2 {
+  typedef T value_type;
 
-    some_comp2() {}
-    some_comp2(const some_comp2&) {}
-    bool operator()(const T&, const T&) const { return false; }
+  some_comp2() {}
+  some_comp2(const some_comp2&) {}
+  bool operator()(const T&, const T&) const { return false; }
 };
 
 #if TEST_STD_VER >= 14
@@ -54,90 +52,85 @@ void swap(some_comp2<T>&, some_comp2<T>&) noexcept {}
 #endif
 
 template <class T>
-struct some_alloc
-{
-    typedef T value_type;
+struct some_alloc {
+  typedef T value_type;
 
-    some_alloc() {}
-    some_alloc(const some_alloc&);
-    void deallocate(void*, unsigned) {}
+  some_alloc() {}
+  some_alloc(const some_alloc&);
+  void deallocate(void*, unsigned) {}
 
-    typedef std::true_type propagate_on_container_swap;
+  typedef std::true_type propagate_on_container_swap;
 };
 
 template <class T>
-struct some_alloc2
-{
-    typedef T value_type;
+struct some_alloc2 {
+  typedef T value_type;
 
-    some_alloc2() {}
-    some_alloc2(const some_alloc2&);
-    void deallocate(void*, unsigned) {}
+  some_alloc2() {}
+  some_alloc2(const some_alloc2&);
+  void deallocate(void*, unsigned) {}
 
-    typedef std::false_type propagate_on_container_swap;
-    typedef std::true_type is_always_equal;
+  typedef std::false_type propagate_on_container_swap;
+  typedef std::true_type is_always_equal;
 };
 
 template <class T>
-struct some_alloc3
-{
-    typedef T value_type;
+struct some_alloc3 {
+  typedef T value_type;
 
-    some_alloc3() {}
-    some_alloc3(const some_alloc3&);
-    void deallocate(void*, unsigned) {}
+  some_alloc3() {}
+  some_alloc3(const some_alloc3&);
+  void deallocate(void*, unsigned) {}
 
-    typedef std::false_type propagate_on_container_swap;
-    typedef std::false_type is_always_equal;
+  typedef std::false_type propagate_on_container_swap;
+  typedef std::false_type is_always_equal;
 };
 
-int main(int, char**)
-{
-    typedef std::pair<const MoveOnly, MoveOnly> V;
-    {
-        typedef std::map<MoveOnly, MoveOnly> C;
-        static_assert(noexcept(swap(std::declval<C&>(), std::declval<C&>())), "");
-    }
+int main(int, char**) {
+  typedef std::pair<const MoveOnly, MoveOnly> V;
+  {
+    typedef std::map<MoveOnly, MoveOnly> C;
+    static_assert(noexcept(swap(std::declval<C&>(), std::declval<C&>())), "");
+  }
 #if defined(_LIBCPP_VERSION)
-    {
-        typedef std::map<MoveOnly, MoveOnly, std::less<MoveOnly>, test_allocator<V>> C;
-        static_assert(noexcept(swap(std::declval<C&>(), std::declval<C&>())), "");
-    }
-    {
-        typedef std::map<MoveOnly, MoveOnly, std::less<MoveOnly>, other_allocator<V>> C;
-        static_assert(noexcept(swap(std::declval<C&>(), std::declval<C&>())), "");
-    }
+  {
+    typedef std::map<MoveOnly, MoveOnly, std::less<MoveOnly>, test_allocator<V>> C;
+    static_assert(noexcept(swap(std::declval<C&>(), std::declval<C&>())), "");
+  }
+  {
+    typedef std::map<MoveOnly, MoveOnly, std::less<MoveOnly>, other_allocator<V>> C;
+    static_assert(noexcept(swap(std::declval<C&>(), std::declval<C&>())), "");
+  }
 #endif // _LIBCPP_VERSION
-    {
-        typedef std::map<MoveOnly, MoveOnly, some_comp<MoveOnly>> C;
-        static_assert(!noexcept(swap(std::declval<C&>(), std::declval<C&>())), "");
-    }
+  {
+    typedef std::map<MoveOnly, MoveOnly, some_comp<MoveOnly>> C;
+    static_assert(!noexcept(swap(std::declval<C&>(), std::declval<C&>())), "");
+  }
 
 #if TEST_STD_VER >= 14
-    { // POCS allocator, throwable swap for comp
-    typedef std::map<MoveOnly, MoveOnly, some_comp <MoveOnly>, some_alloc <V>> C;
+  { // POCS allocator, throwable swap for comp
+    typedef std::map<MoveOnly, MoveOnly, some_comp<MoveOnly>, some_alloc<V>> C;
     static_assert(!noexcept(swap(std::declval<C&>(), std::declval<C&>())), "");
-    }
-    { // always equal allocator, throwable swap for comp
-    typedef std::map<MoveOnly, MoveOnly, some_comp <MoveOnly>, some_alloc2<V>> C;
+  }
+  { // always equal allocator, throwable swap for comp
+    typedef std::map<MoveOnly, MoveOnly, some_comp<MoveOnly>, some_alloc2<V>> C;
     static_assert(!noexcept(swap(std::declval<C&>(), std::declval<C&>())), "");
-    }
-    { // POCS allocator, nothrow swap for comp
-    typedef std::map<MoveOnly, MoveOnly, some_comp2<MoveOnly>, some_alloc <V>> C;
-    static_assert( noexcept(swap(std::declval<C&>(), std::declval<C&>())), "");
-    }
-    { // always equal allocator, nothrow swap for comp
+  }
+  { // POCS allocator, nothrow swap for comp
+    typedef std::map<MoveOnly, MoveOnly, some_comp2<MoveOnly>, some_alloc<V>> C;
+    static_assert(noexcept(swap(std::declval<C&>(), std::declval<C&>())), "");
+  }
+  { // always equal allocator, nothrow swap for comp
     typedef std::map<MoveOnly, MoveOnly, some_comp2<MoveOnly>, some_alloc2<V>> C;
-    static_assert( noexcept(swap(std::declval<C&>(), std::declval<C&>())), "");
-    }
-#if defined(_LIBCPP_VERSION)
-    { // NOT always equal allocator, nothrow swap for comp
+    static_assert(noexcept(swap(std::declval<C&>(), std::declval<C&>())), "");
+  }
+#  if defined(_LIBCPP_VERSION)
+  { // NOT always equal allocator, nothrow swap for comp
     typedef std::map<MoveOnly, MoveOnly, some_comp2<MoveOnly>, some_alloc3<V>> C;
-    static_assert( noexcept(swap(std::declval<C&>(), std::declval<C&>())), "");
-    }
-#endif // _LIBCPP_VERSION
+    static_assert(noexcept(swap(std::declval<C&>(), std::declval<C&>())), "");
+  }
+#  endif // _LIBCPP_VERSION
 #endif
 
-
   return 0;
 }
diff --git a/libcxx/test/std/containers/associative/map/map.value_compare/invoke.pass.cpp b/libcxx/test/std/containers/associative/map/map.value_compare/invoke.pass.cpp
index 2cfbb2ad5c2b2..70fa972812cec 100644
--- a/libcxx/test/std/containers/associative/map/map.value_compare/invoke.pass.cpp
+++ b/libcxx/test/std/containers/associative/map/map.value_compare/invoke.pass.cpp
@@ -19,29 +19,29 @@
 
 template <typename Map>
 struct CallCompMember : Map::value_compare {
-    CallCompMember(const typename Map::value_compare& vc) : Map::value_compare(vc) {}
+  CallCompMember(const typename Map::value_compare& vc) : Map::value_compare(vc) {}
 
-    typedef typename Map::value_type value_type;
-    bool operator()(const value_type& value1, const value_type& value2) const {
-        return this->comp(value1.first, value2.first);
-    }
+  typedef typename Map::value_type value_type;
+  bool operator()(const value_type& value1, const value_type& value2) const {
+    return this->comp(value1.first, value2.first);
+  }
 };
 
 int main(int, char**) {
-    typedef std::map<int, std::string> map_type;
+  typedef std::map<int, std::string> map_type;
 
-    map_type m;
-    std::pair<map_type::iterator, bool> p1 = m.insert(map_type::value_type(1, "abc"));
-    std::pair<map_type::iterator, bool> p2 = m.insert(map_type::value_type(2, "abc"));
+  map_type m;
+  std::pair<map_type::iterator, bool> p1 = m.insert(map_type::value_type(1, "abc"));
+  std::pair<map_type::iterator, bool> p2 = m.insert(map_type::value_type(2, "abc"));
 
-    const map_type::value_compare vc = m.value_comp();
-    CallCompMember<map_type> call_comp = m.value_comp();
+  const map_type::value_compare vc   = m.value_comp();
+  CallCompMember<map_type> call_comp = m.value_comp();
 
-    assert(vc(*p1.first, *p2.first));
-    assert(call_comp(*p1.first, *p2.first));
+  assert(vc(*p1.first, *p2.first));
+  assert(call_comp(*p1.first, *p2.first));
 
-    assert(!vc(*p2.first, *p1.first));
-    assert(!call_comp(*p2.first, *p1.first));
+  assert(!vc(*p2.first, *p1.first));
+  assert(!call_comp(*p2.first, *p1.first));
 
-    return 0;
+  return 0;
 }
diff --git a/libcxx/test/std/containers/associative/map/map.value_compare/types.pass.cpp b/libcxx/test/std/containers/associative/map/map.value_compare/types.pass.cpp
index 52111ddb3f272..89881baf88dca 100644
--- a/libcxx/test/std/containers/associative/map/map.value_compare/types.pass.cpp
+++ b/libcxx/test/std/containers/associative/map/map.value_compare/types.pass.cpp
@@ -18,13 +18,13 @@
 #include "test_macros.h"
 
 int main(int, char**) {
-    typedef std::map<int, std::string> map_type;
-    typedef map_type::value_compare value_compare;
-    typedef map_type::value_type value_type;
+  typedef std::map<int, std::string> map_type;
+  typedef map_type::value_compare value_compare;
+  typedef map_type::value_type value_type;
 
-    ASSERT_SAME_TYPE(value_compare::result_type, bool);
-    ASSERT_SAME_TYPE(value_compare::first_argument_type, value_type);
-    ASSERT_SAME_TYPE(value_compare::second_argument_type, value_type);
+  ASSERT_SAME_TYPE(value_compare::result_type, bool);
+  ASSERT_SAME_TYPE(value_compare::first_argument_type, value_type);
+  ASSERT_SAME_TYPE(value_compare::second_argument_type, value_type);
 
-    return 0;
+  return 0;
 }
diff --git a/libcxx/test/std/containers/associative/map/range_concept_conformance.compile.pass.cpp b/libcxx/test/std/containers/associative/map/range_concept_conformance.compile.pass.cpp
index 4c018ec1b3907..4e4f154adaffb 100644
--- a/libcxx/test/std/containers/associative/map/range_concept_conformance.compile.pass.cpp
+++ b/libcxx/test/std/containers/associative/map/range_concept_conformance.compile.pass.cpp
@@ -17,7 +17,6 @@
 
 using range = std::map<int, int>;
 
-
 static_assert(std::same_as<std::ranges::iterator_t<range>, range::iterator>);
 static_assert(std::ranges::common_range<range>);
 static_assert(std::ranges::bidirectional_range<range>);
diff --git a/libcxx/test/std/containers/associative/map/types.pass.cpp b/libcxx/test/std/containers/associative/map/types.pass.cpp
index 0cfb7c00e813c..7832ff3efd3a4 100644
--- a/libcxx/test/std/containers/associative/map/types.pass.cpp
+++ b/libcxx/test/std/containers/associative/map/types.pass.cpp
@@ -34,9 +34,8 @@
 #include "test_macros.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
+int main(int, char**) {
+  {
     typedef std::map<int, double> C;
     static_assert((std::is_same<C::key_type, int>::value), "");
     static_assert((std::is_same<C::mapped_type, double>::value), "");
@@ -49,9 +48,9 @@ int main(int, char**)
     static_assert((std::is_same<C::const_pointer, const std::pair<const int, double>*>::value), "");
     static_assert((std::is_same<C::size_type, std::size_t>::value), "");
     static_assert((std::is_same<C::difference_type, std::ptrdiff_t>::value), "");
-    }
+  }
 #if TEST_STD_VER >= 11
-    {
+  {
     typedef std::map<int, double, std::less<int>, min_allocator<std::pair<const int, double>>> C;
     static_assert((std::is_same<C::key_type, int>::value), "");
     static_assert((std::is_same<C::mapped_type, double>::value), "");
@@ -62,10 +61,10 @@ int main(int, char**)
     static_assert((std::is_same<C::const_reference, const std::pair<const int, double>&>::value), "");
     static_assert((std::is_same<C::pointer, min_pointer<std::pair<const int, double>>>::value), "");
     static_assert((std::is_same<C::const_pointer, min_pointer<const std::pair<const int, double>>>::value), "");
-//  min_allocator doesn't have a size_type, so one gets synthesized
+    //  min_allocator doesn't have a size_type, so one gets synthesized
     static_assert((std::is_same<C::size_type, std::make_unsigned<C::difference_type>::type>::value), "");
     static_assert((std::is_same<C::difference_type, std::ptrdiff_t>::value), "");
-    }
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/associative/multimap/allocator_mismatch.verify.cpp b/libcxx/test/std/containers/associative/multimap/allocator_mismatch.verify.cpp
index 6fa5f241bb5b8..319f32ddeea26 100644
--- a/libcxx/test/std/containers/associative/multimap/allocator_mismatch.verify.cpp
+++ b/libcxx/test/std/containers/associative/multimap/allocator_mismatch.verify.cpp
@@ -12,4 +12,4 @@
 #include <map>
 
 std::multimap<int, int, std::less<int>, std::allocator<long> > m;
-  // expected-error-re@*:* {{static assertion failed{{.*}}Allocator::value_type must be same type as value_type}}
+// expected-error-re@*:* {{static assertion failed{{.*}}Allocator::value_type must be same type as value_type}}
diff --git a/libcxx/test/std/containers/associative/multimap/empty.pass.cpp b/libcxx/test/std/containers/associative/multimap/empty.pass.cpp
index 45a1d76bd83fa..c183cc385a916 100644
--- a/libcxx/test/std/containers/associative/multimap/empty.pass.cpp
+++ b/libcxx/test/std/containers/associative/multimap/empty.pass.cpp
@@ -18,9 +18,8 @@
 #include "test_macros.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
+int main(int, char**) {
+  {
     typedef std::multimap<int, double> M;
     M m;
     assert(m.empty());
@@ -28,9 +27,9 @@ int main(int, char**)
     assert(!m.empty());
     m.clear();
     assert(m.empty());
-    }
+  }
 #if TEST_STD_VER >= 11
-    {
+  {
     typedef std::multimap<int, double, std::less<int>, min_allocator<std::pair<const int, double>>> M;
     M m;
     assert(m.empty());
@@ -38,7 +37,7 @@ int main(int, char**)
     assert(!m.empty());
     m.clear();
     assert(m.empty());
-    }
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/associative/multimap/empty.verify.cpp b/libcxx/test/std/containers/associative/multimap/empty.verify.cpp
index 8720497c35fdf..2c23a9f38f18b 100644
--- a/libcxx/test/std/containers/associative/multimap/empty.verify.cpp
+++ b/libcxx/test/std/containers/associative/multimap/empty.verify.cpp
@@ -17,6 +17,6 @@
 #include <map>
 
 void f() {
-    std::multimap<int, int> c;
-    c.empty(); // expected-warning {{ignoring return value of function declared with 'nodiscard' attribute}}
+  std::multimap<int, int> c;
+  c.empty(); // expected-warning {{ignoring return value of function declared with 'nodiscard' attribute}}
 }
diff --git a/libcxx/test/std/containers/associative/multimap/get_allocator.pass.cpp b/libcxx/test/std/containers/associative/multimap/get_allocator.pass.cpp
index 41dfaa77c6603..102dd0b5a36cc 100644
--- a/libcxx/test/std/containers/associative/multimap/get_allocator.pass.cpp
+++ b/libcxx/test/std/containers/associative/multimap/get_allocator.pass.cpp
@@ -20,18 +20,17 @@
 #include "test_macros.h"
 
 int main(int, char**) {
-    typedef std::pair<const int, std::string> ValueType;
-    {
-        std::allocator<ValueType> alloc;
-        const std::multimap<int, std::string> m(alloc);
-        assert(m.get_allocator() == alloc);
-    }
-    {
-        other_allocator<ValueType> alloc(1);
-        const std::multimap<int, std::string, std::less<int>,
-                            other_allocator<ValueType> > m(alloc);
-        assert(m.get_allocator() == alloc);
-    }
+  typedef std::pair<const int, std::string> ValueType;
+  {
+    std::allocator<ValueType> alloc;
+    const std::multimap<int, std::string> m(alloc);
+    assert(m.get_allocator() == alloc);
+  }
+  {
+    other_allocator<ValueType> alloc(1);
+    const std::multimap<int, std::string, std::less<int>, other_allocator<ValueType> > m(alloc);
+    assert(m.get_allocator() == alloc);
+  }
 
-    return 0;
+  return 0;
 }
diff --git a/libcxx/test/std/containers/associative/multimap/incomplete_type.pass.cpp b/libcxx/test/std/containers/associative/multimap/incomplete_type.pass.cpp
index 6e7c8e770202e..05b8abf90e2ec 100644
--- a/libcxx/test/std/containers/associative/multimap/incomplete_type.pass.cpp
+++ b/libcxx/test/std/containers/associative/multimap/incomplete_type.pass.cpp
@@ -16,17 +16,17 @@
 #include "test_macros.h"
 
 struct A {
-    typedef std::multimap<A, A> Map;
-    int data;
-    Map m;
-    Map::iterator it;
-    Map::const_iterator cit;
+  typedef std::multimap<A, A> Map;
+  int data;
+  Map m;
+  Map::iterator it;
+  Map::const_iterator cit;
 };
 
 inline bool operator==(A const& L, A const& R) { return &L == &R; }
-inline bool operator<(A const& L, A const& R)  { return L.data < R.data; }
+inline bool operator<(A const& L, A const& R) { return L.data < R.data; }
 int main(int, char**) {
-    A a;
+  A a;
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/associative/multimap/iterator.pass.cpp b/libcxx/test/std/containers/associative/multimap/iterator.pass.cpp
index b12645a3eb1af..ffdc39ff35563 100644
--- a/libcxx/test/std/containers/associative/multimap/iterator.pass.cpp
+++ b/libcxx/test/std/containers/associative/multimap/iterator.pass.cpp
@@ -32,242 +32,137 @@
 #include "test_macros.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef std::pair<const int, double> V;
-        V ar[] =
-        {
-            V(1, 1),
-            V(1, 1.5),
-            V(1, 2),
-            V(2, 1),
-            V(2, 1.5),
-            V(2, 2),
-            V(3, 1),
-            V(3, 1.5),
-            V(3, 2),
-            V(4, 1),
-            V(4, 1.5),
-            V(4, 2),
-            V(5, 1),
-            V(5, 1.5),
-            V(5, 2),
-            V(6, 1),
-            V(6, 1.5),
-            V(6, 2),
-            V(7, 1),
-            V(7, 1.5),
-            V(7, 2),
-            V(8, 1),
-            V(8, 1.5),
-            V(8, 2)
-        };
-        std::multimap<int, double> m(ar, ar+sizeof(ar)/sizeof(ar[0]));
-        assert(static_cast<std::size_t>(std::distance(m.begin(), m.end())) == m.size());
-        assert(static_cast<std::size_t>(std::distance(m.rbegin(), m.rend())) == m.size());
-        std::multimap<int, double>::iterator i;
-        i = m.begin();
-        std::multimap<int, double>::const_iterator k = i;
-        assert(i == k);
-        for (int j = 1; j <= 8; ++j)
-            for (double d = 1; d <= 2; d += .5, ++i)
-            {
-                assert(i->first == j);
-                assert(i->second == d);
-                i->second = 2.5;
-                assert(i->second == 2.5);
-            }
-        assert(i == m.end());
-        for (int j = 8; j >= 1; --j)
-            for (double d = 1; d <= 2; d += .5)
-            {
-                --i;
-                assert(i->first == j);
-                assert(i->second == 2.5);
-                i->second = d;
-                assert(i->second == d);
-            }
-        assert(i == m.begin());
-    }
-    {
-        typedef std::pair<const int, double> V;
-        V ar[] =
-        {
-            V(1, 1),
-            V(1, 1.5),
-            V(1, 2),
-            V(2, 1),
-            V(2, 1.5),
-            V(2, 2),
-            V(3, 1),
-            V(3, 1.5),
-            V(3, 2),
-            V(4, 1),
-            V(4, 1.5),
-            V(4, 2),
-            V(5, 1),
-            V(5, 1.5),
-            V(5, 2),
-            V(6, 1),
-            V(6, 1.5),
-            V(6, 2),
-            V(7, 1),
-            V(7, 1.5),
-            V(7, 2),
-            V(8, 1),
-            V(8, 1.5),
-            V(8, 2)
-        };
-        const std::multimap<int, double> m(ar, ar+sizeof(ar)/sizeof(ar[0]));
-        assert(static_cast<std::size_t>(std::distance(m.begin(), m.end())) == m.size());
-        assert(static_cast<std::size_t>(std::distance(m.cbegin(), m.cend())) == m.size());
-        assert(static_cast<std::size_t>(std::distance(m.rbegin(), m.rend())) == m.size());
-        assert(static_cast<std::size_t>(std::distance(m.crbegin(), m.crend())) == m.size());
-        std::multimap<int, double>::const_iterator i;
-        i = m.begin();
-        for (int j = 1; j <= 8; ++j)
-            for (double d = 1; d <= 2; d += .5, ++i)
-            {
-                assert(i->first == j);
-                assert(i->second == d);
-            }
-        assert(i == m.end());
-        for (int j = 8; j >= 1; --j)
-            for (double d = 2; d >= 1; d -= .5)
-            {
-                --i;
-                assert(i->first == j);
-                assert(i->second == d);
-            }
-        assert(i == m.begin());
-    }
+int main(int, char**) {
+  {
+    typedef std::pair<const int, double> V;
+    V ar[] = {V(1, 1),   V(1, 1.5), V(1, 2),   V(2, 1),   V(2, 1.5), V(2, 2),   V(3, 1),   V(3, 1.5),
+              V(3, 2),   V(4, 1),   V(4, 1.5), V(4, 2),   V(5, 1),   V(5, 1.5), V(5, 2),   V(6, 1),
+              V(6, 1.5), V(6, 2),   V(7, 1),   V(7, 1.5), V(7, 2),   V(8, 1),   V(8, 1.5), V(8, 2)};
+    std::multimap<int, double> m(ar, ar + sizeof(ar) / sizeof(ar[0]));
+    assert(static_cast<std::size_t>(std::distance(m.begin(), m.end())) == m.size());
+    assert(static_cast<std::size_t>(std::distance(m.rbegin(), m.rend())) == m.size());
+    std::multimap<int, double>::iterator i;
+    i                                            = m.begin();
+    std::multimap<int, double>::const_iterator k = i;
+    assert(i == k);
+    for (int j = 1; j <= 8; ++j)
+      for (double d = 1; d <= 2; d += .5, ++i) {
+        assert(i->first == j);
+        assert(i->second == d);
+        i->second = 2.5;
+        assert(i->second == 2.5);
+      }
+    assert(i == m.end());
+    for (int j = 8; j >= 1; --j)
+      for (double d = 1; d <= 2; d += .5) {
+        --i;
+        assert(i->first == j);
+        assert(i->second == 2.5);
+        i->second = d;
+        assert(i->second == d);
+      }
+    assert(i == m.begin());
+  }
+  {
+    typedef std::pair<const int, double> V;
+    V ar[] = {V(1, 1),   V(1, 1.5), V(1, 2),   V(2, 1),   V(2, 1.5), V(2, 2),   V(3, 1),   V(3, 1.5),
+              V(3, 2),   V(4, 1),   V(4, 1.5), V(4, 2),   V(5, 1),   V(5, 1.5), V(5, 2),   V(6, 1),
+              V(6, 1.5), V(6, 2),   V(7, 1),   V(7, 1.5), V(7, 2),   V(8, 1),   V(8, 1.5), V(8, 2)};
+    const std::multimap<int, double> m(ar, ar + sizeof(ar) / sizeof(ar[0]));
+    assert(static_cast<std::size_t>(std::distance(m.begin(), m.end())) == m.size());
+    assert(static_cast<std::size_t>(std::distance(m.cbegin(), m.cend())) == m.size());
+    assert(static_cast<std::size_t>(std::distance(m.rbegin(), m.rend())) == m.size());
+    assert(static_cast<std::size_t>(std::distance(m.crbegin(), m.crend())) == m.size());
+    std::multimap<int, double>::const_iterator i;
+    i = m.begin();
+    for (int j = 1; j <= 8; ++j)
+      for (double d = 1; d <= 2; d += .5, ++i) {
+        assert(i->first == j);
+        assert(i->second == d);
+      }
+    assert(i == m.end());
+    for (int j = 8; j >= 1; --j)
+      for (double d = 2; d >= 1; d -= .5) {
+        --i;
+        assert(i->first == j);
+        assert(i->second == d);
+      }
+    assert(i == m.begin());
+  }
 #if TEST_STD_VER >= 11
-    {
-        typedef std::pair<const int, double> V;
-        V ar[] =
-        {
-            V(1, 1),
-            V(1, 1.5),
-            V(1, 2),
-            V(2, 1),
-            V(2, 1.5),
-            V(2, 2),
-            V(3, 1),
-            V(3, 1.5),
-            V(3, 2),
-            V(4, 1),
-            V(4, 1.5),
-            V(4, 2),
-            V(5, 1),
-            V(5, 1.5),
-            V(5, 2),
-            V(6, 1),
-            V(6, 1.5),
-            V(6, 2),
-            V(7, 1),
-            V(7, 1.5),
-            V(7, 2),
-            V(8, 1),
-            V(8, 1.5),
-            V(8, 2)
-        };
-        std::multimap<int, double, std::less<int>, min_allocator<V>> m(ar, ar+sizeof(ar)/sizeof(ar[0]));
-        assert(static_cast<std::size_t>(std::distance(m.begin(), m.end())) == m.size());
-        assert(static_cast<std::size_t>(std::distance(m.rbegin(), m.rend())) == m.size());
-        std::multimap<int, double, std::less<int>, min_allocator<V>>::iterator i;
-        i = m.begin();
-        std::multimap<int, double, std::less<int>, min_allocator<V>>::const_iterator k = i;
-        assert(i == k);
-        for (int j = 1; j <= 8; ++j)
-            for (double d = 1; d <= 2; d += .5, ++i)
-            {
-                assert(i->first == j);
-                assert(i->second == d);
-                i->second = 2.5;
-                assert(i->second == 2.5);
-            }
-        assert(i == m.end());
-        for (int j = 8; j >= 1; --j)
-            for (double d = 1; d <= 2; d += .5)
-            {
-                --i;
-                assert(i->first == j);
-                assert(i->second == 2.5);
-                i->second = d;
-                assert(i->second == d);
-            }
-        assert(i == m.begin());
-    }
-    {
-        typedef std::pair<const int, double> V;
-        V ar[] =
-        {
-            V(1, 1),
-            V(1, 1.5),
-            V(1, 2),
-            V(2, 1),
-            V(2, 1.5),
-            V(2, 2),
-            V(3, 1),
-            V(3, 1.5),
-            V(3, 2),
-            V(4, 1),
-            V(4, 1.5),
-            V(4, 2),
-            V(5, 1),
-            V(5, 1.5),
-            V(5, 2),
-            V(6, 1),
-            V(6, 1.5),
-            V(6, 2),
-            V(7, 1),
-            V(7, 1.5),
-            V(7, 2),
-            V(8, 1),
-            V(8, 1.5),
-            V(8, 2)
-        };
-        const std::multimap<int, double, std::less<int>, min_allocator<V>> m(ar, ar+sizeof(ar)/sizeof(ar[0]));
-        assert(static_cast<std::size_t>(std::distance(m.begin(), m.end())) == m.size());
-        assert(static_cast<std::size_t>(std::distance(m.cbegin(), m.cend())) == m.size());
-        assert(static_cast<std::size_t>(std::distance(m.rbegin(), m.rend())) == m.size());
-        assert(static_cast<std::size_t>(std::distance(m.crbegin(), m.crend())) == m.size());
-        std::multimap<int, double, std::less<int>, min_allocator<V>>::const_iterator i;
-        i = m.begin();
-        for (int j = 1; j <= 8; ++j)
-            for (double d = 1; d <= 2; d += .5, ++i)
-            {
-                assert(i->first == j);
-                assert(i->second == d);
-            }
-        assert(i == m.end());
-        for (int j = 8; j >= 1; --j)
-            for (double d = 2; d >= 1; d -= .5)
-            {
-                --i;
-                assert(i->first == j);
-                assert(i->second == d);
-            }
-        assert(i == m.begin());
-    }
+  {
+    typedef std::pair<const int, double> V;
+    V ar[] = {V(1, 1),   V(1, 1.5), V(1, 2),   V(2, 1),   V(2, 1.5), V(2, 2),   V(3, 1),   V(3, 1.5),
+              V(3, 2),   V(4, 1),   V(4, 1.5), V(4, 2),   V(5, 1),   V(5, 1.5), V(5, 2),   V(6, 1),
+              V(6, 1.5), V(6, 2),   V(7, 1),   V(7, 1.5), V(7, 2),   V(8, 1),   V(8, 1.5), V(8, 2)};
+    std::multimap<int, double, std::less<int>, min_allocator<V>> m(ar, ar + sizeof(ar) / sizeof(ar[0]));
+    assert(static_cast<std::size_t>(std::distance(m.begin(), m.end())) == m.size());
+    assert(static_cast<std::size_t>(std::distance(m.rbegin(), m.rend())) == m.size());
+    std::multimap<int, double, std::less<int>, min_allocator<V>>::iterator i;
+    i                                                                              = m.begin();
+    std::multimap<int, double, std::less<int>, min_allocator<V>>::const_iterator k = i;
+    assert(i == k);
+    for (int j = 1; j <= 8; ++j)
+      for (double d = 1; d <= 2; d += .5, ++i) {
+        assert(i->first == j);
+        assert(i->second == d);
+        i->second = 2.5;
+        assert(i->second == 2.5);
+      }
+    assert(i == m.end());
+    for (int j = 8; j >= 1; --j)
+      for (double d = 1; d <= 2; d += .5) {
+        --i;
+        assert(i->first == j);
+        assert(i->second == 2.5);
+        i->second = d;
+        assert(i->second == d);
+      }
+    assert(i == m.begin());
+  }
+  {
+    typedef std::pair<const int, double> V;
+    V ar[] = {V(1, 1),   V(1, 1.5), V(1, 2),   V(2, 1),   V(2, 1.5), V(2, 2),   V(3, 1),   V(3, 1.5),
+              V(3, 2),   V(4, 1),   V(4, 1.5), V(4, 2),   V(5, 1),   V(5, 1.5), V(5, 2),   V(6, 1),
+              V(6, 1.5), V(6, 2),   V(7, 1),   V(7, 1.5), V(7, 2),   V(8, 1),   V(8, 1.5), V(8, 2)};
+    const std::multimap<int, double, std::less<int>, min_allocator<V>> m(ar, ar + sizeof(ar) / sizeof(ar[0]));
+    assert(static_cast<std::size_t>(std::distance(m.begin(), m.end())) == m.size());
+    assert(static_cast<std::size_t>(std::distance(m.cbegin(), m.cend())) == m.size());
+    assert(static_cast<std::size_t>(std::distance(m.rbegin(), m.rend())) == m.size());
+    assert(static_cast<std::size_t>(std::distance(m.crbegin(), m.crend())) == m.size());
+    std::multimap<int, double, std::less<int>, min_allocator<V>>::const_iterator i;
+    i = m.begin();
+    for (int j = 1; j <= 8; ++j)
+      for (double d = 1; d <= 2; d += .5, ++i) {
+        assert(i->first == j);
+        assert(i->second == d);
+      }
+    assert(i == m.end());
+    for (int j = 8; j >= 1; --j)
+      for (double d = 2; d >= 1; d -= .5) {
+        --i;
+        assert(i->first == j);
+        assert(i->second == d);
+      }
+    assert(i == m.begin());
+  }
 #endif
 #if TEST_STD_VER > 11
-    { // N3644 testing
-        typedef std::multimap<int, double> C;
-        C::iterator ii1{}, ii2{};
-        C::iterator ii4 = ii1;
-        C::const_iterator cii{};
-        assert ( ii1 == ii2 );
-        assert ( ii1 == ii4 );
+  { // N3644 testing
+    typedef std::multimap<int, double> C;
+    C::iterator ii1{}, ii2{};
+    C::iterator ii4 = ii1;
+    C::const_iterator cii{};
+    assert(ii1 == ii2);
+    assert(ii1 == ii4);
 
-        assert (!(ii1 != ii2 ));
+    assert(!(ii1 != ii2));
 
-        assert ( (ii1 == cii ));
-        assert ( (cii == ii1 ));
-        assert (!(ii1 != cii ));
-        assert (!(cii != ii1 ));
-    }
+    assert((ii1 == cii));
+    assert((cii == ii1));
+    assert(!(ii1 != cii));
+    assert(!(cii != ii1));
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/associative/multimap/iterator_concept_conformance.compile.pass.cpp b/libcxx/test/std/containers/associative/multimap/iterator_concept_conformance.compile.pass.cpp
index bce28fc7fbd39..b5b6a3cbf09da 100644
--- a/libcxx/test/std/containers/associative/multimap/iterator_concept_conformance.compile.pass.cpp
+++ b/libcxx/test/std/containers/associative/multimap/iterator_concept_conformance.compile.pass.cpp
@@ -14,11 +14,11 @@
 
 #include <iterator>
 
-using iterator = std::multimap<int, int>::iterator;
-using const_iterator = std::multimap<int, int>::const_iterator;
-using reverse_iterator = std::multimap<int, int>::reverse_iterator;
+using iterator               = std::multimap<int, int>::iterator;
+using const_iterator         = std::multimap<int, int>::const_iterator;
+using reverse_iterator       = std::multimap<int, int>::reverse_iterator;
 using const_reverse_iterator = std::multimap<int, int>::const_reverse_iterator;
-using value_type = std::pair<const int, int>;
+using value_type             = std::pair<const int, int>;
 
 static_assert(std::bidirectional_iterator<iterator>);
 static_assert(!std::random_access_iterator<iterator>);
@@ -31,9 +31,9 @@ static_assert(!std::sized_sentinel_for<iterator, iterator>);
 static_assert(!std::sized_sentinel_for<iterator, const_iterator>);
 static_assert(!std::sized_sentinel_for<iterator, reverse_iterator>);
 static_assert(!std::sized_sentinel_for<iterator, const_reverse_iterator>);
-static_assert( std::indirectly_movable<iterator, std::pair<int, int>*>);
+static_assert(std::indirectly_movable<iterator, std::pair<int, int>*>);
 static_assert(!std::indirectly_movable_storable<iterator, std::pair<int, int>*>);
-static_assert( std::indirectly_copyable<iterator, std::pair<int, int>*>);
+static_assert(std::indirectly_copyable<iterator, std::pair<int, int>*>);
 static_assert(!std::indirectly_copyable_storable<iterator, std::pair<int, int>*>);
 static_assert(!std::indirectly_swappable<iterator, iterator>);
 
diff --git a/libcxx/test/std/containers/associative/multimap/max_size.pass.cpp b/libcxx/test/std/containers/associative/multimap/max_size.pass.cpp
index f7b8b5e071be9..c6208d27336b5 100644
--- a/libcxx/test/std/containers/associative/multimap/max_size.pass.cpp
+++ b/libcxx/test/std/containers/associative/multimap/max_size.pass.cpp
@@ -20,8 +20,7 @@
 #include "test_allocator.h"
 #include "test_macros.h"
 
-int main(int, char**)
-{
+int main(int, char**) {
   typedef std::pair<const int, int> KV;
   {
     typedef limited_allocator<KV, 10> A;
@@ -33,16 +32,14 @@ int main(int, char**)
   {
     typedef limited_allocator<KV, (std::size_t)-1> A;
     typedef std::multimap<int, int, std::less<int>, A> C;
-    const C::size_type max_dist =
-        static_cast<C::size_type>(std::numeric_limits<C::difference_type>::max());
+    const C::size_type max_dist = static_cast<C::size_type>(std::numeric_limits<C::difference_type>::max());
     C c;
     assert(c.max_size() <= max_dist);
     LIBCPP_ASSERT(c.max_size() == max_dist);
   }
   {
     typedef std::multimap<char, int> C;
-    const C::size_type max_dist =
-        static_cast<C::size_type>(std::numeric_limits<C::difference_type>::max());
+    const C::size_type max_dist = static_cast<C::size_type>(std::numeric_limits<C::difference_type>::max());
     C c;
     assert(c.max_size() <= max_dist);
     assert(c.max_size() <= alloc_max_size(c.get_allocator()));
diff --git a/libcxx/test/std/containers/associative/multimap/multimap.cons/alloc.pass.cpp b/libcxx/test/std/containers/associative/multimap/multimap.cons/alloc.pass.cpp
index 4c16350ecdf5c..10184633a82de 100644
--- a/libcxx/test/std/containers/associative/multimap/multimap.cons/alloc.pass.cpp
+++ b/libcxx/test/std/containers/associative/multimap/multimap.cons/alloc.pass.cpp
@@ -19,33 +19,32 @@
 #include "test_allocator.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
+int main(int, char**) {
+  {
     typedef std::less<int> C;
     typedef test_allocator<std::pair<const int, double> > A;
     std::multimap<int, double, C, A> m(A(5));
     assert(m.empty());
     assert(m.begin() == m.end());
     assert(m.get_allocator() == A(5));
-    }
+  }
 #if TEST_STD_VER >= 11
-    {
+  {
     typedef std::less<int> C;
     typedef min_allocator<std::pair<const int, double> > A;
     std::multimap<int, double, C, A> m(A{});
     assert(m.empty());
     assert(m.begin() == m.end());
     assert(m.get_allocator() == A());
-    }
-    {
+  }
+  {
     typedef std::less<int> C;
     typedef explicit_allocator<std::pair<const int, double> > A;
     std::multimap<int, double, C, A> m(A{});
     assert(m.empty());
     assert(m.begin() == m.end());
     assert(m.get_allocator() == A{});
-    }
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/associative/multimap/multimap.cons/assign_initializer_list.pass.cpp b/libcxx/test/std/containers/associative/multimap/multimap.cons/assign_initializer_list.pass.cpp
index 7a15d780fd6f4..d1de8fab172cf 100644
--- a/libcxx/test/std/containers/associative/multimap/multimap.cons/assign_initializer_list.pass.cpp
+++ b/libcxx/test/std/containers/associative/multimap/multimap.cons/assign_initializer_list.pass.cpp
@@ -20,24 +20,12 @@
 #include "test_macros.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
+int main(int, char**) {
+  {
     typedef std::multimap<int, double> C;
     typedef C::value_type V;
     C m = {{20, 1}};
-    m =
-           {
-               {1, 1},
-               {1, 1.5},
-               {1, 2},
-               {2, 1},
-               {2, 1.5},
-               {2, 2},
-               {3, 1},
-               {3, 1.5},
-               {3, 2}
-           };
+    m   = {{1, 1}, {1, 1.5}, {1, 2}, {2, 1}, {2, 1.5}, {2, 2}, {3, 1}, {3, 1.5}, {3, 2}};
     assert(m.size() == 9);
     assert(std::distance(m.begin(), m.end()) == 9);
     C::const_iterator i = m.cbegin();
@@ -50,23 +38,12 @@ int main(int, char**)
     assert(*++i == V(3, 1));
     assert(*++i == V(3, 1.5));
     assert(*++i == V(3, 2));
-    }
-    {
+  }
+  {
     typedef std::multimap<int, double, std::less<int>, min_allocator<std::pair<const int, double>>> C;
     typedef C::value_type V;
     C m = {{20, 1}};
-    m =
-           {
-               {1, 1},
-               {1, 1.5},
-               {1, 2},
-               {2, 1},
-               {2, 1.5},
-               {2, 2},
-               {3, 1},
-               {3, 1.5},
-               {3, 2}
-           };
+    m   = {{1, 1}, {1, 1.5}, {1, 2}, {2, 1}, {2, 1.5}, {2, 2}, {3, 1}, {3, 1.5}, {3, 2}};
     assert(m.size() == 9);
     assert(std::distance(m.begin(), m.end()) == 9);
     C::const_iterator i = m.cbegin();
@@ -79,7 +56,7 @@ int main(int, char**)
     assert(*++i == V(3, 1));
     assert(*++i == V(3, 1.5));
     assert(*++i == V(3, 2));
-    }
+  }
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/associative/multimap/multimap.cons/compare.pass.cpp b/libcxx/test/std/containers/associative/multimap/multimap.cons/compare.pass.cpp
index ebe1c9d6c7237..84584a427ead0 100644
--- a/libcxx/test/std/containers/associative/multimap/multimap.cons/compare.pass.cpp
+++ b/libcxx/test/std/containers/associative/multimap/multimap.cons/compare.pass.cpp
@@ -19,23 +19,22 @@
 #include "../../../test_compare.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
+int main(int, char**) {
+  {
     typedef test_less<int> C;
     const std::multimap<int, double, C> m(C(3));
     assert(m.empty());
     assert(m.begin() == m.end());
     assert(m.key_comp() == C(3));
-    }
+  }
 #if TEST_STD_VER >= 11
-    {
+  {
     typedef test_less<int> C;
     const std::multimap<int, double, C, min_allocator<std::pair<const int, double>>> m(C(3));
     assert(m.empty());
     assert(m.begin() == m.end());
     assert(m.key_comp() == C(3));
-    }
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/associative/multimap/multimap.cons/compare_alloc.pass.cpp b/libcxx/test/std/containers/associative/multimap/multimap.cons/compare_alloc.pass.cpp
index 7bfb93ee1961f..207e7e271234f 100644
--- a/libcxx/test/std/containers/associative/multimap/multimap.cons/compare_alloc.pass.cpp
+++ b/libcxx/test/std/containers/associative/multimap/multimap.cons/compare_alloc.pass.cpp
@@ -20,9 +20,8 @@
 #include "test_allocator.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
+int main(int, char**) {
+  {
     typedef test_less<int> C;
     typedef test_allocator<std::pair<const int, double> > A;
     std::multimap<int, double, C, A> m(C(4), A(5));
@@ -30,9 +29,9 @@ int main(int, char**)
     assert(m.begin() == m.end());
     assert(m.key_comp() == C(4));
     assert(m.get_allocator() == A(5));
-    }
+  }
 #if TEST_STD_VER >= 11
-    {
+  {
     typedef test_less<int> C;
     typedef min_allocator<std::pair<const int, double> > A;
     std::multimap<int, double, C, A> m(C(4), A());
@@ -40,8 +39,8 @@ int main(int, char**)
     assert(m.begin() == m.end());
     assert(m.key_comp() == C(4));
     assert(m.get_allocator() == A());
-    }
-    {
+  }
+  {
     typedef test_less<int> C;
     typedef explicit_allocator<std::pair<const int, double> > A;
     std::multimap<int, double, C, A> m(C(4), A{});
@@ -49,7 +48,7 @@ int main(int, char**)
     assert(m.begin() == m.end());
     assert(m.key_comp() == C(4));
     assert(m.get_allocator() == A{});
-    }
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/associative/multimap/multimap.cons/compare_copy_constructible.compile.fail.cpp b/libcxx/test/std/containers/associative/multimap/multimap.cons/compare_copy_constructible.compile.fail.cpp
index 23fe479f56cb2..adb866a1abdf6 100644
--- a/libcxx/test/std/containers/associative/multimap/multimap.cons/compare_copy_constructible.compile.fail.cpp
+++ b/libcxx/test/std/containers/associative/multimap/multimap.cons/compare_copy_constructible.compile.fail.cpp
@@ -15,16 +15,16 @@
 
 template <class T>
 struct Comp {
-    bool operator () (const T& lhs, const T& rhs) const { return lhs < rhs; }
+  bool operator()(const T& lhs, const T& rhs) const { return lhs < rhs; }
 
-    Comp () {}
-private:
-    Comp (const Comp &); // declared but not defined
-    };
+  Comp() {}
 
+private:
+  Comp(const Comp&); // declared but not defined
+};
 
 int main(int, char**) {
-    std::multimap<int, int, Comp<int> > m;
+  std::multimap<int, int, Comp<int> > m;
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/associative/multimap/multimap.cons/copy.pass.cpp b/libcxx/test/std/containers/associative/multimap/multimap.cons/copy.pass.cpp
index 9e53d9c15de19..9cffbf8c17850 100644
--- a/libcxx/test/std/containers/associative/multimap/multimap.cons/copy.pass.cpp
+++ b/libcxx/test/std/containers/associative/multimap/multimap.cons/copy.pass.cpp
@@ -20,84 +20,80 @@
 #include "test_allocator.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef std::pair<const int, double> V;
-        V ar[] =
-        {
-            V(1, 1),
-            V(1, 1.5),
-            V(1, 2),
-            V(2, 1),
-            V(2, 1.5),
-            V(2, 2),
-            V(3, 1),
-            V(3, 1.5),
-            V(3, 2),
-        };
-        typedef test_less<int> C;
-        typedef test_allocator<V> A;
-        std::multimap<int, double, C, A> mo(ar, ar+sizeof(ar)/sizeof(ar[0]), C(5), A(7));
-        std::multimap<int, double, C, A> m = mo;
-        assert(m == mo);
-        assert(m.get_allocator() == A(7));
-        assert(m.key_comp() == C(5));
+int main(int, char**) {
+  {
+    typedef std::pair<const int, double> V;
+    V ar[] = {
+        V(1, 1),
+        V(1, 1.5),
+        V(1, 2),
+        V(2, 1),
+        V(2, 1.5),
+        V(2, 2),
+        V(3, 1),
+        V(3, 1.5),
+        V(3, 2),
+    };
+    typedef test_less<int> C;
+    typedef test_allocator<V> A;
+    std::multimap<int, double, C, A> mo(ar, ar + sizeof(ar) / sizeof(ar[0]), C(5), A(7));
+    std::multimap<int, double, C, A> m = mo;
+    assert(m == mo);
+    assert(m.get_allocator() == A(7));
+    assert(m.key_comp() == C(5));
 
-        assert(mo.get_allocator() == A(7));
-        assert(mo.key_comp() == C(5));
-    }
+    assert(mo.get_allocator() == A(7));
+    assert(mo.key_comp() == C(5));
+  }
 #if TEST_STD_VER >= 11
-    {
-        typedef std::pair<const int, double> V;
-        V ar[] =
-        {
-            V(1, 1),
-            V(1, 1.5),
-            V(1, 2),
-            V(2, 1),
-            V(2, 1.5),
-            V(2, 2),
-            V(3, 1),
-            V(3, 1.5),
-            V(3, 2),
-        };
-        typedef test_less<int> C;
-        typedef other_allocator<V> A;
-        std::multimap<int, double, C, A> mo(ar, ar+sizeof(ar)/sizeof(ar[0]), C(5), A(7));
-        std::multimap<int, double, C, A> m = mo;
-        assert(m == mo);
-        assert(m.get_allocator() == A(-2));
-        assert(m.key_comp() == C(5));
+  {
+    typedef std::pair<const int, double> V;
+    V ar[] = {
+        V(1, 1),
+        V(1, 1.5),
+        V(1, 2),
+        V(2, 1),
+        V(2, 1.5),
+        V(2, 2),
+        V(3, 1),
+        V(3, 1.5),
+        V(3, 2),
+    };
+    typedef test_less<int> C;
+    typedef other_allocator<V> A;
+    std::multimap<int, double, C, A> mo(ar, ar + sizeof(ar) / sizeof(ar[0]), C(5), A(7));
+    std::multimap<int, double, C, A> m = mo;
+    assert(m == mo);
+    assert(m.get_allocator() == A(-2));
+    assert(m.key_comp() == C(5));
 
-        assert(mo.get_allocator() == A(7));
-        assert(mo.key_comp() == C(5));
-    }
-    {
-        typedef std::pair<const int, double> V;
-        V ar[] =
-        {
-            V(1, 1),
-            V(1, 1.5),
-            V(1, 2),
-            V(2, 1),
-            V(2, 1.5),
-            V(2, 2),
-            V(3, 1),
-            V(3, 1.5),
-            V(3, 2),
-        };
-        typedef test_less<int> C;
-        typedef min_allocator<V> A;
-        std::multimap<int, double, C, A> mo(ar, ar+sizeof(ar)/sizeof(ar[0]), C(5), A());
-        std::multimap<int, double, C, A> m = mo;
-        assert(m == mo);
-        assert(m.get_allocator() == A());
-        assert(m.key_comp() == C(5));
+    assert(mo.get_allocator() == A(7));
+    assert(mo.key_comp() == C(5));
+  }
+  {
+    typedef std::pair<const int, double> V;
+    V ar[] = {
+        V(1, 1),
+        V(1, 1.5),
+        V(1, 2),
+        V(2, 1),
+        V(2, 1.5),
+        V(2, 2),
+        V(3, 1),
+        V(3, 1.5),
+        V(3, 2),
+    };
+    typedef test_less<int> C;
+    typedef min_allocator<V> A;
+    std::multimap<int, double, C, A> mo(ar, ar + sizeof(ar) / sizeof(ar[0]), C(5), A());
+    std::multimap<int, double, C, A> m = mo;
+    assert(m == mo);
+    assert(m.get_allocator() == A());
+    assert(m.key_comp() == C(5));
 
-        assert(mo.get_allocator() == A());
-        assert(mo.key_comp() == C(5));
-    }
+    assert(mo.get_allocator() == A());
+    assert(mo.key_comp() == C(5));
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/associative/multimap/multimap.cons/copy_alloc.pass.cpp b/libcxx/test/std/containers/associative/multimap/multimap.cons/copy_alloc.pass.cpp
index 9ccf7715d267b..89e6c40aa108d 100644
--- a/libcxx/test/std/containers/associative/multimap/multimap.cons/copy_alloc.pass.cpp
+++ b/libcxx/test/std/containers/associative/multimap/multimap.cons/copy_alloc.pass.cpp
@@ -20,12 +20,10 @@
 #include "test_allocator.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
+int main(int, char**) {
+  {
     typedef std::pair<const int, double> V;
-    V ar[] =
-    {
+    V ar[] = {
         V(1, 1),
         V(1, 1.5),
         V(1, 2),
@@ -38,7 +36,7 @@ int main(int, char**)
     };
     typedef test_less<int> C;
     typedef test_allocator<V> A;
-    std::multimap<int, double, C, A> mo(ar, ar+sizeof(ar)/sizeof(ar[0]), C(5), A(7));
+    std::multimap<int, double, C, A> mo(ar, ar + sizeof(ar) / sizeof(ar[0]), C(5), A(7));
     std::multimap<int, double, C, A> m(mo, A(3));
     assert(m == mo);
     assert(m.get_allocator() == A(3));
@@ -46,12 +44,11 @@ int main(int, char**)
 
     assert(mo.get_allocator() == A(7));
     assert(mo.key_comp() == C(5));
-    }
+  }
 #if TEST_STD_VER >= 11
-    {
+  {
     typedef std::pair<const int, double> V;
-    V ar[] =
-    {
+    V ar[] = {
         V(1, 1),
         V(1, 1.5),
         V(1, 2),
@@ -64,7 +61,7 @@ int main(int, char**)
     };
     typedef test_less<int> C;
     typedef min_allocator<V> A;
-    std::multimap<int, double, C, A> mo(ar, ar+sizeof(ar)/sizeof(ar[0]), C(5), A());
+    std::multimap<int, double, C, A> mo(ar, ar + sizeof(ar) / sizeof(ar[0]), C(5), A());
     std::multimap<int, double, C, A> m(mo, A());
     assert(m == mo);
     assert(m.get_allocator() == A());
@@ -72,11 +69,10 @@ int main(int, char**)
 
     assert(mo.get_allocator() == A());
     assert(mo.key_comp() == C(5));
-    }
-    {
+  }
+  {
     typedef std::pair<const int, double> V;
-    V ar[] =
-    {
+    V ar[] = {
         V(1, 1),
         V(1, 1.5),
         V(1, 2),
@@ -89,7 +85,7 @@ int main(int, char**)
     };
     typedef test_less<int> C;
     typedef explicit_allocator<V> A;
-    std::multimap<int, double, C, A> mo(ar, ar+sizeof(ar)/sizeof(ar[0]), C(5), A{});
+    std::multimap<int, double, C, A> mo(ar, ar + sizeof(ar) / sizeof(ar[0]), C(5), A{});
     std::multimap<int, double, C, A> m(mo, A{});
     assert(m == mo);
     assert(m.get_allocator() == A{});
@@ -97,7 +93,7 @@ int main(int, char**)
 
     assert(mo.get_allocator() == A{});
     assert(mo.key_comp() == C(5));
-    }
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/associative/multimap/multimap.cons/copy_assign.pass.cpp b/libcxx/test/std/containers/associative/multimap/multimap.cons/copy_assign.pass.cpp
index b4ff5e7777b98..463403a412aaa 100644
--- a/libcxx/test/std/containers/associative/multimap/multimap.cons/copy_assign.pass.cpp
+++ b/libcxx/test/std/containers/associative/multimap/multimap.cons/copy_assign.pass.cpp
@@ -20,107 +20,102 @@
 #include "test_allocator.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef std::pair<const int, double> V;
-        V ar[] =
-        {
-            V(1, 1),
-            V(1, 1.5),
-            V(1, 2),
-            V(2, 1),
-            V(2, 1.5),
-            V(2, 2),
-            V(3, 1),
-            V(3, 1.5),
-            V(3, 2),
-        };
-        typedef test_less<int> C;
-        typedef test_allocator<V> A;
-        std::multimap<int, double, C, A> mo(ar, ar+sizeof(ar)/sizeof(ar[0]), C(5), A(2));
-        std::multimap<int, double, C, A> m(ar, ar+sizeof(ar)/sizeof(ar[0])/2, C(3), A(7));
-        m = mo;
-        assert(m == mo);
-        assert(m.get_allocator() == A(7));
-        assert(m.key_comp() == C(5));
+int main(int, char**) {
+  {
+    typedef std::pair<const int, double> V;
+    V ar[] = {
+        V(1, 1),
+        V(1, 1.5),
+        V(1, 2),
+        V(2, 1),
+        V(2, 1.5),
+        V(2, 2),
+        V(3, 1),
+        V(3, 1.5),
+        V(3, 2),
+    };
+    typedef test_less<int> C;
+    typedef test_allocator<V> A;
+    std::multimap<int, double, C, A> mo(ar, ar + sizeof(ar) / sizeof(ar[0]), C(5), A(2));
+    std::multimap<int, double, C, A> m(ar, ar + sizeof(ar) / sizeof(ar[0]) / 2, C(3), A(7));
+    m = mo;
+    assert(m == mo);
+    assert(m.get_allocator() == A(7));
+    assert(m.key_comp() == C(5));
 
-        assert(mo.get_allocator() == A(2));
-        assert(mo.key_comp() == C(5));
-    }
-    {
-        typedef std::pair<const int, double> V;
-        const V ar[] =
-        {
-            V(1, 1),
-            V(1, 1.5),
-            V(1, 2),
-            V(2, 1),
-            V(2, 1.5),
-            V(2, 2),
-            V(3, 1),
-            V(3, 1.5),
-            V(3, 2),
-        };
-        std::multimap<int, double> m(ar, ar+sizeof(ar)/sizeof(ar[0]));
-        std::multimap<int, double> *p = &m;
-        m = *p;
-        assert(m.size() == sizeof(ar)/sizeof(ar[0]));
-        assert(std::equal(m.begin(), m.end(), ar));
-    }
-    {
-        typedef std::pair<const int, double> V;
-        V ar[] =
-        {
-            V(1, 1),
-            V(1, 1.5),
-            V(1, 2),
-            V(2, 1),
-            V(2, 1.5),
-            V(2, 2),
-            V(3, 1),
-            V(3, 1.5),
-            V(3, 2),
-        };
-        typedef test_less<int> C;
-        typedef other_allocator<V> A;
-        std::multimap<int, double, C, A> mo(ar, ar+sizeof(ar)/sizeof(ar[0]), C(5), A(2));
-        std::multimap<int, double, C, A> m(ar, ar+sizeof(ar)/sizeof(ar[0])/2, C(3), A(7));
-        m = mo;
-        assert(m == mo);
-        assert(m.get_allocator() == A(2));
-        assert(m.key_comp() == C(5));
+    assert(mo.get_allocator() == A(2));
+    assert(mo.key_comp() == C(5));
+  }
+  {
+    typedef std::pair<const int, double> V;
+    const V ar[] = {
+        V(1, 1),
+        V(1, 1.5),
+        V(1, 2),
+        V(2, 1),
+        V(2, 1.5),
+        V(2, 2),
+        V(3, 1),
+        V(3, 1.5),
+        V(3, 2),
+    };
+    std::multimap<int, double> m(ar, ar + sizeof(ar) / sizeof(ar[0]));
+    std::multimap<int, double>* p = &m;
+    m                             = *p;
+    assert(m.size() == sizeof(ar) / sizeof(ar[0]));
+    assert(std::equal(m.begin(), m.end(), ar));
+  }
+  {
+    typedef std::pair<const int, double> V;
+    V ar[] = {
+        V(1, 1),
+        V(1, 1.5),
+        V(1, 2),
+        V(2, 1),
+        V(2, 1.5),
+        V(2, 2),
+        V(3, 1),
+        V(3, 1.5),
+        V(3, 2),
+    };
+    typedef test_less<int> C;
+    typedef other_allocator<V> A;
+    std::multimap<int, double, C, A> mo(ar, ar + sizeof(ar) / sizeof(ar[0]), C(5), A(2));
+    std::multimap<int, double, C, A> m(ar, ar + sizeof(ar) / sizeof(ar[0]) / 2, C(3), A(7));
+    m = mo;
+    assert(m == mo);
+    assert(m.get_allocator() == A(2));
+    assert(m.key_comp() == C(5));
 
-        assert(mo.get_allocator() == A(2));
-        assert(mo.key_comp() == C(5));
-    }
+    assert(mo.get_allocator() == A(2));
+    assert(mo.key_comp() == C(5));
+  }
 #if TEST_STD_VER >= 11
-    {
-        typedef std::pair<const int, double> V;
-        V ar[] =
-        {
-            V(1, 1),
-            V(1, 1.5),
-            V(1, 2),
-            V(2, 1),
-            V(2, 1.5),
-            V(2, 2),
-            V(3, 1),
-            V(3, 1.5),
-            V(3, 2),
-        };
-        typedef test_less<int> C;
-        typedef min_allocator<V> A;
-        std::multimap<int, double, C, A> mo(ar, ar+sizeof(ar)/sizeof(ar[0]), C(5), A());
-        std::multimap<int, double, C, A> m(ar, ar+sizeof(ar)/sizeof(ar[0])/2, C(3), A());
-        m = mo;
-        assert(m == mo);
-        assert(m.get_allocator() == A());
-        assert(m.key_comp() == C(5));
+  {
+    typedef std::pair<const int, double> V;
+    V ar[] = {
+        V(1, 1),
+        V(1, 1.5),
+        V(1, 2),
+        V(2, 1),
+        V(2, 1.5),
+        V(2, 2),
+        V(3, 1),
+        V(3, 1.5),
+        V(3, 2),
+    };
+    typedef test_less<int> C;
+    typedef min_allocator<V> A;
+    std::multimap<int, double, C, A> mo(ar, ar + sizeof(ar) / sizeof(ar[0]), C(5), A());
+    std::multimap<int, double, C, A> m(ar, ar + sizeof(ar) / sizeof(ar[0]) / 2, C(3), A());
+    m = mo;
+    assert(m == mo);
+    assert(m.get_allocator() == A());
+    assert(m.key_comp() == C(5));
 
-        assert(mo.get_allocator() == A());
-        assert(mo.key_comp() == C(5));
-    }
+    assert(mo.get_allocator() == A());
+    assert(mo.key_comp() == C(5));
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/associative/multimap/multimap.cons/deduct.pass.cpp b/libcxx/test/std/containers/associative/multimap/multimap.cons/deduct.pass.cpp
index 56bc5400b65ab..2f3db7f0de992 100644
--- a/libcxx/test/std/containers/associative/multimap/multimap.cons/deduct.pass.cpp
+++ b/libcxx/test/std/containers/associative/multimap/multimap.cons/deduct.pass.cpp
@@ -45,114 +45,114 @@
 #include "deduction_guides_sfinae_checks.h"
 #include "test_allocator.h"
 
-using P = std::pair<int, long>;
+using P  = std::pair<int, long>;
 using PC = std::pair<const int, long>;
 
-int main(int, char**)
-{
-    {
-    const P arr[] = { {1,1L}, {2,2L}, {1,1L}, {INT_MAX,1L}, {3,1L} };
+int main(int, char**) {
+  {
+    const P arr[] = {{1, 1L}, {2, 2L}, {1, 1L}, {INT_MAX, 1L}, {3, 1L}};
     std::multimap m(std::begin(arr), std::end(arr));
 
     ASSERT_SAME_TYPE(decltype(m), std::multimap<int, long>);
-    const PC expected_m[] = { {1,1L}, {1,1L}, {2,2L}, {3,1L}, {INT_MAX,1L} };
+    const PC expected_m[] = {{1, 1L}, {1, 1L}, {2, 2L}, {3, 1L}, {INT_MAX, 1L}};
     assert(std::equal(m.begin(), m.end(), std::begin(expected_m), std::end(expected_m)));
-    }
+  }
 
-    {
-    const P arr[] = { {1,1L}, {2,2L}, {1,1L}, {INT_MAX,1L}, {3,1L} };
+  {
+    const P arr[] = {{1, 1L}, {2, 2L}, {1, 1L}, {INT_MAX, 1L}, {3, 1L}};
     std::multimap m(std::begin(arr), std::end(arr), std::greater<int>());
 
     ASSERT_SAME_TYPE(decltype(m), std::multimap<int, long, std::greater<int>>);
-    const PC expected_m[] = { {INT_MAX,1L}, {3,1L}, {2,2L}, {1,1L}, {1,1L} };
+    const PC expected_m[] = {{INT_MAX, 1L}, {3, 1L}, {2, 2L}, {1, 1L}, {1, 1L}};
     assert(std::equal(m.begin(), m.end(), std::begin(expected_m), std::end(expected_m)));
-    }
+  }
 
-    {
-    const P arr[] = { {1,1L}, {2,2L}, {1,1L}, {INT_MAX,1L}, {3,1L} };
+  {
+    const P arr[] = {{1, 1L}, {2, 2L}, {1, 1L}, {INT_MAX, 1L}, {3, 1L}};
     std::multimap m(std::begin(arr), std::end(arr), std::greater<int>(), test_allocator<PC>(0, 42));
 
     ASSERT_SAME_TYPE(decltype(m), std::multimap<int, long, std::greater<int>, test_allocator<PC>>);
-    const PC expected_m[] = { {INT_MAX,1L}, {3,1L}, {2,2L}, {1,1L}, {1,1L} };
+    const PC expected_m[] = {{INT_MAX, 1L}, {3, 1L}, {2, 2L}, {1, 1L}, {1, 1L}};
     assert(std::equal(m.begin(), m.end(), std::begin(expected_m), std::end(expected_m)));
     assert(m.get_allocator().get_id() == 42);
-    }
+  }
 
-    {
+  {
     std::multimap<int, long> source;
     std::multimap m(source);
     ASSERT_SAME_TYPE(decltype(m), decltype(source));
     assert(m.size() == 0);
-    }
+  }
 
-    {
+  {
     std::multimap<int, long> source;
-    std::multimap m{source};  // braces instead of parens
+    std::multimap m{source}; // braces instead of parens
     ASSERT_SAME_TYPE(decltype(m), decltype(source));
     assert(m.size() == 0);
-    }
+  }
 
-    {
+  {
     std::multimap<int, long> source;
     std::multimap m(source, std::map<int, long>::allocator_type());
     ASSERT_SAME_TYPE(decltype(m), decltype(source));
     assert(m.size() == 0);
-    }
+  }
 
-    {
-    std::multimap m{ P{1,1L}, P{2,2L}, P{1,1L}, P{INT_MAX,1L}, P{3,1L} };
+  {
+    std::multimap m{P{1, 1L}, P{2, 2L}, P{1, 1L}, P{INT_MAX, 1L}, P{3, 1L}};
 
     ASSERT_SAME_TYPE(decltype(m), std::multimap<int, long>);
-    const PC expected_m[] = { {1,1L}, {1,1L}, {2,2L}, {3,1L}, {INT_MAX,1L} };
+    const PC expected_m[] = {{1, 1L}, {1, 1L}, {2, 2L}, {3, 1L}, {INT_MAX, 1L}};
     assert(std::equal(m.begin(), m.end(), std::begin(expected_m), std::end(expected_m)));
-    }
+  }
 
-    {
-    std::multimap m({ P{1,1L}, P{2,2L}, P{1,1L}, P{INT_MAX,1L}, P{3,1L} }, std::greater<int>());
+  {
+    std::multimap m({P{1, 1L}, P{2, 2L}, P{1, 1L}, P{INT_MAX, 1L}, P{3, 1L}}, std::greater<int>());
 
     ASSERT_SAME_TYPE(decltype(m), std::multimap<int, long, std::greater<int>>);
-    const PC expected_m[] = { {INT_MAX,1L}, {3,1L}, {2,2L}, {1,1L}, {1,1L} };
+    const PC expected_m[] = {{INT_MAX, 1L}, {3, 1L}, {2, 2L}, {1, 1L}, {1, 1L}};
     assert(std::equal(m.begin(), m.end(), std::begin(expected_m), std::end(expected_m)));
-    }
+  }
 
-    {
-    std::multimap m({ P{1,1L}, P{2,2L}, P{1,1L}, P{INT_MAX,1L}, P{3,1L} }, std::greater<int>(), test_allocator<PC>(0, 43));
+  {
+    std::multimap m(
+        {P{1, 1L}, P{2, 2L}, P{1, 1L}, P{INT_MAX, 1L}, P{3, 1L}}, std::greater<int>(), test_allocator<PC>(0, 43));
 
     ASSERT_SAME_TYPE(decltype(m), std::multimap<int, long, std::greater<int>, test_allocator<PC>>);
-    const PC expected_m[] = { {INT_MAX,1L}, {3,1L}, {2,2L}, {1,1L}, {1,1L} };
+    const PC expected_m[] = {{INT_MAX, 1L}, {3, 1L}, {2, 2L}, {1, 1L}, {1, 1L}};
     assert(std::equal(m.begin(), m.end(), std::begin(expected_m), std::end(expected_m)));
     assert(m.get_allocator().get_id() == 43);
-    }
+  }
 
-    {
-    const P arr[] = { {1,1L}, {2,2L}, {1,1L}, {INT_MAX,1L}, {3,1L} };
+  {
+    const P arr[] = {{1, 1L}, {2, 2L}, {1, 1L}, {INT_MAX, 1L}, {3, 1L}};
     std::multimap m(std::begin(arr), std::end(arr), test_allocator<PC>(0, 44));
 
     ASSERT_SAME_TYPE(decltype(m), std::multimap<int, long, std::less<int>, test_allocator<PC>>);
-    const PC expected_m[] = { {1,1L}, {1,1L}, {2,2L}, {3,1L}, {INT_MAX,1L} };
+    const PC expected_m[] = {{1, 1L}, {1, 1L}, {2, 2L}, {3, 1L}, {INT_MAX, 1L}};
     assert(std::equal(m.begin(), m.end(), std::begin(expected_m), std::end(expected_m)));
     assert(m.get_allocator().get_id() == 44);
-    }
+  }
 
-    {
-    std::multimap m({ P{1,1L}, P{2,2L}, P{1,1L}, P{INT_MAX,1L}, P{3,1L} }, test_allocator<PC>(0, 45));
+  {
+    std::multimap m({P{1, 1L}, P{2, 2L}, P{1, 1L}, P{INT_MAX, 1L}, P{3, 1L}}, test_allocator<PC>(0, 45));
 
     ASSERT_SAME_TYPE(decltype(m), std::multimap<int, long, std::less<int>, test_allocator<PC>>);
-    const PC expected_m[] = { {1,1L}, {1,1L}, {2,2L}, {3,1L}, {INT_MAX,1L} };
+    const PC expected_m[] = {{1, 1L}, {1, 1L}, {2, 2L}, {3, 1L}, {INT_MAX, 1L}};
     assert(std::equal(m.begin(), m.end(), std::begin(expected_m), std::end(expected_m)));
     assert(m.get_allocator().get_id() == 45);
-    }
+  }
 
-    {
+  {
     // Examples from LWG3025
     std::multimap m{std::pair{1, 1}, {2, 2}, {3, 3}};
     ASSERT_SAME_TYPE(decltype(m), std::multimap<int, int>);
 
     std::multimap m2{m.begin(), m.end()};
     ASSERT_SAME_TYPE(decltype(m2), std::multimap<int, int>);
-    }
+  }
 
-    {
+  {
     // Examples from LWG3531
     std::multimap m1{{std::pair{1, 2}, {3, 4}}, std::less<int>()};
     ASSERT_SAME_TYPE(decltype(m1), std::multimap<int, int>);
@@ -160,38 +160,38 @@ int main(int, char**)
     using value_type = std::pair<const int, int>;
     std::multimap m2{{value_type{1, 2}, {3, 4}}, std::less<int>()};
     ASSERT_SAME_TYPE(decltype(m2), std::multimap<int, int>);
-    }
+  }
 
 #if TEST_STD_VER >= 23
-    {
-      using Range = std::array<P, 0>;
-      using Comp = std::greater<int>;
-      using DefaultComp = std::less<int>;
-      using Alloc = test_allocator<PC>;
-
-      { // (from_range, range)
-        std::multimap c(std::from_range, Range());
-        static_assert(std::is_same_v<decltype(c), std::multimap<int, long>>);
-      }
-
-      { // (from_range, range, comp)
-        std::multimap c(std::from_range, Range(), Comp());
-        static_assert(std::is_same_v<decltype(c), std::multimap<int, long, Comp>>);
-      }
-
-      { // (from_range, range, comp, alloc)
-        std::multimap c(std::from_range, Range(), Comp(), Alloc());
-        static_assert(std::is_same_v<decltype(c), std::multimap<int, long, Comp, Alloc>>);
-      }
-
-      { // (from_range, range, alloc)
-        std::multimap c(std::from_range, Range(), Alloc());
-        static_assert(std::is_same_v<decltype(c), std::multimap<int, long, DefaultComp, Alloc>>);
-      }
+  {
+    using Range       = std::array<P, 0>;
+    using Comp        = std::greater<int>;
+    using DefaultComp = std::less<int>;
+    using Alloc       = test_allocator<PC>;
+
+    { // (from_range, range)
+      std::multimap c(std::from_range, Range());
+      static_assert(std::is_same_v<decltype(c), std::multimap<int, long>>);
+    }
+
+    { // (from_range, range, comp)
+      std::multimap c(std::from_range, Range(), Comp());
+      static_assert(std::is_same_v<decltype(c), std::multimap<int, long, Comp>>);
+    }
+
+    { // (from_range, range, comp, alloc)
+      std::multimap c(std::from_range, Range(), Comp(), Alloc());
+      static_assert(std::is_same_v<decltype(c), std::multimap<int, long, Comp, Alloc>>);
+    }
+
+    { // (from_range, range, alloc)
+      std::multimap c(std::from_range, Range(), Alloc());
+      static_assert(std::is_same_v<decltype(c), std::multimap<int, long, DefaultComp, Alloc>>);
     }
+  }
 #endif
 
-    AssociativeContainerDeductionGuidesSfinaeAway<std::multimap, std::multimap<int, long>>();
+  AssociativeContainerDeductionGuidesSfinaeAway<std::multimap, std::multimap<int, long>>();
 
-    return 0;
+  return 0;
 }
diff --git a/libcxx/test/std/containers/associative/multimap/multimap.cons/deduct.verify.cpp b/libcxx/test/std/containers/associative/multimap/multimap.cons/deduct.verify.cpp
index 795ac19240031..03ac8ebf92716 100644
--- a/libcxx/test/std/containers/associative/multimap/multimap.cons/deduct.verify.cpp
+++ b/libcxx/test/std/containers/associative/multimap/multimap.cons/deduct.verify.cpp
@@ -32,74 +32,75 @@
 #include <type_traits>
 
 struct NotAnAllocator {
-    friend bool operator<(NotAnAllocator, NotAnAllocator) { return false; }
+  friend bool operator<(NotAnAllocator, NotAnAllocator) { return false; }
 };
 
-using P = std::pair<int, long>;
+using P  = std::pair<int, long>;
 using PC = std::pair<const int, long>;
 
-int main(int, char**)
-{
-    {
-        // cannot deduce Key and T from nothing
-        std::multimap m; // expected-error-re{{no viable constructor or deduction guide for deduction of template arguments of '{{(std::)?}}multimap'}}
-    }
-    {
-        // cannot deduce Key and T from just (Compare)
-        std::multimap m(std::less<int>{});
-            // expected-error-re@-1{{no viable constructor or deduction guide for deduction of template arguments of '{{(std::)?}}multimap'}}
-    }
-    {
-        // cannot deduce Key and T from just (Compare, Allocator)
-        std::multimap m(std::less<int>{}, std::allocator<PC>{});
-            // expected-error-re@-1{{no viable constructor or deduction guide for deduction of template arguments of '{{(std::)?}}multimap'}}
-    }
-    {
-        // cannot deduce Key and T from just (Allocator)
-        std::multimap m(std::allocator<PC>{});
-            // expected-error-re@-1{{no viable constructor or deduction guide for deduction of template arguments of '{{(std::)?}}multimap'}}
-    }
-    {
-        // refuse to rebind the allocator if Allocator::value_type is not exactly what we expect
-        const P arr[] = { {1,1L}, {2,2L}, {3,3L} };
-        std::multimap m(arr, arr + 3, std::allocator<P>());
-            // expected-error-re@map:* {{static assertion failed{{( due to requirement '.*')?}}{{.*}}Allocator::value_type must be same type as value_type}}
-    }
-    {
-        // cannot convert from some arbitrary unrelated type
-        NotAnAllocator a;
-        std::multimap m(a); // expected-error-re{{no viable constructor or deduction guide for deduction of template arguments of '{{(std::)?}}multimap'}}
-    }
-    {
-        // cannot deduce that the inner braced things should be std::pair and not something else
-        std::multimap m{ {1,1L}, {2,2L}, {3,3L} };
-            // expected-error-re@-1{{no viable constructor or deduction guide for deduction of template arguments of '{{(std::)?}}multimap'}}
-    }
-    {
-        // cannot deduce that the inner braced things should be std::pair and not something else
-        std::multimap m({ {1,1L}, {2,2L}, {3,3L} }, std::less<int>());
-            // expected-error-re@-1{{no viable constructor or deduction guide for deduction of template arguments of '{{(std::)?}}multimap'}}
-    }
-    {
-        // cannot deduce that the inner braced things should be std::pair and not something else
-        std::multimap m({ {1,1L}, {2,2L}, {3,3L} }, std::less<int>(), std::allocator<PC>());
-            // expected-error-re@-1{{no viable constructor or deduction guide for deduction of template arguments of '{{(std::)?}}multimap'}}
-    }
-    {
-        // cannot deduce that the inner braced things should be std::pair and not something else
-        std::multimap m({ {1,1L}, {2,2L}, {3,3L} }, std::allocator<PC>());
-            // expected-error-re@-1{{no viable constructor or deduction guide for deduction of template arguments of '{{(std::)?}}multimap'}}
-    }
-    {
-        // since we have parens, not braces, this deliberately does not find the initializer_list constructor
-        std::multimap m(P{1,1L});
-            // expected-error-re@-1{{no viable constructor or deduction guide for deduction of template arguments of '{{(std::)?}}multimap'}}
-    }
-    {
-        // since we have parens, not braces, this deliberately does not find the initializer_list constructor
-        std::multimap m(PC{1,1L});
-            // expected-error-re@-1{{no viable constructor or deduction guide for deduction of template arguments of '{{(std::)?}}multimap'}}
-    }
+int main(int, char**) {
+  {
+    // cannot deduce Key and T from nothing
+    std::multimap m;
+    // expected-error-re@-1{{no viable constructor or deduction guide for deduction of template arguments of '{{(std::)?}}multimap'}}
+  }
+  {
+    // cannot deduce Key and T from just (Compare)
+    std::multimap m(std::less<int>{});
+    // expected-error-re@-1{{no viable constructor or deduction guide for deduction of template arguments of '{{(std::)?}}multimap'}}
+  }
+  {
+    // cannot deduce Key and T from just (Compare, Allocator)
+    std::multimap m(std::less<int>{}, std::allocator<PC>{});
+    // expected-error-re@-1{{no viable constructor or deduction guide for deduction of template arguments of '{{(std::)?}}multimap'}}
+  }
+  {
+    // cannot deduce Key and T from just (Allocator)
+    std::multimap m(std::allocator<PC>{});
+    // expected-error-re@-1{{no viable constructor or deduction guide for deduction of template arguments of '{{(std::)?}}multimap'}}
+  }
+  {
+    // refuse to rebind the allocator if Allocator::value_type is not exactly what we expect
+    const P arr[] = {{1, 1L}, {2, 2L}, {3, 3L}};
+    std::multimap m(arr, arr + 3, std::allocator<P>());
+    // expected-error-re@map:* {{static assertion failed{{( due to requirement '.*')?}}{{.*}}Allocator::value_type must be same type as value_type}}
+  }
+  {
+    // cannot convert from some arbitrary unrelated type
+    NotAnAllocator a;
+    std::multimap m(a);
+    // expected-error-re@-1{{no viable constructor or deduction guide for deduction of template arguments of '{{(std::)?}}multimap'}}
+  }
+  {
+    // cannot deduce that the inner braced things should be std::pair and not something else
+    std::multimap m{{1, 1L}, {2, 2L}, {3, 3L}};
+    // expected-error-re@-1{{no viable constructor or deduction guide for deduction of template arguments of '{{(std::)?}}multimap'}}
+  }
+  {
+    // cannot deduce that the inner braced things should be std::pair and not something else
+    std::multimap m({{1, 1L}, {2, 2L}, {3, 3L}}, std::less<int>());
+    // expected-error-re@-1{{no viable constructor or deduction guide for deduction of template arguments of '{{(std::)?}}multimap'}}
+  }
+  {
+    // cannot deduce that the inner braced things should be std::pair and not something else
+    std::multimap m({{1, 1L}, {2, 2L}, {3, 3L}}, std::less<int>(), std::allocator<PC>());
+    // expected-error-re@-1{{no viable constructor or deduction guide for deduction of template arguments of '{{(std::)?}}multimap'}}
+  }
+  {
+    // cannot deduce that the inner braced things should be std::pair and not something else
+    std::multimap m({{1, 1L}, {2, 2L}, {3, 3L}}, std::allocator<PC>());
+    // expected-error-re@-1{{no viable constructor or deduction guide for deduction of template arguments of '{{(std::)?}}multimap'}}
+  }
+  {
+    // since we have parens, not braces, this deliberately does not find the initializer_list constructor
+    std::multimap m(P{1, 1L});
+    // expected-error-re@-1{{no viable constructor or deduction guide for deduction of template arguments of '{{(std::)?}}multimap'}}
+  }
+  {
+    // since we have parens, not braces, this deliberately does not find the initializer_list constructor
+    std::multimap m(PC{1, 1L});
+    // expected-error-re@-1{{no viable constructor or deduction guide for deduction of template arguments of '{{(std::)?}}multimap'}}
+  }
 
-    return 0;
+  return 0;
 }
diff --git a/libcxx/test/std/containers/associative/multimap/multimap.cons/deduct_const.pass.cpp b/libcxx/test/std/containers/associative/multimap/multimap.cons/deduct_const.pass.cpp
index c8f25dcd19964..ae1e5622797cd 100644
--- a/libcxx/test/std/containers/associative/multimap/multimap.cons/deduct_const.pass.cpp
+++ b/libcxx/test/std/containers/associative/multimap/multimap.cons/deduct_const.pass.cpp
@@ -35,73 +35,73 @@
 
 #include "test_allocator.h"
 
-using P = std::pair<int, long>;
-using PC = std::pair<const int, long>;
+using P   = std::pair<int, long>;
+using PC  = std::pair<const int, long>;
 using PCC = std::pair<const int, const long>;
 
-int main(int, char**)
-{
-    {
-    const PCC arr[] = { {1,1L}, {2,2L}, {1,1L}, {INT_MAX,1L}, {3,1L} };
+int main(int, char**) {
+  {
+    const PCC arr[] = {{1, 1L}, {2, 2L}, {1, 1L}, {INT_MAX, 1L}, {3, 1L}};
     std::multimap m(std::begin(arr), std::end(arr));
 
     ASSERT_SAME_TYPE(decltype(m), std::multimap<int, const long>);
-    const PCC expected_m[] = { {1,1L}, {1,1L}, {2,2L}, {3,1L}, {INT_MAX,1L} };
+    const PCC expected_m[] = {{1, 1L}, {1, 1L}, {2, 2L}, {3, 1L}, {INT_MAX, 1L}};
     assert(std::equal(m.begin(), m.end(), std::begin(expected_m), std::end(expected_m)));
-    }
+  }
 
-    {
-    const PCC arr[] = { {1,1L}, {2,2L}, {1,1L}, {INT_MAX,1L}, {3,1L} };
+  {
+    const PCC arr[] = {{1, 1L}, {2, 2L}, {1, 1L}, {INT_MAX, 1L}, {3, 1L}};
     std::multimap m(std::begin(arr), std::end(arr), std::greater<int>());
 
     ASSERT_SAME_TYPE(decltype(m), std::multimap<int, const long, std::greater<int>>);
-    const PCC expected_m[] = { {INT_MAX,1L}, {3,1L}, {2,2L}, {1,1L}, {1,1L} };
+    const PCC expected_m[] = {{INT_MAX, 1L}, {3, 1L}, {2, 2L}, {1, 1L}, {1, 1L}};
     assert(std::equal(m.begin(), m.end(), std::begin(expected_m), std::end(expected_m)));
-    }
+  }
 
-    {
-    const PCC arr[] = { {1,1L}, {2,2L}, {1,1L}, {INT_MAX,1L}, {3,1L} };
+  {
+    const PCC arr[] = {{1, 1L}, {2, 2L}, {1, 1L}, {INT_MAX, 1L}, {3, 1L}};
     std::multimap m(std::begin(arr), std::end(arr), std::greater<int>(), test_allocator<PCC>(0, 42));
 
     ASSERT_SAME_TYPE(decltype(m), std::multimap<int, const long, std::greater<int>, test_allocator<PCC>>);
-    const PCC expected_m[] = { {INT_MAX,1L}, {3,1L}, {2,2L}, {1,1L}, {1,1L} };
+    const PCC expected_m[] = {{INT_MAX, 1L}, {3, 1L}, {2, 2L}, {1, 1L}, {1, 1L}};
     assert(std::equal(m.begin(), m.end(), std::begin(expected_m), std::end(expected_m)));
     assert(m.get_allocator().get_id() == 42);
-    }
+  }
 
-    {
-    std::multimap m{ PC{1,1L}, PC{2,2L}, PC{1,1L}, PC{INT_MAX,1L}, PC{3,1L} };
+  {
+    std::multimap m{PC{1, 1L}, PC{2, 2L}, PC{1, 1L}, PC{INT_MAX, 1L}, PC{3, 1L}};
 
     ASSERT_SAME_TYPE(decltype(m), std::multimap<int, long>);
-    const PC expected_m[] = { {1,1L}, {1,1L}, {2,2L}, {3,1L}, {INT_MAX,1L} };
+    const PC expected_m[] = {{1, 1L}, {1, 1L}, {2, 2L}, {3, 1L}, {INT_MAX, 1L}};
     assert(std::equal(m.begin(), m.end(), std::begin(expected_m), std::end(expected_m)));
-    }
+  }
 
-    {
-    std::multimap m({ PC{1,1L}, PC{2,2L}, PC{1,1L}, PC{INT_MAX,1L}, PC{3,1L} }, std::greater<int>());
+  {
+    std::multimap m({PC{1, 1L}, PC{2, 2L}, PC{1, 1L}, PC{INT_MAX, 1L}, PC{3, 1L}}, std::greater<int>());
 
     ASSERT_SAME_TYPE(decltype(m), std::multimap<int, long, std::greater<int>>);
-    const PC expected_m[] = { {INT_MAX,1L}, {3,1L}, {2,2L}, {1,1L}, {1,1L} };
+    const PC expected_m[] = {{INT_MAX, 1L}, {3, 1L}, {2, 2L}, {1, 1L}, {1, 1L}};
     assert(std::equal(m.begin(), m.end(), std::begin(expected_m), std::end(expected_m)));
-    }
+  }
 
-    {
-    std::multimap m({ PC{1,1L}, PC{2,2L}, PC{1,1L}, PC{INT_MAX,1L}, PC{3,1L} }, std::greater<int>(), test_allocator<PC>(0, 43));
+  {
+    std::multimap m(
+        {PC{1, 1L}, PC{2, 2L}, PC{1, 1L}, PC{INT_MAX, 1L}, PC{3, 1L}}, std::greater<int>(), test_allocator<PC>(0, 43));
 
     ASSERT_SAME_TYPE(decltype(m), std::multimap<int, long, std::greater<int>, test_allocator<PC>>);
-    const PC expected_m[] = { {INT_MAX,1L}, {3,1L}, {2,2L}, {1,1L}, {1,1L} };
+    const PC expected_m[] = {{INT_MAX, 1L}, {3, 1L}, {2, 2L}, {1, 1L}, {1, 1L}};
     assert(std::equal(m.begin(), m.end(), std::begin(expected_m), std::end(expected_m)));
     assert(m.get_allocator().get_id() == 43);
-    }
+  }
 
-    {
-    std::multimap m({ PC{1,1L}, PC{2,2L}, PC{1,1L}, PC{INT_MAX,1L}, PC{3,1L} }, test_allocator<PC>(0, 45));
+  {
+    std::multimap m({PC{1, 1L}, PC{2, 2L}, PC{1, 1L}, PC{INT_MAX, 1L}, PC{3, 1L}}, test_allocator<PC>(0, 45));
 
     ASSERT_SAME_TYPE(decltype(m), std::multimap<int, long, std::less<int>, test_allocator<PC>>);
-    const PC expected_m[] = { {1,1L}, {1,1L}, {2,2L}, {3,1L}, {INT_MAX,1L} };
+    const PC expected_m[] = {{1, 1L}, {1, 1L}, {2, 2L}, {3, 1L}, {INT_MAX, 1L}};
     assert(std::equal(m.begin(), m.end(), std::begin(expected_m), std::end(expected_m)));
     assert(m.get_allocator().get_id() == 45);
-    }
+  }
 
-    return 0;
+  return 0;
 }
diff --git a/libcxx/test/std/containers/associative/multimap/multimap.cons/default.pass.cpp b/libcxx/test/std/containers/associative/multimap/multimap.cons/default.pass.cpp
index 5248874c2e48d..f74723b3174f0 100644
--- a/libcxx/test/std/containers/associative/multimap/multimap.cons/default.pass.cpp
+++ b/libcxx/test/std/containers/associative/multimap/multimap.cons/default.pass.cpp
@@ -18,38 +18,37 @@
 #include "test_macros.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
+int main(int, char**) {
+  {
     std::multimap<int, double> m;
     assert(m.empty());
     assert(m.begin() == m.end());
-    }
+  }
 #if TEST_STD_VER >= 11
-    {
+  {
     std::multimap<int, double, std::less<int>, min_allocator<std::pair<const int, double>>> m;
     assert(m.empty());
     assert(m.begin() == m.end());
-    }
-    {
+  }
+  {
     typedef explicit_allocator<std::pair<const int, double>> A;
-        {
-        std::multimap<int, double, std::less<int>, A> m;
-        assert(m.empty());
-        assert(m.begin() == m.end());
-        }
-        {
-        A a;
-        std::multimap<int, double, std::less<int>, A> m(a);
-        assert(m.empty());
-        assert(m.begin() == m.end());
-        }
+    {
+      std::multimap<int, double, std::less<int>, A> m;
+      assert(m.empty());
+      assert(m.begin() == m.end());
     }
     {
+      A a;
+      std::multimap<int, double, std::less<int>, A> m(a);
+      assert(m.empty());
+      assert(m.begin() == m.end());
+    }
+  }
+  {
     std::multimap<int, double> m = {};
     assert(m.empty());
     assert(m.begin() == m.end());
-    }
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/associative/multimap/multimap.cons/default_noexcept.pass.cpp b/libcxx/test/std/containers/associative/multimap/multimap.cons/default_noexcept.pass.cpp
index 6abc125707629..b6c4e0336394d 100644
--- a/libcxx/test/std/containers/associative/multimap/multimap.cons/default_noexcept.pass.cpp
+++ b/libcxx/test/std/containers/associative/multimap/multimap.cons/default_noexcept.pass.cpp
@@ -26,34 +26,32 @@
 #include "test_allocator.h"
 
 template <class T>
-struct some_comp
-{
-    typedef T value_type;
-    some_comp();
-    bool operator()(const T&, const T&) const { return false; }
+struct some_comp {
+  typedef T value_type;
+  some_comp();
+  bool operator()(const T&, const T&) const { return false; }
 };
 
-int main(int, char**)
-{
-    typedef std::pair<const MoveOnly, MoveOnly> V;
+int main(int, char**) {
+  typedef std::pair<const MoveOnly, MoveOnly> V;
 #if defined(_LIBCPP_VERSION)
-    {
-        typedef std::multimap<MoveOnly, MoveOnly> C;
-        static_assert(std::is_nothrow_default_constructible<C>::value, "");
-    }
-    {
-        typedef std::multimap<MoveOnly, MoveOnly, std::less<MoveOnly>, test_allocator<V>> C;
-        static_assert(std::is_nothrow_default_constructible<C>::value, "");
-    }
+  {
+    typedef std::multimap<MoveOnly, MoveOnly> C;
+    static_assert(std::is_nothrow_default_constructible<C>::value, "");
+  }
+  {
+    typedef std::multimap<MoveOnly, MoveOnly, std::less<MoveOnly>, test_allocator<V>> C;
+    static_assert(std::is_nothrow_default_constructible<C>::value, "");
+  }
 #endif // _LIBCPP_VERSION
-    {
-        typedef std::multimap<MoveOnly, MoveOnly, std::less<MoveOnly>, other_allocator<V>> C;
-        static_assert(!std::is_nothrow_default_constructible<C>::value, "");
-    }
-    {
-        typedef std::multimap<MoveOnly, MoveOnly, some_comp<MoveOnly>> C;
-        static_assert(!std::is_nothrow_default_constructible<C>::value, "");
-    }
+  {
+    typedef std::multimap<MoveOnly, MoveOnly, std::less<MoveOnly>, other_allocator<V>> C;
+    static_assert(!std::is_nothrow_default_constructible<C>::value, "");
+  }
+  {
+    typedef std::multimap<MoveOnly, MoveOnly, some_comp<MoveOnly>> C;
+    static_assert(!std::is_nothrow_default_constructible<C>::value, "");
+  }
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/associative/multimap/multimap.cons/dtor_noexcept.pass.cpp b/libcxx/test/std/containers/associative/multimap/multimap.cons/dtor_noexcept.pass.cpp
index 407f966948158..62afae92b6902 100644
--- a/libcxx/test/std/containers/associative/multimap/multimap.cons/dtor_noexcept.pass.cpp
+++ b/libcxx/test/std/containers/associative/multimap/multimap.cons/dtor_noexcept.pass.cpp
@@ -20,33 +20,31 @@
 #include "test_allocator.h"
 
 template <class T>
-struct some_comp
-{
-    typedef T value_type;
-    ~some_comp() noexcept(false);
-    bool operator()(const T&, const T&) const { return false; }
+struct some_comp {
+  typedef T value_type;
+  ~some_comp() noexcept(false);
+  bool operator()(const T&, const T&) const { return false; }
 };
 
-int main(int, char**)
-{
-    typedef std::pair<const MoveOnly, MoveOnly> V;
-    {
-        typedef std::multimap<MoveOnly, MoveOnly> C;
-        static_assert(std::is_nothrow_destructible<C>::value, "");
-    }
-    {
-        typedef std::multimap<MoveOnly, MoveOnly, std::less<MoveOnly>, test_allocator<V>> C;
-        static_assert(std::is_nothrow_destructible<C>::value, "");
-    }
-    {
-        typedef std::multimap<MoveOnly, MoveOnly, std::less<MoveOnly>, other_allocator<V>> C;
-        static_assert(std::is_nothrow_destructible<C>::value, "");
-    }
+int main(int, char**) {
+  typedef std::pair<const MoveOnly, MoveOnly> V;
+  {
+    typedef std::multimap<MoveOnly, MoveOnly> C;
+    static_assert(std::is_nothrow_destructible<C>::value, "");
+  }
+  {
+    typedef std::multimap<MoveOnly, MoveOnly, std::less<MoveOnly>, test_allocator<V>> C;
+    static_assert(std::is_nothrow_destructible<C>::value, "");
+  }
+  {
+    typedef std::multimap<MoveOnly, MoveOnly, std::less<MoveOnly>, other_allocator<V>> C;
+    static_assert(std::is_nothrow_destructible<C>::value, "");
+  }
 #if defined(_LIBCPP_VERSION)
-    {
-        typedef std::multimap<MoveOnly, MoveOnly, some_comp<MoveOnly>> C;
-        static_assert(!std::is_nothrow_destructible<C>::value, "");
-    }
+  {
+    typedef std::multimap<MoveOnly, MoveOnly, some_comp<MoveOnly>> C;
+    static_assert(!std::is_nothrow_destructible<C>::value, "");
+  }
 #endif // _LIBCPP_VERSION
 
   return 0;
diff --git a/libcxx/test/std/containers/associative/multimap/multimap.cons/from_range.pass.cpp b/libcxx/test/std/containers/associative/multimap/multimap.cons/from_range.pass.cpp
index b38078c7e82f2..fd8ca64f1cce6 100644
--- a/libcxx/test/std/containers/associative/multimap/multimap.cons/from_range.pass.cpp
+++ b/libcxx/test/std/containers/associative/multimap/multimap.cons/from_range.pass.cpp
@@ -22,11 +22,9 @@
 #include "test_macros.h"
 
 void test_duplicates() {
-  using T = std::pair<const int, char>;
-  std::array input = {
-    T{1, 'a'}, T{2, 'a'}, T{3, 'a'}, T{3, 'b'}, T{3, 'c'}, T{2, 'b'}, T{4, 'a'}
-  };
-  auto c = std::multimap<int, char>(std::from_range, input);
+  using T          = std::pair<const int, char>;
+  std::array input = {T{1, 'a'}, T{2, 'a'}, T{3, 'a'}, T{3, 'b'}, T{3, 'c'}, T{2, 'b'}, T{4, 'a'}};
+  auto c           = std::multimap<int, char>(std::from_range, input);
   assert(std::ranges::is_permutation(input, c));
 }
 
diff --git a/libcxx/test/std/containers/associative/multimap/multimap.cons/initializer_list.pass.cpp b/libcxx/test/std/containers/associative/multimap/multimap.cons/initializer_list.pass.cpp
index 1c679778941c2..db91af00359e8 100644
--- a/libcxx/test/std/containers/associative/multimap/multimap.cons/initializer_list.pass.cpp
+++ b/libcxx/test/std/containers/associative/multimap/multimap.cons/initializer_list.pass.cpp
@@ -20,23 +20,11 @@
 #include "test_macros.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
+int main(int, char**) {
+  {
     typedef std::multimap<int, double> C;
     typedef C::value_type V;
-    C m =
-           {
-               {1, 1},
-               {1, 1.5},
-               {1, 2},
-               {2, 1},
-               {2, 1.5},
-               {2, 2},
-               {3, 1},
-               {3, 1.5},
-               {3, 2}
-           };
+    C m = {{1, 1}, {1, 1.5}, {1, 2}, {2, 1}, {2, 1.5}, {2, 2}, {3, 1}, {3, 1.5}, {3, 2}};
     assert(m.size() == 9);
     assert(std::distance(m.begin(), m.end()) == 9);
     C::const_iterator i = m.cbegin();
@@ -49,22 +37,11 @@ int main(int, char**)
     assert(*++i == V(3, 1));
     assert(*++i == V(3, 1.5));
     assert(*++i == V(3, 2));
-    }
-    {
+  }
+  {
     typedef std::multimap<int, double, std::less<int>, min_allocator<std::pair<const int, double>>> C;
     typedef C::value_type V;
-    C m =
-           {
-               {1, 1},
-               {1, 1.5},
-               {1, 2},
-               {2, 1},
-               {2, 1.5},
-               {2, 2},
-               {3, 1},
-               {3, 1.5},
-               {3, 2}
-           };
+    C m = {{1, 1}, {1, 1.5}, {1, 2}, {2, 1}, {2, 1.5}, {2, 2}, {3, 1}, {3, 1.5}, {3, 2}};
     assert(m.size() == 9);
     assert(std::distance(m.begin(), m.end()) == 9);
     C::const_iterator i = m.cbegin();
@@ -77,7 +54,7 @@ int main(int, char**)
     assert(*++i == V(3, 1));
     assert(*++i == V(3, 1.5));
     assert(*++i == V(3, 2));
-    }
+  }
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/associative/multimap/multimap.cons/initializer_list_compare.pass.cpp b/libcxx/test/std/containers/associative/multimap/multimap.cons/initializer_list_compare.pass.cpp
index ed8ab93e5289f..0d346d7226353 100644
--- a/libcxx/test/std/containers/associative/multimap/multimap.cons/initializer_list_compare.pass.cpp
+++ b/libcxx/test/std/containers/associative/multimap/multimap.cons/initializer_list_compare.pass.cpp
@@ -20,26 +20,12 @@
 #include "../../../test_compare.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
+int main(int, char**) {
+  {
     typedef test_less<int> Cmp;
     typedef std::multimap<int, double, Cmp> C;
     typedef C::value_type V;
-    C m(
-           {
-               {1, 1},
-               {1, 1.5},
-               {1, 2},
-               {2, 1},
-               {2, 1.5},
-               {2, 2},
-               {3, 1},
-               {3, 1.5},
-               {3, 2}
-           },
-           Cmp(4)
-        );
+    C m({{1, 1}, {1, 1.5}, {1, 2}, {2, 1}, {2, 1.5}, {2, 2}, {3, 1}, {3, 1.5}, {3, 2}}, Cmp(4));
     assert(m.size() == 9);
     assert(std::distance(m.begin(), m.end()) == 9);
     C::const_iterator i = m.cbegin();
@@ -53,25 +39,12 @@ int main(int, char**)
     assert(*++i == V(3, 1.5));
     assert(*++i == V(3, 2));
     assert(m.key_comp() == Cmp(4));
-    }
-    {
+  }
+  {
     typedef test_less<int> Cmp;
     typedef std::multimap<int, double, Cmp, min_allocator<std::pair<const int, double>>> C;
     typedef C::value_type V;
-    C m(
-           {
-               {1, 1},
-               {1, 1.5},
-               {1, 2},
-               {2, 1},
-               {2, 1.5},
-               {2, 2},
-               {3, 1},
-               {3, 1.5},
-               {3, 2}
-           },
-           Cmp(4)
-        );
+    C m({{1, 1}, {1, 1.5}, {1, 2}, {2, 1}, {2, 1.5}, {2, 2}, {3, 1}, {3, 1.5}, {3, 2}}, Cmp(4));
     assert(m.size() == 9);
     assert(std::distance(m.begin(), m.end()) == 9);
     C::const_iterator i = m.cbegin();
@@ -85,7 +58,7 @@ int main(int, char**)
     assert(*++i == V(3, 1.5));
     assert(*++i == V(3, 2));
     assert(m.key_comp() == Cmp(4));
-    }
+  }
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/associative/multimap/multimap.cons/initializer_list_compare_alloc.pass.cpp b/libcxx/test/std/containers/associative/multimap/multimap.cons/initializer_list_compare_alloc.pass.cpp
index 92d6128244c1a..493f1bafbc1b7 100644
--- a/libcxx/test/std/containers/associative/multimap/multimap.cons/initializer_list_compare_alloc.pass.cpp
+++ b/libcxx/test/std/containers/associative/multimap/multimap.cons/initializer_list_compare_alloc.pass.cpp
@@ -21,27 +21,13 @@
 #include "test_allocator.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
+int main(int, char**) {
+  {
     typedef test_less<int> Cmp;
     typedef test_allocator<std::pair<const int, double> > A;
     typedef std::multimap<int, double, Cmp, A> C;
     typedef C::value_type V;
-    C m(
-           {
-               {1, 1},
-               {1, 1.5},
-               {1, 2},
-               {2, 1},
-               {2, 1.5},
-               {2, 2},
-               {3, 1},
-               {3, 1.5},
-               {3, 2}
-           },
-           Cmp(4), A(5)
-        );
+    C m({{1, 1}, {1, 1.5}, {1, 2}, {2, 1}, {2, 1.5}, {2, 2}, {3, 1}, {3, 1.5}, {3, 2}}, Cmp(4), A(5));
     assert(m.size() == 9);
     assert(std::distance(m.begin(), m.end()) == 9);
     C::const_iterator i = m.cbegin();
@@ -56,26 +42,13 @@ int main(int, char**)
     assert(*++i == V(3, 2));
     assert(m.key_comp() == Cmp(4));
     assert(m.get_allocator() == A(5));
-    }
-    {
+  }
+  {
     typedef test_less<int> Cmp;
     typedef min_allocator<std::pair<const int, double> > A;
     typedef std::multimap<int, double, Cmp, A> C;
     typedef C::value_type V;
-    C m(
-           {
-               {1, 1},
-               {1, 1.5},
-               {1, 2},
-               {2, 1},
-               {2, 1.5},
-               {2, 2},
-               {3, 1},
-               {3, 1.5},
-               {3, 2}
-           },
-           Cmp(4), A()
-        );
+    C m({{1, 1}, {1, 1.5}, {1, 2}, {2, 1}, {2, 1.5}, {2, 2}, {3, 1}, {3, 1.5}, {3, 2}}, Cmp(4), A());
     assert(m.size() == 9);
     assert(std::distance(m.begin(), m.end()) == 9);
     C::const_iterator i = m.cbegin();
@@ -90,23 +63,14 @@ int main(int, char**)
     assert(*++i == V(3, 2));
     assert(m.key_comp() == Cmp(4));
     assert(m.get_allocator() == A());
-    }
-    {
+  }
+  {
     typedef test_less<int> C;
     typedef std::pair<const int, double> V;
     typedef min_allocator<V> A;
     typedef std::multimap<int, double, C, A> M;
     A a;
-    M m ({ {1, 1},
-           {1, 1.5},
-           {1, 2},
-           {2, 1},
-           {2, 1.5},
-           {2, 2},
-           {3, 1},
-           {3, 1.5},
-           {3, 2}
-          }, a);
+    M m({{1, 1}, {1, 1.5}, {1, 2}, {2, 1}, {2, 1.5}, {2, 2}, {3, 1}, {3, 1.5}, {3, 2}}, a);
 
     assert(m.size() == 9);
     assert(std::distance(m.begin(), m.end()) == 9);
@@ -121,26 +85,13 @@ int main(int, char**)
     assert(*++i == V(3, 1.5));
     assert(*++i == V(3, 2));
     assert(m.get_allocator() == a);
-    }
-    {
+  }
+  {
     typedef test_less<int> Cmp;
     typedef explicit_allocator<std::pair<const int, double> > A;
     typedef std::multimap<int, double, Cmp, A> C;
     typedef C::value_type V;
-    C m(
-           {
-               {1, 1},
-               {1, 1.5},
-               {1, 2},
-               {2, 1},
-               {2, 1.5},
-               {2, 2},
-               {3, 1},
-               {3, 1.5},
-               {3, 2}
-           },
-           Cmp(4), A{}
-        );
+    C m({{1, 1}, {1, 1.5}, {1, 2}, {2, 1}, {2, 1.5}, {2, 2}, {3, 1}, {3, 1.5}, {3, 2}}, Cmp(4), A{});
     assert(m.size() == 9);
     assert(std::distance(m.begin(), m.end()) == 9);
     C::const_iterator i = m.cbegin();
@@ -155,7 +106,7 @@ int main(int, char**)
     assert(*++i == V(3, 2));
     assert(m.key_comp() == Cmp(4));
     assert(m.get_allocator() == A{});
-    }
+  }
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/associative/multimap/multimap.cons/iter_iter.pass.cpp b/libcxx/test/std/containers/associative/multimap/multimap.cons/iter_iter.pass.cpp
index 92ebc56f160dd..55542914f9a01 100644
--- a/libcxx/test/std/containers/associative/multimap/multimap.cons/iter_iter.pass.cpp
+++ b/libcxx/test/std/containers/associative/multimap/multimap.cons/iter_iter.pass.cpp
@@ -19,12 +19,10 @@
 #include "test_macros.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
+int main(int, char**) {
+  {
     typedef std::pair<const int, double> V;
-    V ar[] =
-    {
+    V ar[] = {
         V(1, 1),
         V(1, 1.5),
         V(1, 2),
@@ -35,7 +33,7 @@ int main(int, char**)
         V(3, 1.5),
         V(3, 2),
     };
-    std::multimap<int, double> m(ar, ar+sizeof(ar)/sizeof(ar[0]));
+    std::multimap<int, double> m(ar, ar + sizeof(ar) / sizeof(ar[0]));
     assert(m.size() == 9);
     assert(std::distance(m.begin(), m.end()) == 9);
     assert(*m.begin() == V(1, 1));
@@ -47,12 +45,11 @@ int main(int, char**)
     assert(*std::next(m.begin(), 6) == V(3, 1));
     assert(*std::next(m.begin(), 7) == V(3, 1.5));
     assert(*std::next(m.begin(), 8) == V(3, 2));
-    }
+  }
 #if TEST_STD_VER >= 11
-    {
+  {
     typedef std::pair<const int, double> V;
-    V ar[] =
-    {
+    V ar[] = {
         V(1, 1),
         V(1, 1.5),
         V(1, 2),
@@ -63,7 +60,7 @@ int main(int, char**)
         V(3, 1.5),
         V(3, 2),
     };
-    std::multimap<int, double, std::less<int>, min_allocator<V>> m(ar, ar+sizeof(ar)/sizeof(ar[0]));
+    std::multimap<int, double, std::less<int>, min_allocator<V>> m(ar, ar + sizeof(ar) / sizeof(ar[0]));
     assert(m.size() == 9);
     assert(std::distance(m.begin(), m.end()) == 9);
     assert(*m.begin() == V(1, 1));
@@ -75,12 +72,11 @@ int main(int, char**)
     assert(*std::next(m.begin(), 6) == V(3, 1));
     assert(*std::next(m.begin(), 7) == V(3, 1.5));
     assert(*std::next(m.begin(), 8) == V(3, 2));
-    }
-#if TEST_STD_VER > 11
-    {
+  }
+#  if TEST_STD_VER > 11
+  {
     typedef std::pair<const int, double> V;
-    V ar[] =
-    {
+    V ar[] = {
         V(1, 1),
         V(1, 1.5),
         V(1, 2),
@@ -93,7 +89,7 @@ int main(int, char**)
     };
     typedef min_allocator<std::pair<const int, double>> A;
     A a;
-    std::multimap<int, double, std::less<int>, A> m(ar, ar+sizeof(ar)/sizeof(ar[0]), a);
+    std::multimap<int, double, std::less<int>, A> m(ar, ar + sizeof(ar) / sizeof(ar[0]), a);
     assert(m.size() == 9);
     assert(std::distance(m.begin(), m.end()) == 9);
     assert(*m.begin() == V(1, 1));
@@ -106,8 +102,8 @@ int main(int, char**)
     assert(*std::next(m.begin(), 7) == V(3, 1.5));
     assert(*std::next(m.begin(), 8) == V(3, 2));
     assert(m.get_allocator() == a);
-    }
-#endif
+  }
+#  endif
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/associative/multimap/multimap.cons/iter_iter_comp.pass.cpp b/libcxx/test/std/containers/associative/multimap/multimap.cons/iter_iter_comp.pass.cpp
index d50df29796ea4..128d54a3d6b4e 100644
--- a/libcxx/test/std/containers/associative/multimap/multimap.cons/iter_iter_comp.pass.cpp
+++ b/libcxx/test/std/containers/associative/multimap/multimap.cons/iter_iter_comp.pass.cpp
@@ -21,12 +21,10 @@
 #include "../../../test_compare.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
+int main(int, char**) {
+  {
     typedef std::pair<const int, double> V;
-    V ar[] =
-    {
+    V ar[] = {
         V(1, 1),
         V(1, 1.5),
         V(1, 2),
@@ -38,7 +36,7 @@ int main(int, char**)
         V(3, 2),
     };
     typedef test_less<int> C;
-    std::multimap<int, double, C> m(ar, ar+sizeof(ar)/sizeof(ar[0]), C(5));
+    std::multimap<int, double, C> m(ar, ar + sizeof(ar) / sizeof(ar[0]), C(5));
     assert(m.key_comp() == C(5));
     assert(m.size() == 9);
     assert(std::distance(m.begin(), m.end()) == 9);
@@ -51,12 +49,11 @@ int main(int, char**)
     assert(*std::next(m.begin(), 6) == V(3, 1));
     assert(*std::next(m.begin(), 7) == V(3, 1.5));
     assert(*std::next(m.begin(), 8) == V(3, 2));
-    }
+  }
 #if TEST_STD_VER >= 11
-    {
+  {
     typedef std::pair<const int, double> V;
-    V ar[] =
-    {
+    V ar[] = {
         V(1, 1),
         V(1, 1.5),
         V(1, 2),
@@ -68,7 +65,7 @@ int main(int, char**)
         V(3, 2),
     };
     typedef test_less<int> C;
-    std::multimap<int, double, C, min_allocator<V>> m(ar, ar+sizeof(ar)/sizeof(ar[0]), C(5));
+    std::multimap<int, double, C, min_allocator<V>> m(ar, ar + sizeof(ar) / sizeof(ar[0]), C(5));
     assert(m.key_comp() == C(5));
     assert(m.size() == 9);
     assert(std::distance(m.begin(), m.end()) == 9);
@@ -81,7 +78,7 @@ int main(int, char**)
     assert(*std::next(m.begin(), 6) == V(3, 1));
     assert(*std::next(m.begin(), 7) == V(3, 1.5));
     assert(*std::next(m.begin(), 8) == V(3, 2));
-    }
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/associative/multimap/multimap.cons/iter_iter_comp_alloc.pass.cpp b/libcxx/test/std/containers/associative/multimap/multimap.cons/iter_iter_comp_alloc.pass.cpp
index 0d0f0b57ed485..68d658c2edf85 100644
--- a/libcxx/test/std/containers/associative/multimap/multimap.cons/iter_iter_comp_alloc.pass.cpp
+++ b/libcxx/test/std/containers/associative/multimap/multimap.cons/iter_iter_comp_alloc.pass.cpp
@@ -22,12 +22,10 @@
 #include "test_allocator.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
+int main(int, char**) {
+  {
     typedef std::pair<const int, double> V;
-    V ar[] =
-    {
+    V ar[] = {
         V(1, 1),
         V(1, 1.5),
         V(1, 2),
@@ -40,7 +38,7 @@ int main(int, char**)
     };
     typedef test_less<int> C;
     typedef test_allocator<V> A;
-    std::multimap<int, double, C, A> m(ar, ar+sizeof(ar)/sizeof(ar[0]), C(5), A(7));
+    std::multimap<int, double, C, A> m(ar, ar + sizeof(ar) / sizeof(ar[0]), C(5), A(7));
     assert(m.get_allocator() == A(7));
     assert(m.key_comp() == C(5));
     assert(m.size() == 9);
@@ -54,12 +52,11 @@ int main(int, char**)
     assert(*std::next(m.begin(), 6) == V(3, 1));
     assert(*std::next(m.begin(), 7) == V(3, 1.5));
     assert(*std::next(m.begin(), 8) == V(3, 2));
-    }
+  }
 #if TEST_STD_VER >= 11
-    {
+  {
     typedef std::pair<const int, double> V;
-    V ar[] =
-    {
+    V ar[] = {
         V(1, 1),
         V(1, 1.5),
         V(1, 2),
@@ -72,7 +69,7 @@ int main(int, char**)
     };
     typedef test_less<int> C;
     typedef min_allocator<V> A;
-    std::multimap<int, double, C, A> m(ar, ar+sizeof(ar)/sizeof(ar[0]), C(5), A());
+    std::multimap<int, double, C, A> m(ar, ar + sizeof(ar) / sizeof(ar[0]), C(5), A());
     assert(m.get_allocator() == A());
     assert(m.key_comp() == C(5));
     assert(m.size() == 9);
@@ -86,11 +83,10 @@ int main(int, char**)
     assert(*std::next(m.begin(), 6) == V(3, 1));
     assert(*std::next(m.begin(), 7) == V(3, 1.5));
     assert(*std::next(m.begin(), 8) == V(3, 2));
-    }
-    {
+  }
+  {
     typedef std::pair<const int, double> V;
-    V ar[] =
-    {
+    V ar[] = {
         V(1, 1),
         V(1, 1.5),
         V(1, 2),
@@ -103,7 +99,7 @@ int main(int, char**)
     };
     typedef test_less<int> C;
     typedef explicit_allocator<V> A;
-    std::multimap<int, double, C, A> m(ar, ar+sizeof(ar)/sizeof(ar[0]), C(5), A{});
+    std::multimap<int, double, C, A> m(ar, ar + sizeof(ar) / sizeof(ar[0]), C(5), A{});
     assert(m.get_allocator() == A{});
     assert(m.key_comp() == C(5));
     assert(m.size() == 9);
@@ -117,7 +113,7 @@ int main(int, char**)
     assert(*std::next(m.begin(), 6) == V(3, 1));
     assert(*std::next(m.begin(), 7) == V(3, 1.5));
     assert(*std::next(m.begin(), 8) == V(3, 2));
-    }
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/associative/multimap/multimap.cons/move.pass.cpp b/libcxx/test/std/containers/associative/multimap/multimap.cons/move.pass.cpp
index 8227b81f0c05c..d229c6f7d5f49 100644
--- a/libcxx/test/std/containers/associative/multimap/multimap.cons/move.pass.cpp
+++ b/libcxx/test/std/containers/associative/multimap/multimap.cons/move.pass.cpp
@@ -22,113 +22,110 @@
 #include "test_allocator.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    typedef std::pair<const int, double> V;
-    {
-        typedef test_less<int> C;
-        typedef test_allocator<V> A;
-        std::multimap<int, double, C, A> mo(C(5), A(7));
-        std::multimap<int, double, C, A> m = std::move(mo);
-        assert(m.get_allocator() == A(7));
-        assert(m.key_comp() == C(5));
-        assert(m.size() == 0);
-        assert(std::distance(m.begin(), m.end()) == 0);
+int main(int, char**) {
+  typedef std::pair<const int, double> V;
+  {
+    typedef test_less<int> C;
+    typedef test_allocator<V> A;
+    std::multimap<int, double, C, A> mo(C(5), A(7));
+    std::multimap<int, double, C, A> m = std::move(mo);
+    assert(m.get_allocator() == A(7));
+    assert(m.key_comp() == C(5));
+    assert(m.size() == 0);
+    assert(std::distance(m.begin(), m.end()) == 0);
 
-        assert(mo.get_allocator() == A(7));
-        assert(mo.get_allocator().get_id() == test_alloc_base::moved_value);
-        assert(mo.key_comp() == C(5));
-        assert(mo.size() == 0);
-        assert(std::distance(mo.begin(), mo.end()) == 0);
-    }
-    {
-        V ar[] =
-        {
-            V(1, 1),
-            V(1, 1.5),
-            V(1, 2),
-            V(2, 1),
-            V(2, 1.5),
-            V(2, 2),
-            V(3, 1),
-            V(3, 1.5),
-            V(3, 2),
-        };
-        typedef test_less<int> C;
-        typedef test_allocator<V> A;
-        std::multimap<int, double, C, A> mo(ar, ar+sizeof(ar)/sizeof(ar[0]), C(5), A(7));
-        std::multimap<int, double, C, A> m = std::move(mo);
-        assert(m.get_allocator() == A(7));
-        assert(m.key_comp() == C(5));
-        assert(m.size() == 9);
-        assert(std::distance(m.begin(), m.end()) == 9);
-        assert(*m.begin() == V(1, 1));
-        assert(*std::next(m.begin()) == V(1, 1.5));
-        assert(*std::next(m.begin(), 2) == V(1, 2));
-        assert(*std::next(m.begin(), 3) == V(2, 1));
-        assert(*std::next(m.begin(), 4) == V(2, 1.5));
-        assert(*std::next(m.begin(), 5) == V(2, 2));
-        assert(*std::next(m.begin(), 6) == V(3, 1));
-        assert(*std::next(m.begin(), 7) == V(3, 1.5));
-        assert(*std::next(m.begin(), 8) == V(3, 2));
+    assert(mo.get_allocator() == A(7));
+    assert(mo.get_allocator().get_id() == test_alloc_base::moved_value);
+    assert(mo.key_comp() == C(5));
+    assert(mo.size() == 0);
+    assert(std::distance(mo.begin(), mo.end()) == 0);
+  }
+  {
+    V ar[] = {
+        V(1, 1),
+        V(1, 1.5),
+        V(1, 2),
+        V(2, 1),
+        V(2, 1.5),
+        V(2, 2),
+        V(3, 1),
+        V(3, 1.5),
+        V(3, 2),
+    };
+    typedef test_less<int> C;
+    typedef test_allocator<V> A;
+    std::multimap<int, double, C, A> mo(ar, ar + sizeof(ar) / sizeof(ar[0]), C(5), A(7));
+    std::multimap<int, double, C, A> m = std::move(mo);
+    assert(m.get_allocator() == A(7));
+    assert(m.key_comp() == C(5));
+    assert(m.size() == 9);
+    assert(std::distance(m.begin(), m.end()) == 9);
+    assert(*m.begin() == V(1, 1));
+    assert(*std::next(m.begin()) == V(1, 1.5));
+    assert(*std::next(m.begin(), 2) == V(1, 2));
+    assert(*std::next(m.begin(), 3) == V(2, 1));
+    assert(*std::next(m.begin(), 4) == V(2, 1.5));
+    assert(*std::next(m.begin(), 5) == V(2, 2));
+    assert(*std::next(m.begin(), 6) == V(3, 1));
+    assert(*std::next(m.begin(), 7) == V(3, 1.5));
+    assert(*std::next(m.begin(), 8) == V(3, 2));
 
-        assert(mo.get_allocator() == A(7));
-        assert(mo.get_allocator().get_id() == test_alloc_base::moved_value);
-        assert(mo.key_comp() == C(5));
-        assert(mo.size() == 0);
-        assert(std::distance(mo.begin(), mo.end()) == 0);
-    }
-    {
-        typedef test_less<int> C;
-        typedef min_allocator<V> A;
-        std::multimap<int, double, C, A> mo(C(5), A());
-        std::multimap<int, double, C, A> m = std::move(mo);
-        assert(m.get_allocator() == A());
-        assert(m.key_comp() == C(5));
-        assert(m.size() == 0);
-        assert(std::distance(m.begin(), m.end()) == 0);
+    assert(mo.get_allocator() == A(7));
+    assert(mo.get_allocator().get_id() == test_alloc_base::moved_value);
+    assert(mo.key_comp() == C(5));
+    assert(mo.size() == 0);
+    assert(std::distance(mo.begin(), mo.end()) == 0);
+  }
+  {
+    typedef test_less<int> C;
+    typedef min_allocator<V> A;
+    std::multimap<int, double, C, A> mo(C(5), A());
+    std::multimap<int, double, C, A> m = std::move(mo);
+    assert(m.get_allocator() == A());
+    assert(m.key_comp() == C(5));
+    assert(m.size() == 0);
+    assert(std::distance(m.begin(), m.end()) == 0);
 
-        assert(mo.get_allocator() == A());
-        assert(mo.key_comp() == C(5));
-        assert(mo.size() == 0);
-        assert(std::distance(mo.begin(), mo.end()) == 0);
-    }
-    {
-        V ar[] =
-        {
-            V(1, 1),
-            V(1, 1.5),
-            V(1, 2),
-            V(2, 1),
-            V(2, 1.5),
-            V(2, 2),
-            V(3, 1),
-            V(3, 1.5),
-            V(3, 2),
-        };
-        typedef test_less<int> C;
-        typedef min_allocator<V> A;
-        std::multimap<int, double, C, A> mo(ar, ar+sizeof(ar)/sizeof(ar[0]), C(5), A());
-        std::multimap<int, double, C, A> m = std::move(mo);
-        assert(m.get_allocator() == A());
-        assert(m.key_comp() == C(5));
-        assert(m.size() == 9);
-        assert(std::distance(m.begin(), m.end()) == 9);
-        assert(*m.begin() == V(1, 1));
-        assert(*std::next(m.begin()) == V(1, 1.5));
-        assert(*std::next(m.begin(), 2) == V(1, 2));
-        assert(*std::next(m.begin(), 3) == V(2, 1));
-        assert(*std::next(m.begin(), 4) == V(2, 1.5));
-        assert(*std::next(m.begin(), 5) == V(2, 2));
-        assert(*std::next(m.begin(), 6) == V(3, 1));
-        assert(*std::next(m.begin(), 7) == V(3, 1.5));
-        assert(*std::next(m.begin(), 8) == V(3, 2));
+    assert(mo.get_allocator() == A());
+    assert(mo.key_comp() == C(5));
+    assert(mo.size() == 0);
+    assert(std::distance(mo.begin(), mo.end()) == 0);
+  }
+  {
+    V ar[] = {
+        V(1, 1),
+        V(1, 1.5),
+        V(1, 2),
+        V(2, 1),
+        V(2, 1.5),
+        V(2, 2),
+        V(3, 1),
+        V(3, 1.5),
+        V(3, 2),
+    };
+    typedef test_less<int> C;
+    typedef min_allocator<V> A;
+    std::multimap<int, double, C, A> mo(ar, ar + sizeof(ar) / sizeof(ar[0]), C(5), A());
+    std::multimap<int, double, C, A> m = std::move(mo);
+    assert(m.get_allocator() == A());
+    assert(m.key_comp() == C(5));
+    assert(m.size() == 9);
+    assert(std::distance(m.begin(), m.end()) == 9);
+    assert(*m.begin() == V(1, 1));
+    assert(*std::next(m.begin()) == V(1, 1.5));
+    assert(*std::next(m.begin(), 2) == V(1, 2));
+    assert(*std::next(m.begin(), 3) == V(2, 1));
+    assert(*std::next(m.begin(), 4) == V(2, 1.5));
+    assert(*std::next(m.begin(), 5) == V(2, 2));
+    assert(*std::next(m.begin(), 6) == V(3, 1));
+    assert(*std::next(m.begin(), 7) == V(3, 1.5));
+    assert(*std::next(m.begin(), 8) == V(3, 2));
 
-        assert(mo.get_allocator() == A());
-        assert(mo.key_comp() == C(5));
-        assert(mo.size() == 0);
-        assert(std::distance(mo.begin(), mo.end()) == 0);
-    }
+    assert(mo.get_allocator() == A());
+    assert(mo.key_comp() == C(5));
+    assert(mo.size() == 0);
+    assert(std::distance(mo.begin(), mo.end()) == 0);
+  }
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/associative/multimap/multimap.cons/move_alloc.pass.cpp b/libcxx/test/std/containers/associative/multimap/multimap.cons/move_alloc.pass.cpp
index e246842ea7d4e..31ff8ff792566 100644
--- a/libcxx/test/std/containers/associative/multimap/multimap.cons/move_alloc.pass.cpp
+++ b/libcxx/test/std/containers/associative/multimap/multimap.cons/move_alloc.pass.cpp
@@ -24,253 +24,131 @@
 #include "min_allocator.h"
 #include "Counter.h"
 
-int main(int, char**)
-{
+int main(int, char**) {
+  {
+    typedef std::pair<MoveOnly, MoveOnly> V;
+    typedef std::pair<const MoveOnly, MoveOnly> VC;
+    typedef test_less<MoveOnly> C;
+    typedef test_allocator<VC> A;
+    typedef std::multimap<MoveOnly, MoveOnly, C, A> M;
+    typedef std::move_iterator<V*> I;
+    V a1[] = {V(1, 1), V(1, 2), V(1, 3), V(2, 1), V(2, 2), V(2, 3), V(3, 1), V(3, 2), V(3, 3)};
+    M m1(I(a1), I(a1 + sizeof(a1) / sizeof(a1[0])), C(5), A(7));
+    V a2[] = {V(1, 1), V(1, 2), V(1, 3), V(2, 1), V(2, 2), V(2, 3), V(3, 1), V(3, 2), V(3, 3)};
+    M m2(I(a2), I(a2 + sizeof(a2) / sizeof(a2[0])), C(5), A(7));
+    M m3(std::move(m1), A(7));
+    assert(m3 == m2);
+    assert(m3.get_allocator() == A(7));
+    assert(m3.key_comp() == C(5));
+    LIBCPP_ASSERT(m1.empty());
+  }
+  {
+    typedef std::pair<MoveOnly, MoveOnly> V;
+    typedef std::pair<const MoveOnly, MoveOnly> VC;
+    typedef test_less<MoveOnly> C;
+    typedef test_allocator<VC> A;
+    typedef std::multimap<MoveOnly, MoveOnly, C, A> M;
+    typedef std::move_iterator<V*> I;
+    V a1[] = {V(1, 1), V(1, 2), V(1, 3), V(2, 1), V(2, 2), V(2, 3), V(3, 1), V(3, 2), V(3, 3)};
+    M m1(I(a1), I(a1 + sizeof(a1) / sizeof(a1[0])), C(5), A(7));
+    V a2[] = {V(1, 1), V(1, 2), V(1, 3), V(2, 1), V(2, 2), V(2, 3), V(3, 1), V(3, 2), V(3, 3)};
+    M m2(I(a2), I(a2 + sizeof(a2) / sizeof(a2[0])), C(5), A(7));
+    M m3(std::move(m1), A(5));
+    assert(m3 == m2);
+    assert(m3.get_allocator() == A(5));
+    assert(m3.key_comp() == C(5));
+    LIBCPP_ASSERT(m1.empty());
+  }
+  {
+    typedef std::pair<MoveOnly, MoveOnly> V;
+    typedef std::pair<const MoveOnly, MoveOnly> VC;
+    typedef test_less<MoveOnly> C;
+    typedef other_allocator<VC> A;
+    typedef std::multimap<MoveOnly, MoveOnly, C, A> M;
+    typedef std::move_iterator<V*> I;
+    V a1[] = {V(1, 1), V(1, 2), V(1, 3), V(2, 1), V(2, 2), V(2, 3), V(3, 1), V(3, 2), V(3, 3)};
+    M m1(I(a1), I(a1 + sizeof(a1) / sizeof(a1[0])), C(5), A(7));
+    V a2[] = {V(1, 1), V(1, 2), V(1, 3), V(2, 1), V(2, 2), V(2, 3), V(3, 1), V(3, 2), V(3, 3)};
+    M m2(I(a2), I(a2 + sizeof(a2) / sizeof(a2[0])), C(5), A(7));
+    M m3(std::move(m1), A(5));
+    assert(m3 == m2);
+    assert(m3.get_allocator() == A(5));
+    assert(m3.key_comp() == C(5));
+    LIBCPP_ASSERT(m1.empty());
+  }
+  {
+    typedef Counter<int> T;
+    typedef std::pair<int, T> V;
+    typedef std::pair<const int, T> VC;
+    typedef test_allocator<VC> A;
+    typedef std::less<int> C;
+    typedef std::multimap<const int, T, C, A> M;
+    typedef V* I;
+    Counter_base::gConstructed = 0;
     {
-        typedef std::pair<MoveOnly, MoveOnly> V;
-        typedef std::pair<const MoveOnly, MoveOnly> VC;
-        typedef test_less<MoveOnly> C;
-        typedef test_allocator<VC> A;
-        typedef std::multimap<MoveOnly, MoveOnly, C, A> M;
-        typedef std::move_iterator<V*> I;
-        V a1[] =
-        {
-            V(1, 1),
-            V(1, 2),
-            V(1, 3),
-            V(2, 1),
-            V(2, 2),
-            V(2, 3),
-            V(3, 1),
-            V(3, 2),
-            V(3, 3)
-        };
-        M m1(I(a1), I(a1+sizeof(a1)/sizeof(a1[0])), C(5), A(7));
-        V a2[] =
-        {
-            V(1, 1),
-            V(1, 2),
-            V(1, 3),
-            V(2, 1),
-            V(2, 2),
-            V(2, 3),
-            V(3, 1),
-            V(3, 2),
-            V(3, 3)
-        };
-        M m2(I(a2), I(a2+sizeof(a2)/sizeof(a2[0])), C(5), A(7));
-        M m3(std::move(m1), A(7));
-        assert(m3 == m2);
-        assert(m3.get_allocator() == A(7));
-        assert(m3.key_comp() == C(5));
-        LIBCPP_ASSERT(m1.empty());
-    }
-    {
-        typedef std::pair<MoveOnly, MoveOnly> V;
-        typedef std::pair<const MoveOnly, MoveOnly> VC;
-        typedef test_less<MoveOnly> C;
-        typedef test_allocator<VC> A;
-        typedef std::multimap<MoveOnly, MoveOnly, C, A> M;
-        typedef std::move_iterator<V*> I;
-        V a1[] =
-        {
-            V(1, 1),
-            V(1, 2),
-            V(1, 3),
-            V(2, 1),
-            V(2, 2),
-            V(2, 3),
-            V(3, 1),
-            V(3, 2),
-            V(3, 3)
-        };
-        M m1(I(a1), I(a1+sizeof(a1)/sizeof(a1[0])), C(5), A(7));
-        V a2[] =
-        {
-            V(1, 1),
-            V(1, 2),
-            V(1, 3),
-            V(2, 1),
-            V(2, 2),
-            V(2, 3),
-            V(3, 1),
-            V(3, 2),
-            V(3, 3)
-        };
-        M m2(I(a2), I(a2+sizeof(a2)/sizeof(a2[0])), C(5), A(7));
-        M m3(std::move(m1), A(5));
-        assert(m3 == m2);
-        assert(m3.get_allocator() == A(5));
-        assert(m3.key_comp() == C(5));
-        LIBCPP_ASSERT(m1.empty());
-    }
-    {
-        typedef std::pair<MoveOnly, MoveOnly> V;
-        typedef std::pair<const MoveOnly, MoveOnly> VC;
-        typedef test_less<MoveOnly> C;
-        typedef other_allocator<VC> A;
-        typedef std::multimap<MoveOnly, MoveOnly, C, A> M;
-        typedef std::move_iterator<V*> I;
-        V a1[] =
-        {
-            V(1, 1),
-            V(1, 2),
-            V(1, 3),
-            V(2, 1),
-            V(2, 2),
-            V(2, 3),
-            V(3, 1),
-            V(3, 2),
-            V(3, 3)
-        };
-        M m1(I(a1), I(a1+sizeof(a1)/sizeof(a1[0])), C(5), A(7));
-        V a2[] =
-        {
-            V(1, 1),
-            V(1, 2),
-            V(1, 3),
-            V(2, 1),
-            V(2, 2),
-            V(2, 3),
-            V(3, 1),
-            V(3, 2),
-            V(3, 3)
-        };
-        M m2(I(a2), I(a2+sizeof(a2)/sizeof(a2[0])), C(5), A(7));
-        M m3(std::move(m1), A(5));
-        assert(m3 == m2);
-        assert(m3.get_allocator() == A(5));
-        assert(m3.key_comp() == C(5));
-        LIBCPP_ASSERT(m1.empty());
-    }
-    {
-        typedef Counter<int> T;
-        typedef std::pair<int, T> V;
-        typedef std::pair<const int, T> VC;
-        typedef test_allocator<VC> A;
-        typedef std::less<int> C;
-        typedef std::multimap<const int, T, C, A> M;
-        typedef V* I;
-        Counter_base::gConstructed = 0;
-        {
-            V a1[] =
-            {
-                V(1, 1),
-                V(1, 2),
-                V(1, 3),
-                V(2, 1),
-                V(2, 2),
-                V(2, 3),
-                V(3, 1),
-                V(3, 2),
-                V(3, 3)
-            };
-            const std::size_t num = sizeof(a1)/sizeof(a1[0]);
-            assert(Counter_base::gConstructed == num);
+      V a1[]                = {V(1, 1), V(1, 2), V(1, 3), V(2, 1), V(2, 2), V(2, 3), V(3, 1), V(3, 2), V(3, 3)};
+      const std::size_t num = sizeof(a1) / sizeof(a1[0]);
+      assert(Counter_base::gConstructed == num);
 
-            M m1(I(a1), I(a1+num), C(), A());
-            assert(Counter_base::gConstructed == 2*num);
+      M m1(I(a1), I(a1 + num), C(), A());
+      assert(Counter_base::gConstructed == 2 * num);
 
-            M m2(m1);
-            assert(m2 == m1);
-            assert(Counter_base::gConstructed == 3*num);
+      M m2(m1);
+      assert(m2 == m1);
+      assert(Counter_base::gConstructed == 3 * num);
 
-            M m3(std::move(m1), A());
-            assert(m3 == m2);
-            LIBCPP_ASSERT(m1.empty());
-            assert(Counter_base::gConstructed >= (int)(3*num));
-            assert(Counter_base::gConstructed <= (int)(4*num));
+      M m3(std::move(m1), A());
+      assert(m3 == m2);
+      LIBCPP_ASSERT(m1.empty());
+      assert(Counter_base::gConstructed >= (int)(3 * num));
+      assert(Counter_base::gConstructed <= (int)(4 * num));
 
-            {
-            M m4(std::move(m2), A(5));
-            assert(Counter_base::gConstructed >= (int)(3*num));
-            assert(Counter_base::gConstructed <= (int)(5*num));
-            assert(m4 == m3);
-            LIBCPP_ASSERT(m2.empty());
-            }
-            assert(Counter_base::gConstructed >= (int)(2*num));
-            assert(Counter_base::gConstructed <= (int)(4*num));
-        }
-        assert(Counter_base::gConstructed == 0);
-    }
-    {
-        typedef std::pair<MoveOnly, MoveOnly> V;
-        typedef std::pair<const MoveOnly, MoveOnly> VC;
-        typedef test_less<MoveOnly> C;
-        typedef min_allocator<VC> A;
-        typedef std::multimap<MoveOnly, MoveOnly, C, A> M;
-        typedef std::move_iterator<V*> I;
-        V a1[] =
-        {
-            V(1, 1),
-            V(1, 2),
-            V(1, 3),
-            V(2, 1),
-            V(2, 2),
-            V(2, 3),
-            V(3, 1),
-            V(3, 2),
-            V(3, 3)
-        };
-        M m1(I(a1), I(a1+sizeof(a1)/sizeof(a1[0])), C(5), A());
-        V a2[] =
-        {
-            V(1, 1),
-            V(1, 2),
-            V(1, 3),
-            V(2, 1),
-            V(2, 2),
-            V(2, 3),
-            V(3, 1),
-            V(3, 2),
-            V(3, 3)
-        };
-        M m2(I(a2), I(a2+sizeof(a2)/sizeof(a2[0])), C(5), A());
-        M m3(std::move(m1), A());
-        assert(m3 == m2);
-        assert(m3.get_allocator() == A());
-        assert(m3.key_comp() == C(5));
-        LIBCPP_ASSERT(m1.empty());
-    }
-    {
-        typedef std::pair<MoveOnly, MoveOnly> V;
-        typedef std::pair<const MoveOnly, MoveOnly> VC;
-        typedef test_less<MoveOnly> C;
-        typedef explicit_allocator<VC> A;
-        typedef std::multimap<MoveOnly, MoveOnly, C, A> M;
-        typedef std::move_iterator<V*> I;
-        V a1[] =
-        {
-            V(1, 1),
-            V(1, 2),
-            V(1, 3),
-            V(2, 1),
-            V(2, 2),
-            V(2, 3),
-            V(3, 1),
-            V(3, 2),
-            V(3, 3)
-        };
-        M m1(I(a1), I(a1+sizeof(a1)/sizeof(a1[0])), C(5), A{});
-        V a2[] =
-        {
-            V(1, 1),
-            V(1, 2),
-            V(1, 3),
-            V(2, 1),
-            V(2, 2),
-            V(2, 3),
-            V(3, 1),
-            V(3, 2),
-            V(3, 3)
-        };
-        M m2(I(a2), I(a2+sizeof(a2)/sizeof(a2[0])), C(5), A{});
-        M m3(std::move(m1), A{});
-        assert(m3 == m2);
-        assert(m3.get_allocator() == A{});
-        assert(m3.key_comp() == C(5));
-        LIBCPP_ASSERT(m1.empty());
+      {
+        M m4(std::move(m2), A(5));
+        assert(Counter_base::gConstructed >= (int)(3 * num));
+        assert(Counter_base::gConstructed <= (int)(5 * num));
+        assert(m4 == m3);
+        LIBCPP_ASSERT(m2.empty());
+      }
+      assert(Counter_base::gConstructed >= (int)(2 * num));
+      assert(Counter_base::gConstructed <= (int)(4 * num));
     }
+    assert(Counter_base::gConstructed == 0);
+  }
+  {
+    typedef std::pair<MoveOnly, MoveOnly> V;
+    typedef std::pair<const MoveOnly, MoveOnly> VC;
+    typedef test_less<MoveOnly> C;
+    typedef min_allocator<VC> A;
+    typedef std::multimap<MoveOnly, MoveOnly, C, A> M;
+    typedef std::move_iterator<V*> I;
+    V a1[] = {V(1, 1), V(1, 2), V(1, 3), V(2, 1), V(2, 2), V(2, 3), V(3, 1), V(3, 2), V(3, 3)};
+    M m1(I(a1), I(a1 + sizeof(a1) / sizeof(a1[0])), C(5), A());
+    V a2[] = {V(1, 1), V(1, 2), V(1, 3), V(2, 1), V(2, 2), V(2, 3), V(3, 1), V(3, 2), V(3, 3)};
+    M m2(I(a2), I(a2 + sizeof(a2) / sizeof(a2[0])), C(5), A());
+    M m3(std::move(m1), A());
+    assert(m3 == m2);
+    assert(m3.get_allocator() == A());
+    assert(m3.key_comp() == C(5));
+    LIBCPP_ASSERT(m1.empty());
+  }
+  {
+    typedef std::pair<MoveOnly, MoveOnly> V;
+    typedef std::pair<const MoveOnly, MoveOnly> VC;
+    typedef test_less<MoveOnly> C;
+    typedef explicit_allocator<VC> A;
+    typedef std::multimap<MoveOnly, MoveOnly, C, A> M;
+    typedef std::move_iterator<V*> I;
+    V a1[] = {V(1, 1), V(1, 2), V(1, 3), V(2, 1), V(2, 2), V(2, 3), V(3, 1), V(3, 2), V(3, 3)};
+    M m1(I(a1), I(a1 + sizeof(a1) / sizeof(a1[0])), C(5), A{});
+    V a2[] = {V(1, 1), V(1, 2), V(1, 3), V(2, 1), V(2, 2), V(2, 3), V(3, 1), V(3, 2), V(3, 3)};
+    M m2(I(a2), I(a2 + sizeof(a2) / sizeof(a2[0])), C(5), A{});
+    M m3(std::move(m1), A{});
+    assert(m3 == m2);
+    assert(m3.get_allocator() == A{});
+    assert(m3.key_comp() == C(5));
+    LIBCPP_ASSERT(m1.empty());
+  }
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/associative/multimap/multimap.cons/move_assign.pass.cpp b/libcxx/test/std/containers/associative/multimap/multimap.cons/move_assign.pass.cpp
index f0a54b0d7cc65..bbf766eec3e40 100644
--- a/libcxx/test/std/containers/associative/multimap/multimap.cons/move_assign.pass.cpp
+++ b/libcxx/test/std/containers/associative/multimap/multimap.cons/move_assign.pass.cpp
@@ -23,168 +23,79 @@
 #include "test_allocator.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef std::pair<MoveOnly, MoveOnly> V;
-        typedef std::pair<const MoveOnly, MoveOnly> VC;
-        typedef test_less<MoveOnly> C;
-        typedef test_allocator<VC> A;
-        typedef std::multimap<MoveOnly, MoveOnly, C, A> M;
-        typedef std::move_iterator<V*> I;
-        V a1[] =
-        {
-            V(1, 1),
-            V(1, 2),
-            V(1, 3),
-            V(2, 1),
-            V(2, 2),
-            V(2, 3),
-            V(3, 1),
-            V(3, 2),
-            V(3, 3)
-        };
-        M m1(I(a1), I(a1+sizeof(a1)/sizeof(a1[0])), C(5), A(7));
-        V a2[] =
-        {
-            V(1, 1),
-            V(1, 2),
-            V(1, 3),
-            V(2, 1),
-            V(2, 2),
-            V(2, 3),
-            V(3, 1),
-            V(3, 2),
-            V(3, 3)
-        };
-        M m2(I(a2), I(a2+sizeof(a2)/sizeof(a2[0])), C(5), A(7));
-        M m3(C(3), A(7));
-        m3 = std::move(m1);
-        assert(m3 == m2);
-        assert(m3.get_allocator() == A(7));
-        assert(m3.key_comp() == C(5));
-        assert(m1.empty());
-    }
-    {
-        typedef std::pair<MoveOnly, MoveOnly> V;
-        typedef std::pair<const MoveOnly, MoveOnly> VC;
-        typedef test_less<MoveOnly> C;
-        typedef test_allocator<VC> A;
-        typedef std::multimap<MoveOnly, MoveOnly, C, A> M;
-        typedef std::move_iterator<V*> I;
-        V a1[] =
-        {
-            V(1, 1),
-            V(1, 2),
-            V(1, 3),
-            V(2, 1),
-            V(2, 2),
-            V(2, 3),
-            V(3, 1),
-            V(3, 2),
-            V(3, 3)
-        };
-        M m1(I(a1), I(a1+sizeof(a1)/sizeof(a1[0])), C(5), A(7));
-        V a2[] =
-        {
-            V(1, 1),
-            V(1, 2),
-            V(1, 3),
-            V(2, 1),
-            V(2, 2),
-            V(2, 3),
-            V(3, 1),
-            V(3, 2),
-            V(3, 3)
-        };
-        M m2(I(a2), I(a2+sizeof(a2)/sizeof(a2[0])), C(5), A(7));
-        M m3(C(3), A(5));
-        m3 = std::move(m1);
-        assert(m3 == m2);
-        assert(m3.get_allocator() == A(5));
-        assert(m3.key_comp() == C(5));
-        LIBCPP_ASSERT(m1.empty());
-    }
-    {
-        typedef std::pair<MoveOnly, MoveOnly> V;
-        typedef std::pair<const MoveOnly, MoveOnly> VC;
-        typedef test_less<MoveOnly> C;
-        typedef other_allocator<VC> A;
-        typedef std::multimap<MoveOnly, MoveOnly, C, A> M;
-        typedef std::move_iterator<V*> I;
-        V a1[] =
-        {
-            V(1, 1),
-            V(1, 2),
-            V(1, 3),
-            V(2, 1),
-            V(2, 2),
-            V(2, 3),
-            V(3, 1),
-            V(3, 2),
-            V(3, 3)
-        };
-        M m1(I(a1), I(a1+sizeof(a1)/sizeof(a1[0])), C(5), A(7));
-        V a2[] =
-        {
-            V(1, 1),
-            V(1, 2),
-            V(1, 3),
-            V(2, 1),
-            V(2, 2),
-            V(2, 3),
-            V(3, 1),
-            V(3, 2),
-            V(3, 3)
-        };
-        M m2(I(a2), I(a2+sizeof(a2)/sizeof(a2[0])), C(5), A(7));
-        M m3(C(3), A(5));
-        m3 = std::move(m1);
-        assert(m3 == m2);
-        assert(m3.get_allocator() == A(7));
-        assert(m3.key_comp() == C(5));
-        assert(m1.empty());
-    }
-    {
-        typedef std::pair<MoveOnly, MoveOnly> V;
-        typedef std::pair<const MoveOnly, MoveOnly> VC;
-        typedef test_less<MoveOnly> C;
-        typedef min_allocator<VC> A;
-        typedef std::multimap<MoveOnly, MoveOnly, C, A> M;
-        typedef std::move_iterator<V*> I;
-        V a1[] =
-        {
-            V(1, 1),
-            V(1, 2),
-            V(1, 3),
-            V(2, 1),
-            V(2, 2),
-            V(2, 3),
-            V(3, 1),
-            V(3, 2),
-            V(3, 3)
-        };
-        M m1(I(a1), I(a1+sizeof(a1)/sizeof(a1[0])), C(5), A());
-        V a2[] =
-        {
-            V(1, 1),
-            V(1, 2),
-            V(1, 3),
-            V(2, 1),
-            V(2, 2),
-            V(2, 3),
-            V(3, 1),
-            V(3, 2),
-            V(3, 3)
-        };
-        M m2(I(a2), I(a2+sizeof(a2)/sizeof(a2[0])), C(5), A());
-        M m3(C(3), A());
-        m3 = std::move(m1);
-        assert(m3 == m2);
-        assert(m3.get_allocator() == A());
-        assert(m3.key_comp() == C(5));
-        assert(m1.empty());
-    }
+int main(int, char**) {
+  {
+    typedef std::pair<MoveOnly, MoveOnly> V;
+    typedef std::pair<const MoveOnly, MoveOnly> VC;
+    typedef test_less<MoveOnly> C;
+    typedef test_allocator<VC> A;
+    typedef std::multimap<MoveOnly, MoveOnly, C, A> M;
+    typedef std::move_iterator<V*> I;
+    V a1[] = {V(1, 1), V(1, 2), V(1, 3), V(2, 1), V(2, 2), V(2, 3), V(3, 1), V(3, 2), V(3, 3)};
+    M m1(I(a1), I(a1 + sizeof(a1) / sizeof(a1[0])), C(5), A(7));
+    V a2[] = {V(1, 1), V(1, 2), V(1, 3), V(2, 1), V(2, 2), V(2, 3), V(3, 1), V(3, 2), V(3, 3)};
+    M m2(I(a2), I(a2 + sizeof(a2) / sizeof(a2[0])), C(5), A(7));
+    M m3(C(3), A(7));
+    m3 = std::move(m1);
+    assert(m3 == m2);
+    assert(m3.get_allocator() == A(7));
+    assert(m3.key_comp() == C(5));
+    assert(m1.empty());
+  }
+  {
+    typedef std::pair<MoveOnly, MoveOnly> V;
+    typedef std::pair<const MoveOnly, MoveOnly> VC;
+    typedef test_less<MoveOnly> C;
+    typedef test_allocator<VC> A;
+    typedef std::multimap<MoveOnly, MoveOnly, C, A> M;
+    typedef std::move_iterator<V*> I;
+    V a1[] = {V(1, 1), V(1, 2), V(1, 3), V(2, 1), V(2, 2), V(2, 3), V(3, 1), V(3, 2), V(3, 3)};
+    M m1(I(a1), I(a1 + sizeof(a1) / sizeof(a1[0])), C(5), A(7));
+    V a2[] = {V(1, 1), V(1, 2), V(1, 3), V(2, 1), V(2, 2), V(2, 3), V(3, 1), V(3, 2), V(3, 3)};
+    M m2(I(a2), I(a2 + sizeof(a2) / sizeof(a2[0])), C(5), A(7));
+    M m3(C(3), A(5));
+    m3 = std::move(m1);
+    assert(m3 == m2);
+    assert(m3.get_allocator() == A(5));
+    assert(m3.key_comp() == C(5));
+    LIBCPP_ASSERT(m1.empty());
+  }
+  {
+    typedef std::pair<MoveOnly, MoveOnly> V;
+    typedef std::pair<const MoveOnly, MoveOnly> VC;
+    typedef test_less<MoveOnly> C;
+    typedef other_allocator<VC> A;
+    typedef std::multimap<MoveOnly, MoveOnly, C, A> M;
+    typedef std::move_iterator<V*> I;
+    V a1[] = {V(1, 1), V(1, 2), V(1, 3), V(2, 1), V(2, 2), V(2, 3), V(3, 1), V(3, 2), V(3, 3)};
+    M m1(I(a1), I(a1 + sizeof(a1) / sizeof(a1[0])), C(5), A(7));
+    V a2[] = {V(1, 1), V(1, 2), V(1, 3), V(2, 1), V(2, 2), V(2, 3), V(3, 1), V(3, 2), V(3, 3)};
+    M m2(I(a2), I(a2 + sizeof(a2) / sizeof(a2[0])), C(5), A(7));
+    M m3(C(3), A(5));
+    m3 = std::move(m1);
+    assert(m3 == m2);
+    assert(m3.get_allocator() == A(7));
+    assert(m3.key_comp() == C(5));
+    assert(m1.empty());
+  }
+  {
+    typedef std::pair<MoveOnly, MoveOnly> V;
+    typedef std::pair<const MoveOnly, MoveOnly> VC;
+    typedef test_less<MoveOnly> C;
+    typedef min_allocator<VC> A;
+    typedef std::multimap<MoveOnly, MoveOnly, C, A> M;
+    typedef std::move_iterator<V*> I;
+    V a1[] = {V(1, 1), V(1, 2), V(1, 3), V(2, 1), V(2, 2), V(2, 3), V(3, 1), V(3, 2), V(3, 3)};
+    M m1(I(a1), I(a1 + sizeof(a1) / sizeof(a1[0])), C(5), A());
+    V a2[] = {V(1, 1), V(1, 2), V(1, 3), V(2, 1), V(2, 2), V(2, 3), V(3, 1), V(3, 2), V(3, 3)};
+    M m2(I(a2), I(a2 + sizeof(a2) / sizeof(a2[0])), C(5), A());
+    M m3(C(3), A());
+    m3 = std::move(m1);
+    assert(m3 == m2);
+    assert(m3.get_allocator() == A());
+    assert(m3.key_comp() == C(5));
+    assert(m1.empty());
+  }
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/associative/multimap/multimap.cons/move_assign_noexcept.compile.pass.cpp b/libcxx/test/std/containers/associative/multimap/multimap.cons/move_assign_noexcept.compile.pass.cpp
new file mode 100644
index 0000000000000..dd4c6f76fbcb6
--- /dev/null
+++ b/libcxx/test/std/containers/associative/multimap/multimap.cons/move_assign_noexcept.compile.pass.cpp
@@ -0,0 +1,62 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// <map>
+
+// multimap& operator=(multimap&& c)
+//     noexcept(
+//          allocator_type::propagate_on_container_move_assignment::value &&
+//          is_nothrow_move_assignable<allocator_type>::value &&
+//          is_nothrow_move_assignable<key_compare>::value);
+
+// This tests a conforming extension
+
+// UNSUPPORTED: c++03
+
+#include <map>
+
+#include "test_macros.h"
+#include "MoveOnly.h"
+#include "test_allocator.h"
+
+template <class T>
+struct some_comp {
+  using value_type = T;
+  some_comp& operator=(const some_comp&);
+  bool operator()(const T&, const T&) const { return false; }
+};
+
+template <class T>
+struct always_equal_alloc {
+  using value_type = T;
+  always_equal_alloc(const always_equal_alloc&);
+  void allocate(std::size_t);
+};
+
+template <class T>
+struct not_always_equal_alloc {
+  int i;
+  using value_type = T;
+  not_always_equal_alloc(const not_always_equal_alloc&);
+  void allocate(std::size_t);
+};
+
+template <template <class> class Alloc>
+using multimap_alloc =
+    std::multimap<MoveOnly, MoveOnly, std::less<MoveOnly>, Alloc<std::pair<const MoveOnly, MoveOnly>>>;
+
+static_assert(std::is_nothrow_move_assignable<multimap_alloc<std::allocator>>::value, "");
+static_assert(!std::is_nothrow_move_assignable<multimap_alloc<test_allocator>>::value, "");
+#if TEST_STD_VER >= 17
+static_assert(std::is_nothrow_move_assignable<multimap_alloc<always_equal_alloc>>::value, "");
+#endif
+static_assert(!std::is_nothrow_move_assignable<multimap_alloc<not_always_equal_alloc>>::value, "");
+#if defined(_LIBCPP_VERSION)
+static_assert(std::is_nothrow_move_assignable<multimap_alloc<other_allocator>>::value, "");
+#endif // _LIBCPP_VERSION
+static_assert(!std::is_nothrow_move_assignable<std::multimap<int, int, some_comp<int>>>::value, "");
diff --git a/libcxx/test/std/containers/associative/multimap/multimap.cons/move_assign_noexcept.pass.cpp b/libcxx/test/std/containers/associative/multimap/multimap.cons/move_assign_noexcept.pass.cpp
deleted file mode 100644
index 4cd8c19fce5f9..0000000000000
--- a/libcxx/test/std/containers/associative/multimap/multimap.cons/move_assign_noexcept.pass.cpp
+++ /dev/null
@@ -1,59 +0,0 @@
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-// <map>
-
-// multimap& operator=(multimap&& c)
-//     noexcept(
-//          allocator_type::propagate_on_container_move_assignment::value &&
-//          is_nothrow_move_assignable<allocator_type>::value &&
-//          is_nothrow_move_assignable<key_compare>::value);
-
-// This tests a conforming extension
-
-// UNSUPPORTED: c++03
-
-#include <map>
-#include <cassert>
-
-#include "test_macros.h"
-#include "MoveOnly.h"
-#include "test_allocator.h"
-
-template <class T>
-struct some_comp
-{
-    typedef T value_type;
-    some_comp& operator=(const some_comp&);
-    bool operator()(const T&, const T&) const { return false; }
-};
-
-int main(int, char**)
-{
-    typedef std::pair<const MoveOnly, MoveOnly> V;
-    {
-        typedef std::multimap<MoveOnly, MoveOnly> C;
-        static_assert(std::is_nothrow_move_assignable<C>::value, "");
-    }
-    {
-        typedef std::multimap<MoveOnly, MoveOnly, std::less<MoveOnly>, test_allocator<V>> C;
-        static_assert(!std::is_nothrow_move_assignable<C>::value, "");
-    }
-#if defined(_LIBCPP_VERSION)
-    {
-        typedef std::multimap<MoveOnly, MoveOnly, std::less<MoveOnly>, other_allocator<V>> C;
-        static_assert(std::is_nothrow_move_assignable<C>::value, "");
-    }
-#endif // _LIBCPP_VERSION
-    {
-        typedef std::multimap<MoveOnly, MoveOnly, some_comp<MoveOnly>> C;
-        static_assert(!std::is_nothrow_move_assignable<C>::value, "");
-    }
-
-  return 0;
-}
diff --git a/libcxx/test/std/containers/associative/multimap/multimap.cons/move_noexcept.pass.cpp b/libcxx/test/std/containers/associative/multimap/multimap.cons/move_noexcept.pass.cpp
index e31348e67ca2f..2af0324575d9c 100644
--- a/libcxx/test/std/containers/associative/multimap/multimap.cons/move_noexcept.pass.cpp
+++ b/libcxx/test/std/containers/associative/multimap/multimap.cons/move_noexcept.pass.cpp
@@ -24,34 +24,32 @@
 #include "test_allocator.h"
 
 template <class T>
-struct some_comp
-{
-    typedef T value_type;
-    some_comp(const some_comp&);
-    bool operator()(const T&, const T&) const { return false; }
+struct some_comp {
+  typedef T value_type;
+  some_comp(const some_comp&);
+  bool operator()(const T&, const T&) const { return false; }
 };
 
-int main(int, char**)
-{
+int main(int, char**) {
 #if defined(_LIBCPP_VERSION)
-    typedef std::pair<const MoveOnly, MoveOnly> V;
-    {
-        typedef std::multimap<MoveOnly, MoveOnly> C;
-        static_assert(std::is_nothrow_move_constructible<C>::value, "");
-    }
-    {
-        typedef std::multimap<MoveOnly, MoveOnly, std::less<MoveOnly>, test_allocator<V>> C;
-        static_assert(std::is_nothrow_move_constructible<C>::value, "");
-    }
-    {
-        typedef std::multimap<MoveOnly, MoveOnly, std::less<MoveOnly>, other_allocator<V>> C;
-        static_assert(std::is_nothrow_move_constructible<C>::value, "");
-    }
+  typedef std::pair<const MoveOnly, MoveOnly> V;
+  {
+    typedef std::multimap<MoveOnly, MoveOnly> C;
+    static_assert(std::is_nothrow_move_constructible<C>::value, "");
+  }
+  {
+    typedef std::multimap<MoveOnly, MoveOnly, std::less<MoveOnly>, test_allocator<V>> C;
+    static_assert(std::is_nothrow_move_constructible<C>::value, "");
+  }
+  {
+    typedef std::multimap<MoveOnly, MoveOnly, std::less<MoveOnly>, other_allocator<V>> C;
+    static_assert(std::is_nothrow_move_constructible<C>::value, "");
+  }
 #endif // _LIBCPP_VERSION
-    {
-        typedef std::multimap<MoveOnly, MoveOnly, some_comp<MoveOnly>> C;
-        static_assert(!std::is_nothrow_move_constructible<C>::value, "");
-    }
+  {
+    typedef std::multimap<MoveOnly, MoveOnly, some_comp<MoveOnly>> C;
+    static_assert(!std::is_nothrow_move_constructible<C>::value, "");
+  }
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/associative/multimap/multimap.erasure/erase_if.pass.cpp b/libcxx/test/std/containers/associative/multimap/multimap.erasure/erase_if.pass.cpp
index 89ab932f47b00..4391bb36f741f 100644
--- a/libcxx/test/std/containers/associative/multimap/multimap.erasure/erase_if.pass.cpp
+++ b/libcxx/test/std/containers/associative/multimap/multimap.erasure/erase_if.pass.cpp
@@ -21,12 +21,11 @@
 
 using Init = std::initializer_list<int>;
 template <typename M>
-M make (Init vals)
-{
-    M ret;
-    for (int v : vals)
-        ret.emplace(static_cast<typename M::key_type>(v), static_cast<typename M::mapped_type>(v + 10));
-    return ret;
+M make(Init vals) {
+  M ret;
+  for (int v : vals)
+    ret.emplace(static_cast<typename M::key_type>(v), static_cast<typename M::mapped_type>(v + 10));
+  return ret;
 }
 
 template <typename M, typename Pred>
@@ -38,52 +37,50 @@ void test0(Init vals, Pred p, Init expected, std::size_t expected_erased_count)
 }
 
 template <typename S>
-void test()
-{
-    auto is1 = [](auto v) { return v.first == 1;};
-    auto is2 = [](auto v) { return v.first == 2;};
-    auto is3 = [](auto v) { return v.first == 3;};
-    auto is4 = [](auto v) { return v.first == 4;};
-    auto True  = [](auto) { return true; };
-    auto False = [](auto) { return false; };
-
-    test0<S>({}, is1, {}, 0);
-
-    test0<S>({1}, is1, {}, 1);
-    test0<S>({1}, is2, {1}, 0);
-
-    test0<S>({1, 2}, is1, {2}, 1);
-    test0<S>({1, 2}, is2, {1}, 1);
-    test0<S>({1, 2}, is3, {1, 2}, 0);
-    test0<S>({1, 1}, is1, {}, 2);
-    test0<S>({1, 1}, is3, {1, 1}, 0);
-
-    test0<S>({1, 2, 3}, is1, {2, 3}, 1);
-    test0<S>({1, 2, 3}, is2, {1, 3}, 1);
-    test0<S>({1, 2, 3}, is3, {1, 2}, 1);
-    test0<S>({1, 2, 3}, is4, {1, 2, 3}, 0);
-
-    test0<S>({1, 1, 1}, is1, {}, 3);
-    test0<S>({1, 1, 1}, is2, {1, 1, 1}, 0);
-    test0<S>({1, 1, 2}, is1, {2}, 2);
-    test0<S>({1, 1, 2}, is2, {1, 1}, 1);
-    test0<S>({1, 1, 2}, is3, {1, 1, 2}, 0);
-    test0<S>({1, 2, 2}, is1, {2, 2}, 1);
-    test0<S>({1, 2, 2}, is2, {1}, 2);
-    test0<S>({1, 2, 2}, is3, {1, 2, 2}, 0);
-
-    test0<S>({1, 2, 3}, True, {}, 3);
-    test0<S>({1, 2, 3}, False, {1, 2, 3}, 0);
+void test() {
+  auto is1   = [](auto v) { return v.first == 1; };
+  auto is2   = [](auto v) { return v.first == 2; };
+  auto is3   = [](auto v) { return v.first == 3; };
+  auto is4   = [](auto v) { return v.first == 4; };
+  auto True  = [](auto) { return true; };
+  auto False = [](auto) { return false; };
+
+  test0<S>({}, is1, {}, 0);
+
+  test0<S>({1}, is1, {}, 1);
+  test0<S>({1}, is2, {1}, 0);
+
+  test0<S>({1, 2}, is1, {2}, 1);
+  test0<S>({1, 2}, is2, {1}, 1);
+  test0<S>({1, 2}, is3, {1, 2}, 0);
+  test0<S>({1, 1}, is1, {}, 2);
+  test0<S>({1, 1}, is3, {1, 1}, 0);
+
+  test0<S>({1, 2, 3}, is1, {2, 3}, 1);
+  test0<S>({1, 2, 3}, is2, {1, 3}, 1);
+  test0<S>({1, 2, 3}, is3, {1, 2}, 1);
+  test0<S>({1, 2, 3}, is4, {1, 2, 3}, 0);
+
+  test0<S>({1, 1, 1}, is1, {}, 3);
+  test0<S>({1, 1, 1}, is2, {1, 1, 1}, 0);
+  test0<S>({1, 1, 2}, is1, {2}, 2);
+  test0<S>({1, 1, 2}, is2, {1, 1}, 1);
+  test0<S>({1, 1, 2}, is3, {1, 1, 2}, 0);
+  test0<S>({1, 2, 2}, is1, {2, 2}, 1);
+  test0<S>({1, 2, 2}, is2, {1}, 2);
+  test0<S>({1, 2, 2}, is3, {1, 2, 2}, 0);
+
+  test0<S>({1, 2, 3}, True, {}, 3);
+  test0<S>({1, 2, 3}, False, {1, 2, 3}, 0);
 }
 
-int main(int, char**)
-{
-    test<std::multimap<int, int>>();
-    test<std::multimap<int, int, std::less<int>, min_allocator<std::pair<const int, int>>>> ();
-    test<std::multimap<int, int, std::less<int>, test_allocator<std::pair<const int, int>>>> ();
+int main(int, char**) {
+  test<std::multimap<int, int>>();
+  test<std::multimap<int, int, std::less<int>, min_allocator<std::pair<const int, int>>>>();
+  test<std::multimap<int, int, std::less<int>, test_allocator<std::pair<const int, int>>>>();
 
-    test<std::multimap<long, short>>();
-    test<std::multimap<short, double>>();
+  test<std::multimap<long, short>>();
+  test<std::multimap<short, double>>();
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/associative/multimap/multimap.modifiers/clear.pass.cpp b/libcxx/test/std/containers/associative/multimap/multimap.modifiers/clear.pass.cpp
index d05a13b5f191e..08dfcea99cf5b 100644
--- a/libcxx/test/std/containers/associative/multimap/multimap.modifiers/clear.pass.cpp
+++ b/libcxx/test/std/containers/associative/multimap/multimap.modifiers/clear.pass.cpp
@@ -18,49 +18,46 @@
 #include "test_macros.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef std::multimap<int, double> M;
-        typedef std::pair<int, double> P;
-        P ar[] =
-        {
-            P(1, 1.5),
-            P(2, 2.5),
-            P(3, 3.5),
-            P(4, 4.5),
-            P(5, 5.5),
-            P(6, 6.5),
-            P(7, 7.5),
-            P(8, 8.5),
-        };
-        M m(ar, ar + sizeof(ar)/sizeof(ar[0]));
-        assert(m.size() == 8);
-        ASSERT_NOEXCEPT(m.clear());
-        m.clear();
-        assert(m.size() == 0);
-    }
+int main(int, char**) {
+  {
+    typedef std::multimap<int, double> M;
+    typedef std::pair<int, double> P;
+    P ar[] = {
+        P(1, 1.5),
+        P(2, 2.5),
+        P(3, 3.5),
+        P(4, 4.5),
+        P(5, 5.5),
+        P(6, 6.5),
+        P(7, 7.5),
+        P(8, 8.5),
+    };
+    M m(ar, ar + sizeof(ar) / sizeof(ar[0]));
+    assert(m.size() == 8);
+    ASSERT_NOEXCEPT(m.clear());
+    m.clear();
+    assert(m.size() == 0);
+  }
 #if TEST_STD_VER >= 11
-    {
-        typedef std::multimap<int, double, std::less<int>, min_allocator<std::pair<const int, double>>> M;
-        typedef std::pair<int, double> P;
-        P ar[] =
-        {
-            P(1, 1.5),
-            P(2, 2.5),
-            P(3, 3.5),
-            P(4, 4.5),
-            P(5, 5.5),
-            P(6, 6.5),
-            P(7, 7.5),
-            P(8, 8.5),
-        };
-        M m(ar, ar + sizeof(ar)/sizeof(ar[0]));
-        assert(m.size() == 8);
-        ASSERT_NOEXCEPT(m.clear());
-        m.clear();
-        assert(m.size() == 0);
-    }
+  {
+    typedef std::multimap<int, double, std::less<int>, min_allocator<std::pair<const int, double>>> M;
+    typedef std::pair<int, double> P;
+    P ar[] = {
+        P(1, 1.5),
+        P(2, 2.5),
+        P(3, 3.5),
+        P(4, 4.5),
+        P(5, 5.5),
+        P(6, 6.5),
+        P(7, 7.5),
+        P(8, 8.5),
+    };
+    M m(ar, ar + sizeof(ar) / sizeof(ar[0]));
+    assert(m.size() == 8);
+    ASSERT_NOEXCEPT(m.clear());
+    m.clear();
+    assert(m.size() == 0);
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/associative/multimap/multimap.modifiers/emplace.pass.cpp b/libcxx/test/std/containers/associative/multimap/multimap.modifiers/emplace.pass.cpp
index 03c1b0441baa4..9a7526c16adf7 100644
--- a/libcxx/test/std/containers/associative/multimap/multimap.modifiers/emplace.pass.cpp
+++ b/libcxx/test/std/containers/associative/multimap/multimap.modifiers/emplace.pass.cpp
@@ -23,128 +23,117 @@
 #include "DefaultOnly.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef std::multimap<int, DefaultOnly> M;
-        typedef M::iterator R;
-        M m;
-        assert(DefaultOnly::count == 0);
-        R r = m.emplace();
-        assert(r == m.begin());
-        assert(m.size() == 1);
-        assert(m.begin()->first == 0);
-        assert(m.begin()->second == DefaultOnly());
-        assert(DefaultOnly::count == 1);
-        r = m.emplace(std::piecewise_construct, std::forward_as_tuple(1),
-                                                std::forward_as_tuple());
-        assert(r == std::next(m.begin()));
-        assert(m.size() == 2);
-        assert(std::next(m.begin())->first == 1);
-        assert(std::next(m.begin())->second == DefaultOnly());
-        assert(DefaultOnly::count == 2);
-        r = m.emplace(std::piecewise_construct, std::forward_as_tuple(1),
-                                                std::forward_as_tuple());
-        assert(r == std::next(m.begin(), 2));
-        assert(m.size() == 3);
-        assert(std::next(m.begin(), 2)->first == 1);
-        assert(std::next(m.begin(), 2)->second == DefaultOnly());
-        assert(DefaultOnly::count == 3);
-    }
+int main(int, char**) {
+  {
+    typedef std::multimap<int, DefaultOnly> M;
+    typedef M::iterator R;
+    M m;
     assert(DefaultOnly::count == 0);
-    {
-        typedef std::multimap<int, Emplaceable> M;
-        typedef M::iterator R;
-        M m;
-        R r = m.emplace(std::piecewise_construct, std::forward_as_tuple(2),
-                                                  std::forward_as_tuple());
-        assert(r == m.begin());
-        assert(m.size() == 1);
-        assert(m.begin()->first == 2);
-        assert(m.begin()->second == Emplaceable());
-        r = m.emplace(std::piecewise_construct, std::forward_as_tuple(1),
-                                                std::forward_as_tuple(2, 3.5));
-        assert(r == m.begin());
-        assert(m.size() == 2);
-        assert(m.begin()->first == 1);
-        assert(m.begin()->second == Emplaceable(2, 3.5));
-        r = m.emplace(std::piecewise_construct, std::forward_as_tuple(1),
-                                                std::forward_as_tuple(3, 3.5));
-        assert(r == std::next(m.begin()));
-        assert(m.size() == 3);
-        assert(r->first == 1);
-        assert(r->second == Emplaceable(3, 3.5));
-    }
-    {
-        typedef std::multimap<int, double> M;
-        typedef M::iterator R;
-        M m;
-        R r = m.emplace(M::value_type(2, 3.5));
-        assert(r == m.begin());
-        assert(m.size() == 1);
-        assert(m.begin()->first == 2);
-        assert(m.begin()->second == 3.5);
-    }
-    {
-        typedef std::multimap<int, DefaultOnly, std::less<int>, min_allocator<std::pair<const int, DefaultOnly>>> M;
-        typedef M::iterator R;
-        M m;
-        assert(DefaultOnly::count == 0);
-        R r = m.emplace();
-        assert(r == m.begin());
-        assert(m.size() == 1);
-        assert(m.begin()->first == 0);
-        assert(m.begin()->second == DefaultOnly());
-        assert(DefaultOnly::count == 1);
-        r = m.emplace(std::piecewise_construct, std::forward_as_tuple(1),
-                                                std::forward_as_tuple());
-        assert(r == std::next(m.begin()));
-        assert(m.size() == 2);
-        assert(std::next(m.begin())->first == 1);
-        assert(std::next(m.begin())->second == DefaultOnly());
-        assert(DefaultOnly::count == 2);
-        r = m.emplace(std::piecewise_construct, std::forward_as_tuple(1),
-                                                std::forward_as_tuple());
-        assert(r == std::next(m.begin(), 2));
-        assert(m.size() == 3);
-        assert(std::next(m.begin(), 2)->first == 1);
-        assert(std::next(m.begin(), 2)->second == DefaultOnly());
-        assert(DefaultOnly::count == 3);
-    }
+    R r = m.emplace();
+    assert(r == m.begin());
+    assert(m.size() == 1);
+    assert(m.begin()->first == 0);
+    assert(m.begin()->second == DefaultOnly());
+    assert(DefaultOnly::count == 1);
+    r = m.emplace(std::piecewise_construct, std::forward_as_tuple(1), std::forward_as_tuple());
+    assert(r == std::next(m.begin()));
+    assert(m.size() == 2);
+    assert(std::next(m.begin())->first == 1);
+    assert(std::next(m.begin())->second == DefaultOnly());
+    assert(DefaultOnly::count == 2);
+    r = m.emplace(std::piecewise_construct, std::forward_as_tuple(1), std::forward_as_tuple());
+    assert(r == std::next(m.begin(), 2));
+    assert(m.size() == 3);
+    assert(std::next(m.begin(), 2)->first == 1);
+    assert(std::next(m.begin(), 2)->second == DefaultOnly());
+    assert(DefaultOnly::count == 3);
+  }
+  assert(DefaultOnly::count == 0);
+  {
+    typedef std::multimap<int, Emplaceable> M;
+    typedef M::iterator R;
+    M m;
+    R r = m.emplace(std::piecewise_construct, std::forward_as_tuple(2), std::forward_as_tuple());
+    assert(r == m.begin());
+    assert(m.size() == 1);
+    assert(m.begin()->first == 2);
+    assert(m.begin()->second == Emplaceable());
+    r = m.emplace(std::piecewise_construct, std::forward_as_tuple(1), std::forward_as_tuple(2, 3.5));
+    assert(r == m.begin());
+    assert(m.size() == 2);
+    assert(m.begin()->first == 1);
+    assert(m.begin()->second == Emplaceable(2, 3.5));
+    r = m.emplace(std::piecewise_construct, std::forward_as_tuple(1), std::forward_as_tuple(3, 3.5));
+    assert(r == std::next(m.begin()));
+    assert(m.size() == 3);
+    assert(r->first == 1);
+    assert(r->second == Emplaceable(3, 3.5));
+  }
+  {
+    typedef std::multimap<int, double> M;
+    typedef M::iterator R;
+    M m;
+    R r = m.emplace(M::value_type(2, 3.5));
+    assert(r == m.begin());
+    assert(m.size() == 1);
+    assert(m.begin()->first == 2);
+    assert(m.begin()->second == 3.5);
+  }
+  {
+    typedef std::multimap<int, DefaultOnly, std::less<int>, min_allocator<std::pair<const int, DefaultOnly>>> M;
+    typedef M::iterator R;
+    M m;
     assert(DefaultOnly::count == 0);
-    {
-        typedef std::multimap<int, Emplaceable, std::less<int>, min_allocator<std::pair<const int, Emplaceable>>> M;
-        typedef M::iterator R;
-        M m;
-        R r = m.emplace(std::piecewise_construct, std::forward_as_tuple(2),
-                                                  std::forward_as_tuple());
-        assert(r == m.begin());
-        assert(m.size() == 1);
-        assert(m.begin()->first == 2);
-        assert(m.begin()->second == Emplaceable());
-        r = m.emplace(std::piecewise_construct, std::forward_as_tuple(1),
-                                                std::forward_as_tuple(2, 3.5));
-        assert(r == m.begin());
-        assert(m.size() == 2);
-        assert(m.begin()->first == 1);
-        assert(m.begin()->second == Emplaceable(2, 3.5));
-        r = m.emplace(std::piecewise_construct, std::forward_as_tuple(1),
-                                                std::forward_as_tuple(3, 3.5));
-        assert(r == std::next(m.begin()));
-        assert(m.size() == 3);
-        assert(r->first == 1);
-        assert(r->second == Emplaceable(3, 3.5));
-    }
-    {
-        typedef std::multimap<int, double, std::less<int>, min_allocator<std::pair<const int, double>>> M;
-        typedef M::iterator R;
-        M m;
-        R r = m.emplace(M::value_type(2, 3.5));
-        assert(r == m.begin());
-        assert(m.size() == 1);
-        assert(m.begin()->first == 2);
-        assert(m.begin()->second == 3.5);
-    }
+    R r = m.emplace();
+    assert(r == m.begin());
+    assert(m.size() == 1);
+    assert(m.begin()->first == 0);
+    assert(m.begin()->second == DefaultOnly());
+    assert(DefaultOnly::count == 1);
+    r = m.emplace(std::piecewise_construct, std::forward_as_tuple(1), std::forward_as_tuple());
+    assert(r == std::next(m.begin()));
+    assert(m.size() == 2);
+    assert(std::next(m.begin())->first == 1);
+    assert(std::next(m.begin())->second == DefaultOnly());
+    assert(DefaultOnly::count == 2);
+    r = m.emplace(std::piecewise_construct, std::forward_as_tuple(1), std::forward_as_tuple());
+    assert(r == std::next(m.begin(), 2));
+    assert(m.size() == 3);
+    assert(std::next(m.begin(), 2)->first == 1);
+    assert(std::next(m.begin(), 2)->second == DefaultOnly());
+    assert(DefaultOnly::count == 3);
+  }
+  assert(DefaultOnly::count == 0);
+  {
+    typedef std::multimap<int, Emplaceable, std::less<int>, min_allocator<std::pair<const int, Emplaceable>>> M;
+    typedef M::iterator R;
+    M m;
+    R r = m.emplace(std::piecewise_construct, std::forward_as_tuple(2), std::forward_as_tuple());
+    assert(r == m.begin());
+    assert(m.size() == 1);
+    assert(m.begin()->first == 2);
+    assert(m.begin()->second == Emplaceable());
+    r = m.emplace(std::piecewise_construct, std::forward_as_tuple(1), std::forward_as_tuple(2, 3.5));
+    assert(r == m.begin());
+    assert(m.size() == 2);
+    assert(m.begin()->first == 1);
+    assert(m.begin()->second == Emplaceable(2, 3.5));
+    r = m.emplace(std::piecewise_construct, std::forward_as_tuple(1), std::forward_as_tuple(3, 3.5));
+    assert(r == std::next(m.begin()));
+    assert(m.size() == 3);
+    assert(r->first == 1);
+    assert(r->second == Emplaceable(3, 3.5));
+  }
+  {
+    typedef std::multimap<int, double, std::less<int>, min_allocator<std::pair<const int, double>>> M;
+    typedef M::iterator R;
+    M m;
+    R r = m.emplace(M::value_type(2, 3.5));
+    assert(r == m.begin());
+    assert(m.size() == 1);
+    assert(m.begin()->first == 2);
+    assert(m.begin()->second == 3.5);
+  }
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/associative/multimap/multimap.modifiers/emplace_hint.pass.cpp b/libcxx/test/std/containers/associative/multimap/multimap.modifiers/emplace_hint.pass.cpp
index ac8aae1444e1c..8b9081082bb23 100644
--- a/libcxx/test/std/containers/associative/multimap/multimap.modifiers/emplace_hint.pass.cpp
+++ b/libcxx/test/std/containers/associative/multimap/multimap.modifiers/emplace_hint.pass.cpp
@@ -23,138 +23,117 @@
 #include "DefaultOnly.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef std::multimap<int, DefaultOnly> M;
-        typedef M::iterator R;
-        M m;
-        assert(DefaultOnly::count == 0);
-        R r = m.emplace_hint(m.cend());
-        assert(r == m.begin());
-        assert(m.size() == 1);
-        assert(m.begin()->first == 0);
-        assert(m.begin()->second == DefaultOnly());
-        assert(DefaultOnly::count == 1);
-        r = m.emplace_hint(m.cend(), std::piecewise_construct,
-                                       std::forward_as_tuple(1),
-                                       std::forward_as_tuple());
-        assert(r == std::next(m.begin()));
-        assert(m.size() == 2);
-        assert(std::next(m.begin())->first == 1);
-        assert(std::next(m.begin())->second == DefaultOnly());
-        assert(DefaultOnly::count == 2);
-        r = m.emplace_hint(m.cend(), std::piecewise_construct,
-                                       std::forward_as_tuple(1),
-                                       std::forward_as_tuple());
-        assert(r == std::next(m.begin(), 2));
-        assert(m.size() == 3);
-        assert(std::next(m.begin(), 2)->first == 1);
-        assert(std::next(m.begin(), 2)->second == DefaultOnly());
-        assert(DefaultOnly::count == 3);
-    }
+int main(int, char**) {
+  {
+    typedef std::multimap<int, DefaultOnly> M;
+    typedef M::iterator R;
+    M m;
     assert(DefaultOnly::count == 0);
-    {
-        typedef std::multimap<int, Emplaceable> M;
-        typedef M::iterator R;
-        M m;
-        R r = m.emplace_hint(m.cend(), std::piecewise_construct,
-                                       std::forward_as_tuple(2),
-                                       std::forward_as_tuple());
-        assert(r == m.begin());
-        assert(m.size() == 1);
-        assert(m.begin()->first == 2);
-        assert(m.begin()->second == Emplaceable());
-        r = m.emplace_hint(m.cbegin(), std::piecewise_construct,
-                                       std::forward_as_tuple(1),
-                                       std::forward_as_tuple(2, 3.5));
-        assert(r == m.begin());
-        assert(m.size() == 2);
-        assert(m.begin()->first == 1);
-        assert(m.begin()->second == Emplaceable(2, 3.5));
-        r = m.emplace_hint(m.cbegin(), std::piecewise_construct,
-                                       std::forward_as_tuple(1),
-                                       std::forward_as_tuple(3, 3.5));
-        assert(r == m.begin());
-        assert(m.size() == 3);
-        assert(r->first == 1);
-        assert(r->second == Emplaceable(3, 3.5));
-    }
-    {
-        typedef std::multimap<int, double> M;
-        typedef M::iterator R;
-        M m;
-        R r = m.emplace_hint(m.cend(), M::value_type(2, 3.5));
-        assert(r == m.begin());
-        assert(m.size() == 1);
-        assert(m.begin()->first == 2);
-        assert(m.begin()->second == 3.5);
-    }
-    {
-        typedef std::multimap<int, DefaultOnly, std::less<int>, min_allocator<std::pair<const int, DefaultOnly>>> M;
-        typedef M::iterator R;
-        M m;
-        assert(DefaultOnly::count == 0);
-        R r = m.emplace_hint(m.cend());
-        assert(r == m.begin());
-        assert(m.size() == 1);
-        assert(m.begin()->first == 0);
-        assert(m.begin()->second == DefaultOnly());
-        assert(DefaultOnly::count == 1);
-        r = m.emplace_hint(m.cend(), std::piecewise_construct,
-                                       std::forward_as_tuple(1),
-                                       std::forward_as_tuple());
-        assert(r == std::next(m.begin()));
-        assert(m.size() == 2);
-        assert(std::next(m.begin())->first == 1);
-        assert(std::next(m.begin())->second == DefaultOnly());
-        assert(DefaultOnly::count == 2);
-        r = m.emplace_hint(m.cend(), std::piecewise_construct,
-                                       std::forward_as_tuple(1),
-                                       std::forward_as_tuple());
-        assert(r == std::next(m.begin(), 2));
-        assert(m.size() == 3);
-        assert(std::next(m.begin(), 2)->first == 1);
-        assert(std::next(m.begin(), 2)->second == DefaultOnly());
-        assert(DefaultOnly::count == 3);
-    }
+    R r = m.emplace_hint(m.cend());
+    assert(r == m.begin());
+    assert(m.size() == 1);
+    assert(m.begin()->first == 0);
+    assert(m.begin()->second == DefaultOnly());
+    assert(DefaultOnly::count == 1);
+    r = m.emplace_hint(m.cend(), std::piecewise_construct, std::forward_as_tuple(1), std::forward_as_tuple());
+    assert(r == std::next(m.begin()));
+    assert(m.size() == 2);
+    assert(std::next(m.begin())->first == 1);
+    assert(std::next(m.begin())->second == DefaultOnly());
+    assert(DefaultOnly::count == 2);
+    r = m.emplace_hint(m.cend(), std::piecewise_construct, std::forward_as_tuple(1), std::forward_as_tuple());
+    assert(r == std::next(m.begin(), 2));
+    assert(m.size() == 3);
+    assert(std::next(m.begin(), 2)->first == 1);
+    assert(std::next(m.begin(), 2)->second == DefaultOnly());
+    assert(DefaultOnly::count == 3);
+  }
+  assert(DefaultOnly::count == 0);
+  {
+    typedef std::multimap<int, Emplaceable> M;
+    typedef M::iterator R;
+    M m;
+    R r = m.emplace_hint(m.cend(), std::piecewise_construct, std::forward_as_tuple(2), std::forward_as_tuple());
+    assert(r == m.begin());
+    assert(m.size() == 1);
+    assert(m.begin()->first == 2);
+    assert(m.begin()->second == Emplaceable());
+    r = m.emplace_hint(m.cbegin(), std::piecewise_construct, std::forward_as_tuple(1), std::forward_as_tuple(2, 3.5));
+    assert(r == m.begin());
+    assert(m.size() == 2);
+    assert(m.begin()->first == 1);
+    assert(m.begin()->second == Emplaceable(2, 3.5));
+    r = m.emplace_hint(m.cbegin(), std::piecewise_construct, std::forward_as_tuple(1), std::forward_as_tuple(3, 3.5));
+    assert(r == m.begin());
+    assert(m.size() == 3);
+    assert(r->first == 1);
+    assert(r->second == Emplaceable(3, 3.5));
+  }
+  {
+    typedef std::multimap<int, double> M;
+    typedef M::iterator R;
+    M m;
+    R r = m.emplace_hint(m.cend(), M::value_type(2, 3.5));
+    assert(r == m.begin());
+    assert(m.size() == 1);
+    assert(m.begin()->first == 2);
+    assert(m.begin()->second == 3.5);
+  }
+  {
+    typedef std::multimap<int, DefaultOnly, std::less<int>, min_allocator<std::pair<const int, DefaultOnly>>> M;
+    typedef M::iterator R;
+    M m;
     assert(DefaultOnly::count == 0);
-    {
-        typedef std::multimap<int, Emplaceable, std::less<int>, min_allocator<std::pair<const int, Emplaceable>>> M;
-        typedef M::iterator R;
-        M m;
-        R r = m.emplace_hint(m.cend(), std::piecewise_construct,
-                                       std::forward_as_tuple(2),
-                                       std::forward_as_tuple());
-        assert(r == m.begin());
-        assert(m.size() == 1);
-        assert(m.begin()->first == 2);
-        assert(m.begin()->second == Emplaceable());
-        r = m.emplace_hint(m.cbegin(), std::piecewise_construct,
-                                       std::forward_as_tuple(1),
-                                       std::forward_as_tuple(2, 3.5));
-        assert(r == m.begin());
-        assert(m.size() == 2);
-        assert(m.begin()->first == 1);
-        assert(m.begin()->second == Emplaceable(2, 3.5));
-        r = m.emplace_hint(m.cbegin(), std::piecewise_construct,
-                                       std::forward_as_tuple(1),
-                                       std::forward_as_tuple(3, 3.5));
-        assert(r == m.begin());
-        assert(m.size() == 3);
-        assert(r->first == 1);
-        assert(r->second == Emplaceable(3, 3.5));
-    }
-    {
-        typedef std::multimap<int, double, std::less<int>, min_allocator<std::pair<const int, double>>> M;
-        typedef M::iterator R;
-        M m;
-        R r = m.emplace_hint(m.cend(), M::value_type(2, 3.5));
-        assert(r == m.begin());
-        assert(m.size() == 1);
-        assert(m.begin()->first == 2);
-        assert(m.begin()->second == 3.5);
-    }
+    R r = m.emplace_hint(m.cend());
+    assert(r == m.begin());
+    assert(m.size() == 1);
+    assert(m.begin()->first == 0);
+    assert(m.begin()->second == DefaultOnly());
+    assert(DefaultOnly::count == 1);
+    r = m.emplace_hint(m.cend(), std::piecewise_construct, std::forward_as_tuple(1), std::forward_as_tuple());
+    assert(r == std::next(m.begin()));
+    assert(m.size() == 2);
+    assert(std::next(m.begin())->first == 1);
+    assert(std::next(m.begin())->second == DefaultOnly());
+    assert(DefaultOnly::count == 2);
+    r = m.emplace_hint(m.cend(), std::piecewise_construct, std::forward_as_tuple(1), std::forward_as_tuple());
+    assert(r == std::next(m.begin(), 2));
+    assert(m.size() == 3);
+    assert(std::next(m.begin(), 2)->first == 1);
+    assert(std::next(m.begin(), 2)->second == DefaultOnly());
+    assert(DefaultOnly::count == 3);
+  }
+  assert(DefaultOnly::count == 0);
+  {
+    typedef std::multimap<int, Emplaceable, std::less<int>, min_allocator<std::pair<const int, Emplaceable>>> M;
+    typedef M::iterator R;
+    M m;
+    R r = m.emplace_hint(m.cend(), std::piecewise_construct, std::forward_as_tuple(2), std::forward_as_tuple());
+    assert(r == m.begin());
+    assert(m.size() == 1);
+    assert(m.begin()->first == 2);
+    assert(m.begin()->second == Emplaceable());
+    r = m.emplace_hint(m.cbegin(), std::piecewise_construct, std::forward_as_tuple(1), std::forward_as_tuple(2, 3.5));
+    assert(r == m.begin());
+    assert(m.size() == 2);
+    assert(m.begin()->first == 1);
+    assert(m.begin()->second == Emplaceable(2, 3.5));
+    r = m.emplace_hint(m.cbegin(), std::piecewise_construct, std::forward_as_tuple(1), std::forward_as_tuple(3, 3.5));
+    assert(r == m.begin());
+    assert(m.size() == 3);
+    assert(r->first == 1);
+    assert(r->second == Emplaceable(3, 3.5));
+  }
+  {
+    typedef std::multimap<int, double, std::less<int>, min_allocator<std::pair<const int, double>>> M;
+    typedef M::iterator R;
+    M m;
+    R r = m.emplace_hint(m.cend(), M::value_type(2, 3.5));
+    assert(r == m.begin());
+    assert(m.size() == 1);
+    assert(m.begin()->first == 2);
+    assert(m.begin()->second == 3.5);
+  }
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/associative/multimap/multimap.modifiers/erase_iter.pass.cpp b/libcxx/test/std/containers/associative/multimap/multimap.modifiers/erase_iter.pass.cpp
index d958f617d1132..67e957e9e3a72 100644
--- a/libcxx/test/std/containers/associative/multimap/multimap.modifiers/erase_iter.pass.cpp
+++ b/libcxx/test/std/containers/associative/multimap/multimap.modifiers/erase_iter.pass.cpp
@@ -18,285 +18,281 @@
 #include "test_macros.h"
 #include "min_allocator.h"
 
-struct TemplateConstructor
-{
-    template<typename T>
-    TemplateConstructor (const T&) {}
+struct TemplateConstructor {
+  template <typename T>
+  TemplateConstructor(const T&) {}
 };
 
 bool operator<(const TemplateConstructor&, const TemplateConstructor&) { return false; }
 
-int main(int, char**)
-{
-    {
-        typedef std::multimap<int, double> M;
-        typedef std::pair<int, double> P;
-        typedef M::iterator I;
-        P ar[] =
-        {
-            P(1, 1),
-            P(1, 1.5),
-            P(1, 2),
-            P(2, 1),
-            P(2, 1.5),
-            P(2, 2),
-            P(3, 1),
-            P(3, 1.5),
-            P(3, 2),
-        };
-        M m(ar, ar + sizeof(ar)/sizeof(ar[0]));
-        assert(m.size() == 9);
-        I i = m.erase(std::next(m.cbegin(), 3));
-        assert(m.size() == 8);
-        assert(i == std::next(m.begin(), 3));
-        assert(m.begin()->first == 1);
-        assert(m.begin()->second == 1);
-        assert(std::next(m.begin())->first == 1);
-        assert(std::next(m.begin())->second == 1.5);
-        assert(std::next(m.begin(), 2)->first == 1);
-        assert(std::next(m.begin(), 2)->second == 2);
-        assert(std::next(m.begin(), 3)->first == 2);
-        assert(std::next(m.begin(), 3)->second == 1.5);
-        assert(std::next(m.begin(), 4)->first == 2);
-        assert(std::next(m.begin(), 4)->second == 2);
-        assert(std::next(m.begin(), 5)->first == 3);
-        assert(std::next(m.begin(), 5)->second == 1);
-        assert(std::next(m.begin(), 6)->first == 3);
-        assert(std::next(m.begin(), 6)->second == 1.5);
-        assert(std::next(m.begin(), 7)->first == 3);
-        assert(std::next(m.begin(), 7)->second == 2);
+int main(int, char**) {
+  {
+    typedef std::multimap<int, double> M;
+    typedef std::pair<int, double> P;
+    typedef M::iterator I;
+    P ar[] = {
+        P(1, 1),
+        P(1, 1.5),
+        P(1, 2),
+        P(2, 1),
+        P(2, 1.5),
+        P(2, 2),
+        P(3, 1),
+        P(3, 1.5),
+        P(3, 2),
+    };
+    M m(ar, ar + sizeof(ar) / sizeof(ar[0]));
+    assert(m.size() == 9);
+    I i = m.erase(std::next(m.cbegin(), 3));
+    assert(m.size() == 8);
+    assert(i == std::next(m.begin(), 3));
+    assert(m.begin()->first == 1);
+    assert(m.begin()->second == 1);
+    assert(std::next(m.begin())->first == 1);
+    assert(std::next(m.begin())->second == 1.5);
+    assert(std::next(m.begin(), 2)->first == 1);
+    assert(std::next(m.begin(), 2)->second == 2);
+    assert(std::next(m.begin(), 3)->first == 2);
+    assert(std::next(m.begin(), 3)->second == 1.5);
+    assert(std::next(m.begin(), 4)->first == 2);
+    assert(std::next(m.begin(), 4)->second == 2);
+    assert(std::next(m.begin(), 5)->first == 3);
+    assert(std::next(m.begin(), 5)->second == 1);
+    assert(std::next(m.begin(), 6)->first == 3);
+    assert(std::next(m.begin(), 6)->second == 1.5);
+    assert(std::next(m.begin(), 7)->first == 3);
+    assert(std::next(m.begin(), 7)->second == 2);
 
-        i = m.erase(std::next(m.cbegin(), 0));
-        assert(m.size() == 7);
-        assert(i == m.begin());
-        assert(std::next(m.begin(), 0)->first == 1);
-        assert(std::next(m.begin(), 0)->second == 1.5);
-        assert(std::next(m.begin(), 1)->first == 1);
-        assert(std::next(m.begin(), 1)->second == 2);
-        assert(std::next(m.begin(), 2)->first == 2);
-        assert(std::next(m.begin(), 2)->second == 1.5);
-        assert(std::next(m.begin(), 3)->first == 2);
-        assert(std::next(m.begin(), 3)->second == 2);
-        assert(std::next(m.begin(), 4)->first == 3);
-        assert(std::next(m.begin(), 4)->second == 1);
-        assert(std::next(m.begin(), 5)->first == 3);
-        assert(std::next(m.begin(), 5)->second == 1.5);
-        assert(std::next(m.begin(), 6)->first == 3);
-        assert(std::next(m.begin(), 6)->second == 2);
+    i = m.erase(std::next(m.cbegin(), 0));
+    assert(m.size() == 7);
+    assert(i == m.begin());
+    assert(std::next(m.begin(), 0)->first == 1);
+    assert(std::next(m.begin(), 0)->second == 1.5);
+    assert(std::next(m.begin(), 1)->first == 1);
+    assert(std::next(m.begin(), 1)->second == 2);
+    assert(std::next(m.begin(), 2)->first == 2);
+    assert(std::next(m.begin(), 2)->second == 1.5);
+    assert(std::next(m.begin(), 3)->first == 2);
+    assert(std::next(m.begin(), 3)->second == 2);
+    assert(std::next(m.begin(), 4)->first == 3);
+    assert(std::next(m.begin(), 4)->second == 1);
+    assert(std::next(m.begin(), 5)->first == 3);
+    assert(std::next(m.begin(), 5)->second == 1.5);
+    assert(std::next(m.begin(), 6)->first == 3);
+    assert(std::next(m.begin(), 6)->second == 2);
 
-        i = m.erase(std::next(m.cbegin(), 5));
-        assert(m.size() == 6);
-        assert(i == std::prev(m.end()));
-        assert(std::next(m.begin(), 0)->first == 1);
-        assert(std::next(m.begin(), 0)->second == 1.5);
-        assert(std::next(m.begin(), 1)->first == 1);
-        assert(std::next(m.begin(), 1)->second == 2);
-        assert(std::next(m.begin(), 2)->first == 2);
-        assert(std::next(m.begin(), 2)->second == 1.5);
-        assert(std::next(m.begin(), 3)->first == 2);
-        assert(std::next(m.begin(), 3)->second == 2);
-        assert(std::next(m.begin(), 4)->first == 3);
-        assert(std::next(m.begin(), 4)->second == 1);
-        assert(std::next(m.begin(), 5)->first == 3);
-        assert(std::next(m.begin(), 5)->second == 2);
+    i = m.erase(std::next(m.cbegin(), 5));
+    assert(m.size() == 6);
+    assert(i == std::prev(m.end()));
+    assert(std::next(m.begin(), 0)->first == 1);
+    assert(std::next(m.begin(), 0)->second == 1.5);
+    assert(std::next(m.begin(), 1)->first == 1);
+    assert(std::next(m.begin(), 1)->second == 2);
+    assert(std::next(m.begin(), 2)->first == 2);
+    assert(std::next(m.begin(), 2)->second == 1.5);
+    assert(std::next(m.begin(), 3)->first == 2);
+    assert(std::next(m.begin(), 3)->second == 2);
+    assert(std::next(m.begin(), 4)->first == 3);
+    assert(std::next(m.begin(), 4)->second == 1);
+    assert(std::next(m.begin(), 5)->first == 3);
+    assert(std::next(m.begin(), 5)->second == 2);
 
-        i = m.erase(std::next(m.cbegin(), 1));
-        assert(m.size() == 5);
-        assert(i == std::next(m.begin()));
-        assert(std::next(m.begin(), 0)->first == 1);
-        assert(std::next(m.begin(), 0)->second == 1.5);
-        assert(std::next(m.begin(), 1)->first == 2);
-        assert(std::next(m.begin(), 1)->second == 1.5);
-        assert(std::next(m.begin(), 2)->first == 2);
-        assert(std::next(m.begin(), 2)->second == 2);
-        assert(std::next(m.begin(), 3)->first == 3);
-        assert(std::next(m.begin(), 3)->second == 1);
-        assert(std::next(m.begin(), 4)->first == 3);
-        assert(std::next(m.begin(), 4)->second == 2);
+    i = m.erase(std::next(m.cbegin(), 1));
+    assert(m.size() == 5);
+    assert(i == std::next(m.begin()));
+    assert(std::next(m.begin(), 0)->first == 1);
+    assert(std::next(m.begin(), 0)->second == 1.5);
+    assert(std::next(m.begin(), 1)->first == 2);
+    assert(std::next(m.begin(), 1)->second == 1.5);
+    assert(std::next(m.begin(), 2)->first == 2);
+    assert(std::next(m.begin(), 2)->second == 2);
+    assert(std::next(m.begin(), 3)->first == 3);
+    assert(std::next(m.begin(), 3)->second == 1);
+    assert(std::next(m.begin(), 4)->first == 3);
+    assert(std::next(m.begin(), 4)->second == 2);
 
-        i = m.erase(std::next(m.cbegin(), 2));
-        assert(m.size() == 4);
-        assert(i == std::next(m.begin(), 2));
-        assert(std::next(m.begin(), 0)->first == 1);
-        assert(std::next(m.begin(), 0)->second == 1.5);
-        assert(std::next(m.begin(), 1)->first == 2);
-        assert(std::next(m.begin(), 1)->second == 1.5);
-        assert(std::next(m.begin(), 2)->first == 3);
-        assert(std::next(m.begin(), 2)->second == 1);
-        assert(std::next(m.begin(), 3)->first == 3);
-        assert(std::next(m.begin(), 3)->second == 2);
+    i = m.erase(std::next(m.cbegin(), 2));
+    assert(m.size() == 4);
+    assert(i == std::next(m.begin(), 2));
+    assert(std::next(m.begin(), 0)->first == 1);
+    assert(std::next(m.begin(), 0)->second == 1.5);
+    assert(std::next(m.begin(), 1)->first == 2);
+    assert(std::next(m.begin(), 1)->second == 1.5);
+    assert(std::next(m.begin(), 2)->first == 3);
+    assert(std::next(m.begin(), 2)->second == 1);
+    assert(std::next(m.begin(), 3)->first == 3);
+    assert(std::next(m.begin(), 3)->second == 2);
 
-        i = m.erase(std::next(m.cbegin(), 2));
-        assert(m.size() == 3);
-        assert(i == std::next(m.begin(), 2));
-        assert(std::next(m.begin(), 0)->first == 1);
-        assert(std::next(m.begin(), 0)->second == 1.5);
-        assert(std::next(m.begin(), 1)->first == 2);
-        assert(std::next(m.begin(), 1)->second == 1.5);
-        assert(std::next(m.begin(), 2)->first == 3);
-        assert(std::next(m.begin(), 2)->second == 2);
+    i = m.erase(std::next(m.cbegin(), 2));
+    assert(m.size() == 3);
+    assert(i == std::next(m.begin(), 2));
+    assert(std::next(m.begin(), 0)->first == 1);
+    assert(std::next(m.begin(), 0)->second == 1.5);
+    assert(std::next(m.begin(), 1)->first == 2);
+    assert(std::next(m.begin(), 1)->second == 1.5);
+    assert(std::next(m.begin(), 2)->first == 3);
+    assert(std::next(m.begin(), 2)->second == 2);
 
-        i = m.erase(std::next(m.cbegin(), 0));
-        assert(m.size() == 2);
-        assert(i == std::next(m.begin(), 0));
-        assert(std::next(m.begin(), 0)->first == 2);
-        assert(std::next(m.begin(), 0)->second == 1.5);
-        assert(std::next(m.begin(), 1)->first == 3);
-        assert(std::next(m.begin(), 1)->second == 2);
+    i = m.erase(std::next(m.cbegin(), 0));
+    assert(m.size() == 2);
+    assert(i == std::next(m.begin(), 0));
+    assert(std::next(m.begin(), 0)->first == 2);
+    assert(std::next(m.begin(), 0)->second == 1.5);
+    assert(std::next(m.begin(), 1)->first == 3);
+    assert(std::next(m.begin(), 1)->second == 2);
 
-        i = m.erase(std::next(m.cbegin(), 1));
-        assert(m.size() == 1);
-        assert(i == m.end());
-        assert(std::next(m.begin(), 0)->first == 2);
-        assert(std::next(m.begin(), 0)->second == 1.5);
+    i = m.erase(std::next(m.cbegin(), 1));
+    assert(m.size() == 1);
+    assert(i == m.end());
+    assert(std::next(m.begin(), 0)->first == 2);
+    assert(std::next(m.begin(), 0)->second == 1.5);
 
-        i = m.erase(m.cbegin());
-        assert(m.size() == 0);
-        assert(i == m.begin());
-        assert(i == m.end());
-    }
+    i = m.erase(m.cbegin());
+    assert(m.size() == 0);
+    assert(i == m.begin());
+    assert(i == m.end());
+  }
 #if TEST_STD_VER >= 11
-    {
-        typedef std::multimap<int, double, std::less<int>, min_allocator<std::pair<const int, double>>> M;
-        typedef std::pair<int, double> P;
-        typedef M::iterator I;
-        P ar[] =
-        {
-            P(1, 1),
-            P(1, 1.5),
-            P(1, 2),
-            P(2, 1),
-            P(2, 1.5),
-            P(2, 2),
-            P(3, 1),
-            P(3, 1.5),
-            P(3, 2),
-        };
-        M m(ar, ar + sizeof(ar)/sizeof(ar[0]));
-        assert(m.size() == 9);
-        I i = m.erase(std::next(m.cbegin(), 3));
-        assert(m.size() == 8);
-        assert(i == std::next(m.begin(), 3));
-        assert(m.begin()->first == 1);
-        assert(m.begin()->second == 1);
-        assert(std::next(m.begin())->first == 1);
-        assert(std::next(m.begin())->second == 1.5);
-        assert(std::next(m.begin(), 2)->first == 1);
-        assert(std::next(m.begin(), 2)->second == 2);
-        assert(std::next(m.begin(), 3)->first == 2);
-        assert(std::next(m.begin(), 3)->second == 1.5);
-        assert(std::next(m.begin(), 4)->first == 2);
-        assert(std::next(m.begin(), 4)->second == 2);
-        assert(std::next(m.begin(), 5)->first == 3);
-        assert(std::next(m.begin(), 5)->second == 1);
-        assert(std::next(m.begin(), 6)->first == 3);
-        assert(std::next(m.begin(), 6)->second == 1.5);
-        assert(std::next(m.begin(), 7)->first == 3);
-        assert(std::next(m.begin(), 7)->second == 2);
+  {
+    typedef std::multimap<int, double, std::less<int>, min_allocator<std::pair<const int, double>>> M;
+    typedef std::pair<int, double> P;
+    typedef M::iterator I;
+    P ar[] = {
+        P(1, 1),
+        P(1, 1.5),
+        P(1, 2),
+        P(2, 1),
+        P(2, 1.5),
+        P(2, 2),
+        P(3, 1),
+        P(3, 1.5),
+        P(3, 2),
+    };
+    M m(ar, ar + sizeof(ar) / sizeof(ar[0]));
+    assert(m.size() == 9);
+    I i = m.erase(std::next(m.cbegin(), 3));
+    assert(m.size() == 8);
+    assert(i == std::next(m.begin(), 3));
+    assert(m.begin()->first == 1);
+    assert(m.begin()->second == 1);
+    assert(std::next(m.begin())->first == 1);
+    assert(std::next(m.begin())->second == 1.5);
+    assert(std::next(m.begin(), 2)->first == 1);
+    assert(std::next(m.begin(), 2)->second == 2);
+    assert(std::next(m.begin(), 3)->first == 2);
+    assert(std::next(m.begin(), 3)->second == 1.5);
+    assert(std::next(m.begin(), 4)->first == 2);
+    assert(std::next(m.begin(), 4)->second == 2);
+    assert(std::next(m.begin(), 5)->first == 3);
+    assert(std::next(m.begin(), 5)->second == 1);
+    assert(std::next(m.begin(), 6)->first == 3);
+    assert(std::next(m.begin(), 6)->second == 1.5);
+    assert(std::next(m.begin(), 7)->first == 3);
+    assert(std::next(m.begin(), 7)->second == 2);
 
-        i = m.erase(std::next(m.cbegin(), 0));
-        assert(m.size() == 7);
-        assert(i == m.begin());
-        assert(std::next(m.begin(), 0)->first == 1);
-        assert(std::next(m.begin(), 0)->second == 1.5);
-        assert(std::next(m.begin(), 1)->first == 1);
-        assert(std::next(m.begin(), 1)->second == 2);
-        assert(std::next(m.begin(), 2)->first == 2);
-        assert(std::next(m.begin(), 2)->second == 1.5);
-        assert(std::next(m.begin(), 3)->first == 2);
-        assert(std::next(m.begin(), 3)->second == 2);
-        assert(std::next(m.begin(), 4)->first == 3);
-        assert(std::next(m.begin(), 4)->second == 1);
-        assert(std::next(m.begin(), 5)->first == 3);
-        assert(std::next(m.begin(), 5)->second == 1.5);
-        assert(std::next(m.begin(), 6)->first == 3);
-        assert(std::next(m.begin(), 6)->second == 2);
+    i = m.erase(std::next(m.cbegin(), 0));
+    assert(m.size() == 7);
+    assert(i == m.begin());
+    assert(std::next(m.begin(), 0)->first == 1);
+    assert(std::next(m.begin(), 0)->second == 1.5);
+    assert(std::next(m.begin(), 1)->first == 1);
+    assert(std::next(m.begin(), 1)->second == 2);
+    assert(std::next(m.begin(), 2)->first == 2);
+    assert(std::next(m.begin(), 2)->second == 1.5);
+    assert(std::next(m.begin(), 3)->first == 2);
+    assert(std::next(m.begin(), 3)->second == 2);
+    assert(std::next(m.begin(), 4)->first == 3);
+    assert(std::next(m.begin(), 4)->second == 1);
+    assert(std::next(m.begin(), 5)->first == 3);
+    assert(std::next(m.begin(), 5)->second == 1.5);
+    assert(std::next(m.begin(), 6)->first == 3);
+    assert(std::next(m.begin(), 6)->second == 2);
 
-        i = m.erase(std::next(m.cbegin(), 5));
-        assert(m.size() == 6);
-        assert(i == std::prev(m.end()));
-        assert(std::next(m.begin(), 0)->first == 1);
-        assert(std::next(m.begin(), 0)->second == 1.5);
-        assert(std::next(m.begin(), 1)->first == 1);
-        assert(std::next(m.begin(), 1)->second == 2);
-        assert(std::next(m.begin(), 2)->first == 2);
-        assert(std::next(m.begin(), 2)->second == 1.5);
-        assert(std::next(m.begin(), 3)->first == 2);
-        assert(std::next(m.begin(), 3)->second == 2);
-        assert(std::next(m.begin(), 4)->first == 3);
-        assert(std::next(m.begin(), 4)->second == 1);
-        assert(std::next(m.begin(), 5)->first == 3);
-        assert(std::next(m.begin(), 5)->second == 2);
+    i = m.erase(std::next(m.cbegin(), 5));
+    assert(m.size() == 6);
+    assert(i == std::prev(m.end()));
+    assert(std::next(m.begin(), 0)->first == 1);
+    assert(std::next(m.begin(), 0)->second == 1.5);
+    assert(std::next(m.begin(), 1)->first == 1);
+    assert(std::next(m.begin(), 1)->second == 2);
+    assert(std::next(m.begin(), 2)->first == 2);
+    assert(std::next(m.begin(), 2)->second == 1.5);
+    assert(std::next(m.begin(), 3)->first == 2);
+    assert(std::next(m.begin(), 3)->second == 2);
+    assert(std::next(m.begin(), 4)->first == 3);
+    assert(std::next(m.begin(), 4)->second == 1);
+    assert(std::next(m.begin(), 5)->first == 3);
+    assert(std::next(m.begin(), 5)->second == 2);
 
-        i = m.erase(std::next(m.cbegin(), 1));
-        assert(m.size() == 5);
-        assert(i == std::next(m.begin()));
-        assert(std::next(m.begin(), 0)->first == 1);
-        assert(std::next(m.begin(), 0)->second == 1.5);
-        assert(std::next(m.begin(), 1)->first == 2);
-        assert(std::next(m.begin(), 1)->second == 1.5);
-        assert(std::next(m.begin(), 2)->first == 2);
-        assert(std::next(m.begin(), 2)->second == 2);
-        assert(std::next(m.begin(), 3)->first == 3);
-        assert(std::next(m.begin(), 3)->second == 1);
-        assert(std::next(m.begin(), 4)->first == 3);
-        assert(std::next(m.begin(), 4)->second == 2);
+    i = m.erase(std::next(m.cbegin(), 1));
+    assert(m.size() == 5);
+    assert(i == std::next(m.begin()));
+    assert(std::next(m.begin(), 0)->first == 1);
+    assert(std::next(m.begin(), 0)->second == 1.5);
+    assert(std::next(m.begin(), 1)->first == 2);
+    assert(std::next(m.begin(), 1)->second == 1.5);
+    assert(std::next(m.begin(), 2)->first == 2);
+    assert(std::next(m.begin(), 2)->second == 2);
+    assert(std::next(m.begin(), 3)->first == 3);
+    assert(std::next(m.begin(), 3)->second == 1);
+    assert(std::next(m.begin(), 4)->first == 3);
+    assert(std::next(m.begin(), 4)->second == 2);
 
-        i = m.erase(std::next(m.cbegin(), 2));
-        assert(m.size() == 4);
-        assert(i == std::next(m.begin(), 2));
-        assert(std::next(m.begin(), 0)->first == 1);
-        assert(std::next(m.begin(), 0)->second == 1.5);
-        assert(std::next(m.begin(), 1)->first == 2);
-        assert(std::next(m.begin(), 1)->second == 1.5);
-        assert(std::next(m.begin(), 2)->first == 3);
-        assert(std::next(m.begin(), 2)->second == 1);
-        assert(std::next(m.begin(), 3)->first == 3);
-        assert(std::next(m.begin(), 3)->second == 2);
+    i = m.erase(std::next(m.cbegin(), 2));
+    assert(m.size() == 4);
+    assert(i == std::next(m.begin(), 2));
+    assert(std::next(m.begin(), 0)->first == 1);
+    assert(std::next(m.begin(), 0)->second == 1.5);
+    assert(std::next(m.begin(), 1)->first == 2);
+    assert(std::next(m.begin(), 1)->second == 1.5);
+    assert(std::next(m.begin(), 2)->first == 3);
+    assert(std::next(m.begin(), 2)->second == 1);
+    assert(std::next(m.begin(), 3)->first == 3);
+    assert(std::next(m.begin(), 3)->second == 2);
 
-        i = m.erase(std::next(m.cbegin(), 2));
-        assert(m.size() == 3);
-        assert(i == std::next(m.begin(), 2));
-        assert(std::next(m.begin(), 0)->first == 1);
-        assert(std::next(m.begin(), 0)->second == 1.5);
-        assert(std::next(m.begin(), 1)->first == 2);
-        assert(std::next(m.begin(), 1)->second == 1.5);
-        assert(std::next(m.begin(), 2)->first == 3);
-        assert(std::next(m.begin(), 2)->second == 2);
+    i = m.erase(std::next(m.cbegin(), 2));
+    assert(m.size() == 3);
+    assert(i == std::next(m.begin(), 2));
+    assert(std::next(m.begin(), 0)->first == 1);
+    assert(std::next(m.begin(), 0)->second == 1.5);
+    assert(std::next(m.begin(), 1)->first == 2);
+    assert(std::next(m.begin(), 1)->second == 1.5);
+    assert(std::next(m.begin(), 2)->first == 3);
+    assert(std::next(m.begin(), 2)->second == 2);
 
-        i = m.erase(std::next(m.cbegin(), 0));
-        assert(m.size() == 2);
-        assert(i == std::next(m.begin(), 0));
-        assert(std::next(m.begin(), 0)->first == 2);
-        assert(std::next(m.begin(), 0)->second == 1.5);
-        assert(std::next(m.begin(), 1)->first == 3);
-        assert(std::next(m.begin(), 1)->second == 2);
+    i = m.erase(std::next(m.cbegin(), 0));
+    assert(m.size() == 2);
+    assert(i == std::next(m.begin(), 0));
+    assert(std::next(m.begin(), 0)->first == 2);
+    assert(std::next(m.begin(), 0)->second == 1.5);
+    assert(std::next(m.begin(), 1)->first == 3);
+    assert(std::next(m.begin(), 1)->second == 2);
 
-        i = m.erase(std::next(m.cbegin(), 1));
-        assert(m.size() == 1);
-        assert(i == m.end());
-        assert(std::next(m.begin(), 0)->first == 2);
-        assert(std::next(m.begin(), 0)->second == 1.5);
+    i = m.erase(std::next(m.cbegin(), 1));
+    assert(m.size() == 1);
+    assert(i == m.end());
+    assert(std::next(m.begin(), 0)->first == 2);
+    assert(std::next(m.begin(), 0)->second == 1.5);
 
-        i = m.erase(m.cbegin());
-        assert(m.size() == 0);
-        assert(i == m.begin());
-        assert(i == m.end());
-    }
+    i = m.erase(m.cbegin());
+    assert(m.size() == 0);
+    assert(i == m.begin());
+    assert(i == m.end());
+  }
 #endif
 #if TEST_STD_VER >= 14
-    {
+  {
     //  This is LWG #2059
-        typedef TemplateConstructor T;
-        typedef std::multimap<T, int> C;
-        typedef C::iterator I;
+    typedef TemplateConstructor T;
+    typedef std::multimap<T, int> C;
+    typedef C::iterator I;
 
-        C c;
-        T a{0};
-        I it = c.find(a);
-        if (it != c.end())
-            c.erase(it);
-    }
+    C c;
+    T a{0};
+    I it = c.find(a);
+    if (it != c.end())
+      c.erase(it);
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/associative/multimap/multimap.modifiers/erase_iter_iter.pass.cpp b/libcxx/test/std/containers/associative/multimap/multimap.modifiers/erase_iter_iter.pass.cpp
index 68a8f2e35ac65..22a34a19d9897 100644
--- a/libcxx/test/std/containers/associative/multimap/multimap.modifiers/erase_iter_iter.pass.cpp
+++ b/libcxx/test/std/containers/associative/multimap/multimap.modifiers/erase_iter_iter.pass.cpp
@@ -18,141 +18,138 @@
 #include "test_macros.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef std::multimap<int, double> M;
-        typedef std::pair<int, double> P;
-        typedef M::iterator I;
-        P ar[] =
-        {
-            P(1, 1.5),
-            P(2, 2.5),
-            P(3, 3.5),
-            P(4, 4.5),
-            P(5, 5.5),
-            P(6, 6.5),
-            P(7, 7.5),
-            P(8, 8.5),
-        };
-        M m(ar, ar + sizeof(ar)/sizeof(ar[0]));
-        assert(m.size() == 8);
-        I i = m.erase(m.cbegin(), m.cbegin());
-        assert(m.size() == 8);
-        assert(i == m.begin());
-        assert(m.begin()->first == 1);
-        assert(m.begin()->second == 1.5);
-        assert(std::next(m.begin())->first == 2);
-        assert(std::next(m.begin())->second == 2.5);
-        assert(std::next(m.begin(), 2)->first == 3);
-        assert(std::next(m.begin(), 2)->second == 3.5);
-        assert(std::next(m.begin(), 3)->first == 4);
-        assert(std::next(m.begin(), 3)->second == 4.5);
-        assert(std::next(m.begin(), 4)->first == 5);
-        assert(std::next(m.begin(), 4)->second == 5.5);
-        assert(std::next(m.begin(), 5)->first == 6);
-        assert(std::next(m.begin(), 5)->second == 6.5);
-        assert(std::next(m.begin(), 6)->first == 7);
-        assert(std::next(m.begin(), 6)->second == 7.5);
-        assert(std::next(m.begin(), 7)->first == 8);
-        assert(std::next(m.begin(), 7)->second == 8.5);
+int main(int, char**) {
+  {
+    typedef std::multimap<int, double> M;
+    typedef std::pair<int, double> P;
+    typedef M::iterator I;
+    P ar[] = {
+        P(1, 1.5),
+        P(2, 2.5),
+        P(3, 3.5),
+        P(4, 4.5),
+        P(5, 5.5),
+        P(6, 6.5),
+        P(7, 7.5),
+        P(8, 8.5),
+    };
+    M m(ar, ar + sizeof(ar) / sizeof(ar[0]));
+    assert(m.size() == 8);
+    I i = m.erase(m.cbegin(), m.cbegin());
+    assert(m.size() == 8);
+    assert(i == m.begin());
+    assert(m.begin()->first == 1);
+    assert(m.begin()->second == 1.5);
+    assert(std::next(m.begin())->first == 2);
+    assert(std::next(m.begin())->second == 2.5);
+    assert(std::next(m.begin(), 2)->first == 3);
+    assert(std::next(m.begin(), 2)->second == 3.5);
+    assert(std::next(m.begin(), 3)->first == 4);
+    assert(std::next(m.begin(), 3)->second == 4.5);
+    assert(std::next(m.begin(), 4)->first == 5);
+    assert(std::next(m.begin(), 4)->second == 5.5);
+    assert(std::next(m.begin(), 5)->first == 6);
+    assert(std::next(m.begin(), 5)->second == 6.5);
+    assert(std::next(m.begin(), 6)->first == 7);
+    assert(std::next(m.begin(), 6)->second == 7.5);
+    assert(std::next(m.begin(), 7)->first == 8);
+    assert(std::next(m.begin(), 7)->second == 8.5);
 
-        i = m.erase(m.cbegin(), std::next(m.cbegin(), 2));
-        assert(m.size() == 6);
-        assert(i == m.begin());
-        assert(std::next(m.begin(), 0)->first == 3);
-        assert(std::next(m.begin(), 0)->second == 3.5);
-        assert(std::next(m.begin(), 1)->first == 4);
-        assert(std::next(m.begin(), 1)->second == 4.5);
-        assert(std::next(m.begin(), 2)->first == 5);
-        assert(std::next(m.begin(), 2)->second == 5.5);
-        assert(std::next(m.begin(), 3)->first == 6);
-        assert(std::next(m.begin(), 3)->second == 6.5);
-        assert(std::next(m.begin(), 4)->first == 7);
-        assert(std::next(m.begin(), 4)->second == 7.5);
-        assert(std::next(m.begin(), 5)->first == 8);
-        assert(std::next(m.begin(), 5)->second == 8.5);
+    i = m.erase(m.cbegin(), std::next(m.cbegin(), 2));
+    assert(m.size() == 6);
+    assert(i == m.begin());
+    assert(std::next(m.begin(), 0)->first == 3);
+    assert(std::next(m.begin(), 0)->second == 3.5);
+    assert(std::next(m.begin(), 1)->first == 4);
+    assert(std::next(m.begin(), 1)->second == 4.5);
+    assert(std::next(m.begin(), 2)->first == 5);
+    assert(std::next(m.begin(), 2)->second == 5.5);
+    assert(std::next(m.begin(), 3)->first == 6);
+    assert(std::next(m.begin(), 3)->second == 6.5);
+    assert(std::next(m.begin(), 4)->first == 7);
+    assert(std::next(m.begin(), 4)->second == 7.5);
+    assert(std::next(m.begin(), 5)->first == 8);
+    assert(std::next(m.begin(), 5)->second == 8.5);
 
-        i = m.erase(std::next(m.cbegin(), 2), std::next(m.cbegin(), 6));
-        assert(m.size() == 2);
-        assert(i == std::next(m.begin(), 2));
-        assert(std::next(m.begin(), 0)->first == 3);
-        assert(std::next(m.begin(), 0)->second == 3.5);
-        assert(std::next(m.begin(), 1)->first == 4);
-        assert(std::next(m.begin(), 1)->second == 4.5);
+    i = m.erase(std::next(m.cbegin(), 2), std::next(m.cbegin(), 6));
+    assert(m.size() == 2);
+    assert(i == std::next(m.begin(), 2));
+    assert(std::next(m.begin(), 0)->first == 3);
+    assert(std::next(m.begin(), 0)->second == 3.5);
+    assert(std::next(m.begin(), 1)->first == 4);
+    assert(std::next(m.begin(), 1)->second == 4.5);
 
-        i = m.erase(m.cbegin(), m.cend());
-        assert(m.size() == 0);
-        assert(i == m.begin());
-        assert(i == m.end());
-    }
+    i = m.erase(m.cbegin(), m.cend());
+    assert(m.size() == 0);
+    assert(i == m.begin());
+    assert(i == m.end());
+  }
 #if TEST_STD_VER >= 11
-    {
-        typedef std::multimap<int, double, std::less<int>, min_allocator<std::pair<const int, double>>> M;
-        typedef std::pair<int, double> P;
-        typedef M::iterator I;
-        P ar[] =
-        {
-            P(1, 1.5),
-            P(2, 2.5),
-            P(3, 3.5),
-            P(4, 4.5),
-            P(5, 5.5),
-            P(6, 6.5),
-            P(7, 7.5),
-            P(8, 8.5),
-        };
-        M m(ar, ar + sizeof(ar)/sizeof(ar[0]));
-        assert(m.size() == 8);
-        I i = m.erase(m.cbegin(), m.cbegin());
-        assert(m.size() == 8);
-        assert(i == m.begin());
-        assert(m.begin()->first == 1);
-        assert(m.begin()->second == 1.5);
-        assert(std::next(m.begin())->first == 2);
-        assert(std::next(m.begin())->second == 2.5);
-        assert(std::next(m.begin(), 2)->first == 3);
-        assert(std::next(m.begin(), 2)->second == 3.5);
-        assert(std::next(m.begin(), 3)->first == 4);
-        assert(std::next(m.begin(), 3)->second == 4.5);
-        assert(std::next(m.begin(), 4)->first == 5);
-        assert(std::next(m.begin(), 4)->second == 5.5);
-        assert(std::next(m.begin(), 5)->first == 6);
-        assert(std::next(m.begin(), 5)->second == 6.5);
-        assert(std::next(m.begin(), 6)->first == 7);
-        assert(std::next(m.begin(), 6)->second == 7.5);
-        assert(std::next(m.begin(), 7)->first == 8);
-        assert(std::next(m.begin(), 7)->second == 8.5);
+  {
+    typedef std::multimap<int, double, std::less<int>, min_allocator<std::pair<const int, double>>> M;
+    typedef std::pair<int, double> P;
+    typedef M::iterator I;
+    P ar[] = {
+        P(1, 1.5),
+        P(2, 2.5),
+        P(3, 3.5),
+        P(4, 4.5),
+        P(5, 5.5),
+        P(6, 6.5),
+        P(7, 7.5),
+        P(8, 8.5),
+    };
+    M m(ar, ar + sizeof(ar) / sizeof(ar[0]));
+    assert(m.size() == 8);
+    I i = m.erase(m.cbegin(), m.cbegin());
+    assert(m.size() == 8);
+    assert(i == m.begin());
+    assert(m.begin()->first == 1);
+    assert(m.begin()->second == 1.5);
+    assert(std::next(m.begin())->first == 2);
+    assert(std::next(m.begin())->second == 2.5);
+    assert(std::next(m.begin(), 2)->first == 3);
+    assert(std::next(m.begin(), 2)->second == 3.5);
+    assert(std::next(m.begin(), 3)->first == 4);
+    assert(std::next(m.begin(), 3)->second == 4.5);
+    assert(std::next(m.begin(), 4)->first == 5);
+    assert(std::next(m.begin(), 4)->second == 5.5);
+    assert(std::next(m.begin(), 5)->first == 6);
+    assert(std::next(m.begin(), 5)->second == 6.5);
+    assert(std::next(m.begin(), 6)->first == 7);
+    assert(std::next(m.begin(), 6)->second == 7.5);
+    assert(std::next(m.begin(), 7)->first == 8);
+    assert(std::next(m.begin(), 7)->second == 8.5);
 
-        i = m.erase(m.cbegin(), std::next(m.cbegin(), 2));
-        assert(m.size() == 6);
-        assert(i == m.begin());
-        assert(std::next(m.begin(), 0)->first == 3);
-        assert(std::next(m.begin(), 0)->second == 3.5);
-        assert(std::next(m.begin(), 1)->first == 4);
-        assert(std::next(m.begin(), 1)->second == 4.5);
-        assert(std::next(m.begin(), 2)->first == 5);
-        assert(std::next(m.begin(), 2)->second == 5.5);
-        assert(std::next(m.begin(), 3)->first == 6);
-        assert(std::next(m.begin(), 3)->second == 6.5);
-        assert(std::next(m.begin(), 4)->first == 7);
-        assert(std::next(m.begin(), 4)->second == 7.5);
-        assert(std::next(m.begin(), 5)->first == 8);
-        assert(std::next(m.begin(), 5)->second == 8.5);
+    i = m.erase(m.cbegin(), std::next(m.cbegin(), 2));
+    assert(m.size() == 6);
+    assert(i == m.begin());
+    assert(std::next(m.begin(), 0)->first == 3);
+    assert(std::next(m.begin(), 0)->second == 3.5);
+    assert(std::next(m.begin(), 1)->first == 4);
+    assert(std::next(m.begin(), 1)->second == 4.5);
+    assert(std::next(m.begin(), 2)->first == 5);
+    assert(std::next(m.begin(), 2)->second == 5.5);
+    assert(std::next(m.begin(), 3)->first == 6);
+    assert(std::next(m.begin(), 3)->second == 6.5);
+    assert(std::next(m.begin(), 4)->first == 7);
+    assert(std::next(m.begin(), 4)->second == 7.5);
+    assert(std::next(m.begin(), 5)->first == 8);
+    assert(std::next(m.begin(), 5)->second == 8.5);
 
-        i = m.erase(std::next(m.cbegin(), 2), std::next(m.cbegin(), 6));
-        assert(m.size() == 2);
-        assert(i == std::next(m.begin(), 2));
-        assert(std::next(m.begin(), 0)->first == 3);
-        assert(std::next(m.begin(), 0)->second == 3.5);
-        assert(std::next(m.begin(), 1)->first == 4);
-        assert(std::next(m.begin(), 1)->second == 4.5);
+    i = m.erase(std::next(m.cbegin(), 2), std::next(m.cbegin(), 6));
+    assert(m.size() == 2);
+    assert(i == std::next(m.begin(), 2));
+    assert(std::next(m.begin(), 0)->first == 3);
+    assert(std::next(m.begin(), 0)->second == 3.5);
+    assert(std::next(m.begin(), 1)->first == 4);
+    assert(std::next(m.begin(), 1)->second == 4.5);
 
-        i = m.erase(m.cbegin(), m.cend());
-        assert(m.size() == 0);
-        assert(i == m.begin());
-        assert(i == m.end());
-    }
+    i = m.erase(m.cbegin(), m.cend());
+    assert(m.size() == 0);
+    assert(i == m.begin());
+    assert(i == m.end());
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/associative/multimap/multimap.modifiers/erase_key.pass.cpp b/libcxx/test/std/containers/associative/multimap/multimap.modifiers/erase_key.pass.cpp
index f00436db264c9..1dde462f096a1 100644
--- a/libcxx/test/std/containers/associative/multimap/multimap.modifiers/erase_key.pass.cpp
+++ b/libcxx/test/std/containers/associative/multimap/multimap.modifiers/erase_key.pass.cpp
@@ -18,137 +18,134 @@
 #include "test_macros.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef std::multimap<int, double> M;
-        typedef std::pair<int, double> P;
-        typedef M::size_type I;
-        P ar[] =
-        {
-            P(1, 1),
-            P(1, 1.5),
-            P(1, 2),
-            P(2, 1),
-            P(2, 1.5),
-            P(2, 2),
-            P(3, 1),
-            P(3, 1.5),
-            P(3, 2),
-        };
-        M m(ar, ar + sizeof(ar)/sizeof(ar[0]));
-        assert(m.size() == 9);
-        I i = m.erase(2);
-        assert(m.size() == 6);
-        assert(i == 3);
-        assert(std::next(m.begin(), 0)->first == 1);
-        assert(std::next(m.begin(), 0)->second == 1);
-        assert(std::next(m.begin(), 1)->first == 1);
-        assert(std::next(m.begin(), 1)->second == 1.5);
-        assert(std::next(m.begin(), 2)->first == 1);
-        assert(std::next(m.begin(), 2)->second == 2);
-        assert(std::next(m.begin(), 3)->first == 3);
-        assert(std::next(m.begin(), 3)->second == 1);
-        assert(std::next(m.begin(), 4)->first == 3);
-        assert(std::next(m.begin(), 4)->second == 1.5);
-        assert(std::next(m.begin(), 5)->first == 3);
-        assert(std::next(m.begin(), 5)->second == 2);
+int main(int, char**) {
+  {
+    typedef std::multimap<int, double> M;
+    typedef std::pair<int, double> P;
+    typedef M::size_type I;
+    P ar[] = {
+        P(1, 1),
+        P(1, 1.5),
+        P(1, 2),
+        P(2, 1),
+        P(2, 1.5),
+        P(2, 2),
+        P(3, 1),
+        P(3, 1.5),
+        P(3, 2),
+    };
+    M m(ar, ar + sizeof(ar) / sizeof(ar[0]));
+    assert(m.size() == 9);
+    I i = m.erase(2);
+    assert(m.size() == 6);
+    assert(i == 3);
+    assert(std::next(m.begin(), 0)->first == 1);
+    assert(std::next(m.begin(), 0)->second == 1);
+    assert(std::next(m.begin(), 1)->first == 1);
+    assert(std::next(m.begin(), 1)->second == 1.5);
+    assert(std::next(m.begin(), 2)->first == 1);
+    assert(std::next(m.begin(), 2)->second == 2);
+    assert(std::next(m.begin(), 3)->first == 3);
+    assert(std::next(m.begin(), 3)->second == 1);
+    assert(std::next(m.begin(), 4)->first == 3);
+    assert(std::next(m.begin(), 4)->second == 1.5);
+    assert(std::next(m.begin(), 5)->first == 3);
+    assert(std::next(m.begin(), 5)->second == 2);
 
-        i = m.erase(2);
-        assert(m.size() == 6);
-        assert(i == 0);
-        assert(std::next(m.begin(), 0)->first == 1);
-        assert(std::next(m.begin(), 0)->second == 1);
-        assert(std::next(m.begin(), 1)->first == 1);
-        assert(std::next(m.begin(), 1)->second == 1.5);
-        assert(std::next(m.begin(), 2)->first == 1);
-        assert(std::next(m.begin(), 2)->second == 2);
-        assert(std::next(m.begin(), 3)->first == 3);
-        assert(std::next(m.begin(), 3)->second == 1);
-        assert(std::next(m.begin(), 4)->first == 3);
-        assert(std::next(m.begin(), 4)->second == 1.5);
-        assert(std::next(m.begin(), 5)->first == 3);
-        assert(std::next(m.begin(), 5)->second == 2);
+    i = m.erase(2);
+    assert(m.size() == 6);
+    assert(i == 0);
+    assert(std::next(m.begin(), 0)->first == 1);
+    assert(std::next(m.begin(), 0)->second == 1);
+    assert(std::next(m.begin(), 1)->first == 1);
+    assert(std::next(m.begin(), 1)->second == 1.5);
+    assert(std::next(m.begin(), 2)->first == 1);
+    assert(std::next(m.begin(), 2)->second == 2);
+    assert(std::next(m.begin(), 3)->first == 3);
+    assert(std::next(m.begin(), 3)->second == 1);
+    assert(std::next(m.begin(), 4)->first == 3);
+    assert(std::next(m.begin(), 4)->second == 1.5);
+    assert(std::next(m.begin(), 5)->first == 3);
+    assert(std::next(m.begin(), 5)->second == 2);
 
-        i = m.erase(3);
-        assert(i == 3);
-        assert(m.size() == 3);
-        assert(std::next(m.begin(), 0)->first == 1);
-        assert(std::next(m.begin(), 0)->second == 1);
-        assert(std::next(m.begin(), 1)->first == 1);
-        assert(std::next(m.begin(), 1)->second == 1.5);
-        assert(std::next(m.begin(), 2)->first == 1);
-        assert(std::next(m.begin(), 2)->second == 2);
+    i = m.erase(3);
+    assert(i == 3);
+    assert(m.size() == 3);
+    assert(std::next(m.begin(), 0)->first == 1);
+    assert(std::next(m.begin(), 0)->second == 1);
+    assert(std::next(m.begin(), 1)->first == 1);
+    assert(std::next(m.begin(), 1)->second == 1.5);
+    assert(std::next(m.begin(), 2)->first == 1);
+    assert(std::next(m.begin(), 2)->second == 2);
 
-        i = m.erase(1);
-        assert(m.size() == 0);
-        assert(i == 3);
-    }
+    i = m.erase(1);
+    assert(m.size() == 0);
+    assert(i == 3);
+  }
 #if TEST_STD_VER >= 11
-    {
-        typedef std::multimap<int, double, std::less<int>, min_allocator<std::pair<const int, double>>> M;
-        typedef std::pair<int, double> P;
-        typedef M::size_type I;
-        P ar[] =
-        {
-            P(1, 1),
-            P(1, 1.5),
-            P(1, 2),
-            P(2, 1),
-            P(2, 1.5),
-            P(2, 2),
-            P(3, 1),
-            P(3, 1.5),
-            P(3, 2),
-        };
-        M m(ar, ar + sizeof(ar)/sizeof(ar[0]));
-        assert(m.size() == 9);
-        I i = m.erase(2);
-        assert(m.size() == 6);
-        assert(i == 3);
-        assert(std::next(m.begin(), 0)->first == 1);
-        assert(std::next(m.begin(), 0)->second == 1);
-        assert(std::next(m.begin(), 1)->first == 1);
-        assert(std::next(m.begin(), 1)->second == 1.5);
-        assert(std::next(m.begin(), 2)->first == 1);
-        assert(std::next(m.begin(), 2)->second == 2);
-        assert(std::next(m.begin(), 3)->first == 3);
-        assert(std::next(m.begin(), 3)->second == 1);
-        assert(std::next(m.begin(), 4)->first == 3);
-        assert(std::next(m.begin(), 4)->second == 1.5);
-        assert(std::next(m.begin(), 5)->first == 3);
-        assert(std::next(m.begin(), 5)->second == 2);
+  {
+    typedef std::multimap<int, double, std::less<int>, min_allocator<std::pair<const int, double>>> M;
+    typedef std::pair<int, double> P;
+    typedef M::size_type I;
+    P ar[] = {
+        P(1, 1),
+        P(1, 1.5),
+        P(1, 2),
+        P(2, 1),
+        P(2, 1.5),
+        P(2, 2),
+        P(3, 1),
+        P(3, 1.5),
+        P(3, 2),
+    };
+    M m(ar, ar + sizeof(ar) / sizeof(ar[0]));
+    assert(m.size() == 9);
+    I i = m.erase(2);
+    assert(m.size() == 6);
+    assert(i == 3);
+    assert(std::next(m.begin(), 0)->first == 1);
+    assert(std::next(m.begin(), 0)->second == 1);
+    assert(std::next(m.begin(), 1)->first == 1);
+    assert(std::next(m.begin(), 1)->second == 1.5);
+    assert(std::next(m.begin(), 2)->first == 1);
+    assert(std::next(m.begin(), 2)->second == 2);
+    assert(std::next(m.begin(), 3)->first == 3);
+    assert(std::next(m.begin(), 3)->second == 1);
+    assert(std::next(m.begin(), 4)->first == 3);
+    assert(std::next(m.begin(), 4)->second == 1.5);
+    assert(std::next(m.begin(), 5)->first == 3);
+    assert(std::next(m.begin(), 5)->second == 2);
 
-        i = m.erase(2);
-        assert(m.size() == 6);
-        assert(i == 0);
-        assert(std::next(m.begin(), 0)->first == 1);
-        assert(std::next(m.begin(), 0)->second == 1);
-        assert(std::next(m.begin(), 1)->first == 1);
-        assert(std::next(m.begin(), 1)->second == 1.5);
-        assert(std::next(m.begin(), 2)->first == 1);
-        assert(std::next(m.begin(), 2)->second == 2);
-        assert(std::next(m.begin(), 3)->first == 3);
-        assert(std::next(m.begin(), 3)->second == 1);
-        assert(std::next(m.begin(), 4)->first == 3);
-        assert(std::next(m.begin(), 4)->second == 1.5);
-        assert(std::next(m.begin(), 5)->first == 3);
-        assert(std::next(m.begin(), 5)->second == 2);
+    i = m.erase(2);
+    assert(m.size() == 6);
+    assert(i == 0);
+    assert(std::next(m.begin(), 0)->first == 1);
+    assert(std::next(m.begin(), 0)->second == 1);
+    assert(std::next(m.begin(), 1)->first == 1);
+    assert(std::next(m.begin(), 1)->second == 1.5);
+    assert(std::next(m.begin(), 2)->first == 1);
+    assert(std::next(m.begin(), 2)->second == 2);
+    assert(std::next(m.begin(), 3)->first == 3);
+    assert(std::next(m.begin(), 3)->second == 1);
+    assert(std::next(m.begin(), 4)->first == 3);
+    assert(std::next(m.begin(), 4)->second == 1.5);
+    assert(std::next(m.begin(), 5)->first == 3);
+    assert(std::next(m.begin(), 5)->second == 2);
 
-        i = m.erase(3);
-        assert(i == 3);
-        assert(m.size() == 3);
-        assert(std::next(m.begin(), 0)->first == 1);
-        assert(std::next(m.begin(), 0)->second == 1);
-        assert(std::next(m.begin(), 1)->first == 1);
-        assert(std::next(m.begin(), 1)->second == 1.5);
-        assert(std::next(m.begin(), 2)->first == 1);
-        assert(std::next(m.begin(), 2)->second == 2);
+    i = m.erase(3);
+    assert(i == 3);
+    assert(m.size() == 3);
+    assert(std::next(m.begin(), 0)->first == 1);
+    assert(std::next(m.begin(), 0)->second == 1);
+    assert(std::next(m.begin(), 1)->first == 1);
+    assert(std::next(m.begin(), 1)->second == 1.5);
+    assert(std::next(m.begin(), 2)->first == 1);
+    assert(std::next(m.begin(), 2)->second == 2);
 
-        i = m.erase(1);
-        assert(m.size() == 0);
-        assert(i == 3);
-    }
+    i = m.erase(1);
+    assert(m.size() == 0);
+    assert(i == 3);
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/associative/multimap/multimap.modifiers/extract_iterator.pass.cpp b/libcxx/test/std/containers/associative/multimap/multimap.modifiers/extract_iterator.pass.cpp
index c4751d81ec953..85beba3015e56 100644
--- a/libcxx/test/std/containers/associative/multimap/multimap.modifiers/extract_iterator.pass.cpp
+++ b/libcxx/test/std/containers/associative/multimap/multimap.modifiers/extract_iterator.pass.cpp
@@ -20,50 +20,44 @@
 #include "Counter.h"
 
 template <class Container>
-void test(Container& c)
-{
-    std::size_t sz = c.size();
-
-    auto some_key = c.cbegin()->first;
-
-    for (auto first = c.cbegin(); first != c.cend();)
-    {
-        auto key_value = first->first;
-        typename Container::node_type t = c.extract(first++);
-        --sz;
-        assert(t.key() == key_value);
-        t.key() = some_key;
-        assert(t.key() == some_key);
-        assert(t.get_allocator() == c.get_allocator());
-        assert(sz == c.size());
-    }
-
-    assert(c.size() == 0);
+void test(Container& c) {
+  std::size_t sz = c.size();
+
+  auto some_key = c.cbegin()->first;
+
+  for (auto first = c.cbegin(); first != c.cend();) {
+    auto key_value                  = first->first;
+    typename Container::node_type t = c.extract(first++);
+    --sz;
+    assert(t.key() == key_value);
+    t.key() = some_key;
+    assert(t.key() == some_key);
+    assert(t.get_allocator() == c.get_allocator());
+    assert(sz == c.size());
+  }
+
+  assert(c.size() == 0);
 }
 
-int main(int, char**)
-{
-    {
-        using map_type = std::multimap<int, int>;
-        map_type m = {{1,1}, {2,2}, {3,3}, {4,4}, {5,5}, {6,6}};
-        test(m);
-    }
-
-    {
-        std::multimap<Counter<int>, Counter<int>> m =
-            {{1,1}, {2,2}, {3,3}, {4,4}, {5,5}, {6,6}};
-        assert(Counter_base::gConstructed == 12);
-        test(m);
-        assert(Counter_base::gConstructed == 0);
-    }
-
-    {
-        using min_alloc_map =
-            std::multimap<int, int, std::less<int>,
-                     min_allocator<std::pair<const int, int>>>;
-        min_alloc_map m = {{1, 1}, {2, 2}, {3, 3}, {4, 4}, {5, 5}, {6, 6}};
-        test(m);
-    }
+int main(int, char**) {
+  {
+    using map_type = std::multimap<int, int>;
+    map_type m     = {{1, 1}, {2, 2}, {3, 3}, {4, 4}, {5, 5}, {6, 6}};
+    test(m);
+  }
+
+  {
+    std::multimap<Counter<int>, Counter<int>> m = {{1, 1}, {2, 2}, {3, 3}, {4, 4}, {5, 5}, {6, 6}};
+    assert(Counter_base::gConstructed == 12);
+    test(m);
+    assert(Counter_base::gConstructed == 0);
+  }
+
+  {
+    using min_alloc_map = std::multimap<int, int, std::less<int>, min_allocator<std::pair<const int, int>>>;
+    min_alloc_map m     = {{1, 1}, {2, 2}, {3, 3}, {4, 4}, {5, 5}, {6, 6}};
+    test(m);
+  }
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/associative/multimap/multimap.modifiers/extract_key.pass.cpp b/libcxx/test/std/containers/associative/multimap/multimap.modifiers/extract_key.pass.cpp
index b558f4e3f1207..b3027b3b0f727 100644
--- a/libcxx/test/std/containers/associative/multimap/multimap.modifiers/extract_key.pass.cpp
+++ b/libcxx/test/std/containers/associative/multimap/multimap.modifiers/extract_key.pass.cpp
@@ -20,59 +20,52 @@
 #include "Counter.h"
 
 template <class Container, class KeyTypeIter>
-void test(Container& c, KeyTypeIter first, KeyTypeIter last)
-{
-    std::size_t sz = c.size();
-    assert((std::size_t)std::distance(first, last) == sz);
+void test(Container& c, KeyTypeIter first, KeyTypeIter last) {
+  std::size_t sz = c.size();
+  assert((std::size_t)std::distance(first, last) == sz);
 
-    for (KeyTypeIter copy = first; copy != last; ++copy)
-    {
-        typename Container::node_type t = c.extract(*copy);
-        assert(!t.empty());
-        --sz;
-        assert(t.key() == *copy);
-        t.key() = *first; // We should be able to mutate key.
-        assert(t.key() == *first);
-        assert(t.get_allocator() == c.get_allocator());
-        assert(sz == c.size());
-    }
+  for (KeyTypeIter copy = first; copy != last; ++copy) {
+    typename Container::node_type t = c.extract(*copy);
+    assert(!t.empty());
+    --sz;
+    assert(t.key() == *copy);
+    t.key() = *first; // We should be able to mutate key.
+    assert(t.key() == *first);
+    assert(t.get_allocator() == c.get_allocator());
+    assert(sz == c.size());
+  }
 
-    assert(c.size() == 0);
+  assert(c.size() == 0);
 
-    for (KeyTypeIter copy = first; copy != last; ++copy)
-    {
-        typename Container::node_type t = c.extract(*copy);
-        assert(t.empty());
-    }
+  for (KeyTypeIter copy = first; copy != last; ++copy) {
+    typename Container::node_type t = c.extract(*copy);
+    assert(t.empty());
+  }
 }
 
-int main(int, char**)
-{
-    {
-        std::multimap<int, int> m = {{1,1}, {2,2}, {3,3}, {4,4}, {5,5}, {6,6}};
-        int keys[] = {1, 2, 3, 4, 5, 6};
-        test(m, std::begin(keys), std::end(keys));
-    }
+int main(int, char**) {
+  {
+    std::multimap<int, int> m = {{1, 1}, {2, 2}, {3, 3}, {4, 4}, {5, 5}, {6, 6}};
+    int keys[]                = {1, 2, 3, 4, 5, 6};
+    test(m, std::begin(keys), std::end(keys));
+  }
 
+  {
+    std::multimap<Counter<int>, Counter<int>> m = {{1, 1}, {2, 2}, {3, 3}, {4, 4}, {5, 5}, {6, 6}};
     {
-        std::multimap<Counter<int>, Counter<int>> m =
-            {{1,1}, {2,2}, {3,3}, {4,4}, {5,5}, {6,6}};
-        {
-            Counter<int> keys[] = {1, 2, 3, 4, 5, 6};
-            assert(Counter_base::gConstructed == 12+6);
-            test(m, std::begin(keys), std::end(keys));
-        }
-        assert(Counter_base::gConstructed == 0);
+      Counter<int> keys[] = {1, 2, 3, 4, 5, 6};
+      assert(Counter_base::gConstructed == 12 + 6);
+      test(m, std::begin(keys), std::end(keys));
     }
+    assert(Counter_base::gConstructed == 0);
+  }
 
-    {
-        using min_alloc_map =
-            std::multimap<int, int, std::less<int>,
-                          min_allocator<std::pair<const int, int>>>;
-        min_alloc_map m = {{1, 1}, {2, 2}, {3, 3}, {4, 4}, {5, 5}, {6, 6}};
-        int keys[] = {1, 2, 3, 4, 5, 6};
-        test(m, std::begin(keys), std::end(keys));
-    }
+  {
+    using min_alloc_map = std::multimap<int, int, std::less<int>, min_allocator<std::pair<const int, int>>>;
+    min_alloc_map m     = {{1, 1}, {2, 2}, {3, 3}, {4, 4}, {5, 5}, {6, 6}};
+    int keys[]          = {1, 2, 3, 4, 5, 6};
+    test(m, std::begin(keys), std::end(keys));
+  }
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/associative/multimap/multimap.modifiers/insert_allocator_requirements.pass.cpp b/libcxx/test/std/containers/associative/multimap/multimap.modifiers/insert_allocator_requirements.pass.cpp
index 446e3529adb7d..68767d002d0ca 100644
--- a/libcxx/test/std/containers/associative/multimap/multimap.modifiers/insert_allocator_requirements.pass.cpp
+++ b/libcxx/test/std/containers/associative/multimap/multimap.modifiers/insert_allocator_requirements.pass.cpp
@@ -20,9 +20,7 @@
 #include "container_test_types.h"
 #include "../../../map_allocator_requirement_test_templates.h"
 
-
-int main(int, char**)
-{
+int main(int, char**) {
   testMultimapInsert<TCT::multimap<> >();
   testMultimapInsertHint<TCT::multimap<> >();
 
diff --git a/libcxx/test/std/containers/associative/multimap/multimap.modifiers/insert_cv.pass.cpp b/libcxx/test/std/containers/associative/multimap/multimap.modifiers/insert_cv.pass.cpp
index f34016303fbe6..a6869a73e1e04 100644
--- a/libcxx/test/std/containers/associative/multimap/multimap.modifiers/insert_cv.pass.cpp
+++ b/libcxx/test/std/containers/associative/multimap/multimap.modifiers/insert_cv.pass.cpp
@@ -20,50 +20,49 @@
 
 template <class Container>
 void do_insert_test() {
-    typedef Container M;
-    typedef typename M::iterator R;
-    typedef typename M::value_type VT;
-    M m;
-    const VT v1(2, 2.5);
-    R r = m.insert(v1);
-    assert(r == m.begin());
-    assert(m.size() == 1);
-    assert(r->first == 2);
-    assert(r->second == 2.5);
+  typedef Container M;
+  typedef typename M::iterator R;
+  typedef typename M::value_type VT;
+  M m;
+  const VT v1(2, 2.5);
+  R r = m.insert(v1);
+  assert(r == m.begin());
+  assert(m.size() == 1);
+  assert(r->first == 2);
+  assert(r->second == 2.5);
 
-    const VT v2(1, 1.5);
-    r = m.insert(v2);
-    assert(r == m.begin());
-    assert(m.size() == 2);
-    assert(r->first == 1);
-    assert(r->second == 1.5);
+  const VT v2(1, 1.5);
+  r = m.insert(v2);
+  assert(r == m.begin());
+  assert(m.size() == 2);
+  assert(r->first == 1);
+  assert(r->second == 1.5);
 
-    const VT v3(3, 3.5);
-    r = m.insert(v3);
-    assert(r == std::prev(m.end()));
-    assert(m.size() == 3);
-    assert(r->first == 3);
-    assert(r->second == 3.5);
+  const VT v3(3, 3.5);
+  r = m.insert(v3);
+  assert(r == std::prev(m.end()));
+  assert(m.size() == 3);
+  assert(r->first == 3);
+  assert(r->second == 3.5);
 
-    const VT v4(3, 3.5);
-    r = m.insert(v4);
-    assert(r == std::prev(m.end()));
-    assert(m.size() == 4);
-    assert(r->first == 3);
-    assert(r->second == 3.5);
+  const VT v4(3, 3.5);
+  r = m.insert(v4);
+  assert(r == std::prev(m.end()));
+  assert(m.size() == 4);
+  assert(r->first == 3);
+  assert(r->second == 3.5);
 }
 
-int main(int, char**)
-{
-    {
-        typedef std::multimap<int, double> Container;
-        do_insert_test<Container>();
-    }
+int main(int, char**) {
+  {
+    typedef std::multimap<int, double> Container;
+    do_insert_test<Container>();
+  }
 #if TEST_STD_VER >= 11
-    {
-        typedef std::multimap<int, double, std::less<int>, min_allocator<std::pair<const int, double>>> Container;
-        do_insert_test<Container>();
-    }
+  {
+    typedef std::multimap<int, double, std::less<int>, min_allocator<std::pair<const int, double>>> Container;
+    do_insert_test<Container>();
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/associative/multimap/multimap.modifiers/insert_initializer_list.pass.cpp b/libcxx/test/std/containers/associative/multimap/multimap.modifiers/insert_initializer_list.pass.cpp
index d5ef12a1f394f..a8f40aa2b09e4 100644
--- a/libcxx/test/std/containers/associative/multimap/multimap.modifiers/insert_initializer_list.pass.cpp
+++ b/libcxx/test/std/containers/associative/multimap/multimap.modifiers/insert_initializer_list.pass.cpp
@@ -20,27 +20,16 @@
 #include "test_macros.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
+int main(int, char**) {
+  {
     typedef std::multimap<int, double> C;
     typedef C::value_type V;
-    C m =
-           {
-               {1, 1},
-               {1, 2},
-               {2, 1},
-               {2, 2},
-               {3, 1},
-               {3, 2}
-           };
-    m.insert(
-               {
-                   {1, 1.5},
-                   {2, 1.5},
-                   {3, 1.5},
-               }
-            );
+    C m = {{1, 1}, {1, 2}, {2, 1}, {2, 2}, {3, 1}, {3, 2}};
+    m.insert({
+        {1, 1.5},
+        {2, 1.5},
+        {3, 1.5},
+    });
     assert(m.size() == 9);
     assert(std::distance(m.begin(), m.end()) == 9);
     C::const_iterator i = m.cbegin();
@@ -53,26 +42,16 @@ int main(int, char**)
     assert(*++i == V(3, 1));
     assert(*++i == V(3, 2));
     assert(*++i == V(3, 1.5));
-    }
-    {
+  }
+  {
     typedef std::multimap<int, double, std::less<int>, min_allocator<std::pair<const int, double>>> C;
     typedef C::value_type V;
-    C m =
-           {
-               {1, 1},
-               {1, 2},
-               {2, 1},
-               {2, 2},
-               {3, 1},
-               {3, 2}
-           };
-    m.insert(
-               {
-                   {1, 1.5},
-                   {2, 1.5},
-                   {3, 1.5},
-               }
-            );
+    C m = {{1, 1}, {1, 2}, {2, 1}, {2, 2}, {3, 1}, {3, 2}};
+    m.insert({
+        {1, 1.5},
+        {2, 1.5},
+        {3, 1.5},
+    });
     assert(m.size() == 9);
     assert(std::distance(m.begin(), m.end()) == 9);
     C::const_iterator i = m.cbegin();
@@ -85,7 +64,7 @@ int main(int, char**)
     assert(*++i == V(3, 1));
     assert(*++i == V(3, 2));
     assert(*++i == V(3, 1.5));
-    }
+  }
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/associative/multimap/multimap.modifiers/insert_iter_cv.pass.cpp b/libcxx/test/std/containers/associative/multimap/multimap.modifiers/insert_iter_cv.pass.cpp
index 81529600ca0af..ef47d841775d3 100644
--- a/libcxx/test/std/containers/associative/multimap/multimap.modifiers/insert_iter_cv.pass.cpp
+++ b/libcxx/test/std/containers/associative/multimap/multimap.modifiers/insert_iter_cv.pass.cpp
@@ -19,49 +19,47 @@
 #include "min_allocator.h"
 
 template <class Container>
-void do_insert_hint_test()
-{
-    typedef Container M;
-    typedef typename M::iterator R;
-    typedef typename M::value_type VT;
-    M m;
-    const VT v1(2, 2.5);
-    R r = m.insert(m.end(), v1);
-    assert(r == m.begin());
-    assert(m.size() == 1);
-    assert(r->first == 2);
-    assert(r->second == 2.5);
+void do_insert_hint_test() {
+  typedef Container M;
+  typedef typename M::iterator R;
+  typedef typename M::value_type VT;
+  M m;
+  const VT v1(2, 2.5);
+  R r = m.insert(m.end(), v1);
+  assert(r == m.begin());
+  assert(m.size() == 1);
+  assert(r->first == 2);
+  assert(r->second == 2.5);
 
-    const VT v2(1, 1.5);
-    r = m.insert(m.end(), v2);
-    assert(r == m.begin());
-    assert(m.size() == 2);
-    assert(r->first == 1);
-    assert(r->second == 1.5);
+  const VT v2(1, 1.5);
+  r = m.insert(m.end(), v2);
+  assert(r == m.begin());
+  assert(m.size() == 2);
+  assert(r->first == 1);
+  assert(r->second == 1.5);
 
-    const VT v3(3, 3.5);
-    r = m.insert(m.end(), v3);
-    assert(r == std::prev(m.end()));
-    assert(m.size() == 3);
-    assert(r->first == 3);
-    assert(r->second == 3.5);
+  const VT v3(3, 3.5);
+  r = m.insert(m.end(), v3);
+  assert(r == std::prev(m.end()));
+  assert(m.size() == 3);
+  assert(r->first == 3);
+  assert(r->second == 3.5);
 
-    const VT v4(3, 4.5);
-    r = m.insert(std::prev(m.end()), v4);
-    assert(r == std::prev(m.end(), 2));
-    assert(m.size() == 4);
-    assert(r->first == 3);
-    assert(r->second == 4.5);
+  const VT v4(3, 4.5);
+  r = m.insert(std::prev(m.end()), v4);
+  assert(r == std::prev(m.end(), 2));
+  assert(m.size() == 4);
+  assert(r->first == 3);
+  assert(r->second == 4.5);
 }
 
-int main(int, char**)
-{
-    do_insert_hint_test<std::multimap<int, double> >();
+int main(int, char**) {
+  do_insert_hint_test<std::multimap<int, double> >();
 #if TEST_STD_VER >= 11
-    {
-        typedef std::multimap<int, double, std::less<int>, min_allocator<std::pair<const int, double>>> M;
-        do_insert_hint_test<M>();
-    }
+  {
+    typedef std::multimap<int, double, std::less<int>, min_allocator<std::pair<const int, double>>> M;
+    do_insert_hint_test<M>();
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/associative/multimap/multimap.modifiers/insert_iter_iter.pass.cpp b/libcxx/test/std/containers/associative/multimap/multimap.modifiers/insert_iter_iter.pass.cpp
index 585f45a9bb852..12df31e9dd6e7 100644
--- a/libcxx/test/std/containers/associative/multimap/multimap.modifiers/insert_iter_iter.pass.cpp
+++ b/libcxx/test/std/containers/associative/multimap/multimap.modifiers/insert_iter_iter.pass.cpp
@@ -20,83 +20,80 @@
 #include "test_iterators.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef std::multimap<int, double> M;
-        typedef std::pair<int, double> P;
-        P ar[] =
-        {
-            P(1, 1),
-            P(1, 1.5),
-            P(1, 2),
-            P(2, 1),
-            P(2, 1.5),
-            P(2, 2),
-            P(3, 1),
-            P(3, 1.5),
-            P(3, 2),
-        };
-        M m;
-        m.insert(cpp17_input_iterator<P*>(ar), cpp17_input_iterator<P*>(ar + sizeof(ar)/sizeof(ar[0])));
-        assert(m.size() == 9);
-        assert(m.begin()->first == 1);
-        assert(m.begin()->second == 1);
-        assert(std::next(m.begin())->first == 1);
-        assert(std::next(m.begin())->second == 1.5);
-        assert(std::next(m.begin(), 2)->first == 1);
-        assert(std::next(m.begin(), 2)->second == 2);
-        assert(std::next(m.begin(), 3)->first == 2);
-        assert(std::next(m.begin(), 3)->second == 1);
-        assert(std::next(m.begin(), 4)->first == 2);
-        assert(std::next(m.begin(), 4)->second == 1.5);
-        assert(std::next(m.begin(), 5)->first == 2);
-        assert(std::next(m.begin(), 5)->second == 2);
-        assert(std::next(m.begin(), 6)->first == 3);
-        assert(std::next(m.begin(), 6)->second == 1);
-        assert(std::next(m.begin(), 7)->first == 3);
-        assert(std::next(m.begin(), 7)->second == 1.5);
-        assert(std::next(m.begin(), 8)->first == 3);
-        assert(std::next(m.begin(), 8)->second == 2);
-    }
+int main(int, char**) {
+  {
+    typedef std::multimap<int, double> M;
+    typedef std::pair<int, double> P;
+    P ar[] = {
+        P(1, 1),
+        P(1, 1.5),
+        P(1, 2),
+        P(2, 1),
+        P(2, 1.5),
+        P(2, 2),
+        P(3, 1),
+        P(3, 1.5),
+        P(3, 2),
+    };
+    M m;
+    m.insert(cpp17_input_iterator<P*>(ar), cpp17_input_iterator<P*>(ar + sizeof(ar) / sizeof(ar[0])));
+    assert(m.size() == 9);
+    assert(m.begin()->first == 1);
+    assert(m.begin()->second == 1);
+    assert(std::next(m.begin())->first == 1);
+    assert(std::next(m.begin())->second == 1.5);
+    assert(std::next(m.begin(), 2)->first == 1);
+    assert(std::next(m.begin(), 2)->second == 2);
+    assert(std::next(m.begin(), 3)->first == 2);
+    assert(std::next(m.begin(), 3)->second == 1);
+    assert(std::next(m.begin(), 4)->first == 2);
+    assert(std::next(m.begin(), 4)->second == 1.5);
+    assert(std::next(m.begin(), 5)->first == 2);
+    assert(std::next(m.begin(), 5)->second == 2);
+    assert(std::next(m.begin(), 6)->first == 3);
+    assert(std::next(m.begin(), 6)->second == 1);
+    assert(std::next(m.begin(), 7)->first == 3);
+    assert(std::next(m.begin(), 7)->second == 1.5);
+    assert(std::next(m.begin(), 8)->first == 3);
+    assert(std::next(m.begin(), 8)->second == 2);
+  }
 #if TEST_STD_VER >= 11
-    {
-        typedef std::multimap<int, double, std::less<int>, min_allocator<std::pair<const int, double>>> M;
-        typedef std::pair<int, double> P;
-        P ar[] =
-        {
-            P(1, 1),
-            P(1, 1.5),
-            P(1, 2),
-            P(2, 1),
-            P(2, 1.5),
-            P(2, 2),
-            P(3, 1),
-            P(3, 1.5),
-            P(3, 2),
-        };
-        M m;
-        m.insert(cpp17_input_iterator<P*>(ar), cpp17_input_iterator<P*>(ar + sizeof(ar)/sizeof(ar[0])));
-        assert(m.size() == 9);
-        assert(m.begin()->first == 1);
-        assert(m.begin()->second == 1);
-        assert(std::next(m.begin())->first == 1);
-        assert(std::next(m.begin())->second == 1.5);
-        assert(std::next(m.begin(), 2)->first == 1);
-        assert(std::next(m.begin(), 2)->second == 2);
-        assert(std::next(m.begin(), 3)->first == 2);
-        assert(std::next(m.begin(), 3)->second == 1);
-        assert(std::next(m.begin(), 4)->first == 2);
-        assert(std::next(m.begin(), 4)->second == 1.5);
-        assert(std::next(m.begin(), 5)->first == 2);
-        assert(std::next(m.begin(), 5)->second == 2);
-        assert(std::next(m.begin(), 6)->first == 3);
-        assert(std::next(m.begin(), 6)->second == 1);
-        assert(std::next(m.begin(), 7)->first == 3);
-        assert(std::next(m.begin(), 7)->second == 1.5);
-        assert(std::next(m.begin(), 8)->first == 3);
-        assert(std::next(m.begin(), 8)->second == 2);
-    }
+  {
+    typedef std::multimap<int, double, std::less<int>, min_allocator<std::pair<const int, double>>> M;
+    typedef std::pair<int, double> P;
+    P ar[] = {
+        P(1, 1),
+        P(1, 1.5),
+        P(1, 2),
+        P(2, 1),
+        P(2, 1.5),
+        P(2, 2),
+        P(3, 1),
+        P(3, 1.5),
+        P(3, 2),
+    };
+    M m;
+    m.insert(cpp17_input_iterator<P*>(ar), cpp17_input_iterator<P*>(ar + sizeof(ar) / sizeof(ar[0])));
+    assert(m.size() == 9);
+    assert(m.begin()->first == 1);
+    assert(m.begin()->second == 1);
+    assert(std::next(m.begin())->first == 1);
+    assert(std::next(m.begin())->second == 1.5);
+    assert(std::next(m.begin(), 2)->first == 1);
+    assert(std::next(m.begin(), 2)->second == 2);
+    assert(std::next(m.begin(), 3)->first == 2);
+    assert(std::next(m.begin(), 3)->second == 1);
+    assert(std::next(m.begin(), 4)->first == 2);
+    assert(std::next(m.begin(), 4)->second == 1.5);
+    assert(std::next(m.begin(), 5)->first == 2);
+    assert(std::next(m.begin(), 5)->second == 2);
+    assert(std::next(m.begin(), 6)->first == 3);
+    assert(std::next(m.begin(), 6)->second == 1);
+    assert(std::next(m.begin(), 7)->first == 3);
+    assert(std::next(m.begin(), 7)->second == 1.5);
+    assert(std::next(m.begin(), 8)->first == 3);
+    assert(std::next(m.begin(), 8)->second == 2);
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/associative/multimap/multimap.modifiers/insert_iter_rv.pass.cpp b/libcxx/test/std/containers/associative/multimap/multimap.modifiers/insert_iter_rv.pass.cpp
index bd90b3c9ba32f..a56f237eb63ac 100644
--- a/libcxx/test/std/containers/associative/multimap/multimap.modifiers/insert_iter_rv.pass.cpp
+++ b/libcxx/test/std/containers/associative/multimap/multimap.modifiers/insert_iter_rv.pass.cpp
@@ -23,78 +23,75 @@
 #include "test_macros.h"
 
 template <class Container, class Pair>
-void do_insert_rv_test()
-{
-    typedef Container M;
-    typedef Pair P;
-    typedef typename M::iterator R;
+void do_insert_rv_test() {
+  typedef Container M;
+  typedef Pair P;
+  typedef typename M::iterator R;
+  M m;
+  R r = m.insert(m.cend(), P(2, 2));
+  assert(r == m.begin());
+  assert(m.size() == 1);
+  assert(r->first == 2);
+  assert(r->second == 2);
+
+  r = m.insert(m.cend(), P(1, 1));
+  assert(r == m.begin());
+  assert(m.size() == 2);
+  assert(r->first == 1);
+  assert(r->second == 1);
+
+  r = m.insert(m.cend(), P(3, 3));
+  assert(r == std::prev(m.end()));
+  assert(m.size() == 3);
+  assert(r->first == 3);
+  assert(r->second == 3);
+
+  r = m.insert(m.cend(), P(3, 2));
+  assert(r == std::prev(m.end()));
+  assert(m.size() == 4);
+  assert(r->first == 3);
+  assert(r->second == 2);
+}
+
+int main(int, char**) {
+  do_insert_rv_test<std::multimap<int, MoveOnly>, std::pair<int, MoveOnly> >();
+  do_insert_rv_test<std::multimap<int, MoveOnly>, std::pair<const int, MoveOnly> >();
+
+  {
+    typedef std::multimap<int, MoveOnly, std::less<int>, min_allocator<std::pair<const int, MoveOnly>>> M;
+    typedef std::pair<int, MoveOnly> P;
+    typedef std::pair<const int, MoveOnly> CP;
+    do_insert_rv_test<M, P>();
+    do_insert_rv_test<M, CP>();
+  }
+  {
+    typedef std::multimap<int, MoveOnly> M;
+    typedef M::iterator R;
     M m;
-    R r = m.insert(m.cend(), P(2, 2));
+    R r = m.insert(m.cend(), {2, MoveOnly(2)});
     assert(r == m.begin());
     assert(m.size() == 1);
     assert(r->first == 2);
     assert(r->second == 2);
 
-    r = m.insert(m.cend(), P(1, 1));
+    r = m.insert(m.cend(), {1, MoveOnly(1)});
     assert(r == m.begin());
     assert(m.size() == 2);
     assert(r->first == 1);
     assert(r->second == 1);
 
-    r = m.insert(m.cend(), P(3, 3));
+    r = m.insert(m.cend(), {3, MoveOnly(3)});
     assert(r == std::prev(m.end()));
     assert(m.size() == 3);
     assert(r->first == 3);
     assert(r->second == 3);
 
-    r = m.insert(m.cend(), P(3, 2));
+    r = m.insert(m.cend(), {3, MoveOnly(2)});
     assert(r == std::prev(m.end()));
     assert(m.size() == 4);
     assert(r->first == 3);
     assert(r->second == 2);
-}
-
-int main(int, char**)
-{
-    do_insert_rv_test<std::multimap<int, MoveOnly>, std::pair<int, MoveOnly> >();
-    do_insert_rv_test<std::multimap<int, MoveOnly>, std::pair<const int, MoveOnly> >();
-
-    {
-        typedef std::multimap<int, MoveOnly, std::less<int>, min_allocator<std::pair<const int, MoveOnly>>> M;
-        typedef std::pair<int, MoveOnly> P;
-        typedef std::pair<const int, MoveOnly> CP;
-        do_insert_rv_test<M, P>();
-        do_insert_rv_test<M, CP>();
-
-    }
-    {
-        typedef std::multimap<int, MoveOnly> M;
-        typedef M::iterator R;
-        M m;
-        R r = m.insert(m.cend(), {2, MoveOnly(2)});
-        assert(r == m.begin());
-        assert(m.size() == 1);
-        assert(r->first == 2);
-        assert(r->second == 2);
-
-        r = m.insert(m.cend(), {1, MoveOnly(1)});
-        assert(r == m.begin());
-        assert(m.size() == 2);
-        assert(r->first == 1);
-        assert(r->second == 1);
-
-        r = m.insert(m.cend(), {3, MoveOnly(3)});
-        assert(r == std::prev(m.end()));
-        assert(m.size() == 3);
-        assert(r->first == 3);
-        assert(r->second == 3);
-
-        r = m.insert(m.cend(), {3, MoveOnly(2)});
-        assert(r == std::prev(m.end()));
-        assert(m.size() == 4);
-        assert(r->first == 3);
-        assert(r->second == 2);
-    }
+  }
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/associative/multimap/multimap.modifiers/insert_node_type.pass.cpp b/libcxx/test/std/containers/associative/multimap/multimap.modifiers/insert_node_type.pass.cpp
index 8334bbef5c958..a488bc6617ccb 100644
--- a/libcxx/test/std/containers/associative/multimap/multimap.modifiers/insert_node_type.pass.cpp
+++ b/libcxx/test/std/containers/associative/multimap/multimap.modifiers/insert_node_type.pass.cpp
@@ -21,60 +21,54 @@
 
 template <class Container>
 typename Container::node_type
-node_factory(typename Container::key_type const& key,
-             typename Container::mapped_type const& mapped)
-{
-    static Container c;
-    auto it = c.insert({key, mapped});
-    return c.extract(it);
+node_factory(typename Container::key_type const& key, typename Container::mapped_type const& mapped) {
+  static Container c;
+  auto it = c.insert({key, mapped});
+  return c.extract(it);
 }
 
 template <class Container>
-void test(Container& c)
-{
-    auto* nf = &node_factory<Container>;
+void test(Container& c) {
+  auto* nf = &node_factory<Container>;
 
-    for (int i = 0; i != 10; ++i)
-    {
-        typename Container::node_type node = nf(i, i + 1);
-        assert(!node.empty());
-        typename Container::iterator it = c.insert(std::move(node));
-        assert(node.empty());
-        assert(it == c.find(i) && it != c.end());
-    }
+  for (int i = 0; i != 10; ++i) {
+    typename Container::node_type node = nf(i, i + 1);
+    assert(!node.empty());
+    typename Container::iterator it = c.insert(std::move(node));
+    assert(node.empty());
+    assert(it == c.find(i) && it != c.end());
+  }
 
-    assert(c.size() == 10);
+  assert(c.size() == 10);
 
-    { // Insert empty node.
-        typename Container::node_type def;
-        auto it = c.insert(std::move(def));
-        assert(def.empty());
-        assert(it == c.end());
-    }
+  { // Insert empty node.
+    typename Container::node_type def;
+    auto it = c.insert(std::move(def));
+    assert(def.empty());
+    assert(it == c.end());
+  }
 
-    { // Insert duplicate node.
-        typename Container::node_type dupl = nf(0, 42);
-        auto it = c.insert(std::move(dupl));
-        assert(dupl.empty());
-        assert(it != c.end());
-        assert(it->second == 42);
-    }
+  { // Insert duplicate node.
+    typename Container::node_type dupl = nf(0, 42);
+    auto it                            = c.insert(std::move(dupl));
+    assert(dupl.empty());
+    assert(it != c.end());
+    assert(it->second == 42);
+  }
 
-    assert(c.size() == 11);
-    assert(c.count(0) == 2);
-    for (int i = 1; i != 10; ++i)
-    {
-        assert(c.count(i) == 1);
-        assert(c.find(i)->second == i + 1);
-    }
+  assert(c.size() == 11);
+  assert(c.count(0) == 2);
+  for (int i = 1; i != 10; ++i) {
+    assert(c.count(i) == 1);
+    assert(c.find(i)->second == i + 1);
+  }
 }
 
-int main(int, char**)
-{
-    std::multimap<int, int> m;
-    test(m);
-    std::multimap<int, int, std::less<int>, min_allocator<std::pair<const int, int>>> m2;
-    test(m2);
+int main(int, char**) {
+  std::multimap<int, int> m;
+  test(m);
+  std::multimap<int, int, std::less<int>, min_allocator<std::pair<const int, int>>> m2;
+  test(m2);
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/associative/multimap/multimap.modifiers/insert_node_type_hint.pass.cpp b/libcxx/test/std/containers/associative/multimap/multimap.modifiers/insert_node_type_hint.pass.cpp
index 68e6b8b795f9d..7e855801045b8 100644
--- a/libcxx/test/std/containers/associative/multimap/multimap.modifiers/insert_node_type_hint.pass.cpp
+++ b/libcxx/test/std/containers/associative/multimap/multimap.modifiers/insert_node_type_hint.pass.cpp
@@ -20,47 +20,41 @@
 
 template <class Container>
 typename Container::node_type
-node_factory(typename Container::key_type const& key,
-             typename Container::mapped_type const& mapped)
-{
-    static Container c;
-    auto it = c.insert({key, mapped});
-    return c.extract(it);
+node_factory(typename Container::key_type const& key, typename Container::mapped_type const& mapped) {
+  static Container c;
+  auto it = c.insert({key, mapped});
+  return c.extract(it);
 }
 
 template <class Container>
-void test(Container& c)
-{
-    auto* nf = &node_factory<Container>;
-
-    for (int i = 0; i != 10; ++i)
-    {
-        typename Container::node_type node = nf(i, i + 1);
-        assert(!node.empty());
-        std::size_t prev = c.size();
-        auto it = c.insert(c.end(), std::move(node));
-        assert(node.empty());
-        assert(prev + 1 == c.size());
-        assert(it == c.find(i));
-        assert(it->first == i);
-        assert(it->second == i + 1);
-    }
-
-    assert(c.size() == 10);
-
-    for (int i = 0; i != 10; ++i)
-    {
-        assert(c.count(i) == 1);
-        assert(c.find(i)->second == i + 1);
-    }
+void test(Container& c) {
+  auto* nf = &node_factory<Container>;
+
+  for (int i = 0; i != 10; ++i) {
+    typename Container::node_type node = nf(i, i + 1);
+    assert(!node.empty());
+    std::size_t prev = c.size();
+    auto it          = c.insert(c.end(), std::move(node));
+    assert(node.empty());
+    assert(prev + 1 == c.size());
+    assert(it == c.find(i));
+    assert(it->first == i);
+    assert(it->second == i + 1);
+  }
+
+  assert(c.size() == 10);
+
+  for (int i = 0; i != 10; ++i) {
+    assert(c.count(i) == 1);
+    assert(c.find(i)->second == i + 1);
+  }
 }
 
-int main(int, char**)
-{
-    std::multimap<int, int> m;
-    test(m);
-    std::multimap<int, int, std::less<int>, min_allocator<std::pair<const int, int>>> m2;
-    test(m2);
+int main(int, char**) {
+  std::multimap<int, int> m;
+  test(m);
+  std::multimap<int, int, std::less<int>, min_allocator<std::pair<const int, int>>> m2;
+  test(m2);
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/associative/multimap/multimap.modifiers/insert_range.pass.cpp b/libcxx/test/std/containers/associative/multimap/multimap.modifiers/insert_range.pass.cpp
index a804cbf0cf206..efd257a4f2c0e 100644
--- a/libcxx/test/std/containers/associative/multimap/multimap.modifiers/insert_range.pass.cpp
+++ b/libcxx/test/std/containers/associative/multimap/multimap.modifiers/insert_range.pass.cpp
@@ -23,10 +23,11 @@
 int main(int, char**) {
   // Note: we want to use a pair with non-const elements for input (an assignable type is a lot more convenient) but
   // have to use the exact `value_type` of the map (that is, `pair<const K, V>`) for the allocator.
-  using Pair = std::pair<int, char>;
+  using Pair      = std::pair<int, char>;
   using ConstPair = std::pair<const int, char>;
   for_all_iterators_and_allocators<ConstPair, const Pair*>([]<class Iter, class Sent, class Alloc>() {
-    test_map_set_insert_range<std::multimap<int, char, test_less<int>, Alloc>, Pair, Iter, Sent>(/*allow_duplicates=*/true);
+    test_map_set_insert_range<std::multimap<int, char, test_less<int>, Alloc>, Pair, Iter, Sent>(
+        /*allow_duplicates=*/true);
   });
 
   static_assert(test_map_constraints_insert_range<std::multimap, int, int, char, double>());
@@ -38,4 +39,3 @@ int main(int, char**) {
 
   return 0;
 }
-
diff --git a/libcxx/test/std/containers/associative/multimap/multimap.modifiers/insert_rv.pass.cpp b/libcxx/test/std/containers/associative/multimap/multimap.modifiers/insert_rv.pass.cpp
index 7933d69d2e3f0..bf32b996e1836 100644
--- a/libcxx/test/std/containers/associative/multimap/multimap.modifiers/insert_rv.pass.cpp
+++ b/libcxx/test/std/containers/associative/multimap/multimap.modifiers/insert_rv.pass.cpp
@@ -23,72 +23,70 @@
 #include "test_macros.h"
 
 template <class Container>
-void do_insert_rv_test()
-{
+void do_insert_rv_test() {
+  typedef std::multimap<int, MoveOnly> M;
+  typedef typename M::iterator R;
+  typedef typename M::value_type VT;
+  M m;
+  R r = m.insert(VT(2, 2));
+  assert(r == m.begin());
+  assert(m.size() == 1);
+  assert(r->first == 2);
+  assert(r->second == 2);
+
+  r = m.insert(VT(1, 1));
+  assert(r == m.begin());
+  assert(m.size() == 2);
+  assert(r->first == 1);
+  assert(r->second == 1);
+
+  r = m.insert(VT(3, 3));
+  assert(r == std::prev(m.end()));
+  assert(m.size() == 3);
+  assert(r->first == 3);
+  assert(r->second == 3);
+
+  r = m.insert(VT(3, 3));
+  assert(r == std::prev(m.end()));
+  assert(m.size() == 4);
+  assert(r->first == 3);
+  assert(r->second == 3);
+}
+
+int main(int, char**) {
+  do_insert_rv_test<std::multimap<int, MoveOnly>>();
+  {
+    typedef std::multimap<int, MoveOnly, std::less<int>, min_allocator<std::pair<const int, MoveOnly>>> M;
+    do_insert_rv_test<M>();
+  }
+  {
     typedef std::multimap<int, MoveOnly> M;
-    typedef typename M::iterator R;
-    typedef typename M::value_type VT;
+    typedef M::iterator R;
     M m;
-    R r = m.insert(VT(2, 2));
+    R r = m.insert({2, MoveOnly(2)});
     assert(r == m.begin());
     assert(m.size() == 1);
     assert(r->first == 2);
     assert(r->second == 2);
 
-    r = m.insert(VT(1, 1));
+    r = m.insert({1, MoveOnly(1)});
     assert(r == m.begin());
     assert(m.size() == 2);
     assert(r->first == 1);
     assert(r->second == 1);
 
-    r = m.insert(VT(3, 3));
+    r = m.insert({3, MoveOnly(3)});
     assert(r == std::prev(m.end()));
     assert(m.size() == 3);
     assert(r->first == 3);
     assert(r->second == 3);
 
-    r = m.insert(VT(3, 3));
+    r = m.insert({3, MoveOnly(3)});
     assert(r == std::prev(m.end()));
     assert(m.size() == 4);
     assert(r->first == 3);
     assert(r->second == 3);
-}
-
-int main(int, char**)
-{
-    do_insert_rv_test<std::multimap<int, MoveOnly>>();
-    {
-        typedef std::multimap<int, MoveOnly, std::less<int>, min_allocator<std::pair<const int, MoveOnly>>> M;
-        do_insert_rv_test<M>();
-    }
-    {
-        typedef std::multimap<int, MoveOnly> M;
-        typedef M::iterator R;
-        M m;
-        R r = m.insert({2, MoveOnly(2)});
-        assert(r == m.begin());
-        assert(m.size() == 1);
-        assert(r->first == 2);
-        assert(r->second == 2);
-
-        r = m.insert({1, MoveOnly(1)});
-        assert(r == m.begin());
-        assert(m.size() == 2);
-        assert(r->first == 1);
-        assert(r->second == 1);
-
-        r = m.insert({3, MoveOnly(3)});
-        assert(r == std::prev(m.end()));
-        assert(m.size() == 3);
-        assert(r->first == 3);
-        assert(r->second == 3);
-
-        r = m.insert({3, MoveOnly(3)});
-        assert(r == std::prev(m.end()));
-        assert(m.size() == 4);
-        assert(r->first == 3);
-        assert(r->second == 3);
-    }
+  }
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/associative/multimap/multimap.modifiers/merge.pass.cpp b/libcxx/test/std/containers/associative/multimap/multimap.modifiers/merge.pass.cpp
index 078932b1106c7..d7952763ca9bb 100644
--- a/libcxx/test/std/containers/associative/multimap/multimap.modifiers/merge.pass.cpp
+++ b/libcxx/test/std/containers/associative/multimap/multimap.modifiers/merge.pass.cpp
@@ -27,124 +27,113 @@
 #include "Counter.h"
 
 template <class Map>
-bool map_equal(const Map& map, Map other)
-{
-    return map == other;
+bool map_equal(const Map& map, Map other) {
+  return map == other;
 }
 
 #ifndef TEST_HAS_NO_EXCEPTIONS
-struct throw_comparator
-{
-    bool& should_throw_;
+struct throw_comparator {
+  bool& should_throw_;
 
-    throw_comparator(bool& should_throw) : should_throw_(should_throw) {}
+  throw_comparator(bool& should_throw) : should_throw_(should_throw) {}
 
-    template <class T>
-    bool operator()(const T& lhs, const T& rhs) const
-    {
-        if (should_throw_)
-            throw 0;
-        return lhs < rhs;
-    }
+  template <class T>
+  bool operator()(const T& lhs, const T& rhs) const {
+    if (should_throw_)
+      throw 0;
+    return lhs < rhs;
+  }
 };
 #endif
 
-int main(int, char**)
-{
-    {
-        std::multimap<int, int> src{{1, 0}, {3, 0}, {5, 0}};
-        std::multimap<int, int> dst{{2, 0}, {4, 0}, {5, 0}};
-        dst.merge(src);
-        assert(map_equal(src, {}));
-        assert(map_equal(dst, {{1, 0}, {2, 0}, {3, 0}, {4, 0}, {5, 0}, {5, 0}}));
-    }
+int main(int, char**) {
+  {
+    std::multimap<int, int> src{{1, 0}, {3, 0}, {5, 0}};
+    std::multimap<int, int> dst{{2, 0}, {4, 0}, {5, 0}};
+    dst.merge(src);
+    assert(map_equal(src, {}));
+    assert(map_equal(dst, {{1, 0}, {2, 0}, {3, 0}, {4, 0}, {5, 0}, {5, 0}}));
+  }
 
 #ifndef TEST_HAS_NO_EXCEPTIONS
-    {
-        bool do_throw = false;
-        typedef std::multimap<Counter<int>, int, throw_comparator> map_type;
-        map_type src({{1, 0}, {3, 0}, {5, 0}}, throw_comparator(do_throw));
-        map_type dst({{2, 0}, {4, 0}, {5, 0}}, throw_comparator(do_throw));
-
-        assert(Counter_base::gConstructed == 6);
-
-        do_throw = true;
-        try
-        {
-            dst.merge(src);
-        }
-        catch (int)
-        {
-            do_throw = false;
-        }
-        assert(!do_throw);
-        assert(map_equal(src, map_type({{1, 0}, {3, 0}, {5, 0}}, throw_comparator(do_throw))));
-        assert(map_equal(dst, map_type({{2, 0}, {4, 0}, {5, 0}}, throw_comparator(do_throw))));
+  {
+    bool do_throw = false;
+    typedef std::multimap<Counter<int>, int, throw_comparator> map_type;
+    map_type src({{1, 0}, {3, 0}, {5, 0}}, throw_comparator(do_throw));
+    map_type dst({{2, 0}, {4, 0}, {5, 0}}, throw_comparator(do_throw));
+
+    assert(Counter_base::gConstructed == 6);
+
+    do_throw = true;
+    try {
+      dst.merge(src);
+    } catch (int) {
+      do_throw = false;
     }
+    assert(!do_throw);
+    assert(map_equal(src, map_type({{1, 0}, {3, 0}, {5, 0}}, throw_comparator(do_throw))));
+    assert(map_equal(dst, map_type({{2, 0}, {4, 0}, {5, 0}}, throw_comparator(do_throw))));
+  }
 #endif
-    assert(Counter_base::gConstructed == 0);
-    struct comparator
-    {
-        comparator() = default;
+  assert(Counter_base::gConstructed == 0);
+  struct comparator {
+    comparator() = default;
 
-        bool operator()(const Counter<int>& lhs, const Counter<int>& rhs) const
-        {
-            return lhs < rhs;
-        }
-    };
-    {
-        typedef std::multimap<Counter<int>, int, std::less<Counter<int>>> first_map_type;
-        typedef std::multimap<Counter<int>, int, comparator> second_map_type;
-        typedef std::map<Counter<int>, int, comparator> third_map_type;
+    bool operator()(const Counter<int>& lhs, const Counter<int>& rhs) const { return lhs < rhs; }
+  };
+  {
+    typedef std::multimap<Counter<int>, int, std::less<Counter<int>>> first_map_type;
+    typedef std::multimap<Counter<int>, int, comparator> second_map_type;
+    typedef std::map<Counter<int>, int, comparator> third_map_type;
 
-        {
-            first_map_type first{{1, 0}, {2, 0}, {3, 0}};
-            second_map_type second{{2, 0}, {3, 0}, {4, 0}};
-            third_map_type third{{1, 0}, {3, 0}};
+    {
+      first_map_type first{{1, 0}, {2, 0}, {3, 0}};
+      second_map_type second{{2, 0}, {3, 0}, {4, 0}};
+      third_map_type third{{1, 0}, {3, 0}};
 
-            assert(Counter_base::gConstructed == 8);
+      assert(Counter_base::gConstructed == 8);
 
-            first.merge(second);
-            first.merge(third);
+      first.merge(second);
+      first.merge(third);
 
-            assert(map_equal(first, {{1, 0}, {1, 0}, {2, 0}, {2, 0}, {3, 0}, {3, 0}, {3, 0}, {4, 0}}));
-            assert(map_equal(second, {}));
-            assert(map_equal(third, {}));
+      assert(map_equal(first, {{1, 0}, {1, 0}, {2, 0}, {2, 0}, {3, 0}, {3, 0}, {3, 0}, {4, 0}}));
+      assert(map_equal(second, {}));
+      assert(map_equal(third, {}));
 
-            assert(Counter_base::gConstructed == 8);
-        }
-        assert(Counter_base::gConstructed == 0);
-        {
-            first_map_type first{{1, 0}, {2, 0}, {3, 0}};
-            second_map_type second{{2, 0}, {3, 0}, {4, 0}};
-            third_map_type third{{1, 0}, {3, 0}};
+      assert(Counter_base::gConstructed == 8);
+    }
+    assert(Counter_base::gConstructed == 0);
+    {
+      first_map_type first{{1, 0}, {2, 0}, {3, 0}};
+      second_map_type second{{2, 0}, {3, 0}, {4, 0}};
+      third_map_type third{{1, 0}, {3, 0}};
 
-            assert(Counter_base::gConstructed == 8);
+      assert(Counter_base::gConstructed == 8);
 
-            first.merge(std::move(second));
-            first.merge(std::move(third));
+      first.merge(std::move(second));
+      first.merge(std::move(third));
 
-            assert(map_equal(first, {{1, 0}, {1, 0}, {2, 0}, {2, 0}, {3, 0}, {3, 0}, {3, 0}, {4, 0}}));
-            assert(map_equal(second, {}));
-            assert(map_equal(third, {}));
+      assert(map_equal(first, {{1, 0}, {1, 0}, {2, 0}, {2, 0}, {3, 0}, {3, 0}, {3, 0}, {4, 0}}));
+      assert(map_equal(second, {}));
+      assert(map_equal(third, {}));
 
-            assert(Counter_base::gConstructed == 8);
-        }
-        assert(Counter_base::gConstructed == 0);
+      assert(Counter_base::gConstructed == 8);
     }
     assert(Counter_base::gConstructed == 0);
+  }
+  assert(Counter_base::gConstructed == 0);
+  {
+    std::multimap<int, int> first;
+    {
+      std::multimap<int, int> second;
+      first.merge(second);
+      first.merge(std::move(second));
+    }
     {
-        std::multimap<int, int> first;
-        {
-            std::multimap<int, int> second;
-            first.merge(second);
-            first.merge(std::move(second));
-        }
-        {
-            std::multimap<int, int> second;
-            first.merge(second);
-            first.merge(std::move(second));
-        }
+      std::multimap<int, int> second;
+      first.merge(second);
+      first.merge(std::move(second));
     }
-    return 0;
+  }
+  return 0;
 }
diff --git a/libcxx/test/std/containers/associative/multimap/multimap.nonmember/op_compare.pass.cpp b/libcxx/test/std/containers/associative/multimap/multimap.nonmember/op_compare.pass.cpp
index be2143c68be7d..fe5f401e19b5c 100644
--- a/libcxx/test/std/containers/associative/multimap/multimap.nonmember/op_compare.pass.cpp
+++ b/libcxx/test/std/containers/associative/multimap/multimap.nonmember/op_compare.pass.cpp
@@ -39,52 +39,52 @@
 #include "test_comparisons.h"
 
 int main(int, char**) {
-    typedef std::multimap<int, std::string> map_type;
-    typedef map_type::value_type value_type;
-    {
-        map_type m1, m2;
-        m1.insert(value_type(1, "abc"));
-        m2.insert(value_type(2, "abc"));
-        const map_type& cm1 = m1, cm2 = m2;
-        assert(testComparisons(cm1, cm2, false, true));
-    }
-    {
-        map_type m1, m2;
-        m1.insert(value_type(1, "abc"));
-        m2.insert(value_type(1, "abc"));
-        const map_type& cm1 = m1, cm2 = m2;
-        assert(testComparisons(cm1, cm2, true, false));
-    }
-    {
-        map_type m1, m2;
-        m1.insert(value_type(1, "ab"));
-        m2.insert(value_type(1, "abc"));
-        const map_type& cm1 = m1, cm2 = m2;
-        assert(testComparisons(cm1, cm2, false, true));
-    }
-    {
-        map_type m1, m2;
-        m1.insert(value_type(1, "abc"));
-        m2.insert(value_type(1, "bcd"));
-        const map_type& cm1 = m1, cm2 = m2;
-        assert(testComparisons(cm1, cm2, false, true));
-    }
-    {
-        map_type m1, m2;
-        m1.insert(value_type(1, "abc"));
-        m2.insert(value_type(1, "abc"));
-        m2.insert(value_type(2, "abc"));
-        const map_type& cm1 = m1, cm2 = m2;
-        assert(testComparisons(cm1, cm2, false, true));
-    }
-    {
-        map_type m1, m2;
-        m1.insert(value_type(1, "abc"));
-        m2.insert(value_type(1, "abc"));
-        m2.insert(value_type(1, "abc"));
-        m2.insert(value_type(1, "bcd"));
-        const map_type& cm1 = m1, cm2 = m2;
-        assert(testComparisons(cm1, cm2, false, true));
-    }
-    return 0;
+  typedef std::multimap<int, std::string> map_type;
+  typedef map_type::value_type value_type;
+  {
+    map_type m1, m2;
+    m1.insert(value_type(1, "abc"));
+    m2.insert(value_type(2, "abc"));
+    const map_type &cm1 = m1, cm2 = m2;
+    assert(testComparisons(cm1, cm2, false, true));
+  }
+  {
+    map_type m1, m2;
+    m1.insert(value_type(1, "abc"));
+    m2.insert(value_type(1, "abc"));
+    const map_type &cm1 = m1, cm2 = m2;
+    assert(testComparisons(cm1, cm2, true, false));
+  }
+  {
+    map_type m1, m2;
+    m1.insert(value_type(1, "ab"));
+    m2.insert(value_type(1, "abc"));
+    const map_type &cm1 = m1, cm2 = m2;
+    assert(testComparisons(cm1, cm2, false, true));
+  }
+  {
+    map_type m1, m2;
+    m1.insert(value_type(1, "abc"));
+    m2.insert(value_type(1, "bcd"));
+    const map_type &cm1 = m1, cm2 = m2;
+    assert(testComparisons(cm1, cm2, false, true));
+  }
+  {
+    map_type m1, m2;
+    m1.insert(value_type(1, "abc"));
+    m2.insert(value_type(1, "abc"));
+    m2.insert(value_type(2, "abc"));
+    const map_type &cm1 = m1, cm2 = m2;
+    assert(testComparisons(cm1, cm2, false, true));
+  }
+  {
+    map_type m1, m2;
+    m1.insert(value_type(1, "abc"));
+    m2.insert(value_type(1, "abc"));
+    m2.insert(value_type(1, "abc"));
+    m2.insert(value_type(1, "bcd"));
+    const map_type &cm1 = m1, cm2 = m2;
+    assert(testComparisons(cm1, cm2, false, true));
+  }
+  return 0;
 }
diff --git a/libcxx/test/std/containers/associative/multimap/multimap.observers/key_comp.pass.cpp b/libcxx/test/std/containers/associative/multimap/multimap.observers/key_comp.pass.cpp
index cc6b18c1b0e01..29cdecd6fef48 100644
--- a/libcxx/test/std/containers/associative/multimap/multimap.observers/key_comp.pass.cpp
+++ b/libcxx/test/std/containers/associative/multimap/multimap.observers/key_comp.pass.cpp
@@ -15,16 +15,16 @@
 #include <string>
 
 int main(int, char**) {
-    typedef std::multimap<int, std::string> map_type;
+  typedef std::multimap<int, std::string> map_type;
 
-    map_type m;
-    map_type::iterator i1 = m.insert(map_type::value_type(1, "abc"));
-    map_type::iterator i2 = m.insert(map_type::value_type(2, "abc"));
+  map_type m;
+  map_type::iterator i1 = m.insert(map_type::value_type(1, "abc"));
+  map_type::iterator i2 = m.insert(map_type::value_type(2, "abc"));
 
-    const map_type& cm = m;
+  const map_type& cm = m;
 
-    assert(cm.key_comp()(i1->first, i2->first));
-    assert(!cm.key_comp()(i2->first, i1->first));
+  assert(cm.key_comp()(i1->first, i2->first));
+  assert(!cm.key_comp()(i2->first, i1->first));
 
-    return 0;
+  return 0;
 }
diff --git a/libcxx/test/std/containers/associative/multimap/multimap.observers/value_comp.pass.cpp b/libcxx/test/std/containers/associative/multimap/multimap.observers/value_comp.pass.cpp
index 3b9ed80598c46..542ed6d71c8d1 100644
--- a/libcxx/test/std/containers/associative/multimap/multimap.observers/value_comp.pass.cpp
+++ b/libcxx/test/std/containers/associative/multimap/multimap.observers/value_comp.pass.cpp
@@ -15,16 +15,16 @@
 #include <string>
 
 int main(int, char**) {
-    typedef std::multimap<int, std::string> map_type;
+  typedef std::multimap<int, std::string> map_type;
 
-    map_type m;
-    map_type::iterator i1 = m.insert(map_type::value_type(1, "abc"));
-    map_type::iterator i2 = m.insert(map_type::value_type(2, "abc"));
+  map_type m;
+  map_type::iterator i1 = m.insert(map_type::value_type(1, "abc"));
+  map_type::iterator i2 = m.insert(map_type::value_type(2, "abc"));
 
-    const map_type& cm = m;
+  const map_type& cm = m;
 
-    assert(cm.value_comp()(*i1, *i2));
-    assert(!cm.value_comp()(*i2, *i1));
+  assert(cm.value_comp()(*i1, *i2));
+  assert(!cm.value_comp()(*i2, *i1));
 
-    return 0;
+  return 0;
 }
diff --git a/libcxx/test/std/containers/associative/multimap/multimap.ops/count.pass.cpp b/libcxx/test/std/containers/associative/multimap/multimap.ops/count.pass.cpp
index 40b855b11e96f..8dfda74577e35 100644
--- a/libcxx/test/std/containers/associative/multimap/multimap.ops/count.pass.cpp
+++ b/libcxx/test/std/containers/associative/multimap/multimap.ops/count.pass.cpp
@@ -20,95 +20,61 @@
 #include "private_constructor.h"
 #include "is_transparent.h"
 
-int main(int, char**)
-{
-    typedef std::pair<const int, double> V;
-    {
+int main(int, char**) {
+  typedef std::pair<const int, double> V;
+  {
     typedef std::multimap<int, double> M;
     {
-        typedef M::size_type R;
-        V ar[] =
-        {
-            V(5, 1),
-            V(5, 2),
-            V(5, 3),
-            V(7, 1),
-            V(7, 2),
-            V(7, 3),
-            V(9, 1),
-            V(9, 2),
-            V(9, 3)
-        };
-        const M m(ar, ar+sizeof(ar)/sizeof(ar[0]));
-        R r = m.count(4);
-        assert(r == 0);
-        r = m.count(5);
-        assert(r == 3);
-        r = m.count(6);
-        assert(r == 0);
-        r = m.count(7);
-        assert(r == 3);
-        r = m.count(8);
-        assert(r == 0);
-        r = m.count(9);
-        assert(r == 3);
-        r = m.count(10);
-        assert(r == 0);
-    }
+      typedef M::size_type R;
+      V ar[] = {V(5, 1), V(5, 2), V(5, 3), V(7, 1), V(7, 2), V(7, 3), V(9, 1), V(9, 2), V(9, 3)};
+      const M m(ar, ar + sizeof(ar) / sizeof(ar[0]));
+      R r = m.count(4);
+      assert(r == 0);
+      r = m.count(5);
+      assert(r == 3);
+      r = m.count(6);
+      assert(r == 0);
+      r = m.count(7);
+      assert(r == 3);
+      r = m.count(8);
+      assert(r == 0);
+      r = m.count(9);
+      assert(r == 3);
+      r = m.count(10);
+      assert(r == 0);
     }
+  }
 #if TEST_STD_VER >= 11
-    {
+  {
     typedef std::multimap<int, double, std::less<int>, min_allocator<std::pair<const int, double>>> M;
     {
-        typedef M::size_type R;
-        V ar[] =
-        {
-            V(5, 1),
-            V(5, 2),
-            V(5, 3),
-            V(7, 1),
-            V(7, 2),
-            V(7, 3),
-            V(9, 1),
-            V(9, 2),
-            V(9, 3)
-        };
-        const M m(ar, ar+sizeof(ar)/sizeof(ar[0]));
-        R r = m.count(4);
-        assert(r == 0);
-        r = m.count(5);
-        assert(r == 3);
-        r = m.count(6);
-        assert(r == 0);
-        r = m.count(7);
-        assert(r == 3);
-        r = m.count(8);
-        assert(r == 0);
-        r = m.count(9);
-        assert(r == 3);
-        r = m.count(10);
-        assert(r == 0);
-    }
+      typedef M::size_type R;
+      V ar[] = {V(5, 1), V(5, 2), V(5, 3), V(7, 1), V(7, 2), V(7, 3), V(9, 1), V(9, 2), V(9, 3)};
+      const M m(ar, ar + sizeof(ar) / sizeof(ar[0]));
+      R r = m.count(4);
+      assert(r == 0);
+      r = m.count(5);
+      assert(r == 3);
+      r = m.count(6);
+      assert(r == 0);
+      r = m.count(7);
+      assert(r == 3);
+      r = m.count(8);
+      assert(r == 0);
+      r = m.count(9);
+      assert(r == 3);
+      r = m.count(10);
+      assert(r == 0);
     }
+  }
 #endif
 
 #if TEST_STD_VER > 11
-    {
+  {
     typedef std::multimap<int, double, std::less<>> M;
     typedef M::size_type R;
-    V ar[] =
-    {
-        V(5, 1),
-        V(5, 2),
-        V(5, 3),
-        V(7, 1),
-        V(7, 2),
-        V(7, 3),
-        V(9, 1),
-        V(9, 2),
-        V(9, 3)
-    };
-    const M m(ar, ar+sizeof(ar)/sizeof(ar[0]));
+    V ar[] = {V(5, 1), V(5, 2), V(5, 3), V(7, 1), V(7, 2), V(7, 3), V(9, 1), V(9, 2), V(9, 3)};
+    const M m(ar, ar + sizeof(ar) / sizeof(ar[0]));
     R r = m.count(4);
     assert(r == 0);
     r = m.count(5);
@@ -138,23 +104,23 @@ int main(int, char**)
     assert(r == 3);
     r = m.count(C2Int(10));
     assert(r == 0);
-    }
+  }
 
-    {
+  {
     typedef PrivateConstructor PC;
     typedef std::multimap<PC, double, std::less<>> M;
     typedef M::size_type R;
 
     M m;
-    m.insert ( std::make_pair<PC, double> ( PC::make(5), 1 ));
-    m.insert ( std::make_pair<PC, double> ( PC::make(5), 2 ));
-    m.insert ( std::make_pair<PC, double> ( PC::make(5), 3 ));
-    m.insert ( std::make_pair<PC, double> ( PC::make(7), 1 ));
-    m.insert ( std::make_pair<PC, double> ( PC::make(7), 2 ));
-    m.insert ( std::make_pair<PC, double> ( PC::make(7), 3 ));
-    m.insert ( std::make_pair<PC, double> ( PC::make(9), 1 ));
-    m.insert ( std::make_pair<PC, double> ( PC::make(9), 2 ));
-    m.insert ( std::make_pair<PC, double> ( PC::make(9), 3 ));
+    m.insert(std::make_pair<PC, double>(PC::make(5), 1));
+    m.insert(std::make_pair<PC, double>(PC::make(5), 2));
+    m.insert(std::make_pair<PC, double>(PC::make(5), 3));
+    m.insert(std::make_pair<PC, double>(PC::make(7), 1));
+    m.insert(std::make_pair<PC, double>(PC::make(7), 2));
+    m.insert(std::make_pair<PC, double>(PC::make(7), 3));
+    m.insert(std::make_pair<PC, double>(PC::make(9), 1));
+    m.insert(std::make_pair<PC, double>(PC::make(9), 2));
+    m.insert(std::make_pair<PC, double>(PC::make(9), 3));
 
     R r = m.count(4);
     assert(r == 0);
@@ -170,7 +136,7 @@ int main(int, char**)
     assert(r == 3);
     r = m.count(10);
     assert(r == 0);
-    }
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/associative/multimap/multimap.ops/count0.pass.cpp b/libcxx/test/std/containers/associative/multimap/multimap.ops/count0.pass.cpp
index bb220d553d513..233d1a11e1d6c 100644
--- a/libcxx/test/std/containers/associative/multimap/multimap.ops/count0.pass.cpp
+++ b/libcxx/test/std/containers/associative/multimap/multimap.ops/count0.pass.cpp
@@ -18,23 +18,21 @@
 // equal_range shall not participate in overload resolution unless the
 // qualified-id Compare::is_transparent is valid and denotes a type
 
-
 #include <map>
 #include <cassert>
 
 #include "test_macros.h"
 #include "is_transparent.h"
 
-int main(int, char**)
-{
-    {
+int main(int, char**) {
+  {
     typedef std::multimap<int, double, transparent_less> M;
     assert(M().count(C2Int{5}) == 0);
-    }
-    {
+  }
+  {
     typedef std::multimap<int, double, transparent_less_not_referenceable> M;
     assert(M().count(C2Int{5}) == 0);
-    }
+  }
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/associative/multimap/multimap.ops/count1.compile.fail.cpp b/libcxx/test/std/containers/associative/multimap/multimap.ops/count1.compile.fail.cpp
index 46231adfdf7d1..48b7cee7b0132 100644
--- a/libcxx/test/std/containers/associative/multimap/multimap.ops/count1.compile.fail.cpp
+++ b/libcxx/test/std/containers/associative/multimap/multimap.ops/count1.compile.fail.cpp
@@ -18,16 +18,14 @@
 // equal_range shall not participate in overload resolution unless the
 // qualified-id Compare::is_transparent is valid and denotes a type
 
-
 #include <map>
 #include <cassert>
 
 #include "test_macros.h"
 #include "is_transparent.h"
 
-int main(int, char**)
-{
-    typedef std::multimap<int, double, transparent_less_no_type> M;
+int main(int, char**) {
+  typedef std::multimap<int, double, transparent_less_no_type> M;
 
-    TEST_IGNORE_NODISCARD M().count(C2Int{5});
+  TEST_IGNORE_NODISCARD M().count(C2Int{5});
 }
diff --git a/libcxx/test/std/containers/associative/multimap/multimap.ops/count2.compile.fail.cpp b/libcxx/test/std/containers/associative/multimap/multimap.ops/count2.compile.fail.cpp
index f7324aa22adf3..dd8b3278ae765 100644
--- a/libcxx/test/std/containers/associative/multimap/multimap.ops/count2.compile.fail.cpp
+++ b/libcxx/test/std/containers/associative/multimap/multimap.ops/count2.compile.fail.cpp
@@ -18,16 +18,14 @@
 // equal_range shall not participate in overload resolution unless the
 // qualified-id Compare::is_transparent is valid and denotes a type
 
-
 #include <map>
 #include <cassert>
 
 #include "test_macros.h"
 #include "is_transparent.h"
 
-int main(int, char**)
-{
-    typedef std::multimap<int, double, transparent_less_private> M;
+int main(int, char**) {
+  typedef std::multimap<int, double, transparent_less_private> M;
 
-    TEST_IGNORE_NODISCARD M().count(C2Int{5});
+  TEST_IGNORE_NODISCARD M().count(C2Int{5});
 }
diff --git a/libcxx/test/std/containers/associative/multimap/multimap.ops/count3.compile.fail.cpp b/libcxx/test/std/containers/associative/multimap/multimap.ops/count3.compile.fail.cpp
index acceaae9db645..1cb7c48fc1087 100644
--- a/libcxx/test/std/containers/associative/multimap/multimap.ops/count3.compile.fail.cpp
+++ b/libcxx/test/std/containers/associative/multimap/multimap.ops/count3.compile.fail.cpp
@@ -18,16 +18,14 @@
 // equal_range shall not participate in overload resolution unless the
 // qualified-id Compare::is_transparent is valid and denotes a type
 
-
 #include <map>
 #include <cassert>
 
 #include "test_macros.h"
 #include "is_transparent.h"
 
-int main(int, char**)
-{
-    typedef std::multimap<int, double, transparent_less_not_a_type> M;
+int main(int, char**) {
+  typedef std::multimap<int, double, transparent_less_not_a_type> M;
 
-    TEST_IGNORE_NODISCARD M().count(C2Int{5});
+  TEST_IGNORE_NODISCARD M().count(C2Int{5});
 }
diff --git a/libcxx/test/std/containers/associative/multimap/multimap.ops/count_transparent.pass.cpp b/libcxx/test/std/containers/associative/multimap/multimap.ops/count_transparent.pass.cpp
index 4bb08c6d928cb..d8dc81884388e 100644
--- a/libcxx/test/std/containers/associative/multimap/multimap.ops/count_transparent.pass.cpp
+++ b/libcxx/test/std/containers/associative/multimap/multimap.ops/count_transparent.pass.cpp
@@ -22,23 +22,15 @@
 struct Comp {
   using is_transparent = void;
 
-  bool operator()(const std::pair<int, int> &lhs,
-                  const std::pair<int, int> &rhs) const {
-    return lhs < rhs;
-  }
-
-  bool operator()(const std::pair<int, int> &lhs, int rhs) const {
-    return lhs.first < rhs;
-  }
-
-  bool operator()(int lhs, const std::pair<int, int> &rhs) const {
-    return lhs < rhs.first;
-  }
+  bool operator()(const std::pair<int, int>& lhs, const std::pair<int, int>& rhs) const { return lhs < rhs; }
+
+  bool operator()(const std::pair<int, int>& lhs, int rhs) const { return lhs.first < rhs; }
+
+  bool operator()(int lhs, const std::pair<int, int>& rhs) const { return lhs < rhs.first; }
 };
 
 int main(int, char**) {
-  std::multimap<std::pair<int, int>, int, Comp> s{
-      {{2, 1}, 1}, {{1, 1}, 2}, {{1, 1}, 3}, {{1, 1}, 4}, {{2, 2}, 5}};
+  std::multimap<std::pair<int, int>, int, Comp> s{{{2, 1}, 1}, {{1, 1}, 2}, {{1, 1}, 3}, {{1, 1}, 4}, {{2, 2}, 5}};
 
   auto cnt = s.count(1);
   assert(cnt == 3);
diff --git a/libcxx/test/std/containers/associative/multimap/multimap.ops/equal_range.pass.cpp b/libcxx/test/std/containers/associative/multimap/multimap.ops/equal_range.pass.cpp
index c134c779a40c7..d06a04e40c6b0 100644
--- a/libcxx/test/std/containers/associative/multimap/multimap.ops/equal_range.pass.cpp
+++ b/libcxx/test/std/containers/associative/multimap/multimap.ops/equal_range.pass.cpp
@@ -21,183 +21,127 @@
 #include "private_constructor.h"
 #include "is_transparent.h"
 
-int main(int, char**)
-{
-    typedef std::pair<const int, double> V;
-    {
+int main(int, char**) {
+  typedef std::pair<const int, double> V;
+  {
     typedef std::multimap<int, double> M;
     {
-        typedef std::pair<M::iterator, M::iterator> R;
-        V ar[] =
-        {
-            V(5, 1),
-            V(5, 2),
-            V(5, 3),
-            V(7, 1),
-            V(7, 2),
-            V(7, 3),
-            V(9, 1),
-            V(9, 2),
-            V(9, 3)
-        };
-        M m(ar, ar+sizeof(ar)/sizeof(ar[0]));
-        R r = m.equal_range(4);
-        assert(r.first == m.begin());
-        assert(r.second == m.begin());
-        r = m.equal_range(5);
-        assert(r.first == m.begin());
-        assert(r.second == std::next(m.begin(), 3));
-        r = m.equal_range(6);
-        assert(r.first == std::next(m.begin(), 3));
-        assert(r.second == std::next(m.begin(), 3));
-        r = m.equal_range(7);
-        assert(r.first == std::next(m.begin(), 3));
-        assert(r.second == std::next(m.begin(), 6));
-        r = m.equal_range(8);
-        assert(r.first == std::next(m.begin(), 6));
-        assert(r.second == std::next(m.begin(), 6));
-        r = m.equal_range(9);
-        assert(r.first == std::next(m.begin(), 6));
-        assert(r.second == std::next(m.begin(), 9));
-        r = m.equal_range(10);
-        assert(r.first == m.end());
-        assert(r.second == m.end());
+      typedef std::pair<M::iterator, M::iterator> R;
+      V ar[] = {V(5, 1), V(5, 2), V(5, 3), V(7, 1), V(7, 2), V(7, 3), V(9, 1), V(9, 2), V(9, 3)};
+      M m(ar, ar + sizeof(ar) / sizeof(ar[0]));
+      R r = m.equal_range(4);
+      assert(r.first == m.begin());
+      assert(r.second == m.begin());
+      r = m.equal_range(5);
+      assert(r.first == m.begin());
+      assert(r.second == std::next(m.begin(), 3));
+      r = m.equal_range(6);
+      assert(r.first == std::next(m.begin(), 3));
+      assert(r.second == std::next(m.begin(), 3));
+      r = m.equal_range(7);
+      assert(r.first == std::next(m.begin(), 3));
+      assert(r.second == std::next(m.begin(), 6));
+      r = m.equal_range(8);
+      assert(r.first == std::next(m.begin(), 6));
+      assert(r.second == std::next(m.begin(), 6));
+      r = m.equal_range(9);
+      assert(r.first == std::next(m.begin(), 6));
+      assert(r.second == std::next(m.begin(), 9));
+      r = m.equal_range(10);
+      assert(r.first == m.end());
+      assert(r.second == m.end());
     }
     {
-        typedef std::pair<M::const_iterator, M::const_iterator> R;
-        V ar[] =
-        {
-            V(5, 1),
-            V(5, 2),
-            V(5, 3),
-            V(7, 1),
-            V(7, 2),
-            V(7, 3),
-            V(9, 1),
-            V(9, 2),
-            V(9, 3)
-        };
-        const M m(ar, ar+sizeof(ar)/sizeof(ar[0]));
-        R r = m.equal_range(4);
-        assert(r.first == m.begin());
-        assert(r.second == m.begin());
-        r = m.equal_range(5);
-        assert(r.first == m.begin());
-        assert(r.second == std::next(m.begin(), 3));
-        r = m.equal_range(6);
-        assert(r.first == std::next(m.begin(), 3));
-        assert(r.second == std::next(m.begin(), 3));
-        r = m.equal_range(7);
-        assert(r.first == std::next(m.begin(), 3));
-        assert(r.second == std::next(m.begin(), 6));
-        r = m.equal_range(8);
-        assert(r.first == std::next(m.begin(), 6));
-        assert(r.second == std::next(m.begin(), 6));
-        r = m.equal_range(9);
-        assert(r.first == std::next(m.begin(), 6));
-        assert(r.second == std::next(m.begin(), 9));
-        r = m.equal_range(10);
-        assert(r.first == m.end());
-        assert(r.second == m.end());
-    }
+      typedef std::pair<M::const_iterator, M::const_iterator> R;
+      V ar[] = {V(5, 1), V(5, 2), V(5, 3), V(7, 1), V(7, 2), V(7, 3), V(9, 1), V(9, 2), V(9, 3)};
+      const M m(ar, ar + sizeof(ar) / sizeof(ar[0]));
+      R r = m.equal_range(4);
+      assert(r.first == m.begin());
+      assert(r.second == m.begin());
+      r = m.equal_range(5);
+      assert(r.first == m.begin());
+      assert(r.second == std::next(m.begin(), 3));
+      r = m.equal_range(6);
+      assert(r.first == std::next(m.begin(), 3));
+      assert(r.second == std::next(m.begin(), 3));
+      r = m.equal_range(7);
+      assert(r.first == std::next(m.begin(), 3));
+      assert(r.second == std::next(m.begin(), 6));
+      r = m.equal_range(8);
+      assert(r.first == std::next(m.begin(), 6));
+      assert(r.second == std::next(m.begin(), 6));
+      r = m.equal_range(9);
+      assert(r.first == std::next(m.begin(), 6));
+      assert(r.second == std::next(m.begin(), 9));
+      r = m.equal_range(10);
+      assert(r.first == m.end());
+      assert(r.second == m.end());
     }
+  }
 #if TEST_STD_VER >= 11
-    {
+  {
     typedef std::multimap<int, double, std::less<int>, min_allocator<std::pair<const int, double>>> M;
     {
-        typedef std::pair<M::iterator, M::iterator> R;
-        V ar[] =
-        {
-            V(5, 1),
-            V(5, 2),
-            V(5, 3),
-            V(7, 1),
-            V(7, 2),
-            V(7, 3),
-            V(9, 1),
-            V(9, 2),
-            V(9, 3)
-        };
-        M m(ar, ar+sizeof(ar)/sizeof(ar[0]));
-        R r = m.equal_range(4);
-        assert(r.first == m.begin());
-        assert(r.second == m.begin());
-        r = m.equal_range(5);
-        assert(r.first == m.begin());
-        assert(r.second == std::next(m.begin(), 3));
-        r = m.equal_range(6);
-        assert(r.first == std::next(m.begin(), 3));
-        assert(r.second == std::next(m.begin(), 3));
-        r = m.equal_range(7);
-        assert(r.first == std::next(m.begin(), 3));
-        assert(r.second == std::next(m.begin(), 6));
-        r = m.equal_range(8);
-        assert(r.first == std::next(m.begin(), 6));
-        assert(r.second == std::next(m.begin(), 6));
-        r = m.equal_range(9);
-        assert(r.first == std::next(m.begin(), 6));
-        assert(r.second == std::next(m.begin(), 9));
-        r = m.equal_range(10);
-        assert(r.first == m.end());
-        assert(r.second == m.end());
+      typedef std::pair<M::iterator, M::iterator> R;
+      V ar[] = {V(5, 1), V(5, 2), V(5, 3), V(7, 1), V(7, 2), V(7, 3), V(9, 1), V(9, 2), V(9, 3)};
+      M m(ar, ar + sizeof(ar) / sizeof(ar[0]));
+      R r = m.equal_range(4);
+      assert(r.first == m.begin());
+      assert(r.second == m.begin());
+      r = m.equal_range(5);
+      assert(r.first == m.begin());
+      assert(r.second == std::next(m.begin(), 3));
+      r = m.equal_range(6);
+      assert(r.first == std::next(m.begin(), 3));
+      assert(r.second == std::next(m.begin(), 3));
+      r = m.equal_range(7);
+      assert(r.first == std::next(m.begin(), 3));
+      assert(r.second == std::next(m.begin(), 6));
+      r = m.equal_range(8);
+      assert(r.first == std::next(m.begin(), 6));
+      assert(r.second == std::next(m.begin(), 6));
+      r = m.equal_range(9);
+      assert(r.first == std::next(m.begin(), 6));
+      assert(r.second == std::next(m.begin(), 9));
+      r = m.equal_range(10);
+      assert(r.first == m.end());
+      assert(r.second == m.end());
     }
     {
-        typedef std::pair<M::const_iterator, M::const_iterator> R;
-        V ar[] =
-        {
-            V(5, 1),
-            V(5, 2),
-            V(5, 3),
-            V(7, 1),
-            V(7, 2),
-            V(7, 3),
-            V(9, 1),
-            V(9, 2),
-            V(9, 3)
-        };
-        const M m(ar, ar+sizeof(ar)/sizeof(ar[0]));
-        R r = m.equal_range(4);
-        assert(r.first == m.begin());
-        assert(r.second == m.begin());
-        r = m.equal_range(5);
-        assert(r.first == m.begin());
-        assert(r.second == std::next(m.begin(), 3));
-        r = m.equal_range(6);
-        assert(r.first == std::next(m.begin(), 3));
-        assert(r.second == std::next(m.begin(), 3));
-        r = m.equal_range(7);
-        assert(r.first == std::next(m.begin(), 3));
-        assert(r.second == std::next(m.begin(), 6));
-        r = m.equal_range(8);
-        assert(r.first == std::next(m.begin(), 6));
-        assert(r.second == std::next(m.begin(), 6));
-        r = m.equal_range(9);
-        assert(r.first == std::next(m.begin(), 6));
-        assert(r.second == std::next(m.begin(), 9));
-        r = m.equal_range(10);
-        assert(r.first == m.end());
-        assert(r.second == m.end());
-    }
+      typedef std::pair<M::const_iterator, M::const_iterator> R;
+      V ar[] = {V(5, 1), V(5, 2), V(5, 3), V(7, 1), V(7, 2), V(7, 3), V(9, 1), V(9, 2), V(9, 3)};
+      const M m(ar, ar + sizeof(ar) / sizeof(ar[0]));
+      R r = m.equal_range(4);
+      assert(r.first == m.begin());
+      assert(r.second == m.begin());
+      r = m.equal_range(5);
+      assert(r.first == m.begin());
+      assert(r.second == std::next(m.begin(), 3));
+      r = m.equal_range(6);
+      assert(r.first == std::next(m.begin(), 3));
+      assert(r.second == std::next(m.begin(), 3));
+      r = m.equal_range(7);
+      assert(r.first == std::next(m.begin(), 3));
+      assert(r.second == std::next(m.begin(), 6));
+      r = m.equal_range(8);
+      assert(r.first == std::next(m.begin(), 6));
+      assert(r.second == std::next(m.begin(), 6));
+      r = m.equal_range(9);
+      assert(r.first == std::next(m.begin(), 6));
+      assert(r.second == std::next(m.begin(), 9));
+      r = m.equal_range(10);
+      assert(r.first == m.end());
+      assert(r.second == m.end());
     }
+  }
 #endif
 #if TEST_STD_VER > 11
-    {
+  {
     typedef std::multimap<int, double, std::less<>> M;
 
     typedef std::pair<M::iterator, M::iterator> R;
-    V ar[] =
-    {
-        V(5, 1),
-        V(5, 2),
-        V(5, 3),
-        V(7, 1),
-        V(7, 2),
-        V(7, 3),
-        V(9, 1),
-        V(9, 2),
-        V(9, 3)
-    };
-    M m(ar, ar+sizeof(ar)/sizeof(ar[0]));
+    V ar[] = {V(5, 1), V(5, 2), V(5, 3), V(7, 1), V(7, 2), V(7, 3), V(9, 1), V(9, 2), V(9, 3)};
+    M m(ar, ar + sizeof(ar) / sizeof(ar[0]));
     R r = m.equal_range(4);
     assert(r.first == m.begin());
     assert(r.second == m.begin());
@@ -241,25 +185,25 @@ int main(int, char**)
     r = m.equal_range(C2Int(10));
     assert(r.first == m.end());
     assert(r.second == m.end());
-    }
+  }
 
-    {
+  {
     typedef PrivateConstructor PC;
     typedef std::multimap<PC, double, std::less<>> M;
     typedef std::pair<M::iterator, M::iterator> R;
 
     M m;
-    m.insert ( std::make_pair<PC, double> ( PC::make(5), 1 ));
-    m.insert ( std::make_pair<PC, double> ( PC::make(5), 2 ));
-    m.insert ( std::make_pair<PC, double> ( PC::make(5), 3 ));
-    m.insert ( std::make_pair<PC, double> ( PC::make(7), 1 ));
-    m.insert ( std::make_pair<PC, double> ( PC::make(7), 2 ));
-    m.insert ( std::make_pair<PC, double> ( PC::make(7), 3 ));
-    m.insert ( std::make_pair<PC, double> ( PC::make(9), 1 ));
-    m.insert ( std::make_pair<PC, double> ( PC::make(9), 2 ));
-    m.insert ( std::make_pair<PC, double> ( PC::make(9), 3 ));
+    m.insert(std::make_pair<PC, double>(PC::make(5), 1));
+    m.insert(std::make_pair<PC, double>(PC::make(5), 2));
+    m.insert(std::make_pair<PC, double>(PC::make(5), 3));
+    m.insert(std::make_pair<PC, double>(PC::make(7), 1));
+    m.insert(std::make_pair<PC, double>(PC::make(7), 2));
+    m.insert(std::make_pair<PC, double>(PC::make(7), 3));
+    m.insert(std::make_pair<PC, double>(PC::make(9), 1));
+    m.insert(std::make_pair<PC, double>(PC::make(9), 2));
+    m.insert(std::make_pair<PC, double>(PC::make(9), 3));
 
-//  assert(m.size() == 9);
+    //  assert(m.size() == 9);
     R r = m.equal_range(4);
     assert(r.first == m.begin());
     assert(r.second == m.begin());
@@ -281,7 +225,7 @@ int main(int, char**)
     r = m.equal_range(10);
     assert(r.first == m.end());
     assert(r.second == m.end());
-    }
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/associative/multimap/multimap.ops/equal_range0.pass.cpp b/libcxx/test/std/containers/associative/multimap/multimap.ops/equal_range0.pass.cpp
index 470ac4aae4905..0bead6c7938db 100644
--- a/libcxx/test/std/containers/associative/multimap/multimap.ops/equal_range0.pass.cpp
+++ b/libcxx/test/std/containers/associative/multimap/multimap.ops/equal_range0.pass.cpp
@@ -19,29 +19,27 @@
 // equal_range shall not participate in overload resolution unless the
 // qualified-id Compare::is_transparent is valid and denotes a type
 
-
 #include <map>
 #include <cassert>
 
 #include "test_macros.h"
 #include "is_transparent.h"
 
-int main(int, char**)
-{
-    {
+int main(int, char**) {
+  {
     typedef std::multimap<int, double, transparent_less> M;
     typedef std::pair<typename M::iterator, typename M::iterator> P;
     M example;
     P result = example.equal_range(C2Int{5});
     assert(result.first == result.second);
-    }
-    {
+  }
+  {
     typedef std::multimap<int, double, transparent_less_not_referenceable> M;
     typedef std::pair<typename M::iterator, typename M::iterator> P;
     M example;
     P result = example.equal_range(C2Int{5});
     assert(result.first == result.second);
-    }
+  }
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/associative/multimap/multimap.ops/equal_range1.compile.fail.cpp b/libcxx/test/std/containers/associative/multimap/multimap.ops/equal_range1.compile.fail.cpp
index dff6e13c2ec8c..31ab8071553de 100644
--- a/libcxx/test/std/containers/associative/multimap/multimap.ops/equal_range1.compile.fail.cpp
+++ b/libcxx/test/std/containers/associative/multimap/multimap.ops/equal_range1.compile.fail.cpp
@@ -19,16 +19,14 @@
 // equal_range shall not participate in overload resolution unless the
 // qualified-id Compare::is_transparent is valid and denotes a type
 
-
 #include <map>
 #include <cassert>
 
 #include "test_macros.h"
 #include "is_transparent.h"
 
-int main(int, char**)
-{
-    typedef std::multimap<int, double, transparent_less_no_type> M;
+int main(int, char**) {
+  typedef std::multimap<int, double, transparent_less_no_type> M;
 
-    TEST_IGNORE_NODISCARD M().equal_range(C2Int{5});
+  TEST_IGNORE_NODISCARD M().equal_range(C2Int{5});
 }
diff --git a/libcxx/test/std/containers/associative/multimap/multimap.ops/equal_range2.compile.fail.cpp b/libcxx/test/std/containers/associative/multimap/multimap.ops/equal_range2.compile.fail.cpp
index a57c8404a5fc8..7b92baaed355b 100644
--- a/libcxx/test/std/containers/associative/multimap/multimap.ops/equal_range2.compile.fail.cpp
+++ b/libcxx/test/std/containers/associative/multimap/multimap.ops/equal_range2.compile.fail.cpp
@@ -19,18 +19,16 @@
 // equal_range shall not participate in overload resolution unless the
 // qualified-id Compare::is_transparent is valid and denotes a type
 
-
 #include <map>
 #include <cassert>
 
 #include "test_macros.h"
 #include "is_transparent.h"
 
-int main(int, char**)
-{
-    {
+int main(int, char**) {
+  {
     typedef std::multimap<int, double, transparent_less_private> M;
 
     TEST_IGNORE_NODISCARD M().equal_range(C2Int{5});
-    }
+  }
 }
diff --git a/libcxx/test/std/containers/associative/multimap/multimap.ops/equal_range3.compile.fail.cpp b/libcxx/test/std/containers/associative/multimap/multimap.ops/equal_range3.compile.fail.cpp
index e215d0fe8b996..a5ce74a8175c8 100644
--- a/libcxx/test/std/containers/associative/multimap/multimap.ops/equal_range3.compile.fail.cpp
+++ b/libcxx/test/std/containers/associative/multimap/multimap.ops/equal_range3.compile.fail.cpp
@@ -19,18 +19,16 @@
 // equal_range shall not participate in overload resolution unless the
 // qualified-id Compare::is_transparent is valid and denotes a type
 
-
 #include <map>
 #include <cassert>
 
 #include "test_macros.h"
 #include "is_transparent.h"
 
-int main(int, char**)
-{
-    {
+int main(int, char**) {
+  {
     typedef std::multimap<int, double, transparent_less_not_a_type> M;
 
     TEST_IGNORE_NODISCARD M().equal_range(C2Int{5});
-    }
+  }
 }
diff --git a/libcxx/test/std/containers/associative/multimap/multimap.ops/equal_range_transparent.pass.cpp b/libcxx/test/std/containers/associative/multimap/multimap.ops/equal_range_transparent.pass.cpp
index 4315d573c6e2c..f2caf36b1a4ee 100644
--- a/libcxx/test/std/containers/associative/multimap/multimap.ops/equal_range_transparent.pass.cpp
+++ b/libcxx/test/std/containers/associative/multimap/multimap.ops/equal_range_transparent.pass.cpp
@@ -25,25 +25,17 @@
 struct Comp {
   using is_transparent = void;
 
-  bool operator()(const std::pair<int, int> &lhs,
-                  const std::pair<int, int> &rhs) const {
-    return lhs < rhs;
-  }
+  bool operator()(const std::pair<int, int>& lhs, const std::pair<int, int>& rhs) const { return lhs < rhs; }
 
-  bool operator()(const std::pair<int, int> &lhs, int rhs) const {
-    return lhs.first < rhs;
-  }
+  bool operator()(const std::pair<int, int>& lhs, int rhs) const { return lhs.first < rhs; }
 
-  bool operator()(int lhs, const std::pair<int, int> &rhs) const {
-    return lhs < rhs.first;
-  }
+  bool operator()(int lhs, const std::pair<int, int>& rhs) const { return lhs < rhs.first; }
 };
 
 int main(int, char**) {
-  std::multimap<std::pair<int, int>, int, Comp> s{
-      {{2, 1}, 1}, {{1, 1}, 2}, {{1, 1}, 3}, {{1, 1}, 4}, {{2, 2}, 5}};
+  std::multimap<std::pair<int, int>, int, Comp> s{{{2, 1}, 1}, {{1, 1}, 2}, {{1, 1}, 3}, {{1, 1}, 4}, {{2, 2}, 5}};
 
-  auto er = s.equal_range(1);
+  auto er   = s.equal_range(1);
   long nels = 0;
 
   for (auto it = er.first; it != er.second; it++) {
diff --git a/libcxx/test/std/containers/associative/multimap/multimap.ops/find.pass.cpp b/libcxx/test/std/containers/associative/multimap/multimap.ops/find.pass.cpp
index 550e49a6ddf9a..6d5018ff5263e 100644
--- a/libcxx/test/std/containers/associative/multimap/multimap.ops/find.pass.cpp
+++ b/libcxx/test/std/containers/associative/multimap/multimap.ops/find.pass.cpp
@@ -21,189 +21,133 @@
 #include "private_constructor.h"
 #include "is_transparent.h"
 
-int main(int, char**)
-{
-    typedef std::pair<const int, double> V;
-    {
+int main(int, char**) {
+  typedef std::pair<const int, double> V;
+  {
     typedef std::multimap<int, double> M;
     {
-        typedef M::iterator R;
-        V ar[] =
-        {
-            V(5, 1),
-            V(5, 2),
-            V(5, 3),
-            V(7, 1),
-            V(7, 2),
-            V(7, 3),
-            V(9, 1),
-            V(9, 2),
-            V(9, 3)
-        };
-        M m(ar, ar+sizeof(ar)/sizeof(ar[0]));
-        R r = m.find(5);
-        assert(r == m.begin());
-        r = m.find(6);
-        assert(r == m.end());
-        r = m.find(7);
-        assert(r == std::next(m.begin(), 3));
-        r = m.find(8);
-        assert(r == m.end());
-        r = m.find(9);
-        assert(r == std::next(m.begin(), 6));
-        r = m.find(10);
-        assert(r == m.end());
+      typedef M::iterator R;
+      V ar[] = {V(5, 1), V(5, 2), V(5, 3), V(7, 1), V(7, 2), V(7, 3), V(9, 1), V(9, 2), V(9, 3)};
+      M m(ar, ar + sizeof(ar) / sizeof(ar[0]));
+      R r = m.find(5);
+      assert(r == m.begin());
+      r = m.find(6);
+      assert(r == m.end());
+      r = m.find(7);
+      assert(r == std::next(m.begin(), 3));
+      r = m.find(8);
+      assert(r == m.end());
+      r = m.find(9);
+      assert(r == std::next(m.begin(), 6));
+      r = m.find(10);
+      assert(r == m.end());
     }
     {
-        typedef M::const_iterator R;
-        V ar[] =
-        {
-            V(5, 1),
-            V(5, 2),
-            V(5, 3),
-            V(7, 1),
-            V(7, 2),
-            V(7, 3),
-            V(9, 1),
-            V(9, 2),
-            V(9, 3)
-        };
-        const M m(ar, ar+sizeof(ar)/sizeof(ar[0]));
-        R r = m.find(5);
-        assert(r == m.begin());
-        r = m.find(6);
-        assert(r == m.end());
-        r = m.find(7);
-        assert(r == std::next(m.begin(), 3));
-        r = m.find(8);
-        assert(r == m.end());
-        r = m.find(9);
-        assert(r == std::next(m.begin(), 6));
-        r = m.find(10);
-        assert(r == m.end());
-    }
+      typedef M::const_iterator R;
+      V ar[] = {V(5, 1), V(5, 2), V(5, 3), V(7, 1), V(7, 2), V(7, 3), V(9, 1), V(9, 2), V(9, 3)};
+      const M m(ar, ar + sizeof(ar) / sizeof(ar[0]));
+      R r = m.find(5);
+      assert(r == m.begin());
+      r = m.find(6);
+      assert(r == m.end());
+      r = m.find(7);
+      assert(r == std::next(m.begin(), 3));
+      r = m.find(8);
+      assert(r == m.end());
+      r = m.find(9);
+      assert(r == std::next(m.begin(), 6));
+      r = m.find(10);
+      assert(r == m.end());
     }
+  }
 #if TEST_STD_VER >= 11
-    {
+  {
     typedef std::multimap<int, double, std::less<int>, min_allocator<std::pair<const int, double>>> M;
     {
-        typedef M::iterator R;
-        V ar[] =
-        {
-            V(5, 1),
-            V(5, 2),
-            V(5, 3),
-            V(7, 1),
-            V(7, 2),
-            V(7, 3),
-            V(9, 1),
-            V(9, 2),
-            V(9, 3)
-        };
-        M m(ar, ar+sizeof(ar)/sizeof(ar[0]));
-        R r = m.find(5);
-        assert(r == m.begin());
-        r = m.find(6);
-        assert(r == m.end());
-        r = m.find(7);
-        assert(r == std::next(m.begin(), 3));
-        r = m.find(8);
-        assert(r == m.end());
-        r = m.find(9);
-        assert(r == std::next(m.begin(), 6));
-        r = m.find(10);
-        assert(r == m.end());
+      typedef M::iterator R;
+      V ar[] = {V(5, 1), V(5, 2), V(5, 3), V(7, 1), V(7, 2), V(7, 3), V(9, 1), V(9, 2), V(9, 3)};
+      M m(ar, ar + sizeof(ar) / sizeof(ar[0]));
+      R r = m.find(5);
+      assert(r == m.begin());
+      r = m.find(6);
+      assert(r == m.end());
+      r = m.find(7);
+      assert(r == std::next(m.begin(), 3));
+      r = m.find(8);
+      assert(r == m.end());
+      r = m.find(9);
+      assert(r == std::next(m.begin(), 6));
+      r = m.find(10);
+      assert(r == m.end());
     }
     {
-        typedef M::const_iterator R;
-        V ar[] =
-        {
-            V(5, 1),
-            V(5, 2),
-            V(5, 3),
-            V(7, 1),
-            V(7, 2),
-            V(7, 3),
-            V(9, 1),
-            V(9, 2),
-            V(9, 3)
-        };
-        const M m(ar, ar+sizeof(ar)/sizeof(ar[0]));
-        R r = m.find(5);
-        assert(r == m.begin());
-        r = m.find(6);
-        assert(r == m.end());
-        r = m.find(7);
-        assert(r == std::next(m.begin(), 3));
-        r = m.find(8);
-        assert(r == m.end());
-        r = m.find(9);
-        assert(r == std::next(m.begin(), 6));
-        r = m.find(10);
-        assert(r == m.end());
-    }
+      typedef M::const_iterator R;
+      V ar[] = {V(5, 1), V(5, 2), V(5, 3), V(7, 1), V(7, 2), V(7, 3), V(9, 1), V(9, 2), V(9, 3)};
+      const M m(ar, ar + sizeof(ar) / sizeof(ar[0]));
+      R r = m.find(5);
+      assert(r == m.begin());
+      r = m.find(6);
+      assert(r == m.end());
+      r = m.find(7);
+      assert(r == std::next(m.begin(), 3));
+      r = m.find(8);
+      assert(r == m.end());
+      r = m.find(9);
+      assert(r == std::next(m.begin(), 6));
+      r = m.find(10);
+      assert(r == m.end());
     }
+  }
 #endif
 #if TEST_STD_VER > 11
-    {
+  {
     typedef std::multimap<int, double, std::less<>> M;
     typedef M::iterator R;
 
-        V ar[] =
-        {
-            V(5, 1),
-            V(5, 2),
-            V(5, 3),
-            V(7, 1),
-            V(7, 2),
-            V(7, 3),
-            V(9, 1),
-            V(9, 2),
-            V(9, 3)
-        };
-        M m(ar, ar+sizeof(ar)/sizeof(ar[0]));
-        R r = m.find(5);
-        assert(r == m.begin());
-        r = m.find(6);
-        assert(r == m.end());
-        r = m.find(7);
-        assert(r == std::next(m.begin(), 3));
-        r = m.find(8);
-        assert(r == m.end());
-        r = m.find(9);
-        assert(r == std::next(m.begin(), 6));
-        r = m.find(10);
-        assert(r == m.end());
+    V ar[] = {V(5, 1), V(5, 2), V(5, 3), V(7, 1), V(7, 2), V(7, 3), V(9, 1), V(9, 2), V(9, 3)};
+    M m(ar, ar + sizeof(ar) / sizeof(ar[0]));
+    R r = m.find(5);
+    assert(r == m.begin());
+    r = m.find(6);
+    assert(r == m.end());
+    r = m.find(7);
+    assert(r == std::next(m.begin(), 3));
+    r = m.find(8);
+    assert(r == m.end());
+    r = m.find(9);
+    assert(r == std::next(m.begin(), 6));
+    r = m.find(10);
+    assert(r == m.end());
 
-        r = m.find(C2Int(5));
-        assert(r == m.begin());
-        r = m.find(C2Int(6));
-        assert(r == m.end());
-        r = m.find(C2Int(7));
-        assert(r == std::next(m.begin(), 3));
-        r = m.find(C2Int(8));
-        assert(r == m.end());
-        r = m.find(C2Int(9));
-        assert(r == std::next(m.begin(), 6));
-        r = m.find(C2Int(10));
-        assert(r == m.end());
-    }
+    r = m.find(C2Int(5));
+    assert(r == m.begin());
+    r = m.find(C2Int(6));
+    assert(r == m.end());
+    r = m.find(C2Int(7));
+    assert(r == std::next(m.begin(), 3));
+    r = m.find(C2Int(8));
+    assert(r == m.end());
+    r = m.find(C2Int(9));
+    assert(r == std::next(m.begin(), 6));
+    r = m.find(C2Int(10));
+    assert(r == m.end());
+  }
 
-    {
+  {
     typedef PrivateConstructor PC;
     typedef std::multimap<PC, double, std::less<>> M;
     typedef M::iterator R;
 
     M m;
-    m.insert ( std::make_pair<PC, double> ( PC::make(5), 1 ));
-    m.insert ( std::make_pair<PC, double> ( PC::make(5), 2 ));
-    m.insert ( std::make_pair<PC, double> ( PC::make(5), 3 ));
-    m.insert ( std::make_pair<PC, double> ( PC::make(7), 1 ));
-    m.insert ( std::make_pair<PC, double> ( PC::make(7), 2 ));
-    m.insert ( std::make_pair<PC, double> ( PC::make(7), 3 ));
-    m.insert ( std::make_pair<PC, double> ( PC::make(9), 1 ));
-    m.insert ( std::make_pair<PC, double> ( PC::make(9), 2 ));
-    m.insert ( std::make_pair<PC, double> ( PC::make(9), 3 ));
+    m.insert(std::make_pair<PC, double>(PC::make(5), 1));
+    m.insert(std::make_pair<PC, double>(PC::make(5), 2));
+    m.insert(std::make_pair<PC, double>(PC::make(5), 3));
+    m.insert(std::make_pair<PC, double>(PC::make(7), 1));
+    m.insert(std::make_pair<PC, double>(PC::make(7), 2));
+    m.insert(std::make_pair<PC, double>(PC::make(7), 3));
+    m.insert(std::make_pair<PC, double>(PC::make(9), 1));
+    m.insert(std::make_pair<PC, double>(PC::make(9), 2));
+    m.insert(std::make_pair<PC, double>(PC::make(9), 3));
 
     R r = m.find(5);
     assert(r == m.begin());
@@ -217,7 +161,7 @@ int main(int, char**)
     assert(r == std::next(m.begin(), 6));
     r = m.find(10);
     assert(r == m.end());
-    }
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/associative/multimap/multimap.ops/find0.pass.cpp b/libcxx/test/std/containers/associative/multimap/multimap.ops/find0.pass.cpp
index d5f3969cd89e8..701d4e314e7a8 100644
--- a/libcxx/test/std/containers/associative/multimap/multimap.ops/find0.pass.cpp
+++ b/libcxx/test/std/containers/associative/multimap/multimap.ops/find0.pass.cpp
@@ -19,25 +19,23 @@
 // equal_range shall not participate in overload resolution unless the
 // qualified-id Compare::is_transparent is valid and denotes a type
 
-
 #include <map>
 #include <cassert>
 
 #include "test_macros.h"
 #include "is_transparent.h"
 
-int main(int, char**)
-{
-    {
+int main(int, char**) {
+  {
     typedef std::multimap<int, double, transparent_less> M;
     M example;
     assert(example.find(C2Int{5}) == example.end());
-    }
-    {
+  }
+  {
     typedef std::multimap<int, double, transparent_less_not_referenceable> M;
     M example;
     assert(example.find(C2Int{5}) == example.end());
-    }
+  }
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/associative/multimap/multimap.ops/find1.compile.fail.cpp b/libcxx/test/std/containers/associative/multimap/multimap.ops/find1.compile.fail.cpp
index 162d334f27593..170bd8b1aaba3 100644
--- a/libcxx/test/std/containers/associative/multimap/multimap.ops/find1.compile.fail.cpp
+++ b/libcxx/test/std/containers/associative/multimap/multimap.ops/find1.compile.fail.cpp
@@ -19,18 +19,16 @@
 // equal_range shall not participate in overload resolution unless the
 // qualified-id Compare::is_transparent is valid and denotes a type
 
-
 #include <map>
 #include <cassert>
 
 #include "test_macros.h"
 #include "is_transparent.h"
 
-int main(int, char**)
-{
-    {
+int main(int, char**) {
+  {
     typedef std::multimap<int, double, transparent_less_no_type> M;
 
     TEST_IGNORE_NODISCARD M().find(C2Int{5});
-    }
+  }
 }
diff --git a/libcxx/test/std/containers/associative/multimap/multimap.ops/find2.compile.fail.cpp b/libcxx/test/std/containers/associative/multimap/multimap.ops/find2.compile.fail.cpp
index fe3756e0bba36..18eaa45ddd741 100644
--- a/libcxx/test/std/containers/associative/multimap/multimap.ops/find2.compile.fail.cpp
+++ b/libcxx/test/std/containers/associative/multimap/multimap.ops/find2.compile.fail.cpp
@@ -19,18 +19,16 @@
 // equal_range shall not participate in overload resolution unless the
 // qualified-id Compare::is_transparent is valid and denotes a type
 
-
 #include <map>
 #include <cassert>
 
 #include "test_macros.h"
 #include "is_transparent.h"
 
-int main(int, char**)
-{
-    {
+int main(int, char**) {
+  {
     typedef std::multimap<int, double, transparent_less_private> M;
 
     TEST_IGNORE_NODISCARD M().find(C2Int{5});
-    }
+  }
 }
diff --git a/libcxx/test/std/containers/associative/multimap/multimap.ops/find3.compile.fail.cpp b/libcxx/test/std/containers/associative/multimap/multimap.ops/find3.compile.fail.cpp
index b5cce48928906..66263aa07a5b0 100644
--- a/libcxx/test/std/containers/associative/multimap/multimap.ops/find3.compile.fail.cpp
+++ b/libcxx/test/std/containers/associative/multimap/multimap.ops/find3.compile.fail.cpp
@@ -19,18 +19,16 @@
 // equal_range shall not participate in overload resolution unless the
 // qualified-id Compare::is_transparent is valid and denotes a type
 
-
 #include <map>
 #include <cassert>
 
 #include "test_macros.h"
 #include "is_transparent.h"
 
-int main(int, char**)
-{
-    {
+int main(int, char**) {
+  {
     typedef std::multimap<int, double, transparent_less_not_a_type> M;
 
     TEST_IGNORE_NODISCARD M().find(C2Int{5});
-    }
+  }
 }
diff --git a/libcxx/test/std/containers/associative/multimap/multimap.ops/lower_bound.pass.cpp b/libcxx/test/std/containers/associative/multimap/multimap.ops/lower_bound.pass.cpp
index aae11c8bba983..704cfff775884 100644
--- a/libcxx/test/std/containers/associative/multimap/multimap.ops/lower_bound.pass.cpp
+++ b/libcxx/test/std/containers/associative/multimap/multimap.ops/lower_bound.pass.cpp
@@ -21,154 +21,98 @@
 #include "private_constructor.h"
 #include "is_transparent.h"
 
-int main(int, char**)
-{
-    typedef std::pair<const int, double> V;
-    {
+int main(int, char**) {
+  typedef std::pair<const int, double> V;
+  {
     typedef std::multimap<int, double> M;
     {
-        typedef M::iterator R;
-        V ar[] =
-        {
-            V(5, 1),
-            V(5, 2),
-            V(5, 3),
-            V(7, 1),
-            V(7, 2),
-            V(7, 3),
-            V(9, 1),
-            V(9, 2),
-            V(9, 3)
-        };
-        M m(ar, ar+sizeof(ar)/sizeof(ar[0]));
-        R r = m.lower_bound(4);
-        assert(r == m.begin());
-        r = m.lower_bound(5);
-        assert(r == m.begin());
-        r = m.lower_bound(6);
-        assert(r == std::next(m.begin(), 3));
-        r = m.lower_bound(7);
-        assert(r == std::next(m.begin(), 3));
-        r = m.lower_bound(8);
-        assert(r == std::next(m.begin(), 6));
-        r = m.lower_bound(9);
-        assert(r == std::next(m.begin(), 6));
-        r = m.lower_bound(10);
-        assert(r == m.end());
+      typedef M::iterator R;
+      V ar[] = {V(5, 1), V(5, 2), V(5, 3), V(7, 1), V(7, 2), V(7, 3), V(9, 1), V(9, 2), V(9, 3)};
+      M m(ar, ar + sizeof(ar) / sizeof(ar[0]));
+      R r = m.lower_bound(4);
+      assert(r == m.begin());
+      r = m.lower_bound(5);
+      assert(r == m.begin());
+      r = m.lower_bound(6);
+      assert(r == std::next(m.begin(), 3));
+      r = m.lower_bound(7);
+      assert(r == std::next(m.begin(), 3));
+      r = m.lower_bound(8);
+      assert(r == std::next(m.begin(), 6));
+      r = m.lower_bound(9);
+      assert(r == std::next(m.begin(), 6));
+      r = m.lower_bound(10);
+      assert(r == m.end());
     }
     {
-        typedef M::const_iterator R;
-        V ar[] =
-        {
-            V(5, 1),
-            V(5, 2),
-            V(5, 3),
-            V(7, 1),
-            V(7, 2),
-            V(7, 3),
-            V(9, 1),
-            V(9, 2),
-            V(9, 3)
-        };
-        const M m(ar, ar+sizeof(ar)/sizeof(ar[0]));
-        R r = m.lower_bound(4);
-        assert(r == m.begin());
-        r = m.lower_bound(5);
-        assert(r == m.begin());
-        r = m.lower_bound(6);
-        assert(r == std::next(m.begin(), 3));
-        r = m.lower_bound(7);
-        assert(r == std::next(m.begin(), 3));
-        r = m.lower_bound(8);
-        assert(r == std::next(m.begin(), 6));
-        r = m.lower_bound(9);
-        assert(r == std::next(m.begin(), 6));
-        r = m.lower_bound(10);
-        assert(r == m.end());
-    }
+      typedef M::const_iterator R;
+      V ar[] = {V(5, 1), V(5, 2), V(5, 3), V(7, 1), V(7, 2), V(7, 3), V(9, 1), V(9, 2), V(9, 3)};
+      const M m(ar, ar + sizeof(ar) / sizeof(ar[0]));
+      R r = m.lower_bound(4);
+      assert(r == m.begin());
+      r = m.lower_bound(5);
+      assert(r == m.begin());
+      r = m.lower_bound(6);
+      assert(r == std::next(m.begin(), 3));
+      r = m.lower_bound(7);
+      assert(r == std::next(m.begin(), 3));
+      r = m.lower_bound(8);
+      assert(r == std::next(m.begin(), 6));
+      r = m.lower_bound(9);
+      assert(r == std::next(m.begin(), 6));
+      r = m.lower_bound(10);
+      assert(r == m.end());
     }
+  }
 #if TEST_STD_VER >= 11
-    {
+  {
     typedef std::multimap<int, double, std::less<int>, min_allocator<std::pair<const int, double>>> M;
     {
-        typedef M::iterator R;
-        V ar[] =
-        {
-            V(5, 1),
-            V(5, 2),
-            V(5, 3),
-            V(7, 1),
-            V(7, 2),
-            V(7, 3),
-            V(9, 1),
-            V(9, 2),
-            V(9, 3)
-        };
-        M m(ar, ar+sizeof(ar)/sizeof(ar[0]));
-        R r = m.lower_bound(4);
-        assert(r == m.begin());
-        r = m.lower_bound(5);
-        assert(r == m.begin());
-        r = m.lower_bound(6);
-        assert(r == std::next(m.begin(), 3));
-        r = m.lower_bound(7);
-        assert(r == std::next(m.begin(), 3));
-        r = m.lower_bound(8);
-        assert(r == std::next(m.begin(), 6));
-        r = m.lower_bound(9);
-        assert(r == std::next(m.begin(), 6));
-        r = m.lower_bound(10);
-        assert(r == m.end());
+      typedef M::iterator R;
+      V ar[] = {V(5, 1), V(5, 2), V(5, 3), V(7, 1), V(7, 2), V(7, 3), V(9, 1), V(9, 2), V(9, 3)};
+      M m(ar, ar + sizeof(ar) / sizeof(ar[0]));
+      R r = m.lower_bound(4);
+      assert(r == m.begin());
+      r = m.lower_bound(5);
+      assert(r == m.begin());
+      r = m.lower_bound(6);
+      assert(r == std::next(m.begin(), 3));
+      r = m.lower_bound(7);
+      assert(r == std::next(m.begin(), 3));
+      r = m.lower_bound(8);
+      assert(r == std::next(m.begin(), 6));
+      r = m.lower_bound(9);
+      assert(r == std::next(m.begin(), 6));
+      r = m.lower_bound(10);
+      assert(r == m.end());
     }
     {
-        typedef M::const_iterator R;
-        V ar[] =
-        {
-            V(5, 1),
-            V(5, 2),
-            V(5, 3),
-            V(7, 1),
-            V(7, 2),
-            V(7, 3),
-            V(9, 1),
-            V(9, 2),
-            V(9, 3)
-        };
-        const M m(ar, ar+sizeof(ar)/sizeof(ar[0]));
-        R r = m.lower_bound(4);
-        assert(r == m.begin());
-        r = m.lower_bound(5);
-        assert(r == m.begin());
-        r = m.lower_bound(6);
-        assert(r == std::next(m.begin(), 3));
-        r = m.lower_bound(7);
-        assert(r == std::next(m.begin(), 3));
-        r = m.lower_bound(8);
-        assert(r == std::next(m.begin(), 6));
-        r = m.lower_bound(9);
-        assert(r == std::next(m.begin(), 6));
-        r = m.lower_bound(10);
-        assert(r == m.end());
-    }
+      typedef M::const_iterator R;
+      V ar[] = {V(5, 1), V(5, 2), V(5, 3), V(7, 1), V(7, 2), V(7, 3), V(9, 1), V(9, 2), V(9, 3)};
+      const M m(ar, ar + sizeof(ar) / sizeof(ar[0]));
+      R r = m.lower_bound(4);
+      assert(r == m.begin());
+      r = m.lower_bound(5);
+      assert(r == m.begin());
+      r = m.lower_bound(6);
+      assert(r == std::next(m.begin(), 3));
+      r = m.lower_bound(7);
+      assert(r == std::next(m.begin(), 3));
+      r = m.lower_bound(8);
+      assert(r == std::next(m.begin(), 6));
+      r = m.lower_bound(9);
+      assert(r == std::next(m.begin(), 6));
+      r = m.lower_bound(10);
+      assert(r == m.end());
     }
+  }
 #endif
 #if TEST_STD_VER > 11
-    {
+  {
     typedef std::multimap<int, double, std::less<>> M;
     typedef M::iterator R;
-    V ar[] =
-    {
-        V(5, 1),
-        V(5, 2),
-        V(5, 3),
-        V(7, 1),
-        V(7, 2),
-        V(7, 3),
-        V(9, 1),
-        V(9, 2),
-        V(9, 3)
-    };
-    M m(ar, ar+sizeof(ar)/sizeof(ar[0]));
+    V ar[] = {V(5, 1), V(5, 2), V(5, 3), V(7, 1), V(7, 2), V(7, 3), V(9, 1), V(9, 2), V(9, 3)};
+    M m(ar, ar + sizeof(ar) / sizeof(ar[0]));
     R r = m.lower_bound(4);
     assert(r == m.begin());
     r = m.lower_bound(5);
@@ -198,23 +142,23 @@ int main(int, char**)
     assert(r == std::next(m.begin(), 6));
     r = m.lower_bound(C2Int(10));
     assert(r == m.end());
-    }
+  }
 
-    {
+  {
     typedef PrivateConstructor PC;
     typedef std::multimap<PC, double, std::less<>> M;
     typedef M::iterator R;
 
     M m;
-    m.insert ( std::make_pair<PC, double> ( PC::make(5), 1 ));
-    m.insert ( std::make_pair<PC, double> ( PC::make(5), 2 ));
-    m.insert ( std::make_pair<PC, double> ( PC::make(5), 3 ));
-    m.insert ( std::make_pair<PC, double> ( PC::make(7), 1 ));
-    m.insert ( std::make_pair<PC, double> ( PC::make(7), 2 ));
-    m.insert ( std::make_pair<PC, double> ( PC::make(7), 3 ));
-    m.insert ( std::make_pair<PC, double> ( PC::make(9), 1 ));
-    m.insert ( std::make_pair<PC, double> ( PC::make(9), 2 ));
-    m.insert ( std::make_pair<PC, double> ( PC::make(9), 3 ));
+    m.insert(std::make_pair<PC, double>(PC::make(5), 1));
+    m.insert(std::make_pair<PC, double>(PC::make(5), 2));
+    m.insert(std::make_pair<PC, double>(PC::make(5), 3));
+    m.insert(std::make_pair<PC, double>(PC::make(7), 1));
+    m.insert(std::make_pair<PC, double>(PC::make(7), 2));
+    m.insert(std::make_pair<PC, double>(PC::make(7), 3));
+    m.insert(std::make_pair<PC, double>(PC::make(9), 1));
+    m.insert(std::make_pair<PC, double>(PC::make(9), 2));
+    m.insert(std::make_pair<PC, double>(PC::make(9), 3));
 
     R r = m.lower_bound(4);
     assert(r == m.begin());
@@ -230,7 +174,7 @@ int main(int, char**)
     assert(r == std::next(m.begin(), 6));
     r = m.lower_bound(10);
     assert(r == m.end());
-    }
+  }
 
 #endif
 
diff --git a/libcxx/test/std/containers/associative/multimap/multimap.ops/lower_bound0.pass.cpp b/libcxx/test/std/containers/associative/multimap/multimap.ops/lower_bound0.pass.cpp
index be6987d2b8127..79f994847f131 100644
--- a/libcxx/test/std/containers/associative/multimap/multimap.ops/lower_bound0.pass.cpp
+++ b/libcxx/test/std/containers/associative/multimap/multimap.ops/lower_bound0.pass.cpp
@@ -19,25 +19,23 @@
 // equal_range shall not participate in overload resolution unless the
 // qualified-id Compare::is_transparent is valid and denotes a type
 
-
 #include <map>
 #include <cassert>
 
 #include "test_macros.h"
 #include "is_transparent.h"
 
-int main(int, char**)
-{
-    {
+int main(int, char**) {
+  {
     typedef std::multimap<int, double, transparent_less> M;
     M example;
     assert(example.lower_bound(C2Int{5}) == example.end());
-    }
-    {
+  }
+  {
     typedef std::multimap<int, double, transparent_less_not_referenceable> M;
     M example;
     assert(example.lower_bound(C2Int{5}) == example.end());
-    }
+  }
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/associative/multimap/multimap.ops/lower_bound1.compile.fail.cpp b/libcxx/test/std/containers/associative/multimap/multimap.ops/lower_bound1.compile.fail.cpp
index d8b34e012e0d2..4622cb00da94f 100644
--- a/libcxx/test/std/containers/associative/multimap/multimap.ops/lower_bound1.compile.fail.cpp
+++ b/libcxx/test/std/containers/associative/multimap/multimap.ops/lower_bound1.compile.fail.cpp
@@ -19,18 +19,16 @@
 // equal_range shall not participate in overload resolution unless the
 // qualified-id Compare::is_transparent is valid and denotes a type
 
-
 #include <map>
 #include <cassert>
 
 #include "test_macros.h"
 #include "is_transparent.h"
 
-int main(int, char**)
-{
-    {
+int main(int, char**) {
+  {
     typedef std::multimap<int, double, transparent_less_no_type> M;
 
     TEST_IGNORE_NODISCARD M().lower_bound(C2Int{5});
-    }
+  }
 }
diff --git a/libcxx/test/std/containers/associative/multimap/multimap.ops/lower_bound2.compile.fail.cpp b/libcxx/test/std/containers/associative/multimap/multimap.ops/lower_bound2.compile.fail.cpp
index 303e17aad1bc0..146e202b7e332 100644
--- a/libcxx/test/std/containers/associative/multimap/multimap.ops/lower_bound2.compile.fail.cpp
+++ b/libcxx/test/std/containers/associative/multimap/multimap.ops/lower_bound2.compile.fail.cpp
@@ -19,18 +19,16 @@
 // equal_range shall not participate in overload resolution unless the
 // qualified-id Compare::is_transparent is valid and denotes a type
 
-
 #include <map>
 #include <cassert>
 
 #include "test_macros.h"
 #include "is_transparent.h"
 
-int main(int, char**)
-{
-    {
+int main(int, char**) {
+  {
     typedef std::multimap<int, double, transparent_less_private> M;
 
     TEST_IGNORE_NODISCARD M().lower_bound(C2Int{5});
-    }
+  }
 }
diff --git a/libcxx/test/std/containers/associative/multimap/multimap.ops/lower_bound3.compile.fail.cpp b/libcxx/test/std/containers/associative/multimap/multimap.ops/lower_bound3.compile.fail.cpp
index 8eb96a99b43c4..9fe4e0e097328 100644
--- a/libcxx/test/std/containers/associative/multimap/multimap.ops/lower_bound3.compile.fail.cpp
+++ b/libcxx/test/std/containers/associative/multimap/multimap.ops/lower_bound3.compile.fail.cpp
@@ -19,18 +19,16 @@
 // equal_range shall not participate in overload resolution unless the
 // qualified-id Compare::is_transparent is valid and denotes a type
 
-
 #include <map>
 #include <cassert>
 
 #include "test_macros.h"
 #include "is_transparent.h"
 
-int main(int, char**)
-{
-    {
+int main(int, char**) {
+  {
     typedef std::multimap<int, double, transparent_less_not_a_type> M;
 
     TEST_IGNORE_NODISCARD M().lower_bound(C2Int{5});
-    }
+  }
 }
diff --git a/libcxx/test/std/containers/associative/multimap/multimap.ops/upper_bound.pass.cpp b/libcxx/test/std/containers/associative/multimap/multimap.ops/upper_bound.pass.cpp
index 35c2541667090..164cfc7c88ac9 100644
--- a/libcxx/test/std/containers/associative/multimap/multimap.ops/upper_bound.pass.cpp
+++ b/libcxx/test/std/containers/associative/multimap/multimap.ops/upper_bound.pass.cpp
@@ -21,154 +21,98 @@
 #include "private_constructor.h"
 #include "is_transparent.h"
 
-int main(int, char**)
-{
-    typedef std::pair<const int, double> V;
-    {
+int main(int, char**) {
+  typedef std::pair<const int, double> V;
+  {
     typedef std::multimap<int, double> M;
     {
-        typedef M::iterator R;
-        V ar[] =
-        {
-            V(5, 1),
-            V(5, 2),
-            V(5, 3),
-            V(7, 1),
-            V(7, 2),
-            V(7, 3),
-            V(9, 1),
-            V(9, 2),
-            V(9, 3)
-        };
-        M m(ar, ar+sizeof(ar)/sizeof(ar[0]));
-        R r = m.upper_bound(4);
-        assert(r == m.begin());
-        r = m.upper_bound(5);
-        assert(r == std::next(m.begin(), 3));
-        r = m.upper_bound(6);
-        assert(r == std::next(m.begin(), 3));
-        r = m.upper_bound(7);
-        assert(r == std::next(m.begin(), 6));
-        r = m.upper_bound(8);
-        assert(r == std::next(m.begin(), 6));
-        r = m.upper_bound(9);
-        assert(r == std::next(m.begin(), 9));
-        r = m.upper_bound(10);
-        assert(r == m.end());
+      typedef M::iterator R;
+      V ar[] = {V(5, 1), V(5, 2), V(5, 3), V(7, 1), V(7, 2), V(7, 3), V(9, 1), V(9, 2), V(9, 3)};
+      M m(ar, ar + sizeof(ar) / sizeof(ar[0]));
+      R r = m.upper_bound(4);
+      assert(r == m.begin());
+      r = m.upper_bound(5);
+      assert(r == std::next(m.begin(), 3));
+      r = m.upper_bound(6);
+      assert(r == std::next(m.begin(), 3));
+      r = m.upper_bound(7);
+      assert(r == std::next(m.begin(), 6));
+      r = m.upper_bound(8);
+      assert(r == std::next(m.begin(), 6));
+      r = m.upper_bound(9);
+      assert(r == std::next(m.begin(), 9));
+      r = m.upper_bound(10);
+      assert(r == m.end());
     }
     {
-        typedef M::const_iterator R;
-        V ar[] =
-        {
-            V(5, 1),
-            V(5, 2),
-            V(5, 3),
-            V(7, 1),
-            V(7, 2),
-            V(7, 3),
-            V(9, 1),
-            V(9, 2),
-            V(9, 3)
-        };
-        const M m(ar, ar+sizeof(ar)/sizeof(ar[0]));
-        R r = m.upper_bound(4);
-        assert(r == m.begin());
-        r = m.upper_bound(5);
-        assert(r == std::next(m.begin(), 3));
-        r = m.upper_bound(6);
-        assert(r == std::next(m.begin(), 3));
-        r = m.upper_bound(7);
-        assert(r == std::next(m.begin(), 6));
-        r = m.upper_bound(8);
-        assert(r == std::next(m.begin(), 6));
-        r = m.upper_bound(9);
-        assert(r == std::next(m.begin(), 9));
-        r = m.upper_bound(10);
-        assert(r == m.end());
-    }
+      typedef M::const_iterator R;
+      V ar[] = {V(5, 1), V(5, 2), V(5, 3), V(7, 1), V(7, 2), V(7, 3), V(9, 1), V(9, 2), V(9, 3)};
+      const M m(ar, ar + sizeof(ar) / sizeof(ar[0]));
+      R r = m.upper_bound(4);
+      assert(r == m.begin());
+      r = m.upper_bound(5);
+      assert(r == std::next(m.begin(), 3));
+      r = m.upper_bound(6);
+      assert(r == std::next(m.begin(), 3));
+      r = m.upper_bound(7);
+      assert(r == std::next(m.begin(), 6));
+      r = m.upper_bound(8);
+      assert(r == std::next(m.begin(), 6));
+      r = m.upper_bound(9);
+      assert(r == std::next(m.begin(), 9));
+      r = m.upper_bound(10);
+      assert(r == m.end());
     }
+  }
 #if TEST_STD_VER >= 11
-    {
+  {
     typedef std::multimap<int, double, std::less<int>, min_allocator<std::pair<const int, double>>> M;
     {
-        typedef M::iterator R;
-        V ar[] =
-        {
-            V(5, 1),
-            V(5, 2),
-            V(5, 3),
-            V(7, 1),
-            V(7, 2),
-            V(7, 3),
-            V(9, 1),
-            V(9, 2),
-            V(9, 3)
-        };
-        M m(ar, ar+sizeof(ar)/sizeof(ar[0]));
-        R r = m.upper_bound(4);
-        assert(r == m.begin());
-        r = m.upper_bound(5);
-        assert(r == std::next(m.begin(), 3));
-        r = m.upper_bound(6);
-        assert(r == std::next(m.begin(), 3));
-        r = m.upper_bound(7);
-        assert(r == std::next(m.begin(), 6));
-        r = m.upper_bound(8);
-        assert(r == std::next(m.begin(), 6));
-        r = m.upper_bound(9);
-        assert(r == std::next(m.begin(), 9));
-        r = m.upper_bound(10);
-        assert(r == m.end());
+      typedef M::iterator R;
+      V ar[] = {V(5, 1), V(5, 2), V(5, 3), V(7, 1), V(7, 2), V(7, 3), V(9, 1), V(9, 2), V(9, 3)};
+      M m(ar, ar + sizeof(ar) / sizeof(ar[0]));
+      R r = m.upper_bound(4);
+      assert(r == m.begin());
+      r = m.upper_bound(5);
+      assert(r == std::next(m.begin(), 3));
+      r = m.upper_bound(6);
+      assert(r == std::next(m.begin(), 3));
+      r = m.upper_bound(7);
+      assert(r == std::next(m.begin(), 6));
+      r = m.upper_bound(8);
+      assert(r == std::next(m.begin(), 6));
+      r = m.upper_bound(9);
+      assert(r == std::next(m.begin(), 9));
+      r = m.upper_bound(10);
+      assert(r == m.end());
     }
     {
-        typedef M::const_iterator R;
-        V ar[] =
-        {
-            V(5, 1),
-            V(5, 2),
-            V(5, 3),
-            V(7, 1),
-            V(7, 2),
-            V(7, 3),
-            V(9, 1),
-            V(9, 2),
-            V(9, 3)
-        };
-        const M m(ar, ar+sizeof(ar)/sizeof(ar[0]));
-        R r = m.upper_bound(4);
-        assert(r == m.begin());
-        r = m.upper_bound(5);
-        assert(r == std::next(m.begin(), 3));
-        r = m.upper_bound(6);
-        assert(r == std::next(m.begin(), 3));
-        r = m.upper_bound(7);
-        assert(r == std::next(m.begin(), 6));
-        r = m.upper_bound(8);
-        assert(r == std::next(m.begin(), 6));
-        r = m.upper_bound(9);
-        assert(r == std::next(m.begin(), 9));
-        r = m.upper_bound(10);
-        assert(r == m.end());
-    }
+      typedef M::const_iterator R;
+      V ar[] = {V(5, 1), V(5, 2), V(5, 3), V(7, 1), V(7, 2), V(7, 3), V(9, 1), V(9, 2), V(9, 3)};
+      const M m(ar, ar + sizeof(ar) / sizeof(ar[0]));
+      R r = m.upper_bound(4);
+      assert(r == m.begin());
+      r = m.upper_bound(5);
+      assert(r == std::next(m.begin(), 3));
+      r = m.upper_bound(6);
+      assert(r == std::next(m.begin(), 3));
+      r = m.upper_bound(7);
+      assert(r == std::next(m.begin(), 6));
+      r = m.upper_bound(8);
+      assert(r == std::next(m.begin(), 6));
+      r = m.upper_bound(9);
+      assert(r == std::next(m.begin(), 9));
+      r = m.upper_bound(10);
+      assert(r == m.end());
     }
+  }
 #endif
 #if TEST_STD_VER > 11
-    {
+  {
     typedef std::multimap<int, double, std::less<>> M;
     typedef M::iterator R;
-    V ar[] =
-    {
-        V(5, 1),
-        V(5, 2),
-        V(5, 3),
-        V(7, 1),
-        V(7, 2),
-        V(7, 3),
-        V(9, 1),
-        V(9, 2),
-        V(9, 3)
-    };
-    M m(ar, ar+sizeof(ar)/sizeof(ar[0]));
+    V ar[] = {V(5, 1), V(5, 2), V(5, 3), V(7, 1), V(7, 2), V(7, 3), V(9, 1), V(9, 2), V(9, 3)};
+    M m(ar, ar + sizeof(ar) / sizeof(ar[0]));
     R r = m.upper_bound(4);
     assert(r == m.begin());
     r = m.upper_bound(5);
@@ -197,23 +141,23 @@ int main(int, char**)
     r = m.upper_bound(C2Int(9));
     assert(r == std::next(m.begin(), 9));
     r = m.upper_bound(C2Int(10));
-    }
+  }
 
-    {
+  {
     typedef PrivateConstructor PC;
     typedef std::multimap<PC, double, std::less<>> M;
     typedef M::iterator R;
 
     M m;
-    m.insert ( std::make_pair<PC, double> ( PC::make(5), 1 ));
-    m.insert ( std::make_pair<PC, double> ( PC::make(5), 2 ));
-    m.insert ( std::make_pair<PC, double> ( PC::make(5), 3 ));
-    m.insert ( std::make_pair<PC, double> ( PC::make(7), 1 ));
-    m.insert ( std::make_pair<PC, double> ( PC::make(7), 2 ));
-    m.insert ( std::make_pair<PC, double> ( PC::make(7), 3 ));
-    m.insert ( std::make_pair<PC, double> ( PC::make(9), 1 ));
-    m.insert ( std::make_pair<PC, double> ( PC::make(9), 2 ));
-    m.insert ( std::make_pair<PC, double> ( PC::make(9), 3 ));
+    m.insert(std::make_pair<PC, double>(PC::make(5), 1));
+    m.insert(std::make_pair<PC, double>(PC::make(5), 2));
+    m.insert(std::make_pair<PC, double>(PC::make(5), 3));
+    m.insert(std::make_pair<PC, double>(PC::make(7), 1));
+    m.insert(std::make_pair<PC, double>(PC::make(7), 2));
+    m.insert(std::make_pair<PC, double>(PC::make(7), 3));
+    m.insert(std::make_pair<PC, double>(PC::make(9), 1));
+    m.insert(std::make_pair<PC, double>(PC::make(9), 2));
+    m.insert(std::make_pair<PC, double>(PC::make(9), 3));
 
     R r = m.upper_bound(4);
     assert(r == m.begin());
@@ -229,7 +173,7 @@ int main(int, char**)
     assert(r == std::next(m.begin(), 9));
     r = m.upper_bound(10);
     assert(r == m.end());
-    }
+  }
 
 #endif
 
diff --git a/libcxx/test/std/containers/associative/multimap/multimap.ops/upper_bound0.pass.cpp b/libcxx/test/std/containers/associative/multimap/multimap.ops/upper_bound0.pass.cpp
index 4ffda7f5ab703..62f52416b9156 100644
--- a/libcxx/test/std/containers/associative/multimap/multimap.ops/upper_bound0.pass.cpp
+++ b/libcxx/test/std/containers/associative/multimap/multimap.ops/upper_bound0.pass.cpp
@@ -19,25 +19,23 @@
 // equal_range shall not participate in overload resolution unless the
 // qualified-id Compare::is_transparent is valid and denotes a type
 
-
 #include <map>
 #include <cassert>
 
 #include "test_macros.h"
 #include "is_transparent.h"
 
-int main(int, char**)
-{
-    {
+int main(int, char**) {
+  {
     typedef std::multimap<int, double, transparent_less> M;
     M example;
     assert(example.upper_bound(C2Int{5}) == example.end());
-    }
-    {
+  }
+  {
     typedef std::multimap<int, double, transparent_less_not_referenceable> M;
     M example;
     assert(example.upper_bound(C2Int{5}) == example.end());
-    }
+  }
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/associative/multimap/multimap.ops/upper_bound1.compile.fail.cpp b/libcxx/test/std/containers/associative/multimap/multimap.ops/upper_bound1.compile.fail.cpp
index 4d09bc5177691..d28143f7cab9d 100644
--- a/libcxx/test/std/containers/associative/multimap/multimap.ops/upper_bound1.compile.fail.cpp
+++ b/libcxx/test/std/containers/associative/multimap/multimap.ops/upper_bound1.compile.fail.cpp
@@ -19,18 +19,16 @@
 // equal_range shall not participate in overload resolution unless the
 // qualified-id Compare::is_transparent is valid and denotes a type
 
-
 #include <map>
 #include <cassert>
 
 #include "test_macros.h"
 #include "is_transparent.h"
 
-int main(int, char**)
-{
-    {
+int main(int, char**) {
+  {
     typedef std::multimap<int, double, transparent_less_no_type> M;
 
     TEST_IGNORE_NODISCARD M().upper_bound(C2Int{5});
-    }
+  }
 }
diff --git a/libcxx/test/std/containers/associative/multimap/multimap.ops/upper_bound2.compile.fail.cpp b/libcxx/test/std/containers/associative/multimap/multimap.ops/upper_bound2.compile.fail.cpp
index 6494ae3789fe4..0f9897996bcca 100644
--- a/libcxx/test/std/containers/associative/multimap/multimap.ops/upper_bound2.compile.fail.cpp
+++ b/libcxx/test/std/containers/associative/multimap/multimap.ops/upper_bound2.compile.fail.cpp
@@ -19,18 +19,16 @@
 // equal_range shall not participate in overload resolution unless the
 // qualified-id Compare::is_transparent is valid and denotes a type
 
-
 #include <map>
 #include <cassert>
 
 #include "test_macros.h"
 #include "is_transparent.h"
 
-int main(int, char**)
-{
-    {
+int main(int, char**) {
+  {
     typedef std::multimap<int, double, transparent_less_private> M;
 
     TEST_IGNORE_NODISCARD M().upper_bound(C2Int{5});
-    }
+  }
 }
diff --git a/libcxx/test/std/containers/associative/multimap/multimap.ops/upper_bound3.compile.fail.cpp b/libcxx/test/std/containers/associative/multimap/multimap.ops/upper_bound3.compile.fail.cpp
index 246ef0886dc35..32e8983c8ce3a 100644
--- a/libcxx/test/std/containers/associative/multimap/multimap.ops/upper_bound3.compile.fail.cpp
+++ b/libcxx/test/std/containers/associative/multimap/multimap.ops/upper_bound3.compile.fail.cpp
@@ -19,18 +19,16 @@
 // equal_range shall not participate in overload resolution unless the
 // qualified-id Compare::is_transparent is valid and denotes a type
 
-
 #include <map>
 #include <cassert>
 
 #include "test_macros.h"
 #include "is_transparent.h"
 
-int main(int, char**)
-{
-    {
+int main(int, char**) {
+  {
     typedef std::multimap<int, double, transparent_less_not_a_type> M;
 
     TEST_IGNORE_NODISCARD M().upper_bound(C2Int{5});
-    }
+  }
 }
diff --git a/libcxx/test/std/containers/associative/multimap/multimap.special/member_swap.pass.cpp b/libcxx/test/std/containers/associative/multimap/multimap.special/member_swap.pass.cpp
index dfe2831890569..e3ebe7980dd52 100644
--- a/libcxx/test/std/containers/associative/multimap/multimap.special/member_swap.pass.cpp
+++ b/libcxx/test/std/containers/associative/multimap/multimap.special/member_swap.pass.cpp
@@ -18,160 +18,95 @@
 #include "test_macros.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    typedef std::pair<const int, double> V;
-    {
+int main(int, char**) {
+  typedef std::pair<const int, double> V;
+  {
     typedef std::multimap<int, double> M;
     {
-        M m1;
-        M m2;
-        M m1_save = m1;
-        M m2_save = m2;
-        m1.swap(m2);
-        assert(m1 == m2_save);
-        assert(m2 == m1_save);
+      M m1;
+      M m2;
+      M m1_save = m1;
+      M m2_save = m2;
+      m1.swap(m2);
+      assert(m1 == m2_save);
+      assert(m2 == m1_save);
     }
     {
-        V ar2[] =
-        {
-            V(5, 5),
-            V(6, 6),
-            V(7, 7),
-            V(8, 8),
-            V(9, 9),
-            V(10, 10),
-            V(11, 11),
-            V(12, 12)
-        };
-        M m1;
-        M m2(ar2, ar2+sizeof(ar2)/sizeof(ar2[0]));
-        M m1_save = m1;
-        M m2_save = m2;
-        m1.swap(m2);
-        assert(m1 == m2_save);
-        assert(m2 == m1_save);
+      V ar2[] = {V(5, 5), V(6, 6), V(7, 7), V(8, 8), V(9, 9), V(10, 10), V(11, 11), V(12, 12)};
+      M m1;
+      M m2(ar2, ar2 + sizeof(ar2) / sizeof(ar2[0]));
+      M m1_save = m1;
+      M m2_save = m2;
+      m1.swap(m2);
+      assert(m1 == m2_save);
+      assert(m2 == m1_save);
     }
     {
-        V ar1[] =
-        {
-            V(1, 1),
-            V(2, 2),
-            V(3, 3),
-            V(4, 4)
-        };
-        M m1(ar1, ar1+sizeof(ar1)/sizeof(ar1[0]));
-        M m2;
-        M m1_save = m1;
-        M m2_save = m2;
-        m1.swap(m2);
-        assert(m1 == m2_save);
-        assert(m2 == m1_save);
+      V ar1[] = {V(1, 1), V(2, 2), V(3, 3), V(4, 4)};
+      M m1(ar1, ar1 + sizeof(ar1) / sizeof(ar1[0]));
+      M m2;
+      M m1_save = m1;
+      M m2_save = m2;
+      m1.swap(m2);
+      assert(m1 == m2_save);
+      assert(m2 == m1_save);
     }
     {
-        V ar1[] =
-        {
-            V(1, 1),
-            V(2, 2),
-            V(3, 3),
-            V(4, 4)
-        };
-        V ar2[] =
-        {
-            V(5, 5),
-            V(6, 6),
-            V(7, 7),
-            V(8, 8),
-            V(9, 9),
-            V(10, 10),
-            V(11, 11),
-            V(12, 12)
-        };
-        M m1(ar1, ar1+sizeof(ar1)/sizeof(ar1[0]));
-        M m2(ar2, ar2+sizeof(ar2)/sizeof(ar2[0]));
-        M m1_save = m1;
-        M m2_save = m2;
-        m1.swap(m2);
-        assert(m1 == m2_save);
-        assert(m2 == m1_save);
-    }
+      V ar1[] = {V(1, 1), V(2, 2), V(3, 3), V(4, 4)};
+      V ar2[] = {V(5, 5), V(6, 6), V(7, 7), V(8, 8), V(9, 9), V(10, 10), V(11, 11), V(12, 12)};
+      M m1(ar1, ar1 + sizeof(ar1) / sizeof(ar1[0]));
+      M m2(ar2, ar2 + sizeof(ar2) / sizeof(ar2[0]));
+      M m1_save = m1;
+      M m2_save = m2;
+      m1.swap(m2);
+      assert(m1 == m2_save);
+      assert(m2 == m1_save);
     }
+  }
 #if TEST_STD_VER >= 11
-    {
+  {
     typedef std::multimap<int, double, std::less<int>, min_allocator<std::pair<const int, double>>> M;
     {
-        M m1;
-        M m2;
-        M m1_save = m1;
-        M m2_save = m2;
-        m1.swap(m2);
-        assert(m1 == m2_save);
-        assert(m2 == m1_save);
+      M m1;
+      M m2;
+      M m1_save = m1;
+      M m2_save = m2;
+      m1.swap(m2);
+      assert(m1 == m2_save);
+      assert(m2 == m1_save);
     }
     {
-        V ar2[] =
-        {
-            V(5, 5),
-            V(6, 6),
-            V(7, 7),
-            V(8, 8),
-            V(9, 9),
-            V(10, 10),
-            V(11, 11),
-            V(12, 12)
-        };
-        M m1;
-        M m2(ar2, ar2+sizeof(ar2)/sizeof(ar2[0]));
-        M m1_save = m1;
-        M m2_save = m2;
-        m1.swap(m2);
-        assert(m1 == m2_save);
-        assert(m2 == m1_save);
+      V ar2[] = {V(5, 5), V(6, 6), V(7, 7), V(8, 8), V(9, 9), V(10, 10), V(11, 11), V(12, 12)};
+      M m1;
+      M m2(ar2, ar2 + sizeof(ar2) / sizeof(ar2[0]));
+      M m1_save = m1;
+      M m2_save = m2;
+      m1.swap(m2);
+      assert(m1 == m2_save);
+      assert(m2 == m1_save);
     }
     {
-        V ar1[] =
-        {
-            V(1, 1),
-            V(2, 2),
-            V(3, 3),
-            V(4, 4)
-        };
-        M m1(ar1, ar1+sizeof(ar1)/sizeof(ar1[0]));
-        M m2;
-        M m1_save = m1;
-        M m2_save = m2;
-        m1.swap(m2);
-        assert(m1 == m2_save);
-        assert(m2 == m1_save);
+      V ar1[] = {V(1, 1), V(2, 2), V(3, 3), V(4, 4)};
+      M m1(ar1, ar1 + sizeof(ar1) / sizeof(ar1[0]));
+      M m2;
+      M m1_save = m1;
+      M m2_save = m2;
+      m1.swap(m2);
+      assert(m1 == m2_save);
+      assert(m2 == m1_save);
     }
     {
-        V ar1[] =
-        {
-            V(1, 1),
-            V(2, 2),
-            V(3, 3),
-            V(4, 4)
-        };
-        V ar2[] =
-        {
-            V(5, 5),
-            V(6, 6),
-            V(7, 7),
-            V(8, 8),
-            V(9, 9),
-            V(10, 10),
-            V(11, 11),
-            V(12, 12)
-        };
-        M m1(ar1, ar1+sizeof(ar1)/sizeof(ar1[0]));
-        M m2(ar2, ar2+sizeof(ar2)/sizeof(ar2[0]));
-        M m1_save = m1;
-        M m2_save = m2;
-        m1.swap(m2);
-        assert(m1 == m2_save);
-        assert(m2 == m1_save);
-    }
+      V ar1[] = {V(1, 1), V(2, 2), V(3, 3), V(4, 4)};
+      V ar2[] = {V(5, 5), V(6, 6), V(7, 7), V(8, 8), V(9, 9), V(10, 10), V(11, 11), V(12, 12)};
+      M m1(ar1, ar1 + sizeof(ar1) / sizeof(ar1[0]));
+      M m2(ar2, ar2 + sizeof(ar2) / sizeof(ar2[0]));
+      M m1_save = m1;
+      M m2_save = m2;
+      m1.swap(m2);
+      assert(m1 == m2_save);
+      assert(m2 == m1_save);
     }
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/associative/multimap/multimap.special/non_member_swap.pass.cpp b/libcxx/test/std/containers/associative/multimap/multimap.special/non_member_swap.pass.cpp
index 82a838d54099a..7e771c9a034d9 100644
--- a/libcxx/test/std/containers/associative/multimap/multimap.special/non_member_swap.pass.cpp
+++ b/libcxx/test/std/containers/associative/multimap/multimap.special/non_member_swap.pass.cpp
@@ -21,262 +21,149 @@
 #include "../../../test_compare.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    typedef std::pair<const int, double> V;
-    {
+int main(int, char**) {
+  typedef std::pair<const int, double> V;
+  {
     typedef std::multimap<int, double> M;
     {
-        M m1;
-        M m2;
-        M m1_save = m1;
-        M m2_save = m2;
-        swap(m1, m2);
-        assert(m1 == m2_save);
-        assert(m2 == m1_save);
-    }
-    {
-        V ar2[] =
-        {
-            V(5, 5),
-            V(6, 6),
-            V(7, 7),
-            V(8, 8),
-            V(9, 9),
-            V(10, 10),
-            V(11, 11),
-            V(12, 12)
-        };
-        M m1;
-        M m2(ar2, ar2+sizeof(ar2)/sizeof(ar2[0]));
-        M m1_save = m1;
-        M m2_save = m2;
-        swap(m1, m2);
-        assert(m1 == m2_save);
-        assert(m2 == m1_save);
-    }
-    {
-        V ar1[] =
-        {
-            V(1, 1),
-            V(2, 2),
-            V(3, 3),
-            V(4, 4)
-        };
-        M m1(ar1, ar1+sizeof(ar1)/sizeof(ar1[0]));
-        M m2;
-        M m1_save = m1;
-        M m2_save = m2;
-        swap(m1, m2);
-        assert(m1 == m2_save);
-        assert(m2 == m1_save);
+      M m1;
+      M m2;
+      M m1_save = m1;
+      M m2_save = m2;
+      swap(m1, m2);
+      assert(m1 == m2_save);
+      assert(m2 == m1_save);
     }
     {
-        V ar1[] =
-        {
-            V(1, 1),
-            V(2, 2),
-            V(3, 3),
-            V(4, 4)
-        };
-        V ar2[] =
-        {
-            V(5, 5),
-            V(6, 6),
-            V(7, 7),
-            V(8, 8),
-            V(9, 9),
-            V(10, 10),
-            V(11, 11),
-            V(12, 12)
-        };
-        M m1(ar1, ar1+sizeof(ar1)/sizeof(ar1[0]));
-        M m2(ar2, ar2+sizeof(ar2)/sizeof(ar2[0]));
-        M m1_save = m1;
-        M m2_save = m2;
-        swap(m1, m2);
-        assert(m1 == m2_save);
-        assert(m2 == m1_save);
-    }
+      V ar2[] = {V(5, 5), V(6, 6), V(7, 7), V(8, 8), V(9, 9), V(10, 10), V(11, 11), V(12, 12)};
+      M m1;
+      M m2(ar2, ar2 + sizeof(ar2) / sizeof(ar2[0]));
+      M m1_save = m1;
+      M m2_save = m2;
+      swap(m1, m2);
+      assert(m1 == m2_save);
+      assert(m2 == m1_save);
     }
     {
-        typedef test_allocator<V> A;
-        typedef test_less<int> C;
-        typedef std::multimap<int, double, C, A> M;
-        V ar1[] =
-        {
-            V(1, 1),
-            V(2, 2),
-            V(3, 3),
-            V(4, 4)
-        };
-        V ar2[] =
-        {
-            V(5, 5),
-            V(6, 6),
-            V(7, 7),
-            V(8, 8),
-            V(9, 9),
-            V(10, 10),
-            V(11, 11),
-            V(12, 12)
-        };
-        M m1(ar1, ar1+sizeof(ar1)/sizeof(ar1[0]), C(1), A(1, 1));
-        M m2(ar2, ar2+sizeof(ar2)/sizeof(ar2[0]), C(2), A(1, 2));
-        M m1_save = m1;
-        M m2_save = m2;
-        swap(m1, m2);
-        assert(m1 == m2_save);
-        assert(m2 == m1_save);
-        assert(m1.key_comp() == C(2));
-        assert(m1.get_allocator().get_id() == 1);
-        assert(m2.key_comp() == C(1));
-        assert(m2.get_allocator().get_id() == 2);
+      V ar1[] = {V(1, 1), V(2, 2), V(3, 3), V(4, 4)};
+      M m1(ar1, ar1 + sizeof(ar1) / sizeof(ar1[0]));
+      M m2;
+      M m1_save = m1;
+      M m2_save = m2;
+      swap(m1, m2);
+      assert(m1 == m2_save);
+      assert(m2 == m1_save);
     }
     {
-        typedef other_allocator<V> A;
-        typedef test_less<int> C;
-        typedef std::multimap<int, double, C, A> M;
-        V ar1[] =
-        {
-            V(1, 1),
-            V(2, 2),
-            V(3, 3),
-            V(4, 4)
-        };
-        V ar2[] =
-        {
-            V(5, 5),
-            V(6, 6),
-            V(7, 7),
-            V(8, 8),
-            V(9, 9),
-            V(10, 10),
-            V(11, 11),
-            V(12, 12)
-        };
-        M m1(ar1, ar1+sizeof(ar1)/sizeof(ar1[0]), C(1), A(1));
-        M m2(ar2, ar2+sizeof(ar2)/sizeof(ar2[0]), C(2), A(2));
-        M m1_save = m1;
-        M m2_save = m2;
-        swap(m1, m2);
-        assert(m1 == m2_save);
-        assert(m2 == m1_save);
-        assert(m1.key_comp() == C(2));
-        assert(m1.get_allocator() == A(2));
-        assert(m2.key_comp() == C(1));
-        assert(m2.get_allocator() == A(1));
+      V ar1[] = {V(1, 1), V(2, 2), V(3, 3), V(4, 4)};
+      V ar2[] = {V(5, 5), V(6, 6), V(7, 7), V(8, 8), V(9, 9), V(10, 10), V(11, 11), V(12, 12)};
+      M m1(ar1, ar1 + sizeof(ar1) / sizeof(ar1[0]));
+      M m2(ar2, ar2 + sizeof(ar2) / sizeof(ar2[0]));
+      M m1_save = m1;
+      M m2_save = m2;
+      swap(m1, m2);
+      assert(m1 == m2_save);
+      assert(m2 == m1_save);
     }
+  }
+  {
+    typedef test_allocator<V> A;
+    typedef test_less<int> C;
+    typedef std::multimap<int, double, C, A> M;
+    V ar1[] = {V(1, 1), V(2, 2), V(3, 3), V(4, 4)};
+    V ar2[] = {V(5, 5), V(6, 6), V(7, 7), V(8, 8), V(9, 9), V(10, 10), V(11, 11), V(12, 12)};
+    M m1(ar1, ar1 + sizeof(ar1) / sizeof(ar1[0]), C(1), A(1, 1));
+    M m2(ar2, ar2 + sizeof(ar2) / sizeof(ar2[0]), C(2), A(1, 2));
+    M m1_save = m1;
+    M m2_save = m2;
+    swap(m1, m2);
+    assert(m1 == m2_save);
+    assert(m2 == m1_save);
+    assert(m1.key_comp() == C(2));
+    assert(m1.get_allocator().get_id() == 1);
+    assert(m2.key_comp() == C(1));
+    assert(m2.get_allocator().get_id() == 2);
+  }
+  {
+    typedef other_allocator<V> A;
+    typedef test_less<int> C;
+    typedef std::multimap<int, double, C, A> M;
+    V ar1[] = {V(1, 1), V(2, 2), V(3, 3), V(4, 4)};
+    V ar2[] = {V(5, 5), V(6, 6), V(7, 7), V(8, 8), V(9, 9), V(10, 10), V(11, 11), V(12, 12)};
+    M m1(ar1, ar1 + sizeof(ar1) / sizeof(ar1[0]), C(1), A(1));
+    M m2(ar2, ar2 + sizeof(ar2) / sizeof(ar2[0]), C(2), A(2));
+    M m1_save = m1;
+    M m2_save = m2;
+    swap(m1, m2);
+    assert(m1 == m2_save);
+    assert(m2 == m1_save);
+    assert(m1.key_comp() == C(2));
+    assert(m1.get_allocator() == A(2));
+    assert(m2.key_comp() == C(1));
+    assert(m2.get_allocator() == A(1));
+  }
 #if TEST_STD_VER >= 11
-    {
+  {
     typedef std::multimap<int, double, std::less<int>, min_allocator<std::pair<const int, double>>> M;
     {
-        M m1;
-        M m2;
-        M m1_save = m1;
-        M m2_save = m2;
-        swap(m1, m2);
-        assert(m1 == m2_save);
-        assert(m2 == m1_save);
+      M m1;
+      M m2;
+      M m1_save = m1;
+      M m2_save = m2;
+      swap(m1, m2);
+      assert(m1 == m2_save);
+      assert(m2 == m1_save);
     }
     {
-        V ar2[] =
-        {
-            V(5, 5),
-            V(6, 6),
-            V(7, 7),
-            V(8, 8),
-            V(9, 9),
-            V(10, 10),
-            V(11, 11),
-            V(12, 12)
-        };
-        M m1;
-        M m2(ar2, ar2+sizeof(ar2)/sizeof(ar2[0]));
-        M m1_save = m1;
-        M m2_save = m2;
-        swap(m1, m2);
-        assert(m1 == m2_save);
-        assert(m2 == m1_save);
+      V ar2[] = {V(5, 5), V(6, 6), V(7, 7), V(8, 8), V(9, 9), V(10, 10), V(11, 11), V(12, 12)};
+      M m1;
+      M m2(ar2, ar2 + sizeof(ar2) / sizeof(ar2[0]));
+      M m1_save = m1;
+      M m2_save = m2;
+      swap(m1, m2);
+      assert(m1 == m2_save);
+      assert(m2 == m1_save);
     }
     {
-        V ar1[] =
-        {
-            V(1, 1),
-            V(2, 2),
-            V(3, 3),
-            V(4, 4)
-        };
-        M m1(ar1, ar1+sizeof(ar1)/sizeof(ar1[0]));
-        M m2;
-        M m1_save = m1;
-        M m2_save = m2;
-        swap(m1, m2);
-        assert(m1 == m2_save);
-        assert(m2 == m1_save);
-    }
-    {
-        V ar1[] =
-        {
-            V(1, 1),
-            V(2, 2),
-            V(3, 3),
-            V(4, 4)
-        };
-        V ar2[] =
-        {
-            V(5, 5),
-            V(6, 6),
-            V(7, 7),
-            V(8, 8),
-            V(9, 9),
-            V(10, 10),
-            V(11, 11),
-            V(12, 12)
-        };
-        M m1(ar1, ar1+sizeof(ar1)/sizeof(ar1[0]));
-        M m2(ar2, ar2+sizeof(ar2)/sizeof(ar2[0]));
-        M m1_save = m1;
-        M m2_save = m2;
-        swap(m1, m2);
-        assert(m1 == m2_save);
-        assert(m2 == m1_save);
-    }
+      V ar1[] = {V(1, 1), V(2, 2), V(3, 3), V(4, 4)};
+      M m1(ar1, ar1 + sizeof(ar1) / sizeof(ar1[0]));
+      M m2;
+      M m1_save = m1;
+      M m2_save = m2;
+      swap(m1, m2);
+      assert(m1 == m2_save);
+      assert(m2 == m1_save);
     }
     {
-        typedef min_allocator<V> A;
-        typedef test_less<int> C;
-        typedef std::multimap<int, double, C, A> M;
-        V ar1[] =
-        {
-            V(1, 1),
-            V(2, 2),
-            V(3, 3),
-            V(4, 4)
-        };
-        V ar2[] =
-        {
-            V(5, 5),
-            V(6, 6),
-            V(7, 7),
-            V(8, 8),
-            V(9, 9),
-            V(10, 10),
-            V(11, 11),
-            V(12, 12)
-        };
-        M m1(ar1, ar1+sizeof(ar1)/sizeof(ar1[0]), C(1), A());
-        M m2(ar2, ar2+sizeof(ar2)/sizeof(ar2[0]), C(2), A());
-        M m1_save = m1;
-        M m2_save = m2;
-        swap(m1, m2);
-        assert(m1 == m2_save);
-        assert(m2 == m1_save);
-        assert(m1.key_comp() == C(2));
-        assert(m1.get_allocator() == A());
-        assert(m2.key_comp() == C(1));
-        assert(m2.get_allocator() == A());
+      V ar1[] = {V(1, 1), V(2, 2), V(3, 3), V(4, 4)};
+      V ar2[] = {V(5, 5), V(6, 6), V(7, 7), V(8, 8), V(9, 9), V(10, 10), V(11, 11), V(12, 12)};
+      M m1(ar1, ar1 + sizeof(ar1) / sizeof(ar1[0]));
+      M m2(ar2, ar2 + sizeof(ar2) / sizeof(ar2[0]));
+      M m1_save = m1;
+      M m2_save = m2;
+      swap(m1, m2);
+      assert(m1 == m2_save);
+      assert(m2 == m1_save);
     }
+  }
+  {
+    typedef min_allocator<V> A;
+    typedef test_less<int> C;
+    typedef std::multimap<int, double, C, A> M;
+    V ar1[] = {V(1, 1), V(2, 2), V(3, 3), V(4, 4)};
+    V ar2[] = {V(5, 5), V(6, 6), V(7, 7), V(8, 8), V(9, 9), V(10, 10), V(11, 11), V(12, 12)};
+    M m1(ar1, ar1 + sizeof(ar1) / sizeof(ar1[0]), C(1), A());
+    M m2(ar2, ar2 + sizeof(ar2) / sizeof(ar2[0]), C(2), A());
+    M m1_save = m1;
+    M m2_save = m2;
+    swap(m1, m2);
+    assert(m1 == m2_save);
+    assert(m2 == m1_save);
+    assert(m1.key_comp() == C(2));
+    assert(m1.get_allocator() == A());
+    assert(m2.key_comp() == C(1));
+    assert(m2.get_allocator() == A());
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/associative/multimap/multimap.special/swap_noexcept.pass.cpp b/libcxx/test/std/containers/associative/multimap/multimap.special/swap_noexcept.pass.cpp
index 00ec83dfd621c..cc8af8bf3a30f 100644
--- a/libcxx/test/std/containers/associative/multimap/multimap.special/swap_noexcept.pass.cpp
+++ b/libcxx/test/std/containers/associative/multimap/multimap.special/swap_noexcept.pass.cpp
@@ -29,23 +29,21 @@
 #include "test_allocator.h"
 
 template <class T>
-struct some_comp
-{
-    typedef T value_type;
+struct some_comp {
+  typedef T value_type;
 
-    some_comp() {}
-    some_comp(const some_comp&) {}
-    bool operator()(const T&, const T&) const { return false; }
+  some_comp() {}
+  some_comp(const some_comp&) {}
+  bool operator()(const T&, const T&) const { return false; }
 };
 
 template <class T>
-struct some_comp2
-{
-    typedef T value_type;
+struct some_comp2 {
+  typedef T value_type;
 
-    some_comp2() {}
-    some_comp2(const some_comp2&) {}
-    bool operator()(const T&, const T&) const { return false; }
+  some_comp2() {}
+  some_comp2(const some_comp2&) {}
+  bool operator()(const T&, const T&) const { return false; }
 };
 
 #if TEST_STD_VER >= 14
@@ -54,88 +52,84 @@ void swap(some_comp2<T>&, some_comp2<T>&) noexcept {}
 #endif
 
 template <class T>
-struct some_alloc
-{
-    typedef T value_type;
+struct some_alloc {
+  typedef T value_type;
 
-    some_alloc() {}
-    some_alloc(const some_alloc&);
-    void deallocate(void*, unsigned) {}
+  some_alloc() {}
+  some_alloc(const some_alloc&);
+  void deallocate(void*, unsigned) {}
 
-    typedef std::true_type propagate_on_container_swap;
+  typedef std::true_type propagate_on_container_swap;
 };
 
 template <class T>
-struct some_alloc2
-{
-    typedef T value_type;
+struct some_alloc2 {
+  typedef T value_type;
 
-    some_alloc2() {}
-    some_alloc2(const some_alloc2&);
-    void deallocate(void*, unsigned) {}
+  some_alloc2() {}
+  some_alloc2(const some_alloc2&);
+  void deallocate(void*, unsigned) {}
 
-    typedef std::false_type propagate_on_container_swap;
-    typedef std::true_type is_always_equal;
+  typedef std::false_type propagate_on_container_swap;
+  typedef std::true_type is_always_equal;
 };
 
 template <class T>
-struct some_alloc3
-{
-    typedef T value_type;
+struct some_alloc3 {
+  typedef T value_type;
 
-    some_alloc3() {}
-    some_alloc3(const some_alloc3&);
-    void deallocate(void*, unsigned) {}
+  some_alloc3() {}
+  some_alloc3(const some_alloc3&);
+  void deallocate(void*, unsigned) {}
 
-    typedef std::false_type propagate_on_container_swap;
-    typedef std::false_type is_always_equal;
+  typedef std::false_type propagate_on_container_swap;
+  typedef std::false_type is_always_equal;
 };
 
-int main(int, char**)
-{
-    typedef std::pair<const MoveOnly, MoveOnly> V;
-    {
-        typedef std::multimap<MoveOnly, MoveOnly> C;
-        static_assert(noexcept(swap(std::declval<C&>(), std::declval<C&>())), "");
-    }
+int main(int, char**) {
+  typedef std::pair<const MoveOnly, MoveOnly> V;
+  {
+    typedef std::multimap<MoveOnly, MoveOnly> C;
+    static_assert(noexcept(swap(std::declval<C&>(), std::declval<C&>())), "");
+  }
 #if defined(_LIBCPP_VERSION)
-    {
-        typedef std::multimap<MoveOnly, MoveOnly, std::less<MoveOnly>, test_allocator<V>> C;
-        static_assert(noexcept(swap(std::declval<C&>(), std::declval<C&>())), "");
-    }
-    {
-        typedef std::multimap<MoveOnly, MoveOnly, std::less<MoveOnly>, other_allocator<V>> C;
-        static_assert(noexcept(swap(std::declval<C&>(), std::declval<C&>())), "");
-    }
+  {
+    typedef std::multimap<MoveOnly, MoveOnly, std::less<MoveOnly>, test_allocator<V>> C;
+    static_assert(noexcept(swap(std::declval<C&>(), std::declval<C&>())), "");
+  }
+  {
+    typedef std::multimap<MoveOnly, MoveOnly, std::less<MoveOnly>, other_allocator<V>> C;
+    static_assert(noexcept(swap(std::declval<C&>(), std::declval<C&>())), "");
+  }
 #endif // _LIBCPP_VERSION
-    {
-        typedef std::multimap<MoveOnly, MoveOnly, some_comp<MoveOnly>> C;
-        static_assert(!noexcept(swap(std::declval<C&>(), std::declval<C&>())), "");
-    }
+  {
+    typedef std::multimap<MoveOnly, MoveOnly, some_comp<MoveOnly>> C;
+    static_assert(!noexcept(swap(std::declval<C&>(), std::declval<C&>())), "");
+  }
 
 #if TEST_STD_VER >= 14
-    { // POCS allocator, throwable swap for comp
-    typedef std::multimap<MoveOnly, MoveOnly, some_comp <MoveOnly>, some_alloc <V>> C;
+  { // POCS allocator, throwable swap for comp
+    typedef std::multimap<MoveOnly, MoveOnly, some_comp<MoveOnly>, some_alloc<V>> C;
     static_assert(!noexcept(swap(std::declval<C&>(), std::declval<C&>())), "");
-    }
-    { // always equal allocator, throwable swap for comp
-    typedef std::multimap<MoveOnly, MoveOnly, some_comp <MoveOnly>, some_alloc2<V>> C;
+  }
+  { // always equal allocator, throwable swap for comp
+    typedef std::multimap<MoveOnly, MoveOnly, some_comp<MoveOnly>, some_alloc2<V>> C;
     static_assert(!noexcept(swap(std::declval<C&>(), std::declval<C&>())), "");
-    }
-    { // POCS allocator, nothrow swap for comp
-    typedef std::multimap<MoveOnly, MoveOnly, some_comp2<MoveOnly>, some_alloc <V>> C;
-    static_assert( noexcept(swap(std::declval<C&>(), std::declval<C&>())), "");
-    }
-    { // always equal allocator, nothrow swap for comp
+  }
+  { // POCS allocator, nothrow swap for comp
+    typedef std::multimap<MoveOnly, MoveOnly, some_comp2<MoveOnly>, some_alloc<V>> C;
+    static_assert(noexcept(swap(std::declval<C&>(), std::declval<C&>())), "");
+  }
+  { // always equal allocator, nothrow swap for comp
     typedef std::multimap<MoveOnly, MoveOnly, some_comp2<MoveOnly>, some_alloc2<V>> C;
-    static_assert( noexcept(swap(std::declval<C&>(), std::declval<C&>())), "");
-    }
-#if defined(_LIBCPP_VERSION)
-    { // NOT always equal allocator, nothrow swap for comp
+    static_assert(noexcept(swap(std::declval<C&>(), std::declval<C&>())), "");
+  }
+#  if defined(_LIBCPP_VERSION)
+  { // NOT always equal allocator, nothrow swap for comp
     typedef std::multimap<MoveOnly, MoveOnly, some_comp2<MoveOnly>, some_alloc3<V>> C;
-    static_assert( noexcept(swap(std::declval<C&>(), std::declval<C&>())), "");
-    }
-#endif // _LIBCPP_VERSION
+    static_assert(noexcept(swap(std::declval<C&>(), std::declval<C&>())), "");
+  }
+#  endif // _LIBCPP_VERSION
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/associative/multimap/multimap.value_compare/invoke.pass.cpp b/libcxx/test/std/containers/associative/multimap/multimap.value_compare/invoke.pass.cpp
index 75a1114ed5e09..67b4743097da8 100644
--- a/libcxx/test/std/containers/associative/multimap/multimap.value_compare/invoke.pass.cpp
+++ b/libcxx/test/std/containers/associative/multimap/multimap.value_compare/invoke.pass.cpp
@@ -19,29 +19,29 @@
 
 template <typename MMap>
 struct CallCompMember : MMap::value_compare {
-    CallCompMember(const typename MMap::value_compare& vc) : MMap::value_compare(vc) {}
+  CallCompMember(const typename MMap::value_compare& vc) : MMap::value_compare(vc) {}
 
-    typedef typename MMap::value_type value_type;
-    bool operator()(const value_type& value1, const value_type& value2) const {
-        return this->comp(value1.first, value2.first);
-    }
+  typedef typename MMap::value_type value_type;
+  bool operator()(const value_type& value1, const value_type& value2) const {
+    return this->comp(value1.first, value2.first);
+  }
 };
 
 int main(int, char**) {
-    typedef std::multimap<int, std::string> map_type;
+  typedef std::multimap<int, std::string> map_type;
 
-    map_type m;
-    map_type::iterator i1 = m.insert(map_type::value_type(1, "abc"));
-    map_type::iterator i2 = m.insert(map_type::value_type(2, "abc"));
+  map_type m;
+  map_type::iterator i1 = m.insert(map_type::value_type(1, "abc"));
+  map_type::iterator i2 = m.insert(map_type::value_type(2, "abc"));
 
-    const map_type::value_compare vc = m.value_comp();
-    CallCompMember<map_type> call_comp = m.value_comp();
+  const map_type::value_compare vc   = m.value_comp();
+  CallCompMember<map_type> call_comp = m.value_comp();
 
-    assert(vc(*i1, *i2));
-    assert(call_comp(*i1, *i2));
+  assert(vc(*i1, *i2));
+  assert(call_comp(*i1, *i2));
 
-    assert(!vc(*i2, *i1));
-    assert(!call_comp(*i2, *i1));
+  assert(!vc(*i2, *i1));
+  assert(!call_comp(*i2, *i1));
 
-    return 0;
+  return 0;
 }
diff --git a/libcxx/test/std/containers/associative/multimap/multimap.value_compare/types.pass.cpp b/libcxx/test/std/containers/associative/multimap/multimap.value_compare/types.pass.cpp
index 0d0c74f29f421..bd1165e61aa9b 100644
--- a/libcxx/test/std/containers/associative/multimap/multimap.value_compare/types.pass.cpp
+++ b/libcxx/test/std/containers/associative/multimap/multimap.value_compare/types.pass.cpp
@@ -18,13 +18,13 @@
 #include "test_macros.h"
 
 int main(int, char**) {
-    typedef std::multimap<int, std::string> map_type;
-    typedef map_type::value_compare value_compare;
-    typedef map_type::value_type value_type;
+  typedef std::multimap<int, std::string> map_type;
+  typedef map_type::value_compare value_compare;
+  typedef map_type::value_type value_type;
 
-    ASSERT_SAME_TYPE(value_compare::result_type, bool);
-    ASSERT_SAME_TYPE(value_compare::first_argument_type, value_type);
-    ASSERT_SAME_TYPE(value_compare::second_argument_type, value_type);
+  ASSERT_SAME_TYPE(value_compare::result_type, bool);
+  ASSERT_SAME_TYPE(value_compare::first_argument_type, value_type);
+  ASSERT_SAME_TYPE(value_compare::second_argument_type, value_type);
 
-    return 0;
+  return 0;
 }
diff --git a/libcxx/test/std/containers/associative/multimap/range_concept_conformance.compile.pass.cpp b/libcxx/test/std/containers/associative/multimap/range_concept_conformance.compile.pass.cpp
index fe382dbc93fc3..ee7385d1d0eff 100644
--- a/libcxx/test/std/containers/associative/multimap/range_concept_conformance.compile.pass.cpp
+++ b/libcxx/test/std/containers/associative/multimap/range_concept_conformance.compile.pass.cpp
@@ -17,7 +17,6 @@
 
 using range = std::multimap<int, int>;
 
-
 static_assert(std::same_as<std::ranges::iterator_t<range>, range::iterator>);
 static_assert(std::ranges::common_range<range>);
 static_assert(std::ranges::bidirectional_range<range>);
diff --git a/libcxx/test/std/containers/associative/multimap/scary.pass.cpp b/libcxx/test/std/containers/associative/multimap/scary.pass.cpp
index 7dcd4e1a09bac..2390d8fed3350 100644
--- a/libcxx/test/std/containers/associative/multimap/scary.pass.cpp
+++ b/libcxx/test/std/containers/associative/multimap/scary.pass.cpp
@@ -16,13 +16,12 @@
 
 #include "test_macros.h"
 
-int main(int, char**)
-{
-    typedef std::map<int, int> M1;
-    typedef std::multimap<int, int> M2;
-    M2::iterator i;
-    M1::iterator j = i;
-    ((void)j);
+int main(int, char**) {
+  typedef std::map<int, int> M1;
+  typedef std::multimap<int, int> M2;
+  M2::iterator i;
+  M1::iterator j = i;
+  ((void)j);
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/associative/multimap/size.pass.cpp b/libcxx/test/std/containers/associative/multimap/size.pass.cpp
index fdb8f72b5f712..77bd807db6b10 100644
--- a/libcxx/test/std/containers/associative/multimap/size.pass.cpp
+++ b/libcxx/test/std/containers/associative/multimap/size.pass.cpp
@@ -18,9 +18,8 @@
 #include "test_macros.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
+int main(int, char**) {
+  {
     typedef std::multimap<int, double> M;
     M m;
     assert(m.size() == 0);
@@ -36,9 +35,9 @@ int main(int, char**)
     assert(m.size() == 1);
     m.erase(m.begin());
     assert(m.size() == 0);
-    }
+  }
 #if TEST_STD_VER >= 11
-    {
+  {
     typedef std::multimap<int, double, std::less<int>, min_allocator<std::pair<const int, double>>> M;
     M m;
     assert(m.size() == 0);
@@ -54,7 +53,7 @@ int main(int, char**)
     assert(m.size() == 1);
     m.erase(m.begin());
     assert(m.size() == 0);
-    }
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/associative/multimap/types.pass.cpp b/libcxx/test/std/containers/associative/multimap/types.pass.cpp
index c9d2f3da6638d..2b182c36c1a7d 100644
--- a/libcxx/test/std/containers/associative/multimap/types.pass.cpp
+++ b/libcxx/test/std/containers/associative/multimap/types.pass.cpp
@@ -34,9 +34,8 @@
 #include "test_macros.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
+int main(int, char**) {
+  {
     typedef std::multimap<int, double> C;
     static_assert((std::is_same<C::key_type, int>::value), "");
     static_assert((std::is_same<C::mapped_type, double>::value), "");
@@ -49,9 +48,9 @@ int main(int, char**)
     static_assert((std::is_same<C::const_pointer, const std::pair<const int, double>*>::value), "");
     static_assert((std::is_same<C::size_type, std::size_t>::value), "");
     static_assert((std::is_same<C::difference_type, std::ptrdiff_t>::value), "");
-    }
+  }
 #if TEST_STD_VER >= 11
-    {
+  {
     typedef std::multimap<int, double, std::less<int>, min_allocator<std::pair<const int, double>>> C;
     static_assert((std::is_same<C::key_type, int>::value), "");
     static_assert((std::is_same<C::mapped_type, double>::value), "");
@@ -62,10 +61,10 @@ int main(int, char**)
     static_assert((std::is_same<C::const_reference, const std::pair<const int, double>&>::value), "");
     static_assert((std::is_same<C::pointer, min_pointer<std::pair<const int, double>>>::value), "");
     static_assert((std::is_same<C::const_pointer, min_pointer<const std::pair<const int, double>>>::value), "");
-//  min_allocator doesn't have a size_type, so one gets synthesized
+    //  min_allocator doesn't have a size_type, so one gets synthesized
     static_assert((std::is_same<C::size_type, std::make_unsigned<C::difference_type>::type>::value), "");
     static_assert((std::is_same<C::difference_type, std::ptrdiff_t>::value), "");
-    }
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/associative/multiset/allocator_mismatch.verify.cpp b/libcxx/test/std/containers/associative/multiset/allocator_mismatch.verify.cpp
index 711dbe049036d..76515aa6c6641 100644
--- a/libcxx/test/std/containers/associative/multiset/allocator_mismatch.verify.cpp
+++ b/libcxx/test/std/containers/associative/multiset/allocator_mismatch.verify.cpp
@@ -12,4 +12,4 @@
 #include <set>
 
 std::multiset<int, std::less<int>, std::allocator<long> > m;
-  // expected-error-re@*:* {{static assertion failed{{.*}}Allocator::value_type must be same type as value_type}}
+// expected-error-re@*:* {{static assertion failed{{.*}}Allocator::value_type must be same type as value_type}}
diff --git a/libcxx/test/std/containers/associative/multiset/clear.pass.cpp b/libcxx/test/std/containers/associative/multiset/clear.pass.cpp
index 93f9fef2caa0a..61a5f25eac608 100644
--- a/libcxx/test/std/containers/associative/multiset/clear.pass.cpp
+++ b/libcxx/test/std/containers/associative/multiset/clear.pass.cpp
@@ -18,49 +18,28 @@
 #include "test_macros.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef std::multiset<int> M;
-        typedef int V;
-        V ar[] =
-        {
-            1,
-            2,
-            3,
-            4,
-            5,
-            6,
-            7,
-            8
-        };
-        M m(ar, ar + sizeof(ar)/sizeof(ar[0]));
-        assert(m.size() == 8);
-        ASSERT_NOEXCEPT(m.clear());
-        m.clear();
-        assert(m.size() == 0);
-    }
+int main(int, char**) {
+  {
+    typedef std::multiset<int> M;
+    typedef int V;
+    V ar[] = {1, 2, 3, 4, 5, 6, 7, 8};
+    M m(ar, ar + sizeof(ar) / sizeof(ar[0]));
+    assert(m.size() == 8);
+    ASSERT_NOEXCEPT(m.clear());
+    m.clear();
+    assert(m.size() == 0);
+  }
 #if TEST_STD_VER >= 11
-    {
-        typedef std::multiset<int, std::less<int>, min_allocator<int>> M;
-        typedef int V;
-        V ar[] =
-        {
-            1,
-            2,
-            3,
-            4,
-            5,
-            6,
-            7,
-            8
-        };
-        M m(ar, ar + sizeof(ar)/sizeof(ar[0]));
-        assert(m.size() == 8);
-        ASSERT_NOEXCEPT(m.clear());
-        m.clear();
-        assert(m.size() == 0);
-    }
+  {
+    typedef std::multiset<int, std::less<int>, min_allocator<int>> M;
+    typedef int V;
+    V ar[] = {1, 2, 3, 4, 5, 6, 7, 8};
+    M m(ar, ar + sizeof(ar) / sizeof(ar[0]));
+    assert(m.size() == 8);
+    ASSERT_NOEXCEPT(m.clear());
+    m.clear();
+    assert(m.size() == 0);
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/associative/multiset/count.pass.cpp b/libcxx/test/std/containers/associative/multiset/count.pass.cpp
index 21575fc12bacd..6ad96b232083c 100644
--- a/libcxx/test/std/containers/associative/multiset/count.pass.cpp
+++ b/libcxx/test/std/containers/associative/multiset/count.pass.cpp
@@ -19,96 +19,62 @@
 #include "min_allocator.h"
 #include "private_constructor.h"
 
-int main(int, char**)
-{
-    {
+int main(int, char**) {
+  {
     typedef int V;
     typedef std::multiset<int> M;
     {
-        typedef M::size_type R;
-        V ar[] =
-        {
-            5,
-            5,
-            5,
-            5,
-            7,
-            7,
-            7,
-            9,
-            9
-        };
-        const M m(ar, ar+sizeof(ar)/sizeof(ar[0]));
-        R r = m.count(4);
-        assert(r == 0);
-        r = m.count(5);
-        assert(r == 4);
-        r = m.count(6);
-        assert(r == 0);
-        r = m.count(7);
-        assert(r == 3);
-        r = m.count(8);
-        assert(r == 0);
-        r = m.count(9);
-        assert(r == 2);
-        r = m.count(10);
-        assert(r == 0);
-    }
+      typedef M::size_type R;
+      V ar[] = {5, 5, 5, 5, 7, 7, 7, 9, 9};
+      const M m(ar, ar + sizeof(ar) / sizeof(ar[0]));
+      R r = m.count(4);
+      assert(r == 0);
+      r = m.count(5);
+      assert(r == 4);
+      r = m.count(6);
+      assert(r == 0);
+      r = m.count(7);
+      assert(r == 3);
+      r = m.count(8);
+      assert(r == 0);
+      r = m.count(9);
+      assert(r == 2);
+      r = m.count(10);
+      assert(r == 0);
     }
+  }
 #if TEST_STD_VER >= 11
-    {
+  {
     typedef int V;
     typedef std::multiset<int, std::less<int>, min_allocator<int>> M;
     {
-        typedef M::size_type R;
-        V ar[] =
-        {
-            5,
-            5,
-            5,
-            5,
-            7,
-            7,
-            7,
-            9,
-            9
-        };
-        const M m(ar, ar+sizeof(ar)/sizeof(ar[0]));
-        R r = m.count(4);
-        assert(r == 0);
-        r = m.count(5);
-        assert(r == 4);
-        r = m.count(6);
-        assert(r == 0);
-        r = m.count(7);
-        assert(r == 3);
-        r = m.count(8);
-        assert(r == 0);
-        r = m.count(9);
-        assert(r == 2);
-        r = m.count(10);
-        assert(r == 0);
-    }
+      typedef M::size_type R;
+      V ar[] = {5, 5, 5, 5, 7, 7, 7, 9, 9};
+      const M m(ar, ar + sizeof(ar) / sizeof(ar[0]));
+      R r = m.count(4);
+      assert(r == 0);
+      r = m.count(5);
+      assert(r == 4);
+      r = m.count(6);
+      assert(r == 0);
+      r = m.count(7);
+      assert(r == 3);
+      r = m.count(8);
+      assert(r == 0);
+      r = m.count(9);
+      assert(r == 2);
+      r = m.count(10);
+      assert(r == 0);
     }
+  }
 #endif
 #if TEST_STD_VER > 11
-    {
+  {
     typedef int V;
     typedef std::multiset<int, std::less<>> M;
     typedef M::size_type R;
-    V ar[] =
-    {
-        5,
-        5,
-        5,
-        5,
-        7,
-        7,
-        7,
-        9,
-        9
-    };
-    const M m(ar, ar+sizeof(ar)/sizeof(ar[0]));
+    V ar[] = {5, 5, 5, 5, 7, 7, 7, 9, 9};
+    const M m(ar, ar + sizeof(ar) / sizeof(ar[0]));
     R r = m.count(4);
     assert(r == 0);
     r = m.count(5);
@@ -123,23 +89,23 @@ int main(int, char**)
     assert(r == 2);
     r = m.count(10);
     assert(r == 0);
-    }
+  }
 
-    {
+  {
     typedef PrivateConstructor V;
     typedef std::multiset<V, std::less<>> M;
     typedef M::size_type R;
 
     M m;
-    m.insert ( V::make ( 5 ));
-    m.insert ( V::make ( 5 ));
-    m.insert ( V::make ( 5 ));
-    m.insert ( V::make ( 5 ));
-    m.insert ( V::make ( 7 ));
-    m.insert ( V::make ( 7 ));
-    m.insert ( V::make ( 7 ));
-    m.insert ( V::make ( 9 ));
-    m.insert ( V::make ( 9 ));
+    m.insert(V::make(5));
+    m.insert(V::make(5));
+    m.insert(V::make(5));
+    m.insert(V::make(5));
+    m.insert(V::make(7));
+    m.insert(V::make(7));
+    m.insert(V::make(7));
+    m.insert(V::make(9));
+    m.insert(V::make(9));
 
     R r = m.count(4);
     assert(r == 0);
@@ -155,7 +121,7 @@ int main(int, char**)
     assert(r == 2);
     r = m.count(10);
     assert(r == 0);
-    }
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/associative/multiset/count_transparent.pass.cpp b/libcxx/test/std/containers/associative/multiset/count_transparent.pass.cpp
index 5e6add9582df2..aea63384887d4 100644
--- a/libcxx/test/std/containers/associative/multiset/count_transparent.pass.cpp
+++ b/libcxx/test/std/containers/associative/multiset/count_transparent.pass.cpp
@@ -22,18 +22,11 @@
 struct Comp {
   using is_transparent = void;
 
-  bool operator()(const std::pair<int, int> &lhs,
-                  const std::pair<int, int> &rhs) const {
-    return lhs < rhs;
-  }
-
-  bool operator()(const std::pair<int, int> &lhs, int rhs) const {
-    return lhs.first < rhs;
-  }
-
-  bool operator()(int lhs, const std::pair<int, int> &rhs) const {
-    return lhs < rhs.first;
-  }
+  bool operator()(const std::pair<int, int>& lhs, const std::pair<int, int>& rhs) const { return lhs < rhs; }
+
+  bool operator()(const std::pair<int, int>& lhs, int rhs) const { return lhs.first < rhs; }
+
+  bool operator()(int lhs, const std::pair<int, int>& rhs) const { return lhs < rhs.first; }
 };
 
 int main(int, char**) {
diff --git a/libcxx/test/std/containers/associative/multiset/emplace.pass.cpp b/libcxx/test/std/containers/associative/multiset/emplace.pass.cpp
index 62ac36650d999..bddfd8f621437 100644
--- a/libcxx/test/std/containers/associative/multiset/emplace.pass.cpp
+++ b/libcxx/test/std/containers/associative/multiset/emplace.pass.cpp
@@ -23,61 +23,60 @@
 #include "DefaultOnly.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef std::multiset<DefaultOnly> M;
-        typedef M::iterator R;
-        M m;
-        assert(DefaultOnly::count == 0);
-        R r = m.emplace();
-        assert(r == m.begin());
-        assert(m.size() == 1);
-        assert(*m.begin() == DefaultOnly());
-        assert(DefaultOnly::count == 1);
-
-        r = m.emplace();
-        assert(r == std::next(m.begin()));
-        assert(m.size() == 2);
-        assert(*m.begin() == DefaultOnly());
-        assert(DefaultOnly::count == 2);
-    }
+int main(int, char**) {
+  {
+    typedef std::multiset<DefaultOnly> M;
+    typedef M::iterator R;
+    M m;
     assert(DefaultOnly::count == 0);
-    {
-        typedef std::multiset<Emplaceable> M;
-        typedef M::iterator R;
-        M m;
-        R r = m.emplace();
-        assert(r == m.begin());
-        assert(m.size() == 1);
-        assert(*m.begin() == Emplaceable());
-        r = m.emplace(2, 3.5);
-        assert(r == std::next(m.begin()));
-        assert(m.size() == 2);
-        assert(*r == Emplaceable(2, 3.5));
-        r = m.emplace(2, 3.5);
-        assert(r == std::next(m.begin(), 2));
-        assert(m.size() == 3);
-        assert(*r == Emplaceable(2, 3.5));
-    }
-    {
-        typedef std::multiset<int> M;
-        typedef M::iterator R;
-        M m;
-        R r = m.emplace(M::value_type(2));
-        assert(r == m.begin());
-        assert(m.size() == 1);
-        assert(*r == 2);
-    }
-    {
-        typedef std::multiset<int, std::less<int>, min_allocator<int>> M;
-        typedef M::iterator R;
-        M m;
-        R r = m.emplace(M::value_type(2));
-        assert(r == m.begin());
-        assert(m.size() == 1);
-        assert(*r == 2);
-    }
+    R r = m.emplace();
+    assert(r == m.begin());
+    assert(m.size() == 1);
+    assert(*m.begin() == DefaultOnly());
+    assert(DefaultOnly::count == 1);
+
+    r = m.emplace();
+    assert(r == std::next(m.begin()));
+    assert(m.size() == 2);
+    assert(*m.begin() == DefaultOnly());
+    assert(DefaultOnly::count == 2);
+  }
+  assert(DefaultOnly::count == 0);
+  {
+    typedef std::multiset<Emplaceable> M;
+    typedef M::iterator R;
+    M m;
+    R r = m.emplace();
+    assert(r == m.begin());
+    assert(m.size() == 1);
+    assert(*m.begin() == Emplaceable());
+    r = m.emplace(2, 3.5);
+    assert(r == std::next(m.begin()));
+    assert(m.size() == 2);
+    assert(*r == Emplaceable(2, 3.5));
+    r = m.emplace(2, 3.5);
+    assert(r == std::next(m.begin(), 2));
+    assert(m.size() == 3);
+    assert(*r == Emplaceable(2, 3.5));
+  }
+  {
+    typedef std::multiset<int> M;
+    typedef M::iterator R;
+    M m;
+    R r = m.emplace(M::value_type(2));
+    assert(r == m.begin());
+    assert(m.size() == 1);
+    assert(*r == 2);
+  }
+  {
+    typedef std::multiset<int, std::less<int>, min_allocator<int>> M;
+    typedef M::iterator R;
+    M m;
+    R r = m.emplace(M::value_type(2));
+    assert(r == m.begin());
+    assert(m.size() == 1);
+    assert(*r == 2);
+  }
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/associative/multiset/emplace_hint.pass.cpp b/libcxx/test/std/containers/associative/multiset/emplace_hint.pass.cpp
index f865d1748a9a5..d8e9d1cd4d1c6 100644
--- a/libcxx/test/std/containers/associative/multiset/emplace_hint.pass.cpp
+++ b/libcxx/test/std/containers/associative/multiset/emplace_hint.pass.cpp
@@ -23,61 +23,60 @@
 #include "DefaultOnly.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef std::multiset<DefaultOnly> M;
-        typedef M::iterator R;
-        M m;
-        assert(DefaultOnly::count == 0);
-        R r = m.emplace_hint(m.cend());
-        assert(r == m.begin());
-        assert(m.size() == 1);
-        assert(*m.begin() == DefaultOnly());
-        assert(DefaultOnly::count == 1);
-
-        r = m.emplace_hint(m.cbegin());
-        assert(r == m.begin());
-        assert(m.size() == 2);
-        assert(*m.begin() == DefaultOnly());
-        assert(DefaultOnly::count == 2);
-    }
+int main(int, char**) {
+  {
+    typedef std::multiset<DefaultOnly> M;
+    typedef M::iterator R;
+    M m;
     assert(DefaultOnly::count == 0);
-    {
-        typedef std::multiset<Emplaceable> M;
-        typedef M::iterator R;
-        M m;
-        R r = m.emplace_hint(m.cend());
-        assert(r == m.begin());
-        assert(m.size() == 1);
-        assert(*m.begin() == Emplaceable());
-        r = m.emplace_hint(m.cend(), 2, 3.5);
-        assert(r == std::next(m.begin()));
-        assert(m.size() == 2);
-        assert(*r == Emplaceable(2, 3.5));
-        r = m.emplace_hint(m.cbegin(), 2, 3.5);
-        assert(r == std::next(m.begin()));
-        assert(m.size() == 3);
-        assert(*r == Emplaceable(2, 3.5));
-    }
-    {
-        typedef std::multiset<int> M;
-        typedef M::iterator R;
-        M m;
-        R r = m.emplace_hint(m.cend(), M::value_type(2));
-        assert(r == m.begin());
-        assert(m.size() == 1);
-        assert(*r == 2);
-    }
-    {
-        typedef std::multiset<int, std::less<int>, min_allocator<int>> M;
-        typedef M::iterator R;
-        M m;
-        R r = m.emplace_hint(m.cend(), M::value_type(2));
-        assert(r == m.begin());
-        assert(m.size() == 1);
-        assert(*r == 2);
-    }
+    R r = m.emplace_hint(m.cend());
+    assert(r == m.begin());
+    assert(m.size() == 1);
+    assert(*m.begin() == DefaultOnly());
+    assert(DefaultOnly::count == 1);
+
+    r = m.emplace_hint(m.cbegin());
+    assert(r == m.begin());
+    assert(m.size() == 2);
+    assert(*m.begin() == DefaultOnly());
+    assert(DefaultOnly::count == 2);
+  }
+  assert(DefaultOnly::count == 0);
+  {
+    typedef std::multiset<Emplaceable> M;
+    typedef M::iterator R;
+    M m;
+    R r = m.emplace_hint(m.cend());
+    assert(r == m.begin());
+    assert(m.size() == 1);
+    assert(*m.begin() == Emplaceable());
+    r = m.emplace_hint(m.cend(), 2, 3.5);
+    assert(r == std::next(m.begin()));
+    assert(m.size() == 2);
+    assert(*r == Emplaceable(2, 3.5));
+    r = m.emplace_hint(m.cbegin(), 2, 3.5);
+    assert(r == std::next(m.begin()));
+    assert(m.size() == 3);
+    assert(*r == Emplaceable(2, 3.5));
+  }
+  {
+    typedef std::multiset<int> M;
+    typedef M::iterator R;
+    M m;
+    R r = m.emplace_hint(m.cend(), M::value_type(2));
+    assert(r == m.begin());
+    assert(m.size() == 1);
+    assert(*r == 2);
+  }
+  {
+    typedef std::multiset<int, std::less<int>, min_allocator<int>> M;
+    typedef M::iterator R;
+    M m;
+    R r = m.emplace_hint(m.cend(), M::value_type(2));
+    assert(r == m.begin());
+    assert(m.size() == 1);
+    assert(*r == 2);
+  }
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/associative/multiset/empty.pass.cpp b/libcxx/test/std/containers/associative/multiset/empty.pass.cpp
index 837c990ccc826..944f610a0c507 100644
--- a/libcxx/test/std/containers/associative/multiset/empty.pass.cpp
+++ b/libcxx/test/std/containers/associative/multiset/empty.pass.cpp
@@ -18,9 +18,8 @@
 #include "test_macros.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
+int main(int, char**) {
+  {
     typedef std::multiset<int> M;
     M m;
     assert(m.empty());
@@ -28,9 +27,9 @@ int main(int, char**)
     assert(!m.empty());
     m.clear();
     assert(m.empty());
-    }
+  }
 #if TEST_STD_VER >= 11
-    {
+  {
     typedef std::multiset<int, std::less<int>, min_allocator<int>> M;
     M m;
     assert(m.empty());
@@ -38,7 +37,7 @@ int main(int, char**)
     assert(!m.empty());
     m.clear();
     assert(m.empty());
-    }
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/associative/multiset/empty.verify.cpp b/libcxx/test/std/containers/associative/multiset/empty.verify.cpp
index 0de93052ad795..90c397011fa8b 100644
--- a/libcxx/test/std/containers/associative/multiset/empty.verify.cpp
+++ b/libcxx/test/std/containers/associative/multiset/empty.verify.cpp
@@ -17,6 +17,6 @@
 #include <set>
 
 void f() {
-    std::multiset<int> c;
-    c.empty(); // expected-warning {{ignoring return value of function declared with 'nodiscard' attribute}}
+  std::multiset<int> c;
+  c.empty(); // expected-warning {{ignoring return value of function declared with 'nodiscard' attribute}}
 }
diff --git a/libcxx/test/std/containers/associative/multiset/equal_range.pass.cpp b/libcxx/test/std/containers/associative/multiset/equal_range.pass.cpp
index 9d1e25d8ab9ee..e5c1d2b1eab98 100644
--- a/libcxx/test/std/containers/associative/multiset/equal_range.pass.cpp
+++ b/libcxx/test/std/containers/associative/multiset/equal_range.pass.cpp
@@ -20,245 +20,189 @@
 #include "min_allocator.h"
 #include "private_constructor.h"
 
-int main(int, char**)
-{
-    {
+int main(int, char**) {
+  {
     typedef int V;
     typedef std::multiset<int> M;
     {
-        typedef std::pair<M::iterator, M::iterator> R;
-        V ar[] =
-        {
-            5,
-            5,
-            5,
-            7,
-            7,
-            7,
-            9,
-            9,
-            9
-        };
-        M m(ar, ar+sizeof(ar)/sizeof(ar[0]));
-        R r = m.equal_range(4);
-        assert(r.first  == std::next(m.begin(), 0));
-        assert(r.second == std::next(m.begin(), 0));
-        r = m.equal_range(5);
-        assert(r.first  == std::next(m.begin(), 0));
-        assert(r.second == std::next(m.begin(), 3));
-        r = m.equal_range(6);
-        assert(r.first  == std::next(m.begin(), 3));
-        assert(r.second == std::next(m.begin(), 3));
-        r = m.equal_range(7);
-        assert(r.first  == std::next(m.begin(), 3));
-        assert(r.second == std::next(m.begin(), 6));
-        r = m.equal_range(8);
-        assert(r.first  == std::next(m.begin(), 6));
-        assert(r.second == std::next(m.begin(), 6));
-        r = m.equal_range(9);
-        assert(r.first  == std::next(m.begin(), 6));
-        assert(r.second == std::next(m.begin(), 9));
-        r = m.equal_range(10);
-        assert(r.first  == std::next(m.begin(), 9));
-        assert(r.second == std::next(m.begin(), 9));
+      typedef std::pair<M::iterator, M::iterator> R;
+      V ar[] = {5, 5, 5, 7, 7, 7, 9, 9, 9};
+      M m(ar, ar + sizeof(ar) / sizeof(ar[0]));
+      R r = m.equal_range(4);
+      assert(r.first == std::next(m.begin(), 0));
+      assert(r.second == std::next(m.begin(), 0));
+      r = m.equal_range(5);
+      assert(r.first == std::next(m.begin(), 0));
+      assert(r.second == std::next(m.begin(), 3));
+      r = m.equal_range(6);
+      assert(r.first == std::next(m.begin(), 3));
+      assert(r.second == std::next(m.begin(), 3));
+      r = m.equal_range(7);
+      assert(r.first == std::next(m.begin(), 3));
+      assert(r.second == std::next(m.begin(), 6));
+      r = m.equal_range(8);
+      assert(r.first == std::next(m.begin(), 6));
+      assert(r.second == std::next(m.begin(), 6));
+      r = m.equal_range(9);
+      assert(r.first == std::next(m.begin(), 6));
+      assert(r.second == std::next(m.begin(), 9));
+      r = m.equal_range(10);
+      assert(r.first == std::next(m.begin(), 9));
+      assert(r.second == std::next(m.begin(), 9));
     }
     {
-        typedef std::pair<M::const_iterator, M::const_iterator> R;
-        V ar[] =
-        {
-            5,
-            5,
-            5,
-            7,
-            7,
-            7,
-            9,
-            9,
-            9
-        };
-        const M m(ar, ar+sizeof(ar)/sizeof(ar[0]));
-        R r = m.equal_range(4);
-        assert(r.first  == std::next(m.begin(), 0));
-        assert(r.second == std::next(m.begin(), 0));
-        r = m.equal_range(5);
-        assert(r.first  == std::next(m.begin(), 0));
-        assert(r.second == std::next(m.begin(), 3));
-        r = m.equal_range(6);
-        assert(r.first  == std::next(m.begin(), 3));
-        assert(r.second == std::next(m.begin(), 3));
-        r = m.equal_range(7);
-        assert(r.first  == std::next(m.begin(), 3));
-        assert(r.second == std::next(m.begin(), 6));
-        r = m.equal_range(8);
-        assert(r.first  == std::next(m.begin(), 6));
-        assert(r.second == std::next(m.begin(), 6));
-        r = m.equal_range(9);
-        assert(r.first  == std::next(m.begin(), 6));
-        assert(r.second == std::next(m.begin(), 9));
-        r = m.equal_range(10);
-        assert(r.first  == std::next(m.begin(), 9));
-        assert(r.second == std::next(m.begin(), 9));
-    }
+      typedef std::pair<M::const_iterator, M::const_iterator> R;
+      V ar[] = {5, 5, 5, 7, 7, 7, 9, 9, 9};
+      const M m(ar, ar + sizeof(ar) / sizeof(ar[0]));
+      R r = m.equal_range(4);
+      assert(r.first == std::next(m.begin(), 0));
+      assert(r.second == std::next(m.begin(), 0));
+      r = m.equal_range(5);
+      assert(r.first == std::next(m.begin(), 0));
+      assert(r.second == std::next(m.begin(), 3));
+      r = m.equal_range(6);
+      assert(r.first == std::next(m.begin(), 3));
+      assert(r.second == std::next(m.begin(), 3));
+      r = m.equal_range(7);
+      assert(r.first == std::next(m.begin(), 3));
+      assert(r.second == std::next(m.begin(), 6));
+      r = m.equal_range(8);
+      assert(r.first == std::next(m.begin(), 6));
+      assert(r.second == std::next(m.begin(), 6));
+      r = m.equal_range(9);
+      assert(r.first == std::next(m.begin(), 6));
+      assert(r.second == std::next(m.begin(), 9));
+      r = m.equal_range(10);
+      assert(r.first == std::next(m.begin(), 9));
+      assert(r.second == std::next(m.begin(), 9));
     }
+  }
 #if TEST_STD_VER >= 11
-    {
+  {
     typedef int V;
     typedef std::multiset<int, std::less<int>, min_allocator<int>> M;
     {
-        typedef std::pair<M::iterator, M::iterator> R;
-        V ar[] =
-        {
-            5,
-            5,
-            5,
-            7,
-            7,
-            7,
-            9,
-            9,
-            9
-        };
-        M m(ar, ar+sizeof(ar)/sizeof(ar[0]));
-        R r = m.equal_range(4);
-        assert(r.first  == std::next(m.begin(), 0));
-        assert(r.second == std::next(m.begin(), 0));
-        r = m.equal_range(5);
-        assert(r.first  == std::next(m.begin(), 0));
-        assert(r.second == std::next(m.begin(), 3));
-        r = m.equal_range(6);
-        assert(r.first  == std::next(m.begin(), 3));
-        assert(r.second == std::next(m.begin(), 3));
-        r = m.equal_range(7);
-        assert(r.first  == std::next(m.begin(), 3));
-        assert(r.second == std::next(m.begin(), 6));
-        r = m.equal_range(8);
-        assert(r.first  == std::next(m.begin(), 6));
-        assert(r.second == std::next(m.begin(), 6));
-        r = m.equal_range(9);
-        assert(r.first  == std::next(m.begin(), 6));
-        assert(r.second == std::next(m.begin(), 9));
-        r = m.equal_range(10);
-        assert(r.first  == std::next(m.begin(), 9));
-        assert(r.second == std::next(m.begin(), 9));
+      typedef std::pair<M::iterator, M::iterator> R;
+      V ar[] = {5, 5, 5, 7, 7, 7, 9, 9, 9};
+      M m(ar, ar + sizeof(ar) / sizeof(ar[0]));
+      R r = m.equal_range(4);
+      assert(r.first == std::next(m.begin(), 0));
+      assert(r.second == std::next(m.begin(), 0));
+      r = m.equal_range(5);
+      assert(r.first == std::next(m.begin(), 0));
+      assert(r.second == std::next(m.begin(), 3));
+      r = m.equal_range(6);
+      assert(r.first == std::next(m.begin(), 3));
+      assert(r.second == std::next(m.begin(), 3));
+      r = m.equal_range(7);
+      assert(r.first == std::next(m.begin(), 3));
+      assert(r.second == std::next(m.begin(), 6));
+      r = m.equal_range(8);
+      assert(r.first == std::next(m.begin(), 6));
+      assert(r.second == std::next(m.begin(), 6));
+      r = m.equal_range(9);
+      assert(r.first == std::next(m.begin(), 6));
+      assert(r.second == std::next(m.begin(), 9));
+      r = m.equal_range(10);
+      assert(r.first == std::next(m.begin(), 9));
+      assert(r.second == std::next(m.begin(), 9));
     }
     {
-        typedef std::pair<M::const_iterator, M::const_iterator> R;
-        V ar[] =
-        {
-            5,
-            5,
-            5,
-            7,
-            7,
-            7,
-            9,
-            9,
-            9
-        };
-        const M m(ar, ar+sizeof(ar)/sizeof(ar[0]));
-        R r = m.equal_range(4);
-        assert(r.first  == std::next(m.begin(), 0));
-        assert(r.second == std::next(m.begin(), 0));
-        r = m.equal_range(5);
-        assert(r.first  == std::next(m.begin(), 0));
-        assert(r.second == std::next(m.begin(), 3));
-        r = m.equal_range(6);
-        assert(r.first  == std::next(m.begin(), 3));
-        assert(r.second == std::next(m.begin(), 3));
-        r = m.equal_range(7);
-        assert(r.first  == std::next(m.begin(), 3));
-        assert(r.second == std::next(m.begin(), 6));
-        r = m.equal_range(8);
-        assert(r.first  == std::next(m.begin(), 6));
-        assert(r.second == std::next(m.begin(), 6));
-        r = m.equal_range(9);
-        assert(r.first  == std::next(m.begin(), 6));
-        assert(r.second == std::next(m.begin(), 9));
-        r = m.equal_range(10);
-        assert(r.first  == std::next(m.begin(), 9));
-        assert(r.second == std::next(m.begin(), 9));
-    }
+      typedef std::pair<M::const_iterator, M::const_iterator> R;
+      V ar[] = {5, 5, 5, 7, 7, 7, 9, 9, 9};
+      const M m(ar, ar + sizeof(ar) / sizeof(ar[0]));
+      R r = m.equal_range(4);
+      assert(r.first == std::next(m.begin(), 0));
+      assert(r.second == std::next(m.begin(), 0));
+      r = m.equal_range(5);
+      assert(r.first == std::next(m.begin(), 0));
+      assert(r.second == std::next(m.begin(), 3));
+      r = m.equal_range(6);
+      assert(r.first == std::next(m.begin(), 3));
+      assert(r.second == std::next(m.begin(), 3));
+      r = m.equal_range(7);
+      assert(r.first == std::next(m.begin(), 3));
+      assert(r.second == std::next(m.begin(), 6));
+      r = m.equal_range(8);
+      assert(r.first == std::next(m.begin(), 6));
+      assert(r.second == std::next(m.begin(), 6));
+      r = m.equal_range(9);
+      assert(r.first == std::next(m.begin(), 6));
+      assert(r.second == std::next(m.begin(), 9));
+      r = m.equal_range(10);
+      assert(r.first == std::next(m.begin(), 9));
+      assert(r.second == std::next(m.begin(), 9));
     }
+  }
 #endif
 #if TEST_STD_VER > 11
-    {
+  {
     typedef int V;
     typedef std::multiset<V, std::less<>> M;
     typedef std::pair<M::iterator, M::iterator> R;
-    V ar[] =
-    {
-        5,
-        5,
-        5,
-        7,
-        7,
-        7,
-        9,
-        9,
-        9
-    };
-    M m(ar, ar+sizeof(ar)/sizeof(ar[0]));
+    V ar[] = {5, 5, 5, 7, 7, 7, 9, 9, 9};
+    M m(ar, ar + sizeof(ar) / sizeof(ar[0]));
     R r = m.equal_range(4);
-    assert(r.first  == std::next(m.begin(), 0));
+    assert(r.first == std::next(m.begin(), 0));
     assert(r.second == std::next(m.begin(), 0));
     r = m.equal_range(5);
-    assert(r.first  == std::next(m.begin(), 0));
+    assert(r.first == std::next(m.begin(), 0));
     assert(r.second == std::next(m.begin(), 3));
     r = m.equal_range(6);
-    assert(r.first  == std::next(m.begin(), 3));
+    assert(r.first == std::next(m.begin(), 3));
     assert(r.second == std::next(m.begin(), 3));
     r = m.equal_range(7);
-    assert(r.first  == std::next(m.begin(), 3));
+    assert(r.first == std::next(m.begin(), 3));
     assert(r.second == std::next(m.begin(), 6));
     r = m.equal_range(8);
-    assert(r.first  == std::next(m.begin(), 6));
+    assert(r.first == std::next(m.begin(), 6));
     assert(r.second == std::next(m.begin(), 6));
     r = m.equal_range(9);
-    assert(r.first  == std::next(m.begin(), 6));
+    assert(r.first == std::next(m.begin(), 6));
     assert(r.second == std::next(m.begin(), 9));
     r = m.equal_range(10);
-    assert(r.first  == std::next(m.begin(), 9));
+    assert(r.first == std::next(m.begin(), 9));
     assert(r.second == std::next(m.begin(), 9));
-    }
+  }
 
-    {
+  {
     typedef PrivateConstructor V;
     typedef std::multiset<V, std::less<>> M;
     typedef std::pair<M::iterator, M::iterator> R;
 
     M m;
-    m.insert ( V::make ( 5 ));
-    m.insert ( V::make ( 5 ));
-    m.insert ( V::make ( 5 ));
-    m.insert ( V::make ( 7 ));
-    m.insert ( V::make ( 7 ));
-    m.insert ( V::make ( 7 ));
-    m.insert ( V::make ( 9 ));
-    m.insert ( V::make ( 9 ));
-    m.insert ( V::make ( 9 ));
+    m.insert(V::make(5));
+    m.insert(V::make(5));
+    m.insert(V::make(5));
+    m.insert(V::make(7));
+    m.insert(V::make(7));
+    m.insert(V::make(7));
+    m.insert(V::make(9));
+    m.insert(V::make(9));
+    m.insert(V::make(9));
 
     R r = m.equal_range(4);
-    assert(r.first  == std::next(m.begin(), 0));
+    assert(r.first == std::next(m.begin(), 0));
     assert(r.second == std::next(m.begin(), 0));
     r = m.equal_range(5);
-    assert(r.first  == std::next(m.begin(), 0));
+    assert(r.first == std::next(m.begin(), 0));
     assert(r.second == std::next(m.begin(), 3));
     r = m.equal_range(6);
-    assert(r.first  == std::next(m.begin(), 3));
+    assert(r.first == std::next(m.begin(), 3));
     assert(r.second == std::next(m.begin(), 3));
     r = m.equal_range(7);
-    assert(r.first  == std::next(m.begin(), 3));
+    assert(r.first == std::next(m.begin(), 3));
     assert(r.second == std::next(m.begin(), 6));
     r = m.equal_range(8);
-    assert(r.first  == std::next(m.begin(), 6));
+    assert(r.first == std::next(m.begin(), 6));
     assert(r.second == std::next(m.begin(), 6));
     r = m.equal_range(9);
-    assert(r.first  == std::next(m.begin(), 6));
+    assert(r.first == std::next(m.begin(), 6));
     assert(r.second == std::next(m.begin(), 9));
     r = m.equal_range(10);
-    assert(r.first  == std::next(m.begin(), 9));
+    assert(r.first == std::next(m.begin(), 9));
     assert(r.second == std::next(m.begin(), 9));
-    }
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/associative/multiset/equal_range_transparent.pass.cpp b/libcxx/test/std/containers/associative/multiset/equal_range_transparent.pass.cpp
index 06b4550933794..6402488a44f22 100644
--- a/libcxx/test/std/containers/associative/multiset/equal_range_transparent.pass.cpp
+++ b/libcxx/test/std/containers/associative/multiset/equal_range_transparent.pass.cpp
@@ -26,24 +26,17 @@
 struct Comp {
   using is_transparent = void;
 
-  bool operator()(const std::pair<int, int> &lhs,
-                  const std::pair<int, int> &rhs) const {
-    return lhs < rhs;
-  }
+  bool operator()(const std::pair<int, int>& lhs, const std::pair<int, int>& rhs) const { return lhs < rhs; }
 
-  bool operator()(const std::pair<int, int> &lhs, int rhs) const {
-    return lhs.first < rhs;
-  }
+  bool operator()(const std::pair<int, int>& lhs, int rhs) const { return lhs.first < rhs; }
 
-  bool operator()(int lhs, const std::pair<int, int> &rhs) const {
-    return lhs < rhs.first;
-  }
+  bool operator()(int lhs, const std::pair<int, int>& rhs) const { return lhs < rhs.first; }
 };
 
 int main(int, char**) {
   std::multiset<std::pair<int, int>, Comp> s{{2, 1}, {1, 1}, {1, 1}, {1, 1}, {2, 2}};
 
-  auto er = s.equal_range(1);
+  auto er   = s.equal_range(1);
   long nels = 0;
 
   for (auto it = er.first; it != er.second; it++) {
diff --git a/libcxx/test/std/containers/associative/multiset/erase_iter.pass.cpp b/libcxx/test/std/containers/associative/multiset/erase_iter.pass.cpp
index a42d48b7847df..31f00feda2977 100644
--- a/libcxx/test/std/containers/associative/multiset/erase_iter.pass.cpp
+++ b/libcxx/test/std/containers/associative/multiset/erase_iter.pass.cpp
@@ -18,187 +18,165 @@
 #include "test_macros.h"
 #include "min_allocator.h"
 
-struct TemplateConstructor
-{
-    template<typename T>
-    TemplateConstructor (const T&) {}
+struct TemplateConstructor {
+  template <typename T>
+  TemplateConstructor(const T&) {}
 };
 
 bool operator<(const TemplateConstructor&, const TemplateConstructor&) { return false; }
 
-int main(int, char**)
-{
-    {
-        typedef std::multiset<int> M;
-        typedef int V;
-        typedef M::iterator I;
-        V ar[] =
-        {
-            1,
-            2,
-            3,
-            4,
-            5,
-            6,
-            7,
-            8
-        };
-        M m(ar, ar + sizeof(ar)/sizeof(ar[0]));
-        assert(m.size() == 8);
-        I i = m.erase(std::next(m.cbegin(), 3));
-        assert(m.size() == 7);
-        assert(i == std::next(m.begin(), 3));
-        assert(*std::next(m.begin(), 0) == 1);
-        assert(*std::next(m.begin(), 1) == 2);
-        assert(*std::next(m.begin(), 2) == 3);
-        assert(*std::next(m.begin(), 3) == 5);
-        assert(*std::next(m.begin(), 4) == 6);
-        assert(*std::next(m.begin(), 5) == 7);
-        assert(*std::next(m.begin(), 6) == 8);
-
-        i = m.erase(std::next(m.cbegin(), 0));
-        assert(m.size() == 6);
-        assert(i == m.begin());
-        assert(*std::next(m.begin(), 0) == 2);
-        assert(*std::next(m.begin(), 1) == 3);
-        assert(*std::next(m.begin(), 2) == 5);
-        assert(*std::next(m.begin(), 3) == 6);
-        assert(*std::next(m.begin(), 4) == 7);
-        assert(*std::next(m.begin(), 5) == 8);
-
-        i = m.erase(std::next(m.cbegin(), 5));
-        assert(m.size() == 5);
-        assert(i == m.end());
-        assert(*std::next(m.begin(), 0) == 2);
-        assert(*std::next(m.begin(), 1) == 3);
-        assert(*std::next(m.begin(), 2) == 5);
-        assert(*std::next(m.begin(), 3) == 6);
-        assert(*std::next(m.begin(), 4) == 7);
-
-        i = m.erase(std::next(m.cbegin(), 1));
-        assert(m.size() == 4);
-        assert(i == std::next(m.begin()));
-        assert(*std::next(m.begin(), 0) == 2);
-        assert(*std::next(m.begin(), 1) == 5);
-        assert(*std::next(m.begin(), 2) == 6);
-        assert(*std::next(m.begin(), 3) == 7);
-
-        i = m.erase(std::next(m.cbegin(), 2));
-        assert(m.size() == 3);
-        assert(i == std::next(m.begin(), 2));
-        assert(*std::next(m.begin(), 0) == 2);
-        assert(*std::next(m.begin(), 1) == 5);
-        assert(*std::next(m.begin(), 2) == 7);
-
-        i = m.erase(std::next(m.cbegin(), 2));
-        assert(m.size() == 2);
-        assert(i == std::next(m.begin(), 2));
-        assert(*std::next(m.begin(), 0) == 2);
-        assert(*std::next(m.begin(), 1) == 5);
-
-        i = m.erase(std::next(m.cbegin(), 0));
-        assert(m.size() == 1);
-        assert(i == std::next(m.begin(), 0));
-        assert(*std::next(m.begin(), 0) == 5);
-
-        i = m.erase(m.cbegin());
-        assert(m.size() == 0);
-        assert(i == m.begin());
-        assert(i == m.end());
-    }
+int main(int, char**) {
+  {
+    typedef std::multiset<int> M;
+    typedef int V;
+    typedef M::iterator I;
+    V ar[] = {1, 2, 3, 4, 5, 6, 7, 8};
+    M m(ar, ar + sizeof(ar) / sizeof(ar[0]));
+    assert(m.size() == 8);
+    I i = m.erase(std::next(m.cbegin(), 3));
+    assert(m.size() == 7);
+    assert(i == std::next(m.begin(), 3));
+    assert(*std::next(m.begin(), 0) == 1);
+    assert(*std::next(m.begin(), 1) == 2);
+    assert(*std::next(m.begin(), 2) == 3);
+    assert(*std::next(m.begin(), 3) == 5);
+    assert(*std::next(m.begin(), 4) == 6);
+    assert(*std::next(m.begin(), 5) == 7);
+    assert(*std::next(m.begin(), 6) == 8);
+
+    i = m.erase(std::next(m.cbegin(), 0));
+    assert(m.size() == 6);
+    assert(i == m.begin());
+    assert(*std::next(m.begin(), 0) == 2);
+    assert(*std::next(m.begin(), 1) == 3);
+    assert(*std::next(m.begin(), 2) == 5);
+    assert(*std::next(m.begin(), 3) == 6);
+    assert(*std::next(m.begin(), 4) == 7);
+    assert(*std::next(m.begin(), 5) == 8);
+
+    i = m.erase(std::next(m.cbegin(), 5));
+    assert(m.size() == 5);
+    assert(i == m.end());
+    assert(*std::next(m.begin(), 0) == 2);
+    assert(*std::next(m.begin(), 1) == 3);
+    assert(*std::next(m.begin(), 2) == 5);
+    assert(*std::next(m.begin(), 3) == 6);
+    assert(*std::next(m.begin(), 4) == 7);
+
+    i = m.erase(std::next(m.cbegin(), 1));
+    assert(m.size() == 4);
+    assert(i == std::next(m.begin()));
+    assert(*std::next(m.begin(), 0) == 2);
+    assert(*std::next(m.begin(), 1) == 5);
+    assert(*std::next(m.begin(), 2) == 6);
+    assert(*std::next(m.begin(), 3) == 7);
+
+    i = m.erase(std::next(m.cbegin(), 2));
+    assert(m.size() == 3);
+    assert(i == std::next(m.begin(), 2));
+    assert(*std::next(m.begin(), 0) == 2);
+    assert(*std::next(m.begin(), 1) == 5);
+    assert(*std::next(m.begin(), 2) == 7);
+
+    i = m.erase(std::next(m.cbegin(), 2));
+    assert(m.size() == 2);
+    assert(i == std::next(m.begin(), 2));
+    assert(*std::next(m.begin(), 0) == 2);
+    assert(*std::next(m.begin(), 1) == 5);
+
+    i = m.erase(std::next(m.cbegin(), 0));
+    assert(m.size() == 1);
+    assert(i == std::next(m.begin(), 0));
+    assert(*std::next(m.begin(), 0) == 5);
+
+    i = m.erase(m.cbegin());
+    assert(m.size() == 0);
+    assert(i == m.begin());
+    assert(i == m.end());
+  }
 #if TEST_STD_VER >= 11
-    {
-        typedef std::multiset<int, std::less<int>, min_allocator<int>> M;
-        typedef int V;
-        typedef M::iterator I;
-        V ar[] =
-        {
-            1,
-            2,
-            3,
-            4,
-            5,
-            6,
-            7,
-            8
-        };
-        M m(ar, ar + sizeof(ar)/sizeof(ar[0]));
-        assert(m.size() == 8);
-        I i = m.erase(std::next(m.cbegin(), 3));
-        assert(m.size() == 7);
-        assert(i == std::next(m.begin(), 3));
-        assert(*std::next(m.begin(), 0) == 1);
-        assert(*std::next(m.begin(), 1) == 2);
-        assert(*std::next(m.begin(), 2) == 3);
-        assert(*std::next(m.begin(), 3) == 5);
-        assert(*std::next(m.begin(), 4) == 6);
-        assert(*std::next(m.begin(), 5) == 7);
-        assert(*std::next(m.begin(), 6) == 8);
-
-        i = m.erase(std::next(m.cbegin(), 0));
-        assert(m.size() == 6);
-        assert(i == m.begin());
-        assert(*std::next(m.begin(), 0) == 2);
-        assert(*std::next(m.begin(), 1) == 3);
-        assert(*std::next(m.begin(), 2) == 5);
-        assert(*std::next(m.begin(), 3) == 6);
-        assert(*std::next(m.begin(), 4) == 7);
-        assert(*std::next(m.begin(), 5) == 8);
-
-        i = m.erase(std::next(m.cbegin(), 5));
-        assert(m.size() == 5);
-        assert(i == m.end());
-        assert(*std::next(m.begin(), 0) == 2);
-        assert(*std::next(m.begin(), 1) == 3);
-        assert(*std::next(m.begin(), 2) == 5);
-        assert(*std::next(m.begin(), 3) == 6);
-        assert(*std::next(m.begin(), 4) == 7);
-
-        i = m.erase(std::next(m.cbegin(), 1));
-        assert(m.size() == 4);
-        assert(i == std::next(m.begin()));
-        assert(*std::next(m.begin(), 0) == 2);
-        assert(*std::next(m.begin(), 1) == 5);
-        assert(*std::next(m.begin(), 2) == 6);
-        assert(*std::next(m.begin(), 3) == 7);
-
-        i = m.erase(std::next(m.cbegin(), 2));
-        assert(m.size() == 3);
-        assert(i == std::next(m.begin(), 2));
-        assert(*std::next(m.begin(), 0) == 2);
-        assert(*std::next(m.begin(), 1) == 5);
-        assert(*std::next(m.begin(), 2) == 7);
-
-        i = m.erase(std::next(m.cbegin(), 2));
-        assert(m.size() == 2);
-        assert(i == std::next(m.begin(), 2));
-        assert(*std::next(m.begin(), 0) == 2);
-        assert(*std::next(m.begin(), 1) == 5);
-
-        i = m.erase(std::next(m.cbegin(), 0));
-        assert(m.size() == 1);
-        assert(i == std::next(m.begin(), 0));
-        assert(*std::next(m.begin(), 0) == 5);
-
-        i = m.erase(m.cbegin());
-        assert(m.size() == 0);
-        assert(i == m.begin());
-        assert(i == m.end());
-    }
+  {
+    typedef std::multiset<int, std::less<int>, min_allocator<int>> M;
+    typedef int V;
+    typedef M::iterator I;
+    V ar[] = {1, 2, 3, 4, 5, 6, 7, 8};
+    M m(ar, ar + sizeof(ar) / sizeof(ar[0]));
+    assert(m.size() == 8);
+    I i = m.erase(std::next(m.cbegin(), 3));
+    assert(m.size() == 7);
+    assert(i == std::next(m.begin(), 3));
+    assert(*std::next(m.begin(), 0) == 1);
+    assert(*std::next(m.begin(), 1) == 2);
+    assert(*std::next(m.begin(), 2) == 3);
+    assert(*std::next(m.begin(), 3) == 5);
+    assert(*std::next(m.begin(), 4) == 6);
+    assert(*std::next(m.begin(), 5) == 7);
+    assert(*std::next(m.begin(), 6) == 8);
+
+    i = m.erase(std::next(m.cbegin(), 0));
+    assert(m.size() == 6);
+    assert(i == m.begin());
+    assert(*std::next(m.begin(), 0) == 2);
+    assert(*std::next(m.begin(), 1) == 3);
+    assert(*std::next(m.begin(), 2) == 5);
+    assert(*std::next(m.begin(), 3) == 6);
+    assert(*std::next(m.begin(), 4) == 7);
+    assert(*std::next(m.begin(), 5) == 8);
+
+    i = m.erase(std::next(m.cbegin(), 5));
+    assert(m.size() == 5);
+    assert(i == m.end());
+    assert(*std::next(m.begin(), 0) == 2);
+    assert(*std::next(m.begin(), 1) == 3);
+    assert(*std::next(m.begin(), 2) == 5);
+    assert(*std::next(m.begin(), 3) == 6);
+    assert(*std::next(m.begin(), 4) == 7);
+
+    i = m.erase(std::next(m.cbegin(), 1));
+    assert(m.size() == 4);
+    assert(i == std::next(m.begin()));
+    assert(*std::next(m.begin(), 0) == 2);
+    assert(*std::next(m.begin(), 1) == 5);
+    assert(*std::next(m.begin(), 2) == 6);
+    assert(*std::next(m.begin(), 3) == 7);
+
+    i = m.erase(std::next(m.cbegin(), 2));
+    assert(m.size() == 3);
+    assert(i == std::next(m.begin(), 2));
+    assert(*std::next(m.begin(), 0) == 2);
+    assert(*std::next(m.begin(), 1) == 5);
+    assert(*std::next(m.begin(), 2) == 7);
+
+    i = m.erase(std::next(m.cbegin(), 2));
+    assert(m.size() == 2);
+    assert(i == std::next(m.begin(), 2));
+    assert(*std::next(m.begin(), 0) == 2);
+    assert(*std::next(m.begin(), 1) == 5);
+
+    i = m.erase(std::next(m.cbegin(), 0));
+    assert(m.size() == 1);
+    assert(i == std::next(m.begin(), 0));
+    assert(*std::next(m.begin(), 0) == 5);
+
+    i = m.erase(m.cbegin());
+    assert(m.size() == 0);
+    assert(i == m.begin());
+    assert(i == m.end());
+  }
 #endif
 #if TEST_STD_VER >= 14
-    {
+  {
     //  This is LWG #2059
-        typedef TemplateConstructor T;
-        typedef std::multiset<T> C;
-        typedef C::iterator I;
-
-        C c;
-        T a{0};
-        I it = c.find(a);
-        if (it != c.end())
-            c.erase(it);
-    }
+    typedef TemplateConstructor T;
+    typedef std::multiset<T> C;
+    typedef C::iterator I;
+
+    C c;
+    T a{0};
+    I it = c.find(a);
+    if (it != c.end())
+      c.erase(it);
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/associative/multiset/erase_iter_iter.pass.cpp b/libcxx/test/std/containers/associative/multiset/erase_iter_iter.pass.cpp
index 54f694926f05e..a3937d13bbb4b 100644
--- a/libcxx/test/std/containers/associative/multiset/erase_iter_iter.pass.cpp
+++ b/libcxx/test/std/containers/associative/multiset/erase_iter_iter.pass.cpp
@@ -18,125 +18,104 @@
 #include "test_macros.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef std::multiset<int> M;
-        typedef int V;
-        typedef M::iterator I;
-        V ar[] =
-        {
-            1,
-            2,
-            3,
-            4,
-            5,
-            6,
-            7,
-            8
-        };
-        M m(ar, ar + sizeof(ar)/sizeof(ar[0]));
-        assert(m.size() == 8);
-        I i = m.erase(std::next(m.cbegin(), 5), std::next(m.cbegin(), 5));
-        assert(m.size() == 8);
-        assert(i == std::next(m.begin(), 5));
-        assert(*std::next(m.begin(), 0) == 1);
-        assert(*std::next(m.begin(), 1) == 2);
-        assert(*std::next(m.begin(), 2) == 3);
-        assert(*std::next(m.begin(), 3) == 4);
-        assert(*std::next(m.begin(), 4) == 5);
-        assert(*std::next(m.begin(), 5) == 6);
-        assert(*std::next(m.begin(), 6) == 7);
-        assert(*std::next(m.begin(), 7) == 8);
+int main(int, char**) {
+  {
+    typedef std::multiset<int> M;
+    typedef int V;
+    typedef M::iterator I;
+    V ar[] = {1, 2, 3, 4, 5, 6, 7, 8};
+    M m(ar, ar + sizeof(ar) / sizeof(ar[0]));
+    assert(m.size() == 8);
+    I i = m.erase(std::next(m.cbegin(), 5), std::next(m.cbegin(), 5));
+    assert(m.size() == 8);
+    assert(i == std::next(m.begin(), 5));
+    assert(*std::next(m.begin(), 0) == 1);
+    assert(*std::next(m.begin(), 1) == 2);
+    assert(*std::next(m.begin(), 2) == 3);
+    assert(*std::next(m.begin(), 3) == 4);
+    assert(*std::next(m.begin(), 4) == 5);
+    assert(*std::next(m.begin(), 5) == 6);
+    assert(*std::next(m.begin(), 6) == 7);
+    assert(*std::next(m.begin(), 7) == 8);
 
-        i = m.erase(std::next(m.cbegin(), 3), std::next(m.cbegin(), 4));
-        assert(m.size() == 7);
-        assert(i == std::next(m.begin(), 3));
-        assert(*std::next(m.begin(), 0) == 1);
-        assert(*std::next(m.begin(), 1) == 2);
-        assert(*std::next(m.begin(), 2) == 3);
-        assert(*std::next(m.begin(), 3) == 5);
-        assert(*std::next(m.begin(), 4) == 6);
-        assert(*std::next(m.begin(), 5) == 7);
-        assert(*std::next(m.begin(), 6) == 8);
+    i = m.erase(std::next(m.cbegin(), 3), std::next(m.cbegin(), 4));
+    assert(m.size() == 7);
+    assert(i == std::next(m.begin(), 3));
+    assert(*std::next(m.begin(), 0) == 1);
+    assert(*std::next(m.begin(), 1) == 2);
+    assert(*std::next(m.begin(), 2) == 3);
+    assert(*std::next(m.begin(), 3) == 5);
+    assert(*std::next(m.begin(), 4) == 6);
+    assert(*std::next(m.begin(), 5) == 7);
+    assert(*std::next(m.begin(), 6) == 8);
 
-        i = m.erase(std::next(m.cbegin(), 2), std::next(m.cbegin(), 5));
-        assert(m.size() == 4);
-        assert(i == std::next(m.begin(), 2));
-        assert(*std::next(m.begin(), 0) == 1);
-        assert(*std::next(m.begin(), 1) == 2);
-        assert(*std::next(m.begin(), 2) == 7);
-        assert(*std::next(m.begin(), 3) == 8);
+    i = m.erase(std::next(m.cbegin(), 2), std::next(m.cbegin(), 5));
+    assert(m.size() == 4);
+    assert(i == std::next(m.begin(), 2));
+    assert(*std::next(m.begin(), 0) == 1);
+    assert(*std::next(m.begin(), 1) == 2);
+    assert(*std::next(m.begin(), 2) == 7);
+    assert(*std::next(m.begin(), 3) == 8);
 
-        i = m.erase(std::next(m.cbegin(), 0), std::next(m.cbegin(), 2));
-        assert(m.size() == 2);
-        assert(i == std::next(m.begin(), 0));
-        assert(*std::next(m.begin(), 0) == 7);
-        assert(*std::next(m.begin(), 1) == 8);
+    i = m.erase(std::next(m.cbegin(), 0), std::next(m.cbegin(), 2));
+    assert(m.size() == 2);
+    assert(i == std::next(m.begin(), 0));
+    assert(*std::next(m.begin(), 0) == 7);
+    assert(*std::next(m.begin(), 1) == 8);
 
-        i = m.erase(m.cbegin(), m.cend());
-        assert(m.size() == 0);
-        assert(i == m.end());
-    }
+    i = m.erase(m.cbegin(), m.cend());
+    assert(m.size() == 0);
+    assert(i == m.end());
+  }
 #if TEST_STD_VER >= 11
-    {
-        typedef std::multiset<int, std::less<int>, min_allocator<int>> M;
-        typedef int V;
-        typedef M::iterator I;
-        V ar[] =
-        {
-            1,
-            2,
-            3,
-            4,
-            5,
-            6,
-            7,
-            8
-        };
-        M m(ar, ar + sizeof(ar)/sizeof(ar[0]));
-        assert(m.size() == 8);
-        I i = m.erase(std::next(m.cbegin(), 5), std::next(m.cbegin(), 5));
-        assert(m.size() == 8);
-        assert(i == std::next(m.begin(), 5));
-        assert(*std::next(m.begin(), 0) == 1);
-        assert(*std::next(m.begin(), 1) == 2);
-        assert(*std::next(m.begin(), 2) == 3);
-        assert(*std::next(m.begin(), 3) == 4);
-        assert(*std::next(m.begin(), 4) == 5);
-        assert(*std::next(m.begin(), 5) == 6);
-        assert(*std::next(m.begin(), 6) == 7);
-        assert(*std::next(m.begin(), 7) == 8);
+  {
+    typedef std::multiset<int, std::less<int>, min_allocator<int>> M;
+    typedef int V;
+    typedef M::iterator I;
+    V ar[] = {1, 2, 3, 4, 5, 6, 7, 8};
+    M m(ar, ar + sizeof(ar) / sizeof(ar[0]));
+    assert(m.size() == 8);
+    I i = m.erase(std::next(m.cbegin(), 5), std::next(m.cbegin(), 5));
+    assert(m.size() == 8);
+    assert(i == std::next(m.begin(), 5));
+    assert(*std::next(m.begin(), 0) == 1);
+    assert(*std::next(m.begin(), 1) == 2);
+    assert(*std::next(m.begin(), 2) == 3);
+    assert(*std::next(m.begin(), 3) == 4);
+    assert(*std::next(m.begin(), 4) == 5);
+    assert(*std::next(m.begin(), 5) == 6);
+    assert(*std::next(m.begin(), 6) == 7);
+    assert(*std::next(m.begin(), 7) == 8);
 
-        i = m.erase(std::next(m.cbegin(), 3), std::next(m.cbegin(), 4));
-        assert(m.size() == 7);
-        assert(i == std::next(m.begin(), 3));
-        assert(*std::next(m.begin(), 0) == 1);
-        assert(*std::next(m.begin(), 1) == 2);
-        assert(*std::next(m.begin(), 2) == 3);
-        assert(*std::next(m.begin(), 3) == 5);
-        assert(*std::next(m.begin(), 4) == 6);
-        assert(*std::next(m.begin(), 5) == 7);
-        assert(*std::next(m.begin(), 6) == 8);
+    i = m.erase(std::next(m.cbegin(), 3), std::next(m.cbegin(), 4));
+    assert(m.size() == 7);
+    assert(i == std::next(m.begin(), 3));
+    assert(*std::next(m.begin(), 0) == 1);
+    assert(*std::next(m.begin(), 1) == 2);
+    assert(*std::next(m.begin(), 2) == 3);
+    assert(*std::next(m.begin(), 3) == 5);
+    assert(*std::next(m.begin(), 4) == 6);
+    assert(*std::next(m.begin(), 5) == 7);
+    assert(*std::next(m.begin(), 6) == 8);
 
-        i = m.erase(std::next(m.cbegin(), 2), std::next(m.cbegin(), 5));
-        assert(m.size() == 4);
-        assert(i == std::next(m.begin(), 2));
-        assert(*std::next(m.begin(), 0) == 1);
-        assert(*std::next(m.begin(), 1) == 2);
-        assert(*std::next(m.begin(), 2) == 7);
-        assert(*std::next(m.begin(), 3) == 8);
+    i = m.erase(std::next(m.cbegin(), 2), std::next(m.cbegin(), 5));
+    assert(m.size() == 4);
+    assert(i == std::next(m.begin(), 2));
+    assert(*std::next(m.begin(), 0) == 1);
+    assert(*std::next(m.begin(), 1) == 2);
+    assert(*std::next(m.begin(), 2) == 7);
+    assert(*std::next(m.begin(), 3) == 8);
 
-        i = m.erase(std::next(m.cbegin(), 0), std::next(m.cbegin(), 2));
-        assert(m.size() == 2);
-        assert(i == std::next(m.begin(), 0));
-        assert(*std::next(m.begin(), 0) == 7);
-        assert(*std::next(m.begin(), 1) == 8);
+    i = m.erase(std::next(m.cbegin(), 0), std::next(m.cbegin(), 2));
+    assert(m.size() == 2);
+    assert(i == std::next(m.begin(), 0));
+    assert(*std::next(m.begin(), 0) == 7);
+    assert(*std::next(m.begin(), 1) == 8);
 
-        i = m.erase(m.cbegin(), m.cend());
-        assert(m.size() == 0);
-        assert(i == m.end());
-    }
+    i = m.erase(m.cbegin(), m.cend());
+    assert(m.size() == 0);
+    assert(i == m.end());
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/associative/multiset/erase_key.pass.cpp b/libcxx/test/std/containers/associative/multiset/erase_key.pass.cpp
index fa5c9d646a652..ba04a238e44b5 100644
--- a/libcxx/test/std/containers/associative/multiset/erase_key.pass.cpp
+++ b/libcxx/test/std/containers/associative/multiset/erase_key.pass.cpp
@@ -18,113 +18,90 @@
 #include "test_macros.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef std::multiset<int> M;
-        typedef int V;
-        typedef M::size_type I;
-        V ar[] =
-        {
-            3,
-            3,
-            3,
-            5,
-            5,
-            5,
-            7,
-            7,
-            7
-        };
-        M m(ar, ar + sizeof(ar)/sizeof(ar[0]));
-        assert(m.size() == 9);
-        I i = m.erase(6);
-        assert(m.size() == 9);
-        assert(i == 0);
-        assert(*std::next(m.begin(), 0) == 3);
-        assert(*std::next(m.begin(), 1) == 3);
-        assert(*std::next(m.begin(), 2) == 3);
-        assert(*std::next(m.begin(), 3) == 5);
-        assert(*std::next(m.begin(), 4) == 5);
-        assert(*std::next(m.begin(), 5) == 5);
-        assert(*std::next(m.begin(), 6) == 7);
-        assert(*std::next(m.begin(), 7) == 7);
-        assert(*std::next(m.begin(), 8) == 7);
+int main(int, char**) {
+  {
+    typedef std::multiset<int> M;
+    typedef int V;
+    typedef M::size_type I;
+    V ar[] = {3, 3, 3, 5, 5, 5, 7, 7, 7};
+    M m(ar, ar + sizeof(ar) / sizeof(ar[0]));
+    assert(m.size() == 9);
+    I i = m.erase(6);
+    assert(m.size() == 9);
+    assert(i == 0);
+    assert(*std::next(m.begin(), 0) == 3);
+    assert(*std::next(m.begin(), 1) == 3);
+    assert(*std::next(m.begin(), 2) == 3);
+    assert(*std::next(m.begin(), 3) == 5);
+    assert(*std::next(m.begin(), 4) == 5);
+    assert(*std::next(m.begin(), 5) == 5);
+    assert(*std::next(m.begin(), 6) == 7);
+    assert(*std::next(m.begin(), 7) == 7);
+    assert(*std::next(m.begin(), 8) == 7);
 
-        i = m.erase(5);
-        assert(m.size() == 6);
-        assert(i == 3);
-        assert(*std::next(m.begin(), 0) == 3);
-        assert(*std::next(m.begin(), 1) == 3);
-        assert(*std::next(m.begin(), 2) == 3);
-        assert(*std::next(m.begin(), 3) == 7);
-        assert(*std::next(m.begin(), 4) == 7);
-        assert(*std::next(m.begin(), 5) == 7);
+    i = m.erase(5);
+    assert(m.size() == 6);
+    assert(i == 3);
+    assert(*std::next(m.begin(), 0) == 3);
+    assert(*std::next(m.begin(), 1) == 3);
+    assert(*std::next(m.begin(), 2) == 3);
+    assert(*std::next(m.begin(), 3) == 7);
+    assert(*std::next(m.begin(), 4) == 7);
+    assert(*std::next(m.begin(), 5) == 7);
 
-        i = m.erase(3);
-        assert(m.size() == 3);
-        assert(i == 3);
-        assert(*std::next(m.begin(), 0) == 7);
-        assert(*std::next(m.begin(), 1) == 7);
-        assert(*std::next(m.begin(), 2) == 7);
+    i = m.erase(3);
+    assert(m.size() == 3);
+    assert(i == 3);
+    assert(*std::next(m.begin(), 0) == 7);
+    assert(*std::next(m.begin(), 1) == 7);
+    assert(*std::next(m.begin(), 2) == 7);
 
-        i = m.erase(7);
-        assert(m.size() == 0);
-        assert(i == 3);
-    }
+    i = m.erase(7);
+    assert(m.size() == 0);
+    assert(i == 3);
+  }
 #if TEST_STD_VER >= 11
-    {
-        typedef std::multiset<int, std::less<int>, min_allocator<int>> M;
-        typedef int V;
-        typedef M::size_type I;
-        V ar[] =
-        {
-            3,
-            3,
-            3,
-            5,
-            5,
-            5,
-            7,
-            7,
-            7
-        };
-        M m(ar, ar + sizeof(ar)/sizeof(ar[0]));
-        assert(m.size() == 9);
-        I i = m.erase(6);
-        assert(m.size() == 9);
-        assert(i == 0);
-        assert(*std::next(m.begin(), 0) == 3);
-        assert(*std::next(m.begin(), 1) == 3);
-        assert(*std::next(m.begin(), 2) == 3);
-        assert(*std::next(m.begin(), 3) == 5);
-        assert(*std::next(m.begin(), 4) == 5);
-        assert(*std::next(m.begin(), 5) == 5);
-        assert(*std::next(m.begin(), 6) == 7);
-        assert(*std::next(m.begin(), 7) == 7);
-        assert(*std::next(m.begin(), 8) == 7);
+  {
+    typedef std::multiset<int, std::less<int>, min_allocator<int>> M;
+    typedef int V;
+    typedef M::size_type I;
+    V ar[] = {3, 3, 3, 5, 5, 5, 7, 7, 7};
+    M m(ar, ar + sizeof(ar) / sizeof(ar[0]));
+    assert(m.size() == 9);
+    I i = m.erase(6);
+    assert(m.size() == 9);
+    assert(i == 0);
+    assert(*std::next(m.begin(), 0) == 3);
+    assert(*std::next(m.begin(), 1) == 3);
+    assert(*std::next(m.begin(), 2) == 3);
+    assert(*std::next(m.begin(), 3) == 5);
+    assert(*std::next(m.begin(), 4) == 5);
+    assert(*std::next(m.begin(), 5) == 5);
+    assert(*std::next(m.begin(), 6) == 7);
+    assert(*std::next(m.begin(), 7) == 7);
+    assert(*std::next(m.begin(), 8) == 7);
 
-        i = m.erase(5);
-        assert(m.size() == 6);
-        assert(i == 3);
-        assert(*std::next(m.begin(), 0) == 3);
-        assert(*std::next(m.begin(), 1) == 3);
-        assert(*std::next(m.begin(), 2) == 3);
-        assert(*std::next(m.begin(), 3) == 7);
-        assert(*std::next(m.begin(), 4) == 7);
-        assert(*std::next(m.begin(), 5) == 7);
+    i = m.erase(5);
+    assert(m.size() == 6);
+    assert(i == 3);
+    assert(*std::next(m.begin(), 0) == 3);
+    assert(*std::next(m.begin(), 1) == 3);
+    assert(*std::next(m.begin(), 2) == 3);
+    assert(*std::next(m.begin(), 3) == 7);
+    assert(*std::next(m.begin(), 4) == 7);
+    assert(*std::next(m.begin(), 5) == 7);
 
-        i = m.erase(3);
-        assert(m.size() == 3);
-        assert(i == 3);
-        assert(*std::next(m.begin(), 0) == 7);
-        assert(*std::next(m.begin(), 1) == 7);
-        assert(*std::next(m.begin(), 2) == 7);
+    i = m.erase(3);
+    assert(m.size() == 3);
+    assert(i == 3);
+    assert(*std::next(m.begin(), 0) == 7);
+    assert(*std::next(m.begin(), 1) == 7);
+    assert(*std::next(m.begin(), 2) == 7);
 
-        i = m.erase(7);
-        assert(m.size() == 0);
-        assert(i == 3);
-    }
+    i = m.erase(7);
+    assert(m.size() == 0);
+    assert(i == 3);
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/associative/multiset/extract_iterator.pass.cpp b/libcxx/test/std/containers/associative/multiset/extract_iterator.pass.cpp
index ac8c5e3b3d459..7f32fa82c46ab 100644
--- a/libcxx/test/std/containers/associative/multiset/extract_iterator.pass.cpp
+++ b/libcxx/test/std/containers/associative/multiset/extract_iterator.pass.cpp
@@ -20,43 +20,40 @@
 #include "Counter.h"
 
 template <class Container>
-void test(Container& c)
-{
-    std::size_t sz = c.size();
-
-    for (auto first = c.cbegin(); first != c.cend();)
-    {
-        auto key_value = *first;
-        typename Container::node_type t = c.extract(first++);
-        --sz;
-        assert(t.value() == key_value);
-        assert(t.get_allocator() == c.get_allocator());
-        assert(sz == c.size());
-    }
-
-    assert(c.size() == 0);
+void test(Container& c) {
+  std::size_t sz = c.size();
+
+  for (auto first = c.cbegin(); first != c.cend();) {
+    auto key_value                  = *first;
+    typename Container::node_type t = c.extract(first++);
+    --sz;
+    assert(t.value() == key_value);
+    assert(t.get_allocator() == c.get_allocator());
+    assert(sz == c.size());
+  }
+
+  assert(c.size() == 0);
 }
 
-int main(int, char**)
-{
-    {
-        using set_type = std::multiset<int>;
-        set_type m = {1, 2, 3, 4, 5, 6};
-        test(m);
-    }
-
-    {
-        std::multiset<Counter<int>> m = {1, 2, 3, 4, 5, 6};
-        assert(Counter_base::gConstructed == 6);
-        test(m);
-        assert(Counter_base::gConstructed == 0);
-    }
-
-    {
-        using min_alloc_set = std::multiset<int, std::less<int>, min_allocator<int>>;
-        min_alloc_set m = {1, 2, 3, 4, 5, 6};
-        test(m);
-    }
+int main(int, char**) {
+  {
+    using set_type = std::multiset<int>;
+    set_type m     = {1, 2, 3, 4, 5, 6};
+    test(m);
+  }
+
+  {
+    std::multiset<Counter<int>> m = {1, 2, 3, 4, 5, 6};
+    assert(Counter_base::gConstructed == 6);
+    test(m);
+    assert(Counter_base::gConstructed == 0);
+  }
+
+  {
+    using min_alloc_set = std::multiset<int, std::less<int>, min_allocator<int>>;
+    min_alloc_set m     = {1, 2, 3, 4, 5, 6};
+    test(m);
+  }
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/associative/multiset/extract_key.pass.cpp b/libcxx/test/std/containers/associative/multiset/extract_key.pass.cpp
index bad5d1bb3d394..24d98cfaaf31a 100644
--- a/libcxx/test/std/containers/associative/multiset/extract_key.pass.cpp
+++ b/libcxx/test/std/containers/associative/multiset/extract_key.pass.cpp
@@ -20,54 +20,50 @@
 #include "Counter.h"
 
 template <class Container, class KeyTypeIter>
-void test(Container& c, KeyTypeIter first, KeyTypeIter last)
-{
-    std::size_t sz = c.size();
-    assert((std::size_t)std::distance(first, last) == sz);
+void test(Container& c, KeyTypeIter first, KeyTypeIter last) {
+  std::size_t sz = c.size();
+  assert((std::size_t)std::distance(first, last) == sz);
 
-    for (KeyTypeIter copy = first; copy != last; ++copy)
-    {
-        typename Container::node_type t = c.extract(*copy);
-        assert(!t.empty());
-        --sz;
-        assert(t.value() == *copy);
-        assert(t.get_allocator() == c.get_allocator());
-        assert(sz == c.size());
-    }
+  for (KeyTypeIter copy = first; copy != last; ++copy) {
+    typename Container::node_type t = c.extract(*copy);
+    assert(!t.empty());
+    --sz;
+    assert(t.value() == *copy);
+    assert(t.get_allocator() == c.get_allocator());
+    assert(sz == c.size());
+  }
 
-    assert(c.size() == 0);
+  assert(c.size() == 0);
 
-    for (KeyTypeIter copy = first; copy != last; ++copy)
-    {
-        typename Container::node_type t = c.extract(*copy);
-        assert(t.empty());
-    }
+  for (KeyTypeIter copy = first; copy != last; ++copy) {
+    typename Container::node_type t = c.extract(*copy);
+    assert(t.empty());
+  }
 }
 
-int main(int, char**)
-{
-    {
-        std::multiset<int> m = {1, 2, 3, 4, 5, 6};
-        int keys[] = {1, 2, 3, 4, 5, 6};
-        test(m, std::begin(keys), std::end(keys));
-    }
+int main(int, char**) {
+  {
+    std::multiset<int> m = {1, 2, 3, 4, 5, 6};
+    int keys[]           = {1, 2, 3, 4, 5, 6};
+    test(m, std::begin(keys), std::end(keys));
+  }
 
+  {
+    std::multiset<Counter<int>> m = {1, 2, 3, 4, 5, 6};
     {
-        std::multiset<Counter<int>> m = {1, 2, 3, 4, 5, 6};
-        {
-            Counter<int> keys[] = {1, 2, 3, 4, 5, 6};
-            assert(Counter_base::gConstructed == 6+6);
-            test(m, std::begin(keys), std::end(keys));
-        }
-        assert(Counter_base::gConstructed == 0);
+      Counter<int> keys[] = {1, 2, 3, 4, 5, 6};
+      assert(Counter_base::gConstructed == 6 + 6);
+      test(m, std::begin(keys), std::end(keys));
     }
+    assert(Counter_base::gConstructed == 0);
+  }
 
-    {
-        using min_alloc_set = std::multiset<int, std::less<int>, min_allocator<int>>;
-        min_alloc_set m = {1, 2, 3, 4, 5, 6};
-        int keys[] = {1, 2, 3, 4, 5, 6};
-        test(m, std::begin(keys), std::end(keys));
-    }
+  {
+    using min_alloc_set = std::multiset<int, std::less<int>, min_allocator<int>>;
+    min_alloc_set m     = {1, 2, 3, 4, 5, 6};
+    int keys[]          = {1, 2, 3, 4, 5, 6};
+    test(m, std::begin(keys), std::end(keys));
+  }
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/associative/multiset/find.pass.cpp b/libcxx/test/std/containers/associative/multiset/find.pass.cpp
index da75b5b00f286..62e6b9dae431d 100644
--- a/libcxx/test/std/containers/associative/multiset/find.pass.cpp
+++ b/libcxx/test/std/containers/associative/multiset/find.pass.cpp
@@ -20,168 +20,117 @@
 #include "min_allocator.h"
 #include "private_constructor.h"
 
-int main(int, char**)
-{
+int main(int, char**) {
+  {
+    typedef int V;
+    typedef std::multiset<int> M;
+    {
+      typedef M::iterator R;
+      V ar[] = {5, 6, 7, 8, 9, 10, 11, 12};
+      M m(ar, ar + sizeof(ar) / sizeof(ar[0]));
+      R r = m.find(5);
+      assert(r == m.begin());
+      r = m.find(6);
+      assert(r == std::next(m.begin()));
+      r = m.find(7);
+      assert(r == std::next(m.begin(), 2));
+      r = m.find(8);
+      assert(r == std::next(m.begin(), 3));
+      r = m.find(9);
+      assert(r == std::next(m.begin(), 4));
+      r = m.find(10);
+      assert(r == std::next(m.begin(), 5));
+      r = m.find(11);
+      assert(r == std::next(m.begin(), 6));
+      r = m.find(12);
+      assert(r == std::next(m.begin(), 7));
+      r = m.find(4);
+      assert(r == std::next(m.begin(), 8));
+    }
     {
-        typedef int V;
-        typedef std::multiset<int> M;
-        {
-            typedef M::iterator R;
-            V ar[] =
-            {
-                5,
-                6,
-                7,
-                8,
-                9,
-                10,
-                11,
-                12
-            };
-            M m(ar, ar+sizeof(ar)/sizeof(ar[0]));
-            R r = m.find(5);
-            assert(r == m.begin());
-            r = m.find(6);
-            assert(r == std::next(m.begin()));
-            r = m.find(7);
-            assert(r == std::next(m.begin(), 2));
-            r = m.find(8);
-            assert(r == std::next(m.begin(), 3));
-            r = m.find(9);
-            assert(r == std::next(m.begin(), 4));
-            r = m.find(10);
-            assert(r == std::next(m.begin(), 5));
-            r = m.find(11);
-            assert(r == std::next(m.begin(), 6));
-            r = m.find(12);
-            assert(r == std::next(m.begin(), 7));
-            r = m.find(4);
-            assert(r == std::next(m.begin(), 8));
-        }
-        {
-            typedef M::const_iterator R;
-            V ar[] =
-            {
-                5,
-                6,
-                7,
-                8,
-                9,
-                10,
-                11,
-                12
-            };
-            const M m(ar, ar+sizeof(ar)/sizeof(ar[0]));
-            R r = m.find(5);
-            assert(r == m.begin());
-            r = m.find(6);
-            assert(r == std::next(m.begin()));
-            r = m.find(7);
-            assert(r == std::next(m.begin(), 2));
-            r = m.find(8);
-            assert(r == std::next(m.begin(), 3));
-            r = m.find(9);
-            assert(r == std::next(m.begin(), 4));
-            r = m.find(10);
-            assert(r == std::next(m.begin(), 5));
-            r = m.find(11);
-            assert(r == std::next(m.begin(), 6));
-            r = m.find(12);
-            assert(r == std::next(m.begin(), 7));
-            r = m.find(4);
-            assert(r == std::next(m.begin(), 8));
-        }
+      typedef M::const_iterator R;
+      V ar[] = {5, 6, 7, 8, 9, 10, 11, 12};
+      const M m(ar, ar + sizeof(ar) / sizeof(ar[0]));
+      R r = m.find(5);
+      assert(r == m.begin());
+      r = m.find(6);
+      assert(r == std::next(m.begin()));
+      r = m.find(7);
+      assert(r == std::next(m.begin(), 2));
+      r = m.find(8);
+      assert(r == std::next(m.begin(), 3));
+      r = m.find(9);
+      assert(r == std::next(m.begin(), 4));
+      r = m.find(10);
+      assert(r == std::next(m.begin(), 5));
+      r = m.find(11);
+      assert(r == std::next(m.begin(), 6));
+      r = m.find(12);
+      assert(r == std::next(m.begin(), 7));
+      r = m.find(4);
+      assert(r == std::next(m.begin(), 8));
     }
+  }
 #if TEST_STD_VER >= 11
+  {
+    typedef int V;
+    typedef std::multiset<int, std::less<int>, min_allocator<int>> M;
+    {
+      typedef M::iterator R;
+      V ar[] = {5, 6, 7, 8, 9, 10, 11, 12};
+      M m(ar, ar + sizeof(ar) / sizeof(ar[0]));
+      R r = m.find(5);
+      assert(r == m.begin());
+      r = m.find(6);
+      assert(r == std::next(m.begin()));
+      r = m.find(7);
+      assert(r == std::next(m.begin(), 2));
+      r = m.find(8);
+      assert(r == std::next(m.begin(), 3));
+      r = m.find(9);
+      assert(r == std::next(m.begin(), 4));
+      r = m.find(10);
+      assert(r == std::next(m.begin(), 5));
+      r = m.find(11);
+      assert(r == std::next(m.begin(), 6));
+      r = m.find(12);
+      assert(r == std::next(m.begin(), 7));
+      r = m.find(4);
+      assert(r == std::next(m.begin(), 8));
+    }
     {
-        typedef int V;
-        typedef std::multiset<int, std::less<int>, min_allocator<int>> M;
-        {
-            typedef M::iterator R;
-            V ar[] =
-            {
-                5,
-                6,
-                7,
-                8,
-                9,
-                10,
-                11,
-                12
-            };
-            M m(ar, ar+sizeof(ar)/sizeof(ar[0]));
-            R r = m.find(5);
-            assert(r == m.begin());
-            r = m.find(6);
-            assert(r == std::next(m.begin()));
-            r = m.find(7);
-            assert(r == std::next(m.begin(), 2));
-            r = m.find(8);
-            assert(r == std::next(m.begin(), 3));
-            r = m.find(9);
-            assert(r == std::next(m.begin(), 4));
-            r = m.find(10);
-            assert(r == std::next(m.begin(), 5));
-            r = m.find(11);
-            assert(r == std::next(m.begin(), 6));
-            r = m.find(12);
-            assert(r == std::next(m.begin(), 7));
-            r = m.find(4);
-            assert(r == std::next(m.begin(), 8));
-        }
-        {
-            typedef M::const_iterator R;
-            V ar[] =
-            {
-                5,
-                6,
-                7,
-                8,
-                9,
-                10,
-                11,
-                12
-            };
-            const M m(ar, ar+sizeof(ar)/sizeof(ar[0]));
-            R r = m.find(5);
-            assert(r == m.begin());
-            r = m.find(6);
-            assert(r == std::next(m.begin()));
-            r = m.find(7);
-            assert(r == std::next(m.begin(), 2));
-            r = m.find(8);
-            assert(r == std::next(m.begin(), 3));
-            r = m.find(9);
-            assert(r == std::next(m.begin(), 4));
-            r = m.find(10);
-            assert(r == std::next(m.begin(), 5));
-            r = m.find(11);
-            assert(r == std::next(m.begin(), 6));
-            r = m.find(12);
-            assert(r == std::next(m.begin(), 7));
-            r = m.find(4);
-            assert(r == std::next(m.begin(), 8));
-        }
+      typedef M::const_iterator R;
+      V ar[] = {5, 6, 7, 8, 9, 10, 11, 12};
+      const M m(ar, ar + sizeof(ar) / sizeof(ar[0]));
+      R r = m.find(5);
+      assert(r == m.begin());
+      r = m.find(6);
+      assert(r == std::next(m.begin()));
+      r = m.find(7);
+      assert(r == std::next(m.begin(), 2));
+      r = m.find(8);
+      assert(r == std::next(m.begin(), 3));
+      r = m.find(9);
+      assert(r == std::next(m.begin(), 4));
+      r = m.find(10);
+      assert(r == std::next(m.begin(), 5));
+      r = m.find(11);
+      assert(r == std::next(m.begin(), 6));
+      r = m.find(12);
+      assert(r == std::next(m.begin(), 7));
+      r = m.find(4);
+      assert(r == std::next(m.begin(), 8));
     }
+  }
 #endif
 #if TEST_STD_VER > 11
-    {
+  {
     typedef int V;
     typedef std::multiset<V, std::less<>> M;
     typedef M::iterator R;
 
-    V ar[] =
-    {
-        5,
-        6,
-        7,
-        8,
-        9,
-        10,
-        11,
-        12
-    };
-    M m(ar, ar+sizeof(ar)/sizeof(ar[0]));
+    V ar[] = {5, 6, 7, 8, 9, 10, 11, 12};
+    M m(ar, ar + sizeof(ar) / sizeof(ar[0]));
     R r = m.find(5);
     assert(r == m.begin());
     r = m.find(6);
@@ -200,22 +149,22 @@ int main(int, char**)
     assert(r == std::next(m.begin(), 7));
     r = m.find(4);
     assert(r == std::next(m.begin(), 8));
-    }
+  }
 
-    {
+  {
     typedef PrivateConstructor V;
     typedef std::multiset<V, std::less<>> M;
     typedef M::iterator R;
 
     M m;
-    m.insert ( V::make ( 5 ));
-    m.insert ( V::make ( 6 ));
-    m.insert ( V::make ( 7 ));
-    m.insert ( V::make ( 8 ));
-    m.insert ( V::make ( 9 ));
-    m.insert ( V::make ( 10 ));
-    m.insert ( V::make ( 11 ));
-    m.insert ( V::make ( 12 ));
+    m.insert(V::make(5));
+    m.insert(V::make(6));
+    m.insert(V::make(7));
+    m.insert(V::make(8));
+    m.insert(V::make(9));
+    m.insert(V::make(10));
+    m.insert(V::make(11));
+    m.insert(V::make(12));
 
     R r = m.find(5);
     assert(r == m.begin());
@@ -235,7 +184,7 @@ int main(int, char**)
     assert(r == std::next(m.begin(), 7));
     r = m.find(4);
     assert(r == std::next(m.begin(), 8));
-    }
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/associative/multiset/get_allocator.pass.cpp b/libcxx/test/std/containers/associative/multiset/get_allocator.pass.cpp
index 3d0d8a4dcb1fb..55bbd567d4360 100644
--- a/libcxx/test/std/containers/associative/multiset/get_allocator.pass.cpp
+++ b/libcxx/test/std/containers/associative/multiset/get_allocator.pass.cpp
@@ -19,15 +19,15 @@
 #include "test_macros.h"
 
 int main(int, char**) {
-    {
-        std::allocator<int> alloc;
-        const std::multiset<int> s(alloc);
-        assert(s.get_allocator() == alloc);
-    }
-    {
-        other_allocator<int> alloc(1);
-        const std::multiset<int, std::less<int>, other_allocator<int> > s(alloc);
-        assert(s.get_allocator() == alloc);
-    }
-    return 0;
+  {
+    std::allocator<int> alloc;
+    const std::multiset<int> s(alloc);
+    assert(s.get_allocator() == alloc);
+  }
+  {
+    other_allocator<int> alloc(1);
+    const std::multiset<int, std::less<int>, other_allocator<int> > s(alloc);
+    assert(s.get_allocator() == alloc);
+  }
+  return 0;
 }
diff --git a/libcxx/test/std/containers/associative/multiset/incomplete_type.pass.cpp b/libcxx/test/std/containers/associative/multiset/incomplete_type.pass.cpp
index 4f2970d5840fb..31cb96ec33e76 100644
--- a/libcxx/test/std/containers/associative/multiset/incomplete_type.pass.cpp
+++ b/libcxx/test/std/containers/associative/multiset/incomplete_type.pass.cpp
@@ -16,17 +16,17 @@
 #include "test_macros.h"
 
 struct A {
-    typedef std::multiset<A> Set;
-    int data;
-    Set m;
-    Set::iterator it;
-    Set::const_iterator cit;
+  typedef std::multiset<A> Set;
+  int data;
+  Set m;
+  Set::iterator it;
+  Set::const_iterator cit;
 };
 
 inline bool operator==(A const& L, A const& R) { return &L == &R; }
-inline bool operator<(A const& L, A const& R)  { return L.data < R.data; }
+inline bool operator<(A const& L, A const& R) { return L.data < R.data; }
 int main(int, char**) {
-    A a;
+  A a;
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/associative/multiset/insert_cv.pass.cpp b/libcxx/test/std/containers/associative/multiset/insert_cv.pass.cpp
index b8a36ec385377..237cf9bb6fac2 100644
--- a/libcxx/test/std/containers/associative/multiset/insert_cv.pass.cpp
+++ b/libcxx/test/std/containers/associative/multiset/insert_cv.pass.cpp
@@ -18,45 +18,43 @@
 #include "test_macros.h"
 #include "min_allocator.h"
 
-template<class Container>
-void do_insert_cv_test()
-{
-    typedef Container M;
-    typedef typename M::iterator R;
-    typedef typename M::value_type VT;
-    M m;
-    const VT v1(2);
-    R r = m.insert(v1);
-    assert(r == m.begin());
-    assert(m.size() == 1);
-    assert(*r == 2);
-
-    const VT v2(1);
-    r = m.insert(v2);
-    assert(r == m.begin());
-    assert(m.size() == 2);
-    assert(*r == 1);
-
-    const VT v3(3);
-    r = m.insert(v3);
-    assert(r == std::prev(m.end()));
-    assert(m.size() == 3);
-    assert(*r == 3);
-
-    r = m.insert(v3);
-    assert(r == std::prev(m.end()));
-    assert(m.size() == 4);
-    assert(*r == 3);
+template <class Container>
+void do_insert_cv_test() {
+  typedef Container M;
+  typedef typename M::iterator R;
+  typedef typename M::value_type VT;
+  M m;
+  const VT v1(2);
+  R r = m.insert(v1);
+  assert(r == m.begin());
+  assert(m.size() == 1);
+  assert(*r == 2);
+
+  const VT v2(1);
+  r = m.insert(v2);
+  assert(r == m.begin());
+  assert(m.size() == 2);
+  assert(*r == 1);
+
+  const VT v3(3);
+  r = m.insert(v3);
+  assert(r == std::prev(m.end()));
+  assert(m.size() == 3);
+  assert(*r == 3);
+
+  r = m.insert(v3);
+  assert(r == std::prev(m.end()));
+  assert(m.size() == 4);
+  assert(*r == 3);
 }
 
-int main(int, char**)
-{
-    do_insert_cv_test<std::multiset<int> >();
+int main(int, char**) {
+  do_insert_cv_test<std::multiset<int> >();
 #if TEST_STD_VER >= 11
-    {
-        typedef std::multiset<int, std::less<int>, min_allocator<int>> M;
-        do_insert_cv_test<M>();
-    }
+  {
+    typedef std::multiset<int, std::less<int>, min_allocator<int>> M;
+    do_insert_cv_test<M>();
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/associative/multiset/insert_emplace_allocator_requirements.pass.cpp b/libcxx/test/std/containers/associative/multiset/insert_emplace_allocator_requirements.pass.cpp
index 232748fc353a3..8085ac7cbb2f1 100644
--- a/libcxx/test/std/containers/associative/multiset/insert_emplace_allocator_requirements.pass.cpp
+++ b/libcxx/test/std/containers/associative/multiset/insert_emplace_allocator_requirements.pass.cpp
@@ -20,8 +20,7 @@
 #include "container_test_types.h"
 #include "../../set_allocator_requirement_test_templates.h"
 
-int main(int, char**)
-{
+int main(int, char**) {
   testMultisetInsert<TCT::multiset<> >();
   testMultisetEmplace<TCT::multiset<> >();
 
diff --git a/libcxx/test/std/containers/associative/multiset/insert_initializer_list.pass.cpp b/libcxx/test/std/containers/associative/multiset/insert_initializer_list.pass.cpp
index 9c870b1298615..8f4f165252d36 100644
--- a/libcxx/test/std/containers/associative/multiset/insert_initializer_list.pass.cpp
+++ b/libcxx/test/std/containers/associative/multiset/insert_initializer_list.pass.cpp
@@ -21,9 +21,8 @@
 #include "test_macros.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
+int main(int, char**) {
+  {
     typedef std::multiset<int> C;
     typedef C::value_type V;
     C m = {10, 8};
@@ -39,8 +38,8 @@ int main(int, char**)
     assert(*++i == V(6));
     assert(*++i == V(8));
     assert(*++i == V(10));
-    }
-    {
+  }
+  {
     typedef std::multiset<int, std::less<int>, min_allocator<int>> C;
     typedef C::value_type V;
     C m = {10, 8};
@@ -56,7 +55,7 @@ int main(int, char**)
     assert(*++i == V(6));
     assert(*++i == V(8));
     assert(*++i == V(10));
-    }
+  }
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/associative/multiset/insert_iter_cv.pass.cpp b/libcxx/test/std/containers/associative/multiset/insert_iter_cv.pass.cpp
index 242e801c2acbf..97c03be6cc57b 100644
--- a/libcxx/test/std/containers/associative/multiset/insert_iter_cv.pass.cpp
+++ b/libcxx/test/std/containers/associative/multiset/insert_iter_cv.pass.cpp
@@ -18,57 +18,56 @@
 #include "test_macros.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef std::multiset<int> M;
-        typedef M::iterator R;
-        M m;
-        R r = m.insert(m.cend(), M::value_type(2));
-        assert(r == m.begin());
-        assert(m.size() == 1);
-        assert(*r == 2);
+int main(int, char**) {
+  {
+    typedef std::multiset<int> M;
+    typedef M::iterator R;
+    M m;
+    R r = m.insert(m.cend(), M::value_type(2));
+    assert(r == m.begin());
+    assert(m.size() == 1);
+    assert(*r == 2);
 
-        r = m.insert(m.cend(), M::value_type(1));
-        assert(r == m.begin());
-        assert(m.size() == 2);
-        assert(*r == 1);
+    r = m.insert(m.cend(), M::value_type(1));
+    assert(r == m.begin());
+    assert(m.size() == 2);
+    assert(*r == 1);
 
-        r = m.insert(m.cend(), M::value_type(3));
-        assert(r == std::prev(m.end()));
-        assert(m.size() == 3);
-        assert(*r == 3);
+    r = m.insert(m.cend(), M::value_type(3));
+    assert(r == std::prev(m.end()));
+    assert(m.size() == 3);
+    assert(*r == 3);
 
-        r = m.insert(m.cend(), M::value_type(3));
-        assert(r == std::prev(m.end()));
-        assert(m.size() == 4);
-        assert(*r == 3);
-    }
+    r = m.insert(m.cend(), M::value_type(3));
+    assert(r == std::prev(m.end()));
+    assert(m.size() == 4);
+    assert(*r == 3);
+  }
 #if TEST_STD_VER >= 11
-    {
-        typedef std::multiset<int, std::less<int>, min_allocator<int>> M;
-        typedef M::iterator R;
-        M m;
-        R r = m.insert(m.cend(), M::value_type(2));
-        assert(r == m.begin());
-        assert(m.size() == 1);
-        assert(*r == 2);
+  {
+    typedef std::multiset<int, std::less<int>, min_allocator<int>> M;
+    typedef M::iterator R;
+    M m;
+    R r = m.insert(m.cend(), M::value_type(2));
+    assert(r == m.begin());
+    assert(m.size() == 1);
+    assert(*r == 2);
 
-        r = m.insert(m.cend(), M::value_type(1));
-        assert(r == m.begin());
-        assert(m.size() == 2);
-        assert(*r == 1);
+    r = m.insert(m.cend(), M::value_type(1));
+    assert(r == m.begin());
+    assert(m.size() == 2);
+    assert(*r == 1);
 
-        r = m.insert(m.cend(), M::value_type(3));
-        assert(r == std::prev(m.end()));
-        assert(m.size() == 3);
-        assert(*r == 3);
+    r = m.insert(m.cend(), M::value_type(3));
+    assert(r == std::prev(m.end()));
+    assert(m.size() == 3);
+    assert(*r == 3);
 
-        r = m.insert(m.cend(), M::value_type(3));
-        assert(r == std::prev(m.end()));
-        assert(m.size() == 4);
-        assert(*r == 3);
-    }
+    r = m.insert(m.cend(), M::value_type(3));
+    assert(r == std::prev(m.end()));
+    assert(m.size() == 4);
+    assert(*r == 3);
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/associative/multiset/insert_iter_iter.pass.cpp b/libcxx/test/std/containers/associative/multiset/insert_iter_iter.pass.cpp
index 23d44b8e375bf..f3dd91794c9d5 100644
--- a/libcxx/test/std/containers/associative/multiset/insert_iter_iter.pass.cpp
+++ b/libcxx/test/std/containers/associative/multiset/insert_iter_iter.pass.cpp
@@ -20,67 +20,42 @@
 #include "test_iterators.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef std::multiset<int> M;
-        typedef int V;
-        V ar[] =
-        {
-            1,
-            1,
-            1,
-            2,
-            2,
-            2,
-            3,
-            3,
-            3
-        };
-        M m;
-        m.insert(cpp17_input_iterator<const V*>(ar),
-                 cpp17_input_iterator<const V*>(ar + sizeof(ar)/sizeof(ar[0])));
-        assert(m.size() == 9);
-        assert(*std::next(m.begin(), 0) == 1);
-        assert(*std::next(m.begin(), 1) == 1);
-        assert(*std::next(m.begin(), 2) == 1);
-        assert(*std::next(m.begin(), 3) == 2);
-        assert(*std::next(m.begin(), 4) == 2);
-        assert(*std::next(m.begin(), 5) == 2);
-        assert(*std::next(m.begin(), 6) == 3);
-        assert(*std::next(m.begin(), 7) == 3);
-        assert(*std::next(m.begin(), 8) == 3);
-    }
+int main(int, char**) {
+  {
+    typedef std::multiset<int> M;
+    typedef int V;
+    V ar[] = {1, 1, 1, 2, 2, 2, 3, 3, 3};
+    M m;
+    m.insert(cpp17_input_iterator<const V*>(ar), cpp17_input_iterator<const V*>(ar + sizeof(ar) / sizeof(ar[0])));
+    assert(m.size() == 9);
+    assert(*std::next(m.begin(), 0) == 1);
+    assert(*std::next(m.begin(), 1) == 1);
+    assert(*std::next(m.begin(), 2) == 1);
+    assert(*std::next(m.begin(), 3) == 2);
+    assert(*std::next(m.begin(), 4) == 2);
+    assert(*std::next(m.begin(), 5) == 2);
+    assert(*std::next(m.begin(), 6) == 3);
+    assert(*std::next(m.begin(), 7) == 3);
+    assert(*std::next(m.begin(), 8) == 3);
+  }
 #if TEST_STD_VER >= 11
-    {
-        typedef std::multiset<int, std::less<int>, min_allocator<int>> M;
-        typedef int V;
-        V ar[] =
-        {
-            1,
-            1,
-            1,
-            2,
-            2,
-            2,
-            3,
-            3,
-            3
-        };
-        M m;
-        m.insert(cpp17_input_iterator<const V*>(ar),
-                 cpp17_input_iterator<const V*>(ar + sizeof(ar)/sizeof(ar[0])));
-        assert(m.size() == 9);
-        assert(*std::next(m.begin(), 0) == 1);
-        assert(*std::next(m.begin(), 1) == 1);
-        assert(*std::next(m.begin(), 2) == 1);
-        assert(*std::next(m.begin(), 3) == 2);
-        assert(*std::next(m.begin(), 4) == 2);
-        assert(*std::next(m.begin(), 5) == 2);
-        assert(*std::next(m.begin(), 6) == 3);
-        assert(*std::next(m.begin(), 7) == 3);
-        assert(*std::next(m.begin(), 8) == 3);
-    }
+  {
+    typedef std::multiset<int, std::less<int>, min_allocator<int>> M;
+    typedef int V;
+    V ar[] = {1, 1, 1, 2, 2, 2, 3, 3, 3};
+    M m;
+    m.insert(cpp17_input_iterator<const V*>(ar), cpp17_input_iterator<const V*>(ar + sizeof(ar) / sizeof(ar[0])));
+    assert(m.size() == 9);
+    assert(*std::next(m.begin(), 0) == 1);
+    assert(*std::next(m.begin(), 1) == 1);
+    assert(*std::next(m.begin(), 2) == 1);
+    assert(*std::next(m.begin(), 3) == 2);
+    assert(*std::next(m.begin(), 4) == 2);
+    assert(*std::next(m.begin(), 5) == 2);
+    assert(*std::next(m.begin(), 6) == 3);
+    assert(*std::next(m.begin(), 7) == 3);
+    assert(*std::next(m.begin(), 8) == 3);
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/associative/multiset/insert_iter_rv.pass.cpp b/libcxx/test/std/containers/associative/multiset/insert_iter_rv.pass.cpp
index 75df25e9b0999..d01f6b9b2c306 100644
--- a/libcxx/test/std/containers/associative/multiset/insert_iter_rv.pass.cpp
+++ b/libcxx/test/std/containers/associative/multiset/insert_iter_rv.pass.cpp
@@ -21,56 +21,55 @@
 #include "MoveOnly.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef std::multiset<MoveOnly> M;
-        typedef M::iterator R;
-        M m;
-        R r = m.insert(m.cend(), M::value_type(2));
-        assert(r == m.begin());
-        assert(m.size() == 1);
-        assert(*r == 2);
+int main(int, char**) {
+  {
+    typedef std::multiset<MoveOnly> M;
+    typedef M::iterator R;
+    M m;
+    R r = m.insert(m.cend(), M::value_type(2));
+    assert(r == m.begin());
+    assert(m.size() == 1);
+    assert(*r == 2);
 
-        r = m.insert(m.cend(), M::value_type(1));
-        assert(r == m.begin());
-        assert(m.size() == 2);
-        assert(*r == 1);
+    r = m.insert(m.cend(), M::value_type(1));
+    assert(r == m.begin());
+    assert(m.size() == 2);
+    assert(*r == 1);
 
-        r = m.insert(m.cend(), M::value_type(3));
-        assert(r == std::prev(m.end()));
-        assert(m.size() == 3);
-        assert(*r == 3);
+    r = m.insert(m.cend(), M::value_type(3));
+    assert(r == std::prev(m.end()));
+    assert(m.size() == 3);
+    assert(*r == 3);
 
-        r = m.insert(m.cend(), M::value_type(3));
-        assert(r == std::prev(m.end()));
-        assert(m.size() == 4);
-        assert(*r == 3);
-    }
-    {
-        typedef std::multiset<MoveOnly, std::less<MoveOnly>, min_allocator<MoveOnly>> M;
-        typedef M::iterator R;
-        M m;
-        R r = m.insert(m.cend(), M::value_type(2));
-        assert(r == m.begin());
-        assert(m.size() == 1);
-        assert(*r == 2);
+    r = m.insert(m.cend(), M::value_type(3));
+    assert(r == std::prev(m.end()));
+    assert(m.size() == 4);
+    assert(*r == 3);
+  }
+  {
+    typedef std::multiset<MoveOnly, std::less<MoveOnly>, min_allocator<MoveOnly>> M;
+    typedef M::iterator R;
+    M m;
+    R r = m.insert(m.cend(), M::value_type(2));
+    assert(r == m.begin());
+    assert(m.size() == 1);
+    assert(*r == 2);
 
-        r = m.insert(m.cend(), M::value_type(1));
-        assert(r == m.begin());
-        assert(m.size() == 2);
-        assert(*r == 1);
+    r = m.insert(m.cend(), M::value_type(1));
+    assert(r == m.begin());
+    assert(m.size() == 2);
+    assert(*r == 1);
 
-        r = m.insert(m.cend(), M::value_type(3));
-        assert(r == std::prev(m.end()));
-        assert(m.size() == 3);
-        assert(*r == 3);
+    r = m.insert(m.cend(), M::value_type(3));
+    assert(r == std::prev(m.end()));
+    assert(m.size() == 3);
+    assert(*r == 3);
 
-        r = m.insert(m.cend(), M::value_type(3));
-        assert(r == std::prev(m.end()));
-        assert(m.size() == 4);
-        assert(*r == 3);
-    }
+    r = m.insert(m.cend(), M::value_type(3));
+    assert(r == std::prev(m.end()));
+    assert(m.size() == 4);
+    assert(*r == 3);
+  }
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/associative/multiset/insert_node_type.pass.cpp b/libcxx/test/std/containers/associative/multiset/insert_node_type.pass.cpp
index bc0d9bcac9361..6971d6b6e5eae 100644
--- a/libcxx/test/std/containers/associative/multiset/insert_node_type.pass.cpp
+++ b/libcxx/test/std/containers/associative/multiset/insert_node_type.pass.cpp
@@ -20,60 +20,54 @@
 #include "min_allocator.h"
 
 template <class Container>
-typename Container::node_type
-node_factory(typename Container::key_type const& key)
-{
-    static Container c;
-    auto it = c.insert(key);
-    return c.extract(it);
+typename Container::node_type node_factory(typename Container::key_type const& key) {
+  static Container c;
+  auto it = c.insert(key);
+  return c.extract(it);
 }
 
 template <class Container>
-void test(Container& c)
-{
-    auto* nf = &node_factory<Container>;
+void test(Container& c) {
+  auto* nf = &node_factory<Container>;
 
-    for (int i = 0; i != 10; ++i)
-    {
-        typename Container::node_type node = nf(i);
-        assert(!node.empty());
-        typename Container::iterator it = c.insert(std::move(node));
-        assert(node.empty());
-        assert(it == c.find(i) && it != c.end());
-        assert(*it == i);
-        assert(node.empty());
-    }
+  for (int i = 0; i != 10; ++i) {
+    typename Container::node_type node = nf(i);
+    assert(!node.empty());
+    typename Container::iterator it = c.insert(std::move(node));
+    assert(node.empty());
+    assert(it == c.find(i) && it != c.end());
+    assert(*it == i);
+    assert(node.empty());
+  }
 
-    assert(c.size() == 10);
+  assert(c.size() == 10);
 
-    { // Insert empty node.
-        typename Container::node_type def;
-        auto it = c.insert(std::move(def));
-        assert(def.empty());
-        assert(it == c.end());
-    }
+  { // Insert empty node.
+    typename Container::node_type def;
+    auto it = c.insert(std::move(def));
+    assert(def.empty());
+    assert(it == c.end());
+  }
 
-    { // Insert duplicate node.
-        typename Container::node_type dupl = nf(0);
-        auto it = c.insert(std::move(dupl));
-        assert(*it == 0);
-    }
+  { // Insert duplicate node.
+    typename Container::node_type dupl = nf(0);
+    auto it                            = c.insert(std::move(dupl));
+    assert(*it == 0);
+  }
 
-    assert(c.size() == 11);
+  assert(c.size() == 11);
 
-    assert(c.count(0) == 2);
-    for (int i = 1; i != 10; ++i)
-    {
-        assert(c.count(i) == 1);
-    }
+  assert(c.count(0) == 2);
+  for (int i = 1; i != 10; ++i) {
+    assert(c.count(i) == 1);
+  }
 }
 
-int main(int, char**)
-{
-    std::multiset<int> m;
-    test(m);
-    std::multiset<int, std::less<int>, min_allocator<int>> m2;
-    test(m2);
+int main(int, char**) {
+  std::multiset<int> m;
+  test(m);
+  std::multiset<int, std::less<int>, min_allocator<int>> m2;
+  test(m2);
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/associative/multiset/insert_node_type_hint.pass.cpp b/libcxx/test/std/containers/associative/multiset/insert_node_type_hint.pass.cpp
index e5c628e8509bc..b0392088beb6b 100644
--- a/libcxx/test/std/containers/associative/multiset/insert_node_type_hint.pass.cpp
+++ b/libcxx/test/std/containers/associative/multiset/insert_node_type_hint.pass.cpp
@@ -19,43 +19,37 @@
 #include "min_allocator.h"
 
 template <class Container>
-typename Container::node_type
-node_factory(typename Container::key_type const& key)
-{
-    static Container c;
-    auto it = c.insert(key);
-    return c.extract(it);
+typename Container::node_type node_factory(typename Container::key_type const& key) {
+  static Container c;
+  auto it = c.insert(key);
+  return c.extract(it);
 }
 
 template <class Container>
-void test(Container& c)
-{
-    auto* nf = &node_factory<Container>;
-
-    for (int i = 0; i != 10; ++i)
-    {
-        typename Container::node_type node = nf(i);
-        assert(!node.empty());
-        std::size_t prev = c.size();
-        auto it = c.insert(c.end(), std::move(node));
-        assert(prev + 1 == c.size());
-        assert(*it == i);
-    }
-
-    assert(c.size() == 10);
-
-    for (int i = 0; i != 10; ++i)
-    {
-        assert(c.count(i) == 1);
-    }
+void test(Container& c) {
+  auto* nf = &node_factory<Container>;
+
+  for (int i = 0; i != 10; ++i) {
+    typename Container::node_type node = nf(i);
+    assert(!node.empty());
+    std::size_t prev = c.size();
+    auto it          = c.insert(c.end(), std::move(node));
+    assert(prev + 1 == c.size());
+    assert(*it == i);
+  }
+
+  assert(c.size() == 10);
+
+  for (int i = 0; i != 10; ++i) {
+    assert(c.count(i) == 1);
+  }
 }
 
-int main(int, char**)
-{
-    std::multiset<int> m;
-    test(m);
-    std::multiset<int, std::less<int>, min_allocator<int>> m2;
-    test(m2);
+int main(int, char**) {
+  std::multiset<int> m;
+  test(m);
+  std::multiset<int, std::less<int>, min_allocator<int>> m2;
+  test(m2);
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/associative/multiset/insert_rv.pass.cpp b/libcxx/test/std/containers/associative/multiset/insert_rv.pass.cpp
index d34c96891d408..88155572f1680 100644
--- a/libcxx/test/std/containers/associative/multiset/insert_rv.pass.cpp
+++ b/libcxx/test/std/containers/associative/multiset/insert_rv.pass.cpp
@@ -21,56 +21,55 @@
 #include "MoveOnly.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef std::multiset<MoveOnly> M;
-        typedef M::iterator R;
-        M m;
-        R r = m.insert(M::value_type(2));
-        assert(r == m.begin());
-        assert(m.size() == 1);
-        assert(*r == 2);
+int main(int, char**) {
+  {
+    typedef std::multiset<MoveOnly> M;
+    typedef M::iterator R;
+    M m;
+    R r = m.insert(M::value_type(2));
+    assert(r == m.begin());
+    assert(m.size() == 1);
+    assert(*r == 2);
 
-        r = m.insert(M::value_type(1));
-        assert(r == m.begin());
-        assert(m.size() == 2);
-        assert(*r == 1);
+    r = m.insert(M::value_type(1));
+    assert(r == m.begin());
+    assert(m.size() == 2);
+    assert(*r == 1);
 
-        r = m.insert(M::value_type(3));
-        assert(r == std::prev(m.end()));
-        assert(m.size() == 3);
-        assert(*r == 3);
+    r = m.insert(M::value_type(3));
+    assert(r == std::prev(m.end()));
+    assert(m.size() == 3);
+    assert(*r == 3);
 
-        r = m.insert(M::value_type(3));
-        assert(r == std::prev(m.end()));
-        assert(m.size() == 4);
-        assert(*r == 3);
-    }
-    {
-        typedef std::multiset<MoveOnly, std::less<MoveOnly>, min_allocator<MoveOnly>> M;
-        typedef M::iterator R;
-        M m;
-        R r = m.insert(M::value_type(2));
-        assert(r == m.begin());
-        assert(m.size() == 1);
-        assert(*r == 2);
+    r = m.insert(M::value_type(3));
+    assert(r == std::prev(m.end()));
+    assert(m.size() == 4);
+    assert(*r == 3);
+  }
+  {
+    typedef std::multiset<MoveOnly, std::less<MoveOnly>, min_allocator<MoveOnly>> M;
+    typedef M::iterator R;
+    M m;
+    R r = m.insert(M::value_type(2));
+    assert(r == m.begin());
+    assert(m.size() == 1);
+    assert(*r == 2);
 
-        r = m.insert(M::value_type(1));
-        assert(r == m.begin());
-        assert(m.size() == 2);
-        assert(*r == 1);
+    r = m.insert(M::value_type(1));
+    assert(r == m.begin());
+    assert(m.size() == 2);
+    assert(*r == 1);
 
-        r = m.insert(M::value_type(3));
-        assert(r == std::prev(m.end()));
-        assert(m.size() == 3);
-        assert(*r == 3);
+    r = m.insert(M::value_type(3));
+    assert(r == std::prev(m.end()));
+    assert(m.size() == 3);
+    assert(*r == 3);
 
-        r = m.insert(M::value_type(3));
-        assert(r == std::prev(m.end()));
-        assert(m.size() == 4);
-        assert(*r == 3);
-    }
+    r = m.insert(M::value_type(3));
+    assert(r == std::prev(m.end()));
+    assert(m.size() == 4);
+    assert(*r == 3);
+  }
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/associative/multiset/iterator.pass.cpp b/libcxx/test/std/containers/associative/multiset/iterator.pass.cpp
index 27a8dbf1a0388..fee8bf9d3eb30 100644
--- a/libcxx/test/std/containers/associative/multiset/iterator.pass.cpp
+++ b/libcxx/test/std/containers/associative/multiset/iterator.pass.cpp
@@ -32,214 +32,109 @@
 #include "test_macros.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef int V;
-        V ar[] =
-        {
-            1,
-            1,
-            1,
-            2,
-            2,
-            2,
-            3,
-            3,
-            3,
-            4,
-            4,
-            4,
-            5,
-            5,
-            5,
-            6,
-            6,
-            6,
-            7,
-            7,
-            7,
-            8,
-            8,
-            8
-        };
-        std::multiset<int> m(ar, ar+sizeof(ar)/sizeof(ar[0]));
-        assert(static_cast<std::size_t>(std::distance(m.begin(), m.end())) == m.size());
-        assert(static_cast<std::size_t>(std::distance(m.rbegin(), m.rend())) == m.size());
-        std::multiset<int>::iterator i;
-        i = m.begin();
-        std::multiset<int>::const_iterator k = i;
-        assert(i == k);
-        for (int j = 1; j <= 8; ++j)
-            for (int n = 0; n < 3; ++n, ++i)
-                assert(*i == j);
-        assert(i == m.end());
-        for (int j = 8; j >= 1; --j)
-            for (int n = 0; n < 3; ++n) {
-                --i;
-                assert(*i == j);
-            }
-        assert(i == m.begin());
-    }
-    {
-        typedef int V;
-        V ar[] =
-        {
-            1,
-            1,
-            1,
-            2,
-            2,
-            2,
-            3,
-            3,
-            3,
-            4,
-            4,
-            4,
-            5,
-            5,
-            5,
-            6,
-            6,
-            6,
-            7,
-            7,
-            7,
-            8,
-            8,
-            8
-        };
-        const std::multiset<int> m(ar, ar+sizeof(ar)/sizeof(ar[0]));
-        assert(static_cast<std::size_t>(std::distance(m.begin(), m.end())) == m.size());
-        assert(static_cast<std::size_t>(std::distance(m.cbegin(), m.cend())) == m.size());
-        assert(static_cast<std::size_t>(std::distance(m.rbegin(), m.rend())) == m.size());
-        assert(static_cast<std::size_t>(std::distance(m.crbegin(), m.crend())) == m.size());
-        std::multiset<int>::const_iterator i;
-        i = m.begin();
-        for (int j = 1; j <= 8; ++j)
-            for (int k = 0; k < 3; ++k, ++i)
-                assert(*i == j);
-        assert(i == m.end());
-        for (int j = 8; j >= 1; --j)
-            for (int k = 0; k < 3; ++k) {
-                --i;
-                assert(*i == j);
-            }
-        assert(i == m.begin());
-    }
+int main(int, char**) {
+  {
+    typedef int V;
+    V ar[] = {1, 1, 1, 2, 2, 2, 3, 3, 3, 4, 4, 4, 5, 5, 5, 6, 6, 6, 7, 7, 7, 8, 8, 8};
+    std::multiset<int> m(ar, ar + sizeof(ar) / sizeof(ar[0]));
+    assert(static_cast<std::size_t>(std::distance(m.begin(), m.end())) == m.size());
+    assert(static_cast<std::size_t>(std::distance(m.rbegin(), m.rend())) == m.size());
+    std::multiset<int>::iterator i;
+    i                                    = m.begin();
+    std::multiset<int>::const_iterator k = i;
+    assert(i == k);
+    for (int j = 1; j <= 8; ++j)
+      for (int n = 0; n < 3; ++n, ++i)
+        assert(*i == j);
+    assert(i == m.end());
+    for (int j = 8; j >= 1; --j)
+      for (int n = 0; n < 3; ++n) {
+        --i;
+        assert(*i == j);
+      }
+    assert(i == m.begin());
+  }
+  {
+    typedef int V;
+    V ar[] = {1, 1, 1, 2, 2, 2, 3, 3, 3, 4, 4, 4, 5, 5, 5, 6, 6, 6, 7, 7, 7, 8, 8, 8};
+    const std::multiset<int> m(ar, ar + sizeof(ar) / sizeof(ar[0]));
+    assert(static_cast<std::size_t>(std::distance(m.begin(), m.end())) == m.size());
+    assert(static_cast<std::size_t>(std::distance(m.cbegin(), m.cend())) == m.size());
+    assert(static_cast<std::size_t>(std::distance(m.rbegin(), m.rend())) == m.size());
+    assert(static_cast<std::size_t>(std::distance(m.crbegin(), m.crend())) == m.size());
+    std::multiset<int>::const_iterator i;
+    i = m.begin();
+    for (int j = 1; j <= 8; ++j)
+      for (int k = 0; k < 3; ++k, ++i)
+        assert(*i == j);
+    assert(i == m.end());
+    for (int j = 8; j >= 1; --j)
+      for (int k = 0; k < 3; ++k) {
+        --i;
+        assert(*i == j);
+      }
+    assert(i == m.begin());
+  }
 #if TEST_STD_VER >= 11
-    {
-        typedef int V;
-        V ar[] =
-        {
-            1,
-            1,
-            1,
-            2,
-            2,
-            2,
-            3,
-            3,
-            3,
-            4,
-            4,
-            4,
-            5,
-            5,
-            5,
-            6,
-            6,
-            6,
-            7,
-            7,
-            7,
-            8,
-            8,
-            8
-        };
-        std::multiset<int, std::less<int>, min_allocator<int>> m(ar, ar+sizeof(ar)/sizeof(ar[0]));
-        assert(static_cast<std::size_t>(std::distance(m.begin(), m.end())) == m.size());
-        assert(static_cast<std::size_t>(std::distance(m.rbegin(), m.rend())) == m.size());
-        std::multiset<int, std::less<int>, min_allocator<int>>::iterator i;
-        i = m.begin();
-        std::multiset<int, std::less<int>, min_allocator<int>>::const_iterator k = i;
-        assert(i == k);
-        for (int j = 1; j <= 8; ++j)
-            for (int n = 0; n < 3; ++n, ++i)
-                assert(*i == j);
-        assert(i == m.end());
-        for (int j = 8; j >= 1; --j)
-            for (int n = 0; n < 3; ++n) {
-                --i;
-                assert(*i == j);
-            }
-        assert(i == m.begin());
-    }
-    {
-        typedef int V;
-        V ar[] =
-        {
-            1,
-            1,
-            1,
-            2,
-            2,
-            2,
-            3,
-            3,
-            3,
-            4,
-            4,
-            4,
-            5,
-            5,
-            5,
-            6,
-            6,
-            6,
-            7,
-            7,
-            7,
-            8,
-            8,
-            8
-        };
-        const std::multiset<int, std::less<int>, min_allocator<int>> m(ar, ar+sizeof(ar)/sizeof(ar[0]));
-        assert(static_cast<std::size_t>(std::distance(m.begin(), m.end())) == m.size());
-        assert(static_cast<std::size_t>(std::distance(m.cbegin(), m.cend())) == m.size());
-        assert(static_cast<std::size_t>(std::distance(m.rbegin(), m.rend())) == m.size());
-        assert(static_cast<std::size_t>(std::distance(m.crbegin(), m.crend())) == m.size());
-        std::multiset<int, std::less<int>, min_allocator<int>>::const_iterator i;
-        i = m.begin();
-        for (int j = 1; j <= 8; ++j)
-            for (int k = 0; k < 3; ++k, ++i)
-                assert(*i == j);
-        assert(i == m.end());
-        for (int j = 8; j >= 1; --j)
-            for (int k = 0; k < 3; ++k) {
-                --i;
-                assert(*i == j);
-            }
-        assert(i == m.begin());
-    }
+  {
+    typedef int V;
+    V ar[] = {1, 1, 1, 2, 2, 2, 3, 3, 3, 4, 4, 4, 5, 5, 5, 6, 6, 6, 7, 7, 7, 8, 8, 8};
+    std::multiset<int, std::less<int>, min_allocator<int>> m(ar, ar + sizeof(ar) / sizeof(ar[0]));
+    assert(static_cast<std::size_t>(std::distance(m.begin(), m.end())) == m.size());
+    assert(static_cast<std::size_t>(std::distance(m.rbegin(), m.rend())) == m.size());
+    std::multiset<int, std::less<int>, min_allocator<int>>::iterator i;
+    i                                                                        = m.begin();
+    std::multiset<int, std::less<int>, min_allocator<int>>::const_iterator k = i;
+    assert(i == k);
+    for (int j = 1; j <= 8; ++j)
+      for (int n = 0; n < 3; ++n, ++i)
+        assert(*i == j);
+    assert(i == m.end());
+    for (int j = 8; j >= 1; --j)
+      for (int n = 0; n < 3; ++n) {
+        --i;
+        assert(*i == j);
+      }
+    assert(i == m.begin());
+  }
+  {
+    typedef int V;
+    V ar[] = {1, 1, 1, 2, 2, 2, 3, 3, 3, 4, 4, 4, 5, 5, 5, 6, 6, 6, 7, 7, 7, 8, 8, 8};
+    const std::multiset<int, std::less<int>, min_allocator<int>> m(ar, ar + sizeof(ar) / sizeof(ar[0]));
+    assert(static_cast<std::size_t>(std::distance(m.begin(), m.end())) == m.size());
+    assert(static_cast<std::size_t>(std::distance(m.cbegin(), m.cend())) == m.size());
+    assert(static_cast<std::size_t>(std::distance(m.rbegin(), m.rend())) == m.size());
+    assert(static_cast<std::size_t>(std::distance(m.crbegin(), m.crend())) == m.size());
+    std::multiset<int, std::less<int>, min_allocator<int>>::const_iterator i;
+    i = m.begin();
+    for (int j = 1; j <= 8; ++j)
+      for (int k = 0; k < 3; ++k, ++i)
+        assert(*i == j);
+    assert(i == m.end());
+    for (int j = 8; j >= 1; --j)
+      for (int k = 0; k < 3; ++k) {
+        --i;
+        assert(*i == j);
+      }
+    assert(i == m.begin());
+  }
 #endif
 #if TEST_STD_VER > 11
-    { // N3644 testing
-        typedef std::multiset<int> C;
-        C::iterator ii1{}, ii2{};
-        C::iterator ii4 = ii1;
-        C::const_iterator cii{};
-        assert ( ii1 == ii2 );
-        assert ( ii1 == ii4 );
+  { // N3644 testing
+    typedef std::multiset<int> C;
+    C::iterator ii1{}, ii2{};
+    C::iterator ii4 = ii1;
+    C::const_iterator cii{};
+    assert(ii1 == ii2);
+    assert(ii1 == ii4);
 
-        assert (!(ii1 != ii2 ));
+    assert(!(ii1 != ii2));
 
-        assert ( (ii1 == cii ));
-        assert ( (cii == ii1 ));
-        assert (!(ii1 != cii ));
-        assert (!(cii != ii1 ));
-    }
+    assert((ii1 == cii));
+    assert((cii == ii1));
+    assert(!(ii1 != cii));
+    assert(!(cii != ii1));
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/associative/multiset/iterator_concept_conformance.compile.pass.cpp b/libcxx/test/std/containers/associative/multiset/iterator_concept_conformance.compile.pass.cpp
index d452afcaeb4ba..81e5e9ddfb0d2 100644
--- a/libcxx/test/std/containers/associative/multiset/iterator_concept_conformance.compile.pass.cpp
+++ b/libcxx/test/std/containers/associative/multiset/iterator_concept_conformance.compile.pass.cpp
@@ -14,11 +14,11 @@
 
 #include <iterator>
 
-using iterator = std::multiset<int>::iterator;
-using const_iterator = std::multiset<int>::const_iterator;
-using reverse_iterator = std::multiset<int>::reverse_iterator;
+using iterator               = std::multiset<int>::iterator;
+using const_iterator         = std::multiset<int>::const_iterator;
+using reverse_iterator       = std::multiset<int>::reverse_iterator;
 using const_reverse_iterator = std::multiset<int>::const_reverse_iterator;
-using value_type = int;
+using value_type             = int;
 
 static_assert(std::bidirectional_iterator<iterator>);
 static_assert(!std::random_access_iterator<iterator>);
diff --git a/libcxx/test/std/containers/associative/multiset/lower_bound.pass.cpp b/libcxx/test/std/containers/associative/multiset/lower_bound.pass.cpp
index d391da8ab0f70..ddd39d8fc0348 100644
--- a/libcxx/test/std/containers/associative/multiset/lower_bound.pass.cpp
+++ b/libcxx/test/std/containers/associative/multiset/lower_bound.pass.cpp
@@ -20,157 +20,101 @@
 #include "min_allocator.h"
 #include "private_constructor.h"
 
-int main(int, char**)
-{
+int main(int, char**) {
+  {
+    typedef int V;
+    typedef std::multiset<int> M;
+    {
+      typedef M::iterator R;
+      V ar[] = {5, 5, 5, 7, 7, 7, 9, 9, 9};
+      M m(ar, ar + sizeof(ar) / sizeof(ar[0]));
+      R r = m.lower_bound(4);
+      assert(r == std::next(m.begin(), 0));
+      r = m.lower_bound(5);
+      assert(r == std::next(m.begin(), 0));
+      r = m.lower_bound(6);
+      assert(r == std::next(m.begin(), 3));
+      r = m.lower_bound(7);
+      assert(r == std::next(m.begin(), 3));
+      r = m.lower_bound(8);
+      assert(r == std::next(m.begin(), 6));
+      r = m.lower_bound(9);
+      assert(r == std::next(m.begin(), 6));
+      r = m.lower_bound(11);
+      assert(r == std::next(m.begin(), 9));
+    }
     {
-        typedef int V;
-        typedef std::multiset<int> M;
-        {
-            typedef M::iterator R;
-            V ar[] =
-            {
-                5,
-                5,
-                5,
-                7,
-                7,
-                7,
-                9,
-                9,
-                9
-            };
-            M m(ar, ar+sizeof(ar)/sizeof(ar[0]));
-            R r = m.lower_bound(4);
-            assert(r == std::next(m.begin(), 0));
-            r = m.lower_bound(5);
-            assert(r == std::next(m.begin(), 0));
-            r = m.lower_bound(6);
-            assert(r == std::next(m.begin(), 3));
-            r = m.lower_bound(7);
-            assert(r == std::next(m.begin(), 3));
-            r = m.lower_bound(8);
-            assert(r == std::next(m.begin(), 6));
-            r = m.lower_bound(9);
-            assert(r == std::next(m.begin(), 6));
-            r = m.lower_bound(11);
-            assert(r == std::next(m.begin(), 9));
-        }
-        {
-            typedef M::const_iterator R;
-            V ar[] =
-            {
-                5,
-                5,
-                5,
-                7,
-                7,
-                7,
-                9,
-                9,
-                9
-            };
-            const M m(ar, ar+sizeof(ar)/sizeof(ar[0]));
-            R r = m.lower_bound(4);
-            assert(r == std::next(m.begin(), 0));
-            r = m.lower_bound(5);
-            assert(r == std::next(m.begin(), 0));
-            r = m.lower_bound(6);
-            assert(r == std::next(m.begin(), 3));
-            r = m.lower_bound(7);
-            assert(r == std::next(m.begin(), 3));
-            r = m.lower_bound(8);
-            assert(r == std::next(m.begin(), 6));
-            r = m.lower_bound(9);
-            assert(r == std::next(m.begin(), 6));
-            r = m.lower_bound(11);
-            assert(r == std::next(m.begin(), 9));
-        }
+      typedef M::const_iterator R;
+      V ar[] = {5, 5, 5, 7, 7, 7, 9, 9, 9};
+      const M m(ar, ar + sizeof(ar) / sizeof(ar[0]));
+      R r = m.lower_bound(4);
+      assert(r == std::next(m.begin(), 0));
+      r = m.lower_bound(5);
+      assert(r == std::next(m.begin(), 0));
+      r = m.lower_bound(6);
+      assert(r == std::next(m.begin(), 3));
+      r = m.lower_bound(7);
+      assert(r == std::next(m.begin(), 3));
+      r = m.lower_bound(8);
+      assert(r == std::next(m.begin(), 6));
+      r = m.lower_bound(9);
+      assert(r == std::next(m.begin(), 6));
+      r = m.lower_bound(11);
+      assert(r == std::next(m.begin(), 9));
     }
+  }
 #if TEST_STD_VER >= 11
+  {
+    typedef int V;
+    typedef std::multiset<int, std::less<int>, min_allocator<int>> M;
+    {
+      typedef M::iterator R;
+      V ar[] = {5, 5, 5, 7, 7, 7, 9, 9, 9};
+      M m(ar, ar + sizeof(ar) / sizeof(ar[0]));
+      R r = m.lower_bound(4);
+      assert(r == std::next(m.begin(), 0));
+      r = m.lower_bound(5);
+      assert(r == std::next(m.begin(), 0));
+      r = m.lower_bound(6);
+      assert(r == std::next(m.begin(), 3));
+      r = m.lower_bound(7);
+      assert(r == std::next(m.begin(), 3));
+      r = m.lower_bound(8);
+      assert(r == std::next(m.begin(), 6));
+      r = m.lower_bound(9);
+      assert(r == std::next(m.begin(), 6));
+      r = m.lower_bound(11);
+      assert(r == std::next(m.begin(), 9));
+    }
     {
-        typedef int V;
-        typedef std::multiset<int, std::less<int>, min_allocator<int>> M;
-        {
-            typedef M::iterator R;
-            V ar[] =
-            {
-                5,
-                5,
-                5,
-                7,
-                7,
-                7,
-                9,
-                9,
-                9
-            };
-            M m(ar, ar+sizeof(ar)/sizeof(ar[0]));
-            R r = m.lower_bound(4);
-            assert(r == std::next(m.begin(), 0));
-            r = m.lower_bound(5);
-            assert(r == std::next(m.begin(), 0));
-            r = m.lower_bound(6);
-            assert(r == std::next(m.begin(), 3));
-            r = m.lower_bound(7);
-            assert(r == std::next(m.begin(), 3));
-            r = m.lower_bound(8);
-            assert(r == std::next(m.begin(), 6));
-            r = m.lower_bound(9);
-            assert(r == std::next(m.begin(), 6));
-            r = m.lower_bound(11);
-            assert(r == std::next(m.begin(), 9));
-        }
-        {
-            typedef M::const_iterator R;
-            V ar[] =
-            {
-                5,
-                5,
-                5,
-                7,
-                7,
-                7,
-                9,
-                9,
-                9
-            };
-            const M m(ar, ar+sizeof(ar)/sizeof(ar[0]));
-            R r = m.lower_bound(4);
-            assert(r == std::next(m.begin(), 0));
-            r = m.lower_bound(5);
-            assert(r == std::next(m.begin(), 0));
-            r = m.lower_bound(6);
-            assert(r == std::next(m.begin(), 3));
-            r = m.lower_bound(7);
-            assert(r == std::next(m.begin(), 3));
-            r = m.lower_bound(8);
-            assert(r == std::next(m.begin(), 6));
-            r = m.lower_bound(9);
-            assert(r == std::next(m.begin(), 6));
-            r = m.lower_bound(11);
-            assert(r == std::next(m.begin(), 9));
-        }
+      typedef M::const_iterator R;
+      V ar[] = {5, 5, 5, 7, 7, 7, 9, 9, 9};
+      const M m(ar, ar + sizeof(ar) / sizeof(ar[0]));
+      R r = m.lower_bound(4);
+      assert(r == std::next(m.begin(), 0));
+      r = m.lower_bound(5);
+      assert(r == std::next(m.begin(), 0));
+      r = m.lower_bound(6);
+      assert(r == std::next(m.begin(), 3));
+      r = m.lower_bound(7);
+      assert(r == std::next(m.begin(), 3));
+      r = m.lower_bound(8);
+      assert(r == std::next(m.begin(), 6));
+      r = m.lower_bound(9);
+      assert(r == std::next(m.begin(), 6));
+      r = m.lower_bound(11);
+      assert(r == std::next(m.begin(), 9));
     }
+  }
 #endif
 #if TEST_STD_VER > 11
-    {
+  {
     typedef int V;
     typedef std::multiset<V, std::less<>> M;
 
     typedef M::iterator R;
-    V ar[] =
-    {
-        5,
-        5,
-        5,
-        7,
-        7,
-        7,
-        9,
-        9,
-        9
-    };
-    M m(ar, ar+sizeof(ar)/sizeof(ar[0]));
+    V ar[] = {5, 5, 5, 7, 7, 7, 9, 9, 9};
+    M m(ar, ar + sizeof(ar) / sizeof(ar[0]));
 
     R r = m.lower_bound(4);
     assert(r == std::next(m.begin(), 0));
@@ -186,23 +130,23 @@ int main(int, char**)
     assert(r == std::next(m.begin(), 6));
     r = m.lower_bound(11);
     assert(r == std::next(m.begin(), 9));
-    }
+  }
 
-    {
+  {
     typedef PrivateConstructor V;
     typedef std::multiset<V, std::less<>> M;
     typedef M::iterator R;
 
     M m;
-    m.insert ( V::make ( 5 ));
-    m.insert ( V::make ( 5 ));
-    m.insert ( V::make ( 5 ));
-    m.insert ( V::make ( 7 ));
-    m.insert ( V::make ( 7 ));
-    m.insert ( V::make ( 7 ));
-    m.insert ( V::make ( 9 ));
-    m.insert ( V::make ( 9 ));
-    m.insert ( V::make ( 9 ));
+    m.insert(V::make(5));
+    m.insert(V::make(5));
+    m.insert(V::make(5));
+    m.insert(V::make(7));
+    m.insert(V::make(7));
+    m.insert(V::make(7));
+    m.insert(V::make(9));
+    m.insert(V::make(9));
+    m.insert(V::make(9));
 
     R r = m.lower_bound(4);
     assert(r == std::next(m.begin(), 0));
@@ -218,7 +162,7 @@ int main(int, char**)
     assert(r == std::next(m.begin(), 6));
     r = m.lower_bound(11);
     assert(r == std::next(m.begin(), 9));
-    }
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/associative/multiset/max_size.pass.cpp b/libcxx/test/std/containers/associative/multiset/max_size.pass.cpp
index 09faf8f9b97e6..a6cff58b090bd 100644
--- a/libcxx/test/std/containers/associative/multiset/max_size.pass.cpp
+++ b/libcxx/test/std/containers/associative/multiset/max_size.pass.cpp
@@ -20,32 +20,29 @@
 #include "test_allocator.h"
 #include "test_macros.h"
 
-int main(int, char**)
-{
-    {
-      typedef limited_allocator<int, 10> A;
-      typedef std::multiset<int, std::less<int>, A> C;
-      C c;
-      assert(c.max_size() <= 10);
-      LIBCPP_ASSERT(c.max_size() == 10);
-    }
-    {
-      typedef limited_allocator<int, (std::size_t)-1> A;
-      typedef std::multiset<int, std::less<int>, A> C;
-      const C::size_type max_dist =
-          static_cast<C::size_type>(std::numeric_limits<C::difference_type>::max());
-      C c;
-      assert(c.max_size() <= max_dist);
-      LIBCPP_ASSERT(c.max_size() == max_dist);
-    }
-    {
-      typedef std::multiset<char> C;
-      const C::size_type max_dist =
-          static_cast<C::size_type>(std::numeric_limits<C::difference_type>::max());
-      C c;
-      assert(c.max_size() <= max_dist);
-      assert(c.max_size() <= alloc_max_size(c.get_allocator()));
-    }
+int main(int, char**) {
+  {
+    typedef limited_allocator<int, 10> A;
+    typedef std::multiset<int, std::less<int>, A> C;
+    C c;
+    assert(c.max_size() <= 10);
+    LIBCPP_ASSERT(c.max_size() == 10);
+  }
+  {
+    typedef limited_allocator<int, (std::size_t)-1> A;
+    typedef std::multiset<int, std::less<int>, A> C;
+    const C::size_type max_dist = static_cast<C::size_type>(std::numeric_limits<C::difference_type>::max());
+    C c;
+    assert(c.max_size() <= max_dist);
+    LIBCPP_ASSERT(c.max_size() == max_dist);
+  }
+  {
+    typedef std::multiset<char> C;
+    const C::size_type max_dist = static_cast<C::size_type>(std::numeric_limits<C::difference_type>::max());
+    C c;
+    assert(c.max_size() <= max_dist);
+    assert(c.max_size() <= alloc_max_size(c.get_allocator()));
+  }
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/associative/multiset/merge.pass.cpp b/libcxx/test/std/containers/associative/multiset/merge.pass.cpp
index 3e99d88995fb8..90eaeb2adbfcb 100644
--- a/libcxx/test/std/containers/associative/multiset/merge.pass.cpp
+++ b/libcxx/test/std/containers/associative/multiset/merge.pass.cpp
@@ -27,123 +27,112 @@
 #include "Counter.h"
 
 template <class Set>
-bool set_equal(const Set& set, Set other)
-{
-    return set == other;
+bool set_equal(const Set& set, Set other) {
+  return set == other;
 }
 
 #ifndef TEST_HAS_NO_EXCEPTIONS
-struct throw_comparator
-{
-    bool& should_throw_;
+struct throw_comparator {
+  bool& should_throw_;
 
-    throw_comparator(bool& should_throw) : should_throw_(should_throw) {}
+  throw_comparator(bool& should_throw) : should_throw_(should_throw) {}
 
-    template <class T>
-    bool operator()(const T& lhs, const T& rhs) const
-    {
-        if (should_throw_)
-            throw 0;
-        return lhs < rhs;
-    }
+  template <class T>
+  bool operator()(const T& lhs, const T& rhs) const {
+    if (should_throw_)
+      throw 0;
+    return lhs < rhs;
+  }
 };
 #endif
 
-int main(int, char**)
-{
-    {
-        std::multiset<int> src{1, 3, 5};
-        std::multiset<int> dst{2, 4, 5};
-        dst.merge(src);
-        assert(set_equal(src, {}));
-        assert(set_equal(dst, {1, 2, 3, 4, 5, 5}));
-    }
+int main(int, char**) {
+  {
+    std::multiset<int> src{1, 3, 5};
+    std::multiset<int> dst{2, 4, 5};
+    dst.merge(src);
+    assert(set_equal(src, {}));
+    assert(set_equal(dst, {1, 2, 3, 4, 5, 5}));
+  }
 
 #ifndef TEST_HAS_NO_EXCEPTIONS
-    {
-        bool do_throw = false;
-        typedef std::multiset<Counter<int>, throw_comparator> set_type;
-        set_type src({1, 3, 5}, throw_comparator(do_throw));
-        set_type dst({2, 4, 5}, throw_comparator(do_throw));
-
-        assert(Counter_base::gConstructed == 6);
-
-        do_throw = true;
-        try
-        {
-            dst.merge(src);
-        }
-        catch (int)
-        {
-            do_throw = false;
-        }
-        assert(!do_throw);
-        assert(set_equal(src, set_type({1, 3, 5}, throw_comparator(do_throw))));
-        assert(set_equal(dst, set_type({2, 4, 5}, throw_comparator(do_throw))));
+  {
+    bool do_throw = false;
+    typedef std::multiset<Counter<int>, throw_comparator> set_type;
+    set_type src({1, 3, 5}, throw_comparator(do_throw));
+    set_type dst({2, 4, 5}, throw_comparator(do_throw));
+
+    assert(Counter_base::gConstructed == 6);
+
+    do_throw = true;
+    try {
+      dst.merge(src);
+    } catch (int) {
+      do_throw = false;
     }
+    assert(!do_throw);
+    assert(set_equal(src, set_type({1, 3, 5}, throw_comparator(do_throw))));
+    assert(set_equal(dst, set_type({2, 4, 5}, throw_comparator(do_throw))));
+  }
 #endif
-    assert(Counter_base::gConstructed == 0);
-    struct comparator
-    {
-        comparator() = default;
+  assert(Counter_base::gConstructed == 0);
+  struct comparator {
+    comparator() = default;
 
-        bool operator()(const Counter<int>& lhs, const Counter<int>& rhs) const
-        {
-            return lhs < rhs;
-        }
-    };
-    {
-        typedef std::multiset<Counter<int>, std::less<Counter<int>>> first_set_type;
-        typedef std::multiset<Counter<int>, comparator> second_set_type;
-        typedef std::set<Counter<int>, comparator> third_set_type;
+    bool operator()(const Counter<int>& lhs, const Counter<int>& rhs) const { return lhs < rhs; }
+  };
+  {
+    typedef std::multiset<Counter<int>, std::less<Counter<int>>> first_set_type;
+    typedef std::multiset<Counter<int>, comparator> second_set_type;
+    typedef std::set<Counter<int>, comparator> third_set_type;
 
-        {
-            first_set_type first{1, 2, 3};
-            second_set_type second{2, 3, 4};
-            third_set_type third{1, 3};
+    {
+      first_set_type first{1, 2, 3};
+      second_set_type second{2, 3, 4};
+      third_set_type third{1, 3};
 
-            assert(Counter_base::gConstructed == 8);
+      assert(Counter_base::gConstructed == 8);
 
-            first.merge(second);
-            first.merge(third);
+      first.merge(second);
+      first.merge(third);
 
-            assert(set_equal(first, {1, 1, 2, 2, 3, 3, 3, 4}));
-            assert(set_equal(second, {}));
-            assert(set_equal(third, {}));
+      assert(set_equal(first, {1, 1, 2, 2, 3, 3, 3, 4}));
+      assert(set_equal(second, {}));
+      assert(set_equal(third, {}));
 
-            assert(Counter_base::gConstructed == 8);
-        }
-        assert(Counter_base::gConstructed == 0);
-        {
-            first_set_type first{1, 2, 3};
-            second_set_type second{2, 3, 4};
-            third_set_type third{1, 3};
+      assert(Counter_base::gConstructed == 8);
+    }
+    assert(Counter_base::gConstructed == 0);
+    {
+      first_set_type first{1, 2, 3};
+      second_set_type second{2, 3, 4};
+      third_set_type third{1, 3};
 
-            assert(Counter_base::gConstructed == 8);
+      assert(Counter_base::gConstructed == 8);
 
-            first.merge(std::move(second));
-            first.merge(std::move(third));
+      first.merge(std::move(second));
+      first.merge(std::move(third));
 
-            assert(set_equal(first, {1, 1, 2, 2, 3, 3, 3, 4}));
-            assert(set_equal(second, {}));
-            assert(set_equal(third, {}));
+      assert(set_equal(first, {1, 1, 2, 2, 3, 3, 3, 4}));
+      assert(set_equal(second, {}));
+      assert(set_equal(third, {}));
 
-            assert(Counter_base::gConstructed == 8);
-        }
-        assert(Counter_base::gConstructed == 0);
+      assert(Counter_base::gConstructed == 8);
+    }
+    assert(Counter_base::gConstructed == 0);
+  }
+  {
+    std::multiset<int> first;
+    {
+      std::multiset<int> second;
+      first.merge(second);
+      first.merge(std::move(second));
     }
     {
-        std::multiset<int> first;
-        {
-            std::multiset<int> second;
-            first.merge(second);
-            first.merge(std::move(second));
-        }
-        {
-            std::set<int> second;
-            first.merge(second);
-            first.merge(std::move(second));
-        }
+      std::set<int> second;
+      first.merge(second);
+      first.merge(std::move(second));
     }
-    return 0;
+  }
+  return 0;
 }
diff --git a/libcxx/test/std/containers/associative/multiset/multiset.cons/alloc.pass.cpp b/libcxx/test/std/containers/associative/multiset/multiset.cons/alloc.pass.cpp
index 8aa09cae659d0..1f05201d706b1 100644
--- a/libcxx/test/std/containers/associative/multiset/multiset.cons/alloc.pass.cpp
+++ b/libcxx/test/std/containers/associative/multiset/multiset.cons/alloc.pass.cpp
@@ -18,14 +18,13 @@
 #include "test_macros.h"
 #include "test_allocator.h"
 
-int main(int, char**)
-{
-    typedef std::less<int> C;
-    typedef test_allocator<int> A;
-    std::multiset<int, C, A> m(A(5));
-    assert(m.empty());
-    assert(m.begin() == m.end());
-    assert(m.get_allocator() == A(5));
+int main(int, char**) {
+  typedef std::less<int> C;
+  typedef test_allocator<int> A;
+  std::multiset<int, C, A> m(A(5));
+  assert(m.empty());
+  assert(m.begin() == m.end());
+  assert(m.get_allocator() == A(5));
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/associative/multiset/multiset.cons/assign_initializer_list.pass.cpp b/libcxx/test/std/containers/associative/multiset/multiset.cons/assign_initializer_list.pass.cpp
index 3c39455e4e17a..7234e61d28bfa 100644
--- a/libcxx/test/std/containers/associative/multiset/multiset.cons/assign_initializer_list.pass.cpp
+++ b/libcxx/test/std/containers/associative/multiset/multiset.cons/assign_initializer_list.pass.cpp
@@ -20,13 +20,12 @@
 #include "test_macros.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
+int main(int, char**) {
+  {
     typedef std::multiset<int> C;
     typedef C::value_type V;
     C m = {10, 8};
-    m = {1, 2, 3, 4, 5, 6};
+    m   = {1, 2, 3, 4, 5, 6};
     assert(m.size() == 6);
     assert(std::distance(m.begin(), m.end()) == 6);
     C::const_iterator i = m.cbegin();
@@ -36,12 +35,12 @@ int main(int, char**)
     assert(*++i == V(4));
     assert(*++i == V(5));
     assert(*++i == V(6));
-    }
-    {
+  }
+  {
     typedef std::multiset<int, std::less<int>, min_allocator<int>> C;
     typedef C::value_type V;
     C m = {10, 8};
-    m = {1, 2, 3, 4, 5, 6};
+    m   = {1, 2, 3, 4, 5, 6};
     assert(m.size() == 6);
     assert(std::distance(m.begin(), m.end()) == 6);
     C::const_iterator i = m.cbegin();
@@ -51,7 +50,7 @@ int main(int, char**)
     assert(*++i == V(4));
     assert(*++i == V(5));
     assert(*++i == V(6));
-    }
+  }
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/associative/multiset/multiset.cons/compare.pass.cpp b/libcxx/test/std/containers/associative/multiset/multiset.cons/compare.pass.cpp
index 6debbd6d3d852..b7da7f66b020d 100644
--- a/libcxx/test/std/containers/associative/multiset/multiset.cons/compare.pass.cpp
+++ b/libcxx/test/std/containers/associative/multiset/multiset.cons/compare.pass.cpp
@@ -18,14 +18,13 @@
 #include "test_macros.h"
 #include "../../../test_compare.h"
 
-int main(int, char**)
-{
-    typedef test_less<int> C;
-    const std::multiset<int, C> m(C(3));
-    assert(m.empty());
-    assert(m.begin() == m.end());
-    assert(m.key_comp() == C(3));
-    assert(m.value_comp() == C(3));
+int main(int, char**) {
+  typedef test_less<int> C;
+  const std::multiset<int, C> m(C(3));
+  assert(m.empty());
+  assert(m.begin() == m.end());
+  assert(m.key_comp() == C(3));
+  assert(m.value_comp() == C(3));
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/associative/multiset/multiset.cons/compare_alloc.pass.cpp b/libcxx/test/std/containers/associative/multiset/multiset.cons/compare_alloc.pass.cpp
index 61458e71d90b4..be8a451fdc405 100644
--- a/libcxx/test/std/containers/associative/multiset/multiset.cons/compare_alloc.pass.cpp
+++ b/libcxx/test/std/containers/associative/multiset/multiset.cons/compare_alloc.pass.cpp
@@ -19,15 +19,14 @@
 #include "../../../test_compare.h"
 #include "test_allocator.h"
 
-int main(int, char**)
-{
-    typedef test_less<int> C;
-    typedef test_allocator<int> A;
-    std::multiset<int, C, A> m(C(4), A(5));
-    assert(m.empty());
-    assert(m.begin() == m.end());
-    assert(m.key_comp() == C(4));
-    assert(m.get_allocator() == A(5));
+int main(int, char**) {
+  typedef test_less<int> C;
+  typedef test_allocator<int> A;
+  std::multiset<int, C, A> m(C(4), A(5));
+  assert(m.empty());
+  assert(m.begin() == m.end());
+  assert(m.key_comp() == C(4));
+  assert(m.get_allocator() == A(5));
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/associative/multiset/multiset.cons/compare_copy_constructible.compile.fail.cpp b/libcxx/test/std/containers/associative/multiset/multiset.cons/compare_copy_constructible.compile.fail.cpp
index ae987c64c85ba..020a11ac24626 100644
--- a/libcxx/test/std/containers/associative/multiset/multiset.cons/compare_copy_constructible.compile.fail.cpp
+++ b/libcxx/test/std/containers/associative/multiset/multiset.cons/compare_copy_constructible.compile.fail.cpp
@@ -15,16 +15,16 @@
 
 template <class T>
 struct Comp {
-    bool operator () (const T& lhs, const T& rhs) const { return lhs < rhs; }
+  bool operator()(const T& lhs, const T& rhs) const { return lhs < rhs; }
 
-    Comp () {}
-private:
-    Comp (const Comp &); // declared but not defined
-    };
+  Comp() {}
 
+private:
+  Comp(const Comp&); // declared but not defined
+};
 
 int main(int, char**) {
-    std::multiset<int, Comp<int> > m;
+  std::multiset<int, Comp<int> > m;
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/associative/multiset/multiset.cons/copy.pass.cpp b/libcxx/test/std/containers/associative/multiset/multiset.cons/copy.pass.cpp
index 9d643199e6b9a..8138227128e8a 100644
--- a/libcxx/test/std/containers/associative/multiset/multiset.cons/copy.pass.cpp
+++ b/libcxx/test/std/containers/associative/multiset/multiset.cons/copy.pass.cpp
@@ -20,101 +20,78 @@
 #include "../../../test_compare.h"
 #include "test_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef int V;
-        V ar[] =
-        {
-            1,
-            1,
-            1,
-            2,
-            2,
-            2,
-            3,
-            3,
-            3
-        };
-        typedef test_less<int> C;
-        typedef test_allocator<V> A;
-        std::multiset<int, C, A> mo(ar, ar+sizeof(ar)/sizeof(ar[0]), C(5), A(7));
-        std::multiset<int, C, A> m = mo;
-        assert(m.get_allocator() == A(7));
-        assert(m.key_comp() == C(5));
-        assert(m.size() == 9);
-        assert(std::distance(m.begin(), m.end()) == 9);
-        assert(*std::next(m.begin(), 0) == 1);
-        assert(*std::next(m.begin(), 1) == 1);
-        assert(*std::next(m.begin(), 2) == 1);
-        assert(*std::next(m.begin(), 3) == 2);
-        assert(*std::next(m.begin(), 4) == 2);
-        assert(*std::next(m.begin(), 5) == 2);
-        assert(*std::next(m.begin(), 6) == 3);
-        assert(*std::next(m.begin(), 7) == 3);
-        assert(*std::next(m.begin(), 8) == 3);
+int main(int, char**) {
+  {
+    typedef int V;
+    V ar[] = {1, 1, 1, 2, 2, 2, 3, 3, 3};
+    typedef test_less<int> C;
+    typedef test_allocator<V> A;
+    std::multiset<int, C, A> mo(ar, ar + sizeof(ar) / sizeof(ar[0]), C(5), A(7));
+    std::multiset<int, C, A> m = mo;
+    assert(m.get_allocator() == A(7));
+    assert(m.key_comp() == C(5));
+    assert(m.size() == 9);
+    assert(std::distance(m.begin(), m.end()) == 9);
+    assert(*std::next(m.begin(), 0) == 1);
+    assert(*std::next(m.begin(), 1) == 1);
+    assert(*std::next(m.begin(), 2) == 1);
+    assert(*std::next(m.begin(), 3) == 2);
+    assert(*std::next(m.begin(), 4) == 2);
+    assert(*std::next(m.begin(), 5) == 2);
+    assert(*std::next(m.begin(), 6) == 3);
+    assert(*std::next(m.begin(), 7) == 3);
+    assert(*std::next(m.begin(), 8) == 3);
 
-        assert(mo.get_allocator() == A(7));
-        assert(mo.key_comp() == C(5));
-        assert(mo.size() == 9);
-        assert(std::distance(mo.begin(), mo.end()) == 9);
-        assert(*std::next(mo.begin(), 0) == 1);
-        assert(*std::next(mo.begin(), 1) == 1);
-        assert(*std::next(mo.begin(), 2) == 1);
-        assert(*std::next(mo.begin(), 3) == 2);
-        assert(*std::next(mo.begin(), 4) == 2);
-        assert(*std::next(mo.begin(), 5) == 2);
-        assert(*std::next(mo.begin(), 6) == 3);
-        assert(*std::next(mo.begin(), 7) == 3);
-        assert(*std::next(mo.begin(), 8) == 3);
-    }
+    assert(mo.get_allocator() == A(7));
+    assert(mo.key_comp() == C(5));
+    assert(mo.size() == 9);
+    assert(std::distance(mo.begin(), mo.end()) == 9);
+    assert(*std::next(mo.begin(), 0) == 1);
+    assert(*std::next(mo.begin(), 1) == 1);
+    assert(*std::next(mo.begin(), 2) == 1);
+    assert(*std::next(mo.begin(), 3) == 2);
+    assert(*std::next(mo.begin(), 4) == 2);
+    assert(*std::next(mo.begin(), 5) == 2);
+    assert(*std::next(mo.begin(), 6) == 3);
+    assert(*std::next(mo.begin(), 7) == 3);
+    assert(*std::next(mo.begin(), 8) == 3);
+  }
 #if TEST_STD_VER >= 11
-    {
-        typedef int V;
-        V ar[] =
-        {
-            1,
-            1,
-            1,
-            2,
-            2,
-            2,
-            3,
-            3,
-            3
-        };
-        typedef test_less<int> C;
-        typedef other_allocator<V> A;
-        std::multiset<int, C, A> mo(ar, ar+sizeof(ar)/sizeof(ar[0]), C(5), A(7));
-        std::multiset<int, C, A> m = mo;
-        assert(m.get_allocator() == A(-2));
-        assert(m.key_comp() == C(5));
-        assert(m.size() == 9);
-        assert(std::distance(m.begin(), m.end()) == 9);
-        assert(*std::next(m.begin(), 0) == 1);
-        assert(*std::next(m.begin(), 1) == 1);
-        assert(*std::next(m.begin(), 2) == 1);
-        assert(*std::next(m.begin(), 3) == 2);
-        assert(*std::next(m.begin(), 4) == 2);
-        assert(*std::next(m.begin(), 5) == 2);
-        assert(*std::next(m.begin(), 6) == 3);
-        assert(*std::next(m.begin(), 7) == 3);
-        assert(*std::next(m.begin(), 8) == 3);
+  {
+    typedef int V;
+    V ar[] = {1, 1, 1, 2, 2, 2, 3, 3, 3};
+    typedef test_less<int> C;
+    typedef other_allocator<V> A;
+    std::multiset<int, C, A> mo(ar, ar + sizeof(ar) / sizeof(ar[0]), C(5), A(7));
+    std::multiset<int, C, A> m = mo;
+    assert(m.get_allocator() == A(-2));
+    assert(m.key_comp() == C(5));
+    assert(m.size() == 9);
+    assert(std::distance(m.begin(), m.end()) == 9);
+    assert(*std::next(m.begin(), 0) == 1);
+    assert(*std::next(m.begin(), 1) == 1);
+    assert(*std::next(m.begin(), 2) == 1);
+    assert(*std::next(m.begin(), 3) == 2);
+    assert(*std::next(m.begin(), 4) == 2);
+    assert(*std::next(m.begin(), 5) == 2);
+    assert(*std::next(m.begin(), 6) == 3);
+    assert(*std::next(m.begin(), 7) == 3);
+    assert(*std::next(m.begin(), 8) == 3);
 
-        assert(mo.get_allocator() == A(7));
-        assert(mo.key_comp() == C(5));
-        assert(mo.size() == 9);
-        assert(std::distance(mo.begin(), mo.end()) == 9);
-        assert(*std::next(mo.begin(), 0) == 1);
-        assert(*std::next(mo.begin(), 1) == 1);
-        assert(*std::next(mo.begin(), 2) == 1);
-        assert(*std::next(mo.begin(), 3) == 2);
-        assert(*std::next(mo.begin(), 4) == 2);
-        assert(*std::next(mo.begin(), 5) == 2);
-        assert(*std::next(mo.begin(), 6) == 3);
-        assert(*std::next(mo.begin(), 7) == 3);
-        assert(*std::next(mo.begin(), 8) == 3);
-    }
+    assert(mo.get_allocator() == A(7));
+    assert(mo.key_comp() == C(5));
+    assert(mo.size() == 9);
+    assert(std::distance(mo.begin(), mo.end()) == 9);
+    assert(*std::next(mo.begin(), 0) == 1);
+    assert(*std::next(mo.begin(), 1) == 1);
+    assert(*std::next(mo.begin(), 2) == 1);
+    assert(*std::next(mo.begin(), 3) == 2);
+    assert(*std::next(mo.begin(), 4) == 2);
+    assert(*std::next(mo.begin(), 5) == 2);
+    assert(*std::next(mo.begin(), 6) == 3);
+    assert(*std::next(mo.begin(), 7) == 3);
+    assert(*std::next(mo.begin(), 8) == 3);
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/associative/multiset/multiset.cons/copy_alloc.pass.cpp b/libcxx/test/std/containers/associative/multiset/multiset.cons/copy_alloc.pass.cpp
index ece846c63ec8e..7570673066829 100644
--- a/libcxx/test/std/containers/associative/multiset/multiset.cons/copy_alloc.pass.cpp
+++ b/libcxx/test/std/containers/associative/multiset/multiset.cons/copy_alloc.pass.cpp
@@ -20,52 +20,40 @@
 #include "../../../test_compare.h"
 #include "test_allocator.h"
 
-int main(int, char**)
-{
-    typedef int V;
-    V ar[] =
-    {
-        1,
-        1,
-        1,
-        2,
-        2,
-        2,
-        3,
-        3,
-        3
-    };
-    typedef test_less<int> C;
-    typedef test_allocator<V> A;
-    std::multiset<int, C, A> mo(ar, ar+sizeof(ar)/sizeof(ar[0]), C(5), A(7));
-    std::multiset<int, C, A> m(mo, A(3));
-    assert(m.get_allocator() == A(3));
-    assert(m.key_comp() == C(5));
-    assert(m.size() == 9);
-    assert(std::distance(m.begin(), m.end()) == 9);
-    assert(*std::next(m.begin(), 0) == 1);
-    assert(*std::next(m.begin(), 1) == 1);
-    assert(*std::next(m.begin(), 2) == 1);
-    assert(*std::next(m.begin(), 3) == 2);
-    assert(*std::next(m.begin(), 4) == 2);
-    assert(*std::next(m.begin(), 5) == 2);
-    assert(*std::next(m.begin(), 6) == 3);
-    assert(*std::next(m.begin(), 7) == 3);
-    assert(*std::next(m.begin(), 8) == 3);
-
-    assert(mo.get_allocator() == A(7));
-    assert(mo.key_comp() == C(5));
-    assert(mo.size() == 9);
-    assert(std::distance(mo.begin(), mo.end()) == 9);
-    assert(*std::next(mo.begin(), 0) == 1);
-    assert(*std::next(mo.begin(), 1) == 1);
-    assert(*std::next(mo.begin(), 2) == 1);
-    assert(*std::next(mo.begin(), 3) == 2);
-    assert(*std::next(mo.begin(), 4) == 2);
-    assert(*std::next(mo.begin(), 5) == 2);
-    assert(*std::next(mo.begin(), 6) == 3);
-    assert(*std::next(mo.begin(), 7) == 3);
-    assert(*std::next(mo.begin(), 8) == 3);
+int main(int, char**) {
+  typedef int V;
+  V ar[] = {1, 1, 1, 2, 2, 2, 3, 3, 3};
+  typedef test_less<int> C;
+  typedef test_allocator<V> A;
+  std::multiset<int, C, A> mo(ar, ar + sizeof(ar) / sizeof(ar[0]), C(5), A(7));
+  std::multiset<int, C, A> m(mo, A(3));
+  assert(m.get_allocator() == A(3));
+  assert(m.key_comp() == C(5));
+  assert(m.size() == 9);
+  assert(std::distance(m.begin(), m.end()) == 9);
+  assert(*std::next(m.begin(), 0) == 1);
+  assert(*std::next(m.begin(), 1) == 1);
+  assert(*std::next(m.begin(), 2) == 1);
+  assert(*std::next(m.begin(), 3) == 2);
+  assert(*std::next(m.begin(), 4) == 2);
+  assert(*std::next(m.begin(), 5) == 2);
+  assert(*std::next(m.begin(), 6) == 3);
+  assert(*std::next(m.begin(), 7) == 3);
+  assert(*std::next(m.begin(), 8) == 3);
+
+  assert(mo.get_allocator() == A(7));
+  assert(mo.key_comp() == C(5));
+  assert(mo.size() == 9);
+  assert(std::distance(mo.begin(), mo.end()) == 9);
+  assert(*std::next(mo.begin(), 0) == 1);
+  assert(*std::next(mo.begin(), 1) == 1);
+  assert(*std::next(mo.begin(), 2) == 1);
+  assert(*std::next(mo.begin(), 3) == 2);
+  assert(*std::next(mo.begin(), 4) == 2);
+  assert(*std::next(mo.begin(), 5) == 2);
+  assert(*std::next(mo.begin(), 6) == 3);
+  assert(*std::next(mo.begin(), 7) == 3);
+  assert(*std::next(mo.begin(), 8) == 3);
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/associative/multiset/multiset.cons/copy_assign.pass.cpp b/libcxx/test/std/containers/associative/multiset/multiset.cons/copy_assign.pass.cpp
index 6835020a98223..d6b394ff66698 100644
--- a/libcxx/test/std/containers/associative/multiset/multiset.cons/copy_assign.pass.cpp
+++ b/libcxx/test/std/containers/associative/multiset/multiset.cons/copy_assign.pass.cpp
@@ -20,122 +20,88 @@
 #include "../../../test_compare.h"
 #include "test_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef int V;
-        V ar[] =
-        {
-            1,
-            1,
-            1,
-            2,
-            2,
-            2,
-            3,
-            3,
-            3
-        };
-        typedef test_less<int> C;
-        typedef test_allocator<V> A;
-        std::multiset<int, C, A> mo(ar, ar+sizeof(ar)/sizeof(ar[0]), C(5), A(2));
-        std::multiset<int, C, A> m(ar, ar+sizeof(ar)/sizeof(ar[0])/2, C(3), A(7));
-        m = mo;
-        assert(m.get_allocator() == A(7));
-        assert(m.key_comp() == C(5));
-        assert(m.size() == 9);
-        assert(std::distance(m.begin(), m.end()) == 9);
-        assert(*std::next(m.begin(), 0) == 1);
-        assert(*std::next(m.begin(), 1) == 1);
-        assert(*std::next(m.begin(), 2) == 1);
-        assert(*std::next(m.begin(), 3) == 2);
-        assert(*std::next(m.begin(), 4) == 2);
-        assert(*std::next(m.begin(), 5) == 2);
-        assert(*std::next(m.begin(), 6) == 3);
-        assert(*std::next(m.begin(), 7) == 3);
-        assert(*std::next(m.begin(), 8) == 3);
+int main(int, char**) {
+  {
+    typedef int V;
+    V ar[] = {1, 1, 1, 2, 2, 2, 3, 3, 3};
+    typedef test_less<int> C;
+    typedef test_allocator<V> A;
+    std::multiset<int, C, A> mo(ar, ar + sizeof(ar) / sizeof(ar[0]), C(5), A(2));
+    std::multiset<int, C, A> m(ar, ar + sizeof(ar) / sizeof(ar[0]) / 2, C(3), A(7));
+    m = mo;
+    assert(m.get_allocator() == A(7));
+    assert(m.key_comp() == C(5));
+    assert(m.size() == 9);
+    assert(std::distance(m.begin(), m.end()) == 9);
+    assert(*std::next(m.begin(), 0) == 1);
+    assert(*std::next(m.begin(), 1) == 1);
+    assert(*std::next(m.begin(), 2) == 1);
+    assert(*std::next(m.begin(), 3) == 2);
+    assert(*std::next(m.begin(), 4) == 2);
+    assert(*std::next(m.begin(), 5) == 2);
+    assert(*std::next(m.begin(), 6) == 3);
+    assert(*std::next(m.begin(), 7) == 3);
+    assert(*std::next(m.begin(), 8) == 3);
 
-        assert(mo.get_allocator() == A(2));
-        assert(mo.key_comp() == C(5));
-        assert(mo.size() == 9);
-        assert(std::distance(mo.begin(), mo.end()) == 9);
-        assert(*std::next(mo.begin(), 0) == 1);
-        assert(*std::next(mo.begin(), 1) == 1);
-        assert(*std::next(mo.begin(), 2) == 1);
-        assert(*std::next(mo.begin(), 3) == 2);
-        assert(*std::next(mo.begin(), 4) == 2);
-        assert(*std::next(mo.begin(), 5) == 2);
-        assert(*std::next(mo.begin(), 6) == 3);
-        assert(*std::next(mo.begin(), 7) == 3);
-        assert(*std::next(mo.begin(), 8) == 3);
-    }
-    {
-        typedef int V;
-        const V ar[] =
-        {
-            1,
-            1,
-            1,
-            2,
-            2,
-            2,
-            3,
-            3,
-            3
-        };
-        std::multiset<int> m(ar, ar+sizeof(ar)/sizeof(ar[0]));
-        std::multiset<int> *p = &m;
-        m = *p;
-        assert(m.size() == 9);
-        assert(std::equal(m.begin(), m.end(), ar));
-    }
-    {
-        typedef int V;
-        V ar[] =
-        {
-            1,
-            1,
-            1,
-            2,
-            2,
-            2,
-            3,
-            3,
-            3
-        };
-        typedef test_less<int> C;
-        typedef other_allocator<V> A;
-        std::multiset<int, C, A> mo(ar, ar+sizeof(ar)/sizeof(ar[0]), C(5), A(2));
-        std::multiset<int, C, A> m(ar, ar+sizeof(ar)/sizeof(ar[0])/2, C(3), A(7));
-        m = mo;
-        assert(m.get_allocator() == A(2));
-        assert(m.key_comp() == C(5));
-        assert(m.size() == 9);
-        assert(std::distance(m.begin(), m.end()) == 9);
-        assert(*std::next(m.begin(), 0) == 1);
-        assert(*std::next(m.begin(), 1) == 1);
-        assert(*std::next(m.begin(), 2) == 1);
-        assert(*std::next(m.begin(), 3) == 2);
-        assert(*std::next(m.begin(), 4) == 2);
-        assert(*std::next(m.begin(), 5) == 2);
-        assert(*std::next(m.begin(), 6) == 3);
-        assert(*std::next(m.begin(), 7) == 3);
-        assert(*std::next(m.begin(), 8) == 3);
+    assert(mo.get_allocator() == A(2));
+    assert(mo.key_comp() == C(5));
+    assert(mo.size() == 9);
+    assert(std::distance(mo.begin(), mo.end()) == 9);
+    assert(*std::next(mo.begin(), 0) == 1);
+    assert(*std::next(mo.begin(), 1) == 1);
+    assert(*std::next(mo.begin(), 2) == 1);
+    assert(*std::next(mo.begin(), 3) == 2);
+    assert(*std::next(mo.begin(), 4) == 2);
+    assert(*std::next(mo.begin(), 5) == 2);
+    assert(*std::next(mo.begin(), 6) == 3);
+    assert(*std::next(mo.begin(), 7) == 3);
+    assert(*std::next(mo.begin(), 8) == 3);
+  }
+  {
+    typedef int V;
+    const V ar[] = {1, 1, 1, 2, 2, 2, 3, 3, 3};
+    std::multiset<int> m(ar, ar + sizeof(ar) / sizeof(ar[0]));
+    std::multiset<int>* p = &m;
+    m                     = *p;
+    assert(m.size() == 9);
+    assert(std::equal(m.begin(), m.end(), ar));
+  }
+  {
+    typedef int V;
+    V ar[] = {1, 1, 1, 2, 2, 2, 3, 3, 3};
+    typedef test_less<int> C;
+    typedef other_allocator<V> A;
+    std::multiset<int, C, A> mo(ar, ar + sizeof(ar) / sizeof(ar[0]), C(5), A(2));
+    std::multiset<int, C, A> m(ar, ar + sizeof(ar) / sizeof(ar[0]) / 2, C(3), A(7));
+    m = mo;
+    assert(m.get_allocator() == A(2));
+    assert(m.key_comp() == C(5));
+    assert(m.size() == 9);
+    assert(std::distance(m.begin(), m.end()) == 9);
+    assert(*std::next(m.begin(), 0) == 1);
+    assert(*std::next(m.begin(), 1) == 1);
+    assert(*std::next(m.begin(), 2) == 1);
+    assert(*std::next(m.begin(), 3) == 2);
+    assert(*std::next(m.begin(), 4) == 2);
+    assert(*std::next(m.begin(), 5) == 2);
+    assert(*std::next(m.begin(), 6) == 3);
+    assert(*std::next(m.begin(), 7) == 3);
+    assert(*std::next(m.begin(), 8) == 3);
 
-        assert(mo.get_allocator() == A(2));
-        assert(mo.key_comp() == C(5));
-        assert(mo.size() == 9);
-        assert(std::distance(mo.begin(), mo.end()) == 9);
-        assert(*std::next(mo.begin(), 0) == 1);
-        assert(*std::next(mo.begin(), 1) == 1);
-        assert(*std::next(mo.begin(), 2) == 1);
-        assert(*std::next(mo.begin(), 3) == 2);
-        assert(*std::next(mo.begin(), 4) == 2);
-        assert(*std::next(mo.begin(), 5) == 2);
-        assert(*std::next(mo.begin(), 6) == 3);
-        assert(*std::next(mo.begin(), 7) == 3);
-        assert(*std::next(mo.begin(), 8) == 3);
-    }
+    assert(mo.get_allocator() == A(2));
+    assert(mo.key_comp() == C(5));
+    assert(mo.size() == 9);
+    assert(std::distance(mo.begin(), mo.end()) == 9);
+    assert(*std::next(mo.begin(), 0) == 1);
+    assert(*std::next(mo.begin(), 1) == 1);
+    assert(*std::next(mo.begin(), 2) == 1);
+    assert(*std::next(mo.begin(), 3) == 2);
+    assert(*std::next(mo.begin(), 4) == 2);
+    assert(*std::next(mo.begin(), 5) == 2);
+    assert(*std::next(mo.begin(), 6) == 3);
+    assert(*std::next(mo.begin(), 7) == 3);
+    assert(*std::next(mo.begin(), 8) == 3);
+  }
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/associative/multiset/multiset.cons/deduct.pass.cpp b/libcxx/test/std/containers/associative/multiset/multiset.cons/deduct.pass.cpp
index 93bff1a41c667..f9797ca571c47 100644
--- a/libcxx/test/std/containers/associative/multiset/multiset.cons/deduct.pass.cpp
+++ b/libcxx/test/std/containers/associative/multiset/multiset.cons/deduct.pass.cpp
@@ -52,38 +52,32 @@ struct NotAnAllocator {
   friend bool operator<(NotAnAllocator, NotAnAllocator) { return false; }
 };
 
-int main(int, char **) {
+int main(int, char**) {
   {
-    const int arr[] = { 1, 2, 1, INT_MAX, 3 };
+    const int arr[] = {1, 2, 1, INT_MAX, 3};
     std::multiset s(std::begin(arr), std::end(arr));
 
     ASSERT_SAME_TYPE(decltype(s), std::multiset<int>);
-    const int expected_s[] = { 1, 1, 2, 3, INT_MAX };
-    assert(std::equal(s.begin(), s.end(), std::begin(expected_s),
-                      std::end(expected_s)));
+    const int expected_s[] = {1, 1, 2, 3, INT_MAX};
+    assert(std::equal(s.begin(), s.end(), std::begin(expected_s), std::end(expected_s)));
   }
 
   {
-    const int arr[] = { 1, 2, 1, INT_MAX, 3 };
+    const int arr[] = {1, 2, 1, INT_MAX, 3};
     std::multiset s(std::begin(arr), std::end(arr), std::greater<int>());
 
     ASSERT_SAME_TYPE(decltype(s), std::multiset<int, std::greater<int> >);
-    const int expected_s[] = { INT_MAX, 3, 2, 1, 1 };
-    assert(std::equal(s.begin(), s.end(), std::begin(expected_s),
-                      std::end(expected_s)));
+    const int expected_s[] = {INT_MAX, 3, 2, 1, 1};
+    assert(std::equal(s.begin(), s.end(), std::begin(expected_s), std::end(expected_s)));
   }
 
   {
-    const int arr[] = { 1, 2, 1, INT_MAX, 3 };
-    std::multiset s(std::begin(arr), std::end(arr), std::greater<int>(),
-                    test_allocator<int>(0, 42));
+    const int arr[] = {1, 2, 1, INT_MAX, 3};
+    std::multiset s(std::begin(arr), std::end(arr), std::greater<int>(), test_allocator<int>(0, 42));
 
-    ASSERT_SAME_TYPE(
-        decltype(s),
-        std::multiset<int, std::greater<int>, test_allocator<int> >);
-    const int expected_s[] = { INT_MAX, 3, 2, 1, 1 };
-    assert(std::equal(s.begin(), s.end(), std::begin(expected_s),
-                      std::end(expected_s)));
+    ASSERT_SAME_TYPE(decltype(s), std::multiset<int, std::greater<int>, test_allocator<int> >);
+    const int expected_s[] = {INT_MAX, 3, 2, 1, 1};
+    assert(std::equal(s.begin(), s.end(), std::begin(expected_s), std::end(expected_s)));
     assert(s.get_allocator().get_id() == 42);
   }
 
@@ -96,7 +90,7 @@ int main(int, char **) {
 
   {
     std::multiset<long> source;
-    std::multiset s{ source };  // braces instead of parens
+    std::multiset s{source}; // braces instead of parens
     ASSERT_SAME_TYPE(decltype(s), std::multiset<long>);
     assert(s.size() == 0);
   }
@@ -109,121 +103,111 @@ int main(int, char **) {
   }
 
   {
-    std::multiset s{ 1, 2, 1, INT_MAX, 3 };
+    std::multiset s{1, 2, 1, INT_MAX, 3};
 
     ASSERT_SAME_TYPE(decltype(s), std::multiset<int>);
-    const int expected_s[] = { 1, 1, 2, 3, INT_MAX };
-    assert(std::equal(s.begin(), s.end(), std::begin(expected_s),
-                      std::end(expected_s)));
+    const int expected_s[] = {1, 1, 2, 3, INT_MAX};
+    assert(std::equal(s.begin(), s.end(), std::begin(expected_s), std::end(expected_s)));
   }
 
   {
-    std::multiset s({ 1, 2, 1, INT_MAX, 3 }, std::greater<int>());
+    std::multiset s({1, 2, 1, INT_MAX, 3}, std::greater<int>());
 
     ASSERT_SAME_TYPE(decltype(s), std::multiset<int, std::greater<int> >);
-    const int expected_s[] = { INT_MAX, 3, 2, 1, 1 };
-    assert(std::equal(s.begin(), s.end(), std::begin(expected_s),
-                      std::end(expected_s)));
+    const int expected_s[] = {INT_MAX, 3, 2, 1, 1};
+    assert(std::equal(s.begin(), s.end(), std::begin(expected_s), std::end(expected_s)));
   }
 
   {
-    std::multiset s({ 1, 2, 1, INT_MAX, 3 }, std::greater<int>(),
-                    test_allocator<int>(0, 43));
+    std::multiset s({1, 2, 1, INT_MAX, 3}, std::greater<int>(), test_allocator<int>(0, 43));
 
-    ASSERT_SAME_TYPE(
-        decltype(s),
-        std::multiset<int, std::greater<int>, test_allocator<int> >);
-    const int expected_s[] = { INT_MAX, 3, 2, 1, 1 };
-    assert(std::equal(s.begin(), s.end(), std::begin(expected_s),
-                      std::end(expected_s)));
+    ASSERT_SAME_TYPE(decltype(s), std::multiset<int, std::greater<int>, test_allocator<int> >);
+    const int expected_s[] = {INT_MAX, 3, 2, 1, 1};
+    assert(std::equal(s.begin(), s.end(), std::begin(expected_s), std::end(expected_s)));
     assert(s.get_allocator().get_id() == 43);
   }
 
   {
-    const int arr[] = { 1, 2, 1, INT_MAX, 3 };
+    const int arr[] = {1, 2, 1, INT_MAX, 3};
     std::multiset s(std::begin(arr), std::end(arr), test_allocator<int>(0, 44));
 
-    ASSERT_SAME_TYPE(decltype(s),
-                     std::multiset<int, std::less<int>, test_allocator<int> >);
-    const int expected_s[] = { 1, 1, 2, 3, INT_MAX };
-    assert(std::equal(s.begin(), s.end(), std::begin(expected_s),
-                      std::end(expected_s)));
+    ASSERT_SAME_TYPE(decltype(s), std::multiset<int, std::less<int>, test_allocator<int> >);
+    const int expected_s[] = {1, 1, 2, 3, INT_MAX};
+    assert(std::equal(s.begin(), s.end(), std::begin(expected_s), std::end(expected_s)));
     assert(s.get_allocator().get_id() == 44);
   }
 
   {
-    std::multiset s({ 1, 2, 1, INT_MAX, 3 }, test_allocator<int>(0, 45));
+    std::multiset s({1, 2, 1, INT_MAX, 3}, test_allocator<int>(0, 45));
 
-    ASSERT_SAME_TYPE(decltype(s),
-                     std::multiset<int, std::less<int>, test_allocator<int> >);
-    const int expected_s[] = { 1, 1, 2, 3, INT_MAX };
-    assert(std::equal(s.begin(), s.end(), std::begin(expected_s),
-                      std::end(expected_s)));
+    ASSERT_SAME_TYPE(decltype(s), std::multiset<int, std::less<int>, test_allocator<int> >);
+    const int expected_s[] = {1, 1, 2, 3, INT_MAX};
+    assert(std::equal(s.begin(), s.end(), std::begin(expected_s), std::end(expected_s)));
     assert(s.get_allocator().get_id() == 45);
   }
 
   {
     NotAnAllocator a;
-    std::multiset s{ a }; // multiset(initializer_list<NotAnAllocator>)
+    std::multiset s{a}; // multiset(initializer_list<NotAnAllocator>)
     ASSERT_SAME_TYPE(decltype(s), std::multiset<NotAnAllocator>);
     assert(s.size() == 1);
   }
 
   {
     std::multiset<long> source;
-    std::multiset s{ source, source }; // multiset(initializer_list<multiset<long>>)
+    std::multiset s{source, source}; // multiset(initializer_list<multiset<long>>)
     ASSERT_SAME_TYPE(decltype(s), std::multiset<std::multiset<long> >);
     assert(s.size() == 2);
   }
 
   {
     NotAnAllocator a;
-    std::multiset s{ a, a }; // multiset(initializer_list<NotAnAllocator>)
+    std::multiset s{a, a}; // multiset(initializer_list<NotAnAllocator>)
     ASSERT_SAME_TYPE(decltype(s), std::multiset<NotAnAllocator>);
     assert(s.size() == 2);
   }
 
   {
-    int source[3] = { 3, 4, 5 };
+    int source[3] = {3, 4, 5};
     std::multiset s(source, source + 3); // multiset(InputIterator, InputIterator)
     ASSERT_SAME_TYPE(decltype(s), std::multiset<int>);
     assert(s.size() == 3);
   }
 
   {
-    int source[3] = { 3, 4, 5 };
-    std::multiset s{ source, source + 3 }; // multiset(initializer_list<int*>)
-    ASSERT_SAME_TYPE(decltype(s), std::multiset<int *>);
+    int source[3] = {3, 4, 5};
+    std::multiset s{source, source + 3}; // multiset(initializer_list<int*>)
+    ASSERT_SAME_TYPE(decltype(s), std::multiset<int*>);
     assert(s.size() == 2);
   }
 
 #if TEST_STD_VER >= 23
-    {
-      using Range = std::array<int, 0>;
-      using Comp = std::greater<int>;
-      using DefaultComp = std::less<int>;
-      using Alloc = test_allocator<int>;
-
-      { // (from_range, range)
-        std::multiset c(std::from_range, Range());
-        static_assert(std::is_same_v<decltype(c), std::multiset<int>>);
-      }
-
-      { // (from_range, range, comp)
-        std::multiset c(std::from_range, Range(), Comp());
-        static_assert(std::is_same_v<decltype(c), std::multiset<int, Comp>>);
-      }
-
-      { // (from_range, range, comp, alloc)
-        std::multiset c(std::from_range, Range(), Comp(), Alloc());
-        static_assert(std::is_same_v<decltype(c), std::multiset<int, Comp, Alloc>>);
-      }
-
-      { // (from_range, range, alloc)
-        std::multiset c(std::from_range, Range(), Alloc());
-        static_assert(std::is_same_v<decltype(c), std::multiset<int, DefaultComp, Alloc>>);
-      }
+  {
+    using Range       = std::array<int, 0>;
+    using Comp        = std::greater<int>;
+    using DefaultComp = std::less<int>;
+    using Alloc       = test_allocator<int>;
+
+    { // (from_range, range)
+      std::multiset c(std::from_range, Range());
+      static_assert(std::is_same_v<decltype(c), std::multiset<int>>);
     }
+
+    { // (from_range, range, comp)
+      std::multiset c(std::from_range, Range(), Comp());
+      static_assert(std::is_same_v<decltype(c), std::multiset<int, Comp>>);
+    }
+
+    { // (from_range, range, comp, alloc)
+      std::multiset c(std::from_range, Range(), Comp(), Alloc());
+      static_assert(std::is_same_v<decltype(c), std::multiset<int, Comp, Alloc>>);
+    }
+
+    { // (from_range, range, alloc)
+      std::multiset c(std::from_range, Range(), Alloc());
+      static_assert(std::is_same_v<decltype(c), std::multiset<int, DefaultComp, Alloc>>);
+    }
+  }
 #endif
 
   AssociativeContainerDeductionGuidesSfinaeAway<std::multiset, std::multiset<int>>();
diff --git a/libcxx/test/std/containers/associative/multiset/multiset.cons/deduct.verify.cpp b/libcxx/test/std/containers/associative/multiset/multiset.cons/deduct.verify.cpp
index 30dd08b048155..6c7484078187c 100644
--- a/libcxx/test/std/containers/associative/multiset/multiset.cons/deduct.verify.cpp
+++ b/libcxx/test/std/containers/associative/multiset/multiset.cons/deduct.verify.cpp
@@ -36,7 +36,7 @@ struct NotAnAllocator {
   friend bool operator<(NotAnAllocator, NotAnAllocator) { return false; }
 };
 
-int main(int, char **) {
+int main(int, char**) {
   {
     // cannot deduce Key from nothing
     std::multiset s;
diff --git a/libcxx/test/std/containers/associative/multiset/multiset.cons/default.pass.cpp b/libcxx/test/std/containers/associative/multiset/multiset.cons/default.pass.cpp
index 793c0f5c7ef9f..4f7dabfea2c68 100644
--- a/libcxx/test/std/containers/associative/multiset/multiset.cons/default.pass.cpp
+++ b/libcxx/test/std/containers/associative/multiset/multiset.cons/default.pass.cpp
@@ -18,38 +18,37 @@
 #include "test_macros.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
+int main(int, char**) {
+  {
     std::multiset<int> m;
     assert(m.empty());
     assert(m.begin() == m.end());
-    }
+  }
 #if TEST_STD_VER >= 11
-    {
+  {
     std::multiset<int, std::less<int>, min_allocator<int>> m;
     assert(m.empty());
     assert(m.begin() == m.end());
-    }
-    {
+  }
+  {
     typedef explicit_allocator<int> A;
-        {
-        std::multiset<int, std::less<int>, A> m;
-        assert(m.empty());
-        assert(m.begin() == m.end());
-        }
-        {
-        A a;
-        std::multiset<int, std::less<int>, A> m(a);
-        assert(m.empty());
-        assert(m.begin() == m.end());
-        }
+    {
+      std::multiset<int, std::less<int>, A> m;
+      assert(m.empty());
+      assert(m.begin() == m.end());
     }
     {
+      A a;
+      std::multiset<int, std::less<int>, A> m(a);
+      assert(m.empty());
+      assert(m.begin() == m.end());
+    }
+  }
+  {
     std::multiset<int> m = {};
     assert(m.empty());
     assert(m.begin() == m.end());
-    }
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/associative/multiset/multiset.cons/default_noexcept.pass.cpp b/libcxx/test/std/containers/associative/multiset/multiset.cons/default_noexcept.pass.cpp
index b6d1cb267846f..052b032d6e798 100644
--- a/libcxx/test/std/containers/associative/multiset/multiset.cons/default_noexcept.pass.cpp
+++ b/libcxx/test/std/containers/associative/multiset/multiset.cons/default_noexcept.pass.cpp
@@ -26,33 +26,31 @@
 #include "test_allocator.h"
 
 template <class T>
-struct some_comp
-{
-    typedef T value_type;
-    some_comp();
-    bool operator()(const T&, const T&) const { return false; }
+struct some_comp {
+  typedef T value_type;
+  some_comp();
+  bool operator()(const T&, const T&) const { return false; }
 };
 
-int main(int, char**)
-{
+int main(int, char**) {
 #if defined(_LIBCPP_VERSION)
-    {
-        typedef std::multiset<MoveOnly> C;
-        static_assert(std::is_nothrow_default_constructible<C>::value, "");
-    }
-    {
-        typedef std::multiset<MoveOnly, std::less<MoveOnly>, test_allocator<MoveOnly>> C;
-        static_assert(std::is_nothrow_default_constructible<C>::value, "");
-    }
+  {
+    typedef std::multiset<MoveOnly> C;
+    static_assert(std::is_nothrow_default_constructible<C>::value, "");
+  }
+  {
+    typedef std::multiset<MoveOnly, std::less<MoveOnly>, test_allocator<MoveOnly>> C;
+    static_assert(std::is_nothrow_default_constructible<C>::value, "");
+  }
 #endif // _LIBCPP_VERSION
-    {
-        typedef std::multiset<MoveOnly, std::less<MoveOnly>, other_allocator<MoveOnly>> C;
-        static_assert(!std::is_nothrow_default_constructible<C>::value, "");
-    }
-    {
-        typedef std::multiset<MoveOnly, some_comp<MoveOnly>> C;
-        static_assert(!std::is_nothrow_default_constructible<C>::value, "");
-    }
+  {
+    typedef std::multiset<MoveOnly, std::less<MoveOnly>, other_allocator<MoveOnly>> C;
+    static_assert(!std::is_nothrow_default_constructible<C>::value, "");
+  }
+  {
+    typedef std::multiset<MoveOnly, some_comp<MoveOnly>> C;
+    static_assert(!std::is_nothrow_default_constructible<C>::value, "");
+  }
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/associative/multiset/multiset.cons/dtor_noexcept.pass.cpp b/libcxx/test/std/containers/associative/multiset/multiset.cons/dtor_noexcept.pass.cpp
index a9f7a648773e7..987eca0706076 100644
--- a/libcxx/test/std/containers/associative/multiset/multiset.cons/dtor_noexcept.pass.cpp
+++ b/libcxx/test/std/containers/associative/multiset/multiset.cons/dtor_noexcept.pass.cpp
@@ -20,32 +20,30 @@
 #include "test_allocator.h"
 
 template <class T>
-struct some_comp
-{
-    typedef T value_type;
-    ~some_comp() noexcept(false);
-    bool operator()(const T&, const T&) const { return false; }
+struct some_comp {
+  typedef T value_type;
+  ~some_comp() noexcept(false);
+  bool operator()(const T&, const T&) const { return false; }
 };
 
-int main(int, char**)
-{
-    {
-        typedef std::multiset<MoveOnly> C;
-        static_assert(std::is_nothrow_destructible<C>::value, "");
-    }
-    {
-        typedef std::multiset<MoveOnly, std::less<MoveOnly>, test_allocator<MoveOnly>> C;
-        static_assert(std::is_nothrow_destructible<C>::value, "");
-    }
-    {
-        typedef std::multiset<MoveOnly, std::less<MoveOnly>, other_allocator<MoveOnly>> C;
-        static_assert(std::is_nothrow_destructible<C>::value, "");
-    }
+int main(int, char**) {
+  {
+    typedef std::multiset<MoveOnly> C;
+    static_assert(std::is_nothrow_destructible<C>::value, "");
+  }
+  {
+    typedef std::multiset<MoveOnly, std::less<MoveOnly>, test_allocator<MoveOnly>> C;
+    static_assert(std::is_nothrow_destructible<C>::value, "");
+  }
+  {
+    typedef std::multiset<MoveOnly, std::less<MoveOnly>, other_allocator<MoveOnly>> C;
+    static_assert(std::is_nothrow_destructible<C>::value, "");
+  }
 #if defined(_LIBCPP_VERSION)
-    {
-        typedef std::multiset<MoveOnly, some_comp<MoveOnly>> C;
-        static_assert(!std::is_nothrow_destructible<C>::value, "");
-    }
+  {
+    typedef std::multiset<MoveOnly, some_comp<MoveOnly>> C;
+    static_assert(!std::is_nothrow_destructible<C>::value, "");
+  }
 #endif // _LIBCPP_VERSION
 
   return 0;
diff --git a/libcxx/test/std/containers/associative/multiset/multiset.cons/from_range.pass.cpp b/libcxx/test/std/containers/associative/multiset/multiset.cons/from_range.pass.cpp
index 74533d7e41abf..15275781a8aac 100644
--- a/libcxx/test/std/containers/associative/multiset/multiset.cons/from_range.pass.cpp
+++ b/libcxx/test/std/containers/associative/multiset/multiset.cons/from_range.pass.cpp
@@ -24,7 +24,7 @@
 
 void test_duplicates() {
   std::array input = {1, 2, 3, 3, 3, 4, 2, 1, 2};
-  auto c = std::multiset<int>(std::from_range, input);
+  auto c           = std::multiset<int>(std::from_range, input);
   assert(std::ranges::is_permutation(input, c));
 }
 
diff --git a/libcxx/test/std/containers/associative/multiset/multiset.cons/initializer_list.pass.cpp b/libcxx/test/std/containers/associative/multiset/multiset.cons/initializer_list.pass.cpp
index fa4a85e20f82a..c860f17bb4196 100644
--- a/libcxx/test/std/containers/associative/multiset/multiset.cons/initializer_list.pass.cpp
+++ b/libcxx/test/std/containers/associative/multiset/multiset.cons/initializer_list.pass.cpp
@@ -20,9 +20,8 @@
 #include "test_macros.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
+int main(int, char**) {
+  {
     typedef std::multiset<int> C;
     typedef C::value_type V;
     C m = {1, 2, 3, 4, 5, 6};
@@ -35,8 +34,8 @@ int main(int, char**)
     assert(*++i == V(4));
     assert(*++i == V(5));
     assert(*++i == V(6));
-    }
-    {
+  }
+  {
     typedef std::multiset<int, std::less<int>, min_allocator<int>> C;
     typedef C::value_type V;
     C m = {1, 2, 3, 4, 5, 6};
@@ -49,12 +48,12 @@ int main(int, char**)
     assert(*++i == V(4));
     assert(*++i == V(5));
     assert(*++i == V(6));
-    }
-    {
+  }
+  {
     typedef std::multiset<int, std::less<int>, min_allocator<int>> C;
     typedef C::value_type V;
     min_allocator<int> a;
-    C m ({1, 2, 3, 4, 5, 6}, a);
+    C m({1, 2, 3, 4, 5, 6}, a);
     assert(m.size() == 6);
     assert(std::distance(m.begin(), m.end()) == 6);
     C::const_iterator i = m.cbegin();
@@ -65,7 +64,7 @@ int main(int, char**)
     assert(*++i == V(5));
     assert(*++i == V(6));
     assert(m.get_allocator() == a);
-    }
+  }
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/associative/multiset/multiset.cons/initializer_list_compare.pass.cpp b/libcxx/test/std/containers/associative/multiset/multiset.cons/initializer_list_compare.pass.cpp
index 4abbddb8c15d4..5bb400cb82be9 100644
--- a/libcxx/test/std/containers/associative/multiset/multiset.cons/initializer_list_compare.pass.cpp
+++ b/libcxx/test/std/containers/associative/multiset/multiset.cons/initializer_list_compare.pass.cpp
@@ -21,22 +21,21 @@
 #include "test_macros.h"
 #include "../../../test_compare.h"
 
-int main(int, char**)
-{
-    typedef test_less<int> Cmp;
-    typedef std::multiset<int, Cmp> C;
-    typedef C::value_type V;
-    C m({1, 2, 3, 4, 5, 6}, Cmp(10));
-    assert(m.size() == 6);
-    assert(std::distance(m.begin(), m.end()) == 6);
-    C::const_iterator i = m.cbegin();
-    assert(*i == V(1));
-    assert(*++i == V(2));
-    assert(*++i == V(3));
-    assert(*++i == V(4));
-    assert(*++i == V(5));
-    assert(*++i == V(6));
-    assert(m.key_comp() == Cmp(10));
+int main(int, char**) {
+  typedef test_less<int> Cmp;
+  typedef std::multiset<int, Cmp> C;
+  typedef C::value_type V;
+  C m({1, 2, 3, 4, 5, 6}, Cmp(10));
+  assert(m.size() == 6);
+  assert(std::distance(m.begin(), m.end()) == 6);
+  C::const_iterator i = m.cbegin();
+  assert(*i == V(1));
+  assert(*++i == V(2));
+  assert(*++i == V(3));
+  assert(*++i == V(4));
+  assert(*++i == V(5));
+  assert(*++i == V(6));
+  assert(m.key_comp() == Cmp(10));
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/associative/multiset/multiset.cons/initializer_list_compare_alloc.pass.cpp b/libcxx/test/std/containers/associative/multiset/multiset.cons/initializer_list_compare_alloc.pass.cpp
index 8234fe3ab7fee..a12e68c1e582b 100644
--- a/libcxx/test/std/containers/associative/multiset/multiset.cons/initializer_list_compare_alloc.pass.cpp
+++ b/libcxx/test/std/containers/associative/multiset/multiset.cons/initializer_list_compare_alloc.pass.cpp
@@ -22,24 +22,23 @@
 #include "../../../test_compare.h"
 #include "test_allocator.h"
 
-int main(int, char**)
-{
-    typedef test_less<int> Cmp;
-    typedef test_allocator<int> A;
-    typedef std::multiset<int, Cmp, A> C;
-    typedef C::value_type V;
-    C m({1, 2, 3, 4, 5, 6}, Cmp(10), A(4));
-    assert(m.size() == 6);
-    assert(std::distance(m.begin(), m.end()) == 6);
-    C::const_iterator i = m.cbegin();
-    assert(*i == V(1));
-    assert(*++i == V(2));
-    assert(*++i == V(3));
-    assert(*++i == V(4));
-    assert(*++i == V(5));
-    assert(*++i == V(6));
-    assert(m.key_comp() == Cmp(10));
-    assert(m.get_allocator() == A(4));
+int main(int, char**) {
+  typedef test_less<int> Cmp;
+  typedef test_allocator<int> A;
+  typedef std::multiset<int, Cmp, A> C;
+  typedef C::value_type V;
+  C m({1, 2, 3, 4, 5, 6}, Cmp(10), A(4));
+  assert(m.size() == 6);
+  assert(std::distance(m.begin(), m.end()) == 6);
+  C::const_iterator i = m.cbegin();
+  assert(*i == V(1));
+  assert(*++i == V(2));
+  assert(*++i == V(3));
+  assert(*++i == V(4));
+  assert(*++i == V(5));
+  assert(*++i == V(6));
+  assert(m.key_comp() == Cmp(10));
+  assert(m.get_allocator() == A(4));
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/associative/multiset/multiset.cons/iter_iter.pass.cpp b/libcxx/test/std/containers/associative/multiset/multiset.cons/iter_iter.pass.cpp
index df28a27df26bc..6b98ceb1c0df7 100644
--- a/libcxx/test/std/containers/associative/multiset/multiset.cons/iter_iter.pass.cpp
+++ b/libcxx/test/std/containers/associative/multiset/multiset.cons/iter_iter.pass.cpp
@@ -20,24 +20,12 @@
 #include "test_iterators.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
+int main(int, char**) {
+  {
     typedef int V;
-    V ar[] =
-    {
-        1,
-        1,
-        1,
-        2,
-        2,
-        2,
-        3,
-        3,
-        3
-    };
-    std::multiset<V> m(cpp17_input_iterator<const int*>(ar),
-                  cpp17_input_iterator<const int*>(ar+sizeof(ar)/sizeof(ar[0])));
+    V ar[] = {1, 1, 1, 2, 2, 2, 3, 3, 3};
+    std::multiset<V> m(
+        cpp17_input_iterator<const int*>(ar), cpp17_input_iterator<const int*>(ar + sizeof(ar) / sizeof(ar[0])));
     assert(m.size() == 9);
     assert(std::distance(m.begin(), m.end()) == 9);
     assert(*std::next(m.begin(), 0) == 1);
@@ -49,24 +37,13 @@ int main(int, char**)
     assert(*std::next(m.begin(), 6) == 3);
     assert(*std::next(m.begin(), 7) == 3);
     assert(*std::next(m.begin(), 8) == 3);
-    }
+  }
 #if TEST_STD_VER >= 11
-    {
+  {
     typedef int V;
-    V ar[] =
-    {
-        1,
-        1,
-        1,
-        2,
-        2,
-        2,
-        3,
-        3,
-        3
-    };
-    std::multiset<V, std::less<V>, min_allocator<V>> m(cpp17_input_iterator<const int*>(ar),
-                  cpp17_input_iterator<const int*>(ar+sizeof(ar)/sizeof(ar[0])));
+    V ar[] = {1, 1, 1, 2, 2, 2, 3, 3, 3};
+    std::multiset<V, std::less<V>, min_allocator<V>> m(
+        cpp17_input_iterator<const int*>(ar), cpp17_input_iterator<const int*>(ar + sizeof(ar) / sizeof(ar[0])));
     assert(m.size() == 9);
     assert(std::distance(m.begin(), m.end()) == 9);
     assert(*std::next(m.begin(), 0) == 1);
@@ -78,7 +55,7 @@ int main(int, char**)
     assert(*std::next(m.begin(), 6) == 3);
     assert(*std::next(m.begin(), 7) == 3);
     assert(*std::next(m.begin(), 8) == 3);
-    }
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/associative/multiset/multiset.cons/iter_iter_alloc.pass.cpp b/libcxx/test/std/containers/associative/multiset/multiset.cons/iter_iter_alloc.pass.cpp
index dcc7e0dd2cc07..01f14d79823fa 100644
--- a/libcxx/test/std/containers/associative/multiset/multiset.cons/iter_iter_alloc.pass.cpp
+++ b/libcxx/test/std/containers/associative/multiset/multiset.cons/iter_iter_alloc.pass.cpp
@@ -22,27 +22,17 @@
 #include "../../../test_compare.h"
 #include "test_allocator.h"
 
-int main(int, char**)
-{
-    {
+int main(int, char**) {
+  {
     typedef int V;
-    V ar[] =
-    {
-        1,
-        1,
-        1,
-        2,
-        2,
-        2,
-        3,
-        3,
-        3
-    };
+    V ar[] = {1, 1, 1, 2, 2, 2, 3, 3, 3};
     typedef test_less<V> C;
     typedef test_allocator<V> A;
-    std::multiset<V, C, A> m(cpp17_input_iterator<const V*>(ar),
-                        cpp17_input_iterator<const V*>(ar+sizeof(ar)/sizeof(ar[0])),
-                        C(5), A(7));
+    std::multiset<V, C, A> m(
+        cpp17_input_iterator<const V*>(ar),
+        cpp17_input_iterator<const V*>(ar + sizeof(ar) / sizeof(ar[0])),
+        C(5),
+        A(7));
     assert(m.value_comp() == C(5));
     assert(m.get_allocator() == A(7));
     assert(m.size() == 9);
@@ -56,26 +46,15 @@ int main(int, char**)
     assert(*std::next(m.begin(), 6) == 3);
     assert(*std::next(m.begin(), 7) == 3);
     assert(*std::next(m.begin(), 8) == 3);
-    }
+  }
 #if TEST_STD_VER > 11
-    {
+  {
     typedef int V;
-    V ar[] =
-    {
-        1,
-        1,
-        1,
-        2,
-        2,
-        2,
-        3,
-        3,
-        3
-    };
+    V ar[] = {1, 1, 1, 2, 2, 2, 3, 3, 3};
     typedef test_allocator<V> A;
     typedef test_less<int> C;
     A a;
-    std::multiset<V, C, A> m(ar, ar+sizeof(ar)/sizeof(ar[0]), a);
+    std::multiset<V, C, A> m(ar, ar + sizeof(ar) / sizeof(ar[0]), a);
 
     assert(m.size() == 9);
     assert(std::distance(m.begin(), m.end()) == 9);
@@ -89,7 +68,7 @@ int main(int, char**)
     assert(*std::next(m.begin(), 7) == 3);
     assert(*std::next(m.begin(), 8) == 3);
     assert(m.get_allocator() == a);
-    }
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/associative/multiset/multiset.cons/iter_iter_comp.pass.cpp b/libcxx/test/std/containers/associative/multiset/multiset.cons/iter_iter_comp.pass.cpp
index b0527f17252f2..a7ce697335944 100644
--- a/libcxx/test/std/containers/associative/multiset/multiset.cons/iter_iter_comp.pass.cpp
+++ b/libcxx/test/std/containers/associative/multiset/multiset.cons/iter_iter_comp.pass.cpp
@@ -20,36 +20,24 @@
 #include "test_iterators.h"
 #include "../../../test_compare.h"
 
-int main(int, char**)
-{
-    typedef int V;
-    V ar[] =
-    {
-        1,
-        1,
-        1,
-        2,
-        2,
-        2,
-        3,
-        3,
-        3
-    };
-    typedef test_less<V> C;
-    std::multiset<V, C> m(cpp17_input_iterator<const V*>(ar),
-                     cpp17_input_iterator<const V*>(ar+sizeof(ar)/sizeof(ar[0])), C(5));
-    assert(m.value_comp() == C(5));
-    assert(m.size() == 9);
-    assert(std::distance(m.begin(), m.end()) == 9);
-    assert(*std::next(m.begin(), 0) == 1);
-    assert(*std::next(m.begin(), 1) == 1);
-    assert(*std::next(m.begin(), 2) == 1);
-    assert(*std::next(m.begin(), 3) == 2);
-    assert(*std::next(m.begin(), 4) == 2);
-    assert(*std::next(m.begin(), 5) == 2);
-    assert(*std::next(m.begin(), 6) == 3);
-    assert(*std::next(m.begin(), 7) == 3);
-    assert(*std::next(m.begin(), 8) == 3);
+int main(int, char**) {
+  typedef int V;
+  V ar[] = {1, 1, 1, 2, 2, 2, 3, 3, 3};
+  typedef test_less<V> C;
+  std::multiset<V, C> m(
+      cpp17_input_iterator<const V*>(ar), cpp17_input_iterator<const V*>(ar + sizeof(ar) / sizeof(ar[0])), C(5));
+  assert(m.value_comp() == C(5));
+  assert(m.size() == 9);
+  assert(std::distance(m.begin(), m.end()) == 9);
+  assert(*std::next(m.begin(), 0) == 1);
+  assert(*std::next(m.begin(), 1) == 1);
+  assert(*std::next(m.begin(), 2) == 1);
+  assert(*std::next(m.begin(), 3) == 2);
+  assert(*std::next(m.begin(), 4) == 2);
+  assert(*std::next(m.begin(), 5) == 2);
+  assert(*std::next(m.begin(), 6) == 3);
+  assert(*std::next(m.begin(), 7) == 3);
+  assert(*std::next(m.begin(), 8) == 3);
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/associative/multiset/multiset.cons/move.pass.cpp b/libcxx/test/std/containers/associative/multiset/multiset.cons/move.pass.cpp
index 63898747180ae..2d71f21e182c9 100644
--- a/libcxx/test/std/containers/associative/multiset/multiset.cons/move.pass.cpp
+++ b/libcxx/test/std/containers/associative/multiset/multiset.cons/move.pass.cpp
@@ -22,100 +22,77 @@
 #include "test_allocator.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef int V;
-        typedef test_less<int> C;
-        typedef test_allocator<V> A;
-        std::multiset<int, C, A> mo(C(5), A(7));
-        std::multiset<int, C, A> m = std::move(mo);
-        assert(m.get_allocator() == A(7));
-        assert(m.key_comp() == C(5));
-        assert(m.size() == 0);
-        assert(std::distance(m.begin(), m.end()) == 0);
+int main(int, char**) {
+  {
+    typedef int V;
+    typedef test_less<int> C;
+    typedef test_allocator<V> A;
+    std::multiset<int, C, A> mo(C(5), A(7));
+    std::multiset<int, C, A> m = std::move(mo);
+    assert(m.get_allocator() == A(7));
+    assert(m.key_comp() == C(5));
+    assert(m.size() == 0);
+    assert(std::distance(m.begin(), m.end()) == 0);
 
-        assert(mo.get_allocator() == A(7));
-        assert(mo.get_allocator().get_id() == test_alloc_base::moved_value);
-        assert(mo.key_comp() == C(5));
-        assert(mo.size() == 0);
-        assert(std::distance(mo.begin(), mo.end()) == 0);
-    }
-    {
-        typedef int V;
-        V ar[] =
-        {
-            1,
-            1,
-            1,
-            2,
-            2,
-            2,
-            3,
-            3,
-            3
-        };
-        typedef test_less<int> C;
-        typedef test_allocator<V> A;
-        std::multiset<int, C, A> mo(ar, ar+sizeof(ar)/sizeof(ar[0]), C(5), A(7));
-        std::multiset<int, C, A> m = std::move(mo);
-        assert(m.get_allocator() == A(7));
-        assert(m.key_comp() == C(5));
-        assert(m.size() == 9);
-        assert(std::distance(m.begin(), m.end()) == 9);
-        assert(*std::next(m.begin(), 0) == 1);
-        assert(*std::next(m.begin(), 1) == 1);
-        assert(*std::next(m.begin(), 2) == 1);
-        assert(*std::next(m.begin(), 3) == 2);
-        assert(*std::next(m.begin(), 4) == 2);
-        assert(*std::next(m.begin(), 5) == 2);
-        assert(*std::next(m.begin(), 6) == 3);
-        assert(*std::next(m.begin(), 7) == 3);
-        assert(*std::next(m.begin(), 8) == 3);
+    assert(mo.get_allocator() == A(7));
+    assert(mo.get_allocator().get_id() == test_alloc_base::moved_value);
+    assert(mo.key_comp() == C(5));
+    assert(mo.size() == 0);
+    assert(std::distance(mo.begin(), mo.end()) == 0);
+  }
+  {
+    typedef int V;
+    V ar[] = {1, 1, 1, 2, 2, 2, 3, 3, 3};
+    typedef test_less<int> C;
+    typedef test_allocator<V> A;
+    std::multiset<int, C, A> mo(ar, ar + sizeof(ar) / sizeof(ar[0]), C(5), A(7));
+    std::multiset<int, C, A> m = std::move(mo);
+    assert(m.get_allocator() == A(7));
+    assert(m.key_comp() == C(5));
+    assert(m.size() == 9);
+    assert(std::distance(m.begin(), m.end()) == 9);
+    assert(*std::next(m.begin(), 0) == 1);
+    assert(*std::next(m.begin(), 1) == 1);
+    assert(*std::next(m.begin(), 2) == 1);
+    assert(*std::next(m.begin(), 3) == 2);
+    assert(*std::next(m.begin(), 4) == 2);
+    assert(*std::next(m.begin(), 5) == 2);
+    assert(*std::next(m.begin(), 6) == 3);
+    assert(*std::next(m.begin(), 7) == 3);
+    assert(*std::next(m.begin(), 8) == 3);
 
-        assert(mo.get_allocator() == A(7));
-        assert(mo.get_allocator().get_id() == test_alloc_base::moved_value);
-        assert(mo.key_comp() == C(5));
-        assert(mo.size() == 0);
-        assert(std::distance(mo.begin(), mo.end()) == 0);
-    }
-    {
-        typedef int V;
-        V ar[] =
-        {
-            1,
-            1,
-            1,
-            2,
-            2,
-            2,
-            3,
-            3,
-            3
-        };
-        typedef test_less<int> C;
-        typedef min_allocator<V> A;
-        std::multiset<int, C, A> mo(ar, ar+sizeof(ar)/sizeof(ar[0]), C(5), A());
-        std::multiset<int, C, A> m = std::move(mo);
-        assert(m.get_allocator() == A());
-        assert(m.key_comp() == C(5));
-        assert(m.size() == 9);
-        assert(std::distance(m.begin(), m.end()) == 9);
-        assert(*std::next(m.begin(), 0) == 1);
-        assert(*std::next(m.begin(), 1) == 1);
-        assert(*std::next(m.begin(), 2) == 1);
-        assert(*std::next(m.begin(), 3) == 2);
-        assert(*std::next(m.begin(), 4) == 2);
-        assert(*std::next(m.begin(), 5) == 2);
-        assert(*std::next(m.begin(), 6) == 3);
-        assert(*std::next(m.begin(), 7) == 3);
-        assert(*std::next(m.begin(), 8) == 3);
+    assert(mo.get_allocator() == A(7));
+    assert(mo.get_allocator().get_id() == test_alloc_base::moved_value);
+    assert(mo.key_comp() == C(5));
+    assert(mo.size() == 0);
+    assert(std::distance(mo.begin(), mo.end()) == 0);
+  }
+  {
+    typedef int V;
+    V ar[] = {1, 1, 1, 2, 2, 2, 3, 3, 3};
+    typedef test_less<int> C;
+    typedef min_allocator<V> A;
+    std::multiset<int, C, A> mo(ar, ar + sizeof(ar) / sizeof(ar[0]), C(5), A());
+    std::multiset<int, C, A> m = std::move(mo);
+    assert(m.get_allocator() == A());
+    assert(m.key_comp() == C(5));
+    assert(m.size() == 9);
+    assert(std::distance(m.begin(), m.end()) == 9);
+    assert(*std::next(m.begin(), 0) == 1);
+    assert(*std::next(m.begin(), 1) == 1);
+    assert(*std::next(m.begin(), 2) == 1);
+    assert(*std::next(m.begin(), 3) == 2);
+    assert(*std::next(m.begin(), 4) == 2);
+    assert(*std::next(m.begin(), 5) == 2);
+    assert(*std::next(m.begin(), 6) == 3);
+    assert(*std::next(m.begin(), 7) == 3);
+    assert(*std::next(m.begin(), 8) == 3);
 
-        assert(mo.get_allocator() == A());
-        assert(mo.key_comp() == C(5));
-        assert(mo.size() == 0);
-        assert(std::distance(mo.begin(), mo.end()) == 0);
-    }
+    assert(mo.get_allocator() == A());
+    assert(mo.key_comp() == C(5));
+    assert(mo.size() == 0);
+    assert(std::distance(mo.begin(), mo.end()) == 0);
+  }
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/associative/multiset/multiset.cons/move_alloc.pass.cpp b/libcxx/test/std/containers/associative/multiset/multiset.cons/move_alloc.pass.cpp
index 090be76483be2..771a7038bf633 100644
--- a/libcxx/test/std/containers/associative/multiset/multiset.cons/move_alloc.pass.cpp
+++ b/libcxx/test/std/containers/associative/multiset/multiset.cons/move_alloc.pass.cpp
@@ -24,170 +24,92 @@
 #include "test_allocator.h"
 #include "Counter.h"
 
-int main(int, char**)
-{
+int main(int, char**) {
+  {
+    typedef MoveOnly V;
+    typedef test_less<MoveOnly> C;
+    typedef test_allocator<V> A;
+    typedef std::multiset<MoveOnly, C, A> M;
+    typedef std::move_iterator<V*> I;
+    V a1[] = {V(1), V(1), V(1), V(2), V(2), V(2), V(3), V(3), V(3)};
+    M m1(I(a1), I(a1 + sizeof(a1) / sizeof(a1[0])), C(5), A(7));
+    V a2[] = {V(1), V(1), V(1), V(2), V(2), V(2), V(3), V(3), V(3)};
+    M m2(I(a2), I(a2 + sizeof(a2) / sizeof(a2[0])), C(5), A(7));
+    M m3(std::move(m1), A(7));
+    assert(m3 == m2);
+    assert(m3.get_allocator() == A(7));
+    assert(m3.key_comp() == C(5));
+    LIBCPP_ASSERT(m1.empty());
+  }
+  {
+    typedef MoveOnly V;
+    typedef test_less<MoveOnly> C;
+    typedef test_allocator<V> A;
+    typedef std::multiset<MoveOnly, C, A> M;
+    typedef std::move_iterator<V*> I;
+    V a1[] = {V(1), V(1), V(1), V(2), V(2), V(2), V(3), V(3), V(3)};
+    M m1(I(a1), I(a1 + sizeof(a1) / sizeof(a1[0])), C(5), A(7));
+    V a2[] = {V(1), V(1), V(1), V(2), V(2), V(2), V(3), V(3), V(3)};
+    M m2(I(a2), I(a2 + sizeof(a2) / sizeof(a2[0])), C(5), A(7));
+    M m3(std::move(m1), A(5));
+    assert(m3 == m2);
+    assert(m3.get_allocator() == A(5));
+    assert(m3.key_comp() == C(5));
+    LIBCPP_ASSERT(m1.empty());
+  }
+  {
+    typedef MoveOnly V;
+    typedef test_less<MoveOnly> C;
+    typedef other_allocator<V> A;
+    typedef std::multiset<MoveOnly, C, A> M;
+    typedef std::move_iterator<V*> I;
+    V a1[] = {V(1), V(1), V(1), V(2), V(2), V(2), V(3), V(3), V(3)};
+    M m1(I(a1), I(a1 + sizeof(a1) / sizeof(a1[0])), C(5), A(7));
+    V a2[] = {V(1), V(1), V(1), V(2), V(2), V(2), V(3), V(3), V(3)};
+    M m2(I(a2), I(a2 + sizeof(a2) / sizeof(a2[0])), C(5), A(7));
+    M m3(std::move(m1), A(5));
+    assert(m3 == m2);
+    assert(m3.get_allocator() == A(5));
+    assert(m3.key_comp() == C(5));
+    LIBCPP_ASSERT(m1.empty());
+  }
+  {
+    typedef Counter<int> V;
+    typedef std::less<V> C;
+    typedef test_allocator<V> A;
+    typedef std::multiset<V, C, A> M;
+    typedef V* I;
+    Counter_base::gConstructed = 0;
     {
-        typedef MoveOnly V;
-        typedef test_less<MoveOnly> C;
-        typedef test_allocator<V> A;
-        typedef std::multiset<MoveOnly, C, A> M;
-        typedef std::move_iterator<V*> I;
-        V a1[] =
-        {
-            V(1),
-            V(1),
-            V(1),
-            V(2),
-            V(2),
-            V(2),
-            V(3),
-            V(3),
-            V(3)
-        };
-        M m1(I(a1), I(a1+sizeof(a1)/sizeof(a1[0])), C(5), A(7));
-        V a2[] =
-        {
-            V(1),
-            V(1),
-            V(1),
-            V(2),
-            V(2),
-            V(2),
-            V(3),
-            V(3),
-            V(3)
-        };
-        M m2(I(a2), I(a2+sizeof(a2)/sizeof(a2[0])), C(5), A(7));
-        M m3(std::move(m1), A(7));
-        assert(m3 == m2);
-        assert(m3.get_allocator() == A(7));
-        assert(m3.key_comp() == C(5));
-        LIBCPP_ASSERT(m1.empty());
-    }
-    {
-        typedef MoveOnly V;
-        typedef test_less<MoveOnly> C;
-        typedef test_allocator<V> A;
-        typedef std::multiset<MoveOnly, C, A> M;
-        typedef std::move_iterator<V*> I;
-        V a1[] =
-        {
-            V(1),
-            V(1),
-            V(1),
-            V(2),
-            V(2),
-            V(2),
-            V(3),
-            V(3),
-            V(3)
-        };
-        M m1(I(a1), I(a1+sizeof(a1)/sizeof(a1[0])), C(5), A(7));
-        V a2[] =
-        {
-            V(1),
-            V(1),
-            V(1),
-            V(2),
-            V(2),
-            V(2),
-            V(3),
-            V(3),
-            V(3)
-        };
-        M m2(I(a2), I(a2+sizeof(a2)/sizeof(a2[0])), C(5), A(7));
-        M m3(std::move(m1), A(5));
-        assert(m3 == m2);
-        assert(m3.get_allocator() == A(5));
-        assert(m3.key_comp() == C(5));
-        LIBCPP_ASSERT(m1.empty());
-    }
-    {
-        typedef MoveOnly V;
-        typedef test_less<MoveOnly> C;
-        typedef other_allocator<V> A;
-        typedef std::multiset<MoveOnly, C, A> M;
-        typedef std::move_iterator<V*> I;
-        V a1[] =
-        {
-            V(1),
-            V(1),
-            V(1),
-            V(2),
-            V(2),
-            V(2),
-            V(3),
-            V(3),
-            V(3)
-        };
-        M m1(I(a1), I(a1+sizeof(a1)/sizeof(a1[0])), C(5), A(7));
-        V a2[] =
-        {
-            V(1),
-            V(1),
-            V(1),
-            V(2),
-            V(2),
-            V(2),
-            V(3),
-            V(3),
-            V(3)
-        };
-        M m2(I(a2), I(a2+sizeof(a2)/sizeof(a2[0])), C(5), A(7));
-        M m3(std::move(m1), A(5));
-        assert(m3 == m2);
-        assert(m3.get_allocator() == A(5));
-        assert(m3.key_comp() == C(5));
-        LIBCPP_ASSERT(m1.empty());
-    }
-    {
-        typedef Counter<int> V;
-        typedef std::less<V> C;
-        typedef test_allocator<V> A;
-        typedef std::multiset<V, C, A> M;
-        typedef V* I;
-        Counter_base::gConstructed = 0;
-        {
-            V a1[] =
-            {
-            V(1),
-            V(1),
-            V(1),
-            V(2),
-            V(2),
-            V(2),
-            V(3),
-            V(3),
-            V(3)
-            };
-            const std::size_t num = sizeof(a1)/sizeof(a1[0]);
-            assert(Counter_base::gConstructed == num);
+      V a1[]                = {V(1), V(1), V(1), V(2), V(2), V(2), V(3), V(3), V(3)};
+      const std::size_t num = sizeof(a1) / sizeof(a1[0]);
+      assert(Counter_base::gConstructed == num);
 
-            M m1(I(a1), I(a1+num), C(), A());
-            assert(Counter_base::gConstructed == 2*num);
+      M m1(I(a1), I(a1 + num), C(), A());
+      assert(Counter_base::gConstructed == 2 * num);
 
-            M m2(m1);
-            assert(m2 == m1);
-            assert(Counter_base::gConstructed == 3*num);
+      M m2(m1);
+      assert(m2 == m1);
+      assert(Counter_base::gConstructed == 3 * num);
 
-            M m3(std::move(m1), A());
-            assert(m3 == m2);
-            LIBCPP_ASSERT(m1.empty());
-            assert(Counter_base::gConstructed >= (int)(3*num));
-            assert(Counter_base::gConstructed <= (int)(4*num));
+      M m3(std::move(m1), A());
+      assert(m3 == m2);
+      LIBCPP_ASSERT(m1.empty());
+      assert(Counter_base::gConstructed >= (int)(3 * num));
+      assert(Counter_base::gConstructed <= (int)(4 * num));
 
-            {
-            M m4(std::move(m2), A(5));
-            assert(Counter_base::gConstructed >= (int)(3*num));
-            assert(Counter_base::gConstructed <= (int)(5*num));
-            assert(m4 == m3);
-            LIBCPP_ASSERT(m2.empty());
-            }
-            assert(Counter_base::gConstructed >= (int)(2*num));
-            assert(Counter_base::gConstructed <= (int)(4*num));
-        }
-        assert(Counter_base::gConstructed == 0);
+      {
+        M m4(std::move(m2), A(5));
+        assert(Counter_base::gConstructed >= (int)(3 * num));
+        assert(Counter_base::gConstructed <= (int)(5 * num));
+        assert(m4 == m3);
+        LIBCPP_ASSERT(m2.empty());
+      }
+      assert(Counter_base::gConstructed >= (int)(2 * num));
+      assert(Counter_base::gConstructed <= (int)(4 * num));
     }
+    assert(Counter_base::gConstructed == 0);
+  }
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/associative/multiset/multiset.cons/move_assign.pass.cpp b/libcxx/test/std/containers/associative/multiset/multiset.cons/move_assign.pass.cpp
index c57a56cd44880..5cdd4ec67b8e0 100644
--- a/libcxx/test/std/containers/associative/multiset/multiset.cons/move_assign.pass.cpp
+++ b/libcxx/test/std/containers/associative/multiset/multiset.cons/move_assign.pass.cpp
@@ -23,164 +23,75 @@
 #include "test_allocator.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef MoveOnly V;
-        typedef test_less<MoveOnly> C;
-        typedef test_allocator<V> A;
-        typedef std::multiset<MoveOnly, C, A> M;
-        typedef std::move_iterator<V*> I;
-        V a1[] =
-        {
-            V(1),
-            V(1),
-            V(1),
-            V(2),
-            V(2),
-            V(2),
-            V(3),
-            V(3),
-            V(3)
-        };
-        M m1(I(a1), I(a1+sizeof(a1)/sizeof(a1[0])), C(5), A(7));
-        V a2[] =
-        {
-            V(1),
-            V(1),
-            V(1),
-            V(2),
-            V(2),
-            V(2),
-            V(3),
-            V(3),
-            V(3)
-        };
-        M m2(I(a2), I(a2+sizeof(a2)/sizeof(a2[0])), C(5), A(7));
-        M m3(C(3), A(7));
-        m3 = std::move(m1);
-        assert(m3 == m2);
-        assert(m3.get_allocator() == A(7));
-        assert(m3.key_comp() == C(5));
-        assert(m1.empty());
-    }
-    {
-        typedef MoveOnly V;
-        typedef test_less<MoveOnly> C;
-        typedef test_allocator<V> A;
-        typedef std::multiset<MoveOnly, C, A> M;
-        typedef std::move_iterator<V*> I;
-        V a1[] =
-        {
-            V(1),
-            V(1),
-            V(1),
-            V(2),
-            V(2),
-            V(2),
-            V(3),
-            V(3),
-            V(3)
-        };
-        M m1(I(a1), I(a1+sizeof(a1)/sizeof(a1[0])), C(5), A(7));
-        V a2[] =
-        {
-            V(1),
-            V(1),
-            V(1),
-            V(2),
-            V(2),
-            V(2),
-            V(3),
-            V(3),
-            V(3)
-        };
-        M m2(I(a2), I(a2+sizeof(a2)/sizeof(a2[0])), C(5), A(7));
-        M m3(C(3), A(5));
-        m3 = std::move(m1);
-        assert(m3 == m2);
-        assert(m3.get_allocator() == A(5));
-        assert(m3.key_comp() == C(5));
-        LIBCPP_ASSERT(m1.empty());
-    }
-    {
-        typedef MoveOnly V;
-        typedef test_less<MoveOnly> C;
-        typedef other_allocator<V> A;
-        typedef std::multiset<MoveOnly, C, A> M;
-        typedef std::move_iterator<V*> I;
-        V a1[] =
-        {
-            V(1),
-            V(1),
-            V(1),
-            V(2),
-            V(2),
-            V(2),
-            V(3),
-            V(3),
-            V(3)
-        };
-        M m1(I(a1), I(a1+sizeof(a1)/sizeof(a1[0])), C(5), A(7));
-        V a2[] =
-        {
-            V(1),
-            V(1),
-            V(1),
-            V(2),
-            V(2),
-            V(2),
-            V(3),
-            V(3),
-            V(3)
-        };
-        M m2(I(a2), I(a2+sizeof(a2)/sizeof(a2[0])), C(5), A(7));
-        M m3(C(3), A(5));
-        m3 = std::move(m1);
-        assert(m3 == m2);
-        assert(m3.get_allocator() == A(7));
-        assert(m3.key_comp() == C(5));
-        assert(m1.empty());
-    }
-    {
-        typedef MoveOnly V;
-        typedef test_less<MoveOnly> C;
-        typedef min_allocator<V> A;
-        typedef std::multiset<MoveOnly, C, A> M;
-        typedef std::move_iterator<V*> I;
-        V a1[] =
-        {
-            V(1),
-            V(1),
-            V(1),
-            V(2),
-            V(2),
-            V(2),
-            V(3),
-            V(3),
-            V(3)
-        };
-        M m1(I(a1), I(a1+sizeof(a1)/sizeof(a1[0])), C(5), A());
-        V a2[] =
-        {
-            V(1),
-            V(1),
-            V(1),
-            V(2),
-            V(2),
-            V(2),
-            V(3),
-            V(3),
-            V(3)
-        };
-        M m2(I(a2), I(a2+sizeof(a2)/sizeof(a2[0])), C(5), A());
-        M m3(C(3), A());
-        m3 = std::move(m1);
-        assert(m3 == m2);
-        assert(m3.get_allocator() == A());
-        assert(m3.key_comp() == C(5));
-        assert(m1.empty());
-    }
+int main(int, char**) {
+  {
+    typedef MoveOnly V;
+    typedef test_less<MoveOnly> C;
+    typedef test_allocator<V> A;
+    typedef std::multiset<MoveOnly, C, A> M;
+    typedef std::move_iterator<V*> I;
+    V a1[] = {V(1), V(1), V(1), V(2), V(2), V(2), V(3), V(3), V(3)};
+    M m1(I(a1), I(a1 + sizeof(a1) / sizeof(a1[0])), C(5), A(7));
+    V a2[] = {V(1), V(1), V(1), V(2), V(2), V(2), V(3), V(3), V(3)};
+    M m2(I(a2), I(a2 + sizeof(a2) / sizeof(a2[0])), C(5), A(7));
+    M m3(C(3), A(7));
+    m3 = std::move(m1);
+    assert(m3 == m2);
+    assert(m3.get_allocator() == A(7));
+    assert(m3.key_comp() == C(5));
+    assert(m1.empty());
+  }
+  {
+    typedef MoveOnly V;
+    typedef test_less<MoveOnly> C;
+    typedef test_allocator<V> A;
+    typedef std::multiset<MoveOnly, C, A> M;
+    typedef std::move_iterator<V*> I;
+    V a1[] = {V(1), V(1), V(1), V(2), V(2), V(2), V(3), V(3), V(3)};
+    M m1(I(a1), I(a1 + sizeof(a1) / sizeof(a1[0])), C(5), A(7));
+    V a2[] = {V(1), V(1), V(1), V(2), V(2), V(2), V(3), V(3), V(3)};
+    M m2(I(a2), I(a2 + sizeof(a2) / sizeof(a2[0])), C(5), A(7));
+    M m3(C(3), A(5));
+    m3 = std::move(m1);
+    assert(m3 == m2);
+    assert(m3.get_allocator() == A(5));
+    assert(m3.key_comp() == C(5));
+    LIBCPP_ASSERT(m1.empty());
+  }
+  {
+    typedef MoveOnly V;
+    typedef test_less<MoveOnly> C;
+    typedef other_allocator<V> A;
+    typedef std::multiset<MoveOnly, C, A> M;
+    typedef std::move_iterator<V*> I;
+    V a1[] = {V(1), V(1), V(1), V(2), V(2), V(2), V(3), V(3), V(3)};
+    M m1(I(a1), I(a1 + sizeof(a1) / sizeof(a1[0])), C(5), A(7));
+    V a2[] = {V(1), V(1), V(1), V(2), V(2), V(2), V(3), V(3), V(3)};
+    M m2(I(a2), I(a2 + sizeof(a2) / sizeof(a2[0])), C(5), A(7));
+    M m3(C(3), A(5));
+    m3 = std::move(m1);
+    assert(m3 == m2);
+    assert(m3.get_allocator() == A(7));
+    assert(m3.key_comp() == C(5));
+    assert(m1.empty());
+  }
+  {
+    typedef MoveOnly V;
+    typedef test_less<MoveOnly> C;
+    typedef min_allocator<V> A;
+    typedef std::multiset<MoveOnly, C, A> M;
+    typedef std::move_iterator<V*> I;
+    V a1[] = {V(1), V(1), V(1), V(2), V(2), V(2), V(3), V(3), V(3)};
+    M m1(I(a1), I(a1 + sizeof(a1) / sizeof(a1[0])), C(5), A());
+    V a2[] = {V(1), V(1), V(1), V(2), V(2), V(2), V(3), V(3), V(3)};
+    M m2(I(a2), I(a2 + sizeof(a2) / sizeof(a2[0])), C(5), A());
+    M m3(C(3), A());
+    m3 = std::move(m1);
+    assert(m3 == m2);
+    assert(m3.get_allocator() == A());
+    assert(m3.key_comp() == C(5));
+    assert(m1.empty());
+  }
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/associative/multiset/multiset.cons/move_assign_noexcept.compile.pass.cpp b/libcxx/test/std/containers/associative/multiset/multiset.cons/move_assign_noexcept.compile.pass.cpp
new file mode 100644
index 0000000000000..930eae1bc9083
--- /dev/null
+++ b/libcxx/test/std/containers/associative/multiset/multiset.cons/move_assign_noexcept.compile.pass.cpp
@@ -0,0 +1,61 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// <set>
+
+// multiset& operator=(multiset&& c)
+//     noexcept(
+//          allocator_type::propagate_on_container_move_assignment::value &&
+//          is_nothrow_move_assignable<allocator_type>::value &&
+//          is_nothrow_move_assignable<key_compare>::value);
+
+// This tests a conforming extension
+
+// UNSUPPORTED: c++03
+
+#include <set>
+
+#include "test_macros.h"
+#include "MoveOnly.h"
+#include "test_allocator.h"
+
+template <class T>
+struct some_comp {
+  using value_type = T;
+  some_comp& operator=(const some_comp&);
+  bool operator()(const T&, const T&) const { return false; }
+};
+
+template <class T>
+struct always_equal_alloc {
+  using value_type = T;
+  always_equal_alloc(const always_equal_alloc&);
+  void allocate(std::size_t);
+};
+
+template <class T>
+struct not_always_equal_alloc {
+  int i;
+  using value_type = T;
+  not_always_equal_alloc(const not_always_equal_alloc&);
+  void allocate(std::size_t);
+};
+
+template <template <class> class Alloc>
+using unordered_set_alloc = std::set<MoveOnly, std::less<MoveOnly>, Alloc<MoveOnly>>;
+
+static_assert(std::is_nothrow_move_assignable<unordered_set_alloc<std::allocator>>::value, "");
+static_assert(!std::is_nothrow_move_assignable<unordered_set_alloc<test_allocator>>::value, "");
+#if TEST_STD_VER >= 17
+static_assert(std::is_nothrow_move_assignable<unordered_set_alloc<always_equal_alloc>>::value, "");
+#endif
+static_assert(!std::is_nothrow_move_assignable<unordered_set_alloc<not_always_equal_alloc>>::value, "");
+#if defined(_LIBCPP_VERSION)
+static_assert(std::is_nothrow_move_assignable<unordered_set_alloc<other_allocator>>::value, "");
+#endif // _LIBCPP_VERSION
+static_assert(!std::is_nothrow_move_assignable<std::set<int, some_comp<int>>>::value, "");
diff --git a/libcxx/test/std/containers/associative/multiset/multiset.cons/move_assign_noexcept.pass.cpp b/libcxx/test/std/containers/associative/multiset/multiset.cons/move_assign_noexcept.pass.cpp
deleted file mode 100644
index a94b5b1abf1e5..0000000000000
--- a/libcxx/test/std/containers/associative/multiset/multiset.cons/move_assign_noexcept.pass.cpp
+++ /dev/null
@@ -1,58 +0,0 @@
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-// <set>
-
-// multiset& operator=(multiset&& c)
-//     noexcept(
-//          allocator_type::propagate_on_container_move_assignment::value &&
-//          is_nothrow_move_assignable<allocator_type>::value &&
-//          is_nothrow_move_assignable<key_compare>::value);
-
-// This tests a conforming extension
-
-// UNSUPPORTED: c++03
-
-#include <set>
-#include <cassert>
-
-#include "test_macros.h"
-#include "MoveOnly.h"
-#include "test_allocator.h"
-
-template <class T>
-struct some_comp
-{
-    typedef T value_type;
-    some_comp& operator=(const some_comp&);
-    bool operator()(const T&, const T&) const { return false; }
-};
-
-int main(int, char**)
-{
-    {
-        typedef std::multiset<MoveOnly> C;
-        static_assert(std::is_nothrow_move_assignable<C>::value, "");
-    }
-    {
-        typedef std::multiset<MoveOnly, std::less<MoveOnly>, test_allocator<MoveOnly>> C;
-        static_assert(!std::is_nothrow_move_assignable<C>::value, "");
-    }
-#if defined(_LIBCPP_VERSION)
-    {
-        typedef std::multiset<MoveOnly, std::less<MoveOnly>, other_allocator<MoveOnly>> C;
-        static_assert(std::is_nothrow_move_assignable<C>::value, "");
-    }
-#endif // _LIBCPP_VERSION
-    {
-        typedef std::multiset<MoveOnly, some_comp<MoveOnly>> C;
-        static_assert(!std::is_nothrow_move_assignable<C>::value, "");
-    }
-
-  return 0;
-}
diff --git a/libcxx/test/std/containers/associative/multiset/multiset.cons/move_noexcept.pass.cpp b/libcxx/test/std/containers/associative/multiset/multiset.cons/move_noexcept.pass.cpp
index 4bf70682d7113..73b4be5005e1c 100644
--- a/libcxx/test/std/containers/associative/multiset/multiset.cons/move_noexcept.pass.cpp
+++ b/libcxx/test/std/containers/associative/multiset/multiset.cons/move_noexcept.pass.cpp
@@ -24,33 +24,31 @@
 #include "test_allocator.h"
 
 template <class T>
-struct some_comp
-{
-    typedef T value_type;
-    some_comp(const some_comp&);
-    bool operator()(const T&, const T&) const { return false; }
+struct some_comp {
+  typedef T value_type;
+  some_comp(const some_comp&);
+  bool operator()(const T&, const T&) const { return false; }
 };
 
-int main(int, char**)
-{
+int main(int, char**) {
 #if defined(_LIBCPP_VERSION)
-    {
-        typedef std::multiset<MoveOnly> C;
-        static_assert(std::is_nothrow_move_constructible<C>::value, "");
-    }
-    {
-        typedef std::multiset<MoveOnly, std::less<MoveOnly>, test_allocator<MoveOnly>> C;
-        static_assert(std::is_nothrow_move_constructible<C>::value, "");
-    }
-    {
-        typedef std::multiset<MoveOnly, std::less<MoveOnly>, other_allocator<MoveOnly>> C;
-        static_assert(std::is_nothrow_move_constructible<C>::value, "");
-    }
+  {
+    typedef std::multiset<MoveOnly> C;
+    static_assert(std::is_nothrow_move_constructible<C>::value, "");
+  }
+  {
+    typedef std::multiset<MoveOnly, std::less<MoveOnly>, test_allocator<MoveOnly>> C;
+    static_assert(std::is_nothrow_move_constructible<C>::value, "");
+  }
+  {
+    typedef std::multiset<MoveOnly, std::less<MoveOnly>, other_allocator<MoveOnly>> C;
+    static_assert(std::is_nothrow_move_constructible<C>::value, "");
+  }
 #endif // _LIBCPP_VERSION
-    {
-        typedef std::multiset<MoveOnly, some_comp<MoveOnly>> C;
-        static_assert(!std::is_nothrow_move_constructible<C>::value, "");
-    }
+  {
+    typedef std::multiset<MoveOnly, some_comp<MoveOnly>> C;
+    static_assert(!std::is_nothrow_move_constructible<C>::value, "");
+  }
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/associative/multiset/multiset.erasure/erase_if.pass.cpp b/libcxx/test/std/containers/associative/multiset/multiset.erasure/erase_if.pass.cpp
index 476f4f54e28e1..cfb304ee3be39 100644
--- a/libcxx/test/std/containers/associative/multiset/multiset.erasure/erase_if.pass.cpp
+++ b/libcxx/test/std/containers/associative/multiset/multiset.erasure/erase_if.pass.cpp
@@ -27,52 +27,50 @@ void test0(S s, Pred p, S expected, std::size_t expected_erased_count) {
 }
 
 template <typename S>
-void test()
-{
-    auto is1 = [](auto v) { return v == 1;};
-    auto is2 = [](auto v) { return v == 2;};
-    auto is3 = [](auto v) { return v == 3;};
-    auto is4 = [](auto v) { return v == 4;};
-    auto True  = [](auto) { return true; };
-    auto False = [](auto) { return false; };
+void test() {
+  auto is1   = [](auto v) { return v == 1; };
+  auto is2   = [](auto v) { return v == 2; };
+  auto is3   = [](auto v) { return v == 3; };
+  auto is4   = [](auto v) { return v == 4; };
+  auto True  = [](auto) { return true; };
+  auto False = [](auto) { return false; };
 
-    test0(S(), is1, S(), 0);
+  test0(S(), is1, S(), 0);
 
-    test0(S({1}), is1, S(), 1);
-    test0(S({1}), is2, S({1}), 0);
+  test0(S({1}), is1, S(), 1);
+  test0(S({1}), is2, S({1}), 0);
 
-    test0(S({1, 2}), is1, S({2}), 1);
-    test0(S({1, 2}), is2, S({1}), 1);
-    test0(S({1, 2}), is3, S({1, 2}), 0);
-    test0(S({1, 1}), is1, S(), 2);
-    test0(S({1, 1}), is3, S({1, 1}), 0);
+  test0(S({1, 2}), is1, S({2}), 1);
+  test0(S({1, 2}), is2, S({1}), 1);
+  test0(S({1, 2}), is3, S({1, 2}), 0);
+  test0(S({1, 1}), is1, S(), 2);
+  test0(S({1, 1}), is3, S({1, 1}), 0);
 
-    test0(S({1, 2, 3}), is1, S({2, 3}), 1);
-    test0(S({1, 2, 3}), is2, S({1, 3}), 1);
-    test0(S({1, 2, 3}), is3, S({1, 2}), 1);
-    test0(S({1, 2, 3}), is4, S({1, 2, 3}), 0);
+  test0(S({1, 2, 3}), is1, S({2, 3}), 1);
+  test0(S({1, 2, 3}), is2, S({1, 3}), 1);
+  test0(S({1, 2, 3}), is3, S({1, 2}), 1);
+  test0(S({1, 2, 3}), is4, S({1, 2, 3}), 0);
 
-    test0(S({1, 1, 1}), is1, S(), 3);
-    test0(S({1, 1, 1}), is2, S({1, 1, 1}), 0);
-    test0(S({1, 1, 2}), is1, S({2}), 2);
-    test0(S({1, 1, 2}), is2, S({1, 1}), 1);
-    test0(S({1, 1, 2}), is3, S({1, 1, 2}), 0);
-    test0(S({1, 2, 2}), is1, S({2, 2}), 1);
-    test0(S({1, 2, 2}), is2, S({1}), 2);
-    test0(S({1, 2, 2}), is3, S({1, 2, 2}), 0);
+  test0(S({1, 1, 1}), is1, S(), 3);
+  test0(S({1, 1, 1}), is2, S({1, 1, 1}), 0);
+  test0(S({1, 1, 2}), is1, S({2}), 2);
+  test0(S({1, 1, 2}), is2, S({1, 1}), 1);
+  test0(S({1, 1, 2}), is3, S({1, 1, 2}), 0);
+  test0(S({1, 2, 2}), is1, S({2, 2}), 1);
+  test0(S({1, 2, 2}), is2, S({1}), 2);
+  test0(S({1, 2, 2}), is3, S({1, 2, 2}), 0);
 
-    test0(S({1, 2, 3}), True, S(), 3);
-    test0(S({1, 2, 3}), False, S({1, 2, 3}), 0);
+  test0(S({1, 2, 3}), True, S(), 3);
+  test0(S({1, 2, 3}), False, S({1, 2, 3}), 0);
 }
 
-int main(int, char**)
-{
-    test<std::multiset<int>>();
-    test<std::multiset<int, std::less<int>, min_allocator<int>>> ();
-    test<std::multiset<int, std::less<int>, test_allocator<int>>> ();
+int main(int, char**) {
+  test<std::multiset<int>>();
+  test<std::multiset<int, std::less<int>, min_allocator<int>>>();
+  test<std::multiset<int, std::less<int>, test_allocator<int>>>();
 
-    test<std::multiset<long>>();
-    test<std::multiset<double>>();
+  test<std::multiset<long>>();
+  test<std::multiset<double>>();
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/associative/multiset/multiset.nonmember/op_compare.pass.cpp b/libcxx/test/std/containers/associative/multiset/multiset.nonmember/op_compare.pass.cpp
index a8011f199775e..9ffb5dc3593c5 100644
--- a/libcxx/test/std/containers/associative/multiset/multiset.nonmember/op_compare.pass.cpp
+++ b/libcxx/test/std/containers/associative/multiset/multiset.nonmember/op_compare.pass.cpp
@@ -39,36 +39,36 @@
 #include "test_comparisons.h"
 
 int main(int, char**) {
-    {
-        std::multiset<int> s1, s2;
-        s1.insert(1);
-        s2.insert(2);
-        const std::multiset<int>& cs1 = s1, cs2 = s2;
-        assert(testComparisons(cs1, cs2, false, true));
-    }
-    {
-        std::multiset<int> s1, s2;
-        s1.insert(1);
-        s2.insert(1);
-        const std::multiset<int>& cs1 = s1, cs2 = s2;
-        assert(testComparisons(cs1, cs2, true, false));
-    }
-    {
-        std::multiset<int> s1, s2;
-        s1.insert(1);
-        s2.insert(1);
-        s2.insert(2);
-        const std::multiset<int>& cs1 = s1, cs2 = s2;
-        assert(testComparisons(cs1, cs2, false, true));
-    }
-    {
-        std::multiset<int> s1, s2;
-        s1.insert(1);
-        s2.insert(1);
-        s2.insert(1);
-        s2.insert(1);
-        const std::multiset<int>& cs1 = s1, cs2 = s2;
-        assert(testComparisons(cs1, cs2, false, true));
-    }
-    return 0;
+  {
+    std::multiset<int> s1, s2;
+    s1.insert(1);
+    s2.insert(2);
+    const std::multiset<int>&cs1 = s1, cs2 = s2;
+    assert(testComparisons(cs1, cs2, false, true));
+  }
+  {
+    std::multiset<int> s1, s2;
+    s1.insert(1);
+    s2.insert(1);
+    const std::multiset<int>&cs1 = s1, cs2 = s2;
+    assert(testComparisons(cs1, cs2, true, false));
+  }
+  {
+    std::multiset<int> s1, s2;
+    s1.insert(1);
+    s2.insert(1);
+    s2.insert(2);
+    const std::multiset<int>&cs1 = s1, cs2 = s2;
+    assert(testComparisons(cs1, cs2, false, true));
+  }
+  {
+    std::multiset<int> s1, s2;
+    s1.insert(1);
+    s2.insert(1);
+    s2.insert(1);
+    s2.insert(1);
+    const std::multiset<int>&cs1 = s1, cs2 = s2;
+    assert(testComparisons(cs1, cs2, false, true));
+  }
+  return 0;
 }
diff --git a/libcxx/test/std/containers/associative/multiset/multiset.observers/comp.pass.cpp b/libcxx/test/std/containers/associative/multiset/multiset.observers/comp.pass.cpp
index ad2a304be703b..63647a052c041 100644
--- a/libcxx/test/std/containers/associative/multiset/multiset.observers/comp.pass.cpp
+++ b/libcxx/test/std/containers/associative/multiset/multiset.observers/comp.pass.cpp
@@ -15,19 +15,19 @@
 #include <cassert>
 
 int main(int, char**) {
-    typedef std::multiset<int> set_type;
+  typedef std::multiset<int> set_type;
 
-    set_type s;
-    set_type::iterator i1 = s.insert(1);
-    set_type::iterator i2 = s.insert(2);
+  set_type s;
+  set_type::iterator i1 = s.insert(1);
+  set_type::iterator i2 = s.insert(2);
 
-    const set_type& cs = s;
+  const set_type& cs = s;
 
-    assert(cs.key_comp()(*i1, *i2));
-    assert(!cs.key_comp()(*i2, *i1));
+  assert(cs.key_comp()(*i1, *i2));
+  assert(!cs.key_comp()(*i2, *i1));
 
-    assert(cs.value_comp()(*i1, *i2));
-    assert(!cs.value_comp()(*i2, *i1));
+  assert(cs.value_comp()(*i1, *i2));
+  assert(!cs.value_comp()(*i2, *i1));
 
-    return 0;
+  return 0;
 }
diff --git a/libcxx/test/std/containers/associative/multiset/multiset.special/member_swap.pass.cpp b/libcxx/test/std/containers/associative/multiset/multiset.special/member_swap.pass.cpp
index 7e7b097fdff16..af32e1882093d 100644
--- a/libcxx/test/std/containers/associative/multiset/multiset.special/member_swap.pass.cpp
+++ b/libcxx/test/std/containers/associative/multiset/multiset.special/member_swap.pass.cpp
@@ -18,161 +18,96 @@
 #include "test_macros.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
+int main(int, char**) {
+  {
     typedef int V;
     typedef std::multiset<int> M;
     {
-        M m1;
-        M m2;
-        M m1_save = m1;
-        M m2_save = m2;
-        m1.swap(m2);
-        assert(m1 == m2_save);
-        assert(m2 == m1_save);
+      M m1;
+      M m2;
+      M m1_save = m1;
+      M m2_save = m2;
+      m1.swap(m2);
+      assert(m1 == m2_save);
+      assert(m2 == m1_save);
     }
     {
-        V ar2[] =
-        {
-            5,
-            6,
-            7,
-            8,
-            9,
-            10,
-            11,
-            12
-        };
-        M m1;
-        M m2(ar2, ar2+sizeof(ar2)/sizeof(ar2[0]));
-        M m1_save = m1;
-        M m2_save = m2;
-        m1.swap(m2);
-        assert(m1 == m2_save);
-        assert(m2 == m1_save);
+      V ar2[] = {5, 6, 7, 8, 9, 10, 11, 12};
+      M m1;
+      M m2(ar2, ar2 + sizeof(ar2) / sizeof(ar2[0]));
+      M m1_save = m1;
+      M m2_save = m2;
+      m1.swap(m2);
+      assert(m1 == m2_save);
+      assert(m2 == m1_save);
     }
     {
-        V ar1[] =
-        {
-            1,
-            2,
-            3,
-            4
-        };
-        M m1(ar1, ar1+sizeof(ar1)/sizeof(ar1[0]));
-        M m2;
-        M m1_save = m1;
-        M m2_save = m2;
-        m1.swap(m2);
-        assert(m1 == m2_save);
-        assert(m2 == m1_save);
+      V ar1[] = {1, 2, 3, 4};
+      M m1(ar1, ar1 + sizeof(ar1) / sizeof(ar1[0]));
+      M m2;
+      M m1_save = m1;
+      M m2_save = m2;
+      m1.swap(m2);
+      assert(m1 == m2_save);
+      assert(m2 == m1_save);
     }
     {
-        V ar1[] =
-        {
-            1,
-            2,
-            3,
-            4
-        };
-        V ar2[] =
-        {
-            5,
-            6,
-            7,
-            8,
-            9,
-            10,
-            11,
-            12
-        };
-        M m1(ar1, ar1+sizeof(ar1)/sizeof(ar1[0]));
-        M m2(ar2, ar2+sizeof(ar2)/sizeof(ar2[0]));
-        M m1_save = m1;
-        M m2_save = m2;
-        m1.swap(m2);
-        assert(m1 == m2_save);
-        assert(m2 == m1_save);
-    }
+      V ar1[] = {1, 2, 3, 4};
+      V ar2[] = {5, 6, 7, 8, 9, 10, 11, 12};
+      M m1(ar1, ar1 + sizeof(ar1) / sizeof(ar1[0]));
+      M m2(ar2, ar2 + sizeof(ar2) / sizeof(ar2[0]));
+      M m1_save = m1;
+      M m2_save = m2;
+      m1.swap(m2);
+      assert(m1 == m2_save);
+      assert(m2 == m1_save);
     }
+  }
 #if TEST_STD_VER >= 11
-    {
+  {
     typedef int V;
     typedef std::multiset<int, std::less<int>, min_allocator<int>> M;
     {
-        M m1;
-        M m2;
-        M m1_save = m1;
-        M m2_save = m2;
-        m1.swap(m2);
-        assert(m1 == m2_save);
-        assert(m2 == m1_save);
+      M m1;
+      M m2;
+      M m1_save = m1;
+      M m2_save = m2;
+      m1.swap(m2);
+      assert(m1 == m2_save);
+      assert(m2 == m1_save);
     }
     {
-        V ar2[] =
-        {
-            5,
-            6,
-            7,
-            8,
-            9,
-            10,
-            11,
-            12
-        };
-        M m1;
-        M m2(ar2, ar2+sizeof(ar2)/sizeof(ar2[0]));
-        M m1_save = m1;
-        M m2_save = m2;
-        m1.swap(m2);
-        assert(m1 == m2_save);
-        assert(m2 == m1_save);
+      V ar2[] = {5, 6, 7, 8, 9, 10, 11, 12};
+      M m1;
+      M m2(ar2, ar2 + sizeof(ar2) / sizeof(ar2[0]));
+      M m1_save = m1;
+      M m2_save = m2;
+      m1.swap(m2);
+      assert(m1 == m2_save);
+      assert(m2 == m1_save);
     }
     {
-        V ar1[] =
-        {
-            1,
-            2,
-            3,
-            4
-        };
-        M m1(ar1, ar1+sizeof(ar1)/sizeof(ar1[0]));
-        M m2;
-        M m1_save = m1;
-        M m2_save = m2;
-        m1.swap(m2);
-        assert(m1 == m2_save);
-        assert(m2 == m1_save);
+      V ar1[] = {1, 2, 3, 4};
+      M m1(ar1, ar1 + sizeof(ar1) / sizeof(ar1[0]));
+      M m2;
+      M m1_save = m1;
+      M m2_save = m2;
+      m1.swap(m2);
+      assert(m1 == m2_save);
+      assert(m2 == m1_save);
     }
     {
-        V ar1[] =
-        {
-            1,
-            2,
-            3,
-            4
-        };
-        V ar2[] =
-        {
-            5,
-            6,
-            7,
-            8,
-            9,
-            10,
-            11,
-            12
-        };
-        M m1(ar1, ar1+sizeof(ar1)/sizeof(ar1[0]));
-        M m2(ar2, ar2+sizeof(ar2)/sizeof(ar2[0]));
-        M m1_save = m1;
-        M m2_save = m2;
-        m1.swap(m2);
-        assert(m1 == m2_save);
-        assert(m2 == m1_save);
-    }
+      V ar1[] = {1, 2, 3, 4};
+      V ar2[] = {5, 6, 7, 8, 9, 10, 11, 12};
+      M m1(ar1, ar1 + sizeof(ar1) / sizeof(ar1[0]));
+      M m2(ar2, ar2 + sizeof(ar2) / sizeof(ar2[0]));
+      M m1_save = m1;
+      M m2_save = m2;
+      m1.swap(m2);
+      assert(m1 == m2_save);
+      assert(m2 == m1_save);
     }
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/associative/multiset/multiset.special/non_member_swap.pass.cpp b/libcxx/test/std/containers/associative/multiset/multiset.special/non_member_swap.pass.cpp
index e77a3b8d053db..53630b3ca0a01 100644
--- a/libcxx/test/std/containers/associative/multiset/multiset.special/non_member_swap.pass.cpp
+++ b/libcxx/test/std/containers/associative/multiset/multiset.special/non_member_swap.pass.cpp
@@ -18,152 +18,87 @@
 #include "test_allocator.h"
 #include "../../../test_compare.h"
 
-int main(int, char**)
-{
-    typedef int V;
-    {
+int main(int, char**) {
+  typedef int V;
+  {
     typedef std::multiset<int> M;
     {
-        M m1;
-        M m2;
-        M m1_save = m1;
-        M m2_save = m2;
-        swap(m1, m2);
-        assert(m1 == m2_save);
-        assert(m2 == m1_save);
-    }
-    {
-        V ar2[] =
-        {
-            5,
-            6,
-            7,
-            8,
-            9,
-            10,
-            11,
-            12
-        };
-        M m1;
-        M m2(ar2, ar2+sizeof(ar2)/sizeof(ar2[0]));
-        M m1_save = m1;
-        M m2_save = m2;
-        swap(m1, m2);
-        assert(m1 == m2_save);
-        assert(m2 == m1_save);
+      M m1;
+      M m2;
+      M m1_save = m1;
+      M m2_save = m2;
+      swap(m1, m2);
+      assert(m1 == m2_save);
+      assert(m2 == m1_save);
     }
     {
-        V ar1[] =
-        {
-            1,
-            2,
-            3,
-            4
-        };
-        M m1(ar1, ar1+sizeof(ar1)/sizeof(ar1[0]));
-        M m2;
-        M m1_save = m1;
-        M m2_save = m2;
-        swap(m1, m2);
-        assert(m1 == m2_save);
-        assert(m2 == m1_save);
-    }
-    {
-        V ar1[] =
-        {
-            1,
-            2,
-            3,
-            4
-        };
-        V ar2[] =
-        {
-            5,
-            6,
-            7,
-            8,
-            9,
-            10,
-            11,
-            12
-        };
-        M m1(ar1, ar1+sizeof(ar1)/sizeof(ar1[0]));
-        M m2(ar2, ar2+sizeof(ar2)/sizeof(ar2[0]));
-        M m1_save = m1;
-        M m2_save = m2;
-        swap(m1, m2);
-        assert(m1 == m2_save);
-        assert(m2 == m1_save);
-    }
+      V ar2[] = {5, 6, 7, 8, 9, 10, 11, 12};
+      M m1;
+      M m2(ar2, ar2 + sizeof(ar2) / sizeof(ar2[0]));
+      M m1_save = m1;
+      M m2_save = m2;
+      swap(m1, m2);
+      assert(m1 == m2_save);
+      assert(m2 == m1_save);
     }
     {
-        typedef test_allocator<V> A;
-        typedef test_less<int> C;
-        typedef std::set<int, C, A> M;
-        V ar1[] =
-        {
-            1,
-            2,
-            3,
-            4
-        };
-        V ar2[] =
-        {
-            5,
-            6,
-            7,
-            8,
-            9,
-            10,
-            11,
-            12
-        };
-        M m1(ar1, ar1+sizeof(ar1)/sizeof(ar1[0]), C(1), A(1, 1));
-        M m2(ar2, ar2+sizeof(ar2)/sizeof(ar2[0]), C(2), A(1, 2));
-        M m1_save = m1;
-        M m2_save = m2;
-        swap(m1, m2);
-        assert(m1 == m2_save);
-        assert(m2 == m1_save);
-        assert(m1.key_comp() == C(2));
-        assert(m1.get_allocator().get_id() == 1);
-        assert(m2.key_comp() == C(1));
-        assert(m2.get_allocator().get_id() == 2);
+      V ar1[] = {1, 2, 3, 4};
+      M m1(ar1, ar1 + sizeof(ar1) / sizeof(ar1[0]));
+      M m2;
+      M m1_save = m1;
+      M m2_save = m2;
+      swap(m1, m2);
+      assert(m1 == m2_save);
+      assert(m2 == m1_save);
     }
     {
-        typedef other_allocator<V> A;
-        typedef test_less<int> C;
-        typedef std::set<int, C, A> M;
-        V ar1[] =
-        {
-            1,
-            2,
-            3,
-            4
-        };
-        V ar2[] =
-        {
-            5,
-            6,
-            7,
-            8,
-            9,
-            10,
-            11,
-            12
-        };
-        M m1(ar1, ar1+sizeof(ar1)/sizeof(ar1[0]), C(1), A(1));
-        M m2(ar2, ar2+sizeof(ar2)/sizeof(ar2[0]), C(2), A(2));
-        M m1_save = m1;
-        M m2_save = m2;
-        swap(m1, m2);
-        assert(m1 == m2_save);
-        assert(m2 == m1_save);
-        assert(m1.key_comp() == C(2));
-        assert(m1.get_allocator() == A(2));
-        assert(m2.key_comp() == C(1));
-        assert(m2.get_allocator() == A(1));
+      V ar1[] = {1, 2, 3, 4};
+      V ar2[] = {5, 6, 7, 8, 9, 10, 11, 12};
+      M m1(ar1, ar1 + sizeof(ar1) / sizeof(ar1[0]));
+      M m2(ar2, ar2 + sizeof(ar2) / sizeof(ar2[0]));
+      M m1_save = m1;
+      M m2_save = m2;
+      swap(m1, m2);
+      assert(m1 == m2_save);
+      assert(m2 == m1_save);
     }
+  }
+  {
+    typedef test_allocator<V> A;
+    typedef test_less<int> C;
+    typedef std::set<int, C, A> M;
+    V ar1[] = {1, 2, 3, 4};
+    V ar2[] = {5, 6, 7, 8, 9, 10, 11, 12};
+    M m1(ar1, ar1 + sizeof(ar1) / sizeof(ar1[0]), C(1), A(1, 1));
+    M m2(ar2, ar2 + sizeof(ar2) / sizeof(ar2[0]), C(2), A(1, 2));
+    M m1_save = m1;
+    M m2_save = m2;
+    swap(m1, m2);
+    assert(m1 == m2_save);
+    assert(m2 == m1_save);
+    assert(m1.key_comp() == C(2));
+    assert(m1.get_allocator().get_id() == 1);
+    assert(m2.key_comp() == C(1));
+    assert(m2.get_allocator().get_id() == 2);
+  }
+  {
+    typedef other_allocator<V> A;
+    typedef test_less<int> C;
+    typedef std::set<int, C, A> M;
+    V ar1[] = {1, 2, 3, 4};
+    V ar2[] = {5, 6, 7, 8, 9, 10, 11, 12};
+    M m1(ar1, ar1 + sizeof(ar1) / sizeof(ar1[0]), C(1), A(1));
+    M m2(ar2, ar2 + sizeof(ar2) / sizeof(ar2[0]), C(2), A(2));
+    M m1_save = m1;
+    M m2_save = m2;
+    swap(m1, m2);
+    assert(m1 == m2_save);
+    assert(m2 == m1_save);
+    assert(m1.key_comp() == C(2));
+    assert(m1.get_allocator() == A(2));
+    assert(m2.key_comp() == C(1));
+    assert(m2.get_allocator() == A(1));
+  }
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/associative/multiset/multiset.special/swap_noexcept.pass.cpp b/libcxx/test/std/containers/associative/multiset/multiset.special/swap_noexcept.pass.cpp
index 467be8e7dafd5..7a7ccf87e9de9 100644
--- a/libcxx/test/std/containers/associative/multiset/multiset.special/swap_noexcept.pass.cpp
+++ b/libcxx/test/std/containers/associative/multiset/multiset.special/swap_noexcept.pass.cpp
@@ -29,23 +29,21 @@
 #include "test_allocator.h"
 
 template <class T>
-struct some_comp
-{
-    typedef T value_type;
+struct some_comp {
+  typedef T value_type;
 
-    some_comp() {}
-    some_comp(const some_comp&) {}
-    bool operator()(const T&, const T&) const { return false; }
+  some_comp() {}
+  some_comp(const some_comp&) {}
+  bool operator()(const T&, const T&) const { return false; }
 };
 
 template <class T>
-struct some_comp2
-{
-    typedef T value_type;
+struct some_comp2 {
+  typedef T value_type;
 
-    some_comp2() {}
-    some_comp2(const some_comp2&) {}
-    bool operator()(const T&, const T&) const { return false; }
+  some_comp2() {}
+  some_comp2(const some_comp2&) {}
+  bool operator()(const T&, const T&) const { return false; }
 };
 
 #if TEST_STD_VER >= 14
@@ -54,87 +52,83 @@ void swap(some_comp2<T>&, some_comp2<T>&) noexcept {}
 #endif
 
 template <class T>
-struct some_alloc
-{
-    typedef T value_type;
+struct some_alloc {
+  typedef T value_type;
 
-    some_alloc() {}
-    some_alloc(const some_alloc&);
-    void deallocate(void*, unsigned) {}
+  some_alloc() {}
+  some_alloc(const some_alloc&);
+  void deallocate(void*, unsigned) {}
 
-    typedef std::true_type propagate_on_container_swap;
+  typedef std::true_type propagate_on_container_swap;
 };
 
 template <class T>
-struct some_alloc2
-{
-    typedef T value_type;
+struct some_alloc2 {
+  typedef T value_type;
 
-    some_alloc2() {}
-    some_alloc2(const some_alloc2&);
-    void deallocate(void*, unsigned) {}
+  some_alloc2() {}
+  some_alloc2(const some_alloc2&);
+  void deallocate(void*, unsigned) {}
 
-    typedef std::false_type propagate_on_container_swap;
-    typedef std::true_type is_always_equal;
+  typedef std::false_type propagate_on_container_swap;
+  typedef std::true_type is_always_equal;
 };
 
 template <class T>
-struct some_alloc3
-{
-    typedef T value_type;
+struct some_alloc3 {
+  typedef T value_type;
 
-    some_alloc3() {}
-    some_alloc3(const some_alloc3&);
-    void deallocate(void*, unsigned) {}
+  some_alloc3() {}
+  some_alloc3(const some_alloc3&);
+  void deallocate(void*, unsigned) {}
 
-    typedef std::false_type propagate_on_container_swap;
-    typedef std::false_type is_always_equal;
+  typedef std::false_type propagate_on_container_swap;
+  typedef std::false_type is_always_equal;
 };
 
-int main(int, char**)
-{
-    {
-        typedef std::multiset<MoveOnly> C;
-        static_assert(noexcept(swap(std::declval<C&>(), std::declval<C&>())), "");
-    }
+int main(int, char**) {
+  {
+    typedef std::multiset<MoveOnly> C;
+    static_assert(noexcept(swap(std::declval<C&>(), std::declval<C&>())), "");
+  }
 #if defined(_LIBCPP_VERSION)
-    {
-        typedef std::multiset<MoveOnly, std::less<MoveOnly>, test_allocator<MoveOnly>> C;
-        static_assert(noexcept(swap(std::declval<C&>(), std::declval<C&>())), "");
-    }
-    {
-        typedef std::multiset<MoveOnly, std::less<MoveOnly>, other_allocator<MoveOnly>> C;
-        static_assert(noexcept(swap(std::declval<C&>(), std::declval<C&>())), "");
-    }
+  {
+    typedef std::multiset<MoveOnly, std::less<MoveOnly>, test_allocator<MoveOnly>> C;
+    static_assert(noexcept(swap(std::declval<C&>(), std::declval<C&>())), "");
+  }
+  {
+    typedef std::multiset<MoveOnly, std::less<MoveOnly>, other_allocator<MoveOnly>> C;
+    static_assert(noexcept(swap(std::declval<C&>(), std::declval<C&>())), "");
+  }
 #endif // _LIBCPP_VERSION
-    {
-        typedef std::multiset<MoveOnly, some_comp<MoveOnly>> C;
-        static_assert(!noexcept(swap(std::declval<C&>(), std::declval<C&>())), "");
-    }
+  {
+    typedef std::multiset<MoveOnly, some_comp<MoveOnly>> C;
+    static_assert(!noexcept(swap(std::declval<C&>(), std::declval<C&>())), "");
+  }
 
 #if TEST_STD_VER >= 14
-    { // POCS allocator, throwable swap for comp
-    typedef std::multiset<MoveOnly, some_comp <MoveOnly>, some_alloc <MoveOnly>> C;
+  { // POCS allocator, throwable swap for comp
+    typedef std::multiset<MoveOnly, some_comp<MoveOnly>, some_alloc<MoveOnly>> C;
     static_assert(!noexcept(swap(std::declval<C&>(), std::declval<C&>())), "");
-    }
-    { // always equal allocator, throwable swap for comp
-    typedef std::multiset<MoveOnly, some_comp <MoveOnly>, some_alloc2<MoveOnly>> C;
+  }
+  { // always equal allocator, throwable swap for comp
+    typedef std::multiset<MoveOnly, some_comp<MoveOnly>, some_alloc2<MoveOnly>> C;
     static_assert(!noexcept(swap(std::declval<C&>(), std::declval<C&>())), "");
-    }
-    { // POCS allocator, nothrow swap for comp
-    typedef std::multiset<MoveOnly, some_comp2<MoveOnly>, some_alloc <MoveOnly>> C;
-    static_assert( noexcept(swap(std::declval<C&>(), std::declval<C&>())), "");
-    }
-    { // always equal allocator, nothrow swap for comp
+  }
+  { // POCS allocator, nothrow swap for comp
+    typedef std::multiset<MoveOnly, some_comp2<MoveOnly>, some_alloc<MoveOnly>> C;
+    static_assert(noexcept(swap(std::declval<C&>(), std::declval<C&>())), "");
+  }
+  { // always equal allocator, nothrow swap for comp
     typedef std::multiset<MoveOnly, some_comp2<MoveOnly>, some_alloc2<MoveOnly>> C;
-    static_assert( noexcept(swap(std::declval<C&>(), std::declval<C&>())), "");
-    }
-#if defined(_LIBCPP_VERSION)
-    { // NOT always equal allocator, nothrow swap for comp
+    static_assert(noexcept(swap(std::declval<C&>(), std::declval<C&>())), "");
+  }
+#  if defined(_LIBCPP_VERSION)
+  { // NOT always equal allocator, nothrow swap for comp
     typedef std::multiset<MoveOnly, some_comp2<MoveOnly>, some_alloc3<MoveOnly>> C;
-    static_assert( noexcept(swap(std::declval<C&>(), std::declval<C&>())), "");
-    }
-#endif // _LIBCPP_VERSION
+    static_assert(noexcept(swap(std::declval<C&>(), std::declval<C&>())), "");
+  }
+#  endif // _LIBCPP_VERSION
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/associative/multiset/scary.pass.cpp b/libcxx/test/std/containers/associative/multiset/scary.pass.cpp
index fc4efd46edee3..4d30c27733647 100644
--- a/libcxx/test/std/containers/associative/multiset/scary.pass.cpp
+++ b/libcxx/test/std/containers/associative/multiset/scary.pass.cpp
@@ -16,13 +16,12 @@
 
 #include "test_macros.h"
 
-int main(int, char**)
-{
-    typedef std::set<int> M1;
-    typedef std::multiset<int> M2;
-    M2::iterator i;
-    M1::iterator j = i;
-    ((void)j);
+int main(int, char**) {
+  typedef std::set<int> M1;
+  typedef std::multiset<int> M2;
+  M2::iterator i;
+  M1::iterator j = i;
+  ((void)j);
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/associative/multiset/size.pass.cpp b/libcxx/test/std/containers/associative/multiset/size.pass.cpp
index d0350251dc38a..405437211b8ca 100644
--- a/libcxx/test/std/containers/associative/multiset/size.pass.cpp
+++ b/libcxx/test/std/containers/associative/multiset/size.pass.cpp
@@ -18,9 +18,8 @@
 #include "test_macros.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
+int main(int, char**) {
+  {
     typedef std::multiset<int> M;
     M m;
     assert(m.size() == 0);
@@ -36,9 +35,9 @@ int main(int, char**)
     assert(m.size() == 1);
     m.erase(m.begin());
     assert(m.size() == 0);
-    }
+  }
 #if TEST_STD_VER >= 11
-    {
+  {
     typedef std::multiset<int, std::less<int>, min_allocator<int>> M;
     M m;
     assert(m.size() == 0);
@@ -54,7 +53,7 @@ int main(int, char**)
     assert(m.size() == 1);
     m.erase(m.begin());
     assert(m.size() == 0);
-    }
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/associative/multiset/types.pass.cpp b/libcxx/test/std/containers/associative/multiset/types.pass.cpp
index a6d0dd58100bb..3595f7a8b5e0d 100644
--- a/libcxx/test/std/containers/associative/multiset/types.pass.cpp
+++ b/libcxx/test/std/containers/associative/multiset/types.pass.cpp
@@ -34,9 +34,8 @@
 #include "test_macros.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
+int main(int, char**) {
+  {
     typedef std::multiset<int> C;
     static_assert((std::is_same<C::key_type, int>::value), "");
     static_assert((std::is_same<C::value_type, int>::value), "");
@@ -49,9 +48,9 @@ int main(int, char**)
     static_assert((std::is_same<C::const_pointer, const int*>::value), "");
     static_assert((std::is_same<C::size_type, std::size_t>::value), "");
     static_assert((std::is_same<C::difference_type, std::ptrdiff_t>::value), "");
-    }
+  }
 #if TEST_STD_VER >= 11
-    {
+  {
     typedef std::multiset<int, std::less<int>, min_allocator<int>> C;
     static_assert((std::is_same<C::key_type, int>::value), "");
     static_assert((std::is_same<C::value_type, int>::value), "");
@@ -62,10 +61,10 @@ int main(int, char**)
     static_assert((std::is_same<C::const_reference, const int&>::value), "");
     static_assert((std::is_same<C::pointer, min_pointer<int>>::value), "");
     static_assert((std::is_same<C::const_pointer, min_pointer<const int>>::value), "");
-//  min_allocator doesn't have a size_type, so one gets synthesized
+    //  min_allocator doesn't have a size_type, so one gets synthesized
     static_assert((std::is_same<C::size_type, std::make_unsigned<C::difference_type>::type>::value), "");
     static_assert((std::is_same<C::difference_type, std::ptrdiff_t>::value), "");
-    }
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/associative/multiset/upper_bound.pass.cpp b/libcxx/test/std/containers/associative/multiset/upper_bound.pass.cpp
index 05ac595ca42cd..c61859b1feffb 100644
--- a/libcxx/test/std/containers/associative/multiset/upper_bound.pass.cpp
+++ b/libcxx/test/std/containers/associative/multiset/upper_bound.pass.cpp
@@ -20,157 +20,101 @@
 #include "min_allocator.h"
 #include "private_constructor.h"
 
-int main(int, char**)
-{
-    {
+int main(int, char**) {
+  {
     typedef int V;
     typedef std::multiset<int> M;
     {
-        typedef M::iterator R;
-        V ar[] =
-        {
-            5,
-            5,
-            5,
-            7,
-            7,
-            7,
-            9,
-            9,
-            9
-        };
-        M m(ar, ar+sizeof(ar)/sizeof(ar[0]));
-        R r = m.upper_bound(4);
-        assert(r == std::next(m.begin(), 0));
-        r = m.upper_bound(5);
-        assert(r == std::next(m.begin(), 3));
-        r = m.upper_bound(6);
-        assert(r == std::next(m.begin(), 3));
-        r = m.upper_bound(7);
-        assert(r == std::next(m.begin(), 6));
-        r = m.upper_bound(8);
-        assert(r == std::next(m.begin(), 6));
-        r = m.upper_bound(9);
-        assert(r == std::next(m.begin(), 9));
-        r = m.upper_bound(11);
-        assert(r == std::next(m.begin(), 9));
+      typedef M::iterator R;
+      V ar[] = {5, 5, 5, 7, 7, 7, 9, 9, 9};
+      M m(ar, ar + sizeof(ar) / sizeof(ar[0]));
+      R r = m.upper_bound(4);
+      assert(r == std::next(m.begin(), 0));
+      r = m.upper_bound(5);
+      assert(r == std::next(m.begin(), 3));
+      r = m.upper_bound(6);
+      assert(r == std::next(m.begin(), 3));
+      r = m.upper_bound(7);
+      assert(r == std::next(m.begin(), 6));
+      r = m.upper_bound(8);
+      assert(r == std::next(m.begin(), 6));
+      r = m.upper_bound(9);
+      assert(r == std::next(m.begin(), 9));
+      r = m.upper_bound(11);
+      assert(r == std::next(m.begin(), 9));
     }
     {
-        typedef M::const_iterator R;
-        V ar[] =
-        {
-            5,
-            5,
-            5,
-            7,
-            7,
-            7,
-            9,
-            9,
-            9
-        };
-        const M m(ar, ar+sizeof(ar)/sizeof(ar[0]));
-        R r = m.upper_bound(4);
-        assert(r == std::next(m.begin(), 0));
-        r = m.upper_bound(5);
-        assert(r == std::next(m.begin(), 3));
-        r = m.upper_bound(6);
-        assert(r == std::next(m.begin(), 3));
-        r = m.upper_bound(7);
-        assert(r == std::next(m.begin(), 6));
-        r = m.upper_bound(8);
-        assert(r == std::next(m.begin(), 6));
-        r = m.upper_bound(9);
-        assert(r == std::next(m.begin(), 9));
-        r = m.upper_bound(11);
-        assert(r == std::next(m.begin(), 9));
-    }
+      typedef M::const_iterator R;
+      V ar[] = {5, 5, 5, 7, 7, 7, 9, 9, 9};
+      const M m(ar, ar + sizeof(ar) / sizeof(ar[0]));
+      R r = m.upper_bound(4);
+      assert(r == std::next(m.begin(), 0));
+      r = m.upper_bound(5);
+      assert(r == std::next(m.begin(), 3));
+      r = m.upper_bound(6);
+      assert(r == std::next(m.begin(), 3));
+      r = m.upper_bound(7);
+      assert(r == std::next(m.begin(), 6));
+      r = m.upper_bound(8);
+      assert(r == std::next(m.begin(), 6));
+      r = m.upper_bound(9);
+      assert(r == std::next(m.begin(), 9));
+      r = m.upper_bound(11);
+      assert(r == std::next(m.begin(), 9));
     }
+  }
 #if TEST_STD_VER >= 11
-    {
+  {
     typedef int V;
     typedef std::multiset<int, std::less<int>, min_allocator<int>> M;
     {
-        typedef M::iterator R;
-        V ar[] =
-        {
-            5,
-            5,
-            5,
-            7,
-            7,
-            7,
-            9,
-            9,
-            9
-        };
-        M m(ar, ar+sizeof(ar)/sizeof(ar[0]));
-        R r = m.upper_bound(4);
-        assert(r == std::next(m.begin(), 0));
-        r = m.upper_bound(5);
-        assert(r == std::next(m.begin(), 3));
-        r = m.upper_bound(6);
-        assert(r == std::next(m.begin(), 3));
-        r = m.upper_bound(7);
-        assert(r == std::next(m.begin(), 6));
-        r = m.upper_bound(8);
-        assert(r == std::next(m.begin(), 6));
-        r = m.upper_bound(9);
-        assert(r == std::next(m.begin(), 9));
-        r = m.upper_bound(11);
-        assert(r == std::next(m.begin(), 9));
+      typedef M::iterator R;
+      V ar[] = {5, 5, 5, 7, 7, 7, 9, 9, 9};
+      M m(ar, ar + sizeof(ar) / sizeof(ar[0]));
+      R r = m.upper_bound(4);
+      assert(r == std::next(m.begin(), 0));
+      r = m.upper_bound(5);
+      assert(r == std::next(m.begin(), 3));
+      r = m.upper_bound(6);
+      assert(r == std::next(m.begin(), 3));
+      r = m.upper_bound(7);
+      assert(r == std::next(m.begin(), 6));
+      r = m.upper_bound(8);
+      assert(r == std::next(m.begin(), 6));
+      r = m.upper_bound(9);
+      assert(r == std::next(m.begin(), 9));
+      r = m.upper_bound(11);
+      assert(r == std::next(m.begin(), 9));
     }
     {
-        typedef M::const_iterator R;
-        V ar[] =
-        {
-            5,
-            5,
-            5,
-            7,
-            7,
-            7,
-            9,
-            9,
-            9
-        };
-        const M m(ar, ar+sizeof(ar)/sizeof(ar[0]));
-        R r = m.upper_bound(4);
-        assert(r == std::next(m.begin(), 0));
-        r = m.upper_bound(5);
-        assert(r == std::next(m.begin(), 3));
-        r = m.upper_bound(6);
-        assert(r == std::next(m.begin(), 3));
-        r = m.upper_bound(7);
-        assert(r == std::next(m.begin(), 6));
-        r = m.upper_bound(8);
-        assert(r == std::next(m.begin(), 6));
-        r = m.upper_bound(9);
-        assert(r == std::next(m.begin(), 9));
-        r = m.upper_bound(11);
-        assert(r == std::next(m.begin(), 9));
-    }
+      typedef M::const_iterator R;
+      V ar[] = {5, 5, 5, 7, 7, 7, 9, 9, 9};
+      const M m(ar, ar + sizeof(ar) / sizeof(ar[0]));
+      R r = m.upper_bound(4);
+      assert(r == std::next(m.begin(), 0));
+      r = m.upper_bound(5);
+      assert(r == std::next(m.begin(), 3));
+      r = m.upper_bound(6);
+      assert(r == std::next(m.begin(), 3));
+      r = m.upper_bound(7);
+      assert(r == std::next(m.begin(), 6));
+      r = m.upper_bound(8);
+      assert(r == std::next(m.begin(), 6));
+      r = m.upper_bound(9);
+      assert(r == std::next(m.begin(), 9));
+      r = m.upper_bound(11);
+      assert(r == std::next(m.begin(), 9));
     }
+  }
 #endif
 #if TEST_STD_VER > 11
-    {
+  {
     typedef int V;
     typedef std::multiset<V, std::less<>> M;
 
     typedef M::iterator R;
-    V ar[] =
-    {
-        5,
-        5,
-        5,
-        7,
-        7,
-        7,
-        9,
-        9,
-        9
-    };
-    M m(ar, ar+sizeof(ar)/sizeof(ar[0]));
+    V ar[] = {5, 5, 5, 7, 7, 7, 9, 9, 9};
+    M m(ar, ar + sizeof(ar) / sizeof(ar[0]));
     R r = m.upper_bound(4);
     assert(r == std::next(m.begin(), 0));
     r = m.upper_bound(5);
@@ -185,23 +129,23 @@ int main(int, char**)
     assert(r == std::next(m.begin(), 9));
     r = m.upper_bound(11);
     assert(r == std::next(m.begin(), 9));
-    }
+  }
 
-    {
+  {
     typedef PrivateConstructor V;
     typedef std::multiset<V, std::less<>> M;
 
     typedef M::iterator R;
     M m;
-    m.insert ( V::make ( 5 ));
-    m.insert ( V::make ( 5 ));
-    m.insert ( V::make ( 5 ));
-    m.insert ( V::make ( 7 ));
-    m.insert ( V::make ( 7 ));
-    m.insert ( V::make ( 7 ));
-    m.insert ( V::make ( 9 ));
-    m.insert ( V::make ( 9 ));
-    m.insert ( V::make ( 9 ));
+    m.insert(V::make(5));
+    m.insert(V::make(5));
+    m.insert(V::make(5));
+    m.insert(V::make(7));
+    m.insert(V::make(7));
+    m.insert(V::make(7));
+    m.insert(V::make(9));
+    m.insert(V::make(9));
+    m.insert(V::make(9));
 
     R r = m.upper_bound(4);
     assert(r == std::next(m.begin(), 0));
@@ -217,7 +161,7 @@ int main(int, char**)
     assert(r == std::next(m.begin(), 9));
     r = m.upper_bound(11);
     assert(r == std::next(m.begin(), 9));
-    }
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/associative/set/allocator_mismatch.verify.cpp b/libcxx/test/std/containers/associative/set/allocator_mismatch.verify.cpp
index 279a07946f005..e36f7743d2cf6 100644
--- a/libcxx/test/std/containers/associative/set/allocator_mismatch.verify.cpp
+++ b/libcxx/test/std/containers/associative/set/allocator_mismatch.verify.cpp
@@ -12,4 +12,4 @@
 #include <set>
 
 std::set<int, std::less<int>, std::allocator<long> > m;
-  // expected-error-re@*:* {{static assertion failed{{.*}}Allocator::value_type must be same type as value_type}}
+// expected-error-re@*:* {{static assertion failed{{.*}}Allocator::value_type must be same type as value_type}}
diff --git a/libcxx/test/std/containers/associative/set/clear.pass.cpp b/libcxx/test/std/containers/associative/set/clear.pass.cpp
index 0650e912fd35c..a53b6a86a660e 100644
--- a/libcxx/test/std/containers/associative/set/clear.pass.cpp
+++ b/libcxx/test/std/containers/associative/set/clear.pass.cpp
@@ -18,49 +18,28 @@
 #include "test_macros.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef std::set<int> M;
-        typedef int V;
-        V ar[] =
-        {
-            1,
-            2,
-            3,
-            4,
-            5,
-            6,
-            7,
-            8
-        };
-        M m(ar, ar + sizeof(ar)/sizeof(ar[0]));
-        assert(m.size() == 8);
-        ASSERT_NOEXCEPT(m.clear());
-        m.clear();
-        assert(m.size() == 0);
-    }
+int main(int, char**) {
+  {
+    typedef std::set<int> M;
+    typedef int V;
+    V ar[] = {1, 2, 3, 4, 5, 6, 7, 8};
+    M m(ar, ar + sizeof(ar) / sizeof(ar[0]));
+    assert(m.size() == 8);
+    ASSERT_NOEXCEPT(m.clear());
+    m.clear();
+    assert(m.size() == 0);
+  }
 #if TEST_STD_VER >= 11
-    {
-        typedef std::set<int, std::less<int>, min_allocator<int>> M;
-        typedef int V;
-        V ar[] =
-        {
-            1,
-            2,
-            3,
-            4,
-            5,
-            6,
-            7,
-            8
-        };
-        M m(ar, ar + sizeof(ar)/sizeof(ar[0]));
-        assert(m.size() == 8);
-        ASSERT_NOEXCEPT(m.clear());
-        m.clear();
-        assert(m.size() == 0);
-    }
+  {
+    typedef std::set<int, std::less<int>, min_allocator<int>> M;
+    typedef int V;
+    V ar[] = {1, 2, 3, 4, 5, 6, 7, 8};
+    M m(ar, ar + sizeof(ar) / sizeof(ar[0]));
+    assert(m.size() == 8);
+    ASSERT_NOEXCEPT(m.clear());
+    m.clear();
+    assert(m.size() == 0);
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/associative/set/contains.pass.cpp b/libcxx/test/std/containers/associative/set/contains.pass.cpp
index d595e380dffca..93e6ff0a28885 100644
--- a/libcxx/test/std/containers/associative/set/contains.pass.cpp
+++ b/libcxx/test/std/containers/associative/set/contains.pass.cpp
@@ -17,27 +17,32 @@
 
 template <typename T, typename V, typename B, typename... Vals>
 void test(B bad, Vals... args) {
-    T set;
-    V vals[] = {args...};
+  T set;
+  V vals[] = {args...};
 
-    for (auto& v : vals) set.insert(v);
-    for (auto& v : vals) assert(set.contains(v));
+  for (auto& v : vals)
+    set.insert(v);
+  for (auto& v : vals)
+    assert(set.contains(v));
 
-    assert(!set.contains(bad));
+  assert(!set.contains(bad));
 }
 
-struct E { int a = 1; double b = 1; char c = 1; };
-
-int main(int, char**)
-{
-    {
-        test<std::set<int>, int>(14, 10, 11, 12, 13);
-        test<std::set<char>, char>('e', 'a', 'b', 'c', 'd');
-    }
-    {
-        test<std::multiset<int>, int>(14, 10, 11, 12, 13);
-        test<std::multiset<char>, char>('e', 'a', 'b', 'c', 'd');
-    }
-
-    return 0;
+struct E {
+  int a    = 1;
+  double b = 1;
+  char c   = 1;
+};
+
+int main(int, char**) {
+  {
+    test<std::set<int>, int>(14, 10, 11, 12, 13);
+    test<std::set<char>, char>('e', 'a', 'b', 'c', 'd');
+  }
+  {
+    test<std::multiset<int>, int>(14, 10, 11, 12, 13);
+    test<std::multiset<char>, char>('e', 'a', 'b', 'c', 'd');
+  }
+
+  return 0;
 }
diff --git a/libcxx/test/std/containers/associative/set/contains_transparent.pass.cpp b/libcxx/test/std/containers/associative/set/contains_transparent.pass.cpp
index e4caa0b7128b4..c5d6b293e8a4b 100644
--- a/libcxx/test/std/containers/associative/set/contains_transparent.pass.cpp
+++ b/libcxx/test/std/containers/associative/set/contains_transparent.pass.cpp
@@ -21,18 +21,11 @@
 struct Comp {
   using is_transparent = void;
 
-  bool operator()(const std::pair<int, int>& lhs,
-                  const std::pair<int, int>& rhs) const {
-    return lhs < rhs;
-  }
-
-  bool operator()(const std::pair<int, int>& lhs, int rhs) const {
-    return lhs.first < rhs;
-  }
-
-  bool operator()(int lhs, const std::pair<int, int>& rhs) const {
-    return lhs < rhs.first;
-  }
+  bool operator()(const std::pair<int, int>& lhs, const std::pair<int, int>& rhs) const { return lhs < rhs; }
+
+  bool operator()(const std::pair<int, int>& lhs, int rhs) const { return lhs.first < rhs; }
+
+  bool operator()(int lhs, const std::pair<int, int>& rhs) const { return lhs < rhs.first; }
 };
 
 template <typename Container>
diff --git a/libcxx/test/std/containers/associative/set/count.pass.cpp b/libcxx/test/std/containers/associative/set/count.pass.cpp
index 06ddf78b96846..abd1e98f1a4ee 100644
--- a/libcxx/test/std/containers/associative/set/count.pass.cpp
+++ b/libcxx/test/std/containers/associative/set/count.pass.cpp
@@ -19,130 +19,99 @@
 #include "min_allocator.h"
 #include "private_constructor.h"
 
-int main(int, char**)
-{
-    {
-        typedef int V;
-        typedef std::set<int> M;
-        typedef M::size_type R;
-        V ar[] =
-        {
-            5,
-            6,
-            7,
-            8,
-            9,
-            10,
-            11,
-            12
-        };
-        const M m(ar, ar+sizeof(ar)/sizeof(ar[0]));
-        R r = m.count(5);
-        assert(r == 1);
-        r = m.count(6);
-        assert(r == 1);
-        r = m.count(7);
-        assert(r == 1);
-        r = m.count(8);
-        assert(r == 1);
-        r = m.count(9);
-        assert(r == 1);
-        r = m.count(10);
-        assert(r == 1);
-        r = m.count(11);
-        assert(r == 1);
-        r = m.count(12);
-        assert(r == 1);
-        r = m.count(4);
-        assert(r == 0);
-    }
+int main(int, char**) {
+  {
+    typedef int V;
+    typedef std::set<int> M;
+    typedef M::size_type R;
+    V ar[] = {5, 6, 7, 8, 9, 10, 11, 12};
+    const M m(ar, ar + sizeof(ar) / sizeof(ar[0]));
+    R r = m.count(5);
+    assert(r == 1);
+    r = m.count(6);
+    assert(r == 1);
+    r = m.count(7);
+    assert(r == 1);
+    r = m.count(8);
+    assert(r == 1);
+    r = m.count(9);
+    assert(r == 1);
+    r = m.count(10);
+    assert(r == 1);
+    r = m.count(11);
+    assert(r == 1);
+    r = m.count(12);
+    assert(r == 1);
+    r = m.count(4);
+    assert(r == 0);
+  }
 #if TEST_STD_VER >= 11
-    {
-        typedef int V;
-        typedef std::set<int, std::less<int>, min_allocator<int>> M;
-        typedef M::size_type R;
-        V ar[] =
-        {
-            5,
-            6,
-            7,
-            8,
-            9,
-            10,
-            11,
-            12
-        };
-        const M m(ar, ar+sizeof(ar)/sizeof(ar[0]));
-        R r = m.count(5);
-        assert(r == 1);
-        r = m.count(6);
-        assert(r == 1);
-        r = m.count(7);
-        assert(r == 1);
-        r = m.count(8);
-        assert(r == 1);
-        r = m.count(9);
-        assert(r == 1);
-        r = m.count(10);
-        assert(r == 1);
-        r = m.count(11);
-        assert(r == 1);
-        r = m.count(12);
-        assert(r == 1);
-        r = m.count(4);
-        assert(r == 0);
-    }
+  {
+    typedef int V;
+    typedef std::set<int, std::less<int>, min_allocator<int>> M;
+    typedef M::size_type R;
+    V ar[] = {5, 6, 7, 8, 9, 10, 11, 12};
+    const M m(ar, ar + sizeof(ar) / sizeof(ar[0]));
+    R r = m.count(5);
+    assert(r == 1);
+    r = m.count(6);
+    assert(r == 1);
+    r = m.count(7);
+    assert(r == 1);
+    r = m.count(8);
+    assert(r == 1);
+    r = m.count(9);
+    assert(r == 1);
+    r = m.count(10);
+    assert(r == 1);
+    r = m.count(11);
+    assert(r == 1);
+    r = m.count(12);
+    assert(r == 1);
+    r = m.count(4);
+    assert(r == 0);
+  }
 #endif
 #if TEST_STD_VER > 11
-    {
-        typedef int V;
-        typedef std::set<int, std::less<>> M;
-        typedef M::size_type R;
-        V ar[] =
-        {
-            5,
-            6,
-            7,
-            8,
-            9,
-            10,
-            11,
-            12
-        };
-        const M m(ar, ar+sizeof(ar)/sizeof(ar[0]));
-        R r = m.count(5);
-        assert(r == 1);
-        r = m.count(6);
-        assert(r == 1);
-        r = m.count(7);
-        assert(r == 1);
-        r = m.count(8);
-        assert(r == 1);
-        r = m.count(9);
-        assert(r == 1);
-        r = m.count(10);
-        assert(r == 1);
-        r = m.count(11);
-        assert(r == 1);
-        r = m.count(12);
-        assert(r == 1);
-        r = m.count(4);
-        assert(r == 0);
-    }
-    {
+  {
+    typedef int V;
+    typedef std::set<int, std::less<>> M;
+    typedef M::size_type R;
+    V ar[] = {5, 6, 7, 8, 9, 10, 11, 12};
+    const M m(ar, ar + sizeof(ar) / sizeof(ar[0]));
+    R r = m.count(5);
+    assert(r == 1);
+    r = m.count(6);
+    assert(r == 1);
+    r = m.count(7);
+    assert(r == 1);
+    r = m.count(8);
+    assert(r == 1);
+    r = m.count(9);
+    assert(r == 1);
+    r = m.count(10);
+    assert(r == 1);
+    r = m.count(11);
+    assert(r == 1);
+    r = m.count(12);
+    assert(r == 1);
+    r = m.count(4);
+    assert(r == 0);
+  }
+  {
     typedef PrivateConstructor V;
     typedef std::set<V, std::less<>> M;
-        typedef M::size_type R;
+    typedef M::size_type R;
 
     M m;
-    m.insert ( V::make ( 5 ));
-    m.insert ( V::make ( 6 ));
-    m.insert ( V::make ( 7 ));
-    m.insert ( V::make ( 8 ));
-    m.insert ( V::make ( 9 ));
-    m.insert ( V::make ( 10 ));
-    m.insert ( V::make ( 11 ));
-    m.insert ( V::make ( 12 ));
+    m.insert(V::make(5));
+    m.insert(V::make(6));
+    m.insert(V::make(7));
+    m.insert(V::make(8));
+    m.insert(V::make(9));
+    m.insert(V::make(10));
+    m.insert(V::make(11));
+    m.insert(V::make(12));
 
     const M& mc = m;
 
@@ -164,9 +133,8 @@ int main(int, char**)
     assert(r == 1);
     r = mc.count(4);
     assert(r == 0);
-    }
+  }
 #endif
 
-
   return 0;
 }
diff --git a/libcxx/test/std/containers/associative/set/count_transparent.pass.cpp b/libcxx/test/std/containers/associative/set/count_transparent.pass.cpp
index 3cb7e5fd312f3..4e0b610292460 100644
--- a/libcxx/test/std/containers/associative/set/count_transparent.pass.cpp
+++ b/libcxx/test/std/containers/associative/set/count_transparent.pass.cpp
@@ -22,18 +22,11 @@
 struct Comp {
   using is_transparent = void;
 
-  bool operator()(const std::pair<int, int> &lhs,
-                  const std::pair<int, int> &rhs) const {
-    return lhs < rhs;
-  }
-
-  bool operator()(const std::pair<int, int> &lhs, int rhs) const {
-    return lhs.first < rhs;
-  }
-
-  bool operator()(int lhs, const std::pair<int, int> &rhs) const {
-    return lhs < rhs.first;
-  }
+  bool operator()(const std::pair<int, int>& lhs, const std::pair<int, int>& rhs) const { return lhs < rhs; }
+
+  bool operator()(const std::pair<int, int>& lhs, int rhs) const { return lhs.first < rhs; }
+
+  bool operator()(int lhs, const std::pair<int, int>& rhs) const { return lhs < rhs.first; }
 };
 
 int main(int, char**) {
diff --git a/libcxx/test/std/containers/associative/set/emplace.pass.cpp b/libcxx/test/std/containers/associative/set/emplace.pass.cpp
index f2f302c05881d..e038de77d48a6 100644
--- a/libcxx/test/std/containers/associative/set/emplace.pass.cpp
+++ b/libcxx/test/std/containers/associative/set/emplace.pass.cpp
@@ -23,68 +23,67 @@
 #include "DefaultOnly.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef std::set<DefaultOnly> M;
-        typedef std::pair<M::iterator, bool> R;
-        M m;
-        assert(DefaultOnly::count == 0);
-        R r = m.emplace();
-        assert(r.second);
-        assert(r.first == m.begin());
-        assert(m.size() == 1);
-        assert(*m.begin() == DefaultOnly());
-        assert(DefaultOnly::count == 1);
-
-        r = m.emplace();
-        assert(!r.second);
-        assert(r.first == m.begin());
-        assert(m.size() == 1);
-        assert(*m.begin() == DefaultOnly());
-        assert(DefaultOnly::count == 1);
-    }
+int main(int, char**) {
+  {
+    typedef std::set<DefaultOnly> M;
+    typedef std::pair<M::iterator, bool> R;
+    M m;
     assert(DefaultOnly::count == 0);
-    {
-        typedef std::set<Emplaceable> M;
-        typedef std::pair<M::iterator, bool> R;
-        M m;
-        R r = m.emplace();
-        assert(r.second);
-        assert(r.first == m.begin());
-        assert(m.size() == 1);
-        assert(*m.begin() == Emplaceable());
-        r = m.emplace(2, 3.5);
-        assert(r.second);
-        assert(r.first == std::next(m.begin()));
-        assert(m.size() == 2);
-        assert(*r.first == Emplaceable(2, 3.5));
-        r = m.emplace(2, 3.5);
-        assert(!r.second);
-        assert(r.first == std::next(m.begin()));
-        assert(m.size() == 2);
-        assert(*r.first == Emplaceable(2, 3.5));
-    }
-    {
-        typedef std::set<int> M;
-        typedef std::pair<M::iterator, bool> R;
-        M m;
-        R r = m.emplace(M::value_type(2));
-        assert(r.second);
-        assert(r.first == m.begin());
-        assert(m.size() == 1);
-        assert(*r.first == 2);
-    }
-    {
-        typedef std::set<int, std::less<int>, min_allocator<int>> M;
-        typedef std::pair<M::iterator, bool> R;
-        M m;
-        R r = m.emplace(M::value_type(2));
-        assert(r.second);
-        assert(r.first == m.begin());
-        assert(m.size() == 1);
-        assert(*r.first == 2);
-    }
+    R r = m.emplace();
+    assert(r.second);
+    assert(r.first == m.begin());
+    assert(m.size() == 1);
+    assert(*m.begin() == DefaultOnly());
+    assert(DefaultOnly::count == 1);
+
+    r = m.emplace();
+    assert(!r.second);
+    assert(r.first == m.begin());
+    assert(m.size() == 1);
+    assert(*m.begin() == DefaultOnly());
+    assert(DefaultOnly::count == 1);
+  }
+  assert(DefaultOnly::count == 0);
+  {
+    typedef std::set<Emplaceable> M;
+    typedef std::pair<M::iterator, bool> R;
+    M m;
+    R r = m.emplace();
+    assert(r.second);
+    assert(r.first == m.begin());
+    assert(m.size() == 1);
+    assert(*m.begin() == Emplaceable());
+    r = m.emplace(2, 3.5);
+    assert(r.second);
+    assert(r.first == std::next(m.begin()));
+    assert(m.size() == 2);
+    assert(*r.first == Emplaceable(2, 3.5));
+    r = m.emplace(2, 3.5);
+    assert(!r.second);
+    assert(r.first == std::next(m.begin()));
+    assert(m.size() == 2);
+    assert(*r.first == Emplaceable(2, 3.5));
+  }
+  {
+    typedef std::set<int> M;
+    typedef std::pair<M::iterator, bool> R;
+    M m;
+    R r = m.emplace(M::value_type(2));
+    assert(r.second);
+    assert(r.first == m.begin());
+    assert(m.size() == 1);
+    assert(*r.first == 2);
+  }
+  {
+    typedef std::set<int, std::less<int>, min_allocator<int>> M;
+    typedef std::pair<M::iterator, bool> R;
+    M m;
+    R r = m.emplace(M::value_type(2));
+    assert(r.second);
+    assert(r.first == m.begin());
+    assert(m.size() == 1);
+    assert(*r.first == 2);
+  }
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/associative/set/emplace_hint.pass.cpp b/libcxx/test/std/containers/associative/set/emplace_hint.pass.cpp
index a7d6857a820b8..9f0d5fbd2c5e5 100644
--- a/libcxx/test/std/containers/associative/set/emplace_hint.pass.cpp
+++ b/libcxx/test/std/containers/associative/set/emplace_hint.pass.cpp
@@ -23,61 +23,60 @@
 #include "DefaultOnly.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef std::set<DefaultOnly> M;
-        typedef M::iterator R;
-        M m;
-        assert(DefaultOnly::count == 0);
-        R r = m.emplace_hint(m.cend());
-        assert(r == m.begin());
-        assert(m.size() == 1);
-        assert(*m.begin() == DefaultOnly());
-        assert(DefaultOnly::count == 1);
-
-        r = m.emplace_hint(m.cbegin());
-        assert(r == m.begin());
-        assert(m.size() == 1);
-        assert(*m.begin() == DefaultOnly());
-        assert(DefaultOnly::count == 1);
-    }
+int main(int, char**) {
+  {
+    typedef std::set<DefaultOnly> M;
+    typedef M::iterator R;
+    M m;
     assert(DefaultOnly::count == 0);
-    {
-        typedef std::set<Emplaceable> M;
-        typedef M::iterator R;
-        M m;
-        R r = m.emplace_hint(m.cend());
-        assert(r == m.begin());
-        assert(m.size() == 1);
-        assert(*m.begin() == Emplaceable());
-        r = m.emplace_hint(m.cend(), 2, 3.5);
-        assert(r == std::next(m.begin()));
-        assert(m.size() == 2);
-        assert(*r == Emplaceable(2, 3.5));
-        r = m.emplace_hint(m.cbegin(), 2, 3.5);
-        assert(r == std::next(m.begin()));
-        assert(m.size() == 2);
-        assert(*r == Emplaceable(2, 3.5));
-    }
-    {
-        typedef std::set<int> M;
-        typedef M::iterator R;
-        M m;
-        R r = m.emplace_hint(m.cend(), M::value_type(2));
-        assert(r == m.begin());
-        assert(m.size() == 1);
-        assert(*r == 2);
-    }
-    {
-        typedef std::set<int, std::less<int>, min_allocator<int>> M;
-        typedef M::iterator R;
-        M m;
-        R r = m.emplace_hint(m.cend(), M::value_type(2));
-        assert(r == m.begin());
-        assert(m.size() == 1);
-        assert(*r == 2);
-    }
+    R r = m.emplace_hint(m.cend());
+    assert(r == m.begin());
+    assert(m.size() == 1);
+    assert(*m.begin() == DefaultOnly());
+    assert(DefaultOnly::count == 1);
+
+    r = m.emplace_hint(m.cbegin());
+    assert(r == m.begin());
+    assert(m.size() == 1);
+    assert(*m.begin() == DefaultOnly());
+    assert(DefaultOnly::count == 1);
+  }
+  assert(DefaultOnly::count == 0);
+  {
+    typedef std::set<Emplaceable> M;
+    typedef M::iterator R;
+    M m;
+    R r = m.emplace_hint(m.cend());
+    assert(r == m.begin());
+    assert(m.size() == 1);
+    assert(*m.begin() == Emplaceable());
+    r = m.emplace_hint(m.cend(), 2, 3.5);
+    assert(r == std::next(m.begin()));
+    assert(m.size() == 2);
+    assert(*r == Emplaceable(2, 3.5));
+    r = m.emplace_hint(m.cbegin(), 2, 3.5);
+    assert(r == std::next(m.begin()));
+    assert(m.size() == 2);
+    assert(*r == Emplaceable(2, 3.5));
+  }
+  {
+    typedef std::set<int> M;
+    typedef M::iterator R;
+    M m;
+    R r = m.emplace_hint(m.cend(), M::value_type(2));
+    assert(r == m.begin());
+    assert(m.size() == 1);
+    assert(*r == 2);
+  }
+  {
+    typedef std::set<int, std::less<int>, min_allocator<int>> M;
+    typedef M::iterator R;
+    M m;
+    R r = m.emplace_hint(m.cend(), M::value_type(2));
+    assert(r == m.begin());
+    assert(m.size() == 1);
+    assert(*r == 2);
+  }
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/associative/set/empty.pass.cpp b/libcxx/test/std/containers/associative/set/empty.pass.cpp
index 8bf1e1295eda4..aaefade5cff7b 100644
--- a/libcxx/test/std/containers/associative/set/empty.pass.cpp
+++ b/libcxx/test/std/containers/associative/set/empty.pass.cpp
@@ -18,9 +18,8 @@
 #include "test_macros.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
+int main(int, char**) {
+  {
     typedef std::set<int> M;
     M m;
     assert(m.empty());
@@ -28,9 +27,9 @@ int main(int, char**)
     assert(!m.empty());
     m.clear();
     assert(m.empty());
-    }
+  }
 #if TEST_STD_VER >= 11
-    {
+  {
     typedef std::set<int, std::less<int>, min_allocator<int>> M;
     M m;
     assert(m.empty());
@@ -38,7 +37,7 @@ int main(int, char**)
     assert(!m.empty());
     m.clear();
     assert(m.empty());
-    }
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/associative/set/empty.verify.cpp b/libcxx/test/std/containers/associative/set/empty.verify.cpp
index e0405f70e67da..67dae7d0a9dc9 100644
--- a/libcxx/test/std/containers/associative/set/empty.verify.cpp
+++ b/libcxx/test/std/containers/associative/set/empty.verify.cpp
@@ -17,6 +17,6 @@
 #include <set>
 
 void f() {
-    std::set<int> c;
-    c.empty(); // expected-warning {{ignoring return value of function declared with 'nodiscard' attribute}}
+  std::set<int> c;
+  c.empty(); // expected-warning {{ignoring return value of function declared with 'nodiscard' attribute}}
 }
diff --git a/libcxx/test/std/containers/associative/set/equal_range.pass.cpp b/libcxx/test/std/containers/associative/set/equal_range.pass.cpp
index c0602ba626afe..239aa3e878426 100644
--- a/libcxx/test/std/containers/associative/set/equal_range.pass.cpp
+++ b/libcxx/test/std/containers/associative/set/equal_range.pass.cpp
@@ -20,161 +20,130 @@
 #include "min_allocator.h"
 #include "private_constructor.h"
 
-int main(int, char**)
-{
-    {
+int main(int, char**) {
+  {
     typedef int V;
     typedef std::set<int> M;
     {
-        typedef std::pair<M::iterator, M::iterator> R;
-        V ar[] =
-        {
-            5,
-            7,
-            9,
-            11,
-            13,
-            15,
-            17,
-            19
-        };
-        M m(ar, ar+sizeof(ar)/sizeof(ar[0]));
-        R r = m.equal_range(5);
-        assert(r.first == std::next(m.begin(), 0));
-        assert(r.second == std::next(m.begin(), 1));
-        r = m.equal_range(7);
-        assert(r.first == std::next(m.begin(), 1));
-        assert(r.second == std::next(m.begin(), 2));
-        r = m.equal_range(9);
-        assert(r.first == std::next(m.begin(), 2));
-        assert(r.second == std::next(m.begin(), 3));
-        r = m.equal_range(11);
-        assert(r.first == std::next(m.begin(), 3));
-        assert(r.second == std::next(m.begin(), 4));
-        r = m.equal_range(13);
-        assert(r.first == std::next(m.begin(), 4));
-        assert(r.second == std::next(m.begin(), 5));
-        r = m.equal_range(15);
-        assert(r.first == std::next(m.begin(), 5));
-        assert(r.second == std::next(m.begin(), 6));
-        r = m.equal_range(17);
-        assert(r.first == std::next(m.begin(), 6));
-        assert(r.second == std::next(m.begin(), 7));
-        r = m.equal_range(19);
-        assert(r.first == std::next(m.begin(), 7));
-        assert(r.second == std::next(m.begin(), 8));
-        r = m.equal_range(4);
-        assert(r.first == std::next(m.begin(), 0));
-        assert(r.second == std::next(m.begin(), 0));
-        r = m.equal_range(6);
-        assert(r.first == std::next(m.begin(), 1));
-        assert(r.second == std::next(m.begin(), 1));
-        r = m.equal_range(8);
-        assert(r.first == std::next(m.begin(), 2));
-        assert(r.second == std::next(m.begin(), 2));
-        r = m.equal_range(10);
-        assert(r.first == std::next(m.begin(), 3));
-        assert(r.second == std::next(m.begin(), 3));
-        r = m.equal_range(12);
-        assert(r.first == std::next(m.begin(), 4));
-        assert(r.second == std::next(m.begin(), 4));
-        r = m.equal_range(14);
-        assert(r.first == std::next(m.begin(), 5));
-        assert(r.second == std::next(m.begin(), 5));
-        r = m.equal_range(16);
-        assert(r.first == std::next(m.begin(), 6));
-        assert(r.second == std::next(m.begin(), 6));
-        r = m.equal_range(18);
-        assert(r.first == std::next(m.begin(), 7));
-        assert(r.second == std::next(m.begin(), 7));
-        r = m.equal_range(20);
-        assert(r.first == std::next(m.begin(), 8));
-        assert(r.second == std::next(m.begin(), 8));
+      typedef std::pair<M::iterator, M::iterator> R;
+      V ar[] = {5, 7, 9, 11, 13, 15, 17, 19};
+      M m(ar, ar + sizeof(ar) / sizeof(ar[0]));
+      R r = m.equal_range(5);
+      assert(r.first == std::next(m.begin(), 0));
+      assert(r.second == std::next(m.begin(), 1));
+      r = m.equal_range(7);
+      assert(r.first == std::next(m.begin(), 1));
+      assert(r.second == std::next(m.begin(), 2));
+      r = m.equal_range(9);
+      assert(r.first == std::next(m.begin(), 2));
+      assert(r.second == std::next(m.begin(), 3));
+      r = m.equal_range(11);
+      assert(r.first == std::next(m.begin(), 3));
+      assert(r.second == std::next(m.begin(), 4));
+      r = m.equal_range(13);
+      assert(r.first == std::next(m.begin(), 4));
+      assert(r.second == std::next(m.begin(), 5));
+      r = m.equal_range(15);
+      assert(r.first == std::next(m.begin(), 5));
+      assert(r.second == std::next(m.begin(), 6));
+      r = m.equal_range(17);
+      assert(r.first == std::next(m.begin(), 6));
+      assert(r.second == std::next(m.begin(), 7));
+      r = m.equal_range(19);
+      assert(r.first == std::next(m.begin(), 7));
+      assert(r.second == std::next(m.begin(), 8));
+      r = m.equal_range(4);
+      assert(r.first == std::next(m.begin(), 0));
+      assert(r.second == std::next(m.begin(), 0));
+      r = m.equal_range(6);
+      assert(r.first == std::next(m.begin(), 1));
+      assert(r.second == std::next(m.begin(), 1));
+      r = m.equal_range(8);
+      assert(r.first == std::next(m.begin(), 2));
+      assert(r.second == std::next(m.begin(), 2));
+      r = m.equal_range(10);
+      assert(r.first == std::next(m.begin(), 3));
+      assert(r.second == std::next(m.begin(), 3));
+      r = m.equal_range(12);
+      assert(r.first == std::next(m.begin(), 4));
+      assert(r.second == std::next(m.begin(), 4));
+      r = m.equal_range(14);
+      assert(r.first == std::next(m.begin(), 5));
+      assert(r.second == std::next(m.begin(), 5));
+      r = m.equal_range(16);
+      assert(r.first == std::next(m.begin(), 6));
+      assert(r.second == std::next(m.begin(), 6));
+      r = m.equal_range(18);
+      assert(r.first == std::next(m.begin(), 7));
+      assert(r.second == std::next(m.begin(), 7));
+      r = m.equal_range(20);
+      assert(r.first == std::next(m.begin(), 8));
+      assert(r.second == std::next(m.begin(), 8));
     }
     {
-        typedef std::pair<M::const_iterator, M::const_iterator> R;
-        V ar[] =
-        {
-            5,
-            7,
-            9,
-            11,
-            13,
-            15,
-            17,
-            19
-        };
-        const M m(ar, ar+sizeof(ar)/sizeof(ar[0]));
-        R r = m.equal_range(5);
-        assert(r.first == std::next(m.begin(), 0));
-        assert(r.second == std::next(m.begin(), 1));
-        r = m.equal_range(7);
-        assert(r.first == std::next(m.begin(), 1));
-        assert(r.second == std::next(m.begin(), 2));
-        r = m.equal_range(9);
-        assert(r.first == std::next(m.begin(), 2));
-        assert(r.second == std::next(m.begin(), 3));
-        r = m.equal_range(11);
-        assert(r.first == std::next(m.begin(), 3));
-        assert(r.second == std::next(m.begin(), 4));
-        r = m.equal_range(13);
-        assert(r.first == std::next(m.begin(), 4));
-        assert(r.second == std::next(m.begin(), 5));
-        r = m.equal_range(15);
-        assert(r.first == std::next(m.begin(), 5));
-        assert(r.second == std::next(m.begin(), 6));
-        r = m.equal_range(17);
-        assert(r.first == std::next(m.begin(), 6));
-        assert(r.second == std::next(m.begin(), 7));
-        r = m.equal_range(19);
-        assert(r.first == std::next(m.begin(), 7));
-        assert(r.second == std::next(m.begin(), 8));
-        r = m.equal_range(4);
-        assert(r.first == std::next(m.begin(), 0));
-        assert(r.second == std::next(m.begin(), 0));
-        r = m.equal_range(6);
-        assert(r.first == std::next(m.begin(), 1));
-        assert(r.second == std::next(m.begin(), 1));
-        r = m.equal_range(8);
-        assert(r.first == std::next(m.begin(), 2));
-        assert(r.second == std::next(m.begin(), 2));
-        r = m.equal_range(10);
-        assert(r.first == std::next(m.begin(), 3));
-        assert(r.second == std::next(m.begin(), 3));
-        r = m.equal_range(12);
-        assert(r.first == std::next(m.begin(), 4));
-        assert(r.second == std::next(m.begin(), 4));
-        r = m.equal_range(14);
-        assert(r.first == std::next(m.begin(), 5));
-        assert(r.second == std::next(m.begin(), 5));
-        r = m.equal_range(16);
-        assert(r.first == std::next(m.begin(), 6));
-        assert(r.second == std::next(m.begin(), 6));
-        r = m.equal_range(18);
-        assert(r.first == std::next(m.begin(), 7));
-        assert(r.second == std::next(m.begin(), 7));
-        r = m.equal_range(20);
-        assert(r.first == std::next(m.begin(), 8));
-        assert(r.second == std::next(m.begin(), 8));
-    }
+      typedef std::pair<M::const_iterator, M::const_iterator> R;
+      V ar[] = {5, 7, 9, 11, 13, 15, 17, 19};
+      const M m(ar, ar + sizeof(ar) / sizeof(ar[0]));
+      R r = m.equal_range(5);
+      assert(r.first == std::next(m.begin(), 0));
+      assert(r.second == std::next(m.begin(), 1));
+      r = m.equal_range(7);
+      assert(r.first == std::next(m.begin(), 1));
+      assert(r.second == std::next(m.begin(), 2));
+      r = m.equal_range(9);
+      assert(r.first == std::next(m.begin(), 2));
+      assert(r.second == std::next(m.begin(), 3));
+      r = m.equal_range(11);
+      assert(r.first == std::next(m.begin(), 3));
+      assert(r.second == std::next(m.begin(), 4));
+      r = m.equal_range(13);
+      assert(r.first == std::next(m.begin(), 4));
+      assert(r.second == std::next(m.begin(), 5));
+      r = m.equal_range(15);
+      assert(r.first == std::next(m.begin(), 5));
+      assert(r.second == std::next(m.begin(), 6));
+      r = m.equal_range(17);
+      assert(r.first == std::next(m.begin(), 6));
+      assert(r.second == std::next(m.begin(), 7));
+      r = m.equal_range(19);
+      assert(r.first == std::next(m.begin(), 7));
+      assert(r.second == std::next(m.begin(), 8));
+      r = m.equal_range(4);
+      assert(r.first == std::next(m.begin(), 0));
+      assert(r.second == std::next(m.begin(), 0));
+      r = m.equal_range(6);
+      assert(r.first == std::next(m.begin(), 1));
+      assert(r.second == std::next(m.begin(), 1));
+      r = m.equal_range(8);
+      assert(r.first == std::next(m.begin(), 2));
+      assert(r.second == std::next(m.begin(), 2));
+      r = m.equal_range(10);
+      assert(r.first == std::next(m.begin(), 3));
+      assert(r.second == std::next(m.begin(), 3));
+      r = m.equal_range(12);
+      assert(r.first == std::next(m.begin(), 4));
+      assert(r.second == std::next(m.begin(), 4));
+      r = m.equal_range(14);
+      assert(r.first == std::next(m.begin(), 5));
+      assert(r.second == std::next(m.begin(), 5));
+      r = m.equal_range(16);
+      assert(r.first == std::next(m.begin(), 6));
+      assert(r.second == std::next(m.begin(), 6));
+      r = m.equal_range(18);
+      assert(r.first == std::next(m.begin(), 7));
+      assert(r.second == std::next(m.begin(), 7));
+      r = m.equal_range(20);
+      assert(r.first == std::next(m.begin(), 8));
+      assert(r.second == std::next(m.begin(), 8));
     }
+  }
 #if TEST_STD_VER >= 11
-    {
+  {
     typedef int V;
     typedef std::set<int, std::less<int>, min_allocator<int>> M;
     typedef std::pair<M::iterator, M::iterator> R;
-    V ar[] =
-    {
-        5,
-        7,
-        9,
-        11,
-        13,
-        15,
-        17,
-        19
-    };
-    M m(ar, ar+sizeof(ar)/sizeof(ar[0]));
+    V ar[] = {5, 7, 9, 11, 13, 15, 17, 19};
+    M m(ar, ar + sizeof(ar) / sizeof(ar[0]));
     R r = m.equal_range(5);
     assert(r.first == std::next(m.begin(), 0));
     assert(r.second == std::next(m.begin(), 1));
@@ -226,93 +195,83 @@ int main(int, char**)
     r = m.equal_range(20);
     assert(r.first == std::next(m.begin(), 8));
     assert(r.second == std::next(m.begin(), 8));
-    }
+  }
 #endif
 #if TEST_STD_VER > 11
-    {
+  {
     typedef int V;
     typedef std::set<V, std::less<>> M;
     {
-        typedef std::pair<M::iterator, M::iterator> R;
-        V ar[] =
-        {
-            5,
-            7,
-            9,
-            11,
-            13,
-            15,
-            17,
-            19
-        };
-        M m(ar, ar+sizeof(ar)/sizeof(ar[0]));
-        R r = m.equal_range(5);
-        assert(r.first == std::next(m.begin(), 0));
-        assert(r.second == std::next(m.begin(), 1));
-        r = m.equal_range(7);
-        assert(r.first == std::next(m.begin(), 1));
-        assert(r.second == std::next(m.begin(), 2));
-        r = m.equal_range(9);
-        assert(r.first == std::next(m.begin(), 2));
-        assert(r.second == std::next(m.begin(), 3));
-        r = m.equal_range(11);
-        assert(r.first == std::next(m.begin(), 3));
-        assert(r.second == std::next(m.begin(), 4));
-        r = m.equal_range(13);
-        assert(r.first == std::next(m.begin(), 4));
-        assert(r.second == std::next(m.begin(), 5));
-        r = m.equal_range(15);
-        assert(r.first == std::next(m.begin(), 5));
-        assert(r.second == std::next(m.begin(), 6));
-        r = m.equal_range(17);
-        assert(r.first == std::next(m.begin(), 6));
-        assert(r.second == std::next(m.begin(), 7));
-        r = m.equal_range(19);
-        assert(r.first == std::next(m.begin(), 7));
-        assert(r.second == std::next(m.begin(), 8));
-        r = m.equal_range(4);
-        assert(r.first == std::next(m.begin(), 0));
-        assert(r.second == std::next(m.begin(), 0));
-        r = m.equal_range(6);
-        assert(r.first == std::next(m.begin(), 1));
-        assert(r.second == std::next(m.begin(), 1));
-        r = m.equal_range(8);
-        assert(r.first == std::next(m.begin(), 2));
-        assert(r.second == std::next(m.begin(), 2));
-        r = m.equal_range(10);
-        assert(r.first == std::next(m.begin(), 3));
-        assert(r.second == std::next(m.begin(), 3));
-        r = m.equal_range(12);
-        assert(r.first == std::next(m.begin(), 4));
-        assert(r.second == std::next(m.begin(), 4));
-        r = m.equal_range(14);
-        assert(r.first == std::next(m.begin(), 5));
-        assert(r.second == std::next(m.begin(), 5));
-        r = m.equal_range(16);
-        assert(r.first == std::next(m.begin(), 6));
-        assert(r.second == std::next(m.begin(), 6));
-        r = m.equal_range(18);
-        assert(r.first == std::next(m.begin(), 7));
-        assert(r.second == std::next(m.begin(), 7));
-        r = m.equal_range(20);
-        assert(r.first == std::next(m.begin(), 8));
-        assert(r.second == std::next(m.begin(), 8));
-        }
+      typedef std::pair<M::iterator, M::iterator> R;
+      V ar[] = {5, 7, 9, 11, 13, 15, 17, 19};
+      M m(ar, ar + sizeof(ar) / sizeof(ar[0]));
+      R r = m.equal_range(5);
+      assert(r.first == std::next(m.begin(), 0));
+      assert(r.second == std::next(m.begin(), 1));
+      r = m.equal_range(7);
+      assert(r.first == std::next(m.begin(), 1));
+      assert(r.second == std::next(m.begin(), 2));
+      r = m.equal_range(9);
+      assert(r.first == std::next(m.begin(), 2));
+      assert(r.second == std::next(m.begin(), 3));
+      r = m.equal_range(11);
+      assert(r.first == std::next(m.begin(), 3));
+      assert(r.second == std::next(m.begin(), 4));
+      r = m.equal_range(13);
+      assert(r.first == std::next(m.begin(), 4));
+      assert(r.second == std::next(m.begin(), 5));
+      r = m.equal_range(15);
+      assert(r.first == std::next(m.begin(), 5));
+      assert(r.second == std::next(m.begin(), 6));
+      r = m.equal_range(17);
+      assert(r.first == std::next(m.begin(), 6));
+      assert(r.second == std::next(m.begin(), 7));
+      r = m.equal_range(19);
+      assert(r.first == std::next(m.begin(), 7));
+      assert(r.second == std::next(m.begin(), 8));
+      r = m.equal_range(4);
+      assert(r.first == std::next(m.begin(), 0));
+      assert(r.second == std::next(m.begin(), 0));
+      r = m.equal_range(6);
+      assert(r.first == std::next(m.begin(), 1));
+      assert(r.second == std::next(m.begin(), 1));
+      r = m.equal_range(8);
+      assert(r.first == std::next(m.begin(), 2));
+      assert(r.second == std::next(m.begin(), 2));
+      r = m.equal_range(10);
+      assert(r.first == std::next(m.begin(), 3));
+      assert(r.second == std::next(m.begin(), 3));
+      r = m.equal_range(12);
+      assert(r.first == std::next(m.begin(), 4));
+      assert(r.second == std::next(m.begin(), 4));
+      r = m.equal_range(14);
+      assert(r.first == std::next(m.begin(), 5));
+      assert(r.second == std::next(m.begin(), 5));
+      r = m.equal_range(16);
+      assert(r.first == std::next(m.begin(), 6));
+      assert(r.second == std::next(m.begin(), 6));
+      r = m.equal_range(18);
+      assert(r.first == std::next(m.begin(), 7));
+      assert(r.second == std::next(m.begin(), 7));
+      r = m.equal_range(20);
+      assert(r.first == std::next(m.begin(), 8));
+      assert(r.second == std::next(m.begin(), 8));
     }
-    {
+  }
+  {
     typedef PrivateConstructor V;
     typedef std::set<V, std::less<>> M;
     typedef std::pair<M::iterator, M::iterator> R;
 
     M m;
-    m.insert ( V::make ( 5 ));
-    m.insert ( V::make ( 7 ));
-    m.insert ( V::make ( 9 ));
-    m.insert ( V::make ( 11 ));
-    m.insert ( V::make ( 13 ));
-    m.insert ( V::make ( 15 ));
-    m.insert ( V::make ( 17 ));
-    m.insert ( V::make ( 19 ));
+    m.insert(V::make(5));
+    m.insert(V::make(7));
+    m.insert(V::make(9));
+    m.insert(V::make(11));
+    m.insert(V::make(13));
+    m.insert(V::make(15));
+    m.insert(V::make(17));
+    m.insert(V::make(19));
 
     R r = m.equal_range(5);
     assert(r.first == std::next(m.begin(), 0));
@@ -365,7 +324,7 @@ int main(int, char**)
     r = m.equal_range(20);
     assert(r.first == std::next(m.begin(), 8));
     assert(r.second == std::next(m.begin(), 8));
-    }
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/associative/set/equal_range_transparent.pass.cpp b/libcxx/test/std/containers/associative/set/equal_range_transparent.pass.cpp
index e4b7833910a17..a44fcf4b74a02 100644
--- a/libcxx/test/std/containers/associative/set/equal_range_transparent.pass.cpp
+++ b/libcxx/test/std/containers/associative/set/equal_range_transparent.pass.cpp
@@ -26,24 +26,17 @@
 struct Comp {
   using is_transparent = void;
 
-  bool operator()(const std::pair<int, int> &lhs,
-                  const std::pair<int, int> &rhs) const {
-    return lhs < rhs;
-  }
+  bool operator()(const std::pair<int, int>& lhs, const std::pair<int, int>& rhs) const { return lhs < rhs; }
 
-  bool operator()(const std::pair<int, int> &lhs, int rhs) const {
-    return lhs.first < rhs;
-  }
+  bool operator()(const std::pair<int, int>& lhs, int rhs) const { return lhs.first < rhs; }
 
-  bool operator()(int lhs, const std::pair<int, int> &rhs) const {
-    return lhs < rhs.first;
-  }
+  bool operator()(int lhs, const std::pair<int, int>& rhs) const { return lhs < rhs.first; }
 };
 
 int main(int, char**) {
   std::set<std::pair<int, int>, Comp> s{{2, 1}, {1, 2}, {1, 3}, {1, 4}, {2, 2}};
 
-  auto er = s.equal_range(1);
+  auto er   = s.equal_range(1);
   long nels = 0;
 
   for (auto it = er.first; it != er.second; it++) {
diff --git a/libcxx/test/std/containers/associative/set/erase_iter.pass.cpp b/libcxx/test/std/containers/associative/set/erase_iter.pass.cpp
index 9d9fa180b7a16..73a4cc4266ec9 100644
--- a/libcxx/test/std/containers/associative/set/erase_iter.pass.cpp
+++ b/libcxx/test/std/containers/associative/set/erase_iter.pass.cpp
@@ -18,187 +18,165 @@
 #include "test_macros.h"
 #include "min_allocator.h"
 
-struct TemplateConstructor
-{
-    template<typename T>
-    TemplateConstructor (const T&) {}
+struct TemplateConstructor {
+  template <typename T>
+  TemplateConstructor(const T&) {}
 };
 
 bool operator<(const TemplateConstructor&, const TemplateConstructor&) { return false; }
 
-int main(int, char**)
-{
-    {
-        typedef std::set<int> M;
-        typedef int V;
-        typedef M::iterator I;
-        V ar[] =
-        {
-            1,
-            2,
-            3,
-            4,
-            5,
-            6,
-            7,
-            8
-        };
-        M m(ar, ar + sizeof(ar)/sizeof(ar[0]));
-        assert(m.size() == 8);
-        I i = m.erase(std::next(m.cbegin(), 3));
-        assert(m.size() == 7);
-        assert(i == std::next(m.begin(), 3));
-        assert(*std::next(m.begin(), 0) == 1);
-        assert(*std::next(m.begin(), 1) == 2);
-        assert(*std::next(m.begin(), 2) == 3);
-        assert(*std::next(m.begin(), 3) == 5);
-        assert(*std::next(m.begin(), 4) == 6);
-        assert(*std::next(m.begin(), 5) == 7);
-        assert(*std::next(m.begin(), 6) == 8);
-
-        i = m.erase(std::next(m.cbegin(), 0));
-        assert(m.size() == 6);
-        assert(i == m.begin());
-        assert(*std::next(m.begin(), 0) == 2);
-        assert(*std::next(m.begin(), 1) == 3);
-        assert(*std::next(m.begin(), 2) == 5);
-        assert(*std::next(m.begin(), 3) == 6);
-        assert(*std::next(m.begin(), 4) == 7);
-        assert(*std::next(m.begin(), 5) == 8);
-
-        i = m.erase(std::next(m.cbegin(), 5));
-        assert(m.size() == 5);
-        assert(i == m.end());
-        assert(*std::next(m.begin(), 0) == 2);
-        assert(*std::next(m.begin(), 1) == 3);
-        assert(*std::next(m.begin(), 2) == 5);
-        assert(*std::next(m.begin(), 3) == 6);
-        assert(*std::next(m.begin(), 4) == 7);
-
-        i = m.erase(std::next(m.cbegin(), 1));
-        assert(m.size() == 4);
-        assert(i == std::next(m.begin()));
-        assert(*std::next(m.begin(), 0) == 2);
-        assert(*std::next(m.begin(), 1) == 5);
-        assert(*std::next(m.begin(), 2) == 6);
-        assert(*std::next(m.begin(), 3) == 7);
-
-        i = m.erase(std::next(m.cbegin(), 2));
-        assert(m.size() == 3);
-        assert(i == std::next(m.begin(), 2));
-        assert(*std::next(m.begin(), 0) == 2);
-        assert(*std::next(m.begin(), 1) == 5);
-        assert(*std::next(m.begin(), 2) == 7);
-
-        i = m.erase(std::next(m.cbegin(), 2));
-        assert(m.size() == 2);
-        assert(i == std::next(m.begin(), 2));
-        assert(*std::next(m.begin(), 0) == 2);
-        assert(*std::next(m.begin(), 1) == 5);
-
-        i = m.erase(std::next(m.cbegin(), 0));
-        assert(m.size() == 1);
-        assert(i == std::next(m.begin(), 0));
-        assert(*std::next(m.begin(), 0) == 5);
-
-        i = m.erase(m.cbegin());
-        assert(m.size() == 0);
-        assert(i == m.begin());
-        assert(i == m.end());
-    }
+int main(int, char**) {
+  {
+    typedef std::set<int> M;
+    typedef int V;
+    typedef M::iterator I;
+    V ar[] = {1, 2, 3, 4, 5, 6, 7, 8};
+    M m(ar, ar + sizeof(ar) / sizeof(ar[0]));
+    assert(m.size() == 8);
+    I i = m.erase(std::next(m.cbegin(), 3));
+    assert(m.size() == 7);
+    assert(i == std::next(m.begin(), 3));
+    assert(*std::next(m.begin(), 0) == 1);
+    assert(*std::next(m.begin(), 1) == 2);
+    assert(*std::next(m.begin(), 2) == 3);
+    assert(*std::next(m.begin(), 3) == 5);
+    assert(*std::next(m.begin(), 4) == 6);
+    assert(*std::next(m.begin(), 5) == 7);
+    assert(*std::next(m.begin(), 6) == 8);
+
+    i = m.erase(std::next(m.cbegin(), 0));
+    assert(m.size() == 6);
+    assert(i == m.begin());
+    assert(*std::next(m.begin(), 0) == 2);
+    assert(*std::next(m.begin(), 1) == 3);
+    assert(*std::next(m.begin(), 2) == 5);
+    assert(*std::next(m.begin(), 3) == 6);
+    assert(*std::next(m.begin(), 4) == 7);
+    assert(*std::next(m.begin(), 5) == 8);
+
+    i = m.erase(std::next(m.cbegin(), 5));
+    assert(m.size() == 5);
+    assert(i == m.end());
+    assert(*std::next(m.begin(), 0) == 2);
+    assert(*std::next(m.begin(), 1) == 3);
+    assert(*std::next(m.begin(), 2) == 5);
+    assert(*std::next(m.begin(), 3) == 6);
+    assert(*std::next(m.begin(), 4) == 7);
+
+    i = m.erase(std::next(m.cbegin(), 1));
+    assert(m.size() == 4);
+    assert(i == std::next(m.begin()));
+    assert(*std::next(m.begin(), 0) == 2);
+    assert(*std::next(m.begin(), 1) == 5);
+    assert(*std::next(m.begin(), 2) == 6);
+    assert(*std::next(m.begin(), 3) == 7);
+
+    i = m.erase(std::next(m.cbegin(), 2));
+    assert(m.size() == 3);
+    assert(i == std::next(m.begin(), 2));
+    assert(*std::next(m.begin(), 0) == 2);
+    assert(*std::next(m.begin(), 1) == 5);
+    assert(*std::next(m.begin(), 2) == 7);
+
+    i = m.erase(std::next(m.cbegin(), 2));
+    assert(m.size() == 2);
+    assert(i == std::next(m.begin(), 2));
+    assert(*std::next(m.begin(), 0) == 2);
+    assert(*std::next(m.begin(), 1) == 5);
+
+    i = m.erase(std::next(m.cbegin(), 0));
+    assert(m.size() == 1);
+    assert(i == std::next(m.begin(), 0));
+    assert(*std::next(m.begin(), 0) == 5);
+
+    i = m.erase(m.cbegin());
+    assert(m.size() == 0);
+    assert(i == m.begin());
+    assert(i == m.end());
+  }
 #if TEST_STD_VER >= 11
-    {
-        typedef std::set<int, std::less<int>, min_allocator<int>> M;
-        typedef int V;
-        typedef M::iterator I;
-        V ar[] =
-        {
-            1,
-            2,
-            3,
-            4,
-            5,
-            6,
-            7,
-            8
-        };
-        M m(ar, ar + sizeof(ar)/sizeof(ar[0]));
-        assert(m.size() == 8);
-        I i = m.erase(std::next(m.cbegin(), 3));
-        assert(m.size() == 7);
-        assert(i == std::next(m.begin(), 3));
-        assert(*std::next(m.begin(), 0) == 1);
-        assert(*std::next(m.begin(), 1) == 2);
-        assert(*std::next(m.begin(), 2) == 3);
-        assert(*std::next(m.begin(), 3) == 5);
-        assert(*std::next(m.begin(), 4) == 6);
-        assert(*std::next(m.begin(), 5) == 7);
-        assert(*std::next(m.begin(), 6) == 8);
-
-        i = m.erase(std::next(m.cbegin(), 0));
-        assert(m.size() == 6);
-        assert(i == m.begin());
-        assert(*std::next(m.begin(), 0) == 2);
-        assert(*std::next(m.begin(), 1) == 3);
-        assert(*std::next(m.begin(), 2) == 5);
-        assert(*std::next(m.begin(), 3) == 6);
-        assert(*std::next(m.begin(), 4) == 7);
-        assert(*std::next(m.begin(), 5) == 8);
-
-        i = m.erase(std::next(m.cbegin(), 5));
-        assert(m.size() == 5);
-        assert(i == m.end());
-        assert(*std::next(m.begin(), 0) == 2);
-        assert(*std::next(m.begin(), 1) == 3);
-        assert(*std::next(m.begin(), 2) == 5);
-        assert(*std::next(m.begin(), 3) == 6);
-        assert(*std::next(m.begin(), 4) == 7);
-
-        i = m.erase(std::next(m.cbegin(), 1));
-        assert(m.size() == 4);
-        assert(i == std::next(m.begin()));
-        assert(*std::next(m.begin(), 0) == 2);
-        assert(*std::next(m.begin(), 1) == 5);
-        assert(*std::next(m.begin(), 2) == 6);
-        assert(*std::next(m.begin(), 3) == 7);
-
-        i = m.erase(std::next(m.cbegin(), 2));
-        assert(m.size() == 3);
-        assert(i == std::next(m.begin(), 2));
-        assert(*std::next(m.begin(), 0) == 2);
-        assert(*std::next(m.begin(), 1) == 5);
-        assert(*std::next(m.begin(), 2) == 7);
-
-        i = m.erase(std::next(m.cbegin(), 2));
-        assert(m.size() == 2);
-        assert(i == std::next(m.begin(), 2));
-        assert(*std::next(m.begin(), 0) == 2);
-        assert(*std::next(m.begin(), 1) == 5);
-
-        i = m.erase(std::next(m.cbegin(), 0));
-        assert(m.size() == 1);
-        assert(i == std::next(m.begin(), 0));
-        assert(*std::next(m.begin(), 0) == 5);
-
-        i = m.erase(m.cbegin());
-        assert(m.size() == 0);
-        assert(i == m.begin());
-        assert(i == m.end());
-    }
+  {
+    typedef std::set<int, std::less<int>, min_allocator<int>> M;
+    typedef int V;
+    typedef M::iterator I;
+    V ar[] = {1, 2, 3, 4, 5, 6, 7, 8};
+    M m(ar, ar + sizeof(ar) / sizeof(ar[0]));
+    assert(m.size() == 8);
+    I i = m.erase(std::next(m.cbegin(), 3));
+    assert(m.size() == 7);
+    assert(i == std::next(m.begin(), 3));
+    assert(*std::next(m.begin(), 0) == 1);
+    assert(*std::next(m.begin(), 1) == 2);
+    assert(*std::next(m.begin(), 2) == 3);
+    assert(*std::next(m.begin(), 3) == 5);
+    assert(*std::next(m.begin(), 4) == 6);
+    assert(*std::next(m.begin(), 5) == 7);
+    assert(*std::next(m.begin(), 6) == 8);
+
+    i = m.erase(std::next(m.cbegin(), 0));
+    assert(m.size() == 6);
+    assert(i == m.begin());
+    assert(*std::next(m.begin(), 0) == 2);
+    assert(*std::next(m.begin(), 1) == 3);
+    assert(*std::next(m.begin(), 2) == 5);
+    assert(*std::next(m.begin(), 3) == 6);
+    assert(*std::next(m.begin(), 4) == 7);
+    assert(*std::next(m.begin(), 5) == 8);
+
+    i = m.erase(std::next(m.cbegin(), 5));
+    assert(m.size() == 5);
+    assert(i == m.end());
+    assert(*std::next(m.begin(), 0) == 2);
+    assert(*std::next(m.begin(), 1) == 3);
+    assert(*std::next(m.begin(), 2) == 5);
+    assert(*std::next(m.begin(), 3) == 6);
+    assert(*std::next(m.begin(), 4) == 7);
+
+    i = m.erase(std::next(m.cbegin(), 1));
+    assert(m.size() == 4);
+    assert(i == std::next(m.begin()));
+    assert(*std::next(m.begin(), 0) == 2);
+    assert(*std::next(m.begin(), 1) == 5);
+    assert(*std::next(m.begin(), 2) == 6);
+    assert(*std::next(m.begin(), 3) == 7);
+
+    i = m.erase(std::next(m.cbegin(), 2));
+    assert(m.size() == 3);
+    assert(i == std::next(m.begin(), 2));
+    assert(*std::next(m.begin(), 0) == 2);
+    assert(*std::next(m.begin(), 1) == 5);
+    assert(*std::next(m.begin(), 2) == 7);
+
+    i = m.erase(std::next(m.cbegin(), 2));
+    assert(m.size() == 2);
+    assert(i == std::next(m.begin(), 2));
+    assert(*std::next(m.begin(), 0) == 2);
+    assert(*std::next(m.begin(), 1) == 5);
+
+    i = m.erase(std::next(m.cbegin(), 0));
+    assert(m.size() == 1);
+    assert(i == std::next(m.begin(), 0));
+    assert(*std::next(m.begin(), 0) == 5);
+
+    i = m.erase(m.cbegin());
+    assert(m.size() == 0);
+    assert(i == m.begin());
+    assert(i == m.end());
+  }
 #endif
 #if TEST_STD_VER >= 14
-    {
+  {
     //  This is LWG #2059
-        typedef TemplateConstructor T;
-        typedef std::set<T> C;
-        typedef C::iterator I;
-
-        C c;
-        T a{0};
-        I it = c.find(a);
-        if (it != c.end())
-            c.erase(it);
-    }
+    typedef TemplateConstructor T;
+    typedef std::set<T> C;
+    typedef C::iterator I;
+
+    C c;
+    T a{0};
+    I it = c.find(a);
+    if (it != c.end())
+      c.erase(it);
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/associative/set/erase_iter_iter.pass.cpp b/libcxx/test/std/containers/associative/set/erase_iter_iter.pass.cpp
index 8508e17bc717b..20c97c28fce56 100644
--- a/libcxx/test/std/containers/associative/set/erase_iter_iter.pass.cpp
+++ b/libcxx/test/std/containers/associative/set/erase_iter_iter.pass.cpp
@@ -18,125 +18,104 @@
 #include "test_macros.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef std::set<int> M;
-        typedef int V;
-        typedef M::iterator I;
-        V ar[] =
-        {
-            1,
-            2,
-            3,
-            4,
-            5,
-            6,
-            7,
-            8
-        };
-        M m(ar, ar + sizeof(ar)/sizeof(ar[0]));
-        assert(m.size() == 8);
-        I i = m.erase(std::next(m.cbegin(), 5), std::next(m.cbegin(), 5));
-        assert(m.size() == 8);
-        assert(i == std::next(m.begin(), 5));
-        assert(*std::next(m.begin(), 0) == 1);
-        assert(*std::next(m.begin(), 1) == 2);
-        assert(*std::next(m.begin(), 2) == 3);
-        assert(*std::next(m.begin(), 3) == 4);
-        assert(*std::next(m.begin(), 4) == 5);
-        assert(*std::next(m.begin(), 5) == 6);
-        assert(*std::next(m.begin(), 6) == 7);
-        assert(*std::next(m.begin(), 7) == 8);
+int main(int, char**) {
+  {
+    typedef std::set<int> M;
+    typedef int V;
+    typedef M::iterator I;
+    V ar[] = {1, 2, 3, 4, 5, 6, 7, 8};
+    M m(ar, ar + sizeof(ar) / sizeof(ar[0]));
+    assert(m.size() == 8);
+    I i = m.erase(std::next(m.cbegin(), 5), std::next(m.cbegin(), 5));
+    assert(m.size() == 8);
+    assert(i == std::next(m.begin(), 5));
+    assert(*std::next(m.begin(), 0) == 1);
+    assert(*std::next(m.begin(), 1) == 2);
+    assert(*std::next(m.begin(), 2) == 3);
+    assert(*std::next(m.begin(), 3) == 4);
+    assert(*std::next(m.begin(), 4) == 5);
+    assert(*std::next(m.begin(), 5) == 6);
+    assert(*std::next(m.begin(), 6) == 7);
+    assert(*std::next(m.begin(), 7) == 8);
 
-        i = m.erase(std::next(m.cbegin(), 3), std::next(m.cbegin(), 4));
-        assert(m.size() == 7);
-        assert(i == std::next(m.begin(), 3));
-        assert(*std::next(m.begin(), 0) == 1);
-        assert(*std::next(m.begin(), 1) == 2);
-        assert(*std::next(m.begin(), 2) == 3);
-        assert(*std::next(m.begin(), 3) == 5);
-        assert(*std::next(m.begin(), 4) == 6);
-        assert(*std::next(m.begin(), 5) == 7);
-        assert(*std::next(m.begin(), 6) == 8);
+    i = m.erase(std::next(m.cbegin(), 3), std::next(m.cbegin(), 4));
+    assert(m.size() == 7);
+    assert(i == std::next(m.begin(), 3));
+    assert(*std::next(m.begin(), 0) == 1);
+    assert(*std::next(m.begin(), 1) == 2);
+    assert(*std::next(m.begin(), 2) == 3);
+    assert(*std::next(m.begin(), 3) == 5);
+    assert(*std::next(m.begin(), 4) == 6);
+    assert(*std::next(m.begin(), 5) == 7);
+    assert(*std::next(m.begin(), 6) == 8);
 
-        i = m.erase(std::next(m.cbegin(), 2), std::next(m.cbegin(), 5));
-        assert(m.size() == 4);
-        assert(i == std::next(m.begin(), 2));
-        assert(*std::next(m.begin(), 0) == 1);
-        assert(*std::next(m.begin(), 1) == 2);
-        assert(*std::next(m.begin(), 2) == 7);
-        assert(*std::next(m.begin(), 3) == 8);
+    i = m.erase(std::next(m.cbegin(), 2), std::next(m.cbegin(), 5));
+    assert(m.size() == 4);
+    assert(i == std::next(m.begin(), 2));
+    assert(*std::next(m.begin(), 0) == 1);
+    assert(*std::next(m.begin(), 1) == 2);
+    assert(*std::next(m.begin(), 2) == 7);
+    assert(*std::next(m.begin(), 3) == 8);
 
-        i = m.erase(std::next(m.cbegin(), 0), std::next(m.cbegin(), 2));
-        assert(m.size() == 2);
-        assert(i == std::next(m.begin(), 0));
-        assert(*std::next(m.begin(), 0) == 7);
-        assert(*std::next(m.begin(), 1) == 8);
+    i = m.erase(std::next(m.cbegin(), 0), std::next(m.cbegin(), 2));
+    assert(m.size() == 2);
+    assert(i == std::next(m.begin(), 0));
+    assert(*std::next(m.begin(), 0) == 7);
+    assert(*std::next(m.begin(), 1) == 8);
 
-        i = m.erase(m.cbegin(), m.cend());
-        assert(m.size() == 0);
-        assert(i == m.end());
-    }
+    i = m.erase(m.cbegin(), m.cend());
+    assert(m.size() == 0);
+    assert(i == m.end());
+  }
 #if TEST_STD_VER >= 11
-    {
-        typedef std::set<int, std::less<int>, min_allocator<int>> M;
-        typedef int V;
-        typedef M::iterator I;
-        V ar[] =
-        {
-            1,
-            2,
-            3,
-            4,
-            5,
-            6,
-            7,
-            8
-        };
-        M m(ar, ar + sizeof(ar)/sizeof(ar[0]));
-        assert(m.size() == 8);
-        I i = m.erase(std::next(m.cbegin(), 5), std::next(m.cbegin(), 5));
-        assert(m.size() == 8);
-        assert(i == std::next(m.begin(), 5));
-        assert(*std::next(m.begin(), 0) == 1);
-        assert(*std::next(m.begin(), 1) == 2);
-        assert(*std::next(m.begin(), 2) == 3);
-        assert(*std::next(m.begin(), 3) == 4);
-        assert(*std::next(m.begin(), 4) == 5);
-        assert(*std::next(m.begin(), 5) == 6);
-        assert(*std::next(m.begin(), 6) == 7);
-        assert(*std::next(m.begin(), 7) == 8);
+  {
+    typedef std::set<int, std::less<int>, min_allocator<int>> M;
+    typedef int V;
+    typedef M::iterator I;
+    V ar[] = {1, 2, 3, 4, 5, 6, 7, 8};
+    M m(ar, ar + sizeof(ar) / sizeof(ar[0]));
+    assert(m.size() == 8);
+    I i = m.erase(std::next(m.cbegin(), 5), std::next(m.cbegin(), 5));
+    assert(m.size() == 8);
+    assert(i == std::next(m.begin(), 5));
+    assert(*std::next(m.begin(), 0) == 1);
+    assert(*std::next(m.begin(), 1) == 2);
+    assert(*std::next(m.begin(), 2) == 3);
+    assert(*std::next(m.begin(), 3) == 4);
+    assert(*std::next(m.begin(), 4) == 5);
+    assert(*std::next(m.begin(), 5) == 6);
+    assert(*std::next(m.begin(), 6) == 7);
+    assert(*std::next(m.begin(), 7) == 8);
 
-        i = m.erase(std::next(m.cbegin(), 3), std::next(m.cbegin(), 4));
-        assert(m.size() == 7);
-        assert(i == std::next(m.begin(), 3));
-        assert(*std::next(m.begin(), 0) == 1);
-        assert(*std::next(m.begin(), 1) == 2);
-        assert(*std::next(m.begin(), 2) == 3);
-        assert(*std::next(m.begin(), 3) == 5);
-        assert(*std::next(m.begin(), 4) == 6);
-        assert(*std::next(m.begin(), 5) == 7);
-        assert(*std::next(m.begin(), 6) == 8);
+    i = m.erase(std::next(m.cbegin(), 3), std::next(m.cbegin(), 4));
+    assert(m.size() == 7);
+    assert(i == std::next(m.begin(), 3));
+    assert(*std::next(m.begin(), 0) == 1);
+    assert(*std::next(m.begin(), 1) == 2);
+    assert(*std::next(m.begin(), 2) == 3);
+    assert(*std::next(m.begin(), 3) == 5);
+    assert(*std::next(m.begin(), 4) == 6);
+    assert(*std::next(m.begin(), 5) == 7);
+    assert(*std::next(m.begin(), 6) == 8);
 
-        i = m.erase(std::next(m.cbegin(), 2), std::next(m.cbegin(), 5));
-        assert(m.size() == 4);
-        assert(i == std::next(m.begin(), 2));
-        assert(*std::next(m.begin(), 0) == 1);
-        assert(*std::next(m.begin(), 1) == 2);
-        assert(*std::next(m.begin(), 2) == 7);
-        assert(*std::next(m.begin(), 3) == 8);
+    i = m.erase(std::next(m.cbegin(), 2), std::next(m.cbegin(), 5));
+    assert(m.size() == 4);
+    assert(i == std::next(m.begin(), 2));
+    assert(*std::next(m.begin(), 0) == 1);
+    assert(*std::next(m.begin(), 1) == 2);
+    assert(*std::next(m.begin(), 2) == 7);
+    assert(*std::next(m.begin(), 3) == 8);
 
-        i = m.erase(std::next(m.cbegin(), 0), std::next(m.cbegin(), 2));
-        assert(m.size() == 2);
-        assert(i == std::next(m.begin(), 0));
-        assert(*std::next(m.begin(), 0) == 7);
-        assert(*std::next(m.begin(), 1) == 8);
+    i = m.erase(std::next(m.cbegin(), 0), std::next(m.cbegin(), 2));
+    assert(m.size() == 2);
+    assert(i == std::next(m.begin(), 0));
+    assert(*std::next(m.begin(), 0) == 7);
+    assert(*std::next(m.begin(), 1) == 8);
 
-        i = m.erase(m.cbegin(), m.cend());
-        assert(m.size() == 0);
-        assert(i == m.end());
-    }
+    i = m.erase(m.cbegin(), m.cend());
+    assert(m.size() == 0);
+    assert(i == m.end());
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/associative/set/erase_key.pass.cpp b/libcxx/test/std/containers/associative/set/erase_key.pass.cpp
index 03b78eb12312f..318c35d21cd87 100644
--- a/libcxx/test/std/containers/associative/set/erase_key.pass.cpp
+++ b/libcxx/test/std/containers/associative/set/erase_key.pass.cpp
@@ -18,187 +18,166 @@
 #include "test_macros.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef std::set<int> M;
-        typedef int V;
-        typedef M::size_type I;
-        V ar[] =
-        {
-            1,
-            2,
-            3,
-            4,
-            5,
-            6,
-            7,
-            8
-        };
-        M m(ar, ar + sizeof(ar)/sizeof(ar[0]));
-        assert(m.size() == 8);
-        I i = m.erase(9);
-        assert(m.size() == 8);
-        assert(i == 0);
-        assert(*std::next(m.begin(), 0) == 1);
-        assert(*std::next(m.begin(), 1) == 2);
-        assert(*std::next(m.begin(), 2) == 3);
-        assert(*std::next(m.begin(), 3) == 4);
-        assert(*std::next(m.begin(), 4) == 5);
-        assert(*std::next(m.begin(), 5) == 6);
-        assert(*std::next(m.begin(), 6) == 7);
-        assert(*std::next(m.begin(), 7) == 8);
-
-        i = m.erase(4);
-        assert(m.size() == 7);
-        assert(i == 1);
-        assert(*std::next(m.begin(), 0) == 1);
-        assert(*std::next(m.begin(), 1) == 2);
-        assert(*std::next(m.begin(), 2) == 3);
-        assert(*std::next(m.begin(), 3) == 5);
-        assert(*std::next(m.begin(), 4) == 6);
-        assert(*std::next(m.begin(), 5) == 7);
-        assert(*std::next(m.begin(), 6) == 8);
-
-        i = m.erase(1);
-        assert(m.size() == 6);
-        assert(i == 1);
-        assert(*std::next(m.begin(), 0) == 2);
-        assert(*std::next(m.begin(), 1) == 3);
-        assert(*std::next(m.begin(), 2) == 5);
-        assert(*std::next(m.begin(), 3) == 6);
-        assert(*std::next(m.begin(), 4) == 7);
-        assert(*std::next(m.begin(), 5) == 8);
-
-        i = m.erase(8);
-        assert(m.size() == 5);
-        assert(i == 1);
-        assert(*std::next(m.begin(), 0) == 2);
-        assert(*std::next(m.begin(), 1) == 3);
-        assert(*std::next(m.begin(), 2) == 5);
-        assert(*std::next(m.begin(), 3) == 6);
-        assert(*std::next(m.begin(), 4) == 7);
-
-        i = m.erase(3);
-        assert(m.size() == 4);
-        assert(i == 1);
-        assert(*std::next(m.begin(), 0) == 2);
-        assert(*std::next(m.begin(), 1) == 5);
-        assert(*std::next(m.begin(), 2) == 6);
-        assert(*std::next(m.begin(), 3) == 7);
-
-        i = m.erase(6);
-        assert(m.size() == 3);
-        assert(i == 1);
-        assert(*std::next(m.begin(), 0) == 2);
-        assert(*std::next(m.begin(), 1) == 5);
-        assert(*std::next(m.begin(), 2) == 7);
-
-        i = m.erase(7);
-        assert(m.size() == 2);
-        assert(i == 1);
-        assert(*std::next(m.begin(), 0) == 2);
-        assert(*std::next(m.begin(), 1) == 5);
-
-        i = m.erase(2);
-        assert(m.size() == 1);
-        assert(i == 1);
-        assert(*std::next(m.begin(), 0) == 5);
-
-        i = m.erase(5);
-        assert(m.size() == 0);
-        assert(i == 1);
-    }
+int main(int, char**) {
+  {
+    typedef std::set<int> M;
+    typedef int V;
+    typedef M::size_type I;
+    V ar[] = {1, 2, 3, 4, 5, 6, 7, 8};
+    M m(ar, ar + sizeof(ar) / sizeof(ar[0]));
+    assert(m.size() == 8);
+    I i = m.erase(9);
+    assert(m.size() == 8);
+    assert(i == 0);
+    assert(*std::next(m.begin(), 0) == 1);
+    assert(*std::next(m.begin(), 1) == 2);
+    assert(*std::next(m.begin(), 2) == 3);
+    assert(*std::next(m.begin(), 3) == 4);
+    assert(*std::next(m.begin(), 4) == 5);
+    assert(*std::next(m.begin(), 5) == 6);
+    assert(*std::next(m.begin(), 6) == 7);
+    assert(*std::next(m.begin(), 7) == 8);
+
+    i = m.erase(4);
+    assert(m.size() == 7);
+    assert(i == 1);
+    assert(*std::next(m.begin(), 0) == 1);
+    assert(*std::next(m.begin(), 1) == 2);
+    assert(*std::next(m.begin(), 2) == 3);
+    assert(*std::next(m.begin(), 3) == 5);
+    assert(*std::next(m.begin(), 4) == 6);
+    assert(*std::next(m.begin(), 5) == 7);
+    assert(*std::next(m.begin(), 6) == 8);
+
+    i = m.erase(1);
+    assert(m.size() == 6);
+    assert(i == 1);
+    assert(*std::next(m.begin(), 0) == 2);
+    assert(*std::next(m.begin(), 1) == 3);
+    assert(*std::next(m.begin(), 2) == 5);
+    assert(*std::next(m.begin(), 3) == 6);
+    assert(*std::next(m.begin(), 4) == 7);
+    assert(*std::next(m.begin(), 5) == 8);
+
+    i = m.erase(8);
+    assert(m.size() == 5);
+    assert(i == 1);
+    assert(*std::next(m.begin(), 0) == 2);
+    assert(*std::next(m.begin(), 1) == 3);
+    assert(*std::next(m.begin(), 2) == 5);
+    assert(*std::next(m.begin(), 3) == 6);
+    assert(*std::next(m.begin(), 4) == 7);
+
+    i = m.erase(3);
+    assert(m.size() == 4);
+    assert(i == 1);
+    assert(*std::next(m.begin(), 0) == 2);
+    assert(*std::next(m.begin(), 1) == 5);
+    assert(*std::next(m.begin(), 2) == 6);
+    assert(*std::next(m.begin(), 3) == 7);
+
+    i = m.erase(6);
+    assert(m.size() == 3);
+    assert(i == 1);
+    assert(*std::next(m.begin(), 0) == 2);
+    assert(*std::next(m.begin(), 1) == 5);
+    assert(*std::next(m.begin(), 2) == 7);
+
+    i = m.erase(7);
+    assert(m.size() == 2);
+    assert(i == 1);
+    assert(*std::next(m.begin(), 0) == 2);
+    assert(*std::next(m.begin(), 1) == 5);
+
+    i = m.erase(2);
+    assert(m.size() == 1);
+    assert(i == 1);
+    assert(*std::next(m.begin(), 0) == 5);
+
+    i = m.erase(5);
+    assert(m.size() == 0);
+    assert(i == 1);
+  }
 #if TEST_STD_VER >= 11
-    {
-        typedef std::set<int, std::less<int>, min_allocator<int>> M;
-        typedef int V;
-        typedef M::size_type I;
-        V ar[] =
-        {
-            1,
-            2,
-            3,
-            4,
-            5,
-            6,
-            7,
-            8
-        };
-        M m(ar, ar + sizeof(ar)/sizeof(ar[0]));
-        assert(m.size() == 8);
-        I i = m.erase(9);
-        assert(m.size() == 8);
-        assert(i == 0);
-        assert(*std::next(m.begin(), 0) == 1);
-        assert(*std::next(m.begin(), 1) == 2);
-        assert(*std::next(m.begin(), 2) == 3);
-        assert(*std::next(m.begin(), 3) == 4);
-        assert(*std::next(m.begin(), 4) == 5);
-        assert(*std::next(m.begin(), 5) == 6);
-        assert(*std::next(m.begin(), 6) == 7);
-        assert(*std::next(m.begin(), 7) == 8);
-
-        i = m.erase(4);
-        assert(m.size() == 7);
-        assert(i == 1);
-        assert(*std::next(m.begin(), 0) == 1);
-        assert(*std::next(m.begin(), 1) == 2);
-        assert(*std::next(m.begin(), 2) == 3);
-        assert(*std::next(m.begin(), 3) == 5);
-        assert(*std::next(m.begin(), 4) == 6);
-        assert(*std::next(m.begin(), 5) == 7);
-        assert(*std::next(m.begin(), 6) == 8);
-
-        i = m.erase(1);
-        assert(m.size() == 6);
-        assert(i == 1);
-        assert(*std::next(m.begin(), 0) == 2);
-        assert(*std::next(m.begin(), 1) == 3);
-        assert(*std::next(m.begin(), 2) == 5);
-        assert(*std::next(m.begin(), 3) == 6);
-        assert(*std::next(m.begin(), 4) == 7);
-        assert(*std::next(m.begin(), 5) == 8);
-
-        i = m.erase(8);
-        assert(m.size() == 5);
-        assert(i == 1);
-        assert(*std::next(m.begin(), 0) == 2);
-        assert(*std::next(m.begin(), 1) == 3);
-        assert(*std::next(m.begin(), 2) == 5);
-        assert(*std::next(m.begin(), 3) == 6);
-        assert(*std::next(m.begin(), 4) == 7);
-
-        i = m.erase(3);
-        assert(m.size() == 4);
-        assert(i == 1);
-        assert(*std::next(m.begin(), 0) == 2);
-        assert(*std::next(m.begin(), 1) == 5);
-        assert(*std::next(m.begin(), 2) == 6);
-        assert(*std::next(m.begin(), 3) == 7);
-
-        i = m.erase(6);
-        assert(m.size() == 3);
-        assert(i == 1);
-        assert(*std::next(m.begin(), 0) == 2);
-        assert(*std::next(m.begin(), 1) == 5);
-        assert(*std::next(m.begin(), 2) == 7);
-
-        i = m.erase(7);
-        assert(m.size() == 2);
-        assert(i == 1);
-        assert(*std::next(m.begin(), 0) == 2);
-        assert(*std::next(m.begin(), 1) == 5);
-
-        i = m.erase(2);
-        assert(m.size() == 1);
-        assert(i == 1);
-        assert(*std::next(m.begin(), 0) == 5);
-
-        i = m.erase(5);
-        assert(m.size() == 0);
-        assert(i == 1);
-    }
+  {
+    typedef std::set<int, std::less<int>, min_allocator<int>> M;
+    typedef int V;
+    typedef M::size_type I;
+    V ar[] = {1, 2, 3, 4, 5, 6, 7, 8};
+    M m(ar, ar + sizeof(ar) / sizeof(ar[0]));
+    assert(m.size() == 8);
+    I i = m.erase(9);
+    assert(m.size() == 8);
+    assert(i == 0);
+    assert(*std::next(m.begin(), 0) == 1);
+    assert(*std::next(m.begin(), 1) == 2);
+    assert(*std::next(m.begin(), 2) == 3);
+    assert(*std::next(m.begin(), 3) == 4);
+    assert(*std::next(m.begin(), 4) == 5);
+    assert(*std::next(m.begin(), 5) == 6);
+    assert(*std::next(m.begin(), 6) == 7);
+    assert(*std::next(m.begin(), 7) == 8);
+
+    i = m.erase(4);
+    assert(m.size() == 7);
+    assert(i == 1);
+    assert(*std::next(m.begin(), 0) == 1);
+    assert(*std::next(m.begin(), 1) == 2);
+    assert(*std::next(m.begin(), 2) == 3);
+    assert(*std::next(m.begin(), 3) == 5);
+    assert(*std::next(m.begin(), 4) == 6);
+    assert(*std::next(m.begin(), 5) == 7);
+    assert(*std::next(m.begin(), 6) == 8);
+
+    i = m.erase(1);
+    assert(m.size() == 6);
+    assert(i == 1);
+    assert(*std::next(m.begin(), 0) == 2);
+    assert(*std::next(m.begin(), 1) == 3);
+    assert(*std::next(m.begin(), 2) == 5);
+    assert(*std::next(m.begin(), 3) == 6);
+    assert(*std::next(m.begin(), 4) == 7);
+    assert(*std::next(m.begin(), 5) == 8);
+
+    i = m.erase(8);
+    assert(m.size() == 5);
+    assert(i == 1);
+    assert(*std::next(m.begin(), 0) == 2);
+    assert(*std::next(m.begin(), 1) == 3);
+    assert(*std::next(m.begin(), 2) == 5);
+    assert(*std::next(m.begin(), 3) == 6);
+    assert(*std::next(m.begin(), 4) == 7);
+
+    i = m.erase(3);
+    assert(m.size() == 4);
+    assert(i == 1);
+    assert(*std::next(m.begin(), 0) == 2);
+    assert(*std::next(m.begin(), 1) == 5);
+    assert(*std::next(m.begin(), 2) == 6);
+    assert(*std::next(m.begin(), 3) == 7);
+
+    i = m.erase(6);
+    assert(m.size() == 3);
+    assert(i == 1);
+    assert(*std::next(m.begin(), 0) == 2);
+    assert(*std::next(m.begin(), 1) == 5);
+    assert(*std::next(m.begin(), 2) == 7);
+
+    i = m.erase(7);
+    assert(m.size() == 2);
+    assert(i == 1);
+    assert(*std::next(m.begin(), 0) == 2);
+    assert(*std::next(m.begin(), 1) == 5);
+
+    i = m.erase(2);
+    assert(m.size() == 1);
+    assert(i == 1);
+    assert(*std::next(m.begin(), 0) == 5);
+
+    i = m.erase(5);
+    assert(m.size() == 0);
+    assert(i == 1);
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/associative/set/extract_iterator.pass.cpp b/libcxx/test/std/containers/associative/set/extract_iterator.pass.cpp
index c6ef11301d7a6..2d7b6e521d961 100644
--- a/libcxx/test/std/containers/associative/set/extract_iterator.pass.cpp
+++ b/libcxx/test/std/containers/associative/set/extract_iterator.pass.cpp
@@ -20,43 +20,40 @@
 #include "Counter.h"
 
 template <class Container>
-void test(Container& c)
-{
-    std::size_t sz = c.size();
-
-    for (auto first = c.cbegin(); first != c.cend();)
-    {
-        auto key_value = *first;
-        typename Container::node_type t = c.extract(first++);
-        --sz;
-        assert(t.value() == key_value);
-        assert(t.get_allocator() == c.get_allocator());
-        assert(sz == c.size());
-    }
-
-    assert(c.size() == 0);
+void test(Container& c) {
+  std::size_t sz = c.size();
+
+  for (auto first = c.cbegin(); first != c.cend();) {
+    auto key_value                  = *first;
+    typename Container::node_type t = c.extract(first++);
+    --sz;
+    assert(t.value() == key_value);
+    assert(t.get_allocator() == c.get_allocator());
+    assert(sz == c.size());
+  }
+
+  assert(c.size() == 0);
 }
 
-int main(int, char**)
-{
-    {
-        using set_type = std::set<int>;
-        set_type m = {1, 2, 3, 4, 5, 6};
-        test(m);
-    }
-
-    {
-        std::set<Counter<int>> m = {1, 2, 3, 4, 5, 6};
-        assert(Counter_base::gConstructed == 6);
-        test(m);
-        assert(Counter_base::gConstructed == 0);
-    }
-
-    {
-        using min_alloc_set = std::set<int, std::less<int>, min_allocator<int>>;
-        min_alloc_set m = {1, 2, 3, 4, 5, 6};
-        test(m);
-    }
+int main(int, char**) {
+  {
+    using set_type = std::set<int>;
+    set_type m     = {1, 2, 3, 4, 5, 6};
+    test(m);
+  }
+
+  {
+    std::set<Counter<int>> m = {1, 2, 3, 4, 5, 6};
+    assert(Counter_base::gConstructed == 6);
+    test(m);
+    assert(Counter_base::gConstructed == 0);
+  }
+
+  {
+    using min_alloc_set = std::set<int, std::less<int>, min_allocator<int>>;
+    min_alloc_set m     = {1, 2, 3, 4, 5, 6};
+    test(m);
+  }
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/associative/set/extract_key.pass.cpp b/libcxx/test/std/containers/associative/set/extract_key.pass.cpp
index f37665b4a2a28..0802387f2c9b3 100644
--- a/libcxx/test/std/containers/associative/set/extract_key.pass.cpp
+++ b/libcxx/test/std/containers/associative/set/extract_key.pass.cpp
@@ -20,54 +20,50 @@
 #include "Counter.h"
 
 template <class Container, class KeyTypeIter>
-void test(Container& c, KeyTypeIter first, KeyTypeIter last)
-{
-    std::size_t sz = c.size();
-    assert((std::size_t)std::distance(first, last) == sz);
+void test(Container& c, KeyTypeIter first, KeyTypeIter last) {
+  std::size_t sz = c.size();
+  assert((std::size_t)std::distance(first, last) == sz);
 
-    for (KeyTypeIter copy = first; copy != last; ++copy)
-    {
-        typename Container::node_type t = c.extract(*copy);
-        assert(!t.empty());
-        --sz;
-        assert(t.value() == *copy);
-        assert(t.get_allocator() == c.get_allocator());
-        assert(sz == c.size());
-    }
+  for (KeyTypeIter copy = first; copy != last; ++copy) {
+    typename Container::node_type t = c.extract(*copy);
+    assert(!t.empty());
+    --sz;
+    assert(t.value() == *copy);
+    assert(t.get_allocator() == c.get_allocator());
+    assert(sz == c.size());
+  }
 
-    assert(c.size() == 0);
+  assert(c.size() == 0);
 
-    for (KeyTypeIter copy = first; copy != last; ++copy)
-    {
-        typename Container::node_type t = c.extract(*copy);
-        assert(t.empty());
-    }
+  for (KeyTypeIter copy = first; copy != last; ++copy) {
+    typename Container::node_type t = c.extract(*copy);
+    assert(t.empty());
+  }
 }
 
-int main(int, char**)
-{
-    {
-        std::set<int> m = {1, 2, 3, 4, 5, 6};
-        int keys[] = {1, 2, 3, 4, 5, 6};
-        test(m, std::begin(keys), std::end(keys));
-    }
+int main(int, char**) {
+  {
+    std::set<int> m = {1, 2, 3, 4, 5, 6};
+    int keys[]      = {1, 2, 3, 4, 5, 6};
+    test(m, std::begin(keys), std::end(keys));
+  }
 
+  {
+    std::set<Counter<int>> m = {1, 2, 3, 4, 5, 6};
     {
-        std::set<Counter<int>> m = {1, 2, 3, 4, 5, 6};
-        {
-            Counter<int> keys[] = {1, 2, 3, 4, 5, 6};
-            assert(Counter_base::gConstructed == 6+6);
-            test(m, std::begin(keys), std::end(keys));
-        }
-        assert(Counter_base::gConstructed == 0);
+      Counter<int> keys[] = {1, 2, 3, 4, 5, 6};
+      assert(Counter_base::gConstructed == 6 + 6);
+      test(m, std::begin(keys), std::end(keys));
     }
+    assert(Counter_base::gConstructed == 0);
+  }
 
-    {
-        using min_alloc_set = std::set<int, std::less<int>, min_allocator<int>>;
-        min_alloc_set m = {1, 2, 3, 4, 5, 6};
-        int keys[] = {1, 2, 3, 4, 5, 6};
-        test(m, std::begin(keys), std::end(keys));
-    }
+  {
+    using min_alloc_set = std::set<int, std::less<int>, min_allocator<int>>;
+    min_alloc_set m     = {1, 2, 3, 4, 5, 6};
+    int keys[]          = {1, 2, 3, 4, 5, 6};
+    test(m, std::begin(keys), std::end(keys));
+  }
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/associative/set/find.pass.cpp b/libcxx/test/std/containers/associative/set/find.pass.cpp
index 081ff94fd0f4a..88ceff0cb144f 100644
--- a/libcxx/test/std/containers/associative/set/find.pass.cpp
+++ b/libcxx/test/std/containers/associative/set/find.pass.cpp
@@ -20,168 +20,117 @@
 #include "min_allocator.h"
 #include "private_constructor.h"
 
-int main(int, char**)
-{
-    {
+int main(int, char**) {
+  {
     typedef int V;
     typedef std::set<int> M;
     {
-        typedef M::iterator R;
-        V ar[] =
-        {
-            5,
-            6,
-            7,
-            8,
-            9,
-            10,
-            11,
-            12
-        };
-        M m(ar, ar+sizeof(ar)/sizeof(ar[0]));
-        R r = m.find(5);
-        assert(r == m.begin());
-        r = m.find(6);
-        assert(r == std::next(m.begin()));
-        r = m.find(7);
-        assert(r == std::next(m.begin(), 2));
-        r = m.find(8);
-        assert(r == std::next(m.begin(), 3));
-        r = m.find(9);
-        assert(r == std::next(m.begin(), 4));
-        r = m.find(10);
-        assert(r == std::next(m.begin(), 5));
-        r = m.find(11);
-        assert(r == std::next(m.begin(), 6));
-        r = m.find(12);
-        assert(r == std::next(m.begin(), 7));
-        r = m.find(4);
-        assert(r == std::next(m.begin(), 8));
+      typedef M::iterator R;
+      V ar[] = {5, 6, 7, 8, 9, 10, 11, 12};
+      M m(ar, ar + sizeof(ar) / sizeof(ar[0]));
+      R r = m.find(5);
+      assert(r == m.begin());
+      r = m.find(6);
+      assert(r == std::next(m.begin()));
+      r = m.find(7);
+      assert(r == std::next(m.begin(), 2));
+      r = m.find(8);
+      assert(r == std::next(m.begin(), 3));
+      r = m.find(9);
+      assert(r == std::next(m.begin(), 4));
+      r = m.find(10);
+      assert(r == std::next(m.begin(), 5));
+      r = m.find(11);
+      assert(r == std::next(m.begin(), 6));
+      r = m.find(12);
+      assert(r == std::next(m.begin(), 7));
+      r = m.find(4);
+      assert(r == std::next(m.begin(), 8));
     }
     {
-        typedef M::const_iterator R;
-        V ar[] =
-        {
-            5,
-            6,
-            7,
-            8,
-            9,
-            10,
-            11,
-            12
-        };
-        const M m(ar, ar+sizeof(ar)/sizeof(ar[0]));
-        R r = m.find(5);
-        assert(r == m.begin());
-        r = m.find(6);
-        assert(r == std::next(m.begin()));
-        r = m.find(7);
-        assert(r == std::next(m.begin(), 2));
-        r = m.find(8);
-        assert(r == std::next(m.begin(), 3));
-        r = m.find(9);
-        assert(r == std::next(m.begin(), 4));
-        r = m.find(10);
-        assert(r == std::next(m.begin(), 5));
-        r = m.find(11);
-        assert(r == std::next(m.begin(), 6));
-        r = m.find(12);
-        assert(r == std::next(m.begin(), 7));
-        r = m.find(4);
-        assert(r == std::next(m.begin(), 8));
-    }
+      typedef M::const_iterator R;
+      V ar[] = {5, 6, 7, 8, 9, 10, 11, 12};
+      const M m(ar, ar + sizeof(ar) / sizeof(ar[0]));
+      R r = m.find(5);
+      assert(r == m.begin());
+      r = m.find(6);
+      assert(r == std::next(m.begin()));
+      r = m.find(7);
+      assert(r == std::next(m.begin(), 2));
+      r = m.find(8);
+      assert(r == std::next(m.begin(), 3));
+      r = m.find(9);
+      assert(r == std::next(m.begin(), 4));
+      r = m.find(10);
+      assert(r == std::next(m.begin(), 5));
+      r = m.find(11);
+      assert(r == std::next(m.begin(), 6));
+      r = m.find(12);
+      assert(r == std::next(m.begin(), 7));
+      r = m.find(4);
+      assert(r == std::next(m.begin(), 8));
     }
+  }
 #if TEST_STD_VER >= 11
-    {
+  {
     typedef int V;
     typedef std::set<int, std::less<int>, min_allocator<int>> M;
     {
-        typedef M::iterator R;
-        V ar[] =
-        {
-            5,
-            6,
-            7,
-            8,
-            9,
-            10,
-            11,
-            12
-        };
-        M m(ar, ar+sizeof(ar)/sizeof(ar[0]));
-        R r = m.find(5);
-        assert(r == m.begin());
-        r = m.find(6);
-        assert(r == std::next(m.begin()));
-        r = m.find(7);
-        assert(r == std::next(m.begin(), 2));
-        r = m.find(8);
-        assert(r == std::next(m.begin(), 3));
-        r = m.find(9);
-        assert(r == std::next(m.begin(), 4));
-        r = m.find(10);
-        assert(r == std::next(m.begin(), 5));
-        r = m.find(11);
-        assert(r == std::next(m.begin(), 6));
-        r = m.find(12);
-        assert(r == std::next(m.begin(), 7));
-        r = m.find(4);
-        assert(r == std::next(m.begin(), 8));
+      typedef M::iterator R;
+      V ar[] = {5, 6, 7, 8, 9, 10, 11, 12};
+      M m(ar, ar + sizeof(ar) / sizeof(ar[0]));
+      R r = m.find(5);
+      assert(r == m.begin());
+      r = m.find(6);
+      assert(r == std::next(m.begin()));
+      r = m.find(7);
+      assert(r == std::next(m.begin(), 2));
+      r = m.find(8);
+      assert(r == std::next(m.begin(), 3));
+      r = m.find(9);
+      assert(r == std::next(m.begin(), 4));
+      r = m.find(10);
+      assert(r == std::next(m.begin(), 5));
+      r = m.find(11);
+      assert(r == std::next(m.begin(), 6));
+      r = m.find(12);
+      assert(r == std::next(m.begin(), 7));
+      r = m.find(4);
+      assert(r == std::next(m.begin(), 8));
     }
     {
-        typedef M::const_iterator R;
-        V ar[] =
-        {
-            5,
-            6,
-            7,
-            8,
-            9,
-            10,
-            11,
-            12
-        };
-        const M m(ar, ar+sizeof(ar)/sizeof(ar[0]));
-        R r = m.find(5);
-        assert(r == m.begin());
-        r = m.find(6);
-        assert(r == std::next(m.begin()));
-        r = m.find(7);
-        assert(r == std::next(m.begin(), 2));
-        r = m.find(8);
-        assert(r == std::next(m.begin(), 3));
-        r = m.find(9);
-        assert(r == std::next(m.begin(), 4));
-        r = m.find(10);
-        assert(r == std::next(m.begin(), 5));
-        r = m.find(11);
-        assert(r == std::next(m.begin(), 6));
-        r = m.find(12);
-        assert(r == std::next(m.begin(), 7));
-        r = m.find(4);
-        assert(r == std::next(m.begin(), 8));
-    }
+      typedef M::const_iterator R;
+      V ar[] = {5, 6, 7, 8, 9, 10, 11, 12};
+      const M m(ar, ar + sizeof(ar) / sizeof(ar[0]));
+      R r = m.find(5);
+      assert(r == m.begin());
+      r = m.find(6);
+      assert(r == std::next(m.begin()));
+      r = m.find(7);
+      assert(r == std::next(m.begin(), 2));
+      r = m.find(8);
+      assert(r == std::next(m.begin(), 3));
+      r = m.find(9);
+      assert(r == std::next(m.begin(), 4));
+      r = m.find(10);
+      assert(r == std::next(m.begin(), 5));
+      r = m.find(11);
+      assert(r == std::next(m.begin(), 6));
+      r = m.find(12);
+      assert(r == std::next(m.begin(), 7));
+      r = m.find(4);
+      assert(r == std::next(m.begin(), 8));
     }
+  }
 #endif
 #if TEST_STD_VER > 11
-    {
+  {
     typedef int V;
     typedef std::set<V, std::less<>> M;
     typedef M::iterator R;
 
-    V ar[] =
-    {
-        5,
-        6,
-        7,
-        8,
-        9,
-        10,
-        11,
-        12
-    };
-    M m(ar, ar+sizeof(ar)/sizeof(ar[0]));
+    V ar[] = {5, 6, 7, 8, 9, 10, 11, 12};
+    M m(ar, ar + sizeof(ar) / sizeof(ar[0]));
     R r = m.find(5);
     assert(r == m.begin());
     r = m.find(6);
@@ -200,22 +149,22 @@ int main(int, char**)
     assert(r == std::next(m.begin(), 7));
     r = m.find(4);
     assert(r == std::next(m.begin(), 8));
-    }
+  }
 
-    {
+  {
     typedef PrivateConstructor V;
     typedef std::set<V, std::less<>> M;
     typedef M::iterator R;
 
     M m;
-    m.insert ( V::make ( 5 ));
-    m.insert ( V::make ( 6 ));
-    m.insert ( V::make ( 7 ));
-    m.insert ( V::make ( 8 ));
-    m.insert ( V::make ( 9 ));
-    m.insert ( V::make ( 10 ));
-    m.insert ( V::make ( 11 ));
-    m.insert ( V::make ( 12 ));
+    m.insert(V::make(5));
+    m.insert(V::make(6));
+    m.insert(V::make(7));
+    m.insert(V::make(8));
+    m.insert(V::make(9));
+    m.insert(V::make(10));
+    m.insert(V::make(11));
+    m.insert(V::make(12));
 
     R r = m.find(5);
     assert(r == m.begin());
@@ -235,7 +184,7 @@ int main(int, char**)
     assert(r == std::next(m.begin(), 7));
     r = m.find(4);
     assert(r == std::next(m.begin(), 8));
-    }
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/associative/set/get_allocator.pass.cpp b/libcxx/test/std/containers/associative/set/get_allocator.pass.cpp
index f6a7d5a97623e..0270c9e82ab0e 100644
--- a/libcxx/test/std/containers/associative/set/get_allocator.pass.cpp
+++ b/libcxx/test/std/containers/associative/set/get_allocator.pass.cpp
@@ -19,15 +19,15 @@
 #include "test_macros.h"
 
 int main(int, char**) {
-    {
-        std::allocator<int> alloc;
-        const std::set<int> s(alloc);
-        assert(s.get_allocator() == alloc);
-    }
-    {
-        other_allocator<int> alloc(1);
-        const std::set<int, std::less<int>, other_allocator<int> > s(alloc);
-        assert(s.get_allocator() == alloc);
-    }
-    return 0;
+  {
+    std::allocator<int> alloc;
+    const std::set<int> s(alloc);
+    assert(s.get_allocator() == alloc);
+  }
+  {
+    other_allocator<int> alloc(1);
+    const std::set<int, std::less<int>, other_allocator<int> > s(alloc);
+    assert(s.get_allocator() == alloc);
+  }
+  return 0;
 }
diff --git a/libcxx/test/std/containers/associative/set/incomplete_type.pass.cpp b/libcxx/test/std/containers/associative/set/incomplete_type.pass.cpp
index d50b56ff1efcd..acbda713619bd 100644
--- a/libcxx/test/std/containers/associative/set/incomplete_type.pass.cpp
+++ b/libcxx/test/std/containers/associative/set/incomplete_type.pass.cpp
@@ -16,17 +16,17 @@
 #include "test_macros.h"
 
 struct A {
-    typedef std::set<A> Set;
-    int data;
-    Set m;
-    Set::iterator it;
-    Set::const_iterator cit;
+  typedef std::set<A> Set;
+  int data;
+  Set m;
+  Set::iterator it;
+  Set::const_iterator cit;
 };
 
 inline bool operator==(A const& L, A const& R) { return &L == &R; }
-inline bool operator<(A const& L, A const& R)  { return L.data < R.data; }
+inline bool operator<(A const& L, A const& R) { return L.data < R.data; }
 int main(int, char**) {
-    A a;
+  A a;
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/associative/set/insert_and_emplace_allocator_requirements.pass.cpp b/libcxx/test/std/containers/associative/set/insert_and_emplace_allocator_requirements.pass.cpp
index 21ae9a30b4598..4711473de4dad 100644
--- a/libcxx/test/std/containers/associative/set/insert_and_emplace_allocator_requirements.pass.cpp
+++ b/libcxx/test/std/containers/associative/set/insert_and_emplace_allocator_requirements.pass.cpp
@@ -21,8 +21,7 @@
 #include "container_test_types.h"
 #include "../../set_allocator_requirement_test_templates.h"
 
-int main(int, char**)
-{
+int main(int, char**) {
   testSetInsert<TCT::set<> >();
   testSetEmplace<TCT::set<> >();
   testSetEmplaceHint<TCT::set<> >();
diff --git a/libcxx/test/std/containers/associative/set/insert_cv.pass.cpp b/libcxx/test/std/containers/associative/set/insert_cv.pass.cpp
index 4ef283410c6b3..2683f2924e760 100644
--- a/libcxx/test/std/containers/associative/set/insert_cv.pass.cpp
+++ b/libcxx/test/std/containers/associative/set/insert_cv.pass.cpp
@@ -18,50 +18,48 @@
 #include "test_macros.h"
 #include "min_allocator.h"
 
-template<class Container>
-void do_insert_cv_test()
-{
-    typedef Container M;
-    typedef std::pair<typename M::iterator, bool> R;
-    typedef typename M::value_type VT;
-    M m;
+template <class Container>
+void do_insert_cv_test() {
+  typedef Container M;
+  typedef std::pair<typename M::iterator, bool> R;
+  typedef typename M::value_type VT;
+  M m;
 
-    const VT v1(2);
-    R r = m.insert(v1);
-    assert(r.second);
-    assert(r.first == m.begin());
-    assert(m.size() == 1);
-    assert(*r.first == 2);
+  const VT v1(2);
+  R r = m.insert(v1);
+  assert(r.second);
+  assert(r.first == m.begin());
+  assert(m.size() == 1);
+  assert(*r.first == 2);
 
-    const VT v2(1);
-    r = m.insert(v2);
-    assert(r.second);
-    assert(r.first == m.begin());
-    assert(m.size() == 2);
-    assert(*r.first == 1);
+  const VT v2(1);
+  r = m.insert(v2);
+  assert(r.second);
+  assert(r.first == m.begin());
+  assert(m.size() == 2);
+  assert(*r.first == 1);
 
-    const VT v3(3);
-    r = m.insert(v3);
-    assert(r.second);
-    assert(r.first == std::prev(m.end()));
-    assert(m.size() == 3);
-    assert(*r.first == 3);
+  const VT v3(3);
+  r = m.insert(v3);
+  assert(r.second);
+  assert(r.first == std::prev(m.end()));
+  assert(m.size() == 3);
+  assert(*r.first == 3);
 
-    r = m.insert(v3);
-    assert(!r.second);
-    assert(r.first == std::prev(m.end()));
-    assert(m.size() == 3);
-    assert(*r.first == 3);
+  r = m.insert(v3);
+  assert(!r.second);
+  assert(r.first == std::prev(m.end()));
+  assert(m.size() == 3);
+  assert(*r.first == 3);
 }
 
-int main(int, char**)
-{
-    do_insert_cv_test<std::set<int> >();
+int main(int, char**) {
+  do_insert_cv_test<std::set<int> >();
 #if TEST_STD_VER >= 11
-    {
-        typedef std::set<int, std::less<int>, min_allocator<int>> M;
-        do_insert_cv_test<M>();
-    }
+  {
+    typedef std::set<int, std::less<int>, min_allocator<int>> M;
+    do_insert_cv_test<M>();
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/associative/set/insert_initializer_list.pass.cpp b/libcxx/test/std/containers/associative/set/insert_initializer_list.pass.cpp
index 913be9aab299b..f3b69a59987e8 100644
--- a/libcxx/test/std/containers/associative/set/insert_initializer_list.pass.cpp
+++ b/libcxx/test/std/containers/associative/set/insert_initializer_list.pass.cpp
@@ -21,9 +21,8 @@
 #include "test_macros.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
+int main(int, char**) {
+  {
     typedef std::set<int> C;
     typedef C::value_type V;
     C m = {10, 8};
@@ -39,8 +38,8 @@ int main(int, char**)
     assert(*++i == V(6));
     assert(*++i == V(8));
     assert(*++i == V(10));
-    }
-    {
+  }
+  {
     typedef std::set<int, std::less<int>, min_allocator<int>> C;
     typedef C::value_type V;
     C m = {10, 8};
@@ -56,7 +55,7 @@ int main(int, char**)
     assert(*++i == V(6));
     assert(*++i == V(8));
     assert(*++i == V(10));
-    }
+  }
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/associative/set/insert_iter_cv.pass.cpp b/libcxx/test/std/containers/associative/set/insert_iter_cv.pass.cpp
index a7c0b654fc878..159487ee0e479 100644
--- a/libcxx/test/std/containers/associative/set/insert_iter_cv.pass.cpp
+++ b/libcxx/test/std/containers/associative/set/insert_iter_cv.pass.cpp
@@ -18,57 +18,56 @@
 #include "test_macros.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef std::set<int> M;
-        typedef M::iterator R;
-        M m;
-        R r = m.insert(m.cend(), M::value_type(2));
-        assert(r == m.begin());
-        assert(m.size() == 1);
-        assert(*r == 2);
+int main(int, char**) {
+  {
+    typedef std::set<int> M;
+    typedef M::iterator R;
+    M m;
+    R r = m.insert(m.cend(), M::value_type(2));
+    assert(r == m.begin());
+    assert(m.size() == 1);
+    assert(*r == 2);
 
-        r = m.insert(m.cend(), M::value_type(1));
-        assert(r == m.begin());
-        assert(m.size() == 2);
-        assert(*r == 1);
+    r = m.insert(m.cend(), M::value_type(1));
+    assert(r == m.begin());
+    assert(m.size() == 2);
+    assert(*r == 1);
 
-        r = m.insert(m.cend(), M::value_type(3));
-        assert(r == std::prev(m.end()));
-        assert(m.size() == 3);
-        assert(*r == 3);
+    r = m.insert(m.cend(), M::value_type(3));
+    assert(r == std::prev(m.end()));
+    assert(m.size() == 3);
+    assert(*r == 3);
 
-        r = m.insert(m.cend(), M::value_type(3));
-        assert(r == std::prev(m.end()));
-        assert(m.size() == 3);
-        assert(*r == 3);
-    }
+    r = m.insert(m.cend(), M::value_type(3));
+    assert(r == std::prev(m.end()));
+    assert(m.size() == 3);
+    assert(*r == 3);
+  }
 #if TEST_STD_VER >= 11
-    {
-        typedef std::set<int, std::less<int>, min_allocator<int>> M;
-        typedef M::iterator R;
-        M m;
-        R r = m.insert(m.cend(), M::value_type(2));
-        assert(r == m.begin());
-        assert(m.size() == 1);
-        assert(*r == 2);
+  {
+    typedef std::set<int, std::less<int>, min_allocator<int>> M;
+    typedef M::iterator R;
+    M m;
+    R r = m.insert(m.cend(), M::value_type(2));
+    assert(r == m.begin());
+    assert(m.size() == 1);
+    assert(*r == 2);
 
-        r = m.insert(m.cend(), M::value_type(1));
-        assert(r == m.begin());
-        assert(m.size() == 2);
-        assert(*r == 1);
+    r = m.insert(m.cend(), M::value_type(1));
+    assert(r == m.begin());
+    assert(m.size() == 2);
+    assert(*r == 1);
 
-        r = m.insert(m.cend(), M::value_type(3));
-        assert(r == std::prev(m.end()));
-        assert(m.size() == 3);
-        assert(*r == 3);
+    r = m.insert(m.cend(), M::value_type(3));
+    assert(r == std::prev(m.end()));
+    assert(m.size() == 3);
+    assert(*r == 3);
 
-        r = m.insert(m.cend(), M::value_type(3));
-        assert(r == std::prev(m.end()));
-        assert(m.size() == 3);
-        assert(*r == 3);
-    }
+    r = m.insert(m.cend(), M::value_type(3));
+    assert(r == std::prev(m.end()));
+    assert(m.size() == 3);
+    assert(*r == 3);
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/associative/set/insert_iter_iter.pass.cpp b/libcxx/test/std/containers/associative/set/insert_iter_iter.pass.cpp
index fe495309ba663..6c9707ca31a81 100644
--- a/libcxx/test/std/containers/associative/set/insert_iter_iter.pass.cpp
+++ b/libcxx/test/std/containers/associative/set/insert_iter_iter.pass.cpp
@@ -20,55 +20,30 @@
 #include "test_iterators.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef std::set<int> M;
-        typedef int V;
-        V ar[] =
-        {
-            1,
-            1,
-            1,
-            2,
-            2,
-            2,
-            3,
-            3,
-            3
-        };
-        M m;
-        m.insert(cpp17_input_iterator<const V*>(ar),
-                 cpp17_input_iterator<const V*>(ar + sizeof(ar)/sizeof(ar[0])));
-        assert(m.size() == 3);
-        assert(*m.begin() == 1);
-        assert(*std::next(m.begin()) == 2);
-        assert(*std::next(m.begin(), 2) == 3);
-    }
+int main(int, char**) {
+  {
+    typedef std::set<int> M;
+    typedef int V;
+    V ar[] = {1, 1, 1, 2, 2, 2, 3, 3, 3};
+    M m;
+    m.insert(cpp17_input_iterator<const V*>(ar), cpp17_input_iterator<const V*>(ar + sizeof(ar) / sizeof(ar[0])));
+    assert(m.size() == 3);
+    assert(*m.begin() == 1);
+    assert(*std::next(m.begin()) == 2);
+    assert(*std::next(m.begin(), 2) == 3);
+  }
 #if TEST_STD_VER >= 11
-    {
-        typedef std::set<int, std::less<int>, min_allocator<int>> M;
-        typedef int V;
-        V ar[] =
-        {
-            1,
-            1,
-            1,
-            2,
-            2,
-            2,
-            3,
-            3,
-            3
-        };
-        M m;
-        m.insert(cpp17_input_iterator<const V*>(ar),
-                 cpp17_input_iterator<const V*>(ar + sizeof(ar)/sizeof(ar[0])));
-        assert(m.size() == 3);
-        assert(*m.begin() == 1);
-        assert(*std::next(m.begin()) == 2);
-        assert(*std::next(m.begin(), 2) == 3);
-    }
+  {
+    typedef std::set<int, std::less<int>, min_allocator<int>> M;
+    typedef int V;
+    V ar[] = {1, 1, 1, 2, 2, 2, 3, 3, 3};
+    M m;
+    m.insert(cpp17_input_iterator<const V*>(ar), cpp17_input_iterator<const V*>(ar + sizeof(ar) / sizeof(ar[0])));
+    assert(m.size() == 3);
+    assert(*m.begin() == 1);
+    assert(*std::next(m.begin()) == 2);
+    assert(*std::next(m.begin(), 2) == 3);
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/associative/set/insert_iter_rv.pass.cpp b/libcxx/test/std/containers/associative/set/insert_iter_rv.pass.cpp
index 5ce5182b868f9..21358f3c40c9c 100644
--- a/libcxx/test/std/containers/associative/set/insert_iter_rv.pass.cpp
+++ b/libcxx/test/std/containers/associative/set/insert_iter_rv.pass.cpp
@@ -22,56 +22,55 @@
 #include "MoveOnly.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef std::set<MoveOnly> M;
-        typedef M::iterator R;
-        M m;
-        R r = m.insert(m.cend(), M::value_type(2));
-        assert(r == m.begin());
-        assert(m.size() == 1);
-        assert(*r == 2);
+int main(int, char**) {
+  {
+    typedef std::set<MoveOnly> M;
+    typedef M::iterator R;
+    M m;
+    R r = m.insert(m.cend(), M::value_type(2));
+    assert(r == m.begin());
+    assert(m.size() == 1);
+    assert(*r == 2);
 
-        r = m.insert(m.cend(), M::value_type(1));
-        assert(r == m.begin());
-        assert(m.size() == 2);
-        assert(*r == 1);
+    r = m.insert(m.cend(), M::value_type(1));
+    assert(r == m.begin());
+    assert(m.size() == 2);
+    assert(*r == 1);
 
-        r = m.insert(m.cend(), M::value_type(3));
-        assert(r == std::prev(m.end()));
-        assert(m.size() == 3);
-        assert(*r == 3);
+    r = m.insert(m.cend(), M::value_type(3));
+    assert(r == std::prev(m.end()));
+    assert(m.size() == 3);
+    assert(*r == 3);
 
-        r = m.insert(m.cend(), M::value_type(3));
-        assert(r == std::prev(m.end()));
-        assert(m.size() == 3);
-        assert(*r == 3);
-    }
-    {
-        typedef std::set<MoveOnly, std::less<MoveOnly>, min_allocator<MoveOnly>> M;
-        typedef M::iterator R;
-        M m;
-        R r = m.insert(m.cend(), M::value_type(2));
-        assert(r == m.begin());
-        assert(m.size() == 1);
-        assert(*r == 2);
+    r = m.insert(m.cend(), M::value_type(3));
+    assert(r == std::prev(m.end()));
+    assert(m.size() == 3);
+    assert(*r == 3);
+  }
+  {
+    typedef std::set<MoveOnly, std::less<MoveOnly>, min_allocator<MoveOnly>> M;
+    typedef M::iterator R;
+    M m;
+    R r = m.insert(m.cend(), M::value_type(2));
+    assert(r == m.begin());
+    assert(m.size() == 1);
+    assert(*r == 2);
 
-        r = m.insert(m.cend(), M::value_type(1));
-        assert(r == m.begin());
-        assert(m.size() == 2);
-        assert(*r == 1);
+    r = m.insert(m.cend(), M::value_type(1));
+    assert(r == m.begin());
+    assert(m.size() == 2);
+    assert(*r == 1);
 
-        r = m.insert(m.cend(), M::value_type(3));
-        assert(r == std::prev(m.end()));
-        assert(m.size() == 3);
-        assert(*r == 3);
+    r = m.insert(m.cend(), M::value_type(3));
+    assert(r == std::prev(m.end()));
+    assert(m.size() == 3);
+    assert(*r == 3);
 
-        r = m.insert(m.cend(), M::value_type(3));
-        assert(r == std::prev(m.end()));
-        assert(m.size() == 3);
-        assert(*r == 3);
-    }
+    r = m.insert(m.cend(), M::value_type(3));
+    assert(r == std::prev(m.end()));
+    assert(m.size() == 3);
+    assert(*r == 3);
+  }
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/associative/set/insert_node_type.pass.cpp b/libcxx/test/std/containers/associative/set/insert_node_type.pass.cpp
index 69caecffef885..9794b4da989d9 100644
--- a/libcxx/test/std/containers/associative/set/insert_node_type.pass.cpp
+++ b/libcxx/test/std/containers/associative/set/insert_node_type.pass.cpp
@@ -42,69 +42,63 @@ void verify_insert_return_type(T&& t) {
 }
 
 template <class Container>
-typename Container::node_type
-node_factory(typename Container::key_type const& key)
-{
-    static Container c;
-    auto p = c.insert(key);
-    assert(p.second);
-    return c.extract(p.first);
+typename Container::node_type node_factory(typename Container::key_type const& key) {
+  static Container c;
+  auto p = c.insert(key);
+  assert(p.second);
+  return c.extract(p.first);
 }
 
 template <class Container>
-void test(Container& c)
-{
-    auto* nf = &node_factory<Container>;
-
-    for (int i = 0; i != 10; ++i)
-    {
-        typename Container::node_type node = nf(i);
-        assert(!node.empty());
-        typename Container::insert_return_type irt = c.insert(std::move(node));
-        assert(node.empty());
-        assert(irt.inserted);
-        assert(irt.node.empty());
-        assert(*irt.position == i);
-        verify_insert_return_type<Container>(irt);
-    }
-
-    assert(c.size() == 10);
-
-    { // Insert empty node.
-        typename Container::node_type def;
-        auto irt = c.insert(std::move(def));
-        assert(def.empty());
-        assert(!irt.inserted);
-        assert(irt.node.empty());
-        assert(irt.position == c.end());
-        verify_insert_return_type<Container>(irt);
-    }
-
-    { // Insert duplicate node.
-        typename Container::node_type dupl = nf(0);
-        auto irt = c.insert(std::move(dupl));
-        assert(dupl.empty());
-        assert(!irt.inserted);
-        assert(!irt.node.empty());
-        assert(irt.position == c.find(0));
-        assert(irt.node.value() == 0);
-        verify_insert_return_type<Container>(irt);
-    }
-
-    assert(c.size() == 10);
-
-    for (int i = 0; i != 10; ++i)
-    {
-        assert(c.count(i) == 1);
-    }
+void test(Container& c) {
+  auto* nf = &node_factory<Container>;
+
+  for (int i = 0; i != 10; ++i) {
+    typename Container::node_type node = nf(i);
+    assert(!node.empty());
+    typename Container::insert_return_type irt = c.insert(std::move(node));
+    assert(node.empty());
+    assert(irt.inserted);
+    assert(irt.node.empty());
+    assert(*irt.position == i);
+    verify_insert_return_type<Container>(irt);
+  }
+
+  assert(c.size() == 10);
+
+  { // Insert empty node.
+    typename Container::node_type def;
+    auto irt = c.insert(std::move(def));
+    assert(def.empty());
+    assert(!irt.inserted);
+    assert(irt.node.empty());
+    assert(irt.position == c.end());
+    verify_insert_return_type<Container>(irt);
+  }
+
+  { // Insert duplicate node.
+    typename Container::node_type dupl = nf(0);
+    auto irt                           = c.insert(std::move(dupl));
+    assert(dupl.empty());
+    assert(!irt.inserted);
+    assert(!irt.node.empty());
+    assert(irt.position == c.find(0));
+    assert(irt.node.value() == 0);
+    verify_insert_return_type<Container>(irt);
+  }
+
+  assert(c.size() == 10);
+
+  for (int i = 0; i != 10; ++i) {
+    assert(c.count(i) == 1);
+  }
 }
 
-int main(int, char**)
-{
-    std::set<int> m;
-    test(m);
-    std::set<int, std::less<int>, min_allocator<int>> m2;
-    test(m2);
+int main(int, char**) {
+  std::set<int> m;
+  test(m);
+  std::set<int, std::less<int>, min_allocator<int>> m2;
+  test(m2);
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/associative/set/insert_node_type_hint.pass.cpp b/libcxx/test/std/containers/associative/set/insert_node_type_hint.pass.cpp
index 6f4cbe97e2ab6..1449be0030fe5 100644
--- a/libcxx/test/std/containers/associative/set/insert_node_type_hint.pass.cpp
+++ b/libcxx/test/std/containers/associative/set/insert_node_type_hint.pass.cpp
@@ -19,45 +19,39 @@
 #include "min_allocator.h"
 
 template <class Container>
-typename Container::node_type
-node_factory(typename Container::key_type const& key)
-{
-    static Container c;
-    auto p = c.insert(key);
-    assert(p.second);
-    return c.extract(p.first);
+typename Container::node_type node_factory(typename Container::key_type const& key) {
+  static Container c;
+  auto p = c.insert(key);
+  assert(p.second);
+  return c.extract(p.first);
 }
 
 template <class Container>
-void test(Container& c)
-{
-    auto* nf = &node_factory<Container>;
-
-    for (int i = 0; i != 10; ++i)
-    {
-        typename Container::node_type node = nf(i);
-        assert(!node.empty());
-        std::size_t prev = c.size();
-        auto it = c.insert(c.end(), std::move(node));
-        assert(node.empty());
-        assert(prev + 1 == c.size());
-        assert(*it == i);
-    }
-
-    assert(c.size() == 10);
-
-    for (int i = 0; i != 10; ++i)
-    {
-        assert(c.count(i) == 1);
-    }
+void test(Container& c) {
+  auto* nf = &node_factory<Container>;
+
+  for (int i = 0; i != 10; ++i) {
+    typename Container::node_type node = nf(i);
+    assert(!node.empty());
+    std::size_t prev = c.size();
+    auto it          = c.insert(c.end(), std::move(node));
+    assert(node.empty());
+    assert(prev + 1 == c.size());
+    assert(*it == i);
+  }
+
+  assert(c.size() == 10);
+
+  for (int i = 0; i != 10; ++i) {
+    assert(c.count(i) == 1);
+  }
 }
 
-int main(int, char**)
-{
-    std::set<int> m;
-    test(m);
-    std::set<int, std::less<int>, min_allocator<int>> m2;
-    test(m2);
+int main(int, char**) {
+  std::set<int> m;
+  test(m);
+  std::set<int, std::less<int>, min_allocator<int>> m2;
+  test(m2);
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/associative/set/insert_rv.pass.cpp b/libcxx/test/std/containers/associative/set/insert_rv.pass.cpp
index 054cdaac899a6..ec18c4379b58d 100644
--- a/libcxx/test/std/containers/associative/set/insert_rv.pass.cpp
+++ b/libcxx/test/std/containers/associative/set/insert_rv.pass.cpp
@@ -21,64 +21,63 @@
 #include "MoveOnly.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef std::set<MoveOnly> M;
-        typedef std::pair<M::iterator, bool> R;
-        M m;
-        R r = m.insert(M::value_type(2));
-        assert(r.second);
-        assert(r.first == m.begin());
-        assert(m.size() == 1);
-        assert(*r.first == 2);
+int main(int, char**) {
+  {
+    typedef std::set<MoveOnly> M;
+    typedef std::pair<M::iterator, bool> R;
+    M m;
+    R r = m.insert(M::value_type(2));
+    assert(r.second);
+    assert(r.first == m.begin());
+    assert(m.size() == 1);
+    assert(*r.first == 2);
 
-        r = m.insert(M::value_type(1));
-        assert(r.second);
-        assert(r.first == m.begin());
-        assert(m.size() == 2);
-        assert(*r.first == 1);
+    r = m.insert(M::value_type(1));
+    assert(r.second);
+    assert(r.first == m.begin());
+    assert(m.size() == 2);
+    assert(*r.first == 1);
 
-        r = m.insert(M::value_type(3));
-        assert(r.second);
-        assert(r.first == std::prev(m.end()));
-        assert(m.size() == 3);
-        assert(*r.first == 3);
+    r = m.insert(M::value_type(3));
+    assert(r.second);
+    assert(r.first == std::prev(m.end()));
+    assert(m.size() == 3);
+    assert(*r.first == 3);
 
-        r = m.insert(M::value_type(3));
-        assert(!r.second);
-        assert(r.first == std::prev(m.end()));
-        assert(m.size() == 3);
-        assert(*r.first == 3);
-    }
-    {
-        typedef std::set<MoveOnly, std::less<MoveOnly>, min_allocator<MoveOnly>> M;
-        typedef std::pair<M::iterator, bool> R;
-        M m;
-        R r = m.insert(M::value_type(2));
-        assert(r.second);
-        assert(r.first == m.begin());
-        assert(m.size() == 1);
-        assert(*r.first == 2);
+    r = m.insert(M::value_type(3));
+    assert(!r.second);
+    assert(r.first == std::prev(m.end()));
+    assert(m.size() == 3);
+    assert(*r.first == 3);
+  }
+  {
+    typedef std::set<MoveOnly, std::less<MoveOnly>, min_allocator<MoveOnly>> M;
+    typedef std::pair<M::iterator, bool> R;
+    M m;
+    R r = m.insert(M::value_type(2));
+    assert(r.second);
+    assert(r.first == m.begin());
+    assert(m.size() == 1);
+    assert(*r.first == 2);
 
-        r = m.insert(M::value_type(1));
-        assert(r.second);
-        assert(r.first == m.begin());
-        assert(m.size() == 2);
-        assert(*r.first == 1);
+    r = m.insert(M::value_type(1));
+    assert(r.second);
+    assert(r.first == m.begin());
+    assert(m.size() == 2);
+    assert(*r.first == 1);
 
-        r = m.insert(M::value_type(3));
-        assert(r.second);
-        assert(r.first == std::prev(m.end()));
-        assert(m.size() == 3);
-        assert(*r.first == 3);
+    r = m.insert(M::value_type(3));
+    assert(r.second);
+    assert(r.first == std::prev(m.end()));
+    assert(m.size() == 3);
+    assert(*r.first == 3);
 
-        r = m.insert(M::value_type(3));
-        assert(!r.second);
-        assert(r.first == std::prev(m.end()));
-        assert(m.size() == 3);
-        assert(*r.first == 3);
-    }
+    r = m.insert(M::value_type(3));
+    assert(!r.second);
+    assert(r.first == std::prev(m.end()));
+    assert(m.size() == 3);
+    assert(*r.first == 3);
+  }
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/associative/set/iterator.pass.cpp b/libcxx/test/std/containers/associative/set/iterator.pass.cpp
index 4ac1187dddc71..1a18cbc426478 100644
--- a/libcxx/test/std/containers/associative/set/iterator.pass.cpp
+++ b/libcxx/test/std/containers/associative/set/iterator.pass.cpp
@@ -32,206 +32,101 @@
 #include "test_macros.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef int V;
-        V ar[] =
-        {
-            1,
-            1,
-            1,
-            2,
-            2,
-            2,
-            3,
-            3,
-            3,
-            4,
-            4,
-            4,
-            5,
-            5,
-            5,
-            6,
-            6,
-            6,
-            7,
-            7,
-            7,
-            8,
-            8,
-            8
-        };
-        std::set<int> m(ar, ar+sizeof(ar)/sizeof(ar[0]));
-        assert(static_cast<std::size_t>(std::distance(m.begin(), m.end())) == m.size());
-        assert(static_cast<std::size_t>(std::distance(m.rbegin(), m.rend())) == m.size());
-        std::set<int>::iterator i;
-        i = m.begin();
-        std::set<int>::const_iterator k = i;
-        assert(i == k);
-        for (int j = 1; j <= static_cast<int>(m.size()); ++j, ++i)
-            assert(*i == j);
-        assert(i == m.end());
-        for (int j = static_cast<int>(m.size()); j >= 1; --j) {
-            --i;
-            assert(*i == j);
-        }
-        assert(i == m.begin());
+int main(int, char**) {
+  {
+    typedef int V;
+    V ar[] = {1, 1, 1, 2, 2, 2, 3, 3, 3, 4, 4, 4, 5, 5, 5, 6, 6, 6, 7, 7, 7, 8, 8, 8};
+    std::set<int> m(ar, ar + sizeof(ar) / sizeof(ar[0]));
+    assert(static_cast<std::size_t>(std::distance(m.begin(), m.end())) == m.size());
+    assert(static_cast<std::size_t>(std::distance(m.rbegin(), m.rend())) == m.size());
+    std::set<int>::iterator i;
+    i                               = m.begin();
+    std::set<int>::const_iterator k = i;
+    assert(i == k);
+    for (int j = 1; j <= static_cast<int>(m.size()); ++j, ++i)
+      assert(*i == j);
+    assert(i == m.end());
+    for (int j = static_cast<int>(m.size()); j >= 1; --j) {
+      --i;
+      assert(*i == j);
     }
-    {
-        typedef int V;
-        V ar[] =
-        {
-            1,
-            1,
-            1,
-            2,
-            2,
-            2,
-            3,
-            3,
-            3,
-            4,
-            4,
-            4,
-            5,
-            5,
-            5,
-            6,
-            6,
-            6,
-            7,
-            7,
-            7,
-            8,
-            8,
-            8
-        };
-        const std::set<int> m(ar, ar+sizeof(ar)/sizeof(ar[0]));
-        assert(static_cast<std::size_t>(std::distance(m.begin(), m.end())) == m.size());
-        assert(static_cast<std::size_t>(std::distance(m.cbegin(), m.cend())) == m.size());
-        assert(static_cast<std::size_t>(std::distance(m.rbegin(), m.rend())) == m.size());
-        assert(static_cast<std::size_t>(std::distance(m.crbegin(), m.crend())) == m.size());
-        std::set<int>::const_iterator i;
-        i = m.begin();
-        for (int j = 1; j <= static_cast<int>(m.size()); ++j, ++i)
-            assert(*i == j);
-        assert(i == m.end());
-        for (int j = static_cast<int>(m.size()); j >= 1; --j) {
-            --i;
-            assert(*i == j);
-        }
-        assert(i == m.begin());
+    assert(i == m.begin());
+  }
+  {
+    typedef int V;
+    V ar[] = {1, 1, 1, 2, 2, 2, 3, 3, 3, 4, 4, 4, 5, 5, 5, 6, 6, 6, 7, 7, 7, 8, 8, 8};
+    const std::set<int> m(ar, ar + sizeof(ar) / sizeof(ar[0]));
+    assert(static_cast<std::size_t>(std::distance(m.begin(), m.end())) == m.size());
+    assert(static_cast<std::size_t>(std::distance(m.cbegin(), m.cend())) == m.size());
+    assert(static_cast<std::size_t>(std::distance(m.rbegin(), m.rend())) == m.size());
+    assert(static_cast<std::size_t>(std::distance(m.crbegin(), m.crend())) == m.size());
+    std::set<int>::const_iterator i;
+    i = m.begin();
+    for (int j = 1; j <= static_cast<int>(m.size()); ++j, ++i)
+      assert(*i == j);
+    assert(i == m.end());
+    for (int j = static_cast<int>(m.size()); j >= 1; --j) {
+      --i;
+      assert(*i == j);
     }
+    assert(i == m.begin());
+  }
 #if TEST_STD_VER >= 11
-    {
-        typedef int V;
-        V ar[] =
-        {
-            1,
-            1,
-            1,
-            2,
-            2,
-            2,
-            3,
-            3,
-            3,
-            4,
-            4,
-            4,
-            5,
-            5,
-            5,
-            6,
-            6,
-            6,
-            7,
-            7,
-            7,
-            8,
-            8,
-            8
-        };
-        std::set<int, std::less<int>, min_allocator<int>> m(ar, ar+sizeof(ar)/sizeof(ar[0]));
-        assert(static_cast<std::size_t>(std::distance(m.begin(), m.end())) == m.size());
-        assert(static_cast<std::size_t>(std::distance(m.rbegin(), m.rend())) == m.size());
-        std::set<int, std::less<int>, min_allocator<int>>::iterator i;
-        i = m.begin();
-        std::set<int, std::less<int>, min_allocator<int>>::const_iterator k = i;
-        assert(i == k);
-        for (int j = 1; j <= static_cast<int>(m.size()); ++j, ++i)
-            assert(*i == j);
-        assert(i == m.end());
-        for (int j = static_cast<int>(m.size()); j >= 1; --j) {
-            --i;
-            assert(*i == j);
-        }
-        assert(i == m.begin());
+  {
+    typedef int V;
+    V ar[] = {1, 1, 1, 2, 2, 2, 3, 3, 3, 4, 4, 4, 5, 5, 5, 6, 6, 6, 7, 7, 7, 8, 8, 8};
+    std::set<int, std::less<int>, min_allocator<int>> m(ar, ar + sizeof(ar) / sizeof(ar[0]));
+    assert(static_cast<std::size_t>(std::distance(m.begin(), m.end())) == m.size());
+    assert(static_cast<std::size_t>(std::distance(m.rbegin(), m.rend())) == m.size());
+    std::set<int, std::less<int>, min_allocator<int>>::iterator i;
+    i                                                                   = m.begin();
+    std::set<int, std::less<int>, min_allocator<int>>::const_iterator k = i;
+    assert(i == k);
+    for (int j = 1; j <= static_cast<int>(m.size()); ++j, ++i)
+      assert(*i == j);
+    assert(i == m.end());
+    for (int j = static_cast<int>(m.size()); j >= 1; --j) {
+      --i;
+      assert(*i == j);
     }
-    {
-        typedef int V;
-        V ar[] =
-        {
-            1,
-            1,
-            1,
-            2,
-            2,
-            2,
-            3,
-            3,
-            3,
-            4,
-            4,
-            4,
-            5,
-            5,
-            5,
-            6,
-            6,
-            6,
-            7,
-            7,
-            7,
-            8,
-            8,
-            8
-        };
-        const std::set<int, std::less<int>, min_allocator<int>> m(ar, ar+sizeof(ar)/sizeof(ar[0]));
-        assert(static_cast<std::size_t>(std::distance(m.begin(), m.end())) == m.size());
-        assert(static_cast<std::size_t>(std::distance(m.cbegin(), m.cend())) == m.size());
-        assert(static_cast<std::size_t>(std::distance(m.rbegin(), m.rend())) == m.size());
-        assert(static_cast<std::size_t>(std::distance(m.crbegin(), m.crend())) == m.size());
-        std::set<int, std::less<int>, min_allocator<int>>::const_iterator i;
-        i = m.begin();
-        for (int j = 1; j <= static_cast<int>(m.size()); ++j, ++i)
-            assert(*i == j);
-        assert(i == m.end());
-        for (int j = static_cast<int>(m.size()); j >= 1; --j) {
-            --i;
-            assert(*i == j);
-        }
-        assert(i == m.begin());
+    assert(i == m.begin());
+  }
+  {
+    typedef int V;
+    V ar[] = {1, 1, 1, 2, 2, 2, 3, 3, 3, 4, 4, 4, 5, 5, 5, 6, 6, 6, 7, 7, 7, 8, 8, 8};
+    const std::set<int, std::less<int>, min_allocator<int>> m(ar, ar + sizeof(ar) / sizeof(ar[0]));
+    assert(static_cast<std::size_t>(std::distance(m.begin(), m.end())) == m.size());
+    assert(static_cast<std::size_t>(std::distance(m.cbegin(), m.cend())) == m.size());
+    assert(static_cast<std::size_t>(std::distance(m.rbegin(), m.rend())) == m.size());
+    assert(static_cast<std::size_t>(std::distance(m.crbegin(), m.crend())) == m.size());
+    std::set<int, std::less<int>, min_allocator<int>>::const_iterator i;
+    i = m.begin();
+    for (int j = 1; j <= static_cast<int>(m.size()); ++j, ++i)
+      assert(*i == j);
+    assert(i == m.end());
+    for (int j = static_cast<int>(m.size()); j >= 1; --j) {
+      --i;
+      assert(*i == j);
     }
+    assert(i == m.begin());
+  }
 #endif
 #if TEST_STD_VER > 11
-    { // N3644 testing
-        typedef std::set<int> C;
-        C::iterator ii1{}, ii2{};
-        C::iterator ii4 = ii1;
-        C::const_iterator cii{};
-        assert ( ii1 == ii2 );
-        assert ( ii1 == ii4 );
+  { // N3644 testing
+    typedef std::set<int> C;
+    C::iterator ii1{}, ii2{};
+    C::iterator ii4 = ii1;
+    C::const_iterator cii{};
+    assert(ii1 == ii2);
+    assert(ii1 == ii4);
 
-        assert (!(ii1 != ii2 ));
+    assert(!(ii1 != ii2));
 
-        assert ( (ii1 == cii ));
-        assert ( (cii == ii1 ));
-        assert (!(ii1 != cii ));
-        assert (!(cii != ii1 ));
-    }
+    assert((ii1 == cii));
+    assert((cii == ii1));
+    assert(!(ii1 != cii));
+    assert(!(cii != ii1));
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/associative/set/iterator_concept_conformance.compile.pass.cpp b/libcxx/test/std/containers/associative/set/iterator_concept_conformance.compile.pass.cpp
index 049acdfed9ccd..ac3737cf5cc94 100644
--- a/libcxx/test/std/containers/associative/set/iterator_concept_conformance.compile.pass.cpp
+++ b/libcxx/test/std/containers/associative/set/iterator_concept_conformance.compile.pass.cpp
@@ -14,11 +14,11 @@
 
 #include <iterator>
 
-using iterator = std::set<int>::iterator;
-using const_iterator = std::set<int>::const_iterator;
-using reverse_iterator = std::set<int>::reverse_iterator;
+using iterator               = std::set<int>::iterator;
+using const_iterator         = std::set<int>::const_iterator;
+using reverse_iterator       = std::set<int>::reverse_iterator;
 using const_reverse_iterator = std::set<int>::const_reverse_iterator;
-using value_type = int;
+using value_type             = int;
 
 static_assert(std::bidirectional_iterator<iterator>);
 static_assert(!std::random_access_iterator<iterator>);
diff --git a/libcxx/test/std/containers/associative/set/lower_bound.pass.cpp b/libcxx/test/std/containers/associative/set/lower_bound.pass.cpp
index 3879504b7ffa5..2f114685fb84d 100644
--- a/libcxx/test/std/containers/associative/set/lower_bound.pass.cpp
+++ b/libcxx/test/std/containers/associative/set/lower_bound.pass.cpp
@@ -20,232 +20,181 @@
 #include "min_allocator.h"
 #include "private_constructor.h"
 
-int main(int, char**)
-{
-    {
+int main(int, char**) {
+  {
     typedef int V;
     typedef std::set<int> M;
     {
-        typedef M::iterator R;
-        V ar[] =
-        {
-            5,
-            7,
-            9,
-            11,
-            13,
-            15,
-            17,
-            19
-        };
-        M m(ar, ar+sizeof(ar)/sizeof(ar[0]));
-        R r = m.lower_bound(5);
-        assert(r == m.begin());
-        r = m.lower_bound(7);
-        assert(r == std::next(m.begin()));
-        r = m.lower_bound(9);
-        assert(r == std::next(m.begin(), 2));
-        r = m.lower_bound(11);
-        assert(r == std::next(m.begin(), 3));
-        r = m.lower_bound(13);
-        assert(r == std::next(m.begin(), 4));
-        r = m.lower_bound(15);
-        assert(r == std::next(m.begin(), 5));
-        r = m.lower_bound(17);
-        assert(r == std::next(m.begin(), 6));
-        r = m.lower_bound(19);
-        assert(r == std::next(m.begin(), 7));
-        r = m.lower_bound(4);
-        assert(r == std::next(m.begin(), 0));
-        r = m.lower_bound(6);
-        assert(r == std::next(m.begin(), 1));
-        r = m.lower_bound(8);
-        assert(r == std::next(m.begin(), 2));
-        r = m.lower_bound(10);
-        assert(r == std::next(m.begin(), 3));
-        r = m.lower_bound(12);
-        assert(r == std::next(m.begin(), 4));
-        r = m.lower_bound(14);
-        assert(r == std::next(m.begin(), 5));
-        r = m.lower_bound(16);
-        assert(r == std::next(m.begin(), 6));
-        r = m.lower_bound(18);
-        assert(r == std::next(m.begin(), 7));
-        r = m.lower_bound(20);
-        assert(r == std::next(m.begin(), 8));
+      typedef M::iterator R;
+      V ar[] = {5, 7, 9, 11, 13, 15, 17, 19};
+      M m(ar, ar + sizeof(ar) / sizeof(ar[0]));
+      R r = m.lower_bound(5);
+      assert(r == m.begin());
+      r = m.lower_bound(7);
+      assert(r == std::next(m.begin()));
+      r = m.lower_bound(9);
+      assert(r == std::next(m.begin(), 2));
+      r = m.lower_bound(11);
+      assert(r == std::next(m.begin(), 3));
+      r = m.lower_bound(13);
+      assert(r == std::next(m.begin(), 4));
+      r = m.lower_bound(15);
+      assert(r == std::next(m.begin(), 5));
+      r = m.lower_bound(17);
+      assert(r == std::next(m.begin(), 6));
+      r = m.lower_bound(19);
+      assert(r == std::next(m.begin(), 7));
+      r = m.lower_bound(4);
+      assert(r == std::next(m.begin(), 0));
+      r = m.lower_bound(6);
+      assert(r == std::next(m.begin(), 1));
+      r = m.lower_bound(8);
+      assert(r == std::next(m.begin(), 2));
+      r = m.lower_bound(10);
+      assert(r == std::next(m.begin(), 3));
+      r = m.lower_bound(12);
+      assert(r == std::next(m.begin(), 4));
+      r = m.lower_bound(14);
+      assert(r == std::next(m.begin(), 5));
+      r = m.lower_bound(16);
+      assert(r == std::next(m.begin(), 6));
+      r = m.lower_bound(18);
+      assert(r == std::next(m.begin(), 7));
+      r = m.lower_bound(20);
+      assert(r == std::next(m.begin(), 8));
     }
     {
-        typedef M::const_iterator R;
-        V ar[] =
-        {
-            5,
-            7,
-            9,
-            11,
-            13,
-            15,
-            17,
-            19
-        };
-        const M m(ar, ar+sizeof(ar)/sizeof(ar[0]));
-        R r = m.lower_bound(5);
-        assert(r == m.begin());
-        r = m.lower_bound(7);
-        assert(r == std::next(m.begin()));
-        r = m.lower_bound(9);
-        assert(r == std::next(m.begin(), 2));
-        r = m.lower_bound(11);
-        assert(r == std::next(m.begin(), 3));
-        r = m.lower_bound(13);
-        assert(r == std::next(m.begin(), 4));
-        r = m.lower_bound(15);
-        assert(r == std::next(m.begin(), 5));
-        r = m.lower_bound(17);
-        assert(r == std::next(m.begin(), 6));
-        r = m.lower_bound(19);
-        assert(r == std::next(m.begin(), 7));
-        r = m.lower_bound(4);
-        assert(r == std::next(m.begin(), 0));
-        r = m.lower_bound(6);
-        assert(r == std::next(m.begin(), 1));
-        r = m.lower_bound(8);
-        assert(r == std::next(m.begin(), 2));
-        r = m.lower_bound(10);
-        assert(r == std::next(m.begin(), 3));
-        r = m.lower_bound(12);
-        assert(r == std::next(m.begin(), 4));
-        r = m.lower_bound(14);
-        assert(r == std::next(m.begin(), 5));
-        r = m.lower_bound(16);
-        assert(r == std::next(m.begin(), 6));
-        r = m.lower_bound(18);
-        assert(r == std::next(m.begin(), 7));
-        r = m.lower_bound(20);
-        assert(r == std::next(m.begin(), 8));
-    }
+      typedef M::const_iterator R;
+      V ar[] = {5, 7, 9, 11, 13, 15, 17, 19};
+      const M m(ar, ar + sizeof(ar) / sizeof(ar[0]));
+      R r = m.lower_bound(5);
+      assert(r == m.begin());
+      r = m.lower_bound(7);
+      assert(r == std::next(m.begin()));
+      r = m.lower_bound(9);
+      assert(r == std::next(m.begin(), 2));
+      r = m.lower_bound(11);
+      assert(r == std::next(m.begin(), 3));
+      r = m.lower_bound(13);
+      assert(r == std::next(m.begin(), 4));
+      r = m.lower_bound(15);
+      assert(r == std::next(m.begin(), 5));
+      r = m.lower_bound(17);
+      assert(r == std::next(m.begin(), 6));
+      r = m.lower_bound(19);
+      assert(r == std::next(m.begin(), 7));
+      r = m.lower_bound(4);
+      assert(r == std::next(m.begin(), 0));
+      r = m.lower_bound(6);
+      assert(r == std::next(m.begin(), 1));
+      r = m.lower_bound(8);
+      assert(r == std::next(m.begin(), 2));
+      r = m.lower_bound(10);
+      assert(r == std::next(m.begin(), 3));
+      r = m.lower_bound(12);
+      assert(r == std::next(m.begin(), 4));
+      r = m.lower_bound(14);
+      assert(r == std::next(m.begin(), 5));
+      r = m.lower_bound(16);
+      assert(r == std::next(m.begin(), 6));
+      r = m.lower_bound(18);
+      assert(r == std::next(m.begin(), 7));
+      r = m.lower_bound(20);
+      assert(r == std::next(m.begin(), 8));
     }
+  }
 #if TEST_STD_VER >= 11
-    {
+  {
     typedef int V;
     typedef std::set<int, std::less<int>, min_allocator<int>> M;
     {
-        typedef M::iterator R;
-        V ar[] =
-        {
-            5,
-            7,
-            9,
-            11,
-            13,
-            15,
-            17,
-            19
-        };
-        M m(ar, ar+sizeof(ar)/sizeof(ar[0]));
-        R r = m.lower_bound(5);
-        assert(r == m.begin());
-        r = m.lower_bound(7);
-        assert(r == std::next(m.begin()));
-        r = m.lower_bound(9);
-        assert(r == std::next(m.begin(), 2));
-        r = m.lower_bound(11);
-        assert(r == std::next(m.begin(), 3));
-        r = m.lower_bound(13);
-        assert(r == std::next(m.begin(), 4));
-        r = m.lower_bound(15);
-        assert(r == std::next(m.begin(), 5));
-        r = m.lower_bound(17);
-        assert(r == std::next(m.begin(), 6));
-        r = m.lower_bound(19);
-        assert(r == std::next(m.begin(), 7));
-        r = m.lower_bound(4);
-        assert(r == std::next(m.begin(), 0));
-        r = m.lower_bound(6);
-        assert(r == std::next(m.begin(), 1));
-        r = m.lower_bound(8);
-        assert(r == std::next(m.begin(), 2));
-        r = m.lower_bound(10);
-        assert(r == std::next(m.begin(), 3));
-        r = m.lower_bound(12);
-        assert(r == std::next(m.begin(), 4));
-        r = m.lower_bound(14);
-        assert(r == std::next(m.begin(), 5));
-        r = m.lower_bound(16);
-        assert(r == std::next(m.begin(), 6));
-        r = m.lower_bound(18);
-        assert(r == std::next(m.begin(), 7));
-        r = m.lower_bound(20);
-        assert(r == std::next(m.begin(), 8));
+      typedef M::iterator R;
+      V ar[] = {5, 7, 9, 11, 13, 15, 17, 19};
+      M m(ar, ar + sizeof(ar) / sizeof(ar[0]));
+      R r = m.lower_bound(5);
+      assert(r == m.begin());
+      r = m.lower_bound(7);
+      assert(r == std::next(m.begin()));
+      r = m.lower_bound(9);
+      assert(r == std::next(m.begin(), 2));
+      r = m.lower_bound(11);
+      assert(r == std::next(m.begin(), 3));
+      r = m.lower_bound(13);
+      assert(r == std::next(m.begin(), 4));
+      r = m.lower_bound(15);
+      assert(r == std::next(m.begin(), 5));
+      r = m.lower_bound(17);
+      assert(r == std::next(m.begin(), 6));
+      r = m.lower_bound(19);
+      assert(r == std::next(m.begin(), 7));
+      r = m.lower_bound(4);
+      assert(r == std::next(m.begin(), 0));
+      r = m.lower_bound(6);
+      assert(r == std::next(m.begin(), 1));
+      r = m.lower_bound(8);
+      assert(r == std::next(m.begin(), 2));
+      r = m.lower_bound(10);
+      assert(r == std::next(m.begin(), 3));
+      r = m.lower_bound(12);
+      assert(r == std::next(m.begin(), 4));
+      r = m.lower_bound(14);
+      assert(r == std::next(m.begin(), 5));
+      r = m.lower_bound(16);
+      assert(r == std::next(m.begin(), 6));
+      r = m.lower_bound(18);
+      assert(r == std::next(m.begin(), 7));
+      r = m.lower_bound(20);
+      assert(r == std::next(m.begin(), 8));
     }
     {
-        typedef M::const_iterator R;
-        V ar[] =
-        {
-            5,
-            7,
-            9,
-            11,
-            13,
-            15,
-            17,
-            19
-        };
-        const M m(ar, ar+sizeof(ar)/sizeof(ar[0]));
-        R r = m.lower_bound(5);
-        assert(r == m.begin());
-        r = m.lower_bound(7);
-        assert(r == std::next(m.begin()));
-        r = m.lower_bound(9);
-        assert(r == std::next(m.begin(), 2));
-        r = m.lower_bound(11);
-        assert(r == std::next(m.begin(), 3));
-        r = m.lower_bound(13);
-        assert(r == std::next(m.begin(), 4));
-        r = m.lower_bound(15);
-        assert(r == std::next(m.begin(), 5));
-        r = m.lower_bound(17);
-        assert(r == std::next(m.begin(), 6));
-        r = m.lower_bound(19);
-        assert(r == std::next(m.begin(), 7));
-        r = m.lower_bound(4);
-        assert(r == std::next(m.begin(), 0));
-        r = m.lower_bound(6);
-        assert(r == std::next(m.begin(), 1));
-        r = m.lower_bound(8);
-        assert(r == std::next(m.begin(), 2));
-        r = m.lower_bound(10);
-        assert(r == std::next(m.begin(), 3));
-        r = m.lower_bound(12);
-        assert(r == std::next(m.begin(), 4));
-        r = m.lower_bound(14);
-        assert(r == std::next(m.begin(), 5));
-        r = m.lower_bound(16);
-        assert(r == std::next(m.begin(), 6));
-        r = m.lower_bound(18);
-        assert(r == std::next(m.begin(), 7));
-        r = m.lower_bound(20);
-        assert(r == std::next(m.begin(), 8));
-    }
+      typedef M::const_iterator R;
+      V ar[] = {5, 7, 9, 11, 13, 15, 17, 19};
+      const M m(ar, ar + sizeof(ar) / sizeof(ar[0]));
+      R r = m.lower_bound(5);
+      assert(r == m.begin());
+      r = m.lower_bound(7);
+      assert(r == std::next(m.begin()));
+      r = m.lower_bound(9);
+      assert(r == std::next(m.begin(), 2));
+      r = m.lower_bound(11);
+      assert(r == std::next(m.begin(), 3));
+      r = m.lower_bound(13);
+      assert(r == std::next(m.begin(), 4));
+      r = m.lower_bound(15);
+      assert(r == std::next(m.begin(), 5));
+      r = m.lower_bound(17);
+      assert(r == std::next(m.begin(), 6));
+      r = m.lower_bound(19);
+      assert(r == std::next(m.begin(), 7));
+      r = m.lower_bound(4);
+      assert(r == std::next(m.begin(), 0));
+      r = m.lower_bound(6);
+      assert(r == std::next(m.begin(), 1));
+      r = m.lower_bound(8);
+      assert(r == std::next(m.begin(), 2));
+      r = m.lower_bound(10);
+      assert(r == std::next(m.begin(), 3));
+      r = m.lower_bound(12);
+      assert(r == std::next(m.begin(), 4));
+      r = m.lower_bound(14);
+      assert(r == std::next(m.begin(), 5));
+      r = m.lower_bound(16);
+      assert(r == std::next(m.begin(), 6));
+      r = m.lower_bound(18);
+      assert(r == std::next(m.begin(), 7));
+      r = m.lower_bound(20);
+      assert(r == std::next(m.begin(), 8));
     }
+  }
 #endif
 #if TEST_STD_VER > 11
-    {
+  {
     typedef int V;
     typedef std::set<V, std::less<>> M;
     typedef M::iterator R;
 
-    V ar[] =
-    {
-        5,
-        7,
-        9,
-        11,
-        13,
-        15,
-        17,
-        19
-    };
-    M m(ar, ar+sizeof(ar)/sizeof(ar[0]));
+    V ar[] = {5, 7, 9, 11, 13, 15, 17, 19};
+    M m(ar, ar + sizeof(ar) / sizeof(ar[0]));
     R r = m.lower_bound(5);
     assert(r == m.begin());
     r = m.lower_bound(7);
@@ -280,22 +229,22 @@ int main(int, char**)
     assert(r == std::next(m.begin(), 7));
     r = m.lower_bound(20);
     assert(r == std::next(m.begin(), 8));
-    }
+  }
 
-    {
+  {
     typedef PrivateConstructor V;
     typedef std::set<V, std::less<>> M;
     typedef M::iterator R;
 
     M m;
-    m.insert ( V::make ( 5 ));
-    m.insert ( V::make ( 7 ));
-    m.insert ( V::make ( 9 ));
-    m.insert ( V::make ( 11 ));
-    m.insert ( V::make ( 13 ));
-    m.insert ( V::make ( 15 ));
-    m.insert ( V::make ( 17 ));
-    m.insert ( V::make ( 19 ));
+    m.insert(V::make(5));
+    m.insert(V::make(7));
+    m.insert(V::make(9));
+    m.insert(V::make(11));
+    m.insert(V::make(13));
+    m.insert(V::make(15));
+    m.insert(V::make(17));
+    m.insert(V::make(19));
 
     R r = m.lower_bound(5);
     assert(r == m.begin());
@@ -331,9 +280,8 @@ int main(int, char**)
     assert(r == std::next(m.begin(), 7));
     r = m.lower_bound(20);
     assert(r == std::next(m.begin(), 8));
-    }
+  }
 #endif
 
-
   return 0;
 }
diff --git a/libcxx/test/std/containers/associative/set/max_size.pass.cpp b/libcxx/test/std/containers/associative/set/max_size.pass.cpp
index f921378df21d1..b00d316c47b35 100644
--- a/libcxx/test/std/containers/associative/set/max_size.pass.cpp
+++ b/libcxx/test/std/containers/associative/set/max_size.pass.cpp
@@ -20,32 +20,29 @@
 #include "test_allocator.h"
 #include "test_macros.h"
 
-int main(int, char**)
-{
-    {
-      typedef limited_allocator<int, 10> A;
-      typedef std::set<int, std::less<int>, A> C;
-      C c;
-      assert(c.max_size() <= 10);
-      LIBCPP_ASSERT(c.max_size() == 10);
-    }
-    {
-      typedef limited_allocator<int, (std::size_t)-1> A;
-      typedef std::set<int, std::less<int>, A> C;
-      const C::size_type max_dist =
-          static_cast<C::size_type>(std::numeric_limits<C::difference_type>::max());
-      C c;
-      assert(c.max_size() <= max_dist);
-      LIBCPP_ASSERT(c.max_size() == max_dist);
-    }
-    {
-      typedef std::set<char> C;
-      const C::size_type max_dist =
-          static_cast<C::size_type>(std::numeric_limits<C::difference_type>::max());
-      C c;
-      assert(c.max_size() <= max_dist);
-      assert(c.max_size() <= alloc_max_size(c.get_allocator()));
-    }
+int main(int, char**) {
+  {
+    typedef limited_allocator<int, 10> A;
+    typedef std::set<int, std::less<int>, A> C;
+    C c;
+    assert(c.max_size() <= 10);
+    LIBCPP_ASSERT(c.max_size() == 10);
+  }
+  {
+    typedef limited_allocator<int, (std::size_t)-1> A;
+    typedef std::set<int, std::less<int>, A> C;
+    const C::size_type max_dist = static_cast<C::size_type>(std::numeric_limits<C::difference_type>::max());
+    C c;
+    assert(c.max_size() <= max_dist);
+    LIBCPP_ASSERT(c.max_size() == max_dist);
+  }
+  {
+    typedef std::set<char> C;
+    const C::size_type max_dist = static_cast<C::size_type>(std::numeric_limits<C::difference_type>::max());
+    C c;
+    assert(c.max_size() <= max_dist);
+    assert(c.max_size() <= alloc_max_size(c.get_allocator()));
+  }
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/associative/set/merge.pass.cpp b/libcxx/test/std/containers/associative/set/merge.pass.cpp
index d73487495f1f1..50141d5444b1f 100644
--- a/libcxx/test/std/containers/associative/set/merge.pass.cpp
+++ b/libcxx/test/std/containers/associative/set/merge.pass.cpp
@@ -27,123 +27,112 @@
 #include "Counter.h"
 
 template <class Set>
-bool set_equal(const Set& set, Set other)
-{
-    return set == other;
+bool set_equal(const Set& set, Set other) {
+  return set == other;
 }
 
 #ifndef TEST_HAS_NO_EXCEPTIONS
-struct throw_comparator
-{
-    bool& should_throw_;
+struct throw_comparator {
+  bool& should_throw_;
 
-    throw_comparator(bool& should_throw) : should_throw_(should_throw) {}
+  throw_comparator(bool& should_throw) : should_throw_(should_throw) {}
 
-    template <class T>
-    bool operator()(const T& lhs, const T& rhs) const
-    {
-        if (should_throw_)
-            throw 0;
-        return lhs < rhs;
-    }
+  template <class T>
+  bool operator()(const T& lhs, const T& rhs) const {
+    if (should_throw_)
+      throw 0;
+    return lhs < rhs;
+  }
 };
 #endif
 
-int main(int, char**)
-{
-    {
-        std::set<int> src{1, 3, 5};
-        std::set<int> dst{2, 4, 5};
-        dst.merge(src);
-        assert(set_equal(src, {5}));
-        assert(set_equal(dst, {1, 2, 3, 4, 5}));
-    }
+int main(int, char**) {
+  {
+    std::set<int> src{1, 3, 5};
+    std::set<int> dst{2, 4, 5};
+    dst.merge(src);
+    assert(set_equal(src, {5}));
+    assert(set_equal(dst, {1, 2, 3, 4, 5}));
+  }
 
 #ifndef TEST_HAS_NO_EXCEPTIONS
-    {
-        bool do_throw = false;
-        typedef std::set<Counter<int>, throw_comparator> set_type;
-        set_type src({1, 3, 5}, throw_comparator(do_throw));
-        set_type dst({2, 4, 5}, throw_comparator(do_throw));
-
-        assert(Counter_base::gConstructed == 6);
-
-        do_throw = true;
-        try
-        {
-            dst.merge(src);
-        }
-        catch (int)
-        {
-            do_throw = false;
-        }
-        assert(!do_throw);
-        assert(set_equal(src, set_type({1, 3, 5}, throw_comparator(do_throw))));
-        assert(set_equal(dst, set_type({2, 4, 5}, throw_comparator(do_throw))));
+  {
+    bool do_throw = false;
+    typedef std::set<Counter<int>, throw_comparator> set_type;
+    set_type src({1, 3, 5}, throw_comparator(do_throw));
+    set_type dst({2, 4, 5}, throw_comparator(do_throw));
+
+    assert(Counter_base::gConstructed == 6);
+
+    do_throw = true;
+    try {
+      dst.merge(src);
+    } catch (int) {
+      do_throw = false;
     }
+    assert(!do_throw);
+    assert(set_equal(src, set_type({1, 3, 5}, throw_comparator(do_throw))));
+    assert(set_equal(dst, set_type({2, 4, 5}, throw_comparator(do_throw))));
+  }
 #endif
-    assert(Counter_base::gConstructed == 0);
-    struct comparator
-    {
-        comparator() = default;
+  assert(Counter_base::gConstructed == 0);
+  struct comparator {
+    comparator() = default;
 
-        bool operator()(const Counter<int>& lhs, const Counter<int>& rhs) const
-        {
-            return lhs < rhs;
-        }
-    };
-    {
-        typedef std::set<Counter<int>, std::less<Counter<int>>> first_set_type;
-        typedef std::set<Counter<int>, comparator> second_set_type;
-        typedef std::multiset<Counter<int>, comparator> third_set_type;
+    bool operator()(const Counter<int>& lhs, const Counter<int>& rhs) const { return lhs < rhs; }
+  };
+  {
+    typedef std::set<Counter<int>, std::less<Counter<int>>> first_set_type;
+    typedef std::set<Counter<int>, comparator> second_set_type;
+    typedef std::multiset<Counter<int>, comparator> third_set_type;
 
-        {
-            first_set_type first{1, 2, 3};
-            second_set_type second{2, 3, 4};
-            third_set_type third{1, 3};
+    {
+      first_set_type first{1, 2, 3};
+      second_set_type second{2, 3, 4};
+      third_set_type third{1, 3};
 
-            assert(Counter_base::gConstructed == 8);
+      assert(Counter_base::gConstructed == 8);
 
-            first.merge(second);
-            first.merge(third);
+      first.merge(second);
+      first.merge(third);
 
-            assert(set_equal(first, {1, 2, 3, 4}));
-            assert(set_equal(second, {2, 3}));
-            assert(set_equal(third, {1, 3}));
+      assert(set_equal(first, {1, 2, 3, 4}));
+      assert(set_equal(second, {2, 3}));
+      assert(set_equal(third, {1, 3}));
 
-            assert(Counter_base::gConstructed == 8);
-        }
-        assert(Counter_base::gConstructed == 0);
-        {
-            first_set_type first{1, 2, 3};
-            second_set_type second{2, 3, 4};
-            third_set_type third{1, 3};
+      assert(Counter_base::gConstructed == 8);
+    }
+    assert(Counter_base::gConstructed == 0);
+    {
+      first_set_type first{1, 2, 3};
+      second_set_type second{2, 3, 4};
+      third_set_type third{1, 3};
 
-            assert(Counter_base::gConstructed == 8);
+      assert(Counter_base::gConstructed == 8);
 
-            first.merge(std::move(second));
-            first.merge(std::move(third));
+      first.merge(std::move(second));
+      first.merge(std::move(third));
 
-            assert(set_equal(first, {1, 2, 3, 4}));
-            assert(set_equal(second, {2, 3}));
-            assert(set_equal(third, {1, 3}));
+      assert(set_equal(first, {1, 2, 3, 4}));
+      assert(set_equal(second, {2, 3}));
+      assert(set_equal(third, {1, 3}));
 
-            assert(Counter_base::gConstructed == 8);
-        }
-        assert(Counter_base::gConstructed == 0);
+      assert(Counter_base::gConstructed == 8);
+    }
+    assert(Counter_base::gConstructed == 0);
+  }
+  {
+    std::set<int> first;
+    {
+      std::set<int> second;
+      first.merge(second);
+      first.merge(std::move(second));
     }
     {
-        std::set<int> first;
-        {
-            std::set<int> second;
-            first.merge(second);
-            first.merge(std::move(second));
-        }
-        {
-            std::multiset<int> second;
-            first.merge(second);
-            first.merge(std::move(second));
-        }
+      std::multiset<int> second;
+      first.merge(second);
+      first.merge(std::move(second));
     }
-    return 0;
+  }
+  return 0;
 }
diff --git a/libcxx/test/std/containers/associative/set/range_concept_conformance.compile.pass.cpp b/libcxx/test/std/containers/associative/set/range_concept_conformance.compile.pass.cpp
index 16f4de96c776c..af5217b750750 100644
--- a/libcxx/test/std/containers/associative/set/range_concept_conformance.compile.pass.cpp
+++ b/libcxx/test/std/containers/associative/set/range_concept_conformance.compile.pass.cpp
@@ -17,7 +17,6 @@
 
 using range = std::set<int>;
 
-
 static_assert(std::same_as<std::ranges::iterator_t<range>, range::iterator>);
 static_assert(std::ranges::bidirectional_range<range>);
 static_assert(!std::ranges::random_access_range<range>);
diff --git a/libcxx/test/std/containers/associative/set/set.cons/alloc.pass.cpp b/libcxx/test/std/containers/associative/set/set.cons/alloc.pass.cpp
index 751a5e9ab1f0d..4321666044a3b 100644
--- a/libcxx/test/std/containers/associative/set/set.cons/alloc.pass.cpp
+++ b/libcxx/test/std/containers/associative/set/set.cons/alloc.pass.cpp
@@ -18,14 +18,13 @@
 #include "test_macros.h"
 #include "test_allocator.h"
 
-int main(int, char**)
-{
-    typedef std::less<int> C;
-    typedef test_allocator<int> A;
-    std::set<int, C, A> m(A(5));
-    assert(m.empty());
-    assert(m.begin() == m.end());
-    assert(m.get_allocator() == A(5));
+int main(int, char**) {
+  typedef std::less<int> C;
+  typedef test_allocator<int> A;
+  std::set<int, C, A> m(A(5));
+  assert(m.empty());
+  assert(m.begin() == m.end());
+  assert(m.get_allocator() == A(5));
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/associative/set/set.cons/assign_initializer_list.pass.cpp b/libcxx/test/std/containers/associative/set/set.cons/assign_initializer_list.pass.cpp
index 8b391ad079abf..d674669fd2559 100644
--- a/libcxx/test/std/containers/associative/set/set.cons/assign_initializer_list.pass.cpp
+++ b/libcxx/test/std/containers/associative/set/set.cons/assign_initializer_list.pass.cpp
@@ -26,7 +26,7 @@ void basic_test() {
     typedef std::set<int> C;
     typedef C::value_type V;
     C m = {10, 8};
-    m = {1, 2, 3, 4, 5, 6};
+    m   = {1, 2, 3, 4, 5, 6};
     assert(m.size() == 6);
     assert(std::distance(m.begin(), m.end()) == 6);
     C::const_iterator i = m.cbegin();
@@ -41,7 +41,7 @@ void basic_test() {
     typedef std::set<int, std::less<int>, min_allocator<int> > C;
     typedef C::value_type V;
     C m = {10, 8};
-    m = {1, 2, 3, 4, 5, 6};
+    m   = {1, 2, 3, 4, 5, 6};
     assert(m.size() == 6);
     assert(std::distance(m.begin(), m.end()) == 6);
     C::const_iterator i = m.cbegin();
diff --git a/libcxx/test/std/containers/associative/set/set.cons/compare.pass.cpp b/libcxx/test/std/containers/associative/set/set.cons/compare.pass.cpp
index 4cd3f3067c841..03452142f5b85 100644
--- a/libcxx/test/std/containers/associative/set/set.cons/compare.pass.cpp
+++ b/libcxx/test/std/containers/associative/set/set.cons/compare.pass.cpp
@@ -18,14 +18,13 @@
 #include "test_macros.h"
 #include "../../../test_compare.h"
 
-int main(int, char**)
-{
-    typedef test_less<int> C;
-    const std::set<int, C> m(C(3));
-    assert(m.empty());
-    assert(m.begin() == m.end());
-    assert(m.key_comp() == C(3));
-    assert(m.value_comp() == C(3));
+int main(int, char**) {
+  typedef test_less<int> C;
+  const std::set<int, C> m(C(3));
+  assert(m.empty());
+  assert(m.begin() == m.end());
+  assert(m.key_comp() == C(3));
+  assert(m.value_comp() == C(3));
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/associative/set/set.cons/compare_alloc.pass.cpp b/libcxx/test/std/containers/associative/set/set.cons/compare_alloc.pass.cpp
index dfd89a16bcd07..2ae0547054537 100644
--- a/libcxx/test/std/containers/associative/set/set.cons/compare_alloc.pass.cpp
+++ b/libcxx/test/std/containers/associative/set/set.cons/compare_alloc.pass.cpp
@@ -19,15 +19,14 @@
 #include "../../../test_compare.h"
 #include "test_allocator.h"
 
-int main(int, char**)
-{
-    typedef test_less<int> C;
-    typedef test_allocator<int> A;
-    std::set<int, C, A> m(C(4), A(5));
-    assert(m.empty());
-    assert(m.begin() == m.end());
-    assert(m.key_comp() == C(4));
-    assert(m.get_allocator() == A(5));
+int main(int, char**) {
+  typedef test_less<int> C;
+  typedef test_allocator<int> A;
+  std::set<int, C, A> m(C(4), A(5));
+  assert(m.empty());
+  assert(m.begin() == m.end());
+  assert(m.key_comp() == C(4));
+  assert(m.get_allocator() == A(5));
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/associative/set/set.cons/compare_copy_constructible.compile.fail.cpp b/libcxx/test/std/containers/associative/set/set.cons/compare_copy_constructible.compile.fail.cpp
index 58f678dd1a415..f9ea45ca1756a 100644
--- a/libcxx/test/std/containers/associative/set/set.cons/compare_copy_constructible.compile.fail.cpp
+++ b/libcxx/test/std/containers/associative/set/set.cons/compare_copy_constructible.compile.fail.cpp
@@ -15,16 +15,16 @@
 
 template <class T>
 struct Comp {
-    bool operator () (const T& lhs, const T& rhs) const { return lhs < rhs; }
+  bool operator()(const T& lhs, const T& rhs) const { return lhs < rhs; }
 
-    Comp () {}
-private:
-    Comp (const Comp &); // declared but not defined
-    };
+  Comp() {}
 
+private:
+  Comp(const Comp&); // declared but not defined
+};
 
 int main(int, char**) {
-    std::set<int, Comp<int> > m;
+  std::set<int, Comp<int> > m;
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/associative/set/set.cons/copy.pass.cpp b/libcxx/test/std/containers/associative/set/set.cons/copy.pass.cpp
index 1d27cfbd3e722..01a1b6ab59af0 100644
--- a/libcxx/test/std/containers/associative/set/set.cons/copy.pass.cpp
+++ b/libcxx/test/std/containers/associative/set/set.cons/copy.pass.cpp
@@ -20,77 +20,54 @@
 #include "../../../test_compare.h"
 #include "test_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef int V;
-        V ar[] =
-        {
-            1,
-            1,
-            1,
-            2,
-            2,
-            2,
-            3,
-            3,
-            3
-        };
-        typedef test_less<int> C;
-        typedef test_allocator<V> A;
-        std::set<int, C, A> mo(ar, ar+sizeof(ar)/sizeof(ar[0]), C(5), A(7));
-        std::set<int, C, A> m = mo;
-        assert(m.get_allocator() == A(7));
-        assert(m.key_comp() == C(5));
-        assert(m.size() == 3);
-        assert(std::distance(m.begin(), m.end()) == 3);
-        assert(*m.begin() == 1);
-        assert(*std::next(m.begin()) == 2);
-        assert(*std::next(m.begin(), 2) == 3);
+int main(int, char**) {
+  {
+    typedef int V;
+    V ar[] = {1, 1, 1, 2, 2, 2, 3, 3, 3};
+    typedef test_less<int> C;
+    typedef test_allocator<V> A;
+    std::set<int, C, A> mo(ar, ar + sizeof(ar) / sizeof(ar[0]), C(5), A(7));
+    std::set<int, C, A> m = mo;
+    assert(m.get_allocator() == A(7));
+    assert(m.key_comp() == C(5));
+    assert(m.size() == 3);
+    assert(std::distance(m.begin(), m.end()) == 3);
+    assert(*m.begin() == 1);
+    assert(*std::next(m.begin()) == 2);
+    assert(*std::next(m.begin(), 2) == 3);
 
-        assert(mo.get_allocator() == A(7));
-        assert(mo.key_comp() == C(5));
-        assert(mo.size() == 3);
-        assert(std::distance(mo.begin(), mo.end()) == 3);
-        assert(*mo.begin() == 1);
-        assert(*std::next(mo.begin()) == 2);
-        assert(*std::next(mo.begin(), 2) == 3);
-    }
+    assert(mo.get_allocator() == A(7));
+    assert(mo.key_comp() == C(5));
+    assert(mo.size() == 3);
+    assert(std::distance(mo.begin(), mo.end()) == 3);
+    assert(*mo.begin() == 1);
+    assert(*std::next(mo.begin()) == 2);
+    assert(*std::next(mo.begin(), 2) == 3);
+  }
 #if TEST_STD_VER >= 11
-    {
-        typedef int V;
-        V ar[] =
-        {
-            1,
-            1,
-            1,
-            2,
-            2,
-            2,
-            3,
-            3,
-            3
-        };
-        typedef test_less<int> C;
-        typedef other_allocator<V> A;
-        std::set<int, C, A> mo(ar, ar+sizeof(ar)/sizeof(ar[0]), C(5), A(7));
-        std::set<int, C, A> m = mo;
-        assert(m.get_allocator() == A(-2));
-        assert(m.key_comp() == C(5));
-        assert(m.size() == 3);
-        assert(std::distance(m.begin(), m.end()) == 3);
-        assert(*m.begin() == 1);
-        assert(*std::next(m.begin()) == 2);
-        assert(*std::next(m.begin(), 2) == 3);
+  {
+    typedef int V;
+    V ar[] = {1, 1, 1, 2, 2, 2, 3, 3, 3};
+    typedef test_less<int> C;
+    typedef other_allocator<V> A;
+    std::set<int, C, A> mo(ar, ar + sizeof(ar) / sizeof(ar[0]), C(5), A(7));
+    std::set<int, C, A> m = mo;
+    assert(m.get_allocator() == A(-2));
+    assert(m.key_comp() == C(5));
+    assert(m.size() == 3);
+    assert(std::distance(m.begin(), m.end()) == 3);
+    assert(*m.begin() == 1);
+    assert(*std::next(m.begin()) == 2);
+    assert(*std::next(m.begin(), 2) == 3);
 
-        assert(mo.get_allocator() == A(7));
-        assert(mo.key_comp() == C(5));
-        assert(mo.size() == 3);
-        assert(std::distance(mo.begin(), mo.end()) == 3);
-        assert(*mo.begin() == 1);
-        assert(*std::next(mo.begin()) == 2);
-        assert(*std::next(mo.begin(), 2) == 3);
-    }
+    assert(mo.get_allocator() == A(7));
+    assert(mo.key_comp() == C(5));
+    assert(mo.size() == 3);
+    assert(std::distance(mo.begin(), mo.end()) == 3);
+    assert(*mo.begin() == 1);
+    assert(*std::next(mo.begin()) == 2);
+    assert(*std::next(mo.begin(), 2) == 3);
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/associative/set/set.cons/copy_alloc.pass.cpp b/libcxx/test/std/containers/associative/set/set.cons/copy_alloc.pass.cpp
index 9d988d84a3795..8f9dd769329c0 100644
--- a/libcxx/test/std/containers/associative/set/set.cons/copy_alloc.pass.cpp
+++ b/libcxx/test/std/containers/associative/set/set.cons/copy_alloc.pass.cpp
@@ -20,40 +20,28 @@
 #include "../../../test_compare.h"
 #include "test_allocator.h"
 
-int main(int, char**)
-{
-    typedef int V;
-    V ar[] =
-    {
-        1,
-        1,
-        1,
-        2,
-        2,
-        2,
-        3,
-        3,
-        3
-    };
-    typedef test_less<int> C;
-    typedef test_allocator<V> A;
-    std::set<int, C, A> mo(ar, ar+sizeof(ar)/sizeof(ar[0]), C(5), A(7));
-    std::set<int, C, A> m(mo, A(3));
-    assert(m.get_allocator() == A(3));
-    assert(m.key_comp() == C(5));
-    assert(m.size() == 3);
-    assert(std::distance(m.begin(), m.end()) == 3);
-    assert(*m.begin() == 1);
-    assert(*std::next(m.begin()) == 2);
-    assert(*std::next(m.begin(), 2) == 3);
-
-    assert(mo.get_allocator() == A(7));
-    assert(mo.key_comp() == C(5));
-    assert(mo.size() == 3);
-    assert(std::distance(mo.begin(), mo.end()) == 3);
-    assert(*mo.begin() == 1);
-    assert(*std::next(mo.begin()) == 2);
-    assert(*std::next(mo.begin(), 2) == 3);
+int main(int, char**) {
+  typedef int V;
+  V ar[] = {1, 1, 1, 2, 2, 2, 3, 3, 3};
+  typedef test_less<int> C;
+  typedef test_allocator<V> A;
+  std::set<int, C, A> mo(ar, ar + sizeof(ar) / sizeof(ar[0]), C(5), A(7));
+  std::set<int, C, A> m(mo, A(3));
+  assert(m.get_allocator() == A(3));
+  assert(m.key_comp() == C(5));
+  assert(m.size() == 3);
+  assert(std::distance(m.begin(), m.end()) == 3);
+  assert(*m.begin() == 1);
+  assert(*std::next(m.begin()) == 2);
+  assert(*std::next(m.begin(), 2) == 3);
+
+  assert(mo.get_allocator() == A(7));
+  assert(mo.key_comp() == C(5));
+  assert(mo.size() == 3);
+  assert(std::distance(mo.begin(), mo.end()) == 3);
+  assert(*mo.begin() == 1);
+  assert(*std::next(mo.begin()) == 2);
+  assert(*std::next(mo.begin(), 2) == 3);
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/associative/set/set.cons/copy_assign.pass.cpp b/libcxx/test/std/containers/associative/set/set.cons/copy_assign.pass.cpp
index 7d35ab62effa6..e21e07734d888 100644
--- a/libcxx/test/std/containers/associative/set/set.cons/copy_assign.pass.cpp
+++ b/libcxx/test/std/containers/associative/set/set.cons/copy_assign.pass.cpp
@@ -20,93 +20,65 @@
 #include "../../../test_compare.h"
 #include "test_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef int V;
-        V ar[] =
-        {
-            1,
-            1,
-            1,
-            2,
-            2,
-            2,
-            3,
-            3,
-            3
-        };
-        typedef test_less<int> C;
-        typedef test_allocator<V> A;
-        std::set<int, C, A> mo(ar, ar+sizeof(ar)/sizeof(ar[0]), C(5), A(2));
-        std::set<int, C, A> m(ar, ar+sizeof(ar)/sizeof(ar[0])/2, C(3), A(7));
-        m = mo;
-        assert(m.get_allocator() == A(7));
-        assert(m.key_comp() == C(5));
-        assert(m.size() == 3);
-        assert(std::distance(m.begin(), m.end()) == 3);
-        assert(*m.begin() == 1);
-        assert(*std::next(m.begin()) == 2);
-        assert(*std::next(m.begin(), 2) == 3);
+int main(int, char**) {
+  {
+    typedef int V;
+    V ar[] = {1, 1, 1, 2, 2, 2, 3, 3, 3};
+    typedef test_less<int> C;
+    typedef test_allocator<V> A;
+    std::set<int, C, A> mo(ar, ar + sizeof(ar) / sizeof(ar[0]), C(5), A(2));
+    std::set<int, C, A> m(ar, ar + sizeof(ar) / sizeof(ar[0]) / 2, C(3), A(7));
+    m = mo;
+    assert(m.get_allocator() == A(7));
+    assert(m.key_comp() == C(5));
+    assert(m.size() == 3);
+    assert(std::distance(m.begin(), m.end()) == 3);
+    assert(*m.begin() == 1);
+    assert(*std::next(m.begin()) == 2);
+    assert(*std::next(m.begin(), 2) == 3);
 
-        assert(mo.get_allocator() == A(2));
-        assert(mo.key_comp() == C(5));
-        assert(mo.size() == 3);
-        assert(std::distance(mo.begin(), mo.end()) == 3);
-        assert(*mo.begin() == 1);
-        assert(*std::next(mo.begin()) == 2);
-        assert(*std::next(mo.begin(), 2) == 3);
-    }
-    {
-        typedef int V;
-        const V ar[] =
-        {
-            1,
-            2,
-            3
-        };
-        std::set<int> m(ar, ar+sizeof(ar)/sizeof(ar[0]));
-        std::set<int> *p = &m;
-        m = *p;
+    assert(mo.get_allocator() == A(2));
+    assert(mo.key_comp() == C(5));
+    assert(mo.size() == 3);
+    assert(std::distance(mo.begin(), mo.end()) == 3);
+    assert(*mo.begin() == 1);
+    assert(*std::next(mo.begin()) == 2);
+    assert(*std::next(mo.begin(), 2) == 3);
+  }
+  {
+    typedef int V;
+    const V ar[] = {1, 2, 3};
+    std::set<int> m(ar, ar + sizeof(ar) / sizeof(ar[0]));
+    std::set<int>* p = &m;
+    m                = *p;
 
-        assert(m.size() == 3);
-        assert(std::equal(m.begin(), m.end(), ar));
-    }
-    {
-        typedef int V;
-        V ar[] =
-        {
-            1,
-            1,
-            1,
-            2,
-            2,
-            2,
-            3,
-            3,
-            3
-        };
-        typedef test_less<int> C;
-        typedef other_allocator<V> A;
-        std::set<int, C, A> mo(ar, ar+sizeof(ar)/sizeof(ar[0]), C(5), A(2));
-        std::set<int, C, A> m(ar, ar+sizeof(ar)/sizeof(ar[0])/2, C(3), A(7));
-        m = mo;
-        assert(m.get_allocator() == A(2));
-        assert(m.key_comp() == C(5));
-        assert(m.size() == 3);
-        assert(std::distance(m.begin(), m.end()) == 3);
-        assert(*m.begin() == 1);
-        assert(*std::next(m.begin()) == 2);
-        assert(*std::next(m.begin(), 2) == 3);
+    assert(m.size() == 3);
+    assert(std::equal(m.begin(), m.end(), ar));
+  }
+  {
+    typedef int V;
+    V ar[] = {1, 1, 1, 2, 2, 2, 3, 3, 3};
+    typedef test_less<int> C;
+    typedef other_allocator<V> A;
+    std::set<int, C, A> mo(ar, ar + sizeof(ar) / sizeof(ar[0]), C(5), A(2));
+    std::set<int, C, A> m(ar, ar + sizeof(ar) / sizeof(ar[0]) / 2, C(3), A(7));
+    m = mo;
+    assert(m.get_allocator() == A(2));
+    assert(m.key_comp() == C(5));
+    assert(m.size() == 3);
+    assert(std::distance(m.begin(), m.end()) == 3);
+    assert(*m.begin() == 1);
+    assert(*std::next(m.begin()) == 2);
+    assert(*std::next(m.begin(), 2) == 3);
 
-        assert(mo.get_allocator() == A(2));
-        assert(mo.key_comp() == C(5));
-        assert(mo.size() == 3);
-        assert(std::distance(mo.begin(), mo.end()) == 3);
-        assert(*mo.begin() == 1);
-        assert(*std::next(mo.begin()) == 2);
-        assert(*std::next(mo.begin(), 2) == 3);
-    }
+    assert(mo.get_allocator() == A(2));
+    assert(mo.key_comp() == C(5));
+    assert(mo.size() == 3);
+    assert(std::distance(mo.begin(), mo.end()) == 3);
+    assert(*mo.begin() == 1);
+    assert(*std::next(mo.begin()) == 2);
+    assert(*std::next(mo.begin(), 2) == 3);
+  }
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/associative/set/set.cons/deduct.pass.cpp b/libcxx/test/std/containers/associative/set/set.cons/deduct.pass.cpp
index 74e72b7b71de5..e26df4503ed41 100644
--- a/libcxx/test/std/containers/associative/set/set.cons/deduct.pass.cpp
+++ b/libcxx/test/std/containers/associative/set/set.cons/deduct.pass.cpp
@@ -52,37 +52,32 @@ struct NotAnAllocator {
   friend bool operator<(NotAnAllocator, NotAnAllocator) { return false; }
 };
 
-int main(int, char **) {
+int main(int, char**) {
   {
-    const int arr[] = { 1, 2, 1, INT_MAX, 3 };
+    const int arr[] = {1, 2, 1, INT_MAX, 3};
     std::set s(std::begin(arr), std::end(arr));
 
     ASSERT_SAME_TYPE(decltype(s), std::set<int>);
-    const int expected_s[] = { 1, 2, 3, INT_MAX };
-    assert(std::equal(s.begin(), s.end(), std::begin(expected_s),
-                      std::end(expected_s)));
+    const int expected_s[] = {1, 2, 3, INT_MAX};
+    assert(std::equal(s.begin(), s.end(), std::begin(expected_s), std::end(expected_s)));
   }
 
   {
-    const int arr[] = { 1, 2, 1, INT_MAX, 3 };
+    const int arr[] = {1, 2, 1, INT_MAX, 3};
     std::set s(std::begin(arr), std::end(arr), std::greater<int>());
 
     ASSERT_SAME_TYPE(decltype(s), std::set<int, std::greater<int> >);
-    const int expected_s[] = { INT_MAX, 3, 2, 1 };
-    assert(std::equal(s.begin(), s.end(), std::begin(expected_s),
-                      std::end(expected_s)));
+    const int expected_s[] = {INT_MAX, 3, 2, 1};
+    assert(std::equal(s.begin(), s.end(), std::begin(expected_s), std::end(expected_s)));
   }
 
   {
-    const int arr[] = { 1, 2, 1, INT_MAX, 3 };
-    std::set s(std::begin(arr), std::end(arr), std::greater<int>(),
-               test_allocator<int>(0, 42));
+    const int arr[] = {1, 2, 1, INT_MAX, 3};
+    std::set s(std::begin(arr), std::end(arr), std::greater<int>(), test_allocator<int>(0, 42));
 
-    ASSERT_SAME_TYPE(decltype(s),
-                     std::set<int, std::greater<int>, test_allocator<int> >);
-    const int expected_s[] = { INT_MAX, 3, 2, 1 };
-    assert(std::equal(s.begin(), s.end(), std::begin(expected_s),
-                      std::end(expected_s)));
+    ASSERT_SAME_TYPE(decltype(s), std::set<int, std::greater<int>, test_allocator<int> >);
+    const int expected_s[] = {INT_MAX, 3, 2, 1};
+    assert(std::equal(s.begin(), s.end(), std::begin(expected_s), std::end(expected_s)));
     assert(s.get_allocator().get_id() == 42);
   }
 
@@ -95,7 +90,7 @@ int main(int, char **) {
 
   {
     std::set<long> source;
-    std::set s{ source };  // braces instead of parens
+    std::set s{source}; // braces instead of parens
     ASSERT_SAME_TYPE(decltype(s), std::set<long>);
     assert(s.size() == 0);
   }
@@ -108,120 +103,111 @@ int main(int, char **) {
   }
 
   {
-    std::set s{ 1, 2, 1, INT_MAX, 3 };
+    std::set s{1, 2, 1, INT_MAX, 3};
 
     ASSERT_SAME_TYPE(decltype(s), std::set<int>);
-    const int expected_s[] = { 1, 2, 3, INT_MAX };
-    assert(std::equal(s.begin(), s.end(), std::begin(expected_s),
-                      std::end(expected_s)));
+    const int expected_s[] = {1, 2, 3, INT_MAX};
+    assert(std::equal(s.begin(), s.end(), std::begin(expected_s), std::end(expected_s)));
   }
 
   {
-    std::set s({ 1, 2, 1, INT_MAX, 3 }, std::greater<int>());
+    std::set s({1, 2, 1, INT_MAX, 3}, std::greater<int>());
 
     ASSERT_SAME_TYPE(decltype(s), std::set<int, std::greater<int> >);
-    const int expected_s[] = { INT_MAX, 3, 2, 1 };
-    assert(std::equal(s.begin(), s.end(), std::begin(expected_s),
-                      std::end(expected_s)));
+    const int expected_s[] = {INT_MAX, 3, 2, 1};
+    assert(std::equal(s.begin(), s.end(), std::begin(expected_s), std::end(expected_s)));
   }
 
   {
-    std::set s({ 1, 2, 1, INT_MAX, 3 }, std::greater<int>(),
-               test_allocator<int>(0, 43));
+    std::set s({1, 2, 1, INT_MAX, 3}, std::greater<int>(), test_allocator<int>(0, 43));
 
-    ASSERT_SAME_TYPE(decltype(s),
-                     std::set<int, std::greater<int>, test_allocator<int> >);
-    const int expected_s[] = { INT_MAX, 3, 2, 1 };
-    assert(std::equal(s.begin(), s.end(), std::begin(expected_s),
-                      std::end(expected_s)));
+    ASSERT_SAME_TYPE(decltype(s), std::set<int, std::greater<int>, test_allocator<int> >);
+    const int expected_s[] = {INT_MAX, 3, 2, 1};
+    assert(std::equal(s.begin(), s.end(), std::begin(expected_s), std::end(expected_s)));
     assert(s.get_allocator().get_id() == 43);
   }
 
   {
-    const int arr[] = { 1, 2, 1, INT_MAX, 3 };
+    const int arr[] = {1, 2, 1, INT_MAX, 3};
     std::set s(std::begin(arr), std::end(arr), test_allocator<int>(0, 44));
 
-    ASSERT_SAME_TYPE(decltype(s),
-                     std::set<int, std::less<int>, test_allocator<int> >);
-    const int expected_s[] = { 1, 2, 3, INT_MAX };
-    assert(std::equal(s.begin(), s.end(), std::begin(expected_s),
-                      std::end(expected_s)));
+    ASSERT_SAME_TYPE(decltype(s), std::set<int, std::less<int>, test_allocator<int> >);
+    const int expected_s[] = {1, 2, 3, INT_MAX};
+    assert(std::equal(s.begin(), s.end(), std::begin(expected_s), std::end(expected_s)));
     assert(s.get_allocator().get_id() == 44);
   }
 
   {
-    std::set s({ 1, 2, 1, INT_MAX, 3 }, test_allocator<int>(0, 45));
+    std::set s({1, 2, 1, INT_MAX, 3}, test_allocator<int>(0, 45));
 
-    ASSERT_SAME_TYPE(decltype(s),
-                     std::set<int, std::less<int>, test_allocator<int> >);
-    const int expected_s[] = { 1, 2, 3, INT_MAX };
-    assert(std::equal(s.begin(), s.end(), std::begin(expected_s),
-                      std::end(expected_s)));
+    ASSERT_SAME_TYPE(decltype(s), std::set<int, std::less<int>, test_allocator<int> >);
+    const int expected_s[] = {1, 2, 3, INT_MAX};
+    assert(std::equal(s.begin(), s.end(), std::begin(expected_s), std::end(expected_s)));
     assert(s.get_allocator().get_id() == 45);
   }
 
   {
     NotAnAllocator a;
-    std::set s{ a }; // set(initializer_list<NotAnAllocator>)
+    std::set s{a}; // set(initializer_list<NotAnAllocator>)
     ASSERT_SAME_TYPE(decltype(s), std::set<NotAnAllocator>);
     assert(s.size() == 1);
   }
 
   {
     std::set<long> source;
-    std::set s{ source, source }; // set(initializer_list<set<long>>)
+    std::set s{source, source}; // set(initializer_list<set<long>>)
     ASSERT_SAME_TYPE(decltype(s), std::set<std::set<long> >);
     assert(s.size() == 1);
   }
 
   {
     NotAnAllocator a;
-    std::set s{ a, a }; // set(initializer_list<NotAnAllocator>)
+    std::set s{a, a}; // set(initializer_list<NotAnAllocator>)
     ASSERT_SAME_TYPE(decltype(s), std::set<NotAnAllocator>);
     assert(s.size() == 1);
   }
 
   {
-    int source[3] = { 3, 4, 5 };
+    int source[3] = {3, 4, 5};
     std::set s(source, source + 3); // set(InputIterator, InputIterator)
     ASSERT_SAME_TYPE(decltype(s), std::set<int>);
     assert(s.size() == 3);
   }
 
   {
-    int source[3] = { 3, 4, 5 };
-    std::set s{ source, source + 3 }; // set(initializer_list<int*>)
-    ASSERT_SAME_TYPE(decltype(s), std::set<int *>);
+    int source[3] = {3, 4, 5};
+    std::set s{source, source + 3}; // set(initializer_list<int*>)
+    ASSERT_SAME_TYPE(decltype(s), std::set<int*>);
     assert(s.size() == 2);
   }
 
 #if TEST_STD_VER >= 23
-    {
-      using Range = std::array<int, 0>;
-      using Comp = std::greater<int>;
-      using DefaultComp = std::less<int>;
-      using Alloc = test_allocator<int>;
-
-      { // (from_range, range)
-        std::set c(std::from_range, Range());
-        static_assert(std::is_same_v<decltype(c), std::set<int>>);
-      }
-
-      { // (from_range, range, comp)
-        std::set c(std::from_range, Range(), Comp());
-        static_assert(std::is_same_v<decltype(c), std::set<int, Comp>>);
-      }
-
-      { // (from_range, range, comp, alloc)
-        std::set c(std::from_range, Range(), Comp(), Alloc());
-        static_assert(std::is_same_v<decltype(c), std::set<int, Comp, Alloc>>);
-      }
-
-      { // (from_range, range, alloc)
-        std::set c(std::from_range, Range(), Alloc());
-        static_assert(std::is_same_v<decltype(c), std::set<int, DefaultComp, Alloc>>);
-      }
+  {
+    using Range       = std::array<int, 0>;
+    using Comp        = std::greater<int>;
+    using DefaultComp = std::less<int>;
+    using Alloc       = test_allocator<int>;
+
+    { // (from_range, range)
+      std::set c(std::from_range, Range());
+      static_assert(std::is_same_v<decltype(c), std::set<int>>);
     }
+
+    { // (from_range, range, comp)
+      std::set c(std::from_range, Range(), Comp());
+      static_assert(std::is_same_v<decltype(c), std::set<int, Comp>>);
+    }
+
+    { // (from_range, range, comp, alloc)
+      std::set c(std::from_range, Range(), Comp(), Alloc());
+      static_assert(std::is_same_v<decltype(c), std::set<int, Comp, Alloc>>);
+    }
+
+    { // (from_range, range, alloc)
+      std::set c(std::from_range, Range(), Alloc());
+      static_assert(std::is_same_v<decltype(c), std::set<int, DefaultComp, Alloc>>);
+    }
+  }
 #endif
 
   AssociativeContainerDeductionGuidesSfinaeAway<std::set, std::set<int>>();
diff --git a/libcxx/test/std/containers/associative/set/set.cons/deduct.verify.cpp b/libcxx/test/std/containers/associative/set/set.cons/deduct.verify.cpp
index 6e250f8fa21e6..8b8d70111e430 100644
--- a/libcxx/test/std/containers/associative/set/set.cons/deduct.verify.cpp
+++ b/libcxx/test/std/containers/associative/set/set.cons/deduct.verify.cpp
@@ -36,7 +36,7 @@ struct NotAnAllocator {
   friend bool operator<(NotAnAllocator, NotAnAllocator) { return false; }
 };
 
-int main(int, char **) {
+int main(int, char**) {
   {
     // cannot deduce Key from nothing
     std::set s;
diff --git a/libcxx/test/std/containers/associative/set/set.cons/default.pass.cpp b/libcxx/test/std/containers/associative/set/set.cons/default.pass.cpp
index 466f5132f3a88..c41d7c3b9d3bd 100644
--- a/libcxx/test/std/containers/associative/set/set.cons/default.pass.cpp
+++ b/libcxx/test/std/containers/associative/set/set.cons/default.pass.cpp
@@ -18,38 +18,37 @@
 #include "test_macros.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
+int main(int, char**) {
+  {
     std::set<int> m;
     assert(m.empty());
     assert(m.begin() == m.end());
-    }
+  }
 #if TEST_STD_VER >= 11
-    {
+  {
     std::set<int, std::less<int>, min_allocator<int>> m;
     assert(m.empty());
     assert(m.begin() == m.end());
-    }
-    {
+  }
+  {
     typedef explicit_allocator<int> A;
-        {
-        std::set<int, std::less<int>, A> m;
-        assert(m.empty());
-        assert(m.begin() == m.end());
-        }
-        {
-        A a;
-        std::set<int, std::less<int>, A> m(a);
-        assert(m.empty());
-        assert(m.begin() == m.end());
-        }
+    {
+      std::set<int, std::less<int>, A> m;
+      assert(m.empty());
+      assert(m.begin() == m.end());
     }
     {
+      A a;
+      std::set<int, std::less<int>, A> m(a);
+      assert(m.empty());
+      assert(m.begin() == m.end());
+    }
+  }
+  {
     std::set<int> m = {};
     assert(m.empty());
     assert(m.begin() == m.end());
-    }
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/associative/set/set.cons/default_noexcept.pass.cpp b/libcxx/test/std/containers/associative/set/set.cons/default_noexcept.pass.cpp
index db97fea722bb1..c01ac3b2fad0b 100644
--- a/libcxx/test/std/containers/associative/set/set.cons/default_noexcept.pass.cpp
+++ b/libcxx/test/std/containers/associative/set/set.cons/default_noexcept.pass.cpp
@@ -26,33 +26,31 @@
 #include "test_allocator.h"
 
 template <class T>
-struct some_comp
-{
-    typedef T value_type;
-    some_comp();
-    bool operator()(const T&, const T&) const { return false; }
+struct some_comp {
+  typedef T value_type;
+  some_comp();
+  bool operator()(const T&, const T&) const { return false; }
 };
 
-int main(int, char**)
-{
+int main(int, char**) {
 #if defined(_LIBCPP_VERSION)
-    {
-        typedef std::set<MoveOnly> C;
-        static_assert(std::is_nothrow_default_constructible<C>::value, "");
-    }
-    {
-        typedef std::set<MoveOnly, std::less<MoveOnly>, test_allocator<MoveOnly>> C;
-        static_assert(std::is_nothrow_default_constructible<C>::value, "");
-    }
+  {
+    typedef std::set<MoveOnly> C;
+    static_assert(std::is_nothrow_default_constructible<C>::value, "");
+  }
+  {
+    typedef std::set<MoveOnly, std::less<MoveOnly>, test_allocator<MoveOnly>> C;
+    static_assert(std::is_nothrow_default_constructible<C>::value, "");
+  }
 #endif // _LIBCPP_VERSION
-    {
-        typedef std::set<MoveOnly, std::less<MoveOnly>, other_allocator<MoveOnly>> C;
-        static_assert(!std::is_nothrow_default_constructible<C>::value, "");
-    }
-    {
-        typedef std::set<MoveOnly, some_comp<MoveOnly>> C;
-        static_assert(!std::is_nothrow_default_constructible<C>::value, "");
-    }
+  {
+    typedef std::set<MoveOnly, std::less<MoveOnly>, other_allocator<MoveOnly>> C;
+    static_assert(!std::is_nothrow_default_constructible<C>::value, "");
+  }
+  {
+    typedef std::set<MoveOnly, some_comp<MoveOnly>> C;
+    static_assert(!std::is_nothrow_default_constructible<C>::value, "");
+  }
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/associative/set/set.cons/dtor_noexcept.pass.cpp b/libcxx/test/std/containers/associative/set/set.cons/dtor_noexcept.pass.cpp
index c9b73731c3078..63c0433477414 100644
--- a/libcxx/test/std/containers/associative/set/set.cons/dtor_noexcept.pass.cpp
+++ b/libcxx/test/std/containers/associative/set/set.cons/dtor_noexcept.pass.cpp
@@ -20,32 +20,30 @@
 #include "test_allocator.h"
 
 template <class T>
-struct some_comp
-{
-    typedef T value_type;
-    ~some_comp() noexcept(false);
-    bool operator()(const T&, const T&) const { return false; }
+struct some_comp {
+  typedef T value_type;
+  ~some_comp() noexcept(false);
+  bool operator()(const T&, const T&) const { return false; }
 };
 
-int main(int, char**)
-{
-    {
-        typedef std::set<MoveOnly> C;
-        static_assert(std::is_nothrow_destructible<C>::value, "");
-    }
-    {
-        typedef std::set<MoveOnly, std::less<MoveOnly>, test_allocator<MoveOnly>> C;
-        static_assert(std::is_nothrow_destructible<C>::value, "");
-    }
-    {
-        typedef std::set<MoveOnly, std::less<MoveOnly>, other_allocator<MoveOnly>> C;
-        static_assert(std::is_nothrow_destructible<C>::value, "");
-    }
+int main(int, char**) {
+  {
+    typedef std::set<MoveOnly> C;
+    static_assert(std::is_nothrow_destructible<C>::value, "");
+  }
+  {
+    typedef std::set<MoveOnly, std::less<MoveOnly>, test_allocator<MoveOnly>> C;
+    static_assert(std::is_nothrow_destructible<C>::value, "");
+  }
+  {
+    typedef std::set<MoveOnly, std::less<MoveOnly>, other_allocator<MoveOnly>> C;
+    static_assert(std::is_nothrow_destructible<C>::value, "");
+  }
 #if defined(_LIBCPP_VERSION)
-    {
-        typedef std::set<MoveOnly, some_comp<MoveOnly>> C;
-        static_assert(!std::is_nothrow_destructible<C>::value, "");
-    }
+  {
+    typedef std::set<MoveOnly, some_comp<MoveOnly>> C;
+    static_assert(!std::is_nothrow_destructible<C>::value, "");
+  }
 #endif // _LIBCPP_VERSION
 
   return 0;
diff --git a/libcxx/test/std/containers/associative/set/set.cons/from_range.pass.cpp b/libcxx/test/std/containers/associative/set/set.cons/from_range.pass.cpp
index 4c9b4aaf26291..3590b75065ad9 100644
--- a/libcxx/test/std/containers/associative/set/set.cons/from_range.pass.cpp
+++ b/libcxx/test/std/containers/associative/set/set.cons/from_range.pass.cpp
@@ -24,13 +24,9 @@
 void test_duplicates() {
   using T = KeyValue;
 
-  std::array input = {
-    T{1, 'a'}, T{2, 'a'}, T{3, 'a'}, T{3, 'b'}, T{3, 'c'}, T{2, 'b'}, T{4, 'a'}
-  };
-  std::array expected = {
-    T{1, 'a'}, T{2, 'b'}, T{3, 'c'}, T{4, 'a'}
-  };
-  auto c = std::set<T>(std::from_range, input);
+  std::array input    = {T{1, 'a'}, T{2, 'a'}, T{3, 'a'}, T{3, 'b'}, T{3, 'c'}, T{2, 'b'}, T{4, 'a'}};
+  std::array expected = {T{1, 'a'}, T{2, 'b'}, T{3, 'c'}, T{4, 'a'}};
+  auto c              = std::set<T>(std::from_range, input);
   assert(std::ranges::is_permutation(expected, c));
 }
 
diff --git a/libcxx/test/std/containers/associative/set/set.cons/initializer_list.pass.cpp b/libcxx/test/std/containers/associative/set/set.cons/initializer_list.pass.cpp
index 4d540c061f369..48495712425be 100644
--- a/libcxx/test/std/containers/associative/set/set.cons/initializer_list.pass.cpp
+++ b/libcxx/test/std/containers/associative/set/set.cons/initializer_list.pass.cpp
@@ -20,9 +20,8 @@
 #include "test_macros.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
+int main(int, char**) {
+  {
     typedef std::set<int> C;
     typedef C::value_type V;
     C m = {1, 2, 3, 4, 5, 6};
@@ -35,8 +34,8 @@ int main(int, char**)
     assert(*++i == V(4));
     assert(*++i == V(5));
     assert(*++i == V(6));
-    }
-    {
+  }
+  {
     typedef std::set<int, std::less<int>, min_allocator<int>> C;
     typedef C::value_type V;
     C m = {1, 2, 3, 4, 5, 6};
@@ -49,7 +48,7 @@ int main(int, char**)
     assert(*++i == V(4));
     assert(*++i == V(5));
     assert(*++i == V(6));
-    }
+  }
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/associative/set/set.cons/initializer_list_compare.pass.cpp b/libcxx/test/std/containers/associative/set/set.cons/initializer_list_compare.pass.cpp
index f097ac8549556..25684966078d6 100644
--- a/libcxx/test/std/containers/associative/set/set.cons/initializer_list_compare.pass.cpp
+++ b/libcxx/test/std/containers/associative/set/set.cons/initializer_list_compare.pass.cpp
@@ -21,22 +21,21 @@
 #include "test_macros.h"
 #include "../../../test_compare.h"
 
-int main(int, char**)
-{
-    typedef test_less<int> Cmp;
-    typedef std::set<int, Cmp> C;
-    typedef C::value_type V;
-    C m({1, 2, 3, 4, 5, 6}, Cmp(10));
-    assert(m.size() == 6);
-    assert(std::distance(m.begin(), m.end()) == 6);
-    C::const_iterator i = m.cbegin();
-    assert(*i == V(1));
-    assert(*++i == V(2));
-    assert(*++i == V(3));
-    assert(*++i == V(4));
-    assert(*++i == V(5));
-    assert(*++i == V(6));
-    assert(m.key_comp() == Cmp(10));
+int main(int, char**) {
+  typedef test_less<int> Cmp;
+  typedef std::set<int, Cmp> C;
+  typedef C::value_type V;
+  C m({1, 2, 3, 4, 5, 6}, Cmp(10));
+  assert(m.size() == 6);
+  assert(std::distance(m.begin(), m.end()) == 6);
+  C::const_iterator i = m.cbegin();
+  assert(*i == V(1));
+  assert(*++i == V(2));
+  assert(*++i == V(3));
+  assert(*++i == V(4));
+  assert(*++i == V(5));
+  assert(*++i == V(6));
+  assert(m.key_comp() == Cmp(10));
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/associative/set/set.cons/initializer_list_compare_alloc.pass.cpp b/libcxx/test/std/containers/associative/set/set.cons/initializer_list_compare_alloc.pass.cpp
index 46785ef3310c1..48649c3b281f4 100644
--- a/libcxx/test/std/containers/associative/set/set.cons/initializer_list_compare_alloc.pass.cpp
+++ b/libcxx/test/std/containers/associative/set/set.cons/initializer_list_compare_alloc.pass.cpp
@@ -23,9 +23,8 @@
 #include "../../../test_compare.h"
 #include "test_allocator.h"
 
-int main(int, char**)
-{
-    {
+int main(int, char**) {
+  {
     typedef test_less<int> Cmp;
     typedef test_allocator<int> A;
     typedef std::set<int, Cmp, A> C;
@@ -42,8 +41,8 @@ int main(int, char**)
     assert(*++i == V(6));
     assert(m.key_comp() == Cmp(10));
     assert(m.get_allocator() == A(4));
-    }
-    {
+  }
+  {
     typedef test_less<int> Cmp;
     typedef test_allocator<int> A;
     typedef std::set<int, Cmp, A> C;
@@ -59,7 +58,7 @@ int main(int, char**)
     assert(*++i == V(5));
     assert(*++i == V(6));
     assert(m.get_allocator() == A(4));
-    }
+  }
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/associative/set/set.cons/iter_iter.pass.cpp b/libcxx/test/std/containers/associative/set/set.cons/iter_iter.pass.cpp
index 9871d0f3902b5..27a4e6a5f527d 100644
--- a/libcxx/test/std/containers/associative/set/set.cons/iter_iter.pass.cpp
+++ b/libcxx/test/std/containers/associative/set/set.cons/iter_iter.pass.cpp
@@ -20,53 +20,30 @@
 #include "test_iterators.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
+int main(int, char**) {
+  {
     typedef int V;
-    V ar[] =
-    {
-        1,
-        1,
-        1,
-        2,
-        2,
-        2,
-        3,
-        3,
-        3
-    };
-    std::set<V> m(cpp17_input_iterator<const int*>(ar),
-                  cpp17_input_iterator<const int*>(ar+sizeof(ar)/sizeof(ar[0])));
+    V ar[] = {1, 1, 1, 2, 2, 2, 3, 3, 3};
+    std::set<V> m(
+        cpp17_input_iterator<const int*>(ar), cpp17_input_iterator<const int*>(ar + sizeof(ar) / sizeof(ar[0])));
     assert(m.size() == 3);
     assert(std::distance(m.begin(), m.end()) == 3);
     assert(*m.begin() == 1);
     assert(*std::next(m.begin()) == 2);
     assert(*std::next(m.begin(), 2) == 3);
-    }
+  }
 #if TEST_STD_VER >= 11
-    {
+  {
     typedef int V;
-    V ar[] =
-    {
-        1,
-        1,
-        1,
-        2,
-        2,
-        2,
-        3,
-        3,
-        3
-    };
-    std::set<V, std::less<int>, min_allocator<int>> m(cpp17_input_iterator<const int*>(ar),
-                  cpp17_input_iterator<const int*>(ar+sizeof(ar)/sizeof(ar[0])));
+    V ar[] = {1, 1, 1, 2, 2, 2, 3, 3, 3};
+    std::set<V, std::less<int>, min_allocator<int>> m(
+        cpp17_input_iterator<const int*>(ar), cpp17_input_iterator<const int*>(ar + sizeof(ar) / sizeof(ar[0])));
     assert(m.size() == 3);
     assert(std::distance(m.begin(), m.end()) == 3);
     assert(*m.begin() == 1);
     assert(*std::next(m.begin()) == 2);
     assert(*std::next(m.begin(), 2) == 3);
-    }
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/associative/set/set.cons/iter_iter_alloc.pass.cpp b/libcxx/test/std/containers/associative/set/set.cons/iter_iter_alloc.pass.cpp
index 2cdad2adab174..dac553073f177 100644
--- a/libcxx/test/std/containers/associative/set/set.cons/iter_iter_alloc.pass.cpp
+++ b/libcxx/test/std/containers/associative/set/set.cons/iter_iter_alloc.pass.cpp
@@ -26,27 +26,17 @@
 #include "../../../test_compare.h"
 #include "test_allocator.h"
 
-int main(int, char**)
-{
-    {
+int main(int, char**) {
+  {
     typedef int V;
-    V ar[] =
-    {
-        1,
-        1,
-        1,
-        2,
-        2,
-        2,
-        3,
-        3,
-        3
-    };
+    V ar[] = {1, 1, 1, 2, 2, 2, 3, 3, 3};
     typedef test_less<V> C;
     typedef test_allocator<V> A;
-    std::set<V, C, A> m(cpp17_input_iterator<const V*>(ar),
-                        cpp17_input_iterator<const V*>(ar+sizeof(ar)/sizeof(ar[0])),
-                        C(5), A(7));
+    std::set<V, C, A> m(
+        cpp17_input_iterator<const V*>(ar),
+        cpp17_input_iterator<const V*>(ar + sizeof(ar) / sizeof(ar[0])),
+        C(5),
+        A(7));
     assert(m.value_comp() == C(5));
     assert(m.get_allocator() == A(7));
     assert(m.size() == 3);
@@ -54,26 +44,15 @@ int main(int, char**)
     assert(*m.begin() == 1);
     assert(*std::next(m.begin()) == 2);
     assert(*std::next(m.begin(), 2) == 3);
-    }
+  }
 #if TEST_STD_VER > 11
-    {
+  {
     typedef int V;
-    V ar[] =
-    {
-        1,
-        1,
-        1,
-        2,
-        2,
-        2,
-        3,
-        3,
-        3
-    };
+    V ar[] = {1, 1, 1, 2, 2, 2, 3, 3, 3};
     typedef test_allocator<V> A;
     typedef test_less<int> C;
     A a(7);
-    std::set<V, C, A> m(ar, ar+sizeof(ar)/sizeof(ar[0]), a);
+    std::set<V, C, A> m(ar, ar + sizeof(ar) / sizeof(ar[0]), a);
 
     assert(m.size() == 3);
     assert(std::distance(m.begin(), m.end()) == 3);
@@ -81,7 +60,7 @@ int main(int, char**)
     assert(*std::next(m.begin()) == 2);
     assert(*std::next(m.begin(), 2) == 3);
     assert(m.get_allocator() == a);
-    }
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/associative/set/set.cons/iter_iter_comp.pass.cpp b/libcxx/test/std/containers/associative/set/set.cons/iter_iter_comp.pass.cpp
index c783282149a00..573bbd316e073 100644
--- a/libcxx/test/std/containers/associative/set/set.cons/iter_iter_comp.pass.cpp
+++ b/libcxx/test/std/containers/associative/set/set.cons/iter_iter_comp.pass.cpp
@@ -20,30 +20,18 @@
 #include "test_iterators.h"
 #include "../../../test_compare.h"
 
-int main(int, char**)
-{
-    typedef int V;
-    V ar[] =
-    {
-        1,
-        1,
-        1,
-        2,
-        2,
-        2,
-        3,
-        3,
-        3
-    };
-    typedef test_less<V> C;
-    std::set<V, C> m(cpp17_input_iterator<const V*>(ar),
-                     cpp17_input_iterator<const V*>(ar+sizeof(ar)/sizeof(ar[0])), C(5));
-    assert(m.value_comp() == C(5));
-    assert(m.size() == 3);
-    assert(std::distance(m.begin(), m.end()) == 3);
-    assert(*m.begin() == 1);
-    assert(*std::next(m.begin()) == 2);
-    assert(*std::next(m.begin(), 2) == 3);
+int main(int, char**) {
+  typedef int V;
+  V ar[] = {1, 1, 1, 2, 2, 2, 3, 3, 3};
+  typedef test_less<V> C;
+  std::set<V, C> m(
+      cpp17_input_iterator<const V*>(ar), cpp17_input_iterator<const V*>(ar + sizeof(ar) / sizeof(ar[0])), C(5));
+  assert(m.value_comp() == C(5));
+  assert(m.size() == 3);
+  assert(std::distance(m.begin(), m.end()) == 3);
+  assert(*m.begin() == 1);
+  assert(*std::next(m.begin()) == 2);
+  assert(*std::next(m.begin(), 2) == 3);
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/associative/set/set.cons/move.pass.cpp b/libcxx/test/std/containers/associative/set/set.cons/move.pass.cpp
index 3d6ab89ccf82f..2c1ec22ab257d 100644
--- a/libcxx/test/std/containers/associative/set/set.cons/move.pass.cpp
+++ b/libcxx/test/std/containers/associative/set/set.cons/move.pass.cpp
@@ -22,88 +22,65 @@
 #include "test_allocator.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef int V;
-        typedef test_less<int> C;
-        typedef test_allocator<V> A;
-        std::set<int, C, A> mo(C(5), A(7));
-        std::set<int, C, A> m = std::move(mo);
-        assert(m.get_allocator() == A(7));
-        assert(m.key_comp() == C(5));
-        assert(m.size() == 0);
-        assert(std::distance(m.begin(), m.end()) == 0);
+int main(int, char**) {
+  {
+    typedef int V;
+    typedef test_less<int> C;
+    typedef test_allocator<V> A;
+    std::set<int, C, A> mo(C(5), A(7));
+    std::set<int, C, A> m = std::move(mo);
+    assert(m.get_allocator() == A(7));
+    assert(m.key_comp() == C(5));
+    assert(m.size() == 0);
+    assert(std::distance(m.begin(), m.end()) == 0);
 
-        assert(mo.get_allocator() == A(7));
-        assert(mo.get_allocator().get_id() == test_alloc_base::moved_value);
-        assert(mo.key_comp() == C(5));
-        assert(mo.size() == 0);
-        assert(std::distance(mo.begin(), mo.end()) == 0);
-    }
-    {
-        typedef int V;
-        V ar[] =
-        {
-            1,
-            1,
-            1,
-            2,
-            2,
-            2,
-            3,
-            3,
-            3
-        };
-        typedef test_less<int> C;
-        typedef test_allocator<V> A;
-        std::set<int, C, A> mo(ar, ar+sizeof(ar)/sizeof(ar[0]), C(5), A(7));
-        std::set<int, C, A> m = std::move(mo);
-        assert(m.get_allocator() == A(7));
-        assert(m.key_comp() == C(5));
-        assert(m.size() == 3);
-        assert(std::distance(m.begin(), m.end()) == 3);
-        assert(*m.begin() == 1);
-        assert(*std::next(m.begin()) == 2);
-        assert(*std::next(m.begin(), 2) == 3);
+    assert(mo.get_allocator() == A(7));
+    assert(mo.get_allocator().get_id() == test_alloc_base::moved_value);
+    assert(mo.key_comp() == C(5));
+    assert(mo.size() == 0);
+    assert(std::distance(mo.begin(), mo.end()) == 0);
+  }
+  {
+    typedef int V;
+    V ar[] = {1, 1, 1, 2, 2, 2, 3, 3, 3};
+    typedef test_less<int> C;
+    typedef test_allocator<V> A;
+    std::set<int, C, A> mo(ar, ar + sizeof(ar) / sizeof(ar[0]), C(5), A(7));
+    std::set<int, C, A> m = std::move(mo);
+    assert(m.get_allocator() == A(7));
+    assert(m.key_comp() == C(5));
+    assert(m.size() == 3);
+    assert(std::distance(m.begin(), m.end()) == 3);
+    assert(*m.begin() == 1);
+    assert(*std::next(m.begin()) == 2);
+    assert(*std::next(m.begin(), 2) == 3);
 
-        assert(mo.get_allocator() == A(7));
-        assert(mo.get_allocator().get_id() == test_alloc_base::moved_value);
-        assert(mo.key_comp() == C(5));
-        assert(mo.size() == 0);
-        assert(std::distance(mo.begin(), mo.end()) == 0);
-    }
-    {
-        typedef int V;
-        V ar[] =
-        {
-            1,
-            1,
-            1,
-            2,
-            2,
-            2,
-            3,
-            3,
-            3
-        };
-        typedef test_less<int> C;
-        typedef min_allocator<V> A;
-        std::set<int, C, A> mo(ar, ar+sizeof(ar)/sizeof(ar[0]), C(5), A());
-        std::set<int, C, A> m = std::move(mo);
-        assert(m.get_allocator() == A());
-        assert(m.key_comp() == C(5));
-        assert(m.size() == 3);
-        assert(std::distance(m.begin(), m.end()) == 3);
-        assert(*m.begin() == 1);
-        assert(*std::next(m.begin()) == 2);
-        assert(*std::next(m.begin(), 2) == 3);
+    assert(mo.get_allocator() == A(7));
+    assert(mo.get_allocator().get_id() == test_alloc_base::moved_value);
+    assert(mo.key_comp() == C(5));
+    assert(mo.size() == 0);
+    assert(std::distance(mo.begin(), mo.end()) == 0);
+  }
+  {
+    typedef int V;
+    V ar[] = {1, 1, 1, 2, 2, 2, 3, 3, 3};
+    typedef test_less<int> C;
+    typedef min_allocator<V> A;
+    std::set<int, C, A> mo(ar, ar + sizeof(ar) / sizeof(ar[0]), C(5), A());
+    std::set<int, C, A> m = std::move(mo);
+    assert(m.get_allocator() == A());
+    assert(m.key_comp() == C(5));
+    assert(m.size() == 3);
+    assert(std::distance(m.begin(), m.end()) == 3);
+    assert(*m.begin() == 1);
+    assert(*std::next(m.begin()) == 2);
+    assert(*std::next(m.begin(), 2) == 3);
 
-        assert(mo.get_allocator() == A());
-        assert(mo.key_comp() == C(5));
-        assert(mo.size() == 0);
-        assert(std::distance(mo.begin(), mo.end()) == 0);
-    }
+    assert(mo.get_allocator() == A());
+    assert(mo.key_comp() == C(5));
+    assert(mo.size() == 0);
+    assert(std::distance(mo.begin(), mo.end()) == 0);
+  }
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/associative/set/set.cons/move_alloc.pass.cpp b/libcxx/test/std/containers/associative/set/set.cons/move_alloc.pass.cpp
index 886cad9fa2123..c85463bd391ca 100644
--- a/libcxx/test/std/containers/associative/set/set.cons/move_alloc.pass.cpp
+++ b/libcxx/test/std/containers/associative/set/set.cons/move_alloc.pass.cpp
@@ -24,171 +24,92 @@
 #include "test_allocator.h"
 #include "Counter.h"
 
-int main(int, char**)
-{
+int main(int, char**) {
+  {
+    typedef MoveOnly V;
+    typedef test_less<MoveOnly> C;
+    typedef test_allocator<V> A;
+    typedef std::set<MoveOnly, C, A> M;
+    typedef std::move_iterator<V*> I;
+    V a1[] = {V(1), V(1), V(1), V(2), V(2), V(2), V(3), V(3), V(3)};
+    M m1(I(a1), I(a1 + sizeof(a1) / sizeof(a1[0])), C(5), A(7));
+    V a2[] = {V(1), V(1), V(1), V(2), V(2), V(2), V(3), V(3), V(3)};
+    M m2(I(a2), I(a2 + sizeof(a2) / sizeof(a2[0])), C(5), A(7));
+    M m3(std::move(m1), A(7));
+    assert(m3 == m2);
+    assert(m3.get_allocator() == A(7));
+    assert(m3.key_comp() == C(5));
+    LIBCPP_ASSERT(m1.empty());
+  }
+  {
+    typedef MoveOnly V;
+    typedef test_less<MoveOnly> C;
+    typedef test_allocator<V> A;
+    typedef std::set<MoveOnly, C, A> M;
+    typedef std::move_iterator<V*> I;
+    V a1[] = {V(1), V(1), V(1), V(2), V(2), V(2), V(3), V(3), V(3)};
+    M m1(I(a1), I(a1 + sizeof(a1) / sizeof(a1[0])), C(5), A(7));
+    V a2[] = {V(1), V(1), V(1), V(2), V(2), V(2), V(3), V(3), V(3)};
+    M m2(I(a2), I(a2 + sizeof(a2) / sizeof(a2[0])), C(5), A(7));
+    M m3(std::move(m1), A(5));
+    assert(m3 == m2);
+    assert(m3.get_allocator() == A(5));
+    assert(m3.key_comp() == C(5));
+    LIBCPP_ASSERT(m1.empty());
+  }
+  {
+    typedef MoveOnly V;
+    typedef test_less<MoveOnly> C;
+    typedef other_allocator<V> A;
+    typedef std::set<MoveOnly, C, A> M;
+    typedef std::move_iterator<V*> I;
+    V a1[] = {V(1), V(1), V(1), V(2), V(2), V(2), V(3), V(3), V(3)};
+    M m1(I(a1), I(a1 + sizeof(a1) / sizeof(a1[0])), C(5), A(7));
+    V a2[] = {V(1), V(1), V(1), V(2), V(2), V(2), V(3), V(3), V(3)};
+    M m2(I(a2), I(a2 + sizeof(a2) / sizeof(a2[0])), C(5), A(7));
+    M m3(std::move(m1), A(5));
+    assert(m3 == m2);
+    assert(m3.get_allocator() == A(5));
+    assert(m3.key_comp() == C(5));
+    LIBCPP_ASSERT(m1.empty());
+  }
+  {
+    typedef Counter<int> V;
+    typedef std::less<V> C;
+    typedef test_allocator<V> A;
+    typedef std::set<V, C, A> M;
+    typedef V* I;
+    Counter_base::gConstructed = 0;
     {
-        typedef MoveOnly V;
-        typedef test_less<MoveOnly> C;
-        typedef test_allocator<V> A;
-        typedef std::set<MoveOnly, C, A> M;
-        typedef std::move_iterator<V*> I;
-        V a1[] =
-        {
-            V(1),
-            V(1),
-            V(1),
-            V(2),
-            V(2),
-            V(2),
-            V(3),
-            V(3),
-            V(3)
-        };
-        M m1(I(a1), I(a1+sizeof(a1)/sizeof(a1[0])), C(5), A(7));
-        V a2[] =
-        {
-            V(1),
-            V(1),
-            V(1),
-            V(2),
-            V(2),
-            V(2),
-            V(3),
-            V(3),
-            V(3)
-        };
-        M m2(I(a2), I(a2+sizeof(a2)/sizeof(a2[0])), C(5), A(7));
-        M m3(std::move(m1), A(7));
-        assert(m3 == m2);
-        assert(m3.get_allocator() == A(7));
-        assert(m3.key_comp() == C(5));
-        LIBCPP_ASSERT(m1.empty());
-    }
-    {
-        typedef MoveOnly V;
-        typedef test_less<MoveOnly> C;
-        typedef test_allocator<V> A;
-        typedef std::set<MoveOnly, C, A> M;
-        typedef std::move_iterator<V*> I;
-        V a1[] =
-        {
-            V(1),
-            V(1),
-            V(1),
-            V(2),
-            V(2),
-            V(2),
-            V(3),
-            V(3),
-            V(3)
-        };
-        M m1(I(a1), I(a1+sizeof(a1)/sizeof(a1[0])), C(5), A(7));
-        V a2[] =
-        {
-            V(1),
-            V(1),
-            V(1),
-            V(2),
-            V(2),
-            V(2),
-            V(3),
-            V(3),
-            V(3)
-        };
-        M m2(I(a2), I(a2+sizeof(a2)/sizeof(a2[0])), C(5), A(7));
-        M m3(std::move(m1), A(5));
-        assert(m3 == m2);
-        assert(m3.get_allocator() == A(5));
-        assert(m3.key_comp() == C(5));
-        LIBCPP_ASSERT(m1.empty());
-    }
-    {
-        typedef MoveOnly V;
-        typedef test_less<MoveOnly> C;
-        typedef other_allocator<V> A;
-        typedef std::set<MoveOnly, C, A> M;
-        typedef std::move_iterator<V*> I;
-        V a1[] =
-        {
-            V(1),
-            V(1),
-            V(1),
-            V(2),
-            V(2),
-            V(2),
-            V(3),
-            V(3),
-            V(3)
-        };
-        M m1(I(a1), I(a1+sizeof(a1)/sizeof(a1[0])), C(5), A(7));
-        V a2[] =
-        {
-            V(1),
-            V(1),
-            V(1),
-            V(2),
-            V(2),
-            V(2),
-            V(3),
-            V(3),
-            V(3)
-        };
-        M m2(I(a2), I(a2+sizeof(a2)/sizeof(a2[0])), C(5), A(7));
-        M m3(std::move(m1), A(5));
-        assert(m3 == m2);
-        assert(m3.get_allocator() == A(5));
-        assert(m3.key_comp() == C(5));
-        LIBCPP_ASSERT(m1.empty());
-    }
-    {
-        typedef Counter<int> V;
-        typedef std::less<V> C;
-        typedef test_allocator<V> A;
-        typedef std::set<V, C, A> M;
-        typedef V* I;
-        Counter_base::gConstructed = 0;
-        {
-            V a1[] =
-            {
-            V(1),
-            V(1),
-            V(1),
-            V(2),
-            V(2),
-            V(2),
-            V(3),
-            V(3),
-            V(3)
-            };
-            const std::size_t num = sizeof(a1)/sizeof(a1[0]);
-            assert(Counter_base::gConstructed == num);
+      V a1[]                = {V(1), V(1), V(1), V(2), V(2), V(2), V(3), V(3), V(3)};
+      const std::size_t num = sizeof(a1) / sizeof(a1[0]);
+      assert(Counter_base::gConstructed == num);
 
-            M m1(I(a1), I(a1+num), C(), A());
-            assert(Counter_base::gConstructed == 3+num);
+      M m1(I(a1), I(a1 + num), C(), A());
+      assert(Counter_base::gConstructed == 3 + num);
 
-            M m2(m1);
-            assert(m2 == m1);
-            assert(Counter_base::gConstructed == 6+num);
+      M m2(m1);
+      assert(m2 == m1);
+      assert(Counter_base::gConstructed == 6 + num);
 
-            M m3(std::move(m1), A());
-            assert(m3 == m2);
-            LIBCPP_ASSERT(m1.empty());
-            assert(Counter_base::gConstructed >= (int)(6+num));
-            assert(Counter_base::gConstructed <= (int)(m1.size()+6+num));
+      M m3(std::move(m1), A());
+      assert(m3 == m2);
+      LIBCPP_ASSERT(m1.empty());
+      assert(Counter_base::gConstructed >= (int)(6 + num));
+      assert(Counter_base::gConstructed <= (int)(m1.size() + 6 + num));
 
-            {
-            M m4(std::move(m2), A(5));
-            assert(Counter_base::gConstructed >= (int)(6+num));
-            assert(Counter_base::gConstructed <= (int)(m1.size()+m2.size()+6+num));
-            assert(m4 == m3);
-            LIBCPP_ASSERT(m2.empty());
-            }
-            assert(Counter_base::gConstructed >= (int)(3+num));
-            assert(Counter_base::gConstructed <= (int)(m1.size()+m2.size()+3+num));
-        }
-        assert(Counter_base::gConstructed == 0);
+      {
+        M m4(std::move(m2), A(5));
+        assert(Counter_base::gConstructed >= (int)(6 + num));
+        assert(Counter_base::gConstructed <= (int)(m1.size() + m2.size() + 6 + num));
+        assert(m4 == m3);
+        LIBCPP_ASSERT(m2.empty());
+      }
+      assert(Counter_base::gConstructed >= (int)(3 + num));
+      assert(Counter_base::gConstructed <= (int)(m1.size() + m2.size() + 3 + num));
     }
-
+    assert(Counter_base::gConstructed == 0);
+  }
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/associative/set/set.cons/move_assign.pass.cpp b/libcxx/test/std/containers/associative/set/set.cons/move_assign.pass.cpp
index ea85051aa4cc2..bacd58c172a12 100644
--- a/libcxx/test/std/containers/associative/set/set.cons/move_assign.pass.cpp
+++ b/libcxx/test/std/containers/associative/set/set.cons/move_assign.pass.cpp
@@ -23,164 +23,75 @@
 #include "test_allocator.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef MoveOnly V;
-        typedef test_less<MoveOnly> C;
-        typedef test_allocator<V> A;
-        typedef std::set<MoveOnly, C, A> M;
-        typedef std::move_iterator<V*> I;
-        V a1[] =
-        {
-            V(1),
-            V(1),
-            V(1),
-            V(2),
-            V(2),
-            V(2),
-            V(3),
-            V(3),
-            V(3)
-        };
-        M m1(I(a1), I(a1+sizeof(a1)/sizeof(a1[0])), C(5), A(7));
-        V a2[] =
-        {
-            V(1),
-            V(1),
-            V(1),
-            V(2),
-            V(2),
-            V(2),
-            V(3),
-            V(3),
-            V(3)
-        };
-        M m2(I(a2), I(a2+sizeof(a2)/sizeof(a2[0])), C(5), A(7));
-        M m3(C(3), A(7));
-        m3 = std::move(m1);
-        assert(m3 == m2);
-        assert(m3.get_allocator() == A(7));
-        assert(m3.key_comp() == C(5));
-        assert(m1.empty());
-    }
-    {
-        typedef MoveOnly V;
-        typedef test_less<MoveOnly> C;
-        typedef test_allocator<V> A;
-        typedef std::set<MoveOnly, C, A> M;
-        typedef std::move_iterator<V*> I;
-        V a1[] =
-        {
-            V(1),
-            V(1),
-            V(1),
-            V(2),
-            V(2),
-            V(2),
-            V(3),
-            V(3),
-            V(3)
-        };
-        M m1(I(a1), I(a1+sizeof(a1)/sizeof(a1[0])), C(5), A(7));
-        V a2[] =
-        {
-            V(1),
-            V(1),
-            V(1),
-            V(2),
-            V(2),
-            V(2),
-            V(3),
-            V(3),
-            V(3)
-        };
-        M m2(I(a2), I(a2+sizeof(a2)/sizeof(a2[0])), C(5), A(7));
-        M m3(C(3), A(5));
-        m3 = std::move(m1);
-        assert(m3 == m2);
-        assert(m3.get_allocator() == A(5));
-        assert(m3.key_comp() == C(5));
-        LIBCPP_ASSERT(m1.empty());
-    }
-    {
-        typedef MoveOnly V;
-        typedef test_less<MoveOnly> C;
-        typedef other_allocator<V> A;
-        typedef std::set<MoveOnly, C, A> M;
-        typedef std::move_iterator<V*> I;
-        V a1[] =
-        {
-            V(1),
-            V(1),
-            V(1),
-            V(2),
-            V(2),
-            V(2),
-            V(3),
-            V(3),
-            V(3)
-        };
-        M m1(I(a1), I(a1+sizeof(a1)/sizeof(a1[0])), C(5), A(7));
-        V a2[] =
-        {
-            V(1),
-            V(1),
-            V(1),
-            V(2),
-            V(2),
-            V(2),
-            V(3),
-            V(3),
-            V(3)
-        };
-        M m2(I(a2), I(a2+sizeof(a2)/sizeof(a2[0])), C(5), A(7));
-        M m3(C(3), A(5));
-        m3 = std::move(m1);
-        assert(m3 == m2);
-        assert(m3.get_allocator() == A(7));
-        assert(m3.key_comp() == C(5));
-        assert(m1.empty());
-    }
-    {
-        typedef MoveOnly V;
-        typedef test_less<MoveOnly> C;
-        typedef min_allocator<V> A;
-        typedef std::set<MoveOnly, C, A> M;
-        typedef std::move_iterator<V*> I;
-        V a1[] =
-        {
-            V(1),
-            V(1),
-            V(1),
-            V(2),
-            V(2),
-            V(2),
-            V(3),
-            V(3),
-            V(3)
-        };
-        M m1(I(a1), I(a1+sizeof(a1)/sizeof(a1[0])), C(5), A());
-        V a2[] =
-        {
-            V(1),
-            V(1),
-            V(1),
-            V(2),
-            V(2),
-            V(2),
-            V(3),
-            V(3),
-            V(3)
-        };
-        M m2(I(a2), I(a2+sizeof(a2)/sizeof(a2[0])), C(5), A());
-        M m3(C(3), A());
-        m3 = std::move(m1);
-        assert(m3 == m2);
-        assert(m3.get_allocator() == A());
-        assert(m3.key_comp() == C(5));
-        assert(m1.empty());
-    }
+int main(int, char**) {
+  {
+    typedef MoveOnly V;
+    typedef test_less<MoveOnly> C;
+    typedef test_allocator<V> A;
+    typedef std::set<MoveOnly, C, A> M;
+    typedef std::move_iterator<V*> I;
+    V a1[] = {V(1), V(1), V(1), V(2), V(2), V(2), V(3), V(3), V(3)};
+    M m1(I(a1), I(a1 + sizeof(a1) / sizeof(a1[0])), C(5), A(7));
+    V a2[] = {V(1), V(1), V(1), V(2), V(2), V(2), V(3), V(3), V(3)};
+    M m2(I(a2), I(a2 + sizeof(a2) / sizeof(a2[0])), C(5), A(7));
+    M m3(C(3), A(7));
+    m3 = std::move(m1);
+    assert(m3 == m2);
+    assert(m3.get_allocator() == A(7));
+    assert(m3.key_comp() == C(5));
+    assert(m1.empty());
+  }
+  {
+    typedef MoveOnly V;
+    typedef test_less<MoveOnly> C;
+    typedef test_allocator<V> A;
+    typedef std::set<MoveOnly, C, A> M;
+    typedef std::move_iterator<V*> I;
+    V a1[] = {V(1), V(1), V(1), V(2), V(2), V(2), V(3), V(3), V(3)};
+    M m1(I(a1), I(a1 + sizeof(a1) / sizeof(a1[0])), C(5), A(7));
+    V a2[] = {V(1), V(1), V(1), V(2), V(2), V(2), V(3), V(3), V(3)};
+    M m2(I(a2), I(a2 + sizeof(a2) / sizeof(a2[0])), C(5), A(7));
+    M m3(C(3), A(5));
+    m3 = std::move(m1);
+    assert(m3 == m2);
+    assert(m3.get_allocator() == A(5));
+    assert(m3.key_comp() == C(5));
+    LIBCPP_ASSERT(m1.empty());
+  }
+  {
+    typedef MoveOnly V;
+    typedef test_less<MoveOnly> C;
+    typedef other_allocator<V> A;
+    typedef std::set<MoveOnly, C, A> M;
+    typedef std::move_iterator<V*> I;
+    V a1[] = {V(1), V(1), V(1), V(2), V(2), V(2), V(3), V(3), V(3)};
+    M m1(I(a1), I(a1 + sizeof(a1) / sizeof(a1[0])), C(5), A(7));
+    V a2[] = {V(1), V(1), V(1), V(2), V(2), V(2), V(3), V(3), V(3)};
+    M m2(I(a2), I(a2 + sizeof(a2) / sizeof(a2[0])), C(5), A(7));
+    M m3(C(3), A(5));
+    m3 = std::move(m1);
+    assert(m3 == m2);
+    assert(m3.get_allocator() == A(7));
+    assert(m3.key_comp() == C(5));
+    assert(m1.empty());
+  }
+  {
+    typedef MoveOnly V;
+    typedef test_less<MoveOnly> C;
+    typedef min_allocator<V> A;
+    typedef std::set<MoveOnly, C, A> M;
+    typedef std::move_iterator<V*> I;
+    V a1[] = {V(1), V(1), V(1), V(2), V(2), V(2), V(3), V(3), V(3)};
+    M m1(I(a1), I(a1 + sizeof(a1) / sizeof(a1[0])), C(5), A());
+    V a2[] = {V(1), V(1), V(1), V(2), V(2), V(2), V(3), V(3), V(3)};
+    M m2(I(a2), I(a2 + sizeof(a2) / sizeof(a2[0])), C(5), A());
+    M m3(C(3), A());
+    m3 = std::move(m1);
+    assert(m3 == m2);
+    assert(m3.get_allocator() == A());
+    assert(m3.key_comp() == C(5));
+    assert(m1.empty());
+  }
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/associative/set/set.cons/move_assign_noexcept.compile.pass.cpp b/libcxx/test/std/containers/associative/set/set.cons/move_assign_noexcept.compile.pass.cpp
new file mode 100644
index 0000000000000..f8a0472fb069c
--- /dev/null
+++ b/libcxx/test/std/containers/associative/set/set.cons/move_assign_noexcept.compile.pass.cpp
@@ -0,0 +1,61 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// <set>
+
+// set& operator=(set&& c)
+//     noexcept(
+//          allocator_type::propagate_on_container_move_assignment::value &&
+//          is_nothrow_move_assignable<allocator_type>::value &&
+//          is_nothrow_move_assignable<key_compare>::value);
+
+// This tests a conforming extension
+
+// UNSUPPORTED: c++03
+
+#include <set>
+
+#include "test_macros.h"
+#include "MoveOnly.h"
+#include "test_allocator.h"
+
+template <class T>
+struct some_comp {
+  using value_type = T;
+  some_comp& operator=(const some_comp&);
+  bool operator()(const T&, const T&) const { return false; }
+};
+
+template <class T>
+struct always_equal_alloc {
+  using value_type = T;
+  always_equal_alloc(const always_equal_alloc&);
+  void allocate(std::size_t);
+};
+
+template <class T>
+struct not_always_equal_alloc {
+  int i;
+  using value_type = T;
+  not_always_equal_alloc(const not_always_equal_alloc&);
+  void allocate(std::size_t);
+};
+
+template <template <class> class Alloc>
+using unordered_map_alloc = std::set<MoveOnly, std::less<MoveOnly>, Alloc<MoveOnly>>;
+
+static_assert(std::is_nothrow_move_assignable<unordered_map_alloc<std::allocator>>::value, "");
+static_assert(!std::is_nothrow_move_assignable<unordered_map_alloc<test_allocator>>::value, "");
+#if TEST_STD_VER >= 17
+static_assert(std::is_nothrow_move_assignable<unordered_map_alloc<always_equal_alloc>>::value, "");
+#endif
+static_assert(!std::is_nothrow_move_assignable<unordered_map_alloc<not_always_equal_alloc>>::value, "");
+#if defined(_LIBCPP_VERSION)
+static_assert(std::is_nothrow_move_assignable<unordered_map_alloc<other_allocator>>::value, "");
+#endif // _LIBCPP_VERSION
+static_assert(!std::is_nothrow_move_assignable<std::set<int, some_comp<int>>>::value, "");
diff --git a/libcxx/test/std/containers/associative/set/set.cons/move_assign_noexcept.pass.cpp b/libcxx/test/std/containers/associative/set/set.cons/move_assign_noexcept.pass.cpp
deleted file mode 100644
index 9c177f219dc7b..0000000000000
--- a/libcxx/test/std/containers/associative/set/set.cons/move_assign_noexcept.pass.cpp
+++ /dev/null
@@ -1,58 +0,0 @@
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-// <set>
-
-// set& operator=(set&& c)
-//     noexcept(
-//          allocator_type::propagate_on_container_move_assignment::value &&
-//          is_nothrow_move_assignable<allocator_type>::value &&
-//          is_nothrow_move_assignable<key_compare>::value);
-
-// This tests a conforming extension
-
-// UNSUPPORTED: c++03
-
-#include <set>
-#include <cassert>
-
-#include "test_macros.h"
-#include "MoveOnly.h"
-#include "test_allocator.h"
-
-template <class T>
-struct some_comp
-{
-    typedef T value_type;
-    some_comp& operator=(const some_comp&);
-    bool operator()(const T&, const T&) const { return false; }
-};
-
-int main(int, char**)
-{
-    {
-        typedef std::set<MoveOnly> C;
-        static_assert(std::is_nothrow_move_assignable<C>::value, "");
-    }
-    {
-        typedef std::set<MoveOnly, std::less<MoveOnly>, test_allocator<MoveOnly>> C;
-        static_assert(!std::is_nothrow_move_assignable<C>::value, "");
-    }
-#if defined(_LIBCPP_VERSION)
-    {
-        typedef std::set<MoveOnly, std::less<MoveOnly>, other_allocator<MoveOnly>> C;
-        static_assert(std::is_nothrow_move_assignable<C>::value, "");
-    }
-#endif // _LIBCPP_VERSION
-    {
-        typedef std::set<MoveOnly, some_comp<MoveOnly>> C;
-        static_assert(!std::is_nothrow_move_assignable<C>::value, "");
-    }
-
-  return 0;
-}
diff --git a/libcxx/test/std/containers/associative/set/set.cons/move_noexcept.pass.cpp b/libcxx/test/std/containers/associative/set/set.cons/move_noexcept.pass.cpp
index d99b853a40119..9325b8d1f40de 100644
--- a/libcxx/test/std/containers/associative/set/set.cons/move_noexcept.pass.cpp
+++ b/libcxx/test/std/containers/associative/set/set.cons/move_noexcept.pass.cpp
@@ -24,33 +24,31 @@
 #include "test_allocator.h"
 
 template <class T>
-struct some_comp
-{
-    typedef T value_type;
-    some_comp(const some_comp&);
-    bool operator()(const T&, const T&) const { return false; }
+struct some_comp {
+  typedef T value_type;
+  some_comp(const some_comp&);
+  bool operator()(const T&, const T&) const { return false; }
 };
 
-int main(int, char**)
-{
+int main(int, char**) {
 #if defined(_LIBCPP_VERSION)
-    {
-        typedef std::set<MoveOnly> C;
-        static_assert(std::is_nothrow_move_constructible<C>::value, "");
-    }
-    {
-        typedef std::set<MoveOnly, std::less<MoveOnly>, test_allocator<MoveOnly>> C;
-        static_assert(std::is_nothrow_move_constructible<C>::value, "");
-    }
-    {
-        typedef std::set<MoveOnly, std::less<MoveOnly>, other_allocator<MoveOnly>> C;
-        static_assert(std::is_nothrow_move_constructible<C>::value, "");
-    }
+  {
+    typedef std::set<MoveOnly> C;
+    static_assert(std::is_nothrow_move_constructible<C>::value, "");
+  }
+  {
+    typedef std::set<MoveOnly, std::less<MoveOnly>, test_allocator<MoveOnly>> C;
+    static_assert(std::is_nothrow_move_constructible<C>::value, "");
+  }
+  {
+    typedef std::set<MoveOnly, std::less<MoveOnly>, other_allocator<MoveOnly>> C;
+    static_assert(std::is_nothrow_move_constructible<C>::value, "");
+  }
 #endif // _LIBCPP_VERSION
-    {
-        typedef std::set<MoveOnly, some_comp<MoveOnly>> C;
-        static_assert(!std::is_nothrow_move_constructible<C>::value, "");
-    }
+  {
+    typedef std::set<MoveOnly, some_comp<MoveOnly>> C;
+    static_assert(!std::is_nothrow_move_constructible<C>::value, "");
+  }
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/associative/set/set.erasure/erase_if.pass.cpp b/libcxx/test/std/containers/associative/set/set.erasure/erase_if.pass.cpp
index 2e0f026376ac5..b076b31059610 100644
--- a/libcxx/test/std/containers/associative/set/set.erasure/erase_if.pass.cpp
+++ b/libcxx/test/std/containers/associative/set/set.erasure/erase_if.pass.cpp
@@ -27,41 +27,39 @@ void test0(S s, Pred p, S expected, std::size_t expected_erased_count) {
 }
 
 template <typename S>
-void test()
-{
-    auto is1 = [](auto v) { return v == 1;};
-    auto is2 = [](auto v) { return v == 2;};
-    auto is3 = [](auto v) { return v == 3;};
-    auto is4 = [](auto v) { return v == 4;};
-    auto True  = [](auto) { return true; };
-    auto False = [](auto) { return false; };
+void test() {
+  auto is1   = [](auto v) { return v == 1; };
+  auto is2   = [](auto v) { return v == 2; };
+  auto is3   = [](auto v) { return v == 3; };
+  auto is4   = [](auto v) { return v == 4; };
+  auto True  = [](auto) { return true; };
+  auto False = [](auto) { return false; };
 
-    test0(S(), is1, S(), 0);
+  test0(S(), is1, S(), 0);
 
-    test0(S({1}), is1, S(), 1);
-    test0(S({1}), is2, S({1}), 0);
+  test0(S({1}), is1, S(), 1);
+  test0(S({1}), is2, S({1}), 0);
 
-    test0(S({1, 2}), is1, S({2}), 1);
-    test0(S({1, 2}), is2, S({1}), 1);
-    test0(S({1, 2}), is3, S({1, 2}), 0);
+  test0(S({1, 2}), is1, S({2}), 1);
+  test0(S({1, 2}), is2, S({1}), 1);
+  test0(S({1, 2}), is3, S({1, 2}), 0);
 
-    test0(S({1, 2, 3}), is1, S({2, 3}), 1);
-    test0(S({1, 2, 3}), is2, S({1, 3}), 1);
-    test0(S({1, 2, 3}), is3, S({1, 2}), 1);
-    test0(S({1, 2, 3}), is4, S({1, 2, 3}), 0);
+  test0(S({1, 2, 3}), is1, S({2, 3}), 1);
+  test0(S({1, 2, 3}), is2, S({1, 3}), 1);
+  test0(S({1, 2, 3}), is3, S({1, 2}), 1);
+  test0(S({1, 2, 3}), is4, S({1, 2, 3}), 0);
 
-    test0(S({1, 2, 3}), True, S(), 3);
-    test0(S({1, 2, 3}), False, S({1, 2, 3}), 0);
+  test0(S({1, 2, 3}), True, S(), 3);
+  test0(S({1, 2, 3}), False, S({1, 2, 3}), 0);
 }
 
-int main(int, char**)
-{
-    test<std::set<int>>();
-    test<std::set<int, std::less<int>, min_allocator<int>>> ();
-    test<std::set<int, std::less<int>, test_allocator<int>>> ();
+int main(int, char**) {
+  test<std::set<int>>();
+  test<std::set<int, std::less<int>, min_allocator<int>>>();
+  test<std::set<int, std::less<int>, test_allocator<int>>>();
 
-    test<std::set<long>>();
-    test<std::set<double>>();
+  test<std::set<long>>();
+  test<std::set<double>>();
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/associative/set/set.nonmember/op_compare.pass.cpp b/libcxx/test/std/containers/associative/set/set.nonmember/op_compare.pass.cpp
index cc427fdd85a0f..1be07ce885298 100644
--- a/libcxx/test/std/containers/associative/set/set.nonmember/op_compare.pass.cpp
+++ b/libcxx/test/std/containers/associative/set/set.nonmember/op_compare.pass.cpp
@@ -39,27 +39,27 @@
 #include "test_comparisons.h"
 
 int main(int, char**) {
-    {
-        std::set<int> s1, s2;
-        s1.insert(1);
-        s2.insert(2);
-        const std::set<int>& cs1 = s1, cs2 = s2;
-        assert(testComparisons(cs1, cs2, false, true));
-    }
-    {
-        std::set<int> s1, s2;
-        s1.insert(1);
-        s2.insert(1);
-        const std::set<int>& cs1 = s1, cs2 = s2;
-        assert(testComparisons(cs1, cs2, true, false));
-    }
-    {
-        std::set<int> s1, s2;
-        s1.insert(1);
-        s2.insert(1);
-        s2.insert(2);
-        const std::set<int>& cs1 = s1, cs2 = s2;
-        assert(testComparisons(cs1, cs2, false, true));
-    }
-    return 0;
+  {
+    std::set<int> s1, s2;
+    s1.insert(1);
+    s2.insert(2);
+    const std::set<int>&cs1 = s1, cs2 = s2;
+    assert(testComparisons(cs1, cs2, false, true));
+  }
+  {
+    std::set<int> s1, s2;
+    s1.insert(1);
+    s2.insert(1);
+    const std::set<int>&cs1 = s1, cs2 = s2;
+    assert(testComparisons(cs1, cs2, true, false));
+  }
+  {
+    std::set<int> s1, s2;
+    s1.insert(1);
+    s2.insert(1);
+    s2.insert(2);
+    const std::set<int>&cs1 = s1, cs2 = s2;
+    assert(testComparisons(cs1, cs2, false, true));
+  }
+  return 0;
 }
diff --git a/libcxx/test/std/containers/associative/set/set.observers/comp.pass.cpp b/libcxx/test/std/containers/associative/set/set.observers/comp.pass.cpp
index 6f573a50df3d8..019dae3955acb 100644
--- a/libcxx/test/std/containers/associative/set/set.observers/comp.pass.cpp
+++ b/libcxx/test/std/containers/associative/set/set.observers/comp.pass.cpp
@@ -15,19 +15,19 @@
 #include <cassert>
 
 int main(int, char**) {
-    typedef std::set<int> set_type;
+  typedef std::set<int> set_type;
 
-    set_type s;
-    std::pair<set_type::iterator, bool> p1 = s.insert(1);
-    std::pair<set_type::iterator, bool> p2 = s.insert(2);
+  set_type s;
+  std::pair<set_type::iterator, bool> p1 = s.insert(1);
+  std::pair<set_type::iterator, bool> p2 = s.insert(2);
 
-    const set_type& cs = s;
+  const set_type& cs = s;
 
-    assert(cs.key_comp()(*p1.first, *p2.first));
-    assert(!cs.key_comp()(*p2.first, *p1.first));
+  assert(cs.key_comp()(*p1.first, *p2.first));
+  assert(!cs.key_comp()(*p2.first, *p1.first));
 
-    assert(cs.value_comp()(*p1.first, *p2.first));
-    assert(!cs.value_comp()(*p2.first, *p1.first));
+  assert(cs.value_comp()(*p1.first, *p2.first));
+  assert(!cs.value_comp()(*p2.first, *p1.first));
 
-    return 0;
+  return 0;
 }
diff --git a/libcxx/test/std/containers/associative/set/set.special/member_swap.pass.cpp b/libcxx/test/std/containers/associative/set/set.special/member_swap.pass.cpp
index ee91ca3bcc34e..db3dc9831b3dd 100644
--- a/libcxx/test/std/containers/associative/set/set.special/member_swap.pass.cpp
+++ b/libcxx/test/std/containers/associative/set/set.special/member_swap.pass.cpp
@@ -18,161 +18,96 @@
 #include "test_macros.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
+int main(int, char**) {
+  {
     typedef int V;
     typedef std::set<int> M;
     {
-        M m1;
-        M m2;
-        M m1_save = m1;
-        M m2_save = m2;
-        m1.swap(m2);
-        assert(m1 == m2_save);
-        assert(m2 == m1_save);
+      M m1;
+      M m2;
+      M m1_save = m1;
+      M m2_save = m2;
+      m1.swap(m2);
+      assert(m1 == m2_save);
+      assert(m2 == m1_save);
     }
     {
-        V ar2[] =
-        {
-            5,
-            6,
-            7,
-            8,
-            9,
-            10,
-            11,
-            12
-        };
-        M m1;
-        M m2(ar2, ar2+sizeof(ar2)/sizeof(ar2[0]));
-        M m1_save = m1;
-        M m2_save = m2;
-        m1.swap(m2);
-        assert(m1 == m2_save);
-        assert(m2 == m1_save);
+      V ar2[] = {5, 6, 7, 8, 9, 10, 11, 12};
+      M m1;
+      M m2(ar2, ar2 + sizeof(ar2) / sizeof(ar2[0]));
+      M m1_save = m1;
+      M m2_save = m2;
+      m1.swap(m2);
+      assert(m1 == m2_save);
+      assert(m2 == m1_save);
     }
     {
-        V ar1[] =
-        {
-            1,
-            2,
-            3,
-            4
-        };
-        M m1(ar1, ar1+sizeof(ar1)/sizeof(ar1[0]));
-        M m2;
-        M m1_save = m1;
-        M m2_save = m2;
-        m1.swap(m2);
-        assert(m1 == m2_save);
-        assert(m2 == m1_save);
+      V ar1[] = {1, 2, 3, 4};
+      M m1(ar1, ar1 + sizeof(ar1) / sizeof(ar1[0]));
+      M m2;
+      M m1_save = m1;
+      M m2_save = m2;
+      m1.swap(m2);
+      assert(m1 == m2_save);
+      assert(m2 == m1_save);
     }
     {
-        V ar1[] =
-        {
-            1,
-            2,
-            3,
-            4
-        };
-        V ar2[] =
-        {
-            5,
-            6,
-            7,
-            8,
-            9,
-            10,
-            11,
-            12
-        };
-        M m1(ar1, ar1+sizeof(ar1)/sizeof(ar1[0]));
-        M m2(ar2, ar2+sizeof(ar2)/sizeof(ar2[0]));
-        M m1_save = m1;
-        M m2_save = m2;
-        m1.swap(m2);
-        assert(m1 == m2_save);
-        assert(m2 == m1_save);
-    }
+      V ar1[] = {1, 2, 3, 4};
+      V ar2[] = {5, 6, 7, 8, 9, 10, 11, 12};
+      M m1(ar1, ar1 + sizeof(ar1) / sizeof(ar1[0]));
+      M m2(ar2, ar2 + sizeof(ar2) / sizeof(ar2[0]));
+      M m1_save = m1;
+      M m2_save = m2;
+      m1.swap(m2);
+      assert(m1 == m2_save);
+      assert(m2 == m1_save);
     }
+  }
 #if TEST_STD_VER >= 11
-    {
+  {
     typedef int V;
     typedef std::set<int, std::less<int>, min_allocator<int>> M;
     {
-        M m1;
-        M m2;
-        M m1_save = m1;
-        M m2_save = m2;
-        m1.swap(m2);
-        assert(m1 == m2_save);
-        assert(m2 == m1_save);
+      M m1;
+      M m2;
+      M m1_save = m1;
+      M m2_save = m2;
+      m1.swap(m2);
+      assert(m1 == m2_save);
+      assert(m2 == m1_save);
     }
     {
-        V ar2[] =
-        {
-            5,
-            6,
-            7,
-            8,
-            9,
-            10,
-            11,
-            12
-        };
-        M m1;
-        M m2(ar2, ar2+sizeof(ar2)/sizeof(ar2[0]));
-        M m1_save = m1;
-        M m2_save = m2;
-        m1.swap(m2);
-        assert(m1 == m2_save);
-        assert(m2 == m1_save);
+      V ar2[] = {5, 6, 7, 8, 9, 10, 11, 12};
+      M m1;
+      M m2(ar2, ar2 + sizeof(ar2) / sizeof(ar2[0]));
+      M m1_save = m1;
+      M m2_save = m2;
+      m1.swap(m2);
+      assert(m1 == m2_save);
+      assert(m2 == m1_save);
     }
     {
-        V ar1[] =
-        {
-            1,
-            2,
-            3,
-            4
-        };
-        M m1(ar1, ar1+sizeof(ar1)/sizeof(ar1[0]));
-        M m2;
-        M m1_save = m1;
-        M m2_save = m2;
-        m1.swap(m2);
-        assert(m1 == m2_save);
-        assert(m2 == m1_save);
+      V ar1[] = {1, 2, 3, 4};
+      M m1(ar1, ar1 + sizeof(ar1) / sizeof(ar1[0]));
+      M m2;
+      M m1_save = m1;
+      M m2_save = m2;
+      m1.swap(m2);
+      assert(m1 == m2_save);
+      assert(m2 == m1_save);
     }
     {
-        V ar1[] =
-        {
-            1,
-            2,
-            3,
-            4
-        };
-        V ar2[] =
-        {
-            5,
-            6,
-            7,
-            8,
-            9,
-            10,
-            11,
-            12
-        };
-        M m1(ar1, ar1+sizeof(ar1)/sizeof(ar1[0]));
-        M m2(ar2, ar2+sizeof(ar2)/sizeof(ar2[0]));
-        M m1_save = m1;
-        M m2_save = m2;
-        m1.swap(m2);
-        assert(m1 == m2_save);
-        assert(m2 == m1_save);
-    }
+      V ar1[] = {1, 2, 3, 4};
+      V ar2[] = {5, 6, 7, 8, 9, 10, 11, 12};
+      M m1(ar1, ar1 + sizeof(ar1) / sizeof(ar1[0]));
+      M m2(ar2, ar2 + sizeof(ar2) / sizeof(ar2[0]));
+      M m1_save = m1;
+      M m2_save = m2;
+      m1.swap(m2);
+      assert(m1 == m2_save);
+      assert(m2 == m1_save);
     }
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/associative/set/set.special/non_member_swap.pass.cpp b/libcxx/test/std/containers/associative/set/set.special/non_member_swap.pass.cpp
index 2c54ecbf4cee3..e663a67035ff5 100644
--- a/libcxx/test/std/containers/associative/set/set.special/non_member_swap.pass.cpp
+++ b/libcxx/test/std/containers/associative/set/set.special/non_member_swap.pass.cpp
@@ -18,152 +18,87 @@
 #include "test_allocator.h"
 #include "../../../test_compare.h"
 
-int main(int, char**)
-{
-    typedef int V;
-    {
+int main(int, char**) {
+  typedef int V;
+  {
     typedef std::set<int> M;
     {
-        M m1;
-        M m2;
-        M m1_save = m1;
-        M m2_save = m2;
-        swap(m1, m2);
-        assert(m1 == m2_save);
-        assert(m2 == m1_save);
-    }
-    {
-        V ar2[] =
-        {
-            5,
-            6,
-            7,
-            8,
-            9,
-            10,
-            11,
-            12
-        };
-        M m1;
-        M m2(ar2, ar2+sizeof(ar2)/sizeof(ar2[0]));
-        M m1_save = m1;
-        M m2_save = m2;
-        swap(m1, m2);
-        assert(m1 == m2_save);
-        assert(m2 == m1_save);
+      M m1;
+      M m2;
+      M m1_save = m1;
+      M m2_save = m2;
+      swap(m1, m2);
+      assert(m1 == m2_save);
+      assert(m2 == m1_save);
     }
     {
-        V ar1[] =
-        {
-            1,
-            2,
-            3,
-            4
-        };
-        M m1(ar1, ar1+sizeof(ar1)/sizeof(ar1[0]));
-        M m2;
-        M m1_save = m1;
-        M m2_save = m2;
-        swap(m1, m2);
-        assert(m1 == m2_save);
-        assert(m2 == m1_save);
-    }
-    {
-        V ar1[] =
-        {
-            1,
-            2,
-            3,
-            4
-        };
-        V ar2[] =
-        {
-            5,
-            6,
-            7,
-            8,
-            9,
-            10,
-            11,
-            12
-        };
-        M m1(ar1, ar1+sizeof(ar1)/sizeof(ar1[0]));
-        M m2(ar2, ar2+sizeof(ar2)/sizeof(ar2[0]));
-        M m1_save = m1;
-        M m2_save = m2;
-        swap(m1, m2);
-        assert(m1 == m2_save);
-        assert(m2 == m1_save);
-    }
+      V ar2[] = {5, 6, 7, 8, 9, 10, 11, 12};
+      M m1;
+      M m2(ar2, ar2 + sizeof(ar2) / sizeof(ar2[0]));
+      M m1_save = m1;
+      M m2_save = m2;
+      swap(m1, m2);
+      assert(m1 == m2_save);
+      assert(m2 == m1_save);
     }
     {
-        typedef test_allocator<V> A;
-        typedef test_less<int> C;
-        typedef std::set<int, C, A> M;
-        V ar1[] =
-        {
-            1,
-            2,
-            3,
-            4
-        };
-        V ar2[] =
-        {
-            5,
-            6,
-            7,
-            8,
-            9,
-            10,
-            11,
-            12
-        };
-        M m1(ar1, ar1+sizeof(ar1)/sizeof(ar1[0]), C(1), A(1, 1));
-        M m2(ar2, ar2+sizeof(ar2)/sizeof(ar2[0]), C(2), A(1, 2));
-        M m1_save = m1;
-        M m2_save = m2;
-        swap(m1, m2);
-        assert(m1 == m2_save);
-        assert(m2 == m1_save);
-        assert(m1.key_comp() == C(2));
-        assert(m1.get_allocator().get_id() == 1);
-        assert(m2.key_comp() == C(1));
-        assert(m2.get_allocator().get_id() == 2);
+      V ar1[] = {1, 2, 3, 4};
+      M m1(ar1, ar1 + sizeof(ar1) / sizeof(ar1[0]));
+      M m2;
+      M m1_save = m1;
+      M m2_save = m2;
+      swap(m1, m2);
+      assert(m1 == m2_save);
+      assert(m2 == m1_save);
     }
     {
-        typedef other_allocator<V> A;
-        typedef test_less<int> C;
-        typedef std::set<int, C, A> M;
-        V ar1[] =
-        {
-            1,
-            2,
-            3,
-            4
-        };
-        V ar2[] =
-        {
-            5,
-            6,
-            7,
-            8,
-            9,
-            10,
-            11,
-            12
-        };
-        M m1(ar1, ar1+sizeof(ar1)/sizeof(ar1[0]), C(1), A(1));
-        M m2(ar2, ar2+sizeof(ar2)/sizeof(ar2[0]), C(2), A(2));
-        M m1_save = m1;
-        M m2_save = m2;
-        swap(m1, m2);
-        assert(m1 == m2_save);
-        assert(m2 == m1_save);
-        assert(m1.key_comp() == C(2));
-        assert(m1.get_allocator() == A(2));
-        assert(m2.key_comp() == C(1));
-        assert(m2.get_allocator() == A(1));
+      V ar1[] = {1, 2, 3, 4};
+      V ar2[] = {5, 6, 7, 8, 9, 10, 11, 12};
+      M m1(ar1, ar1 + sizeof(ar1) / sizeof(ar1[0]));
+      M m2(ar2, ar2 + sizeof(ar2) / sizeof(ar2[0]));
+      M m1_save = m1;
+      M m2_save = m2;
+      swap(m1, m2);
+      assert(m1 == m2_save);
+      assert(m2 == m1_save);
     }
+  }
+  {
+    typedef test_allocator<V> A;
+    typedef test_less<int> C;
+    typedef std::set<int, C, A> M;
+    V ar1[] = {1, 2, 3, 4};
+    V ar2[] = {5, 6, 7, 8, 9, 10, 11, 12};
+    M m1(ar1, ar1 + sizeof(ar1) / sizeof(ar1[0]), C(1), A(1, 1));
+    M m2(ar2, ar2 + sizeof(ar2) / sizeof(ar2[0]), C(2), A(1, 2));
+    M m1_save = m1;
+    M m2_save = m2;
+    swap(m1, m2);
+    assert(m1 == m2_save);
+    assert(m2 == m1_save);
+    assert(m1.key_comp() == C(2));
+    assert(m1.get_allocator().get_id() == 1);
+    assert(m2.key_comp() == C(1));
+    assert(m2.get_allocator().get_id() == 2);
+  }
+  {
+    typedef other_allocator<V> A;
+    typedef test_less<int> C;
+    typedef std::set<int, C, A> M;
+    V ar1[] = {1, 2, 3, 4};
+    V ar2[] = {5, 6, 7, 8, 9, 10, 11, 12};
+    M m1(ar1, ar1 + sizeof(ar1) / sizeof(ar1[0]), C(1), A(1));
+    M m2(ar2, ar2 + sizeof(ar2) / sizeof(ar2[0]), C(2), A(2));
+    M m1_save = m1;
+    M m2_save = m2;
+    swap(m1, m2);
+    assert(m1 == m2_save);
+    assert(m2 == m1_save);
+    assert(m1.key_comp() == C(2));
+    assert(m1.get_allocator() == A(2));
+    assert(m2.key_comp() == C(1));
+    assert(m2.get_allocator() == A(1));
+  }
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/associative/set/set.special/swap_noexcept.pass.cpp b/libcxx/test/std/containers/associative/set/set.special/swap_noexcept.pass.cpp
index b04c77b8a8642..36ab989b76043 100644
--- a/libcxx/test/std/containers/associative/set/set.special/swap_noexcept.pass.cpp
+++ b/libcxx/test/std/containers/associative/set/set.special/swap_noexcept.pass.cpp
@@ -29,23 +29,21 @@
 #include "test_allocator.h"
 
 template <class T>
-struct some_comp
-{
-    typedef T value_type;
+struct some_comp {
+  typedef T value_type;
 
-    some_comp() {}
-    some_comp(const some_comp&) {}
-    bool operator()(const T&, const T&) const { return false; }
+  some_comp() {}
+  some_comp(const some_comp&) {}
+  bool operator()(const T&, const T&) const { return false; }
 };
 
 template <class T>
-struct some_comp2
-{
-    typedef T value_type;
+struct some_comp2 {
+  typedef T value_type;
 
-    some_comp2() {}
-    some_comp2(const some_comp2&) {}
-    bool operator()(const T&, const T&) const { return false; }
+  some_comp2() {}
+  some_comp2(const some_comp2&) {}
+  bool operator()(const T&, const T&) const { return false; }
 };
 
 #if TEST_STD_VER >= 14
@@ -54,89 +52,84 @@ void swap(some_comp2<T>&, some_comp2<T>&) noexcept {}
 #endif
 
 template <class T>
-struct some_alloc
-{
-    typedef T value_type;
+struct some_alloc {
+  typedef T value_type;
 
-    some_alloc() {}
-    some_alloc(const some_alloc&);
-    void deallocate(void*, unsigned) {}
+  some_alloc() {}
+  some_alloc(const some_alloc&);
+  void deallocate(void*, unsigned) {}
 
-    typedef std::true_type propagate_on_container_swap;
+  typedef std::true_type propagate_on_container_swap;
 };
 
 template <class T>
-struct some_alloc2
-{
-    typedef T value_type;
+struct some_alloc2 {
+  typedef T value_type;
 
-    some_alloc2() {}
-    some_alloc2(const some_alloc2&);
-    void deallocate(void*, unsigned) {}
+  some_alloc2() {}
+  some_alloc2(const some_alloc2&);
+  void deallocate(void*, unsigned) {}
 
-    typedef std::false_type propagate_on_container_swap;
-    typedef std::true_type is_always_equal;
+  typedef std::false_type propagate_on_container_swap;
+  typedef std::true_type is_always_equal;
 };
 
 template <class T>
-struct some_alloc3
-{
-    typedef T value_type;
+struct some_alloc3 {
+  typedef T value_type;
 
-    some_alloc3() {}
-    some_alloc3(const some_alloc3&);
-    void deallocate(void*, unsigned) {}
+  some_alloc3() {}
+  some_alloc3(const some_alloc3&);
+  void deallocate(void*, unsigned) {}
 
-    typedef std::false_type propagate_on_container_swap;
-    typedef std::false_type is_always_equal;
+  typedef std::false_type propagate_on_container_swap;
+  typedef std::false_type is_always_equal;
 };
 
-int main(int, char**)
-{
-    {
-        typedef std::set<MoveOnly> C;
-        static_assert(noexcept(swap(std::declval<C&>(), std::declval<C&>())), "");
-    }
+int main(int, char**) {
+  {
+    typedef std::set<MoveOnly> C;
+    static_assert(noexcept(swap(std::declval<C&>(), std::declval<C&>())), "");
+  }
 #if defined(_LIBCPP_VERSION)
-    {
-        typedef std::set<MoveOnly, std::less<MoveOnly>, test_allocator<MoveOnly>> C;
-        static_assert(noexcept(swap(std::declval<C&>(), std::declval<C&>())), "");
-    }
-    {
-        typedef std::set<MoveOnly, std::less<MoveOnly>, other_allocator<MoveOnly>> C;
-        static_assert(noexcept(swap(std::declval<C&>(), std::declval<C&>())), "");
-    }
+  {
+    typedef std::set<MoveOnly, std::less<MoveOnly>, test_allocator<MoveOnly>> C;
+    static_assert(noexcept(swap(std::declval<C&>(), std::declval<C&>())), "");
+  }
+  {
+    typedef std::set<MoveOnly, std::less<MoveOnly>, other_allocator<MoveOnly>> C;
+    static_assert(noexcept(swap(std::declval<C&>(), std::declval<C&>())), "");
+  }
 #endif // _LIBCPP_VERSION
-    {
-        typedef std::set<MoveOnly, some_comp<MoveOnly>> C;
-        static_assert(!noexcept(swap(std::declval<C&>(), std::declval<C&>())), "");
-    }
+  {
+    typedef std::set<MoveOnly, some_comp<MoveOnly>> C;
+    static_assert(!noexcept(swap(std::declval<C&>(), std::declval<C&>())), "");
+  }
 
 #if TEST_STD_VER >= 14
-    { // POCS allocator, throwable swap for comp
-    typedef std::set<MoveOnly, some_comp <MoveOnly>, some_alloc <MoveOnly>> C;
+  { // POCS allocator, throwable swap for comp
+    typedef std::set<MoveOnly, some_comp<MoveOnly>, some_alloc<MoveOnly>> C;
     static_assert(!noexcept(swap(std::declval<C&>(), std::declval<C&>())), "");
-    }
-    { // always equal allocator, throwable swap for comp
-    typedef std::set<MoveOnly, some_comp <MoveOnly>, some_alloc2<MoveOnly>> C;
+  }
+  { // always equal allocator, throwable swap for comp
+    typedef std::set<MoveOnly, some_comp<MoveOnly>, some_alloc2<MoveOnly>> C;
     static_assert(!noexcept(swap(std::declval<C&>(), std::declval<C&>())), "");
-    }
-    { // POCS allocator, nothrow swap for comp
-    typedef std::set<MoveOnly, some_comp2<MoveOnly>, some_alloc <MoveOnly>> C;
-    static_assert( noexcept(swap(std::declval<C&>(), std::declval<C&>())), "");
-    }
-    { // always equal allocator, nothrow swap for comp
+  }
+  { // POCS allocator, nothrow swap for comp
+    typedef std::set<MoveOnly, some_comp2<MoveOnly>, some_alloc<MoveOnly>> C;
+    static_assert(noexcept(swap(std::declval<C&>(), std::declval<C&>())), "");
+  }
+  { // always equal allocator, nothrow swap for comp
     typedef std::set<MoveOnly, some_comp2<MoveOnly>, some_alloc2<MoveOnly>> C;
-    static_assert( noexcept(swap(std::declval<C&>(), std::declval<C&>())), "");
-    }
-#if defined(_LIBCPP_VERSION)
-    { // NOT always equal allocator, nothrow swap for comp
+    static_assert(noexcept(swap(std::declval<C&>(), std::declval<C&>())), "");
+  }
+#  if defined(_LIBCPP_VERSION)
+  { // NOT always equal allocator, nothrow swap for comp
     typedef std::set<MoveOnly, some_comp2<MoveOnly>, some_alloc3<MoveOnly>> C;
-    static_assert( noexcept(swap(std::declval<C&>(), std::declval<C&>())), "");
-    }
-#endif // _LIBCPP_VERSION
+    static_assert(noexcept(swap(std::declval<C&>(), std::declval<C&>())), "");
+  }
+#  endif // _LIBCPP_VERSION
 #endif
 
-
   return 0;
 }
diff --git a/libcxx/test/std/containers/associative/set/size.pass.cpp b/libcxx/test/std/containers/associative/set/size.pass.cpp
index bdce58e98f3fe..1672e1f055c6c 100644
--- a/libcxx/test/std/containers/associative/set/size.pass.cpp
+++ b/libcxx/test/std/containers/associative/set/size.pass.cpp
@@ -18,9 +18,8 @@
 #include "test_macros.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
+int main(int, char**) {
+  {
     typedef std::set<int> M;
     M m;
     assert(m.size() == 0);
@@ -36,9 +35,9 @@ int main(int, char**)
     assert(m.size() == 1);
     m.erase(m.begin());
     assert(m.size() == 0);
-    }
+  }
 #if TEST_STD_VER >= 11
-    {
+  {
     typedef std::set<int, std::less<int>, min_allocator<int>> M;
     M m;
     assert(m.size() == 0);
@@ -54,7 +53,7 @@ int main(int, char**)
     assert(m.size() == 1);
     m.erase(m.begin());
     assert(m.size() == 0);
-    }
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/associative/set/types.pass.cpp b/libcxx/test/std/containers/associative/set/types.pass.cpp
index 2d29a800f9c39..a7c30244a540b 100644
--- a/libcxx/test/std/containers/associative/set/types.pass.cpp
+++ b/libcxx/test/std/containers/associative/set/types.pass.cpp
@@ -34,9 +34,8 @@
 #include "test_macros.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
+int main(int, char**) {
+  {
     typedef std::set<int> C;
     static_assert((std::is_same<C::key_type, int>::value), "");
     static_assert((std::is_same<C::value_type, int>::value), "");
@@ -49,9 +48,9 @@ int main(int, char**)
     static_assert((std::is_same<C::const_pointer, const int*>::value), "");
     static_assert((std::is_same<C::size_type, std::size_t>::value), "");
     static_assert((std::is_same<C::difference_type, std::ptrdiff_t>::value), "");
-    }
+  }
 #if TEST_STD_VER >= 11
-    {
+  {
     typedef std::set<int, std::less<int>, min_allocator<int>> C;
     static_assert((std::is_same<C::key_type, int>::value), "");
     static_assert((std::is_same<C::value_type, int>::value), "");
@@ -62,10 +61,10 @@ int main(int, char**)
     static_assert((std::is_same<C::const_reference, const int&>::value), "");
     static_assert((std::is_same<C::pointer, min_pointer<int>>::value), "");
     static_assert((std::is_same<C::const_pointer, min_pointer<const int>>::value), "");
-//  min_allocator doesn't have a size_type, so one gets synthesized
+    //  min_allocator doesn't have a size_type, so one gets synthesized
     static_assert((std::is_same<C::size_type, std::make_unsigned<C::difference_type>::type>::value), "");
     static_assert((std::is_same<C::difference_type, std::ptrdiff_t>::value), "");
-    }
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/associative/set/upper_bound.pass.cpp b/libcxx/test/std/containers/associative/set/upper_bound.pass.cpp
index 9dec9507ebae7..3543879e07940 100644
--- a/libcxx/test/std/containers/associative/set/upper_bound.pass.cpp
+++ b/libcxx/test/std/containers/associative/set/upper_bound.pass.cpp
@@ -20,232 +20,181 @@
 #include "min_allocator.h"
 #include "private_constructor.h"
 
-int main(int, char**)
-{
-    {
+int main(int, char**) {
+  {
     typedef int V;
     typedef std::set<int> M;
     {
-        typedef M::iterator R;
-        V ar[] =
-        {
-            5,
-            7,
-            9,
-            11,
-            13,
-            15,
-            17,
-            19
-        };
-        M m(ar, ar+sizeof(ar)/sizeof(ar[0]));
-        R r = m.upper_bound(5);
-        assert(r == std::next(m.begin(), 1));
-        r = m.upper_bound(7);
-        assert(r == std::next(m.begin(), 2));
-        r = m.upper_bound(9);
-        assert(r == std::next(m.begin(), 3));
-        r = m.upper_bound(11);
-        assert(r == std::next(m.begin(), 4));
-        r = m.upper_bound(13);
-        assert(r == std::next(m.begin(), 5));
-        r = m.upper_bound(15);
-        assert(r == std::next(m.begin(), 6));
-        r = m.upper_bound(17);
-        assert(r == std::next(m.begin(), 7));
-        r = m.upper_bound(19);
-        assert(r == std::next(m.begin(), 8));
-        r = m.upper_bound(4);
-        assert(r == std::next(m.begin(), 0));
-        r = m.upper_bound(6);
-        assert(r == std::next(m.begin(), 1));
-        r = m.upper_bound(8);
-        assert(r == std::next(m.begin(), 2));
-        r = m.upper_bound(10);
-        assert(r == std::next(m.begin(), 3));
-        r = m.upper_bound(12);
-        assert(r == std::next(m.begin(), 4));
-        r = m.upper_bound(14);
-        assert(r == std::next(m.begin(), 5));
-        r = m.upper_bound(16);
-        assert(r == std::next(m.begin(), 6));
-        r = m.upper_bound(18);
-        assert(r == std::next(m.begin(), 7));
-        r = m.upper_bound(20);
-        assert(r == std::next(m.begin(), 8));
+      typedef M::iterator R;
+      V ar[] = {5, 7, 9, 11, 13, 15, 17, 19};
+      M m(ar, ar + sizeof(ar) / sizeof(ar[0]));
+      R r = m.upper_bound(5);
+      assert(r == std::next(m.begin(), 1));
+      r = m.upper_bound(7);
+      assert(r == std::next(m.begin(), 2));
+      r = m.upper_bound(9);
+      assert(r == std::next(m.begin(), 3));
+      r = m.upper_bound(11);
+      assert(r == std::next(m.begin(), 4));
+      r = m.upper_bound(13);
+      assert(r == std::next(m.begin(), 5));
+      r = m.upper_bound(15);
+      assert(r == std::next(m.begin(), 6));
+      r = m.upper_bound(17);
+      assert(r == std::next(m.begin(), 7));
+      r = m.upper_bound(19);
+      assert(r == std::next(m.begin(), 8));
+      r = m.upper_bound(4);
+      assert(r == std::next(m.begin(), 0));
+      r = m.upper_bound(6);
+      assert(r == std::next(m.begin(), 1));
+      r = m.upper_bound(8);
+      assert(r == std::next(m.begin(), 2));
+      r = m.upper_bound(10);
+      assert(r == std::next(m.begin(), 3));
+      r = m.upper_bound(12);
+      assert(r == std::next(m.begin(), 4));
+      r = m.upper_bound(14);
+      assert(r == std::next(m.begin(), 5));
+      r = m.upper_bound(16);
+      assert(r == std::next(m.begin(), 6));
+      r = m.upper_bound(18);
+      assert(r == std::next(m.begin(), 7));
+      r = m.upper_bound(20);
+      assert(r == std::next(m.begin(), 8));
     }
     {
-        typedef M::const_iterator R;
-        V ar[] =
-        {
-            5,
-            7,
-            9,
-            11,
-            13,
-            15,
-            17,
-            19
-        };
-        const M m(ar, ar+sizeof(ar)/sizeof(ar[0]));
-        R r = m.upper_bound(5);
-        assert(r == std::next(m.begin(), 1));
-        r = m.upper_bound(7);
-        assert(r == std::next(m.begin(), 2));
-        r = m.upper_bound(9);
-        assert(r == std::next(m.begin(), 3));
-        r = m.upper_bound(11);
-        assert(r == std::next(m.begin(), 4));
-        r = m.upper_bound(13);
-        assert(r == std::next(m.begin(), 5));
-        r = m.upper_bound(15);
-        assert(r == std::next(m.begin(), 6));
-        r = m.upper_bound(17);
-        assert(r == std::next(m.begin(), 7));
-        r = m.upper_bound(19);
-        assert(r == std::next(m.begin(), 8));
-        r = m.upper_bound(4);
-        assert(r == std::next(m.begin(), 0));
-        r = m.upper_bound(6);
-        assert(r == std::next(m.begin(), 1));
-        r = m.upper_bound(8);
-        assert(r == std::next(m.begin(), 2));
-        r = m.upper_bound(10);
-        assert(r == std::next(m.begin(), 3));
-        r = m.upper_bound(12);
-        assert(r == std::next(m.begin(), 4));
-        r = m.upper_bound(14);
-        assert(r == std::next(m.begin(), 5));
-        r = m.upper_bound(16);
-        assert(r == std::next(m.begin(), 6));
-        r = m.upper_bound(18);
-        assert(r == std::next(m.begin(), 7));
-        r = m.upper_bound(20);
-        assert(r == std::next(m.begin(), 8));
-    }
+      typedef M::const_iterator R;
+      V ar[] = {5, 7, 9, 11, 13, 15, 17, 19};
+      const M m(ar, ar + sizeof(ar) / sizeof(ar[0]));
+      R r = m.upper_bound(5);
+      assert(r == std::next(m.begin(), 1));
+      r = m.upper_bound(7);
+      assert(r == std::next(m.begin(), 2));
+      r = m.upper_bound(9);
+      assert(r == std::next(m.begin(), 3));
+      r = m.upper_bound(11);
+      assert(r == std::next(m.begin(), 4));
+      r = m.upper_bound(13);
+      assert(r == std::next(m.begin(), 5));
+      r = m.upper_bound(15);
+      assert(r == std::next(m.begin(), 6));
+      r = m.upper_bound(17);
+      assert(r == std::next(m.begin(), 7));
+      r = m.upper_bound(19);
+      assert(r == std::next(m.begin(), 8));
+      r = m.upper_bound(4);
+      assert(r == std::next(m.begin(), 0));
+      r = m.upper_bound(6);
+      assert(r == std::next(m.begin(), 1));
+      r = m.upper_bound(8);
+      assert(r == std::next(m.begin(), 2));
+      r = m.upper_bound(10);
+      assert(r == std::next(m.begin(), 3));
+      r = m.upper_bound(12);
+      assert(r == std::next(m.begin(), 4));
+      r = m.upper_bound(14);
+      assert(r == std::next(m.begin(), 5));
+      r = m.upper_bound(16);
+      assert(r == std::next(m.begin(), 6));
+      r = m.upper_bound(18);
+      assert(r == std::next(m.begin(), 7));
+      r = m.upper_bound(20);
+      assert(r == std::next(m.begin(), 8));
     }
+  }
 #if TEST_STD_VER >= 11
-    {
+  {
     typedef int V;
     typedef std::set<int, std::less<int>, min_allocator<int>> M;
     {
-        typedef M::iterator R;
-        V ar[] =
-        {
-            5,
-            7,
-            9,
-            11,
-            13,
-            15,
-            17,
-            19
-        };
-        M m(ar, ar+sizeof(ar)/sizeof(ar[0]));
-        R r = m.upper_bound(5);
-        assert(r == std::next(m.begin(), 1));
-        r = m.upper_bound(7);
-        assert(r == std::next(m.begin(), 2));
-        r = m.upper_bound(9);
-        assert(r == std::next(m.begin(), 3));
-        r = m.upper_bound(11);
-        assert(r == std::next(m.begin(), 4));
-        r = m.upper_bound(13);
-        assert(r == std::next(m.begin(), 5));
-        r = m.upper_bound(15);
-        assert(r == std::next(m.begin(), 6));
-        r = m.upper_bound(17);
-        assert(r == std::next(m.begin(), 7));
-        r = m.upper_bound(19);
-        assert(r == std::next(m.begin(), 8));
-        r = m.upper_bound(4);
-        assert(r == std::next(m.begin(), 0));
-        r = m.upper_bound(6);
-        assert(r == std::next(m.begin(), 1));
-        r = m.upper_bound(8);
-        assert(r == std::next(m.begin(), 2));
-        r = m.upper_bound(10);
-        assert(r == std::next(m.begin(), 3));
-        r = m.upper_bound(12);
-        assert(r == std::next(m.begin(), 4));
-        r = m.upper_bound(14);
-        assert(r == std::next(m.begin(), 5));
-        r = m.upper_bound(16);
-        assert(r == std::next(m.begin(), 6));
-        r = m.upper_bound(18);
-        assert(r == std::next(m.begin(), 7));
-        r = m.upper_bound(20);
-        assert(r == std::next(m.begin(), 8));
+      typedef M::iterator R;
+      V ar[] = {5, 7, 9, 11, 13, 15, 17, 19};
+      M m(ar, ar + sizeof(ar) / sizeof(ar[0]));
+      R r = m.upper_bound(5);
+      assert(r == std::next(m.begin(), 1));
+      r = m.upper_bound(7);
+      assert(r == std::next(m.begin(), 2));
+      r = m.upper_bound(9);
+      assert(r == std::next(m.begin(), 3));
+      r = m.upper_bound(11);
+      assert(r == std::next(m.begin(), 4));
+      r = m.upper_bound(13);
+      assert(r == std::next(m.begin(), 5));
+      r = m.upper_bound(15);
+      assert(r == std::next(m.begin(), 6));
+      r = m.upper_bound(17);
+      assert(r == std::next(m.begin(), 7));
+      r = m.upper_bound(19);
+      assert(r == std::next(m.begin(), 8));
+      r = m.upper_bound(4);
+      assert(r == std::next(m.begin(), 0));
+      r = m.upper_bound(6);
+      assert(r == std::next(m.begin(), 1));
+      r = m.upper_bound(8);
+      assert(r == std::next(m.begin(), 2));
+      r = m.upper_bound(10);
+      assert(r == std::next(m.begin(), 3));
+      r = m.upper_bound(12);
+      assert(r == std::next(m.begin(), 4));
+      r = m.upper_bound(14);
+      assert(r == std::next(m.begin(), 5));
+      r = m.upper_bound(16);
+      assert(r == std::next(m.begin(), 6));
+      r = m.upper_bound(18);
+      assert(r == std::next(m.begin(), 7));
+      r = m.upper_bound(20);
+      assert(r == std::next(m.begin(), 8));
     }
     {
-        typedef M::const_iterator R;
-        V ar[] =
-        {
-            5,
-            7,
-            9,
-            11,
-            13,
-            15,
-            17,
-            19
-        };
-        const M m(ar, ar+sizeof(ar)/sizeof(ar[0]));
-        R r = m.upper_bound(5);
-        assert(r == std::next(m.begin(), 1));
-        r = m.upper_bound(7);
-        assert(r == std::next(m.begin(), 2));
-        r = m.upper_bound(9);
-        assert(r == std::next(m.begin(), 3));
-        r = m.upper_bound(11);
-        assert(r == std::next(m.begin(), 4));
-        r = m.upper_bound(13);
-        assert(r == std::next(m.begin(), 5));
-        r = m.upper_bound(15);
-        assert(r == std::next(m.begin(), 6));
-        r = m.upper_bound(17);
-        assert(r == std::next(m.begin(), 7));
-        r = m.upper_bound(19);
-        assert(r == std::next(m.begin(), 8));
-        r = m.upper_bound(4);
-        assert(r == std::next(m.begin(), 0));
-        r = m.upper_bound(6);
-        assert(r == std::next(m.begin(), 1));
-        r = m.upper_bound(8);
-        assert(r == std::next(m.begin(), 2));
-        r = m.upper_bound(10);
-        assert(r == std::next(m.begin(), 3));
-        r = m.upper_bound(12);
-        assert(r == std::next(m.begin(), 4));
-        r = m.upper_bound(14);
-        assert(r == std::next(m.begin(), 5));
-        r = m.upper_bound(16);
-        assert(r == std::next(m.begin(), 6));
-        r = m.upper_bound(18);
-        assert(r == std::next(m.begin(), 7));
-        r = m.upper_bound(20);
-        assert(r == std::next(m.begin(), 8));
-    }
+      typedef M::const_iterator R;
+      V ar[] = {5, 7, 9, 11, 13, 15, 17, 19};
+      const M m(ar, ar + sizeof(ar) / sizeof(ar[0]));
+      R r = m.upper_bound(5);
+      assert(r == std::next(m.begin(), 1));
+      r = m.upper_bound(7);
+      assert(r == std::next(m.begin(), 2));
+      r = m.upper_bound(9);
+      assert(r == std::next(m.begin(), 3));
+      r = m.upper_bound(11);
+      assert(r == std::next(m.begin(), 4));
+      r = m.upper_bound(13);
+      assert(r == std::next(m.begin(), 5));
+      r = m.upper_bound(15);
+      assert(r == std::next(m.begin(), 6));
+      r = m.upper_bound(17);
+      assert(r == std::next(m.begin(), 7));
+      r = m.upper_bound(19);
+      assert(r == std::next(m.begin(), 8));
+      r = m.upper_bound(4);
+      assert(r == std::next(m.begin(), 0));
+      r = m.upper_bound(6);
+      assert(r == std::next(m.begin(), 1));
+      r = m.upper_bound(8);
+      assert(r == std::next(m.begin(), 2));
+      r = m.upper_bound(10);
+      assert(r == std::next(m.begin(), 3));
+      r = m.upper_bound(12);
+      assert(r == std::next(m.begin(), 4));
+      r = m.upper_bound(14);
+      assert(r == std::next(m.begin(), 5));
+      r = m.upper_bound(16);
+      assert(r == std::next(m.begin(), 6));
+      r = m.upper_bound(18);
+      assert(r == std::next(m.begin(), 7));
+      r = m.upper_bound(20);
+      assert(r == std::next(m.begin(), 8));
     }
+  }
 #endif
 #if TEST_STD_VER > 11
-    {
+  {
     typedef int V;
     typedef std::set<V, std::less<>> M;
     typedef M::iterator R;
 
-    V ar[] =
-    {
-        5,
-        7,
-        9,
-        11,
-        13,
-        15,
-        17,
-        19
-    };
-    M m(ar, ar+sizeof(ar)/sizeof(ar[0]));
+    V ar[] = {5, 7, 9, 11, 13, 15, 17, 19};
+    M m(ar, ar + sizeof(ar) / sizeof(ar[0]));
     R r = m.upper_bound(5);
     assert(r == std::next(m.begin(), 1));
     r = m.upper_bound(7);
@@ -280,22 +229,22 @@ int main(int, char**)
     assert(r == std::next(m.begin(), 7));
     r = m.upper_bound(20);
     assert(r == std::next(m.begin(), 8));
-    }
+  }
 
-    {
+  {
     typedef PrivateConstructor V;
     typedef std::set<V, std::less<>> M;
     typedef M::iterator R;
 
     M m;
-    m.insert ( V::make ( 5 ));
-    m.insert ( V::make ( 7 ));
-    m.insert ( V::make ( 9 ));
-    m.insert ( V::make ( 11 ));
-    m.insert ( V::make ( 13 ));
-    m.insert ( V::make ( 15 ));
-    m.insert ( V::make ( 17 ));
-    m.insert ( V::make ( 19 ));
+    m.insert(V::make(5));
+    m.insert(V::make(7));
+    m.insert(V::make(9));
+    m.insert(V::make(11));
+    m.insert(V::make(13));
+    m.insert(V::make(15));
+    m.insert(V::make(17));
+    m.insert(V::make(19));
 
     R r = m.upper_bound(5);
     assert(r == std::next(m.begin(), 1));
@@ -331,7 +280,7 @@ int main(int, char**)
     assert(r == std::next(m.begin(), 7));
     r = m.upper_bound(20);
     assert(r == std::next(m.begin(), 8));
-    }
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/check_consecutive.h b/libcxx/test/std/containers/check_consecutive.h
index 0fe46e7f19557..2930e7dbcea98 100644
--- a/libcxx/test/std/containers/check_consecutive.h
+++ b/libcxx/test/std/containers/check_consecutive.h
@@ -18,30 +18,29 @@
 
 // Check consecutive equal values in an unordered_multiset iterator
 template <typename Iter>
-void CheckConsecutiveValues(Iter pos, Iter end, typename Iter::value_type value, std::size_t count)
-{
-    for ( std::size_t i = 0; i < count; ++i )
-    {
-        assert(pos != end);
-        assert(*pos == value);
-        ++pos;
-    }
-    assert(pos == end || *pos != value);
+void CheckConsecutiveValues(Iter pos, Iter end, typename Iter::value_type value, std::size_t count) {
+  for (std::size_t i = 0; i < count; ++i) {
+    assert(pos != end);
+    assert(*pos == value);
+    ++pos;
+  }
+  assert(pos == end || *pos != value);
 }
 
 // Check consecutive equal keys in an unordered_multimap iterator
 template <typename Iter>
-void CheckConsecutiveKeys(Iter pos, Iter end, typename Iter::value_type::first_type key, std::multiset<typename Iter::value_type::second_type>& values)
-{
-    while (!values.empty())
-    {
-        assert(pos != end);
-        assert(pos->first == key);
-        assert(values.find(pos->second) != values.end());
-        values.erase(values.find(pos->second));
-        ++pos;
-    }
-    assert(pos == end || pos->first != key);
+void CheckConsecutiveKeys(Iter pos,
+                          Iter end,
+                          typename Iter::value_type::first_type key,
+                          std::multiset<typename Iter::value_type::second_type>& values) {
+  while (!values.empty()) {
+    assert(pos != end);
+    assert(pos->first == key);
+    assert(values.find(pos->second) != values.end());
+    values.erase(values.find(pos->second));
+    ++pos;
+  }
+  assert(pos == end || pos->first != key);
 }
 
 #endif
diff --git a/libcxx/test/std/containers/container.adaptors/flat.map.syn/sorted_equivalent.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.map.syn/sorted_equivalent.pass.cpp
new file mode 100644
index 0000000000000..d9ee3fbd287b5
--- /dev/null
+++ b/libcxx/test/std/containers/container.adaptors/flat.map.syn/sorted_equivalent.pass.cpp
@@ -0,0 +1,50 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
+
+// <flat_map>
+
+// struct sorted_equivalent_t { explicit sorted_equivalent_t() = default; };
+// inline constexpr sorted_equivalent_t sorted_equivalent{};
+
+#include <cassert>
+#include <concepts>
+#include <flat_map>
+#include <type_traits>
+
+template <class T>
+void implicit_test(T) {}
+
+template <class T>
+concept HasImplicitDefaultCtor = requires { implicit_test<T>({}); };
+
+static_assert(std::is_default_constructible_v<std::sorted_equivalent_t>);
+static_assert(std::is_trivially_default_constructible_v<std::sorted_equivalent_t>);
+static_assert(!HasImplicitDefaultCtor<std::sorted_equivalent_t>);
+
+constexpr bool test() {
+  {
+    [[maybe_unused]] std::sorted_equivalent_t s;
+  }
+  {
+    [[maybe_unused]] std::same_as<const std::sorted_equivalent_t&> decltype(auto) s = (std::sorted_equivalent);
+  }
+  {
+    [[maybe_unused]] std::same_as<const std::sorted_equivalent_t> decltype(auto) copy = std::sorted_equivalent;
+  }
+
+  return true;
+}
+
+int main(int, char**) {
+  test();
+  static_assert(test());
+
+  return 0;
+}
diff --git a/libcxx/test/std/containers/container.adaptors/flat.map.syn/sorted_unique.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.map.syn/sorted_unique.pass.cpp
index c602d2d3d38f7..864fd992c66ee 100644
--- a/libcxx/test/std/containers/container.adaptors/flat.map.syn/sorted_unique.pass.cpp
+++ b/libcxx/test/std/containers/container.adaptors/flat.map.syn/sorted_unique.pass.cpp
@@ -29,9 +29,15 @@ static_assert(std::is_trivially_default_constructible_v<std::sorted_unique_t>);
 static_assert(!HasImplicitDefaultCtor<std::sorted_unique_t>);
 
 constexpr bool test() {
-  { [[maybe_unused]] std::sorted_unique_t s; }
-  { [[maybe_unused]] std::same_as<const std::sorted_unique_t&> decltype(auto) s = (std::sorted_unique); }
-  { [[maybe_unused]] std::same_as<const std::sorted_unique_t> decltype(auto) copy = std::sorted_unique; }
+  {
+    [[maybe_unused]] std::sorted_unique_t s;
+  }
+  {
+    [[maybe_unused]] std::same_as<const std::sorted_unique_t&> decltype(auto) s = (std::sorted_unique);
+  }
+  {
+    [[maybe_unused]] std::same_as<const std::sorted_unique_t> decltype(auto) copy = std::sorted_unique;
+  }
 
   return true;
 }
diff --git a/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.access/at_transparent.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.access/at_transparent.pass.cpp
index 13edca915fd00..456f12e0c0d29 100644
--- a/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.access/at_transparent.pass.cpp
+++ b/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.access/at_transparent.pass.cpp
@@ -103,7 +103,7 @@ int main(int, char**) {
     TransparentComparator c(transparent_used);
     std::flat_map<int, int, TransparentComparator> m(std::sorted_unique, {{1, 1}, {2, 2}, {3, 3}}, c);
     assert(!transparent_used);
-    m.at(Transparent<int>{3});
+    TEST_IGNORE_NODISCARD m.at(Transparent<int>{3});
     assert(transparent_used);
   }
 
diff --git a/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.capacity/empty.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.capacity/empty.pass.cpp
index 5ecc2cf7c917b..05efe063c1e17 100644
--- a/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.capacity/empty.pass.cpp
+++ b/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.capacity/empty.pass.cpp
@@ -25,7 +25,9 @@
 
 template <class KeyContainer, class ValueContainer>
 void test() {
-  using M = std::flat_map<int, double, std::less<int>, KeyContainer, ValueContainer>;
+  using Key   = typename KeyContainer::value_type;
+  using Value = typename ValueContainer::value_type;
+  using M     = std::flat_multimap<Key, Value, std::less<int>, KeyContainer, ValueContainer>;
   M m;
   ASSERT_SAME_TYPE(decltype(m.empty()), bool);
   ASSERT_NOEXCEPT(m.empty());
diff --git a/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.capacity/empty.verify.cpp b/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.capacity/empty.verify.cpp
index cc8016182dcb6..79b943b790d04 100644
--- a/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.capacity/empty.verify.cpp
+++ b/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.capacity/empty.verify.cpp
@@ -14,11 +14,7 @@
 
 #include <flat_map>
 
-#include "test_macros.h"
-
-int main(int, char**) {
+void f() {
   std::flat_map<int, int> c;
   c.empty(); // expected-warning {{ignoring return value of function declared with 'nodiscard' attribute}}
-
-  return 0;
 }
diff --git a/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.cons/deduct.compile.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.cons/deduct.compile.pass.cpp
new file mode 100644
index 0000000000000..190d78f927f34
--- /dev/null
+++ b/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.cons/deduct.compile.pass.cpp
@@ -0,0 +1,52 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
+
+// <flat_map>
+
+// Test CTAD on cases where deduction should fail.
+
+#include <flat_map>
+#include <functional>
+#include <memory>
+#include <utility>
+#include <vector>
+
+struct NotAnAllocator {
+  friend bool operator<(NotAnAllocator, NotAnAllocator) { return false; }
+};
+
+using P  = std::pair<int, long>;
+using PC = std::pair<const int, long>;
+
+template <class... Args>
+concept CanDeductFlatMap = requires { std::flat_map{std::declval<Args>()...}; };
+
+static_assert(CanDeductFlatMap<std::vector<int>, std::vector<int>>);
+
+// cannot deduce Key and T from nothing
+static_assert(!CanDeductFlatMap<>);
+
+// cannot deduce Key and T from just (KeyContainer), even if it's a container of pairs
+static_assert(!CanDeductFlatMap<std::vector<std::pair<int, int>>>);
+
+// cannot deduce Key and T from just (KeyContainer, Allocator)
+static_assert(!CanDeductFlatMap<std::vector<int>, std::allocator<std::pair<const int, int>>>);
+
+// cannot deduce Key and T from just (Compare)
+static_assert(!CanDeductFlatMap<std::less<int>>);
+
+// cannot deduce Key and T from just (Compare, Allocator)
+static_assert(!CanDeductFlatMap<std::less<int>, std::allocator<PC>>);
+
+// cannot deduce Key and T from just (Allocator)
+static_assert(!CanDeductFlatMap<std::allocator<PC>>);
+
+// cannot convert from some arbitrary unrelated type
+static_assert(!CanDeductFlatMap<NotAnAllocator>);
diff --git a/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.cons/deduct.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.cons/deduct.pass.cpp
index d01bee9aae9c0..009392feb3862 100644
--- a/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.cons/deduct.pass.cpp
+++ b/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.cons/deduct.pass.cpp
@@ -169,6 +169,24 @@ void test_iter_iter() {
     std::flat_map m(mo.cbegin(), mo.cend());
     ASSERT_SAME_TYPE(decltype(m), decltype(mo));
   }
+  {
+    std::pair<int, int> source[3] = {{1, 1}, {2, 2}, {3, 3}};
+    std::flat_map s               = {source, source + 3}; // flat_map(InputIterator, InputIterator)
+    ASSERT_SAME_TYPE(decltype(s), std::flat_map<int, int>);
+    assert(s.size() == 3);
+  }
+  {
+    std::pair<int, int> source[3] = {{1, 1}, {2, 2}, {3, 3}};
+    std::flat_map s{source, source + 3}; // flat_map(InputIterator, InputIterator)
+    ASSERT_SAME_TYPE(decltype(s), std::flat_map<int, int>);
+    assert(s.size() == 3);
+  }
+  {
+    std::pair<int, int> source[3] = {{1, 1}, {2, 2}, {3, 3}};
+    std::flat_map s{std::sorted_unique, source, source + 3}; // flat_map(sorted_unique_t, InputIterator, InputIterator)
+    static_assert(std::is_same_v<decltype(s), std::flat_map<int, int>>);
+    assert(s.size() == 3);
+  }
 }
 
 void test_iter_iter_compare() {
@@ -227,6 +245,19 @@ void test_initializer_list() {
     ASSERT_SAME_TYPE(decltype(m), std::flat_map<int, long>);
     assert(std::ranges::equal(m, sorted_arr));
   }
+  {
+    std::flat_map s = {std::make_pair(1, 'a')}; // flat_map(initializer_list<pair<int, char>>)
+    ASSERT_SAME_TYPE(decltype(s), std::flat_map<int, char>);
+    assert(s.size() == 1);
+  }
+  {
+    using M = std::flat_map<int, short>;
+    M m;
+    std::flat_map s = {std::make_pair(m, m)}; // flat_map(initializer_list<pair<M, M>>)
+    ASSERT_SAME_TYPE(decltype(s), std::flat_map<M, M>);
+    assert(s.size() == 1);
+    assert(s[m] == m);
+  }
 }
 
 void test_initializer_list_compare() {
@@ -305,38 +336,6 @@ int main(int, char**) {
   test_from_range_compare();
 
   AssociativeContainerDeductionGuidesSfinaeAway<std::flat_map, std::flat_map<int, short>>();
-  {
-    std::flat_map s = {std::make_pair(1, 'a')}; // flat_map(initializer_list<pair<int, char>>)
-    ASSERT_SAME_TYPE(decltype(s), std::flat_map<int, char>);
-    assert(s.size() == 1);
-  }
-  {
-    using M = std::flat_map<int, short>;
-    M m;
-    std::flat_map s = {std::make_pair(m, m)}; // flat_map(initializer_list<pair<M, M>>)
-    ASSERT_SAME_TYPE(decltype(s), std::flat_map<M, M>);
-    assert(s.size() == 1);
-    assert(s[m] == m);
-  }
-
-  {
-    std::pair<int, int> source[3] = {{1, 1}, {2, 2}, {3, 3}};
-    std::flat_map s               = {source, source + 3}; // flat_map(InputIterator, InputIterator)
-    ASSERT_SAME_TYPE(decltype(s), std::flat_map<int, int>);
-    assert(s.size() == 3);
-  }
-  {
-    std::pair<int, int> source[3] = {{1, 1}, {2, 2}, {3, 3}};
-    std::flat_map s{source, source + 3}; // flat_map(InputIterator, InputIterator)
-    ASSERT_SAME_TYPE(decltype(s), std::flat_map<int, int>);
-    assert(s.size() == 3);
-  }
-  {
-    std::pair<int, int> source[3] = {{1, 1}, {2, 2}, {3, 3}};
-    std::flat_map s{std::sorted_unique, source, source + 3}; // flat_map(sorted_unique_t, InputIterator, InputIterator)
-    static_assert(std::is_same_v<decltype(s), std::flat_map<int, int>>);
-    assert(s.size() == 3);
-  }
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.cons/deduct.verify.cpp b/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.cons/deduct.verify.cpp
index 08244f01cb24e..ed20c1ae715b8 100644
--- a/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.cons/deduct.verify.cpp
+++ b/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.cons/deduct.verify.cpp
@@ -14,56 +14,12 @@
 
 #include <flat_map>
 #include <functional>
-#include <memory>
 #include <utility>
-#include <vector>
-
-struct NotAnAllocator {
-  friend bool operator<(NotAnAllocator, NotAnAllocator) { return false; }
-};
 
 using P  = std::pair<int, long>;
 using PC = std::pair<const int, long>;
 
 void test() {
-  {
-    // cannot deduce Key and T from just (KeyContainer), even if it's a container of pairs
-    std::vector<std::pair<int, int>> v;
-    std::flat_map s(v);
-    // expected-error-re@-1{{{{no viable constructor or deduction guide for deduction of template arguments of '.*flat_map'}}}}
-  }
-  {
-    // cannot deduce Key and T from just (KeyContainer, Allocator)
-    std::vector<int> v;
-    std::flat_map s(v, std::allocator<std::pair<const int, int>>());
-    // expected-error-re@-1{{{{no viable constructor or deduction guide for deduction of template arguments of '.*flat_map'}}}}
-  }
-  {
-    // cannot deduce Key and T from nothing
-    std::flat_map m;
-    // expected-error-re@-1{{{{no viable constructor or deduction guide for deduction of template arguments of '.*flat_map'}}}}
-  }
-  {
-    // cannot deduce Key and T from just (Compare)
-    std::flat_map m(std::less<int>{});
-    // expected-error-re@-1{{{{no viable constructor or deduction guide for deduction of template arguments of '.*flat_map'}}}}
-  }
-  {
-    // cannot deduce Key and T from just (Compare, Allocator)
-    std::flat_map m(std::less<int>{}, std::allocator<PC>{});
-    // expected-error-re@-1{{{{no viable constructor or deduction guide for deduction of template arguments of '.*flat_map'}}}}
-  }
-  {
-    // cannot deduce Key and T from just (Allocator)
-    std::flat_map m(std::allocator<PC>{});
-    // expected-error-re@-1{{{{no viable constructor or deduction guide for deduction of template arguments of '.*flat_map'}}}}
-  }
-  {
-    // cannot convert from some arbitrary unrelated type
-    NotAnAllocator a;
-    std::flat_map m(a);
-    // expected-error-re@-1{{{{no viable constructor or deduction guide for deduction of template arguments of '.*flat_map'}}}}
-  }
   {
     // cannot deduce that the inner braced things should be std::pair and not something else
     std::flat_map m{{1, 1L}, {2, 2L}, {3, 3L}};
diff --git a/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.cons/default_noexcept.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.cons/default_noexcept.pass.cpp
index ac24c8a8ac067..790dfa4a02ed5 100644
--- a/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.cons/default_noexcept.pass.cpp
+++ b/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.cons/default_noexcept.pass.cpp
@@ -37,10 +37,12 @@ int main(int, char**) {
   {
     using C = std::flat_map<MoveOnly, MoveOnly>;
     static_assert(std::is_nothrow_default_constructible_v<C>);
+    C c;
   }
   {
     using C = std::flat_map<MoveOnly, MoveOnly, std::less<MoveOnly>, std::vector<MoveOnly, test_allocator<MoveOnly>>>;
     static_assert(std::is_nothrow_default_constructible_v<C>);
+    C c;
   }
 #endif // _LIBCPP_VERSION
   {
diff --git a/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.cons/dtor_noexcept.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.cons/dtor_noexcept.pass.cpp
index e3ab33a55d95b..1570b0fa14888 100644
--- a/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.cons/dtor_noexcept.pass.cpp
+++ b/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.cons/dtor_noexcept.pass.cpp
@@ -24,28 +24,32 @@
 
 struct ThrowingDtorComp {
   bool operator()(const auto&, const auto&) const;
-  ~ThrowingDtorComp() noexcept(false);
+  ~ThrowingDtorComp() noexcept(false) {}
 };
 
 int main(int, char**) {
   {
     using C = std::flat_map<MoveOnly, MoveOnly>;
     static_assert(std::is_nothrow_destructible_v<C>);
+    C c;
   }
   {
     using V = std::vector<MoveOnly, test_allocator<MoveOnly>>;
     using C = std::flat_map<MoveOnly, MoveOnly, std::less<MoveOnly>, V, V>;
     static_assert(std::is_nothrow_destructible_v<C>);
+    C c;
   }
   {
     using V = std::deque<MoveOnly, other_allocator<MoveOnly>>;
     using C = std::flat_map<MoveOnly, MoveOnly, std::greater<MoveOnly>, V, V>;
     static_assert(std::is_nothrow_destructible_v<C>);
+    C c;
   }
 #if defined(_LIBCPP_VERSION)
   {
     using C = std::flat_map<MoveOnly, MoveOnly, ThrowingDtorComp>;
     static_assert(!std::is_nothrow_destructible_v<C>);
+    C c;
   }
 #endif // _LIBCPP_VERSION
 
diff --git a/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.capacity/empty.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.capacity/empty.pass.cpp
new file mode 100644
index 0000000000000..4fa4fd6a69b94
--- /dev/null
+++ b/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.capacity/empty.pass.cpp
@@ -0,0 +1,51 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
+
+// <flat_map>
+
+// class flat_multimap
+
+// [[nodiscard]] bool empty() const noexcept;
+
+#include <flat_map>
+#include <cassert>
+#include <deque>
+#include <functional>
+#include <utility>
+#include <vector>
+
+#include "MinSequenceContainer.h"
+#include "test_macros.h"
+#include "min_allocator.h"
+
+template <class KeyContainer, class ValueContainer>
+void test() {
+  using Key   = typename KeyContainer::value_type;
+  using Value = typename ValueContainer::value_type;
+  using M     = std::flat_multimap<Key, Value, std::less<int>, KeyContainer, ValueContainer>;
+  M m;
+  ASSERT_SAME_TYPE(decltype(m.empty()), bool);
+  ASSERT_NOEXCEPT(m.empty());
+  assert(m.empty());
+  assert(std::as_const(m).empty());
+  m = {{1, 1.0}, {1, 2.0}};
+  assert(!m.empty());
+  m.clear();
+  assert(m.empty());
+}
+
+int main(int, char**) {
+  test<std::vector<int>, std::vector<double>>();
+  test<std::deque<int>, std::vector<double>>();
+  test<MinSequenceContainer<int>, MinSequenceContainer<double>>();
+  test<std::vector<int, min_allocator<int>>, std::vector<double, min_allocator<double>>>();
+
+  return 0;
+}
diff --git a/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.capacity/empty.verify.cpp b/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.capacity/empty.verify.cpp
new file mode 100644
index 0000000000000..9b7b827c9bec8
--- /dev/null
+++ b/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.capacity/empty.verify.cpp
@@ -0,0 +1,22 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
+
+// <flat_map>
+
+// class flat_multimap
+
+// [[nodiscard]] bool empty() const noexcept;
+
+#include <flat_map>
+
+void f() {
+  std::flat_multimap<int, int> c;
+  c.empty(); // expected-warning {{ignoring return value of function declared with 'nodiscard' attribute}}
+}
diff --git a/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.capacity/max_size.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.capacity/max_size.pass.cpp
new file mode 100644
index 0000000000000..0960c43c5a90a
--- /dev/null
+++ b/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.capacity/max_size.pass.cpp
@@ -0,0 +1,78 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
+
+// <flat_map>
+
+// class flat_multimap
+
+// size_type max_size() const noexcept;
+
+#include <cassert>
+#include <deque>
+#include <flat_map>
+#include <functional>
+#include <limits>
+#include <type_traits>
+#include <vector>
+
+#include "MinSequenceContainer.h"
+#include "test_allocator.h"
+#include "test_macros.h"
+
+int main(int, char**) {
+  {
+    using A1 = limited_allocator<int, 10>;
+    using A2 = limited_allocator<int, 20>;
+    using C  = std::flat_multimap<int, int, std::less<int>, std::vector<int, A1>, std::vector<int, A2>>;
+    ASSERT_SAME_TYPE(C::difference_type, std::ptrdiff_t);
+    ASSERT_SAME_TYPE(C::size_type, std::size_t);
+    const C c;
+    ASSERT_NOEXCEPT(c.max_size());
+    ASSERT_SAME_TYPE(decltype(c.max_size()), C::size_type);
+    assert(c.max_size() <= 10);
+    LIBCPP_ASSERT(c.max_size() == 10);
+  }
+  {
+    using A1 = limited_allocator<int, 10>;
+    using A2 = limited_allocator<int, 20>;
+    using C  = std::flat_multimap<int, int, std::less<int>, std::vector<int, A2>, std::vector<int, A1>>;
+    ASSERT_SAME_TYPE(C::difference_type, std::ptrdiff_t);
+    ASSERT_SAME_TYPE(C::size_type, std::size_t);
+    const C c;
+    ASSERT_NOEXCEPT(c.max_size());
+    ASSERT_SAME_TYPE(decltype(c.max_size()), C::size_type);
+    assert(c.max_size() <= 10);
+    LIBCPP_ASSERT(c.max_size() == 10);
+  }
+  {
+    using A = limited_allocator<int, (size_t)-1>;
+    using C = std::flat_multimap<int, int, std::less<int>, std::vector<int, A>, std::vector<int, A>>;
+    ASSERT_SAME_TYPE(C::difference_type, std::ptrdiff_t);
+    ASSERT_SAME_TYPE(C::size_type, std::size_t);
+    const C::size_type max_dist = static_cast<C::size_type>(std::numeric_limits<C::difference_type>::max());
+    const C c;
+    ASSERT_NOEXCEPT(c.max_size());
+    ASSERT_SAME_TYPE(decltype(c.max_size()), C::size_type);
+    assert(c.max_size() <= max_dist);
+    LIBCPP_ASSERT(c.max_size() == max_dist);
+  }
+  {
+    typedef std::flat_multimap<char, char> C;
+    ASSERT_SAME_TYPE(C::difference_type, std::ptrdiff_t);
+    ASSERT_SAME_TYPE(C::size_type, std::size_t);
+    const C::size_type max_dist = static_cast<C::size_type>(std::numeric_limits<C::difference_type>::max());
+    const C c;
+    ASSERT_NOEXCEPT(c.max_size());
+    ASSERT_SAME_TYPE(decltype(c.max_size()), C::size_type);
+    assert(c.max_size() <= max_dist);
+    assert(c.max_size() <= alloc_max_size(std::allocator<char>()));
+  }
+  return 0;
+}
diff --git a/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.capacity/size.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.capacity/size.pass.cpp
new file mode 100644
index 0000000000000..533f8da631fc8
--- /dev/null
+++ b/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.capacity/size.pass.cpp
@@ -0,0 +1,70 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
+
+// <flat_map>
+
+// class flat_multimap
+
+// size_type size() const noexcept;
+
+#include <cassert>
+#include <deque>
+#include <flat_map>
+#include <functional>
+#include <vector>
+
+#include "MinSequenceContainer.h"
+#include "test_macros.h"
+#include "min_allocator.h"
+
+template <class KeyContainer, class ValueContainer>
+void test() {
+  using M = std::flat_multimap<int, char, std::less<int>, KeyContainer, ValueContainer>;
+  {
+    const M m = {{1, 'a'}, {1, 'b'}, {4, 'd'}, {5, 'e'}, {5, 'h'}};
+    ASSERT_SAME_TYPE(decltype(m.size()), std::size_t);
+    ASSERT_NOEXCEPT(m.size());
+    assert(m.size() == 5);
+  }
+  {
+    const M m = {{1, 'a'}};
+    ASSERT_SAME_TYPE(decltype(m.size()), std::size_t);
+    ASSERT_NOEXCEPT(m.size());
+    assert(m.size() == 1);
+  }
+  {
+    const M m;
+    ASSERT_SAME_TYPE(decltype(m.size()), std::size_t);
+    ASSERT_NOEXCEPT(m.size());
+    assert(m.size() == 0);
+  }
+  {
+    M m;
+    std::size_t s = 1000;
+    for (auto i = 0u; i < s; ++i) {
+      m.emplace(i, 'a');
+    }
+    for (auto i = 0u; i < s; ++i) {
+      m.emplace(i, 'b');
+    }
+    ASSERT_SAME_TYPE(decltype(m.size()), std::size_t);
+    ASSERT_NOEXCEPT(m.size());
+    assert(m.size() == 2 * s);
+  }
+}
+
+int main(int, char**) {
+  test<std::vector<int>, std::vector<char>>();
+  test<std::deque<int>, std::vector<char>>();
+  test<MinSequenceContainer<int>, MinSequenceContainer<char>>();
+  test<std::vector<int, min_allocator<int>>, std::vector<char, min_allocator<char>>>();
+
+  return 0;
+}
diff --git a/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.cons/alloc.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.cons/alloc.pass.cpp
new file mode 100644
index 0000000000000..3e155eb2a1075
--- /dev/null
+++ b/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.cons/alloc.pass.cpp
@@ -0,0 +1,72 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
+
+// <flat_map>
+
+// template<class Allocator>
+//   explicit flat_multimap(const Allocator& a);
+
+#include <cassert>
+#include <flat_map>
+#include <functional>
+#include <vector>
+
+#include "test_macros.h"
+#include "test_allocator.h"
+#include "../../../test_compare.h"
+
+int main(int, char**) {
+  {
+    // The constructors in this subclause shall not participate in overload
+    // resolution unless uses_allocator_v<key_container_type, Alloc> is true
+    // and uses_allocator_v<mapped_container_type, Alloc> is true.
+
+    using C  = test_less<int>;
+    using A1 = test_allocator<int>;
+    using A2 = other_allocator<int>;
+    using V1 = std::vector<int, A1>;
+    using V2 = std::vector<int, A2>;
+    using M1 = std::flat_multimap<int, int, C, V1, V1>;
+    using M2 = std::flat_multimap<int, int, C, V1, V2>;
+    using M3 = std::flat_multimap<int, int, C, V2, V1>;
+    static_assert(std::is_constructible_v<M1, const A1&>);
+    static_assert(!std::is_constructible_v<M1, const A2&>);
+    static_assert(!std::is_constructible_v<M2, const A2&>);
+    static_assert(!std::is_constructible_v<M3, const A2&>);
+  }
+  {
+    // explicit
+    using M =
+        std::flat_multimap<int,
+                           long,
+                           std::less<int>,
+                           std::vector<int, test_allocator<int>>,
+                           std::vector<long, test_allocator<long>>>;
+
+    static_assert(std::is_constructible_v<M, test_allocator<int>>);
+    static_assert(!std::is_convertible_v<test_allocator<int>, M>);
+  }
+  {
+    using A = test_allocator<short>;
+    using M =
+        std::flat_multimap<int,
+                           long,
+                           std::less<int>,
+                           std::vector<int, test_allocator<int>>,
+                           std::vector<long, test_allocator<long>>>;
+    M m(A(0, 5));
+    assert(m.empty());
+    assert(m.begin() == m.end());
+    assert(m.keys().get_allocator().get_id() == 5);
+    assert(m.values().get_allocator().get_id() == 5);
+  }
+
+  return 0;
+}
diff --git a/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.cons/assign_initializer_list.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.cons/assign_initializer_list.pass.cpp
new file mode 100644
index 0000000000000..32f75daae7e38
--- /dev/null
+++ b/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.cons/assign_initializer_list.pass.cpp
@@ -0,0 +1,58 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
+
+// <flat_map>
+
+// flat_multimap& operator=(initializer_list<value_type> il);
+
+#include <algorithm>
+#include <cassert>
+#include <deque>
+#include <flat_map>
+#include <functional>
+#include <ranges>
+#include <vector>
+
+#include "MinSequenceContainer.h"
+#include "test_macros.h"
+#include "min_allocator.h"
+#include "test_allocator.h"
+
+template <class KeyContainer, class ValueContainer>
+void test() {
+  using Key   = typename KeyContainer::value_type;
+  using Value = typename ValueContainer::value_type;
+  using M     = std::flat_multimap<Key, Value, std::less<Key>, KeyContainer, ValueContainer>;
+  {
+    M m = {{8, 8}, {10, 10}};
+    assert(m.size() == 2);
+    m                              = {{3, 0}, {1, 0}, {2, 0}, {2, 1}, {3, 1}, {4, 0}, {3, 2}, {5, 0}, {6, 0}, {5, 1}};
+    std::pair<int, int> expected[] = {{1, 0}, {2, 0}, {2, 1}, {3, 0}, {3, 1}, {3, 2}, {4, 0}, {5, 0}, {5, 1}, {6, 0}};
+    assert(std::ranges::equal(m, expected));
+  }
+  {
+    M m = {{10, 1}, {8, 1}};
+    assert(m.size() == 2);
+    m                                    = {{3, 2}};
+    std::pair<double, double> expected[] = {{3, 2}};
+    assert(std::ranges::equal(m, expected));
+  }
+}
+
+int main(int, char**) {
+  test<std::vector<int>, std::vector<int>>();
+  test<std::vector<int>, std::vector<double>>();
+  test<std::deque<int>, std::vector<double>>();
+  test<MinSequenceContainer<int>, MinSequenceContainer<double>>();
+  test<std::vector<int, min_allocator<int>>, std::vector<double, min_allocator<double>>>();
+  test<std::vector<int, min_allocator<int>>, std::vector<int, min_allocator<int>>>();
+
+  return 0;
+}
diff --git a/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.cons/compare.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.cons/compare.pass.cpp
new file mode 100644
index 0000000000000..1989b8a4ff68a
--- /dev/null
+++ b/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.cons/compare.pass.cpp
@@ -0,0 +1,93 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
+
+// <flat_map>
+
+// explicit flat_multimap(const key_compare& comp);
+// template <class Alloc>
+//   flat_multimap(const key_compare& comp, const Alloc& a);
+
+#include <deque>
+#include <flat_map>
+#include <functional>
+#include <type_traits>
+#include <vector>
+
+#include "test_macros.h"
+#include "../../../test_compare.h"
+#include "test_allocator.h"
+
+int main(int, char**) {
+  {
+    // The constructors in this subclause shall not participate in overload
+    // resolution unless uses_allocator_v<key_container_type, Alloc> is true
+    // and uses_allocator_v<mapped_container_type, Alloc> is true.
+
+    using C  = test_less<int>;
+    using A1 = test_allocator<int>;
+    using A2 = other_allocator<int>;
+    using M1 = std::flat_multimap<int, int, C, std::vector<int, A1>, std::vector<int, A1>>;
+    using M2 = std::flat_multimap<int, int, C, std::vector<int, A1>, std::vector<int, A2>>;
+    using M3 = std::flat_multimap<int, int, C, std::vector<int, A2>, std::vector<int, A1>>;
+    static_assert(std::is_constructible_v<M1, const C&, const A1&>);
+    static_assert(!std::is_constructible_v<M1, const C&, const A2&>);
+    static_assert(!std::is_constructible_v<M2, const C&, const A2&>);
+    static_assert(!std::is_constructible_v<M3, const C&, const A2&>);
+  }
+  {
+    using C = test_less<int>;
+    auto m  = std::flat_multimap<int, char*, C>(C(3));
+    assert(m.empty());
+    assert(m.begin() == m.end());
+    assert(m.key_comp() == C(3));
+  }
+  {
+    // The one-argument ctor is explicit.
+    using C = test_less<int>;
+    static_assert(std::is_constructible_v<std::flat_multimap<int, char*, C>, C>);
+    static_assert(!std::is_convertible_v<C, std::flat_multimap<int, char*, C>>);
+
+    static_assert(std::is_constructible_v<std::flat_multimap<int, char*>, std::less<int>>);
+    static_assert(!std::is_convertible_v<std::less<int>, std::flat_multimap<int, char*>>);
+  }
+  {
+    using C  = test_less<int>;
+    using A1 = test_allocator<int>;
+    using A2 = test_allocator<short>;
+    auto m   = std::flat_multimap<int, short, C, std::vector<int, A1>, std::vector<short, A2>>(C(4), A1(5));
+    assert(m.empty());
+    assert(m.begin() == m.end());
+    assert(m.key_comp() == C(4));
+    assert(m.keys().get_allocator() == A1(5));
+    assert(m.values().get_allocator() == A2(5));
+  }
+  {
+    // explicit(false)
+    using C                                                                         = test_less<int>;
+    using A1                                                                        = test_allocator<int>;
+    using A2                                                                        = test_allocator<short>;
+    std::flat_multimap<int, short, C, std::deque<int, A1>, std::deque<short, A2>> m = {C(4), A1(5)};
+    assert(m.empty());
+    assert(m.begin() == m.end());
+    assert(m.key_comp() == C(4));
+    assert(m.keys().get_allocator() == A1(5));
+    assert(m.values().get_allocator() == A2(5));
+  }
+  {
+    // If an allocator is given, it must be usable by both containers.
+    using A = test_allocator<int>;
+    using M = std::flat_multimap<int, int, std::less<>, std::vector<int>, std::vector<int, A>>;
+    static_assert(std::is_constructible_v<M, std::less<>>);
+    static_assert(!std::is_constructible_v<M, std::less<>, std::allocator<int>>);
+    static_assert(!std::is_constructible_v<M, std::less<>, A>);
+  }
+
+  return 0;
+}
diff --git a/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.cons/containers.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.cons/containers.pass.cpp
new file mode 100644
index 0000000000000..17ee3c3864b1b
--- /dev/null
+++ b/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.cons/containers.pass.cpp
@@ -0,0 +1,187 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
+
+// <flat_map>
+
+// flat_multimap(key_container_type key_cont, mapped_container_type mapped_cont,
+//           const key_compare& comp = key_compare());
+// template<class Allocator>
+//   flat_multimap(const key_container_type& key_cont, const mapped_container_type& mapped_cont,
+//            const Allocator& a);
+// template<class Alloc>
+//   flat_multimap(const key_container_type& key_cont, const mapped_container_type& mapped_cont,
+//            const key_compare& comp, const Alloc& a);
+
+#include <algorithm>
+#include <deque>
+#include <flat_map>
+#include <functional>
+#include <vector>
+
+#include "min_allocator.h"
+#include "MoveOnly.h"
+#include "test_allocator.h"
+#include "test_iterators.h"
+#include "test_macros.h"
+#include "../../../test_compare.h"
+
+struct P {
+  int first;
+  int second;
+  template <class T, class U>
+  bool operator==(const std::pair<T, U>& rhs) const {
+    return MoveOnly(first) == rhs.first && MoveOnly(second) == rhs.second;
+  }
+};
+
+int main(int, char**) {
+  {
+    // The constructors in this subclause shall not participate in overload
+    // resolution unless uses_allocator_v<key_container_type, Alloc> is true
+    // and uses_allocator_v<mapped_container_type, Alloc> is true.
+
+    using C  = test_less<int>;
+    using A1 = test_allocator<int>;
+    using A2 = other_allocator<int>;
+    using V1 = std::vector<int, A1>;
+    using V2 = std::vector<int, A2>;
+    using M1 = std::flat_multimap<int, int, C, V1, V1>;
+    using M2 = std::flat_multimap<int, int, C, V1, V2>;
+    using M3 = std::flat_multimap<int, int, C, V2, V1>;
+    static_assert(std::is_constructible_v<M1, const V1&, const V1&, const A1&>);
+    static_assert(!std::is_constructible_v<M1, const V1&, const V1&, const A2&>);
+    static_assert(!std::is_constructible_v<M2, const V1&, const V2&, const A2&>);
+    static_assert(!std::is_constructible_v<M3, const V2&, const V1&, const A2&>);
+
+    static_assert(std::is_constructible_v<M1, const V1&, const V1&, const C&, const A1&>);
+    static_assert(!std::is_constructible_v<M1, const V1&, const V1&, const C&, const A2&>);
+    static_assert(!std::is_constructible_v<M2, const V1&, const V2&, const C&, const A2&>);
+    static_assert(!std::is_constructible_v<M3, const V2&, const V1&, const C&, const A2&>);
+  }
+  {
+    // flat_multimap(key_container_type , mapped_container_type)
+    using M                         = std::flat_multimap<int, char>;
+    std::vector<int> ks             = {1, 1, 1, 2, 2, 3, 2, 3, 3};
+    std::vector<char> vs            = {1, 2, 3, 4, 5, 6, 7, 8, 9};
+    auto m                          = M(ks, vs);
+    std::pair<int, char> expected[] = {{1, 1}, {1, 2}, {1, 3}, {2, 4}, {2, 5}, {2, 7}, {3, 6}, {3, 8}, {3, 9}};
+    assert(std::ranges::equal(m, expected));
+
+    // explicit(false)
+    M m2 = {ks, vs};
+    assert(m2 == m);
+
+    m = M(std::move(ks), std::move(vs));
+    assert(ks.empty()); // it was moved-from
+    assert(vs.empty()); // it was moved-from
+    assert(std::ranges::equal(m, expected));
+  }
+  {
+    // flat_multimap(key_container_type , mapped_container_type)
+    // move-only
+    P expected[] = {{3, 3}, {3, 2}, {2, 1}, {1, 4}};
+    using Ks     = std::deque<int, min_allocator<int>>;
+    using Vs     = std::vector<MoveOnly, min_allocator<MoveOnly>>;
+    using M      = std::flat_multimap<int, MoveOnly, std::greater<int>, Ks, Vs>;
+    Ks ks        = {1, 3, 3, 2};
+    Vs vs;
+    vs.push_back(4);
+    vs.push_back(3);
+    vs.push_back(2);
+    vs.push_back(1);
+    auto m = M(std::move(ks), std::move(vs));
+    assert(ks.empty()); // it was moved-from
+    assert(vs.empty()); // it was moved-from
+    assert(std::ranges::equal(m, expected, std::equal_to<>()));
+  }
+  {
+    // flat_multimap(key_container_type , mapped_container_type)
+    // container's allocators are used
+    using A = test_allocator<int>;
+    using M = std::flat_multimap<int, int, std::less<int>, std::vector<int, A>, std::deque<int, A>>;
+    auto ks = std::vector<int, A>({1, 1, 1, 2, 2, 3, 2, 3, 3}, A(5));
+    auto vs = std::deque<int, A>({1, 1, 1, 2, 2, 3, 2, 3, 3}, A(6));
+    auto m  = M(std::move(ks), std::move(vs));
+    assert(ks.empty()); // it was moved-from
+    assert(vs.empty()); // it was moved-from
+    std::pair<int, int> expected[] = {{1, 1}, {1, 1}, {1, 1}, {2, 2}, {2, 2}, {2, 2}, {3, 3}, {3, 3}, {3, 3}};
+    assert(std::ranges::equal(m, expected));
+    assert(m.keys().get_allocator() == A(5));
+    assert(m.values().get_allocator() == A(6));
+  }
+  {
+    // flat_multimap(key_container_type , mapped_container_type, key_compare)
+    using C                         = test_less<int>;
+    using M                         = std::flat_multimap<int, char, C>;
+    std::vector<int> ks             = {1, 1, 1, 2, 2, 3, 2, 3, 3};
+    std::vector<char> vs            = {1, 2, 3, 4, 5, 6, 7, 8, 9};
+    auto m                          = M(ks, vs, C(4));
+    std::pair<int, char> expected[] = {{1, 1}, {1, 2}, {1, 3}, {2, 4}, {2, 5}, {2, 7}, {3, 6}, {3, 8}, {3, 9}};
+    assert(std::ranges::equal(m, expected));
+    assert(m.key_comp() == C(4));
+
+    // explicit(false)
+    M m2 = {ks, vs, C(4)};
+    assert(m2 == m);
+    assert(m2.key_comp() == C(4));
+  }
+  {
+    // flat_multimap(key_container_type , mapped_container_type, const Allocator&)
+    using A = test_allocator<int>;
+    using M = std::flat_multimap<int, int, std::less<int>, std::vector<int, A>, std::deque<int, A>>;
+    auto ks = std::vector<int, A>({1, 1, 1, 2, 2, 3, 2, 3, 3}, A(5));
+    auto vs = std::deque<int, A>({1, 1, 1, 2, 2, 3, 2, 3, 3}, A(6));
+    auto m  = M(ks, vs, A(4)); // replaces the allocators
+    assert(!ks.empty());       // it was an lvalue above
+    assert(!vs.empty());       // it was an lvalue above
+    std::pair<int, int> expected[] = {{1, 1}, {1, 1}, {1, 1}, {2, 2}, {2, 2}, {2, 2}, {3, 3}, {3, 3}, {3, 3}};
+    assert(std::ranges::equal(m, expected));
+    assert(m.keys().get_allocator() == A(4));
+    assert(m.values().get_allocator() == A(4));
+  }
+  {
+    // flat_multimap(key_container_type , mapped_container_type, const Allocator&)
+    // explicit(false)
+    using A = test_allocator<int>;
+    using M = std::flat_multimap<int, int, std::less<int>, std::vector<int, A>, std::deque<int, A>>;
+    auto ks = std::vector<int, A>({1, 1, 1, 2, 2, 3, 2, 3, 3}, A(5));
+    auto vs = std::deque<int, A>({1, 1, 1, 2, 2, 3, 2, 3, 3}, A(6));
+    M m     = {ks, vs, A(4)}; // implicit ctor
+    assert(!ks.empty());      // it was an lvalue above
+    assert(!vs.empty());      // it was an lvalue above
+    std::pair<int, int> expected[] = {{1, 1}, {1, 1}, {1, 1}, {2, 2}, {2, 2}, {2, 2}, {3, 3}, {3, 3}, {3, 3}};
+    assert(std::ranges::equal(m, expected));
+    assert(m.keys().get_allocator() == A(4));
+    assert(m.values().get_allocator() == A(4));
+  }
+  {
+    // flat_multimap(key_container_type , mapped_container_type, key_compare, const Allocator&)
+    using C                         = test_less<int>;
+    using A                         = test_allocator<int>;
+    using M                         = std::flat_multimap<int, int, C, std::vector<int, A>, std::vector<int, A>>;
+    std::vector<int, A> ks          = {1, 1, 1, 2, 2, 3, 2, 3, 3};
+    std::vector<int, A> vs          = {1, 2, 3, 4, 5, 6, 7, 8, 9};
+    auto m                          = M(ks, vs, C(4), A(5));
+    std::pair<int, char> expected[] = {{1, 1}, {1, 2}, {1, 3}, {2, 4}, {2, 5}, {2, 7}, {3, 6}, {3, 8}, {3, 9}};
+    assert(std::ranges::equal(m, expected));
+    assert(m.key_comp() == C(4));
+    assert(m.keys().get_allocator() == A(5));
+    assert(m.values().get_allocator() == A(5));
+
+    // explicit(false)
+    M m2 = {ks, vs, C(4), A(5)};
+    assert(m2 == m);
+    assert(m2.key_comp() == C(4));
+    assert(m2.keys().get_allocator() == A(5));
+    assert(m2.values().get_allocator() == A(5));
+  }
+
+  return 0;
+}
diff --git a/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.cons/copy.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.cons/copy.pass.cpp
new file mode 100644
index 0000000000000..0e6d12cd3c569
--- /dev/null
+++ b/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.cons/copy.pass.cpp
@@ -0,0 +1,70 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
+
+// <flat_map>
+
+// flat_multimap(const flat_multimap& m);
+
+#include <cassert>
+#include <flat_map>
+#include <vector>
+
+#include "test_macros.h"
+#include "../../../test_compare.h"
+#include "test_allocator.h"
+
+int main(int, char**) {
+  {
+    using C = test_less<int>;
+    std::vector<int, test_allocator<int>> ks({1, 1, 3, 3, 5}, test_allocator<int>(6));
+    std::vector<char, test_allocator<char>> vs({2, 2, 1, 1, 1}, test_allocator<char>(7));
+    using M = std::flat_multimap<int, char, C, decltype(ks), decltype(vs)>;
+    auto mo = M(ks, vs, C(5));
+    auto m  = mo;
+
+    assert(m.key_comp() == C(5));
+    assert(m.keys() == ks);
+    assert(m.values() == vs);
+    assert(m.keys().get_allocator() == test_allocator<int>(6));
+    assert(m.values().get_allocator() == test_allocator<char>(7));
+
+    // mo is unchanged
+    assert(mo.key_comp() == C(5));
+    assert(mo.keys() == ks);
+    assert(mo.values() == vs);
+    assert(mo.keys().get_allocator() == test_allocator<int>(6));
+    assert(mo.values().get_allocator() == test_allocator<char>(7));
+  }
+  {
+    using C  = test_less<int>;
+    using Ks = std::vector<int, other_allocator<int>>;
+    using Vs = std::vector<char, other_allocator<char>>;
+    auto ks  = Ks({1, 3, 5, 5, 5, 5}, other_allocator<int>(6));
+    auto vs  = Vs({2, 2, 5, 5, 5, 1}, other_allocator<char>(7));
+    using M  = std::flat_multimap<int, char, C, Ks, Vs>;
+    auto mo  = M(Ks(ks, other_allocator<int>(6)), Vs(vs, other_allocator<int>(7)), C(5));
+    auto m   = mo;
+
+    assert(m.key_comp() == C(5));
+    assert(m.keys() == ks);
+    assert(m.values() == vs);
+    assert(m.keys().get_allocator() == other_allocator<int>(-2));
+    assert(m.values().get_allocator() == other_allocator<char>(-2));
+
+    // mo is unchanged
+    assert(mo.key_comp() == C(5));
+    assert(mo.keys() == ks);
+    assert(mo.values() == vs);
+    assert(mo.keys().get_allocator() == other_allocator<int>(6));
+    assert(mo.values().get_allocator() == other_allocator<char>(7));
+  }
+
+  return 0;
+}
diff --git a/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.cons/copy_alloc.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.cons/copy_alloc.pass.cpp
new file mode 100644
index 0000000000000..3047c004d42e9
--- /dev/null
+++ b/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.cons/copy_alloc.pass.cpp
@@ -0,0 +1,67 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
+
+// <flat_map>
+
+// flat_multimap(const flat_multimap&, const allocator_type&);
+
+#include <cassert>
+#include <deque>
+#include <flat_map>
+#include <functional>
+#include <vector>
+
+#include "test_macros.h"
+#include "../../../test_compare.h"
+#include "test_allocator.h"
+
+int main(int, char**) {
+  {
+    // The constructors in this subclause shall not participate in overload
+    // resolution unless uses_allocator_v<key_container_type, Alloc> is true
+    // and uses_allocator_v<mapped_container_type, Alloc> is true.
+
+    using C  = test_less<int>;
+    using A1 = test_allocator<int>;
+    using A2 = other_allocator<int>;
+    using V1 = std::vector<int, A1>;
+    using V2 = std::vector<int, A2>;
+    using M1 = std::flat_multimap<int, int, C, V1, V1>;
+    using M2 = std::flat_multimap<int, int, C, V1, V2>;
+    using M3 = std::flat_multimap<int, int, C, V2, V1>;
+    static_assert(std::is_constructible_v<M1, const M1&, const A1&>);
+    static_assert(!std::is_constructible_v<M1, const M1&, const A2&>);
+    static_assert(!std::is_constructible_v<M2, const M2&, const A2&>);
+    static_assert(!std::is_constructible_v<M3, const M3&, const A2&>);
+  }
+  {
+    using C = test_less<int>;
+    std::vector<int, test_allocator<int>> ks({1, 3, 3, 5, 5}, test_allocator<int>(6));
+    std::vector<char, test_allocator<char>> vs({2, 2, 1, 1, 1}, test_allocator<char>(7));
+    using M = std::flat_multimap<int, char, C, decltype(ks), decltype(vs)>;
+    auto mo = M(ks, vs, C(5));
+    auto m  = M(mo, test_allocator<int>(3));
+
+    assert(m.key_comp() == C(5));
+    assert(m.keys() == ks);
+    assert(m.values() == vs);
+    assert(m.keys().get_allocator() == test_allocator<int>(3));
+    assert(m.values().get_allocator() == test_allocator<char>(3));
+
+    // mo is unchanged
+    assert(mo.key_comp() == C(5));
+    assert(mo.keys() == ks);
+    assert(mo.values() == vs);
+    assert(mo.keys().get_allocator() == test_allocator<int>(6));
+    assert(mo.values().get_allocator() == test_allocator<char>(7));
+  }
+
+  return 0;
+}
diff --git a/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.cons/copy_assign.addressof.compile.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.cons/copy_assign.addressof.compile.pass.cpp
new file mode 100644
index 0000000000000..233a9c6859318
--- /dev/null
+++ b/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.cons/copy_assign.addressof.compile.pass.cpp
@@ -0,0 +1,30 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
+
+// <flat_map>
+
+// flat_multimap& operator=(const flat_multimap& s);
+
+// Validate whether the container can be copy-assigned (move-assigned, swapped)
+// with an ADL-hijacking operator&
+
+#include <flat_map>
+#include <utility>
+
+#include "test_macros.h"
+#include "operator_hijacker.h"
+
+void test() {
+  std::flat_multimap<operator_hijacker, operator_hijacker> so;
+  std::flat_multimap<operator_hijacker, operator_hijacker> s;
+  s = so;
+  s = std::move(so);
+  swap(s, so);
+}
diff --git a/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.cons/copy_assign.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.cons/copy_assign.pass.cpp
new file mode 100644
index 0000000000000..3dd7ebdd38871
--- /dev/null
+++ b/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.cons/copy_assign.pass.cpp
@@ -0,0 +1,81 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
+
+// <flat_map>
+
+// flat_multimap& operator=(const flat_multimap& m);
+
+#include <deque>
+#include <flat_map>
+#include <functional>
+#include <vector>
+
+#include "test_macros.h"
+#include "../../../test_compare.h"
+#include "test_allocator.h"
+
+int main(int, char**) {
+  {
+    // test_allocator is not propagated
+    using C = test_less<int>;
+    std::vector<int, test_allocator<int>> ks({1, 1, 3, 3, 5}, test_allocator<int>(6));
+    std::vector<char, test_allocator<char>> vs({1, 2, 3, 4, 5}, test_allocator<char>(7));
+    using M = std::flat_multimap<int, char, C, decltype(ks), decltype(vs)>;
+    auto mo = M(ks, vs, C(5));
+    auto m  = M({{3, 3}, {4, 4}, {5, 5}}, C(3), test_allocator<int>(2));
+    m       = mo;
+
+    assert(m.key_comp() == C(5));
+    assert(m.keys() == ks);
+    assert(m.values() == vs);
+    assert(m.keys().get_allocator() == test_allocator<int>(2));
+    assert(m.values().get_allocator() == test_allocator<char>(2));
+
+    // mo is unchanged
+    assert(mo.key_comp() == C(5));
+    assert(mo.keys() == ks);
+    assert(mo.values() == vs);
+    assert(mo.keys().get_allocator() == test_allocator<int>(6));
+    assert(mo.values().get_allocator() == test_allocator<char>(7));
+  }
+  {
+    // other_allocator is propagated
+    using C  = test_less<int>;
+    using Ks = std::vector<int, other_allocator<int>>;
+    using Vs = std::vector<char, other_allocator<char>>;
+    auto ks  = Ks({1, 1, 3, 3, 5}, other_allocator<int>(6));
+    auto vs  = Vs({2, 1, 3, 2, 1}, other_allocator<char>(7));
+    using M  = std::flat_multimap<int, char, C, Ks, Vs>;
+    auto mo  = M(Ks(ks, other_allocator<int>(6)), Vs(vs, other_allocator<int>(7)), C(5));
+    auto m   = M({{3, 3}, {4, 4}, {5, 5}}, C(3), other_allocator<int>(2));
+    m        = mo;
+
+    assert(m.key_comp() == C(5));
+    assert(m.keys() == ks);
+    assert(m.values() == vs);
+    assert(m.keys().get_allocator() == other_allocator<int>(6));
+    assert(m.values().get_allocator() == other_allocator<char>(7));
+
+    // mo is unchanged
+    assert(mo.key_comp() == C(5));
+    assert(mo.keys() == ks);
+    assert(mo.values() == vs);
+    assert(mo.keys().get_allocator() == other_allocator<int>(6));
+    assert(mo.values().get_allocator() == other_allocator<char>(7));
+  }
+  {
+    // self-assignment
+    using M = std::flat_multimap<int, int>;
+    M m     = {{1, 1}, {3, 4}};
+    m       = static_cast<const M&>(m);
+    assert((m == M{{1, 1}, {3, 4}}));
+  }
+  return 0;
+}
diff --git a/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.cons/deduct.compile.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.cons/deduct.compile.pass.cpp
new file mode 100644
index 0000000000000..a9d8382bd037c
--- /dev/null
+++ b/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.cons/deduct.compile.pass.cpp
@@ -0,0 +1,52 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
+
+// <flat_map>
+
+// Test CTAD on cases where deduction should fail.
+
+#include <flat_map>
+#include <functional>
+#include <memory>
+#include <utility>
+#include <vector>
+
+struct NotAnAllocator {
+  friend bool operator<(NotAnAllocator, NotAnAllocator) { return false; }
+};
+
+using P  = std::pair<int, long>;
+using PC = std::pair<const int, long>;
+
+template <class... Args>
+concept CanDeductFlatMultimap = requires { std::flat_multimap{std::declval<Args>()...}; };
+
+static_assert(CanDeductFlatMultimap<std::vector<int>, std::vector<int>>);
+
+// cannot deduce Key and T from nothing
+static_assert(!CanDeductFlatMultimap<>);
+
+// cannot deduce Key and T from just (KeyContainer), even if it's a container of pairs
+static_assert(!CanDeductFlatMultimap<std::vector<std::pair<int, int>>>);
+
+// cannot deduce Key and T from just (KeyContainer, Allocator)
+static_assert(!CanDeductFlatMultimap<std::vector<int>, std::allocator<std::pair<const int, int>>>);
+
+// cannot deduce Key and T from just (Compare)
+static_assert(!CanDeductFlatMultimap<std::less<int>>);
+
+// cannot deduce Key and T from just (Compare, Allocator)
+static_assert(!CanDeductFlatMultimap<std::less<int>, std::allocator<PC>>);
+
+// cannot deduce Key and T from just (Allocator)
+static_assert(!CanDeductFlatMultimap<std::allocator<PC>>);
+
+// cannot convert from some arbitrary unrelated type
+static_assert(!CanDeductFlatMultimap<NotAnAllocator>);
diff --git a/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.cons/deduct.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.cons/deduct.pass.cpp
new file mode 100644
index 0000000000000..a718d9cfad5b7
--- /dev/null
+++ b/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.cons/deduct.pass.cpp
@@ -0,0 +1,343 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
+
+// <flat_map>
+
+#include <algorithm>
+#include <cassert>
+#include <climits>
+#include <deque>
+#include <initializer_list>
+#include <list>
+#include <flat_map>
+#include <functional>
+#include <ranges>
+#include <type_traits>
+#include <utility>
+#include <vector>
+
+#include "deduction_guides_sfinae_checks.h"
+#include "test_allocator.h"
+
+using P  = std::pair<int, long>;
+using PC = std::pair<const int, long>;
+
+void test_copy() {
+  {
+    std::flat_multimap<long, short> source = {{1, 2}, {1, 3}};
+    std::flat_multimap s(source);
+    ASSERT_SAME_TYPE(decltype(s), decltype(source));
+    assert(s == source);
+  }
+  {
+    std::flat_multimap<long, short, std::greater<long>> source = {{1, 2}, {1, 3}};
+    std::flat_multimap s{source}; // braces instead of parens
+    ASSERT_SAME_TYPE(decltype(s), decltype(source));
+    assert(s == source);
+  }
+  {
+    std::flat_multimap<long, short, std::greater<long>> source = {{1, 2}, {1, 3}};
+    std::flat_multimap s(source, std::allocator<int>());
+    ASSERT_SAME_TYPE(decltype(s), decltype(source));
+    assert(s == source);
+  }
+}
+
+void test_containers() {
+  std::deque<int, test_allocator<int>> ks({1, 2, 1, 2, 2, INT_MAX, 3}, test_allocator<int>(0, 42));
+  std::deque<short, test_allocator<short>> vs({1, 2, 3, 4, 5, 3, 4}, test_allocator<int>(0, 43));
+  std::deque<int, test_allocator<int>> sorted_ks({1, 1, 2, 2, 2, 3, INT_MAX}, test_allocator<int>(0, 42));
+  std::deque<short, test_allocator<short>> sorted_vs({1, 3, 2, 4, 5, 4, 3}, test_allocator<int>(0, 43));
+  const std::pair<int, short> expected[] = {{1, 1}, {1, 3}, {2, 2}, {2, 4}, {2, 5}, {3, 4}, {INT_MAX, 3}};
+  {
+    std::flat_multimap s(ks, vs);
+
+    ASSERT_SAME_TYPE(decltype(s), std::flat_multimap<int, short, std::less<int>, decltype(ks), decltype(vs)>);
+    assert(std::ranges::equal(s, expected));
+    assert(s.keys().get_allocator().get_id() == 42);
+    assert(s.values().get_allocator().get_id() == 43);
+  }
+  {
+    std::flat_multimap s(std::sorted_equivalent, sorted_ks, sorted_vs);
+
+    ASSERT_SAME_TYPE(decltype(s), std::flat_multimap<int, short, std::less<int>, decltype(ks), decltype(vs)>);
+    assert(std::ranges::equal(s, expected));
+    assert(s.keys().get_allocator().get_id() == 42);
+    assert(s.values().get_allocator().get_id() == 43);
+  }
+  {
+    std::flat_multimap s(ks, vs, test_allocator<long>(0, 44));
+
+    ASSERT_SAME_TYPE(decltype(s), std::flat_multimap<int, short, std::less<int>, decltype(ks), decltype(vs)>);
+    assert(std::ranges::equal(s, expected));
+    assert(s.keys().get_allocator().get_id() == 44);
+    assert(s.values().get_allocator().get_id() == 44);
+  }
+  {
+    std::flat_multimap s(std::sorted_equivalent, sorted_ks, sorted_vs, test_allocator<long>(0, 44));
+
+    ASSERT_SAME_TYPE(decltype(s), std::flat_multimap<int, short, std::less<int>, decltype(ks), decltype(vs)>);
+    assert(std::ranges::equal(s, expected));
+    assert(s.keys().get_allocator().get_id() == 44);
+    assert(s.values().get_allocator().get_id() == 44);
+  }
+}
+
+void test_containers_compare() {
+  std::deque<int, test_allocator<int>> ks({1, 2, 1, 2, 2, INT_MAX, 3}, test_allocator<int>(0, 42));
+  std::deque<short, test_allocator<short>> vs({1, 2, 3, 4, 5, 3, 4}, test_allocator<int>(0, 43));
+  std::deque<int, test_allocator<int>> sorted_ks({INT_MAX, 3, 2, 2, 2, 1, 1}, test_allocator<int>(0, 42));
+  std::deque<short, test_allocator<short>> sorted_vs({3, 4, 2, 4, 5, 1, 3}, test_allocator<int>(0, 43));
+  const std::pair<int, short> expected[] = {{INT_MAX, 3}, {3, 4}, {2, 2}, {2, 4}, {2, 5}, {1, 1}, {1, 3}};
+  {
+    std::flat_multimap s(ks, vs, std::greater<int>());
+
+    ASSERT_SAME_TYPE(decltype(s), std::flat_multimap<int, short, std::greater<int>, decltype(ks), decltype(vs)>);
+    assert(std::ranges::equal(s, expected));
+    assert(s.keys().get_allocator().get_id() == 42);
+    assert(s.values().get_allocator().get_id() == 43);
+  }
+  {
+    std::flat_multimap s(std::sorted_equivalent, sorted_ks, sorted_vs, std::greater<int>());
+
+    ASSERT_SAME_TYPE(decltype(s), std::flat_multimap<int, short, std::greater<int>, decltype(ks), decltype(vs)>);
+    assert(std::ranges::equal(s, expected));
+    assert(s.keys().get_allocator().get_id() == 42);
+    assert(s.values().get_allocator().get_id() == 43);
+  }
+  {
+    std::flat_multimap s(ks, vs, std::greater<int>(), test_allocator<long>(0, 44));
+
+    ASSERT_SAME_TYPE(decltype(s), std::flat_multimap<int, short, std::greater<int>, decltype(ks), decltype(vs)>);
+    assert(std::ranges::equal(s, expected));
+    assert(s.keys().get_allocator().get_id() == 44);
+    assert(s.values().get_allocator().get_id() == 44);
+  }
+  {
+    std::flat_multimap s(
+        std::sorted_equivalent, sorted_ks, sorted_vs, std::greater<int>(), test_allocator<long>(0, 44));
+
+    ASSERT_SAME_TYPE(decltype(s), std::flat_multimap<int, short, std::greater<int>, decltype(ks), decltype(vs)>);
+    assert(std::ranges::equal(s, expected));
+    assert(s.keys().get_allocator().get_id() == 44);
+    assert(s.values().get_allocator().get_id() == 44);
+  }
+}
+
+void test_iter_iter() {
+  const P arr[]          = {{1, 1L}, {2, 2L}, {1, 1L}, {INT_MAX, 1L}, {3, 1L}};
+  const P sorted_arr[]   = {{1, 1L}, {1, 1L}, {2, 2L}, {3, 1L}, {INT_MAX, 1L}};
+  const PC arrc[]        = {{1, 1L}, {2, 2L}, {1, 1L}, {INT_MAX, 1L}, {3, 1L}};
+  const PC sorted_arrc[] = {{1, 1L}, {1, 1L}, {2, 2L}, {3, 1L}, {INT_MAX, 1L}};
+  {
+    std::flat_multimap m(std::begin(arr), std::end(arr));
+
+    ASSERT_SAME_TYPE(decltype(m), std::flat_multimap<int, long>);
+    assert(std::ranges::equal(m, sorted_arr));
+  }
+  {
+    std::flat_multimap m(std::begin(arrc), std::end(arrc));
+
+    ASSERT_SAME_TYPE(decltype(m), std::flat_multimap<int, long>);
+    assert(std::ranges::equal(m, sorted_arr));
+  }
+  {
+    std::flat_multimap m(std::sorted_equivalent, std::begin(sorted_arr), std::end(sorted_arr));
+
+    ASSERT_SAME_TYPE(decltype(m), std::flat_multimap<int, long>);
+    assert(std::ranges::equal(m, sorted_arr));
+  }
+  {
+    std::flat_multimap m(std::sorted_equivalent, std::begin(sorted_arrc), std::end(sorted_arrc));
+
+    ASSERT_SAME_TYPE(decltype(m), std::flat_multimap<int, long>);
+    assert(std::ranges::equal(m, sorted_arr));
+  }
+  {
+    std::flat_multimap<int, short> mo;
+    std::flat_multimap m(mo.begin(), mo.end());
+    ASSERT_SAME_TYPE(decltype(m), decltype(mo));
+  }
+  {
+    std::flat_multimap<int, short> mo;
+    std::flat_multimap m(mo.cbegin(), mo.cend());
+    ASSERT_SAME_TYPE(decltype(m), decltype(mo));
+  }
+  {
+    std::pair<int, int> source[3] = {{1, 1}, {1, 1}, {3, 3}};
+    std::flat_multimap s          = {source, source + 3}; // flat_multimap(InputIterator, InputIterator)
+    ASSERT_SAME_TYPE(decltype(s), std::flat_multimap<int, int>);
+    assert(s.size() == 3);
+  }
+  {
+    std::pair<int, int> source[3] = {{1, 1}, {1, 1}, {3, 3}};
+    std::flat_multimap s{source, source + 3}; // flat_multimap(InputIterator, InputIterator)
+    ASSERT_SAME_TYPE(decltype(s), std::flat_multimap<int, int>);
+    assert(s.size() == 3);
+  }
+  {
+    std::pair<int, int> source[3] = {{1, 1}, {1, 2}, {3, 3}};
+    std::flat_multimap s{
+        std::sorted_equivalent, source, source + 3}; // flat_multimap(sorted_equivalent_t, InputIterator, InputIterator)
+    static_assert(std::is_same_v<decltype(s), std::flat_multimap<int, int>>);
+    assert(s.size() == 3);
+  }
+}
+
+void test_iter_iter_compare() {
+  const P arr[]          = {{1, 1L}, {2, 2L}, {1, 1L}, {INT_MAX, 1L}, {3, 1L}};
+  const P sorted_arr[]   = {{INT_MAX, 1L}, {3, 1L}, {2, 2L}, {1, 1L}, {1, 1L}};
+  const PC arrc[]        = {{1, 1L}, {2, 2L}, {1, 1L}, {INT_MAX, 1L}, {3, 1L}};
+  const PC sorted_arrc[] = {{INT_MAX, 1L}, {3, 1L}, {2, 2L}, {1, 1L}, {1, 1L}};
+  using C                = std::greater<long long>;
+  {
+    std::flat_multimap m(std::begin(arr), std::end(arr), C());
+
+    ASSERT_SAME_TYPE(decltype(m), std::flat_multimap<int, long, C>);
+    assert(std::ranges::equal(m, sorted_arr));
+  }
+  {
+    std::flat_multimap m(std::begin(arrc), std::end(arrc), C());
+
+    ASSERT_SAME_TYPE(decltype(m), std::flat_multimap<int, long, C>);
+    assert(std::ranges::equal(m, sorted_arr));
+  }
+  {
+    std::flat_multimap m(std::sorted_equivalent, std::begin(sorted_arr), std::end(sorted_arr), C());
+
+    ASSERT_SAME_TYPE(decltype(m), std::flat_multimap<int, long, C>);
+    assert(std::ranges::equal(m, sorted_arr));
+  }
+  {
+    std::flat_multimap m(std::sorted_equivalent, std::begin(sorted_arrc), std::end(sorted_arrc), C());
+
+    ASSERT_SAME_TYPE(decltype(m), std::flat_multimap<int, long, C>);
+    assert(std::ranges::equal(m, sorted_arr));
+  }
+  {
+    std::flat_multimap<int, short> mo;
+    std::flat_multimap m(mo.begin(), mo.end(), C());
+    ASSERT_SAME_TYPE(decltype(m), std::flat_multimap<int, short, C>);
+  }
+  {
+    std::flat_multimap<int, short> mo;
+    std::flat_multimap m(mo.cbegin(), mo.cend(), C());
+    ASSERT_SAME_TYPE(decltype(m), std::flat_multimap<int, short, C>);
+  }
+}
+
+void test_initializer_list() {
+  const P sorted_arr[] = {{1, 1L}, {1, 1L}, {2, 2L}, {3, 1L}, {INT_MAX, 1L}};
+  {
+    std::flat_multimap m{std::pair{1, 1L}, {2, 2L}, {1, 1L}, {INT_MAX, 1L}, {3, 1L}};
+
+    ASSERT_SAME_TYPE(decltype(m), std::flat_multimap<int, long>);
+    assert(std::ranges::equal(m, sorted_arr));
+  }
+  {
+    std::flat_multimap m(std::sorted_equivalent, {std::pair{1, 1L}, {1, 1L}, {2, 2L}, {3, 1L}, {INT_MAX, 1L}});
+
+    ASSERT_SAME_TYPE(decltype(m), std::flat_multimap<int, long>);
+    assert(std::ranges::equal(m, sorted_arr));
+  }
+  {
+    std::flat_multimap s = {std::make_pair(1, 'a')}; // flat_multimap(initializer_list<pair<int, char>>)
+    ASSERT_SAME_TYPE(decltype(s), std::flat_multimap<int, char>);
+    assert(s.size() == 1);
+  }
+  {
+    using M = std::flat_multimap<int, short>;
+    M m;
+    std::flat_multimap s = {std::make_pair(m, m)}; // flat_multimap(initializer_list<pair<M, M>>)
+    ASSERT_SAME_TYPE(decltype(s), std::flat_multimap<M, M>);
+    assert(s.size() == 1);
+    assert(s.find(m)->second == m);
+  }
+}
+
+void test_initializer_list_compare() {
+  const P sorted_arr[] = {{INT_MAX, 1L}, {3, 1L}, {2, 2L}, {1, 1L}, {1, 1L}};
+  using C              = std::greater<long long>;
+  {
+    std::flat_multimap m({std::pair{1, 1L}, {2, 2L}, {1, 1L}, {INT_MAX, 1L}, {3, 1L}}, C());
+
+    ASSERT_SAME_TYPE(decltype(m), std::flat_multimap<int, long, C>);
+    assert(std::ranges::equal(m, sorted_arr));
+  }
+  {
+    std::flat_multimap m(std::sorted_equivalent, {std::pair{INT_MAX, 1L}, {3, 1L}, {2, 2L}, {1, 1L}, {1, 1L}}, C());
+
+    ASSERT_SAME_TYPE(decltype(m), std::flat_multimap<int, long, C>);
+    assert(std::ranges::equal(m, sorted_arr));
+  }
+}
+
+void test_from_range() {
+  std::list<std::pair<int, short>> r     = {{1, 1}, {2, 2}, {1, 1}, {INT_MAX, 4}, {3, 5}};
+  const std::pair<int, short> expected[] = {{1, 1}, {1, 1}, {2, 2}, {3, 5}, {INT_MAX, 4}};
+  {
+    std::flat_multimap s(std::from_range, r);
+    ASSERT_SAME_TYPE(decltype(s), std::flat_multimap<int, short, std::less<int>>);
+    assert(std::ranges::equal(s, expected));
+  }
+  {
+    std::flat_multimap s(std::from_range, r, test_allocator<long>(0, 42));
+    ASSERT_SAME_TYPE(
+        decltype(s),
+        std::flat_multimap<int,
+                           short,
+                           std::less<int>,
+                           std::vector<int, test_allocator<int>>,
+                           std::vector<short, test_allocator<short>>>);
+    assert(std::ranges::equal(s, expected));
+    assert(s.keys().get_allocator().get_id() == 42);
+    assert(s.values().get_allocator().get_id() == 42);
+  }
+}
+
+void test_from_range_compare() {
+  std::list<std::pair<int, short>> r     = {{1, 1}, {2, 2}, {1, 1}, {INT_MAX, 4}, {3, 5}};
+  const std::pair<int, short> expected[] = {{INT_MAX, 4}, {3, 5}, {2, 2}, {1, 1}, {1, 1}};
+  {
+    std::flat_multimap s(std::from_range, r, std::greater<int>());
+    ASSERT_SAME_TYPE(decltype(s), std::flat_multimap<int, short, std::greater<int>>);
+    assert(std::ranges::equal(s, expected));
+  }
+  {
+    std::flat_multimap s(std::from_range, r, std::greater<int>(), test_allocator<long>(0, 42));
+    ASSERT_SAME_TYPE(
+        decltype(s),
+        std::flat_multimap<int,
+                           short,
+                           std::greater<int>,
+                           std::vector<int, test_allocator<int>>,
+                           std::vector<short, test_allocator<short>>>);
+    assert(std::ranges::equal(s, expected));
+    assert(s.keys().get_allocator().get_id() == 42);
+    assert(s.values().get_allocator().get_id() == 42);
+  }
+}
+
+int main(int, char**) {
+  // Each test function also tests the sorted_equivalent-prefixed and allocator-suffixed overloads.
+  test_copy();
+  test_containers();
+  test_containers_compare();
+  test_iter_iter();
+  test_iter_iter_compare();
+  test_initializer_list();
+  test_initializer_list_compare();
+  test_from_range();
+  test_from_range_compare();
+
+  AssociativeContainerDeductionGuidesSfinaeAway<std::flat_multimap, std::flat_multimap<int, short>>();
+
+  return 0;
+}
diff --git a/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.cons/deduct.verify.cpp b/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.cons/deduct.verify.cpp
new file mode 100644
index 0000000000000..c25218e890f21
--- /dev/null
+++ b/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.cons/deduct.verify.cpp
@@ -0,0 +1,57 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
+
+// <flat_map>
+
+// Test CTAD on cases where deduction should fail.
+
+#include <flat_map>
+#include <functional>
+#include <utility>
+
+struct NotAnAllocator {
+  friend bool operator<(NotAnAllocator, NotAnAllocator) { return false; }
+};
+
+using P  = std::pair<int, long>;
+using PC = std::pair<const int, long>;
+
+void test() {
+  {
+    // cannot deduce that the inner braced things should be std::pair and not something else
+    std::flat_multimap m{{1, 1L}, {2, 2L}, {3, 3L}};
+    // expected-error-re@-1{{{{no viable constructor or deduction guide for deduction of template arguments of '.*flat_multimap'}}}}
+  }
+  {
+    // cannot deduce that the inner braced things should be std::pair and not something else
+    std::flat_multimap m({{1, 1L}, {2, 2L}, {3, 3L}}, std::less<int>());
+    // expected-error-re@-1{{{{no viable constructor or deduction guide for deduction of template arguments of '.*flat_multimap'}}}}
+  }
+  {
+    // cannot deduce that the inner braced things should be std::pair and not something else
+    std::flat_multimap m({{1, 1L}, {2, 2L}, {3, 3L}}, std::less<int>(), std::allocator<PC>());
+    // expected-error-re@-1{{{{no viable constructor or deduction guide for deduction of template arguments of '.*flat_multimap'}}}}
+  }
+  {
+    // cannot deduce that the inner braced things should be std::pair and not something else
+    std::flat_multimap m({{1, 1L}, {2, 2L}, {3, 3L}}, std::allocator<PC>());
+    // expected-error-re@-1{{{{no viable constructor or deduction guide for deduction of template arguments of '.*flat_multimap'}}}}
+  }
+  {
+    // since we have parens, not braces, this deliberately does not find the initializer_list constructor
+    std::flat_multimap m(P{1, 1L});
+    // expected-error-re@-1{{{{no viable constructor or deduction guide for deduction of template arguments of '.*flat_multimap'}}}}
+  }
+  {
+    // since we have parens, not braces, this deliberately does not find the initializer_list constructor
+    std::flat_multimap m(PC{1, 1L});
+    // expected-error-re@-1{{{{no viable constructor or deduction guide for deduction of template arguments of '.*flat_multimap'}}}}
+  }
+}
diff --git a/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.cons/deduct_pmr.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.cons/deduct_pmr.pass.cpp
new file mode 100644
index 0000000000000..1955a8806631b
--- /dev/null
+++ b/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.cons/deduct_pmr.pass.cpp
@@ -0,0 +1,107 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
+// UNSUPPORTED: availability-pmr-missing
+
+// <flat_map>
+
+#include <algorithm>
+#include <cassert>
+#include <climits>
+#include <deque>
+#include <initializer_list>
+#include <list>
+#include <flat_map>
+#include <functional>
+#include <memory_resource>
+#include <ranges>
+#include <type_traits>
+#include <utility>
+#include <vector>
+
+#include "test_allocator.h"
+
+using P  = std::pair<int, long>;
+using PC = std::pair<const int, long>;
+
+void test_containers() {
+  std::deque<int, test_allocator<int>> ks({1, 2, 1, 2, 2, INT_MAX, 3}, test_allocator<int>(0, 42));
+  std::deque<short, test_allocator<short>> vs({1, 2, 3, 4, 5, 3, 4}, test_allocator<int>(0, 43));
+  std::deque<int, test_allocator<int>> sorted_ks({1, 1, 2, 2, 2, 3, INT_MAX}, test_allocator<int>(0, 42));
+  std::deque<short, test_allocator<short>> sorted_vs({1, 3, 2, 4, 5, 4, 3}, test_allocator<int>(0, 43));
+  const std::pair<int, short> expected[] = {{1, 1}, {1, 3}, {2, 2}, {2, 4}, {2, 5}, {3, 4}, {INT_MAX, 3}};
+  {
+    std::pmr::monotonic_buffer_resource mr;
+    std::pmr::monotonic_buffer_resource mr2;
+    std::pmr::deque<int> pks(ks.begin(), ks.end(), &mr);
+    std::pmr::deque<short> pvs(vs.begin(), vs.end(), &mr);
+    std::flat_multimap s(std::move(pks), std::move(pvs), &mr2);
+
+    ASSERT_SAME_TYPE(
+        decltype(s), std::flat_multimap<int, short, std::less<int>, std::pmr::deque<int>, std::pmr::deque<short>>);
+    assert(std::ranges::equal(s, expected));
+    assert(s.keys().get_allocator().resource() == &mr2);
+    assert(s.values().get_allocator().resource() == &mr2);
+  }
+  {
+    std::pmr::monotonic_buffer_resource mr;
+    std::pmr::monotonic_buffer_resource mr2;
+    std::pmr::deque<int> pks(sorted_ks.begin(), sorted_ks.end(), &mr);
+    std::pmr::deque<short> pvs(sorted_vs.begin(), sorted_vs.end(), &mr);
+    std::flat_multimap s(std::sorted_equivalent, std::move(pks), std::move(pvs), &mr2);
+
+    ASSERT_SAME_TYPE(
+        decltype(s), std::flat_multimap<int, short, std::less<int>, std::pmr::deque<int>, std::pmr::deque<short>>);
+    assert(std::ranges::equal(s, expected));
+    assert(s.keys().get_allocator().resource() == &mr2);
+    assert(s.values().get_allocator().resource() == &mr2);
+  }
+}
+
+void test_containers_compare() {
+  std::deque<int, test_allocator<int>> ks({1, 2, 1, 2, 2, INT_MAX, 3}, test_allocator<int>(0, 42));
+  std::deque<short, test_allocator<short>> vs({1, 2, 3, 4, 5, 3, 4}, test_allocator<int>(0, 43));
+  std::deque<int, test_allocator<int>> sorted_ks({INT_MAX, 3, 2, 2, 2, 1, 1}, test_allocator<int>(0, 42));
+  std::deque<short, test_allocator<short>> sorted_vs({3, 4, 2, 4, 5, 1, 3}, test_allocator<int>(0, 43));
+  const std::pair<int, short> expected[] = {{INT_MAX, 3}, {3, 4}, {2, 2}, {2, 4}, {2, 5}, {1, 1}, {1, 3}};
+
+  {
+    std::pmr::monotonic_buffer_resource mr;
+    std::pmr::monotonic_buffer_resource mr2;
+    std::pmr::deque<int> pks(ks.begin(), ks.end(), &mr);
+    std::pmr::deque<short> pvs(vs.begin(), vs.end(), &mr);
+    std::flat_multimap s(std::move(pks), std::move(pvs), std::greater<int>(), &mr2);
+
+    ASSERT_SAME_TYPE(
+        decltype(s), std::flat_multimap<int, short, std::greater<int>, std::pmr::deque<int>, std::pmr::deque<short>>);
+    assert(std::ranges::equal(s, expected));
+    assert(s.keys().get_allocator().resource() == &mr2);
+    assert(s.values().get_allocator().resource() == &mr2);
+  }
+  {
+    std::pmr::monotonic_buffer_resource mr;
+    std::pmr::monotonic_buffer_resource mr2;
+    std::pmr::deque<int> pks(sorted_ks.begin(), sorted_ks.end(), &mr);
+    std::pmr::deque<short> pvs(sorted_vs.begin(), sorted_vs.end(), &mr);
+    std::flat_multimap s(std::sorted_equivalent, std::move(pks), std::move(pvs), std::greater<int>(), &mr2);
+
+    ASSERT_SAME_TYPE(
+        decltype(s), std::flat_multimap<int, short, std::greater<int>, std::pmr::deque<int>, std::pmr::deque<short>>);
+    assert(std::ranges::equal(s, expected));
+    assert(s.keys().get_allocator().resource() == &mr2);
+    assert(s.values().get_allocator().resource() == &mr2);
+  }
+}
+
+int main(int, char**) {
+  test_containers();
+  test_containers_compare();
+
+  return 0;
+}
diff --git a/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.cons/default.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.cons/default.pass.cpp
new file mode 100644
index 0000000000000..c910f748d95fe
--- /dev/null
+++ b/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.cons/default.pass.cpp
@@ -0,0 +1,72 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
+
+// <flat_map>
+
+// flat_multimap();
+
+#include <cassert>
+#include <deque>
+#include <flat_map>
+#include <functional>
+#include <type_traits>
+#include <vector>
+
+#include "test_macros.h"
+#include "min_allocator.h"
+#include "test_allocator.h"
+
+struct DefaultCtableComp {
+  explicit DefaultCtableComp() { default_constructed_ = true; }
+  bool operator()(int, int) const { return false; }
+  bool default_constructed_ = false;
+};
+
+int main(int, char**) {
+  {
+    std::flat_multimap<int, char*> m;
+    assert(m.empty());
+  }
+  {
+    // explicit(false)
+    std::flat_multimap<int, char*> m = {};
+    assert(m.empty());
+  }
+  {
+    std::flat_multimap<int, char*, DefaultCtableComp, std::deque<int, min_allocator<int>>> m;
+    assert(m.empty());
+    assert(m.begin() == m.end());
+    assert(m.key_comp().default_constructed_);
+  }
+  {
+    using A1 = explicit_allocator<int>;
+    using A2 = explicit_allocator<char*>;
+    {
+      std::flat_multimap<int, char*, DefaultCtableComp, std::vector<int, A1>, std::vector<char*, A2>> m;
+      assert(m.empty());
+      assert(m.key_comp().default_constructed_);
+    }
+    {
+      A1 a1;
+      std::flat_multimap<int, int, DefaultCtableComp, std::vector<int, A1>, std::vector<int, A1>> m(a1);
+      assert(m.empty());
+      assert(m.key_comp().default_constructed_);
+    }
+  }
+  {
+    // If an allocator is given, it must be usable by both containers.
+    using A = test_allocator<int>;
+    using M = std::flat_multimap<int, int, std::less<>, std::vector<int>, std::vector<int, A>>;
+    static_assert(std::is_constructible_v<M>);
+    static_assert(!std::is_constructible_v<M, std::allocator<int>>);
+    static_assert(!std::is_constructible_v<M, A>);
+  }
+  return 0;
+}
diff --git a/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.cons/default_noexcept.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.cons/default_noexcept.pass.cpp
new file mode 100644
index 0000000000000..fa490f120875f
--- /dev/null
+++ b/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.cons/default_noexcept.pass.cpp
@@ -0,0 +1,61 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
+
+// <flat_map>
+
+// flat_multimap()
+//    noexcept(
+//        is_nothrow_default_constructible_v<key_container_type> &&
+//        is_nothrow_default_constructible_v<mapped_container_type> &&
+//        is_nothrow_default_constructible_v<key_compare>);
+
+// This tests a conforming extension
+
+#include <cassert>
+#include <flat_map>
+#include <functional>
+#include <vector>
+
+#include "test_macros.h"
+#include "MoveOnly.h"
+#include "test_allocator.h"
+
+struct ThrowingCtorComp {
+  ThrowingCtorComp() noexcept(false) {}
+  bool operator()(const auto&, const auto&) const { return false; }
+};
+
+int main(int, char**) {
+#if defined(_LIBCPP_VERSION)
+  {
+    using C = std::flat_multimap<MoveOnly, MoveOnly>;
+    static_assert(std::is_nothrow_default_constructible_v<C>);
+    C c;
+  }
+  {
+    using C =
+        std::flat_multimap<MoveOnly, MoveOnly, std::less<MoveOnly>, std::vector<MoveOnly, test_allocator<MoveOnly>>>;
+    static_assert(std::is_nothrow_default_constructible_v<C>);
+    C c;
+  }
+#endif // _LIBCPP_VERSION
+  {
+    using C =
+        std::flat_multimap<MoveOnly, MoveOnly, std::less<MoveOnly>, std::vector<MoveOnly, other_allocator<MoveOnly>>>;
+    static_assert(!std::is_nothrow_default_constructible_v<C>);
+    C c;
+  }
+  {
+    using C = std::flat_multimap<MoveOnly, MoveOnly, ThrowingCtorComp>;
+    static_assert(!std::is_nothrow_default_constructible_v<C>);
+    C c;
+  }
+  return 0;
+}
diff --git a/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.cons/dtor_noexcept.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.cons/dtor_noexcept.pass.cpp
new file mode 100644
index 0000000000000..fd31e440a6614
--- /dev/null
+++ b/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.cons/dtor_noexcept.pass.cpp
@@ -0,0 +1,57 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
+
+// <flat_map>
+
+// ~flat_multimap();
+
+#include <cassert>
+#include <deque>
+#include <flat_map>
+#include <functional>
+#include <vector>
+
+#include "test_macros.h"
+#include "MoveOnly.h"
+#include "test_allocator.h"
+
+struct ThrowingDtorComp {
+  bool operator()(const auto&, const auto&) const;
+  ~ThrowingDtorComp() noexcept(false) {}
+};
+
+int main(int, char**) {
+  {
+    using C = std::flat_multimap<MoveOnly, MoveOnly>;
+    static_assert(std::is_nothrow_destructible_v<C>);
+    C c;
+  }
+  {
+    using V = std::vector<MoveOnly, test_allocator<MoveOnly>>;
+    using C = std::flat_multimap<MoveOnly, MoveOnly, std::less<MoveOnly>, V, V>;
+    static_assert(std::is_nothrow_destructible_v<C>);
+    C c;
+  }
+  {
+    using V = std::deque<MoveOnly, other_allocator<MoveOnly>>;
+    using C = std::flat_multimap<MoveOnly, MoveOnly, std::greater<MoveOnly>, V, V>;
+    static_assert(std::is_nothrow_destructible_v<C>);
+    C c;
+  }
+#if defined(_LIBCPP_VERSION)
+  {
+    using C = std::flat_multimap<MoveOnly, MoveOnly, ThrowingDtorComp>;
+    static_assert(!std::is_nothrow_destructible_v<C>);
+    C c;
+  }
+#endif // _LIBCPP_VERSION
+
+  return 0;
+}
diff --git a/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.cons/initializer_list.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.cons/initializer_list.pass.cpp
new file mode 100644
index 0000000000000..8e89192ec0ea1
--- /dev/null
+++ b/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.cons/initializer_list.pass.cpp
@@ -0,0 +1,159 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
+
+// <flat_map>
+
+// flat_multimap(initializer_list<value_type> il, const key_compare& comp = key_compare());
+// template<class Alloc>
+//    flat_multimap(initializer_list<value_type> il, const Alloc& a);
+// template<class Alloc>
+//    flat_multimap(initializer_list<value_type> il, const key_compare& comp, const Alloc& a);
+
+#include <algorithm>
+#include <cassert>
+#include <deque>
+#include <flat_map>
+#include <functional>
+#include <type_traits>
+#include <vector>
+
+#include "test_macros.h"
+#include "min_allocator.h"
+#include "test_allocator.h"
+
+#include "../../../test_compare.h"
+
+struct DefaultCtableComp {
+  explicit DefaultCtableComp() { default_constructed_ = true; }
+  bool operator()(int, int) const { return false; }
+  bool default_constructed_ = false;
+};
+
+int main(int, char**) {
+  {
+    // The constructors in this subclause shall not participate in overload
+    // resolution unless uses_allocator_v<key_container_type, Alloc> is true
+    // and uses_allocator_v<mapped_container_type, Alloc> is true.
+
+    using C  = test_less<int>;
+    using A1 = test_allocator<int>;
+    using A2 = other_allocator<int>;
+    using V1 = std::vector<int, A1>;
+    using V2 = std::vector<int, A2>;
+    using M1 = std::flat_multimap<int, int, C, V1, V1>;
+    using M2 = std::flat_multimap<int, int, C, V1, V2>;
+    using M3 = std::flat_multimap<int, int, C, V2, V1>;
+    using IL = std::initializer_list<std::pair<int, int>>;
+    static_assert(std::is_constructible_v<M1, IL, const A1&>);
+    static_assert(!std::is_constructible_v<M1, IL, const A2&>);
+    static_assert(!std::is_constructible_v<M2, IL, const A2&>);
+    static_assert(!std::is_constructible_v<M3, IL, const A2&>);
+
+    static_assert(std::is_constructible_v<M1, IL, const C&, const A1&>);
+    static_assert(!std::is_constructible_v<M1, IL, const C&, const A2&>);
+    static_assert(!std::is_constructible_v<M2, IL, const C&, const A2&>);
+    static_assert(!std::is_constructible_v<M3, IL, const C&, const A2&>);
+  }
+
+  {
+    // initializer_list<value_type> needs to match exactly
+    using M = std::flat_multimap<int, short>;
+    using C = typename M::key_compare;
+    static_assert(std::is_constructible_v<M, std::initializer_list<std::pair<int, short>>>);
+    static_assert(std::is_constructible_v<M, std::initializer_list<std::pair<int, short>>, C>);
+    static_assert(std::is_constructible_v<M, std::initializer_list<std::pair<int, short>>, C, std::allocator<int>>);
+    static_assert(std::is_constructible_v<M, std::initializer_list<std::pair<int, short>>, std::allocator<int>>);
+    static_assert(!std::is_constructible_v<M, std::initializer_list<std::pair<const int, short>>>);
+    static_assert(!std::is_constructible_v<M, std::initializer_list<std::pair<const int, short>>, C>);
+    static_assert(
+        !std::is_constructible_v<M, std::initializer_list<std::pair<const int, short>>, C, std::allocator<int>>);
+    static_assert(!std::is_constructible_v<M, std::initializer_list<std::pair<const int, short>>, std::allocator<int>>);
+    static_assert(!std::is_constructible_v<M, std::initializer_list<std::pair<const int, const short>>>);
+    static_assert(!std::is_constructible_v<M, std::initializer_list<std::pair<const int, const short>>, C>);
+    static_assert(
+        !std::is_constructible_v<M, std::initializer_list<std::pair<const int, const short>>, C, std::allocator<int>>);
+    static_assert(
+        !std::is_constructible_v<M, std::initializer_list<std::pair<const int, const short>>, std::allocator<int>>);
+  }
+
+  std::pair<int, short> expected[] = {{1, 1}, {2, 2}, {2, 2}, {3, 3}, {3, 3}, {5, 2}};
+  {
+    // flat_multimap(initializer_list<value_type>);
+    using M                                         = std::flat_multimap<int, short>;
+    std::initializer_list<std::pair<int, short>> il = {{5, 2}, {2, 2}, {2, 2}, {3, 3}, {1, 1}, {3, 3}};
+    M m(il);
+    assert(std::ranges::equal(m, expected));
+  }
+  {
+    // flat_multimap(initializer_list<value_type>);
+    // explicit(false)
+    using M = std::flat_multimap<int, short>;
+    M m     = {{5, 2}, {2, 2}, {2, 2}, {3, 3}, {1, 1}, {3, 3}};
+    assert(std::ranges::equal(m, expected));
+  }
+  {
+    // flat_multimap(initializer_list<value_type>);
+    using M = std::flat_multimap<int, short, std::greater<int>, std::deque<int, min_allocator<int>>>;
+    M m     = {{5, 2}, {2, 2}, {2, 2}, {3, 3}, {1, 1}, {3, 3}};
+    assert(std::equal(m.rbegin(), m.rend(), expected, expected + 6));
+  }
+  {
+    using A = explicit_allocator<int>;
+    {
+      // flat_multimap(initializer_list<value_type>);
+      // different comparator
+      using M = std::flat_multimap<int, int, DefaultCtableComp, std::vector<int, A>, std::deque<int, A>>;
+      M m     = {{1, 1}, {2, 2}, {3, 3}};
+      assert(m.size() == 3);
+
+      std::pair<int, int> expected1[] = {{1, 1}, {2, 2}, {3, 3}};
+      assert(std::ranges::equal(m, expected1));
+      assert(m.key_comp().default_constructed_);
+    }
+    {
+      // flat_multimap(initializer_list<value_type>, const Allocator&);
+      using M = std::flat_multimap<int, int, std::greater<int>, std::deque<int, A>, std::vector<int, A>>;
+      A a;
+      M m({{5, 2}, {2, 2}, {2, 2}, {3, 3}, {1, 1}, {3, 3}}, a);
+      assert(std::equal(m.rbegin(), m.rend(), expected, expected + 6));
+    }
+  }
+  {
+    // flat_multimap(initializer_list<value_type>, const key_compare&);
+    using C = test_less<int>;
+    using M = std::flat_multimap<int, short, C>;
+    auto m  = M({{5, 2}, {2, 2}, {2, 2}, {3, 3}, {1, 1}, {3, 3}}, C(10));
+    assert(std::ranges::equal(m, expected));
+    assert(m.key_comp() == C(10));
+
+    // explicit(false)
+    M m2 = {{{5, 2}, {2, 2}, {2, 2}, {3, 3}, {1, 1}, {3, 3}}, C(10)};
+    assert(m2 == m);
+    assert(m2.key_comp() == C(10));
+  }
+  {
+    // flat_multimap(initializer_list<value_type>, const key_compare&);
+    // Sorting uses the comparator that was passed in
+    using M = std::flat_multimap<int, short, std::function<bool(int, int)>, std::deque<int, min_allocator<int>>>;
+    auto m  = M({{5, 2}, {2, 2}, {2, 2}, {3, 3}, {1, 1}, {3, 3}}, std::greater<int>());
+    assert(std::equal(m.rbegin(), m.rend(), expected, expected + 6));
+    assert(m.key_comp()(2, 1) == true);
+  }
+  {
+    // flat_multimap(initializer_list<value_type> il, const key_compare& comp, const Alloc& a);
+    using A = explicit_allocator<int>;
+    using M = std::flat_multimap<int, int, std::greater<int>, std::deque<int, A>, std::vector<int, A>>;
+    A a;
+    M m({{5, 2}, {2, 2}, {2, 2}, {3, 3}, {1, 1}, {3, 3}}, {}, a);
+    assert(std::equal(m.rbegin(), m.rend(), expected, expected + 6));
+  }
+
+  return 0;
+}
diff --git a/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.cons/iter_iter.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.cons/iter_iter.pass.cpp
new file mode 100644
index 0000000000000..c9c5e6c99d1c8
--- /dev/null
+++ b/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.cons/iter_iter.pass.cpp
@@ -0,0 +1,154 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
+
+// <flat_map>
+
+// template <class InputIterator>
+//   flat_multimap(InputIterator first, InputIterator last, const key_compare& comp = key_compare());
+// template<class InputIterator, class Allocator>
+//   flat_multimap(InputIterator first, InputIterator last, const Allocator& a);
+// template<class InputIterator, class Allocator>
+//   flat_multimap(InputIterator first, InputIterator last, const key_compare& comp, const Allocator& a);
+
+#include <algorithm>
+#include <deque>
+#include <flat_map>
+#include <functional>
+#include <vector>
+
+#include "min_allocator.h"
+#include "test_allocator.h"
+#include "test_iterators.h"
+#include "test_macros.h"
+#include "../../../test_compare.h"
+
+int main(int, char**) {
+  {
+    // The constructors in this subclause shall not participate in overload
+    // resolution unless uses_allocator_v<key_container_type, Alloc> is true
+    // and uses_allocator_v<mapped_container_type, Alloc> is true.
+
+    using C     = test_less<int>;
+    using A1    = test_allocator<int>;
+    using A2    = other_allocator<int>;
+    using V1    = std::vector<int, A1>;
+    using V2    = std::vector<int, A2>;
+    using M1    = std::flat_multimap<int, int, C, V1, V1>;
+    using M2    = std::flat_multimap<int, int, C, V1, V2>;
+    using M3    = std::flat_multimap<int, int, C, V2, V1>;
+    using Iter1 = typename M1::iterator;
+    using Iter2 = typename M2::iterator;
+    using Iter3 = typename M3::iterator;
+    static_assert(std::is_constructible_v<M1, Iter1, Iter1, const A1&>);
+    static_assert(!std::is_constructible_v<M1, Iter1, Iter1, const A2&>);
+    static_assert(!std::is_constructible_v<M2, Iter2, Iter2, const A2&>);
+    static_assert(!std::is_constructible_v<M3, Iter3, Iter3, const A2&>);
+
+    static_assert(std::is_constructible_v<M1, Iter1, Iter1, const C&, const A1&>);
+    static_assert(!std::is_constructible_v<M1, Iter1, Iter1, const C&, const A2&>);
+    static_assert(!std::is_constructible_v<M2, Iter2, Iter2, const C&, const A2&>);
+    static_assert(!std::is_constructible_v<M3, Iter3, Iter3, const C&, const A2&>);
+  }
+
+  using P      = std::pair<int, short>;
+  P ar[]       = {{1, 1}, {1, 2}, {1, 3}, {2, 4}, {2, 5}, {3, 6}, {2, 7}, {3, 8}, {3, 9}};
+  P expected[] = {{1, 1}, {1, 2}, {1, 3}, {2, 4}, {2, 5}, {2, 7}, {3, 6}, {3, 8}, {3, 9}};
+  {
+    // flat_multimap(InputIterator , InputIterator)
+    // cpp17_input_iterator
+    using M = std::flat_multimap<int, short>;
+    auto m  = M(cpp17_input_iterator<const P*>(ar), cpp17_input_iterator<const P*>(ar + 9));
+    assert(std::ranges::equal(m.keys(), expected | std::views::elements<0>));
+    LIBCPP_ASSERT(std::ranges::equal(m, expected));
+
+    // explicit(false)
+    M m2 = {cpp17_input_iterator<const P*>(ar), cpp17_input_iterator<const P*>(ar + 9)};
+    assert(m2 == m);
+  }
+  {
+    // flat_multimap(InputIterator , InputIterator)
+    // greater
+    using M = std::flat_multimap<int, short, std::greater<int>, std::deque<int, min_allocator<int>>, std::deque<short>>;
+    auto m  = M(cpp17_input_iterator<const P*>(ar), cpp17_input_iterator<const P*>(ar + 9));
+    assert((m.keys() == std::deque<int, min_allocator<int>>{3, 3, 3, 2, 2, 2, 1, 1, 1}));
+    LIBCPP_ASSERT((m.values() == std::deque<short>{6, 8, 9, 4, 5, 7, 1, 2, 3}));
+  }
+  {
+    // flat_multimap(InputIterator , InputIterator)
+    // Test when the operands are of array type (also contiguous iterator type)
+    using M = std::flat_multimap<int, short, std::greater<int>, std::vector<int, min_allocator<int>>>;
+    auto m  = M(ar, ar);
+    assert(m.empty());
+  }
+  {
+    // flat_multimap(InputIterator , InputIterator, const key_compare&)
+    using C = test_less<int>;
+    using M = std::flat_multimap<int, short, C, std::vector<int>, std::deque<short>>;
+    auto m  = M(ar, ar + 9, C(3));
+    assert(std::ranges::equal(m.keys(), expected | std::views::elements<0>));
+    LIBCPP_ASSERT(std::ranges::equal(m, expected));
+    assert(m.key_comp() == C(3));
+
+    // explicit(false)
+    M m2 = {ar, ar + 9, C(3)};
+    assert(m2 == m);
+    assert(m2.key_comp() == C(3));
+  }
+  {
+    // flat_multimap(InputIterator , InputIterator, const Allocator&)
+    using A1 = test_allocator<int>;
+    using A2 = test_allocator<short>;
+    using M  = std::flat_multimap<int, short, std::less<int>, std::vector<int, A1>, std::deque<short, A2>>;
+    auto m   = M(ar, ar + 9, A1(5));
+    assert(std::ranges::equal(m.keys(), expected | std::views::elements<0>));
+    LIBCPP_ASSERT(std::ranges::equal(m, expected));
+    assert(m.keys().get_allocator() == A1(5));
+    assert(m.values().get_allocator() == A2(5));
+  }
+  {
+    // flat_multimap(InputIterator , InputIterator, const Allocator&)
+    // explicit(false)
+    using A1 = test_allocator<int>;
+    using A2 = test_allocator<short>;
+    using M  = std::flat_multimap<int, short, std::less<int>, std::vector<int, A1>, std::deque<short, A2>>;
+    M m      = {ar, ar + 9, A1(5)}; // implicit ctor
+    assert(std::ranges::equal(m.keys(), expected | std::views::elements<0>));
+    LIBCPP_ASSERT(std::ranges::equal(m, expected));
+    assert(m.keys().get_allocator() == A1(5));
+    assert(m.values().get_allocator() == A2(5));
+  }
+  {
+    // flat_multimap(InputIterator , InputIterator, const key_compare&, const Allocator&)
+    using C  = test_less<int>;
+    using A1 = test_allocator<int>;
+    using A2 = test_allocator<short>;
+    using M  = std::flat_multimap<int, short, C, std::vector<int, A1>, std::deque<short, A2>>;
+    auto m   = M(ar, ar + 9, C(3), A1(5));
+    assert(std::ranges::equal(m.keys(), expected | std::views::elements<0>));
+    LIBCPP_ASSERT(std::ranges::equal(m, expected));
+    assert(m.key_comp() == C(3));
+    assert(m.keys().get_allocator() == A1(5));
+    assert(m.values().get_allocator() == A2(5));
+  }
+  {
+    // flat_multimap(InputIterator , InputIterator, const key_compare&, const Allocator&)
+    // explicit(false)
+    using A1 = test_allocator<int>;
+    using A2 = test_allocator<short>;
+    using M  = std::flat_multimap<int, short, std::less<int>, std::deque<int, A1>, std::vector<short, A2>>;
+    M m      = {ar, ar + 9, {}, A2(5)}; // implicit ctor
+    assert(std::ranges::equal(m.keys(), expected | std::views::elements<0>));
+    LIBCPP_ASSERT(std::ranges::equal(m, expected));
+    assert(m.keys().get_allocator() == A1(5));
+    assert(m.values().get_allocator() == A2(5));
+  }
+
+  return 0;
+}
diff --git a/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.cons/move.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.cons/move.pass.cpp
new file mode 100644
index 0000000000000..893c9247959d6
--- /dev/null
+++ b/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.cons/move.pass.cpp
@@ -0,0 +1,89 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
+
+// <flat_map>
+
+// flat_multimap(flat_multimap&&);
+
+#include <algorithm>
+#include <deque>
+#include <flat_map>
+#include <functional>
+#include <utility>
+#include <vector>
+
+#include "../helpers.h"
+#include "test_macros.h"
+#include "../../../test_compare.h"
+#include "test_allocator.h"
+#include "min_allocator.h"
+
+int main(int, char**) {
+  {
+    using C = test_less<int>;
+    using A = test_allocator<int>;
+    using M = std::flat_multimap<int, int, C, std::vector<int, A>, std::deque<int, A>>;
+    M mo    = M({{1, 1}, {1, 2}, {3, 1}}, C(5), A(7));
+    M m     = std::move(mo);
+    assert((m == M{{1, 1}, {1, 2}, {3, 1}}));
+    assert(m.key_comp() == C(5));
+    assert(m.keys().get_allocator() == A(7));
+    assert(m.values().get_allocator() == A(7));
+
+    assert(mo.empty());
+    assert(mo.key_comp() == C(5));
+    assert(mo.keys().get_allocator().get_id() == test_alloc_base::moved_value);
+    assert(mo.values().get_allocator().get_id() == test_alloc_base::moved_value);
+  }
+  {
+    using C = test_less<int>;
+    using A = min_allocator<int>;
+    using M = std::flat_multimap<int, int, C, std::vector<int, A>, std::deque<int, A>>;
+    M mo    = M({{1, 1}, {1, 2}, {3, 1}}, C(5), A());
+    M m     = std::move(mo);
+    assert((m == M{{1, 1}, {1, 2}, {3, 1}}));
+    assert(m.key_comp() == C(5));
+    assert(m.keys().get_allocator() == A());
+    assert(m.values().get_allocator() == A());
+
+    assert(mo.empty());
+    assert(mo.key_comp() == C(5));
+    assert(m.keys().get_allocator() == A());
+    assert(m.values().get_allocator() == A());
+  }
+  {
+    // A moved-from flat_multimap maintains its class invariant in the presence of moved-from comparators.
+    using M = std::flat_multimap<int, int, std::function<bool(int, int)>>;
+    M mo    = M({{1, 1}, {1, 2}, {3, 1}}, std::less<int>());
+    M m     = std::move(mo);
+    assert(m.size() == 3);
+    assert(std::is_sorted(m.begin(), m.end(), m.value_comp()));
+    assert(m.key_comp()(1, 2) == true);
+
+    assert(std::is_sorted(mo.begin(), mo.end(), mo.value_comp()));
+    LIBCPP_ASSERT(m.key_comp()(1, 2) == true);
+    LIBCPP_ASSERT(mo.empty());
+    mo.insert({{1, 1}, {1, 2}, {3, 1}}); // insert has no preconditions
+    assert(m == mo);
+  }
+  {
+    // moved-from object maintains invariant if one of underlying container does not clear after move
+    using M = std::flat_multimap<int, int, std::less<>, std::vector<int>, CopyOnlyVector<int>>;
+    M m1    = M({1, 1, 3}, {1, 2, 3});
+    M m2    = std::move(m1);
+    assert(m2.size() == 3);
+    check_invariant(m1);
+    LIBCPP_ASSERT(m1.empty());
+    LIBCPP_ASSERT(m1.keys().size() == 0);
+    LIBCPP_ASSERT(m1.values().size() == 0);
+  }
+
+  return 0;
+}
diff --git a/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.cons/move_alloc.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.cons/move_alloc.pass.cpp
new file mode 100644
index 0000000000000..a0259e805ac5a
--- /dev/null
+++ b/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.cons/move_alloc.pass.cpp
@@ -0,0 +1,82 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
+
+// <flat_map>
+
+// flat_multimap(flat_multimap&&, const allocator_type&);
+
+#include <algorithm>
+#include <deque>
+#include <flat_map>
+#include <functional>
+#include <ranges>
+#include <vector>
+
+#include "../helpers.h"
+#include "test_macros.h"
+#include "../../../test_compare.h"
+#include "test_allocator.h"
+
+int main(int, char**) {
+  {
+    // The constructors in this subclause shall not participate in overload
+    // resolution unless uses_allocator_v<key_container_type, Alloc> is true
+    // and uses_allocator_v<mapped_container_type, Alloc> is true.
+
+    using C  = test_less<int>;
+    using A1 = test_allocator<int>;
+    using A2 = other_allocator<int>;
+    using V1 = std::vector<int, A1>;
+    using V2 = std::vector<int, A2>;
+    using M1 = std::flat_multimap<int, int, C, V1, V1>;
+    using M2 = std::flat_multimap<int, int, C, V1, V2>;
+    using M3 = std::flat_multimap<int, int, C, V2, V1>;
+    static_assert(std::is_constructible_v<M1, M1&&, const A1&>);
+    static_assert(!std::is_constructible_v<M1, M1&&, const A2&>);
+    static_assert(!std::is_constructible_v<M2, M2&&, const A2&>);
+    static_assert(!std::is_constructible_v<M3, M3&&, const A2&>);
+  }
+  {
+    std::pair<int, int> expected[] = {{1, 1}, {1, 2}, {2, 3}, {2, 2}, {3, 1}};
+    using C                        = test_less<int>;
+    using A                        = test_allocator<int>;
+    using M                        = std::flat_multimap<int, int, C, std::vector<int, A>, std::deque<int, A>>;
+    auto mo                        = M(expected, expected + 5, C(5), A(7));
+    auto m                         = M(std::move(mo), A(3));
+
+    assert(m.key_comp() == C(5));
+    assert(m.size() == 5);
+    auto [keys, values] = std::move(m).extract();
+    assert(keys.get_allocator() == A(3));
+    assert(values.get_allocator() == A(3));
+    assert(std::ranges::equal(keys, expected | std::views::elements<0>));
+    assert(std::ranges::equal(values, expected | std::views::elements<1>));
+
+    // The original flat_multimap is moved-from.
+    assert(std::is_sorted(mo.begin(), mo.end(), mo.value_comp()));
+    assert(mo.empty());
+    assert(mo.key_comp() == C(5));
+    assert(mo.keys().get_allocator() == A(7));
+    assert(mo.values().get_allocator() == A(7));
+  }
+  {
+    // moved-from object maintains invariant if one of underlying container does not clear after move
+    using M = std::flat_multimap<int, int, std::less<>, std::vector<int>, CopyOnlyVector<int>>;
+    M m1    = M({1, 1, 3}, {1, 2, 3});
+    M m2(std::move(m1), std::allocator<int>{});
+    assert(m2.size() == 3);
+    check_invariant(m1);
+    LIBCPP_ASSERT(m1.empty());
+    LIBCPP_ASSERT(m1.keys().size() == 0);
+    LIBCPP_ASSERT(m1.values().size() == 0);
+  }
+
+  return 0;
+}
diff --git a/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.cons/move_assign.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.cons/move_assign.pass.cpp
new file mode 100644
index 0000000000000..38200d008c78a
--- /dev/null
+++ b/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.cons/move_assign.pass.cpp
@@ -0,0 +1,74 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
+
+// <flat_map>
+
+// flat_multimap& operator=(flat_multimap&&);
+
+#include <algorithm>
+#include <deque>
+#include <flat_map>
+#include <functional>
+#include <string>
+#include <utility>
+#include <vector>
+
+#include "test_macros.h"
+#include "MoveOnly.h"
+#include "../../../test_compare.h"
+#include "test_allocator.h"
+#include "min_allocator.h"
+
+int main(int, char**) {
+  {
+    using C  = test_less<int>;
+    using A1 = test_allocator<int>;
+    using A2 = test_allocator<char>;
+    using M  = std::flat_multimap<int, char, C, std::vector<int, A1>, std::vector<char, A2>>;
+    M mo     = M({{1, 1}, {1, 3}, {3, 2}}, C(5), A1(7));
+    M m      = M({}, C(3), A1(7));
+    m        = std::move(mo);
+    assert((m == M{{1, 1}, {1, 3}, {3, 2}}));
+    assert(m.key_comp() == C(5));
+    auto [ks, vs] = std::move(m).extract();
+    assert(ks.get_allocator() == A1(7));
+    assert(vs.get_allocator() == A2(7));
+    assert(mo.empty());
+  }
+  {
+    using C  = test_less<int>;
+    using A1 = other_allocator<int>;
+    using A2 = other_allocator<char>;
+    using M  = std::flat_multimap<int, char, C, std::deque<int, A1>, std::deque<char, A2>>;
+    M mo     = M({{4, 5}, {4, 4}}, C(5), A1(7));
+    M m      = M({{1, 1}, {1, 2}, {1, 3}, {4, 4}}, C(3), A1(7));
+    m        = std::move(mo);
+    assert((m == M{{4, 5}, {4, 4}}));
+    assert(m.key_comp() == C(5));
+    auto [ks, vs] = std::move(m).extract();
+    assert(ks.get_allocator() == A1(7));
+    assert(vs.get_allocator() == A2(7));
+    assert(mo.empty());
+  }
+  {
+    using A = min_allocator<int>;
+    using M = std::flat_multimap<int, int, std::greater<int>, std::vector<int, A>, std::vector<int, A>>;
+    M mo    = M({{5, 1}, {5, 2}, {3, 3}}, A());
+    M m     = M({{4, 4}, {4, 3}, {4, 2}, {1, 1}}, A());
+    m       = std::move(mo);
+    assert((m == M{{5, 1}, {5, 2}, {3, 3}}));
+    auto [ks, vs] = std::move(m).extract();
+    assert(ks.get_allocator() == A());
+    assert(vs.get_allocator() == A());
+    assert(mo.empty());
+  }
+
+  return 0;
+}
diff --git a/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.cons/move_assign_clears.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.cons/move_assign_clears.pass.cpp
new file mode 100644
index 0000000000000..bc65dca32899c
--- /dev/null
+++ b/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.cons/move_assign_clears.pass.cpp
@@ -0,0 +1,101 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
+
+// <flat_map>
+
+// flat_multimap& operator=(flat_multimap&&);
+// Preserves the class invariant for the moved-from flat_multimap.
+
+#include <algorithm>
+#include <cassert>
+#include <compare>
+#include <flat_map>
+#include <functional>
+#include <utility>
+#include <vector>
+
+#include "../helpers.h"
+#include "test_macros.h"
+
+struct MoveNegates {
+  int value_    = 0;
+  MoveNegates() = default;
+  MoveNegates(int v) : value_(v) {}
+  MoveNegates(MoveNegates&& rhs) : value_(rhs.value_) { rhs.value_ = -rhs.value_; }
+  MoveNegates& operator=(MoveNegates&& rhs) {
+    value_     = rhs.value_;
+    rhs.value_ = -rhs.value_;
+    return *this;
+  }
+  ~MoveNegates()                             = default;
+  auto operator<=>(const MoveNegates&) const = default;
+};
+
+struct MoveClears {
+  int value_   = 0;
+  MoveClears() = default;
+  MoveClears(int v) : value_(v) {}
+  MoveClears(MoveClears&& rhs) : value_(rhs.value_) { rhs.value_ = 0; }
+  MoveClears& operator=(MoveClears&& rhs) {
+    value_     = rhs.value_;
+    rhs.value_ = 0;
+    return *this;
+  }
+  ~MoveClears()                             = default;
+  auto operator<=>(const MoveClears&) const = default;
+};
+
+int main(int, char**) {
+  {
+    const std::pair<int, int> expected[] = {{1, 1}, {1, 2}, {3, 3}, {3, 4}, {5, 5}, {6, 6}, {7, 7}, {8, 8}};
+    using M = std::flat_multimap<MoveNegates, int, std::less<MoveNegates>, std::vector<MoveNegates>>;
+    M m     = M(expected, expected + 8);
+    M m2    = M(expected, expected + 3);
+
+    m2 = std::move(m);
+
+    assert(std::equal(m2.begin(), m2.end(), expected, expected + 8));
+    LIBCPP_ASSERT(m.empty());
+    check_invariant(m);
+    m.insert({1, 1});
+    m.insert({2, 2});
+    assert(m.contains(1));
+    assert(m.find(2) != m.end());
+  }
+  {
+    const std::pair<int, int> expected[] = {{1, 1}, {1, 2}, {3, 3}, {4, 4}, {5, 5}, {5, 6}, {7, 7}, {8, 8}};
+    using M = std::flat_multimap<MoveClears, int, std::less<MoveClears>, std::vector<MoveClears>>;
+    M m     = M(expected, expected + 8);
+    M m2    = M(expected, expected + 3);
+
+    m2 = std::move(m);
+
+    assert(std::equal(m2.begin(), m2.end(), expected, expected + 8));
+    LIBCPP_ASSERT(m.empty());
+    check_invariant(m);
+    m.insert({1, 1});
+    m.insert({2, 2});
+    assert(m.contains(1));
+    assert(m.find(2) != m.end());
+  }
+  {
+    // moved-from object maintains invariant if one of underlying container does not clear after move
+    using M = std::flat_multimap<int, int, std::less<>, std::vector<int>, CopyOnlyVector<int>>;
+    M m1    = M({1, 1, 3}, {1, 2, 3});
+    M m2    = M({1, 1}, {1, 2});
+    m2      = std::move(m1);
+    assert(m2.size() == 3);
+    check_invariant(m1);
+    LIBCPP_ASSERT(m1.empty());
+    LIBCPP_ASSERT(m1.keys().size() == 0);
+    LIBCPP_ASSERT(m1.values().size() == 0);
+  }
+  return 0;
+}
diff --git a/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.cons/move_assign_noexcept.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.cons/move_assign_noexcept.pass.cpp
new file mode 100644
index 0000000000000..4eb58313f6f72
--- /dev/null
+++ b/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.cons/move_assign_noexcept.pass.cpp
@@ -0,0 +1,110 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
+
+// <flat_map>
+
+// flat_multimap& operator=(flat_multimap&& c)
+//     noexcept(
+//          is_nothrow_move_assignable<key_container_type>::value &&
+//          is_nothrow_move_assignable<mapped_container_type>::value &&
+//          is_nothrow_copy_assignable<key_compare>::value);
+
+// This tests a conforming extension
+
+#include <flat_map>
+#include <functional>
+#include <memory_resource>
+#include <type_traits>
+#include <vector>
+
+#include "MoveOnly.h"
+#include "test_allocator.h"
+#include "test_macros.h"
+
+struct MoveSensitiveComp {
+  MoveSensitiveComp() noexcept(false)                         = default;
+  MoveSensitiveComp(const MoveSensitiveComp&) noexcept(false) = default;
+  MoveSensitiveComp(MoveSensitiveComp&& rhs) { rhs.is_moved_from_ = true; }
+  MoveSensitiveComp& operator=(const MoveSensitiveComp&) noexcept = default;
+  MoveSensitiveComp& operator=(MoveSensitiveComp&& rhs) {
+    rhs.is_moved_from_ = true;
+    return *this;
+  }
+  bool operator()(const auto&, const auto&) const { return false; }
+  bool is_moved_from_ = false;
+};
+
+struct MoveThrowsComp {
+  MoveThrowsComp(MoveThrowsComp&&) noexcept(false);
+  MoveThrowsComp(const MoveThrowsComp&) noexcept(true);
+  MoveThrowsComp& operator=(MoveThrowsComp&&) noexcept(false);
+  MoveThrowsComp& operator=(const MoveThrowsComp&) noexcept(true);
+  bool operator()(const auto&, const auto&) const;
+};
+
+int main(int, char**) {
+  {
+    using C = std::flat_multimap<int, int>;
+    LIBCPP_STATIC_ASSERT(std::is_nothrow_move_assignable_v<C>);
+  }
+  {
+    using C =
+        std::flat_multimap<MoveOnly,
+                           int,
+                           std::less<MoveOnly>,
+                           std::vector<MoveOnly, test_allocator<MoveOnly>>,
+                           std::vector<int, test_allocator<int>>>;
+    static_assert(!std::is_nothrow_move_assignable_v<C>);
+  }
+  {
+    using C =
+        std::flat_multimap<int,
+                           MoveOnly,
+                           std::less<int>,
+                           std::vector<int, test_allocator<int>>,
+                           std::vector<MoveOnly, test_allocator<MoveOnly>>>;
+    static_assert(!std::is_nothrow_move_assignable_v<C>);
+  }
+  {
+    using C =
+        std::flat_multimap<MoveOnly,
+                           int,
+                           std::less<MoveOnly>,
+                           std::vector<MoveOnly, other_allocator<MoveOnly>>,
+                           std::vector<int, other_allocator<int>>>;
+    LIBCPP_STATIC_ASSERT(std::is_nothrow_move_assignable_v<C>);
+  }
+  {
+    using C =
+        std::flat_multimap<int,
+                           MoveOnly,
+                           std::less<int>,
+                           std::vector<int, other_allocator<int>>,
+                           std::vector<MoveOnly, other_allocator<MoveOnly>>>;
+    LIBCPP_STATIC_ASSERT(std::is_nothrow_move_assignable_v<C>);
+  }
+  {
+    // Test with a comparator that throws on move-assignment.
+    using C = std::flat_multimap<int, int, MoveThrowsComp>;
+    LIBCPP_STATIC_ASSERT(!std::is_nothrow_move_assignable_v<C>);
+  }
+  {
+    // Test with a container that throws on move-assignment.
+    using C = std::flat_multimap<int, int, std::less<int>, std::pmr::vector<int>, std::vector<int>>;
+    static_assert(!std::is_nothrow_move_assignable_v<C>);
+  }
+  {
+    // Test with a container that throws on move-assignment.
+    using C = std::flat_multimap<int, int, std::less<int>, std::vector<int>, std::pmr::vector<int>>;
+    static_assert(!std::is_nothrow_move_assignable_v<C>);
+  }
+
+  return 0;
+}
diff --git a/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.cons/move_exceptions.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.cons/move_exceptions.pass.cpp
new file mode 100644
index 0000000000000..c2085e32be532
--- /dev/null
+++ b/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.cons/move_exceptions.pass.cpp
@@ -0,0 +1,71 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
+// UNSUPPORTED: no-exceptions
+
+// <flat_map>
+
+// flat_multimap(flat_multimap&& s);
+// If any member function in [flat.multimap.defn] exits via an exception, the invariant is restored.
+
+#include <algorithm>
+#include <cassert>
+#include <flat_map>
+#include <functional>
+#include <utility>
+#include <vector>
+
+#include "../helpers.h"
+#include "test_macros.h"
+
+static int countdown = 0;
+
+struct EvilContainer : std::vector<int> {
+  EvilContainer() = default;
+  EvilContainer(EvilContainer&& rhs) {
+    // Throw on move-construction.
+    if (--countdown == 0) {
+      rhs.insert(rhs.end(), 0);
+      rhs.insert(rhs.end(), 0);
+      throw 42;
+    }
+  }
+};
+
+int main(int, char**) {
+  {
+    using M   = std::flat_multimap<int, int, std::less<int>, EvilContainer, std::vector<int>>;
+    M mo      = {{1, 1}, {1, 2}, {3, 3}};
+    countdown = 1;
+    try {
+      M m = std::move(mo);
+      assert(false); // not reached
+    } catch (int x) {
+      assert(x == 42);
+    }
+    // The source flat_multimap maintains its class invariant.
+    check_invariant(mo);
+    LIBCPP_ASSERT(mo.empty());
+  }
+  {
+    using M   = std::flat_multimap<int, int, std::less<int>, std::vector<int>, EvilContainer>;
+    M mo      = {{1, 1}, {1, 2}, {3, 3}};
+    countdown = 1;
+    try {
+      M m = std::move(mo);
+      assert(false); // not reached
+    } catch (int x) {
+      assert(x == 42);
+    }
+    // The source flat_multimap maintains its class invariant.
+    check_invariant(mo);
+    LIBCPP_ASSERT(mo.empty());
+  }
+  return 0;
+}
diff --git a/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.cons/move_noexcept.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.cons/move_noexcept.pass.cpp
new file mode 100644
index 0000000000000..e038902e26d52
--- /dev/null
+++ b/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.cons/move_noexcept.pass.cpp
@@ -0,0 +1,104 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
+
+// <flat_map>
+
+// flat_multimap(flat_multimap&&)
+//        noexcept(is_nothrow_move_constructible<key_container_type>::value &&
+//                 is_nothrow_move_constructible<mapped_container_type>::value &&
+//                 is_nothrow_copy_constructible<key_compare>::value);
+
+// This tests a conforming extension
+
+#include <cassert>
+#include <deque>
+#include <flat_map>
+#include <functional>
+#include <memory>
+#include <type_traits>
+#include <vector>
+
+#include "test_macros.h"
+#include "MoveOnly.h"
+#include "test_allocator.h"
+
+template <class T>
+struct ThrowingMoveAllocator {
+  using value_type                                    = T;
+  explicit ThrowingMoveAllocator()                    = default;
+  ThrowingMoveAllocator(const ThrowingMoveAllocator&) = default;
+  ThrowingMoveAllocator(ThrowingMoveAllocator&&) noexcept(false) {}
+  T* allocate(std::ptrdiff_t n) { return std::allocator<T>().allocate(n); }
+  void deallocate(T* p, std::ptrdiff_t n) { return std::allocator<T>().deallocate(p, n); }
+  friend bool operator==(ThrowingMoveAllocator, ThrowingMoveAllocator) = default;
+};
+
+struct ThrowingMoveComp {
+  ThrowingMoveComp() = default;
+  ThrowingMoveComp(const ThrowingMoveComp&) noexcept(true) {}
+  ThrowingMoveComp(ThrowingMoveComp&&) noexcept(false) {}
+  bool operator()(const auto&, const auto&) const { return false; }
+};
+
+struct MoveSensitiveComp {
+  MoveSensitiveComp() noexcept(false)                  = default;
+  MoveSensitiveComp(const MoveSensitiveComp&) noexcept = default;
+  MoveSensitiveComp(MoveSensitiveComp&& rhs) { rhs.is_moved_from_ = true; }
+  MoveSensitiveComp& operator=(const MoveSensitiveComp&) noexcept(false) = default;
+  MoveSensitiveComp& operator=(MoveSensitiveComp&& rhs) {
+    rhs.is_moved_from_ = true;
+    return *this;
+  }
+  bool operator()(const auto&, const auto&) const { return false; }
+  bool is_moved_from_ = false;
+};
+
+int main(int, char**) {
+  {
+    using C = std::flat_multimap<int, int>;
+    LIBCPP_STATIC_ASSERT(std::is_nothrow_move_constructible_v<C>);
+    C c;
+    C d = std::move(c);
+  }
+  {
+    using C = std::flat_multimap<int, int, std::less<int>, std::deque<int, test_allocator<int>>>;
+    LIBCPP_STATIC_ASSERT(std::is_nothrow_move_constructible_v<C>);
+    C c;
+    C d = std::move(c);
+  }
+#if _LIBCPP_VERSION
+  {
+    // Container fails to be nothrow-move-constructible; this relies on libc++'s support for non-nothrow-copyable allocators
+    using C =
+        std::flat_multimap<int, int, std::less<int>, std::deque<int, ThrowingMoveAllocator<int>>, std::vector<int>>;
+    static_assert(!std::is_nothrow_move_constructible_v<std::deque<int, ThrowingMoveAllocator<int>>>);
+    static_assert(!std::is_nothrow_move_constructible_v<C>);
+    C c;
+    C d = std::move(c);
+  }
+  {
+    // Container fails to be nothrow-move-constructible; this relies on libc++'s support for non-nothrow-copyable allocators
+    using C =
+        std::flat_multimap<int, int, std::less<int>, std::vector<int>, std::deque<int, ThrowingMoveAllocator<int>>>;
+    static_assert(!std::is_nothrow_move_constructible_v<std::deque<int, ThrowingMoveAllocator<int>>>);
+    static_assert(!std::is_nothrow_move_constructible_v<C>);
+    C c;
+    C d = std::move(c);
+  }
+#endif // _LIBCPP_VERSION
+  {
+    // Comparator fails to be nothrow-move-constructible
+    using C = std::flat_multimap<int, int, ThrowingMoveComp>;
+    static_assert(!std::is_nothrow_move_constructible_v<C>);
+    C c;
+    C d = std::move(c);
+  }
+  return 0;
+}
diff --git a/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.cons/pmr.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.cons/pmr.pass.cpp
new file mode 100644
index 0000000000000..8b518f6afbda9
--- /dev/null
+++ b/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.cons/pmr.pass.cpp
@@ -0,0 +1,361 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
+// UNSUPPORTED: availability-pmr-missing
+
+// <flat_map>
+
+// Test various constructors with pmr
+
+#include <algorithm>
+#include <cassert>
+#include <deque>
+#include <flat_map>
+#include <functional>
+#include <memory_resource>
+#include <ranges>
+#include <vector>
+#include <string>
+
+#include "test_iterators.h"
+#include "test_macros.h"
+#include "test_allocator.h"
+#include "../../../test_compare.h"
+
+int main(int, char**) {
+  {
+    // flat_multimap(const Allocator& a);
+    using M = std::flat_multimap<int, short, std::less<int>, std::pmr::vector<int>, std::pmr::vector<short>>;
+    std::pmr::monotonic_buffer_resource mr;
+    std::pmr::polymorphic_allocator<int> pa = &mr;
+    auto m1                                 = M(pa);
+    assert(m1.empty());
+    assert(m1.keys().get_allocator() == pa);
+    assert(m1.values().get_allocator() == pa);
+    auto m2 = M(&mr);
+    assert(m2.empty());
+    assert(m2.keys().get_allocator() == pa);
+    assert(m2.values().get_allocator() == pa);
+  }
+  {
+    // flat_multimap(const key_compare& comp, const Alloc& a);
+    using M = std::flat_multimap<int, int, std::function<bool(int, int)>, std::pmr::vector<int>, std::pmr::vector<int>>;
+    std::pmr::monotonic_buffer_resource mr;
+    std::pmr::vector<M> vm(&mr);
+    vm.emplace_back(std::greater<int>());
+    assert(vm[0] == M{});
+    assert(vm[0].key_comp()(2, 1) == true);
+    assert(vm[0].value_comp()({2, 0}, {1, 0}) == true);
+    assert(vm[0].keys().get_allocator().resource() == &mr);
+    assert(vm[0].values().get_allocator().resource() == &mr);
+  }
+  {
+    // flat_multimap(const key_container_type& key_cont, const mapped_container_type& mapped_cont,
+    //          const Allocator& a);
+    using M = std::flat_multimap<int, int, std::less<int>, std::pmr::vector<int>, std::pmr::vector<int>>;
+    std::pmr::monotonic_buffer_resource mr;
+    std::pmr::vector<M> vm(&mr);
+    std::pmr::vector<int> ks = {1, 1, 1, 2, 2, 3, 2, 3, 3};
+    std::pmr::vector<int> vs = {1, 1, 1, 2, 2, 3, 2, 3, 3};
+    assert(ks.get_allocator().resource() != &mr);
+    assert(vs.get_allocator().resource() != &mr);
+    vm.emplace_back(ks, vs);
+    assert(ks.size() == 9); // ks' value is unchanged, since it was an lvalue above
+    assert(vs.size() == 9); // vs' value is unchanged, since it was an lvalue above
+    assert((vm[0] == M{{1, 1}, {1, 1}, {1, 1}, {2, 2}, {2, 2}, {2, 2}, {3, 3}, {3, 3}, {3, 3}}));
+    assert(vm[0].keys().get_allocator().resource() == &mr);
+    assert(vm[0].values().get_allocator().resource() == &mr);
+  }
+  {
+    // flat_multimap(const flat_multimap&, const allocator_type&);
+    using C = test_less<int>;
+    using M = std::flat_multimap<int, int, C, std::pmr::vector<int>, std::pmr::vector<int>>;
+    std::pmr::monotonic_buffer_resource mr1;
+    std::pmr::monotonic_buffer_resource mr2;
+    M mo = M({1, 2, 1}, {2, 2, 1}, C(5), &mr1);
+    M m  = {mo, &mr2}; // also test the implicitness of this constructor
+
+    assert(m.key_comp() == C(5));
+    assert((m.keys() == std::pmr::vector<int>{1, 1, 2}));
+    assert((m.values() == std::pmr::vector<int>{2, 1, 2}));
+    assert(m.keys().get_allocator().resource() == &mr2);
+    assert(m.values().get_allocator().resource() == &mr2);
+
+    // mo is unchanged
+    assert(mo.key_comp() == C(5));
+    assert((mo.keys() == std::pmr::vector<int>{1, 1, 2}));
+    assert((mo.values() == std::pmr::vector<int>{2, 1, 2}));
+    assert(mo.keys().get_allocator().resource() == &mr1);
+    assert(mo.values().get_allocator().resource() == &mr1);
+  }
+  {
+    // flat_multimap(const flat_multimap&, const allocator_type&);
+    using M = std::flat_multimap<int, int, std::less<>, std::pmr::vector<int>, std::pmr::deque<int>>;
+    std::pmr::vector<M> vs;
+    M m = {{1, 2}, {1, 2}, {3, 1}};
+    vs.push_back(m);
+    assert(vs[0] == m);
+  }
+  {
+    // flat_multimap& operator=(const flat_multimap& m);
+    // pmr allocator is not propagated
+    using M = std::flat_multimap<int, int, std::less<>, std::pmr::deque<int>, std::pmr::vector<int>>;
+    std::pmr::monotonic_buffer_resource mr1;
+    std::pmr::monotonic_buffer_resource mr2;
+    M mo = M({{1, 1}, {1, 2}, {3, 3}}, &mr1);
+    M m  = M({{4, 4}, {4, 5}}, &mr2);
+    m    = mo;
+    assert((m == M{{1, 1}, {1, 2}, {3, 3}}));
+    assert(m.keys().get_allocator().resource() == &mr2);
+    assert(m.values().get_allocator().resource() == &mr2);
+
+    // mo is unchanged
+    assert((mo == M{{1, 1}, {1, 2}, {3, 3}}));
+    assert(mo.keys().get_allocator().resource() == &mr1);
+  }
+  {
+    // flat_multimap(const flat_multimap& m);
+    using C = test_less<int>;
+    std::pmr::monotonic_buffer_resource mr;
+    using M = std::flat_multimap<int, int, C, std::pmr::vector<int>, std::pmr::vector<int>>;
+    auto mo = M({{1, 1}, {1, 2}, {3, 3}}, C(5), &mr);
+    auto m  = mo;
+
+    assert(m.key_comp() == C(5));
+    assert((m == M{{1, 1}, {1, 2}, {3, 3}}));
+    auto [ks, vs] = std::move(m).extract();
+    assert(ks.get_allocator().resource() == std::pmr::get_default_resource());
+    assert(vs.get_allocator().resource() == std::pmr::get_default_resource());
+
+    // mo is unchanged
+    assert(mo.key_comp() == C(5));
+    assert((mo == M{{1, 1}, {1, 2}, {3, 3}}));
+    auto [kso, vso] = std::move(mo).extract();
+    assert(kso.get_allocator().resource() == &mr);
+    assert(vso.get_allocator().resource() == &mr);
+  }
+  {
+    //  flat_multimap(initializer_list<value_type> il, const Alloc& a);
+    using M = std::flat_multimap<int, int, std::less<int>, std::pmr::vector<int>, std::pmr::vector<int>>;
+    std::pmr::monotonic_buffer_resource mr;
+    std::pmr::vector<M> vm(&mr);
+    std::initializer_list<M::value_type> il = {{3, 3}, {1, 1}, {4, 4}, {1, 1}, {5, 5}};
+    vm.emplace_back(il);
+    assert((vm[0] == M{{1, 1}, {1, 1}, {3, 3}, {4, 4}, {5, 5}}));
+    assert(vm[0].keys().get_allocator().resource() == &mr);
+    assert(vm[0].values().get_allocator().resource() == &mr);
+  }
+  {
+    //  flat_multimap(initializer_list<value_type> il, const key_compare& comp, const Alloc& a);
+    using C = test_less<int>;
+    using M = std::flat_multimap<int, int, C, std::pmr::vector<int>, std::pmr::deque<int>>;
+    std::pmr::monotonic_buffer_resource mr;
+    std::pmr::vector<M> vm(&mr);
+    std::initializer_list<M::value_type> il = {{3, 3}, {1, 1}, {4, 4}, {1, 1}, {5, 5}};
+    vm.emplace_back(il, C(5));
+    assert((vm[0] == M{{1, 1}, {1, 1}, {3, 3}, {4, 4}, {5, 5}}));
+    assert(vm[0].keys().get_allocator().resource() == &mr);
+    assert(vm[0].values().get_allocator().resource() == &mr);
+    assert(vm[0].key_comp() == C(5));
+  }
+  {
+    // flat_multimap(InputIterator first, InputIterator last, const Allocator& a);
+    using P      = std::pair<int, short>;
+    P ar[]       = {{1, 1}, {1, 2}, {1, 3}, {2, 4}, {2, 5}, {3, 6}, {2, 7}, {3, 8}, {3, 9}};
+    P expected[] = {{1, 1}, {1, 2}, {1, 3}, {2, 4}, {2, 5}, {2, 7}, {3, 6}, {3, 8}, {3, 9}};
+    {
+      //  cpp17 iterator
+      using M = std::flat_multimap<int, short, std::less<int>, std::pmr::vector<int>, std::pmr::vector<short>>;
+      std::pmr::monotonic_buffer_resource mr;
+      std::pmr::vector<M> vm(&mr);
+      vm.emplace_back(cpp17_input_iterator<const P*>(ar), cpp17_input_iterator<const P*>(ar + 9));
+      assert(std::ranges::equal(vm[0].keys(), expected | std::views::elements<0>));
+      LIBCPP_ASSERT(std::ranges::equal(vm[0], expected));
+      assert(vm[0].keys().get_allocator().resource() == &mr);
+      assert(vm[0].values().get_allocator().resource() == &mr);
+    }
+    {
+      using M = std::flat_multimap<int, short, std::less<int>, std::pmr::vector<int>, std::pmr::vector<short>>;
+      std::pmr::monotonic_buffer_resource mr;
+      std::pmr::vector<M> vm(&mr);
+      vm.emplace_back(ar, ar);
+      assert(vm[0].empty());
+      assert(vm[0].keys().get_allocator().resource() == &mr);
+      assert(vm[0].values().get_allocator().resource() == &mr);
+    }
+  }
+  {
+    // flat_multimap(flat_multimap&&, const allocator_type&);
+    std::pair<int, int> expected[] = {{1, 1}, {1, 1}, {2, 2}, {3, 1}};
+    using C                        = test_less<int>;
+    using M                        = std::flat_multimap<int, int, C, std::pmr::vector<int>, std::pmr::deque<int>>;
+    std::pmr::monotonic_buffer_resource mr1;
+    std::pmr::monotonic_buffer_resource mr2;
+    M mo = M({{1, 1}, {3, 1}, {1, 1}, {2, 2}}, C(5), &mr1);
+    M m  = {std::move(mo), &mr2}; // also test the implicitness of this constructor
+
+    assert(m.key_comp() == C(5));
+    assert(m.size() == 4);
+    assert(m.keys().get_allocator().resource() == &mr2);
+    assert(m.values().get_allocator().resource() == &mr2);
+    assert(std::ranges::equal(m, expected));
+
+    // The original flat_multimap is moved-from.
+    assert(std::is_sorted(mo.begin(), mo.end(), mo.value_comp()));
+    assert(mo.key_comp() == C(5));
+    assert(mo.keys().get_allocator().resource() == &mr1);
+    assert(mo.values().get_allocator().resource() == &mr1);
+  }
+  {
+    // flat_multimap(flat_multimap&&, const allocator_type&);
+    using M = std::flat_multimap<int, int, std::less<>, std::pmr::deque<int>, std::pmr::vector<int>>;
+    std::pmr::vector<M> vs;
+    M m = {{1, 1}, {3, 1}, {1, 1}, {2, 2}};
+    vs.push_back(std::move(m));
+    assert((vs[0].keys() == std::pmr::deque<int>{1, 1, 2, 3}));
+    assert((vs[0].values() == std::pmr::vector<int>{1, 1, 2, 1}));
+  }
+  {
+    // flat_multimap& operator=(flat_multimap&&);
+    using M = std::
+        flat_multimap<std::pmr::string, int, std::less<>, std::pmr::vector<std::pmr::string>, std::pmr::vector<int>>;
+    std::pmr::monotonic_buffer_resource mr1;
+    std::pmr::monotonic_buffer_resource mr2;
+    M mo = M({{"short", 1},
+              {"very long string that definitely won't fit in the SSO buffer and therefore becomes empty on move", 2}},
+             &mr1);
+    M m  = M({{"don't care", 3}}, &mr2);
+    m    = std::move(mo);
+    assert(m.size() == 2);
+    assert(std::is_sorted(m.begin(), m.end(), m.value_comp()));
+    assert(m.begin()->first.get_allocator().resource() == &mr2);
+
+    assert(std::is_sorted(mo.begin(), mo.end(), mo.value_comp()));
+    mo.insert({"foo", 1});
+    assert(mo.begin()->first.get_allocator().resource() == &mr1);
+  }
+  {
+    //  flat_multimap(from_range_t, R&&, const Alloc&);
+    using P      = std::pair<int, short>;
+    P ar[]       = {{1, 1}, {1, 2}, {1, 3}, {2, 4}, {2, 5}, {3, 6}, {2, 7}, {3, 8}, {3, 9}};
+    P expected[] = {{1, 1}, {1, 2}, {1, 3}, {2, 4}, {2, 5}, {2, 7}, {3, 6}, {3, 8}, {3, 9}};
+    {
+      // input_range
+      using M    = std::flat_multimap<int, short, std::less<int>, std::pmr::vector<int>, std::pmr::vector<short>>;
+      using Iter = cpp20_input_iterator<const P*>;
+      using Sent = sentinel_wrapper<Iter>;
+      using R    = std::ranges::subrange<Iter, Sent>;
+      std::pmr::monotonic_buffer_resource mr;
+      std::pmr::vector<M> vm(&mr);
+      vm.emplace_back(std::from_range, R(Iter(ar), Sent(Iter(ar + 9))));
+      assert(std::ranges::equal(vm[0].keys(), expected | std::views::elements<0>));
+      LIBCPP_ASSERT(std::ranges::equal(vm[0], expected));
+      assert(vm[0].keys().get_allocator().resource() == &mr);
+      assert(vm[0].values().get_allocator().resource() == &mr);
+    }
+    {
+      using M = std::flat_multimap<int, short, std::less<int>, std::pmr::vector<int>, std::pmr::vector<short>>;
+      using R = std::ranges::subrange<const P*>;
+      std::pmr::monotonic_buffer_resource mr;
+      std::pmr::vector<M> vm(&mr);
+      vm.emplace_back(std::from_range, R(ar, ar));
+      assert(vm[0].empty());
+      assert(vm[0].keys().get_allocator().resource() == &mr);
+      assert(vm[0].values().get_allocator().resource() == &mr);
+    }
+  }
+  {
+    // flat_multimap(sorted_equivalent_t, const key_container_type& key_cont,
+    //          const mapped_container_type& mapped_cont, const Alloc& a);
+    using M = std::flat_multimap<int, int, std::less<int>, std::pmr::vector<int>, std::pmr::vector<int>>;
+    std::pmr::monotonic_buffer_resource mr;
+    std::pmr::vector<M> vm(&mr);
+    std::pmr::vector<int> ks = {1, 1, 4, 10};
+    std::pmr::vector<int> vs = {4, 3, 2, 1};
+    vm.emplace_back(std::sorted_equivalent, ks, vs);
+    assert(!ks.empty()); // it was an lvalue above
+    assert(!vs.empty()); // it was an lvalue above
+    assert((vm[0] == M{{1, 4}, {1, 3}, {4, 2}, {10, 1}}));
+    assert(vm[0].keys().get_allocator().resource() == &mr);
+    assert(vm[0].values().get_allocator().resource() == &mr);
+  }
+  {
+    // flat_multimap(sorted_equivalent_t, const key_container_type& key_cont,
+    //          const mapped_container_type& mapped_cont, const Alloc& a);
+    using M = std::flat_multimap<int, int, std::less<int>, std::pmr::vector<int>, std::pmr::vector<int>>;
+    std::pmr::monotonic_buffer_resource mr;
+    std::pmr::vector<M> vm(&mr);
+    std::pmr::vector<int> ks({1, 1, 4, 10}, &mr);
+    std::pmr::vector<int> vs({4, 3, 2, 1}, &mr);
+    vm.emplace_back(std::sorted_equivalent, ks, vs);
+    assert((vm[0] == M{{1, 4}, {1, 3}, {4, 2}, {10, 1}}));
+    assert(vm[0].keys().get_allocator().resource() == &mr);
+    assert(vm[0].values().get_allocator().resource() == &mr);
+  }
+  {
+    // flat_multimap(sorted_equivalent_t, initializer_list<value_type> il, const Alloc& a);
+    // cpp_17
+    using C = test_less<int>;
+    using M = std::flat_multimap<int, int, C, std::pmr::vector<int>, std::pmr::vector<int>>;
+    std::pmr::monotonic_buffer_resource mr;
+    std::pmr::vector<M> vm(&mr);
+    using P = std::pair<int, int>;
+    P ar[]  = {{1, 1}, {1, 2}, {1, 4}, {5, 5}};
+    vm.emplace_back(
+        std::sorted_equivalent, cpp17_input_iterator<const P*>(ar), cpp17_input_iterator<const P*>(ar + 4), C(3));
+    assert((vm[0] == M{{1, 1}, {1, 2}, {1, 4}, {5, 5}}));
+    assert(vm[0].key_comp() == C(3));
+    assert(vm[0].keys().get_allocator().resource() == &mr);
+    assert(vm[0].values().get_allocator().resource() == &mr);
+  }
+  {
+    // flat_multimap(sorted_equivalent_t, initializer_list<value_type> il, const Alloc& a);
+    using C = test_less<int>;
+    using M = std::flat_multimap<int, int, C, std::pmr::vector<int>, std::pmr::vector<int>>;
+    std::pmr::monotonic_buffer_resource mr;
+    std::pmr::vector<M> vm(&mr);
+    std::pair<int, int> ar[1] = {{42, 42}};
+    vm.emplace_back(std::sorted_equivalent, ar, ar, C(4));
+    assert(vm[0] == M{});
+    assert(vm[0].key_comp() == C(4));
+    assert(vm[0].keys().get_allocator().resource() == &mr);
+    assert(vm[0].values().get_allocator().resource() == &mr);
+  }
+  {
+    // flat_multimap(InputIterator first, InputIterator last, const Alloc& a);
+    // cpp_17
+    using C = test_less<int>;
+    using M = std::flat_multimap<int, int, C, std::pmr::vector<int>, std::pmr::vector<int>>;
+    std::pmr::monotonic_buffer_resource mr;
+    std::pmr::vector<M> vm(&mr);
+    using P = std::pair<int, int>;
+    P ar[]  = {{1, 1}, {1, 2}, {1, 4}, {5, 5}};
+    vm.emplace_back(
+        std::sorted_equivalent, cpp17_input_iterator<const P*>(ar), cpp17_input_iterator<const P*>(ar + 4), C(3));
+    assert((vm[0] == M{{1, 1}, {1, 2}, {1, 4}, {5, 5}}));
+    assert(vm[0].key_comp() == C(3));
+    assert(vm[0].keys().get_allocator().resource() == &mr);
+    assert(vm[0].values().get_allocator().resource() == &mr);
+  }
+  {
+    // flat_multimap(InputIterator first, InputIterator last, const Alloc& a);
+    using C = test_less<int>;
+    using M = std::flat_multimap<int, int, C, std::pmr::vector<int>, std::pmr::vector<int>>;
+    std::pmr::monotonic_buffer_resource mr;
+    std::pmr::vector<M> vm(&mr);
+    std::pair<int, int> ar[1] = {{42, 42}};
+    vm.emplace_back(std::sorted_equivalent, ar, ar, C(4));
+    assert(vm[0] == M{});
+    assert(vm[0].key_comp() == C(4));
+    assert(vm[0].keys().get_allocator().resource() == &mr);
+    assert(vm[0].values().get_allocator().resource() == &mr);
+  }
+
+  return 0;
+}
diff --git a/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.cons/range.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.cons/range.pass.cpp
new file mode 100644
index 0000000000000..de750e2506341
--- /dev/null
+++ b/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.cons/range.pass.cpp
@@ -0,0 +1,227 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
+
+// <flat_map>
+
+// template<container-compatible-range<value_type> R>
+//     flat_multimap(from_range_t, R&&)
+// template<container-compatible-range<value_type> R>
+//     flat_multimap(from_range_t, R&&, const key_compare&)
+// template<container-compatible-range<value_type> R, class Alloc>
+//      flat_multimap(from_range_t, R&&, const Alloc&);
+// template<container-compatible-range<value_type> R, class Alloc>
+//      flat_multimap(from_range_t, R&&, const key_compare&, const Alloc&);
+
+#include <algorithm>
+#include <deque>
+#include <flat_map>
+#include <functional>
+#include <string>
+#include <vector>
+
+#include "min_allocator.h"
+#include "test_allocator.h"
+#include "test_iterators.h"
+#include "test_macros.h"
+#include "../../../test_compare.h"
+
+// test constraint container-compatible-range
+
+template <class V>
+using RangeOf = std::ranges::subrange<V*>;
+using Map     = std::flat_multimap<int, double>;
+
+static_assert(std::is_constructible_v<Map, std::from_range_t, RangeOf<std::pair<int, double>>>);
+static_assert(std::is_constructible_v<Map, std::from_range_t, RangeOf<std::pair<short, double>>>);
+static_assert(!std::is_constructible_v<Map, std::from_range_t, RangeOf<int>>);
+static_assert(!std::is_constructible_v<Map, std::from_range_t, RangeOf<double>>);
+
+static_assert(std::is_constructible_v<Map, std::from_range_t, RangeOf<std::pair<int, double>>, std::less<int>>);
+static_assert(std::is_constructible_v<Map, std::from_range_t, RangeOf<std::pair<short, double>>, std::less<int>>);
+static_assert(!std::is_constructible_v<Map, std::from_range_t, RangeOf<int>, std::less<int>>);
+static_assert(!std::is_constructible_v<Map, std::from_range_t, RangeOf<double>, std::less<int>>);
+
+static_assert(std::is_constructible_v<Map, std::from_range_t, RangeOf<std::pair<int, double>>, std::allocator<int>>);
+static_assert(std::is_constructible_v<Map, std::from_range_t, RangeOf<std::pair<short, double>>, std::allocator<int>>);
+static_assert(!std::is_constructible_v<Map, std::from_range_t, RangeOf<int>, std::allocator<int>>);
+static_assert(!std::is_constructible_v<Map, std::from_range_t, RangeOf<double>, std::allocator<int>>);
+
+static_assert(std::is_constructible_v<Map,
+                                      std::from_range_t,
+                                      RangeOf<std::pair<int, double>>,
+                                      std::less<int>,
+                                      std::allocator<int>>);
+static_assert(std::is_constructible_v<Map,
+                                      std::from_range_t,
+                                      RangeOf<std::pair<short, double>>,
+                                      std::less<int>,
+                                      std::allocator<int>>);
+static_assert(!std::is_constructible_v<Map, std::from_range_t, RangeOf<int>, std::less<int>, std::allocator<int>>);
+static_assert(!std::is_constructible_v<Map, std::from_range_t, RangeOf<double>, std::less<int>, std::allocator<int>>);
+
+int main(int, char**) {
+  {
+    // The constructors in this subclause shall not participate in overload
+    // resolution unless uses_allocator_v<key_container_type, Alloc> is true
+    // and uses_allocator_v<mapped_container_type, Alloc> is true.
+
+    using C  = test_less<int>;
+    using A1 = test_allocator<int>;
+    using A2 = other_allocator<int>;
+    using V1 = std::vector<int, A1>;
+    using V2 = std::vector<int, A2>;
+    using M1 = std::flat_multimap<int, int, C, V1, V1>;
+    using M2 = std::flat_multimap<int, int, C, V1, V2>;
+    using M3 = std::flat_multimap<int, int, C, V2, V1>;
+    static_assert(std::is_constructible_v<M1, std::from_range_t, M1, const A1&>);
+    static_assert(!std::is_constructible_v<M1, std::from_range_t, M1, const A2&>);
+    static_assert(!std::is_constructible_v<M2, std::from_range_t, M2, const A2&>);
+    static_assert(!std::is_constructible_v<M3, std::from_range_t, M3, const A2&>);
+
+    static_assert(std::is_constructible_v<M1, std::from_range_t, M1, const C&, const A1&>);
+    static_assert(!std::is_constructible_v<M1, std::from_range_t, M1, const C&, const A2&>);
+    static_assert(!std::is_constructible_v<M2, std::from_range_t, M2, const C&, const A2&>);
+    static_assert(!std::is_constructible_v<M3, std::from_range_t, M3, const C&, const A2&>);
+  }
+  {
+    // container-compatible-range
+    using C           = test_less<int>;
+    using A1          = test_allocator<int>;
+    using A2          = test_allocator<std::string>;
+    using M           = std::flat_multimap<int, std::string, C, std::vector<int, A1>, std::vector<std::string, A2>>;
+    using Pair        = std::pair<int, std::string>;
+    using PairLike    = std::tuple<int, std::string>;
+    using NonPairLike = int;
+
+    static_assert(std::is_constructible_v<M, std::from_range_t, std::vector<Pair>&>);
+    static_assert(std::is_constructible_v<M, std::from_range_t, std::vector<PairLike>&>);
+    static_assert(!std::is_constructible_v<M, std::from_range_t, std::vector<NonPairLike>&>);
+
+    static_assert(std::is_constructible_v<M, std::from_range_t, std::vector<Pair>&, const C&>);
+    static_assert(std::is_constructible_v<M, std::from_range_t, std::vector<PairLike>&, const C&>);
+    static_assert(!std::is_constructible_v<M, std::from_range_t, std::vector<NonPairLike>&, const C&>);
+
+    static_assert(std::is_constructible_v<M, std::from_range_t, std::vector<Pair>&, const A1&>);
+    static_assert(std::is_constructible_v<M, std::from_range_t, std::vector<PairLike>&, const A1&>);
+    static_assert(!std::is_constructible_v<M, std::from_range_t, std::vector<NonPairLike>&, const A1&>);
+
+    static_assert(std::is_constructible_v<M, std::from_range_t, std::vector<Pair>&, const C&, const A1&>);
+    static_assert(std::is_constructible_v<M, std::from_range_t, std::vector<PairLike>&, const C&, const A1&>);
+    static_assert(!std::is_constructible_v<M, std::from_range_t, std::vector<NonPairLike>&, const C&, const A1&>);
+  }
+
+  using P      = std::pair<int, short>;
+  P ar[]       = {{1, 1}, {1, 2}, {1, 3}, {2, 4}, {2, 5}, {3, 6}, {2, 7}, {3, 8}, {3, 9}};
+  P expected[] = {{1, 1}, {1, 2}, {1, 3}, {2, 4}, {2, 5}, {2, 7}, {3, 6}, {3, 8}, {3, 9}};
+  {
+    // flat_multimap(from_range_t, R&&)
+    // input_range && !common
+    using M    = std::flat_multimap<int, short>;
+    using Iter = cpp20_input_iterator<const P*>;
+    using Sent = sentinel_wrapper<Iter>;
+    using R    = std::ranges::subrange<Iter, Sent>;
+    auto m     = M(std::from_range, R(Iter(ar), Sent(Iter(ar + 9))));
+    assert(std::ranges::equal(m.keys(), expected | std::views::elements<0>));
+    LIBCPP_ASSERT(std::ranges::equal(m, expected));
+
+    // explicit(false)
+    M m2 = {std::from_range, R(Iter(ar), Sent(Iter(ar + 9)))};
+    assert(m2 == m);
+  }
+  {
+    // flat_multimap(from_range_t, R&&)
+    // greater
+    using M = std::flat_multimap<int, short, std::greater<int>, std::deque<int, min_allocator<int>>, std::deque<short>>;
+    using Iter = cpp20_input_iterator<const P*>;
+    using Sent = sentinel_wrapper<Iter>;
+    using R    = std::ranges::subrange<Iter, Sent>;
+    auto m     = M(std::from_range, R(Iter(ar), Sent(Iter(ar + 9))));
+    assert((m.keys() == std::deque<int, min_allocator<int>>{3, 3, 3, 2, 2, 2, 1, 1, 1}));
+    LIBCPP_ASSERT((m.values() == std::deque<short>{6, 8, 9, 4, 5, 7, 1, 2, 3}));
+  }
+  {
+    // flat_multimap(from_range_t, R&&)
+    // contiguous range
+    using M = std::flat_multimap<int, short>;
+    using R = std::ranges::subrange<const P*>;
+    auto m  = M(std::from_range, R(ar, ar + 9));
+    assert(std::ranges::equal(m.keys(), expected | std::views::elements<0>));
+    LIBCPP_ASSERT(std::ranges::equal(m, expected));
+  }
+  {
+    // flat_multimap(from_range_t, R&&, const key_compare&)
+    using C = test_less<int>;
+    using M = std::flat_multimap<int, short, C, std::vector<int>, std::deque<short>>;
+    using R = std::ranges::subrange<const P*>;
+    auto m  = M(std::from_range, R(ar, ar + 9), C(3));
+    assert(std::ranges::equal(m.keys(), expected | std::views::elements<0>));
+    LIBCPP_ASSERT(std::ranges::equal(m, expected));
+    assert(m.key_comp() == C(3));
+
+    // explicit(false)
+    M m2 = {std::from_range, R(ar, ar + 9), C(3)};
+    assert(m2 == m);
+    assert(m2.key_comp() == C(3));
+  }
+  {
+    // flat_multimap(from_range_t, R&&, const Allocator&)
+    using A1 = test_allocator<int>;
+    using A2 = test_allocator<short>;
+    using M  = std::flat_multimap<int, short, std::less<int>, std::vector<int, A1>, std::deque<short, A2>>;
+    using R  = std::ranges::subrange<const P*>;
+    auto m   = M(std::from_range, R(ar, ar + 9), A1(5));
+    assert(std::ranges::equal(m.keys(), expected | std::views::elements<0>));
+    LIBCPP_ASSERT(std::ranges::equal(m, expected));
+    assert(m.keys().get_allocator() == A1(5));
+    assert(m.values().get_allocator() == A2(5));
+  }
+  {
+    // flat_multimap(from_range_t, R&&, const Allocator&)
+    // explicit(false)
+    using A1 = test_allocator<int>;
+    using A2 = test_allocator<short>;
+    using M  = std::flat_multimap<int, short, std::less<int>, std::vector<int, A1>, std::deque<short, A2>>;
+    using R  = std::ranges::subrange<const P*>;
+    M m      = {std::from_range, R(ar, ar + 9), A1(5)}; // implicit ctor
+    assert(std::ranges::equal(m.keys(), expected | std::views::elements<0>));
+    LIBCPP_ASSERT(std::ranges::equal(m, expected));
+    assert(m.keys().get_allocator() == A1(5));
+    assert(m.values().get_allocator() == A2(5));
+  }
+  {
+    // flat_multimap(from_range_t, R&&, const key_compare&, const Allocator&)
+    using C  = test_less<int>;
+    using A1 = test_allocator<int>;
+    using A2 = test_allocator<short>;
+    using M  = std::flat_multimap<int, short, C, std::vector<int, A1>, std::deque<short, A2>>;
+    using R  = std::ranges::subrange<const P*>;
+    auto m   = M(std::from_range, R(ar, ar + 9), C(3), A1(5));
+    assert(std::ranges::equal(m.keys(), expected | std::views::elements<0>));
+    LIBCPP_ASSERT(std::ranges::equal(m, expected));
+    assert(m.key_comp() == C(3));
+    assert(m.keys().get_allocator() == A1(5));
+    assert(m.values().get_allocator() == A2(5));
+  }
+  {
+    // flat_multimap(from_range_t, R&&, const key_compare&, const Allocator&)
+    // explicit(false)
+    using A1 = test_allocator<int>;
+    using A2 = test_allocator<short>;
+    using M  = std::flat_multimap<int, short, std::less<int>, std::deque<int, A1>, std::vector<short, A2>>;
+    using R  = std::ranges::subrange<const P*>;
+    M m      = {std::from_range, R(ar, ar + 9), {}, A2(5)}; // implicit ctor
+    assert(std::ranges::equal(m.keys(), expected | std::views::elements<0>));
+    LIBCPP_ASSERT(std::ranges::equal(m, expected));
+    assert(m.keys().get_allocator() == A1(5));
+    assert(m.values().get_allocator() == A2(5));
+  }
+
+  return 0;
+}
diff --git a/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.cons/sorted_container.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.cons/sorted_container.pass.cpp
new file mode 100644
index 0000000000000..16579f0deed5d
--- /dev/null
+++ b/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.cons/sorted_container.pass.cpp
@@ -0,0 +1,165 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
+
+// <flat_map>
+
+// flat_multimap(sorted_equivalent_t, key_container_type key_cont, mapped_container_type mapped_cont,
+//          const key_compare& comp = key_compare());
+//
+// template<class Alloc>
+//   flat_multimap(sorted_equivalent_t, const key_container_type& key_cont,
+//            const mapped_container_type& mapped_cont, const Alloc& a);
+// template<class Alloc>
+//   flat_multimap(sorted_equivalent_t, const key_container_type& key_cont,
+//            const mapped_container_type& mapped_cont,
+//            const key_compare& comp, const Alloc& a);
+
+#include <deque>
+#include <flat_map>
+#include <functional>
+#include <vector>
+
+#include "min_allocator.h"
+#include "MoveOnly.h"
+#include "test_allocator.h"
+#include "test_iterators.h"
+#include "test_macros.h"
+#include "../../../test_compare.h"
+
+int main(int, char**) {
+  {
+    // The constructors in this subclause shall not participate in overload
+    // resolution unless uses_allocator_v<key_container_type, Alloc> is true
+    // and uses_allocator_v<mapped_container_type, Alloc> is true.
+
+    using C  = test_less<int>;
+    using A1 = test_allocator<int>;
+    using A2 = other_allocator<int>;
+    using V1 = std::vector<int, A1>;
+    using V2 = std::vector<int, A2>;
+    using M1 = std::flat_multimap<int, int, C, V1, V1>;
+    using M2 = std::flat_multimap<int, int, C, V1, V2>;
+    using M3 = std::flat_multimap<int, int, C, V2, V1>;
+    static_assert(std::is_constructible_v<M1, std::sorted_equivalent_t, const V1&, const V1&, const A1&>);
+    static_assert(!std::is_constructible_v<M1, std::sorted_equivalent_t, const V1&, const V1&, const A2&>);
+    static_assert(!std::is_constructible_v<M2, std::sorted_equivalent_t, const V1&, const V2&, const A2&>);
+    static_assert(!std::is_constructible_v<M3, std::sorted_equivalent_t, const V2&, const V1&, const A2&>);
+
+    static_assert(std::is_constructible_v<M1, std::sorted_equivalent_t, const V1&, const V1&, const C&, const A1&>);
+    static_assert(!std::is_constructible_v<M1, std::sorted_equivalent_t, const V1&, const V1&, const C&, const A2&>);
+    static_assert(!std::is_constructible_v<M2, std::sorted_equivalent_t, const V1&, const V2&, const C&, const A2&>);
+    static_assert(!std::is_constructible_v<M3, std::sorted_equivalent_t, const V2&, const V1&, const C&, const A2&>);
+  }
+  {
+    // flat_multimap(sorted_equivalent_t, key_container_type , mapped_container_type)
+    using M              = std::flat_multimap<int, char>;
+    std::vector<int> ks  = {1, 4, 4, 10};
+    std::vector<char> vs = {4, 3, 2, 1};
+    auto ks2             = ks;
+    auto vs2             = vs;
+
+    auto m = M(std::sorted_equivalent, ks, vs);
+    assert((m == M{{1, 4}, {4, 3}, {4, 2}, {10, 1}}));
+    m = M(std::sorted_equivalent, std::move(ks), std::move(vs));
+    assert(ks.empty()); // it was moved-from
+    assert(vs.empty()); // it was moved-from
+    assert((m == M{{1, 4}, {4, 3}, {4, 2}, {10, 1}}));
+
+    // explicit(false)
+    M m2 = {std::sorted_equivalent, std::move(ks2), std::move(vs2)};
+    assert(m == m2);
+  }
+  {
+    // flat_multimap(sorted_equivalent_t, key_container_type , mapped_container_type)
+    // non-default container, comparator and allocator type
+    using Ks = std::deque<int, min_allocator<int>>;
+    using Vs = std::deque<char, min_allocator<char>>;
+    using M  = std::flat_multimap<int, char, std::greater<int>, Ks, Vs>;
+    Ks ks    = {10, 1, 1, 1};
+    Vs vs    = {1, 2, 3, 4};
+    auto m   = M(std::sorted_equivalent, ks, vs);
+    assert((m == M{{1, 2}, {1, 3}, {1, 4}, {10, 1}}));
+    m = M(std::sorted_equivalent, std::move(ks), std::move(vs));
+    assert(ks.empty()); // it was moved-from
+    assert(vs.empty()); // it was moved-from
+    assert((m == M{{1, 2}, {1, 3}, {1, 4}, {10, 1}}));
+  }
+  {
+    // flat_multimap(sorted_equivalent_t, key_container_type , mapped_container_type)
+    // allocator copied into the containers
+    using A = test_allocator<int>;
+    using M = std::flat_multimap<int, int, std::less<int>, std::vector<int, A>, std::deque<int, A>>;
+    auto ks = std::vector<int, A>({2, 2, 4, 10}, A(4));
+    auto vs = std::deque<int, A>({4, 3, 2, 1}, A(5));
+    auto m  = M(std::sorted_equivalent, std::move(ks), std::move(vs));
+    assert(ks.empty()); // it was moved-from
+    assert(vs.empty()); // it was moved-from
+    assert((m == M{{2, 4}, {2, 3}, {4, 2}, {10, 1}}));
+    assert(m.keys().get_allocator() == A(4));
+    assert(m.values().get_allocator() == A(5));
+  }
+  {
+    // flat_multimap(sorted_equivalent_t, key_container_type , mapped_container_type, key_compare)
+    using C              = test_less<int>;
+    using M              = std::flat_multimap<int, char, C>;
+    std::vector<int> ks  = {1, 2, 10, 10};
+    std::vector<char> vs = {4, 3, 2, 1};
+
+    auto m = M(std::sorted_equivalent, ks, vs, C(4));
+    assert((m == M{{1, 4}, {2, 3}, {10, 2}, {10, 1}}));
+    assert(m.key_comp() == C(4));
+
+    // explicit(false)
+    M m2 = {std::sorted_equivalent, ks, vs, C(4)};
+    assert(m2 == m);
+    assert(m2.key_comp() == C(4));
+  }
+  {
+    // flat_multimap(sorted_equivalent_t, key_container_type , mapped_container_type, key_compare, const Allocator&)
+    using C                = test_less<int>;
+    using A                = test_allocator<int>;
+    using M                = std::flat_multimap<int, int, C, std::vector<int, A>, std::vector<int, A>>;
+    std::vector<int, A> ks = {1, 2, 4, 10};
+    std::vector<int, A> vs = {4, 3, 2, 1};
+    auto m                 = M(std::sorted_equivalent, ks, vs, C(4), A(5));
+    assert((m == M{{1, 4}, {2, 3}, {4, 2}, {10, 1}}));
+    assert(m.key_comp() == C(4));
+    assert(m.keys().get_allocator() == A(5));
+    assert(m.values().get_allocator() == A(5));
+
+    // explicit(false)
+    M m2 = {ks, vs, C(4), A(5)};
+    assert(m2 == m);
+    assert(m2.key_comp() == C(4));
+    assert(m2.keys().get_allocator() == A(5));
+    assert(m2.values().get_allocator() == A(5));
+  }
+  {
+    // flat_multimap(sorted_equivalent_t, key_container_type , mapped_container_type, const Allocator&)
+    using A = test_allocator<int>;
+    using M = std::flat_multimap<int, int, std::less<int>, std::vector<int, A>, std::deque<int, A>>;
+    auto ks = std::vector<int, A>({1, 2, 4, 4}, A(4));
+    auto vs = std::deque<int, A>({4, 3, 2, 1}, A(5));
+    auto m  = M(std::sorted_equivalent, ks, vs, A(6)); // replaces the allocators
+    assert(!ks.empty());                               // it was an lvalue above
+    assert(!vs.empty());                               // it was an lvalue above
+    assert((m == M{{1, 4}, {2, 3}, {4, 2}, {4, 1}}));
+    assert(m.keys().get_allocator() == A(6));
+    assert(m.values().get_allocator() == A(6));
+
+    // explicit(false)
+    M m2 = {std::sorted_equivalent, ks, vs, A(6)};
+    assert(m2 == m);
+    assert(m2.keys().get_allocator() == A(6));
+    assert(m2.values().get_allocator() == A(6));
+  }
+
+  return 0;
+}
diff --git a/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.cons/sorted_initializer_list.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.cons/sorted_initializer_list.pass.cpp
new file mode 100644
index 0000000000000..b34313bb3d404
--- /dev/null
+++ b/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.cons/sorted_initializer_list.pass.cpp
@@ -0,0 +1,183 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
+
+// <flat_map>
+
+// template <class InputIterator>
+//   flat_multimap(sorted_equivalent_t s, initializer_list<value_type> il,
+//            const key_compare& comp = key_compare())
+// template<class Alloc>
+//   flat_multimap(sorted_equivalent_t, initializer_list<value_type> il, const Alloc& a);
+// template<class Alloc>
+//   flat_multimap(sorted_equivalent_t, initializer_list<value_type> il,
+//            const key_compare& comp, const Alloc& a);
+
+#include <deque>
+#include <flat_map>
+#include <functional>
+#include <vector>
+
+#include "min_allocator.h"
+#include "test_allocator.h"
+#include "test_iterators.h"
+#include "test_macros.h"
+#include "../../../test_compare.h"
+
+template <class T, class U>
+std::initializer_list<std::pair<T, U>> il = {{1, 1}, {4, 2}, {4, 4}, {5, 5}};
+
+const auto il1 = il<int, int>;
+const auto il2 = il<int, short>;
+const auto il3 = il<short, int>;
+
+int main(int, char**) {
+  {
+    // The constructors in this subclause shall not participate in overload
+    // resolution unless uses_allocator_v<key_container_type, Alloc> is true
+    // and uses_allocator_v<mapped_container_type, Alloc> is true.
+    using C  = test_less<int>;
+    using A1 = test_allocator<int>;
+    using A2 = other_allocator<int>;
+    using V1 = std::vector<int, A1>;
+    using V2 = std::vector<int, A2>;
+    using M1 = std::flat_multimap<int, int, C, V1, V1>;
+    using M2 = std::flat_multimap<int, int, C, V1, V2>;
+    using M3 = std::flat_multimap<int, int, C, V2, V1>;
+    using IL = std::initializer_list<std::pair<int, int>>;
+    static_assert(std::is_constructible_v<M1, std::sorted_equivalent_t, IL, const A1&>);
+    static_assert(!std::is_constructible_v<M1, std::sorted_equivalent_t, IL, const A2&>);
+    static_assert(!std::is_constructible_v<M2, std::sorted_equivalent_t, IL, const A2&>);
+    static_assert(!std::is_constructible_v<M3, std::sorted_equivalent_t, IL, const A2&>);
+
+    static_assert(std::is_constructible_v<M1, std::sorted_equivalent_t, IL, const C&, const A1&>);
+    static_assert(!std::is_constructible_v<M1, std::sorted_equivalent_t, IL, const C&, const A2&>);
+    static_assert(!std::is_constructible_v<M2, std::sorted_equivalent_t, IL, const C&, const A2&>);
+    static_assert(!std::is_constructible_v<M3, std::sorted_equivalent_t, IL, const C&, const A2&>);
+  }
+  {
+    // initializer_list<value_type> needs to match exactly
+    using M = std::flat_multimap<int, short>;
+    using C = typename M::key_compare;
+    static_assert(std::is_constructible_v<M, std::sorted_equivalent_t, std::initializer_list<std::pair<int, short>>>);
+    static_assert(
+        std::is_constructible_v<M, std::sorted_equivalent_t, std::initializer_list<std::pair<int, short>>, C>);
+    static_assert(std::is_constructible_v<M,
+                                          std::sorted_equivalent_t,
+                                          std::initializer_list<std::pair<int, short>>,
+                                          C,
+                                          std::allocator<int>>);
+    static_assert(std::is_constructible_v<M,
+                                          std::sorted_equivalent_t,
+                                          std::initializer_list<std::pair<int, short>>,
+                                          std::allocator<int>>);
+    static_assert(
+        !std::is_constructible_v<M, std::sorted_equivalent_t, std::initializer_list<std::pair<const int, short>>>);
+    static_assert(
+        !std::is_constructible_v<M, std::sorted_equivalent_t, std::initializer_list<std::pair<const int, short>>, C>);
+    static_assert(!std::is_constructible_v<M,
+                                           std::sorted_equivalent_t,
+                                           std::initializer_list<std::pair<const int, short>>,
+                                           C,
+                                           std::allocator<int>>);
+    static_assert(!std::is_constructible_v<M,
+                                           std::sorted_equivalent_t,
+                                           std::initializer_list<std::pair<const int, short>>,
+                                           std::allocator<int>>);
+    static_assert(
+        !std::
+            is_constructible_v<M, std::sorted_equivalent_t, std::initializer_list<std::pair<const int, const short>>>);
+    static_assert(!std::is_constructible_v<M,
+                                           std::sorted_equivalent_t,
+                                           std::initializer_list<std::pair<const int, const short>>,
+                                           C>);
+    static_assert(!std::is_constructible_v<M,
+                                           std::sorted_equivalent_t,
+                                           std::initializer_list<std::pair<const int, const short>>,
+                                           C,
+                                           std::allocator<int>>);
+    static_assert(!std::is_constructible_v<M,
+                                           std::sorted_equivalent_t,
+                                           std::initializer_list<std::pair<const int, const short>>,
+                                           std::allocator<int>>);
+  }
+
+  {
+    // flat_multimap(sorted_equivalent_t, initializer_list<value_type>);
+    using M       = std::flat_multimap<int, int>;
+    auto m        = M(std::sorted_equivalent, il1);
+    auto expected = M{{1, 1}, {4, 2}, {4, 4}, {5, 5}};
+    assert(m == expected);
+
+    // explicit(false)
+    M m2 = {std::sorted_equivalent, il1};
+    assert(m2 == m);
+  }
+  {
+    // flat_multimap(sorted_equivalent_t, initializer_list<value_type>, const key_compare&);
+    using M = std::flat_multimap<int, int, std::function<bool(int, int)>>;
+    auto m  = M(std::sorted_equivalent, il1, std::less<int>());
+    assert(m == M({{1, 1}, {4, 2}, {4, 4}, {5, 5}}, std::less<>()));
+    assert(m.key_comp()(1, 2) == true);
+
+    // explicit(false)
+    M m2 = {std::sorted_equivalent, il1, std::less<int>()};
+    assert(m2 == m);
+  }
+  {
+    // flat_multimap(sorted_equivalent_t, initializer_list<value_type>, const key_compare&);
+    // greater
+    using M = std::flat_multimap<int, int, std::greater<int>, std::deque<int, min_allocator<int>>, std::vector<int>>;
+    std::initializer_list<std::pair<int, int>> il4{{5, 5}, {4, 4}, {1, 2}, {1, 1}};
+    auto m = M(std::sorted_equivalent, il4, std::greater<int>());
+    assert((m == M{{5, 5}, {4, 4}, {1, 2}, {1, 1}}));
+  }
+  {
+    // flat_multimap(sorted_equivalent_t, initializer_list<value_type>,  const Allocator&)
+    using A1      = test_allocator<int>;
+    using A2      = test_allocator<short>;
+    using M       = std::flat_multimap<int, short, std::less<int>, std::vector<int, A1>, std::deque<short, A2>>;
+    auto m        = M(std::sorted_equivalent, il2, A1(5));
+    auto expected = M{{1, 1}, {4, 2}, {4, 4}, {5, 5}};
+    assert(m == expected);
+    assert(m.keys().get_allocator() == A1(5));
+    assert(m.values().get_allocator() == A2(5));
+
+    // explicit(false)
+    M m2 = {std::sorted_equivalent, il2, A1(5)};
+    assert(m2 == m);
+    assert(m2.keys().get_allocator() == A1(5));
+    assert(m2.values().get_allocator() == A2(5));
+  }
+  {
+    // flat_multimap(sorted_equivalent_t, initializer_list<value_type>, const key_compare&, const Allocator&);
+    using C  = test_less<int>;
+    using A1 = test_allocator<int>;
+    using A2 = test_allocator<short>;
+    using M  = std::flat_multimap<int, short, C, std::vector<int, A1>, std::deque<short, A2>>;
+    auto m   = M(std::sorted_equivalent, il2, C(3), A1(5));
+    assert((m == M{{1, 1}, {4, 2}, {4, 4}, {5, 5}}));
+    assert(m.key_comp() == C(3));
+    assert(m.keys().get_allocator() == A1(5));
+    assert(m.values().get_allocator() == A2(5));
+  }
+  {
+    // flat_multimap(sorted_equivalent_t, initializer_list<value_type>, const key_compare&, const Allocator&);
+    // explicit(false)
+    using A1 = test_allocator<short>;
+    using A2 = test_allocator<int>;
+    using M  = std::flat_multimap<short, int, std::less<int>, std::deque<short, A1>, std::vector<int, A2>>;
+    M m      = {std::sorted_equivalent, il3, {}, A1(5)}; // implicit ctor
+    assert((m == M{{1, 1}, {4, 2}, {4, 4}, {5, 5}}));
+    assert(m.keys().get_allocator() == A1(5));
+    assert(m.values().get_allocator() == A2(5));
+  }
+
+  return 0;
+}
diff --git a/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.cons/sorted_iter_iter.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.cons/sorted_iter_iter.pass.cpp
new file mode 100644
index 0000000000000..45c4b3dc675a5
--- /dev/null
+++ b/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.cons/sorted_iter_iter.pass.cpp
@@ -0,0 +1,173 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
+
+// <flat_map>
+
+// template <class InputIterator>
+//   flat_multimap(sorted_equivalent_t, InputIterator first, InputIterator last, const key_compare& comp = key_compare());
+// template<class InputIterator, class Alloc>
+//   flat_multimap(InputIterator first, InputIterator last, const Alloc& a);
+// template<class InputIterator, class Allocator>
+//   flat_multimap(sorted_equivalent_t, InputIterator first, InputIterator last, const key_compare& comp, const Allocator& a);
+
+#include <deque>
+#include <flat_map>
+#include <functional>
+#include <vector>
+
+#include "min_allocator.h"
+#include "test_allocator.h"
+#include "test_iterators.h"
+#include "test_macros.h"
+#include "../../../test_compare.h"
+
+int main(int, char**) {
+  {
+    // The constructors in this subclause shall not participate in overload
+    // resolution unless uses_allocator_v<key_container_type, Alloc> is true
+    // and uses_allocator_v<mapped_container_type, Alloc> is true.
+    using C     = test_less<int>;
+    using A1    = test_allocator<int>;
+    using A2    = other_allocator<int>;
+    using V1    = std::vector<int, A1>;
+    using V2    = std::vector<int, A2>;
+    using M1    = std::flat_multimap<int, int, C, V1, V1>;
+    using M2    = std::flat_multimap<int, int, C, V1, V2>;
+    using M3    = std::flat_multimap<int, int, C, V2, V1>;
+    using Iter1 = typename M1::iterator;
+    using Iter2 = typename M2::iterator;
+    using Iter3 = typename M3::iterator;
+    static_assert(std::is_constructible_v<M1, std::sorted_equivalent_t, Iter1, Iter1, const A1&>);
+    static_assert(!std::is_constructible_v<M1, std::sorted_equivalent_t, Iter1, Iter1, const A2&>);
+    static_assert(!std::is_constructible_v<M2, std::sorted_equivalent_t, Iter2, Iter2, const A2&>);
+    static_assert(!std::is_constructible_v<M3, std::sorted_equivalent_t, Iter3, Iter3, const A2&>);
+
+    static_assert(std::is_constructible_v<M1, std::sorted_equivalent_t, Iter1, Iter1, const C&, const A1&>);
+    static_assert(!std::is_constructible_v<M1, std::sorted_equivalent_t, Iter1, Iter1, const C&, const A2&>);
+    static_assert(!std::is_constructible_v<M2, std::sorted_equivalent_t, Iter2, Iter2, const C&, const A2&>);
+    static_assert(!std::is_constructible_v<M3, std::sorted_equivalent_t, Iter3, Iter3, const C&, const A2&>);
+  }
+  {
+    // flat_multimap(sorted_equivalent_t, InputIterator, InputIterator);
+    // cpp17_input_iterator
+    using M = std::flat_multimap<int, int>;
+    using P = std::pair<int, int>;
+    P ar[]  = {{1, 1}, {4, 4}, {5, 5}, {5, 2}};
+    auto m  = M(std::sorted_equivalent, cpp17_input_iterator<const P*>(ar), cpp17_input_iterator<const P*>(ar + 4));
+    auto expected = M{{1, 1}, {4, 4}, {5, 5}, {5, 2}};
+    assert(m == expected);
+
+    // explicit(false)
+    M m2 = {std::sorted_equivalent, cpp17_input_iterator<const P*>(ar), cpp17_input_iterator<const P*>(ar + 4)};
+    assert(m2 == m);
+  }
+  {
+    // flat_multimap(sorted_equivalent_t, InputIterator, InputIterator);
+    // contiguous iterator
+    using C = test_less<int>;
+    using M =
+        std::flat_multimap<int, int, C, std::vector<int, min_allocator<int>>, std::vector<int, min_allocator<int>>>;
+    std::pair<int, int> ar[] = {{1, 1}, {1, 4}, {2, 2}, {5, 5}};
+    auto m                   = M(std::sorted_equivalent, ar, ar + 4);
+    auto expected            = M{{1, 1}, {1, 4}, {2, 2}, {5, 5}};
+    assert(m == expected);
+  }
+  {
+    // flat_multimap(sorted_equivalent_t, InputIterator, InputIterator, const key_compare&);
+    // cpp_17_input_iterator
+    using M = std::flat_multimap<int, int, std::function<bool(int, int)>>;
+    using P = std::pair<int, int>;
+    P ar[]  = {{1, 1}, {2, 2}, {2, 4}, {5, 5}};
+    auto m  = M(std::sorted_equivalent,
+               cpp17_input_iterator<const P*>(ar),
+               cpp17_input_iterator<const P*>(ar + 4),
+               std::less<int>());
+    assert(m == M({{1, 1}, {2, 2}, {2, 4}, {5, 5}}, std::less<>()));
+    assert(m.key_comp()(1, 2) == true);
+
+    // explicit(false)
+    M m2 = {std::sorted_equivalent,
+            cpp17_input_iterator<const P*>(ar),
+            cpp17_input_iterator<const P*>(ar + 4),
+            std::less<int>()};
+    assert(m2 == m);
+  }
+  {
+    // flat_multimap(sorted_equivalent_t, InputIterator, InputIterator, const key_compare&);
+    // greater
+    using M = std::flat_multimap<int, int, std::greater<int>, std::deque<int, min_allocator<int>>, std::vector<int>>;
+    using P = std::pair<int, int>;
+    P ar[]  = {{5, 5}, {2, 4}, {2, 2}, {1, 1}};
+    auto m  = M(std::sorted_equivalent,
+               cpp17_input_iterator<const P*>(ar),
+               cpp17_input_iterator<const P*>(ar + 4),
+               std::greater<int>());
+    assert((m == M{{5, 5}, {2, 4}, {2, 2}, {1, 1}}));
+  }
+  {
+    // flat_multimap(sorted_equivalent_t, InputIterator, InputIterator, const key_compare&);
+    // contiguous iterator
+    using C = test_less<int>;
+    using M =
+        std::flat_multimap<int, int, C, std::vector<int, min_allocator<int>>, std::vector<int, min_allocator<int>>>;
+    std::pair<int, int> ar[1] = {{42, 42}};
+    auto m                    = M(std::sorted_equivalent, ar, ar, C(5));
+    assert(m.empty());
+    assert(m.key_comp() == C(5));
+  }
+  {
+    // flat_multimap(sorted_equivalent_t, InputIterator , InputIterator, const Allocator&)
+    using A1      = test_allocator<int>;
+    using A2      = test_allocator<short>;
+    using M       = std::flat_multimap<int, short, std::less<int>, std::vector<int, A1>, std::deque<short, A2>>;
+    using P       = std::pair<int, int>;
+    P ar[]        = {{2, 1}, {2, 2}, {4, 4}, {5, 5}};
+    auto m        = M(std::sorted_equivalent, ar, ar + 4, A1(5));
+    auto expected = M{{2, 1}, {2, 2}, {4, 4}, {5, 5}};
+    assert(m == expected);
+    assert(m.keys().get_allocator() == A1(5));
+    assert(m.values().get_allocator() == A2(5));
+
+    // explicit(false)
+    M m2 = {std::sorted_equivalent, ar, ar + 4, A1(5)};
+    assert(m2 == m);
+    assert(m2.keys().get_allocator() == A1(5));
+    assert(m2.values().get_allocator() == A2(5));
+  }
+  {
+    // flat_multimap(sorted_equivalent_t, InputIterator, InputIterator, const key_compare&, const Allocator&);
+    using C  = test_less<int>;
+    using A1 = test_allocator<int>;
+    using A2 = test_allocator<short>;
+    using M  = std::flat_multimap<int, short, C, std::vector<int, A1>, std::deque<short, A2>>;
+    using P  = std::pair<int, int>;
+    P ar[]   = {{1, 1}, {1, 2}, {1, 4}, {1, 5}};
+    auto m   = M(std::sorted_equivalent, ar, ar + 4, C(3), A1(5));
+    assert((m == M{{1, 1}, {1, 2}, {1, 4}, {1, 5}}));
+    assert(m.key_comp() == C(3));
+    assert(m.keys().get_allocator() == A1(5));
+    assert(m.values().get_allocator() == A2(5));
+  }
+  {
+    // flat_multimap(sorted_equivalent_t, InputIterator, InputIterator, const key_compare&, const Allocator&);
+    // explicit(false)
+    using A1 = test_allocator<short>;
+    using A2 = test_allocator<int>;
+    using M  = std::flat_multimap<short, int, std::less<int>, std::deque<short, A1>, std::vector<int, A2>>;
+    using P  = std::pair<int, int>;
+    P ar[]   = {{1, 1}, {1, 2}, {1, 4}, {1, 5}};
+    M m      = {std::sorted_equivalent, ar, ar + 4, {}, A1(5)}; // implicit ctor
+    assert((m == M{{1, 1}, {1, 2}, {1, 4}, {1, 5}}));
+    assert(m.keys().get_allocator() == A1(5));
+    assert(m.values().get_allocator() == A2(5));
+  }
+
+  return 0;
+}
diff --git a/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.erasure/erase_if.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.erasure/erase_if.pass.cpp
new file mode 100644
index 0000000000000..76d5cbd909050
--- /dev/null
+++ b/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.erasure/erase_if.pass.cpp
@@ -0,0 +1,98 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
+
+// <flat_map>
+
+// class flat_multimap
+
+// template<class Key, class T, class Compare, class KeyContainer, class MappedContainer, class Predicate>
+//   typename flat_multimap<Key, T, Compare, KeyContainer, MappedContainer>::size_type
+//   erase_if(flat_multimap<Key, T, Compare, KeyContainer, MappedContainer>& c, Predicate pred);
+
+#include <deque>
+#include <flat_map>
+#include <functional>
+#include <initializer_list>
+#include <vector>
+
+#include "test_macros.h"
+#include "test_allocator.h"
+#include "min_allocator.h"
+
+// Verify that `flat_multimap` (like `multimap`) does NOT support std::erase.
+//
+template <class S>
+concept HasStdErase = requires(S& s, typename S::value_type x) { std::erase(s, x); };
+static_assert(HasStdErase<std::vector<int>>);
+static_assert(!HasStdErase<std::flat_multimap<int, int>>);
+
+template <class M>
+M make(std::initializer_list<int> vals) {
+  M ret;
+  for (int v : vals) {
+    ret.emplace(static_cast<typename M::key_type>(v), static_cast<typename M::mapped_type>(v + 10));
+  }
+  return ret;
+}
+
+template <class M, class Pred>
+void test0(
+    std::initializer_list<int> vals, Pred p, std::initializer_list<int> expected, std::size_t expected_erased_count) {
+  M s = make<M>(vals);
+  ASSERT_SAME_TYPE(typename M::size_type, decltype(std::erase_if(s, p)));
+  assert(expected_erased_count == std::erase_if(s, p));
+  assert(s == make<M>(expected));
+}
+
+template <class S>
+void test() {
+  // Test all the plausible signatures for this predicate.
+  auto is1   = [](typename S::const_reference v) { return v.first == 1; };
+  auto is2   = [](typename S::value_type v) { return v.first == 2; };
+  auto is3   = [](const typename S::value_type& v) { return v.first == 3; };
+  auto is4   = [](auto v) { return v.first == 4; };
+  auto True  = [](const auto&) { return true; };
+  auto False = [](auto&&) { return false; };
+
+  test0<S>({}, is1, {}, 0);
+
+  test0<S>({1}, is1, {}, 1);
+  test0<S>({1, 1}, is1, {}, 2);
+  test0<S>({1, 1}, is2, {1, 1}, 0);
+
+  test0<S>({1, 2}, is1, {2}, 1);
+  test0<S>({1, 2}, is2, {1}, 1);
+  test0<S>({1, 2, 2, 2}, is2, {1}, 3);
+  test0<S>({1, 2, 2, 2}, is3, {1, 2, 2, 2}, 0);
+
+  test0<S>({1, 1, 2, 2, 3, 3}, is1, {2, 2, 3, 3}, 2);
+  test0<S>({1, 1, 2, 2, 3, 3}, is2, {1, 1, 3, 3}, 2);
+  test0<S>({1, 1, 2, 2, 3, 3}, is3, {1, 1, 2, 2}, 2);
+  test0<S>({1, 1, 2, 2, 3, 3}, is4, {1, 1, 2, 2, 3, 3}, 0);
+
+  test0<S>({1, 2, 2, 3, 3, 3}, True, {}, 6);
+  test0<S>({1, 2, 2, 3, 3, 3}, False, {1, 2, 2, 3, 3, 3}, 0);
+}
+
+int main(int, char**) {
+  test<std::flat_multimap<int, char>>();
+  test<std::flat_multimap<int,
+                          char,
+                          std::less<int>,
+                          std::vector<int, min_allocator<int>>,
+                          std::vector<char, min_allocator<char>>>>();
+  test<std::flat_multimap<int, char, std::greater<int>, std::vector<int, test_allocator<int>>>>();
+  test<std::flat_multimap<int, char, std::less<int>, std::deque<int, min_allocator<int>>>>();
+  test<std::flat_multimap<int, char, std::greater<int>, std::deque<int, test_allocator<int>>>>();
+  test<std::flat_multimap<long, int>>();
+  test<std::flat_multimap<double, int>>();
+
+  return 0;
+}
diff --git a/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.erasure/erase_if_exceptions.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.erasure/erase_if_exceptions.pass.cpp
new file mode 100644
index 0000000000000..13b57202f7862
--- /dev/null
+++ b/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.erasure/erase_if_exceptions.pass.cpp
@@ -0,0 +1,157 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
+// UNSUPPORTED: no-exceptions
+
+// <flat_map>
+
+// class flat_multimap
+
+// template<class Key, class T, class Compare, class KeyContainer, class MappedContainer, class Predicate>
+//   typename flat_multimap<Key, T, Compare, KeyContainer, MappedContainer>::size_type
+//   erase_if(flat_multimap<Key, T, Compare, KeyContainer, MappedContainer>& c, Predicate pred);
+// If any member function in [flat.set.defn] exits via an exception, the invariant is restored.
+// (This is not a member function, but let's respect the invariant anyway.)
+
+#include <algorithm>
+#include <cassert>
+#include <deque>
+#include <flat_map>
+#include <functional>
+#include <utility>
+#include <vector>
+
+#include "../helpers.h"
+#include "test_macros.h"
+
+struct Counter {
+  int c1, c2, throws;
+  void tick() {
+    c1 -= 1;
+    if (c1 == 0) {
+      c1 = c2;
+      throws += 1;
+      throw 42;
+    }
+  }
+};
+Counter g_counter = {0, 0, 0};
+
+struct ThrowingAssignment {
+  ThrowingAssignment(int i) : i_(i) {}
+  ThrowingAssignment(const ThrowingAssignment&) = default;
+  ThrowingAssignment& operator=(const ThrowingAssignment& rhs) {
+    g_counter.tick();
+    i_ = rhs.i_;
+    g_counter.tick();
+    return *this;
+  }
+  operator int() const { return i_; }
+  int i_;
+};
+
+struct ThrowingComparator {
+  bool operator()(const ThrowingAssignment& a, const ThrowingAssignment& b) const {
+    g_counter.tick();
+    return a.i_ < b.i_;
+  }
+};
+
+struct ErasurePredicate {
+  bool operator()(const auto& x) const { return (3 <= x.first && x.first <= 5); }
+};
+
+int main(int, char**) {
+  const std::pair<int, int> expected[] = {{1, 1}, {2, 2}, {3, 3}, {3, 3}, {5, 5}, {6, 6}, {7, 7}, {8, 8}};
+  {
+    using M = std::flat_multimap<ThrowingAssignment, int, ThrowingComparator>;
+    for (int first_throw = 1; first_throw < 99; ++first_throw) {
+      for (int second_throw = 1; second_throw < 99; ++second_throw) {
+        g_counter = {0, 0, 0};
+        M m       = M({1, 2, 3, 3, 5, 6, 7, 8}, {1, 2, 3, 3, 5, 6, 7, 8});
+        try {
+          g_counter = {first_throw, second_throw, 0};
+          auto n    = std::erase_if(m, ErasurePredicate());
+          assert(n == 3);
+          // If it didn't throw at all, we're done.
+          g_counter = {0, 0, 0};
+          assert((m == M{{1, 1}, {2, 2}, {6, 6}, {7, 7}, {8, 8}}));
+          first_throw = 99; // "done"
+          break;
+        } catch (int ex) {
+          assert(ex == 42);
+          check_invariant(m);
+          LIBCPP_ASSERT(m.empty() || std::equal(m.begin(), m.end(), expected, expected + 8));
+          if (g_counter.throws == 1) {
+            // We reached the first throw but not the second throw.
+            break;
+          }
+        }
+      }
+    }
+  }
+  {
+    using M = std::flat_multimap<int, ThrowingAssignment, ThrowingComparator>;
+    for (int first_throw = 1; first_throw < 99; ++first_throw) {
+      for (int second_throw = 1; second_throw < 99; ++second_throw) {
+        g_counter = {0, 0, 0};
+        M m       = M({1, 2, 3, 3, 5, 6, 7, 8}, {1, 2, 3, 3, 5, 6, 7, 8});
+        try {
+          g_counter = {first_throw, second_throw, 0};
+          auto n    = std::erase_if(m, ErasurePredicate());
+          assert(n == 3);
+          // If it didn't throw at all, we're done.
+          g_counter = {0, 0, 0};
+          assert((m == M{{1, 1}, {2, 2}, {6, 6}, {7, 7}, {8, 8}}));
+          first_throw = 99; // "done"
+          break;
+        } catch (int ex) {
+          assert(ex == 42);
+          check_invariant(m);
+          LIBCPP_ASSERT(m.empty() || std::equal(m.begin(), m.end(), expected, expected + 8));
+          if (g_counter.throws == 1) {
+            // We reached the first throw but not the second throw.
+            break;
+          }
+        }
+      }
+    }
+  }
+  {
+    using M = std::
+        flat_multimap<ThrowingAssignment, int, ThrowingComparator, std::deque<ThrowingAssignment>, std::deque<int>>;
+    for (int first_throw = 1; first_throw < 99; ++first_throw) {
+      for (int second_throw = 1; second_throw < 99; ++second_throw) {
+        g_counter                                = {0, 0, 0};
+        std::deque<ThrowingAssignment> container = {5, 6, 7, 8};
+        container.insert(container.begin(), {1, 2, 3, 3});
+        M m = M(std::move(container), {1, 2, 3, 3, 5, 6, 7, 8});
+        try {
+          g_counter = {first_throw, second_throw, 0};
+          auto n    = std::erase_if(m, ErasurePredicate());
+          assert(n == 3);
+          // If it didn't throw at all, we're done.
+          g_counter = {0, 0, 0};
+          assert((m == M{{1, 1}, {2, 2}, {6, 6}, {7, 7}, {8, 8}}));
+          first_throw = 99; // "done"
+          break;
+        } catch (int ex) {
+          assert(ex == 42);
+          check_invariant(m);
+          LIBCPP_ASSERT(m.empty() || std::equal(m.begin(), m.end(), expected, expected + 8));
+          if (g_counter.throws == 1) {
+            // We reached the first throw but not the second throw.
+            break;
+          }
+        }
+      }
+    }
+  }
+  return 0;
+}
diff --git a/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.iterators/iterator.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.iterators/iterator.pass.cpp
new file mode 100644
index 0000000000000..c1285955e5db6
--- /dev/null
+++ b/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.iterators/iterator.pass.cpp
@@ -0,0 +1,105 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
+
+// <flat_map>
+
+// class flat_multimap
+
+//       iterator begin()   noexcept;
+// const_iterator begin()   const noexcept
+//       iterator end()     noexcept;
+// const_iterator end()     const noexcept;
+//
+// const_iterator cbegin()  const noexcept;
+// const_iterator cend()    const noexcept;
+
+#include <cassert>
+#include <cstddef>
+#include <deque>
+#include <flat_map>
+#include <functional>
+#include <string>
+
+#include "MinSequenceContainer.h"
+#include "test_macros.h"
+#include "min_allocator.h"
+
+template <class KeyContainer, class ValueContainer>
+void test() {
+  using Key   = typename KeyContainer::value_type;
+  using Value = typename ValueContainer::value_type;
+  using M     = std::flat_multimap<Key, Value, std::less<Key>, KeyContainer, ValueContainer>;
+
+  M m         = {{1, 'a'}, {1, 'z'}, {2, 'b'}, {3, 'a'}, {3, 'b'}, {3, 'c'}, {4, 'd'}};
+  const M& cm = m;
+  ASSERT_SAME_TYPE(decltype(m.begin()), typename M::iterator);
+  ASSERT_SAME_TYPE(decltype(m.cbegin()), typename M::const_iterator);
+  ASSERT_SAME_TYPE(decltype(cm.begin()), typename M::const_iterator);
+  ASSERT_SAME_TYPE(decltype(m.end()), typename M::iterator);
+  ASSERT_SAME_TYPE(decltype(m.cend()), typename M::const_iterator);
+  ASSERT_SAME_TYPE(decltype(cm.end()), typename M::const_iterator);
+  static_assert(noexcept(m.begin()));
+  static_assert(noexcept(cm.begin()));
+  static_assert(noexcept(m.cbegin()));
+  static_assert(noexcept(m.end()));
+  static_assert(noexcept(cm.end()));
+  static_assert(noexcept(m.cend()));
+  assert(m.size() == 7);
+  assert(std::distance(m.begin(), m.end()) == 7);
+  assert(std::distance(cm.begin(), cm.end()) == 7);
+  assert(std::distance(m.cbegin(), m.cend()) == 7);
+  typename M::iterator i;                   // default-construct
+  i                            = m.begin(); // move-assignment
+  typename M::const_iterator k = i;         // converting constructor
+  assert(i == k);                           // comparison
+  assert(i->first == 1);                    // operator->
+  assert(i->second == 'a');                 // operator->
+  ++i;                                      // pre-increment
+  assert(i->first == 1);                    // operator->
+  assert(i->second == 'z');                 // operator->
+  i = i + 3;                                // operator+
+  assert((*i).first == 3);                  // operator*
+  assert((*i).second == 'b');               // operator*
+  i += 3;                                   // operator+=
+  assert(i == m.end());                     // operator==
+  --i;                                      // pre-decrement
+  assert(i->first == 4);                    // operator->
+  assert(i->second == 'd');                 // operator->
+  i = i - 2;                                // operator-
+  assert(i->first == 3);                    // operator->
+  assert(i->second == 'b');                 // operator->
+  i -= 2;                                   // operator-=
+  assert(i > m.begin());                    // operator>
+}
+
+int main(int, char**) {
+  test<std::vector<int>, std::vector<char>>();
+  test<std::deque<int>, std::vector<char>>();
+  test<MinSequenceContainer<int>, MinSequenceContainer<char>>();
+  test<std::vector<int, min_allocator<int>>, std::vector<char, min_allocator<char>>>();
+
+  {
+    // N3644 testing
+    using C = std::flat_multimap<int, char>;
+    C::iterator ii1{}, ii2{};
+    C::iterator ii4 = ii1;
+    C::const_iterator cii{};
+    assert(ii1 == ii2);
+    assert(ii1 == ii4);
+    assert(!(ii1 != ii2));
+
+    assert((ii1 == cii));
+    assert((cii == ii1));
+    assert(!(ii1 != cii));
+    assert(!(cii != ii1));
+  }
+
+  return 0;
+}
diff --git a/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.iterators/iterator_comparison.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.iterators/iterator_comparison.pass.cpp
new file mode 100644
index 0000000000000..f1b2cad743e23
--- /dev/null
+++ b/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.iterators/iterator_comparison.pass.cpp
@@ -0,0 +1,155 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
+
+// <flat_map>
+
+// flat_multimap iterators should be C++20 random access iterators
+
+#include <compare>
+#include <concepts>
+#include <deque>
+#include <flat_map>
+#include <functional>
+#include <vector>
+
+#include "MinSequenceContainer.h"
+#include "test_macros.h"
+#include "min_allocator.h"
+
+template <class KeyContainer, class ValueContainer>
+void test() {
+  using Key   = typename KeyContainer::value_type;
+  using Value = typename ValueContainer::value_type;
+  using M     = std::flat_multimap<Key, Value, std::less<Key>, KeyContainer, ValueContainer>;
+  using KI    = typename KeyContainer::iterator;
+  using I     = M::iterator;
+  using CI    = M::const_iterator;
+  using RI    = M::reverse_iterator;
+  using CRI   = M::const_reverse_iterator;
+
+  static_assert(std::equality_comparable<I>);
+  static_assert(std::equality_comparable<CI>);
+  static_assert(std::equality_comparable<RI>);
+  static_assert(std::equality_comparable<CRI>);
+
+  static_assert(std::totally_ordered<I>);
+  static_assert(std::totally_ordered<CI>);
+  static_assert(std::totally_ordered<RI>);
+  static_assert(std::totally_ordered<CRI>);
+
+  M m = {{1, 'a'}, {2, 'b'}, {2, 'e'}, {3, 'z'}, {3, 'y'}, {3, 'c'}, {4, 'd'}};
+
+  I i1 = m.begin();
+  I i2 = m.begin() + 1;
+
+  assert(i1 == i1);
+  assert(!(i1 != i1));
+  assert(i1 != i2);
+  assert(!(i1 == i2));
+  assert(i1 < i2);
+  assert(!(i1 < i1));
+  assert(i1 <= i1);
+  assert(i1 <= i2);
+  assert(!(i2 <= i1));
+  assert(i2 > i1);
+  assert(!(i2 > i2));
+  assert(i2 >= i1);
+  assert(i2 >= i2);
+  assert(!(i1 >= i2));
+
+  CI ci1 = m.cbegin();
+  CI ci2 = m.cbegin() + 1;
+  assert(ci1 == ci1);
+  assert(!(ci1 != ci1));
+  assert(ci1 != ci2);
+  assert(!(ci1 == ci2));
+  assert(ci1 < ci2);
+  assert(!(ci1 < ci1));
+  assert(ci1 <= ci1);
+  assert(ci1 <= ci2);
+  assert(!(ci2 <= ci1));
+  assert(ci2 > ci1);
+  assert(!(ci2 > ci2));
+  assert(ci2 >= ci1);
+  assert(ci2 >= ci2);
+  assert(!(ci1 >= ci2));
+
+  RI ri1 = m.rbegin();
+  RI ri2 = m.rbegin() + 1;
+  assert(ri1 == ri1);
+  assert(!(ri1 != ri1));
+  assert(ri1 != ri2);
+  assert(!(ri1 == ri2));
+  assert(ri1 < ri2);
+  assert(!(ri1 < ri1));
+  assert(ri1 <= ri1);
+  assert(ri1 <= ri2);
+  assert(!(ri2 <= ri1));
+  assert(ri2 > ri1);
+  assert(!(ri2 > ri2));
+  assert(ri2 >= ri1);
+  assert(ri2 >= ri2);
+  assert(!(ri1 >= ri2));
+
+  CRI cri1 = m.crbegin();
+  CRI cri2 = m.crbegin() + 1;
+  assert(cri1 == cri1);
+  assert(!(cri1 != cri1));
+  assert(cri1 != cri2);
+  assert(!(cri1 == cri2));
+  assert(cri1 < cri2);
+  assert(!(cri1 < cri1));
+  assert(cri1 <= cri1);
+  assert(cri1 <= cri2);
+  assert(!(cri2 <= cri1));
+  assert(cri2 > cri1);
+  assert(!(cri2 > cri2));
+  assert(cri2 >= cri1);
+  assert(cri2 >= cri2);
+  assert(!(cri1 >= cri2));
+
+  if constexpr (std::three_way_comparable<KI>) {
+    static_assert(std::three_way_comparable<I>); // ...of course the wrapped iterators still support <=>.
+    static_assert(std::three_way_comparable<CI>);
+    static_assert(std::three_way_comparable<RI>);
+    static_assert(std::three_way_comparable<CRI>);
+    static_assert(std::same_as<decltype(I() <=> I()), std::strong_ordering>);
+    static_assert(std::same_as<decltype(I() <=> CI()), std::strong_ordering>);
+    static_assert(std::same_as<decltype(CI() <=> CI()), std::strong_ordering>);
+    static_assert(std::same_as<decltype(RI() <=> RI()), std::strong_ordering>);
+    static_assert(std::same_as<decltype(RI() <=> CRI()), std::strong_ordering>);
+    static_assert(std::same_as<decltype(CRI() <=> CRI()), std::strong_ordering>);
+
+    assert(i1 <=> i1 == std::strong_ordering::equivalent);
+    assert(i1 <=> i2 == std::strong_ordering::less);
+    assert(i2 <=> i1 == std::strong_ordering::greater);
+
+    assert(ci1 <=> ci1 == std::strong_ordering::equivalent);
+    assert(ci1 <=> ci2 == std::strong_ordering::less);
+    assert(ci2 <=> ci1 == std::strong_ordering::greater);
+
+    assert(ri1 <=> ri1 == std::strong_ordering::equivalent);
+    assert(ri1 <=> ri2 == std::strong_ordering::less);
+    assert(ri2 <=> ri1 == std::strong_ordering::greater);
+
+    assert(cri1 <=> cri1 == std::strong_ordering::equivalent);
+    assert(cri1 <=> cri2 == std::strong_ordering::less);
+    assert(cri2 <=> cri1 == std::strong_ordering::greater);
+  }
+}
+
+int main(int, char**) {
+  test<std::vector<int>, std::vector<char>>();
+  test<std::deque<int>, std::vector<char>>();
+  test<MinSequenceContainer<int>, MinSequenceContainer<char>>();
+  test<std::vector<int, min_allocator<int>>, std::vector<char, min_allocator<char>>>();
+
+  return 0;
+}
diff --git a/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.iterators/iterator_concept_conformance.compile.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.iterators/iterator_concept_conformance.compile.pass.cpp
new file mode 100644
index 0000000000000..ce578e4def92b
--- /dev/null
+++ b/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.iterators/iterator_concept_conformance.compile.pass.cpp
@@ -0,0 +1,84 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
+
+// <flat_map>
+
+// class flat_multimap
+
+// iterator, const_iterator, reverse_iterator, const_reverse_iterator
+
+#include <flat_map>
+#include <deque>
+#include <functional>
+#include <iterator>
+#include <string>
+#include <vector>
+#include <type_traits>
+
+#include "MinSequenceContainer.h"
+#include "test_macros.h"
+#include "min_allocator.h"
+
+template <class KeyContainer, class ValueContainer>
+void test() {
+  using Key   = typename KeyContainer::value_type;
+  using Value = typename ValueContainer::value_type;
+  using C     = std::flat_multimap<Key, Value, std::less<Key>, KeyContainer, ValueContainer>;
+  using I     = C::iterator;
+  using CI    = C::const_iterator;
+  using RI    = C::reverse_iterator;
+  using CRI   = C::const_reverse_iterator;
+  static_assert(std::random_access_iterator<I>);
+  static_assert(std::random_access_iterator<CI>);
+  static_assert(std::random_access_iterator<RI>);
+  static_assert(std::random_access_iterator<CRI>);
+  static_assert(!std::contiguous_iterator<I>);
+  static_assert(!std::contiguous_iterator<CI>);
+  static_assert(!std::contiguous_iterator<RI>);
+  static_assert(!std::contiguous_iterator<CRI>);
+  static_assert(!std::indirectly_writable<I, std::pair<int, char>>);
+  static_assert(!std::indirectly_writable<CI, std::pair<int, char>>);
+  static_assert(!std::indirectly_writable<RI, std::pair<int, char>>);
+  static_assert(!std::indirectly_writable<CRI, std::pair<int, char>>);
+  static_assert(std::sentinel_for<I, I>);
+  static_assert(std::sentinel_for<I, CI>);
+  static_assert(!std::sentinel_for<I, RI>);
+  static_assert(!std::sentinel_for<I, CRI>);
+  static_assert(std::sentinel_for<CI, I>);
+  static_assert(std::sentinel_for<CI, CI>);
+  static_assert(!std::sentinel_for<CI, RI>);
+  static_assert(!std::sentinel_for<CI, CRI>);
+  static_assert(!std::sentinel_for<RI, I>);
+  static_assert(!std::sentinel_for<RI, CI>);
+  static_assert(std::sentinel_for<RI, RI>);
+  static_assert(std::sentinel_for<RI, CRI>);
+  static_assert(!std::sentinel_for<CRI, I>);
+  static_assert(!std::sentinel_for<CRI, CI>);
+  static_assert(std::sentinel_for<CRI, RI>);
+  static_assert(std::sentinel_for<CRI, CRI>);
+  static_assert(std::indirectly_movable_storable<I, std::pair<int, char>*>);
+  static_assert(std::indirectly_movable_storable<CI, std::pair<int, char>*>);
+  static_assert(std::indirectly_movable_storable<RI, std::pair<int, char>*>);
+  static_assert(std::indirectly_movable_storable<CRI, std::pair<int, char>*>);
+
+#ifdef _LIBCPP_VERSION
+  static_assert(std::is_same_v<typename std::iterator_traits<I>::iterator_category, std::random_access_iterator_tag>);
+  static_assert(std::is_same_v<typename std::iterator_traits<CI>::iterator_category, std::random_access_iterator_tag>);
+  static_assert(std::is_same_v<typename std::iterator_traits<RI>::iterator_category, std::random_access_iterator_tag>);
+  static_assert(std::is_same_v<typename std::iterator_traits<CRI>::iterator_category, std::random_access_iterator_tag>);
+#endif
+}
+
+void test() {
+  test<std::vector<int>, std::vector<char>>();
+  test<std::deque<int>, std::vector<char>>();
+  test<MinSequenceContainer<int>, MinSequenceContainer<char>>();
+  test<std::vector<int, min_allocator<int>>, std::vector<char, min_allocator<char>>>();
+}
diff --git a/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.iterators/range_concept_conformance.compile.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.iterators/range_concept_conformance.compile.pass.cpp
new file mode 100644
index 0000000000000..979c0b090fd66
--- /dev/null
+++ b/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.iterators/range_concept_conformance.compile.pass.cpp
@@ -0,0 +1,55 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
+
+#include <concepts>
+#include <deque>
+#include <flat_map>
+#include <functional>
+#include <ranges>
+#include <string>
+#include <vector>
+#include "MinSequenceContainer.h"
+#include "min_allocator.h"
+
+template <class KeyContainer, class ValueContainer>
+void test() {
+  {
+    using Key   = typename KeyContainer::value_type;
+    using Value = typename ValueContainer::value_type;
+    using C     = std::flat_multimap<Key, Value, std::less<Key>, KeyContainer, ValueContainer>;
+
+    static_assert(std::same_as<std::ranges::iterator_t<C>, typename C::iterator>);
+    static_assert(std::ranges::random_access_range<C>);
+    static_assert(!std::ranges::contiguous_range<C>);
+    static_assert(std::ranges::common_range<C>);
+    static_assert(std::ranges::input_range<C>);
+    static_assert(!std::ranges::view<C>);
+    static_assert(std::ranges::sized_range<C>);
+    static_assert(!std::ranges::borrowed_range<C>);
+    static_assert(std::ranges::viewable_range<C>);
+
+    static_assert(std::same_as<std::ranges::iterator_t<const C>, typename C::const_iterator>);
+    static_assert(std::ranges::random_access_range<const C>);
+    static_assert(!std::ranges::contiguous_range<const C>);
+    static_assert(std::ranges::common_range<const C>);
+    static_assert(std::ranges::input_range<const C>);
+    static_assert(!std::ranges::view<const C>);
+    static_assert(std::ranges::sized_range<const C>);
+    static_assert(!std::ranges::borrowed_range<const C>);
+    static_assert(!std::ranges::viewable_range<const C>);
+  }
+}
+
+void test() {
+  test<std::vector<int>, std::vector<char>>();
+  test<std::deque<int>, std::vector<char>>();
+  test<MinSequenceContainer<int>, MinSequenceContainer<char>>();
+  test<std::vector<int, min_allocator<int>>, std::vector<char, min_allocator<char>>>();
+}
diff --git a/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.iterators/reverse_iterator.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.iterators/reverse_iterator.pass.cpp
new file mode 100644
index 0000000000000..8c1e5451f703f
--- /dev/null
+++ b/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.iterators/reverse_iterator.pass.cpp
@@ -0,0 +1,92 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
+
+// <flat_map>
+
+// class flat_multimap
+
+//       reverse_iterator rbegin() noexcept;
+// const_reverse_iterator rbegin() const noexcept;
+//       reverse_iterator rend()   noexcept;
+// const_reverse_iterator rend()   const noexcept;
+//
+// const_reverse_iterator crbegin() const noexcept;
+// const_reverse_iterator crend()   const noexcept;
+
+#include <cassert>
+#include <cstddef>
+#include <deque>
+#include <flat_map>
+#include <functional>
+#include <vector>
+
+#include <iterator>
+
+#include "test_macros.h"
+#include <iostream>
+
+int main(int, char**) {
+  {
+    using M     = std::flat_multimap<int, char, std::less<int>, std::deque<int>, std::deque<char>>;
+    M m         = {{1, 'a'}, {1, 'b'}, {2, 'c'}, {2, 'd'}, {3, 'e'}, {3, 'f'}, {4, 'g'}, {4, 'h'}};
+    const M& cm = m;
+    ASSERT_SAME_TYPE(decltype(m.rbegin()), M::reverse_iterator);
+    ASSERT_SAME_TYPE(decltype(m.crbegin()), M::const_reverse_iterator);
+    ASSERT_SAME_TYPE(decltype(cm.rbegin()), M::const_reverse_iterator);
+    ASSERT_SAME_TYPE(decltype(m.rend()), M::reverse_iterator);
+    ASSERT_SAME_TYPE(decltype(m.crend()), M::const_reverse_iterator);
+    ASSERT_SAME_TYPE(decltype(cm.rend()), M::const_reverse_iterator);
+    static_assert(noexcept(m.rbegin()));
+    static_assert(noexcept(cm.rbegin()));
+    static_assert(noexcept(m.crbegin()));
+    static_assert(noexcept(m.rend()));
+    static_assert(noexcept(cm.rend()));
+    static_assert(noexcept(m.crend()));
+    assert(m.size() == 8);
+    assert(std::distance(m.rbegin(), m.rend()) == 8);
+    assert(std::distance(cm.rbegin(), cm.rend()) == 8);
+    assert(std::distance(m.crbegin(), m.crend()) == 8);
+    assert(std::distance(cm.crbegin(), cm.crend()) == 8);
+    M::reverse_iterator i; // default-construct
+    ASSERT_SAME_TYPE(decltype(i->first), const int&);
+    ASSERT_SAME_TYPE(decltype(i->second), char&);
+    i                           = m.rbegin(); // move-assignment
+    M::const_reverse_iterator k = i;          // converting constructor
+    assert(i == k);                           // comparison
+    for (int j = 8; j >= 1; --j, ++i) {       // pre-increment
+      assert(i->first == (j + 1) / 2);        // operator->
+      assert(i->second == 'a' + j - 1);
+    }
+    assert(i == m.rend());
+    for (int j = 1; j <= 8; ++j) {
+      --i; // pre-decrement
+      assert((*i).first == (j + 1) / 2);
+      assert((*i).second == 'a' + j - 1);
+    }
+    assert(i == m.rbegin());
+  }
+  {
+    // N3644 testing
+    using C = std::flat_multimap<int, char>;
+    C::reverse_iterator ii1{}, ii2{};
+    C::reverse_iterator ii4 = ii1;
+    C::const_reverse_iterator cii{};
+    assert(ii1 == ii2);
+    assert(ii1 == ii4);
+    assert(!(ii1 != ii2));
+
+    assert((ii1 == cii));
+    assert((cii == ii1));
+    assert(!(ii1 != cii));
+    assert(!(cii != ii1));
+  }
+
+  return 0;
+}
diff --git a/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.modifiers/clear.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.modifiers/clear.pass.cpp
new file mode 100644
index 0000000000000..5b0788b6826fd
--- /dev/null
+++ b/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.modifiers/clear.pass.cpp
@@ -0,0 +1,64 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
+
+// <flat_map>
+
+// class flat_multimap
+
+// void clear() noexcept;
+
+#include <cassert>
+#include <deque>
+#include <flat_map>
+#include <functional>
+#include <vector>
+
+#include "MinSequenceContainer.h"
+#include "../helpers.h"
+#include "test_macros.h"
+#include "min_allocator.h"
+
+// test noexcept
+
+template <class T>
+concept NoExceptClear = requires(T t) {
+  { t.clear() } noexcept;
+};
+
+static_assert(NoExceptClear<std::flat_multimap<int, int>>);
+#ifndef TEST_HAS_NO_EXCEPTIONS
+static_assert(
+    NoExceptClear<std::flat_multimap<int, int, std::less<int>, ThrowOnMoveContainer<int>, ThrowOnMoveContainer<int>>>);
+#endif
+
+template <class KeyContainer, class ValueContainer>
+void test() {
+  using Key   = typename KeyContainer::value_type;
+  using Value = typename ValueContainer::value_type;
+  using M     = std::flat_multimap<Key, Value, std::less<Key>, KeyContainer, ValueContainer>;
+
+  M m = {{5, 2}, {2, 1}, {2, 3}, {2, 1}, {5, 0}};
+  assert(m.size() == 5);
+  ASSERT_NOEXCEPT(m.clear());
+  ASSERT_SAME_TYPE(decltype(m.clear()), void);
+  m.clear();
+  assert(m.size() == 0);
+}
+
+int main(int, char**) {
+  test<std::vector<int>, std::vector<int>>();
+  test<std::vector<int>, std::vector<double>>();
+  test<std::deque<int>, std::vector<double>>();
+  test<MinSequenceContainer<int>, MinSequenceContainer<double>>();
+  test<std::vector<int, min_allocator<int>>, std::vector<double, min_allocator<double>>>();
+  test<std::vector<int, min_allocator<int>>, std::vector<int, min_allocator<int>>>();
+
+  return 0;
+}
diff --git a/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.modifiers/emplace.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.modifiers/emplace.pass.cpp
new file mode 100644
index 0000000000000..9ef0c26e54ba3
--- /dev/null
+++ b/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.modifiers/emplace.pass.cpp
@@ -0,0 +1,158 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
+
+// <flat_map>
+
+// class flat_multimap
+
+// template <class... Args>
+// iterator emplace(Args&&... args);
+
+#include <flat_map>
+#include <cassert>
+#include <deque>
+#include <tuple>
+#include <functional>
+#include <vector>
+
+#include "MinSequenceContainer.h"
+#include "../helpers.h"
+#include "test_macros.h"
+#include "../../../Emplaceable.h"
+#include "DefaultOnly.h"
+#include "min_allocator.h"
+
+// Constraints: is_constructible_v<pair<key_type, mapped_type>, Args...> is true.
+template <class M, class... Args>
+concept CanEmplace = requires(M m, Args&&... args) { m.emplace(std::forward<Args>(args)...); };
+
+using Map = std::flat_multimap<Emplaceable, Emplaceable>;
+static_assert(CanEmplace<Map>);
+static_assert(CanEmplace<Map, Emplaceable, Emplaceable>);
+static_assert(CanEmplace<Map, std::piecewise_construct_t, std::tuple<int, double>, std::tuple<int, double>>);
+static_assert(!CanEmplace<Map, Emplaceable>);
+static_assert(!CanEmplace<Map, int, double>);
+
+template <class KeyContainer, class ValueContainer>
+void test() {
+  using Key   = typename KeyContainer::value_type;
+  using Value = typename ValueContainer::value_type;
+  using M     = std::flat_multimap<Key, Value, std::less<Key>, KeyContainer, ValueContainer>;
+  using R     = typename M::iterator;
+
+  {
+    // was empty
+    M m;
+    std::same_as<R> decltype(auto) r = m.emplace(typename M::value_type(2, 3.5));
+    assert(r == m.begin());
+    assert(m.size() == 1);
+    assert(r->first == 2);
+    assert(r->second == 3.5);
+  }
+  {
+    // key does not exist and inserted at the begin
+    M m                              = {{3, 4.0}, {3, 3.0}, {3, 1.0}, {7, 0.0}};
+    std::same_as<R> decltype(auto) r = m.emplace(typename M::value_type(2, 2.0));
+    assert(r == m.begin());
+    assert(m.size() == 5);
+    assert(r->first == 2);
+    assert(r->second == 2.0);
+  }
+  {
+    // key does not exist and inserted in the middle
+    M m                              = {{1, 4.0}, {1, 3.0}, {3, 1.0}, {4, 0.0}};
+    std::same_as<R> decltype(auto) r = m.emplace(typename M::value_type(2, 2.0));
+    assert(r == m.begin() + 2);
+    assert(m.size() == 5);
+    assert(r->first == 2);
+    assert(r->second == 2.0);
+  }
+  {
+    // key does not exist and inserted at the end
+    M m                              = {{1, 4.0}, {1, 3.0}};
+    std::same_as<R> decltype(auto) r = m.emplace(typename M::value_type(2, 2.0));
+    assert(r == m.begin() + 2);
+    assert(m.size() == 3);
+    assert(r->first == 2);
+    assert(r->second == 2.0);
+  }
+  {
+    // key already exists and original at the begin
+    M m                              = {{2, 4.0}, {2, 3.0}, {5, 1.0}, {6, 0.0}};
+    std::same_as<R> decltype(auto) r = m.emplace(typename M::value_type(2, 2.0));
+    assert(r == m.begin() + 2);
+    assert(m.size() == 5);
+    assert(r->first == 2);
+    assert(r->second == 2.0);
+  }
+  {
+    // key already exists and original in the middle
+    M m                              = {{0, 4.0}, {2, 3.0}, {2, 1.0}, {4, 0.0}};
+    std::same_as<R> decltype(auto) r = m.emplace(typename M::value_type(2, 2.0));
+    assert(r == m.begin() + 3);
+    assert(m.size() == 5);
+    assert(r->first == 2);
+    assert(r->second == 2.0);
+  }
+  {
+    // key already exists and original at the end
+    M m                              = {{0, 4.0}, {1, 3.0}, {2, 1.0}};
+    std::same_as<R> decltype(auto) r = m.emplace(typename M::value_type(2, 2.0));
+    assert(r == m.begin() + 3);
+    assert(m.size() == 4);
+    assert(r->first == 2);
+    assert(r->second == 2.0);
+  }
+}
+
+template <class KeyContainer, class ValueContainer>
+void test_emplaceable() {
+  using M = std::flat_multimap<int, Emplaceable, std::less<int>, KeyContainer, ValueContainer>;
+  using R = typename M::iterator;
+
+  M m;
+  std::same_as<R> decltype(auto) r =
+      m.emplace(std::piecewise_construct, std::forward_as_tuple(2), std::forward_as_tuple());
+  assert(r == m.begin());
+  assert(m.size() == 1);
+  assert(m.begin()->first == 2);
+  assert(m.begin()->second == Emplaceable());
+  r = m.emplace(std::piecewise_construct, std::forward_as_tuple(1), std::forward_as_tuple(2, 3.5));
+  assert(r == m.begin());
+  assert(m.size() == 2);
+  assert(m.begin()->first == 1);
+  assert(m.begin()->second == Emplaceable(2, 3.5));
+  r = m.emplace(std::piecewise_construct, std::forward_as_tuple(1), std::forward_as_tuple(2, 3.5));
+  assert(r == m.begin() + 1);
+  assert(m.size() == 3);
+  assert(m.begin()->first == 1);
+  assert(m.begin()->second == Emplaceable(2, 3.5));
+}
+
+int main(int, char**) {
+  test<std::vector<int>, std::vector<double>>();
+  test<std::deque<int>, std::vector<double>>();
+  test<MinSequenceContainer<int>, MinSequenceContainer<double>>();
+  test<std::vector<int, min_allocator<int>>, std::vector<double, min_allocator<double>>>();
+
+  test_emplaceable<std::vector<int>, std::vector<Emplaceable>>();
+  test_emplaceable<std::deque<int>, std::vector<Emplaceable>>();
+  test_emplaceable<MinSequenceContainer<int>, MinSequenceContainer<Emplaceable>>();
+  test_emplaceable<std::vector<int, min_allocator<int>>, std::vector<Emplaceable, min_allocator<Emplaceable>>>();
+
+  {
+    auto emplace_func = [](auto& m, auto key_arg, auto value_arg) {
+      m.emplace(std::piecewise_construct, std::tuple(key_arg), std::tuple(value_arg));
+    };
+    test_emplace_exception_guarantee(emplace_func);
+  }
+
+  return 0;
+}
diff --git a/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.modifiers/emplace_hint.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.modifiers/emplace_hint.pass.cpp
new file mode 100644
index 0000000000000..588d27ea54f4d
--- /dev/null
+++ b/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.modifiers/emplace_hint.pass.cpp
@@ -0,0 +1,228 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
+
+// <flat_map>
+
+// class flat_multimap
+
+// template <class... Args>
+//   iterator emplace_hint(const_iterator position, Args&&... args);
+
+#include <flat_map>
+#include <cassert>
+#include <deque>
+#include <functional>
+#include <vector>
+
+#include "MinSequenceContainer.h"
+#include "test_macros.h"
+#include "../../../Emplaceable.h"
+#include "DefaultOnly.h"
+#include "min_allocator.h"
+#include "../helpers.h"
+
+#if defined(_LIBCPP_VERSION)
+// spec only specifies `emplace(Args&&...)` is_constructible_v<pair<key_type, mapped_type>, Args...> is true.
+// nothing mentioned for emplace_hint
+template <class M, class... Args>
+concept CanEmplaceHint =
+    requires(M m, typename M::const_iterator i, Args&&... args) { m.emplace_hint(i, std::forward<Args>(args)...); };
+
+using Map = std::flat_multimap<Emplaceable, Emplaceable>;
+static_assert(CanEmplaceHint<Map>);
+static_assert(CanEmplaceHint<Map, Emplaceable, Emplaceable>);
+static_assert(CanEmplaceHint<Map, std::piecewise_construct_t, std::tuple<int, double>, std::tuple<int, double>>);
+static_assert(!CanEmplaceHint<Map, Emplaceable>);
+static_assert(!CanEmplaceHint<Map, int, double>);
+#endif
+
+template <class KeyContainer, class ValueContainer>
+void test() {
+  using Key   = typename KeyContainer::value_type;
+  using Value = typename ValueContainer::value_type;
+  using M     = std::flat_multimap<Key, Value, std::less<Key>, KeyContainer, ValueContainer>;
+  using R     = M::iterator;
+  {
+    // was empty
+    M m;
+    std::same_as<R> decltype(auto) r = m.emplace_hint(m.end(), typename M::value_type(2, 3.5));
+    assert(r == m.begin());
+    assert(m.size() == 1);
+    assert(m.begin()->first == 2);
+    assert(m.begin()->second == 3.5);
+  }
+  {
+    // hint correct and no duplicates
+    M m                              = {{0, 0.0}, {1, 1.0}, {3, 3.0}};
+    auto it                          = m.begin() + 2;
+    std::same_as<R> decltype(auto) r = m.emplace_hint(it, typename M::value_type(2, 2.0));
+    assert(r == m.begin() + 2);
+    assert(m.size() == 4);
+    assert(r->first == 2);
+    assert(r->second == 2.0);
+  }
+  {
+    // hint correct and at the begin
+    M m                              = {{3, 3.0}, {4, 4.0}};
+    auto it                          = m.begin();
+    std::same_as<R> decltype(auto) r = m.emplace_hint(it, typename M::value_type(2, 2.0));
+    assert(r == m.begin());
+    assert(m.size() == 3);
+    assert(r->first == 2);
+    assert(r->second == 2.0);
+  }
+  {
+    // hint correct and at the end
+    M m                              = {{0, 0.0}, {1, 1.0}};
+    auto it                          = m.end();
+    std::same_as<R> decltype(auto) r = m.emplace_hint(it, typename M::value_type(2, 2.0));
+    assert(r == m.begin() + 2);
+    assert(m.size() == 3);
+    assert(r->first == 2);
+    assert(r->second == 2.0);
+  }
+  {
+    // hint correct and at first duplicate
+    M m                              = {{0, 0.0}, {1, 1.0}, {2, 1.9}, {2, 2.1}, {3, 3.0}};
+    auto it                          = m.begin() + 2;
+    std::same_as<R> decltype(auto) r = m.emplace_hint(it, typename M::value_type(2, 2.0));
+    assert(r == m.begin() + 2);
+    assert(m.size() == 6);
+    assert(r->first == 2);
+    assert(r->second == 2.0);
+    assert(std::next(r)->first == 2);
+    assert(std::next(r)->second == 1.9);
+  }
+  {
+    // hint correct and in-between duplicates
+    M m                              = {{0, 0.0}, {1, 1.0}, {2, 1.8}, {2, 1.9}, {2, 2.1}, {3, 3.0}};
+    auto it                          = m.begin() + 4;
+    std::same_as<R> decltype(auto) r = m.emplace_hint(it, typename M::value_type(2, 2.0));
+    assert(r == m.begin() + 4);
+    assert(m.size() == 7);
+    assert(r->first == 2);
+    assert(r->second == 2.0);
+    assert(std::next(r)->first == 2);
+    assert(std::next(r)->second == 2.1);
+  }
+  {
+    // hint correct and after duplicates
+    M m                              = {{0, 0.0}, {1, 1.0}, {2, 1.8}, {2, 1.9}, {2, 2.1}, {3, 3.0}};
+    auto it                          = m.begin() + 5;
+    std::same_as<R> decltype(auto) r = m.emplace_hint(it, typename M::value_type(2, 2.0));
+    assert(r == m.begin() + 5);
+    assert(m.size() == 7);
+    assert(r->first == 2);
+    assert(r->second == 2.0);
+    assert(std::next(r)->first == 3);
+    assert(std::next(r)->second == 3.0);
+  }
+  {
+    // hint incorrect and no duplicates
+    M m                              = {{0, 0.0}, {1, 1.0}, {3, 3.0}};
+    auto it                          = m.begin() + 1;
+    std::same_as<R> decltype(auto) r = m.emplace_hint(it, typename M::value_type(2, 2.0));
+    assert(r == m.begin() + 2);
+    assert(m.size() == 4);
+    assert(r->first == 2);
+    assert(r->second == 2.0);
+  }
+  {
+    // hint incorrect and at the begin
+    M m                              = {{0, 0.0}, {1, 1.0}};
+    auto it                          = m.begin();
+    std::same_as<R> decltype(auto) r = m.emplace_hint(it, typename M::value_type(2, 2.0));
+    assert(r == m.begin() + 2);
+    assert(m.size() == 3);
+    assert(r->first == 2);
+    assert(r->second == 2.0);
+  }
+  {
+    // hint incorrect and at the end
+    M m                              = {{3, 3.0}, {4, 4.0}};
+    auto it                          = m.end();
+    std::same_as<R> decltype(auto) r = m.emplace_hint(it, typename M::value_type(2, 2.0));
+    assert(r == m.begin());
+    assert(m.size() == 3);
+    assert(r->first == 2);
+    assert(r->second == 2.0);
+  }
+  {
+    // hint incorrect and before the first duplicate
+    M m                              = {{0, 0.0}, {1, 1.0}, {2, 1.8}, {2, 1.9}, {2, 2.1}, {3, 3.0}};
+    auto it                          = m.begin();
+    std::same_as<R> decltype(auto) r = m.emplace_hint(it, typename M::value_type(2, 2.0));
+    // the result is as left as possible
+    assert(r == m.begin() + 2);
+    assert(m.size() == 7);
+    assert(r->first == 2);
+    assert(r->second == 2.0);
+    assert(std::next(r)->first == 2);
+    assert(std::next(r)->second == 1.8);
+  }
+  {
+    // hint incorrect and after the last duplicate
+    M m                              = {{0, 0.0}, {1, 1.0}, {2, 1.8}, {2, 1.9}, {2, 2.1}, {3, 3.0}, {4, 4.0}};
+    auto it                          = m.begin() + 6;
+    std::same_as<R> decltype(auto) r = m.emplace_hint(it, typename M::value_type(2, 2.0));
+    // the result is as right as possible
+    assert(r == m.begin() + 5);
+    assert(m.size() == 8);
+    assert(r->first == 2);
+    assert(r->second == 2.0);
+    assert(std::next(r)->first == 3);
+    assert(std::next(r)->second == 3.0);
+  }
+}
+
+template <class KeyContainer, class ValueContainer>
+void test_emplaceable() {
+  using M = std::flat_multimap<int, Emplaceable, std::less<int>, KeyContainer, ValueContainer>;
+  using R = M::iterator;
+
+  M m;
+  ASSERT_SAME_TYPE(decltype(m.emplace_hint(m.cbegin())), R);
+  R r = m.emplace_hint(m.end(), std::piecewise_construct, std::forward_as_tuple(2), std::forward_as_tuple());
+  assert(r == m.begin());
+  assert(m.size() == 1);
+  assert(r->first == 2);
+  assert(r->second == Emplaceable());
+  r = m.emplace_hint(m.end(), std::piecewise_construct, std::forward_as_tuple(1), std::forward_as_tuple(2, 3.5));
+  assert(r == m.begin());
+  assert(m.size() == 2);
+  assert(r->first == 1);
+  assert(r->second == Emplaceable(2, 3.5));
+  r = m.emplace_hint(m.end(), std::piecewise_construct, std::forward_as_tuple(1), std::forward_as_tuple(2, 3.6));
+  assert(r == m.begin() + 1);
+  assert(m.size() == 3);
+  assert(r->first == 1);
+  assert(r->second == Emplaceable(2, 3.6));
+}
+
+int main(int, char**) {
+  test<std::vector<int>, std::vector<double>>();
+  test<std::deque<int>, std::vector<double>>();
+  test<MinSequenceContainer<int>, MinSequenceContainer<double>>();
+  test<std::vector<int, min_allocator<int>>, std::vector<double, min_allocator<double>>>();
+
+  test_emplaceable<std::vector<int>, std::vector<Emplaceable>>();
+  test_emplaceable<std::deque<int>, std::vector<Emplaceable>>();
+  test_emplaceable<MinSequenceContainer<int>, MinSequenceContainer<Emplaceable>>();
+  test_emplaceable<std::vector<int, min_allocator<int>>, std::vector<Emplaceable, min_allocator<Emplaceable>>>();
+
+  {
+    auto emplace_func = [](auto& m, auto key_arg, auto value_arg) {
+      m.emplace_hint(m.begin(), std::piecewise_construct, std::tuple(key_arg), std::tuple(value_arg));
+    };
+    test_emplace_exception_guarantee(emplace_func);
+  }
+
+  return 0;
+}
diff --git a/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.modifiers/erase_iter.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.modifiers/erase_iter.pass.cpp
new file mode 100644
index 0000000000000..78040be2e043d
--- /dev/null
+++ b/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.modifiers/erase_iter.pass.cpp
@@ -0,0 +1,127 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
+
+// <flat_map>
+
+// class flat_multimap
+
+// iterator erase(iterator position);
+// iterator erase(const_iterator position);
+
+#include <compare>
+#include <concepts>
+#include <deque>
+#include <flat_map>
+#include <functional>
+#include <utility>
+#include <vector>
+
+#include "MinSequenceContainer.h"
+#include "../helpers.h"
+#include "test_macros.h"
+#include "min_allocator.h"
+
+template <class KeyContainer, class ValueContainer>
+void test() {
+  using Key   = typename KeyContainer::value_type;
+  using Value = typename ValueContainer::value_type;
+  using M     = std::flat_multimap<Key, Value, std::less<Key>, KeyContainer, ValueContainer>;
+  using P     = std::pair<Key, Value>;
+  using I     = M::iterator;
+
+  P ar[] = {
+      P(1, 1.5),
+      P(2, 2.5),
+      P(2, 2.6),
+      P(3, 3.5),
+      P(4, 4.5),
+      P(4, 4.5),
+      P(4, 4.7),
+      P(5, 5.5),
+      P(6, 6.5),
+      P(7, 7.5),
+      P(8, 8.5),
+  };
+  M m(ar, ar + sizeof(ar) / sizeof(ar[0]));
+  assert(m.size() == 11);
+  std::same_as<I> decltype(auto) i1 = m.erase(std::next(m.cbegin(), 2));
+  assert(m.size() == 10);
+  assert(i1 == std::next(m.begin(), 2));
+  assert(std::ranges::equal(
+      m,
+      std::vector<P>{
+          {1, 1.5}, {2, 2.5}, {3, 3.5}, {4, 4.5}, {4, 4.5}, {4, 4.7}, {5, 5.5}, {6, 6.5}, {7, 7.5}, {8, 8.5}}));
+
+  std::same_as<I> decltype(auto) i2 = m.erase(std::next(m.begin(), 0));
+  assert(m.size() == 9);
+  assert(i2 == m.begin());
+  assert(std::ranges::equal(
+      m, std::vector<P>{{2, 2.5}, {3, 3.5}, {4, 4.5}, {4, 4.5}, {4, 4.7}, {5, 5.5}, {6, 6.5}, {7, 7.5}, {8, 8.5}}));
+
+  std::same_as<I> decltype(auto) i3 = m.erase(std::next(m.cbegin(), 8));
+  assert(m.size() == 8);
+  assert(i3 == m.end());
+  assert(std::ranges::equal(
+      m, std::vector<P>{{2, 2.5}, {3, 3.5}, {4, 4.5}, {4, 4.5}, {4, 4.7}, {5, 5.5}, {6, 6.5}, {7, 7.5}}));
+
+  std::same_as<I> decltype(auto) i4 = m.erase(std::next(m.begin(), 1));
+  assert(m.size() == 7);
+  assert(i4 == std::next(m.begin()));
+  assert(std::ranges::equal(m, std::vector<P>{{2, 2.5}, {4, 4.5}, {4, 4.5}, {4, 4.7}, {5, 5.5}, {6, 6.5}, {7, 7.5}}));
+
+  std::same_as<I> decltype(auto) i5 = m.erase(std::next(m.cbegin(), 2));
+  assert(m.size() == 6);
+  assert(i5 == std::next(m.begin(), 2));
+  assert(std::ranges::equal(m, std::vector<P>{{2, 2.5}, {4, 4.5}, {4, 4.7}, {5, 5.5}, {6, 6.5}, {7, 7.5}}));
+
+  std::same_as<I> decltype(auto) i6 = m.erase(std::next(m.begin(), 2));
+  assert(m.size() == 5);
+  assert(i6 == std::next(m.begin(), 2));
+  assert(std::ranges::equal(m, std::vector<P>{{2, 2.5}, {4, 4.5}, {5, 5.5}, {6, 6.5}, {7, 7.5}}));
+
+  std::same_as<I> decltype(auto) i7 = m.erase(std::next(m.cbegin(), 0));
+  assert(m.size() == 4);
+  assert(i7 == std::next(m.begin(), 0));
+  assert(std::ranges::equal(m, std::vector<P>{{4, 4.5}, {5, 5.5}, {6, 6.5}, {7, 7.5}}));
+
+  std::same_as<I> decltype(auto) i8 = m.erase(std::next(m.cbegin(), 2));
+  assert(m.size() == 3);
+  assert(i8 == std::next(m.begin(), 2));
+  assert(std::ranges::equal(m, std::vector<P>{{4, 4.5}, {5, 5.5}, {7, 7.5}}));
+
+  std::same_as<I> decltype(auto) i9 = m.erase(std::next(m.cbegin(), 2));
+  assert(m.size() == 2);
+  assert(i9 == std::next(m.begin(), 2));
+  assert(std::ranges::equal(m, std::vector<P>{{4, 4.5}, {5, 5.5}}));
+
+  std::same_as<I> decltype(auto) i10 = m.erase(m.cbegin());
+  assert(m.size() == 1);
+  assert(i10 == m.cbegin());
+  assert(std::ranges::equal(m, std::vector<P>{{5, 5.5}}));
+
+  std::same_as<I> decltype(auto) i11 = m.erase(m.begin());
+  assert(m.size() == 0);
+  assert(i11 == m.begin());
+  assert(i11 == m.end());
+}
+
+int main(int, char**) {
+  test<std::vector<int>, std::vector<double>>();
+  test<std::deque<int>, std::vector<double>>();
+  test<MinSequenceContainer<int>, MinSequenceContainer<double>>();
+  test<std::vector<int, min_allocator<int>>, std::vector<double, min_allocator<double>>>();
+
+  {
+    auto erase_function = [](auto& m, auto) { m.erase(m.begin() + 2); };
+    test_erase_exception_guarantee(erase_function);
+  }
+
+  return 0;
+}
diff --git a/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.modifiers/erase_iter_iter.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.modifiers/erase_iter_iter.pass.cpp
new file mode 100644
index 0000000000000..103f38c1c5d4a
--- /dev/null
+++ b/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.modifiers/erase_iter_iter.pass.cpp
@@ -0,0 +1,99 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
+
+// <flat_map>
+
+// class flat_multimap
+
+// iterator erase(const_iterator first, const_iterator last);
+
+#include <compare>
+#include <concepts>
+#include <deque>
+#include <flat_map>
+#include <functional>
+#include <utility>
+#include <vector>
+
+#include "MinSequenceContainer.h"
+#include "../helpers.h"
+#include "test_macros.h"
+#include "min_allocator.h"
+
+template <class KeyContainer, class ValueContainer>
+void test() {
+  using Key   = typename KeyContainer::value_type;
+  using Value = typename ValueContainer::value_type;
+  using M     = std::flat_multimap<Key, Value, std::less<Key>, KeyContainer, ValueContainer>;
+  using P     = std::pair<Key, Value>;
+  using I     = M::iterator;
+
+  P ar[] = {
+      P(1, 1.5),
+      P(2, 2.5),
+      P(2, 2.6),
+      P(3, 3.5),
+      P(3, 3.6),
+      P(3, 3.7),
+      P(4, 4.5),
+      P(5, 5.5),
+      P(6, 6.5),
+      P(7, 7.5),
+      P(8, 8.5),
+  };
+  M m(ar, ar + sizeof(ar) / sizeof(ar[0]));
+  assert(m.size() == 11);
+  std::same_as<I> decltype(auto) i1 = m.erase(m.cbegin(), m.cbegin());
+  assert(m.size() == 11);
+  assert(i1 == m.begin());
+  assert(std::ranges::equal(
+      m,
+      std::vector<P>{
+          {1, 1.5},
+          {2, 2.5},
+          {2, 2.6},
+          {3, 3.5},
+          {3, 3.6},
+          {3, 3.7},
+          {4, 4.5},
+          {5, 5.5},
+          {6, 6.5},
+          {7, 7.5},
+          {8, 8.5}}));
+
+  std::same_as<I> decltype(auto) i2 = m.erase(m.cbegin(), std::next(m.cbegin(), 2));
+  assert(m.size() == 9);
+  assert(i2 == m.begin());
+  assert(std::ranges::equal(
+      m, std::vector<P>{{2, 2.6}, {3, 3.5}, {3, 3.6}, {3, 3.7}, {4, 4.5}, {5, 5.5}, {6, 6.5}, {7, 7.5}, {8, 8.5}}));
+
+  std::same_as<I> decltype(auto) i3 = m.erase(std::next(m.cbegin(), 2), std::next(m.cbegin(), 6));
+  assert(m.size() == 5);
+  assert(i3 == std::next(m.begin(), 2));
+  assert(std::ranges::equal(m, std::vector<P>{{2, 2.6}, {3, 3.5}, {6, 6.5}, {7, 7.5}, {8, 8.5}}));
+
+  std::same_as<I> decltype(auto) i4 = m.erase(m.cbegin(), m.cend());
+  assert(m.size() == 0);
+  assert(i4 == m.begin());
+  assert(i4 == m.end());
+}
+
+int main(int, char**) {
+  test<std::vector<int>, std::vector<double>>();
+  test<std::deque<int>, std::vector<double>>();
+  test<MinSequenceContainer<int>, MinSequenceContainer<double>>();
+  test<std::vector<int, min_allocator<int>>, std::vector<double, min_allocator<double>>>();
+
+  {
+    auto erase_function = [](auto& m, auto) { m.erase(m.begin(), m.begin() + 2); };
+    test_erase_exception_guarantee(erase_function);
+  }
+  return 0;
+}
diff --git a/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.modifiers/erase_key.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.modifiers/erase_key.pass.cpp
new file mode 100644
index 0000000000000..7944996fba1a0
--- /dev/null
+++ b/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.modifiers/erase_key.pass.cpp
@@ -0,0 +1,99 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
+
+// <flat_map>
+
+// class flat_multimap
+
+// size_type erase(const key_type& k);
+
+#include <compare>
+#include <concepts>
+#include <deque>
+#include <flat_map>
+#include <functional>
+#include <utility>
+#include <vector>
+
+#include "MinSequenceContainer.h"
+#include "../helpers.h"
+#include "test_macros.h"
+#include "min_allocator.h"
+
+template <class KeyContainer, class ValueContainer, class Compare = std::less<>>
+void test() {
+  using M = std::flat_multimap<int, char, Compare, KeyContainer, ValueContainer>;
+
+  auto make = [](std::initializer_list<int> il) {
+    M m;
+    for (int i : il) {
+      m.emplace(i, i);
+    }
+    return m;
+  };
+  M m = make({1, 1, 2, 2, 2, 3, 4, 5, 5, 6, 7, 8, 8, 8, 8, 9});
+  ASSERT_SAME_TYPE(decltype(m.erase(9)), typename M::size_type);
+  auto n = m.erase(10);
+  assert(n == 0);
+  assert(m == make({1, 1, 2, 2, 2, 3, 4, 5, 5, 6, 7, 8, 8, 8, 8, 9}));
+  n = m.erase(4);
+  assert(n == 1);
+  assert(m == make({1, 1, 2, 2, 2, 3, 5, 5, 6, 7, 8, 8, 8, 8, 9}));
+  n = m.erase(1);
+  assert(n == 2);
+  assert(m == make({2, 2, 2, 3, 5, 5, 6, 7, 8, 8, 8, 8, 9}));
+  n = m.erase(8);
+  assert(n == 4);
+  assert(m == make({2, 2, 2, 3, 5, 5, 6, 7, 9}));
+  n = m.erase(3);
+  assert(n == 1);
+  assert(m == make({2, 2, 2, 5, 5, 6, 7, 9}));
+  n = m.erase(4);
+  assert(n == 0);
+  assert(m == make({2, 2, 2, 5, 5, 6, 7, 9}));
+  n = m.erase(6);
+  assert(n == 1);
+  assert(m == make({2, 2, 2, 5, 5, 7, 9}));
+  n = m.erase(7);
+  assert(n == 1);
+  assert(m == make({2, 2, 2, 5, 5, 9}));
+  n = m.erase(2);
+  assert(n == 3);
+  assert(m == make({5, 5, 9}));
+  n = m.erase(5);
+  assert(n == 2);
+  assert(m == make({9}));
+  n = m.erase(9);
+  assert(n == 1);
+  assert(m.empty());
+  n = m.erase(1);
+  assert(n == 0);
+  assert(m.empty());
+}
+
+int main(int, char**) {
+  test<std::vector<int>, std::vector<char>>();
+  test<std::vector<int>, std::vector<char>, std::greater<>>();
+  test<std::deque<int>, std::vector<char>>();
+  test<MinSequenceContainer<int>, MinSequenceContainer<char>>();
+  test<std::vector<int, min_allocator<int>>, std::vector<char, min_allocator<char>>>();
+
+  {
+    auto erase_function = [](auto& m, auto key_arg) {
+      using Map = std::decay_t<decltype(m)>;
+      using Key = typename Map::key_type;
+      const Key key{key_arg};
+      m.erase(key);
+    };
+    test_erase_exception_guarantee(erase_function);
+  }
+
+  return 0;
+}
diff --git a/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.modifiers/erase_key_transparent.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.modifiers/erase_key_transparent.pass.cpp
new file mode 100644
index 0000000000000..75a2d205b8f87
--- /dev/null
+++ b/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.modifiers/erase_key_transparent.pass.cpp
@@ -0,0 +1,161 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
+
+// <flat_map>
+
+// class flat_multimap
+
+// size_type erase(K&& k);
+
+#include <compare>
+#include <concepts>
+#include <deque>
+#include <flat_map>
+#include <functional>
+#include <string>
+#include <utility>
+#include <vector>
+
+#include "MinSequenceContainer.h"
+#include "../helpers.h"
+#include "test_macros.h"
+#include "min_allocator.h"
+
+// Constraints: The qualified-id Compare::is_transparent is valid and denotes a type.
+template <class M>
+concept CanErase        = requires(M m, Transparent<int> k) { m.erase(k); };
+using TransparentMap    = std::flat_multimap<int, double, TransparentComparator>;
+using NonTransparentMap = std::flat_multimap<int, double, NonTransparentComparator>;
+static_assert(CanErase<TransparentMap>);
+static_assert(!CanErase<const TransparentMap>);
+static_assert(!CanErase<NonTransparentMap>);
+static_assert(!CanErase<const NonTransparentMap>);
+
+template <class Key, class It>
+struct HeterogeneousKey {
+  explicit HeterogeneousKey(Key key, It it) : key_(key), it_(it) {}
+  operator It() && { return it_; }
+  auto operator<=>(Key key) const { return key_ <=> key; }
+  friend bool operator<(const HeterogeneousKey&, const HeterogeneousKey&) {
+    assert(false);
+    return false;
+  }
+  Key key_;
+  It it_;
+};
+
+template <class KeyContainer, class ValueContainer>
+void test_simple() {
+  using Key   = typename KeyContainer::value_type;
+  using Value = typename ValueContainer::value_type;
+  using M     = std::flat_multimap<Key, Value, std::ranges::less, KeyContainer, ValueContainer>;
+
+  M m = {{1, 1}, {2, 2}, {2, 2}, {3, 3}, {3, 4}, {3, 5}, {4, 4}};
+  ASSERT_SAME_TYPE(decltype(m.erase(9)), typename M::size_type);
+  auto n = m.erase(3); // erase(K&&) [with K=int]
+  assert(n == 3);
+  assert((m == M{{1, 1}, {2, 2}, {2, 2}, {4, 4}}));
+  typename M::key_type lvalue = 2;
+  n                           = m.erase(lvalue); // erase(K&&) [with K=int&]
+  assert(n == 2);
+  assert((m == M{{1, 1}, {4, 4}}));
+  const typename M::key_type const_lvalue = 1;
+  n                                       = m.erase(const_lvalue); // erase(const key_type&)
+  assert(n == 1);
+  assert((m == M{{4, 4}}));
+}
+
+template <class KeyContainer, class ValueContainer>
+void test_transparent_comparator() {
+  using M = std::flat_multimap<std::string, int, TransparentComparator, KeyContainer, ValueContainer>;
+  using P = std::pair<std::string, int>;
+  M m     = {
+      {"alpha", 1}, {"beta", 2}, {"epsilon", 3}, {"epsilon", 4}, {"eta", 4}, {"gamma", 5}, {"gamma", 6}, {"gamma", 7}};
+  ASSERT_SAME_TYPE(decltype(m.erase(Transparent<std::string>{"abc"})), typename M::size_type);
+
+  auto n = m.erase(Transparent<std::string>{"epsilon"});
+  assert(n == 2);
+  assert(std::ranges::equal(
+      m, std::vector<P>{{"alpha", 1}, {"beta", 2}, {"eta", 4}, {"gamma", 5}, {"gamma", 6}, {"gamma", 7}}));
+
+  auto n2 = m.erase(Transparent<std::string>{"aaa"});
+  assert(n2 == 0);
+  assert(std::ranges::equal(
+      m, std::vector<P>{{"alpha", 1}, {"beta", 2}, {"eta", 4}, {"gamma", 5}, {"gamma", 6}, {"gamma", 7}}));
+
+  auto n3 = m.erase(Transparent<std::string>{"gamma"});
+  assert(n3 == 3);
+  assert(std::ranges::equal(m, std::vector<P>{{"alpha", 1}, {"beta", 2}, {"eta", 4}}));
+
+  auto n4 = m.erase(Transparent<std::string>{"alpha"});
+  assert(n4 == 1);
+  assert(std::ranges::equal(m, std::vector<P>{{"beta", 2}, {"eta", 4}}));
+
+  auto n5 = m.erase(Transparent<std::string>{"alpha"});
+  assert(n5 == 0);
+  assert(std::ranges::equal(m, std::vector<P>{{"beta", 2}, {"eta", 4}}));
+
+  auto n6 = m.erase(Transparent<std::string>{"beta"});
+  assert(n6 == 1);
+  assert(std::ranges::equal(m, std::vector<P>{{"eta", 4}}));
+
+  auto n7 = m.erase(Transparent<std::string>{"eta"});
+  assert(n7 == 1);
+  assert(std::ranges::equal(m, std::vector<P>{}));
+
+  auto n8 = m.erase(Transparent<std::string>{"eta"});
+  assert(n8 == 0);
+  assert(std::ranges::equal(m, std::vector<P>{}));
+}
+
+int main(int, char**) {
+  test_simple<std::vector<int>, std::vector<double>>();
+  test_simple<std::deque<int>, std::vector<double>>();
+  test_simple<MinSequenceContainer<int>, MinSequenceContainer<double>>();
+  test_simple<std::vector<int, min_allocator<int>>, std::vector<double, min_allocator<double>>>();
+
+  test_transparent_comparator<std::vector<std::string>, std::vector<int>>();
+  test_transparent_comparator<std::deque<std::string>, std::vector<int>>();
+  test_transparent_comparator<MinSequenceContainer<std::string>, MinSequenceContainer<int>>();
+  test_transparent_comparator<std::vector<std::string, min_allocator<std::string>>,
+                              std::vector<int, min_allocator<int>>>();
+
+  {
+    // P2077's HeterogeneousKey example
+    using M = std::flat_multimap<int, int, std::less<>>;
+    M m     = {{1, 1}, {2, 2}, {3, 3}, {3, 3}, {4, 4}, {5, 5}, {6, 6}, {6, 6}, {7, 7}, {8, 8}, {8, 8}};
+    auto h1 = HeterogeneousKey<int, M::iterator>(8, m.begin());
+    std::same_as<M::size_type> auto n = m.erase(h1); // lvalue is not convertible to It; erase(K&&) is the best match
+    assert(n == 2);
+    assert((m == M{{1, 1}, {2, 2}, {3, 3}, {3, 3}, {4, 4}, {5, 5}, {6, 6}, {6, 6}, {7, 7}}));
+    std::same_as<M::iterator> auto it = m.erase(std::move(h1)); // rvalue is convertible to It; erase(K&&) drops out
+    assert(it == m.begin());
+    assert((m == M{{2, 2}, {3, 3}, {3, 3}, {4, 4}, {5, 5}, {6, 6}, {6, 6}, {7, 7}}));
+  }
+  {
+    bool transparent_used = false;
+    TransparentComparator c(transparent_used);
+    std::flat_multimap<int, int, TransparentComparator> m(std::sorted_equivalent, {{1, 1}, {2, 2}, {3, 3}, {3, 3}}, c);
+    assert(!transparent_used);
+    auto n = m.erase(Transparent<int>{3});
+    assert(n == 2);
+    assert(transparent_used);
+  }
+  {
+    auto erase_transparent = [](auto& m, auto key_arg) {
+      using Map = std::decay_t<decltype(m)>;
+      using Key = typename Map::key_type;
+      m.erase(Transparent<Key>{key_arg});
+    };
+    test_erase_exception_guarantee(erase_transparent);
+  }
+
+  return 0;
+}
diff --git a/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.modifiers/extract.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.modifiers/extract.pass.cpp
new file mode 100644
index 0000000000000..f5ed4a9663a9d
--- /dev/null
+++ b/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.modifiers/extract.pass.cpp
@@ -0,0 +1,93 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
+
+// <flat_map>
+
+// class flat_multimap
+
+// containers extract() &&;
+
+#include <algorithm>
+#include <concepts>
+#include <deque>
+#include <flat_map>
+#include <functional>
+#include <vector>
+
+#include "MinSequenceContainer.h"
+#include "../helpers.h"
+#include "test_macros.h"
+#include "min_allocator.h"
+
+template <class T>
+concept CanExtract = requires(T&& t) { std::forward<T>(t).extract(); };
+
+static_assert(CanExtract<std::flat_multimap<int, int>&&>);
+static_assert(!CanExtract<std::flat_multimap<int, int>&>);
+static_assert(!CanExtract<std::flat_multimap<int, int> const&>);
+static_assert(!CanExtract<std::flat_multimap<int, int> const&&>);
+
+template <class KeyContainer, class ValueContainer>
+void test() {
+  using M = std::flat_multimap<int, int, std::less<int>, KeyContainer, ValueContainer>;
+  M m     = M({1, 2, 2, 2, 3, 3}, {4, 5, 6, 7, 8, 9});
+
+  std::same_as<typename M::containers> auto containers = std::move(m).extract();
+
+  auto expected_keys   = {1, 2, 2, 2, 3, 3};
+  auto expected_values = {4, 5, 6, 7, 8, 9};
+  assert(std::ranges::equal(containers.keys, expected_keys));
+  assert(std::ranges::equal(containers.values, expected_values));
+  check_invariant(m);
+  LIBCPP_ASSERT(m.empty());
+  LIBCPP_ASSERT(m.keys().size() == 0);
+  LIBCPP_ASSERT(m.values().size() == 0);
+}
+
+int main(int, char**) {
+  test<std::vector<int>, std::vector<int>>();
+  test<std::deque<int>, std::vector<int>>();
+  test<MinSequenceContainer<int>, MinSequenceContainer<int>>();
+  test<std::vector<int, min_allocator<int>>, std::vector<int, min_allocator<int>>>();
+  {
+    // extracted object maintains invariant if one of underlying container does not clear after move
+    using M = std::flat_multimap<int, int, std::less<>, std::vector<int>, CopyOnlyVector<int>>;
+    M m     = M({1, 2, 2, 2, 3, 3}, {1, 2, 3, 4, 5, 6});
+    std::same_as<M::containers> auto containers = std::move(m).extract();
+    assert(containers.keys.size() == 6);
+    assert(containers.values.size() == 6);
+    check_invariant(m);
+    LIBCPP_ASSERT(m.empty());
+    LIBCPP_ASSERT(m.keys().size() == 0);
+    LIBCPP_ASSERT(m.values().size() == 0);
+  }
+
+  {
+#ifndef TEST_HAS_NO_EXCEPTIONS
+    using KeyContainer   = std::vector<int>;
+    using ValueContainer = ThrowOnMoveContainer<int>;
+    using M              = std::flat_multimap<int, int, std::ranges::less, KeyContainer, ValueContainer>;
+
+    M m;
+    m.emplace(1, 1);
+    m.emplace(1, 1);
+    try {
+      auto c = std::move(m).extract();
+      assert(false);
+    } catch (int) {
+      check_invariant(m);
+      // In libc++, we try to erase the key after value emplacement failure.
+      // and after erasure failure, we clear the flat_multimap
+      LIBCPP_ASSERT(m.size() == 0);
+    }
+#endif
+  }
+  return 0;
+}
diff --git a/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.modifiers/insert_cv.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.modifiers/insert_cv.pass.cpp
new file mode 100644
index 0000000000000..88c173d8a6917
--- /dev/null
+++ b/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.modifiers/insert_cv.pass.cpp
@@ -0,0 +1,81 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
+
+// <flat_map>
+
+// class flat_multimap
+
+// iterator insert(const value_type& v);
+
+#include <flat_map>
+#include <deque>
+#include <cassert>
+#include <functional>
+
+#include "MinSequenceContainer.h"
+#include "test_macros.h"
+#include "../helpers.h"
+#include "min_allocator.h"
+
+template <class KeyContainer, class ValueContainer>
+void test() {
+  using Key   = typename KeyContainer::value_type;
+  using Value = typename ValueContainer::value_type;
+  using M     = std::flat_multimap<Key, Value, std::less<Key>, KeyContainer, ValueContainer>;
+  using R     = typename M::iterator;
+  using VT    = typename M::value_type;
+  M m;
+
+  const VT v1(2, 2.5);
+  std::same_as<R> decltype(auto) r = m.insert(v1);
+  assert(r == m.begin());
+  assert(m.size() == 1);
+  assert(r->first == 2);
+  assert(r->second == 2.5);
+
+  const VT v2(1, 1.5);
+  r = m.insert(v2);
+  assert(r == m.begin());
+  assert(m.size() == 2);
+  assert(r->first == 1);
+  assert(r->second == 1.5);
+
+  const VT v3(3, 3.5);
+  r = m.insert(v3);
+  assert(r == m.begin() + 2);
+  assert(m.size() == 3);
+  assert(r->first == 3);
+  assert(r->second == 3.5);
+
+  const VT v4(3, 4.5);
+  r = m.insert(v4);
+  assert(r == m.begin() + 3);
+  assert(m.size() == 4);
+  assert(r->first == 3);
+  assert(r->second == 4.5);
+}
+
+int main(int, char**) {
+  test<std::vector<int>, std::vector<double>>();
+  test<std::deque<int>, std::vector<double>>();
+  test<MinSequenceContainer<int>, MinSequenceContainer<double>>();
+  test<std::vector<int, min_allocator<int>>, std::vector<double, min_allocator<double>>>();
+
+  {
+    auto insert_func = [](auto& m, auto key_arg, auto value_arg) {
+      using FlatMap    = std::decay_t<decltype(m)>;
+      using value_type = typename FlatMap::value_type;
+      const value_type p(std::piecewise_construct, std::tuple(key_arg), std::tuple(value_arg));
+      m.insert(p);
+    };
+    test_emplace_exception_guarantee(insert_func);
+  }
+  return 0;
+}
diff --git a/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.modifiers/insert_initializer_list.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.modifiers/insert_initializer_list.pass.cpp
new file mode 100644
index 0000000000000..098b66cc49f18
--- /dev/null
+++ b/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.modifiers/insert_initializer_list.pass.cpp
@@ -0,0 +1,83 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
+
+// <flat_map>
+
+// class flat_multimap
+
+// void insert(initializer_list<value_type> il);
+
+#include <flat_map>
+#include <cassert>
+#include <functional>
+#include <deque>
+
+#include "MinSequenceContainer.h"
+#include "../helpers.h"
+#include "test_macros.h"
+#include "min_allocator.h"
+
+template <class KeyContainer, class ValueContainer>
+void test() {
+  using Key   = typename KeyContainer::value_type;
+  using Value = typename ValueContainer::value_type;
+  using M     = std::flat_multimap<Key, Value, std::less<Key>, KeyContainer, ValueContainer>;
+  using V     = std::pair<int, double>;
+
+  M m = {{1, 1}, {1, 1.5}, {1, 2}, {3, 1}, {3, 1.5}, {3, 2}};
+  m.insert({
+      {4, 1},
+      {4, 1.5},
+      {4, 2},
+      {1, 1},
+      {1, 1.5},
+      {1, 2},
+      {2, 1},
+      {2, 1.5},
+      {2, 2},
+  });
+  assert(m.size() == 15);
+  std::vector<V> expected = {
+      {1, 1},
+      {1, 1.5},
+      {1, 2},
+      {1, 1},
+      {1, 1.5},
+      {1, 2},
+      {2, 1},
+      {2, 1.5},
+      {2, 2},
+      {3, 1},
+      {3, 1.5},
+      {3, 2},
+      {4, 1},
+      {4, 1.5},
+      {4, 2},
+  };
+  assert(std::ranges::equal(m, expected));
+}
+
+int main(int, char**) {
+  test<std::vector<int>, std::vector<double>>();
+  test<std::deque<int>, std::vector<double>>();
+  test<MinSequenceContainer<int>, MinSequenceContainer<double>>();
+  test<std::vector<int, min_allocator<int>>, std::vector<double, min_allocator<double>>>();
+
+  {
+    auto insert_func = [](auto& m, const auto& newValues) {
+      using FlatMap                        = std::decay_t<decltype(m)>;
+      using value_type                     = typename FlatMap::value_type;
+      std::initializer_list<value_type> il = {{newValues[0].first, newValues[0].second}};
+      m.insert(il);
+    };
+    test_insert_range_exception_guarantee(insert_func);
+  }
+  return 0;
+}
diff --git a/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.modifiers/insert_iter_cv.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.modifiers/insert_iter_cv.pass.cpp
new file mode 100644
index 0000000000000..9d645043a15ca
--- /dev/null
+++ b/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.modifiers/insert_iter_cv.pass.cpp
@@ -0,0 +1,95 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
+
+// <flat_map>
+
+// class flat_multimap
+
+// iterator insert(const_iterator position, const value_type& v);
+
+#include <flat_map>
+#include <cassert>
+#include <functional>
+#include <deque>
+
+#include "MinSequenceContainer.h"
+#include "test_macros.h"
+#include "../helpers.h"
+#include "min_allocator.h"
+
+template <class KeyContainer, class ValueContainer>
+void test() {
+  using Key   = typename KeyContainer::value_type;
+  using Value = typename ValueContainer::value_type;
+  using M     = std::flat_multimap<Key, Value, std::less<Key>, KeyContainer, ValueContainer>;
+  using R     = typename M::iterator;
+  using VT    = typename M::value_type;
+
+  M m;
+  const VT v1(2, 2.5);
+  std::same_as<R> decltype(auto) r = m.insert(m.end(), v1);
+  assert(r == m.begin());
+  assert(m.size() == 1);
+  assert(r->first == 2);
+  assert(r->second == 2.5);
+
+  const VT v2(1, 1.5);
+  r = m.insert(m.end(), v2);
+  assert(r == m.begin());
+  assert(m.size() == 2);
+  assert(r->first == 1);
+  assert(r->second == 1.5);
+
+  const VT v3(3, 3.5);
+  r = m.insert(m.end(), v3);
+  assert(r == std::ranges::prev(m.end()));
+  assert(m.size() == 3);
+  assert(r->first == 3);
+  assert(r->second == 3.5);
+
+  const VT v4(3, 4.5);
+  r = m.insert(m.end(), v4);
+  assert(r == std::ranges::prev(m.end()));
+  assert(m.size() == 4);
+  assert(r->first == 3);
+  assert(r->second == 4.5);
+
+  const VT v5(2, 5.5);
+  r = m.insert(m.end(), v5);
+  assert(r == m.begin() + 2);
+  assert(m.size() == 5);
+  assert(r->first == 2);
+  assert(r->second == 5.5);
+
+  const VT v6(2, 6.5);
+  r = m.insert(m.begin(), v6);
+  assert(r == m.begin() + 1);
+  assert(m.size() == 6);
+  assert(r->first == 2);
+  assert(r->second == 6.5);
+}
+
+int main(int, char**) {
+  test<std::vector<int>, std::vector<double>>();
+  test<std::deque<int>, std::vector<double>>();
+  test<MinSequenceContainer<int>, MinSequenceContainer<double>>();
+  test<std::vector<int, min_allocator<int>>, std::vector<double, min_allocator<double>>>();
+
+  {
+    auto insert_func = [](auto& m, auto key_arg, auto value_arg) {
+      using FlatMap    = std::decay_t<decltype(m)>;
+      using value_type = typename FlatMap::value_type;
+      const value_type p(std::piecewise_construct, std::tuple(key_arg), std::tuple(value_arg));
+      m.insert(m.begin(), p);
+    };
+    test_emplace_exception_guarantee(insert_func);
+  }
+  return 0;
+}
diff --git a/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.modifiers/insert_iter_iter.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.modifiers/insert_iter_iter.pass.cpp
new file mode 100644
index 0000000000000..ae031bd010f76
--- /dev/null
+++ b/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.modifiers/insert_iter_iter.pass.cpp
@@ -0,0 +1,109 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
+
+// <flat_map>
+
+// class flat_multimap
+
+// template <class InputIterator>
+//   void insert(InputIterator first, InputIterator last);
+
+#include <flat_map>
+#include <cassert>
+#include <functional>
+#include <deque>
+
+#include "MinSequenceContainer.h"
+#include "../helpers.h"
+#include "test_macros.h"
+#include "test_iterators.h"
+#include "min_allocator.h"
+
+// test constraint InputIterator
+template <class M, class... Args>
+concept CanInsert = requires(M m, Args&&... args) { m.insert(std::forward<Args>(args)...); };
+
+using Map  = std::flat_multimap<int, int>;
+using Pair = std::pair<int, int>;
+
+static_assert(CanInsert<Map, Pair*, Pair*>);
+static_assert(CanInsert<Map, cpp17_input_iterator<Pair*>, cpp17_input_iterator<Pair*>>);
+static_assert(!CanInsert<Map, int, int>);
+static_assert(!CanInsert<Map, cpp20_input_iterator<Pair*>, cpp20_input_iterator<Pair*>>);
+
+template <class KeyContainer, class ValueContainer>
+void test() {
+  using P = std::pair<int, double>;
+  using M = std::flat_multimap<int, double, std::less<int>, KeyContainer, ValueContainer>;
+
+  P ar1[] = {
+      P(2, 1),
+      P(2, 1.5),
+      P(2, 2),
+      P(1, 1),
+      P(1, 1.5),
+      P(1, 2),
+      P(3, 1),
+      P(3, 1.5),
+      P(3, 2),
+  };
+  P ar2[] = {
+      P(4, 1),
+      P(4, 1.5),
+      P(4, 2),
+      P(1, 1),
+      P(1, 1.5),
+      P(1, 2),
+      P(0, 1),
+      P(0, 1.5),
+      P(0, 2),
+  };
+
+  M m;
+  m.insert(cpp17_input_iterator<P*>(ar1), cpp17_input_iterator<P*>(ar1 + sizeof(ar1) / sizeof(ar1[0])));
+  assert(m.size() == 9);
+  std::vector<P> expected{{1, 1}, {1, 1.5}, {1, 2}, {2, 1}, {2, 1.5}, {2, 2}, {3, 1}, {3, 1.5}, {3, 2}};
+  assert(std::ranges::equal(m, expected));
+
+  m.insert(cpp17_input_iterator<P*>(ar2), cpp17_input_iterator<P*>(ar2 + sizeof(ar2) / sizeof(ar2[0])));
+  assert(m.size() == 18);
+  std::vector<P> expected2{
+      {0, 1},
+      {0, 1.5},
+      {0, 2},
+      {1, 1},
+      {1, 1.5},
+      {1, 2},
+      {1, 1},
+      {1, 1.5},
+      {1, 2},
+      {2, 1},
+      {2, 1.5},
+      {2, 2},
+      {3, 1},
+      {3, 1.5},
+      {3, 2},
+      {4, 1},
+      {4, 1.5},
+      {4, 2}};
+  assert(std::ranges::equal(m, expected2));
+}
+int main(int, char**) {
+  test<std::vector<int>, std::vector<double>>();
+  test<std::deque<int>, std::vector<double>>();
+  test<MinSequenceContainer<int>, MinSequenceContainer<double>>();
+  test<std::vector<int, min_allocator<int>>, std::vector<double, min_allocator<double>>>();
+
+  {
+    auto insert_func = [](auto& m, const auto& newValues) { m.insert(newValues.begin(), newValues.end()); };
+    test_insert_range_exception_guarantee(insert_func);
+  }
+  return 0;
+}
diff --git a/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.modifiers/insert_iter_rv.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.modifiers/insert_iter_rv.pass.cpp
new file mode 100644
index 0000000000000..61962f4873aee
--- /dev/null
+++ b/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.modifiers/insert_iter_rv.pass.cpp
@@ -0,0 +1,103 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
+
+// <flat_map>
+
+// class flat_multimap
+
+// iterator insert(const_iterator position, value_type&&);
+
+#include <flat_map>
+#include <cassert>
+#include <deque>
+
+#include "MinSequenceContainer.h"
+#include "MoveOnly.h"
+#include "min_allocator.h"
+#include "../helpers.h"
+#include "test_macros.h"
+
+template <class Container, class Pair>
+void do_insert_iter_rv_test() {
+  using M = Container;
+  using P = Pair;
+  using R = typename M::iterator;
+  M m;
+  std::same_as<R> decltype(auto) r = m.insert(m.end(), P(2, 2));
+  assert(r == m.begin());
+  assert(m.size() == 1);
+  assert(r->first == 2);
+  assert(r->second == 2);
+
+  r = m.insert(m.end(), P(1, 1));
+  assert(r == m.begin());
+  assert(m.size() == 2);
+  assert(r->first == 1);
+  assert(r->second == 1);
+
+  r = m.insert(m.end(), P(3, 3));
+  assert(r == std::ranges::prev(m.end()));
+  assert(m.size() == 3);
+  assert(r->first == 3);
+  assert(r->second == 3);
+
+  r = m.insert(m.end(), P(3, 4));
+  assert(r == std::ranges::prev(m.end()));
+  assert(m.size() == 4);
+  assert(r->first == 3);
+  assert(r->second == 4);
+
+  r = m.insert(m.end(), P(2, 5));
+  assert(r == m.begin() + 2);
+  assert(m.size() == 5);
+  assert(r->first == 2);
+  assert(r->second == 5);
+
+  r = m.insert(m.begin(), P(2, 6));
+  assert(r == m.begin() + 1);
+  assert(m.size() == 6);
+  assert(r->first == 2);
+  assert(r->second == 6);
+}
+
+template <class KeyContainer, class ValueContainer>
+void test() {
+  using Key   = typename KeyContainer::value_type;
+  using Value = typename ValueContainer::value_type;
+  using M     = std::flat_multimap<Key, Value, std::less<Key>, KeyContainer, ValueContainer>;
+  using P     = std::pair<Key, Value>;
+  using CP    = std::pair<const Key, Value>;
+
+  do_insert_iter_rv_test<M, P>();
+  do_insert_iter_rv_test<M, CP>();
+}
+
+int main(int, char**) {
+  test<std::vector<int>, std::vector<double>>();
+  test<std::vector<int>, std::vector<MoveOnly>>();
+  test<std::deque<int>, std::deque<double>>();
+  test<std::deque<int>, std::deque<MoveOnly>>();
+  test<MinSequenceContainer<int>, MinSequenceContainer<double>>();
+  test<MinSequenceContainer<int>, MinSequenceContainer<MoveOnly>>();
+  test<std::vector<int, min_allocator<int>>, std::vector<double, min_allocator<double>>>();
+  test<std::vector<int, min_allocator<int>>, std::vector<MoveOnly, min_allocator<MoveOnly>>>();
+
+  {
+    auto insert_func = [](auto& m, auto key_arg, auto value_arg) {
+      using FlatMap    = std::decay_t<decltype(m)>;
+      using value_type = typename FlatMap::value_type;
+      value_type p(std::piecewise_construct, std::tuple(key_arg), std::tuple(value_arg));
+      m.insert(m.begin(), std::move(p));
+    };
+    test_emplace_exception_guarantee(insert_func);
+  }
+
+  return 0;
+}
diff --git a/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.modifiers/insert_range.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.modifiers/insert_range.pass.cpp
new file mode 100644
index 0000000000000..97b8f17d1094f
--- /dev/null
+++ b/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.modifiers/insert_range.pass.cpp
@@ -0,0 +1,101 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
+
+// <flat_map>
+
+// class flat_multimap
+
+// template<container-compatible-range<value_type> R>
+//   void insert_range(R&& rg);
+
+#include <algorithm>
+#include <deque>
+#include <flat_map>
+#include <functional>
+#include <ranges>
+#include <vector>
+
+#include "MinSequenceContainer.h"
+#include "../helpers.h"
+#include "MoveOnly.h"
+#include "test_macros.h"
+#include "test_iterators.h"
+#include "min_allocator.h"
+
+// test constraint container-compatible-range
+template <class M, class R>
+concept CanInsertRange = requires(M m, R&& r) { m.insert_range(std::forward<R>(r)); };
+
+using Map = std::flat_multimap<int, double>;
+
+static_assert(CanInsertRange<Map, std::ranges::subrange<std::pair<int, double>*>>);
+static_assert(CanInsertRange<Map, std::ranges::subrange<std::pair<short, double>*>>);
+static_assert(!CanInsertRange<Map, std::ranges::subrange<int*>>);
+static_assert(!CanInsertRange<Map, std::ranges::subrange<double*>>);
+
+template <class KeyContainer, class ValueContainer>
+void test() {
+  using Key   = typename KeyContainer::value_type;
+  using Value = typename ValueContainer::value_type;
+
+  {
+    using P                 = std::pair<int, int>;
+    using M                 = std::flat_multimap<Key, Value, std::less<Key>, KeyContainer, ValueContainer>;
+    using It                = forward_iterator<const P*>;
+    M m                     = {{10, 1}, {8, 2}, {5, 3}, {2, 4}, {1, 5}};
+    P ar[]                  = {{3, 1}, {1, 2}, {4, 3}, {1, 4}, {5, 5}, {9, 6}};
+    std::ranges::subrange r = {It(ar), It(ar + 6)};
+    static_assert(std::ranges::common_range<decltype(r)>);
+    m.insert_range(r);
+    std::vector<P> expected = {{1, 5}, {1, 2}, {1, 4}, {2, 4}, {3, 1}, {4, 3}, {5, 3}, {5, 5}, {8, 2}, {9, 6}, {10, 1}};
+    assert(std::ranges::equal(m, expected));
+  }
+  {
+    using P                 = std::pair<int, int>;
+    using M                 = std::flat_multimap<Key, Value, std::greater<>, KeyContainer, ValueContainer>;
+    using It                = cpp20_input_iterator<const P*>;
+    M m                     = {{8, 1}, {5, 2}, {3, 3}, {2, 4}};
+    P ar[]                  = {{3, 1}, {1, 2}, {4, 3}, {1, 4}, {5, 5}, {9, 6}};
+    std::ranges::subrange r = {It(ar), sentinel_wrapper<It>(It(ar + 6))};
+    static_assert(!std::ranges::common_range<decltype(r)>);
+    m.insert_range(r);
+    std::vector<P> expected = {{9, 6}, {8, 1}, {5, 2}, {5, 5}, {4, 3}, {3, 3}, {3, 1}, {2, 4}, {1, 2}, {1, 4}};
+    assert(std::ranges::equal(m, expected));
+  }
+}
+
+int main(int, char**) {
+  test<std::vector<int>, std::vector<int>>();
+  test<std::deque<int>, std::vector<int>>();
+  test<MinSequenceContainer<int>, MinSequenceContainer<int>>();
+  test<std::vector<int, min_allocator<int>>, std::vector<int, min_allocator<int>>>();
+  {
+    // Items are forwarded correctly from the input range (P2767).
+    std::pair<MoveOnly, MoveOnly> a[] = {{3, 3}, {1, 1}, {4, 4}, {1, 1}, {5, 5}};
+    std::flat_multimap<MoveOnly, MoveOnly> m;
+    m.insert_range(a | std::views::as_rvalue);
+    std::pair<MoveOnly, MoveOnly> expected[] = {{1, 1}, {1, 1}, {3, 3}, {4, 4}, {5, 5}};
+    assert(std::ranges::equal(m, expected));
+  }
+  {
+    // The element type of the range doesn't need to be std::pair (P2767).
+    std::pair<int, int> pa[] = {{3, 3}, {1, 1}, {4, 4}, {1, 1}, {5, 5}};
+    std::deque<std::reference_wrapper<std::pair<int, int>>> a(pa, pa + 5);
+    std::flat_multimap<int, int> m;
+    m.insert_range(a);
+    std::pair<int, int> expected[] = {{1, 1}, {1, 1}, {3, 3}, {4, 4}, {5, 5}};
+    assert(std::ranges::equal(m, expected));
+  }
+  {
+    auto insert_func = [](auto& m, const auto& newValues) { m.insert_range(newValues); };
+    test_insert_range_exception_guarantee(insert_func);
+  }
+  return 0;
+}
diff --git a/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.modifiers/insert_rv.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.modifiers/insert_rv.pass.cpp
new file mode 100644
index 0000000000000..573150248ca48
--- /dev/null
+++ b/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.modifiers/insert_rv.pass.cpp
@@ -0,0 +1,116 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
+
+// <flat_map>
+
+// class flat_multimap
+
+// iterator insert( value_type&& v);
+
+#include <flat_map>
+#include <cassert>
+#include <deque>
+
+#include "MinSequenceContainer.h"
+#include "MoveOnly.h"
+#include "min_allocator.h"
+#include "test_macros.h"
+#include "../helpers.h"
+
+template <class Container, class Pair>
+void do_insert_rv_test() {
+  using M = Container;
+  using P = Pair;
+  using R = typename M::iterator;
+  M m;
+  std::same_as<R> decltype(auto) r = m.insert(P(2, 2));
+  assert(r == m.begin());
+  assert(m.size() == 1);
+  assert(r->first == 2);
+  assert(r->second == 2);
+
+  r = m.insert(P(1, 1));
+  assert(r == m.begin());
+  assert(m.size() == 2);
+  assert(r->first == 1);
+  assert(r->second == 1);
+
+  r = m.insert(P(3, 3));
+  assert(r == std::ranges::prev(m.end()));
+  assert(m.size() == 3);
+  assert(r->first == 3);
+  assert(r->second == 3);
+
+  r = m.insert(P(3, 3));
+  assert(r == std::ranges::prev(m.end()));
+  assert(m.size() == 4);
+  assert(r->first == 3);
+  assert(r->second == 3);
+}
+
+template <class KeyContainer, class ValueContainer>
+void test() {
+  using Key   = typename KeyContainer::value_type;
+  using Value = typename ValueContainer::value_type;
+  using M     = std::flat_multimap<Key, Value, TransparentComparator, KeyContainer, ValueContainer>;
+
+  using P  = std::pair<Key, Value>;
+  using CP = std::pair<const Key, Value>;
+
+  do_insert_rv_test<M, P>();
+  do_insert_rv_test<M, CP>();
+}
+
+int main(int, char**) {
+  test<std::vector<int>, std::vector<MoveOnly>>();
+  test<std::deque<int>, std::vector<MoveOnly>>();
+  test<MinSequenceContainer<int>, MinSequenceContainer<MoveOnly>>();
+  test<std::vector<int, min_allocator<int>>, std::vector<MoveOnly, min_allocator<MoveOnly>>>();
+
+  {
+    using M = std::flat_multimap<int, MoveOnly>;
+    using R = M::iterator;
+    M m;
+    R r = m.insert({2, MoveOnly(2)});
+    assert(r == m.begin());
+    assert(m.size() == 1);
+    assert(r->first == 2);
+    assert(r->second == 2);
+
+    r = m.insert({1, MoveOnly(1)});
+    assert(r == m.begin());
+    assert(m.size() == 2);
+    assert(r->first == 1);
+    assert(r->second == 1);
+
+    r = m.insert({3, MoveOnly(3)});
+    assert(r == std::ranges::prev(m.end()));
+    assert(m.size() == 3);
+    assert(r->first == 3);
+    assert(r->second == 3);
+
+    r = m.insert({3, MoveOnly(3)});
+    assert(r == std::ranges::prev(m.end()));
+    assert(m.size() == 4);
+    assert(r->first == 3);
+    assert(r->second == 3);
+  }
+  {
+    auto insert_func = [](auto& m, auto key_arg, auto value_arg) {
+      using FlatMap    = std::decay_t<decltype(m)>;
+      using value_type = typename FlatMap::value_type;
+      value_type p(std::piecewise_construct, std::tuple(key_arg), std::tuple(value_arg));
+      m.insert(std::move(p));
+    };
+    test_emplace_exception_guarantee(insert_func);
+  }
+
+  return 0;
+}
diff --git a/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.modifiers/insert_sorted_initializer_list.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.modifiers/insert_sorted_initializer_list.pass.cpp
new file mode 100644
index 0000000000000..334dff0a0d2f6
--- /dev/null
+++ b/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.modifiers/insert_sorted_initializer_list.pass.cpp
@@ -0,0 +1,66 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
+
+// <flat_map>
+
+// class flat_multimap
+
+// void insert(sorted_equivalent_t, initializer_list<value_type> il);
+
+#include <flat_map>
+#include <cassert>
+#include <functional>
+#include <deque>
+
+#include "MinSequenceContainer.h"
+#include "../helpers.h"
+#include "test_macros.h"
+#include "min_allocator.h"
+
+template <class KeyContainer, class ValueContainer>
+void test() {
+  using Key   = typename KeyContainer::value_type;
+  using Value = typename ValueContainer::value_type;
+  using M     = std::flat_multimap<Key, Value, std::less<Key>, KeyContainer, ValueContainer>;
+
+  using V = std::pair<const Key, Value>;
+  M m     = {{1, 1}, {1, 1.5}, {1, 2}, {3, 1}, {3, 1.5}, {3, 2}};
+  m.insert(std::sorted_equivalent,
+           {
+               {0, 1},
+               {1, 2},
+               {1, 3},
+               {2, 1},
+               {2, 4},
+               {4, 1},
+           });
+  assert(m.size() == 12);
+  V expected[] = {{0, 1}, {1, 1}, {1, 1.5}, {1, 2}, {1, 2}, {1, 3}, {2, 1}, {2, 4}, {3, 1}, {3, 1.5}, {3, 2}, {4, 1}};
+  assert(std::ranges::equal(m, expected));
+}
+
+int main(int, char**) {
+  test<std::vector<int>, std::vector<double>>();
+  test<std::deque<int>, std::vector<double>>();
+  test<MinSequenceContainer<int>, MinSequenceContainer<double>>();
+  test<std::vector<int, min_allocator<int>>, std::vector<double, min_allocator<double>>>();
+
+  {
+    auto insert_func = [](auto& m, const auto& newValues) {
+      using FlatMap                        = std::decay_t<decltype(m)>;
+      using value_type                     = typename FlatMap::value_type;
+      std::initializer_list<value_type> il = {{newValues[0].first, newValues[0].second}};
+      m.insert(std::sorted_equivalent, il);
+    };
+    test_insert_range_exception_guarantee(insert_func);
+  }
+
+  return 0;
+}
diff --git a/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.modifiers/insert_sorted_iter_iter.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.modifiers/insert_sorted_iter_iter.pass.cpp
new file mode 100644
index 0000000000000..37808470a2cf7
--- /dev/null
+++ b/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.modifiers/insert_sorted_iter_iter.pass.cpp
@@ -0,0 +1,94 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
+
+// <flat_map>
+
+// class flat_multimap
+
+// template <class InputIterator>
+//   void insert(sorted_equivalent_t, InputIterator first, InputIterator last);
+
+#include <flat_map>
+#include <cassert>
+#include <functional>
+#include <deque>
+
+#include "MinSequenceContainer.h"
+#include "../helpers.h"
+#include "test_macros.h"
+#include "test_iterators.h"
+#include "min_allocator.h"
+
+// test constraint InputIterator
+template <class M, class... Args>
+concept CanInsert = requires(M m, Args&&... args) { m.insert(std::forward<Args>(args)...); };
+
+using Map  = std::flat_multimap<int, int>;
+using Pair = std::pair<int, int>;
+
+static_assert(CanInsert<Map, std::sorted_equivalent_t, Pair*, Pair*>);
+static_assert(CanInsert<Map, std::sorted_equivalent_t, cpp17_input_iterator<Pair*>, cpp17_input_iterator<Pair*>>);
+static_assert(!CanInsert<Map, std::sorted_equivalent_t, int, int>);
+static_assert(!CanInsert<Map, std::sorted_equivalent_t, cpp20_input_iterator<Pair*>, cpp20_input_iterator<Pair*>>);
+
+template <class KeyContainer, class ValueContainer>
+void test() {
+  using Key   = typename KeyContainer::value_type;
+  using Value = typename ValueContainer::value_type;
+  using M     = std::flat_multimap<Key, Value, std::less<Key>, KeyContainer, ValueContainer>;
+  using P     = std::pair<Key, Value>;
+
+  P ar1[] = {
+      P(1, 1),
+      P(1, 0),
+      P(2, 1),
+      P(2, 3),
+      P(3, 1),
+  };
+
+  P ar2[] = {
+      P(0, 1),
+      P(2, 2),
+      P(2, 5),
+      P(4, 1),
+      P(4, 4),
+  };
+
+  M m;
+  m.insert(std::sorted_equivalent,
+           cpp17_input_iterator<P*>(ar1),
+           cpp17_input_iterator<P*>(ar1 + sizeof(ar1) / sizeof(ar1[0])));
+  assert(m.size() == 5);
+  P expected[] = {{1, 1}, {1, 0}, {2, 1}, {2, 3}, {3, 1}};
+  assert(std::ranges::equal(m, expected));
+
+  m.insert(std::sorted_equivalent,
+           cpp17_input_iterator<P*>(ar2),
+           cpp17_input_iterator<P*>(ar2 + sizeof(ar2) / sizeof(ar2[0])));
+  assert(m.size() == 10);
+  P expected2[] = {{0, 1}, {1, 1}, {1, 0}, {2, 1}, {2, 3}, {2, 2}, {2, 5}, {3, 1}, {4, 1}, {4, 4}};
+  assert(std::ranges::equal(m, expected2));
+}
+
+int main(int, char**) {
+  test<std::vector<int>, std::vector<double>>();
+  test<std::deque<int>, std::vector<double>>();
+  test<MinSequenceContainer<int>, MinSequenceContainer<double>>();
+  test<std::vector<int, min_allocator<int>>, std::vector<double, min_allocator<double>>>();
+
+  {
+    auto insert_func = [](auto& m, const auto& newValues) {
+      m.insert(std::sorted_equivalent, newValues.begin(), newValues.end());
+    };
+    test_insert_range_exception_guarantee(insert_func);
+  }
+
+  return 0;
+}
diff --git a/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.modifiers/insert_transparent.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.modifiers/insert_transparent.pass.cpp
new file mode 100644
index 0000000000000..33ca4d4e30469
--- /dev/null
+++ b/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.modifiers/insert_transparent.pass.cpp
@@ -0,0 +1,135 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
+
+// <flat_map>
+
+// class flat_multimap
+
+// template<class K> iterator insert(P&& x);
+// template<class K> iterator insert(const_iterator hint, P&& x);
+
+#include <algorithm>
+#include <compare>
+#include <concepts>
+#include <deque>
+#include <flat_map>
+#include <functional>
+#include <tuple>
+
+#include "MinSequenceContainer.h"
+#include "../helpers.h"
+#include "test_macros.h"
+#include "test_iterators.h"
+#include "min_allocator.h"
+
+// Constraints: is_constructible_v<pair<key_type, mapped_type>, P> is true.
+template <class M, class... Args>
+concept CanInsert = requires(M m, Args&&... args) { m.insert(std::forward<Args>(args)...); };
+
+using Map  = std::flat_multimap<int, double>;
+using Iter = Map::const_iterator;
+
+static_assert(CanInsert<Map, std::pair<short, double>&&>);
+static_assert(CanInsert<Map, Iter, std::pair<short, double>&&>);
+static_assert(CanInsert<Map, std::tuple<short, double>&&>);
+static_assert(CanInsert<Map, Iter, std::tuple<short, double>&&>);
+static_assert(!CanInsert<Map, int>);
+static_assert(!CanInsert<Map, Iter, int>);
+
+static int expensive_comparisons = 0;
+static int cheap_comparisons     = 0;
+
+struct CompareCounter {
+  int i_ = 0;
+  CompareCounter(int i) : i_(i) {}
+  friend auto operator<=>(const CompareCounter& x, const CompareCounter& y) {
+    expensive_comparisons += 1;
+    return x.i_ <=> y.i_;
+  }
+  bool operator==(const CompareCounter&) const = default;
+  friend auto operator<=>(const CompareCounter& x, int y) {
+    cheap_comparisons += 1;
+    return x.i_ <=> y;
+  }
+};
+
+template <class KeyContainer, class ValueContainer>
+void test() {
+  using Key   = typename KeyContainer::value_type;
+  using Value = typename ValueContainer::value_type;
+  using M     = std::flat_multimap<Key, Value, std::less<Key>, KeyContainer, ValueContainer>;
+
+  {
+    // insert(P&&)
+    //   Unlike flat_set, here we can't use key_compare to compare value_type versus P,
+    //   so we must eagerly convert to value_type.
+    M m                                                 = {{1, 1}, {2, 2}, {3, 1}, {3, 4}, {4, 4}, {5, 5}};
+    expensive_comparisons                               = 0;
+    cheap_comparisons                                   = 0;
+    std::same_as<typename M::iterator> decltype(auto) r = m.insert(std::make_pair(3, 3)); // conversion happens first
+    assert(expensive_comparisons >= 2);
+    assert(cheap_comparisons == 0);
+    assert(r == m.begin() + 4);
+
+    std::pair<int, int> expected[] = {{1, 1}, {2, 2}, {3, 1}, {3, 4}, {3, 3}, {4, 4}, {5, 5}};
+    assert(std::ranges::equal(m, expected));
+  }
+  {
+    // insert(const_iterator, P&&)
+    M m                                        = {{1, 1}, {2, 2}, {3, 1}, {3, 4}, {4, 4}, {5, 5}};
+    expensive_comparisons                      = 0;
+    cheap_comparisons                          = 0;
+    std::same_as<typename M::iterator> auto it = m.insert(m.begin(), std::make_pair(3, 3));
+    assert(expensive_comparisons >= 2);
+    assert(cheap_comparisons == 0);
+    assert(it == m.begin() + 2);
+    std::pair<int, int> expected[] = {{1, 1}, {2, 2}, {3, 3}, {3, 1}, {3, 4}, {4, 4}, {5, 5}};
+    assert(std::ranges::equal(m, expected));
+  }
+}
+
+int main(int, char**) {
+  test<std::vector<CompareCounter>, std::vector<double>>();
+  test<std::deque<CompareCounter>, std::vector<double>>();
+  test<MinSequenceContainer<CompareCounter>, MinSequenceContainer<double>>();
+  test<std::vector<CompareCounter, min_allocator<CompareCounter>>, std::vector<double, min_allocator<double>>>();
+
+  {
+    // no ambiguity between insert(pos, P&&) and insert(first, last)
+    using M = std::flat_multimap<int, int>;
+    struct Evil {
+      operator M::value_type() const;
+      operator M::const_iterator() const;
+    };
+    std::flat_multimap<int, int> m;
+    ASSERT_SAME_TYPE(decltype(m.insert(Evil())), M::iterator);
+    ASSERT_SAME_TYPE(decltype(m.insert(m.begin(), Evil())), M::iterator);
+    ASSERT_SAME_TYPE(decltype(m.insert(m.begin(), m.end())), void);
+  }
+  {
+    auto insert_func = [](auto& m, auto key_arg, auto value_arg) {
+      using FlatMap    = std::decay_t<decltype(m)>;
+      using tuple_type = std::tuple<typename FlatMap::key_type, typename FlatMap::mapped_type>;
+      tuple_type t(key_arg, value_arg);
+      m.insert(t);
+    };
+    test_emplace_exception_guarantee(insert_func);
+  }
+  {
+    auto insert_func_iter = [](auto& m, auto key_arg, auto value_arg) {
+      using FlatMap    = std::decay_t<decltype(m)>;
+      using tuple_type = std::tuple<typename FlatMap::key_type, typename FlatMap::mapped_type>;
+      tuple_type t(key_arg, value_arg);
+      m.insert(m.begin(), t);
+    };
+    test_emplace_exception_guarantee(insert_func_iter);
+  }
+  return 0;
+}
diff --git a/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.modifiers/replace.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.modifiers/replace.pass.cpp
new file mode 100644
index 0000000000000..86fbaff468ab6
--- /dev/null
+++ b/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.modifiers/replace.pass.cpp
@@ -0,0 +1,82 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
+
+// <flat_map>
+
+// class flat_multimap
+
+// void replace(key_container_type&& key_cont, mapped_container_type&& mapped_cont);
+
+#include <algorithm>
+#include <deque>
+#include <concepts>
+#include <flat_map>
+#include <functional>
+
+#include "MinSequenceContainer.h"
+#include "../helpers.h"
+#include "test_macros.h"
+#include "min_allocator.h"
+
+template <class T, class... Args>
+concept CanReplace = requires(T t, Args&&... args) { t.replace(std::forward<Args>(args)...); };
+
+using Map = std::flat_multimap<int, int>;
+static_assert(CanReplace<Map, std::vector<int>, std::vector<int>>);
+static_assert(!CanReplace<Map, const std::vector<int>&, std::vector<int>>);
+static_assert(!CanReplace<Map, std::vector<int>, const std::vector<int>&>);
+static_assert(!CanReplace<Map, const std::vector<int>&, const std::vector<int>&>);
+
+template <class KeyContainer, class ValueContainer>
+void test() {
+  using Key   = typename KeyContainer::value_type;
+  using Value = typename ValueContainer::value_type;
+  using M     = std::flat_multimap<Key, Value, std::less<Key>, KeyContainer, ValueContainer>;
+
+  M m                       = M({1, 1, 3}, {4, 5, 6});
+  KeyContainer new_keys     = {7, 7};
+  ValueContainer new_values = {9, 10};
+  auto expected_keys        = new_keys;
+  auto expected_values      = new_values;
+  m.replace(std::move(new_keys), std::move(new_values));
+  assert(m.size() == 2);
+  assert(std::ranges::equal(m.keys(), expected_keys));
+  assert(std::ranges::equal(m.values(), expected_values));
+}
+
+int main(int, char**) {
+  test<std::vector<int>, std::vector<double>>();
+  test<std::deque<int>, std::vector<double>>();
+  test<MinSequenceContainer<int>, MinSequenceContainer<double>>();
+  test<std::vector<int, min_allocator<int>>, std::vector<double, min_allocator<double>>>();
+
+  {
+#ifndef TEST_HAS_NO_EXCEPTIONS
+    using KeyContainer   = std::vector<int>;
+    using ValueContainer = ThrowOnMoveContainer<int>;
+    using M              = std::flat_multimap<int, int, std::ranges::less, KeyContainer, ValueContainer>;
+
+    M m;
+    m.emplace(1, 1);
+    m.emplace(2, 2);
+    try {
+      KeyContainer new_keys{3, 4};
+      ValueContainer new_values{5, 6};
+      m.replace(std::move(new_keys), std::move(new_values));
+      assert(false);
+    } catch (int) {
+      check_invariant(m);
+      // In libc++, we clear the map
+      LIBCPP_ASSERT(m.size() == 0);
+    }
+#endif
+  }
+  return 0;
+}
diff --git a/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.modifiers/swap_exception.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.modifiers/swap_exception.pass.cpp
new file mode 100644
index 0000000000000..a1252f301309a
--- /dev/null
+++ b/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.modifiers/swap_exception.pass.cpp
@@ -0,0 +1,80 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
+// `check_assertion.h` requires Unix headers and regex support.
+// REQUIRES: has-unix-headers
+// UNSUPPORTED: no-localization
+// UNSUPPORTED: no-exceptions
+
+// <flat_map>
+
+// class flat_multimap
+
+// void swap(flat_multimap& y) noexcept;
+// friend void swap(flat_multimap& x, flat_multimap& y) noexcept
+
+// Test that std::terminate is called if any exception is thrown during swap
+
+#include <flat_map>
+#include <cassert>
+#include <deque>
+#include <functional>
+#include <vector>
+
+#include "test_macros.h"
+#include "../helpers.h"
+#include "check_assertion.h"
+
+template <class F>
+void test_swap_exception_guarantee([[maybe_unused]] F&& swap_function) {
+  {
+    // key swap throws
+    using KeyContainer   = ThrowOnMoveContainer<int>;
+    using ValueContainer = std::vector<int>;
+    using M              = std::flat_multimap<int, int, TransparentComparator, KeyContainer, ValueContainer>;
+
+    M m1, m2;
+    m1.emplace(1, 1);
+    m1.emplace(1, 2);
+    m2.emplace(3, 3);
+    m2.emplace(3, 4);
+    // swap is noexcept
+    EXPECT_STD_TERMINATE([&] { swap_function(m1, m2); });
+  }
+
+  {
+    // value swap throws
+    using KeyContainer   = std::vector<int>;
+    using ValueContainer = ThrowOnMoveContainer<int>;
+    using M              = std::flat_multimap<int, int, TransparentComparator, KeyContainer, ValueContainer>;
+
+    M m1, m2;
+    m1.emplace(1, 1);
+    m1.emplace(1, 2);
+    m2.emplace(3, 3);
+    m2.emplace(3, 4);
+
+    // swap is noexcept
+    EXPECT_STD_TERMINATE([&] { swap_function(m1, m2); });
+  }
+}
+
+int main(int, char**) {
+  {
+    auto swap_func = [](auto& m1, auto& m2) { swap(m1, m2); };
+    test_swap_exception_guarantee(swap_func);
+  }
+
+  {
+    auto swap_func = [](auto& m1, auto& m2) { m1.swap(m2); };
+    test_swap_exception_guarantee(swap_func);
+  }
+
+  return 0;
+}
diff --git a/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.modifiers/swap_free.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.modifiers/swap_free.pass.cpp
new file mode 100644
index 0000000000000..f96155d714dc9
--- /dev/null
+++ b/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.modifiers/swap_free.pass.cpp
@@ -0,0 +1,99 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
+
+// <flat_map>
+
+// class flat_multimap
+
+// friend void swap(flat_multimap& x, flat_multimap& y) noexcept
+
+#include <flat_map>
+#include <cassert>
+#include <deque>
+#include <functional>
+#include <vector>
+
+#include "MinSequenceContainer.h"
+#include "MoveOnly.h"
+#include "min_allocator.h"
+#include "test_macros.h"
+#include "../helpers.h"
+
+// test noexcept
+
+template <class T>
+concept NoExceptAdlSwap = requires(T t1, T t2) {
+  { swap(t1, t2) } noexcept;
+};
+
+static_assert(NoExceptAdlSwap<std::flat_multimap<int, int>>);
+
+#ifndef TEST_HAS_NO_EXCEPTIONS
+static_assert(NoExceptAdlSwap<
+              std::flat_multimap<int, int, std::less<int>, ThrowOnMoveContainer<int>, ThrowOnMoveContainer<int>>>);
+#endif
+
+template <class KeyContainer, class ValueContainer>
+void test() {
+  using Key   = typename KeyContainer::value_type;
+  using Value = typename ValueContainer::value_type;
+  using M     = std::flat_multimap<Key, Value, std::less<Key>, KeyContainer, ValueContainer>;
+  using V     = std::pair<const Key, Value>;
+
+  {
+    M m1;
+    M m2;
+    M m1_save = m1;
+    M m2_save = m2;
+    swap(m1, m2);
+    assert(m1 == m2_save);
+    assert(m2 == m1_save);
+  }
+  {
+    V ar2[] = {V(5, 5), V(5, 6), V(5, 7), V(8, 8), V(9, 9), V(10, 10), V(10, 11), V(10, 12)};
+    M m1;
+    M m2(ar2, ar2 + sizeof(ar2) / sizeof(ar2[0]));
+    M m1_save = m1;
+    M m2_save = m2;
+    swap(m1, m2);
+    assert(m1 == m2_save);
+    assert(m2 == m1_save);
+  }
+  {
+    V ar1[] = {V(1, 1), V(1, 2), V(3, 3), V(4, 4)};
+    M m1(ar1, ar1 + sizeof(ar1) / sizeof(ar1[0]));
+    M m2;
+    M m1_save = m1;
+    M m2_save = m2;
+    swap(m1, m2);
+    assert(m1 == m2_save);
+    assert(m2 == m1_save);
+  }
+  {
+    V ar1[] = {V(1, 1), V(1, 2), V(3, 3), V(4, 4)};
+    V ar2[] = {V(5, 5), V(5, 6), V(5, 7), V(8, 8), V(9, 9), V(10, 10), V(10, 11), V(10, 12)};
+    M m1(ar1, ar1 + sizeof(ar1) / sizeof(ar1[0]));
+    M m2(ar2, ar2 + sizeof(ar2) / sizeof(ar2[0]));
+    M m1_save = m1;
+    M m2_save = m2;
+    swap(m1, m2);
+    assert(m1 == m2_save);
+    assert(m2 == m1_save);
+  }
+}
+
+int main(int, char**) {
+  test<std::vector<int>, std::vector<double>>();
+  test<std::deque<int>, std::vector<double>>();
+  test<MinSequenceContainer<int>, MinSequenceContainer<double>>();
+  test<std::vector<int, min_allocator<int>>, std::vector<double, min_allocator<double>>>();
+
+  return 0;
+}
diff --git a/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.modifiers/swap_member.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.modifiers/swap_member.pass.cpp
new file mode 100644
index 0000000000000..ab7be3b8ac22e
--- /dev/null
+++ b/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.modifiers/swap_member.pass.cpp
@@ -0,0 +1,97 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
+
+// <flat_map>
+
+// class flat_multimap
+
+// void swap(flat_multimap& y) noexcept;
+
+#include <flat_map>
+#include <cassert>
+#include <deque>
+#include <functional>
+#include <vector>
+
+#include "MinSequenceContainer.h"
+#include "MoveOnly.h"
+#include "min_allocator.h"
+#include "test_macros.h"
+#include "../helpers.h"
+
+// test noexcept
+
+template <class T>
+concept NoExceptMemberSwap = requires(T t1, T t2) {
+  { t1.swap(t2) } noexcept;
+};
+
+static_assert(NoExceptMemberSwap<std::flat_multimap<int, int>>);
+#ifndef TEST_HAS_NO_EXCEPTIONS
+static_assert(NoExceptMemberSwap<
+              std::flat_multimap<int, int, std::less<int>, ThrowOnMoveContainer<int>, ThrowOnMoveContainer<int>>>);
+#endif
+
+template <class KeyContainer, class ValueContainer>
+void test() {
+  using Key   = typename KeyContainer::value_type;
+  using Value = typename ValueContainer::value_type;
+  using M     = std::flat_multimap<Key, Value, std::less<Key>, KeyContainer, ValueContainer>;
+  using V     = std::pair<const Key, Value>;
+  {
+    M m1;
+    M m2;
+    M m1_save = m1;
+    M m2_save = m2;
+    m1.swap(m2);
+    assert(m1 == m2_save);
+    assert(m2 == m1_save);
+  }
+  {
+    V ar2[] = {V(5, 5), V(5, 6), V(7, 7), V(8, 8), V(9, 9), V(10, 10), V(10, 11), V(12, 12)};
+    M m1;
+    M m2(ar2, ar2 + sizeof(ar2) / sizeof(ar2[0]));
+    M m1_save = m1;
+    M m2_save = m2;
+    m1.swap(m2);
+    assert(m1 == m2_save);
+    assert(m2 == m1_save);
+  }
+  {
+    V ar1[] = {V(1, 1), V(1, 2), V(3, 3), V(4, 4)};
+    M m1(ar1, ar1 + sizeof(ar1) / sizeof(ar1[0]));
+    M m2;
+    M m1_save = m1;
+    M m2_save = m2;
+    m1.swap(m2);
+    assert(m1 == m2_save);
+    assert(m2 == m1_save);
+  }
+  {
+    V ar1[] = {V(1, 1), V(1, 2), V(3, 3), V(4, 4)};
+    V ar2[] = {V(5, 5), V(5, 6), V(7, 7), V(8, 8), V(9, 9), V(10, 10), V(10, 11), V(12, 12)};
+    M m1(ar1, ar1 + sizeof(ar1) / sizeof(ar1[0]));
+    M m2(ar2, ar2 + sizeof(ar2) / sizeof(ar2[0]));
+    M m1_save = m1;
+    M m2_save = m2;
+    m1.swap(m2);
+    assert(m1 == m2_save);
+    assert(m2 == m1_save);
+  }
+}
+
+int main(int, char**) {
+  test<std::vector<int>, std::vector<double>>();
+  test<std::deque<int>, std::vector<double>>();
+  test<MinSequenceContainer<int>, MinSequenceContainer<double>>();
+  test<std::vector<int, min_allocator<int>>, std::vector<double, min_allocator<double>>>();
+
+  return 0;
+}
diff --git a/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.observers/comp.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.observers/comp.pass.cpp
new file mode 100644
index 0000000000000..47140132c6e47
--- /dev/null
+++ b/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.observers/comp.pass.cpp
@@ -0,0 +1,98 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
+
+// <flat_map>
+
+// class flat_multimap
+
+// key_compare key_comp() const;
+// value_compare value_comp() const;
+
+#include <cassert>
+#include <flat_map>
+#include <functional>
+#include <utility>
+#include <vector>
+
+#include "test_macros.h"
+
+int main(int, char**) {
+  {
+    using M    = std::flat_multimap<int, char>;
+    using Comp = std::less<int>; // the default
+    M m        = {};
+    ASSERT_SAME_TYPE(M::key_compare, Comp);
+    static_assert(!std::is_same_v<M::value_compare, Comp>);
+    ASSERT_SAME_TYPE(decltype(m.key_comp()), Comp);
+    ASSERT_SAME_TYPE(decltype(m.value_comp()), M::value_compare);
+    Comp kc = m.key_comp();
+    assert(kc(1, 2));
+    assert(!kc(2, 1));
+    auto vc = m.value_comp();
+    ASSERT_SAME_TYPE(decltype(vc(std::make_pair(1, 2), std::make_pair(1, 2))), bool);
+    assert(vc({1, '2'}, {2, '1'}));
+    assert(!vc({2, '1'}, {1, '2'}));
+  }
+  {
+    using Comp = std::function<bool(int, int)>;
+    using M    = std::flat_multimap<int, int, Comp>;
+    Comp comp  = std::greater<int>();
+    M m({}, comp);
+    ASSERT_SAME_TYPE(M::key_compare, Comp);
+    ASSERT_SAME_TYPE(decltype(m.key_comp()), Comp);
+    ASSERT_SAME_TYPE(decltype(m.value_comp()), M::value_compare);
+    Comp kc = m.key_comp();
+    assert(!kc(1, 2));
+    assert(kc(2, 1));
+    auto vc = m.value_comp();
+    auto a  = std::make_pair(1, 2);
+    ASSERT_SAME_TYPE(decltype(vc(a, a)), bool);
+    static_assert(!noexcept(vc(a, a)));
+    assert(!vc({1, 2}, {2, 1}));
+    assert(vc({2, 1}, {1, 2}));
+  }
+  {
+    using Comp = std::less<>;
+    using M    = std::flat_multimap<int, int, Comp>;
+    M m        = {};
+    ASSERT_SAME_TYPE(M::key_compare, Comp);
+    ASSERT_SAME_TYPE(decltype(m.key_comp()), Comp);
+    ASSERT_SAME_TYPE(decltype(m.value_comp()), M::value_compare);
+    Comp kc = m.key_comp();
+    assert(kc(1, 2));
+    assert(!kc(2, 1));
+    auto vc = m.value_comp();
+    auto a  = std::make_pair(1, 2);
+    ASSERT_SAME_TYPE(decltype(vc(a, a)), bool);
+    assert(vc({1, 2}, {2, 1}));
+    assert(!vc({2, 1}, {1, 2}));
+  }
+  {
+    using Comp = std::function<bool(const std::vector<int>&, const std::vector<int>&)>;
+    using M    = std::flat_multimap<std::vector<int>, int, Comp>;
+    Comp comp  = [i = 1](const auto& x, const auto& y) { return x[i] < y[i]; };
+    M m({}, comp);
+    auto vc = m.value_comp();
+    static_assert(sizeof(vc) >= sizeof(Comp));
+    comp = nullptr;
+    m    = M({}, nullptr);
+    assert(m.key_comp() == nullptr);
+    // At this point, m.key_comp() is disengaged.
+    // But the std::function captured by copy inside `vc` remains valid.
+    auto a = std::make_pair(std::vector<int>{2, 1, 4}, 42);
+    auto b = std::make_pair(std::vector<int>{1, 2, 3}, 42);
+    auto c = std::make_pair(std::vector<int>{0, 3, 2}, 42);
+    assert(vc(a, b));
+    assert(vc(b, c));
+    assert(!vc(b, a));
+    assert(!vc(c, b));
+  }
+  return 0;
+}
diff --git a/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.observers/keys_values.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.observers/keys_values.pass.cpp
new file mode 100644
index 0000000000000..c7c674c034bca
--- /dev/null
+++ b/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.observers/keys_values.pass.cpp
@@ -0,0 +1,59 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
+
+// <flat_map>
+
+// class flat_multimap
+
+// const key_container_type& keys() const noexcept
+// const mapped_container_type& values() const noexcept
+
+#include <algorithm>
+#include <cassert>
+#include <flat_map>
+#include <functional>
+#include <utility>
+#include <vector>
+#include <deque>
+#include <string>
+
+#include "MinSequenceContainer.h"
+#include "test_macros.h"
+#include "test_allocator.h"
+#include "min_allocator.h"
+
+template <class KeyContainer, class ValueContainer>
+void test() {
+  using Key   = typename KeyContainer::value_type;
+  using Value = typename ValueContainer::value_type;
+  using M     = std::flat_multimap<Key, Value, std::less<Key>, KeyContainer, ValueContainer>;
+
+  const M m                                                 = {{4, 'a'}, {2, 'b'}, {2, 'e'}, {3, 'c'}};
+  std::same_as<const KeyContainer&> decltype(auto) keys     = m.keys();
+  std::same_as<const ValueContainer&> decltype(auto) values = m.values();
+
+  // noexcept
+  static_assert(noexcept(m.keys()));
+  static_assert(noexcept(m.values()));
+
+  auto expected_keys   = {2, 2, 3, 4};
+  auto expected_values = {'b', 'e', 'c', 'a'};
+  assert(std::ranges::equal(keys, expected_keys));
+  assert(std::ranges::equal(values, expected_values));
+}
+
+int main(int, char**) {
+  test<std::vector<int>, std::vector<char>>();
+  test<std::deque<int>, std::vector<char>>();
+  test<MinSequenceContainer<int>, MinSequenceContainer<char>>();
+  test<std::vector<int, min_allocator<int>>, std::vector<char, min_allocator<char>>>();
+
+  return 0;
+}
diff --git a/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.operations/contains.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.operations/contains.pass.cpp
new file mode 100644
index 0000000000000..b3ea0b65a3d93
--- /dev/null
+++ b/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.operations/contains.pass.cpp
@@ -0,0 +1,72 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
+
+// <flat_map>
+
+// class flat_multimap
+
+// bool contains(const key_type& x) const;
+
+#include <cassert>
+#include <deque>
+#include <flat_map>
+#include <functional>
+#include <utility>
+
+#include "MinSequenceContainer.h"
+#include "test_macros.h"
+#include "min_allocator.h"
+
+template <class KeyContainer, class ValueContainer>
+void test() {
+  using Key   = typename KeyContainer::value_type;
+  using Value = typename ValueContainer::value_type;
+  {
+    using M = std::flat_multimap<Key, Value, std::less<>, KeyContainer, ValueContainer>;
+    M m     = {{1, 1}, {2, 2}, {2, 3}, {4, 4}, {5, 5}, {8, 1}, {8, 2}, {8, 8}};
+    assert(!m.contains(0));
+    assert(m.contains(1));
+    assert(m.contains(2));
+    assert(!m.contains(3));
+    assert(m.contains(4));
+    assert(m.contains(5));
+    assert(!m.contains(6));
+    assert(!m.contains(7));
+    assert(std::as_const(m).contains(8));
+    assert(!std::as_const(m).contains(9));
+    m.clear();
+    assert(!m.contains(1));
+  }
+  {
+    using M = std::flat_multimap<Key, Value, std::greater<int>, KeyContainer, ValueContainer>;
+    M m     = {{1, 0}, {2, 0}, {4, 0}, {2, 1}, {5, 1}, {5, 2}, {5, 0}, {8, 0}};
+    assert(!m.contains(0));
+    assert(m.contains(1));
+    assert(m.contains(2));
+    assert(!m.contains(3));
+    assert(m.contains(4));
+    assert(m.contains(5));
+    assert(!m.contains(6));
+    assert(!m.contains(7));
+    assert(std::as_const(m).contains(8));
+    assert(!std::as_const(m).contains(9));
+    m.clear();
+    assert(!m.contains(1));
+  }
+}
+
+int main(int, char**) {
+  test<std::vector<int>, std::vector<int>>();
+  test<std::deque<int>, std::vector<int>>();
+  test<MinSequenceContainer<int>, MinSequenceContainer<int>>();
+  test<std::vector<int, min_allocator<int>>, std::vector<int, min_allocator<int>>>();
+
+  return 0;
+}
diff --git a/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.operations/contains_transparent.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.operations/contains_transparent.pass.cpp
new file mode 100644
index 0000000000000..8a66ec63768d7
--- /dev/null
+++ b/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.operations/contains_transparent.pass.cpp
@@ -0,0 +1,73 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
+
+// <flat_map>
+
+// class flat_multimap
+
+// template<class K> bool contains(const K& x) const;
+
+#include <cassert>
+#include <flat_map>
+#include <string>
+#include <utility>
+#include <deque>
+
+#include "MinSequenceContainer.h"
+#include "../helpers.h"
+#include "test_macros.h"
+#include "min_allocator.h"
+
+// Constraints: The qualified-id Compare::is_transparent is valid and denotes a type.
+template <class M>
+concept CanContains     = requires(M m, Transparent<int> k) { m.contains(k); };
+using TransparentMap    = std::flat_multimap<int, double, TransparentComparator>;
+using NonTransparentMap = std::flat_multimap<int, double, NonTransparentComparator>;
+static_assert(CanContains<TransparentMap>);
+static_assert(CanContains<const TransparentMap>);
+static_assert(!CanContains<NonTransparentMap>);
+static_assert(!CanContains<const NonTransparentMap>);
+
+template <class KeyContainer, class ValueContainer>
+void test() {
+  using Key   = typename KeyContainer::value_type;
+  using Value = typename ValueContainer::value_type;
+  using M     = std::flat_multimap<Key, Value, TransparentComparator, KeyContainer, ValueContainer>;
+
+  M m = {{"alpha", 1}, {"beta", 2}, {"beta", 0}, {"epsilon", 3}, {"eta", 4}, {"eta", 1}, {"gamma", 5}};
+  ASSERT_SAME_TYPE(decltype(m.contains(Transparent<std::string>{"abc"})), bool);
+  ASSERT_SAME_TYPE(decltype(std::as_const(m).contains(Transparent<std::string>{"b"})), bool);
+  assert(m.contains(Transparent<std::string>{"alpha"}) == true);
+  assert(m.contains(Transparent<std::string>{"beta"}) == true);
+  assert(m.contains(Transparent<std::string>{"epsilon"}) == true);
+  assert(m.contains(Transparent<std::string>{"eta"}) == true);
+  assert(m.contains(Transparent<std::string>{"gamma"}) == true);
+  assert(m.contains(Transparent<std::string>{"al"}) == false);
+  assert(m.contains(Transparent<std::string>{""}) == false);
+  assert(m.contains(Transparent<std::string>{"g"}) == false);
+}
+
+int main(int, char**) {
+  test<std::vector<std::string>, std::vector<int>>();
+  test<std::deque<std::string>, std::vector<int>>();
+  test<MinSequenceContainer<std::string>, MinSequenceContainer<int>>();
+  test<std::vector<std::string, min_allocator<std::string>>, std::vector<int, min_allocator<int>>>();
+
+  {
+    bool transparent_used = false;
+    TransparentComparator c(transparent_used);
+    std::flat_multimap<int, int, TransparentComparator> m(std::sorted_equivalent, {{1, 1}, {1, 2}, {2, 2}, {3, 3}}, c);
+    assert(!transparent_used);
+    auto b = m.contains(Transparent<int>{3});
+    assert(b);
+    assert(transparent_used);
+  }
+  return 0;
+}
diff --git a/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.operations/count.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.operations/count.pass.cpp
new file mode 100644
index 0000000000000..59b88428cde3c
--- /dev/null
+++ b/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.operations/count.pass.cpp
@@ -0,0 +1,71 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
+
+// <flat_map>
+
+// class flat_multimap
+
+// size_type count(const key_type& x) const;
+
+#include <cassert>
+#include <deque>
+#include <flat_map>
+#include <functional>
+#include <utility>
+
+#include "MinSequenceContainer.h"
+#include "test_macros.h"
+#include "min_allocator.h"
+
+template <class KeyContainer, class ValueContainer>
+void test() {
+  using Key   = typename KeyContainer::value_type;
+  using Value = typename ValueContainer::value_type;
+
+  {
+    using M = std::flat_multimap<Key, Value, std::less<>, KeyContainer, ValueContainer>;
+    M m     = {{1, 1}, {2, 2}, {2, 2}, {4, 4}, {4, 1}, {4, 3}, {4, 4}, {5, 5}, {8, 8}};
+    ASSERT_SAME_TYPE(decltype(m.count(0)), size_t);
+    assert(m.count(0) == 0);
+    assert(m.count(1) == 1);
+    assert(m.count(2) == 2);
+    assert(m.count(3) == 0);
+    assert(m.count(4) == 4);
+    assert(m.count(5) == 1);
+    assert(m.count(6) == 0);
+    assert(m.count(7) == 0);
+    assert(std::as_const(m).count(8) == 1);
+    assert(std::as_const(m).count(9) == 0);
+  }
+  {
+    using M = std::flat_multimap<Key, Value, std::greater<int>, KeyContainer, ValueContainer>;
+    M m     = {{1, 0}, {2, 0}, {4, 0}, {1, 0}, {1, 2}, {8, 1}, {5, 0}, {8, 0}};
+    ASSERT_SAME_TYPE(decltype(m.count(0)), size_t);
+    assert(m.count(0) == 0);
+    assert(m.count(1) == 3);
+    assert(m.count(2) == 1);
+    assert(m.count(3) == 0);
+    assert(m.count(4) == 1);
+    assert(m.count(5) == 1);
+    assert(m.count(6) == 0);
+    assert(m.count(7) == 0);
+    assert(std::as_const(m).count(8) == 2);
+    assert(std::as_const(m).count(9) == 0);
+  }
+}
+
+int main(int, char**) {
+  test<std::vector<int>, std::vector<int>>();
+  test<std::deque<int>, std::vector<int>>();
+  test<MinSequenceContainer<int>, MinSequenceContainer<int>>();
+  test<std::vector<int, min_allocator<int>>, std::vector<int, min_allocator<int>>>();
+
+  return 0;
+}
diff --git a/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.operations/count_transparent.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.operations/count_transparent.pass.cpp
new file mode 100644
index 0000000000000..41f71065b2f75
--- /dev/null
+++ b/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.operations/count_transparent.pass.cpp
@@ -0,0 +1,83 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
+
+// <flat_map>
+
+// class flat_multimap
+
+// template<class K> size_type count(const K& x) const;
+
+#include <cassert>
+#include <deque>
+#include <flat_map>
+#include <string>
+#include <utility>
+
+#include "MinSequenceContainer.h"
+#include "../helpers.h"
+#include "test_macros.h"
+#include "min_allocator.h"
+
+// Constraints: The qualified-id Compare::is_transparent is valid and denotes a type.
+template <class M>
+concept CanCount        = requires(M m, Transparent<int> k) { m.count(k); };
+using TransparentMap    = std::flat_multimap<int, double, TransparentComparator>;
+using NonTransparentMap = std::flat_multimap<int, double, NonTransparentComparator>;
+static_assert(CanCount<TransparentMap>);
+static_assert(CanCount<const TransparentMap>);
+static_assert(!CanCount<NonTransparentMap>);
+static_assert(!CanCount<const NonTransparentMap>);
+
+template <class KeyContainer, class ValueContainer>
+void test() {
+  using Key   = typename KeyContainer::value_type;
+  using Value = typename ValueContainer::value_type;
+  using M     = std::flat_multimap<Key, Value, TransparentComparator, KeyContainer, ValueContainer>;
+
+  M m = {{"alpha", 1},
+         {"beta", 2},
+         {"beta", 2},
+         {"epsilon", 3},
+         {"eta", 4},
+         {"eta", 1},
+         {"eta", 5},
+         {"gamma", 6},
+         {"gamma", 5}};
+  ASSERT_SAME_TYPE(decltype(m.count(Transparent<std::string>{"abc"})), typename M::size_type);
+  ASSERT_SAME_TYPE(decltype(std::as_const(m).count(Transparent<std::string>{"b"})), typename M::size_type);
+  assert(m.count(Transparent<std::string>{"alpha"}) == 1);
+  assert(m.count(Transparent<std::string>{"beta"}) == 2);
+  assert(m.count(Transparent<std::string>{"epsilon"}) == 1);
+  assert(m.count(Transparent<std::string>{"eta"}) == 3);
+  assert(m.count(Transparent<std::string>{"gamma"}) == 2);
+  assert(m.count(Transparent<std::string>{"al"}) == 0);
+  assert(m.count(Transparent<std::string>{""}) == 0);
+  assert(m.count(Transparent<std::string>{"g"}) == 0);
+}
+
+int main(int, char**) {
+  test<std::vector<std::string>, std::vector<int>>();
+  test<std::deque<std::string>, std::vector<int>>();
+  test<MinSequenceContainer<std::string>, MinSequenceContainer<int>>();
+  test<std::vector<std::string, min_allocator<std::string>>, std::vector<int, min_allocator<int>>>();
+
+  {
+    bool transparent_used = false;
+    TransparentComparator c(transparent_used);
+    std::flat_multimap<int, int, TransparentComparator> m(
+        std::sorted_equivalent, {{1, 1}, {2, 2}, {2, 2}, {3, 3}, {3, 3}}, c);
+    assert(!transparent_used);
+    auto n = m.count(Transparent<int>{3});
+    assert(n == 2);
+    assert(transparent_used);
+  }
+
+  return 0;
+}
diff --git a/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.operations/equal_range.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.operations/equal_range.pass.cpp
new file mode 100644
index 0000000000000..ac369b77a7f3d
--- /dev/null
+++ b/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.operations/equal_range.pass.cpp
@@ -0,0 +1,81 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
+
+// <flat_map>
+
+// class flat_multimap
+
+// pair<iterator,iterator>             equal_range(const key_type& k);
+// pair<const_iterator,const_iterator> equal_range(const key_type& k) const;
+
+#include <cassert>
+#include <deque>
+#include <flat_map>
+#include <functional>
+#include <utility>
+
+#include "MinSequenceContainer.h"
+#include "test_macros.h"
+#include "min_allocator.h"
+
+template <class KeyContainer, class ValueContainer>
+void test() {
+  using Key   = typename KeyContainer::value_type;
+  using Value = typename ValueContainer::value_type;
+  {
+    using M  = std::flat_multimap<Key, Value, std::less<>, KeyContainer, ValueContainer>;
+    using R  = std::pair<typename M::iterator, typename M::iterator>;
+    using CR = std::pair<typename M::const_iterator, typename M::const_iterator>;
+    M m      = {{1, 'a'}, {1, 'a'}, {1, 'A'}, {2, 'b'}, {4, 'd'}, {5, 'E'}, {5, 'e'}, {8, 'h'}, {8, 'z'}};
+    ASSERT_SAME_TYPE(decltype(m.equal_range(0)), R);
+    ASSERT_SAME_TYPE(decltype(std::as_const(m).equal_range(0)), CR);
+    auto begin = m.begin();
+    assert(m.equal_range(0) == std::pair(begin, begin));
+    assert(m.equal_range(1) == std::pair(begin, begin + 3));
+    assert(m.equal_range(2) == std::pair(begin + 3, begin + 4));
+    assert(m.equal_range(3) == std::pair(begin + 4, begin + 4));
+    assert(m.equal_range(4) == std::pair(begin + 4, begin + 5));
+    assert(m.equal_range(5) == std::pair(begin + 5, begin + 7));
+    assert(m.equal_range(6) == std::pair(begin + 7, begin + 7));
+    assert(m.equal_range(7) == std::pair(begin + 7, begin + 7));
+    assert(std::as_const(m).equal_range(8) == std::pair(m.cbegin() + 7, m.cbegin() + 9));
+    assert(std::as_const(m).equal_range(9) == std::pair(m.cbegin() + 9, m.cbegin() + 9));
+  }
+
+  {
+    using M  = std::flat_multimap<Key, Value, std::greater<int>, KeyContainer, ValueContainer>;
+    using R  = std::pair<typename M::iterator, typename M::iterator>;
+    using CR = std::pair<typename M::const_iterator, typename M::const_iterator>;
+    M m      = {
+        {1, 'a'}, {2, 'b'}, {2, 'b'}, {2, 'c'}, {4, 'a'}, {4, 'b'}, {4, 'c'}, {4, 'd'}, {5, 'e'}, {8, 'a'}, {8, 'h'}};
+    ASSERT_SAME_TYPE(decltype(m.equal_range(0)), R);
+    ASSERT_SAME_TYPE(decltype(std::as_const(m).equal_range(0)), CR);
+    auto begin = m.begin();
+    assert(m.equal_range(0) == std::pair(begin + 11, begin + 11));
+    assert(m.equal_range(1) == std::pair(begin + 10, begin + 11));
+    assert(m.equal_range(2) == std::pair(begin + 7, begin + 10));
+    assert(m.equal_range(3) == std::pair(begin + 7, begin + 7));
+    assert(m.equal_range(4) == std::pair(begin + 3, begin + 7));
+    assert(m.equal_range(5) == std::pair(begin + 2, begin + 3));
+    assert(m.equal_range(6) == std::pair(begin + 2, begin + 2));
+    assert(m.equal_range(7) == std::pair(begin + 2, begin + 2));
+    assert(std::as_const(m).equal_range(8) == std::pair(m.cbegin(), m.cbegin() + 2));
+    assert(std::as_const(m).equal_range(9) == std::pair(m.cbegin(), m.cbegin()));
+  }
+}
+
+int main(int, char**) {
+  test<std::vector<int>, std::vector<char>>();
+  test<std::deque<int>, std::vector<char>>();
+  test<MinSequenceContainer<int>, MinSequenceContainer<char>>();
+  test<std::vector<int, min_allocator<int>>, std::vector<char, min_allocator<char>>>();
+
+  return 0;
+}
diff --git a/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.operations/equal_range_transparent.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.operations/equal_range_transparent.pass.cpp
new file mode 100644
index 0000000000000..3666492bb921f
--- /dev/null
+++ b/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.operations/equal_range_transparent.pass.cpp
@@ -0,0 +1,110 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
+
+// <flat_map>
+
+// class flat_multimap
+
+// template<class K> pair<iterator,iterator>             equal_range(const K& x);
+// template<class K> pair<const_iterator,const_iterator> equal_range(const K& x) const;
+
+#include <cassert>
+#include <deque>
+#include <flat_map>
+#include <string>
+#include <utility>
+
+#include "MinSequenceContainer.h"
+#include "../helpers.h"
+#include "test_macros.h"
+#include "min_allocator.h"
+
+// Constraints: The qualified-id Compare::is_transparent is valid and denotes a type.
+template <class M>
+concept CanEqualRange   = requires(M m, Transparent<int> k) { m.equal_range(k); };
+using TransparentMap    = std::flat_multimap<int, double, TransparentComparator>;
+using NonTransparentMap = std::flat_multimap<int, double, NonTransparentComparator>;
+static_assert(CanEqualRange<TransparentMap>);
+static_assert(CanEqualRange<const TransparentMap>);
+static_assert(!CanEqualRange<NonTransparentMap>);
+static_assert(!CanEqualRange<const NonTransparentMap>);
+
+template <class KeyContainer, class ValueContainer>
+void test() {
+  using Key   = typename KeyContainer::value_type;
+  using Value = typename ValueContainer::value_type;
+  using M     = std::flat_multimap<Key, Value, TransparentComparator, KeyContainer, ValueContainer>;
+
+  using R        = std::pair<typename M::iterator, typename M::iterator>;
+  using CR       = std::pair<typename M::const_iterator, typename M::const_iterator>;
+  M m            = {{"alpha", 1},
+                    {"alpha", 1},
+                    {"alpha", 3},
+                    {"beta", 2},
+                    {"epsilon", 3},
+                    {"epsilon", 0},
+                    {"eta", 4},
+                    {"gamma", 5},
+                    {"gamma", 1}};
+  const auto& cm = m;
+  ASSERT_SAME_TYPE(decltype(m.equal_range(Transparent<std::string>{"abc"})), R);
+  ASSERT_SAME_TYPE(decltype(std::as_const(m).equal_range(Transparent<std::string>{"b"})), CR);
+
+  auto test_found = [&](auto&& map, const auto& expected_key, std::initializer_list<Value> expected_values) {
+    auto [first, last] = map.equal_range(Transparent<std::string>{expected_key});
+    auto expected_range =
+        expected_values | std::views::transform([&](auto&& val) { return std::pair(expected_key, val); });
+    assert(std::ranges::equal(std::ranges::subrange(first, last), expected_range));
+  };
+
+  auto test_not_found = [&](auto&& map, const std::string& expected_key, long expected_offset) {
+    auto [first, last] = map.equal_range(Transparent<std::string>{expected_key});
+    assert(first == last);
+    assert(first - m.begin() == expected_offset);
+  };
+
+  test_found(m, "alpha", {1, 1, 3});
+  test_found(m, "beta", {2});
+  test_found(m, "epsilon", {3, 0});
+  test_found(m, "eta", {4});
+  test_found(m, "gamma", {5, 1});
+  test_found(cm, "alpha", {1, 1, 3});
+  test_found(cm, "beta", {2});
+  test_found(cm, "epsilon", {3, 0});
+  test_found(cm, "eta", {4});
+  test_found(cm, "gamma", {5, 1});
+
+  test_not_found(m, "charlie", 4);
+  test_not_found(m, "aaa", 0);
+  test_not_found(m, "zzz", 9);
+  test_not_found(cm, "charlie", 4);
+  test_not_found(cm, "aaa", 0);
+  test_not_found(cm, "zzz", 9);
+}
+
+int main(int, char**) {
+  test<std::vector<std::string>, std::vector<int>>();
+  test<std::deque<std::string>, std::vector<int>>();
+  test<MinSequenceContainer<std::string>, MinSequenceContainer<int>>();
+  test<std::vector<std::string, min_allocator<std::string>>, std::vector<int, min_allocator<int>>>();
+
+  {
+    bool transparent_used = false;
+    TransparentComparator c(transparent_used);
+    std::flat_multimap<int, int, TransparentComparator> m(std::sorted_equivalent, {{1, 1}, {2, 2}, {3, 1}, {3, 3}}, c);
+    assert(!transparent_used);
+    auto p = m.equal_range(Transparent<int>{3});
+    assert(p.first == m.begin() + 2);
+    assert(p.second == m.end());
+    assert(transparent_used);
+  }
+
+  return 0;
+}
diff --git a/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.operations/find.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.operations/find.pass.cpp
new file mode 100644
index 0000000000000..74b7051eb0d7b
--- /dev/null
+++ b/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.operations/find.pass.cpp
@@ -0,0 +1,57 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
+
+// <flat_map>
+
+// class flat_multimap
+
+//       iterator find(const key_type& k);
+// const_iterator find(const key_type& k) const;
+
+#include <cassert>
+#include <deque>
+#include <flat_map>
+#include <functional>
+#include <string>
+#include <utility>
+
+#include "MinSequenceContainer.h"
+#include "test_macros.h"
+#include "min_allocator.h"
+
+template <class KeyContainer, class ValueContainer>
+void test() {
+  using Key   = typename KeyContainer::value_type;
+  using Value = typename ValueContainer::value_type;
+  using M     = std::flat_multimap<Key, Value, std::less<Key>, KeyContainer, ValueContainer>;
+
+  M m = {{1, 'a'}, {1, 'a'}, {1, 'b'}, {2, 'c'}, {2, 'b'}, {4, 'a'}, {4, 'd'}, {5, 'e'}, {8, 'a'}, {8, 'h'}};
+  ASSERT_SAME_TYPE(decltype(m.find(0)), typename M::iterator);
+  ASSERT_SAME_TYPE(decltype(std::as_const(m).find(0)), typename M::const_iterator);
+  assert(m.find(0) == m.end());
+  assert(m.find(1) == m.begin());
+  assert(m.find(2) == m.begin() + 3);
+  assert(m.find(3) == m.end());
+  assert(m.find(4) == m.begin() + 5);
+  assert(m.find(5) == m.begin() + 7);
+  assert(m.find(6) == m.end());
+  assert(m.find(7) == m.end());
+  assert(std::as_const(m).find(8) == m.begin() + 8);
+  assert(std::as_const(m).find(9) == m.end());
+}
+
+int main(int, char**) {
+  test<std::vector<int>, std::vector<char>>();
+  test<std::deque<int>, std::vector<char>>();
+  test<MinSequenceContainer<int>, MinSequenceContainer<char>>();
+  test<std::vector<int, min_allocator<int>>, std::vector<char, min_allocator<char>>>();
+
+  return 0;
+}
diff --git a/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.operations/find_transparent.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.operations/find_transparent.pass.cpp
new file mode 100644
index 0000000000000..be8c6f2e35440
--- /dev/null
+++ b/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.operations/find_transparent.pass.cpp
@@ -0,0 +1,99 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
+
+// <flat_map>
+
+// class flat_multimap
+
+// template<class K> iterator       find(const K& x);
+// template<class K> const_iterator find(const K& x) const;
+
+#include <cassert>
+#include <deque>
+#include <flat_map>
+#include <string>
+#include <utility>
+
+#include "MinSequenceContainer.h"
+#include "../helpers.h"
+#include "test_macros.h"
+#include "min_allocator.h"
+
+// Constraints: The qualified-id Compare::is_transparent is valid and denotes a type.
+template <class M>
+concept CanFind         = requires(M m, Transparent<int> k) { m.find(k); };
+using TransparentMap    = std::flat_multimap<int, double, TransparentComparator>;
+using NonTransparentMap = std::flat_multimap<int, double, NonTransparentComparator>;
+static_assert(CanFind<TransparentMap>);
+static_assert(CanFind<const TransparentMap>);
+static_assert(!CanFind<NonTransparentMap>);
+static_assert(!CanFind<const NonTransparentMap>);
+
+template <class KeyContainer, class ValueContainer>
+void test() {
+  using Key   = typename KeyContainer::value_type;
+  using Value = typename ValueContainer::value_type;
+  using M     = std::flat_multimap<Key, Value, TransparentComparator, KeyContainer, ValueContainer>;
+
+  M m            = {{"alpha", 1},
+                    {"beta", 2},
+                    {"beta", 0},
+                    {"beta", 1},
+                    {"beta", 2},
+                    {"epsilon", 3},
+                    {"epsilon", 1},
+                    {"eta", 4},
+                    {"gamma", 6},
+                    {"gamma", 5}};
+  const auto& cm = m;
+  ASSERT_SAME_TYPE(decltype(m.find(Transparent<std::string>{"abc"})), typename M::iterator);
+  ASSERT_SAME_TYPE(decltype(std::as_const(m).find(Transparent<std::string>{"b"})), typename M::const_iterator);
+
+  auto test_find = [&](auto&& map, const std::string& expected_key, long expected_offset) {
+    auto iter = map.find(Transparent<std::string>{expected_key});
+    assert(iter - map.begin() == expected_offset);
+  };
+
+  test_find(m, "alpha", 0);
+  test_find(m, "beta", 1);
+  test_find(m, "epsilon", 5);
+  test_find(m, "eta", 7);
+  test_find(m, "gamma", 8);
+  test_find(m, "charlie", 10);
+  test_find(m, "aaa", 10);
+  test_find(m, "zzz", 10);
+  test_find(cm, "alpha", 0);
+  test_find(cm, "beta", 1);
+  test_find(cm, "epsilon", 5);
+  test_find(cm, "eta", 7);
+  test_find(cm, "gamma", 8);
+  test_find(cm, "charlie", 10);
+  test_find(cm, "aaa", 10);
+  test_find(cm, "zzz", 10);
+}
+
+int main(int, char**) {
+  test<std::vector<std::string>, std::vector<int>>();
+  test<std::deque<std::string>, std::vector<int>>();
+  test<MinSequenceContainer<std::string>, MinSequenceContainer<int>>();
+  test<std::vector<std::string, min_allocator<std::string>>, std::vector<int, min_allocator<int>>>();
+
+  {
+    bool transparent_used = false;
+    TransparentComparator c(transparent_used);
+    std::flat_multimap<int, int, TransparentComparator> m(std::sorted_equivalent, {{1, 1}, {2, 2}, {3, 3}, {3, 3}}, c);
+    assert(!transparent_used);
+    auto it = m.find(Transparent<int>{3});
+    assert(it != m.end());
+    assert(transparent_used);
+  }
+
+  return 0;
+}
diff --git a/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.operations/lower_bound.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.operations/lower_bound.pass.cpp
new file mode 100644
index 0000000000000..c3befdda7de6e
--- /dev/null
+++ b/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.operations/lower_bound.pass.cpp
@@ -0,0 +1,73 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
+
+// <flat_map>
+
+// class flat_multimap
+
+//       iterator lower_bound(const key_type& k);
+// const_iterator lower_bound(const key_type& k) const;
+
+#include <cassert>
+#include <deque>
+#include <flat_map>
+#include <functional>
+#include <utility>
+
+#include "MinSequenceContainer.h"
+#include "test_macros.h"
+#include "min_allocator.h"
+
+template <class KeyContainer, class ValueContainer>
+void test() {
+  using Key   = typename KeyContainer::value_type;
+  using Value = typename ValueContainer::value_type;
+  {
+    using M = std::flat_multimap<Key, Value, std::less<Key>, KeyContainer, ValueContainer>;
+    M m     = {{1, 'a'}, {2, 'a'}, {2, 'c'}, {2, 'b'}, {4, 'd'}, {5, 'a'}, {5, 'e'}, {8, 'h'}, {8, 'a'}};
+    ASSERT_SAME_TYPE(decltype(m.lower_bound(0)), typename M::iterator);
+    ASSERT_SAME_TYPE(decltype(std::as_const(m).lower_bound(0)), typename M::const_iterator);
+    assert(m.lower_bound(0) == m.begin());
+    assert(m.lower_bound(1) == m.begin());
+    assert(m.lower_bound(2) == m.begin() + 1);
+    assert(m.lower_bound(3) == m.begin() + 4);
+    assert(m.lower_bound(4) == m.begin() + 4);
+    assert(m.lower_bound(5) == m.begin() + 5);
+    assert(m.lower_bound(6) == m.begin() + 7);
+    assert(m.lower_bound(7) == m.begin() + 7);
+    assert(std::as_const(m).lower_bound(8) == m.begin() + 7);
+    assert(std::as_const(m).lower_bound(9) == m.end());
+  }
+  {
+    using M = std::flat_multimap<Key, Value, std::greater<Key>, KeyContainer, ValueContainer>;
+    M m     = {{1, 'a'}, {1, 'b'}, {2, 'b'}, {4, 'd'}, {4, 'a'}, {4, 'e'}, {5, 'e'}, {8, 'a'}, {8, 'h'}};
+    ASSERT_SAME_TYPE(decltype(m.lower_bound(0)), typename M::iterator);
+    ASSERT_SAME_TYPE(decltype(std::as_const(m).lower_bound(0)), typename M::const_iterator);
+    assert(m.lower_bound(0) == m.end());
+    assert(m.lower_bound(1) == m.begin() + 7);
+    assert(m.lower_bound(2) == m.begin() + 6);
+    assert(m.lower_bound(3) == m.begin() + 6);
+    assert(m.lower_bound(4) == m.begin() + 3);
+    assert(m.lower_bound(5) == m.begin() + 2);
+    assert(m.lower_bound(6) == m.begin() + 2);
+    assert(m.lower_bound(7) == m.begin() + 2);
+    assert(std::as_const(m).lower_bound(8) == m.begin());
+    assert(std::as_const(m).lower_bound(9) == m.begin());
+  }
+}
+
+int main(int, char**) {
+  test<std::vector<int>, std::vector<char>>();
+  test<std::deque<int>, std::vector<char>>();
+  test<MinSequenceContainer<int>, MinSequenceContainer<char>>();
+  test<std::vector<int, min_allocator<int>>, std::vector<char, min_allocator<char>>>();
+
+  return 0;
+}
diff --git a/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.operations/lower_bound_transparent.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.operations/lower_bound_transparent.pass.cpp
new file mode 100644
index 0000000000000..b757af132e677
--- /dev/null
+++ b/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.operations/lower_bound_transparent.pass.cpp
@@ -0,0 +1,107 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
+
+// <flat_map>
+
+// class flat_multimap
+
+// template<class K> iterator       lower_bound(const K& x);
+// template<class K> const_iterator lower_bound(const K& x) const;
+
+#include <cassert>
+#include <deque>
+#include <flat_map>
+#include <string>
+#include <utility>
+
+#include "MinSequenceContainer.h"
+#include "../helpers.h"
+#include "test_macros.h"
+#include "min_allocator.h"
+
+// Constraints: The qualified-id Compare::is_transparent is valid and denotes a type.
+template <class M>
+concept CanLowerBound   = requires(M m, Transparent<int> k) { m.lower_bound(k); };
+using TransparentMap    = std::flat_multimap<int, double, TransparentComparator>;
+using NonTransparentMap = std::flat_multimap<int, double, NonTransparentComparator>;
+static_assert(CanLowerBound<TransparentMap>);
+static_assert(CanLowerBound<const TransparentMap>);
+static_assert(!CanLowerBound<NonTransparentMap>);
+static_assert(!CanLowerBound<const NonTransparentMap>);
+
+template <class KeyContainer, class ValueContainer>
+void test() {
+  using Key   = typename KeyContainer::value_type;
+  using Value = typename ValueContainer::value_type;
+  using M     = std::flat_multimap<Key, Value, TransparentComparator, KeyContainer, ValueContainer>;
+
+  M m            = {{"alpha", 1},
+                    {"alpha", 2},
+                    {"alpha", 3},
+                    {"beta", 2},
+                    {"epsilon", 3},
+                    {"epsilon", 4},
+                    {"eta", 4},
+                    {"gamma", 5},
+                    {"gamma", 5},
+                    {"gamma", 5},
+                    {"gamma", 5}};
+  const auto& cm = m;
+  ASSERT_SAME_TYPE(decltype(m.lower_bound(Transparent<std::string>{"abc"})), typename M::iterator);
+  ASSERT_SAME_TYPE(decltype(std::as_const(m).lower_bound(Transparent<std::string>{"b"})), typename M::const_iterator);
+
+  auto test_lower_bound = [&](auto&& map, const std::string& expected_key, long expected_offset) {
+    auto iter = map.lower_bound(Transparent<std::string>{expected_key});
+    assert(iter - map.begin() == expected_offset);
+  };
+
+  test_lower_bound(m, "abc", 0);
+  test_lower_bound(m, "alpha", 0);
+  test_lower_bound(m, "beta", 3);
+  test_lower_bound(m, "bets", 4);
+  test_lower_bound(m, "charlie", 4);
+  test_lower_bound(m, "echo", 4);
+  test_lower_bound(m, "epsilon", 4);
+  test_lower_bound(m, "eta", 6);
+  test_lower_bound(m, "gamma", 7);
+  test_lower_bound(m, "golf", 11);
+  test_lower_bound(m, "zzz", 11);
+
+  test_lower_bound(cm, "abc", 0);
+  test_lower_bound(cm, "alpha", 0);
+  test_lower_bound(cm, "beta", 3);
+  test_lower_bound(cm, "bets", 4);
+  test_lower_bound(cm, "charlie", 4);
+  test_lower_bound(cm, "echo", 4);
+  test_lower_bound(cm, "epsilon", 4);
+  test_lower_bound(cm, "eta", 6);
+  test_lower_bound(cm, "gamma", 7);
+  test_lower_bound(cm, "golf", 11);
+  test_lower_bound(cm, "zzz", 11);
+}
+
+int main(int, char**) {
+  test<std::vector<std::string>, std::vector<int>>();
+  test<std::deque<std::string>, std::vector<int>>();
+  test<MinSequenceContainer<std::string>, MinSequenceContainer<int>>();
+  test<std::vector<std::string, min_allocator<std::string>>, std::vector<int, min_allocator<int>>>();
+
+  {
+    bool transparent_used = false;
+    TransparentComparator c(transparent_used);
+    std::flat_multimap<int, int, TransparentComparator> m(std::sorted_equivalent, {{1, 1}, {2, 2}, {3, 3}}, c);
+    assert(!transparent_used);
+    auto it = m.lower_bound(Transparent<int>{3});
+    assert(it != m.end());
+    assert(transparent_used);
+  }
+
+  return 0;
+}
diff --git a/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.operations/upper_bound.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.operations/upper_bound.pass.cpp
new file mode 100644
index 0000000000000..d73d030236e22
--- /dev/null
+++ b/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.operations/upper_bound.pass.cpp
@@ -0,0 +1,76 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
+
+// <flat_map>
+
+// class flat_multimap
+
+//       iterator upper_bound(const key_type& k);
+// const_iterator upper_bound(const key_type& k) const;
+
+#include <cassert>
+#include <deque>
+#include <flat_map>
+#include <functional>
+#include <utility>
+
+#include "MinSequenceContainer.h"
+#include "test_macros.h"
+#include "min_allocator.h"
+
+template <class KeyContainer, class ValueContainer>
+void test() {
+  using Key   = typename KeyContainer::value_type;
+  using Value = typename ValueContainer::value_type;
+  {
+    using M = std::flat_multimap<Key, Value, std::less<Key>, KeyContainer, ValueContainer>;
+    M m     = {
+        {1, 'a'}, {2, 'b'}, {4, 'd'}, {4, 'e'}, {4, 'a'}, {4, 'b'}, {5, 'e'}, {5, 'a'}, {8, 'a'}, {8, 'b'}, {8, 'h'}};
+    ASSERT_SAME_TYPE(decltype(m.upper_bound(0)), typename M::iterator);
+    ASSERT_SAME_TYPE(decltype(std::as_const(m).upper_bound(0)), typename M::const_iterator);
+    assert(m.upper_bound(0) == m.begin());
+    assert(m.upper_bound(1) == m.begin() + 1);
+    assert(m.upper_bound(2) == m.begin() + 2);
+    assert(m.upper_bound(3) == m.begin() + 2);
+    assert(m.upper_bound(4) == m.begin() + 6);
+    assert(m.upper_bound(5) == m.begin() + 8);
+    assert(m.upper_bound(6) == m.begin() + 8);
+    assert(std::as_const(m).upper_bound(7) == m.begin() + 8);
+    assert(std::as_const(m).upper_bound(8) == m.end());
+    assert(std::as_const(m).upper_bound(9) == m.end());
+  }
+
+  {
+    using M = std::flat_multimap<Key, Value, std::greater<Key>, KeyContainer, ValueContainer>;
+    M m     = {
+        {1, 'a'}, {2, 'b'}, {4, 'd'}, {4, 'e'}, {4, 'a'}, {4, 'b'}, {5, 'e'}, {5, 'a'}, {8, 'a'}, {8, 'b'}, {8, 'h'}};
+    ASSERT_SAME_TYPE(decltype(m.upper_bound(0)), typename M::iterator);
+    ASSERT_SAME_TYPE(decltype(std::as_const(m).upper_bound(0)), typename M::const_iterator);
+    assert(m.upper_bound(0) == m.end());
+    assert(m.upper_bound(1) == m.end());
+    assert(m.upper_bound(2) == m.begin() + 10);
+    assert(m.upper_bound(3) == m.begin() + 9);
+    assert(m.upper_bound(4) == m.begin() + 9);
+    assert(m.upper_bound(5) == m.begin() + 5);
+    assert(m.upper_bound(6) == m.begin() + 3);
+    assert(m.upper_bound(7) == m.begin() + 3);
+    assert(std::as_const(m).upper_bound(8) == m.begin() + 3);
+    assert(std::as_const(m).upper_bound(9) == m.begin());
+  }
+}
+
+int main(int, char**) {
+  test<std::vector<int>, std::vector<char>>();
+  test<std::deque<int>, std::vector<char>>();
+  test<MinSequenceContainer<int>, MinSequenceContainer<char>>();
+  test<std::vector<int, min_allocator<int>>, std::vector<char, min_allocator<char>>>();
+
+  return 0;
+}
diff --git a/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.operations/upper_bound_transparent.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.operations/upper_bound_transparent.pass.cpp
new file mode 100644
index 0000000000000..969489d0fe619
--- /dev/null
+++ b/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.operations/upper_bound_transparent.pass.cpp
@@ -0,0 +1,106 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
+
+// <flat_map>
+
+// class flat_multimap
+
+// template<class K> iterator       upper_bound(const K& x);
+// template<class K> const_iterator upper_bound(const K& x) const;
+
+#include <cassert>
+#include <deque>
+#include <flat_map>
+#include <string>
+#include <utility>
+
+#include "MinSequenceContainer.h"
+#include "../helpers.h"
+#include "test_macros.h"
+#include "min_allocator.h"
+
+// Constraints: The qualified-id Compare::is_transparent is valid and denotes a type.
+template <class M>
+concept CanUpperBound   = requires(M m, Transparent<int> k) { m.upper_bound(k); };
+using TransparentMap    = std::flat_multimap<int, double, TransparentComparator>;
+using NonTransparentMap = std::flat_multimap<int, double, NonTransparentComparator>;
+static_assert(CanUpperBound<TransparentMap>);
+static_assert(CanUpperBound<const TransparentMap>);
+static_assert(!CanUpperBound<NonTransparentMap>);
+static_assert(!CanUpperBound<const NonTransparentMap>);
+
+template <class KeyContainer, class ValueContainer>
+void test() {
+  using Key   = typename KeyContainer::value_type;
+  using Value = typename ValueContainer::value_type;
+  using M     = std::flat_multimap<Key, Value, TransparentComparator, KeyContainer, ValueContainer>;
+
+  M m            = {{"alpha", 1},
+                    {"alpha", 2},
+                    {"alpha", 3},
+                    {"beta", 2},
+                    {"epsilon", 3},
+                    {"epsilon", 4},
+                    {"eta", 4},
+                    {"gamma", 5},
+                    {"gamma", 5},
+                    {"gamma", 5},
+                    {"gamma", 5}};
+  const auto& cm = m;
+  ASSERT_SAME_TYPE(decltype(m.lower_bound(Transparent<std::string>{"abc"})), typename M::iterator);
+  ASSERT_SAME_TYPE(decltype(std::as_const(m).lower_bound(Transparent<std::string>{"b"})), typename M::const_iterator);
+
+  auto test_upper_bound = [&](auto&& map, const std::string& expected_key, long expected_offset) {
+    auto iter = map.upper_bound(Transparent<std::string>{expected_key});
+    assert(iter - map.begin() == expected_offset);
+  };
+
+  test_upper_bound(m, "abc", 0);
+  test_upper_bound(m, "alpha", 3);
+  test_upper_bound(m, "beta", 4);
+  test_upper_bound(m, "bets", 4);
+  test_upper_bound(m, "charlie", 4);
+  test_upper_bound(m, "echo", 4);
+  test_upper_bound(m, "epsilon", 6);
+  test_upper_bound(m, "eta", 7);
+  test_upper_bound(m, "gamma", 11);
+  test_upper_bound(m, "golf", 11);
+  test_upper_bound(m, "zzz", 11);
+
+  test_upper_bound(cm, "abc", 0);
+  test_upper_bound(cm, "alpha", 3);
+  test_upper_bound(cm, "beta", 4);
+  test_upper_bound(cm, "bets", 4);
+  test_upper_bound(cm, "charlie", 4);
+  test_upper_bound(cm, "echo", 4);
+  test_upper_bound(cm, "epsilon", 6);
+  test_upper_bound(cm, "eta", 7);
+  test_upper_bound(cm, "gamma", 11);
+  test_upper_bound(cm, "golf", 11);
+  test_upper_bound(cm, "zzz", 11);
+}
+
+int main(int, char**) {
+  test<std::vector<std::string>, std::vector<int>>();
+  test<std::deque<std::string>, std::vector<int>>();
+  test<MinSequenceContainer<std::string>, MinSequenceContainer<int>>();
+  test<std::vector<std::string, min_allocator<std::string>>, std::vector<int, min_allocator<int>>>();
+  {
+    bool transparent_used = false;
+    TransparentComparator c(transparent_used);
+    std::flat_multimap<int, int, TransparentComparator> m(std::sorted_equivalent, {{1, 1}, {2, 2}, {2, 2}, {3, 3}}, c);
+    assert(!transparent_used);
+    auto it = m.upper_bound(Transparent<int>{2});
+    assert(it == m.begin() + 3);
+    assert(transparent_used);
+  }
+
+  return 0;
+}
diff --git a/libcxx/test/std/containers/container.adaptors/flat.multimap/helpers.h b/libcxx/test/std/containers/container.adaptors/flat.multimap/helpers.h
new file mode 100644
index 0000000000000..252e2454d497c
--- /dev/null
+++ b/libcxx/test/std/containers/container.adaptors/flat.multimap/helpers.h
@@ -0,0 +1,389 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef SUPPORT_FLAT_MULTIMAP_HELPERS_H
+#define SUPPORT_FLAT_MULTIMAP_HELPERS_H
+
+#include <algorithm>
+#include <cassert>
+#include <string>
+#include <vector>
+#include <flat_map>
+
+#include "test_allocator.h"
+#include "test_macros.h"
+
+template <class... Args>
+void check_invariant(const std::flat_multimap<Args...>& m) {
+  assert(m.keys().size() == m.values().size());
+  const auto& keys = m.keys();
+  assert(std::is_sorted(keys.begin(), keys.end(), m.key_comp()));
+}
+
+struct StartsWith {
+  explicit StartsWith(char ch) : lower_(1, ch), upper_(1, ch + 1) {}
+  StartsWith(const StartsWith&)     = delete;
+  void operator=(const StartsWith&) = delete;
+  struct Less {
+    using is_transparent = void;
+    bool operator()(const std::string& a, const std::string& b) const { return a < b; }
+    bool operator()(const StartsWith& a, const std::string& b) const { return a.upper_ <= b; }
+    bool operator()(const std::string& a, const StartsWith& b) const { return a < b.lower_; }
+    bool operator()(const StartsWith&, const StartsWith&) const {
+      assert(false); // should not be called
+      return false;
+    }
+  };
+
+private:
+  std::string lower_;
+  std::string upper_;
+};
+
+template <class T>
+struct CopyOnlyVector : std::vector<T> {
+  using std::vector<T>::vector;
+
+  CopyOnlyVector(const CopyOnlyVector&) = default;
+  CopyOnlyVector(CopyOnlyVector&& other) : CopyOnlyVector(other) {}
+  CopyOnlyVector(CopyOnlyVector&& other, std::vector<T>::allocator_type alloc) : CopyOnlyVector(other, alloc) {}
+
+  CopyOnlyVector& operator=(const CopyOnlyVector&) = default;
+  CopyOnlyVector& operator=(CopyOnlyVector& other) { return this->operator=(other); }
+};
+
+template <class T, bool ConvertibleToT = false>
+struct Transparent {
+  T t;
+
+  operator T() const
+    requires ConvertibleToT
+  {
+    return t;
+  }
+};
+
+template <class T>
+using ConvertibleTransparent = Transparent<T, true>;
+
+template <class T>
+using NonConvertibleTransparent = Transparent<T, false>;
+
+struct TransparentComparator {
+  using is_transparent = void;
+
+  bool* transparent_used  = nullptr;
+  TransparentComparator() = default;
+  TransparentComparator(bool& used) : transparent_used(&used) {}
+
+  template <class T, bool Convertible>
+  bool operator()(const T& t, const Transparent<T, Convertible>& transparent) const {
+    if (transparent_used != nullptr) {
+      *transparent_used = true;
+    }
+    return t < transparent.t;
+  }
+
+  template <class T, bool Convertible>
+  bool operator()(const Transparent<T, Convertible>& transparent, const T& t) const {
+    if (transparent_used != nullptr) {
+      *transparent_used = true;
+    }
+    return transparent.t < t;
+  }
+
+  template <class T>
+  bool operator()(const T& t1, const T& t2) const {
+    return t1 < t2;
+  }
+};
+
+struct NonTransparentComparator {
+  template <class T, bool Convertible>
+  bool operator()(const T&, const Transparent<T, Convertible>&) const;
+
+  template <class T, bool Convertible>
+  bool operator()(const Transparent<T, Convertible>&, const T&) const;
+
+  template <class T>
+  bool operator()(const T&, const T&) const;
+};
+
+struct NoDefaultCtr {
+  NoDefaultCtr() = delete;
+};
+
+#ifndef TEST_HAS_NO_EXCEPTIONS
+template <class T>
+struct EmplaceUnsafeContainer : std::vector<T> {
+  using std::vector<T>::vector;
+
+  template <class... Args>
+  auto emplace(Args&&... args) -> decltype(std::declval<std::vector<T>>().emplace(std::forward<Args>(args)...)) {
+    if (this->size() > 1) {
+      auto it1 = this->begin();
+      auto it2 = it1 + 1;
+      // messing up the container
+      std::iter_swap(it1, it2);
+    }
+
+    throw 42;
+  }
+
+  template <class... Args>
+  auto insert(Args&&... args) -> decltype(std::declval<std::vector<T>>().insert(std::forward<Args>(args)...)) {
+    if (this->size() > 1) {
+      auto it1 = this->begin();
+      auto it2 = it1 + 1;
+      // messing up the container
+      std::iter_swap(it1, it2);
+    }
+
+    throw 42;
+  }
+};
+
+template <class T>
+struct ThrowOnEraseContainer : std::vector<T> {
+  using std::vector<T>::vector;
+
+  template <class... Args>
+  auto erase(Args&&... args) -> decltype(std::declval<std::vector<T>>().erase(std::forward<Args>(args)...)) {
+    throw 42;
+  }
+};
+
+template <class T>
+struct ThrowOnMoveContainer : std::vector<T> {
+  using std::vector<T>::vector;
+
+  ThrowOnMoveContainer(ThrowOnMoveContainer&&) { throw 42; }
+
+  ThrowOnMoveContainer& operator=(ThrowOnMoveContainer&&) { throw 42; }
+};
+
+#endif
+
+template <class F>
+void test_emplace_exception_guarantee([[maybe_unused]] F&& emplace_function) {
+#ifndef TEST_HAS_NO_EXCEPTIONS
+  using C = TransparentComparator;
+  {
+    // Throw on emplace the key, and underlying has strong exception guarantee
+    using KeyContainer = std::vector<int, test_allocator<int>>;
+    using M            = std::flat_multimap<int, int, C, KeyContainer>;
+
+    LIBCPP_STATIC_ASSERT(std::__container_traits<KeyContainer>::__emplacement_has_strong_exception_safety_guarantee);
+
+    test_allocator_statistics stats;
+
+    KeyContainer a({1, 1, 2, 4}, test_allocator<int>{&stats});
+    std::vector<int> b                    = {5, 6, 7, 8};
+    [[maybe_unused]] auto expected_keys   = a;
+    [[maybe_unused]] auto expected_values = b;
+    M m(std::sorted_equivalent, std::move(a), std::move(b));
+
+    stats.throw_after = 1;
+    try {
+      emplace_function(m, 1, 1);
+      assert(false);
+    } catch (const std::bad_alloc&) {
+      check_invariant(m);
+      // In libc++, the flat_multimap is unchanged
+      LIBCPP_ASSERT(m.size() == 4);
+      LIBCPP_ASSERT(m.keys() == expected_keys);
+      LIBCPP_ASSERT(m.values() == expected_values);
+    }
+  }
+  {
+    // Throw on emplace the key, and underlying has no strong exception guarantee
+    using KeyContainer = EmplaceUnsafeContainer<int>;
+    using M            = std::flat_multimap<int, int, C, KeyContainer>;
+
+    LIBCPP_STATIC_ASSERT(!std::__container_traits<KeyContainer>::__emplacement_has_strong_exception_safety_guarantee);
+    KeyContainer a     = {1, 2, 2, 4};
+    std::vector<int> b = {5, 6, 7, 8};
+    M m(std::sorted_equivalent, std::move(a), std::move(b));
+    try {
+      emplace_function(m, 1, 1);
+      assert(false);
+    } catch (int) {
+      check_invariant(m);
+      // In libc++, the flat_multimap is cleared
+      LIBCPP_ASSERT(m.size() == 0);
+    }
+  }
+  {
+    // Throw on emplace the value, and underlying has strong exception guarantee
+    using ValueContainer = std::vector<int, test_allocator<int>>;
+    ;
+    using M = std::flat_multimap<int, int, C, std::vector<int>, ValueContainer>;
+
+    LIBCPP_STATIC_ASSERT(std::__container_traits<ValueContainer>::__emplacement_has_strong_exception_safety_guarantee);
+
+    std::vector<int> a = {1, 3, 3, 4};
+    test_allocator_statistics stats;
+    ValueContainer b({1, 2, 3, 4}, test_allocator<int>{&stats});
+
+    [[maybe_unused]] auto expected_keys   = a;
+    [[maybe_unused]] auto expected_values = b;
+    M m(std::sorted_equivalent, std::move(a), std::move(b));
+
+    stats.throw_after = 1;
+    try {
+      emplace_function(m, 3, 3);
+      assert(false);
+    } catch (const std::bad_alloc&) {
+      check_invariant(m);
+      // In libc++, the emplaced key is erased and the flat_multimap is unchanged
+      LIBCPP_ASSERT(m.size() == 4);
+      LIBCPP_ASSERT(m.keys() == expected_keys);
+      LIBCPP_ASSERT(m.values() == expected_values);
+    }
+  }
+  {
+    // Throw on emplace the value, and underlying has no strong exception guarantee
+    using ValueContainer = EmplaceUnsafeContainer<int>;
+    using M              = std::flat_multimap<int, int, C, std::vector<int>, ValueContainer>;
+
+    LIBCPP_STATIC_ASSERT(!std::__container_traits<ValueContainer>::__emplacement_has_strong_exception_safety_guarantee);
+    std::vector<int> a = {1, 1, 1, 1};
+    ValueContainer b   = {1, 2, 3, 4};
+
+    M m(std::sorted_equivalent, std::move(a), std::move(b));
+
+    try {
+      emplace_function(m, 1, 5);
+      assert(false);
+    } catch (int) {
+      check_invariant(m);
+      // In libc++, the flat_multimap is cleared
+      LIBCPP_ASSERT(m.size() == 0);
+    }
+  }
+  {
+    // Throw on emplace the value, then throw again on erasing the key
+    using KeyContainer   = ThrowOnEraseContainer<int>;
+    using ValueContainer = std::vector<int, test_allocator<int>>;
+    using M              = std::flat_multimap<int, int, C, KeyContainer, ValueContainer>;
+
+    LIBCPP_STATIC_ASSERT(std::__container_traits<ValueContainer>::__emplacement_has_strong_exception_safety_guarantee);
+
+    KeyContainer a = {4, 4, 4, 4};
+    test_allocator_statistics stats;
+    ValueContainer b({1, 2, 3, 4}, test_allocator<int>{&stats});
+
+    M m(std::sorted_equivalent, std::move(a), std::move(b));
+    stats.throw_after = 1;
+    try {
+      emplace_function(m, 0, 0);
+      assert(false);
+    } catch (const std::bad_alloc&) {
+      check_invariant(m);
+      // In libc++, we try to erase the key after value emplacement failure.
+      // and after erasure failure, we clear the flat_multimap
+      LIBCPP_ASSERT(m.size() == 0);
+    }
+  }
+#endif
+}
+
+template <class F>
+void test_insert_range_exception_guarantee([[maybe_unused]] F&& insert_function) {
+#ifndef TEST_HAS_NO_EXCEPTIONS
+  using KeyContainer   = EmplaceUnsafeContainer<int>;
+  using ValueContainer = std::vector<int>;
+  using M              = std::flat_multimap<int, int, std::ranges::less, KeyContainer, ValueContainer>;
+  test_allocator_statistics stats;
+  KeyContainer a{1, 2, 3, 4};
+  ValueContainer b{1, 2, 3, 4};
+  M m(std::sorted_equivalent, std::move(a), std::move(b));
+
+  std::vector<std::pair<int, int>> newValues = {{0, 0}, {1, 1}, {5, 5}, {6, 6}, {7, 7}, {8, 8}};
+  stats.throw_after                          = 1;
+  try {
+    insert_function(m, newValues);
+    assert(false);
+  } catch (int) {
+    check_invariant(m);
+    // In libc++, we clear if anything goes wrong when inserting a range
+    LIBCPP_ASSERT(m.size() == 0);
+  }
+#endif
+}
+
+template <class F>
+void test_erase_exception_guarantee([[maybe_unused]] F&& erase_function) {
+#ifndef TEST_HAS_NO_EXCEPTIONS
+  {
+    // key erase throws
+    using KeyContainer   = ThrowOnEraseContainer<int>;
+    using ValueContainer = std::vector<int>;
+    using M              = std::flat_multimap<int, int, TransparentComparator, KeyContainer, ValueContainer>;
+
+    KeyContainer a{1, 3, 3, 4};
+    ValueContainer b{1, 3, 3, 4};
+    M m(std::sorted_equivalent, std::move(a), std::move(b));
+    try {
+      erase_function(m, 3);
+      assert(false);
+    } catch (int) {
+      check_invariant(m);
+      // In libc++, we clear if anything goes wrong when erasing
+      LIBCPP_ASSERT(m.size() == 0);
+    }
+  }
+  {
+    // key erase throws
+    using KeyContainer   = std::vector<int>;
+    using ValueContainer = ThrowOnEraseContainer<int>;
+    using M              = std::flat_multimap<int, int, TransparentComparator, KeyContainer, ValueContainer>;
+
+    KeyContainer a{1, 3, 3, 4};
+    ValueContainer b{1, 3, 3, 4};
+    M m(std::sorted_equivalent, std::move(a), std::move(b));
+    try {
+      erase_function(m, 3);
+      assert(false);
+    } catch (int) {
+      check_invariant(m);
+      // In libc++, we clear if anything goes wrong when erasing
+      LIBCPP_ASSERT(m.size() == 0);
+    }
+  }
+#endif
+}
+class Moveable {
+  int int_;
+  double double_;
+
+public:
+  Moveable() : int_(0), double_(0) {}
+  Moveable(int i, double d) : int_(i), double_(d) {}
+  Moveable(Moveable&& x) : int_(x.int_), double_(x.double_) {
+    x.int_    = -1;
+    x.double_ = -1;
+  }
+  Moveable& operator=(Moveable&& x) {
+    int_      = x.int_;
+    x.int_    = -1;
+    double_   = x.double_;
+    x.double_ = -1;
+    return *this;
+  }
+
+  Moveable(const Moveable&)            = delete;
+  Moveable& operator=(const Moveable&) = delete;
+  bool operator==(const Moveable& x) const { return int_ == x.int_ && double_ == x.double_; }
+  bool operator<(const Moveable& x) const { return int_ < x.int_ || (int_ == x.int_ && double_ < x.double_); }
+
+  int get() const { return int_; }
+  bool moved() const { return int_ == -1; }
+};
+
+#endif // SUPPORT_FLAT_MULTIMAP_HELPERS_H
diff --git a/libcxx/test/std/containers/container.adaptors/flat.multimap/incomplete_type.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.multimap/incomplete_type.pass.cpp
new file mode 100644
index 0000000000000..e4325b1dfe3ba
--- /dev/null
+++ b/libcxx/test/std/containers/container.adaptors/flat.multimap/incomplete_type.pass.cpp
@@ -0,0 +1,33 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
+
+// <flat_map>
+
+// Check that std::flat_multimap and its iterators can be instantiated with an incomplete
+// type.
+
+#include <flat_map>
+#include <vector>
+
+struct A {
+  using Map = std::flat_multimap<A, A>;
+  int data;
+  Map m;
+  Map::iterator it;
+  Map::const_iterator cit;
+};
+
+// Implement the operator< required in order to instantiate flat_multimap<A, X>
+bool operator<(A const& L, A const& R) { return L.data < R.data; }
+
+int main(int, char**) {
+  A a;
+  return 0;
+}
diff --git a/libcxx/test/std/containers/container.adaptors/flat.multimap/op_compare.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.multimap/op_compare.pass.cpp
new file mode 100644
index 0000000000000..680ff1a127dda
--- /dev/null
+++ b/libcxx/test/std/containers/container.adaptors/flat.multimap/op_compare.pass.cpp
@@ -0,0 +1,133 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
+
+// <flat_map>
+
+// class flat_multimap
+
+// friend bool operator==(const flat_multimap& x, const flat_multimap& y);
+// friend synth-three-way-result<value_type>
+//   operator<=>(const flat_multimap& x, const flat_multimap& y);
+
+#include <algorithm>
+#include <cassert>
+#include <deque>
+#include <compare>
+#include <flat_map>
+#include <functional>
+#include <limits>
+#include <vector>
+
+#include "MinSequenceContainer.h"
+#include "test_macros.h"
+#include "min_allocator.h"
+#include "test_allocator.h"
+#include "test_comparisons.h"
+#include "test_container_comparisons.h"
+
+template <class KeyContainer, class ValueContainer>
+void test() {
+  using Key   = typename KeyContainer::value_type;
+  using Value = typename ValueContainer::value_type;
+
+  {
+    using C = std::flat_multimap<Key, Value>;
+    C s1    = {{1, 1}};
+    C s2    = {{2, 0}}; // {{1,1}} versus {{2,0}}
+    ASSERT_SAME_TYPE(decltype(s1 <=> s2), std::strong_ordering);
+    AssertComparisonsReturnBool<C>();
+    assert(testComparisons(s1, s2, false, true));
+    s2 = {{1, 1}}; // {{1,1}} versus {{1,1}}
+    assert(testComparisons(s1, s2, true, false));
+    s2 = {{1, 1}, {2, 0}}; // {{1,1}} versus {{1,1},{2,0}}
+    assert(testComparisons(s1, s2, false, true));
+    s1 = {{0, 0}, {1, 1}, {2, 2}}; // {{0,0},{1,1},{2,2}} versus {{1,1},{2,0}}
+    assert(testComparisons(s1, s2, false, true));
+    s2 = {{0, 0}, {1, 1}, {2, 3}}; // {{0,0},{1,1},{2,2}} versus {{0,0},{1,1},{2,3}}
+    assert(testComparisons(s1, s2, false, true));
+
+    s1 = {{1, 1}, {1, 1}};
+    s2 = {{1, 1}, {1, 1}};
+    assert(testComparisons(s1, s2, true, false));
+
+    s2 = {{1, 1}, {1, 1}, {2, 2}};
+    assert(testComparisons(s1, s2, false, true));
+
+    s2 = {{1, 1}, {2, 2}, {2, 2}};
+    assert(testComparisons(s1, s2, false, true));
+
+    s2 = {{0, 0}, {1, 1}, {1, 1}};
+    assert(testComparisons(s1, s2, false, false));
+  }
+  {
+    // Comparisons use value_type's native operators, not the comparator
+    using C = std::flat_multimap<Key, Value, std::greater<Key>>;
+    C s1    = {{1, 1}};
+    C s2    = {{2, 0}}; // {{1,1}} versus {{2,0}}
+    ASSERT_SAME_TYPE(decltype(s1 <=> s2), std::strong_ordering);
+    AssertComparisonsReturnBool<C>();
+    assert(testComparisons(s1, s2, false, true));
+    s2 = {{1, 1}}; // {{1,1}} versus {{1,1}}
+    assert(testComparisons(s1, s2, true, false));
+    s2 = {{1, 1}, {2, 0}}; // {{1,1}} versus {{2,0},{1,1}}
+    assert(testComparisons(s1, s2, false, true));
+    s1 = {{0, 0}, {1, 1}, {2, 2}}; // {{2,2},{1,1},{0,0}} versus {2,0},{1,1}}
+    assert(testComparisons(s1, s2, false, false));
+    s2 = {{0, 0}, {1, 1}, {2, 3}}; // {{2,2},{1,1},{0,0}} versus {{2,3},{1,1},{0,0}}
+    assert(testComparisons(s1, s2, false, true));
+  }
+}
+
+int main(int, char**) {
+  test<std::vector<int>, std::vector<int>>();
+  test<std::deque<int>, std::deque<int>>();
+  test<MinSequenceContainer<int>, MinSequenceContainer<int>>();
+  test<std::vector<int, min_allocator<int>>, std::vector<int, min_allocator<int>>>();
+  test<std::vector<int, min_allocator<int>>, std::vector<int, min_allocator<int>>>();
+
+  {
+    using C = std::flat_multimap<double, int>;
+    C s1    = {{1, 1}};
+    C s2    = C(std::sorted_equivalent, {{std::numeric_limits<double>::quiet_NaN(), 2}});
+    ASSERT_SAME_TYPE(decltype(s1 <=> s2), std::partial_ordering);
+    AssertComparisonsReturnBool<C>();
+    assert(testComparisonsComplete(s1, s2, false, false, false));
+  }
+  {
+    using C = std::flat_multimap<int, double>;
+    C s1    = {{1, 1}};
+    C s2    = C(std::sorted_equivalent, {{2, std::numeric_limits<double>::quiet_NaN()}});
+    ASSERT_SAME_TYPE(decltype(s1 <=> s2), std::partial_ordering);
+    AssertComparisonsReturnBool<C>();
+    assert(testComparisonsComplete(s1, s2, false, true, false));
+    s2 = C(std::sorted_equivalent, {{1, std::numeric_limits<double>::quiet_NaN()}});
+    assert(testComparisonsComplete(s1, s2, false, false, false));
+  }
+  {
+    // Comparisons use value_type's native operators, not the comparator
+    struct StrongComp {
+      bool operator()(double a, double b) const { return std::strong_order(a, b) < 0; }
+    };
+    using C = std::flat_multimap<double, double, StrongComp>;
+    C s1    = {{1, 1}};
+    C s2    = {{std::numeric_limits<double>::quiet_NaN(), std::numeric_limits<double>::quiet_NaN()}};
+    ASSERT_SAME_TYPE(decltype(s1 <=> s2), std::partial_ordering);
+    AssertComparisonsReturnBool<C>();
+    assert(testComparisonsComplete(s1, s2, false, false, false));
+    s1 = {{{1, 1}, {std::numeric_limits<double>::quiet_NaN(), 1}}};
+    s2 = {{{std::numeric_limits<double>::quiet_NaN(), 1}, {1, 1}}};
+    assert(std::lexicographical_compare_three_way(
+               s1.keys().begin(), s1.keys().end(), s2.keys().begin(), s2.keys().end(), std::strong_order) ==
+           std::strong_ordering::equal);
+    assert(s1 != s2);
+    assert((s1 <=> s2) == std::partial_ordering::unordered);
+  }
+  return 0;
+}
diff --git a/libcxx/test/std/containers/container.adaptors/flat.multimap/types.compile.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.multimap/types.compile.pass.cpp
new file mode 100644
index 0000000000000..490d51c299793
--- /dev/null
+++ b/libcxx/test/std/containers/container.adaptors/flat.multimap/types.compile.pass.cpp
@@ -0,0 +1,133 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
+
+//  using key_type               = Key;
+//  using mapped_type            = T;
+//  using value_type             = pair<key_type, mapped_type>;
+//  using key_compare            = Compare;
+//  using reference              = pair<const key_type&, mapped_type&>;
+//  using const_reference        = pair<const key_type&, const mapped_type&>;
+//  using size_type              = size_t;
+//  using difference_type        = ptrdiff_t;
+//  using iterator               = implementation-defined; // see [container.requirements]
+//  using const_iterator         = implementation-defined; // see [container.requirements]
+//  using reverse_iterator       = std::reverse_iterator<iterator>;
+//  using const_reverse_iterator = std::reverse_iterator<const_iterator>;
+//  using key_container_type     = KeyContainer;
+//  using mapped_container_type  = MappedContainer;
+
+//  class value_compare;
+
+//  struct containers {
+//    key_container_type keys;
+//    mapped_container_type values;
+//  };
+
+#include <concepts>
+#include <deque>
+#include <flat_map>
+#include <functional>
+#include <ranges>
+#include <string>
+#include <vector>
+#include "min_allocator.h"
+
+void test() {
+  {
+    using M = std::flat_multimap<int, double>;
+    static_assert(std::is_same_v<typename M::key_type, int>);
+    static_assert(std::is_same_v<typename M::mapped_type, double>);
+    static_assert(std::is_same_v<typename M::value_type, std::pair<int, double>>);
+    static_assert(std::is_same_v<typename M::key_compare, std::less<int>>);
+    static_assert(std::is_same_v<typename M::reference, std::pair<const int&, double&>>);
+    static_assert(std::is_same_v<typename M::const_reference, std::pair<const int&, const double&>>);
+    static_assert(std::is_same_v<typename M::size_type, size_t>);
+    static_assert(std::is_same_v<typename M::difference_type, ptrdiff_t>);
+    static_assert(requires { typename M::iterator; });
+    static_assert(requires { typename M::const_iterator; });
+    static_assert(std::is_same_v<typename M::reverse_iterator, std::reverse_iterator<typename M::iterator>>);
+    static_assert(
+        std::is_same_v<typename M::const_reverse_iterator, std::reverse_iterator<typename M::const_iterator>>);
+    static_assert(std::is_same_v<typename M::key_container_type, std::vector<int>>);
+    static_assert(std::is_same_v<typename M::mapped_container_type, std::vector<double>>);
+    static_assert(requires { typename M::value_compare; });
+    static_assert(requires { typename M::containers; });
+    static_assert(std::is_same_v<decltype(M::containers::keys), std::vector<int>>);
+    static_assert(std::is_same_v<decltype(M::containers::values), std::vector<double>>);
+  }
+
+  {
+    struct A {};
+    struct Compare {
+      bool operator()(const std::string&, const std::string&) const;
+    };
+    using M = std::flat_multimap<std::string, A, Compare, std::deque<std::string>, std::deque<A>>;
+    static_assert(std::is_same_v<typename M::key_type, std::string>);
+    static_assert(std::is_same_v<typename M::mapped_type, A>);
+    static_assert(std::is_same_v<typename M::value_type, std::pair<std::string, A>>);
+    static_assert(std::is_same_v<typename M::key_compare, Compare>);
+    static_assert(std::is_same_v<typename M::reference, std::pair<const std::string&, A&>>);
+    static_assert(std::is_same_v<typename M::const_reference, std::pair<const std::string&, const A&>>);
+    static_assert(std::is_same_v<typename M::size_type, size_t>);
+    static_assert(std::is_same_v<typename M::difference_type, ptrdiff_t>);
+    static_assert(requires { typename M::iterator; });
+    static_assert(requires { typename M::const_iterator; });
+    static_assert(std::is_same_v<typename M::reverse_iterator, std::reverse_iterator<typename M::iterator>>);
+    static_assert(
+        std::is_same_v<typename M::const_reverse_iterator, std::reverse_iterator<typename M::const_iterator>>);
+    static_assert(std::is_same_v<typename M::key_container_type, std::deque<std::string>>);
+    static_assert(std::is_same_v<typename M::mapped_container_type, std::deque<A>>);
+    static_assert(requires { typename M::value_compare; });
+    static_assert(requires { typename M::containers; });
+    static_assert(std::is_same_v<decltype(M::containers::keys), std::deque<std::string>>);
+    static_assert(std::is_same_v<decltype(M::containers::values), std::deque<A>>);
+  }
+  {
+    using C = std::flat_multimap<int, short>;
+    static_assert(std::is_same_v<C::key_type, int>);
+    static_assert(std::is_same_v<C::mapped_type, short>);
+    static_assert(std::is_same_v<C::value_type, std::pair<int, short>>);
+    static_assert(std::is_same_v<C::key_compare, std::less<int>>);
+    static_assert(!std::is_same_v<C::value_compare, std::less<int>>);
+    static_assert(std::is_same_v<C::reference, std::pair<const int&, short&>>);
+    static_assert(std::is_same_v<C::const_reference, std::pair<const int&, const short&>>);
+    static_assert(std::random_access_iterator<C::iterator>);
+    static_assert(std::random_access_iterator<C::const_iterator>);
+    static_assert(std::random_access_iterator<C::reverse_iterator>);
+    static_assert(std::random_access_iterator<C::const_reverse_iterator>);
+    static_assert(std::is_same_v<C::reverse_iterator, std::reverse_iterator<C::iterator>>);
+    static_assert(std::is_same_v<C::const_reverse_iterator, std::reverse_iterator<C::const_iterator>>);
+    static_assert(std::is_same_v<C::size_type, std::size_t>);
+    static_assert(std::is_same_v<C::difference_type, std::ptrdiff_t>);
+    static_assert(std::is_same_v<C::key_container_type, std::vector<int>>);
+    static_assert(std::is_same_v<C::mapped_container_type, std::vector<short>>);
+  }
+  {
+    using C = std::flat_multimap<short, int, std::greater<long>, std::deque<short, min_allocator<short>>>;
+    static_assert(std::is_same_v<C::key_type, short>);
+    static_assert(std::is_same_v<C::mapped_type, int>);
+    static_assert(std::is_same_v<C::value_type, std::pair<short, int>>);
+    static_assert(std::is_same_v<C::key_compare, std::greater<long>>);
+    static_assert(!std::is_same_v<C::value_compare, std::greater<long>>);
+    static_assert(std::is_same_v<C::reference, std::pair<const short&, int&>>);
+    static_assert(std::is_same_v<C::const_reference, std::pair<const short&, const int&>>);
+    static_assert(std::random_access_iterator<C::iterator>);
+    static_assert(std::random_access_iterator<C::const_iterator>);
+    static_assert(std::random_access_iterator<C::reverse_iterator>);
+    static_assert(std::random_access_iterator<C::const_reverse_iterator>);
+    static_assert(std::is_same_v<C::reverse_iterator, std::reverse_iterator<C::iterator>>);
+    static_assert(std::is_same_v<C::const_reverse_iterator, std::reverse_iterator<C::const_iterator>>);
+    // size_type is invariably size_t
+    static_assert(std::is_same_v<C::size_type, std::size_t>);
+    static_assert(std::is_same_v<C::difference_type, std::ptrdiff_t>);
+    static_assert(std::is_same_v<C::key_container_type, std::deque<short, min_allocator<short>>>);
+    static_assert(std::is_same_v<C::mapped_container_type, std::vector<int>>);
+  }
+}
diff --git a/libcxx/test/std/containers/container.adaptors/from_range_container_adaptors.h b/libcxx/test/std/containers/container.adaptors/from_range_container_adaptors.h
index 1bb34fb542522..7c864ce3791e8 100644
--- a/libcxx/test/std/containers/container.adaptors/from_range_container_adaptors.h
+++ b/libcxx/test/std/containers/container.adaptors/from_range_container_adaptors.h
@@ -26,7 +26,7 @@
 #include "unwrap_container_adaptor.h"
 
 template <class Container, class Range>
-concept HasFromRangeCtr = requires (Range&& range) {
+concept HasFromRangeCtr = requires(Range&& range) {
   Container(std::from_range, std::forward<Range>(range));
   Container(std::from_range, std::forward<Range>(range), std::allocator<typename Container::value_type>());
 };
@@ -47,8 +47,8 @@ constexpr bool test_constraints() {
   return true;
 }
 
-template <template <class ...> class Adaptor,
-          template <class ...> class UnderlyingContainer,
+template <template <class...> class Adaptor,
+          template <class...> class UnderlyingContainer,
           class T,
           class Iter,
           class Sent,
@@ -80,12 +80,7 @@ constexpr void test_container_adaptor_with_input(std::vector<T>&& input) {
   }
 }
 
-template <template <class ...> class UnderlyingContainer,
-          class T,
-          class Iter,
-          class Sent,
-          class Comp,
-          class Alloc>
+template <template <class...> class UnderlyingContainer, class T, class Iter, class Sent, class Comp, class Alloc>
 constexpr void test_priority_queue_with_input(std::vector<T>&& input) {
   { // (range)
     std::ranges::subrange in(Iter(input.data()), Sent(Iter(input.data() + input.size())));
@@ -146,8 +141,8 @@ constexpr void test_priority_queue_with_input(std::vector<T>&& input) {
   }
 }
 
-template <template <class ...> class Adaptor,
-          template <class ...> class UnderlyingContainer,
+template <template <class...> class Adaptor,
+          template <class...> class UnderlyingContainer,
           class T,
           class Iter,
           class Sent,
@@ -163,12 +158,7 @@ constexpr void test_container_adaptor() {
   test_with_input({5});
 }
 
-template <template <class ...> class UnderlyingContainer,
-          class T,
-          class Iter,
-          class Sent,
-          class Comp,
-          class Alloc>
+template <template <class...> class UnderlyingContainer, class T, class Iter, class Sent, class Comp, class Alloc>
 constexpr void test_priority_queue() {
   auto test_with_input = &test_priority_queue_with_input<UnderlyingContainer, T, Iter, Sent, Comp, Alloc>;
 
@@ -180,7 +170,7 @@ constexpr void test_priority_queue() {
   test_with_input({5});
 }
 
-template <template <class ...> class Container>
+template <template <class...> class Container>
 constexpr void test_container_adaptor_move_only() {
   MoveOnly input[5];
   std::ranges::subrange in(std::move_iterator{input}, std::move_iterator{input + 5});
@@ -188,18 +178,18 @@ constexpr void test_container_adaptor_move_only() {
   [[maybe_unused]] Container<MoveOnly> c(std::from_range, in);
 }
 
-template <template <class ...> class Adaptor>
+template <template <class...> class Adaptor>
 void test_exception_safety_throwing_copy() {
 #if !defined(TEST_HAS_NO_EXCEPTIONS)
   constexpr int ThrowOn = 3;
-  using T = ThrowingCopy<ThrowOn>;
+  using T               = ThrowingCopy<ThrowOn>;
   test_exception_safety_throwing_copy<ThrowOn, /*Size=*/5>([](T* from, T* to) {
     [[maybe_unused]] Adaptor<T, std::vector<T>> c(std::from_range, std::ranges::subrange(from, to));
   });
 #endif
 }
 
-template <template <class ...> class Adaptor, class T>
+template <template <class...> class Adaptor, class T>
 void test_exception_safety_throwing_allocator() {
 #if !defined(TEST_HAS_NO_EXCEPTIONS)
   T in[] = {0, 1};
diff --git a/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.cons.alloc/ctor_alloc.pass.cpp b/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.cons.alloc/ctor_alloc.pass.cpp
index 07b4f46e65dea..903a4ca2c2b4b 100644
--- a/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.cons.alloc/ctor_alloc.pass.cpp
+++ b/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.cons.alloc/ctor_alloc.pass.cpp
@@ -18,33 +18,29 @@
 #include "test_allocator.h"
 
 template <class T>
-struct test
-    : public std::priority_queue<T, std::vector<T, test_allocator<T> > >
-{
-    typedef std::priority_queue<T, std::vector<T, test_allocator<T> > > base;
-    typedef typename base::container_type container_type;
-    typedef typename base::value_compare value_compare;
-
-    explicit test(const test_allocator<int>& a) : base(a) {}
-    test(const value_compare& comp, const test_allocator<int>& a)
-        : base(comp, c, a) {}
-    test(const value_compare& comp, const container_type& container,
-        const test_allocator<int>& a) : base(comp, container, a) {}
+struct test : public std::priority_queue<T, std::vector<T, test_allocator<T> > > {
+  typedef std::priority_queue<T, std::vector<T, test_allocator<T> > > base;
+  typedef typename base::container_type container_type;
+  typedef typename base::value_compare value_compare;
+
+  explicit test(const test_allocator<int>& a) : base(a) {}
+  test(const value_compare& comp, const test_allocator<int>& a) : base(comp, c, a) {}
+  test(const value_compare& comp, const container_type& container, const test_allocator<int>& a)
+      : base(comp, container, a) {}
 #if TEST_STD_VER >= 11
-    test(const value_compare& comp, container_type&& container,
-         const test_allocator<int>& a) : base(comp, std::move(container), a) {}
-    test(test&& q, const test_allocator<int>& a) : base(std::move(q), a) {}
+  test(const value_compare& comp, container_type&& container, const test_allocator<int>& a)
+      : base(comp, std::move(container), a) {}
+  test(test&& q, const test_allocator<int>& a) : base(std::move(q), a) {}
 #endif
-    test_allocator<int> get_allocator() {return c.get_allocator();}
+  test_allocator<int> get_allocator() { return c.get_allocator(); }
 
-    using base::c;
+  using base::c;
 };
 
-int main(int, char**)
-{
-    test<int> q((test_allocator<int>(3)));
-    assert(q.c.get_allocator() == test_allocator<int>(3));
-    assert(q.c.size() == 0);
+int main(int, char**) {
+  test<int> q((test_allocator<int>(3)));
+  assert(q.c.get_allocator() == test_allocator<int>(3));
+  assert(q.c.size() == 0);
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.cons.alloc/ctor_comp_alloc.pass.cpp b/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.cons.alloc/ctor_comp_alloc.pass.cpp
index 87b9ada9b703c..439b19377abab 100644
--- a/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.cons.alloc/ctor_comp_alloc.pass.cpp
+++ b/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.cons.alloc/ctor_comp_alloc.pass.cpp
@@ -18,33 +18,29 @@
 #include "test_allocator.h"
 
 template <class T>
-struct test
-    : public std::priority_queue<T, std::vector<T, test_allocator<T> > >
-{
-    typedef std::priority_queue<T, std::vector<T, test_allocator<T> > > base;
-    typedef typename base::container_type container_type;
-    typedef typename base::value_compare value_compare;
-
-    explicit test(const test_allocator<int>& a) : base(a) {}
-    test(const value_compare& compare, const test_allocator<int>& a)
-        : base(compare, a) {}
-    test(const value_compare& compare, const container_type& container,
-        const test_allocator<int>& a) : base(compare, container, a) {}
+struct test : public std::priority_queue<T, std::vector<T, test_allocator<T> > > {
+  typedef std::priority_queue<T, std::vector<T, test_allocator<T> > > base;
+  typedef typename base::container_type container_type;
+  typedef typename base::value_compare value_compare;
+
+  explicit test(const test_allocator<int>& a) : base(a) {}
+  test(const value_compare& compare, const test_allocator<int>& a) : base(compare, a) {}
+  test(const value_compare& compare, const container_type& container, const test_allocator<int>& a)
+      : base(compare, container, a) {}
 #if TEST_STD_VER >= 11
-    test(const value_compare& compare, container_type&& container,
-         const test_allocator<int>& a) : base(compare, std::move(container), a) {}
-    test(test&& q, const test_allocator<int>& a) : base(std::move(q), a) {}
+  test(const value_compare& compare, container_type&& container, const test_allocator<int>& a)
+      : base(compare, std::move(container), a) {}
+  test(test&& q, const test_allocator<int>& a) : base(std::move(q), a) {}
 #endif
-    test_allocator<int> get_allocator() {return c.get_allocator();}
+  test_allocator<int> get_allocator() { return c.get_allocator(); }
 
-    using base::c;
+  using base::c;
 };
 
-int main(int, char**)
-{
-    test<int> q(std::less<int>(), test_allocator<int>(3));
-    assert(q.c.get_allocator() == test_allocator<int>(3));
-    assert(q.c.size() == 0);
+int main(int, char**) {
+  test<int> q(std::less<int>(), test_allocator<int>(3));
+  assert(q.c.get_allocator() == test_allocator<int>(3));
+  assert(q.c.size() == 0);
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.cons.alloc/ctor_comp_cont_alloc.pass.cpp b/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.cons.alloc/ctor_comp_cont_alloc.pass.cpp
index afec771396799..dddc11c88689b 100644
--- a/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.cons.alloc/ctor_comp_cont_alloc.pass.cpp
+++ b/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.cons.alloc/ctor_comp_cont_alloc.pass.cpp
@@ -19,46 +19,40 @@
 #include "test_allocator.h"
 
 template <class C>
-C
-make(int n)
-{
-    C c;
-    for (int i = 0; i < n; ++i)
-        c.push_back(i);
-    return c;
+C make(int n) {
+  C c;
+  for (int i = 0; i < n; ++i)
+    c.push_back(i);
+  return c;
 }
 
 template <class T>
-struct test
-    : public std::priority_queue<T, std::vector<T, test_allocator<T> > >
-{
-    typedef std::priority_queue<T, std::vector<T, test_allocator<T> > > base;
-    typedef typename base::container_type container_type;
-    typedef typename base::value_compare value_compare;
-
-    explicit test(const test_allocator<int>& a) : base(a) {}
-    test(const value_compare& compare, const test_allocator<int>& a)
-        : base(compare, a) {}
-    test(const value_compare& compare, const container_type& container,
-        const test_allocator<int>& a) : base(compare, container, a) {}
+struct test : public std::priority_queue<T, std::vector<T, test_allocator<T> > > {
+  typedef std::priority_queue<T, std::vector<T, test_allocator<T> > > base;
+  typedef typename base::container_type container_type;
+  typedef typename base::value_compare value_compare;
+
+  explicit test(const test_allocator<int>& a) : base(a) {}
+  test(const value_compare& compare, const test_allocator<int>& a) : base(compare, a) {}
+  test(const value_compare& compare, const container_type& container, const test_allocator<int>& a)
+      : base(compare, container, a) {}
 #if TEST_STD_VER >= 11 // testing rvalue constructor
-    test(const value_compare& compare, container_type&& container,
-         const test_allocator<int>& a) : base(compare, std::move(container), a) {}
-    test(test&& q, const test_allocator<int>& a) : base(std::move(q), a) {}
+  test(const value_compare& compare, container_type&& container, const test_allocator<int>& a)
+      : base(compare, std::move(container), a) {}
+  test(test&& q, const test_allocator<int>& a) : base(std::move(q), a) {}
 #endif
-    test_allocator<int> get_allocator() {return c.get_allocator();}
+  test_allocator<int> get_allocator() { return c.get_allocator(); }
 
-    using base::c;
+  using base::c;
 };
 
-int main(int, char**)
-{
-    typedef std::vector<int, test_allocator<int> > C;
-    C v = make<C>(5);
-    test<int> q(std::less<int>(), v, test_allocator<int>(3));
-    assert(q.c.get_allocator() == test_allocator<int>(3));
-    assert(q.size() == 5);
-    assert(q.top() == 4);
+int main(int, char**) {
+  typedef std::vector<int, test_allocator<int> > C;
+  C v = make<C>(5);
+  test<int> q(std::less<int>(), v, test_allocator<int>(3));
+  assert(q.c.get_allocator() == test_allocator<int>(3));
+  assert(q.size() == 5);
+  assert(q.top() == 4);
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.cons.alloc/ctor_comp_rcont_alloc.pass.cpp b/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.cons.alloc/ctor_comp_rcont_alloc.pass.cpp
index 7ddb5f4d5a3c0..bdb84a5f274da 100644
--- a/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.cons.alloc/ctor_comp_rcont_alloc.pass.cpp
+++ b/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.cons.alloc/ctor_comp_rcont_alloc.pass.cpp
@@ -19,45 +19,39 @@
 #include "test_allocator.h"
 
 template <class C>
-C
-make(int n)
-{
-    C c;
-    for (int i = 0; i < n; ++i)
-        c.push_back(i);
-    return c;
+C make(int n) {
+  C c;
+  for (int i = 0; i < n; ++i)
+    c.push_back(i);
+  return c;
 }
 
 template <class T>
-struct test
-    : public std::priority_queue<T, std::vector<T, test_allocator<T> > >
-{
-    typedef std::priority_queue<T, std::vector<T, test_allocator<T> > > base;
-    typedef typename base::container_type container_type;
-    typedef typename base::value_compare value_compare;
-
-    explicit test(const test_allocator<int>& a) : base(a) {}
-    test(const value_compare& compare, const test_allocator<int>& a)
-        : base(compare, a) {}
-    test(const value_compare& compare, const container_type& container,
-        const test_allocator<int>& a) : base(compare, container, a) {}
+struct test : public std::priority_queue<T, std::vector<T, test_allocator<T> > > {
+  typedef std::priority_queue<T, std::vector<T, test_allocator<T> > > base;
+  typedef typename base::container_type container_type;
+  typedef typename base::value_compare value_compare;
+
+  explicit test(const test_allocator<int>& a) : base(a) {}
+  test(const value_compare& compare, const test_allocator<int>& a) : base(compare, a) {}
+  test(const value_compare& compare, const container_type& container, const test_allocator<int>& a)
+      : base(compare, container, a) {}
 #if TEST_STD_VER >= 11 // testing rvalue ctor
-    test(const value_compare& compare, container_type&& container,
-         const test_allocator<int>& a) : base(compare, std::move(container), a) {}
-    test(test&& q, const test_allocator<int>& a) : base(std::move(q), a) {}
+  test(const value_compare& compare, container_type&& container, const test_allocator<int>& a)
+      : base(compare, std::move(container), a) {}
+  test(test&& q, const test_allocator<int>& a) : base(std::move(q), a) {}
 #endif
-    test_allocator<int> get_allocator() {return c.get_allocator();}
+  test_allocator<int> get_allocator() { return c.get_allocator(); }
 
-    using base::c;
+  using base::c;
 };
 
-int main(int, char**)
-{
-    typedef std::vector<int, test_allocator<int> > C;
-    test<int> q(std::less<int>(), make<C>(5), test_allocator<int>(3));
-    assert(q.c.get_allocator() == test_allocator<int>(3));
-    assert(q.size() == 5);
-    assert(q.top() == 4);
+int main(int, char**) {
+  typedef std::vector<int, test_allocator<int> > C;
+  test<int> q(std::less<int>(), make<C>(5), test_allocator<int>(3));
+  assert(q.c.get_allocator() == test_allocator<int>(3));
+  assert(q.size() == 5);
+  assert(q.top() == 4);
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.cons.alloc/ctor_copy_alloc.pass.cpp b/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.cons.alloc/ctor_copy_alloc.pass.cpp
index 77c9989e91c51..e658d6ce874f2 100644
--- a/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.cons.alloc/ctor_copy_alloc.pass.cpp
+++ b/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.cons.alloc/ctor_copy_alloc.pass.cpp
@@ -15,46 +15,38 @@
 #include <cassert>
 
 template <class C>
-C
-make(int n)
-{
-    C c;
-    for (int i = 0; i < n; ++i)
-        c.push_back(i);
-    return c;
+C make(int n) {
+  C c;
+  for (int i = 0; i < n; ++i)
+    c.push_back(i);
+  return c;
 }
 
 #include "test_macros.h"
 #include "test_allocator.h"
 
 template <class T>
-struct test
-    : public std::priority_queue<T, std::vector<T, test_allocator<T> > >
-{
-    typedef std::priority_queue<T, std::vector<T, test_allocator<T> > > base;
-    typedef typename base::container_type container_type;
-    typedef typename base::value_compare value_compare;
-
-    explicit test(const test_allocator<int>& a) : base(a) {}
-    test(const value_compare& compare, const test_allocator<int>& a)
-        : base(compare, c, a) {}
-    test(const value_compare& compare, const container_type& container,
-         const test_allocator<int>& a) : base(compare, container, a) {}
-    test(const test& q, const test_allocator<int>& a) : base(q, a) {}
-    test_allocator<int> get_allocator() {return c.get_allocator();}
-
-    using base::c;
+struct test : public std::priority_queue<T, std::vector<T, test_allocator<T> > > {
+  typedef std::priority_queue<T, std::vector<T, test_allocator<T> > > base;
+  typedef typename base::container_type container_type;
+  typedef typename base::value_compare value_compare;
+
+  explicit test(const test_allocator<int>& a) : base(a) {}
+  test(const value_compare& compare, const test_allocator<int>& a) : base(compare, c, a) {}
+  test(const value_compare& compare, const container_type& container, const test_allocator<int>& a)
+      : base(compare, container, a) {}
+  test(const test& q, const test_allocator<int>& a) : base(q, a) {}
+  test_allocator<int> get_allocator() { return c.get_allocator(); }
+
+  using base::c;
 };
 
-int main(int, char**)
-{
-    test<int> qo(std::less<int>(),
-                      make<std::vector<int, test_allocator<int> > >(5),
-                      test_allocator<int>(2));
-    test<int> q(qo, test_allocator<int>(6));
-    assert(q.size() == 5);
-    assert(q.c.get_allocator() == test_allocator<int>(6));
-    assert(q.top() == int(4));
+int main(int, char**) {
+  test<int> qo(std::less<int>(), make<std::vector<int, test_allocator<int> > >(5), test_allocator<int>(2));
+  test<int> q(qo, test_allocator<int>(6));
+  assert(q.size() == 5);
+  assert(q.c.get_allocator() == test_allocator<int>(6));
+  assert(q.top() == int(4));
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.cons.alloc/ctor_iter_iter_alloc.pass.cpp b/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.cons.alloc/ctor_iter_iter_alloc.pass.cpp
index e413cb5921465..4786629d46a41 100644
--- a/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.cons.alloc/ctor_iter_iter_alloc.pass.cpp
+++ b/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.cons.alloc/ctor_iter_iter_alloc.pass.cpp
@@ -18,24 +18,23 @@
 #include "test_macros.h"
 #include "test_allocator.h"
 
-template<class T, class Cont, class Comp = std::less<T> >
+template <class T, class Cont, class Comp = std::less<T> >
 struct PQ : std::priority_queue<T, Cont, Comp> {
-    typedef std::priority_queue<T, Cont, Comp> base;
+  typedef std::priority_queue<T, Cont, Comp> base;
 
-    template<class It, class Alloc>
-    explicit PQ(It first, It last, const Alloc& a) : base(first, last, a) {}
+  template <class It, class Alloc>
+  explicit PQ(It first, It last, const Alloc& a) : base(first, last, a) {}
 
-    using base::c;
+  using base::c;
 };
 
-int main(int, char**)
-{
-    int a[] = {3, 5, 2, 0, 6, 8, 1};
-    typedef test_allocator<int> Alloc;
-    PQ<int, std::vector<int, Alloc> > q(a, a+7, Alloc(2));
-    assert(q.size() == 7);
-    assert(q.top() == 8);
-    assert(q.c.get_allocator() == Alloc(2));
+int main(int, char**) {
+  int a[] = {3, 5, 2, 0, 6, 8, 1};
+  typedef test_allocator<int> Alloc;
+  PQ<int, std::vector<int, Alloc> > q(a, a + 7, Alloc(2));
+  assert(q.size() == 7);
+  assert(q.top() == 8);
+  assert(q.c.get_allocator() == Alloc(2));
 
-    return 0;
+  return 0;
 }
diff --git a/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.cons.alloc/ctor_iter_iter_comp_alloc.pass.cpp b/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.cons.alloc/ctor_iter_iter_comp_alloc.pass.cpp
index 5bea9312f61e0..d3e11b30b1f16 100644
--- a/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.cons.alloc/ctor_iter_iter_comp_alloc.pass.cpp
+++ b/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.cons.alloc/ctor_iter_iter_comp_alloc.pass.cpp
@@ -19,24 +19,23 @@
 #include "test_macros.h"
 #include "test_allocator.h"
 
-template<class T, class Cont, class Comp = std::less<T> >
+template <class T, class Cont, class Comp = std::less<T> >
 struct PQ : std::priority_queue<T, Cont, Comp> {
-    typedef std::priority_queue<T, Cont, Comp> base;
+  typedef std::priority_queue<T, Cont, Comp> base;
 
-    template<class It, class Alloc>
-    explicit PQ(It first, It last, const Comp& compare, const Alloc& a) : base(first, last, compare, a) {}
+  template <class It, class Alloc>
+  explicit PQ(It first, It last, const Comp& compare, const Alloc& a) : base(first, last, compare, a) {}
 
-    using base::c;
+  using base::c;
 };
 
-int main(int, char**)
-{
-    int a[] = {3, 5, 2, 0, 6, 8, 1};
-    typedef test_allocator<int> Alloc;
-    PQ<int, std::vector<int, Alloc>, std::greater<int> > q(a, a+7, std::greater<int>(), Alloc(2));
-    assert(q.size() == 7);
-    assert(q.top() == 0);
-    assert(q.c.get_allocator() == Alloc(2));
+int main(int, char**) {
+  int a[] = {3, 5, 2, 0, 6, 8, 1};
+  typedef test_allocator<int> Alloc;
+  PQ<int, std::vector<int, Alloc>, std::greater<int> > q(a, a + 7, std::greater<int>(), Alloc(2));
+  assert(q.size() == 7);
+  assert(q.top() == 0);
+  assert(q.c.get_allocator() == Alloc(2));
 
-    return 0;
+  return 0;
 }
diff --git a/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.cons.alloc/ctor_iter_iter_comp_cont_alloc.pass.cpp b/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.cons.alloc/ctor_iter_iter_comp_cont_alloc.pass.cpp
index 1a7288e3305b0..a989ca1559630 100644
--- a/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.cons.alloc/ctor_iter_iter_comp_cont_alloc.pass.cpp
+++ b/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.cons.alloc/ctor_iter_iter_comp_cont_alloc.pass.cpp
@@ -18,25 +18,25 @@
 #include "test_macros.h"
 #include "test_allocator.h"
 
-template<class T, class Cont, class Comp = std::less<T> >
+template <class T, class Cont, class Comp = std::less<T> >
 struct PQ : std::priority_queue<T, Cont, Comp> {
-    typedef std::priority_queue<T, Cont, Comp> base;
+  typedef std::priority_queue<T, Cont, Comp> base;
 
-    template<class It, class Alloc>
-    explicit PQ(It first, It last, const Comp& compare, const Cont& v, const Alloc& a) : base(first, last, compare, v, a) {}
+  template <class It, class Alloc>
+  explicit PQ(It first, It last, const Comp& compare, const Cont& v, const Alloc& a)
+      : base(first, last, compare, v, a) {}
 
-    using base::c;
+  using base::c;
 };
 
-int main(int, char**)
-{
-    typedef test_allocator<int> Alloc;
-    int a[] = {3, 5, 2, 0, 6, 8, 1};
-    std::vector<int, Alloc> v(a, a+3);
-    PQ<int, std::vector<int, Alloc> > q(a+3, a+7, std::less<int>(), v, Alloc(2));
-    assert(q.size() == 7);
-    assert(q.top() == 8);
-    assert(q.c.get_allocator() == Alloc(2));
-
-    return 0;
+int main(int, char**) {
+  typedef test_allocator<int> Alloc;
+  int a[] = {3, 5, 2, 0, 6, 8, 1};
+  std::vector<int, Alloc> v(a, a + 3);
+  PQ<int, std::vector<int, Alloc> > q(a + 3, a + 7, std::less<int>(), v, Alloc(2));
+  assert(q.size() == 7);
+  assert(q.top() == 8);
+  assert(q.c.get_allocator() == Alloc(2));
+
+  return 0;
 }
diff --git a/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.cons.alloc/ctor_iter_iter_comp_rcont_alloc.pass.cpp b/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.cons.alloc/ctor_iter_iter_comp_rcont_alloc.pass.cpp
index 31595f478be1b..bd373629955ed 100644
--- a/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.cons.alloc/ctor_iter_iter_comp_rcont_alloc.pass.cpp
+++ b/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.cons.alloc/ctor_iter_iter_comp_rcont_alloc.pass.cpp
@@ -21,26 +21,25 @@
 #include "test_allocator.h"
 #include "MoveOnly.h"
 
-template<class T, class Cont, class Comp = std::less<T> >
+template <class T, class Cont, class Comp = std::less<T> >
 struct PQ : std::priority_queue<T, Cont, Comp> {
-    typedef std::priority_queue<T, Cont, Comp> base;
+  typedef std::priority_queue<T, Cont, Comp> base;
 
-    template<class It, class Alloc>
-    explicit PQ(It first, It last, const Comp& compare, Cont&& v, const Alloc& a) : base(first, last, compare, std::move(v), a) {}
+  template <class It, class Alloc>
+  explicit PQ(It first, It last, const Comp& compare, Cont&& v, const Alloc& a)
+      : base(first, last, compare, std::move(v), a) {}
 
-    using base::c;
+  using base::c;
 };
 
-int main(int, char**)
-{
-    using Alloc = test_allocator<MoveOnly>;
-    int a[] = {3, 5, 2, 0, 6, 8, 1};
-    PQ<MoveOnly, std::vector<MoveOnly, Alloc>> q(
-        a+3, a+7, std::less<MoveOnly>(),
-        std::vector<MoveOnly, Alloc>(a, a+3), Alloc(2));
-    assert(q.size() == 7);
-    assert(q.top() == MoveOnly(8));
-    assert(q.c.get_allocator() == Alloc(2));
-
-    return 0;
+int main(int, char**) {
+  using Alloc = test_allocator<MoveOnly>;
+  int a[]     = {3, 5, 2, 0, 6, 8, 1};
+  PQ<MoveOnly, std::vector<MoveOnly, Alloc>> q(
+      a + 3, a + 7, std::less<MoveOnly>(), std::vector<MoveOnly, Alloc>(a, a + 3), Alloc(2));
+  assert(q.size() == 7);
+  assert(q.top() == MoveOnly(8));
+  assert(q.c.get_allocator() == Alloc(2));
+
+  return 0;
 }
diff --git a/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.cons.alloc/ctor_move_alloc.pass.cpp b/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.cons.alloc/ctor_move_alloc.pass.cpp
index c1c05808606ab..95c88cf877a59 100644
--- a/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.cons.alloc/ctor_move_alloc.pass.cpp
+++ b/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.cons.alloc/ctor_move_alloc.pass.cpp
@@ -19,50 +19,41 @@
 #include "test_macros.h"
 #include "MoveOnly.h"
 
-
 template <class C>
-C
-make(int n)
-{
-    C c;
-    for (int i = 0; i < n; ++i)
-        c.push_back(MoveOnly(i));
-    return c;
+C make(int n) {
+  C c;
+  for (int i = 0; i < n; ++i)
+    c.push_back(MoveOnly(i));
+  return c;
 }
 
 #include "test_allocator.h"
 
 template <class T>
-struct test
-    : public std::priority_queue<T, std::vector<T, test_allocator<T> > >
-{
-    typedef std::priority_queue<T, std::vector<T, test_allocator<T> > > base;
-    typedef typename base::container_type container_type;
-    typedef typename base::value_compare value_compare;
-
-    explicit test(const test_allocator<int>& a) : base(a) {}
-    test(const value_compare& compare, const test_allocator<int>& a)
-        : base(compare, c, a) {}
-    test(const value_compare& compare, const container_type& container,
-        const test_allocator<int>& a) : base(compare, container, a) {}
-    test(const value_compare& compare, container_type&& container,
-         const test_allocator<int>& a) : base(compare, std::move(container), a) {}
-    test(test&& q, const test_allocator<int>& a) : base(std::move(q), a) {}
-    test_allocator<int> get_allocator() {return c.get_allocator();}
-
-    using base::c;
+struct test : public std::priority_queue<T, std::vector<T, test_allocator<T> > > {
+  typedef std::priority_queue<T, std::vector<T, test_allocator<T> > > base;
+  typedef typename base::container_type container_type;
+  typedef typename base::value_compare value_compare;
+
+  explicit test(const test_allocator<int>& a) : base(a) {}
+  test(const value_compare& compare, const test_allocator<int>& a) : base(compare, c, a) {}
+  test(const value_compare& compare, const container_type& container, const test_allocator<int>& a)
+      : base(compare, container, a) {}
+  test(const value_compare& compare, container_type&& container, const test_allocator<int>& a)
+      : base(compare, std::move(container), a) {}
+  test(test&& q, const test_allocator<int>& a) : base(std::move(q), a) {}
+  test_allocator<int> get_allocator() { return c.get_allocator(); }
+
+  using base::c;
 };
 
-
-int main(int, char**)
-{
-    test<MoveOnly> qo(std::less<MoveOnly>(),
-                      make<std::vector<MoveOnly, test_allocator<MoveOnly> > >(5),
-                      test_allocator<MoveOnly>(2));
-    test<MoveOnly> q(std::move(qo), test_allocator<MoveOnly>(6));
-    assert(q.size() == 5);
-    assert(q.c.get_allocator() == test_allocator<MoveOnly>(6));
-    assert(q.top() == MoveOnly(4));
+int main(int, char**) {
+  test<MoveOnly> qo(
+      std::less<MoveOnly>(), make<std::vector<MoveOnly, test_allocator<MoveOnly> > >(5), test_allocator<MoveOnly>(2));
+  test<MoveOnly> q(std::move(qo), test_allocator<MoveOnly>(6));
+  assert(q.size() == 5);
+  assert(q.c.get_allocator() == test_allocator<MoveOnly>(6));
+  assert(q.top() == MoveOnly(4));
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.cons/assign_copy.pass.cpp b/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.cons/assign_copy.pass.cpp
index 8bbee85a75983..6acc8aef34569 100644
--- a/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.cons/assign_copy.pass.cpp
+++ b/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.cons/assign_copy.pass.cpp
@@ -17,23 +17,20 @@
 #include "test_macros.h"
 
 template <class C>
-C
-make(int n)
-{
-    C c;
-    for (int i = 0; i < n; ++i)
-        c.push_back(i);
-    return c;
+C make(int n) {
+  C c;
+  for (int i = 0; i < n; ++i)
+    c.push_back(i);
+  return c;
 }
 
-int main(int, char**)
-{
-    std::vector<int> v = make<std::vector<int> >(5);
-    std::priority_queue<int, std::vector<int>, std::greater<int> > qo(std::greater<int>(), v);
-    std::priority_queue<int, std::vector<int>, std::greater<int> > q;
-    q = qo;
-    assert(q.size() == 5);
-    assert(q.top() == 0);
+int main(int, char**) {
+  std::vector<int> v = make<std::vector<int> >(5);
+  std::priority_queue<int, std::vector<int>, std::greater<int> > qo(std::greater<int>(), v);
+  std::priority_queue<int, std::vector<int>, std::greater<int> > q;
+  q = qo;
+  assert(q.size() == 5);
+  assert(q.top() == 0);
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.cons/assign_move.pass.cpp b/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.cons/assign_move.pass.cpp
index 84c3deadd7f2f..fbdc07f530662 100644
--- a/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.cons/assign_move.pass.cpp
+++ b/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.cons/assign_move.pass.cpp
@@ -18,25 +18,20 @@
 #include "test_macros.h"
 #include "MoveOnly.h"
 
-
 template <class C>
-C
-make(int n)
-{
-    C c;
-    for (int i = 0; i < n; ++i)
-        c.push_back(MoveOnly(i));
-    return c;
+C make(int n) {
+  C c;
+  for (int i = 0; i < n; ++i)
+    c.push_back(MoveOnly(i));
+  return c;
 }
 
-
-int main(int, char**)
-{
-    std::priority_queue<MoveOnly> qo(std::less<MoveOnly>(), make<std::vector<MoveOnly> >(5));
-    std::priority_queue<MoveOnly> q;
-    q = std::move(qo);
-    assert(q.size() == 5);
-    assert(q.top() == MoveOnly(4));
+int main(int, char**) {
+  std::priority_queue<MoveOnly> qo(std::less<MoveOnly>(), make<std::vector<MoveOnly> >(5));
+  std::priority_queue<MoveOnly> q;
+  q = std::move(qo);
+  assert(q.size() == 5);
+  assert(q.top() == MoveOnly(4));
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.cons/ctor_comp.pass.cpp b/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.cons/ctor_comp.pass.cpp
index 5c2fc5d89e9ed..8e56ea6234e61 100644
--- a/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.cons/ctor_comp.pass.cpp
+++ b/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.cons/ctor_comp.pass.cpp
@@ -16,24 +16,23 @@
 #include "test_macros.h"
 #include "test_allocator.h"
 #if TEST_STD_VER >= 11
-#include "test_convertible.h"
+#  include "test_convertible.h"
 #endif
 
-int main(int, char**)
-{
-    typedef std::vector<int, limited_allocator<int, 10> > Container;
-    typedef std::less<int> Compare;
-    typedef std::priority_queue<int, Container> Q;
-    Q q((Compare()));
-    assert(q.size() == 0);
-    q.push(1);
-    q.push(2);
-    assert(q.size() == 2);
-    assert(q.top() == 2);
+int main(int, char**) {
+  typedef std::vector<int, limited_allocator<int, 10> > Container;
+  typedef std::less<int> Compare;
+  typedef std::priority_queue<int, Container> Q;
+  Q q((Compare()));
+  assert(q.size() == 0);
+  q.push(1);
+  q.push(2);
+  assert(q.size() == 2);
+  assert(q.top() == 2);
 
 #if TEST_STD_VER >= 11
-    static_assert(!test_convertible<Q, const Compare&>(), "");
+  static_assert(!test_convertible<Q, const Compare&>(), "");
 #endif
 
-    return 0;
+  return 0;
 }
diff --git a/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.cons/ctor_comp_container.pass.cpp b/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.cons/ctor_comp_container.pass.cpp
index 07c50e38968a9..08e74a790f905 100644
--- a/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.cons/ctor_comp_container.pass.cpp
+++ b/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.cons/ctor_comp_container.pass.cpp
@@ -16,33 +16,30 @@
 
 #include "test_macros.h"
 #if TEST_STD_VER >= 11
-#include "test_convertible.h"
+#  include "test_convertible.h"
 #endif
 
 template <class C>
-C
-make(int n)
-{
-    C c;
-    for (int i = 0; i < n; ++i)
-        c.push_back(i);
-    return c;
+C make(int n) {
+  C c;
+  for (int i = 0; i < n; ++i)
+    c.push_back(i);
+  return c;
 }
 
-int main(int, char**)
-{
-    typedef std::vector<int> Container;
-    typedef std::greater<int> Compare;
-    typedef std::priority_queue<int, Container, Compare> Q;
-    Container v = make<Container>(5);
-    Q q(Compare(), v);
-    assert(q.size() == 5);
-    assert(q.top() == 0);
+int main(int, char**) {
+  typedef std::vector<int> Container;
+  typedef std::greater<int> Compare;
+  typedef std::priority_queue<int, Container, Compare> Q;
+  Container v = make<Container>(5);
+  Q q(Compare(), v);
+  assert(q.size() == 5);
+  assert(q.top() == 0);
 
 #if TEST_STD_VER >= 11
-    // It should be explicit, so not convertible before C++20.
-    static_assert(test_convertible<Q, const Compare&, const Container&>(), "");
+  // It should be explicit, so not convertible before C++20.
+  static_assert(test_convertible<Q, const Compare&, const Container&>(), "");
 #endif
 
-    return 0;
+  return 0;
 }
diff --git a/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.cons/ctor_comp_rcontainer.pass.cpp b/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.cons/ctor_comp_rcontainer.pass.cpp
index d555ecab549cd..f0b0eaef2f3d5 100644
--- a/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.cons/ctor_comp_rcontainer.pass.cpp
+++ b/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.cons/ctor_comp_rcontainer.pass.cpp
@@ -21,25 +21,22 @@
 #include "test_convertible.h"
 
 template <class C>
-C
-make(int n)
-{
-    C c;
-    for (int i = 0; i < n; ++i)
-        c.push_back(MoveOnly(i));
-    return c;
+C make(int n) {
+  C c;
+  for (int i = 0; i < n; ++i)
+    c.push_back(MoveOnly(i));
+  return c;
 }
 
-int main(int, char**)
-{
-    typedef std::vector<MoveOnly> Container;
-    typedef std::less<MoveOnly> Compare;
-    typedef std::priority_queue<MoveOnly> Q;
-    Q q(Compare(), make<Container>(5));
-    assert(q.size() == 5);
-    assert(q.top() == MoveOnly(4));
+int main(int, char**) {
+  typedef std::vector<MoveOnly> Container;
+  typedef std::less<MoveOnly> Compare;
+  typedef std::priority_queue<MoveOnly> Q;
+  Q q(Compare(), make<Container>(5));
+  assert(q.size() == 5);
+  assert(q.top() == MoveOnly(4));
 
-    static_assert(test_convertible<Q, const Compare&, Container&&>(), "");
+  static_assert(test_convertible<Q, const Compare&, Container&&>(), "");
 
-    return 0;
+  return 0;
 }
diff --git a/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.cons/ctor_copy.pass.cpp b/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.cons/ctor_copy.pass.cpp
index 5ab4337f8152b..0e9f58ec8b9c9 100644
--- a/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.cons/ctor_copy.pass.cpp
+++ b/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.cons/ctor_copy.pass.cpp
@@ -17,22 +17,19 @@
 #include "test_macros.h"
 
 template <class C>
-C
-make(int n)
-{
-    C c;
-    for (int i = 0; i < n; ++i)
-        c.push_back(i);
-    return c;
+C make(int n) {
+  C c;
+  for (int i = 0; i < n; ++i)
+    c.push_back(i);
+  return c;
 }
 
-int main(int, char**)
-{
-    std::vector<int> v = make<std::vector<int> >(5);
-    std::priority_queue<int, std::vector<int>, std::greater<int> > qo(std::greater<int>(), v);
-    std::priority_queue<int, std::vector<int>, std::greater<int> > q = qo;
-    assert(q.size() == 5);
-    assert(q.top() == 0);
+int main(int, char**) {
+  std::vector<int> v = make<std::vector<int> >(5);
+  std::priority_queue<int, std::vector<int>, std::greater<int> > qo(std::greater<int>(), v);
+  std::priority_queue<int, std::vector<int>, std::greater<int> > q = qo;
+  assert(q.size() == 5);
+  assert(q.top() == 0);
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.cons/ctor_default.pass.cpp b/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.cons/ctor_default.pass.cpp
index e483763ab7b6e..c25caee07ff49 100644
--- a/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.cons/ctor_default.pass.cpp
+++ b/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.cons/ctor_default.pass.cpp
@@ -18,24 +18,23 @@
 #include "test_macros.h"
 #include "test_allocator.h"
 #if TEST_STD_VER >= 11
-#include "test_convertible.h"
+#  include "test_convertible.h"
 #endif
 
-int main(int, char**)
-{
-    typedef std::vector<int, limited_allocator<int, 10> > Container;
-    typedef std::priority_queue<int, Container> Q;
-    Q q;
-    assert(q.size() == 0);
-    q.push(1);
-    q.push(2);
-    assert(q.size() == 2);
-    assert(q.top() == 2);
+int main(int, char**) {
+  typedef std::vector<int, limited_allocator<int, 10> > Container;
+  typedef std::priority_queue<int, Container> Q;
+  Q q;
+  assert(q.size() == 0);
+  q.push(1);
+  q.push(2);
+  assert(q.size() == 2);
+  assert(q.top() == 2);
 
 #if TEST_STD_VER >= 11
-    // It should be explicit, so not convertible before C++20.
-    static_assert(test_convertible<Q>(), "");
+  // It should be explicit, so not convertible before C++20.
+  static_assert(test_convertible<Q>(), "");
 #endif
 
-    return 0;
+  return 0;
 }
diff --git a/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.cons/ctor_iter_constraint.compile.pass.cpp b/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.cons/ctor_iter_constraint.compile.pass.cpp
index 283cbc02883cd..913e944b3f04c 100644
--- a/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.cons/ctor_iter_constraint.compile.pass.cpp
+++ b/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.cons/ctor_iter_constraint.compile.pass.cpp
@@ -30,30 +30,46 @@
 #include <vector>
 
 // Sanity-check that std::vector is constructible from two ints...
-static_assert( std::is_constructible<std::vector<int>,         int*, int*>::value, "");
-static_assert( std::is_constructible<std::vector<int>,         int , int >::value, "");
+static_assert(std::is_constructible<std::vector<int>, int*, int*>::value, "");
+static_assert(std::is_constructible<std::vector<int>, int, int >::value, "");
 
 // ...but std::priority_queue is not.
-static_assert( std::is_constructible<std::priority_queue<int>, int*, int*>::value, "");
-static_assert(!std::is_constructible<std::priority_queue<int>, int , int >::value, "");
+static_assert(std::is_constructible<std::priority_queue<int>, int*, int*>::value, "");
+static_assert(!std::is_constructible<std::priority_queue<int>, int, int >::value, "");
 
-static_assert( std::is_constructible<std::priority_queue<int>, int*, int*, std::less<int>>::value, "");
-static_assert(!std::is_constructible<std::priority_queue<int>, int , int , std::less<int>>::value, "");
+static_assert(std::is_constructible<std::priority_queue<int>, int*, int*, std::less<int>>::value, "");
+static_assert(!std::is_constructible<std::priority_queue<int>, int, int, std::less<int>>::value, "");
 
-static_assert( std::is_constructible<std::priority_queue<int>, int*, int*, std::less<int>, std::vector<int>>::value, "");
-static_assert(!std::is_constructible<std::priority_queue<int>, int , int , std::less<int>, std::vector<int>>::value, "");
+static_assert(std::is_constructible<std::priority_queue<int>, int*, int*, std::less<int>, std::vector<int>>::value, "");
+static_assert(!std::is_constructible<std::priority_queue<int>, int, int, std::less<int>, std::vector<int>>::value, "");
 
-static_assert( std::is_constructible<std::priority_queue<int>, int*, int*, std::less<int>, std::vector<int>&>::value, "");
-static_assert(!std::is_constructible<std::priority_queue<int>, int , int , std::less<int>, std::vector<int>&>::value, "");
+static_assert(std::is_constructible<std::priority_queue<int>, int*, int*, std::less<int>, std::vector<int>&>::value,
+              "");
+static_assert(!std::is_constructible<std::priority_queue<int>, int, int, std::less<int>, std::vector<int>&>::value, "");
 
-static_assert( std::is_constructible<std::priority_queue<int>, int*, int*, std::allocator<int>>::value, "");
-static_assert(!std::is_constructible<std::priority_queue<int>, int , int , std::allocator<int>>::value, "");
+static_assert(std::is_constructible<std::priority_queue<int>, int*, int*, std::allocator<int>>::value, "");
+static_assert(!std::is_constructible<std::priority_queue<int>, int, int, std::allocator<int>>::value, "");
 
-static_assert( std::is_constructible<std::priority_queue<int>, int*, int*, std::less<int>, std::allocator<int>>::value, "");
-static_assert(!std::is_constructible<std::priority_queue<int>, int , int , std::less<int>, std::allocator<int>>::value, "");
+static_assert(std::is_constructible<std::priority_queue<int>, int*, int*, std::less<int>, std::allocator<int>>::value,
+              "");
+static_assert(!std::is_constructible<std::priority_queue<int>, int, int, std::less<int>, std::allocator<int>>::value,
+              "");
 
-static_assert( std::is_constructible<std::priority_queue<int>, int*, int*, std::less<int>, std::vector<int>, std::allocator<int>>::value, "");
-static_assert(!std::is_constructible<std::priority_queue<int>, int , int , std::less<int>, std::vector<int>, std::allocator<int>>::value, "");
+static_assert(
+    std::is_constructible<std::priority_queue<int>, int*, int*, std::less<int>, std::vector<int>, std::allocator<int>>::
+        value,
+    "");
+static_assert(
+    !std::is_constructible<std::priority_queue<int>, int, int, std::less<int>, std::vector<int>, std::allocator<int>>::
+        value,
+    "");
 
-static_assert( std::is_constructible<std::priority_queue<int>, int*, int*, std::less<int>, std::vector<int>&, std::allocator<int>>::value, "");
-static_assert(!std::is_constructible<std::priority_queue<int>, int , int , std::less<int>, std::vector<int>&, std::allocator<int>>::value, "");
+static_assert(
+    std::
+        is_constructible<std::priority_queue<int>, int*, int*, std::less<int>, std::vector<int>&, std::allocator<int>>::
+            value,
+    "");
+static_assert(
+    !std::is_constructible<std::priority_queue<int>, int, int, std::less<int>, std::vector<int>&, std::allocator<int>>::
+        value,
+    "");
diff --git a/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.cons/ctor_iter_iter.pass.cpp b/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.cons/ctor_iter_iter.pass.cpp
index c3838fc97b8cd..9566c18193fbb 100644
--- a/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.cons/ctor_iter_iter.pass.cpp
+++ b/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.cons/ctor_iter_iter.pass.cpp
@@ -17,13 +17,12 @@
 
 #include "test_macros.h"
 
-int main(int, char**)
-{
-    int a[] = {3, 5, 2, 0, 6, 8, 1};
-    int* an = a + sizeof(a)/sizeof(a[0]);
-    std::priority_queue<int> q(a, an);
-    assert(q.size() == static_cast<std::size_t>(an - a));
-    assert(q.top() == 8);
+int main(int, char**) {
+  int a[] = {3, 5, 2, 0, 6, 8, 1};
+  int* an = a + sizeof(a) / sizeof(a[0]);
+  std::priority_queue<int> q(a, an);
+  assert(q.size() == static_cast<std::size_t>(an - a));
+  assert(q.top() == 8);
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.cons/ctor_iter_iter_comp.pass.cpp b/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.cons/ctor_iter_iter_comp.pass.cpp
index 0ae3d102e6e3c..aa3e31b919572 100644
--- a/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.cons/ctor_iter_iter_comp.pass.cpp
+++ b/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.cons/ctor_iter_iter_comp.pass.cpp
@@ -18,14 +18,12 @@
 
 #include "test_macros.h"
 
-int main(int, char**)
-{
-    int a[] = {3, 5, 2, 0, 6, 8, 1};
-    int* an = a + sizeof(a)/sizeof(a[0]);
-    std::priority_queue<int, std::vector<int>, std::greater<int> >
-        q(a, an, std::greater<int>());
-    assert(q.size() == static_cast<std::size_t>(an - a));
-    assert(q.top() == 0);
+int main(int, char**) {
+  int a[] = {3, 5, 2, 0, 6, 8, 1};
+  int* an = a + sizeof(a) / sizeof(a[0]);
+  std::priority_queue<int, std::vector<int>, std::greater<int> > q(a, an, std::greater<int>());
+  assert(q.size() == static_cast<std::size_t>(an - a));
+  assert(q.top() == 0);
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.cons/ctor_iter_iter_comp_cont.pass.cpp b/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.cons/ctor_iter_iter_comp_cont.pass.cpp
index adba98aac85f2..49518488f6c99 100644
--- a/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.cons/ctor_iter_iter_comp_cont.pass.cpp
+++ b/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.cons/ctor_iter_iter_comp_cont.pass.cpp
@@ -17,14 +17,13 @@
 
 #include "test_macros.h"
 
-int main(int, char**)
-{
-    int a[] = {3, 5, 2, 0, 6, 8, 1};
-    const int n = sizeof(a)/sizeof(a[0]);
-    std::vector<int> v(a, a+n/2);
-    std::priority_queue<int> q(a+n/2, a+n, std::less<int>(), v);
-    assert(q.size() == n);
-    assert(q.top() == 8);
+int main(int, char**) {
+  int a[]     = {3, 5, 2, 0, 6, 8, 1};
+  const int n = sizeof(a) / sizeof(a[0]);
+  std::vector<int> v(a, a + n / 2);
+  std::priority_queue<int> q(a + n / 2, a + n, std::less<int>(), v);
+  assert(q.size() == n);
+  assert(q.top() == 8);
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.cons/ctor_iter_iter_comp_rcont.pass.cpp b/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.cons/ctor_iter_iter_comp_rcont.pass.cpp
index e09cb687d2589..85e31054babc8 100644
--- a/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.cons/ctor_iter_iter_comp_rcont.pass.cpp
+++ b/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.cons/ctor_iter_iter_comp_rcont.pass.cpp
@@ -20,15 +20,12 @@
 #include "test_macros.h"
 #include "MoveOnly.h"
 
-int main(int, char**)
-{
-    int a[] = {3, 5, 2, 0, 6, 8, 1};
-    const int n = sizeof(a)/sizeof(a[0]);
-    std::priority_queue<MoveOnly> q(a+n/2, a+n,
-                                    std::less<MoveOnly>(),
-                                    std::vector<MoveOnly>(a, a+n/2));
-    assert(q.size() == n);
-    assert(q.top() == MoveOnly(8));
+int main(int, char**) {
+  int a[]     = {3, 5, 2, 0, 6, 8, 1};
+  const int n = sizeof(a) / sizeof(a[0]);
+  std::priority_queue<MoveOnly> q(a + n / 2, a + n, std::less<MoveOnly>(), std::vector<MoveOnly>(a, a + n / 2));
+  assert(q.size() == n);
+  assert(q.top() == MoveOnly(8));
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.cons/ctor_move.pass.cpp b/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.cons/ctor_move.pass.cpp
index 88a8573c9daf3..74213881e98f3 100644
--- a/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.cons/ctor_move.pass.cpp
+++ b/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.cons/ctor_move.pass.cpp
@@ -18,24 +18,19 @@
 #include "test_macros.h"
 #include "MoveOnly.h"
 
-
 template <class C>
-C
-make(int n)
-{
-    C c;
-    for (int i = 0; i < n; ++i)
-        c.push_back(MoveOnly(i));
-    return c;
+C make(int n) {
+  C c;
+  for (int i = 0; i < n; ++i)
+    c.push_back(MoveOnly(i));
+  return c;
 }
 
-
-int main(int, char**)
-{
-    std::priority_queue<MoveOnly> qo(std::less<MoveOnly>(), make<std::vector<MoveOnly> >(5));
-    std::priority_queue<MoveOnly> q = std::move(qo);
-    assert(q.size() == 5);
-    assert(q.top() == MoveOnly(4));
+int main(int, char**) {
+  std::priority_queue<MoveOnly> qo(std::less<MoveOnly>(), make<std::vector<MoveOnly> >(5));
+  std::priority_queue<MoveOnly> q = std::move(qo);
+  assert(q.size() == 5);
+  assert(q.top() == MoveOnly(4));
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.cons/deduct.pass.cpp b/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.cons/deduct.pass.cpp
index 51954b2e85123..b0099719d84f0 100644
--- a/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.cons/deduct.pass.cpp
+++ b/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.cons/deduct.pass.cpp
@@ -52,378 +52,381 @@
 
 struct A {};
 
-int main(int, char**)
-{
-
-//  Test the explicit deduction guides
-    {
-    std::vector<int> v{0, 1, 2, 3, 4, 5, 6, 7, 8, 9 };
+int main(int, char**) {
+  //  Test the explicit deduction guides
+  {
+    std::vector<int> v{0, 1, 2, 3, 4, 5, 6, 7, 8, 9};
     std::priority_queue pri(std::greater<int>(), v); // priority_queue(Compare, Container)
 
     static_assert(std::is_same_v<decltype(pri), std::priority_queue<int, std::vector<int>, std::greater<int>>>, "");
     assert(pri.size() == v.size());
     assert(pri.top() == 0);
-    }
+  }
 
-    {
-    std::vector<long, test_allocator<long>> v{10, 11, 12, 13, 14, 15, 16, 17, 18, 19 };
-    std::priority_queue pri(std::greater<long>(), v, test_allocator<long>(2)); // priority_queue(Compare, Container, Allocator)
+  {
+    std::vector<long, test_allocator<long>> v{10, 11, 12, 13, 14, 15, 16, 17, 18, 19};
+    std::priority_queue pri(
+        std::greater<long>(), v, test_allocator<long>(2)); // priority_queue(Compare, Container, Allocator)
 
-    static_assert(std::is_same_v<decltype(pri),
-                                 std::priority_queue<long, std::vector<long, test_allocator<long>>, std::greater<long>>>, "");
+    static_assert(
+        std::is_same_v<decltype(pri),
+                       std::priority_queue<long, std::vector<long, test_allocator<long>>, std::greater<long>>>,
+        "");
     assert(pri.size() == v.size());
     assert(pri.top() == 10);
-    }
+  }
 
-    {
-    std::vector<short> v{10, 11, 12, 13, 14, 15, 28, 17, 18, 19 };
+  {
+    std::vector<short> v{10, 11, 12, 13, 14, 15, 28, 17, 18, 19};
     std::priority_queue pri(v.begin(), v.end()); // priority_queue(Iter, Iter)
 
     static_assert(std::is_same_v<decltype(pri), std::priority_queue<short>>, "");
     assert(pri.size() == v.size());
     assert(pri.top() == 28);
-    }
+  }
 
-    {
-    std::vector<double> v{10, 11, 12, 13, 6, 15, 28, 17, 18, 19 };
+  {
+    std::vector<double> v{10, 11, 12, 13, 6, 15, 28, 17, 18, 19};
     std::priority_queue pri(v.begin(), v.end(), std::greater<double>()); // priority_queue(Iter, Iter, Comp)
 
-    static_assert(std::is_same_v<decltype(pri), std::priority_queue<double, std::vector<double>, std::greater<double>>>, "");
+    static_assert(
+        std::is_same_v<decltype(pri), std::priority_queue<double, std::vector<double>, std::greater<double>>>, "");
     assert(pri.size() == v.size());
     assert(pri.top() == 6);
-    }
+  }
 
-    {
-    std::vector<double> v{10, 6, 15, 28, 4, 18, 19 };
+  {
+    std::vector<double> v{10, 6, 15, 28, 4, 18, 19};
     std::deque<double> deq;
-    std::priority_queue pri(v.begin(), v.end(), std::greater<double>(), deq); // priority_queue(Iter, Iter, Comp, Container)
+    std::priority_queue pri(
+        v.begin(), v.end(), std::greater<double>(), deq); // priority_queue(Iter, Iter, Comp, Container)
 
-    static_assert(std::is_same_v<decltype(pri), std::priority_queue<double, std::deque<double>, std::greater<double>>>, "");
+    static_assert(
+        std::is_same_v<decltype(pri), std::priority_queue<double, std::deque<double>, std::greater<double>>>, "");
     assert(pri.size() == v.size());
     assert(pri.top() == 4);
-    }
+  }
 
-//  Test the implicit deduction guides
-    {
-//  We don't expect this one to work - no way to implicitly get value_type
-//  std::priority_queue pri(std::allocator<int>()); // queue (allocator &)
-    }
+  //  Test the implicit deduction guides
+  {
+    //  We don't expect this one to work - no way to implicitly get value_type
+    //  std::priority_queue pri(std::allocator<int>()); // queue (allocator &)
+  }
 
-    {
+  {
     std::priority_queue<float> source;
     std::priority_queue pri(source); // priority_queue(priority_queue &)
     static_assert(std::is_same_v<decltype(pri)::value_type, float>, "");
     static_assert(std::is_same_v<decltype(pri)::container_type, std::vector<float>>, "");
     assert(pri.size() == 0);
+  }
+
+  {
+    typedef short T;
+    typedef std::greater<T> Comp;
+    typedef test_allocator<T> Alloc;
+    typedef std::deque<T, Alloc> Cont;
+    typedef test_allocator<int> ConvertibleToAlloc;
+    static_assert(std::uses_allocator_v<Cont, ConvertibleToAlloc> &&
+                  !std::is_same_v<typename Cont::allocator_type, ConvertibleToAlloc>);
+
+    {
+      Comp comp;
+      Cont cont;
+      std::priority_queue pri(comp, cont, Alloc(2));
+      static_assert(std::is_same_v<decltype(pri), std::priority_queue<T, Cont, Comp>>);
+    }
+
+    {
+      Comp comp;
+      Cont cont;
+      std::priority_queue pri(comp, cont, ConvertibleToAlloc(2));
+      static_assert(std::is_same_v<decltype(pri), std::priority_queue<T, Cont, Comp>>);
     }
 
     {
-        typedef short T;
-        typedef std::greater<T> Comp;
-        typedef test_allocator<T> Alloc;
-        typedef std::deque<T, Alloc> Cont;
-        typedef test_allocator<int> ConvertibleToAlloc;
-        static_assert(std::uses_allocator_v<Cont, ConvertibleToAlloc> &&
-                      !std::is_same_v<typename Cont::allocator_type, ConvertibleToAlloc>);
-
-        {
-        Comp comp;
-        Cont cont;
-        std::priority_queue pri(comp, cont, Alloc(2));
-        static_assert(std::is_same_v<decltype(pri), std::priority_queue<T, Cont, Comp>>);
-        }
-
-        {
-        Comp comp;
-        Cont cont;
-        std::priority_queue pri(comp, cont, ConvertibleToAlloc(2));
-        static_assert(std::is_same_v<decltype(pri), std::priority_queue<T, Cont, Comp>>);
-        }
-
-        {
-        Comp comp;
-        Cont cont;
-        std::priority_queue pri(comp, std::move(cont), Alloc(2));
-        static_assert(std::is_same_v<decltype(pri), std::priority_queue<T, Cont, Comp>>);
-        }
-
-        {
-        Comp comp;
-        Cont cont;
-        std::priority_queue pri(comp, std::move(cont), ConvertibleToAlloc(2));
-        static_assert(std::is_same_v<decltype(pri), std::priority_queue<T, Cont, Comp>>);
-        }
+      Comp comp;
+      Cont cont;
+      std::priority_queue pri(comp, std::move(cont), Alloc(2));
+      static_assert(std::is_same_v<decltype(pri), std::priority_queue<T, Cont, Comp>>);
     }
 
     {
-        typedef short T;
-        typedef signed char ConvertibleToT;
-        typedef std::greater<T> Comp;
-        typedef test_allocator<T> Alloc;
-        typedef std::deque<T, Alloc> Cont;
-        typedef test_allocator<int> ConvertibleToAlloc;
-        static_assert(std::uses_allocator_v<Cont, ConvertibleToAlloc> &&
-                      !std::is_same_v<typename Cont::allocator_type, ConvertibleToAlloc>);
-
-        {
-        std::priority_queue<T, Cont, Comp> source;
-        std::priority_queue pri(source, Alloc(2));
-        static_assert(std::is_same_v<decltype(pri), std::priority_queue<T, Cont, Comp>>);
-        }
-
-        {
-        std::priority_queue<T, Cont, Comp> source;
-        std::priority_queue pri(source, ConvertibleToAlloc(2));
-        static_assert(std::is_same_v<decltype(pri), std::priority_queue<T, Cont, Comp>>);
-        }
-
-        {
-        std::priority_queue<T, Cont, Comp> source;
-        std::priority_queue pri(std::move(source), Alloc(2));
-        static_assert(std::is_same_v<decltype(pri), std::priority_queue<T, Cont, Comp>>);
-        }
-
-        {
-        std::priority_queue<T, Cont, Comp> source;
-        std::priority_queue pri(std::move(source), ConvertibleToAlloc(2));
-        static_assert(std::is_same_v<decltype(pri), std::priority_queue<T, Cont, Comp>>);
-        }
-
-        {
-        Cont cont;
-        std::priority_queue pri(Comp(), cont, Alloc(2));
-        static_assert(std::is_same_v<decltype(pri), std::priority_queue<T, Cont, Comp>>);
-        }
-
-        {
-        Cont cont;
-        std::priority_queue pri(Comp(), cont, ConvertibleToAlloc(2));
-        static_assert(std::is_same_v<decltype(pri), std::priority_queue<T, Cont, Comp>>);
-        }
-
-        {
-        Cont cont;
-        std::priority_queue pri(Comp(), std::move(cont), Alloc(2));
-        static_assert(std::is_same_v<decltype(pri), std::priority_queue<T, Cont, Comp>>);
-        }
-
-        {
-        Cont cont;
-        std::priority_queue pri(Comp(), std::move(cont), ConvertibleToAlloc(2));
-        static_assert(std::is_same_v<decltype(pri), std::priority_queue<T, Cont, Comp>>);
-        }
-
-        {
-        T a[2] = {};
-        std::priority_queue pri(a, a+2, Alloc(2));
-        static_assert(std::is_same_v<decltype(pri), std::priority_queue<T, std::vector<T, Alloc>>>);
-        }
-
-        {
-        T a[2] = {};
-        std::priority_queue pri(a, a+2, Comp(), Alloc(2));
-        static_assert(std::is_same_v<decltype(pri), std::priority_queue<T, std::vector<T, Alloc>, Comp>>);
-        }
-
-        {
-        Cont cont;
-        ConvertibleToT a[2] = {};
-        std::priority_queue pri(a, a+2, Comp(), cont, Alloc(2));
-        static_assert(std::is_same_v<decltype(pri), std::priority_queue<T, Cont, Comp>>);
-        }
-
-        {
-        Cont cont;
-        ConvertibleToT a[2] = {};
-        std::priority_queue pri(a, a+2, Comp(), cont, ConvertibleToAlloc(2));
-        static_assert(std::is_same_v<decltype(pri), std::priority_queue<T, Cont, Comp>>);
-        }
-
-        {
-        Cont cont;
-        ConvertibleToT a[2] = {};
-        std::priority_queue pri(a, a+2, Comp(), std::move(cont), Alloc(2));
-        static_assert(std::is_same_v<decltype(pri), std::priority_queue<T, Cont, Comp>>);
-        }
-
-        {
-        Cont cont;
-        ConvertibleToT a[2] = {};
-        std::priority_queue pri(a, a+2, Comp(), std::move(cont), ConvertibleToAlloc(2));
-        static_assert(std::is_same_v<decltype(pri), std::priority_queue<T, Cont, Comp>>);
-        }
+      Comp comp;
+      Cont cont;
+      std::priority_queue pri(comp, std::move(cont), ConvertibleToAlloc(2));
+      static_assert(std::is_same_v<decltype(pri), std::priority_queue<T, Cont, Comp>>);
+    }
+  }
+
+  {
+    typedef short T;
+    typedef signed char ConvertibleToT;
+    typedef std::greater<T> Comp;
+    typedef test_allocator<T> Alloc;
+    typedef std::deque<T, Alloc> Cont;
+    typedef test_allocator<int> ConvertibleToAlloc;
+    static_assert(std::uses_allocator_v<Cont, ConvertibleToAlloc> &&
+                  !std::is_same_v<typename Cont::allocator_type, ConvertibleToAlloc>);
 
-#if TEST_STD_VER >= 23
-        { // (from_range, range)
-          std::priority_queue c(std::from_range, Cont());
-          static_assert(std::is_same_v<decltype(c), std::priority_queue<T>>);
-        }
-
-        { // (from_range, range, compare)
-          std::priority_queue c(std::from_range, Cont(), Comp());
-          static_assert(std::is_same_v<decltype(c), std::priority_queue<T, std::vector<T>, Comp>>);
-        }
-
-        { // (from_range, range, compare, alloc)
-          std::priority_queue c(std::from_range, Cont(), Comp(), Alloc(2));
-          static_assert(std::is_same_v<decltype(c), std::priority_queue<T, std::vector<T, Alloc>, Comp>>);
-        }
-
-        { // (from_range, range, alloc)
-          std::priority_queue c(std::from_range, Cont(), Alloc(2));
-          static_assert(std::is_same_v<decltype(c), std::priority_queue<T, std::vector<T, Alloc>>>);
-        }
-#endif // TEST_STD_VER >= 23
+    {
+      std::priority_queue<T, Cont, Comp> source;
+      std::priority_queue pri(source, Alloc(2));
+      static_assert(std::is_same_v<decltype(pri), std::priority_queue<T, Cont, Comp>>);
+    }
+
+    {
+      std::priority_queue<T, Cont, Comp> source;
+      std::priority_queue pri(source, ConvertibleToAlloc(2));
+      static_assert(std::is_same_v<decltype(pri), std::priority_queue<T, Cont, Comp>>);
+    }
+
+    {
+      std::priority_queue<T, Cont, Comp> source;
+      std::priority_queue pri(std::move(source), Alloc(2));
+      static_assert(std::is_same_v<decltype(pri), std::priority_queue<T, Cont, Comp>>);
+    }
+
+    {
+      std::priority_queue<T, Cont, Comp> source;
+      std::priority_queue pri(std::move(source), ConvertibleToAlloc(2));
+      static_assert(std::is_same_v<decltype(pri), std::priority_queue<T, Cont, Comp>>);
+    }
+
+    {
+      Cont cont;
+      std::priority_queue pri(Comp(), cont, Alloc(2));
+      static_assert(std::is_same_v<decltype(pri), std::priority_queue<T, Cont, Comp>>);
     }
 
-    // Deduction guides should be SFINAE'd away when given:
-    // - "bad" input iterators (that is, a type not qualifying as an input
-    //   iterator);
-    // - a bad allocator;
-    // - an allocator instead of a comparator;
-    // - an allocator instead of a container;
-    // - an allocator and a container that uses a different allocator.
     {
-        using Comp = std::less<int>;
-        using Cont = std::vector<int>;
-        using Alloc = std::allocator<int>;
-        using Iter = int*;
-
-        // The only requirement in the Standard is that integral types cannot be
-        // considered input iterators, beyond that it is unspecified.
-        using BadIter = int;
+      Cont cont;
+      std::priority_queue pri(Comp(), cont, ConvertibleToAlloc(2));
+      static_assert(std::is_same_v<decltype(pri), std::priority_queue<T, Cont, Comp>>);
+    }
+
+    {
+      Cont cont;
+      std::priority_queue pri(Comp(), std::move(cont), Alloc(2));
+      static_assert(std::is_same_v<decltype(pri), std::priority_queue<T, Cont, Comp>>);
+    }
+
+    {
+      Cont cont;
+      std::priority_queue pri(Comp(), std::move(cont), ConvertibleToAlloc(2));
+      static_assert(std::is_same_v<decltype(pri), std::priority_queue<T, Cont, Comp>>);
+    }
+
+    {
+      T a[2] = {};
+      std::priority_queue pri(a, a + 2, Alloc(2));
+      static_assert(std::is_same_v<decltype(pri), std::priority_queue<T, std::vector<T, Alloc>>>);
+    }
+
+    {
+      T a[2] = {};
+      std::priority_queue pri(a, a + 2, Comp(), Alloc(2));
+      static_assert(std::is_same_v<decltype(pri), std::priority_queue<T, std::vector<T, Alloc>, Comp>>);
+    }
+
+    {
+      Cont cont;
+      ConvertibleToT a[2] = {};
+      std::priority_queue pri(a, a + 2, Comp(), cont, Alloc(2));
+      static_assert(std::is_same_v<decltype(pri), std::priority_queue<T, Cont, Comp>>);
+    }
+
+    {
+      Cont cont;
+      ConvertibleToT a[2] = {};
+      std::priority_queue pri(a, a + 2, Comp(), cont, ConvertibleToAlloc(2));
+      static_assert(std::is_same_v<decltype(pri), std::priority_queue<T, Cont, Comp>>);
+    }
+
+    {
+      Cont cont;
+      ConvertibleToT a[2] = {};
+      std::priority_queue pri(a, a + 2, Comp(), std::move(cont), Alloc(2));
+      static_assert(std::is_same_v<decltype(pri), std::priority_queue<T, Cont, Comp>>);
+    }
+
+    {
+      Cont cont;
+      ConvertibleToT a[2] = {};
+      std::priority_queue pri(a, a + 2, Comp(), std::move(cont), ConvertibleToAlloc(2));
+      static_assert(std::is_same_v<decltype(pri), std::priority_queue<T, Cont, Comp>>);
+    }
+
+#if TEST_STD_VER >= 23
+    { // (from_range, range)
+      std::priority_queue c(std::from_range, Cont());
+      static_assert(std::is_same_v<decltype(c), std::priority_queue<T>>);
+    }
+
+    { // (from_range, range, compare)
+      std::priority_queue c(std::from_range, Cont(), Comp());
+      static_assert(std::is_same_v<decltype(c), std::priority_queue<T, std::vector<T>, Comp>>);
+    }
+
+    { // (from_range, range, compare, alloc)
+      std::priority_queue c(std::from_range, Cont(), Comp(), Alloc(2));
+      static_assert(std::is_same_v<decltype(c), std::priority_queue<T, std::vector<T, Alloc>, Comp>>);
+    }
+
+    { // (from_range, range, alloc)
+      std::priority_queue c(std::from_range, Cont(), Alloc(2));
+      static_assert(std::is_same_v<decltype(c), std::priority_queue<T, std::vector<T, Alloc>>>);
+    }
+#endif // TEST_STD_VER >= 23
+  }
+
+  // Deduction guides should be SFINAE'd away when given:
+  // - "bad" input iterators (that is, a type not qualifying as an input
+  //   iterator);
+  // - a bad allocator;
+  // - an allocator instead of a comparator;
+  // - an allocator instead of a container;
+  // - an allocator and a container that uses a different allocator.
+  {
+    using Comp  = std::less<int>;
+    using Cont  = std::vector<int>;
+    using Alloc = std::allocator<int>;
+    using Iter  = int*;
+
+    // The only requirement in the Standard is that integral types cannot be
+    // considered input iterators, beyond that it is unspecified.
+    using BadIter = int;
 #ifdef _LIBCPP_VERSION
-        struct OutputIter {
-          using iterator_category = std::output_iterator_tag;
-          using value_type = void;
-          using difference_type = void;
-          using pointer = void;
-          using reference = void;
-
-          const OutputIter& operator*() const { return *this; }
-          const OutputIter& operator++() { return *this; }
-          OutputIter operator++(int) const { return *this; }
-        };
+    struct OutputIter {
+      using iterator_category = std::output_iterator_tag;
+      using value_type        = void;
+      using difference_type   = void;
+      using pointer           = void;
+      using reference         = void;
+
+      const OutputIter& operator*() const { return *this; }
+      const OutputIter& operator++() { return *this; }
+      OutputIter operator++(int) const { return *this; }
+    };
 #endif // _LIBCPP_VERSION
 
-        struct BadAlloc {};
-        using AllocAsComp = Alloc;
-        using AllocAsCont = Alloc;
-        using DiffAlloc = test_allocator<int>;
-
-        // (iter, iter)
-        //
-        // Cannot deduce from (BAD_iter, BAD_iter)
-        static_assert(SFINAEs_away<std::priority_queue, BadIter, BadIter>);
-        // Note: (OutputIter, OutputIter) is interpreted as (comp, cont) and fails on accessing
-        // non-existent typedefs in `OutputIter` (as if it were a container). There is no
-        // requirement to SFINAE away bad containers.
-
-        // (iter, iter, comp)
-        //
-        // Cannot deduce from (BAD_iter, BAD_iter, comp)
-        static_assert(SFINAEs_away<std::priority_queue, BadIter, BadIter, Comp>);
-        LIBCPP_STATIC_ASSERT(SFINAEs_away<std::priority_queue, OutputIter, OutputIter, Comp>);
-        // Note: (iter, iter, ALLOC_as_comp) is allowed -- it just calls (iter, iter, alloc).
-
-        // (iter, iter, comp, cont)
-        //
-        // Cannot deduce from (BAD_iter, BAD_iter, comp, cont)
-        static_assert(SFINAEs_away<std::priority_queue, BadIter, BadIter, Comp, Cont>);
-        LIBCPP_STATIC_ASSERT(SFINAEs_away<std::priority_queue, OutputIter, OutputIter, Comp, Cont>);
-        // Cannot deduce from (iter, iter, ALLOC_as_comp, cont)
-        static_assert(SFINAEs_away<std::priority_queue, Iter, Iter, AllocAsComp, Cont>);
-        // Note: (iter, iter, comp, ALLOC_as_cont) is allowed -- it just calls (iter, iter, comp,
-        // alloc).
-
-        // (iter, iter, alloc)
-        //
-        // Cannot deduce from (BAD_iter, BAD_iter, alloc)
-        static_assert(SFINAEs_away<std::priority_queue, BadIter, BadIter, Alloc>);
-        LIBCPP_STATIC_ASSERT(SFINAEs_away<std::priority_queue, OutputIter, OutputIter, Alloc>);
-        // Note: (iter, iter, BAD_alloc) is interpreted as (iter, iter, comp) instead and fails upon
-        // instantiation. There is no requirement to SFINAE away bad comparators.
-
-        // (iter, iter, comp, alloc)
-        //
-        // Cannot deduce from (iter, iter, ALLOC_as_comp, alloc)
-        static_assert(SFINAEs_away<std::priority_queue, Iter, Iter, AllocAsComp, Alloc>);
-        // Note: (iter, iter, comp, BAD_alloc) is interpreted as (iter, iter, comp, cont) instead
-        // and fails upon instantiation. There is no requirement to SFINAE away bad containers.
-
-        // (iter, iter, comp, cont, alloc)
-        //
-        // Cannot deduce from (BAD_iter, BAD_iter, comp, cont, alloc)
-        static_assert(SFINAEs_away<std::priority_queue, BadIter, BadIter, Comp, Cont, Alloc>);
-        LIBCPP_STATIC_ASSERT(
-            SFINAEs_away<std::priority_queue, OutputIter, OutputIter, Comp, Cont, Alloc>);
-        // Cannot deduce from (iter, iter, ALLOC_as_comp, cont, alloc)
-        static_assert(SFINAEs_away<std::priority_queue, Iter, Iter, AllocAsComp, Cont, Alloc>);
-        // Cannot deduce from (iter, iter, comp, ALLOC_as_cont, alloc)
-        static_assert(SFINAEs_away<std::priority_queue, Iter, Iter, Comp, AllocAsCont, Alloc>);
-        // Cannot deduce from (iter, iter, comp, cont, BAD_alloc)
-        static_assert(SFINAEs_away<std::priority_queue, Iter, Iter, Comp, Cont, BadAlloc>);
-        // Cannot deduce from (iter, iter, comp, cont, DIFFERENT_alloc)
-        static_assert(SFINAEs_away<std::priority_queue, Iter, Iter, Comp, Cont, DiffAlloc>);
-
-        // (comp, alloc)
-        //
-        // Cannot deduce from (ALLOC_as_comp, alloc)
-        static_assert(SFINAEs_away<std::priority_queue, AllocAsComp, Alloc>);
-        // Cannot deduce from (comp, BAD_alloc)
-        static_assert(SFINAEs_away<std::priority_queue, Comp, BadAlloc>);
-
-        // (comp, cont, alloc)
-        //
-        // Cannot deduce from (ALLOC_as_comp, cont, alloc)
-        static_assert(SFINAEs_away<std::priority_queue, AllocAsComp, Cont, Alloc>);
-        // Cannot deduce from (comp, ALLOC_as_cont, alloc)
-        static_assert(SFINAEs_away<std::priority_queue, Comp, AllocAsCont, Alloc>);
-        // Cannot deduce from (comp, cont, BAD_alloc)
-        static_assert(SFINAEs_away<std::priority_queue, Comp, Cont, BadAlloc>);
-        // Cannot deduce from (comp, cont, DIFFERENT_alloc)
-        static_assert(SFINAEs_away<std::priority_queue, Comp, Cont, DiffAlloc>);
-
-        // (comp, cont)
-        //
-        // Cannot deduce from (ALLOC_as_comp, cont)
-        static_assert(SFINAEs_away<std::priority_queue, AllocAsComp, Cont>);
-        // Cannot deduce from (comp, ALLOC_as_cont)
-        static_assert(SFINAEs_away<std::priority_queue, Comp, AllocAsCont>);
+    struct BadAlloc {};
+    using AllocAsComp = Alloc;
+    using AllocAsCont = Alloc;
+    using DiffAlloc   = test_allocator<int>;
+
+    // (iter, iter)
+    //
+    // Cannot deduce from (BAD_iter, BAD_iter)
+    static_assert(SFINAEs_away<std::priority_queue, BadIter, BadIter>);
+    // Note: (OutputIter, OutputIter) is interpreted as (comp, cont) and fails on accessing
+    // non-existent typedefs in `OutputIter` (as if it were a container). There is no
+    // requirement to SFINAE away bad containers.
+
+    // (iter, iter, comp)
+    //
+    // Cannot deduce from (BAD_iter, BAD_iter, comp)
+    static_assert(SFINAEs_away<std::priority_queue, BadIter, BadIter, Comp>);
+    LIBCPP_STATIC_ASSERT(SFINAEs_away<std::priority_queue, OutputIter, OutputIter, Comp>);
+    // Note: (iter, iter, ALLOC_as_comp) is allowed -- it just calls (iter, iter, alloc).
+
+    // (iter, iter, comp, cont)
+    //
+    // Cannot deduce from (BAD_iter, BAD_iter, comp, cont)
+    static_assert(SFINAEs_away<std::priority_queue, BadIter, BadIter, Comp, Cont>);
+    LIBCPP_STATIC_ASSERT(SFINAEs_away<std::priority_queue, OutputIter, OutputIter, Comp, Cont>);
+    // Cannot deduce from (iter, iter, ALLOC_as_comp, cont)
+    static_assert(SFINAEs_away<std::priority_queue, Iter, Iter, AllocAsComp, Cont>);
+    // Note: (iter, iter, comp, ALLOC_as_cont) is allowed -- it just calls (iter, iter, comp,
+    // alloc).
+
+    // (iter, iter, alloc)
+    //
+    // Cannot deduce from (BAD_iter, BAD_iter, alloc)
+    static_assert(SFINAEs_away<std::priority_queue, BadIter, BadIter, Alloc>);
+    LIBCPP_STATIC_ASSERT(SFINAEs_away<std::priority_queue, OutputIter, OutputIter, Alloc>);
+    // Note: (iter, iter, BAD_alloc) is interpreted as (iter, iter, comp) instead and fails upon
+    // instantiation. There is no requirement to SFINAE away bad comparators.
+
+    // (iter, iter, comp, alloc)
+    //
+    // Cannot deduce from (iter, iter, ALLOC_as_comp, alloc)
+    static_assert(SFINAEs_away<std::priority_queue, Iter, Iter, AllocAsComp, Alloc>);
+    // Note: (iter, iter, comp, BAD_alloc) is interpreted as (iter, iter, comp, cont) instead
+    // and fails upon instantiation. There is no requirement to SFINAE away bad containers.
+
+    // (iter, iter, comp, cont, alloc)
+    //
+    // Cannot deduce from (BAD_iter, BAD_iter, comp, cont, alloc)
+    static_assert(SFINAEs_away<std::priority_queue, BadIter, BadIter, Comp, Cont, Alloc>);
+    LIBCPP_STATIC_ASSERT(SFINAEs_away<std::priority_queue, OutputIter, OutputIter, Comp, Cont, Alloc>);
+    // Cannot deduce from (iter, iter, ALLOC_as_comp, cont, alloc)
+    static_assert(SFINAEs_away<std::priority_queue, Iter, Iter, AllocAsComp, Cont, Alloc>);
+    // Cannot deduce from (iter, iter, comp, ALLOC_as_cont, alloc)
+    static_assert(SFINAEs_away<std::priority_queue, Iter, Iter, Comp, AllocAsCont, Alloc>);
+    // Cannot deduce from (iter, iter, comp, cont, BAD_alloc)
+    static_assert(SFINAEs_away<std::priority_queue, Iter, Iter, Comp, Cont, BadAlloc>);
+    // Cannot deduce from (iter, iter, comp, cont, DIFFERENT_alloc)
+    static_assert(SFINAEs_away<std::priority_queue, Iter, Iter, Comp, Cont, DiffAlloc>);
+
+    // (comp, alloc)
+    //
+    // Cannot deduce from (ALLOC_as_comp, alloc)
+    static_assert(SFINAEs_away<std::priority_queue, AllocAsComp, Alloc>);
+    // Cannot deduce from (comp, BAD_alloc)
+    static_assert(SFINAEs_away<std::priority_queue, Comp, BadAlloc>);
+
+    // (comp, cont, alloc)
+    //
+    // Cannot deduce from (ALLOC_as_comp, cont, alloc)
+    static_assert(SFINAEs_away<std::priority_queue, AllocAsComp, Cont, Alloc>);
+    // Cannot deduce from (comp, ALLOC_as_cont, alloc)
+    static_assert(SFINAEs_away<std::priority_queue, Comp, AllocAsCont, Alloc>);
+    // Cannot deduce from (comp, cont, BAD_alloc)
+    static_assert(SFINAEs_away<std::priority_queue, Comp, Cont, BadAlloc>);
+    // Cannot deduce from (comp, cont, DIFFERENT_alloc)
+    static_assert(SFINAEs_away<std::priority_queue, Comp, Cont, DiffAlloc>);
+
+    // (comp, cont)
+    //
+    // Cannot deduce from (ALLOC_as_comp, cont)
+    static_assert(SFINAEs_away<std::priority_queue, AllocAsComp, Cont>);
+    // Cannot deduce from (comp, ALLOC_as_cont)
+    static_assert(SFINAEs_away<std::priority_queue, Comp, AllocAsCont>);
 
 #if TEST_STD_VER >= 23
-        using Range = RangeT<int>;
-        using BadRange = BadRangeT<int>;
-
-        // (from_range, range)
-        //
-        // Cannot deduce from (from_range, BAD_range)
-        static_assert(SFINAEs_away<std::priority_queue, BadRange>);
-
-        // (from_range, range, compare)
-        //
-        // Cannot deduce from (from_range, BAD_range, compare)
-        static_assert(SFINAEs_away<std::priority_queue, BadRange, Comp>);
-
-        // (from_range, range, compare, alloc)
-        //
-        // Cannot deduce from (from_range, BAD_range, compare, alloc)
-        static_assert(SFINAEs_away<std::priority_queue, BadRange, Comp, Alloc>);
-        // Cannot deduce from (from_range, range, compare, BAD_alloc)
-        static_assert(SFINAEs_away<std::priority_queue, Range, Comp, BadAlloc>);
-        // Cannot deduce from (from_range, range, compare, DIFFERENT_alloc)
-        static_assert(SFINAEs_away<std::priority_queue, Range, Comp, DiffAlloc>);
-
-        // (from_range, range, alloc)
-        //
-        // Cannot deduce from (from_range, BAD_range, alloc)
-        static_assert(SFINAEs_away<std::priority_queue, BadRange, Alloc>);
-        // Cannot deduce from (from_range, range, BAD_alloc)
-        static_assert(SFINAEs_away<std::priority_queue, Range, BadAlloc>);
-        // Cannot deduce from (from_range, range, DIFFERENT_alloc)
-        static_assert(SFINAEs_away<std::priority_queue, Range, DiffAlloc>);
+    using Range    = RangeT<int>;
+    using BadRange = BadRangeT<int>;
+
+    // (from_range, range)
+    //
+    // Cannot deduce from (from_range, BAD_range)
+    static_assert(SFINAEs_away<std::priority_queue, BadRange>);
+
+    // (from_range, range, compare)
+    //
+    // Cannot deduce from (from_range, BAD_range, compare)
+    static_assert(SFINAEs_away<std::priority_queue, BadRange, Comp>);
+
+    // (from_range, range, compare, alloc)
+    //
+    // Cannot deduce from (from_range, BAD_range, compare, alloc)
+    static_assert(SFINAEs_away<std::priority_queue, BadRange, Comp, Alloc>);
+    // Cannot deduce from (from_range, range, compare, BAD_alloc)
+    static_assert(SFINAEs_away<std::priority_queue, Range, Comp, BadAlloc>);
+    // Cannot deduce from (from_range, range, compare, DIFFERENT_alloc)
+    static_assert(SFINAEs_away<std::priority_queue, Range, Comp, DiffAlloc>);
+
+    // (from_range, range, alloc)
+    //
+    // Cannot deduce from (from_range, BAD_range, alloc)
+    static_assert(SFINAEs_away<std::priority_queue, BadRange, Alloc>);
+    // Cannot deduce from (from_range, range, BAD_alloc)
+    static_assert(SFINAEs_away<std::priority_queue, Range, BadAlloc>);
+    // Cannot deduce from (from_range, range, DIFFERENT_alloc)
+    static_assert(SFINAEs_away<std::priority_queue, Range, DiffAlloc>);
 #endif // TEST_STD_VER >= 23
-    }
+  }
 
-    return 0;
+  return 0;
 }
diff --git a/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.cons/deduct.verify.cpp b/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.cons/deduct.verify.cpp
index 73487597ca56e..057b06b4d3fd7 100644
--- a/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.cons/deduct.verify.cpp
+++ b/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.cons/deduct.verify.cpp
@@ -15,44 +15,46 @@
 #include <cassert>
 #include <cstddef>
 
-
-int main(int, char**)
-{
-//  Test the explicit deduction guides
-    {
-//  queue(Compare, Container, const Alloc);
-//  The '45' is not an allocator
-    std::priority_queue pri(std::greater<int>(), std::deque<int>({1,2,3}), 45);  // expected-error-re {{no viable constructor or deduction guide for deduction of template arguments of '{{(std::)?}}priority_queue'}}
-    }
-
-    {
-//  queue(const queue&, const Alloc&);
-//  The '45' is not an allocator
+int main(int, char**) {
+  //  Test the explicit deduction guides
+  {
+    //  queue(Compare, Container, const Alloc);
+    //  The '45' is not an allocator
+    std::priority_queue pri(std::greater<int>(), std::deque<int>({1, 2, 3}), 45);
+    // expected-error-re@-1 {{no viable constructor or deduction guide for deduction of template arguments of '{{(std::)?}}priority_queue'}}
+  }
+
+  {
+    //  queue(const queue&, const Alloc&);
+    //  The '45' is not an allocator
     std::priority_queue<int> source;
-    std::priority_queue pri(source, 45);  // expected-error-re {{no viable constructor or deduction guide for deduction of template arguments of '{{(std::)?}}priority_queue'}}
-    }
-
-    {
-//  priority_queue(Iter, Iter, Comp)
-//  int is not an iterator
-    std::priority_queue pri(15, 17, std::greater<double>());  // expected-error-re {{no viable constructor or deduction guide for deduction of template arguments of '{{(std::)?}}priority_queue'}}
-    }
-
-    {
-//  priority_queue(Iter, Iter, Comp, Container)
-//  float is not an iterator
-    std::priority_queue pri(23.f, 2.f, std::greater<float>(), std::deque<float>());   // expected-error-re {{no viable constructor or deduction guide for deduction of template arguments of '{{(std::)?}}priority_queue'}}
-    }
-
-//  Test the implicit deduction guides
-    {
-//  priority_queue (allocator &)
-    std::priority_queue pri((std::allocator<int>()));  // expected-error-re {{no viable constructor or deduction guide for deduction of template arguments of '{{(std::)?}}priority_queue'}}
-//  Note: The extra parens are necessary, since otherwise clang decides it is a function declaration.
-//  Also, we can't use {} instead of parens, because that constructs a
-//      stack<allocator<int>, allocator<allocator<int>>>
-    }
-
+    std::priority_queue pri(source, 45);
+    // expected-error-re@-1 {{no viable constructor or deduction guide for deduction of template arguments of '{{(std::)?}}priority_queue'}}
+  }
+
+  {
+    //  priority_queue(Iter, Iter, Comp)
+    //  int is not an iterator
+    std::priority_queue pri(15, 17, std::greater< double>());
+    // expected-error-re@-1 {{no viable constructor or deduction guide for deduction of template arguments of '{{(std::)?}}priority_queue'}}
+  }
+
+  {
+    //  priority_queue(Iter, Iter, Comp, Container)
+    //  float is not an iterator
+    std::priority_queue pri(23.f, 2.f, std::greater<float>(), std::deque< float>());
+    // expected-error-re@-1 {{no viable constructor or deduction guide for deduction of template arguments of '{{(std::)?}}priority_queue'}}
+  }
+
+  //  Test the implicit deduction guides
+  {
+    //  priority_queue (allocator &)
+    std::priority_queue pri((std::allocator< int>()));
+    // expected-error-re@-1 {{no viable constructor or deduction guide for deduction of template arguments of '{{(std::)?}}priority_queue'}}
+    //  Note: The extra parens are necessary, since otherwise clang decides it is a function declaration.
+    //  Also, we can't use {} instead of parens, because that constructs a
+    //      stack<allocator<int>, allocator<allocator<int>>>
+  }
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.cons/default_noexcept.pass.cpp b/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.cons/default_noexcept.pass.cpp
index 80af575a52097..620938b5b7149 100644
--- a/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.cons/default_noexcept.pass.cpp
+++ b/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.cons/default_noexcept.pass.cpp
@@ -16,20 +16,18 @@
 
 // This tests a conforming extension
 
-
 #include <queue>
 #include <cassert>
 
 #include "test_macros.h"
 #include "MoveOnly.h"
 
-int main(int, char**)
-{
+int main(int, char**) {
 #if defined(_LIBCPP_VERSION)
-    {
-        typedef std::priority_queue<MoveOnly> C;
-        static_assert(std::is_nothrow_default_constructible<C>::value, "");
-    }
+  {
+    typedef std::priority_queue<MoveOnly> C;
+    static_assert(std::is_nothrow_default_constructible<C>::value, "");
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.cons/dtor_noexcept.pass.cpp b/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.cons/dtor_noexcept.pass.cpp
index 147e70ef27b95..d292dd24312dd 100644
--- a/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.cons/dtor_noexcept.pass.cpp
+++ b/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.cons/dtor_noexcept.pass.cpp
@@ -19,12 +19,11 @@
 #include "test_macros.h"
 #include "MoveOnly.h"
 
-int main(int, char**)
-{
-    {
-        typedef std::priority_queue<MoveOnly> C;
-        static_assert(std::is_nothrow_destructible<C>::value, "");
-    }
+int main(int, char**) {
+  {
+    typedef std::priority_queue<MoveOnly> C;
+    static_assert(std::is_nothrow_destructible<C>::value, "");
+  }
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.cons/from_range.pass.cpp b/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.cons/from_range.pass.cpp
index de453f72dc8b3..7d3cc2eb6b343 100644
--- a/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.cons/from_range.pass.cpp
+++ b/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.cons/from_range.pass.cpp
@@ -22,7 +22,7 @@
 //   priority_queue(from_range_t, R&& rg, const Alloc&); // since C++23
 
 template <class Range>
-concept PriorityQueueHasFromRangeCtr = requires (Range&& range) {
+concept PriorityQueueHasFromRangeCtr = requires(Range&& range) {
   std::priority_queue<int>(std::from_range, std::forward<Range>(range));
   std::priority_queue<int>(std::from_range, std::forward<Range>(range), std::less<int>());
   std::priority_queue<int>(std::from_range, std::forward<Range>(range), std::less<int>(), std::allocator<int>());
diff --git a/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.cons/move_assign_noexcept.pass.cpp b/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.cons/move_assign_noexcept.pass.cpp
index b24a5e77b27f7..18412d705c9c0 100644
--- a/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.cons/move_assign_noexcept.pass.cpp
+++ b/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.cons/move_assign_noexcept.pass.cpp
@@ -22,12 +22,11 @@
 #include "test_macros.h"
 #include "MoveOnly.h"
 
-int main(int, char**)
-{
-    {
-        typedef std::priority_queue<MoveOnly> C;
-        static_assert(std::is_nothrow_move_assignable<C>::value, "");
-    }
+int main(int, char**) {
+  {
+    typedef std::priority_queue<MoveOnly> C;
+    static_assert(std::is_nothrow_move_assignable<C>::value, "");
+  }
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.cons/move_noexcept.pass.cpp b/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.cons/move_noexcept.pass.cpp
index 582556cea24f0..8dd6de7480b88 100644
--- a/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.cons/move_noexcept.pass.cpp
+++ b/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.cons/move_noexcept.pass.cpp
@@ -22,12 +22,11 @@
 #include "test_macros.h"
 #include "MoveOnly.h"
 
-int main(int, char**)
-{
-    {
-        typedef std::priority_queue<MoveOnly> C;
-        static_assert(std::is_nothrow_move_constructible<C>::value, "");
-    }
+int main(int, char**) {
+  {
+    typedef std::priority_queue<MoveOnly> C;
+    static_assert(std::is_nothrow_move_constructible<C>::value, "");
+  }
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.members/emplace.pass.cpp b/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.members/emplace.pass.cpp
index 4d2c716583e25..f9f88fd843082 100644
--- a/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.members/emplace.pass.cpp
+++ b/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.members/emplace.pass.cpp
@@ -20,15 +20,14 @@
 #include "test_macros.h"
 #include "../../../Emplaceable.h"
 
-int main(int, char**)
-{
-    std::priority_queue<Emplaceable> q;
-    q.emplace(1, 2.5);
-    assert(q.top() == Emplaceable(1, 2.5));
-    q.emplace(3, 4.5);
-    assert(q.top() == Emplaceable(3, 4.5));
-    q.emplace(2, 3.5);
-    assert(q.top() == Emplaceable(3, 4.5));
+int main(int, char**) {
+  std::priority_queue<Emplaceable> q;
+  q.emplace(1, 2.5);
+  assert(q.top() == Emplaceable(1, 2.5));
+  q.emplace(3, 4.5);
+  assert(q.top() == Emplaceable(3, 4.5));
+  q.emplace(2, 3.5);
+  assert(q.top() == Emplaceable(3, 4.5));
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.members/empty.pass.cpp b/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.members/empty.pass.cpp
index 9629bd9b16de7..758c5354aea4d 100644
--- a/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.members/empty.pass.cpp
+++ b/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.members/empty.pass.cpp
@@ -17,14 +17,13 @@
 
 #include "test_macros.h"
 
-int main(int, char**)
-{
-    std::priority_queue<int> q;
-    assert(q.empty());
-    q.push(1);
-    assert(!q.empty());
-    q.pop();
-    assert(q.empty());
+int main(int, char**) {
+  std::priority_queue<int> q;
+  assert(q.empty());
+  q.push(1);
+  assert(!q.empty());
+  q.pop();
+  assert(q.empty());
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.members/empty.verify.cpp b/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.members/empty.verify.cpp
index 793f47d12c20a..5202710846e8b 100644
--- a/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.members/empty.verify.cpp
+++ b/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.members/empty.verify.cpp
@@ -17,6 +17,6 @@
 #include <queue>
 
 void f() {
-    std::priority_queue<int> c;
-    c.empty(); // expected-warning {{ignoring return value of function declared with 'nodiscard' attribute}}
+  std::priority_queue<int> c;
+  c.empty(); // expected-warning {{ignoring return value of function declared with 'nodiscard' attribute}}
 }
diff --git a/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.members/pop.pass.cpp b/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.members/pop.pass.cpp
index fcba8329d7e63..dba6614ad5b69 100644
--- a/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.members/pop.pass.cpp
+++ b/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.members/pop.pass.cpp
@@ -17,21 +17,20 @@
 
 #include "test_macros.h"
 
-int main(int, char**)
-{
-    std::priority_queue<int> q;
-    q.push(1);
-    assert(q.top() == 1);
-    q.push(3);
-    assert(q.top() == 3);
-    q.push(2);
-    assert(q.top() == 3);
-    q.pop();
-    assert(q.top() == 2);
-    q.pop();
-    assert(q.top() == 1);
-    q.pop();
-    assert(q.empty());
+int main(int, char**) {
+  std::priority_queue<int> q;
+  q.push(1);
+  assert(q.top() == 1);
+  q.push(3);
+  assert(q.top() == 3);
+  q.push(2);
+  assert(q.top() == 3);
+  q.pop();
+  assert(q.top() == 2);
+  q.pop();
+  assert(q.top() == 1);
+  q.pop();
+  assert(q.empty());
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.members/push.pass.cpp b/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.members/push.pass.cpp
index 039087c45211c..36ba7d241c5ba 100644
--- a/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.members/push.pass.cpp
+++ b/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.members/push.pass.cpp
@@ -17,15 +17,14 @@
 
 #include "test_macros.h"
 
-int main(int, char**)
-{
-    std::priority_queue<int> q;
-    q.push(1);
-    assert(q.top() == 1);
-    q.push(3);
-    assert(q.top() == 3);
-    q.push(2);
-    assert(q.top() == 3);
+int main(int, char**) {
+  std::priority_queue<int> q;
+  q.push(1);
+  assert(q.top() == 1);
+  q.push(3);
+  assert(q.top() == 3);
+  q.push(2);
+  assert(q.top() == 3);
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.members/push_range.pass.cpp b/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.members/push_range.pass.cpp
index 9a57d930e69d9..26ce56ff5daf6 100644
--- a/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.members/push_range.pass.cpp
+++ b/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.members/push_range.pass.cpp
@@ -30,4 +30,3 @@ int main(int, char**) {
 
   return 0;
 }
-
diff --git a/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.members/push_rvalue.pass.cpp b/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.members/push_rvalue.pass.cpp
index 2f6e7a7b025ef..7700b641a429b 100644
--- a/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.members/push_rvalue.pass.cpp
+++ b/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.members/push_rvalue.pass.cpp
@@ -20,15 +20,14 @@
 #include "test_macros.h"
 #include "MoveOnly.h"
 
-int main(int, char**)
-{
-    std::priority_queue<MoveOnly> q;
-    q.push(1);
-    assert(q.top() == 1);
-    q.push(3);
-    assert(q.top() == 3);
-    q.push(2);
-    assert(q.top() == 3);
+int main(int, char**) {
+  std::priority_queue<MoveOnly> q;
+  q.push(1);
+  assert(q.top() == 1);
+  q.push(3);
+  assert(q.top() == 3);
+  q.push(2);
+  assert(q.top() == 3);
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.members/size.pass.cpp b/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.members/size.pass.cpp
index 6fa21279637d4..2c6b00b87c6ba 100644
--- a/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.members/size.pass.cpp
+++ b/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.members/size.pass.cpp
@@ -17,14 +17,13 @@
 
 #include "test_macros.h"
 
-int main(int, char**)
-{
-    std::priority_queue<int> q;
-    assert(q.size() == 0);
-    q.push(1);
-    assert(q.size() == 1);
-    q.pop();
-    assert(q.size() == 0);
+int main(int, char**) {
+  std::priority_queue<int> q;
+  assert(q.size() == 0);
+  q.push(1);
+  assert(q.size() == 1);
+  q.pop();
+  assert(q.size() == 0);
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.members/swap.pass.cpp b/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.members/swap.pass.cpp
index 295eaf08e0e3a..87e6722d79e35 100644
--- a/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.members/swap.pass.cpp
+++ b/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.members/swap.pass.cpp
@@ -17,17 +17,16 @@
 
 #include "test_macros.h"
 
-int main(int, char**)
-{
-    std::priority_queue<int> q1;
-    std::priority_queue<int> q2;
-    q1.push(1);
-    q1.push(3);
-    q1.push(2);
-    q1.swap(q2);
-    assert(q1.empty());
-    assert(q2.size() == 3);
-    assert(q2.top() == 3);
+int main(int, char**) {
+  std::priority_queue<int> q1;
+  std::priority_queue<int> q2;
+  q1.push(1);
+  q1.push(3);
+  q1.push(2);
+  q1.swap(q2);
+  assert(q1.empty());
+  assert(q2.size() == 3);
+  assert(q2.top() == 3);
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.members/top.pass.cpp b/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.members/top.pass.cpp
index 72153bc532f7f..8b046bc3ccca5 100644
--- a/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.members/top.pass.cpp
+++ b/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.members/top.pass.cpp
@@ -17,15 +17,14 @@
 
 #include "test_macros.h"
 
-int main(int, char**)
-{
-    std::priority_queue<int> q;
-    q.push(1);
-    assert(q.top() == 1);
-    q.push(3);
-    assert(q.top() == 3);
-    q.push(2);
-    assert(q.top() == 3);
+int main(int, char**) {
+  std::priority_queue<int> q;
+  q.push(1);
+  assert(q.top() == 1);
+  q.push(3);
+  assert(q.top() == 3);
+  q.push(2);
+  assert(q.top() == 3);
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.special/swap.pass.cpp b/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.special/swap.pass.cpp
index 1f5b86dc0ecf9..c280601baf17d 100644
--- a/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.special/swap.pass.cpp
+++ b/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.special/swap.pass.cpp
@@ -19,17 +19,16 @@
 
 #include "test_macros.h"
 
-int main(int, char**)
-{
-    std::priority_queue<int> q1;
-    std::priority_queue<int> q2;
-    q1.push(1);
-    q1.push(3);
-    q1.push(2);
-    swap(q1, q2);
-    assert(q1.empty());
-    assert(q2.size() == 3);
-    assert(q2.top() == 3);
+int main(int, char**) {
+  std::priority_queue<int> q1;
+  std::priority_queue<int> q2;
+  q1.push(1);
+  q1.push(3);
+  q1.push(2);
+  swap(q1, q2);
+  assert(q1.empty());
+  assert(q2.size() == 3);
+  assert(q2.top() == 3);
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.special/swap_noexcept.pass.cpp b/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.special/swap_noexcept.pass.cpp
index 579ec0aba7708..7b4678053fcff 100644
--- a/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.special/swap_noexcept.pass.cpp
+++ b/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.special/swap_noexcept.pass.cpp
@@ -23,12 +23,11 @@
 #include "test_macros.h"
 #include "MoveOnly.h"
 
-int main(int, char**)
-{
-    {
-        typedef std::priority_queue<MoveOnly> C;
-        static_assert(noexcept(swap(std::declval<C&>(), std::declval<C&>())), "");
-    }
+int main(int, char**) {
+  {
+    typedef std::priority_queue<MoveOnly> C;
+    static_assert(noexcept(swap(std::declval<C&>(), std::declval<C&>())), "");
+  }
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/container.adaptors/priority.queue/types.compile.fail.cpp b/libcxx/test/std/containers/container.adaptors/priority.queue/types.compile.fail.cpp
index 244028e444d3a..0dfbe06cf1839 100644
--- a/libcxx/test/std/containers/container.adaptors/priority.queue/types.compile.fail.cpp
+++ b/libcxx/test/std/containers/container.adaptors/priority.queue/types.compile.fail.cpp
@@ -27,10 +27,9 @@
 #include <cassert>
 #include <type_traits>
 
-int main(int, char**)
-{
-//  LWG#2566 says that the first template param must match the second one's value type
-    std::priority_queue<double, std::deque<int>> t;
+int main(int, char**) {
+  //  LWG#2566 says that the first template param must match the second one's value type
+  std::priority_queue<double, std::deque<int>> t;
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/container.adaptors/priority.queue/types.pass.cpp b/libcxx/test/std/containers/container.adaptors/priority.queue/types.pass.cpp
index fb801d831e9e8..d90933925a73b 100644
--- a/libcxx/test/std/containers/container.adaptors/priority.queue/types.pass.cpp
+++ b/libcxx/test/std/containers/container.adaptors/priority.queue/types.pass.cpp
@@ -34,38 +34,37 @@
 
 #include "test_macros.h"
 
-struct test
-    : private std::priority_queue<int>
-{
-    test()
-    {
-        c.push_back(1);
-        assert(comp(1, 2));
-    }
+struct test : private std::priority_queue<int> {
+  test() {
+    c.push_back(1);
+    assert(comp(1, 2));
+  }
 };
 
-struct C
-{
-    typedef int value_type;
-    typedef int& reference;
-    typedef const int& const_reference;
-    typedef int size_type;
+struct C {
+  typedef int value_type;
+  typedef int& reference;
+  typedef const int& const_reference;
+  typedef int size_type;
 };
 
-int main(int, char**)
-{
-    static_assert(( std::is_same<std::priority_queue<int>::container_type, std::vector<int> >::value), "");
-    static_assert(( std::is_same<std::priority_queue<int, std::deque<int> >::container_type, std::deque<int> >::value), "");
-    static_assert(( std::is_same<std::priority_queue<int, std::deque<int> >::value_type, int>::value), "");
-    static_assert(( std::is_same<std::priority_queue<int>::reference, std::vector<int>::reference>::value), "");
-    static_assert(( std::is_same<std::priority_queue<int>::const_reference, std::vector<int>::const_reference>::value), "");
-    static_assert(( std::is_same<std::priority_queue<int>::size_type, std::vector<int>::size_type>::value), "");
-    static_assert(( std::is_same<std::priority_queue<int>::value_compare, std::less<int> >::value), "");
-    static_assert(( std::is_same<std::priority_queue<int, std::deque<int> >::value_compare, std::less<int> >::value), "");
-    static_assert(( std::is_same<std::priority_queue<int, std::deque<int>, std::greater<int> >::value_compare, std::greater<int> >::value), "");
-    static_assert(( std::uses_allocator<std::priority_queue<int>, std::allocator<int> >::value), "");
-    static_assert((!std::uses_allocator<std::priority_queue<int, C>, std::allocator<int> >::value), "");
-    test t;
+int main(int, char**) {
+  static_assert((std::is_same<std::priority_queue<int>::container_type, std::vector<int> >::value), "");
+  static_assert(
+      (std::is_same<std::priority_queue<int, std::deque<int> >::container_type, std::deque<int> >::value), "");
+  static_assert((std::is_same<std::priority_queue<int, std::deque<int> >::value_type, int>::value), "");
+  static_assert((std::is_same<std::priority_queue<int>::reference, std::vector<int>::reference>::value), "");
+  static_assert(
+      (std::is_same<std::priority_queue<int>::const_reference, std::vector<int>::const_reference>::value), "");
+  static_assert((std::is_same<std::priority_queue<int>::size_type, std::vector<int>::size_type>::value), "");
+  static_assert((std::is_same<std::priority_queue<int>::value_compare, std::less<int> >::value), "");
+  static_assert((std::is_same<std::priority_queue<int, std::deque<int> >::value_compare, std::less<int> >::value), "");
+  static_assert((std::is_same<std::priority_queue<int, std::deque<int>, std::greater<int> >::value_compare,
+                              std::greater<int> >::value),
+                "");
+  static_assert((std::uses_allocator<std::priority_queue<int>, std::allocator<int> >::value), "");
+  static_assert((!std::uses_allocator<std::priority_queue<int, C>, std::allocator<int> >::value), "");
+  test t;
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/container.adaptors/push_range_container_adaptors.h b/libcxx/test/std/containers/container.adaptors/push_range_container_adaptors.h
index 19e63d23884b2..adf439081a355 100644
--- a/libcxx/test/std/containers/container.adaptors/push_range_container_adaptors.h
+++ b/libcxx/test/std/containers/container.adaptors/push_range_container_adaptors.h
@@ -32,9 +32,7 @@
 #include "unwrap_container_adaptor.h"
 
 template <class Container, class Range>
-concept HasPushRange = requires (Container& c, Range&& range) {
-  c.push_range(range);
-};
+concept HasPushRange = requires(Container& c, Range&& range) { c.push_range(range); };
 
 template <template <class...> class Container, class T, class U>
 constexpr bool test_constraints_push_range() {
@@ -55,55 +53,48 @@ constexpr bool test_constraints_push_range() {
 // Empty container.
 
 template <class T>
-TestCase<T> constexpr EmptyContainer_EmptyRange {
-  .initial = {}, .input = {}, .expected = {}
-};
+TestCase<T> constexpr EmptyContainer_EmptyRange{.initial = {}, .input = {}, .expected = {}};
 
-template <class T> constexpr TestCase<T> EmptyContainer_OneElementRange {
-  .initial = {}, .input = {5}, .expected = {5}
-};
+template <class T>
+constexpr TestCase<T> EmptyContainer_OneElementRange{.initial = {}, .input = {5}, .expected = {5}};
 
-template <class T> constexpr TestCase<T> EmptyContainer_MidRange {
-  .initial = {}, .input = {5, 3, 1, 7, 9}, .expected = {5, 3, 1, 7, 9}
-};
+template <class T>
+constexpr TestCase<T> EmptyContainer_MidRange{.initial = {}, .input = {5, 3, 1, 7, 9}, .expected = {5, 3, 1, 7, 9}};
 
 // One-element container.
 
-template <class T> constexpr TestCase<T> OneElementContainer_EmptyRange {
-  .initial = {3}, .input = {}, .expected = {3}
-};
+template <class T>
+constexpr TestCase<T> OneElementContainer_EmptyRange{.initial = {3}, .input = {}, .expected = {3}};
 
-template <class T> constexpr TestCase<T> OneElementContainer_OneElementRange {
-  .initial = {3}, .input = {-5}, .expected = {3, -5}
-};
+template <class T>
+constexpr TestCase<T> OneElementContainer_OneElementRange{.initial = {3}, .input = {-5}, .expected = {3, -5}};
 
-template <class T> constexpr TestCase<T> OneElementContainer_MidRange {
-  .initial = {3}, .input = {-5, -3, -1, -7, -9}, .expected = {3, -5, -3, -1, -7, -9}
-};
+template <class T>
+constexpr TestCase<T> OneElementContainer_MidRange{
+    .initial = {3}, .input = {-5, -3, -1, -7, -9}, .expected = {3, -5, -3, -1, -7, -9}};
 
 // Full container.
 
-template <class T> constexpr TestCase<T> FullContainer_EmptyRange {
-  .initial = {11, 29, 35, 14, 84}, .input = {}, .expected = {11, 29, 35, 14, 84}
-};
+template <class T>
+constexpr TestCase<T> FullContainer_EmptyRange{
+    .initial = {11, 29, 35, 14, 84}, .input = {}, .expected = {11, 29, 35, 14, 84}};
 
-template <class T> constexpr TestCase<T> FullContainer_OneElementRange {
-  .initial = {11, 29, 35, 14, 84}, .input = {-5}, .expected = {11, 29, 35, 14, 84, -5}
-};
+template <class T>
+constexpr TestCase<T> FullContainer_OneElementRange{
+    .initial = {11, 29, 35, 14, 84}, .input = {-5}, .expected = {11, 29, 35, 14, 84, -5}};
 
-template <class T> constexpr TestCase<T> FullContainer_MidRange {
-  .initial = {11, 29, 35, 14, 84},
-  .input = {-5, -3, -1, -7, -9},
-  .expected = {11, 29, 35, 14, 84, -5, -3, -1, -7, -9}
-};
+template <class T>
+constexpr TestCase<T> FullContainer_MidRange{
+    .initial  = {11, 29, 35, 14, 84},
+    .input    = {-5, -3, -1, -7, -9},
+    .expected = {11, 29, 35, 14, 84, -5, -3, -1, -7, -9}};
 
-template <class T> constexpr TestCase<T> FullContainer_LongRange {
-  .initial = {11, 29, 35, 14, 84},
-  .input = {-5, -3, -1, -7, -9, -19, -48, -56, -13, -14, -29, -88, -17, -1, -5, -11, -89, -21, -33, -48},
-  .expected = {
-      11, 29, 35, 14, 84, -5, -3, -1, -7, -9, -19, -48, -56, -13, -14, -29, -88, -17, -1, -5, -11, -89, -21, -33, -48
-  }
-};
+template <class T>
+constexpr TestCase<T> FullContainer_LongRange{
+    .initial  = {11, 29, 35, 14, 84},
+    .input    = {-5, -3, -1, -7, -9, -19, -48, -56, -13, -14, -29, -88, -17, -1, -5, -11, -89, -21, -33, -48},
+    .expected = {11,  29,  35,  14,  84,  -5, -3, -1,  -7,  -9,  -19, -48, -56,
+                 -13, -14, -29, -88, -17, -1, -5, -11, -89, -21, -33, -48}};
 
 // Container adaptors tests.
 
@@ -159,7 +150,7 @@ constexpr void test_push_range(bool is_result_heapified = false) {
 
 // Move-only types.
 
-template <template <class ...> class Container>
+template <template <class...> class Container>
 constexpr void test_push_range_move_only() {
   MoveOnly input[5];
   std::ranges::subrange in(std::move_iterator{input}, std::move_iterator{input + 5});
@@ -170,25 +161,21 @@ constexpr void test_push_range_move_only() {
 
 // Check that `append_range` is preferred if available and `push_back` is used as a fallback.
 
-enum class InserterChoice {
-  Invalid,
-  PushBack,
-  AppendRange
-};
+enum class InserterChoice { Invalid, PushBack, AppendRange };
 
 template <class T, InserterChoice Inserter>
 struct Container {
   InserterChoice inserter_choice = InserterChoice::Invalid;
 
-  using value_type = T;
-  using iterator = T*;
-  using reference = T&;
+  using value_type      = T;
+  using iterator        = T*;
+  using reference       = T&;
   using const_reference = const T&;
-  using size_type = std::size_t;
+  using size_type       = std::size_t;
 
   static constexpr int Capacity = 8;
-  int size_ = 0;
-  value_type buffer_[Capacity] = {};
+  int size_                     = 0;
+  value_type buffer_[Capacity]  = {};
 
   iterator begin() { return buffer_; }
   iterator end() { return buffer_ + size_; }
@@ -196,15 +183,17 @@ struct Container {
 
   template <class U>
   void push_back(U val)
-  requires (Inserter >= InserterChoice::PushBack) {
+    requires(Inserter >= InserterChoice::PushBack)
+  {
     inserter_choice = InserterChoice::PushBack;
-    buffer_[size_] = val;
+    buffer_[size_]  = val;
     ++size_;
   }
 
   template <std::ranges::input_range Range>
   void append_range(Range&& range)
-  requires (Inserter >= InserterChoice::AppendRange) {
+    requires(Inserter >= InserterChoice::AppendRange)
+  {
     assert(size() + std::ranges::distance(range) <= Capacity);
 
     inserter_choice = InserterChoice::AppendRange;
@@ -218,12 +207,12 @@ struct Container {
   friend bool operator==(const Container&, const Container&) = default;
 };
 
-template <template <class ...> class AdaptorT, class T>
+template <template <class...> class AdaptorT, class T>
 void test_push_range_inserter_choice(bool is_result_heapified = false) {
   { // `append_range` is preferred if available.
     using BaseContainer = Container<T, InserterChoice::AppendRange>;
-    using Adaptor = AdaptorT<T, BaseContainer>;
-    T in[] = {1, 2, 3, 4, 5};
+    using Adaptor       = AdaptorT<T, BaseContainer>;
+    T in[]              = {1, 2, 3, 4, 5};
 
     Adaptor adaptor;
     adaptor.push_range(in);
@@ -241,8 +230,8 @@ void test_push_range_inserter_choice(bool is_result_heapified = false) {
 
   { // `push_back` is used as a fallback (via `back_inserter`).
     using BaseContainer = Container<T, InserterChoice::PushBack>;
-    using Adaptor = AdaptorT<T, BaseContainer>;
-    T in[] = {1, 2, 3, 4, 5};
+    using Adaptor       = AdaptorT<T, BaseContainer>;
+    T in[]              = {1, 2, 3, 4, 5};
 
     Adaptor adaptor;
     adaptor.push_range(in);
@@ -261,11 +250,11 @@ void test_push_range_inserter_choice(bool is_result_heapified = false) {
 
 // Exception safety.
 
-template <template <class ...> class Container>
+template <template <class...> class Container>
 void test_push_range_exception_safety_throwing_copy() {
 #if !defined(TEST_HAS_NO_EXCEPTIONS)
   constexpr int ThrowOn = 3;
-  using T = ThrowingCopy<ThrowOn>;
+  using T               = ThrowingCopy<ThrowOn>;
   test_exception_safety_throwing_copy<ThrowOn, /*Size=*/5>([](auto* from, auto* to) {
     Container<T> c;
     c.push_range(std::ranges::subrange(from, to));
@@ -273,7 +262,7 @@ void test_push_range_exception_safety_throwing_copy() {
 #endif
 }
 
-template <template <class ...> class Adaptor, template <class ...> class BaseContainer, class T>
+template <template <class...> class Adaptor, template <class...> class BaseContainer, class T>
 void test_push_range_exception_safety_throwing_allocator() {
 #if !defined(TEST_HAS_NO_EXCEPTIONS)
   T in[] = {0, 1};
diff --git a/libcxx/test/std/containers/container.adaptors/queue/queue.cons.alloc/ctor_alloc.pass.cpp b/libcxx/test/std/containers/container.adaptors/queue/queue.cons.alloc/ctor_alloc.pass.cpp
index 197256051dc3a..c64d14edb1332 100644
--- a/libcxx/test/std/containers/container.adaptors/queue/queue.cons.alloc/ctor_alloc.pass.cpp
+++ b/libcxx/test/std/containers/container.adaptors/queue/queue.cons.alloc/ctor_alloc.pass.cpp
@@ -17,24 +17,21 @@
 #include "test_macros.h"
 #include "test_allocator.h"
 
-struct test
-    : private std::queue<int, std::deque<int, test_allocator<int> > >
-{
-    typedef std::queue<int, std::deque<int, test_allocator<int> > > base;
+struct test : private std::queue<int, std::deque<int, test_allocator<int> > > {
+  typedef std::queue<int, std::deque<int, test_allocator<int> > > base;
 
-    explicit test(const test_allocator<int>& a) : base(a) {}
-    test(const container_type& container, const test_allocator<int>& a) : base(container, a) {}
+  explicit test(const test_allocator<int>& a) : base(a) {}
+  test(const container_type& container, const test_allocator<int>& a) : base(container, a) {}
 #if TEST_STD_VER >= 11
-    test(container_type&& container, const test_allocator<int>& a) : base(std::move(container), a) {}
-    test(test&& q, const test_allocator<int>& a) : base(std::move(q), a) {}
+  test(container_type&& container, const test_allocator<int>& a) : base(std::move(container), a) {}
+  test(test&& q, const test_allocator<int>& a) : base(std::move(q), a) {}
 #endif
-    test_allocator<int> get_allocator() {return c.get_allocator();}
+  test_allocator<int> get_allocator() { return c.get_allocator(); }
 };
 
-int main(int, char**)
-{
-    test q(test_allocator<int>(3));
-    assert(q.get_allocator() == test_allocator<int>(3));
+int main(int, char**) {
+  test q(test_allocator<int>(3));
+  assert(q.get_allocator() == test_allocator<int>(3));
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/container.adaptors/queue/queue.cons.alloc/ctor_container_alloc.pass.cpp b/libcxx/test/std/containers/container.adaptors/queue/queue.cons.alloc/ctor_container_alloc.pass.cpp
index 674fc6c243f0f..d5611bc017643 100644
--- a/libcxx/test/std/containers/container.adaptors/queue/queue.cons.alloc/ctor_container_alloc.pass.cpp
+++ b/libcxx/test/std/containers/container.adaptors/queue/queue.cons.alloc/ctor_container_alloc.pass.cpp
@@ -19,42 +19,36 @@
 #include "test_allocator.h"
 
 template <class C>
-C
-make(int n)
-{
-    C c;
-    for (int i = 0; i < n; ++i)
-        c.push_back(i);
-    return c;
+C make(int n) {
+  C c;
+  for (int i = 0; i < n; ++i)
+    c.push_back(i);
+  return c;
 }
 
 typedef std::deque<int, test_allocator<int> > C;
 
-struct test
-    : public std::queue<int, C>
-{
-    typedef std::queue<int, C> base;
+struct test : public std::queue<int, C> {
+  typedef std::queue<int, C> base;
 
-    explicit test(const test_allocator<int>& a) : base(a) {}
-    test(const container_type& container, const test_allocator<int>& a) : base(container, a) {}
+  explicit test(const test_allocator<int>& a) : base(a) {}
+  test(const container_type& container, const test_allocator<int>& a) : base(container, a) {}
 #if TEST_STD_VER >= 11
-    test(container_type&& container, const test_allocator<int>& a) : base(std::move(container), a) {}
-    test(test&& q, const test_allocator<int>& a) : base(std::move(q), a) {}
+  test(container_type&& container, const test_allocator<int>& a) : base(std::move(container), a) {}
+  test(test&& q, const test_allocator<int>& a) : base(std::move(q), a) {}
 #endif
-    test_allocator<int> get_allocator() {return c.get_allocator();}
+  test_allocator<int> get_allocator() { return c.get_allocator(); }
 };
 
-int main(int, char**)
-{
-    C d = make<C>(5);
-    test q(d, test_allocator<int>(4));
-    assert(q.get_allocator() == test_allocator<int>(4));
-    assert(q.size() == 5);
-    for (C::size_type i = 0; i < d.size(); ++i)
-    {
-        assert(q.front() == d[i]);
-        q.pop();
-    }
+int main(int, char**) {
+  C d = make<C>(5);
+  test q(d, test_allocator<int>(4));
+  assert(q.get_allocator() == test_allocator<int>(4));
+  assert(q.size() == 5);
+  for (C::size_type i = 0; i < d.size(); ++i) {
+    assert(q.front() == d[i]);
+    q.pop();
+  }
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/container.adaptors/queue/queue.cons.alloc/ctor_queue_alloc.pass.cpp b/libcxx/test/std/containers/container.adaptors/queue/queue.cons.alloc/ctor_queue_alloc.pass.cpp
index a9107f25822dc..a5dc73767b004 100644
--- a/libcxx/test/std/containers/container.adaptors/queue/queue.cons.alloc/ctor_queue_alloc.pass.cpp
+++ b/libcxx/test/std/containers/container.adaptors/queue/queue.cons.alloc/ctor_queue_alloc.pass.cpp
@@ -18,37 +18,32 @@
 #include "test_allocator.h"
 
 template <class C>
-C
-make(int n)
-{
-    C c;
-    for (int i = 0; i < n; ++i)
-        c.push_back(i);
-    return c;
+C make(int n) {
+  C c;
+  for (int i = 0; i < n; ++i)
+    c.push_back(i);
+  return c;
 }
 
 typedef std::deque<int, test_allocator<int> > C;
 
 template <class T>
-struct test
-    : public std::queue<T, C>
-{
-    typedef std::queue<T, C> base;
-    typedef test_allocator<int>      allocator_type;
-    typedef typename base::container_type container_type;
-
-    explicit test(const allocator_type& a) : base(a) {}
-    test(const container_type& container, const allocator_type& a) : base(container, a) {}
-    test(const test& q, const allocator_type& a) : base(q, a) {}
-    allocator_type get_allocator() {return this->c.get_allocator();}
+struct test : public std::queue<T, C> {
+  typedef std::queue<T, C> base;
+  typedef test_allocator<int> allocator_type;
+  typedef typename base::container_type container_type;
+
+  explicit test(const allocator_type& a) : base(a) {}
+  test(const container_type& container, const allocator_type& a) : base(container, a) {}
+  test(const test& q, const allocator_type& a) : base(q, a) {}
+  allocator_type get_allocator() { return this->c.get_allocator(); }
 };
 
-int main(int, char**)
-{
-    test<int> q(make<C>(5), test_allocator<int>(4));
-    test<int> q2(q, test_allocator<int>(5));
-    assert(q2.get_allocator() == test_allocator<int>(5));
-    assert(q2.size() == 5);
+int main(int, char**) {
+  test<int> q(make<C>(5), test_allocator<int>(4));
+  test<int> q2(q, test_allocator<int>(5));
+  assert(q2.get_allocator() == test_allocator<int>(5));
+  assert(q2.size() == 5);
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/container.adaptors/queue/queue.cons.alloc/ctor_rcontainer_alloc.pass.cpp b/libcxx/test/std/containers/container.adaptors/queue/queue.cons.alloc/ctor_rcontainer_alloc.pass.cpp
index 5e78c33df8bbc..b10db553d64fe 100644
--- a/libcxx/test/std/containers/container.adaptors/queue/queue.cons.alloc/ctor_rcontainer_alloc.pass.cpp
+++ b/libcxx/test/std/containers/container.adaptors/queue/queue.cons.alloc/ctor_rcontainer_alloc.pass.cpp
@@ -20,40 +20,33 @@
 #include "test_allocator.h"
 #include "MoveOnly.h"
 
-
 template <class C>
-C
-make(int n)
-{
-    C c;
-    for (int i = 0; i < n; ++i)
-        c.push_back(MoveOnly(i));
-    return c;
+C make(int n) {
+  C c;
+  for (int i = 0; i < n; ++i)
+    c.push_back(MoveOnly(i));
+  return c;
 }
 
 typedef std::deque<MoveOnly, test_allocator<MoveOnly> > C;
 
 template <class T>
-struct test
-    : public std::queue<T, C>
-{
-    typedef std::queue<T, C> base;
-    typedef test_allocator<MoveOnly>      allocator_type;
-    typedef typename base::container_type container_type;
-
-    explicit test(const allocator_type& a) : base(a) {}
-    test(const container_type& container, const allocator_type& a) : base(container, a) {}
-    test(container_type&& container, const allocator_type& a) : base(std::move(container), a) {}
-    test(test&& q, const allocator_type& a) : base(std::move(q), a) {}
-    allocator_type get_allocator() {return this->c.get_allocator();}
+struct test : public std::queue<T, C> {
+  typedef std::queue<T, C> base;
+  typedef test_allocator<MoveOnly> allocator_type;
+  typedef typename base::container_type container_type;
+
+  explicit test(const allocator_type& a) : base(a) {}
+  test(const container_type& container, const allocator_type& a) : base(container, a) {}
+  test(container_type&& container, const allocator_type& a) : base(std::move(container), a) {}
+  test(test&& q, const allocator_type& a) : base(std::move(q), a) {}
+  allocator_type get_allocator() { return this->c.get_allocator(); }
 };
 
-
-int main(int, char**)
-{
-    test<MoveOnly> q(make<C>(5), test_allocator<MoveOnly>(4));
-    assert(q.get_allocator() == test_allocator<MoveOnly>(4));
-    assert(q.size() == 5);
+int main(int, char**) {
+  test<MoveOnly> q(make<C>(5), test_allocator<MoveOnly>(4));
+  assert(q.get_allocator() == test_allocator<MoveOnly>(4));
+  assert(q.size() == 5);
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/container.adaptors/queue/queue.cons.alloc/ctor_rqueue_alloc.pass.cpp b/libcxx/test/std/containers/container.adaptors/queue/queue.cons.alloc/ctor_rqueue_alloc.pass.cpp
index b0604ced6163f..24aef3d79d50a 100644
--- a/libcxx/test/std/containers/container.adaptors/queue/queue.cons.alloc/ctor_rqueue_alloc.pass.cpp
+++ b/libcxx/test/std/containers/container.adaptors/queue/queue.cons.alloc/ctor_rqueue_alloc.pass.cpp
@@ -20,41 +20,34 @@
 #include "test_allocator.h"
 #include "MoveOnly.h"
 
-
 template <class C>
-C
-make(int n)
-{
-    C c;
-    for (int i = 0; i < n; ++i)
-        c.push_back(MoveOnly(i));
-    return c;
+C make(int n) {
+  C c;
+  for (int i = 0; i < n; ++i)
+    c.push_back(MoveOnly(i));
+  return c;
 }
 
 typedef std::deque<MoveOnly, test_allocator<MoveOnly> > C;
 
 template <class T>
-struct test
-    : public std::queue<T, C>
-{
-    typedef std::queue<T, C> base;
-    typedef test_allocator<MoveOnly>      allocator_type;
-    typedef typename base::container_type container_type;
-
-    explicit test(const allocator_type& a) : base(a) {}
-    test(const container_type& container, const allocator_type& a) : base(container, a) {}
-    test(container_type&& container, const allocator_type& a) : base(std::move(container), a) {}
-    test(test&& q, const allocator_type& a) : base(std::move(q), a) {}
-    allocator_type get_allocator() {return this->c.get_allocator();}
+struct test : public std::queue<T, C> {
+  typedef std::queue<T, C> base;
+  typedef test_allocator<MoveOnly> allocator_type;
+  typedef typename base::container_type container_type;
+
+  explicit test(const allocator_type& a) : base(a) {}
+  test(const container_type& container, const allocator_type& a) : base(container, a) {}
+  test(container_type&& container, const allocator_type& a) : base(std::move(container), a) {}
+  test(test&& q, const allocator_type& a) : base(std::move(q), a) {}
+  allocator_type get_allocator() { return this->c.get_allocator(); }
 };
 
-
-int main(int, char**)
-{
-    test<MoveOnly> q(make<C>(5), test_allocator<MoveOnly>(4));
-    test<MoveOnly> q2(std::move(q), test_allocator<MoveOnly>(5));
-    assert(q2.get_allocator() == test_allocator<MoveOnly>(5));
-    assert(q2.size() == 5);
+int main(int, char**) {
+  test<MoveOnly> q(make<C>(5), test_allocator<MoveOnly>(4));
+  test<MoveOnly> q2(std::move(q), test_allocator<MoveOnly>(5));
+  assert(q2.get_allocator() == test_allocator<MoveOnly>(5));
+  assert(q2.size() == 5);
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/container.adaptors/queue/queue.cons/ctor_container.pass.cpp b/libcxx/test/std/containers/container.adaptors/queue/queue.cons/ctor_container.pass.cpp
index 588ed696c8f01..aa32da19099c7 100644
--- a/libcxx/test/std/containers/container.adaptors/queue/queue.cons/ctor_container.pass.cpp
+++ b/libcxx/test/std/containers/container.adaptors/queue/queue.cons/ctor_container.pass.cpp
@@ -16,35 +16,31 @@
 
 #include "test_macros.h"
 #if TEST_STD_VER >= 11
-#include "test_convertible.h"
+#  include "test_convertible.h"
 #endif
 
 template <class C>
-C
-make(int n)
-{
-    C c;
-    for (int i = 0; i < n; ++i)
-        c.push_back(i);
-    return c;
+C make(int n) {
+  C c;
+  for (int i = 0; i < n; ++i)
+    c.push_back(i);
+  return c;
 }
 
-int main(int, char**)
-{
-    typedef std::deque<int> Container;
-    typedef std::queue<int> Q;
-    Container d = make<Container>(5);
-    Q q(d);
-    assert(q.size() == 5);
-    for (std::size_t i = 0; i < d.size(); ++i)
-    {
-        assert(q.front() == d[i]);
-        q.pop();
-    }
+int main(int, char**) {
+  typedef std::deque<int> Container;
+  typedef std::queue<int> Q;
+  Container d = make<Container>(5);
+  Q q(d);
+  assert(q.size() == 5);
+  for (std::size_t i = 0; i < d.size(); ++i) {
+    assert(q.front() == d[i]);
+    q.pop();
+  }
 
 #if TEST_STD_VER >= 11
-    static_assert(!test_convertible<Q, const Container&>(), "");
+  static_assert(!test_convertible<Q, const Container&>(), "");
 #endif
 
-    return 0;
+  return 0;
 }
diff --git a/libcxx/test/std/containers/container.adaptors/queue/queue.cons/ctor_copy.pass.cpp b/libcxx/test/std/containers/container.adaptors/queue/queue.cons/ctor_copy.pass.cpp
index 5c9c012b617a0..8d538f3aa0204 100644
--- a/libcxx/test/std/containers/container.adaptors/queue/queue.cons/ctor_copy.pass.cpp
+++ b/libcxx/test/std/containers/container.adaptors/queue/queue.cons/ctor_copy.pass.cpp
@@ -16,20 +16,17 @@
 #include "test_macros.h"
 
 template <class C>
-C
-make(int n)
-{
-    C c;
-    for (int i = 0; i < n; ++i)
-        c.push_back(i);
-    return c;
+C make(int n) {
+  C c;
+  for (int i = 0; i < n; ++i)
+    c.push_back(i);
+  return c;
 }
 
-int main(int, char**)
-{
-    std::queue<int> q(make<std::deque<int> >(5));
-    std::queue<int> q2 = q;
-    assert(q2 == q);
+int main(int, char**) {
+  std::queue<int> q(make<std::deque<int> >(5));
+  std::queue<int> q2 = q;
+  assert(q2 == q);
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/container.adaptors/queue/queue.cons/ctor_default.pass.cpp b/libcxx/test/std/containers/container.adaptors/queue/queue.cons/ctor_default.pass.cpp
index 17cb4a7936d21..a4ec436457547 100644
--- a/libcxx/test/std/containers/container.adaptors/queue/queue.cons/ctor_default.pass.cpp
+++ b/libcxx/test/std/containers/container.adaptors/queue/queue.cons/ctor_default.pass.cpp
@@ -18,25 +18,24 @@
 #include "test_macros.h"
 #include "test_allocator.h"
 #if TEST_STD_VER >= 11
-#include "test_convertible.h"
+#  include "test_convertible.h"
 #endif
 
-int main(int, char**)
-{
-    typedef std::vector<int, limited_allocator<int, 10> > Container;
-    typedef std::queue<int, Container> Q;
-    Q q;
-    assert(q.size() == 0);
-    q.push(1);
-    q.push(2);
-    assert(q.size() == 2);
-    assert(q.front() == 1);
-    assert(q.back() == 2);
+int main(int, char**) {
+  typedef std::vector<int, limited_allocator<int, 10> > Container;
+  typedef std::queue<int, Container> Q;
+  Q q;
+  assert(q.size() == 0);
+  q.push(1);
+  q.push(2);
+  assert(q.size() == 2);
+  assert(q.front() == 1);
+  assert(q.back() == 2);
 
 #if TEST_STD_VER >= 11
-    // It should be explicit, so not convertible before C++20.
-    static_assert(test_convertible<Q>(), "");
+  // It should be explicit, so not convertible before C++20.
+  static_assert(test_convertible<Q>(), "");
 #endif
 
-    return 0;
+  return 0;
 }
diff --git a/libcxx/test/std/containers/container.adaptors/queue/queue.cons/ctor_iterators.pass.cpp b/libcxx/test/std/containers/container.adaptors/queue/queue.cons/ctor_iterators.pass.cpp
index 5b4a77e90754c..bde15456ebd58 100644
--- a/libcxx/test/std/containers/container.adaptors/queue/queue.cons/ctor_iterators.pass.cpp
+++ b/libcxx/test/std/containers/container.adaptors/queue/queue.cons/ctor_iterators.pass.cpp
@@ -21,8 +21,10 @@
 
 static_assert(!std::is_constructible_v<std::queue<int>, int, int, std::allocator<int>>);
 static_assert(!std::is_constructible_v<std::queue<int>, int*, int*, int>);
-static_assert( std::is_constructible_v<std::queue<int, std::deque<int, test_allocator<int>>>, int*, int*, test_allocator<int>>);
-static_assert(!std::is_constructible_v<std::queue<int, std::deque<int, test_allocator<int>>>, int*, int*, std::allocator<int>>);
+static_assert(
+    std::is_constructible_v<std::queue<int, std::deque<int, test_allocator<int>>>, int*, int*, test_allocator<int>>);
+static_assert(
+    !std::is_constructible_v<std::queue<int, std::deque<int, test_allocator<int>>>, int*, int*, std::allocator<int>>);
 
 template <class T>
 struct alloc : test_allocator<T> {
diff --git a/libcxx/test/std/containers/container.adaptors/queue/queue.cons/ctor_move.pass.cpp b/libcxx/test/std/containers/container.adaptors/queue/queue.cons/ctor_move.pass.cpp
index 0f11733851eed..6d6d495b8e8d7 100644
--- a/libcxx/test/std/containers/container.adaptors/queue/queue.cons/ctor_move.pass.cpp
+++ b/libcxx/test/std/containers/container.adaptors/queue/queue.cons/ctor_move.pass.cpp
@@ -18,24 +18,19 @@
 #include "test_macros.h"
 #include "MoveOnly.h"
 
-
 template <class C>
-C
-make(int n)
-{
-    C c;
-    for (int i = 0; i < n; ++i)
-        c.push_back(MoveOnly(i));
-    return c;
+C make(int n) {
+  C c;
+  for (int i = 0; i < n; ++i)
+    c.push_back(MoveOnly(i));
+  return c;
 }
 
-
-int main(int, char**)
-{
-    std::queue<MoveOnly> q(make<std::deque<MoveOnly> >(5));
-    std::queue<MoveOnly> q2 = std::move(q);
-    assert(q2.size() == 5);
-    assert(q.empty());
+int main(int, char**) {
+  std::queue<MoveOnly> q(make<std::deque<MoveOnly> >(5));
+  std::queue<MoveOnly> q2 = std::move(q);
+  assert(q2.size() == 5);
+  assert(q.empty());
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/container.adaptors/queue/queue.cons/ctor_rcontainer.pass.cpp b/libcxx/test/std/containers/container.adaptors/queue/queue.cons/ctor_rcontainer.pass.cpp
index 22470e5c68c0b..f4f14acbd8a2b 100644
--- a/libcxx/test/std/containers/container.adaptors/queue/queue.cons/ctor_rcontainer.pass.cpp
+++ b/libcxx/test/std/containers/container.adaptors/queue/queue.cons/ctor_rcontainer.pass.cpp
@@ -20,29 +20,26 @@
 #include "test_macros.h"
 #include "MoveOnly.h"
 #if TEST_STD_VER >= 11
-#include "test_convertible.h"
+#  include "test_convertible.h"
 #endif
 
 template <class C>
-C
-make(int n)
-{
-    C c;
-    for (int i = 0; i < n; ++i)
-        c.push_back(MoveOnly(i));
-    return c;
+C make(int n) {
+  C c;
+  for (int i = 0; i < n; ++i)
+    c.push_back(MoveOnly(i));
+  return c;
 }
 
-int main(int, char**)
-{
-    typedef std::deque<MoveOnly> Container;
-    typedef std::queue<MoveOnly> Q;
-    Q q(make<std::deque<MoveOnly> >(5));
-    assert(q.size() == 5);
+int main(int, char**) {
+  typedef std::deque<MoveOnly> Container;
+  typedef std::queue<MoveOnly> Q;
+  Q q(make<std::deque<MoveOnly> >(5));
+  assert(q.size() == 5);
 
 #if TEST_STD_VER >= 11
-    static_assert(!test_convertible<Q, Container&&>(), "");
+  static_assert(!test_convertible<Q, Container&&>(), "");
 #endif
 
-    return 0;
+  return 0;
 }
diff --git a/libcxx/test/std/containers/container.adaptors/queue/queue.cons/deduct.pass.cpp b/libcxx/test/std/containers/container.adaptors/queue/queue.cons/deduct.pass.cpp
index 630d444cbff86..0703248ca5315 100644
--- a/libcxx/test/std/containers/container.adaptors/queue/queue.cons/deduct.pass.cpp
+++ b/libcxx/test/std/containers/container.adaptors/queue/queue.cons/deduct.pass.cpp
@@ -36,140 +36,138 @@
 
 struct A {};
 
-int main(int, char**)
-{
-
-//  Test the explicit deduction guides
-    {
-    std::list<int> l{0, 1, 2, 3, 4, 5, 6, 7, 8, 9 };
+int main(int, char**) {
+  //  Test the explicit deduction guides
+  {
+    std::list<int> l{0, 1, 2, 3, 4, 5, 6, 7, 8, 9};
     std::queue que(l);
 
     static_assert(std::is_same_v<decltype(que), std::queue<int, std::list<int>>>, "");
     assert(que.size() == l.size());
     assert(que.back() == l.back());
-    }
+  }
 
-    {
-    std::list<long, test_allocator<long>> l{10, 11, 12, 13, 14, 15, 16, 17, 18, 19 };
-    std::queue que(l, test_allocator<long>(0,2)); // different allocator
+  {
+    std::list<long, test_allocator<long>> l{10, 11, 12, 13, 14, 15, 16, 17, 18, 19};
+    std::queue que(l, test_allocator<long>(0, 2)); // different allocator
     static_assert(std::is_same_v<decltype(que)::container_type, std::list<long, test_allocator<long>>>, "");
     static_assert(std::is_same_v<decltype(que)::value_type, long>, "");
     assert(que.size() == 10);
     assert(que.back() == 19);
-//  I'd like to assert that we've gotten the right allocator in the queue, but
-//  I don't know how to get at the underlying container.
-    }
+    //  I'd like to assert that we've gotten the right allocator in the queue, but
+    //  I don't know how to get at the underlying container.
+  }
 
-//  Test the implicit deduction guides
-    {
-//  We don't expect this one to work - no way to implicitly get value_type
-//  std::queue que(std::allocator<int>()); // queue (allocator &)
-    }
+  //  Test the implicit deduction guides
+  {
+    //  We don't expect this one to work - no way to implicitly get value_type
+    //  std::queue que(std::allocator<int>()); // queue (allocator &)
+  }
 
-    {
+  {
     std::queue<A> source;
     std::queue que(source); // queue(queue &)
     static_assert(std::is_same_v<decltype(que)::value_type, A>, "");
     static_assert(std::is_same_v<decltype(que)::container_type, std::deque<A>>, "");
     assert(que.size() == 0);
+  }
+
+  {
+    typedef short T;
+    typedef test_allocator<T> Alloc;
+    typedef std::list<T, Alloc> Cont;
+    typedef test_allocator<int> ConvertibleToAlloc;
+    static_assert(std::uses_allocator_v<Cont, ConvertibleToAlloc> &&
+                  !std::is_same_v<typename Cont::allocator_type, ConvertibleToAlloc>);
+
+    {
+      Cont cont;
+      std::queue que(cont, Alloc(2));
+      static_assert(std::is_same_v<decltype(que), std::queue<T, Cont>>);
+    }
+
+    {
+      Cont cont;
+      std::queue que(cont, ConvertibleToAlloc(2));
+      static_assert(std::is_same_v<decltype(que), std::queue<T, Cont>>);
+    }
+
+    {
+      Cont cont;
+      std::queue que(std::move(cont), Alloc(2));
+      static_assert(std::is_same_v<decltype(que), std::queue<T, Cont>>);
+    }
+
+    {
+      Cont cont;
+      std::queue que(std::move(cont), ConvertibleToAlloc(2));
+      static_assert(std::is_same_v<decltype(que), std::queue<T, Cont>>);
+    }
+  }
+
+  {
+    typedef short T;
+    typedef test_allocator<T> Alloc;
+    typedef std::list<T, Alloc> Cont;
+    typedef test_allocator<int> ConvertibleToAlloc;
+    static_assert(std::uses_allocator_v<Cont, ConvertibleToAlloc> &&
+                  !std::is_same_v<typename Cont::allocator_type, ConvertibleToAlloc>);
+
+    {
+      std::queue<T, Cont> source;
+      std::queue que(source, Alloc(2));
+      static_assert(std::is_same_v<decltype(que), std::queue<T, Cont>>);
     }
 
     {
-        typedef short T;
-        typedef test_allocator<T> Alloc;
-        typedef std::list<T, Alloc> Cont;
-        typedef test_allocator<int> ConvertibleToAlloc;
-        static_assert(std::uses_allocator_v<Cont, ConvertibleToAlloc> &&
-                      !std::is_same_v<typename Cont::allocator_type, ConvertibleToAlloc>);
-
-        {
-        Cont cont;
-        std::queue que(cont, Alloc(2));
-        static_assert(std::is_same_v<decltype(que), std::queue<T, Cont>>);
-        }
-
-        {
-        Cont cont;
-        std::queue que(cont, ConvertibleToAlloc(2));
-        static_assert(std::is_same_v<decltype(que), std::queue<T, Cont>>);
-        }
-
-        {
-        Cont cont;
-        std::queue que(std::move(cont), Alloc(2));
-        static_assert(std::is_same_v<decltype(que), std::queue<T, Cont>>);
-        }
-
-        {
-        Cont cont;
-        std::queue que(std::move(cont), ConvertibleToAlloc(2));
-        static_assert(std::is_same_v<decltype(que), std::queue<T, Cont>>);
-        }
+      std::queue<T, Cont> source;
+      std::queue que(source, ConvertibleToAlloc(2));
+      static_assert(std::is_same_v<decltype(que), std::queue<T, Cont>>);
     }
 
     {
-        typedef short T;
-        typedef test_allocator<T> Alloc;
-        typedef std::list<T, Alloc> Cont;
-        typedef test_allocator<int> ConvertibleToAlloc;
-        static_assert(std::uses_allocator_v<Cont, ConvertibleToAlloc> &&
-                      !std::is_same_v<typename Cont::allocator_type, ConvertibleToAlloc>);
-
-        {
-        std::queue<T, Cont> source;
-        std::queue que(source, Alloc(2));
-        static_assert(std::is_same_v<decltype(que), std::queue<T, Cont>>);
-        }
-
-        {
-        std::queue<T, Cont> source;
-        std::queue que(source, ConvertibleToAlloc(2));
-        static_assert(std::is_same_v<decltype(que), std::queue<T, Cont>>);
-        }
-
-        {
-        std::queue<T, Cont> source;
-        std::queue que(std::move(source), Alloc(2));
-        static_assert(std::is_same_v<decltype(que), std::queue<T, Cont>>);
-        }
-
-        {
-        std::queue<T, Cont> source;
-        std::queue que(std::move(source), ConvertibleToAlloc(2));
-        static_assert(std::is_same_v<decltype(que), std::queue<T, Cont>>);
-        }
+      std::queue<T, Cont> source;
+      std::queue que(std::move(source), Alloc(2));
+      static_assert(std::is_same_v<decltype(que), std::queue<T, Cont>>);
     }
 
+    {
+      std::queue<T, Cont> source;
+      std::queue que(std::move(source), ConvertibleToAlloc(2));
+      static_assert(std::is_same_v<decltype(que), std::queue<T, Cont>>);
+    }
+  }
+
 #if TEST_STD_VER >= 23
+  {
+    typedef short T;
+    typedef test_allocator<T> Alloc;
+    std::list<T> a;
+    {
+      std::queue q(a.begin(), a.end());
+      static_assert(std::is_same_v<decltype(q), std::queue<T>>);
+    }
+    {
+      std::queue q(a.begin(), a.end(), Alloc());
+      static_assert(std::is_same_v<decltype(q), std::queue<T, std::deque<T, Alloc>>>);
+    }
+  }
+
+  {
     {
-        typedef short T;
-        typedef test_allocator<T> Alloc;
-        std::list<T> a;
-        {
-        std::queue q(a.begin(), a.end());
-        static_assert(std::is_same_v<decltype(q), std::queue<T>>);
-        }
-        {
-        std::queue q(a.begin(), a.end(), Alloc());
-        static_assert(std::is_same_v<decltype(q), std::queue<T, std::deque<T, Alloc>>>);
-        }
+      std::queue c(std::from_range, std::array<int, 0>());
+      static_assert(std::is_same_v<decltype(c), std::queue<int>>);
     }
 
     {
-      {
-        std::queue c(std::from_range, std::array<int, 0>());
-        static_assert(std::is_same_v<decltype(c), std::queue<int>>);
-      }
-
-      {
-        using Alloc = test_allocator<int>;
-        std::queue c(std::from_range, std::array<int, 0>(), Alloc());
-        static_assert(std::is_same_v<decltype(c), std::queue<int, std::deque<int, Alloc>>>);
-      }
+      using Alloc = test_allocator<int>;
+      std::queue c(std::from_range, std::array<int, 0>(), Alloc());
+      static_assert(std::is_same_v<decltype(c), std::queue<int, std::deque<int, Alloc>>>);
     }
+  }
 #endif
 
-    ContainerAdaptorDeductionGuidesSfinaeAway<std::queue, std::queue<int>>();
+  ContainerAdaptorDeductionGuidesSfinaeAway<std::queue, std::queue<int>>();
 
-    return 0;
+  return 0;
 }
diff --git a/libcxx/test/std/containers/container.adaptors/queue/queue.cons/deduct.verify.cpp b/libcxx/test/std/containers/container.adaptors/queue/queue.cons/deduct.verify.cpp
index f85b2cbeb249c..2216fcf306c4c 100644
--- a/libcxx/test/std/containers/container.adaptors/queue/queue.cons/deduct.verify.cpp
+++ b/libcxx/test/std/containers/container.adaptors/queue/queue.cons/deduct.verify.cpp
@@ -15,32 +15,32 @@
 #include <cassert>
 #include <cstddef>
 
-
-int main(int, char**)
-{
-//  Test the explicit deduction guides
-    {
-//  queue(const Container&, const Alloc&);
-//  The '45' is not an allocator
-    std::queue que(std::list<int>{1,2,3}, 45);  // expected-error-re {{no viable constructor or deduction guide for deduction of template arguments of '{{(std::)?}}queue'}}
-    }
-
-    {
-//  queue(const queue&, const Alloc&);
-//  The '45' is not an allocator
+int main(int, char**) {
+  //  Test the explicit deduction guides
+  {
+    //  queue(const Container&, const Alloc&);
+    //  The '45' is not an allocator
+    std::queue que(std::list<int>{1, 2, 3}, 45);
+    // expected-error-re@-1 {{no viable constructor or deduction guide for deduction of template arguments of '{{(std::)?}}queue'}}
+  }
+
+  {
+    //  queue(const queue&, const Alloc&);
+    //  The '45' is not an allocator
     std::queue<int> source;
-    std::queue que(source, 45);  // expected-error-re {{no viable constructor or deduction guide for deduction of template arguments of '{{(std::)?}}queue'}}
-    }
-
-//  Test the implicit deduction guides
-    {
-//  queue (allocator &)
-    std::queue que((std::allocator<int>()));  // expected-error-re {{no viable constructor or deduction guide for deduction of template arguments of '{{(std::)?}}queue'}}
-//  Note: The extra parens are necessary, since otherwise clang decides it is a function declaration.
-//  Also, we can't use {} instead of parens, because that constructs a
-//      stack<allocator<int>, allocator<allocator<int>>>
-    }
-
+    std::queue que(source, 45);
+    // expected-error-re@-1 {{no viable constructor or deduction guide for deduction of template arguments of '{{(std::)?}}queue'}}
+  }
+
+  //  Test the implicit deduction guides
+  {
+    //  queue (allocator &)
+    std::queue que((std::allocator< int>()));
+    // expected-error-re@-1 {{no viable constructor or deduction guide for deduction of template arguments of '{{(std::)?}}queue'}}
+    //  Note: The extra parens are necessary, since otherwise clang decides it is a function declaration.
+    //  Also, we can't use {} instead of parens, because that constructs a
+    //      stack<allocator<int>, allocator<allocator<int>>>
+  }
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/container.adaptors/queue/queue.cons/default_noexcept.pass.cpp b/libcxx/test/std/containers/container.adaptors/queue/queue.cons/default_noexcept.pass.cpp
index 416e4e38795d3..91c64c33f6207 100644
--- a/libcxx/test/std/containers/container.adaptors/queue/queue.cons/default_noexcept.pass.cpp
+++ b/libcxx/test/std/containers/container.adaptors/queue/queue.cons/default_noexcept.pass.cpp
@@ -21,13 +21,12 @@
 #include "test_macros.h"
 #include "MoveOnly.h"
 
-int main(int, char**)
-{
+int main(int, char**) {
 #if defined(_LIBCPP_VERSION)
-    {
-        typedef std::queue<MoveOnly> C;
-        static_assert(std::is_nothrow_default_constructible<C>::value, "");
-    }
+  {
+    typedef std::queue<MoveOnly> C;
+    static_assert(std::is_nothrow_default_constructible<C>::value, "");
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/container.adaptors/queue/queue.cons/dtor_noexcept.pass.cpp b/libcxx/test/std/containers/container.adaptors/queue/queue.cons/dtor_noexcept.pass.cpp
index 0d53f47961f84..35e1f557f2432 100644
--- a/libcxx/test/std/containers/container.adaptors/queue/queue.cons/dtor_noexcept.pass.cpp
+++ b/libcxx/test/std/containers/container.adaptors/queue/queue.cons/dtor_noexcept.pass.cpp
@@ -19,12 +19,11 @@
 #include "test_macros.h"
 #include "MoveOnly.h"
 
-int main(int, char**)
-{
-    {
-        typedef std::queue<MoveOnly> C;
-        static_assert(std::is_nothrow_destructible<C>::value, "");
-    }
+int main(int, char**) {
+  {
+    typedef std::queue<MoveOnly> C;
+    static_assert(std::is_nothrow_destructible<C>::value, "");
+  }
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/container.adaptors/queue/queue.cons/move_assign_noexcept.pass.cpp b/libcxx/test/std/containers/container.adaptors/queue/queue.cons/move_assign_noexcept.pass.cpp
index 55bd9b4636637..bee13a98a62f7 100644
--- a/libcxx/test/std/containers/container.adaptors/queue/queue.cons/move_assign_noexcept.pass.cpp
+++ b/libcxx/test/std/containers/container.adaptors/queue/queue.cons/move_assign_noexcept.pass.cpp
@@ -21,12 +21,11 @@
 #include "test_macros.h"
 #include "MoveOnly.h"
 
-int main(int, char**)
-{
-    {
-        typedef std::queue<MoveOnly> C;
-        static_assert(std::is_nothrow_move_assignable<C>::value, "");
-    }
+int main(int, char**) {
+  {
+    typedef std::queue<MoveOnly> C;
+    static_assert(std::is_nothrow_move_assignable<C>::value, "");
+  }
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/container.adaptors/queue/queue.cons/move_noexcept.pass.cpp b/libcxx/test/std/containers/container.adaptors/queue/queue.cons/move_noexcept.pass.cpp
index 5a2fd037a009e..8584e8cdabff6 100644
--- a/libcxx/test/std/containers/container.adaptors/queue/queue.cons/move_noexcept.pass.cpp
+++ b/libcxx/test/std/containers/container.adaptors/queue/queue.cons/move_noexcept.pass.cpp
@@ -15,20 +15,18 @@
 
 // This tests a conforming extension
 
-
 #include <queue>
 #include <cassert>
 
 #include "test_macros.h"
 #include "MoveOnly.h"
 
-int main(int, char**)
-{
+int main(int, char**) {
 #if defined(_LIBCPP_VERSION)
-    {
-        typedef std::queue<MoveOnly> C;
-        static_assert(std::is_nothrow_move_constructible<C>::value, "");
-    }
+  {
+    typedef std::queue<MoveOnly> C;
+    static_assert(std::is_nothrow_move_constructible<C>::value, "");
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/container.adaptors/queue/queue.defn/assign_copy.pass.cpp b/libcxx/test/std/containers/container.adaptors/queue/queue.defn/assign_copy.pass.cpp
index 8e1f82f1b05f0..311f4ed2eeb28 100644
--- a/libcxx/test/std/containers/container.adaptors/queue/queue.defn/assign_copy.pass.cpp
+++ b/libcxx/test/std/containers/container.adaptors/queue/queue.defn/assign_copy.pass.cpp
@@ -16,21 +16,18 @@
 #include "test_macros.h"
 
 template <class C>
-C
-make(int n)
-{
-    C c;
-    for (int i = 0; i < n; ++i)
-        c.push_back(i);
-    return c;
+C make(int n) {
+  C c;
+  for (int i = 0; i < n; ++i)
+    c.push_back(i);
+  return c;
 }
 
-int main(int, char**)
-{
-    std::queue<int> q(make<std::deque<int> >(5));
-    std::queue<int> q2;
-    q2 = q;
-    assert(q2 == q);
+int main(int, char**) {
+  std::queue<int> q(make<std::deque<int> >(5));
+  std::queue<int> q2;
+  q2 = q;
+  assert(q2 == q);
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/container.adaptors/queue/queue.defn/assign_move.pass.cpp b/libcxx/test/std/containers/container.adaptors/queue/queue.defn/assign_move.pass.cpp
index d3a7083d0c337..9caaf11717660 100644
--- a/libcxx/test/std/containers/container.adaptors/queue/queue.defn/assign_move.pass.cpp
+++ b/libcxx/test/std/containers/container.adaptors/queue/queue.defn/assign_move.pass.cpp
@@ -18,25 +18,20 @@
 #include "test_macros.h"
 #include "MoveOnly.h"
 
-
 template <class C>
-C
-make(int n)
-{
-    C c;
-    for (int i = 0; i < n; ++i)
-        c.push_back(MoveOnly(i));
-    return c;
+C make(int n) {
+  C c;
+  for (int i = 0; i < n; ++i)
+    c.push_back(MoveOnly(i));
+  return c;
 }
 
-
-int main(int, char**)
-{
-    std::queue<MoveOnly> q(make<std::deque<MoveOnly> >(5));
-    std::queue<MoveOnly> q2;
-    q2 = std::move(q);
-    assert(q2.size() == 5);
-    assert(q.empty());
+int main(int, char**) {
+  std::queue<MoveOnly> q(make<std::deque<MoveOnly> >(5));
+  std::queue<MoveOnly> q2;
+  q2 = std::move(q);
+  assert(q2.size() == 5);
+  assert(q.empty());
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/container.adaptors/queue/queue.defn/back.pass.cpp b/libcxx/test/std/containers/container.adaptors/queue/queue.defn/back.pass.cpp
index 2fa537cae1cd9..d0edb154bdb5c 100644
--- a/libcxx/test/std/containers/container.adaptors/queue/queue.defn/back.pass.cpp
+++ b/libcxx/test/std/containers/container.adaptors/queue/queue.defn/back.pass.cpp
@@ -15,15 +15,14 @@
 
 #include "test_macros.h"
 
-int main(int, char**)
-{
-    std::queue<int> q;
-    assert(q.size() == 0);
-    q.push(1);
-    q.push(2);
-    q.push(3);
-    int& ir = q.back();
-    assert(ir == 3);
+int main(int, char**) {
+  std::queue<int> q;
+  assert(q.size() == 0);
+  q.push(1);
+  q.push(2);
+  q.push(3);
+  int& ir = q.back();
+  assert(ir == 3);
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/container.adaptors/queue/queue.defn/back_const.pass.cpp b/libcxx/test/std/containers/container.adaptors/queue/queue.defn/back_const.pass.cpp
index 822d7dbf08bae..4dc0fa94474a6 100644
--- a/libcxx/test/std/containers/container.adaptors/queue/queue.defn/back_const.pass.cpp
+++ b/libcxx/test/std/containers/container.adaptors/queue/queue.defn/back_const.pass.cpp
@@ -15,16 +15,15 @@
 
 #include "test_macros.h"
 
-int main(int, char**)
-{
-    std::queue<int> q;
-    assert(q.size() == 0);
-    q.push(1);
-    q.push(2);
-    q.push(3);
-    const std::queue<int>& cqr = q;
-    const int& cir = cqr.back();
-    assert(cir == 3);
+int main(int, char**) {
+  std::queue<int> q;
+  assert(q.size() == 0);
+  q.push(1);
+  q.push(2);
+  q.push(3);
+  const std::queue<int>& cqr = q;
+  const int& cir             = cqr.back();
+  assert(cir == 3);
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/container.adaptors/queue/queue.defn/emplace.pass.cpp b/libcxx/test/std/containers/container.adaptors/queue/queue.defn/emplace.pass.cpp
index bdfd9df7978c4..8aea824b11ac4 100644
--- a/libcxx/test/std/containers/container.adaptors/queue/queue.defn/emplace.pass.cpp
+++ b/libcxx/test/std/containers/container.adaptors/queue/queue.defn/emplace.pass.cpp
@@ -14,7 +14,6 @@
 // return type is 'decltype(auto)' in C++17; 'void' before
 // whatever the return type of the underlying container's emplace_back() returns.
 
-
 #include <queue>
 #include <cassert>
 #include <list>
@@ -25,42 +24,41 @@
 
 template <typename Queue>
 void test_return_type() {
-    typedef typename Queue::container_type Container;
-    typedef typename Container::value_type value_type;
-    typedef decltype(std::declval<Queue>().emplace(std::declval<value_type &>())) queue_return_type;
+  typedef typename Queue::container_type Container;
+  typedef typename Container::value_type value_type;
+  typedef decltype(std::declval<Queue>().emplace(std::declval<value_type&>())) queue_return_type;
 
 #if TEST_STD_VER > 14
-    typedef decltype(std::declval<Container>().emplace_back(std::declval<value_type>())) container_return_type;
-    static_assert(std::is_same<queue_return_type, container_return_type>::value, "");
+  typedef decltype(std::declval<Container>().emplace_back(std::declval<value_type>())) container_return_type;
+  static_assert(std::is_same<queue_return_type, container_return_type>::value, "");
 #else
-    static_assert(std::is_same<queue_return_type, void>::value, "");
+  static_assert(std::is_same<queue_return_type, void>::value, "");
 #endif
 }
 
-int main(int, char**)
-{
-    test_return_type<std::queue<int> > ();
-    test_return_type<std::queue<int, std::list<int> > > ();
+int main(int, char**) {
+  test_return_type<std::queue<int> >();
+  test_return_type<std::queue<int, std::list<int> > >();
 
-    std::queue<Emplaceable> q;
+  std::queue<Emplaceable> q;
 #if TEST_STD_VER > 14
-    typedef Emplaceable T;
-    T& r1 = q.emplace(1, 2.5);
-    assert(&r1 == &q.back());
-    T& r2 = q.emplace(2, 3.5);
-    assert(&r2 == &q.back());
-    T& r3 = q.emplace(3, 4.5);
-    assert(&r3 == &q.back());
-    assert(&r1 == &q.front());
+  typedef Emplaceable T;
+  T& r1 = q.emplace(1, 2.5);
+  assert(&r1 == &q.back());
+  T& r2 = q.emplace(2, 3.5);
+  assert(&r2 == &q.back());
+  T& r3 = q.emplace(3, 4.5);
+  assert(&r3 == &q.back());
+  assert(&r1 == &q.front());
 #else
-    q.emplace(1, 2.5);
-    q.emplace(2, 3.5);
-    q.emplace(3, 4.5);
+  q.emplace(1, 2.5);
+  q.emplace(2, 3.5);
+  q.emplace(3, 4.5);
 #endif
 
-    assert(q.size() == 3);
-    assert(q.front() == Emplaceable(1, 2.5));
-    assert(q.back() == Emplaceable(3, 4.5));
+  assert(q.size() == 3);
+  assert(q.front() == Emplaceable(1, 2.5));
+  assert(q.back() == Emplaceable(3, 4.5));
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/container.adaptors/queue/queue.defn/empty.pass.cpp b/libcxx/test/std/containers/container.adaptors/queue/queue.defn/empty.pass.cpp
index 844e7ce8e9638..225ae11a0c9da 100644
--- a/libcxx/test/std/containers/container.adaptors/queue/queue.defn/empty.pass.cpp
+++ b/libcxx/test/std/containers/container.adaptors/queue/queue.defn/empty.pass.cpp
@@ -15,14 +15,13 @@
 
 #include "test_macros.h"
 
-int main(int, char**)
-{
-    std::queue<int> q;
-    assert(q.empty());
-    q.push(1);
-    assert(!q.empty());
-    q.pop();
-    assert(q.empty());
+int main(int, char**) {
+  std::queue<int> q;
+  assert(q.empty());
+  q.push(1);
+  assert(!q.empty());
+  q.pop();
+  assert(q.empty());
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/container.adaptors/queue/queue.defn/empty.verify.cpp b/libcxx/test/std/containers/container.adaptors/queue/queue.defn/empty.verify.cpp
index 5f24ef0e77fc9..27c99b0f379e9 100644
--- a/libcxx/test/std/containers/container.adaptors/queue/queue.defn/empty.verify.cpp
+++ b/libcxx/test/std/containers/container.adaptors/queue/queue.defn/empty.verify.cpp
@@ -17,6 +17,6 @@
 #include <queue>
 
 void f() {
-    std::queue<int> c;
-    c.empty(); // expected-warning {{ignoring return value of function declared with 'nodiscard' attribute}}
+  std::queue<int> c;
+  c.empty(); // expected-warning {{ignoring return value of function declared with 'nodiscard' attribute}}
 }
diff --git a/libcxx/test/std/containers/container.adaptors/queue/queue.defn/front.pass.cpp b/libcxx/test/std/containers/container.adaptors/queue/queue.defn/front.pass.cpp
index d4cbf1e913fa1..fa6c6b5867101 100644
--- a/libcxx/test/std/containers/container.adaptors/queue/queue.defn/front.pass.cpp
+++ b/libcxx/test/std/containers/container.adaptors/queue/queue.defn/front.pass.cpp
@@ -15,15 +15,14 @@
 
 #include "test_macros.h"
 
-int main(int, char**)
-{
-    std::queue<int> q;
-    assert(q.size() == 0);
-    q.push(1);
-    q.push(2);
-    q.push(3);
-    int& ir = q.front();
-    assert(ir == 1);
+int main(int, char**) {
+  std::queue<int> q;
+  assert(q.size() == 0);
+  q.push(1);
+  q.push(2);
+  q.push(3);
+  int& ir = q.front();
+  assert(ir == 1);
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/container.adaptors/queue/queue.defn/front_const.pass.cpp b/libcxx/test/std/containers/container.adaptors/queue/queue.defn/front_const.pass.cpp
index 4e0fc103780af..630f05a22a6e7 100644
--- a/libcxx/test/std/containers/container.adaptors/queue/queue.defn/front_const.pass.cpp
+++ b/libcxx/test/std/containers/container.adaptors/queue/queue.defn/front_const.pass.cpp
@@ -15,16 +15,15 @@
 
 #include "test_macros.h"
 
-int main(int, char**)
-{
-    std::queue<int> q;
-    assert(q.size() == 0);
-    q.push(1);
-    q.push(2);
-    q.push(3);
-    const std::queue<int>& cqr = q;
-    const int& cir = cqr.front();
-    assert(cir == 1);
+int main(int, char**) {
+  std::queue<int> q;
+  assert(q.size() == 0);
+  q.push(1);
+  q.push(2);
+  q.push(3);
+  const std::queue<int>& cqr = q;
+  const int& cir             = cqr.front();
+  assert(cir == 1);
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/container.adaptors/queue/queue.defn/pop.pass.cpp b/libcxx/test/std/containers/container.adaptors/queue/queue.defn/pop.pass.cpp
index 218bec3bcc536..0d6541e8de8f8 100644
--- a/libcxx/test/std/containers/container.adaptors/queue/queue.defn/pop.pass.cpp
+++ b/libcxx/test/std/containers/container.adaptors/queue/queue.defn/pop.pass.cpp
@@ -15,26 +15,25 @@
 
 #include "test_macros.h"
 
-int main(int, char**)
-{
-    std::queue<int> q;
-    assert(q.size() == 0);
-    q.push(1);
-    q.push(2);
-    q.push(3);
-    assert(q.size() == 3);
-    assert(q.front() == 1);
-    assert(q.back() == 3);
-    q.pop();
-    assert(q.size() == 2);
-    assert(q.front() == 2);
-    assert(q.back() == 3);
-    q.pop();
-    assert(q.size() == 1);
-    assert(q.front() == 3);
-    assert(q.back() == 3);
-    q.pop();
-    assert(q.size() == 0);
+int main(int, char**) {
+  std::queue<int> q;
+  assert(q.size() == 0);
+  q.push(1);
+  q.push(2);
+  q.push(3);
+  assert(q.size() == 3);
+  assert(q.front() == 1);
+  assert(q.back() == 3);
+  q.pop();
+  assert(q.size() == 2);
+  assert(q.front() == 2);
+  assert(q.back() == 3);
+  q.pop();
+  assert(q.size() == 1);
+  assert(q.front() == 3);
+  assert(q.back() == 3);
+  q.pop();
+  assert(q.size() == 0);
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/container.adaptors/queue/queue.defn/push.pass.cpp b/libcxx/test/std/containers/container.adaptors/queue/queue.defn/push.pass.cpp
index d772307e36933..901b26a203d98 100644
--- a/libcxx/test/std/containers/container.adaptors/queue/queue.defn/push.pass.cpp
+++ b/libcxx/test/std/containers/container.adaptors/queue/queue.defn/push.pass.cpp
@@ -15,21 +15,20 @@
 
 #include "test_macros.h"
 
-int main(int, char**)
-{
-    std::queue<int> q;
-    q.push(1);
-    assert(q.size() == 1);
-    assert(q.front() == 1);
-    assert(q.back() == 1);
-    q.push(2);
-    assert(q.size() == 2);
-    assert(q.front() == 1);
-    assert(q.back() == 2);
-    q.push(3);
-    assert(q.size() == 3);
-    assert(q.front() == 1);
-    assert(q.back() == 3);
+int main(int, char**) {
+  std::queue<int> q;
+  q.push(1);
+  assert(q.size() == 1);
+  assert(q.front() == 1);
+  assert(q.back() == 1);
+  q.push(2);
+  assert(q.size() == 2);
+  assert(q.front() == 1);
+  assert(q.back() == 2);
+  q.push(3);
+  assert(q.size() == 3);
+  assert(q.front() == 1);
+  assert(q.back() == 3);
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/container.adaptors/queue/queue.defn/push_rv.pass.cpp b/libcxx/test/std/containers/container.adaptors/queue/queue.defn/push_rv.pass.cpp
index 2a75668a3a4b8..4a99c6846dfd2 100644
--- a/libcxx/test/std/containers/container.adaptors/queue/queue.defn/push_rv.pass.cpp
+++ b/libcxx/test/std/containers/container.adaptors/queue/queue.defn/push_rv.pass.cpp
@@ -18,21 +18,20 @@
 #include "test_macros.h"
 #include "MoveOnly.h"
 
-int main(int, char**)
-{
-    std::queue<MoveOnly> q;
-    q.push(MoveOnly(1));
-    assert(q.size() == 1);
-    assert(q.front() == MoveOnly(1));
-    assert(q.back() == MoveOnly(1));
-    q.push(MoveOnly(2));
-    assert(q.size() == 2);
-    assert(q.front() == MoveOnly(1));
-    assert(q.back() == MoveOnly(2));
-    q.push(MoveOnly(3));
-    assert(q.size() == 3);
-    assert(q.front() == MoveOnly(1));
-    assert(q.back() == MoveOnly(3));
+int main(int, char**) {
+  std::queue<MoveOnly> q;
+  q.push(MoveOnly(1));
+  assert(q.size() == 1);
+  assert(q.front() == MoveOnly(1));
+  assert(q.back() == MoveOnly(1));
+  q.push(MoveOnly(2));
+  assert(q.size() == 2);
+  assert(q.front() == MoveOnly(1));
+  assert(q.back() == MoveOnly(2));
+  q.push(MoveOnly(3));
+  assert(q.size() == 3);
+  assert(q.front() == MoveOnly(1));
+  assert(q.back() == MoveOnly(3));
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/container.adaptors/queue/queue.defn/size.pass.cpp b/libcxx/test/std/containers/container.adaptors/queue/queue.defn/size.pass.cpp
index de53c5d669d1f..1824b635e3592 100644
--- a/libcxx/test/std/containers/container.adaptors/queue/queue.defn/size.pass.cpp
+++ b/libcxx/test/std/containers/container.adaptors/queue/queue.defn/size.pass.cpp
@@ -15,12 +15,11 @@
 
 #include "test_macros.h"
 
-int main(int, char**)
-{
-    std::queue<int> q;
-    assert(q.size() == 0);
-    q.push(1);
-    assert(q.size() == 1);
+int main(int, char**) {
+  std::queue<int> q;
+  assert(q.size() == 0);
+  q.push(1);
+  assert(q.size() == 1);
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/container.adaptors/queue/queue.defn/swap.pass.cpp b/libcxx/test/std/containers/container.adaptors/queue/queue.defn/swap.pass.cpp
index e342aeb558a89..2bccea2f5f65a 100644
--- a/libcxx/test/std/containers/container.adaptors/queue/queue.defn/swap.pass.cpp
+++ b/libcxx/test/std/containers/container.adaptors/queue/queue.defn/swap.pass.cpp
@@ -16,24 +16,21 @@
 #include "test_macros.h"
 
 template <class C>
-C
-make(int n)
-{
-    C c;
-    for (int i = 0; i < n; ++i)
-        c.push(i);
-    return c;
+C make(int n) {
+  C c;
+  for (int i = 0; i < n; ++i)
+    c.push(i);
+  return c;
 }
 
-int main(int, char**)
-{
-    std::queue<int> q1 = make<std::queue<int> >(5);
-    std::queue<int> q2 = make<std::queue<int> >(10);
-    std::queue<int> q1_save = q1;
-    std::queue<int> q2_save = q2;
-    q1.swap(q2);
-    assert(q1 == q2_save);
-    assert(q2 == q1_save);
+int main(int, char**) {
+  std::queue<int> q1      = make<std::queue<int> >(5);
+  std::queue<int> q2      = make<std::queue<int> >(10);
+  std::queue<int> q1_save = q1;
+  std::queue<int> q2_save = q2;
+  q1.swap(q2);
+  assert(q1 == q2_save);
+  assert(q2 == q1_save);
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/container.adaptors/queue/queue.defn/types.compile.fail.cpp b/libcxx/test/std/containers/container.adaptors/queue/queue.defn/types.compile.fail.cpp
index 041008ce6aa20..daec08ef62bc6 100644
--- a/libcxx/test/std/containers/container.adaptors/queue/queue.defn/types.compile.fail.cpp
+++ b/libcxx/test/std/containers/container.adaptors/queue/queue.defn/types.compile.fail.cpp
@@ -12,10 +12,9 @@
 #include <cassert>
 #include <type_traits>
 
-int main(int, char**)
-{
-//  LWG#2566 says that the first template param must match the second one's value type
-    std::queue<double, std::deque<int>> t;
+int main(int, char**) {
+  //  LWG#2566 says that the first template param must match the second one's value type
+  std::queue<double, std::deque<int>> t;
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/container.adaptors/queue/queue.defn/types.pass.cpp b/libcxx/test/std/containers/container.adaptors/queue/queue.defn/types.pass.cpp
index bef4060db2da8..782067c56c72d 100644
--- a/libcxx/test/std/containers/container.adaptors/queue/queue.defn/types.pass.cpp
+++ b/libcxx/test/std/containers/container.adaptors/queue/queue.defn/types.pass.cpp
@@ -28,34 +28,27 @@
 
 #include "test_macros.h"
 
-struct test
-    : private std::queue<int>
-{
-    test()
-    {
-        c.push_back(1);
-    }
+struct test : private std::queue<int> {
+  test() { c.push_back(1); }
 };
 
-struct C
-{
-    typedef int value_type;
-    typedef int& reference;
-    typedef const int& const_reference;
-    typedef int size_type;
+struct C {
+  typedef int value_type;
+  typedef int& reference;
+  typedef const int& const_reference;
+  typedef int size_type;
 };
 
-int main(int, char**)
-{
-    static_assert(( std::is_same<std::queue<int>::container_type, std::deque<int> >::value), "");
-    static_assert(( std::is_same<std::queue<int, std::vector<int> >::container_type, std::vector<int> >::value), "");
-    static_assert(( std::is_same<std::queue<int, std::vector<int> >::value_type, int>::value), "");
-    static_assert(( std::is_same<std::queue<int>::reference, std::deque<int>::reference>::value), "");
-    static_assert(( std::is_same<std::queue<int>::const_reference, std::deque<int>::const_reference>::value), "");
-    static_assert(( std::is_same<std::queue<int>::size_type, std::deque<int>::size_type>::value), "");
-    static_assert(( std::uses_allocator<std::queue<int>, std::allocator<int> >::value), "");
-    static_assert((!std::uses_allocator<std::queue<int, C>, std::allocator<int> >::value), "");
-    test t;
+int main(int, char**) {
+  static_assert((std::is_same<std::queue<int>::container_type, std::deque<int> >::value), "");
+  static_assert((std::is_same<std::queue<int, std::vector<int> >::container_type, std::vector<int> >::value), "");
+  static_assert((std::is_same<std::queue<int, std::vector<int> >::value_type, int>::value), "");
+  static_assert((std::is_same<std::queue<int>::reference, std::deque<int>::reference>::value), "");
+  static_assert((std::is_same<std::queue<int>::const_reference, std::deque<int>::const_reference>::value), "");
+  static_assert((std::is_same<std::queue<int>::size_type, std::deque<int>::size_type>::value), "");
+  static_assert((std::uses_allocator<std::queue<int>, std::allocator<int> >::value), "");
+  static_assert((!std::uses_allocator<std::queue<int, C>, std::allocator<int> >::value), "");
+  test t;
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/container.adaptors/queue/queue.ops/eq.pass.cpp b/libcxx/test/std/containers/container.adaptors/queue/queue.ops/eq.pass.cpp
index 8e7825b3fef17..1b1d01fa26d46 100644
--- a/libcxx/test/std/containers/container.adaptors/queue/queue.ops/eq.pass.cpp
+++ b/libcxx/test/std/containers/container.adaptors/queue/queue.ops/eq.pass.cpp
@@ -20,24 +20,21 @@
 #include "test_macros.h"
 
 template <class C>
-C
-make(int n)
-{
-    C c;
-    for (int i = 0; i < n; ++i)
-        c.push(i);
-    return c;
+C make(int n) {
+  C c;
+  for (int i = 0; i < n; ++i)
+    c.push(i);
+  return c;
 }
 
-int main(int, char**)
-{
-    std::queue<int> q1 = make<std::queue<int> >(5);
-    std::queue<int> q2 = make<std::queue<int> >(10);
-    std::queue<int> q1_save = q1;
-    std::queue<int> q2_save = q2;
-    assert(q1 == q1_save);
-    assert(q1 != q2);
-    assert(q2 == q2_save);
+int main(int, char**) {
+  std::queue<int> q1      = make<std::queue<int> >(5);
+  std::queue<int> q2      = make<std::queue<int> >(10);
+  std::queue<int> q1_save = q1;
+  std::queue<int> q2_save = q2;
+  assert(q1 == q1_save);
+  assert(q1 != q2);
+  assert(q2 == q2_save);
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/container.adaptors/queue/queue.ops/lt.pass.cpp b/libcxx/test/std/containers/container.adaptors/queue/queue.ops/lt.pass.cpp
index 246632f75b4de..eb1c8aa3481f5 100644
--- a/libcxx/test/std/containers/container.adaptors/queue/queue.ops/lt.pass.cpp
+++ b/libcxx/test/std/containers/container.adaptors/queue/queue.ops/lt.pass.cpp
@@ -26,23 +26,20 @@
 #include "test_macros.h"
 
 template <class C>
-C
-make(int n)
-{
-    C c;
-    for (int i = 0; i < n; ++i)
-        c.push(i);
-    return c;
+C make(int n) {
+  C c;
+  for (int i = 0; i < n; ++i)
+    c.push(i);
+  return c;
 }
 
-int main(int, char**)
-{
-    std::queue<int> q1 = make<std::queue<int> >(5);
-    std::queue<int> q2 = make<std::queue<int> >(10);
-    assert(q1 < q2);
-    assert(q2 > q1);
-    assert(q1 <= q2);
-    assert(q2 >= q1);
+int main(int, char**) {
+  std::queue<int> q1 = make<std::queue<int> >(5);
+  std::queue<int> q2 = make<std::queue<int> >(10);
+  assert(q1 < q2);
+  assert(q2 > q1);
+  assert(q1 <= q2);
+  assert(q2 >= q1);
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/container.adaptors/queue/queue.special/swap.pass.cpp b/libcxx/test/std/containers/container.adaptors/queue/queue.special/swap.pass.cpp
index 56fbb7418ac9b..1988fba5850ab 100644
--- a/libcxx/test/std/containers/container.adaptors/queue/queue.special/swap.pass.cpp
+++ b/libcxx/test/std/containers/container.adaptors/queue/queue.special/swap.pass.cpp
@@ -17,24 +17,21 @@
 #include "test_macros.h"
 
 template <class C>
-C
-make(int n)
-{
-    C c;
-    for (int i = 0; i < n; ++i)
-        c.push(i);
-    return c;
+C make(int n) {
+  C c;
+  for (int i = 0; i < n; ++i)
+    c.push(i);
+  return c;
 }
 
-int main(int, char**)
-{
-    std::queue<int> q1 = make<std::queue<int> >(5);
-    std::queue<int> q2 = make<std::queue<int> >(10);
-    std::queue<int> q1_save = q1;
-    std::queue<int> q2_save = q2;
-    swap(q1, q2);
-    assert(q1 == q2_save);
-    assert(q2 == q1_save);
+int main(int, char**) {
+  std::queue<int> q1      = make<std::queue<int> >(5);
+  std::queue<int> q2      = make<std::queue<int> >(10);
+  std::queue<int> q1_save = q1;
+  std::queue<int> q2_save = q2;
+  swap(q1, q2);
+  assert(q1 == q2_save);
+  assert(q2 == q1_save);
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/container.adaptors/queue/queue.special/swap_noexcept.pass.cpp b/libcxx/test/std/containers/container.adaptors/queue/queue.special/swap_noexcept.pass.cpp
index 7d07a52cd123d..4d642a29639f7 100644
--- a/libcxx/test/std/containers/container.adaptors/queue/queue.special/swap_noexcept.pass.cpp
+++ b/libcxx/test/std/containers/container.adaptors/queue/queue.special/swap_noexcept.pass.cpp
@@ -22,12 +22,11 @@
 #include "test_macros.h"
 #include "MoveOnly.h"
 
-int main(int, char**)
-{
-    {
-        typedef std::queue<MoveOnly> C;
-        static_assert(noexcept(swap(std::declval<C&>(), std::declval<C&>())), "");
-    }
+int main(int, char**) {
+  {
+    typedef std::queue<MoveOnly> C;
+    static_assert(noexcept(swap(std::declval<C&>(), std::declval<C&>())), "");
+  }
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/container.adaptors/stack/stack.cons.alloc/ctor_alloc.pass.cpp b/libcxx/test/std/containers/container.adaptors/stack/stack.cons.alloc/ctor_alloc.pass.cpp
index 8ec90a50aa809..d756956df8728 100644
--- a/libcxx/test/std/containers/container.adaptors/stack/stack.cons.alloc/ctor_alloc.pass.cpp
+++ b/libcxx/test/std/containers/container.adaptors/stack/stack.cons.alloc/ctor_alloc.pass.cpp
@@ -17,24 +17,21 @@
 #include "test_macros.h"
 #include "test_allocator.h"
 
-struct test
-    : private std::stack<int, std::deque<int, test_allocator<int> > >
-{
-    typedef std::stack<int, std::deque<int, test_allocator<int> > > base;
+struct test : private std::stack<int, std::deque<int, test_allocator<int> > > {
+  typedef std::stack<int, std::deque<int, test_allocator<int> > > base;
 
-    explicit test(const test_allocator<int>& a) : base(a) {}
-    test(const container_type& cont, const test_allocator<int>& a) : base(cont, a) {}
+  explicit test(const test_allocator<int>& a) : base(a) {}
+  test(const container_type& cont, const test_allocator<int>& a) : base(cont, a) {}
 #if TEST_STD_VER >= 11
-    test(container_type&& cont, const test_allocator<int>& a) : base(std::move(cont), a) {}
-    test(test&& q, const test_allocator<int>& a) : base(std::move(q), a) {}
+  test(container_type&& cont, const test_allocator<int>& a) : base(std::move(cont), a) {}
+  test(test&& q, const test_allocator<int>& a) : base(std::move(q), a) {}
 #endif
-    test_allocator<int> get_allocator() {return c.get_allocator();}
+  test_allocator<int> get_allocator() { return c.get_allocator(); }
 };
 
-int main(int, char**)
-{
-    test q(test_allocator<int>(3));
-    assert(q.get_allocator() == test_allocator<int>(3));
+int main(int, char**) {
+  test q(test_allocator<int>(3));
+  assert(q.get_allocator() == test_allocator<int>(3));
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/container.adaptors/stack/stack.cons.alloc/ctor_container_alloc.pass.cpp b/libcxx/test/std/containers/container.adaptors/stack/stack.cons.alloc/ctor_container_alloc.pass.cpp
index 05e48eecb6adb..27a5e187d3d46 100644
--- a/libcxx/test/std/containers/container.adaptors/stack/stack.cons.alloc/ctor_container_alloc.pass.cpp
+++ b/libcxx/test/std/containers/container.adaptors/stack/stack.cons.alloc/ctor_container_alloc.pass.cpp
@@ -19,42 +19,36 @@
 #include "test_allocator.h"
 
 template <class C>
-C
-make(int n)
-{
-    C c;
-    for (int i = 0; i < n; ++i)
-        c.push_back(i);
-    return c;
+C make(int n) {
+  C c;
+  for (int i = 0; i < n; ++i)
+    c.push_back(i);
+  return c;
 }
 
 typedef std::deque<int, test_allocator<int> > C;
 
-struct test
-    : public std::stack<int, C>
-{
-    typedef std::stack<int, C> base;
+struct test : public std::stack<int, C> {
+  typedef std::stack<int, C> base;
 
-    explicit test(const test_allocator<int>& a) : base(a) {}
-    test(const container_type& cont, const test_allocator<int>& a) : base(cont, a) {}
+  explicit test(const test_allocator<int>& a) : base(a) {}
+  test(const container_type& cont, const test_allocator<int>& a) : base(cont, a) {}
 #if TEST_STD_VER >= 11
-    test(container_type&& cont, const test_allocator<int>& a) : base(std::move(cont), a) {}
-    test(test&& q, const test_allocator<int>& a) : base(std::move(q), a) {}
+  test(container_type&& cont, const test_allocator<int>& a) : base(std::move(cont), a) {}
+  test(test&& q, const test_allocator<int>& a) : base(std::move(q), a) {}
 #endif
-    test_allocator<int> get_allocator() {return c.get_allocator();}
+  test_allocator<int> get_allocator() { return c.get_allocator(); }
 };
 
-int main(int, char**)
-{
-    C d = make<C>(5);
-    test q(d, test_allocator<int>(4));
-    assert(q.get_allocator() == test_allocator<int>(4));
-    assert(q.size() == 5);
-    for (C::size_type i = 0; i < d.size(); ++i)
-    {
-        assert(q.top() == d[d.size() - i - 1]);
-        q.pop();
-    }
+int main(int, char**) {
+  C d = make<C>(5);
+  test q(d, test_allocator<int>(4));
+  assert(q.get_allocator() == test_allocator<int>(4));
+  assert(q.size() == 5);
+  for (C::size_type i = 0; i < d.size(); ++i) {
+    assert(q.top() == d[d.size() - i - 1]);
+    q.pop();
+  }
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/container.adaptors/stack/stack.cons.alloc/ctor_copy_alloc.pass.cpp b/libcxx/test/std/containers/container.adaptors/stack/stack.cons.alloc/ctor_copy_alloc.pass.cpp
index 499a3fd2cbf4f..79428cee6faf6 100644
--- a/libcxx/test/std/containers/container.adaptors/stack/stack.cons.alloc/ctor_copy_alloc.pass.cpp
+++ b/libcxx/test/std/containers/container.adaptors/stack/stack.cons.alloc/ctor_copy_alloc.pass.cpp
@@ -18,37 +18,32 @@
 #include "test_allocator.h"
 
 template <class C>
-C
-make(int n)
-{
-    C c;
-    for (int i = 0; i < n; ++i)
-        c.push_back(int(i));
-    return c;
+C make(int n) {
+  C c;
+  for (int i = 0; i < n; ++i)
+    c.push_back(int(i));
+  return c;
 }
 
 typedef std::deque<int, test_allocator<int> > C;
 
 template <class T>
-struct test
-    : public std::stack<T, C>
-{
-    typedef std::stack<T, C> base;
-    typedef test_allocator<int>      allocator_type;
-    typedef typename base::container_type container_type;
-
-    explicit test(const allocator_type& a) : base(a) {}
-    test(const container_type& cont, const allocator_type& a) : base(cont, a) {}
-    test(const test& q, const allocator_type& a) : base(q, a) {}
-    allocator_type get_allocator() {return this->c.get_allocator();}
+struct test : public std::stack<T, C> {
+  typedef std::stack<T, C> base;
+  typedef test_allocator<int> allocator_type;
+  typedef typename base::container_type container_type;
+
+  explicit test(const allocator_type& a) : base(a) {}
+  test(const container_type& cont, const allocator_type& a) : base(cont, a) {}
+  test(const test& q, const allocator_type& a) : base(q, a) {}
+  allocator_type get_allocator() { return this->c.get_allocator(); }
 };
 
-int main(int, char**)
-{
-    test<int> q(make<C>(5), test_allocator<int>(4));
-    test<int> q2(q, test_allocator<int>(5));
-    assert(q2.get_allocator() == test_allocator<int>(5));
-    assert(q2.size() == 5);
+int main(int, char**) {
+  test<int> q(make<C>(5), test_allocator<int>(4));
+  test<int> q2(q, test_allocator<int>(5));
+  assert(q2.get_allocator() == test_allocator<int>(5));
+  assert(q2.size() == 5);
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/container.adaptors/stack/stack.cons.alloc/ctor_rcontainer_alloc.pass.cpp b/libcxx/test/std/containers/container.adaptors/stack/stack.cons.alloc/ctor_rcontainer_alloc.pass.cpp
index eb3a1ac1bf67c..7e453e5f2f998 100644
--- a/libcxx/test/std/containers/container.adaptors/stack/stack.cons.alloc/ctor_rcontainer_alloc.pass.cpp
+++ b/libcxx/test/std/containers/container.adaptors/stack/stack.cons.alloc/ctor_rcontainer_alloc.pass.cpp
@@ -20,40 +20,33 @@
 #include "test_allocator.h"
 #include "MoveOnly.h"
 
-
 template <class C>
-C
-make(int n)
-{
-    C c;
-    for (int i = 0; i < n; ++i)
-        c.push_back(MoveOnly(i));
-    return c;
+C make(int n) {
+  C c;
+  for (int i = 0; i < n; ++i)
+    c.push_back(MoveOnly(i));
+  return c;
 }
 
 typedef std::deque<MoveOnly, test_allocator<MoveOnly> > C;
 
 template <class T>
-struct test
-    : public std::stack<T, C>
-{
-    typedef std::stack<T, C> base;
-    typedef test_allocator<MoveOnly>      allocator_type;
-    typedef typename base::container_type container_type;
-
-    explicit test(const allocator_type& a) : base(a) {}
-    test(const container_type& cont, const allocator_type& a) : base(cont, a) {}
-    test(container_type&& cont, const allocator_type& a) : base(std::move(cont), a) {}
-    test(test&& q, const allocator_type& a) : base(std::move(q), a) {}
-    allocator_type get_allocator() {return this->c.get_allocator();}
+struct test : public std::stack<T, C> {
+  typedef std::stack<T, C> base;
+  typedef test_allocator<MoveOnly> allocator_type;
+  typedef typename base::container_type container_type;
+
+  explicit test(const allocator_type& a) : base(a) {}
+  test(const container_type& cont, const allocator_type& a) : base(cont, a) {}
+  test(container_type&& cont, const allocator_type& a) : base(std::move(cont), a) {}
+  test(test&& q, const allocator_type& a) : base(std::move(q), a) {}
+  allocator_type get_allocator() { return this->c.get_allocator(); }
 };
 
-
-int main(int, char**)
-{
-    test<MoveOnly> q(make<C>(5), test_allocator<MoveOnly>(4));
-    assert(q.get_allocator() == test_allocator<MoveOnly>(4));
-    assert(q.size() == 5);
+int main(int, char**) {
+  test<MoveOnly> q(make<C>(5), test_allocator<MoveOnly>(4));
+  assert(q.get_allocator() == test_allocator<MoveOnly>(4));
+  assert(q.size() == 5);
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/container.adaptors/stack/stack.cons.alloc/ctor_rqueue_alloc.pass.cpp b/libcxx/test/std/containers/container.adaptors/stack/stack.cons.alloc/ctor_rqueue_alloc.pass.cpp
index 849ad046898a7..598a7782d81f8 100644
--- a/libcxx/test/std/containers/container.adaptors/stack/stack.cons.alloc/ctor_rqueue_alloc.pass.cpp
+++ b/libcxx/test/std/containers/container.adaptors/stack/stack.cons.alloc/ctor_rqueue_alloc.pass.cpp
@@ -20,41 +20,34 @@
 #include "test_allocator.h"
 #include "MoveOnly.h"
 
-
 template <class C>
-C
-make(int n)
-{
-    C c;
-    for (int i = 0; i < n; ++i)
-        c.push_back(MoveOnly(i));
-    return c;
+C make(int n) {
+  C c;
+  for (int i = 0; i < n; ++i)
+    c.push_back(MoveOnly(i));
+  return c;
 }
 
 typedef std::deque<MoveOnly, test_allocator<MoveOnly> > C;
 
 template <class T>
-struct test
-    : public std::stack<T, C>
-{
-    typedef std::stack<T, C> base;
-    typedef test_allocator<MoveOnly>      allocator_type;
-    typedef typename base::container_type container_type;
-
-    explicit test(const allocator_type& a) : base(a) {}
-    test(const container_type& cont, const allocator_type& a) : base(cont, a) {}
-    test(container_type&& cont, const allocator_type& a) : base(std::move(cont), a) {}
-    test(test&& q, const allocator_type& a) : base(std::move(q), a) {}
-    allocator_type get_allocator() {return this->c.get_allocator();}
+struct test : public std::stack<T, C> {
+  typedef std::stack<T, C> base;
+  typedef test_allocator<MoveOnly> allocator_type;
+  typedef typename base::container_type container_type;
+
+  explicit test(const allocator_type& a) : base(a) {}
+  test(const container_type& cont, const allocator_type& a) : base(cont, a) {}
+  test(container_type&& cont, const allocator_type& a) : base(std::move(cont), a) {}
+  test(test&& q, const allocator_type& a) : base(std::move(q), a) {}
+  allocator_type get_allocator() { return this->c.get_allocator(); }
 };
 
-
-int main(int, char**)
-{
-    test<MoveOnly> q(make<C>(5), test_allocator<MoveOnly>(4));
-    test<MoveOnly> q2(std::move(q), test_allocator<MoveOnly>(5));
-    assert(q2.get_allocator() == test_allocator<MoveOnly>(5));
-    assert(q2.size() == 5);
+int main(int, char**) {
+  test<MoveOnly> q(make<C>(5), test_allocator<MoveOnly>(4));
+  test<MoveOnly> q2(std::move(q), test_allocator<MoveOnly>(5));
+  assert(q2.get_allocator() == test_allocator<MoveOnly>(5));
+  assert(q2.size() == 5);
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/container.adaptors/stack/stack.cons/ctor_container.pass.cpp b/libcxx/test/std/containers/container.adaptors/stack/stack.cons/ctor_container.pass.cpp
index b77660d4c8750..2bbea6f91c0cc 100644
--- a/libcxx/test/std/containers/container.adaptors/stack/stack.cons/ctor_container.pass.cpp
+++ b/libcxx/test/std/containers/container.adaptors/stack/stack.cons/ctor_container.pass.cpp
@@ -16,35 +16,31 @@
 
 #include "test_macros.h"
 #if TEST_STD_VER >= 11
-#include "test_convertible.h"
+#  include "test_convertible.h"
 #endif
 
 template <class C>
-C
-make(int n)
-{
-    C c;
-    for (int i = 0; i < n; ++i)
-        c.push_back(i);
-    return c;
+C make(int n) {
+  C c;
+  for (int i = 0; i < n; ++i)
+    c.push_back(i);
+  return c;
 }
 
-int main(int, char**)
-{
-    typedef std::deque<int> Container;
-    typedef std::stack<int> Q;
-    Container d = make<Container>(5);
-    Q q(d);
-    assert(q.size() == 5);
-    for (std::size_t i = 0; i < d.size(); ++i)
-    {
-        assert(q.top() == d[d.size() - i - 1]);
-        q.pop();
-    }
+int main(int, char**) {
+  typedef std::deque<int> Container;
+  typedef std::stack<int> Q;
+  Container d = make<Container>(5);
+  Q q(d);
+  assert(q.size() == 5);
+  for (std::size_t i = 0; i < d.size(); ++i) {
+    assert(q.top() == d[d.size() - i - 1]);
+    q.pop();
+  }
 
 #if TEST_STD_VER >= 11
-    static_assert(!test_convertible<Q, const Container&>(), "");
+  static_assert(!test_convertible<Q, const Container&>(), "");
 #endif
 
-    return 0;
+  return 0;
 }
diff --git a/libcxx/test/std/containers/container.adaptors/stack/stack.cons/ctor_copy.pass.cpp b/libcxx/test/std/containers/container.adaptors/stack/stack.cons/ctor_copy.pass.cpp
index e38c12b91b0e9..794ae6fc49b3f 100644
--- a/libcxx/test/std/containers/container.adaptors/stack/stack.cons/ctor_copy.pass.cpp
+++ b/libcxx/test/std/containers/container.adaptors/stack/stack.cons/ctor_copy.pass.cpp
@@ -16,20 +16,17 @@
 #include "test_macros.h"
 
 template <class C>
-C
-make(int n)
-{
-    C c;
-    for (int i = 0; i < n; ++i)
-        c.push_back(i);
-    return c;
+C make(int n) {
+  C c;
+  for (int i = 0; i < n; ++i)
+    c.push_back(i);
+  return c;
 }
 
-int main(int, char**)
-{
-    std::stack<int> q(make<std::deque<int> >(5));
-    std::stack<int> q2 = q;
-    assert(q2 == q);
+int main(int, char**) {
+  std::stack<int> q(make<std::deque<int> >(5));
+  std::stack<int> q2 = q;
+  assert(q2 == q);
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/container.adaptors/stack/stack.cons/ctor_default.pass.cpp b/libcxx/test/std/containers/container.adaptors/stack/stack.cons/ctor_default.pass.cpp
index 4d5aa88e4578a..98fa904adb7ed 100644
--- a/libcxx/test/std/containers/container.adaptors/stack/stack.cons/ctor_default.pass.cpp
+++ b/libcxx/test/std/containers/container.adaptors/stack/stack.cons/ctor_default.pass.cpp
@@ -19,24 +19,23 @@
 #include "test_macros.h"
 #include "test_allocator.h"
 #if TEST_STD_VER >= 11
-#include "test_convertible.h"
+#  include "test_convertible.h"
 #endif
 
-int main(int, char**)
-{
-    typedef std::vector<int, limited_allocator<int, 10> > Container;
-    typedef std::stack<int, Container> Q;
-    Q q;
-    assert(q.size() == 0);
-    q.push(1);
-    q.push(2);
-    assert(q.size() == 2);
-    assert(q.top() == 2);
+int main(int, char**) {
+  typedef std::vector<int, limited_allocator<int, 10> > Container;
+  typedef std::stack<int, Container> Q;
+  Q q;
+  assert(q.size() == 0);
+  q.push(1);
+  q.push(2);
+  assert(q.size() == 2);
+  assert(q.top() == 2);
 
 #if TEST_STD_VER >= 11
-    // It should be explicit, so not convertible before C++20.
-    static_assert(test_convertible<Q>(), "");
+  // It should be explicit, so not convertible before C++20.
+  static_assert(test_convertible<Q>(), "");
 #endif
 
-    return 0;
+  return 0;
 }
diff --git a/libcxx/test/std/containers/container.adaptors/stack/stack.cons/ctor_iterators.pass.cpp b/libcxx/test/std/containers/container.adaptors/stack/stack.cons/ctor_iterators.pass.cpp
index e458b139baaba..5981f9189dbfb 100644
--- a/libcxx/test/std/containers/container.adaptors/stack/stack.cons/ctor_iterators.pass.cpp
+++ b/libcxx/test/std/containers/container.adaptors/stack/stack.cons/ctor_iterators.pass.cpp
@@ -20,8 +20,10 @@
 
 static_assert(!std::is_constructible_v<std::stack<int>, int, int, std::allocator<int>>);
 static_assert(!std::is_constructible_v<std::stack<int>, int*, int*, int>);
-static_assert( std::is_constructible_v<std::stack<int, std::deque<int, test_allocator<int>>>, int*, int*, test_allocator<int>>);
-static_assert(!std::is_constructible_v<std::stack<int, std::deque<int, test_allocator<int>>>, int*, int*, std::allocator<int>>);
+static_assert(
+    std::is_constructible_v<std::stack<int, std::deque<int, test_allocator<int>>>, int*, int*, test_allocator<int>>);
+static_assert(
+    !std::is_constructible_v<std::stack<int, std::deque<int, test_allocator<int>>>, int*, int*, std::allocator<int>>);
 
 template <class T>
 struct alloc : test_allocator<T> {
diff --git a/libcxx/test/std/containers/container.adaptors/stack/stack.cons/ctor_move.pass.cpp b/libcxx/test/std/containers/container.adaptors/stack/stack.cons/ctor_move.pass.cpp
index 39801e9c1c0b8..25318eb37187d 100644
--- a/libcxx/test/std/containers/container.adaptors/stack/stack.cons/ctor_move.pass.cpp
+++ b/libcxx/test/std/containers/container.adaptors/stack/stack.cons/ctor_move.pass.cpp
@@ -18,24 +18,19 @@
 #include "test_macros.h"
 #include "MoveOnly.h"
 
-
 template <class C>
-C
-make(int n)
-{
-    C c;
-    for (int i = 0; i < n; ++i)
-        c.push_back(MoveOnly(i));
-    return c;
+C make(int n) {
+  C c;
+  for (int i = 0; i < n; ++i)
+    c.push_back(MoveOnly(i));
+  return c;
 }
 
-
-int main(int, char**)
-{
-    std::stack<MoveOnly> q(make<std::deque<MoveOnly> >(5));
-    std::stack<MoveOnly> q2 = std::move(q);
-    assert(q2.size() == 5);
-    assert(q.empty());
+int main(int, char**) {
+  std::stack<MoveOnly> q(make<std::deque<MoveOnly> >(5));
+  std::stack<MoveOnly> q2 = std::move(q);
+  assert(q2.size() == 5);
+  assert(q.empty());
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/container.adaptors/stack/stack.cons/ctor_rcontainer.pass.cpp b/libcxx/test/std/containers/container.adaptors/stack/stack.cons/ctor_rcontainer.pass.cpp
index 56dc40e4ed86b..3989685e67884 100644
--- a/libcxx/test/std/containers/container.adaptors/stack/stack.cons/ctor_rcontainer.pass.cpp
+++ b/libcxx/test/std/containers/container.adaptors/stack/stack.cons/ctor_rcontainer.pass.cpp
@@ -20,28 +20,25 @@
 #include "test_macros.h"
 #include "MoveOnly.h"
 #if TEST_STD_VER >= 11
-#include "test_convertible.h"
+#  include "test_convertible.h"
 #endif
 
 template <class C>
-C
-make(int n)
-{
-    C c;
-    for (int i = 0; i < n; ++i)
-        c.push_back(MoveOnly(i));
-    return c;
+C make(int n) {
+  C c;
+  for (int i = 0; i < n; ++i)
+    c.push_back(MoveOnly(i));
+  return c;
 }
 
-int main(int, char**)
-{
-    typedef std::deque<MoveOnly> Container;
-    typedef std::stack<MoveOnly> Q;
-    Q q(make<Container>(5));
-    assert(q.size() == 5);
+int main(int, char**) {
+  typedef std::deque<MoveOnly> Container;
+  typedef std::stack<MoveOnly> Q;
+  Q q(make<Container>(5));
+  assert(q.size() == 5);
 
 #if TEST_STD_VER >= 11
-    static_assert(!test_convertible<Q, Container&&>(), "");
+  static_assert(!test_convertible<Q, Container&&>(), "");
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/container.adaptors/stack/stack.cons/deduct.pass.cpp b/libcxx/test/std/containers/container.adaptors/stack/stack.cons/deduct.pass.cpp
index bc0c594a3c641..2970d732265a1 100644
--- a/libcxx/test/std/containers/container.adaptors/stack/stack.cons/deduct.pass.cpp
+++ b/libcxx/test/std/containers/container.adaptors/stack/stack.cons/deduct.pass.cpp
@@ -39,141 +39,139 @@
 
 struct A {};
 
-int main(int, char**)
-{
-
-//  Test the explicit deduction guides
-    {
-    std::vector<int> v{0, 1, 2, 3, 4, 5, 6, 7, 8, 9 };
+int main(int, char**) {
+  //  Test the explicit deduction guides
+  {
+    std::vector<int> v{0, 1, 2, 3, 4, 5, 6, 7, 8, 9};
     std::stack stk(v);
 
     static_assert(std::is_same_v<decltype(stk), std::stack<int, std::vector<int>>>, "");
     assert(stk.size() == v.size());
     assert(stk.top() == v.back());
-    }
+  }
 
-    {
-    std::list<long, test_allocator<long>> l{10, 11, 12, 13, 14, 15, 16, 17, 18, 19 };
-    std::stack stk(l, test_allocator<long>(0,2)); // different allocator
+  {
+    std::list<long, test_allocator<long>> l{10, 11, 12, 13, 14, 15, 16, 17, 18, 19};
+    std::stack stk(l, test_allocator<long>(0, 2)); // different allocator
     static_assert(std::is_same_v<decltype(stk)::container_type, std::list<long, test_allocator<long>>>, "");
     static_assert(std::is_same_v<decltype(stk)::value_type, long>, "");
     assert(stk.size() == 10);
     assert(stk.top() == 19);
-//  I'd like to assert that we've gotten the right allocator in the stack, but
-//  I don't know how to get at the underlying container.
-    }
+    //  I'd like to assert that we've gotten the right allocator in the stack, but
+    //  I don't know how to get at the underlying container.
+  }
 
-//  Test the implicit deduction guides
+  //  Test the implicit deduction guides
 
-    {
-//  We don't expect this one to work - no way to implicitly get value_type
-//  std::stack stk(std::allocator<int>()); // stack (allocator &)
-    }
+  {
+    //  We don't expect this one to work - no way to implicitly get value_type
+    //  std::stack stk(std::allocator<int>()); // stack (allocator &)
+  }
 
-    {
+  {
     std::stack<A> source;
     std::stack stk(source); // stack(stack &)
     static_assert(std::is_same_v<decltype(stk)::value_type, A>, "");
     static_assert(std::is_same_v<decltype(stk)::container_type, std::deque<A>>, "");
     assert(stk.size() == 0);
+  }
+
+  {
+    typedef short T;
+    typedef test_allocator<T> Alloc;
+    typedef std::list<T, Alloc> Cont;
+    typedef test_allocator<int> ConvertibleToAlloc;
+    static_assert(std::uses_allocator_v<Cont, ConvertibleToAlloc> &&
+                  !std::is_same_v<typename Cont::allocator_type, ConvertibleToAlloc>);
+
+    {
+      Cont cont;
+      std::stack stk(cont, Alloc(2));
+      static_assert(std::is_same_v<decltype(stk), std::stack<T, Cont>>);
+    }
+
+    {
+      Cont cont;
+      std::stack stk(cont, ConvertibleToAlloc(2));
+      static_assert(std::is_same_v<decltype(stk), std::stack<T, Cont>>);
+    }
+
+    {
+      Cont cont;
+      std::stack stk(std::move(cont), Alloc(2));
+      static_assert(std::is_same_v<decltype(stk), std::stack<T, Cont>>);
+    }
+
+    {
+      Cont cont;
+      std::stack stk(std::move(cont), ConvertibleToAlloc(2));
+      static_assert(std::is_same_v<decltype(stk), std::stack<T, Cont>>);
+    }
+  }
+
+  {
+    typedef short T;
+    typedef test_allocator<T> Alloc;
+    typedef std::list<T, Alloc> Cont;
+    typedef test_allocator<int> ConvertibleToAlloc;
+    static_assert(std::uses_allocator_v<Cont, ConvertibleToAlloc> &&
+                  !std::is_same_v<typename Cont::allocator_type, ConvertibleToAlloc>);
+
+    {
+      std::stack<T, Cont> source;
+      std::stack stk(source, Alloc(2));
+      static_assert(std::is_same_v<decltype(stk), std::stack<T, Cont>>);
     }
 
     {
-        typedef short T;
-        typedef test_allocator<T> Alloc;
-        typedef std::list<T, Alloc> Cont;
-        typedef test_allocator<int> ConvertibleToAlloc;
-        static_assert(std::uses_allocator_v<Cont, ConvertibleToAlloc> &&
-                      !std::is_same_v<typename Cont::allocator_type, ConvertibleToAlloc>);
-
-        {
-        Cont cont;
-        std::stack stk(cont, Alloc(2));
-        static_assert(std::is_same_v<decltype(stk), std::stack<T, Cont>>);
-        }
-
-        {
-        Cont cont;
-        std::stack stk(cont, ConvertibleToAlloc(2));
-        static_assert(std::is_same_v<decltype(stk), std::stack<T, Cont>>);
-        }
-
-        {
-        Cont cont;
-        std::stack stk(std::move(cont), Alloc(2));
-        static_assert(std::is_same_v<decltype(stk), std::stack<T, Cont>>);
-        }
-
-        {
-        Cont cont;
-        std::stack stk(std::move(cont), ConvertibleToAlloc(2));
-        static_assert(std::is_same_v<decltype(stk), std::stack<T, Cont>>);
-        }
+      std::stack<T, Cont> source;
+      std::stack stk(source, ConvertibleToAlloc(2));
+      static_assert(std::is_same_v<decltype(stk), std::stack<T, Cont>>);
     }
 
     {
-        typedef short T;
-        typedef test_allocator<T> Alloc;
-        typedef std::list<T, Alloc> Cont;
-        typedef test_allocator<int> ConvertibleToAlloc;
-        static_assert(std::uses_allocator_v<Cont, ConvertibleToAlloc> &&
-                      !std::is_same_v<typename Cont::allocator_type, ConvertibleToAlloc>);
-
-        {
-        std::stack<T, Cont> source;
-        std::stack stk(source, Alloc(2));
-        static_assert(std::is_same_v<decltype(stk), std::stack<T, Cont>>);
-        }
-
-        {
-        std::stack<T, Cont> source;
-        std::stack stk(source, ConvertibleToAlloc(2));
-        static_assert(std::is_same_v<decltype(stk), std::stack<T, Cont>>);
-        }
-
-        {
-        std::stack<T, Cont> source;
-        std::stack stk(std::move(source), Alloc(2));
-        static_assert(std::is_same_v<decltype(stk), std::stack<T, Cont>>);
-        }
-
-        {
-        std::stack<T, Cont> source;
-        std::stack stk(std::move(source), ConvertibleToAlloc(2));
-        static_assert(std::is_same_v<decltype(stk), std::stack<T, Cont>>);
-        }
+      std::stack<T, Cont> source;
+      std::stack stk(std::move(source), Alloc(2));
+      static_assert(std::is_same_v<decltype(stk), std::stack<T, Cont>>);
     }
 
+    {
+      std::stack<T, Cont> source;
+      std::stack stk(std::move(source), ConvertibleToAlloc(2));
+      static_assert(std::is_same_v<decltype(stk), std::stack<T, Cont>>);
+    }
+  }
+
 #if TEST_STD_VER >= 23
+  {
+    typedef short T;
+    typedef test_allocator<T> Alloc;
+    std::list<T> a;
+    {
+      std::stack s(a.begin(), a.end());
+      static_assert(std::is_same_v<decltype(s), std::stack<T>>);
+    }
+    {
+      std::stack s(a.begin(), a.end(), Alloc());
+      static_assert(std::is_same_v<decltype(s), std::stack<T, std::deque<T, Alloc>>>);
+    }
+  }
+
+  {
     {
-        typedef short T;
-        typedef test_allocator<T> Alloc;
-        std::list<T> a;
-        {
-        std::stack s(a.begin(), a.end());
-        static_assert(std::is_same_v<decltype(s), std::stack<T>>);
-        }
-        {
-        std::stack s(a.begin(), a.end(), Alloc());
-        static_assert(std::is_same_v<decltype(s), std::stack<T, std::deque<T, Alloc>>>);
-        }
+      std::stack c(std::from_range, std::array<int, 0>());
+      static_assert(std::is_same_v<decltype(c), std::stack<int>>);
     }
 
     {
-      {
-        std::stack c(std::from_range, std::array<int, 0>());
-        static_assert(std::is_same_v<decltype(c), std::stack<int>>);
-      }
-
-      {
-        using Alloc = test_allocator<int>;
-        std::stack c(std::from_range, std::array<int, 0>(), Alloc());
-        static_assert(std::is_same_v<decltype(c), std::stack<int, std::deque<int, Alloc>>>);
-      }
+      using Alloc = test_allocator<int>;
+      std::stack c(std::from_range, std::array<int, 0>(), Alloc());
+      static_assert(std::is_same_v<decltype(c), std::stack<int, std::deque<int, Alloc>>>);
     }
+  }
 #endif
 
-    ContainerAdaptorDeductionGuidesSfinaeAway<std::stack, std::stack<int>>();
+  ContainerAdaptorDeductionGuidesSfinaeAway<std::stack, std::stack<int>>();
 
-    return 0;
+  return 0;
 }
diff --git a/libcxx/test/std/containers/container.adaptors/stack/stack.cons/deduct.verify.cpp b/libcxx/test/std/containers/container.adaptors/stack/stack.cons/deduct.verify.cpp
index 390df86dd0f5a..dbc0e6c915a6d 100644
--- a/libcxx/test/std/containers/container.adaptors/stack/stack.cons/deduct.verify.cpp
+++ b/libcxx/test/std/containers/container.adaptors/stack/stack.cons/deduct.verify.cpp
@@ -14,39 +14,38 @@
 //    -> vector<typename iterator_traits<InputIterator>::value_type, Allocator>;
 //
 
-
 #include <stack>
 #include <list>
 #include <iterator>
 #include <cassert>
 #include <cstddef>
 
-
-int main(int, char**)
-{
-//  Test the explicit deduction guides
-    {
-//  stack(const Container&, const Alloc&);
-//  The '45' is not an allocator
-    std::stack stk(std::list<int>({1,2,3}), 45);  // expected-error-re {{no viable constructor or deduction guide for deduction of template arguments of '{{(std::)?}}stack'}}
-    }
-
-    {
-//  stack(const stack&, const Alloc&);
-//  The '45' is not an allocator
+int main(int, char**) {
+  //  Test the explicit deduction guides
+  {
+    //  stack(const Container&, const Alloc&);
+    //  The '45' is not an allocator
+    std::stack stk(std::list<int>({1, 2, 3}), 45);
+    // expected-error-re@-1 {{no viable constructor or deduction guide for deduction of template arguments of '{{(std::)?}}stack'}}
+  }
+
+  {
+    //  stack(const stack&, const Alloc&);
+    //  The '45' is not an allocator
     std::stack<int> source;
-    std::stack stk(source, 45);  // expected-error-re {{no viable constructor or deduction guide for deduction of template arguments of '{{(std::)?}}stack'}}
-    }
-
-//  Test the implicit deduction guides
-    {
-//  stack (allocator &)
-    std::stack stk((std::allocator<int>()));  // expected-error-re {{no viable constructor or deduction guide for deduction of template arguments of '{{(std::)?}}stack'}}
-//  Note: The extra parens are necessary, since otherwise clang decides it is a function declaration.
-//  Also, we can't use {} instead of parens, because that constructs a
-//      stack<allocator<int>, allocator<allocator<int>>>
-    }
-
+    std::stack stk(source, 45);
+    // expected-error-re@-1 {{no viable constructor or deduction guide for deduction of template arguments of '{{(std::)?}}stack'}}
+  }
+
+  //  Test the implicit deduction guides
+  {
+    //  stack (allocator &)
+    std::stack stk((std::allocator< int>()));
+    // expected-error-re@-1 {{no viable constructor or deduction guide for deduction of template arguments of '{{(std::)?}}stack'}}
+    //  Note: The extra parens are necessary, since otherwise clang decides it is a function declaration.
+    //  Also, we can't use {} instead of parens, because that constructs a
+    //      stack<allocator<int>, allocator<allocator<int>>>
+  }
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/container.adaptors/stack/stack.cons/default_noexcept.pass.cpp b/libcxx/test/std/containers/container.adaptors/stack/stack.cons/default_noexcept.pass.cpp
index 9cf79a81f9485..2967bbf35426f 100644
--- a/libcxx/test/std/containers/container.adaptors/stack/stack.cons/default_noexcept.pass.cpp
+++ b/libcxx/test/std/containers/container.adaptors/stack/stack.cons/default_noexcept.pass.cpp
@@ -15,20 +15,18 @@
 
 // This tests a conforming extension
 
-
 #include <stack>
 #include <cassert>
 
 #include "test_macros.h"
 #include "MoveOnly.h"
 
-int main(int, char**)
-{
+int main(int, char**) {
 #if defined(_LIBCPP_VERSION)
-    {
-        typedef std::stack<MoveOnly> C;
-        static_assert(std::is_nothrow_default_constructible<C>::value, "");
-    }
+  {
+    typedef std::stack<MoveOnly> C;
+    static_assert(std::is_nothrow_default_constructible<C>::value, "");
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/container.adaptors/stack/stack.cons/dtor_noexcept.pass.cpp b/libcxx/test/std/containers/container.adaptors/stack/stack.cons/dtor_noexcept.pass.cpp
index b6015c1e1ac31..e3dfba01a5007 100644
--- a/libcxx/test/std/containers/container.adaptors/stack/stack.cons/dtor_noexcept.pass.cpp
+++ b/libcxx/test/std/containers/container.adaptors/stack/stack.cons/dtor_noexcept.pass.cpp
@@ -19,12 +19,11 @@
 #include "test_macros.h"
 #include "MoveOnly.h"
 
-int main(int, char**)
-{
-    {
-        typedef std::stack<MoveOnly> C;
-        static_assert(std::is_nothrow_destructible<C>::value, "");
-    }
+int main(int, char**) {
+  {
+    typedef std::stack<MoveOnly> C;
+    static_assert(std::is_nothrow_destructible<C>::value, "");
+  }
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/container.adaptors/stack/stack.cons/move_assign_noexcept.pass.cpp b/libcxx/test/std/containers/container.adaptors/stack/stack.cons/move_assign_noexcept.pass.cpp
index d4da48e835dc4..c00ea2057f5ef 100644
--- a/libcxx/test/std/containers/container.adaptors/stack/stack.cons/move_assign_noexcept.pass.cpp
+++ b/libcxx/test/std/containers/container.adaptors/stack/stack.cons/move_assign_noexcept.pass.cpp
@@ -21,12 +21,11 @@
 #include "test_macros.h"
 #include "MoveOnly.h"
 
-int main(int, char**)
-{
-    {
-        typedef std::stack<MoveOnly> C;
-        static_assert(std::is_nothrow_move_assignable<C>::value, "");
-    }
+int main(int, char**) {
+  {
+    typedef std::stack<MoveOnly> C;
+    static_assert(std::is_nothrow_move_assignable<C>::value, "");
+  }
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/container.adaptors/stack/stack.cons/move_noexcept.pass.cpp b/libcxx/test/std/containers/container.adaptors/stack/stack.cons/move_noexcept.pass.cpp
index 24e66a7076e86..d0f69a2e7bf27 100644
--- a/libcxx/test/std/containers/container.adaptors/stack/stack.cons/move_noexcept.pass.cpp
+++ b/libcxx/test/std/containers/container.adaptors/stack/stack.cons/move_noexcept.pass.cpp
@@ -21,13 +21,12 @@
 #include "test_macros.h"
 #include "MoveOnly.h"
 
-int main(int, char**)
-{
+int main(int, char**) {
 #if defined(_LIBCPP_VERSION)
-    {
-        typedef std::stack<MoveOnly> C;
-        static_assert(std::is_nothrow_move_constructible<C>::value, "");
-    }
+  {
+    typedef std::stack<MoveOnly> C;
+    static_assert(std::is_nothrow_move_constructible<C>::value, "");
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/container.adaptors/stack/stack.defn/assign_copy.pass.cpp b/libcxx/test/std/containers/container.adaptors/stack/stack.defn/assign_copy.pass.cpp
index 81ba10e1dfc47..ee2007ce6b3bc 100644
--- a/libcxx/test/std/containers/container.adaptors/stack/stack.defn/assign_copy.pass.cpp
+++ b/libcxx/test/std/containers/container.adaptors/stack/stack.defn/assign_copy.pass.cpp
@@ -16,21 +16,18 @@
 #include "test_macros.h"
 
 template <class C>
-C
-make(int n)
-{
-    C c;
-    for (int i = 0; i < n; ++i)
-        c.push_back(i);
-    return c;
+C make(int n) {
+  C c;
+  for (int i = 0; i < n; ++i)
+    c.push_back(i);
+  return c;
 }
 
-int main(int, char**)
-{
-    std::stack<int> q(make<std::deque<int> >(5));
-    std::stack<int> q2;
-    q2 = q;
-    assert(q2 == q);
+int main(int, char**) {
+  std::stack<int> q(make<std::deque<int> >(5));
+  std::stack<int> q2;
+  q2 = q;
+  assert(q2 == q);
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/container.adaptors/stack/stack.defn/assign_move.pass.cpp b/libcxx/test/std/containers/container.adaptors/stack/stack.defn/assign_move.pass.cpp
index 81273882244dc..c5c92e6e46356 100644
--- a/libcxx/test/std/containers/container.adaptors/stack/stack.defn/assign_move.pass.cpp
+++ b/libcxx/test/std/containers/container.adaptors/stack/stack.defn/assign_move.pass.cpp
@@ -18,25 +18,20 @@
 #include "test_macros.h"
 #include "MoveOnly.h"
 
-
 template <class C>
-C
-make(int n)
-{
-    C c;
-    for (int i = 0; i < n; ++i)
-        c.push_back(MoveOnly(i));
-    return c;
+C make(int n) {
+  C c;
+  for (int i = 0; i < n; ++i)
+    c.push_back(MoveOnly(i));
+  return c;
 }
 
-
-int main(int, char**)
-{
-    std::stack<MoveOnly> q(make<std::deque<MoveOnly> >(5));
-    std::stack<MoveOnly> q2;
-    q2 = std::move(q);
-    assert(q2.size() == 5);
-    assert(q.empty());
+int main(int, char**) {
+  std::stack<MoveOnly> q(make<std::deque<MoveOnly> >(5));
+  std::stack<MoveOnly> q2;
+  q2 = std::move(q);
+  assert(q2.size() == 5);
+  assert(q.empty());
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/container.adaptors/stack/stack.defn/emplace.pass.cpp b/libcxx/test/std/containers/container.adaptors/stack/stack.defn/emplace.pass.cpp
index 5ce2127a443a3..2a4d9d47e1fba 100644
--- a/libcxx/test/std/containers/container.adaptors/stack/stack.defn/emplace.pass.cpp
+++ b/libcxx/test/std/containers/container.adaptors/stack/stack.defn/emplace.pass.cpp
@@ -24,39 +24,38 @@
 
 template <typename Stack>
 void test_return_type() {
-    typedef typename Stack::container_type Container;
-    typedef typename Container::value_type value_type;
-    typedef decltype(std::declval<Stack>().emplace(std::declval<value_type &>()))     stack_return_type;
+  typedef typename Stack::container_type Container;
+  typedef typename Container::value_type value_type;
+  typedef decltype(std::declval<Stack>().emplace(std::declval<value_type&>())) stack_return_type;
 
 #if TEST_STD_VER > 14
-    typedef decltype(std::declval<Container>().emplace_back(std::declval<value_type>())) container_return_type;
-    static_assert(std::is_same<stack_return_type, container_return_type>::value, "");
+  typedef decltype(std::declval<Container>().emplace_back(std::declval<value_type>())) container_return_type;
+  static_assert(std::is_same<stack_return_type, container_return_type>::value, "");
 #else
-    static_assert(std::is_same<stack_return_type, void>::value, "");
+  static_assert(std::is_same<stack_return_type, void>::value, "");
 #endif
 }
 
-int main(int, char**)
-{
-    test_return_type<std::stack<int> > ();
-    test_return_type<std::stack<int, std::vector<int> > > ();
+int main(int, char**) {
+  test_return_type<std::stack<int> >();
+  test_return_type<std::stack<int, std::vector<int> > >();
 
-    std::stack<Emplaceable> q;
+  std::stack<Emplaceable> q;
 #if TEST_STD_VER > 14
-    typedef Emplaceable T;
-    T& r1 = q.emplace(1, 2.5);
-    assert(&r1 == &q.top());
-    T& r2 = q.emplace(2, 3.5);
-    assert(&r2 == &q.top());
-    T& r3 = q.emplace(3, 4.5);
-    assert(&r3 == &q.top());
+  typedef Emplaceable T;
+  T& r1 = q.emplace(1, 2.5);
+  assert(&r1 == &q.top());
+  T& r2 = q.emplace(2, 3.5);
+  assert(&r2 == &q.top());
+  T& r3 = q.emplace(3, 4.5);
+  assert(&r3 == &q.top());
 #else
-    q.emplace(1, 2.5);
-    q.emplace(2, 3.5);
-    q.emplace(3, 4.5);
+  q.emplace(1, 2.5);
+  q.emplace(2, 3.5);
+  q.emplace(3, 4.5);
 #endif
-    assert(q.size() == 3);
-    assert(q.top() == Emplaceable(3, 4.5));
+  assert(q.size() == 3);
+  assert(q.top() == Emplaceable(3, 4.5));
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/container.adaptors/stack/stack.defn/empty.pass.cpp b/libcxx/test/std/containers/container.adaptors/stack/stack.defn/empty.pass.cpp
index e278c4a4c7a1d..d66c16a56fd0c 100644
--- a/libcxx/test/std/containers/container.adaptors/stack/stack.defn/empty.pass.cpp
+++ b/libcxx/test/std/containers/container.adaptors/stack/stack.defn/empty.pass.cpp
@@ -15,14 +15,13 @@
 
 #include "test_macros.h"
 
-int main(int, char**)
-{
-    std::stack<int> q;
-    assert(q.empty());
-    q.push(1);
-    assert(!q.empty());
-    q.pop();
-    assert(q.empty());
+int main(int, char**) {
+  std::stack<int> q;
+  assert(q.empty());
+  q.push(1);
+  assert(!q.empty());
+  q.pop();
+  assert(q.empty());
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/container.adaptors/stack/stack.defn/empty.verify.cpp b/libcxx/test/std/containers/container.adaptors/stack/stack.defn/empty.verify.cpp
index e9af144c858b1..9100bc2a03e06 100644
--- a/libcxx/test/std/containers/container.adaptors/stack/stack.defn/empty.verify.cpp
+++ b/libcxx/test/std/containers/container.adaptors/stack/stack.defn/empty.verify.cpp
@@ -17,6 +17,6 @@
 #include <stack>
 
 void f() {
-    std::stack<int> c;
-    c.empty(); // expected-warning {{ignoring return value of function declared with 'nodiscard' attribute}}
+  std::stack<int> c;
+  c.empty(); // expected-warning {{ignoring return value of function declared with 'nodiscard' attribute}}
 }
diff --git a/libcxx/test/std/containers/container.adaptors/stack/stack.defn/pop.pass.cpp b/libcxx/test/std/containers/container.adaptors/stack/stack.defn/pop.pass.cpp
index d97dd8f37e085..4af23fa911a0c 100644
--- a/libcxx/test/std/containers/container.adaptors/stack/stack.defn/pop.pass.cpp
+++ b/libcxx/test/std/containers/container.adaptors/stack/stack.defn/pop.pass.cpp
@@ -15,23 +15,22 @@
 
 #include "test_macros.h"
 
-int main(int, char**)
-{
-    std::stack<int> q;
-    assert(q.size() == 0);
-    q.push(1);
-    q.push(2);
-    q.push(3);
-    assert(q.size() == 3);
-    assert(q.top() == 3);
-    q.pop();
-    assert(q.size() == 2);
-    assert(q.top() == 2);
-    q.pop();
-    assert(q.size() == 1);
-    assert(q.top() == 1);
-    q.pop();
-    assert(q.size() == 0);
+int main(int, char**) {
+  std::stack<int> q;
+  assert(q.size() == 0);
+  q.push(1);
+  q.push(2);
+  q.push(3);
+  assert(q.size() == 3);
+  assert(q.top() == 3);
+  q.pop();
+  assert(q.size() == 2);
+  assert(q.top() == 2);
+  q.pop();
+  assert(q.size() == 1);
+  assert(q.top() == 1);
+  q.pop();
+  assert(q.size() == 0);
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/container.adaptors/stack/stack.defn/push.pass.cpp b/libcxx/test/std/containers/container.adaptors/stack/stack.defn/push.pass.cpp
index 91e08bf98a33e..b514033b65242 100644
--- a/libcxx/test/std/containers/container.adaptors/stack/stack.defn/push.pass.cpp
+++ b/libcxx/test/std/containers/container.adaptors/stack/stack.defn/push.pass.cpp
@@ -15,18 +15,17 @@
 
 #include "test_macros.h"
 
-int main(int, char**)
-{
-    std::stack<int> q;
-    q.push(1);
-    assert(q.size() == 1);
-    assert(q.top() == 1);
-    q.push(2);
-    assert(q.size() == 2);
-    assert(q.top() == 2);
-    q.push(3);
-    assert(q.size() == 3);
-    assert(q.top() == 3);
+int main(int, char**) {
+  std::stack<int> q;
+  q.push(1);
+  assert(q.size() == 1);
+  assert(q.top() == 1);
+  q.push(2);
+  assert(q.size() == 2);
+  assert(q.top() == 2);
+  q.push(3);
+  assert(q.size() == 3);
+  assert(q.top() == 3);
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/container.adaptors/stack/stack.defn/push_rv.pass.cpp b/libcxx/test/std/containers/container.adaptors/stack/stack.defn/push_rv.pass.cpp
index 7eeaba7c8685a..12771feccd162 100644
--- a/libcxx/test/std/containers/container.adaptors/stack/stack.defn/push_rv.pass.cpp
+++ b/libcxx/test/std/containers/container.adaptors/stack/stack.defn/push_rv.pass.cpp
@@ -18,18 +18,17 @@
 #include "test_macros.h"
 #include "MoveOnly.h"
 
-int main(int, char**)
-{
-    std::stack<MoveOnly> q;
-    q.push(MoveOnly(1));
-    assert(q.size() == 1);
-    assert(q.top() == MoveOnly(1));
-    q.push(MoveOnly(2));
-    assert(q.size() == 2);
-    assert(q.top() == MoveOnly(2));
-    q.push(MoveOnly(3));
-    assert(q.size() == 3);
-    assert(q.top() == MoveOnly(3));
+int main(int, char**) {
+  std::stack<MoveOnly> q;
+  q.push(MoveOnly(1));
+  assert(q.size() == 1);
+  assert(q.top() == MoveOnly(1));
+  q.push(MoveOnly(2));
+  assert(q.size() == 2);
+  assert(q.top() == MoveOnly(2));
+  q.push(MoveOnly(3));
+  assert(q.size() == 3);
+  assert(q.top() == MoveOnly(3));
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/container.adaptors/stack/stack.defn/size.pass.cpp b/libcxx/test/std/containers/container.adaptors/stack/stack.defn/size.pass.cpp
index 893984ae2297d..8a25d059d26f7 100644
--- a/libcxx/test/std/containers/container.adaptors/stack/stack.defn/size.pass.cpp
+++ b/libcxx/test/std/containers/container.adaptors/stack/stack.defn/size.pass.cpp
@@ -15,12 +15,11 @@
 
 #include "test_macros.h"
 
-int main(int, char**)
-{
-    std::stack<int> q;
-    assert(q.size() == 0);
-    q.push(1);
-    assert(q.size() == 1);
+int main(int, char**) {
+  std::stack<int> q;
+  assert(q.size() == 0);
+  q.push(1);
+  assert(q.size() == 1);
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/container.adaptors/stack/stack.defn/swap.pass.cpp b/libcxx/test/std/containers/container.adaptors/stack/stack.defn/swap.pass.cpp
index 560cc823d44fa..736a02ef98d38 100644
--- a/libcxx/test/std/containers/container.adaptors/stack/stack.defn/swap.pass.cpp
+++ b/libcxx/test/std/containers/container.adaptors/stack/stack.defn/swap.pass.cpp
@@ -16,24 +16,21 @@
 #include "test_macros.h"
 
 template <class C>
-C
-make(int n)
-{
-    C c;
-    for (int i = 0; i < n; ++i)
-        c.push(i);
-    return c;
+C make(int n) {
+  C c;
+  for (int i = 0; i < n; ++i)
+    c.push(i);
+  return c;
 }
 
-int main(int, char**)
-{
-    std::stack<int> q1 = make<std::stack<int> >(5);
-    std::stack<int> q2 = make<std::stack<int> >(10);
-    std::stack<int> q1_save = q1;
-    std::stack<int> q2_save = q2;
-    q1.swap(q2);
-    assert(q1 == q2_save);
-    assert(q2 == q1_save);
+int main(int, char**) {
+  std::stack<int> q1      = make<std::stack<int> >(5);
+  std::stack<int> q2      = make<std::stack<int> >(10);
+  std::stack<int> q1_save = q1;
+  std::stack<int> q2_save = q2;
+  q1.swap(q2);
+  assert(q1 == q2_save);
+  assert(q2 == q1_save);
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/container.adaptors/stack/stack.defn/top.pass.cpp b/libcxx/test/std/containers/container.adaptors/stack/stack.defn/top.pass.cpp
index c5ef96dc2bab8..2226bc557ab5a 100644
--- a/libcxx/test/std/containers/container.adaptors/stack/stack.defn/top.pass.cpp
+++ b/libcxx/test/std/containers/container.adaptors/stack/stack.defn/top.pass.cpp
@@ -15,15 +15,14 @@
 
 #include "test_macros.h"
 
-int main(int, char**)
-{
-    std::stack<int> q;
-    assert(q.size() == 0);
-    q.push(1);
-    q.push(2);
-    q.push(3);
-    int& ir = q.top();
-    assert(ir == 3);
+int main(int, char**) {
+  std::stack<int> q;
+  assert(q.size() == 0);
+  q.push(1);
+  q.push(2);
+  q.push(3);
+  int& ir = q.top();
+  assert(ir == 3);
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/container.adaptors/stack/stack.defn/top_const.pass.cpp b/libcxx/test/std/containers/container.adaptors/stack/stack.defn/top_const.pass.cpp
index 974e3419c262d..59f2a4b7495f2 100644
--- a/libcxx/test/std/containers/container.adaptors/stack/stack.defn/top_const.pass.cpp
+++ b/libcxx/test/std/containers/container.adaptors/stack/stack.defn/top_const.pass.cpp
@@ -15,16 +15,15 @@
 
 #include "test_macros.h"
 
-int main(int, char**)
-{
-    std::stack<int> q;
-    assert(q.size() == 0);
-    q.push(1);
-    q.push(2);
-    q.push(3);
-    const std::stack<int>& cqr = q;
-    const int& cir = cqr.top();
-    assert(cir == 3);
+int main(int, char**) {
+  std::stack<int> q;
+  assert(q.size() == 0);
+  q.push(1);
+  q.push(2);
+  q.push(3);
+  const std::stack<int>& cqr = q;
+  const int& cir             = cqr.top();
+  assert(cir == 3);
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/container.adaptors/stack/stack.defn/types.compile.fail.cpp b/libcxx/test/std/containers/container.adaptors/stack/stack.defn/types.compile.fail.cpp
index d5fe97fd5d75f..ddccbe090afe2 100644
--- a/libcxx/test/std/containers/container.adaptors/stack/stack.defn/types.compile.fail.cpp
+++ b/libcxx/test/std/containers/container.adaptors/stack/stack.defn/types.compile.fail.cpp
@@ -27,10 +27,9 @@
 #include <cassert>
 #include <type_traits>
 
-int main(int, char**)
-{
-//  LWG#2566 says that the first template param must match the second one's value type
-    std::stack<double, std::deque<int>> t;
+int main(int, char**) {
+  //  LWG#2566 says that the first template param must match the second one's value type
+  std::stack<double, std::deque<int>> t;
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/container.adaptors/stack/stack.defn/types.pass.cpp b/libcxx/test/std/containers/container.adaptors/stack/stack.defn/types.pass.cpp
index 944fd42ecccbf..0eae4da418f9a 100644
--- a/libcxx/test/std/containers/container.adaptors/stack/stack.defn/types.pass.cpp
+++ b/libcxx/test/std/containers/container.adaptors/stack/stack.defn/types.pass.cpp
@@ -29,34 +29,27 @@
 
 #include "test_macros.h"
 
-struct test
-    : private std::stack<int>
-{
-    test()
-    {
-        c.push_back(1);
-    }
+struct test : private std::stack<int> {
+  test() { c.push_back(1); }
 };
 
-struct C
-{
-    typedef int value_type;
-    typedef int& reference;
-    typedef const int& const_reference;
-    typedef int size_type;
+struct C {
+  typedef int value_type;
+  typedef int& reference;
+  typedef const int& const_reference;
+  typedef int size_type;
 };
 
-int main(int, char**)
-{
-    static_assert(( std::is_same<std::stack<int>::container_type, std::deque<int> >::value), "");
-    static_assert(( std::is_same<std::stack<int, std::vector<int> >::container_type, std::vector<int> >::value), "");
-    static_assert(( std::is_same<std::stack<int, std::vector<int> >::value_type, int>::value), "");
-    static_assert(( std::is_same<std::stack<int>::reference, std::deque<int>::reference>::value), "");
-    static_assert(( std::is_same<std::stack<int>::const_reference, std::deque<int>::const_reference>::value), "");
-    static_assert(( std::is_same<std::stack<int>::size_type, std::deque<int>::size_type>::value), "");
-    static_assert(( std::uses_allocator<std::stack<int>, std::allocator<int> >::value), "");
-    static_assert((!std::uses_allocator<std::stack<int, C>, std::allocator<int> >::value), "");
-    test t;
+int main(int, char**) {
+  static_assert((std::is_same<std::stack<int>::container_type, std::deque<int> >::value), "");
+  static_assert((std::is_same<std::stack<int, std::vector<int> >::container_type, std::vector<int> >::value), "");
+  static_assert((std::is_same<std::stack<int, std::vector<int> >::value_type, int>::value), "");
+  static_assert((std::is_same<std::stack<int>::reference, std::deque<int>::reference>::value), "");
+  static_assert((std::is_same<std::stack<int>::const_reference, std::deque<int>::const_reference>::value), "");
+  static_assert((std::is_same<std::stack<int>::size_type, std::deque<int>::size_type>::value), "");
+  static_assert((std::uses_allocator<std::stack<int>, std::allocator<int> >::value), "");
+  static_assert((!std::uses_allocator<std::stack<int, C>, std::allocator<int> >::value), "");
+  test t;
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/container.adaptors/stack/stack.ops/eq.pass.cpp b/libcxx/test/std/containers/container.adaptors/stack/stack.ops/eq.pass.cpp
index c826046da96aa..31c15af27faaf 100644
--- a/libcxx/test/std/containers/container.adaptors/stack/stack.ops/eq.pass.cpp
+++ b/libcxx/test/std/containers/container.adaptors/stack/stack.ops/eq.pass.cpp
@@ -20,24 +20,21 @@
 #include "test_macros.h"
 
 template <class C>
-C
-make(int n)
-{
-    C c;
-    for (int i = 0; i < n; ++i)
-        c.push(i);
-    return c;
+C make(int n) {
+  C c;
+  for (int i = 0; i < n; ++i)
+    c.push(i);
+  return c;
 }
 
-int main(int, char**)
-{
-    std::stack<int> q1 = make<std::stack<int> >(5);
-    std::stack<int> q2 = make<std::stack<int> >(10);
-    std::stack<int> q1_save = q1;
-    std::stack<int> q2_save = q2;
-    assert(q1 == q1_save);
-    assert(q1 != q2);
-    assert(q2 == q2_save);
+int main(int, char**) {
+  std::stack<int> q1      = make<std::stack<int> >(5);
+  std::stack<int> q2      = make<std::stack<int> >(10);
+  std::stack<int> q1_save = q1;
+  std::stack<int> q2_save = q2;
+  assert(q1 == q1_save);
+  assert(q1 != q2);
+  assert(q2 == q2_save);
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/container.adaptors/stack/stack.ops/lt.pass.cpp b/libcxx/test/std/containers/container.adaptors/stack/stack.ops/lt.pass.cpp
index 5ac440e8e0247..51259aa59e4c1 100644
--- a/libcxx/test/std/containers/container.adaptors/stack/stack.ops/lt.pass.cpp
+++ b/libcxx/test/std/containers/container.adaptors/stack/stack.ops/lt.pass.cpp
@@ -26,23 +26,20 @@
 #include "test_macros.h"
 
 template <class C>
-C
-make(int n)
-{
-    C c;
-    for (int i = 0; i < n; ++i)
-        c.push(i);
-    return c;
+C make(int n) {
+  C c;
+  for (int i = 0; i < n; ++i)
+    c.push(i);
+  return c;
 }
 
-int main(int, char**)
-{
-    std::stack<int> q1 = make<std::stack<int> >(5);
-    std::stack<int> q2 = make<std::stack<int> >(10);
-    assert(q1 < q2);
-    assert(q2 > q1);
-    assert(q1 <= q2);
-    assert(q2 >= q1);
+int main(int, char**) {
+  std::stack<int> q1 = make<std::stack<int> >(5);
+  std::stack<int> q2 = make<std::stack<int> >(10);
+  assert(q1 < q2);
+  assert(q2 > q1);
+  assert(q1 <= q2);
+  assert(q2 >= q1);
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/container.adaptors/stack/stack.special/swap.pass.cpp b/libcxx/test/std/containers/container.adaptors/stack/stack.special/swap.pass.cpp
index 1f4a382eaaaa3..16e91d1e1ef06 100644
--- a/libcxx/test/std/containers/container.adaptors/stack/stack.special/swap.pass.cpp
+++ b/libcxx/test/std/containers/container.adaptors/stack/stack.special/swap.pass.cpp
@@ -17,24 +17,21 @@
 #include "test_macros.h"
 
 template <class C>
-C
-make(int n)
-{
-    C c;
-    for (int i = 0; i < n; ++i)
-        c.push(i);
-    return c;
+C make(int n) {
+  C c;
+  for (int i = 0; i < n; ++i)
+    c.push(i);
+  return c;
 }
 
-int main(int, char**)
-{
-    std::stack<int> q1 = make<std::stack<int> >(5);
-    std::stack<int> q2 = make<std::stack<int> >(10);
-    std::stack<int> q1_save = q1;
-    std::stack<int> q2_save = q2;
-    swap(q1, q2);
-    assert(q1 == q2_save);
-    assert(q2 == q1_save);
+int main(int, char**) {
+  std::stack<int> q1      = make<std::stack<int> >(5);
+  std::stack<int> q2      = make<std::stack<int> >(10);
+  std::stack<int> q1_save = q1;
+  std::stack<int> q2_save = q2;
+  swap(q1, q2);
+  assert(q1 == q2_save);
+  assert(q2 == q1_save);
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/container.adaptors/stack/stack.special/swap_noexcept.pass.cpp b/libcxx/test/std/containers/container.adaptors/stack/stack.special/swap_noexcept.pass.cpp
index 041944b303554..f2e6be77f16f5 100644
--- a/libcxx/test/std/containers/container.adaptors/stack/stack.special/swap_noexcept.pass.cpp
+++ b/libcxx/test/std/containers/container.adaptors/stack/stack.special/swap_noexcept.pass.cpp
@@ -22,12 +22,11 @@
 #include "test_macros.h"
 #include "MoveOnly.h"
 
-int main(int, char**)
-{
-    {
-        typedef std::stack<MoveOnly> C;
-        static_assert(noexcept(swap(std::declval<C&>(), std::declval<C&>())), "");
-    }
+int main(int, char**) {
+  {
+    typedef std::stack<MoveOnly> C;
+    static_assert(noexcept(swap(std::declval<C&>(), std::declval<C&>())), "");
+  }
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/container.node/node_handle.nodiscard.verify.cpp b/libcxx/test/std/containers/container.node/node_handle.nodiscard.verify.cpp
index 4c47563f6c446..8d01c8edd8f52 100644
--- a/libcxx/test/std/containers/container.node/node_handle.nodiscard.verify.cpp
+++ b/libcxx/test/std/containers/container.node/node_handle.nodiscard.verify.cpp
@@ -21,37 +21,37 @@
 
 void test() {
 #if TEST_STD_VER >= 17
-    {
-        std::map<int, int>::node_type node;
-        node.empty(); // expected-warning {{ignoring return value of function declared with 'nodiscard' attribute}}
-    }
-    {
-        std::multimap<int, int>::node_type node;
-        node.empty(); // expected-warning {{ignoring return value of function declared with 'nodiscard' attribute}}
-    }
+  {
+    std::map<int, int>::node_type node;
+    node.empty(); // expected-warning {{ignoring return value of function declared with 'nodiscard' attribute}}
+  }
+  {
+    std::multimap<int, int>::node_type node;
+    node.empty(); // expected-warning {{ignoring return value of function declared with 'nodiscard' attribute}}
+  }
 #endif // TEST_STD_VER >= 17
-    {
-        std::set<int> node;
-        node.empty(); // expected-warning {{ignoring return value of function declared with 'nodiscard' attribute}}
-    }
-    {
-        std::multiset<int> node;
-        node.empty(); // expected-warning {{ignoring return value of function declared with 'nodiscard' attribute}}
-    }
-    {
-        std::unordered_map<int, int> node;
-        node.empty(); // expected-warning {{ignoring return value of function declared with 'nodiscard' attribute}}
-    }
-    {
-        std::unordered_multimap<int, int> node;
-        node.empty(); // expected-warning {{ignoring return value of function declared with 'nodiscard' attribute}}
-    }
-    {
-        std::unordered_set<int> node;
-        node.empty(); // expected-warning {{ignoring return value of function declared with 'nodiscard' attribute}}
-    }
-    {
-        std::unordered_multiset<int> node;
-        node.empty(); // expected-warning {{ignoring return value of function declared with 'nodiscard' attribute}}
-    }
+  {
+    std::set<int> node;
+    node.empty(); // expected-warning {{ignoring return value of function declared with 'nodiscard' attribute}}
+  }
+  {
+    std::multiset<int> node;
+    node.empty(); // expected-warning {{ignoring return value of function declared with 'nodiscard' attribute}}
+  }
+  {
+    std::unordered_map<int, int> node;
+    node.empty(); // expected-warning {{ignoring return value of function declared with 'nodiscard' attribute}}
+  }
+  {
+    std::unordered_multimap<int, int> node;
+    node.empty(); // expected-warning {{ignoring return value of function declared with 'nodiscard' attribute}}
+  }
+  {
+    std::unordered_set<int> node;
+    node.empty(); // expected-warning {{ignoring return value of function declared with 'nodiscard' attribute}}
+  }
+  {
+    std::unordered_multiset<int> node;
+    node.empty(); // expected-warning {{ignoring return value of function declared with 'nodiscard' attribute}}
+  }
 }
diff --git a/libcxx/test/std/containers/container.node/node_handle.pass.cpp b/libcxx/test/std/containers/container.node/node_handle.pass.cpp
index de9564261a04f..3b1552cdaa5d4 100644
--- a/libcxx/test/std/containers/container.node/node_handle.pass.cpp
+++ b/libcxx/test/std/containers/container.node/node_handle.pass.cpp
@@ -17,46 +17,46 @@
 
 // [container.node.overview] Table 83.
 template <class K, class T, class C1, class C2, class H1, class H2, class E1, class E2, class A_set, class A_map>
-struct node_compatibility_table
-{
-    static constexpr bool value =
-        std::is_same_v<typename std::map<K, T, C1, A_map>::node_type,               typename std::map<K, T, C2, A_map>::node_type> &&
-        std::is_same_v<typename std::map<K, T, C1, A_map>::node_type,               typename std::multimap<K, T, C2, A_map>::node_type> &&
-        std::is_same_v<typename std::set<K, C1, A_set>::node_type,                  typename std::set<K, C2, A_set>::node_type> &&
-        std::is_same_v<typename std::set<K, C1, A_set>::node_type,                  typename std::multiset<K, C2, A_set>::node_type> &&
-        std::is_same_v<typename std::unordered_map<K, T, H1, E1, A_map>::node_type, typename std::unordered_map<K, T, H2, E2, A_map>::node_type> &&
-        std::is_same_v<typename std::unordered_map<K, T, H1, E1, A_map>::node_type, typename std::unordered_multimap<K, T, H2, E2, A_map>::node_type> &&
-        std::is_same_v<typename std::unordered_set<K, H1, E1, A_set>::node_type,    typename std::unordered_set<K, H2, E2, A_set>::node_type> &&
-        std::is_same_v<typename std::unordered_set<K, H1, E1, A_set>::node_type,    typename std::unordered_multiset<K, H2, E2, A_set>::node_type>;
+struct node_compatibility_table {
+  // clang-format off
+  static constexpr bool value =
+      std::is_same_v<typename std::map<K, T, C1, A_map>::node_type,               typename std::map<K, T, C2, A_map>::node_type> &&
+      std::is_same_v<typename std::map<K, T, C1, A_map>::node_type,               typename std::multimap<K, T, C2, A_map>::node_type> &&
+      std::is_same_v<typename std::set<K, C1, A_set>::node_type,                  typename std::set<K, C2, A_set>::node_type> &&
+      std::is_same_v<typename std::set<K, C1, A_set>::node_type,                  typename std::multiset<K, C2, A_set>::node_type> &&
+      std::is_same_v<typename std::unordered_map<K, T, H1, E1, A_map>::node_type, typename std::unordered_map<K, T, H2, E2, A_map>::node_type> &&
+      std::is_same_v<typename std::unordered_map<K, T, H1, E1, A_map>::node_type, typename std::unordered_multimap<K, T, H2, E2, A_map>::node_type> &&
+      std::is_same_v<typename std::unordered_set<K, H1, E1, A_set>::node_type,    typename std::unordered_set<K, H2, E2, A_set>::node_type> &&
+      std::is_same_v<typename std::unordered_set<K, H1, E1, A_set>::node_type,    typename std::unordered_multiset<K, H2, E2, A_set>::node_type>;
+  // clang-format on
 };
 
-template <class T> struct my_hash
-{
-    using argument_type = T;
-    using result_type = std::size_t;
-    my_hash() = default;
-    std::size_t operator()(const T&) const {return 0;}
+template <class T>
+struct my_hash {
+  using argument_type = T;
+  using result_type   = std::size_t;
+  my_hash()           = default;
+  std::size_t operator()(const T&) const { return 0; }
 };
 
-template <class T> struct my_compare
-{
-    my_compare() = default;
-    bool operator()(const T&, const T&) const {return true;}
+template <class T>
+struct my_compare {
+  my_compare() = default;
+  bool operator()(const T&, const T&) const { return true; }
 };
 
-template <class T> struct my_equal
-{
-    my_equal() = default;
-    bool operator()(const T&, const T&) const {return true;}
+template <class T>
+struct my_equal {
+  my_equal() = default;
+  bool operator()(const T&, const T&) const { return true; }
 };
 
-struct Static
-{
-    Static() = default;
-    Static(const Static&) = delete;
-    Static(Static&&) = delete;
-    Static& operator=(const Static&) = delete;
-    Static& operator=(Static&&) = delete;
+struct Static {
+  Static()                         = default;
+  Static(const Static&)            = delete;
+  Static(Static&&)                 = delete;
+  Static& operator=(const Static&) = delete;
+  Static& operator=(Static&&)      = delete;
 };
 
 template <>
@@ -67,78 +67,93 @@ struct std::hash<Static> {
   std::size_t operator()(const Static&) const;
 };
 
-static_assert(node_compatibility_table<
-                  int, int, std::less<int>, std::less<int>, std::hash<int>,
-                  std::hash<int>, std::equal_to<int>, std::equal_to<int>,
-                  std::allocator<int>,
-                  std::allocator<std::pair<const int, int>>>::value,
-              "");
+static_assert(
+    node_compatibility_table< int,
+                              int,
+                              std::less<int>,
+                              std::less<int>,
+                              std::hash<int>,
+                              std::hash<int>,
+                              std::equal_to<int>,
+                              std::equal_to<int>,
+                              std::allocator<int>,
+                              std::allocator<std::pair<const int, int>>>::value,
+    "");
 
 static_assert(
-    node_compatibility_table<int, int, std::less<int>, my_compare<int>,
-                             std::hash<int>, my_hash<int>, std::equal_to<int>,
-                             my_equal<int>, std::allocator<int>,
+    node_compatibility_table<int,
+                             int,
+                             std::less<int>,
+                             my_compare<int>,
+                             std::hash<int>,
+                             my_hash<int>,
+                             std::equal_to<int>,
+                             my_equal<int>,
+                             std::allocator<int>,
                              std::allocator<std::pair<const int, int>>>::value,
     "");
 
-static_assert(node_compatibility_table<
-                  Static, int, my_compare<Static>, std::less<Static>,
-                  my_hash<Static>, std::hash<Static>, my_equal<Static>,
-                  std::equal_to<Static>, min_allocator<Static>,
-                  min_allocator<std::pair<const Static, int>>>::value,
-              "");
+static_assert(
+    node_compatibility_table< Static,
+                              int,
+                              my_compare<Static>,
+                              std::less<Static>,
+                              my_hash<Static>,
+                              std::hash<Static>,
+                              my_equal<Static>,
+                              std::equal_to<Static>,
+                              min_allocator<Static>,
+                              min_allocator<std::pair<const Static, int>>>::value,
+    "");
 
 template <class Container>
-void test_node_handle_operations()
-{
-    Container c;
-
-    typename Container::node_type nt1, nt2 = c.extract(c.emplace().first);
-    assert(nt2.get_allocator() == c.get_allocator());
-    assert(!nt2.empty());
-    assert(nt1.empty());
-    std::swap(nt1, nt2);
-    assert(nt1.get_allocator() == c.get_allocator());
-    assert(nt2.empty());
+void test_node_handle_operations() {
+  Container c;
+
+  typename Container::node_type nt1, nt2 = c.extract(c.emplace().first);
+  assert(nt2.get_allocator() == c.get_allocator());
+  assert(!nt2.empty());
+  assert(nt1.empty());
+  std::swap(nt1, nt2);
+  assert(nt1.get_allocator() == c.get_allocator());
+  assert(nt2.empty());
 }
 
 template <class Container>
-void test_node_handle_operations_multi()
-{
-    Container c;
-
-    typename Container::node_type nt1, nt2 = c.extract(c.emplace());
-    assert(nt2.get_allocator() == c.get_allocator());
-    assert(!nt2.empty());
-    assert(nt1.empty());
-    std::swap(nt1, nt2);
-    assert(nt1.get_allocator() == c.get_allocator());
-    assert(nt2.empty());
+void test_node_handle_operations_multi() {
+  Container c;
+
+  typename Container::node_type nt1, nt2 = c.extract(c.emplace());
+  assert(nt2.get_allocator() == c.get_allocator());
+  assert(!nt2.empty());
+  assert(nt1.empty());
+  std::swap(nt1, nt2);
+  assert(nt1.get_allocator() == c.get_allocator());
+  assert(nt2.empty());
 }
 
-template <class> void test_typedef() {}
+template <class>
+void test_typedef() {}
 
 template <class Container>
-void test_insert_return_type()
-{
-    test_typedef<typename Container::insert_return_type>();
+void test_insert_return_type() {
+  test_typedef<typename Container::insert_return_type>();
 }
 
-int main(int, char**)
-{
-    test_node_handle_operations<std::map<int, int>>();
-    test_node_handle_operations_multi<std::multimap<int, int>>();
-    test_node_handle_operations<std::set<int>>();
-    test_node_handle_operations_multi<std::multiset<int>>();
-    test_node_handle_operations<std::unordered_map<int, int>>();
-    test_node_handle_operations_multi<std::unordered_multimap<int, int>>();
-    test_node_handle_operations<std::unordered_set<int>>();
-    test_node_handle_operations_multi<std::unordered_multiset<int>>();
-
-    test_insert_return_type<std::map<int, int>>();
-    test_insert_return_type<std::set<int>>();
-    test_insert_return_type<std::unordered_map<int, int>>();
-    test_insert_return_type<std::unordered_set<int>>();
+int main(int, char**) {
+  test_node_handle_operations<std::map<int, int>>();
+  test_node_handle_operations_multi<std::multimap<int, int>>();
+  test_node_handle_operations<std::set<int>>();
+  test_node_handle_operations_multi<std::multiset<int>>();
+  test_node_handle_operations<std::unordered_map<int, int>>();
+  test_node_handle_operations_multi<std::unordered_multimap<int, int>>();
+  test_node_handle_operations<std::unordered_set<int>>();
+  test_node_handle_operations_multi<std::unordered_multiset<int>>();
+
+  test_insert_return_type<std::map<int, int>>();
+  test_insert_return_type<std::set<int>>();
+  test_insert_return_type<std::unordered_map<int, int>>();
+  test_insert_return_type<std::unordered_set<int>>();
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/container.requirements/associative.reqmts/associative.reqmts.except/nothing_to_do.pass.cpp b/libcxx/test/std/containers/container.requirements/associative.reqmts/associative.reqmts.except/nothing_to_do.pass.cpp
index 1f764da05d6b5..3e0d2a6570f42 100644
--- a/libcxx/test/std/containers/container.requirements/associative.reqmts/associative.reqmts.except/nothing_to_do.pass.cpp
+++ b/libcxx/test/std/containers/container.requirements/associative.reqmts/associative.reqmts.except/nothing_to_do.pass.cpp
@@ -6,8 +6,4 @@
 //
 //===----------------------------------------------------------------------===//
 
-int main(int, char**)
-{
-
-  return 0;
-}
+int main(int, char**) { return 0; }
diff --git a/libcxx/test/std/containers/container.requirements/container.requirements.dataraces/nothing_to_do.pass.cpp b/libcxx/test/std/containers/container.requirements/container.requirements.dataraces/nothing_to_do.pass.cpp
index 1f764da05d6b5..3e0d2a6570f42 100644
--- a/libcxx/test/std/containers/container.requirements/container.requirements.dataraces/nothing_to_do.pass.cpp
+++ b/libcxx/test/std/containers/container.requirements/container.requirements.dataraces/nothing_to_do.pass.cpp
@@ -6,8 +6,4 @@
 //
 //===----------------------------------------------------------------------===//
 
-int main(int, char**)
-{
-
-  return 0;
-}
+int main(int, char**) { return 0; }
diff --git a/libcxx/test/std/containers/container.requirements/container.requirements.general/allocator_move.pass.cpp b/libcxx/test/std/containers/container.requirements/container.requirements.general/allocator_move.pass.cpp
index 5f2f3cfe16588..5de5579d13067 100644
--- a/libcxx/test/std/containers/container.requirements/container.requirements.general/allocator_move.pass.cpp
+++ b/libcxx/test/std/containers/container.requirements/container.requirements.general/allocator_move.pass.cpp
@@ -93,16 +93,13 @@ int main(int, char**) {
 #else
     int stored_allocators = 1;
 #endif
-    test<std::unordered_set<int, std::hash<int>, std::equal_to<int>,
-                            test_allocator<int> > >(stored_allocators);
-    test<std::unordered_multiset<int, std::hash<int>, std::equal_to<int>,
-                                 test_allocator<int> > >(stored_allocators);
+    test<std::unordered_set<int, std::hash<int>, std::equal_to<int>, test_allocator<int> > >(stored_allocators);
+    test<std::unordered_multiset<int, std::hash<int>, std::equal_to<int>, test_allocator<int> > >(stored_allocators);
 
     using KV = std::pair<const int, int>;
-    test<std::unordered_map<int, int, std::hash<int>, std::equal_to<int>,
-                            test_allocator<KV> > >(stored_allocators);
-    test<std::unordered_multimap<int, int, std::hash<int>, std::equal_to<int>,
-                                 test_allocator<KV> > >(stored_allocators);
+    test<std::unordered_map<int, int, std::hash<int>, std::equal_to<int>, test_allocator<KV> > >(stored_allocators);
+    test<std::unordered_multimap<int, int, std::hash<int>, std::equal_to<int>, test_allocator<KV> > >(
+        stored_allocators);
   }
 
   return 0;
diff --git a/libcxx/test/std/containers/container.requirements/sequence.reqmts/nothing_to_do.pass.cpp b/libcxx/test/std/containers/container.requirements/sequence.reqmts/nothing_to_do.pass.cpp
index 1f764da05d6b5..3e0d2a6570f42 100644
--- a/libcxx/test/std/containers/container.requirements/sequence.reqmts/nothing_to_do.pass.cpp
+++ b/libcxx/test/std/containers/container.requirements/sequence.reqmts/nothing_to_do.pass.cpp
@@ -6,8 +6,4 @@
 //
 //===----------------------------------------------------------------------===//
 
-int main(int, char**)
-{
-
-  return 0;
-}
+int main(int, char**) { return 0; }
diff --git a/libcxx/test/std/containers/container.requirements/unord.req/unord.req.except/nothing_to_do.pass.cpp b/libcxx/test/std/containers/container.requirements/unord.req/unord.req.except/nothing_to_do.pass.cpp
index 1f764da05d6b5..3e0d2a6570f42 100644
--- a/libcxx/test/std/containers/container.requirements/unord.req/unord.req.except/nothing_to_do.pass.cpp
+++ b/libcxx/test/std/containers/container.requirements/unord.req/unord.req.except/nothing_to_do.pass.cpp
@@ -6,8 +6,4 @@
 //
 //===----------------------------------------------------------------------===//
 
-int main(int, char**)
-{
-
-  return 0;
-}
+int main(int, char**) { return 0; }
diff --git a/libcxx/test/std/containers/containers.general/nothing_to_do.pass.cpp b/libcxx/test/std/containers/containers.general/nothing_to_do.pass.cpp
index 1f764da05d6b5..3e0d2a6570f42 100644
--- a/libcxx/test/std/containers/containers.general/nothing_to_do.pass.cpp
+++ b/libcxx/test/std/containers/containers.general/nothing_to_do.pass.cpp
@@ -6,8 +6,4 @@
 //
 //===----------------------------------------------------------------------===//
 
-int main(int, char**)
-{
-
-  return 0;
-}
+int main(int, char**) { return 0; }
diff --git a/libcxx/test/std/containers/exception_safety_helpers.h b/libcxx/test/std/containers/exception_safety_helpers.h
index 08884160d19c9..da247571d303f 100644
--- a/libcxx/test/std/containers/exception_safety_helpers.h
+++ b/libcxx/test/std/containers/exception_safety_helpers.h
@@ -25,9 +25,7 @@ struct ThrowingCopy {
 
   ThrowingCopy() = default;
   ThrowingCopy(int value) : x(value) {}
-  ~ThrowingCopy() {
-    ++destroyed;
-  }
+  ~ThrowingCopy() { ++destroyed; }
 
   ThrowingCopy(const ThrowingCopy& other) : x(other.x) {
     ++created_by_copying;
@@ -45,9 +43,7 @@ struct ThrowingCopy {
   friend bool operator==(const ThrowingCopy& lhs, const ThrowingCopy& rhs) { return lhs.x == rhs.x; }
   friend bool operator<(const ThrowingCopy& lhs, const ThrowingCopy& rhs) { return lhs.x < rhs.x; }
 
-  static void reset() {
-    created_by_copying = destroyed = 0;
-  }
+  static void reset() { created_by_copying = destroyed = 0; }
 };
 
 template <int N>
@@ -59,9 +55,7 @@ int ThrowingCopy<N>::destroyed = 0;
 
 template <int N>
 struct std::hash<ThrowingCopy<N>> {
-  std::size_t operator()(const ThrowingCopy<N>& value) const {
-    return value.x;
-  }
+  std::size_t operator()(const ThrowingCopy<N>& value) const { return value.x; }
 };
 
 template <int ThrowOn, int Size, class Func>
@@ -84,7 +78,7 @@ void test_exception_safety_throwing_copy(Func&& func) {
 // complicate asserting that the expected number of elements was destroyed).
 template <class Container, int ThrowOn, int Size, class Func>
 void test_exception_safety_throwing_copy_container(Func&& func) {
-  using T = ThrowingCopy<ThrowOn>;
+  using T             = ThrowingCopy<ThrowOn>;
   T::throwing_enabled = false;
   T in[Size];
   Container c(in, in + Size);
diff --git a/libcxx/test/std/containers/from_range_helpers.h b/libcxx/test/std/containers/from_range_helpers.h
index e17ea247618bc..edf1d6ce528d7 100644
--- a/libcxx/test/std/containers/from_range_helpers.h
+++ b/libcxx/test/std/containers/from_range_helpers.h
@@ -51,7 +51,7 @@ constexpr auto wrap_input(std::vector<T>& input) {
 }
 
 struct KeyValue {
-  int key; // Only the key is considered for equality comparison.
+  int key;    // Only the key is considered for equality comparison.
   char value; // Allows distinguishing equivalent instances.
 
   bool operator<(const KeyValue& other) const { return key < other.key; }
@@ -60,17 +60,15 @@ struct KeyValue {
 
 template <>
 struct std::hash<KeyValue> {
-  std::size_t operator()(const KeyValue& kv) const {
-    return kv.key;
-  }
+  std::size_t operator()(const KeyValue& kv) const { return kv.key; }
 };
 
 #if !defined(TEST_HAS_NO_EXCEPTIONS)
 
 template <class T>
 struct ThrowingAllocator {
-  using value_type = T;
-  using char_type = T;
+  using value_type      = T;
+  using char_type       = T;
   using is_always_equal = std::false_type;
 
   ThrowingAllocator() = default;
@@ -90,14 +88,13 @@ struct ThrowingAllocator {
 
 template <class T, class Func>
 constexpr void for_all_iterators_and_allocators(Func f) {
-  using Iterators = types::type_list<
-    cpp20_input_iterator<T*>,
-    forward_iterator<T*>,
-    bidirectional_iterator<T*>,
-    random_access_iterator<T*>,
-    contiguous_iterator<T*>,
-    T*
-  >;
+  using Iterators =
+      types::type_list< cpp20_input_iterator<T*>,
+                        forward_iterator<T*>,
+                        bidirectional_iterator<T*>,
+                        random_access_iterator<T*>,
+                        contiguous_iterator<T*>,
+                        T* >;
 
   types::for_each(Iterators{}, [=]<class Iter>() {
     f.template operator()<Iter, sentinel_wrapper<Iter>, std::allocator<T>>();
diff --git a/libcxx/test/std/containers/insert_range_helpers.h b/libcxx/test/std/containers/insert_range_helpers.h
index 531a7df10c7f9..d7d89f5ea6503 100644
--- a/libcxx/test/std/containers/insert_range_helpers.h
+++ b/libcxx/test/std/containers/insert_range_helpers.h
@@ -33,7 +33,7 @@
 // `std::vector`).
 template <class T, std::size_t N = 32>
 class Buffer {
- public:
+public:
   constexpr Buffer() = default;
 
   constexpr Buffer(std::initializer_list<T> input) {
@@ -45,8 +45,9 @@ class Buffer {
   // Makes initializing `Buffer<char>` nicer -- allows writing `buf = "abc"` instead of `buf = {'a', 'b', 'c'}`.
   // To make the two forms equivalent, omits the terminating null.
   template <std::size_t N2>
-  constexpr Buffer(const char (&input) [N2])
-  requires std::same_as<T, char> {
+  constexpr Buffer(const char (&input)[N2])
+    requires std::same_as<T, char>
+  {
     static_assert(N2 <= N);
     std::ranges::copy(input, data_);
     // Omit the terminating null.
@@ -57,9 +58,9 @@ class Buffer {
   constexpr const T* end() const { return data_ + size_; }
   constexpr std::size_t size() const { return size_; }
 
- private:
+private:
   std::size_t size_ = 0;
-  T data_[N] = {};
+  T data_[N]        = {};
 };
 
 template <class T>
@@ -72,14 +73,13 @@ struct TestCase {
 
 template <class T, class PtrT, class Func>
 constexpr void for_all_iterators_and_allocators(Func f) {
-  using Iterators = types::type_list<
-    cpp20_input_iterator<PtrT>,
-    forward_iterator<PtrT>,
-    bidirectional_iterator<PtrT>,
-    random_access_iterator<PtrT>,
-    contiguous_iterator<PtrT>,
-    PtrT
-  >;
+  using Iterators =
+      types::type_list< cpp20_input_iterator<PtrT>,
+                        forward_iterator<PtrT>,
+                        bidirectional_iterator<PtrT>,
+                        random_access_iterator<PtrT>,
+                        contiguous_iterator<PtrT>,
+                        PtrT >;
 
   types::for_each(Iterators{}, [=]<class Iter>() {
     f.template operator()<Iter, sentinel_wrapper<Iter>, std::allocator<T>>();
@@ -100,11 +100,7 @@ constexpr void for_all_iterators_and_allocators(Func f) {
 // too long.
 template <class T, class PtrT, class Func>
 constexpr void for_all_iterators_and_allocators_constexpr(Func f) {
-  using Iterators = types::type_list<
-    cpp20_input_iterator<PtrT>,
-    forward_iterator<PtrT>,
-    PtrT
-  >;
+  using Iterators = types::type_list< cpp20_input_iterator<PtrT>, forward_iterator<PtrT>, PtrT >;
 
   types::for_each(Iterators{}, [=]<class Iter>() {
     f.template operator()<Iter, sentinel_wrapper<Iter>, std::allocator<T>>();
diff --git a/libcxx/test/std/containers/insert_range_maps_sets.h b/libcxx/test/std/containers/insert_range_maps_sets.h
index 82fea93b68fe3..6e344d8ad3b0f 100644
--- a/libcxx/test/std/containers/insert_range_maps_sets.h
+++ b/libcxx/test/std/containers/insert_range_maps_sets.h
@@ -31,9 +31,7 @@
 #include "type_algorithms.h"
 
 template <class Container, class Range>
-concept HasInsertRange = requires (Container& c, Range&& range) {
-  c.insert_range(range);
-};
+concept HasInsertRange = requires(Container& c, Range&& range) { c.insert_range(range); };
 
 template <template <class...> class Container, class T, class U>
 constexpr bool test_set_constraints_insert_range() {
@@ -79,141 +77,127 @@ struct TestCaseMapSet {
 // Empty container.
 
 template <class T>
-TestCaseMapSet<T> constexpr EmptyContainer_EmptyRange {
-  .initial = {}, .input = {}, .expected = {}
-};
+TestCaseMapSet<T> constexpr EmptyContainer_EmptyRange{.initial = {}, .input = {}, .expected = {}};
 
 template <class T>
-TestCaseMapSet<T> constexpr EmptyContainer_OneElementRange {
-  .initial = {}, .input = {1}, .expected = {1}
-};
-template <class K, class V> TestCaseMapSet<std::pair<K, V>>
-constexpr EmptyContainer_OneElementRange<std::pair<K, V>> {
-  .initial = {}, .input = {{1, 'a'}}, .expected = {{1, 'a'}}
-};
+TestCaseMapSet<T> constexpr EmptyContainer_OneElementRange{.initial = {}, .input = {1}, .expected = {1}};
+template <class K, class V>
+TestCaseMapSet<std::pair<K, V>> constexpr EmptyContainer_OneElementRange<std::pair<K, V>>{
+    .initial = {}, .input = {{1, 'a'}}, .expected = {{1, 'a'}}};
 
 template <class T>
-TestCaseMapSet<T> constexpr EmptyContainer_RangeNoDuplicates {
-  .initial = {}, .input = {5, 1, 3, 8, 6}, .expected = {5, 1, 3, 8, 6}
-};
-template <class K, class V> TestCaseMapSet<std::pair<K, V>>
-constexpr EmptyContainer_RangeNoDuplicates<std::pair<K, V>> {
-  .initial = {}, .input = {{5, 'a'}, {1, 'e'}, {3, 'i'}, {8, 'o'}, {6, 'u'}},
-  .expected = {{5, 'a'}, {1, 'e'}, {3, 'i'}, {8, 'o'}, {6, 'u'}}
-};
+TestCaseMapSet<T> constexpr EmptyContainer_RangeNoDuplicates{
+    .initial = {}, .input = {5, 1, 3, 8, 6}, .expected = {5, 1, 3, 8, 6}};
+template <class K, class V>
+TestCaseMapSet<std::pair<K, V>> constexpr EmptyContainer_RangeNoDuplicates<std::pair<K, V>>{
+    .initial  = {},
+    .input    = {{5, 'a'}, {1, 'e'}, {3, 'i'}, {8, 'o'}, {6, 'u'}},
+    .expected = {{5, 'a'}, {1, 'e'}, {3, 'i'}, {8, 'o'}, {6, 'u'}}};
 
 template <class T>
-TestCaseMapSet<T> constexpr EmptyContainer_RangeWithDuplicates {
-  .initial = {},
-  .input = {5, 1, 1, 3, 5, 8, 5, 6, 10},
-  .expected = {5, 1, 3, 8, 6, 10},
-  .expected_multi = {5, 1, 1, 3, 5, 8, 5, 6, 10}
-};
-template <class K, class V> TestCaseMapSet<std::pair<K, V>>
-constexpr EmptyContainer_RangeWithDuplicates<std::pair<K, V>> {
-  .initial = {},
-  .input = {{5, 'a'}, {1, 'a'}, {1, 'b'}, {3, 'a'}, {5, 'b'}, {8, 'a'}, {5, 'c'}, {6, 'a'}, {10, 'b'}},
-  .expected = {{5, 'a'}, {1, 'a'}, {3, 'a'}, {8, 'a'}, {6, 'a'}, {10, 'b'}},
-  .expected_multi = {{5, 'a'}, {1, 'a'}, {1, 'b'}, {3, 'a'}, {5, 'b'}, {8, 'a'}, {5, 'c'}, {6, 'a'}, {10, 'b'}}
-};
+TestCaseMapSet<T> constexpr EmptyContainer_RangeWithDuplicates{
+    .initial        = {},
+    .input          = {5, 1, 1, 3, 5, 8, 5, 6, 10},
+    .expected       = {5, 1, 3, 8, 6, 10},
+    .expected_multi = {5, 1, 1, 3, 5, 8, 5, 6, 10}};
+template <class K, class V>
+TestCaseMapSet<std::pair<K, V>> constexpr EmptyContainer_RangeWithDuplicates<std::pair<K, V>>{
+    .initial        = {},
+    .input          = {{5, 'a'}, {1, 'a'}, {1, 'b'}, {3, 'a'}, {5, 'b'}, {8, 'a'}, {5, 'c'}, {6, 'a'}, {10, 'b'}},
+    .expected       = {{5, 'a'}, {1, 'a'}, {3, 'a'}, {8, 'a'}, {6, 'a'}, {10, 'b'}},
+    .expected_multi = {{5, 'a'}, {1, 'a'}, {1, 'b'}, {3, 'a'}, {5, 'b'}, {8, 'a'}, {5, 'c'}, {6, 'a'}, {10, 'b'}}};
 
 // One-element container.
 
 template <class T>
-TestCaseMapSet<T> constexpr OneElementContainer_EmptyRange {
-  .initial = {10}, .input = {}, .expected = {10}
-};
-template <class K, class V> TestCaseMapSet<std::pair<K, V>>
-constexpr OneElementContainer_EmptyRange<std::pair<K, V>> {
-  .initial = {{10, 'A'}}, .input = {}, .expected = {{10, 'A'}}
-};
+TestCaseMapSet<T> constexpr OneElementContainer_EmptyRange{.initial = {10}, .input = {}, .expected = {10}};
+template <class K, class V>
+TestCaseMapSet<std::pair<K, V>> constexpr OneElementContainer_EmptyRange<std::pair<K, V>>{
+    .initial = {{10, 'A'}}, .input = {}, .expected = {{10, 'A'}}};
 
 template <class T>
-TestCaseMapSet<T> constexpr OneElementContainer_OneElementRange {
-  .initial = {10}, .input = {1}, .expected = {1, 10}
-};
-template <class K, class V> TestCaseMapSet<std::pair<K, V>>
-constexpr OneElementContainer_OneElementRange<std::pair<K, V>> {
-  .initial = {{10, 'A'}}, .input = {{1, 'a'}}, .expected = {{1, 'a'}, {10, 'A'}}
-};
+TestCaseMapSet<T> constexpr OneElementContainer_OneElementRange{.initial = {10}, .input = {1}, .expected = {1, 10}};
+template <class K, class V>
+TestCaseMapSet<std::pair<K, V>> constexpr OneElementContainer_OneElementRange<std::pair<K, V>>{
+    .initial = {{10, 'A'}}, .input = {{1, 'a'}}, .expected = {{1, 'a'}, {10, 'A'}}};
 
 template <class T>
-TestCaseMapSet<T> constexpr OneElementContainer_RangeNoDuplicates {
-  .initial = {10}, .input = {5, 1, 3, 8, 6}, .expected = {5, 1, 3, 8, 6, 10}
-};
-template <class K, class V> TestCaseMapSet<std::pair<K, V>>
-constexpr OneElementContainer_RangeNoDuplicates<std::pair<K, V>> {
-  .initial = {{10, 'A'}}, .input = {{5, 'a'}, {1, 'e'}, {3, 'i'}, {8, 'o'}, {6, 'u'}},
-  .expected = {{5, 'a'}, {1, 'e'}, {3, 'i'}, {8, 'o'}, {6, 'u'}, {10, 'A'}}
-};
+TestCaseMapSet<T> constexpr OneElementContainer_RangeNoDuplicates{
+    .initial = {10}, .input = {5, 1, 3, 8, 6}, .expected = {5, 1, 3, 8, 6, 10}};
+template <class K, class V>
+TestCaseMapSet<std::pair<K, V>> constexpr OneElementContainer_RangeNoDuplicates<std::pair<K, V>>{
+    .initial  = {{10, 'A'}},
+    .input    = {{5, 'a'}, {1, 'e'}, {3, 'i'}, {8, 'o'}, {6, 'u'}},
+    .expected = {{5, 'a'}, {1, 'e'}, {3, 'i'}, {8, 'o'}, {6, 'u'}, {10, 'A'}}};
 
 template <class T>
-TestCaseMapSet<T> constexpr OneElementContainer_RangeWithDuplicates {
-  .initial = {10},
-  .input = {5, 1, 1, 3, 5, 8, 5, 6, 10},
-  .expected = {5, 1, 3, 8, 6, 10},
-  .expected_multi = {5, 1, 1, 3, 5, 8, 5, 6, 10, 10}
-};
-template <class K, class V> TestCaseMapSet<std::pair<K, V>>
-constexpr OneElementContainer_RangeWithDuplicates<std::pair<K, V>> {
-  .initial = {{10, 'A'}},
-  .input = {{5, 'a'}, {1, 'a'}, {1, 'b'}, {3, 'a'}, {5, 'b'}, {8, 'a'}, {5, 'c'}, {6, 'a'}, {10, 'b'}},
-  .expected = {{5, 'a'}, {1, 'a'}, {3, 'a'}, {8, 'a'}, {6, 'a'}, {10, 'A'}},
-  .expected_multi = {
-    {5, 'a'}, {1, 'a'}, {1, 'b'}, {3, 'a'}, {5, 'b'}, {8, 'a'}, {5, 'c'}, {6, 'a'}, {10, 'A'}, {10, 'b'}
-  }
-};
+TestCaseMapSet<T> constexpr OneElementContainer_RangeWithDuplicates{
+    .initial        = {10},
+    .input          = {5, 1, 1, 3, 5, 8, 5, 6, 10},
+    .expected       = {5, 1, 3, 8, 6, 10},
+    .expected_multi = {5, 1, 1, 3, 5, 8, 5, 6, 10, 10}};
+template <class K, class V>
+TestCaseMapSet<std::pair<K, V>> constexpr OneElementContainer_RangeWithDuplicates<std::pair<K, V>>{
+    .initial        = {{10, 'A'}},
+    .input          = {{5, 'a'}, {1, 'a'}, {1, 'b'}, {3, 'a'}, {5, 'b'}, {8, 'a'}, {5, 'c'}, {6, 'a'}, {10, 'b'}},
+    .expected       = {{5, 'a'}, {1, 'a'}, {3, 'a'}, {8, 'a'}, {6, 'a'}, {10, 'A'}},
+    .expected_multi = {
+        {5, 'a'}, {1, 'a'}, {1, 'b'}, {3, 'a'}, {5, 'b'}, {8, 'a'}, {5, 'c'}, {6, 'a'}, {10, 'A'}, {10, 'b'}}};
 
 // N-elements container.
 
 template <class T>
-TestCaseMapSet<T> constexpr NElementsContainer_EmptyRange {
-  .initial = {10, 15, 19, 16}, .input = {}, .expected = {10, 15, 19, 16}
-};
-template <class K, class V> TestCaseMapSet<std::pair<K, V>>
-constexpr NElementsContainer_EmptyRange<std::pair<K, V>> {
-  .initial = {{10, 'A'}, {15, 'B'}, {19, 'C'}, {16, 'D'}}, .input = {},
-  .expected = {{10, 'A'}, {15, 'B'}, {19, 'C'}, {16, 'D'}}
-};
+TestCaseMapSet<T> constexpr NElementsContainer_EmptyRange{
+    .initial = {10, 15, 19, 16}, .input = {}, .expected = {10, 15, 19, 16}};
+template <class K, class V>
+TestCaseMapSet<std::pair<K, V>> constexpr NElementsContainer_EmptyRange<std::pair<K, V>>{
+    .initial  = {{10, 'A'}, {15, 'B'}, {19, 'C'}, {16, 'D'}},
+    .input    = {},
+    .expected = {{10, 'A'}, {15, 'B'}, {19, 'C'}, {16, 'D'}}};
 
 template <class T>
-TestCaseMapSet<T> constexpr NElementsContainer_OneElementRange {
-  .initial = {10, 15, 19, 16}, .input = {1}, .expected = {1, 10, 15, 19, 16}
-};
-template <class K, class V> TestCaseMapSet<std::pair<K, V>>
-constexpr NElementsContainer_OneElementRange<std::pair<K, V>> {
-  .initial = {{10, 'A'}, {15, 'B'}, {19, 'C'}, {16, 'D'}}, .input = {{1, 'a'}},
-  .expected = {{1, 'a'}, {10, 'A'}, {15, 'B'}, {19, 'C'}, {16, 'D'}}
-};
+TestCaseMapSet<T> constexpr NElementsContainer_OneElementRange{
+    .initial = {10, 15, 19, 16}, .input = {1}, .expected = {1, 10, 15, 19, 16}};
+template <class K, class V>
+TestCaseMapSet<std::pair<K, V>> constexpr NElementsContainer_OneElementRange<std::pair<K, V>>{
+    .initial  = {{10, 'A'}, {15, 'B'}, {19, 'C'}, {16, 'D'}},
+    .input    = {{1, 'a'}},
+    .expected = {{1, 'a'}, {10, 'A'}, {15, 'B'}, {19, 'C'}, {16, 'D'}}};
 
 template <class T>
-TestCaseMapSet<T> constexpr NElementsContainer_RangeNoDuplicates {
-  .initial = {10, 15, 19, 16}, .input = {5, 1, 3, 8, 6}, .expected = {5, 1, 3, 8, 6, 10, 15, 19, 16}
-};
-template <class K, class V> TestCaseMapSet<std::pair<K, V>>
-constexpr NElementsContainer_RangeNoDuplicates<std::pair<K, V>> {
-  .initial = {{10, 'A'}, {15, 'B'}, {19, 'C'}, {16, 'D'}},
-  .input = {{5, 'a'}, {1, 'e'}, {3, 'i'}, {8, 'o'}, {6, 'u'}},
-  .expected = {{5, 'a'}, {1, 'e'}, {3, 'i'}, {8, 'o'}, {6, 'u'}, {10, 'A'}, {15, 'B'}, {19, 'C'}, {16, 'D'}}
-};
+TestCaseMapSet<T> constexpr NElementsContainer_RangeNoDuplicates{
+    .initial = {10, 15, 19, 16}, .input = {5, 1, 3, 8, 6}, .expected = {5, 1, 3, 8, 6, 10, 15, 19, 16}};
+template <class K, class V>
+TestCaseMapSet<std::pair<K, V>> constexpr NElementsContainer_RangeNoDuplicates<std::pair<K, V>>{
+    .initial  = {{10, 'A'}, {15, 'B'}, {19, 'C'}, {16, 'D'}},
+    .input    = {{5, 'a'}, {1, 'e'}, {3, 'i'}, {8, 'o'}, {6, 'u'}},
+    .expected = {{5, 'a'}, {1, 'e'}, {3, 'i'}, {8, 'o'}, {6, 'u'}, {10, 'A'}, {15, 'B'}, {19, 'C'}, {16, 'D'}}};
 
 template <class T>
-TestCaseMapSet<T> constexpr NElementsContainer_RangeWithDuplicates {
-  .initial = {10, 15, 19, 16},
-  .input = {5, 1, 1, 3, 5, 8, 5, 6, 10},
-  .expected = {5, 1, 3, 8, 6, 10, 15, 19, 16},
-  .expected_multi = {5, 1, 1, 3, 5, 8, 5, 6, 10, 10, 15, 19, 16}
-};
-template <class K, class V> TestCaseMapSet<std::pair<K, V>>
-constexpr NElementsContainer_RangeWithDuplicates<std::pair<K, V>> {
-  .initial = {{10, 'A'}, {15, 'B'}, {19, 'C'}, {16, 'D'}},
-  .input = {{5, 'a'}, {1, 'a'}, {1, 'b'}, {3, 'a'}, {5, 'b'}, {8, 'a'}, {5, 'c'}, {6, 'a'}, {10, 'b'}},
-  .expected = {{5, 'a'}, {1, 'a'}, {3, 'a'}, {8, 'a'}, {6, 'a'}, {10, 'A'}, {15, 'B'}, {19, 'C'}, {16, 'D'}},
-  .expected_multi = {
-    {5, 'a'}, {1, 'a'}, {1, 'b'}, {3, 'a'}, {5, 'b'}, {8, 'a'}, {5, 'c'}, {6, 'a'}, {10, 'b'},
-    {10, 'A'}, {15, 'B'}, {19, 'C'}, {16, 'D'}
-  }
-};
+TestCaseMapSet<T> constexpr NElementsContainer_RangeWithDuplicates{
+    .initial        = {10, 15, 19, 16},
+    .input          = {5, 1, 1, 3, 5, 8, 5, 6, 10},
+    .expected       = {5, 1, 3, 8, 6, 10, 15, 19, 16},
+    .expected_multi = {5, 1, 1, 3, 5, 8, 5, 6, 10, 10, 15, 19, 16}};
+template <class K, class V>
+TestCaseMapSet<std::pair<K, V>> constexpr NElementsContainer_RangeWithDuplicates<std::pair<K, V>>{
+    .initial        = {{10, 'A'}, {15, 'B'}, {19, 'C'}, {16, 'D'}},
+    .input          = {{5, 'a'}, {1, 'a'}, {1, 'b'}, {3, 'a'}, {5, 'b'}, {8, 'a'}, {5, 'c'}, {6, 'a'}, {10, 'b'}},
+    .expected       = {{5, 'a'}, {1, 'a'}, {3, 'a'}, {8, 'a'}, {6, 'a'}, {10, 'A'}, {15, 'B'}, {19, 'C'}, {16, 'D'}},
+    .expected_multi = {
+        {5, 'a'},
+        {1, 'a'},
+        {1, 'b'},
+        {3, 'a'},
+        {5, 'b'},
+        {8, 'a'},
+        {5, 'c'},
+        {6, 'a'},
+        {10, 'b'},
+        {10, 'A'},
+        {15, 'B'},
+        {19, 'C'},
+        {16, 'D'}}};
 
 template <class Container, class T, class Iter, class Sent>
 void test_map_set_insert_range(bool allow_duplicates = false) {
@@ -265,7 +249,7 @@ void test_map_set_insert_range(bool allow_duplicates = false) {
 
 // Move-only types.
 
-template <template <class ...> class Container>
+template <template <class...> class Container>
 void test_set_insert_range_move_only() {
   MoveOnly input[5];
   std::ranges::subrange in(std::move_iterator{input}, std::move_iterator{input + 5});
@@ -274,7 +258,7 @@ void test_set_insert_range_move_only() {
   c.insert_range(in);
 }
 
-template <template <class ...> class Container>
+template <template <class...> class Container>
 void test_map_insert_range_move_only() {
   using Value = std::pair<const int, MoveOnly>;
   Value input[5];
@@ -286,7 +270,7 @@ void test_map_insert_range_move_only() {
 
 // Exception safety.
 
-template <template <class ...> class Container>
+template <template <class...> class Container>
 void test_set_insert_range_exception_safety_throwing_copy() {
 #if !defined(TEST_HAS_NO_EXCEPTIONS)
   using T = ThrowingCopy<3>;
@@ -305,17 +289,15 @@ void test_set_insert_range_exception_safety_throwing_copy() {
 #endif
 }
 
-template <template <class ...> class Container>
+template <template <class...> class Container>
 void test_map_insert_range_exception_safety_throwing_copy() {
 #if !defined(TEST_HAS_NO_EXCEPTIONS)
   using K = int;
   using V = ThrowingCopy<3>;
 
-  V::throwing_enabled = false;
-  std::pair<const K, V> in[5] = {
-    {1, {}}, {2, {}}, {3, {}}, {4, {}}, {5, {}}
-  };
-  V::throwing_enabled = true;
+  V::throwing_enabled         = false;
+  std::pair<const K, V> in[5] = {{1, {}}, {2, {}}, {3, {}}, {4, {}}, {5, {}}};
+  V::throwing_enabled         = true;
   V::reset();
 
   try {
@@ -330,7 +312,7 @@ void test_map_insert_range_exception_safety_throwing_copy() {
 #endif
 }
 
-template <template <class ...> class Container, class T>
+template <template <class...> class Container, class T>
 void test_assoc_set_insert_range_exception_safety_throwing_allocator() {
 #if !defined(TEST_HAS_NO_EXCEPTIONS)
   T in[] = {1, 2};
@@ -349,7 +331,7 @@ void test_assoc_set_insert_range_exception_safety_throwing_allocator() {
 #endif
 }
 
-template <template <class ...> class Container, class T>
+template <template <class...> class Container, class T>
 void test_unord_set_insert_range_exception_safety_throwing_allocator() {
 #if !defined(TEST_HAS_NO_EXCEPTIONS)
   T in[] = {1, 2};
@@ -368,13 +350,11 @@ void test_unord_set_insert_range_exception_safety_throwing_allocator() {
 #endif
 }
 
-template <template <class ...> class Container, class K, class V>
+template <template <class...> class Container, class K, class V>
 void test_assoc_map_insert_range_exception_safety_throwing_allocator() {
 #if !defined(TEST_HAS_NO_EXCEPTIONS)
   using ValueType = std::pair<const K, V>;
-  ValueType in[] = {
-    ValueType{K{1}, V{1}}
-  };
+  ValueType in[]  = {ValueType{K{1}, V{1}}};
 
   try {
     ThrowingAllocator<ValueType> alloc;
@@ -390,13 +370,11 @@ void test_assoc_map_insert_range_exception_safety_throwing_allocator() {
 #endif
 }
 
-template <template <class ...> class Container, class K, class V>
+template <template <class...> class Container, class K, class V>
 void test_unord_map_insert_range_exception_safety_throwing_allocator() {
 #if !defined(TEST_HAS_NO_EXCEPTIONS)
   using ValueType = std::pair<const K, V>;
-  ValueType in[] = {
-    ValueType{K{1}, V{1}}
-  };
+  ValueType in[]  = {ValueType{K{1}, V{1}}};
 
   try {
     ThrowingAllocator<ValueType> alloc;
diff --git a/libcxx/test/std/containers/iterator.rel_ops.compile.pass.cpp b/libcxx/test/std/containers/iterator.rel_ops.compile.pass.cpp
index 9db2449f2f166..2a8f99e321a78 100644
--- a/libcxx/test/std/containers/iterator.rel_ops.compile.pass.cpp
+++ b/libcxx/test/std/containers/iterator.rel_ops.compile.pass.cpp
@@ -29,73 +29,73 @@
 #include "test_macros.h"
 
 #if TEST_STD_VER >= 17
-#include <string_view>
+#  include <string_view>
 #endif
 
 #if TEST_STD_VER >= 20
-#include <span>
+#  include <span>
 #endif
 
 using namespace std::rel_ops;
 
-template<class It, class ConstIt>
+template <class It, class ConstIt>
 void test_eq(It it, ConstIt cit) {
-    (void)(it == it);
-    (void)(it != it);
-    (void)(it == cit);
-    (void)(it != cit);
-    (void)(cit == it);
-    (void)(cit != it);
-    (void)(cit == cit);
-    (void)(cit != cit);
+  (void)(it == it);
+  (void)(it != it);
+  (void)(it == cit);
+  (void)(it != cit);
+  (void)(cit == it);
+  (void)(cit != it);
+  (void)(cit == cit);
+  (void)(cit != cit);
 }
 
-template<class It, class ConstIt>
+template <class It, class ConstIt>
 void test_lt(It it, ConstIt cit) {
-    (void)(it <  it);
-    (void)(it <= it);
-    (void)(it >  it);
-    (void)(it >= it);
-    (void)(it <  cit);
-    (void)(it <= cit);
-    (void)(it >  cit);
-    (void)(it >= cit);
-    (void)(cit <  it);
-    (void)(cit <= it);
-    (void)(cit >  it);
-    (void)(cit >= it);
-    (void)(cit <  cit);
-    (void)(cit <= cit);
-    (void)(cit >  cit);
-    (void)(cit >= cit);
-
-    // Test subtraction too, even though std::rel_ops shouldn't affect it.
-
-    (void)(it - it);
-    (void)(it - cit);
-    (void)(cit - it);
-    (void)(cit - cit);
+  (void)(it < it);
+  (void)(it <= it);
+  (void)(it > it);
+  (void)(it >= it);
+  (void)(it < cit);
+  (void)(it <= cit);
+  (void)(it > cit);
+  (void)(it >= cit);
+  (void)(cit < it);
+  (void)(cit <= it);
+  (void)(cit > it);
+  (void)(cit >= it);
+  (void)(cit < cit);
+  (void)(cit <= cit);
+  (void)(cit > cit);
+  (void)(cit >= cit);
+
+  // Test subtraction too, even though std::rel_ops shouldn't affect it.
+
+  (void)(it - it);
+  (void)(it - cit);
+  (void)(cit - it);
+  (void)(cit - cit);
 }
 
-template<class Container>
+template <class Container>
 void test_forward() {
-    // There is no need to distinguish "forward" from "bidirectional."
-    // libc++ already can't handle `c.rbegin() >= c.rbegin()` in the
-    // presence of std::rel_ops, and neither can Microsoft nor libstdc++.
-
-    Container c;
-    typename Container::iterator it = c.begin();
-    typename Container::const_iterator cit = c.begin();
-    test_eq(it, cit);
+  // There is no need to distinguish "forward" from "bidirectional."
+  // libc++ already can't handle `c.rbegin() >= c.rbegin()` in the
+  // presence of std::rel_ops, and neither can Microsoft nor libstdc++.
+
+  Container c;
+  typename Container::iterator it        = c.begin();
+  typename Container::const_iterator cit = c.begin();
+  test_eq(it, cit);
 }
 
-template<class Container>
+template <class Container>
 void test_random_access() {
-    Container c;
-    typename Container::iterator it = c.begin();
-    typename Container::const_iterator cit = c.begin();
-    test_eq(it, cit);
-    test_lt(it, cit);
+  Container c;
+  typename Container::iterator it        = c.begin();
+  typename Container::const_iterator cit = c.begin();
+  test_eq(it, cit);
+  test_lt(it, cit);
 }
 
 template void test_random_access<std::array<int, 10> >();
@@ -115,13 +115,13 @@ template void test_random_access<std::vector<int> >();
 
 #if TEST_STD_VER >= 17
 void test_directory_iterators() {
-#ifndef TEST_HAS_NO_FILESYSTEM
-    std::filesystem::directory_iterator it;
-    test_eq(it, it);
+#  ifndef TEST_HAS_NO_FILESYSTEM
+  std::filesystem::directory_iterator it;
+  test_eq(it, it);
 
-    std::filesystem::recursive_directory_iterator rdit;
-    test_eq(rdit, rdit);
-#endif
+  std::filesystem::recursive_directory_iterator rdit;
+  test_eq(rdit, rdit);
+#  endif
 }
 
 template void test_forward<std::filesystem::path>();
@@ -133,9 +133,9 @@ template void test_random_access<std::string_view>();
 
 #if TEST_STD_VER >= 20
 void test_span() {
-    std::span<int> c;
-    std::span<int>::iterator it = c.begin();  // span has no const_iterator
-    test_eq(it, it);
-    test_lt(it, it);
+  std::span<int> c;
+  std::span<int>::iterator it = c.begin(); // span has no const_iterator
+  test_eq(it, it);
+  test_lt(it, it);
 }
 #endif
diff --git a/libcxx/test/std/containers/map_allocator_requirement_test_templates.h b/libcxx/test/std/containers/map_allocator_requirement_test_templates.h
index 9a6f4be0b003a..74dc8b15928a9 100644
--- a/libcxx/test/std/containers/map_allocator_requirement_test_templates.h
+++ b/libcxx/test/std/containers/map_allocator_requirement_test_templates.h
@@ -28,10 +28,8 @@
 #include "count_new.h"
 #include "container_test_types.h"
 
-
 template <class Container>
-void testMapInsert()
-{
+void testMapInsert() {
   typedef typename Container::value_type ValueTp;
   ConstructController* cc = getConstructController();
   cc->reset();
@@ -102,7 +100,7 @@ void testMapInsert()
   {
     // Testing C::insert(std::initializer_list<ValueTp>)
     Container c;
-    std::initializer_list<ValueTp> il = { ValueTp(1, 1), ValueTp(2, 1) };
+    std::initializer_list<ValueTp> il = {ValueTp(1, 1), ValueTp(2, 1)};
     cc->expect<ValueTp const&>(2);
     c.insert(il);
     assert(!cc->unchecked());
@@ -114,7 +112,7 @@ void testMapInsert()
   {
     // Testing C::insert(Iter, Iter) for *Iter = value_type const&
     Container c;
-    const ValueTp ValueList[] = { ValueTp(1, 1), ValueTp(2, 1), ValueTp(3, 1) };
+    const ValueTp ValueList[] = {ValueTp(1, 1), ValueTp(2, 1), ValueTp(3, 1)};
     cc->expect<ValueTp const&>(3);
     c.insert(std::begin(ValueList), std::end(ValueList));
     assert(!cc->unchecked());
@@ -126,14 +124,13 @@ void testMapInsert()
   {
     // Testing C::insert(Iter, Iter) for *Iter = value_type&&
     Container c;
-    ValueTp ValueList[] = { ValueTp(1, 1), ValueTp(2, 1) , ValueTp(3, 1) };
+    ValueTp ValueList[] = {ValueTp(1, 1), ValueTp(2, 1), ValueTp(3, 1)};
     cc->expect<ValueTp&&>(3);
-    c.insert(std::move_iterator<ValueTp*>(std::begin(ValueList)),
-             std::move_iterator<ValueTp*>(std::end(ValueList)));
+    c.insert(std::move_iterator<ValueTp*>(std::begin(ValueList)), std::move_iterator<ValueTp*>(std::end(ValueList)));
     assert(!cc->unchecked());
     {
       DisableAllocationGuard g;
-      ValueTp ValueList2[] = { ValueTp(1, 1), ValueTp(2, 1) , ValueTp(3, 1) };
+      ValueTp ValueList2[] = {ValueTp(1, 1), ValueTp(2, 1), ValueTp(3, 1)};
       c.insert(std::move_iterator<ValueTp*>(std::begin(ValueList2)),
                std::move_iterator<ValueTp*>(std::end(ValueList2)));
     }
@@ -141,7 +138,7 @@ void testMapInsert()
   {
     // Testing C::insert(Iter, Iter) for *Iter = value_type&
     Container c;
-    ValueTp ValueList[] = { ValueTp(1, 1), ValueTp(2, 1) , ValueTp(3, 1) };
+    ValueTp ValueList[] = {ValueTp(1, 1), ValueTp(2, 1), ValueTp(3, 1)};
     cc->expect<ValueTp const&>(3);
     c.insert(std::begin(ValueList), std::end(ValueList));
     assert(!cc->unchecked());
@@ -152,10 +149,8 @@ void testMapInsert()
   }
 }
 
-
 template <class Container>
-void testMapInsertHint()
-{
+void testMapInsertHint() {
   typedef typename Container::value_type ValueTp;
   typedef typename Container::key_type Key;
   typedef typename Container::mapped_type Mapped;
@@ -281,14 +276,10 @@ void testMapInsertHint()
       assert(c.size() == 1);
     }
   }
-
-
 }
 
-
 template <class Container>
-void testMapEmplace()
-{
+void testMapEmplace() {
   typedef typename Container::value_type ValueTp;
   typedef typename Container::key_type Key;
   typedef typename Container::mapped_type Mapped;
@@ -431,10 +422,8 @@ void testMapEmplace()
   }
 }
 
-
 template <class Container>
-void testMapEmplaceHint()
-{
+void testMapEmplaceHint() {
   typedef typename Container::value_type ValueTp;
   typedef typename Container::key_type Key;
   typedef typename Container::mapped_type Mapped;
@@ -616,13 +605,10 @@ void testMapEmplaceHint()
       assert(!cc->unchecked());
     }
   }
-
 }
 
-
 template <class Container>
-void testMultimapInsert()
-{
+void testMultimapInsert() {
   typedef typename Container::value_type ValueTp;
   ConstructController* cc = getConstructController();
   cc->reset();
@@ -660,7 +646,7 @@ void testMultimapInsert()
   {
     // Testing C::insert(std::initializer_list<ValueTp>)
     Container c;
-    std::initializer_list<ValueTp> il = { ValueTp(1, 1), ValueTp(2, 1) };
+    std::initializer_list<ValueTp> il = {ValueTp(1, 1), ValueTp(2, 1)};
     cc->expect<ValueTp const&>(2);
     c.insert(il);
     assert(!cc->unchecked());
@@ -668,7 +654,7 @@ void testMultimapInsert()
   {
     // Testing C::insert(Iter, Iter) for *Iter = value_type const&
     Container c;
-    const ValueTp ValueList[] = { ValueTp(1, 1), ValueTp(2, 1), ValueTp(3, 1) };
+    const ValueTp ValueList[] = {ValueTp(1, 1), ValueTp(2, 1), ValueTp(3, 1)};
     cc->expect<ValueTp const&>(3);
     c.insert(std::begin(ValueList), std::end(ValueList));
     assert(!cc->unchecked());
@@ -676,26 +662,23 @@ void testMultimapInsert()
   {
     // Testing C::insert(Iter, Iter) for *Iter = value_type&&
     Container c;
-    ValueTp ValueList[] = { ValueTp(1, 1), ValueTp(2, 1) , ValueTp(3, 1) };
+    ValueTp ValueList[] = {ValueTp(1, 1), ValueTp(2, 1), ValueTp(3, 1)};
     cc->expect<ValueTp&&>(3);
-    c.insert(std::move_iterator<ValueTp*>(std::begin(ValueList)),
-             std::move_iterator<ValueTp*>(std::end(ValueList)));
+    c.insert(std::move_iterator<ValueTp*>(std::begin(ValueList)), std::move_iterator<ValueTp*>(std::end(ValueList)));
     assert(!cc->unchecked());
   }
   {
     // Testing C::insert(Iter, Iter) for *Iter = value_type&
     Container c;
-    ValueTp ValueList[] = { ValueTp(1, 1), ValueTp(2, 1) , ValueTp(3, 1) };
+    ValueTp ValueList[] = {ValueTp(1, 1), ValueTp(2, 1), ValueTp(3, 1)};
     cc->expect<ValueTp&>(3);
     c.insert(std::begin(ValueList), std::end(ValueList));
     assert(!cc->unchecked());
   }
 }
 
-
 template <class Container>
-void testMultimapInsertHint()
-{
+void testMultimapInsertHint() {
   typedef typename Container::value_type ValueTp;
   ConstructController* cc = getConstructController();
   cc->reset();
diff --git a/libcxx/test/std/containers/sequences/array/aggregate.pass.cpp b/libcxx/test/std/containers/sequences/array/aggregate.pass.cpp
index 9db25a9231bca..582c9a5485c4a 100644
--- a/libcxx/test/std/containers/sequences/array/aggregate.pass.cpp
+++ b/libcxx/test/std/containers/sequences/array/aggregate.pass.cpp
@@ -15,33 +15,35 @@
 #include <type_traits>
 
 template <typename T>
-void check_aggregate()
-{
-    static_assert(std::is_aggregate<std::array<T, 0> >::value, "");
-    static_assert(std::is_aggregate<std::array<T, 1> >::value, "");
-    static_assert(std::is_aggregate<std::array<T, 2> >::value, "");
-    static_assert(std::is_aggregate<std::array<T, 3> >::value, "");
-    static_assert(std::is_aggregate<std::array<T, 4> >::value, "");
+void check_aggregate() {
+  static_assert(std::is_aggregate<std::array<T, 0> >::value, "");
+  static_assert(std::is_aggregate<std::array<T, 1> >::value, "");
+  static_assert(std::is_aggregate<std::array<T, 2> >::value, "");
+  static_assert(std::is_aggregate<std::array<T, 3> >::value, "");
+  static_assert(std::is_aggregate<std::array<T, 4> >::value, "");
 }
 
-struct Empty { };
-struct Trivial { int i; int j; };
+struct Empty {};
+struct Trivial {
+  int i;
+  int j;
+};
 struct NonTrivial {
-    int i; int j;
-    NonTrivial(NonTrivial const&) { }
+  int i;
+  int j;
+  NonTrivial(NonTrivial const&) {}
 };
 
-int main(int, char**)
-{
-    check_aggregate<char>();
-    check_aggregate<int>();
-    check_aggregate<long>();
-    check_aggregate<float>();
-    check_aggregate<double>();
-    check_aggregate<long double>();
-    check_aggregate<Empty>();
-    check_aggregate<Trivial>();
-    check_aggregate<NonTrivial>();
+int main(int, char**) {
+  check_aggregate<char>();
+  check_aggregate<int>();
+  check_aggregate<long>();
+  check_aggregate<float>();
+  check_aggregate<double>();
+  check_aggregate<long double>();
+  check_aggregate<Empty>();
+  check_aggregate<Trivial>();
+  check_aggregate<NonTrivial>();
 
-    return 0;
+  return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/array/array.cons/deduct.pass.cpp b/libcxx/test/std/containers/sequences/array/array.cons/deduct.pass.cpp
index 30dd48e414b67..55ef17c86bca3 100644
--- a/libcxx/test/std/containers/sequences/array/array.cons/deduct.pass.cpp
+++ b/libcxx/test/std/containers/sequences/array/array.cons/deduct.pass.cpp
@@ -20,44 +20,42 @@
 
 #include "test_macros.h"
 
-constexpr bool tests()
-{
-    //  Test the explicit deduction guides
-    {
-        std::array arr{1,2,3};  // array(T, U...)
-        static_assert(std::is_same_v<decltype(arr), std::array<int, 3>>, "");
-        assert(arr[0] == 1);
-        assert(arr[1] == 2);
-        assert(arr[2] == 3);
-    }
-
-    {
-        const long l1 = 42;
-        std::array arr{1L, 4L, 9L, l1}; // array(T, U...)
-        static_assert(std::is_same_v<decltype(arr)::value_type, long>, "");
-        static_assert(arr.size() == 4, "");
-        assert(arr[0] == 1);
-        assert(arr[1] == 4);
-        assert(arr[2] == 9);
-        assert(arr[3] == l1);
-    }
-
-    //  Test the implicit deduction guides
-    {
-        std::array<double, 2> source = {4.0, 5.0};
-        std::array arr(source);   // array(array)
-        static_assert(std::is_same_v<decltype(arr), decltype(source)>, "");
-        static_assert(std::is_same_v<decltype(arr), std::array<double, 2>>, "");
-        assert(arr[0] == 4.0);
-        assert(arr[1] == 5.0);
-    }
-
-    return true;
+constexpr bool tests() {
+  //  Test the explicit deduction guides
+  {
+    std::array arr{1, 2, 3}; // array(T, U...)
+    static_assert(std::is_same_v<decltype(arr), std::array<int, 3>>, "");
+    assert(arr[0] == 1);
+    assert(arr[1] == 2);
+    assert(arr[2] == 3);
+  }
+
+  {
+    const long l1 = 42;
+    std::array arr{1L, 4L, 9L, l1}; // array(T, U...)
+    static_assert(std::is_same_v<decltype(arr)::value_type, long>, "");
+    static_assert(arr.size() == 4, "");
+    assert(arr[0] == 1);
+    assert(arr[1] == 4);
+    assert(arr[2] == 9);
+    assert(arr[3] == l1);
+  }
+
+  //  Test the implicit deduction guides
+  {
+    std::array<double, 2> source = {4.0, 5.0};
+    std::array arr(source); // array(array)
+    static_assert(std::is_same_v<decltype(arr), decltype(source)>, "");
+    static_assert(std::is_same_v<decltype(arr), std::array<double, 2>>, "");
+    assert(arr[0] == 4.0);
+    assert(arr[1] == 5.0);
+  }
+
+  return true;
 }
 
-int main(int, char**)
-{
-    tests();
-    static_assert(tests(), "");
-    return 0;
+int main(int, char**) {
+  tests();
+  static_assert(tests(), "");
+  return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/array/array.cons/deduct.verify.cpp b/libcxx/test/std/containers/sequences/array/array.cons/deduct.verify.cpp
index f59b761fad9c9..92dac11501185 100644
--- a/libcxx/test/std/containers/sequences/array/array.cons/deduct.verify.cpp
+++ b/libcxx/test/std/containers/sequences/array/array.cons/deduct.verify.cpp
@@ -14,18 +14,17 @@
 //
 //  Requires: (is_same_v<T, U> && ...) is true. Otherwise the program is ill-formed.
 
-
 #include <array>
 #include <cassert>
 #include <cstddef>
 
 #include "test_macros.h"
 
-int main(int, char**)
-{
-    {
-    std::array arr{1,2,3L}; // expected-error-re {{no viable constructor or deduction guide for deduction of template arguments of '{{(std::)?}}array'}}
-    }
+int main(int, char**) {
+  {
+    std::array arr{1, 2, 3L};
+    // expected-error-re@-1 {{no viable constructor or deduction guide for deduction of template arguments of '{{(std::)?}}array'}}
+  }
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/array/array.cons/implicit_copy.pass.cpp b/libcxx/test/std/containers/sequences/array/array.cons/implicit_copy.pass.cpp
index 0e1e77fc5f181..11f71052eea59 100644
--- a/libcxx/test/std/containers/sequences/array/array.cons/implicit_copy.pass.cpp
+++ b/libcxx/test/std/containers/sequences/array/array.cons/implicit_copy.pass.cpp
@@ -19,101 +19,102 @@
 // generated operator would be ill-formed; like in the case of a struct with a
 // const member.
 #if TEST_STD_VER < 11
-#   define TEST_NOT_COPY_ASSIGNABLE(T) ((void)0)
+#  define TEST_NOT_COPY_ASSIGNABLE(T) ((void)0)
 #else
-#   define TEST_NOT_COPY_ASSIGNABLE(T) static_assert(!std::is_copy_assignable<T>::value, "")
+#  define TEST_NOT_COPY_ASSIGNABLE(T) static_assert(!std::is_copy_assignable<T>::value, "")
 #endif
 
 struct NoDefault {
-    TEST_CONSTEXPR NoDefault(int) { }
+  TEST_CONSTEXPR NoDefault(int) {}
 };
 
 struct NonTrivialCopy {
-    TEST_CONSTEXPR NonTrivialCopy() { }
-    TEST_CONSTEXPR NonTrivialCopy(NonTrivialCopy const&) { }
-    TEST_CONSTEXPR_CXX14 NonTrivialCopy& operator=(NonTrivialCopy const&) { return *this; }
+  TEST_CONSTEXPR NonTrivialCopy() {}
+  TEST_CONSTEXPR NonTrivialCopy(NonTrivialCopy const&) {}
+  TEST_CONSTEXPR_CXX14 NonTrivialCopy& operator=(NonTrivialCopy const&) { return *this; }
 };
 
-TEST_CONSTEXPR_CXX14 bool tests()
-{
-    {
-        typedef std::array<double, 3> Array;
-        Array array = {1.1, 2.2, 3.3};
-        Array copy = array;
-        copy = array;
-        static_assert(std::is_copy_constructible<Array>::value, "");
-        static_assert(std::is_copy_assignable<Array>::value, "");
-    }
-    {
-        typedef std::array<double const, 3> Array;
-        Array array = {1.1, 2.2, 3.3};
-        Array copy = array; (void)copy;
-        static_assert(std::is_copy_constructible<Array>::value, "");
-        TEST_NOT_COPY_ASSIGNABLE(Array);
-    }
-    {
-        typedef std::array<double, 0> Array;
-        Array array = {};
-        Array copy = array;
-        copy = array;
-        static_assert(std::is_copy_constructible<Array>::value, "");
-        static_assert(std::is_copy_assignable<Array>::value, "");
-    }
-    {
-        // const arrays of size 0 should disable the implicit copy assignment operator.
-        typedef std::array<double const, 0> Array;
-        Array array = {};
-        Array copy = array; (void)copy;
-        static_assert(std::is_copy_constructible<Array>::value, "");
-        TEST_NOT_COPY_ASSIGNABLE(Array);
-    }
-    {
-        typedef std::array<NoDefault, 0> Array;
-        Array array = {};
-        Array copy = array;
-        copy = array;
-        static_assert(std::is_copy_constructible<Array>::value, "");
-        static_assert(std::is_copy_assignable<Array>::value, "");
-    }
-    {
-        typedef std::array<NoDefault const, 0> Array;
-        Array array = {};
-        Array copy = array; (void)copy;
-        static_assert(std::is_copy_constructible<Array>::value, "");
-        TEST_NOT_COPY_ASSIGNABLE(Array);
-    }
+TEST_CONSTEXPR_CXX14 bool tests() {
+  {
+    typedef std::array<double, 3> Array;
+    Array array = {1.1, 2.2, 3.3};
+    Array copy  = array;
+    copy        = array;
+    static_assert(std::is_copy_constructible<Array>::value, "");
+    static_assert(std::is_copy_assignable<Array>::value, "");
+  }
+  {
+    typedef std::array<double const, 3> Array;
+    Array array = {1.1, 2.2, 3.3};
+    Array copy  = array;
+    (void)copy;
+    static_assert(std::is_copy_constructible<Array>::value, "");
+    TEST_NOT_COPY_ASSIGNABLE(Array);
+  }
+  {
+    typedef std::array<double, 0> Array;
+    Array array = {};
+    Array copy  = array;
+    copy        = array;
+    static_assert(std::is_copy_constructible<Array>::value, "");
+    static_assert(std::is_copy_assignable<Array>::value, "");
+  }
+  {
+    // const arrays of size 0 should disable the implicit copy assignment operator.
+    typedef std::array<double const, 0> Array;
+    Array array = {};
+    Array copy  = array;
+    (void)copy;
+    static_assert(std::is_copy_constructible<Array>::value, "");
+    TEST_NOT_COPY_ASSIGNABLE(Array);
+  }
+  {
+    typedef std::array<NoDefault, 0> Array;
+    Array array = {};
+    Array copy  = array;
+    copy        = array;
+    static_assert(std::is_copy_constructible<Array>::value, "");
+    static_assert(std::is_copy_assignable<Array>::value, "");
+  }
+  {
+    typedef std::array<NoDefault const, 0> Array;
+    Array array = {};
+    Array copy  = array;
+    (void)copy;
+    static_assert(std::is_copy_constructible<Array>::value, "");
+    TEST_NOT_COPY_ASSIGNABLE(Array);
+  }
 
-    // Make sure we can implicitly copy a std::array of a non-trivially copyable type
-    {
-        typedef std::array<NonTrivialCopy, 0> Array;
-        Array array = {};
-        Array copy = array;
-        copy = array;
-        static_assert(std::is_copy_constructible<Array>::value, "");
-    }
-    {
-        typedef std::array<NonTrivialCopy, 1> Array;
-        Array array = {};
-        Array copy = array;
-        copy = array;
-        static_assert(std::is_copy_constructible<Array>::value, "");
-    }
-    {
-        typedef std::array<NonTrivialCopy, 2> Array;
-        Array array = {};
-        Array copy = array;
-        copy = array;
-        static_assert(std::is_copy_constructible<Array>::value, "");
-    }
+  // Make sure we can implicitly copy a std::array of a non-trivially copyable type
+  {
+    typedef std::array<NonTrivialCopy, 0> Array;
+    Array array = {};
+    Array copy  = array;
+    copy        = array;
+    static_assert(std::is_copy_constructible<Array>::value, "");
+  }
+  {
+    typedef std::array<NonTrivialCopy, 1> Array;
+    Array array = {};
+    Array copy  = array;
+    copy        = array;
+    static_assert(std::is_copy_constructible<Array>::value, "");
+  }
+  {
+    typedef std::array<NonTrivialCopy, 2> Array;
+    Array array = {};
+    Array copy  = array;
+    copy        = array;
+    static_assert(std::is_copy_constructible<Array>::value, "");
+  }
 
-    return true;
+  return true;
 }
 
-int main(int, char**)
-{
-    tests();
+int main(int, char**) {
+  tests();
 #if TEST_STD_VER >= 14
-    static_assert(tests(), "");
+  static_assert(tests(), "");
 #endif
-    return 0;
+  return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/array/array.cons/initialization.pass.cpp b/libcxx/test/std/containers/sequences/array/array.cons/initialization.pass.cpp
index a23211f5464d6..0bca4299c8516 100644
--- a/libcxx/test/std/containers/sequences/array/array.cons/initialization.pass.cpp
+++ b/libcxx/test/std/containers/sequences/array/array.cons/initialization.pass.cpp
@@ -13,178 +13,215 @@
 #include <type_traits>
 #include "test_macros.h"
 
-
 struct NoDefault {
-    TEST_CONSTEXPR NoDefault(int) { }
+  TEST_CONSTEXPR NoDefault(int) {}
 };
 
 struct test_initialization {
-    template <typename T>
-    TEST_CONSTEXPR_CXX14 void operator()() const
+  template <typename T>
+  TEST_CONSTEXPR_CXX14 void operator()() const {
+    // Check default initalization
     {
-        // Check default initalization
-        {
-            std::array<T, 0> a0; (void)a0;
-            // Before C++20, default initialization doesn't work inside constexpr for
-            // trivially default constructible types. This only apply to non-empty arrays,
-            // since empty arrays don't hold an element of type T.
+      std::array<T, 0> a0;
+      (void)a0;
+      // Before C++20, default initialization doesn't work inside constexpr for
+      // trivially default constructible types. This only apply to non-empty arrays,
+      // since empty arrays don't hold an element of type T.
 #if TEST_STD_VER < 20
-            if (!(TEST_IS_CONSTANT_EVALUATED && std::is_trivially_default_constructible<T>::value))
+      if (!(TEST_IS_CONSTANT_EVALUATED && std::is_trivially_default_constructible<T>::value))
 #endif
-            {
-              std::array<T, 1> a1;
-              (void)a1;
-              std::array<T, 2> a2;
-              (void)a2;
-              std::array<T, 3> a3;
-              (void)a3;
-            }
-
-            std::array<NoDefault, 0> nodefault; (void)nodefault;
-        }
-
-        // A const empty array can also be default-initialized regardless of the type
-        // it contains. For non-empty arrays, this doesn't work whenever T doesn't
-        // have a user-provided default constructor.
-        {
-            const std::array<T, 0> a0; (void)a0;
-            const std::array<NoDefault, 0> nodefault; (void)nodefault;
-        }
-
-        // Check direct-list-initialization syntax (introduced in C++11)
-    #if TEST_STD_VER >= 11
-        {
-            {
-                std::array<T, 0> a0_0{}; (void)a0_0;
-            }
-            {
-                std::array<T, 1> a1_0{}; (void)a1_0;
-                std::array<T, 1> a1_1{T()}; (void)a1_1;
-            }
-            {
-                std::array<T, 2> a2_0{}; (void)a2_0;
-                std::array<T, 2> a2_1{T()}; (void)a2_1;
-                std::array<T, 2> a2_2{T(), T()}; (void)a2_2;
-            }
-            {
-                std::array<T, 3> a3_0{}; (void)a3_0;
-                std::array<T, 3> a3_1{T()}; (void)a3_1;
-                std::array<T, 3> a3_2{T(), T()}; (void)a3_2;
-                std::array<T, 3> a3_3{T(), T(), T()}; (void)a3_3;
-            }
-
-            std::array<NoDefault, 0> nodefault{}; (void)nodefault;
-        }
-    #endif
-
-        // Check copy-list-initialization syntax
-        {
-            {
-                std::array<T, 0> a0_0 = {}; (void)a0_0;
-            }
-            {
-                std::array<T, 1> a1_0 = {}; (void)a1_0;
-                std::array<T, 1> a1_1 = {T()}; (void)a1_1;
-            }
-            {
-                std::array<T, 2> a2_0 = {}; (void)a2_0;
-                std::array<T, 2> a2_1 = {T()}; (void)a2_1;
-                std::array<T, 2> a2_2 = {T(), T()}; (void)a2_2;
-            }
-            {
-                std::array<T, 3> a3_0 = {}; (void)a3_0;
-                std::array<T, 3> a3_1 = {T()}; (void)a3_1;
-                std::array<T, 3> a3_2 = {T(), T()}; (void)a3_2;
-                std::array<T, 3> a3_3 = {T(), T(), T()}; (void)a3_3;
-            }
-
-            std::array<NoDefault, 0> nodefault = {}; (void)nodefault;
-        }
-
-        // Test aggregate initialization
-        {
-            {
-                std::array<T, 0> a0_0 = {{}}; (void)a0_0;
-            }
-            {
-                std::array<T, 1> a1_0 = {{}}; (void)a1_0;
-                std::array<T, 1> a1_1 = {{T()}}; (void)a1_1;
-            }
-            {
-                std::array<T, 2> a2_0 = {{}}; (void)a2_0;
-                std::array<T, 2> a2_1 = {{T()}}; (void)a2_1;
-                std::array<T, 2> a2_2 = {{T(), T()}}; (void)a2_2;
-            }
-            {
-                std::array<T, 3> a3_0 = {{}}; (void)a3_0;
-                std::array<T, 3> a3_1 = {{T()}}; (void)a3_1;
-                std::array<T, 3> a3_2 = {{T(), T()}}; (void)a3_2;
-                std::array<T, 3> a3_3 = {{T(), T(), T()}}; (void)a3_3;
-            }
-
-            // See http://wg21.link/LWG2157
-            std::array<NoDefault, 0> nodefault = {{}}; (void)nodefault;
-        }
+      {
+        std::array<T, 1> a1;
+        (void)a1;
+        std::array<T, 2> a2;
+        (void)a2;
+        std::array<T, 3> a3;
+        (void)a3;
+      }
+
+      std::array<NoDefault, 0> nodefault;
+      (void)nodefault;
     }
-};
 
-// Test construction from an initializer-list
-TEST_CONSTEXPR_CXX14 bool test_initializer_list()
-{
+    // A const empty array can also be default-initialized regardless of the type
+    // it contains. For non-empty arrays, this doesn't work whenever T doesn't
+    // have a user-provided default constructor.
     {
-        std::array<double, 3> const a3_0 = {};
-        assert(a3_0[0] == double());
-        assert(a3_0[1] == double());
-        assert(a3_0[2] == double());
+      const std::array<T, 0> a0;
+      (void)a0;
+      const std::array<NoDefault, 0> nodefault;
+      (void)nodefault;
     }
+
+    // Check direct-list-initialization syntax (introduced in C++11)
+#if TEST_STD_VER >= 11
     {
-        std::array<double, 3> const a3_1 = {1};
-        assert(a3_1[0] == double(1));
-        assert(a3_1[1] == double());
-        assert(a3_1[2] == double());
+      {
+        std::array<T, 0> a0_0{};
+        (void)a0_0;
+      }
+      {
+        std::array<T, 1> a1_0{};
+        (void)a1_0;
+        std::array<T, 1> a1_1{T()};
+        (void)a1_1;
+      }
+      {
+        std::array<T, 2> a2_0{};
+        (void)a2_0;
+        std::array<T, 2> a2_1{T()};
+        (void)a2_1;
+        std::array<T, 2> a2_2{T(), T()};
+        (void)a2_2;
+      }
+      {
+        std::array<T, 3> a3_0{};
+        (void)a3_0;
+        std::array<T, 3> a3_1{T()};
+        (void)a3_1;
+        std::array<T, 3> a3_2{T(), T()};
+        (void)a3_2;
+        std::array<T, 3> a3_3{T(), T(), T()};
+        (void)a3_3;
+      }
+
+      std::array<NoDefault, 0> nodefault{};
+      (void)nodefault;
     }
+#endif
+
+    // Check copy-list-initialization syntax
     {
-        std::array<double, 3> const a3_2 = {1, 2.2};
-        assert(a3_2[0] == double(1));
-        assert(a3_2[1] == 2.2);
-        assert(a3_2[2] == double());
+      {
+        std::array<T, 0> a0_0 = {};
+        (void)a0_0;
+      }
+      {
+        std::array<T, 1> a1_0 = {};
+        (void)a1_0;
+        std::array<T, 1> a1_1 = {T()};
+        (void)a1_1;
+      }
+      {
+        std::array<T, 2> a2_0 = {};
+        (void)a2_0;
+        std::array<T, 2> a2_1 = {T()};
+        (void)a2_1;
+        std::array<T, 2> a2_2 = {T(), T()};
+        (void)a2_2;
+      }
+      {
+        std::array<T, 3> a3_0 = {};
+        (void)a3_0;
+        std::array<T, 3> a3_1 = {T()};
+        (void)a3_1;
+        std::array<T, 3> a3_2 = {T(), T()};
+        (void)a3_2;
+        std::array<T, 3> a3_3 = {T(), T(), T()};
+        (void)a3_3;
+      }
+
+      std::array<NoDefault, 0> nodefault = {};
+      (void)nodefault;
     }
+
+    // Test aggregate initialization
     {
-        std::array<double, 3> const a3_3 = {1, 2, 3.5};
-        assert(a3_3[0] == double(1));
-        assert(a3_3[1] == double(2));
-        assert(a3_3[2] == 3.5);
+      {
+        std::array<T, 0> a0_0 = {{}};
+        (void)a0_0;
+      }
+      {
+        std::array<T, 1> a1_0 = {{}};
+        (void)a1_0;
+        std::array<T, 1> a1_1 = {{T()}};
+        (void)a1_1;
+      }
+      {
+        std::array<T, 2> a2_0 = {{}};
+        (void)a2_0;
+        std::array<T, 2> a2_1 = {{T()}};
+        (void)a2_1;
+        std::array<T, 2> a2_2 = {{T(), T()}};
+        (void)a2_2;
+      }
+      {
+        std::array<T, 3> a3_0 = {{}};
+        (void)a3_0;
+        std::array<T, 3> a3_1 = {{T()}};
+        (void)a3_1;
+        std::array<T, 3> a3_2 = {{T(), T()}};
+        (void)a3_2;
+        std::array<T, 3> a3_3 = {{T(), T(), T()}};
+        (void)a3_3;
+      }
+
+      // See http://wg21.link/LWG2157
+      std::array<NoDefault, 0> nodefault = {{}};
+      (void)nodefault;
     }
+  }
+};
 
-    return true;
+// Test construction from an initializer-list
+TEST_CONSTEXPR_CXX14 bool test_initializer_list() {
+  {
+    std::array<double, 3> const a3_0 = {};
+    assert(a3_0[0] == double());
+    assert(a3_0[1] == double());
+    assert(a3_0[2] == double());
+  }
+  {
+    std::array<double, 3> const a3_1 = {1};
+    assert(a3_1[0] == double(1));
+    assert(a3_1[1] == double());
+    assert(a3_1[2] == double());
+  }
+  {
+    std::array<double, 3> const a3_2 = {1, 2.2};
+    assert(a3_2[0] == double(1));
+    assert(a3_2[1] == 2.2);
+    assert(a3_2[2] == double());
+  }
+  {
+    std::array<double, 3> const a3_3 = {1, 2, 3.5};
+    assert(a3_3[0] == double(1));
+    assert(a3_3[1] == double(2));
+    assert(a3_3[2] == 3.5);
+  }
+
+  return true;
 }
 
-struct Empty { };
-struct Trivial { int i; int j; };
+struct Empty {};
+struct Trivial {
+  int i;
+  int j;
+};
 struct NonTrivial {
-    TEST_CONSTEXPR NonTrivial() { }
-    TEST_CONSTEXPR NonTrivial(NonTrivial const&) { }
+  TEST_CONSTEXPR NonTrivial() {}
+  TEST_CONSTEXPR NonTrivial(NonTrivial const&) {}
 };
 struct NonEmptyNonTrivial {
-    int i; int j;
-    TEST_CONSTEXPR NonEmptyNonTrivial() : i(22), j(33) { }
-    TEST_CONSTEXPR NonEmptyNonTrivial(NonEmptyNonTrivial const&) : i(22), j(33) { }
+  int i;
+  int j;
+  TEST_CONSTEXPR NonEmptyNonTrivial() : i(22), j(33) {}
+  TEST_CONSTEXPR NonEmptyNonTrivial(NonEmptyNonTrivial const&) : i(22), j(33) {}
 };
 
 template <typename F>
-TEST_CONSTEXPR_CXX14 bool with_all_types()
-{
-    F().template operator()<char>();
-    F().template operator()<int>();
-    F().template operator()<long>();
-    F().template operator()<float>();
-    F().template operator()<double>();
-    F().template operator()<long double>();
-    F().template operator()<Empty>();
-    F().template operator()<Trivial>();
-    F().template operator()<NonTrivial>();
-    F().template operator()<NonEmptyNonTrivial>();
-    return true;
+TEST_CONSTEXPR_CXX14 bool with_all_types() {
+  F().template operator()<char>();
+  F().template operator()<int>();
+  F().template operator()<long>();
+  F().template operator()<float>();
+  F().template operator()<double>();
+  F().template operator()<long double>();
+  F().template operator()<Empty>();
+  F().template operator()<Trivial>();
+  F().template operator()<NonTrivial>();
+  F().template operator()<NonEmptyNonTrivial>();
+  return true;
 }
 
 // This is a regression test -- previously, libc++ would implement empty arrays by
@@ -193,22 +230,21 @@ TEST_CONSTEXPR_CXX14 bool with_all_types()
 #if TEST_STD_VER >= 20
 template <class T>
 concept is_list_initializable_int = requires {
-    { T{123} };
+  { T{123} };
 };
 
-struct Foo { };
+struct Foo {};
 static_assert(!is_list_initializable_int<std::array<Foo, 0>>);
 static_assert(!is_list_initializable_int<std::array<Foo, 1>>);
 #endif
 
-int main(int, char**)
-{
-    with_all_types<test_initialization>();
-    test_initializer_list();
+int main(int, char**) {
+  with_all_types<test_initialization>();
+  test_initializer_list();
 #if TEST_STD_VER >= 14
-    static_assert(with_all_types<test_initialization>(), "");
-    static_assert(test_initializer_list(), "");
+  static_assert(with_all_types<test_initialization>(), "");
+  static_assert(test_initializer_list(), "");
 #endif
 
-    return 0;
+  return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/array/array.creation/to_array.pass.cpp b/libcxx/test/std/containers/sequences/array/array.creation/to_array.pass.cpp
index 778fae1ced383..c4a2123115a24 100644
--- a/libcxx/test/std/containers/sequences/array/array.creation/to_array.pass.cpp
+++ b/libcxx/test/std/containers/sequences/array/array.creation/to_array.pass.cpp
@@ -23,8 +23,7 @@
 #include "test_macros.h"
 #include "MoveOnly.h"
 
-constexpr bool tests()
-{
+constexpr bool tests() {
   //  Test deduced type.
   {
     auto arr = std::to_array({1, 2, 3});
@@ -36,7 +35,7 @@ constexpr bool tests()
 
   {
     const long l1 = 42;
-    auto arr = std::to_array({1L, 4L, 9L, l1});
+    auto arr      = std::to_array({1L, 4L, 9L, l1});
     ASSERT_SAME_TYPE(decltype(arr)::value_type, long);
     static_assert(arr.size() == 4, "");
     assert(arr[0] == 1);
@@ -57,7 +56,7 @@ constexpr bool tests()
 
   {
     double source[3] = {4.0, 5.0, 6.0};
-    auto arr = std::to_array(source);
+    auto arr         = std::to_array(source);
     ASSERT_SAME_TYPE(decltype(arr), std::array<double, 3>);
     assert(arr[0] == 4.0);
     assert(arr[1] == 5.0);
@@ -66,7 +65,7 @@ constexpr bool tests()
 
   {
     double source[3] = {4.0, 5.0, 6.0};
-    auto arr = std::to_array(std::move(source));
+    auto arr         = std::to_array(std::move(source));
     ASSERT_SAME_TYPE(decltype(arr), std::array<double, 3>);
     assert(arr[0] == 4.0);
     assert(arr[1] == 5.0);
@@ -116,8 +115,7 @@ constexpr bool tests()
   return true;
 }
 
-int main(int, char**)
-{
+int main(int, char**) {
   tests();
   static_assert(tests(), "");
   return 0;
diff --git a/libcxx/test/std/containers/sequences/array/array.data/data.pass.cpp b/libcxx/test/std/containers/sequences/array/array.data/data.pass.cpp
index b0f9c3a7815e0..34c1e0e8ab398 100644
--- a/libcxx/test/std/containers/sequences/array/array.data/data.pass.cpp
+++ b/libcxx/test/std/containers/sequences/array/array.data/data.pass.cpp
@@ -12,89 +12,87 @@
 
 #include <array>
 #include <cassert>
-#include <cstddef>       // for std::max_align_t
+#include <cstddef> // for std::max_align_t
 #include <cstdint>
 
 #include "test_macros.h"
 
 struct NoDefault {
-    TEST_CONSTEXPR NoDefault(int) { }
+  TEST_CONSTEXPR NoDefault(int) {}
 };
 
 #if TEST_STD_VER < 11
 struct natural_alignment {
-    long t1;
-    long long t2;
-    double t3;
-    long double t4;
+  long t1;
+  long long t2;
+  double t3;
+  long double t4;
 };
 #endif
 
-TEST_CONSTEXPR_CXX17 bool tests()
-{
-    {
-        typedef double T;
-        typedef std::array<T, 3> C;
-        C c = {1, 2, 3.5};
-        ASSERT_NOEXCEPT(c.data());
-        T* p = c.data();
-        assert(p[0] == 1);
-        assert(p[1] == 2);
-        assert(p[2] == 3.5);
-    }
-    {
-        typedef double T;
-        typedef std::array<T, 0> C;
-        C c = {};
-        ASSERT_NOEXCEPT(c.data());
-        T* p = c.data();
-        (void)p;
-    }
-    {
-        typedef double T;
-        typedef std::array<const T, 0> C;
-        C c = {{}};
-        ASSERT_NOEXCEPT(c.data());
-        const T* p = c.data();
-        (void)p;
-        static_assert((std::is_same<decltype(c.data()), const T*>::value), "");
-    }
-    {
-        typedef NoDefault T;
-        typedef std::array<T, 0> C;
-        C c = {};
-        ASSERT_NOEXCEPT(c.data());
-        T* p = c.data();
-        (void)p;
-    }
-    {
-        std::array<int, 5> c = {0, 1, 2, 3, 4};
-        assert(c.data() == &c[0]);
-        assert(*c.data() == c[0]);
-    }
+TEST_CONSTEXPR_CXX17 bool tests() {
+  {
+    typedef double T;
+    typedef std::array<T, 3> C;
+    C c = {1, 2, 3.5};
+    ASSERT_NOEXCEPT(c.data());
+    T* p = c.data();
+    assert(p[0] == 1);
+    assert(p[1] == 2);
+    assert(p[2] == 3.5);
+  }
+  {
+    typedef double T;
+    typedef std::array<T, 0> C;
+    C c = {};
+    ASSERT_NOEXCEPT(c.data());
+    T* p = c.data();
+    (void)p;
+  }
+  {
+    typedef double T;
+    typedef std::array<const T, 0> C;
+    C c = {{}};
+    ASSERT_NOEXCEPT(c.data());
+    const T* p = c.data();
+    (void)p;
+    static_assert((std::is_same<decltype(c.data()), const T*>::value), "");
+  }
+  {
+    typedef NoDefault T;
+    typedef std::array<T, 0> C;
+    C c = {};
+    ASSERT_NOEXCEPT(c.data());
+    T* p = c.data();
+    (void)p;
+  }
+  {
+    std::array<int, 5> c = {0, 1, 2, 3, 4};
+    assert(c.data() == &c[0]);
+    assert(*c.data() == c[0]);
+  }
 
-    return true;
+  return true;
 }
 
-int main(int, char**)
-{
-    tests();
+int main(int, char**) {
+  tests();
 #if TEST_STD_VER >= 17
-    static_assert(tests(), "");
+  static_assert(tests(), "");
 #endif
 
-    // Test the alignment of data()
-    {
+  // Test the alignment of data()
+  {
 #if TEST_STD_VER < 11
-        typedef natural_alignment T;
+    typedef natural_alignment T;
 #else
-        typedef std::max_align_t T;
+    typedef std::max_align_t T;
 #endif
-        typedef std::array<T, 0> C;
-        const C c = {};
-        const T* p = c.data();
-        std::uintptr_t pint = reinterpret_cast<std::uintptr_t>(p);
-        assert(pint % TEST_ALIGNOF(T) == 0);
-    }
-    return 0;
+    typedef std::array<T, 0> C;
+    const C c           = {};
+    const T* p          = c.data();
+    std::uintptr_t pint = reinterpret_cast<std::uintptr_t>(p);
+    assert(pint % TEST_ALIGNOF(T) == 0);
+  }
+  return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/array/array.data/data_const.pass.cpp b/libcxx/test/std/containers/sequences/array/array.data/data_const.pass.cpp
index f6c2e594cc991..41de212e68bf8 100644
--- a/libcxx/test/std/containers/sequences/array/array.data/data_const.pass.cpp
+++ b/libcxx/test/std/containers/sequences/array/array.data/data_const.pass.cpp
@@ -12,81 +12,79 @@
 
 #include <array>
 #include <cassert>
-#include <cstddef>       // for std::max_align_t
+#include <cstddef> // for std::max_align_t
 #include <cstdint>
 
 #include "test_macros.h"
 
 struct NoDefault {
-    TEST_CONSTEXPR NoDefault(int) { }
+  TEST_CONSTEXPR NoDefault(int) {}
 };
 
 #if TEST_STD_VER < 11
 struct natural_alignment {
-    long t1;
-    long long t2;
-    double t3;
-    long double t4;
+  long t1;
+  long long t2;
+  double t3;
+  long double t4;
 };
 #endif
 
-TEST_CONSTEXPR_CXX17 bool tests()
-{
-    {
-        typedef double T;
-        typedef std::array<T, 3> C;
-        const C c = {1, 2, 3.5};
-        ASSERT_NOEXCEPT(c.data());
-        const T* p = c.data();
-        assert(p[0] == 1);
-        assert(p[1] == 2);
-        assert(p[2] == 3.5);
-    }
-    {
-        typedef double T;
-        typedef std::array<T, 0> C;
-        const C c = {};
-        ASSERT_NOEXCEPT(c.data());
-        const T* p = c.data();
-        (void)p;
-    }
-    {
-        typedef NoDefault T;
-        typedef std::array<T, 0> C;
-        const C c = {};
-        ASSERT_NOEXCEPT(c.data());
-        const T* p = c.data();
-        (void)p;
-    }
-    {
-        std::array<int, 5> const c = {0, 1, 2, 3, 4};
-        assert(c.data() == &c[0]);
-        assert(*c.data() == c[0]);
-    }
+TEST_CONSTEXPR_CXX17 bool tests() {
+  {
+    typedef double T;
+    typedef std::array<T, 3> C;
+    const C c = {1, 2, 3.5};
+    ASSERT_NOEXCEPT(c.data());
+    const T* p = c.data();
+    assert(p[0] == 1);
+    assert(p[1] == 2);
+    assert(p[2] == 3.5);
+  }
+  {
+    typedef double T;
+    typedef std::array<T, 0> C;
+    const C c = {};
+    ASSERT_NOEXCEPT(c.data());
+    const T* p = c.data();
+    (void)p;
+  }
+  {
+    typedef NoDefault T;
+    typedef std::array<T, 0> C;
+    const C c = {};
+    ASSERT_NOEXCEPT(c.data());
+    const T* p = c.data();
+    (void)p;
+  }
+  {
+    std::array<int, 5> const c = {0, 1, 2, 3, 4};
+    assert(c.data() == &c[0]);
+    assert(*c.data() == c[0]);
+  }
 
-    return true;
+  return true;
 }
 
-int main(int, char**)
-{
-    tests();
+int main(int, char**) {
+  tests();
 #if TEST_STD_VER >= 17
-    static_assert(tests(), "");
+  static_assert(tests(), "");
 #endif
 
-    // Test the alignment of data()
-    {
+  // Test the alignment of data()
+  {
 #if TEST_STD_VER < 11
-        typedef natural_alignment T;
+    typedef natural_alignment T;
 #else
-        typedef std::max_align_t T;
+    typedef std::max_align_t T;
 #endif
-        typedef std::array<T, 0> C;
-        const C c = {};
-        const T* p = c.data();
-        std::uintptr_t pint = reinterpret_cast<std::uintptr_t>(p);
-        assert(pint % TEST_ALIGNOF(T) == 0);
-    }
+    typedef std::array<T, 0> C;
+    const C c           = {};
+    const T* p          = c.data();
+    std::uintptr_t pint = reinterpret_cast<std::uintptr_t>(p);
+    assert(pint % TEST_ALIGNOF(T) == 0);
+  }
 
-    return 0;
+  return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/array/array.fill/fill.pass.cpp b/libcxx/test/std/containers/sequences/array/array.fill/fill.pass.cpp
index b2d7c4a192409..2bdb01f74bbae 100644
--- a/libcxx/test/std/containers/sequences/array/array.fill/fill.pass.cpp
+++ b/libcxx/test/std/containers/sequences/array/array.fill/fill.pass.cpp
@@ -15,34 +15,32 @@
 
 #include "test_macros.h"
 
-TEST_CONSTEXPR_CXX20 bool tests()
-{
-    {
-        typedef double T;
-        typedef std::array<T, 3> C;
-        C c = {1, 2, 3.5};
-        c.fill(5.5);
-        assert(c.size() == 3);
-        assert(c[0] == 5.5);
-        assert(c[1] == 5.5);
-        assert(c[2] == 5.5);
-    }
+TEST_CONSTEXPR_CXX20 bool tests() {
+  {
+    typedef double T;
+    typedef std::array<T, 3> C;
+    C c = {1, 2, 3.5};
+    c.fill(5.5);
+    assert(c.size() == 3);
+    assert(c[0] == 5.5);
+    assert(c[1] == 5.5);
+    assert(c[2] == 5.5);
+  }
 
-    {
-        typedef double T;
-        typedef std::array<T, 0> C;
-        C c = {};
-        c.fill(5.5);
-        assert(c.size() == 0);
-    }
-    return true;
+  {
+    typedef double T;
+    typedef std::array<T, 0> C;
+    C c = {};
+    c.fill(5.5);
+    assert(c.size() == 0);
+  }
+  return true;
 }
 
-int main(int, char**)
-{
-    tests();
+int main(int, char**) {
+  tests();
 #if TEST_STD_VER >= 20
-    static_assert(tests(), "");
+  static_assert(tests(), "");
 #endif
-    return 0;
+  return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/array/array.size/size.pass.cpp b/libcxx/test/std/containers/sequences/array/array.size/size.pass.cpp
index d39edd1206413..ba0120021de8d 100644
--- a/libcxx/test/std/containers/sequences/array/array.size/size.pass.cpp
+++ b/libcxx/test/std/containers/sequences/array/array.size/size.pass.cpp
@@ -15,42 +15,41 @@
 
 #include "test_macros.h"
 
-int main(int, char**)
-{
-    {
-        typedef double T;
-        typedef std::array<T, 3> C;
-        C c = {1, 2, 3.5};
-        assert(c.size() == 3);
-        assert(c.max_size() == 3);
-        assert(!c.empty());
-    }
-    {
-        typedef double T;
-        typedef std::array<T, 0> C;
-        C c = {};
-        assert(c.size() == 0);
-        assert(c.max_size() == 0);
-        assert(c.empty());
-    }
+int main(int, char**) {
+  {
+    typedef double T;
+    typedef std::array<T, 3> C;
+    C c = {1, 2, 3.5};
+    assert(c.size() == 3);
+    assert(c.max_size() == 3);
+    assert(!c.empty());
+  }
+  {
+    typedef double T;
+    typedef std::array<T, 0> C;
+    C c = {};
+    assert(c.size() == 0);
+    assert(c.max_size() == 0);
+    assert(c.empty());
+  }
 #if TEST_STD_VER >= 11
-    {
-        typedef double T;
-        typedef std::array<T, 3> C;
-        constexpr C c = {1, 2, 3.5};
-        static_assert(c.size() == 3, "");
-        static_assert(c.max_size() == 3, "");
-        static_assert(!c.empty(), "");
-    }
-    {
-        typedef double T;
-        typedef std::array<T, 0> C;
-        constexpr C c = {};
-        static_assert(c.size() == 0, "");
-        static_assert(c.max_size() == 0, "");
-        static_assert(c.empty(), "");
-    }
+  {
+    typedef double T;
+    typedef std::array<T, 3> C;
+    constexpr C c = {1, 2, 3.5};
+    static_assert(c.size() == 3, "");
+    static_assert(c.max_size() == 3, "");
+    static_assert(!c.empty(), "");
+  }
+  {
+    typedef double T;
+    typedef std::array<T, 0> C;
+    constexpr C c = {};
+    static_assert(c.size() == 0, "");
+    static_assert(c.max_size() == 0, "");
+    static_assert(c.empty(), "");
+  }
 #endif
 
-    return 0;
+  return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/array/array.special/swap.pass.cpp b/libcxx/test/std/containers/sequences/array/array.special/swap.pass.cpp
index 94974295dfc10..594723da6d52a 100644
--- a/libcxx/test/std/containers/sequences/array/array.special/swap.pass.cpp
+++ b/libcxx/test/std/containers/sequences/array/array.special/swap.pass.cpp
@@ -17,77 +17,75 @@
 #include "test_macros.h"
 
 struct NonSwappable {
-    TEST_CONSTEXPR NonSwappable() { }
+  TEST_CONSTEXPR NonSwappable() {}
+
 private:
-    NonSwappable(NonSwappable const&);
-    NonSwappable& operator=(NonSwappable const&);
+  NonSwappable(NonSwappable const&);
+  NonSwappable& operator=(NonSwappable const&);
 };
 
 template <class Tp>
-decltype(swap(std::declval<Tp>(), std::declval<Tp>()))
-can_swap_imp(int);
+decltype(swap(std::declval<Tp>(), std::declval<Tp>())) can_swap_imp(int);
 
 template <class Tp>
 std::false_type can_swap_imp(...);
 
 template <class Tp>
-struct can_swap : std::is_same<decltype(can_swap_imp<Tp>(0)), void> { };
+struct can_swap : std::is_same<decltype(can_swap_imp<Tp>(0)), void> {};
 
-TEST_CONSTEXPR_CXX20 bool tests()
-{
-    {
-        typedef double T;
-        typedef std::array<T, 3> C;
-        C c1 = {1, 2, 3.5};
-        C c2 = {4, 5, 6.5};
-        swap(c1, c2);
-        assert(c1.size() == 3);
-        assert(c1[0] == 4);
-        assert(c1[1] == 5);
-        assert(c1[2] == 6.5);
-        assert(c2.size() == 3);
-        assert(c2[0] == 1);
-        assert(c2[1] == 2);
-        assert(c2[2] == 3.5);
-    }
-    {
-        typedef double T;
-        typedef std::array<T, 0> C;
-        C c1 = {};
-        C c2 = {};
-        swap(c1, c2);
-        assert(c1.size() == 0);
-        assert(c2.size() == 0);
-    }
-    {
-        typedef NonSwappable T;
-        typedef std::array<T, 0> C0;
-        static_assert(can_swap<C0&>::value, "");
-        C0 l = {};
-        C0 r = {};
-        swap(l, r);
+TEST_CONSTEXPR_CXX20 bool tests() {
+  {
+    typedef double T;
+    typedef std::array<T, 3> C;
+    C c1 = {1, 2, 3.5};
+    C c2 = {4, 5, 6.5};
+    swap(c1, c2);
+    assert(c1.size() == 3);
+    assert(c1[0] == 4);
+    assert(c1[1] == 5);
+    assert(c1[2] == 6.5);
+    assert(c2.size() == 3);
+    assert(c2[0] == 1);
+    assert(c2[1] == 2);
+    assert(c2[2] == 3.5);
+  }
+  {
+    typedef double T;
+    typedef std::array<T, 0> C;
+    C c1 = {};
+    C c2 = {};
+    swap(c1, c2);
+    assert(c1.size() == 0);
+    assert(c2.size() == 0);
+  }
+  {
+    typedef NonSwappable T;
+    typedef std::array<T, 0> C0;
+    static_assert(can_swap<C0&>::value, "");
+    C0 l = {};
+    C0 r = {};
+    swap(l, r);
 #if TEST_STD_VER >= 11
-        static_assert(noexcept(swap(l, r)), "");
+    static_assert(noexcept(swap(l, r)), "");
 #endif
-    }
+  }
 #if TEST_STD_VER >= 11
-    {
-        // NonSwappable is still considered swappable in C++03 because there
-        // is no access control SFINAE.
-        typedef NonSwappable T;
-        typedef std::array<T, 42> C1;
-        static_assert(!can_swap<C1&>::value, "");
-    }
+  {
+    // NonSwappable is still considered swappable in C++03 because there
+    // is no access control SFINAE.
+    typedef NonSwappable T;
+    typedef std::array<T, 42> C1;
+    static_assert(!can_swap<C1&>::value, "");
+  }
 #endif
 
-    return true;
+  return true;
 }
 
-int main(int, char**)
-{
-    tests();
+int main(int, char**) {
+  tests();
 #if TEST_STD_VER >= 20
-    static_assert(tests(), "");
+  static_assert(tests(), "");
 #endif
-    return 0;
+  return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/array/array.swap/swap.pass.cpp b/libcxx/test/std/containers/sequences/array/array.swap/swap.pass.cpp
index e0133ba8a2951..b67da087a4549 100644
--- a/libcxx/test/std/containers/sequences/array/array.swap/swap.pass.cpp
+++ b/libcxx/test/std/containers/sequences/array/array.swap/swap.pass.cpp
@@ -17,82 +17,81 @@
 #include "test_macros.h"
 
 struct NonSwappable {
-    TEST_CONSTEXPR NonSwappable() { }
+  TEST_CONSTEXPR NonSwappable() {}
+
 private:
-    NonSwappable(NonSwappable const&);
-    NonSwappable& operator=(NonSwappable const&);
+  NonSwappable(NonSwappable const&);
+  NonSwappable& operator=(NonSwappable const&);
 };
 
-TEST_CONSTEXPR_CXX20 bool tests()
-{
-    {
-        typedef double T;
-        typedef std::array<T, 3> C;
-        C c1 = {1, 2, 3.5};
-        C c2 = {4, 5, 6.5};
-        c1.swap(c2);
-        assert(c1.size() == 3);
-        assert(c1[0] == 4);
-        assert(c1[1] == 5);
-        assert(c1[2] == 6.5);
-        assert(c2.size() == 3);
-        assert(c2[0] == 1);
-        assert(c2[1] == 2);
-        assert(c2[2] == 3.5);
-    }
-    {
-        typedef double T;
-        typedef std::array<T, 3> C;
-        C c1 = {1, 2, 3.5};
-        C c2 = {4, 5, 6.5};
-        std::swap(c1, c2);
-        assert(c1.size() == 3);
-        assert(c1[0] == 4);
-        assert(c1[1] == 5);
-        assert(c1[2] == 6.5);
-        assert(c2.size() == 3);
-        assert(c2[0] == 1);
-        assert(c2[1] == 2);
-        assert(c2[2] == 3.5);
-    }
+TEST_CONSTEXPR_CXX20 bool tests() {
+  {
+    typedef double T;
+    typedef std::array<T, 3> C;
+    C c1 = {1, 2, 3.5};
+    C c2 = {4, 5, 6.5};
+    c1.swap(c2);
+    assert(c1.size() == 3);
+    assert(c1[0] == 4);
+    assert(c1[1] == 5);
+    assert(c1[2] == 6.5);
+    assert(c2.size() == 3);
+    assert(c2[0] == 1);
+    assert(c2[1] == 2);
+    assert(c2[2] == 3.5);
+  }
+  {
+    typedef double T;
+    typedef std::array<T, 3> C;
+    C c1 = {1, 2, 3.5};
+    C c2 = {4, 5, 6.5};
+    std::swap(c1, c2);
+    assert(c1.size() == 3);
+    assert(c1[0] == 4);
+    assert(c1[1] == 5);
+    assert(c1[2] == 6.5);
+    assert(c2.size() == 3);
+    assert(c2[0] == 1);
+    assert(c2[1] == 2);
+    assert(c2[2] == 3.5);
+  }
 
-    {
-        typedef double T;
-        typedef std::array<T, 0> C;
-        C c1 = {};
-        C c2 = {};
-        c1.swap(c2);
-        assert(c1.size() == 0);
-        assert(c2.size() == 0);
-    }
-    {
-        typedef double T;
-        typedef std::array<T, 0> C;
-        C c1 = {};
-        C c2 = {};
-        std::swap(c1, c2);
-        assert(c1.size() == 0);
-        assert(c2.size() == 0);
-    }
-    {
-        typedef NonSwappable T;
-        typedef std::array<T, 0> C0;
-        C0 l = {};
-        C0 r = {};
-        l.swap(r);
+  {
+    typedef double T;
+    typedef std::array<T, 0> C;
+    C c1 = {};
+    C c2 = {};
+    c1.swap(c2);
+    assert(c1.size() == 0);
+    assert(c2.size() == 0);
+  }
+  {
+    typedef double T;
+    typedef std::array<T, 0> C;
+    C c1 = {};
+    C c2 = {};
+    std::swap(c1, c2);
+    assert(c1.size() == 0);
+    assert(c2.size() == 0);
+  }
+  {
+    typedef NonSwappable T;
+    typedef std::array<T, 0> C0;
+    C0 l = {};
+    C0 r = {};
+    l.swap(r);
 #if TEST_STD_VER >= 11
-        static_assert(noexcept(l.swap(r)), "");
+    static_assert(noexcept(l.swap(r)), "");
 #endif
-    }
+  }
 
-    return true;
+  return true;
 }
 
-int main(int, char**)
-{
-    tests();
+int main(int, char**) {
+  tests();
 #if TEST_STD_VER >= 20
-    static_assert(tests(), "");
+  static_assert(tests(), "");
 #endif
   return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/array/array.swap/swap.verify.cpp b/libcxx/test/std/containers/sequences/array/array.swap/swap.verify.cpp
index 7816b26578246..4d0ee390726ca 100644
--- a/libcxx/test/std/containers/sequences/array/array.swap/swap.verify.cpp
+++ b/libcxx/test/std/containers/sequences/array/array.swap/swap.verify.cpp
@@ -19,7 +19,7 @@ int main(int, char**) {
   {
     typedef double T;
     typedef std::array<const T, 0> C;
-    C c = {};
+    C c  = {};
     C c2 = {};
     // expected-error-re@array:* {{static assertion failed{{.*}}cannot swap zero-sized array of type 'const T'}}
     c.swap(c2); // expected-note {{requested here}}
diff --git a/libcxx/test/std/containers/sequences/array/array.tuple/get.pass.cpp b/libcxx/test/std/containers/sequences/array/array.tuple/get.pass.cpp
index 04bb1313996df..58bba33250734 100644
--- a/libcxx/test/std/containers/sequences/array/array.tuple/get.pass.cpp
+++ b/libcxx/test/std/containers/sequences/array/array.tuple/get.pass.cpp
@@ -15,57 +15,54 @@
 
 #include "test_macros.h"
 
-template <typename ...T>
-TEST_CONSTEXPR std::array<int, sizeof...(T)> tempArray(T ...args)
-{
-    return {args...};
+template <typename... T>
+TEST_CONSTEXPR std::array<int, sizeof...(T)> tempArray(T... args) {
+  return {args...};
 }
 
-TEST_CONSTEXPR_CXX14 bool tests()
-{
-    {
-        std::array<double, 1> array = {3.3};
-        assert(std::get<0>(array) == 3.3);
-        std::get<0>(array) = 99.1;
-        assert(std::get<0>(array) == 99.1);
-    }
-    {
-        std::array<double, 2> array = {3.3, 4.4};
-        assert(std::get<0>(array) == 3.3);
-        assert(std::get<1>(array) == 4.4);
-        std::get<0>(array) = 99.1;
-        std::get<1>(array) = 99.2;
-        assert(std::get<0>(array) == 99.1);
-        assert(std::get<1>(array) == 99.2);
-    }
-    {
-        std::array<double, 3> array = {3.3, 4.4, 5.5};
-        assert(std::get<0>(array) == 3.3);
-        assert(std::get<1>(array) == 4.4);
-        assert(std::get<2>(array) == 5.5);
-        std::get<1>(array) = 99.2;
-        assert(std::get<0>(array) == 3.3);
-        assert(std::get<1>(array) == 99.2);
-        assert(std::get<2>(array) == 5.5);
-    }
-    {
-        std::array<double, 1> array = {3.3};
-        static_assert(std::is_same<double&, decltype(std::get<0>(array))>::value, "");
-    }
-    {
-        assert(std::get<0>(tempArray(1, 2, 3)) == 1);
-        assert(std::get<1>(tempArray(1, 2, 3)) == 2);
-        assert(std::get<2>(tempArray(1, 2, 3)) == 3);
-    }
+TEST_CONSTEXPR_CXX14 bool tests() {
+  {
+    std::array<double, 1> array = {3.3};
+    assert(std::get<0>(array) == 3.3);
+    std::get<0>(array) = 99.1;
+    assert(std::get<0>(array) == 99.1);
+  }
+  {
+    std::array<double, 2> array = {3.3, 4.4};
+    assert(std::get<0>(array) == 3.3);
+    assert(std::get<1>(array) == 4.4);
+    std::get<0>(array) = 99.1;
+    std::get<1>(array) = 99.2;
+    assert(std::get<0>(array) == 99.1);
+    assert(std::get<1>(array) == 99.2);
+  }
+  {
+    std::array<double, 3> array = {3.3, 4.4, 5.5};
+    assert(std::get<0>(array) == 3.3);
+    assert(std::get<1>(array) == 4.4);
+    assert(std::get<2>(array) == 5.5);
+    std::get<1>(array) = 99.2;
+    assert(std::get<0>(array) == 3.3);
+    assert(std::get<1>(array) == 99.2);
+    assert(std::get<2>(array) == 5.5);
+  }
+  {
+    std::array<double, 1> array = {3.3};
+    static_assert(std::is_same<double&, decltype(std::get<0>(array))>::value, "");
+  }
+  {
+    assert(std::get<0>(tempArray(1, 2, 3)) == 1);
+    assert(std::get<1>(tempArray(1, 2, 3)) == 2);
+    assert(std::get<2>(tempArray(1, 2, 3)) == 3);
+  }
 
-    return true;
+  return true;
 }
 
-int main(int, char**)
-{
-    tests();
+int main(int, char**) {
+  tests();
 #if TEST_STD_VER >= 14
-    static_assert(tests(), "");
+  static_assert(tests(), "");
 #endif
-    return 0;
+  return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/array/array.tuple/get.verify.cpp b/libcxx/test/std/containers/sequences/array/array.tuple/get.verify.cpp
index f3a425e6537cb..169f06f44e19c 100644
--- a/libcxx/test/std/containers/sequences/array/array.tuple/get.verify.cpp
+++ b/libcxx/test/std/containers/sequences/array/array.tuple/get.verify.cpp
@@ -21,7 +21,7 @@
 void f() {
   typedef double T;
   typedef std::array<T, 3> C;
-  C c = {1, 2, 3.5};
+  C c            = {1, 2, 3.5};
   std::get<3>(c) = 5.5; // expected-note {{requested here}}
   // expected-error-re@array:* {{static assertion failed{{( due to requirement '3U[L]{0,2} < 3U[L]{0,2}')?}}{{.*}}Index out of bounds in std::get<> (std::array)}}
 }
diff --git a/libcxx/test/std/containers/sequences/array/array.tuple/get_const.pass.cpp b/libcxx/test/std/containers/sequences/array/array.tuple/get_const.pass.cpp
index 5fc080b8efc86..88c09897aa104 100644
--- a/libcxx/test/std/containers/sequences/array/array.tuple/get_const.pass.cpp
+++ b/libcxx/test/std/containers/sequences/array/array.tuple/get_const.pass.cpp
@@ -15,36 +15,34 @@
 
 #include "test_macros.h"
 
-TEST_CONSTEXPR_CXX14 bool tests()
-{
-    {
-        std::array<double, 1> const array = {3.3};
-        assert(std::get<0>(array) == 3.3);
-    }
-    {
-        std::array<double, 2> const array = {3.3, 4.4};
-        assert(std::get<0>(array) == 3.3);
-        assert(std::get<1>(array) == 4.4);
-    }
-    {
-        std::array<double, 3> const array = {3.3, 4.4, 5.5};
-        assert(std::get<0>(array) == 3.3);
-        assert(std::get<1>(array) == 4.4);
-        assert(std::get<2>(array) == 5.5);
-    }
-    {
-        std::array<double, 1> const array = {3.3};
-        static_assert(std::is_same<double const&, decltype(std::get<0>(array))>::value, "");
-    }
+TEST_CONSTEXPR_CXX14 bool tests() {
+  {
+    std::array<double, 1> const array = {3.3};
+    assert(std::get<0>(array) == 3.3);
+  }
+  {
+    std::array<double, 2> const array = {3.3, 4.4};
+    assert(std::get<0>(array) == 3.3);
+    assert(std::get<1>(array) == 4.4);
+  }
+  {
+    std::array<double, 3> const array = {3.3, 4.4, 5.5};
+    assert(std::get<0>(array) == 3.3);
+    assert(std::get<1>(array) == 4.4);
+    assert(std::get<2>(array) == 5.5);
+  }
+  {
+    std::array<double, 1> const array = {3.3};
+    static_assert(std::is_same<double const&, decltype(std::get<0>(array))>::value, "");
+  }
 
-    return true;
+  return true;
 }
 
-int main(int, char**)
-{
-    tests();
+int main(int, char**) {
+  tests();
 #if TEST_STD_VER >= 14
-    static_assert(tests(), "");
+  static_assert(tests(), "");
 #endif
-    return 0;
+  return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/array/array.tuple/get_const_rv.pass.cpp b/libcxx/test/std/containers/sequences/array/array.tuple/get_const_rv.pass.cpp
index a5121000c18e0..4dd95c65088bd 100644
--- a/libcxx/test/std/containers/sequences/array/array.tuple/get_const_rv.pass.cpp
+++ b/libcxx/test/std/containers/sequences/array/array.tuple/get_const_rv.pass.cpp
@@ -20,29 +20,27 @@
 
 #include "test_macros.h"
 
-int main(int, char**)
-{
-
-    {
-        typedef std::unique_ptr<double> T;
-        typedef std::array<T, 1> C;
-        const C c = {std::unique_ptr<double>(new double(3.5))};
-        static_assert(std::is_same<const T&&, decltype(std::get<0>(std::move(c)))>::value, "");
-        static_assert(noexcept(std::get<0>(std::move(c))), "");
-        const T&& t = std::get<0>(std::move(c));
-        assert(*t == 3.5);
-    }
+int main(int, char**) {
+  {
+    typedef std::unique_ptr<double> T;
+    typedef std::array<T, 1> C;
+    const C c = {std::unique_ptr<double>(new double(3.5))};
+    static_assert(std::is_same<const T&&, decltype(std::get<0>(std::move(c)))>::value, "");
+    static_assert(noexcept(std::get<0>(std::move(c))), "");
+    const T&& t = std::get<0>(std::move(c));
+    assert(*t == 3.5);
+  }
 
 #if TEST_STD_VER >= 14
-    {
-        typedef double T;
-        typedef std::array<T, 3> C;
-        constexpr const C c = {1, 2, 3.5};
-        static_assert(std::get<0>(std::move(c)) == 1, "");
-        static_assert(std::get<1>(std::move(c)) == 2, "");
-        static_assert(std::get<2>(std::move(c)) == 3.5, "");
-    }
+  {
+    typedef double T;
+    typedef std::array<T, 3> C;
+    constexpr const C c = {1, 2, 3.5};
+    static_assert(std::get<0>(std::move(c)) == 1, "");
+    static_assert(std::get<1>(std::move(c)) == 2, "");
+    static_assert(std::get<2>(std::move(c)) == 3.5, "");
+  }
 #endif
 
-    return 0;
+  return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/array/array.tuple/get_rv.pass.cpp b/libcxx/test/std/containers/sequences/array/array.tuple/get_rv.pass.cpp
index 2ab8acbcfdc2a..460219b1e4fce 100644
--- a/libcxx/test/std/containers/sequences/array/array.tuple/get_rv.pass.cpp
+++ b/libcxx/test/std/containers/sequences/array/array.tuple/get_rv.pass.cpp
@@ -19,15 +19,14 @@
 
 #include "test_macros.h"
 
-int main(int, char**)
-{
-    {
-        typedef std::unique_ptr<double> T;
-        typedef std::array<T, 1> C;
-        C c = {std::unique_ptr<double>(new double(3.5))};
-        T t = std::get<0>(std::move(c));
-        assert(*t == 3.5);
-    }
+int main(int, char**) {
+  {
+    typedef std::unique_ptr<double> T;
+    typedef std::array<T, 1> C;
+    C c = {std::unique_ptr<double>(new double(3.5))};
+    T t = std::get<0>(std::move(c));
+    assert(*t == 3.5);
+  }
 
-    return 0;
+  return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/array/array.tuple/tuple_element.pass.cpp b/libcxx/test/std/containers/sequences/array/array.tuple/tuple_element.pass.cpp
index fb7aa7cf02ca5..79451abc933c5 100644
--- a/libcxx/test/std/containers/sequences/array/array.tuple/tuple_element.pass.cpp
+++ b/libcxx/test/std/containers/sequences/array/array.tuple/tuple_element.pass.cpp
@@ -16,42 +16,40 @@
 #include "test_macros.h"
 
 template <class T>
-void test()
-{
-    {
+void test() {
+  {
     typedef T Exp;
     typedef std::array<T, 3> C;
     static_assert((std::is_same<typename std::tuple_element<0, C>::type, Exp>::value), "");
     static_assert((std::is_same<typename std::tuple_element<1, C>::type, Exp>::value), "");
     static_assert((std::is_same<typename std::tuple_element<2, C>::type, Exp>::value), "");
-    }
-    {
+  }
+  {
     typedef T const Exp;
     typedef std::array<T, 3> const C;
     static_assert((std::is_same<typename std::tuple_element<0, C>::type, Exp>::value), "");
     static_assert((std::is_same<typename std::tuple_element<1, C>::type, Exp>::value), "");
     static_assert((std::is_same<typename std::tuple_element<2, C>::type, Exp>::value), "");
-    }
-    {
+  }
+  {
     typedef T volatile Exp;
     typedef std::array<T, 3> volatile C;
     static_assert((std::is_same<typename std::tuple_element<0, C>::type, Exp>::value), "");
     static_assert((std::is_same<typename std::tuple_element<1, C>::type, Exp>::value), "");
     static_assert((std::is_same<typename std::tuple_element<2, C>::type, Exp>::value), "");
-    }
-    {
+  }
+  {
     typedef T const volatile Exp;
     typedef std::array<T, 3> const volatile C;
     static_assert((std::is_same<typename std::tuple_element<0, C>::type, Exp>::value), "");
     static_assert((std::is_same<typename std::tuple_element<1, C>::type, Exp>::value), "");
     static_assert((std::is_same<typename std::tuple_element<2, C>::type, Exp>::value), "");
-    }
+  }
 }
 
-int main(int, char**)
-{
-    test<double>();
-    test<int>();
+int main(int, char**) {
+  test<double>();
+  test<int>();
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/array/array.tuple/tuple_size.pass.cpp b/libcxx/test/std/containers/sequences/array/array.tuple/tuple_size.pass.cpp
index e31407ac5a7d7..ebfcffc699d06 100644
--- a/libcxx/test/std/containers/sequences/array/array.tuple/tuple_size.pass.cpp
+++ b/libcxx/test/std/containers/sequences/array/array.tuple/tuple_size.pass.cpp
@@ -15,31 +15,29 @@
 #include "test_macros.h"
 
 template <class T, std::size_t N>
-void test()
-{
-    {
+void test() {
+  {
     typedef std::array<T, N> C;
     static_assert((std::tuple_size<C>::value == N), "");
-    }
-    {
+  }
+  {
     typedef std::array<T const, N> C;
     static_assert((std::tuple_size<C>::value == N), "");
-    }
-    {
+  }
+  {
     typedef std::array<T volatile, N> C;
     static_assert((std::tuple_size<C>::value == N), "");
-    }
-    {
+  }
+  {
     typedef std::array<T const volatile, N> C;
     static_assert((std::tuple_size<C>::value == N), "");
-    }
+  }
 }
 
-int main(int, char**)
-{
-    test<double, 0>();
-    test<double, 3>();
-    test<double, 5>();
+int main(int, char**) {
+  test<double, 0>();
+  test<double, 3>();
+  test<double, 5>();
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/array/array.zero/tested_elsewhere.pass.cpp b/libcxx/test/std/containers/sequences/array/array.zero/tested_elsewhere.pass.cpp
index af4ab87048eb4..d4c62f0d0493f 100644
--- a/libcxx/test/std/containers/sequences/array/array.zero/tested_elsewhere.pass.cpp
+++ b/libcxx/test/std/containers/sequences/array/array.zero/tested_elsewhere.pass.cpp
@@ -14,8 +14,4 @@
 
 #include "test_macros.h"
 
-int main(int, char**)
-{
-
-  return 0;
-}
+int main(int, char**) { return 0; }
diff --git a/libcxx/test/std/containers/sequences/array/at.pass.cpp b/libcxx/test/std/containers/sequences/array/at.pass.cpp
index 90b5684b1e567..48a3aef2f5953 100644
--- a/libcxx/test/std/containers/sequences/array/at.pass.cpp
+++ b/libcxx/test/std/containers/sequences/array/at.pass.cpp
@@ -14,96 +14,93 @@
 #include <cassert>
 
 #ifndef TEST_HAS_NO_EXCEPTIONS
-#include <stdexcept>
+#  include <stdexcept>
 #endif
 
 #include "test_macros.h"
 
-TEST_CONSTEXPR_CXX17 bool tests()
-{
-    {
-        typedef double T;
-        typedef std::array<T, 3> C;
-        C c = {1, 2, 3.5};
-        typename C::reference r1 = c.at(0);
-        assert(r1 == 1);
-        r1 = 5.5;
-        assert(c[0] == 5.5);
-
-        typename C::reference r2 = c.at(2);
-        assert(r2 == 3.5);
-        r2 = 7.5;
-        assert(c[2] == 7.5);
-    }
-    return true;
+TEST_CONSTEXPR_CXX17 bool tests() {
+  {
+    typedef double T;
+    typedef std::array<T, 3> C;
+    C c                      = {1, 2, 3.5};
+    typename C::reference r1 = c.at(0);
+    assert(r1 == 1);
+    r1 = 5.5;
+    assert(c[0] == 5.5);
+
+    typename C::reference r2 = c.at(2);
+    assert(r2 == 3.5);
+    r2 = 7.5;
+    assert(c[2] == 7.5);
+  }
+  return true;
 }
 
-void test_exceptions()
-{
+void test_exceptions() {
 #ifndef TEST_HAS_NO_EXCEPTIONS
-    {
-        std::array<int, 4> array = {1, 2, 3, 4};
-
-        try {
-            TEST_IGNORE_NODISCARD array.at(4);
-            assert(false);
-        } catch (std::out_of_range const&) {
-            // pass
-        } catch (...) {
-            assert(false);
-        }
-
-        try {
-            TEST_IGNORE_NODISCARD array.at(5);
-            assert(false);
-        } catch (std::out_of_range const&) {
-            // pass
-        } catch (...) {
-            assert(false);
-        }
-
-        try {
-            TEST_IGNORE_NODISCARD array.at(6);
-            assert(false);
-        } catch (std::out_of_range const&) {
-            // pass
-        } catch (...) {
-            assert(false);
-        }
-
-        try {
-            using size_type = decltype(array)::size_type;
-            TEST_IGNORE_NODISCARD array.at(static_cast<size_type>(-1));
-            assert(false);
-        } catch (std::out_of_range const&) {
-            // pass
-        } catch (...) {
-            assert(false);
-        }
+  {
+    std::array<int, 4> array = {1, 2, 3, 4};
+
+    try {
+      TEST_IGNORE_NODISCARD array.at(4);
+      assert(false);
+    } catch (std::out_of_range const&) {
+      // pass
+    } catch (...) {
+      assert(false);
+    }
+
+    try {
+      TEST_IGNORE_NODISCARD array.at(5);
+      assert(false);
+    } catch (std::out_of_range const&) {
+      // pass
+    } catch (...) {
+      assert(false);
     }
 
-    {
-        std::array<int, 0> array = {};
-
-        try {
-            TEST_IGNORE_NODISCARD array.at(0);
-            assert(false);
-        } catch (std::out_of_range const&) {
-            // pass
-        } catch (...) {
-            assert(false);
-        }
+    try {
+      TEST_IGNORE_NODISCARD array.at(6);
+      assert(false);
+    } catch (std::out_of_range const&) {
+      // pass
+    } catch (...) {
+      assert(false);
+    }
+
+    try {
+      using size_type = decltype(array)::size_type;
+      TEST_IGNORE_NODISCARD array.at(static_cast<size_type>(-1));
+      assert(false);
+    } catch (std::out_of_range const&) {
+      // pass
+    } catch (...) {
+      assert(false);
+    }
+  }
+
+  {
+    std::array<int, 0> array = {};
+
+    try {
+      TEST_IGNORE_NODISCARD array.at(0);
+      assert(false);
+    } catch (std::out_of_range const&) {
+      // pass
+    } catch (...) {
+      assert(false);
     }
+  }
 #endif
 }
 
-int main(int, char**)
-{
-    tests();
-    test_exceptions();
+int main(int, char**) {
+  tests();
+  test_exceptions();
 
 #if TEST_STD_VER >= 17
-    static_assert(tests(), "");
+  static_assert(tests(), "");
 #endif
-    return 0;
+  return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/array/at_const.pass.cpp b/libcxx/test/std/containers/sequences/array/at_const.pass.cpp
index 2ddd5a03a6c67..fd12d34512a12 100644
--- a/libcxx/test/std/containers/sequences/array/at_const.pass.cpp
+++ b/libcxx/test/std/containers/sequences/array/at_const.pass.cpp
@@ -14,92 +14,89 @@
 #include <cassert>
 
 #ifndef TEST_HAS_NO_EXCEPTIONS
-#include <stdexcept>
+#  include <stdexcept>
 #endif
 
 #include "test_macros.h"
 
-TEST_CONSTEXPR_CXX14 bool tests()
-{
-    {
-        typedef double T;
-        typedef std::array<T, 3> C;
-        C const c = {1, 2, 3.5};
-        typename C::const_reference r1 = c.at(0);
-        assert(r1 == 1);
-
-        typename C::const_reference r2 = c.at(2);
-        assert(r2 == 3.5);
-    }
-    return true;
+TEST_CONSTEXPR_CXX14 bool tests() {
+  {
+    typedef double T;
+    typedef std::array<T, 3> C;
+    C const c                      = {1, 2, 3.5};
+    typename C::const_reference r1 = c.at(0);
+    assert(r1 == 1);
+
+    typename C::const_reference r2 = c.at(2);
+    assert(r2 == 3.5);
+  }
+  return true;
 }
 
-void test_exceptions()
-{
+void test_exceptions() {
 #ifndef TEST_HAS_NO_EXCEPTIONS
-    {
-        std::array<int, 4> const array = {1, 2, 3, 4};
-
-        try {
-            TEST_IGNORE_NODISCARD array.at(4);
-            assert(false);
-        } catch (std::out_of_range const&) {
-            // pass
-        } catch (...) {
-            assert(false);
-        }
-
-        try {
-            TEST_IGNORE_NODISCARD array.at(5);
-            assert(false);
-        } catch (std::out_of_range const&) {
-            // pass
-        } catch (...) {
-            assert(false);
-        }
-
-        try {
-            TEST_IGNORE_NODISCARD array.at(6);
-            assert(false);
-        } catch (std::out_of_range const&) {
-            // pass
-        } catch (...) {
-            assert(false);
-        }
-
-        try {
-            using size_type = decltype(array)::size_type;
-            TEST_IGNORE_NODISCARD array.at(static_cast<size_type>(-1));
-            assert(false);
-        } catch (std::out_of_range const&) {
-            // pass
-        } catch (...) {
-            assert(false);
-        }
+  {
+    std::array<int, 4> const array = {1, 2, 3, 4};
+
+    try {
+      TEST_IGNORE_NODISCARD array.at(4);
+      assert(false);
+    } catch (std::out_of_range const&) {
+      // pass
+    } catch (...) {
+      assert(false);
+    }
+
+    try {
+      TEST_IGNORE_NODISCARD array.at(5);
+      assert(false);
+    } catch (std::out_of_range const&) {
+      // pass
+    } catch (...) {
+      assert(false);
     }
 
-    {
-        std::array<int, 0> array = {};
-
-        try {
-            TEST_IGNORE_NODISCARD array.at(0);
-            assert(false);
-        } catch (std::out_of_range const&) {
-            // pass
-        } catch (...) {
-            assert(false);
-        }
+    try {
+      TEST_IGNORE_NODISCARD array.at(6);
+      assert(false);
+    } catch (std::out_of_range const&) {
+      // pass
+    } catch (...) {
+      assert(false);
+    }
+
+    try {
+      using size_type = decltype(array)::size_type;
+      TEST_IGNORE_NODISCARD array.at(static_cast<size_type>(-1));
+      assert(false);
+    } catch (std::out_of_range const&) {
+      // pass
+    } catch (...) {
+      assert(false);
+    }
+  }
+
+  {
+    std::array<int, 0> array = {};
+
+    try {
+      TEST_IGNORE_NODISCARD array.at(0);
+      assert(false);
+    } catch (std::out_of_range const&) {
+      // pass
+    } catch (...) {
+      assert(false);
     }
+  }
 #endif
 }
 
-int main(int, char**)
-{
-    tests();
-    test_exceptions();
+int main(int, char**) {
+  tests();
+  test_exceptions();
 
 #if TEST_STD_VER >= 14
-    static_assert(tests(), "");
+  static_assert(tests(), "");
 #endif
-    return 0;
+  return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/array/contiguous.pass.cpp b/libcxx/test/std/containers/sequences/array/contiguous.pass.cpp
index 85e6ed7ae9aee..fda0f9ef138ec 100644
--- a/libcxx/test/std/containers/sequences/array/contiguous.pass.cpp
+++ b/libcxx/test/std/containers/sequences/array/contiguous.pass.cpp
@@ -17,32 +17,29 @@
 #include "test_macros.h"
 
 template <class Container>
-TEST_CONSTEXPR_CXX14 void assert_contiguous(Container const& c)
-{
-    for (std::size_t i = 0; i < c.size(); ++i)
-        assert(*(c.begin() + i) == *(std::addressof(*c.begin()) + i));
+TEST_CONSTEXPR_CXX14 void assert_contiguous(Container const& c) {
+  for (std::size_t i = 0; i < c.size(); ++i)
+    assert(*(c.begin() + i) == *(std::addressof(*c.begin()) + i));
 }
 
-TEST_CONSTEXPR_CXX17 bool tests()
-{
-    assert_contiguous(std::array<double, 0>());
-    assert_contiguous(std::array<double, 1>());
-    assert_contiguous(std::array<double, 2>());
-    assert_contiguous(std::array<double, 3>());
+TEST_CONSTEXPR_CXX17 bool tests() {
+  assert_contiguous(std::array<double, 0>());
+  assert_contiguous(std::array<double, 1>());
+  assert_contiguous(std::array<double, 2>());
+  assert_contiguous(std::array<double, 3>());
 
-    assert_contiguous(std::array<char, 0>());
-    assert_contiguous(std::array<char, 1>());
-    assert_contiguous(std::array<char, 2>());
-    assert_contiguous(std::array<char, 3>());
+  assert_contiguous(std::array<char, 0>());
+  assert_contiguous(std::array<char, 1>());
+  assert_contiguous(std::array<char, 2>());
+  assert_contiguous(std::array<char, 3>());
 
-    return true;
+  return true;
 }
 
-int main(int, char**)
-{
-    tests();
+int main(int, char**) {
+  tests();
 #if TEST_STD_VER >= 17 // begin() & friends are constexpr in >= C++17 only
-    static_assert(tests(), "");
+  static_assert(tests(), "");
 #endif
-    return 0;
+  return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/array/empty.pass.cpp b/libcxx/test/std/containers/sequences/array/empty.pass.cpp
index 8b61575c2e731..a2289760b2e96 100644
--- a/libcxx/test/std/containers/sequences/array/empty.pass.cpp
+++ b/libcxx/test/std/containers/sequences/array/empty.pass.cpp
@@ -17,38 +17,36 @@
 
 #include "test_macros.h"
 
-TEST_CONSTEXPR_CXX14 bool tests()
-{
-    {
-        typedef std::array<int, 2> C;
-        C c = {};
-        ASSERT_NOEXCEPT(c.empty());
-        assert(!c.empty());
-    }
-    {
-        typedef std::array<int, 0> C;
-        C c = {};
-        ASSERT_NOEXCEPT(c.empty());
-        assert(c.empty());
-    }
-
-    return true;
+TEST_CONSTEXPR_CXX14 bool tests() {
+  {
+    typedef std::array<int, 2> C;
+    C c = {};
+    ASSERT_NOEXCEPT(c.empty());
+    assert(!c.empty());
+  }
+  {
+    typedef std::array<int, 0> C;
+    C c = {};
+    ASSERT_NOEXCEPT(c.empty());
+    assert(c.empty());
+  }
+
+  return true;
 }
 
-int main(int, char**)
-{
-    tests();
+int main(int, char**) {
+  tests();
 #if TEST_STD_VER >= 14
-    static_assert(tests(), "");
+  static_assert(tests(), "");
 #endif
 
 #if TEST_STD_VER >= 11
-    // Sanity check for constexpr in C++11
-    {
-        constexpr std::array<int, 3> array = {};
-        static_assert(!array.empty(), "");
-    }
+  // Sanity check for constexpr in C++11
+  {
+    constexpr std::array<int, 3> array = {};
+    static_assert(!array.empty(), "");
+  }
 #endif
 
-    return 0;
+  return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/array/empty.verify.cpp b/libcxx/test/std/containers/sequences/array/empty.verify.cpp
index 9cf8f1156990f..2fc821e08a106 100644
--- a/libcxx/test/std/containers/sequences/array/empty.verify.cpp
+++ b/libcxx/test/std/containers/sequences/array/empty.verify.cpp
@@ -17,8 +17,8 @@
 #include <array>
 
 void f() {
-    std::array<int, 1> c;
-    c.empty(); // expected-warning {{ignoring return value of function declared with 'nodiscard' attribute}}
-    std::array<int, 0> c0;
-    c0.empty(); // expected-warning {{ignoring return value of function declared with 'nodiscard' attribute}}
+  std::array<int, 1> c;
+  c.empty(); // expected-warning {{ignoring return value of function declared with 'nodiscard' attribute}}
+  std::array<int, 0> c0;
+  c0.empty(); // expected-warning {{ignoring return value of function declared with 'nodiscard' attribute}}
 }
diff --git a/libcxx/test/std/containers/sequences/array/front_back.pass.cpp b/libcxx/test/std/containers/sequences/array/front_back.pass.cpp
index c37acd0b78638..4d8cffbcd77a4 100644
--- a/libcxx/test/std/containers/sequences/array/front_back.pass.cpp
+++ b/libcxx/test/std/containers/sequences/array/front_back.pass.cpp
@@ -16,58 +16,56 @@
 
 #include "test_macros.h"
 
-TEST_CONSTEXPR_CXX17 bool tests()
-{
-    {
-        typedef double T;
-        typedef std::array<T, 3> C;
-        C c = {1, 2, 3.5};
+TEST_CONSTEXPR_CXX17 bool tests() {
+  {
+    typedef double T;
+    typedef std::array<T, 3> C;
+    C c = {1, 2, 3.5};
 
-        C::reference r1 = c.front();
-        assert(r1 == 1);
-        r1 = 5.5;
-        assert(c[0] == 5.5);
+    C::reference r1 = c.front();
+    assert(r1 == 1);
+    r1 = 5.5;
+    assert(c[0] == 5.5);
 
-        C::reference r2 = c.back();
-        assert(r2 == 3.5);
-        r2 = 7.5;
-        assert(c[2] == 7.5);
+    C::reference r2 = c.back();
+    assert(r2 == 3.5);
+    r2 = 7.5;
+    assert(c[2] == 7.5);
+  }
+  {
+    typedef double T;
+    typedef std::array<T, 0> C;
+    C c = {};
+    ASSERT_SAME_TYPE(decltype(c.back()), C::reference);
+    LIBCPP_ASSERT_NOEXCEPT(c.back());
+    ASSERT_SAME_TYPE(decltype(c.front()), C::reference);
+    LIBCPP_ASSERT_NOEXCEPT(c.front());
+    if (c.size() > (0)) { // always false
+      TEST_IGNORE_NODISCARD c.front();
+      TEST_IGNORE_NODISCARD c.back();
     }
-    {
-        typedef double T;
-        typedef std::array<T, 0> C;
-        C c = {};
-        ASSERT_SAME_TYPE(decltype(c.back()), C::reference);
-        LIBCPP_ASSERT_NOEXCEPT(c.back());
-        ASSERT_SAME_TYPE(decltype(c.front()), C::reference);
-        LIBCPP_ASSERT_NOEXCEPT(c.front());
-        if (c.size() > (0)) { // always false
-            TEST_IGNORE_NODISCARD c.front();
-            TEST_IGNORE_NODISCARD c.back();
-        }
-    }
-    {
-        typedef double T;
-        typedef std::array<const T, 0> C;
-        C c = {};
-        ASSERT_SAME_TYPE(decltype( c.back()), C::reference);
-        LIBCPP_ASSERT_NOEXCEPT(    c.back());
-        ASSERT_SAME_TYPE(decltype( c.front()), C::reference);
-        LIBCPP_ASSERT_NOEXCEPT(    c.front());
-        if (c.size() > (0)) {
-            TEST_IGNORE_NODISCARD c.front();
-            TEST_IGNORE_NODISCARD c.back();
-        }
+  }
+  {
+    typedef double T;
+    typedef std::array<const T, 0> C;
+    C c = {};
+    ASSERT_SAME_TYPE(decltype(c.back()), C::reference);
+    LIBCPP_ASSERT_NOEXCEPT(c.back());
+    ASSERT_SAME_TYPE(decltype(c.front()), C::reference);
+    LIBCPP_ASSERT_NOEXCEPT(c.front());
+    if (c.size() > (0)) {
+      TEST_IGNORE_NODISCARD c.front();
+      TEST_IGNORE_NODISCARD c.back();
     }
+  }
 
-    return true;
+  return true;
 }
 
-int main(int, char**)
-{
-    tests();
+int main(int, char**) {
+  tests();
 #if TEST_STD_VER >= 17
-    static_assert(tests(), "");
+  static_assert(tests(), "");
 #endif
-    return 0;
+  return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/array/front_back_const.pass.cpp b/libcxx/test/std/containers/sequences/array/front_back_const.pass.cpp
index 9204663f0da3c..d795223df40d7 100644
--- a/libcxx/test/std/containers/sequences/array/front_back_const.pass.cpp
+++ b/libcxx/test/std/containers/sequences/array/front_back_const.pass.cpp
@@ -16,53 +16,51 @@
 
 #include "test_macros.h"
 
-TEST_CONSTEXPR_CXX14 bool tests()
-{
-    {
-        typedef double T;
-        typedef std::array<T, 3> C;
-        C const c = {1, 2, 3.5};
-        C::const_reference r1 = c.front();
-        assert(r1 == 1);
+TEST_CONSTEXPR_CXX14 bool tests() {
+  {
+    typedef double T;
+    typedef std::array<T, 3> C;
+    C const c             = {1, 2, 3.5};
+    C::const_reference r1 = c.front();
+    assert(r1 == 1);
 
-        C::const_reference r2 = c.back();
-        assert(r2 == 3.5);
+    C::const_reference r2 = c.back();
+    assert(r2 == 3.5);
+  }
+  {
+    typedef double T;
+    typedef std::array<T, 0> C;
+    C const c = {};
+    ASSERT_SAME_TYPE(decltype(c.back()), C::const_reference);
+    LIBCPP_ASSERT_NOEXCEPT(c.back());
+    ASSERT_SAME_TYPE(decltype(c.front()), C::const_reference);
+    LIBCPP_ASSERT_NOEXCEPT(c.front());
+    if (c.size() > (0)) { // always false
+      TEST_IGNORE_NODISCARD c.front();
+      TEST_IGNORE_NODISCARD c.back();
     }
-    {
-        typedef double T;
-        typedef std::array<T, 0> C;
-        C const c = {};
-        ASSERT_SAME_TYPE(decltype(c.back()), C::const_reference);
-        LIBCPP_ASSERT_NOEXCEPT(c.back());
-        ASSERT_SAME_TYPE(decltype(c.front()), C::const_reference);
-        LIBCPP_ASSERT_NOEXCEPT(c.front());
-        if (c.size() > (0)) { // always false
-            TEST_IGNORE_NODISCARD c.front();
-            TEST_IGNORE_NODISCARD c.back();
-        }
-    }
-    {
-        typedef double T;
-        typedef std::array<const T, 0> C;
-        C const c = {};
-        ASSERT_SAME_TYPE(decltype(c.back()), C::const_reference);
-        LIBCPP_ASSERT_NOEXCEPT(c.back());
-        ASSERT_SAME_TYPE(decltype(c.front()), C::const_reference);
-        LIBCPP_ASSERT_NOEXCEPT(c.front());
-        if (c.size() > (0)) {
-            TEST_IGNORE_NODISCARD c.front();
-            TEST_IGNORE_NODISCARD c.back();
-        }
+  }
+  {
+    typedef double T;
+    typedef std::array<const T, 0> C;
+    C const c = {};
+    ASSERT_SAME_TYPE(decltype(c.back()), C::const_reference);
+    LIBCPP_ASSERT_NOEXCEPT(c.back());
+    ASSERT_SAME_TYPE(decltype(c.front()), C::const_reference);
+    LIBCPP_ASSERT_NOEXCEPT(c.front());
+    if (c.size() > (0)) {
+      TEST_IGNORE_NODISCARD c.front();
+      TEST_IGNORE_NODISCARD c.back();
     }
+  }
 
-    return true;
+  return true;
 }
 
-int main(int, char**)
-{
-    tests();
+int main(int, char**) {
+  tests();
 #if TEST_STD_VER >= 14
-    static_assert(tests(), "");
+  static_assert(tests(), "");
 #endif
-    return 0;
+  return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/array/indexing.pass.cpp b/libcxx/test/std/containers/sequences/array/indexing.pass.cpp
index 48a2be0bdcbd7..9caa90f2567f5 100644
--- a/libcxx/test/std/containers/sequences/array/indexing.pass.cpp
+++ b/libcxx/test/std/containers/sequences/array/indexing.pass.cpp
@@ -16,59 +16,57 @@
 
 #include "test_macros.h"
 
-TEST_CONSTEXPR_CXX17 bool tests()
-{
-    {
-        typedef double T;
-        typedef std::array<T, 3> C;
-        C c = {1, 2, 3.5};
-        LIBCPP_ASSERT_NOEXCEPT(c[0]);
-        ASSERT_SAME_TYPE(C::reference, decltype(c[0]));
-        C::reference r1 = c[0];
-        assert(r1 == 1);
-        r1 = 5.5;
-        assert(c.front() == 5.5);
+TEST_CONSTEXPR_CXX17 bool tests() {
+  {
+    typedef double T;
+    typedef std::array<T, 3> C;
+    C c = {1, 2, 3.5};
+    LIBCPP_ASSERT_NOEXCEPT(c[0]);
+    ASSERT_SAME_TYPE(C::reference, decltype(c[0]));
+    C::reference r1 = c[0];
+    assert(r1 == 1);
+    r1 = 5.5;
+    assert(c.front() == 5.5);
 
-        C::reference r2 = c[2];
-        assert(r2 == 3.5);
-        r2 = 7.5;
-        assert(c.back() == 7.5);
-    }
+    C::reference r2 = c[2];
+    assert(r2 == 3.5);
+    r2 = 7.5;
+    assert(c.back() == 7.5);
+  }
 
-    // Test operator[] "works" on zero sized arrays
+  // Test operator[] "works" on zero sized arrays
+  {
+    {
+      typedef double T;
+      typedef std::array<T, 0> C;
+      C c = {};
+      LIBCPP_ASSERT_NOEXCEPT(c[0]);
+      ASSERT_SAME_TYPE(C::reference, decltype(c[0]));
+      if (c.size() > (0)) { // always false
+        C::reference r = c[0];
+        (void)r;
+      }
+    }
     {
-        {
-            typedef double T;
-            typedef std::array<T, 0> C;
-            C c = {};
-            LIBCPP_ASSERT_NOEXCEPT(c[0]);
-            ASSERT_SAME_TYPE(C::reference, decltype(c[0]));
-            if (c.size() > (0)) { // always false
-              C::reference r = c[0];
-              (void)r;
-            }
-        }
-        {
-            typedef double T;
-            typedef std::array<const T, 0> C;
-            C c = {};
-            LIBCPP_ASSERT_NOEXCEPT(c[0]);
-            ASSERT_SAME_TYPE(C::reference, decltype(c[0]));
-            if (c.size() > (0)) { // always false
-              C::reference r = c[0];
-              (void)r;
-            }
-        }
+      typedef double T;
+      typedef std::array<const T, 0> C;
+      C c = {};
+      LIBCPP_ASSERT_NOEXCEPT(c[0]);
+      ASSERT_SAME_TYPE(C::reference, decltype(c[0]));
+      if (c.size() > (0)) { // always false
+        C::reference r = c[0];
+        (void)r;
+      }
     }
+  }
 
-    return true;
+  return true;
 }
 
-int main(int, char**)
-{
-    tests();
+int main(int, char**) {
+  tests();
 #if TEST_STD_VER >= 17
-    static_assert(tests(), "");
+  static_assert(tests(), "");
 #endif
-    return 0;
+  return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/array/indexing_const.pass.cpp b/libcxx/test/std/containers/sequences/array/indexing_const.pass.cpp
index ce6f8bb59aac8..78844c60060d8 100644
--- a/libcxx/test/std/containers/sequences/array/indexing_const.pass.cpp
+++ b/libcxx/test/std/containers/sequences/array/indexing_const.pass.cpp
@@ -16,53 +16,51 @@
 
 #include "test_macros.h"
 
-TEST_CONSTEXPR_CXX14 bool tests()
-{
+TEST_CONSTEXPR_CXX14 bool tests() {
+  {
+    typedef double T;
+    typedef std::array<T, 3> C;
+    C const c = {1, 2, 3.5};
+    LIBCPP_ASSERT_NOEXCEPT(c[0]);
+    ASSERT_SAME_TYPE(C::const_reference, decltype(c[0]));
+    C::const_reference r1 = c[0];
+    assert(r1 == 1);
+    C::const_reference r2 = c[2];
+    assert(r2 == 3.5);
+  }
+  // Test operator[] "works" on zero sized arrays
+  {
     {
-        typedef double T;
-        typedef std::array<T, 3> C;
-        C const c = {1, 2, 3.5};
-        LIBCPP_ASSERT_NOEXCEPT(c[0]);
-        ASSERT_SAME_TYPE(C::const_reference, decltype(c[0]));
-        C::const_reference r1 = c[0];
-        assert(r1 == 1);
-        C::const_reference r2 = c[2];
-        assert(r2 == 3.5);
+      typedef double T;
+      typedef std::array<T, 0> C;
+      C const c = {};
+      LIBCPP_ASSERT_NOEXCEPT(c[0]);
+      ASSERT_SAME_TYPE(C::const_reference, decltype(c[0]));
+      if (c.size() > (0)) { // always false
+        C::const_reference r = c[0];
+        (void)r;
+      }
     }
-    // Test operator[] "works" on zero sized arrays
     {
-        {
-            typedef double T;
-            typedef std::array<T, 0> C;
-            C const c = {};
-            LIBCPP_ASSERT_NOEXCEPT(c[0]);
-            ASSERT_SAME_TYPE(C::const_reference, decltype(c[0]));
-            if (c.size() > (0)) { // always false
-                C::const_reference r = c[0];
-                (void)r;
-            }
-        }
-        {
-            typedef double T;
-            typedef std::array<T const, 0> C;
-            C const c = {};
-            LIBCPP_ASSERT_NOEXCEPT(c[0]);
-            ASSERT_SAME_TYPE(C::const_reference, decltype(c[0]));
-            if (c.size() > (0)) { // always false
-              C::const_reference r = c[0];
-              (void)r;
-            }
-        }
+      typedef double T;
+      typedef std::array<T const, 0> C;
+      C const c = {};
+      LIBCPP_ASSERT_NOEXCEPT(c[0]);
+      ASSERT_SAME_TYPE(C::const_reference, decltype(c[0]));
+      if (c.size() > (0)) { // always false
+        C::const_reference r = c[0];
+        (void)r;
+      }
     }
+  }
 
-    return true;
+  return true;
 }
 
-int main(int, char**)
-{
-    tests();
+int main(int, char**) {
+  tests();
 #if TEST_STD_VER >= 14
-    static_assert(tests(), "");
+  static_assert(tests(), "");
 #endif
   return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/array/iterator_concept_conformance.compile.pass.cpp b/libcxx/test/std/containers/sequences/array/iterator_concept_conformance.compile.pass.cpp
index 5ab30616fb5b5..6e048aefc004b 100644
--- a/libcxx/test/std/containers/sequences/array/iterator_concept_conformance.compile.pass.cpp
+++ b/libcxx/test/std/containers/sequences/array/iterator_concept_conformance.compile.pass.cpp
@@ -14,9 +14,9 @@
 
 #include <iterator>
 
-using iterator = std::array<int, 10>::iterator;
-using const_iterator = std::array<int, 10>::const_iterator;
-using reverse_iterator = std::array<int, 10>::reverse_iterator;
+using iterator               = std::array<int, 10>::iterator;
+using const_iterator         = std::array<int, 10>::const_iterator;
+using reverse_iterator       = std::array<int, 10>::reverse_iterator;
 using const_reverse_iterator = std::array<int, 10>::const_reverse_iterator;
 
 static_assert(std::contiguous_iterator<iterator>);
@@ -29,23 +29,23 @@ static_assert(std::sized_sentinel_for<iterator, iterator>);
 static_assert(std::sized_sentinel_for<iterator, const_iterator>);
 static_assert(!std::sized_sentinel_for<iterator, reverse_iterator>);
 static_assert(!std::sized_sentinel_for<iterator, const_reverse_iterator>);
-static_assert( std::indirectly_movable<iterator, iterator>);
-static_assert( std::indirectly_movable_storable<iterator, iterator>);
+static_assert(std::indirectly_movable<iterator, iterator>);
+static_assert(std::indirectly_movable_storable<iterator, iterator>);
 static_assert(!std::indirectly_movable<iterator, const_iterator>);
 static_assert(!std::indirectly_movable_storable<iterator, const_iterator>);
-static_assert( std::indirectly_movable<iterator, reverse_iterator>);
-static_assert( std::indirectly_movable_storable<iterator, reverse_iterator>);
+static_assert(std::indirectly_movable<iterator, reverse_iterator>);
+static_assert(std::indirectly_movable_storable<iterator, reverse_iterator>);
 static_assert(!std::indirectly_movable<iterator, const_reverse_iterator>);
 static_assert(!std::indirectly_movable_storable<iterator, const_reverse_iterator>);
-static_assert( std::indirectly_copyable<iterator, iterator>);
-static_assert( std::indirectly_copyable_storable<iterator, iterator>);
+static_assert(std::indirectly_copyable<iterator, iterator>);
+static_assert(std::indirectly_copyable_storable<iterator, iterator>);
 static_assert(!std::indirectly_copyable<iterator, const_iterator>);
 static_assert(!std::indirectly_copyable_storable<iterator, const_iterator>);
-static_assert( std::indirectly_copyable<iterator, reverse_iterator>);
-static_assert( std::indirectly_copyable_storable<iterator, reverse_iterator>);
+static_assert(std::indirectly_copyable<iterator, reverse_iterator>);
+static_assert(std::indirectly_copyable_storable<iterator, reverse_iterator>);
 static_assert(!std::indirectly_copyable<iterator, const_reverse_iterator>);
 static_assert(!std::indirectly_copyable_storable<iterator, const_reverse_iterator>);
-static_assert( std::indirectly_swappable<iterator, iterator>);
+static_assert(std::indirectly_swappable<iterator, iterator>);
 
 static_assert(std::contiguous_iterator<const_iterator>);
 static_assert(!std::indirectly_writable<const_iterator, int>);
@@ -57,20 +57,20 @@ static_assert(std::sized_sentinel_for<const_iterator, iterator>);
 static_assert(std::sized_sentinel_for<const_iterator, const_iterator>);
 static_assert(!std::sized_sentinel_for<const_iterator, reverse_iterator>);
 static_assert(!std::sized_sentinel_for<const_iterator, const_reverse_iterator>);
-static_assert( std::indirectly_movable<const_iterator, iterator>);
-static_assert( std::indirectly_movable_storable<const_iterator, iterator>);
+static_assert(std::indirectly_movable<const_iterator, iterator>);
+static_assert(std::indirectly_movable_storable<const_iterator, iterator>);
 static_assert(!std::indirectly_movable<const_iterator, const_iterator>);
 static_assert(!std::indirectly_movable_storable<const_iterator, const_iterator>);
-static_assert( std::indirectly_movable<const_iterator, reverse_iterator>);
-static_assert( std::indirectly_movable_storable<const_iterator, reverse_iterator>);
+static_assert(std::indirectly_movable<const_iterator, reverse_iterator>);
+static_assert(std::indirectly_movable_storable<const_iterator, reverse_iterator>);
 static_assert(!std::indirectly_movable<const_iterator, const_reverse_iterator>);
 static_assert(!std::indirectly_movable_storable<const_iterator, const_reverse_iterator>);
-static_assert( std::indirectly_copyable<const_iterator, iterator>);
-static_assert( std::indirectly_copyable_storable<const_iterator, iterator>);
+static_assert(std::indirectly_copyable<const_iterator, iterator>);
+static_assert(std::indirectly_copyable_storable<const_iterator, iterator>);
 static_assert(!std::indirectly_copyable<const_iterator, const_iterator>);
 static_assert(!std::indirectly_copyable_storable<const_iterator, const_iterator>);
-static_assert( std::indirectly_copyable<const_iterator, reverse_iterator>);
-static_assert( std::indirectly_copyable_storable<const_iterator, reverse_iterator>);
+static_assert(std::indirectly_copyable<const_iterator, reverse_iterator>);
+static_assert(std::indirectly_copyable_storable<const_iterator, reverse_iterator>);
 static_assert(!std::indirectly_copyable<const_iterator, const_reverse_iterator>);
 static_assert(!std::indirectly_copyable_storable<const_iterator, const_reverse_iterator>);
 static_assert(!std::indirectly_swappable<const_iterator, const_iterator>);
diff --git a/libcxx/test/std/containers/sequences/array/iterators.pass.cpp b/libcxx/test/std/containers/sequences/array/iterators.pass.cpp
index 710994c68295e..f3857848358f0 100644
--- a/libcxx/test/std/containers/sequences/array/iterators.pass.cpp
+++ b/libcxx/test/std/containers/sequences/array/iterators.pass.cpp
@@ -30,194 +30,193 @@
 #include "test_macros.h"
 
 struct NoDefault {
-    TEST_CONSTEXPR NoDefault(int) { }
+  TEST_CONSTEXPR NoDefault(int) {}
 };
 
 template <class T>
 TEST_CONSTEXPR_CXX17 void check_noexcept(T& c) {
-    ASSERT_NOEXCEPT(c.begin());
-    ASSERT_NOEXCEPT(c.end());
-    ASSERT_NOEXCEPT(c.cbegin());
-    ASSERT_NOEXCEPT(c.cend());
-    ASSERT_NOEXCEPT(c.rbegin());
-    ASSERT_NOEXCEPT(c.rend());
-    ASSERT_NOEXCEPT(c.crbegin());
-    ASSERT_NOEXCEPT(c.crend());
-
-    const T& cc = c; (void)cc;
-    ASSERT_NOEXCEPT(cc.begin());
-    ASSERT_NOEXCEPT(cc.end());
-    ASSERT_NOEXCEPT(cc.rbegin());
-    ASSERT_NOEXCEPT(cc.rend());
+  ASSERT_NOEXCEPT(c.begin());
+  ASSERT_NOEXCEPT(c.end());
+  ASSERT_NOEXCEPT(c.cbegin());
+  ASSERT_NOEXCEPT(c.cend());
+  ASSERT_NOEXCEPT(c.rbegin());
+  ASSERT_NOEXCEPT(c.rend());
+  ASSERT_NOEXCEPT(c.crbegin());
+  ASSERT_NOEXCEPT(c.crend());
+
+  const T& cc = c;
+  (void)cc;
+  ASSERT_NOEXCEPT(cc.begin());
+  ASSERT_NOEXCEPT(cc.end());
+  ASSERT_NOEXCEPT(cc.rbegin());
+  ASSERT_NOEXCEPT(cc.rend());
 }
 
-TEST_CONSTEXPR_CXX17 bool tests()
-{
-    {
-        typedef std::array<int, 5> C;
-        C array = {};
-        check_noexcept(array);
-        typename C::iterator i = array.begin();
-        typename C::const_iterator j = array.cbegin();
-        assert(i == j);
-    }
-    {
-        typedef std::array<int, 0> C;
-        C array = {};
-        check_noexcept(array);
-        typename C::iterator i = array.begin();
-        typename C::const_iterator j = array.cbegin();
-        assert(i == j);
-    }
-
-    {
-        typedef std::array<int, 0> C;
-        C array = {};
-        check_noexcept(array);
-        typename C::iterator i = array.begin();
-        typename C::const_iterator j = array.cbegin();
-        assert(i == array.end());
-        assert(j == array.cend());
-    }
-    {
-        typedef std::array<int, 1> C;
-        C array = {1};
-        check_noexcept(array);
-        typename C::iterator i = array.begin();
-        assert(*i == 1);
-        assert(&*i == array.data());
-        *i = 99;
-        assert(array[0] == 99);
-    }
-    {
-        typedef std::array<int, 2> C;
-        C array = {1, 2};
-        check_noexcept(array);
-        typename C::iterator i = array.begin();
-        assert(*i == 1);
-        assert(&*i == array.data());
-        *i = 99;
-        assert(array[0] == 99);
-        assert(array[1] == 2);
-    }
-    {
-        typedef std::array<double, 3> C;
-        C array = {1, 2, 3.5};
-        check_noexcept(array);
-        typename C::iterator i = array.begin();
-        assert(*i == 1);
-        assert(&*i == array.data());
-        *i = 5.5;
-        assert(array[0] == 5.5);
-        assert(array[1] == 2.0);
-    }
-    {
-        typedef std::array<NoDefault, 0> C;
-        C array = {};
-        typename C::iterator ib = array.begin();
-        typename C::iterator ie = array.end();
-        assert(ib == ie);
-    }
+TEST_CONSTEXPR_CXX17 bool tests() {
+  {
+    typedef std::array<int, 5> C;
+    C array = {};
+    check_noexcept(array);
+    typename C::iterator i       = array.begin();
+    typename C::const_iterator j = array.cbegin();
+    assert(i == j);
+  }
+  {
+    typedef std::array<int, 0> C;
+    C array = {};
+    check_noexcept(array);
+    typename C::iterator i       = array.begin();
+    typename C::const_iterator j = array.cbegin();
+    assert(i == j);
+  }
+
+  {
+    typedef std::array<int, 0> C;
+    C array = {};
+    check_noexcept(array);
+    typename C::iterator i       = array.begin();
+    typename C::const_iterator j = array.cbegin();
+    assert(i == array.end());
+    assert(j == array.cend());
+  }
+  {
+    typedef std::array<int, 1> C;
+    C array = {1};
+    check_noexcept(array);
+    typename C::iterator i = array.begin();
+    assert(*i == 1);
+    assert(&*i == array.data());
+    *i = 99;
+    assert(array[0] == 99);
+  }
+  {
+    typedef std::array<int, 2> C;
+    C array = {1, 2};
+    check_noexcept(array);
+    typename C::iterator i = array.begin();
+    assert(*i == 1);
+    assert(&*i == array.data());
+    *i = 99;
+    assert(array[0] == 99);
+    assert(array[1] == 2);
+  }
+  {
+    typedef std::array<double, 3> C;
+    C array = {1, 2, 3.5};
+    check_noexcept(array);
+    typename C::iterator i = array.begin();
+    assert(*i == 1);
+    assert(&*i == array.data());
+    *i = 5.5;
+    assert(array[0] == 5.5);
+    assert(array[1] == 2.0);
+  }
+  {
+    typedef std::array<NoDefault, 0> C;
+    C array                 = {};
+    typename C::iterator ib = array.begin();
+    typename C::iterator ie = array.end();
+    assert(ib == ie);
+  }
 
 #if TEST_STD_VER >= 14
-    { // N3644 testing
-        {
-            typedef std::array<int, 5> C;
-            C::iterator ii1{}, ii2{};
-            C::iterator ii4 = ii1;
-            C::const_iterator cii{};
-            assert(ii1 == ii2);
-            assert(ii1 == ii4);
-            assert(ii1 == cii);
-
-            assert(!(ii1 != ii2));
-            assert(!(ii1 != cii));
-
-            C c = {};
-            check_noexcept(c);
-            assert(c.begin()   == std::begin(c));
-            assert(c.cbegin()  == std::cbegin(c));
-            assert(c.rbegin()  == std::rbegin(c));
-            assert(c.crbegin() == std::crbegin(c));
-            assert(c.end()     == std::end(c));
-            assert(c.cend()    == std::cend(c));
-            assert(c.rend()    == std::rend(c));
-            assert(c.crend()   == std::crend(c));
-
-            assert(std::begin(c)   != std::end(c));
-            assert(std::rbegin(c)  != std::rend(c));
-            assert(std::cbegin(c)  != std::cend(c));
-            assert(std::crbegin(c) != std::crend(c));
+  { // N3644 testing
+    {
+      typedef std::array<int, 5> C;
+      C::iterator ii1{}, ii2{};
+      C::iterator ii4 = ii1;
+      C::const_iterator cii{};
+      assert(ii1 == ii2);
+      assert(ii1 == ii4);
+      assert(ii1 == cii);
+
+      assert(!(ii1 != ii2));
+      assert(!(ii1 != cii));
+
+      C c = {};
+      check_noexcept(c);
+      assert(c.begin() == std::begin(c));
+      assert(c.cbegin() == std::cbegin(c));
+      assert(c.rbegin() == std::rbegin(c));
+      assert(c.crbegin() == std::crbegin(c));
+      assert(c.end() == std::end(c));
+      assert(c.cend() == std::cend(c));
+      assert(c.rend() == std::rend(c));
+      assert(c.crend() == std::crend(c));
+
+      assert(std::begin(c) != std::end(c));
+      assert(std::rbegin(c) != std::rend(c));
+      assert(std::cbegin(c) != std::cend(c));
+      assert(std::crbegin(c) != std::crend(c));
 
 #  if TEST_STD_VER >= 20
-            // P1614 + LWG3352
-            std::same_as<std::strong_ordering> decltype(auto) r1 = ii1 <=> ii2;
-            assert(r1 == std::strong_ordering::equal);
+      // P1614 + LWG3352
+      std::same_as<std::strong_ordering> decltype(auto) r1 = ii1 <=> ii2;
+      assert(r1 == std::strong_ordering::equal);
 
-            std::same_as<std::strong_ordering> decltype(auto) r2 = cii <=> ii2;
-            assert(r2 == std::strong_ordering::equal);
+      std::same_as<std::strong_ordering> decltype(auto) r2 = cii <=> ii2;
+      assert(r2 == std::strong_ordering::equal);
 #  endif
-        }
-        {
-            typedef std::array<int, 0> C;
-            C::iterator ii1{}, ii2{};
-            C::iterator ii4 = ii1;
-            C::const_iterator cii{};
-            assert(ii1 == ii2);
-            assert(ii1 == ii4);
-
-            assert(!(ii1 != ii2));
-
-            assert( (ii1 == cii));
-            assert( (cii == ii1));
-            assert(!(ii1 != cii));
-            assert(!(cii != ii1));
-            assert(!(ii1 <  cii));
-            assert(!(cii <  ii1));
-            assert( (ii1 <= cii));
-            assert( (cii <= ii1));
-            assert(!(ii1 >  cii));
-            assert(!(cii >  ii1));
-            assert( (ii1 >= cii));
-            assert( (cii >= ii1));
-            assert(cii - ii1 == 0);
-            assert(ii1 - cii == 0);
-
-            C c = {};
-            check_noexcept(c);
-            assert(c.begin()   == std::begin(c));
-            assert(c.cbegin()  == std::cbegin(c));
-            assert(c.rbegin()  == std::rbegin(c));
-            assert(c.crbegin() == std::crbegin(c));
-            assert(c.end()     == std::end(c));
-            assert(c.cend()    == std::cend(c));
-            assert(c.rend()    == std::rend(c));
-            assert(c.crend()   == std::crend(c));
-
-            assert(std::begin(c)   == std::end(c));
-            assert(std::rbegin(c)  == std::rend(c));
-            assert(std::cbegin(c)  == std::cend(c));
-            assert(std::crbegin(c) == std::crend(c));
+    }
+    {
+      typedef std::array<int, 0> C;
+      C::iterator ii1{}, ii2{};
+      C::iterator ii4 = ii1;
+      C::const_iterator cii{};
+      assert(ii1 == ii2);
+      assert(ii1 == ii4);
+
+      assert(!(ii1 != ii2));
+
+      assert((ii1 == cii));
+      assert((cii == ii1));
+      assert(!(ii1 != cii));
+      assert(!(cii != ii1));
+      assert(!(ii1 < cii));
+      assert(!(cii < ii1));
+      assert((ii1 <= cii));
+      assert((cii <= ii1));
+      assert(!(ii1 > cii));
+      assert(!(cii > ii1));
+      assert((ii1 >= cii));
+      assert((cii >= ii1));
+      assert(cii - ii1 == 0);
+      assert(ii1 - cii == 0);
+
+      C c = {};
+      check_noexcept(c);
+      assert(c.begin() == std::begin(c));
+      assert(c.cbegin() == std::cbegin(c));
+      assert(c.rbegin() == std::rbegin(c));
+      assert(c.crbegin() == std::crbegin(c));
+      assert(c.end() == std::end(c));
+      assert(c.cend() == std::cend(c));
+      assert(c.rend() == std::rend(c));
+      assert(c.crend() == std::crend(c));
+
+      assert(std::begin(c) == std::end(c));
+      assert(std::rbegin(c) == std::rend(c));
+      assert(std::cbegin(c) == std::cend(c));
+      assert(std::crbegin(c) == std::crend(c));
 
 #  if TEST_STD_VER >= 20
-            // P1614 + LWG3352
-            std::same_as<std::strong_ordering> decltype(auto) r1 = ii1 <=> ii2;
-            assert(r1 == std::strong_ordering::equal);
+      // P1614 + LWG3352
+      std::same_as<std::strong_ordering> decltype(auto) r1 = ii1 <=> ii2;
+      assert(r1 == std::strong_ordering::equal);
 
-            std::same_as<std::strong_ordering> decltype(auto) r2 = cii <=> ii2;
-            assert(r2 == std::strong_ordering::equal);
+      std::same_as<std::strong_ordering> decltype(auto) r2 = cii <=> ii2;
+      assert(r2 == std::strong_ordering::equal);
 #  endif
-        }
     }
+  }
 #endif
-    return true;
+  return true;
 }
 
-int main(int, char**)
-{
-    tests();
+int main(int, char**) {
+  tests();
 #if TEST_STD_VER >= 17
-    static_assert(tests(), "");
+  static_assert(tests(), "");
 #endif
   return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/array/max_size.pass.cpp b/libcxx/test/std/containers/sequences/array/max_size.pass.cpp
index 8ca8cd44725e2..af6f25559b8dd 100644
--- a/libcxx/test/std/containers/sequences/array/max_size.pass.cpp
+++ b/libcxx/test/std/containers/sequences/array/max_size.pass.cpp
@@ -17,38 +17,36 @@
 
 #include "test_macros.h"
 
-TEST_CONSTEXPR_CXX14 bool tests()
-{
-    {
-        typedef std::array<int, 2> C;
-        C c = {};
-        ASSERT_NOEXCEPT(c.max_size());
-        assert(c.max_size() == 2);
-    }
-    {
-        typedef std::array<int, 0> C;
-        C c = {};
-        ASSERT_NOEXCEPT(c.max_size());
-        assert(c.max_size() == 0);
-    }
-
-    return true;
+TEST_CONSTEXPR_CXX14 bool tests() {
+  {
+    typedef std::array<int, 2> C;
+    C c = {};
+    ASSERT_NOEXCEPT(c.max_size());
+    assert(c.max_size() == 2);
+  }
+  {
+    typedef std::array<int, 0> C;
+    C c = {};
+    ASSERT_NOEXCEPT(c.max_size());
+    assert(c.max_size() == 0);
+  }
+
+  return true;
 }
 
-int main(int, char**)
-{
-    tests();
+int main(int, char**) {
+  tests();
 #if TEST_STD_VER >= 14
-    static_assert(tests(), "");
+  static_assert(tests(), "");
 #endif
 
 #if TEST_STD_VER >= 11
-    // Sanity check for constexpr in C++11
-    {
-        constexpr std::array<int, 3> array = {};
-        static_assert(array.max_size() == 3, "");
-    }
+  // Sanity check for constexpr in C++11
+  {
+    constexpr std::array<int, 3> array = {};
+    static_assert(array.max_size() == 3, "");
+  }
 #endif
 
-    return 0;
+  return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/array/range_concept_conformance.compile.pass.cpp b/libcxx/test/std/containers/sequences/array/range_concept_conformance.compile.pass.cpp
index 6e3f2a2801a2b..a398968d213b3 100644
--- a/libcxx/test/std/containers/sequences/array/range_concept_conformance.compile.pass.cpp
+++ b/libcxx/test/std/containers/sequences/array/range_concept_conformance.compile.pass.cpp
@@ -17,7 +17,6 @@
 
 using range = std::array<int, 10>;
 
-
 static_assert(!std::ranges::view<range>);
 static_assert(std::same_as<std::ranges::iterator_t<range>, range::iterator>);
 static_assert(std::ranges::common_range<range>);
diff --git a/libcxx/test/std/containers/sequences/array/types.pass.cpp b/libcxx/test/std/containers/sequences/array/types.pass.cpp
index c509810507962..f216d3ed15849 100644
--- a/libcxx/test/std/containers/sequences/array/types.pass.cpp
+++ b/libcxx/test/std/containers/sequences/array/types.pass.cpp
@@ -32,63 +32,66 @@
 
 template <class C>
 void test_iterators() {
-    typedef std::iterator_traits<typename C::iterator> ItT;
-    typedef std::iterator_traits<typename C::const_iterator> CItT;
-    static_assert((std::is_same<typename ItT::iterator_category, std::random_access_iterator_tag>::value), "");
-    static_assert((std::is_same<typename ItT::value_type, typename C::value_type>::value), "");
-    static_assert((std::is_same<typename ItT::reference, typename C::reference>::value), "");
-    static_assert((std::is_same<typename ItT::pointer, typename C::pointer>::value), "");
-    static_assert((std::is_same<typename ItT::difference_type, typename C::difference_type>::value), "");
+  typedef std::iterator_traits<typename C::iterator> ItT;
+  typedef std::iterator_traits<typename C::const_iterator> CItT;
+  static_assert((std::is_same<typename ItT::iterator_category, std::random_access_iterator_tag>::value), "");
+  static_assert((std::is_same<typename ItT::value_type, typename C::value_type>::value), "");
+  static_assert((std::is_same<typename ItT::reference, typename C::reference>::value), "");
+  static_assert((std::is_same<typename ItT::pointer, typename C::pointer>::value), "");
+  static_assert((std::is_same<typename ItT::difference_type, typename C::difference_type>::value), "");
 
-    static_assert((std::is_same<typename CItT::iterator_category, std::random_access_iterator_tag>::value), "");
-    static_assert((std::is_same<typename CItT::value_type, typename C::value_type>::value), "");
-    static_assert((std::is_same<typename CItT::reference, typename C::const_reference>::value), "");
-    static_assert((std::is_same<typename CItT::pointer, typename C::const_pointer>::value), "");
-    static_assert((std::is_same<typename CItT::difference_type, typename C::difference_type>::value), "");
+  static_assert((std::is_same<typename CItT::iterator_category, std::random_access_iterator_tag>::value), "");
+  static_assert((std::is_same<typename CItT::value_type, typename C::value_type>::value), "");
+  static_assert((std::is_same<typename CItT::reference, typename C::const_reference>::value), "");
+  static_assert((std::is_same<typename CItT::pointer, typename C::const_pointer>::value), "");
+  static_assert((std::is_same<typename CItT::difference_type, typename C::difference_type>::value), "");
 }
 
-int main(int, char**)
-{
-    {
-        typedef double T;
-        typedef std::array<T, 10> C;
-        static_assert((std::is_same<C::reference, T&>::value), "");
-        static_assert((std::is_same<C::const_reference, const T&>::value), "");
-        test_iterators<C>();
-        static_assert((std::is_same<C::pointer, T*>::value), "");
-        static_assert((std::is_same<C::const_pointer, const T*>::value), "");
-        static_assert((std::is_same<C::size_type, std::size_t>::value), "");
-        static_assert((std::is_same<C::difference_type, std::ptrdiff_t>::value), "");
-        static_assert((std::is_same<C::reverse_iterator, std::reverse_iterator<C::iterator> >::value), "");
-        static_assert((std::is_same<C::const_reverse_iterator, std::reverse_iterator<C::const_iterator> >::value), "");
+int main(int, char**) {
+  {
+    typedef double T;
+    typedef std::array<T, 10> C;
+    static_assert((std::is_same<C::reference, T&>::value), "");
+    static_assert((std::is_same<C::const_reference, const T&>::value), "");
+    test_iterators<C>();
+    static_assert((std::is_same<C::pointer, T*>::value), "");
+    static_assert((std::is_same<C::const_pointer, const T*>::value), "");
+    static_assert((std::is_same<C::size_type, std::size_t>::value), "");
+    static_assert((std::is_same<C::difference_type, std::ptrdiff_t>::value), "");
+    static_assert((std::is_same<C::reverse_iterator, std::reverse_iterator<C::iterator> >::value), "");
+    static_assert((std::is_same<C::const_reverse_iterator, std::reverse_iterator<C::const_iterator> >::value), "");
 
-        static_assert((std::is_signed<typename C::difference_type>::value), "");
-        static_assert((std::is_unsigned<typename C::size_type>::value), "");
-        static_assert((std::is_same<typename C::difference_type,
-            typename std::iterator_traits<typename C::iterator>::difference_type>::value), "");
-        static_assert((std::is_same<typename C::difference_type,
-            typename std::iterator_traits<typename C::const_iterator>::difference_type>::value), "");
-    }
-    {
-        typedef int* T;
-        typedef std::array<T, 0> C;
-        static_assert((std::is_same<C::reference, T&>::value), "");
-        static_assert((std::is_same<C::const_reference, const T&>::value), "");
-        test_iterators<C>();
-        static_assert((std::is_same<C::pointer, T*>::value), "");
-        static_assert((std::is_same<C::const_pointer, const T*>::value), "");
-        static_assert((std::is_same<C::size_type, std::size_t>::value), "");
-        static_assert((std::is_same<C::difference_type, std::ptrdiff_t>::value), "");
-        static_assert((std::is_same<C::reverse_iterator, std::reverse_iterator<C::iterator> >::value), "");
-        static_assert((std::is_same<C::const_reverse_iterator, std::reverse_iterator<C::const_iterator> >::value), "");
+    static_assert((std::is_signed<typename C::difference_type>::value), "");
+    static_assert((std::is_unsigned<typename C::size_type>::value), "");
+    static_assert((std::is_same<typename C::difference_type,
+                                typename std::iterator_traits<typename C::iterator>::difference_type>::value),
+                  "");
+    static_assert((std::is_same<typename C::difference_type,
+                                typename std::iterator_traits<typename C::const_iterator>::difference_type>::value),
+                  "");
+  }
+  {
+    typedef int* T;
+    typedef std::array<T, 0> C;
+    static_assert((std::is_same<C::reference, T&>::value), "");
+    static_assert((std::is_same<C::const_reference, const T&>::value), "");
+    test_iterators<C>();
+    static_assert((std::is_same<C::pointer, T*>::value), "");
+    static_assert((std::is_same<C::const_pointer, const T*>::value), "");
+    static_assert((std::is_same<C::size_type, std::size_t>::value), "");
+    static_assert((std::is_same<C::difference_type, std::ptrdiff_t>::value), "");
+    static_assert((std::is_same<C::reverse_iterator, std::reverse_iterator<C::iterator> >::value), "");
+    static_assert((std::is_same<C::const_reverse_iterator, std::reverse_iterator<C::const_iterator> >::value), "");
 
-        static_assert((std::is_signed<typename C::difference_type>::value), "");
-        static_assert((std::is_unsigned<typename C::size_type>::value), "");
-        static_assert((std::is_same<typename C::difference_type,
-            typename std::iterator_traits<typename C::iterator>::difference_type>::value), "");
-        static_assert((std::is_same<typename C::difference_type,
-            typename std::iterator_traits<typename C::const_iterator>::difference_type>::value), "");
-    }
+    static_assert((std::is_signed<typename C::difference_type>::value), "");
+    static_assert((std::is_unsigned<typename C::size_type>::value), "");
+    static_assert((std::is_same<typename C::difference_type,
+                                typename std::iterator_traits<typename C::iterator>::difference_type>::value),
+                  "");
+    static_assert((std::is_same<typename C::difference_type,
+                                typename std::iterator_traits<typename C::const_iterator>::difference_type>::value),
+                  "");
+  }
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/deque/allocator_mismatch.verify.cpp b/libcxx/test/std/containers/sequences/deque/allocator_mismatch.verify.cpp
index 1096e1dc71369..81648cb1624c1 100644
--- a/libcxx/test/std/containers/sequences/deque/allocator_mismatch.verify.cpp
+++ b/libcxx/test/std/containers/sequences/deque/allocator_mismatch.verify.cpp
@@ -12,4 +12,4 @@
 #include <deque>
 
 std::deque<int, std::allocator<long> > d;
-  // expected-error-re@*:* {{static assertion failed{{.*}}Allocator::value_type must be same type as value_type}}
+// expected-error-re@*:* {{static assertion failed{{.*}}Allocator::value_type must be same type as value_type}}
diff --git a/libcxx/test/std/containers/sequences/deque/compare.pass.cpp b/libcxx/test/std/containers/sequences/deque/compare.pass.cpp
index fc7c07e2d0d48..526e3d38e7dff 100644
--- a/libcxx/test/std/containers/sequences/deque/compare.pass.cpp
+++ b/libcxx/test/std/containers/sequences/deque/compare.pass.cpp
@@ -37,83 +37,82 @@
 
 #include "test_comparisons.h"
 
-int main(int, char**)
-{
-    {
-        const std::deque<int> d1, d2;
-        assert(testComparisons(d1, d2, true, false));
-    }
-    {
-        const std::deque<int> d1(1, 1), d2(1, 1);
-        assert(testComparisons(d1, d2, true, false));
-    }
-    {
-        int items[3] = {1, 2, 3};
-        const std::deque<int> d1(items, items + 3);
-        const std::deque<int> d2(items, items + 3);
-        assert(testComparisons(d1, d2, true, false));
-    }
-    {
-        const std::deque<int> d1(1, 1), d2;
-        assert(testComparisons(d1, d2, false, false));
-    }
-    {
-        const std::deque<int> d1(1, 1), d2(1, 2);
-        assert(testComparisons(d1, d2, false, true));
-    }
-    {
-        int items1[2] = {1, 2};
-        int items2[2] = {1, 3};
-        const std::deque<int> d1(items1, items1 + 2);
-        const std::deque<int> d2(items2, items2 + 2);
-        assert(testComparisons(d1, d2, false, true));
-    }
-    {
-        int items1[2] = {2, 2};
-        int items2[2] = {1, 3};
-        const std::deque<int> d1(items1, items1 + 2);
-        const std::deque<int> d2(items2, items2 + 2);
-        assert(testComparisons(d1, d2, false, false));
-    }
-    {
-        const std::deque<LessAndEqComp> d1, d2;
-        assert(testComparisons(d1, d2, true, false));
-    }
-    {
-        const std::deque<LessAndEqComp> d1(1, LessAndEqComp(1));
-        const std::deque<LessAndEqComp> d2(1, LessAndEqComp(1));
-        assert(testComparisons(d1, d2, true, false));
-    }
-    {
-        LessAndEqComp items[3] = {LessAndEqComp(1), LessAndEqComp(2), LessAndEqComp(3)};
-        const std::deque<LessAndEqComp> d1(items, items + 3);
-        const std::deque<LessAndEqComp> d2(items, items + 3);
-        assert(testComparisons(d1, d2, true, false));
-    }
-    {
-        const std::deque<LessAndEqComp> d1(1, LessAndEqComp(1));
-        const std::deque<LessAndEqComp> d2;
-        assert(testComparisons(d1, d2, false, false));
-    }
-    {
-        const std::deque<LessAndEqComp> d1(1, LessAndEqComp(1));
-        const std::deque<LessAndEqComp> d2(1, LessAndEqComp(2));
-        assert(testComparisons(d1, d2, false, true));
-    }
-    {
-        LessAndEqComp items1[2] = {LessAndEqComp(1), LessAndEqComp(2)};
-        LessAndEqComp items2[2] = {LessAndEqComp(1), LessAndEqComp(3)};
-        const std::deque<LessAndEqComp> d1(items1, items1 + 2);
-        const std::deque<LessAndEqComp> d2(items2, items2 + 2);
-        assert(testComparisons(d1, d2, false, true));
-    }
-    {
-        LessAndEqComp items1[2] = {LessAndEqComp(2), LessAndEqComp(2)};
-        LessAndEqComp items2[2] = {LessAndEqComp(1), LessAndEqComp(3)};
-        const std::deque<LessAndEqComp> d1(items1, items1 + 2);
-        const std::deque<LessAndEqComp> d2(items2, items2 + 2);
-        assert(testComparisons(d1, d2, false, false));
-    }
+int main(int, char**) {
+  {
+    const std::deque<int> d1, d2;
+    assert(testComparisons(d1, d2, true, false));
+  }
+  {
+    const std::deque<int> d1(1, 1), d2(1, 1);
+    assert(testComparisons(d1, d2, true, false));
+  }
+  {
+    int items[3] = {1, 2, 3};
+    const std::deque<int> d1(items, items + 3);
+    const std::deque<int> d2(items, items + 3);
+    assert(testComparisons(d1, d2, true, false));
+  }
+  {
+    const std::deque<int> d1(1, 1), d2;
+    assert(testComparisons(d1, d2, false, false));
+  }
+  {
+    const std::deque<int> d1(1, 1), d2(1, 2);
+    assert(testComparisons(d1, d2, false, true));
+  }
+  {
+    int items1[2] = {1, 2};
+    int items2[2] = {1, 3};
+    const std::deque<int> d1(items1, items1 + 2);
+    const std::deque<int> d2(items2, items2 + 2);
+    assert(testComparisons(d1, d2, false, true));
+  }
+  {
+    int items1[2] = {2, 2};
+    int items2[2] = {1, 3};
+    const std::deque<int> d1(items1, items1 + 2);
+    const std::deque<int> d2(items2, items2 + 2);
+    assert(testComparisons(d1, d2, false, false));
+  }
+  {
+    const std::deque<LessAndEqComp> d1, d2;
+    assert(testComparisons(d1, d2, true, false));
+  }
+  {
+    const std::deque<LessAndEqComp> d1(1, LessAndEqComp(1));
+    const std::deque<LessAndEqComp> d2(1, LessAndEqComp(1));
+    assert(testComparisons(d1, d2, true, false));
+  }
+  {
+    LessAndEqComp items[3] = {LessAndEqComp(1), LessAndEqComp(2), LessAndEqComp(3)};
+    const std::deque<LessAndEqComp> d1(items, items + 3);
+    const std::deque<LessAndEqComp> d2(items, items + 3);
+    assert(testComparisons(d1, d2, true, false));
+  }
+  {
+    const std::deque<LessAndEqComp> d1(1, LessAndEqComp(1));
+    const std::deque<LessAndEqComp> d2;
+    assert(testComparisons(d1, d2, false, false));
+  }
+  {
+    const std::deque<LessAndEqComp> d1(1, LessAndEqComp(1));
+    const std::deque<LessAndEqComp> d2(1, LessAndEqComp(2));
+    assert(testComparisons(d1, d2, false, true));
+  }
+  {
+    LessAndEqComp items1[2] = {LessAndEqComp(1), LessAndEqComp(2)};
+    LessAndEqComp items2[2] = {LessAndEqComp(1), LessAndEqComp(3)};
+    const std::deque<LessAndEqComp> d1(items1, items1 + 2);
+    const std::deque<LessAndEqComp> d2(items2, items2 + 2);
+    assert(testComparisons(d1, d2, false, true));
+  }
+  {
+    LessAndEqComp items1[2] = {LessAndEqComp(2), LessAndEqComp(2)};
+    LessAndEqComp items2[2] = {LessAndEqComp(1), LessAndEqComp(3)};
+    const std::deque<LessAndEqComp> d1(items1, items1 + 2);
+    const std::deque<LessAndEqComp> d2(items2, items2 + 2);
+    assert(testComparisons(d1, d2, false, false));
+  }
 
-    return 0;
+  return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/deque/deque.capacity/access.pass.cpp b/libcxx/test/std/containers/sequences/deque/deque.capacity/access.pass.cpp
index add0456cdbe4e..294663c5bd356 100644
--- a/libcxx/test/std/containers/sequences/deque/deque.capacity/access.pass.cpp
+++ b/libcxx/test/std/containers/sequences/deque/deque.capacity/access.pass.cpp
@@ -29,98 +29,94 @@
 #include "test_macros.h"
 
 template <class C>
-C
-make(int size, int start = 0 )
-{
-    const int b = 4096 / sizeof(int);
-    int init = 0;
-    if (start > 0)
-    {
-        init = (start+1) / b + ((start+1) % b != 0);
-        init *= b;
-        --init;
-    }
-    C c(init, 0);
-    for (int i = 0; i < init-start; ++i)
-        c.pop_back();
-    for (int i = 0; i < size; ++i)
-        c.push_back(i);
-    for (int i = 0; i < start; ++i)
-        c.pop_front();
-    return c;
+C make(int size, int start = 0) {
+  const int b = 4096 / sizeof(int);
+  int init    = 0;
+  if (start > 0) {
+    init = (start + 1) / b + ((start + 1) % b != 0);
+    init *= b;
+    --init;
+  }
+  C c(init, 0);
+  for (int i = 0; i < init - start; ++i)
+    c.pop_back();
+  for (int i = 0; i < size; ++i)
+    c.push_back(i);
+  for (int i = 0; i < start; ++i)
+    c.pop_front();
+  return c;
 }
 
-int main(int, char**)
-{
-    {
-        typedef std::deque<int> C;
-        C c = make<std::deque<int> >(10);
-        ASSERT_SAME_TYPE(decltype(c[0]), C::reference);
-        LIBCPP_ASSERT_NOEXCEPT(   c[0]);
-        LIBCPP_ASSERT_NOEXCEPT(   c.front());
-        ASSERT_SAME_TYPE(decltype(c.front()), C::reference);
-        LIBCPP_ASSERT_NOEXCEPT(   c.back());
-        ASSERT_SAME_TYPE(decltype(c.back()), C::reference);
-        for (int i = 0; i < 10; ++i)
-            assert(c[i] == i);
-        for (int i = 0; i < 10; ++i)
-            assert(c.at(i) == i);
-        assert(c.front() == 0);
-        assert(c.back() == 9);
-        LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c));
-    }
-    {
-        typedef std::deque<int> C;
-        const C c = make<std::deque<int> >(10);
-        ASSERT_SAME_TYPE(decltype(c[0]), C::const_reference);
-        LIBCPP_ASSERT_NOEXCEPT(   c[0]);
-        LIBCPP_ASSERT_NOEXCEPT(   c.front());
-        ASSERT_SAME_TYPE(decltype(c.front()), C::const_reference);
-        LIBCPP_ASSERT_NOEXCEPT(   c.back());
-        ASSERT_SAME_TYPE(decltype(c.back()), C::const_reference);
-        for (int i = 0; i < 10; ++i)
-            assert(c[i] == i);
-        for (int i = 0; i < 10; ++i)
-            assert(c.at(i) == i);
-        assert(c.front() == 0);
-        assert(c.back() == 9);
-        LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c));
-    }
+int main(int, char**) {
+  {
+    typedef std::deque<int> C;
+    C c = make<std::deque<int> >(10);
+    ASSERT_SAME_TYPE(decltype(c[0]), C::reference);
+    LIBCPP_ASSERT_NOEXCEPT(c[0]);
+    LIBCPP_ASSERT_NOEXCEPT(c.front());
+    ASSERT_SAME_TYPE(decltype(c.front()), C::reference);
+    LIBCPP_ASSERT_NOEXCEPT(c.back());
+    ASSERT_SAME_TYPE(decltype(c.back()), C::reference);
+    for (int i = 0; i < 10; ++i)
+      assert(c[i] == i);
+    for (int i = 0; i < 10; ++i)
+      assert(c.at(i) == i);
+    assert(c.front() == 0);
+    assert(c.back() == 9);
+    LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c));
+  }
+  {
+    typedef std::deque<int> C;
+    const C c = make<std::deque<int> >(10);
+    ASSERT_SAME_TYPE(decltype(c[0]), C::const_reference);
+    LIBCPP_ASSERT_NOEXCEPT(c[0]);
+    LIBCPP_ASSERT_NOEXCEPT(c.front());
+    ASSERT_SAME_TYPE(decltype(c.front()), C::const_reference);
+    LIBCPP_ASSERT_NOEXCEPT(c.back());
+    ASSERT_SAME_TYPE(decltype(c.back()), C::const_reference);
+    for (int i = 0; i < 10; ++i)
+      assert(c[i] == i);
+    for (int i = 0; i < 10; ++i)
+      assert(c.at(i) == i);
+    assert(c.front() == 0);
+    assert(c.back() == 9);
+    LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c));
+  }
 #if TEST_STD_VER >= 11
-    {
-        typedef std::deque<int, min_allocator<int>> C;
-        C c = make<std::deque<int, min_allocator<int>> >(10);
-        ASSERT_SAME_TYPE(decltype(c[0]), C::reference);
-        LIBCPP_ASSERT_NOEXCEPT(   c[0]);
-        LIBCPP_ASSERT_NOEXCEPT(   c.front());
-        ASSERT_SAME_TYPE(decltype(c.front()), C::reference);
-        LIBCPP_ASSERT_NOEXCEPT(   c.back());
-        ASSERT_SAME_TYPE(decltype(c.back()), C::reference);
-        for (int i = 0; i < 10; ++i)
-            assert(c[i] == i);
-        for (int i = 0; i < 10; ++i)
-            assert(c.at(i) == i);
-        assert(c.front() == 0);
-        assert(c.back() == 9);
-        LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c));
-    }
-    {
-        typedef std::deque<int, min_allocator<int>> C;
-        const C c = make<std::deque<int, min_allocator<int>> >(10);
-        ASSERT_SAME_TYPE(decltype(c[0]), C::const_reference);
-        LIBCPP_ASSERT_NOEXCEPT(   c[0]);
-        LIBCPP_ASSERT_NOEXCEPT(   c.front());
-        ASSERT_SAME_TYPE(decltype(c.front()), C::const_reference);
-        LIBCPP_ASSERT_NOEXCEPT(   c.back());
-        ASSERT_SAME_TYPE(decltype(c.back()), C::const_reference);
-        for (int i = 0; i < 10; ++i)
-            assert(c[i] == i);
-        for (int i = 0; i < 10; ++i)
-            assert(c.at(i) == i);
-        assert(c.front() == 0);
-        assert(c.back() == 9);
-        LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c));
-    }
+  {
+    typedef std::deque<int, min_allocator<int>> C;
+    C c = make<std::deque<int, min_allocator<int>> >(10);
+    ASSERT_SAME_TYPE(decltype(c[0]), C::reference);
+    LIBCPP_ASSERT_NOEXCEPT(c[0]);
+    LIBCPP_ASSERT_NOEXCEPT(c.front());
+    ASSERT_SAME_TYPE(decltype(c.front()), C::reference);
+    LIBCPP_ASSERT_NOEXCEPT(c.back());
+    ASSERT_SAME_TYPE(decltype(c.back()), C::reference);
+    for (int i = 0; i < 10; ++i)
+      assert(c[i] == i);
+    for (int i = 0; i < 10; ++i)
+      assert(c.at(i) == i);
+    assert(c.front() == 0);
+    assert(c.back() == 9);
+    LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c));
+  }
+  {
+    typedef std::deque<int, min_allocator<int>> C;
+    const C c = make<std::deque<int, min_allocator<int>> >(10);
+    ASSERT_SAME_TYPE(decltype(c[0]), C::const_reference);
+    LIBCPP_ASSERT_NOEXCEPT(c[0]);
+    LIBCPP_ASSERT_NOEXCEPT(c.front());
+    ASSERT_SAME_TYPE(decltype(c.front()), C::const_reference);
+    LIBCPP_ASSERT_NOEXCEPT(c.back());
+    ASSERT_SAME_TYPE(decltype(c.back()), C::const_reference);
+    for (int i = 0; i < 10; ++i)
+      assert(c[i] == i);
+    for (int i = 0; i < 10; ++i)
+      assert(c.at(i) == i);
+    assert(c.front() == 0);
+    assert(c.back() == 9);
+    LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c));
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/sequences/deque/deque.capacity/empty.pass.cpp b/libcxx/test/std/containers/sequences/deque/deque.capacity/empty.pass.cpp
index 57393061c4bac..5e3a6ec1cea82 100644
--- a/libcxx/test/std/containers/sequences/deque/deque.capacity/empty.pass.cpp
+++ b/libcxx/test/std/containers/sequences/deque/deque.capacity/empty.pass.cpp
@@ -19,9 +19,8 @@
 #include "test_macros.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
+int main(int, char**) {
+  {
     typedef std::deque<int> C;
     C c;
     ASSERT_NOEXCEPT(c.empty());
@@ -32,9 +31,9 @@ int main(int, char**)
     c.clear();
     assert(c.empty());
     LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c));
-    }
+  }
 #if TEST_STD_VER >= 11
-    {
+  {
     typedef std::deque<int, min_allocator<int>> C;
     C c;
     ASSERT_NOEXCEPT(c.empty());
@@ -45,7 +44,7 @@ int main(int, char**)
     c.clear();
     assert(c.empty());
     LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c));
-    }
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/sequences/deque/deque.capacity/empty.verify.cpp b/libcxx/test/std/containers/sequences/deque/deque.capacity/empty.verify.cpp
index a2295088c0a9b..9bf3180eb4d8a 100644
--- a/libcxx/test/std/containers/sequences/deque/deque.capacity/empty.verify.cpp
+++ b/libcxx/test/std/containers/sequences/deque/deque.capacity/empty.verify.cpp
@@ -17,6 +17,6 @@
 #include <deque>
 
 void f() {
-    std::deque<int> c;
-    c.empty(); // expected-warning {{ignoring return value of function declared with 'nodiscard' attribute}}
+  std::deque<int> c;
+  c.empty(); // expected-warning {{ignoring return value of function declared with 'nodiscard' attribute}}
 }
diff --git a/libcxx/test/std/containers/sequences/deque/deque.capacity/max_size.pass.cpp b/libcxx/test/std/containers/sequences/deque/deque.capacity/max_size.pass.cpp
index 6f1d0f0491264..39feabe3889b3 100644
--- a/libcxx/test/std/containers/sequences/deque/deque.capacity/max_size.pass.cpp
+++ b/libcxx/test/std/containers/sequences/deque/deque.capacity/max_size.pass.cpp
@@ -31,8 +31,7 @@ int main(int, char**) {
   {
     typedef limited_allocator<int, (std::size_t)-1> A;
     typedef std::deque<int, A> C;
-    const C::size_type max_dist =
-        static_cast<C::size_type>(std::numeric_limits<C::difference_type>::max());
+    const C::size_type max_dist = static_cast<C::size_type>(std::numeric_limits<C::difference_type>::max());
     C c;
     assert(c.max_size() <= max_dist);
     LIBCPP_ASSERT(c.max_size() == max_dist);
@@ -40,8 +39,7 @@ int main(int, char**) {
   }
   {
     typedef std::deque<char> C;
-    const C::size_type max_dist =
-        static_cast<C::size_type>(std::numeric_limits<C::difference_type>::max());
+    const C::size_type max_dist = static_cast<C::size_type>(std::numeric_limits<C::difference_type>::max());
     C c;
     assert(c.max_size() <= max_dist);
     assert(c.max_size() <= alloc_max_size(c.get_allocator()));
diff --git a/libcxx/test/std/containers/sequences/deque/deque.capacity/resize_size.pass.cpp b/libcxx/test/std/containers/sequences/deque/deque.capacity/resize_size.pass.cpp
index a38dd60bd1861..767fe7ae8864e 100644
--- a/libcxx/test/std/containers/sequences/deque/deque.capacity/resize_size.pass.cpp
+++ b/libcxx/test/std/containers/sequences/deque/deque.capacity/resize_size.pass.cpp
@@ -21,82 +21,74 @@
 #include "min_allocator.h"
 
 template <class C>
-C
-make(int size, int start = 0 )
-{
-    const int b = 4096 / sizeof(int);
-    int init = 0;
-    if (start > 0)
-    {
-        init = (start+1) / b + ((start+1) % b != 0);
-        init *= b;
-        --init;
-    }
-    C c(init, 0);
-    for (int i = 0; i < init-start; ++i)
-        c.pop_back();
-    LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c));
-    for (int i = 0; i < size; ++i)
-        c.push_back(i);
-    LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c));
-    for (int i = 0; i < start; ++i)
-        c.pop_front();
-    LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c));
-    return c;
+C make(int size, int start = 0) {
+  const int b = 4096 / sizeof(int);
+  int init    = 0;
+  if (start > 0) {
+    init = (start + 1) / b + ((start + 1) % b != 0);
+    init *= b;
+    --init;
+  }
+  C c(init, 0);
+  for (int i = 0; i < init - start; ++i)
+    c.pop_back();
+  LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c));
+  for (int i = 0; i < size; ++i)
+    c.push_back(i);
+  LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c));
+  for (int i = 0; i < start; ++i)
+    c.pop_front();
+  LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c));
+  return c;
 }
 
 template <class C>
-void
-test(C& c1, int size)
-{
-    typedef typename C::const_iterator CI;
-    typename C::size_type c1_osize = c1.size();
-    c1.resize(size);
-    assert(c1.size() == static_cast<std::size_t>(size));
-    assert(static_cast<std::size_t>(std::distance(c1.begin(), c1.end())) == c1.size());
-    LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c1));
-    CI i = c1.begin();
-    for (int j = 0; static_cast<std::size_t>(j) < std::min(c1_osize, c1.size()); ++j, ++i)
-        assert(*i == j);
-    for (std::size_t j = c1_osize; j < c1.size(); ++j, ++i)
-        assert(*i == 0);
+void test(C& c1, int size) {
+  typedef typename C::const_iterator CI;
+  typename C::size_type c1_osize = c1.size();
+  c1.resize(size);
+  assert(c1.size() == static_cast<std::size_t>(size));
+  assert(static_cast<std::size_t>(std::distance(c1.begin(), c1.end())) == c1.size());
+  LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c1));
+  CI i = c1.begin();
+  for (int j = 0; static_cast<std::size_t>(j) < std::min(c1_osize, c1.size()); ++j, ++i)
+    assert(*i == j);
+  for (std::size_t j = c1_osize; j < c1.size(); ++j, ++i)
+    assert(*i == 0);
 }
 
 template <class C>
-void
-testN(int start, int N, int M)
-{
-    C c1 = make<C>(N, start);
-    test(c1, M);
+void testN(int start, int N, int M) {
+  C c1 = make<C>(N, start);
+  test(c1, M);
 }
 
-int main(int, char**)
-{
-    {
-    int rng[] = {0, 1, 2, 3, 1023, 1024, 1025, 2047, 2048, 2049};
-    const int N = sizeof(rng)/sizeof(rng[0]);
+int main(int, char**) {
+  {
+    int rng[]   = {0, 1, 2, 3, 1023, 1024, 1025, 2047, 2048, 2049};
+    const int N = sizeof(rng) / sizeof(rng[0]);
     for (int i = 0; i < N; ++i)
-        for (int j = 0; j < N; ++j)
-            for (int k = 0; k < N; ++k)
-                testN<std::deque<int> >(rng[i], rng[j], rng[k]);
-    }
+      for (int j = 0; j < N; ++j)
+        for (int k = 0; k < N; ++k)
+          testN<std::deque<int> >(rng[i], rng[j], rng[k]);
+  }
 #if TEST_STD_VER >= 11
-    {
-    int rng[] = {0, 1, 2, 3, 1023, 1024, 1025, 2047, 2048, 2049};
-    const int N = sizeof(rng)/sizeof(rng[0]);
+  {
+    int rng[]   = {0, 1, 2, 3, 1023, 1024, 1025, 2047, 2048, 2049};
+    const int N = sizeof(rng) / sizeof(rng[0]);
     for (int i = 0; i < N; ++i)
-        for (int j = 0; j < N; ++j)
-            for (int k = 0; k < N; ++k)
-                testN<std::deque<int, min_allocator<int>>>(rng[i], rng[j], rng[k]);
-    }
-    {
-    int rng[] = {0, 1, 2, 3, 1023, 1024, 1025, 2047, 2048, 2049};
-    const int N = sizeof(rng)/sizeof(rng[0]);
+      for (int j = 0; j < N; ++j)
+        for (int k = 0; k < N; ++k)
+          testN<std::deque<int, min_allocator<int>>>(rng[i], rng[j], rng[k]);
+  }
+  {
+    int rng[]   = {0, 1, 2, 3, 1023, 1024, 1025, 2047, 2048, 2049};
+    const int N = sizeof(rng) / sizeof(rng[0]);
     for (int i = 0; i < N; ++i)
-        for (int j = 0; j < N; ++j)
-            for (int k = 0; k < N; ++k)
-                testN<std::deque<int, safe_allocator<int>>>(rng[i], rng[j], rng[k]);
-    }
+      for (int j = 0; j < N; ++j)
+        for (int k = 0; k < N; ++k)
+          testN<std::deque<int, safe_allocator<int>>>(rng[i], rng[j], rng[k]);
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/sequences/deque/deque.capacity/resize_size_value.pass.cpp b/libcxx/test/std/containers/sequences/deque/deque.capacity/resize_size_value.pass.cpp
index e640825ee76a7..0abd94759a50f 100644
--- a/libcxx/test/std/containers/sequences/deque/deque.capacity/resize_size_value.pass.cpp
+++ b/libcxx/test/std/containers/sequences/deque/deque.capacity/resize_size_value.pass.cpp
@@ -21,82 +21,74 @@
 #include "min_allocator.h"
 
 template <class C>
-C
-make(int size, int start = 0 )
-{
-    const int b = 4096 / sizeof(int);
-    int init = 0;
-    if (start > 0)
-    {
-        init = (start+1) / b + ((start+1) % b != 0);
-        init *= b;
-        --init;
-    }
-    C c(init, 0);
-    for (int i = 0; i < init-start; ++i)
-        c.pop_back();
-    LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c));
-    for (int i = 0; i < size; ++i)
-        c.push_back(i);
-    LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c));
-    for (int i = 0; i < start; ++i)
-        c.pop_front();
-    LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c));
-    return c;
+C make(int size, int start = 0) {
+  const int b = 4096 / sizeof(int);
+  int init    = 0;
+  if (start > 0) {
+    init = (start + 1) / b + ((start + 1) % b != 0);
+    init *= b;
+    --init;
+  }
+  C c(init, 0);
+  for (int i = 0; i < init - start; ++i)
+    c.pop_back();
+  LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c));
+  for (int i = 0; i < size; ++i)
+    c.push_back(i);
+  LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c));
+  for (int i = 0; i < start; ++i)
+    c.pop_front();
+  LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c));
+  return c;
 }
 
 template <class C>
-void
-test(C& c1, int size, int x)
-{
-    typedef typename C::const_iterator CI;
-    typename C::size_type c1_osize = c1.size();
-    c1.resize(size, x);
-    assert(c1.size() == static_cast<std::size_t>(size));
-    assert(static_cast<std::size_t>(std::distance(c1.begin(), c1.end())) == c1.size());
-    LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c1));
-    CI i = c1.begin();
-    for (int j = 0; static_cast<std::size_t>(j) < std::min(c1_osize, c1.size()); ++j, ++i)
-        assert(*i == j);
-    for (std::size_t j = c1_osize; j < c1.size(); ++j, ++i)
-        assert(*i == x);
+void test(C& c1, int size, int x) {
+  typedef typename C::const_iterator CI;
+  typename C::size_type c1_osize = c1.size();
+  c1.resize(size, x);
+  assert(c1.size() == static_cast<std::size_t>(size));
+  assert(static_cast<std::size_t>(std::distance(c1.begin(), c1.end())) == c1.size());
+  LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c1));
+  CI i = c1.begin();
+  for (int j = 0; static_cast<std::size_t>(j) < std::min(c1_osize, c1.size()); ++j, ++i)
+    assert(*i == j);
+  for (std::size_t j = c1_osize; j < c1.size(); ++j, ++i)
+    assert(*i == x);
 }
 
 template <class C>
-void
-testN(int start, int N, int M)
-{
-    C c1 = make<C>(N, start);
-    test(c1, M, -10);
+void testN(int start, int N, int M) {
+  C c1 = make<C>(N, start);
+  test(c1, M, -10);
 }
 
-int main(int, char**)
-{
-    {
-    int rng[] = {0, 1, 2, 3, 1023, 1024, 1025, 2047, 2048, 2049};
-    const int N = sizeof(rng)/sizeof(rng[0]);
+int main(int, char**) {
+  {
+    int rng[]   = {0, 1, 2, 3, 1023, 1024, 1025, 2047, 2048, 2049};
+    const int N = sizeof(rng) / sizeof(rng[0]);
     for (int i = 0; i < N; ++i)
-        for (int j = 0; j < N; ++j)
-            for (int k = 0; k < N; ++k)
-                testN<std::deque<int> >(rng[i], rng[j], rng[k]);
-    }
+      for (int j = 0; j < N; ++j)
+        for (int k = 0; k < N; ++k)
+          testN<std::deque<int> >(rng[i], rng[j], rng[k]);
+  }
 #if TEST_STD_VER >= 11
-    {
-    int rng[] = {0, 1, 2, 3, 1023, 1024, 1025, 2047, 2048, 2049};
-    const int N = sizeof(rng)/sizeof(rng[0]);
+  {
+    int rng[]   = {0, 1, 2, 3, 1023, 1024, 1025, 2047, 2048, 2049};
+    const int N = sizeof(rng) / sizeof(rng[0]);
     for (int i = 0; i < N; ++i)
-        for (int j = 0; j < N; ++j)
-            for (int k = 0; k < N; ++k)
-                testN<std::deque<int, min_allocator<int>>>(rng[i], rng[j], rng[k]);
-    }
-    {
-    int rng[] = {0, 1, 2, 3, 1023, 1024, 1025, 2047, 2048, 2049};
-    const int N = sizeof(rng)/sizeof(rng[0]);
+      for (int j = 0; j < N; ++j)
+        for (int k = 0; k < N; ++k)
+          testN<std::deque<int, min_allocator<int>>>(rng[i], rng[j], rng[k]);
+  }
+  {
+    int rng[]   = {0, 1, 2, 3, 1023, 1024, 1025, 2047, 2048, 2049};
+    const int N = sizeof(rng) / sizeof(rng[0]);
     for (int i = 0; i < N; ++i)
-        for (int j = 0; j < N; ++j)
-            for (int k = 0; k < N; ++k)
-                testN<std::deque<int, safe_allocator<int>>>(rng[i], rng[j], rng[k]);
-    }
+      for (int j = 0; j < N; ++j)
+        for (int k = 0; k < N; ++k)
+          testN<std::deque<int, safe_allocator<int>>>(rng[i], rng[j], rng[k]);
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/sequences/deque/deque.capacity/shrink_to_fit.pass.cpp b/libcxx/test/std/containers/sequences/deque/deque.capacity/shrink_to_fit.pass.cpp
index 88e8bd53c71d5..34eedf8d050b7 100644
--- a/libcxx/test/std/containers/sequences/deque/deque.capacity/shrink_to_fit.pass.cpp
+++ b/libcxx/test/std/containers/sequences/deque/deque.capacity/shrink_to_fit.pass.cpp
@@ -18,69 +18,61 @@
 #include "min_allocator.h"
 
 template <class C>
-C
-make(int size, int start = 0 )
-{
-    const int b = 4096 / sizeof(int);
-    int init = 0;
-    if (start > 0)
-    {
-        init = (start+1) / b + ((start+1) % b != 0);
-        init *= b;
-        --init;
-    }
-    C c(init, 0);
-    for (int i = 0; i < init-start; ++i)
-        c.pop_back();
-    for (int i = 0; i < size; ++i)
-        c.push_back(i);
-    for (int i = 0; i < start; ++i)
-        c.pop_front();
-    return c;
+C make(int size, int start = 0) {
+  const int b = 4096 / sizeof(int);
+  int init    = 0;
+  if (start > 0) {
+    init = (start + 1) / b + ((start + 1) % b != 0);
+    init *= b;
+    --init;
+  }
+  C c(init, 0);
+  for (int i = 0; i < init - start; ++i)
+    c.pop_back();
+  for (int i = 0; i < size; ++i)
+    c.push_back(i);
+  for (int i = 0; i < start; ++i)
+    c.pop_front();
+  return c;
 }
 
 template <class C>
-void
-test(C& c1)
-{
-    C s = c1;
-    c1.shrink_to_fit();
-    assert(c1 == s);
-    LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c1));
+void test(C& c1) {
+  C s = c1;
+  c1.shrink_to_fit();
+  assert(c1 == s);
+  LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c1));
 }
 
 template <class C>
-void
-testN(int start, int N)
-{
-    C c1 = make<C>(N, start);
-    test(c1);
+void testN(int start, int N) {
+  C c1 = make<C>(N, start);
+  test(c1);
 }
 
-int main(int, char**)
-{
-    {
-    int rng[] = {0, 1, 2, 3, 1023, 1024, 1025, 2047, 2048, 2049};
-    const int N = sizeof(rng)/sizeof(rng[0]);
+int main(int, char**) {
+  {
+    int rng[]   = {0, 1, 2, 3, 1023, 1024, 1025, 2047, 2048, 2049};
+    const int N = sizeof(rng) / sizeof(rng[0]);
     for (int i = 0; i < N; ++i)
-        for (int j = 0; j < N; ++j)
-            testN<std::deque<int> >(rng[i], rng[j]);
-    }
+      for (int j = 0; j < N; ++j)
+        testN<std::deque<int> >(rng[i], rng[j]);
+  }
 #if TEST_STD_VER >= 11
-    {
-    int rng[] = {0, 1, 2, 3, 1023, 1024, 1025, 2047, 2048, 2049};
-    const int N = sizeof(rng)/sizeof(rng[0]);
+  {
+    int rng[]   = {0, 1, 2, 3, 1023, 1024, 1025, 2047, 2048, 2049};
+    const int N = sizeof(rng) / sizeof(rng[0]);
     for (int i = 0; i < N; ++i)
-        for (int j = 0; j < N; ++j)
-            testN<std::deque<int, min_allocator<int>> >(rng[i], rng[j]);
-    }
-    {
-    int rng[] = {0, 1, 2, 3, 1023, 1024, 1025, 2047, 2048, 2049};
-    const int N = sizeof(rng)/sizeof(rng[0]);
+      for (int j = 0; j < N; ++j)
+        testN<std::deque<int, min_allocator<int>> >(rng[i], rng[j]);
+  }
+  {
+    int rng[]   = {0, 1, 2, 3, 1023, 1024, 1025, 2047, 2048, 2049};
+    const int N = sizeof(rng) / sizeof(rng[0]);
     for (int i = 0; i < N; ++i)
-        for (int j = 0; j < N; ++j)
-            testN<std::deque<int, safe_allocator<int>> >(rng[i], rng[j]);
-    }
+      for (int j = 0; j < N; ++j)
+        testN<std::deque<int, safe_allocator<int>> >(rng[i], rng[j]);
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/sequences/deque/deque.capacity/size.pass.cpp b/libcxx/test/std/containers/sequences/deque/deque.capacity/size.pass.cpp
index 21d5ca03a10a2..5c6a34867b39a 100644
--- a/libcxx/test/std/containers/sequences/deque/deque.capacity/size.pass.cpp
+++ b/libcxx/test/std/containers/sequences/deque/deque.capacity/size.pass.cpp
@@ -19,9 +19,8 @@
 #include "test_macros.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
+int main(int, char**) {
+  {
     typedef std::deque<int> C;
     C c;
     ASSERT_NOEXCEPT(c.size());
@@ -45,9 +44,9 @@ int main(int, char**)
     c.erase(c.begin());
     assert(c.size() == 0);
     LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c));
-    }
+  }
 #if TEST_STD_VER >= 11
-    {
+  {
     typedef std::deque<int, min_allocator<int>> C;
     C c;
     ASSERT_NOEXCEPT(c.size());
@@ -71,7 +70,7 @@ int main(int, char**)
     c.erase(c.begin());
     assert(c.size() == 0);
     LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c));
-    }
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/sequences/deque/deque.cons/alloc.pass.cpp b/libcxx/test/std/containers/sequences/deque/deque.cons/alloc.pass.cpp
index 8e3361e35999f..4b19ef3e22173 100644
--- a/libcxx/test/std/containers/sequences/deque/deque.cons/alloc.pass.cpp
+++ b/libcxx/test/std/containers/sequences/deque/deque.cons/alloc.pass.cpp
@@ -20,26 +20,23 @@
 #include "min_allocator.h"
 
 template <class T, class Allocator>
-void
-test(const Allocator& a)
-{
-    std::deque<T, Allocator> d(a);
-    assert(d.size() == 0);
-    assert(d.get_allocator() == a);
-    LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(d));
+void test(const Allocator& a) {
+  std::deque<T, Allocator> d(a);
+  assert(d.size() == 0);
+  assert(d.get_allocator() == a);
+  LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(d));
 }
 
-int main(int, char**)
-{
-    test<int>(std::allocator<int>());
-    test<NotConstructible>(test_allocator<NotConstructible>(3));
+int main(int, char**) {
+  test<int>(std::allocator<int>());
+  test<NotConstructible>(test_allocator<NotConstructible>(3));
 #if TEST_STD_VER >= 11
-    test<int>(min_allocator<int>());
-    test<int>(safe_allocator<int>());
-    test<NotConstructible>(min_allocator<NotConstructible>{});
-    test<NotConstructible>(safe_allocator<NotConstructible>{});
-    test<int>(explicit_allocator<int>());
-    test<NotConstructible>(explicit_allocator<NotConstructible>{});
+  test<int>(min_allocator<int>());
+  test<int>(safe_allocator<int>());
+  test<NotConstructible>(min_allocator<NotConstructible>{});
+  test<NotConstructible>(safe_allocator<NotConstructible>{});
+  test<int>(explicit_allocator<int>());
+  test<NotConstructible>(explicit_allocator<NotConstructible>{});
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/sequences/deque/deque.cons/assign_initializer_list.pass.cpp b/libcxx/test/std/containers/sequences/deque/deque.cons/assign_initializer_list.pass.cpp
index 229034e8ca77f..bc57098463eaa 100644
--- a/libcxx/test/std/containers/sequences/deque/deque.cons/assign_initializer_list.pass.cpp
+++ b/libcxx/test/std/containers/sequences/deque/deque.cons/assign_initializer_list.pass.cpp
@@ -19,9 +19,8 @@
 #include "test_macros.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
+int main(int, char**) {
+  {
     std::deque<int> d;
     d.assign({3, 4, 5, 6});
     assert(d.size() == 4);
@@ -30,8 +29,8 @@ int main(int, char**)
     assert(d[2] == 5);
     assert(d[3] == 6);
     LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(d));
-    }
-    {
+  }
+  {
     std::deque<int, min_allocator<int>> d;
     d.assign({3, 4, 5, 6});
     assert(d.size() == 4);
@@ -40,7 +39,7 @@ int main(int, char**)
     assert(d[2] == 5);
     assert(d[3] == 6);
     LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(d));
-    }
+  }
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/deque/deque.cons/assign_iter_iter.pass.cpp b/libcxx/test/std/containers/sequences/deque/deque.cons/assign_iter_iter.pass.cpp
index c8ccfd6db7e47..c57eb9d08cabe 100644
--- a/libcxx/test/std/containers/sequences/deque/deque.cons/assign_iter_iter.pass.cpp
+++ b/libcxx/test/std/containers/sequences/deque/deque.cons/assign_iter_iter.pass.cpp
@@ -20,103 +20,91 @@
 #include "test_iterators.h"
 #include "min_allocator.h"
 #if TEST_STD_VER >= 11
-#include "emplace_constructible.h"
+#  include "emplace_constructible.h"
 #endif
 
 template <class C>
-C
-make(int size, int start = 0 )
-{
-    const int b = 4096 / sizeof(int);
-    int init = 0;
-    if (start > 0)
-    {
-        init = (start+1) / b + ((start+1) % b != 0);
-        init *= b;
-        --init;
-    }
-    C c(init, 0);
-    for (int i = 0; i < init-start; ++i)
-        c.pop_back();
-    for (int i = 0; i < size; ++i)
-        c.push_back(i);
-    for (int i = 0; i < start; ++i)
-        c.pop_front();
-    return c;
+C make(int size, int start = 0) {
+  const int b = 4096 / sizeof(int);
+  int init    = 0;
+  if (start > 0) {
+    init = (start + 1) / b + ((start + 1) % b != 0);
+    init *= b;
+    --init;
+  }
+  C c(init, 0);
+  for (int i = 0; i < init - start; ++i)
+    c.pop_back();
+  for (int i = 0; i < size; ++i)
+    c.push_back(i);
+  for (int i = 0; i < start; ++i)
+    c.pop_front();
+  return c;
 }
 
 template <class C>
-void
-test(C& c1, const C& c2)
-{
-    c1.assign(c2.begin(), c2.end());
-    assert(static_cast<std::size_t>(std::distance(c1.begin(), c1.end())) == c1.size());
-    assert(c1 == c2);
-    LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c1));
-    LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c2));
+void test(C& c1, const C& c2) {
+  c1.assign(c2.begin(), c2.end());
+  assert(static_cast<std::size_t>(std::distance(c1.begin(), c1.end())) == c1.size());
+  assert(c1 == c2);
+  LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c1));
+  LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c2));
 }
 
 template <class C>
-void
-testN(int start, int N, int M)
-{
-    C c1 = make<C>(N, start);
-    C c2 = make<C>(M);
-    test(c1, c2);
+void testN(int start, int N, int M) {
+  C c1 = make<C>(N, start);
+  C c2 = make<C>(M);
+  test(c1, c2);
 }
 
 template <class C>
-void
-testI(C& c1, const C& c2)
-{
-    typedef typename C::const_iterator CI;
-    typedef cpp17_input_iterator<CI> ICI;
-    c1.assign(ICI(c2.begin()), ICI(c2.end()));
-    assert(static_cast<std::size_t>(std::distance(c1.begin(), c1.end())) == c1.size());
-    assert(c1 == c2);
-    LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c1));
-    LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c2));
+void testI(C& c1, const C& c2) {
+  typedef typename C::const_iterator CI;
+  typedef cpp17_input_iterator<CI> ICI;
+  c1.assign(ICI(c2.begin()), ICI(c2.end()));
+  assert(static_cast<std::size_t>(std::distance(c1.begin(), c1.end())) == c1.size());
+  assert(c1 == c2);
+  LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c1));
+  LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c2));
 }
 
 template <class C>
-void
-testNI(int start, int N, int M)
-{
-    C c1 = make<C>(N, start);
-    C c2 = make<C>(M);
-    testI(c1, c2);
+void testNI(int start, int N, int M) {
+  C c1 = make<C>(N, start);
+  C c2 = make<C>(M);
+  testI(c1, c2);
 }
 
-void basic_test()
-{
-    {
-    int rng[] = {0, 1, 2, 3, 1023, 1024, 1025, 2047, 2048, 2049};
-    const int N = sizeof(rng)/sizeof(rng[0]);
+void basic_test() {
+  {
+    int rng[]   = {0, 1, 2, 3, 1023, 1024, 1025, 2047, 2048, 2049};
+    const int N = sizeof(rng) / sizeof(rng[0]);
     for (int i = 0; i < N; ++i)
-        for (int j = 0; j < N; ++j)
-            for (int k = 0; k < N; ++k)
-                testN<std::deque<int> >(rng[i], rng[j], rng[k]);
+      for (int j = 0; j < N; ++j)
+        for (int k = 0; k < N; ++k)
+          testN<std::deque<int> >(rng[i], rng[j], rng[k]);
     testNI<std::deque<int> >(1500, 2000, 1000);
-    }
+  }
 #if TEST_STD_VER >= 11
-    {
-    int rng[] = {0, 1, 2, 3, 1023, 1024, 1025, 2047, 2048, 2049};
-    const int N = sizeof(rng)/sizeof(rng[0]);
+  {
+    int rng[]   = {0, 1, 2, 3, 1023, 1024, 1025, 2047, 2048, 2049};
+    const int N = sizeof(rng) / sizeof(rng[0]);
     for (int i = 0; i < N; ++i)
-        for (int j = 0; j < N; ++j)
-            for (int k = 0; k < N; ++k)
-                testN<std::deque<int, min_allocator<int>> >(rng[i], rng[j], rng[k]);
+      for (int j = 0; j < N; ++j)
+        for (int k = 0; k < N; ++k)
+          testN<std::deque<int, min_allocator<int>> >(rng[i], rng[j], rng[k]);
     testNI<std::deque<int, min_allocator<int>> >(1500, 2000, 1000);
-    }
-    {
-    int rng[] = {0, 1, 2, 3, 1023, 1024, 1025, 2047, 2048, 2049};
-    const int N = sizeof(rng)/sizeof(rng[0]);
+  }
+  {
+    int rng[]   = {0, 1, 2, 3, 1023, 1024, 1025, 2047, 2048, 2049};
+    const int N = sizeof(rng) / sizeof(rng[0]);
     for (int i = 0; i < N; ++i)
-        for (int j = 0; j < N; ++j)
-            for (int k = 0; k < N; ++k)
-                testN<std::deque<int, safe_allocator<int>> >(rng[i], rng[j], rng[k]);
+      for (int j = 0; j < N; ++j)
+        for (int k = 0; k < N; ++k)
+          testN<std::deque<int, safe_allocator<int>> >(rng[i], rng[j], rng[k]);
     testNI<std::deque<int, safe_allocator<int>> >(1500, 2000, 1000);
-    }
+  }
 #endif
 }
 
diff --git a/libcxx/test/std/containers/sequences/deque/deque.cons/assign_size_value.pass.cpp b/libcxx/test/std/containers/sequences/deque/deque.cons/assign_size_value.pass.cpp
index a5272d010cdea..30277125e46e1 100644
--- a/libcxx/test/std/containers/sequences/deque/deque.cons/assign_size_value.pass.cpp
+++ b/libcxx/test/std/containers/sequences/deque/deque.cons/assign_size_value.pass.cpp
@@ -20,67 +20,59 @@
 #include "min_allocator.h"
 
 template <class C>
-C
-make(int size, int start = 0 )
-{
-    const int b = 4096 / sizeof(int);
-    int init = 0;
-    if (start > 0)
-    {
-        init = (start+1) / b + ((start+1) % b != 0);
-        init *= b;
-        --init;
-    }
-    C c(init, 0);
-    for (int i = 0; i < init-start; ++i)
-        c.pop_back();
-    for (int i = 0; i < size; ++i)
-        c.push_back(i);
-    for (int i = 0; i < start; ++i)
-        c.pop_front();
-    return c;
+C make(int size, int start = 0) {
+  const int b = 4096 / sizeof(int);
+  int init    = 0;
+  if (start > 0) {
+    init = (start + 1) / b + ((start + 1) % b != 0);
+    init *= b;
+    --init;
+  }
+  C c(init, 0);
+  for (int i = 0; i < init - start; ++i)
+    c.pop_back();
+  for (int i = 0; i < size; ++i)
+    c.push_back(i);
+  for (int i = 0; i < start; ++i)
+    c.pop_front();
+  return c;
 }
 
 template <class C>
-void
-test(C& c1, int size, int v)
-{
-    typedef typename C::const_iterator CI;
-    c1.assign(size, v);
-    assert(c1.size() == static_cast<std::size_t>(size));
-    assert(static_cast<std::size_t>(std::distance(c1.begin(), c1.end())) == c1.size());
-    LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c1));
-    for (CI i = c1.begin(); i != c1.end(); ++i)
-        assert(*i == v);
+void test(C& c1, int size, int v) {
+  typedef typename C::const_iterator CI;
+  c1.assign(size, v);
+  assert(c1.size() == static_cast<std::size_t>(size));
+  assert(static_cast<std::size_t>(std::distance(c1.begin(), c1.end())) == c1.size());
+  LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c1));
+  for (CI i = c1.begin(); i != c1.end(); ++i)
+    assert(*i == v);
 }
 
 template <class C>
-void
-testN(int start, int N, int M)
-{
-    C c1 = make<C>(N, start);
-    test(c1, M, -10);
+void testN(int start, int N, int M) {
+  C c1 = make<C>(N, start);
+  test(c1, M, -10);
 }
 
-int main(int, char**)
-{
-    {
-    int rng[] = {0, 1, 2, 3, 1023, 1024, 1025, 2047, 2048, 2049};
-    const int N = sizeof(rng)/sizeof(rng[0]);
+int main(int, char**) {
+  {
+    int rng[]   = {0, 1, 2, 3, 1023, 1024, 1025, 2047, 2048, 2049};
+    const int N = sizeof(rng) / sizeof(rng[0]);
     for (int i = 0; i < N; ++i)
-        for (int j = 0; j < N; ++j)
-            for (int k = 0; k < N; ++k)
-                testN<std::deque<int> >(rng[i], rng[j], rng[k]);
-    }
+      for (int j = 0; j < N; ++j)
+        for (int k = 0; k < N; ++k)
+          testN<std::deque<int> >(rng[i], rng[j], rng[k]);
+  }
 #if TEST_STD_VER >= 11
-    {
-    int rng[] = {0, 1, 2, 3, 1023, 1024, 1025, 2047, 2048, 2049};
-    const int N = sizeof(rng)/sizeof(rng[0]);
+  {
+    int rng[]   = {0, 1, 2, 3, 1023, 1024, 1025, 2047, 2048, 2049};
+    const int N = sizeof(rng) / sizeof(rng[0]);
     for (int i = 0; i < N; ++i)
-        for (int j = 0; j < N; ++j)
-            for (int k = 0; k < N; ++k)
-                testN<std::deque<int, min_allocator<int>> >(rng[i], rng[j], rng[k]);
-    }
+      for (int j = 0; j < N; ++j)
+        for (int k = 0; k < N; ++k)
+          testN<std::deque<int, min_allocator<int>> >(rng[i], rng[j], rng[k]);
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/sequences/deque/deque.cons/copy.pass.cpp b/libcxx/test/std/containers/sequences/deque/deque.cons/copy.pass.cpp
index 44245916c0a31..9f7a429d122a3 100644
--- a/libcxx/test/std/containers/sequences/deque/deque.cons/copy.pass.cpp
+++ b/libcxx/test/std/containers/sequences/deque/deque.cons/copy.pass.cpp
@@ -19,52 +19,49 @@
 #include "min_allocator.h"
 
 template <class C>
-void
-test(const C& x)
-{
-    C c(x);
-    assert(c == x);
-    LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c));
-    LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(x));
+void test(const C& x) {
+  C c(x);
+  assert(c == x);
+  LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c));
+  LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(x));
 }
 
-int main(int, char**)
-{
-    {
-        int ab[] = {3, 4, 2, 8, 0, 1, 44, 34, 45, 96, 80, 1, 13, 31, 45};
-        int* an = ab + sizeof(ab)/sizeof(ab[0]);
-        test(std::deque<int>(ab, an));
-    }
-    {
-        std::deque<int, test_allocator<int> > v(3, 2, test_allocator<int>(5));
-        std::deque<int, test_allocator<int> > v2 = v;
-        assert(v2 == v);
-        assert(v2.get_allocator() == v.get_allocator());
-        LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(v));
-        LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(v2));
-    }
+int main(int, char**) {
+  {
+    int ab[] = {3, 4, 2, 8, 0, 1, 44, 34, 45, 96, 80, 1, 13, 31, 45};
+    int* an  = ab + sizeof(ab) / sizeof(ab[0]);
+    test(std::deque<int>(ab, an));
+  }
+  {
+    std::deque<int, test_allocator<int> > v(3, 2, test_allocator<int>(5));
+    std::deque<int, test_allocator<int> > v2 = v;
+    assert(v2 == v);
+    assert(v2.get_allocator() == v.get_allocator());
+    LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(v));
+    LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(v2));
+  }
 #if TEST_STD_VER >= 11
-    {
-        std::deque<int, other_allocator<int> > v(3, 2, other_allocator<int>(5));
-        std::deque<int, other_allocator<int> > v2 = v;
-        assert(v2 == v);
-        assert(v2.get_allocator() == other_allocator<int>(-2));
-        LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(v));
-        LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(v2));
-    }
-    {
-        int ab[] = {3, 4, 2, 8, 0, 1, 44, 34, 45, 96, 80, 1, 13, 31, 45};
-        int* an = ab + sizeof(ab)/sizeof(ab[0]);
-        test(std::deque<int, min_allocator<int>>(ab, an));
-    }
-    {
-        std::deque<int, min_allocator<int> > v(3, 2, min_allocator<int>());
-        std::deque<int, min_allocator<int> > v2 = v;
-        assert(v2 == v);
-        assert(v2.get_allocator() == v.get_allocator());
-        LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(v));
-        LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(v2));
-    }
+  {
+    std::deque<int, other_allocator<int> > v(3, 2, other_allocator<int>(5));
+    std::deque<int, other_allocator<int> > v2 = v;
+    assert(v2 == v);
+    assert(v2.get_allocator() == other_allocator<int>(-2));
+    LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(v));
+    LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(v2));
+  }
+  {
+    int ab[] = {3, 4, 2, 8, 0, 1, 44, 34, 45, 96, 80, 1, 13, 31, 45};
+    int* an  = ab + sizeof(ab) / sizeof(ab[0]);
+    test(std::deque<int, min_allocator<int>>(ab, an));
+  }
+  {
+    std::deque<int, min_allocator<int> > v(3, 2, min_allocator<int>());
+    std::deque<int, min_allocator<int> > v2 = v;
+    assert(v2 == v);
+    assert(v2.get_allocator() == v.get_allocator());
+    LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(v));
+    LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(v2));
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/sequences/deque/deque.cons/copy_alloc.pass.cpp b/libcxx/test/std/containers/sequences/deque/deque.cons/copy_alloc.pass.cpp
index 3fdfcc3405f0e..fb3ad3c25a8f1 100644
--- a/libcxx/test/std/containers/sequences/deque/deque.cons/copy_alloc.pass.cpp
+++ b/libcxx/test/std/containers/sequences/deque/deque.cons/copy_alloc.pass.cpp
@@ -19,42 +19,35 @@
 #include "min_allocator.h"
 
 template <class C>
-void
-test(const C& x, const typename C::allocator_type& a)
-{
-    C c(x, a);
-    assert(c == x);
-    assert(c.get_allocator() == a);
-    LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c));
+void test(const C& x, const typename C::allocator_type& a) {
+  C c(x, a);
+  assert(c == x);
+  assert(c.get_allocator() == a);
+  LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c));
 }
 
-int main(int, char**)
-{
-    {
-        int ab[] = {3, 4, 2, 8, 0, 1, 44, 34, 45, 96, 80, 1, 13, 31, 45};
-        int* an = ab + sizeof(ab)/sizeof(ab[0]);
-        test(std::deque<int, test_allocator<int> >(ab, an, test_allocator<int>(3)),
-                                                           test_allocator<int>(4));
-    }
-    {
-        int ab[] = {3, 4, 2, 8, 0, 1, 44, 34, 45, 96, 80, 1, 13, 31, 45};
-        int* an = ab + sizeof(ab)/sizeof(ab[0]);
-        test(std::deque<int, other_allocator<int> >(ab, an, other_allocator<int>(3)),
-                                                            other_allocator<int>(4));
-    }
+int main(int, char**) {
+  {
+    int ab[] = {3, 4, 2, 8, 0, 1, 44, 34, 45, 96, 80, 1, 13, 31, 45};
+    int* an  = ab + sizeof(ab) / sizeof(ab[0]);
+    test(std::deque<int, test_allocator<int> >(ab, an, test_allocator<int>(3)), test_allocator<int>(4));
+  }
+  {
+    int ab[] = {3, 4, 2, 8, 0, 1, 44, 34, 45, 96, 80, 1, 13, 31, 45};
+    int* an  = ab + sizeof(ab) / sizeof(ab[0]);
+    test(std::deque<int, other_allocator<int> >(ab, an, other_allocator<int>(3)), other_allocator<int>(4));
+  }
 #if TEST_STD_VER >= 11
-    {
-        int ab[] = {3, 4, 2, 8, 0, 1, 44, 34, 45, 96, 80, 1, 13, 31, 45};
-        int* an = ab + sizeof(ab)/sizeof(ab[0]);
-        test(std::deque<int, min_allocator<int> >(ab, an, min_allocator<int>()),
-                                                          min_allocator<int>());
-    }
-    {
-        int ab[] = {3, 4, 2, 8, 0, 1, 44, 34, 45, 96, 80, 1, 13, 31, 45};
-        int* an = ab + sizeof(ab)/sizeof(ab[0]);
-        test(std::deque<int, safe_allocator<int> >(ab, an, safe_allocator<int>()),
-                                                          safe_allocator<int>());
-    }
+  {
+    int ab[] = {3, 4, 2, 8, 0, 1, 44, 34, 45, 96, 80, 1, 13, 31, 45};
+    int* an  = ab + sizeof(ab) / sizeof(ab[0]);
+    test(std::deque<int, min_allocator<int> >(ab, an, min_allocator<int>()), min_allocator<int>());
+  }
+  {
+    int ab[] = {3, 4, 2, 8, 0, 1, 44, 34, 45, 96, 80, 1, 13, 31, 45};
+    int* an  = ab + sizeof(ab) / sizeof(ab[0]);
+    test(std::deque<int, safe_allocator<int> >(ab, an, safe_allocator<int>()), safe_allocator<int>());
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/sequences/deque/deque.cons/deduct.pass.cpp b/libcxx/test/std/containers/sequences/deque/deque.cons/deduct.pass.cpp
index 096db12590d42..c481ac59ab411 100644
--- a/libcxx/test/std/containers/sequences/deque/deque.cons/deduct.pass.cpp
+++ b/libcxx/test/std/containers/sequences/deque/deque.cons/deduct.pass.cpp
@@ -32,21 +32,19 @@
 
 struct A {};
 
-int main(int, char**)
-{
-
-//  Test the explicit deduction guides
-    {
-    const int arr[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9 };
+int main(int, char**) {
+  //  Test the explicit deduction guides
+  {
+    const int arr[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9};
     std::deque deq(std::begin(arr), std::end(arr));
 
     static_assert(std::is_same_v<decltype(deq), std::deque<int>>, "");
     assert(std::equal(deq.begin(), deq.end(), std::begin(arr), std::end(arr)));
     LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(deq));
-    }
+  }
 
-    {
-    const long arr[] = {INT_MAX, 1L, 2L, 3L };
+  {
+    const long arr[] = {INT_MAX, 1L, 2L, 3L};
     std::deque deq(std::begin(arr), std::end(arr), std::allocator<long>());
     static_assert(std::is_same_v<decltype(deq)::value_type, long>, "");
     assert(deq.size() == 4);
@@ -54,110 +52,110 @@ int main(int, char**)
     assert(deq[1] == 1L);
     assert(deq[2] == 2L);
     LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(deq));
-    }
+  }
 
-//  Test the implicit deduction guides
+  //  Test the implicit deduction guides
 
-    {
-//  We don't expect this one to work.
-//  std::deque deq(std::allocator<int>()); // deque (allocator &)
-    }
+  {
+    //  We don't expect this one to work.
+    //  std::deque deq(std::allocator<int>()); // deque (allocator &)
+  }
 
-    {
+  {
     std::deque deq(1, A{}); // deque (size_type, T)
     static_assert(std::is_same_v<decltype(deq)::value_type, A>, "");
     static_assert(std::is_same_v<decltype(deq)::allocator_type, std::allocator<A>>, "");
     assert(deq.size() == 1);
     LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(deq));
-    }
+  }
 
-    {
+  {
     std::deque deq(1, A{}, test_allocator<A>()); // deque (size_type, T, allocator)
     static_assert(std::is_same_v<decltype(deq)::value_type, A>, "");
     static_assert(std::is_same_v<decltype(deq)::allocator_type, test_allocator<A>>, "");
     assert(deq.size() == 1);
     LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(deq));
-    }
+  }
 
-    {
+  {
     std::deque deq{1U, 2U, 3U, 4U, 5U}; // deque(initializer-list)
     static_assert(std::is_same_v<decltype(deq)::value_type, unsigned>, "");
     assert(deq.size() == 5);
     assert(deq[2] == 3U);
     LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(deq));
-    }
+  }
 
-    {
+  {
     std::deque deq({1.0, 2.0, 3.0, 4.0}, test_allocator<double>()); // deque(initializer-list, allocator)
     static_assert(std::is_same_v<decltype(deq)::value_type, double>, "");
     static_assert(std::is_same_v<decltype(deq)::allocator_type, test_allocator<double>>, "");
     assert(deq.size() == 4);
     assert(deq[3] == 4.0);
     LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(deq));
-    }
+  }
 
-    {
+  {
     std::deque<long double> source;
     std::deque deq(source); // deque(deque &)
     static_assert(std::is_same_v<decltype(deq)::value_type, long double>, "");
     static_assert(std::is_same_v<decltype(deq)::allocator_type, std::allocator<long double>>, "");
     assert(deq.size() == 0);
     LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(deq));
+  }
+
+  {
+    typedef test_allocator<short> Alloc;
+    typedef test_allocator<int> ConvertibleToAlloc;
+
+    {
+      std::deque<short, Alloc> source;
+      std::deque deq(source, Alloc(2));
+      static_assert(std::is_same_v<decltype(deq), decltype(source)>);
+      LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(deq));
+      LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(source));
+    }
+
+    {
+      std::deque<short, Alloc> source;
+      std::deque deq(source, ConvertibleToAlloc(2));
+      static_assert(std::is_same_v<decltype(deq), decltype(source)>);
+      LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(deq));
+      LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(source));
+    }
+
+    {
+      std::deque<short, Alloc> source;
+      std::deque deq(std::move(source), Alloc(2));
+      static_assert(std::is_same_v<decltype(deq), decltype(source)>);
+      LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(deq));
+      LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(source));
     }
 
     {
-        typedef test_allocator<short> Alloc;
-        typedef test_allocator<int> ConvertibleToAlloc;
-
-        {
-        std::deque<short, Alloc> source;
-        std::deque deq(source, Alloc(2));
-        static_assert(std::is_same_v<decltype(deq), decltype(source)>);
-        LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(deq));
-        LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(source));
-        }
-
-        {
-        std::deque<short, Alloc> source;
-        std::deque deq(source, ConvertibleToAlloc(2));
-        static_assert(std::is_same_v<decltype(deq), decltype(source)>);
-        LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(deq));
-        LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(source));
-        }
-
-        {
-        std::deque<short, Alloc> source;
-        std::deque deq(std::move(source), Alloc(2));
-        static_assert(std::is_same_v<decltype(deq), decltype(source)>);
-        LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(deq));
-        LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(source));
-        }
-
-        {
-        std::deque<short, Alloc> source;
-        std::deque deq(std::move(source), ConvertibleToAlloc(2));
-        static_assert(std::is_same_v<decltype(deq), decltype(source)>);
-        LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(deq));
-        LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(source));
-        }
+      std::deque<short, Alloc> source;
+      std::deque deq(std::move(source), ConvertibleToAlloc(2));
+      static_assert(std::is_same_v<decltype(deq), decltype(source)>);
+      LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(deq));
+      LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(source));
     }
+  }
 
 #if TEST_STD_VER >= 23
+  {
+    {
+      std::deque c(std::from_range, std::array<int, 0>());
+      static_assert(std::is_same_v<decltype(c), std::deque<int>>);
+    }
+
     {
-      {
-        std::deque c(std::from_range, std::array<int, 0>());
-        static_assert(std::is_same_v<decltype(c), std::deque<int>>);
-      }
-
-      {
-        using Alloc = test_allocator<int>;
-        std::deque c(std::from_range, std::array<int, 0>(), Alloc());
-        static_assert(std::is_same_v<decltype(c), std::deque<int, Alloc>>);
-      }
+      using Alloc = test_allocator<int>;
+      std::deque c(std::from_range, std::array<int, 0>(), Alloc());
+      static_assert(std::is_same_v<decltype(c), std::deque<int, Alloc>>);
     }
+  }
 #endif
 
-    SequenceContainerDeductionGuidesSfinaeAway<std::deque, std::deque<int>>();
+  SequenceContainerDeductionGuidesSfinaeAway<std::deque, std::deque<int>>();
 
-    return 0;
+  return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/deque/deque.cons/deduct.verify.cpp b/libcxx/test/std/containers/sequences/deque/deque.cons/deduct.verify.cpp
index f65e230112d90..61e58b08ca263 100644
--- a/libcxx/test/std/containers/sequences/deque/deque.cons/deduct.verify.cpp
+++ b/libcxx/test/std/containers/sequences/deque/deque.cons/deduct.verify.cpp
@@ -22,19 +22,18 @@
 
 struct A {};
 
-int main(int, char**)
-{
-//  Test the explicit deduction guides
-
-//  Test the implicit deduction guides
-    {
-//  deque (allocator &)
-    std::deque deq((std::allocator<int>()));  // expected-error-re {{no viable constructor or deduction guide for deduction of template arguments of '{{(std::)?}}deque'}}
-//  Note: The extra parens are necessary, since otherwise clang decides it is a function declaration.
-//  Also, we can't use {} instead of parens, because that constructs a
-//      deque<allocator<int>, allocator<allocator<int>>>
-    }
-
+int main(int, char**) {
+  //  Test the explicit deduction guides
+
+  //  Test the implicit deduction guides
+  {
+    //  deque (allocator &)
+    std::deque deq((std::allocator< int>()));
+    // expected-error-re@-1 {{no viable constructor or deduction guide for deduction of template arguments of '{{(std::)?}}deque'}}
+    //  Note: The extra parens are necessary, since otherwise clang decides it is a function declaration.
+    //  Also, we can't use {} instead of parens, because that constructs a
+    //      deque<allocator<int>, allocator<allocator<int>>>
+  }
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/deque/deque.cons/default.pass.cpp b/libcxx/test/std/containers/sequences/deque/deque.cons/default.pass.cpp
index 493274e932586..6bfd4857f9b0a 100644
--- a/libcxx/test/std/containers/sequences/deque/deque.cons/default.pass.cpp
+++ b/libcxx/test/std/containers/sequences/deque/deque.cons/default.pass.cpp
@@ -20,26 +20,23 @@
 #include "min_allocator.h"
 
 template <class T, class Allocator>
-void
-test()
-{
-    std::deque<T, Allocator> d;
-    assert(d.size() == 0);
-    LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(d));
+void test() {
+  std::deque<T, Allocator> d;
+  assert(d.size() == 0);
+  LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(d));
 #if TEST_STD_VER >= 11
-    std::deque<T, Allocator> d1 = {};
-    assert(d1.size() == 0);
-    LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(d1));
+  std::deque<T, Allocator> d1 = {};
+  assert(d1.size() == 0);
+  LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(d1));
 #endif
 }
 
-int main(int, char**)
-{
-    test<int, std::allocator<int> >();
-    test<NotConstructible, limited_allocator<NotConstructible, 1> >();
+int main(int, char**) {
+  test<int, std::allocator<int> >();
+  test<NotConstructible, limited_allocator<NotConstructible, 1> >();
 #if TEST_STD_VER >= 11
-    test<int, min_allocator<int> >();
-    test<NotConstructible, min_allocator<NotConstructible> >();
+  test<int, min_allocator<int> >();
+  test<NotConstructible, min_allocator<NotConstructible> >();
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/sequences/deque/deque.cons/default_noexcept.pass.cpp b/libcxx/test/std/containers/sequences/deque/deque.cons/default_noexcept.pass.cpp
index 88672fa044fe2..244fef829f521 100644
--- a/libcxx/test/std/containers/sequences/deque/deque.cons/default_noexcept.pass.cpp
+++ b/libcxx/test/std/containers/sequences/deque/deque.cons/default_noexcept.pass.cpp
@@ -23,33 +23,31 @@
 #include "test_allocator.h"
 
 template <class T>
-struct some_alloc
-{
-    typedef T value_type;
-    some_alloc(const some_alloc&);
-    void allocate(std::size_t);
+struct some_alloc {
+  typedef T value_type;
+  some_alloc(const some_alloc&);
+  void allocate(std::size_t);
 };
 
-int main(int, char**)
-{
+int main(int, char**) {
 #if defined(_LIBCPP_VERSION)
-    {
-        typedef std::deque<MoveOnly> C;
-        static_assert(std::is_nothrow_default_constructible<C>::value, "");
-    }
-    {
-        typedef std::deque<MoveOnly, test_allocator<MoveOnly>> C;
-        static_assert(std::is_nothrow_default_constructible<C>::value, "");
-    }
+  {
+    typedef std::deque<MoveOnly> C;
+    static_assert(std::is_nothrow_default_constructible<C>::value, "");
+  }
+  {
+    typedef std::deque<MoveOnly, test_allocator<MoveOnly>> C;
+    static_assert(std::is_nothrow_default_constructible<C>::value, "");
+  }
 #endif // _LIBCPP_VERSION
-    {
-        typedef std::deque<MoveOnly, other_allocator<MoveOnly>> C;
-        static_assert(!std::is_nothrow_default_constructible<C>::value, "");
-    }
-    {
-        typedef std::deque<MoveOnly, some_alloc<MoveOnly>> C;
-        static_assert(!std::is_nothrow_default_constructible<C>::value, "");
-    }
+  {
+    typedef std::deque<MoveOnly, other_allocator<MoveOnly>> C;
+    static_assert(!std::is_nothrow_default_constructible<C>::value, "");
+  }
+  {
+    typedef std::deque<MoveOnly, some_alloc<MoveOnly>> C;
+    static_assert(!std::is_nothrow_default_constructible<C>::value, "");
+  }
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/deque/deque.cons/dtor_noexcept.pass.cpp b/libcxx/test/std/containers/sequences/deque/deque.cons/dtor_noexcept.pass.cpp
index 88fce25a1f7e3..f0a839484f9dc 100644
--- a/libcxx/test/std/containers/sequences/deque/deque.cons/dtor_noexcept.pass.cpp
+++ b/libcxx/test/std/containers/sequences/deque/deque.cons/dtor_noexcept.pass.cpp
@@ -20,33 +20,31 @@
 #include "test_allocator.h"
 
 template <class T>
-struct some_alloc
-{
-    typedef T value_type;
-    some_alloc(const some_alloc&);
-    ~some_alloc() noexcept(false);
-    void allocate(std::size_t);
+struct some_alloc {
+  typedef T value_type;
+  some_alloc(const some_alloc&);
+  ~some_alloc() noexcept(false);
+  void allocate(std::size_t);
 };
 
-int main(int, char**)
-{
-    {
-        typedef std::deque<MoveOnly> C;
-        static_assert(std::is_nothrow_destructible<C>::value, "");
-    }
-    {
-        typedef std::deque<MoveOnly, test_allocator<MoveOnly>> C;
-        static_assert(std::is_nothrow_destructible<C>::value, "");
-    }
-    {
-        typedef std::deque<MoveOnly, other_allocator<MoveOnly>> C;
-        static_assert(std::is_nothrow_destructible<C>::value, "");
-    }
+int main(int, char**) {
+  {
+    typedef std::deque<MoveOnly> C;
+    static_assert(std::is_nothrow_destructible<C>::value, "");
+  }
+  {
+    typedef std::deque<MoveOnly, test_allocator<MoveOnly>> C;
+    static_assert(std::is_nothrow_destructible<C>::value, "");
+  }
+  {
+    typedef std::deque<MoveOnly, other_allocator<MoveOnly>> C;
+    static_assert(std::is_nothrow_destructible<C>::value, "");
+  }
 #if defined(_LIBCPP_VERSION)
-    {
-        typedef std::deque<MoveOnly, some_alloc<MoveOnly>> C;
-        static_assert(!std::is_nothrow_destructible<C>::value, "");
-    }
+  {
+    typedef std::deque<MoveOnly, some_alloc<MoveOnly>> C;
+    static_assert(!std::is_nothrow_destructible<C>::value, "");
+  }
 #endif // _LIBCPP_VERSION
 
   return 0;
diff --git a/libcxx/test/std/containers/sequences/deque/deque.cons/initializer_list.pass.cpp b/libcxx/test/std/containers/sequences/deque/deque.cons/initializer_list.pass.cpp
index bf7bc4bd90acc..d7df936f9413d 100644
--- a/libcxx/test/std/containers/sequences/deque/deque.cons/initializer_list.pass.cpp
+++ b/libcxx/test/std/containers/sequences/deque/deque.cons/initializer_list.pass.cpp
@@ -19,9 +19,8 @@
 #include "test_macros.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
+int main(int, char**) {
+  {
     std::deque<int> d = {3, 4, 5, 6};
     assert(d.size() == 4);
     assert(d[0] == 3);
@@ -29,8 +28,8 @@ int main(int, char**)
     assert(d[2] == 5);
     assert(d[3] == 6);
     LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(d));
-    }
-    {
+  }
+  {
     std::deque<int, min_allocator<int>> d = {3, 4, 5, 6};
     assert(d.size() == 4);
     assert(d[0] == 3);
@@ -38,7 +37,7 @@ int main(int, char**)
     assert(d[2] == 5);
     assert(d[3] == 6);
     LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(d));
-    }
+  }
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/deque/deque.cons/initializer_list_alloc.pass.cpp b/libcxx/test/std/containers/sequences/deque/deque.cons/initializer_list_alloc.pass.cpp
index 99e6195151703..f5f1a23243002 100644
--- a/libcxx/test/std/containers/sequences/deque/deque.cons/initializer_list_alloc.pass.cpp
+++ b/libcxx/test/std/containers/sequences/deque/deque.cons/initializer_list_alloc.pass.cpp
@@ -20,9 +20,8 @@
 #include "test_allocator.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
+int main(int, char**) {
+  {
     std::deque<int, test_allocator<int>> d({3, 4, 5, 6}, test_allocator<int>(3));
     assert(d.get_allocator() == test_allocator<int>(3));
     assert(d.size() == 4);
@@ -31,8 +30,8 @@ int main(int, char**)
     assert(d[2] == 5);
     assert(d[3] == 6);
     LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(d));
-    }
-    {
+  }
+  {
     std::deque<int, min_allocator<int>> d({3, 4, 5, 6}, min_allocator<int>());
     assert(d.get_allocator() == min_allocator<int>());
     assert(d.size() == 4);
@@ -41,7 +40,7 @@ int main(int, char**)
     assert(d[2] == 5);
     assert(d[3] == 6);
     LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(d));
-    }
+  }
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/deque/deque.cons/iter_iter.pass.cpp b/libcxx/test/std/containers/sequences/deque/deque.cons/iter_iter.pass.cpp
index c0df278f00140..1f8a044d0b602 100644
--- a/libcxx/test/std/containers/sequences/deque/deque.cons/iter_iter.pass.cpp
+++ b/libcxx/test/std/containers/sequences/deque/deque.cons/iter_iter.pass.cpp
@@ -20,61 +20,55 @@
 #include "test_iterators.h"
 #include "min_allocator.h"
 #if TEST_STD_VER >= 11
-#include "emplace_constructible.h"
+#  include "emplace_constructible.h"
 #endif
 
 template <class InputIterator>
-void
-test(InputIterator f, InputIterator l)
-{
-    typedef typename std::iterator_traits<InputIterator>::value_type T;
-    typedef std::allocator<T> Allocator;
-    typedef std::deque<T, Allocator> C;
-    typedef typename C::const_iterator const_iterator;
-    C d(f, l);
-    assert(d.size() == static_cast<std::size_t>(std::distance(f, l)));
-    assert(static_cast<std::size_t>(std::distance(d.begin(), d.end())) == d.size());
-    LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(d));
-    for (const_iterator i = d.begin(), e = d.end(); i != e; ++i, ++f)
-        assert(*i == *f);
+void test(InputIterator f, InputIterator l) {
+  typedef typename std::iterator_traits<InputIterator>::value_type T;
+  typedef std::allocator<T> Allocator;
+  typedef std::deque<T, Allocator> C;
+  typedef typename C::const_iterator const_iterator;
+  C d(f, l);
+  assert(d.size() == static_cast<std::size_t>(std::distance(f, l)));
+  assert(static_cast<std::size_t>(std::distance(d.begin(), d.end())) == d.size());
+  LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(d));
+  for (const_iterator i = d.begin(), e = d.end(); i != e; ++i, ++f)
+    assert(*i == *f);
 }
 
 template <class Allocator, class InputIterator>
-void
-test(InputIterator f, InputIterator l)
-{
-    typedef typename std::iterator_traits<InputIterator>::value_type T;
-    typedef std::deque<T, Allocator> C;
-    typedef typename C::const_iterator const_iterator;
-    C d(f, l);
-    assert(d.size() == static_cast<std::size_t>(std::distance(f, l)));
-    assert(static_cast<std::size_t>(std::distance(d.begin(), d.end())) == d.size());
-    LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(d));
-    for (const_iterator i = d.begin(), e = d.end(); i != e; ++i, ++f)
-        assert(*i == *f);
+void test(InputIterator f, InputIterator l) {
+  typedef typename std::iterator_traits<InputIterator>::value_type T;
+  typedef std::deque<T, Allocator> C;
+  typedef typename C::const_iterator const_iterator;
+  C d(f, l);
+  assert(d.size() == static_cast<std::size_t>(std::distance(f, l)));
+  assert(static_cast<std::size_t>(std::distance(d.begin(), d.end())) == d.size());
+  LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(d));
+  for (const_iterator i = d.begin(), e = d.end(); i != e; ++i, ++f)
+    assert(*i == *f);
 }
 
-void basic_test()
-{
-    int ab[] = {3, 4, 2, 8, 0, 1, 44, 34, 45, 96, 80, 1, 13, 31, 45};
-    int* an = ab + sizeof(ab)/sizeof(ab[0]);
-    test(cpp17_input_iterator<const int*>(ab), cpp17_input_iterator<const int*>(an));
-    test(forward_iterator<const int*>(ab), forward_iterator<const int*>(an));
-    test(bidirectional_iterator<const int*>(ab), bidirectional_iterator<const int*>(an));
-    test(random_access_iterator<const int*>(ab), random_access_iterator<const int*>(an));
-    test<limited_allocator<int, 4096> >(ab, an);
+void basic_test() {
+  int ab[] = {3, 4, 2, 8, 0, 1, 44, 34, 45, 96, 80, 1, 13, 31, 45};
+  int* an  = ab + sizeof(ab) / sizeof(ab[0]);
+  test(cpp17_input_iterator<const int*>(ab), cpp17_input_iterator<const int*>(an));
+  test(forward_iterator<const int*>(ab), forward_iterator<const int*>(an));
+  test(bidirectional_iterator<const int*>(ab), bidirectional_iterator<const int*>(an));
+  test(random_access_iterator<const int*>(ab), random_access_iterator<const int*>(an));
+  test<limited_allocator<int, 4096> >(ab, an);
 #if TEST_STD_VER >= 11
-    test<min_allocator<int> >(ab, an);
+  test<min_allocator<int> >(ab, an);
 #endif
 }
 
-
 void test_emplacable_concept() {
 #if TEST_STD_VER >= 11
   int arr1[] = {42};
   int arr2[] = {1, 101, 42};
   {
-    using T = EmplaceConstructibleAndMoveable<int>;
+    using T  = EmplaceConstructibleAndMoveable<int>;
     using It = random_access_iterator<int*>;
     {
       std::deque<T> v(It(arr1), It(std::end(arr1)));
@@ -88,7 +82,7 @@ void test_emplacable_concept() {
     }
   }
   {
-    using T = EmplaceConstructibleAndMoveable<int>;
+    using T  = EmplaceConstructibleAndMoveable<int>;
     using It = cpp17_input_iterator<int*>;
     {
       std::deque<T> v(It(arr1), It(std::end(arr1)));
diff --git a/libcxx/test/std/containers/sequences/deque/deque.cons/iter_iter_alloc.pass.cpp b/libcxx/test/std/containers/sequences/deque/deque.cons/iter_iter_alloc.pass.cpp
index 40cf5cabc735c..61318c3d0f2d3 100644
--- a/libcxx/test/std/containers/sequences/deque/deque.cons/iter_iter_alloc.pass.cpp
+++ b/libcxx/test/std/containers/sequences/deque/deque.cons/iter_iter_alloc.pass.cpp
@@ -21,48 +21,44 @@
 #include "test_allocator.h"
 #include "min_allocator.h"
 #if TEST_STD_VER >= 11
-#include "emplace_constructible.h"
+#  include "emplace_constructible.h"
 #endif
 
 template <class InputIterator, class Allocator>
-void
-test(InputIterator f, InputIterator l, const Allocator& a)
-{
-    typedef typename std::iterator_traits<InputIterator>::value_type T;
-    typedef std::deque<T, Allocator> C;
-    typedef typename C::const_iterator const_iterator;
-    C d(f, l, a);
-    assert(d.get_allocator() == a);
-    assert(d.size() == static_cast<std::size_t>(std::distance(f, l)));
-    assert(static_cast<std::size_t>(std::distance(d.begin(), d.end())) == d.size());
-    LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(d));
-    for (const_iterator i = d.begin(), e = d.end(); i != e; ++i, ++f)
-        assert(*i == *f);
+void test(InputIterator f, InputIterator l, const Allocator& a) {
+  typedef typename std::iterator_traits<InputIterator>::value_type T;
+  typedef std::deque<T, Allocator> C;
+  typedef typename C::const_iterator const_iterator;
+  C d(f, l, a);
+  assert(d.get_allocator() == a);
+  assert(d.size() == static_cast<std::size_t>(std::distance(f, l)));
+  assert(static_cast<std::size_t>(std::distance(d.begin(), d.end())) == d.size());
+  LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(d));
+  for (const_iterator i = d.begin(), e = d.end(); i != e; ++i, ++f)
+    assert(*i == *f);
 }
 
-void basic_test()
-{
-    int ab[] = {3, 4, 2, 8, 0, 1, 44, 34, 45, 96, 80, 1, 13, 31, 45};
-    int* an = ab + sizeof(ab)/sizeof(ab[0]);
-    test(cpp17_input_iterator<const int*>(ab), cpp17_input_iterator<const int*>(an), test_allocator<int>(3));
-    test(forward_iterator<const int*>(ab), forward_iterator<const int*>(an), test_allocator<int>(4));
-    test(bidirectional_iterator<const int*>(ab), bidirectional_iterator<const int*>(an), test_allocator<int>(5));
-    test(random_access_iterator<const int*>(ab), random_access_iterator<const int*>(an), test_allocator<int>(6));
+void basic_test() {
+  int ab[] = {3, 4, 2, 8, 0, 1, 44, 34, 45, 96, 80, 1, 13, 31, 45};
+  int* an  = ab + sizeof(ab) / sizeof(ab[0]);
+  test(cpp17_input_iterator<const int*>(ab), cpp17_input_iterator<const int*>(an), test_allocator<int>(3));
+  test(forward_iterator<const int*>(ab), forward_iterator<const int*>(an), test_allocator<int>(4));
+  test(bidirectional_iterator<const int*>(ab), bidirectional_iterator<const int*>(an), test_allocator<int>(5));
+  test(random_access_iterator<const int*>(ab), random_access_iterator<const int*>(an), test_allocator<int>(6));
 #if TEST_STD_VER >= 11
-    test(cpp17_input_iterator<const int*>(ab), cpp17_input_iterator<const int*>(an), min_allocator<int>());
-    test(forward_iterator<const int*>(ab), forward_iterator<const int*>(an), min_allocator<int>());
-    test(bidirectional_iterator<const int*>(ab), bidirectional_iterator<const int*>(an), min_allocator<int>());
-    test(random_access_iterator<const int*>(ab), random_access_iterator<const int*>(an), min_allocator<int>());
+  test(cpp17_input_iterator<const int*>(ab), cpp17_input_iterator<const int*>(an), min_allocator<int>());
+  test(forward_iterator<const int*>(ab), forward_iterator<const int*>(an), min_allocator<int>());
+  test(bidirectional_iterator<const int*>(ab), bidirectional_iterator<const int*>(an), min_allocator<int>());
+  test(random_access_iterator<const int*>(ab), random_access_iterator<const int*>(an), min_allocator<int>());
 #endif
 }
 
-
 void test_emplacable_concept() {
 #if TEST_STD_VER >= 11
   int arr1[] = {42};
   int arr2[] = {1, 101, 42};
   {
-    using T = EmplaceConstructibleAndMoveable<int>;
+    using T  = EmplaceConstructibleAndMoveable<int>;
     using It = random_access_iterator<int*>;
     std::allocator<T> a;
     {
@@ -79,7 +75,7 @@ void test_emplacable_concept() {
     }
   }
   {
-    using T = EmplaceConstructibleAndMoveable<int>;
+    using T  = EmplaceConstructibleAndMoveable<int>;
     using It = cpp17_input_iterator<int*>;
     std::allocator<T> a;
     {
diff --git a/libcxx/test/std/containers/sequences/deque/deque.cons/move.pass.cpp b/libcxx/test/std/containers/sequences/deque/deque.cons/move.pass.cpp
index b3f08da81ff5b..66ff168cc83b7 100644
--- a/libcxx/test/std/containers/sequences/deque/deque.cons/move.pass.cpp
+++ b/libcxx/test/std/containers/sequences/deque/deque.cons/move.pass.cpp
@@ -21,65 +21,64 @@
 #include "test_allocator.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        int ab[] = {3, 4, 2, 8, 0, 1, 44, 34, 45, 96, 80, 1, 13, 31, 45};
-        int* an = ab + sizeof(ab)/sizeof(ab[0]);
-        typedef test_allocator<MoveOnly> A;
-        std::deque<MoveOnly, A> c1(A(1));
-        for (int* p = ab; p < an; ++p)
-            c1.push_back(MoveOnly(*p));
-        std::deque<MoveOnly, A> c2(A(2));
-        for (int* p = ab; p < an; ++p)
-            c2.push_back(MoveOnly(*p));
-        A old_a = c1.get_allocator();
-        std::deque<MoveOnly, A> c3 = std::move(c1);
-        assert(c2 == c3);
-        assert(c1.size() == 0);
-        assert(c3.get_allocator() == old_a);
-        assert(c1.get_allocator() == A(1));
-        assert(c1.get_allocator().get_id() == test_alloc_base::moved_value);
-        LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c1));
-        LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c2));
-        LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c3));
-    }
-    {
-        int ab[] = {3, 4, 2, 8, 0, 1, 44, 34, 45, 96, 80, 1, 13, 31, 45};
-        int* an = ab + sizeof(ab)/sizeof(ab[0]);
-        typedef other_allocator<MoveOnly> A;
-        std::deque<MoveOnly, A> c1(A(1));
-        for (int* p = ab; p < an; ++p)
-            c1.push_back(MoveOnly(*p));
-        std::deque<MoveOnly, A> c2(A(2));
-        for (int* p = ab; p < an; ++p)
-            c2.push_back(MoveOnly(*p));
-        std::deque<MoveOnly, A> c3 = std::move(c1);
-        assert(c2 == c3);
-        assert(c1.size() == 0);
-        assert(c3.get_allocator() == c1.get_allocator());
-        LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c1));
-        LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c2));
-        LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c3));
-    }
-    {
-        int ab[] = {3, 4, 2, 8, 0, 1, 44, 34, 45, 96, 80, 1, 13, 31, 45};
-        int* an = ab + sizeof(ab)/sizeof(ab[0]);
-        typedef min_allocator<MoveOnly> A;
-        std::deque<MoveOnly, A> c1(A{});
-        for (int* p = ab; p < an; ++p)
-            c1.push_back(MoveOnly(*p));
-        std::deque<MoveOnly, A> c2(A{});
-        for (int* p = ab; p < an; ++p)
-            c2.push_back(MoveOnly(*p));
-        std::deque<MoveOnly, A> c3 = std::move(c1);
-        assert(c2 == c3);
-        assert(c1.size() == 0);
-        assert(c3.get_allocator() == c1.get_allocator());
-        LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c1));
-        LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c2));
-        LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c3));
-    }
+int main(int, char**) {
+  {
+    int ab[] = {3, 4, 2, 8, 0, 1, 44, 34, 45, 96, 80, 1, 13, 31, 45};
+    int* an  = ab + sizeof(ab) / sizeof(ab[0]);
+    typedef test_allocator<MoveOnly> A;
+    std::deque<MoveOnly, A> c1(A(1));
+    for (int* p = ab; p < an; ++p)
+      c1.push_back(MoveOnly(*p));
+    std::deque<MoveOnly, A> c2(A(2));
+    for (int* p = ab; p < an; ++p)
+      c2.push_back(MoveOnly(*p));
+    A old_a                    = c1.get_allocator();
+    std::deque<MoveOnly, A> c3 = std::move(c1);
+    assert(c2 == c3);
+    assert(c1.size() == 0);
+    assert(c3.get_allocator() == old_a);
+    assert(c1.get_allocator() == A(1));
+    assert(c1.get_allocator().get_id() == test_alloc_base::moved_value);
+    LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c1));
+    LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c2));
+    LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c3));
+  }
+  {
+    int ab[] = {3, 4, 2, 8, 0, 1, 44, 34, 45, 96, 80, 1, 13, 31, 45};
+    int* an  = ab + sizeof(ab) / sizeof(ab[0]);
+    typedef other_allocator<MoveOnly> A;
+    std::deque<MoveOnly, A> c1(A(1));
+    for (int* p = ab; p < an; ++p)
+      c1.push_back(MoveOnly(*p));
+    std::deque<MoveOnly, A> c2(A(2));
+    for (int* p = ab; p < an; ++p)
+      c2.push_back(MoveOnly(*p));
+    std::deque<MoveOnly, A> c3 = std::move(c1);
+    assert(c2 == c3);
+    assert(c1.size() == 0);
+    assert(c3.get_allocator() == c1.get_allocator());
+    LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c1));
+    LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c2));
+    LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c3));
+  }
+  {
+    int ab[] = {3, 4, 2, 8, 0, 1, 44, 34, 45, 96, 80, 1, 13, 31, 45};
+    int* an  = ab + sizeof(ab) / sizeof(ab[0]);
+    typedef min_allocator<MoveOnly> A;
+    std::deque<MoveOnly, A> c1(A{});
+    for (int* p = ab; p < an; ++p)
+      c1.push_back(MoveOnly(*p));
+    std::deque<MoveOnly, A> c2(A{});
+    for (int* p = ab; p < an; ++p)
+      c2.push_back(MoveOnly(*p));
+    std::deque<MoveOnly, A> c3 = std::move(c1);
+    assert(c2 == c3);
+    assert(c1.size() == 0);
+    assert(c3.get_allocator() == c1.get_allocator());
+    LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c1));
+    LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c2));
+    LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c3));
+  }
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/deque/deque.cons/move_alloc.pass.cpp b/libcxx/test/std/containers/sequences/deque/deque.cons/move_alloc.pass.cpp
index 25a838f55e441..985d8ad9db67f 100644
--- a/libcxx/test/std/containers/sequences/deque/deque.cons/move_alloc.pass.cpp
+++ b/libcxx/test/std/containers/sequences/deque/deque.cons/move_alloc.pass.cpp
@@ -21,80 +21,79 @@
 #include "test_allocator.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        int ab[] = {3, 4, 2, 8, 0, 1, 44, 34, 45, 96, 80, 1, 13, 31, 45};
-        const int* an = ab + sizeof(ab)/sizeof(ab[0]);
-        typedef test_allocator<MoveOnly> A;
-        std::deque<MoveOnly, A> c1(A(1));
-        for (int* p = ab; p < an; ++p)
-            c1.push_back(MoveOnly(*p));
-        std::deque<MoveOnly, A> c2(A(1));
-        for (int* p = ab; p < an; ++p)
-            c2.push_back(MoveOnly(*p));
-        std::deque<MoveOnly, A> c3(std::move(c1), A(3)); // unequal allocator
-        assert(c2 == c3);
-        assert(c3.get_allocator() == A(3));
-        LIBCPP_ASSERT(c1.size() != 0);
-        LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c1));
-        LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c2));
-        LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c3));
-    }
-    {
-        int ab[] = {3, 4, 2, 8, 0, 1, 44, 34, 45, 96, 80, 1, 13, 31, 45};
-        const int* an = ab + sizeof(ab)/sizeof(ab[0]);
-        typedef test_allocator<MoveOnly> A;
-        std::deque<MoveOnly, A> c1(A(1));
-        for (int* p = ab; p < an; ++p)
-            c1.push_back(MoveOnly(*p));
-        std::deque<MoveOnly, A> c2(A(1));
-        for (int* p = ab; p < an; ++p)
-            c2.push_back(MoveOnly(*p));
-        std::deque<MoveOnly, A> c3(std::move(c1), A(1)); // equal allocator
-        assert(c2 == c3);
-        assert(c3.get_allocator() == A(1));
-        LIBCPP_ASSERT(c1.size() == 0);
-        LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c1));
-        LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c2));
-        LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c3));
-    }
-    {
-        int ab[] = {3, 4, 2, 8, 0, 1, 44, 34, 45, 96, 80, 1, 13, 31, 45};
-        const int* an = ab + sizeof(ab)/sizeof(ab[0]);
-        typedef other_allocator<MoveOnly> A;
-        std::deque<MoveOnly, A> c1(A(1));
-        for (int* p = ab; p < an; ++p)
-            c1.push_back(MoveOnly(*p));
-        std::deque<MoveOnly, A> c2(A(1));
-        for (int* p = ab; p < an; ++p)
-            c2.push_back(MoveOnly(*p));
-        std::deque<MoveOnly, A> c3(std::move(c1), A(3)); // unequal allocator
-        assert(c2 == c3);
-        assert(c3.get_allocator() == A(3));
-        LIBCPP_ASSERT(c1.size() != 0);
-        LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c1));
-        LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c2));
-        LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c3));
-    }
-    {
-        int ab[] = {3, 4, 2, 8, 0, 1, 44, 34, 45, 96, 80, 1, 13, 31, 45};
-        const int* an = ab + sizeof(ab)/sizeof(ab[0]);
-        typedef min_allocator<MoveOnly> A;
-        std::deque<MoveOnly, A> c1(A{});
-        for (int* p = ab; p < an; ++p)
-            c1.push_back(MoveOnly(*p));
-        std::deque<MoveOnly, A> c2(A{});
-        for (int* p = ab; p < an; ++p)
-            c2.push_back(MoveOnly(*p));
-        std::deque<MoveOnly, A> c3(std::move(c1), A());  // equal allocator
-        assert(c2 == c3);
-        assert(c3.get_allocator() == A());
-        LIBCPP_ASSERT(c1.size() == 0);
-        LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c1));
-        LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c2));
-        LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c3));
-    }
+int main(int, char**) {
+  {
+    int ab[]      = {3, 4, 2, 8, 0, 1, 44, 34, 45, 96, 80, 1, 13, 31, 45};
+    const int* an = ab + sizeof(ab) / sizeof(ab[0]);
+    typedef test_allocator<MoveOnly> A;
+    std::deque<MoveOnly, A> c1(A(1));
+    for (int* p = ab; p < an; ++p)
+      c1.push_back(MoveOnly(*p));
+    std::deque<MoveOnly, A> c2(A(1));
+    for (int* p = ab; p < an; ++p)
+      c2.push_back(MoveOnly(*p));
+    std::deque<MoveOnly, A> c3(std::move(c1), A(3)); // unequal allocator
+    assert(c2 == c3);
+    assert(c3.get_allocator() == A(3));
+    LIBCPP_ASSERT(c1.size() != 0);
+    LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c1));
+    LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c2));
+    LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c3));
+  }
+  {
+    int ab[]      = {3, 4, 2, 8, 0, 1, 44, 34, 45, 96, 80, 1, 13, 31, 45};
+    const int* an = ab + sizeof(ab) / sizeof(ab[0]);
+    typedef test_allocator<MoveOnly> A;
+    std::deque<MoveOnly, A> c1(A(1));
+    for (int* p = ab; p < an; ++p)
+      c1.push_back(MoveOnly(*p));
+    std::deque<MoveOnly, A> c2(A(1));
+    for (int* p = ab; p < an; ++p)
+      c2.push_back(MoveOnly(*p));
+    std::deque<MoveOnly, A> c3(std::move(c1), A(1)); // equal allocator
+    assert(c2 == c3);
+    assert(c3.get_allocator() == A(1));
+    LIBCPP_ASSERT(c1.size() == 0);
+    LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c1));
+    LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c2));
+    LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c3));
+  }
+  {
+    int ab[]      = {3, 4, 2, 8, 0, 1, 44, 34, 45, 96, 80, 1, 13, 31, 45};
+    const int* an = ab + sizeof(ab) / sizeof(ab[0]);
+    typedef other_allocator<MoveOnly> A;
+    std::deque<MoveOnly, A> c1(A(1));
+    for (int* p = ab; p < an; ++p)
+      c1.push_back(MoveOnly(*p));
+    std::deque<MoveOnly, A> c2(A(1));
+    for (int* p = ab; p < an; ++p)
+      c2.push_back(MoveOnly(*p));
+    std::deque<MoveOnly, A> c3(std::move(c1), A(3)); // unequal allocator
+    assert(c2 == c3);
+    assert(c3.get_allocator() == A(3));
+    LIBCPP_ASSERT(c1.size() != 0);
+    LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c1));
+    LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c2));
+    LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c3));
+  }
+  {
+    int ab[]      = {3, 4, 2, 8, 0, 1, 44, 34, 45, 96, 80, 1, 13, 31, 45};
+    const int* an = ab + sizeof(ab) / sizeof(ab[0]);
+    typedef min_allocator<MoveOnly> A;
+    std::deque<MoveOnly, A> c1(A{});
+    for (int* p = ab; p < an; ++p)
+      c1.push_back(MoveOnly(*p));
+    std::deque<MoveOnly, A> c2(A{});
+    for (int* p = ab; p < an; ++p)
+      c2.push_back(MoveOnly(*p));
+    std::deque<MoveOnly, A> c3(std::move(c1), A()); // equal allocator
+    assert(c2 == c3);
+    assert(c3.get_allocator() == A());
+    LIBCPP_ASSERT(c1.size() == 0);
+    LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c1));
+    LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c2));
+    LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c3));
+  }
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/deque/deque.cons/move_assign.pass.cpp b/libcxx/test/std/containers/sequences/deque/deque.cons/move_assign.pass.cpp
index 7cf0b63e06b1e..426dea0347101 100644
--- a/libcxx/test/std/containers/sequences/deque/deque.cons/move_assign.pass.cpp
+++ b/libcxx/test/std/containers/sequences/deque/deque.cons/move_assign.pass.cpp
@@ -21,84 +21,83 @@
 #include "test_allocator.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        int ab[] = {3, 4, 2, 8, 0, 1, 44, 34, 45, 96, 80, 1, 13, 31, 45};
-        int* an = ab + sizeof(ab)/sizeof(ab[0]);
-        typedef test_allocator<MoveOnly> A;
-        std::deque<MoveOnly, A> c1(A(5));
-        for (int* p = ab; p < an; ++p)
-            c1.push_back(MoveOnly(*p));
-        std::deque<MoveOnly, A> c2(A(5));
-        for (int* p = ab; p < an; ++p)
-            c2.push_back(MoveOnly(*p));
-        std::deque<MoveOnly, A> c3(A(5));
-        c3 = std::move(c1);
-        assert(c2 == c3);
-        assert(c1.size() == 0);
-        assert(c3.get_allocator() == A(5));
-        LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c1));
-        LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c2));
-        LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c3));
-    }
-    {
-        int ab[] = {3, 4, 2, 8, 0, 1, 44, 34, 45, 96, 80, 1, 13, 31, 45};
-        int* an = ab + sizeof(ab)/sizeof(ab[0]);
-        typedef test_allocator<MoveOnly> A;
-        std::deque<MoveOnly, A> c1(A(5));
-        for (int* p = ab; p < an; ++p)
-            c1.push_back(MoveOnly(*p));
-        std::deque<MoveOnly, A> c2(A(5));
-        for (int* p = ab; p < an; ++p)
-            c2.push_back(MoveOnly(*p));
-        std::deque<MoveOnly, A> c3(A(6));
-        c3 = std::move(c1);
-        assert(c2 == c3);
-        assert(c1.size() != 0);
-        assert(c3.get_allocator() == A(6));
-        LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c1));
-        LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c2));
-        LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c3));
-    }
-    {
-        int ab[] = {3, 4, 2, 8, 0, 1, 44, 34, 45, 96, 80, 1, 13, 31, 45};
-        int* an = ab + sizeof(ab)/sizeof(ab[0]);
-        typedef other_allocator<MoveOnly> A;
-        std::deque<MoveOnly, A> c1(A(5));
-        for (int* p = ab; p < an; ++p)
-            c1.push_back(MoveOnly(*p));
-        std::deque<MoveOnly, A> c2(A(5));
-        for (int* p = ab; p < an; ++p)
-            c2.push_back(MoveOnly(*p));
-        std::deque<MoveOnly, A> c3(A(6));
-        c3 = std::move(c1);
-        assert(c2 == c3);
-        assert(c1.size() == 0);
-        assert(c3.get_allocator() == A(5));
-        LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c1));
-        LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c2));
-        LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c3));
-    }
-    {
-        int ab[] = {3, 4, 2, 8, 0, 1, 44, 34, 45, 96, 80, 1, 13, 31, 45};
-        int* an = ab + sizeof(ab)/sizeof(ab[0]);
-        typedef min_allocator<MoveOnly> A;
-        std::deque<MoveOnly, A> c1(A{});
-        for (int* p = ab; p < an; ++p)
-            c1.push_back(MoveOnly(*p));
-        std::deque<MoveOnly, A> c2(A{});
-        for (int* p = ab; p < an; ++p)
-            c2.push_back(MoveOnly(*p));
-        std::deque<MoveOnly, A> c3(A{});
-        c3 = std::move(c1);
-        assert(c2 == c3);
-        assert(c1.size() == 0);
-        assert(c3.get_allocator() == A());
-        LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c1));
-        LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c2));
-        LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c3));
-    }
+int main(int, char**) {
+  {
+    int ab[] = {3, 4, 2, 8, 0, 1, 44, 34, 45, 96, 80, 1, 13, 31, 45};
+    int* an  = ab + sizeof(ab) / sizeof(ab[0]);
+    typedef test_allocator<MoveOnly> A;
+    std::deque<MoveOnly, A> c1(A(5));
+    for (int* p = ab; p < an; ++p)
+      c1.push_back(MoveOnly(*p));
+    std::deque<MoveOnly, A> c2(A(5));
+    for (int* p = ab; p < an; ++p)
+      c2.push_back(MoveOnly(*p));
+    std::deque<MoveOnly, A> c3(A(5));
+    c3 = std::move(c1);
+    assert(c2 == c3);
+    assert(c1.size() == 0);
+    assert(c3.get_allocator() == A(5));
+    LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c1));
+    LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c2));
+    LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c3));
+  }
+  {
+    int ab[] = {3, 4, 2, 8, 0, 1, 44, 34, 45, 96, 80, 1, 13, 31, 45};
+    int* an  = ab + sizeof(ab) / sizeof(ab[0]);
+    typedef test_allocator<MoveOnly> A;
+    std::deque<MoveOnly, A> c1(A(5));
+    for (int* p = ab; p < an; ++p)
+      c1.push_back(MoveOnly(*p));
+    std::deque<MoveOnly, A> c2(A(5));
+    for (int* p = ab; p < an; ++p)
+      c2.push_back(MoveOnly(*p));
+    std::deque<MoveOnly, A> c3(A(6));
+    c3 = std::move(c1);
+    assert(c2 == c3);
+    assert(c1.size() != 0);
+    assert(c3.get_allocator() == A(6));
+    LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c1));
+    LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c2));
+    LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c3));
+  }
+  {
+    int ab[] = {3, 4, 2, 8, 0, 1, 44, 34, 45, 96, 80, 1, 13, 31, 45};
+    int* an  = ab + sizeof(ab) / sizeof(ab[0]);
+    typedef other_allocator<MoveOnly> A;
+    std::deque<MoveOnly, A> c1(A(5));
+    for (int* p = ab; p < an; ++p)
+      c1.push_back(MoveOnly(*p));
+    std::deque<MoveOnly, A> c2(A(5));
+    for (int* p = ab; p < an; ++p)
+      c2.push_back(MoveOnly(*p));
+    std::deque<MoveOnly, A> c3(A(6));
+    c3 = std::move(c1);
+    assert(c2 == c3);
+    assert(c1.size() == 0);
+    assert(c3.get_allocator() == A(5));
+    LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c1));
+    LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c2));
+    LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c3));
+  }
+  {
+    int ab[] = {3, 4, 2, 8, 0, 1, 44, 34, 45, 96, 80, 1, 13, 31, 45};
+    int* an  = ab + sizeof(ab) / sizeof(ab[0]);
+    typedef min_allocator<MoveOnly> A;
+    std::deque<MoveOnly, A> c1(A{});
+    for (int* p = ab; p < an; ++p)
+      c1.push_back(MoveOnly(*p));
+    std::deque<MoveOnly, A> c2(A{});
+    for (int* p = ab; p < an; ++p)
+      c2.push_back(MoveOnly(*p));
+    std::deque<MoveOnly, A> c3(A{});
+    c3 = std::move(c1);
+    assert(c2 == c3);
+    assert(c1.size() == 0);
+    assert(c3.get_allocator() == A());
+    LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c1));
+    LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c2));
+    LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c3));
+  }
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/deque/deque.cons/move_assign_noexcept.compile.pass.cpp b/libcxx/test/std/containers/sequences/deque/deque.cons/move_assign_noexcept.compile.pass.cpp
new file mode 100644
index 0000000000000..417f44922d37d
--- /dev/null
+++ b/libcxx/test/std/containers/sequences/deque/deque.cons/move_assign_noexcept.compile.pass.cpp
@@ -0,0 +1,49 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// <deque>
+
+// deque& operator=(deque&& c)
+//     noexcept(
+//          allocator_type::propagate_on_container_move_assignment::value &&
+//          is_nothrow_move_assignable<allocator_type>::value);
+
+// This tests a conforming extension
+
+// UNSUPPORTED: c++03
+
+#include <deque>
+
+#include "test_macros.h"
+#include "MoveOnly.h"
+#include "test_allocator.h"
+
+template <class T>
+struct always_equal_alloc {
+  using value_type = T;
+  always_equal_alloc(const always_equal_alloc&);
+  void allocate(std::size_t);
+};
+
+template <class T>
+struct not_always_equal_alloc {
+  int i;
+  using value_type = T;
+  not_always_equal_alloc(const not_always_equal_alloc&);
+  void allocate(std::size_t);
+};
+
+static_assert(std::is_nothrow_move_assignable<std::deque<MoveOnly>>::value, "");
+static_assert(!std::is_nothrow_move_assignable<std::deque<MoveOnly, test_allocator<MoveOnly>>>::value, "");
+#if TEST_STD_VER >= 17
+static_assert(std::is_nothrow_move_assignable<std::deque<MoveOnly, always_equal_alloc<MoveOnly>>>::value, "");
+#endif
+static_assert(!std::is_nothrow_move_assignable<std::deque<MoveOnly, not_always_equal_alloc<MoveOnly>>>::value, "");
+#if defined(_LIBCPP_VERSION)
+static_assert(std::is_nothrow_move_assignable<std::deque<MoveOnly, other_allocator<MoveOnly>>>::value, "");
+#endif // _LIBCPP_VERSION
diff --git a/libcxx/test/std/containers/sequences/deque/deque.cons/move_assign_noexcept.pass.cpp b/libcxx/test/std/containers/sequences/deque/deque.cons/move_assign_noexcept.pass.cpp
deleted file mode 100644
index cbefbf6dae912..0000000000000
--- a/libcxx/test/std/containers/sequences/deque/deque.cons/move_assign_noexcept.pass.cpp
+++ /dev/null
@@ -1,57 +0,0 @@
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-// <deque>
-
-// deque& operator=(deque&& c)
-//     noexcept(
-//          allocator_type::propagate_on_container_move_assignment::value &&
-//          is_nothrow_move_assignable<allocator_type>::value);
-
-// This tests a conforming extension
-
-// UNSUPPORTED: c++03
-
-#include <deque>
-#include <cassert>
-
-#include "test_macros.h"
-#include "MoveOnly.h"
-#include "test_allocator.h"
-
-template <class T>
-struct some_alloc
-{
-    typedef T value_type;
-    some_alloc(const some_alloc&);
-    void allocate(std::size_t);
-};
-
-int main(int, char**)
-{
-    {
-        typedef std::deque<MoveOnly> C;
-        static_assert(std::is_nothrow_move_assignable<C>::value, "");
-    }
-    {
-        typedef std::deque<MoveOnly, test_allocator<MoveOnly>> C;
-        static_assert(!std::is_nothrow_move_assignable<C>::value, "");
-    }
-#if defined(_LIBCPP_VERSION)
-    {
-        typedef std::deque<MoveOnly, other_allocator<MoveOnly>> C;
-        static_assert(std::is_nothrow_move_assignable<C>::value, "");
-    }
-    {
-        typedef std::deque<MoveOnly, some_alloc<MoveOnly>> C;
-        static_assert(!std::is_nothrow_move_assignable<C>::value, "");
-    }
-#endif // _LIBCPP_VERSION
-
-  return 0;
-}
diff --git a/libcxx/test/std/containers/sequences/deque/deque.cons/move_noexcept.pass.cpp b/libcxx/test/std/containers/sequences/deque/deque.cons/move_noexcept.pass.cpp
index add19ec61c8cc..37e8a801e9d71 100644
--- a/libcxx/test/std/containers/sequences/deque/deque.cons/move_noexcept.pass.cpp
+++ b/libcxx/test/std/containers/sequences/deque/deque.cons/move_noexcept.pass.cpp
@@ -23,32 +23,30 @@
 #include "test_allocator.h"
 
 template <class T>
-struct some_alloc
-{
-    typedef T value_type;
-    some_alloc(const some_alloc&);
-    void allocate(std::size_t);
+struct some_alloc {
+  typedef T value_type;
+  some_alloc(const some_alloc&);
+  void allocate(std::size_t);
 };
 
-int main(int, char**)
-{
+int main(int, char**) {
 #if defined(_LIBCPP_VERSION)
-    {
-        typedef std::deque<MoveOnly> C;
-        static_assert(std::is_nothrow_move_constructible<C>::value, "");
-    }
-    {
-        typedef std::deque<MoveOnly, test_allocator<MoveOnly>> C;
-        static_assert(std::is_nothrow_move_constructible<C>::value, "");
-    }
-    {
-        typedef std::deque<MoveOnly, other_allocator<MoveOnly>> C;
-        static_assert(std::is_nothrow_move_constructible<C>::value, "");
-    }
-    {
-        typedef std::deque<MoveOnly, some_alloc<MoveOnly>> C;
-        static_assert(!std::is_nothrow_move_constructible<C>::value, "");
-    }
+  {
+    typedef std::deque<MoveOnly> C;
+    static_assert(std::is_nothrow_move_constructible<C>::value, "");
+  }
+  {
+    typedef std::deque<MoveOnly, test_allocator<MoveOnly>> C;
+    static_assert(std::is_nothrow_move_constructible<C>::value, "");
+  }
+  {
+    typedef std::deque<MoveOnly, other_allocator<MoveOnly>> C;
+    static_assert(std::is_nothrow_move_constructible<C>::value, "");
+  }
+  {
+    typedef std::deque<MoveOnly, some_alloc<MoveOnly>> C;
+    static_assert(!std::is_nothrow_move_constructible<C>::value, "");
+  }
 #endif // _LIBCPP_VERSION
 
   return 0;
diff --git a/libcxx/test/std/containers/sequences/deque/deque.cons/op_equal.pass.cpp b/libcxx/test/std/containers/sequences/deque/deque.cons/op_equal.pass.cpp
index 538d3fc4aa445..05d172268e0bd 100644
--- a/libcxx/test/std/containers/sequences/deque/deque.cons/op_equal.pass.cpp
+++ b/libcxx/test/std/containers/sequences/deque/deque.cons/op_equal.pass.cpp
@@ -18,56 +18,53 @@
 #include "min_allocator.h"
 
 template <class C>
-void
-test(const C& x)
-{
-    C c;
-    c = x;
-    assert(c == x);
-    LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c));
-    LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(x));
+void test(const C& x) {
+  C c;
+  c = x;
+  assert(c == x);
+  LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c));
+  LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(x));
 }
 
-int main(int, char**)
-{
-    {
-        int ab[] = {3, 4, 2, 8, 0, 1, 44, 34, 45, 96, 80, 1, 13, 31, 45};
-        int* an = ab + sizeof(ab)/sizeof(ab[0]);
-        test(std::deque<int>(ab, an));
-    }
-    {
-        std::deque<int, test_allocator<int> > l(3, 2, test_allocator<int>(5));
-        std::deque<int, test_allocator<int> > l2(l, test_allocator<int>(3));
-        l2 = l;
-        assert(l2 == l);
-        assert(l2.get_allocator() == test_allocator<int>(3));
-        LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(l));
-        LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(l2));
-    }
-    {
-        std::deque<int, other_allocator<int> > l(3, 2, other_allocator<int>(5));
-        std::deque<int, other_allocator<int> > l2(l, other_allocator<int>(3));
-        l2 = l;
-        assert(l2 == l);
-        assert(l2.get_allocator() == other_allocator<int>(5));
-        LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(l));
-        LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(l2));
-    }
+int main(int, char**) {
+  {
+    int ab[] = {3, 4, 2, 8, 0, 1, 44, 34, 45, 96, 80, 1, 13, 31, 45};
+    int* an  = ab + sizeof(ab) / sizeof(ab[0]);
+    test(std::deque<int>(ab, an));
+  }
+  {
+    std::deque<int, test_allocator<int> > l(3, 2, test_allocator<int>(5));
+    std::deque<int, test_allocator<int> > l2(l, test_allocator<int>(3));
+    l2 = l;
+    assert(l2 == l);
+    assert(l2.get_allocator() == test_allocator<int>(3));
+    LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(l));
+    LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(l2));
+  }
+  {
+    std::deque<int, other_allocator<int> > l(3, 2, other_allocator<int>(5));
+    std::deque<int, other_allocator<int> > l2(l, other_allocator<int>(3));
+    l2 = l;
+    assert(l2 == l);
+    assert(l2.get_allocator() == other_allocator<int>(5));
+    LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(l));
+    LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(l2));
+  }
 #if TEST_STD_VER >= 11
-    {
-        int ab[] = {3, 4, 2, 8, 0, 1, 44, 34, 45, 96, 80, 1, 13, 31, 45};
-        int* an = ab + sizeof(ab)/sizeof(ab[0]);
-        test(std::deque<int, min_allocator<int>>(ab, an));
-    }
-    {
-        std::deque<int, min_allocator<int> > l(3, 2, min_allocator<int>());
-        std::deque<int, min_allocator<int> > l2(l, min_allocator<int>());
-        l2 = l;
-        assert(l2 == l);
-        assert(l2.get_allocator() == min_allocator<int>());
-        LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(l));
-        LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(l2));
-    }
+  {
+    int ab[] = {3, 4, 2, 8, 0, 1, 44, 34, 45, 96, 80, 1, 13, 31, 45};
+    int* an  = ab + sizeof(ab) / sizeof(ab[0]);
+    test(std::deque<int, min_allocator<int>>(ab, an));
+  }
+  {
+    std::deque<int, min_allocator<int> > l(3, 2, min_allocator<int>());
+    std::deque<int, min_allocator<int> > l2(l, min_allocator<int>());
+    l2 = l;
+    assert(l2 == l);
+    assert(l2.get_allocator() == min_allocator<int>());
+    LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(l));
+    LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(l2));
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/sequences/deque/deque.cons/op_equal_initializer_list.pass.cpp b/libcxx/test/std/containers/sequences/deque/deque.cons/op_equal_initializer_list.pass.cpp
index 2331d3017dcd8..b2760b4a3ff49 100644
--- a/libcxx/test/std/containers/sequences/deque/deque.cons/op_equal_initializer_list.pass.cpp
+++ b/libcxx/test/std/containers/sequences/deque/deque.cons/op_equal_initializer_list.pass.cpp
@@ -19,9 +19,8 @@
 #include "test_macros.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
+int main(int, char**) {
+  {
     std::deque<int> d;
     d = {3, 4, 5, 6};
     assert(d.size() == 4);
@@ -30,8 +29,8 @@ int main(int, char**)
     assert(d[2] == 5);
     assert(d[3] == 6);
     LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(d));
-    }
-    {
+  }
+  {
     std::deque<int, min_allocator<int>> d;
     d = {3, 4, 5, 6};
     assert(d.size() == 4);
@@ -40,7 +39,7 @@ int main(int, char**)
     assert(d[2] == 5);
     assert(d[3] == 6);
     LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(d));
-    }
+  }
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/deque/deque.cons/size.pass.cpp b/libcxx/test/std/containers/sequences/deque/deque.cons/size.pass.cpp
index de1a9843a478e..f8f42bd668f83 100644
--- a/libcxx/test/std/containers/sequences/deque/deque.cons/size.pass.cpp
+++ b/libcxx/test/std/containers/sequences/deque/deque.cons/size.pass.cpp
@@ -21,36 +21,32 @@
 #include "min_allocator.h"
 
 template <class T, class Allocator>
-void
-test2(unsigned n)
-{
+void test2(unsigned n) {
 #if TEST_STD_VER > 11
-    typedef std::deque<T, Allocator> C;
-    typedef typename C::const_iterator const_iterator;
-    assert(DefaultOnly::count == 0);
-    {
+  typedef std::deque<T, Allocator> C;
+  typedef typename C::const_iterator const_iterator;
+  assert(DefaultOnly::count == 0);
+  {
     C d(n, Allocator());
     assert(static_cast<unsigned>(DefaultOnly::count) == n);
     assert(d.size() == n);
     assert(static_cast<std::size_t>(std::distance(d.begin(), d.end())) == d.size());
     LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(d));
     for (const_iterator i = d.begin(), e = d.end(); i != e; ++i)
-        assert(*i == T());
-    }
-    assert(DefaultOnly::count == 0);
+      assert(*i == T());
+  }
+  assert(DefaultOnly::count == 0);
 #else
-    ((void)n);
+  ((void)n);
 #endif
 }
 
 template <class T, class Allocator>
-void
-test1(unsigned n)
-{
-    typedef std::deque<T, Allocator> C;
-    typedef typename C::const_iterator const_iterator;
-    assert(DefaultOnly::count == 0);
-    {
+void test1(unsigned n) {
+  typedef std::deque<T, Allocator> C;
+  typedef typename C::const_iterator const_iterator;
+  assert(DefaultOnly::count == 0);
+  {
     C d(n);
     assert(static_cast<unsigned>(DefaultOnly::count) == n);
     assert(d.size() == n);
@@ -58,65 +54,59 @@ test1(unsigned n)
     LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(d));
 #if TEST_STD_VER >= 11
     for (const_iterator i = d.begin(), e = d.end(); i != e; ++i)
-        assert(*i == T());
+      assert(*i == T());
 #endif
-    }
-    assert(DefaultOnly::count == 0);
+  }
+  assert(DefaultOnly::count == 0);
 }
 
 template <class T, class Allocator>
-void
-test3(unsigned n, Allocator const &alloc = Allocator())
-{
+void test3(unsigned n, Allocator const& alloc = Allocator()) {
 #if TEST_STD_VER > 11
-    typedef std::deque<T, Allocator> C;
-    {
+  typedef std::deque<T, Allocator> C;
+  {
     C d(n, alloc);
     assert(d.size() == n);
     assert(d.get_allocator() == alloc);
     LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(d));
-    }
+  }
 #else
-    ((void)n);
-    ((void)alloc);
+  ((void)n);
+  ((void)alloc);
 #endif
 }
 
 template <class T, class Allocator>
-void
-test(unsigned n)
-{
-    test1<T, Allocator> ( n );
-    test2<T, Allocator> ( n );
+void test(unsigned n) {
+  test1<T, Allocator>(n);
+  test2<T, Allocator>(n);
 }
 
-int main(int, char**)
-{
-    test<DefaultOnly, std::allocator<DefaultOnly> >(0);
-    test<DefaultOnly, std::allocator<DefaultOnly> >(1);
-    test<DefaultOnly, std::allocator<DefaultOnly> >(10);
-    test<DefaultOnly, std::allocator<DefaultOnly> >(1023);
-    test<DefaultOnly, std::allocator<DefaultOnly> >(1024);
-    test<DefaultOnly, std::allocator<DefaultOnly> >(1025);
-    test<DefaultOnly, std::allocator<DefaultOnly> >(2047);
-    test<DefaultOnly, std::allocator<DefaultOnly> >(2048);
-    test<DefaultOnly, std::allocator<DefaultOnly> >(2049);
-    test<DefaultOnly, std::allocator<DefaultOnly> >(4095);
-    test<DefaultOnly, std::allocator<DefaultOnly> >(4096);
-    test<DefaultOnly, std::allocator<DefaultOnly> >(4097);
+int main(int, char**) {
+  test<DefaultOnly, std::allocator<DefaultOnly> >(0);
+  test<DefaultOnly, std::allocator<DefaultOnly> >(1);
+  test<DefaultOnly, std::allocator<DefaultOnly> >(10);
+  test<DefaultOnly, std::allocator<DefaultOnly> >(1023);
+  test<DefaultOnly, std::allocator<DefaultOnly> >(1024);
+  test<DefaultOnly, std::allocator<DefaultOnly> >(1025);
+  test<DefaultOnly, std::allocator<DefaultOnly> >(2047);
+  test<DefaultOnly, std::allocator<DefaultOnly> >(2048);
+  test<DefaultOnly, std::allocator<DefaultOnly> >(2049);
+  test<DefaultOnly, std::allocator<DefaultOnly> >(4095);
+  test<DefaultOnly, std::allocator<DefaultOnly> >(4096);
+  test<DefaultOnly, std::allocator<DefaultOnly> >(4097);
 
-    LIBCPP_ONLY(test1<DefaultOnly, limited_allocator<DefaultOnly, 4096> >(4095));
+  LIBCPP_ONLY(test1<DefaultOnly, limited_allocator<DefaultOnly, 4096> >(4095));
 
 #if TEST_STD_VER >= 11
-    test<DefaultOnly, min_allocator<DefaultOnly> >(4095);
+  test<DefaultOnly, min_allocator<DefaultOnly> >(4095);
 #endif
 
 #if TEST_STD_VER > 11
-    test3<DefaultOnly, std::allocator<DefaultOnly>> (1023);
-    test3<int, std::allocator<int>>(1);
-    test3<int, min_allocator<int>> (3);
+  test3<DefaultOnly, std::allocator<DefaultOnly>>(1023);
+  test3<int, std::allocator<int>>(1);
+  test3<int, min_allocator<int>>(3);
 #endif
 
-
   return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/deque/deque.cons/size_value.pass.cpp b/libcxx/test/std/containers/sequences/deque/deque.cons/size_value.pass.cpp
index 767c544e95561..231aa9c44c663 100644
--- a/libcxx/test/std/containers/sequences/deque/deque.cons/size_value.pass.cpp
+++ b/libcxx/test/std/containers/sequences/deque/deque.cons/size_value.pass.cpp
@@ -20,36 +20,33 @@
 #include "min_allocator.h"
 
 template <class T, class Allocator>
-void
-test(unsigned n, const T& x)
-{
-    typedef std::deque<T, Allocator> C;
-    typedef typename C::const_iterator const_iterator;
-    C d(n, x);
-    assert(d.size() == n);
-    assert(static_cast<std::size_t>(std::distance(d.begin(), d.end())) == d.size());
-    LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(d));
-    for (const_iterator i = d.begin(), e = d.end(); i != e; ++i)
-        assert(*i == x);
+void test(unsigned n, const T& x) {
+  typedef std::deque<T, Allocator> C;
+  typedef typename C::const_iterator const_iterator;
+  C d(n, x);
+  assert(d.size() == n);
+  assert(static_cast<std::size_t>(std::distance(d.begin(), d.end())) == d.size());
+  LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(d));
+  for (const_iterator i = d.begin(), e = d.end(); i != e; ++i)
+    assert(*i == x);
 }
 
-int main(int, char**)
-{
-    test<int, std::allocator<int> >(0, 5);
-    test<int, std::allocator<int> >(1, 10);
-    test<int, std::allocator<int> >(10, 11);
-    test<int, std::allocator<int> >(1023, -11);
-    test<int, std::allocator<int> >(1024, 25);
-    test<int, std::allocator<int> >(1025, 0);
-    test<int, std::allocator<int> >(2047, 110);
-    test<int, std::allocator<int> >(2048, -500);
-    test<int, std::allocator<int> >(2049, 654);
-    test<int, std::allocator<int> >(4095, 78);
-    test<int, std::allocator<int> >(4096, 1165);
-    test<int, std::allocator<int> >(4097, 157);
-    LIBCPP_ONLY(test<int, limited_allocator<int, 4096> >(4095, 90));
+int main(int, char**) {
+  test<int, std::allocator<int> >(0, 5);
+  test<int, std::allocator<int> >(1, 10);
+  test<int, std::allocator<int> >(10, 11);
+  test<int, std::allocator<int> >(1023, -11);
+  test<int, std::allocator<int> >(1024, 25);
+  test<int, std::allocator<int> >(1025, 0);
+  test<int, std::allocator<int> >(2047, 110);
+  test<int, std::allocator<int> >(2048, -500);
+  test<int, std::allocator<int> >(2049, 654);
+  test<int, std::allocator<int> >(4095, 78);
+  test<int, std::allocator<int> >(4096, 1165);
+  test<int, std::allocator<int> >(4097, 157);
+  LIBCPP_ONLY(test<int, limited_allocator<int, 4096> >(4095, 90));
 #if TEST_STD_VER >= 11
-    test<int, min_allocator<int> >(4095, 90);
+  test<int, min_allocator<int> >(4095, 90);
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/sequences/deque/deque.cons/size_value_alloc.pass.cpp b/libcxx/test/std/containers/sequences/deque/deque.cons/size_value_alloc.pass.cpp
index 746d5a221663f..f8ea37e64d9eb 100644
--- a/libcxx/test/std/containers/sequences/deque/deque.cons/size_value_alloc.pass.cpp
+++ b/libcxx/test/std/containers/sequences/deque/deque.cons/size_value_alloc.pass.cpp
@@ -19,23 +19,20 @@
 #include "min_allocator.h"
 
 template <class T, class Allocator>
-void
-test(unsigned n, const T& x, const Allocator& a)
-{
-    typedef std::deque<T, Allocator> C;
-    typedef typename C::const_iterator const_iterator;
-    C d(n, x, a);
-    assert(d.get_allocator() == a);
-    assert(d.size() == n);
-    assert(static_cast<std::size_t>(std::distance(d.begin(), d.end())) == d.size());
-    LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(d));
-    for (const_iterator i = d.begin(), e = d.end(); i != e; ++i)
-        assert(*i == x);
+void test(unsigned n, const T& x, const Allocator& a) {
+  typedef std::deque<T, Allocator> C;
+  typedef typename C::const_iterator const_iterator;
+  C d(n, x, a);
+  assert(d.get_allocator() == a);
+  assert(d.size() == n);
+  assert(static_cast<std::size_t>(std::distance(d.begin(), d.end())) == d.size());
+  LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(d));
+  for (const_iterator i = d.begin(), e = d.end(); i != e; ++i)
+    assert(*i == x);
 }
 
-int main(int, char**)
-{
-    {
+int main(int, char**) {
+  {
     std::allocator<int> a;
     test(0, 5, a);
     test(1, 10, a);
@@ -49,9 +46,9 @@ int main(int, char**)
     test(4095, 78, a);
     test(4096, 1165, a);
     test(4097, 157, a);
-    }
+  }
 #if TEST_STD_VER >= 11
-    {
+  {
     min_allocator<int> a;
     test(0, 5, a);
     test(1, 10, a);
@@ -65,7 +62,7 @@ int main(int, char**)
     test(4095, 78, a);
     test(4096, 1165, a);
     test(4097, 157, a);
-    }
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/sequences/deque/deque.erasure/erase.pass.cpp b/libcxx/test/std/containers/sequences/deque/deque.erasure/erase.pass.cpp
index edb52a79618ec..947e06bd64375 100644
--- a/libcxx/test/std/containers/sequences/deque/deque.erasure/erase.pass.cpp
+++ b/libcxx/test/std/containers/sequences/deque/deque.erasure/erase.pass.cpp
@@ -32,8 +32,7 @@ void test0(S s, U val, S expected, std::size_t expected_erased_count) {
 }
 
 template <class S>
-void test()
-{
+void test() {
   test0(S(), 1, S(), 0);
 
   test0(S({1}), 1, S(), 1);
@@ -67,15 +66,14 @@ void test()
   test0(S({1, 2, 1}), opt(3), S({1, 2, 1}), 0);
 }
 
-int main(int, char**)
-{
-    test<std::deque<int>>();
-    test<std::deque<int, min_allocator<int>>> ();
-    test<std::deque<int, safe_allocator<int>>> ();
-    test<std::deque<int, test_allocator<int>>> ();
+int main(int, char**) {
+  test<std::deque<int>>();
+  test<std::deque<int, min_allocator<int>>>();
+  test<std::deque<int, safe_allocator<int>>>();
+  test<std::deque<int, test_allocator<int>>>();
 
-    test<std::deque<long>>();
-    test<std::deque<double>>();
+  test<std::deque<long>>();
+  test<std::deque<double>>();
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/deque/deque.erasure/erase_if.pass.cpp b/libcxx/test/std/containers/sequences/deque/deque.erasure/erase_if.pass.cpp
index 49709bcc033e2..ed5220422dab2 100644
--- a/libcxx/test/std/containers/sequences/deque/deque.erasure/erase_if.pass.cpp
+++ b/libcxx/test/std/containers/sequences/deque/deque.erasure/erase_if.pass.cpp
@@ -31,53 +31,51 @@ void test0(S s, Pred p, S expected, std::size_t expected_erased_count) {
 }
 
 template <typename S>
-void test()
-{
-    auto is1 = [](auto v) { return v == 1;};
-    auto is2 = [](auto v) { return v == 2;};
-    auto is3 = [](auto v) { return v == 3;};
-    auto is4 = [](auto v) { return v == 4;};
-    auto True  = [](auto) { return true; };
-    auto False = [](auto) { return false; };
-
-    test0(S(), is1, S(), 0);
-
-    test0(S({1}), is1, S(), 1);
-    test0(S({1}), is2, S({1}), 0);
-
-    test0(S({1, 2}), is1, S({2}), 1);
-    test0(S({1, 2}), is2, S({1}), 1);
-    test0(S({1, 2}), is3, S({1, 2}), 0);
-    test0(S({1, 1}), is1, S(), 2);
-    test0(S({1, 1}), is3, S({1, 1}), 0);
-
-    test0(S({1, 2, 3}), is1, S({2, 3}), 1);
-    test0(S({1, 2, 3}), is2, S({1, 3}), 1);
-    test0(S({1, 2, 3}), is3, S({1, 2}), 1);
-    test0(S({1, 2, 3}), is4, S({1, 2, 3}), 0);
-
-    test0(S({1, 1, 1}), is1, S(), 3);
-    test0(S({1, 1, 1}), is2, S({1, 1, 1}), 0);
-    test0(S({1, 1, 2}), is1, S({2}), 2);
-    test0(S({1, 1, 2}), is2, S({1, 1}), 1);
-    test0(S({1, 1, 2}), is3, S({1, 1, 2}), 0);
-    test0(S({1, 2, 2}), is1, S({2, 2}), 1);
-    test0(S({1, 2, 2}), is2, S({1}), 2);
-    test0(S({1, 2, 2}), is3, S({1, 2, 2}), 0);
-
-    test0(S({1, 2, 3}), True, S(), 3);
-    test0(S({1, 2, 3}), False, S({1, 2, 3}), 0);
+void test() {
+  auto is1   = [](auto v) { return v == 1; };
+  auto is2   = [](auto v) { return v == 2; };
+  auto is3   = [](auto v) { return v == 3; };
+  auto is4   = [](auto v) { return v == 4; };
+  auto True  = [](auto) { return true; };
+  auto False = [](auto) { return false; };
+
+  test0(S(), is1, S(), 0);
+
+  test0(S({1}), is1, S(), 1);
+  test0(S({1}), is2, S({1}), 0);
+
+  test0(S({1, 2}), is1, S({2}), 1);
+  test0(S({1, 2}), is2, S({1}), 1);
+  test0(S({1, 2}), is3, S({1, 2}), 0);
+  test0(S({1, 1}), is1, S(), 2);
+  test0(S({1, 1}), is3, S({1, 1}), 0);
+
+  test0(S({1, 2, 3}), is1, S({2, 3}), 1);
+  test0(S({1, 2, 3}), is2, S({1, 3}), 1);
+  test0(S({1, 2, 3}), is3, S({1, 2}), 1);
+  test0(S({1, 2, 3}), is4, S({1, 2, 3}), 0);
+
+  test0(S({1, 1, 1}), is1, S(), 3);
+  test0(S({1, 1, 1}), is2, S({1, 1, 1}), 0);
+  test0(S({1, 1, 2}), is1, S({2}), 2);
+  test0(S({1, 1, 2}), is2, S({1, 1}), 1);
+  test0(S({1, 1, 2}), is3, S({1, 1, 2}), 0);
+  test0(S({1, 2, 2}), is1, S({2, 2}), 1);
+  test0(S({1, 2, 2}), is2, S({1}), 2);
+  test0(S({1, 2, 2}), is3, S({1, 2, 2}), 0);
+
+  test0(S({1, 2, 3}), True, S(), 3);
+  test0(S({1, 2, 3}), False, S({1, 2, 3}), 0);
 }
 
-int main(int, char**)
-{
-    test<std::deque<int>>();
-    test<std::deque<int, min_allocator<int>>> ();
-    test<std::deque<int, safe_allocator<int>>> ();
-    test<std::deque<int, test_allocator<int>>> ();
+int main(int, char**) {
+  test<std::deque<int>>();
+  test<std::deque<int, min_allocator<int>>>();
+  test<std::deque<int, safe_allocator<int>>>();
+  test<std::deque<int, test_allocator<int>>>();
 
-    test<std::deque<long>>();
-    test<std::deque<double>>();
+  test<std::deque<long>>();
+  test<std::deque<double>>();
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/deque/deque.modifiers/clear.pass.cpp b/libcxx/test/std/containers/sequences/deque/deque.modifiers/clear.pass.cpp
index 1e0b04dd24f25..bb2fa6e52af43 100644
--- a/libcxx/test/std/containers/sequences/deque/deque.modifiers/clear.pass.cpp
+++ b/libcxx/test/std/containers/sequences/deque/deque.modifiers/clear.pass.cpp
@@ -18,57 +18,56 @@
 #include "../../../NotConstructible.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef NotConstructible T;
-        typedef std::deque<T> C;
-        C c;
-        ASSERT_NOEXCEPT(c.clear());
-        c.clear();
-        assert(std::distance(c.begin(), c.end()) == 0);
-        LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c));
-    }
-    {
-        typedef int T;
-        typedef std::deque<T> C;
-        const T t[] = {0, 1, 2, 3, 4};
-        C c(std::begin(t), std::end(t));
+int main(int, char**) {
+  {
+    typedef NotConstructible T;
+    typedef std::deque<T> C;
+    C c;
+    ASSERT_NOEXCEPT(c.clear());
+    c.clear();
+    assert(std::distance(c.begin(), c.end()) == 0);
+    LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c));
+  }
+  {
+    typedef int T;
+    typedef std::deque<T> C;
+    const T t[] = {0, 1, 2, 3, 4};
+    C c(std::begin(t), std::end(t));
 
-        ASSERT_NOEXCEPT(c.clear());
-        c.clear();
-        assert(std::distance(c.begin(), c.end()) == 0);
-        LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c));
+    ASSERT_NOEXCEPT(c.clear());
+    c.clear();
+    assert(std::distance(c.begin(), c.end()) == 0);
+    LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c));
 
-        c.clear();
-        assert(std::distance(c.begin(), c.end()) == 0);
-        LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c));
-    }
+    c.clear();
+    assert(std::distance(c.begin(), c.end()) == 0);
+    LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c));
+  }
 #if TEST_STD_VER >= 11
-    {
-        typedef NotConstructible T;
-        typedef std::deque<T, min_allocator<T>> C;
-        C c;
-        ASSERT_NOEXCEPT(c.clear());
-        c.clear();
-        assert(std::distance(c.begin(), c.end()) == 0);
-        LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c));
-    }
-    {
-        typedef int T;
-        typedef std::deque<T, min_allocator<T>> C;
-        const T t[] = {0, 1, 2, 3, 4};
-        C c(std::begin(t), std::end(t));
+  {
+    typedef NotConstructible T;
+    typedef std::deque<T, min_allocator<T>> C;
+    C c;
+    ASSERT_NOEXCEPT(c.clear());
+    c.clear();
+    assert(std::distance(c.begin(), c.end()) == 0);
+    LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c));
+  }
+  {
+    typedef int T;
+    typedef std::deque<T, min_allocator<T>> C;
+    const T t[] = {0, 1, 2, 3, 4};
+    C c(std::begin(t), std::end(t));
 
-        ASSERT_NOEXCEPT(c.clear());
-        c.clear();
-        assert(std::distance(c.begin(), c.end()) == 0);
-        LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c));
+    ASSERT_NOEXCEPT(c.clear());
+    c.clear();
+    assert(std::distance(c.begin(), c.end()) == 0);
+    LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c));
 
-        c.clear();
-        assert(std::distance(c.begin(), c.end()) == 0);
-        LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c));
-    }
+    c.clear();
+    assert(std::distance(c.begin(), c.end()) == 0);
+    LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c));
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/sequences/deque/deque.modifiers/emplace.pass.cpp b/libcxx/test/std/containers/sequences/deque/deque.modifiers/emplace.pass.cpp
index b992610506538..7c90216a9ce56 100644
--- a/libcxx/test/std/containers/sequences/deque/deque.modifiers/emplace.pass.cpp
+++ b/libcxx/test/std/containers/sequences/deque/deque.modifiers/emplace.pass.cpp
@@ -21,97 +21,81 @@
 #include "../../../Emplaceable.h"
 #include "min_allocator.h"
 
-
 template <class C>
-C
-make(int size, int start = 0 )
-{
-    const int b = 4096 / sizeof(int);
-    int init = 0;
-    if (start > 0)
-    {
-        init = (start+1) / b + ((start+1) % b != 0);
-        init *= b;
-        --init;
-    }
-    C c(init);
-    for (int i = 0; i < init-start; ++i)
-        c.pop_back();
-    for (int i = 0; i < size; ++i)
-        c.push_back(Emplaceable());
-    for (int i = 0; i < start; ++i)
-        c.pop_front();
-    return c;
+C make(int size, int start = 0) {
+  const int b = 4096 / sizeof(int);
+  int init    = 0;
+  if (start > 0) {
+    init = (start + 1) / b + ((start + 1) % b != 0);
+    init *= b;
+    --init;
+  }
+  C c(init);
+  for (int i = 0; i < init - start; ++i)
+    c.pop_back();
+  for (int i = 0; i < size; ++i)
+    c.push_back(Emplaceable());
+  for (int i = 0; i < start; ++i)
+    c.pop_front();
+  return c;
 }
 
 template <class C>
-void
-test(int P, C& c1)
-{
-    typedef typename C::const_iterator CI;
-    std::size_t c1_osize = c1.size();
-    CI i = c1.emplace(c1.begin() + P, Emplaceable(1, 2.5));
-    assert(i == c1.begin() + P);
-    assert(c1.size() == c1_osize + 1);
-    assert(static_cast<std::size_t>(std::distance(c1.begin(), c1.end())) == c1.size());
-    assert(*i == Emplaceable(1, 2.5));
-    LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c1));
+void test(int P, C& c1) {
+  typedef typename C::const_iterator CI;
+  std::size_t c1_osize = c1.size();
+  CI i                 = c1.emplace(c1.begin() + P, Emplaceable(1, 2.5));
+  assert(i == c1.begin() + P);
+  assert(c1.size() == c1_osize + 1);
+  assert(static_cast<std::size_t>(std::distance(c1.begin(), c1.end())) == c1.size());
+  assert(*i == Emplaceable(1, 2.5));
+  LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c1));
 }
 
 template <class C>
-void
-testN(int start, int N)
-{
-    for (int i = 0; i <= 3; ++i)
-    {
-        if (0 <= i && i <= N)
-        {
-            C c1 = make<C>(N, start);
-            test(i, c1);
-        }
+void testN(int start, int N) {
+  for (int i = 0; i <= 3; ++i) {
+    if (0 <= i && i <= N) {
+      C c1 = make<C>(N, start);
+      test(i, c1);
     }
-    for (int i = N/2-1; i <= N/2+1; ++i)
-    {
-        if (0 <= i && i <= N)
-        {
-            C c1 = make<C>(N, start);
-            test(i, c1);
-        }
+  }
+  for (int i = N / 2 - 1; i <= N / 2 + 1; ++i) {
+    if (0 <= i && i <= N) {
+      C c1 = make<C>(N, start);
+      test(i, c1);
     }
-    for (int i = N - 3; i <= N; ++i)
-    {
-        if (0 <= i && i <= N)
-        {
-            C c1 = make<C>(N, start);
-            test(i, c1);
-        }
+  }
+  for (int i = N - 3; i <= N; ++i) {
+    if (0 <= i && i <= N) {
+      C c1 = make<C>(N, start);
+      test(i, c1);
     }
+  }
 }
 
-
-int main(int, char**)
-{
-    {
-    int rng[] = {0, 1, 2, 3, 1023, 1024, 1025, 2047, 2048, 2049};
-    const int N = sizeof(rng)/sizeof(rng[0]);
+int main(int, char**) {
+  {
+    int rng[]   = {0, 1, 2, 3, 1023, 1024, 1025, 2047, 2048, 2049};
+    const int N = sizeof(rng) / sizeof(rng[0]);
     for (int i = 0; i < N; ++i)
-        for (int j = 0; j < N; ++j)
-            testN<std::deque<Emplaceable> >(rng[i], rng[j]);
-    }
-    {
-    int rng[] = {0, 1, 2, 3, 1023, 1024, 1025, 2047, 2048, 2049};
-    const int N = sizeof(rng)/sizeof(rng[0]);
+      for (int j = 0; j < N; ++j)
+        testN<std::deque<Emplaceable> >(rng[i], rng[j]);
+  }
+  {
+    int rng[]   = {0, 1, 2, 3, 1023, 1024, 1025, 2047, 2048, 2049};
+    const int N = sizeof(rng) / sizeof(rng[0]);
     for (int i = 0; i < N; ++i)
-        for (int j = 0; j < N; ++j)
-            testN<std::deque<Emplaceable, min_allocator<Emplaceable>> >(rng[i], rng[j]);
-    }
-    {
-    int rng[] = {0, 1, 2, 3, 1023, 1024, 1025, 2047, 2048, 2049};
-    const int N = sizeof(rng)/sizeof(rng[0]);
+      for (int j = 0; j < N; ++j)
+        testN<std::deque<Emplaceable, min_allocator<Emplaceable>> >(rng[i], rng[j]);
+  }
+  {
+    int rng[]   = {0, 1, 2, 3, 1023, 1024, 1025, 2047, 2048, 2049};
+    const int N = sizeof(rng) / sizeof(rng[0]);
     for (int i = 0; i < N; ++i)
-        for (int j = 0; j < N; ++j)
-            testN<std::deque<Emplaceable, safe_allocator<Emplaceable>> >(rng[i], rng[j]);
-    }
+      for (int j = 0; j < N; ++j)
+        testN<std::deque<Emplaceable, safe_allocator<Emplaceable>> >(rng[i], rng[j]);
+  }
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/deque/deque.modifiers/emplace_back.pass.cpp b/libcxx/test/std/containers/sequences/deque/deque.modifiers/emplace_back.pass.cpp
index 1ea3423bd43c6..590ab432dd519 100644
--- a/libcxx/test/std/containers/sequences/deque/deque.modifiers/emplace_back.pass.cpp
+++ b/libcxx/test/std/containers/sequences/deque/deque.modifiers/emplace_back.pass.cpp
@@ -24,89 +24,80 @@
 #include "test_allocator.h"
 
 template <class C>
-C
-make(int size, int start = 0 )
-{
-    const int b = 4096 / sizeof(int);
-    int init = 0;
-    if (start > 0)
-    {
-        init = (start+1) / b + ((start+1) % b != 0);
-        init *= b;
-        --init;
-    }
-    C c(init);
-    for (int i = 0; i < init-start; ++i)
-        c.pop_back();
-    for (int i = 0; i < size; ++i)
-        c.push_back(Emplaceable());
-    for (int i = 0; i < start; ++i)
-        c.pop_front();
-    return c;
+C make(int size, int start = 0) {
+  const int b = 4096 / sizeof(int);
+  int init    = 0;
+  if (start > 0) {
+    init = (start + 1) / b + ((start + 1) % b != 0);
+    init *= b;
+    --init;
+  }
+  C c(init);
+  for (int i = 0; i < init - start; ++i)
+    c.pop_back();
+  for (int i = 0; i < size; ++i)
+    c.push_back(Emplaceable());
+  for (int i = 0; i < start; ++i)
+    c.pop_front();
+  return c;
 }
 
 template <class C>
-void
-test(C& c1)
-{
-    typedef typename C::iterator I;
-    std::size_t c1_osize = c1.size();
+void test(C& c1) {
+  typedef typename C::iterator I;
+  std::size_t c1_osize = c1.size();
 #if TEST_STD_VER > 14
-    typedef typename C::reference Ref;
-    Ref ref = c1.emplace_back(Emplaceable(1, 2.5));
+  typedef typename C::reference Ref;
+  Ref ref = c1.emplace_back(Emplaceable(1, 2.5));
 #else
-              c1.emplace_back(Emplaceable(1, 2.5));
+  c1.emplace_back(Emplaceable(1, 2.5));
 #endif
-    assert(c1.size() == c1_osize + 1);
-    assert(std::distance(c1.begin(), c1.end())
-               == static_cast<std::ptrdiff_t>(c1.size()));
-    I i = c1.end();
-    assert(*--i == Emplaceable(1, 2.5));
-    LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c1));
+  assert(c1.size() == c1_osize + 1);
+  assert(std::distance(c1.begin(), c1.end()) == static_cast<std::ptrdiff_t>(c1.size()));
+  I i = c1.end();
+  assert(*--i == Emplaceable(1, 2.5));
+  LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c1));
 #if TEST_STD_VER > 14
-    assert(&(*i) == &ref);
+  assert(&(*i) == &ref);
 #endif
 }
 
 template <class C>
-void
-testN(int start, int N)
-{
-    C c1 = make<C>(N, start);
-    test(c1);
+void testN(int start, int N) {
+  C c1 = make<C>(N, start);
+  test(c1);
 }
 
-int main(int, char**)
-{
-    {
-    int rng[] = {0, 1, 2, 3, 1023, 1024, 1025, 2047, 2048, 2049};
-    const int N = sizeof(rng)/sizeof(rng[0]);
+int main(int, char**) {
+  {
+    int rng[]   = {0, 1, 2, 3, 1023, 1024, 1025, 2047, 2048, 2049};
+    const int N = sizeof(rng) / sizeof(rng[0]);
     for (int i = 0; i < N; ++i)
-        for (int j = 0; j < N; ++j)
-            testN<std::deque<Emplaceable> >(rng[i], rng[j]);
-    }
-    {
-    int rng[] = {0, 1, 2, 3, 1023, 1024, 1025, 2047, 2048, 2049};
-    const int N = sizeof(rng)/sizeof(rng[0]);
+      for (int j = 0; j < N; ++j)
+        testN<std::deque<Emplaceable> >(rng[i], rng[j]);
+  }
+  {
+    int rng[]   = {0, 1, 2, 3, 1023, 1024, 1025, 2047, 2048, 2049};
+    const int N = sizeof(rng) / sizeof(rng[0]);
     for (int i = 0; i < N; ++i)
-        for (int j = 0; j < N; ++j)
-            testN<std::deque<Emplaceable, min_allocator<Emplaceable>> >(rng[i], rng[j]);
-    }
-    {
-        std::deque<Tag_X, TaggingAllocator<Tag_X>> c;
-        c.emplace_back();
-        assert(c.size() == 1);
-        LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c));
-        c.emplace_back(1, 2, 3);
-        assert(c.size() == 2);
-        LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c));
-        c.emplace_front();
-        assert(c.size() == 3);
-        LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c));
-        c.emplace_front(1, 2, 3);
-        assert(c.size() == 4);
-        LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c));
-    }
+      for (int j = 0; j < N; ++j)
+        testN<std::deque<Emplaceable, min_allocator<Emplaceable>> >(rng[i], rng[j]);
+  }
+  {
+    std::deque<Tag_X, TaggingAllocator<Tag_X>> c;
+    c.emplace_back();
+    assert(c.size() == 1);
+    LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c));
+    c.emplace_back(1, 2, 3);
+    assert(c.size() == 2);
+    LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c));
+    c.emplace_front();
+    assert(c.size() == 3);
+    LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c));
+    c.emplace_front(1, 2, 3);
+    assert(c.size() == 4);
+    LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c));
+  }
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/deque/deque.modifiers/emplace_front.pass.cpp b/libcxx/test/std/containers/sequences/deque/deque.modifiers/emplace_front.pass.cpp
index 26926e411c96a..3fbaee1fc1587 100644
--- a/libcxx/test/std/containers/sequences/deque/deque.modifiers/emplace_front.pass.cpp
+++ b/libcxx/test/std/containers/sequences/deque/deque.modifiers/emplace_front.pass.cpp
@@ -24,90 +24,80 @@
 #include "test_allocator.h"
 
 template <class C>
-C
-make(int size, int start = 0 )
-{
-    const int b = 4096 / sizeof(int);
-    int init = 0;
-    if (start > 0)
-    {
-        init = (start+1) / b + ((start+1) % b != 0);
-        init *= b;
-        --init;
-    }
-    C c(init);
-    for (int i = 0; i < init-start; ++i)
-        c.pop_back();
-    for (int i = 0; i < size; ++i)
-        c.push_back(Emplaceable());
-    for (int i = 0; i < start; ++i)
-        c.pop_front();
-    return c;
+C make(int size, int start = 0) {
+  const int b = 4096 / sizeof(int);
+  int init    = 0;
+  if (start > 0) {
+    init = (start + 1) / b + ((start + 1) % b != 0);
+    init *= b;
+    --init;
+  }
+  C c(init);
+  for (int i = 0; i < init - start; ++i)
+    c.pop_back();
+  for (int i = 0; i < size; ++i)
+    c.push_back(Emplaceable());
+  for (int i = 0; i < start; ++i)
+    c.pop_front();
+  return c;
 }
 
 template <class C>
-void
-test(C& c1)
-{
-    typedef typename C::iterator I;
-    std::size_t c1_osize = c1.size();
+void test(C& c1) {
+  typedef typename C::iterator I;
+  std::size_t c1_osize = c1.size();
 #if TEST_STD_VER > 14
-    typedef typename C::reference Ref;
-    Ref res_ref = c1.emplace_front(Emplaceable(1, 2.5));
+  typedef typename C::reference Ref;
+  Ref res_ref = c1.emplace_front(Emplaceable(1, 2.5));
 #else
-                  c1.emplace_front(Emplaceable(1, 2.5));
+  c1.emplace_front(Emplaceable(1, 2.5));
 #endif
-    assert(c1.size() == c1_osize + 1);
-    assert(std::distance(c1.begin(), c1.end())
-               == static_cast<std::ptrdiff_t>(c1.size()));
-    I i = c1.begin();
-    assert(*i == Emplaceable(1, 2.5));
-    LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c1));
+  assert(c1.size() == c1_osize + 1);
+  assert(std::distance(c1.begin(), c1.end()) == static_cast<std::ptrdiff_t>(c1.size()));
+  I i = c1.begin();
+  assert(*i == Emplaceable(1, 2.5));
+  LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c1));
 #if TEST_STD_VER > 14
-    assert(&res_ref == &(*i));
+  assert(&res_ref == &(*i));
 #endif
 }
 
 template <class C>
-void
-testN(int start, int N)
-{
-    C c1 = make<C>(N, start);
-    test(c1);
+void testN(int start, int N) {
+  C c1 = make<C>(N, start);
+  test(c1);
 }
 
-
-int main(int, char**)
-{
-    {
-    int rng[] = {0, 1, 2, 3, 1023, 1024, 1025, 2047, 2048, 2049};
-    const int N = sizeof(rng)/sizeof(rng[0]);
+int main(int, char**) {
+  {
+    int rng[]   = {0, 1, 2, 3, 1023, 1024, 1025, 2047, 2048, 2049};
+    const int N = sizeof(rng) / sizeof(rng[0]);
     for (int i = 0; i < N; ++i)
-        for (int j = 0; j < N; ++j)
-            testN<std::deque<Emplaceable> >(rng[i], rng[j]);
-    }
-    {
-    int rng[] = {0, 1, 2, 3, 1023, 1024, 1025, 2047, 2048, 2049};
-    const int N = sizeof(rng)/sizeof(rng[0]);
+      for (int j = 0; j < N; ++j)
+        testN<std::deque<Emplaceable> >(rng[i], rng[j]);
+  }
+  {
+    int rng[]   = {0, 1, 2, 3, 1023, 1024, 1025, 2047, 2048, 2049};
+    const int N = sizeof(rng) / sizeof(rng[0]);
     for (int i = 0; i < N; ++i)
-        for (int j = 0; j < N; ++j)
-            testN<std::deque<Emplaceable, min_allocator<Emplaceable>> >(rng[i], rng[j]);
-    }
-    {
-        std::deque<Tag_X, TaggingAllocator<Tag_X>> c;
-        c.emplace_front();
-        assert(c.size() == 1);
-        LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c));
-        c.emplace_front(1, 2, 3);
-        assert(c.size() == 2);
-        LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c));
-        c.emplace_front();
-        assert(c.size() == 3);
-        LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c));
-        c.emplace_front(1, 2, 3);
-        assert(c.size() == 4);
-        LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c));
-    }
+      for (int j = 0; j < N; ++j)
+        testN<std::deque<Emplaceable, min_allocator<Emplaceable>> >(rng[i], rng[j]);
+  }
+  {
+    std::deque<Tag_X, TaggingAllocator<Tag_X>> c;
+    c.emplace_front();
+    assert(c.size() == 1);
+    LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c));
+    c.emplace_front(1, 2, 3);
+    assert(c.size() == 2);
+    LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c));
+    c.emplace_front();
+    assert(c.size() == 3);
+    LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c));
+    c.emplace_front(1, 2, 3);
+    assert(c.size() == 4);
+    LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c));
+  }
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/deque/deque.modifiers/erase_iter.invalidation.pass.cpp b/libcxx/test/std/containers/sequences/deque/deque.modifiers/erase_iter.invalidation.pass.cpp
index 22f67b2f92873..b538b0f037ab1 100644
--- a/libcxx/test/std/containers/sequences/deque/deque.modifiers/erase_iter.invalidation.pass.cpp
+++ b/libcxx/test/std/containers/sequences/deque/deque.modifiers/erase_iter.invalidation.pass.cpp
@@ -20,56 +20,52 @@
 #include "test_macros.h"
 
 template <typename C>
-void del_at_start(C c)
-{
-    typename C::iterator first = c.begin();
-    typename C::iterator it1 = first + 1;
-    typename C::iterator it2 = c.end() - 1;
+void del_at_start(C c) {
+  typename C::iterator first = c.begin();
+  typename C::iterator it1   = first + 1;
+  typename C::iterator it2   = c.end() - 1;
 
-    c.erase (first);
+  c.erase(first);
 
-    typename C::iterator it3 = c.begin();
-    typename C::iterator it4 = c.end() - 1;
-    assert(  it1 ==   it3);
-    assert( *it1 ==  *it3);
-    assert(&*it1 == &*it3);
-    assert(  it2 ==   it4);
-    assert( *it2 ==  *it4);
-    assert(&*it2 == &*it4);
+  typename C::iterator it3 = c.begin();
+  typename C::iterator it4 = c.end() - 1;
+  assert(it1 == it3);
+  assert(*it1 == *it3);
+  assert(&*it1 == &*it3);
+  assert(it2 == it4);
+  assert(*it2 == *it4);
+  assert(&*it2 == &*it4);
 }
 
 template <typename C>
-void del_at_end(C c)
-{
-    typename C::iterator first = c.end() - 1;
-    typename C::iterator it1 = c.begin();
-    typename C::iterator it2 = first - 1;
+void del_at_end(C c) {
+  typename C::iterator first = c.end() - 1;
+  typename C::iterator it1   = c.begin();
+  typename C::iterator it2   = first - 1;
 
-    c.erase (first);
+  c.erase(first);
 
-    typename C::iterator it3 = c.begin();
-    typename C::iterator it4 = c.end() - 1;
-    assert(  it1 ==   it3);
-    assert( *it1 ==  *it3);
-    assert(&*it1 == &*it3);
-    assert(  it2 ==   it4);
-    assert( *it2 ==  *it4);
-    assert(&*it2 == &*it4);
-    LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c));
+  typename C::iterator it3 = c.begin();
+  typename C::iterator it4 = c.end() - 1;
+  assert(it1 == it3);
+  assert(*it1 == *it3);
+  assert(&*it1 == &*it3);
+  assert(it2 == it4);
+  assert(*it2 == *it4);
+  assert(&*it2 == &*it4);
+  LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c));
 }
 
-int main(int, char**)
-{
-    std::deque<int> queue;
-    for (int i = 0; i < 20; ++i)
-        queue.push_back(i);
+int main(int, char**) {
+  std::deque<int> queue;
+  for (int i = 0; i < 20; ++i)
+    queue.push_back(i);
 
-    while (queue.size() > 1)
-    {
-        del_at_start(queue);
-        del_at_end(queue);
-        queue.pop_back();
-    }
+  while (queue.size() > 1) {
+    del_at_start(queue);
+    del_at_end(queue);
+    queue.pop_back();
+  }
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/deque/deque.modifiers/erase_iter.pass.cpp b/libcxx/test/std/containers/sequences/deque/deque.modifiers/erase_iter.pass.cpp
index 6b19d5b8bc69b..2d40f0c449f4d 100644
--- a/libcxx/test/std/containers/sequences/deque/deque.modifiers/erase_iter.pass.cpp
+++ b/libcxx/test/std/containers/sequences/deque/deque.modifiers/erase_iter.pass.cpp
@@ -22,106 +22,109 @@
 
 #ifndef TEST_HAS_NO_EXCEPTIONS
 struct Throws {
-    Throws() : v_(0) {}
-    Throws(int v) : v_(v) {}
-    Throws(const Throws  &rhs) : v_(rhs.v_) { if (sThrows) throw 1; }
-    Throws(      Throws &&rhs) : v_(rhs.v_) { if (sThrows) throw 1; }
-    Throws& operator=(const Throws  &rhs) { v_ = rhs.v_; return *this; }
-    Throws& operator=(      Throws &&rhs) { v_ = rhs.v_; return *this; }
-    int v_;
+  Throws() : v_(0) {}
+  Throws(int v) : v_(v) {}
+  Throws(const Throws& rhs) : v_(rhs.v_) {
+    if (sThrows)
+      throw 1;
+  }
+  Throws(Throws&& rhs) : v_(rhs.v_) {
+    if (sThrows)
+      throw 1;
+  }
+  Throws& operator=(const Throws& rhs) {
+    v_ = rhs.v_;
+    return *this;
+  }
+  Throws& operator=(Throws&& rhs) {
+    v_ = rhs.v_;
+    return *this;
+  }
+  int v_;
 
-    static bool sThrows;
-    };
+  static bool sThrows;
+};
 
 bool Throws::sThrows = false;
 #endif
 
 template <class C>
-C
-make(int size, int start = 0 )
-{
-    const int b = 4096 / sizeof(int);
-    int init = 0;
-    if (start > 0)
-    {
-        init = (start+1) / b + ((start+1) % b != 0);
-        init *= b;
-        --init;
-    }
-    C c(init, 0);
-    for (int i = 0; i < init-start; ++i)
-        c.pop_back();
-    for (int i = 0; i < size; ++i)
-        c.push_back(i);
-    for (int i = 0; i < start; ++i)
-        c.pop_front();
-    return c;
+C make(int size, int start = 0) {
+  const int b = 4096 / sizeof(int);
+  int init    = 0;
+  if (start > 0) {
+    init = (start + 1) / b + ((start + 1) % b != 0);
+    init *= b;
+    --init;
+  }
+  C c(init, 0);
+  for (int i = 0; i < init - start; ++i)
+    c.pop_back();
+  for (int i = 0; i < size; ++i)
+    c.push_back(i);
+  for (int i = 0; i < start; ++i)
+    c.pop_front();
+  return c;
 }
 
 template <class C>
-void
-test(int P, C& c1)
-{
-    typedef typename C::iterator I;
-    assert(static_cast<std::size_t>(P) < c1.size());
-    std::size_t c1_osize = c1.size();
-    I i = c1.erase(c1.cbegin() + P);
-    assert(i == c1.begin() + P);
-    assert(c1.size() == c1_osize - 1);
-    assert(static_cast<std::size_t>(std::distance(c1.begin(), c1.end())) == c1.size());
-    i = c1.begin();
-    int j = 0;
-    for (; j < P; ++j, ++i)
-        assert(*i == j);
-    for (++j; static_cast<std::size_t>(j) < c1_osize; ++j, ++i)
-        assert(*i == j);
+void test(int P, C& c1) {
+  typedef typename C::iterator I;
+  assert(static_cast<std::size_t>(P) < c1.size());
+  std::size_t c1_osize = c1.size();
+  I i                  = c1.erase(c1.cbegin() + P);
+  assert(i == c1.begin() + P);
+  assert(c1.size() == c1_osize - 1);
+  assert(static_cast<std::size_t>(std::distance(c1.begin(), c1.end())) == c1.size());
+  i     = c1.begin();
+  int j = 0;
+  for (; j < P; ++j, ++i)
+    assert(*i == j);
+  for (++j; static_cast<std::size_t>(j) < c1_osize; ++j, ++i)
+    assert(*i == j);
 }
 
 template <class C>
-void
-testN(int start, int N)
-{
-    int pstep = std::max(N / std::max(std::min(N, 10), 1), 1);
-    for (int p = 0; p < N; p += pstep)
-    {
-        C c1 = make<C>(N, start);
-        test(p, c1);
-    }
+void testN(int start, int N) {
+  int pstep = std::max(N / std::max(std::min(N, 10), 1), 1);
+  for (int p = 0; p < N; p += pstep) {
+    C c1 = make<C>(N, start);
+    test(p, c1);
+  }
 }
 
-int main(int, char**)
-{
-    {
-    int rng[] = {0, 1, 2, 3, 1023, 1024, 1025, 2047, 2048, 2049};
-    const int N = sizeof(rng)/sizeof(rng[0]);
+int main(int, char**) {
+  {
+    int rng[]   = {0, 1, 2, 3, 1023, 1024, 1025, 2047, 2048, 2049};
+    const int N = sizeof(rng) / sizeof(rng[0]);
     for (int i = 0; i < N; ++i)
-        for (int j = 0; j < N; ++j)
-            testN<std::deque<int> >(rng[i], rng[j]);
-    }
+      for (int j = 0; j < N; ++j)
+        testN<std::deque<int> >(rng[i], rng[j]);
+  }
 #if TEST_STD_VER >= 11
-    {
-    int rng[] = {0, 1, 2, 3, 1023, 1024, 1025, 2047, 2048, 2049};
-    const int N = sizeof(rng)/sizeof(rng[0]);
+  {
+    int rng[]   = {0, 1, 2, 3, 1023, 1024, 1025, 2047, 2048, 2049};
+    const int N = sizeof(rng) / sizeof(rng[0]);
     for (int i = 0; i < N; ++i)
-        for (int j = 0; j < N; ++j)
-            testN<std::deque<int, min_allocator<int>> >(rng[i], rng[j]);
-    }
+      for (int j = 0; j < N; ++j)
+        testN<std::deque<int, min_allocator<int>> >(rng[i], rng[j]);
+  }
 #endif
 
 #ifndef TEST_HAS_NO_EXCEPTIONS
-// Test for LWG2953:
-// Throws: Nothing unless an exception is thrown by the assignment operator of T.
-// (which includes move assignment)
-    {
+  // Test for LWG2953:
+  // Throws: Nothing unless an exception is thrown by the assignment operator of T.
+  // (which includes move assignment)
+  {
     Throws arr[] = {1, 2, 3};
-    std::deque<Throws> v(arr, arr+3);
+    std::deque<Throws> v(arr, arr + 3);
     Throws::sThrows = true;
     v.erase(v.begin());
     v.erase(--v.end());
     v.erase(v.begin());
     assert(v.size() == 0);
     LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(v));
-    }
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/sequences/deque/deque.modifiers/erase_iter_iter.invalidation.pass.cpp b/libcxx/test/std/containers/sequences/deque/deque.modifiers/erase_iter_iter.invalidation.pass.cpp
index 2920b7d83a1a6..320d4771a2fc7 100644
--- a/libcxx/test/std/containers/sequences/deque/deque.modifiers/erase_iter_iter.invalidation.pass.cpp
+++ b/libcxx/test/std/containers/sequences/deque/deque.modifiers/erase_iter_iter.invalidation.pass.cpp
@@ -13,7 +13,6 @@
 //  Erasing items from the beginning or the end of a deque shall not invalidate iterators
 //  to items that were not erased.
 
-
 #include "asan_testing.h"
 #include <deque>
 #include <cstdint>
@@ -22,63 +21,57 @@
 #include "test_macros.h"
 
 template <typename C>
-void del_at_start(C c, std::size_t num)
-{
-    typename C::iterator first = c.begin();
-    typename C::iterator last  = first + num;
-    typename C::iterator it1 = last;
-    typename C::iterator it2 = c.end() - 1;
-
-    c.erase (first, last);
-
-    typename C::iterator it3 = c.begin();
-    typename C::iterator it4 = c.end() - 1;
-    assert(  it1 ==   it3);
-    assert( *it1 ==  *it3);
-    assert(&*it1 == &*it3);
-    assert(  it2 ==   it4);
-    assert( *it2 ==  *it4);
-    assert(&*it2 == &*it4);
-    LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c));
+void del_at_start(C c, std::size_t num) {
+  typename C::iterator first = c.begin();
+  typename C::iterator last  = first + num;
+  typename C::iterator it1   = last;
+  typename C::iterator it2   = c.end() - 1;
+
+  c.erase(first, last);
+
+  typename C::iterator it3 = c.begin();
+  typename C::iterator it4 = c.end() - 1;
+  assert(it1 == it3);
+  assert(*it1 == *it3);
+  assert(&*it1 == &*it3);
+  assert(it2 == it4);
+  assert(*it2 == *it4);
+  assert(&*it2 == &*it4);
+  LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c));
 }
 
 template <typename C>
-void del_at_end(C c, std::size_t num)
-{
-    typename C::iterator last  = c.end();
-    typename C::iterator first = last - num;
-    typename C::iterator it1 = c.begin();
-    typename C::iterator it2 = first - 1;
-
-    c.erase (first, last);
-
-    typename C::iterator it3 = c.begin();
-    typename C::iterator it4 = c.end() - 1;
-    assert(  it1 ==   it3);
-    assert( *it1 ==  *it3);
-    assert(&*it1 == &*it3);
-    assert(  it2 ==   it4);
-    assert( *it2 ==  *it4);
-    assert(&*it2 == &*it4);
-    LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c));
+void del_at_end(C c, std::size_t num) {
+  typename C::iterator last  = c.end();
+  typename C::iterator first = last - num;
+  typename C::iterator it1   = c.begin();
+  typename C::iterator it2   = first - 1;
+
+  c.erase(first, last);
+
+  typename C::iterator it3 = c.begin();
+  typename C::iterator it4 = c.end() - 1;
+  assert(it1 == it3);
+  assert(*it1 == *it3);
+  assert(&*it1 == &*it3);
+  assert(it2 == it4);
+  assert(*it2 == *it4);
+  assert(&*it2 == &*it4);
+  LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c));
 }
 
+int main(int, char**) {
+  std::deque<int> queue;
+  for (int i = 0; i < 20; ++i)
+    queue.push_back(i);
 
-int main(int, char**)
-{
-    std::deque<int> queue;
-    for (int i = 0; i < 20; ++i)
-        queue.push_back(i);
-
-    while (queue.size() > 1)
-    {
-        for (std::size_t i = 1; i < queue.size(); ++i)
-        {
-            del_at_start(queue, i);
-            del_at_end  (queue, i);
-        }
-        queue.pop_back();
+  while (queue.size() > 1) {
+    for (std::size_t i = 1; i < queue.size(); ++i) {
+      del_at_start(queue, i);
+      del_at_end(queue, i);
     }
+    queue.pop_back();
+  }
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/deque/deque.modifiers/erase_iter_iter.pass.cpp b/libcxx/test/std/containers/sequences/deque/deque.modifiers/erase_iter_iter.pass.cpp
index 51e774cb59194..1eaa7a6c72a57 100644
--- a/libcxx/test/std/containers/sequences/deque/deque.modifiers/erase_iter_iter.pass.cpp
+++ b/libcxx/test/std/containers/sequences/deque/deque.modifiers/erase_iter_iter.pass.cpp
@@ -24,104 +24,105 @@
 
 #ifndef TEST_HAS_NO_EXCEPTIONS
 struct Throws {
-    Throws() : v_(0) {}
-    Throws(int v) : v_(v) {}
-    Throws(const Throws  &rhs) : v_(rhs.v_) { if (sThrows) throw 1; }
-    Throws(      Throws &&rhs) : v_(rhs.v_) { if (sThrows) throw 1; }
-    Throws& operator=(const Throws  &rhs) { v_ = rhs.v_; return *this; }
-    Throws& operator=(      Throws &&rhs) { v_ = rhs.v_; return *this; }
-    int v_;
+  Throws() : v_(0) {}
+  Throws(int v) : v_(v) {}
+  Throws(const Throws& rhs) : v_(rhs.v_) {
+    if (sThrows)
+      throw 1;
+  }
+  Throws(Throws&& rhs) : v_(rhs.v_) {
+    if (sThrows)
+      throw 1;
+  }
+  Throws& operator=(const Throws& rhs) {
+    v_ = rhs.v_;
+    return *this;
+  }
+  Throws& operator=(Throws&& rhs) {
+    v_ = rhs.v_;
+    return *this;
+  }
+  int v_;
 
-    static bool sThrows;
-    };
+  static bool sThrows;
+};
 
 bool Throws::sThrows = false;
 #endif
 
-
 template <class C>
-C
-make(int size, int start = 0 )
-{
-    const int b = 4096 / sizeof(int);
-    int init = 0;
-    if (start > 0)
-    {
-        init = (start+1) / b + ((start+1) % b != 0);
-        init *= b;
-        --init;
-    }
-    C c(init, 0);
-    for (int i = 0; i < init-start; ++i)
-        c.pop_back();
-    for (int i = 0; i < size; ++i)
-        c.push_back(i);
-    for (int i = 0; i < start; ++i)
-        c.pop_front();
-    return c;
+C make(int size, int start = 0) {
+  const int b = 4096 / sizeof(int);
+  int init    = 0;
+  if (start > 0) {
+    init = (start + 1) / b + ((start + 1) % b != 0);
+    init *= b;
+    --init;
+  }
+  C c(init, 0);
+  for (int i = 0; i < init - start; ++i)
+    c.pop_back();
+  for (int i = 0; i < size; ++i)
+    c.push_back(i);
+  for (int i = 0; i < start; ++i)
+    c.pop_front();
+  return c;
 }
 
 template <class C>
-void
-test(int P, C& c1, int size)
-{
-    typedef typename C::iterator I;
-    assert(static_cast<std::size_t>(P + size) <= c1.size());
-    std::size_t c1_osize = c1.size();
-    I i = c1.erase(c1.cbegin() + P, c1.cbegin() + (P + size));
-    assert(i == c1.begin() + P);
-    assert(c1.size() == c1_osize - size);
-    assert(static_cast<std::size_t>(std::distance(c1.begin(), c1.end())) == c1.size());
-    LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c1));
-    i = c1.begin();
-    int j = 0;
-    for (; j < P; ++j, ++i)
-        assert(*i == j);
-    for (j += size; static_cast<std::size_t>(j) < c1_osize; ++j, ++i)
-        assert(*i == j);
+void test(int P, C& c1, int size) {
+  typedef typename C::iterator I;
+  assert(static_cast<std::size_t>(P + size) <= c1.size());
+  std::size_t c1_osize = c1.size();
+  I i                  = c1.erase(c1.cbegin() + P, c1.cbegin() + (P + size));
+  assert(i == c1.begin() + P);
+  assert(c1.size() == c1_osize - size);
+  assert(static_cast<std::size_t>(std::distance(c1.begin(), c1.end())) == c1.size());
+  LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c1));
+  i     = c1.begin();
+  int j = 0;
+  for (; j < P; ++j, ++i)
+    assert(*i == j);
+  for (j += size; static_cast<std::size_t>(j) < c1_osize; ++j, ++i)
+    assert(*i == j);
 }
 
 template <class C>
-void
-testN(int start, int N)
-{
-    int pstep = std::max(N / std::max(std::min(N, 10), 1), 1);
-    for (int p = 0; p <= N; p += pstep)
-    {
-        int sstep = std::max((N - p) / std::max(std::min(N - p, 10), 1), 1);
-        for (int s = 0; s <= N - p; s += sstep)
-        {
-            C c1 = make<C>(N, start);
-            test(p, c1, s);
-        }
+void testN(int start, int N) {
+  int pstep = std::max(N / std::max(std::min(N, 10), 1), 1);
+  for (int p = 0; p <= N; p += pstep) {
+    int sstep = std::max((N - p) / std::max(std::min(N - p, 10), 1), 1);
+    for (int s = 0; s <= N - p; s += sstep) {
+      C c1 = make<C>(N, start);
+      test(p, c1, s);
     }
+  }
 }
 
-int main(int, char**)
-{
-    {
-    int rng[] = {0, 1, 2, 3, 1023, 1024, 1025, 2047, 2048, 2049};
-    const int N = sizeof(rng)/sizeof(rng[0]);
+int main(int, char**) {
+  {
+    int rng[]   = {0, 1, 2, 3, 1023, 1024, 1025, 2047, 2048, 2049};
+    const int N = sizeof(rng) / sizeof(rng[0]);
     for (int i = 0; i < N; ++i)
-        for (int j = 0; j < N; ++j)
-            testN<std::deque<int> >(rng[i], rng[j]);
-    }
+      for (int j = 0; j < N; ++j)
+        testN<std::deque<int> >(rng[i], rng[j]);
+  }
 #if TEST_STD_VER >= 11
-    {
-    int rng[] = {0, 1, 2, 3, 1023, 1024, 1025, 2047, 2048, 2049};
-    const int N = sizeof(rng)/sizeof(rng[0]);
+  {
+    int rng[]   = {0, 1, 2, 3, 1023, 1024, 1025, 2047, 2048, 2049};
+    const int N = sizeof(rng) / sizeof(rng[0]);
     for (int i = 0; i < N; ++i)
-        for (int j = 0; j < N; ++j)
-            testN<std::deque<int, min_allocator<int>> >(rng[i], rng[j]);
-    }
+      for (int j = 0; j < N; ++j)
+        testN<std::deque<int, min_allocator<int>> >(rng[i], rng[j]);
+  }
 #endif
 #ifndef TEST_HAS_NO_EXCEPTIONS
-// Test for LWG2953:
-// Throws: Nothing unless an exception is thrown by the assignment operator of T.
-// (which includes move assignment)
-    {
+  // Test for LWG2953:
+  // Throws: Nothing unless an exception is thrown by the assignment operator of T.
+  // (which includes move assignment)
+  {
     Throws arr[] = {1, 2, 3};
-    std::deque<Throws> v(arr, arr+3);
+    std::deque<Throws> v(arr, arr + 3);
     Throws::sThrows = true;
     v.erase(v.begin(), --v.end());
     assert(v.size() == 1);
@@ -129,7 +130,7 @@ int main(int, char**)
     v.erase(v.begin(), v.end());
     assert(v.size() == 0);
     LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(v));
-    }
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/sequences/deque/deque.modifiers/insert_iter_initializer_list.pass.cpp b/libcxx/test/std/containers/sequences/deque/deque.modifiers/insert_iter_initializer_list.pass.cpp
index 080a60cec1a3c..117ce63b93d7e 100644
--- a/libcxx/test/std/containers/sequences/deque/deque.modifiers/insert_iter_initializer_list.pass.cpp
+++ b/libcxx/test/std/containers/sequences/deque/deque.modifiers/insert_iter_initializer_list.pass.cpp
@@ -19,9 +19,8 @@
 #include "test_macros.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
+int main(int, char**) {
+  {
     std::deque<int> d(10, 1);
     std::deque<int>::iterator i = d.insert(d.cbegin() + 2, {3, 4, 5, 6});
     assert(d.size() == 14);
@@ -41,8 +40,8 @@ int main(int, char**)
     assert(d[12] == 1);
     assert(d[13] == 1);
     LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(d));
-    }
-    {
+  }
+  {
     std::deque<int, min_allocator<int>> d(10, 1);
     std::deque<int, min_allocator<int>>::iterator i = d.insert(d.cbegin() + 2, {3, 4, 5, 6});
     assert(d.size() == 14);
@@ -62,7 +61,7 @@ int main(int, char**)
     assert(d[12] == 1);
     assert(d[13] == 1);
     LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(d));
-    }
+  }
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/deque/deque.modifiers/insert_iter_iter.pass.cpp b/libcxx/test/std/containers/sequences/deque/deque.modifiers/insert_iter_iter.pass.cpp
index b99c2b87cc5fe..3941c1e8bc6c5 100644
--- a/libcxx/test/std/containers/sequences/deque/deque.modifiers/insert_iter_iter.pass.cpp
+++ b/libcxx/test/std/containers/sequences/deque/deque.modifiers/insert_iter_iter.pass.cpp
@@ -31,37 +31,32 @@
 #include "min_allocator.h"
 
 template <class C>
-C
-make(int size, int start = 0 )
-{
-    const int b = 4096 / sizeof(int);
-    int init = 0;
-    if (start > 0)
-    {
-        init = (start+1) / b + ((start+1) % b != 0);
-        init *= b;
-        --init;
-    }
-    C c(init, 0);
-    for (int i = 0; i < init-start; ++i)
-        c.pop_back();
-    for (int i = 0; i < size; ++i)
-        c.push_back(i);
-    for (int i = 0; i < start; ++i)
-        c.pop_front();
-    return c;
+C make(int size, int start = 0) {
+  const int b = 4096 / sizeof(int);
+  int init    = 0;
+  if (start > 0) {
+    init = (start + 1) / b + ((start + 1) % b != 0);
+    init *= b;
+    --init;
+  }
+  C c(init, 0);
+  for (int i = 0; i < init - start; ++i)
+    c.pop_back();
+  for (int i = 0; i < size; ++i)
+    c.push_back(i);
+  for (int i = 0; i < start; ++i)
+    c.pop_front();
+  return c;
 }
 
 template <class C>
-void
-test(int P, const C& c0, const C& c2)
-{
-    {
+void test(int P, const C& c0, const C& c2) {
+  {
     typedef typename C::const_iterator CI;
     typedef cpp17_input_iterator<CI> BCI;
-    C c1 = c0;
+    C c1                 = c0;
     std::size_t c1_osize = c1.size();
-    CI i = c1.insert(c1.begin() + P, BCI(c2.begin()), BCI(c2.end()));
+    CI i                 = c1.insert(c1.begin() + P, BCI(c2.begin()), BCI(c2.end()));
     assert(i == c1.begin() + P);
     assert(c1.size() == c1_osize + c2.size());
     assert(static_cast<std::size_t>(std::distance(c1.begin(), c1.end())) == c1.size());
@@ -69,242 +64,211 @@ test(int P, const C& c0, const C& c2)
     LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c2));
     i = c1.begin();
     for (int j = 0; j < P; ++j, ++i)
-        assert(*i == j);
+      assert(*i == j);
     for (int j = 0; static_cast<std::size_t>(j) < c2.size(); ++j, ++i)
-        assert(*i == j);
+      assert(*i == j);
     for (int j = P; static_cast<std::size_t>(j) < c1_osize; ++j, ++i)
-        assert(*i == j);
-    }
-    {
+      assert(*i == j);
+  }
+  {
     typedef typename C::const_iterator CI;
     typedef forward_iterator<CI> BCI;
-    C c1 = c0;
+    C c1                 = c0;
     std::size_t c1_osize = c1.size();
-    CI i = c1.insert(c1.begin() + P, BCI(c2.begin()), BCI(c2.end()));
+    CI i                 = c1.insert(c1.begin() + P, BCI(c2.begin()), BCI(c2.end()));
     assert(i == c1.begin() + P);
     assert(c1.size() == c1_osize + c2.size());
     assert(static_cast<std::size_t>(std::distance(c1.begin(), c1.end())) == c1.size());
     i = c1.begin();
     for (int j = 0; j < P; ++j, ++i)
-        assert(*i == j);
+      assert(*i == j);
     for (int j = 0; static_cast<std::size_t>(j) < c2.size(); ++j, ++i)
-        assert(*i == j);
+      assert(*i == j);
     for (int j = P; static_cast<std::size_t>(j) < c1_osize; ++j, ++i)
-        assert(*i == j);
-    }
-    {
+      assert(*i == j);
+  }
+  {
     typedef typename C::const_iterator CI;
     typedef bidirectional_iterator<CI> BCI;
-    C c1 = c0;
+    C c1                 = c0;
     std::size_t c1_osize = c1.size();
-    CI i = c1.insert(c1.begin() + P, BCI(c2.begin()), BCI(c2.end()));
+    CI i                 = c1.insert(c1.begin() + P, BCI(c2.begin()), BCI(c2.end()));
     assert(i == c1.begin() + P);
     assert(c1.size() == c1_osize + c2.size());
     assert(static_cast<std::size_t>(std::distance(c1.begin(), c1.end())) == c1.size());
     i = c1.begin();
     for (int j = 0; j < P; ++j, ++i)
-        assert(*i == j);
+      assert(*i == j);
     for (int j = 0; static_cast<std::size_t>(j) < c2.size(); ++j, ++i)
-        assert(*i == j);
+      assert(*i == j);
     for (int j = P; static_cast<std::size_t>(j) < c1_osize; ++j, ++i)
-        assert(*i == j);
-    }
+      assert(*i == j);
+  }
 }
 
 template <class C>
-void
-testN(int start, int N, int M)
-{
-    for (int i = 0; i <= 3; ++i)
-    {
-        if (0 <= i && i <= N)
-        {
-            C c1 = make<C>(N, start);
-            C c2 = make<C>(M);
-            test(i, c1, c2);
-        }
+void testN(int start, int N, int M) {
+  for (int i = 0; i <= 3; ++i) {
+    if (0 <= i && i <= N) {
+      C c1 = make<C>(N, start);
+      C c2 = make<C>(M);
+      test(i, c1, c2);
     }
-    for (int i = M-1; i <= M+1; ++i)
-    {
-        if (0 <= i && i <= N)
-        {
-            C c1 = make<C>(N, start);
-            C c2 = make<C>(M);
-            test(i, c1, c2);
-        }
+  }
+  for (int i = M - 1; i <= M + 1; ++i) {
+    if (0 <= i && i <= N) {
+      C c1 = make<C>(N, start);
+      C c2 = make<C>(M);
+      test(i, c1, c2);
     }
-    for (int i = N/2-1; i <= N/2+1; ++i)
-    {
-        if (0 <= i && i <= N)
-        {
-            C c1 = make<C>(N, start);
-            C c2 = make<C>(M);
-            test(i, c1, c2);
-        }
+  }
+  for (int i = N / 2 - 1; i <= N / 2 + 1; ++i) {
+    if (0 <= i && i <= N) {
+      C c1 = make<C>(N, start);
+      C c2 = make<C>(M);
+      test(i, c1, c2);
     }
-    for (int i = N - M - 1; i <= N - M + 1; ++i)
-    {
-        if (0 <= i && i <= N)
-        {
-            C c1 = make<C>(N, start);
-            C c2 = make<C>(M);
-            test(i, c1, c2);
-        }
+  }
+  for (int i = N - M - 1; i <= N - M + 1; ++i) {
+    if (0 <= i && i <= N) {
+      C c1 = make<C>(N, start);
+      C c2 = make<C>(M);
+      test(i, c1, c2);
     }
-    for (int i = N - M - 1; i <= N - M + 1; ++i)
-    {
-        if (0 <= i && i <= N)
-        {
-            C c1 = make<C>(N, start);
-            C c2 = make<C>(M);
-            test(i, c1, c2);
-        }
+  }
+  for (int i = N - M - 1; i <= N - M + 1; ++i) {
+    if (0 <= i && i <= N) {
+      C c1 = make<C>(N, start);
+      C c2 = make<C>(M);
+      test(i, c1, c2);
     }
-    for (int i = N - 3; i <= N; ++i)
-    {
-        if (0 <= i && i <= N)
-        {
-            C c1 = make<C>(N, start);
-            C c2 = make<C>(M);
-            test(i, c1, c2);
-        }
+  }
+  for (int i = N - 3; i <= N; ++i) {
+    if (0 <= i && i <= N) {
+      C c1 = make<C>(N, start);
+      C c2 = make<C>(M);
+      test(i, c1, c2);
     }
+  }
 }
 
 template <class C>
-void
-testI(int P, C& c1, const C& c2)
-{
-    typedef typename C::const_iterator CI;
-    typedef cpp17_input_iterator<CI> ICI;
-    std::size_t c1_osize = c1.size();
-    CI i = c1.insert(c1.begin() + P, ICI(c2.begin()), ICI(c2.end()));
-    assert(i == c1.begin() + P);
-    assert(c1.size() == c1_osize + c2.size());
-    assert(static_cast<std::size_t>(std::distance(c1.begin(), c1.end())) == c1.size());
-    LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c1));
-    LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c2));
-    i = c1.begin();
-    for (int j = 0; j < P; ++j, ++i)
-        assert(*i == j);
-    for (int j = 0; static_cast<std::size_t>(j) < c2.size(); ++j, ++i)
-        assert(*i == j);
-    for (int j = P; static_cast<std::size_t>(j) < c1_osize; ++j, ++i)
-        assert(*i == j);
+void testI(int P, C& c1, const C& c2) {
+  typedef typename C::const_iterator CI;
+  typedef cpp17_input_iterator<CI> ICI;
+  std::size_t c1_osize = c1.size();
+  CI i                 = c1.insert(c1.begin() + P, ICI(c2.begin()), ICI(c2.end()));
+  assert(i == c1.begin() + P);
+  assert(c1.size() == c1_osize + c2.size());
+  assert(static_cast<std::size_t>(std::distance(c1.begin(), c1.end())) == c1.size());
+  LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c1));
+  LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c2));
+  i = c1.begin();
+  for (int j = 0; j < P; ++j, ++i)
+    assert(*i == j);
+  for (int j = 0; static_cast<std::size_t>(j) < c2.size(); ++j, ++i)
+    assert(*i == j);
+  for (int j = P; static_cast<std::size_t>(j) < c1_osize; ++j, ++i)
+    assert(*i == j);
 }
 
 template <class C>
-void
-testNI(int start, int N, int M)
-{
-    for (int i = 0; i <= 3; ++i)
-    {
-        if (0 <= i && i <= N)
-        {
-            C c1 = make<C>(N, start);
-            C c2 = make<C>(M);
-            testI(i, c1, c2);
-        }
+void testNI(int start, int N, int M) {
+  for (int i = 0; i <= 3; ++i) {
+    if (0 <= i && i <= N) {
+      C c1 = make<C>(N, start);
+      C c2 = make<C>(M);
+      testI(i, c1, c2);
     }
-    for (int i = M-1; i <= M+1; ++i)
-    {
-        if (0 <= i && i <= N)
-        {
-            C c1 = make<C>(N, start);
-            C c2 = make<C>(M);
-            testI(i, c1, c2);
-        }
+  }
+  for (int i = M - 1; i <= M + 1; ++i) {
+    if (0 <= i && i <= N) {
+      C c1 = make<C>(N, start);
+      C c2 = make<C>(M);
+      testI(i, c1, c2);
     }
-    for (int i = N/2-1; i <= N/2+1; ++i)
-    {
-        if (0 <= i && i <= N)
-        {
-            C c1 = make<C>(N, start);
-            C c2 = make<C>(M);
-            testI(i, c1, c2);
-        }
+  }
+  for (int i = N / 2 - 1; i <= N / 2 + 1; ++i) {
+    if (0 <= i && i <= N) {
+      C c1 = make<C>(N, start);
+      C c2 = make<C>(M);
+      testI(i, c1, c2);
     }
-    for (int i = N - M - 1; i <= N - M + 1; ++i)
-    {
-        if (0 <= i && i <= N)
-        {
-            C c1 = make<C>(N, start);
-            C c2 = make<C>(M);
-            testI(i, c1, c2);
-        }
+  }
+  for (int i = N - M - 1; i <= N - M + 1; ++i) {
+    if (0 <= i && i <= N) {
+      C c1 = make<C>(N, start);
+      C c2 = make<C>(M);
+      testI(i, c1, c2);
     }
-    for (int i = N - 3; i <= N; ++i)
-    {
-        if (0 <= i && i <= N)
-        {
-            C c1 = make<C>(N, start);
-            C c2 = make<C>(M);
-            testI(i, c1, c2);
-        }
+  }
+  for (int i = N - 3; i <= N; ++i) {
+    if (0 <= i && i <= N) {
+      C c1 = make<C>(N, start);
+      C c2 = make<C>(M);
+      testI(i, c1, c2);
     }
+  }
 }
 
 template <class C>
-void
-test_move()
-{
+void test_move() {
 #if TEST_STD_VER >= 11
-    C c;
-    typedef typename C::const_iterator CI;
-    {
-        MoveOnly mo(0);
-        typedef MoveOnly* I;
-        c.insert(c.end(), std::move_iterator<I>(&mo), std::move_iterator<I>(&mo+1));
-    }
-    int j = 0;
-    for (CI i = c.begin(); i != c.end(); ++i, (void) ++j)
-        assert(*i == MoveOnly(j));
-    {
-        MoveOnly mo(1);
-        typedef cpp17_input_iterator<MoveOnly*> I;
-        c.insert(c.end(), std::move_iterator<I>(I(&mo)), std::move_iterator<I>(I(&mo+1)));
-    }
-    j = 0;
-    for (CI i = c.begin(); i != c.end(); ++i, (void) ++j)
-        assert(*i == MoveOnly(j));
+  C c;
+  typedef typename C::const_iterator CI;
+  {
+    MoveOnly mo(0);
+    typedef MoveOnly* I;
+    c.insert(c.end(), std::move_iterator<I>(&mo), std::move_iterator<I>(&mo + 1));
+  }
+  int j = 0;
+  for (CI i = c.begin(); i != c.end(); ++i, (void)++j)
+    assert(*i == MoveOnly(j));
+  {
+    MoveOnly mo(1);
+    typedef cpp17_input_iterator<MoveOnly*> I;
+    c.insert(c.end(), std::move_iterator<I>(I(&mo)), std::move_iterator<I>(I(&mo + 1)));
+  }
+  j = 0;
+  for (CI i = c.begin(); i != c.end(); ++i, (void)++j)
+    assert(*i == MoveOnly(j));
 #endif
 }
 
-int main(int, char**)
-{
-    {
-    int rng[] = {0, 1, 2, 3, 1023, 1024, 1025, 2047, 2048, 2049};
-    const int N = sizeof(rng)/sizeof(rng[0]);
+int main(int, char**) {
+  {
+    int rng[]   = {0, 1, 2, 3, 1023, 1024, 1025, 2047, 2048, 2049};
+    const int N = sizeof(rng) / sizeof(rng[0]);
     for (int i = 0; i < N; ++i)
-        for (int j = 0; j < N; ++j)
-            for (int k = 0; k < N; ++k)
-                testN<std::deque<int> >(rng[i], rng[j], rng[k]);
+      for (int j = 0; j < N; ++j)
+        for (int k = 0; k < N; ++k)
+          testN<std::deque<int> >(rng[i], rng[j], rng[k]);
     testNI<std::deque<int> >(1500, 2000, 1000);
 #if TEST_STD_VER >= 11
     test_move<std::deque<MoveOnly, limited_allocator<MoveOnly, 2000> > >();
 #endif
-    }
+  }
 #if TEST_STD_VER >= 11
-    {
-    int rng[] = {0, 1, 2, 3, 1023, 1024, 1025, 2047, 2048, 2049};
-    const int N = sizeof(rng)/sizeof(rng[0]);
+  {
+    int rng[]   = {0, 1, 2, 3, 1023, 1024, 1025, 2047, 2048, 2049};
+    const int N = sizeof(rng) / sizeof(rng[0]);
     for (int i = 0; i < N; ++i)
-        for (int j = 0; j < N; ++j)
-            for (int k = 0; k < N; ++k)
-                testN<std::deque<int, min_allocator<int>> >(rng[i], rng[j], rng[k]);
+      for (int j = 0; j < N; ++j)
+        for (int k = 0; k < N; ++k)
+          testN<std::deque<int, min_allocator<int>> >(rng[i], rng[j], rng[k]);
     testNI<std::deque<int> >(1500, 2000, 1000);
     test_move<std::deque<MoveOnly, min_allocator<MoveOnly> > >();
-    }
-    {
-    int rng[] = {0, 1, 2, 3, 1023, 1024, 1025, 2047, 2048, 2049};
-    const int N = sizeof(rng)/sizeof(rng[0]);
+  }
+  {
+    int rng[]   = {0, 1, 2, 3, 1023, 1024, 1025, 2047, 2048, 2049};
+    const int N = sizeof(rng) / sizeof(rng[0]);
     for (int i = 0; i < N; ++i)
-        for (int j = 0; j < N; ++j)
-            for (int k = 0; k < N; ++k)
-                testN<std::deque<int, safe_allocator<int>> >(rng[i], rng[j], rng[k]);
+      for (int j = 0; j < N; ++j)
+        for (int k = 0; k < N; ++k)
+          testN<std::deque<int, safe_allocator<int>> >(rng[i], rng[j], rng[k]);
     testNI<std::deque<int> >(1500, 2000, 1000);
     test_move<std::deque<MoveOnly, safe_allocator<MoveOnly> > >();
-    }
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/sequences/deque/deque.modifiers/insert_rvalue.pass.cpp b/libcxx/test/std/containers/sequences/deque/deque.modifiers/insert_rvalue.pass.cpp
index 24092029a7453..ff4654cf0ccb3 100644
--- a/libcxx/test/std/containers/sequences/deque/deque.modifiers/insert_rvalue.pass.cpp
+++ b/libcxx/test/std/containers/sequences/deque/deque.modifiers/insert_rvalue.pass.cpp
@@ -21,102 +21,87 @@
 #include "MoveOnly.h"
 #include "min_allocator.h"
 
-
 template <class C>
-C
-make(int size, int start = 0 )
-{
-    const int b = 4096 / sizeof(int);
-    int init = 0;
-    if (start > 0)
-    {
-        init = (start+1) / b + ((start+1) % b != 0);
-        init *= b;
-        --init;
-    }
-    C c(init);
-    for (int i = 0; i < init-start; ++i)
-        c.pop_back();
-    for (int i = 0; i < size; ++i)
-        c.push_back(MoveOnly(i));
-    for (int i = 0; i < start; ++i)
-        c.pop_front();
-    return c;
+C make(int size, int start = 0) {
+  const int b = 4096 / sizeof(int);
+  int init    = 0;
+  if (start > 0) {
+    init = (start + 1) / b + ((start + 1) % b != 0);
+    init *= b;
+    --init;
+  }
+  C c(init);
+  for (int i = 0; i < init - start; ++i)
+    c.pop_back();
+  for (int i = 0; i < size; ++i)
+    c.push_back(MoveOnly(i));
+  for (int i = 0; i < start; ++i)
+    c.pop_front();
+  return c;
 }
 
 template <class C>
-void
-test(int P, C& c1, int x)
-{
-    typedef typename C::const_iterator CI;
-    std::size_t c1_osize = c1.size();
-    CI i = c1.insert(c1.begin() + P, MoveOnly(x));
-    assert(i == c1.begin() + P);
-    assert(c1.size() == c1_osize + 1);
-    assert(static_cast<std::size_t>(std::distance(c1.begin(), c1.end())) == c1.size());
-    LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c1));
-    i = c1.begin();
-    for (int j = 0; j < P; ++j, (void) ++i)
-        assert(*i == MoveOnly(j));
-    assert(*i == MoveOnly(x));
-    ++i;
-    for (int j = P; static_cast<std::size_t>(j) < c1_osize; ++j, (void) ++i)
-        assert(*i == MoveOnly(j));
+void test(int P, C& c1, int x) {
+  typedef typename C::const_iterator CI;
+  std::size_t c1_osize = c1.size();
+  CI i                 = c1.insert(c1.begin() + P, MoveOnly(x));
+  assert(i == c1.begin() + P);
+  assert(c1.size() == c1_osize + 1);
+  assert(static_cast<std::size_t>(std::distance(c1.begin(), c1.end())) == c1.size());
+  LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c1));
+  i = c1.begin();
+  for (int j = 0; j < P; ++j, (void)++i)
+    assert(*i == MoveOnly(j));
+  assert(*i == MoveOnly(x));
+  ++i;
+  for (int j = P; static_cast<std::size_t>(j) < c1_osize; ++j, (void)++i)
+    assert(*i == MoveOnly(j));
 }
 
 template <class C>
-void
-testN(int start, int N)
-{
-    for (int i = 0; i <= 3; ++i)
-    {
-        if (0 <= i && i <= N)
-        {
-            C c1 = make<C>(N, start);
-            test(i, c1, -10);
-        }
+void testN(int start, int N) {
+  for (int i = 0; i <= 3; ++i) {
+    if (0 <= i && i <= N) {
+      C c1 = make<C>(N, start);
+      test(i, c1, -10);
     }
-    for (int i = N/2-1; i <= N/2+1; ++i)
-    {
-        if (0 <= i && i <= N)
-        {
-            C c1 = make<C>(N, start);
-            test(i, c1, -10);
-        }
+  }
+  for (int i = N / 2 - 1; i <= N / 2 + 1; ++i) {
+    if (0 <= i && i <= N) {
+      C c1 = make<C>(N, start);
+      test(i, c1, -10);
     }
-    for (int i = N - 3; i <= N; ++i)
-    {
-        if (0 <= i && i <= N)
-        {
-            C c1 = make<C>(N, start);
-            test(i, c1, -10);
-        }
+  }
+  for (int i = N - 3; i <= N; ++i) {
+    if (0 <= i && i <= N) {
+      C c1 = make<C>(N, start);
+      test(i, c1, -10);
     }
+  }
 }
 
-int main(int, char**)
-{
-    {
-    int rng[] = {0, 1, 2, 3, 1023, 1024, 1025, 2047, 2048, 2049};
-    const int N = sizeof(rng)/sizeof(rng[0]);
+int main(int, char**) {
+  {
+    int rng[]   = {0, 1, 2, 3, 1023, 1024, 1025, 2047, 2048, 2049};
+    const int N = sizeof(rng) / sizeof(rng[0]);
     for (int i = 0; i < N; ++i)
-        for (int j = 0; j < N; ++j)
-            testN<std::deque<MoveOnly> >(rng[i], rng[j]);
-    }
-    {
-    int rng[] = {0, 1, 2, 3, 1023, 1024, 1025, 2047, 2048, 2049};
-    const int N = sizeof(rng)/sizeof(rng[0]);
+      for (int j = 0; j < N; ++j)
+        testN<std::deque<MoveOnly> >(rng[i], rng[j]);
+  }
+  {
+    int rng[]   = {0, 1, 2, 3, 1023, 1024, 1025, 2047, 2048, 2049};
+    const int N = sizeof(rng) / sizeof(rng[0]);
     for (int i = 0; i < N; ++i)
-        for (int j = 0; j < N; ++j)
-            testN<std::deque<MoveOnly, min_allocator<MoveOnly>> >(rng[i], rng[j]);
-    }
-    {
-    int rng[] = {0, 1, 2, 3, 1023, 1024, 1025, 2047, 2048, 2049};
-    const int N = sizeof(rng)/sizeof(rng[0]);
+      for (int j = 0; j < N; ++j)
+        testN<std::deque<MoveOnly, min_allocator<MoveOnly>> >(rng[i], rng[j]);
+  }
+  {
+    int rng[]   = {0, 1, 2, 3, 1023, 1024, 1025, 2047, 2048, 2049};
+    const int N = sizeof(rng) / sizeof(rng[0]);
     for (int i = 0; i < N; ++i)
-        for (int j = 0; j < N; ++j)
-            testN<std::deque<MoveOnly, safe_allocator<MoveOnly>> >(rng[i], rng[j]);
-    }
+      for (int j = 0; j < N; ++j)
+        testN<std::deque<MoveOnly, safe_allocator<MoveOnly>> >(rng[i], rng[j]);
+  }
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/deque/deque.modifiers/insert_size_value.pass.cpp b/libcxx/test/std/containers/sequences/deque/deque.modifiers/insert_size_value.pass.cpp
index 555cb902d91da..05a16a9e76387 100644
--- a/libcxx/test/std/containers/sequences/deque/deque.modifiers/insert_size_value.pass.cpp
+++ b/libcxx/test/std/containers/sequences/deque/deque.modifiers/insert_size_value.pass.cpp
@@ -21,149 +21,127 @@
 #include "min_allocator.h"
 
 template <class C>
-C
-make(int size, int start = 0 )
-{
-    const int b = 4096 / sizeof(int);
-    int init = 0;
-    if (start > 0)
-    {
-        init = (start+1) / b + ((start+1) % b != 0);
-        init *= b;
-        --init;
-    }
-    C c(init, 0);
-    for (int i = 0; i < init-start; ++i)
-        c.pop_back();
-    for (int i = 0; i < size; ++i)
-        c.push_back(i);
-    for (int i = 0; i < start; ++i)
-        c.pop_front();
-    return c;
+C make(int size, int start = 0) {
+  const int b = 4096 / sizeof(int);
+  int init    = 0;
+  if (start > 0) {
+    init = (start + 1) / b + ((start + 1) % b != 0);
+    init *= b;
+    --init;
+  }
+  C c(init, 0);
+  for (int i = 0; i < init - start; ++i)
+    c.pop_back();
+  for (int i = 0; i < size; ++i)
+    c.push_back(i);
+  for (int i = 0; i < start; ++i)
+    c.pop_front();
+  return c;
 }
 
 template <class C>
-void
-test(int P, C& c1, int size, int x)
-{
-    typedef typename C::const_iterator CI;
-    std::size_t c1_osize = c1.size();
-    CI i = c1.insert(c1.begin() + P, size, x);
-    assert(i == c1.begin() + P);
-    assert(c1.size() == c1_osize + size);
-    assert(static_cast<std::size_t>(std::distance(c1.begin(), c1.end())) == c1.size());
-    LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c1));
-    i = c1.begin();
-    for (int j = 0; j < P; ++j, ++i)
-        assert(*i == j);
-    for (int j = 0; j < size; ++j, ++i)
-        assert(*i == x);
-    for (int j = P; static_cast<std::size_t>(j) < c1_osize; ++j, ++i)
-        assert(*i == j);
+void test(int P, C& c1, int size, int x) {
+  typedef typename C::const_iterator CI;
+  std::size_t c1_osize = c1.size();
+  CI i                 = c1.insert(c1.begin() + P, size, x);
+  assert(i == c1.begin() + P);
+  assert(c1.size() == c1_osize + size);
+  assert(static_cast<std::size_t>(std::distance(c1.begin(), c1.end())) == c1.size());
+  LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c1));
+  i = c1.begin();
+  for (int j = 0; j < P; ++j, ++i)
+    assert(*i == j);
+  for (int j = 0; j < size; ++j, ++i)
+    assert(*i == x);
+  for (int j = P; static_cast<std::size_t>(j) < c1_osize; ++j, ++i)
+    assert(*i == j);
 }
 
 template <class C>
-void
-testN(int start, int N, int M)
-{
-    for (int i = 0; i <= 3; ++i)
-    {
-        if (0 <= i && i <= N)
-        {
-            C c1 = make<C>(N, start);
-            test(i, c1, M, -10);
-        }
+void testN(int start, int N, int M) {
+  for (int i = 0; i <= 3; ++i) {
+    if (0 <= i && i <= N) {
+      C c1 = make<C>(N, start);
+      test(i, c1, M, -10);
     }
-    for (int i = M-1; i <= M+1; ++i)
-    {
-        if (0 <= i && i <= N)
-        {
-            C c1 = make<C>(N, start);
-            test(i, c1, M, -10);
-        }
+  }
+  for (int i = M - 1; i <= M + 1; ++i) {
+    if (0 <= i && i <= N) {
+      C c1 = make<C>(N, start);
+      test(i, c1, M, -10);
     }
-    for (int i = N/2-1; i <= N/2+1; ++i)
-    {
-        if (0 <= i && i <= N)
-        {
-            C c1 = make<C>(N, start);
-            test(i, c1, M, -10);
-        }
+  }
+  for (int i = N / 2 - 1; i <= N / 2 + 1; ++i) {
+    if (0 <= i && i <= N) {
+      C c1 = make<C>(N, start);
+      test(i, c1, M, -10);
     }
-    for (int i = N - M - 1; i <= N - M + 1; ++i)
-    {
-        if (0 <= i && i <= N)
-        {
-            C c1 = make<C>(N, start);
-            test(i, c1, M, -10);
-        }
+  }
+  for (int i = N - M - 1; i <= N - M + 1; ++i) {
+    if (0 <= i && i <= N) {
+      C c1 = make<C>(N, start);
+      test(i, c1, M, -10);
     }
-    for (int i = N - 3; i <= N; ++i)
-    {
-        if (0 <= i && i <= N)
-        {
-            C c1 = make<C>(N, start);
-            test(i, c1, M, -10);
-        }
+  }
+  for (int i = N - 3; i <= N; ++i) {
+    if (0 <= i && i <= N) {
+      C c1 = make<C>(N, start);
+      test(i, c1, M, -10);
     }
+  }
 }
 
 template <class C>
-void
-self_reference_test()
-{
-    typedef typename C::const_iterator CI;
-    for (int i = 0; i < 20; ++i)
-    {
-        for (int j = 0; j < 20; ++j)
-        {
-            C c = make<C>(20);
-            CI it = c.cbegin() + i;
-            CI jt = c.cbegin() + j;
-            c.insert(it, 5, *jt);
-            assert(c.size() == 25);
-            assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-            it = c.cbegin();
-            for (int k = 0; k < i; ++k, ++it)
-                assert(*it == k);
-            for (int k = 0; k < 5; ++k, ++it)
-                assert(*it == j);
-            for (int k = i; k < 20; ++k, ++it)
-                assert(*it == k);
-        }
+void self_reference_test() {
+  typedef typename C::const_iterator CI;
+  for (int i = 0; i < 20; ++i) {
+    for (int j = 0; j < 20; ++j) {
+      C c   = make<C>(20);
+      CI it = c.cbegin() + i;
+      CI jt = c.cbegin() + j;
+      c.insert(it, 5, *jt);
+      assert(c.size() == 25);
+      assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
+      it = c.cbegin();
+      for (int k = 0; k < i; ++k, ++it)
+        assert(*it == k);
+      for (int k = 0; k < 5; ++k, ++it)
+        assert(*it == j);
+      for (int k = i; k < 20; ++k, ++it)
+        assert(*it == k);
     }
+  }
 }
 
-int main(int, char**)
-{
-    {
-    int rng[] = {0, 1, 2, 3, 1023, 1024, 1025, 2047, 2048, 2049};
-    const int N = sizeof(rng)/sizeof(rng[0]);
+int main(int, char**) {
+  {
+    int rng[]   = {0, 1, 2, 3, 1023, 1024, 1025, 2047, 2048, 2049};
+    const int N = sizeof(rng) / sizeof(rng[0]);
     for (int i = 0; i < N; ++i)
-        for (int j = 0; j < N; ++j)
-            for (int k = 0; k < N; ++k)
-                testN<std::deque<int> >(rng[i], rng[j], rng[k]);
+      for (int j = 0; j < N; ++j)
+        for (int k = 0; k < N; ++k)
+          testN<std::deque<int> >(rng[i], rng[j], rng[k]);
     self_reference_test<std::deque<int> >();
-    }
+  }
 #if TEST_STD_VER >= 11
-    {
-    int rng[] = {0, 1, 2, 3, 1023, 1024, 1025, 2047, 2048, 2049};
-    const int N = sizeof(rng)/sizeof(rng[0]);
+  {
+    int rng[]   = {0, 1, 2, 3, 1023, 1024, 1025, 2047, 2048, 2049};
+    const int N = sizeof(rng) / sizeof(rng[0]);
     for (int i = 0; i < N; ++i)
-        for (int j = 0; j < N; ++j)
-            for (int k = 0; k < N; ++k)
-                testN<std::deque<int, min_allocator<int>> >(rng[i], rng[j], rng[k]);
+      for (int j = 0; j < N; ++j)
+        for (int k = 0; k < N; ++k)
+          testN<std::deque<int, min_allocator<int>> >(rng[i], rng[j], rng[k]);
     self_reference_test<std::deque<int, min_allocator<int>> >();
-    }
-    {
-    int rng[] = {0, 1, 2, 3, 1023, 1024, 1025, 2047, 2048, 2049};
-    const int N = sizeof(rng)/sizeof(rng[0]);
+  }
+  {
+    int rng[]   = {0, 1, 2, 3, 1023, 1024, 1025, 2047, 2048, 2049};
+    const int N = sizeof(rng) / sizeof(rng[0]);
     for (int i = 0; i < N; ++i)
-        for (int j = 0; j < N; ++j)
-            for (int k = 0; k < N; ++k)
-                testN<std::deque<int, safe_allocator<int>> >(rng[i], rng[j], rng[k]);
+      for (int j = 0; j < N; ++j)
+        for (int k = 0; k < N; ++k)
+          testN<std::deque<int, safe_allocator<int>> >(rng[i], rng[j], rng[k]);
     self_reference_test<std::deque<int, safe_allocator<int>> >();
-    }
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/sequences/deque/deque.modifiers/insert_value.pass.cpp b/libcxx/test/std/containers/sequences/deque/deque.modifiers/insert_value.pass.cpp
index 250e19418e45b..b9440acb4986a 100644
--- a/libcxx/test/std/containers/sequences/deque/deque.modifiers/insert_value.pass.cpp
+++ b/libcxx/test/std/containers/sequences/deque/deque.modifiers/insert_value.pass.cpp
@@ -19,130 +19,112 @@
 #include "min_allocator.h"
 
 template <class C>
-C
-make(int size, int start = 0 )
-{
-    const int b = 4096 / sizeof(int);
-    int init = 0;
-    if (start > 0)
-    {
-        init = (start+1) / b + ((start+1) % b != 0);
-        init *= b;
-        --init;
-    }
-    C c(init, 0);
-    for (int i = 0; i < init-start; ++i)
-        c.pop_back();
-    for (int i = 0; i < size; ++i)
-        c.push_back(i);
-    for (int i = 0; i < start; ++i)
-        c.pop_front();
-    return c;
+C make(int size, int start = 0) {
+  const int b = 4096 / sizeof(int);
+  int init    = 0;
+  if (start > 0) {
+    init = (start + 1) / b + ((start + 1) % b != 0);
+    init *= b;
+    --init;
+  }
+  C c(init, 0);
+  for (int i = 0; i < init - start; ++i)
+    c.pop_back();
+  for (int i = 0; i < size; ++i)
+    c.push_back(i);
+  for (int i = 0; i < start; ++i)
+    c.pop_front();
+  return c;
 }
 
 template <class C>
-void
-test(int P, C& c1, int x)
-{
-    typedef typename C::const_iterator CI;
-    std::size_t c1_osize = c1.size();
-    CI i = c1.insert(c1.begin() + P, x);
-    assert(i == c1.begin() + P);
-    assert(c1.size() == c1_osize + 1);
-    assert(static_cast<std::size_t>(std::distance(c1.begin(), c1.end())) == c1.size());
-    LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c1));
-    i = c1.begin();
-    for (int j = 0; j < P; ++j, ++i)
-        assert(*i == j);
-    assert(*i == x);
-    ++i;
-    for (int j = P; static_cast<std::size_t>(j) < c1_osize; ++j, ++i)
-        assert(*i == j);
+void test(int P, C& c1, int x) {
+  typedef typename C::const_iterator CI;
+  std::size_t c1_osize = c1.size();
+  CI i                 = c1.insert(c1.begin() + P, x);
+  assert(i == c1.begin() + P);
+  assert(c1.size() == c1_osize + 1);
+  assert(static_cast<std::size_t>(std::distance(c1.begin(), c1.end())) == c1.size());
+  LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c1));
+  i = c1.begin();
+  for (int j = 0; j < P; ++j, ++i)
+    assert(*i == j);
+  assert(*i == x);
+  ++i;
+  for (int j = P; static_cast<std::size_t>(j) < c1_osize; ++j, ++i)
+    assert(*i == j);
 }
 
 template <class C>
-void
-testN(int start, int N)
-{
-    for (int i = 0; i <= 3; ++i)
-    {
-        if (0 <= i && i <= N)
-        {
-            C c1 = make<C>(N, start);
-            test(i, c1, -10);
-        }
+void testN(int start, int N) {
+  for (int i = 0; i <= 3; ++i) {
+    if (0 <= i && i <= N) {
+      C c1 = make<C>(N, start);
+      test(i, c1, -10);
     }
-    for (int i = N/2-1; i <= N/2+1; ++i)
-    {
-        if (0 <= i && i <= N)
-        {
-            C c1 = make<C>(N, start);
-            test(i, c1, -10);
-        }
+  }
+  for (int i = N / 2 - 1; i <= N / 2 + 1; ++i) {
+    if (0 <= i && i <= N) {
+      C c1 = make<C>(N, start);
+      test(i, c1, -10);
     }
-    for (int i = N - 3; i <= N; ++i)
-    {
-        if (0 <= i && i <= N)
-        {
-            C c1 = make<C>(N, start);
-            test(i, c1, -10);
-        }
+  }
+  for (int i = N - 3; i <= N; ++i) {
+    if (0 <= i && i <= N) {
+      C c1 = make<C>(N, start);
+      test(i, c1, -10);
     }
+  }
 }
 
 template <class C>
-void
-self_reference_test()
-{
-    typedef typename C::const_iterator CI;
-    for (int i = 0; i < 20; ++i)
-    {
-        for (int j = 0; j < 20; ++j)
-        {
-            C c = make<C>(20);
-            CI it = c.cbegin() + i;
-            CI jt = c.cbegin() + j;
-            c.insert(it, *jt);
-            assert(c.size() == 21);
-            assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-            it = c.cbegin();
-            for (int k = 0; k < i; ++k, ++it)
-                assert(*it == k);
-            assert(*it == j);
-            ++it;
-            for (int k = i; k < 20; ++k, ++it)
-                assert(*it == k);
-        }
+void self_reference_test() {
+  typedef typename C::const_iterator CI;
+  for (int i = 0; i < 20; ++i) {
+    for (int j = 0; j < 20; ++j) {
+      C c   = make<C>(20);
+      CI it = c.cbegin() + i;
+      CI jt = c.cbegin() + j;
+      c.insert(it, *jt);
+      assert(c.size() == 21);
+      assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
+      it = c.cbegin();
+      for (int k = 0; k < i; ++k, ++it)
+        assert(*it == k);
+      assert(*it == j);
+      ++it;
+      for (int k = i; k < 20; ++k, ++it)
+        assert(*it == k);
     }
+  }
 }
 
-int main(int, char**)
-{
-    {
-    int rng[] = {0, 1, 2, 3, 1023, 1024, 1025, 2047, 2048, 2049};
-    const int N = sizeof(rng)/sizeof(rng[0]);
+int main(int, char**) {
+  {
+    int rng[]   = {0, 1, 2, 3, 1023, 1024, 1025, 2047, 2048, 2049};
+    const int N = sizeof(rng) / sizeof(rng[0]);
     for (int i = 0; i < N; ++i)
-        for (int j = 0; j < N; ++j)
-            testN<std::deque<int> >(rng[i], rng[j]);
+      for (int j = 0; j < N; ++j)
+        testN<std::deque<int> >(rng[i], rng[j]);
     self_reference_test<std::deque<int> >();
-    }
+  }
 #if TEST_STD_VER >= 11
-    {
-    int rng[] = {0, 1, 2, 3, 1023, 1024, 1025, 2047, 2048, 2049};
-    const int N = sizeof(rng)/sizeof(rng[0]);
+  {
+    int rng[]   = {0, 1, 2, 3, 1023, 1024, 1025, 2047, 2048, 2049};
+    const int N = sizeof(rng) / sizeof(rng[0]);
     for (int i = 0; i < N; ++i)
-        for (int j = 0; j < N; ++j)
-            testN<std::deque<int, min_allocator<int>> >(rng[i], rng[j]);
+      for (int j = 0; j < N; ++j)
+        testN<std::deque<int, min_allocator<int>> >(rng[i], rng[j]);
     self_reference_test<std::deque<int, min_allocator<int>> >();
-    }
-    {
-    int rng[] = {0, 1, 2, 3, 1023, 1024, 1025, 2047, 2048, 2049};
-    const int N = sizeof(rng)/sizeof(rng[0]);
+  }
+  {
+    int rng[]   = {0, 1, 2, 3, 1023, 1024, 1025, 2047, 2048, 2049};
+    const int N = sizeof(rng) / sizeof(rng[0]);
     for (int i = 0; i < N; ++i)
-        for (int j = 0; j < N; ++j)
-            testN<std::deque<int, safe_allocator<int>> >(rng[i], rng[j]);
+      for (int j = 0; j < N; ++j)
+        testN<std::deque<int, safe_allocator<int>> >(rng[i], rng[j]);
     self_reference_test<std::deque<int, safe_allocator<int>> >();
-    }
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/sequences/deque/deque.modifiers/pop_back.invalidation.pass.cpp b/libcxx/test/std/containers/sequences/deque/deque.modifiers/pop_back.invalidation.pass.cpp
index 3071d00a9dbff..3d0133e362b6a 100644
--- a/libcxx/test/std/containers/sequences/deque/deque.modifiers/pop_back.invalidation.pass.cpp
+++ b/libcxx/test/std/containers/sequences/deque/deque.modifiers/pop_back.invalidation.pass.cpp
@@ -20,36 +20,33 @@
 #include "test_macros.h"
 
 template <typename C>
-void test(C c)
-{
-    typename C::iterator it1 = c.begin();
-    typename C::iterator it2 = c.end() - 2;
-
-    c.pop_back();
-
-    typename C::iterator it3 = c.begin();
-    typename C::iterator it4 = c.end() - 1;
-    assert(  it1 ==   it3);
-    assert( *it1 ==  *it3);
-    assert(&*it1 == &*it3);
-    assert(  it2 ==   it4);
-    assert( *it2 ==  *it4);
-    assert(&*it2 == &*it4);
-    LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c));
+void test(C c) {
+  typename C::iterator it1 = c.begin();
+  typename C::iterator it2 = c.end() - 2;
+
+  c.pop_back();
+
+  typename C::iterator it3 = c.begin();
+  typename C::iterator it4 = c.end() - 1;
+  assert(it1 == it3);
+  assert(*it1 == *it3);
+  assert(&*it1 == &*it3);
+  assert(it2 == it4);
+  assert(*it2 == *it4);
+  assert(&*it2 == &*it4);
+  LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c));
 }
 
-int main(int, char**)
-{
-    std::deque<int> queue;
-    for (int i = 0; i < 4098; ++i)
-        queue.push_back(i);
-
-    while (queue.size() > 1)
-    {
-        test(queue);
-        queue.pop_back();
-        LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(queue));
-    }
+int main(int, char**) {
+  std::deque<int> queue;
+  for (int i = 0; i < 4098; ++i)
+    queue.push_back(i);
+
+  while (queue.size() > 1) {
+    test(queue);
+    queue.pop_back();
+    LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(queue));
+  }
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/deque/deque.modifiers/pop_back.pass.cpp b/libcxx/test/std/containers/sequences/deque/deque.modifiers/pop_back.pass.cpp
index b027f40d8dd46..301d8e3342bb6 100644
--- a/libcxx/test/std/containers/sequences/deque/deque.modifiers/pop_back.pass.cpp
+++ b/libcxx/test/std/containers/sequences/deque/deque.modifiers/pop_back.pass.cpp
@@ -19,77 +19,68 @@
 #include "min_allocator.h"
 
 template <class C>
-C
-make(int size, int start = 0 )
-{
-    const int b = 4096 / sizeof(int);
-    int init = 0;
-    if (start > 0)
-    {
-        init = (start+1) / b + ((start+1) % b != 0);
-        init *= b;
-        --init;
-    }
-    C c(init, 0);
-    for (int i = 0; i < init-start; ++i)
-        c.pop_back();
-    for (int i = 0; i < size; ++i)
-        c.push_back(i);
-    for (int i = 0; i < start; ++i)
-        c.pop_front();
-    return c;
+C make(int size, int start = 0) {
+  const int b = 4096 / sizeof(int);
+  int init    = 0;
+  if (start > 0) {
+    init = (start + 1) / b + ((start + 1) % b != 0);
+    init *= b;
+    --init;
+  }
+  C c(init, 0);
+  for (int i = 0; i < init - start; ++i)
+    c.pop_back();
+  for (int i = 0; i < size; ++i)
+    c.push_back(i);
+  for (int i = 0; i < start; ++i)
+    c.pop_front();
+  return c;
 }
 
 template <class C>
-void
-test(C& c1)
-{
-    typedef typename C::iterator I;
-    std::size_t c1_osize = c1.size();
-    c1.pop_back();
-    assert(c1.size() == c1_osize - 1);
-    assert(static_cast<std::size_t>(std::distance(c1.begin(), c1.end())) == c1.size());
-    LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c1));
-    I i = c1.begin();
-    for (int j = 0; static_cast<std::size_t>(j) < c1.size(); ++j, ++i)
-        assert(*i == j);
+void test(C& c1) {
+  typedef typename C::iterator I;
+  std::size_t c1_osize = c1.size();
+  c1.pop_back();
+  assert(c1.size() == c1_osize - 1);
+  assert(static_cast<std::size_t>(std::distance(c1.begin(), c1.end())) == c1.size());
+  LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c1));
+  I i = c1.begin();
+  for (int j = 0; static_cast<std::size_t>(j) < c1.size(); ++j, ++i)
+    assert(*i == j);
 }
 
 template <class C>
-void
-testN(int start, int N)
-{
-    if (N != 0)
-    {
-        C c1 = make<C>(N, start);
-        test(c1);
-    }
+void testN(int start, int N) {
+  if (N != 0) {
+    C c1 = make<C>(N, start);
+    test(c1);
+  }
 }
 
-int main(int, char**)
-{
-    {
-    int rng[] = {0, 1, 2, 3, 1023, 1024, 1025, 2047, 2048, 2049};
-    const int N = sizeof(rng)/sizeof(rng[0]);
+int main(int, char**) {
+  {
+    int rng[]   = {0, 1, 2, 3, 1023, 1024, 1025, 2047, 2048, 2049};
+    const int N = sizeof(rng) / sizeof(rng[0]);
     for (int i = 0; i < N; ++i)
-        for (int j = 0; j < N; ++j)
-            testN<std::deque<int> >(rng[i], rng[j]);
-    }
+      for (int j = 0; j < N; ++j)
+        testN<std::deque<int> >(rng[i], rng[j]);
+  }
 #if TEST_STD_VER >= 11
-    {
-    int rng[] = {0, 1, 2, 3, 1023, 1024, 1025, 2047, 2048, 2049};
-    const int N = sizeof(rng)/sizeof(rng[0]);
+  {
+    int rng[]   = {0, 1, 2, 3, 1023, 1024, 1025, 2047, 2048, 2049};
+    const int N = sizeof(rng) / sizeof(rng[0]);
     for (int i = 0; i < N; ++i)
-        for (int j = 0; j < N; ++j)
-            testN<std::deque<int, min_allocator<int>> >(rng[i], rng[j]);
-    }
-    {
-    int rng[] = {0, 1, 2, 3, 1023, 1024, 1025, 2047, 2048, 2049};
-    const int N = sizeof(rng)/sizeof(rng[0]);
+      for (int j = 0; j < N; ++j)
+        testN<std::deque<int, min_allocator<int>> >(rng[i], rng[j]);
+  }
+  {
+    int rng[]   = {0, 1, 2, 3, 1023, 1024, 1025, 2047, 2048, 2049};
+    const int N = sizeof(rng) / sizeof(rng[0]);
     for (int i = 0; i < N; ++i)
-        for (int j = 0; j < N; ++j)
-            testN<std::deque<int, safe_allocator<int>> >(rng[i], rng[j]);
-    }
+      for (int j = 0; j < N; ++j)
+        testN<std::deque<int, safe_allocator<int>> >(rng[i], rng[j]);
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/sequences/deque/deque.modifiers/pop_front.invalidation.pass.cpp b/libcxx/test/std/containers/sequences/deque/deque.modifiers/pop_front.invalidation.pass.cpp
index 4e77cdca430d2..c857c3038d49e 100644
--- a/libcxx/test/std/containers/sequences/deque/deque.modifiers/pop_front.invalidation.pass.cpp
+++ b/libcxx/test/std/containers/sequences/deque/deque.modifiers/pop_front.invalidation.pass.cpp
@@ -20,35 +20,32 @@
 #include "test_macros.h"
 
 template <typename C>
-void test(C c)
-{
-    typename C::iterator it1 = c.begin() + 1;
-    typename C::iterator it2 = c.end() - 1;
-
-    c.pop_front();
-
-    typename C::iterator it3 = c.begin();
-    typename C::iterator it4 = c.end() - 1;
-    assert(  it1 ==   it3);
-    assert( *it1 ==  *it3);
-    assert(&*it1 == &*it3);
-    assert(  it2 ==   it4);
-    assert( *it2 ==  *it4);
-    assert(&*it2 == &*it4);
+void test(C c) {
+  typename C::iterator it1 = c.begin() + 1;
+  typename C::iterator it2 = c.end() - 1;
+
+  c.pop_front();
+
+  typename C::iterator it3 = c.begin();
+  typename C::iterator it4 = c.end() - 1;
+  assert(it1 == it3);
+  assert(*it1 == *it3);
+  assert(&*it1 == &*it3);
+  assert(it2 == it4);
+  assert(*it2 == *it4);
+  assert(&*it2 == &*it4);
 }
 
-int main(int, char**)
-{
-    std::deque<int> queue;
-    for (int i = 0; i < 4098; ++i)
-        queue.push_back(i);
-
-    while (queue.size() > 1)
-    {
-        test(queue);
-        queue.pop_back();
-        LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(queue));
-    }
+int main(int, char**) {
+  std::deque<int> queue;
+  for (int i = 0; i < 4098; ++i)
+    queue.push_back(i);
+
+  while (queue.size() > 1) {
+    test(queue);
+    queue.pop_back();
+    LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(queue));
+  }
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/deque/deque.modifiers/pop_front.pass.cpp b/libcxx/test/std/containers/sequences/deque/deque.modifiers/pop_front.pass.cpp
index e28242c8a4890..2bd390ac315e0 100644
--- a/libcxx/test/std/containers/sequences/deque/deque.modifiers/pop_front.pass.cpp
+++ b/libcxx/test/std/containers/sequences/deque/deque.modifiers/pop_front.pass.cpp
@@ -19,70 +19,61 @@
 #include "min_allocator.h"
 
 template <class C>
-C
-make(int size, int start = 0 )
-{
-    const int b = 4096 / sizeof(int);
-    int init = 0;
-    if (start > 0)
-    {
-        init = (start+1) / b + ((start+1) % b != 0);
-        init *= b;
-        --init;
-    }
-    C c(init, 0);
-    for (int i = 0; i < init-start; ++i)
-        c.pop_back();
-    for (int i = 0; i < size; ++i)
-        c.push_back(i);
-    for (int i = 0; i < start; ++i)
-        c.pop_front();
-    return c;
+C make(int size, int start = 0) {
+  const int b = 4096 / sizeof(int);
+  int init    = 0;
+  if (start > 0) {
+    init = (start + 1) / b + ((start + 1) % b != 0);
+    init *= b;
+    --init;
+  }
+  C c(init, 0);
+  for (int i = 0; i < init - start; ++i)
+    c.pop_back();
+  for (int i = 0; i < size; ++i)
+    c.push_back(i);
+  for (int i = 0; i < start; ++i)
+    c.pop_front();
+  return c;
 }
 
 template <class C>
-void
-test(C& c1)
-{
-    typedef typename C::iterator I;
-    std::size_t c1_osize = c1.size();
-    c1.pop_front();
-    assert(c1.size() == c1_osize - 1);
-    assert(static_cast<std::size_t>(std::distance(c1.begin(), c1.end())) == c1.size());
-    LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c1));
-    I i = c1.begin();
-    for (int j = 1; static_cast<std::size_t>(j) < c1.size(); ++j, ++i)
-        assert(*i == j);
+void test(C& c1) {
+  typedef typename C::iterator I;
+  std::size_t c1_osize = c1.size();
+  c1.pop_front();
+  assert(c1.size() == c1_osize - 1);
+  assert(static_cast<std::size_t>(std::distance(c1.begin(), c1.end())) == c1.size());
+  LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c1));
+  I i = c1.begin();
+  for (int j = 1; static_cast<std::size_t>(j) < c1.size(); ++j, ++i)
+    assert(*i == j);
 }
 
 template <class C>
-void
-testN(int start, int N)
-{
-    if (N != 0)
-    {
-        C c1 = make<C>(N, start);
-        test(c1);
-    }
+void testN(int start, int N) {
+  if (N != 0) {
+    C c1 = make<C>(N, start);
+    test(c1);
+  }
 }
 
-int main(int, char**)
-{
-    {
-    int rng[] = {0, 1, 2, 3, 1023, 1024, 1025, 2047, 2048, 2049};
-    const int N = sizeof(rng)/sizeof(rng[0]);
+int main(int, char**) {
+  {
+    int rng[]   = {0, 1, 2, 3, 1023, 1024, 1025, 2047, 2048, 2049};
+    const int N = sizeof(rng) / sizeof(rng[0]);
     for (int i = 0; i < N; ++i)
-        for (int j = 0; j < N; ++j)
-            testN<std::deque<int> >(rng[i], rng[j]);
-    }
+      for (int j = 0; j < N; ++j)
+        testN<std::deque<int> >(rng[i], rng[j]);
+  }
 #if TEST_STD_VER >= 11
-    {
-    int rng[] = {0, 1, 2, 3, 1023, 1024, 1025, 2047, 2048, 2049};
-    const int N = sizeof(rng)/sizeof(rng[0]);
+  {
+    int rng[]   = {0, 1, 2, 3, 1023, 1024, 1025, 2047, 2048, 2049};
+    const int N = sizeof(rng) / sizeof(rng[0]);
     for (int i = 0; i < N; ++i)
-        for (int j = 0; j < N; ++j)
-            testN<std::deque<int, min_allocator<int>> >(rng[i], rng[j]);
-    }
+      for (int j = 0; j < N; ++j)
+        testN<std::deque<int, min_allocator<int>> >(rng[i], rng[j]);
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/sequences/deque/deque.modifiers/push_back.pass.cpp b/libcxx/test/std/containers/sequences/deque/deque.modifiers/push_back.pass.cpp
index 70aa1baa1efcb..fff3433ab4dd4 100644
--- a/libcxx/test/std/containers/sequences/deque/deque.modifiers/push_back.pass.cpp
+++ b/libcxx/test/std/containers/sequences/deque/deque.modifiers/push_back.pass.cpp
@@ -20,57 +20,51 @@
 #include "min_allocator.h"
 
 template <class C>
-C
-make(int size, int start = 0 )
-{
-    const int b = 4096 / sizeof(int);
-    int init = 0;
-    if (start > 0)
-    {
-        init = (start+1) / b + ((start+1) % b != 0);
-        init *= b;
-        --init;
-    }
-    C c(init, 0);
-    for (int i = 0; i < init-start; ++i)
-        c.pop_back();
-    for (int i = 0; i < size; ++i)
-        c.push_back(i);
-    for (int i = 0; i < start; ++i)
-        c.pop_front();
-    return c;
+C make(int size, int start = 0) {
+  const int b = 4096 / sizeof(int);
+  int init    = 0;
+  if (start > 0) {
+    init = (start + 1) / b + ((start + 1) % b != 0);
+    init *= b;
+    --init;
+  }
+  C c(init, 0);
+  for (int i = 0; i < init - start; ++i)
+    c.pop_back();
+  for (int i = 0; i < size; ++i)
+    c.push_back(i);
+  for (int i = 0; i < start; ++i)
+    c.pop_front();
+  return c;
 }
 
 template <class C>
-void test(int size)
-{
-    int rng[] = {0, 1, 2, 3, 1023, 1024, 1025, 2046, 2047, 2048, 2049};
-    const int N = sizeof(rng)/sizeof(rng[0]);
-    for (int j = 0; j < N; ++j)
-    {
-        C c = make<C>(size, rng[j]);
-        typename C::const_iterator it = c.begin();
-        LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c));
-        for (int i = 0; i < size; ++i, ++it)
-            assert(*it == i);
-    }
+void test(int size) {
+  int rng[]   = {0, 1, 2, 3, 1023, 1024, 1025, 2046, 2047, 2048, 2049};
+  const int N = sizeof(rng) / sizeof(rng[0]);
+  for (int j = 0; j < N; ++j) {
+    C c                           = make<C>(size, rng[j]);
+    typename C::const_iterator it = c.begin();
+    LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c));
+    for (int i = 0; i < size; ++i, ++it)
+      assert(*it == i);
+  }
 }
 
-int main(int, char**)
-{
-    {
-    int rng[] = {0, 1, 2, 3, 1023, 1024, 1025, 2046, 2047, 2048, 2049, 4094, 4095, 4096};
-    const int N = sizeof(rng)/sizeof(rng[0]);
+int main(int, char**) {
+  {
+    int rng[]   = {0, 1, 2, 3, 1023, 1024, 1025, 2046, 2047, 2048, 2049, 4094, 4095, 4096};
+    const int N = sizeof(rng) / sizeof(rng[0]);
     for (int j = 0; j < N; ++j)
-        test<std::deque<int> >(rng[j]);
-    }
+      test<std::deque<int> >(rng[j]);
+  }
 #if TEST_STD_VER >= 11
-    {
-    int rng[] = {0, 1, 2, 3, 1023, 1024, 1025, 2046, 2047, 2048, 2049, 4094, 4095, 4096};
-    const int N = sizeof(rng)/sizeof(rng[0]);
+  {
+    int rng[]   = {0, 1, 2, 3, 1023, 1024, 1025, 2046, 2047, 2048, 2049, 4094, 4095, 4096};
+    const int N = sizeof(rng) / sizeof(rng[0]);
     for (int j = 0; j < N; ++j)
-        test<std::deque<int, min_allocator<int>> >(rng[j]);
-    }
+      test<std::deque<int, min_allocator<int>> >(rng[j]);
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/sequences/deque/deque.modifiers/push_back_exception_safety.pass.cpp b/libcxx/test/std/containers/sequences/deque/deque.modifiers/push_back_exception_safety.pass.cpp
index 3e65f993f6506..10da5b02f96f1 100644
--- a/libcxx/test/std/containers/sequences/deque/deque.modifiers/push_back_exception_safety.pass.cpp
+++ b/libcxx/test/std/containers/sequences/deque/deque.modifiers/push_back_exception_safety.pass.cpp
@@ -20,19 +20,26 @@
 static bool gCopyConstructorShouldThrow = false;
 
 class CMyClass {
-    public: CMyClass(int tag);
-    public: CMyClass(const CMyClass& iOther);
-    public: ~CMyClass();
+public:
+  CMyClass(int tag);
 
-    bool equal(const CMyClass &rhs) const
-        { return fTag == rhs.fTag && fMagicValue == rhs.fMagicValue; }
+public:
+  CMyClass(const CMyClass& iOther);
 
-    private:
-        int fMagicValue;
-        int fTag;
+public:
+  ~CMyClass();
 
-    private: static int kStartedConstructionMagicValue;
-    private: static int kFinishedConstructionMagicValue;
+  bool equal(const CMyClass& rhs) const { return fTag == rhs.fTag && fMagicValue == rhs.fMagicValue; }
+
+private:
+  int fMagicValue;
+  int fTag;
+
+private:
+  static int kStartedConstructionMagicValue;
+
+private:
+  static int kFinishedConstructionMagicValue;
 };
 
 // Value for fMagicValue when the constructor has started running, but not yet finished
@@ -40,35 +47,30 @@ int CMyClass::kStartedConstructionMagicValue = 0;
 // Value for fMagicValue when the constructor has finished running
 int CMyClass::kFinishedConstructionMagicValue = 12345;
 
-CMyClass::CMyClass(int tag) :
-    fMagicValue(kStartedConstructionMagicValue), fTag(tag)
-{
-    // Signal that the constructor has finished running
-    fMagicValue = kFinishedConstructionMagicValue;
+CMyClass::CMyClass(int tag) : fMagicValue(kStartedConstructionMagicValue), fTag(tag) {
+  // Signal that the constructor has finished running
+  fMagicValue = kFinishedConstructionMagicValue;
 }
 
-CMyClass::CMyClass(const CMyClass& iOther) :
-    fMagicValue(kStartedConstructionMagicValue), fTag(iOther.fTag)
-{
-    // If requested, throw an exception _before_ setting fMagicValue to kFinishedConstructionMagicValue
-    if (gCopyConstructorShouldThrow) {
-      throw std::exception();
-    }
-    // Signal that the constructor has finished running
-    fMagicValue = kFinishedConstructionMagicValue;
+CMyClass::CMyClass(const CMyClass& iOther) : fMagicValue(kStartedConstructionMagicValue), fTag(iOther.fTag) {
+  // If requested, throw an exception _before_ setting fMagicValue to kFinishedConstructionMagicValue
+  if (gCopyConstructorShouldThrow) {
+    throw std::exception();
+  }
+  // Signal that the constructor has finished running
+  fMagicValue = kFinishedConstructionMagicValue;
 }
 
 CMyClass::~CMyClass() {
-    // Only instances for which the constructor has finished running should be destructed
-    assert(fMagicValue == kFinishedConstructionMagicValue);
+  // Only instances for which the constructor has finished running should be destructed
+  assert(fMagicValue == kFinishedConstructionMagicValue);
 }
 
-bool operator==(const CMyClass &lhs, const CMyClass &rhs) { return lhs.equal(rhs); }
+bool operator==(const CMyClass& lhs, const CMyClass& rhs) { return lhs.equal(rhs); }
 
-int main(int, char**)
-{
-    CMyClass instance(42);
-    {
+int main(int, char**) {
+  CMyClass instance(42);
+  {
     std::deque<CMyClass> vec;
 
     vec.push_back(instance);
@@ -76,16 +78,15 @@ int main(int, char**)
 
     gCopyConstructorShouldThrow = true;
     try {
-        vec.push_back(instance);
-        assert(false);
-    }
-    catch (...) {
+      vec.push_back(instance);
+      assert(false);
+    } catch (...) {
       gCopyConstructorShouldThrow = false;
       assert(vec == vec2);
     }
-    }
+  }
 
-    {
+  {
     test_allocator_statistics alloc_stats;
     typedef std::deque<CMyClass, test_allocator<CMyClass> > C;
     C vec((test_allocator<CMyClass>(&alloc_stats)));
@@ -93,13 +94,12 @@ int main(int, char**)
 
     alloc_stats.throw_after = 1;
     try {
-        vec.push_back(instance);
-        assert(false);
-    }
-    catch (...) {
-        assert(vec==vec2);
-    }
+      vec.push_back(instance);
+      assert(false);
+    } catch (...) {
+      assert(vec == vec2);
     }
+  }
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/deque/deque.modifiers/push_back_rvalue.pass.cpp b/libcxx/test/std/containers/sequences/deque/deque.modifiers/push_back_rvalue.pass.cpp
index 817379b62ae07..c9e0ca7a6d7e0 100644
--- a/libcxx/test/std/containers/sequences/deque/deque.modifiers/push_back_rvalue.pass.cpp
+++ b/libcxx/test/std/containers/sequences/deque/deque.modifiers/push_back_rvalue.pass.cpp
@@ -22,65 +22,57 @@
 #include "MoveOnly.h"
 #include "min_allocator.h"
 
-
 template <class C>
-C
-make(int size, int start = 0 )
-{
-    const int b = 4096 / sizeof(int);
-    int init = 0;
-    if (start > 0)
-    {
-        init = (start+1) / b + ((start+1) % b != 0);
-        init *= b;
-        --init;
-    }
-    C c(init);
-    for (int i = 0; i < init-start; ++i)
-        c.pop_back();
-    for (int i = 0; i < size; ++i)
-        c.push_back(MoveOnly(i));
-    for (int i = 0; i < start; ++i)
-        c.pop_front();
-    return c;
+C make(int size, int start = 0) {
+  const int b = 4096 / sizeof(int);
+  int init    = 0;
+  if (start > 0) {
+    init = (start + 1) / b + ((start + 1) % b != 0);
+    init *= b;
+    --init;
+  }
+  C c(init);
+  for (int i = 0; i < init - start; ++i)
+    c.pop_back();
+  for (int i = 0; i < size; ++i)
+    c.push_back(MoveOnly(i));
+  for (int i = 0; i < start; ++i)
+    c.pop_front();
+  return c;
 }
 
 template <class C>
-void test(int size)
-{
-    int rng[] = {0, 1, 2, 3, 1023, 1024, 1025, 2046, 2047, 2048, 2049};
-    const int N = sizeof(rng)/sizeof(rng[0]);
-    for (int j = 0; j < N; ++j)
-    {
-        C c = make<C>(size, rng[j]);
-        typename C::const_iterator it = c.begin();
-        LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c));
-        for (int i = 0; i < size; ++i, (void) ++it)
-            assert(*it == MoveOnly(i));
-    }
+void test(int size) {
+  int rng[]   = {0, 1, 2, 3, 1023, 1024, 1025, 2046, 2047, 2048, 2049};
+  const int N = sizeof(rng) / sizeof(rng[0]);
+  for (int j = 0; j < N; ++j) {
+    C c                           = make<C>(size, rng[j]);
+    typename C::const_iterator it = c.begin();
+    LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c));
+    for (int i = 0; i < size; ++i, (void)++it)
+      assert(*it == MoveOnly(i));
+  }
 }
 
-
-int main(int, char**)
-{
-    {
-    int rng[] = {0, 1, 2, 3, 1023, 1024, 1025, 2046, 2047, 2048, 2049, 4094, 4095, 4096};
-    const int N = sizeof(rng)/sizeof(rng[0]);
+int main(int, char**) {
+  {
+    int rng[]   = {0, 1, 2, 3, 1023, 1024, 1025, 2046, 2047, 2048, 2049, 4094, 4095, 4096};
+    const int N = sizeof(rng) / sizeof(rng[0]);
     for (int j = 0; j < N; ++j)
-        test<std::deque<MoveOnly> >(rng[j]);
-    }
-    {
-    int rng[] = {0, 1, 2, 3, 1023, 1024, 1025, 2046, 2047, 2048, 2049, 4094, 4095, 4096};
-    const int N = sizeof(rng)/sizeof(rng[0]);
+      test<std::deque<MoveOnly> >(rng[j]);
+  }
+  {
+    int rng[]   = {0, 1, 2, 3, 1023, 1024, 1025, 2046, 2047, 2048, 2049, 4094, 4095, 4096};
+    const int N = sizeof(rng) / sizeof(rng[0]);
     for (int j = 0; j < N; ++j)
-        test<std::deque<MoveOnly, min_allocator<MoveOnly>> >(rng[j]);
-    }
-    {
-    int rng[] = {0, 1, 2, 3, 1023, 1024, 1025, 2046, 2047, 2048, 2049, 4094, 4095, 4096};
-    const int N = sizeof(rng)/sizeof(rng[0]);
+      test<std::deque<MoveOnly, min_allocator<MoveOnly>> >(rng[j]);
+  }
+  {
+    int rng[]   = {0, 1, 2, 3, 1023, 1024, 1025, 2046, 2047, 2048, 2049, 4094, 4095, 4096};
+    const int N = sizeof(rng) / sizeof(rng[0]);
     for (int j = 0; j < N; ++j)
-        test<std::deque<MoveOnly, safe_allocator<MoveOnly>> >(rng[j]);
-    }
+      test<std::deque<MoveOnly, safe_allocator<MoveOnly>> >(rng[j]);
+  }
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/deque/deque.modifiers/push_front.pass.cpp b/libcxx/test/std/containers/sequences/deque/deque.modifiers/push_front.pass.cpp
index 2d1246b1507e7..1fb8341895a35 100644
--- a/libcxx/test/std/containers/sequences/deque/deque.modifiers/push_front.pass.cpp
+++ b/libcxx/test/std/containers/sequences/deque/deque.modifiers/push_front.pass.cpp
@@ -19,76 +19,68 @@
 #include "min_allocator.h"
 
 template <class C>
-C
-make(int size, int start = 0 )
-{
-    const int b = 4096 / sizeof(int);
-    int init = 0;
-    if (start > 0)
-    {
-        init = (start+1) / b + ((start+1) % b != 0);
-        init *= b;
-        --init;
-    }
-    C c(init, 0);
-    for (int i = 0; i < init-start; ++i)
-        c.pop_back();
-    for (int i = 0; i < size; ++i)
-        c.push_back(i);
-    for (int i = 0; i < start; ++i)
-        c.pop_front();
-    return c;
+C make(int size, int start = 0) {
+  const int b = 4096 / sizeof(int);
+  int init    = 0;
+  if (start > 0) {
+    init = (start + 1) / b + ((start + 1) % b != 0);
+    init *= b;
+    --init;
+  }
+  C c(init, 0);
+  for (int i = 0; i < init - start; ++i)
+    c.pop_back();
+  for (int i = 0; i < size; ++i)
+    c.push_back(i);
+  for (int i = 0; i < start; ++i)
+    c.pop_front();
+  return c;
 }
 
 template <class C>
-void
-test(C& c1, int x)
-{
-    typedef typename C::iterator I;
-    std::size_t c1_osize = c1.size();
-    c1.push_front(x);
-    assert(c1.size() == c1_osize + 1);
-    assert(static_cast<std::size_t>(std::distance(c1.begin(), c1.end())) == c1.size());
-    I i = c1.begin();
-    assert(*i == x);
-    LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c1));
-    ++i;
-    for (int j = 0; static_cast<std::size_t>(j) < c1_osize; ++j, ++i)
-        assert(*i == j);
+void test(C& c1, int x) {
+  typedef typename C::iterator I;
+  std::size_t c1_osize = c1.size();
+  c1.push_front(x);
+  assert(c1.size() == c1_osize + 1);
+  assert(static_cast<std::size_t>(std::distance(c1.begin(), c1.end())) == c1.size());
+  I i = c1.begin();
+  assert(*i == x);
+  LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c1));
+  ++i;
+  for (int j = 0; static_cast<std::size_t>(j) < c1_osize; ++j, ++i)
+    assert(*i == j);
 }
 
 template <class C>
-void
-testN(int start, int N)
-{
-    C c1 = make<C>(N, start);
-    test(c1, -10);
+void testN(int start, int N) {
+  C c1 = make<C>(N, start);
+  test(c1, -10);
 }
 
-int main(int, char**)
-{
-    {
-    int rng[] = {0, 1, 2, 3, 1023, 1024, 1025, 2047, 2048, 2049};
-    const int N = sizeof(rng)/sizeof(rng[0]);
+int main(int, char**) {
+  {
+    int rng[]   = {0, 1, 2, 3, 1023, 1024, 1025, 2047, 2048, 2049};
+    const int N = sizeof(rng) / sizeof(rng[0]);
     for (int i = 0; i < N; ++i)
-        for (int j = 0; j < N; ++j)
-            testN<std::deque<int> >(rng[i], rng[j]);
-    }
+      for (int j = 0; j < N; ++j)
+        testN<std::deque<int> >(rng[i], rng[j]);
+  }
 #if TEST_STD_VER >= 11
-   {
-    int rng[] = {0, 1, 2, 3, 1023, 1024, 1025, 2047, 2048, 2049};
-    const int N = sizeof(rng)/sizeof(rng[0]);
+  {
+    int rng[]   = {0, 1, 2, 3, 1023, 1024, 1025, 2047, 2048, 2049};
+    const int N = sizeof(rng) / sizeof(rng[0]);
     for (int i = 0; i < N; ++i)
-        for (int j = 0; j < N; ++j)
-            testN<std::deque<int, min_allocator<int>> >(rng[i], rng[j]);
-    }
-    {
-    int rng[] = {0, 1, 2, 3, 1023, 1024, 1025, 2047, 2048, 2049};
-    const int N = sizeof(rng)/sizeof(rng[0]);
+      for (int j = 0; j < N; ++j)
+        testN<std::deque<int, min_allocator<int>> >(rng[i], rng[j]);
+  }
+  {
+    int rng[]   = {0, 1, 2, 3, 1023, 1024, 1025, 2047, 2048, 2049};
+    const int N = sizeof(rng) / sizeof(rng[0]);
     for (int i = 0; i < N; ++i)
-        for (int j = 0; j < N; ++j)
-            testN<std::deque<int, safe_allocator<int>> >(rng[i], rng[j]);
-    }
+      for (int j = 0; j < N; ++j)
+        testN<std::deque<int, safe_allocator<int>> >(rng[i], rng[j]);
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/sequences/deque/deque.modifiers/push_front_exception_safety.pass.cpp b/libcxx/test/std/containers/sequences/deque/deque.modifiers/push_front_exception_safety.pass.cpp
index c1d7723bbbf25..f4b77bcf09f62 100644
--- a/libcxx/test/std/containers/sequences/deque/deque.modifiers/push_front_exception_safety.pass.cpp
+++ b/libcxx/test/std/containers/sequences/deque/deque.modifiers/push_front_exception_safety.pass.cpp
@@ -20,18 +20,26 @@
 static bool gCopyConstructorShouldThrow = false;
 
 class CMyClass {
-    public: CMyClass(int tag);
-    public: CMyClass(const CMyClass& iOther);
-    public: ~CMyClass();
-
-    bool equal(const CMyClass &rhs) const
-        { return fTag == rhs.fTag && fMagicValue == rhs.fMagicValue; }
-    private:
-        int fMagicValue;
-        int fTag;
-
-    private: static int kStartedConstructionMagicValue;
-    private: static int kFinishedConstructionMagicValue;
+public:
+  CMyClass(int tag);
+
+public:
+  CMyClass(const CMyClass& iOther);
+
+public:
+  ~CMyClass();
+
+  bool equal(const CMyClass& rhs) const { return fTag == rhs.fTag && fMagicValue == rhs.fMagicValue; }
+
+private:
+  int fMagicValue;
+  int fTag;
+
+private:
+  static int kStartedConstructionMagicValue;
+
+private:
+  static int kFinishedConstructionMagicValue;
 };
 
 // Value for fMagicValue when the constructor has started running, but not yet finished
@@ -39,35 +47,30 @@ int CMyClass::kStartedConstructionMagicValue = 0;
 // Value for fMagicValue when the constructor has finished running
 int CMyClass::kFinishedConstructionMagicValue = 12345;
 
-CMyClass::CMyClass(int tag) :
-    fMagicValue(kStartedConstructionMagicValue), fTag(tag)
-{
-    // Signal that the constructor has finished running
-    fMagicValue = kFinishedConstructionMagicValue;
+CMyClass::CMyClass(int tag) : fMagicValue(kStartedConstructionMagicValue), fTag(tag) {
+  // Signal that the constructor has finished running
+  fMagicValue = kFinishedConstructionMagicValue;
 }
 
-CMyClass::CMyClass(const CMyClass& iOther) :
-    fMagicValue(kStartedConstructionMagicValue), fTag(iOther.fTag)
-{
-    // If requested, throw an exception _before_ setting fMagicValue to kFinishedConstructionMagicValue
-    if (gCopyConstructorShouldThrow) {
-      throw std::exception();
-    }
-    // Signal that the constructor has finished running
-    fMagicValue = kFinishedConstructionMagicValue;
+CMyClass::CMyClass(const CMyClass& iOther) : fMagicValue(kStartedConstructionMagicValue), fTag(iOther.fTag) {
+  // If requested, throw an exception _before_ setting fMagicValue to kFinishedConstructionMagicValue
+  if (gCopyConstructorShouldThrow) {
+    throw std::exception();
+  }
+  // Signal that the constructor has finished running
+  fMagicValue = kFinishedConstructionMagicValue;
 }
 
 CMyClass::~CMyClass() {
-    // Only instances for which the constructor has finished running should be destructed
-    assert(fMagicValue == kFinishedConstructionMagicValue);
+  // Only instances for which the constructor has finished running should be destructed
+  assert(fMagicValue == kFinishedConstructionMagicValue);
 }
 
-bool operator==(const CMyClass &lhs, const CMyClass &rhs) { return lhs.equal(rhs); }
+bool operator==(const CMyClass& lhs, const CMyClass& rhs) { return lhs.equal(rhs); }
 
-int main(int, char**)
-{
-    CMyClass instance(42);
-    {
+int main(int, char**) {
+  CMyClass instance(42);
+  {
     std::deque<CMyClass> vec;
 
     vec.push_front(instance);
@@ -75,16 +78,15 @@ int main(int, char**)
 
     gCopyConstructorShouldThrow = true;
     try {
-        vec.push_front(instance);
-        assert(false);
-    }
-    catch (...) {
+      vec.push_front(instance);
+      assert(false);
+    } catch (...) {
       gCopyConstructorShouldThrow = false;
       assert(vec == vec2);
     }
-    }
+  }
 
-    {
+  {
     test_allocator_statistics alloc_stats;
     typedef std::deque<CMyClass, test_allocator<CMyClass> > C;
     C vec((test_allocator<CMyClass>(&alloc_stats)));
@@ -92,13 +94,12 @@ int main(int, char**)
 
     alloc_stats.throw_after = 1;
     try {
-        vec.push_front(instance);
-        assert(false);
-    }
-    catch (...) {
-        assert(vec==vec2);
-    }
+      vec.push_front(instance);
+      assert(false);
+    } catch (...) {
+      assert(vec == vec2);
     }
+  }
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/deque/deque.modifiers/push_front_rvalue.pass.cpp b/libcxx/test/std/containers/sequences/deque/deque.modifiers/push_front_rvalue.pass.cpp
index 2755433e58cbf..418a0c976e66c 100644
--- a/libcxx/test/std/containers/sequences/deque/deque.modifiers/push_front_rvalue.pass.cpp
+++ b/libcxx/test/std/containers/sequences/deque/deque.modifiers/push_front_rvalue.pass.cpp
@@ -21,71 +21,61 @@
 #include "MoveOnly.h"
 #include "min_allocator.h"
 
-
 template <class C>
-C
-make(int size, int start = 0 )
-{
-    const int b = 4096 / sizeof(int);
-    int init = 0;
-    if (start > 0)
-    {
-        init = (start+1) / b + ((start+1) % b != 0);
-        init *= b;
-        --init;
-    }
-    C c(init);
-    for (int i = 0; i < init-start; ++i)
-        c.pop_back();
-    for (int i = 0; i < size; ++i)
-        c.push_back(MoveOnly(i));
-    for (int i = 0; i < start; ++i)
-        c.pop_front();
-    return c;
+C make(int size, int start = 0) {
+  const int b = 4096 / sizeof(int);
+  int init    = 0;
+  if (start > 0) {
+    init = (start + 1) / b + ((start + 1) % b != 0);
+    init *= b;
+    --init;
+  }
+  C c(init);
+  for (int i = 0; i < init - start; ++i)
+    c.pop_back();
+  for (int i = 0; i < size; ++i)
+    c.push_back(MoveOnly(i));
+  for (int i = 0; i < start; ++i)
+    c.pop_front();
+  return c;
 }
 
 template <class C>
-void
-test(C& c1, int x)
-{
-    typedef typename C::iterator I;
-    std::size_t c1_osize = c1.size();
-    c1.push_front(MoveOnly(x));
-    assert(c1.size() == c1_osize + 1);
-    assert(static_cast<std::size_t>(std::distance(c1.begin(), c1.end())) == c1.size());
-    I i = c1.begin();
-    assert(*i == MoveOnly(x));
-    LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c1));
-    ++i;
-    for (int j = 0; static_cast<std::size_t>(j) < c1_osize; ++j, (void) ++i)
-        assert(*i == MoveOnly(j));
+void test(C& c1, int x) {
+  typedef typename C::iterator I;
+  std::size_t c1_osize = c1.size();
+  c1.push_front(MoveOnly(x));
+  assert(c1.size() == c1_osize + 1);
+  assert(static_cast<std::size_t>(std::distance(c1.begin(), c1.end())) == c1.size());
+  I i = c1.begin();
+  assert(*i == MoveOnly(x));
+  LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c1));
+  ++i;
+  for (int j = 0; static_cast<std::size_t>(j) < c1_osize; ++j, (void)++i)
+    assert(*i == MoveOnly(j));
 }
 
 template <class C>
-void
-testN(int start, int N)
-{
-    C c1 = make<C>(N, start);
-    test(c1, -10);
+void testN(int start, int N) {
+  C c1 = make<C>(N, start);
+  test(c1, -10);
 }
 
-
-int main(int, char**)
-{
-    {
-    int rng[] = {0, 1, 2, 3, 1023, 1024, 1025, 2047, 2048, 2049};
-    const int N = sizeof(rng)/sizeof(rng[0]);
+int main(int, char**) {
+  {
+    int rng[]   = {0, 1, 2, 3, 1023, 1024, 1025, 2047, 2048, 2049};
+    const int N = sizeof(rng) / sizeof(rng[0]);
     for (int i = 0; i < N; ++i)
-        for (int j = 0; j < N; ++j)
-            testN<std::deque<MoveOnly> >(rng[i], rng[j]);
-    }
-    {
-    int rng[] = {0, 1, 2, 3, 1023, 1024, 1025, 2047, 2048, 2049};
-    const int N = sizeof(rng)/sizeof(rng[0]);
+      for (int j = 0; j < N; ++j)
+        testN<std::deque<MoveOnly> >(rng[i], rng[j]);
+  }
+  {
+    int rng[]   = {0, 1, 2, 3, 1023, 1024, 1025, 2047, 2048, 2049};
+    const int N = sizeof(rng) / sizeof(rng[0]);
     for (int i = 0; i < N; ++i)
-        for (int j = 0; j < N; ++j)
-            testN<std::deque<MoveOnly, safe_allocator<MoveOnly>> >(rng[i], rng[j]);
-    }
+      for (int j = 0; j < N; ++j)
+        testN<std::deque<MoveOnly, safe_allocator<MoveOnly>> >(rng[i], rng[j]);
+  }
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/deque/deque.special/copy.pass.cpp b/libcxx/test/std/containers/sequences/deque/deque.special/copy.pass.cpp
index e472a1cb59d47..382d6e530b24f 100644
--- a/libcxx/test/std/containers/sequences/deque/deque.special/copy.pass.cpp
+++ b/libcxx/test/std/containers/sequences/deque/deque.special/copy.pass.cpp
@@ -23,80 +23,75 @@
 #include "min_allocator.h"
 
 template <class C>
-C
-make(int size, int start = 0 )
-{
-    const int b = 4096 / sizeof(int);
-    int init = 0;
-    if (start > 0)
-    {
-        init = (start+1) / b + ((start+1) % b != 0);
-        init *= b;
-        --init;
-    }
-    C c(init, 0);
-    for (int i = 0; i < init-start; ++i)
-        c.pop_back();
-    for (int i = 0; i < size; ++i)
-        c.push_back(i);
-    for (int i = 0; i < start; ++i)
-        c.pop_front();
-    return c;
+C make(int size, int start = 0) {
+  const int b = 4096 / sizeof(int);
+  int init    = 0;
+  if (start > 0) {
+    init = (start + 1) / b + ((start + 1) % b != 0);
+    init *= b;
+    --init;
+  }
+  C c(init, 0);
+  for (int i = 0; i < init - start; ++i)
+    c.pop_back();
+  for (int i = 0; i < size; ++i)
+    c.push_back(i);
+  for (int i = 0; i < start; ++i)
+    c.pop_front();
+  return c;
 }
 
 template <class C>
-void testN(int start, int N)
-{
-    typedef typename C::iterator I;
-    typedef typename C::const_iterator CI;
-    typedef random_access_iterator<I> RAI;
-    typedef random_access_iterator<CI> RACI;
-    typedef cpp17_input_iterator<CI> ICI;
-    C c1 = make<C>(N, start);
-    C c2 = make<C>(N);
-    assert(std::copy(c1.cbegin(), c1.cend(), c2.begin()) == c2.end());
-    assert(c1 == c2);
-    LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c1));
-    LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c2));
-    assert(std::copy(c2.cbegin(), c2.cend(), c1.begin()) == c1.end());
-    assert(c1 == c2);
-    LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c1));
-    LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c2));
-    assert(std::copy(c1.cbegin(), c1.cend(), RAI(c2.begin())) == RAI(c2.end()));
-    assert(c1 == c2);
-    LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c1));
-    LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c2));
-    assert(std::copy(c2.cbegin(), c2.cend(), RAI(c1.begin())) == RAI(c1.end()));
-    assert(c1 == c2);
-    LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c1));
-    LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c2));
-    assert(std::copy(RACI(c1.cbegin()), RACI(c1.cend()), c2.begin()) == c2.end());
-    assert(c1 == c2);
-    LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c1));
-    LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c2));
-    assert(std::copy(ICI(c2.cbegin()), ICI(c2.cend()), c1.begin()) == c1.end());
-    assert(c1 == c2);
-    LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c1));
-    LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c2));
+void testN(int start, int N) {
+  typedef typename C::iterator I;
+  typedef typename C::const_iterator CI;
+  typedef random_access_iterator<I> RAI;
+  typedef random_access_iterator<CI> RACI;
+  typedef cpp17_input_iterator<CI> ICI;
+  C c1 = make<C>(N, start);
+  C c2 = make<C>(N);
+  assert(std::copy(c1.cbegin(), c1.cend(), c2.begin()) == c2.end());
+  assert(c1 == c2);
+  LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c1));
+  LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c2));
+  assert(std::copy(c2.cbegin(), c2.cend(), c1.begin()) == c1.end());
+  assert(c1 == c2);
+  LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c1));
+  LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c2));
+  assert(std::copy(c1.cbegin(), c1.cend(), RAI(c2.begin())) == RAI(c2.end()));
+  assert(c1 == c2);
+  LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c1));
+  LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c2));
+  assert(std::copy(c2.cbegin(), c2.cend(), RAI(c1.begin())) == RAI(c1.end()));
+  assert(c1 == c2);
+  LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c1));
+  LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c2));
+  assert(std::copy(RACI(c1.cbegin()), RACI(c1.cend()), c2.begin()) == c2.end());
+  assert(c1 == c2);
+  LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c1));
+  LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c2));
+  assert(std::copy(ICI(c2.cbegin()), ICI(c2.cend()), c1.begin()) == c1.end());
+  assert(c1 == c2);
+  LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c1));
+  LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c2));
 }
 
-int main(int, char**)
-{
-    {
-    int rng[] = {0, 1, 2, 3, 1023, 1024, 1025, 2047, 2048, 2049};
-    const int N = sizeof(rng)/sizeof(rng[0]);
+int main(int, char**) {
+  {
+    int rng[]   = {0, 1, 2, 3, 1023, 1024, 1025, 2047, 2048, 2049};
+    const int N = sizeof(rng) / sizeof(rng[0]);
     for (int i = 0; i < N; ++i)
-        for (int j = 0; j < N; ++j)
-            testN<std::deque<int> >(rng[i], rng[j]);
-    }
+      for (int j = 0; j < N; ++j)
+        testN<std::deque<int> >(rng[i], rng[j]);
+  }
 #if TEST_STD_VER >= 11
-    {
-    int rng[] = {0, 1, 2, 3, 1023, 1024, 1025, 2047, 2048, 2049};
-    const int N = sizeof(rng)/sizeof(rng[0]);
+  {
+    int rng[]   = {0, 1, 2, 3, 1023, 1024, 1025, 2047, 2048, 2049};
+    const int N = sizeof(rng) / sizeof(rng[0]);
     for (int i = 0; i < N; ++i)
-        for (int j = 0; j < N; ++j)
-            testN<std::deque<int, min_allocator<int>> >(rng[i], rng[j]);
-    }
+      for (int j = 0; j < N; ++j)
+        testN<std::deque<int, min_allocator<int>> >(rng[i], rng[j]);
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/sequences/deque/deque.special/copy_backward.pass.cpp b/libcxx/test/std/containers/sequences/deque/deque.special/copy_backward.pass.cpp
index bba4d173745cb..1414f03a549c1 100644
--- a/libcxx/test/std/containers/sequences/deque/deque.special/copy_backward.pass.cpp
+++ b/libcxx/test/std/containers/sequences/deque/deque.special/copy_backward.pass.cpp
@@ -23,79 +23,74 @@
 #include "min_allocator.h"
 
 template <class C>
-C
-make(int size, int start = 0 )
-{
-    const int b = 4096 / sizeof(int);
-    int init = 0;
-    if (start > 0)
-    {
-        init = (start+1) / b + ((start+1) % b != 0);
-        init *= b;
-        --init;
-    }
-    C c(init, 0);
-    for (int i = 0; i < init-start; ++i)
-        c.pop_back();
-    for (int i = 0; i < size; ++i)
-        c.push_back(i);
-    for (int i = 0; i < start; ++i)
-        c.pop_front();
-    return c;
+C make(int size, int start = 0) {
+  const int b = 4096 / sizeof(int);
+  int init    = 0;
+  if (start > 0) {
+    init = (start + 1) / b + ((start + 1) % b != 0);
+    init *= b;
+    --init;
+  }
+  C c(init, 0);
+  for (int i = 0; i < init - start; ++i)
+    c.pop_back();
+  for (int i = 0; i < size; ++i)
+    c.push_back(i);
+  for (int i = 0; i < start; ++i)
+    c.pop_front();
+  return c;
 }
 
 template <class C>
-void testN(int start, int N)
-{
-    typedef typename C::iterator I;
-    typedef typename C::const_iterator CI;
-    typedef random_access_iterator<I> RAI;
-    typedef random_access_iterator<CI> RACI;
-    C c1 = make<C>(N, start);
-    C c2 = make<C>(N);
-    assert(std::copy_backward(c1.cbegin(), c1.cend(), c2.end()) == c2.begin());
-    assert(c1 == c2);
-    LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c1));
-    LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c2));
-    assert(std::copy_backward(c2.cbegin(), c2.cend(), c1.end()) == c1.begin());
-    assert(c1 == c2);
-    LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c1));
-    LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c2));
-    assert(std::copy_backward(c1.cbegin(), c1.cend(), RAI(c2.end())) == RAI(c2.begin()));
-    assert(c1 == c2);
-    LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c1));
-    LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c2));
-    assert(std::copy_backward(c2.cbegin(), c2.cend(), RAI(c1.end())) == RAI(c1.begin()));
-    assert(c1 == c2);
-    LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c1));
-    LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c2));
-    assert(std::copy_backward(RACI(c1.cbegin()), RACI(c1.cend()), c2.end()) == c2.begin());
-    assert(c1 == c2);
-    LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c1));
-    LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c2));
-    assert(std::copy_backward(RACI(c2.cbegin()), RACI(c2.cend()), c1.end()) == c1.begin());
-    assert(c1 == c2);
-    LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c1));
-    LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c2));
+void testN(int start, int N) {
+  typedef typename C::iterator I;
+  typedef typename C::const_iterator CI;
+  typedef random_access_iterator<I> RAI;
+  typedef random_access_iterator<CI> RACI;
+  C c1 = make<C>(N, start);
+  C c2 = make<C>(N);
+  assert(std::copy_backward(c1.cbegin(), c1.cend(), c2.end()) == c2.begin());
+  assert(c1 == c2);
+  LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c1));
+  LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c2));
+  assert(std::copy_backward(c2.cbegin(), c2.cend(), c1.end()) == c1.begin());
+  assert(c1 == c2);
+  LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c1));
+  LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c2));
+  assert(std::copy_backward(c1.cbegin(), c1.cend(), RAI(c2.end())) == RAI(c2.begin()));
+  assert(c1 == c2);
+  LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c1));
+  LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c2));
+  assert(std::copy_backward(c2.cbegin(), c2.cend(), RAI(c1.end())) == RAI(c1.begin()));
+  assert(c1 == c2);
+  LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c1));
+  LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c2));
+  assert(std::copy_backward(RACI(c1.cbegin()), RACI(c1.cend()), c2.end()) == c2.begin());
+  assert(c1 == c2);
+  LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c1));
+  LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c2));
+  assert(std::copy_backward(RACI(c2.cbegin()), RACI(c2.cend()), c1.end()) == c1.begin());
+  assert(c1 == c2);
+  LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c1));
+  LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c2));
 }
 
-int main(int, char**)
-{
-    {
-    int rng[] = {0, 1, 2, 3, 1023, 1024, 1025, 2047, 2048, 2049};
-    const int N = sizeof(rng)/sizeof(rng[0]);
+int main(int, char**) {
+  {
+    int rng[]   = {0, 1, 2, 3, 1023, 1024, 1025, 2047, 2048, 2049};
+    const int N = sizeof(rng) / sizeof(rng[0]);
     for (int i = 0; i < N; ++i)
-        for (int j = 0; j < N; ++j)
-            testN<std::deque<int> >(rng[i], rng[j]);
-    }
+      for (int j = 0; j < N; ++j)
+        testN<std::deque<int> >(rng[i], rng[j]);
+  }
 #if TEST_STD_VER >= 11
-    {
-    int rng[] = {0, 1, 2, 3, 1023, 1024, 1025, 2047, 2048, 2049};
-    const int N = sizeof(rng)/sizeof(rng[0]);
+  {
+    int rng[]   = {0, 1, 2, 3, 1023, 1024, 1025, 2047, 2048, 2049};
+    const int N = sizeof(rng) / sizeof(rng[0]);
     for (int i = 0; i < N; ++i)
-        for (int j = 0; j < N; ++j)
-            testN<std::deque<int, min_allocator<int>> >(rng[i], rng[j]);
-    }
+      for (int j = 0; j < N; ++j)
+        testN<std::deque<int, min_allocator<int>> >(rng[i], rng[j]);
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/sequences/deque/deque.special/move.pass.cpp b/libcxx/test/std/containers/sequences/deque/deque.special/move.pass.cpp
index 497fac1063b6f..4b24141db3cee 100644
--- a/libcxx/test/std/containers/sequences/deque/deque.special/move.pass.cpp
+++ b/libcxx/test/std/containers/sequences/deque/deque.special/move.pass.cpp
@@ -23,79 +23,74 @@
 #include "min_allocator.h"
 
 template <class C>
-C
-make(int size, int start = 0 )
-{
-    const int b = 4096 / sizeof(int);
-    int init = 0;
-    if (start > 0)
-    {
-        init = (start+1) / b + ((start+1) % b != 0);
-        init *= b;
-        --init;
-    }
-    C c(init, 0);
-    for (int i = 0; i < init-start; ++i)
-        c.pop_back();
-    for (int i = 0; i < size; ++i)
-        c.push_back(i);
-    for (int i = 0; i < start; ++i)
-        c.pop_front();
-    return c;
+C make(int size, int start = 0) {
+  const int b = 4096 / sizeof(int);
+  int init    = 0;
+  if (start > 0) {
+    init = (start + 1) / b + ((start + 1) % b != 0);
+    init *= b;
+    --init;
+  }
+  C c(init, 0);
+  for (int i = 0; i < init - start; ++i)
+    c.pop_back();
+  for (int i = 0; i < size; ++i)
+    c.push_back(i);
+  for (int i = 0; i < start; ++i)
+    c.pop_front();
+  return c;
 }
 
 template <class C>
-void testN(int start, int N)
-{
-    typedef typename C::iterator I;
-    typedef typename C::const_iterator CI;
-    typedef random_access_iterator<I> RAI;
-    typedef random_access_iterator<CI> RACI;
-    C c1 = make<C>(N, start);
-    C c2 = make<C>(N);
-    assert(std::move(c1.cbegin(), c1.cend(), c2.begin()) == c2.end());
-    assert(c1 == c2);
-    LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c1));
-    LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c2));
-    assert(std::move(c2.cbegin(), c2.cend(), c1.begin()) == c1.end());
-    assert(c1 == c2);
-    LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c1));
-    LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c2));
-    assert(std::move(c1.cbegin(), c1.cend(), RAI(c2.begin())) == RAI(c2.end()));
-    assert(c1 == c2);
-    LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c1));
-    LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c2));
-    assert(std::move(c2.cbegin(), c2.cend(), RAI(c1.begin())) == RAI(c1.end()));
-    assert(c1 == c2);
-    LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c1));
-    LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c2));
-    assert(std::move(RACI(c1.cbegin()), RACI(c1.cend()), c2.begin()) == c2.end());
-    assert(c1 == c2);
-    LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c1));
-    LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c2));
-    assert(std::move(RACI(c2.cbegin()), RACI(c2.cend()), c1.begin()) == c1.end());
-    assert(c1 == c2);
-    LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c1));
-    LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c2));
+void testN(int start, int N) {
+  typedef typename C::iterator I;
+  typedef typename C::const_iterator CI;
+  typedef random_access_iterator<I> RAI;
+  typedef random_access_iterator<CI> RACI;
+  C c1 = make<C>(N, start);
+  C c2 = make<C>(N);
+  assert(std::move(c1.cbegin(), c1.cend(), c2.begin()) == c2.end());
+  assert(c1 == c2);
+  LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c1));
+  LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c2));
+  assert(std::move(c2.cbegin(), c2.cend(), c1.begin()) == c1.end());
+  assert(c1 == c2);
+  LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c1));
+  LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c2));
+  assert(std::move(c1.cbegin(), c1.cend(), RAI(c2.begin())) == RAI(c2.end()));
+  assert(c1 == c2);
+  LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c1));
+  LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c2));
+  assert(std::move(c2.cbegin(), c2.cend(), RAI(c1.begin())) == RAI(c1.end()));
+  assert(c1 == c2);
+  LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c1));
+  LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c2));
+  assert(std::move(RACI(c1.cbegin()), RACI(c1.cend()), c2.begin()) == c2.end());
+  assert(c1 == c2);
+  LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c1));
+  LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c2));
+  assert(std::move(RACI(c2.cbegin()), RACI(c2.cend()), c1.begin()) == c1.end());
+  assert(c1 == c2);
+  LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c1));
+  LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c2));
 }
 
-int main(int, char**)
-{
-    {
-    int rng[] = {0, 1, 2, 3, 1023, 1024, 1025, 2047, 2048, 2049};
-    const int N = sizeof(rng)/sizeof(rng[0]);
+int main(int, char**) {
+  {
+    int rng[]   = {0, 1, 2, 3, 1023, 1024, 1025, 2047, 2048, 2049};
+    const int N = sizeof(rng) / sizeof(rng[0]);
     for (int i = 0; i < N; ++i)
-        for (int j = 0; j < N; ++j)
-            testN<std::deque<int> >(rng[i], rng[j]);
-    }
+      for (int j = 0; j < N; ++j)
+        testN<std::deque<int> >(rng[i], rng[j]);
+  }
 #if TEST_STD_VER >= 11
-    {
-    int rng[] = {0, 1, 2, 3, 1023, 1024, 1025, 2047, 2048, 2049};
-    const int N = sizeof(rng)/sizeof(rng[0]);
+  {
+    int rng[]   = {0, 1, 2, 3, 1023, 1024, 1025, 2047, 2048, 2049};
+    const int N = sizeof(rng) / sizeof(rng[0]);
     for (int i = 0; i < N; ++i)
-        for (int j = 0; j < N; ++j)
-            testN<std::deque<int, min_allocator<int>> >(rng[i], rng[j]);
-    }
+      for (int j = 0; j < N; ++j)
+        testN<std::deque<int, min_allocator<int>> >(rng[i], rng[j]);
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/sequences/deque/deque.special/move_backward.pass.cpp b/libcxx/test/std/containers/sequences/deque/deque.special/move_backward.pass.cpp
index 5a261304ff684..726abe7617aef 100644
--- a/libcxx/test/std/containers/sequences/deque/deque.special/move_backward.pass.cpp
+++ b/libcxx/test/std/containers/sequences/deque/deque.special/move_backward.pass.cpp
@@ -23,79 +23,74 @@
 #include "min_allocator.h"
 
 template <class C>
-C
-make(int size, int start = 0 )
-{
-    const int b = 4096 / sizeof(int);
-    int init = 0;
-    if (start > 0)
-    {
-        init = (start+1) / b + ((start+1) % b != 0);
-        init *= b;
-        --init;
-    }
-    C c(init, 0);
-    for (int i = 0; i < init-start; ++i)
-        c.pop_back();
-    for (int i = 0; i < size; ++i)
-        c.push_back(i);
-    for (int i = 0; i < start; ++i)
-        c.pop_front();
-    return c;
+C make(int size, int start = 0) {
+  const int b = 4096 / sizeof(int);
+  int init    = 0;
+  if (start > 0) {
+    init = (start + 1) / b + ((start + 1) % b != 0);
+    init *= b;
+    --init;
+  }
+  C c(init, 0);
+  for (int i = 0; i < init - start; ++i)
+    c.pop_back();
+  for (int i = 0; i < size; ++i)
+    c.push_back(i);
+  for (int i = 0; i < start; ++i)
+    c.pop_front();
+  return c;
 }
 
 template <class C>
-void testN(int start, int N)
-{
-    typedef typename C::iterator I;
-    typedef typename C::const_iterator CI;
-    typedef random_access_iterator<I> RAI;
-    typedef random_access_iterator<CI> RACI;
-    C c1 = make<C>(N, start);
-    C c2 = make<C>(N);
-    assert(std::move_backward(c1.cbegin(), c1.cend(), c2.end()) == c2.begin());
-    assert(c1 == c2);
-    LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c1));
-    LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c2));
-    assert(std::move_backward(c2.cbegin(), c2.cend(), c1.end()) == c1.begin());
-    assert(c1 == c2);
-    LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c1));
-    LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c2));
-    assert(std::move_backward(c1.cbegin(), c1.cend(), RAI(c2.end())) == RAI(c2.begin()));
-    assert(c1 == c2);
-    LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c1));
-    LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c2));
-    assert(std::move_backward(c2.cbegin(), c2.cend(), RAI(c1.end())) == RAI(c1.begin()));
-    assert(c1 == c2);
-    LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c1));
-    LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c2));
-    assert(std::move_backward(RACI(c1.cbegin()), RACI(c1.cend()), c2.end()) == c2.begin());
-    assert(c1 == c2);
-    LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c1));
-    LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c2));
-    assert(std::move_backward(RACI(c2.cbegin()), RACI(c2.cend()), c1.end()) == c1.begin());
-    assert(c1 == c2);
-    LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c1));
-    LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c2));
+void testN(int start, int N) {
+  typedef typename C::iterator I;
+  typedef typename C::const_iterator CI;
+  typedef random_access_iterator<I> RAI;
+  typedef random_access_iterator<CI> RACI;
+  C c1 = make<C>(N, start);
+  C c2 = make<C>(N);
+  assert(std::move_backward(c1.cbegin(), c1.cend(), c2.end()) == c2.begin());
+  assert(c1 == c2);
+  LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c1));
+  LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c2));
+  assert(std::move_backward(c2.cbegin(), c2.cend(), c1.end()) == c1.begin());
+  assert(c1 == c2);
+  LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c1));
+  LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c2));
+  assert(std::move_backward(c1.cbegin(), c1.cend(), RAI(c2.end())) == RAI(c2.begin()));
+  assert(c1 == c2);
+  LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c1));
+  LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c2));
+  assert(std::move_backward(c2.cbegin(), c2.cend(), RAI(c1.end())) == RAI(c1.begin()));
+  assert(c1 == c2);
+  LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c1));
+  LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c2));
+  assert(std::move_backward(RACI(c1.cbegin()), RACI(c1.cend()), c2.end()) == c2.begin());
+  assert(c1 == c2);
+  LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c1));
+  LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c2));
+  assert(std::move_backward(RACI(c2.cbegin()), RACI(c2.cend()), c1.end()) == c1.begin());
+  assert(c1 == c2);
+  LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c1));
+  LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c2));
 }
 
-int main(int, char**)
-{
-    {
-    int rng[] = {0, 1, 2, 3, 1023, 1024, 1025, 2047, 2048, 2049};
-    const int N = sizeof(rng)/sizeof(rng[0]);
+int main(int, char**) {
+  {
+    int rng[]   = {0, 1, 2, 3, 1023, 1024, 1025, 2047, 2048, 2049};
+    const int N = sizeof(rng) / sizeof(rng[0]);
     for (int i = 0; i < N; ++i)
-        for (int j = 0; j < N; ++j)
-            testN<std::deque<int> >(rng[i], rng[j]);
-    }
+      for (int j = 0; j < N; ++j)
+        testN<std::deque<int> >(rng[i], rng[j]);
+  }
 #if TEST_STD_VER >= 11
-    {
-    int rng[] = {0, 1, 2, 3, 1023, 1024, 1025, 2047, 2048, 2049};
-    const int N = sizeof(rng)/sizeof(rng[0]);
+  {
+    int rng[]   = {0, 1, 2, 3, 1023, 1024, 1025, 2047, 2048, 2049};
+    const int N = sizeof(rng) / sizeof(rng[0]);
     for (int i = 0; i < N; ++i)
-        for (int j = 0; j < N; ++j)
-            testN<std::deque<int, min_allocator<int> > >(rng[i], rng[j]);
-    }
+      for (int j = 0; j < N; ++j)
+        testN<std::deque<int, min_allocator<int> > >(rng[i], rng[j]);
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/sequences/deque/deque.special/swap.pass.cpp b/libcxx/test/std/containers/sequences/deque/deque.special/swap.pass.cpp
index e5816326ffac6..07004315a0bfb 100644
--- a/libcxx/test/std/containers/sequences/deque/deque.special/swap.pass.cpp
+++ b/libcxx/test/std/containers/sequences/deque/deque.special/swap.pass.cpp
@@ -19,104 +19,99 @@
 #include "min_allocator.h"
 
 template <class C>
-C
-make(int size, int start = 0 )
-{
-    const int b = 4096 / sizeof(int);
-    int init = 0;
-    if (start > 0)
-    {
-        init = (start+1) / b + ((start+1) % b != 0);
-        init *= b;
-        --init;
-    }
-    C c(init, 0);
-    for (int i = 0; i < init-start; ++i)
-        c.pop_back();
-    for (int i = 0; i < size; ++i)
-        c.push_back(i);
-    for (int i = 0; i < start; ++i)
-        c.pop_front();
-    return c;
+C make(int size, int start = 0) {
+  const int b = 4096 / sizeof(int);
+  int init    = 0;
+  if (start > 0) {
+    init = (start + 1) / b + ((start + 1) % b != 0);
+    init *= b;
+    --init;
+  }
+  C c(init, 0);
+  for (int i = 0; i < init - start; ++i)
+    c.pop_back();
+  for (int i = 0; i < size; ++i)
+    c.push_back(i);
+  for (int i = 0; i < start; ++i)
+    c.pop_front();
+  return c;
 }
 
 template <class C>
-void testN(int start, int N, int M)
-{
-    C c1 = make<C>(N, start);
-    C c2 = make<C>(M);
-    C c1_save = c1;
-    C c2_save = c2;
+void testN(int start, int N, int M) {
+  C c1      = make<C>(N, start);
+  C c2      = make<C>(M);
+  C c1_save = c1;
+  C c2_save = c2;
+  swap(c1, c2);
+  assert(c1 == c2_save);
+  assert(c2 == c1_save);
+  LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c1));
+  LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c2));
+  LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c1_save));
+  LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c2_save));
+}
+
+int main(int, char**) {
+  {
+    int rng[]   = {0, 1, 2, 3, 1023, 1024, 1025, 2047, 2048, 2049};
+    const int N = sizeof(rng) / sizeof(rng[0]);
+    for (int i = 0; i < N; ++i)
+      for (int j = 0; j < N; ++j)
+        for (int k = 0; k < N; ++k)
+          testN<std::deque<int> >(rng[i], rng[j], rng[k]);
+  }
+  {
+    int a1[] = {1, 3, 7, 9, 10};
+    int a2[] = {0, 2, 4, 5, 6, 8, 11};
+    typedef test_allocator<int> A;
+    std::deque<int, A> c1(a1, a1 + sizeof(a1) / sizeof(a1[0]), A(1, 1));
+    std::deque<int, A> c2(a2, a2 + sizeof(a2) / sizeof(a2[0]), A(1, 2));
     swap(c1, c2);
-    assert(c1 == c2_save);
-    assert(c2 == c1_save);
+    assert((c1 == std::deque<int, A>(a2, a2 + sizeof(a2) / sizeof(a2[0]))));
+    assert(c1.get_allocator().get_id() == 1);
+    assert((c2 == std::deque<int, A>(a1, a1 + sizeof(a1) / sizeof(a1[0]))));
+    assert(c2.get_allocator().get_id() == 2);
     LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c1));
     LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c2));
-    LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c1_save));
-    LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c2_save));
-}
-
-int main(int, char**)
-{
-    {
-        int rng[] = {0, 1, 2, 3, 1023, 1024, 1025, 2047, 2048, 2049};
-        const int N = sizeof(rng)/sizeof(rng[0]);
-        for (int i = 0; i < N; ++i)
-            for (int j = 0; j < N; ++j)
-                for (int k = 0; k < N; ++k)
-                    testN<std::deque<int> >(rng[i], rng[j], rng[k]);
-    }
-    {
-        int a1[] = {1, 3, 7, 9, 10};
-        int a2[] = {0, 2, 4, 5, 6, 8, 11};
-        typedef test_allocator<int> A;
-        std::deque<int, A> c1(a1, a1+sizeof(a1)/sizeof(a1[0]), A(1, 1));
-        std::deque<int, A> c2(a2, a2+sizeof(a2)/sizeof(a2[0]), A(1, 2));
-        swap(c1, c2);
-        assert((c1 == std::deque<int, A>(a2, a2+sizeof(a2)/sizeof(a2[0]))));
-        assert(c1.get_allocator().get_id() == 1);
-        assert((c2 == std::deque<int, A>(a1, a1+sizeof(a1)/sizeof(a1[0]))));
-        assert(c2.get_allocator().get_id() == 2);
-        LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c1));
-        LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c2));
-    }
-    {
-        int a1[] = {1, 3, 7, 9, 10};
-        int a2[] = {0, 2, 4, 5, 6, 8, 11};
-        typedef other_allocator<int> A;
-        std::deque<int, A> c1(a1, a1+sizeof(a1)/sizeof(a1[0]), A(1));
-        std::deque<int, A> c2(a2, a2+sizeof(a2)/sizeof(a2[0]), A(2));
-        swap(c1, c2);
-        assert((c1 == std::deque<int, A>(a2, a2+sizeof(a2)/sizeof(a2[0]))));
-        assert(c1.get_allocator() == A(2));
-        assert((c2 == std::deque<int, A>(a1, a1+sizeof(a1)/sizeof(a1[0]))));
-        assert(c2.get_allocator() == A(1));
-        LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c1));
-        LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c2));
-    }
+  }
+  {
+    int a1[] = {1, 3, 7, 9, 10};
+    int a2[] = {0, 2, 4, 5, 6, 8, 11};
+    typedef other_allocator<int> A;
+    std::deque<int, A> c1(a1, a1 + sizeof(a1) / sizeof(a1[0]), A(1));
+    std::deque<int, A> c2(a2, a2 + sizeof(a2) / sizeof(a2[0]), A(2));
+    swap(c1, c2);
+    assert((c1 == std::deque<int, A>(a2, a2 + sizeof(a2) / sizeof(a2[0]))));
+    assert(c1.get_allocator() == A(2));
+    assert((c2 == std::deque<int, A>(a1, a1 + sizeof(a1) / sizeof(a1[0]))));
+    assert(c2.get_allocator() == A(1));
+    LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c1));
+    LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c2));
+  }
 #if TEST_STD_VER >= 11
-    {
-        int rng[] = {0, 1, 2, 3, 1023, 1024, 1025, 2047, 2048, 2049};
-        const int N = sizeof(rng)/sizeof(rng[0]);
-        for (int i = 0; i < N; ++i)
-            for (int j = 0; j < N; ++j)
-                for (int k = 0; k < N; ++k)
-                    testN<std::deque<int, min_allocator<int>> >(rng[i], rng[j], rng[k]);
-    }
-    {
-        int a1[] = {1, 3, 7, 9, 10};
-        int a2[] = {0, 2, 4, 5, 6, 8, 11};
-        typedef min_allocator<int> A;
-        std::deque<int, A> c1(a1, a1+sizeof(a1)/sizeof(a1[0]), A());
-        std::deque<int, A> c2(a2, a2+sizeof(a2)/sizeof(a2[0]), A());
-        swap(c1, c2);
-        assert((c1 == std::deque<int, A>(a2, a2+sizeof(a2)/sizeof(a2[0]))));
-        assert(c1.get_allocator() == A());
-        assert((c2 == std::deque<int, A>(a1, a1+sizeof(a1)/sizeof(a1[0]))));
-        assert(c2.get_allocator() == A());
-        LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c1));
-        LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c2));
-    }
+  {
+    int rng[]   = {0, 1, 2, 3, 1023, 1024, 1025, 2047, 2048, 2049};
+    const int N = sizeof(rng) / sizeof(rng[0]);
+    for (int i = 0; i < N; ++i)
+      for (int j = 0; j < N; ++j)
+        for (int k = 0; k < N; ++k)
+          testN<std::deque<int, min_allocator<int>> >(rng[i], rng[j], rng[k]);
+  }
+  {
+    int a1[] = {1, 3, 7, 9, 10};
+    int a2[] = {0, 2, 4, 5, 6, 8, 11};
+    typedef min_allocator<int> A;
+    std::deque<int, A> c1(a1, a1 + sizeof(a1) / sizeof(a1[0]), A());
+    std::deque<int, A> c2(a2, a2 + sizeof(a2) / sizeof(a2[0]), A());
+    swap(c1, c2);
+    assert((c1 == std::deque<int, A>(a2, a2 + sizeof(a2) / sizeof(a2[0]))));
+    assert(c1.get_allocator() == A());
+    assert((c2 == std::deque<int, A>(a1, a1 + sizeof(a1) / sizeof(a1[0]))));
+    assert(c2.get_allocator() == A());
+    LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c1));
+    LIBCPP_ASSERT(is_double_ended_contiguous_container_asan_correct(c2));
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/sequences/deque/deque.special/swap_noexcept.pass.cpp b/libcxx/test/std/containers/sequences/deque/deque.special/swap_noexcept.pass.cpp
index d6257b13aaaec..6347496d67c15 100644
--- a/libcxx/test/std/containers/sequences/deque/deque.special/swap_noexcept.pass.cpp
+++ b/libcxx/test/std/containers/sequences/deque/deque.special/swap_noexcept.pass.cpp
@@ -28,65 +28,61 @@
 #include "test_allocator.h"
 
 template <class T>
-struct some_alloc
-{
-    typedef T value_type;
+struct some_alloc {
+  typedef T value_type;
 
-    some_alloc() {}
-    some_alloc(const some_alloc&);
-    void allocate(std::size_t);
-    void deallocate(void*, unsigned) {}
+  some_alloc() {}
+  some_alloc(const some_alloc&);
+  void allocate(std::size_t);
+  void deallocate(void*, unsigned) {}
 
-    typedef std::true_type propagate_on_container_swap;
+  typedef std::true_type propagate_on_container_swap;
 };
 
 template <class T>
-struct some_alloc2
-{
-    typedef T value_type;
+struct some_alloc2 {
+  typedef T value_type;
 
-    some_alloc2() {}
-    some_alloc2(const some_alloc2&);
-    void allocate(std::size_t);
-    void deallocate(void*, unsigned) {}
+  some_alloc2() {}
+  some_alloc2(const some_alloc2&);
+  void allocate(std::size_t);
+  void deallocate(void*, unsigned) {}
 
-    typedef std::false_type propagate_on_container_swap;
-    typedef std::true_type is_always_equal;
+  typedef std::false_type propagate_on_container_swap;
+  typedef std::true_type is_always_equal;
 };
 
-int main(int, char**)
-{
-    {
-        typedef std::deque<MoveOnly> C;
-        static_assert(noexcept(swap(std::declval<C&>(), std::declval<C&>())), "");
-    }
+int main(int, char**) {
+  {
+    typedef std::deque<MoveOnly> C;
+    static_assert(noexcept(swap(std::declval<C&>(), std::declval<C&>())), "");
+  }
 #if defined(_LIBCPP_VERSION)
-    {
-        typedef std::deque<MoveOnly, test_allocator<MoveOnly>> C;
-        static_assert(noexcept(swap(std::declval<C&>(), std::declval<C&>())), "");
-    }
-    {
-        typedef std::deque<MoveOnly, other_allocator<MoveOnly>> C;
-        static_assert(noexcept(swap(std::declval<C&>(), std::declval<C&>())), "");
-    }
+  {
+    typedef std::deque<MoveOnly, test_allocator<MoveOnly>> C;
+    static_assert(noexcept(swap(std::declval<C&>(), std::declval<C&>())), "");
+  }
+  {
+    typedef std::deque<MoveOnly, other_allocator<MoveOnly>> C;
+    static_assert(noexcept(swap(std::declval<C&>(), std::declval<C&>())), "");
+  }
 #endif // _LIBCPP_VERSION
-    {
-        typedef std::deque<MoveOnly, some_alloc<MoveOnly>> C;
+  {
+    typedef std::deque<MoveOnly, some_alloc<MoveOnly>> C;
 #if TEST_STD_VER >= 14
     //  In C++14, if POCS is set, swapping the allocator is required not to throw
-        static_assert( noexcept(swap(std::declval<C&>(), std::declval<C&>())), "");
+    static_assert(noexcept(swap(std::declval<C&>(), std::declval<C&>())), "");
 #else
-        static_assert(!noexcept(swap(std::declval<C&>(), std::declval<C&>())), "");
+    static_assert(!noexcept(swap(std::declval<C&>(), std::declval<C&>())), "");
 #endif
-    }
+  }
 #if TEST_STD_VER >= 14
-    {
-        typedef std::deque<MoveOnly, some_alloc2<MoveOnly>> C;
+  {
+    typedef std::deque<MoveOnly, some_alloc2<MoveOnly>> C;
     //  if the allocators are always equal, then the swap can be noexcept
-        static_assert( noexcept(swap(std::declval<C&>(), std::declval<C&>())), "");
-    }
+    static_assert(noexcept(swap(std::declval<C&>(), std::declval<C&>())), "");
+  }
 #endif
 
-
   return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/deque/get_allocator.pass.cpp b/libcxx/test/std/containers/sequences/deque/get_allocator.pass.cpp
index e42dbb7d75153..6b8bc6c555bcd 100644
--- a/libcxx/test/std/containers/sequences/deque/get_allocator.pass.cpp
+++ b/libcxx/test/std/containers/sequences/deque/get_allocator.pass.cpp
@@ -19,16 +19,16 @@
 #include "test_macros.h"
 
 int main(int, char**) {
-    {
-        std::allocator<int> alloc;
-        const std::deque<int> d(alloc);
-        assert(d.get_allocator() == alloc);
-    }
-    {
-        other_allocator<int> alloc(1);
-        const std::deque<int, other_allocator<int> > d(alloc);
-        assert(d.get_allocator() == alloc);
-    }
+  {
+    std::allocator<int> alloc;
+    const std::deque<int> d(alloc);
+    assert(d.get_allocator() == alloc);
+  }
+  {
+    other_allocator<int> alloc(1);
+    const std::deque<int, other_allocator<int> > d(alloc);
+    assert(d.get_allocator() == alloc);
+  }
 
-    return 0;
+  return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/deque/iterator_concept_conformance.compile.pass.cpp b/libcxx/test/std/containers/sequences/deque/iterator_concept_conformance.compile.pass.cpp
index 3b0aa36f02aac..02c1ecf78b0af 100644
--- a/libcxx/test/std/containers/sequences/deque/iterator_concept_conformance.compile.pass.cpp
+++ b/libcxx/test/std/containers/sequences/deque/iterator_concept_conformance.compile.pass.cpp
@@ -14,11 +14,11 @@
 
 #include <iterator>
 
-using iterator = std::deque<int>::iterator;
-using const_iterator = std::deque<int>::const_iterator;
-using reverse_iterator = std::deque<int>::reverse_iterator;
+using iterator               = std::deque<int>::iterator;
+using const_iterator         = std::deque<int>::const_iterator;
+using reverse_iterator       = std::deque<int>::reverse_iterator;
 using const_reverse_iterator = std::deque<int>::const_reverse_iterator;
-using value_type = int;
+using value_type             = int;
 
 static_assert(std::random_access_iterator<iterator>);
 static_assert(!std::contiguous_iterator<iterator>);
@@ -31,20 +31,20 @@ static_assert(std::sized_sentinel_for<iterator, iterator>);
 static_assert(std::sized_sentinel_for<iterator, const_iterator>);
 static_assert(!std::sized_sentinel_for<iterator, reverse_iterator>);
 static_assert(!std::sized_sentinel_for<iterator, const_reverse_iterator>);
-static_assert( std::indirectly_movable<iterator, iterator>);
-static_assert( std::indirectly_movable_storable<iterator, iterator>);
+static_assert(std::indirectly_movable<iterator, iterator>);
+static_assert(std::indirectly_movable_storable<iterator, iterator>);
 static_assert(!std::indirectly_movable<iterator, const_iterator>);
 static_assert(!std::indirectly_movable_storable<iterator, const_iterator>);
-static_assert( std::indirectly_movable<iterator, reverse_iterator>);
-static_assert( std::indirectly_movable_storable<iterator, reverse_iterator>);
+static_assert(std::indirectly_movable<iterator, reverse_iterator>);
+static_assert(std::indirectly_movable_storable<iterator, reverse_iterator>);
 static_assert(!std::indirectly_movable<iterator, const_reverse_iterator>);
 static_assert(!std::indirectly_movable_storable<iterator, const_reverse_iterator>);
-static_assert( std::indirectly_copyable<iterator, iterator>);
-static_assert( std::indirectly_copyable_storable<iterator, iterator>);
+static_assert(std::indirectly_copyable<iterator, iterator>);
+static_assert(std::indirectly_copyable_storable<iterator, iterator>);
 static_assert(!std::indirectly_copyable<iterator, const_iterator>);
 static_assert(!std::indirectly_copyable_storable<iterator, const_iterator>);
-static_assert( std::indirectly_copyable<iterator, reverse_iterator>);
-static_assert( std::indirectly_copyable_storable<iterator, reverse_iterator>);
+static_assert(std::indirectly_copyable<iterator, reverse_iterator>);
+static_assert(std::indirectly_copyable_storable<iterator, reverse_iterator>);
 static_assert(!std::indirectly_copyable<iterator, const_reverse_iterator>);
 static_assert(!std::indirectly_copyable_storable<iterator, const_reverse_iterator>);
 static_assert(std::indirectly_swappable<iterator, iterator>);
@@ -60,20 +60,20 @@ static_assert(std::sized_sentinel_for<const_iterator, iterator>);
 static_assert(std::sized_sentinel_for<const_iterator, const_iterator>);
 static_assert(!std::sized_sentinel_for<const_iterator, reverse_iterator>);
 static_assert(!std::sized_sentinel_for<const_iterator, const_reverse_iterator>);
-static_assert( std::indirectly_movable<const_iterator, iterator>);
-static_assert( std::indirectly_movable_storable<const_iterator, iterator>);
+static_assert(std::indirectly_movable<const_iterator, iterator>);
+static_assert(std::indirectly_movable_storable<const_iterator, iterator>);
 static_assert(!std::indirectly_movable<const_iterator, const_iterator>);
 static_assert(!std::indirectly_movable_storable<const_iterator, const_iterator>);
-static_assert( std::indirectly_movable<const_iterator, reverse_iterator>);
-static_assert( std::indirectly_movable_storable<const_iterator, reverse_iterator>);
+static_assert(std::indirectly_movable<const_iterator, reverse_iterator>);
+static_assert(std::indirectly_movable_storable<const_iterator, reverse_iterator>);
 static_assert(!std::indirectly_movable<const_iterator, const_reverse_iterator>);
 static_assert(!std::indirectly_movable_storable<const_iterator, const_reverse_iterator>);
-static_assert( std::indirectly_copyable<const_iterator, iterator>);
-static_assert( std::indirectly_copyable_storable<const_iterator, iterator>);
+static_assert(std::indirectly_copyable<const_iterator, iterator>);
+static_assert(std::indirectly_copyable_storable<const_iterator, iterator>);
 static_assert(!std::indirectly_copyable<const_iterator, const_iterator>);
 static_assert(!std::indirectly_copyable_storable<const_iterator, const_iterator>);
-static_assert( std::indirectly_copyable<const_iterator, reverse_iterator>);
-static_assert( std::indirectly_copyable_storable<const_iterator, reverse_iterator>);
+static_assert(std::indirectly_copyable<const_iterator, reverse_iterator>);
+static_assert(std::indirectly_copyable_storable<const_iterator, reverse_iterator>);
 static_assert(!std::indirectly_copyable<const_iterator, const_reverse_iterator>);
 static_assert(!std::indirectly_copyable_storable<const_iterator, const_reverse_iterator>);
 static_assert(!std::indirectly_swappable<const_iterator, const_iterator>);
diff --git a/libcxx/test/std/containers/sequences/deque/iterators.pass.cpp b/libcxx/test/std/containers/sequences/deque/iterators.pass.cpp
index 484a2961fdb0c..337fd688ebcdb 100644
--- a/libcxx/test/std/containers/sequences/deque/iterators.pass.cpp
+++ b/libcxx/test/std/containers/sequences/deque/iterators.pass.cpp
@@ -22,9 +22,8 @@
 #include "test_macros.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
+int main(int, char**) {
+  {
     typedef std::deque<int> C;
     C c;
     C::iterator i;
@@ -32,9 +31,9 @@ int main(int, char**)
     C::const_iterator j;
     j = c.cbegin();
     assert(i == j);
-    }
+  }
 #if TEST_STD_VER >= 11
-    {
+  {
     typedef std::deque<int, min_allocator<int>> C;
     C c;
     C::iterator i;
@@ -62,48 +61,48 @@ int main(int, char**)
     std::same_as<std::strong_ordering> decltype(auto) r1 = i <=> j;
     assert(r1 == std::strong_ordering::equal);
 #  endif
-    }
+  }
 #endif
 #if TEST_STD_VER > 11
-    { // N3644 testing
-        std::deque<int>::iterator ii1{}, ii2{};
-        std::deque<int>::iterator ii4 = ii1;
-        std::deque<int>::const_iterator cii{};
-        assert ( ii1 == ii2 );
-        assert ( ii1 == ii4 );
-
-        assert (!(ii1 != ii2 ));
-
-        assert ( (ii1 == cii ));
-        assert ( (cii == ii1 ));
-        assert (!(ii1 != cii ));
-        assert (!(cii != ii1 ));
-        assert (!(ii1 <  cii ));
-        assert (!(cii <  ii1 ));
-        assert ( (ii1 <= cii ));
-        assert ( (cii <= ii1 ));
-        assert (!(ii1 >  cii ));
-        assert (!(cii >  ii1 ));
-        assert ( (ii1 >= cii ));
-        assert ( (cii >= ii1 ));
-        assert (cii - ii1 == 0);
-        assert (ii1 - cii == 0);
-
-//         std::deque<int> c;
-//         assert ( ii1 != c.cbegin());
-//         assert ( cii != c.begin());
-//         assert ( cii != c.cend());
-//         assert ( ii1 != c.end());
+  { // N3644 testing
+    std::deque<int>::iterator ii1{}, ii2{};
+    std::deque<int>::iterator ii4 = ii1;
+    std::deque<int>::const_iterator cii{};
+    assert(ii1 == ii2);
+    assert(ii1 == ii4);
+
+    assert(!(ii1 != ii2));
+
+    assert((ii1 == cii));
+    assert((cii == ii1));
+    assert(!(ii1 != cii));
+    assert(!(cii != ii1));
+    assert(!(ii1 < cii));
+    assert(!(cii < ii1));
+    assert((ii1 <= cii));
+    assert((cii <= ii1));
+    assert(!(ii1 > cii));
+    assert(!(cii > ii1));
+    assert((ii1 >= cii));
+    assert((cii >= ii1));
+    assert(cii - ii1 == 0);
+    assert(ii1 - cii == 0);
+
+    //         std::deque<int> c;
+    //         assert ( ii1 != c.cbegin());
+    //         assert ( cii != c.begin());
+    //         assert ( cii != c.cend());
+    //         assert ( ii1 != c.end());
 
 #  if TEST_STD_VER >= 20
-        // P1614 + LWG3352
-        std::same_as<std::strong_ordering> decltype(auto) r1 = ii1 <=> ii2;
-        assert(r1 == std::strong_ordering::equal);
+    // P1614 + LWG3352
+    std::same_as<std::strong_ordering> decltype(auto) r1 = ii1 <=> ii2;
+    assert(r1 == std::strong_ordering::equal);
 
-        std::same_as<std::strong_ordering> decltype(auto) r2 = cii <=> ii2;
-        assert(r2 == std::strong_ordering::equal);
+    std::same_as<std::strong_ordering> decltype(auto) r2 = cii <=> ii2;
+    assert(r2 == std::strong_ordering::equal);
 #  endif // TEST_STD_VER > 20
-    }
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/sequences/deque/range_concept_conformance.compile.pass.cpp b/libcxx/test/std/containers/sequences/deque/range_concept_conformance.compile.pass.cpp
index b413badb9de1e..925ff8d097fc3 100644
--- a/libcxx/test/std/containers/sequences/deque/range_concept_conformance.compile.pass.cpp
+++ b/libcxx/test/std/containers/sequences/deque/range_concept_conformance.compile.pass.cpp
@@ -17,7 +17,6 @@
 
 using range = std::deque<int>;
 
-
 static_assert(std::same_as<std::ranges::iterator_t<range>, range::iterator>);
 static_assert(std::ranges::common_range<range>);
 static_assert(std::ranges::random_access_range<range>);
diff --git a/libcxx/test/std/containers/sequences/deque/types.pass.cpp b/libcxx/test/std/containers/sequences/deque/types.pass.cpp
index 8c14de0c77440..8184d55873d20 100644
--- a/libcxx/test/std/containers/sequences/deque/types.pass.cpp
+++ b/libcxx/test/std/containers/sequences/deque/types.pass.cpp
@@ -38,76 +38,73 @@
 #include "min_allocator.h"
 
 template <class T, class Allocator>
-void
-test()
-{
-    typedef std::deque<T, Allocator> C;
+void test() {
+  typedef std::deque<T, Allocator> C;
 
-    static_assert((std::is_same<typename C::value_type, T>::value), "");
-    static_assert(
-        (std::is_same<typename C::value_type, typename std::allocator_traits<Allocator>::value_type>::value), "");
-    static_assert((std::is_same<typename C::allocator_type, Allocator>::value), "");
-    static_assert(
-        (std::is_same<typename C::size_type, typename std::allocator_traits<Allocator>::size_type>::value), "");
-    static_assert(
-        (std::is_same<typename C::difference_type, typename std::allocator_traits<Allocator>::difference_type>::value),
-        "");
-    static_assert(
-        (std::is_same<typename C::reference, typename std::allocator_traits<Allocator>::value_type&>::value), "");
-    static_assert((std::is_same<typename C::const_reference,
-                                const typename std::allocator_traits<Allocator>::value_type&>::value),
-                  "");
-    static_assert((std::is_same<typename C::pointer, typename std::allocator_traits<Allocator>::pointer>::value), "");
-    static_assert(
-        (std::is_same<typename C::const_pointer, typename std::allocator_traits<Allocator>::const_pointer>::value), "");
-    static_assert((std::is_same<
-        typename std::iterator_traits<typename C::iterator>::iterator_category,
-        std::random_access_iterator_tag>::value), "");
-    static_assert((std::is_same<
-        typename std::iterator_traits<typename C::const_iterator>::iterator_category,
-        std::random_access_iterator_tag>::value), "");
-    static_assert((std::is_same<
-        typename C::reverse_iterator,
-        std::reverse_iterator<typename C::iterator> >::value), "");
-    static_assert((std::is_same<
-        typename C::const_reverse_iterator,
-        std::reverse_iterator<typename C::const_iterator> >::value), "");
-    static_assert((std::is_signed<typename C::difference_type>::value), "");
-    static_assert((std::is_unsigned<typename C::size_type>::value), "");
-    static_assert((std::is_same<typename C::difference_type,
-        typename std::iterator_traits<typename C::iterator>::difference_type>::value), "");
-    static_assert((std::is_same<typename C::difference_type,
-        typename std::iterator_traits<typename C::const_iterator>::difference_type>::value), "");
+  static_assert((std::is_same<typename C::value_type, T>::value), "");
+  static_assert(
+      (std::is_same<typename C::value_type, typename std::allocator_traits<Allocator>::value_type>::value), "");
+  static_assert((std::is_same<typename C::allocator_type, Allocator>::value), "");
+  static_assert((std::is_same<typename C::size_type, typename std::allocator_traits<Allocator>::size_type>::value), "");
+  static_assert(
+      (std::is_same<typename C::difference_type, typename std::allocator_traits<Allocator>::difference_type>::value),
+      "");
+  static_assert(
+      (std::is_same<typename C::reference, typename std::allocator_traits<Allocator>::value_type&>::value), "");
+  static_assert(
+      (std::is_same<typename C::const_reference, const typename std::allocator_traits<Allocator>::value_type&>::value),
+      "");
+  static_assert((std::is_same<typename C::pointer, typename std::allocator_traits<Allocator>::pointer>::value), "");
+  static_assert(
+      (std::is_same<typename C::const_pointer, typename std::allocator_traits<Allocator>::const_pointer>::value), "");
+  static_assert((std::is_same< typename std::iterator_traits<typename C::iterator>::iterator_category,
+                               std::random_access_iterator_tag>::value),
+                "");
+  static_assert((std::is_same< typename std::iterator_traits<typename C::const_iterator>::iterator_category,
+                               std::random_access_iterator_tag>::value),
+                "");
+  static_assert((std::is_same< typename C::reverse_iterator, std::reverse_iterator<typename C::iterator> >::value), "");
+  static_assert(
+      (std::is_same< typename C::const_reverse_iterator, std::reverse_iterator<typename C::const_iterator> >::value),
+      "");
+  static_assert((std::is_signed<typename C::difference_type>::value), "");
+  static_assert((std::is_unsigned<typename C::size_type>::value), "");
+  static_assert((std::is_same<typename C::difference_type,
+                              typename std::iterator_traits<typename C::iterator>::difference_type>::value),
+                "");
+  static_assert((std::is_same<typename C::difference_type,
+                              typename std::iterator_traits<typename C::const_iterator>::difference_type>::value),
+                "");
 }
 
-int main(int, char**)
-{
-    test<int, test_allocator<int> >();
-    test<int*, std::allocator<int*> >();
-    test<Copyable, test_allocator<Copyable> >();
-    static_assert((std::is_same<std::deque<char>::allocator_type,
-                                std::allocator<char> >::value), "");
+int main(int, char**) {
+  test<int, test_allocator<int> >();
+  test<int*, std::allocator<int*> >();
+  test<Copyable, test_allocator<Copyable> >();
+  static_assert((std::is_same<std::deque<char>::allocator_type, std::allocator<char> >::value), "");
 
 #if TEST_STD_VER >= 11
-    {
-        typedef std::deque<short, min_allocator<short>> C;
-        static_assert((std::is_same<C::value_type, short>::value), "");
-        static_assert((std::is_same<C::allocator_type, min_allocator<C::value_type> >::value), "");
-        static_assert((std::is_same<C::reference, C::value_type&>::value), "");
-        static_assert((std::is_same<C::const_reference, const C::value_type&>::value), "");
-        static_assert((std::is_same<C::pointer, min_pointer<C::value_type>>::value), "");
-        static_assert((std::is_same<C::const_pointer, min_pointer<const C::value_type>>::value), "");
-//  min_allocator doesn't have a size_type, so one gets synthesized
-        static_assert((std::is_same<C::size_type, std::make_unsigned<C::difference_type>::type>::value), "");
-        static_assert((std::is_same<C::difference_type, std::ptrdiff_t>::value), "");
+  {
+    typedef std::deque<short, min_allocator<short>> C;
+    static_assert((std::is_same<C::value_type, short>::value), "");
+    static_assert((std::is_same<C::allocator_type, min_allocator<C::value_type> >::value), "");
+    static_assert((std::is_same<C::reference, C::value_type&>::value), "");
+    static_assert((std::is_same<C::const_reference, const C::value_type&>::value), "");
+    static_assert((std::is_same<C::pointer, min_pointer<C::value_type>>::value), "");
+    static_assert((std::is_same<C::const_pointer, min_pointer<const C::value_type>>::value), "");
+    //  min_allocator doesn't have a size_type, so one gets synthesized
+    static_assert((std::is_same<C::size_type, std::make_unsigned<C::difference_type>::type>::value), "");
+    static_assert((std::is_same<C::difference_type, std::ptrdiff_t>::value), "");
 
-        static_assert((std::is_signed<typename C::difference_type>::value), "");
-        static_assert((std::is_unsigned<typename C::size_type>::value), "");
-        static_assert((std::is_same<typename C::difference_type,
-            typename std::iterator_traits<typename C::iterator>::difference_type>::value), "");
-        static_assert((std::is_same<typename C::difference_type,
-            typename std::iterator_traits<typename C::const_iterator>::difference_type>::value), "");
-    }
+    static_assert((std::is_signed<typename C::difference_type>::value), "");
+    static_assert((std::is_unsigned<typename C::size_type>::value), "");
+    static_assert((std::is_same<typename C::difference_type,
+                                typename std::iterator_traits<typename C::iterator>::difference_type>::value),
+                  "");
+    static_assert((std::is_same<typename C::difference_type,
+                                typename std::iterator_traits<typename C::const_iterator>::difference_type>::value),
+                  "");
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/sequences/forwardlist/allocator_mismatch.verify.cpp b/libcxx/test/std/containers/sequences/forwardlist/allocator_mismatch.verify.cpp
index d7133a1c5ac0f..96c3a11264dc3 100644
--- a/libcxx/test/std/containers/sequences/forwardlist/allocator_mismatch.verify.cpp
+++ b/libcxx/test/std/containers/sequences/forwardlist/allocator_mismatch.verify.cpp
@@ -12,4 +12,4 @@
 #include <forward_list>
 
 std::forward_list<int, std::allocator<long> > fl;
-  // expected-error-re@*:* {{static assertion failed{{.*}}Allocator::value_type must be same type as value_type}}
+// expected-error-re@*:* {{static assertion failed{{.*}}Allocator::value_type must be same type as value_type}}
diff --git a/libcxx/test/std/containers/sequences/forwardlist/empty.pass.cpp b/libcxx/test/std/containers/sequences/forwardlist/empty.pass.cpp
index 727904c987133..dbc0631d11930 100644
--- a/libcxx/test/std/containers/sequences/forwardlist/empty.pass.cpp
+++ b/libcxx/test/std/containers/sequences/forwardlist/empty.pass.cpp
@@ -18,9 +18,8 @@
 #include "test_macros.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
+int main(int, char**) {
+  {
     typedef std::forward_list<int> C;
     C c;
     ASSERT_NOEXCEPT(c.empty());
@@ -29,9 +28,9 @@ int main(int, char**)
     assert(!c.empty());
     c.clear();
     assert(c.empty());
-    }
+  }
 #if TEST_STD_VER >= 11
-    {
+  {
     typedef std::forward_list<int, min_allocator<int>> C;
     C c;
     ASSERT_NOEXCEPT(c.empty());
@@ -40,7 +39,7 @@ int main(int, char**)
     assert(!c.empty());
     c.clear();
     assert(c.empty());
-    }
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/sequences/forwardlist/empty.verify.cpp b/libcxx/test/std/containers/sequences/forwardlist/empty.verify.cpp
index 366094b09e885..d3755acecebb9 100644
--- a/libcxx/test/std/containers/sequences/forwardlist/empty.verify.cpp
+++ b/libcxx/test/std/containers/sequences/forwardlist/empty.verify.cpp
@@ -17,6 +17,6 @@
 #include <forward_list>
 
 void f() {
-    std::forward_list<int> c;
-    c.empty(); // expected-warning {{ignoring return value of function declared with 'nodiscard' attribute}}
+  std::forward_list<int> c;
+  c.empty(); // expected-warning {{ignoring return value of function declared with 'nodiscard' attribute}}
 }
diff --git a/libcxx/test/std/containers/sequences/forwardlist/exception_safety.pass.cpp b/libcxx/test/std/containers/sequences/forwardlist/exception_safety.pass.cpp
index a6f1a15bdd85c..faf42202e951a 100644
--- a/libcxx/test/std/containers/sequences/forwardlist/exception_safety.pass.cpp
+++ b/libcxx/test/std/containers/sequences/forwardlist/exception_safety.pass.cpp
@@ -55,23 +55,23 @@
 #include "test_macros.h"
 
 #if TEST_STD_VER >= 23
-#include <ranges>
+#  include <ranges>
 #endif
 
 int main(int, char**) {
   {
     constexpr int ThrowOn = 1;
-    constexpr int Size = 1;
-    using T = ThrowingCopy<ThrowOn>;
+    constexpr int Size    = 1;
+    using T               = ThrowingCopy<ThrowOn>;
 
     // void push_front(const value_type& v);
-    test_exception_safety_throwing_copy<ThrowOn, Size>([](T* from, T*){
+    test_exception_safety_throwing_copy<ThrowOn, Size>([](T* from, T*) {
       std::forward_list<T> c;
       c.push_front(*from);
     });
 
     // iterator insert_after(const_iterator p, const value_type& v);
-    test_exception_safety_throwing_copy</*ThrowOn=*/1, Size>([](T* from, T*){
+    test_exception_safety_throwing_copy</*ThrowOn=*/1, Size>([](T* from, T*) {
       std::forward_list<T> c;
       c.insert_after(c.before_begin(), *from);
     });
@@ -79,26 +79,26 @@ int main(int, char**) {
 
   {
     constexpr int ThrowOn = 3;
-    constexpr int Size = 5;
-    using T = ThrowingCopy<ThrowOn>;
-    using C = std::forward_list<T>;
-    using Alloc = std::allocator<T>;
+    constexpr int Size    = 5;
+    using T               = ThrowingCopy<ThrowOn>;
+    using C               = std::forward_list<T>;
+    using Alloc           = std::allocator<T>;
 
     // forward_list(size_type n, const value_type& v);
-    test_exception_safety_throwing_copy<ThrowOn, Size>([](T* from, T*){
+    test_exception_safety_throwing_copy<ThrowOn, Size>([](T* from, T*) {
       std::forward_list<T> c(Size, *from);
       (void)c;
     });
 
     // forward_list(size_type n, const value_type& v, const allocator_type& a);
-    test_exception_safety_throwing_copy<ThrowOn, Size>([](T* from, T*){
+    test_exception_safety_throwing_copy<ThrowOn, Size>([](T* from, T*) {
       std::forward_list<T> c(Size, *from, Alloc());
       (void)c;
     });
 
     // template <class InputIterator>
     //     forward_list(InputIterator first, InputIterator last);
-    test_exception_safety_throwing_copy<ThrowOn, Size>([](T* from, T* to){
+    test_exception_safety_throwing_copy<ThrowOn, Size>([](T* from, T* to) {
       std::forward_list<T> c(from, to);
       (void)c;
     });
@@ -106,7 +106,7 @@ int main(int, char**) {
 #if TEST_STD_VER >= 23
     // template<container-compatible-range<T> R>
     //     forward_list(from_range_t, R&& rg, const Allocator& = Allocator()); // C++23
-    test_exception_safety_throwing_copy<ThrowOn, Size>([](T* from, T* to){
+    test_exception_safety_throwing_copy<ThrowOn, Size>([](T* from, T* to) {
       {
         std::forward_list<T> c(std::from_range, std::ranges::subrange(from, to));
         (void)c;
@@ -121,7 +121,7 @@ int main(int, char**) {
 
     // template <class InputIterator>
     //     forward_list(InputIterator first, InputIterator last, const allocator_type& a);
-    test_exception_safety_throwing_copy<ThrowOn, Size>([](T* from, T* to){
+    test_exception_safety_throwing_copy<ThrowOn, Size>([](T* from, T* to) {
       std::forward_list<T> c(from, to, Alloc());
       (void)c;
     });
diff --git a/libcxx/test/std/containers/sequences/forwardlist/forwardlist.access/front.pass.cpp b/libcxx/test/std/containers/sequences/forwardlist/forwardlist.access/front.pass.cpp
index 74eb3e65bb83c..757db7d957f5f 100644
--- a/libcxx/test/std/containers/sequences/forwardlist/forwardlist.access/front.pass.cpp
+++ b/libcxx/test/std/containers/sequences/forwardlist/forwardlist.access/front.pass.cpp
@@ -18,45 +18,44 @@
 #include "test_macros.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef int T;
-        typedef std::forward_list<T> C;
-        const T t[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9};
-        C c(std::begin(t), std::end(t));
-        assert(c.front() == 0);
-        c.front() = 10;
-        assert(c.front() == 10);
-        assert(*c.begin() == 10);
-    }
-    {
-        typedef int T;
-        typedef std::forward_list<T> C;
-        const T t[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9};
-        const C c(std::begin(t), std::end(t));
-        assert(c.front() == 0);
-        assert(*c.begin() == 0);
-    }
+int main(int, char**) {
+  {
+    typedef int T;
+    typedef std::forward_list<T> C;
+    const T t[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9};
+    C c(std::begin(t), std::end(t));
+    assert(c.front() == 0);
+    c.front() = 10;
+    assert(c.front() == 10);
+    assert(*c.begin() == 10);
+  }
+  {
+    typedef int T;
+    typedef std::forward_list<T> C;
+    const T t[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9};
+    const C c(std::begin(t), std::end(t));
+    assert(c.front() == 0);
+    assert(*c.begin() == 0);
+  }
 #if TEST_STD_VER >= 11
-    {
-        typedef int T;
-        typedef std::forward_list<T, min_allocator<T>> C;
-        const T t[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9};
-        C c(std::begin(t), std::end(t));
-        assert(c.front() == 0);
-        c.front() = 10;
-        assert(c.front() == 10);
-        assert(*c.begin() == 10);
-    }
-    {
-        typedef int T;
-        typedef std::forward_list<T, min_allocator<T>> C;
-        const T t[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9};
-        const C c(std::begin(t), std::end(t));
-        assert(c.front() == 0);
-        assert(*c.begin() == 0);
-    }
+  {
+    typedef int T;
+    typedef std::forward_list<T, min_allocator<T>> C;
+    const T t[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9};
+    C c(std::begin(t), std::end(t));
+    assert(c.front() == 0);
+    c.front() = 10;
+    assert(c.front() == 10);
+    assert(*c.begin() == 10);
+  }
+  {
+    typedef int T;
+    typedef std::forward_list<T, min_allocator<T>> C;
+    const T t[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9};
+    const C c(std::begin(t), std::end(t));
+    assert(c.front() == 0);
+    assert(*c.begin() == 0);
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/sequences/forwardlist/forwardlist.cons/alloc.compile.fail.cpp b/libcxx/test/std/containers/sequences/forwardlist/forwardlist.cons/alloc.compile.fail.cpp
index bf43ee82ca069..31893a1b95994 100644
--- a/libcxx/test/std/containers/sequences/forwardlist/forwardlist.cons/alloc.compile.fail.cpp
+++ b/libcxx/test/std/containers/sequences/forwardlist/forwardlist.cons/alloc.compile.fail.cpp
@@ -16,16 +16,15 @@
 #include "test_allocator.h"
 #include "../../../NotConstructible.h"
 
-int main(int, char**)
-{
-    {
-        typedef test_allocator<NotConstructible> A;
-        typedef A::value_type T;
-        typedef std::forward_list<T, A> C;
-        C c = A(12);
-        assert(c.get_allocator() == A(12));
-        assert(c.empty());
-    }
+int main(int, char**) {
+  {
+    typedef test_allocator<NotConstructible> A;
+    typedef A::value_type T;
+    typedef std::forward_list<T, A> C;
+    C c = A(12);
+    assert(c.get_allocator() == A(12));
+    assert(c.empty());
+  }
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/forwardlist/forwardlist.cons/alloc.pass.cpp b/libcxx/test/std/containers/sequences/forwardlist/forwardlist.cons/alloc.pass.cpp
index d2dda09d82cd9..bfb330fdaf9fc 100644
--- a/libcxx/test/std/containers/sequences/forwardlist/forwardlist.cons/alloc.pass.cpp
+++ b/libcxx/test/std/containers/sequences/forwardlist/forwardlist.cons/alloc.pass.cpp
@@ -18,33 +18,32 @@
 #include "../../../NotConstructible.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef test_allocator<NotConstructible> A;
-        typedef A::value_type T;
-        typedef std::forward_list<T, A> C;
-        C c(A(12));
-        assert(c.get_allocator() == A(12));
-        assert(c.empty());
-    }
+int main(int, char**) {
+  {
+    typedef test_allocator<NotConstructible> A;
+    typedef A::value_type T;
+    typedef std::forward_list<T, A> C;
+    C c(A(12));
+    assert(c.get_allocator() == A(12));
+    assert(c.empty());
+  }
 #if TEST_STD_VER >= 11
-    {
-        typedef min_allocator<NotConstructible> A;
-        typedef A::value_type T;
-        typedef std::forward_list<T, A> C;
-        C c(A{});
-        assert(c.get_allocator() == A());
-        assert(c.empty());
-    }
-    {
-        typedef explicit_allocator<NotConstructible> A;
-        typedef A::value_type T;
-        typedef std::forward_list<T, A> C;
-        C c(A{});
-        assert(c.get_allocator() == A());
-        assert(c.empty());
-    }
+  {
+    typedef min_allocator<NotConstructible> A;
+    typedef A::value_type T;
+    typedef std::forward_list<T, A> C;
+    C c(A{});
+    assert(c.get_allocator() == A());
+    assert(c.empty());
+  }
+  {
+    typedef explicit_allocator<NotConstructible> A;
+    typedef A::value_type T;
+    typedef std::forward_list<T, A> C;
+    C c(A{});
+    assert(c.get_allocator() == A());
+    assert(c.empty());
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/sequences/forwardlist/forwardlist.cons/assign_copy.pass.cpp b/libcxx/test/std/containers/sequences/forwardlist/forwardlist.cons/assign_copy.pass.cpp
index 07d34a1ac6ba5..27d450c63dcae 100644
--- a/libcxx/test/std/containers/sequences/forwardlist/forwardlist.cons/assign_copy.pass.cpp
+++ b/libcxx/test/std/containers/sequences/forwardlist/forwardlist.cons/assign_copy.pass.cpp
@@ -18,130 +18,129 @@
 #include "test_allocator.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef int T;
-        typedef test_allocator<int> A;
-        typedef std::forward_list<T, A> C;
-        const T t0[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9};
-        const T t1[] = {10, 11, 12, 13};
-        C c0(std::begin(t0), std::end(t0), A(10));
-        C c1(std::begin(t1), std::end(t1), A(10));
-        c1 = c0;
-        assert(c1 == c0);
-        assert(c1.get_allocator() == A(10));
-    }
-    {
-        typedef int T;
-        typedef test_allocator<int> A;
-        typedef std::forward_list<T, A> C;
-        const T t0[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9};
-        const T t1[] = {10, 11, 12, 13};
-        C c0(std::begin(t0), std::end(t0), A(10));
-        C c1(std::begin(t1), std::end(t1), A(11));
-        c1 = c0;
-        assert(c1 == c0);
-        assert(c1.get_allocator() == A(11));
-    }
-    {
-        typedef int T;
-        typedef test_allocator<int> A;
-        typedef std::forward_list<T, A> C;
-        const T t0[] = {10, 11, 12, 13};
-        const T t1[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9};
-        C c0(std::begin(t0), std::end(t0), A(10));
-        C c1(std::begin(t1), std::end(t1), A(10));
-        c1 = c0;
-        assert(c1 == c0);
-        assert(c1.get_allocator() == A(10));
-    }
-    {
-        typedef int T;
-        typedef test_allocator<int> A;
-        typedef std::forward_list<T, A> C;
-        const T t0[] = {10, 11, 12, 13};
-        const T t1[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9};
-        C c0(std::begin(t0), std::end(t0), A(10));
-        C c1(std::begin(t1), std::end(t1), A(11));
-        c1 = c0;
-        assert(c1 == c0);
-        assert(c1.get_allocator() == A(11));
-    }
+int main(int, char**) {
+  {
+    typedef int T;
+    typedef test_allocator<int> A;
+    typedef std::forward_list<T, A> C;
+    const T t0[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9};
+    const T t1[] = {10, 11, 12, 13};
+    C c0(std::begin(t0), std::end(t0), A(10));
+    C c1(std::begin(t1), std::end(t1), A(10));
+    c1 = c0;
+    assert(c1 == c0);
+    assert(c1.get_allocator() == A(10));
+  }
+  {
+    typedef int T;
+    typedef test_allocator<int> A;
+    typedef std::forward_list<T, A> C;
+    const T t0[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9};
+    const T t1[] = {10, 11, 12, 13};
+    C c0(std::begin(t0), std::end(t0), A(10));
+    C c1(std::begin(t1), std::end(t1), A(11));
+    c1 = c0;
+    assert(c1 == c0);
+    assert(c1.get_allocator() == A(11));
+  }
+  {
+    typedef int T;
+    typedef test_allocator<int> A;
+    typedef std::forward_list<T, A> C;
+    const T t0[] = {10, 11, 12, 13};
+    const T t1[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9};
+    C c0(std::begin(t0), std::end(t0), A(10));
+    C c1(std::begin(t1), std::end(t1), A(10));
+    c1 = c0;
+    assert(c1 == c0);
+    assert(c1.get_allocator() == A(10));
+  }
+  {
+    typedef int T;
+    typedef test_allocator<int> A;
+    typedef std::forward_list<T, A> C;
+    const T t0[] = {10, 11, 12, 13};
+    const T t1[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9};
+    C c0(std::begin(t0), std::end(t0), A(10));
+    C c1(std::begin(t1), std::end(t1), A(11));
+    c1 = c0;
+    assert(c1 == c0);
+    assert(c1.get_allocator() == A(11));
+  }
 
-    {
-        typedef int T;
-        typedef other_allocator<int> A;
-        typedef std::forward_list<T, A> C;
-        const T t0[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9};
-        const T t1[] = {10, 11, 12, 13};
-        C c0(std::begin(t0), std::end(t0), A(10));
-        C c1(std::begin(t1), std::end(t1), A(10));
-        c1 = c0;
-        assert(c1 == c0);
-        assert(c1.get_allocator() == A(10));
-    }
-    {
-        typedef int T;
-        typedef other_allocator<int> A;
-        typedef std::forward_list<T, A> C;
-        const T t0[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9};
-        const T t1[] = {10, 11, 12, 13};
-        C c0(std::begin(t0), std::end(t0), A(10));
-        C c1(std::begin(t1), std::end(t1), A(11));
-        c1 = c0;
-        assert(c1 == c0);
-        assert(c1.get_allocator() == A(10));
-    }
-    {
-        typedef int T;
-        typedef other_allocator<int> A;
-        typedef std::forward_list<T, A> C;
-        const T t0[] = {10, 11, 12, 13};
-        const T t1[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9};
-        C c0(std::begin(t0), std::end(t0), A(10));
-        C c1(std::begin(t1), std::end(t1), A(10));
-        c1 = c0;
-        assert(c1 == c0);
-        assert(c1.get_allocator() == A(10));
-    }
-    {
-        typedef int T;
-        typedef other_allocator<int> A;
-        typedef std::forward_list<T, A> C;
-        const T t0[] = {10, 11, 12, 13};
-        const T t1[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9};
-        C c0(std::begin(t0), std::end(t0), A(10));
-        C c1(std::begin(t1), std::end(t1), A(11));
-        c1 = c0;
-        assert(c1 == c0);
-        assert(c1.get_allocator() == A(10));
-    }
+  {
+    typedef int T;
+    typedef other_allocator<int> A;
+    typedef std::forward_list<T, A> C;
+    const T t0[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9};
+    const T t1[] = {10, 11, 12, 13};
+    C c0(std::begin(t0), std::end(t0), A(10));
+    C c1(std::begin(t1), std::end(t1), A(10));
+    c1 = c0;
+    assert(c1 == c0);
+    assert(c1.get_allocator() == A(10));
+  }
+  {
+    typedef int T;
+    typedef other_allocator<int> A;
+    typedef std::forward_list<T, A> C;
+    const T t0[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9};
+    const T t1[] = {10, 11, 12, 13};
+    C c0(std::begin(t0), std::end(t0), A(10));
+    C c1(std::begin(t1), std::end(t1), A(11));
+    c1 = c0;
+    assert(c1 == c0);
+    assert(c1.get_allocator() == A(10));
+  }
+  {
+    typedef int T;
+    typedef other_allocator<int> A;
+    typedef std::forward_list<T, A> C;
+    const T t0[] = {10, 11, 12, 13};
+    const T t1[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9};
+    C c0(std::begin(t0), std::end(t0), A(10));
+    C c1(std::begin(t1), std::end(t1), A(10));
+    c1 = c0;
+    assert(c1 == c0);
+    assert(c1.get_allocator() == A(10));
+  }
+  {
+    typedef int T;
+    typedef other_allocator<int> A;
+    typedef std::forward_list<T, A> C;
+    const T t0[] = {10, 11, 12, 13};
+    const T t1[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9};
+    C c0(std::begin(t0), std::end(t0), A(10));
+    C c1(std::begin(t1), std::end(t1), A(11));
+    c1 = c0;
+    assert(c1 == c0);
+    assert(c1.get_allocator() == A(10));
+  }
 #if TEST_STD_VER >= 11
-    {
-        typedef int T;
-        typedef min_allocator<int> A;
-        typedef std::forward_list<T, A> C;
-        const T t0[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9};
-        const T t1[] = {10, 11, 12, 13};
-        C c0(std::begin(t0), std::end(t0), A());
-        C c1(std::begin(t1), std::end(t1), A());
-        c1 = c0;
-        assert(c1 == c0);
-        assert(c1.get_allocator() == A());
-    }
-    {
-        typedef int T;
-        typedef min_allocator<int> A;
-        typedef std::forward_list<T, A> C;
-        const T t0[] = {10, 11, 12, 13};
-        const T t1[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9};
-        C c0(std::begin(t0), std::end(t0), A());
-        C c1(std::begin(t1), std::end(t1), A());
-        c1 = c0;
-        assert(c1 == c0);
-        assert(c1.get_allocator() == A());
-    }
+  {
+    typedef int T;
+    typedef min_allocator<int> A;
+    typedef std::forward_list<T, A> C;
+    const T t0[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9};
+    const T t1[] = {10, 11, 12, 13};
+    C c0(std::begin(t0), std::end(t0), A());
+    C c1(std::begin(t1), std::end(t1), A());
+    c1 = c0;
+    assert(c1 == c0);
+    assert(c1.get_allocator() == A());
+  }
+  {
+    typedef int T;
+    typedef min_allocator<int> A;
+    typedef std::forward_list<T, A> C;
+    const T t0[] = {10, 11, 12, 13};
+    const T t1[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9};
+    C c0(std::begin(t0), std::end(t0), A());
+    C c1(std::begin(t1), std::end(t1), A());
+    c1 = c0;
+    assert(c1 == c0);
+    assert(c1.get_allocator() == A());
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/sequences/forwardlist/forwardlist.cons/assign_init.pass.cpp b/libcxx/test/std/containers/sequences/forwardlist/forwardlist.cons/assign_init.pass.cpp
index 6d6b2fa58c454..1cdcca82d3352 100644
--- a/libcxx/test/std/containers/sequences/forwardlist/forwardlist.cons/assign_init.pass.cpp
+++ b/libcxx/test/std/containers/sequences/forwardlist/forwardlist.cons/assign_init.pass.cpp
@@ -19,52 +19,51 @@
 #include "test_macros.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef int T;
-        typedef std::forward_list<T> C;
-        const T t1[] = {10, 11, 12, 13};
-        C c(std::begin(t1), std::end(t1));
-        c.assign({0, 1, 2, 3, 4, 5, 6, 7, 8, 9});
-        int n = 0;
-        for (C::const_iterator i = c.cbegin(); i != c.cend(); ++i, ++n)
-            assert(*i == n);
-        assert(n == 10);
-    }
-    {
-        typedef int T;
-        typedef std::forward_list<T> C;
-        const T t1[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9};
-        C c(std::begin(t1), std::end(t1));
-        c.assign({10, 11, 12, 13});
-        int n = 0;
-        for (C::const_iterator i = c.cbegin(); i != c.cend(); ++i, ++n)
-            assert(*i == 10+n);
-        assert(n == 4);
-    }
-    {
-        typedef int T;
-        typedef std::forward_list<T, min_allocator<T>> C;
-        const T t1[] = {10, 11, 12, 13};
-        C c(std::begin(t1), std::end(t1));
-        c.assign({0, 1, 2, 3, 4, 5, 6, 7, 8, 9});
-        int n = 0;
-        for (C::const_iterator i = c.cbegin(); i != c.cend(); ++i, ++n)
-            assert(*i == n);
-        assert(n == 10);
-    }
-    {
-        typedef int T;
-        typedef std::forward_list<T, min_allocator<T>> C;
-        const T t1[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9};
-        C c(std::begin(t1), std::end(t1));
-        c.assign({10, 11, 12, 13});
-        int n = 0;
-        for (C::const_iterator i = c.cbegin(); i != c.cend(); ++i, ++n)
-            assert(*i == 10+n);
-        assert(n == 4);
-    }
+int main(int, char**) {
+  {
+    typedef int T;
+    typedef std::forward_list<T> C;
+    const T t1[] = {10, 11, 12, 13};
+    C c(std::begin(t1), std::end(t1));
+    c.assign({0, 1, 2, 3, 4, 5, 6, 7, 8, 9});
+    int n = 0;
+    for (C::const_iterator i = c.cbegin(); i != c.cend(); ++i, ++n)
+      assert(*i == n);
+    assert(n == 10);
+  }
+  {
+    typedef int T;
+    typedef std::forward_list<T> C;
+    const T t1[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9};
+    C c(std::begin(t1), std::end(t1));
+    c.assign({10, 11, 12, 13});
+    int n = 0;
+    for (C::const_iterator i = c.cbegin(); i != c.cend(); ++i, ++n)
+      assert(*i == 10 + n);
+    assert(n == 4);
+  }
+  {
+    typedef int T;
+    typedef std::forward_list<T, min_allocator<T>> C;
+    const T t1[] = {10, 11, 12, 13};
+    C c(std::begin(t1), std::end(t1));
+    c.assign({0, 1, 2, 3, 4, 5, 6, 7, 8, 9});
+    int n = 0;
+    for (C::const_iterator i = c.cbegin(); i != c.cend(); ++i, ++n)
+      assert(*i == n);
+    assert(n == 10);
+  }
+  {
+    typedef int T;
+    typedef std::forward_list<T, min_allocator<T>> C;
+    const T t1[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9};
+    C c(std::begin(t1), std::end(t1));
+    c.assign({10, 11, 12, 13});
+    int n = 0;
+    for (C::const_iterator i = c.cbegin(); i != c.cend(); ++i, ++n)
+      assert(*i == 10 + n);
+    assert(n == 4);
+  }
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/forwardlist/forwardlist.cons/assign_move.pass.cpp b/libcxx/test/std/containers/sequences/forwardlist/forwardlist.cons/assign_move.pass.cpp
index 1b2aabf25907d..998a7e11ef343 100644
--- a/libcxx/test/std/containers/sequences/forwardlist/forwardlist.cons/assign_move.pass.cpp
+++ b/libcxx/test/std/containers/sequences/forwardlist/forwardlist.cons/assign_move.pass.cpp
@@ -21,179 +21,178 @@
 #include "MoveOnly.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef MoveOnly T;
-        typedef test_allocator<T> A;
-        typedef std::forward_list<T, A> C;
-        T t0[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9};
-        T t1[] = {10, 11, 12, 13};
-        typedef std::move_iterator<T*> I;
-        C c0(I(std::begin(t0)), I(std::end(t0)), A(10));
-        C c1(I(std::begin(t1)), I(std::end(t1)), A(10));
-        c1 = std::move(c0);
-        int n = 0;
-        for (C::const_iterator i = c1.cbegin(); i != c1.cend(); ++i, (void) ++n)
-            assert(*i == n);
-        assert(n == 10);
-        assert(c1.get_allocator() == A(10));
-        assert(c0.empty());
-    }
-    {
-        typedef MoveOnly T;
-        typedef test_allocator<T> A;
-        typedef std::forward_list<T, A> C;
-        T t0[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9};
-        T t1[] = {10, 11, 12, 13};
-        typedef std::move_iterator<T*> I;
-        C c0(I(std::begin(t0)), I(std::end(t0)), A(10));
-        C c1(I(std::begin(t1)), I(std::end(t1)), A(11));
-        c1 = std::move(c0);
-        int n = 0;
-        for (C::const_iterator i = c1.cbegin(); i != c1.cend(); ++i, (void) ++n)
-            assert(*i == n);
-        assert(n == 10);
-        assert(c1.get_allocator() == A(11));
-        assert(!c0.empty());
-    }
-    {
-        typedef MoveOnly T;
-        typedef test_allocator<T> A;
-        typedef std::forward_list<T, A> C;
-        T t0[] = {10, 11, 12, 13};
-        T t1[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9};
-        typedef std::move_iterator<T*> I;
-        C c0(I(std::begin(t0)), I(std::end(t0)), A(10));
-        C c1(I(std::begin(t1)), I(std::end(t1)), A(10));
-        c1 = std::move(c0);
-        int n = 0;
-        for (C::const_iterator i = c1.cbegin(); i != c1.cend(); ++i, (void) ++n)
-            assert(*i == 10+n);
-        assert(n == 4);
-        assert(c1.get_allocator() == A(10));
-        assert(c0.empty());
-    }
-    {
-        typedef MoveOnly T;
-        typedef test_allocator<T> A;
-        typedef std::forward_list<T, A> C;
-        T t0[] = {10, 11, 12, 13};
-        T t1[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9};
-        typedef std::move_iterator<T*> I;
-        C c0(I(std::begin(t0)), I(std::end(t0)), A(10));
-        C c1(I(std::begin(t1)), I(std::end(t1)), A(11));
-        c1 = std::move(c0);
-        int n = 0;
-        for (C::const_iterator i = c1.cbegin(); i != c1.cend(); ++i, (void) ++n)
-            assert(*i == 10+n);
-        assert(n == 4);
-        assert(c1.get_allocator() == A(11));
-        assert(!c0.empty());
-    }
+int main(int, char**) {
+  {
+    typedef MoveOnly T;
+    typedef test_allocator<T> A;
+    typedef std::forward_list<T, A> C;
+    T t0[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9};
+    T t1[] = {10, 11, 12, 13};
+    typedef std::move_iterator<T*> I;
+    C c0(I(std::begin(t0)), I(std::end(t0)), A(10));
+    C c1(I(std::begin(t1)), I(std::end(t1)), A(10));
+    c1    = std::move(c0);
+    int n = 0;
+    for (C::const_iterator i = c1.cbegin(); i != c1.cend(); ++i, (void)++n)
+      assert(*i == n);
+    assert(n == 10);
+    assert(c1.get_allocator() == A(10));
+    assert(c0.empty());
+  }
+  {
+    typedef MoveOnly T;
+    typedef test_allocator<T> A;
+    typedef std::forward_list<T, A> C;
+    T t0[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9};
+    T t1[] = {10, 11, 12, 13};
+    typedef std::move_iterator<T*> I;
+    C c0(I(std::begin(t0)), I(std::end(t0)), A(10));
+    C c1(I(std::begin(t1)), I(std::end(t1)), A(11));
+    c1    = std::move(c0);
+    int n = 0;
+    for (C::const_iterator i = c1.cbegin(); i != c1.cend(); ++i, (void)++n)
+      assert(*i == n);
+    assert(n == 10);
+    assert(c1.get_allocator() == A(11));
+    assert(!c0.empty());
+  }
+  {
+    typedef MoveOnly T;
+    typedef test_allocator<T> A;
+    typedef std::forward_list<T, A> C;
+    T t0[] = {10, 11, 12, 13};
+    T t1[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9};
+    typedef std::move_iterator<T*> I;
+    C c0(I(std::begin(t0)), I(std::end(t0)), A(10));
+    C c1(I(std::begin(t1)), I(std::end(t1)), A(10));
+    c1    = std::move(c0);
+    int n = 0;
+    for (C::const_iterator i = c1.cbegin(); i != c1.cend(); ++i, (void)++n)
+      assert(*i == 10 + n);
+    assert(n == 4);
+    assert(c1.get_allocator() == A(10));
+    assert(c0.empty());
+  }
+  {
+    typedef MoveOnly T;
+    typedef test_allocator<T> A;
+    typedef std::forward_list<T, A> C;
+    T t0[] = {10, 11, 12, 13};
+    T t1[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9};
+    typedef std::move_iterator<T*> I;
+    C c0(I(std::begin(t0)), I(std::end(t0)), A(10));
+    C c1(I(std::begin(t1)), I(std::end(t1)), A(11));
+    c1    = std::move(c0);
+    int n = 0;
+    for (C::const_iterator i = c1.cbegin(); i != c1.cend(); ++i, (void)++n)
+      assert(*i == 10 + n);
+    assert(n == 4);
+    assert(c1.get_allocator() == A(11));
+    assert(!c0.empty());
+  }
 
-    {
-        typedef MoveOnly T;
-        typedef other_allocator<T> A;
-        typedef std::forward_list<T, A> C;
-        T t0[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9};
-        T t1[] = {10, 11, 12, 13};
-        typedef std::move_iterator<T*> I;
-        C c0(I(std::begin(t0)), I(std::end(t0)), A(10));
-        C c1(I(std::begin(t1)), I(std::end(t1)), A(10));
-        c1 = std::move(c0);
-        int n = 0;
-        for (C::const_iterator i = c1.cbegin(); i != c1.cend(); ++i, (void) ++n)
-            assert(*i == n);
-        assert(n == 10);
-        assert(c1.get_allocator() == A(10));
-        assert(c0.empty());
-    }
-    {
-        typedef MoveOnly T;
-        typedef other_allocator<T> A;
-        typedef std::forward_list<T, A> C;
-        T t0[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9};
-        T t1[] = {10, 11, 12, 13};
-        typedef std::move_iterator<T*> I;
-        C c0(I(std::begin(t0)), I(std::end(t0)), A(10));
-        C c1(I(std::begin(t1)), I(std::end(t1)), A(11));
-        c1 = std::move(c0);
-        int n = 0;
-        for (C::const_iterator i = c1.cbegin(); i != c1.cend(); ++i, (void) ++n)
-            assert(*i == n);
-        assert(n == 10);
-        assert(c1.get_allocator() == A(10));
-        assert(c0.empty());
-    }
-    {
-        typedef MoveOnly T;
-        typedef other_allocator<T> A;
-        typedef std::forward_list<T, A> C;
-        T t0[] = {10, 11, 12, 13};
-        T t1[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9};
-        typedef std::move_iterator<T*> I;
-        C c0(I(std::begin(t0)), I(std::end(t0)), A(10));
-        C c1(I(std::begin(t1)), I(std::end(t1)), A(10));
-        c1 = std::move(c0);
-        int n = 0;
-        for (C::const_iterator i = c1.cbegin(); i != c1.cend(); ++i, (void) ++n)
-            assert(*i == 10+n);
-        assert(n == 4);
-        assert(c1.get_allocator() == A(10));
-        assert(c0.empty());
-    }
-    {
-        typedef MoveOnly T;
-        typedef other_allocator<T> A;
-        typedef std::forward_list<T, A> C;
-        T t0[] = {10, 11, 12, 13};
-        T t1[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9};
-        typedef std::move_iterator<T*> I;
-        C c0(I(std::begin(t0)), I(std::end(t0)), A(10));
-        C c1(I(std::begin(t1)), I(std::end(t1)), A(11));
-        c1 = std::move(c0);
-        int n = 0;
-        for (C::const_iterator i = c1.cbegin(); i != c1.cend(); ++i, (void) ++n)
-            assert(*i == 10+n);
-        assert(n == 4);
-        assert(c1.get_allocator() == A(10));
-        assert(c0.empty());
-    }
-    {
-        typedef MoveOnly T;
-        typedef min_allocator<T> A;
-        typedef std::forward_list<T, A> C;
-        T t0[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9};
-        T t1[] = {10, 11, 12, 13};
-        typedef std::move_iterator<T*> I;
-        C c0(I(std::begin(t0)), I(std::end(t0)), A());
-        C c1(I(std::begin(t1)), I(std::end(t1)), A());
-        c1 = std::move(c0);
-        int n = 0;
-        for (C::const_iterator i = c1.cbegin(); i != c1.cend(); ++i, (void) ++n)
-            assert(*i == n);
-        assert(n == 10);
-        assert(c1.get_allocator() == A());
-        assert(c0.empty());
-    }
-    {
-        typedef MoveOnly T;
-        typedef min_allocator<T> A;
-        typedef std::forward_list<T, A> C;
-        T t0[] = {10, 11, 12, 13};
-        T t1[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9};
-        typedef std::move_iterator<T*> I;
-        C c0(I(std::begin(t0)), I(std::end(t0)), A());
-        C c1(I(std::begin(t1)), I(std::end(t1)), A());
-        c1 = std::move(c0);
-        int n = 0;
-        for (C::const_iterator i = c1.cbegin(); i != c1.cend(); ++i, (void) ++n)
-            assert(*i == 10+n);
-        assert(n == 4);
-        assert(c1.get_allocator() == A());
-        assert(c0.empty());
-    }
+  {
+    typedef MoveOnly T;
+    typedef other_allocator<T> A;
+    typedef std::forward_list<T, A> C;
+    T t0[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9};
+    T t1[] = {10, 11, 12, 13};
+    typedef std::move_iterator<T*> I;
+    C c0(I(std::begin(t0)), I(std::end(t0)), A(10));
+    C c1(I(std::begin(t1)), I(std::end(t1)), A(10));
+    c1    = std::move(c0);
+    int n = 0;
+    for (C::const_iterator i = c1.cbegin(); i != c1.cend(); ++i, (void)++n)
+      assert(*i == n);
+    assert(n == 10);
+    assert(c1.get_allocator() == A(10));
+    assert(c0.empty());
+  }
+  {
+    typedef MoveOnly T;
+    typedef other_allocator<T> A;
+    typedef std::forward_list<T, A> C;
+    T t0[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9};
+    T t1[] = {10, 11, 12, 13};
+    typedef std::move_iterator<T*> I;
+    C c0(I(std::begin(t0)), I(std::end(t0)), A(10));
+    C c1(I(std::begin(t1)), I(std::end(t1)), A(11));
+    c1    = std::move(c0);
+    int n = 0;
+    for (C::const_iterator i = c1.cbegin(); i != c1.cend(); ++i, (void)++n)
+      assert(*i == n);
+    assert(n == 10);
+    assert(c1.get_allocator() == A(10));
+    assert(c0.empty());
+  }
+  {
+    typedef MoveOnly T;
+    typedef other_allocator<T> A;
+    typedef std::forward_list<T, A> C;
+    T t0[] = {10, 11, 12, 13};
+    T t1[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9};
+    typedef std::move_iterator<T*> I;
+    C c0(I(std::begin(t0)), I(std::end(t0)), A(10));
+    C c1(I(std::begin(t1)), I(std::end(t1)), A(10));
+    c1    = std::move(c0);
+    int n = 0;
+    for (C::const_iterator i = c1.cbegin(); i != c1.cend(); ++i, (void)++n)
+      assert(*i == 10 + n);
+    assert(n == 4);
+    assert(c1.get_allocator() == A(10));
+    assert(c0.empty());
+  }
+  {
+    typedef MoveOnly T;
+    typedef other_allocator<T> A;
+    typedef std::forward_list<T, A> C;
+    T t0[] = {10, 11, 12, 13};
+    T t1[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9};
+    typedef std::move_iterator<T*> I;
+    C c0(I(std::begin(t0)), I(std::end(t0)), A(10));
+    C c1(I(std::begin(t1)), I(std::end(t1)), A(11));
+    c1    = std::move(c0);
+    int n = 0;
+    for (C::const_iterator i = c1.cbegin(); i != c1.cend(); ++i, (void)++n)
+      assert(*i == 10 + n);
+    assert(n == 4);
+    assert(c1.get_allocator() == A(10));
+    assert(c0.empty());
+  }
+  {
+    typedef MoveOnly T;
+    typedef min_allocator<T> A;
+    typedef std::forward_list<T, A> C;
+    T t0[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9};
+    T t1[] = {10, 11, 12, 13};
+    typedef std::move_iterator<T*> I;
+    C c0(I(std::begin(t0)), I(std::end(t0)), A());
+    C c1(I(std::begin(t1)), I(std::end(t1)), A());
+    c1    = std::move(c0);
+    int n = 0;
+    for (C::const_iterator i = c1.cbegin(); i != c1.cend(); ++i, (void)++n)
+      assert(*i == n);
+    assert(n == 10);
+    assert(c1.get_allocator() == A());
+    assert(c0.empty());
+  }
+  {
+    typedef MoveOnly T;
+    typedef min_allocator<T> A;
+    typedef std::forward_list<T, A> C;
+    T t0[] = {10, 11, 12, 13};
+    T t1[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9};
+    typedef std::move_iterator<T*> I;
+    C c0(I(std::begin(t0)), I(std::end(t0)), A());
+    C c1(I(std::begin(t1)), I(std::end(t1)), A());
+    c1    = std::move(c0);
+    int n = 0;
+    for (C::const_iterator i = c1.cbegin(); i != c1.cend(); ++i, (void)++n)
+      assert(*i == 10 + n);
+    assert(n == 4);
+    assert(c1.get_allocator() == A());
+    assert(c0.empty());
+  }
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/forwardlist/forwardlist.cons/assign_op_init.pass.cpp b/libcxx/test/std/containers/sequences/forwardlist/forwardlist.cons/assign_op_init.pass.cpp
index b6de0e97dfaeb..a22d6c4985bc5 100644
--- a/libcxx/test/std/containers/sequences/forwardlist/forwardlist.cons/assign_op_init.pass.cpp
+++ b/libcxx/test/std/containers/sequences/forwardlist/forwardlist.cons/assign_op_init.pass.cpp
@@ -19,52 +19,51 @@
 #include "test_macros.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef int T;
-        typedef std::forward_list<T> C;
-        const T t1[] = {10, 11, 12, 13};
-        C c(std::begin(t1), std::end(t1));
-        c = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9};
-        int n = 0;
-        for (C::const_iterator i = c.cbegin(); i != c.cend(); ++i, ++n)
-            assert(*i == n);
-        assert(n == 10);
-    }
-    {
-        typedef int T;
-        typedef std::forward_list<T> C;
-        const T t1[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9};
-        C c(std::begin(t1), std::end(t1));
-        c = {10, 11, 12, 13};
-        int n = 0;
-        for (C::const_iterator i = c.cbegin(); i != c.cend(); ++i, ++n)
-            assert(*i == 10+n);
-        assert(n == 4);
-    }
-    {
-        typedef int T;
-        typedef std::forward_list<T, min_allocator<T>> C;
-        const T t1[] = {10, 11, 12, 13};
-        C c(std::begin(t1), std::end(t1));
-        c = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9};
-        int n = 0;
-        for (C::const_iterator i = c.cbegin(); i != c.cend(); ++i, ++n)
-            assert(*i == n);
-        assert(n == 10);
-    }
-    {
-        typedef int T;
-        typedef std::forward_list<T, min_allocator<T>> C;
-        const T t1[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9};
-        C c(std::begin(t1), std::end(t1));
-        c = {10, 11, 12, 13};
-        int n = 0;
-        for (C::const_iterator i = c.cbegin(); i != c.cend(); ++i, ++n)
-            assert(*i == 10+n);
-        assert(n == 4);
-    }
+int main(int, char**) {
+  {
+    typedef int T;
+    typedef std::forward_list<T> C;
+    const T t1[] = {10, 11, 12, 13};
+    C c(std::begin(t1), std::end(t1));
+    c     = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9};
+    int n = 0;
+    for (C::const_iterator i = c.cbegin(); i != c.cend(); ++i, ++n)
+      assert(*i == n);
+    assert(n == 10);
+  }
+  {
+    typedef int T;
+    typedef std::forward_list<T> C;
+    const T t1[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9};
+    C c(std::begin(t1), std::end(t1));
+    c     = {10, 11, 12, 13};
+    int n = 0;
+    for (C::const_iterator i = c.cbegin(); i != c.cend(); ++i, ++n)
+      assert(*i == 10 + n);
+    assert(n == 4);
+  }
+  {
+    typedef int T;
+    typedef std::forward_list<T, min_allocator<T>> C;
+    const T t1[] = {10, 11, 12, 13};
+    C c(std::begin(t1), std::end(t1));
+    c     = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9};
+    int n = 0;
+    for (C::const_iterator i = c.cbegin(); i != c.cend(); ++i, ++n)
+      assert(*i == n);
+    assert(n == 10);
+  }
+  {
+    typedef int T;
+    typedef std::forward_list<T, min_allocator<T>> C;
+    const T t1[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9};
+    C c(std::begin(t1), std::end(t1));
+    c     = {10, 11, 12, 13};
+    int n = 0;
+    for (C::const_iterator i = c.cbegin(); i != c.cend(); ++i, ++n)
+      assert(*i == 10 + n);
+    assert(n == 4);
+  }
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/forwardlist/forwardlist.cons/assign_range.pass.cpp b/libcxx/test/std/containers/sequences/forwardlist/forwardlist.cons/assign_range.pass.cpp
index 552cf04434645..9a35328740790 100644
--- a/libcxx/test/std/containers/sequences/forwardlist/forwardlist.cons/assign_range.pass.cpp
+++ b/libcxx/test/std/containers/sequences/forwardlist/forwardlist.cons/assign_range.pass.cpp
@@ -19,61 +19,60 @@
 #include "test_iterators.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef int T;
-        typedef std::forward_list<T> C;
-        const T t0[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9};
-        const T t1[] = {10, 11, 12, 13};
-        C c(std::begin(t1), std::end(t1));
-        typedef cpp17_input_iterator<const T*> I;
-        c.assign(I(std::begin(t0)), I(std::end(t0)));
-        int n = 0;
-        for (C::const_iterator i = c.cbegin(); i != c.cend(); ++i, ++n)
-            assert(*i == n);
-        assert(n == 10);
-    }
-    {
-        typedef int T;
-        typedef std::forward_list<T> C;
-        const T t0[] = {10, 11, 12, 13};
-        const T t1[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9};
-        C c(std::begin(t1), std::end(t1));
-        typedef cpp17_input_iterator<const T*> I;
-        c.assign(I(std::begin(t0)), I(std::end(t0)));
-        int n = 0;
-        for (C::const_iterator i = c.cbegin(); i != c.cend(); ++i, ++n)
-            assert(*i == 10+n);
-        assert(n == 4);
-    }
+int main(int, char**) {
+  {
+    typedef int T;
+    typedef std::forward_list<T> C;
+    const T t0[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9};
+    const T t1[] = {10, 11, 12, 13};
+    C c(std::begin(t1), std::end(t1));
+    typedef cpp17_input_iterator<const T*> I;
+    c.assign(I(std::begin(t0)), I(std::end(t0)));
+    int n = 0;
+    for (C::const_iterator i = c.cbegin(); i != c.cend(); ++i, ++n)
+      assert(*i == n);
+    assert(n == 10);
+  }
+  {
+    typedef int T;
+    typedef std::forward_list<T> C;
+    const T t0[] = {10, 11, 12, 13};
+    const T t1[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9};
+    C c(std::begin(t1), std::end(t1));
+    typedef cpp17_input_iterator<const T*> I;
+    c.assign(I(std::begin(t0)), I(std::end(t0)));
+    int n = 0;
+    for (C::const_iterator i = c.cbegin(); i != c.cend(); ++i, ++n)
+      assert(*i == 10 + n);
+    assert(n == 4);
+  }
 #if TEST_STD_VER >= 11
-    {
-        typedef int T;
-        typedef std::forward_list<T, min_allocator<T>> C;
-        const T t0[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9};
-        const T t1[] = {10, 11, 12, 13};
-        C c(std::begin(t1), std::end(t1));
-        typedef cpp17_input_iterator<const T*> I;
-        c.assign(I(std::begin(t0)), I(std::end(t0)));
-        int n = 0;
-        for (C::const_iterator i = c.cbegin(); i != c.cend(); ++i, ++n)
-            assert(*i == n);
-        assert(n == 10);
-    }
-    {
-        typedef int T;
-        typedef std::forward_list<T, min_allocator<T>> C;
-        const T t0[] = {10, 11, 12, 13};
-        const T t1[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9};
-        C c(std::begin(t1), std::end(t1));
-        typedef cpp17_input_iterator<const T*> I;
-        c.assign(I(std::begin(t0)), I(std::end(t0)));
-        int n = 0;
-        for (C::const_iterator i = c.cbegin(); i != c.cend(); ++i, (void) ++n)
-            assert(*i == 10+n);
-        assert(n == 4);
-    }
+  {
+    typedef int T;
+    typedef std::forward_list<T, min_allocator<T>> C;
+    const T t0[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9};
+    const T t1[] = {10, 11, 12, 13};
+    C c(std::begin(t1), std::end(t1));
+    typedef cpp17_input_iterator<const T*> I;
+    c.assign(I(std::begin(t0)), I(std::end(t0)));
+    int n = 0;
+    for (C::const_iterator i = c.cbegin(); i != c.cend(); ++i, ++n)
+      assert(*i == n);
+    assert(n == 10);
+  }
+  {
+    typedef int T;
+    typedef std::forward_list<T, min_allocator<T>> C;
+    const T t0[] = {10, 11, 12, 13};
+    const T t1[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9};
+    C c(std::begin(t1), std::end(t1));
+    typedef cpp17_input_iterator<const T*> I;
+    c.assign(I(std::begin(t0)), I(std::end(t0)));
+    int n = 0;
+    for (C::const_iterator i = c.cbegin(); i != c.cend(); ++i, (void)++n)
+      assert(*i == 10 + n);
+    assert(n == 4);
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/sequences/forwardlist/forwardlist.cons/assign_size_value.pass.cpp b/libcxx/test/std/containers/sequences/forwardlist/forwardlist.cons/assign_size_value.pass.cpp
index 4824dd8890aaf..b0fbfa3249e5e 100644
--- a/libcxx/test/std/containers/sequences/forwardlist/forwardlist.cons/assign_size_value.pass.cpp
+++ b/libcxx/test/std/containers/sequences/forwardlist/forwardlist.cons/assign_size_value.pass.cpp
@@ -17,53 +17,52 @@
 #include "test_macros.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef int T;
-        typedef std::forward_list<T> C;
-        const T t1[] = {10, 11, 12, 13};
-        C c(std::begin(t1), std::end(t1));
-        c.assign(10, 1);
-        int n = 0;
-        for (C::const_iterator i = c.cbegin(); i != c.cend(); ++i, ++n)
-            assert(*i == 1);
-        assert(n == 10);
-    }
-    {
-        typedef int T;
-        typedef std::forward_list<T> C;
-        const T t1[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9};
-        C c(std::begin(t1), std::end(t1));
-        c.assign(4, 10);
-        int n = 0;
-        for (C::const_iterator i = c.cbegin(); i != c.cend(); ++i, ++n)
-            assert(*i == 10);
-        assert(n == 4);
-    }
+int main(int, char**) {
+  {
+    typedef int T;
+    typedef std::forward_list<T> C;
+    const T t1[] = {10, 11, 12, 13};
+    C c(std::begin(t1), std::end(t1));
+    c.assign(10, 1);
+    int n = 0;
+    for (C::const_iterator i = c.cbegin(); i != c.cend(); ++i, ++n)
+      assert(*i == 1);
+    assert(n == 10);
+  }
+  {
+    typedef int T;
+    typedef std::forward_list<T> C;
+    const T t1[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9};
+    C c(std::begin(t1), std::end(t1));
+    c.assign(4, 10);
+    int n = 0;
+    for (C::const_iterator i = c.cbegin(); i != c.cend(); ++i, ++n)
+      assert(*i == 10);
+    assert(n == 4);
+  }
 #if TEST_STD_VER >= 11
-    {
-        typedef int T;
-        typedef std::forward_list<T, min_allocator<T>> C;
-        const T t1[] = {10, 11, 12, 13};
-        C c(std::begin(t1), std::end(t1));
-        c.assign(10, 1);
-        int n = 0;
-        for (C::const_iterator i = c.cbegin(); i != c.cend(); ++i, ++n)
-            assert(*i == 1);
-        assert(n == 10);
-    }
-    {
-        typedef int T;
-        typedef std::forward_list<T, min_allocator<T>> C;
-        const T t1[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9};
-        C c(std::begin(t1), std::end(t1));
-        c.assign(4, 10);
-        int n = 0;
-        for (C::const_iterator i = c.cbegin(); i != c.cend(); ++i, ++n)
-            assert(*i == 10);
-        assert(n == 4);
-    }
+  {
+    typedef int T;
+    typedef std::forward_list<T, min_allocator<T>> C;
+    const T t1[] = {10, 11, 12, 13};
+    C c(std::begin(t1), std::end(t1));
+    c.assign(10, 1);
+    int n = 0;
+    for (C::const_iterator i = c.cbegin(); i != c.cend(); ++i, ++n)
+      assert(*i == 1);
+    assert(n == 10);
+  }
+  {
+    typedef int T;
+    typedef std::forward_list<T, min_allocator<T>> C;
+    const T t1[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9};
+    C c(std::begin(t1), std::end(t1));
+    c.assign(4, 10);
+    int n = 0;
+    for (C::const_iterator i = c.cbegin(); i != c.cend(); ++i, ++n)
+      assert(*i == 10);
+    assert(n == 4);
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/sequences/forwardlist/forwardlist.cons/copy.pass.cpp b/libcxx/test/std/containers/sequences/forwardlist/forwardlist.cons/copy.pass.cpp
index 681629a2c5e34..22d5054b9ae18 100644
--- a/libcxx/test/std/containers/sequences/forwardlist/forwardlist.cons/copy.pass.cpp
+++ b/libcxx/test/std/containers/sequences/forwardlist/forwardlist.cons/copy.pass.cpp
@@ -18,51 +18,50 @@
 #include "test_allocator.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef int T;
-        typedef test_allocator<int> A;
-        typedef std::forward_list<T, A> C;
-        const T t[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9};
-        C c0(std::begin(t), std::end(t), A(10));
-        C c = c0;
-        int n = 0;
-        for (C::const_iterator i = c.begin(), e = c.end(); i != e; ++i, ++n)
-            assert(*i == n);
-        assert(n == std::end(t) - std::begin(t));
-        assert(c == c0);
-        assert(c.get_allocator() == A(10));
-    }
+int main(int, char**) {
+  {
+    typedef int T;
+    typedef test_allocator<int> A;
+    typedef std::forward_list<T, A> C;
+    const T t[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9};
+    C c0(std::begin(t), std::end(t), A(10));
+    C c   = c0;
+    int n = 0;
+    for (C::const_iterator i = c.begin(), e = c.end(); i != e; ++i, ++n)
+      assert(*i == n);
+    assert(n == std::end(t) - std::begin(t));
+    assert(c == c0);
+    assert(c.get_allocator() == A(10));
+  }
 #if TEST_STD_VER >= 11
-    {
-        typedef int T;
-        typedef other_allocator<int> A;
-        typedef std::forward_list<T, A> C;
-        const T t[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9};
-        C c0(std::begin(t), std::end(t), A(10));
-        C c = c0;
-        int n = 0;
-        for (C::const_iterator i = c.begin(), e = c.end(); i != e; ++i, ++n)
-            assert(*i == n);
-        assert(n == std::end(t) - std::begin(t));
-        assert(c == c0);
-        assert(c.get_allocator() == A(-2));
-    }
-    {
-        typedef int T;
-        typedef min_allocator<int> A;
-        typedef std::forward_list<T, A> C;
-        const T t[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9};
-        C c0(std::begin(t), std::end(t), A());
-        C c = c0;
-        int n = 0;
-        for (C::const_iterator i = c.begin(), e = c.end(); i != e; ++i, ++n)
-            assert(*i == n);
-        assert(n == std::end(t) - std::begin(t));
-        assert(c == c0);
-        assert(c.get_allocator() == A());
-    }
+  {
+    typedef int T;
+    typedef other_allocator<int> A;
+    typedef std::forward_list<T, A> C;
+    const T t[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9};
+    C c0(std::begin(t), std::end(t), A(10));
+    C c   = c0;
+    int n = 0;
+    for (C::const_iterator i = c.begin(), e = c.end(); i != e; ++i, ++n)
+      assert(*i == n);
+    assert(n == std::end(t) - std::begin(t));
+    assert(c == c0);
+    assert(c.get_allocator() == A(-2));
+  }
+  {
+    typedef int T;
+    typedef min_allocator<int> A;
+    typedef std::forward_list<T, A> C;
+    const T t[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9};
+    C c0(std::begin(t), std::end(t), A());
+    C c   = c0;
+    int n = 0;
+    for (C::const_iterator i = c.begin(), e = c.end(); i != e; ++i, ++n)
+      assert(*i == n);
+    assert(n == std::end(t) - std::begin(t));
+    assert(c == c0);
+    assert(c.get_allocator() == A());
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/sequences/forwardlist/forwardlist.cons/copy_alloc.pass.cpp b/libcxx/test/std/containers/sequences/forwardlist/forwardlist.cons/copy_alloc.pass.cpp
index 3eb33a66d15eb..a61233e4b5d22 100644
--- a/libcxx/test/std/containers/sequences/forwardlist/forwardlist.cons/copy_alloc.pass.cpp
+++ b/libcxx/test/std/containers/sequences/forwardlist/forwardlist.cons/copy_alloc.pass.cpp
@@ -18,51 +18,50 @@
 #include "test_allocator.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef int T;
-        typedef test_allocator<int> A;
-        typedef std::forward_list<T, A> C;
-        const T t[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9};
-        C c0(std::begin(t), std::end(t), A(10));
-        C c(c0, A(9));
-        int n = 0;
-        for (C::const_iterator i = c.begin(), e = c.end(); i != e; ++i, ++n)
-            assert(*i == n);
-        assert(n == std::end(t) - std::begin(t));
-        assert(c == c0);
-        assert(c.get_allocator() == A(9));
-    }
-    {
-        typedef int T;
-        typedef other_allocator<int> A;
-        typedef std::forward_list<T, A> C;
-        const T t[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9};
-        C c0(std::begin(t), std::end(t), A(10));
-        C c(c0, A(9));
-        int n = 0;
-        for (C::const_iterator i = c.begin(), e = c.end(); i != e; ++i, ++n)
-            assert(*i == n);
-        assert(n == std::end(t) - std::begin(t));
-        assert(c == c0);
-        assert(c.get_allocator() == A(9));
-    }
+int main(int, char**) {
+  {
+    typedef int T;
+    typedef test_allocator<int> A;
+    typedef std::forward_list<T, A> C;
+    const T t[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9};
+    C c0(std::begin(t), std::end(t), A(10));
+    C c(c0, A(9));
+    int n = 0;
+    for (C::const_iterator i = c.begin(), e = c.end(); i != e; ++i, ++n)
+      assert(*i == n);
+    assert(n == std::end(t) - std::begin(t));
+    assert(c == c0);
+    assert(c.get_allocator() == A(9));
+  }
+  {
+    typedef int T;
+    typedef other_allocator<int> A;
+    typedef std::forward_list<T, A> C;
+    const T t[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9};
+    C c0(std::begin(t), std::end(t), A(10));
+    C c(c0, A(9));
+    int n = 0;
+    for (C::const_iterator i = c.begin(), e = c.end(); i != e; ++i, ++n)
+      assert(*i == n);
+    assert(n == std::end(t) - std::begin(t));
+    assert(c == c0);
+    assert(c.get_allocator() == A(9));
+  }
 #if TEST_STD_VER >= 11
-    {
-        typedef int T;
-        typedef min_allocator<int> A;
-        typedef std::forward_list<T, A> C;
-        const T t[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9};
-        C c0(std::begin(t), std::end(t), A());
-        C c(c0, A());
-        int n = 0;
-        for (C::const_iterator i = c.begin(), e = c.end(); i != e; ++i, ++n)
-            assert(*i == n);
-        assert(n == std::end(t) - std::begin(t));
-        assert(c == c0);
-        assert(c.get_allocator() == A());
-    }
+  {
+    typedef int T;
+    typedef min_allocator<int> A;
+    typedef std::forward_list<T, A> C;
+    const T t[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9};
+    C c0(std::begin(t), std::end(t), A());
+    C c(c0, A());
+    int n = 0;
+    for (C::const_iterator i = c.begin(), e = c.end(); i != e; ++i, ++n)
+      assert(*i == n);
+    assert(n == std::end(t) - std::begin(t));
+    assert(c == c0);
+    assert(c.get_allocator() == A());
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/sequences/forwardlist/forwardlist.cons/deduct.pass.cpp b/libcxx/test/std/containers/sequences/forwardlist/forwardlist.cons/deduct.pass.cpp
index 81b70a95df43c..2358e62db55b7 100644
--- a/libcxx/test/std/containers/sequences/forwardlist/forwardlist.cons/deduct.pass.cpp
+++ b/libcxx/test/std/containers/sequences/forwardlist/forwardlist.cons/deduct.pass.cpp
@@ -32,20 +32,18 @@
 
 struct A {};
 
-int main(int, char**)
-{
-
-//  Test the explicit deduction guides
-    {
-    const int arr[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9 };
+int main(int, char**) {
+  //  Test the explicit deduction guides
+  {
+    const int arr[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9};
     std::forward_list fwl(std::begin(arr), std::end(arr));
 
     static_assert(std::is_same_v<decltype(fwl), std::forward_list<int>>, "");
     assert(std::equal(fwl.begin(), fwl.end(), std::begin(arr), std::end(arr)));
-    }
+  }
 
-    {
-    const long arr[] = {INT_MAX, 1L, 2L, 3L };
+  {
+    const long arr[] = {INT_MAX, 1L, 2L, 3L};
     std::forward_list fwl(std::begin(arr), std::end(arr), std::allocator<long>());
     static_assert(std::is_same_v<decltype(fwl)::value_type, long>, "");
     assert(std::distance(fwl.begin(), fwl.end()) == 4); // no size for forward_list
@@ -53,39 +51,39 @@ int main(int, char**)
     assert(*it++ == INT_MAX);
     assert(*it++ == 1L);
     assert(*it++ == 2L);
-    }
+  }
 
-//  Test the implicit deduction guides
+  //  Test the implicit deduction guides
 
-    {
-//  We don't expect this one to work.
-//  std::forward_list fwl(std::allocator<int>()); // deque (allocator &)
-    }
+  {
+    //  We don't expect this one to work.
+    //  std::forward_list fwl(std::allocator<int>()); // deque (allocator &)
+  }
 
-    {
+  {
     std::forward_list fwl(1, A{}); // deque (size_type, T)
     static_assert(std::is_same_v<decltype(fwl)::value_type, A>, "");
     static_assert(std::is_same_v<decltype(fwl)::allocator_type, std::allocator<A>>, "");
     assert(std::distance(fwl.begin(), fwl.end()) == 1); // no size for forward_list
-    }
+  }
 
-    {
+  {
     std::forward_list fwl(1, A{}, test_allocator<A>()); // deque (size_type, T, allocator)
     static_assert(std::is_same_v<decltype(fwl)::value_type, A>, "");
     static_assert(std::is_same_v<decltype(fwl)::allocator_type, test_allocator<A>>, "");
     assert(std::distance(fwl.begin(), fwl.end()) == 1); // no size for forward_list
-    }
+  }
 
-    {
+  {
     std::forward_list fwl{1U, 2U, 3U, 4U, 5U}; // deque(initializer-list)
     static_assert(std::is_same_v<decltype(fwl)::value_type, unsigned>, "");
     assert(std::distance(fwl.begin(), fwl.end()) == 5); // no size for forward_list
     auto it = fwl.begin();
     std::advance(it, 2);
     assert(*it == 3U);
-    }
+  }
 
-    {
+  {
     std::forward_list fwl({1.0, 2.0, 3.0, 4.0}, test_allocator<double>()); // deque(initializer-list, allocator)
     static_assert(std::is_same_v<decltype(fwl)::value_type, double>, "");
     static_assert(std::is_same_v<decltype(fwl)::allocator_type, test_allocator<double>>, "");
@@ -93,61 +91,61 @@ int main(int, char**)
     auto it = fwl.begin();
     std::advance(it, 3);
     assert(*it == 4.0);
-    }
+  }
 
-    {
+  {
     std::forward_list<long double> source;
     std::forward_list fwl(source); // deque(deque &)
     static_assert(std::is_same_v<decltype(fwl)::value_type, long double>, "");
     static_assert(std::is_same_v<decltype(fwl)::allocator_type, std::allocator<long double>>, "");
     assert(std::distance(fwl.begin(), fwl.end()) == 0); // no size for forward_list
+  }
+
+  {
+    typedef test_allocator<short> Alloc;
+    typedef test_allocator<int> ConvertibleToAlloc;
+
+    {
+      std::forward_list<short, Alloc> source;
+      std::forward_list fwl(source, Alloc(2));
+      static_assert(std::is_same_v<decltype(fwl), decltype(source)>);
+    }
+
+    {
+      std::forward_list<short, Alloc> source;
+      std::forward_list fwl(source, ConvertibleToAlloc(2));
+      static_assert(std::is_same_v<decltype(fwl), decltype(source)>);
+    }
+
+    {
+      std::forward_list<short, Alloc> source;
+      std::forward_list fwl(std::move(source), Alloc(2));
+      static_assert(std::is_same_v<decltype(fwl), decltype(source)>);
     }
 
     {
-        typedef test_allocator<short> Alloc;
-        typedef test_allocator<int> ConvertibleToAlloc;
-
-        {
-        std::forward_list<short, Alloc> source;
-        std::forward_list fwl(source, Alloc(2));
-        static_assert(std::is_same_v<decltype(fwl), decltype(source)>);
-        }
-
-        {
-        std::forward_list<short, Alloc> source;
-        std::forward_list fwl(source, ConvertibleToAlloc(2));
-        static_assert(std::is_same_v<decltype(fwl), decltype(source)>);
-        }
-
-        {
-        std::forward_list<short, Alloc> source;
-        std::forward_list fwl(std::move(source), Alloc(2));
-        static_assert(std::is_same_v<decltype(fwl), decltype(source)>);
-        }
-
-        {
-        std::forward_list<short, Alloc> source;
-        std::forward_list fwl(std::move(source), ConvertibleToAlloc(2));
-        static_assert(std::is_same_v<decltype(fwl), decltype(source)>);
-        }
+      std::forward_list<short, Alloc> source;
+      std::forward_list fwl(std::move(source), ConvertibleToAlloc(2));
+      static_assert(std::is_same_v<decltype(fwl), decltype(source)>);
     }
+  }
 
 #if TEST_STD_VER >= 23
+  {
+    {
+      std::forward_list c(std::from_range, std::array<int, 0>());
+      static_assert(std::is_same_v<decltype(c), std::forward_list<int>>);
+    }
+
     {
-      {
-        std::forward_list c(std::from_range, std::array<int, 0>());
-        static_assert(std::is_same_v<decltype(c), std::forward_list<int>>);
-      }
-
-      {
-        using Alloc = test_allocator<int>;
-        std::forward_list c(std::from_range, std::array<int, 0>(), Alloc());
-        static_assert(std::is_same_v<decltype(c), std::forward_list<int, Alloc>>);
-      }
+      using Alloc = test_allocator<int>;
+      std::forward_list c(std::from_range, std::array<int, 0>(), Alloc());
+      static_assert(std::is_same_v<decltype(c), std::forward_list<int, Alloc>>);
     }
+  }
 #endif
 
-    SequenceContainerDeductionGuidesSfinaeAway<std::forward_list, std::forward_list<int>>();
+  SequenceContainerDeductionGuidesSfinaeAway<std::forward_list, std::forward_list<int>>();
 
-    return 0;
+  return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/forwardlist/forwardlist.cons/deduct.verify.cpp b/libcxx/test/std/containers/sequences/forwardlist/forwardlist.cons/deduct.verify.cpp
index b3c3f73270f32..7726787d0fd3d 100644
--- a/libcxx/test/std/containers/sequences/forwardlist/forwardlist.cons/deduct.verify.cpp
+++ b/libcxx/test/std/containers/sequences/forwardlist/forwardlist.cons/deduct.verify.cpp
@@ -22,19 +22,18 @@
 
 struct A {};
 
-int main(int, char**)
-{
-//  Test the explicit deduction guides
-
-//  Test the implicit deduction guides
-    {
-//  forward_list (allocator &)
-    std::forward_list fwl((std::allocator<int>()));  // expected-error-re {{no viable constructor or deduction guide for deduction of template arguments of '{{(std::)?}}forward_list'}}
-//  Note: The extra parens are necessary, since otherwise clang decides it is a function declaration.
-//  Also, we can't use {} instead of parens, because that constructs a
-//      forward_list<allocator<int>, allocator<allocator<int>>>
-    }
-
+int main(int, char**) {
+  //  Test the explicit deduction guides
+
+  //  Test the implicit deduction guides
+  {
+    //  forward_list (allocator &)
+    std::forward_list fwl((std::allocator< int>()));
+    // expected-error-re@-1 {{no viable constructor or deduction guide for deduction of template arguments of '{{(std::)?}}forward_list'}}
+    //  Note: The extra parens are necessary, since otherwise clang decides it is a function declaration.
+    //  Also, we can't use {} instead of parens, because that constructs a
+    //      forward_list<allocator<int>, allocator<allocator<int>>>
+  }
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/forwardlist/forwardlist.cons/default.pass.cpp b/libcxx/test/std/containers/sequences/forwardlist/forwardlist.cons/default.pass.cpp
index 7313909af5a4f..b493a89b78003 100644
--- a/libcxx/test/std/containers/sequences/forwardlist/forwardlist.cons/default.pass.cpp
+++ b/libcxx/test/std/containers/sequences/forwardlist/forwardlist.cons/default.pass.cpp
@@ -16,27 +16,26 @@
 #include "test_macros.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef int T;
-        typedef std::forward_list<T> C;
-        C c;
-        assert(c.empty());
-    }
+int main(int, char**) {
+  {
+    typedef int T;
+    typedef std::forward_list<T> C;
+    C c;
+    assert(c.empty());
+  }
 #if TEST_STD_VER >= 11
-    {
-        typedef int T;
-        typedef std::forward_list<T, min_allocator<T>> C;
-        C c;
-        assert(c.empty());
-    }
-    {
-        typedef int T;
-        typedef std::forward_list<T> C;
-        C c = {};
-        assert(c.empty());
-    }
+  {
+    typedef int T;
+    typedef std::forward_list<T, min_allocator<T>> C;
+    C c;
+    assert(c.empty());
+  }
+  {
+    typedef int T;
+    typedef std::forward_list<T> C;
+    C c = {};
+    assert(c.empty());
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/sequences/forwardlist/forwardlist.cons/default_noexcept.pass.cpp b/libcxx/test/std/containers/sequences/forwardlist/forwardlist.cons/default_noexcept.pass.cpp
index b131cc9cdcafc..8ac8771835725 100644
--- a/libcxx/test/std/containers/sequences/forwardlist/forwardlist.cons/default_noexcept.pass.cpp
+++ b/libcxx/test/std/containers/sequences/forwardlist/forwardlist.cons/default_noexcept.pass.cpp
@@ -23,33 +23,31 @@
 #include "test_allocator.h"
 
 template <class T>
-struct some_alloc
-{
-    typedef T value_type;
-    some_alloc(const some_alloc&);
-    void allocate(std::size_t);
+struct some_alloc {
+  typedef T value_type;
+  some_alloc(const some_alloc&);
+  void allocate(std::size_t);
 };
 
-int main(int, char**)
-{
+int main(int, char**) {
 #if defined(_LIBCPP_VERSION)
-    {
-        typedef std::forward_list<MoveOnly> C;
-        static_assert(std::is_nothrow_default_constructible<C>::value, "");
-    }
-    {
-        typedef std::forward_list<MoveOnly, test_allocator<MoveOnly>> C;
-        static_assert(std::is_nothrow_default_constructible<C>::value, "");
-    }
+  {
+    typedef std::forward_list<MoveOnly> C;
+    static_assert(std::is_nothrow_default_constructible<C>::value, "");
+  }
+  {
+    typedef std::forward_list<MoveOnly, test_allocator<MoveOnly>> C;
+    static_assert(std::is_nothrow_default_constructible<C>::value, "");
+  }
 #endif // _LIBCPP_VERSION
-    {
-        typedef std::forward_list<MoveOnly, other_allocator<MoveOnly>> C;
-        static_assert(!std::is_nothrow_default_constructible<C>::value, "");
-    }
-    {
-        typedef std::forward_list<MoveOnly, some_alloc<MoveOnly>> C;
-        static_assert(!std::is_nothrow_default_constructible<C>::value, "");
-    }
+  {
+    typedef std::forward_list<MoveOnly, other_allocator<MoveOnly>> C;
+    static_assert(!std::is_nothrow_default_constructible<C>::value, "");
+  }
+  {
+    typedef std::forward_list<MoveOnly, some_alloc<MoveOnly>> C;
+    static_assert(!std::is_nothrow_default_constructible<C>::value, "");
+  }
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/forwardlist/forwardlist.cons/default_recursive.pass.cpp b/libcxx/test/std/containers/sequences/forwardlist/forwardlist.cons/default_recursive.pass.cpp
index affe050564b87..c2f0570da848e 100644
--- a/libcxx/test/std/containers/sequences/forwardlist/forwardlist.cons/default_recursive.pass.cpp
+++ b/libcxx/test/std/containers/sequences/forwardlist/forwardlist.cons/default_recursive.pass.cpp
@@ -16,13 +16,8 @@
 
 #include "test_macros.h"
 
-struct X
-{
-    std::forward_list<X> q;
+struct X {
+  std::forward_list<X> q;
 };
 
-int main(int, char**)
-{
-
-  return 0;
-}
+int main(int, char**) { return 0; }
diff --git a/libcxx/test/std/containers/sequences/forwardlist/forwardlist.cons/dtor_noexcept.pass.cpp b/libcxx/test/std/containers/sequences/forwardlist/forwardlist.cons/dtor_noexcept.pass.cpp
index db8f0d22e62f7..64c60af1fdcb7 100644
--- a/libcxx/test/std/containers/sequences/forwardlist/forwardlist.cons/dtor_noexcept.pass.cpp
+++ b/libcxx/test/std/containers/sequences/forwardlist/forwardlist.cons/dtor_noexcept.pass.cpp
@@ -20,33 +20,31 @@
 #include "test_allocator.h"
 
 template <class T>
-struct some_alloc
-{
-    typedef T value_type;
-    some_alloc(const some_alloc&);
-    ~some_alloc() noexcept(false);
-    void allocate(std::size_t);
+struct some_alloc {
+  typedef T value_type;
+  some_alloc(const some_alloc&);
+  ~some_alloc() noexcept(false);
+  void allocate(std::size_t);
 };
 
-int main(int, char**)
-{
-    {
-        typedef std::forward_list<MoveOnly> C;
-        static_assert(std::is_nothrow_destructible<C>::value, "");
-    }
-    {
-        typedef std::forward_list<MoveOnly, test_allocator<MoveOnly>> C;
-        static_assert(std::is_nothrow_destructible<C>::value, "");
-    }
-    {
-        typedef std::forward_list<MoveOnly, other_allocator<MoveOnly>> C;
-        static_assert(std::is_nothrow_destructible<C>::value, "");
-    }
+int main(int, char**) {
+  {
+    typedef std::forward_list<MoveOnly> C;
+    static_assert(std::is_nothrow_destructible<C>::value, "");
+  }
+  {
+    typedef std::forward_list<MoveOnly, test_allocator<MoveOnly>> C;
+    static_assert(std::is_nothrow_destructible<C>::value, "");
+  }
+  {
+    typedef std::forward_list<MoveOnly, other_allocator<MoveOnly>> C;
+    static_assert(std::is_nothrow_destructible<C>::value, "");
+  }
 #if defined(_LIBCPP_VERSION)
-    {
-        typedef std::forward_list<MoveOnly, some_alloc<MoveOnly>> C;
-        static_assert(!std::is_nothrow_destructible<C>::value, "");
-    }
+  {
+    typedef std::forward_list<MoveOnly, some_alloc<MoveOnly>> C;
+    static_assert(!std::is_nothrow_destructible<C>::value, "");
+  }
 #endif // _LIBCPP_VERSION
 
   return 0;
diff --git a/libcxx/test/std/containers/sequences/forwardlist/forwardlist.cons/init.pass.cpp b/libcxx/test/std/containers/sequences/forwardlist/forwardlist.cons/init.pass.cpp
index 8db81e220e58b..b42242b0a83d4 100644
--- a/libcxx/test/std/containers/sequences/forwardlist/forwardlist.cons/init.pass.cpp
+++ b/libcxx/test/std/containers/sequences/forwardlist/forwardlist.cons/init.pass.cpp
@@ -18,26 +18,25 @@
 #include "test_macros.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef int T;
-        typedef std::forward_list<T> C;
-        C c = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9};
-        int n = 0;
-        for (C::const_iterator i = c.begin(), e = c.end(); i != e; ++i, ++n)
-            assert(*i == n);
-        assert(n == 10);
-    }
-    {
-        typedef int T;
-        typedef std::forward_list<T, min_allocator<T>> C;
-        C c = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9};
-        int n = 0;
-        for (C::const_iterator i = c.begin(), e = c.end(); i != e; ++i, ++n)
-            assert(*i == n);
-        assert(n == 10);
-    }
+int main(int, char**) {
+  {
+    typedef int T;
+    typedef std::forward_list<T> C;
+    C c   = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9};
+    int n = 0;
+    for (C::const_iterator i = c.begin(), e = c.end(); i != e; ++i, ++n)
+      assert(*i == n);
+    assert(n == 10);
+  }
+  {
+    typedef int T;
+    typedef std::forward_list<T, min_allocator<T>> C;
+    C c   = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9};
+    int n = 0;
+    for (C::const_iterator i = c.begin(), e = c.end(); i != e; ++i, ++n)
+      assert(*i == n);
+    assert(n == 10);
+  }
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/forwardlist/forwardlist.cons/init_alloc.pass.cpp b/libcxx/test/std/containers/sequences/forwardlist/forwardlist.cons/init_alloc.pass.cpp
index f742ba18c6ce6..0b29cbfa9254d 100644
--- a/libcxx/test/std/containers/sequences/forwardlist/forwardlist.cons/init_alloc.pass.cpp
+++ b/libcxx/test/std/containers/sequences/forwardlist/forwardlist.cons/init_alloc.pass.cpp
@@ -19,30 +19,29 @@
 #include "test_allocator.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef int T;
-        typedef test_allocator<T> A;
-        typedef std::forward_list<T, A> C;
-        C c({0, 1, 2, 3, 4, 5, 6, 7, 8, 9}, A(14));
-        int n = 0;
-        for (C::const_iterator i = c.begin(), e = c.end(); i != e; ++i, ++n)
-            assert(*i == n);
-        assert(n == 10);
-        assert(c.get_allocator() == A(14));
-    }
-    {
-        typedef int T;
-        typedef min_allocator<T> A;
-        typedef std::forward_list<T, A> C;
-        C c({0, 1, 2, 3, 4, 5, 6, 7, 8, 9}, A());
-        int n = 0;
-        for (C::const_iterator i = c.begin(), e = c.end(); i != e; ++i, ++n)
-            assert(*i == n);
-        assert(n == 10);
-        assert(c.get_allocator() == A());
-    }
+int main(int, char**) {
+  {
+    typedef int T;
+    typedef test_allocator<T> A;
+    typedef std::forward_list<T, A> C;
+    C c({0, 1, 2, 3, 4, 5, 6, 7, 8, 9}, A(14));
+    int n = 0;
+    for (C::const_iterator i = c.begin(), e = c.end(); i != e; ++i, ++n)
+      assert(*i == n);
+    assert(n == 10);
+    assert(c.get_allocator() == A(14));
+  }
+  {
+    typedef int T;
+    typedef min_allocator<T> A;
+    typedef std::forward_list<T, A> C;
+    C c({0, 1, 2, 3, 4, 5, 6, 7, 8, 9}, A());
+    int n = 0;
+    for (C::const_iterator i = c.begin(), e = c.end(); i != e; ++i, ++n)
+      assert(*i == n);
+    assert(n == 10);
+    assert(c.get_allocator() == A());
+  }
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/forwardlist/forwardlist.cons/move.pass.cpp b/libcxx/test/std/containers/sequences/forwardlist/forwardlist.cons/move.pass.cpp
index c64baf79f46e0..762e252ca76fe 100644
--- a/libcxx/test/std/containers/sequences/forwardlist/forwardlist.cons/move.pass.cpp
+++ b/libcxx/test/std/containers/sequences/forwardlist/forwardlist.cons/move.pass.cpp
@@ -21,53 +21,52 @@
 #include "MoveOnly.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef MoveOnly T;
-        typedef test_allocator<T> A;
-        typedef std::forward_list<T, A> C;
-        T t[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9};
-        typedef std::move_iterator<T*> I;
-        C c0(I(std::begin(t)), I(std::end(t)), A(10));
-        C c = std::move(c0);
-        int n = 0;
-        for (C::const_iterator i = c.begin(), e = c.end(); i != e; ++i, (void) ++n)
-            assert(*i == n);
-        assert(n == std::end(t) - std::begin(t));
-        assert(c0.empty());
-        assert(c.get_allocator() == A(10));
-    }
-    {
-        typedef MoveOnly T;
-        typedef other_allocator<T> A;
-        typedef std::forward_list<T, A> C;
-        T t[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9};
-        typedef std::move_iterator<T*> I;
-        C c0(I(std::begin(t)), I(std::end(t)), A(10));
-        C c = std::move(c0);
-        int n = 0;
-        for (C::const_iterator i = c.begin(), e = c.end(); i != e; ++i, (void) ++n)
-            assert(*i == n);
-        assert(n == std::end(t) - std::begin(t));
-        assert(c0.empty());
-        assert(c.get_allocator() == A(10));
-    }
-    {
-        typedef MoveOnly T;
-        typedef min_allocator<T> A;
-        typedef std::forward_list<T, A> C;
-        T t[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9};
-        typedef std::move_iterator<T*> I;
-        C c0(I(std::begin(t)), I(std::end(t)), A());
-        C c = std::move(c0);
-        int n = 0;
-        for (C::const_iterator i = c.begin(), e = c.end(); i != e; ++i, (void) ++n)
-            assert(*i == n);
-        assert(n == std::end(t) - std::begin(t));
-        assert(c0.empty());
-        assert(c.get_allocator() == A());
-    }
+int main(int, char**) {
+  {
+    typedef MoveOnly T;
+    typedef test_allocator<T> A;
+    typedef std::forward_list<T, A> C;
+    T t[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9};
+    typedef std::move_iterator<T*> I;
+    C c0(I(std::begin(t)), I(std::end(t)), A(10));
+    C c   = std::move(c0);
+    int n = 0;
+    for (C::const_iterator i = c.begin(), e = c.end(); i != e; ++i, (void)++n)
+      assert(*i == n);
+    assert(n == std::end(t) - std::begin(t));
+    assert(c0.empty());
+    assert(c.get_allocator() == A(10));
+  }
+  {
+    typedef MoveOnly T;
+    typedef other_allocator<T> A;
+    typedef std::forward_list<T, A> C;
+    T t[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9};
+    typedef std::move_iterator<T*> I;
+    C c0(I(std::begin(t)), I(std::end(t)), A(10));
+    C c   = std::move(c0);
+    int n = 0;
+    for (C::const_iterator i = c.begin(), e = c.end(); i != e; ++i, (void)++n)
+      assert(*i == n);
+    assert(n == std::end(t) - std::begin(t));
+    assert(c0.empty());
+    assert(c.get_allocator() == A(10));
+  }
+  {
+    typedef MoveOnly T;
+    typedef min_allocator<T> A;
+    typedef std::forward_list<T, A> C;
+    T t[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9};
+    typedef std::move_iterator<T*> I;
+    C c0(I(std::begin(t)), I(std::end(t)), A());
+    C c   = std::move(c0);
+    int n = 0;
+    for (C::const_iterator i = c.begin(), e = c.end(); i != e; ++i, (void)++n)
+      assert(*i == n);
+    assert(n == std::end(t) - std::begin(t));
+    assert(c0.empty());
+    assert(c.get_allocator() == A());
+  }
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/forwardlist/forwardlist.cons/move_alloc.pass.cpp b/libcxx/test/std/containers/sequences/forwardlist/forwardlist.cons/move_alloc.pass.cpp
index 56eaa56d52b22..a9bc2cb12f288 100644
--- a/libcxx/test/std/containers/sequences/forwardlist/forwardlist.cons/move_alloc.pass.cpp
+++ b/libcxx/test/std/containers/sequences/forwardlist/forwardlist.cons/move_alloc.pass.cpp
@@ -21,53 +21,52 @@
 #include "MoveOnly.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef MoveOnly T;
-        typedef test_allocator<T> A;
-        typedef std::forward_list<T, A> C;
-        T t[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9};
-        typedef std::move_iterator<T*> I;
-        C c0(I(std::begin(t)), I(std::end(t)), A(10));
-        C c(std::move(c0), A(10));
-        unsigned n = 0;
-        for (C::const_iterator i = c.begin(), e = c.end(); i != e; ++i, (void) ++n)
-            assert(*i == n);
-        assert(n == static_cast<unsigned>(std::end(t) - std::begin(t)));
-        assert(c0.empty());
-        assert(c.get_allocator() == A(10));
-    }
-    {
-        typedef MoveOnly T;
-        typedef test_allocator<T> A;
-        typedef std::forward_list<T, A> C;
-        T t[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9};
-        typedef std::move_iterator<T*> I;
-        C c0(I(std::begin(t)), I(std::end(t)), A(10));
-        C c(std::move(c0), A(9));
-        unsigned n = 0;
-        for (C::const_iterator i = c.begin(), e = c.end(); i != e; ++i, (void) ++n)
-            assert(*i == n);
-        assert(n == static_cast<unsigned>(std::end(t) - std::begin(t)));
-        assert(!c0.empty());
-        assert(c.get_allocator() == A(9));
-    }
-    {
-        typedef MoveOnly T;
-        typedef min_allocator<T> A;
-        typedef std::forward_list<T, A> C;
-        T t[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9};
-        typedef std::move_iterator<T*> I;
-        C c0(I(std::begin(t)), I(std::end(t)), A());
-        C c(std::move(c0), A());
-        unsigned n = 0;
-        for (C::const_iterator i = c.begin(), e = c.end(); i != e; ++i, (void) ++n)
-            assert(*i == n);
-        assert(n == static_cast<unsigned>(std::end(t) - std::begin(t)));
-        assert(c0.empty());
-        assert(c.get_allocator() == A());
-    }
+int main(int, char**) {
+  {
+    typedef MoveOnly T;
+    typedef test_allocator<T> A;
+    typedef std::forward_list<T, A> C;
+    T t[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9};
+    typedef std::move_iterator<T*> I;
+    C c0(I(std::begin(t)), I(std::end(t)), A(10));
+    C c(std::move(c0), A(10));
+    unsigned n = 0;
+    for (C::const_iterator i = c.begin(), e = c.end(); i != e; ++i, (void)++n)
+      assert(*i == n);
+    assert(n == static_cast<unsigned>(std::end(t) - std::begin(t)));
+    assert(c0.empty());
+    assert(c.get_allocator() == A(10));
+  }
+  {
+    typedef MoveOnly T;
+    typedef test_allocator<T> A;
+    typedef std::forward_list<T, A> C;
+    T t[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9};
+    typedef std::move_iterator<T*> I;
+    C c0(I(std::begin(t)), I(std::end(t)), A(10));
+    C c(std::move(c0), A(9));
+    unsigned n = 0;
+    for (C::const_iterator i = c.begin(), e = c.end(); i != e; ++i, (void)++n)
+      assert(*i == n);
+    assert(n == static_cast<unsigned>(std::end(t) - std::begin(t)));
+    assert(!c0.empty());
+    assert(c.get_allocator() == A(9));
+  }
+  {
+    typedef MoveOnly T;
+    typedef min_allocator<T> A;
+    typedef std::forward_list<T, A> C;
+    T t[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9};
+    typedef std::move_iterator<T*> I;
+    C c0(I(std::begin(t)), I(std::end(t)), A());
+    C c(std::move(c0), A());
+    unsigned n = 0;
+    for (C::const_iterator i = c.begin(), e = c.end(); i != e; ++i, (void)++n)
+      assert(*i == n);
+    assert(n == static_cast<unsigned>(std::end(t) - std::begin(t)));
+    assert(c0.empty());
+    assert(c.get_allocator() == A());
+  }
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/forwardlist/forwardlist.cons/move_assign_noexcept.compile.pass.cpp b/libcxx/test/std/containers/sequences/forwardlist/forwardlist.cons/move_assign_noexcept.compile.pass.cpp
new file mode 100644
index 0000000000000..b73177a00d466
--- /dev/null
+++ b/libcxx/test/std/containers/sequences/forwardlist/forwardlist.cons/move_assign_noexcept.compile.pass.cpp
@@ -0,0 +1,50 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// <forward_list>
+
+// forward_list& operator=(forward_list&& c)
+//     noexcept(
+//          allocator_type::propagate_on_container_move_assignment::value &&
+//          is_nothrow_move_assignable<allocator_type>::value);
+
+// This tests a conforming extension
+
+// UNSUPPORTED: c++03
+
+#include <forward_list>
+
+#include "test_macros.h"
+#include "MoveOnly.h"
+#include "test_allocator.h"
+
+template <class T>
+struct always_equal_alloc {
+  using value_type = T;
+  always_equal_alloc(const always_equal_alloc&);
+  void allocate(std::size_t);
+};
+
+template <class T>
+struct not_always_equal_alloc {
+  int i;
+  using value_type = T;
+  not_always_equal_alloc(const not_always_equal_alloc&);
+  void allocate(std::size_t);
+};
+
+static_assert(std::is_nothrow_move_assignable<std::forward_list<MoveOnly>>::value, "");
+static_assert(!std::is_nothrow_move_assignable<std::forward_list<MoveOnly, test_allocator<MoveOnly>>>::value, "");
+#if TEST_STD_VER >= 17
+static_assert(std::is_nothrow_move_assignable<std::forward_list<MoveOnly, always_equal_alloc<MoveOnly>>>::value, "");
+#endif
+static_assert(!std::is_nothrow_move_assignable<std::forward_list<MoveOnly, not_always_equal_alloc<MoveOnly>>>::value,
+              "");
+#if defined(_LIBCPP_VERSION)
+static_assert(std::is_nothrow_move_assignable<std::forward_list<MoveOnly, other_allocator<MoveOnly>>>::value, "");
+#endif // _LIBCPP_VERSION
diff --git a/libcxx/test/std/containers/sequences/forwardlist/forwardlist.cons/move_assign_noexcept.pass.cpp b/libcxx/test/std/containers/sequences/forwardlist/forwardlist.cons/move_assign_noexcept.pass.cpp
deleted file mode 100644
index f361e373d23ff..0000000000000
--- a/libcxx/test/std/containers/sequences/forwardlist/forwardlist.cons/move_assign_noexcept.pass.cpp
+++ /dev/null
@@ -1,57 +0,0 @@
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-// <forward_list>
-
-// forward_list& operator=(forward_list&& c)
-//     noexcept(
-//          allocator_type::propagate_on_container_move_assignment::value &&
-//          is_nothrow_move_assignable<allocator_type>::value);
-
-// This tests a conforming extension
-
-// UNSUPPORTED: c++03
-
-#include <forward_list>
-#include <cassert>
-
-#include "test_macros.h"
-#include "MoveOnly.h"
-#include "test_allocator.h"
-
-template <class T>
-struct some_alloc
-{
-    typedef T value_type;
-    some_alloc(const some_alloc&);
-    void allocate(std::size_t);
-};
-
-int main(int, char**)
-{
-    {
-        typedef std::forward_list<MoveOnly> C;
-        static_assert(std::is_nothrow_move_assignable<C>::value, "");
-    }
-    {
-        typedef std::forward_list<MoveOnly, test_allocator<MoveOnly>> C;
-        static_assert(!std::is_nothrow_move_assignable<C>::value, "");
-    }
-#if defined(_LIBCPP_VERSION)
-    {
-        typedef std::forward_list<MoveOnly, other_allocator<MoveOnly>> C;
-        static_assert(std::is_nothrow_move_assignable<C>::value, "");
-    }
-    {
-        typedef std::forward_list<MoveOnly, some_alloc<MoveOnly>> C;
-        static_assert(!std::is_nothrow_move_assignable<C>::value, "");
-    }
-#endif // _LIBCPP_VERSION
-
-  return 0;
-}
diff --git a/libcxx/test/std/containers/sequences/forwardlist/forwardlist.cons/move_noexcept.pass.cpp b/libcxx/test/std/containers/sequences/forwardlist/forwardlist.cons/move_noexcept.pass.cpp
index f132f0ed0b25d..8dcf00b49c78b 100644
--- a/libcxx/test/std/containers/sequences/forwardlist/forwardlist.cons/move_noexcept.pass.cpp
+++ b/libcxx/test/std/containers/sequences/forwardlist/forwardlist.cons/move_noexcept.pass.cpp
@@ -23,32 +23,30 @@
 #include "test_allocator.h"
 
 template <class T>
-struct some_alloc
-{
-    typedef T value_type;
-    some_alloc(const some_alloc&);
-    void allocate(std::size_t);
+struct some_alloc {
+  typedef T value_type;
+  some_alloc(const some_alloc&);
+  void allocate(std::size_t);
 };
 
-int main(int, char**)
-{
+int main(int, char**) {
 #if defined(_LIBCPP_VERSION)
-    {
-        typedef std::forward_list<MoveOnly> C;
-        static_assert(std::is_nothrow_move_constructible<C>::value, "");
-    }
-    {
-        typedef std::forward_list<MoveOnly, test_allocator<MoveOnly>> C;
-        static_assert(std::is_nothrow_move_constructible<C>::value, "");
-    }
-    {
-        typedef std::forward_list<MoveOnly, other_allocator<MoveOnly>> C;
-        static_assert(std::is_nothrow_move_constructible<C>::value, "");
-    }
-    {
-        typedef std::forward_list<MoveOnly, some_alloc<MoveOnly>> C;
-        static_assert(!std::is_nothrow_move_constructible<C>::value, "");
-    }
+  {
+    typedef std::forward_list<MoveOnly> C;
+    static_assert(std::is_nothrow_move_constructible<C>::value, "");
+  }
+  {
+    typedef std::forward_list<MoveOnly, test_allocator<MoveOnly>> C;
+    static_assert(std::is_nothrow_move_constructible<C>::value, "");
+  }
+  {
+    typedef std::forward_list<MoveOnly, other_allocator<MoveOnly>> C;
+    static_assert(std::is_nothrow_move_constructible<C>::value, "");
+  }
+  {
+    typedef std::forward_list<MoveOnly, some_alloc<MoveOnly>> C;
+    static_assert(!std::is_nothrow_move_constructible<C>::value, "");
+  }
 #endif // _LIBCPP_VERSION
 
   return 0;
diff --git a/libcxx/test/std/containers/sequences/forwardlist/forwardlist.cons/range.pass.cpp b/libcxx/test/std/containers/sequences/forwardlist/forwardlist.cons/range.pass.cpp
index e8c54c84c2c59..ebd0e6a5bd1e0 100644
--- a/libcxx/test/std/containers/sequences/forwardlist/forwardlist.cons/range.pass.cpp
+++ b/libcxx/test/std/containers/sequences/forwardlist/forwardlist.cons/range.pass.cpp
@@ -19,31 +19,30 @@
 #include "test_iterators.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef int T;
-        typedef std::forward_list<T> C;
-        typedef cpp17_input_iterator<const T*> I;
-        const T t[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9};
-        C c(I(std::begin(t)), I(std::end(t)));
-        int n = 0;
-        for (C::const_iterator i = c.begin(), e = c.end(); i != e; ++i, ++n)
-            assert(*i == n);
-        assert(n == std::end(t) - std::begin(t));
-    }
+int main(int, char**) {
+  {
+    typedef int T;
+    typedef std::forward_list<T> C;
+    typedef cpp17_input_iterator<const T*> I;
+    const T t[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9};
+    C c(I(std::begin(t)), I(std::end(t)));
+    int n = 0;
+    for (C::const_iterator i = c.begin(), e = c.end(); i != e; ++i, ++n)
+      assert(*i == n);
+    assert(n == std::end(t) - std::begin(t));
+  }
 #if TEST_STD_VER >= 11
-    {
-        typedef int T;
-        typedef std::forward_list<T, min_allocator<T>> C;
-        typedef cpp17_input_iterator<const T*> I;
-        const T t[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9};
-        C c(I(std::begin(t)), I(std::end(t)));
-        int n = 0;
-        for (C::const_iterator i = c.begin(), e = c.end(); i != e; ++i, ++n)
-            assert(*i == n);
-        assert(n == std::end(t) - std::begin(t));
-    }
+  {
+    typedef int T;
+    typedef std::forward_list<T, min_allocator<T>> C;
+    typedef cpp17_input_iterator<const T*> I;
+    const T t[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9};
+    C c(I(std::begin(t)), I(std::end(t)));
+    int n = 0;
+    for (C::const_iterator i = c.begin(), e = c.end(); i != e; ++i, ++n)
+      assert(*i == n);
+    assert(n == std::end(t) - std::begin(t));
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/sequences/forwardlist/forwardlist.cons/range_alloc.pass.cpp b/libcxx/test/std/containers/sequences/forwardlist/forwardlist.cons/range_alloc.pass.cpp
index 67712fc5e5d60..4a28041ad2cbc 100644
--- a/libcxx/test/std/containers/sequences/forwardlist/forwardlist.cons/range_alloc.pass.cpp
+++ b/libcxx/test/std/containers/sequences/forwardlist/forwardlist.cons/range_alloc.pass.cpp
@@ -21,35 +21,34 @@
 #include "test_iterators.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef int T;
-        typedef test_allocator<T> A;
-        typedef std::forward_list<T, A> C;
-        typedef cpp17_input_iterator<const T*> I;
-        const T t[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9};
-        C c(I(std::begin(t)), I(std::end(t)), A(13));
-        int n = 0;
-        for (C::const_iterator i = c.begin(), e = c.end(); i != e; ++i, ++n)
-            assert(*i == n);
-        assert(n == std::end(t) - std::begin(t));
-        assert(c.get_allocator() == A(13));
-    }
+int main(int, char**) {
+  {
+    typedef int T;
+    typedef test_allocator<T> A;
+    typedef std::forward_list<T, A> C;
+    typedef cpp17_input_iterator<const T*> I;
+    const T t[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9};
+    C c(I(std::begin(t)), I(std::end(t)), A(13));
+    int n = 0;
+    for (C::const_iterator i = c.begin(), e = c.end(); i != e; ++i, ++n)
+      assert(*i == n);
+    assert(n == std::end(t) - std::begin(t));
+    assert(c.get_allocator() == A(13));
+  }
 #if TEST_STD_VER >= 11
-    {
-        typedef int T;
-        typedef min_allocator<T> A;
-        typedef std::forward_list<T, A> C;
-        typedef cpp17_input_iterator<const T*> I;
-        const T t[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9};
-        C c(I(std::begin(t)), I(std::end(t)), A());
-        int n = 0;
-        for (C::const_iterator i = c.begin(), e = c.end(); i != e; ++i, ++n)
-            assert(*i == n);
-        assert(n == std::end(t) - std::begin(t));
-        assert(c.get_allocator() == A());
-    }
+  {
+    typedef int T;
+    typedef min_allocator<T> A;
+    typedef std::forward_list<T, A> C;
+    typedef cpp17_input_iterator<const T*> I;
+    const T t[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9};
+    C c(I(std::begin(t)), I(std::end(t)), A());
+    int n = 0;
+    for (C::const_iterator i = c.begin(), e = c.end(); i != e; ++i, ++n)
+      assert(*i == n);
+    assert(n == std::end(t) - std::begin(t));
+    assert(c.get_allocator() == A());
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/sequences/forwardlist/forwardlist.cons/size.pass.cpp b/libcxx/test/std/containers/sequences/forwardlist/forwardlist.cons/size.pass.cpp
index 3c27a6dfa34f1..81b128d2149e3 100644
--- a/libcxx/test/std/containers/sequences/forwardlist/forwardlist.cons/size.pass.cpp
+++ b/libcxx/test/std/containers/sequences/forwardlist/forwardlist.cons/size.pass.cpp
@@ -20,55 +20,53 @@
 #include "min_allocator.h"
 
 template <class T, class Allocator>
-void check_allocator(unsigned n, Allocator const &alloc = Allocator())
-{
+void check_allocator(unsigned n, Allocator const& alloc = Allocator()) {
 #if TEST_STD_VER > 11
-    typedef std::forward_list<T, Allocator> C;
-    C d(n, alloc);
-    assert(d.get_allocator() == alloc);
-    assert(static_cast<std::size_t>(std::distance(d.begin(), d.end())) == n);
+  typedef std::forward_list<T, Allocator> C;
+  C d(n, alloc);
+  assert(d.get_allocator() == alloc);
+  assert(static_cast<std::size_t>(std::distance(d.begin(), d.end())) == n);
 #else
-    ((void)n);
-    ((void)alloc);
+  ((void)n);
+  ((void)alloc);
 #endif
 }
 
-int main(int, char**)
-{
-    { // test that the ctor is explicit
-      typedef std::forward_list<DefaultOnly> C;
-      static_assert((std::is_constructible<C, std::size_t>::value), "");
-      static_assert((!std::is_convertible<std::size_t, C>::value), "");
-    }
-    {
-        typedef DefaultOnly T;
-        typedef std::forward_list<T> C;
-        unsigned N = 10;
-        C c(N);
-        unsigned n = 0;
+int main(int, char**) {
+  { // test that the ctor is explicit
+    typedef std::forward_list<DefaultOnly> C;
+    static_assert((std::is_constructible<C, std::size_t>::value), "");
+    static_assert((!std::is_convertible<std::size_t, C>::value), "");
+  }
+  {
+    typedef DefaultOnly T;
+    typedef std::forward_list<T> C;
+    unsigned N = 10;
+    C c(N);
+    unsigned n = 0;
 
-        for (C::const_iterator i = c.begin(), e = c.end(); i != e; ++i, ++n) {
+    for (C::const_iterator i = c.begin(), e = c.end(); i != e; ++i, ++n) {
 #if TEST_STD_VER >= 11
-            assert(*i == T());
+      assert(*i == T());
 #else
-            ((void)0);
+      ((void)0);
 #endif
-        }
-        assert(n == N);
     }
+    assert(n == N);
+  }
 #if TEST_STD_VER >= 11
-    {
-        typedef DefaultOnly T;
-        typedef std::forward_list<T, min_allocator<T>> C;
-        unsigned N = 10;
-        C c(N);
-        unsigned n = 0;
-        for (C::const_iterator i = c.begin(), e = c.end(); i != e; ++i, ++n)
-            assert(*i == T());
-        assert(n == N);
-        check_allocator<T, min_allocator<T>> ( 0 );
-        check_allocator<T, min_allocator<T>> ( 3 );
-    }
+  {
+    typedef DefaultOnly T;
+    typedef std::forward_list<T, min_allocator<T>> C;
+    unsigned N = 10;
+    C c(N);
+    unsigned n = 0;
+    for (C::const_iterator i = c.begin(), e = c.end(); i != e; ++i, ++n)
+      assert(*i == T());
+    assert(n == N);
+    check_allocator<T, min_allocator<T>>(0);
+    check_allocator<T, min_allocator<T>>(3);
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/sequences/forwardlist/forwardlist.cons/size_value.pass.cpp b/libcxx/test/std/containers/sequences/forwardlist/forwardlist.cons/size_value.pass.cpp
index 4da7394a8b66e..663422d1c3c30 100644
--- a/libcxx/test/std/containers/sequences/forwardlist/forwardlist.cons/size_value.pass.cpp
+++ b/libcxx/test/std/containers/sequences/forwardlist/forwardlist.cons/size_value.pass.cpp
@@ -16,31 +16,30 @@
 #include "test_macros.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef int T;
-        typedef std::forward_list<T> C;
-        T v(6);
-        unsigned N = 10;
-        C c(N, v);
-        unsigned n = 0;
-        for (C::const_iterator i = c.begin(), e = c.end(); i != e; ++i, ++n)
-            assert(*i == v);
-        assert(n == N);
-    }
+int main(int, char**) {
+  {
+    typedef int T;
+    typedef std::forward_list<T> C;
+    T v(6);
+    unsigned N = 10;
+    C c(N, v);
+    unsigned n = 0;
+    for (C::const_iterator i = c.begin(), e = c.end(); i != e; ++i, ++n)
+      assert(*i == v);
+    assert(n == N);
+  }
 #if TEST_STD_VER >= 11
-    {
-        typedef int T;
-        typedef std::forward_list<T, min_allocator<T>> C;
-        T v(6);
-        unsigned N = 10;
-        C c(N, v);
-        unsigned n = 0;
-        for (C::const_iterator i = c.begin(), e = c.end(); i != e; ++i, ++n)
-            assert(*i == v);
-        assert(n == N);
-    }
+  {
+    typedef int T;
+    typedef std::forward_list<T, min_allocator<T>> C;
+    T v(6);
+    unsigned N = 10;
+    C c(N, v);
+    unsigned n = 0;
+    for (C::const_iterator i = c.begin(), e = c.end(); i != e; ++i, ++n)
+      assert(*i == v);
+    assert(n == N);
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/sequences/forwardlist/forwardlist.cons/size_value_alloc.pass.cpp b/libcxx/test/std/containers/sequences/forwardlist/forwardlist.cons/size_value_alloc.pass.cpp
index fb37e8dbdbbd9..af7f7471d4c98 100644
--- a/libcxx/test/std/containers/sequences/forwardlist/forwardlist.cons/size_value_alloc.pass.cpp
+++ b/libcxx/test/std/containers/sequences/forwardlist/forwardlist.cons/size_value_alloc.pass.cpp
@@ -17,35 +17,34 @@
 #include "test_allocator.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef test_allocator<int> A;
-        typedef A::value_type T;
-        typedef std::forward_list<T, A> C;
-        T v(6);
-        unsigned N = 10;
-        C c(N, v, A(12));
-        unsigned n = 0;
-        for (C::const_iterator i = c.begin(), e = c.end(); i != e; ++i, ++n)
-            assert(*i == v);
-        assert(n == N);
-        assert(c.get_allocator() == A(12));
-    }
+int main(int, char**) {
+  {
+    typedef test_allocator<int> A;
+    typedef A::value_type T;
+    typedef std::forward_list<T, A> C;
+    T v(6);
+    unsigned N = 10;
+    C c(N, v, A(12));
+    unsigned n = 0;
+    for (C::const_iterator i = c.begin(), e = c.end(); i != e; ++i, ++n)
+      assert(*i == v);
+    assert(n == N);
+    assert(c.get_allocator() == A(12));
+  }
 #if TEST_STD_VER >= 11
-    {
-        typedef min_allocator<int> A;
-        typedef A::value_type T;
-        typedef std::forward_list<T, A> C;
-        T v(6);
-        unsigned N = 10;
-        C c(N, v, A());
-        unsigned n = 0;
-        for (C::const_iterator i = c.begin(), e = c.end(); i != e; ++i, ++n)
-            assert(*i == v);
-        assert(n == N);
-        assert(c.get_allocator() == A());
-    }
+  {
+    typedef min_allocator<int> A;
+    typedef A::value_type T;
+    typedef std::forward_list<T, A> C;
+    T v(6);
+    unsigned N = 10;
+    C c(N, v, A());
+    unsigned n = 0;
+    for (C::const_iterator i = c.begin(), e = c.end(); i != e; ++i, ++n)
+      assert(*i == v);
+    assert(n == N);
+    assert(c.get_allocator() == A());
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/sequences/forwardlist/forwardlist.erasure/erase.pass.cpp b/libcxx/test/std/containers/sequences/forwardlist/forwardlist.erasure/erase.pass.cpp
index df9aa4139f736..1044d779220ee 100644
--- a/libcxx/test/std/containers/sequences/forwardlist/forwardlist.erasure/erase.pass.cpp
+++ b/libcxx/test/std/containers/sequences/forwardlist/forwardlist.erasure/erase.pass.cpp
@@ -28,8 +28,7 @@ void test0(S s, U val, S expected, std::size_t expected_erased_count) {
 }
 
 template <class S>
-void test()
-{
+void test() {
   test0(S(), 1, S(), 0);
 
   test0(S({1}), 1, S(), 1);
@@ -63,14 +62,13 @@ void test()
   test0(S({1, 2, 1}), opt(3), S({1, 2, 1}), 0);
 }
 
-int main(int, char**)
-{
-    test<std::forward_list<int>>();
-    test<std::forward_list<int, min_allocator<int>>> ();
-    test<std::forward_list<int, test_allocator<int>>> ();
+int main(int, char**) {
+  test<std::forward_list<int>>();
+  test<std::forward_list<int, min_allocator<int>>>();
+  test<std::forward_list<int, test_allocator<int>>>();
 
-    test<std::forward_list<long>>();
-    test<std::forward_list<double>>();
+  test<std::forward_list<long>>();
+  test<std::forward_list<double>>();
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/forwardlist/forwardlist.erasure/erase_if.pass.cpp b/libcxx/test/std/containers/sequences/forwardlist/forwardlist.erasure/erase_if.pass.cpp
index 756febd1a1f17..c4f45a1069a2b 100644
--- a/libcxx/test/std/containers/sequences/forwardlist/forwardlist.erasure/erase_if.pass.cpp
+++ b/libcxx/test/std/containers/sequences/forwardlist/forwardlist.erasure/erase_if.pass.cpp
@@ -27,52 +27,50 @@ void test0(S s, Pred p, S expected, std::size_t expected_erased_count) {
 }
 
 template <typename S>
-void test()
-{
-    auto is1 = [](auto v) { return v == 1;};
-    auto is2 = [](auto v) { return v == 2;};
-    auto is3 = [](auto v) { return v == 3;};
-    auto is4 = [](auto v) { return v == 4;};
-    auto True  = [](auto) { return true; };
-    auto False = [](auto) { return false; };
+void test() {
+  auto is1   = [](auto v) { return v == 1; };
+  auto is2   = [](auto v) { return v == 2; };
+  auto is3   = [](auto v) { return v == 3; };
+  auto is4   = [](auto v) { return v == 4; };
+  auto True  = [](auto) { return true; };
+  auto False = [](auto) { return false; };
 
-    test0(S(), is1, S(), 0);
+  test0(S(), is1, S(), 0);
 
-    test0(S({1}), is1, S(), 1);
-    test0(S({1}), is2, S({1}), 0);
+  test0(S({1}), is1, S(), 1);
+  test0(S({1}), is2, S({1}), 0);
 
-    test0(S({1, 2}), is1, S({2}), 1);
-    test0(S({1, 2}), is2, S({1}), 1);
-    test0(S({1, 2}), is3, S({1, 2}), 0);
-    test0(S({1, 1}), is1, S(), 2);
-    test0(S({1, 1}), is3, S({1, 1}), 0);
+  test0(S({1, 2}), is1, S({2}), 1);
+  test0(S({1, 2}), is2, S({1}), 1);
+  test0(S({1, 2}), is3, S({1, 2}), 0);
+  test0(S({1, 1}), is1, S(), 2);
+  test0(S({1, 1}), is3, S({1, 1}), 0);
 
-    test0(S({1, 2, 3}), is1, S({2, 3}), 1);
-    test0(S({1, 2, 3}), is2, S({1, 3}), 1);
-    test0(S({1, 2, 3}), is3, S({1, 2}), 1);
-    test0(S({1, 2, 3}), is4, S({1, 2, 3}), 0);
+  test0(S({1, 2, 3}), is1, S({2, 3}), 1);
+  test0(S({1, 2, 3}), is2, S({1, 3}), 1);
+  test0(S({1, 2, 3}), is3, S({1, 2}), 1);
+  test0(S({1, 2, 3}), is4, S({1, 2, 3}), 0);
 
-    test0(S({1, 1, 1}), is1, S(), 3);
-    test0(S({1, 1, 1}), is2, S({1, 1, 1}), 0);
-    test0(S({1, 1, 2}), is1, S({2}), 2);
-    test0(S({1, 1, 2}), is2, S({1, 1}), 1);
-    test0(S({1, 1, 2}), is3, S({1, 1, 2}), 0);
-    test0(S({1, 2, 2}), is1, S({2, 2}), 1);
-    test0(S({1, 2, 2}), is2, S({1}), 2);
-    test0(S({1, 2, 2}), is3, S({1, 2, 2}), 0);
+  test0(S({1, 1, 1}), is1, S(), 3);
+  test0(S({1, 1, 1}), is2, S({1, 1, 1}), 0);
+  test0(S({1, 1, 2}), is1, S({2}), 2);
+  test0(S({1, 1, 2}), is2, S({1, 1}), 1);
+  test0(S({1, 1, 2}), is3, S({1, 1, 2}), 0);
+  test0(S({1, 2, 2}), is1, S({2, 2}), 1);
+  test0(S({1, 2, 2}), is2, S({1}), 2);
+  test0(S({1, 2, 2}), is3, S({1, 2, 2}), 0);
 
-    test0(S({1, 2, 3}), True, S(), 3);
-    test0(S({1, 2, 3}), False, S({1, 2, 3}), 0);
+  test0(S({1, 2, 3}), True, S(), 3);
+  test0(S({1, 2, 3}), False, S({1, 2, 3}), 0);
 }
 
-int main(int, char**)
-{
-    test<std::forward_list<int>>();
-    test<std::forward_list<int, min_allocator<int>>> ();
-    test<std::forward_list<int, test_allocator<int>>> ();
+int main(int, char**) {
+  test<std::forward_list<int>>();
+  test<std::forward_list<int, min_allocator<int>>>();
+  test<std::forward_list<int, test_allocator<int>>>();
 
-    test<std::forward_list<long>>();
-    test<std::forward_list<double>>();
+  test<std::forward_list<long>>();
+  test<std::forward_list<double>>();
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/forwardlist/forwardlist.iter/before_begin.pass.cpp b/libcxx/test/std/containers/sequences/forwardlist/forwardlist.iter/before_begin.pass.cpp
index d633a64d6a541..d66d2cd879515 100644
--- a/libcxx/test/std/containers/sequences/forwardlist/forwardlist.iter/before_begin.pass.cpp
+++ b/libcxx/test/std/containers/sequences/forwardlist/forwardlist.iter/before_begin.pass.cpp
@@ -19,87 +19,86 @@
 #include "test_macros.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef int T;
-        typedef std::forward_list<T> C;
-        C c;
-        C::iterator i = c.before_begin();
-        assert(std::distance(i, c.end()) == 1);
-    }
-    {
-        typedef int T;
-        typedef std::forward_list<T> C;
-        const C c;
-        C::const_iterator i = c.before_begin();
-        assert(std::distance(i, c.end()) == 1);
-    }
-    {
-        typedef int T;
-        typedef std::forward_list<T> C;
-        const C c;
-        C::const_iterator i = c.cbefore_begin();
-        assert(std::distance(i, c.end()) == 1);
-        assert(c.cbefore_begin() == c.before_begin());
-    }
-    {
-        typedef int T;
-        typedef std::forward_list<T> C;
-        const T t[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9};
-        C c(std::begin(t), std::end(t));
-        C::iterator i = c.before_begin();
-        assert(std::distance(i, c.end()) == 11);
-        assert(std::next(c.before_begin()) == c.begin());
-    }
-    {
-        typedef int T;
-        typedef std::forward_list<T> C;
-        const T t[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9};
-        const C c(std::begin(t), std::end(t));
-        C::const_iterator i = c.before_begin();
-        assert(std::distance(i, c.end()) == 11);
-    }
+int main(int, char**) {
+  {
+    typedef int T;
+    typedef std::forward_list<T> C;
+    C c;
+    C::iterator i = c.before_begin();
+    assert(std::distance(i, c.end()) == 1);
+  }
+  {
+    typedef int T;
+    typedef std::forward_list<T> C;
+    const C c;
+    C::const_iterator i = c.before_begin();
+    assert(std::distance(i, c.end()) == 1);
+  }
+  {
+    typedef int T;
+    typedef std::forward_list<T> C;
+    const C c;
+    C::const_iterator i = c.cbefore_begin();
+    assert(std::distance(i, c.end()) == 1);
+    assert(c.cbefore_begin() == c.before_begin());
+  }
+  {
+    typedef int T;
+    typedef std::forward_list<T> C;
+    const T t[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9};
+    C c(std::begin(t), std::end(t));
+    C::iterator i = c.before_begin();
+    assert(std::distance(i, c.end()) == 11);
+    assert(std::next(c.before_begin()) == c.begin());
+  }
+  {
+    typedef int T;
+    typedef std::forward_list<T> C;
+    const T t[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9};
+    const C c(std::begin(t), std::end(t));
+    C::const_iterator i = c.before_begin();
+    assert(std::distance(i, c.end()) == 11);
+  }
 #if TEST_STD_VER >= 11
-    {
-        typedef int T;
-        typedef std::forward_list<T, min_allocator<T>> C;
-        C c;
-        C::iterator i = c.before_begin();
-        assert(std::distance(i, c.end()) == 1);
-    }
-    {
-        typedef int T;
-        typedef std::forward_list<T, min_allocator<T>> C;
-        const C c;
-        C::const_iterator i = c.before_begin();
-        assert(std::distance(i, c.end()) == 1);
-    }
-    {
-        typedef int T;
-        typedef std::forward_list<T, min_allocator<T>> C;
-        const C c;
-        C::const_iterator i = c.cbefore_begin();
-        assert(std::distance(i, c.end()) == 1);
-        assert(c.cbefore_begin() == c.before_begin());
-    }
-    {
-        typedef int T;
-        typedef std::forward_list<T, min_allocator<T>> C;
-        const T t[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9};
-        C c(std::begin(t), std::end(t));
-        C::iterator i = c.before_begin();
-        assert(std::distance(i, c.end()) == 11);
-        assert(std::next(c.before_begin()) == c.begin());
-    }
-    {
-        typedef int T;
-        typedef std::forward_list<T, min_allocator<T>> C;
-        const T t[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9};
-        const C c(std::begin(t), std::end(t));
-        C::const_iterator i = c.before_begin();
-        assert(std::distance(i, c.end()) == 11);
-    }
+  {
+    typedef int T;
+    typedef std::forward_list<T, min_allocator<T>> C;
+    C c;
+    C::iterator i = c.before_begin();
+    assert(std::distance(i, c.end()) == 1);
+  }
+  {
+    typedef int T;
+    typedef std::forward_list<T, min_allocator<T>> C;
+    const C c;
+    C::const_iterator i = c.before_begin();
+    assert(std::distance(i, c.end()) == 1);
+  }
+  {
+    typedef int T;
+    typedef std::forward_list<T, min_allocator<T>> C;
+    const C c;
+    C::const_iterator i = c.cbefore_begin();
+    assert(std::distance(i, c.end()) == 1);
+    assert(c.cbefore_begin() == c.before_begin());
+  }
+  {
+    typedef int T;
+    typedef std::forward_list<T, min_allocator<T>> C;
+    const T t[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9};
+    C c(std::begin(t), std::end(t));
+    C::iterator i = c.before_begin();
+    assert(std::distance(i, c.end()) == 11);
+    assert(std::next(c.before_begin()) == c.begin());
+  }
+  {
+    typedef int T;
+    typedef std::forward_list<T, min_allocator<T>> C;
+    const T t[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9};
+    const C c(std::begin(t), std::end(t));
+    C::const_iterator i = c.before_begin();
+    assert(std::distance(i, c.end()) == 11);
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/sequences/forwardlist/forwardlist.iter/iterator_concept_conformance.compile.pass.cpp b/libcxx/test/std/containers/sequences/forwardlist/forwardlist.iter/iterator_concept_conformance.compile.pass.cpp
index 4a1c455df6622..23563e80177a6 100644
--- a/libcxx/test/std/containers/sequences/forwardlist/forwardlist.iter/iterator_concept_conformance.compile.pass.cpp
+++ b/libcxx/test/std/containers/sequences/forwardlist/forwardlist.iter/iterator_concept_conformance.compile.pass.cpp
@@ -14,9 +14,9 @@
 
 #include <iterator>
 
-using iterator = std::forward_list<int>::iterator;
+using iterator       = std::forward_list<int>::iterator;
 using const_iterator = std::forward_list<int>::const_iterator;
-using value_type = int;
+using value_type     = int;
 
 static_assert(std::forward_iterator<iterator>);
 static_assert(!std::bidirectional_iterator<iterator>);
@@ -25,12 +25,12 @@ static_assert(std::sentinel_for<iterator, iterator>);
 static_assert(std::sentinel_for<iterator, const_iterator>);
 static_assert(!std::sized_sentinel_for<iterator, iterator>);
 static_assert(!std::sized_sentinel_for<iterator, const_iterator>);
-static_assert( std::indirectly_movable<iterator, iterator>);
-static_assert( std::indirectly_movable_storable<iterator, iterator>);
+static_assert(std::indirectly_movable<iterator, iterator>);
+static_assert(std::indirectly_movable_storable<iterator, iterator>);
 static_assert(!std::indirectly_movable<iterator, const_iterator>);
 static_assert(!std::indirectly_movable_storable<iterator, const_iterator>);
-static_assert( std::indirectly_copyable<iterator, iterator>);
-static_assert( std::indirectly_copyable_storable<iterator, iterator>);
+static_assert(std::indirectly_copyable<iterator, iterator>);
+static_assert(std::indirectly_copyable_storable<iterator, iterator>);
 static_assert(!std::indirectly_copyable<iterator, const_iterator>);
 static_assert(!std::indirectly_copyable_storable<iterator, const_iterator>);
 static_assert(std::indirectly_swappable<iterator, iterator>);
@@ -42,12 +42,12 @@ static_assert(std::sentinel_for<const_iterator, iterator>);
 static_assert(std::sentinel_for<const_iterator, const_iterator>);
 static_assert(!std::sized_sentinel_for<const_iterator, iterator>);
 static_assert(!std::sized_sentinel_for<const_iterator, const_iterator>);
-static_assert( std::indirectly_movable<const_iterator, iterator>);
-static_assert( std::indirectly_movable_storable<const_iterator, iterator>);
+static_assert(std::indirectly_movable<const_iterator, iterator>);
+static_assert(std::indirectly_movable_storable<const_iterator, iterator>);
 static_assert(!std::indirectly_movable<const_iterator, const_iterator>);
 static_assert(!std::indirectly_movable_storable<const_iterator, const_iterator>);
-static_assert( std::indirectly_copyable<const_iterator, iterator>);
-static_assert( std::indirectly_copyable_storable<const_iterator, iterator>);
+static_assert(std::indirectly_copyable<const_iterator, iterator>);
+static_assert(std::indirectly_copyable_storable<const_iterator, iterator>);
 static_assert(!std::indirectly_copyable<const_iterator, const_iterator>);
 static_assert(!std::indirectly_copyable_storable<const_iterator, const_iterator>);
 static_assert(!std::indirectly_swappable<const_iterator, const_iterator>);
diff --git a/libcxx/test/std/containers/sequences/forwardlist/forwardlist.iter/iterators.pass.cpp b/libcxx/test/std/containers/sequences/forwardlist/forwardlist.iter/iterators.pass.cpp
index e5441109bf373..135689b2321c3 100644
--- a/libcxx/test/std/containers/sequences/forwardlist/forwardlist.iter/iterators.pass.cpp
+++ b/libcxx/test/std/containers/sequences/forwardlist/forwardlist.iter/iterators.pass.cpp
@@ -22,125 +22,124 @@
 #include "test_macros.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef int T;
-        typedef std::forward_list<T> C;
-        C c;
-        C::iterator i = c.begin();
-        C::iterator j = c.end();
-        assert(std::distance(i, j) == 0);
-        assert(i == j);
-    }
-    {
-        typedef int T;
-        typedef std::forward_list<T> C;
-        const C c;
-        C::const_iterator i = c.begin();
-        C::const_iterator j = c.end();
-        assert(std::distance(i, j) == 0);
-        assert(i == j);
-    }
-    {
-        typedef int T;
-        typedef std::forward_list<T> C;
-        C c;
-        C::const_iterator i = c.cbegin();
-        C::const_iterator j = c.cend();
-        assert(std::distance(i, j) == 0);
-        assert(i == j);
-        assert(i == c.end());
-    }
-    {
-        typedef int T;
-        typedef std::forward_list<T> C;
-        const T t[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9};
-        C c(std::begin(t), std::end(t));
-        C::iterator i = c.begin();
-        assert(*i == 0);
-        ++i;
-        assert(*i == 1);
-        *i = 10;
-        assert(*i == 10);
-        assert(std::distance(c.begin(), c.end()) == 10);
-    }
-    {
-        typedef int T;
-        typedef std::forward_list<T> C;
-        C::iterator i;
-        C::const_iterator j;
-    }
+int main(int, char**) {
+  {
+    typedef int T;
+    typedef std::forward_list<T> C;
+    C c;
+    C::iterator i = c.begin();
+    C::iterator j = c.end();
+    assert(std::distance(i, j) == 0);
+    assert(i == j);
+  }
+  {
+    typedef int T;
+    typedef std::forward_list<T> C;
+    const C c;
+    C::const_iterator i = c.begin();
+    C::const_iterator j = c.end();
+    assert(std::distance(i, j) == 0);
+    assert(i == j);
+  }
+  {
+    typedef int T;
+    typedef std::forward_list<T> C;
+    C c;
+    C::const_iterator i = c.cbegin();
+    C::const_iterator j = c.cend();
+    assert(std::distance(i, j) == 0);
+    assert(i == j);
+    assert(i == c.end());
+  }
+  {
+    typedef int T;
+    typedef std::forward_list<T> C;
+    const T t[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9};
+    C c(std::begin(t), std::end(t));
+    C::iterator i = c.begin();
+    assert(*i == 0);
+    ++i;
+    assert(*i == 1);
+    *i = 10;
+    assert(*i == 10);
+    assert(std::distance(c.begin(), c.end()) == 10);
+  }
+  {
+    typedef int T;
+    typedef std::forward_list<T> C;
+    C::iterator i;
+    C::const_iterator j;
+  }
 #if TEST_STD_VER >= 11
-    {
-        typedef int T;
-        typedef std::forward_list<T, min_allocator<T>> C;
-        C c;
-        C::iterator i = c.begin();
-        C::iterator j = c.end();
-        assert(std::distance(i, j) == 0);
-        assert(i == j);
-    }
-    {
-        typedef int T;
-        typedef std::forward_list<T, min_allocator<T>> C;
-        const C c;
-        C::const_iterator i = c.begin();
-        C::const_iterator j = c.end();
-        assert(std::distance(i, j) == 0);
-        assert(i == j);
-    }
-    {
-        typedef int T;
-        typedef std::forward_list<T, min_allocator<T>> C;
-        C c;
-        C::const_iterator i = c.cbegin();
-        C::const_iterator j = c.cend();
-        assert(std::distance(i, j) == 0);
-        assert(i == j);
-        assert(i == c.end());
-    }
-    {
-        typedef int T;
-        typedef std::forward_list<T, min_allocator<T>> C;
-        const T t[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9};
-        C c(std::begin(t), std::end(t));
-        C::iterator i = c.begin();
-        assert(*i == 0);
-        ++i;
-        assert(*i == 1);
-        *i = 10;
-        assert(*i == 10);
-        assert(std::distance(c.begin(), c.end()) == 10);
-    }
-    {
-        typedef int T;
-        typedef std::forward_list<T, min_allocator<T>> C;
-        C::iterator i;
-        C::const_iterator j;
-    }
+  {
+    typedef int T;
+    typedef std::forward_list<T, min_allocator<T>> C;
+    C c;
+    C::iterator i = c.begin();
+    C::iterator j = c.end();
+    assert(std::distance(i, j) == 0);
+    assert(i == j);
+  }
+  {
+    typedef int T;
+    typedef std::forward_list<T, min_allocator<T>> C;
+    const C c;
+    C::const_iterator i = c.begin();
+    C::const_iterator j = c.end();
+    assert(std::distance(i, j) == 0);
+    assert(i == j);
+  }
+  {
+    typedef int T;
+    typedef std::forward_list<T, min_allocator<T>> C;
+    C c;
+    C::const_iterator i = c.cbegin();
+    C::const_iterator j = c.cend();
+    assert(std::distance(i, j) == 0);
+    assert(i == j);
+    assert(i == c.end());
+  }
+  {
+    typedef int T;
+    typedef std::forward_list<T, min_allocator<T>> C;
+    const T t[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9};
+    C c(std::begin(t), std::end(t));
+    C::iterator i = c.begin();
+    assert(*i == 0);
+    ++i;
+    assert(*i == 1);
+    *i = 10;
+    assert(*i == 10);
+    assert(std::distance(c.begin(), c.end()) == 10);
+  }
+  {
+    typedef int T;
+    typedef std::forward_list<T, min_allocator<T>> C;
+    C::iterator i;
+    C::const_iterator j;
+  }
 #endif
 #if TEST_STD_VER > 11
-    { // N3644 testing
-        std::forward_list<int>::iterator ii1{}, ii2{};
-        std::forward_list<int>::iterator ii4 = ii1;
-        std::forward_list<int>::const_iterator cii{};
-        assert ( ii1 == ii2 );
-        assert ( ii1 == ii4 );
+  { // N3644 testing
+    std::forward_list<int>::iterator ii1{}, ii2{};
+    std::forward_list<int>::iterator ii4 = ii1;
+    std::forward_list<int>::const_iterator cii{};
+    assert(ii1 == ii2);
+    assert(ii1 == ii4);
 
-        assert (!(ii1 != ii2 ));
+    assert(!(ii1 != ii2));
 
-        assert ( (ii1 == cii ));
-        assert ( (cii == ii1 ));
-        assert (!(ii1 != cii ));
-        assert (!(cii != ii1 ));
+    assert((ii1 == cii));
+    assert((cii == ii1));
+    assert(!(ii1 != cii));
+    assert(!(cii != ii1));
 
-//         std::forward_list<int> c;
-//         assert ( ii1 != c.cbegin());
-//         assert ( cii != c.begin());
-//         assert ( cii != c.cend());
-//         assert ( ii1 != c.end());
-    }
+    //         std::forward_list<int> c;
+    //         assert ( ii1 != c.cbegin());
+    //         assert ( cii != c.begin());
+    //         assert ( cii != c.cend());
+    //         assert ( ii1 != c.end());
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/sequences/forwardlist/forwardlist.modifiers/clear.pass.cpp b/libcxx/test/std/containers/sequences/forwardlist/forwardlist.modifiers/clear.pass.cpp
index 55ce87cfe174c..9f6d34b701df7 100644
--- a/libcxx/test/std/containers/sequences/forwardlist/forwardlist.modifiers/clear.pass.cpp
+++ b/libcxx/test/std/containers/sequences/forwardlist/forwardlist.modifiers/clear.pass.cpp
@@ -18,51 +18,50 @@
 #include "../../../NotConstructible.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef NotConstructible T;
-        typedef std::forward_list<T> C;
-        C c;
-        ASSERT_NOEXCEPT(c.clear());
-        c.clear();
-        assert(std::distance(c.begin(), c.end()) == 0);
-    }
-    {
-        typedef int T;
-        typedef std::forward_list<T> C;
-        const T t[] = {0, 1, 2, 3, 4};
-        C c(std::begin(t), std::end(t));
+int main(int, char**) {
+  {
+    typedef NotConstructible T;
+    typedef std::forward_list<T> C;
+    C c;
+    ASSERT_NOEXCEPT(c.clear());
+    c.clear();
+    assert(std::distance(c.begin(), c.end()) == 0);
+  }
+  {
+    typedef int T;
+    typedef std::forward_list<T> C;
+    const T t[] = {0, 1, 2, 3, 4};
+    C c(std::begin(t), std::end(t));
 
-        ASSERT_NOEXCEPT(c.clear());
-        c.clear();
-        assert(std::distance(c.begin(), c.end()) == 0);
+    ASSERT_NOEXCEPT(c.clear());
+    c.clear();
+    assert(std::distance(c.begin(), c.end()) == 0);
 
-        c.clear();
-        assert(std::distance(c.begin(), c.end()) == 0);
-    }
+    c.clear();
+    assert(std::distance(c.begin(), c.end()) == 0);
+  }
 #if TEST_STD_VER >= 11
-    {
-        typedef NotConstructible T;
-        typedef std::forward_list<T, min_allocator<T>> C;
-        C c;
-        ASSERT_NOEXCEPT(c.clear());
-        c.clear();
-        assert(std::distance(c.begin(), c.end()) == 0);
-    }
-    {
-        typedef int T;
-        typedef std::forward_list<T, min_allocator<T>> C;
-        const T t[] = {0, 1, 2, 3, 4};
-        C c(std::begin(t), std::end(t));
+  {
+    typedef NotConstructible T;
+    typedef std::forward_list<T, min_allocator<T>> C;
+    C c;
+    ASSERT_NOEXCEPT(c.clear());
+    c.clear();
+    assert(std::distance(c.begin(), c.end()) == 0);
+  }
+  {
+    typedef int T;
+    typedef std::forward_list<T, min_allocator<T>> C;
+    const T t[] = {0, 1, 2, 3, 4};
+    C c(std::begin(t), std::end(t));
 
-        ASSERT_NOEXCEPT(c.clear());
-        c.clear();
-        assert(std::distance(c.begin(), c.end()) == 0);
+    ASSERT_NOEXCEPT(c.clear());
+    c.clear();
+    assert(std::distance(c.begin(), c.end()) == 0);
 
-        c.clear();
-        assert(std::distance(c.begin(), c.end()) == 0);
-    }
+    c.clear();
+    assert(std::distance(c.begin(), c.end()) == 0);
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/sequences/forwardlist/forwardlist.modifiers/emplace_after.pass.cpp b/libcxx/test/std/containers/sequences/forwardlist/forwardlist.modifiers/emplace_after.pass.cpp
index 0d45323a5f7a9..f77d47ee7c74f 100644
--- a/libcxx/test/std/containers/sequences/forwardlist/forwardlist.modifiers/emplace_after.pass.cpp
+++ b/libcxx/test/std/containers/sequences/forwardlist/forwardlist.modifiers/emplace_after.pass.cpp
@@ -20,70 +20,69 @@
 #include "../../../Emplaceable.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef Emplaceable T;
-        typedef std::forward_list<T> C;
-        typedef C::iterator I;
-        C c;
-        I i = c.emplace_after(c.cbefore_begin());
-        assert(i == c.begin());
-        assert(c.front() == Emplaceable());
-        assert(std::distance(c.begin(), c.end()) == 1);
+int main(int, char**) {
+  {
+    typedef Emplaceable T;
+    typedef std::forward_list<T> C;
+    typedef C::iterator I;
+    C c;
+    I i = c.emplace_after(c.cbefore_begin());
+    assert(i == c.begin());
+    assert(c.front() == Emplaceable());
+    assert(std::distance(c.begin(), c.end()) == 1);
 
-        i = c.emplace_after(c.cbegin(), 1, 2.5);
-        assert(i == std::next(c.begin()));
-        assert(c.front() == Emplaceable());
-        assert(*std::next(c.begin()) == Emplaceable(1, 2.5));
-        assert(std::distance(c.begin(), c.end()) == 2);
+    i = c.emplace_after(c.cbegin(), 1, 2.5);
+    assert(i == std::next(c.begin()));
+    assert(c.front() == Emplaceable());
+    assert(*std::next(c.begin()) == Emplaceable(1, 2.5));
+    assert(std::distance(c.begin(), c.end()) == 2);
 
-        i = c.emplace_after(std::next(c.cbegin()), 2, 3.5);
-        assert(i == std::next(c.begin(), 2));
-        assert(c.front() == Emplaceable());
-        assert(*std::next(c.begin()) == Emplaceable(1, 2.5));
-        assert(*std::next(c.begin(), 2) == Emplaceable(2, 3.5));
-        assert(std::distance(c.begin(), c.end()) == 3);
+    i = c.emplace_after(std::next(c.cbegin()), 2, 3.5);
+    assert(i == std::next(c.begin(), 2));
+    assert(c.front() == Emplaceable());
+    assert(*std::next(c.begin()) == Emplaceable(1, 2.5));
+    assert(*std::next(c.begin(), 2) == Emplaceable(2, 3.5));
+    assert(std::distance(c.begin(), c.end()) == 3);
 
-        i = c.emplace_after(c.cbegin(), 3, 4.5);
-        assert(i == std::next(c.begin()));
-        assert(c.front() == Emplaceable());
-        assert(*std::next(c.begin(), 1) == Emplaceable(3, 4.5));
-        assert(*std::next(c.begin(), 2) == Emplaceable(1, 2.5));
-        assert(*std::next(c.begin(), 3) == Emplaceable(2, 3.5));
-        assert(std::distance(c.begin(), c.end()) == 4);
-    }
-    {
-        typedef Emplaceable T;
-        typedef std::forward_list<T, min_allocator<T>> C;
-        typedef C::iterator I;
-        C c;
-        I i = c.emplace_after(c.cbefore_begin());
-        assert(i == c.begin());
-        assert(c.front() == Emplaceable());
-        assert(std::distance(c.begin(), c.end()) == 1);
+    i = c.emplace_after(c.cbegin(), 3, 4.5);
+    assert(i == std::next(c.begin()));
+    assert(c.front() == Emplaceable());
+    assert(*std::next(c.begin(), 1) == Emplaceable(3, 4.5));
+    assert(*std::next(c.begin(), 2) == Emplaceable(1, 2.5));
+    assert(*std::next(c.begin(), 3) == Emplaceable(2, 3.5));
+    assert(std::distance(c.begin(), c.end()) == 4);
+  }
+  {
+    typedef Emplaceable T;
+    typedef std::forward_list<T, min_allocator<T>> C;
+    typedef C::iterator I;
+    C c;
+    I i = c.emplace_after(c.cbefore_begin());
+    assert(i == c.begin());
+    assert(c.front() == Emplaceable());
+    assert(std::distance(c.begin(), c.end()) == 1);
 
-        i = c.emplace_after(c.cbegin(), 1, 2.5);
-        assert(i == std::next(c.begin()));
-        assert(c.front() == Emplaceable());
-        assert(*std::next(c.begin()) == Emplaceable(1, 2.5));
-        assert(std::distance(c.begin(), c.end()) == 2);
+    i = c.emplace_after(c.cbegin(), 1, 2.5);
+    assert(i == std::next(c.begin()));
+    assert(c.front() == Emplaceable());
+    assert(*std::next(c.begin()) == Emplaceable(1, 2.5));
+    assert(std::distance(c.begin(), c.end()) == 2);
 
-        i = c.emplace_after(std::next(c.cbegin()), 2, 3.5);
-        assert(i == std::next(c.begin(), 2));
-        assert(c.front() == Emplaceable());
-        assert(*std::next(c.begin()) == Emplaceable(1, 2.5));
-        assert(*std::next(c.begin(), 2) == Emplaceable(2, 3.5));
-        assert(std::distance(c.begin(), c.end()) == 3);
+    i = c.emplace_after(std::next(c.cbegin()), 2, 3.5);
+    assert(i == std::next(c.begin(), 2));
+    assert(c.front() == Emplaceable());
+    assert(*std::next(c.begin()) == Emplaceable(1, 2.5));
+    assert(*std::next(c.begin(), 2) == Emplaceable(2, 3.5));
+    assert(std::distance(c.begin(), c.end()) == 3);
 
-        i = c.emplace_after(c.cbegin(), 3, 4.5);
-        assert(i == std::next(c.begin()));
-        assert(c.front() == Emplaceable());
-        assert(*std::next(c.begin(), 1) == Emplaceable(3, 4.5));
-        assert(*std::next(c.begin(), 2) == Emplaceable(1, 2.5));
-        assert(*std::next(c.begin(), 3) == Emplaceable(2, 3.5));
-        assert(std::distance(c.begin(), c.end()) == 4);
-    }
+    i = c.emplace_after(c.cbegin(), 3, 4.5);
+    assert(i == std::next(c.begin()));
+    assert(c.front() == Emplaceable());
+    assert(*std::next(c.begin(), 1) == Emplaceable(3, 4.5));
+    assert(*std::next(c.begin(), 2) == Emplaceable(1, 2.5));
+    assert(*std::next(c.begin(), 3) == Emplaceable(2, 3.5));
+    assert(std::distance(c.begin(), c.end()) == 4);
+  }
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/forwardlist/forwardlist.modifiers/emplace_front.pass.cpp b/libcxx/test/std/containers/sequences/forwardlist/forwardlist.modifiers/emplace_front.pass.cpp
index 47e79cb5136bb..cd3bb20c52ae5 100644
--- a/libcxx/test/std/containers/sequences/forwardlist/forwardlist.modifiers/emplace_front.pass.cpp
+++ b/libcxx/test/std/containers/sequences/forwardlist/forwardlist.modifiers/emplace_front.pass.cpp
@@ -21,52 +21,51 @@
 #include "../../../Emplaceable.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef Emplaceable T;
-        typedef std::forward_list<T> C;
-        C c;
+int main(int, char**) {
+  {
+    typedef Emplaceable T;
+    typedef std::forward_list<T> C;
+    C c;
 #if TEST_STD_VER > 14
-        T& r1 = c.emplace_front();
-        assert(c.front() == Emplaceable());
-        assert(&r1 == &c.front());
-        assert(std::distance(c.begin(), c.end()) == 1);
-        T& r2 = c.emplace_front(1, 2.5);
-        assert(c.front() == Emplaceable(1, 2.5));
-        assert(&r2 == &c.front());
+    T& r1 = c.emplace_front();
+    assert(c.front() == Emplaceable());
+    assert(&r1 == &c.front());
+    assert(std::distance(c.begin(), c.end()) == 1);
+    T& r2 = c.emplace_front(1, 2.5);
+    assert(c.front() == Emplaceable(1, 2.5));
+    assert(&r2 == &c.front());
 #else
-        c.emplace_front();
-        assert(c.front() == Emplaceable());
-        assert(std::distance(c.begin(), c.end()) == 1);
-        c.emplace_front(1, 2.5);
-        assert(c.front() == Emplaceable(1, 2.5));
+    c.emplace_front();
+    assert(c.front() == Emplaceable());
+    assert(std::distance(c.begin(), c.end()) == 1);
+    c.emplace_front(1, 2.5);
+    assert(c.front() == Emplaceable(1, 2.5));
 #endif
-        assert(*std::next(c.begin()) == Emplaceable());
-        assert(std::distance(c.begin(), c.end()) == 2);
-    }
-    {
-        typedef Emplaceable T;
-        typedef std::forward_list<T, min_allocator<T>> C;
-        C c;
+    assert(*std::next(c.begin()) == Emplaceable());
+    assert(std::distance(c.begin(), c.end()) == 2);
+  }
+  {
+    typedef Emplaceable T;
+    typedef std::forward_list<T, min_allocator<T>> C;
+    C c;
 #if TEST_STD_VER > 14
-        T& r1 = c.emplace_front();
-        assert(c.front() == Emplaceable());
-        assert(&r1 == &c.front());
-        assert(std::distance(c.begin(), c.end()) == 1);
-        T& r2 = c.emplace_front(1, 2.5);
-        assert(c.front() == Emplaceable(1, 2.5));
-        assert(&r2 == &c.front());
+    T& r1 = c.emplace_front();
+    assert(c.front() == Emplaceable());
+    assert(&r1 == &c.front());
+    assert(std::distance(c.begin(), c.end()) == 1);
+    T& r2 = c.emplace_front(1, 2.5);
+    assert(c.front() == Emplaceable(1, 2.5));
+    assert(&r2 == &c.front());
 #else
-        c.emplace_front();
-        assert(c.front() == Emplaceable());
-        assert(std::distance(c.begin(), c.end()) == 1);
-        c.emplace_front(1, 2.5);
-        assert(c.front() == Emplaceable(1, 2.5));
+    c.emplace_front();
+    assert(c.front() == Emplaceable());
+    assert(std::distance(c.begin(), c.end()) == 1);
+    c.emplace_front(1, 2.5);
+    assert(c.front() == Emplaceable(1, 2.5));
 #endif
-        assert(*std::next(c.begin()) == Emplaceable());
-        assert(std::distance(c.begin(), c.end()) == 2);
-    }
+    assert(*std::next(c.begin()) == Emplaceable());
+    assert(std::distance(c.begin(), c.end()) == 2);
+  }
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/forwardlist/forwardlist.modifiers/erase_after_many.pass.cpp b/libcxx/test/std/containers/sequences/forwardlist/forwardlist.modifiers/erase_after_many.pass.cpp
index 51476559b60ab..e85951798526d 100644
--- a/libcxx/test/std/containers/sequences/forwardlist/forwardlist.modifiers/erase_after_many.pass.cpp
+++ b/libcxx/test/std/containers/sequences/forwardlist/forwardlist.modifiers/erase_after_many.pass.cpp
@@ -17,141 +17,140 @@
 #include "test_macros.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef int T;
-        typedef std::forward_list<T> C;
-        const T t[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9};
-        C c(std::begin(t), std::end(t));
-
-        C::iterator i = c.erase_after(std::next(c.cbefore_begin(), 4), std::next(c.cbefore_begin(), 4));
-        assert(i == std::next(c.cbefore_begin(), 4));
-        assert(std::distance(c.begin(), c.end()) == 10);
-        assert(*std::next(c.begin(), 0) == 0);
-        assert(*std::next(c.begin(), 1) == 1);
-        assert(*std::next(c.begin(), 2) == 2);
-        assert(*std::next(c.begin(), 3) == 3);
-        assert(*std::next(c.begin(), 4) == 4);
-        assert(*std::next(c.begin(), 5) == 5);
-        assert(*std::next(c.begin(), 6) == 6);
-        assert(*std::next(c.begin(), 7) == 7);
-        assert(*std::next(c.begin(), 8) == 8);
-        assert(*std::next(c.begin(), 9) == 9);
-
-        i = c.erase_after(std::next(c.cbefore_begin(), 2), std::next(c.cbefore_begin(), 5));
-        assert(i == std::next(c.begin(), 2));
-        assert(std::distance(c.begin(), c.end()) == 8);
-        assert(*std::next(c.begin(), 0) == 0);
-        assert(*std::next(c.begin(), 1) == 1);
-        assert(*std::next(c.begin(), 2) == 4);
-        assert(*std::next(c.begin(), 3) == 5);
-        assert(*std::next(c.begin(), 4) == 6);
-        assert(*std::next(c.begin(), 5) == 7);
-        assert(*std::next(c.begin(), 6) == 8);
-        assert(*std::next(c.begin(), 7) == 9);
-
-        i = c.erase_after(std::next(c.cbefore_begin(), 2), std::next(c.cbefore_begin(), 3));
-        assert(i == std::next(c.begin(), 2));
-        assert(std::distance(c.begin(), c.end()) == 8);
-        assert(*std::next(c.begin(), 0) == 0);
-        assert(*std::next(c.begin(), 1) == 1);
-        assert(*std::next(c.begin(), 2) == 4);
-        assert(*std::next(c.begin(), 3) == 5);
-        assert(*std::next(c.begin(), 4) == 6);
-        assert(*std::next(c.begin(), 5) == 7);
-        assert(*std::next(c.begin(), 6) == 8);
-        assert(*std::next(c.begin(), 7) == 9);
-
-        i = c.erase_after(std::next(c.cbefore_begin(), 5), std::next(c.cbefore_begin(), 9));
-        assert(i == c.end());
-        assert(std::distance(c.begin(), c.end()) == 5);
-        assert(*std::next(c.begin(), 0) == 0);
-        assert(*std::next(c.begin(), 1) == 1);
-        assert(*std::next(c.begin(), 2) == 4);
-        assert(*std::next(c.begin(), 3) == 5);
-        assert(*std::next(c.begin(), 4) == 6);
-
-        i = c.erase_after(std::next(c.cbefore_begin(), 0), std::next(c.cbefore_begin(), 2));
-        assert(i == c.begin());
-        assert(std::distance(c.begin(), c.end()) == 4);
-        assert(*std::next(c.begin(), 0) == 1);
-        assert(*std::next(c.begin(), 1) == 4);
-        assert(*std::next(c.begin(), 2) == 5);
-        assert(*std::next(c.begin(), 3) == 6);
-
-        i = c.erase_after(std::next(c.cbefore_begin(), 0), std::next(c.cbefore_begin(), 5));
-        assert(i == c.begin());
-        assert(i == c.end());
-        assert(std::distance(c.begin(), c.end()) == 0);
-    }
+int main(int, char**) {
+  {
+    typedef int T;
+    typedef std::forward_list<T> C;
+    const T t[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9};
+    C c(std::begin(t), std::end(t));
+
+    C::iterator i = c.erase_after(std::next(c.cbefore_begin(), 4), std::next(c.cbefore_begin(), 4));
+    assert(i == std::next(c.cbefore_begin(), 4));
+    assert(std::distance(c.begin(), c.end()) == 10);
+    assert(*std::next(c.begin(), 0) == 0);
+    assert(*std::next(c.begin(), 1) == 1);
+    assert(*std::next(c.begin(), 2) == 2);
+    assert(*std::next(c.begin(), 3) == 3);
+    assert(*std::next(c.begin(), 4) == 4);
+    assert(*std::next(c.begin(), 5) == 5);
+    assert(*std::next(c.begin(), 6) == 6);
+    assert(*std::next(c.begin(), 7) == 7);
+    assert(*std::next(c.begin(), 8) == 8);
+    assert(*std::next(c.begin(), 9) == 9);
+
+    i = c.erase_after(std::next(c.cbefore_begin(), 2), std::next(c.cbefore_begin(), 5));
+    assert(i == std::next(c.begin(), 2));
+    assert(std::distance(c.begin(), c.end()) == 8);
+    assert(*std::next(c.begin(), 0) == 0);
+    assert(*std::next(c.begin(), 1) == 1);
+    assert(*std::next(c.begin(), 2) == 4);
+    assert(*std::next(c.begin(), 3) == 5);
+    assert(*std::next(c.begin(), 4) == 6);
+    assert(*std::next(c.begin(), 5) == 7);
+    assert(*std::next(c.begin(), 6) == 8);
+    assert(*std::next(c.begin(), 7) == 9);
+
+    i = c.erase_after(std::next(c.cbefore_begin(), 2), std::next(c.cbefore_begin(), 3));
+    assert(i == std::next(c.begin(), 2));
+    assert(std::distance(c.begin(), c.end()) == 8);
+    assert(*std::next(c.begin(), 0) == 0);
+    assert(*std::next(c.begin(), 1) == 1);
+    assert(*std::next(c.begin(), 2) == 4);
+    assert(*std::next(c.begin(), 3) == 5);
+    assert(*std::next(c.begin(), 4) == 6);
+    assert(*std::next(c.begin(), 5) == 7);
+    assert(*std::next(c.begin(), 6) == 8);
+    assert(*std::next(c.begin(), 7) == 9);
+
+    i = c.erase_after(std::next(c.cbefore_begin(), 5), std::next(c.cbefore_begin(), 9));
+    assert(i == c.end());
+    assert(std::distance(c.begin(), c.end()) == 5);
+    assert(*std::next(c.begin(), 0) == 0);
+    assert(*std::next(c.begin(), 1) == 1);
+    assert(*std::next(c.begin(), 2) == 4);
+    assert(*std::next(c.begin(), 3) == 5);
+    assert(*std::next(c.begin(), 4) == 6);
+
+    i = c.erase_after(std::next(c.cbefore_begin(), 0), std::next(c.cbefore_begin(), 2));
+    assert(i == c.begin());
+    assert(std::distance(c.begin(), c.end()) == 4);
+    assert(*std::next(c.begin(), 0) == 1);
+    assert(*std::next(c.begin(), 1) == 4);
+    assert(*std::next(c.begin(), 2) == 5);
+    assert(*std::next(c.begin(), 3) == 6);
+
+    i = c.erase_after(std::next(c.cbefore_begin(), 0), std::next(c.cbefore_begin(), 5));
+    assert(i == c.begin());
+    assert(i == c.end());
+    assert(std::distance(c.begin(), c.end()) == 0);
+  }
 #if TEST_STD_VER >= 11
-    {
-        typedef int T;
-        typedef std::forward_list<T, min_allocator<T>> C;
-        const T t[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9};
-        C c(std::begin(t), std::end(t));
-
-        C::iterator i = c.erase_after(std::next(c.cbefore_begin(), 4), std::next(c.cbefore_begin(), 4));
-        assert(i == std::next(c.cbefore_begin(), 4));
-        assert(std::distance(c.begin(), c.end()) == 10);
-        assert(*std::next(c.begin(), 0) == 0);
-        assert(*std::next(c.begin(), 1) == 1);
-        assert(*std::next(c.begin(), 2) == 2);
-        assert(*std::next(c.begin(), 3) == 3);
-        assert(*std::next(c.begin(), 4) == 4);
-        assert(*std::next(c.begin(), 5) == 5);
-        assert(*std::next(c.begin(), 6) == 6);
-        assert(*std::next(c.begin(), 7) == 7);
-        assert(*std::next(c.begin(), 8) == 8);
-        assert(*std::next(c.begin(), 9) == 9);
-
-        i = c.erase_after(std::next(c.cbefore_begin(), 2), std::next(c.cbefore_begin(), 5));
-        assert(i == std::next(c.begin(), 2));
-        assert(std::distance(c.begin(), c.end()) == 8);
-        assert(*std::next(c.begin(), 0) == 0);
-        assert(*std::next(c.begin(), 1) == 1);
-        assert(*std::next(c.begin(), 2) == 4);
-        assert(*std::next(c.begin(), 3) == 5);
-        assert(*std::next(c.begin(), 4) == 6);
-        assert(*std::next(c.begin(), 5) == 7);
-        assert(*std::next(c.begin(), 6) == 8);
-        assert(*std::next(c.begin(), 7) == 9);
-
-        i = c.erase_after(std::next(c.cbefore_begin(), 2), std::next(c.cbefore_begin(), 3));
-        assert(i == std::next(c.begin(), 2));
-        assert(std::distance(c.begin(), c.end()) == 8);
-        assert(*std::next(c.begin(), 0) == 0);
-        assert(*std::next(c.begin(), 1) == 1);
-        assert(*std::next(c.begin(), 2) == 4);
-        assert(*std::next(c.begin(), 3) == 5);
-        assert(*std::next(c.begin(), 4) == 6);
-        assert(*std::next(c.begin(), 5) == 7);
-        assert(*std::next(c.begin(), 6) == 8);
-        assert(*std::next(c.begin(), 7) == 9);
-
-        i = c.erase_after(std::next(c.cbefore_begin(), 5), std::next(c.cbefore_begin(), 9));
-        assert(i == c.end());
-        assert(std::distance(c.begin(), c.end()) == 5);
-        assert(*std::next(c.begin(), 0) == 0);
-        assert(*std::next(c.begin(), 1) == 1);
-        assert(*std::next(c.begin(), 2) == 4);
-        assert(*std::next(c.begin(), 3) == 5);
-        assert(*std::next(c.begin(), 4) == 6);
-
-        i = c.erase_after(std::next(c.cbefore_begin(), 0), std::next(c.cbefore_begin(), 2));
-        assert(i == c.begin());
-        assert(std::distance(c.begin(), c.end()) == 4);
-        assert(*std::next(c.begin(), 0) == 1);
-        assert(*std::next(c.begin(), 1) == 4);
-        assert(*std::next(c.begin(), 2) == 5);
-        assert(*std::next(c.begin(), 3) == 6);
-
-        i = c.erase_after(std::next(c.cbefore_begin(), 0), std::next(c.cbefore_begin(), 5));
-        assert(i == c.begin());
-        assert(i == c.end());
-        assert(std::distance(c.begin(), c.end()) == 0);
-    }
+  {
+    typedef int T;
+    typedef std::forward_list<T, min_allocator<T>> C;
+    const T t[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9};
+    C c(std::begin(t), std::end(t));
+
+    C::iterator i = c.erase_after(std::next(c.cbefore_begin(), 4), std::next(c.cbefore_begin(), 4));
+    assert(i == std::next(c.cbefore_begin(), 4));
+    assert(std::distance(c.begin(), c.end()) == 10);
+    assert(*std::next(c.begin(), 0) == 0);
+    assert(*std::next(c.begin(), 1) == 1);
+    assert(*std::next(c.begin(), 2) == 2);
+    assert(*std::next(c.begin(), 3) == 3);
+    assert(*std::next(c.begin(), 4) == 4);
+    assert(*std::next(c.begin(), 5) == 5);
+    assert(*std::next(c.begin(), 6) == 6);
+    assert(*std::next(c.begin(), 7) == 7);
+    assert(*std::next(c.begin(), 8) == 8);
+    assert(*std::next(c.begin(), 9) == 9);
+
+    i = c.erase_after(std::next(c.cbefore_begin(), 2), std::next(c.cbefore_begin(), 5));
+    assert(i == std::next(c.begin(), 2));
+    assert(std::distance(c.begin(), c.end()) == 8);
+    assert(*std::next(c.begin(), 0) == 0);
+    assert(*std::next(c.begin(), 1) == 1);
+    assert(*std::next(c.begin(), 2) == 4);
+    assert(*std::next(c.begin(), 3) == 5);
+    assert(*std::next(c.begin(), 4) == 6);
+    assert(*std::next(c.begin(), 5) == 7);
+    assert(*std::next(c.begin(), 6) == 8);
+    assert(*std::next(c.begin(), 7) == 9);
+
+    i = c.erase_after(std::next(c.cbefore_begin(), 2), std::next(c.cbefore_begin(), 3));
+    assert(i == std::next(c.begin(), 2));
+    assert(std::distance(c.begin(), c.end()) == 8);
+    assert(*std::next(c.begin(), 0) == 0);
+    assert(*std::next(c.begin(), 1) == 1);
+    assert(*std::next(c.begin(), 2) == 4);
+    assert(*std::next(c.begin(), 3) == 5);
+    assert(*std::next(c.begin(), 4) == 6);
+    assert(*std::next(c.begin(), 5) == 7);
+    assert(*std::next(c.begin(), 6) == 8);
+    assert(*std::next(c.begin(), 7) == 9);
+
+    i = c.erase_after(std::next(c.cbefore_begin(), 5), std::next(c.cbefore_begin(), 9));
+    assert(i == c.end());
+    assert(std::distance(c.begin(), c.end()) == 5);
+    assert(*std::next(c.begin(), 0) == 0);
+    assert(*std::next(c.begin(), 1) == 1);
+    assert(*std::next(c.begin(), 2) == 4);
+    assert(*std::next(c.begin(), 3) == 5);
+    assert(*std::next(c.begin(), 4) == 6);
+
+    i = c.erase_after(std::next(c.cbefore_begin(), 0), std::next(c.cbefore_begin(), 2));
+    assert(i == c.begin());
+    assert(std::distance(c.begin(), c.end()) == 4);
+    assert(*std::next(c.begin(), 0) == 1);
+    assert(*std::next(c.begin(), 1) == 4);
+    assert(*std::next(c.begin(), 2) == 5);
+    assert(*std::next(c.begin(), 3) == 6);
+
+    i = c.erase_after(std::next(c.cbefore_begin(), 0), std::next(c.cbefore_begin(), 5));
+    assert(i == c.begin());
+    assert(i == c.end());
+    assert(std::distance(c.begin(), c.end()) == 0);
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/sequences/forwardlist/forwardlist.modifiers/erase_after_one.pass.cpp b/libcxx/test/std/containers/sequences/forwardlist/forwardlist.modifiers/erase_after_one.pass.cpp
index ce035955862b8..892228e76def7 100644
--- a/libcxx/test/std/containers/sequences/forwardlist/forwardlist.modifiers/erase_after_one.pass.cpp
+++ b/libcxx/test/std/containers/sequences/forwardlist/forwardlist.modifiers/erase_after_one.pass.cpp
@@ -17,83 +17,82 @@
 #include "test_macros.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef int T;
-        typedef std::forward_list<T> C;
-        const T t[] = {0, 1, 2, 3, 4};
-        C c(std::begin(t), std::end(t));
-
-        C::iterator i = c.erase_after(std::next(c.cbefore_begin(), 4));
-        assert(i == c.end());
-        assert(std::distance(c.begin(), c.end()) == 4);
-        assert(*std::next(c.begin(), 0) == 0);
-        assert(*std::next(c.begin(), 1) == 1);
-        assert(*std::next(c.begin(), 2) == 2);
-        assert(*std::next(c.begin(), 3) == 3);
-
-        i = c.erase_after(std::next(c.cbefore_begin(), 0));
-        assert(i == c.begin());
-        assert(std::distance(c.begin(), c.end()) == 3);
-        assert(*std::next(c.begin(), 0) == 1);
-        assert(*std::next(c.begin(), 1) == 2);
-        assert(*std::next(c.begin(), 2) == 3);
-
-        i = c.erase_after(std::next(c.cbefore_begin(), 1));
-        assert(i == std::next(c.begin()));
-        assert(std::distance(c.begin(), c.end()) == 2);
-        assert(*std::next(c.begin(), 0) == 1);
-        assert(*std::next(c.begin(), 1) == 3);
-
-        i = c.erase_after(std::next(c.cbefore_begin(), 1));
-        assert(i == c.end());
-        assert(std::distance(c.begin(), c.end()) == 1);
-        assert(*std::next(c.begin(), 0) == 1);
-
-        i = c.erase_after(std::next(c.cbefore_begin(), 0));
-        assert(i == c.begin());
-        assert(i == c.end());
-        assert(std::distance(c.begin(), c.end()) == 0);
-    }
+int main(int, char**) {
+  {
+    typedef int T;
+    typedef std::forward_list<T> C;
+    const T t[] = {0, 1, 2, 3, 4};
+    C c(std::begin(t), std::end(t));
+
+    C::iterator i = c.erase_after(std::next(c.cbefore_begin(), 4));
+    assert(i == c.end());
+    assert(std::distance(c.begin(), c.end()) == 4);
+    assert(*std::next(c.begin(), 0) == 0);
+    assert(*std::next(c.begin(), 1) == 1);
+    assert(*std::next(c.begin(), 2) == 2);
+    assert(*std::next(c.begin(), 3) == 3);
+
+    i = c.erase_after(std::next(c.cbefore_begin(), 0));
+    assert(i == c.begin());
+    assert(std::distance(c.begin(), c.end()) == 3);
+    assert(*std::next(c.begin(), 0) == 1);
+    assert(*std::next(c.begin(), 1) == 2);
+    assert(*std::next(c.begin(), 2) == 3);
+
+    i = c.erase_after(std::next(c.cbefore_begin(), 1));
+    assert(i == std::next(c.begin()));
+    assert(std::distance(c.begin(), c.end()) == 2);
+    assert(*std::next(c.begin(), 0) == 1);
+    assert(*std::next(c.begin(), 1) == 3);
+
+    i = c.erase_after(std::next(c.cbefore_begin(), 1));
+    assert(i == c.end());
+    assert(std::distance(c.begin(), c.end()) == 1);
+    assert(*std::next(c.begin(), 0) == 1);
+
+    i = c.erase_after(std::next(c.cbefore_begin(), 0));
+    assert(i == c.begin());
+    assert(i == c.end());
+    assert(std::distance(c.begin(), c.end()) == 0);
+  }
 #if TEST_STD_VER >= 11
-    {
-        typedef int T;
-        typedef std::forward_list<T, min_allocator<T>> C;
-        const T t[] = {0, 1, 2, 3, 4};
-        C c(std::begin(t), std::end(t));
-
-        C::iterator i = c.erase_after(std::next(c.cbefore_begin(), 4));
-        assert(i == c.end());
-        assert(std::distance(c.begin(), c.end()) == 4);
-        assert(*std::next(c.begin(), 0) == 0);
-        assert(*std::next(c.begin(), 1) == 1);
-        assert(*std::next(c.begin(), 2) == 2);
-        assert(*std::next(c.begin(), 3) == 3);
-
-        i = c.erase_after(std::next(c.cbefore_begin(), 0));
-        assert(i == c.begin());
-        assert(std::distance(c.begin(), c.end()) == 3);
-        assert(*std::next(c.begin(), 0) == 1);
-        assert(*std::next(c.begin(), 1) == 2);
-        assert(*std::next(c.begin(), 2) == 3);
-
-        i = c.erase_after(std::next(c.cbefore_begin(), 1));
-        assert(i == std::next(c.begin()));
-        assert(std::distance(c.begin(), c.end()) == 2);
-        assert(*std::next(c.begin(), 0) == 1);
-        assert(*std::next(c.begin(), 1) == 3);
-
-        i = c.erase_after(std::next(c.cbefore_begin(), 1));
-        assert(i == c.end());
-        assert(std::distance(c.begin(), c.end()) == 1);
-        assert(*std::next(c.begin(), 0) == 1);
-
-        i = c.erase_after(std::next(c.cbefore_begin(), 0));
-        assert(i == c.begin());
-        assert(i == c.end());
-        assert(std::distance(c.begin(), c.end()) == 0);
-    }
+  {
+    typedef int T;
+    typedef std::forward_list<T, min_allocator<T>> C;
+    const T t[] = {0, 1, 2, 3, 4};
+    C c(std::begin(t), std::end(t));
+
+    C::iterator i = c.erase_after(std::next(c.cbefore_begin(), 4));
+    assert(i == c.end());
+    assert(std::distance(c.begin(), c.end()) == 4);
+    assert(*std::next(c.begin(), 0) == 0);
+    assert(*std::next(c.begin(), 1) == 1);
+    assert(*std::next(c.begin(), 2) == 2);
+    assert(*std::next(c.begin(), 3) == 3);
+
+    i = c.erase_after(std::next(c.cbefore_begin(), 0));
+    assert(i == c.begin());
+    assert(std::distance(c.begin(), c.end()) == 3);
+    assert(*std::next(c.begin(), 0) == 1);
+    assert(*std::next(c.begin(), 1) == 2);
+    assert(*std::next(c.begin(), 2) == 3);
+
+    i = c.erase_after(std::next(c.cbefore_begin(), 1));
+    assert(i == std::next(c.begin()));
+    assert(std::distance(c.begin(), c.end()) == 2);
+    assert(*std::next(c.begin(), 0) == 1);
+    assert(*std::next(c.begin(), 1) == 3);
+
+    i = c.erase_after(std::next(c.cbefore_begin(), 1));
+    assert(i == c.end());
+    assert(std::distance(c.begin(), c.end()) == 1);
+    assert(*std::next(c.begin(), 0) == 1);
+
+    i = c.erase_after(std::next(c.cbefore_begin(), 0));
+    assert(i == c.begin());
+    assert(i == c.end());
+    assert(std::distance(c.begin(), c.end()) == 0);
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/sequences/forwardlist/forwardlist.modifiers/insert_after_const.pass.cpp b/libcxx/test/std/containers/sequences/forwardlist/forwardlist.modifiers/insert_after_const.pass.cpp
index 46cbc0b6b9fa5..8443158413e7f 100644
--- a/libcxx/test/std/containers/sequences/forwardlist/forwardlist.modifiers/insert_after_const.pass.cpp
+++ b/libcxx/test/std/containers/sequences/forwardlist/forwardlist.modifiers/insert_after_const.pass.cpp
@@ -16,73 +16,72 @@
 #include "test_macros.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef int T;
-        typedef std::forward_list<T> C;
-        typedef C::iterator I;
-        C c;
-        I i = c.insert_after(c.cbefore_begin(), 0);
-        assert(i == c.begin());
-        assert(c.front() == 0);
-        assert(c.front() == 0);
-        assert(std::distance(c.begin(), c.end()) == 1);
+int main(int, char**) {
+  {
+    typedef int T;
+    typedef std::forward_list<T> C;
+    typedef C::iterator I;
+    C c;
+    I i = c.insert_after(c.cbefore_begin(), 0);
+    assert(i == c.begin());
+    assert(c.front() == 0);
+    assert(c.front() == 0);
+    assert(std::distance(c.begin(), c.end()) == 1);
 
-        i = c.insert_after(c.cbegin(), 1);
-        assert(i == std::next(c.begin()));
-        assert(c.front() == 0);
-        assert(*std::next(c.begin()) == 1);
-        assert(std::distance(c.begin(), c.end()) == 2);
+    i = c.insert_after(c.cbegin(), 1);
+    assert(i == std::next(c.begin()));
+    assert(c.front() == 0);
+    assert(*std::next(c.begin()) == 1);
+    assert(std::distance(c.begin(), c.end()) == 2);
 
-        i = c.insert_after(std::next(c.cbegin()), 2);
-        assert(i == std::next(c.begin(), 2));
-        assert(c.front() == 0);
-        assert(*std::next(c.begin()) == 1);
-        assert(*std::next(c.begin(), 2) == 2);
-        assert(std::distance(c.begin(), c.end()) == 3);
+    i = c.insert_after(std::next(c.cbegin()), 2);
+    assert(i == std::next(c.begin(), 2));
+    assert(c.front() == 0);
+    assert(*std::next(c.begin()) == 1);
+    assert(*std::next(c.begin(), 2) == 2);
+    assert(std::distance(c.begin(), c.end()) == 3);
 
-        i = c.insert_after(c.cbegin(), 3);
-        assert(i == std::next(c.begin()));
-        assert(c.front() == 0);
-        assert(*std::next(c.begin(), 1) == 3);
-        assert(*std::next(c.begin(), 2) == 1);
-        assert(*std::next(c.begin(), 3) == 2);
-        assert(std::distance(c.begin(), c.end()) == 4);
-    }
+    i = c.insert_after(c.cbegin(), 3);
+    assert(i == std::next(c.begin()));
+    assert(c.front() == 0);
+    assert(*std::next(c.begin(), 1) == 3);
+    assert(*std::next(c.begin(), 2) == 1);
+    assert(*std::next(c.begin(), 3) == 2);
+    assert(std::distance(c.begin(), c.end()) == 4);
+  }
 #if TEST_STD_VER >= 11
-    {
-        typedef int T;
-        typedef std::forward_list<T, min_allocator<T>> C;
-        typedef C::iterator I;
-        C c;
-        I i = c.insert_after(c.cbefore_begin(), 0);
-        assert(i == c.begin());
-        assert(c.front() == 0);
-        assert(c.front() == 0);
-        assert(std::distance(c.begin(), c.end()) == 1);
+  {
+    typedef int T;
+    typedef std::forward_list<T, min_allocator<T>> C;
+    typedef C::iterator I;
+    C c;
+    I i = c.insert_after(c.cbefore_begin(), 0);
+    assert(i == c.begin());
+    assert(c.front() == 0);
+    assert(c.front() == 0);
+    assert(std::distance(c.begin(), c.end()) == 1);
 
-        i = c.insert_after(c.cbegin(), 1);
-        assert(i == std::next(c.begin()));
-        assert(c.front() == 0);
-        assert(*std::next(c.begin()) == 1);
-        assert(std::distance(c.begin(), c.end()) == 2);
+    i = c.insert_after(c.cbegin(), 1);
+    assert(i == std::next(c.begin()));
+    assert(c.front() == 0);
+    assert(*std::next(c.begin()) == 1);
+    assert(std::distance(c.begin(), c.end()) == 2);
 
-        i = c.insert_after(std::next(c.cbegin()), 2);
-        assert(i == std::next(c.begin(), 2));
-        assert(c.front() == 0);
-        assert(*std::next(c.begin()) == 1);
-        assert(*std::next(c.begin(), 2) == 2);
-        assert(std::distance(c.begin(), c.end()) == 3);
+    i = c.insert_after(std::next(c.cbegin()), 2);
+    assert(i == std::next(c.begin(), 2));
+    assert(c.front() == 0);
+    assert(*std::next(c.begin()) == 1);
+    assert(*std::next(c.begin(), 2) == 2);
+    assert(std::distance(c.begin(), c.end()) == 3);
 
-        i = c.insert_after(c.cbegin(), 3);
-        assert(i == std::next(c.begin()));
-        assert(c.front() == 0);
-        assert(*std::next(c.begin(), 1) == 3);
-        assert(*std::next(c.begin(), 2) == 1);
-        assert(*std::next(c.begin(), 3) == 2);
-        assert(std::distance(c.begin(), c.end()) == 4);
-    }
+    i = c.insert_after(c.cbegin(), 3);
+    assert(i == std::next(c.begin()));
+    assert(c.front() == 0);
+    assert(*std::next(c.begin(), 1) == 3);
+    assert(*std::next(c.begin(), 2) == 1);
+    assert(*std::next(c.begin(), 3) == 2);
+    assert(std::distance(c.begin(), c.end()) == 4);
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/sequences/forwardlist/forwardlist.modifiers/insert_after_init.pass.cpp b/libcxx/test/std/containers/sequences/forwardlist/forwardlist.modifiers/insert_after_init.pass.cpp
index 193cb24886ba7..de924a10c18f0 100644
--- a/libcxx/test/std/containers/sequences/forwardlist/forwardlist.modifiers/insert_after_init.pass.cpp
+++ b/libcxx/test/std/containers/sequences/forwardlist/forwardlist.modifiers/insert_after_init.pass.cpp
@@ -18,58 +18,57 @@
 #include "test_macros.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef int T;
-        typedef std::forward_list<T> C;
-        typedef C::iterator I;
-        C c;
-        I i = c.insert_after(c.cbefore_begin(), {});
-        assert(i == c.before_begin());
-        assert(std::distance(c.begin(), c.end()) == 0);
+int main(int, char**) {
+  {
+    typedef int T;
+    typedef std::forward_list<T> C;
+    typedef C::iterator I;
+    C c;
+    I i = c.insert_after(c.cbefore_begin(), {});
+    assert(i == c.before_begin());
+    assert(std::distance(c.begin(), c.end()) == 0);
 
-        i = c.insert_after(c.cbefore_begin(), {0, 1, 2});
-        assert(i == std::next(c.before_begin(), 3));
-        assert(std::distance(c.begin(), c.end()) == 3);
-        assert(*std::next(c.begin(), 0) == 0);
-        assert(*std::next(c.begin(), 1) == 1);
-        assert(*std::next(c.begin(), 2) == 2);
+    i = c.insert_after(c.cbefore_begin(), {0, 1, 2});
+    assert(i == std::next(c.before_begin(), 3));
+    assert(std::distance(c.begin(), c.end()) == 3);
+    assert(*std::next(c.begin(), 0) == 0);
+    assert(*std::next(c.begin(), 1) == 1);
+    assert(*std::next(c.begin(), 2) == 2);
 
-        i = c.insert_after(c.begin(), {3, 4});
-        assert(i == std::next(c.begin(), 2));
-        assert(std::distance(c.begin(), c.end()) == 5);
-        assert(*std::next(c.begin(), 0) == 0);
-        assert(*std::next(c.begin(), 1) == 3);
-        assert(*std::next(c.begin(), 2) == 4);
-        assert(*std::next(c.begin(), 3) == 1);
-        assert(*std::next(c.begin(), 4) == 2);
-    }
-    {
-        typedef int T;
-        typedef std::forward_list<T, min_allocator<T>> C;
-        typedef C::iterator I;
-        C c;
-        I i = c.insert_after(c.cbefore_begin(), {});
-        assert(i == c.before_begin());
-        assert(std::distance(c.begin(), c.end()) == 0);
+    i = c.insert_after(c.begin(), {3, 4});
+    assert(i == std::next(c.begin(), 2));
+    assert(std::distance(c.begin(), c.end()) == 5);
+    assert(*std::next(c.begin(), 0) == 0);
+    assert(*std::next(c.begin(), 1) == 3);
+    assert(*std::next(c.begin(), 2) == 4);
+    assert(*std::next(c.begin(), 3) == 1);
+    assert(*std::next(c.begin(), 4) == 2);
+  }
+  {
+    typedef int T;
+    typedef std::forward_list<T, min_allocator<T>> C;
+    typedef C::iterator I;
+    C c;
+    I i = c.insert_after(c.cbefore_begin(), {});
+    assert(i == c.before_begin());
+    assert(std::distance(c.begin(), c.end()) == 0);
 
-        i = c.insert_after(c.cbefore_begin(), {0, 1, 2});
-        assert(i == std::next(c.before_begin(), 3));
-        assert(std::distance(c.begin(), c.end()) == 3);
-        assert(*std::next(c.begin(), 0) == 0);
-        assert(*std::next(c.begin(), 1) == 1);
-        assert(*std::next(c.begin(), 2) == 2);
+    i = c.insert_after(c.cbefore_begin(), {0, 1, 2});
+    assert(i == std::next(c.before_begin(), 3));
+    assert(std::distance(c.begin(), c.end()) == 3);
+    assert(*std::next(c.begin(), 0) == 0);
+    assert(*std::next(c.begin(), 1) == 1);
+    assert(*std::next(c.begin(), 2) == 2);
 
-        i = c.insert_after(c.begin(), {3, 4});
-        assert(i == std::next(c.begin(), 2));
-        assert(std::distance(c.begin(), c.end()) == 5);
-        assert(*std::next(c.begin(), 0) == 0);
-        assert(*std::next(c.begin(), 1) == 3);
-        assert(*std::next(c.begin(), 2) == 4);
-        assert(*std::next(c.begin(), 3) == 1);
-        assert(*std::next(c.begin(), 4) == 2);
-    }
+    i = c.insert_after(c.begin(), {3, 4});
+    assert(i == std::next(c.begin(), 2));
+    assert(std::distance(c.begin(), c.end()) == 5);
+    assert(*std::next(c.begin(), 0) == 0);
+    assert(*std::next(c.begin(), 1) == 3);
+    assert(*std::next(c.begin(), 2) == 4);
+    assert(*std::next(c.begin(), 3) == 1);
+    assert(*std::next(c.begin(), 4) == 2);
+  }
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/forwardlist/forwardlist.modifiers/insert_after_range.pass.cpp b/libcxx/test/std/containers/sequences/forwardlist/forwardlist.modifiers/insert_after_range.pass.cpp
index 8d70b0bdb41c0..af810d0f6961c 100644
--- a/libcxx/test/std/containers/sequences/forwardlist/forwardlist.modifiers/insert_after_range.pass.cpp
+++ b/libcxx/test/std/containers/sequences/forwardlist/forwardlist.modifiers/insert_after_range.pass.cpp
@@ -19,63 +19,62 @@
 #include "test_iterators.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef int T;
-        typedef std::forward_list<T> C;
-        typedef C::iterator I;
-        typedef cpp17_input_iterator<const T*> J;
-        C c;
-        const T t[] = {0, 1, 2, 3, 4};
-        I i = c.insert_after(c.cbefore_begin(), J(t), J(t));
-        assert(i == c.before_begin());
-        assert(std::distance(c.begin(), c.end()) == 0);
+int main(int, char**) {
+  {
+    typedef int T;
+    typedef std::forward_list<T> C;
+    typedef C::iterator I;
+    typedef cpp17_input_iterator<const T*> J;
+    C c;
+    const T t[] = {0, 1, 2, 3, 4};
+    I i         = c.insert_after(c.cbefore_begin(), J(t), J(t));
+    assert(i == c.before_begin());
+    assert(std::distance(c.begin(), c.end()) == 0);
 
-        i = c.insert_after(c.cbefore_begin(), J(t), J(t+3));
-        assert(i == std::next(c.before_begin(), 3));
-        assert(std::distance(c.begin(), c.end()) == 3);
-        assert(*std::next(c.begin(), 0) == 0);
-        assert(*std::next(c.begin(), 1) == 1);
-        assert(*std::next(c.begin(), 2) == 2);
+    i = c.insert_after(c.cbefore_begin(), J(t), J(t + 3));
+    assert(i == std::next(c.before_begin(), 3));
+    assert(std::distance(c.begin(), c.end()) == 3);
+    assert(*std::next(c.begin(), 0) == 0);
+    assert(*std::next(c.begin(), 1) == 1);
+    assert(*std::next(c.begin(), 2) == 2);
 
-        i = c.insert_after(c.begin(), J(t+3), J(t+5));
-        assert(i == std::next(c.begin(), 2));
-        assert(std::distance(c.begin(), c.end()) == 5);
-        assert(*std::next(c.begin(), 0) == 0);
-        assert(*std::next(c.begin(), 1) == 3);
-        assert(*std::next(c.begin(), 2) == 4);
-        assert(*std::next(c.begin(), 3) == 1);
-        assert(*std::next(c.begin(), 4) == 2);
-    }
+    i = c.insert_after(c.begin(), J(t + 3), J(t + 5));
+    assert(i == std::next(c.begin(), 2));
+    assert(std::distance(c.begin(), c.end()) == 5);
+    assert(*std::next(c.begin(), 0) == 0);
+    assert(*std::next(c.begin(), 1) == 3);
+    assert(*std::next(c.begin(), 2) == 4);
+    assert(*std::next(c.begin(), 3) == 1);
+    assert(*std::next(c.begin(), 4) == 2);
+  }
 #if TEST_STD_VER >= 11
-    {
-        typedef int T;
-        typedef std::forward_list<T, min_allocator<T>> C;
-        typedef C::iterator I;
-        typedef cpp17_input_iterator<const T*> J;
-        C c;
-        const T t[] = {0, 1, 2, 3, 4};
-        I i = c.insert_after(c.cbefore_begin(), J(t), J(t));
-        assert(i == c.before_begin());
-        assert(std::distance(c.begin(), c.end()) == 0);
+  {
+    typedef int T;
+    typedef std::forward_list<T, min_allocator<T>> C;
+    typedef C::iterator I;
+    typedef cpp17_input_iterator<const T*> J;
+    C c;
+    const T t[] = {0, 1, 2, 3, 4};
+    I i         = c.insert_after(c.cbefore_begin(), J(t), J(t));
+    assert(i == c.before_begin());
+    assert(std::distance(c.begin(), c.end()) == 0);
 
-        i = c.insert_after(c.cbefore_begin(), J(t), J(t+3));
-        assert(i == std::next(c.before_begin(), 3));
-        assert(std::distance(c.begin(), c.end()) == 3);
-        assert(*std::next(c.begin(), 0) == 0);
-        assert(*std::next(c.begin(), 1) == 1);
-        assert(*std::next(c.begin(), 2) == 2);
+    i = c.insert_after(c.cbefore_begin(), J(t), J(t + 3));
+    assert(i == std::next(c.before_begin(), 3));
+    assert(std::distance(c.begin(), c.end()) == 3);
+    assert(*std::next(c.begin(), 0) == 0);
+    assert(*std::next(c.begin(), 1) == 1);
+    assert(*std::next(c.begin(), 2) == 2);
 
-        i = c.insert_after(c.begin(), J(t+3), J(t+5));
-        assert(i == std::next(c.begin(), 2));
-        assert(std::distance(c.begin(), c.end()) == 5);
-        assert(*std::next(c.begin(), 0) == 0);
-        assert(*std::next(c.begin(), 1) == 3);
-        assert(*std::next(c.begin(), 2) == 4);
-        assert(*std::next(c.begin(), 3) == 1);
-        assert(*std::next(c.begin(), 4) == 2);
-    }
+    i = c.insert_after(c.begin(), J(t + 3), J(t + 5));
+    assert(i == std::next(c.begin(), 2));
+    assert(std::distance(c.begin(), c.end()) == 5);
+    assert(*std::next(c.begin(), 0) == 0);
+    assert(*std::next(c.begin(), 1) == 3);
+    assert(*std::next(c.begin(), 2) == 4);
+    assert(*std::next(c.begin(), 3) == 1);
+    assert(*std::next(c.begin(), 4) == 2);
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/sequences/forwardlist/forwardlist.modifiers/insert_after_rv.pass.cpp b/libcxx/test/std/containers/sequences/forwardlist/forwardlist.modifiers/insert_after_rv.pass.cpp
index 083bdd90b0aa5..acd4bc73f724e 100644
--- a/libcxx/test/std/containers/sequences/forwardlist/forwardlist.modifiers/insert_after_rv.pass.cpp
+++ b/libcxx/test/std/containers/sequences/forwardlist/forwardlist.modifiers/insert_after_rv.pass.cpp
@@ -19,72 +19,71 @@
 #include "MoveOnly.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef MoveOnly T;
-        typedef std::forward_list<T> C;
-        typedef C::iterator I;
-        C c;
-        I i = c.insert_after(c.cbefore_begin(), 0);
-        assert(i == c.begin());
-        assert(c.front() == 0);
-        assert(c.front() == 0);
-        assert(std::distance(c.begin(), c.end()) == 1);
+int main(int, char**) {
+  {
+    typedef MoveOnly T;
+    typedef std::forward_list<T> C;
+    typedef C::iterator I;
+    C c;
+    I i = c.insert_after(c.cbefore_begin(), 0);
+    assert(i == c.begin());
+    assert(c.front() == 0);
+    assert(c.front() == 0);
+    assert(std::distance(c.begin(), c.end()) == 1);
 
-        i = c.insert_after(c.cbegin(), 1);
-        assert(i == std::next(c.begin()));
-        assert(c.front() == 0);
-        assert(*std::next(c.begin()) == 1);
-        assert(std::distance(c.begin(), c.end()) == 2);
+    i = c.insert_after(c.cbegin(), 1);
+    assert(i == std::next(c.begin()));
+    assert(c.front() == 0);
+    assert(*std::next(c.begin()) == 1);
+    assert(std::distance(c.begin(), c.end()) == 2);
 
-        i = c.insert_after(std::next(c.cbegin()), 2);
-        assert(i == std::next(c.begin(), 2));
-        assert(c.front() == 0);
-        assert(*std::next(c.begin()) == 1);
-        assert(*std::next(c.begin(), 2) == 2);
-        assert(std::distance(c.begin(), c.end()) == 3);
+    i = c.insert_after(std::next(c.cbegin()), 2);
+    assert(i == std::next(c.begin(), 2));
+    assert(c.front() == 0);
+    assert(*std::next(c.begin()) == 1);
+    assert(*std::next(c.begin(), 2) == 2);
+    assert(std::distance(c.begin(), c.end()) == 3);
 
-        i = c.insert_after(c.cbegin(), 3);
-        assert(i == std::next(c.begin()));
-        assert(c.front() == 0);
-        assert(*std::next(c.begin(), 1) == 3);
-        assert(*std::next(c.begin(), 2) == 1);
-        assert(*std::next(c.begin(), 3) == 2);
-        assert(std::distance(c.begin(), c.end()) == 4);
-    }
-    {
-        typedef MoveOnly T;
-        typedef std::forward_list<T, min_allocator<T>> C;
-        typedef C::iterator I;
-        C c;
-        I i = c.insert_after(c.cbefore_begin(), 0);
-        assert(i == c.begin());
-        assert(c.front() == 0);
-        assert(c.front() == 0);
-        assert(std::distance(c.begin(), c.end()) == 1);
+    i = c.insert_after(c.cbegin(), 3);
+    assert(i == std::next(c.begin()));
+    assert(c.front() == 0);
+    assert(*std::next(c.begin(), 1) == 3);
+    assert(*std::next(c.begin(), 2) == 1);
+    assert(*std::next(c.begin(), 3) == 2);
+    assert(std::distance(c.begin(), c.end()) == 4);
+  }
+  {
+    typedef MoveOnly T;
+    typedef std::forward_list<T, min_allocator<T>> C;
+    typedef C::iterator I;
+    C c;
+    I i = c.insert_after(c.cbefore_begin(), 0);
+    assert(i == c.begin());
+    assert(c.front() == 0);
+    assert(c.front() == 0);
+    assert(std::distance(c.begin(), c.end()) == 1);
 
-        i = c.insert_after(c.cbegin(), 1);
-        assert(i == std::next(c.begin()));
-        assert(c.front() == 0);
-        assert(*std::next(c.begin()) == 1);
-        assert(std::distance(c.begin(), c.end()) == 2);
+    i = c.insert_after(c.cbegin(), 1);
+    assert(i == std::next(c.begin()));
+    assert(c.front() == 0);
+    assert(*std::next(c.begin()) == 1);
+    assert(std::distance(c.begin(), c.end()) == 2);
 
-        i = c.insert_after(std::next(c.cbegin()), 2);
-        assert(i == std::next(c.begin(), 2));
-        assert(c.front() == 0);
-        assert(*std::next(c.begin()) == 1);
-        assert(*std::next(c.begin(), 2) == 2);
-        assert(std::distance(c.begin(), c.end()) == 3);
+    i = c.insert_after(std::next(c.cbegin()), 2);
+    assert(i == std::next(c.begin(), 2));
+    assert(c.front() == 0);
+    assert(*std::next(c.begin()) == 1);
+    assert(*std::next(c.begin(), 2) == 2);
+    assert(std::distance(c.begin(), c.end()) == 3);
 
-        i = c.insert_after(c.cbegin(), 3);
-        assert(i == std::next(c.begin()));
-        assert(c.front() == 0);
-        assert(*std::next(c.begin(), 1) == 3);
-        assert(*std::next(c.begin(), 2) == 1);
-        assert(*std::next(c.begin(), 3) == 2);
-        assert(std::distance(c.begin(), c.end()) == 4);
-    }
+    i = c.insert_after(c.cbegin(), 3);
+    assert(i == std::next(c.begin()));
+    assert(c.front() == 0);
+    assert(*std::next(c.begin(), 1) == 3);
+    assert(*std::next(c.begin(), 2) == 1);
+    assert(*std::next(c.begin(), 3) == 2);
+    assert(std::distance(c.begin(), c.end()) == 4);
+  }
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/forwardlist/forwardlist.modifiers/insert_after_size_value.pass.cpp b/libcxx/test/std/containers/sequences/forwardlist/forwardlist.modifiers/insert_after_size_value.pass.cpp
index 1ef7f16b70292..2506f04311e0e 100644
--- a/libcxx/test/std/containers/sequences/forwardlist/forwardlist.modifiers/insert_after_size_value.pass.cpp
+++ b/libcxx/test/std/containers/sequences/forwardlist/forwardlist.modifiers/insert_after_size_value.pass.cpp
@@ -16,59 +16,58 @@
 #include "test_macros.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef int T;
-        typedef std::forward_list<T> C;
-        typedef C::iterator I;
-        C c;
-        I i = c.insert_after(c.cbefore_begin(), 0, 0);
-        assert(i == c.before_begin());
-        assert(std::distance(c.begin(), c.end()) == 0);
+int main(int, char**) {
+  {
+    typedef int T;
+    typedef std::forward_list<T> C;
+    typedef C::iterator I;
+    C c;
+    I i = c.insert_after(c.cbefore_begin(), 0, 0);
+    assert(i == c.before_begin());
+    assert(std::distance(c.begin(), c.end()) == 0);
 
-        i = c.insert_after(c.cbefore_begin(), 3, 3);
-        assert(i == std::next(c.before_begin(), 3));
-        assert(std::distance(c.begin(), c.end()) == 3);
-        assert(*std::next(c.begin(), 0) == 3);
-        assert(*std::next(c.begin(), 1) == 3);
-        assert(*std::next(c.begin(), 2) == 3);
+    i = c.insert_after(c.cbefore_begin(), 3, 3);
+    assert(i == std::next(c.before_begin(), 3));
+    assert(std::distance(c.begin(), c.end()) == 3);
+    assert(*std::next(c.begin(), 0) == 3);
+    assert(*std::next(c.begin(), 1) == 3);
+    assert(*std::next(c.begin(), 2) == 3);
 
-        i = c.insert_after(c.begin(), 2, 2);
-        assert(i == std::next(c.begin(), 2));
-        assert(std::distance(c.begin(), c.end()) == 5);
-        assert(*std::next(c.begin(), 0) == 3);
-        assert(*std::next(c.begin(), 1) == 2);
-        assert(*std::next(c.begin(), 2) == 2);
-        assert(*std::next(c.begin(), 3) == 3);
-        assert(*std::next(c.begin(), 4) == 3);
-    }
+    i = c.insert_after(c.begin(), 2, 2);
+    assert(i == std::next(c.begin(), 2));
+    assert(std::distance(c.begin(), c.end()) == 5);
+    assert(*std::next(c.begin(), 0) == 3);
+    assert(*std::next(c.begin(), 1) == 2);
+    assert(*std::next(c.begin(), 2) == 2);
+    assert(*std::next(c.begin(), 3) == 3);
+    assert(*std::next(c.begin(), 4) == 3);
+  }
 #if TEST_STD_VER >= 11
-    {
-        typedef int T;
-        typedef std::forward_list<T, min_allocator<T>> C;
-        typedef C::iterator I;
-        C c;
-        I i = c.insert_after(c.cbefore_begin(), 0, 0);
-        assert(i == c.before_begin());
-        assert(std::distance(c.begin(), c.end()) == 0);
+  {
+    typedef int T;
+    typedef std::forward_list<T, min_allocator<T>> C;
+    typedef C::iterator I;
+    C c;
+    I i = c.insert_after(c.cbefore_begin(), 0, 0);
+    assert(i == c.before_begin());
+    assert(std::distance(c.begin(), c.end()) == 0);
 
-        i = c.insert_after(c.cbefore_begin(), 3, 3);
-        assert(i == std::next(c.before_begin(), 3));
-        assert(std::distance(c.begin(), c.end()) == 3);
-        assert(*std::next(c.begin(), 0) == 3);
-        assert(*std::next(c.begin(), 1) == 3);
-        assert(*std::next(c.begin(), 2) == 3);
+    i = c.insert_after(c.cbefore_begin(), 3, 3);
+    assert(i == std::next(c.before_begin(), 3));
+    assert(std::distance(c.begin(), c.end()) == 3);
+    assert(*std::next(c.begin(), 0) == 3);
+    assert(*std::next(c.begin(), 1) == 3);
+    assert(*std::next(c.begin(), 2) == 3);
 
-        i = c.insert_after(c.begin(), 2, 2);
-        assert(i == std::next(c.begin(), 2));
-        assert(std::distance(c.begin(), c.end()) == 5);
-        assert(*std::next(c.begin(), 0) == 3);
-        assert(*std::next(c.begin(), 1) == 2);
-        assert(*std::next(c.begin(), 2) == 2);
-        assert(*std::next(c.begin(), 3) == 3);
-        assert(*std::next(c.begin(), 4) == 3);
-    }
+    i = c.insert_after(c.begin(), 2, 2);
+    assert(i == std::next(c.begin(), 2));
+    assert(std::distance(c.begin(), c.end()) == 5);
+    assert(*std::next(c.begin(), 0) == 3);
+    assert(*std::next(c.begin(), 1) == 2);
+    assert(*std::next(c.begin(), 2) == 2);
+    assert(*std::next(c.begin(), 3) == 3);
+    assert(*std::next(c.begin(), 4) == 3);
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/sequences/forwardlist/forwardlist.modifiers/insert_range_after.pass.cpp b/libcxx/test/std/containers/sequences/forwardlist/forwardlist.modifiers/insert_range_after.pass.cpp
index 1b9133e7c3a7a..25f4c43f38486 100644
--- a/libcxx/test/std/containers/sequences/forwardlist/forwardlist.modifiers/insert_range_after.pass.cpp
+++ b/libcxx/test/std/containers/sequences/forwardlist/forwardlist.modifiers/insert_range_after.pass.cpp
@@ -17,9 +17,7 @@
 #include "test_macros.h"
 
 template <class Container, class Range>
-concept HasInsertRangeAfter = requires (Container& c, Range&& range) {
-  c.insert_range_after(c.begin(), range);
-};
+concept HasInsertRangeAfter = requires(Container& c, Range&& range) { c.insert_range_after(c.begin(), range); };
 
 template <template <class...> class Container, class T, class U>
 constexpr bool test_constraints_insert_range_after() {
@@ -53,12 +51,12 @@ constexpr void test_sequence_insert_range_after() {
     return std::ranges::next(c.before_begin(), test_case.index + test_case.input.size());
   };
 
-  { // Empty container.
+  {   // Empty container.
     { // empty_c.insert_range_after(end, empty_range)
       auto& test_case = EmptyContainer_EmptyRange<T>;
 
       Container c(test_case.initial.begin(), test_case.initial.end());
-      auto in = wrap_input<Iter, Sent>(test_case.input);
+      auto in  = wrap_input<Iter, Sent>(test_case.input);
       auto pos = get_insert_pos(c, test_case);
 
       auto result = c.insert_range_after(pos, in);
@@ -70,7 +68,7 @@ constexpr void test_sequence_insert_range_after() {
       auto& test_case = EmptyContainer_OneElementRange<T>;
 
       Container c(test_case.initial.begin(), test_case.initial.end());
-      auto in = wrap_input<Iter, Sent>(test_case.input);
+      auto in  = wrap_input<Iter, Sent>(test_case.input);
       auto pos = get_insert_pos(c, test_case);
 
       auto result = c.insert_range_after(pos, in);
@@ -82,7 +80,7 @@ constexpr void test_sequence_insert_range_after() {
       auto& test_case = EmptyContainer_MidRange<T>;
 
       Container c(test_case.initial.begin(), test_case.initial.end());
-      auto in = wrap_input<Iter, Sent>(test_case.input);
+      auto in  = wrap_input<Iter, Sent>(test_case.input);
       auto pos = get_insert_pos(c, test_case);
 
       auto result = c.insert_range_after(pos, in);
@@ -91,12 +89,12 @@ constexpr void test_sequence_insert_range_after() {
     }
   }
 
-  { // One-element container.
+  {   // One-element container.
     { // one_element_c.insert_range_after(begin, empty_range)
       auto& test_case = OneElementContainer_Begin_EmptyRange<T>;
 
       Container c(test_case.initial.begin(), test_case.initial.end());
-      auto in = wrap_input<Iter, Sent>(test_case.input);
+      auto in  = wrap_input<Iter, Sent>(test_case.input);
       auto pos = get_insert_pos(c, test_case);
 
       auto result = c.insert_range_after(pos, in);
@@ -108,7 +106,7 @@ constexpr void test_sequence_insert_range_after() {
       auto& test_case = OneElementContainer_End_EmptyRange<T>;
 
       Container c(test_case.initial.begin(), test_case.initial.end());
-      auto in = wrap_input<Iter, Sent>(test_case.input);
+      auto in  = wrap_input<Iter, Sent>(test_case.input);
       auto pos = get_insert_pos(c, test_case);
 
       auto result = c.insert_range_after(pos, in);
@@ -120,7 +118,7 @@ constexpr void test_sequence_insert_range_after() {
       auto& test_case = OneElementContainer_Begin_OneElementRange<T>;
 
       Container c(test_case.initial.begin(), test_case.initial.end());
-      auto in = wrap_input<Iter, Sent>(test_case.input);
+      auto in  = wrap_input<Iter, Sent>(test_case.input);
       auto pos = get_insert_pos(c, test_case);
 
       auto result = c.insert_range_after(pos, in);
@@ -132,7 +130,7 @@ constexpr void test_sequence_insert_range_after() {
       auto& test_case = OneElementContainer_End_OneElementRange<T>;
 
       Container c(test_case.initial.begin(), test_case.initial.end());
-      auto in = wrap_input<Iter, Sent>(test_case.input);
+      auto in  = wrap_input<Iter, Sent>(test_case.input);
       auto pos = get_insert_pos(c, test_case);
 
       auto result = c.insert_range_after(pos, in);
@@ -144,7 +142,7 @@ constexpr void test_sequence_insert_range_after() {
       auto& test_case = OneElementContainer_Begin_MidRange<T>;
 
       Container c(test_case.initial.begin(), test_case.initial.end());
-      auto in = wrap_input<Iter, Sent>(test_case.input);
+      auto in  = wrap_input<Iter, Sent>(test_case.input);
       auto pos = get_insert_pos(c, test_case);
 
       auto result = c.insert_range_after(pos, in);
@@ -156,7 +154,7 @@ constexpr void test_sequence_insert_range_after() {
       auto& test_case = OneElementContainer_End_MidRange<T>;
 
       Container c(test_case.initial.begin(), test_case.initial.end());
-      auto in = wrap_input<Iter, Sent>(test_case.input);
+      auto in  = wrap_input<Iter, Sent>(test_case.input);
       auto pos = get_insert_pos(c, test_case);
 
       auto result = c.insert_range_after(pos, in);
@@ -165,12 +163,12 @@ constexpr void test_sequence_insert_range_after() {
     }
   }
 
-  { // Full container.
+  {   // Full container.
     { // full_container.insert_range_after(begin, empty_range)
       auto& test_case = FullContainer_Begin_EmptyRange<T>;
 
       Container c(test_case.initial.begin(), test_case.initial.end());
-      auto in = wrap_input<Iter, Sent>(test_case.input);
+      auto in  = wrap_input<Iter, Sent>(test_case.input);
       auto pos = get_insert_pos(c, test_case);
 
       auto result = c.insert_range_after(pos, in);
@@ -182,7 +180,7 @@ constexpr void test_sequence_insert_range_after() {
       auto& test_case = FullContainer_Mid_EmptyRange<T>;
 
       Container c(test_case.initial.begin(), test_case.initial.end());
-      auto in = wrap_input<Iter, Sent>(test_case.input);
+      auto in  = wrap_input<Iter, Sent>(test_case.input);
       auto pos = get_insert_pos(c, test_case);
 
       auto result = c.insert_range_after(pos, in);
@@ -194,7 +192,7 @@ constexpr void test_sequence_insert_range_after() {
       auto& test_case = FullContainer_End_EmptyRange<T>;
 
       Container c(test_case.initial.begin(), test_case.initial.end());
-      auto in = wrap_input<Iter, Sent>(test_case.input);
+      auto in  = wrap_input<Iter, Sent>(test_case.input);
       auto pos = get_insert_pos(c, test_case);
 
       auto result = c.insert_range_after(pos, in);
@@ -206,7 +204,7 @@ constexpr void test_sequence_insert_range_after() {
       auto& test_case = FullContainer_Begin_OneElementRange<T>;
 
       Container c(test_case.initial.begin(), test_case.initial.end());
-      auto in = wrap_input<Iter, Sent>(test_case.input);
+      auto in  = wrap_input<Iter, Sent>(test_case.input);
       auto pos = get_insert_pos(c, test_case);
 
       auto result = c.insert_range_after(pos, in);
@@ -218,7 +216,7 @@ constexpr void test_sequence_insert_range_after() {
       auto& test_case = FullContainer_Mid_OneElementRange<T>;
 
       Container c(test_case.initial.begin(), test_case.initial.end());
-      auto in = wrap_input<Iter, Sent>(test_case.input);
+      auto in  = wrap_input<Iter, Sent>(test_case.input);
       auto pos = get_insert_pos(c, test_case);
 
       auto result = c.insert_range_after(pos, in);
@@ -230,7 +228,7 @@ constexpr void test_sequence_insert_range_after() {
       auto& test_case = FullContainer_End_OneElementRange<T>;
 
       Container c(test_case.initial.begin(), test_case.initial.end());
-      auto in = wrap_input<Iter, Sent>(test_case.input);
+      auto in  = wrap_input<Iter, Sent>(test_case.input);
       auto pos = get_insert_pos(c, test_case);
 
       auto result = c.insert_range_after(pos, in);
@@ -242,7 +240,7 @@ constexpr void test_sequence_insert_range_after() {
       auto& test_case = FullContainer_Begin_MidRange<T>;
 
       Container c(test_case.initial.begin(), test_case.initial.end());
-      auto in = wrap_input<Iter, Sent>(test_case.input);
+      auto in  = wrap_input<Iter, Sent>(test_case.input);
       auto pos = get_insert_pos(c, test_case);
 
       auto result = c.insert_range_after(pos, in);
@@ -254,7 +252,7 @@ constexpr void test_sequence_insert_range_after() {
       auto& test_case = FullContainer_Mid_MidRange<T>;
 
       Container c(test_case.initial.begin(), test_case.initial.end());
-      auto in = wrap_input<Iter, Sent>(test_case.input);
+      auto in  = wrap_input<Iter, Sent>(test_case.input);
       auto pos = get_insert_pos(c, test_case);
 
       auto result = c.insert_range_after(pos, in);
@@ -266,7 +264,7 @@ constexpr void test_sequence_insert_range_after() {
       auto& test_case = FullContainer_End_MidRange<T>;
 
       Container c(test_case.initial.begin(), test_case.initial.end());
-      auto in = wrap_input<Iter, Sent>(test_case.input);
+      auto in  = wrap_input<Iter, Sent>(test_case.input);
       auto pos = get_insert_pos(c, test_case);
 
       auto result = c.insert_range_after(pos, in);
@@ -278,7 +276,7 @@ constexpr void test_sequence_insert_range_after() {
       auto& test_case = FullContainer_Begin_LongRange<T>;
 
       Container c(test_case.initial.begin(), test_case.initial.end());
-      auto in = wrap_input<Iter, Sent>(test_case.input);
+      auto in  = wrap_input<Iter, Sent>(test_case.input);
       auto pos = get_insert_pos(c, test_case);
 
       auto result = c.insert_range_after(pos, in);
@@ -290,7 +288,7 @@ constexpr void test_sequence_insert_range_after() {
       auto& test_case = FullContainer_Mid_LongRange<T>;
 
       Container c(test_case.initial.begin(), test_case.initial.end());
-      auto in = wrap_input<Iter, Sent>(test_case.input);
+      auto in  = wrap_input<Iter, Sent>(test_case.input);
       auto pos = get_insert_pos(c, test_case);
 
       auto result = c.insert_range_after(pos, in);
@@ -302,7 +300,7 @@ constexpr void test_sequence_insert_range_after() {
       auto& test_case = FullContainer_End_LongRange<T>;
 
       Container c(test_case.initial.begin(), test_case.initial.end());
-      auto in = wrap_input<Iter, Sent>(test_case.input);
+      auto in  = wrap_input<Iter, Sent>(test_case.input);
       auto pos = get_insert_pos(c, test_case);
 
       auto result = c.insert_range_after(pos, in);
diff --git a/libcxx/test/std/containers/sequences/forwardlist/forwardlist.modifiers/pop_front.pass.cpp b/libcxx/test/std/containers/sequences/forwardlist/forwardlist.modifiers/pop_front.pass.cpp
index 5022353cac76c..98c7a26341179 100644
--- a/libcxx/test/std/containers/sequences/forwardlist/forwardlist.modifiers/pop_front.pass.cpp
+++ b/libcxx/test/std/containers/sequences/forwardlist/forwardlist.modifiers/pop_front.pass.cpp
@@ -17,59 +17,58 @@
 #include "MoveOnly.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef int T;
-        typedef std::forward_list<T> C;
-        typedef std::forward_list<T> C;
-        C c;
-        c.push_front(1);
-        c.push_front(3);
-        c.pop_front();
-        assert(std::distance(c.begin(), c.end()) == 1);
-        assert(c.front() == 1);
-        c.pop_front();
-        assert(std::distance(c.begin(), c.end()) == 0);
-    }
+int main(int, char**) {
+  {
+    typedef int T;
+    typedef std::forward_list<T> C;
+    typedef std::forward_list<T> C;
+    C c;
+    c.push_front(1);
+    c.push_front(3);
+    c.pop_front();
+    assert(std::distance(c.begin(), c.end()) == 1);
+    assert(c.front() == 1);
+    c.pop_front();
+    assert(std::distance(c.begin(), c.end()) == 0);
+  }
 #if TEST_STD_VER >= 11
-    {
-        typedef MoveOnly T;
-        typedef std::forward_list<T> C;
-        C c;
-        c.push_front(1);
-        c.push_front(3);
-        c.pop_front();
-        assert(std::distance(c.begin(), c.end()) == 1);
-        assert(c.front() == 1);
-        c.pop_front();
-        assert(std::distance(c.begin(), c.end()) == 0);
-    }
-    {
-        typedef int T;
-        typedef std::forward_list<T, min_allocator<T>> C;
-        typedef std::forward_list<T, min_allocator<T>> C;
-        C c;
-        c.push_front(1);
-        c.push_front(3);
-        c.pop_front();
-        assert(std::distance(c.begin(), c.end()) == 1);
-        assert(c.front() == 1);
-        c.pop_front();
-        assert(std::distance(c.begin(), c.end()) == 0);
-    }
-    {
-        typedef MoveOnly T;
-        typedef std::forward_list<T, min_allocator<T>> C;
-        C c;
-        c.push_front(1);
-        c.push_front(3);
-        c.pop_front();
-        assert(std::distance(c.begin(), c.end()) == 1);
-        assert(c.front() == 1);
-        c.pop_front();
-        assert(std::distance(c.begin(), c.end()) == 0);
-    }
+  {
+    typedef MoveOnly T;
+    typedef std::forward_list<T> C;
+    C c;
+    c.push_front(1);
+    c.push_front(3);
+    c.pop_front();
+    assert(std::distance(c.begin(), c.end()) == 1);
+    assert(c.front() == 1);
+    c.pop_front();
+    assert(std::distance(c.begin(), c.end()) == 0);
+  }
+  {
+    typedef int T;
+    typedef std::forward_list<T, min_allocator<T>> C;
+    typedef std::forward_list<T, min_allocator<T>> C;
+    C c;
+    c.push_front(1);
+    c.push_front(3);
+    c.pop_front();
+    assert(std::distance(c.begin(), c.end()) == 1);
+    assert(c.front() == 1);
+    c.pop_front();
+    assert(std::distance(c.begin(), c.end()) == 0);
+  }
+  {
+    typedef MoveOnly T;
+    typedef std::forward_list<T, min_allocator<T>> C;
+    C c;
+    c.push_front(1);
+    c.push_front(3);
+    c.pop_front();
+    assert(std::distance(c.begin(), c.end()) == 1);
+    assert(c.front() == 1);
+    c.pop_front();
+    assert(std::distance(c.begin(), c.end()) == 0);
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/sequences/forwardlist/forwardlist.modifiers/push_front_const.pass.cpp b/libcxx/test/std/containers/sequences/forwardlist/forwardlist.modifiers/push_front_const.pass.cpp
index e2a1638d3dd31..f99c40fa0c1a0 100644
--- a/libcxx/test/std/containers/sequences/forwardlist/forwardlist.modifiers/push_front_const.pass.cpp
+++ b/libcxx/test/std/containers/sequences/forwardlist/forwardlist.modifiers/push_front_const.pass.cpp
@@ -16,33 +16,32 @@
 #include "test_macros.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef int T;
-        typedef std::forward_list<T> C;
-        C c;
-        c.push_front(1);
-        assert(c.front() == 1);
-        assert(std::distance(c.begin(), c.end()) == 1);
-        c.push_front(3);
-        assert(c.front() == 3);
-        assert(*std::next(c.begin()) == 1);
-        assert(std::distance(c.begin(), c.end()) == 2);
-    }
+int main(int, char**) {
+  {
+    typedef int T;
+    typedef std::forward_list<T> C;
+    C c;
+    c.push_front(1);
+    assert(c.front() == 1);
+    assert(std::distance(c.begin(), c.end()) == 1);
+    c.push_front(3);
+    assert(c.front() == 3);
+    assert(*std::next(c.begin()) == 1);
+    assert(std::distance(c.begin(), c.end()) == 2);
+  }
 #if TEST_STD_VER >= 11
-    {
-        typedef int T;
-        typedef std::forward_list<T, min_allocator<T>> C;
-        C c;
-        c.push_front(1);
-        assert(c.front() == 1);
-        assert(std::distance(c.begin(), c.end()) == 1);
-        c.push_front(3);
-        assert(c.front() == 3);
-        assert(*std::next(c.begin()) == 1);
-        assert(std::distance(c.begin(), c.end()) == 2);
-    }
+  {
+    typedef int T;
+    typedef std::forward_list<T, min_allocator<T>> C;
+    C c;
+    c.push_front(1);
+    assert(c.front() == 1);
+    assert(std::distance(c.begin(), c.end()) == 1);
+    c.push_front(3);
+    assert(c.front() == 3);
+    assert(*std::next(c.begin()) == 1);
+    assert(std::distance(c.begin(), c.end()) == 2);
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/sequences/forwardlist/forwardlist.modifiers/push_front_exception_safety.pass.cpp b/libcxx/test/std/containers/sequences/forwardlist/forwardlist.modifiers/push_front_exception_safety.pass.cpp
index 2d0f11c2bf922..467037465eedd 100644
--- a/libcxx/test/std/containers/sequences/forwardlist/forwardlist.modifiers/push_front_exception_safety.pass.cpp
+++ b/libcxx/test/std/containers/sequences/forwardlist/forwardlist.modifiers/push_front_exception_safety.pass.cpp
@@ -21,14 +21,23 @@
 static bool gCopyConstructorShouldThrow = false;
 
 class CMyClass {
-    public: CMyClass();
-    public: CMyClass(const CMyClass& iOther);
-    public: ~CMyClass();
+public:
+  CMyClass();
 
-    private: int fMagicValue;
+public:
+  CMyClass(const CMyClass& iOther);
 
-    private: static int kStartedConstructionMagicValue;
-    private: static int kFinishedConstructionMagicValue;
+public:
+  ~CMyClass();
+
+private:
+  int fMagicValue;
+
+private:
+  static int kStartedConstructionMagicValue;
+
+private:
+  static int kFinishedConstructionMagicValue;
 };
 
 // Value for fMagicValue when the constructor has started running, but not yet finished
@@ -36,42 +45,36 @@ int CMyClass::kStartedConstructionMagicValue = 0;
 // Value for fMagicValue when the constructor has finished running
 int CMyClass::kFinishedConstructionMagicValue = 12345;
 
-CMyClass::CMyClass() :
-    fMagicValue(kStartedConstructionMagicValue)
-{
-    // Signal that the constructor has finished running
-    fMagicValue = kFinishedConstructionMagicValue;
+CMyClass::CMyClass() : fMagicValue(kStartedConstructionMagicValue) {
+  // Signal that the constructor has finished running
+  fMagicValue = kFinishedConstructionMagicValue;
 }
 
-CMyClass::CMyClass(const CMyClass& /*iOther*/) :
-    fMagicValue(kStartedConstructionMagicValue)
-{
-    // If requested, throw an exception _before_ setting fMagicValue to kFinishedConstructionMagicValue
-    if (gCopyConstructorShouldThrow) {
-      throw std::exception();
-    }
-    // Signal that the constructor has finished running
-    fMagicValue = kFinishedConstructionMagicValue;
+CMyClass::CMyClass(const CMyClass& /*iOther*/) : fMagicValue(kStartedConstructionMagicValue) {
+  // If requested, throw an exception _before_ setting fMagicValue to kFinishedConstructionMagicValue
+  if (gCopyConstructorShouldThrow) {
+    throw std::exception();
+  }
+  // Signal that the constructor has finished running
+  fMagicValue = kFinishedConstructionMagicValue;
 }
 
 CMyClass::~CMyClass() {
-    // Only instances for which the constructor has finished running should be destructed
-    assert(fMagicValue == kFinishedConstructionMagicValue);
+  // Only instances for which the constructor has finished running should be destructed
+  assert(fMagicValue == kFinishedConstructionMagicValue);
 }
 
-int main(int, char**)
-{
-    CMyClass instance;
-    std::forward_list<CMyClass> vec;
+int main(int, char**) {
+  CMyClass instance;
+  std::forward_list<CMyClass> vec;
 
-    vec.push_front(instance);
+  vec.push_front(instance);
 
-    gCopyConstructorShouldThrow = true;
-    try {
-        vec.push_front(instance);
-    }
-    catch (...) {
-    }
+  gCopyConstructorShouldThrow = true;
+  try {
+    vec.push_front(instance);
+  } catch (...) {
+  }
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/forwardlist/forwardlist.modifiers/push_front_rv.pass.cpp b/libcxx/test/std/containers/sequences/forwardlist/forwardlist.modifiers/push_front_rv.pass.cpp
index 7294a454c2946..d3156c5fdd38a 100644
--- a/libcxx/test/std/containers/sequences/forwardlist/forwardlist.modifiers/push_front_rv.pass.cpp
+++ b/libcxx/test/std/containers/sequences/forwardlist/forwardlist.modifiers/push_front_rv.pass.cpp
@@ -19,32 +19,31 @@
 #include "MoveOnly.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef MoveOnly T;
-        typedef std::forward_list<T> C;
-        C c;
-        c.push_front(1);
-        assert(c.front() == 1);
-        assert(std::distance(c.begin(), c.end()) == 1);
-        c.push_front(3);
-        assert(c.front() == 3);
-        assert(*std::next(c.begin()) == 1);
-        assert(std::distance(c.begin(), c.end()) == 2);
-    }
-    {
-        typedef MoveOnly T;
-        typedef std::forward_list<T, min_allocator<T>> C;
-        C c;
-        c.push_front(1);
-        assert(c.front() == 1);
-        assert(std::distance(c.begin(), c.end()) == 1);
-        c.push_front(3);
-        assert(c.front() == 3);
-        assert(*std::next(c.begin()) == 1);
-        assert(std::distance(c.begin(), c.end()) == 2);
-    }
+int main(int, char**) {
+  {
+    typedef MoveOnly T;
+    typedef std::forward_list<T> C;
+    C c;
+    c.push_front(1);
+    assert(c.front() == 1);
+    assert(std::distance(c.begin(), c.end()) == 1);
+    c.push_front(3);
+    assert(c.front() == 3);
+    assert(*std::next(c.begin()) == 1);
+    assert(std::distance(c.begin(), c.end()) == 2);
+  }
+  {
+    typedef MoveOnly T;
+    typedef std::forward_list<T, min_allocator<T>> C;
+    C c;
+    c.push_front(1);
+    assert(c.front() == 1);
+    assert(std::distance(c.begin(), c.end()) == 1);
+    c.push_front(3);
+    assert(c.front() == 3);
+    assert(*std::next(c.begin()) == 1);
+    assert(std::distance(c.begin(), c.end()) == 2);
+  }
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/forwardlist/forwardlist.modifiers/resize_size.pass.cpp b/libcxx/test/std/containers/sequences/forwardlist/forwardlist.modifiers/resize_size.pass.cpp
index 606440ca21e6c..2dacf458d7d9d 100644
--- a/libcxx/test/std/containers/sequences/forwardlist/forwardlist.modifiers/resize_size.pass.cpp
+++ b/libcxx/test/std/containers/sequences/forwardlist/forwardlist.modifiers/resize_size.pass.cpp
@@ -18,99 +18,98 @@
 #include "DefaultOnly.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef DefaultOnly T;
-        typedef std::forward_list<T> C;
-        C c;
-        c.resize(0);
-        assert(std::distance(c.begin(), c.end()) == 0);
-        c.resize(10);
-        assert(std::distance(c.begin(), c.end()) == 10);
-        c.resize(20);
-        assert(std::distance(c.begin(), c.end()) == 20);
-        c.resize(5);
-        assert(std::distance(c.begin(), c.end()) == 5);
-        c.resize(0);
-        assert(std::distance(c.begin(), c.end()) == 0);
-    }
-    {
-        typedef int T;
-        typedef std::forward_list<T> C;
-        const T t[] = {0, 1, 2, 3, 4};
-        C c(std::begin(t), std::end(t));
+int main(int, char**) {
+  {
+    typedef DefaultOnly T;
+    typedef std::forward_list<T> C;
+    C c;
+    c.resize(0);
+    assert(std::distance(c.begin(), c.end()) == 0);
+    c.resize(10);
+    assert(std::distance(c.begin(), c.end()) == 10);
+    c.resize(20);
+    assert(std::distance(c.begin(), c.end()) == 20);
+    c.resize(5);
+    assert(std::distance(c.begin(), c.end()) == 5);
+    c.resize(0);
+    assert(std::distance(c.begin(), c.end()) == 0);
+  }
+  {
+    typedef int T;
+    typedef std::forward_list<T> C;
+    const T t[] = {0, 1, 2, 3, 4};
+    C c(std::begin(t), std::end(t));
 
-        c.resize(3);
-        assert(std::distance(c.begin(), c.end()) == 3);
-        assert(*std::next(c.begin(), 0) == 0);
-        assert(*std::next(c.begin(), 1) == 1);
-        assert(*std::next(c.begin(), 2) == 2);
+    c.resize(3);
+    assert(std::distance(c.begin(), c.end()) == 3);
+    assert(*std::next(c.begin(), 0) == 0);
+    assert(*std::next(c.begin(), 1) == 1);
+    assert(*std::next(c.begin(), 2) == 2);
 
-        c.resize(6);
-        assert(std::distance(c.begin(), c.end()) == 6);
-        assert(*std::next(c.begin(), 0) == 0);
-        assert(*std::next(c.begin(), 1) == 1);
-        assert(*std::next(c.begin(), 2) == 2);
-        assert(*std::next(c.begin(), 3) == 0);
-        assert(*std::next(c.begin(), 4) == 0);
-        assert(*std::next(c.begin(), 5) == 0);
+    c.resize(6);
+    assert(std::distance(c.begin(), c.end()) == 6);
+    assert(*std::next(c.begin(), 0) == 0);
+    assert(*std::next(c.begin(), 1) == 1);
+    assert(*std::next(c.begin(), 2) == 2);
+    assert(*std::next(c.begin(), 3) == 0);
+    assert(*std::next(c.begin(), 4) == 0);
+    assert(*std::next(c.begin(), 5) == 0);
 
-        c.resize(6);
-        assert(std::distance(c.begin(), c.end()) == 6);
-        assert(*std::next(c.begin(), 0) == 0);
-        assert(*std::next(c.begin(), 1) == 1);
-        assert(*std::next(c.begin(), 2) == 2);
-        assert(*std::next(c.begin(), 3) == 0);
-        assert(*std::next(c.begin(), 4) == 0);
-        assert(*std::next(c.begin(), 5) == 0);
-    }
+    c.resize(6);
+    assert(std::distance(c.begin(), c.end()) == 6);
+    assert(*std::next(c.begin(), 0) == 0);
+    assert(*std::next(c.begin(), 1) == 1);
+    assert(*std::next(c.begin(), 2) == 2);
+    assert(*std::next(c.begin(), 3) == 0);
+    assert(*std::next(c.begin(), 4) == 0);
+    assert(*std::next(c.begin(), 5) == 0);
+  }
 #if TEST_STD_VER >= 11
-    {
-        typedef DefaultOnly T;
-        typedef std::forward_list<T, min_allocator<T>> C;
-        C c;
-        c.resize(0);
-        assert(std::distance(c.begin(), c.end()) == 0);
-        c.resize(10);
-        assert(std::distance(c.begin(), c.end()) == 10);
-        c.resize(20);
-        assert(std::distance(c.begin(), c.end()) == 20);
-        c.resize(5);
-        assert(std::distance(c.begin(), c.end()) == 5);
-        c.resize(0);
-        assert(std::distance(c.begin(), c.end()) == 0);
-    }
-    {
-        typedef int T;
-        typedef std::forward_list<T, min_allocator<T>> C;
-        const T t[] = {0, 1, 2, 3, 4};
-        C c(std::begin(t), std::end(t));
+  {
+    typedef DefaultOnly T;
+    typedef std::forward_list<T, min_allocator<T>> C;
+    C c;
+    c.resize(0);
+    assert(std::distance(c.begin(), c.end()) == 0);
+    c.resize(10);
+    assert(std::distance(c.begin(), c.end()) == 10);
+    c.resize(20);
+    assert(std::distance(c.begin(), c.end()) == 20);
+    c.resize(5);
+    assert(std::distance(c.begin(), c.end()) == 5);
+    c.resize(0);
+    assert(std::distance(c.begin(), c.end()) == 0);
+  }
+  {
+    typedef int T;
+    typedef std::forward_list<T, min_allocator<T>> C;
+    const T t[] = {0, 1, 2, 3, 4};
+    C c(std::begin(t), std::end(t));
 
-        c.resize(3);
-        assert(std::distance(c.begin(), c.end()) == 3);
-        assert(*std::next(c.begin(), 0) == 0);
-        assert(*std::next(c.begin(), 1) == 1);
-        assert(*std::next(c.begin(), 2) == 2);
+    c.resize(3);
+    assert(std::distance(c.begin(), c.end()) == 3);
+    assert(*std::next(c.begin(), 0) == 0);
+    assert(*std::next(c.begin(), 1) == 1);
+    assert(*std::next(c.begin(), 2) == 2);
 
-        c.resize(6);
-        assert(std::distance(c.begin(), c.end()) == 6);
-        assert(*std::next(c.begin(), 0) == 0);
-        assert(*std::next(c.begin(), 1) == 1);
-        assert(*std::next(c.begin(), 2) == 2);
-        assert(*std::next(c.begin(), 3) == 0);
-        assert(*std::next(c.begin(), 4) == 0);
-        assert(*std::next(c.begin(), 5) == 0);
+    c.resize(6);
+    assert(std::distance(c.begin(), c.end()) == 6);
+    assert(*std::next(c.begin(), 0) == 0);
+    assert(*std::next(c.begin(), 1) == 1);
+    assert(*std::next(c.begin(), 2) == 2);
+    assert(*std::next(c.begin(), 3) == 0);
+    assert(*std::next(c.begin(), 4) == 0);
+    assert(*std::next(c.begin(), 5) == 0);
 
-        c.resize(6);
-        assert(std::distance(c.begin(), c.end()) == 6);
-        assert(*std::next(c.begin(), 0) == 0);
-        assert(*std::next(c.begin(), 1) == 1);
-        assert(*std::next(c.begin(), 2) == 2);
-        assert(*std::next(c.begin(), 3) == 0);
-        assert(*std::next(c.begin(), 4) == 0);
-        assert(*std::next(c.begin(), 5) == 0);
-    }
+    c.resize(6);
+    assert(std::distance(c.begin(), c.end()) == 6);
+    assert(*std::next(c.begin(), 0) == 0);
+    assert(*std::next(c.begin(), 1) == 1);
+    assert(*std::next(c.begin(), 2) == 2);
+    assert(*std::next(c.begin(), 3) == 0);
+    assert(*std::next(c.begin(), 4) == 0);
+    assert(*std::next(c.begin(), 5) == 0);
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/sequences/forwardlist/forwardlist.modifiers/resize_size_value.pass.cpp b/libcxx/test/std/containers/sequences/forwardlist/forwardlist.modifiers/resize_size_value.pass.cpp
index 3c0b64e85a1bf..a6af763e6937f 100644
--- a/libcxx/test/std/containers/sequences/forwardlist/forwardlist.modifiers/resize_size_value.pass.cpp
+++ b/libcxx/test/std/containers/sequences/forwardlist/forwardlist.modifiers/resize_size_value.pass.cpp
@@ -19,85 +19,84 @@
 #include "min_allocator.h"
 
 #if TEST_STD_VER >= 11
-#include "container_test_types.h"
+#  include "container_test_types.h"
 #endif
 
-int main(int, char**)
-{
-    {
-        typedef int T;
-        typedef std::forward_list<T> C;
-        const T t[] = {0, 1, 2, 3, 4};
-        C c(std::begin(t), std::end(t));
+int main(int, char**) {
+  {
+    typedef int T;
+    typedef std::forward_list<T> C;
+    const T t[] = {0, 1, 2, 3, 4};
+    C c(std::begin(t), std::end(t));
 
-        c.resize(3, 10);
-        assert(std::distance(c.begin(), c.end()) == 3);
-        assert(*std::next(c.begin(), 0) == 0);
-        assert(*std::next(c.begin(), 1) == 1);
-        assert(*std::next(c.begin(), 2) == 2);
+    c.resize(3, 10);
+    assert(std::distance(c.begin(), c.end()) == 3);
+    assert(*std::next(c.begin(), 0) == 0);
+    assert(*std::next(c.begin(), 1) == 1);
+    assert(*std::next(c.begin(), 2) == 2);
 
-        c.resize(6, 10);
-        assert(std::distance(c.begin(), c.end()) == 6);
-        assert(*std::next(c.begin(), 0) == 0);
-        assert(*std::next(c.begin(), 1) == 1);
-        assert(*std::next(c.begin(), 2) == 2);
-        assert(*std::next(c.begin(), 3) == 10);
-        assert(*std::next(c.begin(), 4) == 10);
-        assert(*std::next(c.begin(), 5) == 10);
+    c.resize(6, 10);
+    assert(std::distance(c.begin(), c.end()) == 6);
+    assert(*std::next(c.begin(), 0) == 0);
+    assert(*std::next(c.begin(), 1) == 1);
+    assert(*std::next(c.begin(), 2) == 2);
+    assert(*std::next(c.begin(), 3) == 10);
+    assert(*std::next(c.begin(), 4) == 10);
+    assert(*std::next(c.begin(), 5) == 10);
 
-        c.resize(6, 12);
-        assert(std::distance(c.begin(), c.end()) == 6);
-        assert(*std::next(c.begin(), 0) == 0);
-        assert(*std::next(c.begin(), 1) == 1);
-        assert(*std::next(c.begin(), 2) == 2);
-        assert(*std::next(c.begin(), 3) == 10);
-        assert(*std::next(c.begin(), 4) == 10);
-        assert(*std::next(c.begin(), 5) == 10);
-    }
+    c.resize(6, 12);
+    assert(std::distance(c.begin(), c.end()) == 6);
+    assert(*std::next(c.begin(), 0) == 0);
+    assert(*std::next(c.begin(), 1) == 1);
+    assert(*std::next(c.begin(), 2) == 2);
+    assert(*std::next(c.begin(), 3) == 10);
+    assert(*std::next(c.begin(), 4) == 10);
+    assert(*std::next(c.begin(), 5) == 10);
+  }
 #if TEST_STD_VER >= 11
-    {
-        typedef int T;
-        typedef std::forward_list<T, min_allocator<T>> C;
-        const T t[] = {0, 1, 2, 3, 4};
-        C c(std::begin(t), std::end(t));
+  {
+    typedef int T;
+    typedef std::forward_list<T, min_allocator<T>> C;
+    const T t[] = {0, 1, 2, 3, 4};
+    C c(std::begin(t), std::end(t));
 
-        c.resize(3, 10);
-        assert(std::distance(c.begin(), c.end()) == 3);
-        assert(*std::next(c.begin(), 0) == 0);
-        assert(*std::next(c.begin(), 1) == 1);
-        assert(*std::next(c.begin(), 2) == 2);
+    c.resize(3, 10);
+    assert(std::distance(c.begin(), c.end()) == 3);
+    assert(*std::next(c.begin(), 0) == 0);
+    assert(*std::next(c.begin(), 1) == 1);
+    assert(*std::next(c.begin(), 2) == 2);
 
-        c.resize(6, 10);
-        assert(std::distance(c.begin(), c.end()) == 6);
-        assert(*std::next(c.begin(), 0) == 0);
-        assert(*std::next(c.begin(), 1) == 1);
-        assert(*std::next(c.begin(), 2) == 2);
-        assert(*std::next(c.begin(), 3) == 10);
-        assert(*std::next(c.begin(), 4) == 10);
-        assert(*std::next(c.begin(), 5) == 10);
+    c.resize(6, 10);
+    assert(std::distance(c.begin(), c.end()) == 6);
+    assert(*std::next(c.begin(), 0) == 0);
+    assert(*std::next(c.begin(), 1) == 1);
+    assert(*std::next(c.begin(), 2) == 2);
+    assert(*std::next(c.begin(), 3) == 10);
+    assert(*std::next(c.begin(), 4) == 10);
+    assert(*std::next(c.begin(), 5) == 10);
 
-        c.resize(6, 12);
-        assert(std::distance(c.begin(), c.end()) == 6);
-        assert(*std::next(c.begin(), 0) == 0);
-        assert(*std::next(c.begin(), 1) == 1);
-        assert(*std::next(c.begin(), 2) == 2);
-        assert(*std::next(c.begin(), 3) == 10);
-        assert(*std::next(c.begin(), 4) == 10);
-        assert(*std::next(c.begin(), 5) == 10);
-    }
+    c.resize(6, 12);
+    assert(std::distance(c.begin(), c.end()) == 6);
+    assert(*std::next(c.begin(), 0) == 0);
+    assert(*std::next(c.begin(), 1) == 1);
+    assert(*std::next(c.begin(), 2) == 2);
+    assert(*std::next(c.begin(), 3) == 10);
+    assert(*std::next(c.begin(), 4) == 10);
+    assert(*std::next(c.begin(), 5) == 10);
+  }
+  {
+    // Test that the allocator's construct method is being used to
+    // construct the new elements and that it's called exactly N times.
+    typedef std::forward_list<int, ContainerTestAllocator<int, int>> Container;
+    ConstructController* cc = getConstructController();
+    cc->reset();
     {
-        // Test that the allocator's construct method is being used to
-        // construct the new elements and that it's called exactly N times.
-        typedef std::forward_list<int, ContainerTestAllocator<int, int>> Container;
-        ConstructController* cc = getConstructController();
-        cc->reset();
-        {
-            Container c;
-            cc->expect<int const&>(6);
-            c.resize(6, 42);
-            assert(!cc->unchecked());
-        }
+      Container c;
+      cc->expect<int const&>(6);
+      c.resize(6, 42);
+      assert(!cc->unchecked());
     }
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/sequences/forwardlist/forwardlist.ops/merge_rvalue.pass.cpp b/libcxx/test/std/containers/sequences/forwardlist/forwardlist.ops/merge_rvalue.pass.cpp
index 3a7a8744ad46a..d5084eccd98da 100644
--- a/libcxx/test/std/containers/sequences/forwardlist/forwardlist.ops/merge_rvalue.pass.cpp
+++ b/libcxx/test/std/containers/sequences/forwardlist/forwardlist.ops/merge_rvalue.pass.cpp
@@ -57,9 +57,9 @@ int main(int, char**) {
     const T to[3] = {0, 1, 2};
 
     C c2(std::begin(to), std::end(to));
-    I io[3] = {c2.begin(), ++c2.begin(), ++ ++c2.begin()};
+    I io[3]                         = {c2.begin(), ++c2.begin(), ++ ++c2.begin()};
     std::reference_wrapper<T> ro[3] = {*io[0], *io[1], *io[2]};
-    P po[3] = {&*io[0], &*io[1], &*io[2]};
+    P po[3]                         = {&*io[0], &*io[1], &*io[2]};
 
     C c1;
     c1.merge(std::move(c2));
diff --git a/libcxx/test/std/containers/sequences/forwardlist/forwardlist.ops/merge_rvalue_pred.pass.cpp b/libcxx/test/std/containers/sequences/forwardlist/forwardlist.ops/merge_rvalue_pred.pass.cpp
index f1ddbfb620d29..235707c65370d 100644
--- a/libcxx/test/std/containers/sequences/forwardlist/forwardlist.ops/merge_rvalue_pred.pass.cpp
+++ b/libcxx/test/std/containers/sequences/forwardlist/forwardlist.ops/merge_rvalue_pred.pass.cpp
@@ -57,9 +57,9 @@ int main(int, char**) {
     const T to[3] = {2, 1, 0};
 
     C c2(std::begin(to), std::end(to));
-    I io[3] = {c2.begin(), ++c2.begin(), ++ ++c2.begin()};
+    I io[3]                         = {c2.begin(), ++c2.begin(), ++ ++c2.begin()};
     std::reference_wrapper<T> ro[3] = {*io[0], *io[1], *io[2]};
-    P po[3] = {&*io[0], &*io[1], &*io[2]};
+    P po[3]                         = {&*io[0], &*io[1], &*io[2]};
 
     C c1;
     c1.merge(std::move(c2), std::greater<T>());
diff --git a/libcxx/test/std/containers/sequences/forwardlist/forwardlist.ops/remove.pass.cpp b/libcxx/test/std/containers/sequences/forwardlist/forwardlist.ops/remove.pass.cpp
index c475468549cfe..ec3bf845dcc5a 100644
--- a/libcxx/test/std/containers/sequences/forwardlist/forwardlist.ops/remove.pass.cpp
+++ b/libcxx/test/std/containers/sequences/forwardlist/forwardlist.ops/remove.pass.cpp
@@ -19,154 +19,156 @@
 #include "min_allocator.h"
 
 template <class L>
-void do_remove(L &l, const typename L::value_type &value, typename L::size_type expected)
-{
-    typename L::size_type old_size = std::distance(l.begin(), l.end());
+void do_remove(L& l, const typename L::value_type& value, typename L::size_type expected) {
+  typename L::size_type old_size = std::distance(l.begin(), l.end());
 #if TEST_STD_VER > 17
-    ASSERT_SAME_TYPE(decltype(l.remove(value)), typename L::size_type);
-    assert(l.remove(value) == expected);
+  ASSERT_SAME_TYPE(decltype(l.remove(value)), typename L::size_type);
+  assert(l.remove(value) == expected);
 #else
-    ASSERT_SAME_TYPE(decltype(l.remove(value)), void);
-    l.remove(value);
+  ASSERT_SAME_TYPE(decltype(l.remove(value)), void);
+  l.remove(value);
 #endif
-    assert(old_size - std::distance(l.begin(), l.end()) == expected);
+  assert(old_size - std::distance(l.begin(), l.end()) == expected);
 }
 
-
 struct S {
-    S(int i) : i_(new int(i)) {}
-    S(const S &rhs) : i_(new int(*rhs.i_)) {}
-    S& operator = (const S &rhs) { *i_ = *rhs.i_; return *this; }
-    ~S () { delete i_; i_ = NULL; }
-    bool operator == (const S &rhs) const { return *i_ == *rhs.i_; }
-    int get () const { return *i_; }
-    int *i_;
-    };
-
+  S(int i) : i_(new int(i)) {}
+  S(const S& rhs) : i_(new int(*rhs.i_)) {}
+  S& operator=(const S& rhs) {
+    *i_ = *rhs.i_;
+    return *this;
+  }
+  ~S() {
+    delete i_;
+    i_ = NULL;
+  }
+  bool operator==(const S& rhs) const { return *i_ == *rhs.i_; }
+  int get() const { return *i_; }
+  int* i_;
+};
 
-int main(int, char**)
-{
-    {
-        typedef int T;
-        typedef std::forward_list<T> C;
-        const T t1[] = {0, 5, 5, 0, 0, 0, 5};
-        const T t2[] = {5, 5, 5};
-        C c1(std::begin(t1), std::end(t1));
-        C c2(std::begin(t2), std::end(t2));
-        do_remove(c1, 0, 4);
-        assert(c1 == c2);
-    }
-    {
-        typedef int T;
-        typedef std::forward_list<T> C;
-        const T t1[] = {0, 0, 0, 0};
-        C c1(std::begin(t1), std::end(t1));
-        C c2;
-        do_remove(c1, 0, 4);
-        assert(c1 == c2);
-    }
-    {
-        typedef int T;
-        typedef std::forward_list<T> C;
-        const T t1[] = {5, 5, 5};
-        const T t2[] = {5, 5, 5};
-        C c1(std::begin(t1), std::end(t1));
-        C c2(std::begin(t2), std::end(t2));
-        do_remove(c1, 0, 0);
-        assert(c1 == c2);
-    }
-    {
-        typedef int T;
-        typedef std::forward_list<T> C;
-        C c1;
-        C c2;
-        do_remove(c1, 0, 0);
-        assert(c1 == c2);
-    }
-    {
-        typedef int T;
-        typedef std::forward_list<T> C;
-        const T t1[] = {5, 5, 5, 0};
-        const T t2[] = {5, 5, 5};
-        C c1(std::begin(t1), std::end(t1));
-        C c2(std::begin(t2), std::end(t2));
-        do_remove(c1, 0, 1);
-        assert(c1 == c2);
-    }
-    {  // LWG issue #526
+int main(int, char**) {
+  {
+    typedef int T;
+    typedef std::forward_list<T> C;
+    const T t1[] = {0, 5, 5, 0, 0, 0, 5};
+    const T t2[] = {5, 5, 5};
+    C c1(std::begin(t1), std::end(t1));
+    C c2(std::begin(t2), std::end(t2));
+    do_remove(c1, 0, 4);
+    assert(c1 == c2);
+  }
+  {
+    typedef int T;
+    typedef std::forward_list<T> C;
+    const T t1[] = {0, 0, 0, 0};
+    C c1(std::begin(t1), std::end(t1));
+    C c2;
+    do_remove(c1, 0, 4);
+    assert(c1 == c2);
+  }
+  {
+    typedef int T;
+    typedef std::forward_list<T> C;
+    const T t1[] = {5, 5, 5};
+    const T t2[] = {5, 5, 5};
+    C c1(std::begin(t1), std::end(t1));
+    C c2(std::begin(t2), std::end(t2));
+    do_remove(c1, 0, 0);
+    assert(c1 == c2);
+  }
+  {
+    typedef int T;
+    typedef std::forward_list<T> C;
+    C c1;
+    C c2;
+    do_remove(c1, 0, 0);
+    assert(c1 == c2);
+  }
+  {
+    typedef int T;
+    typedef std::forward_list<T> C;
+    const T t1[] = {5, 5, 5, 0};
+    const T t2[] = {5, 5, 5};
+    C c1(std::begin(t1), std::end(t1));
+    C c2(std::begin(t2), std::end(t2));
+    do_remove(c1, 0, 1);
+    assert(c1 == c2);
+  }
+  { // LWG issue #526
     typedef int T;
     typedef std::forward_list<T> C;
     int t1[] = {1, 2, 1, 3, 5, 8, 11};
-    int t2[] = {   2,    3, 5, 8, 11};
+    int t2[] = {2, 3, 5, 8, 11};
     C c1(std::begin(t1), std::end(t1));
     C c2(std::begin(t2), std::end(t2));
     do_remove(c1, c1.front(), 2);
     assert(c1 == c2);
-    }
-    {
+  }
+  {
     typedef S T;
     typedef std::forward_list<T> C;
     int t1[] = {1, 2, 1, 3, 5, 8, 11, 1};
-    int t2[] = {   2,    3, 5, 8, 11   };
+    int t2[] = {2, 3, 5, 8, 11};
     C c;
-    for(int *ip = std::end(t1); ip != std::begin(t1);)
-        c.push_front(S(*--ip));
+    for (int* ip = std::end(t1); ip != std::begin(t1);)
+      c.push_front(S(*--ip));
     do_remove(c, c.front(), 3);
     C::const_iterator it = c.begin();
-    for(int *ip = std::begin(t2); ip != std::end(t2); ++ip, ++it) {
-        assert ( it != c.end());
-        assert ( *ip == it->get());
-        }
-    assert ( it == c.end ());
+    for (int* ip = std::begin(t2); ip != std::end(t2); ++ip, ++it) {
+      assert(it != c.end());
+      assert(*ip == it->get());
     }
+    assert(it == c.end());
+  }
 #if TEST_STD_VER >= 11
-    {
-        typedef int T;
-        typedef std::forward_list<T, min_allocator<T>> C;
-        const T t1[] = {0, 5, 5, 0, 0, 0, 5};
-        const T t2[] = {5, 5, 5};
-        C c1(std::begin(t1), std::end(t1));
-        C c2(std::begin(t2), std::end(t2));
-        do_remove(c1, 0, 4);
-        assert(c1 == c2);
-    }
-    {
-        typedef int T;
-        typedef std::forward_list<T, min_allocator<T>> C;
-        const T t1[] = {0, 0, 0, 0};
-        C c1(std::begin(t1), std::end(t1));
-        C c2;
-        do_remove(c1, 0, 4);
-        assert(c1 == c2);
-    }
-    {
-        typedef int T;
-        typedef std::forward_list<T, min_allocator<T>> C;
-        const T t1[] = {5, 5, 5};
-        const T t2[] = {5, 5, 5};
-        C c1(std::begin(t1), std::end(t1));
-        C c2(std::begin(t2), std::end(t2));
-        do_remove(c1, 0, 0);
-        assert(c1 == c2);
-    }
-    {
-        typedef int T;
-        typedef std::forward_list<T, min_allocator<T>> C;
-        C c1;
-        C c2;
-        do_remove(c1, 0, 0);
-        assert(c1 == c2);
-    }
-    {
-        typedef int T;
-        typedef std::forward_list<T, min_allocator<T>> C;
-        const T t1[] = {5, 5, 5, 0};
-        const T t2[] = {5, 5, 5};
-        C c1(std::begin(t1), std::end(t1));
-        C c2(std::begin(t2), std::end(t2));
-        do_remove(c1, 0, 1);
-        assert(c1 == c2);
-    }
+  {
+    typedef int T;
+    typedef std::forward_list<T, min_allocator<T>> C;
+    const T t1[] = {0, 5, 5, 0, 0, 0, 5};
+    const T t2[] = {5, 5, 5};
+    C c1(std::begin(t1), std::end(t1));
+    C c2(std::begin(t2), std::end(t2));
+    do_remove(c1, 0, 4);
+    assert(c1 == c2);
+  }
+  {
+    typedef int T;
+    typedef std::forward_list<T, min_allocator<T>> C;
+    const T t1[] = {0, 0, 0, 0};
+    C c1(std::begin(t1), std::end(t1));
+    C c2;
+    do_remove(c1, 0, 4);
+    assert(c1 == c2);
+  }
+  {
+    typedef int T;
+    typedef std::forward_list<T, min_allocator<T>> C;
+    const T t1[] = {5, 5, 5};
+    const T t2[] = {5, 5, 5};
+    C c1(std::begin(t1), std::end(t1));
+    C c2(std::begin(t2), std::end(t2));
+    do_remove(c1, 0, 0);
+    assert(c1 == c2);
+  }
+  {
+    typedef int T;
+    typedef std::forward_list<T, min_allocator<T>> C;
+    C c1;
+    C c2;
+    do_remove(c1, 0, 0);
+    assert(c1 == c2);
+  }
+  {
+    typedef int T;
+    typedef std::forward_list<T, min_allocator<T>> C;
+    const T t1[] = {5, 5, 5, 0};
+    const T t2[] = {5, 5, 5};
+    C c1(std::begin(t1), std::end(t1));
+    C c2(std::begin(t2), std::end(t2));
+    do_remove(c1, 0, 1);
+    assert(c1 == c2);
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/sequences/forwardlist/forwardlist.ops/remove_if.pass.cpp b/libcxx/test/std/containers/sequences/forwardlist/forwardlist.ops/remove_if.pass.cpp
index 4e5b9bce14eb4..c6325baea2590 100644
--- a/libcxx/test/std/containers/sequences/forwardlist/forwardlist.ops/remove_if.pass.cpp
+++ b/libcxx/test/std/containers/sequences/forwardlist/forwardlist.ops/remove_if.pass.cpp
@@ -21,25 +21,20 @@
 #include "min_allocator.h"
 #include "counting_predicates.h"
 
-
 template <class L, class Predicate>
-void do_remove_if(L &l, Predicate pred, typename L::size_type expected)
-{
-    typename L::size_type old_size = std::distance(l.begin(), l.end());
+void do_remove_if(L& l, Predicate pred, typename L::size_type expected) {
+  typename L::size_type old_size = std::distance(l.begin(), l.end());
 #if TEST_STD_VER > 17
-    ASSERT_SAME_TYPE(decltype(l.remove_if(pred)), typename L::size_type);
-    assert(l.remove_if(pred) == expected);
+  ASSERT_SAME_TYPE(decltype(l.remove_if(pred)), typename L::size_type);
+  assert(l.remove_if(pred) == expected);
 #else
-    ASSERT_SAME_TYPE(decltype(l.remove_if(pred)), void);
-    l.remove_if(pred);
+  ASSERT_SAME_TYPE(decltype(l.remove_if(pred)), void);
+  l.remove_if(pred);
 #endif
-    assert(old_size - std::distance(l.begin(), l.end()) == expected);
+  assert(old_size - std::distance(l.begin(), l.end()) == expected);
 }
 
-bool g(int i)
-{
-    return i < 3;
-}
+bool g(int i) { return i < 3; }
 
 struct PredLWG526 {
   PredLWG526(int i) : i_(i) {}
@@ -50,148 +45,146 @@ struct PredLWG526 {
   int i_;
 };
 
-int main(int, char**)
-{
-    {
-        typedef int T;
-        typedef unary_counting_predicate<bool(*)(T), T> Predicate;
-        typedef std::forward_list<T> C;
-        const T t1[] = {0, 5, 5, 0, 0, 0, 5};
-        const T t2[] = {5, 5, 5};
-        C c1(std::begin(t1), std::end(t1));
-        C c2(std::begin(t2), std::end(t2));
-        Predicate cp(g);
-        do_remove_if(c1, std::ref(cp), 4);
-        assert(c1 == c2);
-        assert(cp.count() == static_cast<std::size_t>(std::distance(std::begin(t1), std::end(t1))));
-    }
-    {
-        typedef int T;
-        typedef unary_counting_predicate<bool(*)(T), T> Predicate;
-        typedef std::forward_list<T> C;
-        const T t1[] = {0, 0, 0, 0};
-        C c1(std::begin(t1), std::end(t1));
-        C c2;
-        Predicate cp(g);
-        do_remove_if(c1, std::ref(cp), 4);
-        assert(c1 == c2);
-        assert(cp.count() == static_cast<std::size_t>(std::distance(std::begin(t1), std::end(t1))));
-    }
-    {
-        typedef int T;
-        typedef unary_counting_predicate<bool(*)(T), T> Predicate;
-        typedef std::forward_list<T> C;
-        const T t1[] = {5, 5, 5};
-        const T t2[] = {5, 5, 5};
-        C c1(std::begin(t1), std::end(t1));
-        C c2(std::begin(t2), std::end(t2));
-        Predicate cp(g);
-        do_remove_if(c1, std::ref(cp), 0);
-        assert(c1 == c2);
-        assert(cp.count() == static_cast<std::size_t>(std::distance(std::begin(t1), std::end(t1))));
-    }
-    {
-        typedef int T;
-        typedef unary_counting_predicate<bool(*)(T), T> Predicate;
-        typedef std::forward_list<T> C;
-        C c1;
-        C c2;
-        Predicate cp(g);
-        do_remove_if(c1, std::ref(cp), 0);
-        assert(c1 == c2);
-        assert(cp.count() == 0);
-    }
-    {
-        typedef int T;
-        typedef unary_counting_predicate<bool(*)(T), T> Predicate;
-        typedef std::forward_list<T> C;
-        const T t1[] = {5, 5, 5, 0};
-        const T t2[] = {5, 5, 5};
-        C c1(std::begin(t1), std::end(t1));
-        C c2(std::begin(t2), std::end(t2));
-        Predicate cp(g);
-        do_remove_if(c1, std::ref(cp), 1);
-        assert(c1 == c2);
-        assert(cp.count() == static_cast<std::size_t>(std::distance(std::begin(t1), std::end(t1))));
-    }
+int main(int, char**) {
+  {
+    typedef int T;
+    typedef unary_counting_predicate<bool (*)(T), T> Predicate;
+    typedef std::forward_list<T> C;
+    const T t1[] = {0, 5, 5, 0, 0, 0, 5};
+    const T t2[] = {5, 5, 5};
+    C c1(std::begin(t1), std::end(t1));
+    C c2(std::begin(t2), std::end(t2));
+    Predicate cp(g);
+    do_remove_if(c1, std::ref(cp), 4);
+    assert(c1 == c2);
+    assert(cp.count() == static_cast<std::size_t>(std::distance(std::begin(t1), std::end(t1))));
+  }
+  {
+    typedef int T;
+    typedef unary_counting_predicate<bool (*)(T), T> Predicate;
+    typedef std::forward_list<T> C;
+    const T t1[] = {0, 0, 0, 0};
+    C c1(std::begin(t1), std::end(t1));
+    C c2;
+    Predicate cp(g);
+    do_remove_if(c1, std::ref(cp), 4);
+    assert(c1 == c2);
+    assert(cp.count() == static_cast<std::size_t>(std::distance(std::begin(t1), std::end(t1))));
+  }
+  {
+    typedef int T;
+    typedef unary_counting_predicate<bool (*)(T), T> Predicate;
+    typedef std::forward_list<T> C;
+    const T t1[] = {5, 5, 5};
+    const T t2[] = {5, 5, 5};
+    C c1(std::begin(t1), std::end(t1));
+    C c2(std::begin(t2), std::end(t2));
+    Predicate cp(g);
+    do_remove_if(c1, std::ref(cp), 0);
+    assert(c1 == c2);
+    assert(cp.count() == static_cast<std::size_t>(std::distance(std::begin(t1), std::end(t1))));
+  }
+  {
+    typedef int T;
+    typedef unary_counting_predicate<bool (*)(T), T> Predicate;
+    typedef std::forward_list<T> C;
+    C c1;
+    C c2;
+    Predicate cp(g);
+    do_remove_if(c1, std::ref(cp), 0);
+    assert(c1 == c2);
+    assert(cp.count() == 0);
+  }
+  {
+    typedef int T;
+    typedef unary_counting_predicate<bool (*)(T), T> Predicate;
+    typedef std::forward_list<T> C;
+    const T t1[] = {5, 5, 5, 0};
+    const T t2[] = {5, 5, 5};
+    C c1(std::begin(t1), std::end(t1));
+    C c2(std::begin(t2), std::end(t2));
+    Predicate cp(g);
+    do_remove_if(c1, std::ref(cp), 1);
+    assert(c1 == c2);
+    assert(cp.count() == static_cast<std::size_t>(std::distance(std::begin(t1), std::end(t1))));
+  }
 
-    { // LWG issue #526
+  { // LWG issue #526
     int a1[] = {1, 2, 1, 3, 5, 8, 11};
-    int a2[] = {   2,    3, 5, 8, 11};
+    int a2[] = {2, 3, 5, 8, 11};
     std::forward_list<PredLWG526> c(a1, a1 + 7);
     do_remove_if(c, std::ref(c.front()), 2);
-    for (std::size_t i = 0; i < 5; ++i)
-    {
-        assert(!c.empty());
-        assert(c.front() == a2[i]);
-        c.pop_front();
+    for (std::size_t i = 0; i < 5; ++i) {
+      assert(!c.empty());
+      assert(c.front() == a2[i]);
+      c.pop_front();
     }
     assert(c.empty());
-    }
+  }
 
 #if TEST_STD_VER >= 11
-    {
-        typedef int T;
-        typedef unary_counting_predicate<bool(*)(T), T> Predicate;
-        typedef std::forward_list<T, min_allocator<T>> C;
-        const T t1[] = {0, 5, 5, 0, 0, 0, 5};
-        const T t2[] = {5, 5, 5};
-        C c1(std::begin(t1), std::end(t1));
-        C c2(std::begin(t2), std::end(t2));
-        Predicate cp(g);
-        do_remove_if(c1, std::ref(cp), 4);
-        assert(c1 == c2);
-        assert(cp.count() == static_cast<std::size_t>(std::distance(std::begin(t1), std::end(t1))));
-    }
-    {
-        typedef int T;
-        typedef unary_counting_predicate<bool(*)(T), T> Predicate;
-        typedef std::forward_list<T, min_allocator<T>> C;
-        const T t1[] = {0, 0, 0, 0};
-        C c1(std::begin(t1), std::end(t1));
-        C c2;
-        Predicate cp(g);
-        do_remove_if(c1, std::ref(cp), 4);
-        assert(c1 == c2);
-        assert(cp.count() == static_cast<std::size_t>(std::distance(std::begin(t1), std::end(t1))));
-    }
-    {
-        typedef int T;
-        typedef unary_counting_predicate<bool(*)(T), T> Predicate;
-        typedef std::forward_list<T, min_allocator<T>> C;
-        const T t1[] = {5, 5, 5};
-        const T t2[] = {5, 5, 5};
-        C c1(std::begin(t1), std::end(t1));
-        C c2(std::begin(t2), std::end(t2));
-        Predicate cp(g);
-        do_remove_if(c1, std::ref(cp), 0);
-        assert(c1 == c2);
-        assert(cp.count() == static_cast<std::size_t>(std::distance(std::begin(t1), std::end(t1))));
-    }
-    {
-        typedef int T;
-        typedef unary_counting_predicate<bool(*)(T), T> Predicate;
-        typedef std::forward_list<T, min_allocator<T>> C;
-        C c1;
-        C c2;
-        Predicate cp(g);
-        do_remove_if(c1, std::ref(cp), 0);
-        assert(c1 == c2);
-        assert(cp.count() == 0);
-    }
-    {
-        typedef int T;
-        typedef unary_counting_predicate<bool(*)(T), T> Predicate;
-        typedef std::forward_list<T, min_allocator<T>> C;
-        const T t1[] = {5, 5, 5, 0};
-        const T t2[] = {5, 5, 5};
-        C c1(std::begin(t1), std::end(t1));
-        C c2(std::begin(t2), std::end(t2));
-        Predicate cp(g);
-        do_remove_if(c1, std::ref(cp), 1);
-        assert(c1 == c2);
-        assert(cp.count() == static_cast<std::size_t>(std::distance(std::begin(t1), std::end(t1))));
-    }
+  {
+    typedef int T;
+    typedef unary_counting_predicate<bool (*)(T), T> Predicate;
+    typedef std::forward_list<T, min_allocator<T>> C;
+    const T t1[] = {0, 5, 5, 0, 0, 0, 5};
+    const T t2[] = {5, 5, 5};
+    C c1(std::begin(t1), std::end(t1));
+    C c2(std::begin(t2), std::end(t2));
+    Predicate cp(g);
+    do_remove_if(c1, std::ref(cp), 4);
+    assert(c1 == c2);
+    assert(cp.count() == static_cast<std::size_t>(std::distance(std::begin(t1), std::end(t1))));
+  }
+  {
+    typedef int T;
+    typedef unary_counting_predicate<bool (*)(T), T> Predicate;
+    typedef std::forward_list<T, min_allocator<T>> C;
+    const T t1[] = {0, 0, 0, 0};
+    C c1(std::begin(t1), std::end(t1));
+    C c2;
+    Predicate cp(g);
+    do_remove_if(c1, std::ref(cp), 4);
+    assert(c1 == c2);
+    assert(cp.count() == static_cast<std::size_t>(std::distance(std::begin(t1), std::end(t1))));
+  }
+  {
+    typedef int T;
+    typedef unary_counting_predicate<bool (*)(T), T> Predicate;
+    typedef std::forward_list<T, min_allocator<T>> C;
+    const T t1[] = {5, 5, 5};
+    const T t2[] = {5, 5, 5};
+    C c1(std::begin(t1), std::end(t1));
+    C c2(std::begin(t2), std::end(t2));
+    Predicate cp(g);
+    do_remove_if(c1, std::ref(cp), 0);
+    assert(c1 == c2);
+    assert(cp.count() == static_cast<std::size_t>(std::distance(std::begin(t1), std::end(t1))));
+  }
+  {
+    typedef int T;
+    typedef unary_counting_predicate<bool (*)(T), T> Predicate;
+    typedef std::forward_list<T, min_allocator<T>> C;
+    C c1;
+    C c2;
+    Predicate cp(g);
+    do_remove_if(c1, std::ref(cp), 0);
+    assert(c1 == c2);
+    assert(cp.count() == 0);
+  }
+  {
+    typedef int T;
+    typedef unary_counting_predicate<bool (*)(T), T> Predicate;
+    typedef std::forward_list<T, min_allocator<T>> C;
+    const T t1[] = {5, 5, 5, 0};
+    const T t2[] = {5, 5, 5};
+    C c1(std::begin(t1), std::end(t1));
+    C c2(std::begin(t2), std::end(t2));
+    Predicate cp(g);
+    do_remove_if(c1, std::ref(cp), 1);
+    assert(c1 == c2);
+    assert(cp.count() == static_cast<std::size_t>(std::distance(std::begin(t1), std::end(t1))));
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/sequences/forwardlist/forwardlist.ops/reverse.pass.cpp b/libcxx/test/std/containers/sequences/forwardlist/forwardlist.ops/reverse.pass.cpp
index c337d138ef799..0d0656897f34e 100644
--- a/libcxx/test/std/containers/sequences/forwardlist/forwardlist.ops/reverse.pass.cpp
+++ b/libcxx/test/std/containers/sequences/forwardlist/forwardlist.ops/reverse.pass.cpp
@@ -19,25 +19,23 @@
 #include "min_allocator.h"
 
 template <class C>
-void test(int N)
-{
-    C c;
-    for (int i = 0; i < N; ++i)
-        c.push_front(i);
-    c.reverse();
-    assert(std::distance(c.begin(), c.end()) == N);
-    typename C::const_iterator j = c.begin();
-    for (int i = 0; i < N; ++i, ++j)
-        assert(*j == i);
+void test(int N) {
+  C c;
+  for (int i = 0; i < N; ++i)
+    c.push_front(i);
+  c.reverse();
+  assert(std::distance(c.begin(), c.end()) == N);
+  typename C::const_iterator j = c.begin();
+  for (int i = 0; i < N; ++i, ++j)
+    assert(*j == i);
 }
 
-int main(int, char**)
-{
-    for (int i = 0; i < 10; ++i)
-        test<std::forward_list<int> >(i);
+int main(int, char**) {
+  for (int i = 0; i < 10; ++i)
+    test<std::forward_list<int> >(i);
 #if TEST_STD_VER >= 11
-    for (int i = 0; i < 10; ++i)
-        test<std::forward_list<int, min_allocator<int>> >(i);
+  for (int i = 0; i < 10; ++i)
+    test<std::forward_list<int, min_allocator<int>> >(i);
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/sequences/forwardlist/forwardlist.ops/sort.pass.cpp b/libcxx/test/std/containers/sequences/forwardlist/forwardlist.ops/sort.pass.cpp
index 4789e73584b6b..95d6147be9b0c 100644
--- a/libcxx/test/std/containers/sequences/forwardlist/forwardlist.ops/sort.pass.cpp
+++ b/libcxx/test/std/containers/sequences/forwardlist/forwardlist.ops/sort.pass.cpp
@@ -23,73 +23,68 @@
 std::mt19937 randomness;
 
 template <class C>
-void test(int N)
-{
-    typedef typename C::value_type T;
-    typedef std::vector<T> V;
-    V v;
-    for (int i = 0; i < N; ++i)
-        v.push_back(i);
-    std::shuffle(v.begin(), v.end(), randomness);
-    C c(v.begin(), v.end());
-    c.sort();
-    assert(std::distance(c.begin(), c.end()) == N);
-    typename C::const_iterator j = c.begin();
-    for (int i = 0; i < N; ++i, ++j)
-        assert(*j == i);
+void test(int N) {
+  typedef typename C::value_type T;
+  typedef std::vector<T> V;
+  V v;
+  for (int i = 0; i < N; ++i)
+    v.push_back(i);
+  std::shuffle(v.begin(), v.end(), randomness);
+  C c(v.begin(), v.end());
+  c.sort();
+  assert(std::distance(c.begin(), c.end()) == N);
+  typename C::const_iterator j = c.begin();
+  for (int i = 0; i < N; ++i, ++j)
+    assert(*j == i);
 }
 
-struct Payload
-{
-    int val;
-    int side;
-    Payload(int v) : val(v), side(0) {}
-    Payload(int v, int s) : val(v), side(s) {}
-    bool operator< (const Payload &rhs) const { return val <  rhs.val;}
-//     bool operator==(const Payload &rhs) const { return val == rhs.val;}
+struct Payload {
+  int val;
+  int side;
+  Payload(int v) : val(v), side(0) {}
+  Payload(int v, int s) : val(v), side(s) {}
+  bool operator<(const Payload& rhs) const { return val < rhs.val; }
+  //     bool operator==(const Payload &rhs) const { return val == rhs.val;}
 };
 
-void test_stable(int N)
-{
-    typedef Payload T;
-    typedef std::forward_list<T> C;
-    typedef std::vector<T> V;
-    V v;
-    for (int i = 0; i < N; ++i)
-        v.push_back(Payload(i/2));
-    std::shuffle(v.begin(), v.end(), randomness);
-    for (int i = 0; i < N; ++i)
-        v[i].side = i;
+void test_stable(int N) {
+  typedef Payload T;
+  typedef std::forward_list<T> C;
+  typedef std::vector<T> V;
+  V v;
+  for (int i = 0; i < N; ++i)
+    v.push_back(Payload(i / 2));
+  std::shuffle(v.begin(), v.end(), randomness);
+  for (int i = 0; i < N; ++i)
+    v[i].side = i;
 
-    C c(v.begin(), v.end());
-    c.sort();
-    assert(std::distance(c.begin(), c.end()) == N);
+  C c(v.begin(), v.end());
+  c.sort();
+  assert(std::distance(c.begin(), c.end()) == N);
 
-//  Are we sorted?
-    typename C::const_iterator j = c.begin();
-    for (int i = 0; i < N; ++i, ++j)
-        assert(j->val == i/2);
+  //  Are we sorted?
+  typename C::const_iterator j = c.begin();
+  for (int i = 0; i < N; ++i, ++j)
+    assert(j->val == i / 2);
 
-//  Are we stable?
-    for (C::const_iterator it = c.begin(); it != c.end(); ++it)
-    {
-        C::const_iterator next = std::next(it);
-        if (next != c.end() && it->val == next->val)
-            assert(it->side < next->side);
-    }
+  //  Are we stable?
+  for (C::const_iterator it = c.begin(); it != c.end(); ++it) {
+    C::const_iterator next = std::next(it);
+    if (next != c.end() && it->val == next->val)
+      assert(it->side < next->side);
+  }
 }
 
-int main(int, char**)
-{
-    for (int i = 0; i < 40; ++i)
-        test<std::forward_list<int> >(i);
+int main(int, char**) {
+  for (int i = 0; i < 40; ++i)
+    test<std::forward_list<int> >(i);
 #if TEST_STD_VER >= 11
-    for (int i = 0; i < 40; ++i)
-        test<std::forward_list<int, min_allocator<int>> >(i);
+  for (int i = 0; i < 40; ++i)
+    test<std::forward_list<int, min_allocator<int>> >(i);
 #endif
 
-    for (int i = 0; i < 40; ++i)
-        test_stable(i);
+  for (int i = 0; i < 40; ++i)
+    test_stable(i);
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/forwardlist/forwardlist.ops/sort_pred.pass.cpp b/libcxx/test/std/containers/sequences/forwardlist/forwardlist.ops/sort_pred.pass.cpp
index 6db1599448608..2644aed6fe65b 100644
--- a/libcxx/test/std/containers/sequences/forwardlist/forwardlist.ops/sort_pred.pass.cpp
+++ b/libcxx/test/std/containers/sequences/forwardlist/forwardlist.ops/sort_pred.pass.cpp
@@ -23,75 +23,70 @@
 
 std::mt19937 randomness;
 
-struct Payload
-{
-    int val;
-    int side;
-    Payload(int v) : val(v), side(0) {}
-    Payload(int v, int s) : val(v), side(s) {}
-    bool operator< (const Payload &rhs) const { return val <  rhs.val;}
+struct Payload {
+  int val;
+  int side;
+  Payload(int v) : val(v), side(0) {}
+  Payload(int v, int s) : val(v), side(s) {}
+  bool operator<(const Payload& rhs) const { return val < rhs.val; }
 };
 
-bool greater(const Payload &lhs, const Payload &rhs) { return lhs.val >  rhs.val; }
+bool greater(const Payload& lhs, const Payload& rhs) { return lhs.val > rhs.val; }
 
-void test_stable(int N)
-{
-    typedef Payload T;
-    typedef std::forward_list<T> C;
-    typedef std::vector<T> V;
-    V v;
-    for (int i = 0; i < N; ++i)
-        v.push_back(Payload(i/2));
-    std::shuffle(v.begin(), v.end(), randomness);
-    for (int i = 0; i < N; ++i)
-        v[i].side = i;
+void test_stable(int N) {
+  typedef Payload T;
+  typedef std::forward_list<T> C;
+  typedef std::vector<T> V;
+  V v;
+  for (int i = 0; i < N; ++i)
+    v.push_back(Payload(i / 2));
+  std::shuffle(v.begin(), v.end(), randomness);
+  for (int i = 0; i < N; ++i)
+    v[i].side = i;
 
-    C c(v.begin(), v.end());
-    c.sort(greater);
-    assert(std::distance(c.begin(), c.end()) == N);
+  C c(v.begin(), v.end());
+  c.sort(greater);
+  assert(std::distance(c.begin(), c.end()) == N);
 
-//  Are we sorted?
-    typename C::const_iterator j = c.begin();
-    for (int i = 0; i < N; ++i, ++j)
-        assert(j->val == (N-1-i)/2);
+  //  Are we sorted?
+  typename C::const_iterator j = c.begin();
+  for (int i = 0; i < N; ++i, ++j)
+    assert(j->val == (N - 1 - i) / 2);
 
-//  Are we stable?
-    for (C::const_iterator it = c.begin(); it != c.end(); ++it)
-    {
-        C::const_iterator next = std::next(it);
-        if (next != c.end() && it->val == next->val)
-            assert(it->side < next->side);
-    }
+  //  Are we stable?
+  for (C::const_iterator it = c.begin(); it != c.end(); ++it) {
+    C::const_iterator next = std::next(it);
+    if (next != c.end() && it->val == next->val)
+      assert(it->side < next->side);
+  }
 }
 
 template <class C>
-void test(int N)
-{
-    typedef typename C::value_type T;
-    typedef std::vector<T> V;
-    V v;
-    for (int i = 0; i < N; ++i)
-        v.push_back(i);
-    std::shuffle(v.begin(), v.end(), randomness);
-    C c(v.begin(), v.end());
-    c.sort(std::greater<T>());
-    assert(std::distance(c.begin(), c.end()) == N);
-    typename C::const_iterator j = c.begin();
-    for (int i = 0; i < N; ++i, ++j)
-        assert(*j == N-1-i);
+void test(int N) {
+  typedef typename C::value_type T;
+  typedef std::vector<T> V;
+  V v;
+  for (int i = 0; i < N; ++i)
+    v.push_back(i);
+  std::shuffle(v.begin(), v.end(), randomness);
+  C c(v.begin(), v.end());
+  c.sort(std::greater<T>());
+  assert(std::distance(c.begin(), c.end()) == N);
+  typename C::const_iterator j = c.begin();
+  for (int i = 0; i < N; ++i, ++j)
+    assert(*j == N - 1 - i);
 }
 
-int main(int, char**)
-{
-    for (int i = 0; i < 40; ++i)
-        test<std::forward_list<int> >(i);
+int main(int, char**) {
+  for (int i = 0; i < 40; ++i)
+    test<std::forward_list<int> >(i);
 #if TEST_STD_VER >= 11
-    for (int i = 0; i < 40; ++i)
-        test<std::forward_list<int, min_allocator<int>> >(i);
+  for (int i = 0; i < 40; ++i)
+    test<std::forward_list<int, min_allocator<int>> >(i);
 #endif
 
-    for (int i = 0; i < 40; ++i)
-        test_stable(i);
+  for (int i = 0; i < 40; ++i)
+    test_stable(i);
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/forwardlist/forwardlist.ops/splice_after_flist.pass.cpp b/libcxx/test/std/containers/sequences/forwardlist/forwardlist.ops/splice_after_flist.pass.cpp
index d5d95b96919ea..4c91d7397adf0 100644
--- a/libcxx/test/std/containers/sequences/forwardlist/forwardlist.ops/splice_after_flist.pass.cpp
+++ b/libcxx/test/std/containers/sequences/forwardlist/forwardlist.ops/splice_after_flist.pass.cpp
@@ -19,59 +19,52 @@
 #include "min_allocator.h"
 
 typedef int T;
-const T t1[] = {0, 1, 2, 3, 4, 5, 6, 7};
-const T t2[] = {10, 11, 12, 13, 14, 15};
+const T t1[]                 = {0, 1, 2, 3, 4, 5, 6, 7};
+const T t2[]                 = {10, 11, 12, 13, 14, 15};
 const std::ptrdiff_t size_t1 = std::end(t1) - std::begin(t1);
 const std::ptrdiff_t size_t2 = std::end(t2) - std::begin(t2);
 
 template <class C>
-void
-testd(const C& c, int p, int l)
-{
-    typename C::const_iterator i = c.begin();
-    int n1 = 0;
-    for (; n1 < p; ++n1, ++i)
-        assert(*i == t1[n1]);
-    for (int n2 = 0; n2 < l; ++n2, ++i)
-        assert(*i == t2[n2]);
-    for (; n1 < size_t1; ++n1, ++i)
-        assert(*i == t1[n1]);
-    assert(std::distance(c.begin(), c.end()) == size_t1 + l);
+void testd(const C& c, int p, int l) {
+  typename C::const_iterator i = c.begin();
+  int n1                       = 0;
+  for (; n1 < p; ++n1, ++i)
+    assert(*i == t1[n1]);
+  for (int n2 = 0; n2 < l; ++n2, ++i)
+    assert(*i == t2[n2]);
+  for (; n1 < size_t1; ++n1, ++i)
+    assert(*i == t1[n1]);
+  assert(std::distance(c.begin(), c.end()) == size_t1 + l);
 }
 
-int main(int, char**)
-{
-    {
+int main(int, char**) {
+  {
     // splicing different containers
     typedef std::forward_list<T> C;
-    for (int l = 0; l <= size_t2; ++l)
-    {
-        for (int p = 0; p <= size_t1; ++p)
-        {
-            C c1(std::begin(t1), std::end(t1));
-            C c2(t2, t2+l);
+    for (int l = 0; l <= size_t2; ++l) {
+      for (int p = 0; p <= size_t1; ++p) {
+        C c1(std::begin(t1), std::end(t1));
+        C c2(t2, t2 + l);
 
-            c1.splice_after(std::next(c1.cbefore_begin(), p), std::move(c2));
-            testd(c1, p, l);
-        }
-    }
+        c1.splice_after(std::next(c1.cbefore_begin(), p), std::move(c2));
+        testd(c1, p, l);
+      }
     }
+  }
 #if TEST_STD_VER >= 11
-    {
+  {
     // splicing different containers
     typedef std::forward_list<T, min_allocator<T>> C;
-    for (int l = 0; l <= size_t2; ++l)
-    {
-        for (int p = 0; p <= size_t1; ++p)
-        {
-            C c1(std::begin(t1), std::end(t1));
-            C c2(t2, t2+l);
+    for (int l = 0; l <= size_t2; ++l) {
+      for (int p = 0; p <= size_t1; ++p) {
+        C c1(std::begin(t1), std::end(t1));
+        C c2(t2, t2 + l);
 
-            c1.splice_after(std::next(c1.cbefore_begin(), p), std::move(c2));
-            testd(c1, p, l);
-        }
-    }
+        c1.splice_after(std::next(c1.cbefore_begin(), p), std::move(c2));
+        testd(c1, p, l);
+      }
     }
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/sequences/forwardlist/forwardlist.ops/splice_after_one.pass.cpp b/libcxx/test/std/containers/sequences/forwardlist/forwardlist.ops/splice_after_one.pass.cpp
index f54ff2ca1deaa..bb8bdea632547 100644
--- a/libcxx/test/std/containers/sequences/forwardlist/forwardlist.ops/splice_after_one.pass.cpp
+++ b/libcxx/test/std/containers/sequences/forwardlist/forwardlist.ops/splice_after_one.pass.cpp
@@ -19,123 +19,102 @@
 #include "min_allocator.h"
 
 typedef int T;
-const T t1[] = {0, 1, 2, 3, 4, 5, 6, 7};
-const T t2[] = {10, 11, 12};
+const T t1[]                 = {0, 1, 2, 3, 4, 5, 6, 7};
+const T t2[]                 = {10, 11, 12};
 const std::ptrdiff_t size_t1 = std::end(t1) - std::begin(t1);
 const std::ptrdiff_t size_t2 = std::end(t2) - std::begin(t2);
 
 template <class C>
-void
-testd(const C& c, int p, int f)
-{
-    typename C::const_iterator i = c.begin();
-    int n1 = 0;
-    for (; n1 < p; ++n1, ++i)
-        assert(*i == t1[n1]);
-    for (int n2 = f; n2 < f+1; ++n2, ++i)
-        assert(*i == t2[n2]);
-    for (; n1 < size_t1; ++n1, ++i)
-        assert(*i == t1[n1]);
-    assert(std::distance(c.begin(), c.end()) == size_t1 + 1);
+void testd(const C& c, int p, int f) {
+  typename C::const_iterator i = c.begin();
+  int n1                       = 0;
+  for (; n1 < p; ++n1, ++i)
+    assert(*i == t1[n1]);
+  for (int n2 = f; n2 < f + 1; ++n2, ++i)
+    assert(*i == t2[n2]);
+  for (; n1 < size_t1; ++n1, ++i)
+    assert(*i == t1[n1]);
+  assert(std::distance(c.begin(), c.end()) == size_t1 + 1);
 }
 
 template <class C>
-void
-tests(const C& c, int p, int f)
-{
-    typename C::const_iterator i = c.begin();
-    int n = 0;
-    if (p == f || p == f+1)
-    {
-        for (n = 0; n < size_t1; ++n, ++i)
-            assert(*i == t1[n]);
-    }
-    else if (p < f)
-    {
-        for (n = 0; n < p; ++n, ++i)
-            assert(*i == t1[n]);
-        for (n = f; n < f+1; ++n, ++i)
-            assert(*i == t1[n]);
-        for (n = p; n < f; ++n, ++i)
-            assert(*i == t1[n]);
-        for (n = f+1; n < size_t1; ++n, ++i)
-            assert(*i == t1[n]);
-    }
-    else // p > f+1
-    {
-        for (n = 0; n < f; ++n, ++i)
-            assert(*i == t1[n]);
-        for (n = f+1; n < p; ++n, ++i)
-            assert(*i == t1[n]);
-        for (n = f; n < f+1; ++n, ++i)
-            assert(*i == t1[n]);
-        for (n = p; n < size_t1; ++n, ++i)
-            assert(*i == t1[n]);
-    }
-    assert(std::distance(c.begin(), c.end()) == size_t1);
+void tests(const C& c, int p, int f) {
+  typename C::const_iterator i = c.begin();
+  int n                        = 0;
+  if (p == f || p == f + 1) {
+    for (n = 0; n < size_t1; ++n, ++i)
+      assert(*i == t1[n]);
+  } else if (p < f) {
+    for (n = 0; n < p; ++n, ++i)
+      assert(*i == t1[n]);
+    for (n = f; n < f + 1; ++n, ++i)
+      assert(*i == t1[n]);
+    for (n = p; n < f; ++n, ++i)
+      assert(*i == t1[n]);
+    for (n = f + 1; n < size_t1; ++n, ++i)
+      assert(*i == t1[n]);
+  } else // p > f+1
+  {
+    for (n = 0; n < f; ++n, ++i)
+      assert(*i == t1[n]);
+    for (n = f + 1; n < p; ++n, ++i)
+      assert(*i == t1[n]);
+    for (n = f; n < f + 1; ++n, ++i)
+      assert(*i == t1[n]);
+    for (n = p; n < size_t1; ++n, ++i)
+      assert(*i == t1[n]);
+  }
+  assert(std::distance(c.begin(), c.end()) == size_t1);
 }
 
-int main(int, char**)
-{
-    {
+int main(int, char**) {
+  {
     // splicing different containers
     typedef std::forward_list<T> C;
-    for (int f = 0; f <= size_t2-1; ++f)
-    {
-        for (int p = 0; p <= size_t1; ++p)
-        {
-            C c1(std::begin(t1), std::end(t1));
-            C c2(std::begin(t2), std::end(t2));
+    for (int f = 0; f <= size_t2 - 1; ++f) {
+      for (int p = 0; p <= size_t1; ++p) {
+        C c1(std::begin(t1), std::end(t1));
+        C c2(std::begin(t2), std::end(t2));
 
-            c1.splice_after(std::next(c1.cbefore_begin(), p), std::move(c2),
-                  std::next(c2.cbefore_begin(), f));
-            testd(c1, p, f);
-        }
+        c1.splice_after(std::next(c1.cbefore_begin(), p), std::move(c2), std::next(c2.cbefore_begin(), f));
+        testd(c1, p, f);
+      }
     }
 
     // splicing within same container
-    for (int f = 0; f <= size_t1-1; ++f)
-    {
-        for (int p = 0; p <= size_t1; ++p)
-        {
-            C c1(std::begin(t1), std::end(t1));
+    for (int f = 0; f <= size_t1 - 1; ++f) {
+      for (int p = 0; p <= size_t1; ++p) {
+        C c1(std::begin(t1), std::end(t1));
 
-            c1.splice_after(std::next(c1.cbefore_begin(), p), std::move(c1),
-                  std::next(c1.cbefore_begin(), f));
-            tests(c1, p, f);
-        }
-    }
+        c1.splice_after(std::next(c1.cbefore_begin(), p), std::move(c1), std::next(c1.cbefore_begin(), f));
+        tests(c1, p, f);
+      }
     }
+  }
 #if TEST_STD_VER >= 11
-    {
+  {
     // splicing different containers
     typedef std::forward_list<T, min_allocator<T>> C;
-    for (int f = 0; f <= size_t2-1; ++f)
-    {
-        for (int p = 0; p <= size_t1; ++p)
-        {
-            C c1(std::begin(t1), std::end(t1));
-            C c2(std::begin(t2), std::end(t2));
+    for (int f = 0; f <= size_t2 - 1; ++f) {
+      for (int p = 0; p <= size_t1; ++p) {
+        C c1(std::begin(t1), std::end(t1));
+        C c2(std::begin(t2), std::end(t2));
 
-            c1.splice_after(std::next(c1.cbefore_begin(), p), std::move(c2),
-                  std::next(c2.cbefore_begin(), f));
-            testd(c1, p, f);
-        }
+        c1.splice_after(std::next(c1.cbefore_begin(), p), std::move(c2), std::next(c2.cbefore_begin(), f));
+        testd(c1, p, f);
+      }
     }
 
     // splicing within same container
-    for (int f = 0; f <= size_t1-1; ++f)
-    {
-        for (int p = 0; p <= size_t1; ++p)
-        {
-            C c1(std::begin(t1), std::end(t1));
+    for (int f = 0; f <= size_t1 - 1; ++f) {
+      for (int p = 0; p <= size_t1; ++p) {
+        C c1(std::begin(t1), std::end(t1));
 
-            c1.splice_after(std::next(c1.cbefore_begin(), p), std::move(c1),
-                  std::next(c1.cbefore_begin(), f));
-            tests(c1, p, f);
-        }
-    }
+        c1.splice_after(std::next(c1.cbefore_begin(), p), std::move(c1), std::next(c1.cbefore_begin(), f));
+        tests(c1, p, f);
+      }
     }
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/sequences/forwardlist/forwardlist.ops/splice_after_range.pass.cpp b/libcxx/test/std/containers/sequences/forwardlist/forwardlist.ops/splice_after_range.pass.cpp
index f2ab6e5faa19d..99b3ed1c7836b 100644
--- a/libcxx/test/std/containers/sequences/forwardlist/forwardlist.ops/splice_after_range.pass.cpp
+++ b/libcxx/test/std/containers/sequences/forwardlist/forwardlist.ops/splice_after_range.pass.cpp
@@ -20,152 +20,141 @@
 #include "min_allocator.h"
 
 typedef std::ptrdiff_t T;
-const T t1[] = {0, 1, 2, 3, 4, 5, 6, 7};
-const T t2[] = {10, 11, 12, 13, 14, 15};
+const T t1[]                 = {0, 1, 2, 3, 4, 5, 6, 7};
+const T t2[]                 = {10, 11, 12, 13, 14, 15};
 const std::ptrdiff_t size_t1 = std::end(t1) - std::begin(t1);
 const std::ptrdiff_t size_t2 = std::end(t2) - std::begin(t2);
 
 template <class C>
-void
-testd(const C& c, std::ptrdiff_t p, ptrdiff_t f, ptrdiff_t l)
-{
-    typename C::const_iterator i = c.begin();
-    std::ptrdiff_t n1 = 0;
-    for (; n1 < p; ++n1, ++i)
-        assert(*i == t1[n1]);
-    for (std::ptrdiff_t n2 = f; n2 < l-1; ++n2, ++i)
-        assert(*i == t2[n2]);
-    for (; n1 < size_t1; ++n1, ++i)
-        assert(*i == t1[n1]);
-    assert(std::distance(c.begin(), c.end()) == size_t1 + (l > f+1 ? l-1-f : 0));
+void testd(const C& c, std::ptrdiff_t p, ptrdiff_t f, ptrdiff_t l) {
+  typename C::const_iterator i = c.begin();
+  std::ptrdiff_t n1            = 0;
+  for (; n1 < p; ++n1, ++i)
+    assert(*i == t1[n1]);
+  for (std::ptrdiff_t n2 = f; n2 < l - 1; ++n2, ++i)
+    assert(*i == t2[n2]);
+  for (; n1 < size_t1; ++n1, ++i)
+    assert(*i == t1[n1]);
+  assert(std::distance(c.begin(), c.end()) == size_t1 + (l > f + 1 ? l - 1 - f : 0));
 }
 
 template <class C>
-void
-tests(const C& c, std::ptrdiff_t p, ptrdiff_t f, ptrdiff_t l)
-{
-    typename C::const_iterator i = c.begin();
-    std::ptrdiff_t n = 0;
-    std::ptrdiff_t d = l > f+1 ? l-1-f : 0;
-    if (d == 0 || p == f)
-    {
-        for (n = 0; n < size_t1; ++n, ++i)
-            assert(*i == t1[n]);
-    }
-    else if (p < f)
-    {
-        for (n = 0; n < p; ++n, ++i)
-            assert(*i == t1[n]);
-        for (n = f; n < l-1; ++n, ++i)
-            assert(*i == t1[n]);
-        for (n = p; n < f; ++n, ++i)
-            assert(*i == t1[n]);
-        for (n = l-1; n < size_t1; ++n, ++i)
-            assert(*i == t1[n]);
-    }
-    else // p > f
-    {
-        for (n = 0; n < f; ++n, ++i)
-            assert(*i == t1[n]);
-        for (n = l-1; n < p; ++n, ++i)
-            assert(*i == t1[n]);
-        for (n = f; n < l-1; ++n, ++i)
-            assert(*i == t1[n]);
-        for (n = p; n < size_t1; ++n, ++i)
-            assert(*i == t1[n]);
-    }
-    assert(std::distance(c.begin(), c.end()) == size_t1);
+void tests(const C& c, std::ptrdiff_t p, ptrdiff_t f, ptrdiff_t l) {
+  typename C::const_iterator i = c.begin();
+  std::ptrdiff_t n             = 0;
+  std::ptrdiff_t d             = l > f + 1 ? l - 1 - f : 0;
+  if (d == 0 || p == f) {
+    for (n = 0; n < size_t1; ++n, ++i)
+      assert(*i == t1[n]);
+  } else if (p < f) {
+    for (n = 0; n < p; ++n, ++i)
+      assert(*i == t1[n]);
+    for (n = f; n < l - 1; ++n, ++i)
+      assert(*i == t1[n]);
+    for (n = p; n < f; ++n, ++i)
+      assert(*i == t1[n]);
+    for (n = l - 1; n < size_t1; ++n, ++i)
+      assert(*i == t1[n]);
+  } else // p > f
+  {
+    for (n = 0; n < f; ++n, ++i)
+      assert(*i == t1[n]);
+    for (n = l - 1; n < p; ++n, ++i)
+      assert(*i == t1[n]);
+    for (n = f; n < l - 1; ++n, ++i)
+      assert(*i == t1[n]);
+    for (n = p; n < size_t1; ++n, ++i)
+      assert(*i == t1[n]);
+  }
+  assert(std::distance(c.begin(), c.end()) == size_t1);
 }
 
-int main(int, char**)
-{
-    {
+int main(int, char**) {
+  {
     // splicing different containers
     typedef std::forward_list<T> C;
-    for (std::ptrdiff_t f = 0; f <= size_t2+1; ++f)
-    {
-        for (std::ptrdiff_t l = f; l <= size_t2+1; ++l)
-        {
-            for (std::ptrdiff_t p = 0; p <= size_t1; ++p)
-            {
-                C c1(std::begin(t1), std::end(t1));
-                C c2(std::begin(t2), std::end(t2));
-
-                c1.splice_after(std::next(c1.cbefore_begin(), p), std::move(c2),
-                      std::next(c2.cbefore_begin(), f), std::next(c2.cbefore_begin(), l));
-                testd(c1, p, f, l);
-            }
+    for (std::ptrdiff_t f = 0; f <= size_t2 + 1; ++f) {
+      for (std::ptrdiff_t l = f; l <= size_t2 + 1; ++l) {
+        for (std::ptrdiff_t p = 0; p <= size_t1; ++p) {
+          C c1(std::begin(t1), std::end(t1));
+          C c2(std::begin(t2), std::end(t2));
+
+          c1.splice_after(std::next(c1.cbefore_begin(), p),
+                          std::move(c2),
+                          std::next(c2.cbefore_begin(), f),
+                          std::next(c2.cbefore_begin(), l));
+          testd(c1, p, f, l);
         }
+      }
     }
 
     // splicing within same container
-    for (std::ptrdiff_t f = 0; f <= size_t1+1; ++f)
-    {
-        for (std::ptrdiff_t l = f; l <= size_t1; ++l)
-        {
-            for (std::ptrdiff_t p = 0; p <= f; ++p)
-            {
-                C c1(std::begin(t1), std::end(t1));
-
-                c1.splice_after(std::next(c1.cbefore_begin(), p), std::move(c1),
-                      std::next(c1.cbefore_begin(), f), std::next(c1.cbefore_begin(), l));
-                tests(c1, p, f, l);
-            }
-            for (std::ptrdiff_t p = l; p <= size_t1; ++p)
-            {
-                C c1(std::begin(t1), std::end(t1));
-
-                c1.splice_after(std::next(c1.cbefore_begin(), p), std::move(c1),
-                      std::next(c1.cbefore_begin(), f), std::next(c1.cbefore_begin(), l));
-                tests(c1, p, f, l);
-            }
+    for (std::ptrdiff_t f = 0; f <= size_t1 + 1; ++f) {
+      for (std::ptrdiff_t l = f; l <= size_t1; ++l) {
+        for (std::ptrdiff_t p = 0; p <= f; ++p) {
+          C c1(std::begin(t1), std::end(t1));
+
+          c1.splice_after(std::next(c1.cbefore_begin(), p),
+                          std::move(c1),
+                          std::next(c1.cbefore_begin(), f),
+                          std::next(c1.cbefore_begin(), l));
+          tests(c1, p, f, l);
         }
+        for (std::ptrdiff_t p = l; p <= size_t1; ++p) {
+          C c1(std::begin(t1), std::end(t1));
+
+          c1.splice_after(std::next(c1.cbefore_begin(), p),
+                          std::move(c1),
+                          std::next(c1.cbefore_begin(), f),
+                          std::next(c1.cbefore_begin(), l));
+          tests(c1, p, f, l);
+        }
+      }
     }
-    }
+  }
 #if TEST_STD_VER >= 11
-    {
+  {
     // splicing different containers
     typedef std::forward_list<T, min_allocator<T>> C;
-    for (std::ptrdiff_t f = 0; f <= size_t2+1; ++f)
-    {
-        for (std::ptrdiff_t l = f; l <= size_t2+1; ++l)
-        {
-            for (std::ptrdiff_t p = 0; p <= size_t1; ++p)
-            {
-                C c1(std::begin(t1), std::end(t1));
-                C c2(std::begin(t2), std::end(t2));
-
-                c1.splice_after(std::next(c1.cbefore_begin(), p), std::move(c2),
-                      std::next(c2.cbefore_begin(), f), std::next(c2.cbefore_begin(), l));
-                testd(c1, p, f, l);
-            }
+    for (std::ptrdiff_t f = 0; f <= size_t2 + 1; ++f) {
+      for (std::ptrdiff_t l = f; l <= size_t2 + 1; ++l) {
+        for (std::ptrdiff_t p = 0; p <= size_t1; ++p) {
+          C c1(std::begin(t1), std::end(t1));
+          C c2(std::begin(t2), std::end(t2));
+
+          c1.splice_after(std::next(c1.cbefore_begin(), p),
+                          std::move(c2),
+                          std::next(c2.cbefore_begin(), f),
+                          std::next(c2.cbefore_begin(), l));
+          testd(c1, p, f, l);
         }
+      }
     }
 
     // splicing within same container
-    for (std::ptrdiff_t f = 0; f <= size_t1+1; ++f)
-    {
-        for (std::ptrdiff_t l = f; l <= size_t1; ++l)
-        {
-            for (std::ptrdiff_t p = 0; p <= f; ++p)
-            {
-                C c1(std::begin(t1), std::end(t1));
-
-                c1.splice_after(std::next(c1.cbefore_begin(), p), std::move(c1),
-                      std::next(c1.cbefore_begin(), f), std::next(c1.cbefore_begin(), l));
-                tests(c1, p, f, l);
-            }
-            for (std::ptrdiff_t p = l; p <= size_t1; ++p)
-            {
-                C c1(std::begin(t1), std::end(t1));
-
-                c1.splice_after(std::next(c1.cbefore_begin(), p), std::move(c1),
-                      std::next(c1.cbefore_begin(), f), std::next(c1.cbefore_begin(), l));
-                tests(c1, p, f, l);
-            }
+    for (std::ptrdiff_t f = 0; f <= size_t1 + 1; ++f) {
+      for (std::ptrdiff_t l = f; l <= size_t1; ++l) {
+        for (std::ptrdiff_t p = 0; p <= f; ++p) {
+          C c1(std::begin(t1), std::end(t1));
+
+          c1.splice_after(std::next(c1.cbefore_begin(), p),
+                          std::move(c1),
+                          std::next(c1.cbefore_begin(), f),
+                          std::next(c1.cbefore_begin(), l));
+          tests(c1, p, f, l);
         }
+        for (std::ptrdiff_t p = l; p <= size_t1; ++p) {
+          C c1(std::begin(t1), std::end(t1));
+
+          c1.splice_after(std::next(c1.cbefore_begin(), p),
+                          std::move(c1),
+                          std::next(c1.cbefore_begin(), f),
+                          std::next(c1.cbefore_begin(), l));
+          tests(c1, p, f, l);
+        }
+      }
     }
-    }
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/sequences/forwardlist/forwardlist.ops/unique.pass.cpp b/libcxx/test/std/containers/sequences/forwardlist/forwardlist.ops/unique.pass.cpp
index 793fcf382da43..ebd1a79cdb4bc 100644
--- a/libcxx/test/std/containers/sequences/forwardlist/forwardlist.ops/unique.pass.cpp
+++ b/libcxx/test/std/containers/sequences/forwardlist/forwardlist.ops/unique.pass.cpp
@@ -19,118 +19,116 @@
 #include "min_allocator.h"
 
 template <class L>
-void do_unique(L &l, typename L::size_type expected)
-{
-    typename L::size_type old_size = std::distance(l.begin(), l.end());
+void do_unique(L& l, typename L::size_type expected) {
+  typename L::size_type old_size = std::distance(l.begin(), l.end());
 #if TEST_STD_VER > 17
-    ASSERT_SAME_TYPE(decltype(l.unique()), typename L::size_type);
-    assert(l.unique() == expected);
+  ASSERT_SAME_TYPE(decltype(l.unique()), typename L::size_type);
+  assert(l.unique() == expected);
 #else
-    ASSERT_SAME_TYPE(decltype(l.unique()), void);
-    l.unique();
+  ASSERT_SAME_TYPE(decltype(l.unique()), void);
+  l.unique();
 #endif
-    assert(old_size - std::distance(l.begin(), l.end()) == expected);
+  assert(old_size - std::distance(l.begin(), l.end()) == expected);
 }
 
-int main(int, char**)
-{
-    {
-        typedef int T;
-        typedef std::forward_list<T> C;
-        const T t1[] = {0, 5, 5, 0, 0, 0, 5};
-        const T t2[] = {0, 5, 0, 5};
-        C c1(std::begin(t1), std::end(t1));
-        C c2(std::begin(t2), std::end(t2));
-        do_unique(c1, 3);
-        assert(c1 == c2);
-    }
-    {
-        typedef int T;
-        typedef std::forward_list<T> C;
-        const T t1[] = {0, 0, 0, 0};
-        const T t2[] = {0};
-        C c1(std::begin(t1), std::end(t1));
-        C c2(std::begin(t2), std::end(t2));
-        do_unique(c1, 3);
-        assert(c1 == c2);
-    }
-    {
-        typedef int T;
-        typedef std::forward_list<T> C;
-        const T t1[] = {5, 5, 5};
-        const T t2[] = {5};
-        C c1(std::begin(t1), std::end(t1));
-        C c2(std::begin(t2), std::end(t2));
-        do_unique(c1, 2);
-        assert(c1 == c2);
-    }
-    {
-        typedef int T;
-        typedef std::forward_list<T> C;
-        C c1;
-        C c2;
-        do_unique(c1, 0);
-        assert(c1 == c2);
-    }
-    {
-        typedef int T;
-        typedef std::forward_list<T> C;
-        const T t1[] = {5, 5, 5, 0};
-        const T t2[] = {5, 0};
-        C c1(std::begin(t1), std::end(t1));
-        C c2(std::begin(t2), std::end(t2));
-        do_unique(c1, 2);
-        assert(c1 == c2);
-    }
+int main(int, char**) {
+  {
+    typedef int T;
+    typedef std::forward_list<T> C;
+    const T t1[] = {0, 5, 5, 0, 0, 0, 5};
+    const T t2[] = {0, 5, 0, 5};
+    C c1(std::begin(t1), std::end(t1));
+    C c2(std::begin(t2), std::end(t2));
+    do_unique(c1, 3);
+    assert(c1 == c2);
+  }
+  {
+    typedef int T;
+    typedef std::forward_list<T> C;
+    const T t1[] = {0, 0, 0, 0};
+    const T t2[] = {0};
+    C c1(std::begin(t1), std::end(t1));
+    C c2(std::begin(t2), std::end(t2));
+    do_unique(c1, 3);
+    assert(c1 == c2);
+  }
+  {
+    typedef int T;
+    typedef std::forward_list<T> C;
+    const T t1[] = {5, 5, 5};
+    const T t2[] = {5};
+    C c1(std::begin(t1), std::end(t1));
+    C c2(std::begin(t2), std::end(t2));
+    do_unique(c1, 2);
+    assert(c1 == c2);
+  }
+  {
+    typedef int T;
+    typedef std::forward_list<T> C;
+    C c1;
+    C c2;
+    do_unique(c1, 0);
+    assert(c1 == c2);
+  }
+  {
+    typedef int T;
+    typedef std::forward_list<T> C;
+    const T t1[] = {5, 5, 5, 0};
+    const T t2[] = {5, 0};
+    C c1(std::begin(t1), std::end(t1));
+    C c2(std::begin(t2), std::end(t2));
+    do_unique(c1, 2);
+    assert(c1 == c2);
+  }
 #if TEST_STD_VER >= 11
-    {
-        typedef int T;
-        typedef std::forward_list<T, min_allocator<T>> C;
-        const T t1[] = {0, 5, 5, 0, 0, 0, 5};
-        const T t2[] = {0, 5, 0, 5};
-        C c1(std::begin(t1), std::end(t1));
-        C c2(std::begin(t2), std::end(t2));
-        do_unique(c1, 3);
-        assert(c1 == c2);
-    }
-    {
-        typedef int T;
-        typedef std::forward_list<T, min_allocator<T>> C;
-        const T t1[] = {0, 0, 0, 0};
-        const T t2[] = {0};
-        C c1(std::begin(t1), std::end(t1));
-        C c2(std::begin(t2), std::end(t2));
-        do_unique(c1, 3);
-        assert(c1 == c2);
-    }
-    {
-        typedef int T;
-        typedef std::forward_list<T, min_allocator<T>> C;
-        const T t1[] = {5, 5, 5};
-        const T t2[] = {5};
-        C c1(std::begin(t1), std::end(t1));
-        C c2(std::begin(t2), std::end(t2));
-        do_unique(c1, 2);
-        assert(c1 == c2);
-    }
-    {
-        typedef int T;
-        typedef std::forward_list<T, min_allocator<T>> C;
-        C c1;
-        C c2;
-        do_unique(c1, 0);
-        assert(c1 == c2);
-    }
-    {
-        typedef int T;
-        typedef std::forward_list<T, min_allocator<T>> C;
-        const T t1[] = {5, 5, 5, 0};
-        const T t2[] = {5, 0};
-        C c1(std::begin(t1), std::end(t1));
-        C c2(std::begin(t2), std::end(t2));
-        do_unique(c1, 2);
-        assert(c1 == c2);
-    }
+  {
+    typedef int T;
+    typedef std::forward_list<T, min_allocator<T>> C;
+    const T t1[] = {0, 5, 5, 0, 0, 0, 5};
+    const T t2[] = {0, 5, 0, 5};
+    C c1(std::begin(t1), std::end(t1));
+    C c2(std::begin(t2), std::end(t2));
+    do_unique(c1, 3);
+    assert(c1 == c2);
+  }
+  {
+    typedef int T;
+    typedef std::forward_list<T, min_allocator<T>> C;
+    const T t1[] = {0, 0, 0, 0};
+    const T t2[] = {0};
+    C c1(std::begin(t1), std::end(t1));
+    C c2(std::begin(t2), std::end(t2));
+    do_unique(c1, 3);
+    assert(c1 == c2);
+  }
+  {
+    typedef int T;
+    typedef std::forward_list<T, min_allocator<T>> C;
+    const T t1[] = {5, 5, 5};
+    const T t2[] = {5};
+    C c1(std::begin(t1), std::end(t1));
+    C c2(std::begin(t2), std::end(t2));
+    do_unique(c1, 2);
+    assert(c1 == c2);
+  }
+  {
+    typedef int T;
+    typedef std::forward_list<T, min_allocator<T>> C;
+    C c1;
+    C c2;
+    do_unique(c1, 0);
+    assert(c1 == c2);
+  }
+  {
+    typedef int T;
+    typedef std::forward_list<T, min_allocator<T>> C;
+    const T t1[] = {5, 5, 5, 0};
+    const T t2[] = {5, 0};
+    C c1(std::begin(t1), std::end(t1));
+    C c2(std::begin(t2), std::end(t2));
+    do_unique(c1, 2);
+    assert(c1 == c2);
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/sequences/forwardlist/forwardlist.ops/unique_pred.pass.cpp b/libcxx/test/std/containers/sequences/forwardlist/forwardlist.ops/unique_pred.pass.cpp
index 3216717811d86..408cbf6ae9c20 100644
--- a/libcxx/test/std/containers/sequences/forwardlist/forwardlist.ops/unique_pred.pass.cpp
+++ b/libcxx/test/std/containers/sequences/forwardlist/forwardlist.ops/unique_pred.pass.cpp
@@ -20,20 +20,18 @@
 #include "min_allocator.h"
 
 template <class L, class Predicate>
-void do_unique(L &l, Predicate pred, typename L::size_type expected)
-{
-    typename L::size_type old_size = std::distance(l.begin(), l.end());
+void do_unique(L& l, Predicate pred, typename L::size_type expected) {
+  typename L::size_type old_size = std::distance(l.begin(), l.end());
 #if TEST_STD_VER > 17
-    ASSERT_SAME_TYPE(decltype(l.unique(pred)), typename L::size_type);
-    assert(l.unique(pred) == expected);
+  ASSERT_SAME_TYPE(decltype(l.unique(pred)), typename L::size_type);
+  assert(l.unique(pred) == expected);
 #else
-    ASSERT_SAME_TYPE(decltype(l.unique(pred)), void);
-    l.unique(pred);
+  ASSERT_SAME_TYPE(decltype(l.unique(pred)), void);
+  l.unique(pred);
 #endif
-    assert(old_size - std::distance(l.begin(), l.end()) == expected);
+  assert(old_size - std::distance(l.begin(), l.end()) == expected);
 }
 
-
 struct PredLWG526 {
   PredLWG526(int i) : i_(i) {}
   ~PredLWG526() { i_ = -32767; }
@@ -43,126 +41,120 @@ struct PredLWG526 {
   int i_;
 };
 
+bool g(int x, int y) { return x == y; }
 
-bool g(int x, int y)
-{
-    return x == y;
-}
-
-int main(int, char**)
-{
-    {
-        typedef int T;
-        typedef std::forward_list<T> C;
-        const T t1[] = {0, 5, 5, 0, 0, 0, 5};
-        const T t2[] = {0, 5, 0, 5};
-        C c1(std::begin(t1), std::end(t1));
-        C c2(std::begin(t2), std::end(t2));
-        do_unique(c1, g, 3);
-        assert(c1 == c2);
-    }
-    {
-        typedef int T;
-        typedef std::forward_list<T> C;
-        const T t1[] = {0, 0, 0, 0};
-        const T t2[] = {0};
-        C c1(std::begin(t1), std::end(t1));
-        C c2(std::begin(t2), std::end(t2));
-        do_unique(c1, g, 3);
-        assert(c1 == c2);
-    }
-    {
-        typedef int T;
-        typedef std::forward_list<T> C;
-        const T t1[] = {5, 5, 5};
-        const T t2[] = {5};
-        C c1(std::begin(t1), std::end(t1));
-        C c2(std::begin(t2), std::end(t2));
-        do_unique(c1, g, 2);
-        assert(c1 == c2);
-    }
-    {
-        typedef int T;
-        typedef std::forward_list<T> C;
-        C c1;
-        C c2;
-        do_unique(c1, g, 0);
-        assert(c1 == c2);
-    }
-    {
-        typedef int T;
-        typedef std::forward_list<T> C;
-        const T t1[] = {5, 5, 5, 0};
-        const T t2[] = {5, 0};
-        C c1(std::begin(t1), std::end(t1));
-        C c2(std::begin(t2), std::end(t2));
-        do_unique(c1, g, 2);
-        assert(c1 == c2);
-    }
+int main(int, char**) {
+  {
+    typedef int T;
+    typedef std::forward_list<T> C;
+    const T t1[] = {0, 5, 5, 0, 0, 0, 5};
+    const T t2[] = {0, 5, 0, 5};
+    C c1(std::begin(t1), std::end(t1));
+    C c2(std::begin(t2), std::end(t2));
+    do_unique(c1, g, 3);
+    assert(c1 == c2);
+  }
+  {
+    typedef int T;
+    typedef std::forward_list<T> C;
+    const T t1[] = {0, 0, 0, 0};
+    const T t2[] = {0};
+    C c1(std::begin(t1), std::end(t1));
+    C c2(std::begin(t2), std::end(t2));
+    do_unique(c1, g, 3);
+    assert(c1 == c2);
+  }
+  {
+    typedef int T;
+    typedef std::forward_list<T> C;
+    const T t1[] = {5, 5, 5};
+    const T t2[] = {5};
+    C c1(std::begin(t1), std::end(t1));
+    C c2(std::begin(t2), std::end(t2));
+    do_unique(c1, g, 2);
+    assert(c1 == c2);
+  }
+  {
+    typedef int T;
+    typedef std::forward_list<T> C;
+    C c1;
+    C c2;
+    do_unique(c1, g, 0);
+    assert(c1 == c2);
+  }
+  {
+    typedef int T;
+    typedef std::forward_list<T> C;
+    const T t1[] = {5, 5, 5, 0};
+    const T t2[] = {5, 0};
+    C c1(std::begin(t1), std::end(t1));
+    C c2(std::begin(t2), std::end(t2));
+    do_unique(c1, g, 2);
+    assert(c1 == c2);
+  }
 
-    { // LWG issue #526
+  { // LWG issue #526
     int a1[] = {1, 1, 1, 2, 3, 5, 2, 11};
-    int a2[] = {1,       2, 3, 5, 2, 11};
+    int a2[] = {1, 2, 3, 5, 2, 11};
     std::forward_list<PredLWG526> c1(a1, a1 + 8);
     do_unique(c1, std::ref(c1.front()), 2);
-    for (std::size_t i = 0; i < 6; ++i)
-    {
-        assert(!c1.empty());
-        assert(c1.front() == a2[i]);
-        c1.pop_front();
+    for (std::size_t i = 0; i < 6; ++i) {
+      assert(!c1.empty());
+      assert(c1.front() == a2[i]);
+      c1.pop_front();
     }
     assert(c1.empty());
-    }
+  }
 
 #if TEST_STD_VER >= 11
-    {
-        typedef int T;
-        typedef std::forward_list<T, min_allocator<T>> C;
-        const T t1[] = {0, 5, 5, 0, 0, 0, 5};
-        const T t2[] = {0, 5, 0, 5};
-        C c1(std::begin(t1), std::end(t1));
-        C c2(std::begin(t2), std::end(t2));
-        do_unique(c1, g, 3);
-        assert(c1 == c2);
-    }
-    {
-        typedef int T;
-        typedef std::forward_list<T, min_allocator<T>> C;
-        const T t1[] = {0, 0, 0, 0};
-        const T t2[] = {0};
-        C c1(std::begin(t1), std::end(t1));
-        C c2(std::begin(t2), std::end(t2));
-        do_unique(c1, g, 3);
-        assert(c1 == c2);
-    }
-    {
-        typedef int T;
-        typedef std::forward_list<T, min_allocator<T>> C;
-        const T t1[] = {5, 5, 5};
-        const T t2[] = {5};
-        C c1(std::begin(t1), std::end(t1));
-        C c2(std::begin(t2), std::end(t2));
-        do_unique(c1, g, 2);
-        assert(c1 == c2);
-    }
-    {
-        typedef int T;
-        typedef std::forward_list<T, min_allocator<T>> C;
-        C c1;
-        C c2;
-        do_unique(c1, g, 0);
-        assert(c1 == c2);
-    }
-    {
-        typedef int T;
-        typedef std::forward_list<T, min_allocator<T>> C;
-        const T t1[] = {5, 5, 5, 0};
-        const T t2[] = {5, 0};
-        C c1(std::begin(t1), std::end(t1));
-        C c2(std::begin(t2), std::end(t2));
-        do_unique(c1, g, 2);
-        assert(c1 == c2);
-    }
+  {
+    typedef int T;
+    typedef std::forward_list<T, min_allocator<T>> C;
+    const T t1[] = {0, 5, 5, 0, 0, 0, 5};
+    const T t2[] = {0, 5, 0, 5};
+    C c1(std::begin(t1), std::end(t1));
+    C c2(std::begin(t2), std::end(t2));
+    do_unique(c1, g, 3);
+    assert(c1 == c2);
+  }
+  {
+    typedef int T;
+    typedef std::forward_list<T, min_allocator<T>> C;
+    const T t1[] = {0, 0, 0, 0};
+    const T t2[] = {0};
+    C c1(std::begin(t1), std::end(t1));
+    C c2(std::begin(t2), std::end(t2));
+    do_unique(c1, g, 3);
+    assert(c1 == c2);
+  }
+  {
+    typedef int T;
+    typedef std::forward_list<T, min_allocator<T>> C;
+    const T t1[] = {5, 5, 5};
+    const T t2[] = {5};
+    C c1(std::begin(t1), std::end(t1));
+    C c2(std::begin(t2), std::end(t2));
+    do_unique(c1, g, 2);
+    assert(c1 == c2);
+  }
+  {
+    typedef int T;
+    typedef std::forward_list<T, min_allocator<T>> C;
+    C c1;
+    C c2;
+    do_unique(c1, g, 0);
+    assert(c1 == c2);
+  }
+  {
+    typedef int T;
+    typedef std::forward_list<T, min_allocator<T>> C;
+    const T t1[] = {5, 5, 5, 0};
+    const T t2[] = {5, 0};
+    C c1(std::begin(t1), std::end(t1));
+    C c2(std::begin(t2), std::end(t2));
+    do_unique(c1, g, 2);
+    assert(c1 == c2);
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/sequences/forwardlist/forwardlist.spec/equal.pass.cpp b/libcxx/test/std/containers/sequences/forwardlist/forwardlist.spec/equal.pass.cpp
index b21035f7dd746..ef6b72ee360a9 100644
--- a/libcxx/test/std/containers/sequences/forwardlist/forwardlist.spec/equal.pass.cpp
+++ b/libcxx/test/std/containers/sequences/forwardlist/forwardlist.spec/equal.pass.cpp
@@ -25,36 +25,33 @@
 #include "min_allocator.h"
 
 template <class C>
-void test(int N, int M)
-{
-    C c1;
-    for (int i = 0; i < N; ++i)
-        c1.push_front(i);
-    C c2;
-    for (int i = 0; i < M; ++i)
-        c2.push_front(i);
-    if (N == M)
-        assert(c1 == c2);
-    else
-        assert(c1 != c2);
-    c2 = c1;
+void test(int N, int M) {
+  C c1;
+  for (int i = 0; i < N; ++i)
+    c1.push_front(i);
+  C c2;
+  for (int i = 0; i < M; ++i)
+    c2.push_front(i);
+  if (N == M)
     assert(c1 == c2);
-    if (N > 0)
-    {
-        c2.front() = N+1;
-        assert(c1 != c2);
-    }
+  else
+    assert(c1 != c2);
+  c2 = c1;
+  assert(c1 == c2);
+  if (N > 0) {
+    c2.front() = N + 1;
+    assert(c1 != c2);
+  }
 }
 
-int main(int, char**)
-{
-    for (int i = 0; i < 10; ++i)
-        for (int j = 0; j < 10; ++j)
-            test<std::forward_list<int> >(i, j);
+int main(int, char**) {
+  for (int i = 0; i < 10; ++i)
+    for (int j = 0; j < 10; ++j)
+      test<std::forward_list<int> >(i, j);
 #if TEST_STD_VER >= 11
-    for (int i = 0; i < 10; ++i)
-        for (int j = 0; j < 10; ++j)
-            test<std::forward_list<int, min_allocator<int>> >(i, j);
+  for (int i = 0; i < 10; ++i)
+    for (int j = 0; j < 10; ++j)
+      test<std::forward_list<int, min_allocator<int>> >(i, j);
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/sequences/forwardlist/forwardlist.spec/member_swap.pass.cpp b/libcxx/test/std/containers/sequences/forwardlist/forwardlist.spec/member_swap.pass.cpp
index e7eea2e87bfec..e50f9e6e9e473 100644
--- a/libcxx/test/std/containers/sequences/forwardlist/forwardlist.spec/member_swap.pass.cpp
+++ b/libcxx/test/std/containers/sequences/forwardlist/forwardlist.spec/member_swap.pass.cpp
@@ -18,244 +18,243 @@
 #include "test_allocator.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef int T;
-        typedef test_allocator<T> A;
-        typedef std::forward_list<T, A> C;
-        const T t1[] = {0, 1, 2, 3, 4, 5};
-        C c1(std::begin(t1), std::end(t1), A(1, 1));
-        const T t2[] = {10, 11, 12};
-        C c2(std::begin(t2), std::end(t2), A(1, 2));
-        c1.swap(c2);
+int main(int, char**) {
+  {
+    typedef int T;
+    typedef test_allocator<T> A;
+    typedef std::forward_list<T, A> C;
+    const T t1[] = {0, 1, 2, 3, 4, 5};
+    C c1(std::begin(t1), std::end(t1), A(1, 1));
+    const T t2[] = {10, 11, 12};
+    C c2(std::begin(t2), std::end(t2), A(1, 2));
+    c1.swap(c2);
 
-        assert(std::distance(c1.begin(), c1.end()) == 3);
-        assert(*std::next(c1.begin(), 0) == 10);
-        assert(*std::next(c1.begin(), 1) == 11);
-        assert(*std::next(c1.begin(), 2) == 12);
-        assert(c1.get_allocator().get_id() == 1);
+    assert(std::distance(c1.begin(), c1.end()) == 3);
+    assert(*std::next(c1.begin(), 0) == 10);
+    assert(*std::next(c1.begin(), 1) == 11);
+    assert(*std::next(c1.begin(), 2) == 12);
+    assert(c1.get_allocator().get_id() == 1);
 
-        assert(std::distance(c2.begin(), c2.end()) == 6);
-        assert(*std::next(c2.begin(), 0) == 0);
-        assert(*std::next(c2.begin(), 1) == 1);
-        assert(*std::next(c2.begin(), 2) == 2);
-        assert(*std::next(c2.begin(), 3) == 3);
-        assert(*std::next(c2.begin(), 4) == 4);
-        assert(*std::next(c2.begin(), 5) == 5);
-        assert(c2.get_allocator().get_id() == 2);
-    }
-    {
-        typedef int T;
-        typedef test_allocator<T> A;
-        typedef std::forward_list<T, A> C;
-        const T t1[] = {0, 1, 2, 3, 4, 5};
-        C c1(std::begin(t1), std::end(t1), A(1, 1));
-        C c2(A(1, 2));
-        c1.swap(c2);
+    assert(std::distance(c2.begin(), c2.end()) == 6);
+    assert(*std::next(c2.begin(), 0) == 0);
+    assert(*std::next(c2.begin(), 1) == 1);
+    assert(*std::next(c2.begin(), 2) == 2);
+    assert(*std::next(c2.begin(), 3) == 3);
+    assert(*std::next(c2.begin(), 4) == 4);
+    assert(*std::next(c2.begin(), 5) == 5);
+    assert(c2.get_allocator().get_id() == 2);
+  }
+  {
+    typedef int T;
+    typedef test_allocator<T> A;
+    typedef std::forward_list<T, A> C;
+    const T t1[] = {0, 1, 2, 3, 4, 5};
+    C c1(std::begin(t1), std::end(t1), A(1, 1));
+    C c2(A(1, 2));
+    c1.swap(c2);
 
-        assert(std::distance(c1.begin(), c1.end()) == 0);
-        assert(c1.get_allocator().get_id() == 1);
+    assert(std::distance(c1.begin(), c1.end()) == 0);
+    assert(c1.get_allocator().get_id() == 1);
 
-        assert(std::distance(c2.begin(), c2.end()) == 6);
-        assert(*std::next(c2.begin(), 0) == 0);
-        assert(*std::next(c2.begin(), 1) == 1);
-        assert(*std::next(c2.begin(), 2) == 2);
-        assert(*std::next(c2.begin(), 3) == 3);
-        assert(*std::next(c2.begin(), 4) == 4);
-        assert(*std::next(c2.begin(), 5) == 5);
-        assert(c2.get_allocator().get_id() == 2);
-    }
-    {
-        typedef int T;
-        typedef test_allocator<T> A;
-        typedef std::forward_list<T, A> C;
-        C c1(A(1, 1));
-        const T t2[] = {10, 11, 12};
-        C c2(std::begin(t2), std::end(t2), A(1, 2));
-        c1.swap(c2);
+    assert(std::distance(c2.begin(), c2.end()) == 6);
+    assert(*std::next(c2.begin(), 0) == 0);
+    assert(*std::next(c2.begin(), 1) == 1);
+    assert(*std::next(c2.begin(), 2) == 2);
+    assert(*std::next(c2.begin(), 3) == 3);
+    assert(*std::next(c2.begin(), 4) == 4);
+    assert(*std::next(c2.begin(), 5) == 5);
+    assert(c2.get_allocator().get_id() == 2);
+  }
+  {
+    typedef int T;
+    typedef test_allocator<T> A;
+    typedef std::forward_list<T, A> C;
+    C c1(A(1, 1));
+    const T t2[] = {10, 11, 12};
+    C c2(std::begin(t2), std::end(t2), A(1, 2));
+    c1.swap(c2);
 
-        assert(std::distance(c1.begin(), c1.end()) == 3);
-        assert(*std::next(c1.begin(), 0) == 10);
-        assert(*std::next(c1.begin(), 1) == 11);
-        assert(*std::next(c1.begin(), 2) == 12);
-        assert(c1.get_allocator().get_id() == 1);
+    assert(std::distance(c1.begin(), c1.end()) == 3);
+    assert(*std::next(c1.begin(), 0) == 10);
+    assert(*std::next(c1.begin(), 1) == 11);
+    assert(*std::next(c1.begin(), 2) == 12);
+    assert(c1.get_allocator().get_id() == 1);
 
-        assert(std::distance(c2.begin(), c2.end()) == 0);
-        assert(c2.get_allocator().get_id() == 2);
-    }
-    {
-        typedef int T;
-        typedef test_allocator<T> A;
-        typedef std::forward_list<T, A> C;
-        C c1(A(1, 1));
-        C c2(A(1, 2));
-        c1.swap(c2);
+    assert(std::distance(c2.begin(), c2.end()) == 0);
+    assert(c2.get_allocator().get_id() == 2);
+  }
+  {
+    typedef int T;
+    typedef test_allocator<T> A;
+    typedef std::forward_list<T, A> C;
+    C c1(A(1, 1));
+    C c2(A(1, 2));
+    c1.swap(c2);
 
-        assert(std::distance(c1.begin(), c1.end()) == 0);
-        assert(c1.get_allocator().get_id() == 1);
+    assert(std::distance(c1.begin(), c1.end()) == 0);
+    assert(c1.get_allocator().get_id() == 1);
 
-        assert(std::distance(c2.begin(), c2.end()) == 0);
-        assert(c2.get_allocator().get_id() == 2);
-    }
+    assert(std::distance(c2.begin(), c2.end()) == 0);
+    assert(c2.get_allocator().get_id() == 2);
+  }
 
-    {
-        typedef int T;
-        typedef other_allocator<T> A;
-        typedef std::forward_list<T, A> C;
-        const T t1[] = {0, 1, 2, 3, 4, 5};
-        C c1(std::begin(t1), std::end(t1), A(1));
-        const T t2[] = {10, 11, 12};
-        C c2(std::begin(t2), std::end(t2), A(2));
-        c1.swap(c2);
+  {
+    typedef int T;
+    typedef other_allocator<T> A;
+    typedef std::forward_list<T, A> C;
+    const T t1[] = {0, 1, 2, 3, 4, 5};
+    C c1(std::begin(t1), std::end(t1), A(1));
+    const T t2[] = {10, 11, 12};
+    C c2(std::begin(t2), std::end(t2), A(2));
+    c1.swap(c2);
 
-        assert(std::distance(c1.begin(), c1.end()) == 3);
-        assert(*std::next(c1.begin(), 0) == 10);
-        assert(*std::next(c1.begin(), 1) == 11);
-        assert(*std::next(c1.begin(), 2) == 12);
-        assert(c1.get_allocator() == A(2));
+    assert(std::distance(c1.begin(), c1.end()) == 3);
+    assert(*std::next(c1.begin(), 0) == 10);
+    assert(*std::next(c1.begin(), 1) == 11);
+    assert(*std::next(c1.begin(), 2) == 12);
+    assert(c1.get_allocator() == A(2));
 
-        assert(std::distance(c2.begin(), c2.end()) == 6);
-        assert(*std::next(c2.begin(), 0) == 0);
-        assert(*std::next(c2.begin(), 1) == 1);
-        assert(*std::next(c2.begin(), 2) == 2);
-        assert(*std::next(c2.begin(), 3) == 3);
-        assert(*std::next(c2.begin(), 4) == 4);
-        assert(*std::next(c2.begin(), 5) == 5);
-        assert(c2.get_allocator() == A(1));
-    }
-    {
-        typedef int T;
-        typedef other_allocator<T> A;
-        typedef std::forward_list<T, A> C;
-        const T t1[] = {0, 1, 2, 3, 4, 5};
-        C c1(std::begin(t1), std::end(t1), A(1));
-        C c2(A(2));
-        c1.swap(c2);
+    assert(std::distance(c2.begin(), c2.end()) == 6);
+    assert(*std::next(c2.begin(), 0) == 0);
+    assert(*std::next(c2.begin(), 1) == 1);
+    assert(*std::next(c2.begin(), 2) == 2);
+    assert(*std::next(c2.begin(), 3) == 3);
+    assert(*std::next(c2.begin(), 4) == 4);
+    assert(*std::next(c2.begin(), 5) == 5);
+    assert(c2.get_allocator() == A(1));
+  }
+  {
+    typedef int T;
+    typedef other_allocator<T> A;
+    typedef std::forward_list<T, A> C;
+    const T t1[] = {0, 1, 2, 3, 4, 5};
+    C c1(std::begin(t1), std::end(t1), A(1));
+    C c2(A(2));
+    c1.swap(c2);
 
-        assert(std::distance(c1.begin(), c1.end()) == 0);
-        assert(c1.get_allocator() == A(2));
+    assert(std::distance(c1.begin(), c1.end()) == 0);
+    assert(c1.get_allocator() == A(2));
 
-        assert(std::distance(c2.begin(), c2.end()) == 6);
-        assert(*std::next(c2.begin(), 0) == 0);
-        assert(*std::next(c2.begin(), 1) == 1);
-        assert(*std::next(c2.begin(), 2) == 2);
-        assert(*std::next(c2.begin(), 3) == 3);
-        assert(*std::next(c2.begin(), 4) == 4);
-        assert(*std::next(c2.begin(), 5) == 5);
-        assert(c2.get_allocator() == A(1));
-    }
-    {
-        typedef int T;
-        typedef other_allocator<T> A;
-        typedef std::forward_list<T, A> C;
-        C c1(A(1));
-        const T t2[] = {10, 11, 12};
-        C c2(std::begin(t2), std::end(t2), A(2));
-        c1.swap(c2);
+    assert(std::distance(c2.begin(), c2.end()) == 6);
+    assert(*std::next(c2.begin(), 0) == 0);
+    assert(*std::next(c2.begin(), 1) == 1);
+    assert(*std::next(c2.begin(), 2) == 2);
+    assert(*std::next(c2.begin(), 3) == 3);
+    assert(*std::next(c2.begin(), 4) == 4);
+    assert(*std::next(c2.begin(), 5) == 5);
+    assert(c2.get_allocator() == A(1));
+  }
+  {
+    typedef int T;
+    typedef other_allocator<T> A;
+    typedef std::forward_list<T, A> C;
+    C c1(A(1));
+    const T t2[] = {10, 11, 12};
+    C c2(std::begin(t2), std::end(t2), A(2));
+    c1.swap(c2);
 
-        assert(std::distance(c1.begin(), c1.end()) == 3);
-        assert(*std::next(c1.begin(), 0) == 10);
-        assert(*std::next(c1.begin(), 1) == 11);
-        assert(*std::next(c1.begin(), 2) == 12);
-        assert(c1.get_allocator() == A(2));
+    assert(std::distance(c1.begin(), c1.end()) == 3);
+    assert(*std::next(c1.begin(), 0) == 10);
+    assert(*std::next(c1.begin(), 1) == 11);
+    assert(*std::next(c1.begin(), 2) == 12);
+    assert(c1.get_allocator() == A(2));
 
-        assert(std::distance(c2.begin(), c2.end()) == 0);
-        assert(c2.get_allocator() == A(1));
-    }
-    {
-        typedef int T;
-        typedef other_allocator<T> A;
-        typedef std::forward_list<T, A> C;
-        C c1(A(1));
-        C c2(A(2));
-        c1.swap(c2);
+    assert(std::distance(c2.begin(), c2.end()) == 0);
+    assert(c2.get_allocator() == A(1));
+  }
+  {
+    typedef int T;
+    typedef other_allocator<T> A;
+    typedef std::forward_list<T, A> C;
+    C c1(A(1));
+    C c2(A(2));
+    c1.swap(c2);
 
-        assert(std::distance(c1.begin(), c1.end()) == 0);
-        assert(c1.get_allocator() == A(2));
+    assert(std::distance(c1.begin(), c1.end()) == 0);
+    assert(c1.get_allocator() == A(2));
 
-        assert(std::distance(c2.begin(), c2.end()) == 0);
-        assert(c2.get_allocator() == A(1));
-    }
+    assert(std::distance(c2.begin(), c2.end()) == 0);
+    assert(c2.get_allocator() == A(1));
+  }
 #if TEST_STD_VER >= 11
-    {
-        typedef int T;
-        typedef min_allocator<T> A;
-        typedef std::forward_list<T, A> C;
-        const T t1[] = {0, 1, 2, 3, 4, 5};
-        C c1(std::begin(t1), std::end(t1), A());
-        const T t2[] = {10, 11, 12};
-        C c2(std::begin(t2), std::end(t2), A());
-        c1.swap(c2);
+  {
+    typedef int T;
+    typedef min_allocator<T> A;
+    typedef std::forward_list<T, A> C;
+    const T t1[] = {0, 1, 2, 3, 4, 5};
+    C c1(std::begin(t1), std::end(t1), A());
+    const T t2[] = {10, 11, 12};
+    C c2(std::begin(t2), std::end(t2), A());
+    c1.swap(c2);
 
-        assert(std::distance(c1.begin(), c1.end()) == 3);
-        assert(*std::next(c1.begin(), 0) == 10);
-        assert(*std::next(c1.begin(), 1) == 11);
-        assert(*std::next(c1.begin(), 2) == 12);
-        assert(c1.get_allocator() == A());
+    assert(std::distance(c1.begin(), c1.end()) == 3);
+    assert(*std::next(c1.begin(), 0) == 10);
+    assert(*std::next(c1.begin(), 1) == 11);
+    assert(*std::next(c1.begin(), 2) == 12);
+    assert(c1.get_allocator() == A());
 
-        assert(std::distance(c2.begin(), c2.end()) == 6);
-        assert(*std::next(c2.begin(), 0) == 0);
-        assert(*std::next(c2.begin(), 1) == 1);
-        assert(*std::next(c2.begin(), 2) == 2);
-        assert(*std::next(c2.begin(), 3) == 3);
-        assert(*std::next(c2.begin(), 4) == 4);
-        assert(*std::next(c2.begin(), 5) == 5);
-        assert(c2.get_allocator() == A());
-    }
-    {
-        typedef int T;
-        typedef min_allocator<T> A;
-        typedef std::forward_list<T, A> C;
-        const T t1[] = {0, 1, 2, 3, 4, 5};
-        C c1(std::begin(t1), std::end(t1), A());
-        C c2(A{});
-        c1.swap(c2);
+    assert(std::distance(c2.begin(), c2.end()) == 6);
+    assert(*std::next(c2.begin(), 0) == 0);
+    assert(*std::next(c2.begin(), 1) == 1);
+    assert(*std::next(c2.begin(), 2) == 2);
+    assert(*std::next(c2.begin(), 3) == 3);
+    assert(*std::next(c2.begin(), 4) == 4);
+    assert(*std::next(c2.begin(), 5) == 5);
+    assert(c2.get_allocator() == A());
+  }
+  {
+    typedef int T;
+    typedef min_allocator<T> A;
+    typedef std::forward_list<T, A> C;
+    const T t1[] = {0, 1, 2, 3, 4, 5};
+    C c1(std::begin(t1), std::end(t1), A());
+    C c2(A{});
+    c1.swap(c2);
 
-        assert(std::distance(c1.begin(), c1.end()) == 0);
-        assert(c1.get_allocator() == A());
+    assert(std::distance(c1.begin(), c1.end()) == 0);
+    assert(c1.get_allocator() == A());
 
-        assert(std::distance(c2.begin(), c2.end()) == 6);
-        assert(*std::next(c2.begin(), 0) == 0);
-        assert(*std::next(c2.begin(), 1) == 1);
-        assert(*std::next(c2.begin(), 2) == 2);
-        assert(*std::next(c2.begin(), 3) == 3);
-        assert(*std::next(c2.begin(), 4) == 4);
-        assert(*std::next(c2.begin(), 5) == 5);
-        assert(c2.get_allocator() == A());
-    }
-    {
-        typedef int T;
-        typedef min_allocator<T> A;
-        typedef std::forward_list<T, A> C;
-        C c1(A{});
-        const T t2[] = {10, 11, 12};
-        C c2(std::begin(t2), std::end(t2), A());
-        c1.swap(c2);
+    assert(std::distance(c2.begin(), c2.end()) == 6);
+    assert(*std::next(c2.begin(), 0) == 0);
+    assert(*std::next(c2.begin(), 1) == 1);
+    assert(*std::next(c2.begin(), 2) == 2);
+    assert(*std::next(c2.begin(), 3) == 3);
+    assert(*std::next(c2.begin(), 4) == 4);
+    assert(*std::next(c2.begin(), 5) == 5);
+    assert(c2.get_allocator() == A());
+  }
+  {
+    typedef int T;
+    typedef min_allocator<T> A;
+    typedef std::forward_list<T, A> C;
+    C c1(A{});
+    const T t2[] = {10, 11, 12};
+    C c2(std::begin(t2), std::end(t2), A());
+    c1.swap(c2);
 
-        assert(std::distance(c1.begin(), c1.end()) == 3);
-        assert(*std::next(c1.begin(), 0) == 10);
-        assert(*std::next(c1.begin(), 1) == 11);
-        assert(*std::next(c1.begin(), 2) == 12);
-        assert(c1.get_allocator() == A());
+    assert(std::distance(c1.begin(), c1.end()) == 3);
+    assert(*std::next(c1.begin(), 0) == 10);
+    assert(*std::next(c1.begin(), 1) == 11);
+    assert(*std::next(c1.begin(), 2) == 12);
+    assert(c1.get_allocator() == A());
 
-        assert(std::distance(c2.begin(), c2.end()) == 0);
-        assert(c2.get_allocator() == A());
-    }
-    {
-        typedef int T;
-        typedef min_allocator<T> A;
-        typedef std::forward_list<T, A> C;
-        C c1(A{});
-        C c2(A{});
-        c1.swap(c2);
+    assert(std::distance(c2.begin(), c2.end()) == 0);
+    assert(c2.get_allocator() == A());
+  }
+  {
+    typedef int T;
+    typedef min_allocator<T> A;
+    typedef std::forward_list<T, A> C;
+    C c1(A{});
+    C c2(A{});
+    c1.swap(c2);
 
-        assert(std::distance(c1.begin(), c1.end()) == 0);
-        assert(c1.get_allocator() == A());
+    assert(std::distance(c1.begin(), c1.end()) == 0);
+    assert(c1.get_allocator() == A());
 
-        assert(std::distance(c2.begin(), c2.end()) == 0);
-        assert(c2.get_allocator() == A());
-    }
+    assert(std::distance(c2.begin(), c2.end()) == 0);
+    assert(c2.get_allocator() == A());
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/sequences/forwardlist/forwardlist.spec/non_member_swap.pass.cpp b/libcxx/test/std/containers/sequences/forwardlist/forwardlist.spec/non_member_swap.pass.cpp
index 54d26d0a64917..cae6950436dee 100644
--- a/libcxx/test/std/containers/sequences/forwardlist/forwardlist.spec/non_member_swap.pass.cpp
+++ b/libcxx/test/std/containers/sequences/forwardlist/forwardlist.spec/non_member_swap.pass.cpp
@@ -19,244 +19,243 @@
 #include "test_allocator.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef int T;
-        typedef test_allocator<T> A;
-        typedef std::forward_list<T, A> C;
-        const T t1[] = {0, 1, 2, 3, 4, 5};
-        C c1(std::begin(t1), std::end(t1), A(1, 1));
-        const T t2[] = {10, 11, 12};
-        C c2(std::begin(t2), std::end(t2), A(1, 2));
-        swap(c1, c2);
+int main(int, char**) {
+  {
+    typedef int T;
+    typedef test_allocator<T> A;
+    typedef std::forward_list<T, A> C;
+    const T t1[] = {0, 1, 2, 3, 4, 5};
+    C c1(std::begin(t1), std::end(t1), A(1, 1));
+    const T t2[] = {10, 11, 12};
+    C c2(std::begin(t2), std::end(t2), A(1, 2));
+    swap(c1, c2);
 
-        assert(std::distance(c1.begin(), c1.end()) == 3);
-        assert(*std::next(c1.begin(), 0) == 10);
-        assert(*std::next(c1.begin(), 1) == 11);
-        assert(*std::next(c1.begin(), 2) == 12);
-        assert(c1.get_allocator().get_id() == 1);
+    assert(std::distance(c1.begin(), c1.end()) == 3);
+    assert(*std::next(c1.begin(), 0) == 10);
+    assert(*std::next(c1.begin(), 1) == 11);
+    assert(*std::next(c1.begin(), 2) == 12);
+    assert(c1.get_allocator().get_id() == 1);
 
-        assert(std::distance(c2.begin(), c2.end()) == 6);
-        assert(*std::next(c2.begin(), 0) == 0);
-        assert(*std::next(c2.begin(), 1) == 1);
-        assert(*std::next(c2.begin(), 2) == 2);
-        assert(*std::next(c2.begin(), 3) == 3);
-        assert(*std::next(c2.begin(), 4) == 4);
-        assert(*std::next(c2.begin(), 5) == 5);
-        assert(c2.get_allocator().get_id() == 2);
-    }
-    {
-        typedef int T;
-        typedef test_allocator<T> A;
-        typedef std::forward_list<T, A> C;
-        const T t1[] = {0, 1, 2, 3, 4, 5};
-        C c1(std::begin(t1), std::end(t1), A(1, 1));
-        C c2(A(1, 2));
-        swap(c1, c2);
+    assert(std::distance(c2.begin(), c2.end()) == 6);
+    assert(*std::next(c2.begin(), 0) == 0);
+    assert(*std::next(c2.begin(), 1) == 1);
+    assert(*std::next(c2.begin(), 2) == 2);
+    assert(*std::next(c2.begin(), 3) == 3);
+    assert(*std::next(c2.begin(), 4) == 4);
+    assert(*std::next(c2.begin(), 5) == 5);
+    assert(c2.get_allocator().get_id() == 2);
+  }
+  {
+    typedef int T;
+    typedef test_allocator<T> A;
+    typedef std::forward_list<T, A> C;
+    const T t1[] = {0, 1, 2, 3, 4, 5};
+    C c1(std::begin(t1), std::end(t1), A(1, 1));
+    C c2(A(1, 2));
+    swap(c1, c2);
 
-        assert(std::distance(c1.begin(), c1.end()) == 0);
-        assert(c1.get_allocator().get_id() == 1);
+    assert(std::distance(c1.begin(), c1.end()) == 0);
+    assert(c1.get_allocator().get_id() == 1);
 
-        assert(std::distance(c2.begin(), c2.end()) == 6);
-        assert(*std::next(c2.begin(), 0) == 0);
-        assert(*std::next(c2.begin(), 1) == 1);
-        assert(*std::next(c2.begin(), 2) == 2);
-        assert(*std::next(c2.begin(), 3) == 3);
-        assert(*std::next(c2.begin(), 4) == 4);
-        assert(*std::next(c2.begin(), 5) == 5);
-        assert(c2.get_allocator().get_id() == 2);
-    }
-    {
-        typedef int T;
-        typedef test_allocator<T> A;
-        typedef std::forward_list<T, A> C;
-        C c1(A(1, 1));
-        const T t2[] = {10, 11, 12};
-        C c2(std::begin(t2), std::end(t2), A(1, 2));
-        swap(c1, c2);
+    assert(std::distance(c2.begin(), c2.end()) == 6);
+    assert(*std::next(c2.begin(), 0) == 0);
+    assert(*std::next(c2.begin(), 1) == 1);
+    assert(*std::next(c2.begin(), 2) == 2);
+    assert(*std::next(c2.begin(), 3) == 3);
+    assert(*std::next(c2.begin(), 4) == 4);
+    assert(*std::next(c2.begin(), 5) == 5);
+    assert(c2.get_allocator().get_id() == 2);
+  }
+  {
+    typedef int T;
+    typedef test_allocator<T> A;
+    typedef std::forward_list<T, A> C;
+    C c1(A(1, 1));
+    const T t2[] = {10, 11, 12};
+    C c2(std::begin(t2), std::end(t2), A(1, 2));
+    swap(c1, c2);
 
-        assert(std::distance(c1.begin(), c1.end()) == 3);
-        assert(*std::next(c1.begin(), 0) == 10);
-        assert(*std::next(c1.begin(), 1) == 11);
-        assert(*std::next(c1.begin(), 2) == 12);
-        assert(c1.get_allocator().get_id() == 1);
+    assert(std::distance(c1.begin(), c1.end()) == 3);
+    assert(*std::next(c1.begin(), 0) == 10);
+    assert(*std::next(c1.begin(), 1) == 11);
+    assert(*std::next(c1.begin(), 2) == 12);
+    assert(c1.get_allocator().get_id() == 1);
 
-        assert(std::distance(c2.begin(), c2.end()) == 0);
-        assert(c2.get_allocator().get_id() == 2);
-    }
-    {
-        typedef int T;
-        typedef test_allocator<T> A;
-        typedef std::forward_list<T, A> C;
-        C c1(A(1, 1));
-        C c2(A(1, 2));
-        swap(c1, c2);
+    assert(std::distance(c2.begin(), c2.end()) == 0);
+    assert(c2.get_allocator().get_id() == 2);
+  }
+  {
+    typedef int T;
+    typedef test_allocator<T> A;
+    typedef std::forward_list<T, A> C;
+    C c1(A(1, 1));
+    C c2(A(1, 2));
+    swap(c1, c2);
 
-        assert(std::distance(c1.begin(), c1.end()) == 0);
-        assert(c1.get_allocator().get_id() == 1);
+    assert(std::distance(c1.begin(), c1.end()) == 0);
+    assert(c1.get_allocator().get_id() == 1);
 
-        assert(std::distance(c2.begin(), c2.end()) == 0);
-        assert(c2.get_allocator().get_id() == 2);
-    }
+    assert(std::distance(c2.begin(), c2.end()) == 0);
+    assert(c2.get_allocator().get_id() == 2);
+  }
 
-    {
-        typedef int T;
-        typedef other_allocator<T> A;
-        typedef std::forward_list<T, A> C;
-        const T t1[] = {0, 1, 2, 3, 4, 5};
-        C c1(std::begin(t1), std::end(t1), A(1));
-        const T t2[] = {10, 11, 12};
-        C c2(std::begin(t2), std::end(t2), A(2));
-        swap(c1, c2);
+  {
+    typedef int T;
+    typedef other_allocator<T> A;
+    typedef std::forward_list<T, A> C;
+    const T t1[] = {0, 1, 2, 3, 4, 5};
+    C c1(std::begin(t1), std::end(t1), A(1));
+    const T t2[] = {10, 11, 12};
+    C c2(std::begin(t2), std::end(t2), A(2));
+    swap(c1, c2);
 
-        assert(std::distance(c1.begin(), c1.end()) == 3);
-        assert(*std::next(c1.begin(), 0) == 10);
-        assert(*std::next(c1.begin(), 1) == 11);
-        assert(*std::next(c1.begin(), 2) == 12);
-        assert(c1.get_allocator() == A(2));
+    assert(std::distance(c1.begin(), c1.end()) == 3);
+    assert(*std::next(c1.begin(), 0) == 10);
+    assert(*std::next(c1.begin(), 1) == 11);
+    assert(*std::next(c1.begin(), 2) == 12);
+    assert(c1.get_allocator() == A(2));
 
-        assert(std::distance(c2.begin(), c2.end()) == 6);
-        assert(*std::next(c2.begin(), 0) == 0);
-        assert(*std::next(c2.begin(), 1) == 1);
-        assert(*std::next(c2.begin(), 2) == 2);
-        assert(*std::next(c2.begin(), 3) == 3);
-        assert(*std::next(c2.begin(), 4) == 4);
-        assert(*std::next(c2.begin(), 5) == 5);
-        assert(c2.get_allocator() == A(1));
-    }
-    {
-        typedef int T;
-        typedef other_allocator<T> A;
-        typedef std::forward_list<T, A> C;
-        const T t1[] = {0, 1, 2, 3, 4, 5};
-        C c1(std::begin(t1), std::end(t1), A(1));
-        C c2(A(2));
-        swap(c1, c2);
+    assert(std::distance(c2.begin(), c2.end()) == 6);
+    assert(*std::next(c2.begin(), 0) == 0);
+    assert(*std::next(c2.begin(), 1) == 1);
+    assert(*std::next(c2.begin(), 2) == 2);
+    assert(*std::next(c2.begin(), 3) == 3);
+    assert(*std::next(c2.begin(), 4) == 4);
+    assert(*std::next(c2.begin(), 5) == 5);
+    assert(c2.get_allocator() == A(1));
+  }
+  {
+    typedef int T;
+    typedef other_allocator<T> A;
+    typedef std::forward_list<T, A> C;
+    const T t1[] = {0, 1, 2, 3, 4, 5};
+    C c1(std::begin(t1), std::end(t1), A(1));
+    C c2(A(2));
+    swap(c1, c2);
 
-        assert(std::distance(c1.begin(), c1.end()) == 0);
-        assert(c1.get_allocator() == A(2));
+    assert(std::distance(c1.begin(), c1.end()) == 0);
+    assert(c1.get_allocator() == A(2));
 
-        assert(std::distance(c2.begin(), c2.end()) == 6);
-        assert(*std::next(c2.begin(), 0) == 0);
-        assert(*std::next(c2.begin(), 1) == 1);
-        assert(*std::next(c2.begin(), 2) == 2);
-        assert(*std::next(c2.begin(), 3) == 3);
-        assert(*std::next(c2.begin(), 4) == 4);
-        assert(*std::next(c2.begin(), 5) == 5);
-        assert(c2.get_allocator() == A(1));
-    }
-    {
-        typedef int T;
-        typedef other_allocator<T> A;
-        typedef std::forward_list<T, A> C;
-        C c1(A(1));
-        const T t2[] = {10, 11, 12};
-        C c2(std::begin(t2), std::end(t2), A(2));
-        swap(c1, c2);
+    assert(std::distance(c2.begin(), c2.end()) == 6);
+    assert(*std::next(c2.begin(), 0) == 0);
+    assert(*std::next(c2.begin(), 1) == 1);
+    assert(*std::next(c2.begin(), 2) == 2);
+    assert(*std::next(c2.begin(), 3) == 3);
+    assert(*std::next(c2.begin(), 4) == 4);
+    assert(*std::next(c2.begin(), 5) == 5);
+    assert(c2.get_allocator() == A(1));
+  }
+  {
+    typedef int T;
+    typedef other_allocator<T> A;
+    typedef std::forward_list<T, A> C;
+    C c1(A(1));
+    const T t2[] = {10, 11, 12};
+    C c2(std::begin(t2), std::end(t2), A(2));
+    swap(c1, c2);
 
-        assert(std::distance(c1.begin(), c1.end()) == 3);
-        assert(*std::next(c1.begin(), 0) == 10);
-        assert(*std::next(c1.begin(), 1) == 11);
-        assert(*std::next(c1.begin(), 2) == 12);
-        assert(c1.get_allocator() == A(2));
+    assert(std::distance(c1.begin(), c1.end()) == 3);
+    assert(*std::next(c1.begin(), 0) == 10);
+    assert(*std::next(c1.begin(), 1) == 11);
+    assert(*std::next(c1.begin(), 2) == 12);
+    assert(c1.get_allocator() == A(2));
 
-        assert(std::distance(c2.begin(), c2.end()) == 0);
-        assert(c2.get_allocator() == A(1));
-    }
-    {
-        typedef int T;
-        typedef other_allocator<T> A;
-        typedef std::forward_list<T, A> C;
-        C c1(A(1));
-        C c2(A(2));
-        swap(c1, c2);
+    assert(std::distance(c2.begin(), c2.end()) == 0);
+    assert(c2.get_allocator() == A(1));
+  }
+  {
+    typedef int T;
+    typedef other_allocator<T> A;
+    typedef std::forward_list<T, A> C;
+    C c1(A(1));
+    C c2(A(2));
+    swap(c1, c2);
 
-        assert(std::distance(c1.begin(), c1.end()) == 0);
-        assert(c1.get_allocator() == A(2));
+    assert(std::distance(c1.begin(), c1.end()) == 0);
+    assert(c1.get_allocator() == A(2));
 
-        assert(std::distance(c2.begin(), c2.end()) == 0);
-        assert(c2.get_allocator() == A(1));
-    }
+    assert(std::distance(c2.begin(), c2.end()) == 0);
+    assert(c2.get_allocator() == A(1));
+  }
 #if TEST_STD_VER >= 11
-    {
-        typedef int T;
-        typedef min_allocator<T> A;
-        typedef std::forward_list<T, A> C;
-        const T t1[] = {0, 1, 2, 3, 4, 5};
-        C c1(std::begin(t1), std::end(t1), A());
-        const T t2[] = {10, 11, 12};
-        C c2(std::begin(t2), std::end(t2), A());
-        swap(c1, c2);
+  {
+    typedef int T;
+    typedef min_allocator<T> A;
+    typedef std::forward_list<T, A> C;
+    const T t1[] = {0, 1, 2, 3, 4, 5};
+    C c1(std::begin(t1), std::end(t1), A());
+    const T t2[] = {10, 11, 12};
+    C c2(std::begin(t2), std::end(t2), A());
+    swap(c1, c2);
 
-        assert(std::distance(c1.begin(), c1.end()) == 3);
-        assert(*std::next(c1.begin(), 0) == 10);
-        assert(*std::next(c1.begin(), 1) == 11);
-        assert(*std::next(c1.begin(), 2) == 12);
-        assert(c1.get_allocator() == A());
+    assert(std::distance(c1.begin(), c1.end()) == 3);
+    assert(*std::next(c1.begin(), 0) == 10);
+    assert(*std::next(c1.begin(), 1) == 11);
+    assert(*std::next(c1.begin(), 2) == 12);
+    assert(c1.get_allocator() == A());
 
-        assert(std::distance(c2.begin(), c2.end()) == 6);
-        assert(*std::next(c2.begin(), 0) == 0);
-        assert(*std::next(c2.begin(), 1) == 1);
-        assert(*std::next(c2.begin(), 2) == 2);
-        assert(*std::next(c2.begin(), 3) == 3);
-        assert(*std::next(c2.begin(), 4) == 4);
-        assert(*std::next(c2.begin(), 5) == 5);
-        assert(c2.get_allocator() == A());
-    }
-    {
-        typedef int T;
-        typedef min_allocator<T> A;
-        typedef std::forward_list<T, A> C;
-        const T t1[] = {0, 1, 2, 3, 4, 5};
-        C c1(std::begin(t1), std::end(t1), A());
-        C c2(A{});
-        swap(c1, c2);
+    assert(std::distance(c2.begin(), c2.end()) == 6);
+    assert(*std::next(c2.begin(), 0) == 0);
+    assert(*std::next(c2.begin(), 1) == 1);
+    assert(*std::next(c2.begin(), 2) == 2);
+    assert(*std::next(c2.begin(), 3) == 3);
+    assert(*std::next(c2.begin(), 4) == 4);
+    assert(*std::next(c2.begin(), 5) == 5);
+    assert(c2.get_allocator() == A());
+  }
+  {
+    typedef int T;
+    typedef min_allocator<T> A;
+    typedef std::forward_list<T, A> C;
+    const T t1[] = {0, 1, 2, 3, 4, 5};
+    C c1(std::begin(t1), std::end(t1), A());
+    C c2(A{});
+    swap(c1, c2);
 
-        assert(std::distance(c1.begin(), c1.end()) == 0);
-        assert(c1.get_allocator() == A());
+    assert(std::distance(c1.begin(), c1.end()) == 0);
+    assert(c1.get_allocator() == A());
 
-        assert(std::distance(c2.begin(), c2.end()) == 6);
-        assert(*std::next(c2.begin(), 0) == 0);
-        assert(*std::next(c2.begin(), 1) == 1);
-        assert(*std::next(c2.begin(), 2) == 2);
-        assert(*std::next(c2.begin(), 3) == 3);
-        assert(*std::next(c2.begin(), 4) == 4);
-        assert(*std::next(c2.begin(), 5) == 5);
-        assert(c2.get_allocator() == A());
-    }
-    {
-        typedef int T;
-        typedef min_allocator<T> A;
-        typedef std::forward_list<T, A> C;
-        C c1(A{});
-        const T t2[] = {10, 11, 12};
-        C c2(std::begin(t2), std::end(t2), A());
-        swap(c1, c2);
+    assert(std::distance(c2.begin(), c2.end()) == 6);
+    assert(*std::next(c2.begin(), 0) == 0);
+    assert(*std::next(c2.begin(), 1) == 1);
+    assert(*std::next(c2.begin(), 2) == 2);
+    assert(*std::next(c2.begin(), 3) == 3);
+    assert(*std::next(c2.begin(), 4) == 4);
+    assert(*std::next(c2.begin(), 5) == 5);
+    assert(c2.get_allocator() == A());
+  }
+  {
+    typedef int T;
+    typedef min_allocator<T> A;
+    typedef std::forward_list<T, A> C;
+    C c1(A{});
+    const T t2[] = {10, 11, 12};
+    C c2(std::begin(t2), std::end(t2), A());
+    swap(c1, c2);
 
-        assert(std::distance(c1.begin(), c1.end()) == 3);
-        assert(*std::next(c1.begin(), 0) == 10);
-        assert(*std::next(c1.begin(), 1) == 11);
-        assert(*std::next(c1.begin(), 2) == 12);
-        assert(c1.get_allocator() == A());
+    assert(std::distance(c1.begin(), c1.end()) == 3);
+    assert(*std::next(c1.begin(), 0) == 10);
+    assert(*std::next(c1.begin(), 1) == 11);
+    assert(*std::next(c1.begin(), 2) == 12);
+    assert(c1.get_allocator() == A());
 
-        assert(std::distance(c2.begin(), c2.end()) == 0);
-        assert(c2.get_allocator() == A());
-    }
-    {
-        typedef int T;
-        typedef min_allocator<T> A;
-        typedef std::forward_list<T, A> C;
-        C c1(A{});
-        C c2(A{});
-        swap(c1, c2);
+    assert(std::distance(c2.begin(), c2.end()) == 0);
+    assert(c2.get_allocator() == A());
+  }
+  {
+    typedef int T;
+    typedef min_allocator<T> A;
+    typedef std::forward_list<T, A> C;
+    C c1(A{});
+    C c2(A{});
+    swap(c1, c2);
 
-        assert(std::distance(c1.begin(), c1.end()) == 0);
-        assert(c1.get_allocator() == A());
+    assert(std::distance(c1.begin(), c1.end()) == 0);
+    assert(c1.get_allocator() == A());
 
-        assert(std::distance(c2.begin(), c2.end()) == 0);
-        assert(c2.get_allocator() == A());
-    }
+    assert(std::distance(c2.begin(), c2.end()) == 0);
+    assert(c2.get_allocator() == A());
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/sequences/forwardlist/forwardlist.spec/relational.pass.cpp b/libcxx/test/std/containers/sequences/forwardlist/forwardlist.spec/relational.pass.cpp
index 9ca19486a54b9..d16acadaeb893 100644
--- a/libcxx/test/std/containers/sequences/forwardlist/forwardlist.spec/relational.pass.cpp
+++ b/libcxx/test/std/containers/sequences/forwardlist/forwardlist.spec/relational.pass.cpp
@@ -33,33 +33,31 @@
 #include "min_allocator.h"
 
 template <class C>
-void test(int N, int M)
-{
-    C c1;
-    for (int i = 0; i < N; ++i)
-        c1.push_front(i);
-    C c2;
-    for (int i = 0; i < M; ++i)
-        c2.push_front(i);
-    if (N < M)
-        assert(c1 < c2);
-    if (N <= M)
-        assert(c1 <= c2);
-    if (N >= M)
-        assert(c1 >= c2);
-    if (N > M)
-        assert(c1 > c2);
+void test(int N, int M) {
+  C c1;
+  for (int i = 0; i < N; ++i)
+    c1.push_front(i);
+  C c2;
+  for (int i = 0; i < M; ++i)
+    c2.push_front(i);
+  if (N < M)
+    assert(c1 < c2);
+  if (N <= M)
+    assert(c1 <= c2);
+  if (N >= M)
+    assert(c1 >= c2);
+  if (N > M)
+    assert(c1 > c2);
 }
 
-int main(int, char**)
-{
-    for (int i = 0; i < 10; ++i)
-        for (int j = 0; j < 10; ++j)
-            test<std::forward_list<int> >(i, j);
+int main(int, char**) {
+  for (int i = 0; i < 10; ++i)
+    for (int j = 0; j < 10; ++j)
+      test<std::forward_list<int> >(i, j);
 #if TEST_STD_VER >= 11
-    for (int i = 0; i < 10; ++i)
-        for (int j = 0; j < 10; ++j)
-            test<std::forward_list<int, min_allocator<int>> >(i, j);
+  for (int i = 0; i < 10; ++i)
+    for (int j = 0; j < 10; ++j)
+      test<std::forward_list<int, min_allocator<int>> >(i, j);
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/sequences/forwardlist/forwardlist.spec/swap_noexcept.pass.cpp b/libcxx/test/std/containers/sequences/forwardlist/forwardlist.spec/swap_noexcept.pass.cpp
index 76f8b6213908b..b83ec3c3c1210 100644
--- a/libcxx/test/std/containers/sequences/forwardlist/forwardlist.spec/swap_noexcept.pass.cpp
+++ b/libcxx/test/std/containers/sequences/forwardlist/forwardlist.spec/swap_noexcept.pass.cpp
@@ -28,63 +28,60 @@
 #include "test_allocator.h"
 
 template <class T>
-struct some_alloc
-{
-    typedef T value_type;
+struct some_alloc {
+  typedef T value_type;
 
-    some_alloc() {}
-    some_alloc(const some_alloc&);
-    void allocate(std::size_t);
-    void deallocate(void*, unsigned) {}
+  some_alloc() {}
+  some_alloc(const some_alloc&);
+  void allocate(std::size_t);
+  void deallocate(void*, unsigned) {}
 
-    typedef std::true_type propagate_on_container_swap;
+  typedef std::true_type propagate_on_container_swap;
 };
 
 template <class T>
-struct some_alloc2
-{
-    typedef T value_type;
+struct some_alloc2 {
+  typedef T value_type;
 
-    some_alloc2() {}
-    some_alloc2(const some_alloc2&);
-    void allocate(std::size_t);
-    void deallocate(void*, unsigned) {}
+  some_alloc2() {}
+  some_alloc2(const some_alloc2&);
+  void allocate(std::size_t);
+  void deallocate(void*, unsigned) {}
 
-    typedef std::false_type propagate_on_container_swap;
-    typedef std::true_type is_always_equal;
+  typedef std::false_type propagate_on_container_swap;
+  typedef std::true_type is_always_equal;
 };
 
-int main(int, char**)
-{
-    {
-        typedef std::forward_list<MoveOnly> C;
-        static_assert(noexcept(swap(std::declval<C&>(), std::declval<C&>())), "");
-    }
+int main(int, char**) {
+  {
+    typedef std::forward_list<MoveOnly> C;
+    static_assert(noexcept(swap(std::declval<C&>(), std::declval<C&>())), "");
+  }
 #if defined(_LIBCPP_VERSION)
-    {
-        typedef std::forward_list<MoveOnly, test_allocator<MoveOnly>> C;
-        static_assert(noexcept(swap(std::declval<C&>(), std::declval<C&>())), "");
-    }
-    {
-        typedef std::forward_list<MoveOnly, other_allocator<MoveOnly>> C;
-        static_assert(noexcept(swap(std::declval<C&>(), std::declval<C&>())), "");
-    }
+  {
+    typedef std::forward_list<MoveOnly, test_allocator<MoveOnly>> C;
+    static_assert(noexcept(swap(std::declval<C&>(), std::declval<C&>())), "");
+  }
+  {
+    typedef std::forward_list<MoveOnly, other_allocator<MoveOnly>> C;
+    static_assert(noexcept(swap(std::declval<C&>(), std::declval<C&>())), "");
+  }
 #endif // _LIBCPP_VERSION
-    {
-        typedef std::forward_list<MoveOnly, some_alloc<MoveOnly>> C;
+  {
+    typedef std::forward_list<MoveOnly, some_alloc<MoveOnly>> C;
 #if TEST_STD_VER >= 14
     //  In C++14, if POCS is set, swapping the allocator is required not to throw
-        static_assert( noexcept(swap(std::declval<C&>(), std::declval<C&>())), "");
+    static_assert(noexcept(swap(std::declval<C&>(), std::declval<C&>())), "");
 #else
-        static_assert(!noexcept(swap(std::declval<C&>(), std::declval<C&>())), "");
+    static_assert(!noexcept(swap(std::declval<C&>(), std::declval<C&>())), "");
 #endif
-    }
+  }
 #if TEST_STD_VER >= 14
-    {
-        typedef std::forward_list<MoveOnly, some_alloc2<MoveOnly>> C;
+  {
+    typedef std::forward_list<MoveOnly, some_alloc2<MoveOnly>> C;
     //  if the allocators are always equal, then the swap can be noexcept
-        static_assert( noexcept(swap(std::declval<C&>(), std::declval<C&>())), "");
-    }
+    static_assert(noexcept(swap(std::declval<C&>(), std::declval<C&>())), "");
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/sequences/forwardlist/get_allocator.pass.cpp b/libcxx/test/std/containers/sequences/forwardlist/get_allocator.pass.cpp
index 9d88e7c0fe878..f37f5c2f513bd 100644
--- a/libcxx/test/std/containers/sequences/forwardlist/get_allocator.pass.cpp
+++ b/libcxx/test/std/containers/sequences/forwardlist/get_allocator.pass.cpp
@@ -19,16 +19,16 @@
 #include "test_macros.h"
 
 int main(int, char**) {
-    {
-        std::allocator<int> alloc;
-        const std::forward_list<int> fl(alloc);
-        assert(fl.get_allocator() == alloc);
-    }
-    {
-        other_allocator<int> alloc(1);
-        const std::forward_list<int, other_allocator<int> > fl(alloc);
-        assert(fl.get_allocator() == alloc);
-    }
+  {
+    std::allocator<int> alloc;
+    const std::forward_list<int> fl(alloc);
+    assert(fl.get_allocator() == alloc);
+  }
+  {
+    other_allocator<int> alloc(1);
+    const std::forward_list<int, other_allocator<int> > fl(alloc);
+    assert(fl.get_allocator() == alloc);
+  }
 
-    return 0;
+  return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/forwardlist/incomplete.pass.cpp b/libcxx/test/std/containers/sequences/forwardlist/incomplete.pass.cpp
index 2bdfad777e8df..b7be03f1062dc 100644
--- a/libcxx/test/std/containers/sequences/forwardlist/incomplete.pass.cpp
+++ b/libcxx/test/std/containers/sequences/forwardlist/incomplete.pass.cpp
@@ -33,19 +33,18 @@ struct B {
 };
 #endif
 
-int main(int, char**)
-{
+int main(int, char**) {
   {
     A a;
     assert(a.d.empty());
-    a.it = a.d.begin();
+    a.it  = a.d.begin();
     a.it2 = a.d.cbefore_begin();
   }
 #if TEST_STD_VER >= 11
   {
     B b;
     assert(b.d.empty());
-    b.it = b.d.begin();
+    b.it  = b.d.begin();
     b.it2 = b.d.cbefore_begin();
   }
 #endif
diff --git a/libcxx/test/std/containers/sequences/forwardlist/max_size.pass.cpp b/libcxx/test/std/containers/sequences/forwardlist/max_size.pass.cpp
index c5145fde52af7..5ba0d61f104e0 100644
--- a/libcxx/test/std/containers/sequences/forwardlist/max_size.pass.cpp
+++ b/libcxx/test/std/containers/sequences/forwardlist/max_size.pass.cpp
@@ -18,32 +18,29 @@
 #include "test_allocator.h"
 #include "test_macros.h"
 
-int main(int, char**)
-{
-    {
-      typedef limited_allocator<int, 10> A;
-      typedef std::forward_list<int, A> C;
-      C c;
-      assert(c.max_size() <= 10);
-      LIBCPP_ASSERT(c.max_size() == 10);
-    }
-    {
-      typedef limited_allocator<int, (std::size_t)-1> A;
-      typedef std::forward_list<int, A> C;
-      const C::size_type max_dist =
-          static_cast<C::size_type>(std::numeric_limits<C::difference_type>::max());
-      C c;
-      assert(c.max_size() <= max_dist);
-      LIBCPP_ASSERT(c.max_size() == max_dist);
-    }
-    {
-      typedef std::forward_list<char> C;
-      const C::size_type max_dist =
-          static_cast<C::size_type>(std::numeric_limits<C::difference_type>::max());
-      C c;
-      assert(c.max_size() <= max_dist);
-      assert(c.max_size() <= alloc_max_size(c.get_allocator()));
-    }
+int main(int, char**) {
+  {
+    typedef limited_allocator<int, 10> A;
+    typedef std::forward_list<int, A> C;
+    C c;
+    assert(c.max_size() <= 10);
+    LIBCPP_ASSERT(c.max_size() == 10);
+  }
+  {
+    typedef limited_allocator<int, (std::size_t)-1> A;
+    typedef std::forward_list<int, A> C;
+    const C::size_type max_dist = static_cast<C::size_type>(std::numeric_limits<C::difference_type>::max());
+    C c;
+    assert(c.max_size() <= max_dist);
+    LIBCPP_ASSERT(c.max_size() == max_dist);
+  }
+  {
+    typedef std::forward_list<char> C;
+    const C::size_type max_dist = static_cast<C::size_type>(std::numeric_limits<C::difference_type>::max());
+    C c;
+    assert(c.max_size() <= max_dist);
+    assert(c.max_size() <= alloc_max_size(c.get_allocator()));
+  }
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/forwardlist/range_concept_conformance.compile.pass.cpp b/libcxx/test/std/containers/sequences/forwardlist/range_concept_conformance.compile.pass.cpp
index f3e7fd258a63f..53078c91cf885 100644
--- a/libcxx/test/std/containers/sequences/forwardlist/range_concept_conformance.compile.pass.cpp
+++ b/libcxx/test/std/containers/sequences/forwardlist/range_concept_conformance.compile.pass.cpp
@@ -17,7 +17,6 @@
 
 using range = std::forward_list<int>;
 
-
 static_assert(std::same_as<std::ranges::iterator_t<range>, range::iterator>);
 static_assert(std::ranges::common_range<range>);
 static_assert(std::ranges::forward_range<range>);
diff --git a/libcxx/test/std/containers/sequences/forwardlist/types.pass.cpp b/libcxx/test/std/containers/sequences/forwardlist/types.pass.cpp
index 116891d4980be..d2e48b351fc73 100644
--- a/libcxx/test/std/containers/sequences/forwardlist/types.pass.cpp
+++ b/libcxx/test/std/containers/sequences/forwardlist/types.pass.cpp
@@ -50,11 +50,12 @@ static_assert(std::is_same<my_derived<int, min_allocator<int>>::base, my_base>::
 static_assert(std::is_same<my_derived<my_base, min_allocator<my_base>>::base, my_base>::value, "");
 #endif
 
-struct A { std::forward_list<A> v; }; // incomplete type support
+struct A {
+  std::forward_list<A> v;
+}; // incomplete type support
 
-int main(int, char**)
-{
-    {
+int main(int, char**) {
+  {
     typedef std::forward_list<char> C;
     static_assert((std::is_same<C::value_type, char>::value), "");
     static_assert((std::is_same<C::allocator_type, std::allocator<char> >::value), "");
@@ -68,12 +69,14 @@ int main(int, char**)
     static_assert((std::is_signed<typename C::difference_type>::value), "");
     static_assert((std::is_unsigned<typename C::size_type>::value), "");
     static_assert((std::is_same<typename C::difference_type,
-        typename std::iterator_traits<typename C::iterator>::difference_type>::value), "");
+                                typename std::iterator_traits<typename C::iterator>::difference_type>::value),
+                  "");
     static_assert((std::is_same<typename C::difference_type,
-        typename std::iterator_traits<typename C::const_iterator>::difference_type>::value), "");
-    }
+                                typename std::iterator_traits<typename C::const_iterator>::difference_type>::value),
+                  "");
+  }
 #if TEST_STD_VER >= 11
-    {
+  {
     typedef std::forward_list<char, min_allocator<char>> C;
     static_assert((std::is_same<C::value_type, char>::value), "");
     static_assert((std::is_same<C::allocator_type, min_allocator<char> >::value), "");
@@ -81,17 +84,19 @@ int main(int, char**)
     static_assert((std::is_same<C::const_reference, const char&>::value), "");
     static_assert((std::is_same<C::pointer, min_pointer<char>>::value), "");
     static_assert((std::is_same<C::const_pointer, min_pointer<const char>>::value), "");
-//  min_allocator doesn't have a size_type, so one gets synthesized
+    //  min_allocator doesn't have a size_type, so one gets synthesized
     static_assert((std::is_same<C::size_type, std::make_unsigned<C::difference_type>::type>::value), "");
     static_assert((std::is_same<C::difference_type, std::ptrdiff_t>::value), "");
 
     static_assert((std::is_signed<typename C::difference_type>::value), "");
     static_assert((std::is_unsigned<typename C::size_type>::value), "");
     static_assert((std::is_same<typename C::difference_type,
-        typename std::iterator_traits<typename C::iterator>::difference_type>::value), "");
+                                typename std::iterator_traits<typename C::iterator>::difference_type>::value),
+                  "");
     static_assert((std::is_same<typename C::difference_type,
-        typename std::iterator_traits<typename C::const_iterator>::difference_type>::value), "");
-    }
+                                typename std::iterator_traits<typename C::const_iterator>::difference_type>::value),
+                  "");
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/sequences/from_range_sequence_containers.h b/libcxx/test/std/containers/sequences/from_range_sequence_containers.h
index df66bc9180807..0d04d26ef3c31 100644
--- a/libcxx/test/std/containers/sequences/from_range_sequence_containers.h
+++ b/libcxx/test/std/containers/sequences/from_range_sequence_containers.h
@@ -26,10 +26,10 @@
 #include "test_macros.h"
 
 template <class T>
-concept HasSize = requires (const T& value) { value.size(); };
+concept HasSize = requires(const T& value) { value.size(); };
 
 template <class Container, class Range>
-concept HasFromRangeCtr = requires (Range&& range) {
+concept HasFromRangeCtr = requires(Range&& range) {
   Container(std::from_range, std::forward<Range>(range));
   Container(std::from_range, std::forward<Range>(range), std::allocator<typename Container::value_type>());
 };
@@ -53,7 +53,7 @@ constexpr bool test_constraints() {
 }
 
 // Note: `std::array` is used to avoid dealing with `vector<bool>`.
-template <template <class ...> class Container,
+template <template <class...> class Container,
           class T,
           class Iter,
           class Sent,
@@ -86,12 +86,7 @@ constexpr void test_sequence_container_with_input(std::array<T, N>&& input, Vali
   }
 }
 
-template <template <class ...> class Container,
-          class T,
-          class Iter,
-          class Sent,
-          class Alloc,
-          class ValidateFunc>
+template <template <class...> class Container, class T, class Iter, class Sent, class Alloc, class ValidateFunc>
 constexpr void test_sequence_container(ValidateFunc validate) {
   // Normal input.
   test_sequence_container_with_input<Container, T, Iter, Sent, Alloc>(std::array{0, 5, 12, 7, -1, 8, 26}, validate);
@@ -101,7 +96,7 @@ constexpr void test_sequence_container(ValidateFunc validate) {
   test_sequence_container_with_input<Container, T, Iter, Sent, Alloc>(std::array{5}, validate);
 }
 
-template <template <class ...> class Container>
+template <template <class...> class Container>
 constexpr void test_sequence_container_move_only() {
   MoveOnly input[5];
   std::ranges::subrange in(std::move_iterator{input}, std::move_iterator{input + 5});
@@ -109,10 +104,7 @@ constexpr void test_sequence_container_move_only() {
   [[maybe_unused]] Container<MoveOnly> c(std::from_range, in);
 }
 
-template <class Iter,
-          class Sent,
-          class Alloc,
-          class ValidateFunc>
+template <class Iter, class Sent, class Alloc, class ValidateFunc>
 constexpr void test_vector_bool(ValidateFunc validate) {
   // Normal input.
   test_sequence_container_with_input<std::vector, bool, Iter, Sent, Alloc>(
@@ -123,18 +115,18 @@ constexpr void test_vector_bool(ValidateFunc validate) {
   test_sequence_container_with_input<std::vector, bool, Iter, Sent, Alloc>(std::array{true}, validate);
 }
 
-template <template <class ...> class Container>
+template <template <class...> class Container>
 void test_exception_safety_throwing_copy() {
 #if !defined(TEST_HAS_NO_EXCEPTIONS)
   constexpr int ThrowOn = 3;
-  using T = ThrowingCopy<ThrowOn>;
+  using T               = ThrowingCopy<ThrowOn>;
   test_exception_safety_throwing_copy<ThrowOn, /*Size=*/5>([](T* from, T* to) {
     [[maybe_unused]] Container<T> c(std::from_range, std::ranges::subrange(from, to));
   });
 #endif
 }
 
-template <template <class ...> class Container, class T>
+template <template <class...> class Container, class T>
 void test_exception_safety_throwing_allocator() {
 #if !defined(TEST_HAS_NO_EXCEPTIONS)
   T in[] = {0, 1};
diff --git a/libcxx/test/std/containers/sequences/insert_range_sequence_containers.h b/libcxx/test/std/containers/sequences/insert_range_sequence_containers.h
index 352ee474cae76..9f404c46df778 100644
--- a/libcxx/test/std/containers/sequences/insert_range_sequence_containers.h
+++ b/libcxx/test/std/containers/sequences/insert_range_sequence_containers.h
@@ -31,9 +31,7 @@
 #include "type_algorithms.h"
 
 template <class Container, class Range>
-concept HasInsertRange = requires (Container& c, Range&& range) {
-  c.insert_range(c.end(), range);
-};
+concept HasInsertRange = requires(Container& c, Range&& range) { c.insert_range(c.end(), range); };
 
 template <template <class...> class Container, class T, class U>
 constexpr bool test_constraints_insert_range() {
@@ -52,9 +50,7 @@ constexpr bool test_constraints_insert_range() {
 }
 
 template <class Container, class Range>
-concept HasAppendRange = requires (Container& c, Range&& range) {
-  c.append_range(range);
-};
+concept HasAppendRange = requires(Container& c, Range&& range) { c.append_range(range); };
 
 template <template <class...> class Container, class T, class U>
 constexpr bool test_constraints_append_range() {
@@ -73,9 +69,7 @@ constexpr bool test_constraints_append_range() {
 }
 
 template <class Container, class Range>
-concept HasPrependRange = requires (Container& c, Range&& range) {
-  c.prepend_range(range);
-};
+concept HasPrependRange = requires(Container& c, Range&& range) { c.prepend_range(range); };
 
 template <template <class...> class Container, class T, class U>
 constexpr bool test_constraints_prepend_range() {
@@ -94,9 +88,7 @@ constexpr bool test_constraints_prepend_range() {
 }
 
 template <class Container, class Range>
-concept HasAssignRange = requires (Container& c, Range&& range) {
-  c.assign_range(range);
-};
+concept HasAssignRange = requires(Container& c, Range&& range) { c.assign_range(range); };
 
 template <template <class...> class Container, class T, class U>
 constexpr bool test_constraints_assign_range() {
@@ -117,322 +109,301 @@ constexpr bool test_constraints_assign_range() {
 // Empty container.
 
 template <class T>
-TestCase<T> constexpr EmptyContainer_EmptyRange {
-  .initial = {}, .index = 0, .input = {}, .expected = {}
-};
+TestCase<T> constexpr EmptyContainer_EmptyRange{.initial = {}, .index = 0, .input = {}, .expected = {}};
 // Note: specializations for `bool` still use `vector<int>` for inputs. This is to avoid dealing with `vector<bool>` and
 // its iterators over proxy types.
-template <> constexpr TestCase<int> EmptyContainer_EmptyRange<bool> {
-  .initial = {}, .index = 0, .input = {}, .expected = {}
-};
-
-template <class T> constexpr TestCase<T> EmptyContainer_OneElementRange;
-template <> constexpr TestCase<int> EmptyContainer_OneElementRange<int> {
-  .initial = {}, .index = 0, .input = {5}, .expected = {5}
-};
-template <> constexpr TestCase<char> EmptyContainer_OneElementRange<char> {
-  .initial = {}, .index = 0, .input = "a", .expected = "a"
-};
-template <> constexpr TestCase<int> EmptyContainer_OneElementRange<bool> {
-  .initial = {}, .index = 0, .input = {true}, .expected = {true}
-};
-
-template <class T> constexpr TestCase<T> EmptyContainer_MidRange;
-template <> constexpr TestCase<int> EmptyContainer_MidRange<int> {
-  .initial = {}, .index = 0, .input = {5, 3, 1, 7, 9}, .expected = {5, 3, 1, 7, 9}
-};
-template <> constexpr TestCase<char> EmptyContainer_MidRange<char> {
-  .initial = {}, .index = 0, .input = "aeiou", .expected = "aeiou"
-};
-template <> constexpr TestCase<int> EmptyContainer_MidRange<bool> {
-  .initial = {}, .index = 0, .input = {1, 1, 0, 1, 1}, .expected = {1, 1, 0, 1, 1}
-};
+template <>
+constexpr TestCase<int> EmptyContainer_EmptyRange<bool>{.initial = {}, .index = 0, .input = {}, .expected = {}};
+
+template <class T>
+constexpr TestCase<T> EmptyContainer_OneElementRange;
+template <>
+constexpr TestCase<int> EmptyContainer_OneElementRange<int>{.initial = {}, .index = 0, .input = {5}, .expected = {5}};
+template <>
+constexpr TestCase<char> EmptyContainer_OneElementRange<char>{.initial = {}, .index = 0, .input = "a", .expected = "a"};
+template <>
+constexpr TestCase<int> EmptyContainer_OneElementRange<bool>{
+    .initial = {}, .index = 0, .input = {true}, .expected = {true}};
+
+template <class T>
+constexpr TestCase<T> EmptyContainer_MidRange;
+template <>
+constexpr TestCase<int> EmptyContainer_MidRange<int>{
+    .initial = {}, .index = 0, .input = {5, 3, 1, 7, 9}, .expected = {5, 3, 1, 7, 9}};
+template <>
+constexpr TestCase<char> EmptyContainer_MidRange<char>{
+    .initial = {}, .index = 0, .input = "aeiou", .expected = "aeiou"};
+template <>
+constexpr TestCase<int> EmptyContainer_MidRange<bool>{
+    .initial = {}, .index = 0, .input = {1, 1, 0, 1, 1}, .expected = {1, 1, 0, 1, 1}};
 
 // One-element container.
 
-template <class T> constexpr TestCase<T> OneElementContainer_Begin_EmptyRange;
-template <> constexpr TestCase<int> OneElementContainer_Begin_EmptyRange<int> {
-  .initial = {3}, .index = 0, .input = {}, .expected = {3}
-};
-template <> constexpr TestCase<char> OneElementContainer_Begin_EmptyRange<char> {
-  .initial = "B", .index = 0, .input = {}, .expected = "B"
-};
-template <> constexpr TestCase<int> OneElementContainer_Begin_EmptyRange<bool> {
-  .initial = {0}, .index = 0, .input = {}, .expected = {0}
-};
-
-template <class T> constexpr TestCase<T> OneElementContainer_End_EmptyRange;
-template <> constexpr TestCase<int> OneElementContainer_End_EmptyRange<int> {
-  .initial = {3}, .index = 1, .input = {}, .expected = {3}
-};
-template <> constexpr TestCase<char> OneElementContainer_End_EmptyRange<char> {
-  .initial = "B", .index = 1, .input = {}, .expected = "B"
-};
-template <> constexpr TestCase<int> OneElementContainer_End_EmptyRange<bool> {
-  .initial = {0}, .index = 1, .input = {}, .expected = {0}
-};
-
-template <class T> constexpr TestCase<T> OneElementContainer_Begin_OneElementRange;
-template <> constexpr TestCase<int> OneElementContainer_Begin_OneElementRange<int> {
-  .initial = {3}, .index = 0, .input = {-5}, .expected = {-5, 3}
-};
-template <> constexpr TestCase<char> OneElementContainer_Begin_OneElementRange<char> {
-  .initial = "B", .index = 0, .input = "a", .expected = "aB"
-};
-template <> constexpr TestCase<int> OneElementContainer_Begin_OneElementRange<bool> {
-  .initial = {0}, .index = 0, .input = {1}, .expected = {1, 0}
-};
-
-template <class T> constexpr TestCase<T> OneElementContainer_End_OneElementRange;
-template <> constexpr TestCase<int> OneElementContainer_End_OneElementRange<int> {
-  .initial = {3}, .index = 1, .input = {-5}, .expected = {3, -5}
-};
-template <> constexpr TestCase<char> OneElementContainer_End_OneElementRange<char> {
-  .initial = "B", .index = 1, .input = "a", .expected = "Ba"
-};
-template <> constexpr TestCase<int> OneElementContainer_End_OneElementRange<bool> {
-  .initial = {0}, .index = 1, .input = {1}, .expected = {0, 1}
-};
-
-template <class T> constexpr TestCase<T> OneElementContainer_Begin_MidRange;
-template <> constexpr TestCase<int> OneElementContainer_Begin_MidRange<int> {
-  .initial = {3}, .index = 0, .input = {-5, -3, -1, -7, -9}, .expected = {-5, -3, -1, -7, -9, 3}
-};
-template <> constexpr TestCase<char> OneElementContainer_Begin_MidRange<char> {
-  .initial = "B", .index = 0, .input = "aeiou", .expected = "aeiouB"
-};
-template <> constexpr TestCase<int> OneElementContainer_Begin_MidRange<bool> {
-  .initial = {0}, .index = 0, .input = {1, 1, 0, 1, 1}, .expected = {1, 1, 0, 1, 1, 0}
-};
-
-template <class T> constexpr TestCase<T> OneElementContainer_End_MidRange;
-template <> constexpr TestCase<int> OneElementContainer_End_MidRange<int> {
-  .initial = {3}, .index = 1, .input = {-5, -3, -1, -7, -9}, .expected = {3, -5, -3, -1, -7, -9}
-};
-template <> constexpr TestCase<char> OneElementContainer_End_MidRange<char> {
-  .initial = "B", .index = 1, .input = "aeiou", .expected = "Baeiou"
-};
-template <> constexpr TestCase<int> OneElementContainer_End_MidRange<bool> {
-  .initial = {0}, .index = 1, .input = {1, 1, 0, 1, 1}, .expected = {0, 1, 1, 0, 1, 1}
-};
+template <class T>
+constexpr TestCase<T> OneElementContainer_Begin_EmptyRange;
+template <>
+constexpr TestCase<int> OneElementContainer_Begin_EmptyRange<int>{
+    .initial = {3}, .index = 0, .input = {}, .expected = {3}};
+template <>
+constexpr TestCase<char> OneElementContainer_Begin_EmptyRange<char>{
+    .initial = "B", .index = 0, .input = {}, .expected = "B"};
+template <>
+constexpr TestCase<int> OneElementContainer_Begin_EmptyRange<bool>{
+    .initial = {0}, .index = 0, .input = {}, .expected = {0}};
+
+template <class T>
+constexpr TestCase<T> OneElementContainer_End_EmptyRange;
+template <>
+constexpr TestCase<int> OneElementContainer_End_EmptyRange<int>{
+    .initial = {3}, .index = 1, .input = {}, .expected = {3}};
+template <>
+constexpr TestCase<char> OneElementContainer_End_EmptyRange<char>{
+    .initial = "B", .index = 1, .input = {}, .expected = "B"};
+template <>
+constexpr TestCase<int> OneElementContainer_End_EmptyRange<bool>{
+    .initial = {0}, .index = 1, .input = {}, .expected = {0}};
+
+template <class T>
+constexpr TestCase<T> OneElementContainer_Begin_OneElementRange;
+template <>
+constexpr TestCase<int> OneElementContainer_Begin_OneElementRange<int>{
+    .initial = {3}, .index = 0, .input = {-5}, .expected = {-5, 3}};
+template <>
+constexpr TestCase<char> OneElementContainer_Begin_OneElementRange<char>{
+    .initial = "B", .index = 0, .input = "a", .expected = "aB"};
+template <>
+constexpr TestCase<int> OneElementContainer_Begin_OneElementRange<bool>{
+    .initial = {0}, .index = 0, .input = {1}, .expected = {1, 0}};
+
+template <class T>
+constexpr TestCase<T> OneElementContainer_End_OneElementRange;
+template <>
+constexpr TestCase<int> OneElementContainer_End_OneElementRange<int>{
+    .initial = {3}, .index = 1, .input = {-5}, .expected = {3, -5}};
+template <>
+constexpr TestCase<char> OneElementContainer_End_OneElementRange<char>{
+    .initial = "B", .index = 1, .input = "a", .expected = "Ba"};
+template <>
+constexpr TestCase<int> OneElementContainer_End_OneElementRange<bool>{
+    .initial = {0}, .index = 1, .input = {1}, .expected = {0, 1}};
+
+template <class T>
+constexpr TestCase<T> OneElementContainer_Begin_MidRange;
+template <>
+constexpr TestCase<int> OneElementContainer_Begin_MidRange<int>{
+    .initial = {3}, .index = 0, .input = {-5, -3, -1, -7, -9}, .expected = {-5, -3, -1, -7, -9, 3}};
+template <>
+constexpr TestCase<char> OneElementContainer_Begin_MidRange<char>{
+    .initial = "B", .index = 0, .input = "aeiou", .expected = "aeiouB"};
+template <>
+constexpr TestCase<int> OneElementContainer_Begin_MidRange<bool>{
+    .initial = {0}, .index = 0, .input = {1, 1, 0, 1, 1}, .expected = {1, 1, 0, 1, 1, 0}};
+
+template <class T>
+constexpr TestCase<T> OneElementContainer_End_MidRange;
+template <>
+constexpr TestCase<int> OneElementContainer_End_MidRange<int>{
+    .initial = {3}, .index = 1, .input = {-5, -3, -1, -7, -9}, .expected = {3, -5, -3, -1, -7, -9}};
+template <>
+constexpr TestCase<char> OneElementContainer_End_MidRange<char>{
+    .initial = "B", .index = 1, .input = "aeiou", .expected = "Baeiou"};
+template <>
+constexpr TestCase<int> OneElementContainer_End_MidRange<bool>{
+    .initial = {0}, .index = 1, .input = {1, 1, 0, 1, 1}, .expected = {0, 1, 1, 0, 1, 1}};
 
 // Full container / empty range.
 
-template <class T> constexpr TestCase<T> FullContainer_Begin_EmptyRange;
-template <> constexpr TestCase<int> FullContainer_Begin_EmptyRange<int> {
-  .initial = {11, 29, 35, 14, 84}, .index = 0, .input = {}, .expected = {11, 29, 35, 14, 84}
-};
-template <> constexpr TestCase<char> FullContainer_Begin_EmptyRange<char> {
-  .initial = "_BCD_", .index = 0, .input = {}, .expected = "_BCD_"
-};
-template <> constexpr TestCase<int> FullContainer_Begin_EmptyRange<bool> {
-  .initial = {0, 0, 1, 0, 0}, .index = 0, .input = {}, .expected = {0, 0, 1, 0, 0}
-};
-
-template <class T> constexpr TestCase<T> FullContainer_Mid_EmptyRange;
-template <> constexpr TestCase<int> FullContainer_Mid_EmptyRange<int> {
-  .initial = {11, 29, 35, 14, 84}, .index = 2, .input = {}, .expected = {11, 29, 35, 14, 84}
-};
-template <> constexpr TestCase<char> FullContainer_Mid_EmptyRange<char> {
-  .initial = "_BCD_", .index = 2, .input = {}, .expected = "_BCD_"
-};
-template <> constexpr TestCase<int> FullContainer_Mid_EmptyRange<bool> {
-  .initial = {0, 0, 1, 0, 0}, .index = 2, .input = {}, .expected = {0, 0, 1, 0, 0}
-};
-
-template <class T> constexpr TestCase<T> FullContainer_End_EmptyRange;
-template <> constexpr TestCase<int> FullContainer_End_EmptyRange<int> {
-  .initial = {11, 29, 35, 14, 84}, .index = 5, .input = {}, .expected = {11, 29, 35, 14, 84}
-};
-template <> constexpr TestCase<char> FullContainer_End_EmptyRange<char> {
-  .initial = "_BCD_", .index = 5, .input = {}, .expected = "_BCD_"
-};
-template <> constexpr TestCase<int> FullContainer_End_EmptyRange<bool> {
-  .initial = {0, 0, 1, 0, 0}, .index = 5, .input = {}, .expected = {0, 0, 1, 0, 0}
-};
+template <class T>
+constexpr TestCase<T> FullContainer_Begin_EmptyRange;
+template <>
+constexpr TestCase<int> FullContainer_Begin_EmptyRange<int>{
+    .initial = {11, 29, 35, 14, 84}, .index = 0, .input = {}, .expected = {11, 29, 35, 14, 84}};
+template <>
+constexpr TestCase<char> FullContainer_Begin_EmptyRange<char>{
+    .initial = "_BCD_", .index = 0, .input = {}, .expected = "_BCD_"};
+template <>
+constexpr TestCase<int> FullContainer_Begin_EmptyRange<bool>{
+    .initial = {0, 0, 1, 0, 0}, .index = 0, .input = {}, .expected = {0, 0, 1, 0, 0}};
+
+template <class T>
+constexpr TestCase<T> FullContainer_Mid_EmptyRange;
+template <>
+constexpr TestCase<int> FullContainer_Mid_EmptyRange<int>{
+    .initial = {11, 29, 35, 14, 84}, .index = 2, .input = {}, .expected = {11, 29, 35, 14, 84}};
+template <>
+constexpr TestCase<char> FullContainer_Mid_EmptyRange<char>{
+    .initial = "_BCD_", .index = 2, .input = {}, .expected = "_BCD_"};
+template <>
+constexpr TestCase<int> FullContainer_Mid_EmptyRange<bool>{
+    .initial = {0, 0, 1, 0, 0}, .index = 2, .input = {}, .expected = {0, 0, 1, 0, 0}};
+
+template <class T>
+constexpr TestCase<T> FullContainer_End_EmptyRange;
+template <>
+constexpr TestCase<int> FullContainer_End_EmptyRange<int>{
+    .initial = {11, 29, 35, 14, 84}, .index = 5, .input = {}, .expected = {11, 29, 35, 14, 84}};
+template <>
+constexpr TestCase<char> FullContainer_End_EmptyRange<char>{
+    .initial = "_BCD_", .index = 5, .input = {}, .expected = "_BCD_"};
+template <>
+constexpr TestCase<int> FullContainer_End_EmptyRange<bool>{
+    .initial = {0, 0, 1, 0, 0}, .index = 5, .input = {}, .expected = {0, 0, 1, 0, 0}};
 
 // Full container / one-element range.
 
-template <class T> constexpr TestCase<T> FullContainer_Begin_OneElementRange;
-template <> constexpr TestCase<int> FullContainer_Begin_OneElementRange<int> {
-  .initial = {11, 29, 35, 14, 84}, .index = 0, .input = {-5}, .expected = {-5, 11, 29, 35, 14, 84}
-};
-template <> constexpr TestCase<char> FullContainer_Begin_OneElementRange<char> {
-  .initial = "_BCD_", .index = 0, .input = "a", .expected = "a_BCD_"
-};
-template <> constexpr TestCase<int> FullContainer_Begin_OneElementRange<bool> {
-  .initial = {0, 0, 1, 0, 0}, .index = 0, .input = {1}, .expected = {1, 0, 0, 1, 0, 0}
-};
-
-template <class T> constexpr TestCase<T> FullContainer_Mid_OneElementRange;
-template <> constexpr TestCase<int> FullContainer_Mid_OneElementRange<int> {
-  .initial = {11, 29, 35, 14, 84}, .index = 2, .input = {-5}, .expected = {11, 29, -5, 35, 14, 84}
-};
-template <> constexpr TestCase<char> FullContainer_Mid_OneElementRange<char> {
-  .initial = "_BCD_", .index = 2, .input = "a", .expected = "_BaCD_"
-};
-template <> constexpr TestCase<int> FullContainer_Mid_OneElementRange<bool> {
-  .initial = {0, 0, 1, 0, 0}, .index = 2, .input = {1}, .expected = {0, 0, 1, 1, 0, 0}
-};
-
-template <class T> constexpr TestCase<T> FullContainer_End_OneElementRange;
-template <> constexpr TestCase<int> FullContainer_End_OneElementRange<int> {
-  .initial = {11, 29, 35, 14, 84}, .index = 5, .input = {-5}, .expected = {11, 29, 35, 14, 84, -5}
-};
-template <> constexpr TestCase<char> FullContainer_End_OneElementRange<char> {
-  .initial = "_BCD_", .index = 5, .input = "a", .expected = "_BCD_a"
-};
-template <> constexpr TestCase<int> FullContainer_End_OneElementRange<bool> {
-  .initial = {0, 0, 1, 0, 0}, .index = 5, .input = {1}, .expected = {0, 0, 1, 0, 0, 1}
-};
+template <class T>
+constexpr TestCase<T> FullContainer_Begin_OneElementRange;
+template <>
+constexpr TestCase<int> FullContainer_Begin_OneElementRange<int>{
+    .initial = {11, 29, 35, 14, 84}, .index = 0, .input = {-5}, .expected = {-5, 11, 29, 35, 14, 84}};
+template <>
+constexpr TestCase<char> FullContainer_Begin_OneElementRange<char>{
+    .initial = "_BCD_", .index = 0, .input = "a", .expected = "a_BCD_"};
+template <>
+constexpr TestCase<int> FullContainer_Begin_OneElementRange<bool>{
+    .initial = {0, 0, 1, 0, 0}, .index = 0, .input = {1}, .expected = {1, 0, 0, 1, 0, 0}};
+
+template <class T>
+constexpr TestCase<T> FullContainer_Mid_OneElementRange;
+template <>
+constexpr TestCase<int> FullContainer_Mid_OneElementRange<int>{
+    .initial = {11, 29, 35, 14, 84}, .index = 2, .input = {-5}, .expected = {11, 29, -5, 35, 14, 84}};
+template <>
+constexpr TestCase<char> FullContainer_Mid_OneElementRange<char>{
+    .initial = "_BCD_", .index = 2, .input = "a", .expected = "_BaCD_"};
+template <>
+constexpr TestCase<int> FullContainer_Mid_OneElementRange<bool>{
+    .initial = {0, 0, 1, 0, 0}, .index = 2, .input = {1}, .expected = {0, 0, 1, 1, 0, 0}};
+
+template <class T>
+constexpr TestCase<T> FullContainer_End_OneElementRange;
+template <>
+constexpr TestCase<int> FullContainer_End_OneElementRange<int>{
+    .initial = {11, 29, 35, 14, 84}, .index = 5, .input = {-5}, .expected = {11, 29, 35, 14, 84, -5}};
+template <>
+constexpr TestCase<char> FullContainer_End_OneElementRange<char>{
+    .initial = "_BCD_", .index = 5, .input = "a", .expected = "_BCD_a"};
+template <>
+constexpr TestCase<int> FullContainer_End_OneElementRange<bool>{
+    .initial = {0, 0, 1, 0, 0}, .index = 5, .input = {1}, .expected = {0, 0, 1, 0, 0, 1}};
 
 // Full container / mid-sized range.
 
-template <class T> constexpr TestCase<T> FullContainer_Begin_MidRange;
-template <> constexpr TestCase<int> FullContainer_Begin_MidRange<int> {
-  .initial = {11, 29, 35, 14, 84},
-  .index = 0,
-  .input = {-5, -3, -1, -7, -9},
-  .expected = {-5, -3, -1, -7, -9, 11, 29, 35, 14, 84}
-};
-template <> constexpr TestCase<char> FullContainer_Begin_MidRange<char> {
-  .initial = "_BCD_",
-  .index = 0,
-  .input = "aeiou",
-  .expected = "aeiou_BCD_"
-};
-template <> constexpr TestCase<int> FullContainer_Begin_MidRange<bool> {
-  .initial = {0, 0, 1, 0, 1},
-  .index = 0,
-  .input = {1, 1, 0, 1, 1},
-  .expected = {1, 1, 0, 1, 1, 0, 0, 1, 0, 1}
-};
-
-template <class T> constexpr TestCase<T> FullContainer_Mid_MidRange;
-template <> constexpr TestCase<int> FullContainer_Mid_MidRange<int> {
-  .initial = {11, 29, 35, 14, 84},
-  .index = 2,
-  .input = {-5, -3, -1, -7, -9},
-  .expected = {11, 29, -5, -3, -1, -7, -9, 35, 14, 84}
-};
-template <> constexpr TestCase<char> FullContainer_Mid_MidRange<char> {
-  .initial = "_BCD_",
-  .index = 2,
-  .input = "aeiou",
-  .expected = "_BaeiouCD_"
-};
-template <> constexpr TestCase<int> FullContainer_Mid_MidRange<bool> {
-  .initial = {0, 0, 1, 0, 1},
-  .index = 2,
-  .input = {1, 1, 0, 1, 1},
-  .expected = {0, 0, 1, 1, 0, 1, 1, 1, 0, 1}
-};
-
-template <class T> constexpr TestCase<T> FullContainer_End_MidRange;
-template <> constexpr TestCase<int> FullContainer_End_MidRange<int> {
-  .initial = {11, 29, 35, 14, 84},
-  .index = 5,
-  .input = {-5, -3, -1, -7, -9},
-  .expected = {11, 29, 35, 14, 84, -5, -3, -1, -7, -9}
-};
-template <> constexpr TestCase<char> FullContainer_End_MidRange<char> {
-  .initial = "_BCD_",
-  .index = 5,
-  .input = "aeiou",
-  .expected = "_BCD_aeiou"
-};
-template <> constexpr TestCase<int> FullContainer_End_MidRange<bool> {
-  .initial = {0, 0, 1, 0, 1},
-  .index = 5,
-  .input = {1, 1, 0, 1, 1},
-  .expected = {0, 0, 1, 0, 1, 1, 1, 0, 1, 1}
-};
+template <class T>
+constexpr TestCase<T> FullContainer_Begin_MidRange;
+template <>
+constexpr TestCase<int> FullContainer_Begin_MidRange<int>{
+    .initial  = {11, 29, 35, 14, 84},
+    .index    = 0,
+    .input    = {-5, -3, -1, -7, -9},
+    .expected = {-5, -3, -1, -7, -9, 11, 29, 35, 14, 84}};
+template <>
+constexpr TestCase<char> FullContainer_Begin_MidRange<char>{
+    .initial = "_BCD_", .index = 0, .input = "aeiou", .expected = "aeiou_BCD_"};
+template <>
+constexpr TestCase<int> FullContainer_Begin_MidRange<bool>{
+    .initial = {0, 0, 1, 0, 1}, .index = 0, .input = {1, 1, 0, 1, 1}, .expected = {1, 1, 0, 1, 1, 0, 0, 1, 0, 1}};
+
+template <class T>
+constexpr TestCase<T> FullContainer_Mid_MidRange;
+template <>
+constexpr TestCase<int> FullContainer_Mid_MidRange<int>{
+    .initial  = {11, 29, 35, 14, 84},
+    .index    = 2,
+    .input    = {-5, -3, -1, -7, -9},
+    .expected = {11, 29, -5, -3, -1, -7, -9, 35, 14, 84}};
+template <>
+constexpr TestCase<char> FullContainer_Mid_MidRange<char>{
+    .initial = "_BCD_", .index = 2, .input = "aeiou", .expected = "_BaeiouCD_"};
+template <>
+constexpr TestCase<int> FullContainer_Mid_MidRange<bool>{
+    .initial = {0, 0, 1, 0, 1}, .index = 2, .input = {1, 1, 0, 1, 1}, .expected = {0, 0, 1, 1, 0, 1, 1, 1, 0, 1}};
+
+template <class T>
+constexpr TestCase<T> FullContainer_End_MidRange;
+template <>
+constexpr TestCase<int> FullContainer_End_MidRange<int>{
+    .initial  = {11, 29, 35, 14, 84},
+    .index    = 5,
+    .input    = {-5, -3, -1, -7, -9},
+    .expected = {11, 29, 35, 14, 84, -5, -3, -1, -7, -9}};
+template <>
+constexpr TestCase<char> FullContainer_End_MidRange<char>{
+    .initial = "_BCD_", .index = 5, .input = "aeiou", .expected = "_BCD_aeiou"};
+template <>
+constexpr TestCase<int> FullContainer_End_MidRange<bool>{
+    .initial = {0, 0, 1, 0, 1}, .index = 5, .input = {1, 1, 0, 1, 1}, .expected = {0, 0, 1, 0, 1, 1, 1, 0, 1, 1}};
 
 // Full container / long range.
 
-template <class T> constexpr TestCase<T> FullContainer_Begin_LongRange;
-template <> constexpr TestCase<int> FullContainer_Begin_LongRange<int> {
-  .initial = {11, 29, 35, 14, 84},
-  .index = 0,
-  .input = {-5, -3, -1, -7, -9, -19, -48, -56, -13, -14, -29, -88, -17, -1, -5, -11, -89, -21, -33, -48},
-  .expected = {
-      -5, -3, -1, -7, -9, -19, -48, -56, -13, -14, -29, -88, -17, -1, -5, -11, -89, -21, -33, -48, 11, 29, 35, 14, 84
-  }
-};
-template <> constexpr TestCase<char> FullContainer_Begin_LongRange<char> {
-  .initial = "_BCD_",
-  .index = 0,
-  .input = "aeiouqwxyz5781964203",
-  .expected = "aeiouqwxyz5781964203_BCD_"
-};
-template <> constexpr TestCase<int> FullContainer_Begin_LongRange<bool> {
-  .initial = {0, 0, 1, 0, 0},
-  .index = 0,
-  .input = {1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0},
-  .expected = {
-      1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0
-  }
-};
-
-template <class T> constexpr TestCase<T> FullContainer_Mid_LongRange;
-template <> constexpr TestCase<int> FullContainer_Mid_LongRange<int> {
-  .initial = {11, 29, 35, 14, 84},
-  .index = 2,
-  .input = {-5, -3, -1, -7, -9, -19, -48, -56, -13, -14, -29, -88, -17, -1, -5, -11, -89, -21, -33, -48},
-  .expected = {
-      11, 29, -5, -3, -1, -7, -9, -19, -48, -56, -13, -14, -29, -88, -17, -1, -5, -11, -89, -21, -33, -48, 35, 14, 84
-  }
-};
-template <> constexpr TestCase<char> FullContainer_Mid_LongRange<char> {
-  .initial = "_BCD_",
-  .index = 2,
-  .input = "aeiouqwxyz5781964203",
-  .expected = "_Baeiouqwxyz5781964203CD_"
-};
-template <> constexpr TestCase<int> FullContainer_Mid_LongRange<bool> {
-  .initial = {0, 0, 1, 0, 0},
-  .index = 2,
-  .input = {1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0},
-  .expected = {
-      0, 0, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 1, 0, 0
-  }
-};
-
-template <class T> constexpr TestCase<T> FullContainer_End_LongRange;
-template <> constexpr TestCase<int> FullContainer_End_LongRange<int> {
-  .initial = {11, 29, 35, 14, 84},
-  .index = 5,
-  .input = {-5, -3, -1, -7, -9, -19, -48, -56, -13, -14, -29, -88, -17, -1, -5, -11, -89, -21, -33, -48},
-  .expected = {
-      11, 29, 35, 14, 84, -5, -3, -1, -7, -9, -19, -48, -56, -13, -14, -29, -88, -17, -1, -5, -11, -89, -21, -33, -48
-  }
-};
-template <> constexpr TestCase<char> FullContainer_End_LongRange<char> {
-  .initial = "_BCD_",
-  .index = 5,
-  .input = "aeiouqwxyz5781964203",
-  .expected = "_BCD_aeiouqwxyz5781964203"
-};
-template <> constexpr TestCase<int> FullContainer_End_LongRange<bool> {
-  .initial = {0, 0, 1, 0, 1},
-  .index = 5,
-  .input = {1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0},
-  .expected = {
-      0, 0, 1, 0, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0
-  }
-};
+template <class T>
+constexpr TestCase<T> FullContainer_Begin_LongRange;
+template <>
+constexpr TestCase<int> FullContainer_Begin_LongRange<int>{
+    .initial  = {11, 29, 35, 14, 84},
+    .index    = 0,
+    .input    = {-5, -3, -1, -7, -9, -19, -48, -56, -13, -14, -29, -88, -17, -1, -5, -11, -89, -21, -33, -48},
+    .expected = {-5, -3, -1,  -7,  -9,  -19, -48, -56, -13, -14, -29, -88, -17,
+                 -1, -5, -11, -89, -21, -33, -48, 11,  29,  35,  14,  84}};
+template <>
+constexpr TestCase<char> FullContainer_Begin_LongRange<char>{
+    .initial = "_BCD_", .index = 0, .input = "aeiouqwxyz5781964203", .expected = "aeiouqwxyz5781964203_BCD_"};
+template <>
+constexpr TestCase<int> FullContainer_Begin_LongRange<bool>{
+    .initial  = {0, 0, 1, 0, 0},
+    .index    = 0,
+    .input    = {1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0},
+    .expected = {1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0}};
+
+template <class T>
+constexpr TestCase<T> FullContainer_Mid_LongRange;
+template <>
+constexpr TestCase<int> FullContainer_Mid_LongRange<int>{
+    .initial  = {11, 29, 35, 14, 84},
+    .index    = 2,
+    .input    = {-5, -3, -1, -7, -9, -19, -48, -56, -13, -14, -29, -88, -17, -1, -5, -11, -89, -21, -33, -48},
+    .expected = {11,  29,  -5, -3, -1,  -7,  -9,  -19, -48, -56, -13, -14, -29,
+                 -88, -17, -1, -5, -11, -89, -21, -33, -48, 35,  14,  84}};
+template <>
+constexpr TestCase<char> FullContainer_Mid_LongRange<char>{
+    .initial = "_BCD_", .index = 2, .input = "aeiouqwxyz5781964203", .expected = "_Baeiouqwxyz5781964203CD_"};
+template <>
+constexpr TestCase<int> FullContainer_Mid_LongRange<bool>{
+    .initial  = {0, 0, 1, 0, 0},
+    .index    = 2,
+    .input    = {1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0},
+    .expected = {0, 0, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 1, 0, 0}};
+
+template <class T>
+constexpr TestCase<T> FullContainer_End_LongRange;
+template <>
+constexpr TestCase<int> FullContainer_End_LongRange<int>{
+    .initial  = {11, 29, 35, 14, 84},
+    .index    = 5,
+    .input    = {-5, -3, -1, -7, -9, -19, -48, -56, -13, -14, -29, -88, -17, -1, -5, -11, -89, -21, -33, -48},
+    .expected = {11,  29,  35,  14,  84,  -5, -3, -1,  -7,  -9,  -19, -48, -56,
+                 -13, -14, -29, -88, -17, -1, -5, -11, -89, -21, -33, -48}};
+template <>
+constexpr TestCase<char> FullContainer_End_LongRange<char>{
+    .initial = "_BCD_", .index = 5, .input = "aeiouqwxyz5781964203", .expected = "_BCD_aeiouqwxyz5781964203"};
+template <>
+constexpr TestCase<int> FullContainer_End_LongRange<bool>{
+    .initial  = {0, 0, 1, 0, 1},
+    .index    = 5,
+    .input    = {1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0},
+    .expected = {0, 0, 1, 0, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0}};
 
 // Sequence containers tests.
 
 template <class Container, class Iter, class Sent, class Validate>
 constexpr void test_sequence_insert_range(Validate validate) {
-  using T = typename Container::value_type;
+  using T      = typename Container::value_type;
   using D      = typename Container::difference_type;
   auto get_pos = [](auto& c, auto& test_case) { return std::ranges::next(c.begin(), static_cast<D>(test_case.index)); };
 
   auto test = [&](auto& test_case) {
     Container c(test_case.initial.begin(), test_case.initial.end());
-    auto in = wrap_input<Iter, Sent>(test_case.input);
+    auto in  = wrap_input<Iter, Sent>(test_case.input);
     auto pos = get_pos(c, test_case);
 
     auto result = c.insert_range(pos, in);
@@ -583,13 +554,13 @@ template <class Container, class Iter, class Sent, class Validate>
 constexpr void test_sequence_assign_range(Validate validate) {
   using T = typename Container::value_type;
 
-  auto& initial_empty = EmptyContainer_EmptyRange<T>.initial;
+  auto& initial_empty       = EmptyContainer_EmptyRange<T>.initial;
   auto& initial_one_element = OneElementContainer_Begin_EmptyRange<T>.initial;
-  auto& initial_full = FullContainer_Begin_EmptyRange<T>.initial;
-  auto& input_empty = FullContainer_Begin_EmptyRange<T>.input;
-  auto& input_one_element = FullContainer_Begin_OneElementRange<T>.input;
-  auto& input_mid_range = FullContainer_Begin_MidRange<T>.input;
-  auto& input_long_range = FullContainer_Begin_LongRange<T>.input;
+  auto& initial_full        = FullContainer_Begin_EmptyRange<T>.initial;
+  auto& input_empty         = FullContainer_Begin_EmptyRange<T>.input;
+  auto& input_one_element   = FullContainer_Begin_OneElementRange<T>.input;
+  auto& input_mid_range     = FullContainer_Begin_MidRange<T>.input;
+  auto& input_long_range    = FullContainer_Begin_LongRange<T>.input;
 
   auto test = [&](auto& initial, auto& input) {
     Container c(initial.begin(), initial.end());
@@ -636,7 +607,7 @@ constexpr void test_sequence_assign_range(Validate validate) {
 
 // Move-only types.
 
-template <template <class ...> class Container>
+template <template <class...> class Container>
 constexpr void test_sequence_insert_range_move_only() {
   MoveOnly input[5];
   std::ranges::subrange in(std::move_iterator{input}, std::move_iterator{input + 5});
@@ -645,7 +616,7 @@ constexpr void test_sequence_insert_range_move_only() {
   c.insert_range(c.end(), in);
 }
 
-template <template <class ...> class Container>
+template <template <class...> class Container>
 constexpr void test_sequence_prepend_range_move_only() {
   MoveOnly input[5];
   std::ranges::subrange in(std::move_iterator{input}, std::move_iterator{input + 5});
@@ -654,7 +625,7 @@ constexpr void test_sequence_prepend_range_move_only() {
   c.prepend_range(in);
 }
 
-template <template <class ...> class Container>
+template <template <class...> class Container>
 constexpr void test_sequence_append_range_move_only() {
   MoveOnly input[5];
   std::ranges::subrange in(std::move_iterator{input}, std::move_iterator{input + 5});
@@ -663,7 +634,7 @@ constexpr void test_sequence_append_range_move_only() {
   c.append_range(in);
 }
 
-template <template <class ...> class Container>
+template <template <class...> class Container>
 constexpr void test_sequence_assign_range_move_only() {
   MoveOnly input[5];
   std::ranges::subrange in(std::move_iterator{input}, std::move_iterator{input + 5});
@@ -674,11 +645,11 @@ constexpr void test_sequence_assign_range_move_only() {
 
 // Exception safety.
 
-template <template <class ...> class Container>
+template <template <class...> class Container>
 void test_insert_range_exception_safety_throwing_copy() {
 #if !defined(TEST_HAS_NO_EXCEPTIONS)
   constexpr int ThrowOn = 3;
-  using T = ThrowingCopy<ThrowOn>;
+  using T               = ThrowingCopy<ThrowOn>;
   test_exception_safety_throwing_copy<ThrowOn, /*Size=*/5>([](T* from, T* to) {
     Container<T> c;
     c.insert_range(c.end(), std::ranges::subrange(from, to));
@@ -686,7 +657,7 @@ void test_insert_range_exception_safety_throwing_copy() {
 #endif
 }
 
-template <template <class ...> class Container, class T>
+template <template <class...> class Container, class T>
 void test_insert_range_exception_safety_throwing_allocator() {
 #if !defined(TEST_HAS_NO_EXCEPTIONS)
   T in[] = {0, 1};
@@ -705,11 +676,11 @@ void test_insert_range_exception_safety_throwing_allocator() {
 #endif
 }
 
-template <template <class ...> class Container>
+template <template <class...> class Container>
 void test_prepend_range_exception_safety_throwing_copy() {
 #if !defined(TEST_HAS_NO_EXCEPTIONS)
   constexpr int ThrowOn = 3;
-  using T = ThrowingCopy<ThrowOn>;
+  using T               = ThrowingCopy<ThrowOn>;
   test_exception_safety_throwing_copy<ThrowOn, /*Size=*/5>([](T* from, T* to) {
     Container<T> c;
     c.prepend_range(std::ranges::subrange(from, to));
@@ -717,7 +688,7 @@ void test_prepend_range_exception_safety_throwing_copy() {
 #endif
 }
 
-template <template <class ...> class Container, class T>
+template <template <class...> class Container, class T>
 void test_prepend_range_exception_safety_throwing_allocator() {
 #if !defined(TEST_HAS_NO_EXCEPTIONS)
   T in[] = {0, 1};
@@ -736,11 +707,11 @@ void test_prepend_range_exception_safety_throwing_allocator() {
 #endif
 }
 
-template <template <class ...> class Container>
+template <template <class...> class Container>
 void test_append_range_exception_safety_throwing_copy() {
 #if !defined(TEST_HAS_NO_EXCEPTIONS)
   constexpr int ThrowOn = 3;
-  using T = ThrowingCopy<ThrowOn>;
+  using T               = ThrowingCopy<ThrowOn>;
   test_exception_safety_throwing_copy<ThrowOn, /*Size=*/5>([](T* from, T* to) {
     Container<T> c;
     c.append_range(std::ranges::subrange(from, to));
@@ -748,7 +719,7 @@ void test_append_range_exception_safety_throwing_copy() {
 #endif
 }
 
-template <template <class ...> class Container, class T>
+template <template <class...> class Container, class T>
 void test_append_range_exception_safety_throwing_allocator() {
 #if !defined(TEST_HAS_NO_EXCEPTIONS)
   T in[] = {0, 1};
@@ -767,11 +738,11 @@ void test_append_range_exception_safety_throwing_allocator() {
 #endif
 }
 
-template <template <class ...> class Container>
+template <template <class...> class Container>
 void test_assign_range_exception_safety_throwing_copy() {
 #if !defined(TEST_HAS_NO_EXCEPTIONS)
   constexpr int ThrowOn = 3;
-  using T = ThrowingCopy<ThrowOn>;
+  using T               = ThrowingCopy<ThrowOn>;
   test_exception_safety_throwing_copy<ThrowOn, /*Size=*/5>([](T* from, T* to) {
     Container<T> c;
     c.assign_range(std::ranges::subrange(from, to));
@@ -779,7 +750,7 @@ void test_assign_range_exception_safety_throwing_copy() {
 #endif
 }
 
-template <template <class ...> class Container, class T>
+template <template <class...> class Container, class T>
 void test_assign_range_exception_safety_throwing_allocator() {
 #if !defined(TEST_HAS_NO_EXCEPTIONS)
   T in[] = {0, 1};
diff --git a/libcxx/test/std/containers/sequences/list/allocator_mismatch.verify.cpp b/libcxx/test/std/containers/sequences/list/allocator_mismatch.verify.cpp
index 981e2face7ede..ff67c03261002 100644
--- a/libcxx/test/std/containers/sequences/list/allocator_mismatch.verify.cpp
+++ b/libcxx/test/std/containers/sequences/list/allocator_mismatch.verify.cpp
@@ -12,4 +12,4 @@
 #include <list>
 
 std::list<int, std::allocator<long> > l;
-  // expected-error-re@*:* {{static assertion failed{{.*}}Allocator::value_type must be same type as value_type}}
+// expected-error-re@*:* {{static assertion failed{{.*}}Allocator::value_type must be same type as value_type}}
diff --git a/libcxx/test/std/containers/sequences/list/compare.pass.cpp b/libcxx/test/std/containers/sequences/list/compare.pass.cpp
index 1d6dd80745447..ce00f57733bf9 100644
--- a/libcxx/test/std/containers/sequences/list/compare.pass.cpp
+++ b/libcxx/test/std/containers/sequences/list/compare.pass.cpp
@@ -38,80 +38,80 @@
 #include "test_comparisons.h"
 
 int main(int, char**) {
-    {
-        const std::list<int> l1, l2;
-        assert(testComparisons(l1, l2, true, false));
-    }
-    {
-        const std::list<int> l1(1, 1), l2(1, 1);
-        assert(testComparisons(l1, l2, true, false));
-    }
-    {
-        int items[3] = {1, 2, 3};
-        const std::list<int> l1(items, items + 3);
-        const std::list<int> l2(items, items + 3);
-        assert(testComparisons(l1, l2, true, false));
-    }
-    {
-        const std::list<int> l1(1, 1), l2;
-        assert(testComparisons(l1, l2, false, false));
-    }
-    {
-        const std::list<int> l1(1, 1), l2(1, 2);
-        assert(testComparisons(l1, l2, false, true));
-    }
-    {
-        int items1[2] = {1, 2};
-        int items2[2] = {1, 3};
-        const std::list<int> l1(items1, items1 + 2);
-        const std::list<int> l2(items2, items2 + 2);
-        assert(testComparisons(l1, l2, false, true));
-    }
-    {
-        int items1[2] = {2, 2};
-        int items2[2] = {1, 3};
-        const std::list<int> l1(items1, items1 + 2);
-        const std::list<int> l2(items2, items2 + 2);
-        assert(testComparisons(l1, l2, false, false));
-    }
-    {
-        const std::list<LessAndEqComp> l1, l2;
-        assert(testComparisons(l1, l2, true, false));
-    }
-    {
-        const std::list<LessAndEqComp> l1(1, LessAndEqComp(1));
-        const std::list<LessAndEqComp> l2(1, LessAndEqComp(1));
-        assert(testComparisons(l1, l2, true, false));
-    }
-    {
-        LessAndEqComp items[3] = {LessAndEqComp(1), LessAndEqComp(2), LessAndEqComp(3)};
-        const std::list<LessAndEqComp> l1(items, items + 3);
-        const std::list<LessAndEqComp> l2(items, items + 3);
-        assert(testComparisons(l1, l2, true, false));
-    }
-    {
-        const std::list<LessAndEqComp> l1(1, LessAndEqComp(1));
-        const std::list<LessAndEqComp> l2;
-        assert(testComparisons(l1, l2, false, false));
-    }
-    {
-        const std::list<LessAndEqComp> l1(1, LessAndEqComp(1));
-        const std::list<LessAndEqComp> l2(1, LessAndEqComp(2));
-        assert(testComparisons(l1, l2, false, true));
-    }
-    {
-        LessAndEqComp items1[2] = {LessAndEqComp(1), LessAndEqComp(2)};
-        LessAndEqComp items2[2] = {LessAndEqComp(1), LessAndEqComp(3)};
-        const std::list<LessAndEqComp> l1(items1, items1 + 2);
-        const std::list<LessAndEqComp> l2(items2, items2 + 2);
-        assert(testComparisons(l1, l2, false, true));
-    }
-    {
-        LessAndEqComp items1[2] = {LessAndEqComp(2), LessAndEqComp(2)};
-        LessAndEqComp items2[2] = {LessAndEqComp(1), LessAndEqComp(3)};
-        const std::list<LessAndEqComp> l1(items1, items1 + 2);
-        const std::list<LessAndEqComp> l2(items2, items2 + 2);
-        assert(testComparisons(l1, l2, false, false));
-    }
-    return 0;
+  {
+    const std::list<int> l1, l2;
+    assert(testComparisons(l1, l2, true, false));
+  }
+  {
+    const std::list<int> l1(1, 1), l2(1, 1);
+    assert(testComparisons(l1, l2, true, false));
+  }
+  {
+    int items[3] = {1, 2, 3};
+    const std::list<int> l1(items, items + 3);
+    const std::list<int> l2(items, items + 3);
+    assert(testComparisons(l1, l2, true, false));
+  }
+  {
+    const std::list<int> l1(1, 1), l2;
+    assert(testComparisons(l1, l2, false, false));
+  }
+  {
+    const std::list<int> l1(1, 1), l2(1, 2);
+    assert(testComparisons(l1, l2, false, true));
+  }
+  {
+    int items1[2] = {1, 2};
+    int items2[2] = {1, 3};
+    const std::list<int> l1(items1, items1 + 2);
+    const std::list<int> l2(items2, items2 + 2);
+    assert(testComparisons(l1, l2, false, true));
+  }
+  {
+    int items1[2] = {2, 2};
+    int items2[2] = {1, 3};
+    const std::list<int> l1(items1, items1 + 2);
+    const std::list<int> l2(items2, items2 + 2);
+    assert(testComparisons(l1, l2, false, false));
+  }
+  {
+    const std::list<LessAndEqComp> l1, l2;
+    assert(testComparisons(l1, l2, true, false));
+  }
+  {
+    const std::list<LessAndEqComp> l1(1, LessAndEqComp(1));
+    const std::list<LessAndEqComp> l2(1, LessAndEqComp(1));
+    assert(testComparisons(l1, l2, true, false));
+  }
+  {
+    LessAndEqComp items[3] = {LessAndEqComp(1), LessAndEqComp(2), LessAndEqComp(3)};
+    const std::list<LessAndEqComp> l1(items, items + 3);
+    const std::list<LessAndEqComp> l2(items, items + 3);
+    assert(testComparisons(l1, l2, true, false));
+  }
+  {
+    const std::list<LessAndEqComp> l1(1, LessAndEqComp(1));
+    const std::list<LessAndEqComp> l2;
+    assert(testComparisons(l1, l2, false, false));
+  }
+  {
+    const std::list<LessAndEqComp> l1(1, LessAndEqComp(1));
+    const std::list<LessAndEqComp> l2(1, LessAndEqComp(2));
+    assert(testComparisons(l1, l2, false, true));
+  }
+  {
+    LessAndEqComp items1[2] = {LessAndEqComp(1), LessAndEqComp(2)};
+    LessAndEqComp items2[2] = {LessAndEqComp(1), LessAndEqComp(3)};
+    const std::list<LessAndEqComp> l1(items1, items1 + 2);
+    const std::list<LessAndEqComp> l2(items2, items2 + 2);
+    assert(testComparisons(l1, l2, false, true));
+  }
+  {
+    LessAndEqComp items1[2] = {LessAndEqComp(2), LessAndEqComp(2)};
+    LessAndEqComp items2[2] = {LessAndEqComp(1), LessAndEqComp(3)};
+    const std::list<LessAndEqComp> l1(items1, items1 + 2);
+    const std::list<LessAndEqComp> l2(items2, items2 + 2);
+    assert(testComparisons(l1, l2, false, false));
+  }
+  return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/list/exception_safety.pass.cpp b/libcxx/test/std/containers/sequences/list/exception_safety.pass.cpp
index c734d18f2bb2f..9820792f748c2 100644
--- a/libcxx/test/std/containers/sequences/list/exception_safety.pass.cpp
+++ b/libcxx/test/std/containers/sequences/list/exception_safety.pass.cpp
@@ -58,29 +58,29 @@
 #include "test_macros.h"
 
 #if TEST_STD_VER >= 23
-#include <ranges>
+#  include <ranges>
 #endif
 
 int main(int, char**) {
   {
     constexpr int ThrowOn = 1;
-    constexpr int Size = 1;
-    using T = ThrowingCopy<ThrowOn>;
+    constexpr int Size    = 1;
+    using T               = ThrowingCopy<ThrowOn>;
 
     // void push_front(const value_type& v);
-    test_exception_safety_throwing_copy<ThrowOn, Size>([](T* from, T*){
+    test_exception_safety_throwing_copy<ThrowOn, Size>([](T* from, T*) {
       std::list<T> c;
       c.push_front(*from);
     });
 
     // void push_back(const value_type& v);
-    test_exception_safety_throwing_copy<ThrowOn, Size>([](T* from, T*){
+    test_exception_safety_throwing_copy<ThrowOn, Size>([](T* from, T*) {
       std::list<T> c;
       c.push_back(*from);
     });
 
     // iterator insert(const_iterator p, const value_type& v);
-    test_exception_safety_throwing_copy</*ThrowOn=*/1, Size>([](T* from, T*){
+    test_exception_safety_throwing_copy</*ThrowOn=*/1, Size>([](T* from, T*) {
       std::list<T> c;
       c.insert(c.end(), *from);
     });
@@ -88,33 +88,33 @@ int main(int, char**) {
 
   {
     constexpr int ThrowOn = 3;
-    constexpr int Size = 5;
-    using T = ThrowingCopy<ThrowOn>;
-    using C = std::list<T>;
-    using Alloc = std::allocator<T>;
+    constexpr int Size    = 5;
+    using T               = ThrowingCopy<ThrowOn>;
+    using C               = std::list<T>;
+    using Alloc           = std::allocator<T>;
 
     // list(size_type n, const value_type& v);
-    test_exception_safety_throwing_copy<ThrowOn, Size>([](T* from, T*){
+    test_exception_safety_throwing_copy<ThrowOn, Size>([](T* from, T*) {
       std::list<T> c(Size, *from);
       (void)c;
     });
 
     // list(size_type n, const value_type& v, const allocator_type& a);
-    test_exception_safety_throwing_copy<ThrowOn, Size>([](T* from, T*){
+    test_exception_safety_throwing_copy<ThrowOn, Size>([](T* from, T*) {
       std::list<T> c(Size, *from, Alloc());
       (void)c;
     });
 
     // template <class InputIterator>
     //     list(InputIterator first, InputIterator last);
-    test_exception_safety_throwing_copy<ThrowOn, Size>([](T* from, T* to){
+    test_exception_safety_throwing_copy<ThrowOn, Size>([](T* from, T* to) {
       std::list<T> c(from, to);
       (void)c;
     });
 
     // template <class InputIterator>
     //     list(InputIterator first, InputIterator last, const allocator_type& a);
-    test_exception_safety_throwing_copy<ThrowOn, Size>([](T* from, T* to){
+    test_exception_safety_throwing_copy<ThrowOn, Size>([](T* from, T* to) {
       std::list<T> c(from, to, Alloc());
       (void)c;
     });
@@ -122,7 +122,7 @@ int main(int, char**) {
 #if TEST_STD_VER >= 23
     // template<container-compatible-range<T> R>
     //     list(from_range_t, R&& rg, const Allocator& = Allocator()); // C++23
-    test_exception_safety_throwing_copy<ThrowOn, Size>([](T* from, T* to){
+    test_exception_safety_throwing_copy<ThrowOn, Size>([](T* from, T* to) {
       {
         std::list<T> c(std::from_range, std::ranges::subrange(from, to));
         (void)c;
diff --git a/libcxx/test/std/containers/sequences/list/get_allocator.pass.cpp b/libcxx/test/std/containers/sequences/list/get_allocator.pass.cpp
index 895fc219086a8..f1002f2ca8113 100644
--- a/libcxx/test/std/containers/sequences/list/get_allocator.pass.cpp
+++ b/libcxx/test/std/containers/sequences/list/get_allocator.pass.cpp
@@ -19,16 +19,16 @@
 #include "test_macros.h"
 
 int main(int, char**) {
-    {
-        std::allocator<int> alloc;
-        const std::list<int> l(alloc);
-        assert(l.get_allocator() == alloc);
-    }
-    {
-        other_allocator<int> alloc(1);
-        const std::list<int, other_allocator<int> > l(alloc);
-        assert(l.get_allocator() == alloc);
-    }
+  {
+    std::allocator<int> alloc;
+    const std::list<int> l(alloc);
+    assert(l.get_allocator() == alloc);
+  }
+  {
+    other_allocator<int> alloc(1);
+    const std::list<int, other_allocator<int> > l(alloc);
+    assert(l.get_allocator() == alloc);
+  }
 
-    return 0;
+  return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/list/incomplete_type.pass.cpp b/libcxx/test/std/containers/sequences/list/incomplete_type.pass.cpp
index 103117fb40fd3..1802e53ecf387 100644
--- a/libcxx/test/std/containers/sequences/list/incomplete_type.pass.cpp
+++ b/libcxx/test/std/containers/sequences/list/incomplete_type.pass.cpp
@@ -16,15 +16,15 @@
 #include "test_macros.h"
 
 struct A {
-    std::list<A> l;
-    std::list<A>::iterator it;
-    std::list<A>::const_iterator cit;
-    std::list<A>::reverse_iterator rit;
-    std::list<A>::const_reverse_iterator crit;
+  std::list<A> l;
+  std::list<A>::iterator it;
+  std::list<A>::const_iterator cit;
+  std::list<A>::reverse_iterator rit;
+  std::list<A>::const_reverse_iterator crit;
 };
 
 int main(int, char**) {
-    A a;
+  A a;
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/list/iterator_concept_conformance.compile.pass.cpp b/libcxx/test/std/containers/sequences/list/iterator_concept_conformance.compile.pass.cpp
index bfa31feb416df..102c7ef7f33d8 100644
--- a/libcxx/test/std/containers/sequences/list/iterator_concept_conformance.compile.pass.cpp
+++ b/libcxx/test/std/containers/sequences/list/iterator_concept_conformance.compile.pass.cpp
@@ -14,11 +14,11 @@
 
 #include <iterator>
 
-using iterator = std::list<int>::iterator;
-using const_iterator = std::list<int>::const_iterator;
-using reverse_iterator = std::list<int>::reverse_iterator;
+using iterator               = std::list<int>::iterator;
+using const_iterator         = std::list<int>::const_iterator;
+using reverse_iterator       = std::list<int>::reverse_iterator;
 using const_reverse_iterator = std::list<int>::const_reverse_iterator;
-using value_type = int;
+using value_type             = int;
 
 static_assert(std::bidirectional_iterator<iterator>);
 static_assert(!std::random_access_iterator<iterator>);
@@ -31,20 +31,20 @@ static_assert(!std::sized_sentinel_for<iterator, iterator>);
 static_assert(!std::sized_sentinel_for<iterator, const_iterator>);
 static_assert(!std::sized_sentinel_for<iterator, reverse_iterator>);
 static_assert(!std::sized_sentinel_for<iterator, const_reverse_iterator>);
-static_assert( std::indirectly_movable<iterator, iterator>);
-static_assert( std::indirectly_movable_storable<iterator, iterator>);
+static_assert(std::indirectly_movable<iterator, iterator>);
+static_assert(std::indirectly_movable_storable<iterator, iterator>);
 static_assert(!std::indirectly_movable<iterator, const_iterator>);
 static_assert(!std::indirectly_movable_storable<iterator, const_iterator>);
-static_assert( std::indirectly_movable<iterator, reverse_iterator>);
-static_assert( std::indirectly_movable_storable<iterator, reverse_iterator>);
+static_assert(std::indirectly_movable<iterator, reverse_iterator>);
+static_assert(std::indirectly_movable_storable<iterator, reverse_iterator>);
 static_assert(!std::indirectly_movable<iterator, const_reverse_iterator>);
 static_assert(!std::indirectly_movable_storable<iterator, const_reverse_iterator>);
-static_assert( std::indirectly_copyable<iterator, iterator>);
-static_assert( std::indirectly_copyable_storable<iterator, iterator>);
+static_assert(std::indirectly_copyable<iterator, iterator>);
+static_assert(std::indirectly_copyable_storable<iterator, iterator>);
 static_assert(!std::indirectly_copyable<iterator, const_iterator>);
 static_assert(!std::indirectly_copyable_storable<iterator, const_iterator>);
-static_assert( std::indirectly_copyable<iterator, reverse_iterator>);
-static_assert( std::indirectly_copyable_storable<iterator, reverse_iterator>);
+static_assert(std::indirectly_copyable<iterator, reverse_iterator>);
+static_assert(std::indirectly_copyable_storable<iterator, reverse_iterator>);
 static_assert(!std::indirectly_copyable<iterator, const_reverse_iterator>);
 static_assert(!std::indirectly_copyable_storable<iterator, const_reverse_iterator>);
 static_assert(std::indirectly_swappable<iterator, iterator>);
@@ -60,20 +60,20 @@ static_assert(!std::sized_sentinel_for<const_iterator, iterator>);
 static_assert(!std::sized_sentinel_for<const_iterator, const_iterator>);
 static_assert(!std::sized_sentinel_for<const_iterator, reverse_iterator>);
 static_assert(!std::sized_sentinel_for<const_iterator, const_reverse_iterator>);
-static_assert( std::indirectly_movable<const_iterator, iterator>);
-static_assert( std::indirectly_movable_storable<const_iterator, iterator>);
+static_assert(std::indirectly_movable<const_iterator, iterator>);
+static_assert(std::indirectly_movable_storable<const_iterator, iterator>);
 static_assert(!std::indirectly_movable<const_iterator, const_iterator>);
 static_assert(!std::indirectly_movable_storable<const_iterator, const_iterator>);
-static_assert( std::indirectly_movable<const_iterator, reverse_iterator>);
-static_assert( std::indirectly_movable_storable<const_iterator, reverse_iterator>);
+static_assert(std::indirectly_movable<const_iterator, reverse_iterator>);
+static_assert(std::indirectly_movable_storable<const_iterator, reverse_iterator>);
 static_assert(!std::indirectly_movable<const_iterator, const_reverse_iterator>);
 static_assert(!std::indirectly_movable_storable<const_iterator, const_reverse_iterator>);
-static_assert( std::indirectly_copyable<const_iterator, iterator>);
-static_assert( std::indirectly_copyable_storable<const_iterator, iterator>);
+static_assert(std::indirectly_copyable<const_iterator, iterator>);
+static_assert(std::indirectly_copyable_storable<const_iterator, iterator>);
 static_assert(!std::indirectly_copyable<const_iterator, const_iterator>);
 static_assert(!std::indirectly_copyable_storable<const_iterator, const_iterator>);
-static_assert( std::indirectly_copyable<const_iterator, reverse_iterator>);
-static_assert( std::indirectly_copyable_storable<const_iterator, reverse_iterator>);
+static_assert(std::indirectly_copyable<const_iterator, reverse_iterator>);
+static_assert(std::indirectly_copyable_storable<const_iterator, reverse_iterator>);
 static_assert(!std::indirectly_copyable<const_iterator, const_reverse_iterator>);
 static_assert(!std::indirectly_copyable_storable<const_iterator, const_reverse_iterator>);
 static_assert(!std::indirectly_swappable<const_iterator, const_iterator>);
diff --git a/libcxx/test/std/containers/sequences/list/iterators.pass.cpp b/libcxx/test/std/containers/sequences/list/iterators.pass.cpp
index 0fe92dfba694f..deaae31f2d27c 100644
--- a/libcxx/test/std/containers/sequences/list/iterators.pass.cpp
+++ b/libcxx/test/std/containers/sequences/list/iterators.pass.cpp
@@ -22,136 +22,133 @@
 #include "test_macros.h"
 #include "min_allocator.h"
 
-struct A
-{
-    int first;
-    int second;
+struct A {
+  int first;
+  int second;
 };
 
-int main(int, char**)
-{
-    {
-        typedef int T;
-        typedef std::list<T> C;
-        C c;
-        C::iterator i = c.begin();
-        C::iterator j = c.end();
-        assert(std::distance(i, j) == 0);
-        assert(i == j);
-    }
-    {
-        typedef int T;
-        typedef std::list<T> C;
-        const C c;
-        C::const_iterator i = c.begin();
-        C::const_iterator j = c.end();
-        assert(std::distance(i, j) == 0);
-        assert(i == j);
-    }
-    {
-        typedef int T;
-        typedef std::list<T> C;
-        C c;
-        C::const_iterator i = c.cbegin();
-        C::const_iterator j = c.cend();
-        assert(std::distance(i, j) == 0);
-        assert(i == j);
-        assert(i == c.end());
-    }
-    {
-        typedef int T;
-        typedef std::list<T> C;
-        const T t[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9};
-        C c(std::begin(t), std::end(t));
-        C::iterator i = c.begin();
-        assert(*i == 0);
-        ++i;
-        assert(*i == 1);
-        *i = 10;
-        assert(*i == 10);
-        assert(std::distance(c.begin(), c.end()) == 10);
-    }
-    {
-        typedef int T;
-        typedef std::list<T> C;
-        C::iterator i;
-        C::const_iterator j;
-    }
+int main(int, char**) {
+  {
+    typedef int T;
+    typedef std::list<T> C;
+    C c;
+    C::iterator i = c.begin();
+    C::iterator j = c.end();
+    assert(std::distance(i, j) == 0);
+    assert(i == j);
+  }
+  {
+    typedef int T;
+    typedef std::list<T> C;
+    const C c;
+    C::const_iterator i = c.begin();
+    C::const_iterator j = c.end();
+    assert(std::distance(i, j) == 0);
+    assert(i == j);
+  }
+  {
+    typedef int T;
+    typedef std::list<T> C;
+    C c;
+    C::const_iterator i = c.cbegin();
+    C::const_iterator j = c.cend();
+    assert(std::distance(i, j) == 0);
+    assert(i == j);
+    assert(i == c.end());
+  }
+  {
+    typedef int T;
+    typedef std::list<T> C;
+    const T t[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9};
+    C c(std::begin(t), std::end(t));
+    C::iterator i = c.begin();
+    assert(*i == 0);
+    ++i;
+    assert(*i == 1);
+    *i = 10;
+    assert(*i == 10);
+    assert(std::distance(c.begin(), c.end()) == 10);
+  }
+  {
+    typedef int T;
+    typedef std::list<T> C;
+    C::iterator i;
+    C::const_iterator j;
+  }
 #if TEST_STD_VER >= 11
-    {
-        typedef int T;
-        typedef std::list<T, min_allocator<T>> C;
-        C c;
-        C::iterator i = c.begin();
-        C::iterator j = c.end();
-        assert(std::distance(i, j) == 0);
-        assert(i == j);
-    }
-    {
-        typedef int T;
-        typedef std::list<T, min_allocator<T>> C;
-        const C c;
-        C::const_iterator i = c.begin();
-        C::const_iterator j = c.end();
-        assert(std::distance(i, j) == 0);
-        assert(i == j);
-    }
-    {
-        typedef int T;
-        typedef std::list<T, min_allocator<T>> C;
-        C c;
-        C::const_iterator i = c.cbegin();
-        C::const_iterator j = c.cend();
-        assert(std::distance(i, j) == 0);
-        assert(i == j);
-        assert(i == c.end());
-    }
-    {
-        typedef int T;
-        typedef std::list<T, min_allocator<T>> C;
-        const T t[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9};
-        C c(std::begin(t), std::end(t));
-        C::iterator i = c.begin();
-        assert(*i == 0);
-        ++i;
-        assert(*i == 1);
-        *i = 10;
-        assert(*i == 10);
-        assert(std::distance(c.begin(), c.end()) == 10);
-    }
-    {
-        typedef int T;
-        typedef std::list<T, min_allocator<T>> C;
-        C::iterator i;
-        C::const_iterator j;
-    }
-    {
-        typedef A T;
-        typedef std::list<T, min_allocator<T>> C;
-        C c = {A{1, 2}};
-        C::iterator i = c.begin();
-        i->first = 3;
-        C::const_iterator j = i;
-        assert(j->first == 3);
-    }
+  {
+    typedef int T;
+    typedef std::list<T, min_allocator<T>> C;
+    C c;
+    C::iterator i = c.begin();
+    C::iterator j = c.end();
+    assert(std::distance(i, j) == 0);
+    assert(i == j);
+  }
+  {
+    typedef int T;
+    typedef std::list<T, min_allocator<T>> C;
+    const C c;
+    C::const_iterator i = c.begin();
+    C::const_iterator j = c.end();
+    assert(std::distance(i, j) == 0);
+    assert(i == j);
+  }
+  {
+    typedef int T;
+    typedef std::list<T, min_allocator<T>> C;
+    C c;
+    C::const_iterator i = c.cbegin();
+    C::const_iterator j = c.cend();
+    assert(std::distance(i, j) == 0);
+    assert(i == j);
+    assert(i == c.end());
+  }
+  {
+    typedef int T;
+    typedef std::list<T, min_allocator<T>> C;
+    const T t[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9};
+    C c(std::begin(t), std::end(t));
+    C::iterator i = c.begin();
+    assert(*i == 0);
+    ++i;
+    assert(*i == 1);
+    *i = 10;
+    assert(*i == 10);
+    assert(std::distance(c.begin(), c.end()) == 10);
+  }
+  {
+    typedef int T;
+    typedef std::list<T, min_allocator<T>> C;
+    C::iterator i;
+    C::const_iterator j;
+  }
+  {
+    typedef A T;
+    typedef std::list<T, min_allocator<T>> C;
+    C c                 = {A{1, 2}};
+    C::iterator i       = c.begin();
+    i->first            = 3;
+    C::const_iterator j = i;
+    assert(j->first == 3);
+  }
 #endif
 #if TEST_STD_VER > 11
-    {
-        std::list<int>::iterator ii1{}, ii2{};
-        std::list<int>::iterator ii4 = ii1;
-        std::list<int>::const_iterator cii{};
-        assert ( ii1 == ii2 );
-        assert ( ii1 == ii4 );
+  {
+    std::list<int>::iterator ii1{}, ii2{};
+    std::list<int>::iterator ii4 = ii1;
+    std::list<int>::const_iterator cii{};
+    assert(ii1 == ii2);
+    assert(ii1 == ii4);
 
-        assert (!(ii1 != ii2 ));
+    assert(!(ii1 != ii2));
 
-        assert ( (ii1 == cii ));
-        assert ( (cii == ii1 ));
-        assert (!(ii1 != cii ));
-        assert (!(cii != ii1 ));
-    }
+    assert((ii1 == cii));
+    assert((cii == ii1));
+    assert(!(ii1 != cii));
+    assert(!(cii != ii1));
+  }
 #endif
 
-
   return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/list/list.capacity/empty.pass.cpp b/libcxx/test/std/containers/sequences/list/list.capacity/empty.pass.cpp
index 7619ec50853ad..50ca23ff9c56c 100644
--- a/libcxx/test/std/containers/sequences/list/list.capacity/empty.pass.cpp
+++ b/libcxx/test/std/containers/sequences/list/list.capacity/empty.pass.cpp
@@ -18,9 +18,8 @@
 #include "test_macros.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
+int main(int, char**) {
+  {
     typedef std::list<int> C;
     C c;
     ASSERT_NOEXCEPT(c.empty());
@@ -29,9 +28,9 @@ int main(int, char**)
     assert(!c.empty());
     c.clear();
     assert(c.empty());
-    }
+  }
 #if TEST_STD_VER >= 11
-    {
+  {
     typedef std::list<int, min_allocator<int>> C;
     C c;
     ASSERT_NOEXCEPT(c.empty());
@@ -40,7 +39,7 @@ int main(int, char**)
     assert(!c.empty());
     c.clear();
     assert(c.empty());
-    }
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/sequences/list/list.capacity/empty.verify.cpp b/libcxx/test/std/containers/sequences/list/list.capacity/empty.verify.cpp
index 77d4e5ab21387..4ecb4cff6853c 100644
--- a/libcxx/test/std/containers/sequences/list/list.capacity/empty.verify.cpp
+++ b/libcxx/test/std/containers/sequences/list/list.capacity/empty.verify.cpp
@@ -17,6 +17,6 @@
 #include <list>
 
 void f() {
-    std::list<int> c;
-    c.empty(); // expected-warning {{ignoring return value of function declared with 'nodiscard' attribute}}
+  std::list<int> c;
+  c.empty(); // expected-warning {{ignoring return value of function declared with 'nodiscard' attribute}}
 }
diff --git a/libcxx/test/std/containers/sequences/list/list.capacity/max_size.pass.cpp b/libcxx/test/std/containers/sequences/list/list.capacity/max_size.pass.cpp
index c39fefe1cb11f..74c2ccfb14421 100644
--- a/libcxx/test/std/containers/sequences/list/list.capacity/max_size.pass.cpp
+++ b/libcxx/test/std/containers/sequences/list/list.capacity/max_size.pass.cpp
@@ -29,16 +29,14 @@ int main(int, char**) {
   {
     typedef limited_allocator<int, (std::size_t)-1> A;
     typedef std::list<int, A> C;
-    const C::size_type max_dist =
-        static_cast<C::size_type>(std::numeric_limits<C::difference_type>::max());
+    const C::size_type max_dist = static_cast<C::size_type>(std::numeric_limits<C::difference_type>::max());
     C c;
     assert(c.max_size() <= max_dist);
     LIBCPP_ASSERT(c.max_size() == max_dist);
   }
   {
     typedef std::list<char> C;
-    const C::size_type max_dist =
-        static_cast<C::size_type>(std::numeric_limits<C::difference_type>::max());
+    const C::size_type max_dist = static_cast<C::size_type>(std::numeric_limits<C::difference_type>::max());
     C c;
     assert(c.max_size() <= max_dist);
     assert(c.max_size() <= alloc_max_size(c.get_allocator()));
diff --git a/libcxx/test/std/containers/sequences/list/list.capacity/resize_size.pass.cpp b/libcxx/test/std/containers/sequences/list/list.capacity/resize_size.pass.cpp
index b27b9a3f5ace9..754d931646cc6 100644
--- a/libcxx/test/std/containers/sequences/list/list.capacity/resize_size.pass.cpp
+++ b/libcxx/test/std/containers/sequences/list/list.capacity/resize_size.pass.cpp
@@ -16,64 +16,63 @@
 #include "DefaultOnly.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        std::list<int> l(5, 2);
-        l.resize(2);
-        assert(l.size() == 2);
-        assert(std::distance(l.begin(), l.end()) == 2);
-        assert(l == std::list<int>(2, 2));
-    }
-    {
-        std::list<int> l(5, 2);
-        l.resize(10);
-        assert(l.size() == 10);
-        assert(std::distance(l.begin(), l.end()) == 10);
-        assert(l.front() == 2);
-        assert(l.back() == 0);
-    }
+int main(int, char**) {
+  {
+    std::list<int> l(5, 2);
+    l.resize(2);
+    assert(l.size() == 2);
+    assert(std::distance(l.begin(), l.end()) == 2);
+    assert(l == std::list<int>(2, 2));
+  }
+  {
+    std::list<int> l(5, 2);
+    l.resize(10);
+    assert(l.size() == 10);
+    assert(std::distance(l.begin(), l.end()) == 10);
+    assert(l.front() == 2);
+    assert(l.back() == 0);
+  }
 #if TEST_STD_VER >= 11
-    {
-        std::list<DefaultOnly> l(10);
-        l.resize(5);
-        assert(l.size() == 5);
-        assert(std::distance(l.begin(), l.end()) == 5);
-    }
-    {
-        std::list<DefaultOnly> l(10);
-        l.resize(20);
-        assert(l.size() == 20);
-        assert(std::distance(l.begin(), l.end()) == 20);
-    }
-    {
-        std::list<int, min_allocator<int>> l(5, 2);
-        l.resize(2);
-        assert(l.size() == 2);
-        assert(std::distance(l.begin(), l.end()) == 2);
-        assert((l == std::list<int, min_allocator<int>>(2, 2)));
-    }
-    {
-        std::list<int, min_allocator<int>> l(5, 2);
-        l.resize(10);
-        assert(l.size() == 10);
-        assert(std::distance(l.begin(), l.end()) == 10);
-        assert(l.front() == 2);
-        assert(l.back() == 0);
-    }
-    {
-        std::list<DefaultOnly, min_allocator<DefaultOnly>> l(10);
-        l.resize(5);
-        assert(l.size() == 5);
-        assert(std::distance(l.begin(), l.end()) == 5);
-    }
-    {
-        std::list<DefaultOnly, min_allocator<DefaultOnly>> l(10);
-        l.resize(20);
-        assert(l.size() == 20);
-        assert(std::distance(l.begin(), l.end()) == 20);
-    }
+  {
+    std::list<DefaultOnly> l(10);
+    l.resize(5);
+    assert(l.size() == 5);
+    assert(std::distance(l.begin(), l.end()) == 5);
+  }
+  {
+    std::list<DefaultOnly> l(10);
+    l.resize(20);
+    assert(l.size() == 20);
+    assert(std::distance(l.begin(), l.end()) == 20);
+  }
+  {
+    std::list<int, min_allocator<int>> l(5, 2);
+    l.resize(2);
+    assert(l.size() == 2);
+    assert(std::distance(l.begin(), l.end()) == 2);
+    assert((l == std::list<int, min_allocator<int>>(2, 2)));
+  }
+  {
+    std::list<int, min_allocator<int>> l(5, 2);
+    l.resize(10);
+    assert(l.size() == 10);
+    assert(std::distance(l.begin(), l.end()) == 10);
+    assert(l.front() == 2);
+    assert(l.back() == 0);
+  }
+  {
+    std::list<DefaultOnly, min_allocator<DefaultOnly>> l(10);
+    l.resize(5);
+    assert(l.size() == 5);
+    assert(std::distance(l.begin(), l.end()) == 5);
+  }
+  {
+    std::list<DefaultOnly, min_allocator<DefaultOnly>> l(10);
+    l.resize(20);
+    assert(l.size() == 20);
+    assert(std::distance(l.begin(), l.end()) == 20);
+  }
 #endif
 
-    return 0;
+  return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/list/list.capacity/resize_size_value.pass.cpp b/libcxx/test/std/containers/sequences/list/list.capacity/resize_size_value.pass.cpp
index 112287a550b3d..95fccddeca540 100644
--- a/libcxx/test/std/containers/sequences/list/list.capacity/resize_size_value.pass.cpp
+++ b/libcxx/test/std/containers/sequences/list/list.capacity/resize_size_value.pass.cpp
@@ -16,39 +16,38 @@
 #include "DefaultOnly.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        std::list<double> l(5, 2);
-        l.resize(2, 3.5);
-        assert(l.size() == 2);
-        assert(std::distance(l.begin(), l.end()) == 2);
-        assert(l == std::list<double>(2, 2));
-    }
-    {
-        std::list<double> l(5, 2);
-        l.resize(10, 3.5);
-        assert(l.size() == 10);
-        assert(std::distance(l.begin(), l.end()) == 10);
-        assert(l.front() == 2);
-        assert(l.back() == 3.5);
-    }
+int main(int, char**) {
+  {
+    std::list<double> l(5, 2);
+    l.resize(2, 3.5);
+    assert(l.size() == 2);
+    assert(std::distance(l.begin(), l.end()) == 2);
+    assert(l == std::list<double>(2, 2));
+  }
+  {
+    std::list<double> l(5, 2);
+    l.resize(10, 3.5);
+    assert(l.size() == 10);
+    assert(std::distance(l.begin(), l.end()) == 10);
+    assert(l.front() == 2);
+    assert(l.back() == 3.5);
+  }
 #if TEST_STD_VER >= 11
-    {
-        std::list<double, min_allocator<double>> l(5, 2);
-        l.resize(2, 3.5);
-        assert(l.size() == 2);
-        assert(std::distance(l.begin(), l.end()) == 2);
-        assert((l == std::list<double, min_allocator<double>>(2, 2)));
-    }
-    {
-        std::list<double, min_allocator<double>> l(5, 2);
-        l.resize(10, 3.5);
-        assert(l.size() == 10);
-        assert(std::distance(l.begin(), l.end()) == 10);
-        assert(l.front() == 2);
-        assert(l.back() == 3.5);
-    }
+  {
+    std::list<double, min_allocator<double>> l(5, 2);
+    l.resize(2, 3.5);
+    assert(l.size() == 2);
+    assert(std::distance(l.begin(), l.end()) == 2);
+    assert((l == std::list<double, min_allocator<double>>(2, 2)));
+  }
+  {
+    std::list<double, min_allocator<double>> l(5, 2);
+    l.resize(10, 3.5);
+    assert(l.size() == 10);
+    assert(std::distance(l.begin(), l.end()) == 10);
+    assert(l.front() == 2);
+    assert(l.back() == 3.5);
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/sequences/list/list.capacity/size.pass.cpp b/libcxx/test/std/containers/sequences/list/list.capacity/size.pass.cpp
index d4801e76c2255..930331205a9a5 100644
--- a/libcxx/test/std/containers/sequences/list/list.capacity/size.pass.cpp
+++ b/libcxx/test/std/containers/sequences/list/list.capacity/size.pass.cpp
@@ -18,9 +18,8 @@
 #include "test_macros.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
+int main(int, char**) {
+  {
     typedef std::list<int> C;
     C c;
     ASSERT_NOEXCEPT(c.size());
@@ -37,9 +36,9 @@ int main(int, char**)
     assert(c.size() == 1);
     c.erase(c.begin());
     assert(c.size() == 0);
-    }
+  }
 #if TEST_STD_VER >= 11
-    {
+  {
     typedef std::list<int, min_allocator<int>> C;
     C c;
     ASSERT_NOEXCEPT(c.size());
@@ -56,7 +55,7 @@ int main(int, char**)
     assert(c.size() == 1);
     c.erase(c.begin());
     assert(c.size() == 0);
-    }
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/sequences/list/list.cons/assign_copy.pass.cpp b/libcxx/test/std/containers/sequences/list/list.cons/assign_copy.pass.cpp
index 39369d4f8295c..ca468d870998e 100644
--- a/libcxx/test/std/containers/sequences/list/list.cons/assign_copy.pass.cpp
+++ b/libcxx/test/std/containers/sequences/list/list.cons/assign_copy.pass.cpp
@@ -16,30 +16,29 @@
 #include "test_allocator.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        std::list<int, test_allocator<int> > l(3, 2, test_allocator<int>(5));
-        std::list<int, test_allocator<int> > l2(l, test_allocator<int>(3));
-        l2 = l;
-        assert(l2 == l);
-        assert(l2.get_allocator() == test_allocator<int>(3));
-    }
-    {
-        std::list<int, other_allocator<int> > l(3, 2, other_allocator<int>(5));
-        std::list<int, other_allocator<int> > l2(l, other_allocator<int>(3));
-        l2 = l;
-        assert(l2 == l);
-        assert(l2.get_allocator() == other_allocator<int>(5));
-    }
+int main(int, char**) {
+  {
+    std::list<int, test_allocator<int> > l(3, 2, test_allocator<int>(5));
+    std::list<int, test_allocator<int> > l2(l, test_allocator<int>(3));
+    l2 = l;
+    assert(l2 == l);
+    assert(l2.get_allocator() == test_allocator<int>(3));
+  }
+  {
+    std::list<int, other_allocator<int> > l(3, 2, other_allocator<int>(5));
+    std::list<int, other_allocator<int> > l2(l, other_allocator<int>(3));
+    l2 = l;
+    assert(l2 == l);
+    assert(l2.get_allocator() == other_allocator<int>(5));
+  }
 #if TEST_STD_VER >= 11
-    {
-        std::list<int, min_allocator<int> > l(3, 2, min_allocator<int>());
-        std::list<int, min_allocator<int> > l2(l, min_allocator<int>());
-        l2 = l;
-        assert(l2 == l);
-        assert(l2.get_allocator() == min_allocator<int>());
-    }
+  {
+    std::list<int, min_allocator<int> > l(3, 2, min_allocator<int>());
+    std::list<int, min_allocator<int> > l2(l, min_allocator<int>());
+    l2 = l;
+    assert(l2 == l);
+    assert(l2.get_allocator() == min_allocator<int>());
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/sequences/list/list.cons/assign_initializer_list.pass.cpp b/libcxx/test/std/containers/sequences/list/list.cons/assign_initializer_list.pass.cpp
index 6aeec2b165efc..d4c1120df6226 100644
--- a/libcxx/test/std/containers/sequences/list/list.cons/assign_initializer_list.pass.cpp
+++ b/libcxx/test/std/containers/sequences/list/list.cons/assign_initializer_list.pass.cpp
@@ -18,9 +18,8 @@
 #include "test_macros.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
+int main(int, char**) {
+  {
     std::list<int> d;
     d.assign({3, 4, 5, 6});
     assert(d.size() == 4);
@@ -29,8 +28,8 @@ int main(int, char**)
     assert(*i++ == 4);
     assert(*i++ == 5);
     assert(*i++ == 6);
-    }
-    {
+  }
+  {
     std::list<int, min_allocator<int>> d;
     d.assign({3, 4, 5, 6});
     assert(d.size() == 4);
@@ -39,7 +38,7 @@ int main(int, char**)
     assert(*i++ == 4);
     assert(*i++ == 5);
     assert(*i++ == 6);
-    }
+  }
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/list/list.cons/assign_move.pass.cpp b/libcxx/test/std/containers/sequences/list/list.cons/assign_move.pass.cpp
index 1e459a2bdca8d..87faaaac2b210 100644
--- a/libcxx/test/std/containers/sequences/list/list.cons/assign_move.pass.cpp
+++ b/libcxx/test/std/containers/sequences/list/list.cons/assign_move.pass.cpp
@@ -19,70 +19,65 @@
 #include "test_allocator.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        std::list<MoveOnly, test_allocator<MoveOnly> > l(test_allocator<MoveOnly>(5));
-        std::list<MoveOnly, test_allocator<MoveOnly> > lo(test_allocator<MoveOnly>(5));
-        for (int i = 1; i <= 3; ++i)
-        {
-            l.push_back(i);
-            lo.push_back(i);
-        }
-        std::list<MoveOnly, test_allocator<MoveOnly> > l2(test_allocator<MoveOnly>(5));
-        std::list<MoveOnly, test_allocator<MoveOnly> >::iterator it = l.begin();
-        l2 = std::move(l);
-        assert(l2 == lo);
-        assert(l.empty());
-        assert(l2.get_allocator() == lo.get_allocator());
-        assert(it == l2.begin());  // Iterators remain valid
+int main(int, char**) {
+  {
+    std::list<MoveOnly, test_allocator<MoveOnly> > l(test_allocator<MoveOnly>(5));
+    std::list<MoveOnly, test_allocator<MoveOnly> > lo(test_allocator<MoveOnly>(5));
+    for (int i = 1; i <= 3; ++i) {
+      l.push_back(i);
+      lo.push_back(i);
     }
-    {
-        std::list<MoveOnly, test_allocator<MoveOnly> > l(test_allocator<MoveOnly>(5));
-        std::list<MoveOnly, test_allocator<MoveOnly> > lo(test_allocator<MoveOnly>(5));
-        for (int i = 1; i <= 3; ++i)
-        {
-            l.push_back(i);
-            lo.push_back(i);
-        }
-        std::list<MoveOnly, test_allocator<MoveOnly> > l2(test_allocator<MoveOnly>(6));
-        l2 = std::move(l);
-        assert(l2 == lo);
-        assert(!l.empty());
-        assert(l2.get_allocator() == test_allocator<MoveOnly>(6));
+    std::list<MoveOnly, test_allocator<MoveOnly> > l2(test_allocator<MoveOnly>(5));
+    std::list<MoveOnly, test_allocator<MoveOnly> >::iterator it = l.begin();
+    l2                                                          = std::move(l);
+    assert(l2 == lo);
+    assert(l.empty());
+    assert(l2.get_allocator() == lo.get_allocator());
+    assert(it == l2.begin()); // Iterators remain valid
+  }
+  {
+    std::list<MoveOnly, test_allocator<MoveOnly> > l(test_allocator<MoveOnly>(5));
+    std::list<MoveOnly, test_allocator<MoveOnly> > lo(test_allocator<MoveOnly>(5));
+    for (int i = 1; i <= 3; ++i) {
+      l.push_back(i);
+      lo.push_back(i);
     }
-    {
-        std::list<MoveOnly, other_allocator<MoveOnly> > l(other_allocator<MoveOnly>(5));
-        std::list<MoveOnly, other_allocator<MoveOnly> > lo(other_allocator<MoveOnly>(5));
-        for (int i = 1; i <= 3; ++i)
-        {
-            l.push_back(i);
-            lo.push_back(i);
-        }
-        std::list<MoveOnly, other_allocator<MoveOnly> > l2(other_allocator<MoveOnly>(6));
-        std::list<MoveOnly, other_allocator<MoveOnly> >::iterator it = l.begin();
-        l2 = std::move(l);
-        assert(l2 == lo);
-        assert(l.empty());
-        assert(l2.get_allocator() == lo.get_allocator());
-        assert(it == l2.begin());  // Iterators remain valid
+    std::list<MoveOnly, test_allocator<MoveOnly> > l2(test_allocator<MoveOnly>(6));
+    l2 = std::move(l);
+    assert(l2 == lo);
+    assert(!l.empty());
+    assert(l2.get_allocator() == test_allocator<MoveOnly>(6));
+  }
+  {
+    std::list<MoveOnly, other_allocator<MoveOnly> > l(other_allocator<MoveOnly>(5));
+    std::list<MoveOnly, other_allocator<MoveOnly> > lo(other_allocator<MoveOnly>(5));
+    for (int i = 1; i <= 3; ++i) {
+      l.push_back(i);
+      lo.push_back(i);
     }
-    {
-        std::list<MoveOnly, min_allocator<MoveOnly> > l(min_allocator<MoveOnly>{});
-        std::list<MoveOnly, min_allocator<MoveOnly> > lo(min_allocator<MoveOnly>{});
-        for (int i = 1; i <= 3; ++i)
-        {
-            l.push_back(i);
-            lo.push_back(i);
-        }
-        std::list<MoveOnly, min_allocator<MoveOnly> > l2(min_allocator<MoveOnly>{});
-        std::list<MoveOnly, min_allocator<MoveOnly> >::iterator it = l.begin();
-        l2 = std::move(l);
-        assert(l2 == lo);
-        assert(l.empty());
-        assert(l2.get_allocator() == lo.get_allocator());
-        assert(it == l2.begin());  // Iterators remain valid
+    std::list<MoveOnly, other_allocator<MoveOnly> > l2(other_allocator<MoveOnly>(6));
+    std::list<MoveOnly, other_allocator<MoveOnly> >::iterator it = l.begin();
+    l2                                                           = std::move(l);
+    assert(l2 == lo);
+    assert(l.empty());
+    assert(l2.get_allocator() == lo.get_allocator());
+    assert(it == l2.begin()); // Iterators remain valid
+  }
+  {
+    std::list<MoveOnly, min_allocator<MoveOnly> > l(min_allocator<MoveOnly>{});
+    std::list<MoveOnly, min_allocator<MoveOnly> > lo(min_allocator<MoveOnly>{});
+    for (int i = 1; i <= 3; ++i) {
+      l.push_back(i);
+      lo.push_back(i);
     }
+    std::list<MoveOnly, min_allocator<MoveOnly> > l2(min_allocator<MoveOnly>{});
+    std::list<MoveOnly, min_allocator<MoveOnly> >::iterator it = l.begin();
+    l2                                                         = std::move(l);
+    assert(l2 == lo);
+    assert(l.empty());
+    assert(l2.get_allocator() == lo.get_allocator());
+    assert(it == l2.begin()); // Iterators remain valid
+  }
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/list/list.cons/copy.pass.cpp b/libcxx/test/std/containers/sequences/list/list.cons/copy.pass.cpp
index 153cd2dbc96b6..de52da0fefabf 100644
--- a/libcxx/test/std/containers/sequences/list/list.cons/copy.pass.cpp
+++ b/libcxx/test/std/containers/sequences/list/list.cons/copy.pass.cpp
@@ -18,37 +18,36 @@
 #include "test_allocator.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        std::list<int> l(3, 2);
-        std::list<int> l2 = l;
-        assert(l2 == l);
-    }
-    {
-        std::list<int, test_allocator<int> > l(3, 2, test_allocator<int>(5));
-        std::list<int, test_allocator<int> > l2 = l;
-        assert(l2 == l);
-        assert(l2.get_allocator() == l.get_allocator());
-    }
+int main(int, char**) {
+  {
+    std::list<int> l(3, 2);
+    std::list<int> l2 = l;
+    assert(l2 == l);
+  }
+  {
+    std::list<int, test_allocator<int> > l(3, 2, test_allocator<int>(5));
+    std::list<int, test_allocator<int> > l2 = l;
+    assert(l2 == l);
+    assert(l2.get_allocator() == l.get_allocator());
+  }
 #if TEST_STD_VER >= 11
-    {
-        std::list<int, other_allocator<int> > l(3, 2, other_allocator<int>(5));
-        std::list<int, other_allocator<int> > l2 = l;
-        assert(l2 == l);
-        assert(l2.get_allocator() == other_allocator<int>(-2));
-    }
-    {
-        std::list<int, min_allocator<int>> l(3, 2);
-        std::list<int, min_allocator<int>> l2 = l;
-        assert(l2 == l);
-    }
-    {
-        std::list<int, min_allocator<int> > l(3, 2, min_allocator<int>());
-        std::list<int, min_allocator<int> > l2 = l;
-        assert(l2 == l);
-        assert(l2.get_allocator() == l.get_allocator());
-    }
+  {
+    std::list<int, other_allocator<int> > l(3, 2, other_allocator<int>(5));
+    std::list<int, other_allocator<int> > l2 = l;
+    assert(l2 == l);
+    assert(l2.get_allocator() == other_allocator<int>(-2));
+  }
+  {
+    std::list<int, min_allocator<int>> l(3, 2);
+    std::list<int, min_allocator<int>> l2 = l;
+    assert(l2 == l);
+  }
+  {
+    std::list<int, min_allocator<int> > l(3, 2, min_allocator<int>());
+    std::list<int, min_allocator<int> > l2 = l;
+    assert(l2 == l);
+    assert(l2.get_allocator() == l.get_allocator());
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/sequences/list/list.cons/copy_alloc.pass.cpp b/libcxx/test/std/containers/sequences/list/list.cons/copy_alloc.pass.cpp
index 99d7879de15c9..0d6c6f431f09d 100644
--- a/libcxx/test/std/containers/sequences/list/list.cons/copy_alloc.pass.cpp
+++ b/libcxx/test/std/containers/sequences/list/list.cons/copy_alloc.pass.cpp
@@ -17,27 +17,26 @@
 #include "test_allocator.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        std::list<int, test_allocator<int> > l(3, 2, test_allocator<int>(5));
-        std::list<int, test_allocator<int> > l2(l, test_allocator<int>(3));
-        assert(l2 == l);
-        assert(l2.get_allocator() == test_allocator<int>(3));
-    }
-    {
-        std::list<int, other_allocator<int> > l(3, 2, other_allocator<int>(5));
-        std::list<int, other_allocator<int> > l2(l, other_allocator<int>(3));
-        assert(l2 == l);
-        assert(l2.get_allocator() == other_allocator<int>(3));
-    }
+int main(int, char**) {
+  {
+    std::list<int, test_allocator<int> > l(3, 2, test_allocator<int>(5));
+    std::list<int, test_allocator<int> > l2(l, test_allocator<int>(3));
+    assert(l2 == l);
+    assert(l2.get_allocator() == test_allocator<int>(3));
+  }
+  {
+    std::list<int, other_allocator<int> > l(3, 2, other_allocator<int>(5));
+    std::list<int, other_allocator<int> > l2(l, other_allocator<int>(3));
+    assert(l2 == l);
+    assert(l2.get_allocator() == other_allocator<int>(3));
+  }
 #if TEST_STD_VER >= 11
-    {
-        std::list<int, min_allocator<int> > l(3, 2, min_allocator<int>());
-        std::list<int, min_allocator<int> > l2(l, min_allocator<int>());
-        assert(l2 == l);
-        assert(l2.get_allocator() == min_allocator<int>());
-    }
+  {
+    std::list<int, min_allocator<int> > l(3, 2, min_allocator<int>());
+    std::list<int, min_allocator<int> > l2(l, min_allocator<int>());
+    assert(l2 == l);
+    assert(l2.get_allocator() == min_allocator<int>());
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/sequences/list/list.cons/deduct.pass.cpp b/libcxx/test/std/containers/sequences/list/list.cons/deduct.pass.cpp
index 2210a9ea6e699..836c140d2e52c 100644
--- a/libcxx/test/std/containers/sequences/list/list.cons/deduct.pass.cpp
+++ b/libcxx/test/std/containers/sequences/list/list.cons/deduct.pass.cpp
@@ -31,20 +31,18 @@
 
 struct A {};
 
-int main(int, char**)
-{
-
-//  Test the explicit deduction guides
-    {
-    const int arr[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9 };
+int main(int, char**) {
+  //  Test the explicit deduction guides
+  {
+    const int arr[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9};
     std::list lst(std::begin(arr), std::end(arr));
 
     static_assert(std::is_same_v<decltype(lst), std::list<int>>, "");
     assert(std::equal(lst.begin(), lst.end(), std::begin(arr), std::end(arr)));
-    }
+  }
 
-    {
-    const long arr[] = {INT_MAX, 1L, 2L, 3L };
+  {
+    const long arr[] = {INT_MAX, 1L, 2L, 3L};
     std::list lst(std::begin(arr), std::end(arr), std::allocator<long>());
     static_assert(std::is_same_v<decltype(lst)::value_type, long>, "");
     assert(lst.size() == 4);
@@ -52,39 +50,39 @@ int main(int, char**)
     assert(*it++ == INT_MAX);
     assert(*it++ == 1L);
     assert(*it++ == 2L);
-    }
+  }
 
-//  Test the implicit deduction guides
+  //  Test the implicit deduction guides
 
-    {
-//  We don't expect this one to work.
-//  std::list lst(std::allocator<int>()); // list (allocator &)
-    }
+  {
+    //  We don't expect this one to work.
+    //  std::list lst(std::allocator<int>()); // list (allocator &)
+  }
 
-    {
+  {
     std::list lst(1, A{}); // list (size_type, T)
     static_assert(std::is_same_v<decltype(lst)::value_type, A>, "");
     static_assert(std::is_same_v<decltype(lst)::allocator_type, std::allocator<A>>, "");
     assert(lst.size() == 1);
-    }
+  }
 
-    {
+  {
     std::list lst(1, A{}, test_allocator<A>()); // list (size_type, T, allocator)
     static_assert(std::is_same_v<decltype(lst)::value_type, A>, "");
     static_assert(std::is_same_v<decltype(lst)::allocator_type, test_allocator<A>>, "");
     assert(lst.size() == 1);
-    }
+  }
 
-    {
+  {
     std::list lst{1U, 2U, 3U, 4U, 5U}; // list(initializer-list)
     static_assert(std::is_same_v<decltype(lst)::value_type, unsigned>, "");
     assert(lst.size() == 5);
     auto it = lst.begin();
     std::advance(it, 2);
     assert(*it == 3U);
-    }
+  }
 
-    {
+  {
     std::list lst({1.0, 2.0, 3.0, 4.0}, test_allocator<double>()); // list(initializer-list, allocator)
     static_assert(std::is_same_v<decltype(lst)::value_type, double>, "");
     static_assert(std::is_same_v<decltype(lst)::allocator_type, test_allocator<double>>, "");
@@ -92,61 +90,61 @@ int main(int, char**)
     auto it = lst.begin();
     std::advance(it, 3);
     assert(*it == 4.0);
-    }
+  }
 
-    {
+  {
     std::list<long double> source;
     std::list lst(source); // list(list &)
     static_assert(std::is_same_v<decltype(lst)::value_type, long double>, "");
     static_assert(std::is_same_v<decltype(lst)::allocator_type, std::allocator<long double>>, "");
     assert(lst.size() == 0);
+  }
+
+  {
+    typedef test_allocator<short> Alloc;
+    typedef test_allocator<int> ConvertibleToAlloc;
+
+    {
+      std::list<short, Alloc> source;
+      std::list lst(source, Alloc(2));
+      static_assert(std::is_same_v<decltype(lst), decltype(source)>);
+    }
+
+    {
+      std::list<short, Alloc> source;
+      std::list lst(source, ConvertibleToAlloc(2));
+      static_assert(std::is_same_v<decltype(lst), decltype(source)>);
+    }
+
+    {
+      std::list<short, Alloc> source;
+      std::list lst(std::move(source), Alloc(2));
+      static_assert(std::is_same_v<decltype(lst), decltype(source)>);
     }
 
     {
-        typedef test_allocator<short> Alloc;
-        typedef test_allocator<int> ConvertibleToAlloc;
-
-        {
-        std::list<short, Alloc> source;
-        std::list lst(source, Alloc(2));
-        static_assert(std::is_same_v<decltype(lst), decltype(source)>);
-        }
-
-        {
-        std::list<short, Alloc> source;
-        std::list lst(source, ConvertibleToAlloc(2));
-        static_assert(std::is_same_v<decltype(lst), decltype(source)>);
-        }
-
-        {
-        std::list<short, Alloc> source;
-        std::list lst(std::move(source), Alloc(2));
-        static_assert(std::is_same_v<decltype(lst), decltype(source)>);
-        }
-
-        {
-        std::list<short, Alloc> source;
-        std::list lst(std::move(source), ConvertibleToAlloc(2));
-        static_assert(std::is_same_v<decltype(lst), decltype(source)>);
-        }
+      std::list<short, Alloc> source;
+      std::list lst(std::move(source), ConvertibleToAlloc(2));
+      static_assert(std::is_same_v<decltype(lst), decltype(source)>);
     }
+  }
 
 #if TEST_STD_VER >= 23
+  {
+    {
+      std::list c(std::from_range, std::array<int, 0>());
+      static_assert(std::is_same_v<decltype(c), std::list<int>>);
+    }
+
     {
-      {
-        std::list c(std::from_range, std::array<int, 0>());
-        static_assert(std::is_same_v<decltype(c), std::list<int>>);
-      }
-
-      {
-        using Alloc = test_allocator<int>;
-        std::list c(std::from_range, std::array<int, 0>(), Alloc());
-        static_assert(std::is_same_v<decltype(c), std::list<int, Alloc>>);
-      }
+      using Alloc = test_allocator<int>;
+      std::list c(std::from_range, std::array<int, 0>(), Alloc());
+      static_assert(std::is_same_v<decltype(c), std::list<int, Alloc>>);
     }
+  }
 #endif
 
-    SequenceContainerDeductionGuidesSfinaeAway<std::list, std::list<int>>();
+  SequenceContainerDeductionGuidesSfinaeAway<std::list, std::list<int>>();
 
-    return 0;
+  return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/list/list.cons/deduct.verify.cpp b/libcxx/test/std/containers/sequences/list/list.cons/deduct.verify.cpp
index 370cd38612ab0..1b15ed14fe9a0 100644
--- a/libcxx/test/std/containers/sequences/list/list.cons/deduct.verify.cpp
+++ b/libcxx/test/std/containers/sequences/list/list.cons/deduct.verify.cpp
@@ -22,19 +22,18 @@
 
 struct A {};
 
-int main(int, char**)
-{
-//  Test the explicit deduction guides
-
-//  Test the implicit deduction guides
-    {
-//  list (allocator &)
-    std::list lst((std::allocator<int>()));  // expected-error-re {{no viable constructor or deduction guide for deduction of template arguments of '{{(std::)?}}list'}}
-//  Note: The extra parens are necessary, since otherwise clang decides it is a function declaration.
-//  Also, we can't use {} instead of parens, because that constructs a
-//      deque<allocator<int>, allocator<allocator<int>>>
-    }
-
+int main(int, char**) {
+  //  Test the explicit deduction guides
+
+  //  Test the implicit deduction guides
+  {
+    //  list (allocator &)
+    std::list lst((std::allocator< int>()));
+    // expected-error-re@-1 {{no viable constructor or deduction guide for deduction of template arguments of '{{(std::)?}}list'}}
+    //  Note: The extra parens are necessary, since otherwise clang decides it is a function declaration.
+    //  Also, we can't use {} instead of parens, because that constructs a
+    //      deque<allocator<int>, allocator<allocator<int>>>
+  }
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/list/list.cons/default.pass.cpp b/libcxx/test/std/containers/sequences/list/list.cons/default.pass.cpp
index 32f2729b70960..0cfd8f1e9c594 100644
--- a/libcxx/test/std/containers/sequences/list/list.cons/default.pass.cpp
+++ b/libcxx/test/std/containers/sequences/list/list.cons/default.pass.cpp
@@ -16,54 +16,53 @@
 #include "DefaultOnly.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        std::list<int> l;
-        assert(l.size() == 0);
-        assert(std::distance(l.begin(), l.end()) == 0);
-    }
-    {
-        std::list<DefaultOnly> l;
-        assert(l.size() == 0);
-        assert(std::distance(l.begin(), l.end()) == 0);
-    }
-    {
-        std::list<int> l((std::allocator<int>()));
-        assert(l.size() == 0);
-        assert(std::distance(l.begin(), l.end()) == 0);
-    }
+int main(int, char**) {
+  {
+    std::list<int> l;
+    assert(l.size() == 0);
+    assert(std::distance(l.begin(), l.end()) == 0);
+  }
+  {
+    std::list<DefaultOnly> l;
+    assert(l.size() == 0);
+    assert(std::distance(l.begin(), l.end()) == 0);
+  }
+  {
+    std::list<int> l((std::allocator<int>()));
+    assert(l.size() == 0);
+    assert(std::distance(l.begin(), l.end()) == 0);
+  }
 #if TEST_STD_VER >= 11
-    {
-        std::list<int, min_allocator<int>> l;
-        assert(l.size() == 0);
-        assert(std::distance(l.begin(), l.end()) == 0);
-    }
-    {
-        std::list<DefaultOnly, min_allocator<DefaultOnly>> l;
-        assert(l.size() == 0);
-        assert(std::distance(l.begin(), l.end()) == 0);
-    }
-    {
-        std::list<int, min_allocator<int>> l((min_allocator<int>()));
-        assert(l.size() == 0);
-        assert(std::distance(l.begin(), l.end()) == 0);
-    }
-    {
-        std::list<int> l = {};
-        assert(l.size() == 0);
-        assert(std::distance(l.begin(), l.end()) == 0);
-    }
-    {
-        std::list<int, explicit_allocator<int>> l;
-        assert(l.size() == 0);
-        assert(std::distance(l.begin(), l.end()) == 0);
-    }
-    {
-        std::list<int, explicit_allocator<int>> l((explicit_allocator<int>()));
-        assert(l.size() == 0);
-        assert(std::distance(l.begin(), l.end()) == 0);
-    }
+  {
+    std::list<int, min_allocator<int>> l;
+    assert(l.size() == 0);
+    assert(std::distance(l.begin(), l.end()) == 0);
+  }
+  {
+    std::list<DefaultOnly, min_allocator<DefaultOnly>> l;
+    assert(l.size() == 0);
+    assert(std::distance(l.begin(), l.end()) == 0);
+  }
+  {
+    std::list<int, min_allocator<int>> l((min_allocator<int>()));
+    assert(l.size() == 0);
+    assert(std::distance(l.begin(), l.end()) == 0);
+  }
+  {
+    std::list<int> l = {};
+    assert(l.size() == 0);
+    assert(std::distance(l.begin(), l.end()) == 0);
+  }
+  {
+    std::list<int, explicit_allocator<int>> l;
+    assert(l.size() == 0);
+    assert(std::distance(l.begin(), l.end()) == 0);
+  }
+  {
+    std::list<int, explicit_allocator<int>> l((explicit_allocator<int>()));
+    assert(l.size() == 0);
+    assert(std::distance(l.begin(), l.end()) == 0);
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/sequences/list/list.cons/default_noexcept.pass.cpp b/libcxx/test/std/containers/sequences/list/list.cons/default_noexcept.pass.cpp
index f21225777af5e..b5e6f3c4d617b 100644
--- a/libcxx/test/std/containers/sequences/list/list.cons/default_noexcept.pass.cpp
+++ b/libcxx/test/std/containers/sequences/list/list.cons/default_noexcept.pass.cpp
@@ -23,33 +23,31 @@
 #include "test_allocator.h"
 
 template <class T>
-struct some_alloc
-{
-    typedef T value_type;
-    some_alloc(const some_alloc&);
-    void allocate(std::size_t);
+struct some_alloc {
+  typedef T value_type;
+  some_alloc(const some_alloc&);
+  void allocate(std::size_t);
 };
 
-int main(int, char**)
-{
+int main(int, char**) {
 #if defined(_LIBCPP_VERSION)
-    {
-        typedef std::list<MoveOnly> C;
-        static_assert(std::is_nothrow_default_constructible<C>::value, "");
-    }
-    {
-        typedef std::list<MoveOnly, test_allocator<MoveOnly>> C;
-        static_assert(std::is_nothrow_default_constructible<C>::value, "");
-    }
+  {
+    typedef std::list<MoveOnly> C;
+    static_assert(std::is_nothrow_default_constructible<C>::value, "");
+  }
+  {
+    typedef std::list<MoveOnly, test_allocator<MoveOnly>> C;
+    static_assert(std::is_nothrow_default_constructible<C>::value, "");
+  }
 #endif // _LIBCPP_VERSION
-    {
-        typedef std::list<MoveOnly, other_allocator<MoveOnly>> C;
-        static_assert(!std::is_nothrow_default_constructible<C>::value, "");
-    }
-    {
-        typedef std::list<MoveOnly, some_alloc<MoveOnly>> C;
-        static_assert(!std::is_nothrow_default_constructible<C>::value, "");
-    }
+  {
+    typedef std::list<MoveOnly, other_allocator<MoveOnly>> C;
+    static_assert(!std::is_nothrow_default_constructible<C>::value, "");
+  }
+  {
+    typedef std::list<MoveOnly, some_alloc<MoveOnly>> C;
+    static_assert(!std::is_nothrow_default_constructible<C>::value, "");
+  }
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/list/list.cons/default_stack_alloc.pass.cpp b/libcxx/test/std/containers/sequences/list/list.cons/default_stack_alloc.pass.cpp
index 67c886b4ee7bc..e31a58d7b9a5c 100644
--- a/libcxx/test/std/containers/sequences/list/list.cons/default_stack_alloc.pass.cpp
+++ b/libcxx/test/std/containers/sequences/list/list.cons/default_stack_alloc.pass.cpp
@@ -16,34 +16,33 @@
 #include "test_allocator.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        std::list<int> l;
-        assert(l.size() == 0);
-        assert(std::distance(l.begin(), l.end()) == 0);
-    }
-    {
-        std::list<int> l((std::allocator<int>()));
-        assert(l.size() == 0);
-        assert(std::distance(l.begin(), l.end()) == 0);
-    }
-    {
-        std::list<int, limited_allocator<int, 4> > l;
-        assert(l.size() == 0);
-        assert(std::distance(l.begin(), l.end()) == 0);
-    }
+int main(int, char**) {
+  {
+    std::list<int> l;
+    assert(l.size() == 0);
+    assert(std::distance(l.begin(), l.end()) == 0);
+  }
+  {
+    std::list<int> l((std::allocator<int>()));
+    assert(l.size() == 0);
+    assert(std::distance(l.begin(), l.end()) == 0);
+  }
+  {
+    std::list<int, limited_allocator<int, 4> > l;
+    assert(l.size() == 0);
+    assert(std::distance(l.begin(), l.end()) == 0);
+  }
 #if TEST_STD_VER >= 11
-    {
-        std::list<int, min_allocator<int>> l;
-        assert(l.size() == 0);
-        assert(std::distance(l.begin(), l.end()) == 0);
-    }
-    {
-        std::list<int, min_allocator<int>> l((min_allocator<int>()));
-        assert(l.size() == 0);
-        assert(std::distance(l.begin(), l.end()) == 0);
-    }
+  {
+    std::list<int, min_allocator<int>> l;
+    assert(l.size() == 0);
+    assert(std::distance(l.begin(), l.end()) == 0);
+  }
+  {
+    std::list<int, min_allocator<int>> l((min_allocator<int>()));
+    assert(l.size() == 0);
+    assert(std::distance(l.begin(), l.end()) == 0);
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/sequences/list/list.cons/dtor.pass.cpp b/libcxx/test/std/containers/sequences/list/list.cons/dtor.pass.cpp
index 8d04a0938a1b2..5dedb54769046 100644
--- a/libcxx/test/std/containers/sequences/list/list.cons/dtor.pass.cpp
+++ b/libcxx/test/std/containers/sequences/list/list.cons/dtor.pass.cpp
@@ -19,27 +19,25 @@
 
 #include "test_macros.h"
 
-
 std::set<int> destroyed;
 
 struct Foo {
-    explicit Foo(int i) : value(i) { }
-    ~Foo() { destroyed.insert(value); }
-    int value;
+  explicit Foo(int i) : value(i) {}
+  ~Foo() { destroyed.insert(value); }
+  int value;
 };
 
-int main(int, char**)
-{
-    {
-        std::list<Foo> list;
-        list.emplace_back(1);
-        list.emplace_back(2);
-        list.emplace_back(3);
-        assert(destroyed.empty());
-    }
-    assert(destroyed.count(1) == 1);
-    assert(destroyed.count(2) == 1);
-    assert(destroyed.count(3) == 1);
-
-    return 0;
+int main(int, char**) {
+  {
+    std::list<Foo> list;
+    list.emplace_back(1);
+    list.emplace_back(2);
+    list.emplace_back(3);
+    assert(destroyed.empty());
+  }
+  assert(destroyed.count(1) == 1);
+  assert(destroyed.count(2) == 1);
+  assert(destroyed.count(3) == 1);
+
+  return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/list/list.cons/dtor_noexcept.pass.cpp b/libcxx/test/std/containers/sequences/list/list.cons/dtor_noexcept.pass.cpp
index ae7e264c64a3a..44e6ddd722a70 100644
--- a/libcxx/test/std/containers/sequences/list/list.cons/dtor_noexcept.pass.cpp
+++ b/libcxx/test/std/containers/sequences/list/list.cons/dtor_noexcept.pass.cpp
@@ -20,33 +20,31 @@
 #include "test_allocator.h"
 
 template <class T>
-struct some_alloc
-{
-    typedef T value_type;
-    some_alloc(const some_alloc&);
-    ~some_alloc() noexcept(false);
-    void allocate(std::size_t);
+struct some_alloc {
+  typedef T value_type;
+  some_alloc(const some_alloc&);
+  ~some_alloc() noexcept(false);
+  void allocate(std::size_t);
 };
 
-int main(int, char**)
-{
-    {
-        typedef std::list<MoveOnly> C;
-        static_assert(std::is_nothrow_destructible<C>::value, "");
-    }
-    {
-        typedef std::list<MoveOnly, test_allocator<MoveOnly>> C;
-        static_assert(std::is_nothrow_destructible<C>::value, "");
-    }
-    {
-        typedef std::list<MoveOnly, other_allocator<MoveOnly>> C;
-        static_assert(std::is_nothrow_destructible<C>::value, "");
-    }
+int main(int, char**) {
+  {
+    typedef std::list<MoveOnly> C;
+    static_assert(std::is_nothrow_destructible<C>::value, "");
+  }
+  {
+    typedef std::list<MoveOnly, test_allocator<MoveOnly>> C;
+    static_assert(std::is_nothrow_destructible<C>::value, "");
+  }
+  {
+    typedef std::list<MoveOnly, other_allocator<MoveOnly>> C;
+    static_assert(std::is_nothrow_destructible<C>::value, "");
+  }
 #if defined(_LIBCPP_VERSION)
-    {
-        typedef std::list<MoveOnly, some_alloc<MoveOnly>> C;
-        static_assert(!std::is_nothrow_destructible<C>::value, "");
-    }
+  {
+    typedef std::list<MoveOnly, some_alloc<MoveOnly>> C;
+    static_assert(!std::is_nothrow_destructible<C>::value, "");
+  }
 #endif // _LIBCPP_VERSION
 
   return 0;
diff --git a/libcxx/test/std/containers/sequences/list/list.cons/from_range.pass.cpp b/libcxx/test/std/containers/sequences/list/list.cons/from_range.pass.cpp
index 42b7475c00784..cc5ed5729b57b 100644
--- a/libcxx/test/std/containers/sequences/list/list.cons/from_range.pass.cpp
+++ b/libcxx/test/std/containers/sequences/list/list.cons/from_range.pass.cpp
@@ -31,4 +31,3 @@ int main(int, char**) {
 
   return 0;
 }
-
diff --git a/libcxx/test/std/containers/sequences/list/list.cons/initializer_list.pass.cpp b/libcxx/test/std/containers/sequences/list/list.cons/initializer_list.pass.cpp
index 8583d67dd2f3f..3ba90d1337e94 100644
--- a/libcxx/test/std/containers/sequences/list/list.cons/initializer_list.pass.cpp
+++ b/libcxx/test/std/containers/sequences/list/list.cons/initializer_list.pass.cpp
@@ -18,9 +18,8 @@
 #include "test_macros.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
+int main(int, char**) {
+  {
     std::list<int> d = {3, 4, 5, 6};
     assert(d.size() == 4);
     std::list<int>::iterator i = d.begin();
@@ -28,8 +27,8 @@ int main(int, char**)
     assert(*i++ == 4);
     assert(*i++ == 5);
     assert(*i++ == 6);
-    }
-    {
+  }
+  {
     std::list<int, min_allocator<int>> d = {3, 4, 5, 6};
     assert(d.size() == 4);
     std::list<int, min_allocator<int>>::iterator i = d.begin();
@@ -37,7 +36,7 @@ int main(int, char**)
     assert(*i++ == 4);
     assert(*i++ == 5);
     assert(*i++ == 6);
-    }
+  }
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/list/list.cons/initializer_list_alloc.pass.cpp b/libcxx/test/std/containers/sequences/list/list.cons/initializer_list_alloc.pass.cpp
index 0e12793ec2834..e4779eb5a6401 100644
--- a/libcxx/test/std/containers/sequences/list/list.cons/initializer_list_alloc.pass.cpp
+++ b/libcxx/test/std/containers/sequences/list/list.cons/initializer_list_alloc.pass.cpp
@@ -19,9 +19,8 @@
 #include "test_allocator.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
+int main(int, char**) {
+  {
     std::list<int, test_allocator<int>> d({3, 4, 5, 6}, test_allocator<int>(3));
     assert(d.get_allocator() == test_allocator<int>(3));
     assert(d.size() == 4);
@@ -30,8 +29,8 @@ int main(int, char**)
     assert(*i++ == 4);
     assert(*i++ == 5);
     assert(*i++ == 6);
-    }
-    {
+  }
+  {
     std::list<int, min_allocator<int>> d({3, 4, 5, 6}, min_allocator<int>());
     assert(d.get_allocator() == min_allocator<int>());
     assert(d.size() == 4);
@@ -40,7 +39,7 @@ int main(int, char**)
     assert(*i++ == 4);
     assert(*i++ == 5);
     assert(*i++ == 6);
-    }
+  }
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/list/list.cons/input_iterator.pass.cpp b/libcxx/test/std/containers/sequences/list/list.cons/input_iterator.pass.cpp
index d13696f1d0ac4..c99069f92f51d 100644
--- a/libcxx/test/std/containers/sequences/list/list.cons/input_iterator.pass.cpp
+++ b/libcxx/test/std/containers/sequences/list/list.cons/input_iterator.pass.cpp
@@ -18,77 +18,75 @@
 #include "test_allocator.h"
 #include "min_allocator.h"
 #if TEST_STD_VER >= 11
-#include "emplace_constructible.h"
-#include "container_test_types.h"
+#  include "emplace_constructible.h"
+#  include "container_test_types.h"
 #endif
 
-void basic_test()
-{
-    {
-        int a[] = {0, 1, 2, 3};
-        std::list<int> l(cpp17_input_iterator<const int*>(a),
-                         cpp17_input_iterator<const int*>(a + sizeof(a)/sizeof(a[0])));
-        assert(l.size() == sizeof(a)/sizeof(a[0]));
-        assert(std::distance(l.begin(), l.end()) == sizeof(a)/sizeof(a[0]));
-        int j = 0;
-        for (std::list<int>::const_iterator i = l.begin(), e = l.end(); i != e; ++i, ++j)
-            assert(*i == j);
-    }
-    {
-        int a[] = {0, 1, 2, 3};
-        std::list<int> l(cpp17_input_iterator<const int*>(a),
-                         cpp17_input_iterator<const int*>(a + sizeof(a)/sizeof(a[0])),
-                         std::allocator<int>());
-        assert(l.size() == sizeof(a)/sizeof(a[0]));
-        assert(std::distance(l.begin(), l.end()) == sizeof(a)/sizeof(a[0]));
-        int j = 0;
-        for (std::list<int>::const_iterator i = l.begin(), e = l.end(); i != e; ++i, ++j)
-            assert(*i == j);
-    }
-    {
-        int a[] = {0, 1, 2, 3};
-        // Add 2 for implementations that dynamically allocate a sentinel node and container proxy.
-        std::list<int, limited_allocator<int, sizeof(a)/sizeof(a[0]) + 2> > l(cpp17_input_iterator<const int*>(a),
-                         cpp17_input_iterator<const int*>(a + sizeof(a)/sizeof(a[0])));
-        assert(l.size() == sizeof(a)/sizeof(a[0]));
-        assert(std::distance(l.begin(), l.end()) == sizeof(a)/sizeof(a[0]));
-        int j = 0;
-        for (std::list<int>::const_iterator i = l.begin(), e = l.end(); i != e; ++i, ++j)
-            assert(*i == j);
-    }
+void basic_test() {
+  {
+    int a[] = {0, 1, 2, 3};
+    std::list<int> l(
+        cpp17_input_iterator<const int*>(a), cpp17_input_iterator<const int*>(a + sizeof(a) / sizeof(a[0])));
+    assert(l.size() == sizeof(a) / sizeof(a[0]));
+    assert(std::distance(l.begin(), l.end()) == sizeof(a) / sizeof(a[0]));
+    int j = 0;
+    for (std::list<int>::const_iterator i = l.begin(), e = l.end(); i != e; ++i, ++j)
+      assert(*i == j);
+  }
+  {
+    int a[] = {0, 1, 2, 3};
+    std::list<int> l(cpp17_input_iterator<const int*>(a),
+                     cpp17_input_iterator<const int*>(a + sizeof(a) / sizeof(a[0])),
+                     std::allocator<int>());
+    assert(l.size() == sizeof(a) / sizeof(a[0]));
+    assert(std::distance(l.begin(), l.end()) == sizeof(a) / sizeof(a[0]));
+    int j = 0;
+    for (std::list<int>::const_iterator i = l.begin(), e = l.end(); i != e; ++i, ++j)
+      assert(*i == j);
+  }
+  {
+    int a[] = {0, 1, 2, 3};
+    // Add 2 for implementations that dynamically allocate a sentinel node and container proxy.
+    std::list<int, limited_allocator<int, sizeof(a) / sizeof(a[0]) + 2> > l(
+        cpp17_input_iterator<const int*>(a), cpp17_input_iterator<const int*>(a + sizeof(a) / sizeof(a[0])));
+    assert(l.size() == sizeof(a) / sizeof(a[0]));
+    assert(std::distance(l.begin(), l.end()) == sizeof(a) / sizeof(a[0]));
+    int j = 0;
+    for (std::list<int>::const_iterator i = l.begin(), e = l.end(); i != e; ++i, ++j)
+      assert(*i == j);
+  }
 #if TEST_STD_VER >= 11
-    {
-        int a[] = {0, 1, 2, 3};
-        std::list<int, min_allocator<int>> l(cpp17_input_iterator<const int*>(a),
-                         cpp17_input_iterator<const int*>(a + sizeof(a)/sizeof(a[0])));
-        assert(l.size() == sizeof(a)/sizeof(a[0]));
-        assert(std::distance(l.begin(), l.end()) == sizeof(a)/sizeof(a[0]));
-        int j = 0;
-        for (std::list<int, min_allocator<int>>::const_iterator i = l.begin(), e = l.end(); i != e; ++i, ++j)
-            assert(*i == j);
-    }
-    {
-        int a[] = {0, 1, 2, 3};
-        std::list<int, min_allocator<int>> l(cpp17_input_iterator<const int*>(a),
-                         cpp17_input_iterator<const int*>(a + sizeof(a)/sizeof(a[0])),
-                         min_allocator<int>());
-        assert(l.size() == sizeof(a)/sizeof(a[0]));
-        assert(std::distance(l.begin(), l.end()) == sizeof(a)/sizeof(a[0]));
-        int j = 0;
-        for (std::list<int, min_allocator<int>>::const_iterator i = l.begin(), e = l.end(); i != e; ++i, ++j)
-            assert(*i == j);
-    }
+  {
+    int a[] = {0, 1, 2, 3};
+    std::list<int, min_allocator<int>> l(
+        cpp17_input_iterator<const int*>(a), cpp17_input_iterator<const int*>(a + sizeof(a) / sizeof(a[0])));
+    assert(l.size() == sizeof(a) / sizeof(a[0]));
+    assert(std::distance(l.begin(), l.end()) == sizeof(a) / sizeof(a[0]));
+    int j = 0;
+    for (std::list<int, min_allocator<int>>::const_iterator i = l.begin(), e = l.end(); i != e; ++i, ++j)
+      assert(*i == j);
+  }
+  {
+    int a[] = {0, 1, 2, 3};
+    std::list<int, min_allocator<int>> l(
+        cpp17_input_iterator<const int*>(a),
+        cpp17_input_iterator<const int*>(a + sizeof(a) / sizeof(a[0])),
+        min_allocator<int>());
+    assert(l.size() == sizeof(a) / sizeof(a[0]));
+    assert(std::distance(l.begin(), l.end()) == sizeof(a) / sizeof(a[0]));
+    int j = 0;
+    for (std::list<int, min_allocator<int>>::const_iterator i = l.begin(), e = l.end(); i != e; ++i, ++j)
+      assert(*i == j);
+  }
 #endif
 }
 
-
-
 void test_emplacable_concept() {
 #if TEST_STD_VER >= 11
   int arr1[] = {42};
   int arr2[] = {1, 101, 42};
   {
-    using T = EmplaceConstructible<int>;
+    using T  = EmplaceConstructible<int>;
     using It = random_access_iterator<int*>;
     {
       std::list<T> v(It(arr1), It(std::end(arr1)));
@@ -106,7 +104,7 @@ void test_emplacable_concept() {
     }
   }
   {
-    using T = EmplaceConstructible<int>;
+    using T  = EmplaceConstructible<int>;
     using It = cpp17_input_iterator<int*>;
     {
       std::list<T> v(It(arr1), It(std::end(arr1)));
@@ -128,14 +126,12 @@ void test_emplacable_concept() {
 #endif
 }
 
-
-
 void test_emplacable_concept_with_alloc() {
 #if TEST_STD_VER >= 11
   int arr1[] = {42};
   int arr2[] = {1, 101, 42};
   {
-    using T = EmplaceConstructible<int>;
+    using T  = EmplaceConstructible<int>;
     using It = random_access_iterator<int*>;
     std::allocator<T> a;
     {
@@ -154,7 +150,7 @@ void test_emplacable_concept_with_alloc() {
     }
   }
   {
-    using T = EmplaceConstructible<int>;
+    using T  = EmplaceConstructible<int>;
     using It = cpp17_input_iterator<int*>;
     std::allocator<T> a;
     {
@@ -182,7 +178,7 @@ void test_ctor_under_alloc() {
   int arr1[] = {42};
   int arr2[] = {1, 101, 42};
   {
-    using C = TCT::list<>;
+    using C  = TCT::list<>;
     using It = forward_iterator<int*>;
     {
       ExpectConstructGuard<int&> G(1);
@@ -194,7 +190,7 @@ void test_ctor_under_alloc() {
     }
   }
   {
-    using C = TCT::list<>;
+    using C  = TCT::list<>;
     using It = cpp17_input_iterator<int*>;
     {
       ExpectConstructGuard<int&> G(1);
@@ -213,8 +209,8 @@ void test_ctor_under_alloc_with_alloc() {
   int arr1[] = {42};
   int arr2[] = {1, 101, 42};
   {
-    using C = TCT::list<>;
-    using It = forward_iterator<int*>;
+    using C     = TCT::list<>;
+    using It    = forward_iterator<int*>;
     using Alloc = typename C::allocator_type;
     Alloc a;
     {
@@ -227,8 +223,8 @@ void test_ctor_under_alloc_with_alloc() {
     }
   }
   {
-    using C = TCT::list<>;
-    using It = cpp17_input_iterator<int*>;
+    using C     = TCT::list<>;
+    using It    = cpp17_input_iterator<int*>;
     using Alloc = typename C::allocator_type;
     Alloc a;
     {
@@ -243,8 +239,6 @@ void test_ctor_under_alloc_with_alloc() {
 #endif
 }
 
-
-
 int main(int, char**) {
   basic_test();
   test_emplacable_concept();
diff --git a/libcxx/test/std/containers/sequences/list/list.cons/move.pass.cpp b/libcxx/test/std/containers/sequences/list/list.cons/move.pass.cpp
index d5e09ca902fee..6703390f10b94 100644
--- a/libcxx/test/std/containers/sequences/list/list.cons/move.pass.cpp
+++ b/libcxx/test/std/containers/sequences/list/list.cons/move.pass.cpp
@@ -19,53 +19,49 @@
 #include "test_allocator.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        std::list<MoveOnly, test_allocator<MoveOnly> > l(test_allocator<MoveOnly>(5));
-        std::list<MoveOnly, test_allocator<MoveOnly> > lo(test_allocator<MoveOnly>(5));
-        for (int i = 1; i <= 3; ++i)
-        {
-            l.push_back(i);
-            lo.push_back(i);
-        }
-        std::list<MoveOnly, test_allocator<MoveOnly> >::iterator it = l.begin();
-        std::list<MoveOnly, test_allocator<MoveOnly> > l2 = std::move(l);
-        assert(l2 == lo);
-        assert(l.empty());
-        assert(l2.get_allocator() == lo.get_allocator());
-        assert(it == l2.begin());  // Iterators remain valid
+int main(int, char**) {
+  {
+    std::list<MoveOnly, test_allocator<MoveOnly> > l(test_allocator<MoveOnly>(5));
+    std::list<MoveOnly, test_allocator<MoveOnly> > lo(test_allocator<MoveOnly>(5));
+    for (int i = 1; i <= 3; ++i) {
+      l.push_back(i);
+      lo.push_back(i);
     }
-    {
-        std::list<MoveOnly, other_allocator<MoveOnly> > l(other_allocator<MoveOnly>(5));
-        std::list<MoveOnly, other_allocator<MoveOnly> > lo(other_allocator<MoveOnly>(5));
-        for (int i = 1; i <= 3; ++i)
-        {
-            l.push_back(i);
-            lo.push_back(i);
-        }
-        std::list<MoveOnly, other_allocator<MoveOnly> >::iterator it = l.begin();
-        std::list<MoveOnly, other_allocator<MoveOnly> > l2 = std::move(l);
-        assert(l2 == lo);
-        assert(l.empty());
-        assert(l2.get_allocator() == lo.get_allocator());
-        assert(it == l2.begin());  // Iterators remain valid
+    std::list<MoveOnly, test_allocator<MoveOnly> >::iterator it = l.begin();
+    std::list<MoveOnly, test_allocator<MoveOnly> > l2           = std::move(l);
+    assert(l2 == lo);
+    assert(l.empty());
+    assert(l2.get_allocator() == lo.get_allocator());
+    assert(it == l2.begin()); // Iterators remain valid
+  }
+  {
+    std::list<MoveOnly, other_allocator<MoveOnly> > l(other_allocator<MoveOnly>(5));
+    std::list<MoveOnly, other_allocator<MoveOnly> > lo(other_allocator<MoveOnly>(5));
+    for (int i = 1; i <= 3; ++i) {
+      l.push_back(i);
+      lo.push_back(i);
     }
-    {
-        std::list<MoveOnly, min_allocator<MoveOnly> > l(min_allocator<MoveOnly>{});
-        std::list<MoveOnly, min_allocator<MoveOnly> > lo(min_allocator<MoveOnly>{});
-        for (int i = 1; i <= 3; ++i)
-        {
-            l.push_back(i);
-            lo.push_back(i);
-        }
-        std::list<MoveOnly, min_allocator<MoveOnly> >::iterator it = l.begin();
-        std::list<MoveOnly, min_allocator<MoveOnly> > l2 = std::move(l);
-        assert(l2 == lo);
-        assert(l.empty());
-        assert(l2.get_allocator() == lo.get_allocator());
-        assert(it == l2.begin());  // Iterators remain valid
+    std::list<MoveOnly, other_allocator<MoveOnly> >::iterator it = l.begin();
+    std::list<MoveOnly, other_allocator<MoveOnly> > l2           = std::move(l);
+    assert(l2 == lo);
+    assert(l.empty());
+    assert(l2.get_allocator() == lo.get_allocator());
+    assert(it == l2.begin()); // Iterators remain valid
+  }
+  {
+    std::list<MoveOnly, min_allocator<MoveOnly> > l(min_allocator<MoveOnly>{});
+    std::list<MoveOnly, min_allocator<MoveOnly> > lo(min_allocator<MoveOnly>{});
+    for (int i = 1; i <= 3; ++i) {
+      l.push_back(i);
+      lo.push_back(i);
     }
+    std::list<MoveOnly, min_allocator<MoveOnly> >::iterator it = l.begin();
+    std::list<MoveOnly, min_allocator<MoveOnly> > l2           = std::move(l);
+    assert(l2 == lo);
+    assert(l.empty());
+    assert(l2.get_allocator() == lo.get_allocator());
+    assert(it == l2.begin()); // Iterators remain valid
+  }
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/list/list.cons/move_alloc.pass.cpp b/libcxx/test/std/containers/sequences/list/list.cons/move_alloc.pass.cpp
index 694fccdf038dd..f6a1f2c33a63a 100644
--- a/libcxx/test/std/containers/sequences/list/list.cons/move_alloc.pass.cpp
+++ b/libcxx/test/std/containers/sequences/list/list.cons/move_alloc.pass.cpp
@@ -19,60 +19,55 @@
 #include "test_allocator.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        std::list<MoveOnly, test_allocator<MoveOnly> > l(test_allocator<MoveOnly>(5));
-        std::list<MoveOnly, test_allocator<MoveOnly> > lo(test_allocator<MoveOnly>(5));
-        for (int i = 1; i <= 3; ++i)
-        {
-            l.push_back(i);
-            lo.push_back(i);
-        }
-        std::list<MoveOnly, test_allocator<MoveOnly> > l2(std::move(l), test_allocator<MoveOnly>(6));
-        assert(l2 == lo);
-        assert(!l.empty());
-        assert(l2.get_allocator() == test_allocator<MoveOnly>(6));
+int main(int, char**) {
+  {
+    std::list<MoveOnly, test_allocator<MoveOnly> > l(test_allocator<MoveOnly>(5));
+    std::list<MoveOnly, test_allocator<MoveOnly> > lo(test_allocator<MoveOnly>(5));
+    for (int i = 1; i <= 3; ++i) {
+      l.push_back(i);
+      lo.push_back(i);
     }
-    {
-        std::list<MoveOnly, test_allocator<MoveOnly> > l(test_allocator<MoveOnly>(5));
-        std::list<MoveOnly, test_allocator<MoveOnly> > lo(test_allocator<MoveOnly>(5));
-        for (int i = 1; i <= 3; ++i)
-        {
-            l.push_back(i);
-            lo.push_back(i);
-        }
-        std::list<MoveOnly, test_allocator<MoveOnly> > l2(std::move(l), test_allocator<MoveOnly>(5));
-        assert(l2 == lo);
-        assert(l.empty());
-        assert(l2.get_allocator() == test_allocator<MoveOnly>(5));
+    std::list<MoveOnly, test_allocator<MoveOnly> > l2(std::move(l), test_allocator<MoveOnly>(6));
+    assert(l2 == lo);
+    assert(!l.empty());
+    assert(l2.get_allocator() == test_allocator<MoveOnly>(6));
+  }
+  {
+    std::list<MoveOnly, test_allocator<MoveOnly> > l(test_allocator<MoveOnly>(5));
+    std::list<MoveOnly, test_allocator<MoveOnly> > lo(test_allocator<MoveOnly>(5));
+    for (int i = 1; i <= 3; ++i) {
+      l.push_back(i);
+      lo.push_back(i);
     }
-    {
-        std::list<MoveOnly, other_allocator<MoveOnly> > l(other_allocator<MoveOnly>(5));
-        std::list<MoveOnly, other_allocator<MoveOnly> > lo(other_allocator<MoveOnly>(5));
-        for (int i = 1; i <= 3; ++i)
-        {
-            l.push_back(i);
-            lo.push_back(i);
-        }
-        std::list<MoveOnly, other_allocator<MoveOnly> > l2(std::move(l), other_allocator<MoveOnly>(4));
-        assert(l2 == lo);
-        assert(!l.empty());
-        assert(l2.get_allocator() == other_allocator<MoveOnly>(4));
+    std::list<MoveOnly, test_allocator<MoveOnly> > l2(std::move(l), test_allocator<MoveOnly>(5));
+    assert(l2 == lo);
+    assert(l.empty());
+    assert(l2.get_allocator() == test_allocator<MoveOnly>(5));
+  }
+  {
+    std::list<MoveOnly, other_allocator<MoveOnly> > l(other_allocator<MoveOnly>(5));
+    std::list<MoveOnly, other_allocator<MoveOnly> > lo(other_allocator<MoveOnly>(5));
+    for (int i = 1; i <= 3; ++i) {
+      l.push_back(i);
+      lo.push_back(i);
     }
-    {
-        std::list<MoveOnly, min_allocator<MoveOnly> > l(min_allocator<MoveOnly>{});
-        std::list<MoveOnly, min_allocator<MoveOnly> > lo(min_allocator<MoveOnly>{});
-        for (int i = 1; i <= 3; ++i)
-        {
-            l.push_back(i);
-            lo.push_back(i);
-        }
-        std::list<MoveOnly, min_allocator<MoveOnly> > l2(std::move(l), min_allocator<MoveOnly>());
-        assert(l2 == lo);
-        assert(l.empty());
-        assert(l2.get_allocator() == min_allocator<MoveOnly>());
+    std::list<MoveOnly, other_allocator<MoveOnly> > l2(std::move(l), other_allocator<MoveOnly>(4));
+    assert(l2 == lo);
+    assert(!l.empty());
+    assert(l2.get_allocator() == other_allocator<MoveOnly>(4));
+  }
+  {
+    std::list<MoveOnly, min_allocator<MoveOnly> > l(min_allocator<MoveOnly>{});
+    std::list<MoveOnly, min_allocator<MoveOnly> > lo(min_allocator<MoveOnly>{});
+    for (int i = 1; i <= 3; ++i) {
+      l.push_back(i);
+      lo.push_back(i);
     }
+    std::list<MoveOnly, min_allocator<MoveOnly> > l2(std::move(l), min_allocator<MoveOnly>());
+    assert(l2 == lo);
+    assert(l.empty());
+    assert(l2.get_allocator() == min_allocator<MoveOnly>());
+  }
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/list/list.cons/move_assign_noexcept.compile.pass.cpp b/libcxx/test/std/containers/sequences/list/list.cons/move_assign_noexcept.compile.pass.cpp
new file mode 100644
index 0000000000000..8da5cfc4716e4
--- /dev/null
+++ b/libcxx/test/std/containers/sequences/list/list.cons/move_assign_noexcept.compile.pass.cpp
@@ -0,0 +1,49 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// <list>
+
+// list& operator=(list&& c)
+//     noexcept(
+//          allocator_type::propagate_on_container_move_assignment::value &&
+//          is_nothrow_move_assignable<allocator_type>::value);
+
+// This tests a conforming extension
+
+// UNSUPPORTED: c++03
+
+#include <list>
+
+#include "test_macros.h"
+#include "MoveOnly.h"
+#include "test_allocator.h"
+
+template <class T>
+struct always_equal_alloc {
+  using value_type = T;
+  always_equal_alloc(const always_equal_alloc&);
+  void allocate(std::size_t);
+};
+
+template <class T>
+struct not_always_equal_alloc {
+  int i;
+  using value_type = T;
+  not_always_equal_alloc(const not_always_equal_alloc&);
+  void allocate(std::size_t);
+};
+
+static_assert(std::is_nothrow_move_assignable<std::list<MoveOnly>>::value, "");
+static_assert(!std::is_nothrow_move_assignable<std::list<MoveOnly, test_allocator<MoveOnly>>>::value, "");
+#if TEST_STD_VER >= 17
+static_assert(std::is_nothrow_move_assignable<std::list<MoveOnly, always_equal_alloc<MoveOnly>>>::value, "");
+#endif
+static_assert(!std::is_nothrow_move_assignable<std::list<MoveOnly, not_always_equal_alloc<MoveOnly>>>::value, "");
+#if defined(_LIBCPP_VERSION)
+static_assert(std::is_nothrow_move_assignable<std::list<MoveOnly, other_allocator<MoveOnly>>>::value, "");
+#endif // _LIBCPP_VERSION
diff --git a/libcxx/test/std/containers/sequences/list/list.cons/move_assign_noexcept.pass.cpp b/libcxx/test/std/containers/sequences/list/list.cons/move_assign_noexcept.pass.cpp
deleted file mode 100644
index 9df8413bb19c4..0000000000000
--- a/libcxx/test/std/containers/sequences/list/list.cons/move_assign_noexcept.pass.cpp
+++ /dev/null
@@ -1,57 +0,0 @@
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-// <list>
-
-// list& operator=(list&& c)
-//     noexcept(
-//          allocator_type::propagate_on_container_move_assignment::value &&
-//          is_nothrow_move_assignable<allocator_type>::value);
-
-// This tests a conforming extension
-
-// UNSUPPORTED: c++03
-
-#include <list>
-#include <cassert>
-
-#include "test_macros.h"
-#include "MoveOnly.h"
-#include "test_allocator.h"
-
-template <class T>
-struct some_alloc
-{
-    typedef T value_type;
-    some_alloc(const some_alloc&);
-    void allocate(std::size_t);
-};
-
-int main(int, char**)
-{
-    {
-        typedef std::list<MoveOnly> C;
-        static_assert(std::is_nothrow_move_assignable<C>::value, "");
-    }
-    {
-        typedef std::list<MoveOnly, test_allocator<MoveOnly>> C;
-        static_assert(!std::is_nothrow_move_assignable<C>::value, "");
-    }
-#if defined(_LIBCPP_VERSION)
-    {
-        typedef std::list<MoveOnly, other_allocator<MoveOnly>> C;
-        static_assert(std::is_nothrow_move_assignable<C>::value, "");
-    }
-    {
-        typedef std::list<MoveOnly, some_alloc<MoveOnly>> C;
-        static_assert(!std::is_nothrow_move_assignable<C>::value, "");
-    }
-#endif // _LIBCPP_VERSION
-
-  return 0;
-}
diff --git a/libcxx/test/std/containers/sequences/list/list.cons/move_noexcept.pass.cpp b/libcxx/test/std/containers/sequences/list/list.cons/move_noexcept.pass.cpp
index 5c0b4c2b36d25..c33ef906be400 100644
--- a/libcxx/test/std/containers/sequences/list/list.cons/move_noexcept.pass.cpp
+++ b/libcxx/test/std/containers/sequences/list/list.cons/move_noexcept.pass.cpp
@@ -23,33 +23,31 @@
 #include "test_allocator.h"
 
 template <class T>
-struct some_alloc
-{
-    typedef T value_type;
-    some_alloc(const some_alloc&);
-    void allocate(std::size_t);
+struct some_alloc {
+  typedef T value_type;
+  some_alloc(const some_alloc&);
+  void allocate(std::size_t);
 };
 
-int main(int, char**)
-{
+int main(int, char**) {
 #if defined(_LIBCPP_VERSION)
-    {
-        typedef std::list<MoveOnly> C;
-        static_assert(std::is_nothrow_move_constructible<C>::value, "");
-    }
-    {
-        typedef std::list<MoveOnly, test_allocator<MoveOnly>> C;
-        static_assert(std::is_nothrow_move_constructible<C>::value, "");
-    }
-    {
-        typedef std::list<MoveOnly, other_allocator<MoveOnly>> C;
-        static_assert(std::is_nothrow_move_constructible<C>::value, "");
-    }
+  {
+    typedef std::list<MoveOnly> C;
+    static_assert(std::is_nothrow_move_constructible<C>::value, "");
+  }
+  {
+    typedef std::list<MoveOnly, test_allocator<MoveOnly>> C;
+    static_assert(std::is_nothrow_move_constructible<C>::value, "");
+  }
+  {
+    typedef std::list<MoveOnly, other_allocator<MoveOnly>> C;
+    static_assert(std::is_nothrow_move_constructible<C>::value, "");
+  }
 #endif // _LIBCPP_VERSION
-    {
-        typedef std::list<MoveOnly, some_alloc<MoveOnly>> C;
-        static_assert(!std::is_nothrow_move_constructible<C>::value, "");
-    }
+  {
+    typedef std::list<MoveOnly, some_alloc<MoveOnly>> C;
+    static_assert(!std::is_nothrow_move_constructible<C>::value, "");
+  }
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/list/list.cons/op_equal_initializer_list.pass.cpp b/libcxx/test/std/containers/sequences/list/list.cons/op_equal_initializer_list.pass.cpp
index 08cbf2acbae26..a9ab30b82640c 100644
--- a/libcxx/test/std/containers/sequences/list/list.cons/op_equal_initializer_list.pass.cpp
+++ b/libcxx/test/std/containers/sequences/list/list.cons/op_equal_initializer_list.pass.cpp
@@ -17,9 +17,8 @@
 #include "test_macros.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
+int main(int, char**) {
+  {
     std::list<int> d;
     d = {3, 4, 5, 6};
     assert(d.size() == 4);
@@ -28,8 +27,8 @@ int main(int, char**)
     assert(*i++ == 4);
     assert(*i++ == 5);
     assert(*i++ == 6);
-    }
-    {
+  }
+  {
     std::list<int, min_allocator<int>> d;
     d = {3, 4, 5, 6};
     assert(d.size() == 4);
@@ -38,7 +37,7 @@ int main(int, char**)
     assert(*i++ == 4);
     assert(*i++ == 5);
     assert(*i++ == 6);
-    }
+  }
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/list/list.cons/size_type.pass.cpp b/libcxx/test/std/containers/sequences/list/list.cons/size_type.pass.cpp
index ef365d09fd435..937a86a27e058 100644
--- a/libcxx/test/std/containers/sequences/list/list.cons/size_type.pass.cpp
+++ b/libcxx/test/std/containers/sequences/list/list.cons/size_type.pass.cpp
@@ -19,86 +19,82 @@
 #include "min_allocator.h"
 
 template <class T, class Allocator>
-void
-test3(unsigned n, Allocator const &alloc = Allocator())
-{
+void test3(unsigned n, Allocator const& alloc = Allocator()) {
 #if TEST_STD_VER > 11
-    typedef std::list<T, Allocator> C;
-    {
+  typedef std::list<T, Allocator> C;
+  {
     C d(n, alloc);
     assert(d.size() == n);
     assert(static_cast<std::size_t>(std::distance(d.begin(), d.end())) == n);
     assert(d.get_allocator() == alloc);
-    }
+  }
 #else
-    ((void)n);
-    ((void)alloc);
+  ((void)n);
+  ((void)alloc);
 #endif
 }
 
-
-int main(int, char**)
-{
-    {
-        std::list<int> l(3);
-        assert(l.size() == 3);
-        assert(std::distance(l.begin(), l.end()) == 3);
-        std::list<int>::const_iterator i = l.begin();
-        assert(*i == 0);
-        ++i;
-        assert(*i == 0);
-        ++i;
-        assert(*i == 0);
-    }
-    {
-        // Add 2 for implementations that dynamically allocate a sentinel node and container proxy.
-        std::list<int, limited_allocator<int, 3 + 2> > l(3);
-        assert(l.size() == 3);
-        assert(std::distance(l.begin(), l.end()) == 3);
-        std::list<int>::const_iterator i = l.begin();
-        assert(*i == 0);
-        ++i;
-        assert(*i == 0);
-        ++i;
-        assert(*i == 0);
-    }
+int main(int, char**) {
+  {
+    std::list<int> l(3);
+    assert(l.size() == 3);
+    assert(std::distance(l.begin(), l.end()) == 3);
+    std::list<int>::const_iterator i = l.begin();
+    assert(*i == 0);
+    ++i;
+    assert(*i == 0);
+    ++i;
+    assert(*i == 0);
+  }
+  {
+    // Add 2 for implementations that dynamically allocate a sentinel node and container proxy.
+    std::list<int, limited_allocator<int, 3 + 2> > l(3);
+    assert(l.size() == 3);
+    assert(std::distance(l.begin(), l.end()) == 3);
+    std::list<int>::const_iterator i = l.begin();
+    assert(*i == 0);
+    ++i;
+    assert(*i == 0);
+    ++i;
+    assert(*i == 0);
+  }
 #if TEST_STD_VER > 11
-    {
-        typedef std::list<int, min_allocator<int> > C;
-        C l(3, min_allocator<int> ());
-        assert(l.size() == 3);
-        assert(std::distance(l.begin(), l.end()) == 3);
-        C::const_iterator i = l.begin();
-        assert(*i == 0);
-        ++i;
-        assert(*i == 0);
-        ++i;
-        assert(*i == 0);
-        test3<int, min_allocator<int>> (3);
-    }
+  {
+    typedef std::list<int, min_allocator<int> > C;
+    C l(3, min_allocator<int>());
+    assert(l.size() == 3);
+    assert(std::distance(l.begin(), l.end()) == 3);
+    C::const_iterator i = l.begin();
+    assert(*i == 0);
+    ++i;
+    assert(*i == 0);
+    ++i;
+    assert(*i == 0);
+    test3<int, min_allocator<int>>(3);
+  }
 #endif
 #if TEST_STD_VER >= 11
-    {
-        std::list<DefaultOnly> l(3);
-        assert(l.size() == 3);
-        assert(std::distance(l.begin(), l.end()) == 3);
-    }
-    {
-        std::list<int, min_allocator<int>> l(3);
-        assert(l.size() == 3);
-        assert(std::distance(l.begin(), l.end()) == 3);
-        std::list<int, min_allocator<int>>::const_iterator i = l.begin();
-        assert(*i == 0);
-        ++i;
-        assert(*i == 0);
-        ++i;
-        assert(*i == 0);
-    }
-    {
-        std::list<DefaultOnly, min_allocator<DefaultOnly>> l(3);
-        assert(l.size() == 3);
-        assert(std::distance(l.begin(), l.end()) == 3);
-    }
+  {
+    std::list<DefaultOnly> l(3);
+    assert(l.size() == 3);
+    assert(std::distance(l.begin(), l.end()) == 3);
+  }
+  {
+    std::list<int, min_allocator<int>> l(3);
+    assert(l.size() == 3);
+    assert(std::distance(l.begin(), l.end()) == 3);
+    std::list<int, min_allocator<int>>::const_iterator i = l.begin();
+    assert(*i == 0);
+    ++i;
+    assert(*i == 0);
+    ++i;
+    assert(*i == 0);
+  }
+  {
+    std::list<DefaultOnly, min_allocator<DefaultOnly>> l(3);
+    assert(l.size() == 3);
+    assert(std::distance(l.begin(), l.end()) == 3);
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/sequences/list/list.cons/size_value_alloc.pass.cpp b/libcxx/test/std/containers/sequences/list/list.cons/size_value_alloc.pass.cpp
index bced58d0208ba..ff7982ce147d5 100644
--- a/libcxx/test/std/containers/sequences/list/list.cons/size_value_alloc.pass.cpp
+++ b/libcxx/test/std/containers/sequences/list/list.cons/size_value_alloc.pass.cpp
@@ -17,65 +17,64 @@
 #include "test_allocator.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        std::list<int> l(3, 2);
-        assert(l.size() == 3);
-        assert(std::distance(l.begin(), l.end()) == 3);
-        std::list<int>::const_iterator i = l.begin();
-        assert(*i == 2);
-        ++i;
-        assert(*i == 2);
-        ++i;
-        assert(*i == 2);
-    }
-    {
-        std::list<int> l(3, 2, std::allocator<int>());
-        assert(l.size() == 3);
-        assert(std::distance(l.begin(), l.end()) == 3);
-        std::list<int>::const_iterator i = l.begin();
-        assert(*i == 2);
-        ++i;
-        assert(*i == 2);
-        ++i;
-        assert(*i == 2);
-    }
-    {
-        // Add 2 for implementations that dynamically allocate a sentinel node and container proxy.
-        std::list<int, limited_allocator<int, 3 + 2> > l(3, 2);
-        assert(l.size() == 3);
-        assert(std::distance(l.begin(), l.end()) == 3);
-        std::list<int>::const_iterator i = l.begin();
-        assert(*i == 2);
-        ++i;
-        assert(*i == 2);
-        ++i;
-        assert(*i == 2);
-    }
+int main(int, char**) {
+  {
+    std::list<int> l(3, 2);
+    assert(l.size() == 3);
+    assert(std::distance(l.begin(), l.end()) == 3);
+    std::list<int>::const_iterator i = l.begin();
+    assert(*i == 2);
+    ++i;
+    assert(*i == 2);
+    ++i;
+    assert(*i == 2);
+  }
+  {
+    std::list<int> l(3, 2, std::allocator<int>());
+    assert(l.size() == 3);
+    assert(std::distance(l.begin(), l.end()) == 3);
+    std::list<int>::const_iterator i = l.begin();
+    assert(*i == 2);
+    ++i;
+    assert(*i == 2);
+    ++i;
+    assert(*i == 2);
+  }
+  {
+    // Add 2 for implementations that dynamically allocate a sentinel node and container proxy.
+    std::list<int, limited_allocator<int, 3 + 2> > l(3, 2);
+    assert(l.size() == 3);
+    assert(std::distance(l.begin(), l.end()) == 3);
+    std::list<int>::const_iterator i = l.begin();
+    assert(*i == 2);
+    ++i;
+    assert(*i == 2);
+    ++i;
+    assert(*i == 2);
+  }
 #if TEST_STD_VER >= 11
-    {
-        std::list<int, min_allocator<int>> l(3, 2);
-        assert(l.size() == 3);
-        assert(std::distance(l.begin(), l.end()) == 3);
-        std::list<int, min_allocator<int>>::const_iterator i = l.begin();
-        assert(*i == 2);
-        ++i;
-        assert(*i == 2);
-        ++i;
-        assert(*i == 2);
-    }
-    {
-        std::list<int, min_allocator<int>> l(3, 2, min_allocator<int>());
-        assert(l.size() == 3);
-        assert(std::distance(l.begin(), l.end()) == 3);
-        std::list<int, min_allocator<int>>::const_iterator i = l.begin();
-        assert(*i == 2);
-        ++i;
-        assert(*i == 2);
-        ++i;
-        assert(*i == 2);
-    }
+  {
+    std::list<int, min_allocator<int>> l(3, 2);
+    assert(l.size() == 3);
+    assert(std::distance(l.begin(), l.end()) == 3);
+    std::list<int, min_allocator<int>>::const_iterator i = l.begin();
+    assert(*i == 2);
+    ++i;
+    assert(*i == 2);
+    ++i;
+    assert(*i == 2);
+  }
+  {
+    std::list<int, min_allocator<int>> l(3, 2, min_allocator<int>());
+    assert(l.size() == 3);
+    assert(std::distance(l.begin(), l.end()) == 3);
+    std::list<int, min_allocator<int>>::const_iterator i = l.begin();
+    assert(*i == 2);
+    ++i;
+    assert(*i == 2);
+    ++i;
+    assert(*i == 2);
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/sequences/list/list.erasure/erase.pass.cpp b/libcxx/test/std/containers/sequences/list/list.erasure/erase.pass.cpp
index 977cf24d1e11a..77f9f89560375 100644
--- a/libcxx/test/std/containers/sequences/list/list.erasure/erase.pass.cpp
+++ b/libcxx/test/std/containers/sequences/list/list.erasure/erase.pass.cpp
@@ -28,8 +28,7 @@ void test0(S s, U val, S expected, std::size_t expected_erased_count) {
 }
 
 template <class S>
-void test()
-{
+void test() {
   test0(S(), 1, S(), 0);
 
   test0(S({1}), 1, S(), 1);
@@ -63,14 +62,13 @@ void test()
   test0(S({1, 2, 1}), opt(3), S({1, 2, 1}), 0);
 }
 
-int main(int, char**)
-{
-    test<std::list<int>>();
-    test<std::list<int, min_allocator<int>>> ();
-    test<std::list<int, test_allocator<int>>> ();
+int main(int, char**) {
+  test<std::list<int>>();
+  test<std::list<int, min_allocator<int>>>();
+  test<std::list<int, test_allocator<int>>>();
 
-    test<std::list<long>>();
-    test<std::list<double>>();
+  test<std::list<long>>();
+  test<std::list<double>>();
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/list/list.erasure/erase_if.pass.cpp b/libcxx/test/std/containers/sequences/list/list.erasure/erase_if.pass.cpp
index db8fd8b21189b..5352a2f454f80 100644
--- a/libcxx/test/std/containers/sequences/list/list.erasure/erase_if.pass.cpp
+++ b/libcxx/test/std/containers/sequences/list/list.erasure/erase_if.pass.cpp
@@ -27,52 +27,50 @@ void test0(S s, Pred p, S expected, std::size_t expected_erased_count) {
 }
 
 template <typename S>
-void test()
-{
-    auto is1 = [](auto v) { return v == 1;};
-    auto is2 = [](auto v) { return v == 2;};
-    auto is3 = [](auto v) { return v == 3;};
-    auto is4 = [](auto v) { return v == 4;};
-    auto True  = [](auto) { return true; };
-    auto False = [](auto) { return false; };
+void test() {
+  auto is1   = [](auto v) { return v == 1; };
+  auto is2   = [](auto v) { return v == 2; };
+  auto is3   = [](auto v) { return v == 3; };
+  auto is4   = [](auto v) { return v == 4; };
+  auto True  = [](auto) { return true; };
+  auto False = [](auto) { return false; };
 
-    test0(S(), is1, S(), 0);
+  test0(S(), is1, S(), 0);
 
-    test0(S({1}), is1, S(), 1);
-    test0(S({1}), is2, S({1}), 0);
+  test0(S({1}), is1, S(), 1);
+  test0(S({1}), is2, S({1}), 0);
 
-    test0(S({1, 2}), is1, S({2}), 1);
-    test0(S({1, 2}), is2, S({1}), 1);
-    test0(S({1, 2}), is3, S({1, 2}), 0);
-    test0(S({1, 1}), is1, S(), 2);
-    test0(S({1, 1}), is3, S({1, 1}), 0);
+  test0(S({1, 2}), is1, S({2}), 1);
+  test0(S({1, 2}), is2, S({1}), 1);
+  test0(S({1, 2}), is3, S({1, 2}), 0);
+  test0(S({1, 1}), is1, S(), 2);
+  test0(S({1, 1}), is3, S({1, 1}), 0);
 
-    test0(S({1, 2, 3}), is1, S({2, 3}), 1);
-    test0(S({1, 2, 3}), is2, S({1, 3}), 1);
-    test0(S({1, 2, 3}), is3, S({1, 2}), 1);
-    test0(S({1, 2, 3}), is4, S({1, 2, 3}), 0);
+  test0(S({1, 2, 3}), is1, S({2, 3}), 1);
+  test0(S({1, 2, 3}), is2, S({1, 3}), 1);
+  test0(S({1, 2, 3}), is3, S({1, 2}), 1);
+  test0(S({1, 2, 3}), is4, S({1, 2, 3}), 0);
 
-    test0(S({1, 1, 1}), is1, S(), 3);
-    test0(S({1, 1, 1}), is2, S({1, 1, 1}), 0);
-    test0(S({1, 1, 2}), is1, S({2}), 2);
-    test0(S({1, 1, 2}), is2, S({1, 1}), 1);
-    test0(S({1, 1, 2}), is3, S({1, 1, 2}), 0);
-    test0(S({1, 2, 2}), is1, S({2, 2}), 1);
-    test0(S({1, 2, 2}), is2, S({1}), 2);
-    test0(S({1, 2, 2}), is3, S({1, 2, 2}), 0);
+  test0(S({1, 1, 1}), is1, S(), 3);
+  test0(S({1, 1, 1}), is2, S({1, 1, 1}), 0);
+  test0(S({1, 1, 2}), is1, S({2}), 2);
+  test0(S({1, 1, 2}), is2, S({1, 1}), 1);
+  test0(S({1, 1, 2}), is3, S({1, 1, 2}), 0);
+  test0(S({1, 2, 2}), is1, S({2, 2}), 1);
+  test0(S({1, 2, 2}), is2, S({1}), 2);
+  test0(S({1, 2, 2}), is3, S({1, 2, 2}), 0);
 
-    test0(S({1, 2, 3}), True, S(), 3);
-    test0(S({1, 2, 3}), False, S({1, 2, 3}), 0);
+  test0(S({1, 2, 3}), True, S(), 3);
+  test0(S({1, 2, 3}), False, S({1, 2, 3}), 0);
 }
 
-int main(int, char**)
-{
-    test<std::list<int>>();
-    test<std::list<int, min_allocator<int>>> ();
-    test<std::list<int, test_allocator<int>>> ();
+int main(int, char**) {
+  test<std::list<int>>();
+  test<std::list<int, min_allocator<int>>>();
+  test<std::list<int, test_allocator<int>>>();
 
-    test<std::list<long>>();
-    test<std::list<double>>();
+  test<std::list<long>>();
+  test<std::list<double>>();
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/list/list.modifiers/clear.pass.cpp b/libcxx/test/std/containers/sequences/list/list.modifiers/clear.pass.cpp
index 0afd0a5483512..5931fd62d037c 100644
--- a/libcxx/test/std/containers/sequences/list/list.modifiers/clear.pass.cpp
+++ b/libcxx/test/std/containers/sequences/list/list.modifiers/clear.pass.cpp
@@ -16,23 +16,22 @@
 #include "test_macros.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
+int main(int, char**) {
+  {
     int a[] = {1, 2, 3};
-    std::list<int> c(a, a+3);
+    std::list<int> c(a, a + 3);
     ASSERT_NOEXCEPT(c.clear());
     c.clear();
     assert(c.empty());
-    }
+  }
 #if TEST_STD_VER >= 11
-    {
+  {
     int a[] = {1, 2, 3};
-    std::list<int, min_allocator<int>> c(a, a+3);
+    std::list<int, min_allocator<int>> c(a, a + 3);
     ASSERT_NOEXCEPT(c.clear());
     c.clear();
     assert(c.empty());
-    }
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/sequences/list/list.modifiers/emplace.pass.cpp b/libcxx/test/std/containers/sequences/list/list.modifiers/emplace.pass.cpp
index 2b6e5999369ae..2f83aa0d317b5 100644
--- a/libcxx/test/std/containers/sequences/list/list.modifiers/emplace.pass.cpp
+++ b/libcxx/test/std/containers/sequences/list/list.modifiers/emplace.pass.cpp
@@ -12,31 +12,28 @@
 
 // template <class... Args> void emplace(const_iterator p, Args&&... args);
 
-
 #include <list>
 #include <cassert>
 
 #include "test_macros.h"
 #include "min_allocator.h"
 
-class A
-{
-    int i_;
-    double d_;
+class A {
+  int i_;
+  double d_;
+
+  A(const A&);
+  A& operator=(const A&);
 
-    A(const A&);
-    A& operator=(const A&);
 public:
-    A(int i, double d)
-        : i_(i), d_(d) {}
+  A(int i, double d) : i_(i), d_(d) {}
 
-    int geti() const {return i_;}
-    double getd() const {return d_;}
+  int geti() const { return i_; }
+  double getd() const { return d_; }
 };
 
-int main(int, char**)
-{
-    {
+int main(int, char**) {
+  {
     std::list<A> c;
     c.emplace(c.cbegin(), 2, 3.5);
     assert(c.size() == 1);
@@ -48,8 +45,8 @@ int main(int, char**)
     assert(c.front().getd() == 3.5);
     assert(c.back().geti() == 3);
     assert(c.back().getd() == 4.5);
-    }
-    {
+  }
+  {
     std::list<A, min_allocator<A>> c;
     c.emplace(c.cbegin(), 2, 3.5);
     assert(c.size() == 1);
@@ -61,8 +58,7 @@ int main(int, char**)
     assert(c.front().getd() == 3.5);
     assert(c.back().geti() == 3);
     assert(c.back().getd() == 4.5);
-    }
-
+  }
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/list/list.modifiers/emplace_back.pass.cpp b/libcxx/test/std/containers/sequences/list/list.modifiers/emplace_back.pass.cpp
index 0d7965be64ded..900f8b83d3e69 100644
--- a/libcxx/test/std/containers/sequences/list/list.modifiers/emplace_back.pass.cpp
+++ b/libcxx/test/std/containers/sequences/list/list.modifiers/emplace_back.pass.cpp
@@ -19,24 +19,22 @@
 #include "test_macros.h"
 #include "min_allocator.h"
 
-class A
-{
-    int i_;
-    double d_;
+class A {
+  int i_;
+  double d_;
+
+  A(const A&);
+  A& operator=(const A&);
 
-    A(const A&);
-    A& operator=(const A&);
 public:
-    A(int i, double d)
-        : i_(i), d_(d) {}
+  A(int i, double d) : i_(i), d_(d) {}
 
-    int geti() const {return i_;}
-    double getd() const {return d_;}
+  int geti() const { return i_; }
+  double getd() const { return d_; }
 };
 
-int main(int, char**)
-{
-    {
+int main(int, char**) {
+  {
     std::list<A> c;
 #if TEST_STD_VER > 14
     A& r1 = c.emplace_back(2, 3.5);
@@ -59,8 +57,8 @@ int main(int, char**)
     assert(c.front().getd() == 3.5);
     assert(c.back().geti() == 3);
     assert(c.back().getd() == 4.5);
-    }
-    {
+  }
+  {
     std::list<A, min_allocator<A>> c;
 #if TEST_STD_VER > 14
     A& r1 = c.emplace_back(2, 3.5);
@@ -83,7 +81,7 @@ int main(int, char**)
     assert(c.front().getd() == 3.5);
     assert(c.back().geti() == 3);
     assert(c.back().getd() == 4.5);
-    }
+  }
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/list/list.modifiers/emplace_front.pass.cpp b/libcxx/test/std/containers/sequences/list/list.modifiers/emplace_front.pass.cpp
index 7d6d18686cb57..665f5077bd429 100644
--- a/libcxx/test/std/containers/sequences/list/list.modifiers/emplace_front.pass.cpp
+++ b/libcxx/test/std/containers/sequences/list/list.modifiers/emplace_front.pass.cpp
@@ -19,24 +19,22 @@
 #include "test_macros.h"
 #include "min_allocator.h"
 
-class A
-{
-    int i_;
-    double d_;
+class A {
+  int i_;
+  double d_;
+
+  A(const A&);
+  A& operator=(const A&);
 
-    A(const A&);
-    A& operator=(const A&);
 public:
-    A(int i, double d)
-        : i_(i), d_(d) {}
+  A(int i, double d) : i_(i), d_(d) {}
 
-    int geti() const {return i_;}
-    double getd() const {return d_;}
+  int geti() const { return i_; }
+  double getd() const { return d_; }
 };
 
-int main(int, char**)
-{
-    {
+int main(int, char**) {
+  {
     std::list<A> c;
 #if TEST_STD_VER > 14
     A& r1 = c.emplace_front(2, 3.5);
@@ -59,9 +57,9 @@ int main(int, char**)
     assert(c.front().getd() == 4.5);
     assert(c.back().geti() == 2);
     assert(c.back().getd() == 3.5);
-    }
+  }
 
-    {
+  {
     std::list<A, min_allocator<A>> c;
 #if TEST_STD_VER > 14
     A& r1 = c.emplace_front(2, 3.5);
@@ -84,7 +82,7 @@ int main(int, char**)
     assert(c.front().getd() == 4.5);
     assert(c.back().geti() == 2);
     assert(c.back().getd() == 3.5);
-    }
+  }
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/list/list.modifiers/erase_iter.pass.cpp b/libcxx/test/std/containers/sequences/list/list.modifiers/erase_iter.pass.cpp
index b559f8e7118fa..ba139b4367d73 100644
--- a/libcxx/test/std/containers/sequences/list/list.modifiers/erase_iter.pass.cpp
+++ b/libcxx/test/std/containers/sequences/list/list.modifiers/erase_iter.pass.cpp
@@ -16,11 +16,10 @@
 #include "test_macros.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
+int main(int, char**) {
+  {
     int a1[] = {1, 2, 3};
-    std::list<int> l1(a1, a1+3);
+    std::list<int> l1(a1, a1 + 3);
     std::list<int>::const_iterator i = l1.begin();
     ++i;
     std::list<int>::iterator j = l1.erase(i);
@@ -38,11 +37,11 @@ int main(int, char**)
     assert(j == l1.end());
     assert(l1.size() == 0);
     assert(std::distance(l1.begin(), l1.end()) == 0);
-    }
+  }
 #if TEST_STD_VER >= 11
-    {
+  {
     int a1[] = {1, 2, 3};
-    std::list<int, min_allocator<int>> l1(a1, a1+3);
+    std::list<int, min_allocator<int>> l1(a1, a1 + 3);
     std::list<int, min_allocator<int>>::const_iterator i = l1.begin();
     ++i;
     std::list<int, min_allocator<int>>::iterator j = l1.erase(i);
@@ -60,7 +59,7 @@ int main(int, char**)
     assert(j == l1.end());
     assert(l1.size() == 0);
     assert(std::distance(l1.begin(), l1.end()) == 0);
-    }
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/sequences/list/list.modifiers/erase_iter_iter.pass.cpp b/libcxx/test/std/containers/sequences/list/list.modifiers/erase_iter_iter.pass.cpp
index d8ea38632d48f..cc8d537032d04 100644
--- a/libcxx/test/std/containers/sequences/list/list.modifiers/erase_iter_iter.pass.cpp
+++ b/libcxx/test/std/containers/sequences/list/list.modifiers/erase_iter_iter.pass.cpp
@@ -16,70 +16,69 @@
 #include "test_macros.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    int a1[] = {1, 2, 3};
-    {
-        std::list<int> l1(a1, a1+3);
-        std::list<int>::iterator i = l1.erase(l1.cbegin(), l1.cbegin());
-        assert(l1.size() == 3);
-        assert(std::distance(l1.cbegin(), l1.cend()) == 3);
-        assert(i == l1.begin());
-    }
-    {
-        std::list<int> l1(a1, a1+3);
-        std::list<int>::iterator i = l1.erase(l1.cbegin(), std::next(l1.cbegin()));
-        assert(l1.size() == 2);
-        assert(std::distance(l1.cbegin(), l1.cend()) == 2);
-        assert(i == l1.begin());
-        assert(l1 == std::list<int>(a1+1, a1+3));
-    }
-    {
-        std::list<int> l1(a1, a1+3);
-        std::list<int>::iterator i = l1.erase(l1.cbegin(), std::next(l1.cbegin(), 2));
-        assert(l1.size() == 1);
-        assert(std::distance(l1.cbegin(), l1.cend()) == 1);
-        assert(i == l1.begin());
-        assert(l1 == std::list<int>(a1+2, a1+3));
-    }
-    {
-        std::list<int> l1(a1, a1+3);
-        std::list<int>::iterator i = l1.erase(l1.cbegin(), std::next(l1.cbegin(), 3));
-        assert(l1.size() == 0);
-        assert(std::distance(l1.cbegin(), l1.cend()) == 0);
-        assert(i == l1.begin());
-    }
+int main(int, char**) {
+  int a1[] = {1, 2, 3};
+  {
+    std::list<int> l1(a1, a1 + 3);
+    std::list<int>::iterator i = l1.erase(l1.cbegin(), l1.cbegin());
+    assert(l1.size() == 3);
+    assert(std::distance(l1.cbegin(), l1.cend()) == 3);
+    assert(i == l1.begin());
+  }
+  {
+    std::list<int> l1(a1, a1 + 3);
+    std::list<int>::iterator i = l1.erase(l1.cbegin(), std::next(l1.cbegin()));
+    assert(l1.size() == 2);
+    assert(std::distance(l1.cbegin(), l1.cend()) == 2);
+    assert(i == l1.begin());
+    assert(l1 == std::list<int>(a1 + 1, a1 + 3));
+  }
+  {
+    std::list<int> l1(a1, a1 + 3);
+    std::list<int>::iterator i = l1.erase(l1.cbegin(), std::next(l1.cbegin(), 2));
+    assert(l1.size() == 1);
+    assert(std::distance(l1.cbegin(), l1.cend()) == 1);
+    assert(i == l1.begin());
+    assert(l1 == std::list<int>(a1 + 2, a1 + 3));
+  }
+  {
+    std::list<int> l1(a1, a1 + 3);
+    std::list<int>::iterator i = l1.erase(l1.cbegin(), std::next(l1.cbegin(), 3));
+    assert(l1.size() == 0);
+    assert(std::distance(l1.cbegin(), l1.cend()) == 0);
+    assert(i == l1.begin());
+  }
 #if TEST_STD_VER >= 11
-    {
-        std::list<int, min_allocator<int>> l1(a1, a1+3);
-        std::list<int, min_allocator<int>>::iterator i = l1.erase(l1.cbegin(), l1.cbegin());
-        assert(l1.size() == 3);
-        assert(std::distance(l1.cbegin(), l1.cend()) == 3);
-        assert(i == l1.begin());
-    }
-    {
-        std::list<int, min_allocator<int>> l1(a1, a1+3);
-        std::list<int, min_allocator<int>>::iterator i = l1.erase(l1.cbegin(), std::next(l1.cbegin()));
-        assert(l1.size() == 2);
-        assert(std::distance(l1.cbegin(), l1.cend()) == 2);
-        assert(i == l1.begin());
-        assert((l1 == std::list<int, min_allocator<int>>(a1+1, a1+3)));
-    }
-    {
-        std::list<int, min_allocator<int>> l1(a1, a1+3);
-        std::list<int, min_allocator<int>>::iterator i = l1.erase(l1.cbegin(), std::next(l1.cbegin(), 2));
-        assert(l1.size() == 1);
-        assert(std::distance(l1.cbegin(), l1.cend()) == 1);
-        assert(i == l1.begin());
-        assert((l1 == std::list<int, min_allocator<int>>(a1+2, a1+3)));
-    }
-    {
-        std::list<int, min_allocator<int>> l1(a1, a1+3);
-        std::list<int, min_allocator<int>>::iterator i = l1.erase(l1.cbegin(), std::next(l1.cbegin(), 3));
-        assert(l1.size() == 0);
-        assert(std::distance(l1.cbegin(), l1.cend()) == 0);
-        assert(i == l1.begin());
-    }
+  {
+    std::list<int, min_allocator<int>> l1(a1, a1 + 3);
+    std::list<int, min_allocator<int>>::iterator i = l1.erase(l1.cbegin(), l1.cbegin());
+    assert(l1.size() == 3);
+    assert(std::distance(l1.cbegin(), l1.cend()) == 3);
+    assert(i == l1.begin());
+  }
+  {
+    std::list<int, min_allocator<int>> l1(a1, a1 + 3);
+    std::list<int, min_allocator<int>>::iterator i = l1.erase(l1.cbegin(), std::next(l1.cbegin()));
+    assert(l1.size() == 2);
+    assert(std::distance(l1.cbegin(), l1.cend()) == 2);
+    assert(i == l1.begin());
+    assert((l1 == std::list<int, min_allocator<int>>(a1 + 1, a1 + 3)));
+  }
+  {
+    std::list<int, min_allocator<int>> l1(a1, a1 + 3);
+    std::list<int, min_allocator<int>>::iterator i = l1.erase(l1.cbegin(), std::next(l1.cbegin(), 2));
+    assert(l1.size() == 1);
+    assert(std::distance(l1.cbegin(), l1.cend()) == 1);
+    assert(i == l1.begin());
+    assert((l1 == std::list<int, min_allocator<int>>(a1 + 2, a1 + 3)));
+  }
+  {
+    std::list<int, min_allocator<int>> l1(a1, a1 + 3);
+    std::list<int, min_allocator<int>>::iterator i = l1.erase(l1.cbegin(), std::next(l1.cbegin(), 3));
+    assert(l1.size() == 0);
+    assert(std::distance(l1.cbegin(), l1.cend()) == 0);
+    assert(i == l1.begin());
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/sequences/list/list.modifiers/insert_iter_initializer_list.pass.cpp b/libcxx/test/std/containers/sequences/list/list.modifiers/insert_iter_initializer_list.pass.cpp
index 32adf3a42d7c4..8bd01c940d958 100644
--- a/libcxx/test/std/containers/sequences/list/list.modifiers/insert_iter_initializer_list.pass.cpp
+++ b/libcxx/test/std/containers/sequences/list/list.modifiers/insert_iter_initializer_list.pass.cpp
@@ -18,9 +18,8 @@
 #include "test_macros.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
+int main(int, char**) {
+  {
     std::list<int> d(10, 1);
     std::list<int>::iterator i = d.insert(std::next(d.cbegin(), 2), {3, 4, 5, 6});
     assert(d.size() == 14);
@@ -40,8 +39,8 @@ int main(int, char**)
     assert(*i++ == 1);
     assert(*i++ == 1);
     assert(*i++ == 1);
-    }
-    {
+  }
+  {
     std::list<int, min_allocator<int>> d(10, 1);
     std::list<int, min_allocator<int>>::iterator i = d.insert(std::next(d.cbegin(), 2), {3, 4, 5, 6});
     assert(d.size() == 14);
@@ -61,7 +60,7 @@ int main(int, char**)
     assert(*i++ == 1);
     assert(*i++ == 1);
     assert(*i++ == 1);
-    }
+  }
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/list/list.modifiers/insert_iter_iter_iter.pass.cpp b/libcxx/test/std/containers/sequences/list/list.modifiers/insert_iter_iter_iter.pass.cpp
index d1177e94fbb7e..bab125ca62094 100644
--- a/libcxx/test/std/containers/sequences/list/list.modifiers/insert_iter_iter_iter.pass.cpp
+++ b/libcxx/test/std/containers/sequences/list/list.modifiers/insert_iter_iter_iter.pass.cpp
@@ -22,70 +22,66 @@
 
 template <class List>
 void test() {
-    int a1[] = {1, 2, 3};
-    List l1;
-    typename List::iterator i = l1.insert(l1.begin(), a1, a1+3);
-    assert(i == l1.begin());
-    assert(l1.size() == 3);
-    assert(std::distance(l1.begin(), l1.end()) == 3);
-    i = l1.begin();
-    assert(*i == 1);
-    ++i;
-    assert(*i == 2);
-    ++i;
-    assert(*i == 3);
-    int a2[] = {4, 5, 6};
-    i = l1.insert(i, a2, a2+3);
-    assert(*i == 4);
-    assert(l1.size() == 6);
-    assert(std::distance(l1.begin(), l1.end()) == 6);
-    i = l1.begin();
-    assert(*i == 1);
-    ++i;
-    assert(*i == 2);
-    ++i;
-    assert(*i == 4);
-    ++i;
-    assert(*i == 5);
-    ++i;
-    assert(*i == 6);
-    ++i;
-    assert(*i == 3);
+  int a1[] = {1, 2, 3};
+  List l1;
+  typename List::iterator i = l1.insert(l1.begin(), a1, a1 + 3);
+  assert(i == l1.begin());
+  assert(l1.size() == 3);
+  assert(std::distance(l1.begin(), l1.end()) == 3);
+  i = l1.begin();
+  assert(*i == 1);
+  ++i;
+  assert(*i == 2);
+  ++i;
+  assert(*i == 3);
+  int a2[] = {4, 5, 6};
+  i        = l1.insert(i, a2, a2 + 3);
+  assert(*i == 4);
+  assert(l1.size() == 6);
+  assert(std::distance(l1.begin(), l1.end()) == 6);
+  i = l1.begin();
+  assert(*i == 1);
+  ++i;
+  assert(*i == 2);
+  ++i;
+  assert(*i == 4);
+  ++i;
+  assert(*i == 5);
+  ++i;
+  assert(*i == 6);
+  ++i;
+  assert(*i == 3);
 
 #if !defined(TEST_HAS_NO_EXCEPTIONS) && !defined(DISABLE_NEW_COUNT)
-    globalMemCounter.throw_after = 2;
-    int save_count = globalMemCounter.outstanding_new;
-    try
-    {
-        i = l1.insert(i, a2, a2+3);
-        assert(false);
-    }
-    catch (...)
-    {
-    }
-    assert(globalMemCounter.checkOutstandingNewEq(save_count));
-    assert(l1.size() == 6);
-    assert(std::distance(l1.begin(), l1.end()) == 6);
-    i = l1.begin();
-    assert(*i == 1);
-    ++i;
-    assert(*i == 2);
-    ++i;
-    assert(*i == 4);
-    ++i;
-    assert(*i == 5);
-    ++i;
-    assert(*i == 6);
-    ++i;
-    assert(*i == 3);
+  globalMemCounter.throw_after = 2;
+  int save_count               = globalMemCounter.outstanding_new;
+  try {
+    i = l1.insert(i, a2, a2 + 3);
+    assert(false);
+  } catch (...) {
+  }
+  assert(globalMemCounter.checkOutstandingNewEq(save_count));
+  assert(l1.size() == 6);
+  assert(std::distance(l1.begin(), l1.end()) == 6);
+  i = l1.begin();
+  assert(*i == 1);
+  ++i;
+  assert(*i == 2);
+  ++i;
+  assert(*i == 4);
+  ++i;
+  assert(*i == 5);
+  ++i;
+  assert(*i == 6);
+  ++i;
+  assert(*i == 3);
 #endif
 }
 
-int main(int, char**)
-{
-    test<std::list<int> >();
+int main(int, char**) {
+  test<std::list<int> >();
 #if TEST_STD_VER >= 11
-    test<std::list<int, min_allocator<int>>>();
+  test<std::list<int, min_allocator<int>>>();
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/sequences/list/list.modifiers/insert_iter_rvalue.pass.cpp b/libcxx/test/std/containers/sequences/list/list.modifiers/insert_iter_rvalue.pass.cpp
index 5a548916aef8a..8bb513208eb7f 100644
--- a/libcxx/test/std/containers/sequences/list/list.modifiers/insert_iter_rvalue.pass.cpp
+++ b/libcxx/test/std/containers/sequences/list/list.modifiers/insert_iter_rvalue.pass.cpp
@@ -19,9 +19,8 @@
 #include "MoveOnly.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
+int main(int, char**) {
+  {
     std::list<MoveOnly> l1;
     l1.insert(l1.cend(), MoveOnly(1));
     assert(l1.size() == 1);
@@ -30,8 +29,8 @@ int main(int, char**)
     assert(l1.size() == 2);
     assert(l1.front() == MoveOnly(2));
     assert(l1.back() == MoveOnly(1));
-    }
-    {
+  }
+  {
     std::list<MoveOnly, min_allocator<MoveOnly>> l1;
     l1.insert(l1.cend(), MoveOnly(1));
     assert(l1.size() == 1);
@@ -40,7 +39,7 @@ int main(int, char**)
     assert(l1.size() == 2);
     assert(l1.front() == MoveOnly(2));
     assert(l1.back() == MoveOnly(1));
-    }
+  }
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/list/list.modifiers/insert_iter_size_value.pass.cpp b/libcxx/test/std/containers/sequences/list/list.modifiers/insert_iter_size_value.pass.cpp
index 9fda96d016078..32ee7a73406dc 100644
--- a/libcxx/test/std/containers/sequences/list/list.modifiers/insert_iter_size_value.pass.cpp
+++ b/libcxx/test/std/containers/sequences/list/list.modifiers/insert_iter_size_value.pass.cpp
@@ -22,33 +22,29 @@
 
 template <class List>
 void test() {
-    int a1[] = {1, 2, 3};
-    int a2[] = {1, 4, 4, 4, 4, 4, 2, 3};
-    List l1(a1, a1+3);
-    typename List::iterator i = l1.insert(std::next(l1.cbegin()), 5, 4);
-    assert(i == std::next(l1.begin()));
-    assert(l1 == List(a2, a2+8));
+  int a1[] = {1, 2, 3};
+  int a2[] = {1, 4, 4, 4, 4, 4, 2, 3};
+  List l1(a1, a1 + 3);
+  typename List::iterator i = l1.insert(std::next(l1.cbegin()), 5, 4);
+  assert(i == std::next(l1.begin()));
+  assert(l1 == List(a2, a2 + 8));
 #ifndef TEST_HAS_NO_EXCEPTIONS
-    globalMemCounter.throw_after = 4;
-    int save_count = globalMemCounter.outstanding_new;
-    try
-    {
-        i = l1.insert(i, 5, 5);
-        assert(false);
-    }
-    catch (...)
-    {
-    }
-    assert(globalMemCounter.checkOutstandingNewEq(save_count));
-    assert(l1 == List(a2, a2+8));
+  globalMemCounter.throw_after = 4;
+  int save_count               = globalMemCounter.outstanding_new;
+  try {
+    i = l1.insert(i, 5, 5);
+    assert(false);
+  } catch (...) {
+  }
+  assert(globalMemCounter.checkOutstandingNewEq(save_count));
+  assert(l1 == List(a2, a2 + 8));
 #endif
 }
 
-int main(int, char**)
-{
-    test<std::list<int> >();
+int main(int, char**) {
+  test<std::list<int> >();
 #if TEST_STD_VER >= 11
-    test<std::list<int, min_allocator<int>>>();
+  test<std::list<int, min_allocator<int>>>();
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/sequences/list/list.modifiers/insert_iter_value.pass.cpp b/libcxx/test/std/containers/sequences/list/list.modifiers/insert_iter_value.pass.cpp
index 3e0edef65aaf0..129fe05cb39d2 100644
--- a/libcxx/test/std/containers/sequences/list/list.modifiers/insert_iter_value.pass.cpp
+++ b/libcxx/test/std/containers/sequences/list/list.modifiers/insert_iter_value.pass.cpp
@@ -19,38 +19,33 @@
 #include "count_new.h"
 
 template <class List>
-void test()
-{
-    int a1[] = {1, 2, 3};
-    int a2[] = {1, 4, 2, 3};
-    List l1(a1, a1+3);
-    typename List::iterator i = l1.insert(std::next(l1.cbegin()), 4);
-    assert(i == std::next(l1.begin()));
-    assert(l1.size() == 4);
-    assert(std::distance(l1.begin(), l1.end()) == 4);
-    assert(l1 == List(a2, a2+4));
+void test() {
+  int a1[] = {1, 2, 3};
+  int a2[] = {1, 4, 2, 3};
+  List l1(a1, a1 + 3);
+  typename List::iterator i = l1.insert(std::next(l1.cbegin()), 4);
+  assert(i == std::next(l1.begin()));
+  assert(l1.size() == 4);
+  assert(std::distance(l1.begin(), l1.end()) == 4);
+  assert(l1 == List(a2, a2 + 4));
 
 #if !defined(TEST_HAS_NO_EXCEPTIONS) && !defined(DISABLE_NEW_COUNT)
-    globalMemCounter.throw_after = 0;
-    int save_count = globalMemCounter.outstanding_new;
-    try
-    {
-        i = l1.insert(i, 5);
-        assert(false);
-    }
-    catch (...)
-    {
-    }
-    assert(globalMemCounter.checkOutstandingNewEq(save_count));
-    assert(l1 == List(a2, a2+4));
+  globalMemCounter.throw_after = 0;
+  int save_count               = globalMemCounter.outstanding_new;
+  try {
+    i = l1.insert(i, 5);
+    assert(false);
+  } catch (...) {
+  }
+  assert(globalMemCounter.checkOutstandingNewEq(save_count));
+  assert(l1 == List(a2, a2 + 4));
 #endif
 }
 
-int main(int, char**)
-{
-    test<std::list<int> >();
+int main(int, char**) {
+  test<std::list<int> >();
 #if TEST_STD_VER >= 11
-    test<std::list<int, min_allocator<int>>>();
+  test<std::list<int, min_allocator<int>>>();
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/sequences/list/list.modifiers/pop_back.pass.cpp b/libcxx/test/std/containers/sequences/list/list.modifiers/pop_back.pass.cpp
index c4c88d421d608..aaa225b147767 100644
--- a/libcxx/test/std/containers/sequences/list/list.modifiers/pop_back.pass.cpp
+++ b/libcxx/test/std/containers/sequences/list/list.modifiers/pop_back.pass.cpp
@@ -16,29 +16,28 @@
 #include "test_macros.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
+int main(int, char**) {
+  {
     int a[] = {1, 2, 3};
-    std::list<int> c(a, a+3);
+    std::list<int> c(a, a + 3);
     c.pop_back();
-    assert(c == std::list<int>(a, a+2));
+    assert(c == std::list<int>(a, a + 2));
     c.pop_back();
-    assert(c == std::list<int>(a, a+1));
+    assert(c == std::list<int>(a, a + 1));
     c.pop_back();
     assert(c.empty());
-    }
+  }
 #if TEST_STD_VER >= 11
-    {
+  {
     int a[] = {1, 2, 3};
-    std::list<int, min_allocator<int>> c(a, a+3);
+    std::list<int, min_allocator<int>> c(a, a + 3);
     c.pop_back();
-    assert((c == std::list<int, min_allocator<int>>(a, a+2)));
+    assert((c == std::list<int, min_allocator<int>>(a, a + 2)));
     c.pop_back();
-    assert((c == std::list<int, min_allocator<int>>(a, a+1)));
+    assert((c == std::list<int, min_allocator<int>>(a, a + 1)));
     c.pop_back();
     assert(c.empty());
-    }
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/sequences/list/list.modifiers/pop_front.pass.cpp b/libcxx/test/std/containers/sequences/list/list.modifiers/pop_front.pass.cpp
index 7f20e0a4c11b6..33b8ff35c524c 100644
--- a/libcxx/test/std/containers/sequences/list/list.modifiers/pop_front.pass.cpp
+++ b/libcxx/test/std/containers/sequences/list/list.modifiers/pop_front.pass.cpp
@@ -16,29 +16,28 @@
 #include "test_macros.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
+int main(int, char**) {
+  {
     int a[] = {1, 2, 3};
-    std::list<int> c(a, a+3);
+    std::list<int> c(a, a + 3);
     c.pop_front();
-    assert(c == std::list<int>(a+1, a+3));
+    assert(c == std::list<int>(a + 1, a + 3));
     c.pop_front();
-    assert(c == std::list<int>(a+2, a+3));
+    assert(c == std::list<int>(a + 2, a + 3));
     c.pop_front();
     assert(c.empty());
-    }
+  }
 #if TEST_STD_VER >= 11
-    {
+  {
     int a[] = {1, 2, 3};
-    std::list<int, min_allocator<int>> c(a, a+3);
+    std::list<int, min_allocator<int>> c(a, a + 3);
     c.pop_front();
-    assert((c == std::list<int, min_allocator<int>>(a+1, a+3)));
+    assert((c == std::list<int, min_allocator<int>>(a + 1, a + 3)));
     c.pop_front();
-    assert((c == std::list<int, min_allocator<int>>(a+2, a+3)));
+    assert((c == std::list<int, min_allocator<int>>(a + 2, a + 3)));
     c.pop_front();
     assert(c.empty());
-    }
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/sequences/list/list.modifiers/push_back.pass.cpp b/libcxx/test/std/containers/sequences/list/list.modifiers/push_back.pass.cpp
index 29f22a566bf34..582f4a200ac26 100644
--- a/libcxx/test/std/containers/sequences/list/list.modifiers/push_back.pass.cpp
+++ b/libcxx/test/std/containers/sequences/list/list.modifiers/push_back.pass.cpp
@@ -16,23 +16,22 @@
 #include "test_macros.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
+int main(int, char**) {
+  {
     std::list<int> c;
     for (int i = 0; i < 5; ++i)
-        c.push_back(i);
+      c.push_back(i);
     int a[] = {0, 1, 2, 3, 4};
-    assert(c == std::list<int>(a, a+5));
-    }
+    assert(c == std::list<int>(a, a + 5));
+  }
 #if TEST_STD_VER >= 11
-    {
+  {
     std::list<int, min_allocator<int>> c;
     for (int i = 0; i < 5; ++i)
-        c.push_back(i);
+      c.push_back(i);
     int a[] = {0, 1, 2, 3, 4};
-    assert((c == std::list<int, min_allocator<int>>(a, a+5)));
-    }
+    assert((c == std::list<int, min_allocator<int>>(a, a + 5)));
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/sequences/list/list.modifiers/push_back_exception_safety.pass.cpp b/libcxx/test/std/containers/sequences/list/list.modifiers/push_back_exception_safety.pass.cpp
index 0ef4bc4676e88..ca53f160eea81 100644
--- a/libcxx/test/std/containers/sequences/list/list.modifiers/push_back_exception_safety.pass.cpp
+++ b/libcxx/test/std/containers/sequences/list/list.modifiers/push_back_exception_safety.pass.cpp
@@ -21,14 +21,23 @@
 static bool gCopyConstructorShouldThrow = false;
 
 class CMyClass {
-    public: CMyClass();
-    public: CMyClass(const CMyClass& iOther);
-    public: ~CMyClass();
+public:
+  CMyClass();
 
-    private: int fMagicValue;
+public:
+  CMyClass(const CMyClass& iOther);
 
-    private: static int kStartedConstructionMagicValue;
-    private: static int kFinishedConstructionMagicValue;
+public:
+  ~CMyClass();
+
+private:
+  int fMagicValue;
+
+private:
+  static int kStartedConstructionMagicValue;
+
+private:
+  static int kFinishedConstructionMagicValue;
 };
 
 // Value for fMagicValue when the constructor has started running, but not yet finished
@@ -36,42 +45,36 @@ int CMyClass::kStartedConstructionMagicValue = 0;
 // Value for fMagicValue when the constructor has finished running
 int CMyClass::kFinishedConstructionMagicValue = 12345;
 
-CMyClass::CMyClass() :
-    fMagicValue(kStartedConstructionMagicValue)
-{
-    // Signal that the constructor has finished running
-    fMagicValue = kFinishedConstructionMagicValue;
+CMyClass::CMyClass() : fMagicValue(kStartedConstructionMagicValue) {
+  // Signal that the constructor has finished running
+  fMagicValue = kFinishedConstructionMagicValue;
 }
 
-CMyClass::CMyClass(const CMyClass& /*iOther*/) :
-    fMagicValue(kStartedConstructionMagicValue)
-{
-    // If requested, throw an exception _before_ setting fMagicValue to kFinishedConstructionMagicValue
-    if (gCopyConstructorShouldThrow) {
-      throw std::exception();
-    }
-    // Signal that the constructor has finished running
-    fMagicValue = kFinishedConstructionMagicValue;
+CMyClass::CMyClass(const CMyClass& /*iOther*/) : fMagicValue(kStartedConstructionMagicValue) {
+  // If requested, throw an exception _before_ setting fMagicValue to kFinishedConstructionMagicValue
+  if (gCopyConstructorShouldThrow) {
+    throw std::exception();
+  }
+  // Signal that the constructor has finished running
+  fMagicValue = kFinishedConstructionMagicValue;
 }
 
 CMyClass::~CMyClass() {
-    // Only instances for which the constructor has finished running should be destructed
-    assert(fMagicValue == kFinishedConstructionMagicValue);
+  // Only instances for which the constructor has finished running should be destructed
+  assert(fMagicValue == kFinishedConstructionMagicValue);
 }
 
-int main(int, char**)
-{
-    CMyClass instance;
-    std::list<CMyClass> vec;
+int main(int, char**) {
+  CMyClass instance;
+  std::list<CMyClass> vec;
 
-    vec.push_back(instance);
+  vec.push_back(instance);
 
-    gCopyConstructorShouldThrow = true;
-    try {
-        vec.push_back(instance);
-    }
-    catch (...) {
-    }
+  gCopyConstructorShouldThrow = true;
+  try {
+    vec.push_back(instance);
+  } catch (...) {
+  }
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/list/list.modifiers/push_back_rvalue.pass.cpp b/libcxx/test/std/containers/sequences/list/list.modifiers/push_back_rvalue.pass.cpp
index 8adeb0d44e2b0..6a31d81d694f3 100644
--- a/libcxx/test/std/containers/sequences/list/list.modifiers/push_back_rvalue.pass.cpp
+++ b/libcxx/test/std/containers/sequences/list/list.modifiers/push_back_rvalue.pass.cpp
@@ -19,9 +19,8 @@
 #include "MoveOnly.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
+int main(int, char**) {
+  {
     std::list<MoveOnly> l1;
     l1.push_back(MoveOnly(1));
     assert(l1.size() == 1);
@@ -30,8 +29,8 @@ int main(int, char**)
     assert(l1.size() == 2);
     assert(l1.front() == MoveOnly(1));
     assert(l1.back() == MoveOnly(2));
-    }
-    {
+  }
+  {
     std::list<MoveOnly, min_allocator<MoveOnly>> l1;
     l1.push_back(MoveOnly(1));
     assert(l1.size() == 1);
@@ -40,7 +39,7 @@ int main(int, char**)
     assert(l1.size() == 2);
     assert(l1.front() == MoveOnly(1));
     assert(l1.back() == MoveOnly(2));
-    }
+  }
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/list/list.modifiers/push_front.pass.cpp b/libcxx/test/std/containers/sequences/list/list.modifiers/push_front.pass.cpp
index 2b6393179a209..3b5f74a217a2f 100644
--- a/libcxx/test/std/containers/sequences/list/list.modifiers/push_front.pass.cpp
+++ b/libcxx/test/std/containers/sequences/list/list.modifiers/push_front.pass.cpp
@@ -16,23 +16,22 @@
 #include "test_macros.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
+int main(int, char**) {
+  {
     std::list<int> c;
     for (int i = 0; i < 5; ++i)
-        c.push_front(i);
+      c.push_front(i);
     int a[] = {4, 3, 2, 1, 0};
-    assert(c == std::list<int>(a, a+5));
-    }
+    assert(c == std::list<int>(a, a + 5));
+  }
 #if TEST_STD_VER >= 11
-    {
+  {
     std::list<int, min_allocator<int>> c;
     for (int i = 0; i < 5; ++i)
-        c.push_front(i);
+      c.push_front(i);
     int a[] = {4, 3, 2, 1, 0};
-    assert((c == std::list<int, min_allocator<int>>(a, a+5)));
-    }
+    assert((c == std::list<int, min_allocator<int>>(a, a + 5)));
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/sequences/list/list.modifiers/push_front_exception_safety.pass.cpp b/libcxx/test/std/containers/sequences/list/list.modifiers/push_front_exception_safety.pass.cpp
index 054559d177748..8922cab79d54f 100644
--- a/libcxx/test/std/containers/sequences/list/list.modifiers/push_front_exception_safety.pass.cpp
+++ b/libcxx/test/std/containers/sequences/list/list.modifiers/push_front_exception_safety.pass.cpp
@@ -21,14 +21,23 @@
 static bool gCopyConstructorShouldThrow = false;
 
 class CMyClass {
-    public: CMyClass();
-    public: CMyClass(const CMyClass& iOther);
-    public: ~CMyClass();
+public:
+  CMyClass();
 
-    private: int fMagicValue;
+public:
+  CMyClass(const CMyClass& iOther);
 
-    private: static int kStartedConstructionMagicValue;
-    private: static int kFinishedConstructionMagicValue;
+public:
+  ~CMyClass();
+
+private:
+  int fMagicValue;
+
+private:
+  static int kStartedConstructionMagicValue;
+
+private:
+  static int kFinishedConstructionMagicValue;
 };
 
 // Value for fMagicValue when the constructor has started running, but not yet finished
@@ -36,42 +45,36 @@ int CMyClass::kStartedConstructionMagicValue = 0;
 // Value for fMagicValue when the constructor has finished running
 int CMyClass::kFinishedConstructionMagicValue = 12345;
 
-CMyClass::CMyClass() :
-    fMagicValue(kStartedConstructionMagicValue)
-{
-    // Signal that the constructor has finished running
-    fMagicValue = kFinishedConstructionMagicValue;
+CMyClass::CMyClass() : fMagicValue(kStartedConstructionMagicValue) {
+  // Signal that the constructor has finished running
+  fMagicValue = kFinishedConstructionMagicValue;
 }
 
-CMyClass::CMyClass(const CMyClass& /*iOther*/) :
-    fMagicValue(kStartedConstructionMagicValue)
-{
-    // If requested, throw an exception _before_ setting fMagicValue to kFinishedConstructionMagicValue
-    if (gCopyConstructorShouldThrow) {
-      throw std::exception();
-    }
-    // Signal that the constructor has finished running
-    fMagicValue = kFinishedConstructionMagicValue;
+CMyClass::CMyClass(const CMyClass& /*iOther*/) : fMagicValue(kStartedConstructionMagicValue) {
+  // If requested, throw an exception _before_ setting fMagicValue to kFinishedConstructionMagicValue
+  if (gCopyConstructorShouldThrow) {
+    throw std::exception();
+  }
+  // Signal that the constructor has finished running
+  fMagicValue = kFinishedConstructionMagicValue;
 }
 
 CMyClass::~CMyClass() {
-    // Only instances for which the constructor has finished running should be destructed
-    assert(fMagicValue == kFinishedConstructionMagicValue);
+  // Only instances for which the constructor has finished running should be destructed
+  assert(fMagicValue == kFinishedConstructionMagicValue);
 }
 
-int main(int, char**)
-{
-    CMyClass instance;
-    std::list<CMyClass> vec;
+int main(int, char**) {
+  CMyClass instance;
+  std::list<CMyClass> vec;
 
-    vec.push_front(instance);
+  vec.push_front(instance);
 
-    gCopyConstructorShouldThrow = true;
-    try {
-        vec.push_front(instance);
-    }
-    catch (...) {
-    }
+  gCopyConstructorShouldThrow = true;
+  try {
+    vec.push_front(instance);
+  } catch (...) {
+  }
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/list/list.modifiers/push_front_rvalue.pass.cpp b/libcxx/test/std/containers/sequences/list/list.modifiers/push_front_rvalue.pass.cpp
index b3d0a24eaa45f..0d41b8fd8553d 100644
--- a/libcxx/test/std/containers/sequences/list/list.modifiers/push_front_rvalue.pass.cpp
+++ b/libcxx/test/std/containers/sequences/list/list.modifiers/push_front_rvalue.pass.cpp
@@ -19,9 +19,8 @@
 #include "MoveOnly.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
+int main(int, char**) {
+  {
     std::list<MoveOnly> l1;
     l1.push_front(MoveOnly(1));
     assert(l1.size() == 1);
@@ -30,8 +29,8 @@ int main(int, char**)
     assert(l1.size() == 2);
     assert(l1.front() == MoveOnly(2));
     assert(l1.back() == MoveOnly(1));
-    }
-    {
+  }
+  {
     std::list<MoveOnly, min_allocator<MoveOnly>> l1;
     l1.push_front(MoveOnly(1));
     assert(l1.size() == 1);
@@ -40,7 +39,7 @@ int main(int, char**)
     assert(l1.size() == 2);
     assert(l1.front() == MoveOnly(2));
     assert(l1.back() == MoveOnly(1));
-    }
+  }
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/list/list.ops/merge.pass.cpp b/libcxx/test/std/containers/sequences/list/list.ops/merge.pass.cpp
index 5faa3f34a3626..7f82f65fd493b 100644
--- a/libcxx/test/std/containers/sequences/list/list.ops/merge.pass.cpp
+++ b/libcxx/test/std/containers/sequences/list/list.ops/merge.pass.cpp
@@ -17,37 +17,36 @@
 #include "test_macros.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
+int main(int, char**) {
+  {
     int a1[] = {1, 3, 7, 9, 10};
     int a2[] = {0, 2, 4, 5, 6, 8, 11};
     int a3[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11};
-    std::list<int> c1(a1, a1+sizeof(a1)/sizeof(a1[0]));
-    std::list<int> c2(a2, a2+sizeof(a2)/sizeof(a2[0]));
+    std::list<int> c1(a1, a1 + sizeof(a1) / sizeof(a1[0]));
+    std::list<int> c2(a2, a2 + sizeof(a2) / sizeof(a2[0]));
     c1.merge(c2);
-    assert(c1 == std::list<int>(a3, a3+sizeof(a3)/sizeof(a3[0])));
+    assert(c1 == std::list<int>(a3, a3 + sizeof(a3) / sizeof(a3[0])));
     assert(c2.empty());
-    }
+  }
 
-    {
+  {
     int a1[] = {1, 3, 7, 9, 10};
-    std::list<int> c1(a1, a1+sizeof(a1)/sizeof(a1[0]));
+    std::list<int> c1(a1, a1 + sizeof(a1) / sizeof(a1[0]));
     c1.merge(c1);
-    assert((c1 == std::list<int>(a1, a1+sizeof(a1)/sizeof(a1[0]))));
-    }
+    assert((c1 == std::list<int>(a1, a1 + sizeof(a1) / sizeof(a1[0]))));
+  }
 
 #if TEST_STD_VER >= 11
-    {
+  {
     int a1[] = {1, 3, 7, 9, 10};
     int a2[] = {0, 2, 4, 5, 6, 8, 11};
     int a3[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11};
-    std::list<int, min_allocator<int>> c1(a1, a1+sizeof(a1)/sizeof(a1[0]));
-    std::list<int, min_allocator<int>> c2(a2, a2+sizeof(a2)/sizeof(a2[0]));
+    std::list<int, min_allocator<int>> c1(a1, a1 + sizeof(a1) / sizeof(a1[0]));
+    std::list<int, min_allocator<int>> c2(a2, a2 + sizeof(a2) / sizeof(a2[0]));
     c1.merge(c2);
-    assert((c1 == std::list<int, min_allocator<int>>(a3, a3+sizeof(a3)/sizeof(a3[0]))));
+    assert((c1 == std::list<int, min_allocator<int>>(a3, a3 + sizeof(a3) / sizeof(a3[0]))));
     assert(c2.empty());
-    }
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/sequences/list/list.ops/merge_comp.pass.cpp b/libcxx/test/std/containers/sequences/list/list.ops/merge_comp.pass.cpp
index 3d4f8cc7fc2aa..13241909c6e5b 100644
--- a/libcxx/test/std/containers/sequences/list/list.ops/merge_comp.pass.cpp
+++ b/libcxx/test/std/containers/sequences/list/list.ops/merge_comp.pass.cpp
@@ -18,36 +18,35 @@
 #include "test_macros.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
+int main(int, char**) {
+  {
     int a1[] = {10, 9, 7, 3, 1};
     int a2[] = {11, 8, 6, 5, 4, 2, 0};
     int a3[] = {11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0};
-    std::list<int> c1(a1, a1+sizeof(a1)/sizeof(a1[0]));
-    std::list<int> c2(a2, a2+sizeof(a2)/sizeof(a2[0]));
+    std::list<int> c1(a1, a1 + sizeof(a1) / sizeof(a1[0]));
+    std::list<int> c2(a2, a2 + sizeof(a2) / sizeof(a2[0]));
     c1.merge(c2, std::greater<int>());
-    assert(c1 == std::list<int>(a3, a3+sizeof(a3)/sizeof(a3[0])));
+    assert(c1 == std::list<int>(a3, a3 + sizeof(a3) / sizeof(a3[0])));
     assert(c2.empty());
-    }
-    {
+  }
+  {
     int a1[] = {10, 9, 7, 3, 1};
-    std::list<int> c1(a1, a1+sizeof(a1)/sizeof(a1[0]));
+    std::list<int> c1(a1, a1 + sizeof(a1) / sizeof(a1[0]));
     c1.merge(c1, std::greater<int>());
-    assert((c1 == std::list<int>(a1, a1+sizeof(a1)/sizeof(a1[0]))));
-    }
+    assert((c1 == std::list<int>(a1, a1 + sizeof(a1) / sizeof(a1[0]))));
+  }
 
 #if TEST_STD_VER >= 11
-    {
+  {
     int a1[] = {10, 9, 7, 3, 1};
     int a2[] = {11, 8, 6, 5, 4, 2, 0};
     int a3[] = {11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0};
-    std::list<int, min_allocator<int>> c1(a1, a1+sizeof(a1)/sizeof(a1[0]));
-    std::list<int, min_allocator<int>> c2(a2, a2+sizeof(a2)/sizeof(a2[0]));
+    std::list<int, min_allocator<int>> c1(a1, a1 + sizeof(a1) / sizeof(a1[0]));
+    std::list<int, min_allocator<int>> c2(a2, a2 + sizeof(a2) / sizeof(a2[0]));
     c1.merge(c2, std::greater<int>());
-    assert((c1 == std::list<int, min_allocator<int>>(a3, a3+sizeof(a3)/sizeof(a3[0]))));
+    assert((c1 == std::list<int, min_allocator<int>>(a3, a3 + sizeof(a3) / sizeof(a3[0]))));
     assert(c2.empty());
-    }
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/sequences/list/list.ops/remove.pass.cpp b/libcxx/test/std/containers/sequences/list/list.ops/remove.pass.cpp
index 0bfcb6e89d650..238ea9b69ea2e 100644
--- a/libcxx/test/std/containers/sequences/list/list.ops/remove.pass.cpp
+++ b/libcxx/test/std/containers/sequences/list/list.ops/remove.pass.cpp
@@ -19,8 +19,8 @@
 
 struct S {
   S(int i) : i_(new int(i)) {}
-  S(const S &rhs) : i_(new int(*rhs.i_)) {}
-  S &operator=(const S &rhs) {
+  S(const S& rhs) : i_(new int(*rhs.i_)) {}
+  S& operator=(const S& rhs) {
     *i_ = *rhs.i_;
     return *this;
   }
@@ -28,9 +28,9 @@ struct S {
     delete i_;
     i_ = NULL;
   }
-  bool operator==(const S &rhs) const { return *i_ == *rhs.i_; }
+  bool operator==(const S& rhs) const { return *i_ == *rhs.i_; }
   int get() const { return *i_; }
-  int *i_;
+  int* i_;
 };
 
 int main(int, char**) {
@@ -43,7 +43,7 @@ int main(int, char**) {
     assert(c.remove(3) == 1);
     ASSERT_SAME_TYPE(L::size_type, decltype(c.remove(3)));
 #else
-    ASSERT_SAME_TYPE(void,         decltype(c.remove(3)));
+    ASSERT_SAME_TYPE(void, decltype(c.remove(3)));
     c.remove(3);
 #endif
 
@@ -60,7 +60,7 @@ int main(int, char**) {
     int a1[] = {1, 2, 1, 3, 5, 8, 11, 1};
     int a2[] = {2, 3, 5, 8, 11};
     std::list<S> c;
-    for (int *ip = a1; ip < a1 + 8; ++ip)
+    for (int* ip = a1; ip < a1 + 8; ++ip)
       c.push_back(S(*ip));
 #if TEST_STD_VER > 17
     assert(c.remove(c.front()) == 3);
@@ -68,7 +68,7 @@ int main(int, char**) {
     c.remove(c.front());
 #endif
     std::list<S>::const_iterator it = c.begin();
-    for (int *ip = a2; ip < a2 + 5; ++ip, ++it) {
+    for (int* ip = a2; ip < a2 + 5; ++ip, ++it) {
       assert(it != c.end());
       assert(*ip == it->get());
     }
@@ -92,11 +92,11 @@ int main(int, char**) {
     int a1[] = {1, 2, 3, 4};
     int a2[] = {1, 2, 4};
     std::list<int, min_allocator<int>> c(a1, a1 + 4);
-#if TEST_STD_VER > 17
+#  if TEST_STD_VER > 17
     assert(c.remove(3) == 1);
-#else
+#  else
     c.remove(3);
-#endif
+#  endif
     assert((c == std::list<int, min_allocator<int>>(a2, a2 + 3)));
   }
 #endif
diff --git a/libcxx/test/std/containers/sequences/list/list.ops/remove_if.pass.cpp b/libcxx/test/std/containers/sequences/list/list.ops/remove_if.pass.cpp
index 4050466a6c727..510cb361142b4 100644
--- a/libcxx/test/std/containers/sequences/list/list.ops/remove_if.pass.cpp
+++ b/libcxx/test/std/containers/sequences/list/list.ops/remove_if.pass.cpp
@@ -19,15 +19,9 @@
 #include "min_allocator.h"
 #include "counting_predicates.h"
 
-bool even(int i)
-{
-    return i % 2 == 0;
-}
+bool even(int i) { return i % 2 == 0; }
 
-bool g(int i)
-{
-    return i < 3;
-}
+bool g(int i) { return i < 3; }
 
 struct PredLWG526 {
   PredLWG526(int i) : i_(i) {}
@@ -38,15 +32,14 @@ struct PredLWG526 {
   int i_;
 };
 
-typedef unary_counting_predicate<bool(*)(int), int> Predicate;
+typedef unary_counting_predicate<bool (*)(int), int> Predicate;
 
-int main(int, char**)
-{
-    {
+int main(int, char**) {
+  {
     int a1[] = {1, 2, 3, 4};
     int a2[] = {3, 4};
     typedef std::list<int> L;
-    L c(a1, a1+4);
+    L c(a1, a1 + 4);
     Predicate cp(g);
 #if TEST_STD_VER > 17
     ASSERT_SAME_TYPE(L::size_type, decltype(c.remove_if(std::ref(cp))));
@@ -55,49 +48,48 @@ int main(int, char**)
     ASSERT_SAME_TYPE(void, decltype(c.remove_if(std::ref(cp))));
     c.remove_if(std::ref(cp));
 #endif
-    assert(c == std::list<int>(a2, a2+2));
+    assert(c == std::list<int>(a2, a2 + 2));
     assert(cp.count() == 4);
-    }
-    {
+  }
+  {
     int a1[] = {1, 2, 3, 4};
     int a2[] = {1, 3};
-    std::list<int> c(a1, a1+4);
+    std::list<int> c(a1, a1 + 4);
     Predicate cp(even);
 #if TEST_STD_VER > 17
     assert(c.remove_if(std::ref(cp)) == 2);
 #else
     c.remove_if(std::ref(cp));
 #endif
-    assert(c == std::list<int>(a2, a2+2));
+    assert(c == std::list<int>(a2, a2 + 2));
     assert(cp.count() == 4);
-    }
-    { // LWG issue #526
+  }
+  { // LWG issue #526
     int a1[] = {1, 2, 1, 3, 5, 8, 11};
     int a2[] = {2, 3, 5, 8, 11};
     std::list<PredLWG526> c(a1, a1 + 7);
     c.remove_if(std::ref(c.front()));
     assert(c.size() == 5);
-    for (std::size_t i = 0; i < c.size(); ++i)
-    {
-        assert(c.front() == a2[i]);
-        c.pop_front();
-    }
+    for (std::size_t i = 0; i < c.size(); ++i) {
+      assert(c.front() == a2[i]);
+      c.pop_front();
     }
+  }
 
 #if TEST_STD_VER >= 11
-    {
+  {
     int a1[] = {1, 2, 3, 4};
     int a2[] = {3, 4};
-    std::list<int, min_allocator<int>> c(a1, a1+4);
+    std::list<int, min_allocator<int>> c(a1, a1 + 4);
     Predicate cp(g);
-#if TEST_STD_VER > 17
+#  if TEST_STD_VER > 17
     assert(c.remove_if(std::ref(cp)) == 2);
-#else
+#  else
     c.remove_if(std::ref(cp));
-#endif
-    assert((c == std::list<int, min_allocator<int>>(a2, a2+2)));
+#  endif
+    assert((c == std::list<int, min_allocator<int>>(a2, a2 + 2)));
     assert(cp.count() == 4);
-    }
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/sequences/list/list.ops/reverse.pass.cpp b/libcxx/test/std/containers/sequences/list/list.ops/reverse.pass.cpp
index 6ebd7f9b611f7..5b91ad0224be6 100644
--- a/libcxx/test/std/containers/sequences/list/list.ops/reverse.pass.cpp
+++ b/libcxx/test/std/containers/sequences/list/list.ops/reverse.pass.cpp
@@ -16,23 +16,22 @@
 #include "test_macros.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
+int main(int, char**) {
+  {
     int a1[] = {11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0};
     int a2[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11};
-    std::list<int> c1(a1, a1+sizeof(a1)/sizeof(a1[0]));
+    std::list<int> c1(a1, a1 + sizeof(a1) / sizeof(a1[0]));
     c1.reverse();
-    assert(c1 == std::list<int>(a2, a2+sizeof(a2)/sizeof(a2[0])));
-    }
+    assert(c1 == std::list<int>(a2, a2 + sizeof(a2) / sizeof(a2[0])));
+  }
 #if TEST_STD_VER >= 11
-    {
+  {
     int a1[] = {11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0};
     int a2[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11};
-    std::list<int, min_allocator<int>> c1(a1, a1+sizeof(a1)/sizeof(a1[0]));
+    std::list<int, min_allocator<int>> c1(a1, a1 + sizeof(a1) / sizeof(a1[0]));
     c1.reverse();
-    assert((c1 == std::list<int, min_allocator<int>>(a2, a2+sizeof(a2)/sizeof(a2[0]))));
-    }
+    assert((c1 == std::list<int, min_allocator<int>>(a2, a2 + sizeof(a2) / sizeof(a2[0]))));
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/sequences/list/list.ops/sort.pass.cpp b/libcxx/test/std/containers/sequences/list/list.ops/sort.pass.cpp
index 6349a3d850671..892419f6ac964 100644
--- a/libcxx/test/std/containers/sequences/list/list.ops/sort.pass.cpp
+++ b/libcxx/test/std/containers/sequences/list/list.ops/sort.pass.cpp
@@ -21,68 +21,63 @@
 
 std::mt19937 randomness;
 
-struct Payload
-{
-    int val;
-    int side;
-    Payload(int v) : val(v), side(0) {}
-    Payload(int v, int s) : val(v), side(s) {}
-    bool operator< (const Payload &rhs) const { return val <  rhs.val;}
-//     bool operator==(const Payload &rhs) const { return val == rhs.val;}
+struct Payload {
+  int val;
+  int side;
+  Payload(int v) : val(v), side(0) {}
+  Payload(int v, int s) : val(v), side(s) {}
+  bool operator<(const Payload& rhs) const { return val < rhs.val; }
+  //     bool operator==(const Payload &rhs) const { return val == rhs.val;}
 };
 
-void test_stable(int N)
-{
-    typedef Payload T;
-    typedef std::list<T> C;
-    typedef std::vector<T> V;
-    V v;
-    for (int i = 0; i < N; ++i)
-        v.push_back(Payload(i/2));
-    std::shuffle(v.begin(), v.end(), randomness);
-    for (int i = 0; i < N; ++i)
-        v[i].side = i;
+void test_stable(int N) {
+  typedef Payload T;
+  typedef std::list<T> C;
+  typedef std::vector<T> V;
+  V v;
+  for (int i = 0; i < N; ++i)
+    v.push_back(Payload(i / 2));
+  std::shuffle(v.begin(), v.end(), randomness);
+  for (int i = 0; i < N; ++i)
+    v[i].side = i;
 
-    C c(v.begin(), v.end());
-    c.sort();
-    assert(std::distance(c.begin(), c.end()) == N);
+  C c(v.begin(), v.end());
+  c.sort();
+  assert(std::distance(c.begin(), c.end()) == N);
 
-//  Are we sorted?
-    typename C::const_iterator j = c.begin();
-    for (int i = 0; i < N; ++i, ++j)
-        assert(j->val == i/2);
+  //  Are we sorted?
+  typename C::const_iterator j = c.begin();
+  for (int i = 0; i < N; ++i, ++j)
+    assert(j->val == i / 2);
 
-//  Are we stable?
-    for (C::const_iterator it = c.begin(); it != c.end(); ++it)
-    {
-        C::const_iterator next = std::next(it);
-        if (next != c.end() && it->val == next->val)
-            assert(it->side < next->side);
-    }
+  //  Are we stable?
+  for (C::const_iterator it = c.begin(); it != c.end(); ++it) {
+    C::const_iterator next = std::next(it);
+    if (next != c.end() && it->val == next->val)
+      assert(it->side < next->side);
+  }
 }
 
-
-int main(int, char**)
-{
-    {
+int main(int, char**) {
+  {
     int a1[] = {4, 8, 1, 0, 5, 7, 2, 3, 6, 11, 10, 9};
     int a2[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11};
-    std::list<int> c1(a1, a1+sizeof(a1)/sizeof(a1[0]));
+    std::list<int> c1(a1, a1 + sizeof(a1) / sizeof(a1[0]));
     c1.sort();
-    assert(c1 == std::list<int>(a2, a2+sizeof(a2)/sizeof(a2[0])));
-    }
+    assert(c1 == std::list<int>(a2, a2 + sizeof(a2) / sizeof(a2[0])));
+  }
 #if TEST_STD_VER >= 11
-    {
+  {
     int a1[] = {4, 8, 1, 0, 5, 7, 2, 3, 6, 11, 10, 9};
     int a2[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11};
-    std::list<int, min_allocator<int>> c1(a1, a1+sizeof(a1)/sizeof(a1[0]));
+    std::list<int, min_allocator<int>> c1(a1, a1 + sizeof(a1) / sizeof(a1[0]));
     c1.sort();
-    assert((c1 == std::list<int, min_allocator<int>>(a2, a2+sizeof(a2)/sizeof(a2[0]))));
-    }
+    assert((c1 == std::list<int, min_allocator<int>>(a2, a2 + sizeof(a2) / sizeof(a2[0]))));
+  }
 #endif
 
-    for (int i = 0; i < 40; ++i)
-        test_stable(i);
+  for (int i = 0; i < 40; ++i)
+    test_stable(i);
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/list/list.ops/sort_comp.pass.cpp b/libcxx/test/std/containers/sequences/list/list.ops/sort_comp.pass.cpp
index 8345823f4db9d..4997022819919 100644
--- a/libcxx/test/std/containers/sequences/list/list.ops/sort_comp.pass.cpp
+++ b/libcxx/test/std/containers/sequences/list/list.ops/sort_comp.pass.cpp
@@ -12,7 +12,7 @@
 
 #include <list>
 #include <functional>
-#include <algorithm>  // for is_permutation
+#include <algorithm> // for is_permutation
 #include <random>
 #include <vector>
 #include <cassert>
@@ -25,105 +25,97 @@ std::mt19937 randomness;
 #ifndef TEST_HAS_NO_EXCEPTIONS
 template <typename T>
 struct throwingLess {
-    throwingLess() : num_(1) {}
-    throwingLess(int num) : num_(num) {}
+  throwingLess() : num_(1) {}
+  throwingLess(int num) : num_(num) {}
 
-    bool operator() (const T& lhs, const T& rhs) const
-    {
-    if ( --num_ == 0) throw 1;
+  bool operator()(const T& lhs, const T& rhs) const {
+    if (--num_ == 0)
+      throw 1;
     return lhs < rhs;
-    }
+  }
 
-    mutable int num_;
-    };
+  mutable int num_;
+};
 #endif
 
-
-struct Payload
-{
-    int val;
-    int side;
-    Payload(int v) : val(v), side(0) {}
-    Payload(int v, int s) : val(v), side(s) {}
-    bool operator< (const Payload &rhs) const { return val <  rhs.val;}
+struct Payload {
+  int val;
+  int side;
+  Payload(int v) : val(v), side(0) {}
+  Payload(int v, int s) : val(v), side(s) {}
+  bool operator<(const Payload& rhs) const { return val < rhs.val; }
 };
 
-bool greater(const Payload &lhs, const Payload &rhs) { return lhs.val >  rhs.val; }
-
-void test_stable(int N)
-{
-    typedef Payload T;
-    typedef std::list<T> C;
-    typedef std::vector<T> V;
-    V v;
-    for (int i = 0; i < N; ++i)
-        v.push_back(Payload(i/2));
-    std::shuffle(v.begin(), v.end(), randomness);
-    for (int i = 0; i < N; ++i)
-        v[i].side = i;
-
-    C c(v.begin(), v.end());
-    c.sort(greater);
-    assert(std::distance(c.begin(), c.end()) == N);
-
-//  Are we sorted?
-    typename C::const_iterator j = c.begin();
-    for (int i = 0; i < N; ++i, ++j)
-        assert(j->val == (N-1-i)/2);
-
-//  Are we stable?
-    for (C::const_iterator it = c.begin(); it != c.end(); ++it)
-    {
-        C::const_iterator next = std::next(it);
-        if (next != c.end() && it->val == next->val)
-            assert(it->side < next->side);
-    }
+bool greater(const Payload& lhs, const Payload& rhs) { return lhs.val > rhs.val; }
+
+void test_stable(int N) {
+  typedef Payload T;
+  typedef std::list<T> C;
+  typedef std::vector<T> V;
+  V v;
+  for (int i = 0; i < N; ++i)
+    v.push_back(Payload(i / 2));
+  std::shuffle(v.begin(), v.end(), randomness);
+  for (int i = 0; i < N; ++i)
+    v[i].side = i;
+
+  C c(v.begin(), v.end());
+  c.sort(greater);
+  assert(std::distance(c.begin(), c.end()) == N);
+
+  //  Are we sorted?
+  typename C::const_iterator j = c.begin();
+  for (int i = 0; i < N; ++i, ++j)
+    assert(j->val == (N - 1 - i) / 2);
+
+  //  Are we stable?
+  for (C::const_iterator it = c.begin(); it != c.end(); ++it) {
+    C::const_iterator next = std::next(it);
+    if (next != c.end() && it->val == next->val)
+      assert(it->side < next->side);
+  }
 }
 
-
-int main(int, char**)
-{
-    {
+int main(int, char**) {
+  {
     int a1[] = {4, 8, 1, 0, 5, 7, 2, 3, 6, 11, 10, 9};
     int a2[] = {11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0};
-    std::list<int> c1(a1, a1+sizeof(a1)/sizeof(a1[0]));
+    std::list<int> c1(a1, a1 + sizeof(a1) / sizeof(a1[0]));
     c1.sort(std::greater<int>());
-    assert(c1 == std::list<int>(a2, a2+sizeof(a2)/sizeof(a2[0])));
-    }
+    assert(c1 == std::list<int>(a2, a2 + sizeof(a2) / sizeof(a2[0])));
+  }
 
 //  Test with throwing comparison; make sure that nothing is lost.
 //  This is (sort of) LWG #2824
 #ifndef TEST_HAS_NO_EXCEPTIONS
-    {
-    int a1[] = {4, 8, 1, 0, 5, 7, 2, 3, 6, 11, 10, 9};
-    const int sz = sizeof(a1)/sizeof(a1[0]);
-    for (int i = 0; i < 10; ++i)
-        {
-        std::list<int> c1(a1, a1 + sz);
-        try
-            {
-            throwingLess<int> comp(i);
-            c1.sort(std::cref(comp));
-            }
-        catch (int) {}
-        assert((c1.size() == sz));
-        assert((std::is_permutation(c1.begin(), c1.end(), a1)));
-        }
+  {
+    int a1[]     = {4, 8, 1, 0, 5, 7, 2, 3, 6, 11, 10, 9};
+    const int sz = sizeof(a1) / sizeof(a1[0]);
+    for (int i = 0; i < 10; ++i) {
+      std::list<int> c1(a1, a1 + sz);
+      try {
+        throwingLess<int> comp(i);
+        c1.sort(std::cref(comp));
+      } catch (int) {
+      }
+      assert((c1.size() == sz));
+      assert((std::is_permutation(c1.begin(), c1.end(), a1)));
     }
+  }
 #endif
 
 #if TEST_STD_VER >= 11
-    {
+  {
     int a1[] = {4, 8, 1, 0, 5, 7, 2, 3, 6, 11, 10, 9};
     int a2[] = {11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0};
-    std::list<int, min_allocator<int>> c1(a1, a1+sizeof(a1)/sizeof(a1[0]));
+    std::list<int, min_allocator<int>> c1(a1, a1 + sizeof(a1) / sizeof(a1[0]));
     c1.sort(std::greater<int>());
-    assert((c1 == std::list<int, min_allocator<int>>(a2, a2+sizeof(a2)/sizeof(a2[0]))));
-    }
+    assert((c1 == std::list<int, min_allocator<int>>(a2, a2 + sizeof(a2) / sizeof(a2[0]))));
+  }
 #endif
 
-    for (int i = 0; i < 40; ++i)
-        test_stable(i);
+  for (int i = 0; i < 40; ++i)
+    test_stable(i);
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/list/list.ops/splice_pos_list.pass.cpp b/libcxx/test/std/containers/sequences/list/list.ops/splice_pos_list.pass.cpp
index ff0f225e8b4ac..4b40876e3bb7d 100644
--- a/libcxx/test/std/containers/sequences/list/list.ops/splice_pos_list.pass.cpp
+++ b/libcxx/test/std/containers/sequences/list/list.ops/splice_pos_list.pass.cpp
@@ -16,769 +16,768 @@
 #include "test_macros.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    int a1[] = {1, 2, 3};
-    int a2[] = {4, 5, 6};
-    {
-        std::list<int> l1;
-        std::list<int> l2;
-        l1.splice(l1.end(), l2);
-        assert(l1.size() == 0);
-        assert(std::distance(l1.begin(), l1.end()) == 0);
-        assert(l2.size() == 0);
-        assert(std::distance(l2.begin(), l2.end()) == 0);
-    }
-    {
-        std::list<int> l1;
-        std::list<int> l2(a2, a2+1);
-        l1.splice(l1.end(), l2);
-        assert(l1.size() == 1);
-        assert(std::distance(l1.begin(), l1.end()) == 1);
-        assert(l2.size() == 0);
-        assert(std::distance(l2.begin(), l2.end()) == 0);
-        std::list<int>::const_iterator i = l1.begin();
-        assert(*i == 4);
-    }
-    {
-        std::list<int> l1;
-        std::list<int> l2(a2, a2+2);
-        l1.splice(l1.end(), l2);
-        assert(l1.size() == 2);
-        assert(std::distance(l1.begin(), l1.end()) == 2);
-        assert(l2.size() == 0);
-        assert(std::distance(l2.begin(), l2.end()) == 0);
-        std::list<int>::const_iterator i = l1.begin();
-        assert(*i == 4);
-        ++i;
-        assert(*i == 5);
-    }
-    {
-        std::list<int> l1;
-        std::list<int> l2(a2, a2+3);
-        l1.splice(l1.end(), l2);
-        assert(l1.size() == 3);
-        assert(std::distance(l1.begin(), l1.end()) == 3);
-        assert(l2.size() == 0);
-        assert(std::distance(l2.begin(), l2.end()) == 0);
-        std::list<int>::const_iterator i = l1.begin();
-        assert(*i == 4);
-        ++i;
-        assert(*i == 5);
-        ++i;
-        assert(*i == 6);
-    }
-    {
-        std::list<int> l1(a1, a1+1);
-        std::list<int> l2;
-        l1.splice(l1.begin(), l2);
-        assert(l1.size() == 1);
-        assert(std::distance(l1.begin(), l1.end()) == 1);
-        assert(l2.size() == 0);
-        assert(std::distance(l2.begin(), l2.end()) == 0);
-        std::list<int>::const_iterator i = l1.begin();
-        assert(*i == 1);
-    }
-    {
-        std::list<int> l1(a1, a1+1);
-        std::list<int> l2;
-        l1.splice(l1.end(), l2);
-        assert(l1.size() == 1);
-        assert(std::distance(l1.begin(), l1.end()) == 1);
-        assert(l2.size() == 0);
-        assert(std::distance(l2.begin(), l2.end()) == 0);
-        std::list<int>::const_iterator i = l1.begin();
-        assert(*i == 1);
-    }
-    {
-        std::list<int> l1(a1, a1+1);
-        std::list<int> l2(a2, a2+1);
-        l1.splice(l1.begin(), l2);
-        assert(l1.size() == 2);
-        assert(std::distance(l1.begin(), l1.end()) == 2);
-        assert(l2.size() == 0);
-        assert(std::distance(l2.begin(), l2.end()) == 0);
-        std::list<int>::const_iterator i = l1.begin();
-        assert(*i == 4);
-        ++i;
-        assert(*i == 1);
-    }
-    {
-        std::list<int> l1(a1, a1+1);
-        std::list<int> l2(a2, a2+1);
-        l1.splice(l1.end(), l2);
-        assert(l1.size() == 2);
-        assert(std::distance(l1.begin(), l1.end()) == 2);
-        assert(l2.size() == 0);
-        assert(std::distance(l2.begin(), l2.end()) == 0);
-        std::list<int>::const_iterator i = l1.begin();
-        assert(*i == 1);
-        ++i;
-        assert(*i == 4);
-    }
-    {
-        std::list<int> l1(a1, a1+1);
-        std::list<int> l2(a2, a2+2);
-        l1.splice(l1.begin(), l2);
-        assert(l1.size() == 3);
-        assert(std::distance(l1.begin(), l1.end()) == 3);
-        assert(l2.size() == 0);
-        assert(std::distance(l2.begin(), l2.end()) == 0);
-        std::list<int>::const_iterator i = l1.begin();
-        assert(*i == 4);
-        ++i;
-        assert(*i == 5);
-        ++i;
-        assert(*i == 1);
-    }
-    {
-        std::list<int> l1(a1, a1+1);
-        std::list<int> l2(a2, a2+2);
-        l1.splice(l1.end(), l2);
-        assert(l1.size() == 3);
-        assert(std::distance(l1.begin(), l1.end()) == 3);
-        assert(l2.size() == 0);
-        assert(std::distance(l2.begin(), l2.end()) == 0);
-        std::list<int>::const_iterator i = l1.begin();
-        assert(*i == 1);
-        ++i;
-        assert(*i == 4);
-        ++i;
-        assert(*i == 5);
-    }
-    {
-        std::list<int> l1(a1, a1+1);
-        std::list<int> l2(a2, a2+3);
-        l1.splice(l1.begin(), l2);
-        assert(l1.size() == 4);
-        assert(std::distance(l1.begin(), l1.end()) == 4);
-        assert(l2.size() == 0);
-        assert(std::distance(l2.begin(), l2.end()) == 0);
-        std::list<int>::const_iterator i = l1.begin();
-        assert(*i == 4);
-        ++i;
-        assert(*i == 5);
-        ++i;
-        assert(*i == 6);
-        ++i;
-        assert(*i == 1);
-    }
-    {
-        std::list<int> l1(a1, a1+1);
-        std::list<int> l2(a2, a2+3);
-        l1.splice(l1.end(), l2);
-        assert(l1.size() == 4);
-        assert(std::distance(l1.begin(), l1.end()) == 4);
-        assert(l2.size() == 0);
-        assert(std::distance(l2.begin(), l2.end()) == 0);
-        std::list<int>::const_iterator i = l1.begin();
-        assert(*i == 1);
-        ++i;
-        assert(*i == 4);
-        ++i;
-        assert(*i == 5);
-        ++i;
-        assert(*i == 6);
-    }
-    {
-        std::list<int> l1(a1, a1+2);
-        std::list<int> l2;
-        l1.splice(l1.begin(), l2);
-        assert(l1.size() == 2);
-        assert(std::distance(l1.begin(), l1.end()) == 2);
-        assert(l2.size() == 0);
-        assert(std::distance(l2.begin(), l2.end()) == 0);
-        std::list<int>::const_iterator i = l1.begin();
-        assert(*i == 1);
-        ++i;
-        assert(*i == 2);
-    }
-    {
-        std::list<int> l1(a1, a1+2);
-        std::list<int> l2;
-        l1.splice(std::next(l1.begin()), l2);
-        assert(l1.size() == 2);
-        assert(std::distance(l1.begin(), l1.end()) == 2);
-        assert(l2.size() == 0);
-        assert(std::distance(l2.begin(), l2.end()) == 0);
-        std::list<int>::const_iterator i = l1.begin();
-        assert(*i == 1);
-        ++i;
-        assert(*i == 2);
-    }
-    {
-        std::list<int> l1(a1, a1+2);
-        std::list<int> l2;
-        l1.splice(std::next(l1.begin(), 2), l2);
-        assert(l1.size() == 2);
-        assert(std::distance(l1.begin(), l1.end()) == 2);
-        assert(l2.size() == 0);
-        assert(std::distance(l2.begin(), l2.end()) == 0);
-        std::list<int>::const_iterator i = l1.begin();
-        assert(*i == 1);
-        ++i;
-        assert(*i == 2);
-    }
-    {
-        std::list<int> l1(a1, a1+2);
-        std::list<int> l2(a2, a2+1);
-        l1.splice(l1.begin(), l2);
-        assert(l1.size() == 3);
-        assert(std::distance(l1.begin(), l1.end()) == 3);
-        assert(l2.size() == 0);
-        assert(std::distance(l2.begin(), l2.end()) == 0);
-        std::list<int>::const_iterator i = l1.begin();
-        assert(*i == 4);
-        ++i;
-        assert(*i == 1);
-        ++i;
-        assert(*i == 2);
-    }
-    {
-        std::list<int> l1(a1, a1+2);
-        std::list<int> l2(a2, a2+1);
-        l1.splice(std::next(l1.begin()), l2);
-        assert(l1.size() == 3);
-        assert(std::distance(l1.begin(), l1.end()) == 3);
-        assert(l2.size() == 0);
-        assert(std::distance(l2.begin(), l2.end()) == 0);
-        std::list<int>::const_iterator i = l1.begin();
-        assert(*i == 1);
-        ++i;
-        assert(*i == 4);
-        ++i;
-        assert(*i == 2);
-    }
-    {
-        std::list<int> l1(a1, a1+2);
-        std::list<int> l2(a2, a2+1);
-        l1.splice(std::next(l1.begin(), 2), l2);
-        assert(l1.size() == 3);
-        assert(std::distance(l1.begin(), l1.end()) == 3);
-        assert(l2.size() == 0);
-        assert(std::distance(l2.begin(), l2.end()) == 0);
-        std::list<int>::const_iterator i = l1.begin();
-        assert(*i == 1);
-        ++i;
-        assert(*i == 2);
-        ++i;
-        assert(*i == 4);
-    }
-    {
-        std::list<int> l1(a1, a1+2);
-        std::list<int> l2(a2, a2+2);
-        l1.splice(l1.begin(), l2);
-        assert(l1.size() == 4);
-        assert(std::distance(l1.begin(), l1.end()) == 4);
-        assert(l2.size() == 0);
-        assert(std::distance(l2.begin(), l2.end()) == 0);
-        std::list<int>::const_iterator i = l1.begin();
-        assert(*i == 4);
-        ++i;
-        assert(*i == 5);
-        ++i;
-        assert(*i == 1);
-        ++i;
-        assert(*i == 2);
-    }
-    {
-        std::list<int> l1(a1, a1+2);
-        std::list<int> l2(a2, a2+2);
-        l1.splice(std::next(l1.begin()), l2);
-        assert(l1.size() == 4);
-        assert(std::distance(l1.begin(), l1.end()) == 4);
-        assert(l2.size() == 0);
-        assert(std::distance(l2.begin(), l2.end()) == 0);
-        std::list<int>::const_iterator i = l1.begin();
-        assert(*i == 1);
-        ++i;
-        assert(*i == 4);
-        ++i;
-        assert(*i == 5);
-        ++i;
-        assert(*i == 2);
-    }
-    {
-        std::list<int> l1(a1, a1+2);
-        std::list<int> l2(a2, a2+2);
-        l1.splice(std::next(l1.begin(), 2), l2);
-        assert(l1.size() == 4);
-        assert(std::distance(l1.begin(), l1.end()) == 4);
-        assert(l2.size() == 0);
-        assert(std::distance(l2.begin(), l2.end()) == 0);
-        std::list<int>::const_iterator i = l1.begin();
-        assert(*i == 1);
-        ++i;
-        assert(*i == 2);
-        ++i;
-        assert(*i == 4);
-        ++i;
-        assert(*i == 5);
-    }
-    {
-        std::list<int> l1(a1, a1+3);
-        std::list<int> l2(a2, a2+3);
-        l1.splice(l1.begin(), l2);
-        assert(l1.size() == 6);
-        assert(std::distance(l1.begin(), l1.end()) == 6);
-        assert(l2.size() == 0);
-        assert(std::distance(l2.begin(), l2.end()) == 0);
-        std::list<int>::const_iterator i = l1.begin();
-        assert(*i == 4);
-        ++i;
-        assert(*i == 5);
-        ++i;
-        assert(*i == 6);
-        ++i;
-        assert(*i == 1);
-        ++i;
-        assert(*i == 2);
-        ++i;
-        assert(*i == 3);
-    }
-    {
-        std::list<int> l1(a1, a1+3);
-        std::list<int> l2(a2, a2+3);
-        l1.splice(std::next(l1.begin()), l2);
-        assert(l1.size() == 6);
-        assert(std::distance(l1.begin(), l1.end()) == 6);
-        assert(l2.size() == 0);
-        assert(std::distance(l2.begin(), l2.end()) == 0);
-        std::list<int>::const_iterator i = l1.begin();
-        assert(*i == 1);
-        ++i;
-        assert(*i == 4);
-        ++i;
-        assert(*i == 5);
-        ++i;
-        assert(*i == 6);
-        ++i;
-        assert(*i == 2);
-        ++i;
-        assert(*i == 3);
-    }
-    {
-        std::list<int> l1(a1, a1+3);
-        std::list<int> l2(a2, a2+3);
-        l1.splice(std::next(l1.begin(), 2), l2);
-        assert(l1.size() == 6);
-        assert(std::distance(l1.begin(), l1.end()) == 6);
-        assert(l2.size() == 0);
-        assert(std::distance(l2.begin(), l2.end()) == 0);
-        std::list<int>::const_iterator i = l1.begin();
-        assert(*i == 1);
-        ++i;
-        assert(*i == 2);
-        ++i;
-        assert(*i == 4);
-        ++i;
-        assert(*i == 5);
-        ++i;
-        assert(*i == 6);
-        ++i;
-        assert(*i == 3);
-    }
-    {
-        std::list<int> l1(a1, a1+3);
-        std::list<int> l2(a2, a2+3);
-        l1.splice(std::next(l1.begin(), 3), l2);
-        assert(l1.size() == 6);
-        assert(std::distance(l1.begin(), l1.end()) == 6);
-        assert(l2.size() == 0);
-        assert(std::distance(l2.begin(), l2.end()) == 0);
-        std::list<int>::const_iterator i = l1.begin();
-        assert(*i == 1);
-        ++i;
-        assert(*i == 2);
-        ++i;
-        assert(*i == 3);
-        ++i;
-        assert(*i == 4);
-        ++i;
-        assert(*i == 5);
-        ++i;
-        assert(*i == 6);
-    }
+int main(int, char**) {
+  int a1[] = {1, 2, 3};
+  int a2[] = {4, 5, 6};
+  {
+    std::list<int> l1;
+    std::list<int> l2;
+    l1.splice(l1.end(), l2);
+    assert(l1.size() == 0);
+    assert(std::distance(l1.begin(), l1.end()) == 0);
+    assert(l2.size() == 0);
+    assert(std::distance(l2.begin(), l2.end()) == 0);
+  }
+  {
+    std::list<int> l1;
+    std::list<int> l2(a2, a2 + 1);
+    l1.splice(l1.end(), l2);
+    assert(l1.size() == 1);
+    assert(std::distance(l1.begin(), l1.end()) == 1);
+    assert(l2.size() == 0);
+    assert(std::distance(l2.begin(), l2.end()) == 0);
+    std::list<int>::const_iterator i = l1.begin();
+    assert(*i == 4);
+  }
+  {
+    std::list<int> l1;
+    std::list<int> l2(a2, a2 + 2);
+    l1.splice(l1.end(), l2);
+    assert(l1.size() == 2);
+    assert(std::distance(l1.begin(), l1.end()) == 2);
+    assert(l2.size() == 0);
+    assert(std::distance(l2.begin(), l2.end()) == 0);
+    std::list<int>::const_iterator i = l1.begin();
+    assert(*i == 4);
+    ++i;
+    assert(*i == 5);
+  }
+  {
+    std::list<int> l1;
+    std::list<int> l2(a2, a2 + 3);
+    l1.splice(l1.end(), l2);
+    assert(l1.size() == 3);
+    assert(std::distance(l1.begin(), l1.end()) == 3);
+    assert(l2.size() == 0);
+    assert(std::distance(l2.begin(), l2.end()) == 0);
+    std::list<int>::const_iterator i = l1.begin();
+    assert(*i == 4);
+    ++i;
+    assert(*i == 5);
+    ++i;
+    assert(*i == 6);
+  }
+  {
+    std::list<int> l1(a1, a1 + 1);
+    std::list<int> l2;
+    l1.splice(l1.begin(), l2);
+    assert(l1.size() == 1);
+    assert(std::distance(l1.begin(), l1.end()) == 1);
+    assert(l2.size() == 0);
+    assert(std::distance(l2.begin(), l2.end()) == 0);
+    std::list<int>::const_iterator i = l1.begin();
+    assert(*i == 1);
+  }
+  {
+    std::list<int> l1(a1, a1 + 1);
+    std::list<int> l2;
+    l1.splice(l1.end(), l2);
+    assert(l1.size() == 1);
+    assert(std::distance(l1.begin(), l1.end()) == 1);
+    assert(l2.size() == 0);
+    assert(std::distance(l2.begin(), l2.end()) == 0);
+    std::list<int>::const_iterator i = l1.begin();
+    assert(*i == 1);
+  }
+  {
+    std::list<int> l1(a1, a1 + 1);
+    std::list<int> l2(a2, a2 + 1);
+    l1.splice(l1.begin(), l2);
+    assert(l1.size() == 2);
+    assert(std::distance(l1.begin(), l1.end()) == 2);
+    assert(l2.size() == 0);
+    assert(std::distance(l2.begin(), l2.end()) == 0);
+    std::list<int>::const_iterator i = l1.begin();
+    assert(*i == 4);
+    ++i;
+    assert(*i == 1);
+  }
+  {
+    std::list<int> l1(a1, a1 + 1);
+    std::list<int> l2(a2, a2 + 1);
+    l1.splice(l1.end(), l2);
+    assert(l1.size() == 2);
+    assert(std::distance(l1.begin(), l1.end()) == 2);
+    assert(l2.size() == 0);
+    assert(std::distance(l2.begin(), l2.end()) == 0);
+    std::list<int>::const_iterator i = l1.begin();
+    assert(*i == 1);
+    ++i;
+    assert(*i == 4);
+  }
+  {
+    std::list<int> l1(a1, a1 + 1);
+    std::list<int> l2(a2, a2 + 2);
+    l1.splice(l1.begin(), l2);
+    assert(l1.size() == 3);
+    assert(std::distance(l1.begin(), l1.end()) == 3);
+    assert(l2.size() == 0);
+    assert(std::distance(l2.begin(), l2.end()) == 0);
+    std::list<int>::const_iterator i = l1.begin();
+    assert(*i == 4);
+    ++i;
+    assert(*i == 5);
+    ++i;
+    assert(*i == 1);
+  }
+  {
+    std::list<int> l1(a1, a1 + 1);
+    std::list<int> l2(a2, a2 + 2);
+    l1.splice(l1.end(), l2);
+    assert(l1.size() == 3);
+    assert(std::distance(l1.begin(), l1.end()) == 3);
+    assert(l2.size() == 0);
+    assert(std::distance(l2.begin(), l2.end()) == 0);
+    std::list<int>::const_iterator i = l1.begin();
+    assert(*i == 1);
+    ++i;
+    assert(*i == 4);
+    ++i;
+    assert(*i == 5);
+  }
+  {
+    std::list<int> l1(a1, a1 + 1);
+    std::list<int> l2(a2, a2 + 3);
+    l1.splice(l1.begin(), l2);
+    assert(l1.size() == 4);
+    assert(std::distance(l1.begin(), l1.end()) == 4);
+    assert(l2.size() == 0);
+    assert(std::distance(l2.begin(), l2.end()) == 0);
+    std::list<int>::const_iterator i = l1.begin();
+    assert(*i == 4);
+    ++i;
+    assert(*i == 5);
+    ++i;
+    assert(*i == 6);
+    ++i;
+    assert(*i == 1);
+  }
+  {
+    std::list<int> l1(a1, a1 + 1);
+    std::list<int> l2(a2, a2 + 3);
+    l1.splice(l1.end(), l2);
+    assert(l1.size() == 4);
+    assert(std::distance(l1.begin(), l1.end()) == 4);
+    assert(l2.size() == 0);
+    assert(std::distance(l2.begin(), l2.end()) == 0);
+    std::list<int>::const_iterator i = l1.begin();
+    assert(*i == 1);
+    ++i;
+    assert(*i == 4);
+    ++i;
+    assert(*i == 5);
+    ++i;
+    assert(*i == 6);
+  }
+  {
+    std::list<int> l1(a1, a1 + 2);
+    std::list<int> l2;
+    l1.splice(l1.begin(), l2);
+    assert(l1.size() == 2);
+    assert(std::distance(l1.begin(), l1.end()) == 2);
+    assert(l2.size() == 0);
+    assert(std::distance(l2.begin(), l2.end()) == 0);
+    std::list<int>::const_iterator i = l1.begin();
+    assert(*i == 1);
+    ++i;
+    assert(*i == 2);
+  }
+  {
+    std::list<int> l1(a1, a1 + 2);
+    std::list<int> l2;
+    l1.splice(std::next(l1.begin()), l2);
+    assert(l1.size() == 2);
+    assert(std::distance(l1.begin(), l1.end()) == 2);
+    assert(l2.size() == 0);
+    assert(std::distance(l2.begin(), l2.end()) == 0);
+    std::list<int>::const_iterator i = l1.begin();
+    assert(*i == 1);
+    ++i;
+    assert(*i == 2);
+  }
+  {
+    std::list<int> l1(a1, a1 + 2);
+    std::list<int> l2;
+    l1.splice(std::next(l1.begin(), 2), l2);
+    assert(l1.size() == 2);
+    assert(std::distance(l1.begin(), l1.end()) == 2);
+    assert(l2.size() == 0);
+    assert(std::distance(l2.begin(), l2.end()) == 0);
+    std::list<int>::const_iterator i = l1.begin();
+    assert(*i == 1);
+    ++i;
+    assert(*i == 2);
+  }
+  {
+    std::list<int> l1(a1, a1 + 2);
+    std::list<int> l2(a2, a2 + 1);
+    l1.splice(l1.begin(), l2);
+    assert(l1.size() == 3);
+    assert(std::distance(l1.begin(), l1.end()) == 3);
+    assert(l2.size() == 0);
+    assert(std::distance(l2.begin(), l2.end()) == 0);
+    std::list<int>::const_iterator i = l1.begin();
+    assert(*i == 4);
+    ++i;
+    assert(*i == 1);
+    ++i;
+    assert(*i == 2);
+  }
+  {
+    std::list<int> l1(a1, a1 + 2);
+    std::list<int> l2(a2, a2 + 1);
+    l1.splice(std::next(l1.begin()), l2);
+    assert(l1.size() == 3);
+    assert(std::distance(l1.begin(), l1.end()) == 3);
+    assert(l2.size() == 0);
+    assert(std::distance(l2.begin(), l2.end()) == 0);
+    std::list<int>::const_iterator i = l1.begin();
+    assert(*i == 1);
+    ++i;
+    assert(*i == 4);
+    ++i;
+    assert(*i == 2);
+  }
+  {
+    std::list<int> l1(a1, a1 + 2);
+    std::list<int> l2(a2, a2 + 1);
+    l1.splice(std::next(l1.begin(), 2), l2);
+    assert(l1.size() == 3);
+    assert(std::distance(l1.begin(), l1.end()) == 3);
+    assert(l2.size() == 0);
+    assert(std::distance(l2.begin(), l2.end()) == 0);
+    std::list<int>::const_iterator i = l1.begin();
+    assert(*i == 1);
+    ++i;
+    assert(*i == 2);
+    ++i;
+    assert(*i == 4);
+  }
+  {
+    std::list<int> l1(a1, a1 + 2);
+    std::list<int> l2(a2, a2 + 2);
+    l1.splice(l1.begin(), l2);
+    assert(l1.size() == 4);
+    assert(std::distance(l1.begin(), l1.end()) == 4);
+    assert(l2.size() == 0);
+    assert(std::distance(l2.begin(), l2.end()) == 0);
+    std::list<int>::const_iterator i = l1.begin();
+    assert(*i == 4);
+    ++i;
+    assert(*i == 5);
+    ++i;
+    assert(*i == 1);
+    ++i;
+    assert(*i == 2);
+  }
+  {
+    std::list<int> l1(a1, a1 + 2);
+    std::list<int> l2(a2, a2 + 2);
+    l1.splice(std::next(l1.begin()), l2);
+    assert(l1.size() == 4);
+    assert(std::distance(l1.begin(), l1.end()) == 4);
+    assert(l2.size() == 0);
+    assert(std::distance(l2.begin(), l2.end()) == 0);
+    std::list<int>::const_iterator i = l1.begin();
+    assert(*i == 1);
+    ++i;
+    assert(*i == 4);
+    ++i;
+    assert(*i == 5);
+    ++i;
+    assert(*i == 2);
+  }
+  {
+    std::list<int> l1(a1, a1 + 2);
+    std::list<int> l2(a2, a2 + 2);
+    l1.splice(std::next(l1.begin(), 2), l2);
+    assert(l1.size() == 4);
+    assert(std::distance(l1.begin(), l1.end()) == 4);
+    assert(l2.size() == 0);
+    assert(std::distance(l2.begin(), l2.end()) == 0);
+    std::list<int>::const_iterator i = l1.begin();
+    assert(*i == 1);
+    ++i;
+    assert(*i == 2);
+    ++i;
+    assert(*i == 4);
+    ++i;
+    assert(*i == 5);
+  }
+  {
+    std::list<int> l1(a1, a1 + 3);
+    std::list<int> l2(a2, a2 + 3);
+    l1.splice(l1.begin(), l2);
+    assert(l1.size() == 6);
+    assert(std::distance(l1.begin(), l1.end()) == 6);
+    assert(l2.size() == 0);
+    assert(std::distance(l2.begin(), l2.end()) == 0);
+    std::list<int>::const_iterator i = l1.begin();
+    assert(*i == 4);
+    ++i;
+    assert(*i == 5);
+    ++i;
+    assert(*i == 6);
+    ++i;
+    assert(*i == 1);
+    ++i;
+    assert(*i == 2);
+    ++i;
+    assert(*i == 3);
+  }
+  {
+    std::list<int> l1(a1, a1 + 3);
+    std::list<int> l2(a2, a2 + 3);
+    l1.splice(std::next(l1.begin()), l2);
+    assert(l1.size() == 6);
+    assert(std::distance(l1.begin(), l1.end()) == 6);
+    assert(l2.size() == 0);
+    assert(std::distance(l2.begin(), l2.end()) == 0);
+    std::list<int>::const_iterator i = l1.begin();
+    assert(*i == 1);
+    ++i;
+    assert(*i == 4);
+    ++i;
+    assert(*i == 5);
+    ++i;
+    assert(*i == 6);
+    ++i;
+    assert(*i == 2);
+    ++i;
+    assert(*i == 3);
+  }
+  {
+    std::list<int> l1(a1, a1 + 3);
+    std::list<int> l2(a2, a2 + 3);
+    l1.splice(std::next(l1.begin(), 2), l2);
+    assert(l1.size() == 6);
+    assert(std::distance(l1.begin(), l1.end()) == 6);
+    assert(l2.size() == 0);
+    assert(std::distance(l2.begin(), l2.end()) == 0);
+    std::list<int>::const_iterator i = l1.begin();
+    assert(*i == 1);
+    ++i;
+    assert(*i == 2);
+    ++i;
+    assert(*i == 4);
+    ++i;
+    assert(*i == 5);
+    ++i;
+    assert(*i == 6);
+    ++i;
+    assert(*i == 3);
+  }
+  {
+    std::list<int> l1(a1, a1 + 3);
+    std::list<int> l2(a2, a2 + 3);
+    l1.splice(std::next(l1.begin(), 3), l2);
+    assert(l1.size() == 6);
+    assert(std::distance(l1.begin(), l1.end()) == 6);
+    assert(l2.size() == 0);
+    assert(std::distance(l2.begin(), l2.end()) == 0);
+    std::list<int>::const_iterator i = l1.begin();
+    assert(*i == 1);
+    ++i;
+    assert(*i == 2);
+    ++i;
+    assert(*i == 3);
+    ++i;
+    assert(*i == 4);
+    ++i;
+    assert(*i == 5);
+    ++i;
+    assert(*i == 6);
+  }
 #if TEST_STD_VER >= 11
-    {
-        std::list<int, min_allocator<int>> l1;
-        std::list<int, min_allocator<int>> l2;
-        l1.splice(l1.end(), l2);
-        assert(l1.size() == 0);
-        assert(std::distance(l1.begin(), l1.end()) == 0);
-        assert(l2.size() == 0);
-        assert(std::distance(l2.begin(), l2.end()) == 0);
-    }
-    {
-        std::list<int, min_allocator<int>> l1;
-        std::list<int, min_allocator<int>> l2(a2, a2+1);
-        l1.splice(l1.end(), l2);
-        assert(l1.size() == 1);
-        assert(std::distance(l1.begin(), l1.end()) == 1);
-        assert(l2.size() == 0);
-        assert(std::distance(l2.begin(), l2.end()) == 0);
-        std::list<int, min_allocator<int>>::const_iterator i = l1.begin();
-        assert(*i == 4);
-    }
-    {
-        std::list<int, min_allocator<int>> l1;
-        std::list<int, min_allocator<int>> l2(a2, a2+2);
-        l1.splice(l1.end(), l2);
-        assert(l1.size() == 2);
-        assert(std::distance(l1.begin(), l1.end()) == 2);
-        assert(l2.size() == 0);
-        assert(std::distance(l2.begin(), l2.end()) == 0);
-        std::list<int, min_allocator<int>>::const_iterator i = l1.begin();
-        assert(*i == 4);
-        ++i;
-        assert(*i == 5);
-    }
-    {
-        std::list<int, min_allocator<int>> l1;
-        std::list<int, min_allocator<int>> l2(a2, a2+3);
-        l1.splice(l1.end(), l2);
-        assert(l1.size() == 3);
-        assert(std::distance(l1.begin(), l1.end()) == 3);
-        assert(l2.size() == 0);
-        assert(std::distance(l2.begin(), l2.end()) == 0);
-        std::list<int, min_allocator<int>>::const_iterator i = l1.begin();
-        assert(*i == 4);
-        ++i;
-        assert(*i == 5);
-        ++i;
-        assert(*i == 6);
-    }
-    {
-        std::list<int, min_allocator<int>> l1(a1, a1+1);
-        std::list<int, min_allocator<int>> l2;
-        l1.splice(l1.begin(), l2);
-        assert(l1.size() == 1);
-        assert(std::distance(l1.begin(), l1.end()) == 1);
-        assert(l2.size() == 0);
-        assert(std::distance(l2.begin(), l2.end()) == 0);
-        std::list<int, min_allocator<int>>::const_iterator i = l1.begin();
-        assert(*i == 1);
-    }
-    {
-        std::list<int, min_allocator<int>> l1(a1, a1+1);
-        std::list<int, min_allocator<int>> l2;
-        l1.splice(l1.end(), l2);
-        assert(l1.size() == 1);
-        assert(std::distance(l1.begin(), l1.end()) == 1);
-        assert(l2.size() == 0);
-        assert(std::distance(l2.begin(), l2.end()) == 0);
-        std::list<int, min_allocator<int>>::const_iterator i = l1.begin();
-        assert(*i == 1);
-    }
-    {
-        std::list<int, min_allocator<int>> l1(a1, a1+1);
-        std::list<int, min_allocator<int>> l2(a2, a2+1);
-        l1.splice(l1.begin(), l2);
-        assert(l1.size() == 2);
-        assert(std::distance(l1.begin(), l1.end()) == 2);
-        assert(l2.size() == 0);
-        assert(std::distance(l2.begin(), l2.end()) == 0);
-        std::list<int, min_allocator<int>>::const_iterator i = l1.begin();
-        assert(*i == 4);
-        ++i;
-        assert(*i == 1);
-    }
-    {
-        std::list<int, min_allocator<int>> l1(a1, a1+1);
-        std::list<int, min_allocator<int>> l2(a2, a2+1);
-        l1.splice(l1.end(), l2);
-        assert(l1.size() == 2);
-        assert(std::distance(l1.begin(), l1.end()) == 2);
-        assert(l2.size() == 0);
-        assert(std::distance(l2.begin(), l2.end()) == 0);
-        std::list<int, min_allocator<int>>::const_iterator i = l1.begin();
-        assert(*i == 1);
-        ++i;
-        assert(*i == 4);
-    }
-    {
-        std::list<int, min_allocator<int>> l1(a1, a1+1);
-        std::list<int, min_allocator<int>> l2(a2, a2+2);
-        l1.splice(l1.begin(), l2);
-        assert(l1.size() == 3);
-        assert(std::distance(l1.begin(), l1.end()) == 3);
-        assert(l2.size() == 0);
-        assert(std::distance(l2.begin(), l2.end()) == 0);
-        std::list<int, min_allocator<int>>::const_iterator i = l1.begin();
-        assert(*i == 4);
-        ++i;
-        assert(*i == 5);
-        ++i;
-        assert(*i == 1);
-    }
-    {
-        std::list<int, min_allocator<int>> l1(a1, a1+1);
-        std::list<int, min_allocator<int>> l2(a2, a2+2);
-        l1.splice(l1.end(), l2);
-        assert(l1.size() == 3);
-        assert(std::distance(l1.begin(), l1.end()) == 3);
-        assert(l2.size() == 0);
-        assert(std::distance(l2.begin(), l2.end()) == 0);
-        std::list<int, min_allocator<int>>::const_iterator i = l1.begin();
-        assert(*i == 1);
-        ++i;
-        assert(*i == 4);
-        ++i;
-        assert(*i == 5);
-    }
-    {
-        std::list<int, min_allocator<int>> l1(a1, a1+1);
-        std::list<int, min_allocator<int>> l2(a2, a2+3);
-        l1.splice(l1.begin(), l2);
-        assert(l1.size() == 4);
-        assert(std::distance(l1.begin(), l1.end()) == 4);
-        assert(l2.size() == 0);
-        assert(std::distance(l2.begin(), l2.end()) == 0);
-        std::list<int, min_allocator<int>>::const_iterator i = l1.begin();
-        assert(*i == 4);
-        ++i;
-        assert(*i == 5);
-        ++i;
-        assert(*i == 6);
-        ++i;
-        assert(*i == 1);
-    }
-    {
-        std::list<int, min_allocator<int>> l1(a1, a1+1);
-        std::list<int, min_allocator<int>> l2(a2, a2+3);
-        l1.splice(l1.end(), l2);
-        assert(l1.size() == 4);
-        assert(std::distance(l1.begin(), l1.end()) == 4);
-        assert(l2.size() == 0);
-        assert(std::distance(l2.begin(), l2.end()) == 0);
-        std::list<int, min_allocator<int>>::const_iterator i = l1.begin();
-        assert(*i == 1);
-        ++i;
-        assert(*i == 4);
-        ++i;
-        assert(*i == 5);
-        ++i;
-        assert(*i == 6);
-    }
-    {
-        std::list<int, min_allocator<int>> l1(a1, a1+2);
-        std::list<int, min_allocator<int>> l2;
-        l1.splice(l1.begin(), l2);
-        assert(l1.size() == 2);
-        assert(std::distance(l1.begin(), l1.end()) == 2);
-        assert(l2.size() == 0);
-        assert(std::distance(l2.begin(), l2.end()) == 0);
-        std::list<int, min_allocator<int>>::const_iterator i = l1.begin();
-        assert(*i == 1);
-        ++i;
-        assert(*i == 2);
-    }
-    {
-        std::list<int, min_allocator<int>> l1(a1, a1+2);
-        std::list<int, min_allocator<int>> l2;
-        l1.splice(std::next(l1.begin()), l2);
-        assert(l1.size() == 2);
-        assert(std::distance(l1.begin(), l1.end()) == 2);
-        assert(l2.size() == 0);
-        assert(std::distance(l2.begin(), l2.end()) == 0);
-        std::list<int, min_allocator<int>>::const_iterator i = l1.begin();
-        assert(*i == 1);
-        ++i;
-        assert(*i == 2);
-    }
-    {
-        std::list<int, min_allocator<int>> l1(a1, a1+2);
-        std::list<int, min_allocator<int>> l2;
-        l1.splice(std::next(l1.begin(), 2), l2);
-        assert(l1.size() == 2);
-        assert(std::distance(l1.begin(), l1.end()) == 2);
-        assert(l2.size() == 0);
-        assert(std::distance(l2.begin(), l2.end()) == 0);
-        std::list<int, min_allocator<int>>::const_iterator i = l1.begin();
-        assert(*i == 1);
-        ++i;
-        assert(*i == 2);
-    }
-    {
-        std::list<int, min_allocator<int>> l1(a1, a1+2);
-        std::list<int, min_allocator<int>> l2(a2, a2+1);
-        l1.splice(l1.begin(), l2);
-        assert(l1.size() == 3);
-        assert(std::distance(l1.begin(), l1.end()) == 3);
-        assert(l2.size() == 0);
-        assert(std::distance(l2.begin(), l2.end()) == 0);
-        std::list<int, min_allocator<int>>::const_iterator i = l1.begin();
-        assert(*i == 4);
-        ++i;
-        assert(*i == 1);
-        ++i;
-        assert(*i == 2);
-    }
-    {
-        std::list<int, min_allocator<int>> l1(a1, a1+2);
-        std::list<int, min_allocator<int>> l2(a2, a2+1);
-        l1.splice(std::next(l1.begin()), l2);
-        assert(l1.size() == 3);
-        assert(std::distance(l1.begin(), l1.end()) == 3);
-        assert(l2.size() == 0);
-        assert(std::distance(l2.begin(), l2.end()) == 0);
-        std::list<int, min_allocator<int>>::const_iterator i = l1.begin();
-        assert(*i == 1);
-        ++i;
-        assert(*i == 4);
-        ++i;
-        assert(*i == 2);
-    }
-    {
-        std::list<int, min_allocator<int>> l1(a1, a1+2);
-        std::list<int, min_allocator<int>> l2(a2, a2+1);
-        l1.splice(std::next(l1.begin(), 2), l2);
-        assert(l1.size() == 3);
-        assert(std::distance(l1.begin(), l1.end()) == 3);
-        assert(l2.size() == 0);
-        assert(std::distance(l2.begin(), l2.end()) == 0);
-        std::list<int, min_allocator<int>>::const_iterator i = l1.begin();
-        assert(*i == 1);
-        ++i;
-        assert(*i == 2);
-        ++i;
-        assert(*i == 4);
-    }
-    {
-        std::list<int, min_allocator<int>> l1(a1, a1+2);
-        std::list<int, min_allocator<int>> l2(a2, a2+2);
-        l1.splice(l1.begin(), l2);
-        assert(l1.size() == 4);
-        assert(std::distance(l1.begin(), l1.end()) == 4);
-        assert(l2.size() == 0);
-        assert(std::distance(l2.begin(), l2.end()) == 0);
-        std::list<int, min_allocator<int>>::const_iterator i = l1.begin();
-        assert(*i == 4);
-        ++i;
-        assert(*i == 5);
-        ++i;
-        assert(*i == 1);
-        ++i;
-        assert(*i == 2);
-    }
-    {
-        std::list<int, min_allocator<int>> l1(a1, a1+2);
-        std::list<int, min_allocator<int>> l2(a2, a2+2);
-        l1.splice(std::next(l1.begin()), l2);
-        assert(l1.size() == 4);
-        assert(std::distance(l1.begin(), l1.end()) == 4);
-        assert(l2.size() == 0);
-        assert(std::distance(l2.begin(), l2.end()) == 0);
-        std::list<int, min_allocator<int>>::const_iterator i = l1.begin();
-        assert(*i == 1);
-        ++i;
-        assert(*i == 4);
-        ++i;
-        assert(*i == 5);
-        ++i;
-        assert(*i == 2);
-    }
-    {
-        std::list<int, min_allocator<int>> l1(a1, a1+2);
-        std::list<int, min_allocator<int>> l2(a2, a2+2);
-        l1.splice(std::next(l1.begin(), 2), l2);
-        assert(l1.size() == 4);
-        assert(std::distance(l1.begin(), l1.end()) == 4);
-        assert(l2.size() == 0);
-        assert(std::distance(l2.begin(), l2.end()) == 0);
-        std::list<int, min_allocator<int>>::const_iterator i = l1.begin();
-        assert(*i == 1);
-        ++i;
-        assert(*i == 2);
-        ++i;
-        assert(*i == 4);
-        ++i;
-        assert(*i == 5);
-    }
-    {
-        std::list<int, min_allocator<int>> l1(a1, a1+3);
-        std::list<int, min_allocator<int>> l2(a2, a2+3);
-        l1.splice(l1.begin(), l2);
-        assert(l1.size() == 6);
-        assert(std::distance(l1.begin(), l1.end()) == 6);
-        assert(l2.size() == 0);
-        assert(std::distance(l2.begin(), l2.end()) == 0);
-        std::list<int, min_allocator<int>>::const_iterator i = l1.begin();
-        assert(*i == 4);
-        ++i;
-        assert(*i == 5);
-        ++i;
-        assert(*i == 6);
-        ++i;
-        assert(*i == 1);
-        ++i;
-        assert(*i == 2);
-        ++i;
-        assert(*i == 3);
-    }
-    {
-        std::list<int, min_allocator<int>> l1(a1, a1+3);
-        std::list<int, min_allocator<int>> l2(a2, a2+3);
-        l1.splice(std::next(l1.begin()), l2);
-        assert(l1.size() == 6);
-        assert(std::distance(l1.begin(), l1.end()) == 6);
-        assert(l2.size() == 0);
-        assert(std::distance(l2.begin(), l2.end()) == 0);
-        std::list<int, min_allocator<int>>::const_iterator i = l1.begin();
-        assert(*i == 1);
-        ++i;
-        assert(*i == 4);
-        ++i;
-        assert(*i == 5);
-        ++i;
-        assert(*i == 6);
-        ++i;
-        assert(*i == 2);
-        ++i;
-        assert(*i == 3);
-    }
-    {
-        std::list<int, min_allocator<int>> l1(a1, a1+3);
-        std::list<int, min_allocator<int>> l2(a2, a2+3);
-        l1.splice(std::next(l1.begin(), 2), l2);
-        assert(l1.size() == 6);
-        assert(std::distance(l1.begin(), l1.end()) == 6);
-        assert(l2.size() == 0);
-        assert(std::distance(l2.begin(), l2.end()) == 0);
-        std::list<int, min_allocator<int>>::const_iterator i = l1.begin();
-        assert(*i == 1);
-        ++i;
-        assert(*i == 2);
-        ++i;
-        assert(*i == 4);
-        ++i;
-        assert(*i == 5);
-        ++i;
-        assert(*i == 6);
-        ++i;
-        assert(*i == 3);
-    }
-    {
-        std::list<int, min_allocator<int>> l1(a1, a1+3);
-        std::list<int, min_allocator<int>> l2(a2, a2+3);
-        l1.splice(std::next(l1.begin(), 3), l2);
-        assert(l1.size() == 6);
-        assert(std::distance(l1.begin(), l1.end()) == 6);
-        assert(l2.size() == 0);
-        assert(std::distance(l2.begin(), l2.end()) == 0);
-        std::list<int, min_allocator<int>>::const_iterator i = l1.begin();
-        assert(*i == 1);
-        ++i;
-        assert(*i == 2);
-        ++i;
-        assert(*i == 3);
-        ++i;
-        assert(*i == 4);
-        ++i;
-        assert(*i == 5);
-        ++i;
-        assert(*i == 6);
-    }
+  {
+    std::list<int, min_allocator<int>> l1;
+    std::list<int, min_allocator<int>> l2;
+    l1.splice(l1.end(), l2);
+    assert(l1.size() == 0);
+    assert(std::distance(l1.begin(), l1.end()) == 0);
+    assert(l2.size() == 0);
+    assert(std::distance(l2.begin(), l2.end()) == 0);
+  }
+  {
+    std::list<int, min_allocator<int>> l1;
+    std::list<int, min_allocator<int>> l2(a2, a2 + 1);
+    l1.splice(l1.end(), l2);
+    assert(l1.size() == 1);
+    assert(std::distance(l1.begin(), l1.end()) == 1);
+    assert(l2.size() == 0);
+    assert(std::distance(l2.begin(), l2.end()) == 0);
+    std::list<int, min_allocator<int>>::const_iterator i = l1.begin();
+    assert(*i == 4);
+  }
+  {
+    std::list<int, min_allocator<int>> l1;
+    std::list<int, min_allocator<int>> l2(a2, a2 + 2);
+    l1.splice(l1.end(), l2);
+    assert(l1.size() == 2);
+    assert(std::distance(l1.begin(), l1.end()) == 2);
+    assert(l2.size() == 0);
+    assert(std::distance(l2.begin(), l2.end()) == 0);
+    std::list<int, min_allocator<int>>::const_iterator i = l1.begin();
+    assert(*i == 4);
+    ++i;
+    assert(*i == 5);
+  }
+  {
+    std::list<int, min_allocator<int>> l1;
+    std::list<int, min_allocator<int>> l2(a2, a2 + 3);
+    l1.splice(l1.end(), l2);
+    assert(l1.size() == 3);
+    assert(std::distance(l1.begin(), l1.end()) == 3);
+    assert(l2.size() == 0);
+    assert(std::distance(l2.begin(), l2.end()) == 0);
+    std::list<int, min_allocator<int>>::const_iterator i = l1.begin();
+    assert(*i == 4);
+    ++i;
+    assert(*i == 5);
+    ++i;
+    assert(*i == 6);
+  }
+  {
+    std::list<int, min_allocator<int>> l1(a1, a1 + 1);
+    std::list<int, min_allocator<int>> l2;
+    l1.splice(l1.begin(), l2);
+    assert(l1.size() == 1);
+    assert(std::distance(l1.begin(), l1.end()) == 1);
+    assert(l2.size() == 0);
+    assert(std::distance(l2.begin(), l2.end()) == 0);
+    std::list<int, min_allocator<int>>::const_iterator i = l1.begin();
+    assert(*i == 1);
+  }
+  {
+    std::list<int, min_allocator<int>> l1(a1, a1 + 1);
+    std::list<int, min_allocator<int>> l2;
+    l1.splice(l1.end(), l2);
+    assert(l1.size() == 1);
+    assert(std::distance(l1.begin(), l1.end()) == 1);
+    assert(l2.size() == 0);
+    assert(std::distance(l2.begin(), l2.end()) == 0);
+    std::list<int, min_allocator<int>>::const_iterator i = l1.begin();
+    assert(*i == 1);
+  }
+  {
+    std::list<int, min_allocator<int>> l1(a1, a1 + 1);
+    std::list<int, min_allocator<int>> l2(a2, a2 + 1);
+    l1.splice(l1.begin(), l2);
+    assert(l1.size() == 2);
+    assert(std::distance(l1.begin(), l1.end()) == 2);
+    assert(l2.size() == 0);
+    assert(std::distance(l2.begin(), l2.end()) == 0);
+    std::list<int, min_allocator<int>>::const_iterator i = l1.begin();
+    assert(*i == 4);
+    ++i;
+    assert(*i == 1);
+  }
+  {
+    std::list<int, min_allocator<int>> l1(a1, a1 + 1);
+    std::list<int, min_allocator<int>> l2(a2, a2 + 1);
+    l1.splice(l1.end(), l2);
+    assert(l1.size() == 2);
+    assert(std::distance(l1.begin(), l1.end()) == 2);
+    assert(l2.size() == 0);
+    assert(std::distance(l2.begin(), l2.end()) == 0);
+    std::list<int, min_allocator<int>>::const_iterator i = l1.begin();
+    assert(*i == 1);
+    ++i;
+    assert(*i == 4);
+  }
+  {
+    std::list<int, min_allocator<int>> l1(a1, a1 + 1);
+    std::list<int, min_allocator<int>> l2(a2, a2 + 2);
+    l1.splice(l1.begin(), l2);
+    assert(l1.size() == 3);
+    assert(std::distance(l1.begin(), l1.end()) == 3);
+    assert(l2.size() == 0);
+    assert(std::distance(l2.begin(), l2.end()) == 0);
+    std::list<int, min_allocator<int>>::const_iterator i = l1.begin();
+    assert(*i == 4);
+    ++i;
+    assert(*i == 5);
+    ++i;
+    assert(*i == 1);
+  }
+  {
+    std::list<int, min_allocator<int>> l1(a1, a1 + 1);
+    std::list<int, min_allocator<int>> l2(a2, a2 + 2);
+    l1.splice(l1.end(), l2);
+    assert(l1.size() == 3);
+    assert(std::distance(l1.begin(), l1.end()) == 3);
+    assert(l2.size() == 0);
+    assert(std::distance(l2.begin(), l2.end()) == 0);
+    std::list<int, min_allocator<int>>::const_iterator i = l1.begin();
+    assert(*i == 1);
+    ++i;
+    assert(*i == 4);
+    ++i;
+    assert(*i == 5);
+  }
+  {
+    std::list<int, min_allocator<int>> l1(a1, a1 + 1);
+    std::list<int, min_allocator<int>> l2(a2, a2 + 3);
+    l1.splice(l1.begin(), l2);
+    assert(l1.size() == 4);
+    assert(std::distance(l1.begin(), l1.end()) == 4);
+    assert(l2.size() == 0);
+    assert(std::distance(l2.begin(), l2.end()) == 0);
+    std::list<int, min_allocator<int>>::const_iterator i = l1.begin();
+    assert(*i == 4);
+    ++i;
+    assert(*i == 5);
+    ++i;
+    assert(*i == 6);
+    ++i;
+    assert(*i == 1);
+  }
+  {
+    std::list<int, min_allocator<int>> l1(a1, a1 + 1);
+    std::list<int, min_allocator<int>> l2(a2, a2 + 3);
+    l1.splice(l1.end(), l2);
+    assert(l1.size() == 4);
+    assert(std::distance(l1.begin(), l1.end()) == 4);
+    assert(l2.size() == 0);
+    assert(std::distance(l2.begin(), l2.end()) == 0);
+    std::list<int, min_allocator<int>>::const_iterator i = l1.begin();
+    assert(*i == 1);
+    ++i;
+    assert(*i == 4);
+    ++i;
+    assert(*i == 5);
+    ++i;
+    assert(*i == 6);
+  }
+  {
+    std::list<int, min_allocator<int>> l1(a1, a1 + 2);
+    std::list<int, min_allocator<int>> l2;
+    l1.splice(l1.begin(), l2);
+    assert(l1.size() == 2);
+    assert(std::distance(l1.begin(), l1.end()) == 2);
+    assert(l2.size() == 0);
+    assert(std::distance(l2.begin(), l2.end()) == 0);
+    std::list<int, min_allocator<int>>::const_iterator i = l1.begin();
+    assert(*i == 1);
+    ++i;
+    assert(*i == 2);
+  }
+  {
+    std::list<int, min_allocator<int>> l1(a1, a1 + 2);
+    std::list<int, min_allocator<int>> l2;
+    l1.splice(std::next(l1.begin()), l2);
+    assert(l1.size() == 2);
+    assert(std::distance(l1.begin(), l1.end()) == 2);
+    assert(l2.size() == 0);
+    assert(std::distance(l2.begin(), l2.end()) == 0);
+    std::list<int, min_allocator<int>>::const_iterator i = l1.begin();
+    assert(*i == 1);
+    ++i;
+    assert(*i == 2);
+  }
+  {
+    std::list<int, min_allocator<int>> l1(a1, a1 + 2);
+    std::list<int, min_allocator<int>> l2;
+    l1.splice(std::next(l1.begin(), 2), l2);
+    assert(l1.size() == 2);
+    assert(std::distance(l1.begin(), l1.end()) == 2);
+    assert(l2.size() == 0);
+    assert(std::distance(l2.begin(), l2.end()) == 0);
+    std::list<int, min_allocator<int>>::const_iterator i = l1.begin();
+    assert(*i == 1);
+    ++i;
+    assert(*i == 2);
+  }
+  {
+    std::list<int, min_allocator<int>> l1(a1, a1 + 2);
+    std::list<int, min_allocator<int>> l2(a2, a2 + 1);
+    l1.splice(l1.begin(), l2);
+    assert(l1.size() == 3);
+    assert(std::distance(l1.begin(), l1.end()) == 3);
+    assert(l2.size() == 0);
+    assert(std::distance(l2.begin(), l2.end()) == 0);
+    std::list<int, min_allocator<int>>::const_iterator i = l1.begin();
+    assert(*i == 4);
+    ++i;
+    assert(*i == 1);
+    ++i;
+    assert(*i == 2);
+  }
+  {
+    std::list<int, min_allocator<int>> l1(a1, a1 + 2);
+    std::list<int, min_allocator<int>> l2(a2, a2 + 1);
+    l1.splice(std::next(l1.begin()), l2);
+    assert(l1.size() == 3);
+    assert(std::distance(l1.begin(), l1.end()) == 3);
+    assert(l2.size() == 0);
+    assert(std::distance(l2.begin(), l2.end()) == 0);
+    std::list<int, min_allocator<int>>::const_iterator i = l1.begin();
+    assert(*i == 1);
+    ++i;
+    assert(*i == 4);
+    ++i;
+    assert(*i == 2);
+  }
+  {
+    std::list<int, min_allocator<int>> l1(a1, a1 + 2);
+    std::list<int, min_allocator<int>> l2(a2, a2 + 1);
+    l1.splice(std::next(l1.begin(), 2), l2);
+    assert(l1.size() == 3);
+    assert(std::distance(l1.begin(), l1.end()) == 3);
+    assert(l2.size() == 0);
+    assert(std::distance(l2.begin(), l2.end()) == 0);
+    std::list<int, min_allocator<int>>::const_iterator i = l1.begin();
+    assert(*i == 1);
+    ++i;
+    assert(*i == 2);
+    ++i;
+    assert(*i == 4);
+  }
+  {
+    std::list<int, min_allocator<int>> l1(a1, a1 + 2);
+    std::list<int, min_allocator<int>> l2(a2, a2 + 2);
+    l1.splice(l1.begin(), l2);
+    assert(l1.size() == 4);
+    assert(std::distance(l1.begin(), l1.end()) == 4);
+    assert(l2.size() == 0);
+    assert(std::distance(l2.begin(), l2.end()) == 0);
+    std::list<int, min_allocator<int>>::const_iterator i = l1.begin();
+    assert(*i == 4);
+    ++i;
+    assert(*i == 5);
+    ++i;
+    assert(*i == 1);
+    ++i;
+    assert(*i == 2);
+  }
+  {
+    std::list<int, min_allocator<int>> l1(a1, a1 + 2);
+    std::list<int, min_allocator<int>> l2(a2, a2 + 2);
+    l1.splice(std::next(l1.begin()), l2);
+    assert(l1.size() == 4);
+    assert(std::distance(l1.begin(), l1.end()) == 4);
+    assert(l2.size() == 0);
+    assert(std::distance(l2.begin(), l2.end()) == 0);
+    std::list<int, min_allocator<int>>::const_iterator i = l1.begin();
+    assert(*i == 1);
+    ++i;
+    assert(*i == 4);
+    ++i;
+    assert(*i == 5);
+    ++i;
+    assert(*i == 2);
+  }
+  {
+    std::list<int, min_allocator<int>> l1(a1, a1 + 2);
+    std::list<int, min_allocator<int>> l2(a2, a2 + 2);
+    l1.splice(std::next(l1.begin(), 2), l2);
+    assert(l1.size() == 4);
+    assert(std::distance(l1.begin(), l1.end()) == 4);
+    assert(l2.size() == 0);
+    assert(std::distance(l2.begin(), l2.end()) == 0);
+    std::list<int, min_allocator<int>>::const_iterator i = l1.begin();
+    assert(*i == 1);
+    ++i;
+    assert(*i == 2);
+    ++i;
+    assert(*i == 4);
+    ++i;
+    assert(*i == 5);
+  }
+  {
+    std::list<int, min_allocator<int>> l1(a1, a1 + 3);
+    std::list<int, min_allocator<int>> l2(a2, a2 + 3);
+    l1.splice(l1.begin(), l2);
+    assert(l1.size() == 6);
+    assert(std::distance(l1.begin(), l1.end()) == 6);
+    assert(l2.size() == 0);
+    assert(std::distance(l2.begin(), l2.end()) == 0);
+    std::list<int, min_allocator<int>>::const_iterator i = l1.begin();
+    assert(*i == 4);
+    ++i;
+    assert(*i == 5);
+    ++i;
+    assert(*i == 6);
+    ++i;
+    assert(*i == 1);
+    ++i;
+    assert(*i == 2);
+    ++i;
+    assert(*i == 3);
+  }
+  {
+    std::list<int, min_allocator<int>> l1(a1, a1 + 3);
+    std::list<int, min_allocator<int>> l2(a2, a2 + 3);
+    l1.splice(std::next(l1.begin()), l2);
+    assert(l1.size() == 6);
+    assert(std::distance(l1.begin(), l1.end()) == 6);
+    assert(l2.size() == 0);
+    assert(std::distance(l2.begin(), l2.end()) == 0);
+    std::list<int, min_allocator<int>>::const_iterator i = l1.begin();
+    assert(*i == 1);
+    ++i;
+    assert(*i == 4);
+    ++i;
+    assert(*i == 5);
+    ++i;
+    assert(*i == 6);
+    ++i;
+    assert(*i == 2);
+    ++i;
+    assert(*i == 3);
+  }
+  {
+    std::list<int, min_allocator<int>> l1(a1, a1 + 3);
+    std::list<int, min_allocator<int>> l2(a2, a2 + 3);
+    l1.splice(std::next(l1.begin(), 2), l2);
+    assert(l1.size() == 6);
+    assert(std::distance(l1.begin(), l1.end()) == 6);
+    assert(l2.size() == 0);
+    assert(std::distance(l2.begin(), l2.end()) == 0);
+    std::list<int, min_allocator<int>>::const_iterator i = l1.begin();
+    assert(*i == 1);
+    ++i;
+    assert(*i == 2);
+    ++i;
+    assert(*i == 4);
+    ++i;
+    assert(*i == 5);
+    ++i;
+    assert(*i == 6);
+    ++i;
+    assert(*i == 3);
+  }
+  {
+    std::list<int, min_allocator<int>> l1(a1, a1 + 3);
+    std::list<int, min_allocator<int>> l2(a2, a2 + 3);
+    l1.splice(std::next(l1.begin(), 3), l2);
+    assert(l1.size() == 6);
+    assert(std::distance(l1.begin(), l1.end()) == 6);
+    assert(l2.size() == 0);
+    assert(std::distance(l2.begin(), l2.end()) == 0);
+    std::list<int, min_allocator<int>>::const_iterator i = l1.begin();
+    assert(*i == 1);
+    ++i;
+    assert(*i == 2);
+    ++i;
+    assert(*i == 3);
+    ++i;
+    assert(*i == 4);
+    ++i;
+    assert(*i == 5);
+    ++i;
+    assert(*i == 6);
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/sequences/list/list.ops/splice_pos_list_iter.pass.cpp b/libcxx/test/std/containers/sequences/list/list.ops/splice_pos_list_iter.pass.cpp
index 08b78e14afa1b..db71fe17a06eb 100644
--- a/libcxx/test/std/containers/sequences/list/list.ops/splice_pos_list_iter.pass.cpp
+++ b/libcxx/test/std/containers/sequences/list/list.ops/splice_pos_list_iter.pass.cpp
@@ -16,323 +16,322 @@
 #include "test_macros.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    int a1[] = {1, 2, 3};
-    int a2[] = {4, 5, 6};
-    {
-        std::list<int> l1;
-        std::list<int> l2(a2, a2+1);
-        l1.splice(l1.end(), l2, l2.begin());
-        assert(l1.size() == 1);
-        assert(std::distance(l1.begin(), l1.end()) == 1);
-        assert(l2.size() == 0);
-        assert(std::distance(l2.begin(), l2.end()) == 0);
-        std::list<int>::const_iterator i = l1.begin();
-        assert(*i == 4);
-    }
-    {
-        std::list<int> l1;
-        std::list<int> l2(a2, a2+2);
-        l1.splice(l1.end(), l2, l2.begin());
-        assert(l1.size() == 1);
-        assert(std::distance(l1.begin(), l1.end()) == 1);
-        assert(l2.size() == 1);
-        assert(std::distance(l2.begin(), l2.end()) == 1);
-        std::list<int>::const_iterator i = l1.begin();
-        assert(*i == 4);
-        i = l2.begin();
-        assert(*i == 5);
-    }
-    {
-        std::list<int> l1;
-        std::list<int> l2(a2, a2+2);
-        l1.splice(l1.end(), l2, std::next(l2.begin()));
-        assert(l1.size() == 1);
-        assert(std::distance(l1.begin(), l1.end()) == 1);
-        assert(l2.size() == 1);
-        assert(std::distance(l2.begin(), l2.end()) == 1);
-        std::list<int>::const_iterator i = l1.begin();
-        assert(*i == 5);
-        i = l2.begin();
-        assert(*i == 4);
-    }
-    {
-        std::list<int> l1;
-        std::list<int> l2(a2, a2+3);
-        l1.splice(l1.end(), l2, l2.begin());
-        assert(l1.size() == 1);
-        assert(std::distance(l1.begin(), l1.end()) == 1);
-        assert(l2.size() == 2);
-        assert(std::distance(l2.begin(), l2.end()) == 2);
-        std::list<int>::const_iterator i = l1.begin();
-        assert(*i == 4);
-        i = l2.begin();
-        assert(*i == 5);
-        ++i;
-        assert(*i == 6);
-    }
-    {
-        std::list<int> l1;
-        std::list<int> l2(a2, a2+3);
-        l1.splice(l1.end(), l2, std::next(l2.begin()));
-        assert(l1.size() == 1);
-        assert(std::distance(l1.begin(), l1.end()) == 1);
-        assert(l2.size() == 2);
-        assert(std::distance(l2.begin(), l2.end()) == 2);
-        std::list<int>::const_iterator i = l1.begin();
-        assert(*i == 5);
-        i = l2.begin();
-        assert(*i == 4);
-        ++i;
-        assert(*i == 6);
-    }
-    {
-        std::list<int> l1;
-        std::list<int> l2(a2, a2+3);
-        l1.splice(l1.end(), l2, std::next(l2.begin(), 2));
-        assert(l1.size() == 1);
-        assert(std::distance(l1.begin(), l1.end()) == 1);
-        assert(l2.size() == 2);
-        assert(std::distance(l2.begin(), l2.end()) == 2);
-        std::list<int>::const_iterator i = l1.begin();
-        assert(*i == 6);
-        i = l2.begin();
-        assert(*i == 4);
-        ++i;
-        assert(*i == 5);
-    }
-    {
-        std::list<int> l1(a1, a1+1);
-        l1.splice(l1.begin(), l1, l1.begin());
-        assert(l1.size() == 1);
-        assert(std::distance(l1.begin(), l1.end()) == 1);
-        std::list<int>::const_iterator i = l1.begin();
-        assert(*i == 1);
-    }
-    {
-        std::list<int> l1(a1, a1+1);
-        std::list<int> l2(a2, a2+1);
-        l1.splice(l1.begin(), l2, l2.begin());
-        assert(l1.size() == 2);
-        assert(std::distance(l1.begin(), l1.end()) == 2);
-        assert(l2.size() == 0);
-        assert(std::distance(l2.begin(), l2.end()) == 0);
-        std::list<int>::const_iterator i = l1.begin();
-        assert(*i == 4);
-        ++i;
-        assert(*i == 1);
-    }
-    {
-        std::list<int> l1(a1, a1+1);
-        std::list<int> l2(a2, a2+1);
-        l1.splice(std::next(l1.begin()), l2, l2.begin());
-        assert(l1.size() == 2);
-        assert(std::distance(l1.begin(), l1.end()) == 2);
-        assert(l2.size() == 0);
-        assert(std::distance(l2.begin(), l2.end()) == 0);
-        std::list<int>::const_iterator i = l1.begin();
-        assert(*i == 1);
-        ++i;
-        assert(*i == 4);
-    }
-    {
-        std::list<int> l1(a1, a1+2);
-        l1.splice(l1.begin(), l1, l1.begin());
-        assert(l1.size() == 2);
-        assert(std::distance(l1.begin(), l1.end()) == 2);
-        std::list<int>::const_iterator i = l1.begin();
-        assert(*i == 1);
-        ++i;
-        assert(*i == 2);
-    }
-    {
-        std::list<int> l1(a1, a1+2);
-        l1.splice(l1.begin(), l1, std::next(l1.begin()));
-        assert(l1.size() == 2);
-        assert(std::distance(l1.begin(), l1.end()) == 2);
-        std::list<int>::const_iterator i = l1.begin();
-        assert(*i == 2);
-        ++i;
-        assert(*i == 1);
-    }
-    {
-        std::list<int> l1(a1, a1+2);
-        l1.splice(std::next(l1.begin()), l1, l1.begin());
-        assert(l1.size() == 2);
-        assert(std::distance(l1.begin(), l1.end()) == 2);
-        std::list<int>::const_iterator i = l1.begin();
-        assert(*i == 1);
-        ++i;
-        assert(*i == 2);
-    }
-    {
-        std::list<int> l1(a1, a1+2);
-        l1.splice(std::next(l1.begin()), l1, std::next(l1.begin()));
-        assert(l1.size() == 2);
-        assert(std::distance(l1.begin(), l1.end()) == 2);
-        std::list<int>::const_iterator i = l1.begin();
-        assert(*i == 1);
-        ++i;
-        assert(*i == 2);
-    }
+int main(int, char**) {
+  int a1[] = {1, 2, 3};
+  int a2[] = {4, 5, 6};
+  {
+    std::list<int> l1;
+    std::list<int> l2(a2, a2 + 1);
+    l1.splice(l1.end(), l2, l2.begin());
+    assert(l1.size() == 1);
+    assert(std::distance(l1.begin(), l1.end()) == 1);
+    assert(l2.size() == 0);
+    assert(std::distance(l2.begin(), l2.end()) == 0);
+    std::list<int>::const_iterator i = l1.begin();
+    assert(*i == 4);
+  }
+  {
+    std::list<int> l1;
+    std::list<int> l2(a2, a2 + 2);
+    l1.splice(l1.end(), l2, l2.begin());
+    assert(l1.size() == 1);
+    assert(std::distance(l1.begin(), l1.end()) == 1);
+    assert(l2.size() == 1);
+    assert(std::distance(l2.begin(), l2.end()) == 1);
+    std::list<int>::const_iterator i = l1.begin();
+    assert(*i == 4);
+    i = l2.begin();
+    assert(*i == 5);
+  }
+  {
+    std::list<int> l1;
+    std::list<int> l2(a2, a2 + 2);
+    l1.splice(l1.end(), l2, std::next(l2.begin()));
+    assert(l1.size() == 1);
+    assert(std::distance(l1.begin(), l1.end()) == 1);
+    assert(l2.size() == 1);
+    assert(std::distance(l2.begin(), l2.end()) == 1);
+    std::list<int>::const_iterator i = l1.begin();
+    assert(*i == 5);
+    i = l2.begin();
+    assert(*i == 4);
+  }
+  {
+    std::list<int> l1;
+    std::list<int> l2(a2, a2 + 3);
+    l1.splice(l1.end(), l2, l2.begin());
+    assert(l1.size() == 1);
+    assert(std::distance(l1.begin(), l1.end()) == 1);
+    assert(l2.size() == 2);
+    assert(std::distance(l2.begin(), l2.end()) == 2);
+    std::list<int>::const_iterator i = l1.begin();
+    assert(*i == 4);
+    i = l2.begin();
+    assert(*i == 5);
+    ++i;
+    assert(*i == 6);
+  }
+  {
+    std::list<int> l1;
+    std::list<int> l2(a2, a2 + 3);
+    l1.splice(l1.end(), l2, std::next(l2.begin()));
+    assert(l1.size() == 1);
+    assert(std::distance(l1.begin(), l1.end()) == 1);
+    assert(l2.size() == 2);
+    assert(std::distance(l2.begin(), l2.end()) == 2);
+    std::list<int>::const_iterator i = l1.begin();
+    assert(*i == 5);
+    i = l2.begin();
+    assert(*i == 4);
+    ++i;
+    assert(*i == 6);
+  }
+  {
+    std::list<int> l1;
+    std::list<int> l2(a2, a2 + 3);
+    l1.splice(l1.end(), l2, std::next(l2.begin(), 2));
+    assert(l1.size() == 1);
+    assert(std::distance(l1.begin(), l1.end()) == 1);
+    assert(l2.size() == 2);
+    assert(std::distance(l2.begin(), l2.end()) == 2);
+    std::list<int>::const_iterator i = l1.begin();
+    assert(*i == 6);
+    i = l2.begin();
+    assert(*i == 4);
+    ++i;
+    assert(*i == 5);
+  }
+  {
+    std::list<int> l1(a1, a1 + 1);
+    l1.splice(l1.begin(), l1, l1.begin());
+    assert(l1.size() == 1);
+    assert(std::distance(l1.begin(), l1.end()) == 1);
+    std::list<int>::const_iterator i = l1.begin();
+    assert(*i == 1);
+  }
+  {
+    std::list<int> l1(a1, a1 + 1);
+    std::list<int> l2(a2, a2 + 1);
+    l1.splice(l1.begin(), l2, l2.begin());
+    assert(l1.size() == 2);
+    assert(std::distance(l1.begin(), l1.end()) == 2);
+    assert(l2.size() == 0);
+    assert(std::distance(l2.begin(), l2.end()) == 0);
+    std::list<int>::const_iterator i = l1.begin();
+    assert(*i == 4);
+    ++i;
+    assert(*i == 1);
+  }
+  {
+    std::list<int> l1(a1, a1 + 1);
+    std::list<int> l2(a2, a2 + 1);
+    l1.splice(std::next(l1.begin()), l2, l2.begin());
+    assert(l1.size() == 2);
+    assert(std::distance(l1.begin(), l1.end()) == 2);
+    assert(l2.size() == 0);
+    assert(std::distance(l2.begin(), l2.end()) == 0);
+    std::list<int>::const_iterator i = l1.begin();
+    assert(*i == 1);
+    ++i;
+    assert(*i == 4);
+  }
+  {
+    std::list<int> l1(a1, a1 + 2);
+    l1.splice(l1.begin(), l1, l1.begin());
+    assert(l1.size() == 2);
+    assert(std::distance(l1.begin(), l1.end()) == 2);
+    std::list<int>::const_iterator i = l1.begin();
+    assert(*i == 1);
+    ++i;
+    assert(*i == 2);
+  }
+  {
+    std::list<int> l1(a1, a1 + 2);
+    l1.splice(l1.begin(), l1, std::next(l1.begin()));
+    assert(l1.size() == 2);
+    assert(std::distance(l1.begin(), l1.end()) == 2);
+    std::list<int>::const_iterator i = l1.begin();
+    assert(*i == 2);
+    ++i;
+    assert(*i == 1);
+  }
+  {
+    std::list<int> l1(a1, a1 + 2);
+    l1.splice(std::next(l1.begin()), l1, l1.begin());
+    assert(l1.size() == 2);
+    assert(std::distance(l1.begin(), l1.end()) == 2);
+    std::list<int>::const_iterator i = l1.begin();
+    assert(*i == 1);
+    ++i;
+    assert(*i == 2);
+  }
+  {
+    std::list<int> l1(a1, a1 + 2);
+    l1.splice(std::next(l1.begin()), l1, std::next(l1.begin()));
+    assert(l1.size() == 2);
+    assert(std::distance(l1.begin(), l1.end()) == 2);
+    std::list<int>::const_iterator i = l1.begin();
+    assert(*i == 1);
+    ++i;
+    assert(*i == 2);
+  }
 #if TEST_STD_VER >= 11
-    {
-        std::list<int, min_allocator<int>> l1;
-        std::list<int, min_allocator<int>> l2(a2, a2+1);
-        l1.splice(l1.end(), l2, l2.begin());
-        assert(l1.size() == 1);
-        assert(std::distance(l1.begin(), l1.end()) == 1);
-        assert(l2.size() == 0);
-        assert(std::distance(l2.begin(), l2.end()) == 0);
-        std::list<int, min_allocator<int>>::const_iterator i = l1.begin();
-        assert(*i == 4);
-    }
-    {
-        std::list<int, min_allocator<int>> l1;
-        std::list<int, min_allocator<int>> l2(a2, a2+2);
-        l1.splice(l1.end(), l2, l2.begin());
-        assert(l1.size() == 1);
-        assert(std::distance(l1.begin(), l1.end()) == 1);
-        assert(l2.size() == 1);
-        assert(std::distance(l2.begin(), l2.end()) == 1);
-        std::list<int, min_allocator<int>>::const_iterator i = l1.begin();
-        assert(*i == 4);
-        i = l2.begin();
-        assert(*i == 5);
-    }
-    {
-        std::list<int, min_allocator<int>> l1;
-        std::list<int, min_allocator<int>> l2(a2, a2+2);
-        l1.splice(l1.end(), l2, std::next(l2.begin()));
-        assert(l1.size() == 1);
-        assert(std::distance(l1.begin(), l1.end()) == 1);
-        assert(l2.size() == 1);
-        assert(std::distance(l2.begin(), l2.end()) == 1);
-        std::list<int, min_allocator<int>>::const_iterator i = l1.begin();
-        assert(*i == 5);
-        i = l2.begin();
-        assert(*i == 4);
-    }
-    {
-        std::list<int, min_allocator<int>> l1;
-        std::list<int, min_allocator<int>> l2(a2, a2+3);
-        l1.splice(l1.end(), l2, l2.begin());
-        assert(l1.size() == 1);
-        assert(std::distance(l1.begin(), l1.end()) == 1);
-        assert(l2.size() == 2);
-        assert(std::distance(l2.begin(), l2.end()) == 2);
-        std::list<int, min_allocator<int>>::const_iterator i = l1.begin();
-        assert(*i == 4);
-        i = l2.begin();
-        assert(*i == 5);
-        ++i;
-        assert(*i == 6);
-    }
-    {
-        std::list<int, min_allocator<int>> l1;
-        std::list<int, min_allocator<int>> l2(a2, a2+3);
-        l1.splice(l1.end(), l2, std::next(l2.begin()));
-        assert(l1.size() == 1);
-        assert(std::distance(l1.begin(), l1.end()) == 1);
-        assert(l2.size() == 2);
-        assert(std::distance(l2.begin(), l2.end()) == 2);
-        std::list<int, min_allocator<int>>::const_iterator i = l1.begin();
-        assert(*i == 5);
-        i = l2.begin();
-        assert(*i == 4);
-        ++i;
-        assert(*i == 6);
-    }
-    {
-        std::list<int, min_allocator<int>> l1;
-        std::list<int, min_allocator<int>> l2(a2, a2+3);
-        l1.splice(l1.end(), l2, std::next(l2.begin(), 2));
-        assert(l1.size() == 1);
-        assert(std::distance(l1.begin(), l1.end()) == 1);
-        assert(l2.size() == 2);
-        assert(std::distance(l2.begin(), l2.end()) == 2);
-        std::list<int, min_allocator<int>>::const_iterator i = l1.begin();
-        assert(*i == 6);
-        i = l2.begin();
-        assert(*i == 4);
-        ++i;
-        assert(*i == 5);
-    }
-    {
-        std::list<int, min_allocator<int>> l1(a1, a1+1);
-        l1.splice(l1.begin(), l1, l1.begin());
-        assert(l1.size() == 1);
-        assert(std::distance(l1.begin(), l1.end()) == 1);
-        std::list<int, min_allocator<int>>::const_iterator i = l1.begin();
-        assert(*i == 1);
-    }
-    {
-        std::list<int, min_allocator<int>> l1(a1, a1+1);
-        std::list<int, min_allocator<int>> l2(a2, a2+1);
-        l1.splice(l1.begin(), l2, l2.begin());
-        assert(l1.size() == 2);
-        assert(std::distance(l1.begin(), l1.end()) == 2);
-        assert(l2.size() == 0);
-        assert(std::distance(l2.begin(), l2.end()) == 0);
-        std::list<int, min_allocator<int>>::const_iterator i = l1.begin();
-        assert(*i == 4);
-        ++i;
-        assert(*i == 1);
-    }
-    {
-        std::list<int, min_allocator<int>> l1(a1, a1+1);
-        std::list<int, min_allocator<int>> l2(a2, a2+1);
-        l1.splice(std::next(l1.begin()), l2, l2.begin());
-        assert(l1.size() == 2);
-        assert(std::distance(l1.begin(), l1.end()) == 2);
-        assert(l2.size() == 0);
-        assert(std::distance(l2.begin(), l2.end()) == 0);
-        std::list<int, min_allocator<int>>::const_iterator i = l1.begin();
-        assert(*i == 1);
-        ++i;
-        assert(*i == 4);
-    }
-    {
-        std::list<int, min_allocator<int>> l1(a1, a1+2);
-        l1.splice(l1.begin(), l1, l1.begin());
-        assert(l1.size() == 2);
-        assert(std::distance(l1.begin(), l1.end()) == 2);
-        std::list<int, min_allocator<int>>::const_iterator i = l1.begin();
-        assert(*i == 1);
-        ++i;
-        assert(*i == 2);
-    }
-    {
-        std::list<int, min_allocator<int>> l1(a1, a1+2);
-        l1.splice(l1.begin(), l1, std::next(l1.begin()));
-        assert(l1.size() == 2);
-        assert(std::distance(l1.begin(), l1.end()) == 2);
-        std::list<int, min_allocator<int>>::const_iterator i = l1.begin();
-        assert(*i == 2);
-        ++i;
-        assert(*i == 1);
-    }
-    {
-        std::list<int, min_allocator<int>> l1(a1, a1+2);
-        l1.splice(std::next(l1.begin()), l1, l1.begin());
-        assert(l1.size() == 2);
-        assert(std::distance(l1.begin(), l1.end()) == 2);
-        std::list<int, min_allocator<int>>::const_iterator i = l1.begin();
-        assert(*i == 1);
-        ++i;
-        assert(*i == 2);
-    }
-    {
-        std::list<int, min_allocator<int>> l1(a1, a1+2);
-        l1.splice(std::next(l1.begin()), l1, std::next(l1.begin()));
-        assert(l1.size() == 2);
-        assert(std::distance(l1.begin(), l1.end()) == 2);
-        std::list<int, min_allocator<int>>::const_iterator i = l1.begin();
-        assert(*i == 1);
-        ++i;
-        assert(*i == 2);
-    }
+  {
+    std::list<int, min_allocator<int>> l1;
+    std::list<int, min_allocator<int>> l2(a2, a2 + 1);
+    l1.splice(l1.end(), l2, l2.begin());
+    assert(l1.size() == 1);
+    assert(std::distance(l1.begin(), l1.end()) == 1);
+    assert(l2.size() == 0);
+    assert(std::distance(l2.begin(), l2.end()) == 0);
+    std::list<int, min_allocator<int>>::const_iterator i = l1.begin();
+    assert(*i == 4);
+  }
+  {
+    std::list<int, min_allocator<int>> l1;
+    std::list<int, min_allocator<int>> l2(a2, a2 + 2);
+    l1.splice(l1.end(), l2, l2.begin());
+    assert(l1.size() == 1);
+    assert(std::distance(l1.begin(), l1.end()) == 1);
+    assert(l2.size() == 1);
+    assert(std::distance(l2.begin(), l2.end()) == 1);
+    std::list<int, min_allocator<int>>::const_iterator i = l1.begin();
+    assert(*i == 4);
+    i = l2.begin();
+    assert(*i == 5);
+  }
+  {
+    std::list<int, min_allocator<int>> l1;
+    std::list<int, min_allocator<int>> l2(a2, a2 + 2);
+    l1.splice(l1.end(), l2, std::next(l2.begin()));
+    assert(l1.size() == 1);
+    assert(std::distance(l1.begin(), l1.end()) == 1);
+    assert(l2.size() == 1);
+    assert(std::distance(l2.begin(), l2.end()) == 1);
+    std::list<int, min_allocator<int>>::const_iterator i = l1.begin();
+    assert(*i == 5);
+    i = l2.begin();
+    assert(*i == 4);
+  }
+  {
+    std::list<int, min_allocator<int>> l1;
+    std::list<int, min_allocator<int>> l2(a2, a2 + 3);
+    l1.splice(l1.end(), l2, l2.begin());
+    assert(l1.size() == 1);
+    assert(std::distance(l1.begin(), l1.end()) == 1);
+    assert(l2.size() == 2);
+    assert(std::distance(l2.begin(), l2.end()) == 2);
+    std::list<int, min_allocator<int>>::const_iterator i = l1.begin();
+    assert(*i == 4);
+    i = l2.begin();
+    assert(*i == 5);
+    ++i;
+    assert(*i == 6);
+  }
+  {
+    std::list<int, min_allocator<int>> l1;
+    std::list<int, min_allocator<int>> l2(a2, a2 + 3);
+    l1.splice(l1.end(), l2, std::next(l2.begin()));
+    assert(l1.size() == 1);
+    assert(std::distance(l1.begin(), l1.end()) == 1);
+    assert(l2.size() == 2);
+    assert(std::distance(l2.begin(), l2.end()) == 2);
+    std::list<int, min_allocator<int>>::const_iterator i = l1.begin();
+    assert(*i == 5);
+    i = l2.begin();
+    assert(*i == 4);
+    ++i;
+    assert(*i == 6);
+  }
+  {
+    std::list<int, min_allocator<int>> l1;
+    std::list<int, min_allocator<int>> l2(a2, a2 + 3);
+    l1.splice(l1.end(), l2, std::next(l2.begin(), 2));
+    assert(l1.size() == 1);
+    assert(std::distance(l1.begin(), l1.end()) == 1);
+    assert(l2.size() == 2);
+    assert(std::distance(l2.begin(), l2.end()) == 2);
+    std::list<int, min_allocator<int>>::const_iterator i = l1.begin();
+    assert(*i == 6);
+    i = l2.begin();
+    assert(*i == 4);
+    ++i;
+    assert(*i == 5);
+  }
+  {
+    std::list<int, min_allocator<int>> l1(a1, a1 + 1);
+    l1.splice(l1.begin(), l1, l1.begin());
+    assert(l1.size() == 1);
+    assert(std::distance(l1.begin(), l1.end()) == 1);
+    std::list<int, min_allocator<int>>::const_iterator i = l1.begin();
+    assert(*i == 1);
+  }
+  {
+    std::list<int, min_allocator<int>> l1(a1, a1 + 1);
+    std::list<int, min_allocator<int>> l2(a2, a2 + 1);
+    l1.splice(l1.begin(), l2, l2.begin());
+    assert(l1.size() == 2);
+    assert(std::distance(l1.begin(), l1.end()) == 2);
+    assert(l2.size() == 0);
+    assert(std::distance(l2.begin(), l2.end()) == 0);
+    std::list<int, min_allocator<int>>::const_iterator i = l1.begin();
+    assert(*i == 4);
+    ++i;
+    assert(*i == 1);
+  }
+  {
+    std::list<int, min_allocator<int>> l1(a1, a1 + 1);
+    std::list<int, min_allocator<int>> l2(a2, a2 + 1);
+    l1.splice(std::next(l1.begin()), l2, l2.begin());
+    assert(l1.size() == 2);
+    assert(std::distance(l1.begin(), l1.end()) == 2);
+    assert(l2.size() == 0);
+    assert(std::distance(l2.begin(), l2.end()) == 0);
+    std::list<int, min_allocator<int>>::const_iterator i = l1.begin();
+    assert(*i == 1);
+    ++i;
+    assert(*i == 4);
+  }
+  {
+    std::list<int, min_allocator<int>> l1(a1, a1 + 2);
+    l1.splice(l1.begin(), l1, l1.begin());
+    assert(l1.size() == 2);
+    assert(std::distance(l1.begin(), l1.end()) == 2);
+    std::list<int, min_allocator<int>>::const_iterator i = l1.begin();
+    assert(*i == 1);
+    ++i;
+    assert(*i == 2);
+  }
+  {
+    std::list<int, min_allocator<int>> l1(a1, a1 + 2);
+    l1.splice(l1.begin(), l1, std::next(l1.begin()));
+    assert(l1.size() == 2);
+    assert(std::distance(l1.begin(), l1.end()) == 2);
+    std::list<int, min_allocator<int>>::const_iterator i = l1.begin();
+    assert(*i == 2);
+    ++i;
+    assert(*i == 1);
+  }
+  {
+    std::list<int, min_allocator<int>> l1(a1, a1 + 2);
+    l1.splice(std::next(l1.begin()), l1, l1.begin());
+    assert(l1.size() == 2);
+    assert(std::distance(l1.begin(), l1.end()) == 2);
+    std::list<int, min_allocator<int>>::const_iterator i = l1.begin();
+    assert(*i == 1);
+    ++i;
+    assert(*i == 2);
+  }
+  {
+    std::list<int, min_allocator<int>> l1(a1, a1 + 2);
+    l1.splice(std::next(l1.begin()), l1, std::next(l1.begin()));
+    assert(l1.size() == 2);
+    assert(std::distance(l1.begin(), l1.end()) == 2);
+    std::list<int, min_allocator<int>>::const_iterator i = l1.begin();
+    assert(*i == 1);
+    ++i;
+    assert(*i == 2);
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/sequences/list/list.ops/splice_pos_list_iter_iter.pass.cpp b/libcxx/test/std/containers/sequences/list/list.ops/splice_pos_list_iter_iter.pass.cpp
index ebfdd50a2b545..b77b6a26440d5 100644
--- a/libcxx/test/std/containers/sequences/list/list.ops/splice_pos_list_iter_iter.pass.cpp
+++ b/libcxx/test/std/containers/sequences/list/list.ops/splice_pos_list_iter_iter.pass.cpp
@@ -16,203 +16,202 @@
 #include "test_macros.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    int a1[] = {1, 2, 3};
-    int a2[] = {4, 5, 6};
-    {
-        std::list<int> l1(a1, a1+3);
-        l1.splice(l1.begin(), l1, std::next(l1.begin()), std::next(l1.begin()));
-        assert(l1.size() == 3);
-        assert(std::distance(l1.begin(), l1.end()) == 3);
-        std::list<int>::const_iterator i = l1.begin();
-        assert(*i == 1);
-        ++i;
-        assert(*i == 2);
-        ++i;
-        assert(*i == 3);
-    }
-    {
-        std::list<int> l1(a1, a1+3);
-        l1.splice(l1.begin(), l1, std::next(l1.begin()), std::next(l1.begin(), 2));
-        assert(l1.size() == 3);
-        assert(std::distance(l1.begin(), l1.end()) == 3);
-        std::list<int>::const_iterator i = l1.begin();
-        assert(*i == 2);
-        ++i;
-        assert(*i == 1);
-        ++i;
-        assert(*i == 3);
-    }
-    {
-        std::list<int> l1(a1, a1+3);
-        l1.splice(l1.begin(), l1, std::next(l1.begin()), std::next(l1.begin(), 3));
-        assert(l1.size() == 3);
-        assert(std::distance(l1.begin(), l1.end()) == 3);
-        std::list<int>::const_iterator i = l1.begin();
-        assert(*i == 2);
-        ++i;
-        assert(*i == 3);
-        ++i;
-        assert(*i == 1);
-    }
-    {
-        std::list<int> l1(a1, a1+3);
-        std::list<int> l2(a2, a2+3);
-        l1.splice(l1.begin(), l2, std::next(l2.begin()), l2.end());
-        assert(l1.size() == 5);
-        assert(std::distance(l1.begin(), l1.end()) == 5);
-        std::list<int>::const_iterator i = l1.begin();
-        assert(*i == 5);
-        ++i;
-        assert(*i == 6);
-        ++i;
-        assert(*i == 1);
-        ++i;
-        assert(*i == 2);
-        ++i;
-        assert(*i == 3);
-        assert(l2.size() == 1);
-        i = l2.begin();
-        assert(*i == 4);
-    }
-    {
-        std::list<int> l1(a1, a1+3);
-        std::list<int> l2(a2, a2+3);
-        l1.splice(std::next(l1.begin()), l2, std::next(l2.begin()), l2.end());
-        assert(l1.size() == 5);
-        assert(std::distance(l1.begin(), l1.end()) == 5);
-        std::list<int>::const_iterator i = l1.begin();
-        assert(*i == 1);
-        ++i;
-        assert(*i == 5);
-        ++i;
-        assert(*i == 6);
-        ++i;
-        assert(*i == 2);
-        ++i;
-        assert(*i == 3);
-        assert(l2.size() == 1);
-        i = l2.begin();
-        assert(*i == 4);
-    }
-    {
-        std::list<int> l1(a1, a1+3);
-        std::list<int> l2(a2, a2+3);
-        l1.splice(l1.end(), l2, std::next(l2.begin()), l2.end());
-        assert(l1.size() == 5);
-        assert(std::distance(l1.begin(), l1.end()) == 5);
-        std::list<int>::const_iterator i = l1.begin();
-        assert(*i == 1);
-        ++i;
-        assert(*i == 2);
-        ++i;
-        assert(*i == 3);
-        ++i;
-        assert(*i == 5);
-        ++i;
-        assert(*i == 6);
-        assert(l2.size() == 1);
-        i = l2.begin();
-        assert(*i == 4);
-    }
+int main(int, char**) {
+  int a1[] = {1, 2, 3};
+  int a2[] = {4, 5, 6};
+  {
+    std::list<int> l1(a1, a1 + 3);
+    l1.splice(l1.begin(), l1, std::next(l1.begin()), std::next(l1.begin()));
+    assert(l1.size() == 3);
+    assert(std::distance(l1.begin(), l1.end()) == 3);
+    std::list<int>::const_iterator i = l1.begin();
+    assert(*i == 1);
+    ++i;
+    assert(*i == 2);
+    ++i;
+    assert(*i == 3);
+  }
+  {
+    std::list<int> l1(a1, a1 + 3);
+    l1.splice(l1.begin(), l1, std::next(l1.begin()), std::next(l1.begin(), 2));
+    assert(l1.size() == 3);
+    assert(std::distance(l1.begin(), l1.end()) == 3);
+    std::list<int>::const_iterator i = l1.begin();
+    assert(*i == 2);
+    ++i;
+    assert(*i == 1);
+    ++i;
+    assert(*i == 3);
+  }
+  {
+    std::list<int> l1(a1, a1 + 3);
+    l1.splice(l1.begin(), l1, std::next(l1.begin()), std::next(l1.begin(), 3));
+    assert(l1.size() == 3);
+    assert(std::distance(l1.begin(), l1.end()) == 3);
+    std::list<int>::const_iterator i = l1.begin();
+    assert(*i == 2);
+    ++i;
+    assert(*i == 3);
+    ++i;
+    assert(*i == 1);
+  }
+  {
+    std::list<int> l1(a1, a1 + 3);
+    std::list<int> l2(a2, a2 + 3);
+    l1.splice(l1.begin(), l2, std::next(l2.begin()), l2.end());
+    assert(l1.size() == 5);
+    assert(std::distance(l1.begin(), l1.end()) == 5);
+    std::list<int>::const_iterator i = l1.begin();
+    assert(*i == 5);
+    ++i;
+    assert(*i == 6);
+    ++i;
+    assert(*i == 1);
+    ++i;
+    assert(*i == 2);
+    ++i;
+    assert(*i == 3);
+    assert(l2.size() == 1);
+    i = l2.begin();
+    assert(*i == 4);
+  }
+  {
+    std::list<int> l1(a1, a1 + 3);
+    std::list<int> l2(a2, a2 + 3);
+    l1.splice(std::next(l1.begin()), l2, std::next(l2.begin()), l2.end());
+    assert(l1.size() == 5);
+    assert(std::distance(l1.begin(), l1.end()) == 5);
+    std::list<int>::const_iterator i = l1.begin();
+    assert(*i == 1);
+    ++i;
+    assert(*i == 5);
+    ++i;
+    assert(*i == 6);
+    ++i;
+    assert(*i == 2);
+    ++i;
+    assert(*i == 3);
+    assert(l2.size() == 1);
+    i = l2.begin();
+    assert(*i == 4);
+  }
+  {
+    std::list<int> l1(a1, a1 + 3);
+    std::list<int> l2(a2, a2 + 3);
+    l1.splice(l1.end(), l2, std::next(l2.begin()), l2.end());
+    assert(l1.size() == 5);
+    assert(std::distance(l1.begin(), l1.end()) == 5);
+    std::list<int>::const_iterator i = l1.begin();
+    assert(*i == 1);
+    ++i;
+    assert(*i == 2);
+    ++i;
+    assert(*i == 3);
+    ++i;
+    assert(*i == 5);
+    ++i;
+    assert(*i == 6);
+    assert(l2.size() == 1);
+    i = l2.begin();
+    assert(*i == 4);
+  }
 #if TEST_STD_VER >= 11
-    {
-        std::list<int, min_allocator<int>> l1(a1, a1+3);
-        l1.splice(l1.begin(), l1, std::next(l1.begin()), std::next(l1.begin()));
-        assert(l1.size() == 3);
-        assert(std::distance(l1.begin(), l1.end()) == 3);
-        std::list<int, min_allocator<int>>::const_iterator i = l1.begin();
-        assert(*i == 1);
-        ++i;
-        assert(*i == 2);
-        ++i;
-        assert(*i == 3);
-    }
-    {
-        std::list<int, min_allocator<int>> l1(a1, a1+3);
-        l1.splice(l1.begin(), l1, std::next(l1.begin()), std::next(l1.begin(), 2));
-        assert(l1.size() == 3);
-        assert(std::distance(l1.begin(), l1.end()) == 3);
-        std::list<int, min_allocator<int>>::const_iterator i = l1.begin();
-        assert(*i == 2);
-        ++i;
-        assert(*i == 1);
-        ++i;
-        assert(*i == 3);
-    }
-    {
-        std::list<int, min_allocator<int>> l1(a1, a1+3);
-        l1.splice(l1.begin(), l1, std::next(l1.begin()), std::next(l1.begin(), 3));
-        assert(l1.size() == 3);
-        assert(std::distance(l1.begin(), l1.end()) == 3);
-        std::list<int, min_allocator<int>>::const_iterator i = l1.begin();
-        assert(*i == 2);
-        ++i;
-        assert(*i == 3);
-        ++i;
-        assert(*i == 1);
-    }
-    {
-        std::list<int, min_allocator<int>> l1(a1, a1+3);
-        std::list<int, min_allocator<int>> l2(a2, a2+3);
-        l1.splice(l1.begin(), l2, std::next(l2.begin()), l2.end());
-        assert(l1.size() == 5);
-        assert(std::distance(l1.begin(), l1.end()) == 5);
-        std::list<int, min_allocator<int>>::const_iterator i = l1.begin();
-        assert(*i == 5);
-        ++i;
-        assert(*i == 6);
-        ++i;
-        assert(*i == 1);
-        ++i;
-        assert(*i == 2);
-        ++i;
-        assert(*i == 3);
-        assert(l2.size() == 1);
-        i = l2.begin();
-        assert(*i == 4);
-    }
-    {
-        std::list<int, min_allocator<int>> l1(a1, a1+3);
-        std::list<int, min_allocator<int>> l2(a2, a2+3);
-        l1.splice(std::next(l1.begin()), l2, std::next(l2.begin()), l2.end());
-        assert(l1.size() == 5);
-        assert(std::distance(l1.begin(), l1.end()) == 5);
-        std::list<int, min_allocator<int>>::const_iterator i = l1.begin();
-        assert(*i == 1);
-        ++i;
-        assert(*i == 5);
-        ++i;
-        assert(*i == 6);
-        ++i;
-        assert(*i == 2);
-        ++i;
-        assert(*i == 3);
-        assert(l2.size() == 1);
-        i = l2.begin();
-        assert(*i == 4);
-    }
-    {
-        std::list<int, min_allocator<int>> l1(a1, a1+3);
-        std::list<int, min_allocator<int>> l2(a2, a2+3);
-        l1.splice(l1.end(), l2, std::next(l2.begin()), l2.end());
-        assert(l1.size() == 5);
-        assert(std::distance(l1.begin(), l1.end()) == 5);
-        std::list<int, min_allocator<int>>::const_iterator i = l1.begin();
-        assert(*i == 1);
-        ++i;
-        assert(*i == 2);
-        ++i;
-        assert(*i == 3);
-        ++i;
-        assert(*i == 5);
-        ++i;
-        assert(*i == 6);
-        assert(l2.size() == 1);
-        i = l2.begin();
-        assert(*i == 4);
-    }
+  {
+    std::list<int, min_allocator<int>> l1(a1, a1 + 3);
+    l1.splice(l1.begin(), l1, std::next(l1.begin()), std::next(l1.begin()));
+    assert(l1.size() == 3);
+    assert(std::distance(l1.begin(), l1.end()) == 3);
+    std::list<int, min_allocator<int>>::const_iterator i = l1.begin();
+    assert(*i == 1);
+    ++i;
+    assert(*i == 2);
+    ++i;
+    assert(*i == 3);
+  }
+  {
+    std::list<int, min_allocator<int>> l1(a1, a1 + 3);
+    l1.splice(l1.begin(), l1, std::next(l1.begin()), std::next(l1.begin(), 2));
+    assert(l1.size() == 3);
+    assert(std::distance(l1.begin(), l1.end()) == 3);
+    std::list<int, min_allocator<int>>::const_iterator i = l1.begin();
+    assert(*i == 2);
+    ++i;
+    assert(*i == 1);
+    ++i;
+    assert(*i == 3);
+  }
+  {
+    std::list<int, min_allocator<int>> l1(a1, a1 + 3);
+    l1.splice(l1.begin(), l1, std::next(l1.begin()), std::next(l1.begin(), 3));
+    assert(l1.size() == 3);
+    assert(std::distance(l1.begin(), l1.end()) == 3);
+    std::list<int, min_allocator<int>>::const_iterator i = l1.begin();
+    assert(*i == 2);
+    ++i;
+    assert(*i == 3);
+    ++i;
+    assert(*i == 1);
+  }
+  {
+    std::list<int, min_allocator<int>> l1(a1, a1 + 3);
+    std::list<int, min_allocator<int>> l2(a2, a2 + 3);
+    l1.splice(l1.begin(), l2, std::next(l2.begin()), l2.end());
+    assert(l1.size() == 5);
+    assert(std::distance(l1.begin(), l1.end()) == 5);
+    std::list<int, min_allocator<int>>::const_iterator i = l1.begin();
+    assert(*i == 5);
+    ++i;
+    assert(*i == 6);
+    ++i;
+    assert(*i == 1);
+    ++i;
+    assert(*i == 2);
+    ++i;
+    assert(*i == 3);
+    assert(l2.size() == 1);
+    i = l2.begin();
+    assert(*i == 4);
+  }
+  {
+    std::list<int, min_allocator<int>> l1(a1, a1 + 3);
+    std::list<int, min_allocator<int>> l2(a2, a2 + 3);
+    l1.splice(std::next(l1.begin()), l2, std::next(l2.begin()), l2.end());
+    assert(l1.size() == 5);
+    assert(std::distance(l1.begin(), l1.end()) == 5);
+    std::list<int, min_allocator<int>>::const_iterator i = l1.begin();
+    assert(*i == 1);
+    ++i;
+    assert(*i == 5);
+    ++i;
+    assert(*i == 6);
+    ++i;
+    assert(*i == 2);
+    ++i;
+    assert(*i == 3);
+    assert(l2.size() == 1);
+    i = l2.begin();
+    assert(*i == 4);
+  }
+  {
+    std::list<int, min_allocator<int>> l1(a1, a1 + 3);
+    std::list<int, min_allocator<int>> l2(a2, a2 + 3);
+    l1.splice(l1.end(), l2, std::next(l2.begin()), l2.end());
+    assert(l1.size() == 5);
+    assert(std::distance(l1.begin(), l1.end()) == 5);
+    std::list<int, min_allocator<int>>::const_iterator i = l1.begin();
+    assert(*i == 1);
+    ++i;
+    assert(*i == 2);
+    ++i;
+    assert(*i == 3);
+    ++i;
+    assert(*i == 5);
+    ++i;
+    assert(*i == 6);
+    assert(l2.size() == 1);
+    i = l2.begin();
+    assert(*i == 4);
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/sequences/list/list.ops/unique.pass.cpp b/libcxx/test/std/containers/sequences/list/list.ops/unique.pass.cpp
index f78f6f152ae91..c08e348218f95 100644
--- a/libcxx/test/std/containers/sequences/list/list.ops/unique.pass.cpp
+++ b/libcxx/test/std/containers/sequences/list/list.ops/unique.pass.cpp
@@ -17,34 +17,33 @@
 #include "test_macros.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
+int main(int, char**) {
+  {
     int a1[] = {2, 1, 1, 4, 4, 4, 4, 3, 3};
     int a2[] = {2, 1, 4, 3};
     typedef std::list<int> L;
-    L c(a1, a1+sizeof(a1)/sizeof(a1[0]));
+    L c(a1, a1 + sizeof(a1) / sizeof(a1[0]));
 #if TEST_STD_VER > 17
     ASSERT_SAME_TYPE(L::size_type, decltype(c.unique()));
     assert(c.unique() == 5);
 #else
-    ASSERT_SAME_TYPE(void,         decltype(c.unique()));
+    ASSERT_SAME_TYPE(void, decltype(c.unique()));
     c.unique();
 #endif
-    assert(c == std::list<int>(a2, a2+4));
-    }
+    assert(c == std::list<int>(a2, a2 + 4));
+  }
 #if TEST_STD_VER >= 11
-    {
+  {
     int a1[] = {2, 1, 1, 4, 4, 4, 4, 3, 3};
     int a2[] = {2, 1, 4, 3};
-    std::list<int, min_allocator<int>> c(a1, a1+sizeof(a1)/sizeof(a1[0]));
-#if TEST_STD_VER > 17
+    std::list<int, min_allocator<int>> c(a1, a1 + sizeof(a1) / sizeof(a1[0]));
+#  if TEST_STD_VER > 17
     assert(c.unique() == 5);
-#else
+#  else
     c.unique();
-#endif
-    assert((c == std::list<int, min_allocator<int>>(a2, a2+4)));
-    }
+#  endif
+    assert((c == std::list<int, min_allocator<int>>(a2, a2 + 4)));
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/sequences/list/list.ops/unique_pred.pass.cpp b/libcxx/test/std/containers/sequences/list/list.ops/unique_pred.pass.cpp
index 316b7c1b44d8d..1d3a8e0c426aa 100644
--- a/libcxx/test/std/containers/sequences/list/list.ops/unique_pred.pass.cpp
+++ b/libcxx/test/std/containers/sequences/list/list.ops/unique_pred.pass.cpp
@@ -18,10 +18,7 @@
 #include "test_macros.h"
 #include "min_allocator.h"
 
-bool g(int x, int y)
-{
-    return x == y;
-}
+bool g(int x, int y) { return x == y; }
 
 struct PredLWG526 {
   PredLWG526(int i) : i_(i) {}
@@ -32,26 +29,25 @@ struct PredLWG526 {
   int i_;
 };
 
-int main(int, char**)
-{
-    {
+int main(int, char**) {
+  {
     int a1[] = {2, 1, 1, 4, 4, 4, 4, 3, 3};
     int a2[] = {2, 1, 4, 3};
     typedef std::list<int> L;
-    L c(a1, a1+sizeof(a1)/sizeof(a1[0]));
+    L c(a1, a1 + sizeof(a1) / sizeof(a1[0]));
 #if TEST_STD_VER > 17
     ASSERT_SAME_TYPE(L::size_type, decltype(c.unique(g)));
     assert(c.unique(g) == 5);
 #else
-    ASSERT_SAME_TYPE(void,         decltype(c.unique(g)));
+    ASSERT_SAME_TYPE(void, decltype(c.unique(g)));
     c.unique(g);
 #endif
-    assert(c == std::list<int>(a2, a2+4));
-    }
+    assert(c == std::list<int>(a2, a2 + 4));
+  }
 
-    { // LWG issue #526
+  { // LWG issue #526
     int a1[] = {1, 1, 1, 2, 3, 5, 5, 2, 11};
-    int a2[] = {1,       2, 3, 5,    2, 11};
+    int a2[] = {1, 2, 3, 5, 2, 11};
     std::list<PredLWG526> c(a1, a1 + 9);
 #if TEST_STD_VER > 17
     assert(c.unique(std::ref(c.front())) == 3);
@@ -59,25 +55,24 @@ int main(int, char**)
     c.unique(std::ref(c.front()));
 #endif
     assert(c.size() == 6);
-    for (std::size_t i = 0; i < c.size(); ++i)
-    {
-        assert(c.front() == a2[i]);
-        c.pop_front();
-    }
+    for (std::size_t i = 0; i < c.size(); ++i) {
+      assert(c.front() == a2[i]);
+      c.pop_front();
     }
+  }
 
 #if TEST_STD_VER >= 11
-    {
+  {
     int a1[] = {2, 1, 1, 4, 4, 4, 4, 3, 3};
     int a2[] = {2, 1, 4, 3};
-    std::list<int, min_allocator<int>> c(a1, a1+sizeof(a1)/sizeof(a1[0]));
-#if TEST_STD_VER > 17
+    std::list<int, min_allocator<int>> c(a1, a1 + sizeof(a1) / sizeof(a1[0]));
+#  if TEST_STD_VER > 17
     assert(c.unique(g) == 5);
-#else
+#  else
     c.unique(g);
-#endif
-    assert((c == std::list<int, min_allocator<int>>(a2, a2+4)));
-    }
+#  endif
+    assert((c == std::list<int, min_allocator<int>>(a2, a2 + 4)));
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/sequences/list/list.special/swap.pass.cpp b/libcxx/test/std/containers/sequences/list/list.special/swap.pass.cpp
index 6e6eb21246eeb..1e9c71131d80a 100644
--- a/libcxx/test/std/containers/sequences/list/list.special/swap.pass.cpp
+++ b/libcxx/test/std/containers/sequences/list/list.special/swap.pass.cpp
@@ -17,122 +17,121 @@
 #include "test_allocator.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        int a1[] = {1, 3, 7, 9, 10};
-        int a2[] = {0, 2, 4, 5, 6, 8, 11};
-        std::list<int> c1(a1, a1 + 5);
-        std::list<int> c2(a2, a2 + 7);
-        std::list<int>::iterator it1 = c1.begin();
-        std::list<int>::const_iterator it2 = c2.begin();
-        swap(c1, c2);
-        assert(c1 == std::list<int>(a2, a2 + 7));
-        assert(c2 == std::list<int>(a1, a1 + 5));
-        assert(it1 == c2.begin()); // Iterators remain valid
-        assert(it2 == c1.begin()); // Iterators remain valid
-    }
-    {
-        int a2[] = {0, 2, 4, 5, 6, 8, 11};
-        std::list<int> c1;
-        std::list<int> c2(a2, a2 + 7);
-        swap(c1, c2);
-        assert(c1 == std::list<int>(a2, a2 + 7));
-        assert(c2.empty());
-        assert(std::distance(c2.begin(), c2.end()) == 0);
-    }
-    {
-        int a1[] = {1, 3, 7, 9, 10};
-        std::list<int> c1(a1, a1 + 5);
-        std::list<int> c2;
-        swap(c1, c2);
-        assert(c1.empty());
-        assert(std::distance(c1.begin(), c1.end()) == 0);
-        assert(c2 == std::list<int>(a1, a1 + 5));
-    }
-    {
-        std::list<int> c1;
-        std::list<int> c2;
-        swap(c1, c2);
-        assert(c1.empty());
-        assert(std::distance(c1.begin(), c1.end()) == 0);
-        assert(c2.empty());
-        assert(std::distance(c2.begin(), c2.end()) == 0);
-    }
-    {
-        int a1[] = {1, 3, 7, 9, 10};
-        int a2[] = {0, 2, 4, 5, 6, 8, 11};
-        typedef test_allocator<int> A;
-        std::list<int, A> c1(a1, a1 + 5, A(1));
-        std::list<int, A> c2(a2, a2 + 7, A(1));
-        swap(c1, c2);
-        assert((c1 == std::list<int, A>(a2, a2 + 7)));
-        assert(c1.get_allocator() == A(1));
-        assert((c2 == std::list<int, A>(a1, a1 + 5)));
-        assert(c2.get_allocator() == A(1));
-    }
-    {
-        int a1[] = {1, 3, 7, 9, 10};
-        int a2[] = {0, 2, 4, 5, 6, 8, 11};
-        typedef other_allocator<int> A;
-        std::list<int, A> c1(a1, a1 + 5, A(1));
-        std::list<int, A> c2(a2, a2 + 7, A(2));
-        swap(c1, c2);
-        assert((c1 == std::list<int, A>(a2, a2 + 7)));
-        assert(c1.get_allocator() == A(2));
-        assert((c2 == std::list<int, A>(a1, a1 + 5)));
-        assert(c2.get_allocator() == A(1));
-    }
+int main(int, char**) {
+  {
+    int a1[] = {1, 3, 7, 9, 10};
+    int a2[] = {0, 2, 4, 5, 6, 8, 11};
+    std::list<int> c1(a1, a1 + 5);
+    std::list<int> c2(a2, a2 + 7);
+    std::list<int>::iterator it1       = c1.begin();
+    std::list<int>::const_iterator it2 = c2.begin();
+    swap(c1, c2);
+    assert(c1 == std::list<int>(a2, a2 + 7));
+    assert(c2 == std::list<int>(a1, a1 + 5));
+    assert(it1 == c2.begin()); // Iterators remain valid
+    assert(it2 == c1.begin()); // Iterators remain valid
+  }
+  {
+    int a2[] = {0, 2, 4, 5, 6, 8, 11};
+    std::list<int> c1;
+    std::list<int> c2(a2, a2 + 7);
+    swap(c1, c2);
+    assert(c1 == std::list<int>(a2, a2 + 7));
+    assert(c2.empty());
+    assert(std::distance(c2.begin(), c2.end()) == 0);
+  }
+  {
+    int a1[] = {1, 3, 7, 9, 10};
+    std::list<int> c1(a1, a1 + 5);
+    std::list<int> c2;
+    swap(c1, c2);
+    assert(c1.empty());
+    assert(std::distance(c1.begin(), c1.end()) == 0);
+    assert(c2 == std::list<int>(a1, a1 + 5));
+  }
+  {
+    std::list<int> c1;
+    std::list<int> c2;
+    swap(c1, c2);
+    assert(c1.empty());
+    assert(std::distance(c1.begin(), c1.end()) == 0);
+    assert(c2.empty());
+    assert(std::distance(c2.begin(), c2.end()) == 0);
+  }
+  {
+    int a1[] = {1, 3, 7, 9, 10};
+    int a2[] = {0, 2, 4, 5, 6, 8, 11};
+    typedef test_allocator<int> A;
+    std::list<int, A> c1(a1, a1 + 5, A(1));
+    std::list<int, A> c2(a2, a2 + 7, A(1));
+    swap(c1, c2);
+    assert((c1 == std::list<int, A>(a2, a2 + 7)));
+    assert(c1.get_allocator() == A(1));
+    assert((c2 == std::list<int, A>(a1, a1 + 5)));
+    assert(c2.get_allocator() == A(1));
+  }
+  {
+    int a1[] = {1, 3, 7, 9, 10};
+    int a2[] = {0, 2, 4, 5, 6, 8, 11};
+    typedef other_allocator<int> A;
+    std::list<int, A> c1(a1, a1 + 5, A(1));
+    std::list<int, A> c2(a2, a2 + 7, A(2));
+    swap(c1, c2);
+    assert((c1 == std::list<int, A>(a2, a2 + 7)));
+    assert(c1.get_allocator() == A(2));
+    assert((c2 == std::list<int, A>(a1, a1 + 5)));
+    assert(c2.get_allocator() == A(1));
+  }
 #if TEST_STD_VER >= 11
-    {
-        int a1[] = {1, 3, 7, 9, 10};
-        int a2[] = {0, 2, 4, 5, 6, 8, 11};
-        std::list<int, min_allocator<int>> c1(a1, a1 + 5);
-        std::list<int, min_allocator<int>> c2(a2, a2 + 7);
-        swap(c1, c2);
-        assert((c1 == std::list<int, min_allocator<int>>(a2, a2 + 7)));
-        assert((c2 == std::list<int, min_allocator<int>>(a1, a1 + 5)));
-    }
-    {
-        int a2[] = {0, 2, 4, 5, 6, 8, 11};
-        std::list<int, min_allocator<int>> c1;
-        std::list<int, min_allocator<int>> c2(a2, a2 + 7);
-        swap(c1, c2);
-        assert((c1 == std::list<int, min_allocator<int>>(a2, a2 + 7)));
-        assert(c2.empty());
-        assert(std::distance(c2.begin(), c2.end()) == 0);
-    }
-    {
-        int a1[] = {1, 3, 7, 9, 10};
-        std::list<int, min_allocator<int>> c1(a1, a1 + 5);
-        std::list<int, min_allocator<int>> c2;
-        swap(c1, c2);
-        assert(c1.empty());
-        assert(std::distance(c1.begin(), c1.end()) == 0);
-        assert((c2 == std::list<int, min_allocator<int>>(a1, a1 + 5)));
-    }
-    {
-        std::list<int, min_allocator<int>> c1;
-        std::list<int, min_allocator<int>> c2;
-        swap(c1, c2);
-        assert(c1.empty());
-        assert(std::distance(c1.begin(), c1.end()) == 0);
-        assert(c2.empty());
-        assert(std::distance(c2.begin(), c2.end()) == 0);
-    }
-    {
-        int a1[] = {1, 3, 7, 9, 10};
-        int a2[] = {0, 2, 4, 5, 6, 8, 11};
-        typedef min_allocator<int> A;
-        std::list<int, A> c1(a1, a1 + 5, A());
-        std::list<int, A> c2(a2, a2 + 7, A());
-        swap(c1, c2);
-        assert((c1 == std::list<int, A>(a2, a2 + 7)));
-        assert(c1.get_allocator() == A());
-        assert((c2 == std::list<int, A>(a1, a1 + 5)));
-        assert(c2.get_allocator() == A());
-    }
+  {
+    int a1[] = {1, 3, 7, 9, 10};
+    int a2[] = {0, 2, 4, 5, 6, 8, 11};
+    std::list<int, min_allocator<int>> c1(a1, a1 + 5);
+    std::list<int, min_allocator<int>> c2(a2, a2 + 7);
+    swap(c1, c2);
+    assert((c1 == std::list<int, min_allocator<int>>(a2, a2 + 7)));
+    assert((c2 == std::list<int, min_allocator<int>>(a1, a1 + 5)));
+  }
+  {
+    int a2[] = {0, 2, 4, 5, 6, 8, 11};
+    std::list<int, min_allocator<int>> c1;
+    std::list<int, min_allocator<int>> c2(a2, a2 + 7);
+    swap(c1, c2);
+    assert((c1 == std::list<int, min_allocator<int>>(a2, a2 + 7)));
+    assert(c2.empty());
+    assert(std::distance(c2.begin(), c2.end()) == 0);
+  }
+  {
+    int a1[] = {1, 3, 7, 9, 10};
+    std::list<int, min_allocator<int>> c1(a1, a1 + 5);
+    std::list<int, min_allocator<int>> c2;
+    swap(c1, c2);
+    assert(c1.empty());
+    assert(std::distance(c1.begin(), c1.end()) == 0);
+    assert((c2 == std::list<int, min_allocator<int>>(a1, a1 + 5)));
+  }
+  {
+    std::list<int, min_allocator<int>> c1;
+    std::list<int, min_allocator<int>> c2;
+    swap(c1, c2);
+    assert(c1.empty());
+    assert(std::distance(c1.begin(), c1.end()) == 0);
+    assert(c2.empty());
+    assert(std::distance(c2.begin(), c2.end()) == 0);
+  }
+  {
+    int a1[] = {1, 3, 7, 9, 10};
+    int a2[] = {0, 2, 4, 5, 6, 8, 11};
+    typedef min_allocator<int> A;
+    std::list<int, A> c1(a1, a1 + 5, A());
+    std::list<int, A> c2(a2, a2 + 7, A());
+    swap(c1, c2);
+    assert((c1 == std::list<int, A>(a2, a2 + 7)));
+    assert(c1.get_allocator() == A());
+    assert((c2 == std::list<int, A>(a1, a1 + 5)));
+    assert(c2.get_allocator() == A());
+  }
 #endif
 
-    return 0;
+  return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/list/list.special/swap_noexcept.pass.cpp b/libcxx/test/std/containers/sequences/list/list.special/swap_noexcept.pass.cpp
index ae76236c5d5cc..a4b1622a04bee 100644
--- a/libcxx/test/std/containers/sequences/list/list.special/swap_noexcept.pass.cpp
+++ b/libcxx/test/std/containers/sequences/list/list.special/swap_noexcept.pass.cpp
@@ -28,65 +28,61 @@
 #include "test_allocator.h"
 
 template <class T>
-struct some_alloc
-{
-    typedef T value_type;
+struct some_alloc {
+  typedef T value_type;
 
-    some_alloc() {}
-    some_alloc(const some_alloc&);
-    void allocate(std::size_t);
-    void deallocate(void*, unsigned) {}
+  some_alloc() {}
+  some_alloc(const some_alloc&);
+  void allocate(std::size_t);
+  void deallocate(void*, unsigned) {}
 
-    typedef std::true_type propagate_on_container_swap;
+  typedef std::true_type propagate_on_container_swap;
 };
 
 template <class T>
-struct some_alloc2
-{
-    typedef T value_type;
+struct some_alloc2 {
+  typedef T value_type;
 
-    some_alloc2() {}
-    some_alloc2(const some_alloc2&);
-    void allocate(std::size_t);
-    void deallocate(void*, unsigned) {}
+  some_alloc2() {}
+  some_alloc2(const some_alloc2&);
+  void allocate(std::size_t);
+  void deallocate(void*, unsigned) {}
 
-    typedef std::false_type propagate_on_container_swap;
-    typedef std::true_type is_always_equal;
+  typedef std::false_type propagate_on_container_swap;
+  typedef std::true_type is_always_equal;
 };
 
-int main(int, char**)
-{
-    {
-        typedef std::list<MoveOnly> C;
-        static_assert(noexcept(swap(std::declval<C&>(), std::declval<C&>())), "");
-    }
+int main(int, char**) {
+  {
+    typedef std::list<MoveOnly> C;
+    static_assert(noexcept(swap(std::declval<C&>(), std::declval<C&>())), "");
+  }
 #if defined(_LIBCPP_VERSION)
-    {
-        typedef std::list<MoveOnly, test_allocator<MoveOnly>> C;
-        static_assert(noexcept(swap(std::declval<C&>(), std::declval<C&>())), "");
-    }
-    {
-        typedef std::list<MoveOnly, other_allocator<MoveOnly>> C;
-        static_assert(noexcept(swap(std::declval<C&>(), std::declval<C&>())), "");
-    }
+  {
+    typedef std::list<MoveOnly, test_allocator<MoveOnly>> C;
+    static_assert(noexcept(swap(std::declval<C&>(), std::declval<C&>())), "");
+  }
+  {
+    typedef std::list<MoveOnly, other_allocator<MoveOnly>> C;
+    static_assert(noexcept(swap(std::declval<C&>(), std::declval<C&>())), "");
+  }
 #endif // _LIBCPP_VERSION
-    {
-        typedef std::list<MoveOnly, some_alloc<MoveOnly>> C;
+  {
+    typedef std::list<MoveOnly, some_alloc<MoveOnly>> C;
 #if TEST_STD_VER >= 14
     //  In C++14, if POCS is set, swapping the allocator is required not to throw
-        static_assert( noexcept(swap(std::declval<C&>(), std::declval<C&>())), "");
+    static_assert(noexcept(swap(std::declval<C&>(), std::declval<C&>())), "");
 #else
-        static_assert(!noexcept(swap(std::declval<C&>(), std::declval<C&>())), "");
+    static_assert(!noexcept(swap(std::declval<C&>(), std::declval<C&>())), "");
 #endif
-    }
+  }
 #if TEST_STD_VER >= 14
-    {
-        typedef std::list<MoveOnly, some_alloc2<MoveOnly>> C;
+  {
+    typedef std::list<MoveOnly, some_alloc2<MoveOnly>> C;
     //  if the allocators are always equal, then the swap can be noexcept
-        static_assert( noexcept(swap(std::declval<C&>(), std::declval<C&>())), "");
-    }
+    static_assert(noexcept(swap(std::declval<C&>(), std::declval<C&>())), "");
+  }
 #endif
 
-
   return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/list/range_concept_conformance.compile.pass.cpp b/libcxx/test/std/containers/sequences/list/range_concept_conformance.compile.pass.cpp
index 20484efc5e09a..6f62dedb878d5 100644
--- a/libcxx/test/std/containers/sequences/list/range_concept_conformance.compile.pass.cpp
+++ b/libcxx/test/std/containers/sequences/list/range_concept_conformance.compile.pass.cpp
@@ -17,7 +17,6 @@
 
 using range = std::list<int>;
 
-
 static_assert(std::same_as<std::ranges::iterator_t<range>, range::iterator>);
 static_assert(std::ranges::common_range<range>);
 static_assert(std::ranges::bidirectional_range<range>);
diff --git a/libcxx/test/std/containers/sequences/list/types.pass.cpp b/libcxx/test/std/containers/sequences/list/types.pass.cpp
index a2d49080f1bcb..755c8bf4df29d 100644
--- a/libcxx/test/std/containers/sequences/list/types.pass.cpp
+++ b/libcxx/test/std/containers/sequences/list/types.pass.cpp
@@ -47,11 +47,12 @@ static_assert(std::is_same<my_derived<int, min_allocator<int>>::base, my_base>::
 static_assert(std::is_same<my_derived<my_base, min_allocator<my_base>>::base, my_base>::value, "");
 #endif
 
-struct A { std::list<A> v; }; // incomplete type support
+struct A {
+  std::list<A> v;
+}; // incomplete type support
 
-int main(int, char**)
-{
-    {
+int main(int, char**) {
+  {
     typedef std::list<int> C;
     static_assert((std::is_same<C::value_type, int>::value), "");
     static_assert((std::is_same<C::allocator_type, std::allocator<int> >::value), "");
@@ -65,13 +66,15 @@ int main(int, char**)
     static_assert((std::is_signed<typename C::difference_type>::value), "");
     static_assert((std::is_unsigned<typename C::size_type>::value), "");
     static_assert((std::is_same<typename C::difference_type,
-        typename std::iterator_traits<typename C::iterator>::difference_type>::value), "");
+                                typename std::iterator_traits<typename C::iterator>::difference_type>::value),
+                  "");
     static_assert((std::is_same<typename C::difference_type,
-        typename std::iterator_traits<typename C::const_iterator>::difference_type>::value), "");
-    }
+                                typename std::iterator_traits<typename C::const_iterator>::difference_type>::value),
+                  "");
+  }
 
 #if TEST_STD_VER >= 11
-    {
+  {
     typedef std::list<int, min_allocator<int>> C;
     static_assert((std::is_same<C::value_type, int>::value), "");
     static_assert((std::is_same<C::allocator_type, min_allocator<int> >::value), "");
@@ -83,10 +86,12 @@ int main(int, char**)
     static_assert((std::is_signed<typename C::difference_type>::value), "");
     static_assert((std::is_unsigned<typename C::size_type>::value), "");
     static_assert((std::is_same<typename C::difference_type,
-        typename std::iterator_traits<typename C::iterator>::difference_type>::value), "");
+                                typename std::iterator_traits<typename C::iterator>::difference_type>::value),
+                  "");
     static_assert((std::is_same<typename C::difference_type,
-        typename std::iterator_traits<typename C::const_iterator>::difference_type>::value), "");
-    }
+                                typename std::iterator_traits<typename C::const_iterator>::difference_type>::value),
+                  "");
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/sequences/vector.bool/append_range.pass.cpp b/libcxx/test/std/containers/sequences/vector.bool/append_range.pass.cpp
index aafeec7669449..84047ea15c6cf 100644
--- a/libcxx/test/std/containers/sequences/vector.bool/append_range.pass.cpp
+++ b/libcxx/test/std/containers/sequences/vector.bool/append_range.pass.cpp
@@ -31,12 +31,12 @@ constexpr bool test() {
     });
   });
 
-  { // Vector may or may not need to reallocate because of the insertion -- make sure to test both cases.
+  {   // Vector may or may not need to reallocate because of the insertion -- make sure to test both cases.
     { // Ensure reallocation happens.
-      constexpr int N = 255;
-      bool in[N] = {};
+      constexpr int N     = 255;
+      bool in[N]          = {};
       std::vector<bool> v = {0, 0, 0, 1, 1, 0, 0, 0};
-      auto initial = v;
+      auto initial        = v;
       assert(v.capacity() < v.size() + std::ranges::size(in));
 
       v.append_range(in);
@@ -48,7 +48,7 @@ constexpr bool test() {
     }
 
     { // Ensure no reallocation happens.
-      bool in[] = {1, 1, 1, 1, 0, 0, 1, 1, 1, 1};
+      bool in[]           = {1, 1, 1, 1, 0, 0, 1, 1, 1, 1};
       std::vector<bool> v = {0, 0, 0, 1, 1, 0, 0, 0};
       v.reserve(v.size() + std::ranges::size(in));
       assert(v.capacity() >= v.size() + std::ranges::size(in));
diff --git a/libcxx/test/std/containers/sequences/vector.bool/assign_copy.pass.cpp b/libcxx/test/std/containers/sequences/vector.bool/assign_copy.pass.cpp
index c4866ea4c9b45..2144478601897 100644
--- a/libcxx/test/std/containers/sequences/vector.bool/assign_copy.pass.cpp
+++ b/libcxx/test/std/containers/sequences/vector.bool/assign_copy.pass.cpp
@@ -16,40 +16,38 @@
 #include "test_allocator.h"
 #include "min_allocator.h"
 
-TEST_CONSTEXPR_CXX20 bool tests()
-{
-    {
-        std::vector<bool, test_allocator<bool> > l(3, true, test_allocator<bool>(5));
-        std::vector<bool, test_allocator<bool> > l2(l, test_allocator<bool>(3));
-        l2 = l;
-        assert(l2 == l);
-        assert(l2.get_allocator() == test_allocator<bool>(3));
-    }
-    {
-        std::vector<bool, other_allocator<bool> > l(3, true, other_allocator<bool>(5));
-        std::vector<bool, other_allocator<bool> > l2(l, other_allocator<bool>(3));
-        l2 = l;
-        assert(l2 == l);
-        assert(l2.get_allocator() == other_allocator<bool>(5));
-    }
+TEST_CONSTEXPR_CXX20 bool tests() {
+  {
+    std::vector<bool, test_allocator<bool> > l(3, true, test_allocator<bool>(5));
+    std::vector<bool, test_allocator<bool> > l2(l, test_allocator<bool>(3));
+    l2 = l;
+    assert(l2 == l);
+    assert(l2.get_allocator() == test_allocator<bool>(3));
+  }
+  {
+    std::vector<bool, other_allocator<bool> > l(3, true, other_allocator<bool>(5));
+    std::vector<bool, other_allocator<bool> > l2(l, other_allocator<bool>(3));
+    l2 = l;
+    assert(l2 == l);
+    assert(l2.get_allocator() == other_allocator<bool>(5));
+  }
 #if TEST_STD_VER >= 11
-    {
-        std::vector<bool, min_allocator<bool> > l(3, true, min_allocator<bool>());
-        std::vector<bool, min_allocator<bool> > l2(l, min_allocator<bool>());
-        l2 = l;
-        assert(l2 == l);
-        assert(l2.get_allocator() == min_allocator<bool>());
-    }
+  {
+    std::vector<bool, min_allocator<bool> > l(3, true, min_allocator<bool>());
+    std::vector<bool, min_allocator<bool> > l2(l, min_allocator<bool>());
+    l2 = l;
+    assert(l2 == l);
+    assert(l2.get_allocator() == min_allocator<bool>());
+  }
 #endif
 
-    return true;
+  return true;
 }
 
-int main(int, char**)
-{
-    tests();
+int main(int, char**) {
+  tests();
 #if TEST_STD_VER > 17
-    static_assert(tests());
+  static_assert(tests());
 #endif
-    return 0;
+  return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/vector.bool/assign_initializer_list.pass.cpp b/libcxx/test/std/containers/sequences/vector.bool/assign_initializer_list.pass.cpp
index 0f5ec9061f52d..3c48c0cceba0f 100644
--- a/libcxx/test/std/containers/sequences/vector.bool/assign_initializer_list.pass.cpp
+++ b/libcxx/test/std/containers/sequences/vector.bool/assign_initializer_list.pass.cpp
@@ -18,9 +18,8 @@
 #include "test_macros.h"
 #include "min_allocator.h"
 
-TEST_CONSTEXPR_CXX20 bool tests()
-{
-    {
+TEST_CONSTEXPR_CXX20 bool tests() {
+  {
     std::vector<bool> d;
     d.assign({true, false, false, true});
     assert(d.size() == 4);
@@ -28,8 +27,8 @@ TEST_CONSTEXPR_CXX20 bool tests()
     assert(d[1] == false);
     assert(d[2] == false);
     assert(d[3] == true);
-    }
-    {
+  }
+  {
     std::vector<bool, min_allocator<bool>> d;
     d.assign({true, false, false, true});
     assert(d.size() == 4);
@@ -37,16 +36,15 @@ TEST_CONSTEXPR_CXX20 bool tests()
     assert(d[1] == false);
     assert(d[2] == false);
     assert(d[3] == true);
-    }
+  }
 
-    return true;
+  return true;
 }
 
-int main(int, char**)
-{
-    tests();
+int main(int, char**) {
+  tests();
 #if TEST_STD_VER > 17
-    static_assert(tests());
+  static_assert(tests());
 #endif
-    return 0;
+  return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/vector.bool/assign_move.pass.cpp b/libcxx/test/std/containers/sequences/vector.bool/assign_move.pass.cpp
index 10271efc3f403..48ef5e3a8c262 100644
--- a/libcxx/test/std/containers/sequences/vector.bool/assign_move.pass.cpp
+++ b/libcxx/test/std/containers/sequences/vector.bool/assign_move.pass.cpp
@@ -18,73 +18,67 @@
 #include "test_allocator.h"
 #include "min_allocator.h"
 
-TEST_CONSTEXPR_CXX20 bool tests()
-{
-    {
-        std::vector<bool, test_allocator<bool> > l(test_allocator<bool>(5));
-        std::vector<bool, test_allocator<bool> > lo(test_allocator<bool>(5));
-        for (int i = 1; i <= 3; ++i)
-        {
-            l.push_back(i);
-            lo.push_back(i);
-        }
-        std::vector<bool, test_allocator<bool> > l2(test_allocator<bool>(5));
-        l2 = std::move(l);
-        assert(l2 == lo);
-        LIBCPP_ASSERT(l.empty());
-        assert(l2.get_allocator() == lo.get_allocator());
+TEST_CONSTEXPR_CXX20 bool tests() {
+  {
+    std::vector<bool, test_allocator<bool> > l(test_allocator<bool>(5));
+    std::vector<bool, test_allocator<bool> > lo(test_allocator<bool>(5));
+    for (int i = 1; i <= 3; ++i) {
+      l.push_back(i);
+      lo.push_back(i);
     }
-    {
-        std::vector<bool, test_allocator<bool> > l(test_allocator<bool>(5));
-        std::vector<bool, test_allocator<bool> > lo(test_allocator<bool>(5));
-        for (int i = 1; i <= 3; ++i)
-        {
-            l.push_back(i);
-            lo.push_back(i);
-        }
-        std::vector<bool, test_allocator<bool> > l2(test_allocator<bool>(6));
-        l2 = std::move(l);
-        assert(l2 == lo);
-        assert(!l.empty());
-        assert(l2.get_allocator() == test_allocator<bool>(6));
+    std::vector<bool, test_allocator<bool> > l2(test_allocator<bool>(5));
+    l2 = std::move(l);
+    assert(l2 == lo);
+    LIBCPP_ASSERT(l.empty());
+    assert(l2.get_allocator() == lo.get_allocator());
+  }
+  {
+    std::vector<bool, test_allocator<bool> > l(test_allocator<bool>(5));
+    std::vector<bool, test_allocator<bool> > lo(test_allocator<bool>(5));
+    for (int i = 1; i <= 3; ++i) {
+      l.push_back(i);
+      lo.push_back(i);
     }
-    {
-        std::vector<bool, other_allocator<bool> > l(other_allocator<bool>(5));
-        std::vector<bool, other_allocator<bool> > lo(other_allocator<bool>(5));
-        for (int i = 1; i <= 3; ++i)
-        {
-            l.push_back(i);
-            lo.push_back(i);
-        }
-        std::vector<bool, other_allocator<bool> > l2(other_allocator<bool>(6));
-        l2 = std::move(l);
-        assert(l2 == lo);
-        assert(l.empty());
-        assert(l2.get_allocator() == lo.get_allocator());
+    std::vector<bool, test_allocator<bool> > l2(test_allocator<bool>(6));
+    l2 = std::move(l);
+    assert(l2 == lo);
+    assert(!l.empty());
+    assert(l2.get_allocator() == test_allocator<bool>(6));
+  }
+  {
+    std::vector<bool, other_allocator<bool> > l(other_allocator<bool>(5));
+    std::vector<bool, other_allocator<bool> > lo(other_allocator<bool>(5));
+    for (int i = 1; i <= 3; ++i) {
+      l.push_back(i);
+      lo.push_back(i);
     }
-    {
-        std::vector<bool, min_allocator<bool> > l(min_allocator<bool>{});
-        std::vector<bool, min_allocator<bool> > lo(min_allocator<bool>{});
-        for (int i = 1; i <= 3; ++i)
-        {
-            l.push_back(i);
-            lo.push_back(i);
-        }
-        std::vector<bool, min_allocator<bool> > l2(min_allocator<bool>{});
-        l2 = std::move(l);
-        assert(l2 == lo);
-        assert(l.empty());
-        assert(l2.get_allocator() == lo.get_allocator());
+    std::vector<bool, other_allocator<bool> > l2(other_allocator<bool>(6));
+    l2 = std::move(l);
+    assert(l2 == lo);
+    assert(l.empty());
+    assert(l2.get_allocator() == lo.get_allocator());
+  }
+  {
+    std::vector<bool, min_allocator<bool> > l(min_allocator<bool>{});
+    std::vector<bool, min_allocator<bool> > lo(min_allocator<bool>{});
+    for (int i = 1; i <= 3; ++i) {
+      l.push_back(i);
+      lo.push_back(i);
     }
+    std::vector<bool, min_allocator<bool> > l2(min_allocator<bool>{});
+    l2 = std::move(l);
+    assert(l2 == lo);
+    assert(l.empty());
+    assert(l2.get_allocator() == lo.get_allocator());
+  }
 
-    return true;
+  return true;
 }
 
-int main(int, char**)
-{
-    tests();
+int main(int, char**) {
+  tests();
 #if TEST_STD_VER > 17
-    static_assert(tests());
+  static_assert(tests());
 #endif
-    return 0;
+  return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/vector.bool/assign_range.pass.cpp b/libcxx/test/std/containers/sequences/vector.bool/assign_range.pass.cpp
index a8cbb32c2655e..1e215e49b53fd 100644
--- a/libcxx/test/std/containers/sequences/vector.bool/assign_range.pass.cpp
+++ b/libcxx/test/std/containers/sequences/vector.bool/assign_range.pass.cpp
@@ -32,10 +32,10 @@ constexpr bool test() {
     });
   });
 
-  { // Vector may or may not need to reallocate because of the assignment -- make sure to test both cases.
+  {   // Vector may or may not need to reallocate because of the assignment -- make sure to test both cases.
     { // Ensure reallocation happens. Note that `vector<bool>` typically reserves a lot of capacity.
-      constexpr int N = 255;
-      bool in[N] = {};
+      constexpr int N     = 255;
+      bool in[N]          = {};
       std::vector<bool> v = {0, 0, 0, 1, 1, 0, 0, 0};
       assert(v.capacity() < v.size() + std::ranges::size(in));
 
@@ -44,7 +44,7 @@ constexpr bool test() {
     }
 
     { // Ensure no reallocation happens.
-      bool in[] = {1, 1, 0, 1, 1};
+      bool in[]           = {1, 1, 0, 1, 1};
       std::vector<bool> v = {0, 0, 0, 1, 1, 0, 0, 0};
 
       v.assign_range(in);
diff --git a/libcxx/test/std/containers/sequences/vector.bool/capacity.pass.cpp b/libcxx/test/std/containers/sequences/vector.bool/capacity.pass.cpp
index 986453a189049..46946bf07456c 100644
--- a/libcxx/test/std/containers/sequences/vector.bool/capacity.pass.cpp
+++ b/libcxx/test/std/containers/sequences/vector.bool/capacity.pass.cpp
@@ -17,39 +17,37 @@
 #include "test_macros.h"
 #include "min_allocator.h"
 
-TEST_CONSTEXPR_CXX20 bool tests()
-{
-    {
-        std::vector<bool> v;
-        assert(v.capacity() == 0);
-    }
-    {
-        std::vector<bool> v(100);
-        assert(v.capacity() >= 100);
-        v.push_back(0);
-        assert(v.capacity() >= 101);
-    }
+TEST_CONSTEXPR_CXX20 bool tests() {
+  {
+    std::vector<bool> v;
+    assert(v.capacity() == 0);
+  }
+  {
+    std::vector<bool> v(100);
+    assert(v.capacity() >= 100);
+    v.push_back(0);
+    assert(v.capacity() >= 101);
+  }
 #if TEST_STD_VER >= 11
-    {
-        std::vector<bool, min_allocator<bool>> v;
-        assert(v.capacity() == 0);
-    }
-    {
-        std::vector<bool, min_allocator<bool>> v(100);
-        assert(v.capacity() >= 100);
-        v.push_back(0);
-        assert(v.capacity() >= 101);
-    }
+  {
+    std::vector<bool, min_allocator<bool>> v;
+    assert(v.capacity() == 0);
+  }
+  {
+    std::vector<bool, min_allocator<bool>> v(100);
+    assert(v.capacity() >= 100);
+    v.push_back(0);
+    assert(v.capacity() >= 101);
+  }
 #endif
 
-    return true;
+  return true;
 }
 
-int main(int, char**)
-{
-    tests();
+int main(int, char**) {
+  tests();
 #if TEST_STD_VER > 17
-    static_assert(tests());
+  static_assert(tests());
 #endif
-    return 0;
+  return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/vector.bool/compare.pass.cpp b/libcxx/test/std/containers/sequences/vector.bool/compare.pass.cpp
index 0f818029c1ff9..5300447c53579 100644
--- a/libcxx/test/std/containers/sequences/vector.bool/compare.pass.cpp
+++ b/libcxx/test/std/containers/sequences/vector.bool/compare.pass.cpp
@@ -21,69 +21,69 @@
 #include "test_comparisons.h"
 
 TEST_CONSTEXPR_CXX20 bool test() {
-    typedef std::vector<bool> VB;
-    {
-        const VB v1, v2;
-        assert(testComparisons(v1, v2, true, false));
-    }
-    {
-        const VB v1(1, true);
-        const VB v2(1, true);
-        assert(testComparisons(v1, v2, true, false));
-    }
-    {
-        const VB v1(1, false);
-        const VB v2(1, true);
-        assert(testComparisons(v1, v2, false, true));
-    }
-    {
-        const VB v1, v2(1, true);
-        assert(testComparisons(v1, v2, false, true));
-    }
-    {
-        bool items1[3] = {false, true, false};
-        bool items2[3] = {false, true, true};
-        const VB v1(items1, items1 + 3);
-        const VB v2(items2, items2 + 3);
-        assert(testComparisons(v1, v2, false, true));
-    }
-    {
-        bool items1[3] = {false, false, false};
-        bool items2[3] = {false, true, false};
-        const VB v1(items1, items1 + 3);
-        const VB v2(items2, items2 + 3);
-        assert(testComparisons(v1, v2, false, true));
-    }
-    {
-        bool items1[2] = {false, true};
-        bool items2[3] = {false, true, false};
-        const VB v1(items1, items1 + 2);
-        const VB v2(items2, items2 + 3);
-        assert(testComparisons(v1, v2, false, true));
-    }
-    {
-        bool items[3] = {false, true, false};
-        const VB v1(items, items + 3);
-        const VB v2(1, true);
-        assert(testComparisons(v1, v2, false, true));
-    }
-    {
-        assert( (std::vector<bool>() == std::vector<bool>()));
-        assert(!(std::vector<bool>() != std::vector<bool>()));
-        assert(!(std::vector<bool>() < std::vector<bool>()));
-        assert( (std::vector<bool>() <= std::vector<bool>()));
-        assert(!(std::vector<bool>() > std::vector<bool>()));
-        assert( (std::vector<bool>() >= std::vector<bool>()));
-    }
+  typedef std::vector<bool> VB;
+  {
+    const VB v1, v2;
+    assert(testComparisons(v1, v2, true, false));
+  }
+  {
+    const VB v1(1, true);
+    const VB v2(1, true);
+    assert(testComparisons(v1, v2, true, false));
+  }
+  {
+    const VB v1(1, false);
+    const VB v2(1, true);
+    assert(testComparisons(v1, v2, false, true));
+  }
+  {
+    const VB v1, v2(1, true);
+    assert(testComparisons(v1, v2, false, true));
+  }
+  {
+    bool items1[3] = {false, true, false};
+    bool items2[3] = {false, true, true};
+    const VB v1(items1, items1 + 3);
+    const VB v2(items2, items2 + 3);
+    assert(testComparisons(v1, v2, false, true));
+  }
+  {
+    bool items1[3] = {false, false, false};
+    bool items2[3] = {false, true, false};
+    const VB v1(items1, items1 + 3);
+    const VB v2(items2, items2 + 3);
+    assert(testComparisons(v1, v2, false, true));
+  }
+  {
+    bool items1[2] = {false, true};
+    bool items2[3] = {false, true, false};
+    const VB v1(items1, items1 + 2);
+    const VB v2(items2, items2 + 3);
+    assert(testComparisons(v1, v2, false, true));
+  }
+  {
+    bool items[3] = {false, true, false};
+    const VB v1(items, items + 3);
+    const VB v2(1, true);
+    assert(testComparisons(v1, v2, false, true));
+  }
+  {
+    assert((std::vector<bool>() == std::vector<bool>()));
+    assert(!(std::vector<bool>() != std::vector<bool>()));
+    assert(!(std::vector<bool>() < std::vector<bool>()));
+    assert((std::vector<bool>() <= std::vector<bool>()));
+    assert(!(std::vector<bool>() > std::vector<bool>()));
+    assert((std::vector<bool>() >= std::vector<bool>()));
+  }
 
-    return true;
+  return true;
 }
 
 int main(int, char**) {
-    test();
+  test();
 #if TEST_STD_VER > 17
-    static_assert(test());
+  static_assert(test());
 #endif
 
-    return 0;
+  return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/vector.bool/const_reference.pass.cpp b/libcxx/test/std/containers/sequences/vector.bool/const_reference.pass.cpp
index c2cb7f0298ddb..dd81f3208daa2 100644
--- a/libcxx/test/std/containers/sequences/vector.bool/const_reference.pass.cpp
+++ b/libcxx/test/std/containers/sequences/vector.bool/const_reference.pass.cpp
@@ -33,7 +33,7 @@ TEST_CONSTEXPR_CXX20 bool test() {
 int main(int, char**) {
   test();
 #if TEST_STD_VER > 17
-    static_assert(test());
+  static_assert(test());
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/sequences/vector.bool/construct_default.pass.cpp b/libcxx/test/std/containers/sequences/vector.bool/construct_default.pass.cpp
index 8f07524d070bb..09d33369aebcd 100644
--- a/libcxx/test/std/containers/sequences/vector.bool/construct_default.pass.cpp
+++ b/libcxx/test/std/containers/sequences/vector.bool/construct_default.pass.cpp
@@ -15,7 +15,6 @@
 // For vector<>, this was added to the standard by N4258,
 //   but vector<bool> was not changed.
 
-
 #include <vector>
 #include <cassert>
 
@@ -24,64 +23,62 @@
 #include "min_allocator.h"
 
 template <class C>
-TEST_CONSTEXPR_CXX20 void test0()
-{
+TEST_CONSTEXPR_CXX20 void test0() {
 #if TEST_STD_VER > 14
-    LIBCPP_STATIC_ASSERT((noexcept(C{})), "" );
+  LIBCPP_STATIC_ASSERT((noexcept(C{})), "");
 #elif TEST_STD_VER >= 11
-    LIBCPP_STATIC_ASSERT((noexcept(C()) == noexcept(typename C::allocator_type())), "" );
+  LIBCPP_STATIC_ASSERT((noexcept(C()) == noexcept(typename C::allocator_type())), "");
 #endif
-    C c;
-    LIBCPP_ASSERT(c.__invariants());
-    assert(c.empty());
-    assert(c.get_allocator() == typename C::allocator_type());
+  C c;
+  LIBCPP_ASSERT(c.__invariants());
+  assert(c.empty());
+  assert(c.get_allocator() == typename C::allocator_type());
 #if TEST_STD_VER >= 11
-    C c1 = {};
-    LIBCPP_ASSERT(c1.__invariants());
-    assert(c1.empty());
-    assert(c1.get_allocator() == typename C::allocator_type());
+  C c1 = {};
+  LIBCPP_ASSERT(c1.__invariants());
+  assert(c1.empty());
+  assert(c1.get_allocator() == typename C::allocator_type());
 #endif
 }
 
 template <class C>
-TEST_CONSTEXPR_CXX20 void test1(const typename C::allocator_type& a)
-{
+TEST_CONSTEXPR_CXX20 void test1(const typename C::allocator_type& a) {
 #if TEST_STD_VER > 14
-    LIBCPP_STATIC_ASSERT((noexcept(C{typename C::allocator_type{}})), "" );
+  LIBCPP_STATIC_ASSERT((noexcept(C{typename C::allocator_type{}})), "");
 #elif TEST_STD_VER >= 11
-    LIBCPP_STATIC_ASSERT((noexcept(C(typename C::allocator_type())) == std::is_nothrow_copy_constructible<typename C::allocator_type>::value), "" );
+  LIBCPP_STATIC_ASSERT((noexcept(C(typename C::allocator_type())) ==
+                        std::is_nothrow_copy_constructible<typename C::allocator_type>::value),
+                       "");
 #endif
-    C c(a);
-    LIBCPP_ASSERT(c.__invariants());
-    assert(c.empty());
-    assert(c.get_allocator() == a);
+  C c(a);
+  LIBCPP_ASSERT(c.__invariants());
+  assert(c.empty());
+  assert(c.get_allocator() == a);
 }
 
-TEST_CONSTEXPR_CXX20 bool tests()
-{
-    {
+TEST_CONSTEXPR_CXX20 bool tests() {
+  {
     test0<std::vector<bool> >();
     test1<std::vector<bool, test_allocator<bool> > >(test_allocator<bool>(3));
-    }
+  }
 #if TEST_STD_VER >= 11
-    {
+  {
     test0<std::vector<bool, min_allocator<bool>> >();
     test1<std::vector<bool, min_allocator<bool> > >(min_allocator<bool>());
-    }
-    {
+  }
+  {
     test0<std::vector<bool, explicit_allocator<bool>> >();
     test1<std::vector<bool, explicit_allocator<bool> > >(explicit_allocator<bool>());
-    }
+  }
 #endif
 
-    return true;
+  return true;
 }
 
-int main(int, char**)
-{
-    tests();
+int main(int, char**) {
+  tests();
 #if TEST_STD_VER > 17
-    static_assert(tests());
+  static_assert(tests());
 #endif
-    return 0;
+  return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/vector.bool/construct_iter_iter.pass.cpp b/libcxx/test/std/containers/sequences/vector.bool/construct_iter_iter.pass.cpp
index 2f520d5abbf3e..e9fb2e6ecfbac 100644
--- a/libcxx/test/std/containers/sequences/vector.bool/construct_iter_iter.pass.cpp
+++ b/libcxx/test/std/containers/sequences/vector.bool/construct_iter_iter.pass.cpp
@@ -20,40 +20,40 @@
 #include "min_allocator.h"
 
 template <class C, class Iterator>
-TEST_CONSTEXPR_CXX20 void test(Iterator first, Iterator last)
-{
-    C c(first, last);
-    LIBCPP_ASSERT(c.__invariants());
-    assert(c.size() == static_cast<std::size_t>(std::distance(first, last)));
-    for (typename C::const_iterator i = c.cbegin(), e = c.cend(); i != e; ++i, ++first)
-        assert(*i == *first);
+TEST_CONSTEXPR_CXX20 void test(Iterator first, Iterator last) {
+  C c(first, last);
+  LIBCPP_ASSERT(c.__invariants());
+  assert(c.size() == static_cast<std::size_t>(std::distance(first, last)));
+  for (typename C::const_iterator i = c.cbegin(), e = c.cend(); i != e; ++i, ++first)
+    assert(*i == *first);
 }
 
-TEST_CONSTEXPR_CXX20 bool tests()
-{
-    bool a[] = {0, 1, 0, 0, 1, 1, 1, 0, 0, 1, 0, 0, 0, 1, 1, 0, 1, 0};
-    bool* an = a + sizeof(a)/sizeof(a[0]);
-    test<std::vector<bool> >(cpp17_input_iterator<const bool*>(a), cpp17_input_iterator<const bool*>(an));
-    test<std::vector<bool> >(forward_iterator<const bool*>(a), forward_iterator<const bool*>(an));
-    test<std::vector<bool> >(bidirectional_iterator<const bool*>(a), bidirectional_iterator<const bool*>(an));
-    test<std::vector<bool> >(random_access_iterator<const bool*>(a), random_access_iterator<const bool*>(an));
-    test<std::vector<bool> >(a, an);
+TEST_CONSTEXPR_CXX20 bool tests() {
+  bool a[] = {0, 1, 0, 0, 1, 1, 1, 0, 0, 1, 0, 0, 0, 1, 1, 0, 1, 0};
+  bool* an = a + sizeof(a) / sizeof(a[0]);
+  test<std::vector<bool> >(cpp17_input_iterator<const bool*>(a), cpp17_input_iterator<const bool*>(an));
+  test<std::vector<bool> >(forward_iterator<const bool*>(a), forward_iterator<const bool*>(an));
+  test<std::vector<bool> >(bidirectional_iterator<const bool*>(a), bidirectional_iterator<const bool*>(an));
+  test<std::vector<bool> >(random_access_iterator<const bool*>(a), random_access_iterator<const bool*>(an));
+  test<std::vector<bool> >(a, an);
 #if TEST_STD_VER >= 11
-    test<std::vector<bool, min_allocator<bool>> >(cpp17_input_iterator<const bool*>(a), cpp17_input_iterator<const bool*>(an));
-    test<std::vector<bool, min_allocator<bool>> >(forward_iterator<const bool*>(a), forward_iterator<const bool*>(an));
-    test<std::vector<bool, min_allocator<bool>> >(bidirectional_iterator<const bool*>(a), bidirectional_iterator<const bool*>(an));
-    test<std::vector<bool, min_allocator<bool>> >(random_access_iterator<const bool*>(a), random_access_iterator<const bool*>(an));
-    test<std::vector<bool, min_allocator<bool>> >(a, an);
+  test<std::vector<bool, min_allocator<bool>> >(
+      cpp17_input_iterator<const bool*>(a), cpp17_input_iterator<const bool*>(an));
+  test<std::vector<bool, min_allocator<bool>> >(forward_iterator<const bool*>(a), forward_iterator<const bool*>(an));
+  test<std::vector<bool, min_allocator<bool>> >(
+      bidirectional_iterator<const bool*>(a), bidirectional_iterator<const bool*>(an));
+  test<std::vector<bool, min_allocator<bool>> >(
+      random_access_iterator<const bool*>(a), random_access_iterator<const bool*>(an));
+  test<std::vector<bool, min_allocator<bool>> >(a, an);
 #endif
 
-    return true;
+  return true;
 }
 
-int main(int, char**)
-{
-    tests();
+int main(int, char**) {
+  tests();
 #if TEST_STD_VER > 17
-    static_assert(tests());
+  static_assert(tests());
 #endif
-    return 0;
+  return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/vector.bool/construct_iter_iter_alloc.pass.cpp b/libcxx/test/std/containers/sequences/vector.bool/construct_iter_iter_alloc.pass.cpp
index 57e1cb2e0b502..71a176a0a64ba 100644
--- a/libcxx/test/std/containers/sequences/vector.bool/construct_iter_iter_alloc.pass.cpp
+++ b/libcxx/test/std/containers/sequences/vector.bool/construct_iter_iter_alloc.pass.cpp
@@ -21,46 +21,47 @@
 #include "min_allocator.h"
 
 template <class C, class Iterator>
-TEST_CONSTEXPR_CXX20 void test(Iterator first, Iterator last, const typename C::allocator_type& a)
-{
-    C c(first, last, a);
-    LIBCPP_ASSERT(c.__invariants());
-    assert(c.size() == static_cast<std::size_t>(std::distance(first, last)));
-    for (typename C::const_iterator i = c.cbegin(), e = c.cend(); i != e; ++i, ++first)
-        assert(*i == *first);
+TEST_CONSTEXPR_CXX20 void test(Iterator first, Iterator last, const typename C::allocator_type& a) {
+  C c(first, last, a);
+  LIBCPP_ASSERT(c.__invariants());
+  assert(c.size() == static_cast<std::size_t>(std::distance(first, last)));
+  for (typename C::const_iterator i = c.cbegin(), e = c.cend(); i != e; ++i, ++first)
+    assert(*i == *first);
 }
 
-TEST_CONSTEXPR_CXX20 bool tests()
-{
-    bool a[] = {0, 1, 0, 0, 1, 1, 1, 0, 0, 1, 0, 0, 0, 1, 1, 0, 1, 0};
-    bool* an = a + sizeof(a)/sizeof(a[0]);
-    {
+TEST_CONSTEXPR_CXX20 bool tests() {
+  bool a[] = {0, 1, 0, 0, 1, 1, 1, 0, 0, 1, 0, 0, 0, 1, 1, 0, 1, 0};
+  bool* an = a + sizeof(a) / sizeof(a[0]);
+  {
     std::allocator<bool> alloc;
     test<std::vector<bool> >(cpp17_input_iterator<const bool*>(a), cpp17_input_iterator<const bool*>(an), alloc);
     test<std::vector<bool> >(forward_iterator<const bool*>(a), forward_iterator<const bool*>(an), alloc);
     test<std::vector<bool> >(bidirectional_iterator<const bool*>(a), bidirectional_iterator<const bool*>(an), alloc);
     test<std::vector<bool> >(random_access_iterator<const bool*>(a), random_access_iterator<const bool*>(an), alloc);
     test<std::vector<bool> >(a, an, alloc);
-    }
+  }
 #if TEST_STD_VER >= 11
-    {
+  {
     min_allocator<bool> alloc;
-    test<std::vector<bool, min_allocator<bool>> >(cpp17_input_iterator<const bool*>(a), cpp17_input_iterator<const bool*>(an), alloc);
-    test<std::vector<bool, min_allocator<bool>> >(forward_iterator<const bool*>(a), forward_iterator<const bool*>(an), alloc);
-    test<std::vector<bool, min_allocator<bool>> >(bidirectional_iterator<const bool*>(a), bidirectional_iterator<const bool*>(an), alloc);
-    test<std::vector<bool, min_allocator<bool>> >(random_access_iterator<const bool*>(a), random_access_iterator<const bool*>(an), alloc);
+    test<std::vector<bool, min_allocator<bool>> >(
+        cpp17_input_iterator<const bool*>(a), cpp17_input_iterator<const bool*>(an), alloc);
+    test<std::vector<bool, min_allocator<bool>> >(
+        forward_iterator<const bool*>(a), forward_iterator<const bool*>(an), alloc);
+    test<std::vector<bool, min_allocator<bool>> >(
+        bidirectional_iterator<const bool*>(a), bidirectional_iterator<const bool*>(an), alloc);
+    test<std::vector<bool, min_allocator<bool>> >(
+        random_access_iterator<const bool*>(a), random_access_iterator<const bool*>(an), alloc);
     test<std::vector<bool, min_allocator<bool>> >(a, an, alloc);
-    }
+  }
 #endif
 
-    return true;
+  return true;
 }
 
-int main(int, char**)
-{
-    tests();
+int main(int, char**) {
+  tests();
 #if TEST_STD_VER > 17
-    static_assert(tests());
+  static_assert(tests());
 #endif
-    return 0;
+  return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/vector.bool/construct_size.pass.cpp b/libcxx/test/std/containers/sequences/vector.bool/construct_size.pass.cpp
index bcc5a16859d36..80b3fe307c93a 100644
--- a/libcxx/test/std/containers/sequences/vector.bool/construct_size.pass.cpp
+++ b/libcxx/test/std/containers/sequences/vector.bool/construct_size.pass.cpp
@@ -19,16 +19,15 @@
 #include "test_allocator.h"
 
 template <class C>
-TEST_CONSTEXPR_CXX20 void test2(typename C::size_type n,
-                                typename C::allocator_type const& a = typename C::allocator_type ())
-{
+TEST_CONSTEXPR_CXX20 void
+test2(typename C::size_type n, typename C::allocator_type const& a = typename C::allocator_type()) {
 #if TEST_STD_VER >= 14
-    C c(n, a);
-    LIBCPP_ASSERT(c.__invariants());
-    assert(c.size() == n);
-    assert(c.get_allocator() == a);
-    for (typename C::const_iterator i = c.cbegin(), e = c.cend(); i != e; ++i)
-        assert(*i == typename C::value_type());
+  C c(n, a);
+  LIBCPP_ASSERT(c.__invariants());
+  assert(c.size() == n);
+  assert(c.get_allocator() == a);
+  for (typename C::const_iterator i = c.cbegin(), e = c.cend(); i != e; ++i)
+    assert(*i == typename C::value_type());
 #else
   ((void)n);
   ((void)a);
@@ -36,39 +35,35 @@ TEST_CONSTEXPR_CXX20 void test2(typename C::size_type n,
 }
 
 template <class C>
-TEST_CONSTEXPR_CXX20 void test1(typename C::size_type n)
-{
-    C c(n);
-    LIBCPP_ASSERT(c.__invariants());
-    assert(c.size() == n);
-    assert(c.get_allocator() == typename C::allocator_type());
-    for (typename C::const_iterator i = c.cbegin(), e = c.cend(); i != e; ++i)
-        assert(*i == typename C::value_type());
+TEST_CONSTEXPR_CXX20 void test1(typename C::size_type n) {
+  C c(n);
+  LIBCPP_ASSERT(c.__invariants());
+  assert(c.size() == n);
+  assert(c.get_allocator() == typename C::allocator_type());
+  for (typename C::const_iterator i = c.cbegin(), e = c.cend(); i != e; ++i)
+    assert(*i == typename C::value_type());
 }
 
 template <class C>
-TEST_CONSTEXPR_CXX20 void test(typename C::size_type n)
-{
-    test1<C> ( n );
-    test2<C> ( n );
+TEST_CONSTEXPR_CXX20 void test(typename C::size_type n) {
+  test1<C>(n);
+  test2<C>(n);
 }
 
-TEST_CONSTEXPR_CXX20 bool tests()
-{
-    test<std::vector<bool> >(50);
+TEST_CONSTEXPR_CXX20 bool tests() {
+  test<std::vector<bool> >(50);
 #if TEST_STD_VER >= 11
-    test<std::vector<bool, min_allocator<bool>> >(50);
-    test2<std::vector<bool, test_allocator<bool>> >( 100, test_allocator<bool>(23));
+  test<std::vector<bool, min_allocator<bool>> >(50);
+  test2<std::vector<bool, test_allocator<bool>> >(100, test_allocator<bool>(23));
 #endif
 
-    return true;
+  return true;
 }
 
-int main(int, char**)
-{
-    tests();
+int main(int, char**) {
+  tests();
 #if TEST_STD_VER > 17
-    static_assert(tests());
+  static_assert(tests());
 #endif
-    return 0;
+  return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/vector.bool/construct_size_value.pass.cpp b/libcxx/test/std/containers/sequences/vector.bool/construct_size_value.pass.cpp
index 629e8ba57ca85..17d3ea2c5dd8d 100644
--- a/libcxx/test/std/containers/sequences/vector.bool/construct_size_value.pass.cpp
+++ b/libcxx/test/std/containers/sequences/vector.bool/construct_size_value.pass.cpp
@@ -18,30 +18,27 @@
 #include "min_allocator.h"
 
 template <class C>
-TEST_CONSTEXPR_CXX20 void test(typename C::size_type n, const typename C::value_type& x)
-{
-    C c(n, x);
-    LIBCPP_ASSERT(c.__invariants());
-    assert(c.size() == n);
-    for (typename C::const_iterator i = c.cbegin(), e = c.cend(); i != e; ++i)
-        assert(*i == x);
+TEST_CONSTEXPR_CXX20 void test(typename C::size_type n, const typename C::value_type& x) {
+  C c(n, x);
+  LIBCPP_ASSERT(c.__invariants());
+  assert(c.size() == n);
+  for (typename C::const_iterator i = c.cbegin(), e = c.cend(); i != e; ++i)
+    assert(*i == x);
 }
 
-TEST_CONSTEXPR_CXX20 bool tests()
-{
-    test<std::vector<bool> >(50, true);
+TEST_CONSTEXPR_CXX20 bool tests() {
+  test<std::vector<bool> >(50, true);
 #if TEST_STD_VER >= 11
-    test<std::vector<bool, min_allocator<bool>> >(50, true);
+  test<std::vector<bool, min_allocator<bool>> >(50, true);
 #endif
 
-    return true;
+  return true;
 }
 
-int main(int, char**)
-{
-    tests();
+int main(int, char**) {
+  tests();
 #if TEST_STD_VER > 17
-    static_assert(tests());
+  static_assert(tests());
 #endif
-    return 0;
+  return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/vector.bool/construct_size_value_alloc.pass.cpp b/libcxx/test/std/containers/sequences/vector.bool/construct_size_value_alloc.pass.cpp
index 94a44ad282b71..f63b7b35c2daa 100644
--- a/libcxx/test/std/containers/sequences/vector.bool/construct_size_value_alloc.pass.cpp
+++ b/libcxx/test/std/containers/sequences/vector.bool/construct_size_value_alloc.pass.cpp
@@ -18,33 +18,29 @@
 #include "min_allocator.h"
 
 template <class C>
-TEST_CONSTEXPR_CXX20 void test(typename C::size_type n,
-                                    const typename C::value_type& x,
-                                    const typename C::allocator_type& a)
-{
-    C c(n, x, a);
-    LIBCPP_ASSERT(c.__invariants());
-    assert(a == c.get_allocator());
-    assert(c.size() == n);
-    for (typename C::const_iterator i = c.cbegin(), e = c.cend(); i != e; ++i)
-        assert(*i == x);
+TEST_CONSTEXPR_CXX20 void
+test(typename C::size_type n, const typename C::value_type& x, const typename C::allocator_type& a) {
+  C c(n, x, a);
+  LIBCPP_ASSERT(c.__invariants());
+  assert(a == c.get_allocator());
+  assert(c.size() == n);
+  for (typename C::const_iterator i = c.cbegin(), e = c.cend(); i != e; ++i)
+    assert(*i == x);
 }
 
-TEST_CONSTEXPR_CXX20 bool tests()
-{
-    test<std::vector<bool> >(50, true, std::allocator<bool>());
+TEST_CONSTEXPR_CXX20 bool tests() {
+  test<std::vector<bool> >(50, true, std::allocator<bool>());
 #if TEST_STD_VER >= 11
-    test<std::vector<bool, min_allocator<bool>> >(50, true, min_allocator<bool>());
+  test<std::vector<bool, min_allocator<bool>> >(50, true, min_allocator<bool>());
 #endif
 
-    return true;
+  return true;
 }
 
-int main(int, char**)
-{
-    tests();
+int main(int, char**) {
+  tests();
 #if TEST_STD_VER > 17
-    static_assert(tests());
+  static_assert(tests());
 #endif
-    return 0;
+  return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/vector.bool/copy.pass.cpp b/libcxx/test/std/containers/sequences/vector.bool/copy.pass.cpp
index bfd37e956dbdf..15b26e8054df8 100644
--- a/libcxx/test/std/containers/sequences/vector.bool/copy.pass.cpp
+++ b/libcxx/test/std/containers/sequences/vector.bool/copy.pass.cpp
@@ -19,56 +19,53 @@
 #include "min_allocator.h"
 
 template <class C>
-TEST_CONSTEXPR_CXX20 void test(const C& x)
-{
-    typename C::size_type s = x.size();
-    C c(x);
-    LIBCPP_ASSERT(c.__invariants());
-    assert(c.size() == s);
-    assert(c == x);
+TEST_CONSTEXPR_CXX20 void test(const C& x) {
+  typename C::size_type s = x.size();
+  C c(x);
+  LIBCPP_ASSERT(c.__invariants());
+  assert(c.size() == s);
+  assert(c == x);
 }
 
-TEST_CONSTEXPR_CXX20 bool tests()
-{
-    {
-        bool a[] = {0, 1, 0, 0, 1, 1, 1, 0, 0, 1, 0, 0, 0, 1, 1, 0, 1, 0};
-        bool* an = a + sizeof(a)/sizeof(a[0]);
-        test(std::vector<bool>(a, an));
-    }
-    {
-        std::vector<bool, test_allocator<bool> > v(3, true, test_allocator<bool>(5));
-        std::vector<bool, test_allocator<bool> > v2 = v;
-        assert(v2 == v);
-        assert(v2.get_allocator() == v.get_allocator());
-    }
+TEST_CONSTEXPR_CXX20 bool tests() {
+  {
+    bool a[] = {0, 1, 0, 0, 1, 1, 1, 0, 0, 1, 0, 0, 0, 1, 1, 0, 1, 0};
+    bool* an = a + sizeof(a) / sizeof(a[0]);
+    test(std::vector<bool>(a, an));
+  }
+  {
+    std::vector<bool, test_allocator<bool> > v(3, true, test_allocator<bool>(5));
+    std::vector<bool, test_allocator<bool> > v2 = v;
+    assert(v2 == v);
+    assert(v2.get_allocator() == v.get_allocator());
+  }
 #if TEST_STD_VER >= 11
-    {
-        std::vector<bool, other_allocator<bool> > v(3, true, other_allocator<bool>(5));
-        std::vector<bool, other_allocator<bool> > v2 = v;
-        assert(v2 == v);
-        assert(v2.get_allocator() == other_allocator<bool>(-2));
-    }
-    {
-        bool a[] = {0, 1, 0, 0, 1, 1, 1, 0, 0, 1, 0, 0, 0, 1, 1, 0, 1, 0};
-        bool* an = a + sizeof(a)/sizeof(a[0]);
-        test(std::vector<bool, min_allocator<bool>>(a, an));
-    }
-    {
-        std::vector<bool, min_allocator<bool> > v(3, true, min_allocator<bool>());
-        std::vector<bool, min_allocator<bool> > v2 = v;
-        assert(v2 == v);
-        assert(v2.get_allocator() == v.get_allocator());
-    }
+  {
+    std::vector<bool, other_allocator<bool> > v(3, true, other_allocator<bool>(5));
+    std::vector<bool, other_allocator<bool> > v2 = v;
+    assert(v2 == v);
+    assert(v2.get_allocator() == other_allocator<bool>(-2));
+  }
+  {
+    bool a[] = {0, 1, 0, 0, 1, 1, 1, 0, 0, 1, 0, 0, 0, 1, 1, 0, 1, 0};
+    bool* an = a + sizeof(a) / sizeof(a[0]);
+    test(std::vector<bool, min_allocator<bool>>(a, an));
+  }
+  {
+    std::vector<bool, min_allocator<bool> > v(3, true, min_allocator<bool>());
+    std::vector<bool, min_allocator<bool> > v2 = v;
+    assert(v2 == v);
+    assert(v2.get_allocator() == v.get_allocator());
+  }
 #endif
 
-    return true;
+  return true;
 }
 
-int main(int, char**)
-{
-    tests();
+int main(int, char**) {
+  tests();
 #if TEST_STD_VER > 17
-    static_assert(tests());
+  static_assert(tests());
 #endif
-    return 0;
+  return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/vector.bool/copy_alloc.pass.cpp b/libcxx/test/std/containers/sequences/vector.bool/copy_alloc.pass.cpp
index 468bf5d866004..d22b7c5d58447 100644
--- a/libcxx/test/std/containers/sequences/vector.bool/copy_alloc.pass.cpp
+++ b/libcxx/test/std/containers/sequences/vector.bool/copy_alloc.pass.cpp
@@ -18,56 +18,53 @@
 #include "min_allocator.h"
 
 template <class C>
-TEST_CONSTEXPR_CXX20 void test(const C& x, const typename C::allocator_type& a)
-{
-    typename C::size_type s = x.size();
-    C c(x, a);
-    LIBCPP_ASSERT(c.__invariants());
-    assert(c.size() == s);
-    assert(c == x);
+TEST_CONSTEXPR_CXX20 void test(const C& x, const typename C::allocator_type& a) {
+  typename C::size_type s = x.size();
+  C c(x, a);
+  LIBCPP_ASSERT(c.__invariants());
+  assert(c.size() == s);
+  assert(c == x);
 }
 
-TEST_CONSTEXPR_CXX20 bool tests()
-{
-    {
-        bool a[] = {0, 1, 0, 0, 1, 1, 1, 0, 0, 1, 0, 0, 0, 1, 1, 0, 1, 0};
-        bool* an = a + sizeof(a)/sizeof(a[0]);
-        test(std::vector<bool>(a, an), std::allocator<bool>());
-    }
-    {
-        std::vector<bool, test_allocator<bool> > l(3, true, test_allocator<bool>(5));
-        std::vector<bool, test_allocator<bool> > l2(l, test_allocator<bool>(3));
-        assert(l2 == l);
-        assert(l2.get_allocator() == test_allocator<bool>(3));
-    }
-    {
-        std::vector<bool, other_allocator<bool> > l(3, true, other_allocator<bool>(5));
-        std::vector<bool, other_allocator<bool> > l2(l, other_allocator<bool>(3));
-        assert(l2 == l);
-        assert(l2.get_allocator() == other_allocator<bool>(3));
-    }
+TEST_CONSTEXPR_CXX20 bool tests() {
+  {
+    bool a[] = {0, 1, 0, 0, 1, 1, 1, 0, 0, 1, 0, 0, 0, 1, 1, 0, 1, 0};
+    bool* an = a + sizeof(a) / sizeof(a[0]);
+    test(std::vector<bool>(a, an), std::allocator<bool>());
+  }
+  {
+    std::vector<bool, test_allocator<bool> > l(3, true, test_allocator<bool>(5));
+    std::vector<bool, test_allocator<bool> > l2(l, test_allocator<bool>(3));
+    assert(l2 == l);
+    assert(l2.get_allocator() == test_allocator<bool>(3));
+  }
+  {
+    std::vector<bool, other_allocator<bool> > l(3, true, other_allocator<bool>(5));
+    std::vector<bool, other_allocator<bool> > l2(l, other_allocator<bool>(3));
+    assert(l2 == l);
+    assert(l2.get_allocator() == other_allocator<bool>(3));
+  }
 #if TEST_STD_VER >= 11
-    {
-        bool a[] = {0, 1, 0, 0, 1, 1, 1, 0, 0, 1, 0, 0, 0, 1, 1, 0, 1, 0};
-        bool* an = a + sizeof(a)/sizeof(a[0]);
-        test(std::vector<bool, min_allocator<bool>>(a, an), min_allocator<bool>());
-    }
-    {
-        std::vector<bool, min_allocator<bool> > l(3, true, min_allocator<bool>());
-        std::vector<bool, min_allocator<bool> > l2(l, min_allocator<bool>());
-        assert(l2 == l);
-        assert(l2.get_allocator() == min_allocator<bool>());
-    }
+  {
+    bool a[] = {0, 1, 0, 0, 1, 1, 1, 0, 0, 1, 0, 0, 0, 1, 1, 0, 1, 0};
+    bool* an = a + sizeof(a) / sizeof(a[0]);
+    test(std::vector<bool, min_allocator<bool>>(a, an), min_allocator<bool>());
+  }
+  {
+    std::vector<bool, min_allocator<bool> > l(3, true, min_allocator<bool>());
+    std::vector<bool, min_allocator<bool> > l2(l, min_allocator<bool>());
+    assert(l2 == l);
+    assert(l2.get_allocator() == min_allocator<bool>());
+  }
 #endif
 
   return true;
 }
 
-int main(int, char**)
-{
-    tests();
+int main(int, char**) {
+  tests();
 #if TEST_STD_VER > 17
-    static_assert(tests());
+  static_assert(tests());
 #endif
-    return 0;
+  return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/vector.bool/default_noexcept.pass.cpp b/libcxx/test/std/containers/sequences/vector.bool/default_noexcept.pass.cpp
index a43b6483b0535..0801709625090 100644
--- a/libcxx/test/std/containers/sequences/vector.bool/default_noexcept.pass.cpp
+++ b/libcxx/test/std/containers/sequences/vector.bool/default_noexcept.pass.cpp
@@ -16,7 +16,6 @@
 // For vector<>, this was added to the standard by N4258,
 //   but vector<bool> was not changed.
 
-
 #include <vector>
 #include <cassert>
 
@@ -24,31 +23,29 @@
 #include "test_allocator.h"
 
 template <class T>
-struct some_alloc
-{
-    typedef T value_type;
-    some_alloc(const some_alloc&);
+struct some_alloc {
+  typedef T value_type;
+  some_alloc(const some_alloc&);
 };
 
-int main(int, char**)
-{
+int main(int, char**) {
 #if defined(_LIBCPP_VERSION)
-    {
-        typedef std::vector<bool> C;
-        static_assert(std::is_nothrow_default_constructible<C>::value, "");
-    }
-    {
-        typedef std::vector<bool, test_allocator<bool>> C;
-        static_assert(std::is_nothrow_default_constructible<C>::value, "");
-    }
-    {
-        typedef std::vector<bool, other_allocator<bool>> C;
-        static_assert(!std::is_nothrow_default_constructible<C>::value, "");
-    }
-    {
-        typedef std::vector<bool, some_alloc<bool>> C;
-        static_assert(!std::is_nothrow_default_constructible<C>::value, "");
-    }
+  {
+    typedef std::vector<bool> C;
+    static_assert(std::is_nothrow_default_constructible<C>::value, "");
+  }
+  {
+    typedef std::vector<bool, test_allocator<bool>> C;
+    static_assert(std::is_nothrow_default_constructible<C>::value, "");
+  }
+  {
+    typedef std::vector<bool, other_allocator<bool>> C;
+    static_assert(!std::is_nothrow_default_constructible<C>::value, "");
+  }
+  {
+    typedef std::vector<bool, some_alloc<bool>> C;
+    static_assert(!std::is_nothrow_default_constructible<C>::value, "");
+  }
 #endif // _LIBCPP_VERSION
 
   return 0;
diff --git a/libcxx/test/std/containers/sequences/vector.bool/dtor_noexcept.pass.cpp b/libcxx/test/std/containers/sequences/vector.bool/dtor_noexcept.pass.cpp
index 569a95f8bc9f6..f8f3c76f8c3d9 100644
--- a/libcxx/test/std/containers/sequences/vector.bool/dtor_noexcept.pass.cpp
+++ b/libcxx/test/std/containers/sequences/vector.bool/dtor_noexcept.pass.cpp
@@ -19,32 +19,30 @@
 #include "test_allocator.h"
 
 template <class T>
-struct some_alloc
-{
-    typedef T value_type;
-    some_alloc(const some_alloc&);
-    ~some_alloc() noexcept(false);
+struct some_alloc {
+  typedef T value_type;
+  some_alloc(const some_alloc&);
+  ~some_alloc() noexcept(false);
 };
 
-int main(int, char**)
-{
-    {
-        typedef std::vector<bool> C;
-        static_assert(std::is_nothrow_destructible<C>::value, "");
-    }
-    {
-        typedef std::vector<bool, test_allocator<bool>> C;
-        static_assert(std::is_nothrow_destructible<C>::value, "");
-    }
-    {
-        typedef std::vector<bool, other_allocator<bool>> C;
-        static_assert(std::is_nothrow_destructible<C>::value, "");
-    }
+int main(int, char**) {
+  {
+    typedef std::vector<bool> C;
+    static_assert(std::is_nothrow_destructible<C>::value, "");
+  }
+  {
+    typedef std::vector<bool, test_allocator<bool>> C;
+    static_assert(std::is_nothrow_destructible<C>::value, "");
+  }
+  {
+    typedef std::vector<bool, other_allocator<bool>> C;
+    static_assert(std::is_nothrow_destructible<C>::value, "");
+  }
 #if defined(_LIBCPP_VERSION)
-    {
-        typedef std::vector<bool, some_alloc<bool>> C;
-        static_assert(!std::is_nothrow_destructible<C>::value, "");
-    }
+  {
+    typedef std::vector<bool, some_alloc<bool>> C;
+    static_assert(!std::is_nothrow_destructible<C>::value, "");
+  }
 #endif // _LIBCPP_VERSION
 
   return 0;
diff --git a/libcxx/test/std/containers/sequences/vector.bool/emplace.pass.cpp b/libcxx/test/std/containers/sequences/vector.bool/emplace.pass.cpp
index 134af5a7979a1..810a1968605d8 100644
--- a/libcxx/test/std/containers/sequences/vector.bool/emplace.pass.cpp
+++ b/libcxx/test/std/containers/sequences/vector.bool/emplace.pass.cpp
@@ -17,62 +17,60 @@
 #include "test_macros.h"
 #include "min_allocator.h"
 
-TEST_CONSTEXPR_CXX20 bool tests()
-{
-    {
-        typedef std::vector<bool> C;
-        C c;
+TEST_CONSTEXPR_CXX20 bool tests() {
+  {
+    typedef std::vector<bool> C;
+    C c;
 
-        C::iterator i = c.emplace(c.cbegin());
-        assert(i == c.begin());
-        assert(c.size() == 1);
-        assert(c.front() == false);
+    C::iterator i = c.emplace(c.cbegin());
+    assert(i == c.begin());
+    assert(c.size() == 1);
+    assert(c.front() == false);
 
-        i = c.emplace(c.cend(), true);
-        assert(i == c.end()-1);
-        assert(c.size() == 2);
-        assert(c.front() == false);
-        assert(c.back() == true);
+    i = c.emplace(c.cend(), true);
+    assert(i == c.end() - 1);
+    assert(c.size() == 2);
+    assert(c.front() == false);
+    assert(c.back() == true);
 
-        i = c.emplace(c.cbegin()+1, true);
-        assert(i == c.begin()+1);
-        assert(c.size() == 3);
-        assert(c.front() == false);
-        assert(c[1] == true);
-        assert(c.back() == true);
-    }
-    {
-        typedef std::vector<bool, min_allocator<bool>> C;
-        C c;
+    i = c.emplace(c.cbegin() + 1, true);
+    assert(i == c.begin() + 1);
+    assert(c.size() == 3);
+    assert(c.front() == false);
+    assert(c[1] == true);
+    assert(c.back() == true);
+  }
+  {
+    typedef std::vector<bool, min_allocator<bool>> C;
+    C c;
 
-        C::iterator i = c.emplace(c.cbegin());
-        assert(i == c.begin());
-        assert(c.size() == 1);
-        assert(c.front() == false);
+    C::iterator i = c.emplace(c.cbegin());
+    assert(i == c.begin());
+    assert(c.size() == 1);
+    assert(c.front() == false);
 
-        i = c.emplace(c.cend(), true);
-        assert(i == c.end()-1);
-        assert(c.size() == 2);
-        assert(c.front() == false);
-        assert(c.back() == true);
+    i = c.emplace(c.cend(), true);
+    assert(i == c.end() - 1);
+    assert(c.size() == 2);
+    assert(c.front() == false);
+    assert(c.back() == true);
 
-        i = c.emplace(c.cbegin()+1, true);
-        assert(i == c.begin()+1);
-        assert(c.size() == 3);
-        assert(c.size() == 3);
-        assert(c.front() == false);
-        assert(c[1] == true);
-        assert(c.back() == true);
-    }
+    i = c.emplace(c.cbegin() + 1, true);
+    assert(i == c.begin() + 1);
+    assert(c.size() == 3);
+    assert(c.size() == 3);
+    assert(c.front() == false);
+    assert(c[1] == true);
+    assert(c.back() == true);
+  }
 
-    return true;
+  return true;
 }
 
-int main(int, char**)
-{
-    tests();
+int main(int, char**) {
+  tests();
 #if TEST_STD_VER > 17
-    static_assert(tests());
+  static_assert(tests());
 #endif
-    return 0;
+  return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/vector.bool/emplace_back.pass.cpp b/libcxx/test/std/containers/sequences/vector.bool/emplace_back.pass.cpp
index fccf6e5fb962c..87da6fd9f6019 100644
--- a/libcxx/test/std/containers/sequences/vector.bool/emplace_back.pass.cpp
+++ b/libcxx/test/std/containers/sequences/vector.bool/emplace_back.pass.cpp
@@ -18,84 +18,82 @@
 #include "test_macros.h"
 #include "min_allocator.h"
 
-TEST_CONSTEXPR_CXX20 bool tests()
-{
-    {
-        typedef std::vector<bool> C;
-        C c;
+TEST_CONSTEXPR_CXX20 bool tests() {
+  {
+    typedef std::vector<bool> C;
+    C c;
 #if TEST_STD_VER > 14
-        typedef C::reference Ref;
-        Ref r1 = c.emplace_back();
-        assert(c.size() == 1);
-        assert(c.front() == false);
-        r1 = true;
-        assert(c.front() == true);
-        r1 = false;
-        Ref r2 = c.emplace_back(true);
-        assert(c.size() == 2);
-        assert(c.front() == false);
-        assert(c.back() == true);
-        r2 = false;
-        assert(c.back() == false);
-        r2 = true;
+    typedef C::reference Ref;
+    Ref r1 = c.emplace_back();
+    assert(c.size() == 1);
+    assert(c.front() == false);
+    r1 = true;
+    assert(c.front() == true);
+    r1     = false;
+    Ref r2 = c.emplace_back(true);
+    assert(c.size() == 2);
+    assert(c.front() == false);
+    assert(c.back() == true);
+    r2 = false;
+    assert(c.back() == false);
+    r2 = true;
 #else
-        c.emplace_back();
-        assert(c.size() == 1);
-        assert(c.front() == false);
-        c.emplace_back(true);
-        assert(c.size() == 2);
-        assert(c.front() == false);
-        assert(c.back() == true);
+    c.emplace_back();
+    assert(c.size() == 1);
+    assert(c.front() == false);
+    c.emplace_back(true);
+    assert(c.size() == 2);
+    assert(c.front() == false);
+    assert(c.back() == true);
 #endif
-        c.emplace_back(true);
-        assert(c.size() == 3);
-        assert(c.front() == false);
-        assert(c[1] == true);
-        assert(c.back() == true);
-    }
-    {
-        typedef std::vector<bool, min_allocator<bool>> C;
-        C c;
+    c.emplace_back(true);
+    assert(c.size() == 3);
+    assert(c.front() == false);
+    assert(c[1] == true);
+    assert(c.back() == true);
+  }
+  {
+    typedef std::vector<bool, min_allocator<bool>> C;
+    C c;
 
 #if TEST_STD_VER > 14
-        typedef C::reference Ref;
-        Ref r1 = c.emplace_back();
-        assert(c.size() == 1);
-        assert(c.front() == false);
-        r1 = true;
-        assert(c.front() == true);
-        r1 = false;
-        Ref r2 = c.emplace_back(true);
-        assert(c.size() == 2);
-        assert(c.front() == false);
-        assert(c.back() == true);
-        r2 = false;
-        assert(c.back() == false);
-        r2 = true;
+    typedef C::reference Ref;
+    Ref r1 = c.emplace_back();
+    assert(c.size() == 1);
+    assert(c.front() == false);
+    r1 = true;
+    assert(c.front() == true);
+    r1     = false;
+    Ref r2 = c.emplace_back(true);
+    assert(c.size() == 2);
+    assert(c.front() == false);
+    assert(c.back() == true);
+    r2 = false;
+    assert(c.back() == false);
+    r2 = true;
 #else
-        c.emplace_back();
-        assert(c.size() == 1);
-        assert(c.front() == false);
-        c.emplace_back(true);
-        assert(c.size() == 2);
-        assert(c.front() == false);
-        assert(c.back() == true);
+    c.emplace_back();
+    assert(c.size() == 1);
+    assert(c.front() == false);
+    c.emplace_back(true);
+    assert(c.size() == 2);
+    assert(c.front() == false);
+    assert(c.back() == true);
 #endif
-        c.emplace_back(true);
-        assert(c.size() == 3);
-        assert(c.front() == false);
-        assert(c[1] == true);
-        assert(c.back() == true);
-    }
+    c.emplace_back(true);
+    assert(c.size() == 3);
+    assert(c.front() == false);
+    assert(c[1] == true);
+    assert(c.back() == true);
+  }
 
-    return true;
+  return true;
 }
 
-int main(int, char**)
-{
-    tests();
+int main(int, char**) {
+  tests();
 #if TEST_STD_VER > 17
-    static_assert(tests());
+  static_assert(tests());
 #endif
-    return 0;
+  return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/vector.bool/empty.pass.cpp b/libcxx/test/std/containers/sequences/vector.bool/empty.pass.cpp
index ec4d2f97045dd..806a04cf061ad 100644
--- a/libcxx/test/std/containers/sequences/vector.bool/empty.pass.cpp
+++ b/libcxx/test/std/containers/sequences/vector.bool/empty.pass.cpp
@@ -18,9 +18,8 @@
 #include "test_macros.h"
 #include "min_allocator.h"
 
-TEST_CONSTEXPR_CXX20 bool tests()
-{
-    {
+TEST_CONSTEXPR_CXX20 bool tests() {
+  {
     typedef std::vector<bool> C;
     C c;
     ASSERT_NOEXCEPT(c.empty());
@@ -29,9 +28,9 @@ TEST_CONSTEXPR_CXX20 bool tests()
     assert(!c.empty());
     c.clear();
     assert(c.empty());
-    }
+  }
 #if TEST_STD_VER >= 11
-    {
+  {
     typedef std::vector<bool, min_allocator<bool>> C;
     C c;
     ASSERT_NOEXCEPT(c.empty());
@@ -40,17 +39,16 @@ TEST_CONSTEXPR_CXX20 bool tests()
     assert(!c.empty());
     c.clear();
     assert(c.empty());
-    }
+  }
 #endif
 
-    return true;
+  return true;
 }
 
-int main(int, char**)
-{
-    tests();
+int main(int, char**) {
+  tests();
 #if TEST_STD_VER > 17
-    static_assert(tests());
+  static_assert(tests());
 #endif
-    return 0;
+  return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/vector.bool/empty.verify.cpp b/libcxx/test/std/containers/sequences/vector.bool/empty.verify.cpp
index 167e095b798f4..71b17ff68d192 100644
--- a/libcxx/test/std/containers/sequences/vector.bool/empty.verify.cpp
+++ b/libcxx/test/std/containers/sequences/vector.bool/empty.verify.cpp
@@ -17,6 +17,6 @@
 #include <vector>
 
 void f() {
-    std::vector<bool> c;
-    c.empty(); // expected-warning {{ignoring return value of function declared with 'nodiscard' attribute}}
+  std::vector<bool> c;
+  c.empty(); // expected-warning {{ignoring return value of function declared with 'nodiscard' attribute}}
 }
diff --git a/libcxx/test/std/containers/sequences/vector.bool/enabled_hash.pass.cpp b/libcxx/test/std/containers/sequences/vector.bool/enabled_hash.pass.cpp
index 66361202f8c00..41cedd68fe50e 100644
--- a/libcxx/test/std/containers/sequences/vector.bool/enabled_hash.pass.cpp
+++ b/libcxx/test/std/containers/sequences/vector.bool/enabled_hash.pass.cpp
@@ -30,7 +30,7 @@ int main(int, char**) {
   test_library_hash_specializations_available();
   test();
 #if TEST_STD_VER > 17
-    static_assert(test());
+  static_assert(test());
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/sequences/vector.bool/erase_iter.pass.cpp b/libcxx/test/std/containers/sequences/vector.bool/erase_iter.pass.cpp
index 42d1e76a9c770..2479d4c048013 100644
--- a/libcxx/test/std/containers/sequences/vector.bool/erase_iter.pass.cpp
+++ b/libcxx/test/std/containers/sequences/vector.bool/erase_iter.pass.cpp
@@ -18,11 +18,10 @@
 #include "test_macros.h"
 #include "min_allocator.h"
 
-TEST_CONSTEXPR_CXX20 bool tests()
-{
-    bool a1[] = {1, 0, 1};
-    {
-    std::vector<bool> l1(a1, a1+3);
+TEST_CONSTEXPR_CXX20 bool tests() {
+  bool a1[] = {1, 0, 1};
+  {
+    std::vector<bool> l1(a1, a1 + 3);
     std::vector<bool>::const_iterator i = l1.begin();
     ++i;
     std::vector<bool>::iterator j = l1.erase(i);
@@ -40,10 +39,10 @@ TEST_CONSTEXPR_CXX20 bool tests()
     assert(j == l1.end());
     assert(l1.size() == 0);
     assert(std::distance(l1.begin(), l1.end()) == 0);
-    }
+  }
 #if TEST_STD_VER >= 11
-    {
-    std::vector<bool, min_allocator<bool>> l1(a1, a1+3);
+  {
+    std::vector<bool, min_allocator<bool>> l1(a1, a1 + 3);
     std::vector<bool, min_allocator<bool>>::const_iterator i = l1.begin();
     ++i;
     std::vector<bool, min_allocator<bool>>::iterator j = l1.erase(i);
@@ -61,17 +60,16 @@ TEST_CONSTEXPR_CXX20 bool tests()
     assert(j == l1.end());
     assert(l1.size() == 0);
     assert(std::distance(l1.begin(), l1.end()) == 0);
-    }
+  }
 #endif
 
-    return true;
+  return true;
 }
 
-int main(int, char**)
-{
-    tests();
+int main(int, char**) {
+  tests();
 #if TEST_STD_VER > 17
-    static_assert(tests());
+  static_assert(tests());
 #endif
-    return 0;
+  return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/vector.bool/erase_iter_iter.pass.cpp b/libcxx/test/std/containers/sequences/vector.bool/erase_iter_iter.pass.cpp
index c19ce9987e1e1..1afd481b40efe 100644
--- a/libcxx/test/std/containers/sequences/vector.bool/erase_iter_iter.pass.cpp
+++ b/libcxx/test/std/containers/sequences/vector.bool/erase_iter_iter.pass.cpp
@@ -18,80 +18,78 @@
 #include "test_macros.h"
 #include "min_allocator.h"
 
-TEST_CONSTEXPR_CXX20 bool tests()
-{
-    bool a1[] = {1, 0, 1};
-    {
-        std::vector<bool> l1(a1, a1+3);
-        std::vector<bool>::iterator i = l1.erase(l1.cbegin(), l1.cbegin());
-        assert(l1.size() == 3);
-        assert(std::distance(l1.cbegin(), l1.cend()) == 3);
-        assert(i == l1.begin());
-    }
-    {
-        std::vector<bool> l1(a1, a1+3);
-        std::vector<bool>::iterator i = l1.erase(l1.cbegin(), std::next(l1.cbegin()));
-        assert(l1.size() == 2);
-        assert(std::distance(l1.cbegin(), l1.cend()) == 2);
-        assert(i == l1.begin());
-        assert(l1 == std::vector<bool>(a1+1, a1+3));
-    }
-    {
-        std::vector<bool> l1(a1, a1+3);
-        std::vector<bool>::iterator i = l1.erase(l1.cbegin(), std::next(l1.cbegin(), 2));
-        assert(l1.size() == 1);
-        assert(std::distance(l1.cbegin(), l1.cend()) == 1);
-        assert(i == l1.begin());
-        assert(l1 == std::vector<bool>(a1+2, a1+3));
-    }
-    {
-        std::vector<bool> l1(a1, a1+3);
-        std::vector<bool>::iterator i = l1.erase(l1.cbegin(), std::next(l1.cbegin(), 3));
-        assert(l1.size() == 0);
-        assert(std::distance(l1.cbegin(), l1.cend()) == 0);
-        assert(i == l1.begin());
-    }
+TEST_CONSTEXPR_CXX20 bool tests() {
+  bool a1[] = {1, 0, 1};
+  {
+    std::vector<bool> l1(a1, a1 + 3);
+    std::vector<bool>::iterator i = l1.erase(l1.cbegin(), l1.cbegin());
+    assert(l1.size() == 3);
+    assert(std::distance(l1.cbegin(), l1.cend()) == 3);
+    assert(i == l1.begin());
+  }
+  {
+    std::vector<bool> l1(a1, a1 + 3);
+    std::vector<bool>::iterator i = l1.erase(l1.cbegin(), std::next(l1.cbegin()));
+    assert(l1.size() == 2);
+    assert(std::distance(l1.cbegin(), l1.cend()) == 2);
+    assert(i == l1.begin());
+    assert(l1 == std::vector<bool>(a1 + 1, a1 + 3));
+  }
+  {
+    std::vector<bool> l1(a1, a1 + 3);
+    std::vector<bool>::iterator i = l1.erase(l1.cbegin(), std::next(l1.cbegin(), 2));
+    assert(l1.size() == 1);
+    assert(std::distance(l1.cbegin(), l1.cend()) == 1);
+    assert(i == l1.begin());
+    assert(l1 == std::vector<bool>(a1 + 2, a1 + 3));
+  }
+  {
+    std::vector<bool> l1(a1, a1 + 3);
+    std::vector<bool>::iterator i = l1.erase(l1.cbegin(), std::next(l1.cbegin(), 3));
+    assert(l1.size() == 0);
+    assert(std::distance(l1.cbegin(), l1.cend()) == 0);
+    assert(i == l1.begin());
+  }
 #if TEST_STD_VER >= 11
-    {
-        std::vector<bool, min_allocator<bool>> l1(a1, a1+3);
-        std::vector<bool, min_allocator<bool>>::iterator i = l1.erase(l1.cbegin(), l1.cbegin());
-        assert(l1.size() == 3);
-        assert(std::distance(l1.cbegin(), l1.cend()) == 3);
-        assert(i == l1.begin());
-    }
-    {
-        std::vector<bool, min_allocator<bool>> l1(a1, a1+3);
-        std::vector<bool, min_allocator<bool>>::iterator i = l1.erase(l1.cbegin(), std::next(l1.cbegin()));
-        assert(l1.size() == 2);
-        assert(std::distance(l1.cbegin(), l1.cend()) == 2);
-        assert(i == l1.begin());
-        assert((l1 == std::vector<bool, min_allocator<bool>>(a1+1, a1+3)));
-    }
-    {
-        std::vector<bool, min_allocator<bool>> l1(a1, a1+3);
-        std::vector<bool, min_allocator<bool>>::iterator i = l1.erase(l1.cbegin(), std::next(l1.cbegin(), 2));
-        assert(l1.size() == 1);
-        assert(std::distance(l1.cbegin(), l1.cend()) == 1);
-        assert(i == l1.begin());
-        assert((l1 == std::vector<bool, min_allocator<bool>>(a1+2, a1+3)));
-    }
-    {
-        std::vector<bool, min_allocator<bool>> l1(a1, a1+3);
-        std::vector<bool, min_allocator<bool>>::iterator i = l1.erase(l1.cbegin(), std::next(l1.cbegin(), 3));
-        assert(l1.size() == 0);
-        assert(std::distance(l1.cbegin(), l1.cend()) == 0);
-        assert(i == l1.begin());
-    }
+  {
+    std::vector<bool, min_allocator<bool>> l1(a1, a1 + 3);
+    std::vector<bool, min_allocator<bool>>::iterator i = l1.erase(l1.cbegin(), l1.cbegin());
+    assert(l1.size() == 3);
+    assert(std::distance(l1.cbegin(), l1.cend()) == 3);
+    assert(i == l1.begin());
+  }
+  {
+    std::vector<bool, min_allocator<bool>> l1(a1, a1 + 3);
+    std::vector<bool, min_allocator<bool>>::iterator i = l1.erase(l1.cbegin(), std::next(l1.cbegin()));
+    assert(l1.size() == 2);
+    assert(std::distance(l1.cbegin(), l1.cend()) == 2);
+    assert(i == l1.begin());
+    assert((l1 == std::vector<bool, min_allocator<bool>>(a1 + 1, a1 + 3)));
+  }
+  {
+    std::vector<bool, min_allocator<bool>> l1(a1, a1 + 3);
+    std::vector<bool, min_allocator<bool>>::iterator i = l1.erase(l1.cbegin(), std::next(l1.cbegin(), 2));
+    assert(l1.size() == 1);
+    assert(std::distance(l1.cbegin(), l1.cend()) == 1);
+    assert(i == l1.begin());
+    assert((l1 == std::vector<bool, min_allocator<bool>>(a1 + 2, a1 + 3)));
+  }
+  {
+    std::vector<bool, min_allocator<bool>> l1(a1, a1 + 3);
+    std::vector<bool, min_allocator<bool>>::iterator i = l1.erase(l1.cbegin(), std::next(l1.cbegin(), 3));
+    assert(l1.size() == 0);
+    assert(std::distance(l1.cbegin(), l1.cend()) == 0);
+    assert(i == l1.begin());
+  }
 #endif
 
-    return true;
+  return true;
 }
 
-int main(int, char**)
-{
-    tests();
+int main(int, char**) {
+  tests();
 #if TEST_STD_VER > 17
-    static_assert(tests());
+  static_assert(tests());
 #endif
-    return 0;
+  return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/vector.bool/find.pass.cpp b/libcxx/test/std/containers/sequences/vector.bool/find.pass.cpp
index 73cca82e79f11..744496fcc0234 100644
--- a/libcxx/test/std/containers/sequences/vector.bool/find.pass.cpp
+++ b/libcxx/test/std/containers/sequences/vector.bool/find.pass.cpp
@@ -20,35 +20,31 @@
 
 #include "test_macros.h"
 
-TEST_CONSTEXPR_CXX20 bool tests()
-{
-    {
-        for (unsigned i = 1; i < 256; ++i)
-        {
-            std::vector<bool> b(i,true);
-            std::vector<bool>::iterator j = std::find(b.begin()+1, b.end(), false);
-            assert(static_cast<std::size_t>(j-b.begin()) == i);
-            assert(b.end() == j);
-        }
+TEST_CONSTEXPR_CXX20 bool tests() {
+  {
+    for (unsigned i = 1; i < 256; ++i) {
+      std::vector<bool> b(i, true);
+      std::vector<bool>::iterator j = std::find(b.begin() + 1, b.end(), false);
+      assert(static_cast<std::size_t>(j - b.begin()) == i);
+      assert(b.end() == j);
     }
-    {
-        for (unsigned i = 1; i < 256; ++i)
-        {
-            std::vector<bool> b(i,false);
-            std::vector<bool>::iterator j = std::find(b.begin()+1, b.end(), true);
-            assert(static_cast<std::size_t>(j-b.begin()) == i);
-            assert(b.end() == j);
-        }
+  }
+  {
+    for (unsigned i = 1; i < 256; ++i) {
+      std::vector<bool> b(i, false);
+      std::vector<bool>::iterator j = std::find(b.begin() + 1, b.end(), true);
+      assert(static_cast<std::size_t>(j - b.begin()) == i);
+      assert(b.end() == j);
     }
+  }
 
-    return true;
+  return true;
 }
 
-int main(int, char**)
-{
-    tests();
+int main(int, char**) {
+  tests();
 #if TEST_STD_VER > 17
-    static_assert(tests());
+  static_assert(tests());
 #endif
-    return 0;
+  return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/vector.bool/get_allocator.pass.cpp b/libcxx/test/std/containers/sequences/vector.bool/get_allocator.pass.cpp
index a088cd8119abc..efd4310f9f88d 100644
--- a/libcxx/test/std/containers/sequences/vector.bool/get_allocator.pass.cpp
+++ b/libcxx/test/std/containers/sequences/vector.bool/get_allocator.pass.cpp
@@ -19,25 +19,25 @@
 #include "test_macros.h"
 
 TEST_CONSTEXPR_CXX20 bool test() {
-    {
-        std::allocator<bool> alloc;
-        const std::vector<bool> vb(alloc);
-        assert(vb.get_allocator() == alloc);
-    }
-    {
-        other_allocator<bool> alloc(1);
-        const std::vector<bool, other_allocator<bool> > vb(alloc);
-        assert(vb.get_allocator() == alloc);
-    }
-
-    return true;
+  {
+    std::allocator<bool> alloc;
+    const std::vector<bool> vb(alloc);
+    assert(vb.get_allocator() == alloc);
+  }
+  {
+    other_allocator<bool> alloc(1);
+    const std::vector<bool, other_allocator<bool> > vb(alloc);
+    assert(vb.get_allocator() == alloc);
+  }
+
+  return true;
 }
 
 int main(int, char**) {
   test();
 #if TEST_STD_VER > 17
-    static_assert(test());
+  static_assert(test());
 #endif
 
-    return 0;
+  return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/vector.bool/initializer_list.pass.cpp b/libcxx/test/std/containers/sequences/vector.bool/initializer_list.pass.cpp
index 9b13922240d57..83ca3efd51984 100644
--- a/libcxx/test/std/containers/sequences/vector.bool/initializer_list.pass.cpp
+++ b/libcxx/test/std/containers/sequences/vector.bool/initializer_list.pass.cpp
@@ -18,33 +18,31 @@
 #include "test_macros.h"
 #include "min_allocator.h"
 
-TEST_CONSTEXPR_CXX20 bool tests()
-{
-    {
+TEST_CONSTEXPR_CXX20 bool tests() {
+  {
     std::vector<bool> d = {true, false, false, true};
     assert(d.size() == 4);
     assert(d[0] == true);
     assert(d[1] == false);
     assert(d[2] == false);
     assert(d[3] == true);
-    }
-    {
+  }
+  {
     std::vector<bool, min_allocator<bool>> d = {true, false, false, true};
     assert(d.size() == 4);
     assert(d[0] == true);
     assert(d[1] == false);
     assert(d[2] == false);
     assert(d[3] == true);
-    }
+  }
 
-    return true;
+  return true;
 }
 
-int main(int, char**)
-{
-    tests();
+int main(int, char**) {
+  tests();
 #if TEST_STD_VER > 17
-    static_assert(tests());
+  static_assert(tests());
 #endif
-    return 0;
+  return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/vector.bool/initializer_list_alloc.pass.cpp b/libcxx/test/std/containers/sequences/vector.bool/initializer_list_alloc.pass.cpp
index 0d5dd8a79fc53..b26c62c647f0a 100644
--- a/libcxx/test/std/containers/sequences/vector.bool/initializer_list_alloc.pass.cpp
+++ b/libcxx/test/std/containers/sequences/vector.bool/initializer_list_alloc.pass.cpp
@@ -19,9 +19,8 @@
 #include "test_allocator.h"
 #include "min_allocator.h"
 
-TEST_CONSTEXPR_CXX20 bool tests()
-{
-    {
+TEST_CONSTEXPR_CXX20 bool tests() {
+  {
     std::vector<bool, test_allocator<bool>> d({true, false, false, true}, test_allocator<bool>(3));
     assert(d.get_allocator() == test_allocator<bool>(3));
     assert(d.size() == 4);
@@ -29,8 +28,8 @@ TEST_CONSTEXPR_CXX20 bool tests()
     assert(d[1] == false);
     assert(d[2] == false);
     assert(d[3] == true);
-    }
-    {
+  }
+  {
     std::vector<bool, min_allocator<bool>> d({true, false, false, true}, min_allocator<bool>());
     assert(d.get_allocator() == min_allocator<bool>());
     assert(d.size() == 4);
@@ -38,16 +37,15 @@ TEST_CONSTEXPR_CXX20 bool tests()
     assert(d[1] == false);
     assert(d[2] == false);
     assert(d[3] == true);
-    }
+  }
 
-    return true;
+  return true;
 }
 
-int main(int, char**)
-{
-    tests();
+int main(int, char**) {
+  tests();
 #if TEST_STD_VER > 17
-    static_assert(tests());
+  static_assert(tests());
 #endif
-    return 0;
+  return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/vector.bool/insert_iter_initializer_list.pass.cpp b/libcxx/test/std/containers/sequences/vector.bool/insert_iter_initializer_list.pass.cpp
index 2462e05f20360..2642770675b4e 100644
--- a/libcxx/test/std/containers/sequences/vector.bool/insert_iter_initializer_list.pass.cpp
+++ b/libcxx/test/std/containers/sequences/vector.bool/insert_iter_initializer_list.pass.cpp
@@ -18,9 +18,8 @@
 #include "test_macros.h"
 #include "min_allocator.h"
 
-TEST_CONSTEXPR_CXX20 bool tests()
-{
-    {
+TEST_CONSTEXPR_CXX20 bool tests() {
+  {
     std::vector<bool> d(10, true);
     std::vector<bool>::iterator i = d.insert(d.cbegin() + 2, {false, true, true, false});
     assert(d.size() == 14);
@@ -39,8 +38,8 @@ TEST_CONSTEXPR_CXX20 bool tests()
     assert(d[11] == true);
     assert(d[12] == true);
     assert(d[13] == true);
-    }
-    {
+  }
+  {
     std::vector<bool, min_allocator<bool>> d(10, true);
     std::vector<bool, min_allocator<bool>>::iterator i = d.insert(d.cbegin() + 2, {false, true, true, false});
     assert(d.size() == 14);
@@ -59,16 +58,15 @@ TEST_CONSTEXPR_CXX20 bool tests()
     assert(d[11] == true);
     assert(d[12] == true);
     assert(d[13] == true);
-    }
+  }
 
-    return true;
+  return true;
 }
 
-int main(int, char**)
-{
-    tests();
+int main(int, char**) {
+  tests();
 #if TEST_STD_VER > 17
-    static_assert(tests());
+  static_assert(tests());
 #endif
-    return 0;
+  return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/vector.bool/insert_iter_iter_iter.pass.cpp b/libcxx/test/std/containers/sequences/vector.bool/insert_iter_iter_iter.pass.cpp
index d3e1297aeec92..3ab437c9a1546 100644
--- a/libcxx/test/std/containers/sequences/vector.bool/insert_iter_iter_iter.pass.cpp
+++ b/libcxx/test/std/containers/sequences/vector.bool/insert_iter_iter_iter.pass.cpp
@@ -20,136 +20,138 @@
 #include "test_iterators.h"
 #include "min_allocator.h"
 
-TEST_CONSTEXPR_CXX20 bool tests()
-{
-    {
-        std::vector<bool> v(100);
-        bool a[] = {1, 0, 0, 1, 1};
-        const unsigned N = sizeof(a)/sizeof(a[0]);
-        std::vector<bool>::iterator i = v.insert(v.cbegin() + 10, cpp17_input_iterator<const bool*>(a),
-                                        cpp17_input_iterator<const bool*>(a+N));
-        assert(v.size() == 100 + N);
-        assert(i == v.begin() + 10);
-        std::size_t j;
-        for (j = 0; j < 10; ++j)
-            assert(v[j] == 0);
-        for (std::size_t k = 0; k < N; ++j, ++k)
-            assert(v[j] == a[k]);
-        for (; j < v.size(); ++j)
-            assert(v[j] == 0);
-    }
-    {
-        std::vector<bool> v(100);
-        bool a[] = {1, 0, 0, 1, 1};
-        const unsigned N = sizeof(a)/sizeof(a[0]);
-        std::vector<bool>::iterator i = v.insert(v.cbegin() + 10, forward_iterator<const bool*>(a),
-                                        forward_iterator<const bool*>(a+N));
-        assert(v.size() == 100 + N);
-        assert(i == v.begin() + 10);
-        int j;
-        for (j = 0; j < 10; ++j)
-            assert(v[j] == 0);
-        for (std::size_t k = 0; k < N; ++j, ++k)
-            assert(v[j] == a[k]);
-        for (; j < 105; ++j)
-            assert(v[j] == 0);
-    }
-    {
-        std::vector<bool> v(100);
-        while(v.size() < v.capacity()) v.push_back(false);
-        std::size_t sz = v.size();
-        bool a[] = {1, 0, 0, 1, 1};
-        const unsigned N = sizeof(a)/sizeof(a[0]);
-        std::vector<bool>::iterator i = v.insert(v.cbegin() + 10, forward_iterator<const bool*>(a),
-                                        forward_iterator<const bool*>(a+N));
-        assert(v.size() == sz + N);
-        assert(i == v.begin() + 10);
-        std::size_t j;
-        for (j = 0; j < 10; ++j)
-            assert(v[j] == 0);
-        for (std::size_t k = 0; k < N; ++j, ++k)
-            assert(v[j] == a[k]);
-        for (; j < v.size(); ++j)
-            assert(v[j] == 0);
-    }
-    {
-        std::vector<bool> v(100);
-        while(v.size() < v.capacity()) v.push_back(false);
-        v.pop_back(); v.pop_back(); v.pop_back();
-        std::size_t sz = v.size();
-        bool a[] = {1, 0, 0, 1, 1};
-        const unsigned N = sizeof(a)/sizeof(a[0]);
-        std::vector<bool>::iterator i = v.insert(v.cbegin() + 10, forward_iterator<const bool*>(a),
-                                        forward_iterator<const bool*>(a+N));
-        assert(v.size() == sz + N);
-        assert(i == v.begin() + 10);
-        std::size_t j;
-        for (j = 0; j < 10; ++j)
-            assert(v[j] == 0);
-        for (std::size_t k = 0; k < N; ++j, ++k)
-            assert(v[j] == a[k]);
-        for (; j < v.size(); ++j)
-            assert(v[j] == 0);
-    }
+TEST_CONSTEXPR_CXX20 bool tests() {
+  {
+    std::vector<bool> v(100);
+    bool a[]         = {1, 0, 0, 1, 1};
+    const unsigned N = sizeof(a) / sizeof(a[0]);
+    std::vector<bool>::iterator i =
+        v.insert(v.cbegin() + 10, cpp17_input_iterator<const bool*>(a), cpp17_input_iterator<const bool*>(a + N));
+    assert(v.size() == 100 + N);
+    assert(i == v.begin() + 10);
+    std::size_t j;
+    for (j = 0; j < 10; ++j)
+      assert(v[j] == 0);
+    for (std::size_t k = 0; k < N; ++j, ++k)
+      assert(v[j] == a[k]);
+    for (; j < v.size(); ++j)
+      assert(v[j] == 0);
+  }
+  {
+    std::vector<bool> v(100);
+    bool a[]         = {1, 0, 0, 1, 1};
+    const unsigned N = sizeof(a) / sizeof(a[0]);
+    std::vector<bool>::iterator i =
+        v.insert(v.cbegin() + 10, forward_iterator<const bool*>(a), forward_iterator<const bool*>(a + N));
+    assert(v.size() == 100 + N);
+    assert(i == v.begin() + 10);
+    int j;
+    for (j = 0; j < 10; ++j)
+      assert(v[j] == 0);
+    for (std::size_t k = 0; k < N; ++j, ++k)
+      assert(v[j] == a[k]);
+    for (; j < 105; ++j)
+      assert(v[j] == 0);
+  }
+  {
+    std::vector<bool> v(100);
+    while (v.size() < v.capacity())
+      v.push_back(false);
+    std::size_t sz   = v.size();
+    bool a[]         = {1, 0, 0, 1, 1};
+    const unsigned N = sizeof(a) / sizeof(a[0]);
+    std::vector<bool>::iterator i =
+        v.insert(v.cbegin() + 10, forward_iterator<const bool*>(a), forward_iterator<const bool*>(a + N));
+    assert(v.size() == sz + N);
+    assert(i == v.begin() + 10);
+    std::size_t j;
+    for (j = 0; j < 10; ++j)
+      assert(v[j] == 0);
+    for (std::size_t k = 0; k < N; ++j, ++k)
+      assert(v[j] == a[k]);
+    for (; j < v.size(); ++j)
+      assert(v[j] == 0);
+  }
+  {
+    std::vector<bool> v(100);
+    while (v.size() < v.capacity())
+      v.push_back(false);
+    v.pop_back();
+    v.pop_back();
+    v.pop_back();
+    std::size_t sz   = v.size();
+    bool a[]         = {1, 0, 0, 1, 1};
+    const unsigned N = sizeof(a) / sizeof(a[0]);
+    std::vector<bool>::iterator i =
+        v.insert(v.cbegin() + 10, forward_iterator<const bool*>(a), forward_iterator<const bool*>(a + N));
+    assert(v.size() == sz + N);
+    assert(i == v.begin() + 10);
+    std::size_t j;
+    for (j = 0; j < 10; ++j)
+      assert(v[j] == 0);
+    for (std::size_t k = 0; k < N; ++j, ++k)
+      assert(v[j] == a[k]);
+    for (; j < v.size(); ++j)
+      assert(v[j] == 0);
+  }
 #if TEST_STD_VER >= 11
-    {
-        std::vector<bool, explicit_allocator<bool>> v(100);
-        bool a[] = {1, 0, 0, 1, 1};
-        const unsigned N = sizeof(a)/sizeof(a[0]);
-        std::vector<bool, explicit_allocator<bool>>::iterator i = v.insert(v.cbegin() + 10, cpp17_input_iterator<const bool*>(a),
-                                        cpp17_input_iterator<const bool*>(a+N));
-        assert(v.size() == 100 + N);
-        assert(i == v.begin() + 10);
-        std::size_t j;
-        for (j = 0; j < 10; ++j)
-            assert(v[j] == 0);
-        for (std::size_t k = 0; k < N; ++j, ++k)
-            assert(v[j] == a[k]);
-        for (; j < v.size(); ++j)
-            assert(v[j] == 0);
-    }
-    {
-        std::vector<bool, min_allocator<bool>> v(100);
-        bool a[] = {1, 0, 0, 1, 1};
-        const unsigned N = sizeof(a)/sizeof(a[0]);
-        std::vector<bool, min_allocator<bool>>::iterator i = v.insert(v.cbegin() + 10, cpp17_input_iterator<const bool*>(a),
-                                        cpp17_input_iterator<const bool*>(a+N));
-        assert(v.size() == 100 + N);
-        assert(i == v.begin() + 10);
-        std::size_t j;
-        for (j = 0; j < 10; ++j)
-            assert(v[j] == 0);
-        for (std::size_t k = 0; k < N; ++j, ++k)
-            assert(v[j] == a[k]);
-        for (; j < v.size(); ++j)
-            assert(v[j] == 0);
-    }
-    {
-        std::vector<bool, min_allocator<bool>> v(100);
-        bool a[] = {1, 0, 0, 1, 1};
-        const unsigned N = sizeof(a)/sizeof(a[0]);
-        std::vector<bool, min_allocator<bool>>::iterator i = v.insert(v.cbegin() + 10, forward_iterator<const bool*>(a),
-                                        forward_iterator<const bool*>(a+N));
-        assert(v.size() == 100 + N);
-        assert(i == v.begin() + 10);
-        std::size_t j;
-        for (j = 0; j < 10; ++j)
-            assert(v[j] == 0);
-        for (std::size_t k = 0; k < N; ++j, ++k)
-            assert(v[j] == a[k]);
-        for (; j < v.size(); ++j)
-            assert(v[j] == 0);
-    }
+  {
+    std::vector<bool, explicit_allocator<bool>> v(100);
+    bool a[]         = {1, 0, 0, 1, 1};
+    const unsigned N = sizeof(a) / sizeof(a[0]);
+    std::vector<bool, explicit_allocator<bool>>::iterator i =
+        v.insert(v.cbegin() + 10, cpp17_input_iterator<const bool*>(a), cpp17_input_iterator<const bool*>(a + N));
+    assert(v.size() == 100 + N);
+    assert(i == v.begin() + 10);
+    std::size_t j;
+    for (j = 0; j < 10; ++j)
+      assert(v[j] == 0);
+    for (std::size_t k = 0; k < N; ++j, ++k)
+      assert(v[j] == a[k]);
+    for (; j < v.size(); ++j)
+      assert(v[j] == 0);
+  }
+  {
+    std::vector<bool, min_allocator<bool>> v(100);
+    bool a[]         = {1, 0, 0, 1, 1};
+    const unsigned N = sizeof(a) / sizeof(a[0]);
+    std::vector<bool, min_allocator<bool>>::iterator i =
+        v.insert(v.cbegin() + 10, cpp17_input_iterator<const bool*>(a), cpp17_input_iterator<const bool*>(a + N));
+    assert(v.size() == 100 + N);
+    assert(i == v.begin() + 10);
+    std::size_t j;
+    for (j = 0; j < 10; ++j)
+      assert(v[j] == 0);
+    for (std::size_t k = 0; k < N; ++j, ++k)
+      assert(v[j] == a[k]);
+    for (; j < v.size(); ++j)
+      assert(v[j] == 0);
+  }
+  {
+    std::vector<bool, min_allocator<bool>> v(100);
+    bool a[]         = {1, 0, 0, 1, 1};
+    const unsigned N = sizeof(a) / sizeof(a[0]);
+    std::vector<bool, min_allocator<bool>>::iterator i =
+        v.insert(v.cbegin() + 10, forward_iterator<const bool*>(a), forward_iterator<const bool*>(a + N));
+    assert(v.size() == 100 + N);
+    assert(i == v.begin() + 10);
+    std::size_t j;
+    for (j = 0; j < 10; ++j)
+      assert(v[j] == 0);
+    for (std::size_t k = 0; k < N; ++j, ++k)
+      assert(v[j] == a[k]);
+    for (; j < v.size(); ++j)
+      assert(v[j] == 0);
+  }
 #endif
 
-    return true;
+  return true;
 }
 
-int main(int, char**)
-{
-    tests();
+int main(int, char**) {
+  tests();
 #if TEST_STD_VER > 17
-    static_assert(tests());
+  static_assert(tests());
 #endif
-    return 0;
+  return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/vector.bool/insert_iter_size_value.pass.cpp b/libcxx/test/std/containers/sequences/vector.bool/insert_iter_size_value.pass.cpp
index 484c57eadb653..51481fbea1daf 100644
--- a/libcxx/test/std/containers/sequences/vector.bool/insert_iter_size_value.pass.cpp
+++ b/libcxx/test/std/containers/sequences/vector.bool/insert_iter_size_value.pass.cpp
@@ -18,83 +18,83 @@
 #include "test_macros.h"
 #include "min_allocator.h"
 
-TEST_CONSTEXPR_CXX20 bool tests()
-{
-    {
-        std::vector<bool> v(100);
-        std::vector<bool>::iterator i = v.insert(v.cbegin() + 10, 5, 1);
-        assert(v.size() == 105);
-        assert(i == v.begin() + 10);
-        std::size_t j;
-        for (j = 0; j < 10; ++j)
-            assert(v[j] == 0);
-        for (; j < 15; ++j)
-            assert(v[j] == 1);
-        for (++j; j < v.size(); ++j)
-            assert(v[j] == 0);
-    }
-    {
-        std::vector<bool> v(100);
-        while(v.size() < v.capacity()) v.push_back(false);
-        std::size_t sz = v.size();
-        std::vector<bool>::iterator i = v.insert(v.cbegin() + 10, 5, 1);
-        assert(v.size() == sz + 5);
-        assert(i == v.begin() + 10);
-        std::size_t j;
-        for (j = 0; j < 10; ++j)
-            assert(v[j] == 0);
-        for (; j < 15; ++j)
-            assert(v[j] == 1);
-        for (++j; j < v.size(); ++j)
-            assert(v[j] == 0);
-    }
-    {
-        std::vector<bool> v(100);
-        while(v.size() < v.capacity()) v.push_back(false);
-        v.pop_back(); v.pop_back();
-        std::size_t sz = v.size();
-        std::vector<bool>::iterator i = v.insert(v.cbegin() + 10, 5, 1);
-        assert(v.size() == sz + 5);
-        assert(i == v.begin() + 10);
-        std::size_t j;
-        for (j = 0; j < 10; ++j)
-            assert(v[j] == 0);
-        for (; j < 15; ++j)
-            assert(v[j] == 1);
-        for (++j; j < v.size(); ++j)
-            assert(v[j] == 0);
-    }
+TEST_CONSTEXPR_CXX20 bool tests() {
+  {
+    std::vector<bool> v(100);
+    std::vector<bool>::iterator i = v.insert(v.cbegin() + 10, 5, 1);
+    assert(v.size() == 105);
+    assert(i == v.begin() + 10);
+    std::size_t j;
+    for (j = 0; j < 10; ++j)
+      assert(v[j] == 0);
+    for (; j < 15; ++j)
+      assert(v[j] == 1);
+    for (++j; j < v.size(); ++j)
+      assert(v[j] == 0);
+  }
+  {
+    std::vector<bool> v(100);
+    while (v.size() < v.capacity())
+      v.push_back(false);
+    std::size_t sz                = v.size();
+    std::vector<bool>::iterator i = v.insert(v.cbegin() + 10, 5, 1);
+    assert(v.size() == sz + 5);
+    assert(i == v.begin() + 10);
+    std::size_t j;
+    for (j = 0; j < 10; ++j)
+      assert(v[j] == 0);
+    for (; j < 15; ++j)
+      assert(v[j] == 1);
+    for (++j; j < v.size(); ++j)
+      assert(v[j] == 0);
+  }
+  {
+    std::vector<bool> v(100);
+    while (v.size() < v.capacity())
+      v.push_back(false);
+    v.pop_back();
+    v.pop_back();
+    std::size_t sz                = v.size();
+    std::vector<bool>::iterator i = v.insert(v.cbegin() + 10, 5, 1);
+    assert(v.size() == sz + 5);
+    assert(i == v.begin() + 10);
+    std::size_t j;
+    for (j = 0; j < 10; ++j)
+      assert(v[j] == 0);
+    for (; j < 15; ++j)
+      assert(v[j] == 1);
+    for (++j; j < v.size(); ++j)
+      assert(v[j] == 0);
+  }
 #if TEST_STD_VER >= 11
-    {
-        std::vector<bool, explicit_allocator<bool>> v(10);
-        std::vector<bool, explicit_allocator<bool>>::iterator i
-            = v.insert(v.cbegin() + 10, 5, 1);
-        assert(v.size() == 15);
-        assert(i == v.begin() + 10);
-    }
-    {
-        std::vector<bool, min_allocator<bool>> v(100);
-        std::vector<bool, min_allocator<bool>>::iterator i = v.insert(v.cbegin() + 10, 5, 1);
-        assert(v.size() == 105);
-        assert(i == v.begin() + 10);
-        std::size_t j;
-        for (j = 0; j < 10; ++j)
-            assert(v[j] == 0);
-        for (; j < 15; ++j)
-            assert(v[j] == 1);
-        for (++j; j < v.size(); ++j)
-            assert(v[j] == 0);
-    }
+  {
+    std::vector<bool, explicit_allocator<bool>> v(10);
+    std::vector<bool, explicit_allocator<bool>>::iterator i = v.insert(v.cbegin() + 10, 5, 1);
+    assert(v.size() == 15);
+    assert(i == v.begin() + 10);
+  }
+  {
+    std::vector<bool, min_allocator<bool>> v(100);
+    std::vector<bool, min_allocator<bool>>::iterator i = v.insert(v.cbegin() + 10, 5, 1);
+    assert(v.size() == 105);
+    assert(i == v.begin() + 10);
+    std::size_t j;
+    for (j = 0; j < 10; ++j)
+      assert(v[j] == 0);
+    for (; j < 15; ++j)
+      assert(v[j] == 1);
+    for (++j; j < v.size(); ++j)
+      assert(v[j] == 0);
+  }
 #endif
 
-    return true;
+  return true;
 }
 
-int main(int, char**)
-{
-    tests();
+int main(int, char**) {
+  tests();
 #if TEST_STD_VER > 17
-    static_assert(tests());
+  static_assert(tests());
 #endif
-    return 0;
+  return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/vector.bool/insert_iter_value.pass.cpp b/libcxx/test/std/containers/sequences/vector.bool/insert_iter_value.pass.cpp
index bb59de3b185a9..70c1e7867eec5 100644
--- a/libcxx/test/std/containers/sequences/vector.bool/insert_iter_value.pass.cpp
+++ b/libcxx/test/std/containers/sequences/vector.bool/insert_iter_value.pass.cpp
@@ -18,80 +18,80 @@
 #include "test_macros.h"
 #include "min_allocator.h"
 
-TEST_CONSTEXPR_CXX20 bool tests()
-{
-    {
-        std::vector<bool> v(100);
-        std::vector<bool>::iterator i = v.insert(v.cbegin() + 10, 1);
-        assert(v.size() == 101);
-        assert(i == v.begin() + 10);
-        std::size_t j;
-        for (j = 0; j < 10; ++j)
-            assert(v[j] == 0);
-        assert(v[j] == 1);
-        for (++j; j < v.size(); ++j)
-            assert(v[j] == 0);
-    }
-    {
-        std::vector<bool> v(100);
-        while(v.size() < v.capacity()) v.push_back(false);
-        std::size_t sz = v.size();
-        std::vector<bool>::iterator i = v.insert(v.cbegin() + 10, 1);
-        assert(v.size() == sz + 1);
-        assert(i == v.begin() + 10);
-        std::size_t j;
-        for (j = 0; j < 10; ++j)
-            assert(v[j] == 0);
-        assert(v[j] == 1);
-        for (++j; j < v.size(); ++j)
-            assert(v[j] == 0);
-    }
-    {
-        std::vector<bool> v(100);
-        while(v.size() < v.capacity()) v.push_back(false);
-        v.pop_back(); v.pop_back();
-        std::size_t sz = v.size();
-        std::vector<bool>::iterator i = v.insert(v.cbegin() + 10, 1);
-        assert(v.size() == sz + 1);
-        assert(i == v.begin() + 10);
-        std::size_t j;
-        for (j = 0; j < 10; ++j)
-            assert(v[j] == 0);
-        assert(v[j] == 1);
-        for (++j; j < v.size(); ++j)
-            assert(v[j] == 0);
-    }
+TEST_CONSTEXPR_CXX20 bool tests() {
+  {
+    std::vector<bool> v(100);
+    std::vector<bool>::iterator i = v.insert(v.cbegin() + 10, 1);
+    assert(v.size() == 101);
+    assert(i == v.begin() + 10);
+    std::size_t j;
+    for (j = 0; j < 10; ++j)
+      assert(v[j] == 0);
+    assert(v[j] == 1);
+    for (++j; j < v.size(); ++j)
+      assert(v[j] == 0);
+  }
+  {
+    std::vector<bool> v(100);
+    while (v.size() < v.capacity())
+      v.push_back(false);
+    std::size_t sz                = v.size();
+    std::vector<bool>::iterator i = v.insert(v.cbegin() + 10, 1);
+    assert(v.size() == sz + 1);
+    assert(i == v.begin() + 10);
+    std::size_t j;
+    for (j = 0; j < 10; ++j)
+      assert(v[j] == 0);
+    assert(v[j] == 1);
+    for (++j; j < v.size(); ++j)
+      assert(v[j] == 0);
+  }
+  {
+    std::vector<bool> v(100);
+    while (v.size() < v.capacity())
+      v.push_back(false);
+    v.pop_back();
+    v.pop_back();
+    std::size_t sz                = v.size();
+    std::vector<bool>::iterator i = v.insert(v.cbegin() + 10, 1);
+    assert(v.size() == sz + 1);
+    assert(i == v.begin() + 10);
+    std::size_t j;
+    for (j = 0; j < 10; ++j)
+      assert(v[j] == 0);
+    assert(v[j] == 1);
+    for (++j; j < v.size(); ++j)
+      assert(v[j] == 0);
+  }
 #if TEST_STD_VER >= 11
-    {
-        std::vector<bool, explicit_allocator<bool>> v(10);
-        std::vector<bool, explicit_allocator<bool>>::iterator i
-            = v.insert(v.cbegin() + 10, 1);
-        assert(v.size() == 11);
-        assert(i == v.begin() + 10);
-        assert(*i == 1);
-    }
-    {
-        std::vector<bool, min_allocator<bool>> v(100);
-        std::vector<bool, min_allocator<bool>>::iterator i = v.insert(v.cbegin() + 10, 1);
-        assert(v.size() == 101);
-        assert(i == v.begin() + 10);
-        std::size_t j;
-        for (j = 0; j < 10; ++j)
-            assert(v[j] == 0);
-        assert(v[j] == 1);
-        for (++j; j < v.size(); ++j)
-            assert(v[j] == 0);
-    }
+  {
+    std::vector<bool, explicit_allocator<bool>> v(10);
+    std::vector<bool, explicit_allocator<bool>>::iterator i = v.insert(v.cbegin() + 10, 1);
+    assert(v.size() == 11);
+    assert(i == v.begin() + 10);
+    assert(*i == 1);
+  }
+  {
+    std::vector<bool, min_allocator<bool>> v(100);
+    std::vector<bool, min_allocator<bool>>::iterator i = v.insert(v.cbegin() + 10, 1);
+    assert(v.size() == 101);
+    assert(i == v.begin() + 10);
+    std::size_t j;
+    for (j = 0; j < 10; ++j)
+      assert(v[j] == 0);
+    assert(v[j] == 1);
+    for (++j; j < v.size(); ++j)
+      assert(v[j] == 0);
+  }
 #endif
 
-    return true;
+  return true;
 }
 
-int main(int, char**)
-{
-    tests();
+int main(int, char**) {
+  tests();
 #if TEST_STD_VER > 17
-    static_assert(tests());
+  static_assert(tests());
 #endif
-    return 0;
+  return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/vector.bool/insert_range.pass.cpp b/libcxx/test/std/containers/sequences/vector.bool/insert_range.pass.cpp
index d8f19d47e9942..e9a84150feb82 100644
--- a/libcxx/test/std/containers/sequences/vector.bool/insert_range.pass.cpp
+++ b/libcxx/test/std/containers/sequences/vector.bool/insert_range.pass.cpp
@@ -32,12 +32,12 @@ constexpr bool test() {
     });
   });
 
-  { // Vector may or may not need to reallocate because of the insertion -- make sure to test both cases.
+  {   // Vector may or may not need to reallocate because of the insertion -- make sure to test both cases.
     { // Ensure reallocation happens.
-      constexpr int N = 255;
-      bool in[N] = {};
+      constexpr int N     = 255;
+      bool in[N]          = {};
       std::vector<bool> v = {0, 0, 0, 1, 1, 0, 0, 0};
-      auto initial = v;
+      auto initial        = v;
       assert(v.capacity() < v.size() + std::ranges::size(in));
 
       v.insert_range(v.end(), in);
@@ -49,7 +49,7 @@ constexpr bool test() {
     }
 
     { // Ensure no reallocation happens.
-      bool in[] = {1, 1, 1, 1, 0, 0, 1, 1, 1, 1};
+      bool in[]           = {1, 1, 1, 1, 0, 0, 1, 1, 1, 1};
       std::vector<bool> v = {0, 0, 0, 1, 1, 0, 0, 0};
       v.reserve(v.size() + std::ranges::size(in));
       assert(v.capacity() >= v.size() + std::ranges::size(in));
diff --git a/libcxx/test/std/containers/sequences/vector.bool/iterator_concept_conformance.compile.pass.cpp b/libcxx/test/std/containers/sequences/vector.bool/iterator_concept_conformance.compile.pass.cpp
index ba2627e09bd45..c66f6a9e60353 100644
--- a/libcxx/test/std/containers/sequences/vector.bool/iterator_concept_conformance.compile.pass.cpp
+++ b/libcxx/test/std/containers/sequences/vector.bool/iterator_concept_conformance.compile.pass.cpp
@@ -15,27 +15,27 @@
 
 #include "test_macros.h"
 
-using iterator = std::vector<bool>::iterator;
-using const_iterator = std::vector<bool>::const_iterator;
-using reverse_iterator = std::vector<bool>::reverse_iterator;
+using iterator               = std::vector<bool>::iterator;
+using const_iterator         = std::vector<bool>::const_iterator;
+using reverse_iterator       = std::vector<bool>::reverse_iterator;
 using const_reverse_iterator = std::vector<bool>::const_reverse_iterator;
-using value_type = bool;
+using value_type             = bool;
 
-static_assert( std::random_access_iterator<iterator>);
-static_assert( std::random_access_iterator<reverse_iterator>);
+static_assert(std::random_access_iterator<iterator>);
+static_assert(std::random_access_iterator<reverse_iterator>);
 static_assert(!std::contiguous_iterator<iterator>);
 static_assert(!std::contiguous_iterator<reverse_iterator>);
 #if TEST_STD_VER > 20
-static_assert( std::indirectly_writable<iterator, value_type>);
+static_assert(std::indirectly_writable<iterator, value_type>);
 #else
 static_assert(!std::indirectly_writable<iterator, value_type>);
 #endif
-static_assert( std::sentinel_for<iterator, iterator>);
-static_assert( std::sentinel_for<iterator, const_iterator>);
+static_assert(std::sentinel_for<iterator, iterator>);
+static_assert(std::sentinel_for<iterator, const_iterator>);
 static_assert(!std::sentinel_for<iterator, reverse_iterator>);
 static_assert(!std::sentinel_for<iterator, const_reverse_iterator>);
-static_assert( std::sized_sentinel_for<iterator, iterator>);
-static_assert( std::sized_sentinel_for<iterator, const_iterator>);
+static_assert(std::sized_sentinel_for<iterator, iterator>);
+static_assert(std::sized_sentinel_for<iterator, const_iterator>);
 static_assert(!std::sized_sentinel_for<iterator, reverse_iterator>);
 static_assert(!std::sized_sentinel_for<iterator, const_reverse_iterator>);
 static_assert(std::indirectly_movable<iterator, bool*>);
@@ -44,17 +44,17 @@ static_assert(std::indirectly_copyable<iterator, bool*>);
 static_assert(std::indirectly_copyable_storable<iterator, bool*>);
 static_assert(std::indirectly_swappable<iterator, iterator>);
 
-static_assert( std::random_access_iterator<const_iterator>);
-static_assert( std::random_access_iterator<const_reverse_iterator>);
+static_assert(std::random_access_iterator<const_iterator>);
+static_assert(std::random_access_iterator<const_reverse_iterator>);
 static_assert(!std::contiguous_iterator<const_iterator>);
 static_assert(!std::contiguous_iterator<const_reverse_iterator>);
 static_assert(!std::indirectly_writable<const_iterator, value_type>);
-static_assert( std::sentinel_for<const_iterator, iterator>);
-static_assert( std::sentinel_for<const_iterator, const_iterator>);
+static_assert(std::sentinel_for<const_iterator, iterator>);
+static_assert(std::sentinel_for<const_iterator, const_iterator>);
 static_assert(!std::sentinel_for<const_iterator, reverse_iterator>);
 static_assert(!std::sentinel_for<const_iterator, const_reverse_iterator>);
-static_assert( std::sized_sentinel_for<const_iterator, iterator>);
-static_assert( std::sized_sentinel_for<const_iterator, const_iterator>);
+static_assert(std::sized_sentinel_for<const_iterator, iterator>);
+static_assert(std::sized_sentinel_for<const_iterator, const_iterator>);
 static_assert(!std::sized_sentinel_for<const_iterator, reverse_iterator>);
 static_assert(!std::sized_sentinel_for<const_iterator, const_reverse_iterator>);
 static_assert(!std::indirectly_swappable<const_iterator, const_iterator>);
diff --git a/libcxx/test/std/containers/sequences/vector.bool/iterators.pass.cpp b/libcxx/test/std/containers/sequences/vector.bool/iterators.pass.cpp
index 1e4877e8d2443..278d70284e0b1 100644
--- a/libcxx/test/std/containers/sequences/vector.bool/iterators.pass.cpp
+++ b/libcxx/test/std/containers/sequences/vector.bool/iterators.pass.cpp
@@ -22,163 +22,161 @@
 #include "test_macros.h"
 #include "min_allocator.h"
 
-TEST_CONSTEXPR_CXX20 bool tests()
-{
-    using IterRefT = std::iterator_traits<std::vector<bool>::iterator>::reference;
-    ASSERT_SAME_TYPE(IterRefT, std::vector<bool>::reference);
+TEST_CONSTEXPR_CXX20 bool tests() {
+  using IterRefT = std::iterator_traits<std::vector<bool>::iterator>::reference;
+  ASSERT_SAME_TYPE(IterRefT, std::vector<bool>::reference);
 
-    using ConstIterRefT = std::iterator_traits<std::vector<bool>::const_iterator>::reference;
+  using ConstIterRefT = std::iterator_traits<std::vector<bool>::const_iterator>::reference;
 #if !defined(_LIBCPP_VERSION) || defined(_LIBCPP_ABI_BITSET_VECTOR_BOOL_CONST_SUBSCRIPT_RETURN_BOOL)
-    ASSERT_SAME_TYPE(ConstIterRefT, bool);
+  ASSERT_SAME_TYPE(ConstIterRefT, bool);
 #else
-    ASSERT_SAME_TYPE(ConstIterRefT, std::__bit_const_reference<std::vector<bool> >);
+  ASSERT_SAME_TYPE(ConstIterRefT, std::__bit_const_reference<std::vector<bool> >);
 #endif
-    {
-        typedef bool T;
-        typedef std::vector<T> C;
-        C c;
-        C::iterator i = c.begin();
-        C::iterator j = c.end();
-        assert(std::distance(i, j) == 0);
-        assert(i == j);
-    }
-    {
-        typedef bool T;
-        typedef std::vector<T> C;
-        const C c;
-        C::const_iterator i = c.begin();
-        C::const_iterator j = c.end();
-        assert(std::distance(i, j) == 0);
-        assert(i == j);
-    }
-    {
-        typedef bool T;
-        typedef std::vector<T> C;
-        C c;
-        C::const_iterator i = c.cbegin();
-        C::const_iterator j = c.cend();
-        assert(std::distance(i, j) == 0);
-        assert(i == j);
-        assert(i == c.end());
-    }
-    {
-        typedef bool T;
-        typedef std::vector<T> C;
-        C::iterator i;
-        C::const_iterator j;
-        (void) i;
-        (void) j;
-    }
+  {
+    typedef bool T;
+    typedef std::vector<T> C;
+    C c;
+    C::iterator i = c.begin();
+    C::iterator j = c.end();
+    assert(std::distance(i, j) == 0);
+    assert(i == j);
+  }
+  {
+    typedef bool T;
+    typedef std::vector<T> C;
+    const C c;
+    C::const_iterator i = c.begin();
+    C::const_iterator j = c.end();
+    assert(std::distance(i, j) == 0);
+    assert(i == j);
+  }
+  {
+    typedef bool T;
+    typedef std::vector<T> C;
+    C c;
+    C::const_iterator i = c.cbegin();
+    C::const_iterator j = c.cend();
+    assert(std::distance(i, j) == 0);
+    assert(i == j);
+    assert(i == c.end());
+  }
+  {
+    typedef bool T;
+    typedef std::vector<T> C;
+    C::iterator i;
+    C::const_iterator j;
+    (void)i;
+    (void)j;
+  }
 #if TEST_STD_VER >= 11
-    {
-        typedef bool T;
-        typedef std::vector<T, min_allocator<T>> C;
-        C c;
-        C::iterator i = c.begin();
-        C::iterator j = c.end();
-        assert(std::distance(i, j) == 0);
+  {
+    typedef bool T;
+    typedef std::vector<T, min_allocator<T>> C;
+    C c;
+    C::iterator i = c.begin();
+    C::iterator j = c.end();
+    assert(std::distance(i, j) == 0);
 
-        assert(i == j);
-        assert(!(i != j));
+    assert(i == j);
+    assert(!(i != j));
 
-        assert(!(i < j));
-        assert((i <= j));
+    assert(!(i < j));
+    assert((i <= j));
 
-        assert(!(i > j));
-        assert((i >= j));
+    assert(!(i > j));
+    assert((i >= j));
 
 #  if TEST_STD_VER >= 20
-        // P1614 + LWG3352
-        std::same_as<std::strong_ordering> decltype(auto) r = i <=> j;
-        assert(r == std::strong_ordering::equal);
+    // P1614 + LWG3352
+    std::same_as<std::strong_ordering> decltype(auto) r = i <=> j;
+    assert(r == std::strong_ordering::equal);
 #  endif
-    }
-    {
-        typedef bool T;
-        typedef std::vector<T, min_allocator<T>> C;
-        const C c;
-        C::const_iterator i = c.begin();
-        C::const_iterator j = c.end();
-        assert(std::distance(i, j) == 0);
+  }
+  {
+    typedef bool T;
+    typedef std::vector<T, min_allocator<T>> C;
+    const C c;
+    C::const_iterator i = c.begin();
+    C::const_iterator j = c.end();
+    assert(std::distance(i, j) == 0);
 
-        assert(i == j);
-        assert(!(i != j));
+    assert(i == j);
+    assert(!(i != j));
 
-        assert(!(i < j));
-        assert((i <= j));
+    assert(!(i < j));
+    assert((i <= j));
 
-        assert(!(i > j));
-        assert((i >= j));
+    assert(!(i > j));
+    assert((i >= j));
 
 #  if TEST_STD_VER >= 20
-        // P1614 + LWG3352
-        std::same_as<std::strong_ordering> decltype(auto) r = i <=> j;
-        assert(r == std::strong_ordering::equal);
+    // P1614 + LWG3352
+    std::same_as<std::strong_ordering> decltype(auto) r = i <=> j;
+    assert(r == std::strong_ordering::equal);
 #  endif
-    }
-    {
-        typedef bool T;
-        typedef std::vector<T, min_allocator<T>> C;
-        C c;
-        C::const_iterator i = c.cbegin();
-        C::const_iterator j = c.cend();
-        assert(std::distance(i, j) == 0);
-        assert(i == j);
-        assert(i == c.end());
-    }
-    {
-        typedef bool T;
-        typedef std::vector<T, min_allocator<T>> C;
-        C::iterator i;
-        C::const_iterator j;
-        (void) i;
-        (void) j;
-    }
+  }
+  {
+    typedef bool T;
+    typedef std::vector<T, min_allocator<T>> C;
+    C c;
+    C::const_iterator i = c.cbegin();
+    C::const_iterator j = c.cend();
+    assert(std::distance(i, j) == 0);
+    assert(i == j);
+    assert(i == c.end());
+  }
+  {
+    typedef bool T;
+    typedef std::vector<T, min_allocator<T>> C;
+    C::iterator i;
+    C::const_iterator j;
+    (void)i;
+    (void)j;
+  }
 #endif
 #if TEST_STD_VER > 11
-    { // N3644 testing
-        std::vector<bool>::iterator ii1{}, ii2{};
-        std::vector<bool>::iterator ii4 = ii1;
-        std::vector<bool>::const_iterator cii{};
-        assert ( ii1 == ii2 );
-        assert ( ii1 == ii4 );
-
-        assert (!(ii1 != ii2 ));
-
-        assert ( (ii1 == cii ));
-        assert ( (cii == ii1 ));
-        assert (!(ii1 != cii ));
-        assert (!(cii != ii1 ));
-        assert (!(ii1 <  cii ));
-        assert (!(cii <  ii1 ));
-        assert ( (ii1 <= cii ));
-        assert ( (cii <= ii1 ));
-        assert (!(ii1 >  cii ));
-        assert (!(cii >  ii1 ));
-        assert ( (ii1 >= cii ));
-        assert ( (cii >= ii1 ));
-        assert (cii - ii1 == 0);
-        assert (ii1 - cii == 0);
+  { // N3644 testing
+    std::vector<bool>::iterator ii1{}, ii2{};
+    std::vector<bool>::iterator ii4 = ii1;
+    std::vector<bool>::const_iterator cii{};
+    assert(ii1 == ii2);
+    assert(ii1 == ii4);
+
+    assert(!(ii1 != ii2));
+
+    assert((ii1 == cii));
+    assert((cii == ii1));
+    assert(!(ii1 != cii));
+    assert(!(cii != ii1));
+    assert(!(ii1 < cii));
+    assert(!(cii < ii1));
+    assert((ii1 <= cii));
+    assert((cii <= ii1));
+    assert(!(ii1 > cii));
+    assert(!(cii > ii1));
+    assert((ii1 >= cii));
+    assert((cii >= ii1));
+    assert(cii - ii1 == 0);
+    assert(ii1 - cii == 0);
 
 #  if TEST_STD_VER >= 20
-        // P1614 + LWG3352
-        std::same_as<std::strong_ordering> decltype(auto) r1 = ii1 <=> ii2;
-        assert(r1 == std::strong_ordering::equal);
+    // P1614 + LWG3352
+    std::same_as<std::strong_ordering> decltype(auto) r1 = ii1 <=> ii2;
+    assert(r1 == std::strong_ordering::equal);
 
-        std::same_as<std::strong_ordering> decltype(auto) r2 = cii <=> ii2;
-        assert(r2 == std::strong_ordering::equal);
+    std::same_as<std::strong_ordering> decltype(auto) r2 = cii <=> ii2;
+    assert(r2 == std::strong_ordering::equal);
 #  endif // TEST_STD_VER > 20
-    }
+  }
 #endif
 
-    return true;
+  return true;
 }
 
-int main(int, char**)
-{
-    tests();
+int main(int, char**) {
+  tests();
 #if TEST_STD_VER > 17
-    static_assert(tests());
+  static_assert(tests());
 #endif
-    return 0;
+  return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/vector.bool/max_size.pass.cpp b/libcxx/test/std/containers/sequences/vector.bool/max_size.pass.cpp
new file mode 100644
index 0000000000000..86a163e0381ae
--- /dev/null
+++ b/libcxx/test/std/containers/sequences/vector.bool/max_size.pass.cpp
@@ -0,0 +1,105 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// <vector>
+// vector<bool>
+
+// size_type max_size() const;
+
+#include <algorithm>
+#include <cassert>
+#include <climits>
+#include <cstdint>
+#include <limits>
+#include <memory>
+#include <type_traits>
+#include <vector>
+
+#include "min_allocator.h"
+#include "sized_allocator.h"
+#include "test_allocator.h"
+#include "test_macros.h"
+
+#if TEST_STD_VER >= 11
+
+template <typename Alloc>
+TEST_CONSTEXPR_CXX20 void test(const std::vector<bool, Alloc>& v) {
+  using Vector             = std::vector<bool, Alloc>;
+  using size_type          = typename Vector::size_type;
+  using difference_type    = typename Vector::difference_type;
+  const size_type max_dist = static_cast<size_type>(std::numeric_limits<difference_type>::max());
+  assert(v.max_size() <= max_dist);
+
+  // The following check is specific to libc++ implementation details and is not portable to libstdc++
+  // and MSVC STL, as they use different types for the underlying word storage.
+#  if defined(_LIBCPP_VERSION)
+  using storage_type        = typename Vector::__storage_type;
+  using storage_alloc       = typename std::allocator_traits<Alloc>::template rebind_alloc<storage_type>;
+  using storage_traits      = typename std::allocator_traits<Alloc>::template rebind_traits<storage_type>;
+  const size_type max_alloc = storage_traits::max_size(storage_alloc(v.get_allocator()));
+  std::size_t bits_per_word = sizeof(storage_type) * CHAR_BIT;
+  const size_type max_size  = max_dist / bits_per_word < max_alloc ? max_dist : max_alloc * bits_per_word;
+  assert(v.max_size() / bits_per_word <= max_alloc); // max_alloc * bits_per_word may overflow
+  assert(v.max_size() == max_size);
+#  endif // defined(_LIBCPP_VERSION)
+}
+
+#endif // TEST_STD_VER >= 11
+
+TEST_CONSTEXPR_CXX20 bool tests() {
+  // The following check is specific to libc++ implementation details and is not portable to libstdc++
+  // and MSVC STL, as they use different types for the underlying word storage.
+#if defined(_LIBCPP_VERSION)
+  // Test cases where v.max_size() is determined by allocator::max_size()
+  {
+    using Alloc        = limited_allocator<bool, 10>;
+    using Vector       = std::vector<bool, Alloc>;
+    using storage_type = Vector::__storage_type;
+    Vector v;
+    std::size_t bits_per_word = sizeof(storage_type) * CHAR_BIT;
+    assert(v.max_size() == 10 * bits_per_word);
+  }
+#endif // defined(_LIBCPP_VERSION)
+
+#if TEST_STD_VER >= 11
+
+  // Test with various allocators and different `size_type`s
+  {
+    test(std::vector<bool>());
+    test(std::vector<bool, std::allocator<int> >());
+    test(std::vector<bool, min_allocator<bool> >());
+    test(std::vector<bool, test_allocator<bool> >(test_allocator<bool>(1)));
+    test(std::vector<bool, other_allocator<bool> >(other_allocator<bool>(5)));
+    test(std::vector<bool, sized_allocator<bool, std::uint8_t, std::int8_t> >());
+    test(std::vector<bool, sized_allocator<bool, std::uint16_t, std::int16_t> >());
+    test(std::vector<bool, sized_allocator<bool, std::uint32_t, std::int32_t> >());
+    test(std::vector<bool, sized_allocator<bool, std::uint64_t, std::int64_t> >());
+    test(std::vector<bool, limited_allocator<bool, static_cast<std::size_t>(-1)> >());
+  }
+
+  // Test cases to identify incorrect implementations that unconditionally compute an internal-to-external
+  // capacity in a way that can overflow, leading to incorrect results.
+  {
+    test(std::vector<bool, limited_allocator<bool, static_cast<std::size_t>(-1) / 61> >());
+    test(std::vector<bool, limited_allocator<bool, static_cast<std::size_t>(-1) / 63> >());
+  }
+
+#endif // TEST_STD_VER >= 11
+
+  return true;
+}
+
+int main(int, char**) {
+  tests();
+
+#if TEST_STD_VER >= 20
+  static_assert(tests());
+#endif
+
+  return 0;
+}
diff --git a/libcxx/test/std/containers/sequences/vector.bool/move.pass.cpp b/libcxx/test/std/containers/sequences/vector.bool/move.pass.cpp
index 56de337225815..a69f0b5738986 100644
--- a/libcxx/test/std/containers/sequences/vector.bool/move.pass.cpp
+++ b/libcxx/test/std/containers/sequences/vector.bool/move.pass.cpp
@@ -18,88 +18,83 @@
 #include "test_allocator.h"
 #include "min_allocator.h"
 
-TEST_CONSTEXPR_CXX20 bool tests()
-{
-    test_allocator_statistics alloc_stats;
-    {
-        std::vector<bool, test_allocator<bool> > l(test_allocator<bool>(5, &alloc_stats));
-        std::vector<bool, test_allocator<bool> > lo(test_allocator<bool>(5, &alloc_stats));
-        for (int i = 1; i <= 3; ++i)
-        {
-            l.push_back(true);
-            lo.push_back(true);
-        }
-        std::vector<bool, test_allocator<bool> > l2 = std::move(l);
-        assert(l2 == lo);
-        assert(l.empty());
-        assert(l2.get_allocator() == lo.get_allocator());
+TEST_CONSTEXPR_CXX20 bool tests() {
+  test_allocator_statistics alloc_stats;
+  {
+    std::vector<bool, test_allocator<bool> > l(test_allocator<bool>(5, &alloc_stats));
+    std::vector<bool, test_allocator<bool> > lo(test_allocator<bool>(5, &alloc_stats));
+    for (int i = 1; i <= 3; ++i) {
+      l.push_back(true);
+      lo.push_back(true);
     }
-    {
-        std::vector<bool, other_allocator<bool> > l(other_allocator<bool>(5));
-        std::vector<bool, other_allocator<bool> > lo(other_allocator<bool>(5));
-        for (int i = 1; i <= 3; ++i)
-        {
-            l.push_back(true);
-            lo.push_back(true);
-        }
-        std::vector<bool, other_allocator<bool> > l2 = std::move(l);
-        assert(l2 == lo);
-        assert(l.empty());
-        assert(l2.get_allocator() == lo.get_allocator());
+    std::vector<bool, test_allocator<bool> > l2 = std::move(l);
+    assert(l2 == lo);
+    assert(l.empty());
+    assert(l2.get_allocator() == lo.get_allocator());
+  }
+  {
+    std::vector<bool, other_allocator<bool> > l(other_allocator<bool>(5));
+    std::vector<bool, other_allocator<bool> > lo(other_allocator<bool>(5));
+    for (int i = 1; i <= 3; ++i) {
+      l.push_back(true);
+      lo.push_back(true);
     }
-    {
-        std::vector<bool, min_allocator<bool> > l(min_allocator<bool>{});
-        std::vector<bool, min_allocator<bool> > lo(min_allocator<bool>{});
-        for (int i = 1; i <= 3; ++i)
-        {
-            l.push_back(true);
-            lo.push_back(true);
-        }
-        std::vector<bool, min_allocator<bool> > l2 = std::move(l);
-        assert(l2 == lo);
-        assert(l.empty());
-        assert(l2.get_allocator() == lo.get_allocator());
+    std::vector<bool, other_allocator<bool> > l2 = std::move(l);
+    assert(l2 == lo);
+    assert(l.empty());
+    assert(l2.get_allocator() == lo.get_allocator());
+  }
+  {
+    std::vector<bool, min_allocator<bool> > l(min_allocator<bool>{});
+    std::vector<bool, min_allocator<bool> > lo(min_allocator<bool>{});
+    for (int i = 1; i <= 3; ++i) {
+      l.push_back(true);
+      lo.push_back(true);
     }
+    std::vector<bool, min_allocator<bool> > l2 = std::move(l);
+    assert(l2 == lo);
+    assert(l.empty());
+    assert(l2.get_allocator() == lo.get_allocator());
+  }
+  {
+    alloc_stats.clear();
+    using Vect   = std::vector<bool, test_allocator<bool> >;
+    using AllocT = Vect::allocator_type;
+    Vect v(test_allocator<bool>(42, 101, &alloc_stats));
+    assert(alloc_stats.count == 1);
     {
-      alloc_stats.clear();
-      using Vect = std::vector<bool, test_allocator<bool> >;
-      using AllocT = Vect::allocator_type;
-      Vect v(test_allocator<bool>(42, 101, &alloc_stats));
-      assert(alloc_stats.count == 1);
-      {
-        const AllocT& a = v.get_allocator();
-        assert(alloc_stats.count == 2);
-        assert(a.get_data() == 42);
-        assert(a.get_id() == 101);
-      }
-      assert(alloc_stats.count == 1);
-      alloc_stats.clear_ctor_counters();
-
-      Vect v2 = std::move(v);
+      const AllocT& a = v.get_allocator();
       assert(alloc_stats.count == 2);
-      assert(alloc_stats.copied == 0);
-      assert(alloc_stats.moved == 1);
-      {
-        const AllocT& a1 = v.get_allocator();
-        assert(a1.get_id() == test_alloc_base::moved_value);
-        assert(a1.get_data() == 42);
+      assert(a.get_data() == 42);
+      assert(a.get_id() == 101);
+    }
+    assert(alloc_stats.count == 1);
+    alloc_stats.clear_ctor_counters();
+
+    Vect v2 = std::move(v);
+    assert(alloc_stats.count == 2);
+    assert(alloc_stats.copied == 0);
+    assert(alloc_stats.moved == 1);
+    {
+      const AllocT& a1 = v.get_allocator();
+      assert(a1.get_id() == test_alloc_base::moved_value);
+      assert(a1.get_data() == 42);
 
-        const AllocT& a2 = v2.get_allocator();
-        assert(a2.get_id() == 101);
-        assert(a2.get_data() == 42);
+      const AllocT& a2 = v2.get_allocator();
+      assert(a2.get_id() == 101);
+      assert(a2.get_data() == 42);
 
-        assert(a1 == a2);
-      }
+      assert(a1 == a2);
     }
+  }
 
-    return true;
+  return true;
 }
 
-int main(int, char**)
-{
-    tests();
+int main(int, char**) {
+  tests();
 #if TEST_STD_VER > 17
-    static_assert(tests());
+  static_assert(tests());
 #endif
-    return 0;
+  return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/vector.bool/move_alloc.pass.cpp b/libcxx/test/std/containers/sequences/vector.bool/move_alloc.pass.cpp
index 1f41657eb56ea..01e7410040686 100644
--- a/libcxx/test/std/containers/sequences/vector.bool/move_alloc.pass.cpp
+++ b/libcxx/test/std/containers/sequences/vector.bool/move_alloc.pass.cpp
@@ -18,69 +18,63 @@
 #include "test_allocator.h"
 #include "min_allocator.h"
 
-TEST_CONSTEXPR_CXX20 bool tests()
-{
-    {
-        std::vector<bool, test_allocator<bool> > l(test_allocator<bool>(5));
-        std::vector<bool, test_allocator<bool> > lo(test_allocator<bool>(5));
-        for (int i = 1; i <= 3; ++i)
-        {
-            l.push_back(i);
-            lo.push_back(i);
-        }
-        std::vector<bool, test_allocator<bool> > l2(std::move(l), test_allocator<bool>(6));
-        assert(l2 == lo);
-        assert(!l.empty());
-        assert(l2.get_allocator() == test_allocator<bool>(6));
+TEST_CONSTEXPR_CXX20 bool tests() {
+  {
+    std::vector<bool, test_allocator<bool> > l(test_allocator<bool>(5));
+    std::vector<bool, test_allocator<bool> > lo(test_allocator<bool>(5));
+    for (int i = 1; i <= 3; ++i) {
+      l.push_back(i);
+      lo.push_back(i);
     }
-    {
-        std::vector<bool, test_allocator<bool> > l(test_allocator<bool>(5));
-        std::vector<bool, test_allocator<bool> > lo(test_allocator<bool>(5));
-        for (int i = 1; i <= 3; ++i)
-        {
-            l.push_back(i);
-            lo.push_back(i);
-        }
-        std::vector<bool, test_allocator<bool> > l2(std::move(l), test_allocator<bool>(5));
-        assert(l2 == lo);
-        assert(l.empty());
-        assert(l2.get_allocator() == test_allocator<bool>(5));
+    std::vector<bool, test_allocator<bool> > l2(std::move(l), test_allocator<bool>(6));
+    assert(l2 == lo);
+    assert(!l.empty());
+    assert(l2.get_allocator() == test_allocator<bool>(6));
+  }
+  {
+    std::vector<bool, test_allocator<bool> > l(test_allocator<bool>(5));
+    std::vector<bool, test_allocator<bool> > lo(test_allocator<bool>(5));
+    for (int i = 1; i <= 3; ++i) {
+      l.push_back(i);
+      lo.push_back(i);
     }
-    {
-        std::vector<bool, other_allocator<bool> > l(other_allocator<bool>(5));
-        std::vector<bool, other_allocator<bool> > lo(other_allocator<bool>(5));
-        for (int i = 1; i <= 3; ++i)
-        {
-            l.push_back(i);
-            lo.push_back(i);
-        }
-        std::vector<bool, other_allocator<bool> > l2(std::move(l), other_allocator<bool>(4));
-        assert(l2 == lo);
-        assert(!l.empty());
-        assert(l2.get_allocator() == other_allocator<bool>(4));
+    std::vector<bool, test_allocator<bool> > l2(std::move(l), test_allocator<bool>(5));
+    assert(l2 == lo);
+    assert(l.empty());
+    assert(l2.get_allocator() == test_allocator<bool>(5));
+  }
+  {
+    std::vector<bool, other_allocator<bool> > l(other_allocator<bool>(5));
+    std::vector<bool, other_allocator<bool> > lo(other_allocator<bool>(5));
+    for (int i = 1; i <= 3; ++i) {
+      l.push_back(i);
+      lo.push_back(i);
     }
-    {
-        std::vector<bool, min_allocator<bool> > l(min_allocator<bool>{});
-        std::vector<bool, min_allocator<bool> > lo(min_allocator<bool>{});
-        for (int i = 1; i <= 3; ++i)
-        {
-            l.push_back(i);
-            lo.push_back(i);
-        }
-        std::vector<bool, min_allocator<bool> > l2(std::move(l), min_allocator<bool>());
-        assert(l2 == lo);
-        assert(l.empty());
-        assert(l2.get_allocator() == min_allocator<bool>());
+    std::vector<bool, other_allocator<bool> > l2(std::move(l), other_allocator<bool>(4));
+    assert(l2 == lo);
+    assert(!l.empty());
+    assert(l2.get_allocator() == other_allocator<bool>(4));
+  }
+  {
+    std::vector<bool, min_allocator<bool> > l(min_allocator<bool>{});
+    std::vector<bool, min_allocator<bool> > lo(min_allocator<bool>{});
+    for (int i = 1; i <= 3; ++i) {
+      l.push_back(i);
+      lo.push_back(i);
     }
+    std::vector<bool, min_allocator<bool> > l2(std::move(l), min_allocator<bool>());
+    assert(l2 == lo);
+    assert(l.empty());
+    assert(l2.get_allocator() == min_allocator<bool>());
+  }
 
-    return true;
+  return true;
 }
 
-int main(int, char**)
-{
-    tests();
+int main(int, char**) {
+  tests();
 #if TEST_STD_VER > 17
-    static_assert(tests());
+  static_assert(tests());
 #endif
-    return 0;
+  return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/vector.bool/move_assign_noexcept.pass.cpp b/libcxx/test/std/containers/sequences/vector.bool/move_assign_noexcept.pass.cpp
index c40d1e557267f..5a69213c3b4ef 100644
--- a/libcxx/test/std/containers/sequences/vector.bool/move_assign_noexcept.pass.cpp
+++ b/libcxx/test/std/containers/sequences/vector.bool/move_assign_noexcept.pass.cpp
@@ -24,78 +24,74 @@
 #include "test_allocator.h"
 
 template <class T>
-struct some_alloc
-{
-    typedef T value_type;
-    some_alloc(const some_alloc&);
+struct some_alloc {
+  typedef T value_type;
+  some_alloc(const some_alloc&);
 };
 
 template <class T>
-struct some_alloc2
-{
-    typedef T value_type;
+struct some_alloc2 {
+  typedef T value_type;
 
-    some_alloc2() {}
-    some_alloc2(const some_alloc2&);
-    void deallocate(void*, unsigned) {}
+  some_alloc2() {}
+  some_alloc2(const some_alloc2&);
+  void deallocate(void*, unsigned) {}
 
-    typedef std::false_type propagate_on_container_move_assignment;
-    typedef std::true_type is_always_equal;
+  typedef std::false_type propagate_on_container_move_assignment;
+  typedef std::true_type is_always_equal;
 };
 
 template <class T>
-struct some_alloc3
-{
-    typedef T value_type;
+struct some_alloc3 {
+  typedef T value_type;
 
-    some_alloc3() {}
-    some_alloc3(const some_alloc3&);
-    void deallocate(void*, unsigned) {}
+  some_alloc3() {}
+  some_alloc3(const some_alloc3&);
+  void deallocate(void*, unsigned) {}
 
-    typedef std::false_type propagate_on_container_move_assignment;
-    typedef std::false_type is_always_equal;
+  typedef std::false_type propagate_on_container_move_assignment;
+  typedef std::false_type is_always_equal;
 };
 
-int main(int, char**)
-{
+int main(int, char**) {
 #if defined(_LIBCPP_VERSION)
-    {
-        typedef std::vector<bool> C;
-        static_assert(std::is_nothrow_move_assignable<C>::value, "");
-    }
+  {
+    typedef std::vector<bool> C;
+    static_assert(std::is_nothrow_move_assignable<C>::value, "");
+  }
 #endif // _LIBCPP_VERSION
-    {
-        typedef std::vector<bool, test_allocator<bool>> C;
-        static_assert(!std::is_nothrow_move_assignable<C>::value, "");
-    }
+  {
+    typedef std::vector<bool, test_allocator<bool>> C;
+    static_assert(!std::is_nothrow_move_assignable<C>::value, "");
+  }
 #if defined(_LIBCPP_VERSION)
-    {
-        typedef std::vector<bool, other_allocator<bool>> C;
-        static_assert(std::is_nothrow_move_assignable<C>::value, "");
-    }
+  {
+    typedef std::vector<bool, other_allocator<bool>> C;
+    static_assert(std::is_nothrow_move_assignable<C>::value, "");
+  }
 #endif // _LIBCPP_VERSION
-    {
+  {
 #if TEST_STD_VER > 14
-#if defined(_LIBCPP_VERSION)
-        typedef std::vector<bool, some_alloc<bool>> C;
-        static_assert( std::is_nothrow_move_assignable<C>::value, "");
-#endif // _LIBCPP_VERSION
+#  if defined(_LIBCPP_VERSION)
+    typedef std::vector<bool, some_alloc<bool>> C;
+    static_assert(std::is_nothrow_move_assignable<C>::value, "");
+#  endif // _LIBCPP_VERSION
 #else
-        typedef std::vector<bool, some_alloc<bool>> C;
-        static_assert(!std::is_nothrow_move_assignable<C>::value, "");
+    typedef std::vector<bool, some_alloc<bool>> C;
+    static_assert(!std::is_nothrow_move_assignable<C>::value, "");
 #endif
-    }
+  }
 #if TEST_STD_VER > 14
-#if defined(_LIBCPP_VERSION)
-    {  // POCMA false, is_always_equal true
-        typedef std::vector<bool, some_alloc2<bool>> C;
-        static_assert( std::is_nothrow_move_assignable<C>::value, "");
-    }
-#endif // _LIBCPP_VERSION
-    {  // POCMA false, is_always_equal false
-        typedef std::vector<bool, some_alloc3<bool>> C;
-        static_assert(!std::is_nothrow_move_assignable<C>::value, "");
-    }
+#  if defined(_LIBCPP_VERSION)
+  { // POCMA false, is_always_equal true
+    typedef std::vector<bool, some_alloc2<bool>> C;
+    static_assert(std::is_nothrow_move_assignable<C>::value, "");
+  }
+#  endif // _LIBCPP_VERSION
+  {      // POCMA false, is_always_equal false
+    typedef std::vector<bool, some_alloc3<bool>> C;
+    static_assert(!std::is_nothrow_move_assignable<C>::value, "");
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/sequences/vector.bool/move_noexcept.pass.cpp b/libcxx/test/std/containers/sequences/vector.bool/move_noexcept.pass.cpp
index ccd8185d3f3d6..d0d231f48953a 100644
--- a/libcxx/test/std/containers/sequences/vector.bool/move_noexcept.pass.cpp
+++ b/libcxx/test/std/containers/sequences/vector.bool/move_noexcept.pass.cpp
@@ -22,40 +22,38 @@
 #include "test_allocator.h"
 
 template <class T>
-struct some_alloc
-{
-    typedef T value_type;
-    some_alloc(const some_alloc&);
+struct some_alloc {
+  typedef T value_type;
+  some_alloc(const some_alloc&);
 };
 
-int main(int, char**)
-{
+int main(int, char**) {
 #if defined(_LIBCPP_VERSION)
-    {
-        typedef std::vector<bool> C;
-        static_assert(std::is_nothrow_move_constructible<C>::value, "");
-    }
-    {
-        typedef std::vector<bool, test_allocator<bool>> C;
-        static_assert(std::is_nothrow_move_constructible<C>::value, "");
-    }
-    {
-        typedef std::vector<bool, other_allocator<bool>> C;
-        static_assert(std::is_nothrow_move_constructible<C>::value, "");
-    }
+  {
+    typedef std::vector<bool> C;
+    static_assert(std::is_nothrow_move_constructible<C>::value, "");
+  }
+  {
+    typedef std::vector<bool, test_allocator<bool>> C;
+    static_assert(std::is_nothrow_move_constructible<C>::value, "");
+  }
+  {
+    typedef std::vector<bool, other_allocator<bool>> C;
+    static_assert(std::is_nothrow_move_constructible<C>::value, "");
+  }
 #endif // _LIBCPP_VERSION
-    {
+  {
     //  In C++17, move constructors for allocators are not allowed to throw
 #if TEST_STD_VER > 14
-#if defined(_LIBCPP_VERSION)
-        typedef std::vector<bool, some_alloc<bool>> C;
-        static_assert( std::is_nothrow_move_constructible<C>::value, "");
-#endif // _LIBCPP_VERSION
+#  if defined(_LIBCPP_VERSION)
+    typedef std::vector<bool, some_alloc<bool>> C;
+    static_assert(std::is_nothrow_move_constructible<C>::value, "");
+#  endif // _LIBCPP_VERSION
 #else
-        typedef std::vector<bool, some_alloc<bool>> C;
-        static_assert(!std::is_nothrow_move_constructible<C>::value, "");
+    typedef std::vector<bool, some_alloc<bool>> C;
+    static_assert(!std::is_nothrow_move_constructible<C>::value, "");
 #endif
-    }
+  }
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/vector.bool/op_equal_initializer_list.pass.cpp b/libcxx/test/std/containers/sequences/vector.bool/op_equal_initializer_list.pass.cpp
index 618357120c6bd..bffccffbc03dc 100644
--- a/libcxx/test/std/containers/sequences/vector.bool/op_equal_initializer_list.pass.cpp
+++ b/libcxx/test/std/containers/sequences/vector.bool/op_equal_initializer_list.pass.cpp
@@ -18,9 +18,8 @@
 #include "test_macros.h"
 #include "min_allocator.h"
 
-TEST_CONSTEXPR_CXX20 bool tests()
-{
-    {
+TEST_CONSTEXPR_CXX20 bool tests() {
+  {
     std::vector<bool> d;
     d = {true, false, false, true};
     assert(d.size() == 4);
@@ -28,8 +27,8 @@ TEST_CONSTEXPR_CXX20 bool tests()
     assert(d[1] == false);
     assert(d[2] == false);
     assert(d[3] == true);
-    }
-    {
+  }
+  {
     std::vector<bool, min_allocator<bool>> d;
     d = {true, false, false, true};
     assert(d.size() == 4);
@@ -37,16 +36,15 @@ TEST_CONSTEXPR_CXX20 bool tests()
     assert(d[1] == false);
     assert(d[2] == false);
     assert(d[3] == true);
-    }
+  }
 
-    return true;
+  return true;
 }
 
-int main(int, char**)
-{
-    tests();
+int main(int, char**) {
+  tests();
 #if TEST_STD_VER > 17
-    static_assert(tests());
+  static_assert(tests());
 #endif
-    return 0;
+  return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/vector.bool/push_back.pass.cpp b/libcxx/test/std/containers/sequences/vector.bool/push_back.pass.cpp
index 25f8fb6b29dfe..ca892154f9c36 100644
--- a/libcxx/test/std/containers/sequences/vector.bool/push_back.pass.cpp
+++ b/libcxx/test/std/containers/sequences/vector.bool/push_back.pass.cpp
@@ -18,43 +18,39 @@
 #include "test_macros.h"
 #include "min_allocator.h"
 
-TEST_CONSTEXPR_CXX20 bool tests()
-{
-    {
-        bool a[] = {0, 1, 1, 0, 1, 0, 0};
-        const unsigned N = sizeof(a)/sizeof(a[0]);
-        std::vector<bool> c;
-        for (unsigned i = 0; i < N; ++i)
-        {
-            c.push_back(a[i]);
-            assert(c.size() == i+1);
-            for (std::size_t j = 0; j < c.size(); ++j)
-                assert(c[j] == a[j]);
-        }
+TEST_CONSTEXPR_CXX20 bool tests() {
+  {
+    bool a[]         = {0, 1, 1, 0, 1, 0, 0};
+    const unsigned N = sizeof(a) / sizeof(a[0]);
+    std::vector<bool> c;
+    for (unsigned i = 0; i < N; ++i) {
+      c.push_back(a[i]);
+      assert(c.size() == i + 1);
+      for (std::size_t j = 0; j < c.size(); ++j)
+        assert(c[j] == a[j]);
     }
+  }
 #if TEST_STD_VER >= 11
-    {
-        bool a[] = {0, 1, 1, 0, 1, 0, 0};
-        const unsigned N = sizeof(a)/sizeof(a[0]);
-        std::vector<bool, min_allocator<bool>> c;
-        for (unsigned i = 0; i < N; ++i)
-        {
-            c.push_back(a[i]);
-            assert(c.size() == i+1);
-            for (std::size_t j = 0; j < c.size(); ++j)
-                assert(c[j] == a[j]);
-        }
+  {
+    bool a[]         = {0, 1, 1, 0, 1, 0, 0};
+    const unsigned N = sizeof(a) / sizeof(a[0]);
+    std::vector<bool, min_allocator<bool>> c;
+    for (unsigned i = 0; i < N; ++i) {
+      c.push_back(a[i]);
+      assert(c.size() == i + 1);
+      for (std::size_t j = 0; j < c.size(); ++j)
+        assert(c[j] == a[j]);
     }
+  }
 #endif
 
-    return true;
+  return true;
 }
 
-int main(int, char**)
-{
-    tests();
+int main(int, char**) {
+  tests();
 #if TEST_STD_VER > 17
-    static_assert(tests());
+  static_assert(tests());
 #endif
-    return 0;
+  return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/vector.bool/range_concept_conformance.compile.pass.cpp b/libcxx/test/std/containers/sequences/vector.bool/range_concept_conformance.compile.pass.cpp
index 410cf834bf724..dc2332df84f11 100644
--- a/libcxx/test/std/containers/sequences/vector.bool/range_concept_conformance.compile.pass.cpp
+++ b/libcxx/test/std/containers/sequences/vector.bool/range_concept_conformance.compile.pass.cpp
@@ -17,7 +17,6 @@
 
 using range = std::vector<bool>;
 
-
 static_assert(std::same_as<std::ranges::iterator_t<range>, range::iterator>);
 static_assert(std::ranges::common_range<range>);
 static_assert(std::ranges::random_access_range<range>);
diff --git a/libcxx/test/std/containers/sequences/vector.bool/reference.swap.pass.cpp b/libcxx/test/std/containers/sequences/vector.bool/reference.swap.pass.cpp
index 8879d1f1d358c..a0476b6fc305b 100644
--- a/libcxx/test/std/containers/sequences/vector.bool/reference.swap.pass.cpp
+++ b/libcxx/test/std/containers/sequences/vector.bool/reference.swap.pass.cpp
@@ -16,34 +16,31 @@
 
 #include "test_macros.h"
 
-TEST_CONSTEXPR_CXX20 bool tests()
-{
+TEST_CONSTEXPR_CXX20 bool tests() {
+  bool a[] = {false, true, false, true};
+  bool* an = a + sizeof(a) / sizeof(a[0]);
 
-    bool a[] = {false, true, false, true};
-    bool* an = a + sizeof(a)/sizeof(a[0]);
-
-    std::vector<bool> v(a, an);
-    std::vector<bool>::reference r1 = v[0];
-    std::vector<bool>::reference r2 = v[3];
+  std::vector<bool> v(a, an);
+  std::vector<bool>::reference r1 = v[0];
+  std::vector<bool>::reference r2 = v[3];
 
 #if TEST_STD_VER >= 11
-    static_assert((noexcept(v.swap(r1,r2))), "");
+  static_assert((noexcept(v.swap(r1, r2))), "");
 #endif
 
-    assert(!r1);
-    assert( r2);
-    v.swap(r1, r2);
-    assert( r1);
-    assert(!r2);
+  assert(!r1);
+  assert(r2);
+  v.swap(r1, r2);
+  assert(r1);
+  assert(!r2);
 
-    return true;
+  return true;
 }
 
-int main(int, char**)
-{
-    tests();
+int main(int, char**) {
+  tests();
 #if TEST_STD_VER > 17
-    static_assert(tests());
+  static_assert(tests());
 #endif
-    return 0;
+  return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/vector.bool/reference/assign_bool.pass.cpp b/libcxx/test/std/containers/sequences/vector.bool/reference/assign_bool.pass.cpp
index 64dcc3c608214..0ce88b393c516 100644
--- a/libcxx/test/std/containers/sequences/vector.bool/reference/assign_bool.pass.cpp
+++ b/libcxx/test/std/containers/sequences/vector.bool/reference/assign_bool.pass.cpp
@@ -20,7 +20,7 @@ TEST_CONSTEXPR_CXX20 bool test() {
   typedef std::vector<bool>::reference Ref;
   vec.push_back(true);
   vec.push_back(false);
-  Ref ref = vec[0];
+  Ref ref        = vec[0];
   const Ref cref = vec[0];
 
   assert(ref);
@@ -48,7 +48,7 @@ TEST_CONSTEXPR_CXX20 bool test() {
 int main(int, char**) {
   test();
 #if TEST_STD_VER > 17
-    static_assert(test());
+  static_assert(test());
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/sequences/vector.bool/reference/assign_copy.pass.cpp b/libcxx/test/std/containers/sequences/vector.bool/reference/assign_copy.pass.cpp
index 7568eb2f3cc3e..a877aea16e6ab 100644
--- a/libcxx/test/std/containers/sequences/vector.bool/reference/assign_copy.pass.cpp
+++ b/libcxx/test/std/containers/sequences/vector.bool/reference/assign_copy.pass.cpp
@@ -23,19 +23,19 @@ TEST_CONSTEXPR_CXX20 bool test() {
   vec.push_back(false);
   Ref ref1 = vec[0];
   Ref ref2 = vec[1];
-  ref2 = ref1;
+  ref2     = ref1;
   // Ref&
   {
     vec[0] = false;
     vec[1] = true;
-    ref1 = ref2;
+    ref1   = ref2;
     assert(vec[0]);
     assert(vec[1]);
   }
   {
     vec[0] = true;
     vec[1] = false;
-    ref1 = ref2;
+    ref1   = ref2;
     assert(!vec[0]);
     assert(!vec[1]);
   }
@@ -43,14 +43,14 @@ TEST_CONSTEXPR_CXX20 bool test() {
   {
     vec[0] = false;
     vec[1] = true;
-    ref1 = std::move(ref2);
+    ref1   = std::move(ref2);
     assert(vec[0]);
     assert(vec[1]);
   }
   {
     vec[0] = true;
     vec[1] = false;
-    ref1 = std::move(ref2);
+    ref1   = std::move(ref2);
     assert(!vec[0]);
     assert(!vec[1]);
   }
@@ -58,14 +58,14 @@ TEST_CONSTEXPR_CXX20 bool test() {
   {
     vec[0] = false;
     vec[1] = true;
-    ref1 = static_cast<const Ref&>(ref2);
+    ref1   = static_cast<const Ref&>(ref2);
     assert(vec[0]);
     assert(vec[1]);
   }
   {
     vec[0] = true;
     vec[1] = false;
-    ref1 = static_cast<const Ref&>(ref2);
+    ref1   = static_cast<const Ref&>(ref2);
     assert(!vec[0]);
     assert(!vec[1]);
   }
@@ -75,7 +75,7 @@ TEST_CONSTEXPR_CXX20 bool test() {
 int main(int, char**) {
   test();
 #if TEST_STD_VER > 17
-    static_assert(test());
+  static_assert(test());
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/sequences/vector.bool/reference/ctor_copy.pass.cpp b/libcxx/test/std/containers/sequences/vector.bool/reference/ctor_copy.pass.cpp
index 46ac18ab493b7..5b7303828b5e8 100644
--- a/libcxx/test/std/containers/sequences/vector.bool/reference/ctor_copy.pass.cpp
+++ b/libcxx/test/std/containers/sequences/vector.bool/reference/ctor_copy.pass.cpp
@@ -19,7 +19,7 @@ TEST_CONSTEXPR_CXX20 bool test() {
   std::vector<bool> vec;
   typedef std::vector<bool>::reference Ref;
   vec.push_back(true);
-  Ref ref = vec[0];
+  Ref ref  = vec[0];
   Ref ref2 = ref;
   assert(ref == ref2 && ref2);
   ref.flip();
@@ -31,7 +31,7 @@ TEST_CONSTEXPR_CXX20 bool test() {
 int main(int, char**) {
   test();
 #if TEST_STD_VER > 17
-    static_assert(test());
+  static_assert(test());
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/sequences/vector.bool/reference/flip.pass.cpp b/libcxx/test/std/containers/sequences/vector.bool/reference/flip.pass.cpp
index 080975656ac9e..4c5b15ea0c1ba 100644
--- a/libcxx/test/std/containers/sequences/vector.bool/reference/flip.pass.cpp
+++ b/libcxx/test/std/containers/sequences/vector.bool/reference/flip.pass.cpp
@@ -36,7 +36,7 @@ TEST_CONSTEXPR_CXX20 bool test() {
 int main(int, char**) {
   test();
 #if TEST_STD_VER > 17
-    static_assert(test());
+  static_assert(test());
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/sequences/vector.bool/reference/operator_bool.pass.cpp b/libcxx/test/std/containers/sequences/vector.bool/reference/operator_bool.pass.cpp
index 02abd0e6be766..2fa00f8f92646 100644
--- a/libcxx/test/std/containers/sequences/vector.bool/reference/operator_bool.pass.cpp
+++ b/libcxx/test/std/containers/sequences/vector.bool/reference/operator_bool.pass.cpp
@@ -23,9 +23,9 @@ TEST_CONSTEXPR_CXX20 bool test() {
 
   vec.push_back(true);
   vec.push_back(false);
-  Ref true_ref = vec[0];
+  Ref true_ref  = vec[0];
   Ref false_ref = vec[1];
-  bool b = true_ref;
+  bool b        = true_ref;
   assert(b);
   assert(true_ref);
   assert(!false_ref);
@@ -36,7 +36,7 @@ TEST_CONSTEXPR_CXX20 bool test() {
 int main(int, char**) {
   test();
 #if TEST_STD_VER > 17
-    static_assert(test());
+  static_assert(test());
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/sequences/vector.bool/reserve.pass.cpp b/libcxx/test/std/containers/sequences/vector.bool/reserve.pass.cpp
index e179fb95aac22..e0caeb0083fbb 100644
--- a/libcxx/test/std/containers/sequences/vector.bool/reserve.pass.cpp
+++ b/libcxx/test/std/containers/sequences/vector.bool/reserve.pass.cpp
@@ -19,71 +19,69 @@
 #include "min_allocator.h"
 #include "test_allocator.h"
 
-TEST_CONSTEXPR_CXX20 bool tests()
-{
-    {
-        std::vector<bool> v;
-        v.reserve(10);
-        assert(v.capacity() >= 10);
-    }
-    {
-        std::vector<bool> v(100);
-        assert(v.capacity() >= 100);
-        v.reserve(50);
-        assert(v.size() == 100);
-        assert(v.capacity() >= 100);
-        v.reserve(150);
-        assert(v.size() == 100);
-        assert(v.capacity() >= 150);
-    }
+TEST_CONSTEXPR_CXX20 bool tests() {
+  {
+    std::vector<bool> v;
+    v.reserve(10);
+    assert(v.capacity() >= 10);
+  }
+  {
+    std::vector<bool> v(100);
+    assert(v.capacity() >= 100);
+    v.reserve(50);
+    assert(v.size() == 100);
+    assert(v.capacity() >= 100);
+    v.reserve(150);
+    assert(v.size() == 100);
+    assert(v.capacity() >= 150);
+  }
 #if TEST_STD_VER >= 11
-    {
-        std::vector<bool, min_allocator<bool>> v;
-        v.reserve(10);
-        assert(v.capacity() >= 10);
-    }
-    {
-        std::vector<bool, explicit_allocator<bool>> v;
-        v.reserve(10);
-        assert(v.capacity() >= 10);
-    }
-    {
-        std::vector<bool, min_allocator<bool>> v(100);
-        assert(v.capacity() >= 100);
-        v.reserve(50);
-        assert(v.size() == 100);
-        assert(v.capacity() >= 100);
-        v.reserve(150);
-        assert(v.size() == 100);
-        assert(v.capacity() >= 150);
-    }
+  {
+    std::vector<bool, min_allocator<bool>> v;
+    v.reserve(10);
+    assert(v.capacity() >= 10);
+  }
+  {
+    std::vector<bool, explicit_allocator<bool>> v;
+    v.reserve(10);
+    assert(v.capacity() >= 10);
+  }
+  {
+    std::vector<bool, min_allocator<bool>> v(100);
+    assert(v.capacity() >= 100);
+    v.reserve(50);
+    assert(v.size() == 100);
+    assert(v.capacity() >= 100);
+    v.reserve(150);
+    assert(v.size() == 100);
+    assert(v.capacity() >= 150);
+  }
 #endif
 #ifndef TEST_HAS_NO_EXCEPTIONS
-    if (!TEST_IS_CONSTANT_EVALUATED) {
-        std::vector<bool, limited_allocator<bool, 10> > v;
-        v.reserve(5);
-        try {
-            // A typical implementation would allocate chunks of bits.
-            // In libc++ the chunk has the same size as the machine word. It is
-            // reasonable to assume that in practice no implementation would use
-            // 64 kB or larger chunks.
-            v.reserve(10 * 65536);
-            assert(false);
-        } catch (const std::length_error&) {
-            // no-op
-        }
-        assert(v.capacity() >= 5);
+  if (!TEST_IS_CONSTANT_EVALUATED) {
+    std::vector<bool, limited_allocator<bool, 10> > v;
+    v.reserve(5);
+    try {
+      // A typical implementation would allocate chunks of bits.
+      // In libc++ the chunk has the same size as the machine word. It is
+      // reasonable to assume that in practice no implementation would use
+      // 64 kB or larger chunks.
+      v.reserve(10 * 65536);
+      assert(false);
+    } catch (const std::length_error&) {
+      // no-op
     }
+    assert(v.capacity() >= 5);
+  }
 #endif
 
-    return true;
+  return true;
 }
 
-int main(int, char**)
-{
-    tests();
+int main(int, char**) {
+  tests();
 #if TEST_STD_VER > 17
-    static_assert(tests());
+  static_assert(tests());
 #endif
-    return 0;
+  return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/vector.bool/resize_size.pass.cpp b/libcxx/test/std/containers/sequences/vector.bool/resize_size.pass.cpp
index ef83dbf0a9741..690537a3bbbf0 100644
--- a/libcxx/test/std/containers/sequences/vector.bool/resize_size.pass.cpp
+++ b/libcxx/test/std/containers/sequences/vector.bool/resize_size.pass.cpp
@@ -17,51 +17,49 @@
 #include "test_macros.h"
 #include "min_allocator.h"
 
-TEST_CONSTEXPR_CXX20 bool tests()
-{
-    {
-        std::vector<bool> v(100);
-        v.resize(50);
-        assert(v.size() == 50);
-        assert(v.capacity() >= 100);
-        v.resize(200);
-        assert(v.size() == 200);
-        assert(v.capacity() >= 200);
-        v.reserve(400);
-        v.resize(300);  // check the case when resizing and we already have room
-        assert(v.size() == 300);
-        assert(v.capacity() >= 400);
-    }
+TEST_CONSTEXPR_CXX20 bool tests() {
+  {
+    std::vector<bool> v(100);
+    v.resize(50);
+    assert(v.size() == 50);
+    assert(v.capacity() >= 100);
+    v.resize(200);
+    assert(v.size() == 200);
+    assert(v.capacity() >= 200);
+    v.reserve(400);
+    v.resize(300); // check the case when resizing and we already have room
+    assert(v.size() == 300);
+    assert(v.capacity() >= 400);
+  }
 #if TEST_STD_VER >= 11
-    {
-        std::vector<bool, explicit_allocator<bool>> v;
-        v.resize(10);
-        assert(v.size() == 10);
-        assert(v.capacity() >= 10);
-    }
-    {
-        std::vector<bool, min_allocator<bool>> v(100);
-        v.resize(50);
-        assert(v.size() == 50);
-        assert(v.capacity() >= 100);
-        v.resize(200);
-        assert(v.size() == 200);
-        assert(v.capacity() >= 200);
-        v.reserve(400);
-        v.resize(300);  // check the case when resizing and we already have room
-        assert(v.size() == 300);
-        assert(v.capacity() >= 400);
-    }
+  {
+    std::vector<bool, explicit_allocator<bool>> v;
+    v.resize(10);
+    assert(v.size() == 10);
+    assert(v.capacity() >= 10);
+  }
+  {
+    std::vector<bool, min_allocator<bool>> v(100);
+    v.resize(50);
+    assert(v.size() == 50);
+    assert(v.capacity() >= 100);
+    v.resize(200);
+    assert(v.size() == 200);
+    assert(v.capacity() >= 200);
+    v.reserve(400);
+    v.resize(300); // check the case when resizing and we already have room
+    assert(v.size() == 300);
+    assert(v.capacity() >= 400);
+  }
 #endif
 
-    return true;
+  return true;
 }
 
-int main(int, char**)
-{
-    tests();
+int main(int, char**) {
+  tests();
 #if TEST_STD_VER > 17
-    static_assert(tests());
+  static_assert(tests());
 #endif
-    return 0;
+  return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/vector.bool/resize_size_value.pass.cpp b/libcxx/test/std/containers/sequences/vector.bool/resize_size_value.pass.cpp
index fa7ae3d3420a4..138a837e8a2f6 100644
--- a/libcxx/test/std/containers/sequences/vector.bool/resize_size_value.pass.cpp
+++ b/libcxx/test/std/containers/sequences/vector.bool/resize_size_value.pass.cpp
@@ -17,47 +17,45 @@
 #include "test_macros.h"
 #include "min_allocator.h"
 
-TEST_CONSTEXPR_CXX20 bool tests()
-{
-    {
-        std::vector<bool> v(100);
-        v.resize(50, 1);
-        assert(v.size() == 50);
-        assert(v.capacity() >= 100);
-        assert(v == std::vector<bool>(50));
-        v.resize(200, 1);
-        assert(v.size() == 200);
-        assert(v.capacity() >= 200);
-        for (unsigned i = 0; i < 50; ++i)
-            assert(v[i] == 0);
-        for (unsigned i = 50; i < 200; ++i)
-            assert(v[i] == 1);
-    }
+TEST_CONSTEXPR_CXX20 bool tests() {
+  {
+    std::vector<bool> v(100);
+    v.resize(50, 1);
+    assert(v.size() == 50);
+    assert(v.capacity() >= 100);
+    assert(v == std::vector<bool>(50));
+    v.resize(200, 1);
+    assert(v.size() == 200);
+    assert(v.capacity() >= 200);
+    for (unsigned i = 0; i < 50; ++i)
+      assert(v[i] == 0);
+    for (unsigned i = 50; i < 200; ++i)
+      assert(v[i] == 1);
+  }
 #if TEST_STD_VER >= 11
-    {
-        std::vector<bool, min_allocator<bool>> v(100);
-        v.resize(50, 1);
-        assert(v.size() == 50);
-        assert(v.capacity() >= 100);
-        assert((v == std::vector<bool, min_allocator<bool>>(50)));
-        v.resize(200, 1);
-        assert(v.size() == 200);
-        assert(v.capacity() >= 200);
-        for (unsigned i = 0; i < 50; ++i)
-            assert(v[i] == 0);
-        for (unsigned i = 50; i < 200; ++i)
-            assert(v[i] == 1);
-    }
+  {
+    std::vector<bool, min_allocator<bool>> v(100);
+    v.resize(50, 1);
+    assert(v.size() == 50);
+    assert(v.capacity() >= 100);
+    assert((v == std::vector<bool, min_allocator<bool>>(50)));
+    v.resize(200, 1);
+    assert(v.size() == 200);
+    assert(v.capacity() >= 200);
+    for (unsigned i = 0; i < 50; ++i)
+      assert(v[i] == 0);
+    for (unsigned i = 50; i < 200; ++i)
+      assert(v[i] == 1);
+  }
 #endif
 
-    return true;
+  return true;
 }
 
-int main(int, char**)
-{
-    tests();
+int main(int, char**) {
+  tests();
 #if TEST_STD_VER > 17
-    static_assert(tests());
+  static_assert(tests());
 #endif
-    return 0;
+  return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/vector.bool/size.pass.cpp b/libcxx/test/std/containers/sequences/vector.bool/size.pass.cpp
index 33f8b72dd4722..aeac3293ec606 100644
--- a/libcxx/test/std/containers/sequences/vector.bool/size.pass.cpp
+++ b/libcxx/test/std/containers/sequences/vector.bool/size.pass.cpp
@@ -18,9 +18,8 @@
 #include "test_macros.h"
 #include "min_allocator.h"
 
-TEST_CONSTEXPR_CXX20 bool tests()
-{
-    {
+TEST_CONSTEXPR_CXX20 bool tests() {
+  {
     typedef std::vector<bool> C;
     C c;
     ASSERT_NOEXCEPT(c.size());
@@ -37,9 +36,9 @@ TEST_CONSTEXPR_CXX20 bool tests()
     assert(c.size() == 1);
     c.erase(c.begin());
     assert(c.size() == 0);
-    }
+  }
 #if TEST_STD_VER >= 11
-    {
+  {
     typedef std::vector<bool, min_allocator<bool>> C;
     C c;
     ASSERT_NOEXCEPT(c.size());
@@ -56,17 +55,16 @@ TEST_CONSTEXPR_CXX20 bool tests()
     assert(c.size() == 1);
     c.erase(c.begin());
     assert(c.size() == 0);
-    }
+  }
 #endif
 
-    return true;
+  return true;
 }
 
-int main(int, char**)
-{
-    tests();
+int main(int, char**) {
+  tests();
 #if TEST_STD_VER > 17
-    static_assert(tests());
+  static_assert(tests());
 #endif
-    return 0;
+  return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/vector.bool/swap.pass.cpp b/libcxx/test/std/containers/sequences/vector.bool/swap.pass.cpp
index edd28ef308471..9b29a245e389a 100644
--- a/libcxx/test/std/containers/sequences/vector.bool/swap.pass.cpp
+++ b/libcxx/test/std/containers/sequences/vector.bool/swap.pass.cpp
@@ -17,93 +17,91 @@
 #include "test_allocator.h"
 #include "min_allocator.h"
 
-TEST_CONSTEXPR_CXX20 bool tests()
-{
-    {
-        std::vector<bool> v1(100);
-        std::vector<bool> v2(200);
-        v1.swap(v2);
-        assert(v1.size() == 200);
-        assert(v1.capacity() >= 200);
-        assert(v2.size() == 100);
-        assert(v2.capacity() >= 100);
-    }
-    {
-        typedef test_allocator<bool> A;
-        std::vector<bool, A> v1(100, true, A(1, 1));
-        std::vector<bool, A> v2(200, false, A(1, 2));
-        swap(v1, v2);
-        assert(v1.size() == 200);
-        assert(v1.capacity() >= 200);
-        assert(v2.size() == 100);
-        assert(v2.capacity() >= 100);
-        assert(v1.get_allocator().get_id() == 1);
-        assert(v2.get_allocator().get_id() == 2);
-    }
-    {
-        typedef other_allocator<bool> A;
-        std::vector<bool, A> v1(100, true, A(1));
-        std::vector<bool, A> v2(200, false, A(2));
-        swap(v1, v2);
-        assert(v1.size() == 200);
-        assert(v1.capacity() >= 200);
-        assert(v2.size() == 100);
-        assert(v2.capacity() >= 100);
-        assert(v1.get_allocator() == A(2));
-        assert(v2.get_allocator() == A(1));
-    }
-    {
-        std::vector<bool> v(2);
-        std::vector<bool>::reference r1 = v[0];
-        std::vector<bool>::reference r2 = v[1];
-        r1 = true;
-        using std::swap;
-        swap(r1, r2);
-        assert(v[0] == false);
-        assert(v[1] == true);
-    }
+TEST_CONSTEXPR_CXX20 bool tests() {
+  {
+    std::vector<bool> v1(100);
+    std::vector<bool> v2(200);
+    v1.swap(v2);
+    assert(v1.size() == 200);
+    assert(v1.capacity() >= 200);
+    assert(v2.size() == 100);
+    assert(v2.capacity() >= 100);
+  }
+  {
+    typedef test_allocator<bool> A;
+    std::vector<bool, A> v1(100, true, A(1, 1));
+    std::vector<bool, A> v2(200, false, A(1, 2));
+    swap(v1, v2);
+    assert(v1.size() == 200);
+    assert(v1.capacity() >= 200);
+    assert(v2.size() == 100);
+    assert(v2.capacity() >= 100);
+    assert(v1.get_allocator().get_id() == 1);
+    assert(v2.get_allocator().get_id() == 2);
+  }
+  {
+    typedef other_allocator<bool> A;
+    std::vector<bool, A> v1(100, true, A(1));
+    std::vector<bool, A> v2(200, false, A(2));
+    swap(v1, v2);
+    assert(v1.size() == 200);
+    assert(v1.capacity() >= 200);
+    assert(v2.size() == 100);
+    assert(v2.capacity() >= 100);
+    assert(v1.get_allocator() == A(2));
+    assert(v2.get_allocator() == A(1));
+  }
+  {
+    std::vector<bool> v(2);
+    std::vector<bool>::reference r1 = v[0];
+    std::vector<bool>::reference r2 = v[1];
+    r1                              = true;
+    using std::swap;
+    swap(r1, r2);
+    assert(v[0] == false);
+    assert(v[1] == true);
+  }
 #if TEST_STD_VER >= 11
-    {
-        std::vector<bool, min_allocator<bool>> v1(100);
-        std::vector<bool, min_allocator<bool>> v2(200);
-        v1.swap(v2);
-        assert(v1.size() == 200);
-        assert(v1.capacity() >= 200);
-        assert(v2.size() == 100);
-        assert(v2.capacity() >= 100);
-    }
-    {
-        typedef min_allocator<bool> A;
-        std::vector<bool, A> v1(100, true, A());
-        std::vector<bool, A> v2(200, false, A());
-        swap(v1, v2);
-        assert(v1.size() == 200);
-        assert(v1.capacity() >= 200);
-        assert(v2.size() == 100);
-        assert(v2.capacity() >= 100);
-        assert(v1.get_allocator() == A());
-        assert(v2.get_allocator() == A());
-    }
-    {
-        std::vector<bool, min_allocator<bool>> v(2);
-        std::vector<bool, min_allocator<bool>>::reference r1 = v[0];
-        std::vector<bool, min_allocator<bool>>::reference r2 = v[1];
-        r1 = true;
-        using std::swap;
-        swap(r1, r2);
-        assert(v[0] == false);
-        assert(v[1] == true);
-    }
+  {
+    std::vector<bool, min_allocator<bool>> v1(100);
+    std::vector<bool, min_allocator<bool>> v2(200);
+    v1.swap(v2);
+    assert(v1.size() == 200);
+    assert(v1.capacity() >= 200);
+    assert(v2.size() == 100);
+    assert(v2.capacity() >= 100);
+  }
+  {
+    typedef min_allocator<bool> A;
+    std::vector<bool, A> v1(100, true, A());
+    std::vector<bool, A> v2(200, false, A());
+    swap(v1, v2);
+    assert(v1.size() == 200);
+    assert(v1.capacity() >= 200);
+    assert(v2.size() == 100);
+    assert(v2.capacity() >= 100);
+    assert(v1.get_allocator() == A());
+    assert(v2.get_allocator() == A());
+  }
+  {
+    std::vector<bool, min_allocator<bool>> v(2);
+    std::vector<bool, min_allocator<bool>>::reference r1 = v[0];
+    std::vector<bool, min_allocator<bool>>::reference r2 = v[1];
+    r1                                                   = true;
+    using std::swap;
+    swap(r1, r2);
+    assert(v[0] == false);
+    assert(v[1] == true);
+  }
 #endif
 
-    return true;
+  return true;
 }
 
-int main(int, char**)
-{
-    tests();
+int main(int, char**) {
+  tests();
 #if TEST_STD_VER > 17
-    static_assert(tests());
+  static_assert(tests());
 #endif
-    return 0;
+  return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/vector.bool/swap_noexcept.pass.cpp b/libcxx/test/std/containers/sequences/vector.bool/swap_noexcept.pass.cpp
index 40f7ae05be9b9..1683191403709 100644
--- a/libcxx/test/std/containers/sequences/vector.bool/swap_noexcept.pass.cpp
+++ b/libcxx/test/std/containers/sequences/vector.bool/swap_noexcept.pass.cpp
@@ -28,66 +28,63 @@
 #include "test_allocator.h"
 
 template <class T>
-struct some_alloc
-{
-    typedef T value_type;
+struct some_alloc {
+  typedef T value_type;
 
-    some_alloc() {}
-    some_alloc(const some_alloc&);
-    void deallocate(void*, unsigned) {}
+  some_alloc() {}
+  some_alloc(const some_alloc&);
+  void deallocate(void*, unsigned) {}
 
-    typedef std::true_type propagate_on_container_swap;
+  typedef std::true_type propagate_on_container_swap;
 };
 
 template <class T>
-struct some_alloc2
-{
-    typedef T value_type;
+struct some_alloc2 {
+  typedef T value_type;
 
-    some_alloc2() {}
-    some_alloc2(const some_alloc2&);
-    void deallocate(void*, unsigned) {}
+  some_alloc2() {}
+  some_alloc2(const some_alloc2&);
+  void deallocate(void*, unsigned) {}
 
-    typedef std::false_type propagate_on_container_swap;
-    typedef std::true_type is_always_equal;
+  typedef std::false_type propagate_on_container_swap;
+  typedef std::true_type is_always_equal;
 };
 
-int main(int, char**)
-{
+int main(int, char**) {
 #if defined(_LIBCPP_VERSION)
-    {
-        typedef std::vector<bool> C;
-        static_assert(noexcept(swap(std::declval<C&>(), std::declval<C&>())), "");
-    }
-    {
-        typedef std::vector<bool, test_allocator<bool>> C;
-        static_assert(noexcept(swap(std::declval<C&>(), std::declval<C&>())), "");
-    }
-    {
-        typedef std::vector<bool, other_allocator<bool>> C;
-        static_assert(noexcept(swap(std::declval<C&>(), std::declval<C&>())), "");
-    }
+  {
+    typedef std::vector<bool> C;
+    static_assert(noexcept(swap(std::declval<C&>(), std::declval<C&>())), "");
+  }
+  {
+    typedef std::vector<bool, test_allocator<bool>> C;
+    static_assert(noexcept(swap(std::declval<C&>(), std::declval<C&>())), "");
+  }
+  {
+    typedef std::vector<bool, other_allocator<bool>> C;
+    static_assert(noexcept(swap(std::declval<C&>(), std::declval<C&>())), "");
+  }
 #endif // _LIBCPP_VERSION
-    {
+  {
 #if TEST_STD_VER >= 14
-#if defined(_LIBCPP_VERSION)
+#  if defined(_LIBCPP_VERSION)
     //  In C++14, if POCS is set, swapping the allocator is required not to throw
-        typedef std::vector<bool, some_alloc<bool>> C;
-        static_assert( noexcept(swap(std::declval<C&>(), std::declval<C&>())), "");
-#endif // _LIBCPP_VERSION
+    typedef std::vector<bool, some_alloc<bool>> C;
+    static_assert(noexcept(swap(std::declval<C&>(), std::declval<C&>())), "");
+#  endif // _LIBCPP_VERSION
 #else
-        typedef std::vector<bool, some_alloc<bool>> C;
-        static_assert(!noexcept(swap(std::declval<C&>(), std::declval<C&>())), "");
+    typedef std::vector<bool, some_alloc<bool>> C;
+    static_assert(!noexcept(swap(std::declval<C&>(), std::declval<C&>())), "");
 #endif
-    }
+  }
 #if TEST_STD_VER >= 14
-#if defined(_LIBCPP_VERSION)
-    {
-        typedef std::vector<bool, some_alloc2<bool>> C;
+#  if defined(_LIBCPP_VERSION)
+  {
+    typedef std::vector<bool, some_alloc2<bool>> C;
     //  if the allocators are always equal, then the swap can be noexcept
-        static_assert( noexcept(swap(std::declval<C&>(), std::declval<C&>())), "");
-    }
-#endif // _LIBCPP_VERSION
+    static_assert(noexcept(swap(std::declval<C&>(), std::declval<C&>())), "");
+  }
+#  endif // _LIBCPP_VERSION
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/sequences/vector.bool/types.pass.cpp b/libcxx/test/std/containers/sequences/vector.bool/types.pass.cpp
index d75d715147352..4aaf02ef31f8b 100644
--- a/libcxx/test/std/containers/sequences/vector.bool/types.pass.cpp
+++ b/libcxx/test/std/containers/sequences/vector.bool/types.pass.cpp
@@ -37,49 +37,47 @@
 #include "min_allocator.h"
 
 template <class Allocator>
-void
-test()
-{
-    typedef std::vector<bool, Allocator> C;
+void test() {
+  typedef std::vector<bool, Allocator> C;
 
-    static_assert((std::is_same<typename C::value_type, bool>::value), "");
-    static_assert((std::is_same<typename C::value_type, typename Allocator::value_type>::value), "");
-    static_assert((std::is_same<typename C::allocator_type, Allocator>::value), "");
-    static_assert((std::is_same<typename C::size_type, typename std::allocator_traits<Allocator>::size_type>::value), "");
-    static_assert((std::is_same<typename C::difference_type, typename std::allocator_traits<Allocator>::difference_type>::value), "");
+  static_assert((std::is_same<typename C::value_type, bool>::value), "");
+  static_assert((std::is_same<typename C::value_type, typename Allocator::value_type>::value), "");
+  static_assert((std::is_same<typename C::allocator_type, Allocator>::value), "");
+  static_assert((std::is_same<typename C::size_type, typename std::allocator_traits<Allocator>::size_type>::value), "");
+  static_assert(
+      (std::is_same<typename C::difference_type, typename std::allocator_traits<Allocator>::difference_type>::value),
+      "");
 
-    static_assert((std::is_signed<typename C::difference_type>::value), "");
-    static_assert((std::is_unsigned<typename C::size_type>::value), "");
-    static_assert((std::is_same<typename C::difference_type,
-        typename std::iterator_traits<typename C::iterator>::difference_type>::value), "");
-    static_assert((std::is_same<typename C::difference_type,
-        typename std::iterator_traits<typename C::const_iterator>::difference_type>::value), "");
+  static_assert((std::is_signed<typename C::difference_type>::value), "");
+  static_assert((std::is_unsigned<typename C::size_type>::value), "");
+  static_assert((std::is_same<typename C::difference_type,
+                              typename std::iterator_traits<typename C::iterator>::difference_type>::value),
+                "");
+  static_assert((std::is_same<typename C::difference_type,
+                              typename std::iterator_traits<typename C::const_iterator>::difference_type>::value),
+                "");
 
-   static_assert((std::is_same<
-        typename std::iterator_traits<typename C::iterator>::iterator_category,
-        std::random_access_iterator_tag>::value), "");
-    static_assert((std::is_same<
-        typename std::iterator_traits<typename C::const_iterator>::iterator_category,
-        std::random_access_iterator_tag>::value), "");
-    static_assert((std::is_same<
-        typename C::reverse_iterator,
-        std::reverse_iterator<typename C::iterator> >::value), "");
-    static_assert((std::is_same<
-        typename C::const_reverse_iterator,
-        std::reverse_iterator<typename C::const_iterator> >::value), "");
+  static_assert((std::is_same< typename std::iterator_traits<typename C::iterator>::iterator_category,
+                               std::random_access_iterator_tag>::value),
+                "");
+  static_assert((std::is_same< typename std::iterator_traits<typename C::const_iterator>::iterator_category,
+                               std::random_access_iterator_tag>::value),
+                "");
+  static_assert((std::is_same< typename C::reverse_iterator, std::reverse_iterator<typename C::iterator> >::value), "");
+  static_assert(
+      (std::is_same< typename C::const_reverse_iterator, std::reverse_iterator<typename C::const_iterator> >::value),
+      "");
 #if !defined(_LIBCPP_VERSION) || defined(_LIBCPP_ABI_BITSET_VECTOR_BOOL_CONST_SUBSCRIPT_RETURN_BOOL)
-    static_assert(std::is_same<typename C::const_reference, bool>::value, "");
+  static_assert(std::is_same<typename C::const_reference, bool>::value, "");
 #endif
 }
 
-int main(int, char**)
-{
-    test<test_allocator<bool> >();
-    test<std::allocator<bool> >();
-    static_assert((std::is_same<std::vector<bool>::allocator_type,
-                                std::allocator<bool> >::value), "");
+int main(int, char**) {
+  test<test_allocator<bool> >();
+  test<std::allocator<bool> >();
+  static_assert((std::is_same<std::vector<bool>::allocator_type, std::allocator<bool> >::value), "");
 #if TEST_STD_VER >= 11
-    test<min_allocator<bool> >();
+  test<min_allocator<bool> >();
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/sequences/vector.bool/vector_bool.pass.cpp b/libcxx/test/std/containers/sequences/vector.bool/vector_bool.pass.cpp
index b927f2a4dbdde..e270869a8320f 100644
--- a/libcxx/test/std/containers/sequences/vector.bool/vector_bool.pass.cpp
+++ b/libcxx/test/std/containers/sequences/vector.bool/vector_bool.pass.cpp
@@ -24,14 +24,13 @@
 #include "test_macros.h"
 #include "min_allocator.h"
 
-TEST_CONSTEXPR_CXX20 bool tests()
-{
-    {
+TEST_CONSTEXPR_CXX20 bool tests() {
+  {
     typedef std::vector<bool> T;
     typedef std::hash<T> H;
 #if TEST_STD_VER <= 14
-    static_assert((std::is_same<H::argument_type, T>::value), "" );
-    static_assert((std::is_same<H::result_type, std::size_t>::value), "" );
+    static_assert((std::is_same<H::argument_type, T>::value), "");
+    static_assert((std::is_same<H::result_type, std::size_t>::value), "");
 #endif
     ASSERT_NOEXCEPT(H()(T()));
 
@@ -39,31 +38,30 @@ TEST_CONSTEXPR_CXX20 bool tests()
     T vb(std::begin(ba), std::end(ba));
     H h;
     assert(h(vb) != 0);
-    }
+  }
 #if TEST_STD_VER >= 11
-    {
+  {
     typedef std::vector<bool, min_allocator<bool>> T;
     typedef std::hash<T> H;
-#if TEST_STD_VER <= 14
-    static_assert((std::is_same<H::argument_type, T>::value), "" );
-    static_assert((std::is_same<H::result_type, std::size_t>::value), "" );
-#endif
+#  if TEST_STD_VER <= 14
+    static_assert((std::is_same<H::argument_type, T>::value), "");
+    static_assert((std::is_same<H::result_type, std::size_t>::value), "");
+#  endif
     ASSERT_NOEXCEPT(H()(T()));
     bool ba[] = {true, false, true, true, false};
     T vb(std::begin(ba), std::end(ba));
     H h;
     assert(h(vb) != 0);
-    }
+  }
 #endif
 
-    return true;
+  return true;
 }
 
-int main(int, char**)
-{
-    tests();
+int main(int, char**) {
+  tests();
 #if TEST_STD_VER > 17
-    static_assert(tests());
+  static_assert(tests());
 #endif
-    return 0;
+  return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/vector/access.pass.cpp b/libcxx/test/std/containers/sequences/vector/access.pass.cpp
index 026cf92c2d723..efee03d342f54 100644
--- a/libcxx/test/std/containers/sequences/vector/access.pass.cpp
+++ b/libcxx/test/std/containers/sequences/vector/access.pass.cpp
@@ -30,101 +30,102 @@
 
 template <class C>
 TEST_CONSTEXPR_CXX20 C make(int size, int start) {
-    C c;
-    for (int i = 0; i < size; ++i)
-        c.push_back(start + i);
-    return c;
+  C c;
+  for (int i = 0; i < size; ++i)
+    c.push_back(start + i);
+  return c;
 }
 
 template <class Vector>
 TEST_CONSTEXPR_CXX20 void test_get_basic(Vector& c, int start_value) {
-    const int n = static_cast<int>(c.size());
-    for (int i = 0; i < n; ++i)
-        assert(c[i] == start_value + i);
-    for (int i = 0; i < n; ++i)
-        assert(c.at(i) == start_value + i);
+  const int n = static_cast<int>(c.size());
+  for (int i = 0; i < n; ++i)
+    assert(c[i] == start_value + i);
+  for (int i = 0; i < n; ++i)
+    assert(c.at(i) == start_value + i);
 
 #ifndef TEST_HAS_NO_EXCEPTIONS
-    if (!TEST_IS_CONSTANT_EVALUATED) {
-        try {
-            TEST_IGNORE_NODISCARD c.at(n);
-            assert(false);
-        } catch (const std::out_of_range&) {}
+  if (!TEST_IS_CONSTANT_EVALUATED) {
+    try {
+      TEST_IGNORE_NODISCARD c.at(n);
+      assert(false);
+    } catch (const std::out_of_range&) {
     }
+  }
 #endif
 
-    assert(c.front() == start_value);
-    assert(c.back() == start_value + n - 1);
+  assert(c.front() == start_value);
+  assert(c.back() == start_value + n - 1);
 }
 
 template <class Vector>
 TEST_CONSTEXPR_CXX20 void test_get() {
-    int start_value = 35;
-    Vector c = make<Vector>(10, start_value);
-    const Vector& cc = c;
-    test_get_basic(c, start_value);
-    test_get_basic(cc, start_value);
+  int start_value  = 35;
+  Vector c         = make<Vector>(10, start_value);
+  const Vector& cc = c;
+  test_get_basic(c, start_value);
+  test_get_basic(cc, start_value);
 }
 
 template <class Vector>
 TEST_CONSTEXPR_CXX20 void test_set() {
-    int start_value = 35;
-    const int n = 10;
-    Vector c = make<Vector>(n, start_value);
-
-    for (int i = 0; i < n; ++i) {
-        assert(c[i] == start_value + i);
-        c[i] = start_value + i + 1;
-        assert(c[i] == start_value + i + 1);
-    }
-    for (int i = 0; i < n; ++i) {
-        assert(c.at(i) == start_value + i + 1);
-        c.at(i) = start_value + i + 2;
-        assert(c.at(i) == start_value + i + 2);
-    }
-
-    assert(c.front() == start_value + 2);
-    c.front() = start_value + 3;
-    assert(c.front() == start_value + 3);
-
-    assert(c.back() == start_value + n + 1);
-    c.back() = start_value + n + 2;
-    assert(c.back() == start_value + n + 2);
+  int start_value = 35;
+  const int n     = 10;
+  Vector c        = make<Vector>(n, start_value);
+
+  for (int i = 0; i < n; ++i) {
+    assert(c[i] == start_value + i);
+    c[i] = start_value + i + 1;
+    assert(c[i] == start_value + i + 1);
+  }
+  for (int i = 0; i < n; ++i) {
+    assert(c.at(i) == start_value + i + 1);
+    c.at(i) = start_value + i + 2;
+    assert(c.at(i) == start_value + i + 2);
+  }
+
+  assert(c.front() == start_value + 2);
+  c.front() = start_value + 3;
+  assert(c.front() == start_value + 3);
+
+  assert(c.back() == start_value + n + 1);
+  c.back() = start_value + n + 2;
+  assert(c.back() == start_value + n + 2);
 }
 
 template <class Vector>
 TEST_CONSTEXPR_CXX20 void test() {
-    test_get<Vector>();
-    test_set<Vector>();
+  test_get<Vector>();
+  test_set<Vector>();
 
-    Vector c;
-    const Vector& cc = c;
-    ASSERT_SAME_TYPE(typename Vector::reference, decltype(c[0]));
-    ASSERT_SAME_TYPE(typename Vector::const_reference, decltype(cc[0]));
+  Vector c;
+  const Vector& cc = c;
+  ASSERT_SAME_TYPE(typename Vector::reference, decltype(c[0]));
+  ASSERT_SAME_TYPE(typename Vector::const_reference, decltype(cc[0]));
 
-    ASSERT_SAME_TYPE(typename Vector::reference, decltype(c.at(0)));
-    ASSERT_SAME_TYPE(typename Vector::const_reference, decltype(cc.at(0)));
+  ASSERT_SAME_TYPE(typename Vector::reference, decltype(c.at(0)));
+  ASSERT_SAME_TYPE(typename Vector::const_reference, decltype(cc.at(0)));
 
-    ASSERT_SAME_TYPE(typename Vector::reference, decltype(c.front()));
-    ASSERT_SAME_TYPE(typename Vector::const_reference, decltype(cc.front()));
+  ASSERT_SAME_TYPE(typename Vector::reference, decltype(c.front()));
+  ASSERT_SAME_TYPE(typename Vector::const_reference, decltype(cc.front()));
 
-    ASSERT_SAME_TYPE(typename Vector::reference, decltype(c.back()));
-    ASSERT_SAME_TYPE(typename Vector::const_reference, decltype(cc.back()));
+  ASSERT_SAME_TYPE(typename Vector::reference, decltype(c.back()));
+  ASSERT_SAME_TYPE(typename Vector::const_reference, decltype(cc.back()));
 }
 
 TEST_CONSTEXPR_CXX20 bool tests() {
-    test<std::vector<int> >();
+  test<std::vector<int> >();
 #if TEST_STD_VER >= 11
-    test<std::vector<int, min_allocator<int> > >();
-    test<std::vector<int, safe_allocator<int> > >();
+  test<std::vector<int, min_allocator<int> > >();
+  test<std::vector<int, safe_allocator<int> > >();
 #endif
-    return true;
+  return true;
 }
 
 int main(int, char**) {
-    tests();
+  tests();
 #if TEST_STD_VER > 17
-    static_assert(tests());
+  static_assert(tests());
 #endif
-    return 0;
+  return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/vector/addressof.compile.pass.cpp b/libcxx/test/std/containers/sequences/vector/addressof.compile.pass.cpp
index 120b7b289af93..86f8d0e17e665 100644
--- a/libcxx/test/std/containers/sequences/vector/addressof.compile.pass.cpp
+++ b/libcxx/test/std/containers/sequences/vector/addressof.compile.pass.cpp
@@ -39,8 +39,12 @@ void test(
   v = std::move(v);
 
   // construction
-  { Vector v2(std::move(v)); }
-  { Vector v2(std::move(v), std::allocator<operator_hijacker>()); }
+  {
+    Vector v2(std::move(v));
+  }
+  {
+    Vector v2(std::move(v), std::allocator<operator_hijacker>());
+  }
 
   // swap
   v.swap(v);
diff --git a/libcxx/test/std/containers/sequences/vector/allocator_mismatch.verify.cpp b/libcxx/test/std/containers/sequences/vector/allocator_mismatch.verify.cpp
index 97edbf4d64f67..eb23c67689ee3 100644
--- a/libcxx/test/std/containers/sequences/vector/allocator_mismatch.verify.cpp
+++ b/libcxx/test/std/containers/sequences/vector/allocator_mismatch.verify.cpp
@@ -12,4 +12,4 @@
 #include <vector>
 
 std::vector<int, std::allocator<long> > v;
-  // expected-error-re@*:* {{static assertion failed{{.*}}Allocator::value_type must be same type as value_type}}
+// expected-error-re@*:* {{static assertion failed{{.*}}Allocator::value_type must be same type as value_type}}
diff --git a/libcxx/test/std/containers/sequences/vector/compare.pass.cpp b/libcxx/test/std/containers/sequences/vector/compare.pass.cpp
index 917095e1f138d..413767e52a4d6 100644
--- a/libcxx/test/std/containers/sequences/vector/compare.pass.cpp
+++ b/libcxx/test/std/containers/sequences/vector/compare.pass.cpp
@@ -21,109 +21,109 @@
 #include "test_comparisons.h"
 
 TEST_CONSTEXPR_CXX20 bool test() {
-    {
-        const std::vector<int> c1, c2;
-        assert(testComparisons(c1, c2, true, false));
-    }
-    {
-        const std::vector<int> c1(1, 1), c2(1, 2);
-        assert(testComparisons(c1, c2, false, true));
-    }
-    {
-        const std::vector<int> c1, c2(1, 2);
-        assert(testComparisons(c1, c2, false, true));
-    }
-    {
-        int items1[3] = {1, 2, 1};
-        int items2[3] = {1, 2, 2};
-        const std::vector<int> c1(items1, items1 + 3);
-        const std::vector<int> c2(items2, items2 + 3);
-        assert(testComparisons(c1, c2, false, true));
-    }
-    {
-        int items1[3] = {3, 2, 3};
-        int items2[3] = {3, 1, 3};
-        const std::vector<int> c1(items1, items1 + 3);
-        const std::vector<int> c2(items2, items2 + 3);
+  {
+    const std::vector<int> c1, c2;
+    assert(testComparisons(c1, c2, true, false));
+  }
+  {
+    const std::vector<int> c1(1, 1), c2(1, 2);
+    assert(testComparisons(c1, c2, false, true));
+  }
+  {
+    const std::vector<int> c1, c2(1, 2);
+    assert(testComparisons(c1, c2, false, true));
+  }
+  {
+    int items1[3] = {1, 2, 1};
+    int items2[3] = {1, 2, 2};
+    const std::vector<int> c1(items1, items1 + 3);
+    const std::vector<int> c2(items2, items2 + 3);
+    assert(testComparisons(c1, c2, false, true));
+  }
+  {
+    int items1[3] = {3, 2, 3};
+    int items2[3] = {3, 1, 3};
+    const std::vector<int> c1(items1, items1 + 3);
+    const std::vector<int> c2(items2, items2 + 3);
 
-        assert(testComparisons(c1, c2, false, false));
-    }
-    {
-        int items1[2] = {1, 2};
-        int items2[3] = {1, 2, 0};
-        const std::vector<int> c1(items1, items1 + 2);
-        const std::vector<int> c2(items2, items2 + 3);
-        assert(testComparisons(c1, c2, false, true));
-    }
-    {
-        int items1[3] = {1, 2, 0};
-        const std::vector<int> c1(items1, items1 + 3);
-        const std::vector<int> c2(1, 3);
-        assert(testComparisons(c1, c2, false, true));
-    }
-    {
-        const std::vector<LessAndEqComp> c1, c2;
-        assert(testComparisons(c1, c2, true, false));
-    }
-    {
-        const std::vector<LessAndEqComp> c1(1, LessAndEqComp(1));
-        const std::vector<LessAndEqComp> c2(1, LessAndEqComp(1));
-        assert(testComparisons(c1, c2, true, false));
-    }
-    {
-        const std::vector<LessAndEqComp> c1(1, LessAndEqComp(1));
-        const std::vector<LessAndEqComp> c2(1, LessAndEqComp(2));
-        assert(testComparisons(c1, c2, false, true));
-    }
-    {
-        const std::vector<LessAndEqComp> c1;
-        const std::vector<LessAndEqComp> c2(1, LessAndEqComp(2));
-        assert(testComparisons(c1, c2, false, true));
-    }
-    {
-        LessAndEqComp items1[3] = {LessAndEqComp(1), LessAndEqComp(2), LessAndEqComp(2)};
-        LessAndEqComp items2[3] = {LessAndEqComp(1), LessAndEqComp(2), LessAndEqComp(1)};
-        const std::vector<LessAndEqComp> c1(items1, items1 + 3);
-        const std::vector<LessAndEqComp> c2(items2, items2 + 3);
-        assert(testComparisons(c1, c2, false, false));
-    }
-    {
-        LessAndEqComp items1[3] = {LessAndEqComp(3), LessAndEqComp(3), LessAndEqComp(3)};
-        LessAndEqComp items2[3] = {LessAndEqComp(3), LessAndEqComp(2), LessAndEqComp(3)};
-        const std::vector<LessAndEqComp> c1(items1, items1 + 3);
-        const std::vector<LessAndEqComp> c2(items2, items2 + 3);
-        assert(testComparisons(c1, c2, false, false));
-    }
-    {
-        LessAndEqComp items1[2] = {LessAndEqComp(1), LessAndEqComp(2)};
-        LessAndEqComp items2[3] = {LessAndEqComp(1), LessAndEqComp(2), LessAndEqComp(0)};
-        const std::vector<LessAndEqComp> c1(items1, items1 + 2);
-        const std::vector<LessAndEqComp> c2(items2, items2 + 3);
-        assert(testComparisons(c1, c2, false, true));
-    }
-    {
-        LessAndEqComp items1[3] = {LessAndEqComp(1), LessAndEqComp(2), LessAndEqComp(0)};
-        const std::vector<LessAndEqComp> c1(items1, items1 + 3);
-        const std::vector<LessAndEqComp> c2(1, LessAndEqComp(3));
-        assert(testComparisons(c1, c2, false, true));
-    }
-    {
-        assert((std::vector<int>() == std::vector<int>()));
-        assert(!(std::vector<int>() != std::vector<int>()));
-        assert(!(std::vector<int>() < std::vector<int>()));
-        assert((std::vector<int>() <= std::vector<int>()));
-        assert(!(std::vector<int>() > std::vector<int>()));
-        assert((std::vector<int>() >= std::vector<int>()));
-    }
+    assert(testComparisons(c1, c2, false, false));
+  }
+  {
+    int items1[2] = {1, 2};
+    int items2[3] = {1, 2, 0};
+    const std::vector<int> c1(items1, items1 + 2);
+    const std::vector<int> c2(items2, items2 + 3);
+    assert(testComparisons(c1, c2, false, true));
+  }
+  {
+    int items1[3] = {1, 2, 0};
+    const std::vector<int> c1(items1, items1 + 3);
+    const std::vector<int> c2(1, 3);
+    assert(testComparisons(c1, c2, false, true));
+  }
+  {
+    const std::vector<LessAndEqComp> c1, c2;
+    assert(testComparisons(c1, c2, true, false));
+  }
+  {
+    const std::vector<LessAndEqComp> c1(1, LessAndEqComp(1));
+    const std::vector<LessAndEqComp> c2(1, LessAndEqComp(1));
+    assert(testComparisons(c1, c2, true, false));
+  }
+  {
+    const std::vector<LessAndEqComp> c1(1, LessAndEqComp(1));
+    const std::vector<LessAndEqComp> c2(1, LessAndEqComp(2));
+    assert(testComparisons(c1, c2, false, true));
+  }
+  {
+    const std::vector<LessAndEqComp> c1;
+    const std::vector<LessAndEqComp> c2(1, LessAndEqComp(2));
+    assert(testComparisons(c1, c2, false, true));
+  }
+  {
+    LessAndEqComp items1[3] = {LessAndEqComp(1), LessAndEqComp(2), LessAndEqComp(2)};
+    LessAndEqComp items2[3] = {LessAndEqComp(1), LessAndEqComp(2), LessAndEqComp(1)};
+    const std::vector<LessAndEqComp> c1(items1, items1 + 3);
+    const std::vector<LessAndEqComp> c2(items2, items2 + 3);
+    assert(testComparisons(c1, c2, false, false));
+  }
+  {
+    LessAndEqComp items1[3] = {LessAndEqComp(3), LessAndEqComp(3), LessAndEqComp(3)};
+    LessAndEqComp items2[3] = {LessAndEqComp(3), LessAndEqComp(2), LessAndEqComp(3)};
+    const std::vector<LessAndEqComp> c1(items1, items1 + 3);
+    const std::vector<LessAndEqComp> c2(items2, items2 + 3);
+    assert(testComparisons(c1, c2, false, false));
+  }
+  {
+    LessAndEqComp items1[2] = {LessAndEqComp(1), LessAndEqComp(2)};
+    LessAndEqComp items2[3] = {LessAndEqComp(1), LessAndEqComp(2), LessAndEqComp(0)};
+    const std::vector<LessAndEqComp> c1(items1, items1 + 2);
+    const std::vector<LessAndEqComp> c2(items2, items2 + 3);
+    assert(testComparisons(c1, c2, false, true));
+  }
+  {
+    LessAndEqComp items1[3] = {LessAndEqComp(1), LessAndEqComp(2), LessAndEqComp(0)};
+    const std::vector<LessAndEqComp> c1(items1, items1 + 3);
+    const std::vector<LessAndEqComp> c2(1, LessAndEqComp(3));
+    assert(testComparisons(c1, c2, false, true));
+  }
+  {
+    assert((std::vector<int>() == std::vector<int>()));
+    assert(!(std::vector<int>() != std::vector<int>()));
+    assert(!(std::vector<int>() < std::vector<int>()));
+    assert((std::vector<int>() <= std::vector<int>()));
+    assert(!(std::vector<int>() > std::vector<int>()));
+    assert((std::vector<int>() >= std::vector<int>()));
+  }
 
-    return true;
+  return true;
 }
 
 int main(int, char**) {
-    test();
+  test();
 #if TEST_STD_VER > 17
-    static_assert(test());
+  static_assert(test());
 #endif
 
-    return 0;
+  return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/vector/contiguous.pass.cpp b/libcxx/test/std/containers/sequences/vector/contiguous.pass.cpp
index 3c992ba9bb594..898f824e65ac0 100644
--- a/libcxx/test/std/containers/sequences/vector/contiguous.pass.cpp
+++ b/libcxx/test/std/containers/sequences/vector/contiguous.pass.cpp
@@ -18,53 +18,50 @@
 #include "min_allocator.h"
 
 template <class C>
-TEST_CONSTEXPR_CXX20 void test_contiguous(const C &c)
-{
-    for ( std::size_t i = 0; i < c.size(); ++i )
-        assert ( *(c.begin() + static_cast<typename C::difference_type>(i)) == *(std::addressof(*c.begin()) + i));
+TEST_CONSTEXPR_CXX20 void test_contiguous(const C& c) {
+  for (std::size_t i = 0; i < c.size(); ++i)
+    assert(*(c.begin() + static_cast<typename C::difference_type>(i)) == *(std::addressof(*c.begin()) + i));
 }
 
-TEST_CONSTEXPR_CXX20 bool tests()
-{
-    {
+TEST_CONSTEXPR_CXX20 bool tests() {
+  {
     typedef int T;
     typedef std::vector<T> C;
     test_contiguous(C());
     test_contiguous(C(3, 5));
-    }
+  }
 
-    {
+  {
     typedef double T;
     typedef test_allocator<T> A;
     typedef std::vector<T, A> C;
     test_contiguous(C(A(3)));
     test_contiguous(C(7, 9.0, A(5)));
-    }
+  }
 #if TEST_STD_VER >= 11
-    {
+  {
     typedef double T;
     typedef min_allocator<T> A;
     typedef std::vector<T, A> C;
     test_contiguous(C(A{}));
     test_contiguous(C(9, 11.0, A{}));
-    }
-    {
-      typedef double T;
-      typedef safe_allocator<T> A;
-      typedef std::vector<T, A> C;
-      test_contiguous(C(A{}));
-      test_contiguous(C(9, 11.0, A{}));
-    }
+  }
+  {
+    typedef double T;
+    typedef safe_allocator<T> A;
+    typedef std::vector<T, A> C;
+    test_contiguous(C(A{}));
+    test_contiguous(C(9, 11.0, A{}));
+  }
 #endif
 
-    return true;
+  return true;
 }
 
-int main(int, char**)
-{
-    tests();
+int main(int, char**) {
+  tests();
 #if TEST_STD_VER > 17
-    static_assert(tests());
+  static_assert(tests());
 #endif
-    return 0;
+  return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/vector/get_allocator.pass.cpp b/libcxx/test/std/containers/sequences/vector/get_allocator.pass.cpp
index c0003acae75ef..0398ff8bb1fd8 100644
--- a/libcxx/test/std/containers/sequences/vector/get_allocator.pass.cpp
+++ b/libcxx/test/std/containers/sequences/vector/get_allocator.pass.cpp
@@ -19,25 +19,25 @@
 #include "test_macros.h"
 
 TEST_CONSTEXPR_CXX20 bool test() {
-    {
-        std::allocator<int> alloc;
-        const std::vector<int> v(alloc);
-        assert(v.get_allocator() == alloc);
-    }
-    {
-        other_allocator<int> alloc(1);
-        const std::vector<int, other_allocator<int> > v(alloc);
-        assert(v.get_allocator() == alloc);
-    }
-
-    return true;
+  {
+    std::allocator<int> alloc;
+    const std::vector<int> v(alloc);
+    assert(v.get_allocator() == alloc);
+  }
+  {
+    other_allocator<int> alloc(1);
+    const std::vector<int, other_allocator<int> > v(alloc);
+    assert(v.get_allocator() == alloc);
+  }
+
+  return true;
 }
 
 int main(int, char**) {
-    test();
+  test();
 #if TEST_STD_VER > 17
-    static_assert(test());
+  static_assert(test());
 #endif
 
-    return 0;
+  return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/vector/iterator_concept_conformance.compile.pass.cpp b/libcxx/test/std/containers/sequences/vector/iterator_concept_conformance.compile.pass.cpp
index f0dc2ba834ae1..41b5bde5e9ad3 100644
--- a/libcxx/test/std/containers/sequences/vector/iterator_concept_conformance.compile.pass.cpp
+++ b/libcxx/test/std/containers/sequences/vector/iterator_concept_conformance.compile.pass.cpp
@@ -14,68 +14,68 @@
 
 #include <iterator>
 
-using iterator = std::vector<int>::iterator;
-using const_iterator = std::vector<int>::const_iterator;
-using reverse_iterator = std::vector<int>::reverse_iterator;
+using iterator               = std::vector<int>::iterator;
+using const_iterator         = std::vector<int>::const_iterator;
+using reverse_iterator       = std::vector<int>::reverse_iterator;
 using const_reverse_iterator = std::vector<int>::const_reverse_iterator;
-using value_type = int;
+using value_type             = int;
 
-static_assert( std::contiguous_iterator<iterator>);
-static_assert( std::random_access_iterator<reverse_iterator>);
+static_assert(std::contiguous_iterator<iterator>);
+static_assert(std::random_access_iterator<reverse_iterator>);
 static_assert(!std::contiguous_iterator<reverse_iterator>);
-static_assert( std::indirectly_writable<iterator, value_type>);
-static_assert( std::sentinel_for<iterator, iterator>);
-static_assert( std::sentinel_for<iterator, const_iterator>);
+static_assert(std::indirectly_writable<iterator, value_type>);
+static_assert(std::sentinel_for<iterator, iterator>);
+static_assert(std::sentinel_for<iterator, const_iterator>);
 static_assert(!std::sentinel_for<iterator, reverse_iterator>);
 static_assert(!std::sentinel_for<iterator, const_reverse_iterator>);
-static_assert( std::sized_sentinel_for<iterator, iterator>);
-static_assert( std::sized_sentinel_for<iterator, const_iterator>);
+static_assert(std::sized_sentinel_for<iterator, iterator>);
+static_assert(std::sized_sentinel_for<iterator, const_iterator>);
 static_assert(!std::sized_sentinel_for<iterator, reverse_iterator>);
 static_assert(!std::sized_sentinel_for<iterator, const_reverse_iterator>);
-static_assert( std::indirectly_movable<iterator, iterator>);
-static_assert( std::indirectly_movable_storable<iterator, iterator>);
+static_assert(std::indirectly_movable<iterator, iterator>);
+static_assert(std::indirectly_movable_storable<iterator, iterator>);
 static_assert(!std::indirectly_movable<iterator, const_iterator>);
 static_assert(!std::indirectly_movable_storable<iterator, const_iterator>);
-static_assert( std::indirectly_movable<iterator, reverse_iterator>);
-static_assert( std::indirectly_movable_storable<iterator, reverse_iterator>);
+static_assert(std::indirectly_movable<iterator, reverse_iterator>);
+static_assert(std::indirectly_movable_storable<iterator, reverse_iterator>);
 static_assert(!std::indirectly_movable<iterator, const_reverse_iterator>);
 static_assert(!std::indirectly_movable_storable<iterator, const_reverse_iterator>);
-static_assert( std::indirectly_copyable<iterator, iterator>);
-static_assert( std::indirectly_copyable_storable<iterator, iterator>);
+static_assert(std::indirectly_copyable<iterator, iterator>);
+static_assert(std::indirectly_copyable_storable<iterator, iterator>);
 static_assert(!std::indirectly_copyable<iterator, const_iterator>);
 static_assert(!std::indirectly_copyable_storable<iterator, const_iterator>);
-static_assert( std::indirectly_copyable<iterator, reverse_iterator>);
-static_assert( std::indirectly_copyable_storable<iterator, reverse_iterator>);
+static_assert(std::indirectly_copyable<iterator, reverse_iterator>);
+static_assert(std::indirectly_copyable_storable<iterator, reverse_iterator>);
 static_assert(!std::indirectly_copyable<iterator, const_reverse_iterator>);
 static_assert(!std::indirectly_copyable_storable<iterator, const_reverse_iterator>);
 static_assert(std::indirectly_swappable<iterator, iterator>);
 
-static_assert( std::contiguous_iterator<const_iterator>);
-static_assert( std::random_access_iterator<const_reverse_iterator>);
+static_assert(std::contiguous_iterator<const_iterator>);
+static_assert(std::random_access_iterator<const_reverse_iterator>);
 static_assert(!std::contiguous_iterator<const_reverse_iterator>);
 static_assert(!std::indirectly_writable<const_iterator, value_type>);
-static_assert( std::sentinel_for<const_iterator, iterator>);
-static_assert( std::sentinel_for<const_iterator, const_iterator>);
+static_assert(std::sentinel_for<const_iterator, iterator>);
+static_assert(std::sentinel_for<const_iterator, const_iterator>);
 static_assert(!std::sentinel_for<const_iterator, reverse_iterator>);
 static_assert(!std::sentinel_for<const_iterator, const_reverse_iterator>);
-static_assert( std::sized_sentinel_for<const_iterator, iterator>);
-static_assert( std::sized_sentinel_for<const_iterator, const_iterator>);
+static_assert(std::sized_sentinel_for<const_iterator, iterator>);
+static_assert(std::sized_sentinel_for<const_iterator, const_iterator>);
 static_assert(!std::sized_sentinel_for<const_iterator, reverse_iterator>);
 static_assert(!std::sized_sentinel_for<const_iterator, const_reverse_iterator>);
-static_assert( std::indirectly_movable<const_iterator, iterator>);
-static_assert( std::indirectly_movable_storable<const_iterator, iterator>);
+static_assert(std::indirectly_movable<const_iterator, iterator>);
+static_assert(std::indirectly_movable_storable<const_iterator, iterator>);
 static_assert(!std::indirectly_movable<const_iterator, const_iterator>);
 static_assert(!std::indirectly_movable_storable<const_iterator, const_iterator>);
-static_assert( std::indirectly_movable<const_iterator, reverse_iterator>);
-static_assert( std::indirectly_movable_storable<const_iterator, reverse_iterator>);
+static_assert(std::indirectly_movable<const_iterator, reverse_iterator>);
+static_assert(std::indirectly_movable_storable<const_iterator, reverse_iterator>);
 static_assert(!std::indirectly_movable<const_iterator, const_reverse_iterator>);
 static_assert(!std::indirectly_movable_storable<const_iterator, const_reverse_iterator>);
-static_assert( std::indirectly_copyable<const_iterator, iterator>);
-static_assert( std::indirectly_copyable_storable<const_iterator, iterator>);
+static_assert(std::indirectly_copyable<const_iterator, iterator>);
+static_assert(std::indirectly_copyable_storable<const_iterator, iterator>);
 static_assert(!std::indirectly_copyable<const_iterator, const_iterator>);
 static_assert(!std::indirectly_copyable_storable<const_iterator, const_iterator>);
-static_assert( std::indirectly_copyable<const_iterator, reverse_iterator>);
-static_assert( std::indirectly_copyable_storable<const_iterator, reverse_iterator>);
+static_assert(std::indirectly_copyable<const_iterator, reverse_iterator>);
+static_assert(std::indirectly_copyable_storable<const_iterator, reverse_iterator>);
 static_assert(!std::indirectly_copyable<const_iterator, const_reverse_iterator>);
 static_assert(!std::indirectly_copyable_storable<const_iterator, const_reverse_iterator>);
 static_assert(!std::indirectly_swappable<const_iterator, const_iterator>);
diff --git a/libcxx/test/std/containers/sequences/vector/iterators.pass.cpp b/libcxx/test/std/containers/sequences/vector/iterators.pass.cpp
index 0aa7ad0d42ed7..5b0b9f4d1d763 100644
--- a/libcxx/test/std/containers/sequences/vector/iterators.pass.cpp
+++ b/libcxx/test/std/containers/sequences/vector/iterators.pass.cpp
@@ -22,206 +22,203 @@
 #include "test_macros.h"
 #include "min_allocator.h"
 
-struct A
-{
-    int first;
-    int second;
+struct A {
+  int first;
+  int second;
 };
 
-TEST_CONSTEXPR_CXX20 bool tests()
-{
-    {
-        typedef int T;
-        typedef std::vector<T> C;
-        C c;
-        C::iterator i = c.begin();
-        C::iterator j = c.end();
-        assert(std::distance(i, j) == 0);
-        assert(i == j);
-    }
-    {
-        typedef int T;
-        typedef std::vector<T> C;
-        const C c;
-        C::const_iterator i = c.begin();
-        C::const_iterator j = c.end();
-        assert(std::distance(i, j) == 0);
-        assert(i == j);
-    }
-    {
-        typedef int T;
-        typedef std::vector<T> C;
-        C c;
-        C::const_iterator i = c.cbegin();
-        C::const_iterator j = c.cend();
-        assert(std::distance(i, j) == 0);
-        assert(i == j);
-        assert(i == c.end());
-    }
-    {
-        typedef int T;
-        typedef std::vector<T> C;
-        const T t[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9};
-        C c(std::begin(t), std::end(t));
-        C::iterator i = c.begin();
-        assert(*i == 0);
-        ++i;
-        assert(*i == 1);
-        *i = 10;
-        assert(*i == 10);
-        assert(std::distance(c.begin(), c.end()) == 10);
-    }
-    {
-        typedef int T;
-        typedef std::vector<T> C;
-        C::iterator i;
-        C::const_iterator j;
-        (void) i;
-        (void) j;
-    }
+TEST_CONSTEXPR_CXX20 bool tests() {
+  {
+    typedef int T;
+    typedef std::vector<T> C;
+    C c;
+    C::iterator i = c.begin();
+    C::iterator j = c.end();
+    assert(std::distance(i, j) == 0);
+    assert(i == j);
+  }
+  {
+    typedef int T;
+    typedef std::vector<T> C;
+    const C c;
+    C::const_iterator i = c.begin();
+    C::const_iterator j = c.end();
+    assert(std::distance(i, j) == 0);
+    assert(i == j);
+  }
+  {
+    typedef int T;
+    typedef std::vector<T> C;
+    C c;
+    C::const_iterator i = c.cbegin();
+    C::const_iterator j = c.cend();
+    assert(std::distance(i, j) == 0);
+    assert(i == j);
+    assert(i == c.end());
+  }
+  {
+    typedef int T;
+    typedef std::vector<T> C;
+    const T t[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9};
+    C c(std::begin(t), std::end(t));
+    C::iterator i = c.begin();
+    assert(*i == 0);
+    ++i;
+    assert(*i == 1);
+    *i = 10;
+    assert(*i == 10);
+    assert(std::distance(c.begin(), c.end()) == 10);
+  }
+  {
+    typedef int T;
+    typedef std::vector<T> C;
+    C::iterator i;
+    C::const_iterator j;
+    (void)i;
+    (void)j;
+  }
 #if TEST_STD_VER >= 11
-    {
-        typedef int T;
-        typedef std::vector<T, min_allocator<T>> C;
-        C c;
-        C::iterator i = c.begin();
-        C::iterator j = c.end();
-        assert(std::distance(i, j) == 0);
+  {
+    typedef int T;
+    typedef std::vector<T, min_allocator<T>> C;
+    C c;
+    C::iterator i = c.begin();
+    C::iterator j = c.end();
+    assert(std::distance(i, j) == 0);
 
-        assert(i == j);
-        assert(!(i != j));
+    assert(i == j);
+    assert(!(i != j));
 
-        assert(!(i < j));
-        assert((i <= j));
+    assert(!(i < j));
+    assert((i <= j));
 
-        assert(!(i > j));
-        assert((i >= j));
+    assert(!(i > j));
+    assert((i >= j));
 
 #  if TEST_STD_VER >= 20
-        // P1614 + LWG3352
-        // When the allocator does not have operator<=> then the iterator uses a
-        // fallback to provide operator<=>.
-        // Make sure to test with an allocator that does not have operator<=>.
-        static_assert(!std::three_way_comparable<min_allocator<int>, std::strong_ordering>);
-        static_assert(std::three_way_comparable<typename C::iterator, std::strong_ordering>);
-
-        std::same_as<std::strong_ordering> decltype(auto) r1 = i <=> j;
-        assert(r1 == std::strong_ordering::equal);
+    // P1614 + LWG3352
+    // When the allocator does not have operator<=> then the iterator uses a
+    // fallback to provide operator<=>.
+    // Make sure to test with an allocator that does not have operator<=>.
+    static_assert(!std::three_way_comparable<min_allocator<int>, std::strong_ordering>);
+    static_assert(std::three_way_comparable<typename C::iterator, std::strong_ordering>);
+
+    std::same_as<std::strong_ordering> decltype(auto) r1 = i <=> j;
+    assert(r1 == std::strong_ordering::equal);
 #  endif
-    }
-    {
-        typedef int T;
-        typedef std::vector<T, min_allocator<T>> C;
-        const C c;
-        C::const_iterator i = c.begin();
-        C::const_iterator j = c.end();
-        assert(std::distance(i, j) == 0);
+  }
+  {
+    typedef int T;
+    typedef std::vector<T, min_allocator<T>> C;
+    const C c;
+    C::const_iterator i = c.begin();
+    C::const_iterator j = c.end();
+    assert(std::distance(i, j) == 0);
 
-        assert(i == j);
-        assert(!(i != j));
+    assert(i == j);
+    assert(!(i != j));
 
-        assert(!(i < j));
-        assert((i <= j));
+    assert(!(i < j));
+    assert((i <= j));
 
-        assert(!(i > j));
-        assert((i >= j));
+    assert(!(i > j));
+    assert((i >= j));
 
 #  if TEST_STD_VER >= 20
-        // When the allocator does not have operator<=> then the iterator uses a
-        // fallback to provide operator<=>.
-        // Make sure to test with an allocator that does not have operator<=>.
-        static_assert(!std::three_way_comparable<min_allocator<int>, std::strong_ordering>);
-        static_assert(std::three_way_comparable<typename C::iterator, std::strong_ordering>);
-
-        std::same_as<std::strong_ordering> decltype(auto) r1 = i <=> j;
-        assert(r1 == std::strong_ordering::equal);
+    // When the allocator does not have operator<=> then the iterator uses a
+    // fallback to provide operator<=>.
+    // Make sure to test with an allocator that does not have operator<=>.
+    static_assert(!std::three_way_comparable<min_allocator<int>, std::strong_ordering>);
+    static_assert(std::three_way_comparable<typename C::iterator, std::strong_ordering>);
+
+    std::same_as<std::strong_ordering> decltype(auto) r1 = i <=> j;
+    assert(r1 == std::strong_ordering::equal);
 #  endif
-    }
-    {
-        typedef int T;
-        typedef std::vector<T, min_allocator<T>> C;
-        C c;
-        C::const_iterator i = c.cbegin();
-        C::const_iterator j = c.cend();
-        assert(std::distance(i, j) == 0);
-        assert(i == j);
-        assert(i == c.end());
-    }
-    {
-        typedef int T;
-        typedef std::vector<T, min_allocator<T>> C;
-        const T t[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9};
-        C c(std::begin(t), std::end(t));
-        C::iterator i = c.begin();
-        assert(*i == 0);
-        ++i;
-        assert(*i == 1);
-        *i = 10;
-        assert(*i == 10);
-        assert(std::distance(c.begin(), c.end()) == 10);
-    }
-    {
-        typedef int T;
-        typedef std::vector<T, min_allocator<T>> C;
-        C::iterator i;
-        C::const_iterator j;
-        (void) i;
-        (void) j;
-    }
-    {
-        typedef A T;
-        typedef std::vector<T, min_allocator<T>> C;
-        C c = {A{1, 2}};
-        C::iterator i = c.begin();
-        i->first = 3;
-        C::const_iterator j = i;
-        assert(j->first == 3);
-    }
+  }
+  {
+    typedef int T;
+    typedef std::vector<T, min_allocator<T>> C;
+    C c;
+    C::const_iterator i = c.cbegin();
+    C::const_iterator j = c.cend();
+    assert(std::distance(i, j) == 0);
+    assert(i == j);
+    assert(i == c.end());
+  }
+  {
+    typedef int T;
+    typedef std::vector<T, min_allocator<T>> C;
+    const T t[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9};
+    C c(std::begin(t), std::end(t));
+    C::iterator i = c.begin();
+    assert(*i == 0);
+    ++i;
+    assert(*i == 1);
+    *i = 10;
+    assert(*i == 10);
+    assert(std::distance(c.begin(), c.end()) == 10);
+  }
+  {
+    typedef int T;
+    typedef std::vector<T, min_allocator<T>> C;
+    C::iterator i;
+    C::const_iterator j;
+    (void)i;
+    (void)j;
+  }
+  {
+    typedef A T;
+    typedef std::vector<T, min_allocator<T>> C;
+    C c                 = {A{1, 2}};
+    C::iterator i       = c.begin();
+    i->first            = 3;
+    C::const_iterator j = i;
+    assert(j->first == 3);
+  }
 #endif
 #if TEST_STD_VER > 11
-    { // N3644 testing
-        typedef std::vector<int> C;
-        C::iterator ii1{}, ii2{};
-        C::iterator ii4 = ii1;
-        C::const_iterator cii{};
-        assert ( ii1 == ii2 );
-        assert ( ii1 == ii4 );
-
-        assert (!(ii1 != ii2 ));
-
-        assert ( (ii1 == cii ));
-        assert ( (cii == ii1 ));
-        assert (!(ii1 != cii ));
-        assert (!(cii != ii1 ));
-        assert (!(ii1 <  cii ));
-        assert (!(cii <  ii1 ));
-        assert ( (ii1 <= cii ));
-        assert ( (cii <= ii1 ));
-        assert (!(ii1 >  cii ));
-        assert (!(cii >  ii1 ));
-        assert ( (ii1 >= cii ));
-        assert ( (cii >= ii1 ));
-        assert (cii - ii1 == 0);
-        assert (ii1 - cii == 0);
+  { // N3644 testing
+    typedef std::vector<int> C;
+    C::iterator ii1{}, ii2{};
+    C::iterator ii4 = ii1;
+    C::const_iterator cii{};
+    assert(ii1 == ii2);
+    assert(ii1 == ii4);
+
+    assert(!(ii1 != ii2));
+
+    assert((ii1 == cii));
+    assert((cii == ii1));
+    assert(!(ii1 != cii));
+    assert(!(cii != ii1));
+    assert(!(ii1 < cii));
+    assert(!(cii < ii1));
+    assert((ii1 <= cii));
+    assert((cii <= ii1));
+    assert(!(ii1 > cii));
+    assert(!(cii > ii1));
+    assert((ii1 >= cii));
+    assert((cii >= ii1));
+    assert(cii - ii1 == 0);
+    assert(ii1 - cii == 0);
 #  if TEST_STD_VER >= 20
-        // P1614 + LWG3352
-        std::same_as<std::strong_ordering> decltype(auto) r1 = ii1 <=> ii2;
-        assert(r1 == std::strong_ordering::equal);
+    // P1614 + LWG3352
+    std::same_as<std::strong_ordering> decltype(auto) r1 = ii1 <=> ii2;
+    assert(r1 == std::strong_ordering::equal);
 
-        std::same_as<std::strong_ordering> decltype(auto) r2 = cii <=> ii2;
-        assert(r2 == std::strong_ordering::equal);
+    std::same_as<std::strong_ordering> decltype(auto) r2 = cii <=> ii2;
+    assert(r2 == std::strong_ordering::equal);
 #  endif // TEST_STD_VER > 20
-    }
+  }
 #endif // TEST_STD_VER > 11
 
-    return true;
+  return true;
 }
 
-int main(int, char**)
-{
-    tests();
+int main(int, char**) {
+  tests();
 #if TEST_STD_VER > 17
-    static_assert(tests());
+  static_assert(tests());
 #endif
-    return 0;
+  return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/vector/range_concept_conformance.compile.pass.cpp b/libcxx/test/std/containers/sequences/vector/range_concept_conformance.compile.pass.cpp
index 36c68f73494a9..47db59240a4ed 100644
--- a/libcxx/test/std/containers/sequences/vector/range_concept_conformance.compile.pass.cpp
+++ b/libcxx/test/std/containers/sequences/vector/range_concept_conformance.compile.pass.cpp
@@ -17,7 +17,6 @@
 
 using range = std::vector<int>;
 
-
 static_assert(std::same_as<std::ranges::iterator_t<range>, range::iterator>);
 static_assert(std::ranges::common_range<range>);
 static_assert(std::ranges::random_access_range<range>);
diff --git a/libcxx/test/std/containers/sequences/vector/reverse_iterators.pass.cpp b/libcxx/test/std/containers/sequences/vector/reverse_iterators.pass.cpp
index 1591f912f3372..5baeac5bf2549 100644
--- a/libcxx/test/std/containers/sequences/vector/reverse_iterators.pass.cpp
+++ b/libcxx/test/std/containers/sequences/vector/reverse_iterators.pass.cpp
@@ -23,64 +23,64 @@
 
 template <class Vector>
 TEST_CONSTEXPR_CXX20 void check_vector_reverse_iterators() {
+  {
+    Vector vec;
+    assert(vec.rbegin() == vec.rend());
+    assert(vec.crbegin() == vec.crend());
+  }
+  {
+    const int n = 10;
+    Vector vec;
+    const Vector& cvec = vec;
+    vec.reserve(n);
+    for (int i = 0; i < n; ++i)
+      vec.push_back(i);
     {
-        Vector vec;
-        assert(vec.rbegin() == vec.rend());
-        assert(vec.crbegin() == vec.crend());
+      int iterations = 0;
+
+      for (typename Vector::const_reverse_iterator it = vec.crbegin(); it != vec.crend(); ++it) {
+        assert(*it == (n - iterations - 1));
+        ++iterations;
+      }
+      assert(iterations == n);
     }
     {
-        const int n = 10;
-        Vector vec;
-        const Vector& cvec = vec;
-        vec.reserve(n);
-        for (int i = 0; i < n; ++i)
-            vec.push_back(i);
-        {
-            int iterations = 0;
-
-            for (typename Vector::const_reverse_iterator it = vec.crbegin(); it != vec.crend(); ++it) {
-                assert(*it == (n - iterations - 1));
-                ++iterations;
-            }
-            assert(iterations == n);
-        }
-        {
-            assert(cvec.rbegin() == vec.crbegin());
-            assert(cvec.rend() == vec.crend());
-        }
-        {
-            int iterations = 0;
-
-            for (typename Vector::reverse_iterator it = vec.rbegin(); it != vec.rend(); ++it) {
-                assert(*it == (n - iterations - 1));
-                *it = 40;
-                assert(*it == 40);
-                ++iterations;
-            }
-            assert(iterations == n);
-        }
+      assert(cvec.rbegin() == vec.crbegin());
+      assert(cvec.rend() == vec.crend());
+    }
+    {
+      int iterations = 0;
 
-        assert(std::distance(vec.rbegin(), vec.rend()) == n);
-        assert(std::distance(cvec.rbegin(), cvec.rend()) == n);
-        assert(std::distance(vec.crbegin(), vec.crend()) == n);
-        assert(std::distance(cvec.crbegin(), cvec.crend()) == n);
+      for (typename Vector::reverse_iterator it = vec.rbegin(); it != vec.rend(); ++it) {
+        assert(*it == (n - iterations - 1));
+        *it = 40;
+        assert(*it == 40);
+        ++iterations;
+      }
+      assert(iterations == n);
     }
+
+    assert(std::distance(vec.rbegin(), vec.rend()) == n);
+    assert(std::distance(cvec.rbegin(), cvec.rend()) == n);
+    assert(std::distance(vec.crbegin(), vec.crend()) == n);
+    assert(std::distance(cvec.crbegin(), cvec.crend()) == n);
+  }
 }
 
 TEST_CONSTEXPR_CXX20 bool test() {
-    check_vector_reverse_iterators<std::vector<int> >();
+  check_vector_reverse_iterators<std::vector<int> >();
 #if TEST_STD_VER >= 11
-    check_vector_reverse_iterators<std::vector<int, min_allocator<int> > >();
+  check_vector_reverse_iterators<std::vector<int, min_allocator<int> > >();
 #endif
 
-    return true;
+  return true;
 }
 
 int main(int, char**) {
-    test();
+  test();
 #if TEST_STD_VER > 17
-    static_assert(test());
+  static_assert(test());
 #endif
 
-    return 0;
+  return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/vector/trivial_relocation.pass.cpp b/libcxx/test/std/containers/sequences/vector/trivial_relocation.pass.cpp
new file mode 100644
index 0000000000000..fbd597d07d6e3
--- /dev/null
+++ b/libcxx/test/std/containers/sequences/vector/trivial_relocation.pass.cpp
@@ -0,0 +1,71 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// <vector>
+
+// Make sure we don't miscompile vector operations for types that shouldn't be considered
+// trivially relocatable.
+
+#include <vector>
+#include <cassert>
+#include <cstddef>
+#include <type_traits>
+#include <utility>
+
+#include "test_macros.h"
+
+struct Tracker {
+  std::size_t move_constructs = 0;
+};
+
+struct [[clang::trivial_abi]] Inner {
+  TEST_CONSTEXPR_CXX20 explicit Inner(Tracker* tracker) : tracker_(tracker) {}
+  TEST_CONSTEXPR_CXX20 Inner(const Inner& rhs) : tracker_(rhs.tracker_) { tracker_->move_constructs += 1; }
+  TEST_CONSTEXPR_CXX20 Inner(Inner&& rhs) : tracker_(rhs.tracker_) { tracker_->move_constructs += 1; }
+  Tracker* tracker_;
+};
+
+// Even though this type contains a trivial_abi type, it is not trivially move-constructible,
+// so we should not attempt to optimize its move construction + destroy using trivial relocation.
+struct NotTriviallyMovable {
+  TEST_CONSTEXPR_CXX20 explicit NotTriviallyMovable(Tracker* tracker) : inner_(tracker) {}
+  TEST_CONSTEXPR_CXX20 NotTriviallyMovable(NotTriviallyMovable&& other) : inner_(std::move(other.inner_)) {}
+  Inner inner_;
+};
+static_assert(!std::is_trivially_copyable<NotTriviallyMovable>::value, "");
+LIBCPP_STATIC_ASSERT(!std::__libcpp_is_trivially_relocatable<NotTriviallyMovable>::value, "");
+
+TEST_CONSTEXPR_CXX20 bool tests() {
+  Tracker track;
+  std::vector<NotTriviallyMovable> v;
+
+  // Fill the vector at its capacity, such that any subsequent push_back would require growing.
+  v.reserve(5);
+  std::size_t const capacity = v.capacity(); // could technically be more than 5
+  while (v.size() < v.capacity()) {
+    v.emplace_back(&track);
+  }
+  assert(track.move_constructs == 0);
+  assert(v.capacity() == capacity);
+  assert(v.size() == capacity);
+
+  // Force a reallocation of the buffer + relocalization of the elements.
+  // All the existing elements of the vector should be move-constructed to their new location.
+  v.emplace_back(&track);
+  assert(track.move_constructs == capacity);
+
+  return true;
+}
+
+int main(int, char**) {
+  tests();
+#if TEST_STD_VER >= 20
+  static_assert(tests());
+#endif
+  return 0;
+}
diff --git a/libcxx/test/std/containers/sequences/vector/types.pass.cpp b/libcxx/test/std/containers/sequences/vector/types.pass.cpp
index f4d7fa088842f..c3aa60bd30c7e 100644
--- a/libcxx/test/std/containers/sequences/vector/types.pass.cpp
+++ b/libcxx/test/std/containers/sequences/vector/types.pass.cpp
@@ -37,66 +37,60 @@
 #include "../../Copyable.h"
 #include "min_allocator.h"
 
-struct A { std::vector<A> v; }; // incomplete type support
+struct A {
+  std::vector<A> v;
+}; // incomplete type support
 
 template <class T, class Allocator>
-void
-test()
-{
-    typedef std::vector<T, Allocator> C;
+void test() {
+  typedef std::vector<T, Allocator> C;
 
-//  TODO: These tests should use allocator_traits to get stuff, rather than
-//  blindly pulling typedefs out of the allocator. This is why we can't call
-//  test<int, min_allocator<int>>() below.
-    static_assert((std::is_same<typename C::value_type, T>::value), "");
-    static_assert(
-        (std::is_same<typename C::value_type, typename std::allocator_traits<Allocator>::value_type>::value), "");
-    static_assert((std::is_same<typename C::allocator_type, Allocator>::value), "");
-    static_assert(
-        (std::is_same<typename C::size_type, typename std::allocator_traits<Allocator>::size_type>::value), "");
-    static_assert(
-        (std::is_same<typename C::difference_type, typename std::allocator_traits<Allocator>::difference_type>::value),
-        "");
-    static_assert(
-        (std::is_same<typename C::reference, typename std::allocator_traits<Allocator>::value_type&>::value), "");
-    static_assert((std::is_same<typename C::const_reference,
-                                const typename std::allocator_traits<Allocator>::value_type&>::value),
-                  "");
-    static_assert((std::is_same<typename C::pointer, typename std::allocator_traits<Allocator>::pointer>::value), "");
-    static_assert(
-        (std::is_same<typename C::const_pointer, typename std::allocator_traits<Allocator>::const_pointer>::value), "");
+  //  TODO: These tests should use allocator_traits to get stuff, rather than
+  //  blindly pulling typedefs out of the allocator. This is why we can't call
+  //  test<int, min_allocator<int>>() below.
+  static_assert((std::is_same<typename C::value_type, T>::value), "");
+  static_assert(
+      (std::is_same<typename C::value_type, typename std::allocator_traits<Allocator>::value_type>::value), "");
+  static_assert((std::is_same<typename C::allocator_type, Allocator>::value), "");
+  static_assert((std::is_same<typename C::size_type, typename std::allocator_traits<Allocator>::size_type>::value), "");
+  static_assert(
+      (std::is_same<typename C::difference_type, typename std::allocator_traits<Allocator>::difference_type>::value),
+      "");
+  static_assert(
+      (std::is_same<typename C::reference, typename std::allocator_traits<Allocator>::value_type&>::value), "");
+  static_assert(
+      (std::is_same<typename C::const_reference, const typename std::allocator_traits<Allocator>::value_type&>::value),
+      "");
+  static_assert((std::is_same<typename C::pointer, typename std::allocator_traits<Allocator>::pointer>::value), "");
+  static_assert(
+      (std::is_same<typename C::const_pointer, typename std::allocator_traits<Allocator>::const_pointer>::value), "");
 
-    static_assert((std::is_signed<typename C::difference_type>::value), "");
-    static_assert((std::is_unsigned<typename C::size_type>::value), "");
-//     static_assert((std::is_same<typename C::difference_type,
-//         typename std::iterator_traits<typename C::iterator>::difference_type>::value), "");
-//     static_assert((std::is_same<typename C::difference_type,
-//         typename std::iterator_traits<typename C::const_iterator>::difference_type>::value), "");
+  static_assert((std::is_signed<typename C::difference_type>::value), "");
+  static_assert((std::is_unsigned<typename C::size_type>::value), "");
+  //     static_assert((std::is_same<typename C::difference_type,
+  //         typename std::iterator_traits<typename C::iterator>::difference_type>::value), "");
+  //     static_assert((std::is_same<typename C::difference_type,
+  //         typename std::iterator_traits<typename C::const_iterator>::difference_type>::value), "");
 
-    static_assert((std::is_same<
-        typename std::iterator_traits<typename C::iterator>::iterator_category,
-        std::random_access_iterator_tag>::value), "");
-    static_assert((std::is_same<
-        typename std::iterator_traits<typename C::const_iterator>::iterator_category,
-        std::random_access_iterator_tag>::value), "");
-    static_assert((std::is_same<
-        typename C::reverse_iterator,
-        std::reverse_iterator<typename C::iterator> >::value), "");
-    static_assert((std::is_same<
-        typename C::const_reverse_iterator,
-        std::reverse_iterator<typename C::const_iterator> >::value), "");
+  static_assert((std::is_same< typename std::iterator_traits<typename C::iterator>::iterator_category,
+                               std::random_access_iterator_tag>::value),
+                "");
+  static_assert((std::is_same< typename std::iterator_traits<typename C::const_iterator>::iterator_category,
+                               std::random_access_iterator_tag>::value),
+                "");
+  static_assert((std::is_same< typename C::reverse_iterator, std::reverse_iterator<typename C::iterator> >::value), "");
+  static_assert(
+      (std::is_same< typename C::const_reverse_iterator, std::reverse_iterator<typename C::const_iterator> >::value),
+      "");
 }
 
-int main(int, char**)
-{
-    test<int, test_allocator<int> >();
-    test<int*, std::allocator<int*> >();
-    test<Copyable, test_allocator<Copyable> >();
-    static_assert((std::is_same<std::vector<char>::allocator_type,
-                                std::allocator<char> >::value), "");
+int main(int, char**) {
+  test<int, test_allocator<int> >();
+  test<int*, std::allocator<int*> >();
+  test<Copyable, test_allocator<Copyable> >();
+  static_assert((std::is_same<std::vector<char>::allocator_type, std::allocator<char> >::value), "");
 #if TEST_STD_VER >= 11
-    {
-
+  {
     typedef std::vector<int, min_allocator<int> > C;
     static_assert((std::is_same<C::value_type, int>::value), "");
     static_assert((std::is_same<C::allocator_type, min_allocator<int> >::value), "");
@@ -107,11 +101,11 @@ int main(int, char**)
 
     static_assert((std::is_signed<typename C::difference_type>::value), "");
     static_assert((std::is_unsigned<typename C::size_type>::value), "");
-//     static_assert((std::is_same<typename C::difference_type,
-//         typename std::iterator_traits<typename C::iterator>::difference_type>::value), "");
-//     static_assert((std::is_same<typename C::difference_type,
-//         typename std::iterator_traits<typename C::const_iterator>::difference_type>::value), "");
-    }
+    //     static_assert((std::is_same<typename C::difference_type,
+    //         typename std::iterator_traits<typename C::iterator>::difference_type>::value), "");
+    //     static_assert((std::is_same<typename C::difference_type,
+    //         typename std::iterator_traits<typename C::const_iterator>::difference_type>::value), "");
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/sequences/vector/vector.capacity/capacity.pass.cpp b/libcxx/test/std/containers/sequences/vector/vector.capacity/capacity.pass.cpp
index b2eb33c6c5f53..b28d31d9f9bb6 100644
--- a/libcxx/test/std/containers/sequences/vector/vector.capacity/capacity.pass.cpp
+++ b/libcxx/test/std/containers/sequences/vector/vector.capacity/capacity.pass.cpp
@@ -17,43 +17,41 @@
 #include "min_allocator.h"
 #include "asan_testing.h"
 
-TEST_CONSTEXPR_CXX20 bool tests()
-{
-    {
-        std::vector<int> v;
-        assert(v.capacity() == 0);
-        assert(is_contiguous_container_asan_correct(v));
-    }
-    {
-        std::vector<int> v(100);
-        assert(v.capacity() == 100);
-        v.push_back(0);
-        assert(v.capacity() > 101);
-        assert(is_contiguous_container_asan_correct(v));
-    }
+TEST_CONSTEXPR_CXX20 bool tests() {
+  {
+    std::vector<int> v;
+    assert(v.capacity() == 0);
+    assert(is_contiguous_container_asan_correct(v));
+  }
+  {
+    std::vector<int> v(100);
+    assert(v.capacity() == 100);
+    v.push_back(0);
+    assert(v.capacity() > 101);
+    assert(is_contiguous_container_asan_correct(v));
+  }
 #if TEST_STD_VER >= 11
-    {
-        std::vector<int, min_allocator<int>> v;
-        assert(v.capacity() == 0);
-        assert(is_contiguous_container_asan_correct(v));
-    }
-    {
-        std::vector<int, min_allocator<int>> v(100);
-        assert(v.capacity() == 100);
-        v.push_back(0);
-        assert(v.capacity() > 101);
-        assert(is_contiguous_container_asan_correct(v));
-    }
+  {
+    std::vector<int, min_allocator<int>> v;
+    assert(v.capacity() == 0);
+    assert(is_contiguous_container_asan_correct(v));
+  }
+  {
+    std::vector<int, min_allocator<int>> v(100);
+    assert(v.capacity() == 100);
+    v.push_back(0);
+    assert(v.capacity() > 101);
+    assert(is_contiguous_container_asan_correct(v));
+  }
 #endif
 
-    return true;
+  return true;
 }
 
-int main(int, char**)
-{
-    tests();
+int main(int, char**) {
+  tests();
 #if TEST_STD_VER > 17
-    static_assert(tests());
+  static_assert(tests());
 #endif
-    return 0;
+  return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/vector/vector.capacity/empty.pass.cpp b/libcxx/test/std/containers/sequences/vector/vector.capacity/empty.pass.cpp
index ad4d7b310af76..b691a74a04208 100644
--- a/libcxx/test/std/containers/sequences/vector/vector.capacity/empty.pass.cpp
+++ b/libcxx/test/std/containers/sequences/vector/vector.capacity/empty.pass.cpp
@@ -19,7 +19,7 @@
 #include "min_allocator.h"
 
 TEST_CONSTEXPR_CXX20 bool tests() {
-    {
+  {
     typedef std::vector<int> C;
     C c;
     ASSERT_NOEXCEPT(c.empty());
@@ -28,9 +28,9 @@ TEST_CONSTEXPR_CXX20 bool tests() {
     assert(!c.empty());
     c.clear();
     assert(c.empty());
-    }
+  }
 #if TEST_STD_VER >= 11
-    {
+  {
     typedef std::vector<int, min_allocator<int>> C;
     C c;
     ASSERT_NOEXCEPT(c.empty());
@@ -39,27 +39,26 @@ TEST_CONSTEXPR_CXX20 bool tests() {
     assert(!c.empty());
     c.clear();
     assert(c.empty());
-    }
-    {
-      typedef std::vector<int, safe_allocator<int>> C;
-      C c;
-      ASSERT_NOEXCEPT(c.empty());
-      assert(c.empty());
-      c.push_back(C::value_type(1));
-      assert(!c.empty());
-      c.clear();
-      assert(c.empty());
-    }
+  }
+  {
+    typedef std::vector<int, safe_allocator<int>> C;
+    C c;
+    ASSERT_NOEXCEPT(c.empty());
+    assert(c.empty());
+    c.push_back(C::value_type(1));
+    assert(!c.empty());
+    c.clear();
+    assert(c.empty());
+  }
 #endif
 
-    return true;
+  return true;
 }
 
-int main(int, char**)
-{
-    tests();
+int main(int, char**) {
+  tests();
 #if TEST_STD_VER > 17
-    static_assert(tests());
+  static_assert(tests());
 #endif
-    return 0;
+  return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/vector/vector.capacity/empty.verify.cpp b/libcxx/test/std/containers/sequences/vector/vector.capacity/empty.verify.cpp
index 4cea844f1e357..0de3913b0c5c2 100644
--- a/libcxx/test/std/containers/sequences/vector/vector.capacity/empty.verify.cpp
+++ b/libcxx/test/std/containers/sequences/vector/vector.capacity/empty.verify.cpp
@@ -17,6 +17,6 @@
 #include <vector>
 
 void f() {
-    std::vector<int> c;
-    c.empty(); // expected-warning {{ignoring return value of function declared with 'nodiscard' attribute}}
+  std::vector<int> c;
+  c.empty(); // expected-warning {{ignoring return value of function declared with 'nodiscard' attribute}}
 }
diff --git a/libcxx/test/std/containers/sequences/vector/vector.capacity/max_size.pass.cpp b/libcxx/test/std/containers/sequences/vector/vector.capacity/max_size.pass.cpp
index 33abaa04b1207..a3f44f33028e4 100644
--- a/libcxx/test/std/containers/sequences/vector/vector.capacity/max_size.pass.cpp
+++ b/libcxx/test/std/containers/sequences/vector/vector.capacity/max_size.pass.cpp
@@ -10,16 +10,36 @@
 
 // size_type max_size() const;
 
+#include <algorithm>
 #include <cassert>
+#include <cstdint>
 #include <limits>
 #include <type_traits>
 #include <vector>
 
+#include "min_allocator.h"
+#include "sized_allocator.h"
 #include "test_allocator.h"
 #include "test_macros.h"
 
+#if TEST_STD_VER >= 11
 
-TEST_CONSTEXPR_CXX20 bool test() {
+template <typename T, typename Alloc>
+TEST_CONSTEXPR_CXX20 void test(const std::vector<T, Alloc>& v) {
+  using Vector              = std::vector<T, Alloc>;
+  using alloc_traits        = std::allocator_traits<typename Vector::allocator_type>;
+  using size_type           = typename Vector::size_type;
+  using difference_type     = typename Vector::difference_type;
+  const size_type max_dist  = static_cast<size_type>(std::numeric_limits<difference_type>::max());
+  const size_type max_alloc = alloc_traits::max_size(v.get_allocator());
+  assert(v.max_size() <= max_dist);
+  assert(v.max_size() <= max_alloc);
+  LIBCPP_ASSERT(v.max_size() == std::min<size_type>(max_dist, max_alloc));
+}
+
+#endif // TEST_STD_VER >= 11
+
+TEST_CONSTEXPR_CXX20 bool tests() {
   {
     typedef limited_allocator<int, 10> A;
     typedef std::vector<int, A> C;
@@ -30,29 +50,48 @@ TEST_CONSTEXPR_CXX20 bool test() {
   {
     typedef limited_allocator<int, (std::size_t)-1> A;
     typedef std::vector<int, A> C;
-    const C::size_type max_dist =
-        static_cast<C::size_type>(std::numeric_limits<C::difference_type>::max());
+    const C::size_type max_dist = static_cast<C::size_type>(std::numeric_limits<C::difference_type>::max());
     C c;
     assert(c.max_size() <= max_dist);
     LIBCPP_ASSERT(c.max_size() == max_dist);
   }
   {
     typedef std::vector<char> C;
-    const C::size_type max_dist =
-        static_cast<C::size_type>(std::numeric_limits<C::difference_type>::max());
+    const C::size_type max_dist = static_cast<C::size_type>(std::numeric_limits<C::difference_type>::max());
     C c;
     assert(c.max_size() <= max_dist);
     assert(c.max_size() <= alloc_max_size(c.get_allocator()));
+    LIBCPP_ASSERT(c.max_size() == std::min(max_dist, alloc_max_size(c.get_allocator())));
+  }
+
+#if TEST_STD_VER >= 11
+
+  // Test with various allocators and diffrent size_type
+  {
+    test(std::vector<int>());
+    test(std::vector<short, std::allocator<short> >());
+    test(std::vector<unsigned, min_allocator<unsigned> >());
+    test(std::vector<char, test_allocator<char> >(test_allocator<char>(1)));
+    test(std::vector<std::size_t, other_allocator<std::size_t> >(other_allocator<std::size_t>(5)));
+    test(std::vector<int, sized_allocator<int, std::uint8_t, std::int8_t> >());
+    test(std::vector<int, sized_allocator<int, std::uint16_t, std::int16_t> >());
+    test(std::vector<int, sized_allocator<int, std::uint32_t, std::int32_t> >());
+    test(std::vector<int, sized_allocator<int, std::uint64_t, std::int64_t> >());
+    test(std::vector<int, limited_allocator<int, static_cast<std::size_t>(-1)> >());
+    test(std::vector<int, limited_allocator<int, static_cast<std::size_t>(-1) / 2> >());
+    test(std::vector<int, limited_allocator<int, static_cast<std::size_t>(-1) / 4> >());
   }
 
+#endif // TEST_STD_VER >= 11
+
   return true;
 }
 
 int main(int, char**) {
-  test();
+  tests();
 
 #if TEST_STD_VER > 17
-  static_assert(test());
+  static_assert(tests());
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/sequences/vector/vector.capacity/resize_size.pass.cpp b/libcxx/test/std/containers/sequences/vector/vector.capacity/resize_size.pass.cpp
index 967daf99f27c7..ddd15a700cab2 100644
--- a/libcxx/test/std/containers/sequences/vector/vector.capacity/resize_size.pass.cpp
+++ b/libcxx/test/std/containers/sequences/vector/vector.capacity/resize_size.pass.cpp
@@ -20,86 +20,85 @@
 #include "asan_testing.h"
 
 TEST_CONSTEXPR_CXX20 bool tests() {
-    {
-        std::vector<int> v(100);
-        v.resize(50);
-        assert(v.size() == 50);
-        assert(v.capacity() == 100);
-        assert(is_contiguous_container_asan_correct(v));
-        v.resize(200);
-        assert(v.size() == 200);
-        assert(v.capacity() >= 200);
-        assert(is_contiguous_container_asan_correct(v));
-    }
-    {
-        // Add 1 for implementations that dynamically allocate a container proxy.
-        std::vector<int, limited_allocator<int, 300 + 1> > v(100);
-        v.resize(50);
-        assert(v.size() == 50);
-        assert(v.capacity() == 100);
-        assert(is_contiguous_container_asan_correct(v));
-        v.resize(200);
-        assert(v.size() == 200);
-        assert(v.capacity() >= 200);
-        assert(is_contiguous_container_asan_correct(v));
-    }
+  {
+    std::vector<int> v(100);
+    v.resize(50);
+    assert(v.size() == 50);
+    assert(v.capacity() == 100);
+    assert(is_contiguous_container_asan_correct(v));
+    v.resize(200);
+    assert(v.size() == 200);
+    assert(v.capacity() >= 200);
+    assert(is_contiguous_container_asan_correct(v));
+  }
+  {
+    // Add 1 for implementations that dynamically allocate a container proxy.
+    std::vector<int, limited_allocator<int, 300 + 1> > v(100);
+    v.resize(50);
+    assert(v.size() == 50);
+    assert(v.capacity() == 100);
+    assert(is_contiguous_container_asan_correct(v));
+    v.resize(200);
+    assert(v.size() == 200);
+    assert(v.capacity() >= 200);
+    assert(is_contiguous_container_asan_correct(v));
+  }
 #if TEST_STD_VER >= 11
-    {
-        std::vector<MoveOnly> v(100);
-        v.resize(50);
-        assert(v.size() == 50);
-        assert(v.capacity() == 100);
-        assert(is_contiguous_container_asan_correct(v));
-        v.resize(200);
-        assert(v.size() == 200);
-        assert(v.capacity() >= 200);
-        assert(is_contiguous_container_asan_correct(v));
-    }
-    {
-        // Add 1 for implementations that dynamically allocate a container proxy.
-        std::vector<MoveOnly, limited_allocator<MoveOnly, 300 + 1> > v(100);
-        v.resize(50);
-        assert(v.size() == 50);
-        assert(v.capacity() == 100);
-        assert(is_contiguous_container_asan_correct(v));
-        v.resize(200);
-        assert(v.size() == 200);
-        assert(v.capacity() >= 200);
-        assert(is_contiguous_container_asan_correct(v));
-    }
-    {
-        std::vector<MoveOnly, min_allocator<MoveOnly>> v(100);
-        v.resize(50);
-        assert(v.size() == 50);
-        assert(v.capacity() == 100);
-        assert(is_contiguous_container_asan_correct(v));
-        v.resize(200);
-        assert(v.size() == 200);
-        assert(v.capacity() >= 200);
-        assert(is_contiguous_container_asan_correct(v));
-    }
-    {
-      std::vector<int, safe_allocator<int>> v(100);
-      v.resize(50);
-      assert(v.size() == 50);
-      assert(v.capacity() == 100);
-      assert(is_contiguous_container_asan_correct(v));
-      v.resize(200);
-      assert(v.size() == 200);
-      assert(v.capacity() >= 200);
-      assert(is_contiguous_container_asan_correct(v));
-    }
+  {
+    std::vector<MoveOnly> v(100);
+    v.resize(50);
+    assert(v.size() == 50);
+    assert(v.capacity() == 100);
+    assert(is_contiguous_container_asan_correct(v));
+    v.resize(200);
+    assert(v.size() == 200);
+    assert(v.capacity() >= 200);
+    assert(is_contiguous_container_asan_correct(v));
+  }
+  {
+    // Add 1 for implementations that dynamically allocate a container proxy.
+    std::vector<MoveOnly, limited_allocator<MoveOnly, 300 + 1> > v(100);
+    v.resize(50);
+    assert(v.size() == 50);
+    assert(v.capacity() == 100);
+    assert(is_contiguous_container_asan_correct(v));
+    v.resize(200);
+    assert(v.size() == 200);
+    assert(v.capacity() >= 200);
+    assert(is_contiguous_container_asan_correct(v));
+  }
+  {
+    std::vector<MoveOnly, min_allocator<MoveOnly>> v(100);
+    v.resize(50);
+    assert(v.size() == 50);
+    assert(v.capacity() == 100);
+    assert(is_contiguous_container_asan_correct(v));
+    v.resize(200);
+    assert(v.size() == 200);
+    assert(v.capacity() >= 200);
+    assert(is_contiguous_container_asan_correct(v));
+  }
+  {
+    std::vector<int, safe_allocator<int>> v(100);
+    v.resize(50);
+    assert(v.size() == 50);
+    assert(v.capacity() == 100);
+    assert(is_contiguous_container_asan_correct(v));
+    v.resize(200);
+    assert(v.size() == 200);
+    assert(v.capacity() >= 200);
+    assert(is_contiguous_container_asan_correct(v));
+  }
 #endif
 
-    return true;
+  return true;
 }
 
-int main(int, char**)
-{
-    tests();
+int main(int, char**) {
+  tests();
 
 #if TEST_STD_VER > 17
-    static_assert(tests());
+  static_assert(tests());
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/sequences/vector/vector.capacity/resize_size_value.pass.cpp b/libcxx/test/std/containers/sequences/vector/vector.capacity/resize_size_value.pass.cpp
index a880bcf1f9499..f4005eadd369e 100644
--- a/libcxx/test/std/containers/sequences/vector/vector.capacity/resize_size_value.pass.cpp
+++ b/libcxx/test/std/containers/sequences/vector/vector.capacity/resize_size_value.pass.cpp
@@ -18,97 +18,96 @@
 #include "asan_testing.h"
 
 TEST_CONSTEXPR_CXX20 bool tests() {
-    {
-        std::vector<int> v(100);
-        v.resize(50, 1);
-        assert(v.size() == 50);
-        assert(v.capacity() == 100);
-        assert(v == std::vector<int>(50));
-        v.resize(200, 1);
-        assert(v.size() == 200);
-        assert(v.capacity() >= 200);
-        assert(is_contiguous_container_asan_correct(v));
-        for (unsigned i = 0; i < 50; ++i)
-            assert(v[i] == 0);
-        for (unsigned i = 50; i < 200; ++i)
-            assert(v[i] == 1);
-    }
-    {
-        // Add 1 for implementations that dynamically allocate a container proxy.
-        std::vector<int, limited_allocator<int, 300 + 1> > v(100);
-        v.resize(50, 1);
-        assert(v.size() == 50);
-        assert(v.capacity() == 100);
-        v.resize(200, 1);
-        assert(v.size() == 200);
-        assert(v.capacity() >= 200);
-        assert(is_contiguous_container_asan_correct(v));
-    }
+  {
+    std::vector<int> v(100);
+    v.resize(50, 1);
+    assert(v.size() == 50);
+    assert(v.capacity() == 100);
+    assert(v == std::vector<int>(50));
+    v.resize(200, 1);
+    assert(v.size() == 200);
+    assert(v.capacity() >= 200);
+    assert(is_contiguous_container_asan_correct(v));
+    for (unsigned i = 0; i < 50; ++i)
+      assert(v[i] == 0);
+    for (unsigned i = 50; i < 200; ++i)
+      assert(v[i] == 1);
+  }
+  {
+    // Add 1 for implementations that dynamically allocate a container proxy.
+    std::vector<int, limited_allocator<int, 300 + 1> > v(100);
+    v.resize(50, 1);
+    assert(v.size() == 50);
+    assert(v.capacity() == 100);
+    v.resize(200, 1);
+    assert(v.size() == 200);
+    assert(v.capacity() >= 200);
+    assert(is_contiguous_container_asan_correct(v));
+  }
 #if TEST_STD_VER >= 11
-    {
-        std::vector<int, min_allocator<int>> v(100);
-        v.resize(50, 1);
-        assert(v.size() == 50);
-        assert(v.capacity() == 100);
-        assert(is_contiguous_container_asan_correct(v));
-        assert((v == std::vector<int, min_allocator<int>>(50)));
-        v.resize(200, 1);
-        assert(v.size() == 200);
-        assert(v.capacity() >= 200);
-        assert(is_contiguous_container_asan_correct(v));
-        for (unsigned i = 0; i < 50; ++i)
-            assert(v[i] == 0);
-        for (unsigned i = 50; i < 200; ++i)
-            assert(v[i] == 1);
-    }
-    {
-        std::vector<int, min_allocator<int>> v(100);
-        v.resize(50, 1);
-        assert(v.size() == 50);
-        assert(v.capacity() == 100);
-        assert(is_contiguous_container_asan_correct(v));
-        v.resize(200, 1);
-        assert(v.size() == 200);
-        assert(v.capacity() >= 200);
-        assert(is_contiguous_container_asan_correct(v));
-    }
-    {
-      std::vector<int, safe_allocator<int>> v(100);
-      v.resize(50, 1);
-      assert(v.size() == 50);
-      assert(v.capacity() == 100);
-      assert(is_contiguous_container_asan_correct(v));
-      assert((v == std::vector<int, safe_allocator<int>>(50)));
-      v.resize(200, 1);
-      assert(v.size() == 200);
-      assert(v.capacity() >= 200);
-      assert(is_contiguous_container_asan_correct(v));
-      for (unsigned i = 0; i < 50; ++i)
-        assert(v[i] == 0);
-      for (unsigned i = 50; i < 200; ++i)
-        assert(v[i] == 1);
-    }
-    {
-      std::vector<int, safe_allocator<int>> v(100);
-      v.resize(50, 1);
-      assert(v.size() == 50);
-      assert(v.capacity() == 100);
-      assert(is_contiguous_container_asan_correct(v));
-      v.resize(200, 1);
-      assert(v.size() == 200);
-      assert(v.capacity() >= 200);
-      assert(is_contiguous_container_asan_correct(v));
-    }
+  {
+    std::vector<int, min_allocator<int>> v(100);
+    v.resize(50, 1);
+    assert(v.size() == 50);
+    assert(v.capacity() == 100);
+    assert(is_contiguous_container_asan_correct(v));
+    assert((v == std::vector<int, min_allocator<int>>(50)));
+    v.resize(200, 1);
+    assert(v.size() == 200);
+    assert(v.capacity() >= 200);
+    assert(is_contiguous_container_asan_correct(v));
+    for (unsigned i = 0; i < 50; ++i)
+      assert(v[i] == 0);
+    for (unsigned i = 50; i < 200; ++i)
+      assert(v[i] == 1);
+  }
+  {
+    std::vector<int, min_allocator<int>> v(100);
+    v.resize(50, 1);
+    assert(v.size() == 50);
+    assert(v.capacity() == 100);
+    assert(is_contiguous_container_asan_correct(v));
+    v.resize(200, 1);
+    assert(v.size() == 200);
+    assert(v.capacity() >= 200);
+    assert(is_contiguous_container_asan_correct(v));
+  }
+  {
+    std::vector<int, safe_allocator<int>> v(100);
+    v.resize(50, 1);
+    assert(v.size() == 50);
+    assert(v.capacity() == 100);
+    assert(is_contiguous_container_asan_correct(v));
+    assert((v == std::vector<int, safe_allocator<int>>(50)));
+    v.resize(200, 1);
+    assert(v.size() == 200);
+    assert(v.capacity() >= 200);
+    assert(is_contiguous_container_asan_correct(v));
+    for (unsigned i = 0; i < 50; ++i)
+      assert(v[i] == 0);
+    for (unsigned i = 50; i < 200; ++i)
+      assert(v[i] == 1);
+  }
+  {
+    std::vector<int, safe_allocator<int>> v(100);
+    v.resize(50, 1);
+    assert(v.size() == 50);
+    assert(v.capacity() == 100);
+    assert(is_contiguous_container_asan_correct(v));
+    v.resize(200, 1);
+    assert(v.size() == 200);
+    assert(v.capacity() >= 200);
+    assert(is_contiguous_container_asan_correct(v));
+  }
 #endif
 
-    return true;
+  return true;
 }
 
-int main(int, char**)
-{
-    tests();
+int main(int, char**) {
+  tests();
 #if TEST_STD_VER > 17
-    static_assert(tests());
+  static_assert(tests());
 #endif
-    return 0;
+  return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/vector/vector.capacity/size.pass.cpp b/libcxx/test/std/containers/sequences/vector/vector.capacity/size.pass.cpp
index f38aab1e13845..89f3404a2ca32 100644
--- a/libcxx/test/std/containers/sequences/vector/vector.capacity/size.pass.cpp
+++ b/libcxx/test/std/containers/sequences/vector/vector.capacity/size.pass.cpp
@@ -18,9 +18,8 @@
 #include "test_macros.h"
 #include "min_allocator.h"
 
-TEST_CONSTEXPR_CXX20 bool tests()
-{
-    {
+TEST_CONSTEXPR_CXX20 bool tests() {
+  {
     typedef std::vector<int> C;
     C c;
     ASSERT_NOEXCEPT(c.size());
@@ -37,9 +36,9 @@ TEST_CONSTEXPR_CXX20 bool tests()
     assert(c.size() == 1);
     c.erase(c.begin());
     assert(c.size() == 0);
-    }
+  }
 #if TEST_STD_VER >= 11
-    {
+  {
     typedef std::vector<int, min_allocator<int>> C;
     C c;
     ASSERT_NOEXCEPT(c.size());
@@ -56,35 +55,34 @@ TEST_CONSTEXPR_CXX20 bool tests()
     assert(c.size() == 1);
     c.erase(c.begin());
     assert(c.size() == 0);
-    }
-    {
-      typedef std::vector<int, safe_allocator<int>> C;
-      C c;
-      ASSERT_NOEXCEPT(c.size());
-      assert(c.size() == 0);
-      c.push_back(C::value_type(2));
-      assert(c.size() == 1);
-      c.push_back(C::value_type(1));
-      assert(c.size() == 2);
-      c.push_back(C::value_type(3));
-      assert(c.size() == 3);
-      c.erase(c.begin());
-      assert(c.size() == 2);
-      c.erase(c.begin());
-      assert(c.size() == 1);
-      c.erase(c.begin());
-      assert(c.size() == 0);
-    }
+  }
+  {
+    typedef std::vector<int, safe_allocator<int>> C;
+    C c;
+    ASSERT_NOEXCEPT(c.size());
+    assert(c.size() == 0);
+    c.push_back(C::value_type(2));
+    assert(c.size() == 1);
+    c.push_back(C::value_type(1));
+    assert(c.size() == 2);
+    c.push_back(C::value_type(3));
+    assert(c.size() == 3);
+    c.erase(c.begin());
+    assert(c.size() == 2);
+    c.erase(c.begin());
+    assert(c.size() == 1);
+    c.erase(c.begin());
+    assert(c.size() == 0);
+  }
 #endif
 
-    return true;
+  return true;
 }
 
-int main(int, char**)
-{
-    tests();
+int main(int, char**) {
+  tests();
 #if TEST_STD_VER > 17
-    static_assert(tests());
+  static_assert(tests());
 #endif
   return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/vector/vector.capacity/swap.pass.cpp b/libcxx/test/std/containers/sequences/vector/vector.capacity/swap.pass.cpp
index 1b2ac76eb2b5f..d5e0f7028c3ed 100644
--- a/libcxx/test/std/containers/sequences/vector/vector.capacity/swap.pass.cpp
+++ b/libcxx/test/std/containers/sequences/vector/vector.capacity/swap.pass.cpp
@@ -18,56 +18,55 @@
 #include "asan_testing.h"
 
 TEST_CONSTEXPR_CXX20 bool tests() {
-    {
-        std::vector<int> v1(100);
-        std::vector<int> v2(200);
-        assert(is_contiguous_container_asan_correct(v1));
-        assert(is_contiguous_container_asan_correct(v2));
-        v1.swap(v2);
-        assert(v1.size() == 200);
-        assert(v1.capacity() == 200);
-        assert(is_contiguous_container_asan_correct(v1));
-        assert(v2.size() == 100);
-        assert(v2.capacity() == 100);
-        assert(is_contiguous_container_asan_correct(v2));
-    }
+  {
+    std::vector<int> v1(100);
+    std::vector<int> v2(200);
+    assert(is_contiguous_container_asan_correct(v1));
+    assert(is_contiguous_container_asan_correct(v2));
+    v1.swap(v2);
+    assert(v1.size() == 200);
+    assert(v1.capacity() == 200);
+    assert(is_contiguous_container_asan_correct(v1));
+    assert(v2.size() == 100);
+    assert(v2.capacity() == 100);
+    assert(is_contiguous_container_asan_correct(v2));
+  }
 #if TEST_STD_VER >= 11
-    {
-        std::vector<int, min_allocator<int>> v1(100);
-        std::vector<int, min_allocator<int>> v2(200);
-        assert(is_contiguous_container_asan_correct(v1));
-        assert(is_contiguous_container_asan_correct(v2));
-        v1.swap(v2);
-        assert(v1.size() == 200);
-        assert(v1.capacity() == 200);
-        assert(is_contiguous_container_asan_correct(v1));
-        assert(v2.size() == 100);
-        assert(v2.capacity() == 100);
-        assert(is_contiguous_container_asan_correct(v2));
-    }
-    {
-      std::vector<int, safe_allocator<int>> v1(100);
-      std::vector<int, safe_allocator<int>> v2(200);
-      assert(is_contiguous_container_asan_correct(v1));
-      assert(is_contiguous_container_asan_correct(v2));
-      v1.swap(v2);
-      assert(v1.size() == 200);
-      assert(v1.capacity() == 200);
-      assert(is_contiguous_container_asan_correct(v1));
-      assert(v2.size() == 100);
-      assert(v2.capacity() == 100);
-      assert(is_contiguous_container_asan_correct(v2));
-    }
+  {
+    std::vector<int, min_allocator<int>> v1(100);
+    std::vector<int, min_allocator<int>> v2(200);
+    assert(is_contiguous_container_asan_correct(v1));
+    assert(is_contiguous_container_asan_correct(v2));
+    v1.swap(v2);
+    assert(v1.size() == 200);
+    assert(v1.capacity() == 200);
+    assert(is_contiguous_container_asan_correct(v1));
+    assert(v2.size() == 100);
+    assert(v2.capacity() == 100);
+    assert(is_contiguous_container_asan_correct(v2));
+  }
+  {
+    std::vector<int, safe_allocator<int>> v1(100);
+    std::vector<int, safe_allocator<int>> v2(200);
+    assert(is_contiguous_container_asan_correct(v1));
+    assert(is_contiguous_container_asan_correct(v2));
+    v1.swap(v2);
+    assert(v1.size() == 200);
+    assert(v1.capacity() == 200);
+    assert(is_contiguous_container_asan_correct(v1));
+    assert(v2.size() == 100);
+    assert(v2.capacity() == 100);
+    assert(is_contiguous_container_asan_correct(v2));
+  }
 #endif
 
-    return true;
+  return true;
 }
 
-int main(int, char**)
-{
-    tests();
+int main(int, char**) {
+  tests();
 #if TEST_STD_VER > 17
-    static_assert(tests());
+  static_assert(tests());
 #endif
-    return 0;
+  return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/vector/vector.cons/assign_copy.pass.cpp b/libcxx/test/std/containers/sequences/vector/vector.cons/assign_copy.pass.cpp
index e0e227e8dc0c6..064d807c36ce0 100644
--- a/libcxx/test/std/containers/sequences/vector/vector.cons/assign_copy.pass.cpp
+++ b/libcxx/test/std/containers/sequences/vector/vector.cons/assign_copy.pass.cpp
@@ -18,83 +18,82 @@
 #include "allocators.h"
 
 TEST_CONSTEXPR_CXX20 bool tests() {
-    {
-        std::vector<int, test_allocator<int> > l(3, 2, test_allocator<int>(5));
-        std::vector<int, test_allocator<int> > l2(l, test_allocator<int>(3));
-        l2 = l;
-        assert(l2 == l);
-        assert(l2.get_allocator() == test_allocator<int>(3));
-    }
-    {
-        std::vector<int, other_allocator<int> > l(3, 2, other_allocator<int>(5));
-        std::vector<int, other_allocator<int> > l2(l, other_allocator<int>(3));
-        l2 = l;
-        assert(l2 == l);
-        assert(l2.get_allocator() == other_allocator<int>(5));
-    }
+  {
+    std::vector<int, test_allocator<int> > l(3, 2, test_allocator<int>(5));
+    std::vector<int, test_allocator<int> > l2(l, test_allocator<int>(3));
+    l2 = l;
+    assert(l2 == l);
+    assert(l2.get_allocator() == test_allocator<int>(3));
+  }
+  {
+    std::vector<int, other_allocator<int> > l(3, 2, other_allocator<int>(5));
+    std::vector<int, other_allocator<int> > l2(l, other_allocator<int>(3));
+    l2 = l;
+    assert(l2 == l);
+    assert(l2.get_allocator() == other_allocator<int>(5));
+  }
 #if TEST_STD_VER >= 11
-    {
-        // Test with Allocator::propagate_on_container_copy_assignment == false_type
-        using Alloc = NonPOCCAAllocator<int>;
-        bool copy_assigned_into = false;
-        std::vector<int, Alloc> l(3, 2, Alloc(5, nullptr));
-        std::vector<int, Alloc> l2(l, Alloc(3, &copy_assigned_into));
-        assert(!copy_assigned_into);
-        l2 = l;
-        assert(!copy_assigned_into);
-        assert(l2 == l);
-        assert(l2.get_allocator() == Alloc(3, nullptr));
-    }
-    {
-        // Test with Allocator::propagate_on_container_copy_assignment == true_type
-        // and equal allocators
-        using Alloc = POCCAAllocator<int>;
-        bool copy_assigned_into = false;
-        std::vector<int, Alloc> l(3, 2, Alloc(5, nullptr));
-        std::vector<int, Alloc> l2(l, Alloc(5, &copy_assigned_into));
-        assert(!copy_assigned_into);
-        l2 = l;
-        assert(copy_assigned_into);
-        assert(l2 == l);
-        assert(l2.get_allocator() == Alloc(5, nullptr));
-    }
-    {
-        // Test with Allocator::propagate_on_container_copy_assignment == true_type
-        // and unequal allocators
-        using Alloc = POCCAAllocator<int>;
-        bool copy_assigned_into = false;
-        std::vector<int, Alloc> l(3, 2, Alloc(5, nullptr));
-        std::vector<int, Alloc> l2(l, Alloc(3, &copy_assigned_into));
-        assert(!copy_assigned_into);
-        l2 = l;
-        assert(copy_assigned_into);
-        assert(l2 == l);
-        assert(l2.get_allocator() == Alloc(5, nullptr));
-    }
-    {
-        std::vector<int, min_allocator<int> > l(3, 2, min_allocator<int>());
-        std::vector<int, min_allocator<int> > l2(l, min_allocator<int>());
-        l2 = l;
-        assert(l2 == l);
-        assert(l2.get_allocator() == min_allocator<int>());
-    }
-    {
-      std::vector<int, safe_allocator<int> > l(3, 2, safe_allocator<int>());
-      std::vector<int, safe_allocator<int> > l2(l, safe_allocator<int>());
-      l2 = l;
-      assert(l2 == l);
-      assert(l2.get_allocator() == safe_allocator<int>());
-    }
+  {
+    // Test with Allocator::propagate_on_container_copy_assignment == false_type
+    using Alloc             = NonPOCCAAllocator<int>;
+    bool copy_assigned_into = false;
+    std::vector<int, Alloc> l(3, 2, Alloc(5, nullptr));
+    std::vector<int, Alloc> l2(l, Alloc(3, &copy_assigned_into));
+    assert(!copy_assigned_into);
+    l2 = l;
+    assert(!copy_assigned_into);
+    assert(l2 == l);
+    assert(l2.get_allocator() == Alloc(3, nullptr));
+  }
+  {
+    // Test with Allocator::propagate_on_container_copy_assignment == true_type
+    // and equal allocators
+    using Alloc             = POCCAAllocator<int>;
+    bool copy_assigned_into = false;
+    std::vector<int, Alloc> l(3, 2, Alloc(5, nullptr));
+    std::vector<int, Alloc> l2(l, Alloc(5, &copy_assigned_into));
+    assert(!copy_assigned_into);
+    l2 = l;
+    assert(copy_assigned_into);
+    assert(l2 == l);
+    assert(l2.get_allocator() == Alloc(5, nullptr));
+  }
+  {
+    // Test with Allocator::propagate_on_container_copy_assignment == true_type
+    // and unequal allocators
+    using Alloc             = POCCAAllocator<int>;
+    bool copy_assigned_into = false;
+    std::vector<int, Alloc> l(3, 2, Alloc(5, nullptr));
+    std::vector<int, Alloc> l2(l, Alloc(3, &copy_assigned_into));
+    assert(!copy_assigned_into);
+    l2 = l;
+    assert(copy_assigned_into);
+    assert(l2 == l);
+    assert(l2.get_allocator() == Alloc(5, nullptr));
+  }
+  {
+    std::vector<int, min_allocator<int> > l(3, 2, min_allocator<int>());
+    std::vector<int, min_allocator<int> > l2(l, min_allocator<int>());
+    l2 = l;
+    assert(l2 == l);
+    assert(l2.get_allocator() == min_allocator<int>());
+  }
+  {
+    std::vector<int, safe_allocator<int> > l(3, 2, safe_allocator<int>());
+    std::vector<int, safe_allocator<int> > l2(l, safe_allocator<int>());
+    l2 = l;
+    assert(l2 == l);
+    assert(l2.get_allocator() == safe_allocator<int>());
+  }
 #endif
 
-    return true;
+  return true;
 }
 
-int main(int, char**)
-{
-    tests();
+int main(int, char**) {
+  tests();
 #if TEST_STD_VER > 17
-    static_assert(tests());
+  static_assert(tests());
 #endif
-    return 0;
+  return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/vector/vector.cons/assign_initializer_list.pass.cpp b/libcxx/test/std/containers/sequences/vector/vector.cons/assign_initializer_list.pass.cpp
index 19a1b77603552..0be881875be2d 100644
--- a/libcxx/test/std/containers/sequences/vector/vector.cons/assign_initializer_list.pass.cpp
+++ b/libcxx/test/std/containers/sequences/vector/vector.cons/assign_initializer_list.pass.cpp
@@ -20,43 +20,41 @@
 #include "asan_testing.h"
 
 template <typename Vec>
-TEST_CONSTEXPR_CXX20 void test(Vec &v)
-{
-    v.assign({3, 4, 5, 6});
-    assert(v.size() == 4);
-    assert(is_contiguous_container_asan_correct(v));
-    assert(v[0] == 3);
-    assert(v[1] == 4);
-    assert(v[2] == 5);
-    assert(v[3] == 6);
+TEST_CONSTEXPR_CXX20 void test(Vec& v) {
+  v.assign({3, 4, 5, 6});
+  assert(v.size() == 4);
+  assert(is_contiguous_container_asan_correct(v));
+  assert(v[0] == 3);
+  assert(v[1] == 4);
+  assert(v[2] == 5);
+  assert(v[3] == 6);
 }
 
 TEST_CONSTEXPR_CXX20 bool tests() {
-    {
+  {
     typedef std::vector<int> V;
     V d1;
     V d2;
-    d2.reserve(10);  // no reallocation during assign.
+    d2.reserve(10); // no reallocation during assign.
     test(d1);
     test(d2);
-    }
-    {
+  }
+  {
     typedef std::vector<int, min_allocator<int>> V;
     V d1;
     V d2;
-    d2.reserve(10);  // no reallocation during assign.
+    d2.reserve(10); // no reallocation during assign.
     test(d1);
     test(d2);
-    }
+  }
 
-    return true;
+  return true;
 }
 
-int main(int, char**)
-{
-    tests();
+int main(int, char**) {
+  tests();
 #if TEST_STD_VER > 17
-    static_assert(tests());
+  static_assert(tests());
 #endif
-    return 0;
+  return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/vector/vector.cons/assign_move.pass.cpp b/libcxx/test/std/containers/sequences/vector/vector.cons/assign_move.pass.cpp
index 58081fa0ecef5..aad7908168105 100644
--- a/libcxx/test/std/containers/sequences/vector/vector.cons/assign_move.pass.cpp
+++ b/libcxx/test/std/containers/sequences/vector/vector.cons/assign_move.pass.cpp
@@ -21,107 +21,102 @@
 #include "asan_testing.h"
 
 TEST_CONSTEXPR_CXX20 bool tests() {
-    {
-        std::vector<MoveOnly, test_allocator<MoveOnly> > l(test_allocator<MoveOnly>(5));
-        std::vector<MoveOnly, test_allocator<MoveOnly> > lo(test_allocator<MoveOnly>(5));
-        for (int i = 1; i <= 3; ++i)
-        {
-            l.push_back(i);
-            lo.push_back(i);
-        }
-        assert(is_contiguous_container_asan_correct(l));
-        assert(is_contiguous_container_asan_correct(lo));
-        std::vector<MoveOnly, test_allocator<MoveOnly> > l2(test_allocator<MoveOnly>(5));
-        l2 = std::move(l);
-        assert(l2 == lo);
-        assert(l.empty());
-        assert(l2.get_allocator() == lo.get_allocator());
-        assert(is_contiguous_container_asan_correct(l2));
+  {
+    std::vector<MoveOnly, test_allocator<MoveOnly> > l(test_allocator<MoveOnly>(5));
+    std::vector<MoveOnly, test_allocator<MoveOnly> > lo(test_allocator<MoveOnly>(5));
+    for (int i = 1; i <= 3; ++i) {
+      l.push_back(i);
+      lo.push_back(i);
     }
-    {
-        std::vector<MoveOnly, test_allocator<MoveOnly> > l(test_allocator<MoveOnly>(5));
-        std::vector<MoveOnly, test_allocator<MoveOnly> > lo(test_allocator<MoveOnly>(5));
-        assert(is_contiguous_container_asan_correct(l));
-        assert(is_contiguous_container_asan_correct(lo));
-        for (int i = 1; i <= 3; ++i)
-        {
-            l.push_back(i);
-            lo.push_back(i);
-        }
-        assert(is_contiguous_container_asan_correct(l));
-        assert(is_contiguous_container_asan_correct(lo));
-        std::vector<MoveOnly, test_allocator<MoveOnly> > l2(test_allocator<MoveOnly>(6));
-        l2 = std::move(l);
-        assert(l2 == lo);
-        assert(!l.empty());
-        assert(l2.get_allocator() == test_allocator<MoveOnly>(6));
-        assert(is_contiguous_container_asan_correct(l2));
+    assert(is_contiguous_container_asan_correct(l));
+    assert(is_contiguous_container_asan_correct(lo));
+    std::vector<MoveOnly, test_allocator<MoveOnly> > l2(test_allocator<MoveOnly>(5));
+    l2 = std::move(l);
+    assert(l2 == lo);
+    assert(l.empty());
+    assert(l2.get_allocator() == lo.get_allocator());
+    assert(is_contiguous_container_asan_correct(l2));
+  }
+  {
+    std::vector<MoveOnly, test_allocator<MoveOnly> > l(test_allocator<MoveOnly>(5));
+    std::vector<MoveOnly, test_allocator<MoveOnly> > lo(test_allocator<MoveOnly>(5));
+    assert(is_contiguous_container_asan_correct(l));
+    assert(is_contiguous_container_asan_correct(lo));
+    for (int i = 1; i <= 3; ++i) {
+      l.push_back(i);
+      lo.push_back(i);
     }
-    {
-        std::vector<MoveOnly, other_allocator<MoveOnly> > l(other_allocator<MoveOnly>(5));
-        std::vector<MoveOnly, other_allocator<MoveOnly> > lo(other_allocator<MoveOnly>(5));
-        assert(is_contiguous_container_asan_correct(l));
-        assert(is_contiguous_container_asan_correct(lo));
-        for (int i = 1; i <= 3; ++i)
-        {
-            l.push_back(i);
-            lo.push_back(i);
-        }
-        assert(is_contiguous_container_asan_correct(l));
-        assert(is_contiguous_container_asan_correct(lo));
-        std::vector<MoveOnly, other_allocator<MoveOnly> > l2(other_allocator<MoveOnly>(6));
-        l2 = std::move(l);
-        assert(l2 == lo);
-        assert(l.empty());
-        assert(l2.get_allocator() == lo.get_allocator());
-        assert(is_contiguous_container_asan_correct(l2));
+    assert(is_contiguous_container_asan_correct(l));
+    assert(is_contiguous_container_asan_correct(lo));
+    std::vector<MoveOnly, test_allocator<MoveOnly> > l2(test_allocator<MoveOnly>(6));
+    l2 = std::move(l);
+    assert(l2 == lo);
+    assert(!l.empty());
+    assert(l2.get_allocator() == test_allocator<MoveOnly>(6));
+    assert(is_contiguous_container_asan_correct(l2));
+  }
+  {
+    std::vector<MoveOnly, other_allocator<MoveOnly> > l(other_allocator<MoveOnly>(5));
+    std::vector<MoveOnly, other_allocator<MoveOnly> > lo(other_allocator<MoveOnly>(5));
+    assert(is_contiguous_container_asan_correct(l));
+    assert(is_contiguous_container_asan_correct(lo));
+    for (int i = 1; i <= 3; ++i) {
+      l.push_back(i);
+      lo.push_back(i);
     }
-    {
-        std::vector<MoveOnly, min_allocator<MoveOnly> > l((min_allocator<MoveOnly>()));
-        std::vector<MoveOnly, min_allocator<MoveOnly> > lo((min_allocator<MoveOnly>()));
-        assert(is_contiguous_container_asan_correct(l));
-        assert(is_contiguous_container_asan_correct(lo));
-        for (int i = 1; i <= 3; ++i)
-        {
-            l.push_back(i);
-            lo.push_back(i);
-        }
-        assert(is_contiguous_container_asan_correct(l));
-        assert(is_contiguous_container_asan_correct(lo));
-        std::vector<MoveOnly, min_allocator<MoveOnly> > l2((min_allocator<MoveOnly>()));
-        l2 = std::move(l);
-        assert(l2 == lo);
-        assert(l.empty());
-        assert(l2.get_allocator() == lo.get_allocator());
-        assert(is_contiguous_container_asan_correct(l2));
+    assert(is_contiguous_container_asan_correct(l));
+    assert(is_contiguous_container_asan_correct(lo));
+    std::vector<MoveOnly, other_allocator<MoveOnly> > l2(other_allocator<MoveOnly>(6));
+    l2 = std::move(l);
+    assert(l2 == lo);
+    assert(l.empty());
+    assert(l2.get_allocator() == lo.get_allocator());
+    assert(is_contiguous_container_asan_correct(l2));
+  }
+  {
+    std::vector<MoveOnly, min_allocator<MoveOnly> > l((min_allocator<MoveOnly>()));
+    std::vector<MoveOnly, min_allocator<MoveOnly> > lo((min_allocator<MoveOnly>()));
+    assert(is_contiguous_container_asan_correct(l));
+    assert(is_contiguous_container_asan_correct(lo));
+    for (int i = 1; i <= 3; ++i) {
+      l.push_back(i);
+      lo.push_back(i);
     }
-    {
-      std::vector<MoveOnly, safe_allocator<MoveOnly> > l((safe_allocator<MoveOnly>()));
-      std::vector<MoveOnly, safe_allocator<MoveOnly> > lo((safe_allocator<MoveOnly>()));
-      assert(is_contiguous_container_asan_correct(l));
-      assert(is_contiguous_container_asan_correct(lo));
-      for (int i = 1; i <= 3; ++i) {
-        l.push_back(i);
-        lo.push_back(i);
-      }
-      assert(is_contiguous_container_asan_correct(l));
-      assert(is_contiguous_container_asan_correct(lo));
-      std::vector<MoveOnly, safe_allocator<MoveOnly> > l2((safe_allocator<MoveOnly>()));
-      l2 = std::move(l);
-      assert(l2 == lo);
-      assert(l.empty());
-      assert(l2.get_allocator() == lo.get_allocator());
-      assert(is_contiguous_container_asan_correct(l2));
+    assert(is_contiguous_container_asan_correct(l));
+    assert(is_contiguous_container_asan_correct(lo));
+    std::vector<MoveOnly, min_allocator<MoveOnly> > l2((min_allocator<MoveOnly>()));
+    l2 = std::move(l);
+    assert(l2 == lo);
+    assert(l.empty());
+    assert(l2.get_allocator() == lo.get_allocator());
+    assert(is_contiguous_container_asan_correct(l2));
+  }
+  {
+    std::vector<MoveOnly, safe_allocator<MoveOnly> > l((safe_allocator<MoveOnly>()));
+    std::vector<MoveOnly, safe_allocator<MoveOnly> > lo((safe_allocator<MoveOnly>()));
+    assert(is_contiguous_container_asan_correct(l));
+    assert(is_contiguous_container_asan_correct(lo));
+    for (int i = 1; i <= 3; ++i) {
+      l.push_back(i);
+      lo.push_back(i);
     }
+    assert(is_contiguous_container_asan_correct(l));
+    assert(is_contiguous_container_asan_correct(lo));
+    std::vector<MoveOnly, safe_allocator<MoveOnly> > l2((safe_allocator<MoveOnly>()));
+    l2 = std::move(l);
+    assert(l2 == lo);
+    assert(l.empty());
+    assert(l2.get_allocator() == lo.get_allocator());
+    assert(is_contiguous_container_asan_correct(l2));
+  }
 
-    return true;
+  return true;
 }
 
-int main(int, char**)
-{
-    tests();
+int main(int, char**) {
+  tests();
 #if TEST_STD_VER > 17
-    static_assert(tests());
+  static_assert(tests());
 #endif
-    return 0;
+  return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/vector/vector.cons/assign_size_value.pass.cpp b/libcxx/test/std/containers/sequences/vector/vector.cons/assign_size_value.pass.cpp
index 896aa40c0db95..8526cf587ed3f 100644
--- a/libcxx/test/std/containers/sequences/vector/vector.cons/assign_size_value.pass.cpp
+++ b/libcxx/test/std/containers/sequences/vector/vector.cons/assign_size_value.pass.cpp
@@ -21,48 +21,46 @@
 TEST_CONSTEXPR bool is6(int x) { return x == 6; }
 
 template <typename Vec>
-TEST_CONSTEXPR_CXX20 void test(Vec &v)
-{
-    v.assign(5, 6);
-    assert(v.size() == 5);
-    assert(is_contiguous_container_asan_correct(v));
-    assert(std::all_of(v.begin(), v.end(), is6));
+TEST_CONSTEXPR_CXX20 void test(Vec& v) {
+  v.assign(5, 6);
+  assert(v.size() == 5);
+  assert(is_contiguous_container_asan_correct(v));
+  assert(std::all_of(v.begin(), v.end(), is6));
 }
 
 TEST_CONSTEXPR_CXX20 bool tests() {
-    {
+  {
     typedef std::vector<int> V;
     V d1;
     V d2;
-    d2.reserve(10);  // no reallocation during assign.
+    d2.reserve(10); // no reallocation during assign.
     test(d1);
     test(d2);
-    }
-    {
+  }
+  {
     std::vector<int> vec;
     vec.reserve(32);
     vec.resize(16); // destruction during assign
     test(vec);
-    }
+  }
 #if TEST_STD_VER >= 11
-    {
+  {
     typedef std::vector<int, min_allocator<int>> V;
     V d1;
     V d2;
-    d2.reserve(10);  // no reallocation during assign.
+    d2.reserve(10); // no reallocation during assign.
     test(d1);
     test(d2);
-    }
+  }
 #endif
 
-    return true;
+  return true;
 }
 
-int main(int, char**)
-{
-    tests();
+int main(int, char**) {
+  tests();
 #if TEST_STD_VER > 17
-    static_assert(tests());
+  static_assert(tests());
 #endif
-    return 0;
+  return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/vector/vector.cons/construct_default.pass.cpp b/libcxx/test/std/containers/sequences/vector/vector.cons/construct_default.pass.cpp
index 9a135b64a336c..9be4e95156fa9 100644
--- a/libcxx/test/std/containers/sequences/vector/vector.cons/construct_default.pass.cpp
+++ b/libcxx/test/std/containers/sequences/vector/vector.cons/construct_default.pass.cpp
@@ -22,88 +22,85 @@
 #include "asan_testing.h"
 
 template <class C>
-TEST_CONSTEXPR_CXX20 void test0()
-{
+TEST_CONSTEXPR_CXX20 void test0() {
 #if TEST_STD_VER > 14
-    static_assert((noexcept(C{})), "" );
+  static_assert((noexcept(C{})), "");
 #elif TEST_STD_VER >= 11
-    static_assert((noexcept(C()) == noexcept(typename C::allocator_type())), "" );
+  static_assert((noexcept(C()) == noexcept(typename C::allocator_type())), "");
 #endif
-    C c;
-    LIBCPP_ASSERT(c.__invariants());
-    assert(c.empty());
-    assert(c.get_allocator() == typename C::allocator_type());
-    LIBCPP_ASSERT(is_contiguous_container_asan_correct(c));
+  C c;
+  LIBCPP_ASSERT(c.__invariants());
+  assert(c.empty());
+  assert(c.get_allocator() == typename C::allocator_type());
+  LIBCPP_ASSERT(is_contiguous_container_asan_correct(c));
 #if TEST_STD_VER >= 11
-    C c1 = {};
-    LIBCPP_ASSERT(c1.__invariants());
-    assert(c1.empty());
-    assert(c1.get_allocator() == typename C::allocator_type());
-    LIBCPP_ASSERT(is_contiguous_container_asan_correct(c1));
+  C c1 = {};
+  LIBCPP_ASSERT(c1.__invariants());
+  assert(c1.empty());
+  assert(c1.get_allocator() == typename C::allocator_type());
+  LIBCPP_ASSERT(is_contiguous_container_asan_correct(c1));
 #endif
 }
 
 template <class C>
-TEST_CONSTEXPR_CXX20 void test1(const typename C::allocator_type& a)
-{
+TEST_CONSTEXPR_CXX20 void test1(const typename C::allocator_type& a) {
 #if TEST_STD_VER > 14
-    static_assert((noexcept(C{typename C::allocator_type{}})), "" );
+  static_assert((noexcept(C{typename C::allocator_type{}})), "");
 #elif TEST_STD_VER >= 11
-    static_assert((noexcept(C(typename C::allocator_type())) == std::is_nothrow_copy_constructible<typename C::allocator_type>::value), "" );
+  static_assert((noexcept(C(typename C::allocator_type())) ==
+                 std::is_nothrow_copy_constructible<typename C::allocator_type>::value),
+                "");
 #endif
-    C c(a);
-    LIBCPP_ASSERT(c.__invariants());
-    assert(c.empty());
-    assert(c.get_allocator() == a);
-    LIBCPP_ASSERT(is_contiguous_container_asan_correct(c));
+  C c(a);
+  LIBCPP_ASSERT(c.__invariants());
+  assert(c.empty());
+  assert(c.get_allocator() == a);
+  LIBCPP_ASSERT(is_contiguous_container_asan_correct(c));
 }
 
 TEST_CONSTEXPR_CXX20 bool tests() {
-    {
+  {
     test0<std::vector<int> >();
     test0<std::vector<NotConstructible> >();
     test1<std::vector<int, test_allocator<int> > >(test_allocator<int>(3));
-    test1<std::vector<NotConstructible, test_allocator<NotConstructible> > >
-        (test_allocator<NotConstructible>(5));
-    }
-    {
-        std::vector<int, limited_allocator<int, 10> > v;
-        assert(v.empty());
-    }
+    test1<std::vector<NotConstructible, test_allocator<NotConstructible> > >(test_allocator<NotConstructible>(5));
+  }
+  {
+    std::vector<int, limited_allocator<int, 10> > v;
+    assert(v.empty());
+  }
 #if TEST_STD_VER >= 11
-    {
+  {
     test0<std::vector<int, min_allocator<int>> >();
     test0<std::vector<NotConstructible, min_allocator<NotConstructible>> >();
     test1<std::vector<int, min_allocator<int> > >(min_allocator<int>{});
-    test1<std::vector<NotConstructible, min_allocator<NotConstructible> > >
-        (min_allocator<NotConstructible>{});
-    }
-    {
-        std::vector<int, min_allocator<int> > v;
-        assert(v.empty());
-    }
+    test1<std::vector<NotConstructible, min_allocator<NotConstructible> > >(min_allocator<NotConstructible>{});
+  }
+  {
+    std::vector<int, min_allocator<int> > v;
+    assert(v.empty());
+  }
 
-    {
+  {
     test0<std::vector<int, explicit_allocator<int>> >();
     test0<std::vector<NotConstructible, explicit_allocator<NotConstructible>> >();
     test1<std::vector<int, explicit_allocator<int> > >(explicit_allocator<int>{});
-    test1<std::vector<NotConstructible, explicit_allocator<NotConstructible> > >
-        (explicit_allocator<NotConstructible>{});
-    }
-    {
-        std::vector<int, explicit_allocator<int> > v;
-        assert(v.empty());
-    }
+    test1<std::vector<NotConstructible, explicit_allocator<NotConstructible> > >(
+        explicit_allocator<NotConstructible>{});
+  }
+  {
+    std::vector<int, explicit_allocator<int> > v;
+    assert(v.empty());
+  }
 #endif
 
-    return true;
+  return true;
 }
 
-int main(int, char**)
-{
-    tests();
+int main(int, char**) {
+  tests();
 #if TEST_STD_VER > 17
-    static_assert(tests());
+  static_assert(tests());
 #endif
-    return 0;
+  return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/vector/vector.cons/construct_iter_iter.pass.cpp b/libcxx/test/std/containers/sequences/vector/vector.cons/construct_iter_iter.pass.cpp
index 3a66bac344c87..bac2ea2b4ca1e 100644
--- a/libcxx/test/std/containers/sequences/vector/vector.cons/construct_iter_iter.pass.cpp
+++ b/libcxx/test/std/containers/sequences/vector/vector.cons/construct_iter_iter.pass.cpp
@@ -20,8 +20,8 @@
 #include "min_allocator.h"
 #include "asan_testing.h"
 #if TEST_STD_VER >= 11
-#include "emplace_constructible.h"
-#include "container_test_types.h"
+#  include "emplace_constructible.h"
+#  include "container_test_types.h"
 #endif
 
 template <class C, class Iterator>
@@ -31,9 +31,8 @@ TEST_CONSTEXPR_CXX20 void test(Iterator first, Iterator last) {
     LIBCPP_ASSERT(c.__invariants());
     assert(c.size() == static_cast<std::size_t>(std::distance(first, last)));
     LIBCPP_ASSERT(is_contiguous_container_asan_correct(c));
-    for (typename C::const_iterator i = c.cbegin(), e = c.cend(); i != e;
-      ++i, ++first)
-    assert(*i == *first);
+    for (typename C::const_iterator i = c.cbegin(), e = c.cend(); i != e; ++i, ++first)
+      assert(*i == *first);
   }
   // Test with an empty range
   {
@@ -47,14 +46,10 @@ TEST_CONSTEXPR_CXX20 void test(Iterator first, Iterator last) {
 TEST_CONSTEXPR_CXX20 void basic_test_cases() {
   int a[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 8, 7, 6, 5, 4, 3, 1, 0};
   int* an = a + sizeof(a) / sizeof(a[0]);
-  test<std::vector<int> >(cpp17_input_iterator<const int*>(a),
-                          cpp17_input_iterator<const int*>(an));
-  test<std::vector<int> >(forward_iterator<const int*>(a),
-                          forward_iterator<const int*>(an));
-  test<std::vector<int> >(bidirectional_iterator<const int*>(a),
-                          bidirectional_iterator<const int*>(an));
-  test<std::vector<int> >(random_access_iterator<const int*>(a),
-                          random_access_iterator<const int*>(an));
+  test<std::vector<int> >(cpp17_input_iterator<const int*>(a), cpp17_input_iterator<const int*>(an));
+  test<std::vector<int> >(forward_iterator<const int*>(a), forward_iterator<const int*>(an));
+  test<std::vector<int> >(bidirectional_iterator<const int*>(a), bidirectional_iterator<const int*>(an));
+  test<std::vector<int> >(random_access_iterator<const int*>(a), random_access_iterator<const int*>(an));
   test<std::vector<int> >(a, an);
 
   test<std::vector<int, limited_allocator<int, 63> > >(
@@ -63,23 +58,18 @@ TEST_CONSTEXPR_CXX20 void basic_test_cases() {
   test<std::vector<int, limited_allocator<int, 18 + 1> > >(
       forward_iterator<const int*>(a), forward_iterator<const int*>(an));
   test<std::vector<int, limited_allocator<int, 18 + 1> > >(
-      bidirectional_iterator<const int*>(a),
-      bidirectional_iterator<const int*>(an));
+      bidirectional_iterator<const int*>(a), bidirectional_iterator<const int*>(an));
   test<std::vector<int, limited_allocator<int, 18 + 1> > >(
-      random_access_iterator<const int*>(a),
-      random_access_iterator<const int*>(an));
+      random_access_iterator<const int*>(a), random_access_iterator<const int*>(an));
   test<std::vector<int, limited_allocator<int, 18 + 1> > >(a, an);
 #if TEST_STD_VER >= 11
-  test<std::vector<int, min_allocator<int> > >(cpp17_input_iterator<const int*>(a),
-                                               cpp17_input_iterator<const int*>(an));
   test<std::vector<int, min_allocator<int> > >(
-      forward_iterator<const int*>(a), forward_iterator<const int*>(an));
+      cpp17_input_iterator<const int*>(a), cpp17_input_iterator<const int*>(an));
+  test<std::vector<int, min_allocator<int> > >(forward_iterator<const int*>(a), forward_iterator<const int*>(an));
   test<std::vector<int, min_allocator<int> > >(
-      bidirectional_iterator<const int*>(a),
-      bidirectional_iterator<const int*>(an));
+      bidirectional_iterator<const int*>(a), bidirectional_iterator<const int*>(an));
   test<std::vector<int, min_allocator<int> > >(
-      random_access_iterator<const int*>(a),
-      random_access_iterator<const int*>(an));
+      random_access_iterator<const int*>(a), random_access_iterator<const int*>(an));
   test<std::vector<int> >(a, an);
   test<std::vector<int, safe_allocator<int> > >(
       cpp17_input_iterator<const int*>(a), cpp17_input_iterator<const int*>(an));
@@ -102,7 +92,7 @@ TEST_CONSTEXPR_CXX20 void emplaceable_concept_tests() {
   int arr1[] = {42};
   int arr2[] = {1, 101, 42};
   {
-    using T = EmplaceConstructible<int>;
+    using T  = EmplaceConstructible<int>;
     using It = forward_iterator<int*>;
     {
       std::vector<T> v(It(arr1), It(std::end(arr1)));
@@ -116,7 +106,7 @@ TEST_CONSTEXPR_CXX20 void emplaceable_concept_tests() {
     }
   }
   {
-    using T = EmplaceConstructibleAndMoveInsertable<int>;
+    using T  = EmplaceConstructibleAndMoveInsertable<int>;
     using It = cpp17_input_iterator<int*>;
     {
       std::vector<T> v(It(arr1), It(std::end(arr1)));
@@ -141,7 +131,7 @@ void test_ctor_under_alloc() {
   int arr1[] = {42};
   int arr2[] = {1, 101, 42};
   {
-    using C = TCT::vector<>;
+    using C  = TCT::vector<>;
     using It = forward_iterator<int*>;
     {
       ExpectConstructGuard<int&> G(1);
@@ -153,7 +143,7 @@ void test_ctor_under_alloc() {
     }
   }
   {
-    using C = TCT::vector<>;
+    using C  = TCT::vector<>;
     using It = cpp17_input_iterator<int*>;
     {
       ExpectConstructGuard<int&> G(1);
@@ -168,9 +158,15 @@ void test_ctor_under_alloc() {
 }
 
 // In C++03, you can't instantiate a template with a local type.
-struct B1 { int x; };
-struct B2 { int y; };
-struct Der : B1, B2 { int z; };
+struct B1 {
+  int x;
+};
+struct B2 {
+  int y;
+};
+struct Der : B1, B2 {
+  int z;
+};
 
 // Initialize a vector with a different value type.
 TEST_CONSTEXPR_CXX20 void test_ctor_with_different_value_type() {
@@ -188,7 +184,7 @@ TEST_CONSTEXPR_CXX20 void test_ctor_with_different_value_type() {
   }
   {
     Der z;
-    Der *array[1] = { &z };
+    Der* array[1] = {&z};
     // Though the types Der* and B2* are very similar, initialization still cannot
     // be done with `memcpy`.
     std::vector<B2*> v(array, array + 1);
@@ -196,7 +192,7 @@ TEST_CONSTEXPR_CXX20 void test_ctor_with_different_value_type() {
   }
   {
     // Though the types are different, initialization can be done with `memcpy`.
-    std::int32_t array[1] = { -1 };
+    std::int32_t array[1] = {-1};
     std::vector<std::uint32_t> v(array, array + 1);
     assert(v[0] == 4294967295U);
   }
@@ -210,12 +206,11 @@ TEST_CONSTEXPR_CXX20 bool tests() {
   return true;
 }
 
-int main(int, char**)
-{
-    tests();
-    test_ctor_under_alloc();
+int main(int, char**) {
+  tests();
+  test_ctor_under_alloc();
 #if TEST_STD_VER > 17
-    static_assert(tests());
+  static_assert(tests());
 #endif
-    return 0;
+  return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/vector/vector.cons/construct_iter_iter_alloc.pass.cpp b/libcxx/test/std/containers/sequences/vector/vector.cons/construct_iter_iter_alloc.pass.cpp
index 42fb5f7c054b6..de325040f4a19 100644
--- a/libcxx/test/std/containers/sequences/vector/vector.cons/construct_iter_iter_alloc.pass.cpp
+++ b/libcxx/test/std/containers/sequences/vector/vector.cons/construct_iter_iter_alloc.pass.cpp
@@ -21,8 +21,8 @@
 #include "min_allocator.h"
 #include "asan_testing.h"
 #if TEST_STD_VER >= 11
-#include "emplace_constructible.h"
-#include "container_test_types.h"
+#  include "emplace_constructible.h"
+#  include "container_test_types.h"
 #endif
 
 template <class C, class Iterator, class A>
@@ -31,8 +31,7 @@ TEST_CONSTEXPR_CXX20 void test(Iterator first, Iterator last, const A& a) {
   LIBCPP_ASSERT(c.__invariants());
   assert(c.size() == static_cast<std::size_t>(std::distance(first, last)));
   LIBCPP_ASSERT(is_contiguous_container_asan_correct(c));
-  for (typename C::const_iterator i = c.cbegin(), e = c.cend(); i != e;
-       ++i, ++first)
+  for (typename C::const_iterator i = c.cbegin(), e = c.cend(); i != e; ++i, ++first)
     assert(*i == *first);
 }
 
@@ -54,14 +53,10 @@ TEST_CONSTEXPR_CXX20 void basic_tests() {
     int a[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 8, 7, 6, 5, 4, 3, 1, 0};
     int* an = a + sizeof(a) / sizeof(a[0]);
     std::allocator<int> alloc;
-    test<std::vector<int> >(cpp17_input_iterator<const int*>(a),
-                            cpp17_input_iterator<const int*>(an), alloc);
-    test<std::vector<int> >(forward_iterator<const int*>(a),
-                            forward_iterator<const int*>(an), alloc);
-    test<std::vector<int> >(bidirectional_iterator<const int*>(a),
-                            bidirectional_iterator<const int*>(an), alloc);
-    test<std::vector<int> >(random_access_iterator<const int*>(a),
-                            random_access_iterator<const int*>(an), alloc);
+    test<std::vector<int> >(cpp17_input_iterator<const int*>(a), cpp17_input_iterator<const int*>(an), alloc);
+    test<std::vector<int> >(forward_iterator<const int*>(a), forward_iterator<const int*>(an), alloc);
+    test<std::vector<int> >(bidirectional_iterator<const int*>(a), bidirectional_iterator<const int*>(an), alloc);
+    test<std::vector<int> >(random_access_iterator<const int*>(a), random_access_iterator<const int*>(an), alloc);
     test<std::vector<int> >(a, an, alloc);
   }
 #if TEST_STD_VER >= 11
@@ -72,14 +67,11 @@ TEST_CONSTEXPR_CXX20 void basic_tests() {
     test<std::vector<int, min_allocator<int> > >(
         cpp17_input_iterator<const int*>(a), cpp17_input_iterator<const int*>(an), alloc);
     test<std::vector<int, min_allocator<int> > >(
-        forward_iterator<const int*>(a), forward_iterator<const int*>(an),
-        alloc);
+        forward_iterator<const int*>(a), forward_iterator<const int*>(an), alloc);
     test<std::vector<int, min_allocator<int> > >(
-        bidirectional_iterator<const int*>(a),
-        bidirectional_iterator<const int*>(an), alloc);
+        bidirectional_iterator<const int*>(a), bidirectional_iterator<const int*>(an), alloc);
     test<std::vector<int, min_allocator<int> > >(
-        random_access_iterator<const int*>(a),
-        random_access_iterator<const int*>(an), alloc);
+        random_access_iterator<const int*>(a), random_access_iterator<const int*>(an), alloc);
     test<std::vector<int, min_allocator<int> > >(a, an, alloc);
     test<std::vector<int, implicit_conv_allocator<int> > >(a, an, nullptr);
   }
@@ -112,8 +104,8 @@ TEST_CONSTEXPR_CXX20 void emplaceable_concept_tests() {
   int arr1[] = {42};
   int arr2[] = {1, 101, 42};
   {
-    using T = EmplaceConstructible<int>;
-    using It = forward_iterator<int*>;
+    using T     = EmplaceConstructible<int>;
+    using It    = forward_iterator<int*>;
     using Alloc = std::allocator<T>;
     Alloc a;
     {
@@ -128,8 +120,8 @@ TEST_CONSTEXPR_CXX20 void emplaceable_concept_tests() {
     }
   }
   {
-    using T = EmplaceConstructibleAndMoveInsertable<int>;
-    using It = cpp17_input_iterator<int*>;
+    using T     = EmplaceConstructibleAndMoveInsertable<int>;
+    using It    = cpp17_input_iterator<int*>;
     using Alloc = std::allocator<T>;
     Alloc a;
     {
@@ -153,8 +145,8 @@ void test_ctor_under_alloc() {
   int arr1[] = {42};
   int arr2[] = {1, 101, 42};
   {
-    using C = TCT::vector<>;
-    using It = forward_iterator<int*>;
+    using C     = TCT::vector<>;
+    using It    = forward_iterator<int*>;
     using Alloc = typename C::allocator_type;
     Alloc a;
     {
@@ -167,8 +159,8 @@ void test_ctor_under_alloc() {
     }
   }
   {
-    using C = TCT::vector<>;
-    using It = cpp17_input_iterator<int*>;
+    using C     = TCT::vector<>;
+    using It    = cpp17_input_iterator<int*>;
     using Alloc = typename C::allocator_type;
     Alloc a;
     {
diff --git a/libcxx/test/std/containers/sequences/vector/vector.cons/construct_size.pass.cpp b/libcxx/test/std/containers/sequences/vector/vector.cons/construct_size.pass.cpp
index 1dc7cd9bc7c1b..cbae3653d995f 100644
--- a/libcxx/test/std/containers/sequences/vector/vector.cons/construct_size.pass.cpp
+++ b/libcxx/test/std/containers/sequences/vector/vector.cons/construct_size.pass.cpp
@@ -21,69 +21,67 @@
 #include "asan_testing.h"
 
 template <class C>
-TEST_CONSTEXPR_CXX20
-void test(typename C::size_type n,
-          typename C::allocator_type const& a = typename C::allocator_type())
-{
-    (void)a;
-    // Test without a custom allocator
-    {
-        C c(n);
-        LIBCPP_ASSERT(c.__invariants());
-        assert(c.size() == n);
-        assert(c.get_allocator() == typename C::allocator_type());
-        LIBCPP_ASSERT(is_contiguous_container_asan_correct(c));
+TEST_CONSTEXPR_CXX20 void
+test(typename C::size_type n, typename C::allocator_type const& a = typename C::allocator_type()) {
+  (void)a;
+  // Test without a custom allocator
+  {
+    C c(n);
+    LIBCPP_ASSERT(c.__invariants());
+    assert(c.size() == n);
+    assert(c.get_allocator() == typename C::allocator_type());
+    LIBCPP_ASSERT(is_contiguous_container_asan_correct(c));
 #if TEST_STD_VER >= 11
-        for (typename C::const_iterator i = c.cbegin(), e = c.cend(); i != e; ++i)
-            assert(*i == typename C::value_type());
+    for (typename C::const_iterator i = c.cbegin(), e = c.cend(); i != e; ++i)
+      assert(*i == typename C::value_type());
 #endif
-    }
+  }
 
-    // Test with a custom allocator
+  // Test with a custom allocator
 #if TEST_STD_VER >= 14
-    {
-        C c(n, a);
-        LIBCPP_ASSERT(c.__invariants());
-        assert(c.size() == n);
-        assert(c.get_allocator() == a);
-        LIBCPP_ASSERT(is_contiguous_container_asan_correct(c));
-        for (typename C::const_iterator i = c.cbegin(), e = c.cend(); i != e; ++i)
-            assert(*i == typename C::value_type());
-    }
+  {
+    C c(n, a);
+    LIBCPP_ASSERT(c.__invariants());
+    assert(c.size() == n);
+    assert(c.get_allocator() == a);
+    LIBCPP_ASSERT(is_contiguous_container_asan_correct(c));
+    for (typename C::const_iterator i = c.cbegin(), e = c.cend(); i != e; ++i)
+      assert(*i == typename C::value_type());
+  }
 #endif
 }
 
 TEST_CONSTEXPR_CXX20 bool tests() {
-    test<std::vector<int> >(0);
-    test<std::vector<int> >(50);
+  test<std::vector<int> >(0);
+  test<std::vector<int> >(50);
 #if TEST_STD_VER >= 11
-    test<std::vector<int, min_allocator<int>>>(0);
-    test<std::vector<int, min_allocator<int>>>(50);
-    test<std::vector<int, safe_allocator<int>>>(0);
-    test<std::vector<int, safe_allocator<int>>>(50);
+  test<std::vector<int, min_allocator<int>>>(0);
+  test<std::vector<int, min_allocator<int>>>(50);
+  test<std::vector<int, safe_allocator<int>>>(0);
+  test<std::vector<int, safe_allocator<int>>>(50);
 #endif
 
-    return true;
+  return true;
 }
 
 int main(int, char**) {
-    tests();
+  tests();
 #if TEST_STD_VER > 17
-    static_assert(tests());
+  static_assert(tests());
 #endif
-    test<std::vector<DefaultOnly> >(0);
-    test<std::vector<DefaultOnly> >(500);
-    assert(DefaultOnly::count == 0);
+  test<std::vector<DefaultOnly> >(0);
+  test<std::vector<DefaultOnly> >(500);
+  assert(DefaultOnly::count == 0);
 
 #if TEST_STD_VER >= 11
-    test<std::vector<DefaultOnly, min_allocator<DefaultOnly>>>(0);
-    test<std::vector<DefaultOnly, min_allocator<DefaultOnly>>>(500);
-    test<std::vector<DefaultOnly, safe_allocator<DefaultOnly>>>(0);
-    test<std::vector<DefaultOnly, safe_allocator<DefaultOnly>>>(500);
-    test<std::vector<DefaultOnly, test_allocator<DefaultOnly>>>(0, test_allocator<DefaultOnly>(23));
-    test<std::vector<DefaultOnly, test_allocator<DefaultOnly>>>(100, test_allocator<DefaultOnly>(23));
-    assert(DefaultOnly::count == 0);
+  test<std::vector<DefaultOnly, min_allocator<DefaultOnly>>>(0);
+  test<std::vector<DefaultOnly, min_allocator<DefaultOnly>>>(500);
+  test<std::vector<DefaultOnly, safe_allocator<DefaultOnly>>>(0);
+  test<std::vector<DefaultOnly, safe_allocator<DefaultOnly>>>(500);
+  test<std::vector<DefaultOnly, test_allocator<DefaultOnly>>>(0, test_allocator<DefaultOnly>(23));
+  test<std::vector<DefaultOnly, test_allocator<DefaultOnly>>>(100, test_allocator<DefaultOnly>(23));
+  assert(DefaultOnly::count == 0);
 #endif
 
-    return 0;
+  return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/vector/vector.cons/construct_size_value.pass.cpp b/libcxx/test/std/containers/sequences/vector/vector.cons/construct_size_value.pass.cpp
index 0f24e77cd47e0..de2df027dd530 100644
--- a/libcxx/test/std/containers/sequences/vector/vector.cons/construct_size_value.pass.cpp
+++ b/libcxx/test/std/containers/sequences/vector/vector.cons/construct_size_value.pass.cpp
@@ -19,39 +19,36 @@
 #include "asan_testing.h"
 
 template <class C>
-TEST_CONSTEXPR_CXX20 void
-test(typename C::size_type n, const typename C::value_type& x)
-{
-    C c(n, x);
-    LIBCPP_ASSERT(c.__invariants());
-    assert(c.size() == n);
-    LIBCPP_ASSERT(is_contiguous_container_asan_correct(c));
-    for (typename C::const_iterator i = c.cbegin(), e = c.cend(); i != e; ++i)
-        assert(*i == x);
+TEST_CONSTEXPR_CXX20 void test(typename C::size_type n, const typename C::value_type& x) {
+  C c(n, x);
+  LIBCPP_ASSERT(c.__invariants());
+  assert(c.size() == n);
+  LIBCPP_ASSERT(is_contiguous_container_asan_correct(c));
+  for (typename C::const_iterator i = c.cbegin(), e = c.cend(); i != e; ++i)
+    assert(*i == x);
 }
 
 TEST_CONSTEXPR_CXX20 bool tests() {
-    test<std::vector<int> >(0, 3);
-    test<std::vector<int> >(50, 3);
-    // Add 1 for implementations that dynamically allocate a container proxy.
-    test<std::vector<int, limited_allocator<int, 50 + 1> > >(0, 5);
-    test<std::vector<int, limited_allocator<int, 50 + 1> > >(50, 5);
-    test<std::vector<int, limited_allocator<int, 50 + 1> > >(50, 5);
+  test<std::vector<int> >(0, 3);
+  test<std::vector<int> >(50, 3);
+  // Add 1 for implementations that dynamically allocate a container proxy.
+  test<std::vector<int, limited_allocator<int, 50 + 1> > >(0, 5);
+  test<std::vector<int, limited_allocator<int, 50 + 1> > >(50, 5);
+  test<std::vector<int, limited_allocator<int, 50 + 1> > >(50, 5);
 #if TEST_STD_VER >= 11
-    test<std::vector<int, min_allocator<int>> >(0, 3);
-    test<std::vector<int, min_allocator<int>> >(50, 3);
-    test<std::vector<int, safe_allocator<int>> >(0, 3);
-    test<std::vector<int, safe_allocator<int>> >(50, 3);
+  test<std::vector<int, min_allocator<int>> >(0, 3);
+  test<std::vector<int, min_allocator<int>> >(50, 3);
+  test<std::vector<int, safe_allocator<int>> >(0, 3);
+  test<std::vector<int, safe_allocator<int>> >(50, 3);
 #endif
 
-    return true;
+  return true;
 }
 
-int main(int, char**)
-{
-    tests();
+int main(int, char**) {
+  tests();
 #if TEST_STD_VER > 17
-    static_assert(tests());
+  static_assert(tests());
 #endif
-    return 0;
+  return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/vector/vector.cons/construct_size_value_alloc.pass.cpp b/libcxx/test/std/containers/sequences/vector/vector.cons/construct_size_value_alloc.pass.cpp
index eb34de8bc9b31..c6d66d4983170 100644
--- a/libcxx/test/std/containers/sequences/vector/vector.cons/construct_size_value_alloc.pass.cpp
+++ b/libcxx/test/std/containers/sequences/vector/vector.cons/construct_size_value_alloc.pass.cpp
@@ -19,36 +19,33 @@
 
 template <class C>
 TEST_CONSTEXPR_CXX20 void
-test(typename C::size_type n, const typename C::value_type& x,
-     const typename C::allocator_type& a)
-{
-    C c(n, x, a);
-    LIBCPP_ASSERT(c.__invariants());
-    assert(a == c.get_allocator());
-    assert(c.size() == n);
-    LIBCPP_ASSERT(is_contiguous_container_asan_correct(c));
-    for (typename C::const_iterator i = c.cbegin(), e = c.cend(); i != e; ++i)
-        assert(*i == x);
+test(typename C::size_type n, const typename C::value_type& x, const typename C::allocator_type& a) {
+  C c(n, x, a);
+  LIBCPP_ASSERT(c.__invariants());
+  assert(a == c.get_allocator());
+  assert(c.size() == n);
+  LIBCPP_ASSERT(is_contiguous_container_asan_correct(c));
+  for (typename C::const_iterator i = c.cbegin(), e = c.cend(); i != e; ++i)
+    assert(*i == x);
 }
 
 TEST_CONSTEXPR_CXX20 bool tests() {
-    test<std::vector<int> >(0, 3, std::allocator<int>());
-    test<std::vector<int> >(50, 3, std::allocator<int>());
+  test<std::vector<int> >(0, 3, std::allocator<int>());
+  test<std::vector<int> >(50, 3, std::allocator<int>());
 #if TEST_STD_VER >= 11
-    test<std::vector<int, min_allocator<int>> >(0, 3, min_allocator<int>());
-    test<std::vector<int, min_allocator<int>> >(50, 3, min_allocator<int>());
-    test<std::vector<int, safe_allocator<int>> >(0, 3, safe_allocator<int>());
-    test<std::vector<int, safe_allocator<int>> >(50, 3, safe_allocator<int>());
+  test<std::vector<int, min_allocator<int>> >(0, 3, min_allocator<int>());
+  test<std::vector<int, min_allocator<int>> >(50, 3, min_allocator<int>());
+  test<std::vector<int, safe_allocator<int>> >(0, 3, safe_allocator<int>());
+  test<std::vector<int, safe_allocator<int>> >(50, 3, safe_allocator<int>());
 #endif
 
-    return true;
+  return true;
 }
 
-int main(int, char**)
-{
-    tests();
+int main(int, char**) {
+  tests();
 #if TEST_STD_VER > 17
-    static_assert(tests());
+  static_assert(tests());
 #endif
-    return 0;
+  return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/vector/vector.cons/copy.move_only.verify.cpp b/libcxx/test/std/containers/sequences/vector/vector.cons/copy.move_only.verify.cpp
index af8ca01370f08..ce3173d4dd062 100644
--- a/libcxx/test/std/containers/sequences/vector/vector.cons/copy.move_only.verify.cpp
+++ b/libcxx/test/std/containers/sequences/vector/vector.cons/copy.move_only.verify.cpp
@@ -15,6 +15,7 @@
 #include "MoveOnly.h"
 
 void f() {
-    std::vector<MoveOnly> v;
-    std::vector<MoveOnly> copy = v; // expected-error-re@* {{{{(no matching function for call to '__construct_at')|(call to deleted constructor of 'MoveOnly')}}}}
+  std::vector<MoveOnly> v;
+  std::vector<MoveOnly> copy =
+      v; // expected-error-re@* {{{{(no matching function for call to '__construct_at')|(call to deleted constructor of 'MoveOnly')}}}}
 }
diff --git a/libcxx/test/std/containers/sequences/vector/vector.cons/copy.pass.cpp b/libcxx/test/std/containers/sequences/vector/vector.cons/copy.pass.cpp
index b0e626381454e..ce5b42575b46f 100644
--- a/libcxx/test/std/containers/sequences/vector/vector.cons/copy.pass.cpp
+++ b/libcxx/test/std/containers/sequences/vector/vector.cons/copy.pass.cpp
@@ -19,101 +19,98 @@
 #include "asan_testing.h"
 
 template <class C>
-TEST_CONSTEXPR_CXX20 void
-test(const C& x)
-{
-    typename C::size_type s = x.size();
-    C c(x);
-    LIBCPP_ASSERT(c.__invariants());
-    assert(c.size() == s);
-    assert(c == x);
-    LIBCPP_ASSERT(is_contiguous_container_asan_correct(c));
+TEST_CONSTEXPR_CXX20 void test(const C& x) {
+  typename C::size_type s = x.size();
+  C c(x);
+  LIBCPP_ASSERT(c.__invariants());
+  assert(c.size() == s);
+  assert(c == x);
+  LIBCPP_ASSERT(is_contiguous_container_asan_correct(c));
 }
 
 TEST_CONSTEXPR_CXX20 bool tests() {
-    {
-        int a[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 8, 7, 6, 5, 4, 3, 1, 0};
-        int* an = a + sizeof(a)/sizeof(a[0]);
-        test(std::vector<int>(a, an));
-    }
-    {
-        std::vector<int, test_allocator<int> > v(3, 2, test_allocator<int>(5));
-        std::vector<int, test_allocator<int> > v2 = v;
-        assert(is_contiguous_container_asan_correct(v));
-        assert(is_contiguous_container_asan_correct(v2));
-        assert(v2 == v);
-        assert(v2.get_allocator() == v.get_allocator());
-        assert(is_contiguous_container_asan_correct(v));
-        assert(is_contiguous_container_asan_correct(v2));
-    }
-    {
-        // Test copy ctor with empty source
-        std::vector<int, test_allocator<int> > v(test_allocator<int>(5));
-        std::vector<int, test_allocator<int> > v2 = v;
-        assert(is_contiguous_container_asan_correct(v));
-        assert(is_contiguous_container_asan_correct(v2));
-        assert(v2 == v);
-        assert(v2.get_allocator() == v.get_allocator());
-        assert(is_contiguous_container_asan_correct(v));
-        assert(is_contiguous_container_asan_correct(v2));
-        assert(v2.empty());
-    }
+  {
+    int a[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 8, 7, 6, 5, 4, 3, 1, 0};
+    int* an = a + sizeof(a) / sizeof(a[0]);
+    test(std::vector<int>(a, an));
+  }
+  {
+    std::vector<int, test_allocator<int> > v(3, 2, test_allocator<int>(5));
+    std::vector<int, test_allocator<int> > v2 = v;
+    assert(is_contiguous_container_asan_correct(v));
+    assert(is_contiguous_container_asan_correct(v2));
+    assert(v2 == v);
+    assert(v2.get_allocator() == v.get_allocator());
+    assert(is_contiguous_container_asan_correct(v));
+    assert(is_contiguous_container_asan_correct(v2));
+  }
+  {
+    // Test copy ctor with empty source
+    std::vector<int, test_allocator<int> > v(test_allocator<int>(5));
+    std::vector<int, test_allocator<int> > v2 = v;
+    assert(is_contiguous_container_asan_correct(v));
+    assert(is_contiguous_container_asan_correct(v2));
+    assert(v2 == v);
+    assert(v2.get_allocator() == v.get_allocator());
+    assert(is_contiguous_container_asan_correct(v));
+    assert(is_contiguous_container_asan_correct(v2));
+    assert(v2.empty());
+  }
 #if TEST_STD_VER >= 11
-    {
-        std::vector<int, other_allocator<int> > v(3, 2, other_allocator<int>(5));
-        std::vector<int, other_allocator<int> > v2 = v;
-        assert(is_contiguous_container_asan_correct(v));
-        assert(is_contiguous_container_asan_correct(v2));
-        assert(v2 == v);
-        assert(v2.get_allocator() == other_allocator<int>(-2));
-        assert(is_contiguous_container_asan_correct(v));
-        assert(is_contiguous_container_asan_correct(v2));
-    }
-    {
-        int a[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 8, 7, 6, 5, 4, 3, 1, 0};
-        int* an = a + sizeof(a)/sizeof(a[0]);
-        test(std::vector<int, min_allocator<int>>(a, an));
-        test(std::vector<int, safe_allocator<int>>(a, an));
-    }
-    {
-        std::vector<int, min_allocator<int> > v(3, 2, min_allocator<int>());
-        std::vector<int, min_allocator<int> > v2 = v;
-        assert(is_contiguous_container_asan_correct(v));
-        assert(is_contiguous_container_asan_correct(v2));
-        assert(v2 == v);
-        assert(v2.get_allocator() == v.get_allocator());
-        assert(is_contiguous_container_asan_correct(v));
-        assert(is_contiguous_container_asan_correct(v2));
-    }
-    {
-      std::vector<int, safe_allocator<int> > v(3, 2, safe_allocator<int>());
-      std::vector<int, safe_allocator<int> > v2 = v;
-      assert(is_contiguous_container_asan_correct(v));
-      assert(is_contiguous_container_asan_correct(v2));
-      assert(v2 == v);
-      assert(v2.get_allocator() == v.get_allocator());
-      assert(is_contiguous_container_asan_correct(v));
-      assert(is_contiguous_container_asan_correct(v2));
-    }
+  {
+    std::vector<int, other_allocator<int> > v(3, 2, other_allocator<int>(5));
+    std::vector<int, other_allocator<int> > v2 = v;
+    assert(is_contiguous_container_asan_correct(v));
+    assert(is_contiguous_container_asan_correct(v2));
+    assert(v2 == v);
+    assert(v2.get_allocator() == other_allocator<int>(-2));
+    assert(is_contiguous_container_asan_correct(v));
+    assert(is_contiguous_container_asan_correct(v2));
+  }
+  {
+    int a[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 8, 7, 6, 5, 4, 3, 1, 0};
+    int* an = a + sizeof(a) / sizeof(a[0]);
+    test(std::vector<int, min_allocator<int>>(a, an));
+    test(std::vector<int, safe_allocator<int>>(a, an));
+  }
+  {
+    std::vector<int, min_allocator<int> > v(3, 2, min_allocator<int>());
+    std::vector<int, min_allocator<int> > v2 = v;
+    assert(is_contiguous_container_asan_correct(v));
+    assert(is_contiguous_container_asan_correct(v2));
+    assert(v2 == v);
+    assert(v2.get_allocator() == v.get_allocator());
+    assert(is_contiguous_container_asan_correct(v));
+    assert(is_contiguous_container_asan_correct(v2));
+  }
+  {
+    std::vector<int, safe_allocator<int> > v(3, 2, safe_allocator<int>());
+    std::vector<int, safe_allocator<int> > v2 = v;
+    assert(is_contiguous_container_asan_correct(v));
+    assert(is_contiguous_container_asan_correct(v2));
+    assert(v2 == v);
+    assert(v2.get_allocator() == v.get_allocator());
+    assert(is_contiguous_container_asan_correct(v));
+    assert(is_contiguous_container_asan_correct(v2));
+  }
 #endif
 
-    return true;
+  return true;
 }
 
 void test_copy_from_volatile_src() {
-    volatile int src[] = {1, 2, 3};
-    std::vector<int> v(src, src + 3);
-    assert(v[0] == 1);
-    assert(v[1] == 2);
-    assert(v[2] == 3);
+  volatile int src[] = {1, 2, 3};
+  std::vector<int> v(src, src + 3);
+  assert(v[0] == 1);
+  assert(v[1] == 2);
+  assert(v[2] == 3);
 }
 
-int main(int, char**)
-{
-    tests();
+int main(int, char**) {
+  tests();
 #if TEST_STD_VER > 17
-    static_assert(tests());
+  static_assert(tests());
 #endif
-    test_copy_from_volatile_src();
-    return 0;
+  test_copy_from_volatile_src();
+  return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/vector/vector.cons/copy_alloc.pass.cpp b/libcxx/test/std/containers/sequences/vector/vector.cons/copy_alloc.pass.cpp
index 0c03c50aed2e8..2c598d5dc426f 100644
--- a/libcxx/test/std/containers/sequences/vector/vector.cons/copy_alloc.pass.cpp
+++ b/libcxx/test/std/containers/sequences/vector/vector.cons/copy_alloc.pass.cpp
@@ -19,72 +19,69 @@
 #include "asan_testing.h"
 
 template <class C>
-TEST_CONSTEXPR_CXX20 void
-test(const C& x, const typename C::allocator_type& a)
-{
-    typename C::size_type s = x.size();
-    C c(x, a);
-    LIBCPP_ASSERT(c.__invariants());
-    assert(c.size() == s);
-    assert(c == x);
-    LIBCPP_ASSERT(is_contiguous_container_asan_correct(c));
+TEST_CONSTEXPR_CXX20 void test(const C& x, const typename C::allocator_type& a) {
+  typename C::size_type s = x.size();
+  C c(x, a);
+  LIBCPP_ASSERT(c.__invariants());
+  assert(c.size() == s);
+  assert(c == x);
+  LIBCPP_ASSERT(is_contiguous_container_asan_correct(c));
 }
 
 TEST_CONSTEXPR_CXX20 bool tests() {
-    {
-        int a[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 8, 7, 6, 5, 4, 3, 1, 0};
-        int* an = a + sizeof(a)/sizeof(a[0]);
-        test(std::vector<int>(a, an), std::allocator<int>());
-    }
-    {
-        std::vector<int, test_allocator<int> > l(3, 2, test_allocator<int>(5));
-        std::vector<int, test_allocator<int> > l2(l, test_allocator<int>(3));
-        assert(l2 == l);
-        assert(l2.get_allocator() == test_allocator<int>(3));
-    }
-    {
-        std::vector<int, other_allocator<int> > l(3, 2, other_allocator<int>(5));
-        std::vector<int, other_allocator<int> > l2(l, other_allocator<int>(3));
-        assert(l2 == l);
-        assert(l2.get_allocator() == other_allocator<int>(3));
-    }
-    {
-        // Test copy ctor with allocator and empty source
-        std::vector<int, other_allocator<int> > l(other_allocator<int>(5));
-        std::vector<int, other_allocator<int> > l2(l, other_allocator<int>(3));
-        assert(l2 == l);
-        assert(l2.get_allocator() == other_allocator<int>(3));
-        assert(l2.empty());
-    }
+  {
+    int a[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 8, 7, 6, 5, 4, 3, 1, 0};
+    int* an = a + sizeof(a) / sizeof(a[0]);
+    test(std::vector<int>(a, an), std::allocator<int>());
+  }
+  {
+    std::vector<int, test_allocator<int> > l(3, 2, test_allocator<int>(5));
+    std::vector<int, test_allocator<int> > l2(l, test_allocator<int>(3));
+    assert(l2 == l);
+    assert(l2.get_allocator() == test_allocator<int>(3));
+  }
+  {
+    std::vector<int, other_allocator<int> > l(3, 2, other_allocator<int>(5));
+    std::vector<int, other_allocator<int> > l2(l, other_allocator<int>(3));
+    assert(l2 == l);
+    assert(l2.get_allocator() == other_allocator<int>(3));
+  }
+  {
+    // Test copy ctor with allocator and empty source
+    std::vector<int, other_allocator<int> > l(other_allocator<int>(5));
+    std::vector<int, other_allocator<int> > l2(l, other_allocator<int>(3));
+    assert(l2 == l);
+    assert(l2.get_allocator() == other_allocator<int>(3));
+    assert(l2.empty());
+  }
 #if TEST_STD_VER >= 11
-    {
-        int a[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 8, 7, 6, 5, 4, 3, 1, 0};
-        int* an = a + sizeof(a)/sizeof(a[0]);
-        test(std::vector<int, min_allocator<int>>(a, an), min_allocator<int>());
-        test(std::vector<int, safe_allocator<int>>(a, an), safe_allocator<int>());
-    }
-    {
-        std::vector<int, min_allocator<int> > l(3, 2, min_allocator<int>());
-        std::vector<int, min_allocator<int> > l2(l, min_allocator<int>());
-        assert(l2 == l);
-        assert(l2.get_allocator() == min_allocator<int>());
-    }
-    {
-      std::vector<int, safe_allocator<int> > l(3, 2, safe_allocator<int>());
-      std::vector<int, safe_allocator<int> > l2(l, safe_allocator<int>());
-      assert(l2 == l);
-      assert(l2.get_allocator() == safe_allocator<int>());
-    }
+  {
+    int a[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 8, 7, 6, 5, 4, 3, 1, 0};
+    int* an = a + sizeof(a) / sizeof(a[0]);
+    test(std::vector<int, min_allocator<int>>(a, an), min_allocator<int>());
+    test(std::vector<int, safe_allocator<int>>(a, an), safe_allocator<int>());
+  }
+  {
+    std::vector<int, min_allocator<int> > l(3, 2, min_allocator<int>());
+    std::vector<int, min_allocator<int> > l2(l, min_allocator<int>());
+    assert(l2 == l);
+    assert(l2.get_allocator() == min_allocator<int>());
+  }
+  {
+    std::vector<int, safe_allocator<int> > l(3, 2, safe_allocator<int>());
+    std::vector<int, safe_allocator<int> > l2(l, safe_allocator<int>());
+    assert(l2 == l);
+    assert(l2.get_allocator() == safe_allocator<int>());
+  }
 #endif
 
-    return true;
+  return true;
 }
 
-int main(int, char**)
-{
-    tests();
+int main(int, char**) {
+  tests();
 #if TEST_STD_VER > 17
-    static_assert(tests());
+  static_assert(tests());
 #endif
-    return 0;
+  return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/vector/vector.cons/deduct.pass.cpp b/libcxx/test/std/containers/sequences/vector/vector.cons/deduct.pass.cpp
index 322161d517502..2408369c368ae 100644
--- a/libcxx/test/std/containers/sequences/vector/vector.cons/deduct.pass.cpp
+++ b/libcxx/test/std/containers/sequences/vector/vector.cons/deduct.pass.cpp
@@ -34,140 +34,139 @@
 struct A {};
 
 TEST_CONSTEXPR_CXX20 bool tests() {
-
-//  Test the explicit deduction guides
-    {
-    const int arr[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9 };
+  //  Test the explicit deduction guides
+  {
+    const int arr[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9};
     std::vector vec(std::begin(arr), std::end(arr));
 
     static_assert(std::is_same_v<decltype(vec), std::vector<int>>, "");
     assert(std::equal(vec.begin(), vec.end(), std::begin(arr), std::end(arr)));
-    }
+  }
 
-    {
-    const long arr[] = {INT_MAX, 1L, 2L, 3L };
+  {
+    const long arr[] = {INT_MAX, 1L, 2L, 3L};
     std::vector vec(std::begin(arr), std::end(arr), std::allocator<long>());
     static_assert(std::is_same_v<decltype(vec)::value_type, long>, "");
     assert(vec.size() == 4);
     assert(vec[0] == INT_MAX);
     assert(vec[1] == 1L);
     assert(vec[2] == 2L);
-    }
+  }
 
-//  Test the implicit deduction guides
+  //  Test the implicit deduction guides
 
-    {
-//  We don't expect this one to work.
-//  std::vector vec(std::allocator<int>()); // vector (allocator &)
-    }
+  {
+    //  We don't expect this one to work.
+    //  std::vector vec(std::allocator<int>()); // vector (allocator &)
+  }
 
-    {
+  {
     std::vector vec(1, A{}); // vector (size_type, T)
     static_assert(std::is_same_v<decltype(vec)::value_type, A>, "");
     static_assert(std::is_same_v<decltype(vec)::allocator_type, std::allocator<A>>, "");
     assert(vec.size() == 1);
-    }
+  }
 
-    {
+  {
     std::vector vec(1, A{}, test_allocator<A>()); // vector (size_type, T, allocator)
     static_assert(std::is_same_v<decltype(vec)::value_type, A>, "");
     static_assert(std::is_same_v<decltype(vec)::allocator_type, test_allocator<A>>, "");
     assert(vec.size() == 1);
-    }
+  }
 
-    {
+  {
     std::vector vec{1U, 2U, 3U, 4U, 5U}; // vector(initializer-list)
     static_assert(std::is_same_v<decltype(vec)::value_type, unsigned>, "");
     assert(vec.size() == 5);
     assert(vec[2] == 3U);
-    }
+  }
 
-    {
+  {
     std::vector vec({1.0, 2.0, 3.0, 4.0}, test_allocator<double>()); // vector(initializer-list, allocator)
     static_assert(std::is_same_v<decltype(vec)::value_type, double>, "");
     static_assert(std::is_same_v<decltype(vec)::allocator_type, test_allocator<double>>, "");
     assert(vec.size() == 4);
     assert(vec[3] == 4.0);
-    }
+  }
 
-    {
+  {
     std::vector<long double> source;
     std::vector vec(source); // vector(vector &)
     static_assert(std::is_same_v<decltype(vec)::value_type, long double>, "");
     static_assert(std::is_same_v<decltype(vec)::allocator_type, std::allocator<long double>>, "");
     assert(vec.size() == 0);
-    }
+  }
 
 #if TEST_STD_VER >= 23
+  {
     {
-      {
-        std::vector c(std::from_range, std::array<int, 0>());
-        static_assert(std::is_same_v<decltype(c), std::vector<int>>);
-      }
-
-      {
-        using Alloc = test_allocator<int>;
-        std::vector c(std::from_range, std::array<int, 0>(), Alloc());
-        static_assert(std::is_same_v<decltype(c), std::vector<int, Alloc>>);
-      }
+      std::vector c(std::from_range, std::array<int, 0>());
+      static_assert(std::is_same_v<decltype(c), std::vector<int>>);
     }
-#endif
 
-//  A couple of vector<bool> tests, too!
     {
+      using Alloc = test_allocator<int>;
+      std::vector c(std::from_range, std::array<int, 0>(), Alloc());
+      static_assert(std::is_same_v<decltype(c), std::vector<int, Alloc>>);
+    }
+  }
+#endif
+
+  //  A couple of vector<bool> tests, too!
+  {
     std::vector vec(3, true); // vector(initializer-list)
     static_assert(std::is_same_v<decltype(vec)::value_type, bool>, "");
     static_assert(std::is_same_v<decltype(vec)::allocator_type, std::allocator<bool>>, "");
     assert(vec.size() == 3);
     assert(vec[0] && vec[1] && vec[2]);
-    }
+  }
 
-    {
+  {
     std::vector<bool> source;
     std::vector vec(source); // vector(vector &)
     static_assert(std::is_same_v<decltype(vec)::value_type, bool>, "");
     static_assert(std::is_same_v<decltype(vec)::allocator_type, std::allocator<bool>>, "");
     assert(vec.size() == 0);
+  }
+
+  {
+    typedef test_allocator<short> Alloc;
+    typedef test_allocator<int> ConvertibleToAlloc;
+
+    {
+      std::vector<short, Alloc> source;
+      std::vector vec(source, Alloc(2));
+      static_assert(std::is_same_v<decltype(vec), decltype(source)>);
+    }
+
+    {
+      std::vector<short, Alloc> source;
+      std::vector vec(source, ConvertibleToAlloc(2));
+      static_assert(std::is_same_v<decltype(vec), decltype(source)>);
+    }
+
+    {
+      std::vector<short, Alloc> source;
+      std::vector vec(std::move(source), Alloc(2));
+      static_assert(std::is_same_v<decltype(vec), decltype(source)>);
     }
 
     {
-        typedef test_allocator<short> Alloc;
-        typedef test_allocator<int> ConvertibleToAlloc;
-
-        {
-        std::vector<short, Alloc> source;
-        std::vector vec(source, Alloc(2));
-        static_assert(std::is_same_v<decltype(vec), decltype(source)>);
-        }
-
-        {
-        std::vector<short, Alloc> source;
-        std::vector vec(source, ConvertibleToAlloc(2));
-        static_assert(std::is_same_v<decltype(vec), decltype(source)>);
-        }
-
-        {
-        std::vector<short, Alloc> source;
-        std::vector vec(std::move(source), Alloc(2));
-        static_assert(std::is_same_v<decltype(vec), decltype(source)>);
-        }
-
-        {
-        std::vector<short, Alloc> source;
-        std::vector vec(std::move(source), ConvertibleToAlloc(2));
-        static_assert(std::is_same_v<decltype(vec), decltype(source)>);
-        }
+      std::vector<short, Alloc> source;
+      std::vector vec(std::move(source), ConvertibleToAlloc(2));
+      static_assert(std::is_same_v<decltype(vec), decltype(source)>);
     }
+  }
 
-    SequenceContainerDeductionGuidesSfinaeAway<std::vector, std::vector<int>>();
+  SequenceContainerDeductionGuidesSfinaeAway<std::vector, std::vector<int>>();
 
-    return true;
+  return true;
 }
 
 int main(int, char**) {
-    tests();
+  tests();
 #if TEST_STD_VER > 17
-    static_assert(tests());
+  static_assert(tests());
 #endif
-    return 0;
+  return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/vector/vector.cons/default.recursive.pass.cpp b/libcxx/test/std/containers/sequences/vector/vector.cons/default.recursive.pass.cpp
index 310f9ccfc9c6c..0dd33525978c1 100644
--- a/libcxx/test/std/containers/sequences/vector/vector.cons/default.recursive.pass.cpp
+++ b/libcxx/test/std/containers/sequences/vector/vector.cons/default.recursive.pass.cpp
@@ -14,13 +14,8 @@
 
 #include "test_macros.h"
 
-struct X
-{
-    std::vector<X> q;
+struct X {
+  std::vector<X> q;
 };
 
-int main(int, char**)
-{
-
-    return 0;
-}
+int main(int, char**) { return 0; }
diff --git a/libcxx/test/std/containers/sequences/vector/vector.cons/default_noexcept.pass.cpp b/libcxx/test/std/containers/sequences/vector/vector.cons/default_noexcept.pass.cpp
index 054ea6430f82a..c22776edcd6d1 100644
--- a/libcxx/test/std/containers/sequences/vector/vector.cons/default_noexcept.pass.cpp
+++ b/libcxx/test/std/containers/sequences/vector/vector.cons/default_noexcept.pass.cpp
@@ -14,7 +14,6 @@
 
 // This *was* a conforming extension, but it was adopted in N4258.
 
-
 #include <vector>
 #include <cassert>
 
@@ -23,39 +22,37 @@
 #include "test_allocator.h"
 
 template <class T>
-struct some_alloc
-{
-    typedef T value_type;
-    some_alloc(const some_alloc&);
-    void allocate(std::size_t);
+struct some_alloc {
+  typedef T value_type;
+  some_alloc(const some_alloc&);
+  void allocate(std::size_t);
 };
 
 TEST_CONSTEXPR_CXX20 bool tests() {
-    {
-        typedef std::vector<MoveOnly> C;
-        static_assert(std::is_nothrow_default_constructible<C>::value, "");
-    }
-    {
-        typedef std::vector<MoveOnly, test_allocator<MoveOnly>> C;
-        static_assert(std::is_nothrow_default_constructible<C>::value, "");
-    }
-    {
-        typedef std::vector<MoveOnly, other_allocator<MoveOnly>> C;
-        static_assert(!std::is_nothrow_default_constructible<C>::value, "");
-    }
-    {
-        typedef std::vector<MoveOnly, some_alloc<MoveOnly>> C;
-        static_assert(!std::is_nothrow_default_constructible<C>::value, "");
-    }
-
-    return true;
+  {
+    typedef std::vector<MoveOnly> C;
+    static_assert(std::is_nothrow_default_constructible<C>::value, "");
+  }
+  {
+    typedef std::vector<MoveOnly, test_allocator<MoveOnly>> C;
+    static_assert(std::is_nothrow_default_constructible<C>::value, "");
+  }
+  {
+    typedef std::vector<MoveOnly, other_allocator<MoveOnly>> C;
+    static_assert(!std::is_nothrow_default_constructible<C>::value, "");
+  }
+  {
+    typedef std::vector<MoveOnly, some_alloc<MoveOnly>> C;
+    static_assert(!std::is_nothrow_default_constructible<C>::value, "");
+  }
+
+  return true;
 }
 
-int main(int, char**)
-{
-    tests();
+int main(int, char**) {
+  tests();
 #if TEST_STD_VER > 17
-    static_assert(tests());
+  static_assert(tests());
 #endif
-    return 0;
+  return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/vector/vector.cons/dtor_noexcept.pass.cpp b/libcxx/test/std/containers/sequences/vector/vector.cons/dtor_noexcept.pass.cpp
index 1b98b7598b97b..4a7e2cf445676 100644
--- a/libcxx/test/std/containers/sequences/vector/vector.cons/dtor_noexcept.pass.cpp
+++ b/libcxx/test/std/containers/sequences/vector/vector.cons/dtor_noexcept.pass.cpp
@@ -20,43 +20,40 @@
 #include "test_allocator.h"
 
 template <class T>
-struct some_alloc
-{
-    typedef T value_type;
-    some_alloc(const some_alloc&);
-    ~some_alloc() noexcept(false);
-    void allocate(std::size_t);
+struct some_alloc {
+  typedef T value_type;
+  some_alloc(const some_alloc&);
+  ~some_alloc() noexcept(false);
+  void allocate(std::size_t);
 };
 
-TEST_CONSTEXPR_CXX20 bool tests()
-{
-    {
-        typedef std::vector<MoveOnly> C;
-        static_assert(std::is_nothrow_destructible<C>::value, "");
-    }
-    {
-        typedef std::vector<MoveOnly, test_allocator<MoveOnly>> C;
-        static_assert(std::is_nothrow_destructible<C>::value, "");
-    }
-    {
-        typedef std::vector<MoveOnly, other_allocator<MoveOnly>> C;
-        static_assert(std::is_nothrow_destructible<C>::value, "");
-    }
+TEST_CONSTEXPR_CXX20 bool tests() {
+  {
+    typedef std::vector<MoveOnly> C;
+    static_assert(std::is_nothrow_destructible<C>::value, "");
+  }
+  {
+    typedef std::vector<MoveOnly, test_allocator<MoveOnly>> C;
+    static_assert(std::is_nothrow_destructible<C>::value, "");
+  }
+  {
+    typedef std::vector<MoveOnly, other_allocator<MoveOnly>> C;
+    static_assert(std::is_nothrow_destructible<C>::value, "");
+  }
 #if defined(_LIBCPP_VERSION)
-    {
-        typedef std::vector<MoveOnly, some_alloc<MoveOnly>> C;
-        static_assert(!std::is_nothrow_destructible<C>::value, "");
-    }
+  {
+    typedef std::vector<MoveOnly, some_alloc<MoveOnly>> C;
+    static_assert(!std::is_nothrow_destructible<C>::value, "");
+  }
 #endif // _LIBCPP_VERSION
 
-    return true;
+  return true;
 }
 
-int main(int, char**)
-{
-    tests();
+int main(int, char**) {
+  tests();
 #if TEST_STD_VER > 17
-    static_assert(tests());
+  static_assert(tests());
 #endif
-    return 0;
+  return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/vector/vector.cons/initializer_list.pass.cpp b/libcxx/test/std/containers/sequences/vector/vector.cons/initializer_list.pass.cpp
index af2f3e40a91b5..02bb7f3a3ea9c 100644
--- a/libcxx/test/std/containers/sequences/vector/vector.cons/initializer_list.pass.cpp
+++ b/libcxx/test/std/containers/sequences/vector/vector.cons/initializer_list.pass.cpp
@@ -18,9 +18,8 @@
 #include "min_allocator.h"
 #include "asan_testing.h"
 
-TEST_CONSTEXPR_CXX20 bool tests()
-{
-    {
+TEST_CONSTEXPR_CXX20 bool tests() {
+  {
     std::vector<int> d = {3, 4, 5, 6};
     assert(d.size() == 4);
     assert(is_contiguous_container_asan_correct(d));
@@ -28,8 +27,8 @@ TEST_CONSTEXPR_CXX20 bool tests()
     assert(d[1] == 4);
     assert(d[2] == 5);
     assert(d[3] == 6);
-    }
-    {
+  }
+  {
     std::vector<int, min_allocator<int>> d = {3, 4, 5, 6};
     assert(d.size() == 4);
     assert(is_contiguous_container_asan_correct(d));
@@ -37,25 +36,24 @@ TEST_CONSTEXPR_CXX20 bool tests()
     assert(d[1] == 4);
     assert(d[2] == 5);
     assert(d[3] == 6);
-    }
-    {
-      std::vector<int, safe_allocator<int>> d = {3, 4, 5, 6};
-      assert(d.size() == 4);
-      assert(is_contiguous_container_asan_correct(d));
-      assert(d[0] == 3);
-      assert(d[1] == 4);
-      assert(d[2] == 5);
-      assert(d[3] == 6);
-    }
+  }
+  {
+    std::vector<int, safe_allocator<int>> d = {3, 4, 5, 6};
+    assert(d.size() == 4);
+    assert(is_contiguous_container_asan_correct(d));
+    assert(d[0] == 3);
+    assert(d[1] == 4);
+    assert(d[2] == 5);
+    assert(d[3] == 6);
+  }
 
-    return true;
+  return true;
 }
 
-int main(int, char**)
-{
-    tests();
+int main(int, char**) {
+  tests();
 #if TEST_STD_VER > 17
-    static_assert(tests());
+  static_assert(tests());
 #endif
-    return 0;
+  return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/vector/vector.cons/initializer_list_alloc.pass.cpp b/libcxx/test/std/containers/sequences/vector/vector.cons/initializer_list_alloc.pass.cpp
index 8396e84681620..bfc9b5a03a617 100644
--- a/libcxx/test/std/containers/sequences/vector/vector.cons/initializer_list_alloc.pass.cpp
+++ b/libcxx/test/std/containers/sequences/vector/vector.cons/initializer_list_alloc.pass.cpp
@@ -20,9 +20,8 @@
 #include "min_allocator.h"
 #include "asan_testing.h"
 
-TEST_CONSTEXPR_CXX20 bool tests()
-{
-    {
+TEST_CONSTEXPR_CXX20 bool tests() {
+  {
     std::vector<int, test_allocator<int>> d({3, 4, 5, 6}, test_allocator<int>(3));
     assert(d.get_allocator() == test_allocator<int>(3));
     assert(d.size() == 4);
@@ -31,8 +30,8 @@ TEST_CONSTEXPR_CXX20 bool tests()
     assert(d[1] == 4);
     assert(d[2] == 5);
     assert(d[3] == 6);
-    }
-    {
+  }
+  {
     std::vector<int, min_allocator<int>> d({3, 4, 5, 6}, min_allocator<int>());
     assert(d.get_allocator() == min_allocator<int>());
     assert(d.size() == 4);
@@ -41,38 +40,37 @@ TEST_CONSTEXPR_CXX20 bool tests()
     assert(d[1] == 4);
     assert(d[2] == 5);
     assert(d[3] == 6);
-    }
-    {
+  }
+  {
     std::vector<int, min_allocator<int>> d({}, min_allocator<int>());
     assert(d.size() == 0);
     assert(d.empty());
     assert(is_contiguous_container_asan_correct(d));
-    }
-    {
-      std::vector<int, safe_allocator<int>> d({3, 4, 5, 6}, safe_allocator<int>());
-      assert(d.get_allocator() == safe_allocator<int>());
-      assert(d.size() == 4);
-      assert(is_contiguous_container_asan_correct(d));
-      assert(d[0] == 3);
-      assert(d[1] == 4);
-      assert(d[2] == 5);
-      assert(d[3] == 6);
-    }
-    {
-      std::vector<int, safe_allocator<int>> d({}, safe_allocator<int>());
-      assert(d.size() == 0);
-      assert(d.empty());
-      assert(is_contiguous_container_asan_correct(d));
-    }
+  }
+  {
+    std::vector<int, safe_allocator<int>> d({3, 4, 5, 6}, safe_allocator<int>());
+    assert(d.get_allocator() == safe_allocator<int>());
+    assert(d.size() == 4);
+    assert(is_contiguous_container_asan_correct(d));
+    assert(d[0] == 3);
+    assert(d[1] == 4);
+    assert(d[2] == 5);
+    assert(d[3] == 6);
+  }
+  {
+    std::vector<int, safe_allocator<int>> d({}, safe_allocator<int>());
+    assert(d.size() == 0);
+    assert(d.empty());
+    assert(is_contiguous_container_asan_correct(d));
+  }
 
-    return true;
+  return true;
 }
 
-int main(int, char**)
-{
-    tests();
+int main(int, char**) {
+  tests();
 #if TEST_STD_VER > 17
-    static_assert(tests());
+  static_assert(tests());
 #endif
-    return 0;
+  return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/vector/vector.cons/move.pass.cpp b/libcxx/test/std/containers/sequences/vector/vector.cons/move.pass.cpp
index c8b8701cc491b..6e83696b45bac 100644
--- a/libcxx/test/std/containers/sequences/vector/vector.cons/move.pass.cpp
+++ b/libcxx/test/std/containers/sequences/vector/vector.cons/move.pass.cpp
@@ -21,153 +21,148 @@
 #include "min_allocator.h"
 #include "asan_testing.h"
 
-TEST_CONSTEXPR_CXX20 bool tests()
-{
-    test_allocator_statistics alloc_stats;
-    {
-        std::vector<MoveOnly, test_allocator<MoveOnly> > l(test_allocator<MoveOnly>(5, &alloc_stats));
-        std::vector<MoveOnly, test_allocator<MoveOnly> > lo(test_allocator<MoveOnly>(5, &alloc_stats));
-        assert(is_contiguous_container_asan_correct(l));
-        assert(is_contiguous_container_asan_correct(lo));
-        for (int i = 1; i <= 3; ++i)
-        {
-            l.push_back(i);
-            lo.push_back(i);
-        }
-        assert(is_contiguous_container_asan_correct(l));
-        assert(is_contiguous_container_asan_correct(lo));
-        std::vector<MoveOnly, test_allocator<MoveOnly> > l2 = std::move(l);
-        assert(l2 == lo);
-        assert(l.empty());
-        assert(l2.get_allocator() == lo.get_allocator());
-        assert(is_contiguous_container_asan_correct(l2));
-    }
-    {
-        std::vector<MoveOnly, other_allocator<MoveOnly> > l(other_allocator<MoveOnly>(5));
-        std::vector<MoveOnly, other_allocator<MoveOnly> > lo(other_allocator<MoveOnly>(5));
-        assert(is_contiguous_container_asan_correct(l));
-        assert(is_contiguous_container_asan_correct(lo));
-        for (int i = 1; i <= 3; ++i)
-        {
-            l.push_back(i);
-            lo.push_back(i);
-        }
-        assert(is_contiguous_container_asan_correct(l));
-        assert(is_contiguous_container_asan_correct(lo));
-        std::vector<MoveOnly, other_allocator<MoveOnly> > l2 = std::move(l);
-        assert(l2 == lo);
-        assert(l.empty());
-        assert(l2.get_allocator() == lo.get_allocator());
-        assert(is_contiguous_container_asan_correct(l2));
-    }
-    {
-        int a1[] = {1, 3, 7, 9, 10};
-        std::vector<int> c1(a1, a1+sizeof(a1)/sizeof(a1[0]));
-        assert(is_contiguous_container_asan_correct(c1));
-        std::vector<int>::const_iterator i = c1.begin();
-        std::vector<int> c2 = std::move(c1);
-        assert(is_contiguous_container_asan_correct(c2));
-        std::vector<int>::iterator j = c2.erase(i);
-        assert(*j == 3);
-        assert(is_contiguous_container_asan_correct(c2));
+TEST_CONSTEXPR_CXX20 bool tests() {
+  test_allocator_statistics alloc_stats;
+  {
+    std::vector<MoveOnly, test_allocator<MoveOnly> > l(test_allocator<MoveOnly>(5, &alloc_stats));
+    std::vector<MoveOnly, test_allocator<MoveOnly> > lo(test_allocator<MoveOnly>(5, &alloc_stats));
+    assert(is_contiguous_container_asan_correct(l));
+    assert(is_contiguous_container_asan_correct(lo));
+    for (int i = 1; i <= 3; ++i) {
+      l.push_back(i);
+      lo.push_back(i);
     }
-    {
-        std::vector<MoveOnly, min_allocator<MoveOnly> > l((min_allocator<MoveOnly>()));
-        std::vector<MoveOnly, min_allocator<MoveOnly> > lo((min_allocator<MoveOnly>()));
-        assert(is_contiguous_container_asan_correct(l));
-        assert(is_contiguous_container_asan_correct(lo));
-        for (int i = 1; i <= 3; ++i)
-        {
-            l.push_back(i);
-            lo.push_back(i);
-        }
-        assert(is_contiguous_container_asan_correct(l));
-        assert(is_contiguous_container_asan_correct(lo));
-        std::vector<MoveOnly, min_allocator<MoveOnly> > l2 = std::move(l);
-        assert(l2 == lo);
-        assert(l.empty());
-        assert(l2.get_allocator() == lo.get_allocator());
-        assert(is_contiguous_container_asan_correct(l2));
+    assert(is_contiguous_container_asan_correct(l));
+    assert(is_contiguous_container_asan_correct(lo));
+    std::vector<MoveOnly, test_allocator<MoveOnly> > l2 = std::move(l);
+    assert(l2 == lo);
+    assert(l.empty());
+    assert(l2.get_allocator() == lo.get_allocator());
+    assert(is_contiguous_container_asan_correct(l2));
+  }
+  {
+    std::vector<MoveOnly, other_allocator<MoveOnly> > l(other_allocator<MoveOnly>(5));
+    std::vector<MoveOnly, other_allocator<MoveOnly> > lo(other_allocator<MoveOnly>(5));
+    assert(is_contiguous_container_asan_correct(l));
+    assert(is_contiguous_container_asan_correct(lo));
+    for (int i = 1; i <= 3; ++i) {
+      l.push_back(i);
+      lo.push_back(i);
     }
-    {
-        int a1[] = {1, 3, 7, 9, 10};
-        std::vector<int, min_allocator<int> > c1(a1, a1+sizeof(a1)/sizeof(a1[0]));
-        assert(is_contiguous_container_asan_correct(c1));
-        std::vector<int, min_allocator<int> >::const_iterator i = c1.begin();
-        std::vector<int, min_allocator<int> > c2 = std::move(c1);
-        assert(is_contiguous_container_asan_correct(c2));
-        std::vector<int, min_allocator<int> >::iterator j = c2.erase(i);
-        assert(*j == 3);
-        assert(is_contiguous_container_asan_correct(c2));
+    assert(is_contiguous_container_asan_correct(l));
+    assert(is_contiguous_container_asan_correct(lo));
+    std::vector<MoveOnly, other_allocator<MoveOnly> > l2 = std::move(l);
+    assert(l2 == lo);
+    assert(l.empty());
+    assert(l2.get_allocator() == lo.get_allocator());
+    assert(is_contiguous_container_asan_correct(l2));
+  }
+  {
+    int a1[] = {1, 3, 7, 9, 10};
+    std::vector<int> c1(a1, a1 + sizeof(a1) / sizeof(a1[0]));
+    assert(is_contiguous_container_asan_correct(c1));
+    std::vector<int>::const_iterator i = c1.begin();
+    std::vector<int> c2                = std::move(c1);
+    assert(is_contiguous_container_asan_correct(c2));
+    std::vector<int>::iterator j = c2.erase(i);
+    assert(*j == 3);
+    assert(is_contiguous_container_asan_correct(c2));
+  }
+  {
+    std::vector<MoveOnly, min_allocator<MoveOnly> > l((min_allocator<MoveOnly>()));
+    std::vector<MoveOnly, min_allocator<MoveOnly> > lo((min_allocator<MoveOnly>()));
+    assert(is_contiguous_container_asan_correct(l));
+    assert(is_contiguous_container_asan_correct(lo));
+    for (int i = 1; i <= 3; ++i) {
+      l.push_back(i);
+      lo.push_back(i);
     }
-    {
-      std::vector<MoveOnly, safe_allocator<MoveOnly> > l((safe_allocator<MoveOnly>()));
-      std::vector<MoveOnly, safe_allocator<MoveOnly> > lo((safe_allocator<MoveOnly>()));
-      assert(is_contiguous_container_asan_correct(l));
-      assert(is_contiguous_container_asan_correct(lo));
-      for (int i = 1; i <= 3; ++i) {
-        l.push_back(i);
-        lo.push_back(i);
-      }
-      assert(is_contiguous_container_asan_correct(l));
-      assert(is_contiguous_container_asan_correct(lo));
-      std::vector<MoveOnly, safe_allocator<MoveOnly> > l2 = std::move(l);
-      assert(l2 == lo);
-      assert(l.empty());
-      assert(l2.get_allocator() == lo.get_allocator());
-      assert(is_contiguous_container_asan_correct(l2));
+    assert(is_contiguous_container_asan_correct(l));
+    assert(is_contiguous_container_asan_correct(lo));
+    std::vector<MoveOnly, min_allocator<MoveOnly> > l2 = std::move(l);
+    assert(l2 == lo);
+    assert(l.empty());
+    assert(l2.get_allocator() == lo.get_allocator());
+    assert(is_contiguous_container_asan_correct(l2));
+  }
+  {
+    int a1[] = {1, 3, 7, 9, 10};
+    std::vector<int, min_allocator<int> > c1(a1, a1 + sizeof(a1) / sizeof(a1[0]));
+    assert(is_contiguous_container_asan_correct(c1));
+    std::vector<int, min_allocator<int> >::const_iterator i = c1.begin();
+    std::vector<int, min_allocator<int> > c2                = std::move(c1);
+    assert(is_contiguous_container_asan_correct(c2));
+    std::vector<int, min_allocator<int> >::iterator j = c2.erase(i);
+    assert(*j == 3);
+    assert(is_contiguous_container_asan_correct(c2));
+  }
+  {
+    std::vector<MoveOnly, safe_allocator<MoveOnly> > l((safe_allocator<MoveOnly>()));
+    std::vector<MoveOnly, safe_allocator<MoveOnly> > lo((safe_allocator<MoveOnly>()));
+    assert(is_contiguous_container_asan_correct(l));
+    assert(is_contiguous_container_asan_correct(lo));
+    for (int i = 1; i <= 3; ++i) {
+      l.push_back(i);
+      lo.push_back(i);
     }
+    assert(is_contiguous_container_asan_correct(l));
+    assert(is_contiguous_container_asan_correct(lo));
+    std::vector<MoveOnly, safe_allocator<MoveOnly> > l2 = std::move(l);
+    assert(l2 == lo);
+    assert(l.empty());
+    assert(l2.get_allocator() == lo.get_allocator());
+    assert(is_contiguous_container_asan_correct(l2));
+  }
+  {
+    int a1[] = {1, 3, 7, 9, 10};
+    std::vector<int, safe_allocator<int> > c1(a1, a1 + sizeof(a1) / sizeof(a1[0]));
+    assert(is_contiguous_container_asan_correct(c1));
+    std::vector<int, safe_allocator<int> >::const_iterator i = c1.begin();
+    std::vector<int, safe_allocator<int> > c2                = std::move(c1);
+    assert(is_contiguous_container_asan_correct(c2));
+    std::vector<int, safe_allocator<int> >::iterator j = c2.erase(i);
+    assert(*j == 3);
+    assert(is_contiguous_container_asan_correct(c2));
+  }
+  {
+    alloc_stats.clear();
+    using Vect = std::vector<int, test_allocator<int> >;
+    Vect v(test_allocator<int>(42, 101, &alloc_stats));
+    assert(alloc_stats.count == 1);
+    assert(alloc_stats.copied == 1);
+    assert(alloc_stats.moved == 0);
     {
-      int a1[] = {1, 3, 7, 9, 10};
-      std::vector<int, safe_allocator<int> > c1(a1, a1 + sizeof(a1) / sizeof(a1[0]));
-      assert(is_contiguous_container_asan_correct(c1));
-      std::vector<int, safe_allocator<int> >::const_iterator i = c1.begin();
-      std::vector<int, safe_allocator<int> > c2                = std::move(c1);
-      assert(is_contiguous_container_asan_correct(c2));
-      std::vector<int, safe_allocator<int> >::iterator j = c2.erase(i);
-      assert(*j == 3);
-      assert(is_contiguous_container_asan_correct(c2));
+      const test_allocator<int>& a = v.get_allocator();
+      assert(a.get_data() == 42);
+      assert(a.get_id() == 101);
     }
-    {
-      alloc_stats.clear();
-      using Vect = std::vector<int, test_allocator<int> >;
-      Vect v(test_allocator<int>(42, 101, &alloc_stats));
-      assert(alloc_stats.count == 1);
-      assert(alloc_stats.copied == 1);
-      assert(alloc_stats.moved == 0);
-      {
-        const test_allocator<int>& a = v.get_allocator();
-        assert(a.get_data() == 42);
-        assert(a.get_id() == 101);
-      }
-      assert(alloc_stats.count == 1);
-      alloc_stats.clear_ctor_counters();
+    assert(alloc_stats.count == 1);
+    alloc_stats.clear_ctor_counters();
 
-      Vect v2 = std::move(v);
-      assert(alloc_stats.count == 2);
-      assert(alloc_stats.copied == 0);
-      assert(alloc_stats.moved == 1);
-      {
-        const test_allocator<int>& a1 = v.get_allocator();
-        assert(a1.get_id() == test_alloc_base::moved_value);
-        assert(a1.get_data() == 42);
+    Vect v2 = std::move(v);
+    assert(alloc_stats.count == 2);
+    assert(alloc_stats.copied == 0);
+    assert(alloc_stats.moved == 1);
+    {
+      const test_allocator<int>& a1 = v.get_allocator();
+      assert(a1.get_id() == test_alloc_base::moved_value);
+      assert(a1.get_data() == 42);
 
-        const test_allocator<int>& a2 = v2.get_allocator();
-        assert(a2.get_id() == 101);
-        assert(a2.get_data() == 42);
+      const test_allocator<int>& a2 = v2.get_allocator();
+      assert(a2.get_id() == 101);
+      assert(a2.get_data() == 42);
 
-        assert(a1 == a2);
-      }
+      assert(a1 == a2);
     }
+  }
 
-    return true;
+  return true;
 }
 
-int main(int, char**)
-{
-    tests();
+int main(int, char**) {
+  tests();
 #if TEST_STD_VER > 17
-    static_assert(tests());
+  static_assert(tests());
 #endif
-    return 0;
+  return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/vector/vector.cons/move_alloc.pass.cpp b/libcxx/test/std/containers/sequences/vector/vector.cons/move_alloc.pass.cpp
index fd0b995b54d54..8066f122e41ac 100644
--- a/libcxx/test/std/containers/sequences/vector/vector.cons/move_alloc.pass.cpp
+++ b/libcxx/test/std/containers/sequences/vector/vector.cons/move_alloc.pass.cpp
@@ -20,106 +20,100 @@
 #include "min_allocator.h"
 #include "asan_testing.h"
 
-TEST_CONSTEXPR_CXX20 bool tests()
-{
-    {
-        std::vector<MoveOnly, test_allocator<MoveOnly> > l(test_allocator<MoveOnly>(5));
-        std::vector<MoveOnly, test_allocator<MoveOnly> > lo(test_allocator<MoveOnly>(5));
-        assert(is_contiguous_container_asan_correct(l));
-        assert(is_contiguous_container_asan_correct(lo));
-        for (int i = 1; i <= 3; ++i)
-        {
-            l.push_back(i);
-            lo.push_back(i);
-        }
-        assert(is_contiguous_container_asan_correct(l));
-        assert(is_contiguous_container_asan_correct(lo));
-        std::vector<MoveOnly, test_allocator<MoveOnly> > l2(std::move(l), test_allocator<MoveOnly>(6));
-        assert(l2 == lo);
-        assert(!l.empty());
-        assert(l2.get_allocator() == test_allocator<MoveOnly>(6));
-        assert(is_contiguous_container_asan_correct(l2));
+TEST_CONSTEXPR_CXX20 bool tests() {
+  {
+    std::vector<MoveOnly, test_allocator<MoveOnly> > l(test_allocator<MoveOnly>(5));
+    std::vector<MoveOnly, test_allocator<MoveOnly> > lo(test_allocator<MoveOnly>(5));
+    assert(is_contiguous_container_asan_correct(l));
+    assert(is_contiguous_container_asan_correct(lo));
+    for (int i = 1; i <= 3; ++i) {
+      l.push_back(i);
+      lo.push_back(i);
     }
-    {
-        std::vector<MoveOnly, test_allocator<MoveOnly> > l(test_allocator<MoveOnly>(5));
-        std::vector<MoveOnly, test_allocator<MoveOnly> > lo(test_allocator<MoveOnly>(5));
-        assert(is_contiguous_container_asan_correct(l));
-        assert(is_contiguous_container_asan_correct(lo));
-        for (int i = 1; i <= 3; ++i)
-        {
-            l.push_back(i);
-            lo.push_back(i);
-        }
-        assert(is_contiguous_container_asan_correct(l));
-        assert(is_contiguous_container_asan_correct(lo));
-        std::vector<MoveOnly, test_allocator<MoveOnly> > l2(std::move(l), test_allocator<MoveOnly>(5));
-        assert(l2 == lo);
-        assert(l.empty());
-        assert(l2.get_allocator() == test_allocator<MoveOnly>(5));
-        assert(is_contiguous_container_asan_correct(l2));
+    assert(is_contiguous_container_asan_correct(l));
+    assert(is_contiguous_container_asan_correct(lo));
+    std::vector<MoveOnly, test_allocator<MoveOnly> > l2(std::move(l), test_allocator<MoveOnly>(6));
+    assert(l2 == lo);
+    assert(!l.empty());
+    assert(l2.get_allocator() == test_allocator<MoveOnly>(6));
+    assert(is_contiguous_container_asan_correct(l2));
+  }
+  {
+    std::vector<MoveOnly, test_allocator<MoveOnly> > l(test_allocator<MoveOnly>(5));
+    std::vector<MoveOnly, test_allocator<MoveOnly> > lo(test_allocator<MoveOnly>(5));
+    assert(is_contiguous_container_asan_correct(l));
+    assert(is_contiguous_container_asan_correct(lo));
+    for (int i = 1; i <= 3; ++i) {
+      l.push_back(i);
+      lo.push_back(i);
     }
-    {
-        std::vector<MoveOnly, other_allocator<MoveOnly> > l(other_allocator<MoveOnly>(5));
-        std::vector<MoveOnly, other_allocator<MoveOnly> > lo(other_allocator<MoveOnly>(5));
-        assert(is_contiguous_container_asan_correct(l));
-        assert(is_contiguous_container_asan_correct(lo));
-        for (int i = 1; i <= 3; ++i)
-        {
-            l.push_back(i);
-            lo.push_back(i);
-        }
-        assert(is_contiguous_container_asan_correct(l));
-        assert(is_contiguous_container_asan_correct(lo));
-        std::vector<MoveOnly, other_allocator<MoveOnly> > l2(std::move(l), other_allocator<MoveOnly>(4));
-        assert(l2 == lo);
-        assert(!l.empty());
-        assert(l2.get_allocator() == other_allocator<MoveOnly>(4));
-        assert(is_contiguous_container_asan_correct(l2));
+    assert(is_contiguous_container_asan_correct(l));
+    assert(is_contiguous_container_asan_correct(lo));
+    std::vector<MoveOnly, test_allocator<MoveOnly> > l2(std::move(l), test_allocator<MoveOnly>(5));
+    assert(l2 == lo);
+    assert(l.empty());
+    assert(l2.get_allocator() == test_allocator<MoveOnly>(5));
+    assert(is_contiguous_container_asan_correct(l2));
+  }
+  {
+    std::vector<MoveOnly, other_allocator<MoveOnly> > l(other_allocator<MoveOnly>(5));
+    std::vector<MoveOnly, other_allocator<MoveOnly> > lo(other_allocator<MoveOnly>(5));
+    assert(is_contiguous_container_asan_correct(l));
+    assert(is_contiguous_container_asan_correct(lo));
+    for (int i = 1; i <= 3; ++i) {
+      l.push_back(i);
+      lo.push_back(i);
     }
-    {
-        std::vector<MoveOnly, min_allocator<MoveOnly> > l((min_allocator<MoveOnly>()));
-        std::vector<MoveOnly, min_allocator<MoveOnly> > lo((min_allocator<MoveOnly>()));
-        assert(is_contiguous_container_asan_correct(l));
-        assert(is_contiguous_container_asan_correct(lo));
-        for (int i = 1; i <= 3; ++i)
-        {
-            l.push_back(i);
-            lo.push_back(i);
-        }
-        assert(is_contiguous_container_asan_correct(l));
-        assert(is_contiguous_container_asan_correct(lo));
-        std::vector<MoveOnly, min_allocator<MoveOnly> > l2(std::move(l), min_allocator<MoveOnly>());
-        assert(l2 == lo);
-        assert(l.empty());
-        assert(l2.get_allocator() == min_allocator<MoveOnly>());
-        assert(is_contiguous_container_asan_correct(l2));
+    assert(is_contiguous_container_asan_correct(l));
+    assert(is_contiguous_container_asan_correct(lo));
+    std::vector<MoveOnly, other_allocator<MoveOnly> > l2(std::move(l), other_allocator<MoveOnly>(4));
+    assert(l2 == lo);
+    assert(!l.empty());
+    assert(l2.get_allocator() == other_allocator<MoveOnly>(4));
+    assert(is_contiguous_container_asan_correct(l2));
+  }
+  {
+    std::vector<MoveOnly, min_allocator<MoveOnly> > l((min_allocator<MoveOnly>()));
+    std::vector<MoveOnly, min_allocator<MoveOnly> > lo((min_allocator<MoveOnly>()));
+    assert(is_contiguous_container_asan_correct(l));
+    assert(is_contiguous_container_asan_correct(lo));
+    for (int i = 1; i <= 3; ++i) {
+      l.push_back(i);
+      lo.push_back(i);
     }
-    {
-      std::vector<MoveOnly, safe_allocator<MoveOnly> > l((safe_allocator<MoveOnly>()));
-      std::vector<MoveOnly, safe_allocator<MoveOnly> > lo((safe_allocator<MoveOnly>()));
-      assert(is_contiguous_container_asan_correct(l));
-      assert(is_contiguous_container_asan_correct(lo));
-      for (int i = 1; i <= 3; ++i) {
-        l.push_back(i);
-        lo.push_back(i);
-      }
-      assert(is_contiguous_container_asan_correct(l));
-      assert(is_contiguous_container_asan_correct(lo));
-      std::vector<MoveOnly, safe_allocator<MoveOnly> > l2(std::move(l), safe_allocator<MoveOnly>());
-      assert(l2 == lo);
-      assert(l.empty());
-      assert(l2.get_allocator() == safe_allocator<MoveOnly>());
-      assert(is_contiguous_container_asan_correct(l2));
+    assert(is_contiguous_container_asan_correct(l));
+    assert(is_contiguous_container_asan_correct(lo));
+    std::vector<MoveOnly, min_allocator<MoveOnly> > l2(std::move(l), min_allocator<MoveOnly>());
+    assert(l2 == lo);
+    assert(l.empty());
+    assert(l2.get_allocator() == min_allocator<MoveOnly>());
+    assert(is_contiguous_container_asan_correct(l2));
+  }
+  {
+    std::vector<MoveOnly, safe_allocator<MoveOnly> > l((safe_allocator<MoveOnly>()));
+    std::vector<MoveOnly, safe_allocator<MoveOnly> > lo((safe_allocator<MoveOnly>()));
+    assert(is_contiguous_container_asan_correct(l));
+    assert(is_contiguous_container_asan_correct(lo));
+    for (int i = 1; i <= 3; ++i) {
+      l.push_back(i);
+      lo.push_back(i);
     }
+    assert(is_contiguous_container_asan_correct(l));
+    assert(is_contiguous_container_asan_correct(lo));
+    std::vector<MoveOnly, safe_allocator<MoveOnly> > l2(std::move(l), safe_allocator<MoveOnly>());
+    assert(l2 == lo);
+    assert(l.empty());
+    assert(l2.get_allocator() == safe_allocator<MoveOnly>());
+    assert(is_contiguous_container_asan_correct(l2));
+  }
 
-    return true;
+  return true;
 }
 
-int main(int, char**)
-{
-    tests();
+int main(int, char**) {
+  tests();
 #if TEST_STD_VER > 17
-    static_assert(tests());
+  static_assert(tests());
 #endif
-    return 0;
+  return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/vector/vector.cons/move_assign_noexcept.pass.cpp b/libcxx/test/std/containers/sequences/vector/vector.cons/move_assign_noexcept.pass.cpp
index f466809bf2dd0..1719cbeba37db 100644
--- a/libcxx/test/std/containers/sequences/vector/vector.cons/move_assign_noexcept.pass.cpp
+++ b/libcxx/test/std/containers/sequences/vector/vector.cons/move_assign_noexcept.pass.cpp
@@ -25,76 +25,71 @@
 #include "test_allocator.h"
 
 template <class T>
-struct some_alloc
-{
-    typedef T value_type;
-    some_alloc(const some_alloc&);
-    void allocate(std::size_t);
+struct some_alloc {
+  typedef T value_type;
+  some_alloc(const some_alloc&);
+  void allocate(std::size_t);
 };
 
 template <class T>
-struct some_alloc2
-{
-    typedef T value_type;
+struct some_alloc2 {
+  typedef T value_type;
 
-    some_alloc2() {}
-    some_alloc2(const some_alloc2&);
-    void allocate(std::size_t);
-    void deallocate(void*, unsigned) {}
+  some_alloc2() {}
+  some_alloc2(const some_alloc2&);
+  void allocate(std::size_t);
+  void deallocate(void*, unsigned) {}
 
-    typedef std::false_type propagate_on_container_move_assignment;
-    typedef std::true_type is_always_equal;
+  typedef std::false_type propagate_on_container_move_assignment;
+  typedef std::true_type is_always_equal;
 };
 
 template <class T>
-struct some_alloc3
-{
-    typedef T value_type;
+struct some_alloc3 {
+  typedef T value_type;
 
-    some_alloc3() {}
-    some_alloc3(const some_alloc3&);
-    void allocate(std::size_t);
-    void deallocate(void*, unsigned) {}
+  some_alloc3() {}
+  some_alloc3(const some_alloc3&);
+  void allocate(std::size_t);
+  void deallocate(void*, unsigned) {}
 
-    typedef std::false_type propagate_on_container_move_assignment;
-    typedef std::false_type is_always_equal;
+  typedef std::false_type propagate_on_container_move_assignment;
+  typedef std::false_type is_always_equal;
 };
 
-
-int main(int, char**)
-{
-    {
-        typedef std::vector<MoveOnly> C;
-        static_assert(std::is_nothrow_move_assignable<C>::value, "");
-    }
-    {
-        typedef std::vector<MoveOnly, test_allocator<MoveOnly>> C;
-        static_assert(!std::is_nothrow_move_assignable<C>::value, "");
-    }
-    {
-        typedef std::vector<MoveOnly, other_allocator<MoveOnly>> C;
-        static_assert(std::is_nothrow_move_assignable<C>::value, "");
-    }
-    {
-        typedef std::vector<MoveOnly, some_alloc<MoveOnly>> C;
+int main(int, char**) {
+  {
+    typedef std::vector<MoveOnly> C;
+    static_assert(std::is_nothrow_move_assignable<C>::value, "");
+  }
+  {
+    typedef std::vector<MoveOnly, test_allocator<MoveOnly>> C;
+    static_assert(!std::is_nothrow_move_assignable<C>::value, "");
+  }
+  {
+    typedef std::vector<MoveOnly, other_allocator<MoveOnly>> C;
+    static_assert(std::is_nothrow_move_assignable<C>::value, "");
+  }
+  {
+    typedef std::vector<MoveOnly, some_alloc<MoveOnly>> C;
     //  In C++17, move assignment for allocators are not allowed to throw
 #if TEST_STD_VER > 14
-        static_assert( std::is_nothrow_move_assignable<C>::value, "");
+    static_assert(std::is_nothrow_move_assignable<C>::value, "");
 #else
-        static_assert(!std::is_nothrow_move_assignable<C>::value, "");
+    static_assert(!std::is_nothrow_move_assignable<C>::value, "");
 #endif
-    }
+  }
 
 #if TEST_STD_VER > 14
-    {  // POCMA false, is_always_equal true
-        typedef std::vector<MoveOnly, some_alloc2<MoveOnly>> C;
-        static_assert( std::is_nothrow_move_assignable<C>::value, "");
-    }
-    {  // POCMA false, is_always_equal false
-        typedef std::vector<MoveOnly, some_alloc3<MoveOnly>> C;
-        static_assert(!std::is_nothrow_move_assignable<C>::value, "");
-    }
+  { // POCMA false, is_always_equal true
+    typedef std::vector<MoveOnly, some_alloc2<MoveOnly>> C;
+    static_assert(std::is_nothrow_move_assignable<C>::value, "");
+  }
+  { // POCMA false, is_always_equal false
+    typedef std::vector<MoveOnly, some_alloc3<MoveOnly>> C;
+    static_assert(!std::is_nothrow_move_assignable<C>::value, "");
+  }
 #endif
 
-    return 0;
+  return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/vector/vector.cons/move_noexcept.pass.cpp b/libcxx/test/std/containers/sequences/vector/vector.cons/move_noexcept.pass.cpp
index c2bfa5234065d..13dc6d41684fa 100644
--- a/libcxx/test/std/containers/sequences/vector/vector.cons/move_noexcept.pass.cpp
+++ b/libcxx/test/std/containers/sequences/vector/vector.cons/move_noexcept.pass.cpp
@@ -23,36 +23,34 @@
 #include "test_allocator.h"
 
 template <class T>
-struct some_alloc
-{
-    typedef T value_type;
-    some_alloc(const some_alloc&);
-    void allocate(std::size_t);
+struct some_alloc {
+  typedef T value_type;
+  some_alloc(const some_alloc&);
+  void allocate(std::size_t);
 };
 
-int main(int, char**)
-{
-    {
-        typedef std::vector<MoveOnly> C;
-        static_assert(std::is_nothrow_move_constructible<C>::value, "");
-    }
-    {
-        typedef std::vector<MoveOnly, test_allocator<MoveOnly>> C;
-        static_assert(std::is_nothrow_move_constructible<C>::value, "");
-    }
-    {
-        typedef std::vector<MoveOnly, other_allocator<MoveOnly>> C;
-        static_assert(std::is_nothrow_move_constructible<C>::value, "");
-    }
-    {
-        typedef std::vector<MoveOnly, some_alloc<MoveOnly>> C;
+int main(int, char**) {
+  {
+    typedef std::vector<MoveOnly> C;
+    static_assert(std::is_nothrow_move_constructible<C>::value, "");
+  }
+  {
+    typedef std::vector<MoveOnly, test_allocator<MoveOnly>> C;
+    static_assert(std::is_nothrow_move_constructible<C>::value, "");
+  }
+  {
+    typedef std::vector<MoveOnly, other_allocator<MoveOnly>> C;
+    static_assert(std::is_nothrow_move_constructible<C>::value, "");
+  }
+  {
+    typedef std::vector<MoveOnly, some_alloc<MoveOnly>> C;
     //  In C++17, move constructors for allocators are not allowed to throw
 #if TEST_STD_VER > 14
-        static_assert( std::is_nothrow_move_constructible<C>::value, "");
+    static_assert(std::is_nothrow_move_constructible<C>::value, "");
 #else
-        static_assert(!std::is_nothrow_move_constructible<C>::value, "");
+    static_assert(!std::is_nothrow_move_constructible<C>::value, "");
 #endif
-    }
+  }
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/vector/vector.cons/op_equal_initializer_list.pass.cpp b/libcxx/test/std/containers/sequences/vector/vector.cons/op_equal_initializer_list.pass.cpp
index f3d7b6a21b62e..893f909cf8c6c 100644
--- a/libcxx/test/std/containers/sequences/vector/vector.cons/op_equal_initializer_list.pass.cpp
+++ b/libcxx/test/std/containers/sequences/vector/vector.cons/op_equal_initializer_list.pass.cpp
@@ -19,9 +19,8 @@
 #include "min_allocator.h"
 #include "asan_testing.h"
 
-TEST_CONSTEXPR_CXX20 bool tests()
-{
-    {
+TEST_CONSTEXPR_CXX20 bool tests() {
+  {
     std::vector<int> d;
     d = {3, 4, 5, 6};
     assert(d.size() == 4);
@@ -30,8 +29,8 @@ TEST_CONSTEXPR_CXX20 bool tests()
     assert(d[1] == 4);
     assert(d[2] == 5);
     assert(d[3] == 6);
-    }
-    {
+  }
+  {
     std::vector<int, min_allocator<int>> d;
     d = {3, 4, 5, 6};
     assert(d.size() == 4);
@@ -40,16 +39,15 @@ TEST_CONSTEXPR_CXX20 bool tests()
     assert(d[1] == 4);
     assert(d[2] == 5);
     assert(d[3] == 6);
-    }
+  }
 
-    return true;
+  return true;
 }
 
-int main(int, char**)
-{
-    tests();
+int main(int, char**) {
+  tests();
 #if TEST_STD_VER > 17
-    static_assert(tests());
+  static_assert(tests());
 #endif
-    return 0;
+  return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/vector/vector.data/data.pass.cpp b/libcxx/test/std/containers/sequences/vector/vector.data/data.pass.cpp
index 64b85424e54a5..50c4f5df8333e 100644
--- a/libcxx/test/std/containers/sequences/vector/vector.data/data.pass.cpp
+++ b/libcxx/test/std/containers/sequences/vector/vector.data/data.pass.cpp
@@ -18,72 +18,73 @@
 #include "asan_testing.h"
 
 struct Nasty {
-    TEST_CONSTEXPR Nasty() : i_(0) {}
-    TEST_CONSTEXPR Nasty(int i) : i_(i) {}
-    TEST_CONSTEXPR_CXX20 ~Nasty() {}
+  TEST_CONSTEXPR Nasty() : i_(0) {}
+  TEST_CONSTEXPR Nasty(int i) : i_(i) {}
+  TEST_CONSTEXPR_CXX20 ~Nasty() {}
 
-    Nasty * operator&() const { assert(false); return nullptr; }
-    int i_;
+  Nasty* operator&() const {
+    assert(false);
+    return nullptr;
+  }
+  int i_;
 };
 
-TEST_CONSTEXPR_CXX20 bool tests()
-{
-    {
-        std::vector<int> v;
-        assert(v.data() == 0);
-        assert(is_contiguous_container_asan_correct(v));
-    }
-    {
-        std::vector<int> v(100);
-        assert(v.data() == std::addressof(v.front()));
-        assert(is_contiguous_container_asan_correct(v));
-    }
-    {
-        std::vector<Nasty> v(100);
-        assert(v.data() == std::addressof(v.front()));
-        assert(is_contiguous_container_asan_correct(v));
-    }
+TEST_CONSTEXPR_CXX20 bool tests() {
+  {
+    std::vector<int> v;
+    assert(v.data() == 0);
+    assert(is_contiguous_container_asan_correct(v));
+  }
+  {
+    std::vector<int> v(100);
+    assert(v.data() == std::addressof(v.front()));
+    assert(is_contiguous_container_asan_correct(v));
+  }
+  {
+    std::vector<Nasty> v(100);
+    assert(v.data() == std::addressof(v.front()));
+    assert(is_contiguous_container_asan_correct(v));
+  }
 #if TEST_STD_VER >= 11
-    {
-        std::vector<int, min_allocator<int>> v;
-        assert(v.data() == 0);
-        assert(is_contiguous_container_asan_correct(v));
-    }
-    {
-        std::vector<int, min_allocator<int>> v(100);
-        assert(v.data() == std::addressof(v.front()));
-        assert(is_contiguous_container_asan_correct(v));
-    }
-    {
-        std::vector<Nasty, min_allocator<Nasty>> v(100);
-        assert(v.data() == std::addressof(v.front()));
-        assert(is_contiguous_container_asan_correct(v));
-    }
-    {
-      std::vector<int, safe_allocator<int>> v;
-      assert(v.data() == 0);
-      assert(is_contiguous_container_asan_correct(v));
-    }
-    {
-      std::vector<int, safe_allocator<int>> v(100);
-      assert(v.data() == std::addressof(v.front()));
-      assert(is_contiguous_container_asan_correct(v));
-    }
-    {
-      std::vector<Nasty, safe_allocator<Nasty>> v(100);
-      assert(v.data() == std::addressof(v.front()));
-      assert(is_contiguous_container_asan_correct(v));
-    }
+  {
+    std::vector<int, min_allocator<int>> v;
+    assert(v.data() == 0);
+    assert(is_contiguous_container_asan_correct(v));
+  }
+  {
+    std::vector<int, min_allocator<int>> v(100);
+    assert(v.data() == std::addressof(v.front()));
+    assert(is_contiguous_container_asan_correct(v));
+  }
+  {
+    std::vector<Nasty, min_allocator<Nasty>> v(100);
+    assert(v.data() == std::addressof(v.front()));
+    assert(is_contiguous_container_asan_correct(v));
+  }
+  {
+    std::vector<int, safe_allocator<int>> v;
+    assert(v.data() == 0);
+    assert(is_contiguous_container_asan_correct(v));
+  }
+  {
+    std::vector<int, safe_allocator<int>> v(100);
+    assert(v.data() == std::addressof(v.front()));
+    assert(is_contiguous_container_asan_correct(v));
+  }
+  {
+    std::vector<Nasty, safe_allocator<Nasty>> v(100);
+    assert(v.data() == std::addressof(v.front()));
+    assert(is_contiguous_container_asan_correct(v));
+  }
 #endif
 
-    return true;
+  return true;
 }
 
-int main(int, char**)
-{
-    tests();
+int main(int, char**) {
+  tests();
 #if TEST_STD_VER > 17
-    static_assert(tests());
+  static_assert(tests());
 #endif
-    return 0;
+  return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/vector/vector.data/data_const.pass.cpp b/libcxx/test/std/containers/sequences/vector/vector.data/data_const.pass.cpp
index 0cf0b22047352..6b45fbcfaf0c5 100644
--- a/libcxx/test/std/containers/sequences/vector/vector.data/data_const.pass.cpp
+++ b/libcxx/test/std/containers/sequences/vector/vector.data/data_const.pass.cpp
@@ -18,72 +18,73 @@
 #include "asan_testing.h"
 
 struct Nasty {
-    TEST_CONSTEXPR Nasty() : i_(0) {}
-    TEST_CONSTEXPR Nasty(int i) : i_(i) {}
-    TEST_CONSTEXPR_CXX20 ~Nasty() {}
+  TEST_CONSTEXPR Nasty() : i_(0) {}
+  TEST_CONSTEXPR Nasty(int i) : i_(i) {}
+  TEST_CONSTEXPR_CXX20 ~Nasty() {}
 
-    Nasty * operator&() const { assert(false); return nullptr; }
-    int i_;
+  Nasty* operator&() const {
+    assert(false);
+    return nullptr;
+  }
+  int i_;
 };
 
-TEST_CONSTEXPR_CXX20 bool tests()
-{
-    {
-        const std::vector<int> v;
-        assert(v.data() == 0);
-        assert(is_contiguous_container_asan_correct(v));
-    }
-    {
-        const std::vector<int> v(100);
-        assert(v.data() == std::addressof(v.front()));
-        assert(is_contiguous_container_asan_correct(v));
-    }
-    {
-        const std::vector<Nasty> v(100);
-        assert(v.data() == std::addressof(v.front()));
-        assert(is_contiguous_container_asan_correct(v));
-    }
+TEST_CONSTEXPR_CXX20 bool tests() {
+  {
+    const std::vector<int> v;
+    assert(v.data() == 0);
+    assert(is_contiguous_container_asan_correct(v));
+  }
+  {
+    const std::vector<int> v(100);
+    assert(v.data() == std::addressof(v.front()));
+    assert(is_contiguous_container_asan_correct(v));
+  }
+  {
+    const std::vector<Nasty> v(100);
+    assert(v.data() == std::addressof(v.front()));
+    assert(is_contiguous_container_asan_correct(v));
+  }
 #if TEST_STD_VER >= 11
-    {
-        const std::vector<int, min_allocator<int>> v;
-        assert(v.data() == 0);
-        assert(is_contiguous_container_asan_correct(v));
-    }
-    {
-        const std::vector<int, min_allocator<int>> v(100);
-        assert(v.data() == &v.front());
-        assert(is_contiguous_container_asan_correct(v));
-    }
-    {
-        const std::vector<Nasty, min_allocator<Nasty>> v(100);
-        assert(v.data() == std::addressof(v.front()));
-        assert(is_contiguous_container_asan_correct(v));
-    }
-    {
-      const std::vector<int, safe_allocator<int>> v;
-      assert(v.data() == 0);
-      assert(is_contiguous_container_asan_correct(v));
-    }
-    {
-      const std::vector<int, safe_allocator<int>> v(100);
-      assert(v.data() == &v.front());
-      assert(is_contiguous_container_asan_correct(v));
-    }
-    {
-      const std::vector<Nasty, safe_allocator<Nasty>> v(100);
-      assert(v.data() == std::addressof(v.front()));
-      assert(is_contiguous_container_asan_correct(v));
-    }
+  {
+    const std::vector<int, min_allocator<int>> v;
+    assert(v.data() == 0);
+    assert(is_contiguous_container_asan_correct(v));
+  }
+  {
+    const std::vector<int, min_allocator<int>> v(100);
+    assert(v.data() == &v.front());
+    assert(is_contiguous_container_asan_correct(v));
+  }
+  {
+    const std::vector<Nasty, min_allocator<Nasty>> v(100);
+    assert(v.data() == std::addressof(v.front()));
+    assert(is_contiguous_container_asan_correct(v));
+  }
+  {
+    const std::vector<int, safe_allocator<int>> v;
+    assert(v.data() == 0);
+    assert(is_contiguous_container_asan_correct(v));
+  }
+  {
+    const std::vector<int, safe_allocator<int>> v(100);
+    assert(v.data() == &v.front());
+    assert(is_contiguous_container_asan_correct(v));
+  }
+  {
+    const std::vector<Nasty, safe_allocator<Nasty>> v(100);
+    assert(v.data() == std::addressof(v.front()));
+    assert(is_contiguous_container_asan_correct(v));
+  }
 #endif
 
-    return true;
+  return true;
 }
 
-int main(int, char**)
-{
-    tests();
+int main(int, char**) {
+  tests();
 #if TEST_STD_VER > 17
-    static_assert(tests());
+  static_assert(tests());
 #endif
-    return 0;
+  return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/vector/vector.erasure/erase.pass.cpp b/libcxx/test/std/containers/sequences/vector/vector.erasure/erase.pass.cpp
index 88b27d9634c54..175d56c1d09de 100644
--- a/libcxx/test/std/containers/sequences/vector/vector.erasure/erase.pass.cpp
+++ b/libcxx/test/std/containers/sequences/vector/vector.erasure/erase.pass.cpp
@@ -28,8 +28,7 @@ TEST_CONSTEXPR_CXX20 void test0(S s, U val, S expected, std::size_t expected_era
 }
 
 template <class S>
-TEST_CONSTEXPR_CXX20 void test()
-{
+TEST_CONSTEXPR_CXX20 void test() {
   test0(S(), 1, S(), 0);
 
   test0(S({1}), 1, S(), 1);
@@ -63,21 +62,19 @@ TEST_CONSTEXPR_CXX20 void test()
   test0(S({1, 2, 1}), opt(3), S({1, 2, 1}), 0);
 }
 
-TEST_CONSTEXPR_CXX20 bool tests()
-{
-    test<std::vector<int>>();
-    test<std::vector<int, min_allocator<int>>> ();
-    test<std::vector<int, test_allocator<int>>> ();
-    test<std::vector<int, safe_allocator<int>>>();
+TEST_CONSTEXPR_CXX20 bool tests() {
+  test<std::vector<int>>();
+  test<std::vector<int, min_allocator<int>>>();
+  test<std::vector<int, test_allocator<int>>>();
+  test<std::vector<int, safe_allocator<int>>>();
 
-    test<std::vector<long>>();
-    test<std::vector<double>>();
+  test<std::vector<long>>();
+  test<std::vector<double>>();
 
-    return true;
+  return true;
 }
 
-int main(int, char**)
-{
+int main(int, char**) {
   tests();
 #if TEST_STD_VER > 17
   static_assert(tests());
diff --git a/libcxx/test/std/containers/sequences/vector/vector.erasure/erase_if.pass.cpp b/libcxx/test/std/containers/sequences/vector/vector.erasure/erase_if.pass.cpp
index 015824a944571..189986fe3fd4d 100644
--- a/libcxx/test/std/containers/sequences/vector/vector.erasure/erase_if.pass.cpp
+++ b/libcxx/test/std/containers/sequences/vector/vector.erasure/erase_if.pass.cpp
@@ -27,62 +27,59 @@ TEST_CONSTEXPR_CXX20 void test0(S s, Pred p, S expected, std::size_t expected_er
 }
 
 template <typename S>
-TEST_CONSTEXPR_CXX20 void test()
-{
-    auto is1 = [](auto v) { return v == 1;};
-    auto is2 = [](auto v) { return v == 2;};
-    auto is3 = [](auto v) { return v == 3;};
-    auto is4 = [](auto v) { return v == 4;};
-    auto True  = [](auto) { return true; };
-    auto False = [](auto) { return false; };
-
-    test0(S(), is1, S(), 0);
-
-    test0(S({1}), is1, S(), 1);
-    test0(S({1}), is2, S({1}), 0);
-
-    test0(S({1, 2}), is1, S({2}), 1);
-    test0(S({1, 2}), is2, S({1}), 1);
-    test0(S({1, 2}), is3, S({1, 2}), 0);
-    test0(S({1, 1}), is1, S(), 2);
-    test0(S({1, 1}), is3, S({1, 1}), 0);
-
-    test0(S({1, 2, 3}), is1, S({2, 3}), 1);
-    test0(S({1, 2, 3}), is2, S({1, 3}), 1);
-    test0(S({1, 2, 3}), is3, S({1, 2}), 1);
-    test0(S({1, 2, 3}), is4, S({1, 2, 3}), 0);
-
-    test0(S({1, 1, 1}), is1, S(), 3);
-    test0(S({1, 1, 1}), is2, S({1, 1, 1}), 0);
-    test0(S({1, 1, 2}), is1, S({2}), 2);
-    test0(S({1, 1, 2}), is2, S({1, 1}), 1);
-    test0(S({1, 1, 2}), is3, S({1, 1, 2}), 0);
-    test0(S({1, 2, 2}), is1, S({2, 2}), 1);
-    test0(S({1, 2, 2}), is2, S({1}), 2);
-    test0(S({1, 2, 2}), is3, S({1, 2, 2}), 0);
-
-    test0(S({1, 2, 3}), True, S(), 3);
-    test0(S({1, 2, 3}), False, S({1, 2, 3}), 0);
+TEST_CONSTEXPR_CXX20 void test() {
+  auto is1   = [](auto v) { return v == 1; };
+  auto is2   = [](auto v) { return v == 2; };
+  auto is3   = [](auto v) { return v == 3; };
+  auto is4   = [](auto v) { return v == 4; };
+  auto True  = [](auto) { return true; };
+  auto False = [](auto) { return false; };
+
+  test0(S(), is1, S(), 0);
+
+  test0(S({1}), is1, S(), 1);
+  test0(S({1}), is2, S({1}), 0);
+
+  test0(S({1, 2}), is1, S({2}), 1);
+  test0(S({1, 2}), is2, S({1}), 1);
+  test0(S({1, 2}), is3, S({1, 2}), 0);
+  test0(S({1, 1}), is1, S(), 2);
+  test0(S({1, 1}), is3, S({1, 1}), 0);
+
+  test0(S({1, 2, 3}), is1, S({2, 3}), 1);
+  test0(S({1, 2, 3}), is2, S({1, 3}), 1);
+  test0(S({1, 2, 3}), is3, S({1, 2}), 1);
+  test0(S({1, 2, 3}), is4, S({1, 2, 3}), 0);
+
+  test0(S({1, 1, 1}), is1, S(), 3);
+  test0(S({1, 1, 1}), is2, S({1, 1, 1}), 0);
+  test0(S({1, 1, 2}), is1, S({2}), 2);
+  test0(S({1, 1, 2}), is2, S({1, 1}), 1);
+  test0(S({1, 1, 2}), is3, S({1, 1, 2}), 0);
+  test0(S({1, 2, 2}), is1, S({2, 2}), 1);
+  test0(S({1, 2, 2}), is2, S({1}), 2);
+  test0(S({1, 2, 2}), is3, S({1, 2, 2}), 0);
+
+  test0(S({1, 2, 3}), True, S(), 3);
+  test0(S({1, 2, 3}), False, S({1, 2, 3}), 0);
 }
 
-TEST_CONSTEXPR_CXX20 bool tests()
-{
-    test<std::vector<int>>();
-    test<std::vector<int, min_allocator<int>>> ();
-    test<std::vector<int, test_allocator<int>>> ();
-    test<std::vector<int, safe_allocator<int>>>();
+TEST_CONSTEXPR_CXX20 bool tests() {
+  test<std::vector<int>>();
+  test<std::vector<int, min_allocator<int>>>();
+  test<std::vector<int, test_allocator<int>>>();
+  test<std::vector<int, safe_allocator<int>>>();
 
-    test<std::vector<long>>();
-    test<std::vector<double>>();
+  test<std::vector<long>>();
+  test<std::vector<double>>();
 
-    return true;
+  return true;
 }
 
-int main(int, char**)
-{
-    tests();
+int main(int, char**) {
+  tests();
 #if TEST_STD_VER > 17
-    static_assert(tests());
+  static_assert(tests());
 #endif
-    return 0;
+  return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/vector/vector.modifiers/append_range.pass.cpp b/libcxx/test/std/containers/sequences/vector/vector.modifiers/append_range.pass.cpp
index 69e6df6fcffa8..7a5031e4676f2 100644
--- a/libcxx/test/std/containers/sequences/vector/vector.modifiers/append_range.pass.cpp
+++ b/libcxx/test/std/containers/sequences/vector/vector.modifiers/append_range.pass.cpp
@@ -34,9 +34,9 @@ constexpr bool test() {
   });
   test_sequence_append_range_move_only<std::vector>();
 
-  { // Vector may or may not need to reallocate because of the insertion -- make sure to test both cases.
+  {   // Vector may or may not need to reallocate because of the insertion -- make sure to test both cases.
     { // Ensure reallocation happens.
-      int in[] = {-1, -2, -3, -4, -5, -6, -7, -8, -9, -10};
+      int in[]           = {-1, -2, -3, -4, -5, -6, -7, -8, -9, -10};
       std::vector<int> v = {1, 2, 3, 4, 5, 6, 7, 8};
       v.shrink_to_fit();
       assert(v.capacity() < v.size() + std::ranges::size(in));
@@ -46,7 +46,7 @@ constexpr bool test() {
     }
 
     { // Ensure no reallocation happens.
-      int in[] = {-1, -2, -3, -4, -5, -6, -7, -8, -9, -10};
+      int in[]           = {-1, -2, -3, -4, -5, -6, -7, -8, -9, -10};
       std::vector<int> v = {1, 2, 3, 4, 5, 6, 7, 8};
       v.reserve(v.size() + std::ranges::size(in));
       assert(v.capacity() >= v.size() + std::ranges::size(in));
diff --git a/libcxx/test/std/containers/sequences/vector/vector.modifiers/assign_range.pass.cpp b/libcxx/test/std/containers/sequences/vector/vector.modifiers/assign_range.pass.cpp
index 6326ffca96966..132343f455615 100644
--- a/libcxx/test/std/containers/sequences/vector/vector.modifiers/assign_range.pass.cpp
+++ b/libcxx/test/std/containers/sequences/vector/vector.modifiers/assign_range.pass.cpp
@@ -35,9 +35,9 @@ constexpr bool test() {
   });
   test_sequence_assign_range_move_only<std::vector>();
 
-  { // Vector may or may not need to reallocate because of the assignment -- make sure to test both cases.
+  {   // Vector may or may not need to reallocate because of the assignment -- make sure to test both cases.
     { // Ensure reallocation happens.
-      int in[] = {-1, -2, -3, -4, -5, -6, -7, -8, -9, -10};
+      int in[]           = {-1, -2, -3, -4, -5, -6, -7, -8, -9, -10};
       std::vector<int> v = {1, 2, 3, 4, 5, 6, 7, 8};
       v.shrink_to_fit();
       assert(v.capacity() < v.size() + std::ranges::size(in));
@@ -47,7 +47,7 @@ constexpr bool test() {
     }
 
     { // Ensure no reallocation happens -- the input range is shorter than the vector.
-      int in[] = {-1, -2, -3, -4, -5};
+      int in[]           = {-1, -2, -3, -4, -5};
       std::vector<int> v = {1, 2, 3, 4, 5, 6, 7, 8};
 
       v.assign_range(in);
@@ -55,7 +55,7 @@ constexpr bool test() {
     }
 
     { // Ensure no reallocation happens -- the input range is longer than the vector but within capacity.
-      int in[] = {-1, -2, -3, -4, -5, -6, -7, -8};
+      int in[]           = {-1, -2, -3, -4, -5, -6, -7, -8};
       std::vector<int> v = {1, 2, 3, 4, 5};
       v.reserve(std::ranges::size(in));
       assert(v.capacity() >= std::ranges::size(in));
diff --git a/libcxx/test/std/containers/sequences/vector/vector.modifiers/clear.pass.cpp b/libcxx/test/std/containers/sequences/vector/vector.modifiers/clear.pass.cpp
index 9390b8309736f..3c8ed17cafbd4 100644
--- a/libcxx/test/std/containers/sequences/vector/vector.modifiers/clear.pass.cpp
+++ b/libcxx/test/std/containers/sequences/vector/vector.modifiers/clear.pass.cpp
@@ -17,46 +17,44 @@
 #include "min_allocator.h"
 #include "asan_testing.h"
 
-TEST_CONSTEXPR_CXX20 bool tests()
-{
-    {
+TEST_CONSTEXPR_CXX20 bool tests() {
+  {
     int a[] = {1, 2, 3};
-    std::vector<int> c(a, a+3);
+    std::vector<int> c(a, a + 3);
     ASSERT_NOEXCEPT(c.clear());
     c.clear();
     assert(c.empty());
     LIBCPP_ASSERT(c.__invariants());
     LIBCPP_ASSERT(is_contiguous_container_asan_correct(c));
-    }
+  }
 #if TEST_STD_VER >= 11
-    {
+  {
     int a[] = {1, 2, 3};
-    std::vector<int, min_allocator<int>> c(a, a+3);
+    std::vector<int, min_allocator<int>> c(a, a + 3);
     ASSERT_NOEXCEPT(c.clear());
     c.clear();
     assert(c.empty());
     LIBCPP_ASSERT(c.__invariants());
     LIBCPP_ASSERT(is_contiguous_container_asan_correct(c));
-    }
-    {
-      int a[] = {1, 2, 3};
-      std::vector<int, safe_allocator<int>> c(a, a + 3);
-      ASSERT_NOEXCEPT(c.clear());
-      c.clear();
-      assert(c.empty());
-      LIBCPP_ASSERT(c.__invariants());
-      LIBCPP_ASSERT(is_contiguous_container_asan_correct(c));
-    }
+  }
+  {
+    int a[] = {1, 2, 3};
+    std::vector<int, safe_allocator<int>> c(a, a + 3);
+    ASSERT_NOEXCEPT(c.clear());
+    c.clear();
+    assert(c.empty());
+    LIBCPP_ASSERT(c.__invariants());
+    LIBCPP_ASSERT(is_contiguous_container_asan_correct(c));
+  }
 #endif
 
-    return true;
+  return true;
 }
 
-int main(int, char**)
-{
-    tests();
+int main(int, char**) {
+  tests();
 #if TEST_STD_VER > 17
-    static_assert(tests());
+  static_assert(tests());
 #endif
-    return 0;
+  return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/vector/vector.modifiers/emplace.pass.cpp b/libcxx/test/std/containers/sequences/vector/vector.modifiers/emplace.pass.cpp
index 4f5e36abb6bb8..667ce483806e7 100644
--- a/libcxx/test/std/containers/sequences/vector/vector.modifiers/emplace.pass.cpp
+++ b/libcxx/test/std/containers/sequences/vector/vector.modifiers/emplace.pass.cpp
@@ -20,151 +20,144 @@
 #include "min_allocator.h"
 #include "asan_testing.h"
 
-class A
-{
-    int i_;
-    double d_;
+class A {
+  int i_;
+  double d_;
+
+  A(const A&);
+  A& operator=(const A&);
 
-    A(const A&);
-    A& operator=(const A&);
 public:
-    TEST_CONSTEXPR_CXX14 A(int i, double d)
-        : i_(i), d_(d) {}
+  TEST_CONSTEXPR_CXX14 A(int i, double d) : i_(i), d_(d) {}
 
-    TEST_CONSTEXPR_CXX14 A(A&& a)
-        : i_(a.i_),
-          d_(a.d_)
-    {
-        a.i_ = 0;
-        a.d_ = 0;
-    }
+  TEST_CONSTEXPR_CXX14 A(A&& a) : i_(a.i_), d_(a.d_) {
+    a.i_ = 0;
+    a.d_ = 0;
+  }
 
-    TEST_CONSTEXPR_CXX14 A& operator=(A&& a)
-    {
-        i_ = a.i_;
-        d_ = a.d_;
-        a.i_ = 0;
-        a.d_ = 0;
-        return *this;
-    }
+  TEST_CONSTEXPR_CXX14 A& operator=(A&& a) {
+    i_   = a.i_;
+    d_   = a.d_;
+    a.i_ = 0;
+    a.d_ = 0;
+    return *this;
+  }
 
-    TEST_CONSTEXPR_CXX14 int geti() const {return i_;}
-    TEST_CONSTEXPR_CXX14 double getd() const {return d_;}
+  TEST_CONSTEXPR_CXX14 int geti() const { return i_; }
+  TEST_CONSTEXPR_CXX14 double getd() const { return d_; }
 };
 
-TEST_CONSTEXPR_CXX20 bool tests()
-{
-    {
-        std::vector<A> c;
-        std::vector<A>::iterator i = c.emplace(c.cbegin(), 2, 3.5);
-        assert(i == c.begin());
-        assert(c.size() == 1);
-        assert(c.front().geti() == 2);
-        assert(c.front().getd() == 3.5);
-        assert(is_contiguous_container_asan_correct(c));
-        i = c.emplace(c.cend(), 3, 4.5);
-        assert(i == c.end()-1);
-        assert(c.size() == 2);
-        assert(c.front().geti() == 2);
-        assert(c.front().getd() == 3.5);
-        assert(c.back().geti() == 3);
-        assert(c.back().getd() == 4.5);
-        assert(is_contiguous_container_asan_correct(c));
-        i = c.emplace(c.cbegin()+1, 4, 6.5);
-        assert(i == c.begin()+1);
-        assert(c.size() == 3);
-        assert(c.front().geti() == 2);
-        assert(c.front().getd() == 3.5);
-        assert(c[1].geti() == 4);
-        assert(c[1].getd() == 6.5);
-        assert(c.back().geti() == 3);
-        assert(c.back().getd() == 4.5);
-        assert(is_contiguous_container_asan_correct(c));
-    }
-    {
-        std::vector<A, limited_allocator<A, 7> > c;
-        std::vector<A, limited_allocator<A, 7> >::iterator i = c.emplace(c.cbegin(), 2, 3.5);
-        assert(i == c.begin());
-        assert(c.size() == 1);
-        assert(c.front().geti() == 2);
-        assert(c.front().getd() == 3.5);
-        assert(is_contiguous_container_asan_correct(c));
-        i = c.emplace(c.cend(), 3, 4.5);
-        assert(i == c.end()-1);
-        assert(c.size() == 2);
-        assert(c.front().geti() == 2);
-        assert(c.front().getd() == 3.5);
-        assert(c.back().geti() == 3);
-        assert(c.back().getd() == 4.5);
-        assert(is_contiguous_container_asan_correct(c));
-        i = c.emplace(c.cbegin()+1, 4, 6.5);
-        assert(i == c.begin()+1);
-        assert(c.size() == 3);
-        assert(c.front().geti() == 2);
-        assert(c.front().getd() == 3.5);
-        assert(c[1].geti() == 4);
-        assert(c[1].getd() == 6.5);
-        assert(c.back().geti() == 3);
-        assert(c.back().getd() == 4.5);
-        assert(is_contiguous_container_asan_correct(c));
-    }
-    {
-        std::vector<A, min_allocator<A> > c;
-        std::vector<A, min_allocator<A> >::iterator i = c.emplace(c.cbegin(), 2, 3.5);
-        assert(i == c.begin());
-        assert(c.size() == 1);
-        assert(c.front().geti() == 2);
-        assert(c.front().getd() == 3.5);
-        i = c.emplace(c.cend(), 3, 4.5);
-        assert(i == c.end()-1);
-        assert(c.size() == 2);
-        assert(c.front().geti() == 2);
-        assert(c.front().getd() == 3.5);
-        assert(c.back().geti() == 3);
-        assert(c.back().getd() == 4.5);
-        i = c.emplace(c.cbegin()+1, 4, 6.5);
-        assert(i == c.begin()+1);
-        assert(c.size() == 3);
-        assert(c.front().geti() == 2);
-        assert(c.front().getd() == 3.5);
-        assert(c[1].geti() == 4);
-        assert(c[1].getd() == 6.5);
-        assert(c.back().geti() == 3);
-        assert(c.back().getd() == 4.5);
-    }
-    {
-      std::vector<A, safe_allocator<A> > c;
-      std::vector<A, safe_allocator<A> >::iterator i = c.emplace(c.cbegin(), 2, 3.5);
-      assert(i == c.begin());
-      assert(c.size() == 1);
-      assert(c.front().geti() == 2);
-      assert(c.front().getd() == 3.5);
-      i = c.emplace(c.cend(), 3, 4.5);
-      assert(i == c.end() - 1);
-      assert(c.size() == 2);
-      assert(c.front().geti() == 2);
-      assert(c.front().getd() == 3.5);
-      assert(c.back().geti() == 3);
-      assert(c.back().getd() == 4.5);
-      i = c.emplace(c.cbegin() + 1, 4, 6.5);
-      assert(i == c.begin() + 1);
-      assert(c.size() == 3);
-      assert(c.front().geti() == 2);
-      assert(c.front().getd() == 3.5);
-      assert(c[1].geti() == 4);
-      assert(c[1].getd() == 6.5);
-      assert(c.back().geti() == 3);
-      assert(c.back().getd() == 4.5);
-    }
+TEST_CONSTEXPR_CXX20 bool tests() {
+  {
+    std::vector<A> c;
+    std::vector<A>::iterator i = c.emplace(c.cbegin(), 2, 3.5);
+    assert(i == c.begin());
+    assert(c.size() == 1);
+    assert(c.front().geti() == 2);
+    assert(c.front().getd() == 3.5);
+    assert(is_contiguous_container_asan_correct(c));
+    i = c.emplace(c.cend(), 3, 4.5);
+    assert(i == c.end() - 1);
+    assert(c.size() == 2);
+    assert(c.front().geti() == 2);
+    assert(c.front().getd() == 3.5);
+    assert(c.back().geti() == 3);
+    assert(c.back().getd() == 4.5);
+    assert(is_contiguous_container_asan_correct(c));
+    i = c.emplace(c.cbegin() + 1, 4, 6.5);
+    assert(i == c.begin() + 1);
+    assert(c.size() == 3);
+    assert(c.front().geti() == 2);
+    assert(c.front().getd() == 3.5);
+    assert(c[1].geti() == 4);
+    assert(c[1].getd() == 6.5);
+    assert(c.back().geti() == 3);
+    assert(c.back().getd() == 4.5);
+    assert(is_contiguous_container_asan_correct(c));
+  }
+  {
+    std::vector<A, limited_allocator<A, 7> > c;
+    std::vector<A, limited_allocator<A, 7> >::iterator i = c.emplace(c.cbegin(), 2, 3.5);
+    assert(i == c.begin());
+    assert(c.size() == 1);
+    assert(c.front().geti() == 2);
+    assert(c.front().getd() == 3.5);
+    assert(is_contiguous_container_asan_correct(c));
+    i = c.emplace(c.cend(), 3, 4.5);
+    assert(i == c.end() - 1);
+    assert(c.size() == 2);
+    assert(c.front().geti() == 2);
+    assert(c.front().getd() == 3.5);
+    assert(c.back().geti() == 3);
+    assert(c.back().getd() == 4.5);
+    assert(is_contiguous_container_asan_correct(c));
+    i = c.emplace(c.cbegin() + 1, 4, 6.5);
+    assert(i == c.begin() + 1);
+    assert(c.size() == 3);
+    assert(c.front().geti() == 2);
+    assert(c.front().getd() == 3.5);
+    assert(c[1].geti() == 4);
+    assert(c[1].getd() == 6.5);
+    assert(c.back().geti() == 3);
+    assert(c.back().getd() == 4.5);
+    assert(is_contiguous_container_asan_correct(c));
+  }
+  {
+    std::vector<A, min_allocator<A> > c;
+    std::vector<A, min_allocator<A> >::iterator i = c.emplace(c.cbegin(), 2, 3.5);
+    assert(i == c.begin());
+    assert(c.size() == 1);
+    assert(c.front().geti() == 2);
+    assert(c.front().getd() == 3.5);
+    i = c.emplace(c.cend(), 3, 4.5);
+    assert(i == c.end() - 1);
+    assert(c.size() == 2);
+    assert(c.front().geti() == 2);
+    assert(c.front().getd() == 3.5);
+    assert(c.back().geti() == 3);
+    assert(c.back().getd() == 4.5);
+    i = c.emplace(c.cbegin() + 1, 4, 6.5);
+    assert(i == c.begin() + 1);
+    assert(c.size() == 3);
+    assert(c.front().geti() == 2);
+    assert(c.front().getd() == 3.5);
+    assert(c[1].geti() == 4);
+    assert(c[1].getd() == 6.5);
+    assert(c.back().geti() == 3);
+    assert(c.back().getd() == 4.5);
+  }
+  {
+    std::vector<A, safe_allocator<A> > c;
+    std::vector<A, safe_allocator<A> >::iterator i = c.emplace(c.cbegin(), 2, 3.5);
+    assert(i == c.begin());
+    assert(c.size() == 1);
+    assert(c.front().geti() == 2);
+    assert(c.front().getd() == 3.5);
+    i = c.emplace(c.cend(), 3, 4.5);
+    assert(i == c.end() - 1);
+    assert(c.size() == 2);
+    assert(c.front().geti() == 2);
+    assert(c.front().getd() == 3.5);
+    assert(c.back().geti() == 3);
+    assert(c.back().getd() == 4.5);
+    i = c.emplace(c.cbegin() + 1, 4, 6.5);
+    assert(i == c.begin() + 1);
+    assert(c.size() == 3);
+    assert(c.front().geti() == 2);
+    assert(c.front().getd() == 3.5);
+    assert(c[1].geti() == 4);
+    assert(c[1].getd() == 6.5);
+    assert(c.back().geti() == 3);
+    assert(c.back().getd() == 4.5);
+  }
 
-    return true;
+  return true;
 }
 
-int main(int, char**)
-{
-    tests();
+int main(int, char**) {
+  tests();
 #if TEST_STD_VER > 17
-    static_assert(tests());
+  static_assert(tests());
 #endif
-    return 0;
+  return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/vector/vector.modifiers/emplace_back.pass.cpp b/libcxx/test/std/containers/sequences/vector/vector.modifiers/emplace_back.pass.cpp
index 730ed32bd7334..43216a1512fd6 100644
--- a/libcxx/test/std/containers/sequences/vector/vector.modifiers/emplace_back.pass.cpp
+++ b/libcxx/test/std/containers/sequences/vector/vector.modifiers/emplace_back.pass.cpp
@@ -21,177 +21,169 @@
 #include "test_allocator.h"
 #include "asan_testing.h"
 
-class A
-{
-    int i_;
-    double d_;
+class A {
+  int i_;
+  double d_;
 
 public:
-    A(const A&) = delete;
-    A& operator=(const A&) = delete;
+  A(const A&)            = delete;
+  A& operator=(const A&) = delete;
 
-    TEST_CONSTEXPR_CXX14 A(int i, double d)
-        : i_(i), d_(d) {}
+  TEST_CONSTEXPR_CXX14 A(int i, double d) : i_(i), d_(d) {}
 
-    TEST_CONSTEXPR_CXX14 A(A&& a)
-        : i_(a.i_),
-          d_(a.d_)
-    {
-        a.i_ = 0;
-        a.d_ = 0;
-    }
+  TEST_CONSTEXPR_CXX14 A(A&& a) : i_(a.i_), d_(a.d_) {
+    a.i_ = 0;
+    a.d_ = 0;
+  }
 
-    TEST_CONSTEXPR_CXX14 A& operator=(A&& a)
-    {
-        i_ = a.i_;
-        d_ = a.d_;
-        a.i_ = 0;
-        a.d_ = 0;
-        return *this;
-    }
+  TEST_CONSTEXPR_CXX14 A& operator=(A&& a) {
+    i_   = a.i_;
+    d_   = a.d_;
+    a.i_ = 0;
+    a.d_ = 0;
+    return *this;
+  }
 
-    TEST_CONSTEXPR_CXX14 int geti() const {return i_;}
-    TEST_CONSTEXPR_CXX14 double getd() const {return d_;}
+  TEST_CONSTEXPR_CXX14 int geti() const { return i_; }
+  TEST_CONSTEXPR_CXX14 double getd() const { return d_; }
 };
 
-TEST_CONSTEXPR_CXX20 bool tests()
-{
-    {
-        std::vector<A> c;
+TEST_CONSTEXPR_CXX20 bool tests() {
+  {
+    std::vector<A> c;
 #if TEST_STD_VER > 14
-        A& r1 = c.emplace_back(2, 3.5);
-        assert(c.size() == 1);
-        assert(&r1 == &c.back());
-        assert(c.front().geti() == 2);
-        assert(c.front().getd() == 3.5);
-        assert(is_contiguous_container_asan_correct(c));
-        A& r2 = c.emplace_back(3, 4.5);
-        assert(c.size() == 2);
-        assert(&r2 == &c.back());
+    A& r1 = c.emplace_back(2, 3.5);
+    assert(c.size() == 1);
+    assert(&r1 == &c.back());
+    assert(c.front().geti() == 2);
+    assert(c.front().getd() == 3.5);
+    assert(is_contiguous_container_asan_correct(c));
+    A& r2 = c.emplace_back(3, 4.5);
+    assert(c.size() == 2);
+    assert(&r2 == &c.back());
 #else
-        c.emplace_back(2, 3.5);
-        assert(c.size() == 1);
-        assert(c.front().geti() == 2);
-        assert(c.front().getd() == 3.5);
-        assert(is_contiguous_container_asan_correct(c));
-        c.emplace_back(3, 4.5);
-        assert(c.size() == 2);
+    c.emplace_back(2, 3.5);
+    assert(c.size() == 1);
+    assert(c.front().geti() == 2);
+    assert(c.front().getd() == 3.5);
+    assert(is_contiguous_container_asan_correct(c));
+    c.emplace_back(3, 4.5);
+    assert(c.size() == 2);
 #endif
-        assert(c.front().geti() == 2);
-        assert(c.front().getd() == 3.5);
-        assert(c.back().geti() == 3);
-        assert(c.back().getd() == 4.5);
-        assert(is_contiguous_container_asan_correct(c));
-    }
-    {
-        std::vector<A, limited_allocator<A, 4> > c;
+    assert(c.front().geti() == 2);
+    assert(c.front().getd() == 3.5);
+    assert(c.back().geti() == 3);
+    assert(c.back().getd() == 4.5);
+    assert(is_contiguous_container_asan_correct(c));
+  }
+  {
+    std::vector<A, limited_allocator<A, 4> > c;
 #if TEST_STD_VER > 14
-        A& r1 = c.emplace_back(2, 3.5);
-        assert(c.size() == 1);
-        assert(&r1 == &c.back());
-        assert(c.front().geti() == 2);
-        assert(c.front().getd() == 3.5);
-        assert(is_contiguous_container_asan_correct(c));
-        A& r2 = c.emplace_back(3, 4.5);
-        assert(c.size() == 2);
-        assert(&r2 == &c.back());
+    A& r1 = c.emplace_back(2, 3.5);
+    assert(c.size() == 1);
+    assert(&r1 == &c.back());
+    assert(c.front().geti() == 2);
+    assert(c.front().getd() == 3.5);
+    assert(is_contiguous_container_asan_correct(c));
+    A& r2 = c.emplace_back(3, 4.5);
+    assert(c.size() == 2);
+    assert(&r2 == &c.back());
 #else
-        c.emplace_back(2, 3.5);
-        assert(c.size() == 1);
-        assert(c.front().geti() == 2);
-        assert(c.front().getd() == 3.5);
-        assert(is_contiguous_container_asan_correct(c));
-        c.emplace_back(3, 4.5);
-        assert(c.size() == 2);
+    c.emplace_back(2, 3.5);
+    assert(c.size() == 1);
+    assert(c.front().geti() == 2);
+    assert(c.front().getd() == 3.5);
+    assert(is_contiguous_container_asan_correct(c));
+    c.emplace_back(3, 4.5);
+    assert(c.size() == 2);
 #endif
-        assert(c.front().geti() == 2);
-        assert(c.front().getd() == 3.5);
-        assert(c.back().geti() == 3);
-        assert(c.back().getd() == 4.5);
-        assert(is_contiguous_container_asan_correct(c));
-    }
-    {
-        std::vector<A, min_allocator<A> > c;
+    assert(c.front().geti() == 2);
+    assert(c.front().getd() == 3.5);
+    assert(c.back().geti() == 3);
+    assert(c.back().getd() == 4.5);
+    assert(is_contiguous_container_asan_correct(c));
+  }
+  {
+    std::vector<A, min_allocator<A> > c;
 #if TEST_STD_VER > 14
-        A& r1 = c.emplace_back(2, 3.5);
-        assert(c.size() == 1);
-        assert(&r1 == &c.back());
-        assert(c.front().geti() == 2);
-        assert(c.front().getd() == 3.5);
-        assert(is_contiguous_container_asan_correct(c));
-        A& r2 = c.emplace_back(3, 4.5);
-        assert(c.size() == 2);
-        assert(&r2 == &c.back());
+    A& r1 = c.emplace_back(2, 3.5);
+    assert(c.size() == 1);
+    assert(&r1 == &c.back());
+    assert(c.front().geti() == 2);
+    assert(c.front().getd() == 3.5);
+    assert(is_contiguous_container_asan_correct(c));
+    A& r2 = c.emplace_back(3, 4.5);
+    assert(c.size() == 2);
+    assert(&r2 == &c.back());
 #else
-        c.emplace_back(2, 3.5);
-        assert(c.size() == 1);
-        assert(c.front().geti() == 2);
-        assert(c.front().getd() == 3.5);
-        assert(is_contiguous_container_asan_correct(c));
-        c.emplace_back(3, 4.5);
-        assert(c.size() == 2);
+    c.emplace_back(2, 3.5);
+    assert(c.size() == 1);
+    assert(c.front().geti() == 2);
+    assert(c.front().getd() == 3.5);
+    assert(is_contiguous_container_asan_correct(c));
+    c.emplace_back(3, 4.5);
+    assert(c.size() == 2);
 #endif
-        assert(c.front().geti() == 2);
-        assert(c.front().getd() == 3.5);
-        assert(c.back().geti() == 3);
-        assert(c.back().getd() == 4.5);
-        assert(is_contiguous_container_asan_correct(c));
-    }
-    {
-      std::vector<A, safe_allocator<A> > c;
+    assert(c.front().geti() == 2);
+    assert(c.front().getd() == 3.5);
+    assert(c.back().geti() == 3);
+    assert(c.back().getd() == 4.5);
+    assert(is_contiguous_container_asan_correct(c));
+  }
+  {
+    std::vector<A, safe_allocator<A> > c;
 #if TEST_STD_VER > 14
-      A& r1 = c.emplace_back(2, 3.5);
-      assert(c.size() == 1);
-      assert(&r1 == &c.back());
-      assert(c.front().geti() == 2);
-      assert(c.front().getd() == 3.5);
-      assert(is_contiguous_container_asan_correct(c));
-      A& r2 = c.emplace_back(3, 4.5);
-      assert(c.size() == 2);
-      assert(&r2 == &c.back());
+    A& r1 = c.emplace_back(2, 3.5);
+    assert(c.size() == 1);
+    assert(&r1 == &c.back());
+    assert(c.front().geti() == 2);
+    assert(c.front().getd() == 3.5);
+    assert(is_contiguous_container_asan_correct(c));
+    A& r2 = c.emplace_back(3, 4.5);
+    assert(c.size() == 2);
+    assert(&r2 == &c.back());
 #else
-        c.emplace_back(2, 3.5);
-        assert(c.size() == 1);
-        assert(c.front().geti() == 2);
-        assert(c.front().getd() == 3.5);
-        assert(is_contiguous_container_asan_correct(c));
-        c.emplace_back(3, 4.5);
-        assert(c.size() == 2);
+    c.emplace_back(2, 3.5);
+    assert(c.size() == 1);
+    assert(c.front().geti() == 2);
+    assert(c.front().getd() == 3.5);
+    assert(is_contiguous_container_asan_correct(c));
+    c.emplace_back(3, 4.5);
+    assert(c.size() == 2);
 #endif
-        assert(c.front().geti() == 2);
-        assert(c.front().getd() == 3.5);
-        assert(c.back().geti() == 3);
-        assert(c.back().getd() == 4.5);
-        assert(is_contiguous_container_asan_correct(c));
-    }
-    {
-        std::vector<Tag_X, TaggingAllocator<Tag_X> > c;
-        c.emplace_back();
-        assert(c.size() == 1);
-        c.emplace_back(1, 2, 3);
-        assert(c.size() == 2);
-        assert(is_contiguous_container_asan_correct(c));
-    }
+    assert(c.front().geti() == 2);
+    assert(c.front().getd() == 3.5);
+    assert(c.back().geti() == 3);
+    assert(c.back().getd() == 4.5);
+    assert(is_contiguous_container_asan_correct(c));
+  }
+  {
+    std::vector<Tag_X, TaggingAllocator<Tag_X> > c;
+    c.emplace_back();
+    assert(c.size() == 1);
+    c.emplace_back(1, 2, 3);
+    assert(c.size() == 2);
+    assert(is_contiguous_container_asan_correct(c));
+  }
 
-    { // LWG 2164
-        int arr[] = {0, 1, 2, 3, 4};
-        int sz = 5;
-        std::vector<int> c(arr, arr+sz);
-        while (c.size() < c.capacity())
-            c.push_back(sz++);
-        c.emplace_back(c.front());
-        assert(c.back() == 0);
-        for (int i = 0; i < sz; ++i)
-            assert(c[i] == i);
-    }
-    return true;
+  { // LWG 2164
+    int arr[] = {0, 1, 2, 3, 4};
+    int sz    = 5;
+    std::vector<int> c(arr, arr + sz);
+    while (c.size() < c.capacity())
+      c.push_back(sz++);
+    c.emplace_back(c.front());
+    assert(c.back() == 0);
+    for (int i = 0; i < sz; ++i)
+      assert(c[i] == i);
+  }
+  return true;
 }
 
-int main(int, char**)
-{
-    tests();
+int main(int, char**) {
+  tests();
 #if TEST_STD_VER > 17
-    static_assert(tests());
+  static_assert(tests());
 #endif
-    return 0;
+  return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/vector/vector.modifiers/emplace_extra.pass.cpp b/libcxx/test/std/containers/sequences/vector/vector.modifiers/emplace_extra.pass.cpp
index f186a0d97ae39..43990b148cad6 100644
--- a/libcxx/test/std/containers/sequences/vector/vector.modifiers/emplace_extra.pass.cpp
+++ b/libcxx/test/std/containers/sequences/vector/vector.modifiers/emplace_extra.pass.cpp
@@ -20,82 +20,82 @@
 #include "asan_testing.h"
 
 TEST_CONSTEXPR_CXX20 bool tests() {
-    {
-        std::vector<int> v;
-        v.reserve(3);
-        assert(is_contiguous_container_asan_correct(v));
-        v = { 1, 2, 3 };
-        v.emplace(v.begin(), v.back());
-        assert(v[0] == 3);
-        assert(is_contiguous_container_asan_correct(v));
-    }
-    {
-        std::vector<int> v;
-        v.reserve(4);
-        assert(is_contiguous_container_asan_correct(v));
-        v = { 1, 2, 3 };
-        v.emplace(v.begin(), v.back());
-        assert(v[0] == 3);
-        assert(is_contiguous_container_asan_correct(v));
-    }
-    {
-        std::vector<int, min_allocator<int>> v;
-        v.reserve(3);
-        assert(is_contiguous_container_asan_correct(v));
-        v = { 1, 2, 3 };
-        v.emplace(v.begin(), v.back());
-        assert(v[0] == 3);
-        assert(is_contiguous_container_asan_correct(v));
-    }
-    {
-        std::vector<int, min_allocator<int>> v;
-        v.reserve(4);
-        assert(is_contiguous_container_asan_correct(v));
-        v = { 1, 2, 3 };
-        v.emplace(v.begin(), v.back());
-        assert(v[0] == 3);
-        assert(is_contiguous_container_asan_correct(v));
-    }
-    {
-      std::vector<int, safe_allocator<int>> v;
-      v.reserve(3);
-      assert(is_contiguous_container_asan_correct(v));
-      v = {1, 2, 3};
-      v.emplace(v.begin(), v.back());
-      assert(v[0] == 3);
-      assert(is_contiguous_container_asan_correct(v));
-    }
-    {
-      std::vector<int, safe_allocator<int>> v;
-      v.reserve(4);
-      assert(is_contiguous_container_asan_correct(v));
-      v = {1, 2, 3};
-      v.emplace(v.begin(), v.back());
-      assert(v[0] == 3);
-      assert(is_contiguous_container_asan_correct(v));
-    }
-    {
-        std::vector<int> v;
-        v.reserve(8);
-        std::size_t old_capacity = v.capacity();
-        assert(old_capacity >= 8);
+  {
+    std::vector<int> v;
+    v.reserve(3);
+    assert(is_contiguous_container_asan_correct(v));
+    v = {1, 2, 3};
+    v.emplace(v.begin(), v.back());
+    assert(v[0] == 3);
+    assert(is_contiguous_container_asan_correct(v));
+  }
+  {
+    std::vector<int> v;
+    v.reserve(4);
+    assert(is_contiguous_container_asan_correct(v));
+    v = {1, 2, 3};
+    v.emplace(v.begin(), v.back());
+    assert(v[0] == 3);
+    assert(is_contiguous_container_asan_correct(v));
+  }
+  {
+    std::vector<int, min_allocator<int>> v;
+    v.reserve(3);
+    assert(is_contiguous_container_asan_correct(v));
+    v = {1, 2, 3};
+    v.emplace(v.begin(), v.back());
+    assert(v[0] == 3);
+    assert(is_contiguous_container_asan_correct(v));
+  }
+  {
+    std::vector<int, min_allocator<int>> v;
+    v.reserve(4);
+    assert(is_contiguous_container_asan_correct(v));
+    v = {1, 2, 3};
+    v.emplace(v.begin(), v.back());
+    assert(v[0] == 3);
+    assert(is_contiguous_container_asan_correct(v));
+  }
+  {
+    std::vector<int, safe_allocator<int>> v;
+    v.reserve(3);
+    assert(is_contiguous_container_asan_correct(v));
+    v = {1, 2, 3};
+    v.emplace(v.begin(), v.back());
+    assert(v[0] == 3);
+    assert(is_contiguous_container_asan_correct(v));
+  }
+  {
+    std::vector<int, safe_allocator<int>> v;
+    v.reserve(4);
+    assert(is_contiguous_container_asan_correct(v));
+    v = {1, 2, 3};
+    v.emplace(v.begin(), v.back());
+    assert(v[0] == 3);
+    assert(is_contiguous_container_asan_correct(v));
+  }
+  {
+    std::vector<int> v;
+    v.reserve(8);
+    std::size_t old_capacity = v.capacity();
+    assert(old_capacity >= 8);
 
-        v.resize(4); // keep the existing capacity
-        assert(v.capacity() == old_capacity);
+    v.resize(4); // keep the existing capacity
+    assert(v.capacity() == old_capacity);
 
-        v.emplace(v.cend(), 42);
-        assert(v.size() == 5);
-        assert(v.capacity() == old_capacity);
-        assert(v[4] == 42);
-    }
+    v.emplace(v.cend(), 42);
+    assert(v.size() == 5);
+    assert(v.capacity() == old_capacity);
+    assert(v[4] == 42);
+  }
 
-    return true;
+  return true;
 }
 
 int main(int, char**) {
-    tests();
+  tests();
 #if TEST_STD_VER > 17
-    static_assert(tests());
+  static_assert(tests());
 #endif
-    return 0;
+  return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/vector/vector.modifiers/erase_iter.pass.cpp b/libcxx/test/std/containers/sequences/vector/vector.modifiers/erase_iter.pass.cpp
index f0157eb74b90f..0c65bd1034bb8 100644
--- a/libcxx/test/std/containers/sequences/vector/vector.modifiers/erase_iter.pass.cpp
+++ b/libcxx/test/std/containers/sequences/vector/vector.modifiers/erase_iter.pass.cpp
@@ -23,9 +23,9 @@
 template <template <class> class Allocator, class T>
 TEST_CONSTEXPR_CXX20 void tests() {
   {
-    T arr[]             = {T(1), T(2), T(3), T(4), T(5)};
-    using Vector        = std::vector<T, Allocator<T> >;
-    using Iterator      = typename Vector::iterator;
+    T arr[]        = {T(1), T(2), T(3), T(4), T(5)};
+    using Vector   = std::vector<T, Allocator<T> >;
+    using Iterator = typename Vector::iterator;
 
     {
       Vector v(arr, arr + 5);
diff --git a/libcxx/test/std/containers/sequences/vector/vector.modifiers/insert_iter_initializer_list.pass.cpp b/libcxx/test/std/containers/sequences/vector/vector.modifiers/insert_iter_initializer_list.pass.cpp
index 92584892001a6..12d189896df8d 100644
--- a/libcxx/test/std/containers/sequences/vector/vector.modifiers/insert_iter_initializer_list.pass.cpp
+++ b/libcxx/test/std/containers/sequences/vector/vector.modifiers/insert_iter_initializer_list.pass.cpp
@@ -54,8 +54,7 @@ void test_throwing() {
 }
 #endif // TEST_HAS_NO_EXCEPTIONS
 
-TEST_CONSTEXPR_CXX20 bool tests()
-{
+TEST_CONSTEXPR_CXX20 bool tests() {
   {
     std::vector<int> d(10, 1);
     std::vector<int>::iterator i = d.insert(d.cbegin() + 2, {3, 4, 5, 6});
@@ -99,7 +98,7 @@ TEST_CONSTEXPR_CXX20 bool tests()
     assert(d[13] == 1);
   }
 
-    return true;
+  return true;
 }
 
 int main(int, char**) {
diff --git a/libcxx/test/std/containers/sequences/vector/vector.modifiers/insert_iter_lvalue.pass.cpp b/libcxx/test/std/containers/sequences/vector/vector.modifiers/insert_iter_lvalue.pass.cpp
index b5582df5bd2c4..93a3ce3918615 100644
--- a/libcxx/test/std/containers/sequences/vector/vector.modifiers/insert_iter_lvalue.pass.cpp
+++ b/libcxx/test/std/containers/sequences/vector/vector.modifiers/insert_iter_lvalue.pass.cpp
@@ -20,124 +20,125 @@
 #include "asan_testing.h"
 
 TEST_CONSTEXPR_CXX20 bool test() {
+  {
+    std::vector<int> v(100);
+    const int lvalue             = 1;
+    std::vector<int>::iterator i = v.insert(v.cbegin() + 10, lvalue);
+    assert(v.size() == 101);
+    assert(is_contiguous_container_asan_correct(v));
+    assert(i == v.begin() + 10);
+    int j;
+    for (j = 0; j < 10; ++j)
+      assert(v[j] == 0);
+    assert(v[j] == 1);
+    for (++j; j < 101; ++j)
+      assert(v[j] == 0);
+  }
+  {
+    const std::size_t n = 100;
+    std::vector<int> v(n);
+    v.reserve(n + 1);
+    const int lvalue = 1;
 
-    {
-        std::vector<int> v(100);
-        const int lvalue = 1;
-        std::vector<int>::iterator i = v.insert(v.cbegin() + 10, lvalue);
-        assert(v.size() == 101);
-        assert(is_contiguous_container_asan_correct(v));
-        assert(i == v.begin() + 10);
-        int j;
-        for (j = 0; j < 10; ++j)
-            assert(v[j] == 0);
-        assert(v[j] == 1);
-        for (++j; j < 101; ++j)
-            assert(v[j] == 0);
-    }
-    {
-        const std::size_t n = 100;
-        std::vector<int> v(n);
-        v.reserve(n + 1);
-        const int lvalue = 1;
-
-        // no reallocation expected
-        std::vector<int>::iterator it = v.insert(v.cbegin() + n, lvalue);
+    // no reallocation expected
+    std::vector<int>::iterator it = v.insert(v.cbegin() + n, lvalue);
 
-        assert(v.size() == n + 1);
-        assert(is_contiguous_container_asan_correct(v));
-        assert(it == v.begin() + n);
-        for (std::size_t i = 0; i < n; ++i) {
-            assert(v[i] == 0);
-        }
-        assert(v[n] == lvalue);
-    }
-    {
-        std::vector<int> v(100);
-        while(v.size() < v.capacity()) v.push_back(0); // force reallocation
-        std::size_t sz = v.size();
-        const int lvalue = 1;
-        std::vector<int>::iterator i = v.insert(v.cbegin() + 10, lvalue);
-        assert(v.size() == sz + 1);
-        assert(is_contiguous_container_asan_correct(v));
-        assert(i == v.begin() + 10);
-        std::size_t j;
-        for (j = 0; j < 10; ++j)
-            assert(v[j] == 0);
-        assert(v[j] == 1);
-        for (++j; j < v.size(); ++j)
-            assert(v[j] == 0);
-    }
-    {
-        std::vector<int> v(100);
-        while(v.size() < v.capacity()) v.push_back(0);
-        v.pop_back(); v.pop_back(); // force no reallocation
-        std::size_t sz = v.size();
-        const int lvalue = 1;
-        std::vector<int>::iterator i = v.insert(v.cbegin() + 10, lvalue);
-        assert(v.size() == sz + 1);
-        assert(is_contiguous_container_asan_correct(v));
-        assert(i == v.begin() + 10);
-        std::size_t j;
-        for (j = 0; j < 10; ++j)
-            assert(v[j] == 0);
-        assert(v[j] == 1);
-        for (++j; j < v.size(); ++j)
-            assert(v[j] == 0);
-    }
-    {
-        std::vector<int, limited_allocator<int, 300> > v(100);
-        const int lvalue = 1;
-        std::vector<int, limited_allocator<int, 300> >::iterator i = v.insert(v.cbegin() + 10, lvalue);
-        assert(v.size() == 101);
-        assert(is_contiguous_container_asan_correct(v));
-        assert(i == v.begin() + 10);
-        int j;
-        for (j = 0; j < 10; ++j)
-            assert(v[j] == 0);
-        assert(v[j] == 1);
-        for (++j; j < 101; ++j)
-            assert(v[j] == 0);
+    assert(v.size() == n + 1);
+    assert(is_contiguous_container_asan_correct(v));
+    assert(it == v.begin() + n);
+    for (std::size_t i = 0; i < n; ++i) {
+      assert(v[i] == 0);
     }
+    assert(v[n] == lvalue);
+  }
+  {
+    std::vector<int> v(100);
+    while (v.size() < v.capacity())
+      v.push_back(0); // force reallocation
+    std::size_t sz               = v.size();
+    const int lvalue             = 1;
+    std::vector<int>::iterator i = v.insert(v.cbegin() + 10, lvalue);
+    assert(v.size() == sz + 1);
+    assert(is_contiguous_container_asan_correct(v));
+    assert(i == v.begin() + 10);
+    std::size_t j;
+    for (j = 0; j < 10; ++j)
+      assert(v[j] == 0);
+    assert(v[j] == 1);
+    for (++j; j < v.size(); ++j)
+      assert(v[j] == 0);
+  }
+  {
+    std::vector<int> v(100);
+    while (v.size() < v.capacity())
+      v.push_back(0);
+    v.pop_back();
+    v.pop_back(); // force no reallocation
+    std::size_t sz               = v.size();
+    const int lvalue             = 1;
+    std::vector<int>::iterator i = v.insert(v.cbegin() + 10, lvalue);
+    assert(v.size() == sz + 1);
+    assert(is_contiguous_container_asan_correct(v));
+    assert(i == v.begin() + 10);
+    std::size_t j;
+    for (j = 0; j < 10; ++j)
+      assert(v[j] == 0);
+    assert(v[j] == 1);
+    for (++j; j < v.size(); ++j)
+      assert(v[j] == 0);
+  }
+  {
+    std::vector<int, limited_allocator<int, 300> > v(100);
+    const int lvalue                                           = 1;
+    std::vector<int, limited_allocator<int, 300> >::iterator i = v.insert(v.cbegin() + 10, lvalue);
+    assert(v.size() == 101);
+    assert(is_contiguous_container_asan_correct(v));
+    assert(i == v.begin() + 10);
+    int j;
+    for (j = 0; j < 10; ++j)
+      assert(v[j] == 0);
+    assert(v[j] == 1);
+    for (++j; j < 101; ++j)
+      assert(v[j] == 0);
+  }
 #if TEST_STD_VER >= 11
-    {
-        std::vector<int, min_allocator<int>> v(100);
-        const int lvalue = 1;
-        std::vector<int, min_allocator<int>>::iterator i = v.insert(v.cbegin() + 10, lvalue);
-        assert(v.size() == 101);
-        assert(is_contiguous_container_asan_correct(v));
-        assert(i == v.begin() + 10);
-        int j;
-        for (j = 0; j < 10; ++j)
-            assert(v[j] == 0);
-        assert(v[j] == 1);
-        for (++j; j < 101; ++j)
-            assert(v[j] == 0);
-    }
-    {
-      std::vector<int, safe_allocator<int>> v(100);
-      const int lvalue                                  = 1;
-      std::vector<int, safe_allocator<int>>::iterator i = v.insert(v.cbegin() + 10, lvalue);
-      assert(v.size() == 101);
-      assert(is_contiguous_container_asan_correct(v));
-      assert(i == v.begin() + 10);
-      int j;
-      for (j = 0; j < 10; ++j)
-        assert(v[j] == 0);
-      assert(v[j] == 1);
-      for (++j; j < 101; ++j)
-        assert(v[j] == 0);
-    }
+  {
+    std::vector<int, min_allocator<int>> v(100);
+    const int lvalue                                 = 1;
+    std::vector<int, min_allocator<int>>::iterator i = v.insert(v.cbegin() + 10, lvalue);
+    assert(v.size() == 101);
+    assert(is_contiguous_container_asan_correct(v));
+    assert(i == v.begin() + 10);
+    int j;
+    for (j = 0; j < 10; ++j)
+      assert(v[j] == 0);
+    assert(v[j] == 1);
+    for (++j; j < 101; ++j)
+      assert(v[j] == 0);
+  }
+  {
+    std::vector<int, safe_allocator<int>> v(100);
+    const int lvalue                                  = 1;
+    std::vector<int, safe_allocator<int>>::iterator i = v.insert(v.cbegin() + 10, lvalue);
+    assert(v.size() == 101);
+    assert(is_contiguous_container_asan_correct(v));
+    assert(i == v.begin() + 10);
+    int j;
+    for (j = 0; j < 10; ++j)
+      assert(v[j] == 0);
+    assert(v[j] == 1);
+    for (++j; j < 101; ++j)
+      assert(v[j] == 0);
+  }
 #endif
 
-    return true;
+  return true;
 }
 
-int main(int, char**)
-{
-    test();
+int main(int, char**) {
+  test();
 #if TEST_STD_VER > 17
-    static_assert(test());
+  static_assert(test());
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/sequences/vector/vector.modifiers/insert_iter_rvalue.pass.cpp b/libcxx/test/std/containers/sequences/vector/vector.modifiers/insert_iter_rvalue.pass.cpp
index 4619d7a7d1b6f..570e182cb6689 100644
--- a/libcxx/test/std/containers/sequences/vector/vector.modifiers/insert_iter_rvalue.pass.cpp
+++ b/libcxx/test/std/containers/sequences/vector/vector.modifiers/insert_iter_rvalue.pass.cpp
@@ -21,69 +21,67 @@
 #include "min_allocator.h"
 #include "asan_testing.h"
 
-TEST_CONSTEXPR_CXX20 bool tests()
-{
-    {
-        std::vector<MoveOnly> v(100);
-        std::vector<MoveOnly>::iterator i = v.insert(v.cbegin() + 10, MoveOnly(3));
-        assert(v.size() == 101);
-        assert(is_contiguous_container_asan_correct(v));
-        assert(i == v.begin() + 10);
-        int j;
-        for (j = 0; j < 10; ++j)
-            assert(v[j] == MoveOnly());
-        assert(v[j] == MoveOnly(3));
-        for (++j; j < 101; ++j)
-            assert(v[j] == MoveOnly());
-    }
-    {
-        std::vector<MoveOnly, limited_allocator<MoveOnly, 300> > v(100);
-        std::vector<MoveOnly, limited_allocator<MoveOnly, 300> >::iterator i = v.insert(v.cbegin() + 10, MoveOnly(3));
-        assert(v.size() == 101);
-        assert(is_contiguous_container_asan_correct(v));
-        assert(i == v.begin() + 10);
-        int j;
-        for (j = 0; j < 10; ++j)
-            assert(v[j] == MoveOnly());
-        assert(v[j] == MoveOnly(3));
-        for (++j; j < 101; ++j)
-            assert(v[j] == MoveOnly());
-    }
-    {
-        std::vector<MoveOnly, min_allocator<MoveOnly> > v(100);
-        std::vector<MoveOnly, min_allocator<MoveOnly> >::iterator i = v.insert(v.cbegin() + 10, MoveOnly(3));
-        assert(v.size() == 101);
-        assert(is_contiguous_container_asan_correct(v));
-        assert(i == v.begin() + 10);
-        int j;
-        for (j = 0; j < 10; ++j)
-            assert(v[j] == MoveOnly());
-        assert(v[j] == MoveOnly(3));
-        for (++j; j < 101; ++j)
-            assert(v[j] == MoveOnly());
-    }
-    {
-      std::vector<MoveOnly, safe_allocator<MoveOnly> > v(100);
-      std::vector<MoveOnly, safe_allocator<MoveOnly> >::iterator i = v.insert(v.cbegin() + 10, MoveOnly(3));
-      assert(v.size() == 101);
-      assert(is_contiguous_container_asan_correct(v));
-      assert(i == v.begin() + 10);
-      int j;
-      for (j = 0; j < 10; ++j)
-        assert(v[j] == MoveOnly());
-      assert(v[j] == MoveOnly(3));
-      for (++j; j < 101; ++j)
-        assert(v[j] == MoveOnly());
-    }
+TEST_CONSTEXPR_CXX20 bool tests() {
+  {
+    std::vector<MoveOnly> v(100);
+    std::vector<MoveOnly>::iterator i = v.insert(v.cbegin() + 10, MoveOnly(3));
+    assert(v.size() == 101);
+    assert(is_contiguous_container_asan_correct(v));
+    assert(i == v.begin() + 10);
+    int j;
+    for (j = 0; j < 10; ++j)
+      assert(v[j] == MoveOnly());
+    assert(v[j] == MoveOnly(3));
+    for (++j; j < 101; ++j)
+      assert(v[j] == MoveOnly());
+  }
+  {
+    std::vector<MoveOnly, limited_allocator<MoveOnly, 300> > v(100);
+    std::vector<MoveOnly, limited_allocator<MoveOnly, 300> >::iterator i = v.insert(v.cbegin() + 10, MoveOnly(3));
+    assert(v.size() == 101);
+    assert(is_contiguous_container_asan_correct(v));
+    assert(i == v.begin() + 10);
+    int j;
+    for (j = 0; j < 10; ++j)
+      assert(v[j] == MoveOnly());
+    assert(v[j] == MoveOnly(3));
+    for (++j; j < 101; ++j)
+      assert(v[j] == MoveOnly());
+  }
+  {
+    std::vector<MoveOnly, min_allocator<MoveOnly> > v(100);
+    std::vector<MoveOnly, min_allocator<MoveOnly> >::iterator i = v.insert(v.cbegin() + 10, MoveOnly(3));
+    assert(v.size() == 101);
+    assert(is_contiguous_container_asan_correct(v));
+    assert(i == v.begin() + 10);
+    int j;
+    for (j = 0; j < 10; ++j)
+      assert(v[j] == MoveOnly());
+    assert(v[j] == MoveOnly(3));
+    for (++j; j < 101; ++j)
+      assert(v[j] == MoveOnly());
+  }
+  {
+    std::vector<MoveOnly, safe_allocator<MoveOnly> > v(100);
+    std::vector<MoveOnly, safe_allocator<MoveOnly> >::iterator i = v.insert(v.cbegin() + 10, MoveOnly(3));
+    assert(v.size() == 101);
+    assert(is_contiguous_container_asan_correct(v));
+    assert(i == v.begin() + 10);
+    int j;
+    for (j = 0; j < 10; ++j)
+      assert(v[j] == MoveOnly());
+    assert(v[j] == MoveOnly(3));
+    for (++j; j < 101; ++j)
+      assert(v[j] == MoveOnly());
+  }
 
-    return true;
+  return true;
 }
 
-int main(int, char**)
-{
-    tests();
+int main(int, char**) {
+  tests();
 #if TEST_STD_VER > 17
-    static_assert(tests());
+  static_assert(tests());
 #endif
-    return 0;
+  return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/vector/vector.modifiers/insert_iter_size_value.pass.cpp b/libcxx/test/std/containers/sequences/vector/vector.modifiers/insert_iter_size_value.pass.cpp
index 8325fcfd0670f..8173285d4996b 100644
--- a/libcxx/test/std/containers/sequences/vector/vector.modifiers/insert_iter_size_value.pass.cpp
+++ b/libcxx/test/std/containers/sequences/vector/vector.modifiers/insert_iter_size_value.pass.cpp
@@ -19,107 +19,106 @@
 #include "min_allocator.h"
 #include "asan_testing.h"
 
-TEST_CONSTEXPR_CXX20 bool tests()
-{
-    {
-        std::vector<int> v(100);
-        std::vector<int>::iterator i = v.insert(v.cbegin() + 10, 5, 1);
-        assert(v.size() == 105);
-        assert(is_contiguous_container_asan_correct(v));
-        assert(i == v.begin() + 10);
-        int j;
-        for (j = 0; j < 10; ++j)
-            assert(v[j] == 0);
-        for (; j < 15; ++j)
-            assert(v[j] == 1);
-        for (++j; j < 105; ++j)
-            assert(v[j] == 0);
-    }
-    {
-        std::vector<int> v(100);
-        while(v.size() < v.capacity()) v.push_back(0); // force reallocation
-        std::size_t sz = v.size();
-        std::vector<int>::iterator i = v.insert(v.cbegin() + 10, 5, 1);
-        assert(v.size() == sz + 5);
-        assert(is_contiguous_container_asan_correct(v));
-        assert(i == v.begin() + 10);
-        std::size_t j;
-        for (j = 0; j < 10; ++j)
-            assert(v[j] == 0);
-        for (; j < 15; ++j)
-            assert(v[j] == 1);
-        for (++j; j < v.size(); ++j)
-            assert(v[j] == 0);
-    }
-    {
-        std::vector<int> v(100);
-        v.reserve(128); // force no reallocation
-        std::size_t sz = v.size();
-        std::vector<int>::iterator i = v.insert(v.cbegin() + 10, 5, 1);
-        assert(v.size() == sz + 5);
-        assert(is_contiguous_container_asan_correct(v));
-        assert(i == v.begin() + 10);
-        std::size_t j;
-        for (j = 0; j < 10; ++j)
-            assert(v[j] == 0);
-        for (; j < 15; ++j)
-            assert(v[j] == 1);
-        for (++j; j < v.size(); ++j)
-            assert(v[j] == 0);
-    }
-    {
-        std::vector<int, limited_allocator<int, 300> > v(100);
-        std::vector<int, limited_allocator<int, 300> >::iterator i = v.insert(v.cbegin() + 10, 5, 1);
-        assert(v.size() == 105);
-        assert(is_contiguous_container_asan_correct(v));
-        assert(i == v.begin() + 10);
-        int j;
-        for (j = 0; j < 10; ++j)
-            assert(v[j] == 0);
-        for (; j < 15; ++j)
-            assert(v[j] == 1);
-        for (++j; j < 105; ++j)
-            assert(v[j] == 0);
-    }
+TEST_CONSTEXPR_CXX20 bool tests() {
+  {
+    std::vector<int> v(100);
+    std::vector<int>::iterator i = v.insert(v.cbegin() + 10, 5, 1);
+    assert(v.size() == 105);
+    assert(is_contiguous_container_asan_correct(v));
+    assert(i == v.begin() + 10);
+    int j;
+    for (j = 0; j < 10; ++j)
+      assert(v[j] == 0);
+    for (; j < 15; ++j)
+      assert(v[j] == 1);
+    for (++j; j < 105; ++j)
+      assert(v[j] == 0);
+  }
+  {
+    std::vector<int> v(100);
+    while (v.size() < v.capacity())
+      v.push_back(0); // force reallocation
+    std::size_t sz               = v.size();
+    std::vector<int>::iterator i = v.insert(v.cbegin() + 10, 5, 1);
+    assert(v.size() == sz + 5);
+    assert(is_contiguous_container_asan_correct(v));
+    assert(i == v.begin() + 10);
+    std::size_t j;
+    for (j = 0; j < 10; ++j)
+      assert(v[j] == 0);
+    for (; j < 15; ++j)
+      assert(v[j] == 1);
+    for (++j; j < v.size(); ++j)
+      assert(v[j] == 0);
+  }
+  {
+    std::vector<int> v(100);
+    v.reserve(128); // force no reallocation
+    std::size_t sz               = v.size();
+    std::vector<int>::iterator i = v.insert(v.cbegin() + 10, 5, 1);
+    assert(v.size() == sz + 5);
+    assert(is_contiguous_container_asan_correct(v));
+    assert(i == v.begin() + 10);
+    std::size_t j;
+    for (j = 0; j < 10; ++j)
+      assert(v[j] == 0);
+    for (; j < 15; ++j)
+      assert(v[j] == 1);
+    for (++j; j < v.size(); ++j)
+      assert(v[j] == 0);
+  }
+  {
+    std::vector<int, limited_allocator<int, 300> > v(100);
+    std::vector<int, limited_allocator<int, 300> >::iterator i = v.insert(v.cbegin() + 10, 5, 1);
+    assert(v.size() == 105);
+    assert(is_contiguous_container_asan_correct(v));
+    assert(i == v.begin() + 10);
+    int j;
+    for (j = 0; j < 10; ++j)
+      assert(v[j] == 0);
+    for (; j < 15; ++j)
+      assert(v[j] == 1);
+    for (++j; j < 105; ++j)
+      assert(v[j] == 0);
+  }
 #if TEST_STD_VER >= 11
-    {
-        std::vector<int, min_allocator<int>> v(100);
-        std::vector<int, min_allocator<int>>::iterator i = v.insert(v.cbegin() + 10, 5, 1);
-        assert(v.size() == 105);
-        assert(is_contiguous_container_asan_correct(v));
-        assert(i == v.begin() + 10);
-        int j;
-        for (j = 0; j < 10; ++j)
-            assert(v[j] == 0);
-        for (; j < 15; ++j)
-            assert(v[j] == 1);
-        for (++j; j < 105; ++j)
-            assert(v[j] == 0);
-    }
-    {
-      std::vector<int, safe_allocator<int>> v(100);
-      std::vector<int, safe_allocator<int>>::iterator i = v.insert(v.cbegin() + 10, 5, 1);
-      assert(v.size() == 105);
-      assert(is_contiguous_container_asan_correct(v));
-      assert(i == v.begin() + 10);
-      int j;
-      for (j = 0; j < 10; ++j)
-        assert(v[j] == 0);
-      for (; j < 15; ++j)
-        assert(v[j] == 1);
-      for (++j; j < 105; ++j)
-        assert(v[j] == 0);
-    }
+  {
+    std::vector<int, min_allocator<int>> v(100);
+    std::vector<int, min_allocator<int>>::iterator i = v.insert(v.cbegin() + 10, 5, 1);
+    assert(v.size() == 105);
+    assert(is_contiguous_container_asan_correct(v));
+    assert(i == v.begin() + 10);
+    int j;
+    for (j = 0; j < 10; ++j)
+      assert(v[j] == 0);
+    for (; j < 15; ++j)
+      assert(v[j] == 1);
+    for (++j; j < 105; ++j)
+      assert(v[j] == 0);
+  }
+  {
+    std::vector<int, safe_allocator<int>> v(100);
+    std::vector<int, safe_allocator<int>>::iterator i = v.insert(v.cbegin() + 10, 5, 1);
+    assert(v.size() == 105);
+    assert(is_contiguous_container_asan_correct(v));
+    assert(i == v.begin() + 10);
+    int j;
+    for (j = 0; j < 10; ++j)
+      assert(v[j] == 0);
+    for (; j < 15; ++j)
+      assert(v[j] == 1);
+    for (++j; j < 105; ++j)
+      assert(v[j] == 0);
+  }
 #endif
 
-    return true;
+  return true;
 }
 
-int main(int, char**)
-{
-    tests();
+int main(int, char**) {
+  tests();
 #if TEST_STD_VER > 17
-    static_assert(tests());
+  static_assert(tests());
 #endif
-    return 0;
+  return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/vector/vector.modifiers/insert_range.pass.cpp b/libcxx/test/std/containers/sequences/vector/vector.modifiers/insert_range.pass.cpp
index 25ffc33aad424..ea0573df73751 100644
--- a/libcxx/test/std/containers/sequences/vector/vector.modifiers/insert_range.pass.cpp
+++ b/libcxx/test/std/containers/sequences/vector/vector.modifiers/insert_range.pass.cpp
@@ -34,9 +34,9 @@ constexpr bool test() {
   });
   test_sequence_insert_range_move_only<std::vector>();
 
-  { // Vector may or may not need to reallocate because of the insertion -- make sure to test both cases.
+  {   // Vector may or may not need to reallocate because of the insertion -- make sure to test both cases.
     { // Ensure reallocation happens.
-      int in[] = {-1, -2, -3, -4, -5, -6, -7, -8, -9, -10};
+      int in[]           = {-1, -2, -3, -4, -5, -6, -7, -8, -9, -10};
       std::vector<int> v = {1, 2, 3, 4, 5, 6, 7, 8};
       v.shrink_to_fit();
       assert(v.capacity() < v.size() + std::ranges::size(in));
@@ -46,7 +46,7 @@ constexpr bool test() {
     }
 
     { // Ensure no reallocation happens.
-      int in[] = {-1, -2, -3, -4, -5, -6, -7, -8, -9, -10};
+      int in[]           = {-1, -2, -3, -4, -5, -6, -7, -8, -9, -10};
       std::vector<int> v = {1, 2, 3, 4, 5, 6, 7, 8};
       v.reserve(v.size() + std::ranges::size(in));
       assert(v.capacity() >= v.size() + std::ranges::size(in));
diff --git a/libcxx/test/std/containers/sequences/vector/vector.modifiers/pop_back.pass.cpp b/libcxx/test/std/containers/sequences/vector/vector.modifiers/pop_back.pass.cpp
index 9482920f0fe60..9717666079d80 100644
--- a/libcxx/test/std/containers/sequences/vector/vector.modifiers/pop_back.pass.cpp
+++ b/libcxx/test/std/containers/sequences/vector/vector.modifiers/pop_back.pass.cpp
@@ -17,47 +17,43 @@
 #include "test_allocator.h"
 #include "min_allocator.h"
 
-
-TEST_CONSTEXPR_CXX20 bool tests()
-{
-    {
-        std::vector<int> c;
-        c.push_back(1);
-        assert(c.size() == 1);
-        c.pop_back();
-        assert(c.size() == 0);
-
-    }
+TEST_CONSTEXPR_CXX20 bool tests() {
+  {
+    std::vector<int> c;
+    c.push_back(1);
+    assert(c.size() == 1);
+    c.pop_back();
+    assert(c.size() == 0);
+  }
 #if TEST_STD_VER >= 11
-    {
-        std::vector<int, min_allocator<int>> c;
-        c.push_back(1);
-        assert(c.size() == 1);
-        c.pop_back();
-        assert(c.size() == 0);
-    }
+  {
+    std::vector<int, min_allocator<int>> c;
+    c.push_back(1);
+    assert(c.size() == 1);
+    c.pop_back();
+    assert(c.size() == 0);
+  }
 #endif
 
-    { // LWG 526
-        int arr[] = {0, 1, 2, 3, 4};
-        int sz = 5;
-        std::vector<int> c(arr, arr+sz);
-        while (c.size() < c.capacity())
-            c.push_back(sz++);
-        c.push_back(c.front());
-        assert(c.back() == 0);
-        for (int i = 0; i < sz; ++i)
-            assert(c[i] == i);
-    }
-
-    return true;
+  { // LWG 526
+    int arr[] = {0, 1, 2, 3, 4};
+    int sz    = 5;
+    std::vector<int> c(arr, arr + sz);
+    while (c.size() < c.capacity())
+      c.push_back(sz++);
+    c.push_back(c.front());
+    assert(c.back() == 0);
+    for (int i = 0; i < sz; ++i)
+      assert(c[i] == i);
+  }
+
+  return true;
 }
 
-int main(int, char**)
-{
-    tests();
+int main(int, char**) {
+  tests();
 #if TEST_STD_VER > 17
-    static_assert(tests());
+  static_assert(tests());
 #endif
-    return 0;
+  return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/vector/vector.modifiers/push_back.pass.cpp b/libcxx/test/std/containers/sequences/vector/vector.modifiers/push_back.pass.cpp
index 2490fded523d6..1e77055abd6d3 100644
--- a/libcxx/test/std/containers/sequences/vector/vector.modifiers/push_back.pass.cpp
+++ b/libcxx/test/std/containers/sequences/vector/vector.modifiers/push_back.pass.cpp
@@ -18,106 +18,104 @@
 #include "min_allocator.h"
 #include "asan_testing.h"
 
-TEST_CONSTEXPR_CXX20 bool tests()
-{
-    {
-        std::vector<int> c;
-        c.push_back(0);
-        assert(c.size() == 1);
-        assert(is_contiguous_container_asan_correct(c));
-        for (int j = 0; static_cast<std::size_t>(j) < c.size(); ++j)
-            assert(c[j] == j);
-        c.push_back(1);
-        assert(c.size() == 2);
-        assert(is_contiguous_container_asan_correct(c));
-        for (int j = 0; static_cast<std::size_t>(j) < c.size(); ++j)
-            assert(c[j] == j);
-        c.push_back(2);
-        assert(c.size() == 3);
-        assert(is_contiguous_container_asan_correct(c));
-        for (int j = 0; static_cast<std::size_t>(j) < c.size(); ++j)
-            assert(c[j] == j);
-        c.push_back(3);
-        assert(c.size() == 4);
-        assert(is_contiguous_container_asan_correct(c));
-        for (int j = 0; static_cast<std::size_t>(j) < c.size(); ++j)
-            assert(c[j] == j);
-        c.push_back(4);
-        assert(c.size() == 5);
-        assert(is_contiguous_container_asan_correct(c));
-        for (int j = 0; static_cast<std::size_t>(j) < c.size(); ++j)
-            assert(c[j] == j);
-    }
-    {
-        // libc++ needs 15 because it grows by 2x (1 + 2 + 4 + 8).
-        // Use 17 for implementations that dynamically allocate a container proxy
-        // and grow by 1.5x (1 for proxy + 1 + 2 + 3 + 4 + 6).
-        std::vector<int, limited_allocator<int, 17> > c;
-        c.push_back(0);
-        assert(c.size() == 1);
-        assert(is_contiguous_container_asan_correct(c));
-        for (int j = 0; static_cast<std::size_t>(j) < c.size(); ++j)
-            assert(c[j] == j);
-        c.push_back(1);
-        assert(c.size() == 2);
-        assert(is_contiguous_container_asan_correct(c));
-        for (int j = 0; static_cast<std::size_t>(j) < c.size(); ++j)
-            assert(c[j] == j);
-        c.push_back(2);
-        assert(c.size() == 3);
-        assert(is_contiguous_container_asan_correct(c));
-        for (int j = 0; static_cast<std::size_t>(j) < c.size(); ++j)
-            assert(c[j] == j);
-        c.push_back(3);
-        assert(c.size() == 4);
-        assert(is_contiguous_container_asan_correct(c));
-        for (int j = 0; static_cast<std::size_t>(j) < c.size(); ++j)
-            assert(c[j] == j);
-        c.push_back(4);
-        assert(c.size() == 5);
-        assert(is_contiguous_container_asan_correct(c));
-        for (int j = 0; static_cast<std::size_t>(j) < c.size(); ++j)
-            assert(c[j] == j);
-    }
+TEST_CONSTEXPR_CXX20 bool tests() {
+  {
+    std::vector<int> c;
+    c.push_back(0);
+    assert(c.size() == 1);
+    assert(is_contiguous_container_asan_correct(c));
+    for (int j = 0; static_cast<std::size_t>(j) < c.size(); ++j)
+      assert(c[j] == j);
+    c.push_back(1);
+    assert(c.size() == 2);
+    assert(is_contiguous_container_asan_correct(c));
+    for (int j = 0; static_cast<std::size_t>(j) < c.size(); ++j)
+      assert(c[j] == j);
+    c.push_back(2);
+    assert(c.size() == 3);
+    assert(is_contiguous_container_asan_correct(c));
+    for (int j = 0; static_cast<std::size_t>(j) < c.size(); ++j)
+      assert(c[j] == j);
+    c.push_back(3);
+    assert(c.size() == 4);
+    assert(is_contiguous_container_asan_correct(c));
+    for (int j = 0; static_cast<std::size_t>(j) < c.size(); ++j)
+      assert(c[j] == j);
+    c.push_back(4);
+    assert(c.size() == 5);
+    assert(is_contiguous_container_asan_correct(c));
+    for (int j = 0; static_cast<std::size_t>(j) < c.size(); ++j)
+      assert(c[j] == j);
+  }
+  {
+    // libc++ needs 15 because it grows by 2x (1 + 2 + 4 + 8).
+    // Use 17 for implementations that dynamically allocate a container proxy
+    // and grow by 1.5x (1 for proxy + 1 + 2 + 3 + 4 + 6).
+    std::vector<int, limited_allocator<int, 17> > c;
+    c.push_back(0);
+    assert(c.size() == 1);
+    assert(is_contiguous_container_asan_correct(c));
+    for (int j = 0; static_cast<std::size_t>(j) < c.size(); ++j)
+      assert(c[j] == j);
+    c.push_back(1);
+    assert(c.size() == 2);
+    assert(is_contiguous_container_asan_correct(c));
+    for (int j = 0; static_cast<std::size_t>(j) < c.size(); ++j)
+      assert(c[j] == j);
+    c.push_back(2);
+    assert(c.size() == 3);
+    assert(is_contiguous_container_asan_correct(c));
+    for (int j = 0; static_cast<std::size_t>(j) < c.size(); ++j)
+      assert(c[j] == j);
+    c.push_back(3);
+    assert(c.size() == 4);
+    assert(is_contiguous_container_asan_correct(c));
+    for (int j = 0; static_cast<std::size_t>(j) < c.size(); ++j)
+      assert(c[j] == j);
+    c.push_back(4);
+    assert(c.size() == 5);
+    assert(is_contiguous_container_asan_correct(c));
+    for (int j = 0; static_cast<std::size_t>(j) < c.size(); ++j)
+      assert(c[j] == j);
+  }
 #if TEST_STD_VER >= 11
-    {
-        std::vector<int, min_allocator<int>> c;
-        c.push_back(0);
-        assert(c.size() == 1);
-        assert(is_contiguous_container_asan_correct(c));
-        for (int j = 0; static_cast<std::size_t>(j) < c.size(); ++j)
-            assert(c[j] == j);
-        c.push_back(1);
-        assert(c.size() == 2);
-        assert(is_contiguous_container_asan_correct(c));
-        for (int j = 0; static_cast<std::size_t>(j) < c.size(); ++j)
-            assert(c[j] == j);
-        c.push_back(2);
-        assert(c.size() == 3);
-        assert(is_contiguous_container_asan_correct(c));
-        for (int j = 0; static_cast<std::size_t>(j) < c.size(); ++j)
-            assert(c[j] == j);
-        c.push_back(3);
-        assert(c.size() == 4);
-        assert(is_contiguous_container_asan_correct(c));
-        for (int j = 0; static_cast<std::size_t>(j) < c.size(); ++j)
-            assert(c[j] == j);
-        c.push_back(4);
-        assert(c.size() == 5);
-        assert(is_contiguous_container_asan_correct(c));
-        for (int j = 0; static_cast<std::size_t>(j) < c.size(); ++j)
-            assert(c[j] == j);
-    }
+  {
+    std::vector<int, min_allocator<int>> c;
+    c.push_back(0);
+    assert(c.size() == 1);
+    assert(is_contiguous_container_asan_correct(c));
+    for (int j = 0; static_cast<std::size_t>(j) < c.size(); ++j)
+      assert(c[j] == j);
+    c.push_back(1);
+    assert(c.size() == 2);
+    assert(is_contiguous_container_asan_correct(c));
+    for (int j = 0; static_cast<std::size_t>(j) < c.size(); ++j)
+      assert(c[j] == j);
+    c.push_back(2);
+    assert(c.size() == 3);
+    assert(is_contiguous_container_asan_correct(c));
+    for (int j = 0; static_cast<std::size_t>(j) < c.size(); ++j)
+      assert(c[j] == j);
+    c.push_back(3);
+    assert(c.size() == 4);
+    assert(is_contiguous_container_asan_correct(c));
+    for (int j = 0; static_cast<std::size_t>(j) < c.size(); ++j)
+      assert(c[j] == j);
+    c.push_back(4);
+    assert(c.size() == 5);
+    assert(is_contiguous_container_asan_correct(c));
+    for (int j = 0; static_cast<std::size_t>(j) < c.size(); ++j)
+      assert(c[j] == j);
+  }
 #endif
 
-    return true;
+  return true;
 }
 
-int main(int, char**)
-{
-    tests();
+int main(int, char**) {
+  tests();
 #if TEST_STD_VER > 17
-    static_assert(tests());
+  static_assert(tests());
 #endif
-    return 0;
+  return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/vector/vector.modifiers/push_back_exception_safety.pass.cpp b/libcxx/test/std/containers/sequences/vector/vector.modifiers/push_back_exception_safety.pass.cpp
index 9e25611290cc6..cbdc55a700b75 100644
--- a/libcxx/test/std/containers/sequences/vector/vector.modifiers/push_back_exception_safety.pass.cpp
+++ b/libcxx/test/std/containers/sequences/vector/vector.modifiers/push_back_exception_safety.pass.cpp
@@ -20,18 +20,26 @@
 static bool gCopyConstructorShouldThrow = false;
 
 class CMyClass {
-    public: CMyClass(int tag);
-    public: CMyClass(const CMyClass& iOther);
-    public: ~CMyClass();
-
-    bool equal(const CMyClass &rhs) const
-        { return fTag == rhs.fTag && fMagicValue == rhs.fMagicValue; }
-    private:
-        int fMagicValue;
-        int fTag;
-
-    private: static int kStartedConstructionMagicValue;
-    private: static int kFinishedConstructionMagicValue;
+public:
+  CMyClass(int tag);
+
+public:
+  CMyClass(const CMyClass& iOther);
+
+public:
+  ~CMyClass();
+
+  bool equal(const CMyClass& rhs) const { return fTag == rhs.fTag && fMagicValue == rhs.fMagicValue; }
+
+private:
+  int fMagicValue;
+  int fTag;
+
+private:
+  static int kStartedConstructionMagicValue;
+
+private:
+  static int kFinishedConstructionMagicValue;
 };
 
 // Value for fMagicValue when the constructor has started running, but not yet finished
@@ -39,51 +47,45 @@ int CMyClass::kStartedConstructionMagicValue = 0;
 // Value for fMagicValue when the constructor has finished running
 int CMyClass::kFinishedConstructionMagicValue = 12345;
 
-CMyClass::CMyClass(int tag) :
-    fMagicValue(kStartedConstructionMagicValue), fTag(tag)
-{
-    // Signal that the constructor has finished running
-    fMagicValue = kFinishedConstructionMagicValue;
+CMyClass::CMyClass(int tag) : fMagicValue(kStartedConstructionMagicValue), fTag(tag) {
+  // Signal that the constructor has finished running
+  fMagicValue = kFinishedConstructionMagicValue;
 }
 
-CMyClass::CMyClass(const CMyClass& iOther) :
-    fMagicValue(kStartedConstructionMagicValue), fTag(iOther.fTag)
-{
-    // If requested, throw an exception _before_ setting fMagicValue to kFinishedConstructionMagicValue
-    if (gCopyConstructorShouldThrow) {
-        TEST_THROW(std::exception());
-    }
-    // Signal that the constructor has finished running
-    fMagicValue = kFinishedConstructionMagicValue;
+CMyClass::CMyClass(const CMyClass& iOther) : fMagicValue(kStartedConstructionMagicValue), fTag(iOther.fTag) {
+  // If requested, throw an exception _before_ setting fMagicValue to kFinishedConstructionMagicValue
+  if (gCopyConstructorShouldThrow) {
+    TEST_THROW(std::exception());
+  }
+  // Signal that the constructor has finished running
+  fMagicValue = kFinishedConstructionMagicValue;
 }
 
 CMyClass::~CMyClass() {
-    // Only instances for which the constructor has finished running should be destructed
-    assert(fMagicValue == kFinishedConstructionMagicValue);
+  // Only instances for which the constructor has finished running should be destructed
+  assert(fMagicValue == kFinishedConstructionMagicValue);
 }
 
-bool operator==(const CMyClass &lhs, const CMyClass &rhs) { return lhs.equal(rhs); }
+bool operator==(const CMyClass& lhs, const CMyClass& rhs) { return lhs.equal(rhs); }
 
-int main(int, char**)
-{
-    CMyClass instance(42);
-    std::vector<CMyClass> vec;
+int main(int, char**) {
+  CMyClass instance(42);
+  std::vector<CMyClass> vec;
 
-    vec.push_back(instance);
-    std::vector<CMyClass> vec2(vec);
-    assert(is_contiguous_container_asan_correct(vec));
-    assert(is_contiguous_container_asan_correct(vec2));
+  vec.push_back(instance);
+  std::vector<CMyClass> vec2(vec);
+  assert(is_contiguous_container_asan_correct(vec));
+  assert(is_contiguous_container_asan_correct(vec2));
 
 #ifndef TEST_HAS_NO_EXCEPTIONS
-    gCopyConstructorShouldThrow = true;
-    try {
-        vec.push_back(instance);
-        assert(false);
-    }
-    catch (...) {
-        assert(vec==vec2);
-        assert(is_contiguous_container_asan_correct(vec));
-    }
+  gCopyConstructorShouldThrow = true;
+  try {
+    vec.push_back(instance);
+    assert(false);
+  } catch (...) {
+    assert(vec == vec2);
+    assert(is_contiguous_container_asan_correct(vec));
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/sequences/vector/vector.modifiers/push_back_rvalue.pass.cpp b/libcxx/test/std/containers/sequences/vector/vector.modifiers/push_back_rvalue.pass.cpp
index 92b629b3cdb4f..e97d5a9c448da 100644
--- a/libcxx/test/std/containers/sequences/vector/vector.modifiers/push_back_rvalue.pass.cpp
+++ b/libcxx/test/std/containers/sequences/vector/vector.modifiers/push_back_rvalue.pass.cpp
@@ -21,104 +21,102 @@
 #include "min_allocator.h"
 #include "asan_testing.h"
 
-TEST_CONSTEXPR_CXX20 bool tests()
-{
-    {
-        std::vector<MoveOnly> c;
-        c.push_back(MoveOnly(0));
-        assert(c.size() == 1);
-        assert(is_contiguous_container_asan_correct(c));
-        for (int j = 0; static_cast<std::size_t>(j) < c.size(); ++j)
-            assert(c[j] == MoveOnly(j));
-        c.push_back(MoveOnly(1));
-        assert(c.size() == 2);
-        assert(is_contiguous_container_asan_correct(c));
-        for (int j = 0; static_cast<std::size_t>(j) < c.size(); ++j)
-            assert(c[j] == MoveOnly(j));
-        c.push_back(MoveOnly(2));
-        assert(c.size() == 3);
-        assert(is_contiguous_container_asan_correct(c));
-        for (int j = 0; static_cast<std::size_t>(j) < c.size(); ++j)
-            assert(c[j] == MoveOnly(j));
-        c.push_back(MoveOnly(3));
-        assert(c.size() == 4);
-        assert(is_contiguous_container_asan_correct(c));
-        for (int j = 0; static_cast<std::size_t>(j) < c.size(); ++j)
-            assert(c[j] == MoveOnly(j));
-        c.push_back(MoveOnly(4));
-        assert(c.size() == 5);
-        assert(is_contiguous_container_asan_correct(c));
-        for (int j = 0; static_cast<std::size_t>(j) < c.size(); ++j)
-            assert(c[j] == MoveOnly(j));
-    }
-    {
-        // libc++ needs 15 because it grows by 2x (1 + 2 + 4 + 8).
-        // Use 17 for implementations that dynamically allocate a container proxy
-        // and grow by 1.5x (1 for proxy + 1 + 2 + 3 + 4 + 6).
-        std::vector<MoveOnly, limited_allocator<MoveOnly, 17> > c;
-        c.push_back(MoveOnly(0));
-        assert(c.size() == 1);
-        assert(is_contiguous_container_asan_correct(c));
-        for (int j = 0; static_cast<std::size_t>(j) < c.size(); ++j)
-            assert(c[j] == MoveOnly(j));
-        c.push_back(MoveOnly(1));
-        assert(c.size() == 2);
-        assert(is_contiguous_container_asan_correct(c));
-        for (int j = 0; static_cast<std::size_t>(j) < c.size(); ++j)
-            assert(c[j] == MoveOnly(j));
-        c.push_back(MoveOnly(2));
-        assert(c.size() == 3);
-        assert(is_contiguous_container_asan_correct(c));
-        for (int j = 0; static_cast<std::size_t>(j) < c.size(); ++j)
-            assert(c[j] == MoveOnly(j));
-        c.push_back(MoveOnly(3));
-        assert(c.size() == 4);
-        assert(is_contiguous_container_asan_correct(c));
-        for (int j = 0; static_cast<std::size_t>(j) < c.size(); ++j)
-            assert(c[j] == MoveOnly(j));
-        c.push_back(MoveOnly(4));
-        assert(c.size() == 5);
-        assert(is_contiguous_container_asan_correct(c));
-        for (int j = 0; static_cast<std::size_t>(j) < c.size(); ++j)
-            assert(c[j] == MoveOnly(j));
-    }
-    {
-        std::vector<MoveOnly, min_allocator<MoveOnly> > c;
-        c.push_back(MoveOnly(0));
-        assert(c.size() == 1);
-        assert(is_contiguous_container_asan_correct(c));
-        for (int j = 0; static_cast<std::size_t>(j) < c.size(); ++j)
-            assert(c[j] == MoveOnly(j));
-        c.push_back(MoveOnly(1));
-        assert(c.size() == 2);
-        assert(is_contiguous_container_asan_correct(c));
-        for (int j = 0; static_cast<std::size_t>(j) < c.size(); ++j)
-            assert(c[j] == MoveOnly(j));
-        c.push_back(MoveOnly(2));
-        assert(c.size() == 3);
-        assert(is_contiguous_container_asan_correct(c));
-        for (int j = 0; static_cast<std::size_t>(j) < c.size(); ++j)
-            assert(c[j] == MoveOnly(j));
-        c.push_back(MoveOnly(3));
-        assert(c.size() == 4);
-        assert(is_contiguous_container_asan_correct(c));
-        for (int j = 0; static_cast<std::size_t>(j) < c.size(); ++j)
-            assert(c[j] == MoveOnly(j));
-        c.push_back(MoveOnly(4));
-        assert(c.size() == 5);
-        assert(is_contiguous_container_asan_correct(c));
-        for (int j = 0; static_cast<std::size_t>(j) < c.size(); ++j)
-            assert(c[j] == MoveOnly(j));
-    }
+TEST_CONSTEXPR_CXX20 bool tests() {
+  {
+    std::vector<MoveOnly> c;
+    c.push_back(MoveOnly(0));
+    assert(c.size() == 1);
+    assert(is_contiguous_container_asan_correct(c));
+    for (int j = 0; static_cast<std::size_t>(j) < c.size(); ++j)
+      assert(c[j] == MoveOnly(j));
+    c.push_back(MoveOnly(1));
+    assert(c.size() == 2);
+    assert(is_contiguous_container_asan_correct(c));
+    for (int j = 0; static_cast<std::size_t>(j) < c.size(); ++j)
+      assert(c[j] == MoveOnly(j));
+    c.push_back(MoveOnly(2));
+    assert(c.size() == 3);
+    assert(is_contiguous_container_asan_correct(c));
+    for (int j = 0; static_cast<std::size_t>(j) < c.size(); ++j)
+      assert(c[j] == MoveOnly(j));
+    c.push_back(MoveOnly(3));
+    assert(c.size() == 4);
+    assert(is_contiguous_container_asan_correct(c));
+    for (int j = 0; static_cast<std::size_t>(j) < c.size(); ++j)
+      assert(c[j] == MoveOnly(j));
+    c.push_back(MoveOnly(4));
+    assert(c.size() == 5);
+    assert(is_contiguous_container_asan_correct(c));
+    for (int j = 0; static_cast<std::size_t>(j) < c.size(); ++j)
+      assert(c[j] == MoveOnly(j));
+  }
+  {
+    // libc++ needs 15 because it grows by 2x (1 + 2 + 4 + 8).
+    // Use 17 for implementations that dynamically allocate a container proxy
+    // and grow by 1.5x (1 for proxy + 1 + 2 + 3 + 4 + 6).
+    std::vector<MoveOnly, limited_allocator<MoveOnly, 17> > c;
+    c.push_back(MoveOnly(0));
+    assert(c.size() == 1);
+    assert(is_contiguous_container_asan_correct(c));
+    for (int j = 0; static_cast<std::size_t>(j) < c.size(); ++j)
+      assert(c[j] == MoveOnly(j));
+    c.push_back(MoveOnly(1));
+    assert(c.size() == 2);
+    assert(is_contiguous_container_asan_correct(c));
+    for (int j = 0; static_cast<std::size_t>(j) < c.size(); ++j)
+      assert(c[j] == MoveOnly(j));
+    c.push_back(MoveOnly(2));
+    assert(c.size() == 3);
+    assert(is_contiguous_container_asan_correct(c));
+    for (int j = 0; static_cast<std::size_t>(j) < c.size(); ++j)
+      assert(c[j] == MoveOnly(j));
+    c.push_back(MoveOnly(3));
+    assert(c.size() == 4);
+    assert(is_contiguous_container_asan_correct(c));
+    for (int j = 0; static_cast<std::size_t>(j) < c.size(); ++j)
+      assert(c[j] == MoveOnly(j));
+    c.push_back(MoveOnly(4));
+    assert(c.size() == 5);
+    assert(is_contiguous_container_asan_correct(c));
+    for (int j = 0; static_cast<std::size_t>(j) < c.size(); ++j)
+      assert(c[j] == MoveOnly(j));
+  }
+  {
+    std::vector<MoveOnly, min_allocator<MoveOnly> > c;
+    c.push_back(MoveOnly(0));
+    assert(c.size() == 1);
+    assert(is_contiguous_container_asan_correct(c));
+    for (int j = 0; static_cast<std::size_t>(j) < c.size(); ++j)
+      assert(c[j] == MoveOnly(j));
+    c.push_back(MoveOnly(1));
+    assert(c.size() == 2);
+    assert(is_contiguous_container_asan_correct(c));
+    for (int j = 0; static_cast<std::size_t>(j) < c.size(); ++j)
+      assert(c[j] == MoveOnly(j));
+    c.push_back(MoveOnly(2));
+    assert(c.size() == 3);
+    assert(is_contiguous_container_asan_correct(c));
+    for (int j = 0; static_cast<std::size_t>(j) < c.size(); ++j)
+      assert(c[j] == MoveOnly(j));
+    c.push_back(MoveOnly(3));
+    assert(c.size() == 4);
+    assert(is_contiguous_container_asan_correct(c));
+    for (int j = 0; static_cast<std::size_t>(j) < c.size(); ++j)
+      assert(c[j] == MoveOnly(j));
+    c.push_back(MoveOnly(4));
+    assert(c.size() == 5);
+    assert(is_contiguous_container_asan_correct(c));
+    for (int j = 0; static_cast<std::size_t>(j) < c.size(); ++j)
+      assert(c[j] == MoveOnly(j));
+  }
 
-    return true;
+  return true;
 }
 
-int main(int, char**)
-{
-    tests();
+int main(int, char**) {
+  tests();
 #if TEST_STD_VER > 17
-    static_assert(tests());
+  static_assert(tests());
 #endif
-    return 0;
+  return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/vector/vector.modifiers/resize_not_move_insertable.verify.cpp b/libcxx/test/std/containers/sequences/vector/vector.modifiers/resize_not_move_insertable.verify.cpp
index 8eb055283fdf4..acc0127ac73e2 100644
--- a/libcxx/test/std/containers/sequences/vector/vector.modifiers/resize_not_move_insertable.verify.cpp
+++ b/libcxx/test/std/containers/sequences/vector/vector.modifiers/resize_not_move_insertable.verify.cpp
@@ -19,12 +19,12 @@
 template <int>
 class BadUserNoCookie {
 public:
-  BadUserNoCookie() { }
+  BadUserNoCookie() {}
 
-  BadUserNoCookie(BadUserNoCookie&&) = delete;
+  BadUserNoCookie(BadUserNoCookie&&)            = delete;
   BadUserNoCookie& operator=(BadUserNoCookie&&) = delete;
 
-  BadUserNoCookie(const BadUserNoCookie&) = default;
+  BadUserNoCookie(const BadUserNoCookie&)            = default;
   BadUserNoCookie& operator=(const BadUserNoCookie&) = default;
 };
 
@@ -35,7 +35,6 @@ int main(int, char**) {
   // expected-error@* 0-2 {{call to deleted constructor}}
   // expected-error@* 0-2 {{no matching function for call to '__construct_at'}}
   {
-
     std::vector<BadUserNoCookie<1> > x;
     x.emplace_back();
   }
diff --git a/libcxx/test/std/containers/sequences/vector/vector.special/swap.pass.cpp b/libcxx/test/std/containers/sequences/vector/vector.special/swap.pass.cpp
index 98aece64c80e7..0da2f820ab1d7 100644
--- a/libcxx/test/std/containers/sequences/vector/vector.special/swap.pass.cpp
+++ b/libcxx/test/std/containers/sequences/vector/vector.special/swap.pass.cpp
@@ -94,105 +94,103 @@ TEST_CONSTEXPR_CXX20 void test_with_allocator() {
   }
 }
 
-TEST_CONSTEXPR_CXX20 bool tests()
-{
-    {
-        int a1[] = {1, 3, 7, 9, 10};
-        int a2[] = {0, 2, 4, 5, 6, 8, 11};
-        std::vector<int> c1(a1, a1+sizeof(a1)/sizeof(a1[0]));
-        std::vector<int> c2(a2, a2+sizeof(a2)/sizeof(a2[0]));
-        assert(is_contiguous_container_asan_correct(c1));
-        assert(is_contiguous_container_asan_correct(c2));
-        swap(c1, c2);
-        assert(c1 == std::vector<int>(a2, a2+sizeof(a2)/sizeof(a2[0])));
-        assert(c2 == std::vector<int>(a1, a1+sizeof(a1)/sizeof(a1[0])));
-        assert(is_contiguous_container_asan_correct(c1));
-        assert(is_contiguous_container_asan_correct(c2));
-    }
-    {
-        int a1[] = {1, 3, 7, 9, 10};
-        int a2[] = {0, 2, 4, 5, 6, 8, 11};
-        std::vector<int> c1(a1, a1);
-        std::vector<int> c2(a2, a2+sizeof(a2)/sizeof(a2[0]));
-        assert(is_contiguous_container_asan_correct(c1));
-        assert(is_contiguous_container_asan_correct(c2));
-        swap(c1, c2);
-        assert(c1 == std::vector<int>(a2, a2+sizeof(a2)/sizeof(a2[0])));
-        assert(c2.empty());
-        assert(std::distance(c2.begin(), c2.end()) == 0);
-        assert(is_contiguous_container_asan_correct(c1));
-        assert(is_contiguous_container_asan_correct(c2));
-    }
-    {
-        int a1[] = {1, 3, 7, 9, 10};
-        int a2[] = {0, 2, 4, 5, 6, 8, 11};
-        std::vector<int> c1(a1, a1+sizeof(a1)/sizeof(a1[0]));
-        std::vector<int> c2(a2, a2);
-        assert(is_contiguous_container_asan_correct(c1));
-        assert(is_contiguous_container_asan_correct(c2));
-        swap(c1, c2);
-        assert(c1.empty());
-        assert(std::distance(c1.begin(), c1.end()) == 0);
-        assert(c2 == std::vector<int>(a1, a1+sizeof(a1)/sizeof(a1[0])));
-        assert(is_contiguous_container_asan_correct(c1));
-        assert(is_contiguous_container_asan_correct(c2));
-    }
-    {
-        int a1[] = {1, 3, 7, 9, 10};
-        int a2[] = {0, 2, 4, 5, 6, 8, 11};
-        std::vector<int> c1(a1, a1);
-        std::vector<int> c2(a2, a2);
-        assert(is_contiguous_container_asan_correct(c1));
-        assert(is_contiguous_container_asan_correct(c2));
-        swap(c1, c2);
-        assert(c1.empty());
-        assert(std::distance(c1.begin(), c1.end()) == 0);
-        assert(c2.empty());
-        assert(std::distance(c2.begin(), c2.end()) == 0);
-        assert(is_contiguous_container_asan_correct(c1));
-        assert(is_contiguous_container_asan_correct(c2));
-    }
-    {
-        int a1[] = {1, 3, 7, 9, 10};
-        int a2[] = {0, 2, 4, 5, 6, 8, 11};
-        typedef test_allocator<int> A;
-        std::vector<int, A> c1(a1, a1+sizeof(a1)/sizeof(a1[0]), A(1, 1));
-        std::vector<int, A> c2(a2, a2+sizeof(a2)/sizeof(a2[0]), A(1, 2));
-        swap(c1, c2);
-        assert((c1 == std::vector<int, A>(a2, a2+sizeof(a2)/sizeof(a2[0]))));
-        assert(c1.get_allocator().get_id() == 1);
-        assert((c2 == std::vector<int, A>(a1, a1+sizeof(a1)/sizeof(a1[0]))));
-        assert(c2.get_allocator().get_id() == 2);
-    }
-    {
-        int a1[] = {1, 3, 7, 9, 10};
-        int a2[] = {0, 2, 4, 5, 6, 8, 11};
-        typedef other_allocator<int> A;
-        std::vector<int, A> c1(a1, a1+sizeof(a1)/sizeof(a1[0]), A(1));
-        std::vector<int, A> c2(a2, a2+sizeof(a2)/sizeof(a2[0]), A(2));
-        assert(is_contiguous_container_asan_correct(c1));
-        assert(is_contiguous_container_asan_correct(c2));
-        swap(c1, c2);
-        assert((c1 == std::vector<int, A>(a2, a2+sizeof(a2)/sizeof(a2[0]))));
-        assert(c1.get_allocator() == A(2));
-        assert((c2 == std::vector<int, A>(a1, a1+sizeof(a1)/sizeof(a1[0]))));
-        assert(c2.get_allocator() == A(1));
-        assert(is_contiguous_container_asan_correct(c1));
-        assert(is_contiguous_container_asan_correct(c2));
-    }
+TEST_CONSTEXPR_CXX20 bool tests() {
+  {
+    int a1[] = {1, 3, 7, 9, 10};
+    int a2[] = {0, 2, 4, 5, 6, 8, 11};
+    std::vector<int> c1(a1, a1 + sizeof(a1) / sizeof(a1[0]));
+    std::vector<int> c2(a2, a2 + sizeof(a2) / sizeof(a2[0]));
+    assert(is_contiguous_container_asan_correct(c1));
+    assert(is_contiguous_container_asan_correct(c2));
+    swap(c1, c2);
+    assert(c1 == std::vector<int>(a2, a2 + sizeof(a2) / sizeof(a2[0])));
+    assert(c2 == std::vector<int>(a1, a1 + sizeof(a1) / sizeof(a1[0])));
+    assert(is_contiguous_container_asan_correct(c1));
+    assert(is_contiguous_container_asan_correct(c2));
+  }
+  {
+    int a1[] = {1, 3, 7, 9, 10};
+    int a2[] = {0, 2, 4, 5, 6, 8, 11};
+    std::vector<int> c1(a1, a1);
+    std::vector<int> c2(a2, a2 + sizeof(a2) / sizeof(a2[0]));
+    assert(is_contiguous_container_asan_correct(c1));
+    assert(is_contiguous_container_asan_correct(c2));
+    swap(c1, c2);
+    assert(c1 == std::vector<int>(a2, a2 + sizeof(a2) / sizeof(a2[0])));
+    assert(c2.empty());
+    assert(std::distance(c2.begin(), c2.end()) == 0);
+    assert(is_contiguous_container_asan_correct(c1));
+    assert(is_contiguous_container_asan_correct(c2));
+  }
+  {
+    int a1[] = {1, 3, 7, 9, 10};
+    int a2[] = {0, 2, 4, 5, 6, 8, 11};
+    std::vector<int> c1(a1, a1 + sizeof(a1) / sizeof(a1[0]));
+    std::vector<int> c2(a2, a2);
+    assert(is_contiguous_container_asan_correct(c1));
+    assert(is_contiguous_container_asan_correct(c2));
+    swap(c1, c2);
+    assert(c1.empty());
+    assert(std::distance(c1.begin(), c1.end()) == 0);
+    assert(c2 == std::vector<int>(a1, a1 + sizeof(a1) / sizeof(a1[0])));
+    assert(is_contiguous_container_asan_correct(c1));
+    assert(is_contiguous_container_asan_correct(c2));
+  }
+  {
+    int a1[] = {1, 3, 7, 9, 10};
+    int a2[] = {0, 2, 4, 5, 6, 8, 11};
+    std::vector<int> c1(a1, a1);
+    std::vector<int> c2(a2, a2);
+    assert(is_contiguous_container_asan_correct(c1));
+    assert(is_contiguous_container_asan_correct(c2));
+    swap(c1, c2);
+    assert(c1.empty());
+    assert(std::distance(c1.begin(), c1.end()) == 0);
+    assert(c2.empty());
+    assert(std::distance(c2.begin(), c2.end()) == 0);
+    assert(is_contiguous_container_asan_correct(c1));
+    assert(is_contiguous_container_asan_correct(c2));
+  }
+  {
+    int a1[] = {1, 3, 7, 9, 10};
+    int a2[] = {0, 2, 4, 5, 6, 8, 11};
+    typedef test_allocator<int> A;
+    std::vector<int, A> c1(a1, a1 + sizeof(a1) / sizeof(a1[0]), A(1, 1));
+    std::vector<int, A> c2(a2, a2 + sizeof(a2) / sizeof(a2[0]), A(1, 2));
+    swap(c1, c2);
+    assert((c1 == std::vector<int, A>(a2, a2 + sizeof(a2) / sizeof(a2[0]))));
+    assert(c1.get_allocator().get_id() == 1);
+    assert((c2 == std::vector<int, A>(a1, a1 + sizeof(a1) / sizeof(a1[0]))));
+    assert(c2.get_allocator().get_id() == 2);
+  }
+  {
+    int a1[] = {1, 3, 7, 9, 10};
+    int a2[] = {0, 2, 4, 5, 6, 8, 11};
+    typedef other_allocator<int> A;
+    std::vector<int, A> c1(a1, a1 + sizeof(a1) / sizeof(a1[0]), A(1));
+    std::vector<int, A> c2(a2, a2 + sizeof(a2) / sizeof(a2[0]), A(2));
+    assert(is_contiguous_container_asan_correct(c1));
+    assert(is_contiguous_container_asan_correct(c2));
+    swap(c1, c2);
+    assert((c1 == std::vector<int, A>(a2, a2 + sizeof(a2) / sizeof(a2[0]))));
+    assert(c1.get_allocator() == A(2));
+    assert((c2 == std::vector<int, A>(a1, a1 + sizeof(a1) / sizeof(a1[0]))));
+    assert(c2.get_allocator() == A(1));
+    assert(is_contiguous_container_asan_correct(c1));
+    assert(is_contiguous_container_asan_correct(c2));
+  }
 #if TEST_STD_VER >= 11
-    test_with_allocator<min_allocator<int>>();
-    test_with_allocator<safe_allocator<int>>();
+  test_with_allocator<min_allocator<int>>();
+  test_with_allocator<safe_allocator<int>>();
 #endif
 
-    return true;
+  return true;
 }
 
-int main(int, char**)
-{
-    tests();
+int main(int, char**) {
+  tests();
 #if TEST_STD_VER > 17
-    static_assert(tests());
+  static_assert(tests());
 #endif
-    return 0;
+  return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/vector/vector.special/swap_noexcept.compile.pass.cpp b/libcxx/test/std/containers/sequences/vector/vector.special/swap_noexcept.compile.pass.cpp
index d38a00c789f8e..48e083e42b15f 100644
--- a/libcxx/test/std/containers/sequences/vector/vector.special/swap_noexcept.compile.pass.cpp
+++ b/libcxx/test/std/containers/sequences/vector/vector.special/swap_noexcept.compile.pass.cpp
@@ -29,62 +29,59 @@
 #include "test_allocator.h"
 
 template <class T>
-struct some_alloc
-{
-    typedef T value_type;
+struct some_alloc {
+  typedef T value_type;
 
-    some_alloc() {}
-    some_alloc(const some_alloc&);
-    void allocate(std::size_t);
-    void deallocate(void*, unsigned) {}
+  some_alloc() {}
+  some_alloc(const some_alloc&);
+  void allocate(std::size_t);
+  void deallocate(void*, unsigned) {}
 
-    typedef std::true_type propagate_on_container_swap;
+  typedef std::true_type propagate_on_container_swap;
 };
 
 template <class T>
-struct some_alloc2
-{
-    typedef T value_type;
+struct some_alloc2 {
+  typedef T value_type;
 
-    some_alloc2() {}
-    some_alloc2(const some_alloc2&);
-    void allocate(std::size_t);
-    void deallocate(void*, unsigned) {}
+  some_alloc2() {}
+  some_alloc2(const some_alloc2&);
+  void allocate(std::size_t);
+  void deallocate(void*, unsigned) {}
 
-    typedef std::false_type propagate_on_container_swap;
-    typedef std::true_type is_always_equal;
+  typedef std::false_type propagate_on_container_swap;
+  typedef std::true_type is_always_equal;
 };
 
-void test()
-{
-    {
-        typedef std::vector<MoveOnly> C;
-        static_assert(noexcept(swap(std::declval<C&>(), std::declval<C&>())), "");
-    }
+void test() {
+  {
+    typedef std::vector<MoveOnly> C;
+    static_assert(noexcept(swap(std::declval<C&>(), std::declval<C&>())), "");
+  }
 #if defined(_LIBCPP_VERSION)
-    {
-        typedef std::vector<MoveOnly, test_allocator<MoveOnly>> C;
-        static_assert(noexcept(swap(std::declval<C&>(), std::declval<C&>())), "");
-    }
+  {
+    typedef std::vector<MoveOnly, test_allocator<MoveOnly>> C;
+    static_assert(noexcept(swap(std::declval<C&>(), std::declval<C&>())), "");
+  }
 #endif
-    {
-        typedef std::vector<MoveOnly, other_allocator<MoveOnly>> C;
-        static_assert(noexcept(swap(std::declval<C&>(), std::declval<C&>())), "");
-    }
-    {
-        typedef std::vector<MoveOnly, some_alloc<MoveOnly>> C;
+  {
+    typedef std::vector<MoveOnly, other_allocator<MoveOnly>> C;
+    static_assert(noexcept(swap(std::declval<C&>(), std::declval<C&>())), "");
+  }
+  {
+    typedef std::vector<MoveOnly, some_alloc<MoveOnly>> C;
 #if TEST_STD_VER >= 14
-        //  In C++14, if POCS is set, swapping the allocator is required not to throw
-        static_assert( noexcept(swap(std::declval<C&>(), std::declval<C&>())), "");
+    //  In C++14, if POCS is set, swapping the allocator is required not to throw
+    static_assert(noexcept(swap(std::declval<C&>(), std::declval<C&>())), "");
 #else
-        static_assert(!noexcept(swap(std::declval<C&>(), std::declval<C&>())), "");
+    static_assert(!noexcept(swap(std::declval<C&>(), std::declval<C&>())), "");
 #endif
-    }
+  }
 #if TEST_STD_VER >= 14
-    {
-        typedef std::vector<MoveOnly, some_alloc2<MoveOnly>> C;
-        // If the allocators are always equal, then the swap can be noexcept
-        static_assert( noexcept(swap(std::declval<C&>(), std::declval<C&>())), "");
-    }
+  {
+    typedef std::vector<MoveOnly, some_alloc2<MoveOnly>> C;
+    // If the allocators are always equal, then the swap can be noexcept
+    static_assert(noexcept(swap(std::declval<C&>(), std::declval<C&>())), "");
+  }
 #endif
 }
diff --git a/libcxx/test/std/containers/set_allocator_requirement_test_templates.h b/libcxx/test/std/containers/set_allocator_requirement_test_templates.h
index ad05ee458e3cd..9ab5c50e9012e 100644
--- a/libcxx/test/std/containers/set_allocator_requirement_test_templates.h
+++ b/libcxx/test/std/containers/set_allocator_requirement_test_templates.h
@@ -19,7 +19,6 @@
 // emplace(...);
 // emplace_hint(...);
 
-
 #include <cassert>
 #include <iterator>
 
@@ -27,10 +26,8 @@
 #include "count_new.h"
 #include "container_test_types.h"
 
-
 template <class Container>
-void testSetInsert()
-{
+void testSetInsert() {
   typedef typename Container::value_type ValueTp;
   ConstructController* cc = getConstructController();
   cc->reset();
@@ -89,7 +86,7 @@ void testSetInsert()
   {
     // Testing C::insert(std::initializer_list<ValueTp>)"
     Container c;
-    std::initializer_list<ValueTp> il = { ValueTp(1), ValueTp(2) };
+    std::initializer_list<ValueTp> il = {ValueTp(1), ValueTp(2)};
     cc->expect<ValueTp const&>(2);
     c.insert(il);
     assert(!cc->unchecked());
@@ -101,7 +98,7 @@ void testSetInsert()
   {
     // Testing C::insert(Iter, Iter) for *Iter = value_type const&"
     Container c;
-    const ValueTp ValueList[] = { ValueTp(1), ValueTp(2), ValueTp(3) };
+    const ValueTp ValueList[] = {ValueTp(1), ValueTp(2), ValueTp(3)};
     cc->expect<ValueTp const&>(3);
     c.insert(std::begin(ValueList), std::end(ValueList));
     assert(!cc->unchecked());
@@ -113,14 +110,13 @@ void testSetInsert()
   {
     // Testing C::insert(Iter, Iter) for *Iter = value_type&&"
     Container c;
-    ValueTp ValueList[] = { ValueTp(1), ValueTp(2) , ValueTp(3) };
+    ValueTp ValueList[] = {ValueTp(1), ValueTp(2), ValueTp(3)};
     cc->expect<ValueTp&&>(3);
-    c.insert(std::move_iterator<ValueTp*>(std::begin(ValueList)),
-             std::move_iterator<ValueTp*>(std::end(ValueList)));
+    c.insert(std::move_iterator<ValueTp*>(std::begin(ValueList)), std::move_iterator<ValueTp*>(std::end(ValueList)));
     assert(!cc->unchecked());
     {
       DisableAllocationGuard g;
-      ValueTp ValueList2[] = { ValueTp(1), ValueTp(2) , ValueTp(3) };
+      ValueTp ValueList2[] = {ValueTp(1), ValueTp(2), ValueTp(3)};
       c.insert(std::move_iterator<ValueTp*>(std::begin(ValueList2)),
                std::move_iterator<ValueTp*>(std::end(ValueList2)));
     }
@@ -128,7 +124,7 @@ void testSetInsert()
   {
     // Testing C::insert(Iter, Iter) for *Iter = value_type&"
     Container c;
-    ValueTp ValueList[] = { ValueTp(1), ValueTp(2) , ValueTp(3) };
+    ValueTp ValueList[] = {ValueTp(1), ValueTp(2), ValueTp(3)};
     cc->expect<ValueTp const&>(3);
     c.insert(std::begin(ValueList), std::end(ValueList));
     assert(!cc->unchecked());
@@ -139,10 +135,8 @@ void testSetInsert()
   }
 }
 
-
 template <class Container>
-void testSetEmplace()
-{
+void testSetEmplace() {
   typedef typename Container::value_type ValueTp;
   ConstructController* cc = getConstructController();
   cc->reset();
@@ -200,10 +194,8 @@ void testSetEmplace()
   }
 }
 
-
 template <class Container>
-void testSetEmplaceHint()
-{
+void testSetEmplaceHint() {
   typedef typename Container::value_type ValueTp;
   typedef Container C;
   typedef typename C::iterator It;
@@ -279,10 +271,8 @@ void testSetEmplaceHint()
   }
 }
 
-
 template <class Container>
-void testMultisetInsert()
-{
+void testMultisetInsert() {
   typedef typename Container::value_type ValueTp;
   ConstructController* cc = getConstructController();
   cc->reset();
@@ -313,7 +303,7 @@ void testMultisetInsert()
   {
     // Testing C::insert(std::initializer_list<ValueTp>)"
     Container c;
-    std::initializer_list<ValueTp> il = { ValueTp(1), ValueTp(2) };
+    std::initializer_list<ValueTp> il = {ValueTp(1), ValueTp(2)};
     cc->expect<ValueTp const&>(2);
     c.insert(il);
     assert(!cc->unchecked());
@@ -321,7 +311,7 @@ void testMultisetInsert()
   {
     // Testing C::insert(Iter, Iter) for *Iter = value_type const&"
     Container c;
-    const ValueTp ValueList[] = { ValueTp(1), ValueTp(2), ValueTp(3) };
+    const ValueTp ValueList[] = {ValueTp(1), ValueTp(2), ValueTp(3)};
     cc->expect<ValueTp const&>(3);
     c.insert(std::begin(ValueList), std::end(ValueList));
     assert(!cc->unchecked());
@@ -329,26 +319,23 @@ void testMultisetInsert()
   {
     // Testing C::insert(Iter, Iter) for *Iter = value_type&&"
     Container c;
-    ValueTp ValueList[] = { ValueTp(1), ValueTp(2) , ValueTp(3) };
+    ValueTp ValueList[] = {ValueTp(1), ValueTp(2), ValueTp(3)};
     cc->expect<ValueTp&&>(3);
-    c.insert(std::move_iterator<ValueTp*>(std::begin(ValueList)),
-             std::move_iterator<ValueTp*>(std::end(ValueList)));
+    c.insert(std::move_iterator<ValueTp*>(std::begin(ValueList)), std::move_iterator<ValueTp*>(std::end(ValueList)));
     assert(!cc->unchecked());
   }
   {
     // Testing C::insert(Iter, Iter) for *Iter = value_type&"
     Container c;
-    ValueTp ValueList[] = { ValueTp(1), ValueTp(2) , ValueTp(1) };
+    ValueTp ValueList[] = {ValueTp(1), ValueTp(2), ValueTp(1)};
     cc->expect<ValueTp&>(3);
     c.insert(std::begin(ValueList), std::end(ValueList));
     assert(!cc->unchecked());
   }
 }
 
-
 template <class Container>
-void testMultisetEmplace()
-{
+void testMultisetEmplace() {
   typedef typename Container::value_type ValueTp;
   ConstructController* cc = getConstructController();
   cc->reset();
diff --git a/libcxx/test/std/containers/test_compare.h b/libcxx/test/std/containers/test_compare.h
index 1b4c4916b3374..18e0b27dc954d 100644
--- a/libcxx/test/std/containers/test_compare.h
+++ b/libcxx/test/std/containers/test_compare.h
@@ -10,27 +10,21 @@
 #define TEST_COMPARE_H
 
 template <class T>
-struct test_equal_to
-{
-    int data_;
-    explicit test_equal_to() : data_(0) {}
-    explicit test_equal_to(int data) : data_(data) {}
-    bool operator()(const T& a, const T& b) const
-        { return a == b; }
-    friend bool operator==(const test_equal_to& a, const test_equal_to& b)
-        { return a.data_ == b.data_; }
+struct test_equal_to {
+  int data_;
+  explicit test_equal_to() : data_(0) {}
+  explicit test_equal_to(int data) : data_(data) {}
+  bool operator()(const T& a, const T& b) const { return a == b; }
+  friend bool operator==(const test_equal_to& a, const test_equal_to& b) { return a.data_ == b.data_; }
 };
 
 template <class T>
-struct test_less
-{
-    int data_;
-    explicit test_less() : data_(0) {}
-    explicit test_less(int data) : data_(data) {}
-    bool operator()(const T& a, const T& b) const
-        { return a < b; }
-    friend bool operator==(const test_less& a, const test_less& b)
-        { return a.data_ == b.data_; }
+struct test_less {
+  int data_;
+  explicit test_less() : data_(0) {}
+  explicit test_less(int data) : data_(data) {}
+  bool operator()(const T& a, const T& b) const { return a < b; }
+  friend bool operator==(const test_less& a, const test_less& b) { return a.data_ == b.data_; }
 };
 
 #endif // TEST_COMPARE_H
diff --git a/libcxx/test/std/containers/test_hash.h b/libcxx/test/std/containers/test_hash.h
index 64195804a8d47..99ed907a36bf2 100644
--- a/libcxx/test/std/containers/test_hash.h
+++ b/libcxx/test/std/containers/test_hash.h
@@ -13,17 +13,15 @@
 #include <utility>
 
 template <class T>
-class test_hash
-{
-    int data_;
+class test_hash {
+  int data_;
+
 public:
-    explicit test_hash(int data = 0) : data_(data) {}
+  explicit test_hash(int data = 0) : data_(data) {}
 
-    std::size_t operator()(const T& x) const
-        {return std::hash<T>()(x);}
+  std::size_t operator()(const T& x) const { return std::hash<T>()(x); }
 
-    bool operator==(const test_hash& c) const
-        {return data_ == c.data_;}
+  bool operator==(const test_hash& c) const { return data_ == c.data_; }
 };
 
 #endif // TEST_HASH_H
diff --git a/libcxx/test/std/containers/unord/from_range_unordered_containers.h b/libcxx/test/std/containers/unord/from_range_unordered_containers.h
index 7eb2a07e499c7..b8aa73383354d 100644
--- a/libcxx/test/std/containers/unord/from_range_unordered_containers.h
+++ b/libcxx/test/std/containers/unord/from_range_unordered_containers.h
@@ -41,7 +41,7 @@
 //     : unordered-container(from_range, std::forward<R>(rg), n, hf, key_equal(), a) { }       // C++23
 
 template <class Container, class Range>
-concept HasFromRangeCtr = requires (Range&& range) {
+concept HasFromRangeCtr = requires(Range&& range) {
   // (from_range, range)
   Container(std::from_range, std::forward<Range>(range));
   // (from_range, range, n)
@@ -49,15 +49,25 @@ concept HasFromRangeCtr = requires (Range&& range) {
   // (from_range, range, n, hash)
   Container(std::from_range, std::forward<Range>(range), 0, std::hash<typename Container::key_type>());
   // (from_range, range, n, hash, equal)
-  Container(std::from_range, std::forward<Range>(range), 0, std::hash<typename Container::key_type>(),
+  Container(std::from_range,
+            std::forward<Range>(range),
+            0,
+            std::hash<typename Container::key_type>(),
             std::equal_to<typename Container::key_type>());
   // (from_range, range, n, hash, equal, alloc)
-  Container(std::from_range, std::forward<Range>(range), 0, std::hash<typename Container::key_type>(),
-            std::equal_to<typename Container::key_type>(), std::allocator<typename Container::value_type>());
+  Container(std::from_range,
+            std::forward<Range>(range),
+            0,
+            std::hash<typename Container::key_type>(),
+            std::equal_to<typename Container::key_type>(),
+            std::allocator<typename Container::value_type>());
   // (from_range, range, n, alloc)
   Container(std::from_range, std::forward<Range>(range), 0, std::allocator<typename Container::value_type>());
   // (from_range, range, n, hash, alloc)
-  Container(std::from_range, std::forward<Range>(range), 0, std::hash<typename Container::key_type>(),
+  Container(std::from_range,
+            std::forward<Range>(range),
+            0,
+            std::hash<typename Container::key_type>(),
             std::allocator<typename Container::value_type>());
 };
 
@@ -78,7 +88,7 @@ constexpr bool test_map_constraints() {
   return true;
 }
 
-template <template <class ...> class Container,
+template <template <class...> class Container,
           class K,
           class V,
           class Iter,
@@ -88,7 +98,7 @@ template <template <class ...> class Container,
           class Alloc,
           class ValueType = std::pair<const K, V>>
 void test_unordered_map_with_input(std::vector<ValueType>&& input) {
-  using DefaultHash = std::hash<int>;
+  using DefaultHash  = std::hash<int>;
   using DefaultEqual = std::equal_to<int>;
 
   auto validate = [](auto&& c) {
@@ -175,7 +185,7 @@ void test_unordered_map_with_input(std::vector<ValueType>&& input) {
   }
 }
 
-template <template <class ...> class Container,
+template <template <class...> class Container,
           class K,
           class V,
           class Iter,
@@ -194,7 +204,7 @@ void test_unordered_map() {
   test_with_input({{1, 2}});
 }
 
-template <template <class ...> class Container>
+template <template <class...> class Container>
 void test_unordered_map_move_only() {
   std::pair<const int, MoveOnly> input[5];
   std::ranges::subrange in(std::move_iterator{input}, std::move_iterator{input + 5});
@@ -202,17 +212,15 @@ void test_unordered_map_move_only() {
   [[maybe_unused]] Container<int, MoveOnly> c(std::from_range, in);
 }
 
-template <template <class ...> class Container>
+template <template <class...> class Container>
 void test_map_exception_safety_throwing_copy() {
 #if !defined(TEST_HAS_NO_EXCEPTIONS)
   using K = int;
   using V = ThrowingCopy<3>;
 
-  V::throwing_enabled = false;
-  std::pair<const K, V> in[5] = {
-    {1, {}}, {2, {}}, {3, {}}, {4, {}}, {5, {}}
-  };
-  V::throwing_enabled = true;
+  V::throwing_enabled         = false;
+  std::pair<const K, V> in[5] = {{1, {}}, {2, {}}, {3, {}}, {4, {}}, {5, {}}};
+  V::throwing_enabled         = true;
   V::reset();
 
   try {
@@ -226,20 +234,18 @@ void test_map_exception_safety_throwing_copy() {
 #endif
 }
 
-template <template <class ...> class Container, class K, class V>
+template <template <class...> class Container, class K, class V>
 void test_map_exception_safety_throwing_allocator() {
 #if !defined(TEST_HAS_NO_EXCEPTIONS)
   using ValueType = std::pair<const K, V>;
-  ValueType in[] = {
-    ValueType{K{1}, V{1}}
-  };
+  ValueType in[]  = {ValueType{K{1}, V{1}}};
 
   try {
     ThrowingAllocator<ValueType> alloc;
 
     globalMemCounter.reset();
-    Container<K, V, test_hash<K>, test_equal_to<K>, ThrowingAllocator<ValueType>>
-        c(std::from_range, in, /*n=*/0, alloc);
+    Container<K, V, test_hash<K>, test_equal_to<K>, ThrowingAllocator<ValueType>> c(
+        std::from_range, in, /*n=*/0, alloc);
     assert(false); // The constructor call should throw.
 
   } catch (int) {
@@ -249,7 +255,7 @@ void test_map_exception_safety_throwing_allocator() {
 }
 
 template <class Container, class Range>
-concept SetHasFromRangeCtr = requires (Range&& range) {
+concept SetHasFromRangeCtr = requires(Range&& range) {
   // (from_range, range)
   Container(std::from_range, std::forward<Range>(range));
   // (from_range, range, n)
@@ -257,15 +263,25 @@ concept SetHasFromRangeCtr = requires (Range&& range) {
   // (from_range, range, n, hash)
   Container(std::from_range, std::forward<Range>(range), 0, std::hash<typename Container::value_type>());
   // (from_range, range, n, hash, equal)
-  Container(std::from_range, std::forward<Range>(range), 0, std::hash<typename Container::value_type>(),
+  Container(std::from_range,
+            std::forward<Range>(range),
+            0,
+            std::hash<typename Container::value_type>(),
             std::equal_to<typename Container::value_type>());
   // (from_range, range, n, hash, equal, alloc)
-  Container(std::from_range, std::forward<Range>(range), 0, std::hash<typename Container::value_type>(),
-            std::equal_to<typename Container::value_type>(), std::allocator<typename Container::value_type>());
+  Container(std::from_range,
+            std::forward<Range>(range),
+            0,
+            std::hash<typename Container::value_type>(),
+            std::equal_to<typename Container::value_type>(),
+            std::allocator<typename Container::value_type>());
   // (from_range, range, n, alloc)
   Container(std::from_range, std::forward<Range>(range), 0, std::allocator<typename Container::value_type>());
   // (from_range, range, n, hash, alloc)
-  Container(std::from_range, std::forward<Range>(range), 0, std::hash<typename Container::value_type>(),
+  Container(std::from_range,
+            std::forward<Range>(range),
+            0,
+            std::hash<typename Container::value_type>(),
             std::allocator<typename Container::value_type>());
 };
 
@@ -283,15 +299,9 @@ constexpr bool test_set_constraints() {
   return true;
 }
 
-template <template <class ...> class Container,
-          class T,
-          class Iter,
-          class Sent,
-          class Hash,
-          class Equal,
-          class Alloc>
+template <template <class...> class Container, class T, class Iter, class Sent, class Hash, class Equal, class Alloc>
 void test_unordered_set_with_input(std::vector<T>&& input) {
-  using DefaultHash = std::hash<int>;
+  using DefaultHash  = std::hash<int>;
   using DefaultEqual = std::equal_to<int>;
 
   auto validate = [](auto&& c) {
@@ -378,13 +388,7 @@ void test_unordered_set_with_input(std::vector<T>&& input) {
   }
 }
 
-template <template <class ...> class Container,
-          class T,
-          class Iter,
-          class Sent,
-          class Hash,
-          class Equal,
-          class Alloc>
+template <template <class...> class Container, class T, class Iter, class Sent, class Hash, class Equal, class Alloc>
 void test_unordered_set() {
   auto test_with_input = &test_unordered_set_with_input<Container, T, Iter, Sent, Hash, Equal, Alloc>;
 
@@ -396,7 +400,7 @@ void test_unordered_set() {
   test_with_input({5});
 }
 
-template <template <class ...> class Container>
+template <template <class...> class Container>
 void test_unordered_set_move_only() {
   MoveOnly input[5];
   std::ranges::subrange in(std::move_iterator{input}, std::move_iterator{input + 5});
@@ -404,7 +408,7 @@ void test_unordered_set_move_only() {
   [[maybe_unused]] Container<MoveOnly> c(std::from_range, in);
 }
 
-template <template <class ...> class Container>
+template <template <class...> class Container>
 void test_set_exception_safety_throwing_copy() {
 #if !defined(TEST_HAS_NO_EXCEPTIONS)
   using T = ThrowingCopy<3>;
@@ -422,7 +426,7 @@ void test_set_exception_safety_throwing_copy() {
 #endif
 }
 
-template <template <class ...> class Container, class T>
+template <template <class...> class Container, class T>
 void test_set_exception_safety_throwing_allocator() {
 #if !defined(TEST_HAS_NO_EXCEPTIONS)
   T in[] = {1, 2, 3};
diff --git a/libcxx/test/std/containers/unord/iterator_difference_type.pass.cpp b/libcxx/test/std/containers/unord/iterator_difference_type.pass.cpp
index fc5ccbee34f22..b69127282f4b0 100644
--- a/libcxx/test/std/containers/unord/iterator_difference_type.pass.cpp
+++ b/libcxx/test/std/containers/unord/iterator_difference_type.pass.cpp
@@ -14,7 +14,6 @@
 #include "min_allocator.h"
 #include "test_allocator.h"
 
-
 template <class Map, class ValueTp, class PtrT, class CPtrT>
 void testUnorderedMap() {
   typedef typename Map::difference_type Diff;
@@ -48,13 +47,10 @@ void testUnorderedMap() {
   }
 }
 
-
 template <class Set, class ValueTp, class CPtrT>
 void testUnorderedSet() {
-  static_assert((std::is_convertible<typename Set::iterator,
-                                     typename Set::const_iterator>::value), "");
-  static_assert((std::is_convertible<typename Set::local_iterator,
-                                     typename Set::const_local_iterator>::value), "");
+  static_assert((std::is_convertible<typename Set::iterator, typename Set::const_iterator>::value), "");
+  static_assert((std::is_convertible<typename Set::local_iterator, typename Set::const_local_iterator>::value), "");
   typedef typename Set::difference_type Diff;
   {
     typedef typename Set::iterator It;
@@ -62,7 +58,6 @@ void testUnorderedSet() {
     static_assert((std::is_same<typename It::reference, ValueTp const&>::value), "");
     static_assert((std::is_same<typename It::pointer, CPtrT>::value), "");
     static_assert((std::is_same<typename It::difference_type, Diff>::value), "");
-
   }
   {
     typedef typename Set::local_iterator It;
diff --git a/libcxx/test/std/containers/unord/unord.map/allocator_mismatch.verify.cpp b/libcxx/test/std/containers/unord/unord.map/allocator_mismatch.verify.cpp
index e57505d889f37..189c44cec52fb 100644
--- a/libcxx/test/std/containers/unord/unord.map/allocator_mismatch.verify.cpp
+++ b/libcxx/test/std/containers/unord/unord.map/allocator_mismatch.verify.cpp
@@ -12,4 +12,4 @@
 #include <unordered_map>
 
 std::unordered_map<int, int, std::hash<int>, std::less<int>, std::allocator<long> > m;
-  // expected-error-re@*:* {{static assertion failed{{.*}}Allocator::value_type must be same type as value_type}}
+// expected-error-re@*:* {{static assertion failed{{.*}}Allocator::value_type must be same type as value_type}}
diff --git a/libcxx/test/std/containers/unord/unord.map/bucket.pass.cpp b/libcxx/test/std/containers/unord/unord.map/bucket.pass.cpp
index 14e5b2fe0ad36..a842828c7f649 100644
--- a/libcxx/test/std/containers/unord/unord.map/bucket.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.map/bucket.pass.cpp
@@ -22,47 +22,48 @@
 #include "test_macros.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef std::unordered_map<int, std::string> C;
-        typedef std::pair<int, std::string> P;
-        P a[] =
-        {
-            P(1, "one"),
-            P(2, "two"),
-            P(3, "three"),
-            P(4, "four"),
-            P(1, "four"),
-            P(2, "four"),
-        };
-        const C c(std::begin(a), std::end(a));
-        std::size_t bc = c.bucket_count();
-        assert(bc >= 5);
-        for (std::size_t i = 0; i < 13; ++i)
-            LIBCPP_ASSERT(c.bucket(i) == i % bc);
-    }
+int main(int, char**) {
+  {
+    typedef std::unordered_map<int, std::string> C;
+    typedef std::pair<int, std::string> P;
+    P a[] = {
+        P(1, "one"),
+        P(2, "two"),
+        P(3, "three"),
+        P(4, "four"),
+        P(1, "four"),
+        P(2, "four"),
+    };
+    const C c(std::begin(a), std::end(a));
+    std::size_t bc = c.bucket_count();
+    assert(bc >= 5);
+    for (std::size_t i = 0; i < 13; ++i)
+      LIBCPP_ASSERT(c.bucket(i) == i % bc);
+  }
 #if TEST_STD_VER >= 11
-    {
-        typedef std::unordered_map<int, std::string, std::hash<int>, std::equal_to<int>,
-                            min_allocator<std::pair<const int, std::string>>> C;
-        typedef std::pair<int, std::string> P;
-        P a[] =
-        {
-            P(1, "one"),
-            P(2, "two"),
-            P(3, "three"),
-            P(4, "four"),
-            P(1, "four"),
-            P(2, "four"),
-        };
-        const C c(std::begin(a), std::end(a));
-        std::size_t bc = c.bucket_count();
-        assert(bc >= 5);
-        for (std::size_t i = 0; i < 13; ++i)
-            LIBCPP_ASSERT(c.bucket(i) == i % bc);
-    }
+  {
+    typedef std::unordered_map<int,
+                               std::string,
+                               std::hash<int>,
+                               std::equal_to<int>,
+                               min_allocator<std::pair<const int, std::string>>>
+        C;
+    typedef std::pair<int, std::string> P;
+    P a[] = {
+        P(1, "one"),
+        P(2, "two"),
+        P(3, "three"),
+        P(4, "four"),
+        P(1, "four"),
+        P(2, "four"),
+    };
+    const C c(std::begin(a), std::end(a));
+    std::size_t bc = c.bucket_count();
+    assert(bc >= 5);
+    for (std::size_t i = 0; i < 13; ++i)
+      LIBCPP_ASSERT(c.bucket(i) == i % bc);
+  }
 #endif
 
-    return 0;
+  return 0;
 }
diff --git a/libcxx/test/std/containers/unord/unord.map/bucket_count.pass.cpp b/libcxx/test/std/containers/unord/unord.map/bucket_count.pass.cpp
index 6283408cef6f0..23ce829655714 100644
--- a/libcxx/test/std/containers/unord/unord.map/bucket_count.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.map/bucket_count.pass.cpp
@@ -22,55 +22,60 @@
 #include "test_macros.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef std::unordered_map<int, std::string> C;
-        const C c;
-        LIBCPP_ASSERT(c.bucket_count() == 0);
-    }
-    {
-        typedef std::unordered_map<int, std::string> C;
-        typedef std::pair<int, std::string> P;
-        P a[] =
-        {
-            P(10, "ten"),
-            P(20, "twenty"),
-            P(30, "thirty"),
-            P(40, "forty"),
-            P(50, "fifty"),
-            P(60, "sixty"),
-            P(70, "seventy"),
-            P(80, "eighty"),
-        };
-        const C c(std::begin(a), std::end(a));
-        assert(c.bucket_count() >= 8);
-    }
+int main(int, char**) {
+  {
+    typedef std::unordered_map<int, std::string> C;
+    const C c;
+    LIBCPP_ASSERT(c.bucket_count() == 0);
+  }
+  {
+    typedef std::unordered_map<int, std::string> C;
+    typedef std::pair<int, std::string> P;
+    P a[] = {
+        P(10, "ten"),
+        P(20, "twenty"),
+        P(30, "thirty"),
+        P(40, "forty"),
+        P(50, "fifty"),
+        P(60, "sixty"),
+        P(70, "seventy"),
+        P(80, "eighty"),
+    };
+    const C c(std::begin(a), std::end(a));
+    assert(c.bucket_count() >= 8);
+  }
 #if TEST_STD_VER >= 11
-    {
-        typedef std::unordered_map<int, std::string, std::hash<int>, std::equal_to<int>,
-                            min_allocator<std::pair<const int, std::string>>> C;
-        const C c;
-        LIBCPP_ASSERT(c.bucket_count() == 0);
-    }
-    {
-        typedef std::unordered_map<int, std::string, std::hash<int>, std::equal_to<int>,
-                            min_allocator<std::pair<const int, std::string>>> C;
-        typedef std::pair<int, std::string> P;
-        P a[] =
-        {
-            P(10, "ten"),
-            P(20, "twenty"),
-            P(30, "thirty"),
-            P(40, "forty"),
-            P(50, "fifty"),
-            P(60, "sixty"),
-            P(70, "seventy"),
-            P(80, "eighty"),
-        };
-        const C c(std::begin(a), std::end(a));
-        assert(c.bucket_count() >= 8);
-    }
+  {
+    typedef std::unordered_map<int,
+                               std::string,
+                               std::hash<int>,
+                               std::equal_to<int>,
+                               min_allocator<std::pair<const int, std::string>>>
+        C;
+    const C c;
+    LIBCPP_ASSERT(c.bucket_count() == 0);
+  }
+  {
+    typedef std::unordered_map<int,
+                               std::string,
+                               std::hash<int>,
+                               std::equal_to<int>,
+                               min_allocator<std::pair<const int, std::string>>>
+        C;
+    typedef std::pair<int, std::string> P;
+    P a[] = {
+        P(10, "ten"),
+        P(20, "twenty"),
+        P(30, "thirty"),
+        P(40, "forty"),
+        P(50, "fifty"),
+        P(60, "sixty"),
+        P(70, "seventy"),
+        P(80, "eighty"),
+    };
+    const C c(std::begin(a), std::end(a));
+    assert(c.bucket_count() >= 8);
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/unord/unord.map/bucket_size.pass.cpp b/libcxx/test/std/containers/unord/unord.map/bucket_size.pass.cpp
index 110ac5e45eec1..b1ad906d05405 100644
--- a/libcxx/test/std/containers/unord/unord.map/bucket_size.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.map/bucket_size.pass.cpp
@@ -22,51 +22,52 @@
 #include "test_macros.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef std::unordered_map<int, std::string> C;
-        typedef std::pair<int, std::string> P;
-        P a[] =
-        {
-            P(1, "one"),
-            P(2, "two"),
-            P(3, "three"),
-            P(4, "four"),
-            P(1, "four"),
-            P(2, "four"),
-        };
-        const C c(std::begin(a), std::end(a));
-        assert(c.bucket_count() >= 5);
-        LIBCPP_ASSERT(c.bucket_size(0) == 0);
-        LIBCPP_ASSERT(c.bucket_size(1) == 1);
-        LIBCPP_ASSERT(c.bucket_size(2) == 1);
-        LIBCPP_ASSERT(c.bucket_size(3) == 1);
-        LIBCPP_ASSERT(c.bucket_size(4) == 1);
-    }
+int main(int, char**) {
+  {
+    typedef std::unordered_map<int, std::string> C;
+    typedef std::pair<int, std::string> P;
+    P a[] = {
+        P(1, "one"),
+        P(2, "two"),
+        P(3, "three"),
+        P(4, "four"),
+        P(1, "four"),
+        P(2, "four"),
+    };
+    const C c(std::begin(a), std::end(a));
+    assert(c.bucket_count() >= 5);
+    LIBCPP_ASSERT(c.bucket_size(0) == 0);
+    LIBCPP_ASSERT(c.bucket_size(1) == 1);
+    LIBCPP_ASSERT(c.bucket_size(2) == 1);
+    LIBCPP_ASSERT(c.bucket_size(3) == 1);
+    LIBCPP_ASSERT(c.bucket_size(4) == 1);
+  }
 #if TEST_STD_VER >= 11
-    {
-        typedef std::unordered_map<int, std::string, std::hash<int>, std::equal_to<int>,
-                            min_allocator<std::pair<const int, std::string>>> C;
-        typedef std::pair<int, std::string> P;
-        P a[] =
-        {
-            P(1, "one"),
-            P(2, "two"),
-            P(3, "three"),
-            P(4, "four"),
-            P(1, "four"),
-            P(2, "four"),
-        };
-        const C c(std::begin(a), std::end(a));
-        assert(c.bucket_count() >= 5);
-        LIBCPP_ASSERT(c.bucket_size(0) == 0);
-        LIBCPP_ASSERT(c.bucket_size(1) == 1);
-        LIBCPP_ASSERT(c.bucket_size(2) == 1);
-        LIBCPP_ASSERT(c.bucket_size(3) == 1);
-        LIBCPP_ASSERT(c.bucket_size(4) == 1);
-    }
+  {
+    typedef std::unordered_map<int,
+                               std::string,
+                               std::hash<int>,
+                               std::equal_to<int>,
+                               min_allocator<std::pair<const int, std::string>>>
+        C;
+    typedef std::pair<int, std::string> P;
+    P a[] = {
+        P(1, "one"),
+        P(2, "two"),
+        P(3, "three"),
+        P(4, "four"),
+        P(1, "four"),
+        P(2, "four"),
+    };
+    const C c(std::begin(a), std::end(a));
+    assert(c.bucket_count() >= 5);
+    LIBCPP_ASSERT(c.bucket_size(0) == 0);
+    LIBCPP_ASSERT(c.bucket_size(1) == 1);
+    LIBCPP_ASSERT(c.bucket_size(2) == 1);
+    LIBCPP_ASSERT(c.bucket_size(3) == 1);
+    LIBCPP_ASSERT(c.bucket_size(4) == 1);
+  }
 #endif
 
-    return 0;
+  return 0;
 }
diff --git a/libcxx/test/std/containers/unord/unord.map/compare.pass.cpp b/libcxx/test/std/containers/unord/unord.map/compare.pass.cpp
index a8474f8f0bdc8..c122ca75ad238 100644
--- a/libcxx/test/std/containers/unord/unord.map/compare.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.map/compare.pass.cpp
@@ -21,8 +21,9 @@
 #include "test_macros.h"
 
 struct Key {
-  template <typename T> Key(const T&) {}
-  bool operator== (const Key&) const { return true; }
+  template <typename T>
+  Key(const T&) {}
+  bool operator==(const Key&) const { return true; }
 };
 
 template <>
@@ -30,16 +31,15 @@ struct std::hash<Key> {
   std::size_t operator()(Key const&) const { return 0; }
 };
 
-int main(int, char**)
-{
-    typedef std::unordered_map<Key, int> MapT;
-    typedef MapT::iterator Iter;
-    MapT map;
-    Iter it = map.find(Key(0));
-    assert(it == map.end());
-    std::pair<Iter, bool> result = map.insert(std::make_pair(Key(0), 42));
-    assert(result.second);
-    assert(result.first->second == 42);
+int main(int, char**) {
+  typedef std::unordered_map<Key, int> MapT;
+  typedef MapT::iterator Iter;
+  MapT map;
+  Iter it = map.find(Key(0));
+  assert(it == map.end());
+  std::pair<Iter, bool> result = map.insert(std::make_pair(Key(0), 42));
+  assert(result.second);
+  assert(result.first->second == 42);
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/unord/unord.map/contains.pass.cpp b/libcxx/test/std/containers/unord/unord.map/contains.pass.cpp
index e757517fc8d13..5fde843433f1f 100644
--- a/libcxx/test/std/containers/unord/unord.map/contains.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.map/contains.pass.cpp
@@ -17,45 +17,44 @@
 
 template <typename T, typename P, typename B, typename... Pairs>
 void test(B bad, Pairs... args) {
-    T map;
-    P pairs[] = {args...};
+  T map;
+  P pairs[] = {args...};
 
-    for (auto& p : pairs) map.insert(p);
-    for (auto& p : pairs) assert(map.contains(p.first));
+  for (auto& p : pairs)
+    map.insert(p);
+  for (auto& p : pairs)
+    assert(map.contains(p.first));
 
-    assert(!map.contains(bad));
+  assert(!map.contains(bad));
 }
 
-struct E { int a = 1; double b = 1; char c = 1; };
-
-int main(int, char**)
-{
-    {
-        test<std::unordered_map<char, int>, std::pair<char, int> >(
-            'e', std::make_pair('a', 10), std::make_pair('b', 11),
-            std::make_pair('c', 12), std::make_pair('d', 13));
-
-        test<std::unordered_map<char, char>, std::pair<char, char> >(
-            'e', std::make_pair('a', 'a'), std::make_pair('b', 'a'),
-            std::make_pair('c', 'a'), std::make_pair('d', 'b'));
-
-        test<std::unordered_map<int, E>, std::pair<int, E> >(
-            -1, std::make_pair(1, E{}), std::make_pair(2, E{}),
-            std::make_pair(3, E{}), std::make_pair(4, E{}));
-    }
-    {
-        test<std::unordered_multimap<char, int>, std::pair<char, int> >(
-            'e', std::make_pair('a', 10), std::make_pair('b', 11),
-            std::make_pair('c', 12), std::make_pair('d', 13));
-
-        test<std::unordered_multimap<char, char>, std::pair<char, char> >(
-            'e', std::make_pair('a', 'a'), std::make_pair('b', 'a'),
-            std::make_pair('c', 'a'), std::make_pair('d', 'b'));
-
-        test<std::unordered_multimap<int, E>, std::pair<int, E> >(
-            -1, std::make_pair(1, E{}), std::make_pair(2, E{}),
-            std::make_pair(3, E{}), std::make_pair(4, E{}));
-    }
-
-    return 0;
+struct E {
+  int a    = 1;
+  double b = 1;
+  char c   = 1;
+};
+
+int main(int, char**) {
+  {
+    test<std::unordered_map<char, int>, std::pair<char, int> >(
+        'e', std::make_pair('a', 10), std::make_pair('b', 11), std::make_pair('c', 12), std::make_pair('d', 13));
+
+    test<std::unordered_map<char, char>, std::pair<char, char> >(
+        'e', std::make_pair('a', 'a'), std::make_pair('b', 'a'), std::make_pair('c', 'a'), std::make_pair('d', 'b'));
+
+    test<std::unordered_map<int, E>, std::pair<int, E> >(
+        -1, std::make_pair(1, E{}), std::make_pair(2, E{}), std::make_pair(3, E{}), std::make_pair(4, E{}));
+  }
+  {
+    test<std::unordered_multimap<char, int>, std::pair<char, int> >(
+        'e', std::make_pair('a', 10), std::make_pair('b', 11), std::make_pair('c', 12), std::make_pair('d', 13));
+
+    test<std::unordered_multimap<char, char>, std::pair<char, char> >(
+        'e', std::make_pair('a', 'a'), std::make_pair('b', 'a'), std::make_pair('c', 'a'), std::make_pair('d', 'b'));
+
+    test<std::unordered_multimap<int, E>, std::pair<int, E> >(
+        -1, std::make_pair(1, E{}), std::make_pair(2, E{}), std::make_pair(3, E{}), std::make_pair(4, E{}));
+  }
+
+  return 0;
 }
diff --git a/libcxx/test/std/containers/unord/unord.map/contains.transparent.pass.cpp b/libcxx/test/std/containers/unord/unord.map/contains.transparent.pass.cpp
index b8742cdb4b6c4..a774995892e25 100644
--- a/libcxx/test/std/containers/unord/unord.map/contains.transparent.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.map/contains.transparent.pass.cpp
@@ -20,44 +20,43 @@
 #include <unordered_map>
 #include "test_transparent_unordered.h"
 
-int main(int, char**)
-{
-    using key_type = StoredType<int>;
-
-    {
-      // Make sure conversions don't happen for transparent non-final hasher and key_equal
-      using M = unord_map_type<std::unordered_map, transparent_hash, std::equal_to<>>;
-      test_transparent_contains<M>({{1, 2}, {2, 3}});
-      test_transparent_contains<const M>({{1, 2}, {2, 3}});
-    }
-
-    {
-      // Make sure conversions don't happen for transparent final hasher and key_equal
-      using M = unord_map_type<std::unordered_map, transparent_hash_final, transparent_equal_final>;
-      test_transparent_contains<M>({{1, 2}, {2, 3}});
-      test_transparent_contains<const M>({{1, 2}, {2, 3}});
-    }
-
-    {
-      // Make sure conversions do happen for non-transparent hasher
-      using M = unord_map_type<std::unordered_map, non_transparent_hash, std::equal_to<>>;
-      test_non_transparent_contains<M>({{1, 2}, {2, 3}});
-      test_non_transparent_contains<const M>({{1, 2}, {2, 3}});
-    }
-
-    {
-      // Make sure conversions do happen for non-transparent key_equal
-      using M = unord_map_type<std::unordered_map, transparent_hash, std::equal_to<key_type>>;
-      test_non_transparent_contains<M>({{1, 2}, {2, 3}});
-      test_non_transparent_contains<const M>({{1, 2}, {2, 3}});
-    }
-
-    {
-      // Make sure conversions do happen for both non-transparent hasher and key_equal
-      using M = unord_map_type<std::unordered_map, non_transparent_hash, std::equal_to<key_type>>;
-      test_non_transparent_contains<M>({{1, 2}, {2, 3}});
-      test_non_transparent_contains<const M>({{1, 2}, {2, 3}});
-    }
-
-    return 0;
+int main(int, char**) {
+  using key_type = StoredType<int>;
+
+  {
+    // Make sure conversions don't happen for transparent non-final hasher and key_equal
+    using M = unord_map_type<std::unordered_map, transparent_hash, std::equal_to<>>;
+    test_transparent_contains<M>({{1, 2}, {2, 3}});
+    test_transparent_contains<const M>({{1, 2}, {2, 3}});
+  }
+
+  {
+    // Make sure conversions don't happen for transparent final hasher and key_equal
+    using M = unord_map_type<std::unordered_map, transparent_hash_final, transparent_equal_final>;
+    test_transparent_contains<M>({{1, 2}, {2, 3}});
+    test_transparent_contains<const M>({{1, 2}, {2, 3}});
+  }
+
+  {
+    // Make sure conversions do happen for non-transparent hasher
+    using M = unord_map_type<std::unordered_map, non_transparent_hash, std::equal_to<>>;
+    test_non_transparent_contains<M>({{1, 2}, {2, 3}});
+    test_non_transparent_contains<const M>({{1, 2}, {2, 3}});
+  }
+
+  {
+    // Make sure conversions do happen for non-transparent key_equal
+    using M = unord_map_type<std::unordered_map, transparent_hash, std::equal_to<key_type>>;
+    test_non_transparent_contains<M>({{1, 2}, {2, 3}});
+    test_non_transparent_contains<const M>({{1, 2}, {2, 3}});
+  }
+
+  {
+    // Make sure conversions do happen for both non-transparent hasher and key_equal
+    using M = unord_map_type<std::unordered_map, non_transparent_hash, std::equal_to<key_type>>;
+    test_non_transparent_contains<M>({{1, 2}, {2, 3}});
+    test_non_transparent_contains<const M>({{1, 2}, {2, 3}});
+  }
+
+  return 0;
 }
diff --git a/libcxx/test/std/containers/unord/unord.map/count.pass.cpp b/libcxx/test/std/containers/unord/unord.map/count.pass.cpp
index a003208ab0256..34bbbd6d2b243 100644
--- a/libcxx/test/std/containers/unord/unord.map/count.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.map/count.pass.cpp
@@ -22,46 +22,47 @@
 #include "test_macros.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef std::unordered_map<int, std::string> C;
-        typedef std::pair<int, std::string> P;
-        P a[] =
-        {
-            P(10, "ten"),
-            P(20, "twenty"),
-            P(30, "thirty"),
-            P(40, "forty"),
-            P(50, "fifty"),
-            P(60, "sixty"),
-            P(70, "seventy"),
-            P(80, "eighty"),
-        };
-        const C c(std::begin(a), std::end(a));
-        assert(c.count(30) == 1);
-        assert(c.count(5) == 0);
-    }
+int main(int, char**) {
+  {
+    typedef std::unordered_map<int, std::string> C;
+    typedef std::pair<int, std::string> P;
+    P a[] = {
+        P(10, "ten"),
+        P(20, "twenty"),
+        P(30, "thirty"),
+        P(40, "forty"),
+        P(50, "fifty"),
+        P(60, "sixty"),
+        P(70, "seventy"),
+        P(80, "eighty"),
+    };
+    const C c(std::begin(a), std::end(a));
+    assert(c.count(30) == 1);
+    assert(c.count(5) == 0);
+  }
 #if TEST_STD_VER >= 11
-    {
-        typedef std::unordered_map<int, std::string, std::hash<int>, std::equal_to<int>,
-                            min_allocator<std::pair<const int, std::string>>> C;
-        typedef std::pair<int, std::string> P;
-        P a[] =
-        {
-            P(10, "ten"),
-            P(20, "twenty"),
-            P(30, "thirty"),
-            P(40, "forty"),
-            P(50, "fifty"),
-            P(60, "sixty"),
-            P(70, "seventy"),
-            P(80, "eighty"),
-        };
-        const C c(std::begin(a), std::end(a));
-        assert(c.count(30) == 1);
-        assert(c.count(5) == 0);
-    }
+  {
+    typedef std::unordered_map<int,
+                               std::string,
+                               std::hash<int>,
+                               std::equal_to<int>,
+                               min_allocator<std::pair<const int, std::string>>>
+        C;
+    typedef std::pair<int, std::string> P;
+    P a[] = {
+        P(10, "ten"),
+        P(20, "twenty"),
+        P(30, "thirty"),
+        P(40, "forty"),
+        P(50, "fifty"),
+        P(60, "sixty"),
+        P(70, "seventy"),
+        P(80, "eighty"),
+    };
+    const C c(std::begin(a), std::end(a));
+    assert(c.count(30) == 1);
+    assert(c.count(5) == 0);
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/unord/unord.map/count.transparent.pass.cpp b/libcxx/test/std/containers/unord/unord.map/count.transparent.pass.cpp
index 1ae5a94d256a5..ad7dafedfeb48 100644
--- a/libcxx/test/std/containers/unord/unord.map/count.transparent.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.map/count.transparent.pass.cpp
@@ -21,44 +21,43 @@
 
 #include "test_transparent_unordered.h"
 
-int main(int, char**)
-{
-    using key_type = StoredType<int>;
-
-    {
-      // Make sure conversions don't happen for transparent non-final hasher and key_equal
-      using M = unord_map_type<std::unordered_map, transparent_hash, std::equal_to<>>;
-      test_transparent_count<M>({{1, 2}, {2, 3}});
-      test_transparent_count<const M>({{1, 2}, {2, 3}});
-    }
-
-    {
-      // Make sure conversions don't happen for transparent final hasher and key_equal
-      using M = unord_map_type<std::unordered_map, transparent_hash_final, transparent_equal_final>;
-      test_transparent_count<M>({{1, 2}, {2, 3}});
-      test_transparent_count<const M>({{1, 2}, {2, 3}});
-    }
-
-    {
-      // Make sure conversions do happen for non-transparent hasher
-      using M = unord_map_type<std::unordered_map, non_transparent_hash, std::equal_to<>>;
-      test_non_transparent_count<M>({{1, 2}, {2, 3}});
-      test_non_transparent_count<const M>({{1, 2}, {2, 3}});
-    }
-
-    {
-      // Make sure conversions do happen for non-transparent key_equal
-      using M = unord_map_type<std::unordered_map, transparent_hash, std::equal_to<key_type>>;
-      test_non_transparent_count<M>({{1, 2}, {2, 3}});
-      test_non_transparent_count<const M>({{1, 2}, {2, 3}});
-    }
-
-    {
-      // Make sure conversions do happen for both non-transparent hasher and key_equal
-      using M = unord_map_type<std::unordered_map, non_transparent_hash, std::equal_to<key_type>>;
-      test_non_transparent_count<M>({{1, 2}, {2, 3}});
-      test_non_transparent_count<const M>({{1, 2}, {2, 3}});
-    }
-
-    return 0;
+int main(int, char**) {
+  using key_type = StoredType<int>;
+
+  {
+    // Make sure conversions don't happen for transparent non-final hasher and key_equal
+    using M = unord_map_type<std::unordered_map, transparent_hash, std::equal_to<>>;
+    test_transparent_count<M>({{1, 2}, {2, 3}});
+    test_transparent_count<const M>({{1, 2}, {2, 3}});
+  }
+
+  {
+    // Make sure conversions don't happen for transparent final hasher and key_equal
+    using M = unord_map_type<std::unordered_map, transparent_hash_final, transparent_equal_final>;
+    test_transparent_count<M>({{1, 2}, {2, 3}});
+    test_transparent_count<const M>({{1, 2}, {2, 3}});
+  }
+
+  {
+    // Make sure conversions do happen for non-transparent hasher
+    using M = unord_map_type<std::unordered_map, non_transparent_hash, std::equal_to<>>;
+    test_non_transparent_count<M>({{1, 2}, {2, 3}});
+    test_non_transparent_count<const M>({{1, 2}, {2, 3}});
+  }
+
+  {
+    // Make sure conversions do happen for non-transparent key_equal
+    using M = unord_map_type<std::unordered_map, transparent_hash, std::equal_to<key_type>>;
+    test_non_transparent_count<M>({{1, 2}, {2, 3}});
+    test_non_transparent_count<const M>({{1, 2}, {2, 3}});
+  }
+
+  {
+    // Make sure conversions do happen for both non-transparent hasher and key_equal
+    using M = unord_map_type<std::unordered_map, non_transparent_hash, std::equal_to<key_type>>;
+    test_non_transparent_count<M>({{1, 2}, {2, 3}});
+    test_non_transparent_count<const M>({{1, 2}, {2, 3}});
+  }
+
+  return 0;
 }
diff --git a/libcxx/test/std/containers/unord/unord.map/empty.pass.cpp b/libcxx/test/std/containers/unord/unord.map/empty.pass.cpp
index da6d48df2d57c..bf2219f34a811 100644
--- a/libcxx/test/std/containers/unord/unord.map/empty.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.map/empty.pass.cpp
@@ -18,9 +18,8 @@
 #include "test_macros.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
+int main(int, char**) {
+  {
     typedef std::unordered_map<int, double> M;
     M m;
     ASSERT_NOEXCEPT(m.empty());
@@ -29,10 +28,12 @@ int main(int, char**)
     assert(!m.empty());
     m.clear();
     assert(m.empty());
-    }
+  }
 #if TEST_STD_VER >= 11
-    {
-    typedef std::unordered_map<int, double, std::hash<int>, std::equal_to<int>, min_allocator<std::pair<const int, double>>> M;
+  {
+    typedef std::
+        unordered_map<int, double, std::hash<int>, std::equal_to<int>, min_allocator<std::pair<const int, double>>>
+            M;
     M m;
     ASSERT_NOEXCEPT(m.empty());
     assert(m.empty());
@@ -40,7 +41,7 @@ int main(int, char**)
     assert(!m.empty());
     m.clear();
     assert(m.empty());
-    }
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/unord/unord.map/empty.verify.cpp b/libcxx/test/std/containers/unord/unord.map/empty.verify.cpp
index ada96fdf89157..da8f23cf9da0d 100644
--- a/libcxx/test/std/containers/unord/unord.map/empty.verify.cpp
+++ b/libcxx/test/std/containers/unord/unord.map/empty.verify.cpp
@@ -17,6 +17,6 @@
 #include <unordered_map>
 
 void f() {
-    std::unordered_map<int, int> c;
-    c.empty(); // expected-warning {{ignoring return value of function declared with 'nodiscard' attribute}}
+  std::unordered_map<int, int> c;
+  c.empty(); // expected-warning {{ignoring return value of function declared with 'nodiscard' attribute}}
 }
diff --git a/libcxx/test/std/containers/unord/unord.map/eq.pass.cpp b/libcxx/test/std/containers/unord/unord.map/eq.pass.cpp
index 6fd22ab992ee3..51c804c9292d1 100644
--- a/libcxx/test/std/containers/unord/unord.map/eq.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.map/eq.pass.cpp
@@ -28,173 +28,180 @@
 
 #include "test_comparisons.h"
 
-int main(int, char**)
-{
-    {
-        typedef std::unordered_map<int, std::string> C;
-        typedef std::pair<int, std::string> P;
-        P a[] =
-        {
-            P(10, "ten"),
-            P(20, "twenty"),
-            P(30, "thirty"),
-            P(40, "forty"),
-            P(50, "fifty"),
-            P(60, "sixty"),
-            P(70, "seventy"),
-            P(80, "eighty"),
-        };
-        const C c1(std::begin(a), std::end(a));
-        const C c2;
-        assert(testEquality(c1, c2, false));
-    }
-    {
-        typedef std::unordered_map<int, std::string> C;
-        typedef std::pair<int, std::string> P;
-        P a[] =
-        {
-            P(10, "ten"),
-            P(20, "twenty"),
-            P(30, "thirty"),
-            P(40, "forty"),
-            P(50, "fifty"),
-            P(60, "sixty"),
-            P(70, "seventy"),
-            P(80, "eighty"),
-        };
-        const C c1(std::begin(a), std::end(a));
-        const C c2 = c1;
-        assert(testEquality(c1, c2, true));
-    }
-    {
-        typedef std::unordered_map<int, std::string> C;
-        typedef std::pair<int, std::string> P;
-        P a[] =
-        {
-            P(10, "ten"),
-            P(20, "twenty"),
-            P(30, "thirty"),
-            P(40, "forty"),
-            P(50, "fifty"),
-            P(60, "sixty"),
-            P(70, "seventy"),
-            P(80, "eighty"),
-        };
-        C c1(std::begin(a), std::end(a));
-        C c2 = c1;
-        c2.rehash(30);
-        assert(testEquality(c1, c2, true));
-        c2.insert(P(90, "ninety"));
-        assert(testEquality(c1, c2, false));
-        c1.insert(P(90, "ninety"));
-        assert(testEquality(c1, c2, true));
-    }
-    {
-        typedef std::unordered_map<int, std::string> C;
-        typedef std::pair<int, std::string> P;
-        P a[] =
-        {
-            P(10, "ten"),
-            P(20, "twenty"),
-            P(30, "thirty"),
-            P(40, "forty"),
-            P(50, "fifty"),
-            P(60, "sixty"),
-            P(70, "seventy"),
-            P(80, "eighty"),
-        };
-        C c1(std::begin(a), std::end(a));
-        C c2 = c1;
-        assert(testEquality(c1, c2, true));
-        c1.insert(P(90, "ninety"));
-        c2.insert(P(100, "onehundred"));
-        assert(testEquality(c1, c2, false));
-    }
+int main(int, char**) {
+  {
+    typedef std::unordered_map<int, std::string> C;
+    typedef std::pair<int, std::string> P;
+    P a[] = {
+        P(10, "ten"),
+        P(20, "twenty"),
+        P(30, "thirty"),
+        P(40, "forty"),
+        P(50, "fifty"),
+        P(60, "sixty"),
+        P(70, "seventy"),
+        P(80, "eighty"),
+    };
+    const C c1(std::begin(a), std::end(a));
+    const C c2;
+    assert(testEquality(c1, c2, false));
+  }
+  {
+    typedef std::unordered_map<int, std::string> C;
+    typedef std::pair<int, std::string> P;
+    P a[] = {
+        P(10, "ten"),
+        P(20, "twenty"),
+        P(30, "thirty"),
+        P(40, "forty"),
+        P(50, "fifty"),
+        P(60, "sixty"),
+        P(70, "seventy"),
+        P(80, "eighty"),
+    };
+    const C c1(std::begin(a), std::end(a));
+    const C c2 = c1;
+    assert(testEquality(c1, c2, true));
+  }
+  {
+    typedef std::unordered_map<int, std::string> C;
+    typedef std::pair<int, std::string> P;
+    P a[] = {
+        P(10, "ten"),
+        P(20, "twenty"),
+        P(30, "thirty"),
+        P(40, "forty"),
+        P(50, "fifty"),
+        P(60, "sixty"),
+        P(70, "seventy"),
+        P(80, "eighty"),
+    };
+    C c1(std::begin(a), std::end(a));
+    C c2 = c1;
+    c2.rehash(30);
+    assert(testEquality(c1, c2, true));
+    c2.insert(P(90, "ninety"));
+    assert(testEquality(c1, c2, false));
+    c1.insert(P(90, "ninety"));
+    assert(testEquality(c1, c2, true));
+  }
+  {
+    typedef std::unordered_map<int, std::string> C;
+    typedef std::pair<int, std::string> P;
+    P a[] = {
+        P(10, "ten"),
+        P(20, "twenty"),
+        P(30, "thirty"),
+        P(40, "forty"),
+        P(50, "fifty"),
+        P(60, "sixty"),
+        P(70, "seventy"),
+        P(80, "eighty"),
+    };
+    C c1(std::begin(a), std::end(a));
+    C c2 = c1;
+    assert(testEquality(c1, c2, true));
+    c1.insert(P(90, "ninety"));
+    c2.insert(P(100, "onehundred"));
+    assert(testEquality(c1, c2, false));
+  }
 #if TEST_STD_VER >= 11
-    {
-        typedef std::unordered_map<int, std::string, std::hash<int>, std::equal_to<int>,
-                            min_allocator<std::pair<const int, std::string>>> C;
-        typedef std::pair<int, std::string> P;
-        P a[] =
-        {
-            P(10, "ten"),
-            P(20, "twenty"),
-            P(30, "thirty"),
-            P(40, "forty"),
-            P(50, "fifty"),
-            P(60, "sixty"),
-            P(70, "seventy"),
-            P(80, "eighty"),
-        };
-        const C c1(std::begin(a), std::end(a));
-        const C c2;
-        assert(testEquality(c1, c2, false));
-    }
-    {
-        typedef std::unordered_map<int, std::string, std::hash<int>, std::equal_to<int>,
-                            min_allocator<std::pair<const int, std::string>>> C;
-        typedef std::pair<int, std::string> P;
-        P a[] =
-        {
-            P(10, "ten"),
-            P(20, "twenty"),
-            P(30, "thirty"),
-            P(40, "forty"),
-            P(50, "fifty"),
-            P(60, "sixty"),
-            P(70, "seventy"),
-            P(80, "eighty"),
-        };
-        const C c1(std::begin(a), std::end(a));
-        const C c2 = c1;
-        assert(testEquality(c1, c2, true));
-    }
-    {
-        typedef std::unordered_map<int, std::string, std::hash<int>, std::equal_to<int>,
-                            min_allocator<std::pair<const int, std::string>>> C;
-        typedef std::pair<int, std::string> P;
-        P a[] =
-        {
-            P(10, "ten"),
-            P(20, "twenty"),
-            P(30, "thirty"),
-            P(40, "forty"),
-            P(50, "fifty"),
-            P(60, "sixty"),
-            P(70, "seventy"),
-            P(80, "eighty"),
-        };
-        C c1(std::begin(a), std::end(a));
-        C c2 = c1;
-        c2.rehash(30);
-        assert(testEquality(c1, c2, true));
-        c2.insert(P(90, "ninety"));
-        assert(testEquality(c1, c2, false));
-        c1.insert(P(90, "ninety"));
-        assert(testEquality(c1, c2, true));
-    }
-    {
-        typedef std::unordered_map<int, std::string, std::hash<int>, std::equal_to<int>,
-                            min_allocator<std::pair<const int, std::string>>> C;
-        typedef std::pair<int, std::string> P;
-        P a[] =
-        {
-            P(10, "ten"),
-            P(20, "twenty"),
-            P(30, "thirty"),
-            P(40, "forty"),
-            P(50, "fifty"),
-            P(60, "sixty"),
-            P(70, "seventy"),
-            P(80, "eighty"),
-        };
-        C c1(std::begin(a), std::end(a));
-        C c2 = c1;
-        assert(testEquality(c1, c2, true));
-        c1.insert(P(90, "ninety"));
-        c2.insert(P(100, "onehundred"));
-        assert(testEquality(c1, c2, false));
-    }
+  {
+    typedef std::unordered_map<int,
+                               std::string,
+                               std::hash<int>,
+                               std::equal_to<int>,
+                               min_allocator<std::pair<const int, std::string>>>
+        C;
+    typedef std::pair<int, std::string> P;
+    P a[] = {
+        P(10, "ten"),
+        P(20, "twenty"),
+        P(30, "thirty"),
+        P(40, "forty"),
+        P(50, "fifty"),
+        P(60, "sixty"),
+        P(70, "seventy"),
+        P(80, "eighty"),
+    };
+    const C c1(std::begin(a), std::end(a));
+    const C c2;
+    assert(testEquality(c1, c2, false));
+  }
+  {
+    typedef std::unordered_map<int,
+                               std::string,
+                               std::hash<int>,
+                               std::equal_to<int>,
+                               min_allocator<std::pair<const int, std::string>>>
+        C;
+    typedef std::pair<int, std::string> P;
+    P a[] = {
+        P(10, "ten"),
+        P(20, "twenty"),
+        P(30, "thirty"),
+        P(40, "forty"),
+        P(50, "fifty"),
+        P(60, "sixty"),
+        P(70, "seventy"),
+        P(80, "eighty"),
+    };
+    const C c1(std::begin(a), std::end(a));
+    const C c2 = c1;
+    assert(testEquality(c1, c2, true));
+  }
+  {
+    typedef std::unordered_map<int,
+                               std::string,
+                               std::hash<int>,
+                               std::equal_to<int>,
+                               min_allocator<std::pair<const int, std::string>>>
+        C;
+    typedef std::pair<int, std::string> P;
+    P a[] = {
+        P(10, "ten"),
+        P(20, "twenty"),
+        P(30, "thirty"),
+        P(40, "forty"),
+        P(50, "fifty"),
+        P(60, "sixty"),
+        P(70, "seventy"),
+        P(80, "eighty"),
+    };
+    C c1(std::begin(a), std::end(a));
+    C c2 = c1;
+    c2.rehash(30);
+    assert(testEquality(c1, c2, true));
+    c2.insert(P(90, "ninety"));
+    assert(testEquality(c1, c2, false));
+    c1.insert(P(90, "ninety"));
+    assert(testEquality(c1, c2, true));
+  }
+  {
+    typedef std::unordered_map<int,
+                               std::string,
+                               std::hash<int>,
+                               std::equal_to<int>,
+                               min_allocator<std::pair<const int, std::string>>>
+        C;
+    typedef std::pair<int, std::string> P;
+    P a[] = {
+        P(10, "ten"),
+        P(20, "twenty"),
+        P(30, "thirty"),
+        P(40, "forty"),
+        P(50, "fifty"),
+        P(60, "sixty"),
+        P(70, "seventy"),
+        P(80, "eighty"),
+    };
+    C c1(std::begin(a), std::end(a));
+    C c2 = c1;
+    assert(testEquality(c1, c2, true));
+    c1.insert(P(90, "ninety"));
+    c2.insert(P(100, "onehundred"));
+    assert(testEquality(c1, c2, false));
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/unord/unord.map/equal_range.transparent.pass.cpp b/libcxx/test/std/containers/unord/unord.map/equal_range.transparent.pass.cpp
index 07b34f3110e3e..5504a5a998f1b 100644
--- a/libcxx/test/std/containers/unord/unord.map/equal_range.transparent.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.map/equal_range.transparent.pass.cpp
@@ -21,44 +21,43 @@
 
 #include "test_transparent_unordered.h"
 
-int main(int, char**)
-{
-    using key_type = StoredType<int>;
-
-    {
-      // Make sure conversions don't happen for transparent non-final hasher and key_equal
-      using M = unord_map_type<std::unordered_map, transparent_hash, std::equal_to<>>;
-      test_transparent_equal_range<M>({{1, 2}, {2, 3}});
-      test_transparent_equal_range<const M>({{1, 2}, {2, 3}});
-    }
-
-    {
-      // Make sure conversions don't happen for transparent final hasher and key_equal
-      using M = unord_map_type<std::unordered_map, transparent_hash_final, transparent_equal_final>;
-      test_transparent_equal_range<M>({{1, 2}, {2, 3}});
-      test_transparent_equal_range<const M>({{1, 2}, {2, 3}});
-    }
-
-    {
-      // Make sure conversions do happen for non-transparent hasher
-      using M = unord_map_type<std::unordered_map, non_transparent_hash, std::equal_to<>>;
-      test_non_transparent_equal_range<M>({{1, 2}, {2, 3}});
-      test_non_transparent_equal_range<const M>({{1, 2}, {2, 3}});
-    }
-
-    {
-      // Make sure conversions do happen for non-transparent key_equal
-      using M = unord_map_type<std::unordered_map, transparent_hash, std::equal_to<key_type>>;
-      test_non_transparent_equal_range<M>({{1, 2}, {2, 3}});
-      test_non_transparent_equal_range<const M>({{1, 2}, {2, 3}});
-    }
-
-    {
-      // Make sure conversions do happen for both non-transparent hasher and key_equal
-      using M = unord_map_type<std::unordered_map, non_transparent_hash, std::equal_to<key_type>>;
-      test_non_transparent_equal_range<M>({{1, 2}, {2, 3}});
-      test_non_transparent_equal_range<const M>({{1, 2}, {2, 3}});
-    }
-
-    return 0;
+int main(int, char**) {
+  using key_type = StoredType<int>;
+
+  {
+    // Make sure conversions don't happen for transparent non-final hasher and key_equal
+    using M = unord_map_type<std::unordered_map, transparent_hash, std::equal_to<>>;
+    test_transparent_equal_range<M>({{1, 2}, {2, 3}});
+    test_transparent_equal_range<const M>({{1, 2}, {2, 3}});
+  }
+
+  {
+    // Make sure conversions don't happen for transparent final hasher and key_equal
+    using M = unord_map_type<std::unordered_map, transparent_hash_final, transparent_equal_final>;
+    test_transparent_equal_range<M>({{1, 2}, {2, 3}});
+    test_transparent_equal_range<const M>({{1, 2}, {2, 3}});
+  }
+
+  {
+    // Make sure conversions do happen for non-transparent hasher
+    using M = unord_map_type<std::unordered_map, non_transparent_hash, std::equal_to<>>;
+    test_non_transparent_equal_range<M>({{1, 2}, {2, 3}});
+    test_non_transparent_equal_range<const M>({{1, 2}, {2, 3}});
+  }
+
+  {
+    // Make sure conversions do happen for non-transparent key_equal
+    using M = unord_map_type<std::unordered_map, transparent_hash, std::equal_to<key_type>>;
+    test_non_transparent_equal_range<M>({{1, 2}, {2, 3}});
+    test_non_transparent_equal_range<const M>({{1, 2}, {2, 3}});
+  }
+
+  {
+    // Make sure conversions do happen for both non-transparent hasher and key_equal
+    using M = unord_map_type<std::unordered_map, non_transparent_hash, std::equal_to<key_type>>;
+    test_non_transparent_equal_range<M>({{1, 2}, {2, 3}});
+    test_non_transparent_equal_range<const M>({{1, 2}, {2, 3}});
+  }
+
+  return 0;
 }
diff --git a/libcxx/test/std/containers/unord/unord.map/equal_range_const.pass.cpp b/libcxx/test/std/containers/unord/unord.map/equal_range_const.pass.cpp
index eb8beaec3db14..c4c3925fea063 100644
--- a/libcxx/test/std/containers/unord/unord.map/equal_range_const.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.map/equal_range_const.pass.cpp
@@ -22,56 +22,57 @@
 #include "test_macros.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef std::unordered_map<int, std::string> C;
-        typedef C::const_iterator I;
-        typedef std::pair<int, std::string> P;
-        P a[] =
-        {
-            P(10, "ten"),
-            P(20, "twenty"),
-            P(30, "thirty"),
-            P(40, "forty"),
-            P(50, "fifty"),
-            P(60, "sixty"),
-            P(70, "seventy"),
-            P(80, "eighty"),
-        };
-        const C c(std::begin(a), std::end(a));
-        std::pair<I, I> r = c.equal_range(30);
-        assert(std::distance(r.first, r.second) == 1);
-        assert(r.first->first == 30);
-        assert(r.first->second == "thirty");
-        r = c.equal_range(5);
-        assert(std::distance(r.first, r.second) == 0);
-    }
+int main(int, char**) {
+  {
+    typedef std::unordered_map<int, std::string> C;
+    typedef C::const_iterator I;
+    typedef std::pair<int, std::string> P;
+    P a[] = {
+        P(10, "ten"),
+        P(20, "twenty"),
+        P(30, "thirty"),
+        P(40, "forty"),
+        P(50, "fifty"),
+        P(60, "sixty"),
+        P(70, "seventy"),
+        P(80, "eighty"),
+    };
+    const C c(std::begin(a), std::end(a));
+    std::pair<I, I> r = c.equal_range(30);
+    assert(std::distance(r.first, r.second) == 1);
+    assert(r.first->first == 30);
+    assert(r.first->second == "thirty");
+    r = c.equal_range(5);
+    assert(std::distance(r.first, r.second) == 0);
+  }
 #if TEST_STD_VER >= 11
-    {
-        typedef std::unordered_map<int, std::string, std::hash<int>, std::equal_to<int>,
-                            min_allocator<std::pair<const int, std::string>>> C;
-        typedef C::const_iterator I;
-        typedef std::pair<int, std::string> P;
-        P a[] =
-        {
-            P(10, "ten"),
-            P(20, "twenty"),
-            P(30, "thirty"),
-            P(40, "forty"),
-            P(50, "fifty"),
-            P(60, "sixty"),
-            P(70, "seventy"),
-            P(80, "eighty"),
-        };
-        const C c(std::begin(a), std::end(a));
-        std::pair<I, I> r = c.equal_range(30);
-        assert(std::distance(r.first, r.second) == 1);
-        assert(r.first->first == 30);
-        assert(r.first->second == "thirty");
-        r = c.equal_range(5);
-        assert(std::distance(r.first, r.second) == 0);
-    }
+  {
+    typedef std::unordered_map<int,
+                               std::string,
+                               std::hash<int>,
+                               std::equal_to<int>,
+                               min_allocator<std::pair<const int, std::string>>>
+        C;
+    typedef C::const_iterator I;
+    typedef std::pair<int, std::string> P;
+    P a[] = {
+        P(10, "ten"),
+        P(20, "twenty"),
+        P(30, "thirty"),
+        P(40, "forty"),
+        P(50, "fifty"),
+        P(60, "sixty"),
+        P(70, "seventy"),
+        P(80, "eighty"),
+    };
+    const C c(std::begin(a), std::end(a));
+    std::pair<I, I> r = c.equal_range(30);
+    assert(std::distance(r.first, r.second) == 1);
+    assert(r.first->first == 30);
+    assert(r.first->second == "thirty");
+    r = c.equal_range(5);
+    assert(std::distance(r.first, r.second) == 0);
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/unord/unord.map/equal_range_non_const.pass.cpp b/libcxx/test/std/containers/unord/unord.map/equal_range_non_const.pass.cpp
index f0e268edfc716..fc188f748edf9 100644
--- a/libcxx/test/std/containers/unord/unord.map/equal_range_non_const.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.map/equal_range_non_const.pass.cpp
@@ -22,56 +22,57 @@
 #include "test_macros.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef std::unordered_map<int, std::string> C;
-        typedef C::iterator I;
-        typedef std::pair<int, std::string> P;
-        P a[] =
-        {
-            P(10, "ten"),
-            P(20, "twenty"),
-            P(30, "thirty"),
-            P(40, "forty"),
-            P(50, "fifty"),
-            P(60, "sixty"),
-            P(70, "seventy"),
-            P(80, "eighty"),
-        };
-        C c(std::begin(a), std::end(a));
-        std::pair<I, I> r = c.equal_range(30);
-        assert(std::distance(r.first, r.second) == 1);
-        assert(r.first->first == 30);
-        assert(r.first->second == "thirty");
-        r = c.equal_range(5);
-        assert(std::distance(r.first, r.second) == 0);
-    }
+int main(int, char**) {
+  {
+    typedef std::unordered_map<int, std::string> C;
+    typedef C::iterator I;
+    typedef std::pair<int, std::string> P;
+    P a[] = {
+        P(10, "ten"),
+        P(20, "twenty"),
+        P(30, "thirty"),
+        P(40, "forty"),
+        P(50, "fifty"),
+        P(60, "sixty"),
+        P(70, "seventy"),
+        P(80, "eighty"),
+    };
+    C c(std::begin(a), std::end(a));
+    std::pair<I, I> r = c.equal_range(30);
+    assert(std::distance(r.first, r.second) == 1);
+    assert(r.first->first == 30);
+    assert(r.first->second == "thirty");
+    r = c.equal_range(5);
+    assert(std::distance(r.first, r.second) == 0);
+  }
 #if TEST_STD_VER >= 11
-    {
-        typedef std::unordered_map<int, std::string, std::hash<int>, std::equal_to<int>,
-                            min_allocator<std::pair<const int, std::string>>> C;
-        typedef C::iterator I;
-        typedef std::pair<int, std::string> P;
-        P a[] =
-        {
-            P(10, "ten"),
-            P(20, "twenty"),
-            P(30, "thirty"),
-            P(40, "forty"),
-            P(50, "fifty"),
-            P(60, "sixty"),
-            P(70, "seventy"),
-            P(80, "eighty"),
-        };
-        C c(std::begin(a), std::end(a));
-        std::pair<I, I> r = c.equal_range(30);
-        assert(std::distance(r.first, r.second) == 1);
-        assert(r.first->first == 30);
-        assert(r.first->second == "thirty");
-        r = c.equal_range(5);
-        assert(std::distance(r.first, r.second) == 0);
-    }
+  {
+    typedef std::unordered_map<int,
+                               std::string,
+                               std::hash<int>,
+                               std::equal_to<int>,
+                               min_allocator<std::pair<const int, std::string>>>
+        C;
+    typedef C::iterator I;
+    typedef std::pair<int, std::string> P;
+    P a[] = {
+        P(10, "ten"),
+        P(20, "twenty"),
+        P(30, "thirty"),
+        P(40, "forty"),
+        P(50, "fifty"),
+        P(60, "sixty"),
+        P(70, "seventy"),
+        P(80, "eighty"),
+    };
+    C c(std::begin(a), std::end(a));
+    std::pair<I, I> r = c.equal_range(30);
+    assert(std::distance(r.first, r.second) == 1);
+    assert(r.first->first == 30);
+    assert(r.first->second == "thirty");
+    r = c.equal_range(5);
+    assert(std::distance(r.first, r.second) == 0);
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/unord/unord.map/erase_if.pass.cpp b/libcxx/test/std/containers/unord/unord.map/erase_if.pass.cpp
index 329ead0811944..f12ca7f9f268c 100644
--- a/libcxx/test/std/containers/unord/unord.map/erase_if.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.map/erase_if.pass.cpp
@@ -22,12 +22,11 @@
 
 using Init = std::initializer_list<int>;
 template <typename M>
-M make (Init vals)
-{
-    M ret;
-    for (int v : vals)
-        ret[static_cast<typename M::key_type>(v)] = static_cast<typename M::mapped_type>(v + 10);
-    return ret;
+M make(Init vals) {
+  M ret;
+  for (int v : vals)
+    ret[static_cast<typename M::key_type>(v)] = static_cast<typename M::mapped_type>(v + 10);
+  return ret;
 }
 
 template <typename M, typename Pred>
@@ -40,41 +39,39 @@ void test0(Init vals, Pred p, Init expected, std::size_t expected_erased_count)
 }
 
 template <typename S>
-void test()
-{
-    auto is1 = [](auto v) { return v.first == 1;};
-    auto is2 = [](auto v) { return v.first == 2;};
-    auto is3 = [](auto v) { return v.first == 3;};
-    auto is4 = [](auto v) { return v.first == 4;};
-    auto True  = [](auto) { return true; };
-    auto False = [](auto) { return false; };
+void test() {
+  auto is1   = [](auto v) { return v.first == 1; };
+  auto is2   = [](auto v) { return v.first == 2; };
+  auto is3   = [](auto v) { return v.first == 3; };
+  auto is4   = [](auto v) { return v.first == 4; };
+  auto True  = [](auto) { return true; };
+  auto False = [](auto) { return false; };
 
-    test0<S>({}, is1, {}, 0);
+  test0<S>({}, is1, {}, 0);
 
-    test0<S>({1}, is1, {}, 1);
-    test0<S>({1}, is2, {1}, 0);
+  test0<S>({1}, is1, {}, 1);
+  test0<S>({1}, is2, {1}, 0);
 
-    test0<S>({1, 2}, is1, {2}, 1);
-    test0<S>({1, 2}, is2, {1}, 1);
-    test0<S>({1, 2}, is3, {1, 2}, 0);
+  test0<S>({1, 2}, is1, {2}, 1);
+  test0<S>({1, 2}, is2, {1}, 1);
+  test0<S>({1, 2}, is3, {1, 2}, 0);
 
-    test0<S>({1, 2, 3}, is1, {2, 3}, 1);
-    test0<S>({1, 2, 3}, is2, {1, 3}, 1);
-    test0<S>({1, 2, 3}, is3, {1, 2}, 1);
-    test0<S>({1, 2, 3}, is4, {1, 2, 3}, 0);
+  test0<S>({1, 2, 3}, is1, {2, 3}, 1);
+  test0<S>({1, 2, 3}, is2, {1, 3}, 1);
+  test0<S>({1, 2, 3}, is3, {1, 2}, 1);
+  test0<S>({1, 2, 3}, is4, {1, 2, 3}, 0);
 
-    test0<S>({1, 2, 3}, True, {}, 3);
-    test0<S>({1, 2, 3}, False, {1, 2, 3}, 0);
+  test0<S>({1, 2, 3}, True, {}, 3);
+  test0<S>({1, 2, 3}, False, {1, 2, 3}, 0);
 }
 
-int main(int, char**)
-{
-    test<std::unordered_map<int, int>>();
-    test<std::unordered_map<int, int, std::hash<int>, std::equal_to<int>, min_allocator<std::pair<const int, int>>>> ();
-    test<std::unordered_map<int, int, std::hash<int>, std::equal_to<int>, test_allocator<std::pair<const int, int>>>> ();
+int main(int, char**) {
+  test<std::unordered_map<int, int>>();
+  test<std::unordered_map<int, int, std::hash<int>, std::equal_to<int>, min_allocator<std::pair<const int, int>>>>();
+  test<std::unordered_map<int, int, std::hash<int>, std::equal_to<int>, test_allocator<std::pair<const int, int>>>>();
 
-    test<std::unordered_map<long, short>>();
-    test<std::unordered_map<short, double>>();
+  test<std::unordered_map<long, short>>();
+  test<std::unordered_map<short, double>>();
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/unord/unord.map/find.transparent.pass.cpp b/libcxx/test/std/containers/unord/unord.map/find.transparent.pass.cpp
index 623b86f6ab530..e8d590dd392f6 100644
--- a/libcxx/test/std/containers/unord/unord.map/find.transparent.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.map/find.transparent.pass.cpp
@@ -21,44 +21,43 @@
 
 #include "test_transparent_unordered.h"
 
-int main(int, char**)
-{
-    using key_type = StoredType<int>;
-
-    {
-      // Make sure conversions don't happen for transparent non-final hasher and key_equal
-      using M = unord_map_type<std::unordered_map, transparent_hash, std::equal_to<>>;
-      test_transparent_find<M>({{1, 2}, {2, 3}});
-      test_transparent_find<const M>({{1, 2}, {2, 3}});
-    }
-
-    {
-      // Make sure conversions don't happen for transparent final hasher and key_equal
-      using M = unord_map_type<std::unordered_map, transparent_hash_final, transparent_equal_final>;
-      test_transparent_find<M>({{1, 2}, {2, 3}});
-      test_transparent_find<const M>({{1, 2}, {2, 3}});
-    }
-
-    {
-      // Make sure conversions do happen for non-transparent hasher
-      using M = unord_map_type<std::unordered_map, non_transparent_hash, std::equal_to<>>;
-      test_non_transparent_find<M>({{1, 2}, {2, 3}});
-      test_non_transparent_find<const M>({{1, 2}, {2, 3}});
-    }
-
-    {
-      // Make sure conversions do happen for non-transparent key_equal
-      using M = unord_map_type<std::unordered_map, transparent_hash, std::equal_to<key_type>>;
-      test_non_transparent_find<M>({{1, 2}, {2, 3}});
-      test_non_transparent_find<const M>({{1, 2}, {2, 3}});
-    }
-
-    {
-      // Make sure conversions do happen for both non-transparent hasher and key_equal
-      using M = unord_map_type<std::unordered_map, non_transparent_hash, std::equal_to<key_type>>;
-      test_non_transparent_find<M>({{1, 2}, {2, 3}});
-      test_non_transparent_find<const M>({{1, 2}, {2, 3}});
-    }
-
-    return 0;
+int main(int, char**) {
+  using key_type = StoredType<int>;
+
+  {
+    // Make sure conversions don't happen for transparent non-final hasher and key_equal
+    using M = unord_map_type<std::unordered_map, transparent_hash, std::equal_to<>>;
+    test_transparent_find<M>({{1, 2}, {2, 3}});
+    test_transparent_find<const M>({{1, 2}, {2, 3}});
+  }
+
+  {
+    // Make sure conversions don't happen for transparent final hasher and key_equal
+    using M = unord_map_type<std::unordered_map, transparent_hash_final, transparent_equal_final>;
+    test_transparent_find<M>({{1, 2}, {2, 3}});
+    test_transparent_find<const M>({{1, 2}, {2, 3}});
+  }
+
+  {
+    // Make sure conversions do happen for non-transparent hasher
+    using M = unord_map_type<std::unordered_map, non_transparent_hash, std::equal_to<>>;
+    test_non_transparent_find<M>({{1, 2}, {2, 3}});
+    test_non_transparent_find<const M>({{1, 2}, {2, 3}});
+  }
+
+  {
+    // Make sure conversions do happen for non-transparent key_equal
+    using M = unord_map_type<std::unordered_map, transparent_hash, std::equal_to<key_type>>;
+    test_non_transparent_find<M>({{1, 2}, {2, 3}});
+    test_non_transparent_find<const M>({{1, 2}, {2, 3}});
+  }
+
+  {
+    // Make sure conversions do happen for both non-transparent hasher and key_equal
+    using M = unord_map_type<std::unordered_map, non_transparent_hash, std::equal_to<key_type>>;
+    test_non_transparent_find<M>({{1, 2}, {2, 3}});
+    test_non_transparent_find<const M>({{1, 2}, {2, 3}});
+  }
+
+  return 0;
 }
diff --git a/libcxx/test/std/containers/unord/unord.map/find_const.pass.cpp b/libcxx/test/std/containers/unord/unord.map/find_const.pass.cpp
index fa8417b72becd..5fed665cc75c7 100644
--- a/libcxx/test/std/containers/unord/unord.map/find_const.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.map/find_const.pass.cpp
@@ -22,52 +22,53 @@
 #include "test_macros.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef std::unordered_map<int, std::string> C;
-        typedef std::pair<int, std::string> P;
-        P a[] =
-        {
-            P(10, "ten"),
-            P(20, "twenty"),
-            P(30, "thirty"),
-            P(40, "forty"),
-            P(50, "fifty"),
-            P(60, "sixty"),
-            P(70, "seventy"),
-            P(80, "eighty"),
-        };
-        const C c(std::begin(a), std::end(a));
-        C::const_iterator i = c.find(30);
-        assert(i->first == 30);
-        assert(i->second == "thirty");
-        i = c.find(5);
-        assert(i == c.cend());
-    }
+int main(int, char**) {
+  {
+    typedef std::unordered_map<int, std::string> C;
+    typedef std::pair<int, std::string> P;
+    P a[] = {
+        P(10, "ten"),
+        P(20, "twenty"),
+        P(30, "thirty"),
+        P(40, "forty"),
+        P(50, "fifty"),
+        P(60, "sixty"),
+        P(70, "seventy"),
+        P(80, "eighty"),
+    };
+    const C c(std::begin(a), std::end(a));
+    C::const_iterator i = c.find(30);
+    assert(i->first == 30);
+    assert(i->second == "thirty");
+    i = c.find(5);
+    assert(i == c.cend());
+  }
 #if TEST_STD_VER >= 11
-    {
-        typedef std::unordered_map<int, std::string, std::hash<int>, std::equal_to<int>,
-                            min_allocator<std::pair<const int, std::string>>> C;
-        typedef std::pair<int, std::string> P;
-        P a[] =
-        {
-            P(10, "ten"),
-            P(20, "twenty"),
-            P(30, "thirty"),
-            P(40, "forty"),
-            P(50, "fifty"),
-            P(60, "sixty"),
-            P(70, "seventy"),
-            P(80, "eighty"),
-        };
-        const C c(std::begin(a), std::end(a));
-        C::const_iterator i = c.find(30);
-        assert(i->first == 30);
-        assert(i->second == "thirty");
-        i = c.find(5);
-        assert(i == c.cend());
-    }
+  {
+    typedef std::unordered_map<int,
+                               std::string,
+                               std::hash<int>,
+                               std::equal_to<int>,
+                               min_allocator<std::pair<const int, std::string>>>
+        C;
+    typedef std::pair<int, std::string> P;
+    P a[] = {
+        P(10, "ten"),
+        P(20, "twenty"),
+        P(30, "thirty"),
+        P(40, "forty"),
+        P(50, "fifty"),
+        P(60, "sixty"),
+        P(70, "seventy"),
+        P(80, "eighty"),
+    };
+    const C c(std::begin(a), std::end(a));
+    C::const_iterator i = c.find(30);
+    assert(i->first == 30);
+    assert(i->second == "thirty");
+    i = c.find(5);
+    assert(i == c.cend());
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/unord/unord.map/find_non_const.pass.cpp b/libcxx/test/std/containers/unord/unord.map/find_non_const.pass.cpp
index 00c5605c2b491..28b3a71f7b0db 100644
--- a/libcxx/test/std/containers/unord/unord.map/find_non_const.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.map/find_non_const.pass.cpp
@@ -22,52 +22,53 @@
 #include "test_macros.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef std::unordered_map<int, std::string> C;
-        typedef std::pair<int, std::string> P;
-        P a[] =
-        {
-            P(10, "ten"),
-            P(20, "twenty"),
-            P(30, "thirty"),
-            P(40, "forty"),
-            P(50, "fifty"),
-            P(60, "sixty"),
-            P(70, "seventy"),
-            P(80, "eighty"),
-        };
-        C c(std::begin(a), std::end(a));
-        C::iterator i = c.find(30);
-        assert(i->first == 30);
-        assert(i->second == "thirty");
-        i = c.find(5);
-        assert(i == c.end());
-    }
+int main(int, char**) {
+  {
+    typedef std::unordered_map<int, std::string> C;
+    typedef std::pair<int, std::string> P;
+    P a[] = {
+        P(10, "ten"),
+        P(20, "twenty"),
+        P(30, "thirty"),
+        P(40, "forty"),
+        P(50, "fifty"),
+        P(60, "sixty"),
+        P(70, "seventy"),
+        P(80, "eighty"),
+    };
+    C c(std::begin(a), std::end(a));
+    C::iterator i = c.find(30);
+    assert(i->first == 30);
+    assert(i->second == "thirty");
+    i = c.find(5);
+    assert(i == c.end());
+  }
 #if TEST_STD_VER >= 11
-    {
-        typedef std::unordered_map<int, std::string, std::hash<int>, std::equal_to<int>,
-                            min_allocator<std::pair<const int, std::string>>> C;
-        typedef std::pair<int, std::string> P;
-        P a[] =
-        {
-            P(10, "ten"),
-            P(20, "twenty"),
-            P(30, "thirty"),
-            P(40, "forty"),
-            P(50, "fifty"),
-            P(60, "sixty"),
-            P(70, "seventy"),
-            P(80, "eighty"),
-        };
-        C c(std::begin(a), std::end(a));
-        C::iterator i = c.find(30);
-        assert(i->first == 30);
-        assert(i->second == "thirty");
-        i = c.find(5);
-        assert(i == c.end());
-    }
+  {
+    typedef std::unordered_map<int,
+                               std::string,
+                               std::hash<int>,
+                               std::equal_to<int>,
+                               min_allocator<std::pair<const int, std::string>>>
+        C;
+    typedef std::pair<int, std::string> P;
+    P a[] = {
+        P(10, "ten"),
+        P(20, "twenty"),
+        P(30, "thirty"),
+        P(40, "forty"),
+        P(50, "fifty"),
+        P(60, "sixty"),
+        P(70, "seventy"),
+        P(80, "eighty"),
+    };
+    C c(std::begin(a), std::end(a));
+    C::iterator i = c.find(30);
+    assert(i->first == 30);
+    assert(i->second == "thirty");
+    i = c.find(5);
+    assert(i == c.end());
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/unord/unord.map/get_allocator.pass.cpp b/libcxx/test/std/containers/unord/unord.map/get_allocator.pass.cpp
index 64836abfff033..04d67cfaa9995 100644
--- a/libcxx/test/std/containers/unord/unord.map/get_allocator.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.map/get_allocator.pass.cpp
@@ -20,19 +20,18 @@
 #include "test_macros.h"
 
 int main(int, char**) {
-    typedef std::pair<const int, std::string> ValueType;
-    {
-        std::allocator<ValueType> alloc;
-        const std::unordered_map<int, std::string> m(alloc);
-        assert(m.get_allocator() == alloc);
-    }
-    {
-        other_allocator<ValueType> alloc(1);
-        const std::unordered_map<int, std::string, std::hash<int>,
-                                 std::equal_to<int>,
-                                 other_allocator<ValueType> > m(alloc);
-        assert(m.get_allocator() == alloc);
-    }
+  typedef std::pair<const int, std::string> ValueType;
+  {
+    std::allocator<ValueType> alloc;
+    const std::unordered_map<int, std::string> m(alloc);
+    assert(m.get_allocator() == alloc);
+  }
+  {
+    other_allocator<ValueType> alloc(1);
+    const std::unordered_map<int, std::string, std::hash<int>, std::equal_to<int>, other_allocator<ValueType> > m(
+        alloc);
+    assert(m.get_allocator() == alloc);
+  }
 
-    return 0;
+  return 0;
 }
diff --git a/libcxx/test/std/containers/unord/unord.map/incomplete_type.pass.cpp b/libcxx/test/std/containers/unord/unord.map/incomplete_type.pass.cpp
index cf2cc679f704c..4b4d14d3b04f4 100644
--- a/libcxx/test/std/containers/unord/unord.map/incomplete_type.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.map/incomplete_type.pass.cpp
@@ -19,22 +19,22 @@
 template <class Tp>
 struct MyHash {
   MyHash() {}
-  std::size_t operator()(Tp const&) const {return 42;}
+  std::size_t operator()(Tp const&) const { return 42; }
 };
 
 struct A {
-    typedef std::unordered_map<A, A, MyHash<A> > Map;
-    Map m;
-    Map::iterator it;
-    Map::const_iterator cit;
-    Map::local_iterator lit;
-    Map::const_local_iterator clit;
+  typedef std::unordered_map<A, A, MyHash<A> > Map;
+  Map m;
+  Map::iterator it;
+  Map::const_iterator cit;
+  Map::local_iterator lit;
+  Map::const_local_iterator clit;
 };
 
 inline bool operator==(A const& L, A const& R) { return &L == &R; }
 
 int main(int, char**) {
-    A a;
+  A a;
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/unord/unord.map/iterator.operators.addressof.compile.pass.cpp b/libcxx/test/std/containers/unord/unord.map/iterator.operators.addressof.compile.pass.cpp
index 856b78293a107..73ac37a98d1b9 100644
--- a/libcxx/test/std/containers/unord/unord.map/iterator.operators.addressof.compile.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.map/iterator.operators.addressof.compile.pass.cpp
@@ -33,14 +33,14 @@ void test() {
 
 void test() {
   {
-    using I = std::unordered_map<operator_hijacker, operator_hijacker>::iterator;
+    using I  = std::unordered_map<operator_hijacker, operator_hijacker>::iterator;
     using CI = std::unordered_map<operator_hijacker, operator_hijacker>::const_iterator;
     test<I, I>();
     test<CI, I>();
     test<CI, CI>();
   }
   {
-    using IL = std::unordered_map<operator_hijacker, operator_hijacker>::local_iterator;
+    using IL  = std::unordered_map<operator_hijacker, operator_hijacker>::local_iterator;
     using CIL = std::unordered_map<operator_hijacker, operator_hijacker>::const_local_iterator;
     test<IL, IL>();
     test<CIL, IL>();
diff --git a/libcxx/test/std/containers/unord/unord.map/iterator_concept_conformance.compile.pass.cpp b/libcxx/test/std/containers/unord/unord.map/iterator_concept_conformance.compile.pass.cpp
index ff4b487e4aca4..4a184d4d772ad 100644
--- a/libcxx/test/std/containers/unord/unord.map/iterator_concept_conformance.compile.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.map/iterator_concept_conformance.compile.pass.cpp
@@ -16,11 +16,11 @@
 
 #include "test_macros.h"
 
-using iterator = std::unordered_map<int, int>::iterator;
-using const_iterator = std::unordered_map<int, int>::const_iterator;
-using local_iterator = std::unordered_map<int, int>::local_iterator;
+using iterator             = std::unordered_map<int, int>::iterator;
+using const_iterator       = std::unordered_map<int, int>::const_iterator;
+using local_iterator       = std::unordered_map<int, int>::local_iterator;
 using const_local_iterator = std::unordered_map<int, int>::const_local_iterator;
-using value_type = std::pair<const int, int>;
+using value_type           = std::pair<const int, int>;
 
 static_assert(std::forward_iterator<iterator>);
 LIBCPP_STATIC_ASSERT(!std::bidirectional_iterator<iterator>);
diff --git a/libcxx/test/std/containers/unord/unord.map/iterators.pass.cpp b/libcxx/test/std/containers/unord/unord.map/iterators.pass.cpp
index 0b4e02e339f62..5296273d0eeb7 100644
--- a/libcxx/test/std/containers/unord/unord.map/iterators.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.map/iterators.pass.cpp
@@ -27,104 +27,107 @@
 #include "test_macros.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef std::unordered_map<int, std::string> C;
-        typedef std::pair<int, std::string> P;
-        P a[] =
-        {
-            P(1, "one"),
-            P(2, "two"),
-            P(3, "three"),
-            P(4, "four"),
-            P(1, "four"),
-            P(2, "four"),
-        };
-        C c(a, a + sizeof(a)/sizeof(a[0]));
-        assert(c.bucket_count() >= 5);
-        assert(c.size() == 4);
-        assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-        assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
-        C::iterator i;
-    }
-    {
-        typedef std::unordered_map<int, std::string> C;
-        typedef std::pair<int, std::string> P;
-        P a[] =
-        {
-            P(1, "one"),
-            P(2, "two"),
-            P(3, "three"),
-            P(4, "four"),
-            P(1, "four"),
-            P(2, "four"),
-        };
-        const C c(a, a + sizeof(a)/sizeof(a[0]));
-        assert(c.bucket_count() >= 5);
-        assert(c.size() == 4);
-        assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-        assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
-        C::const_iterator i;
-    }
+int main(int, char**) {
+  {
+    typedef std::unordered_map<int, std::string> C;
+    typedef std::pair<int, std::string> P;
+    P a[] = {
+        P(1, "one"),
+        P(2, "two"),
+        P(3, "three"),
+        P(4, "four"),
+        P(1, "four"),
+        P(2, "four"),
+    };
+    C c(a, a + sizeof(a) / sizeof(a[0]));
+    assert(c.bucket_count() >= 5);
+    assert(c.size() == 4);
+    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
+    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
+    C::iterator i;
+  }
+  {
+    typedef std::unordered_map<int, std::string> C;
+    typedef std::pair<int, std::string> P;
+    P a[] = {
+        P(1, "one"),
+        P(2, "two"),
+        P(3, "three"),
+        P(4, "four"),
+        P(1, "four"),
+        P(2, "four"),
+    };
+    const C c(a, a + sizeof(a) / sizeof(a[0]));
+    assert(c.bucket_count() >= 5);
+    assert(c.size() == 4);
+    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
+    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
+    C::const_iterator i;
+  }
 #if TEST_STD_VER >= 11
-    {
-        typedef std::unordered_map<int, std::string, std::hash<int>, std::equal_to<int>,
-                            min_allocator<std::pair<const int, std::string>>> C;
-        typedef std::pair<int, std::string> P;
-        P a[] =
-        {
-            P(1, "one"),
-            P(2, "two"),
-            P(3, "three"),
-            P(4, "four"),
-            P(1, "four"),
-            P(2, "four"),
-        };
-        C c(a, a + sizeof(a)/sizeof(a[0]));
-        assert(c.bucket_count() >= 5);
-        assert(c.size() == 4);
-        assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-        assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
-        C::iterator i;
-    }
-    {
-        typedef std::unordered_map<int, std::string, std::hash<int>, std::equal_to<int>,
-                            min_allocator<std::pair<const int, std::string>>> C;
-        typedef std::pair<int, std::string> P;
-        P a[] =
-        {
-            P(1, "one"),
-            P(2, "two"),
-            P(3, "three"),
-            P(4, "four"),
-            P(1, "four"),
-            P(2, "four"),
-        };
-        const C c(a, a + sizeof(a)/sizeof(a[0]));
-        assert(c.bucket_count() >= 5);
-        assert(c.size() == 4);
-        assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-        assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
-        C::const_iterator i;
-    }
+  {
+    typedef std::unordered_map<int,
+                               std::string,
+                               std::hash<int>,
+                               std::equal_to<int>,
+                               min_allocator<std::pair<const int, std::string>>>
+        C;
+    typedef std::pair<int, std::string> P;
+    P a[] = {
+        P(1, "one"),
+        P(2, "two"),
+        P(3, "three"),
+        P(4, "four"),
+        P(1, "four"),
+        P(2, "four"),
+    };
+    C c(a, a + sizeof(a) / sizeof(a[0]));
+    assert(c.bucket_count() >= 5);
+    assert(c.size() == 4);
+    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
+    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
+    C::iterator i;
+  }
+  {
+    typedef std::unordered_map<int,
+                               std::string,
+                               std::hash<int>,
+                               std::equal_to<int>,
+                               min_allocator<std::pair<const int, std::string>>>
+        C;
+    typedef std::pair<int, std::string> P;
+    P a[] = {
+        P(1, "one"),
+        P(2, "two"),
+        P(3, "three"),
+        P(4, "four"),
+        P(1, "four"),
+        P(2, "four"),
+    };
+    const C c(a, a + sizeof(a) / sizeof(a[0]));
+    assert(c.bucket_count() >= 5);
+    assert(c.size() == 4);
+    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
+    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
+    C::const_iterator i;
+  }
 #endif
 #if TEST_STD_VER > 11
-    { // N3644 testing
-        typedef std::unordered_map<int,double> C;
-        C::iterator ii1{}, ii2{};
-        C::iterator ii4 = ii1;
-        C::const_iterator cii{};
-        assert ( ii1 == ii2 );
-        assert ( ii1 == ii4 );
+  { // N3644 testing
+    typedef std::unordered_map<int, double> C;
+    C::iterator ii1{}, ii2{};
+    C::iterator ii4 = ii1;
+    C::const_iterator cii{};
+    assert(ii1 == ii2);
+    assert(ii1 == ii4);
 
-        assert (!(ii1 != ii2 ));
+    assert(!(ii1 != ii2));
 
-        assert ( (ii1 == cii ));
-        assert ( (cii == ii1 ));
-        assert (!(ii1 != cii ));
-        assert (!(cii != ii1 ));
-    }
+    assert((ii1 == cii));
+    assert((cii == ii1));
+    assert(!(ii1 != cii));
+    assert(!(cii != ii1));
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/unord/unord.map/load_factor.pass.cpp b/libcxx/test/std/containers/unord/unord.map/load_factor.pass.cpp
index 6bb7f89f3a13e..12a4b777167d2 100644
--- a/libcxx/test/std/containers/unord/unord.map/load_factor.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.map/load_factor.pass.cpp
@@ -24,55 +24,60 @@
 #include "test_macros.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef std::unordered_map<int, std::string> C;
-        typedef std::pair<int, std::string> P;
-        P a[] =
-        {
-            P(10, "ten"),
-            P(20, "twenty"),
-            P(30, "thirty"),
-            P(40, "forty"),
-            P(50, "fifty"),
-            P(60, "sixty"),
-            P(70, "seventy"),
-            P(80, "eighty"),
-        };
-        const C c(std::begin(a), std::end(a));
-        assert(std::fabs(c.load_factor() - (float)c.size()/c.bucket_count()) < FLT_EPSILON);
-    }
-    {
-        typedef std::unordered_map<int, std::string> C;
-        const C c;
-        assert(c.load_factor() == 0);
-    }
+int main(int, char**) {
+  {
+    typedef std::unordered_map<int, std::string> C;
+    typedef std::pair<int, std::string> P;
+    P a[] = {
+        P(10, "ten"),
+        P(20, "twenty"),
+        P(30, "thirty"),
+        P(40, "forty"),
+        P(50, "fifty"),
+        P(60, "sixty"),
+        P(70, "seventy"),
+        P(80, "eighty"),
+    };
+    const C c(std::begin(a), std::end(a));
+    assert(std::fabs(c.load_factor() - (float)c.size() / c.bucket_count()) < FLT_EPSILON);
+  }
+  {
+    typedef std::unordered_map<int, std::string> C;
+    const C c;
+    assert(c.load_factor() == 0);
+  }
 #if TEST_STD_VER >= 11
-    {
-        typedef std::unordered_map<int, std::string, std::hash<int>, std::equal_to<int>,
-                            min_allocator<std::pair<const int, std::string>>> C;
-        typedef std::pair<int, std::string> P;
-        P a[] =
-        {
-            P(10, "ten"),
-            P(20, "twenty"),
-            P(30, "thirty"),
-            P(40, "forty"),
-            P(50, "fifty"),
-            P(60, "sixty"),
-            P(70, "seventy"),
-            P(80, "eighty"),
-        };
-        const C c(std::begin(a), std::end(a));
-        assert(std::fabs(c.load_factor() - (float)c.size()/c.bucket_count()) < FLT_EPSILON);
-    }
-    {
-        typedef std::unordered_map<int, std::string, std::hash<int>, std::equal_to<int>,
-                            min_allocator<std::pair<const int, std::string>>> C;
-        const C c;
-        assert(c.load_factor() == 0);
-    }
+  {
+    typedef std::unordered_map<int,
+                               std::string,
+                               std::hash<int>,
+                               std::equal_to<int>,
+                               min_allocator<std::pair<const int, std::string>>>
+        C;
+    typedef std::pair<int, std::string> P;
+    P a[] = {
+        P(10, "ten"),
+        P(20, "twenty"),
+        P(30, "thirty"),
+        P(40, "forty"),
+        P(50, "fifty"),
+        P(60, "sixty"),
+        P(70, "seventy"),
+        P(80, "eighty"),
+    };
+    const C c(std::begin(a), std::end(a));
+    assert(std::fabs(c.load_factor() - (float)c.size() / c.bucket_count()) < FLT_EPSILON);
+  }
+  {
+    typedef std::unordered_map<int,
+                               std::string,
+                               std::hash<int>,
+                               std::equal_to<int>,
+                               min_allocator<std::pair<const int, std::string>>>
+        C;
+    const C c;
+    assert(c.load_factor() == 0);
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/unord/unord.map/local_iterators.pass.cpp b/libcxx/test/std/containers/unord/unord.map/local_iterators.pass.cpp
index 5da3a0b3d932c..a976e3c5d84a7 100644
--- a/libcxx/test/std/containers/unord/unord.map/local_iterators.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.map/local_iterators.pass.cpp
@@ -26,397 +26,404 @@
 #include "test_macros.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef std::unordered_map<int, std::string> C;
-        typedef std::pair<int, std::string> P;
-        typedef C::local_iterator I;
-        P a[] =
-        {
-            P(1, "one"),
-            P(2, "two"),
-            P(3, "three"),
-            P(4, "four"),
-            P(1, "four"),
-            P(2, "four"),
-        };
-        C c(a, a + sizeof(a)/sizeof(a[0]));
-        assert(c.bucket_count() >= 5);
-        C::size_type b = c.bucket(0);
-        I i = c.begin(b);
-        I j = c.end(b);
-        assert(std::distance(i, j) == 0);
-
-        b = c.bucket(1);
-        i = c.begin(b);
-        j = c.end(b);
-        assert(std::distance(i, j) == 1);
-        assert(i->first == 1);
-        assert(i->second == "one");
-
-        b = c.bucket(2);
-        i = c.begin(b);
-        j = c.end(b);
-        assert(std::distance(i, j) == 1);
-        assert(i->first == 2);
-        assert(i->second == "two");
-
-        b = c.bucket(3);
-        i = c.begin(b);
-        j = c.end(b);
-        assert(std::distance(i, j) == 1);
-        assert(i->first == 3);
-        assert(i->second == "three");
-
-        b = c.bucket(4);
-        i = c.begin(b);
-        j = c.end(b);
-        assert(std::distance(i, j) == 1);
-        assert(i->first == 4);
-        assert(i->second == "four");
-    }
-    {
-        typedef std::unordered_map<int, std::string> C;
-        typedef std::pair<int, std::string> P;
-        typedef C::const_local_iterator I;
-        P a[] =
-        {
-            P(1, "one"),
-            P(2, "two"),
-            P(3, "three"),
-            P(4, "four"),
-            P(1, "four"),
-            P(2, "four"),
-        };
-        const C c(a, a + sizeof(a)/sizeof(a[0]));
-        assert(c.bucket_count() >= 5);
-        C::size_type b = c.bucket(0);
-        I i = c.begin(b);
-        I j = c.end(b);
-        assert(std::distance(i, j) == 0);
-
-        b = c.bucket(1);
-        i = c.begin(b);
-        j = c.end(b);
-        assert(std::distance(i, j) == 1);
-        assert(i->first == 1);
-        assert(i->second == "one");
-
-        b = c.bucket(2);
-        i = c.begin(b);
-        j = c.end(b);
-        assert(std::distance(i, j) == 1);
-        assert(i->first == 2);
-        assert(i->second == "two");
-
-        b = c.bucket(3);
-        i = c.begin(b);
-        j = c.end(b);
-        assert(std::distance(i, j) == 1);
-        assert(i->first == 3);
-        assert(i->second == "three");
-
-        b = c.bucket(4);
-        i = c.begin(b);
-        j = c.end(b);
-        assert(std::distance(i, j) == 1);
-        assert(i->first == 4);
-        assert(i->second == "four");
-    }
-    {
-        typedef std::unordered_map<int, std::string> C;
-        typedef std::pair<int, std::string> P;
-        typedef C::const_local_iterator I;
-        P a[] =
-        {
-            P(1, "one"),
-            P(2, "two"),
-            P(3, "three"),
-            P(4, "four"),
-            P(1, "four"),
-            P(2, "four"),
-        };
-        C c(a, a + sizeof(a)/sizeof(a[0]));
-        assert(c.bucket_count() >= 5);
-        C::size_type b = c.bucket(0);
-        I i = c.cbegin(b);
-        I j = c.cend(b);
-        assert(std::distance(i, j) == 0);
-
-        b = c.bucket(1);
-        i = c.cbegin(b);
-        j = c.cend(b);
-        assert(std::distance(i, j) == 1);
-        assert(i->first == 1);
-        assert(i->second == "one");
-
-        b = c.bucket(2);
-        i = c.cbegin(b);
-        j = c.cend(b);
-        assert(std::distance(i, j) == 1);
-        assert(i->first == 2);
-        assert(i->second == "two");
-
-        b = c.bucket(3);
-        i = c.cbegin(b);
-        j = c.cend(b);
-        assert(std::distance(i, j) == 1);
-        assert(i->first == 3);
-        assert(i->second == "three");
-
-        b = c.bucket(4);
-        i = c.cbegin(b);
-        j = c.cend(b);
-        assert(std::distance(i, j) == 1);
-        assert(i->first == 4);
-        assert(i->second == "four");
-    }
-    {
-        typedef std::unordered_map<int, std::string> C;
-        typedef std::pair<int, std::string> P;
-        typedef C::const_local_iterator I;
-        P a[] =
-        {
-            P(1, "one"),
-            P(2, "two"),
-            P(3, "three"),
-            P(4, "four"),
-            P(1, "four"),
-            P(2, "four"),
-        };
-        const C c(a, a + sizeof(a)/sizeof(a[0]));
-        assert(c.bucket_count() >= 5);
-        C::size_type b = c.bucket(0);
-        I i = c.cbegin(b);
-        I j = c.cend(b);
-        assert(std::distance(i, j) == 0);
-
-        b = c.bucket(1);
-        i = c.cbegin(b);
-        j = c.cend(b);
-        assert(std::distance(i, j) == 1);
-        assert(i->first == 1);
-        assert(i->second == "one");
-
-        b = c.bucket(2);
-        i = c.cbegin(b);
-        j = c.cend(b);
-        assert(std::distance(i, j) == 1);
-        assert(i->first == 2);
-        assert(i->second == "two");
-
-        b = c.bucket(3);
-        i = c.cbegin(b);
-        j = c.cend(b);
-        assert(std::distance(i, j) == 1);
-        assert(i->first == 3);
-        assert(i->second == "three");
-
-        b = c.bucket(4);
-        i = c.cbegin(b);
-        j = c.cend(b);
-        assert(std::distance(i, j) == 1);
-        assert(i->first == 4);
-        assert(i->second == "four");
-    }
+int main(int, char**) {
+  {
+    typedef std::unordered_map<int, std::string> C;
+    typedef std::pair<int, std::string> P;
+    typedef C::local_iterator I;
+    P a[] = {
+        P(1, "one"),
+        P(2, "two"),
+        P(3, "three"),
+        P(4, "four"),
+        P(1, "four"),
+        P(2, "four"),
+    };
+    C c(a, a + sizeof(a) / sizeof(a[0]));
+    assert(c.bucket_count() >= 5);
+    C::size_type b = c.bucket(0);
+    I i            = c.begin(b);
+    I j            = c.end(b);
+    assert(std::distance(i, j) == 0);
+
+    b = c.bucket(1);
+    i = c.begin(b);
+    j = c.end(b);
+    assert(std::distance(i, j) == 1);
+    assert(i->first == 1);
+    assert(i->second == "one");
+
+    b = c.bucket(2);
+    i = c.begin(b);
+    j = c.end(b);
+    assert(std::distance(i, j) == 1);
+    assert(i->first == 2);
+    assert(i->second == "two");
+
+    b = c.bucket(3);
+    i = c.begin(b);
+    j = c.end(b);
+    assert(std::distance(i, j) == 1);
+    assert(i->first == 3);
+    assert(i->second == "three");
+
+    b = c.bucket(4);
+    i = c.begin(b);
+    j = c.end(b);
+    assert(std::distance(i, j) == 1);
+    assert(i->first == 4);
+    assert(i->second == "four");
+  }
+  {
+    typedef std::unordered_map<int, std::string> C;
+    typedef std::pair<int, std::string> P;
+    typedef C::const_local_iterator I;
+    P a[] = {
+        P(1, "one"),
+        P(2, "two"),
+        P(3, "three"),
+        P(4, "four"),
+        P(1, "four"),
+        P(2, "four"),
+    };
+    const C c(a, a + sizeof(a) / sizeof(a[0]));
+    assert(c.bucket_count() >= 5);
+    C::size_type b = c.bucket(0);
+    I i            = c.begin(b);
+    I j            = c.end(b);
+    assert(std::distance(i, j) == 0);
+
+    b = c.bucket(1);
+    i = c.begin(b);
+    j = c.end(b);
+    assert(std::distance(i, j) == 1);
+    assert(i->first == 1);
+    assert(i->second == "one");
+
+    b = c.bucket(2);
+    i = c.begin(b);
+    j = c.end(b);
+    assert(std::distance(i, j) == 1);
+    assert(i->first == 2);
+    assert(i->second == "two");
+
+    b = c.bucket(3);
+    i = c.begin(b);
+    j = c.end(b);
+    assert(std::distance(i, j) == 1);
+    assert(i->first == 3);
+    assert(i->second == "three");
+
+    b = c.bucket(4);
+    i = c.begin(b);
+    j = c.end(b);
+    assert(std::distance(i, j) == 1);
+    assert(i->first == 4);
+    assert(i->second == "four");
+  }
+  {
+    typedef std::unordered_map<int, std::string> C;
+    typedef std::pair<int, std::string> P;
+    typedef C::const_local_iterator I;
+    P a[] = {
+        P(1, "one"),
+        P(2, "two"),
+        P(3, "three"),
+        P(4, "four"),
+        P(1, "four"),
+        P(2, "four"),
+    };
+    C c(a, a + sizeof(a) / sizeof(a[0]));
+    assert(c.bucket_count() >= 5);
+    C::size_type b = c.bucket(0);
+    I i            = c.cbegin(b);
+    I j            = c.cend(b);
+    assert(std::distance(i, j) == 0);
+
+    b = c.bucket(1);
+    i = c.cbegin(b);
+    j = c.cend(b);
+    assert(std::distance(i, j) == 1);
+    assert(i->first == 1);
+    assert(i->second == "one");
+
+    b = c.bucket(2);
+    i = c.cbegin(b);
+    j = c.cend(b);
+    assert(std::distance(i, j) == 1);
+    assert(i->first == 2);
+    assert(i->second == "two");
+
+    b = c.bucket(3);
+    i = c.cbegin(b);
+    j = c.cend(b);
+    assert(std::distance(i, j) == 1);
+    assert(i->first == 3);
+    assert(i->second == "three");
+
+    b = c.bucket(4);
+    i = c.cbegin(b);
+    j = c.cend(b);
+    assert(std::distance(i, j) == 1);
+    assert(i->first == 4);
+    assert(i->second == "four");
+  }
+  {
+    typedef std::unordered_map<int, std::string> C;
+    typedef std::pair<int, std::string> P;
+    typedef C::const_local_iterator I;
+    P a[] = {
+        P(1, "one"),
+        P(2, "two"),
+        P(3, "three"),
+        P(4, "four"),
+        P(1, "four"),
+        P(2, "four"),
+    };
+    const C c(a, a + sizeof(a) / sizeof(a[0]));
+    assert(c.bucket_count() >= 5);
+    C::size_type b = c.bucket(0);
+    I i            = c.cbegin(b);
+    I j            = c.cend(b);
+    assert(std::distance(i, j) == 0);
+
+    b = c.bucket(1);
+    i = c.cbegin(b);
+    j = c.cend(b);
+    assert(std::distance(i, j) == 1);
+    assert(i->first == 1);
+    assert(i->second == "one");
+
+    b = c.bucket(2);
+    i = c.cbegin(b);
+    j = c.cend(b);
+    assert(std::distance(i, j) == 1);
+    assert(i->first == 2);
+    assert(i->second == "two");
+
+    b = c.bucket(3);
+    i = c.cbegin(b);
+    j = c.cend(b);
+    assert(std::distance(i, j) == 1);
+    assert(i->first == 3);
+    assert(i->second == "three");
+
+    b = c.bucket(4);
+    i = c.cbegin(b);
+    j = c.cend(b);
+    assert(std::distance(i, j) == 1);
+    assert(i->first == 4);
+    assert(i->second == "four");
+  }
 #if TEST_STD_VER >= 11
-    {
-        typedef std::unordered_map<int, std::string, std::hash<int>, std::equal_to<int>,
-                            min_allocator<std::pair<const int, std::string>>> C;
-        typedef std::pair<int, std::string> P;
-        typedef C::local_iterator I;
-        P a[] =
-        {
-            P(1, "one"),
-            P(2, "two"),
-            P(3, "three"),
-            P(4, "four"),
-            P(1, "four"),
-            P(2, "four"),
-        };
-        C c(a, a + sizeof(a)/sizeof(a[0]));
-        assert(c.bucket_count() >= 5);
-        C::size_type b = c.bucket(0);
-        I i = c.begin(b);
-        I j = c.end(b);
-        assert(std::distance(i, j) == 0);
-
-        b = c.bucket(1);
-        i = c.begin(b);
-        j = c.end(b);
-        assert(std::distance(i, j) == 1);
-        assert(i->first == 1);
-        assert(i->second == "one");
-
-        b = c.bucket(2);
-        i = c.begin(b);
-        j = c.end(b);
-        assert(std::distance(i, j) == 1);
-        assert(i->first == 2);
-        assert(i->second == "two");
-
-        b = c.bucket(3);
-        i = c.begin(b);
-        j = c.end(b);
-        assert(std::distance(i, j) == 1);
-        assert(i->first == 3);
-        assert(i->second == "three");
-
-        b = c.bucket(4);
-        i = c.begin(b);
-        j = c.end(b);
-        assert(std::distance(i, j) == 1);
-        assert(i->first == 4);
-        assert(i->second == "four");
-    }
-    {
-        typedef std::unordered_map<int, std::string, std::hash<int>, std::equal_to<int>,
-                            min_allocator<std::pair<const int, std::string>>> C;
-        typedef std::pair<int, std::string> P;
-        typedef C::const_local_iterator I;
-        P a[] =
-        {
-            P(1, "one"),
-            P(2, "two"),
-            P(3, "three"),
-            P(4, "four"),
-            P(1, "four"),
-            P(2, "four"),
-        };
-        const C c(a, a + sizeof(a)/sizeof(a[0]));
-        assert(c.bucket_count() >= 5);
-        C::size_type b = c.bucket(0);
-        I i = c.begin(b);
-        I j = c.end(b);
-        assert(std::distance(i, j) == 0);
-
-        b = c.bucket(1);
-        i = c.begin(b);
-        j = c.end(b);
-        assert(std::distance(i, j) == 1);
-        assert(i->first == 1);
-        assert(i->second == "one");
-
-        b = c.bucket(2);
-        i = c.begin(b);
-        j = c.end(b);
-        assert(std::distance(i, j) == 1);
-        assert(i->first == 2);
-        assert(i->second == "two");
-
-        b = c.bucket(3);
-        i = c.begin(b);
-        j = c.end(b);
-        assert(std::distance(i, j) == 1);
-        assert(i->first == 3);
-        assert(i->second == "three");
-
-        b = c.bucket(4);
-        i = c.begin(b);
-        j = c.end(b);
-        assert(std::distance(i, j) == 1);
-        assert(i->first == 4);
-        assert(i->second == "four");
-    }
-    {
-        typedef std::unordered_map<int, std::string, std::hash<int>, std::equal_to<int>,
-                            min_allocator<std::pair<const int, std::string>>> C;
-        typedef std::pair<int, std::string> P;
-        typedef C::const_local_iterator I;
-        P a[] =
-        {
-            P(1, "one"),
-            P(2, "two"),
-            P(3, "three"),
-            P(4, "four"),
-            P(1, "four"),
-            P(2, "four"),
-        };
-        C c(a, a + sizeof(a)/sizeof(a[0]));
-        assert(c.bucket_count() >= 5);
-        C::size_type b = c.bucket(0);
-        I i = c.cbegin(b);
-        I j = c.cend(b);
-        assert(std::distance(i, j) == 0);
-
-        b = c.bucket(1);
-        i = c.cbegin(b);
-        j = c.cend(b);
-        assert(std::distance(i, j) == 1);
-        assert(i->first == 1);
-        assert(i->second == "one");
-
-        b = c.bucket(2);
-        i = c.cbegin(b);
-        j = c.cend(b);
-        assert(std::distance(i, j) == 1);
-        assert(i->first == 2);
-        assert(i->second == "two");
-
-        b = c.bucket(3);
-        i = c.cbegin(b);
-        j = c.cend(b);
-        assert(std::distance(i, j) == 1);
-        assert(i->first == 3);
-        assert(i->second == "three");
-
-        b = c.bucket(4);
-        i = c.cbegin(b);
-        j = c.cend(b);
-        assert(std::distance(i, j) == 1);
-        assert(i->first == 4);
-        assert(i->second == "four");
-    }
-    {
-        typedef std::unordered_map<int, std::string, std::hash<int>, std::equal_to<int>,
-                            min_allocator<std::pair<const int, std::string>>> C;
-        typedef std::pair<int, std::string> P;
-        typedef C::const_local_iterator I;
-        P a[] =
-        {
-            P(1, "one"),
-            P(2, "two"),
-            P(3, "three"),
-            P(4, "four"),
-            P(1, "four"),
-            P(2, "four"),
-        };
-        const C c(a, a + sizeof(a)/sizeof(a[0]));
-        assert(c.bucket_count() >= 5);
-        C::size_type b = c.bucket(0);
-        I i = c.cbegin(b);
-        I j = c.cend(b);
-        assert(std::distance(i, j) == 0);
-
-        b = c.bucket(1);
-        i = c.cbegin(b);
-        j = c.cend(b);
-        assert(std::distance(i, j) == 1);
-        assert(i->first == 1);
-        assert(i->second == "one");
-
-        b = c.bucket(2);
-        i = c.cbegin(b);
-        j = c.cend(b);
-        assert(std::distance(i, j) == 1);
-        assert(i->first == 2);
-        assert(i->second == "two");
-
-        b = c.bucket(3);
-        i = c.cbegin(b);
-        j = c.cend(b);
-        assert(std::distance(i, j) == 1);
-        assert(i->first == 3);
-        assert(i->second == "three");
-
-        b = c.bucket(4);
-        i = c.cbegin(b);
-        j = c.cend(b);
-        assert(std::distance(i, j) == 1);
-        assert(i->first == 4);
-        assert(i->second == "four");
-    }
+  {
+    typedef std::unordered_map<int,
+                               std::string,
+                               std::hash<int>,
+                               std::equal_to<int>,
+                               min_allocator<std::pair<const int, std::string>>>
+        C;
+    typedef std::pair<int, std::string> P;
+    typedef C::local_iterator I;
+    P a[] = {
+        P(1, "one"),
+        P(2, "two"),
+        P(3, "three"),
+        P(4, "four"),
+        P(1, "four"),
+        P(2, "four"),
+    };
+    C c(a, a + sizeof(a) / sizeof(a[0]));
+    assert(c.bucket_count() >= 5);
+    C::size_type b = c.bucket(0);
+    I i            = c.begin(b);
+    I j            = c.end(b);
+    assert(std::distance(i, j) == 0);
+
+    b = c.bucket(1);
+    i = c.begin(b);
+    j = c.end(b);
+    assert(std::distance(i, j) == 1);
+    assert(i->first == 1);
+    assert(i->second == "one");
+
+    b = c.bucket(2);
+    i = c.begin(b);
+    j = c.end(b);
+    assert(std::distance(i, j) == 1);
+    assert(i->first == 2);
+    assert(i->second == "two");
+
+    b = c.bucket(3);
+    i = c.begin(b);
+    j = c.end(b);
+    assert(std::distance(i, j) == 1);
+    assert(i->first == 3);
+    assert(i->second == "three");
+
+    b = c.bucket(4);
+    i = c.begin(b);
+    j = c.end(b);
+    assert(std::distance(i, j) == 1);
+    assert(i->first == 4);
+    assert(i->second == "four");
+  }
+  {
+    typedef std::unordered_map<int,
+                               std::string,
+                               std::hash<int>,
+                               std::equal_to<int>,
+                               min_allocator<std::pair<const int, std::string>>>
+        C;
+    typedef std::pair<int, std::string> P;
+    typedef C::const_local_iterator I;
+    P a[] = {
+        P(1, "one"),
+        P(2, "two"),
+        P(3, "three"),
+        P(4, "four"),
+        P(1, "four"),
+        P(2, "four"),
+    };
+    const C c(a, a + sizeof(a) / sizeof(a[0]));
+    assert(c.bucket_count() >= 5);
+    C::size_type b = c.bucket(0);
+    I i            = c.begin(b);
+    I j            = c.end(b);
+    assert(std::distance(i, j) == 0);
+
+    b = c.bucket(1);
+    i = c.begin(b);
+    j = c.end(b);
+    assert(std::distance(i, j) == 1);
+    assert(i->first == 1);
+    assert(i->second == "one");
+
+    b = c.bucket(2);
+    i = c.begin(b);
+    j = c.end(b);
+    assert(std::distance(i, j) == 1);
+    assert(i->first == 2);
+    assert(i->second == "two");
+
+    b = c.bucket(3);
+    i = c.begin(b);
+    j = c.end(b);
+    assert(std::distance(i, j) == 1);
+    assert(i->first == 3);
+    assert(i->second == "three");
+
+    b = c.bucket(4);
+    i = c.begin(b);
+    j = c.end(b);
+    assert(std::distance(i, j) == 1);
+    assert(i->first == 4);
+    assert(i->second == "four");
+  }
+  {
+    typedef std::unordered_map<int,
+                               std::string,
+                               std::hash<int>,
+                               std::equal_to<int>,
+                               min_allocator<std::pair<const int, std::string>>>
+        C;
+    typedef std::pair<int, std::string> P;
+    typedef C::const_local_iterator I;
+    P a[] = {
+        P(1, "one"),
+        P(2, "two"),
+        P(3, "three"),
+        P(4, "four"),
+        P(1, "four"),
+        P(2, "four"),
+    };
+    C c(a, a + sizeof(a) / sizeof(a[0]));
+    assert(c.bucket_count() >= 5);
+    C::size_type b = c.bucket(0);
+    I i            = c.cbegin(b);
+    I j            = c.cend(b);
+    assert(std::distance(i, j) == 0);
+
+    b = c.bucket(1);
+    i = c.cbegin(b);
+    j = c.cend(b);
+    assert(std::distance(i, j) == 1);
+    assert(i->first == 1);
+    assert(i->second == "one");
+
+    b = c.bucket(2);
+    i = c.cbegin(b);
+    j = c.cend(b);
+    assert(std::distance(i, j) == 1);
+    assert(i->first == 2);
+    assert(i->second == "two");
+
+    b = c.bucket(3);
+    i = c.cbegin(b);
+    j = c.cend(b);
+    assert(std::distance(i, j) == 1);
+    assert(i->first == 3);
+    assert(i->second == "three");
+
+    b = c.bucket(4);
+    i = c.cbegin(b);
+    j = c.cend(b);
+    assert(std::distance(i, j) == 1);
+    assert(i->first == 4);
+    assert(i->second == "four");
+  }
+  {
+    typedef std::unordered_map<int,
+                               std::string,
+                               std::hash<int>,
+                               std::equal_to<int>,
+                               min_allocator<std::pair<const int, std::string>>>
+        C;
+    typedef std::pair<int, std::string> P;
+    typedef C::const_local_iterator I;
+    P a[] = {
+        P(1, "one"),
+        P(2, "two"),
+        P(3, "three"),
+        P(4, "four"),
+        P(1, "four"),
+        P(2, "four"),
+    };
+    const C c(a, a + sizeof(a) / sizeof(a[0]));
+    assert(c.bucket_count() >= 5);
+    C::size_type b = c.bucket(0);
+    I i            = c.cbegin(b);
+    I j            = c.cend(b);
+    assert(std::distance(i, j) == 0);
+
+    b = c.bucket(1);
+    i = c.cbegin(b);
+    j = c.cend(b);
+    assert(std::distance(i, j) == 1);
+    assert(i->first == 1);
+    assert(i->second == "one");
+
+    b = c.bucket(2);
+    i = c.cbegin(b);
+    j = c.cend(b);
+    assert(std::distance(i, j) == 1);
+    assert(i->first == 2);
+    assert(i->second == "two");
+
+    b = c.bucket(3);
+    i = c.cbegin(b);
+    j = c.cend(b);
+    assert(std::distance(i, j) == 1);
+    assert(i->first == 3);
+    assert(i->second == "three");
+
+    b = c.bucket(4);
+    i = c.cbegin(b);
+    j = c.cend(b);
+    assert(std::distance(i, j) == 1);
+    assert(i->first == 4);
+    assert(i->second == "four");
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/unord/unord.map/max_bucket_count.pass.cpp b/libcxx/test/std/containers/unord/unord.map/max_bucket_count.pass.cpp
index eb8f3e9d2ed5d..b0fec10a3047b 100644
--- a/libcxx/test/std/containers/unord/unord.map/max_bucket_count.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.map/max_bucket_count.pass.cpp
@@ -21,20 +21,23 @@
 #include "test_macros.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef std::unordered_map<int, std::string> C;
-        const C c;
-        assert(c.max_bucket_count() > 0);
-    }
+int main(int, char**) {
+  {
+    typedef std::unordered_map<int, std::string> C;
+    const C c;
+    assert(c.max_bucket_count() > 0);
+  }
 #if TEST_STD_VER >= 11
-    {
-        typedef std::unordered_map<int, std::string, std::hash<int>, std::equal_to<int>,
-                            min_allocator<std::pair<const int, std::string>>> C;
-        const C c;
-        assert(c.max_bucket_count() > 0);
-    }
+  {
+    typedef std::unordered_map<int,
+                               std::string,
+                               std::hash<int>,
+                               std::equal_to<int>,
+                               min_allocator<std::pair<const int, std::string>>>
+        C;
+    const C c;
+    assert(c.max_bucket_count() > 0);
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/unord/unord.map/max_load_factor.pass.cpp b/libcxx/test/std/containers/unord/unord.map/max_load_factor.pass.cpp
index 070ba1e329bf2..53e17478c5eef 100644
--- a/libcxx/test/std/containers/unord/unord.map/max_load_factor.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.map/max_load_factor.pass.cpp
@@ -22,36 +22,43 @@
 #include "test_macros.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef std::unordered_map<int, std::string> C;
-        const C c;
-        assert(c.max_load_factor() == 1);
-    }
-    {
-        typedef std::unordered_map<int, std::string> C;
-        C c;
-        assert(c.max_load_factor() == 1);
-        c.max_load_factor(2.5);
-        assert(c.max_load_factor() == 2.5);
-    }
+int main(int, char**) {
+  {
+    typedef std::unordered_map<int, std::string> C;
+    const C c;
+    assert(c.max_load_factor() == 1);
+  }
+  {
+    typedef std::unordered_map<int, std::string> C;
+    C c;
+    assert(c.max_load_factor() == 1);
+    c.max_load_factor(2.5);
+    assert(c.max_load_factor() == 2.5);
+  }
 #if TEST_STD_VER >= 11
-    {
-        typedef std::unordered_map<int, std::string, std::hash<int>, std::equal_to<int>,
-                            min_allocator<std::pair<const int, std::string>>> C;
-        const C c;
-        assert(c.max_load_factor() == 1);
-    }
-    {
-        typedef std::unordered_map<int, std::string, std::hash<int>, std::equal_to<int>,
-                            min_allocator<std::pair<const int, std::string>>> C;
-        C c;
-        assert(c.max_load_factor() == 1);
-        c.max_load_factor(2.5);
-        assert(c.max_load_factor() == 2.5);
-    }
+  {
+    typedef std::unordered_map<int,
+                               std::string,
+                               std::hash<int>,
+                               std::equal_to<int>,
+                               min_allocator<std::pair<const int, std::string>>>
+        C;
+    const C c;
+    assert(c.max_load_factor() == 1);
+  }
+  {
+    typedef std::unordered_map<int,
+                               std::string,
+                               std::hash<int>,
+                               std::equal_to<int>,
+                               min_allocator<std::pair<const int, std::string>>>
+        C;
+    C c;
+    assert(c.max_load_factor() == 1);
+    c.max_load_factor(2.5);
+    assert(c.max_load_factor() == 2.5);
+  }
 #endif
 
-    return 0;
+  return 0;
 }
diff --git a/libcxx/test/std/containers/unord/unord.map/max_size.pass.cpp b/libcxx/test/std/containers/unord/unord.map/max_size.pass.cpp
index 33c009b67dffd..0c8453e27f7dd 100644
--- a/libcxx/test/std/containers/unord/unord.map/max_size.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.map/max_size.pass.cpp
@@ -20,35 +20,30 @@
 #include "test_allocator.h"
 #include "test_macros.h"
 
-int main(int, char**)
-{
+int main(int, char**) {
   typedef std::pair<const int, int> KV;
   {
     typedef limited_allocator<KV, 10> A;
-    typedef std::unordered_map<int, int, std::hash<int>, std::equal_to<int>, A>
-        C;
+    typedef std::unordered_map<int, int, std::hash<int>, std::equal_to<int>, A> C;
     C c;
     assert(c.max_size() <= 10);
     LIBCPP_ASSERT(c.max_size() == 10);
   }
   {
     typedef limited_allocator<KV, (std::size_t)-1> A;
-    typedef std::unordered_map<int, int, std::hash<int>, std::equal_to<int>, A>
-        C;
-    const C::size_type max_dist =
-        static_cast<C::size_type>(std::numeric_limits<C::difference_type>::max());
+    typedef std::unordered_map<int, int, std::hash<int>, std::equal_to<int>, A> C;
+    const C::size_type max_dist = static_cast<C::size_type>(std::numeric_limits<C::difference_type>::max());
     C c;
     assert(c.max_size() <= max_dist);
     LIBCPP_ASSERT(c.max_size() == max_dist);
-    }
-    {
-      typedef std::unordered_map<char, int> C;
-    const C::size_type max_dist =
-        static_cast<C::size_type>(std::numeric_limits<C::difference_type>::max());
-      C c;
-      assert(c.max_size() <= max_dist);
-      assert(c.max_size() <= alloc_max_size(c.get_allocator()));
-    }
+  }
+  {
+    typedef std::unordered_map<char, int> C;
+    const C::size_type max_dist = static_cast<C::size_type>(std::numeric_limits<C::difference_type>::max());
+    C c;
+    assert(c.max_size() <= max_dist);
+    assert(c.max_size() <= alloc_max_size(c.get_allocator()));
+  }
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/unord/unord.map/range_concept_conformance.compile.pass.cpp b/libcxx/test/std/containers/unord/unord.map/range_concept_conformance.compile.pass.cpp
index 765277ced8724..0cf9732659585 100644
--- a/libcxx/test/std/containers/unord/unord.map/range_concept_conformance.compile.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.map/range_concept_conformance.compile.pass.cpp
@@ -19,7 +19,6 @@
 
 using range = std::unordered_map<int, int>;
 
-
 static_assert(std::same_as<std::ranges::iterator_t<range>, range::iterator>);
 static_assert(std::ranges::common_range<range>);
 static_assert(std::ranges::forward_range<range>);
diff --git a/libcxx/test/std/containers/unord/unord.map/rehash.pass.cpp b/libcxx/test/std/containers/unord/unord.map/rehash.pass.cpp
index 737e40d6c41cc..5a05d37022421 100644
--- a/libcxx/test/std/containers/unord/unord.map/rehash.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.map/rehash.pass.cpp
@@ -22,83 +22,82 @@
 #include "min_allocator.h"
 
 template <class C>
-void rehash_postcondition(const C& c, std::size_t n)
-{
-    assert(c.bucket_count() >= c.size() / c.max_load_factor() && c.bucket_count() >= n);
+void rehash_postcondition(const C& c, std::size_t n) {
+  assert(c.bucket_count() >= c.size() / c.max_load_factor() && c.bucket_count() >= n);
 }
 
 template <class C>
-void test(const C& c)
-{
-    assert(c.size() == 4);
-    assert(c.at(1) == "one");
-    assert(c.at(2) == "two");
-    assert(c.at(3) == "three");
-    assert(c.at(4) == "four");
+void test(const C& c) {
+  assert(c.size() == 4);
+  assert(c.at(1) == "one");
+  assert(c.at(2) == "two");
+  assert(c.at(3) == "three");
+  assert(c.at(4) == "four");
 }
 
-int main(int, char**)
-{
-    {
-        typedef std::unordered_map<int, std::string> C;
-        typedef std::pair<int, std::string> P;
-        P a[] =
-        {
-            P(1, "one"),
-            P(2, "two"),
-            P(3, "three"),
-            P(4, "four"),
-            P(1, "four"),
-            P(2, "four"),
-        };
-        C c(a, a + sizeof(a)/sizeof(a[0]));
-        test(c);
-        assert(c.bucket_count() >= 5);
-        c.rehash(3);
-        rehash_postcondition(c, 3);
-        LIBCPP_ASSERT(c.bucket_count() == 5);
-        test(c);
-        c.max_load_factor(2);
-        c.rehash(3);
-        rehash_postcondition(c, 3);
-        LIBCPP_ASSERT(c.bucket_count() == 3);
-        test(c);
-        c.rehash(31);
-        rehash_postcondition(c, 31);
-        LIBCPP_ASSERT(c.bucket_count() == 31);
-        test(c);
-    }
+int main(int, char**) {
+  {
+    typedef std::unordered_map<int, std::string> C;
+    typedef std::pair<int, std::string> P;
+    P a[] = {
+        P(1, "one"),
+        P(2, "two"),
+        P(3, "three"),
+        P(4, "four"),
+        P(1, "four"),
+        P(2, "four"),
+    };
+    C c(a, a + sizeof(a) / sizeof(a[0]));
+    test(c);
+    assert(c.bucket_count() >= 5);
+    c.rehash(3);
+    rehash_postcondition(c, 3);
+    LIBCPP_ASSERT(c.bucket_count() == 5);
+    test(c);
+    c.max_load_factor(2);
+    c.rehash(3);
+    rehash_postcondition(c, 3);
+    LIBCPP_ASSERT(c.bucket_count() == 3);
+    test(c);
+    c.rehash(31);
+    rehash_postcondition(c, 31);
+    LIBCPP_ASSERT(c.bucket_count() == 31);
+    test(c);
+  }
 #if TEST_STD_VER >= 11
-    {
-        typedef std::unordered_map<int, std::string, std::hash<int>, std::equal_to<int>,
-                            min_allocator<std::pair<const int, std::string>>> C;
-        typedef std::pair<int, std::string> P;
-        P a[] =
-        {
-            P(1, "one"),
-            P(2, "two"),
-            P(3, "three"),
-            P(4, "four"),
-            P(1, "four"),
-            P(2, "four"),
-        };
-        C c(a, a + sizeof(a)/sizeof(a[0]));
-        test(c);
-        assert(c.bucket_count() >= 5);
-        c.rehash(3);
-        rehash_postcondition(c, 3);
-        LIBCPP_ASSERT(c.bucket_count() == 5);
-        test(c);
-        c.max_load_factor(2);
-        c.rehash(3);
-        rehash_postcondition(c, 3);
-        LIBCPP_ASSERT(c.bucket_count() == 3);
-        test(c);
-        c.rehash(31);
-        rehash_postcondition(c, 31);
-        LIBCPP_ASSERT(c.bucket_count() == 31);
-        test(c);
-    }
+  {
+    typedef std::unordered_map<int,
+                               std::string,
+                               std::hash<int>,
+                               std::equal_to<int>,
+                               min_allocator<std::pair<const int, std::string>>>
+        C;
+    typedef std::pair<int, std::string> P;
+    P a[] = {
+        P(1, "one"),
+        P(2, "two"),
+        P(3, "three"),
+        P(4, "four"),
+        P(1, "four"),
+        P(2, "four"),
+    };
+    C c(a, a + sizeof(a) / sizeof(a[0]));
+    test(c);
+    assert(c.bucket_count() >= 5);
+    c.rehash(3);
+    rehash_postcondition(c, 3);
+    LIBCPP_ASSERT(c.bucket_count() == 5);
+    test(c);
+    c.max_load_factor(2);
+    c.rehash(3);
+    rehash_postcondition(c, 3);
+    LIBCPP_ASSERT(c.bucket_count() == 3);
+    test(c);
+    c.rehash(31);
+    rehash_postcondition(c, 31);
+    LIBCPP_ASSERT(c.bucket_count() == 31);
+    test(c);
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/unord/unord.map/reserve.pass.cpp b/libcxx/test/std/containers/unord/unord.map/reserve.pass.cpp
index 358da35526306..ed56619d44d2b 100644
--- a/libcxx/test/std/containers/unord/unord.map/reserve.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.map/reserve.pass.cpp
@@ -22,88 +22,86 @@
 #include "min_allocator.h"
 
 template <class C>
-void test(const C& c)
-{
-    assert(c.size() == 4);
-    assert(c.at(1) == "one");
-    assert(c.at(2) == "two");
-    assert(c.at(3) == "three");
-    assert(c.at(4) == "four");
+void test(const C& c) {
+  assert(c.size() == 4);
+  assert(c.at(1) == "one");
+  assert(c.at(2) == "two");
+  assert(c.at(3) == "three");
+  assert(c.at(4) == "four");
 }
 
 void reserve_invariant(std::size_t n) // LWG #2156
 {
-    for (std::size_t i = 0; i < n; ++i)
-    {
-        std::unordered_map<std::size_t, size_t> c;
-        c.reserve(n);
-        std::size_t buckets = c.bucket_count();
-        for (std::size_t j = 0; j < i; ++j)
-        {
-            c[i] = i;
-            assert(buckets == c.bucket_count());
-        }
+  for (std::size_t i = 0; i < n; ++i) {
+    std::unordered_map<std::size_t, size_t> c;
+    c.reserve(n);
+    std::size_t buckets = c.bucket_count();
+    for (std::size_t j = 0; j < i; ++j) {
+      c[i] = i;
+      assert(buckets == c.bucket_count());
     }
+  }
 }
 
-int main(int, char**)
-{
-    {
-        typedef std::unordered_map<int, std::string> C;
-        typedef std::pair<int, std::string> P;
-        P a[] =
-        {
-            P(1, "one"),
-            P(2, "two"),
-            P(3, "three"),
-            P(4, "four"),
-            P(1, "four"),
-            P(2, "four"),
-        };
-        C c(a, a + sizeof(a)/sizeof(a[0]));
-        test(c);
-        assert(c.bucket_count() >= 5);
-        c.reserve(3);
-        LIBCPP_ASSERT(c.bucket_count() == 5);
-        test(c);
-        c.max_load_factor(2);
-        c.reserve(3);
-        assert(c.bucket_count() >= 2);
-        test(c);
-        c.reserve(31);
-        assert(c.bucket_count() >= 16);
-        test(c);
-    }
+int main(int, char**) {
+  {
+    typedef std::unordered_map<int, std::string> C;
+    typedef std::pair<int, std::string> P;
+    P a[] = {
+        P(1, "one"),
+        P(2, "two"),
+        P(3, "three"),
+        P(4, "four"),
+        P(1, "four"),
+        P(2, "four"),
+    };
+    C c(a, a + sizeof(a) / sizeof(a[0]));
+    test(c);
+    assert(c.bucket_count() >= 5);
+    c.reserve(3);
+    LIBCPP_ASSERT(c.bucket_count() == 5);
+    test(c);
+    c.max_load_factor(2);
+    c.reserve(3);
+    assert(c.bucket_count() >= 2);
+    test(c);
+    c.reserve(31);
+    assert(c.bucket_count() >= 16);
+    test(c);
+  }
 #if TEST_STD_VER >= 11
-    {
-        typedef std::unordered_map<int, std::string, std::hash<int>, std::equal_to<int>,
-                            min_allocator<std::pair<const int, std::string>>> C;
-        typedef std::pair<int, std::string> P;
-        P a[] =
-        {
-            P(1, "one"),
-            P(2, "two"),
-            P(3, "three"),
-            P(4, "four"),
-            P(1, "four"),
-            P(2, "four"),
-        };
-        C c(a, a + sizeof(a)/sizeof(a[0]));
-        test(c);
-        assert(c.bucket_count() >= 5);
-        c.reserve(3);
-        LIBCPP_ASSERT(c.bucket_count() == 5);
-        test(c);
-        c.max_load_factor(2);
-        c.reserve(3);
-        assert(c.bucket_count() >= 2);
-        test(c);
-        c.reserve(31);
-        assert(c.bucket_count() >= 16);
-        test(c);
-    }
+  {
+    typedef std::unordered_map<int,
+                               std::string,
+                               std::hash<int>,
+                               std::equal_to<int>,
+                               min_allocator<std::pair<const int, std::string>>>
+        C;
+    typedef std::pair<int, std::string> P;
+    P a[] = {
+        P(1, "one"),
+        P(2, "two"),
+        P(3, "three"),
+        P(4, "four"),
+        P(1, "four"),
+        P(2, "four"),
+    };
+    C c(a, a + sizeof(a) / sizeof(a[0]));
+    test(c);
+    assert(c.bucket_count() >= 5);
+    c.reserve(3);
+    LIBCPP_ASSERT(c.bucket_count() == 5);
+    test(c);
+    c.max_load_factor(2);
+    c.reserve(3);
+    assert(c.bucket_count() >= 2);
+    test(c);
+    c.reserve(31);
+    assert(c.bucket_count() >= 16);
+    test(c);
+  }
 #endif
-    reserve_invariant(20);
+  reserve_invariant(20);
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/unord/unord.map/size.pass.cpp b/libcxx/test/std/containers/unord/unord.map/size.pass.cpp
index d4e7cb97f6491..aa160b2c51c45 100644
--- a/libcxx/test/std/containers/unord/unord.map/size.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.map/size.pass.cpp
@@ -18,9 +18,8 @@
 #include "test_macros.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
+int main(int, char**) {
+  {
     typedef std::unordered_map<int, double> M;
     M m;
     ASSERT_NOEXCEPT(m.size());
@@ -37,10 +36,12 @@ int main(int, char**)
     assert(m.size() == 1);
     m.erase(m.begin());
     assert(m.size() == 0);
-    }
+  }
 #if TEST_STD_VER >= 11
-    {
-    typedef std::unordered_map<int, double, std::hash<int>, std::equal_to<int>, min_allocator<std::pair<const int, double>>> M;
+  {
+    typedef std::
+        unordered_map<int, double, std::hash<int>, std::equal_to<int>, min_allocator<std::pair<const int, double>>>
+            M;
     M m;
     ASSERT_NOEXCEPT(m.size());
     assert(m.size() == 0);
@@ -56,7 +57,7 @@ int main(int, char**)
     assert(m.size() == 1);
     m.erase(m.begin());
     assert(m.size() == 0);
-    }
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/unord/unord.map/swap_member.pass.cpp b/libcxx/test/std/containers/unord/unord.map/swap_member.pass.cpp
index 955aa31a88ae3..d27c9b92938a2 100644
--- a/libcxx/test/std/containers/unord/unord.map/swap_member.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.map/swap_member.pass.cpp
@@ -26,547 +26,534 @@
 #include "test_allocator.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef test_hash<int> Hash;
-        typedef test_equal_to<int> Compare;
-        typedef test_allocator<std::pair<const int, std::string> > Alloc;
-        typedef std::unordered_map<int, std::string, Hash, Compare, Alloc> C;
-        C c1(0, Hash(1), Compare(1), Alloc(1, 1));
-        C c2(0, Hash(2), Compare(2), Alloc(1, 2));
-        c2.max_load_factor(2);
-        c1.swap(c2);
-
-        LIBCPP_ASSERT(c1.bucket_count() == 0);
-        assert(c1.size() == 0);
-        assert(c1.hash_function() == Hash(2));
-        assert(c1.key_eq() == Compare(2));
-        assert(c1.get_allocator().get_id() == 1);
-        assert(static_cast<std::size_t>(std::distance(c1.begin(), c1.end())) == c1.size());
-        assert(static_cast<std::size_t>(std::distance(c1.cbegin(), c1.cend())) == c1.size());
-        assert(c1.max_load_factor() == 2);
-
-        LIBCPP_ASSERT(c2.bucket_count() == 0);
-        assert(c2.size() == 0);
-        assert(c2.hash_function() == Hash(1));
-        assert(c2.key_eq() == Compare(1));
-        assert(c2.get_allocator().get_id() == 2);
-        assert(static_cast<std::size_t>(std::distance(c2.begin(), c2.end())) == c2.size());
-        assert(static_cast<std::size_t>(std::distance(c2.cbegin(), c2.cend())) == c2.size());
-        assert(c2.max_load_factor() == 1);
-    }
-    {
-        typedef test_hash<int> Hash;
-        typedef test_equal_to<int> Compare;
-        typedef test_allocator<std::pair<const int, std::string> > Alloc;
-        typedef std::unordered_map<int, std::string, Hash, Compare, Alloc> C;
-        typedef std::pair<int, std::string> P;
-        P a2[] =
-        {
-            P(10, "ten"),
-            P(20, "twenty"),
-            P(30, "thirty"),
-            P(40, "forty"),
-            P(50, "fifty"),
-            P(60, "sixty"),
-            P(70, "seventy"),
-            P(80, "eighty"),
-        };
-        C c1(0, Hash(1), Compare(1), Alloc(1, 1));
-        C c2(std::begin(a2), std::end(a2), 0, Hash(2), Compare(2), Alloc(1, 2));
-        c2.max_load_factor(2);
-        c1.swap(c2);
-
-        assert(c1.bucket_count() >= 8);
-        assert(c1.size() == 8);
-        assert(c1.at(10) == "ten");
-        assert(c1.at(20) == "twenty");
-        assert(c1.at(30) == "thirty");
-        assert(c1.at(40) == "forty");
-        assert(c1.at(50) == "fifty");
-        assert(c1.at(60) == "sixty");
-        assert(c1.at(70) == "seventy");
-        assert(c1.at(80) == "eighty");
-        assert(c1.hash_function() == Hash(2));
-        assert(c1.key_eq() == Compare(2));
-        assert(c1.get_allocator().get_id() == 1);
-        assert(static_cast<std::size_t>(std::distance(c1.begin(), c1.end())) == c1.size());
-        assert(static_cast<std::size_t>(std::distance(c1.cbegin(), c1.cend())) == c1.size());
-        assert(c1.max_load_factor() == 2);
-
-        LIBCPP_ASSERT(c2.bucket_count() == 0);
-        assert(c2.size() == 0);
-        assert(c2.hash_function() == Hash(1));
-        assert(c2.key_eq() == Compare(1));
-        assert(c2.get_allocator().get_id() == 2);
-        assert(static_cast<std::size_t>(std::distance(c2.begin(), c2.end())) == c2.size());
-        assert(static_cast<std::size_t>(std::distance(c2.cbegin(), c2.cend())) == c2.size());
-        assert(c2.max_load_factor() == 1);
-    }
-    {
-        typedef test_hash<int> Hash;
-        typedef test_equal_to<int> Compare;
-        typedef test_allocator<std::pair<const int, std::string> > Alloc;
-        typedef std::unordered_map<int, std::string, Hash, Compare, Alloc> C;
-        typedef std::pair<int, std::string> P;
-        P a1[] =
-        {
-            P(1, "one"),
-            P(2, "two"),
-            P(3, "three"),
-            P(4, "four"),
-            P(1, "four"),
-            P(2, "four"),
-        };
-        C c1(std::begin(a1), std::end(a1), 0, Hash(1), Compare(1), Alloc(1, 1));
-        C c2(0, Hash(2), Compare(2), Alloc(1, 2));
-        c2.max_load_factor(2);
-        c1.swap(c2);
-
-        LIBCPP_ASSERT(c1.bucket_count() == 0);
-        assert(c1.size() == 0);
-        assert(c1.hash_function() == Hash(2));
-        assert(c1.key_eq() == Compare(2));
-        assert(c1.get_allocator().get_id() == 1);
-        assert(static_cast<std::size_t>(std::distance(c1.begin(), c1.end())) == c1.size());
-        assert(static_cast<std::size_t>(std::distance(c1.cbegin(), c1.cend())) == c1.size());
-        assert(c1.max_load_factor() == 2);
-
-        assert(c2.bucket_count() >= 4);
-        assert(c2.size() == 4);
-        assert(c2.at(1) == "one");
-        assert(c2.at(2) == "two");
-        assert(c2.at(3) == "three");
-        assert(c2.at(4) == "four");
-        assert(c2.hash_function() == Hash(1));
-        assert(c2.key_eq() == Compare(1));
-        assert(c2.get_allocator().get_id() == 2);
-        assert(static_cast<std::size_t>(std::distance(c2.begin(), c2.end())) == c2.size());
-        assert(static_cast<std::size_t>(std::distance(c2.cbegin(), c2.cend())) == c2.size());
-        assert(c2.max_load_factor() == 1);
-    }
-    {
-        typedef test_hash<int> Hash;
-        typedef test_equal_to<int> Compare;
-        typedef test_allocator<std::pair<const int, std::string> > Alloc;
-        typedef std::unordered_map<int, std::string, Hash, Compare, Alloc> C;
-        typedef std::pair<int, std::string> P;
-        P a1[] =
-        {
-            P(1, "one"),
-            P(2, "two"),
-            P(3, "three"),
-            P(4, "four"),
-            P(1, "four"),
-            P(2, "four"),
-        };
-        P a2[] =
-        {
-            P(10, "ten"),
-            P(20, "twenty"),
-            P(30, "thirty"),
-            P(40, "forty"),
-            P(50, "fifty"),
-            P(60, "sixty"),
-            P(70, "seventy"),
-            P(80, "eighty"),
-        };
-        C c1(std::begin(a1), std::end(a1), 0, Hash(1), Compare(1), Alloc(1, 1));
-        C c2(std::begin(a2), std::end(a2), 0, Hash(2), Compare(2), Alloc(1, 2));
-        c2.max_load_factor(2);
-        c1.swap(c2);
-
-        assert(c1.bucket_count() >= 8);
-        assert(c1.size() == 8);
-        assert(c1.at(10) == "ten");
-        assert(c1.at(20) == "twenty");
-        assert(c1.at(30) == "thirty");
-        assert(c1.at(40) == "forty");
-        assert(c1.at(50) == "fifty");
-        assert(c1.at(60) == "sixty");
-        assert(c1.at(70) == "seventy");
-        assert(c1.at(80) == "eighty");
-        assert(c1.hash_function() == Hash(2));
-        assert(c1.key_eq() == Compare(2));
-        assert(c1.get_allocator().get_id() == 1);
-        assert(static_cast<std::size_t>(std::distance(c1.begin(), c1.end())) == c1.size());
-        assert(static_cast<std::size_t>(std::distance(c1.cbegin(), c1.cend())) == c1.size());
-        assert(c1.max_load_factor() == 2);
-
-        assert(c2.bucket_count() >= 4);
-        assert(c2.size() == 4);
-        assert(c2.at(1) == "one");
-        assert(c2.at(2) == "two");
-        assert(c2.at(3) == "three");
-        assert(c2.at(4) == "four");
-        assert(c2.hash_function() == Hash(1));
-        assert(c2.key_eq() == Compare(1));
-        assert(c2.get_allocator().get_id() == 2);
-        assert(static_cast<std::size_t>(std::distance(c2.begin(), c2.end())) == c2.size());
-        assert(static_cast<std::size_t>(std::distance(c2.cbegin(), c2.cend())) == c2.size());
-        assert(c2.max_load_factor() == 1);
-    }
-
-    {
-        typedef test_hash<int> Hash;
-        typedef test_equal_to<int> Compare;
-        typedef other_allocator<std::pair<const int, std::string> > Alloc;
-        typedef std::unordered_map<int, std::string, Hash, Compare, Alloc> C;
-        C c1(0, Hash(1), Compare(1), Alloc(1));
-        C c2(0, Hash(2), Compare(2), Alloc(2));
-        c2.max_load_factor(2);
-        c1.swap(c2);
-
-        LIBCPP_ASSERT(c1.bucket_count() == 0);
-        assert(c1.size() == 0);
-        assert(c1.hash_function() == Hash(2));
-        assert(c1.key_eq() == Compare(2));
-        assert(c1.get_allocator() == Alloc(2));
-        assert(static_cast<std::size_t>(std::distance(c1.begin(), c1.end())) == c1.size());
-        assert(static_cast<std::size_t>(std::distance(c1.cbegin(), c1.cend())) == c1.size());
-        assert(c1.max_load_factor() == 2);
-
-        LIBCPP_ASSERT(c2.bucket_count() == 0);
-        assert(c2.size() == 0);
-        assert(c2.hash_function() == Hash(1));
-        assert(c2.key_eq() == Compare(1));
-        assert(c2.get_allocator() == Alloc(1));
-        assert(static_cast<std::size_t>(std::distance(c2.begin(), c2.end())) == c2.size());
-        assert(static_cast<std::size_t>(std::distance(c2.cbegin(), c2.cend())) == c2.size());
-        assert(c2.max_load_factor() == 1);
-    }
-    {
-        typedef test_hash<int> Hash;
-        typedef test_equal_to<int> Compare;
-        typedef other_allocator<std::pair<const int, std::string> > Alloc;
-        typedef std::unordered_map<int, std::string, Hash, Compare, Alloc> C;
-        typedef std::pair<int, std::string> P;
-        P a2[] =
-        {
-            P(10, "ten"),
-            P(20, "twenty"),
-            P(30, "thirty"),
-            P(40, "forty"),
-            P(50, "fifty"),
-            P(60, "sixty"),
-            P(70, "seventy"),
-            P(80, "eighty"),
-        };
-        C c1(0, Hash(1), Compare(1), Alloc(1));
-        C c2(std::begin(a2), std::end(a2), 0, Hash(2), Compare(2), Alloc(2));
-        c2.max_load_factor(2);
-        c1.swap(c2);
-
-        assert(c1.bucket_count() >= 8);
-        assert(c1.size() == 8);
-        assert(c1.at(10) == "ten");
-        assert(c1.at(20) == "twenty");
-        assert(c1.at(30) == "thirty");
-        assert(c1.at(40) == "forty");
-        assert(c1.at(50) == "fifty");
-        assert(c1.at(60) == "sixty");
-        assert(c1.at(70) == "seventy");
-        assert(c1.at(80) == "eighty");
-        assert(c1.hash_function() == Hash(2));
-        assert(c1.key_eq() == Compare(2));
-        assert(c1.get_allocator() == Alloc(2));
-        assert(static_cast<std::size_t>(std::distance(c1.begin(), c1.end())) == c1.size());
-        assert(static_cast<std::size_t>(std::distance(c1.cbegin(), c1.cend())) == c1.size());
-        assert(c1.max_load_factor() == 2);
-
-        LIBCPP_ASSERT(c2.bucket_count() == 0);
-        assert(c2.size() == 0);
-        assert(c2.hash_function() == Hash(1));
-        assert(c2.key_eq() == Compare(1));
-        assert(c2.get_allocator() == Alloc(1));
-        assert(static_cast<std::size_t>(std::distance(c2.begin(), c2.end())) == c2.size());
-        assert(static_cast<std::size_t>(std::distance(c2.cbegin(), c2.cend())) == c2.size());
-        assert(c2.max_load_factor() == 1);
-    }
-    {
-        typedef test_hash<int> Hash;
-        typedef test_equal_to<int> Compare;
-        typedef other_allocator<std::pair<const int, std::string> > Alloc;
-        typedef std::unordered_map<int, std::string, Hash, Compare, Alloc> C;
-        typedef std::pair<int, std::string> P;
-        P a1[] =
-        {
-            P(1, "one"),
-            P(2, "two"),
-            P(3, "three"),
-            P(4, "four"),
-            P(1, "four"),
-            P(2, "four"),
-        };
-        C c1(std::begin(a1), std::end(a1), 0, Hash(1), Compare(1), Alloc(1));
-        C c2(0, Hash(2), Compare(2), Alloc(2));
-        c2.max_load_factor(2);
-        c1.swap(c2);
-
-        LIBCPP_ASSERT(c1.bucket_count() == 0);
-        assert(c1.size() == 0);
-        assert(c1.hash_function() == Hash(2));
-        assert(c1.key_eq() == Compare(2));
-        assert(c1.get_allocator() == Alloc(2));
-        assert(static_cast<std::size_t>(std::distance(c1.begin(), c1.end())) == c1.size());
-        assert(static_cast<std::size_t>(std::distance(c1.cbegin(), c1.cend())) == c1.size());
-        assert(c1.max_load_factor() == 2);
-
-        assert(c2.bucket_count() >= 4);
-        assert(c2.size() == 4);
-        assert(c2.at(1) == "one");
-        assert(c2.at(2) == "two");
-        assert(c2.at(3) == "three");
-        assert(c2.at(4) == "four");
-        assert(c2.hash_function() == Hash(1));
-        assert(c2.key_eq() == Compare(1));
-        assert(c2.get_allocator() == Alloc(1));
-        assert(static_cast<std::size_t>(std::distance(c2.begin(), c2.end())) == c2.size());
-        assert(static_cast<std::size_t>(std::distance(c2.cbegin(), c2.cend())) == c2.size());
-        assert(c2.max_load_factor() == 1);
-    }
-    {
-        typedef test_hash<int> Hash;
-        typedef test_equal_to<int> Compare;
-        typedef other_allocator<std::pair<const int, std::string> > Alloc;
-        typedef std::unordered_map<int, std::string, Hash, Compare, Alloc> C;
-        typedef std::pair<int, std::string> P;
-        P a1[] =
-        {
-            P(1, "one"),
-            P(2, "two"),
-            P(3, "three"),
-            P(4, "four"),
-            P(1, "four"),
-            P(2, "four"),
-        };
-        P a2[] =
-        {
-            P(10, "ten"),
-            P(20, "twenty"),
-            P(30, "thirty"),
-            P(40, "forty"),
-            P(50, "fifty"),
-            P(60, "sixty"),
-            P(70, "seventy"),
-            P(80, "eighty"),
-        };
-        C c1(std::begin(a1), std::end(a1), 0, Hash(1), Compare(1), Alloc(1));
-        C c2(std::begin(a2), std::end(a2), 0, Hash(2), Compare(2), Alloc(2));
-        c2.max_load_factor(2);
-        c1.swap(c2);
-
-        assert(c1.bucket_count() >= 8);
-        assert(c1.size() == 8);
-        assert(c1.at(10) == "ten");
-        assert(c1.at(20) == "twenty");
-        assert(c1.at(30) == "thirty");
-        assert(c1.at(40) == "forty");
-        assert(c1.at(50) == "fifty");
-        assert(c1.at(60) == "sixty");
-        assert(c1.at(70) == "seventy");
-        assert(c1.at(80) == "eighty");
-        assert(c1.hash_function() == Hash(2));
-        assert(c1.key_eq() == Compare(2));
-        assert(c1.get_allocator() == Alloc(2));
-        assert(static_cast<std::size_t>(std::distance(c1.begin(), c1.end())) == c1.size());
-        assert(static_cast<std::size_t>(std::distance(c1.cbegin(), c1.cend())) == c1.size());
-        assert(c1.max_load_factor() == 2);
-
-        assert(c2.bucket_count() >= 4);
-        assert(c2.size() == 4);
-        assert(c2.at(1) == "one");
-        assert(c2.at(2) == "two");
-        assert(c2.at(3) == "three");
-        assert(c2.at(4) == "four");
-        assert(c2.hash_function() == Hash(1));
-        assert(c2.key_eq() == Compare(1));
-        assert(c2.get_allocator() == Alloc(1));
-        assert(static_cast<std::size_t>(std::distance(c2.begin(), c2.end())) == c2.size());
-        assert(static_cast<std::size_t>(std::distance(c2.cbegin(), c2.cend())) == c2.size());
-        assert(c2.max_load_factor() == 1);
-    }
+int main(int, char**) {
+  {
+    typedef test_hash<int> Hash;
+    typedef test_equal_to<int> Compare;
+    typedef test_allocator<std::pair<const int, std::string> > Alloc;
+    typedef std::unordered_map<int, std::string, Hash, Compare, Alloc> C;
+    C c1(0, Hash(1), Compare(1), Alloc(1, 1));
+    C c2(0, Hash(2), Compare(2), Alloc(1, 2));
+    c2.max_load_factor(2);
+    c1.swap(c2);
+
+    LIBCPP_ASSERT(c1.bucket_count() == 0);
+    assert(c1.size() == 0);
+    assert(c1.hash_function() == Hash(2));
+    assert(c1.key_eq() == Compare(2));
+    assert(c1.get_allocator().get_id() == 1);
+    assert(static_cast<std::size_t>(std::distance(c1.begin(), c1.end())) == c1.size());
+    assert(static_cast<std::size_t>(std::distance(c1.cbegin(), c1.cend())) == c1.size());
+    assert(c1.max_load_factor() == 2);
+
+    LIBCPP_ASSERT(c2.bucket_count() == 0);
+    assert(c2.size() == 0);
+    assert(c2.hash_function() == Hash(1));
+    assert(c2.key_eq() == Compare(1));
+    assert(c2.get_allocator().get_id() == 2);
+    assert(static_cast<std::size_t>(std::distance(c2.begin(), c2.end())) == c2.size());
+    assert(static_cast<std::size_t>(std::distance(c2.cbegin(), c2.cend())) == c2.size());
+    assert(c2.max_load_factor() == 1);
+  }
+  {
+    typedef test_hash<int> Hash;
+    typedef test_equal_to<int> Compare;
+    typedef test_allocator<std::pair<const int, std::string> > Alloc;
+    typedef std::unordered_map<int, std::string, Hash, Compare, Alloc> C;
+    typedef std::pair<int, std::string> P;
+    P a2[] = {
+        P(10, "ten"),
+        P(20, "twenty"),
+        P(30, "thirty"),
+        P(40, "forty"),
+        P(50, "fifty"),
+        P(60, "sixty"),
+        P(70, "seventy"),
+        P(80, "eighty"),
+    };
+    C c1(0, Hash(1), Compare(1), Alloc(1, 1));
+    C c2(std::begin(a2), std::end(a2), 0, Hash(2), Compare(2), Alloc(1, 2));
+    c2.max_load_factor(2);
+    c1.swap(c2);
+
+    assert(c1.bucket_count() >= 8);
+    assert(c1.size() == 8);
+    assert(c1.at(10) == "ten");
+    assert(c1.at(20) == "twenty");
+    assert(c1.at(30) == "thirty");
+    assert(c1.at(40) == "forty");
+    assert(c1.at(50) == "fifty");
+    assert(c1.at(60) == "sixty");
+    assert(c1.at(70) == "seventy");
+    assert(c1.at(80) == "eighty");
+    assert(c1.hash_function() == Hash(2));
+    assert(c1.key_eq() == Compare(2));
+    assert(c1.get_allocator().get_id() == 1);
+    assert(static_cast<std::size_t>(std::distance(c1.begin(), c1.end())) == c1.size());
+    assert(static_cast<std::size_t>(std::distance(c1.cbegin(), c1.cend())) == c1.size());
+    assert(c1.max_load_factor() == 2);
+
+    LIBCPP_ASSERT(c2.bucket_count() == 0);
+    assert(c2.size() == 0);
+    assert(c2.hash_function() == Hash(1));
+    assert(c2.key_eq() == Compare(1));
+    assert(c2.get_allocator().get_id() == 2);
+    assert(static_cast<std::size_t>(std::distance(c2.begin(), c2.end())) == c2.size());
+    assert(static_cast<std::size_t>(std::distance(c2.cbegin(), c2.cend())) == c2.size());
+    assert(c2.max_load_factor() == 1);
+  }
+  {
+    typedef test_hash<int> Hash;
+    typedef test_equal_to<int> Compare;
+    typedef test_allocator<std::pair<const int, std::string> > Alloc;
+    typedef std::unordered_map<int, std::string, Hash, Compare, Alloc> C;
+    typedef std::pair<int, std::string> P;
+    P a1[] = {
+        P(1, "one"),
+        P(2, "two"),
+        P(3, "three"),
+        P(4, "four"),
+        P(1, "four"),
+        P(2, "four"),
+    };
+    C c1(std::begin(a1), std::end(a1), 0, Hash(1), Compare(1), Alloc(1, 1));
+    C c2(0, Hash(2), Compare(2), Alloc(1, 2));
+    c2.max_load_factor(2);
+    c1.swap(c2);
+
+    LIBCPP_ASSERT(c1.bucket_count() == 0);
+    assert(c1.size() == 0);
+    assert(c1.hash_function() == Hash(2));
+    assert(c1.key_eq() == Compare(2));
+    assert(c1.get_allocator().get_id() == 1);
+    assert(static_cast<std::size_t>(std::distance(c1.begin(), c1.end())) == c1.size());
+    assert(static_cast<std::size_t>(std::distance(c1.cbegin(), c1.cend())) == c1.size());
+    assert(c1.max_load_factor() == 2);
+
+    assert(c2.bucket_count() >= 4);
+    assert(c2.size() == 4);
+    assert(c2.at(1) == "one");
+    assert(c2.at(2) == "two");
+    assert(c2.at(3) == "three");
+    assert(c2.at(4) == "four");
+    assert(c2.hash_function() == Hash(1));
+    assert(c2.key_eq() == Compare(1));
+    assert(c2.get_allocator().get_id() == 2);
+    assert(static_cast<std::size_t>(std::distance(c2.begin(), c2.end())) == c2.size());
+    assert(static_cast<std::size_t>(std::distance(c2.cbegin(), c2.cend())) == c2.size());
+    assert(c2.max_load_factor() == 1);
+  }
+  {
+    typedef test_hash<int> Hash;
+    typedef test_equal_to<int> Compare;
+    typedef test_allocator<std::pair<const int, std::string> > Alloc;
+    typedef std::unordered_map<int, std::string, Hash, Compare, Alloc> C;
+    typedef std::pair<int, std::string> P;
+    P a1[] = {
+        P(1, "one"),
+        P(2, "two"),
+        P(3, "three"),
+        P(4, "four"),
+        P(1, "four"),
+        P(2, "four"),
+    };
+    P a2[] = {
+        P(10, "ten"),
+        P(20, "twenty"),
+        P(30, "thirty"),
+        P(40, "forty"),
+        P(50, "fifty"),
+        P(60, "sixty"),
+        P(70, "seventy"),
+        P(80, "eighty"),
+    };
+    C c1(std::begin(a1), std::end(a1), 0, Hash(1), Compare(1), Alloc(1, 1));
+    C c2(std::begin(a2), std::end(a2), 0, Hash(2), Compare(2), Alloc(1, 2));
+    c2.max_load_factor(2);
+    c1.swap(c2);
+
+    assert(c1.bucket_count() >= 8);
+    assert(c1.size() == 8);
+    assert(c1.at(10) == "ten");
+    assert(c1.at(20) == "twenty");
+    assert(c1.at(30) == "thirty");
+    assert(c1.at(40) == "forty");
+    assert(c1.at(50) == "fifty");
+    assert(c1.at(60) == "sixty");
+    assert(c1.at(70) == "seventy");
+    assert(c1.at(80) == "eighty");
+    assert(c1.hash_function() == Hash(2));
+    assert(c1.key_eq() == Compare(2));
+    assert(c1.get_allocator().get_id() == 1);
+    assert(static_cast<std::size_t>(std::distance(c1.begin(), c1.end())) == c1.size());
+    assert(static_cast<std::size_t>(std::distance(c1.cbegin(), c1.cend())) == c1.size());
+    assert(c1.max_load_factor() == 2);
+
+    assert(c2.bucket_count() >= 4);
+    assert(c2.size() == 4);
+    assert(c2.at(1) == "one");
+    assert(c2.at(2) == "two");
+    assert(c2.at(3) == "three");
+    assert(c2.at(4) == "four");
+    assert(c2.hash_function() == Hash(1));
+    assert(c2.key_eq() == Compare(1));
+    assert(c2.get_allocator().get_id() == 2);
+    assert(static_cast<std::size_t>(std::distance(c2.begin(), c2.end())) == c2.size());
+    assert(static_cast<std::size_t>(std::distance(c2.cbegin(), c2.cend())) == c2.size());
+    assert(c2.max_load_factor() == 1);
+  }
+
+  {
+    typedef test_hash<int> Hash;
+    typedef test_equal_to<int> Compare;
+    typedef other_allocator<std::pair<const int, std::string> > Alloc;
+    typedef std::unordered_map<int, std::string, Hash, Compare, Alloc> C;
+    C c1(0, Hash(1), Compare(1), Alloc(1));
+    C c2(0, Hash(2), Compare(2), Alloc(2));
+    c2.max_load_factor(2);
+    c1.swap(c2);
+
+    LIBCPP_ASSERT(c1.bucket_count() == 0);
+    assert(c1.size() == 0);
+    assert(c1.hash_function() == Hash(2));
+    assert(c1.key_eq() == Compare(2));
+    assert(c1.get_allocator() == Alloc(2));
+    assert(static_cast<std::size_t>(std::distance(c1.begin(), c1.end())) == c1.size());
+    assert(static_cast<std::size_t>(std::distance(c1.cbegin(), c1.cend())) == c1.size());
+    assert(c1.max_load_factor() == 2);
+
+    LIBCPP_ASSERT(c2.bucket_count() == 0);
+    assert(c2.size() == 0);
+    assert(c2.hash_function() == Hash(1));
+    assert(c2.key_eq() == Compare(1));
+    assert(c2.get_allocator() == Alloc(1));
+    assert(static_cast<std::size_t>(std::distance(c2.begin(), c2.end())) == c2.size());
+    assert(static_cast<std::size_t>(std::distance(c2.cbegin(), c2.cend())) == c2.size());
+    assert(c2.max_load_factor() == 1);
+  }
+  {
+    typedef test_hash<int> Hash;
+    typedef test_equal_to<int> Compare;
+    typedef other_allocator<std::pair<const int, std::string> > Alloc;
+    typedef std::unordered_map<int, std::string, Hash, Compare, Alloc> C;
+    typedef std::pair<int, std::string> P;
+    P a2[] = {
+        P(10, "ten"),
+        P(20, "twenty"),
+        P(30, "thirty"),
+        P(40, "forty"),
+        P(50, "fifty"),
+        P(60, "sixty"),
+        P(70, "seventy"),
+        P(80, "eighty"),
+    };
+    C c1(0, Hash(1), Compare(1), Alloc(1));
+    C c2(std::begin(a2), std::end(a2), 0, Hash(2), Compare(2), Alloc(2));
+    c2.max_load_factor(2);
+    c1.swap(c2);
+
+    assert(c1.bucket_count() >= 8);
+    assert(c1.size() == 8);
+    assert(c1.at(10) == "ten");
+    assert(c1.at(20) == "twenty");
+    assert(c1.at(30) == "thirty");
+    assert(c1.at(40) == "forty");
+    assert(c1.at(50) == "fifty");
+    assert(c1.at(60) == "sixty");
+    assert(c1.at(70) == "seventy");
+    assert(c1.at(80) == "eighty");
+    assert(c1.hash_function() == Hash(2));
+    assert(c1.key_eq() == Compare(2));
+    assert(c1.get_allocator() == Alloc(2));
+    assert(static_cast<std::size_t>(std::distance(c1.begin(), c1.end())) == c1.size());
+    assert(static_cast<std::size_t>(std::distance(c1.cbegin(), c1.cend())) == c1.size());
+    assert(c1.max_load_factor() == 2);
+
+    LIBCPP_ASSERT(c2.bucket_count() == 0);
+    assert(c2.size() == 0);
+    assert(c2.hash_function() == Hash(1));
+    assert(c2.key_eq() == Compare(1));
+    assert(c2.get_allocator() == Alloc(1));
+    assert(static_cast<std::size_t>(std::distance(c2.begin(), c2.end())) == c2.size());
+    assert(static_cast<std::size_t>(std::distance(c2.cbegin(), c2.cend())) == c2.size());
+    assert(c2.max_load_factor() == 1);
+  }
+  {
+    typedef test_hash<int> Hash;
+    typedef test_equal_to<int> Compare;
+    typedef other_allocator<std::pair<const int, std::string> > Alloc;
+    typedef std::unordered_map<int, std::string, Hash, Compare, Alloc> C;
+    typedef std::pair<int, std::string> P;
+    P a1[] = {
+        P(1, "one"),
+        P(2, "two"),
+        P(3, "three"),
+        P(4, "four"),
+        P(1, "four"),
+        P(2, "four"),
+    };
+    C c1(std::begin(a1), std::end(a1), 0, Hash(1), Compare(1), Alloc(1));
+    C c2(0, Hash(2), Compare(2), Alloc(2));
+    c2.max_load_factor(2);
+    c1.swap(c2);
+
+    LIBCPP_ASSERT(c1.bucket_count() == 0);
+    assert(c1.size() == 0);
+    assert(c1.hash_function() == Hash(2));
+    assert(c1.key_eq() == Compare(2));
+    assert(c1.get_allocator() == Alloc(2));
+    assert(static_cast<std::size_t>(std::distance(c1.begin(), c1.end())) == c1.size());
+    assert(static_cast<std::size_t>(std::distance(c1.cbegin(), c1.cend())) == c1.size());
+    assert(c1.max_load_factor() == 2);
+
+    assert(c2.bucket_count() >= 4);
+    assert(c2.size() == 4);
+    assert(c2.at(1) == "one");
+    assert(c2.at(2) == "two");
+    assert(c2.at(3) == "three");
+    assert(c2.at(4) == "four");
+    assert(c2.hash_function() == Hash(1));
+    assert(c2.key_eq() == Compare(1));
+    assert(c2.get_allocator() == Alloc(1));
+    assert(static_cast<std::size_t>(std::distance(c2.begin(), c2.end())) == c2.size());
+    assert(static_cast<std::size_t>(std::distance(c2.cbegin(), c2.cend())) == c2.size());
+    assert(c2.max_load_factor() == 1);
+  }
+  {
+    typedef test_hash<int> Hash;
+    typedef test_equal_to<int> Compare;
+    typedef other_allocator<std::pair<const int, std::string> > Alloc;
+    typedef std::unordered_map<int, std::string, Hash, Compare, Alloc> C;
+    typedef std::pair<int, std::string> P;
+    P a1[] = {
+        P(1, "one"),
+        P(2, "two"),
+        P(3, "three"),
+        P(4, "four"),
+        P(1, "four"),
+        P(2, "four"),
+    };
+    P a2[] = {
+        P(10, "ten"),
+        P(20, "twenty"),
+        P(30, "thirty"),
+        P(40, "forty"),
+        P(50, "fifty"),
+        P(60, "sixty"),
+        P(70, "seventy"),
+        P(80, "eighty"),
+    };
+    C c1(std::begin(a1), std::end(a1), 0, Hash(1), Compare(1), Alloc(1));
+    C c2(std::begin(a2), std::end(a2), 0, Hash(2), Compare(2), Alloc(2));
+    c2.max_load_factor(2);
+    c1.swap(c2);
+
+    assert(c1.bucket_count() >= 8);
+    assert(c1.size() == 8);
+    assert(c1.at(10) == "ten");
+    assert(c1.at(20) == "twenty");
+    assert(c1.at(30) == "thirty");
+    assert(c1.at(40) == "forty");
+    assert(c1.at(50) == "fifty");
+    assert(c1.at(60) == "sixty");
+    assert(c1.at(70) == "seventy");
+    assert(c1.at(80) == "eighty");
+    assert(c1.hash_function() == Hash(2));
+    assert(c1.key_eq() == Compare(2));
+    assert(c1.get_allocator() == Alloc(2));
+    assert(static_cast<std::size_t>(std::distance(c1.begin(), c1.end())) == c1.size());
+    assert(static_cast<std::size_t>(std::distance(c1.cbegin(), c1.cend())) == c1.size());
+    assert(c1.max_load_factor() == 2);
+
+    assert(c2.bucket_count() >= 4);
+    assert(c2.size() == 4);
+    assert(c2.at(1) == "one");
+    assert(c2.at(2) == "two");
+    assert(c2.at(3) == "three");
+    assert(c2.at(4) == "four");
+    assert(c2.hash_function() == Hash(1));
+    assert(c2.key_eq() == Compare(1));
+    assert(c2.get_allocator() == Alloc(1));
+    assert(static_cast<std::size_t>(std::distance(c2.begin(), c2.end())) == c2.size());
+    assert(static_cast<std::size_t>(std::distance(c2.cbegin(), c2.cend())) == c2.size());
+    assert(c2.max_load_factor() == 1);
+  }
 #if TEST_STD_VER >= 11
-    {
-        typedef test_hash<int> Hash;
-        typedef test_equal_to<int> Compare;
-        typedef min_allocator<std::pair<const int, std::string> > Alloc;
-        typedef std::unordered_map<int, std::string, Hash, Compare, Alloc> C;
-        C c1(0, Hash(1), Compare(1), Alloc());
-        C c2(0, Hash(2), Compare(2), Alloc());
-        c2.max_load_factor(2);
-        c1.swap(c2);
-
-        LIBCPP_ASSERT(c1.bucket_count() == 0);
-        assert(c1.size() == 0);
-        assert(c1.hash_function() == Hash(2));
-        assert(c1.key_eq() == Compare(2));
-        assert(c1.get_allocator() == Alloc());
-        assert(static_cast<std::size_t>(std::distance(c1.begin(), c1.end())) == c1.size());
-        assert(static_cast<std::size_t>(std::distance(c1.cbegin(), c1.cend())) == c1.size());
-        assert(c1.max_load_factor() == 2);
-
-        LIBCPP_ASSERT(c2.bucket_count() == 0);
-        assert(c2.size() == 0);
-        assert(c2.hash_function() == Hash(1));
-        assert(c2.key_eq() == Compare(1));
-        assert(c2.get_allocator() == Alloc());
-        assert(static_cast<std::size_t>(std::distance(c2.begin(), c2.end())) == c2.size());
-        assert(static_cast<std::size_t>(std::distance(c2.cbegin(), c2.cend())) == c2.size());
-        assert(c2.max_load_factor() == 1);
-    }
-    {
-        typedef test_hash<int> Hash;
-        typedef test_equal_to<int> Compare;
-        typedef min_allocator<std::pair<const int, std::string> > Alloc;
-        typedef std::unordered_map<int, std::string, Hash, Compare, Alloc> C;
-        typedef std::pair<int, std::string> P;
-        P a2[] =
-        {
-            P(10, "ten"),
-            P(20, "twenty"),
-            P(30, "thirty"),
-            P(40, "forty"),
-            P(50, "fifty"),
-            P(60, "sixty"),
-            P(70, "seventy"),
-            P(80, "eighty"),
-        };
-        C c1(0, Hash(1), Compare(1), Alloc());
-        C c2(std::begin(a2), std::end(a2), 0, Hash(2), Compare(2), Alloc());
-        c2.max_load_factor(2);
-        c1.swap(c2);
-
-        assert(c1.bucket_count() >= 8);
-        assert(c1.size() == 8);
-        assert(c1.at(10) == "ten");
-        assert(c1.at(20) == "twenty");
-        assert(c1.at(30) == "thirty");
-        assert(c1.at(40) == "forty");
-        assert(c1.at(50) == "fifty");
-        assert(c1.at(60) == "sixty");
-        assert(c1.at(70) == "seventy");
-        assert(c1.at(80) == "eighty");
-        assert(c1.hash_function() == Hash(2));
-        assert(c1.key_eq() == Compare(2));
-        assert(c1.get_allocator() == Alloc());
-        assert(static_cast<std::size_t>(std::distance(c1.begin(), c1.end())) == c1.size());
-        assert(static_cast<std::size_t>(std::distance(c1.cbegin(), c1.cend())) == c1.size());
-        assert(c1.max_load_factor() == 2);
-
-        LIBCPP_ASSERT(c2.bucket_count() == 0);
-        assert(c2.size() == 0);
-        assert(c2.hash_function() == Hash(1));
-        assert(c2.key_eq() == Compare(1));
-        assert(c2.get_allocator() == Alloc());
-        assert(static_cast<std::size_t>(std::distance(c2.begin(), c2.end())) == c2.size());
-        assert(static_cast<std::size_t>(std::distance(c2.cbegin(), c2.cend())) == c2.size());
-        assert(c2.max_load_factor() == 1);
-    }
-    {
-        typedef test_hash<int> Hash;
-        typedef test_equal_to<int> Compare;
-        typedef min_allocator<std::pair<const int, std::string> > Alloc;
-        typedef std::unordered_map<int, std::string, Hash, Compare, Alloc> C;
-        typedef std::pair<int, std::string> P;
-        P a1[] =
-        {
-            P(1, "one"),
-            P(2, "two"),
-            P(3, "three"),
-            P(4, "four"),
-            P(1, "four"),
-            P(2, "four"),
-        };
-        C c1(std::begin(a1), std::end(a1), 0, Hash(1), Compare(1), Alloc());
-        C c2(0, Hash(2), Compare(2), Alloc());
-        c2.max_load_factor(2);
-        c1.swap(c2);
-
-        LIBCPP_ASSERT(c1.bucket_count() == 0);
-        assert(c1.size() == 0);
-        assert(c1.hash_function() == Hash(2));
-        assert(c1.key_eq() == Compare(2));
-        assert(c1.get_allocator() == Alloc());
-        assert(static_cast<std::size_t>(std::distance(c1.begin(), c1.end())) == c1.size());
-        assert(static_cast<std::size_t>(std::distance(c1.cbegin(), c1.cend())) == c1.size());
-        assert(c1.max_load_factor() == 2);
-
-        assert(c2.bucket_count() >= 4);
-        assert(c2.size() == 4);
-        assert(c2.at(1) == "one");
-        assert(c2.at(2) == "two");
-        assert(c2.at(3) == "three");
-        assert(c2.at(4) == "four");
-        assert(c2.hash_function() == Hash(1));
-        assert(c2.key_eq() == Compare(1));
-        assert(c2.get_allocator() == Alloc());
-        assert(static_cast<std::size_t>(std::distance(c2.begin(), c2.end())) == c2.size());
-        assert(static_cast<std::size_t>(std::distance(c2.cbegin(), c2.cend())) == c2.size());
-        assert(c2.max_load_factor() == 1);
-    }
-    {
-        typedef test_hash<int> Hash;
-        typedef test_equal_to<int> Compare;
-        typedef min_allocator<std::pair<const int, std::string> > Alloc;
-        typedef std::unordered_map<int, std::string, Hash, Compare, Alloc> C;
-        typedef std::pair<int, std::string> P;
-        P a1[] =
-        {
-            P(1, "one"),
-            P(2, "two"),
-            P(3, "three"),
-            P(4, "four"),
-            P(1, "four"),
-            P(2, "four"),
-        };
-        P a2[] =
-        {
-            P(10, "ten"),
-            P(20, "twenty"),
-            P(30, "thirty"),
-            P(40, "forty"),
-            P(50, "fifty"),
-            P(60, "sixty"),
-            P(70, "seventy"),
-            P(80, "eighty"),
-        };
-        C c1(std::begin(a1), std::end(a1), 0, Hash(1), Compare(1), Alloc());
-        C c2(std::begin(a2), std::end(a2), 0, Hash(2), Compare(2), Alloc());
-        c2.max_load_factor(2);
-        c1.swap(c2);
-
-        assert(c1.bucket_count() >= 8);
-        assert(c1.size() == 8);
-        assert(c1.at(10) == "ten");
-        assert(c1.at(20) == "twenty");
-        assert(c1.at(30) == "thirty");
-        assert(c1.at(40) == "forty");
-        assert(c1.at(50) == "fifty");
-        assert(c1.at(60) == "sixty");
-        assert(c1.at(70) == "seventy");
-        assert(c1.at(80) == "eighty");
-        assert(c1.hash_function() == Hash(2));
-        assert(c1.key_eq() == Compare(2));
-        assert(c1.get_allocator() == Alloc());
-        assert(static_cast<std::size_t>(std::distance(c1.begin(), c1.end())) == c1.size());
-        assert(static_cast<std::size_t>(std::distance(c1.cbegin(), c1.cend())) == c1.size());
-        assert(c1.max_load_factor() == 2);
-
-        assert(c2.bucket_count() >= 4);
-        assert(c2.size() == 4);
-        assert(c2.at(1) == "one");
-        assert(c2.at(2) == "two");
-        assert(c2.at(3) == "three");
-        assert(c2.at(4) == "four");
-        assert(c2.hash_function() == Hash(1));
-        assert(c2.key_eq() == Compare(1));
-        assert(c2.get_allocator() == Alloc());
-        assert(static_cast<std::size_t>(std::distance(c2.begin(), c2.end())) == c2.size());
-        assert(static_cast<std::size_t>(std::distance(c2.cbegin(), c2.cend())) == c2.size());
-        assert(c2.max_load_factor() == 1);
-    }
+  {
+    typedef test_hash<int> Hash;
+    typedef test_equal_to<int> Compare;
+    typedef min_allocator<std::pair<const int, std::string> > Alloc;
+    typedef std::unordered_map<int, std::string, Hash, Compare, Alloc> C;
+    C c1(0, Hash(1), Compare(1), Alloc());
+    C c2(0, Hash(2), Compare(2), Alloc());
+    c2.max_load_factor(2);
+    c1.swap(c2);
+
+    LIBCPP_ASSERT(c1.bucket_count() == 0);
+    assert(c1.size() == 0);
+    assert(c1.hash_function() == Hash(2));
+    assert(c1.key_eq() == Compare(2));
+    assert(c1.get_allocator() == Alloc());
+    assert(static_cast<std::size_t>(std::distance(c1.begin(), c1.end())) == c1.size());
+    assert(static_cast<std::size_t>(std::distance(c1.cbegin(), c1.cend())) == c1.size());
+    assert(c1.max_load_factor() == 2);
+
+    LIBCPP_ASSERT(c2.bucket_count() == 0);
+    assert(c2.size() == 0);
+    assert(c2.hash_function() == Hash(1));
+    assert(c2.key_eq() == Compare(1));
+    assert(c2.get_allocator() == Alloc());
+    assert(static_cast<std::size_t>(std::distance(c2.begin(), c2.end())) == c2.size());
+    assert(static_cast<std::size_t>(std::distance(c2.cbegin(), c2.cend())) == c2.size());
+    assert(c2.max_load_factor() == 1);
+  }
+  {
+    typedef test_hash<int> Hash;
+    typedef test_equal_to<int> Compare;
+    typedef min_allocator<std::pair<const int, std::string> > Alloc;
+    typedef std::unordered_map<int, std::string, Hash, Compare, Alloc> C;
+    typedef std::pair<int, std::string> P;
+    P a2[] = {
+        P(10, "ten"),
+        P(20, "twenty"),
+        P(30, "thirty"),
+        P(40, "forty"),
+        P(50, "fifty"),
+        P(60, "sixty"),
+        P(70, "seventy"),
+        P(80, "eighty"),
+    };
+    C c1(0, Hash(1), Compare(1), Alloc());
+    C c2(std::begin(a2), std::end(a2), 0, Hash(2), Compare(2), Alloc());
+    c2.max_load_factor(2);
+    c1.swap(c2);
+
+    assert(c1.bucket_count() >= 8);
+    assert(c1.size() == 8);
+    assert(c1.at(10) == "ten");
+    assert(c1.at(20) == "twenty");
+    assert(c1.at(30) == "thirty");
+    assert(c1.at(40) == "forty");
+    assert(c1.at(50) == "fifty");
+    assert(c1.at(60) == "sixty");
+    assert(c1.at(70) == "seventy");
+    assert(c1.at(80) == "eighty");
+    assert(c1.hash_function() == Hash(2));
+    assert(c1.key_eq() == Compare(2));
+    assert(c1.get_allocator() == Alloc());
+    assert(static_cast<std::size_t>(std::distance(c1.begin(), c1.end())) == c1.size());
+    assert(static_cast<std::size_t>(std::distance(c1.cbegin(), c1.cend())) == c1.size());
+    assert(c1.max_load_factor() == 2);
+
+    LIBCPP_ASSERT(c2.bucket_count() == 0);
+    assert(c2.size() == 0);
+    assert(c2.hash_function() == Hash(1));
+    assert(c2.key_eq() == Compare(1));
+    assert(c2.get_allocator() == Alloc());
+    assert(static_cast<std::size_t>(std::distance(c2.begin(), c2.end())) == c2.size());
+    assert(static_cast<std::size_t>(std::distance(c2.cbegin(), c2.cend())) == c2.size());
+    assert(c2.max_load_factor() == 1);
+  }
+  {
+    typedef test_hash<int> Hash;
+    typedef test_equal_to<int> Compare;
+    typedef min_allocator<std::pair<const int, std::string> > Alloc;
+    typedef std::unordered_map<int, std::string, Hash, Compare, Alloc> C;
+    typedef std::pair<int, std::string> P;
+    P a1[] = {
+        P(1, "one"),
+        P(2, "two"),
+        P(3, "three"),
+        P(4, "four"),
+        P(1, "four"),
+        P(2, "four"),
+    };
+    C c1(std::begin(a1), std::end(a1), 0, Hash(1), Compare(1), Alloc());
+    C c2(0, Hash(2), Compare(2), Alloc());
+    c2.max_load_factor(2);
+    c1.swap(c2);
+
+    LIBCPP_ASSERT(c1.bucket_count() == 0);
+    assert(c1.size() == 0);
+    assert(c1.hash_function() == Hash(2));
+    assert(c1.key_eq() == Compare(2));
+    assert(c1.get_allocator() == Alloc());
+    assert(static_cast<std::size_t>(std::distance(c1.begin(), c1.end())) == c1.size());
+    assert(static_cast<std::size_t>(std::distance(c1.cbegin(), c1.cend())) == c1.size());
+    assert(c1.max_load_factor() == 2);
+
+    assert(c2.bucket_count() >= 4);
+    assert(c2.size() == 4);
+    assert(c2.at(1) == "one");
+    assert(c2.at(2) == "two");
+    assert(c2.at(3) == "three");
+    assert(c2.at(4) == "four");
+    assert(c2.hash_function() == Hash(1));
+    assert(c2.key_eq() == Compare(1));
+    assert(c2.get_allocator() == Alloc());
+    assert(static_cast<std::size_t>(std::distance(c2.begin(), c2.end())) == c2.size());
+    assert(static_cast<std::size_t>(std::distance(c2.cbegin(), c2.cend())) == c2.size());
+    assert(c2.max_load_factor() == 1);
+  }
+  {
+    typedef test_hash<int> Hash;
+    typedef test_equal_to<int> Compare;
+    typedef min_allocator<std::pair<const int, std::string> > Alloc;
+    typedef std::unordered_map<int, std::string, Hash, Compare, Alloc> C;
+    typedef std::pair<int, std::string> P;
+    P a1[] = {
+        P(1, "one"),
+        P(2, "two"),
+        P(3, "three"),
+        P(4, "four"),
+        P(1, "four"),
+        P(2, "four"),
+    };
+    P a2[] = {
+        P(10, "ten"),
+        P(20, "twenty"),
+        P(30, "thirty"),
+        P(40, "forty"),
+        P(50, "fifty"),
+        P(60, "sixty"),
+        P(70, "seventy"),
+        P(80, "eighty"),
+    };
+    C c1(std::begin(a1), std::end(a1), 0, Hash(1), Compare(1), Alloc());
+    C c2(std::begin(a2), std::end(a2), 0, Hash(2), Compare(2), Alloc());
+    c2.max_load_factor(2);
+    c1.swap(c2);
+
+    assert(c1.bucket_count() >= 8);
+    assert(c1.size() == 8);
+    assert(c1.at(10) == "ten");
+    assert(c1.at(20) == "twenty");
+    assert(c1.at(30) == "thirty");
+    assert(c1.at(40) == "forty");
+    assert(c1.at(50) == "fifty");
+    assert(c1.at(60) == "sixty");
+    assert(c1.at(70) == "seventy");
+    assert(c1.at(80) == "eighty");
+    assert(c1.hash_function() == Hash(2));
+    assert(c1.key_eq() == Compare(2));
+    assert(c1.get_allocator() == Alloc());
+    assert(static_cast<std::size_t>(std::distance(c1.begin(), c1.end())) == c1.size());
+    assert(static_cast<std::size_t>(std::distance(c1.cbegin(), c1.cend())) == c1.size());
+    assert(c1.max_load_factor() == 2);
+
+    assert(c2.bucket_count() >= 4);
+    assert(c2.size() == 4);
+    assert(c2.at(1) == "one");
+    assert(c2.at(2) == "two");
+    assert(c2.at(3) == "three");
+    assert(c2.at(4) == "four");
+    assert(c2.hash_function() == Hash(1));
+    assert(c2.key_eq() == Compare(1));
+    assert(c2.get_allocator() == Alloc());
+    assert(static_cast<std::size_t>(std::distance(c2.begin(), c2.end())) == c2.size());
+    assert(static_cast<std::size_t>(std::distance(c2.cbegin(), c2.cend())) == c2.size());
+    assert(c2.max_load_factor() == 1);
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/unord/unord.map/types.pass.cpp b/libcxx/test/std/containers/unord/unord.map/types.pass.cpp
index 403a0c1599ada..c883d35f85ee6 100644
--- a/libcxx/test/std/containers/unord/unord.map/types.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.map/types.pass.cpp
@@ -33,41 +33,41 @@
 #include "test_macros.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef std::unordered_map<char, short> C;
-        static_assert((std::is_same<C::key_type, char>::value), "");
-        static_assert((std::is_same<C::mapped_type, short>::value), "");
-        static_assert((std::is_same<C::hasher, std::hash<C::key_type> >::value), "");
-        static_assert((std::is_same<C::key_equal, std::equal_to<C::key_type> >::value), "");
-        static_assert((std::is_same<C::allocator_type, std::allocator<C::value_type> >::value), "");
-        static_assert((std::is_same<C::value_type, std::pair<const C::key_type, C::mapped_type> >::value), "");
-        static_assert((std::is_same<C::reference, C::value_type&>::value), "");
-        static_assert((std::is_same<C::const_reference, const C::value_type&>::value), "");
-        static_assert((std::is_same<C::pointer, C::value_type*>::value), "");
-        static_assert((std::is_same<C::const_pointer, const C::value_type*>::value), "");
-        static_assert((std::is_same<C::size_type, std::size_t>::value), "");
-        static_assert((std::is_same<C::difference_type, std::ptrdiff_t>::value), "");
-    }
+int main(int, char**) {
+  {
+    typedef std::unordered_map<char, short> C;
+    static_assert((std::is_same<C::key_type, char>::value), "");
+    static_assert((std::is_same<C::mapped_type, short>::value), "");
+    static_assert((std::is_same<C::hasher, std::hash<C::key_type> >::value), "");
+    static_assert((std::is_same<C::key_equal, std::equal_to<C::key_type> >::value), "");
+    static_assert((std::is_same<C::allocator_type, std::allocator<C::value_type> >::value), "");
+    static_assert((std::is_same<C::value_type, std::pair<const C::key_type, C::mapped_type> >::value), "");
+    static_assert((std::is_same<C::reference, C::value_type&>::value), "");
+    static_assert((std::is_same<C::const_reference, const C::value_type&>::value), "");
+    static_assert((std::is_same<C::pointer, C::value_type*>::value), "");
+    static_assert((std::is_same<C::const_pointer, const C::value_type*>::value), "");
+    static_assert((std::is_same<C::size_type, std::size_t>::value), "");
+    static_assert((std::is_same<C::difference_type, std::ptrdiff_t>::value), "");
+  }
 #if TEST_STD_VER >= 11
-    {
-        typedef std::unordered_map<char, short, std::hash<char>, std::equal_to<char>,
-                            min_allocator<std::pair<const char, short>>> C;
-        static_assert((std::is_same<C::key_type, char>::value), "");
-        static_assert((std::is_same<C::mapped_type, short>::value), "");
-        static_assert((std::is_same<C::hasher, std::hash<C::key_type> >::value), "");
-        static_assert((std::is_same<C::key_equal, std::equal_to<C::key_type> >::value), "");
-        static_assert((std::is_same<C::allocator_type, min_allocator<C::value_type> >::value), "");
-        static_assert((std::is_same<C::value_type, std::pair<const C::key_type, C::mapped_type> >::value), "");
-        static_assert((std::is_same<C::reference, C::value_type&>::value), "");
-        static_assert((std::is_same<C::const_reference, const C::value_type&>::value), "");
-        static_assert((std::is_same<C::pointer, min_pointer<C::value_type>>::value), "");
-        static_assert((std::is_same<C::const_pointer, min_pointer<const C::value_type>>::value), "");
+  {
+    typedef std::
+        unordered_map<char, short, std::hash<char>, std::equal_to<char>, min_allocator<std::pair<const char, short>>>
+            C;
+    static_assert((std::is_same<C::key_type, char>::value), "");
+    static_assert((std::is_same<C::mapped_type, short>::value), "");
+    static_assert((std::is_same<C::hasher, std::hash<C::key_type> >::value), "");
+    static_assert((std::is_same<C::key_equal, std::equal_to<C::key_type> >::value), "");
+    static_assert((std::is_same<C::allocator_type, min_allocator<C::value_type> >::value), "");
+    static_assert((std::is_same<C::value_type, std::pair<const C::key_type, C::mapped_type> >::value), "");
+    static_assert((std::is_same<C::reference, C::value_type&>::value), "");
+    static_assert((std::is_same<C::const_reference, const C::value_type&>::value), "");
+    static_assert((std::is_same<C::pointer, min_pointer<C::value_type>>::value), "");
+    static_assert((std::is_same<C::const_pointer, min_pointer<const C::value_type>>::value), "");
     //  min_allocator doesn't have a size_type, so one gets synthesized
-        static_assert((std::is_same<C::size_type, std::make_unsigned<C::difference_type>::type>::value), "");
-        static_assert((std::is_same<C::difference_type, std::ptrdiff_t>::value), "");
-    }
+    static_assert((std::is_same<C::size_type, std::make_unsigned<C::difference_type>::type>::value), "");
+    static_assert((std::is_same<C::difference_type, std::ptrdiff_t>::value), "");
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/unord/unord.map/unord.map.cnstr/allocator.pass.cpp b/libcxx/test/std/containers/unord/unord.map/unord.map.cnstr/allocator.pass.cpp
index af97adf739eb7..601471f364dbe 100644
--- a/libcxx/test/std/containers/unord/unord.map/unord.map.cnstr/allocator.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.map/unord.map.cnstr/allocator.pass.cpp
@@ -24,107 +24,105 @@
 #include "test_allocator.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef std::unordered_map<NotConstructible, NotConstructible,
-                                   test_hash<NotConstructible>,
-                                   test_equal_to<NotConstructible>,
-                                   test_allocator<std::pair<const NotConstructible,
-                                                                  NotConstructible> >
-                                   > C;
-        C c(test_allocator<std::pair<const NotConstructible, NotConstructible> >(10));
-        LIBCPP_ASSERT(c.bucket_count() == 0);
-        assert(c.hash_function() == test_hash<NotConstructible>());
-        assert(c.key_eq() == test_equal_to<NotConstructible>());
-        assert(c.get_allocator() ==
-               (test_allocator<std::pair<const NotConstructible, NotConstructible> >(10)));
-        assert(c.size() == 0);
-        assert(c.empty());
-        assert(std::distance(c.begin(), c.end()) == 0);
-        assert(c.load_factor() == 0);
-        assert(c.max_load_factor() == 1);
-    }
+int main(int, char**) {
+  {
+    typedef std::unordered_map<NotConstructible,
+                               NotConstructible,
+                               test_hash<NotConstructible>,
+                               test_equal_to<NotConstructible>,
+                               test_allocator<std::pair<const NotConstructible, NotConstructible> > >
+        C;
+    C c(test_allocator<std::pair<const NotConstructible, NotConstructible> >(10));
+    LIBCPP_ASSERT(c.bucket_count() == 0);
+    assert(c.hash_function() == test_hash<NotConstructible>());
+    assert(c.key_eq() == test_equal_to<NotConstructible>());
+    assert(c.get_allocator() == (test_allocator<std::pair<const NotConstructible, NotConstructible> >(10)));
+    assert(c.size() == 0);
+    assert(c.empty());
+    assert(std::distance(c.begin(), c.end()) == 0);
+    assert(c.load_factor() == 0);
+    assert(c.max_load_factor() == 1);
+  }
 #if TEST_STD_VER >= 11
-    {
-        typedef std::unordered_map<NotConstructible, NotConstructible,
-                                   test_hash<NotConstructible>,
-                                   test_equal_to<NotConstructible>,
-                                   min_allocator<std::pair<const NotConstructible,
-                                                                 NotConstructible> >
-                                   > C;
-        C c(min_allocator<std::pair<const NotConstructible, NotConstructible> >{});
-        LIBCPP_ASSERT(c.bucket_count() == 0);
-        assert(c.hash_function() == test_hash<NotConstructible>());
-        assert(c.key_eq() == test_equal_to<NotConstructible>());
-        assert(c.get_allocator() ==
-               (min_allocator<std::pair<const NotConstructible, NotConstructible> >()));
-        assert(c.size() == 0);
-        assert(c.empty());
-        assert(std::distance(c.begin(), c.end()) == 0);
-        assert(c.load_factor() == 0);
-        assert(c.max_load_factor() == 1);
-    }
-    {
-        typedef explicit_allocator<std::pair<const NotConstructible, NotConstructible>> A;
-        typedef std::unordered_map<NotConstructible, NotConstructible,
-                                   test_hash<NotConstructible>,
-                                   test_equal_to<NotConstructible>,
-                                   A
-                                   > C;
-        C c(A{});
-        LIBCPP_ASSERT(c.bucket_count() == 0);
-        assert(c.hash_function() == test_hash<NotConstructible>());
-        assert(c.key_eq() == test_equal_to<NotConstructible>());
-        assert(c.get_allocator() == A{});
-        assert(c.size() == 0);
-        assert(c.empty());
-        assert(std::distance(c.begin(), c.end()) == 0);
-        assert(c.load_factor() == 0);
-        assert(c.max_load_factor() == 1);
-    }
-#if TEST_STD_VER > 11
-    {
-        typedef NotConstructible T;
-        typedef test_allocator<std::pair<const T, T>> A;
-        typedef test_hash<T> HF;
-        typedef test_equal_to<T> Comp;
-        typedef std::unordered_map<T, T, HF, Comp, A> C;
+  {
+    typedef std::unordered_map<NotConstructible,
+                               NotConstructible,
+                               test_hash<NotConstructible>,
+                               test_equal_to<NotConstructible>,
+                               min_allocator<std::pair<const NotConstructible, NotConstructible> > >
+        C;
+    C c(min_allocator<std::pair<const NotConstructible, NotConstructible> >{});
+    LIBCPP_ASSERT(c.bucket_count() == 0);
+    assert(c.hash_function() == test_hash<NotConstructible>());
+    assert(c.key_eq() == test_equal_to<NotConstructible>());
+    assert(c.get_allocator() == (min_allocator<std::pair<const NotConstructible, NotConstructible> >()));
+    assert(c.size() == 0);
+    assert(c.empty());
+    assert(std::distance(c.begin(), c.end()) == 0);
+    assert(c.load_factor() == 0);
+    assert(c.max_load_factor() == 1);
+  }
+  {
+    typedef explicit_allocator<std::pair<const NotConstructible, NotConstructible>> A;
+    typedef std::unordered_map<NotConstructible,
+                               NotConstructible,
+                               test_hash<NotConstructible>,
+                               test_equal_to<NotConstructible>,
+                               A >
+        C;
+    C c(A{});
+    LIBCPP_ASSERT(c.bucket_count() == 0);
+    assert(c.hash_function() == test_hash<NotConstructible>());
+    assert(c.key_eq() == test_equal_to<NotConstructible>());
+    assert(c.get_allocator() == A{});
+    assert(c.size() == 0);
+    assert(c.empty());
+    assert(std::distance(c.begin(), c.end()) == 0);
+    assert(c.load_factor() == 0);
+    assert(c.max_load_factor() == 1);
+  }
+#  if TEST_STD_VER > 11
+  {
+    typedef NotConstructible T;
+    typedef test_allocator<std::pair<const T, T>> A;
+    typedef test_hash<T> HF;
+    typedef test_equal_to<T> Comp;
+    typedef std::unordered_map<T, T, HF, Comp, A> C;
 
-        A a(10);
-        C c(2, a);
-        LIBCPP_ASSERT(c.bucket_count() == 2);
-        assert(c.hash_function() == HF());
-        assert(c.key_eq() == Comp());
-        assert(c.get_allocator() == a);
-        assert(c.size() == 0);
-        assert(c.empty());
-        assert(std::distance(c.begin(), c.end()) == 0);
-        assert(c.load_factor() == 0);
-        assert(c.max_load_factor() == 1);
-    }
-    {
-        typedef NotConstructible T;
-        typedef test_allocator<std::pair<const T, T>> A;
-        typedef test_hash<T> HF;
-        typedef test_equal_to<T> Comp;
-        typedef std::unordered_map<T, T, HF, Comp, A> C;
+    A a(10);
+    C c(2, a);
+    LIBCPP_ASSERT(c.bucket_count() == 2);
+    assert(c.hash_function() == HF());
+    assert(c.key_eq() == Comp());
+    assert(c.get_allocator() == a);
+    assert(c.size() == 0);
+    assert(c.empty());
+    assert(std::distance(c.begin(), c.end()) == 0);
+    assert(c.load_factor() == 0);
+    assert(c.max_load_factor() == 1);
+  }
+  {
+    typedef NotConstructible T;
+    typedef test_allocator<std::pair<const T, T>> A;
+    typedef test_hash<T> HF;
+    typedef test_equal_to<T> Comp;
+    typedef std::unordered_map<T, T, HF, Comp, A> C;
 
-        A a(10);
-        HF hf(12);
-        C c(2, hf, a);
-        LIBCPP_ASSERT(c.bucket_count() == 2);
-        assert(c.hash_function() == hf);
-        assert(!(c.hash_function() == HF()));
-        assert(c.key_eq() == Comp());
-        assert(c.get_allocator() == a);
-        assert(c.size() == 0);
-        assert(c.empty());
-        assert(std::distance(c.begin(), c.end()) == 0);
-        assert(c.load_factor() == 0);
-        assert(c.max_load_factor() == 1);
-    }
-#endif
+    A a(10);
+    HF hf(12);
+    C c(2, hf, a);
+    LIBCPP_ASSERT(c.bucket_count() == 2);
+    assert(c.hash_function() == hf);
+    assert(!(c.hash_function() == HF()));
+    assert(c.key_eq() == Comp());
+    assert(c.get_allocator() == a);
+    assert(c.size() == 0);
+    assert(c.empty());
+    assert(std::distance(c.begin(), c.end()) == 0);
+    assert(c.load_factor() == 0);
+    assert(c.max_load_factor() == 1);
+  }
+#  endif
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/unord/unord.map/unord.map.cnstr/assign_copy.pass.cpp b/libcxx/test/std/containers/unord/unord.map/unord.map.cnstr/assign_copy.pass.cpp
index d0b290d54ca22..34dec07b03e08 100644
--- a/libcxx/test/std/containers/unord/unord.map/unord.map.cnstr/assign_copy.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.map/unord.map.cnstr/assign_copy.pass.cpp
@@ -28,162 +28,115 @@
 #include "test_allocator.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef test_allocator<std::pair<const int, std::string> > A;
-        typedef std::unordered_map<int, std::string,
-                                   test_hash<int>,
-                                   test_equal_to<int>,
-                                   A
-                                   > C;
-        typedef std::pair<int, std::string> P;
-        P a[] =
-        {
-            P(1, "one"),
-            P(2, "two"),
-            P(3, "three"),
-            P(4, "four"),
-            P(1, "four"),
-            P(2, "four"),
-        };
-        C c0(a, a + sizeof(a)/sizeof(a[0]),
-            7,
-            test_hash<int>(8),
-            test_equal_to<int>(9),
-            A(10)
-           );
-        C c(a, a + 2,
-            7,
-            test_hash<int>(2),
-            test_equal_to<int>(3),
-            A(4)
-           );
-        c = c0;
-        LIBCPP_ASSERT(c.bucket_count() == 7);
-        assert(c.size() == 4);
-        assert(c.at(1) == "one");
-        assert(c.at(2) == "two");
-        assert(c.at(3) == "three");
-        assert(c.at(4) == "four");
-        assert(c.hash_function() == test_hash<int>(8));
-        assert(c.key_eq() == test_equal_to<int>(9));
-        assert(c.get_allocator() == A(4));
-        assert(!c.empty());
-        assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-        assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
-        assert(fabs(c.load_factor() - (float)c.size()/c.bucket_count()) < FLT_EPSILON);
-        assert(c.max_load_factor() == 1);
-    }
-    {
-        typedef std::unordered_map<int, std::string> C;
-        typedef std::pair<const int, std::string> P;
-        const P a[] =
-        {
-            P(1, "one"),
-            P(2, "two"),
-            P(3, "three"),
-            P(4, "four"),
-            P(1, "four"),
-            P(2, "four"),
-        };
-        C c(a, a + sizeof(a)/sizeof(a[0]));
-        C *p = &c;
-        c = *p;
-        assert(c.size() == 4);
-        assert(std::is_permutation(c.begin(), c.end(), a));
-    }
-    {
-        typedef other_allocator<std::pair<const int, std::string> > A;
-        typedef std::unordered_map<int, std::string,
-                                   test_hash<int>,
-                                   test_equal_to<int>,
-                                   A
-                                   > C;
-        typedef std::pair<int, std::string> P;
-        P a[] =
-        {
-            P(1, "one"),
-            P(2, "two"),
-            P(3, "three"),
-            P(4, "four"),
-            P(1, "four"),
-            P(2, "four"),
-        };
-        C c0(a, a + sizeof(a)/sizeof(a[0]),
-            7,
-            test_hash<int>(8),
-            test_equal_to<int>(9),
-            A(10)
-           );
-        C c(a, a + 2,
-            7,
-            test_hash<int>(2),
-            test_equal_to<int>(3),
-            A(4)
-           );
-        c = c0;
-        assert(c.bucket_count() >= 5);
-        assert(c.size() == 4);
-        assert(c.at(1) == "one");
-        assert(c.at(2) == "two");
-        assert(c.at(3) == "three");
-        assert(c.at(4) == "four");
-        assert(c.hash_function() == test_hash<int>(8));
-        assert(c.key_eq() == test_equal_to<int>(9));
-        assert(c.get_allocator() == A(10));
-        assert(!c.empty());
-        assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-        assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
-        assert(fabs(c.load_factor() - (float)c.size()/c.bucket_count()) < FLT_EPSILON);
-        assert(c.max_load_factor() == 1);
-    }
+int main(int, char**) {
+  {
+    typedef test_allocator<std::pair<const int, std::string> > A;
+    typedef std::unordered_map<int, std::string, test_hash<int>, test_equal_to<int>, A > C;
+    typedef std::pair<int, std::string> P;
+    P a[] = {
+        P(1, "one"),
+        P(2, "two"),
+        P(3, "three"),
+        P(4, "four"),
+        P(1, "four"),
+        P(2, "four"),
+    };
+    C c0(a, a + sizeof(a) / sizeof(a[0]), 7, test_hash<int>(8), test_equal_to<int>(9), A(10));
+    C c(a, a + 2, 7, test_hash<int>(2), test_equal_to<int>(3), A(4));
+    c = c0;
+    LIBCPP_ASSERT(c.bucket_count() == 7);
+    assert(c.size() == 4);
+    assert(c.at(1) == "one");
+    assert(c.at(2) == "two");
+    assert(c.at(3) == "three");
+    assert(c.at(4) == "four");
+    assert(c.hash_function() == test_hash<int>(8));
+    assert(c.key_eq() == test_equal_to<int>(9));
+    assert(c.get_allocator() == A(4));
+    assert(!c.empty());
+    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
+    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
+    assert(fabs(c.load_factor() - (float)c.size() / c.bucket_count()) < FLT_EPSILON);
+    assert(c.max_load_factor() == 1);
+  }
+  {
+    typedef std::unordered_map<int, std::string> C;
+    typedef std::pair<const int, std::string> P;
+    const P a[] = {
+        P(1, "one"),
+        P(2, "two"),
+        P(3, "three"),
+        P(4, "four"),
+        P(1, "four"),
+        P(2, "four"),
+    };
+    C c(a, a + sizeof(a) / sizeof(a[0]));
+    C* p = &c;
+    c    = *p;
+    assert(c.size() == 4);
+    assert(std::is_permutation(c.begin(), c.end(), a));
+  }
+  {
+    typedef other_allocator<std::pair<const int, std::string> > A;
+    typedef std::unordered_map<int, std::string, test_hash<int>, test_equal_to<int>, A > C;
+    typedef std::pair<int, std::string> P;
+    P a[] = {
+        P(1, "one"),
+        P(2, "two"),
+        P(3, "three"),
+        P(4, "four"),
+        P(1, "four"),
+        P(2, "four"),
+    };
+    C c0(a, a + sizeof(a) / sizeof(a[0]), 7, test_hash<int>(8), test_equal_to<int>(9), A(10));
+    C c(a, a + 2, 7, test_hash<int>(2), test_equal_to<int>(3), A(4));
+    c = c0;
+    assert(c.bucket_count() >= 5);
+    assert(c.size() == 4);
+    assert(c.at(1) == "one");
+    assert(c.at(2) == "two");
+    assert(c.at(3) == "three");
+    assert(c.at(4) == "four");
+    assert(c.hash_function() == test_hash<int>(8));
+    assert(c.key_eq() == test_equal_to<int>(9));
+    assert(c.get_allocator() == A(10));
+    assert(!c.empty());
+    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
+    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
+    assert(fabs(c.load_factor() - (float)c.size() / c.bucket_count()) < FLT_EPSILON);
+    assert(c.max_load_factor() == 1);
+  }
 #if TEST_STD_VER >= 11
-    {
-        typedef min_allocator<std::pair<const int, std::string> > A;
-        typedef std::unordered_map<int, std::string,
-                                   test_hash<int>,
-                                   test_equal_to<int>,
-                                   A
-                                   > C;
-        typedef std::pair<int, std::string> P;
-        P a[] =
-        {
-            P(1, "one"),
-            P(2, "two"),
-            P(3, "three"),
-            P(4, "four"),
-            P(1, "four"),
-            P(2, "four"),
-        };
-        C c0(a, a + sizeof(a)/sizeof(a[0]),
-            7,
-            test_hash<int>(8),
-            test_equal_to<int>(9),
-            A()
-           );
-        C c(a, a + 2,
-            7,
-            test_hash<int>(2),
-            test_equal_to<int>(3),
-            A()
-           );
-        c = c0;
-        LIBCPP_ASSERT(c.bucket_count() == 7);
-        assert(c.size() == 4);
-        assert(c.at(1) == "one");
-        assert(c.at(2) == "two");
-        assert(c.at(3) == "three");
-        assert(c.at(4) == "four");
-        assert(c.hash_function() == test_hash<int>(8));
-        assert(c.key_eq() == test_equal_to<int>(9));
-        assert(c.get_allocator() == A());
-        assert(!c.empty());
-        assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-        assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
-        assert(fabs(c.load_factor() - (float)c.size()/c.bucket_count()) < FLT_EPSILON);
-        assert(c.max_load_factor() == 1);
-    }
+  {
+    typedef min_allocator<std::pair<const int, std::string> > A;
+    typedef std::unordered_map<int, std::string, test_hash<int>, test_equal_to<int>, A > C;
+    typedef std::pair<int, std::string> P;
+    P a[] = {
+        P(1, "one"),
+        P(2, "two"),
+        P(3, "three"),
+        P(4, "four"),
+        P(1, "four"),
+        P(2, "four"),
+    };
+    C c0(a, a + sizeof(a) / sizeof(a[0]), 7, test_hash<int>(8), test_equal_to<int>(9), A());
+    C c(a, a + 2, 7, test_hash<int>(2), test_equal_to<int>(3), A());
+    c = c0;
+    LIBCPP_ASSERT(c.bucket_count() == 7);
+    assert(c.size() == 4);
+    assert(c.at(1) == "one");
+    assert(c.at(2) == "two");
+    assert(c.at(3) == "three");
+    assert(c.at(4) == "four");
+    assert(c.hash_function() == test_hash<int>(8));
+    assert(c.key_eq() == test_equal_to<int>(9));
+    assert(c.get_allocator() == A());
+    assert(!c.empty());
+    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
+    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
+    assert(fabs(c.load_factor() - (float)c.size() / c.bucket_count()) < FLT_EPSILON);
+    assert(c.max_load_factor() == 1);
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/unord/unord.map/unord.map.cnstr/assign_init.pass.cpp b/libcxx/test/std/containers/unord/unord.map/unord.map.cnstr/assign_init.pass.cpp
index 4aa56c4162f37..ed3aa49fcddbe 100644
--- a/libcxx/test/std/containers/unord/unord.map/unord.map.cnstr/assign_init.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.map/unord.map.cnstr/assign_init.pass.cpp
@@ -28,72 +28,63 @@
 #include "../../../test_hash.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef std::allocator<std::pair<const int, std::string> > A;
-        typedef std::unordered_map<int, std::string,
-                                   test_hash<int>,
-                                   test_equal_to<int>,
-                                   A
-                                   > C;
-        typedef std::pair<int, std::string> P;
-        C c =   {
-                    P(4, "four"),
-                    P(1, "four"),
-                    P(2, "four"),
-                };
-        c =     {
-                    P(1, "one"),
-                    P(2, "two"),
-                    P(3, "three"),
-                    P(4, "four"),
-                    P(1, "four"),
-                    P(2, "four"),
-                };
-        assert(c.bucket_count() >= 5);
-        assert(c.size() == 4);
-        assert(c.at(1) == "one");
-        assert(c.at(2) == "two");
-        assert(c.at(3) == "three");
-        assert(c.at(4) == "four");
-        assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-        assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
-        assert(fabs(c.load_factor() - (float)c.size()/c.bucket_count()) < FLT_EPSILON);
-        assert(c.max_load_factor() == 1);
-    }
-    {
-        typedef min_allocator<std::pair<const int, std::string> > A;
-        typedef std::unordered_map<int, std::string,
-                                   test_hash<int>,
-                                   test_equal_to<int>,
-                                   A
-                                   > C;
-        typedef std::pair<int, std::string> P;
-        C c =   {
-                    P(4, "four"),
-                    P(1, "four"),
-                    P(2, "four"),
-                };
-        c =     {
-                    P(1, "one"),
-                    P(2, "two"),
-                    P(3, "three"),
-                    P(4, "four"),
-                    P(1, "four"),
-                    P(2, "four"),
-                };
-        assert(c.bucket_count() >= 5);
-        assert(c.size() == 4);
-        assert(c.at(1) == "one");
-        assert(c.at(2) == "two");
-        assert(c.at(3) == "three");
-        assert(c.at(4) == "four");
-        assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-        assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
-        assert(fabs(c.load_factor() - (float)c.size()/c.bucket_count()) < FLT_EPSILON);
-        assert(c.max_load_factor() == 1);
-    }
+int main(int, char**) {
+  {
+    typedef std::allocator<std::pair<const int, std::string> > A;
+    typedef std::unordered_map<int, std::string, test_hash<int>, test_equal_to<int>, A > C;
+    typedef std::pair<int, std::string> P;
+    C c = {
+        P(4, "four"),
+        P(1, "four"),
+        P(2, "four"),
+    };
+    c = {
+        P(1, "one"),
+        P(2, "two"),
+        P(3, "three"),
+        P(4, "four"),
+        P(1, "four"),
+        P(2, "four"),
+    };
+    assert(c.bucket_count() >= 5);
+    assert(c.size() == 4);
+    assert(c.at(1) == "one");
+    assert(c.at(2) == "two");
+    assert(c.at(3) == "three");
+    assert(c.at(4) == "four");
+    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
+    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
+    assert(fabs(c.load_factor() - (float)c.size() / c.bucket_count()) < FLT_EPSILON);
+    assert(c.max_load_factor() == 1);
+  }
+  {
+    typedef min_allocator<std::pair<const int, std::string> > A;
+    typedef std::unordered_map<int, std::string, test_hash<int>, test_equal_to<int>, A > C;
+    typedef std::pair<int, std::string> P;
+    C c = {
+        P(4, "four"),
+        P(1, "four"),
+        P(2, "four"),
+    };
+    c = {
+        P(1, "one"),
+        P(2, "two"),
+        P(3, "three"),
+        P(4, "four"),
+        P(1, "four"),
+        P(2, "four"),
+    };
+    assert(c.bucket_count() >= 5);
+    assert(c.size() == 4);
+    assert(c.at(1) == "one");
+    assert(c.at(2) == "two");
+    assert(c.at(3) == "three");
+    assert(c.at(4) == "four");
+    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
+    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
+    assert(fabs(c.load_factor() - (float)c.size() / c.bucket_count()) < FLT_EPSILON);
+    assert(c.max_load_factor() == 1);
+  }
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/unord/unord.map/unord.map.cnstr/assign_move.pass.cpp b/libcxx/test/std/containers/unord/unord.map/unord.map.cnstr/assign_move.pass.cpp
index dc859e8fd067a..39e0fd0feb267 100644
--- a/libcxx/test/std/containers/unord/unord.map/unord.map.cnstr/assign_move.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.map/unord.map.cnstr/assign_move.pass.cpp
@@ -29,197 +29,136 @@
 #include "test_allocator.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef test_allocator<std::pair<const int, std::string> > A;
-        typedef std::unordered_map<int, std::string,
-                                   test_hash<int>,
-                                   test_equal_to<int>,
-                                   A
-                                   > C;
-        typedef std::pair<int, std::string> P;
-        P a[] =
-        {
-            P(1, "one"),
-            P(2, "two"),
-            P(3, "three"),
-            P(4, "four"),
-            P(1, "four"),
-            P(2, "four"),
-        };
-        C c0(a, a + sizeof(a)/sizeof(a[0]),
-            7,
-            test_hash<int>(8),
-            test_equal_to<int>(9),
-            A(10)
-           );
-        C c(a, a + 2,
-            7,
-            test_hash<int>(2),
-            test_equal_to<int>(3),
-            A(4)
-           );
-        c = std::move(c0);
-        LIBCPP_ASSERT(c.bucket_count() == 7);
-        assert(c.size() == 4);
-        assert(c.at(1) == "one");
-        assert(c.at(2) == "two");
-        assert(c.at(3) == "three");
-        assert(c.at(4) == "four");
-        assert(c.hash_function() == test_hash<int>(8));
-        assert(c.key_eq() == test_equal_to<int>(9));
-        assert(c.get_allocator() == A(4));
-        assert(!c.empty());
-        assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-        assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
-        assert(fabs(c.load_factor() - (float)c.size()/c.bucket_count()) < FLT_EPSILON);
-        assert(c.max_load_factor() == 1);
-    }
-    {
-        typedef test_allocator<std::pair<const int, std::string> > A;
-        typedef std::unordered_map<int, std::string,
-                                   test_hash<int>,
-                                   test_equal_to<int>,
-                                   A
-                                   > C;
-        typedef std::pair<int, std::string> P;
-        P a[] =
-        {
-            P(1, "one"),
-            P(2, "two"),
-            P(3, "three"),
-            P(4, "four"),
-            P(1, "four"),
-            P(2, "four"),
-        };
-        C c0(a, a + sizeof(a)/sizeof(a[0]),
-            7,
-            test_hash<int>(8),
-            test_equal_to<int>(9),
-            A(10)
-           );
-        C c(a, a + 2,
-            7,
-            test_hash<int>(2),
-            test_equal_to<int>(3),
-            A(10)
-           );
-        C::iterator it0 = c0.begin();
-        c = std::move(c0);
-        LIBCPP_ASSERT(c.bucket_count() == 7);
-        assert(c.size() == 4);
-        assert(c.at(1) == "one");
-        assert(c.at(2) == "two");
-        assert(c.at(3) == "three");
-        assert(c.at(4) == "four");
-        assert(c.hash_function() == test_hash<int>(8));
-        assert(c.key_eq() == test_equal_to<int>(9));
-        assert(c.get_allocator() == A(10));
-        assert(!c.empty());
-        assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-        assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
-        assert(fabs(c.load_factor() - (float)c.size()/c.bucket_count()) < FLT_EPSILON);
-        assert(c.max_load_factor() == 1);
-        assert(c0.size() == 0);
-        assert(it0 == c.begin()); // Iterators remain valid
-    }
-    {
-        typedef other_allocator<std::pair<const int, std::string> > A;
-        typedef std::unordered_map<int, std::string,
-                                   test_hash<int>,
-                                   test_equal_to<int>,
-                                   A
-                                   > C;
-        typedef std::pair<int, std::string> P;
-        P a[] =
-        {
-            P(1, "one"),
-            P(2, "two"),
-            P(3, "three"),
-            P(4, "four"),
-            P(1, "four"),
-            P(2, "four"),
-        };
-        C c0(a, a + sizeof(a)/sizeof(a[0]),
-            7,
-            test_hash<int>(8),
-            test_equal_to<int>(9),
-            A(10)
-           );
-        C c(a, a + 2,
-            7,
-            test_hash<int>(2),
-            test_equal_to<int>(3),
-            A(4)
-           );
-        C::iterator it0 = c0.begin();
-        c = std::move(c0);
-        LIBCPP_ASSERT(c.bucket_count() == 7);
-        assert(c.size() == 4);
-        assert(c.at(1) == "one");
-        assert(c.at(2) == "two");
-        assert(c.at(3) == "three");
-        assert(c.at(4) == "four");
-        assert(c.hash_function() == test_hash<int>(8));
-        assert(c.key_eq() == test_equal_to<int>(9));
-        assert(c.get_allocator() == A(10));
-        assert(!c.empty());
-        assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-        assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
-        assert(fabs(c.load_factor() - (float)c.size()/c.bucket_count()) < FLT_EPSILON);
-        assert(c.max_load_factor() == 1);
-        assert(c0.size() == 0);
-        assert(it0 == c.begin()); // Iterators remain valid
-    }
-    {
-        typedef min_allocator<std::pair<const int, std::string> > A;
-        typedef std::unordered_map<int, std::string,
-                                   test_hash<int>,
-                                   test_equal_to<int>,
-                                   A
-                                   > C;
-        typedef std::pair<int, std::string> P;
-        P a[] =
-        {
-            P(1, "one"),
-            P(2, "two"),
-            P(3, "three"),
-            P(4, "four"),
-            P(1, "four"),
-            P(2, "four"),
-        };
-        C c0(a, a + sizeof(a)/sizeof(a[0]),
-            7,
-            test_hash<int>(8),
-            test_equal_to<int>(9),
-            A()
-           );
-        C c(a, a + 2,
-            7,
-            test_hash<int>(2),
-            test_equal_to<int>(3),
-            A()
-           );
-        C::iterator it0 = c0.begin();
-        c = std::move(c0);
-        LIBCPP_ASSERT(c.bucket_count() == 7);
-        assert(c.size() == 4);
-        assert(c.at(1) == "one");
-        assert(c.at(2) == "two");
-        assert(c.at(3) == "three");
-        assert(c.at(4) == "four");
-        assert(c.hash_function() == test_hash<int>(8));
-        assert(c.key_eq() == test_equal_to<int>(9));
-        assert(c.get_allocator() == A());
-        assert(!c.empty());
-        assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-        assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
-        assert(fabs(c.load_factor() - (float)c.size()/c.bucket_count()) < FLT_EPSILON);
-        assert(c.max_load_factor() == 1);
-        assert(c0.size() == 0);
-        assert(it0 == c.begin()); // Iterators remain valid
-    }
+int main(int, char**) {
+  {
+    typedef test_allocator<std::pair<const int, std::string> > A;
+    typedef std::unordered_map<int, std::string, test_hash<int>, test_equal_to<int>, A > C;
+    typedef std::pair<int, std::string> P;
+    P a[] = {
+        P(1, "one"),
+        P(2, "two"),
+        P(3, "three"),
+        P(4, "four"),
+        P(1, "four"),
+        P(2, "four"),
+    };
+    C c0(a, a + sizeof(a) / sizeof(a[0]), 7, test_hash<int>(8), test_equal_to<int>(9), A(10));
+    C c(a, a + 2, 7, test_hash<int>(2), test_equal_to<int>(3), A(4));
+    c = std::move(c0);
+    LIBCPP_ASSERT(c.bucket_count() == 7);
+    assert(c.size() == 4);
+    assert(c.at(1) == "one");
+    assert(c.at(2) == "two");
+    assert(c.at(3) == "three");
+    assert(c.at(4) == "four");
+    assert(c.hash_function() == test_hash<int>(8));
+    assert(c.key_eq() == test_equal_to<int>(9));
+    assert(c.get_allocator() == A(4));
+    assert(!c.empty());
+    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
+    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
+    assert(fabs(c.load_factor() - (float)c.size() / c.bucket_count()) < FLT_EPSILON);
+    assert(c.max_load_factor() == 1);
+  }
+  {
+    typedef test_allocator<std::pair<const int, std::string> > A;
+    typedef std::unordered_map<int, std::string, test_hash<int>, test_equal_to<int>, A > C;
+    typedef std::pair<int, std::string> P;
+    P a[] = {
+        P(1, "one"),
+        P(2, "two"),
+        P(3, "three"),
+        P(4, "four"),
+        P(1, "four"),
+        P(2, "four"),
+    };
+    C c0(a, a + sizeof(a) / sizeof(a[0]), 7, test_hash<int>(8), test_equal_to<int>(9), A(10));
+    C c(a, a + 2, 7, test_hash<int>(2), test_equal_to<int>(3), A(10));
+    C::iterator it0 = c0.begin();
+    c               = std::move(c0);
+    LIBCPP_ASSERT(c.bucket_count() == 7);
+    assert(c.size() == 4);
+    assert(c.at(1) == "one");
+    assert(c.at(2) == "two");
+    assert(c.at(3) == "three");
+    assert(c.at(4) == "four");
+    assert(c.hash_function() == test_hash<int>(8));
+    assert(c.key_eq() == test_equal_to<int>(9));
+    assert(c.get_allocator() == A(10));
+    assert(!c.empty());
+    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
+    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
+    assert(fabs(c.load_factor() - (float)c.size() / c.bucket_count()) < FLT_EPSILON);
+    assert(c.max_load_factor() == 1);
+    assert(c0.size() == 0);
+    assert(it0 == c.begin()); // Iterators remain valid
+  }
+  {
+    typedef other_allocator<std::pair<const int, std::string> > A;
+    typedef std::unordered_map<int, std::string, test_hash<int>, test_equal_to<int>, A > C;
+    typedef std::pair<int, std::string> P;
+    P a[] = {
+        P(1, "one"),
+        P(2, "two"),
+        P(3, "three"),
+        P(4, "four"),
+        P(1, "four"),
+        P(2, "four"),
+    };
+    C c0(a, a + sizeof(a) / sizeof(a[0]), 7, test_hash<int>(8), test_equal_to<int>(9), A(10));
+    C c(a, a + 2, 7, test_hash<int>(2), test_equal_to<int>(3), A(4));
+    C::iterator it0 = c0.begin();
+    c               = std::move(c0);
+    LIBCPP_ASSERT(c.bucket_count() == 7);
+    assert(c.size() == 4);
+    assert(c.at(1) == "one");
+    assert(c.at(2) == "two");
+    assert(c.at(3) == "three");
+    assert(c.at(4) == "four");
+    assert(c.hash_function() == test_hash<int>(8));
+    assert(c.key_eq() == test_equal_to<int>(9));
+    assert(c.get_allocator() == A(10));
+    assert(!c.empty());
+    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
+    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
+    assert(fabs(c.load_factor() - (float)c.size() / c.bucket_count()) < FLT_EPSILON);
+    assert(c.max_load_factor() == 1);
+    assert(c0.size() == 0);
+    assert(it0 == c.begin()); // Iterators remain valid
+  }
+  {
+    typedef min_allocator<std::pair<const int, std::string> > A;
+    typedef std::unordered_map<int, std::string, test_hash<int>, test_equal_to<int>, A > C;
+    typedef std::pair<int, std::string> P;
+    P a[] = {
+        P(1, "one"),
+        P(2, "two"),
+        P(3, "three"),
+        P(4, "four"),
+        P(1, "four"),
+        P(2, "four"),
+    };
+    C c0(a, a + sizeof(a) / sizeof(a[0]), 7, test_hash<int>(8), test_equal_to<int>(9), A());
+    C c(a, a + 2, 7, test_hash<int>(2), test_equal_to<int>(3), A());
+    C::iterator it0 = c0.begin();
+    c               = std::move(c0);
+    LIBCPP_ASSERT(c.bucket_count() == 7);
+    assert(c.size() == 4);
+    assert(c.at(1) == "one");
+    assert(c.at(2) == "two");
+    assert(c.at(3) == "three");
+    assert(c.at(4) == "four");
+    assert(c.hash_function() == test_hash<int>(8));
+    assert(c.key_eq() == test_equal_to<int>(9));
+    assert(c.get_allocator() == A());
+    assert(!c.empty());
+    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
+    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
+    assert(fabs(c.load_factor() - (float)c.size() / c.bucket_count()) < FLT_EPSILON);
+    assert(c.max_load_factor() == 1);
+    assert(c0.size() == 0);
+    assert(it0 == c.begin()); // Iterators remain valid
+  }
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/unord/unord.map/unord.map.cnstr/compare_copy_constructible.compile.fail.cpp b/libcxx/test/std/containers/unord/unord.map/unord.map.cnstr/compare_copy_constructible.compile.fail.cpp
index 22c0d088314ce..5591d7d66a3ac 100644
--- a/libcxx/test/std/containers/unord/unord.map/unord.map.cnstr/compare_copy_constructible.compile.fail.cpp
+++ b/libcxx/test/std/containers/unord/unord.map/unord.map.cnstr/compare_copy_constructible.compile.fail.cpp
@@ -18,16 +18,16 @@
 
 template <class T>
 struct Comp {
-    bool operator () (const T& lhs, const T& rhs) const { return lhs == rhs; }
+  bool operator()(const T& lhs, const T& rhs) const { return lhs == rhs; }
 
-    Comp () {}
-private:
-    Comp (const Comp &); // declared but not defined
-    };
+  Comp() {}
 
+private:
+  Comp(const Comp&); // declared but not defined
+};
 
 int main(int, char**) {
-    std::unordered_map<int, int, std::hash<int>, Comp<int> > m;
+  std::unordered_map<int, int, std::hash<int>, Comp<int> > m;
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/unord/unord.map/unord.map.cnstr/copy.pass.cpp b/libcxx/test/std/containers/unord/unord.map/unord.map.cnstr/copy.pass.cpp
index 6dad15ef23deb..793807babbdca 100644
--- a/libcxx/test/std/containers/unord/unord.map/unord.map.cnstr/copy.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.map/unord.map.cnstr/copy.pass.cpp
@@ -27,126 +27,122 @@
 #include "test_allocator.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef std::unordered_map<int, std::string,
-                                   test_hash<int>,
-                                   test_equal_to<int>,
-                                   test_allocator<std::pair<const int, std::string> >
-                                   > C;
-        typedef std::pair<int, std::string> P;
-        P a[] =
-        {
-            P(1, "one"),
-            P(2, "two"),
-            P(3, "three"),
-            P(4, "four"),
-            P(1, "four"),
-            P(2, "four"),
-        };
-        C c0(a, a + sizeof(a)/sizeof(a[0]),
-            7,
-            test_hash<int>(8),
-            test_equal_to<int>(9),
-            test_allocator<std::pair<const int, std::string> >(10)
-           );
-        C c = c0;
-        LIBCPP_ASSERT(c.bucket_count() == 7);
-        assert(c.size() == 4);
-        assert(c.at(1) == "one");
-        assert(c.at(2) == "two");
-        assert(c.at(3) == "three");
-        assert(c.at(4) == "four");
-        assert(c.hash_function() == test_hash<int>(8));
-        assert(c.key_eq() == test_equal_to<int>(9));
-        assert(c.get_allocator() ==
-               (test_allocator<std::pair<const int, std::string> >(10)));
-        assert(!c.empty());
-        assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-        assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
-        assert(std::fabs(c.load_factor() - (float)c.size()/c.bucket_count()) < FLT_EPSILON);
-        assert(c.max_load_factor() == 1);
-    }
+int main(int, char**) {
+  {
+    typedef std::unordered_map<int,
+                               std::string,
+                               test_hash<int>,
+                               test_equal_to<int>,
+                               test_allocator<std::pair<const int, std::string> > >
+        C;
+    typedef std::pair<int, std::string> P;
+    P a[] = {
+        P(1, "one"),
+        P(2, "two"),
+        P(3, "three"),
+        P(4, "four"),
+        P(1, "four"),
+        P(2, "four"),
+    };
+    C c0(a,
+         a + sizeof(a) / sizeof(a[0]),
+         7,
+         test_hash<int>(8),
+         test_equal_to<int>(9),
+         test_allocator<std::pair<const int, std::string> >(10));
+    C c = c0;
+    LIBCPP_ASSERT(c.bucket_count() == 7);
+    assert(c.size() == 4);
+    assert(c.at(1) == "one");
+    assert(c.at(2) == "two");
+    assert(c.at(3) == "three");
+    assert(c.at(4) == "four");
+    assert(c.hash_function() == test_hash<int>(8));
+    assert(c.key_eq() == test_equal_to<int>(9));
+    assert(c.get_allocator() == (test_allocator<std::pair<const int, std::string> >(10)));
+    assert(!c.empty());
+    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
+    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
+    assert(std::fabs(c.load_factor() - (float)c.size() / c.bucket_count()) < FLT_EPSILON);
+    assert(c.max_load_factor() == 1);
+  }
 #if TEST_STD_VER >= 11
-    {
-        typedef std::unordered_map<int, std::string,
-                                   test_hash<int>,
-                                   test_equal_to<int>,
-                                   other_allocator<std::pair<const int, std::string> >
-                                   > C;
-        typedef std::pair<int, std::string> P;
-        P a[] =
-        {
-            P(1, "one"),
-            P(2, "two"),
-            P(3, "three"),
-            P(4, "four"),
-            P(1, "four"),
-            P(2, "four"),
-        };
-        C c0(a, a + sizeof(a)/sizeof(a[0]),
-            7,
-            test_hash<int>(8),
-            test_equal_to<int>(9),
-            other_allocator<std::pair<const int, std::string> >(10)
-           );
-        C c = c0;
-        LIBCPP_ASSERT(c.bucket_count() == 7);
-        assert(c.size() == 4);
-        assert(c.at(1) == "one");
-        assert(c.at(2) == "two");
-        assert(c.at(3) == "three");
-        assert(c.at(4) == "four");
-        assert(c.hash_function() == test_hash<int>(8));
-        assert(c.key_eq() == test_equal_to<int>(9));
-        assert(c.get_allocator() ==
-               (other_allocator<std::pair<const int, std::string> >(-2)));
-        assert(!c.empty());
-        assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-        assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
-        assert(std::fabs(c.load_factor() - (float)c.size()/c.bucket_count()) < FLT_EPSILON);
-        assert(c.max_load_factor() == 1);
-    }
-    {
-        typedef std::unordered_map<int, std::string,
-                                   test_hash<int>,
-                                   test_equal_to<int>,
-                                   min_allocator<std::pair<const int, std::string> >
-                                   > C;
-        typedef std::pair<int, std::string> P;
-        P a[] =
-        {
-            P(1, "one"),
-            P(2, "two"),
-            P(3, "three"),
-            P(4, "four"),
-            P(1, "four"),
-            P(2, "four"),
-        };
-        C c0(a, a + sizeof(a)/sizeof(a[0]),
-            7,
-            test_hash<int>(8),
-            test_equal_to<int>(9),
-            min_allocator<std::pair<const int, std::string> >()
-           );
-        C c = c0;
-        LIBCPP_ASSERT(c.bucket_count() == 7);
-        assert(c.size() == 4);
-        assert(c.at(1) == "one");
-        assert(c.at(2) == "two");
-        assert(c.at(3) == "three");
-        assert(c.at(4) == "four");
-        assert(c.hash_function() == test_hash<int>(8));
-        assert(c.key_eq() == test_equal_to<int>(9));
-        assert(c.get_allocator() ==
-               (min_allocator<std::pair<const int, std::string> >()));
-        assert(!c.empty());
-        assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-        assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
-        assert(std::fabs(c.load_factor() - (float)c.size()/c.bucket_count()) < FLT_EPSILON);
-        assert(c.max_load_factor() == 1);
-    }
+  {
+    typedef std::unordered_map<int,
+                               std::string,
+                               test_hash<int>,
+                               test_equal_to<int>,
+                               other_allocator<std::pair<const int, std::string> > >
+        C;
+    typedef std::pair<int, std::string> P;
+    P a[] = {
+        P(1, "one"),
+        P(2, "two"),
+        P(3, "three"),
+        P(4, "four"),
+        P(1, "four"),
+        P(2, "four"),
+    };
+    C c0(a,
+         a + sizeof(a) / sizeof(a[0]),
+         7,
+         test_hash<int>(8),
+         test_equal_to<int>(9),
+         other_allocator<std::pair<const int, std::string> >(10));
+    C c = c0;
+    LIBCPP_ASSERT(c.bucket_count() == 7);
+    assert(c.size() == 4);
+    assert(c.at(1) == "one");
+    assert(c.at(2) == "two");
+    assert(c.at(3) == "three");
+    assert(c.at(4) == "four");
+    assert(c.hash_function() == test_hash<int>(8));
+    assert(c.key_eq() == test_equal_to<int>(9));
+    assert(c.get_allocator() == (other_allocator<std::pair<const int, std::string> >(-2)));
+    assert(!c.empty());
+    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
+    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
+    assert(std::fabs(c.load_factor() - (float)c.size() / c.bucket_count()) < FLT_EPSILON);
+    assert(c.max_load_factor() == 1);
+  }
+  {
+    typedef std::unordered_map<int,
+                               std::string,
+                               test_hash<int>,
+                               test_equal_to<int>,
+                               min_allocator<std::pair<const int, std::string> > >
+        C;
+    typedef std::pair<int, std::string> P;
+    P a[] = {
+        P(1, "one"),
+        P(2, "two"),
+        P(3, "three"),
+        P(4, "four"),
+        P(1, "four"),
+        P(2, "four"),
+    };
+    C c0(a,
+         a + sizeof(a) / sizeof(a[0]),
+         7,
+         test_hash<int>(8),
+         test_equal_to<int>(9),
+         min_allocator<std::pair<const int, std::string> >());
+    C c = c0;
+    LIBCPP_ASSERT(c.bucket_count() == 7);
+    assert(c.size() == 4);
+    assert(c.at(1) == "one");
+    assert(c.at(2) == "two");
+    assert(c.at(3) == "three");
+    assert(c.at(4) == "four");
+    assert(c.hash_function() == test_hash<int>(8));
+    assert(c.key_eq() == test_equal_to<int>(9));
+    assert(c.get_allocator() == (min_allocator<std::pair<const int, std::string> >()));
+    assert(!c.empty());
+    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
+    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
+    assert(std::fabs(c.load_factor() - (float)c.size() / c.bucket_count()) < FLT_EPSILON);
+    assert(c.max_load_factor() == 1);
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/unord/unord.map/unord.map.cnstr/copy_alloc.pass.cpp b/libcxx/test/std/containers/unord/unord.map/unord.map.cnstr/copy_alloc.pass.cpp
index 1d9cf9f5932e6..65c49f4fdcd00 100644
--- a/libcxx/test/std/containers/unord/unord.map/unord.map.cnstr/copy_alloc.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.map/unord.map.cnstr/copy_alloc.pass.cpp
@@ -27,126 +27,113 @@
 #include "test_allocator.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef std::unordered_map<int, std::string,
-                                   test_hash<int>,
-                                   test_equal_to<int>,
-                                   test_allocator<std::pair<const int, std::string> >
-                                   > C;
-        typedef std::pair<int, std::string> P;
-        P a[] =
-        {
-            P(1, "one"),
-            P(2, "two"),
-            P(3, "three"),
-            P(4, "four"),
-            P(1, "four"),
-            P(2, "four"),
-        };
-        C c0(a, a + sizeof(a)/sizeof(a[0]),
-            7,
-            test_hash<int>(8),
-            test_equal_to<int>(9),
-            test_allocator<std::pair<const int, std::string> >(10)
-           );
-        C c(c0, test_allocator<std::pair<const int, std::string> >(5));
-        LIBCPP_ASSERT(c.bucket_count() == 7);
-        assert(c.size() == 4);
-        assert(c.at(1) == "one");
-        assert(c.at(2) == "two");
-        assert(c.at(3) == "three");
-        assert(c.at(4) == "four");
-        assert(c.hash_function() == test_hash<int>(8));
-        assert(c.key_eq() == test_equal_to<int>(9));
-        assert(c.get_allocator() ==
-               (test_allocator<std::pair<const int, std::string> >(5)));
-        assert(!c.empty());
-        assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-        assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
-        assert(std::fabs(c.load_factor() - (float)c.size()/c.bucket_count()) < FLT_EPSILON);
-        assert(c.max_load_factor() == 1);
-    }
+int main(int, char**) {
+  {
+    typedef std::unordered_map<int,
+                               std::string,
+                               test_hash<int>,
+                               test_equal_to<int>,
+                               test_allocator<std::pair<const int, std::string> > >
+        C;
+    typedef std::pair<int, std::string> P;
+    P a[] = {
+        P(1, "one"),
+        P(2, "two"),
+        P(3, "three"),
+        P(4, "four"),
+        P(1, "four"),
+        P(2, "four"),
+    };
+    C c0(a,
+         a + sizeof(a) / sizeof(a[0]),
+         7,
+         test_hash<int>(8),
+         test_equal_to<int>(9),
+         test_allocator<std::pair<const int, std::string> >(10));
+    C c(c0, test_allocator<std::pair<const int, std::string> >(5));
+    LIBCPP_ASSERT(c.bucket_count() == 7);
+    assert(c.size() == 4);
+    assert(c.at(1) == "one");
+    assert(c.at(2) == "two");
+    assert(c.at(3) == "three");
+    assert(c.at(4) == "four");
+    assert(c.hash_function() == test_hash<int>(8));
+    assert(c.key_eq() == test_equal_to<int>(9));
+    assert(c.get_allocator() == (test_allocator<std::pair<const int, std::string> >(5)));
+    assert(!c.empty());
+    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
+    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
+    assert(std::fabs(c.load_factor() - (float)c.size() / c.bucket_count()) < FLT_EPSILON);
+    assert(c.max_load_factor() == 1);
+  }
 #if TEST_STD_VER >= 11
-    {
-        typedef std::unordered_map<int, std::string,
-                                   test_hash<int>,
-                                   test_equal_to<int>,
-                                   min_allocator<std::pair<const int, std::string> >
-                                   > C;
-        typedef std::pair<int, std::string> P;
-        P a[] =
-        {
-            P(1, "one"),
-            P(2, "two"),
-            P(3, "three"),
-            P(4, "four"),
-            P(1, "four"),
-            P(2, "four"),
-        };
-        C c0(a, a + sizeof(a)/sizeof(a[0]),
-            7,
-            test_hash<int>(8),
-            test_equal_to<int>(9),
-            min_allocator<std::pair<const int, std::string> >()
-           );
-        C c(c0, min_allocator<std::pair<const int, std::string> >());
-        LIBCPP_ASSERT(c.bucket_count() == 7);
-        assert(c.size() == 4);
-        assert(c.at(1) == "one");
-        assert(c.at(2) == "two");
-        assert(c.at(3) == "three");
-        assert(c.at(4) == "four");
-        assert(c.hash_function() == test_hash<int>(8));
-        assert(c.key_eq() == test_equal_to<int>(9));
-        assert(c.get_allocator() ==
-               (min_allocator<std::pair<const int, std::string> >()));
-        assert(!c.empty());
-        assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-        assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
-        assert(std::fabs(c.load_factor() - (float)c.size()/c.bucket_count()) < FLT_EPSILON);
-        assert(c.max_load_factor() == 1);
-    }
-    {
-        typedef explicit_allocator<std::pair<const int, std::string>> A;
-        typedef std::unordered_map<int, std::string,
-                                   test_hash<int>,
-                                   test_equal_to<int>,
-                                   A
-                                   > C;
-        typedef std::pair<int, std::string> P;
-        P a[] =
-        {
-            P(1, "one"),
-            P(2, "two"),
-            P(3, "three"),
-            P(4, "four"),
-            P(1, "four"),
-            P(2, "four"),
-        };
-        C c0(a, a + sizeof(a)/sizeof(a[0]),
-            7,
-            test_hash<int>(8),
-            test_equal_to<int>(9),
-            A{}
-           );
-        C c(c0, A{});
-        LIBCPP_ASSERT(c.bucket_count() == 7);
-        assert(c.size() == 4);
-        assert(c.at(1) == "one");
-        assert(c.at(2) == "two");
-        assert(c.at(3) == "three");
-        assert(c.at(4) == "four");
-        assert(c.hash_function() == test_hash<int>(8));
-        assert(c.key_eq() == test_equal_to<int>(9));
-        assert(c.get_allocator() == A{});
-        assert(!c.empty());
-        assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-        assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
-        assert(std::fabs(c.load_factor() - (float)c.size()/c.bucket_count()) < FLT_EPSILON);
-        assert(c.max_load_factor() == 1);
-    }
+  {
+    typedef std::unordered_map<int,
+                               std::string,
+                               test_hash<int>,
+                               test_equal_to<int>,
+                               min_allocator<std::pair<const int, std::string> > >
+        C;
+    typedef std::pair<int, std::string> P;
+    P a[] = {
+        P(1, "one"),
+        P(2, "two"),
+        P(3, "three"),
+        P(4, "four"),
+        P(1, "four"),
+        P(2, "four"),
+    };
+    C c0(a,
+         a + sizeof(a) / sizeof(a[0]),
+         7,
+         test_hash<int>(8),
+         test_equal_to<int>(9),
+         min_allocator<std::pair<const int, std::string> >());
+    C c(c0, min_allocator<std::pair<const int, std::string> >());
+    LIBCPP_ASSERT(c.bucket_count() == 7);
+    assert(c.size() == 4);
+    assert(c.at(1) == "one");
+    assert(c.at(2) == "two");
+    assert(c.at(3) == "three");
+    assert(c.at(4) == "four");
+    assert(c.hash_function() == test_hash<int>(8));
+    assert(c.key_eq() == test_equal_to<int>(9));
+    assert(c.get_allocator() == (min_allocator<std::pair<const int, std::string> >()));
+    assert(!c.empty());
+    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
+    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
+    assert(std::fabs(c.load_factor() - (float)c.size() / c.bucket_count()) < FLT_EPSILON);
+    assert(c.max_load_factor() == 1);
+  }
+  {
+    typedef explicit_allocator<std::pair<const int, std::string>> A;
+    typedef std::unordered_map<int, std::string, test_hash<int>, test_equal_to<int>, A > C;
+    typedef std::pair<int, std::string> P;
+    P a[] = {
+        P(1, "one"),
+        P(2, "two"),
+        P(3, "three"),
+        P(4, "four"),
+        P(1, "four"),
+        P(2, "four"),
+    };
+    C c0(a, a + sizeof(a) / sizeof(a[0]), 7, test_hash<int>(8), test_equal_to<int>(9), A{});
+    C c(c0, A{});
+    LIBCPP_ASSERT(c.bucket_count() == 7);
+    assert(c.size() == 4);
+    assert(c.at(1) == "one");
+    assert(c.at(2) == "two");
+    assert(c.at(3) == "three");
+    assert(c.at(4) == "four");
+    assert(c.hash_function() == test_hash<int>(8));
+    assert(c.key_eq() == test_equal_to<int>(9));
+    assert(c.get_allocator() == A{});
+    assert(!c.empty());
+    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
+    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
+    assert(std::fabs(c.load_factor() - (float)c.size() / c.bucket_count()) < FLT_EPSILON);
+    assert(c.max_load_factor() == 1);
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/unord/unord.map/unord.map.cnstr/deduct.pass.cpp b/libcxx/test/std/containers/unord/unord.map/unord.map.cnstr/deduct.pass.cpp
index 873b77e06a8d8..b2ef6be5523a5 100644
--- a/libcxx/test/std/containers/unord/unord.map/unord.map.cnstr/deduct.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.map/unord.map.cnstr/deduct.pass.cpp
@@ -90,152 +90,168 @@
 #include "deduction_guides_sfinae_checks.h"
 #include "test_allocator.h"
 
-using P = std::pair<int, long>;
+using P  = std::pair<int, long>;
 using PC = std::pair<const int, long>;
 
-int main(int, char**)
-{
-    const PC expected_m[] = { {1,1}, {2,2}, {3,1}, {INT_MAX,1} };
+int main(int, char**) {
+  const PC expected_m[] = {{1, 1}, {2, 2}, {3, 1}, {INT_MAX, 1}};
 
-    {
-    const P arr[] = { {1,1}, {2,2}, {1,1}, {INT_MAX,1}, {3,1} };
+  {
+    const P arr[] = {{1, 1}, {2, 2}, {1, 1}, {INT_MAX, 1}, {3, 1}};
     std::unordered_map m(std::begin(arr), std::end(arr));
     ASSERT_SAME_TYPE(decltype(m), std::unordered_map<int, long>);
     assert(std::is_permutation(m.begin(), m.end(), std::begin(expected_m), std::end(expected_m)));
-    }
+  }
 
-    {
-    const P arr[] = { {1,1}, {2,2}, {1,1}, {INT_MAX,1}, {3,1} };
+  {
+    const P arr[] = {{1, 1}, {2, 2}, {1, 1}, {INT_MAX, 1}, {3, 1}};
     std::unordered_map m(std::begin(arr), std::end(arr), 42);
     ASSERT_SAME_TYPE(decltype(m), std::unordered_map<int, long>);
     assert(std::is_permutation(m.begin(), m.end(), std::begin(expected_m), std::end(expected_m)));
-    }
+  }
 
-    {
-    const P arr[] = { {1,1}, {2,2}, {1,1}, {INT_MAX,1}, {3,1} };
+  {
+    const P arr[] = {{1, 1}, {2, 2}, {1, 1}, {INT_MAX, 1}, {3, 1}};
     std::unordered_map m(std::begin(arr), std::end(arr), 42, std::hash<long long>());
     ASSERT_SAME_TYPE(decltype(m), std::unordered_map<int, long, std::hash<long long>, std::equal_to<int>>);
     assert(std::is_permutation(m.begin(), m.end(), std::begin(expected_m), std::end(expected_m)));
-    }
+  }
 
-    {
-    const P arr[] = { {1,1}, {2,2}, {1,1}, {INT_MAX,1}, {3,1} };
+  {
+    const P arr[] = {{1, 1}, {2, 2}, {1, 1}, {INT_MAX, 1}, {3, 1}};
     std::unordered_map m(std::begin(arr), std::end(arr), 42, std::hash<long long>(), std::equal_to<>());
     ASSERT_SAME_TYPE(decltype(m), std::unordered_map<int, long, std::hash<long long>, std::equal_to<>>);
     assert(std::is_permutation(m.begin(), m.end(), std::begin(expected_m), std::end(expected_m)));
-    }
-
-    {
-    const P arr[] = { {1,1}, {2,2}, {1,1}, {INT_MAX,1}, {3,1} };
-    std::unordered_map m(std::begin(arr), std::end(arr), 42, std::hash<long long>(), std::equal_to<>(), test_allocator<PC>(0, 41));
-    ASSERT_SAME_TYPE(decltype(m), std::unordered_map<int, long, std::hash<long long>, std::equal_to<>, test_allocator<PC>>);
+  }
+
+  {
+    const P arr[] = {{1, 1}, {2, 2}, {1, 1}, {INT_MAX, 1}, {3, 1}};
+    std::unordered_map m(
+        std::begin(arr), std::end(arr), 42, std::hash<long long>(), std::equal_to<>(), test_allocator<PC>(0, 41));
+    ASSERT_SAME_TYPE(
+        decltype(m), std::unordered_map<int, long, std::hash<long long>, std::equal_to<>, test_allocator<PC>>);
     assert(std::is_permutation(m.begin(), m.end(), std::begin(expected_m), std::end(expected_m)));
     assert(m.get_allocator().get_id() == 41);
-    }
+  }
 
-    {
+  {
     std::unordered_map<int, long> source;
     std::unordered_map m(source);
     ASSERT_SAME_TYPE(decltype(m), decltype(source));
     assert(m.size() == 0);
-    }
+  }
 
-    {
+  {
     std::unordered_map<int, long> source;
-    std::unordered_map m{source};  // braces instead of parens
+    std::unordered_map m{source}; // braces instead of parens
     ASSERT_SAME_TYPE(decltype(m), decltype(source));
     assert(m.size() == 0);
-    }
+  }
 
-    {
+  {
     std::unordered_map<int, long, std::hash<long long>, std::equal_to<>, test_allocator<PC>> source;
     test_allocator<PC> a(0, 42);
     std::unordered_map m(source, a);
     ASSERT_SAME_TYPE(decltype(m), decltype(source));
     assert(m.get_allocator().get_id() == 42);
     assert(m.size() == 0);
-    }
+  }
 
-    {
+  {
     std::unordered_map<int, long, std::hash<long long>, std::equal_to<>, test_allocator<PC>> source;
     test_allocator<PC> a(0, 43);
-    std::unordered_map m{source, a};  // braces instead of parens
+    std::unordered_map m{source, a}; // braces instead of parens
     ASSERT_SAME_TYPE(decltype(m), decltype(source));
     assert(m.get_allocator().get_id() == 43);
     assert(m.size() == 0);
-    }
+  }
 
-    {
-    std::unordered_map m { P{1,1L}, P{2,2L}, P{1,1L}, P{INT_MAX,1L}, P{3,1L} };
+  {
+    std::unordered_map m{P{1, 1L}, P{2, 2L}, P{1, 1L}, P{INT_MAX, 1L}, P{3, 1L}};
     ASSERT_SAME_TYPE(decltype(m), std::unordered_map<int, long>);
     assert(std::is_permutation(m.begin(), m.end(), std::begin(expected_m), std::end(expected_m)));
-    }
+  }
 
-    {
-    std::unordered_map m({ P{1,1L}, P{2,2L}, P{1,1L}, P{INT_MAX,1L}, P{3,1L} }, 42);
+  {
+    std::unordered_map m({P{1, 1L}, P{2, 2L}, P{1, 1L}, P{INT_MAX, 1L}, P{3, 1L}}, 42);
     ASSERT_SAME_TYPE(decltype(m), std::unordered_map<int, long>);
     assert(std::is_permutation(m.begin(), m.end(), std::begin(expected_m), std::end(expected_m)));
-    }
+  }
 
-    {
-    std::unordered_map m({ P{1,1L}, P{2,2L}, P{1,1L}, P{INT_MAX,1L}, P{3,1L} }, 42, std::hash<long long>());
+  {
+    std::unordered_map m({P{1, 1L}, P{2, 2L}, P{1, 1L}, P{INT_MAX, 1L}, P{3, 1L}}, 42, std::hash<long long>());
     ASSERT_SAME_TYPE(decltype(m), std::unordered_map<int, long, std::hash<long long>>);
     assert(std::is_permutation(m.begin(), m.end(), std::begin(expected_m), std::end(expected_m)));
-    }
+  }
 
-    {
-    std::unordered_map m({ P{1,1L}, P{2,2L}, P{1,1L}, P{INT_MAX,1L}, P{3,1L} }, 42, std::hash<long long>(), std::equal_to<>());
+  {
+    std::unordered_map m(
+        {P{1, 1L}, P{2, 2L}, P{1, 1L}, P{INT_MAX, 1L}, P{3, 1L}}, 42, std::hash<long long>(), std::equal_to<>());
     ASSERT_SAME_TYPE(decltype(m), std::unordered_map<int, long, std::hash<long long>, std::equal_to<>>);
     assert(std::is_permutation(m.begin(), m.end(), std::begin(expected_m), std::end(expected_m)));
-    }
-
-    {
-    std::unordered_map m({ P{1,1L}, P{2,2L}, P{1,1L}, P{INT_MAX,1L}, P{3,1L} }, 42, std::hash<long long>(), std::equal_to<>(), test_allocator<PC>(0, 44));
-    ASSERT_SAME_TYPE(decltype(m), std::unordered_map<int, long, std::hash<long long>, std::equal_to<>, test_allocator<PC>>);
+  }
+
+  {
+    std::unordered_map m(
+        {P{1, 1L}, P{2, 2L}, P{1, 1L}, P{INT_MAX, 1L}, P{3, 1L}},
+        42,
+        std::hash<long long>(),
+        std::equal_to<>(),
+        test_allocator<PC>(0, 44));
+    ASSERT_SAME_TYPE(
+        decltype(m), std::unordered_map<int, long, std::hash<long long>, std::equal_to<>, test_allocator<PC>>);
     assert(std::is_permutation(m.begin(), m.end(), std::begin(expected_m), std::end(expected_m)));
     assert(m.get_allocator().get_id() == 44);
-    }
+  }
 
-    {
-    const P arr[] = { {1,1}, {2,2}, {1,1}, {INT_MAX,1}, {3,1} };
+  {
+    const P arr[] = {{1, 1}, {2, 2}, {1, 1}, {INT_MAX, 1}, {3, 1}};
     std::unordered_map m(std::begin(arr), std::end(arr), 42, test_allocator<PC>(0, 45));
-    ASSERT_SAME_TYPE(decltype(m), std::unordered_map<int, long, std::hash<int>, std::equal_to<int>, test_allocator<PC>>);
+    ASSERT_SAME_TYPE(
+        decltype(m), std::unordered_map<int, long, std::hash<int>, std::equal_to<int>, test_allocator<PC>>);
     assert(std::is_permutation(m.begin(), m.end(), std::begin(expected_m), std::end(expected_m)));
     assert(m.get_allocator().get_id() == 45);
-    }
+  }
 
-    {
-    const P arr[] = { {1,1}, {2,2}, {1,1}, {INT_MAX,1}, {3,1} };
+  {
+    const P arr[] = {{1, 1}, {2, 2}, {1, 1}, {INT_MAX, 1}, {3, 1}};
     std::unordered_map m(std::begin(arr), std::end(arr), 42, std::hash<long long>(), test_allocator<PC>(0, 46));
-    ASSERT_SAME_TYPE(decltype(m), std::unordered_map<int, long, std::hash<long long>, std::equal_to<int>, test_allocator<PC>>);
+    ASSERT_SAME_TYPE(
+        decltype(m), std::unordered_map<int, long, std::hash<long long>, std::equal_to<int>, test_allocator<PC>>);
     assert(std::is_permutation(m.begin(), m.end(), std::begin(expected_m), std::end(expected_m)));
     assert(m.get_allocator().get_id() == 46);
-    }
+  }
 
-    {
-    std::unordered_map m({ P{1,1L}, P{2,2L}, P{1,1L}, P{INT_MAX,1L}, P{3,1L} }, 42, test_allocator<PC>(0, 47));
-    ASSERT_SAME_TYPE(decltype(m), std::unordered_map<int, long, std::hash<int>, std::equal_to<int>, test_allocator<PC>>);
+  {
+    std::unordered_map m({P{1, 1L}, P{2, 2L}, P{1, 1L}, P{INT_MAX, 1L}, P{3, 1L}}, 42, test_allocator<PC>(0, 47));
+    ASSERT_SAME_TYPE(
+        decltype(m), std::unordered_map<int, long, std::hash<int>, std::equal_to<int>, test_allocator<PC>>);
     assert(std::is_permutation(m.begin(), m.end(), std::begin(expected_m), std::end(expected_m)));
     assert(m.get_allocator().get_id() == 47);
-    }
-
-    {
-    std::unordered_map m({ P{1,1L}, P{2,2L}, P{1,1L}, P{INT_MAX,1L}, P{3,1L} }, 42, std::hash<long long>(), test_allocator<PC>(0, 48));
-    ASSERT_SAME_TYPE(decltype(m), std::unordered_map<int, long, std::hash<long long>, std::equal_to<int>, test_allocator<PC>>);
+  }
+
+  {
+    std::unordered_map m(
+        {P{1, 1L}, P{2, 2L}, P{1, 1L}, P{INT_MAX, 1L}, P{3, 1L}},
+        42,
+        std::hash<long long>(),
+        test_allocator<PC>(0, 48));
+    ASSERT_SAME_TYPE(
+        decltype(m), std::unordered_map<int, long, std::hash<long long>, std::equal_to<int>, test_allocator<PC>>);
     assert(std::is_permutation(m.begin(), m.end(), std::begin(expected_m), std::end(expected_m)));
     assert(m.get_allocator().get_id() == 48);
-    }
+  }
 
-    {
+  {
     // Examples from LWG3025
     std::unordered_map m{std::pair{1, 1}, {2, 2}, {3, 3}};
     ASSERT_SAME_TYPE(decltype(m), std::unordered_map<int, int>);
 
     std::unordered_map m2{m.begin(), m.end()};
     ASSERT_SAME_TYPE(decltype(m2), std::unordered_map<int, int>);
-    }
+  }
 
-    {
+  {
     // Examples from LWG3531
     std::unordered_map m1{{std::pair{1, 2}, {3, 4}}, 0};
     ASSERT_SAME_TYPE(decltype(m1), std::unordered_map<int, int>);
@@ -243,61 +259,61 @@ int main(int, char**)
     using value_type = std::pair<const int, int>;
     std::unordered_map m2{{value_type{1, 2}, {3, 4}}, 0};
     ASSERT_SAME_TYPE(decltype(m2), std::unordered_map<int, int>);
-    }
+  }
 
 #if TEST_STD_VER >= 23
-    {
-      using Range = std::array<P, 0>;
-      using Pred = test_equal_to<int>;
-      using DefaultPred = std::equal_to<int>;
-      using Hash = test_hash<int>;
-      using DefaultHash = std::hash<int>;
-      using Alloc = test_allocator<PC>;
-
-      { // (from_range, range)
-        std::unordered_map c(std::from_range, Range());
-        static_assert(std::is_same_v<decltype(c), std::unordered_map<int, long>>);
-      }
-
-      { // (from_range, range, n)
-        std::unordered_map c(std::from_range, Range(), std::size_t());
-        static_assert(std::is_same_v<decltype(c), std::unordered_map<int, long>>);
-      }
-
-      { // (from_range, range, n, hash)
-        std::unordered_map c(std::from_range, Range(), std::size_t(), Hash());
-        static_assert(std::is_same_v<decltype(c), std::unordered_map<int, long, Hash>>);
-      }
-
-      { // (from_range, range, n, hash, pred)
-        std::unordered_map c(std::from_range, Range(), std::size_t(), Hash(), Pred());
-        static_assert(std::is_same_v<decltype(c), std::unordered_map<int, long, Hash, Pred>>);
-      }
-
-      { // (from_range, range, n, hash, pred, alloc)
-        std::unordered_map c(std::from_range, Range(), std::size_t(), Hash(), Pred(), Alloc());
-        static_assert(std::is_same_v<decltype(c), std::unordered_map<int, long, Hash, Pred, Alloc>>);
-      }
-
-      { // (from_range, range, n, alloc)
-        std::unordered_map c(std::from_range, Range(), std::size_t(), Alloc());
-        static_assert(std::is_same_v<decltype(c), std::unordered_map<int, long, DefaultHash, DefaultPred, Alloc>>);
-      }
-
-      // TODO(LWG 2713): uncomment this test once the constructor is added.
-      { // (from_range, range, alloc)
-        //std::unordered_map c(std::from_range, Range(), Alloc());
-        //static_assert(std::is_same_v<decltype(c), std::unordered_map<int, long, DefaultHash, DefaultPred, Alloc>>);
-      }
-
-      { // (from_range, range, n, hash, alloc)
-        std::unordered_map c(std::from_range, Range(), std::size_t(), Hash(), Alloc());
-        static_assert(std::is_same_v<decltype(c), std::unordered_map<int, long, Hash, DefaultPred, Alloc>>);
-      }
+  {
+    using Range       = std::array<P, 0>;
+    using Pred        = test_equal_to<int>;
+    using DefaultPred = std::equal_to<int>;
+    using Hash        = test_hash<int>;
+    using DefaultHash = std::hash<int>;
+    using Alloc       = test_allocator<PC>;
+
+    { // (from_range, range)
+      std::unordered_map c(std::from_range, Range());
+      static_assert(std::is_same_v<decltype(c), std::unordered_map<int, long>>);
+    }
+
+    { // (from_range, range, n)
+      std::unordered_map c(std::from_range, Range(), std::size_t());
+      static_assert(std::is_same_v<decltype(c), std::unordered_map<int, long>>);
+    }
+
+    { // (from_range, range, n, hash)
+      std::unordered_map c(std::from_range, Range(), std::size_t(), Hash());
+      static_assert(std::is_same_v<decltype(c), std::unordered_map<int, long, Hash>>);
+    }
+
+    { // (from_range, range, n, hash, pred)
+      std::unordered_map c(std::from_range, Range(), std::size_t(), Hash(), Pred());
+      static_assert(std::is_same_v<decltype(c), std::unordered_map<int, long, Hash, Pred>>);
+    }
+
+    { // (from_range, range, n, hash, pred, alloc)
+      std::unordered_map c(std::from_range, Range(), std::size_t(), Hash(), Pred(), Alloc());
+      static_assert(std::is_same_v<decltype(c), std::unordered_map<int, long, Hash, Pred, Alloc>>);
+    }
+
+    { // (from_range, range, n, alloc)
+      std::unordered_map c(std::from_range, Range(), std::size_t(), Alloc());
+      static_assert(std::is_same_v<decltype(c), std::unordered_map<int, long, DefaultHash, DefaultPred, Alloc>>);
+    }
+
+    // TODO(LWG 2713): uncomment this test once the constructor is added.
+    { // (from_range, range, alloc)
+      //std::unordered_map c(std::from_range, Range(), Alloc());
+      //static_assert(std::is_same_v<decltype(c), std::unordered_map<int, long, DefaultHash, DefaultPred, Alloc>>);
+    }
+
+    { // (from_range, range, n, hash, alloc)
+      std::unordered_map c(std::from_range, Range(), std::size_t(), Hash(), Alloc());
+      static_assert(std::is_same_v<decltype(c), std::unordered_map<int, long, Hash, DefaultPred, Alloc>>);
     }
+  }
 #endif
 
-    UnorderedContainerDeductionGuidesSfinaeAway<std::unordered_map, std::unordered_map<int, long>>();
+  UnorderedContainerDeductionGuidesSfinaeAway<std::unordered_map, std::unordered_map<int, long>>();
 
-    return 0;
+  return 0;
 }
diff --git a/libcxx/test/std/containers/unord/unord.map/unord.map.cnstr/deduct.verify.cpp b/libcxx/test/std/containers/unord/unord.map/unord.map.cnstr/deduct.verify.cpp
index dc0ffd26813d9..5bb2b28fa09f5 100644
--- a/libcxx/test/std/containers/unord/unord.map/unord.map.cnstr/deduct.verify.cpp
+++ b/libcxx/test/std/containers/unord/unord.map/unord.map.cnstr/deduct.verify.cpp
@@ -59,47 +59,48 @@
 #include <functional>
 #include <unordered_map>
 
-int main(int, char**)
-{
-    using P = std::pair<const int, int>;
-    {
-        // cannot deduce Key from nothing
-        std::unordered_map m; // expected-error-re{{no viable constructor or deduction guide for deduction of template arguments of '{{(std::)?}}unordered_map'}}
-    }
-    {
-        // cannot deduce Key from just (Size)
-        std::unordered_map m(42); // expected-error-re{{no viable constructor or deduction guide for deduction of template arguments of '{{(std::)?}}unordered_map'}}
-    }
-    {
-        // cannot deduce Key from just (Size, Hash)
-        std::unordered_map m(42, std::hash<int>());
-            // expected-error-re@-1{{no viable constructor or deduction guide for deduction of template arguments of '{{(std::)?}}unordered_map'}}
-    }
-    {
-        // cannot deduce Key from just (Size, Hash, Pred)
-        std::unordered_map m(42, std::hash<int>(), std::equal_to<int>());
-            // expected-error-re@-1{{no viable constructor or deduction guide for deduction of template arguments of '{{(std::)?}}unordered_map'}}
-    }
-    {
-        // cannot deduce Key from just (Size, Hash, Pred, Allocator)
-        std::unordered_map m(42, std::hash<int>(), std::equal_to<int>(), std::allocator<P>());
-            // expected-error-re@-1{{no viable constructor or deduction guide for deduction of template arguments of '{{(std::)?}}unordered_map'}}
-    }
-    {
-        // cannot deduce Key from just (Allocator)
-        std::unordered_map m(std::allocator<P>{});
-            // expected-error-re@-1{{no viable constructor or deduction guide for deduction of template arguments of '{{(std::)?}}unordered_map'}}
-    }
-    {
-        // cannot deduce Key from just (Size, Allocator)
-        std::unordered_map m(42, std::allocator<P>());
-            // expected-error-re@-1{{no viable constructor or deduction guide for deduction of template arguments of '{{(std::)?}}unordered_map'}}
-    }
-    {
-        // cannot deduce Key from just (Size, Hash, Allocator)
-        std::unordered_map m(42, std::hash<int>(), std::allocator<P>());
-            // expected-error-re@-1{{no viable constructor or deduction guide for deduction of template arguments of '{{(std::)?}}unordered_map'}}
-    }
+int main(int, char**) {
+  using P = std::pair<const int, int>;
+  {
+    // cannot deduce Key from nothing
+    std::unordered_map m;
+    // expected-error-re@-1{{no viable constructor or deduction guide for deduction of template arguments of '{{(std::)?}}unordered_map'}}
+  }
+  {
+    // cannot deduce Key from just (Size)
+    std::unordered_map m(42);
+    // expected-error-re@-1{{no viable constructor or deduction guide for deduction of template arguments of '{{(std::)?}}unordered_map'}}
+  }
+  {
+    // cannot deduce Key from just (Size, Hash)
+    std::unordered_map m(42, std::hash<int>());
+    // expected-error-re@-1{{no viable constructor or deduction guide for deduction of template arguments of '{{(std::)?}}unordered_map'}}
+  }
+  {
+    // cannot deduce Key from just (Size, Hash, Pred)
+    std::unordered_map m(42, std::hash<int>(), std::equal_to<int>());
+    // expected-error-re@-1{{no viable constructor or deduction guide for deduction of template arguments of '{{(std::)?}}unordered_map'}}
+  }
+  {
+    // cannot deduce Key from just (Size, Hash, Pred, Allocator)
+    std::unordered_map m(42, std::hash<int>(), std::equal_to<int>(), std::allocator<P>());
+    // expected-error-re@-1{{no viable constructor or deduction guide for deduction of template arguments of '{{(std::)?}}unordered_map'}}
+  }
+  {
+    // cannot deduce Key from just (Allocator)
+    std::unordered_map m(std::allocator<P>{});
+    // expected-error-re@-1{{no viable constructor or deduction guide for deduction of template arguments of '{{(std::)?}}unordered_map'}}
+  }
+  {
+    // cannot deduce Key from just (Size, Allocator)
+    std::unordered_map m(42, std::allocator<P>());
+    // expected-error-re@-1{{no viable constructor or deduction guide for deduction of template arguments of '{{(std::)?}}unordered_map'}}
+  }
+  {
+    // cannot deduce Key from just (Size, Hash, Allocator)
+    std::unordered_map m(42, std::hash<int>(), std::allocator<P>());
+    // expected-error-re@-1{{no viable constructor or deduction guide for deduction of template arguments of '{{(std::)?}}unordered_map'}}
+  }
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/unord/unord.map/unord.map.cnstr/deduct_const.pass.cpp b/libcxx/test/std/containers/unord/unord.map/unord.map.cnstr/deduct_const.pass.cpp
index 547eef02961f8..75e55256bc25f 100644
--- a/libcxx/test/std/containers/unord/unord.map/unord.map.cnstr/deduct_const.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.map/unord.map.cnstr/deduct_const.pass.cpp
@@ -64,109 +64,123 @@
 
 #include "test_allocator.h"
 
-using P = std::pair<int, long>;
+using P  = std::pair<int, long>;
 using PC = std::pair<const int, long>;
 
-int main(int, char**)
-{
-    const PC expected_m[] = { {1,1L}, {2,2L}, {3,1L}, {INT_MAX,1L} };
+int main(int, char**) {
+  const PC expected_m[] = {{1, 1L}, {2, 2L}, {3, 1L}, {INT_MAX, 1L}};
 
-    {
-    const PC arr[] = { {1,1L}, {2,2L}, {1,1L}, {INT_MAX,1L}, {3,1L} };
+  {
+    const PC arr[] = {{1, 1L}, {2, 2L}, {1, 1L}, {INT_MAX, 1L}, {3, 1L}};
     std::unordered_map m(std::begin(arr), std::end(arr));
     ASSERT_SAME_TYPE(decltype(m), std::unordered_map<int, long>);
     assert(std::is_permutation(m.begin(), m.end(), std::begin(expected_m), std::end(expected_m)));
-    }
+  }
 
-    {
-    const PC arr[] = { {1,1L}, {2,2L}, {1,1L}, {INT_MAX,1L}, {3,1L} };
+  {
+    const PC arr[] = {{1, 1L}, {2, 2L}, {1, 1L}, {INT_MAX, 1L}, {3, 1L}};
     std::unordered_map m(std::begin(arr), std::end(arr), 42);
     ASSERT_SAME_TYPE(decltype(m), std::unordered_map<int, long>);
     assert(std::is_permutation(m.begin(), m.end(), std::begin(expected_m), std::end(expected_m)));
-    }
+  }
 
-    {
-    const PC arr[] = { {1,1L}, {2,2L}, {1,1L}, {INT_MAX,1L}, {3,1L} };
+  {
+    const PC arr[] = {{1, 1L}, {2, 2L}, {1, 1L}, {INT_MAX, 1L}, {3, 1L}};
     std::unordered_map m(std::begin(arr), std::end(arr), 42, std::hash<short>());
     ASSERT_SAME_TYPE(decltype(m), std::unordered_map<int, long, std::hash<short>, std::equal_to<int>>);
     assert(std::is_permutation(m.begin(), m.end(), std::begin(expected_m), std::end(expected_m)));
-    }
+  }
 
-    {
-    const PC arr[] = { {1,1L}, {2,2L}, {1,1L}, {INT_MAX,1L}, {3,1L} };
+  {
+    const PC arr[] = {{1, 1L}, {2, 2L}, {1, 1L}, {INT_MAX, 1L}, {3, 1L}};
     std::unordered_map m(std::begin(arr), std::end(arr), 42, std::hash<short>(), std::equal_to<>());
     ASSERT_SAME_TYPE(decltype(m), std::unordered_map<int, long, std::hash<short>, std::equal_to<>>);
     assert(std::is_permutation(m.begin(), m.end(), std::begin(expected_m), std::end(expected_m)));
-    }
+  }
 
-    {
-    const PC arr[] = { {1,1L}, {2,2L}, {1,1L}, {INT_MAX,1L}, {3,1L} };
-    std::unordered_map m(std::begin(arr), std::end(arr), 42, std::hash<short>(), std::equal_to<>(), test_allocator<PC>(0, 41));
+  {
+    const PC arr[] = {{1, 1L}, {2, 2L}, {1, 1L}, {INT_MAX, 1L}, {3, 1L}};
+    std::unordered_map m(
+        std::begin(arr), std::end(arr), 42, std::hash<short>(), std::equal_to<>(), test_allocator<PC>(0, 41));
     ASSERT_SAME_TYPE(decltype(m), std::unordered_map<int, long, std::hash<short>, std::equal_to<>, test_allocator<PC>>);
     assert(std::is_permutation(m.begin(), m.end(), std::begin(expected_m), std::end(expected_m)));
     assert(m.get_allocator().get_id() == 41);
-    }
+  }
 
-    {
-    std::unordered_map m { PC{1,1L}, PC{2,2L}, PC{1,1L}, PC{INT_MAX,1L}, PC{3,1L} };
+  {
+    std::unordered_map m{PC{1, 1L}, PC{2, 2L}, PC{1, 1L}, PC{INT_MAX, 1L}, PC{3, 1L}};
     ASSERT_SAME_TYPE(decltype(m), std::unordered_map<int, long>);
     assert(std::is_permutation(m.begin(), m.end(), std::begin(expected_m), std::end(expected_m)));
-    }
+  }
 
-    {
-    std::unordered_map m({ PC{1,1L}, PC{2,2L}, PC{1,1L}, PC{INT_MAX,1L}, PC{3,1L} }, 42);
+  {
+    std::unordered_map m({PC{1, 1L}, PC{2, 2L}, PC{1, 1L}, PC{INT_MAX, 1L}, PC{3, 1L}}, 42);
     ASSERT_SAME_TYPE(decltype(m), std::unordered_map<int, long>);
     assert(std::is_permutation(m.begin(), m.end(), std::begin(expected_m), std::end(expected_m)));
-    }
+  }
 
-    {
-    std::unordered_map m({ PC{1,1L}, PC{2,2L}, PC{1,1L}, PC{INT_MAX,1L}, PC{3,1L} }, 42, std::hash<short>());
+  {
+    std::unordered_map m({PC{1, 1L}, PC{2, 2L}, PC{1, 1L}, PC{INT_MAX, 1L}, PC{3, 1L}}, 42, std::hash<short>());
     ASSERT_SAME_TYPE(decltype(m), std::unordered_map<int, long, std::hash<short>>);
     assert(std::is_permutation(m.begin(), m.end(), std::begin(expected_m), std::end(expected_m)));
-    }
+  }
 
-    {
-    std::unordered_map m({ PC{1,1L}, PC{2,2L}, PC{1,1L}, PC{INT_MAX,1L}, PC{3,1L} }, 42, std::hash<short>(), std::equal_to<>());
+  {
+    std::unordered_map m(
+        {PC{1, 1L}, PC{2, 2L}, PC{1, 1L}, PC{INT_MAX, 1L}, PC{3, 1L}}, 42, std::hash<short>(), std::equal_to<>());
     ASSERT_SAME_TYPE(decltype(m), std::unordered_map<int, long, std::hash<short>, std::equal_to<>>);
     assert(std::is_permutation(m.begin(), m.end(), std::begin(expected_m), std::end(expected_m)));
-    }
-
-    {
-    std::unordered_map m({ PC{1,1L}, PC{2,2L}, PC{1,1L}, PC{INT_MAX,1L}, PC{3,1L} }, 42, std::hash<short>(), std::equal_to<>(), test_allocator<PC>(0, 44));
+  }
+
+  {
+    std::unordered_map m(
+        {PC{1, 1L}, PC{2, 2L}, PC{1, 1L}, PC{INT_MAX, 1L}, PC{3, 1L}},
+        42,
+        std::hash<short>(),
+        std::equal_to<>(),
+        test_allocator<PC>(0, 44));
     ASSERT_SAME_TYPE(decltype(m), std::unordered_map<int, long, std::hash<short>, std::equal_to<>, test_allocator<PC>>);
     assert(std::is_permutation(m.begin(), m.end(), std::begin(expected_m), std::end(expected_m)));
     assert(m.get_allocator().get_id() == 44);
-    }
+  }
 
-    {
-    const PC arr[] = { {1,1}, {2,2}, {1,1}, {INT_MAX,1}, {3,1} };
+  {
+    const PC arr[] = {{1, 1}, {2, 2}, {1, 1}, {INT_MAX, 1}, {3, 1}};
     std::unordered_map m(std::begin(arr), std::end(arr), 42, test_allocator<PC>(0, 45));
-    ASSERT_SAME_TYPE(decltype(m), std::unordered_map<int, long, std::hash<int>, std::equal_to<int>, test_allocator<PC>>);
+    ASSERT_SAME_TYPE(
+        decltype(m), std::unordered_map<int, long, std::hash<int>, std::equal_to<int>, test_allocator<PC>>);
     assert(std::is_permutation(m.begin(), m.end(), std::begin(expected_m), std::end(expected_m)));
     assert(m.get_allocator().get_id() == 45);
-    }
+  }
 
-    {
-    const PC arr[] = { {1,1}, {2,2}, {1,1}, {INT_MAX,1}, {3,1} };
+  {
+    const PC arr[] = {{1, 1}, {2, 2}, {1, 1}, {INT_MAX, 1}, {3, 1}};
     std::unordered_map m(std::begin(arr), std::end(arr), 42, std::hash<short>(), test_allocator<PC>(0, 46));
-    ASSERT_SAME_TYPE(decltype(m), std::unordered_map<int, long, std::hash<short>, std::equal_to<int>, test_allocator<PC>>);
+    ASSERT_SAME_TYPE(
+        decltype(m), std::unordered_map<int, long, std::hash<short>, std::equal_to<int>, test_allocator<PC>>);
     assert(std::is_permutation(m.begin(), m.end(), std::begin(expected_m), std::end(expected_m)));
     assert(m.get_allocator().get_id() == 46);
-    }
+  }
 
-    {
-    std::unordered_map m({ PC{1,1L}, PC{2,2L}, PC{1,1L}, PC{INT_MAX,1L}, PC{3,1L} }, 42, test_allocator<PC>(0, 47));
-    ASSERT_SAME_TYPE(decltype(m), std::unordered_map<int, long, std::hash<int>, std::equal_to<int>, test_allocator<PC>>);
+  {
+    std::unordered_map m({PC{1, 1L}, PC{2, 2L}, PC{1, 1L}, PC{INT_MAX, 1L}, PC{3, 1L}}, 42, test_allocator<PC>(0, 47));
+    ASSERT_SAME_TYPE(
+        decltype(m), std::unordered_map<int, long, std::hash<int>, std::equal_to<int>, test_allocator<PC>>);
     assert(std::is_permutation(m.begin(), m.end(), std::begin(expected_m), std::end(expected_m)));
     assert(m.get_allocator().get_id() == 47);
-    }
-
-    {
-    std::unordered_map m({ PC{1,1L}, PC{2,2L}, PC{1,1L}, PC{INT_MAX,1L}, PC{3,1L} }, 42, std::hash<short>(), test_allocator<PC>(0, 48));
-    ASSERT_SAME_TYPE(decltype(m), std::unordered_map<int, long, std::hash<short>, std::equal_to<int>, test_allocator<PC>>);
+  }
+
+  {
+    std::unordered_map m(
+        {PC{1, 1L}, PC{2, 2L}, PC{1, 1L}, PC{INT_MAX, 1L}, PC{3, 1L}},
+        42,
+        std::hash<short>(),
+        test_allocator<PC>(0, 48));
+    ASSERT_SAME_TYPE(
+        decltype(m), std::unordered_map<int, long, std::hash<short>, std::equal_to<int>, test_allocator<PC>>);
     assert(std::is_permutation(m.begin(), m.end(), std::begin(expected_m), std::end(expected_m)));
     assert(m.get_allocator().get_id() == 48);
-    }
+  }
 
-    return 0;
+  return 0;
 }
diff --git a/libcxx/test/std/containers/unord/unord.map/unord.map.cnstr/default.pass.cpp b/libcxx/test/std/containers/unord/unord.map/unord.map.cnstr/default.pass.cpp
index a4bdb52be307e..7ce5cc401347d 100644
--- a/libcxx/test/std/containers/unord/unord.map/unord.map.cnstr/default.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.map/unord.map.cnstr/default.pass.cpp
@@ -24,89 +24,87 @@
 #include "test_allocator.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef std::unordered_map<NotConstructible, NotConstructible,
-                                   test_hash<NotConstructible>,
-                                   test_equal_to<NotConstructible>,
-                                   test_allocator<std::pair<const NotConstructible,
-                                                                  NotConstructible> >
-                                   > C;
-        C c;
-        LIBCPP_ASSERT(c.bucket_count() == 0);
-        assert(c.hash_function() == test_hash<NotConstructible>());
-        assert(c.key_eq() == test_equal_to<NotConstructible>());
-        assert(c.get_allocator() ==
-               (test_allocator<std::pair<const NotConstructible, NotConstructible> >()));
-        assert(c.size() == 0);
-        assert(c.empty());
-        assert(std::distance(c.begin(), c.end()) == 0);
-        assert(c.load_factor() == 0);
-        assert(c.max_load_factor() == 1);
-    }
+int main(int, char**) {
+  {
+    typedef std::unordered_map<NotConstructible,
+                               NotConstructible,
+                               test_hash<NotConstructible>,
+                               test_equal_to<NotConstructible>,
+                               test_allocator<std::pair<const NotConstructible, NotConstructible> > >
+        C;
+    C c;
+    LIBCPP_ASSERT(c.bucket_count() == 0);
+    assert(c.hash_function() == test_hash<NotConstructible>());
+    assert(c.key_eq() == test_equal_to<NotConstructible>());
+    assert(c.get_allocator() == (test_allocator<std::pair<const NotConstructible, NotConstructible> >()));
+    assert(c.size() == 0);
+    assert(c.empty());
+    assert(std::distance(c.begin(), c.end()) == 0);
+    assert(c.load_factor() == 0);
+    assert(c.max_load_factor() == 1);
+  }
 #if TEST_STD_VER >= 11
+  {
+    typedef std::unordered_map<NotConstructible,
+                               NotConstructible,
+                               test_hash<NotConstructible>,
+                               test_equal_to<NotConstructible>,
+                               min_allocator<std::pair<const NotConstructible, NotConstructible> > >
+        C;
+    C c;
+    LIBCPP_ASSERT(c.bucket_count() == 0);
+    assert(c.hash_function() == test_hash<NotConstructible>());
+    assert(c.key_eq() == test_equal_to<NotConstructible>());
+    assert(c.get_allocator() == (min_allocator<std::pair<const NotConstructible, NotConstructible> >()));
+    assert(c.size() == 0);
+    assert(c.empty());
+    assert(std::distance(c.begin(), c.end()) == 0);
+    assert(c.load_factor() == 0);
+    assert(c.max_load_factor() == 1);
+  }
+  {
+    typedef explicit_allocator<std::pair<const NotConstructible, NotConstructible>> A;
+    typedef std::unordered_map<NotConstructible,
+                               NotConstructible,
+                               test_hash<NotConstructible>,
+                               test_equal_to<NotConstructible>,
+                               A >
+        C;
     {
-        typedef std::unordered_map<NotConstructible, NotConstructible,
-                                   test_hash<NotConstructible>,
-                                   test_equal_to<NotConstructible>,
-                                   min_allocator<std::pair<const NotConstructible,
-                                                                  NotConstructible> >
-                                   > C;
-        C c;
-        LIBCPP_ASSERT(c.bucket_count() == 0);
-        assert(c.hash_function() == test_hash<NotConstructible>());
-        assert(c.key_eq() == test_equal_to<NotConstructible>());
-        assert(c.get_allocator() ==
-               (min_allocator<std::pair<const NotConstructible, NotConstructible> >()));
-        assert(c.size() == 0);
-        assert(c.empty());
-        assert(std::distance(c.begin(), c.end()) == 0);
-        assert(c.load_factor() == 0);
-        assert(c.max_load_factor() == 1);
-    }
-    {
-        typedef explicit_allocator<std::pair<const NotConstructible, NotConstructible>> A;
-        typedef std::unordered_map<NotConstructible, NotConstructible,
-                                   test_hash<NotConstructible>,
-                                   test_equal_to<NotConstructible>,
-                                   A
-                                   > C;
-        {
-        C c;
-        LIBCPP_ASSERT(c.bucket_count() == 0);
-        assert(c.hash_function() == test_hash<NotConstructible>());
-        assert(c.key_eq() == test_equal_to<NotConstructible>());
-        assert(c.get_allocator() == A());
-        assert(c.size() == 0);
-        assert(c.empty());
-        assert(std::distance(c.begin(), c.end()) == 0);
-        assert(c.load_factor() == 0);
-        assert(c.max_load_factor() == 1);
-        }
-        {
-        A a;
-        C c(a);
-        LIBCPP_ASSERT(c.bucket_count() == 0);
-        assert(c.hash_function() == test_hash<NotConstructible>());
-        assert(c.key_eq() == test_equal_to<NotConstructible>());
-        assert(c.get_allocator() == a);
-        assert(c.size() == 0);
-        assert(c.empty());
-        assert(std::distance(c.begin(), c.end()) == 0);
-        assert(c.load_factor() == 0);
-        assert(c.max_load_factor() == 1);
-        }
+      C c;
+      LIBCPP_ASSERT(c.bucket_count() == 0);
+      assert(c.hash_function() == test_hash<NotConstructible>());
+      assert(c.key_eq() == test_equal_to<NotConstructible>());
+      assert(c.get_allocator() == A());
+      assert(c.size() == 0);
+      assert(c.empty());
+      assert(std::distance(c.begin(), c.end()) == 0);
+      assert(c.load_factor() == 0);
+      assert(c.max_load_factor() == 1);
     }
     {
-        std::unordered_map<int, int> c = {};
-        LIBCPP_ASSERT(c.bucket_count() == 0);
-        assert(c.size() == 0);
-        assert(c.empty());
-        assert(std::distance(c.begin(), c.end()) == 0);
-        assert(c.load_factor() == 0);
-        assert(c.max_load_factor() == 1);
+      A a;
+      C c(a);
+      LIBCPP_ASSERT(c.bucket_count() == 0);
+      assert(c.hash_function() == test_hash<NotConstructible>());
+      assert(c.key_eq() == test_equal_to<NotConstructible>());
+      assert(c.get_allocator() == a);
+      assert(c.size() == 0);
+      assert(c.empty());
+      assert(std::distance(c.begin(), c.end()) == 0);
+      assert(c.load_factor() == 0);
+      assert(c.max_load_factor() == 1);
     }
+  }
+  {
+    std::unordered_map<int, int> c = {};
+    LIBCPP_ASSERT(c.bucket_count() == 0);
+    assert(c.size() == 0);
+    assert(c.empty());
+    assert(std::distance(c.begin(), c.end()) == 0);
+    assert(c.load_factor() == 0);
+    assert(c.max_load_factor() == 1);
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/unord/unord.map/unord.map.cnstr/default_noexcept.pass.cpp b/libcxx/test/std/containers/unord/unord.map/unord.map.cnstr/default_noexcept.pass.cpp
index 2fff7fdbac6e4..f41d18a6c622c 100644
--- a/libcxx/test/std/containers/unord/unord.map/unord.map.cnstr/default_noexcept.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.map/unord.map.cnstr/default_noexcept.pass.cpp
@@ -27,51 +27,55 @@
 #include "../../../test_hash.h"
 
 template <class T>
-struct some_comp
-{
-    typedef T value_type;
-    some_comp();
-    some_comp(const some_comp&);
-    bool operator()(const T&, const T&) const { return false; }
+struct some_comp {
+  typedef T value_type;
+  some_comp();
+  some_comp(const some_comp&);
+  bool operator()(const T&, const T&) const { return false; }
 };
 
 template <class T>
-struct some_hash
-{
-    typedef T value_type;
-    some_hash();
-    some_hash(const some_hash&);
+struct some_hash {
+  typedef T value_type;
+  some_hash();
+  some_hash(const some_hash&);
 
-    std::size_t operator()(T const&) const;
+  std::size_t operator()(T const&) const;
 };
 
-int main(int, char**)
-{
+int main(int, char**) {
 #if defined(_LIBCPP_VERSION)
-    {
-        typedef std::unordered_map<MoveOnly, MoveOnly> C;
-        static_assert(std::is_nothrow_default_constructible<C>::value, "");
-    }
-    {
-        typedef std::unordered_map<MoveOnly, MoveOnly, std::hash<MoveOnly>,
-                           std::equal_to<MoveOnly>, test_allocator<std::pair<const MoveOnly, MoveOnly>>> C;
-        static_assert(std::is_nothrow_default_constructible<C>::value, "");
-    }
+  {
+    typedef std::unordered_map<MoveOnly, MoveOnly> C;
+    static_assert(std::is_nothrow_default_constructible<C>::value, "");
+  }
+  {
+    typedef std::unordered_map<MoveOnly,
+                               MoveOnly,
+                               std::hash<MoveOnly>,
+                               std::equal_to<MoveOnly>,
+                               test_allocator<std::pair<const MoveOnly, MoveOnly>>>
+        C;
+    static_assert(std::is_nothrow_default_constructible<C>::value, "");
+  }
 #endif // _LIBCPP_VERSION
-    {
-        typedef std::unordered_map<MoveOnly, MoveOnly, std::hash<MoveOnly>,
-                          std::equal_to<MoveOnly>, other_allocator<std::pair<const MoveOnly, MoveOnly>>> C;
-        static_assert(!std::is_nothrow_default_constructible<C>::value, "");
-    }
-    {
-        typedef std::unordered_map<MoveOnly, MoveOnly, some_hash<MoveOnly>> C;
-        static_assert(!std::is_nothrow_default_constructible<C>::value, "");
-    }
-    {
-        typedef std::unordered_map<MoveOnly, MoveOnly, std::hash<MoveOnly>,
-                                                         some_comp<MoveOnly>> C;
-        static_assert(!std::is_nothrow_default_constructible<C>::value, "");
-    }
+  {
+    typedef std::unordered_map<MoveOnly,
+                               MoveOnly,
+                               std::hash<MoveOnly>,
+                               std::equal_to<MoveOnly>,
+                               other_allocator<std::pair<const MoveOnly, MoveOnly>>>
+        C;
+    static_assert(!std::is_nothrow_default_constructible<C>::value, "");
+  }
+  {
+    typedef std::unordered_map<MoveOnly, MoveOnly, some_hash<MoveOnly>> C;
+    static_assert(!std::is_nothrow_default_constructible<C>::value, "");
+  }
+  {
+    typedef std::unordered_map<MoveOnly, MoveOnly, std::hash<MoveOnly>, some_comp<MoveOnly>> C;
+    static_assert(!std::is_nothrow_default_constructible<C>::value, "");
+  }
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/unord/unord.map/unord.map.cnstr/dtor_noexcept.pass.cpp b/libcxx/test/std/containers/unord/unord.map/unord.map.cnstr/dtor_noexcept.pass.cpp
index 0775b2842a73e..5797599b1f9e4 100644
--- a/libcxx/test/std/containers/unord/unord.map/unord.map.cnstr/dtor_noexcept.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.map/unord.map.cnstr/dtor_noexcept.pass.cpp
@@ -20,50 +20,54 @@
 #include "test_allocator.h"
 
 template <class T>
-struct some_comp
-{
-    typedef T value_type;
-    ~some_comp() noexcept(false);
-    bool operator()(const T&, const T&) const { return false; }
+struct some_comp {
+  typedef T value_type;
+  ~some_comp() noexcept(false);
+  bool operator()(const T&, const T&) const { return false; }
 };
 
 template <class T>
-struct some_hash
-{
-    typedef T value_type;
-    some_hash();
-    some_hash(const some_hash&);
-    ~some_hash() noexcept(false);
+struct some_hash {
+  typedef T value_type;
+  some_hash();
+  some_hash(const some_hash&);
+  ~some_hash() noexcept(false);
 
-    std::size_t operator()(T const&) const;
+  std::size_t operator()(T const&) const;
 };
 
-int main(int, char**)
-{
-    {
-        typedef std::unordered_map<MoveOnly, MoveOnly> C;
-        static_assert(std::is_nothrow_destructible<C>::value, "");
-    }
-    {
-        typedef std::unordered_map<MoveOnly, MoveOnly, std::hash<MoveOnly>,
-                           std::equal_to<MoveOnly>, test_allocator<std::pair<const MoveOnly, MoveOnly>>> C;
-        static_assert(std::is_nothrow_destructible<C>::value, "");
-    }
-    {
-        typedef std::unordered_map<MoveOnly, MoveOnly, std::hash<MoveOnly>,
-                          std::equal_to<MoveOnly>, other_allocator<std::pair<const MoveOnly, MoveOnly>>> C;
-        static_assert(std::is_nothrow_destructible<C>::value, "");
-    }
+int main(int, char**) {
+  {
+    typedef std::unordered_map<MoveOnly, MoveOnly> C;
+    static_assert(std::is_nothrow_destructible<C>::value, "");
+  }
+  {
+    typedef std::unordered_map<MoveOnly,
+                               MoveOnly,
+                               std::hash<MoveOnly>,
+                               std::equal_to<MoveOnly>,
+                               test_allocator<std::pair<const MoveOnly, MoveOnly>>>
+        C;
+    static_assert(std::is_nothrow_destructible<C>::value, "");
+  }
+  {
+    typedef std::unordered_map<MoveOnly,
+                               MoveOnly,
+                               std::hash<MoveOnly>,
+                               std::equal_to<MoveOnly>,
+                               other_allocator<std::pair<const MoveOnly, MoveOnly>>>
+        C;
+    static_assert(std::is_nothrow_destructible<C>::value, "");
+  }
 #if defined(_LIBCPP_VERSION)
-    {
-        typedef std::unordered_map<MoveOnly, MoveOnly, some_hash<MoveOnly>> C;
-        static_assert(!std::is_nothrow_destructible<C>::value, "");
-    }
-    {
-        typedef std::unordered_map<MoveOnly, MoveOnly, std::hash<MoveOnly>,
-                                                         some_comp<MoveOnly>> C;
-        static_assert(!std::is_nothrow_destructible<C>::value, "");
-    }
+  {
+    typedef std::unordered_map<MoveOnly, MoveOnly, some_hash<MoveOnly>> C;
+    static_assert(!std::is_nothrow_destructible<C>::value, "");
+  }
+  {
+    typedef std::unordered_map<MoveOnly, MoveOnly, std::hash<MoveOnly>, some_comp<MoveOnly>> C;
+    static_assert(!std::is_nothrow_destructible<C>::value, "");
+  }
 #endif // _LIBCPP_VERSION
 
   return 0;
diff --git a/libcxx/test/std/containers/unord/unord.map/unord.map.cnstr/from_range.pass.cpp b/libcxx/test/std/containers/unord/unord.map/unord.map.cnstr/from_range.pass.cpp
index f48057c12f31c..f38990a14626e 100644
--- a/libcxx/test/std/containers/unord/unord.map/unord.map.cnstr/from_range.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.map/unord.map.cnstr/from_range.pass.cpp
@@ -30,13 +30,9 @@
 void test_duplicates() {
   using T = std::pair<const int, char>;
 
-  std::array input = {
-    T{1, 'a'}, T{2, 'a'}, T{3, 'a'}, T{3, 'b'}, T{3, 'c'}, T{2, 'b'}, T{4, 'a'}
-  };
-  std::array expected = {
-    T{1, 'a'}, T{2, 'a'}, T{3, 'a'}, T{4, 'a'}
-  };
-  auto c = std::unordered_map<int, char>(std::from_range, input);
+  std::array input    = {T{1, 'a'}, T{2, 'a'}, T{3, 'a'}, T{3, 'b'}, T{3, 'c'}, T{2, 'b'}, T{4, 'a'}};
+  std::array expected = {T{1, 'a'}, T{2, 'a'}, T{3, 'a'}, T{4, 'a'}};
+  auto c              = std::unordered_map<int, char>(std::from_range, input);
   assert(std::ranges::is_permutation(expected, c));
 }
 
diff --git a/libcxx/test/std/containers/unord/unord.map/unord.map.cnstr/hash_copy_constructible.compile.fail.cpp b/libcxx/test/std/containers/unord/unord.map/unord.map.cnstr/hash_copy_constructible.compile.fail.cpp
index 589314a13cdcd..b5db5a477f42d 100644
--- a/libcxx/test/std/containers/unord/unord.map/unord.map.cnstr/hash_copy_constructible.compile.fail.cpp
+++ b/libcxx/test/std/containers/unord/unord.map/unord.map.cnstr/hash_copy_constructible.compile.fail.cpp
@@ -18,16 +18,16 @@
 
 template <class T>
 struct Hash {
-    std::size_t operator () (const T& lhs) const { return 0; }
+  std::size_t operator()(const T& lhs) const { return 0; }
+
+  Hash() {}
 
-    Hash () {}
 private:
-    Hash (const Hash &); // declared but not defined
+  Hash(const Hash&); // declared but not defined
 };
 
-
 int main(int, char**) {
-    std::unordered_map<int, int, Hash<int> > m;
+  std::unordered_map<int, int, Hash<int> > m;
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/unord/unord.map/unord.map.cnstr/init.pass.cpp b/libcxx/test/std/containers/unord/unord.map/unord.map.cnstr/init.pass.cpp
index f20ba84d7799a..203248404c235 100644
--- a/libcxx/test/std/containers/unord/unord.map/unord.map.cnstr/init.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.map/unord.map.cnstr/init.pass.cpp
@@ -29,135 +29,141 @@
 #include "test_allocator.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef std::unordered_map<int, std::string,
-                                   test_hash<int>,
-                                   test_equal_to<int>,
-                                   test_allocator<std::pair<const int, std::string> >
-                                   > C;
-        typedef std::pair<int, std::string> P;
-        C c = {
-                P(1, "one"),
-                P(2, "two"),
-                P(3, "three"),
-                P(4, "four"),
-                P(1, "four"),
-                P(2, "four"),
-              };
-        assert(c.bucket_count() >= 5);
-        assert(c.size() == 4);
-        assert(c.at(1) == "one");
-        assert(c.at(2) == "two");
-        assert(c.at(3) == "three");
-        assert(c.at(4) == "four");
-        assert(c.hash_function() == test_hash<int>());
-        assert(c.key_eq() == test_equal_to<int>());
-        assert(c.get_allocator() ==
-               (test_allocator<std::pair<const int, std::string> >()));
-        assert(!c.empty());
-        assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-        assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
-        assert(fabs(c.load_factor() - (float)c.size()/c.bucket_count()) < FLT_EPSILON);
-        assert(c.max_load_factor() == 1);
-    }
-    {
-        typedef std::unordered_map<int, std::string,
-                                   test_hash<int>,
-                                   test_equal_to<int>,
-                                   min_allocator<std::pair<const int, std::string> >
-                                   > C;
-        typedef std::pair<int, std::string> P;
-        C c = {
-                P(1, "one"),
-                P(2, "two"),
-                P(3, "three"),
-                P(4, "four"),
-                P(1, "four"),
-                P(2, "four"),
-              };
-        assert(c.bucket_count() >= 5);
-        assert(c.size() == 4);
-        assert(c.at(1) == "one");
-        assert(c.at(2) == "two");
-        assert(c.at(3) == "three");
-        assert(c.at(4) == "four");
-        assert(c.hash_function() == test_hash<int>());
-        assert(c.key_eq() == test_equal_to<int>());
-        assert(c.get_allocator() ==
-               (min_allocator<std::pair<const int, std::string> >()));
-        assert(!c.empty());
-        assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-        assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
-        assert(fabs(c.load_factor() - (float)c.size()/c.bucket_count()) < FLT_EPSILON);
-        assert(c.max_load_factor() == 1);
-    }
+int main(int, char**) {
+  {
+    typedef std::unordered_map<int,
+                               std::string,
+                               test_hash<int>,
+                               test_equal_to<int>,
+                               test_allocator<std::pair<const int, std::string> > >
+        C;
+    typedef std::pair<int, std::string> P;
+    C c = {
+        P(1, "one"),
+        P(2, "two"),
+        P(3, "three"),
+        P(4, "four"),
+        P(1, "four"),
+        P(2, "four"),
+    };
+    assert(c.bucket_count() >= 5);
+    assert(c.size() == 4);
+    assert(c.at(1) == "one");
+    assert(c.at(2) == "two");
+    assert(c.at(3) == "three");
+    assert(c.at(4) == "four");
+    assert(c.hash_function() == test_hash<int>());
+    assert(c.key_eq() == test_equal_to<int>());
+    assert(c.get_allocator() == (test_allocator<std::pair<const int, std::string> >()));
+    assert(!c.empty());
+    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
+    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
+    assert(fabs(c.load_factor() - (float)c.size() / c.bucket_count()) < FLT_EPSILON);
+    assert(c.max_load_factor() == 1);
+  }
+  {
+    typedef std::unordered_map<int,
+                               std::string,
+                               test_hash<int>,
+                               test_equal_to<int>,
+                               min_allocator<std::pair<const int, std::string> > >
+        C;
+    typedef std::pair<int, std::string> P;
+    C c = {
+        P(1, "one"),
+        P(2, "two"),
+        P(3, "three"),
+        P(4, "four"),
+        P(1, "four"),
+        P(2, "four"),
+    };
+    assert(c.bucket_count() >= 5);
+    assert(c.size() == 4);
+    assert(c.at(1) == "one");
+    assert(c.at(2) == "two");
+    assert(c.at(3) == "three");
+    assert(c.at(4) == "four");
+    assert(c.hash_function() == test_hash<int>());
+    assert(c.key_eq() == test_equal_to<int>());
+    assert(c.get_allocator() == (min_allocator<std::pair<const int, std::string> >()));
+    assert(!c.empty());
+    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
+    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
+    assert(fabs(c.load_factor() - (float)c.size() / c.bucket_count()) < FLT_EPSILON);
+    assert(c.max_load_factor() == 1);
+  }
 #if TEST_STD_VER > 11
-    {
-        typedef std::pair<int, std::string> P;
-        typedef test_allocator<std::pair<const int, std::string>> A;
-        typedef test_hash<int> HF;
-        typedef test_equal_to<int> Comp;
-        typedef std::unordered_map<int, std::string, HF, Comp, A> C;
+  {
+    typedef std::pair<int, std::string> P;
+    typedef test_allocator<std::pair<const int, std::string>> A;
+    typedef test_hash<int> HF;
+    typedef test_equal_to<int> Comp;
+    typedef std::unordered_map<int, std::string, HF, Comp, A> C;
 
-        A a(42);
-        C c ( {
-                P(1, "one"),
-                P(2, "two"),
-                P(3, "three"),
-                P(4, "four"),
-                P(1, "four"),
-                P(2, "four"),
-              }, 12, a);
-        assert(c.bucket_count() >= 12);
-        assert(c.size() == 4);
-        assert(c.at(1) == "one");
-        assert(c.at(2) == "two");
-        assert(c.at(3) == "three");
-        assert(c.at(4) == "four");
-        assert(c.hash_function() == test_hash<int>());
-        assert(c.key_eq() == test_equal_to<int>());
-        assert(c.get_allocator() == a);
-        assert(!c.empty());
-        assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-        assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
-        assert(fabs(c.load_factor() - (float)c.size()/c.bucket_count()) < FLT_EPSILON);
-        assert(c.max_load_factor() == 1);
-    }
-    {
-        typedef std::pair<int, std::string> P;
-        typedef test_allocator<std::pair<const int, std::string>> A;
-        typedef test_hash<int> HF;
-        typedef test_equal_to<int> Comp;
-        typedef std::unordered_map<int, std::string, HF, Comp, A> C;
+    A a(42);
+    C c(
+        {
+            P(1, "one"),
+            P(2, "two"),
+            P(3, "three"),
+            P(4, "four"),
+            P(1, "four"),
+            P(2, "four"),
+        },
+        12,
+        a);
+    assert(c.bucket_count() >= 12);
+    assert(c.size() == 4);
+    assert(c.at(1) == "one");
+    assert(c.at(2) == "two");
+    assert(c.at(3) == "three");
+    assert(c.at(4) == "four");
+    assert(c.hash_function() == test_hash<int>());
+    assert(c.key_eq() == test_equal_to<int>());
+    assert(c.get_allocator() == a);
+    assert(!c.empty());
+    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
+    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
+    assert(fabs(c.load_factor() - (float)c.size() / c.bucket_count()) < FLT_EPSILON);
+    assert(c.max_load_factor() == 1);
+  }
+  {
+    typedef std::pair<int, std::string> P;
+    typedef test_allocator<std::pair<const int, std::string>> A;
+    typedef test_hash<int> HF;
+    typedef test_equal_to<int> Comp;
+    typedef std::unordered_map<int, std::string, HF, Comp, A> C;
 
-        HF hf(42);
-        A a(43);
-        C c ( {
-                P(1, "one"),
-                P(2, "two"),
-                P(3, "three"),
-                P(4, "four"),
-                P(1, "four"),
-                P(2, "four"),
-              }, 12, hf, a);
-        assert(c.bucket_count() >= 12);
-        assert(c.size() == 4);
-        assert(c.at(1) == "one");
-        assert(c.at(2) == "two");
-        assert(c.at(3) == "three");
-        assert(c.at(4) == "four");
-        assert(c.hash_function() == hf);
-        assert(!(c.hash_function() == test_hash<int>()));
-        assert(c.key_eq() == test_equal_to<int>());
-        assert(c.get_allocator() == a);
-        assert(!c.empty());
-        assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-        assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
-        assert(fabs(c.load_factor() - (float)c.size()/c.bucket_count()) < FLT_EPSILON);
-        assert(c.max_load_factor() == 1);
-    }
+    HF hf(42);
+    A a(43);
+    C c(
+        {
+            P(1, "one"),
+            P(2, "two"),
+            P(3, "three"),
+            P(4, "four"),
+            P(1, "four"),
+            P(2, "four"),
+        },
+        12,
+        hf,
+        a);
+    assert(c.bucket_count() >= 12);
+    assert(c.size() == 4);
+    assert(c.at(1) == "one");
+    assert(c.at(2) == "two");
+    assert(c.at(3) == "three");
+    assert(c.at(4) == "four");
+    assert(c.hash_function() == hf);
+    assert(!(c.hash_function() == test_hash<int>()));
+    assert(c.key_eq() == test_equal_to<int>());
+    assert(c.get_allocator() == a);
+    assert(!c.empty());
+    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
+    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
+    assert(fabs(c.load_factor() - (float)c.size() / c.bucket_count()) < FLT_EPSILON);
+    assert(c.max_load_factor() == 1);
+  }
 #endif // TEST_STD_VER > 11
 
   return 0;
diff --git a/libcxx/test/std/containers/unord/unord.map/unord.map.cnstr/init_size.pass.cpp b/libcxx/test/std/containers/unord/unord.map/unord.map.cnstr/init_size.pass.cpp
index dc26c81c369cc..73c706a5b281c 100644
--- a/libcxx/test/std/containers/unord/unord.map/unord.map.cnstr/init_size.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.map/unord.map.cnstr/init_size.pass.cpp
@@ -29,74 +29,73 @@
 #include "test_allocator.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef std::unordered_map<int, std::string,
-                                   test_hash<int>,
-                                   test_equal_to<int>,
-                                   test_allocator<std::pair<const int, std::string> >
-                                   > C;
-        typedef std::pair<int, std::string> P;
-        C c({
-                P(1, "one"),
-                P(2, "two"),
-                P(3, "three"),
-                P(4, "four"),
-                P(1, "four"),
-                P(2, "four"),
-            },
-            7
-           );
-        LIBCPP_ASSERT(c.bucket_count() == 7);
-        assert(c.size() == 4);
-        assert(c.at(1) == "one");
-        assert(c.at(2) == "two");
-        assert(c.at(3) == "three");
-        assert(c.at(4) == "four");
-        assert(c.hash_function() == test_hash<int>());
-        assert(c.key_eq() == test_equal_to<int>());
-        assert(c.get_allocator() ==
-               (test_allocator<std::pair<const int, std::string> >()));
-        assert(!c.empty());
-        assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-        assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
-        assert(std::fabs(c.load_factor() - (float)c.size()/c.bucket_count()) < FLT_EPSILON);
-        assert(c.max_load_factor() == 1);
-    }
-    {
-        typedef std::unordered_map<int, std::string,
-                                   test_hash<int>,
-                                   test_equal_to<int>,
-                                   min_allocator<std::pair<const int, std::string> >
-                                   > C;
-        typedef std::pair<int, std::string> P;
-        C c({
-                P(1, "one"),
-                P(2, "two"),
-                P(3, "three"),
-                P(4, "four"),
-                P(1, "four"),
-                P(2, "four"),
-            },
-            7
-           );
-        LIBCPP_ASSERT(c.bucket_count() == 7);
-        assert(c.size() == 4);
-        assert(c.at(1) == "one");
-        assert(c.at(2) == "two");
-        assert(c.at(3) == "three");
-        assert(c.at(4) == "four");
-        assert(c.hash_function() == test_hash<int>());
-        assert(c.key_eq() == test_equal_to<int>());
-        assert(c.get_allocator() ==
-               (min_allocator<std::pair<const int, std::string> >()));
-        assert(!c.empty());
-        assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-        assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
-        assert(std::fabs(c.load_factor() - (float)c.size()/c.bucket_count()) < FLT_EPSILON);
-        assert(c.max_load_factor() == 1);
-    }
+int main(int, char**) {
+  {
+    typedef std::unordered_map<int,
+                               std::string,
+                               test_hash<int>,
+                               test_equal_to<int>,
+                               test_allocator<std::pair<const int, std::string> > >
+        C;
+    typedef std::pair<int, std::string> P;
+    C c(
+        {
+            P(1, "one"),
+            P(2, "two"),
+            P(3, "three"),
+            P(4, "four"),
+            P(1, "four"),
+            P(2, "four"),
+        },
+        7);
+    LIBCPP_ASSERT(c.bucket_count() == 7);
+    assert(c.size() == 4);
+    assert(c.at(1) == "one");
+    assert(c.at(2) == "two");
+    assert(c.at(3) == "three");
+    assert(c.at(4) == "four");
+    assert(c.hash_function() == test_hash<int>());
+    assert(c.key_eq() == test_equal_to<int>());
+    assert(c.get_allocator() == (test_allocator<std::pair<const int, std::string> >()));
+    assert(!c.empty());
+    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
+    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
+    assert(std::fabs(c.load_factor() - (float)c.size() / c.bucket_count()) < FLT_EPSILON);
+    assert(c.max_load_factor() == 1);
+  }
+  {
+    typedef std::unordered_map<int,
+                               std::string,
+                               test_hash<int>,
+                               test_equal_to<int>,
+                               min_allocator<std::pair<const int, std::string> > >
+        C;
+    typedef std::pair<int, std::string> P;
+    C c(
+        {
+            P(1, "one"),
+            P(2, "two"),
+            P(3, "three"),
+            P(4, "four"),
+            P(1, "four"),
+            P(2, "four"),
+        },
+        7);
+    LIBCPP_ASSERT(c.bucket_count() == 7);
+    assert(c.size() == 4);
+    assert(c.at(1) == "one");
+    assert(c.at(2) == "two");
+    assert(c.at(3) == "three");
+    assert(c.at(4) == "four");
+    assert(c.hash_function() == test_hash<int>());
+    assert(c.key_eq() == test_equal_to<int>());
+    assert(c.get_allocator() == (min_allocator<std::pair<const int, std::string> >()));
+    assert(!c.empty());
+    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
+    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
+    assert(std::fabs(c.load_factor() - (float)c.size() / c.bucket_count()) < FLT_EPSILON);
+    assert(c.max_load_factor() == 1);
+  }
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/unord/unord.map/unord.map.cnstr/init_size_allocator.pass.cpp b/libcxx/test/std/containers/unord/unord.map/unord.map.cnstr/init_size_allocator.pass.cpp
index 7e25ba298d21c..e29ecea327971 100644
--- a/libcxx/test/std/containers/unord/unord.map/unord.map.cnstr/init_size_allocator.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.map/unord.map.cnstr/init_size_allocator.pass.cpp
@@ -32,45 +32,41 @@
 
 template <class Allocator>
 void test(const Allocator& alloc) {
-    typedef std::unordered_map<int, std::string,
-                               test_hash<int>,
-                               test_equal_to<int>,
-                               Allocator
-                               > C;
-    typedef std::pair<int, std::string> P;
+  typedef std::unordered_map<int, std::string, test_hash<int>, test_equal_to<int>, Allocator > C;
+  typedef std::pair<int, std::string> P;
 
-    C c({
-            P(1, "one"),
-            P(2, "two"),
-            P(3, "three"),
-            P(4, "four"),
-            P(1, "four"),
-            P(2, "four"),
-         },
-         7,
-         alloc);
-    LIBCPP_ASSERT(c.bucket_count() == 7);
-    assert(c.size() == 4);
-    assert(c.at(1) == "one");
-    assert(c.at(2) == "two");
-    assert(c.at(3) == "three");
-    assert(c.at(4) == "four");
-    assert(c.hash_function() == test_hash<int>());
-    assert(c.key_eq() == test_equal_to<int>());
-    assert(c.get_allocator() == alloc);
-    assert(!c.empty());
-    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-    assert(std::fabs(c.load_factor() - (float)c.size() / c.bucket_count()) < FLT_EPSILON);
-    assert(c.max_load_factor() == 1);
+  C c(
+      {
+          P(1, "one"),
+          P(2, "two"),
+          P(3, "three"),
+          P(4, "four"),
+          P(1, "four"),
+          P(2, "four"),
+      },
+      7,
+      alloc);
+  LIBCPP_ASSERT(c.bucket_count() == 7);
+  assert(c.size() == 4);
+  assert(c.at(1) == "one");
+  assert(c.at(2) == "two");
+  assert(c.at(3) == "three");
+  assert(c.at(4) == "four");
+  assert(c.hash_function() == test_hash<int>());
+  assert(c.key_eq() == test_equal_to<int>());
+  assert(c.get_allocator() == alloc);
+  assert(!c.empty());
+  assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
+  assert(std::fabs(c.load_factor() - (float)c.size() / c.bucket_count()) < FLT_EPSILON);
+  assert(c.max_load_factor() == 1);
 }
 
-int main(int, char**)
-{
-    typedef std::pair<const int, std::string> P;
+int main(int, char**) {
+  typedef std::pair<const int, std::string> P;
 
-    test(test_allocator<P>(10));
-    test(min_allocator<P>());
-    test(explicit_allocator<P>());
+  test(test_allocator<P>(10));
+  test(min_allocator<P>());
+  test(explicit_allocator<P>());
 
-    return 0;
+  return 0;
 }
diff --git a/libcxx/test/std/containers/unord/unord.map/unord.map.cnstr/init_size_hash.pass.cpp b/libcxx/test/std/containers/unord/unord.map/unord.map.cnstr/init_size_hash.pass.cpp
index 319af0e0e39b3..5ae90df90e3d0 100644
--- a/libcxx/test/std/containers/unord/unord.map/unord.map.cnstr/init_size_hash.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.map/unord.map.cnstr/init_size_hash.pass.cpp
@@ -29,76 +29,75 @@
 #include "test_allocator.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef std::unordered_map<int, std::string,
-                                   test_hash<int>,
-                                   test_equal_to<int>,
-                                   test_allocator<std::pair<const int, std::string> >
-                                   > C;
-        typedef std::pair<int, std::string> P;
-        C c({
-                P(1, "one"),
-                P(2, "two"),
-                P(3, "three"),
-                P(4, "four"),
-                P(1, "four"),
-                P(2, "four"),
-            },
-            7,
-            test_hash<int>(8)
-           );
-        LIBCPP_ASSERT(c.bucket_count() == 7);
-        assert(c.size() == 4);
-        assert(c.at(1) == "one");
-        assert(c.at(2) == "two");
-        assert(c.at(3) == "three");
-        assert(c.at(4) == "four");
-        assert(c.hash_function() == test_hash<int>(8));
-        assert(c.key_eq() == test_equal_to<int>());
-        assert(c.get_allocator() ==
-               (test_allocator<std::pair<const int, std::string> >()));
-        assert(!c.empty());
-        assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-        assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
-        assert(std::fabs(c.load_factor() - (float)c.size()/c.bucket_count()) < FLT_EPSILON);
-        assert(c.max_load_factor() == 1);
-    }
-    {
-        typedef std::unordered_map<int, std::string,
-                                   test_hash<int>,
-                                   test_equal_to<int>,
-                                   min_allocator<std::pair<const int, std::string> >
-                                   > C;
-        typedef std::pair<int, std::string> P;
-        C c({
-                P(1, "one"),
-                P(2, "two"),
-                P(3, "three"),
-                P(4, "four"),
-                P(1, "four"),
-                P(2, "four"),
-            },
-            7,
-            test_hash<int>(8)
-           );
-        LIBCPP_ASSERT(c.bucket_count() == 7);
-        assert(c.size() == 4);
-        assert(c.at(1) == "one");
-        assert(c.at(2) == "two");
-        assert(c.at(3) == "three");
-        assert(c.at(4) == "four");
-        assert(c.hash_function() == test_hash<int>(8));
-        assert(c.key_eq() == test_equal_to<int>());
-        assert(c.get_allocator() ==
-               (min_allocator<std::pair<const int, std::string> >()));
-        assert(!c.empty());
-        assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-        assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
-        assert(std::fabs(c.load_factor() - (float)c.size()/c.bucket_count()) < FLT_EPSILON);
-        assert(c.max_load_factor() == 1);
-    }
+int main(int, char**) {
+  {
+    typedef std::unordered_map<int,
+                               std::string,
+                               test_hash<int>,
+                               test_equal_to<int>,
+                               test_allocator<std::pair<const int, std::string> > >
+        C;
+    typedef std::pair<int, std::string> P;
+    C c(
+        {
+            P(1, "one"),
+            P(2, "two"),
+            P(3, "three"),
+            P(4, "four"),
+            P(1, "four"),
+            P(2, "four"),
+        },
+        7,
+        test_hash<int>(8));
+    LIBCPP_ASSERT(c.bucket_count() == 7);
+    assert(c.size() == 4);
+    assert(c.at(1) == "one");
+    assert(c.at(2) == "two");
+    assert(c.at(3) == "three");
+    assert(c.at(4) == "four");
+    assert(c.hash_function() == test_hash<int>(8));
+    assert(c.key_eq() == test_equal_to<int>());
+    assert(c.get_allocator() == (test_allocator<std::pair<const int, std::string> >()));
+    assert(!c.empty());
+    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
+    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
+    assert(std::fabs(c.load_factor() - (float)c.size() / c.bucket_count()) < FLT_EPSILON);
+    assert(c.max_load_factor() == 1);
+  }
+  {
+    typedef std::unordered_map<int,
+                               std::string,
+                               test_hash<int>,
+                               test_equal_to<int>,
+                               min_allocator<std::pair<const int, std::string> > >
+        C;
+    typedef std::pair<int, std::string> P;
+    C c(
+        {
+            P(1, "one"),
+            P(2, "two"),
+            P(3, "three"),
+            P(4, "four"),
+            P(1, "four"),
+            P(2, "four"),
+        },
+        7,
+        test_hash<int>(8));
+    LIBCPP_ASSERT(c.bucket_count() == 7);
+    assert(c.size() == 4);
+    assert(c.at(1) == "one");
+    assert(c.at(2) == "two");
+    assert(c.at(3) == "three");
+    assert(c.at(4) == "four");
+    assert(c.hash_function() == test_hash<int>(8));
+    assert(c.key_eq() == test_equal_to<int>());
+    assert(c.get_allocator() == (min_allocator<std::pair<const int, std::string> >()));
+    assert(!c.empty());
+    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
+    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
+    assert(std::fabs(c.load_factor() - (float)c.size() / c.bucket_count()) < FLT_EPSILON);
+    assert(c.max_load_factor() == 1);
+  }
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/unord/unord.map/unord.map.cnstr/init_size_hash_allocator.pass.cpp b/libcxx/test/std/containers/unord/unord.map/unord.map.cnstr/init_size_hash_allocator.pass.cpp
index 70a631dbaef86..c2adf98c7eaca 100644
--- a/libcxx/test/std/containers/unord/unord.map/unord.map.cnstr/init_size_hash_allocator.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.map/unord.map.cnstr/init_size_hash_allocator.pass.cpp
@@ -33,46 +33,42 @@
 
 template <class Allocator>
 void test(const Allocator& alloc) {
-    typedef std::unordered_map<int, std::string,
-                               test_hash<int>,
-                               test_equal_to<int>,
-                               Allocator
-                               > C;
-    typedef std::pair<int, std::string> P;
+  typedef std::unordered_map<int, std::string, test_hash<int>, test_equal_to<int>, Allocator > C;
+  typedef std::pair<int, std::string> P;
 
-    C c({
-            P(1, "one"),
-            P(2, "two"),
-            P(3, "three"),
-            P(4, "four"),
-            P(1, "four"),
-            P(2, "four"),
-         },
-         7,
-         test_hash<int>(5),
-         alloc);
-    LIBCPP_ASSERT(c.bucket_count() == 7);
-    assert(c.size() == 4);
-    assert(c.at(1) == "one");
-    assert(c.at(2) == "two");
-    assert(c.at(3) == "three");
-    assert(c.at(4) == "four");
-    assert(c.hash_function() == test_hash<int>(5));
-    assert(c.key_eq() == test_equal_to<int>());
-    assert(c.get_allocator() == alloc);
-    assert(!c.empty());
-    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-    assert(std::fabs(c.load_factor() - (float)c.size() / c.bucket_count()) < FLT_EPSILON);
-    assert(c.max_load_factor() == 1);
+  C c(
+      {
+          P(1, "one"),
+          P(2, "two"),
+          P(3, "three"),
+          P(4, "four"),
+          P(1, "four"),
+          P(2, "four"),
+      },
+      7,
+      test_hash<int>(5),
+      alloc);
+  LIBCPP_ASSERT(c.bucket_count() == 7);
+  assert(c.size() == 4);
+  assert(c.at(1) == "one");
+  assert(c.at(2) == "two");
+  assert(c.at(3) == "three");
+  assert(c.at(4) == "four");
+  assert(c.hash_function() == test_hash<int>(5));
+  assert(c.key_eq() == test_equal_to<int>());
+  assert(c.get_allocator() == alloc);
+  assert(!c.empty());
+  assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
+  assert(std::fabs(c.load_factor() - (float)c.size() / c.bucket_count()) < FLT_EPSILON);
+  assert(c.max_load_factor() == 1);
 }
 
-int main(int, char**)
-{
-    typedef std::pair<const int, std::string> P;
+int main(int, char**) {
+  typedef std::pair<const int, std::string> P;
 
-    test(test_allocator<P>(10));
-    test(min_allocator<P>());
-    test(explicit_allocator<P>());
+  test(test_allocator<P>(10));
+  test(min_allocator<P>());
+  test(explicit_allocator<P>());
 
-    return 0;
+  return 0;
 }
diff --git a/libcxx/test/std/containers/unord/unord.map/unord.map.cnstr/init_size_hash_equal.pass.cpp b/libcxx/test/std/containers/unord/unord.map/unord.map.cnstr/init_size_hash_equal.pass.cpp
index 20098ba04e0a7..fa84a89dd328e 100644
--- a/libcxx/test/std/containers/unord/unord.map/unord.map.cnstr/init_size_hash_equal.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.map/unord.map.cnstr/init_size_hash_equal.pass.cpp
@@ -30,78 +30,77 @@
 #include "test_allocator.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef std::unordered_map<int, std::string,
-                                   test_hash<int>,
-                                   test_equal_to<int>,
-                                   test_allocator<std::pair<const int, std::string> >
-                                   > C;
-        typedef std::pair<int, std::string> P;
-        C c({
-                P(1, "one"),
-                P(2, "two"),
-                P(3, "three"),
-                P(4, "four"),
-                P(1, "four"),
-                P(2, "four"),
-            },
-            7,
-            test_hash<int>(8),
-            test_equal_to<int>(9)
-           );
-        LIBCPP_ASSERT(c.bucket_count() == 7);
-        assert(c.size() == 4);
-        assert(c.at(1) == "one");
-        assert(c.at(2) == "two");
-        assert(c.at(3) == "three");
-        assert(c.at(4) == "four");
-        assert(c.hash_function() == test_hash<int>(8));
-        assert(c.key_eq() == test_equal_to<int>(9));
-        assert(c.get_allocator() ==
-               (test_allocator<std::pair<const int, std::string> >()));
-        assert(!c.empty());
-        assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-        assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
-        assert(std::fabs(c.load_factor() - (float)c.size()/c.bucket_count()) < FLT_EPSILON);
-        assert(c.max_load_factor() == 1);
-    }
-    {
-        typedef std::unordered_map<int, std::string,
-                                   test_hash<int>,
-                                   test_equal_to<int>,
-                                   min_allocator<std::pair<const int, std::string> >
-                                   > C;
-        typedef std::pair<int, std::string> P;
-        C c({
-                P(1, "one"),
-                P(2, "two"),
-                P(3, "three"),
-                P(4, "four"),
-                P(1, "four"),
-                P(2, "four"),
-            },
-            7,
-            test_hash<int>(8),
-            test_equal_to<int>(9)
-           );
-        LIBCPP_ASSERT(c.bucket_count() == 7);
-        assert(c.size() == 4);
-        assert(c.at(1) == "one");
-        assert(c.at(2) == "two");
-        assert(c.at(3) == "three");
-        assert(c.at(4) == "four");
-        assert(c.hash_function() == test_hash<int>(8));
-        assert(c.key_eq() == test_equal_to<int>(9));
-        assert(c.get_allocator() ==
-               (min_allocator<std::pair<const int, std::string> >()));
-        assert(!c.empty());
-        assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-        assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
-        assert(std::fabs(c.load_factor() - (float)c.size()/c.bucket_count()) < FLT_EPSILON);
-        assert(c.max_load_factor() == 1);
-    }
+int main(int, char**) {
+  {
+    typedef std::unordered_map<int,
+                               std::string,
+                               test_hash<int>,
+                               test_equal_to<int>,
+                               test_allocator<std::pair<const int, std::string> > >
+        C;
+    typedef std::pair<int, std::string> P;
+    C c(
+        {
+            P(1, "one"),
+            P(2, "two"),
+            P(3, "three"),
+            P(4, "four"),
+            P(1, "four"),
+            P(2, "four"),
+        },
+        7,
+        test_hash<int>(8),
+        test_equal_to<int>(9));
+    LIBCPP_ASSERT(c.bucket_count() == 7);
+    assert(c.size() == 4);
+    assert(c.at(1) == "one");
+    assert(c.at(2) == "two");
+    assert(c.at(3) == "three");
+    assert(c.at(4) == "four");
+    assert(c.hash_function() == test_hash<int>(8));
+    assert(c.key_eq() == test_equal_to<int>(9));
+    assert(c.get_allocator() == (test_allocator<std::pair<const int, std::string> >()));
+    assert(!c.empty());
+    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
+    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
+    assert(std::fabs(c.load_factor() - (float)c.size() / c.bucket_count()) < FLT_EPSILON);
+    assert(c.max_load_factor() == 1);
+  }
+  {
+    typedef std::unordered_map<int,
+                               std::string,
+                               test_hash<int>,
+                               test_equal_to<int>,
+                               min_allocator<std::pair<const int, std::string> > >
+        C;
+    typedef std::pair<int, std::string> P;
+    C c(
+        {
+            P(1, "one"),
+            P(2, "two"),
+            P(3, "three"),
+            P(4, "four"),
+            P(1, "four"),
+            P(2, "four"),
+        },
+        7,
+        test_hash<int>(8),
+        test_equal_to<int>(9));
+    LIBCPP_ASSERT(c.bucket_count() == 7);
+    assert(c.size() == 4);
+    assert(c.at(1) == "one");
+    assert(c.at(2) == "two");
+    assert(c.at(3) == "three");
+    assert(c.at(4) == "four");
+    assert(c.hash_function() == test_hash<int>(8));
+    assert(c.key_eq() == test_equal_to<int>(9));
+    assert(c.get_allocator() == (min_allocator<std::pair<const int, std::string> >()));
+    assert(!c.empty());
+    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
+    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
+    assert(std::fabs(c.load_factor() - (float)c.size() / c.bucket_count()) < FLT_EPSILON);
+    assert(c.max_load_factor() == 1);
+  }
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/unord/unord.map/unord.map.cnstr/init_size_hash_equal_allocator.pass.cpp b/libcxx/test/std/containers/unord/unord.map/unord.map.cnstr/init_size_hash_equal_allocator.pass.cpp
index f235ae0cd55bd..e9ba444d4f8cf 100644
--- a/libcxx/test/std/containers/unord/unord.map/unord.map.cnstr/init_size_hash_equal_allocator.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.map/unord.map.cnstr/init_size_hash_equal_allocator.pass.cpp
@@ -30,116 +30,111 @@
 #include "test_allocator.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef std::unordered_map<int, std::string,
-                                   test_hash<int>,
-                                   test_equal_to<int>,
-                                   test_allocator<std::pair<const int, std::string> >
-                                   > C;
-        typedef std::pair<int, std::string> P;
-        C c({
-                P(1, "one"),
-                P(2, "two"),
-                P(3, "three"),
-                P(4, "four"),
-                P(1, "four"),
-                P(2, "four"),
-            },
-            7,
-            test_hash<int>(8),
-            test_equal_to<int>(9),
-            test_allocator<std::pair<const int, std::string> >(10)
-           );
-        LIBCPP_ASSERT(c.bucket_count() == 7);
-        assert(c.size() == 4);
-        assert(c.at(1) == "one");
-        assert(c.at(2) == "two");
-        assert(c.at(3) == "three");
-        assert(c.at(4) == "four");
-        assert(c.hash_function() == test_hash<int>(8));
-        assert(c.key_eq() == test_equal_to<int>(9));
-        assert(c.get_allocator() ==
-               (test_allocator<std::pair<const int, std::string> >(10)));
-        assert(!c.empty());
-        assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-        assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
-        assert(std::fabs(c.load_factor() - (float)c.size()/c.bucket_count()) < FLT_EPSILON);
-        assert(c.max_load_factor() == 1);
-    }
-    {
-        typedef std::unordered_map<int, std::string,
-                                   test_hash<int>,
-                                   test_equal_to<int>,
-                                   min_allocator<std::pair<const int, std::string> >
-                                   > C;
-        typedef std::pair<int, std::string> P;
-        C c({
-                P(1, "one"),
-                P(2, "two"),
-                P(3, "three"),
-                P(4, "four"),
-                P(1, "four"),
-                P(2, "four"),
-            },
-            7,
-            test_hash<int>(8),
-            test_equal_to<int>(9),
-            min_allocator<std::pair<const int, std::string> >()
-           );
-        LIBCPP_ASSERT(c.bucket_count() == 7);
-        assert(c.size() == 4);
-        assert(c.at(1) == "one");
-        assert(c.at(2) == "two");
-        assert(c.at(3) == "three");
-        assert(c.at(4) == "four");
-        assert(c.hash_function() == test_hash<int>(8));
-        assert(c.key_eq() == test_equal_to<int>(9));
-        assert(c.get_allocator() ==
-               (min_allocator<std::pair<const int, std::string> >()));
-        assert(!c.empty());
-        assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-        assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
-        assert(std::fabs(c.load_factor() - (float)c.size()/c.bucket_count()) < FLT_EPSILON);
-        assert(c.max_load_factor() == 1);
-    }
-    {
-        typedef explicit_allocator<std::pair<const int, std::string>> A;
-        typedef std::unordered_map<int, std::string,
-                                   test_hash<int>,
-                                   test_equal_to<int>,
-                                   A
-                                   > C;
-        typedef std::pair<int, std::string> P;
-        C c({
-                P(1, "one"),
-                P(2, "two"),
-                P(3, "three"),
-                P(4, "four"),
-                P(1, "four"),
-                P(2, "four"),
-            },
-            7,
-            test_hash<int>(8),
-            test_equal_to<int>(9),
-            A{}
-           );
-        LIBCPP_ASSERT(c.bucket_count() == 7);
-        assert(c.size() == 4);
-        assert(c.at(1) == "one");
-        assert(c.at(2) == "two");
-        assert(c.at(3) == "three");
-        assert(c.at(4) == "four");
-        assert(c.hash_function() == test_hash<int>(8));
-        assert(c.key_eq() == test_equal_to<int>(9));
-        assert(c.get_allocator() == A{});
-        assert(!c.empty());
-        assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-        assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
-        assert(std::fabs(c.load_factor() - (float)c.size()/c.bucket_count()) < FLT_EPSILON);
-        assert(c.max_load_factor() == 1);
-    }
+int main(int, char**) {
+  {
+    typedef std::unordered_map<int,
+                               std::string,
+                               test_hash<int>,
+                               test_equal_to<int>,
+                               test_allocator<std::pair<const int, std::string> > >
+        C;
+    typedef std::pair<int, std::string> P;
+    C c(
+        {
+            P(1, "one"),
+            P(2, "two"),
+            P(3, "three"),
+            P(4, "four"),
+            P(1, "four"),
+            P(2, "four"),
+        },
+        7,
+        test_hash<int>(8),
+        test_equal_to<int>(9),
+        test_allocator<std::pair<const int, std::string> >(10));
+    LIBCPP_ASSERT(c.bucket_count() == 7);
+    assert(c.size() == 4);
+    assert(c.at(1) == "one");
+    assert(c.at(2) == "two");
+    assert(c.at(3) == "three");
+    assert(c.at(4) == "four");
+    assert(c.hash_function() == test_hash<int>(8));
+    assert(c.key_eq() == test_equal_to<int>(9));
+    assert(c.get_allocator() == (test_allocator<std::pair<const int, std::string> >(10)));
+    assert(!c.empty());
+    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
+    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
+    assert(std::fabs(c.load_factor() - (float)c.size() / c.bucket_count()) < FLT_EPSILON);
+    assert(c.max_load_factor() == 1);
+  }
+  {
+    typedef std::unordered_map<int,
+                               std::string,
+                               test_hash<int>,
+                               test_equal_to<int>,
+                               min_allocator<std::pair<const int, std::string> > >
+        C;
+    typedef std::pair<int, std::string> P;
+    C c(
+        {
+            P(1, "one"),
+            P(2, "two"),
+            P(3, "three"),
+            P(4, "four"),
+            P(1, "four"),
+            P(2, "four"),
+        },
+        7,
+        test_hash<int>(8),
+        test_equal_to<int>(9),
+        min_allocator<std::pair<const int, std::string> >());
+    LIBCPP_ASSERT(c.bucket_count() == 7);
+    assert(c.size() == 4);
+    assert(c.at(1) == "one");
+    assert(c.at(2) == "two");
+    assert(c.at(3) == "three");
+    assert(c.at(4) == "four");
+    assert(c.hash_function() == test_hash<int>(8));
+    assert(c.key_eq() == test_equal_to<int>(9));
+    assert(c.get_allocator() == (min_allocator<std::pair<const int, std::string> >()));
+    assert(!c.empty());
+    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
+    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
+    assert(std::fabs(c.load_factor() - (float)c.size() / c.bucket_count()) < FLT_EPSILON);
+    assert(c.max_load_factor() == 1);
+  }
+  {
+    typedef explicit_allocator<std::pair<const int, std::string>> A;
+    typedef std::unordered_map<int, std::string, test_hash<int>, test_equal_to<int>, A > C;
+    typedef std::pair<int, std::string> P;
+    C c(
+        {
+            P(1, "one"),
+            P(2, "two"),
+            P(3, "three"),
+            P(4, "four"),
+            P(1, "four"),
+            P(2, "four"),
+        },
+        7,
+        test_hash<int>(8),
+        test_equal_to<int>(9),
+        A{});
+    LIBCPP_ASSERT(c.bucket_count() == 7);
+    assert(c.size() == 4);
+    assert(c.at(1) == "one");
+    assert(c.at(2) == "two");
+    assert(c.at(3) == "three");
+    assert(c.at(4) == "four");
+    assert(c.hash_function() == test_hash<int>(8));
+    assert(c.key_eq() == test_equal_to<int>(9));
+    assert(c.get_allocator() == A{});
+    assert(!c.empty());
+    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
+    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
+    assert(std::fabs(c.load_factor() - (float)c.size() / c.bucket_count()) < FLT_EPSILON);
+    assert(c.max_load_factor() == 1);
+  }
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/unord/unord.map/unord.map.cnstr/iter_iter.pass.cpp b/libcxx/test/std/containers/unord/unord.map/unord.map.cnstr/iter_iter.pass.cpp
index 120826f02e937..da49abaf8b5b0 100644
--- a/libcxx/test/std/containers/unord/unord.map/unord.map.cnstr/iter_iter.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.map/unord.map.cnstr/iter_iter.pass.cpp
@@ -30,144 +30,139 @@
 #include "test_allocator.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef std::unordered_map<int, std::string,
-                                   test_hash<int>,
-                                   test_equal_to<int>,
-                                   test_allocator<std::pair<const int, std::string> >
-                                   > C;
-        typedef std::pair<int, std::string> P;
-        P a[] =
-        {
-            P(1, "one"),
-            P(2, "two"),
-            P(3, "three"),
-            P(4, "four"),
-            P(1, "four"),
-            P(2, "four"),
-        };
-        C c(cpp17_input_iterator<P*>(a), cpp17_input_iterator<P*>(a + sizeof(a)/sizeof(a[0])));
-        assert(c.bucket_count() >= 5);
-        assert(c.size() == 4);
-        assert(c.at(1) == "one");
-        assert(c.at(2) == "two");
-        assert(c.at(3) == "three");
-        assert(c.at(4) == "four");
-        assert(c.hash_function() == test_hash<int>());
-        assert(c.key_eq() == test_equal_to<int>());
-        assert(c.get_allocator() ==
-               (test_allocator<std::pair<const int, std::string> >()));
-        assert(!c.empty());
-        assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-        assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
-        assert(fabs(c.load_factor() - (float)c.size()/c.bucket_count()) < FLT_EPSILON);
-        assert(c.max_load_factor() == 1);
-    }
+int main(int, char**) {
+  {
+    typedef std::unordered_map<int,
+                               std::string,
+                               test_hash<int>,
+                               test_equal_to<int>,
+                               test_allocator<std::pair<const int, std::string> > >
+        C;
+    typedef std::pair<int, std::string> P;
+    P a[] = {
+        P(1, "one"),
+        P(2, "two"),
+        P(3, "three"),
+        P(4, "four"),
+        P(1, "four"),
+        P(2, "four"),
+    };
+    C c(cpp17_input_iterator<P*>(a), cpp17_input_iterator<P*>(a + sizeof(a) / sizeof(a[0])));
+    assert(c.bucket_count() >= 5);
+    assert(c.size() == 4);
+    assert(c.at(1) == "one");
+    assert(c.at(2) == "two");
+    assert(c.at(3) == "three");
+    assert(c.at(4) == "four");
+    assert(c.hash_function() == test_hash<int>());
+    assert(c.key_eq() == test_equal_to<int>());
+    assert(c.get_allocator() == (test_allocator<std::pair<const int, std::string> >()));
+    assert(!c.empty());
+    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
+    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
+    assert(fabs(c.load_factor() - (float)c.size() / c.bucket_count()) < FLT_EPSILON);
+    assert(c.max_load_factor() == 1);
+  }
 #if TEST_STD_VER >= 11
-    {
-        typedef std::unordered_map<int, std::string,
-                                   test_hash<int>,
-                                   test_equal_to<int>,
-                                   min_allocator<std::pair<const int, std::string> >
-                                   > C;
-        typedef std::pair<int, std::string> P;
-        P a[] =
-        {
-            P(1, "one"),
-            P(2, "two"),
-            P(3, "three"),
-            P(4, "four"),
-            P(1, "four"),
-            P(2, "four"),
-        };
-        C c(cpp17_input_iterator<P*>(a), cpp17_input_iterator<P*>(a + sizeof(a)/sizeof(a[0])));
-        assert(c.bucket_count() >= 5);
-        assert(c.size() == 4);
-        assert(c.at(1) == "one");
-        assert(c.at(2) == "two");
-        assert(c.at(3) == "three");
-        assert(c.at(4) == "four");
-        assert(c.hash_function() == test_hash<int>());
-        assert(c.key_eq() == test_equal_to<int>());
-        assert(c.get_allocator() ==
-               (min_allocator<std::pair<const int, std::string> >()));
-        assert(!c.empty());
-        assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-        assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
-        assert(fabs(c.load_factor() - (float)c.size()/c.bucket_count()) < FLT_EPSILON);
-        assert(c.max_load_factor() == 1);
-    }
-#if TEST_STD_VER > 11
-    {
-        typedef std::pair<int, std::string> P;
-        typedef test_allocator<std::pair<const int, std::string>> A;
-        typedef test_hash<int> HF;
-        typedef test_equal_to<int> Comp;
-        typedef std::unordered_map<int, std::string, HF, Comp, A> C;
+  {
+    typedef std::unordered_map<int,
+                               std::string,
+                               test_hash<int>,
+                               test_equal_to<int>,
+                               min_allocator<std::pair<const int, std::string> > >
+        C;
+    typedef std::pair<int, std::string> P;
+    P a[] = {
+        P(1, "one"),
+        P(2, "two"),
+        P(3, "three"),
+        P(4, "four"),
+        P(1, "four"),
+        P(2, "four"),
+    };
+    C c(cpp17_input_iterator<P*>(a), cpp17_input_iterator<P*>(a + sizeof(a) / sizeof(a[0])));
+    assert(c.bucket_count() >= 5);
+    assert(c.size() == 4);
+    assert(c.at(1) == "one");
+    assert(c.at(2) == "two");
+    assert(c.at(3) == "three");
+    assert(c.at(4) == "four");
+    assert(c.hash_function() == test_hash<int>());
+    assert(c.key_eq() == test_equal_to<int>());
+    assert(c.get_allocator() == (min_allocator<std::pair<const int, std::string> >()));
+    assert(!c.empty());
+    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
+    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
+    assert(fabs(c.load_factor() - (float)c.size() / c.bucket_count()) < FLT_EPSILON);
+    assert(c.max_load_factor() == 1);
+  }
+#  if TEST_STD_VER > 11
+  {
+    typedef std::pair<int, std::string> P;
+    typedef test_allocator<std::pair<const int, std::string>> A;
+    typedef test_hash<int> HF;
+    typedef test_equal_to<int> Comp;
+    typedef std::unordered_map<int, std::string, HF, Comp, A> C;
 
-        P arr[] =
-        {
-            P(1, "one"),
-            P(2, "two"),
-            P(3, "three"),
-            P(4, "four"),
-            P(1, "four"),
-            P(2, "four"),
-        };
-        C c(cpp17_input_iterator<P*>(arr), cpp17_input_iterator<P*>(arr + sizeof(arr)/sizeof(arr[0])), 14);
-        assert(c.bucket_count() >= 14);
-        assert(c.size() == 4);
-        assert(c.at(1) == "one");
-        assert(c.at(2) == "two");
-        assert(c.at(3) == "three");
-        assert(c.at(4) == "four");
-        assert(c.hash_function() == HF());
-        assert(c.key_eq() == Comp());
-        assert(c.get_allocator() == A());
-        assert(!c.empty());
-        assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-        assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
-        assert(fabs(c.load_factor() - (float)c.size()/c.bucket_count()) < FLT_EPSILON);
-        assert(c.max_load_factor() == 1);
-    }
-    {
-        typedef std::pair<int, std::string> P;
-        typedef test_allocator<std::pair<const int, std::string>> A;
-        typedef test_hash<int> HF;
-        typedef test_equal_to<int> Comp;
-        typedef std::unordered_map<int, std::string, HF, Comp, A> C;
+    P arr[] = {
+        P(1, "one"),
+        P(2, "two"),
+        P(3, "three"),
+        P(4, "four"),
+        P(1, "four"),
+        P(2, "four"),
+    };
+    C c(cpp17_input_iterator<P*>(arr), cpp17_input_iterator<P*>(arr + sizeof(arr) / sizeof(arr[0])), 14);
+    assert(c.bucket_count() >= 14);
+    assert(c.size() == 4);
+    assert(c.at(1) == "one");
+    assert(c.at(2) == "two");
+    assert(c.at(3) == "three");
+    assert(c.at(4) == "four");
+    assert(c.hash_function() == HF());
+    assert(c.key_eq() == Comp());
+    assert(c.get_allocator() == A());
+    assert(!c.empty());
+    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
+    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
+    assert(fabs(c.load_factor() - (float)c.size() / c.bucket_count()) < FLT_EPSILON);
+    assert(c.max_load_factor() == 1);
+  }
+  {
+    typedef std::pair<int, std::string> P;
+    typedef test_allocator<std::pair<const int, std::string>> A;
+    typedef test_hash<int> HF;
+    typedef test_equal_to<int> Comp;
+    typedef std::unordered_map<int, std::string, HF, Comp, A> C;
 
-        P arr[] =
-        {
-            P(1, "one"),
-            P(2, "two"),
-            P(3, "three"),
-            P(4, "four"),
-            P(1, "four"),
-            P(2, "four"),
-        };
-        HF hf(42);
-        A a(43);
-        C c(cpp17_input_iterator<P*>(arr), cpp17_input_iterator<P*>(arr + sizeof(arr)/sizeof(arr[0])), 14, hf, a);
-        assert(c.bucket_count() >= 14);
-        assert(c.size() == 4);
-        assert(c.at(1) == "one");
-        assert(c.at(2) == "two");
-        assert(c.at(3) == "three");
-        assert(c.at(4) == "four");
-        assert(c.hash_function() == hf);
-        assert(!(c.hash_function() == HF()));
-        assert(c.key_eq() == Comp());
-        assert(c.get_allocator() == a);
-        assert(!c.empty());
-        assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-        assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
-        assert(fabs(c.load_factor() - (float)c.size()/c.bucket_count()) < FLT_EPSILON);
-        assert(c.max_load_factor() == 1);
-    }
-#endif
+    P arr[] = {
+        P(1, "one"),
+        P(2, "two"),
+        P(3, "three"),
+        P(4, "four"),
+        P(1, "four"),
+        P(2, "four"),
+    };
+    HF hf(42);
+    A a(43);
+    C c(cpp17_input_iterator<P*>(arr), cpp17_input_iterator<P*>(arr + sizeof(arr) / sizeof(arr[0])), 14, hf, a);
+    assert(c.bucket_count() >= 14);
+    assert(c.size() == 4);
+    assert(c.at(1) == "one");
+    assert(c.at(2) == "two");
+    assert(c.at(3) == "three");
+    assert(c.at(4) == "four");
+    assert(c.hash_function() == hf);
+    assert(!(c.hash_function() == HF()));
+    assert(c.key_eq() == Comp());
+    assert(c.get_allocator() == a);
+    assert(!c.empty());
+    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
+    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
+    assert(fabs(c.load_factor() - (float)c.size() / c.bucket_count()) < FLT_EPSILON);
+    assert(c.max_load_factor() == 1);
+  }
+#  endif
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/unord/unord.map/unord.map.cnstr/iter_iter_size.pass.cpp b/libcxx/test/std/containers/unord/unord.map/unord.map.cnstr/iter_iter_size.pass.cpp
index f06feda28078c..91a3b9ef66520 100644
--- a/libcxx/test/std/containers/unord/unord.map/unord.map.cnstr/iter_iter_size.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.map/unord.map.cnstr/iter_iter_size.pass.cpp
@@ -30,79 +30,72 @@
 #include "test_allocator.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef std::unordered_map<int, std::string,
-                                   test_hash<int>,
-                                   test_equal_to<int>,
-                                   test_allocator<std::pair<const int, std::string> >
-                                   > C;
-        typedef std::pair<int, std::string> P;
-        P a[] =
-        {
-            P(1, "one"),
-            P(2, "two"),
-            P(3, "three"),
-            P(4, "four"),
-            P(1, "four"),
-            P(2, "four"),
-        };
-        C c(cpp17_input_iterator<P*>(a), cpp17_input_iterator<P*>(a + sizeof(a)/sizeof(a[0])),
-            10
-           );
-        LIBCPP_ASSERT(c.bucket_count() == 11);
-        assert(c.size() == 4);
-        assert(c.at(1) == "one");
-        assert(c.at(2) == "two");
-        assert(c.at(3) == "three");
-        assert(c.at(4) == "four");
-        assert(c.hash_function() == test_hash<int>());
-        assert(c.key_eq() == test_equal_to<int>());
-        assert(c.get_allocator() ==
-               (test_allocator<std::pair<const int, std::string> >()));
-        assert(!c.empty());
-        assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-        assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
-        assert(std::fabs(c.load_factor() - (float)c.size()/c.bucket_count()) < FLT_EPSILON);
-        assert(c.max_load_factor() == 1);
-    }
+int main(int, char**) {
+  {
+    typedef std::unordered_map<int,
+                               std::string,
+                               test_hash<int>,
+                               test_equal_to<int>,
+                               test_allocator<std::pair<const int, std::string> > >
+        C;
+    typedef std::pair<int, std::string> P;
+    P a[] = {
+        P(1, "one"),
+        P(2, "two"),
+        P(3, "three"),
+        P(4, "four"),
+        P(1, "four"),
+        P(2, "four"),
+    };
+    C c(cpp17_input_iterator<P*>(a), cpp17_input_iterator<P*>(a + sizeof(a) / sizeof(a[0])), 10);
+    LIBCPP_ASSERT(c.bucket_count() == 11);
+    assert(c.size() == 4);
+    assert(c.at(1) == "one");
+    assert(c.at(2) == "two");
+    assert(c.at(3) == "three");
+    assert(c.at(4) == "four");
+    assert(c.hash_function() == test_hash<int>());
+    assert(c.key_eq() == test_equal_to<int>());
+    assert(c.get_allocator() == (test_allocator<std::pair<const int, std::string> >()));
+    assert(!c.empty());
+    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
+    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
+    assert(std::fabs(c.load_factor() - (float)c.size() / c.bucket_count()) < FLT_EPSILON);
+    assert(c.max_load_factor() == 1);
+  }
 #if TEST_STD_VER >= 11
-    {
-        typedef std::unordered_map<int, std::string,
-                                   test_hash<int>,
-                                   test_equal_to<int>,
-                                   min_allocator<std::pair<const int, std::string> >
-                                   > C;
-        typedef std::pair<int, std::string> P;
-        P a[] =
-        {
-            P(1, "one"),
-            P(2, "two"),
-            P(3, "three"),
-            P(4, "four"),
-            P(1, "four"),
-            P(2, "four"),
-        };
-        C c(cpp17_input_iterator<P*>(a), cpp17_input_iterator<P*>(a + sizeof(a)/sizeof(a[0])),
-            10
-           );
-        LIBCPP_ASSERT(c.bucket_count() == 11);
-        assert(c.size() == 4);
-        assert(c.at(1) == "one");
-        assert(c.at(2) == "two");
-        assert(c.at(3) == "three");
-        assert(c.at(4) == "four");
-        assert(c.hash_function() == test_hash<int>());
-        assert(c.key_eq() == test_equal_to<int>());
-        assert(c.get_allocator() ==
-               (min_allocator<std::pair<const int, std::string> >()));
-        assert(!c.empty());
-        assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-        assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
-        assert(std::fabs(c.load_factor() - (float)c.size()/c.bucket_count()) < FLT_EPSILON);
-        assert(c.max_load_factor() == 1);
-    }
+  {
+    typedef std::unordered_map<int,
+                               std::string,
+                               test_hash<int>,
+                               test_equal_to<int>,
+                               min_allocator<std::pair<const int, std::string> > >
+        C;
+    typedef std::pair<int, std::string> P;
+    P a[] = {
+        P(1, "one"),
+        P(2, "two"),
+        P(3, "three"),
+        P(4, "four"),
+        P(1, "four"),
+        P(2, "four"),
+    };
+    C c(cpp17_input_iterator<P*>(a), cpp17_input_iterator<P*>(a + sizeof(a) / sizeof(a[0])), 10);
+    LIBCPP_ASSERT(c.bucket_count() == 11);
+    assert(c.size() == 4);
+    assert(c.at(1) == "one");
+    assert(c.at(2) == "two");
+    assert(c.at(3) == "three");
+    assert(c.at(4) == "four");
+    assert(c.hash_function() == test_hash<int>());
+    assert(c.key_eq() == test_equal_to<int>());
+    assert(c.get_allocator() == (min_allocator<std::pair<const int, std::string> >()));
+    assert(!c.empty());
+    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
+    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
+    assert(std::fabs(c.load_factor() - (float)c.size() / c.bucket_count()) < FLT_EPSILON);
+    assert(c.max_load_factor() == 1);
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/unord/unord.map/unord.map.cnstr/iter_iter_size_allocator.pass.cpp b/libcxx/test/std/containers/unord/unord.map/unord.map.cnstr/iter_iter_size_allocator.pass.cpp
index 11ca33816e679..1039e1d92be79 100644
--- a/libcxx/test/std/containers/unord/unord.map/unord.map.cnstr/iter_iter_size_allocator.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.map/unord.map.cnstr/iter_iter_size_allocator.pass.cpp
@@ -33,49 +33,40 @@
 #include "min_allocator.h"
 
 template <class Allocator>
-void test(const Allocator& alloc)
-{
-    typedef std::unordered_map<int, std::string,
-                               test_hash<int>,
-                               test_equal_to<int>,
-                               Allocator
-                               > C;
-    typedef std::pair<int, std::string> P;
-    P a[] =
-        {
-            P(1, "one"),
-            P(2, "two"),
-            P(3, "three"),
-            P(4, "four"),
-            P(1, "four"),
-            P(2, "four"),
-        };
+void test(const Allocator& alloc) {
+  typedef std::unordered_map<int, std::string, test_hash<int>, test_equal_to<int>, Allocator > C;
+  typedef std::pair<int, std::string> P;
+  P a[] = {
+      P(1, "one"),
+      P(2, "two"),
+      P(3, "three"),
+      P(4, "four"),
+      P(1, "four"),
+      P(2, "four"),
+  };
 
-    C c(cpp17_input_iterator<P*>(a), cpp17_input_iterator<P*>(a + sizeof(a) / sizeof(a[0])),
-        7,
-        alloc);
+  C c(cpp17_input_iterator<P*>(a), cpp17_input_iterator<P*>(a + sizeof(a) / sizeof(a[0])), 7, alloc);
 
-    LIBCPP_ASSERT(c.bucket_count() == 7);
-    assert(c.size() == 4);
-    assert(c.at(1) == "one");
-    assert(c.at(2) == "two");
-    assert(c.at(3) == "three");
-    assert(c.at(4) == "four");
-    assert(c.hash_function() == test_hash<int>());
-    assert(c.key_eq() == test_equal_to<int>());
-    assert(c.get_allocator() == alloc);
-    assert(!c.empty());
-    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-    assert(std::fabs(c.load_factor() - (float)c.size() / c.bucket_count()) < FLT_EPSILON);
-    assert(c.max_load_factor() == 1);
+  LIBCPP_ASSERT(c.bucket_count() == 7);
+  assert(c.size() == 4);
+  assert(c.at(1) == "one");
+  assert(c.at(2) == "two");
+  assert(c.at(3) == "three");
+  assert(c.at(4) == "four");
+  assert(c.hash_function() == test_hash<int>());
+  assert(c.key_eq() == test_equal_to<int>());
+  assert(c.get_allocator() == alloc);
+  assert(!c.empty());
+  assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
+  assert(std::fabs(c.load_factor() - (float)c.size() / c.bucket_count()) < FLT_EPSILON);
+  assert(c.max_load_factor() == 1);
 }
 
-int main(int, char**)
-{
-    typedef std::pair<const int, std::string> P;
-    test(test_allocator<P>(10));
-    test(min_allocator<P>());
-    test(explicit_allocator<P>());
+int main(int, char**) {
+  typedef std::pair<const int, std::string> P;
+  test(test_allocator<P>(10));
+  test(min_allocator<P>());
+  test(explicit_allocator<P>());
 
-    return 0;
+  return 0;
 }
diff --git a/libcxx/test/std/containers/unord/unord.map/unord.map.cnstr/iter_iter_size_hash.pass.cpp b/libcxx/test/std/containers/unord/unord.map/unord.map.cnstr/iter_iter_size_hash.pass.cpp
index 75ae4cd121243..f0a3358e80aaa 100644
--- a/libcxx/test/std/containers/unord/unord.map/unord.map.cnstr/iter_iter_size_hash.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.map/unord.map.cnstr/iter_iter_size_hash.pass.cpp
@@ -31,81 +31,72 @@
 #include "test_allocator.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef std::unordered_map<int, std::string,
-                                   test_hash<int>,
-                                   test_equal_to<int>,
-                                   test_allocator<std::pair<const int, std::string> >
-                                   > C;
-        typedef std::pair<int, std::string> P;
-        P a[] =
-        {
-            P(1, "one"),
-            P(2, "two"),
-            P(3, "three"),
-            P(4, "four"),
-            P(1, "four"),
-            P(2, "four"),
-        };
-        C c(cpp17_input_iterator<P*>(a), cpp17_input_iterator<P*>(a + sizeof(a)/sizeof(a[0])),
-            7,
-            test_hash<int>(8)
-           );
-        LIBCPP_ASSERT(c.bucket_count() == 7);
-        assert(c.size() == 4);
-        assert(c.at(1) == "one");
-        assert(c.at(2) == "two");
-        assert(c.at(3) == "three");
-        assert(c.at(4) == "four");
-        assert(c.hash_function() == test_hash<int>(8));
-        assert(c.key_eq() == test_equal_to<int>());
-        assert(c.get_allocator() ==
-               (test_allocator<std::pair<const int, std::string> >()));
-        assert(!c.empty());
-        assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-        assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
-        assert(std::fabs(c.load_factor() - (float)c.size()/c.bucket_count()) < FLT_EPSILON);
-        assert(c.max_load_factor() == 1);
-    }
+int main(int, char**) {
+  {
+    typedef std::unordered_map<int,
+                               std::string,
+                               test_hash<int>,
+                               test_equal_to<int>,
+                               test_allocator<std::pair<const int, std::string> > >
+        C;
+    typedef std::pair<int, std::string> P;
+    P a[] = {
+        P(1, "one"),
+        P(2, "two"),
+        P(3, "three"),
+        P(4, "four"),
+        P(1, "four"),
+        P(2, "four"),
+    };
+    C c(cpp17_input_iterator<P*>(a), cpp17_input_iterator<P*>(a + sizeof(a) / sizeof(a[0])), 7, test_hash<int>(8));
+    LIBCPP_ASSERT(c.bucket_count() == 7);
+    assert(c.size() == 4);
+    assert(c.at(1) == "one");
+    assert(c.at(2) == "two");
+    assert(c.at(3) == "three");
+    assert(c.at(4) == "four");
+    assert(c.hash_function() == test_hash<int>(8));
+    assert(c.key_eq() == test_equal_to<int>());
+    assert(c.get_allocator() == (test_allocator<std::pair<const int, std::string> >()));
+    assert(!c.empty());
+    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
+    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
+    assert(std::fabs(c.load_factor() - (float)c.size() / c.bucket_count()) < FLT_EPSILON);
+    assert(c.max_load_factor() == 1);
+  }
 #if TEST_STD_VER >= 11
-    {
-        typedef std::unordered_map<int, std::string,
-                                   test_hash<int>,
-                                   test_equal_to<int>,
-                                   min_allocator<std::pair<const int, std::string> >
-                                   > C;
-        typedef std::pair<int, std::string> P;
-        P a[] =
-        {
-            P(1, "one"),
-            P(2, "two"),
-            P(3, "three"),
-            P(4, "four"),
-            P(1, "four"),
-            P(2, "four"),
-        };
-        C c(cpp17_input_iterator<P*>(a), cpp17_input_iterator<P*>(a + sizeof(a)/sizeof(a[0])),
-            7,
-            test_hash<int>(8)
-           );
-        LIBCPP_ASSERT(c.bucket_count() == 7);
-        assert(c.size() == 4);
-        assert(c.at(1) == "one");
-        assert(c.at(2) == "two");
-        assert(c.at(3) == "three");
-        assert(c.at(4) == "four");
-        assert(c.hash_function() == test_hash<int>(8));
-        assert(c.key_eq() == test_equal_to<int>());
-        assert(c.get_allocator() ==
-               (min_allocator<std::pair<const int, std::string> >()));
-        assert(!c.empty());
-        assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-        assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
-        assert(std::fabs(c.load_factor() - (float)c.size()/c.bucket_count()) < FLT_EPSILON);
-        assert(c.max_load_factor() == 1);
-    }
+  {
+    typedef std::unordered_map<int,
+                               std::string,
+                               test_hash<int>,
+                               test_equal_to<int>,
+                               min_allocator<std::pair<const int, std::string> > >
+        C;
+    typedef std::pair<int, std::string> P;
+    P a[] = {
+        P(1, "one"),
+        P(2, "two"),
+        P(3, "three"),
+        P(4, "four"),
+        P(1, "four"),
+        P(2, "four"),
+    };
+    C c(cpp17_input_iterator<P*>(a), cpp17_input_iterator<P*>(a + sizeof(a) / sizeof(a[0])), 7, test_hash<int>(8));
+    LIBCPP_ASSERT(c.bucket_count() == 7);
+    assert(c.size() == 4);
+    assert(c.at(1) == "one");
+    assert(c.at(2) == "two");
+    assert(c.at(3) == "three");
+    assert(c.at(4) == "four");
+    assert(c.hash_function() == test_hash<int>(8));
+    assert(c.key_eq() == test_equal_to<int>());
+    assert(c.get_allocator() == (min_allocator<std::pair<const int, std::string> >()));
+    assert(!c.empty());
+    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
+    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
+    assert(std::fabs(c.load_factor() - (float)c.size() / c.bucket_count()) < FLT_EPSILON);
+    assert(c.max_load_factor() == 1);
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/unord/unord.map/unord.map.cnstr/iter_iter_size_hash_allocator.pass.cpp b/libcxx/test/std/containers/unord/unord.map/unord.map.cnstr/iter_iter_size_hash_allocator.pass.cpp
index 3a18bdf0dda74..b6e06df9c8fe2 100644
--- a/libcxx/test/std/containers/unord/unord.map/unord.map.cnstr/iter_iter_size_hash_allocator.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.map/unord.map.cnstr/iter_iter_size_hash_allocator.pass.cpp
@@ -34,50 +34,40 @@
 #include "min_allocator.h"
 
 template <class Allocator>
-void test(const Allocator& alloc)
-{
-    typedef std::unordered_map<int, std::string,
-                               test_hash<int>,
-                               test_equal_to<int>,
-                               Allocator
-                               > C;
-    typedef std::pair<int, std::string> P;
-    P a[] =
-        {
-            P(1, "one"),
-            P(2, "two"),
-            P(3, "three"),
-            P(4, "four"),
-            P(1, "four"),
-            P(2, "four"),
-        };
+void test(const Allocator& alloc) {
+  typedef std::unordered_map<int, std::string, test_hash<int>, test_equal_to<int>, Allocator > C;
+  typedef std::pair<int, std::string> P;
+  P a[] = {
+      P(1, "one"),
+      P(2, "two"),
+      P(3, "three"),
+      P(4, "four"),
+      P(1, "four"),
+      P(2, "four"),
+  };
 
-    C c(cpp17_input_iterator<P*>(a), cpp17_input_iterator<P*>(a + sizeof(a) / sizeof(a[0])),
-        7,
-        test_hash<int>(5),
-        alloc);
+  C c(cpp17_input_iterator<P*>(a), cpp17_input_iterator<P*>(a + sizeof(a) / sizeof(a[0])), 7, test_hash<int>(5), alloc);
 
-    LIBCPP_ASSERT(c.bucket_count() == 7);
-    assert(c.size() == 4);
-    assert(c.at(1) == "one");
-    assert(c.at(2) == "two");
-    assert(c.at(3) == "three");
-    assert(c.at(4) == "four");
-    assert(c.hash_function() == test_hash<int>(5));
-    assert(c.key_eq() == test_equal_to<int>());
-    assert(c.get_allocator() == alloc);
-    assert(!c.empty());
-    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-    assert(std::fabs(c.load_factor() - (float)c.size() / c.bucket_count()) < FLT_EPSILON);
-    assert(c.max_load_factor() == 1);
+  LIBCPP_ASSERT(c.bucket_count() == 7);
+  assert(c.size() == 4);
+  assert(c.at(1) == "one");
+  assert(c.at(2) == "two");
+  assert(c.at(3) == "three");
+  assert(c.at(4) == "four");
+  assert(c.hash_function() == test_hash<int>(5));
+  assert(c.key_eq() == test_equal_to<int>());
+  assert(c.get_allocator() == alloc);
+  assert(!c.empty());
+  assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
+  assert(std::fabs(c.load_factor() - (float)c.size() / c.bucket_count()) < FLT_EPSILON);
+  assert(c.max_load_factor() == 1);
 }
 
-int main(int, char**)
-{
-    typedef std::pair<const int, std::string> P;
-    test(test_allocator<P>(10));
-    test(min_allocator<P>());
-    test(explicit_allocator<P>());
+int main(int, char**) {
+  typedef std::pair<const int, std::string> P;
+  test(test_allocator<P>(10));
+  test(min_allocator<P>());
+  test(explicit_allocator<P>());
 
-    return 0;
+  return 0;
 }
diff --git a/libcxx/test/std/containers/unord/unord.map/unord.map.cnstr/iter_iter_size_hash_equal.pass.cpp b/libcxx/test/std/containers/unord/unord.map/unord.map.cnstr/iter_iter_size_hash_equal.pass.cpp
index 25cfaae1a8891..7976e40341000 100644
--- a/libcxx/test/std/containers/unord/unord.map/unord.map.cnstr/iter_iter_size_hash_equal.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.map/unord.map.cnstr/iter_iter_size_hash_equal.pass.cpp
@@ -31,83 +31,80 @@
 #include "test_allocator.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef std::unordered_map<int, std::string,
-                                   test_hash<int>,
-                                   test_equal_to<int>,
-                                   test_allocator<std::pair<const int, std::string> >
-                                   > C;
-        typedef std::pair<int, std::string> P;
-        P a[] =
-        {
-            P(1, "one"),
-            P(2, "two"),
-            P(3, "three"),
-            P(4, "four"),
-            P(1, "four"),
-            P(2, "four"),
-        };
-        C c(cpp17_input_iterator<P*>(a), cpp17_input_iterator<P*>(a + sizeof(a)/sizeof(a[0])),
-            7,
-            test_hash<int>(8),
-            test_equal_to<int>(9)
-           );
-        LIBCPP_ASSERT(c.bucket_count() == 7);
-        assert(c.size() == 4);
-        assert(c.at(1) == "one");
-        assert(c.at(2) == "two");
-        assert(c.at(3) == "three");
-        assert(c.at(4) == "four");
-        assert(c.hash_function() == test_hash<int>(8));
-        assert(c.key_eq() == test_equal_to<int>(9));
-        assert(c.get_allocator() ==
-               (test_allocator<std::pair<const int, std::string> >()));
-        assert(!c.empty());
-        assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-        assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
-        assert(std::fabs(c.load_factor() - (float)c.size()/c.bucket_count()) < FLT_EPSILON);
-        assert(c.max_load_factor() == 1);
-    }
+int main(int, char**) {
+  {
+    typedef std::unordered_map<int,
+                               std::string,
+                               test_hash<int>,
+                               test_equal_to<int>,
+                               test_allocator<std::pair<const int, std::string> > >
+        C;
+    typedef std::pair<int, std::string> P;
+    P a[] = {
+        P(1, "one"),
+        P(2, "two"),
+        P(3, "three"),
+        P(4, "four"),
+        P(1, "four"),
+        P(2, "four"),
+    };
+    C c(cpp17_input_iterator<P*>(a),
+        cpp17_input_iterator<P*>(a + sizeof(a) / sizeof(a[0])),
+        7,
+        test_hash<int>(8),
+        test_equal_to<int>(9));
+    LIBCPP_ASSERT(c.bucket_count() == 7);
+    assert(c.size() == 4);
+    assert(c.at(1) == "one");
+    assert(c.at(2) == "two");
+    assert(c.at(3) == "three");
+    assert(c.at(4) == "four");
+    assert(c.hash_function() == test_hash<int>(8));
+    assert(c.key_eq() == test_equal_to<int>(9));
+    assert(c.get_allocator() == (test_allocator<std::pair<const int, std::string> >()));
+    assert(!c.empty());
+    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
+    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
+    assert(std::fabs(c.load_factor() - (float)c.size() / c.bucket_count()) < FLT_EPSILON);
+    assert(c.max_load_factor() == 1);
+  }
 #if TEST_STD_VER >= 11
-    {
-        typedef std::unordered_map<int, std::string,
-                                   test_hash<int>,
-                                   test_equal_to<int>,
-                                   min_allocator<std::pair<const int, std::string> >
-                                   > C;
-        typedef std::pair<int, std::string> P;
-        P a[] =
-        {
-            P(1, "one"),
-            P(2, "two"),
-            P(3, "three"),
-            P(4, "four"),
-            P(1, "four"),
-            P(2, "four"),
-        };
-        C c(cpp17_input_iterator<P*>(a), cpp17_input_iterator<P*>(a + sizeof(a)/sizeof(a[0])),
-            7,
-            test_hash<int>(8),
-            test_equal_to<int>(9)
-           );
-        LIBCPP_ASSERT(c.bucket_count() == 7);
-        assert(c.size() == 4);
-        assert(c.at(1) == "one");
-        assert(c.at(2) == "two");
-        assert(c.at(3) == "three");
-        assert(c.at(4) == "four");
-        assert(c.hash_function() == test_hash<int>(8));
-        assert(c.key_eq() == test_equal_to<int>(9));
-        assert(c.get_allocator() ==
-               (min_allocator<std::pair<const int, std::string> >()));
-        assert(!c.empty());
-        assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-        assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
-        assert(std::fabs(c.load_factor() - (float)c.size()/c.bucket_count()) < FLT_EPSILON);
-        assert(c.max_load_factor() == 1);
-    }
+  {
+    typedef std::unordered_map<int,
+                               std::string,
+                               test_hash<int>,
+                               test_equal_to<int>,
+                               min_allocator<std::pair<const int, std::string> > >
+        C;
+    typedef std::pair<int, std::string> P;
+    P a[] = {
+        P(1, "one"),
+        P(2, "two"),
+        P(3, "three"),
+        P(4, "four"),
+        P(1, "four"),
+        P(2, "four"),
+    };
+    C c(cpp17_input_iterator<P*>(a),
+        cpp17_input_iterator<P*>(a + sizeof(a) / sizeof(a[0])),
+        7,
+        test_hash<int>(8),
+        test_equal_to<int>(9));
+    LIBCPP_ASSERT(c.bucket_count() == 7);
+    assert(c.size() == 4);
+    assert(c.at(1) == "one");
+    assert(c.at(2) == "two");
+    assert(c.at(3) == "three");
+    assert(c.at(4) == "four");
+    assert(c.hash_function() == test_hash<int>(8));
+    assert(c.key_eq() == test_equal_to<int>(9));
+    assert(c.get_allocator() == (min_allocator<std::pair<const int, std::string> >()));
+    assert(!c.empty());
+    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
+    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
+    assert(std::fabs(c.load_factor() - (float)c.size() / c.bucket_count()) < FLT_EPSILON);
+    assert(c.max_load_factor() == 1);
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/unord/unord.map/unord.map.cnstr/iter_iter_size_hash_equal_allocator.pass.cpp b/libcxx/test/std/containers/unord/unord.map/unord.map.cnstr/iter_iter_size_hash_equal_allocator.pass.cpp
index 8abdd0dab566d..2660d2013a0e3 100644
--- a/libcxx/test/std/containers/unord/unord.map/unord.map.cnstr/iter_iter_size_hash_equal_allocator.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.map/unord.map.cnstr/iter_iter_size_hash_equal_allocator.pass.cpp
@@ -32,123 +32,115 @@
 #include "test_allocator.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef std::unordered_map<int, std::string,
-                                   test_hash<int>,
-                                   test_equal_to<int>,
-                                   test_allocator<std::pair<const int, std::string> >
-                                   > C;
-        typedef std::pair<int, std::string> P;
-        P a[] =
-        {
-            P(1, "one"),
-            P(2, "two"),
-            P(3, "three"),
-            P(4, "four"),
-            P(1, "four"),
-            P(2, "four"),
-        };
-        C c(cpp17_input_iterator<P*>(a), cpp17_input_iterator<P*>(a + sizeof(a)/sizeof(a[0])),
-            7,
-            test_hash<int>(8),
-            test_equal_to<int>(9),
-            test_allocator<std::pair<const int, std::string> >(10)
-           );
-        LIBCPP_ASSERT(c.bucket_count() == 7);
-        assert(c.size() == 4);
-        assert(c.at(1) == "one");
-        assert(c.at(2) == "two");
-        assert(c.at(3) == "three");
-        assert(c.at(4) == "four");
-        assert(c.hash_function() == test_hash<int>(8));
-        assert(c.key_eq() == test_equal_to<int>(9));
-        assert(c.get_allocator() ==
-               (test_allocator<std::pair<const int, std::string> >(10)));
-        assert(!c.empty());
-        assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-        assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
-        assert(std::fabs(c.load_factor() - (float)c.size()/c.bucket_count()) < FLT_EPSILON);
-        assert(c.max_load_factor() == 1);
-    }
+int main(int, char**) {
+  {
+    typedef std::unordered_map<int,
+                               std::string,
+                               test_hash<int>,
+                               test_equal_to<int>,
+                               test_allocator<std::pair<const int, std::string> > >
+        C;
+    typedef std::pair<int, std::string> P;
+    P a[] = {
+        P(1, "one"),
+        P(2, "two"),
+        P(3, "three"),
+        P(4, "four"),
+        P(1, "four"),
+        P(2, "four"),
+    };
+    C c(cpp17_input_iterator<P*>(a),
+        cpp17_input_iterator<P*>(a + sizeof(a) / sizeof(a[0])),
+        7,
+        test_hash<int>(8),
+        test_equal_to<int>(9),
+        test_allocator<std::pair<const int, std::string> >(10));
+    LIBCPP_ASSERT(c.bucket_count() == 7);
+    assert(c.size() == 4);
+    assert(c.at(1) == "one");
+    assert(c.at(2) == "two");
+    assert(c.at(3) == "three");
+    assert(c.at(4) == "four");
+    assert(c.hash_function() == test_hash<int>(8));
+    assert(c.key_eq() == test_equal_to<int>(9));
+    assert(c.get_allocator() == (test_allocator<std::pair<const int, std::string> >(10)));
+    assert(!c.empty());
+    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
+    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
+    assert(std::fabs(c.load_factor() - (float)c.size() / c.bucket_count()) < FLT_EPSILON);
+    assert(c.max_load_factor() == 1);
+  }
 #if TEST_STD_VER >= 11
-    {
-        typedef std::unordered_map<int, std::string,
-                                   test_hash<int>,
-                                   test_equal_to<int>,
-                                   min_allocator<std::pair<const int, std::string> >
-                                   > C;
-        typedef std::pair<int, std::string> P;
-        P a[] =
-        {
-            P(1, "one"),
-            P(2, "two"),
-            P(3, "three"),
-            P(4, "four"),
-            P(1, "four"),
-            P(2, "four"),
-        };
-        C c(cpp17_input_iterator<P*>(a), cpp17_input_iterator<P*>(a + sizeof(a)/sizeof(a[0])),
-            7,
-            test_hash<int>(8),
-            test_equal_to<int>(9),
-            min_allocator<std::pair<const int, std::string> >()
-           );
-        LIBCPP_ASSERT(c.bucket_count() == 7);
-        assert(c.size() == 4);
-        assert(c.at(1) == "one");
-        assert(c.at(2) == "two");
-        assert(c.at(3) == "three");
-        assert(c.at(4) == "four");
-        assert(c.hash_function() == test_hash<int>(8));
-        assert(c.key_eq() == test_equal_to<int>(9));
-        assert(c.get_allocator() ==
-               (min_allocator<std::pair<const int, std::string> >()));
-        assert(!c.empty());
-        assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-        assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
-        assert(std::fabs(c.load_factor() - (float)c.size()/c.bucket_count()) < FLT_EPSILON);
-        assert(c.max_load_factor() == 1);
-    }
-    {
-        typedef explicit_allocator<std::pair<const int, std::string>> A;
-        typedef std::unordered_map<int, std::string,
-                                   test_hash<int>,
-                                   test_equal_to<int>,
-                                   A
-                                   > C;
-        typedef std::pair<int, std::string> P;
-        P a[] =
-        {
-            P(1, "one"),
-            P(2, "two"),
-            P(3, "three"),
-            P(4, "four"),
-            P(1, "four"),
-            P(2, "four"),
-        };
-        C c(cpp17_input_iterator<P*>(a), cpp17_input_iterator<P*>(a + sizeof(a)/sizeof(a[0])),
-            7,
-            test_hash<int>(8),
-            test_equal_to<int>(9),
-            A{}
-           );
-        LIBCPP_ASSERT(c.bucket_count() == 7);
-        assert(c.size() == 4);
-        assert(c.at(1) == "one");
-        assert(c.at(2) == "two");
-        assert(c.at(3) == "three");
-        assert(c.at(4) == "four");
-        assert(c.hash_function() == test_hash<int>(8));
-        assert(c.key_eq() == test_equal_to<int>(9));
-        assert(c.get_allocator() == A{});
-        assert(!c.empty());
-        assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-        assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
-        assert(std::fabs(c.load_factor() - (float)c.size()/c.bucket_count()) < FLT_EPSILON);
-        assert(c.max_load_factor() == 1);
-    }
+  {
+    typedef std::unordered_map<int,
+                               std::string,
+                               test_hash<int>,
+                               test_equal_to<int>,
+                               min_allocator<std::pair<const int, std::string> > >
+        C;
+    typedef std::pair<int, std::string> P;
+    P a[] = {
+        P(1, "one"),
+        P(2, "two"),
+        P(3, "three"),
+        P(4, "four"),
+        P(1, "four"),
+        P(2, "four"),
+    };
+    C c(cpp17_input_iterator<P*>(a),
+        cpp17_input_iterator<P*>(a + sizeof(a) / sizeof(a[0])),
+        7,
+        test_hash<int>(8),
+        test_equal_to<int>(9),
+        min_allocator<std::pair<const int, std::string> >());
+    LIBCPP_ASSERT(c.bucket_count() == 7);
+    assert(c.size() == 4);
+    assert(c.at(1) == "one");
+    assert(c.at(2) == "two");
+    assert(c.at(3) == "three");
+    assert(c.at(4) == "four");
+    assert(c.hash_function() == test_hash<int>(8));
+    assert(c.key_eq() == test_equal_to<int>(9));
+    assert(c.get_allocator() == (min_allocator<std::pair<const int, std::string> >()));
+    assert(!c.empty());
+    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
+    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
+    assert(std::fabs(c.load_factor() - (float)c.size() / c.bucket_count()) < FLT_EPSILON);
+    assert(c.max_load_factor() == 1);
+  }
+  {
+    typedef explicit_allocator<std::pair<const int, std::string>> A;
+    typedef std::unordered_map<int, std::string, test_hash<int>, test_equal_to<int>, A > C;
+    typedef std::pair<int, std::string> P;
+    P a[] = {
+        P(1, "one"),
+        P(2, "two"),
+        P(3, "three"),
+        P(4, "four"),
+        P(1, "four"),
+        P(2, "four"),
+    };
+    C c(cpp17_input_iterator<P*>(a),
+        cpp17_input_iterator<P*>(a + sizeof(a) / sizeof(a[0])),
+        7,
+        test_hash<int>(8),
+        test_equal_to<int>(9),
+        A{});
+    LIBCPP_ASSERT(c.bucket_count() == 7);
+    assert(c.size() == 4);
+    assert(c.at(1) == "one");
+    assert(c.at(2) == "two");
+    assert(c.at(3) == "three");
+    assert(c.at(4) == "four");
+    assert(c.hash_function() == test_hash<int>(8));
+    assert(c.key_eq() == test_equal_to<int>(9));
+    assert(c.get_allocator() == A{});
+    assert(!c.empty());
+    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
+    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
+    assert(std::fabs(c.load_factor() - (float)c.size() / c.bucket_count()) < FLT_EPSILON);
+    assert(c.max_load_factor() == 1);
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/unord/unord.map/unord.map.cnstr/move.pass.cpp b/libcxx/test/std/containers/unord/unord.map/unord.map.cnstr/move.pass.cpp
index 336cdfb8d215a..2a2a3531716ed 100644
--- a/libcxx/test/std/containers/unord/unord.map/unord.map.cnstr/move.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.map/unord.map.cnstr/move.pass.cpp
@@ -29,150 +29,139 @@
 #include "test_allocator.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef std::unordered_map<int, std::string,
-                                   test_hash<int>,
-                                   test_equal_to<int>,
-                                   test_allocator<std::pair<const int, std::string> >
-                                   > C;
-        C c0(7,
-            test_hash<int>(8),
-            test_equal_to<int>(9),
-            test_allocator<std::pair<const int, std::string> >(10)
-           );
-        C::iterator it0 = c0.begin();
-        C c = std::move(c0);
-        LIBCPP_ASSERT(c.bucket_count() == 7);
-        assert(c.size() == 0);
-        assert(c.hash_function() == test_hash<int>(8));
-        assert(c.key_eq() == test_equal_to<int>(9));
-        assert(c.get_allocator() ==
-               (test_allocator<std::pair<const int, std::string> >(10)));
-        assert(c.empty());
-        assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-        assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
-        assert(c.load_factor() == 0);
-        assert(c.max_load_factor() == 1);
-        assert(it0 == c.begin()); // Iterators remain valid
+int main(int, char**) {
+  {
+    typedef std::unordered_map<int,
+                               std::string,
+                               test_hash<int>,
+                               test_equal_to<int>,
+                               test_allocator<std::pair<const int, std::string> > >
+        C;
+    C c0(7, test_hash<int>(8), test_equal_to<int>(9), test_allocator<std::pair<const int, std::string> >(10));
+    C::iterator it0 = c0.begin();
+    C c             = std::move(c0);
+    LIBCPP_ASSERT(c.bucket_count() == 7);
+    assert(c.size() == 0);
+    assert(c.hash_function() == test_hash<int>(8));
+    assert(c.key_eq() == test_equal_to<int>(9));
+    assert(c.get_allocator() == (test_allocator<std::pair<const int, std::string> >(10)));
+    assert(c.empty());
+    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
+    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
+    assert(c.load_factor() == 0);
+    assert(c.max_load_factor() == 1);
+    assert(it0 == c.begin()); // Iterators remain valid
 
-        assert(c0.empty());
-    }
-    {
-        typedef std::unordered_map<int, std::string,
-                                   test_hash<int>,
-                                   test_equal_to<int>,
-                                   test_allocator<std::pair<const int, std::string> >
-                                   > C;
-        typedef std::pair<int, std::string> P;
-        P a[] =
-        {
-            P(1, "one"),
-            P(2, "two"),
-            P(3, "three"),
-            P(4, "four"),
-            P(1, "four"),
-            P(2, "four"),
-        };
-        C c0(a, a + sizeof(a)/sizeof(a[0]),
-            7,
-            test_hash<int>(8),
-            test_equal_to<int>(9),
-            test_allocator<std::pair<const int, std::string> >(10)
-           );
-        C::iterator it0 = c0.begin();
-        C c = std::move(c0);
-        LIBCPP_ASSERT(c.bucket_count() == 7);
-        assert(c.size() == 4);
-        assert(c.at(1) == "one");
-        assert(c.at(2) == "two");
-        assert(c.at(3) == "three");
-        assert(c.at(4) == "four");
-        assert(c.hash_function() == test_hash<int>(8));
-        assert(c.key_eq() == test_equal_to<int>(9));
-        assert(c.get_allocator() ==
-               (test_allocator<std::pair<const int, std::string> >(10)));
-        assert(!c.empty());
-        assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-        assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
-        assert(std::fabs(c.load_factor() - (float)c.size()/c.bucket_count()) < FLT_EPSILON);
-        assert(c.max_load_factor() == 1);
-        assert(it0 == c.begin()); // Iterators remain valid
+    assert(c0.empty());
+  }
+  {
+    typedef std::unordered_map<int,
+                               std::string,
+                               test_hash<int>,
+                               test_equal_to<int>,
+                               test_allocator<std::pair<const int, std::string> > >
+        C;
+    typedef std::pair<int, std::string> P;
+    P a[] = {
+        P(1, "one"),
+        P(2, "two"),
+        P(3, "three"),
+        P(4, "four"),
+        P(1, "four"),
+        P(2, "four"),
+    };
+    C c0(a,
+         a + sizeof(a) / sizeof(a[0]),
+         7,
+         test_hash<int>(8),
+         test_equal_to<int>(9),
+         test_allocator<std::pair<const int, std::string> >(10));
+    C::iterator it0 = c0.begin();
+    C c             = std::move(c0);
+    LIBCPP_ASSERT(c.bucket_count() == 7);
+    assert(c.size() == 4);
+    assert(c.at(1) == "one");
+    assert(c.at(2) == "two");
+    assert(c.at(3) == "three");
+    assert(c.at(4) == "four");
+    assert(c.hash_function() == test_hash<int>(8));
+    assert(c.key_eq() == test_equal_to<int>(9));
+    assert(c.get_allocator() == (test_allocator<std::pair<const int, std::string> >(10)));
+    assert(!c.empty());
+    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
+    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
+    assert(std::fabs(c.load_factor() - (float)c.size() / c.bucket_count()) < FLT_EPSILON);
+    assert(c.max_load_factor() == 1);
+    assert(it0 == c.begin()); // Iterators remain valid
 
-        assert(c0.empty());
-    }
-    {
-        typedef std::unordered_map<int, std::string,
-                                   test_hash<int>,
-                                   test_equal_to<int>,
-                                   min_allocator<std::pair<const int, std::string> >
-                                   > C;
-        C c0(7,
-            test_hash<int>(8),
-            test_equal_to<int>(9),
-            min_allocator<std::pair<const int, std::string> >()
-           );
-        C::iterator it0 = c0.begin();
-        C c = std::move(c0);
-        LIBCPP_ASSERT(c.bucket_count() == 7);
-        assert(c.size() == 0);
-        assert(c.hash_function() == test_hash<int>(8));
-        assert(c.key_eq() == test_equal_to<int>(9));
-        assert(c.get_allocator() ==
-               (min_allocator<std::pair<const int, std::string> >()));
-        assert(c.empty());
-        assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-        assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
-        assert(c.load_factor() == 0);
-        assert(c.max_load_factor() == 1);
-        assert(it0 == c.begin()); // Iterators remain valid
+    assert(c0.empty());
+  }
+  {
+    typedef std::unordered_map<int,
+                               std::string,
+                               test_hash<int>,
+                               test_equal_to<int>,
+                               min_allocator<std::pair<const int, std::string> > >
+        C;
+    C c0(7, test_hash<int>(8), test_equal_to<int>(9), min_allocator<std::pair<const int, std::string> >());
+    C::iterator it0 = c0.begin();
+    C c             = std::move(c0);
+    LIBCPP_ASSERT(c.bucket_count() == 7);
+    assert(c.size() == 0);
+    assert(c.hash_function() == test_hash<int>(8));
+    assert(c.key_eq() == test_equal_to<int>(9));
+    assert(c.get_allocator() == (min_allocator<std::pair<const int, std::string> >()));
+    assert(c.empty());
+    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
+    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
+    assert(c.load_factor() == 0);
+    assert(c.max_load_factor() == 1);
+    assert(it0 == c.begin()); // Iterators remain valid
 
-        assert(c0.empty());
-    }
-    {
-        typedef std::unordered_map<int, std::string,
-                                   test_hash<int>,
-                                   test_equal_to<int>,
-                                   min_allocator<std::pair<const int, std::string> >
-                                   > C;
-        typedef std::pair<int, std::string> P;
-        P a[] =
-        {
-            P(1, "one"),
-            P(2, "two"),
-            P(3, "three"),
-            P(4, "four"),
-            P(1, "four"),
-            P(2, "four"),
-        };
-        C c0(a, a + sizeof(a)/sizeof(a[0]),
-            7,
-            test_hash<int>(8),
-            test_equal_to<int>(9),
-            min_allocator<std::pair<const int, std::string> >()
-           );
-        C::iterator it0 = c0.begin();
-        C c = std::move(c0);
-        LIBCPP_ASSERT(c.bucket_count() == 7);
-        assert(c.size() == 4);
-        assert(c.at(1) == "one");
-        assert(c.at(2) == "two");
-        assert(c.at(3) == "three");
-        assert(c.at(4) == "four");
-        assert(c.hash_function() == test_hash<int>(8));
-        assert(c.key_eq() == test_equal_to<int>(9));
-        assert(c.get_allocator() ==
-               (min_allocator<std::pair<const int, std::string> >()));
-        assert(!c.empty());
-        assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-        assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
-        assert(std::fabs(c.load_factor() - (float)c.size()/c.bucket_count()) < FLT_EPSILON);
-        assert(c.max_load_factor() == 1);
-        assert(it0 == c.begin()); // Iterators remain valid
+    assert(c0.empty());
+  }
+  {
+    typedef std::unordered_map<int,
+                               std::string,
+                               test_hash<int>,
+                               test_equal_to<int>,
+                               min_allocator<std::pair<const int, std::string> > >
+        C;
+    typedef std::pair<int, std::string> P;
+    P a[] = {
+        P(1, "one"),
+        P(2, "two"),
+        P(3, "three"),
+        P(4, "four"),
+        P(1, "four"),
+        P(2, "four"),
+    };
+    C c0(a,
+         a + sizeof(a) / sizeof(a[0]),
+         7,
+         test_hash<int>(8),
+         test_equal_to<int>(9),
+         min_allocator<std::pair<const int, std::string> >());
+    C::iterator it0 = c0.begin();
+    C c             = std::move(c0);
+    LIBCPP_ASSERT(c.bucket_count() == 7);
+    assert(c.size() == 4);
+    assert(c.at(1) == "one");
+    assert(c.at(2) == "two");
+    assert(c.at(3) == "three");
+    assert(c.at(4) == "four");
+    assert(c.hash_function() == test_hash<int>(8));
+    assert(c.key_eq() == test_equal_to<int>(9));
+    assert(c.get_allocator() == (min_allocator<std::pair<const int, std::string> >()));
+    assert(!c.empty());
+    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
+    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
+    assert(std::fabs(c.load_factor() - (float)c.size() / c.bucket_count()) < FLT_EPSILON);
+    assert(c.max_load_factor() == 1);
+    assert(it0 == c.begin()); // Iterators remain valid
 
-        assert(c0.empty());
-    }
+    assert(c0.empty());
+  }
 
-    return 0;
+  return 0;
 }
diff --git a/libcxx/test/std/containers/unord/unord.map/unord.map.cnstr/move_alloc.addressof.compile.pass.cpp b/libcxx/test/std/containers/unord/unord.map/unord.map.cnstr/move_alloc.addressof.compile.pass.cpp
index 1fec0ee5d0f4b..90c69b3855fab 100644
--- a/libcxx/test/std/containers/unord/unord.map/unord.map.cnstr/move_alloc.addressof.compile.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.map/unord.map.cnstr/move_alloc.addressof.compile.pass.cpp
@@ -28,8 +28,12 @@
 
 void test() {
   using A = test_allocator<std::pair<const operator_hijacker, operator_hijacker>>;
-  using C = std::unordered_map<operator_hijacker, operator_hijacker, std::hash<operator_hijacker>,
-                               std::equal_to<operator_hijacker>, A>;
+  using C =
+      std::unordered_map<operator_hijacker,
+                         operator_hijacker,
+                         std::hash<operator_hijacker>,
+                         std::equal_to<operator_hijacker>,
+                         A>;
 
   C mo;
   C m(std::move(mo), A());
diff --git a/libcxx/test/std/containers/unord/unord.map/unord.map.cnstr/move_alloc.pass.cpp b/libcxx/test/std/containers/unord/unord.map/unord.map.cnstr/move_alloc.pass.cpp
index 89f2716f9cd2c..154b11f3da9f4 100644
--- a/libcxx/test/std/containers/unord/unord.map/unord.map.cnstr/move_alloc.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.map/unord.map.cnstr/move_alloc.pass.cpp
@@ -29,172 +29,131 @@
 #include "test_allocator.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef std::pair<int, std::string> P;
-        typedef test_allocator<std::pair<const int, std::string>> A;
-        typedef std::unordered_map<int, std::string,
-                                   test_hash<int>,
-                                   test_equal_to<int>,
-                                   A
-                                   > C;
-        P a[] =
-        {
-            P(1, "one"),
-            P(2, "two"),
-            P(3, "three"),
-            P(4, "four"),
-            P(1, "four"),
-            P(2, "four"),
-        };
-        C c0(a, a + sizeof(a)/sizeof(a[0]),
-            7,
-            test_hash<int>(8),
-            test_equal_to<int>(9),
-            A(10)
-           );
-        C c(std::move(c0), A(12));
-        assert(c.bucket_count() >= 5);
-        assert(c.size() == 4);
-        assert(c.at(1) == "one");
-        assert(c.at(2) == "two");
-        assert(c.at(3) == "three");
-        assert(c.at(4) == "four");
-        assert(c.hash_function() == test_hash<int>(8));
-        assert(c.key_eq() == test_equal_to<int>(9));
-        assert(c.get_allocator() == A(12));
-        assert(!c.empty());
-        assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-        assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
-        assert(std::fabs(c.load_factor() - (float)c.size()/c.bucket_count()) < FLT_EPSILON);
-        assert(c.max_load_factor() == 1);
+int main(int, char**) {
+  {
+    typedef std::pair<int, std::string> P;
+    typedef test_allocator<std::pair<const int, std::string>> A;
+    typedef std::unordered_map<int, std::string, test_hash<int>, test_equal_to<int>, A > C;
+    P a[] = {
+        P(1, "one"),
+        P(2, "two"),
+        P(3, "three"),
+        P(4, "four"),
+        P(1, "four"),
+        P(2, "four"),
+    };
+    C c0(a, a + sizeof(a) / sizeof(a[0]), 7, test_hash<int>(8), test_equal_to<int>(9), A(10));
+    C c(std::move(c0), A(12));
+    assert(c.bucket_count() >= 5);
+    assert(c.size() == 4);
+    assert(c.at(1) == "one");
+    assert(c.at(2) == "two");
+    assert(c.at(3) == "three");
+    assert(c.at(4) == "four");
+    assert(c.hash_function() == test_hash<int>(8));
+    assert(c.key_eq() == test_equal_to<int>(9));
+    assert(c.get_allocator() == A(12));
+    assert(!c.empty());
+    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
+    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
+    assert(std::fabs(c.load_factor() - (float)c.size() / c.bucket_count()) < FLT_EPSILON);
+    assert(c.max_load_factor() == 1);
 
-        assert(c0.empty());
-    }
-    {
-        typedef std::pair<int, std::string> P;
-        typedef test_allocator<std::pair<const int, std::string>> A;
-        typedef std::unordered_map<int, std::string,
-                                   test_hash<int>,
-                                   test_equal_to<int>,
-                                   A
-                                   > C;
-        P a[] =
-        {
-            P(1, "one"),
-            P(2, "two"),
-            P(3, "three"),
-            P(4, "four"),
-            P(1, "four"),
-            P(2, "four"),
-        };
-        C c0(a, a + sizeof(a)/sizeof(a[0]),
-            7,
-            test_hash<int>(8),
-            test_equal_to<int>(9),
-            A(10)
-           );
-        C c(std::move(c0), A(10));
-        LIBCPP_ASSERT(c.bucket_count() == 7);
-        assert(c.size() == 4);
-        assert(c.at(1) == "one");
-        assert(c.at(2) == "two");
-        assert(c.at(3) == "three");
-        assert(c.at(4) == "four");
-        assert(c.hash_function() == test_hash<int>(8));
-        assert(c.key_eq() == test_equal_to<int>(9));
-        assert(c.get_allocator() == A(10));
-        assert(!c.empty());
-        assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-        assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
-        assert(std::fabs(c.load_factor() - (float)c.size()/c.bucket_count()) < FLT_EPSILON);
-        assert(c.max_load_factor() == 1);
+    assert(c0.empty());
+  }
+  {
+    typedef std::pair<int, std::string> P;
+    typedef test_allocator<std::pair<const int, std::string>> A;
+    typedef std::unordered_map<int, std::string, test_hash<int>, test_equal_to<int>, A > C;
+    P a[] = {
+        P(1, "one"),
+        P(2, "two"),
+        P(3, "three"),
+        P(4, "four"),
+        P(1, "four"),
+        P(2, "four"),
+    };
+    C c0(a, a + sizeof(a) / sizeof(a[0]), 7, test_hash<int>(8), test_equal_to<int>(9), A(10));
+    C c(std::move(c0), A(10));
+    LIBCPP_ASSERT(c.bucket_count() == 7);
+    assert(c.size() == 4);
+    assert(c.at(1) == "one");
+    assert(c.at(2) == "two");
+    assert(c.at(3) == "three");
+    assert(c.at(4) == "four");
+    assert(c.hash_function() == test_hash<int>(8));
+    assert(c.key_eq() == test_equal_to<int>(9));
+    assert(c.get_allocator() == A(10));
+    assert(!c.empty());
+    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
+    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
+    assert(std::fabs(c.load_factor() - (float)c.size() / c.bucket_count()) < FLT_EPSILON);
+    assert(c.max_load_factor() == 1);
 
-        assert(c0.empty());
-    }
-    {
-        typedef std::pair<int, std::string> P;
-        typedef min_allocator<std::pair<const int, std::string>> A;
-        typedef std::unordered_map<int, std::string,
-                                   test_hash<int>,
-                                   test_equal_to<int>,
-                                   A
-                                   > C;
-        P a[] =
-        {
-            P(1, "one"),
-            P(2, "two"),
-            P(3, "three"),
-            P(4, "four"),
-            P(1, "four"),
-            P(2, "four"),
-        };
-        C c0(a, a + sizeof(a)/sizeof(a[0]),
-            7,
-            test_hash<int>(8),
-            test_equal_to<int>(9),
-            A()
-           );
-        C c(std::move(c0), A());
-        LIBCPP_ASSERT(c.bucket_count() == 7);
-        assert(c.size() == 4);
-        assert(c.at(1) == "one");
-        assert(c.at(2) == "two");
-        assert(c.at(3) == "three");
-        assert(c.at(4) == "four");
-        assert(c.hash_function() == test_hash<int>(8));
-        assert(c.key_eq() == test_equal_to<int>(9));
-        assert(c.get_allocator() == A());
-        assert(!c.empty());
-        assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-        assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
-        assert(std::fabs(c.load_factor() - (float)c.size()/c.bucket_count()) < FLT_EPSILON);
-        assert(c.max_load_factor() == 1);
+    assert(c0.empty());
+  }
+  {
+    typedef std::pair<int, std::string> P;
+    typedef min_allocator<std::pair<const int, std::string>> A;
+    typedef std::unordered_map<int, std::string, test_hash<int>, test_equal_to<int>, A > C;
+    P a[] = {
+        P(1, "one"),
+        P(2, "two"),
+        P(3, "three"),
+        P(4, "four"),
+        P(1, "four"),
+        P(2, "four"),
+    };
+    C c0(a, a + sizeof(a) / sizeof(a[0]), 7, test_hash<int>(8), test_equal_to<int>(9), A());
+    C c(std::move(c0), A());
+    LIBCPP_ASSERT(c.bucket_count() == 7);
+    assert(c.size() == 4);
+    assert(c.at(1) == "one");
+    assert(c.at(2) == "two");
+    assert(c.at(3) == "three");
+    assert(c.at(4) == "four");
+    assert(c.hash_function() == test_hash<int>(8));
+    assert(c.key_eq() == test_equal_to<int>(9));
+    assert(c.get_allocator() == A());
+    assert(!c.empty());
+    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
+    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
+    assert(std::fabs(c.load_factor() - (float)c.size() / c.bucket_count()) < FLT_EPSILON);
+    assert(c.max_load_factor() == 1);
 
-        assert(c0.empty());
-    }
-    {
-        typedef std::pair<int, std::string> P;
-        typedef explicit_allocator<std::pair<const int, std::string>> A;
-        typedef std::unordered_map<int, std::string,
-                                   test_hash<int>,
-                                   test_equal_to<int>,
-                                   A
-                                   > C;
-        P a[] =
-        {
-            P(1, "one"),
-            P(2, "two"),
-            P(3, "three"),
-            P(4, "four"),
-            P(1, "four"),
-            P(2, "four"),
-        };
-        C c0(a, a + sizeof(a)/sizeof(a[0]),
-            7,
-            test_hash<int>(8),
-            test_equal_to<int>(9),
-            A{}
-           );
-        C c(std::move(c0), A{});
-        LIBCPP_ASSERT(c.bucket_count() == 7);
-        assert(c.size() == 4);
-        assert(c.at(1) == "one");
-        assert(c.at(2) == "two");
-        assert(c.at(3) == "three");
-        assert(c.at(4) == "four");
-        assert(c.hash_function() == test_hash<int>(8));
-        assert(c.key_eq() == test_equal_to<int>(9));
-        assert(c.get_allocator() == A{});
-        assert(!c.empty());
-        assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-        assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
-        assert(std::fabs(c.load_factor() - (float)c.size()/c.bucket_count()) < FLT_EPSILON);
-        assert(c.max_load_factor() == 1);
+    assert(c0.empty());
+  }
+  {
+    typedef std::pair<int, std::string> P;
+    typedef explicit_allocator<std::pair<const int, std::string>> A;
+    typedef std::unordered_map<int, std::string, test_hash<int>, test_equal_to<int>, A > C;
+    P a[] = {
+        P(1, "one"),
+        P(2, "two"),
+        P(3, "three"),
+        P(4, "four"),
+        P(1, "four"),
+        P(2, "four"),
+    };
+    C c0(a, a + sizeof(a) / sizeof(a[0]), 7, test_hash<int>(8), test_equal_to<int>(9), A{});
+    C c(std::move(c0), A{});
+    LIBCPP_ASSERT(c.bucket_count() == 7);
+    assert(c.size() == 4);
+    assert(c.at(1) == "one");
+    assert(c.at(2) == "two");
+    assert(c.at(3) == "three");
+    assert(c.at(4) == "four");
+    assert(c.hash_function() == test_hash<int>(8));
+    assert(c.key_eq() == test_equal_to<int>(9));
+    assert(c.get_allocator() == A{});
+    assert(!c.empty());
+    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
+    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
+    assert(std::fabs(c.load_factor() - (float)c.size() / c.bucket_count()) < FLT_EPSILON);
+    assert(c.max_load_factor() == 1);
 
-        assert(c0.empty());
-    }
+    assert(c0.empty());
+  }
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/unord/unord.map/unord.map.cnstr/move_assign_noexcept.compile.pass.cpp b/libcxx/test/std/containers/unord/unord.map/unord.map.cnstr/move_assign_noexcept.compile.pass.cpp
new file mode 100644
index 0000000000000..098e83d3521e5
--- /dev/null
+++ b/libcxx/test/std/containers/unord/unord.map/unord.map.cnstr/move_assign_noexcept.compile.pass.cpp
@@ -0,0 +1,78 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// <unordered_map>
+
+// unordered_map& operator=(unordered_map&& c)
+//     noexcept(
+//          allocator_type::propagate_on_container_move_assignment::value &&
+//          is_nothrow_move_assignable<allocator_type>::value &&
+//          is_nothrow_move_assignable<key_compare>::value);
+
+// This tests a conforming extension
+
+// UNSUPPORTED: c++03
+
+#include <unordered_map>
+
+#include "test_macros.h"
+#include "MoveOnly.h"
+#include "test_allocator.h"
+
+template <class T>
+struct some_hash {
+  using value_type = T;
+  some_hash();
+  some_hash(const some_hash&);
+  some_hash& operator=(const some_hash&);
+
+  std::size_t operator()(T const&) const;
+};
+
+template <class T>
+struct some_comp {
+  using value_type = T;
+  some_comp& operator=(const some_comp&);
+  bool operator()(const T&, const T&) const { return false; }
+};
+
+template <class T>
+struct always_equal_alloc {
+  using value_type = T;
+  always_equal_alloc(const always_equal_alloc&);
+  void allocate(std::size_t);
+};
+
+template <class T>
+struct not_always_equal_alloc {
+  int i;
+  using value_type = T;
+  not_always_equal_alloc(const not_always_equal_alloc&);
+  void allocate(std::size_t);
+};
+
+template <template <class> class Alloc>
+using unordered_set_alloc =
+    std::unordered_map<MoveOnly,
+                       MoveOnly,
+                       std::hash<MoveOnly>,
+                       std::equal_to<MoveOnly>,
+                       Alloc<std::pair<const MoveOnly, MoveOnly>>>;
+
+static_assert(std::is_nothrow_move_assignable<unordered_set_alloc<std::allocator>>::value, "");
+static_assert(!std::is_nothrow_move_assignable<unordered_set_alloc<test_allocator>>::value, "");
+#if TEST_STD_VER >= 17
+static_assert(std::is_nothrow_move_assignable<unordered_set_alloc<always_equal_alloc>>::value, "");
+#endif
+static_assert(!std::is_nothrow_move_assignable<unordered_set_alloc<not_always_equal_alloc>>::value, "");
+#if defined(_LIBCPP_VERSION)
+static_assert(std::is_nothrow_move_assignable<unordered_set_alloc<other_allocator>>::value, "");
+#endif // _LIBCPP_VERSION
+static_assert(!std::is_nothrow_move_assignable<std::unordered_map<int, int, some_hash<int>>>::value, "");
+static_assert(!std::is_nothrow_move_assignable<std::unordered_map<int, int, std::hash<int>, some_comp<int>>>::value,
+              "");
diff --git a/libcxx/test/std/containers/unord/unord.map/unord.map.cnstr/move_assign_noexcept.pass.cpp b/libcxx/test/std/containers/unord/unord.map/unord.map.cnstr/move_assign_noexcept.pass.cpp
deleted file mode 100644
index 9d50537216716..0000000000000
--- a/libcxx/test/std/containers/unord/unord.map/unord.map.cnstr/move_assign_noexcept.pass.cpp
+++ /dev/null
@@ -1,76 +0,0 @@
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-// <unordered_map>
-
-// unordered_map& operator=(unordered_map&& c)
-//     noexcept(
-//          allocator_type::propagate_on_container_move_assignment::value &&
-//          is_nothrow_move_assignable<allocator_type>::value &&
-//          is_nothrow_move_assignable<key_compare>::value);
-
-// This tests a conforming extension
-
-// UNSUPPORTED: c++03
-
-#include <unordered_map>
-#include <cassert>
-
-#include "test_macros.h"
-#include "MoveOnly.h"
-#include "test_allocator.h"
-
-template <class T>
-struct some_comp
-{
-    typedef T value_type;
-    some_comp& operator=(const some_comp&);
-    bool operator()(const T&, const T&) const { return false; }
-};
-
-template <class T>
-struct some_hash
-{
-    typedef T value_type;
-    some_hash();
-    some_hash(const some_hash&);
-    some_hash& operator=(const some_hash&);
-
-    std::size_t operator()(T const&) const;
-};
-
-int main(int, char**)
-{
-    {
-        typedef std::unordered_map<MoveOnly, MoveOnly> C;
-        static_assert(std::is_nothrow_move_assignable<C>::value, "");
-    }
-    {
-        typedef std::unordered_map<MoveOnly, MoveOnly, std::hash<MoveOnly>,
-                           std::equal_to<MoveOnly>, test_allocator<std::pair<const MoveOnly, MoveOnly>>> C;
-        static_assert(!std::is_nothrow_move_assignable<C>::value, "");
-    }
-#if defined(_LIBCPP_VERSION)
-    {
-        typedef std::unordered_map<MoveOnly, MoveOnly, std::hash<MoveOnly>,
-                          std::equal_to<MoveOnly>, other_allocator<std::pair<const MoveOnly, MoveOnly>>> C;
-        static_assert(std::is_nothrow_move_assignable<C>::value, "");
-    }
-#endif // _LIBCPP_VERSION
-    {
-        typedef std::unordered_map<MoveOnly, MoveOnly, some_hash<MoveOnly>> C;
-        static_assert(!std::is_nothrow_move_assignable<C>::value, "");
-    }
-    {
-        typedef std::unordered_map<MoveOnly, MoveOnly, std::hash<MoveOnly>,
-                                                         some_comp<MoveOnly>> C;
-        static_assert(!std::is_nothrow_move_assignable<C>::value, "");
-    }
-
-  return 0;
-}
diff --git a/libcxx/test/std/containers/unord/unord.map/unord.map.cnstr/move_noexcept.pass.cpp b/libcxx/test/std/containers/unord/unord.map/unord.map.cnstr/move_noexcept.pass.cpp
index 8458627e75481..82b510db2fa26 100644
--- a/libcxx/test/std/containers/unord/unord.map/unord.map.cnstr/move_noexcept.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.map/unord.map.cnstr/move_noexcept.pass.cpp
@@ -24,50 +24,54 @@
 #include "test_allocator.h"
 
 template <class T>
-struct some_comp
-{
-    typedef T value_type;
-    some_comp(const some_comp&);
-    bool operator()(const T&, const T&) const { return false; }
+struct some_comp {
+  typedef T value_type;
+  some_comp(const some_comp&);
+  bool operator()(const T&, const T&) const { return false; }
 };
 
 template <class T>
-struct some_hash
-{
-    typedef T value_type;
-    some_hash();
-    some_hash(const some_hash&);
+struct some_hash {
+  typedef T value_type;
+  some_hash();
+  some_hash(const some_hash&);
 
-    std::size_t operator()(T const&) const;
+  std::size_t operator()(T const&) const;
 };
 
-int main(int, char**)
-{
+int main(int, char**) {
 #if defined(_LIBCPP_VERSION)
-    {
-        typedef std::unordered_map<MoveOnly, MoveOnly> C;
-        static_assert(std::is_nothrow_move_constructible<C>::value, "");
-    }
-    {
-        typedef std::unordered_map<MoveOnly, MoveOnly, std::hash<MoveOnly>,
-                           std::equal_to<MoveOnly>, test_allocator<std::pair<const MoveOnly, MoveOnly>>> C;
-        static_assert(std::is_nothrow_move_constructible<C>::value, "");
-    }
-    {
-        typedef std::unordered_map<MoveOnly, MoveOnly, std::hash<MoveOnly>,
-                          std::equal_to<MoveOnly>, other_allocator<std::pair<const MoveOnly, MoveOnly>>> C;
-        static_assert(std::is_nothrow_move_constructible<C>::value, "");
-    }
+  {
+    typedef std::unordered_map<MoveOnly, MoveOnly> C;
+    static_assert(std::is_nothrow_move_constructible<C>::value, "");
+  }
+  {
+    typedef std::unordered_map<MoveOnly,
+                               MoveOnly,
+                               std::hash<MoveOnly>,
+                               std::equal_to<MoveOnly>,
+                               test_allocator<std::pair<const MoveOnly, MoveOnly>>>
+        C;
+    static_assert(std::is_nothrow_move_constructible<C>::value, "");
+  }
+  {
+    typedef std::unordered_map<MoveOnly,
+                               MoveOnly,
+                               std::hash<MoveOnly>,
+                               std::equal_to<MoveOnly>,
+                               other_allocator<std::pair<const MoveOnly, MoveOnly>>>
+        C;
+    static_assert(std::is_nothrow_move_constructible<C>::value, "");
+  }
 #endif // _LIBCPP_VERSION
-    {
-        typedef std::unordered_map<MoveOnly, MoveOnly, some_hash<MoveOnly>> C;
-        static_assert(!std::is_nothrow_move_constructible<C>::value, "");
-    }
-    {
-        typedef std::unordered_map<MoveOnly, MoveOnly, std::hash<MoveOnly>,
-                                                         some_comp<MoveOnly>> C;
-        static_assert(!std::is_nothrow_move_constructible<C>::value, "");
-    }
+  {
+    typedef std::unordered_map<MoveOnly, MoveOnly, some_hash<MoveOnly>> C;
+    static_assert(!std::is_nothrow_move_constructible<C>::value, "");
+  }
+  {
+    typedef std::unordered_map<MoveOnly, MoveOnly, std::hash<MoveOnly>, some_comp<MoveOnly>> C;
+    static_assert(!std::is_nothrow_move_constructible<C>::value, "");
+  }
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/unord/unord.map/unord.map.cnstr/size.compile.fail.cpp b/libcxx/test/std/containers/unord/unord.map/unord.map.cnstr/size.compile.fail.cpp
index 371987d6e2d2a..a203f7ba3c224 100644
--- a/libcxx/test/std/containers/unord/unord.map/unord.map.cnstr/size.compile.fail.cpp
+++ b/libcxx/test/std/containers/unord/unord.map/unord.map.cnstr/size.compile.fail.cpp
@@ -24,47 +24,44 @@
 #include "test_allocator.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef std::unordered_map<NotConstructible, NotConstructible,
-                                   test_hash<NotConstructible>,
-                                   test_equal_to<NotConstructible>,
-                                   test_allocator<std::pair<const NotConstructible,
-                                                                  NotConstructible> >
-                                   > C;
-        C c = 7;
-        LIBCPP_ASSERT(c.bucket_count() == 7);
-        assert(c.hash_function() == test_hash<NotConstructible>());
-        assert(c.key_eq() == test_equal_to<NotConstructible>());
-        assert(c.get_allocator() ==
-               (test_allocator<std::pair<const NotConstructible, NotConstructible> >()));
-        assert(c.size() == 0);
-        assert(c.empty());
-        assert(std::distance(c.begin(), c.end()) == 0);
-        assert(c.load_factor() == 0);
-        assert(c.max_load_factor() == 1);
-    }
+int main(int, char**) {
+  {
+    typedef std::unordered_map<NotConstructible,
+                               NotConstructible,
+                               test_hash<NotConstructible>,
+                               test_equal_to<NotConstructible>,
+                               test_allocator<std::pair<const NotConstructible, NotConstructible> > >
+        C;
+    C c = 7;
+    LIBCPP_ASSERT(c.bucket_count() == 7);
+    assert(c.hash_function() == test_hash<NotConstructible>());
+    assert(c.key_eq() == test_equal_to<NotConstructible>());
+    assert(c.get_allocator() == (test_allocator<std::pair<const NotConstructible, NotConstructible> >()));
+    assert(c.size() == 0);
+    assert(c.empty());
+    assert(std::distance(c.begin(), c.end()) == 0);
+    assert(c.load_factor() == 0);
+    assert(c.max_load_factor() == 1);
+  }
 #if TEST_STD_VER >= 11
-    {
-        typedef std::unordered_map<NotConstructible, NotConstructible,
-                                   test_hash<NotConstructible>,
-                                   test_equal_to<NotConstructible>,
-                                   min_allocator<std::pair<const NotConstructible,
-                                                                 NotConstructible> >
-                                   > C;
-        C c = 7;
-        LIBCPP_ASSERT(c.bucket_count() == 7);
-        assert(c.hash_function() == test_hash<NotConstructible>());
-        assert(c.key_eq() == test_equal_to<NotConstructible>());
-        assert(c.get_allocator() ==
-               (min_allocator<std::pair<const NotConstructible, NotConstructible> >()));
-        assert(c.size() == 0);
-        assert(c.empty());
-        assert(std::distance(c.begin(), c.end()) == 0);
-        assert(c.load_factor() == 0);
-        assert(c.max_load_factor() == 1);
-    }
+  {
+    typedef std::unordered_map<NotConstructible,
+                               NotConstructible,
+                               test_hash<NotConstructible>,
+                               test_equal_to<NotConstructible>,
+                               min_allocator<std::pair<const NotConstructible, NotConstructible> > >
+        C;
+    C c = 7;
+    LIBCPP_ASSERT(c.bucket_count() == 7);
+    assert(c.hash_function() == test_hash<NotConstructible>());
+    assert(c.key_eq() == test_equal_to<NotConstructible>());
+    assert(c.get_allocator() == (min_allocator<std::pair<const NotConstructible, NotConstructible> >()));
+    assert(c.size() == 0);
+    assert(c.empty());
+    assert(std::distance(c.begin(), c.end()) == 0);
+    assert(c.load_factor() == 0);
+    assert(c.max_load_factor() == 1);
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/unord/unord.map/unord.map.cnstr/size.pass.cpp b/libcxx/test/std/containers/unord/unord.map/unord.map.cnstr/size.pass.cpp
index 7328b954021ab..f7f7ca54a9a2a 100644
--- a/libcxx/test/std/containers/unord/unord.map/unord.map.cnstr/size.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.map/unord.map.cnstr/size.pass.cpp
@@ -24,47 +24,44 @@
 #include "test_allocator.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef std::unordered_map<NotConstructible, NotConstructible,
-                                   test_hash<NotConstructible>,
-                                   test_equal_to<NotConstructible>,
-                                   test_allocator<std::pair<const NotConstructible,
-                                                                  NotConstructible> >
-                                   > C;
-        C c(7);
-        LIBCPP_ASSERT(c.bucket_count() == 7);
-        assert(c.hash_function() == test_hash<NotConstructible>());
-        assert(c.key_eq() == test_equal_to<NotConstructible>());
-        assert(c.get_allocator() ==
-               (test_allocator<std::pair<const NotConstructible, NotConstructible> >()));
-        assert(c.size() == 0);
-        assert(c.empty());
-        assert(std::distance(c.begin(), c.end()) == 0);
-        assert(c.load_factor() == 0);
-        assert(c.max_load_factor() == 1);
-    }
+int main(int, char**) {
+  {
+    typedef std::unordered_map<NotConstructible,
+                               NotConstructible,
+                               test_hash<NotConstructible>,
+                               test_equal_to<NotConstructible>,
+                               test_allocator<std::pair<const NotConstructible, NotConstructible> > >
+        C;
+    C c(7);
+    LIBCPP_ASSERT(c.bucket_count() == 7);
+    assert(c.hash_function() == test_hash<NotConstructible>());
+    assert(c.key_eq() == test_equal_to<NotConstructible>());
+    assert(c.get_allocator() == (test_allocator<std::pair<const NotConstructible, NotConstructible> >()));
+    assert(c.size() == 0);
+    assert(c.empty());
+    assert(std::distance(c.begin(), c.end()) == 0);
+    assert(c.load_factor() == 0);
+    assert(c.max_load_factor() == 1);
+  }
 #if TEST_STD_VER >= 11
-    {
-        typedef std::unordered_map<NotConstructible, NotConstructible,
-                                   test_hash<NotConstructible>,
-                                   test_equal_to<NotConstructible>,
-                                   min_allocator<std::pair<const NotConstructible,
-                                                                 NotConstructible> >
-                                   > C;
-        C c(7);
-        LIBCPP_ASSERT(c.bucket_count() == 7);
-        assert(c.hash_function() == test_hash<NotConstructible>());
-        assert(c.key_eq() == test_equal_to<NotConstructible>());
-        assert(c.get_allocator() ==
-               (min_allocator<std::pair<const NotConstructible, NotConstructible> >()));
-        assert(c.size() == 0);
-        assert(c.empty());
-        assert(std::distance(c.begin(), c.end()) == 0);
-        assert(c.load_factor() == 0);
-        assert(c.max_load_factor() == 1);
-    }
+  {
+    typedef std::unordered_map<NotConstructible,
+                               NotConstructible,
+                               test_hash<NotConstructible>,
+                               test_equal_to<NotConstructible>,
+                               min_allocator<std::pair<const NotConstructible, NotConstructible> > >
+        C;
+    C c(7);
+    LIBCPP_ASSERT(c.bucket_count() == 7);
+    assert(c.hash_function() == test_hash<NotConstructible>());
+    assert(c.key_eq() == test_equal_to<NotConstructible>());
+    assert(c.get_allocator() == (min_allocator<std::pair<const NotConstructible, NotConstructible> >()));
+    assert(c.size() == 0);
+    assert(c.empty());
+    assert(std::distance(c.begin(), c.end()) == 0);
+    assert(c.load_factor() == 0);
+    assert(c.max_load_factor() == 1);
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/unord/unord.map/unord.map.cnstr/size_allocator.pass.cpp b/libcxx/test/std/containers/unord/unord.map/unord.map.cnstr/size_allocator.pass.cpp
index fcc896014fa50..450fb7b5e40d4 100644
--- a/libcxx/test/std/containers/unord/unord.map/unord.map.cnstr/size_allocator.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.map/unord.map.cnstr/size_allocator.pass.cpp
@@ -29,31 +29,31 @@
 
 template <class Allocator>
 void test(const Allocator& alloc) {
-    typedef std::unordered_map<NotConstructible, NotConstructible,
-                               test_hash<NotConstructible>,
-                               test_equal_to<NotConstructible>,
-                               Allocator
-                               > C;
-
-    C c(7, alloc);
-    LIBCPP_ASSERT(c.bucket_count() == 7);
-    assert(c.hash_function() == test_hash<NotConstructible>());
-    assert(c.key_eq() == test_equal_to<NotConstructible>());
-    assert(c.get_allocator() == alloc);
-    assert(c.size() == 0);
-    assert(c.empty());
-    assert(std::distance(c.begin(), c.end()) == 0);
-    assert(c.load_factor() == 0);
-    assert(c.max_load_factor() == 1);
+  typedef std::unordered_map<NotConstructible,
+                             NotConstructible,
+                             test_hash<NotConstructible>,
+                             test_equal_to<NotConstructible>,
+                             Allocator >
+      C;
+
+  C c(7, alloc);
+  LIBCPP_ASSERT(c.bucket_count() == 7);
+  assert(c.hash_function() == test_hash<NotConstructible>());
+  assert(c.key_eq() == test_equal_to<NotConstructible>());
+  assert(c.get_allocator() == alloc);
+  assert(c.size() == 0);
+  assert(c.empty());
+  assert(std::distance(c.begin(), c.end()) == 0);
+  assert(c.load_factor() == 0);
+  assert(c.max_load_factor() == 1);
 }
 
-int main(int, char**)
-{
-    typedef std::pair<const NotConstructible, NotConstructible> V;
+int main(int, char**) {
+  typedef std::pair<const NotConstructible, NotConstructible> V;
 
-    test(test_allocator<V>(10));
-    test(min_allocator<V>());
-    test(explicit_allocator<V>());
+  test(test_allocator<V>(10));
+  test(min_allocator<V>());
+  test(explicit_allocator<V>());
 
-    return 0;
+  return 0;
 }
diff --git a/libcxx/test/std/containers/unord/unord.map/unord.map.cnstr/size_hash.pass.cpp b/libcxx/test/std/containers/unord/unord.map/unord.map.cnstr/size_hash.pass.cpp
index ee0cf5dc4ebe4..1b78599b087d4 100644
--- a/libcxx/test/std/containers/unord/unord.map/unord.map.cnstr/size_hash.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.map/unord.map.cnstr/size_hash.pass.cpp
@@ -24,51 +24,44 @@
 #include "test_allocator.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef std::unordered_map<NotConstructible, NotConstructible,
-                                   test_hash<NotConstructible>,
-                                   test_equal_to<NotConstructible>,
-                                   test_allocator<std::pair<const NotConstructible,
-                                                                  NotConstructible> >
-                                   > C;
-        C c(7,
-            test_hash<NotConstructible>(8)
-           );
-        LIBCPP_ASSERT(c.bucket_count() == 7);
-        assert(c.hash_function() == test_hash<NotConstructible>(8));
-        assert(c.key_eq() == test_equal_to<NotConstructible>());
-        assert(c.get_allocator() ==
-               (test_allocator<std::pair<const NotConstructible, NotConstructible> >()));
-        assert(c.size() == 0);
-        assert(c.empty());
-        assert(std::distance(c.begin(), c.end()) == 0);
-        assert(c.load_factor() == 0);
-        assert(c.max_load_factor() == 1);
-    }
+int main(int, char**) {
+  {
+    typedef std::unordered_map<NotConstructible,
+                               NotConstructible,
+                               test_hash<NotConstructible>,
+                               test_equal_to<NotConstructible>,
+                               test_allocator<std::pair<const NotConstructible, NotConstructible> > >
+        C;
+    C c(7, test_hash<NotConstructible>(8));
+    LIBCPP_ASSERT(c.bucket_count() == 7);
+    assert(c.hash_function() == test_hash<NotConstructible>(8));
+    assert(c.key_eq() == test_equal_to<NotConstructible>());
+    assert(c.get_allocator() == (test_allocator<std::pair<const NotConstructible, NotConstructible> >()));
+    assert(c.size() == 0);
+    assert(c.empty());
+    assert(std::distance(c.begin(), c.end()) == 0);
+    assert(c.load_factor() == 0);
+    assert(c.max_load_factor() == 1);
+  }
 #if TEST_STD_VER >= 11
-    {
-        typedef std::unordered_map<NotConstructible, NotConstructible,
-                                   test_hash<NotConstructible>,
-                                   test_equal_to<NotConstructible>,
-                                   min_allocator<std::pair<const NotConstructible,
-                                                                 NotConstructible> >
-                                   > C;
-        C c(7,
-            test_hash<NotConstructible>(8)
-           );
-        LIBCPP_ASSERT(c.bucket_count() == 7);
-        assert(c.hash_function() == test_hash<NotConstructible>(8));
-        assert(c.key_eq() == test_equal_to<NotConstructible>());
-        assert(c.get_allocator() ==
-               (min_allocator<std::pair<const NotConstructible, NotConstructible> >()));
-        assert(c.size() == 0);
-        assert(c.empty());
-        assert(std::distance(c.begin(), c.end()) == 0);
-        assert(c.load_factor() == 0);
-        assert(c.max_load_factor() == 1);
-    }
+  {
+    typedef std::unordered_map<NotConstructible,
+                               NotConstructible,
+                               test_hash<NotConstructible>,
+                               test_equal_to<NotConstructible>,
+                               min_allocator<std::pair<const NotConstructible, NotConstructible> > >
+        C;
+    C c(7, test_hash<NotConstructible>(8));
+    LIBCPP_ASSERT(c.bucket_count() == 7);
+    assert(c.hash_function() == test_hash<NotConstructible>(8));
+    assert(c.key_eq() == test_equal_to<NotConstructible>());
+    assert(c.get_allocator() == (min_allocator<std::pair<const NotConstructible, NotConstructible> >()));
+    assert(c.size() == 0);
+    assert(c.empty());
+    assert(std::distance(c.begin(), c.end()) == 0);
+    assert(c.load_factor() == 0);
+    assert(c.max_load_factor() == 1);
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/unord/unord.map/unord.map.cnstr/size_hash_allocator.pass.cpp b/libcxx/test/std/containers/unord/unord.map/unord.map.cnstr/size_hash_allocator.pass.cpp
index 219d1466c55e1..a9f73f3f45fe2 100644
--- a/libcxx/test/std/containers/unord/unord.map/unord.map.cnstr/size_hash_allocator.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.map/unord.map.cnstr/size_hash_allocator.pass.cpp
@@ -29,31 +29,31 @@
 
 template <class Allocator>
 void test(const Allocator& alloc) {
-    typedef std::unordered_map<NotConstructible, NotConstructible,
-                               test_hash<NotConstructible>,
-                               test_equal_to<NotConstructible>,
-                               Allocator
-                               > C;
-
-    C c(7, test_hash<NotConstructible>(8), alloc);
-    LIBCPP_ASSERT(c.bucket_count() == 7);
-    assert(c.hash_function() == test_hash<NotConstructible>(8));
-    assert(c.key_eq() == test_equal_to<NotConstructible>());
-    assert(c.get_allocator() == alloc);
-    assert(c.size() == 0);
-    assert(c.empty());
-    assert(std::distance(c.begin(), c.end()) == 0);
-    assert(c.load_factor() == 0);
-    assert(c.max_load_factor() == 1);
+  typedef std::unordered_map<NotConstructible,
+                             NotConstructible,
+                             test_hash<NotConstructible>,
+                             test_equal_to<NotConstructible>,
+                             Allocator >
+      C;
+
+  C c(7, test_hash<NotConstructible>(8), alloc);
+  LIBCPP_ASSERT(c.bucket_count() == 7);
+  assert(c.hash_function() == test_hash<NotConstructible>(8));
+  assert(c.key_eq() == test_equal_to<NotConstructible>());
+  assert(c.get_allocator() == alloc);
+  assert(c.size() == 0);
+  assert(c.empty());
+  assert(std::distance(c.begin(), c.end()) == 0);
+  assert(c.load_factor() == 0);
+  assert(c.max_load_factor() == 1);
 }
 
-int main(int, char**)
-{
-    typedef std::pair<const NotConstructible, NotConstructible> V;
+int main(int, char**) {
+  typedef std::pair<const NotConstructible, NotConstructible> V;
 
-    test(test_allocator<V>(10));
-    test(min_allocator<V>());
-    test(explicit_allocator<V>());
+  test(test_allocator<V>(10));
+  test(min_allocator<V>());
+  test(explicit_allocator<V>());
 
-    return 0;
+  return 0;
 }
diff --git a/libcxx/test/std/containers/unord/unord.map/unord.map.cnstr/size_hash_equal.pass.cpp b/libcxx/test/std/containers/unord/unord.map/unord.map.cnstr/size_hash_equal.pass.cpp
index 1c81b0e4af2cc..5410be404c88a 100644
--- a/libcxx/test/std/containers/unord/unord.map/unord.map.cnstr/size_hash_equal.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.map/unord.map.cnstr/size_hash_equal.pass.cpp
@@ -24,53 +24,44 @@
 #include "test_allocator.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef std::unordered_map<NotConstructible, NotConstructible,
-                                   test_hash<NotConstructible>,
-                                   test_equal_to<NotConstructible>,
-                                   test_allocator<std::pair<const NotConstructible,
-                                                                  NotConstructible> >
-                                   > C;
-        C c(7,
-            test_hash<NotConstructible>(8),
-            test_equal_to<NotConstructible>(9)
-           );
-        LIBCPP_ASSERT(c.bucket_count() == 7);
-        assert(c.hash_function() == test_hash<NotConstructible>(8));
-        assert(c.key_eq() == test_equal_to<NotConstructible>(9));
-        assert(c.get_allocator() ==
-               (test_allocator<std::pair<const NotConstructible, NotConstructible> >()));
-        assert(c.size() == 0);
-        assert(c.empty());
-        assert(std::distance(c.begin(), c.end()) == 0);
-        assert(c.load_factor() == 0);
-        assert(c.max_load_factor() == 1);
-    }
+int main(int, char**) {
+  {
+    typedef std::unordered_map<NotConstructible,
+                               NotConstructible,
+                               test_hash<NotConstructible>,
+                               test_equal_to<NotConstructible>,
+                               test_allocator<std::pair<const NotConstructible, NotConstructible> > >
+        C;
+    C c(7, test_hash<NotConstructible>(8), test_equal_to<NotConstructible>(9));
+    LIBCPP_ASSERT(c.bucket_count() == 7);
+    assert(c.hash_function() == test_hash<NotConstructible>(8));
+    assert(c.key_eq() == test_equal_to<NotConstructible>(9));
+    assert(c.get_allocator() == (test_allocator<std::pair<const NotConstructible, NotConstructible> >()));
+    assert(c.size() == 0);
+    assert(c.empty());
+    assert(std::distance(c.begin(), c.end()) == 0);
+    assert(c.load_factor() == 0);
+    assert(c.max_load_factor() == 1);
+  }
 #if TEST_STD_VER >= 11
-    {
-        typedef std::unordered_map<NotConstructible, NotConstructible,
-                                   test_hash<NotConstructible>,
-                                   test_equal_to<NotConstructible>,
-                                   min_allocator<std::pair<const NotConstructible,
-                                                                 NotConstructible> >
-                                   > C;
-        C c(7,
-            test_hash<NotConstructible>(8),
-            test_equal_to<NotConstructible>(9)
-           );
-        LIBCPP_ASSERT(c.bucket_count() == 7);
-        assert(c.hash_function() == test_hash<NotConstructible>(8));
-        assert(c.key_eq() == test_equal_to<NotConstructible>(9));
-        assert(c.get_allocator() ==
-               (min_allocator<std::pair<const NotConstructible, NotConstructible> >()));
-        assert(c.size() == 0);
-        assert(c.empty());
-        assert(std::distance(c.begin(), c.end()) == 0);
-        assert(c.load_factor() == 0);
-        assert(c.max_load_factor() == 1);
-    }
+  {
+    typedef std::unordered_map<NotConstructible,
+                               NotConstructible,
+                               test_hash<NotConstructible>,
+                               test_equal_to<NotConstructible>,
+                               min_allocator<std::pair<const NotConstructible, NotConstructible> > >
+        C;
+    C c(7, test_hash<NotConstructible>(8), test_equal_to<NotConstructible>(9));
+    LIBCPP_ASSERT(c.bucket_count() == 7);
+    assert(c.hash_function() == test_hash<NotConstructible>(8));
+    assert(c.key_eq() == test_equal_to<NotConstructible>(9));
+    assert(c.get_allocator() == (min_allocator<std::pair<const NotConstructible, NotConstructible> >()));
+    assert(c.size() == 0);
+    assert(c.empty());
+    assert(std::distance(c.begin(), c.end()) == 0);
+    assert(c.load_factor() == 0);
+    assert(c.max_load_factor() == 1);
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/unord/unord.map/unord.map.cnstr/size_hash_equal_allocator.pass.cpp b/libcxx/test/std/containers/unord/unord.map/unord.map.cnstr/size_hash_equal_allocator.pass.cpp
index 3a329c9962ce4..324aa0cddb77e 100644
--- a/libcxx/test/std/containers/unord/unord.map/unord.map.cnstr/size_hash_equal_allocator.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.map/unord.map.cnstr/size_hash_equal_allocator.pass.cpp
@@ -24,77 +24,69 @@
 #include "test_allocator.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef std::unordered_map<NotConstructible, NotConstructible,
-                                   test_hash<NotConstructible>,
-                                   test_equal_to<NotConstructible>,
-                                   test_allocator<std::pair<const NotConstructible,
-                                                                  NotConstructible> >
-                                   > C;
-        C c(7,
-            test_hash<NotConstructible>(8),
-            test_equal_to<NotConstructible>(9),
-            test_allocator<std::pair<const NotConstructible, NotConstructible> >(10)
-           );
-        LIBCPP_ASSERT(c.bucket_count() == 7);
-        assert(c.hash_function() == test_hash<NotConstructible>(8));
-        assert(c.key_eq() == test_equal_to<NotConstructible>(9));
-        assert(c.get_allocator() ==
-               (test_allocator<std::pair<const NotConstructible, NotConstructible> >(10)));
-        assert(c.size() == 0);
-        assert(c.empty());
-        assert(std::distance(c.begin(), c.end()) == 0);
-        assert(c.load_factor() == 0);
-        assert(c.max_load_factor() == 1);
-    }
+int main(int, char**) {
+  {
+    typedef std::unordered_map<NotConstructible,
+                               NotConstructible,
+                               test_hash<NotConstructible>,
+                               test_equal_to<NotConstructible>,
+                               test_allocator<std::pair<const NotConstructible, NotConstructible> > >
+        C;
+    C c(7,
+        test_hash<NotConstructible>(8),
+        test_equal_to<NotConstructible>(9),
+        test_allocator<std::pair<const NotConstructible, NotConstructible> >(10));
+    LIBCPP_ASSERT(c.bucket_count() == 7);
+    assert(c.hash_function() == test_hash<NotConstructible>(8));
+    assert(c.key_eq() == test_equal_to<NotConstructible>(9));
+    assert(c.get_allocator() == (test_allocator<std::pair<const NotConstructible, NotConstructible> >(10)));
+    assert(c.size() == 0);
+    assert(c.empty());
+    assert(std::distance(c.begin(), c.end()) == 0);
+    assert(c.load_factor() == 0);
+    assert(c.max_load_factor() == 1);
+  }
 #if TEST_STD_VER >= 11
-    {
-        typedef std::unordered_map<NotConstructible, NotConstructible,
-                                   test_hash<NotConstructible>,
-                                   test_equal_to<NotConstructible>,
-                                   min_allocator<std::pair<const NotConstructible,
-                                                                 NotConstructible> >
-                                   > C;
-        C c(7,
-            test_hash<NotConstructible>(8),
-            test_equal_to<NotConstructible>(9),
-            min_allocator<std::pair<const NotConstructible, NotConstructible> >()
-           );
-        LIBCPP_ASSERT(c.bucket_count() == 7);
-        assert(c.hash_function() == test_hash<NotConstructible>(8));
-        assert(c.key_eq() == test_equal_to<NotConstructible>(9));
-        assert(c.get_allocator() ==
-               (min_allocator<std::pair<const NotConstructible, NotConstructible> >()));
-        assert(c.size() == 0);
-        assert(c.empty());
-        assert(std::distance(c.begin(), c.end()) == 0);
-        assert(c.load_factor() == 0);
-        assert(c.max_load_factor() == 1);
-    }
-    {
-        typedef explicit_allocator<std::pair<const NotConstructible, NotConstructible> > A;
-        typedef std::unordered_map<NotConstructible, NotConstructible,
-                                   test_hash<NotConstructible>,
-                                   test_equal_to<NotConstructible>,
-                                   A
-                                   > C;
-        C c(7,
-            test_hash<NotConstructible>(8),
-            test_equal_to<NotConstructible>(9),
-            A{}
-           );
-        LIBCPP_ASSERT(c.bucket_count() == 7);
-        assert(c.hash_function() == test_hash<NotConstructible>(8));
-        assert(c.key_eq() == test_equal_to<NotConstructible>(9));
-        assert(c.get_allocator() == A{});
-        assert(c.size() == 0);
-        assert(c.empty());
-        assert(std::distance(c.begin(), c.end()) == 0);
-        assert(c.load_factor() == 0);
-        assert(c.max_load_factor() == 1);
-    }
+  {
+    typedef std::unordered_map<NotConstructible,
+                               NotConstructible,
+                               test_hash<NotConstructible>,
+                               test_equal_to<NotConstructible>,
+                               min_allocator<std::pair<const NotConstructible, NotConstructible> > >
+        C;
+    C c(7,
+        test_hash<NotConstructible>(8),
+        test_equal_to<NotConstructible>(9),
+        min_allocator<std::pair<const NotConstructible, NotConstructible> >());
+    LIBCPP_ASSERT(c.bucket_count() == 7);
+    assert(c.hash_function() == test_hash<NotConstructible>(8));
+    assert(c.key_eq() == test_equal_to<NotConstructible>(9));
+    assert(c.get_allocator() == (min_allocator<std::pair<const NotConstructible, NotConstructible> >()));
+    assert(c.size() == 0);
+    assert(c.empty());
+    assert(std::distance(c.begin(), c.end()) == 0);
+    assert(c.load_factor() == 0);
+    assert(c.max_load_factor() == 1);
+  }
+  {
+    typedef explicit_allocator<std::pair<const NotConstructible, NotConstructible> > A;
+    typedef std::unordered_map<NotConstructible,
+                               NotConstructible,
+                               test_hash<NotConstructible>,
+                               test_equal_to<NotConstructible>,
+                               A >
+        C;
+    C c(7, test_hash<NotConstructible>(8), test_equal_to<NotConstructible>(9), A{});
+    LIBCPP_ASSERT(c.bucket_count() == 7);
+    assert(c.hash_function() == test_hash<NotConstructible>(8));
+    assert(c.key_eq() == test_equal_to<NotConstructible>(9));
+    assert(c.get_allocator() == A{});
+    assert(c.size() == 0);
+    assert(c.empty());
+    assert(std::distance(c.begin(), c.end()) == 0);
+    assert(c.load_factor() == 0);
+    assert(c.max_load_factor() == 1);
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/unord/unord.map/unord.map.elem/at.pass.cpp b/libcxx/test/std/containers/unord/unord.map/unord.map.elem/at.pass.cpp
index fe6ef8bc7e85c..a08cba29086f1 100644
--- a/libcxx/test/std/containers/unord/unord.map/unord.map.elem/at.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.map/unord.map.elem/at.pass.cpp
@@ -24,121 +24,112 @@
 #include "min_allocator.h"
 #include "test_macros.h"
 
-int main(int, char**)
-{
-    {
-        typedef std::unordered_map<int, std::string> C;
-        typedef std::pair<int, std::string> P;
-        P a[] =
-        {
-            P(1, "one"),
-            P(2, "two"),
-            P(3, "three"),
-            P(4, "four"),
-            P(1, "four"),
-            P(2, "four"),
-        };
-        C c(a, a + sizeof(a)/sizeof(a[0]));
-        assert(c.size() == 4);
-        c.at(1) = "ONE";
-        assert(c.at(1) == "ONE");
+int main(int, char**) {
+  {
+    typedef std::unordered_map<int, std::string> C;
+    typedef std::pair<int, std::string> P;
+    P a[] = {
+        P(1, "one"),
+        P(2, "two"),
+        P(3, "three"),
+        P(4, "four"),
+        P(1, "four"),
+        P(2, "four"),
+    };
+    C c(a, a + sizeof(a) / sizeof(a[0]));
+    assert(c.size() == 4);
+    c.at(1) = "ONE";
+    assert(c.at(1) == "ONE");
 #ifndef TEST_HAS_NO_EXCEPTIONS
-        try
-        {
-            c.at(11) = "eleven";
-            assert(false);
-        }
-        catch (std::out_of_range&)
-        {
-        }
-        assert(c.size() == 4);
-#endif
+    try {
+      c.at(11) = "eleven";
+      assert(false);
+    } catch (std::out_of_range&) {
     }
-    {
-        typedef std::unordered_map<int, std::string> C;
-        typedef std::pair<int, std::string> P;
-        P a[] =
-        {
-            P(1, "one"),
-            P(2, "two"),
-            P(3, "three"),
-            P(4, "four"),
-            P(1, "four"),
-            P(2, "four"),
-        };
-        const C c(a, a + sizeof(a)/sizeof(a[0]));
-        assert(c.size() == 4);
-        assert(c.at(1) == "one");
-#ifndef TEST_HAS_NO_EXCEPTIONS
-        try
-        {
-            TEST_IGNORE_NODISCARD c.at(11);
-            assert(false);
-        }
-        catch (std::out_of_range&)
-        {
-        }
-        assert(c.size() == 4);
+    assert(c.size() == 4);
 #endif
-    }
-#if TEST_STD_VER >= 11
-    {
-        typedef std::unordered_map<int, std::string, std::hash<int>, std::equal_to<int>,
-                            min_allocator<std::pair<const int, std::string>>> C;
-        typedef std::pair<int, std::string> P;
-        P a[] =
-        {
-            P(1, "one"),
-            P(2, "two"),
-            P(3, "three"),
-            P(4, "four"),
-            P(1, "four"),
-            P(2, "four"),
-        };
-        C c(a, a + sizeof(a)/sizeof(a[0]));
-        assert(c.size() == 4);
-        c.at(1) = "ONE";
-        assert(c.at(1) == "ONE");
+  }
+  {
+    typedef std::unordered_map<int, std::string> C;
+    typedef std::pair<int, std::string> P;
+    P a[] = {
+        P(1, "one"),
+        P(2, "two"),
+        P(3, "three"),
+        P(4, "four"),
+        P(1, "four"),
+        P(2, "four"),
+    };
+    const C c(a, a + sizeof(a) / sizeof(a[0]));
+    assert(c.size() == 4);
+    assert(c.at(1) == "one");
 #ifndef TEST_HAS_NO_EXCEPTIONS
-        try
-        {
-            c.at(11) = "eleven";
-            assert(false);
-        }
-        catch (std::out_of_range&)
-        {
-        }
-        assert(c.size() == 4);
-#endif
+    try {
+      TEST_IGNORE_NODISCARD c.at(11);
+      assert(false);
+    } catch (std::out_of_range&) {
     }
-    {
-        typedef std::unordered_map<int, std::string, std::hash<int>, std::equal_to<int>,
-                            min_allocator<std::pair<const int, std::string>>> C;
-        typedef std::pair<int, std::string> P;
-        P a[] =
-        {
-            P(1, "one"),
-            P(2, "two"),
-            P(3, "three"),
-            P(4, "four"),
-            P(1, "four"),
-            P(2, "four"),
-        };
-        const C c(a, a + sizeof(a)/sizeof(a[0]));
-        assert(c.size() == 4);
-        assert(c.at(1) == "one");
-#ifndef TEST_HAS_NO_EXCEPTIONS
-        try
-        {
-            TEST_IGNORE_NODISCARD c.at(11);
-            assert(false);
-        }
-        catch (std::out_of_range&)
-        {
-        }
-        assert(c.size() == 4);
+    assert(c.size() == 4);
 #endif
+  }
+#if TEST_STD_VER >= 11
+  {
+    typedef std::unordered_map<int,
+                               std::string,
+                               std::hash<int>,
+                               std::equal_to<int>,
+                               min_allocator<std::pair<const int, std::string>>>
+        C;
+    typedef std::pair<int, std::string> P;
+    P a[] = {
+        P(1, "one"),
+        P(2, "two"),
+        P(3, "three"),
+        P(4, "four"),
+        P(1, "four"),
+        P(2, "four"),
+    };
+    C c(a, a + sizeof(a) / sizeof(a[0]));
+    assert(c.size() == 4);
+    c.at(1) = "ONE";
+    assert(c.at(1) == "ONE");
+#  ifndef TEST_HAS_NO_EXCEPTIONS
+    try {
+      c.at(11) = "eleven";
+      assert(false);
+    } catch (std::out_of_range&) {
+    }
+    assert(c.size() == 4);
+#  endif
+  }
+  {
+    typedef std::unordered_map<int,
+                               std::string,
+                               std::hash<int>,
+                               std::equal_to<int>,
+                               min_allocator<std::pair<const int, std::string>>>
+        C;
+    typedef std::pair<int, std::string> P;
+    P a[] = {
+        P(1, "one"),
+        P(2, "two"),
+        P(3, "three"),
+        P(4, "four"),
+        P(1, "four"),
+        P(2, "four"),
+    };
+    const C c(a, a + sizeof(a) / sizeof(a[0]));
+    assert(c.size() == 4);
+    assert(c.at(1) == "one");
+#  ifndef TEST_HAS_NO_EXCEPTIONS
+    try {
+      TEST_IGNORE_NODISCARD c.at(11);
+      assert(false);
+    } catch (std::out_of_range&) {
     }
+    assert(c.size() == 4);
+#  endif
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/unord/unord.map/unord.map.elem/index.pass.cpp b/libcxx/test/std/containers/unord/unord.map/unord.map.elem/index.pass.cpp
index 9ca0edb076731..a9f9fb7ae84f8 100644
--- a/libcxx/test/std/containers/unord/unord.map/unord.map.elem/index.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.map/unord.map.elem/index.pass.cpp
@@ -25,139 +25,142 @@
 #include "count_new.h"
 
 #if TEST_STD_VER >= 11
-#include "container_test_types.h"
+#  include "container_test_types.h"
 #endif
 
-int main(int, char**)
-{
-    {
-        typedef std::unordered_map<int, std::string> C;
-        typedef std::pair<int, std::string> P;
-        P a[] =
-        {
-            P(1, "one"),
-            P(2, "two"),
-            P(3, "three"),
-            P(4, "four"),
-            P(1, "four"),
-            P(2, "four"),
-        };
-        C c(a, a + sizeof(a)/sizeof(a[0]));
-        assert(c.size() == 4);
-        c[1] = "ONE";
-        assert(c.at(1) == "ONE");
-        c[11] = "eleven";
-        assert(c.size() == 5);
-        assert(c.at(11) == "eleven");
-    }
+int main(int, char**) {
+  {
+    typedef std::unordered_map<int, std::string> C;
+    typedef std::pair<int, std::string> P;
+    P a[] = {
+        P(1, "one"),
+        P(2, "two"),
+        P(3, "three"),
+        P(4, "four"),
+        P(1, "four"),
+        P(2, "four"),
+    };
+    C c(a, a + sizeof(a) / sizeof(a[0]));
+    assert(c.size() == 4);
+    c[1] = "ONE";
+    assert(c.at(1) == "ONE");
+    c[11] = "eleven";
+    assert(c.size() == 5);
+    assert(c.at(11) == "eleven");
+  }
 #if TEST_STD_VER >= 11
+  {
+    typedef std::unordered_map<MoveOnly, std::string> C;
+    typedef std::pair<int, std::string> P;
+    P a[] = {
+        P(1, "one"),
+        P(2, "two"),
+        P(3, "three"),
+        P(4, "four"),
+        P(1, "four"),
+        P(2, "four"),
+    };
+    C c(a, a + sizeof(a) / sizeof(a[0]));
+    assert(c.size() == 4);
+    c[1] = "ONE";
+    assert(c.at(1) == "ONE");
+    c[11] = "eleven";
+    assert(c.size() == 5);
+    assert(c.at(11) == "eleven");
+  }
+  {
+    typedef std::unordered_map<int,
+                               std::string,
+                               std::hash<int>,
+                               std::equal_to<int>,
+                               min_allocator<std::pair<const int, std::string>>>
+        C;
+    typedef std::pair<int, std::string> P;
+    P a[] = {
+        P(1, "one"),
+        P(2, "two"),
+        P(3, "three"),
+        P(4, "four"),
+        P(1, "four"),
+        P(2, "four"),
+    };
+    C c(a, a + sizeof(a) / sizeof(a[0]));
+    assert(c.size() == 4);
+    c[1] = "ONE";
+    assert(c.at(1) == "ONE");
+    c[11] = "eleven";
+    assert(c.size() == 5);
+    assert(c.at(11) == "eleven");
+  }
+
+  {
+    typedef std::unordered_map<MoveOnly,
+                               std::string,
+                               std::hash<MoveOnly>,
+                               std::equal_to<MoveOnly>,
+                               min_allocator<std::pair<const MoveOnly, std::string>>>
+        C;
+    typedef std::pair<int, std::string> P;
+    P a[] = {
+        P(1, "one"),
+        P(2, "two"),
+        P(3, "three"),
+        P(4, "four"),
+        P(1, "four"),
+        P(2, "four"),
+    };
+    C c(a, a + sizeof(a) / sizeof(a[0]));
+    assert(c.size() == 4);
+    c[1] = "ONE";
+    assert(c.at(1) == "ONE");
+    c[11] = "eleven";
+    assert(c.size() == 5);
+    assert(c.at(11) == "eleven");
+  }
+  {
+    using Container         = TCT::unordered_map<>;
+    using Key               = Container::key_type;
+    using MappedType        = Container::mapped_type;
+    ConstructController* cc = getConstructController();
+    cc->reset();
     {
-        typedef std::unordered_map<MoveOnly, std::string> C;
-        typedef std::pair<int, std::string> P;
-        P a[] =
-        {
-            P(1, "one"),
-            P(2, "two"),
-            P(3, "three"),
-            P(4, "four"),
-            P(1, "four"),
-            P(2, "four"),
-        };
-        C c(a, a + sizeof(a)/sizeof(a[0]));
-        assert(c.size() == 4);
-        c[1] = "ONE";
-        assert(c.at(1) == "ONE");
-        c[11] = "eleven";
-        assert(c.size() == 5);
-        assert(c.at(11) == "eleven");
-    }
-    {
-        typedef std::unordered_map<int, std::string, std::hash<int>, std::equal_to<int>,
-                            min_allocator<std::pair<const int, std::string>>> C;
-        typedef std::pair<int, std::string> P;
-        P a[] =
-        {
-            P(1, "one"),
-            P(2, "two"),
-            P(3, "three"),
-            P(4, "four"),
-            P(1, "four"),
-            P(2, "four"),
-        };
-        C c(a, a + sizeof(a)/sizeof(a[0]));
-        assert(c.size() == 4);
-        c[1] = "ONE";
-        assert(c.at(1) == "ONE");
-        c[11] = "eleven";
-        assert(c.size() == 5);
-        assert(c.at(11) == "eleven");
+      Container c;
+      const Key k(1);
+      cc->expect<std::piecewise_construct_t const&, std::tuple<Key const&>&&, std::tuple<>&&>();
+      MappedType& mref = c[k];
+      assert(!cc->unchecked());
+      {
+        DisableAllocationGuard g;
+        MappedType& mref2 = c[k];
+        assert(&mref == &mref2);
+      }
     }
-
     {
-        typedef std::unordered_map<MoveOnly, std::string, std::hash<MoveOnly>, std::equal_to<MoveOnly>,
-                            min_allocator<std::pair<const MoveOnly, std::string>>> C;
-        typedef std::pair<int, std::string> P;
-        P a[] =
-        {
-            P(1, "one"),
-            P(2, "two"),
-            P(3, "three"),
-            P(4, "four"),
-            P(1, "four"),
-            P(2, "four"),
-        };
-        C c(a, a + sizeof(a)/sizeof(a[0]));
-        assert(c.size() == 4);
-        c[1] = "ONE";
-        assert(c.at(1) == "ONE");
-        c[11] = "eleven";
-        assert(c.size() == 5);
-        assert(c.at(11) == "eleven");
+      Container c;
+      Key k(1);
+      cc->expect<std::piecewise_construct_t const&, std::tuple<Key const&>&&, std::tuple<>&&>();
+      MappedType& mref = c[k];
+      assert(!cc->unchecked());
+      {
+        DisableAllocationGuard g;
+        MappedType& mref2 = c[k];
+        assert(&mref == &mref2);
+      }
     }
     {
-        using Container = TCT::unordered_map<>;
-        using Key = Container::key_type;
-        using MappedType = Container::mapped_type;
-        ConstructController* cc = getConstructController();
-        cc->reset();
-        {
-            Container c;
-            const Key k(1);
-            cc->expect<std::piecewise_construct_t const&, std::tuple<Key const&>&&, std::tuple<>&&>();
-            MappedType& mref = c[k];
-            assert(!cc->unchecked());
-            {
-                DisableAllocationGuard g;
-                MappedType& mref2 = c[k];
-                assert(&mref == &mref2);
-            }
-        }
-        {
-            Container c;
-            Key k(1);
-            cc->expect<std::piecewise_construct_t const&, std::tuple<Key const&>&&, std::tuple<>&&>();
-            MappedType& mref = c[k];
-            assert(!cc->unchecked());
-            {
-                DisableAllocationGuard g;
-                MappedType& mref2 = c[k];
-                assert(&mref == &mref2);
-            }
-        }
-        {
-            Container c;
-            Key k(1);
-            cc->expect<std::piecewise_construct_t const&, std::tuple<Key &&>&&, std::tuple<>&&>();
-            MappedType& mref = c[std::move(k)];
-            assert(!cc->unchecked());
-            {
-                Key k2(1);
-                DisableAllocationGuard g;
-                MappedType& mref2 = c[std::move(k2)];
-                assert(&mref == &mref2);
-            }
-        }
+      Container c;
+      Key k(1);
+      cc->expect<std::piecewise_construct_t const&, std::tuple<Key&&>&&, std::tuple<>&&>();
+      MappedType& mref = c[std::move(k)];
+      assert(!cc->unchecked());
+      {
+        Key k2(1);
+        DisableAllocationGuard g;
+        MappedType& mref2 = c[std::move(k2)];
+        assert(&mref == &mref2);
+      }
     }
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/unord/unord.map/unord.map.elem/index_tuple.pass.cpp b/libcxx/test/std/containers/unord/unord.map/unord.map.elem/index_tuple.pass.cpp
index 78b5fbd36fe39..ac7bf64125333 100644
--- a/libcxx/test/std/containers/unord/unord.map/unord.map.elem/index_tuple.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.map/unord.map.elem/index_tuple.pass.cpp
@@ -23,12 +23,12 @@
 #include <unordered_map>
 
 struct my_hash {
-    std::size_t operator()(const std::tuple<int, int>&) const { return 0; }
+  std::size_t operator()(const std::tuple<int, int>&) const { return 0; }
 };
 
 int main(int, char**) {
-    std::unordered_map<std::tuple<int, int>, std::size_t, my_hash> m;
-    m[std::make_tuple(2, 3)] = 7;
+  std::unordered_map<std::tuple<int, int>, std::size_t, my_hash> m;
+  m[std::make_tuple(2, 3)] = 7;
 
-    return 0;
+  return 0;
 }
diff --git a/libcxx/test/std/containers/unord/unord.map/unord.map.modifiers/clear.pass.cpp b/libcxx/test/std/containers/unord/unord.map/unord.map.modifiers/clear.pass.cpp
index 64fe72ecdacdb..9d2f23a2d789a 100644
--- a/libcxx/test/std/containers/unord/unord.map/unord.map.modifiers/clear.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.map/unord.map.modifiers/clear.pass.cpp
@@ -21,44 +21,45 @@
 #include "test_macros.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef std::unordered_map<int, std::string> C;
-        typedef std::pair<int, std::string> P;
-        P a[] =
-        {
-            P(1, "one"),
-            P(2, "two"),
-            P(3, "three"),
-            P(4, "four"),
-            P(1, "four"),
-            P(2, "four"),
-        };
-        C c(a, a + sizeof(a)/sizeof(a[0]));
-        ASSERT_NOEXCEPT(c.clear());
-        c.clear();
-        assert(c.size() == 0);
-    }
+int main(int, char**) {
+  {
+    typedef std::unordered_map<int, std::string> C;
+    typedef std::pair<int, std::string> P;
+    P a[] = {
+        P(1, "one"),
+        P(2, "two"),
+        P(3, "three"),
+        P(4, "four"),
+        P(1, "four"),
+        P(2, "four"),
+    };
+    C c(a, a + sizeof(a) / sizeof(a[0]));
+    ASSERT_NOEXCEPT(c.clear());
+    c.clear();
+    assert(c.size() == 0);
+  }
 #if TEST_STD_VER >= 11
-    {
-        typedef std::unordered_map<int, std::string, std::hash<int>, std::equal_to<int>,
-                            min_allocator<std::pair<const int, std::string>>> C;
-        typedef std::pair<int, std::string> P;
-        P a[] =
-        {
-            P(1, "one"),
-            P(2, "two"),
-            P(3, "three"),
-            P(4, "four"),
-            P(1, "four"),
-            P(2, "four"),
-        };
-        C c(a, a + sizeof(a)/sizeof(a[0]));
-        ASSERT_NOEXCEPT(c.clear());
-        c.clear();
-        assert(c.size() == 0);
-    }
+  {
+    typedef std::unordered_map<int,
+                               std::string,
+                               std::hash<int>,
+                               std::equal_to<int>,
+                               min_allocator<std::pair<const int, std::string>>>
+        C;
+    typedef std::pair<int, std::string> P;
+    P a[] = {
+        P(1, "one"),
+        P(2, "two"),
+        P(3, "three"),
+        P(4, "four"),
+        P(1, "four"),
+        P(2, "four"),
+    };
+    C c(a, a + sizeof(a) / sizeof(a[0]));
+    ASSERT_NOEXCEPT(c.clear());
+    c.clear();
+    assert(c.size() == 0);
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/unord/unord.map/unord.map.modifiers/emplace.pass.cpp b/libcxx/test/std/containers/unord/unord.map/unord.map.modifiers/emplace.pass.cpp
index 11bbaf039b538..5615088fe30cc 100644
--- a/libcxx/test/std/containers/unord/unord.map/unord.map.modifiers/emplace.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.map/unord.map.modifiers/emplace.pass.cpp
@@ -24,57 +24,56 @@
 #include "../../../Emplaceable.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef std::unordered_map<int, Emplaceable> C;
-        typedef std::pair<C::iterator, bool> R;
-        C c;
-        R r = c.emplace(std::piecewise_construct, std::forward_as_tuple(3),
-                                                  std::forward_as_tuple());
-        assert(r.second);
-        assert(c.size() == 1);
-        assert(r.first->first == 3);
-        assert(r.first->second == Emplaceable());
+int main(int, char**) {
+  {
+    typedef std::unordered_map<int, Emplaceable> C;
+    typedef std::pair<C::iterator, bool> R;
+    C c;
+    R r = c.emplace(std::piecewise_construct, std::forward_as_tuple(3), std::forward_as_tuple());
+    assert(r.second);
+    assert(c.size() == 1);
+    assert(r.first->first == 3);
+    assert(r.first->second == Emplaceable());
 
-        r = c.emplace(std::pair<const int, Emplaceable>(4, Emplaceable(5, 6)));
-        assert(r.second);
-        assert(c.size() == 2);
-        assert(r.first->first == 4);
-        assert(r.first->second == Emplaceable(5, 6));
+    r = c.emplace(std::pair<const int, Emplaceable>(4, Emplaceable(5, 6)));
+    assert(r.second);
+    assert(c.size() == 2);
+    assert(r.first->first == 4);
+    assert(r.first->second == Emplaceable(5, 6));
 
-        r = c.emplace(std::piecewise_construct, std::forward_as_tuple(5),
-                                               std::forward_as_tuple(6, 7));
-        assert(r.second);
-        assert(c.size() == 3);
-        assert(r.first->first == 5);
-        assert(r.first->second == Emplaceable(6, 7));
-    }
-    {
-        typedef std::unordered_map<int, Emplaceable, std::hash<int>, std::equal_to<int>,
-                            min_allocator<std::pair<const int, Emplaceable>>> C;
-        typedef std::pair<C::iterator, bool> R;
-        C c;
-        R r = c.emplace(std::piecewise_construct, std::forward_as_tuple(3),
-                                                  std::forward_as_tuple());
-        assert(r.second);
-        assert(c.size() == 1);
-        assert(r.first->first == 3);
-        assert(r.first->second == Emplaceable());
+    r = c.emplace(std::piecewise_construct, std::forward_as_tuple(5), std::forward_as_tuple(6, 7));
+    assert(r.second);
+    assert(c.size() == 3);
+    assert(r.first->first == 5);
+    assert(r.first->second == Emplaceable(6, 7));
+  }
+  {
+    typedef std::unordered_map<int,
+                               Emplaceable,
+                               std::hash<int>,
+                               std::equal_to<int>,
+                               min_allocator<std::pair<const int, Emplaceable>>>
+        C;
+    typedef std::pair<C::iterator, bool> R;
+    C c;
+    R r = c.emplace(std::piecewise_construct, std::forward_as_tuple(3), std::forward_as_tuple());
+    assert(r.second);
+    assert(c.size() == 1);
+    assert(r.first->first == 3);
+    assert(r.first->second == Emplaceable());
 
-        r = c.emplace(std::pair<const int, Emplaceable>(4, Emplaceable(5, 6)));
-        assert(r.second);
-        assert(c.size() == 2);
-        assert(r.first->first == 4);
-        assert(r.first->second == Emplaceable(5, 6));
+    r = c.emplace(std::pair<const int, Emplaceable>(4, Emplaceable(5, 6)));
+    assert(r.second);
+    assert(c.size() == 2);
+    assert(r.first->first == 4);
+    assert(r.first->second == Emplaceable(5, 6));
 
-        r = c.emplace(std::piecewise_construct, std::forward_as_tuple(5),
-                                               std::forward_as_tuple(6, 7));
-        assert(r.second);
-        assert(c.size() == 3);
-        assert(r.first->first == 5);
-        assert(r.first->second == Emplaceable(6, 7));
-    }
+    r = c.emplace(std::piecewise_construct, std::forward_as_tuple(5), std::forward_as_tuple(6, 7));
+    assert(r.second);
+    assert(c.size() == 3);
+    assert(r.first->first == 5);
+    assert(r.first->second == Emplaceable(6, 7));
+  }
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/unord/unord.map/unord.map.modifiers/emplace_hint.pass.cpp b/libcxx/test/std/containers/unord/unord.map/unord.map.modifiers/emplace_hint.pass.cpp
index 4277a1bec2a50..3e41213d1f2e3 100644
--- a/libcxx/test/std/containers/unord/unord.map/unord.map.modifiers/emplace_hint.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.map/unord.map.modifiers/emplace_hint.pass.cpp
@@ -17,7 +17,6 @@
 // template <class... Args>
 //     iterator emplace_hint(const_iterator p, Args&&... args);
 
-
 #include <unordered_map>
 #include <cassert>
 
@@ -25,53 +24,52 @@
 #include "../../../Emplaceable.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef std::unordered_map<int, Emplaceable> C;
-        typedef C::iterator R;
-        C c;
-        C::const_iterator e = c.end();
-        R r = c.emplace_hint(e, std::piecewise_construct, std::forward_as_tuple(3),
-                                                          std::forward_as_tuple());
-        assert(c.size() == 1);
-        assert(r->first == 3);
-        assert(r->second == Emplaceable());
+int main(int, char**) {
+  {
+    typedef std::unordered_map<int, Emplaceable> C;
+    typedef C::iterator R;
+    C c;
+    C::const_iterator e = c.end();
+    R r = c.emplace_hint(e, std::piecewise_construct, std::forward_as_tuple(3), std::forward_as_tuple());
+    assert(c.size() == 1);
+    assert(r->first == 3);
+    assert(r->second == Emplaceable());
 
-        r = c.emplace_hint(c.end(), std::pair<const int, Emplaceable>(4, Emplaceable(5, 6)));
-        assert(c.size() == 2);
-        assert(r->first == 4);
-        assert(r->second == Emplaceable(5, 6));
+    r = c.emplace_hint(c.end(), std::pair<const int, Emplaceable>(4, Emplaceable(5, 6)));
+    assert(c.size() == 2);
+    assert(r->first == 4);
+    assert(r->second == Emplaceable(5, 6));
 
-        r = c.emplace_hint(c.end(), std::piecewise_construct, std::forward_as_tuple(5),
-                                                       std::forward_as_tuple(6, 7));
-        assert(c.size() == 3);
-        assert(r->first == 5);
-        assert(r->second == Emplaceable(6, 7));
-    }
-    {
-        typedef std::unordered_map<int, Emplaceable, std::hash<int>, std::equal_to<int>,
-                            min_allocator<std::pair<const int, Emplaceable>>> C;
-        typedef C::iterator R;
-        C c;
-        C::const_iterator e = c.end();
-        R r = c.emplace_hint(e, std::piecewise_construct, std::forward_as_tuple(3),
-                                                          std::forward_as_tuple());
-        assert(c.size() == 1);
-        assert(r->first == 3);
-        assert(r->second == Emplaceable());
+    r = c.emplace_hint(c.end(), std::piecewise_construct, std::forward_as_tuple(5), std::forward_as_tuple(6, 7));
+    assert(c.size() == 3);
+    assert(r->first == 5);
+    assert(r->second == Emplaceable(6, 7));
+  }
+  {
+    typedef std::unordered_map<int,
+                               Emplaceable,
+                               std::hash<int>,
+                               std::equal_to<int>,
+                               min_allocator<std::pair<const int, Emplaceable>>>
+        C;
+    typedef C::iterator R;
+    C c;
+    C::const_iterator e = c.end();
+    R r = c.emplace_hint(e, std::piecewise_construct, std::forward_as_tuple(3), std::forward_as_tuple());
+    assert(c.size() == 1);
+    assert(r->first == 3);
+    assert(r->second == Emplaceable());
 
-        r = c.emplace_hint(c.end(), std::pair<const int, Emplaceable>(4, Emplaceable(5, 6)));
-        assert(c.size() == 2);
-        assert(r->first == 4);
-        assert(r->second == Emplaceable(5, 6));
+    r = c.emplace_hint(c.end(), std::pair<const int, Emplaceable>(4, Emplaceable(5, 6)));
+    assert(c.size() == 2);
+    assert(r->first == 4);
+    assert(r->second == Emplaceable(5, 6));
 
-        r = c.emplace_hint(c.end(), std::piecewise_construct, std::forward_as_tuple(5),
-                                                       std::forward_as_tuple(6, 7));
-        assert(c.size() == 3);
-        assert(r->first == 5);
-        assert(r->second == Emplaceable(6, 7));
-    }
+    r = c.emplace_hint(c.end(), std::piecewise_construct, std::forward_as_tuple(5), std::forward_as_tuple(6, 7));
+    assert(c.size() == 3);
+    assert(r->first == 5);
+    assert(r->second == Emplaceable(6, 7));
+  }
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/unord/unord.map/unord.map.modifiers/erase_const_iter.pass.cpp b/libcxx/test/std/containers/unord/unord.map/unord.map.modifiers/erase_const_iter.pass.cpp
index 42cc9d1408378..f6ab2411789f0 100644
--- a/libcxx/test/std/containers/unord/unord.map/unord.map.modifiers/erase_const_iter.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.map/unord.map.modifiers/erase_const_iter.pass.cpp
@@ -21,73 +21,75 @@
 #include "test_macros.h"
 #include "min_allocator.h"
 
-struct TemplateConstructor
-{
-    template<typename T>
-    TemplateConstructor (const T&) {}
+struct TemplateConstructor {
+  template <typename T>
+  TemplateConstructor(const T&) {}
 };
 
 bool operator==(const TemplateConstructor&, const TemplateConstructor&) { return false; }
-struct Hash { std::size_t operator() (const TemplateConstructor &) const { return 0; } };
+struct Hash {
+  std::size_t operator()(const TemplateConstructor&) const { return 0; }
+};
 
-int main(int, char**)
-{
-    {
-        typedef std::unordered_map<int, std::string> C;
-        typedef std::pair<int, std::string> P;
-        P a[] =
-        {
-            P(1, "one"),
-            P(2, "two"),
-            P(3, "three"),
-            P(4, "four"),
-            P(1, "four"),
-            P(2, "four"),
-        };
-        C c(a, a + sizeof(a)/sizeof(a[0]));
-        C::const_iterator i = c.find(2);
-        c.erase(i);
-        assert(c.size() == 3);
-        assert(c.at(1) == "one");
-        assert(c.at(3) == "three");
-        assert(c.at(4) == "four");
-    }
+int main(int, char**) {
+  {
+    typedef std::unordered_map<int, std::string> C;
+    typedef std::pair<int, std::string> P;
+    P a[] = {
+        P(1, "one"),
+        P(2, "two"),
+        P(3, "three"),
+        P(4, "four"),
+        P(1, "four"),
+        P(2, "four"),
+    };
+    C c(a, a + sizeof(a) / sizeof(a[0]));
+    C::const_iterator i = c.find(2);
+    c.erase(i);
+    assert(c.size() == 3);
+    assert(c.at(1) == "one");
+    assert(c.at(3) == "three");
+    assert(c.at(4) == "four");
+  }
 #if TEST_STD_VER >= 11
-    {
-        typedef std::unordered_map<int, std::string, std::hash<int>, std::equal_to<int>,
-                            min_allocator<std::pair<const int, std::string>>> C;
-        typedef std::pair<int, std::string> P;
-        P a[] =
-        {
-            P(1, "one"),
-            P(2, "two"),
-            P(3, "three"),
-            P(4, "four"),
-            P(1, "four"),
-            P(2, "four"),
-        };
-        C c(a, a + sizeof(a)/sizeof(a[0]));
-        C::const_iterator i = c.find(2);
-        c.erase(i);
-        assert(c.size() == 3);
-        assert(c.at(1) == "one");
-        assert(c.at(3) == "three");
-        assert(c.at(4) == "four");
-    }
+  {
+    typedef std::unordered_map<int,
+                               std::string,
+                               std::hash<int>,
+                               std::equal_to<int>,
+                               min_allocator<std::pair<const int, std::string>>>
+        C;
+    typedef std::pair<int, std::string> P;
+    P a[] = {
+        P(1, "one"),
+        P(2, "two"),
+        P(3, "three"),
+        P(4, "four"),
+        P(1, "four"),
+        P(2, "four"),
+    };
+    C c(a, a + sizeof(a) / sizeof(a[0]));
+    C::const_iterator i = c.find(2);
+    c.erase(i);
+    assert(c.size() == 3);
+    assert(c.at(1) == "one");
+    assert(c.at(3) == "three");
+    assert(c.at(4) == "four");
+  }
 #endif
 #if TEST_STD_VER >= 14
-    {
+  {
     //  This is LWG #2059
-        typedef TemplateConstructor T;
-        typedef std::unordered_map<T, int, Hash> C;
-        typedef C::iterator I;
+    typedef TemplateConstructor T;
+    typedef std::unordered_map<T, int, Hash> C;
+    typedef C::iterator I;
 
-        C m;
-        T a{0};
-        I it = m.find(a);
-        if (it != m.end())
-            m.erase(it);
-    }
+    C m;
+    T a{0};
+    I it = m.find(a);
+    if (it != m.end())
+      m.erase(it);
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/unord/unord.map/unord.map.modifiers/erase_key.pass.cpp b/libcxx/test/std/containers/unord/unord.map/unord.map.modifiers/erase_key.pass.cpp
index 305d149a58374..e78c48ce5ccfd 100644
--- a/libcxx/test/std/containers/unord/unord.map/unord.map.modifiers/erase_key.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.map/unord.map.modifiers/erase_key.pass.cpp
@@ -23,156 +23,156 @@
 
 #if TEST_STD_VER >= 11
 template <typename Unordered>
-bool only_deletions ( const Unordered &whole, const Unordered &part ) {
-    typename Unordered::const_iterator w = whole.begin();
-    typename Unordered::const_iterator p = part.begin();
+bool only_deletions(const Unordered& whole, const Unordered& part) {
+  typename Unordered::const_iterator w = whole.begin();
+  typename Unordered::const_iterator p = part.begin();
 
-    while ( w != whole.end () && p != part.end()) {
-        if ( *w == *p )
-            p++;
-        w++;
-        }
+  while (w != whole.end() && p != part.end()) {
+    if (*w == *p)
+      p++;
+    w++;
+  }
 
-    return p == part.end();
+  return p == part.end();
 }
 #endif
 
-
-int main(int, char**)
-{
-    {
-        typedef std::unordered_map<int, std::string> C;
-        typedef std::pair<int, std::string> P;
-        P a[] =
-        {
-            P(1, "one"),
-            P(2, "two"),
-            P(3, "three"),
-            P(4, "four"),
-            P(1, "four"),
-            P(2, "four"),
-        };
-        C c(a, a + sizeof(a)/sizeof(a[0]));
-        assert(c.erase(5) == 0);
-        assert(c.size() == 4);
-        assert(c.at(1) == "one");
-        assert(c.at(2) == "two");
-        assert(c.at(3) == "three");
-        assert(c.at(4) == "four");
-
-        assert(c.erase(2) == 1);
-        assert(c.size() == 3);
-        assert(c.at(1) == "one");
-        assert(c.at(3) == "three");
-        assert(c.at(4) == "four");
-
-        assert(c.erase(2) == 0);
-        assert(c.size() == 3);
-        assert(c.at(1) == "one");
-        assert(c.at(3) == "three");
-        assert(c.at(4) == "four");
-
-        assert(c.erase(4) == 1);
-        assert(c.size() == 2);
-        assert(c.at(1) == "one");
-        assert(c.at(3) == "three");
-
-        assert(c.erase(4) == 0);
-        assert(c.size() == 2);
-        assert(c.at(1) == "one");
-        assert(c.at(3) == "three");
-
-        assert(c.erase(1) == 1);
-        assert(c.size() == 1);
-        assert(c.at(3) == "three");
-
-        assert(c.erase(1) == 0);
-        assert(c.size() == 1);
-        assert(c.at(3) == "three");
-
-        assert(c.erase(3) == 1);
-        assert(c.size() == 0);
-
-        assert(c.erase(3) == 0);
-        assert(c.size() == 0);
-    }
+int main(int, char**) {
+  {
+    typedef std::unordered_map<int, std::string> C;
+    typedef std::pair<int, std::string> P;
+    P a[] = {
+        P(1, "one"),
+        P(2, "two"),
+        P(3, "three"),
+        P(4, "four"),
+        P(1, "four"),
+        P(2, "four"),
+    };
+    C c(a, a + sizeof(a) / sizeof(a[0]));
+    assert(c.erase(5) == 0);
+    assert(c.size() == 4);
+    assert(c.at(1) == "one");
+    assert(c.at(2) == "two");
+    assert(c.at(3) == "three");
+    assert(c.at(4) == "four");
+
+    assert(c.erase(2) == 1);
+    assert(c.size() == 3);
+    assert(c.at(1) == "one");
+    assert(c.at(3) == "three");
+    assert(c.at(4) == "four");
+
+    assert(c.erase(2) == 0);
+    assert(c.size() == 3);
+    assert(c.at(1) == "one");
+    assert(c.at(3) == "three");
+    assert(c.at(4) == "four");
+
+    assert(c.erase(4) == 1);
+    assert(c.size() == 2);
+    assert(c.at(1) == "one");
+    assert(c.at(3) == "three");
+
+    assert(c.erase(4) == 0);
+    assert(c.size() == 2);
+    assert(c.at(1) == "one");
+    assert(c.at(3) == "three");
+
+    assert(c.erase(1) == 1);
+    assert(c.size() == 1);
+    assert(c.at(3) == "three");
+
+    assert(c.erase(1) == 0);
+    assert(c.size() == 1);
+    assert(c.at(3) == "three");
+
+    assert(c.erase(3) == 1);
+    assert(c.size() == 0);
+
+    assert(c.erase(3) == 0);
+    assert(c.size() == 0);
+  }
 #if TEST_STD_VER >= 11
-    {
-        typedef std::unordered_map<int, std::string, std::hash<int>, std::equal_to<int>,
-                            min_allocator<std::pair<const int, std::string>>> C;
-        typedef std::pair<int, std::string> P;
-        P a[] =
-        {
-            P(1, "one"),
-            P(2, "two"),
-            P(3, "three"),
-            P(4, "four"),
-            P(1, "four"),
-            P(2, "four"),
-        };
-        C c(a, a + sizeof(a)/sizeof(a[0]));
-        assert(c.erase(5) == 0);
-        assert(c.size() == 4);
-        assert(c.at(1) == "one");
-        assert(c.at(2) == "two");
-        assert(c.at(3) == "three");
-        assert(c.at(4) == "four");
-
-        assert(c.erase(2) == 1);
-        assert(c.size() == 3);
-        assert(c.at(1) == "one");
-        assert(c.at(3) == "three");
-        assert(c.at(4) == "four");
-
-        assert(c.erase(2) == 0);
-        assert(c.size() == 3);
-        assert(c.at(1) == "one");
-        assert(c.at(3) == "three");
-        assert(c.at(4) == "four");
-
-        assert(c.erase(4) == 1);
-        assert(c.size() == 2);
-        assert(c.at(1) == "one");
-        assert(c.at(3) == "three");
-
-        assert(c.erase(4) == 0);
-        assert(c.size() == 2);
-        assert(c.at(1) == "one");
-        assert(c.at(3) == "three");
-
-        assert(c.erase(1) == 1);
-        assert(c.size() == 1);
-        assert(c.at(3) == "three");
-
-        assert(c.erase(1) == 0);
-        assert(c.size() == 1);
-        assert(c.at(3) == "three");
-
-        assert(c.erase(3) == 1);
-        assert(c.size() == 0);
-
-        assert(c.erase(3) == 0);
-        assert(c.size() == 0);
-    }
-    {
+  {
+    typedef std::unordered_map<int,
+                               std::string,
+                               std::hash<int>,
+                               std::equal_to<int>,
+                               min_allocator<std::pair<const int, std::string>>>
+        C;
+    typedef std::pair<int, std::string> P;
+    P a[] = {
+        P(1, "one"),
+        P(2, "two"),
+        P(3, "three"),
+        P(4, "four"),
+        P(1, "four"),
+        P(2, "four"),
+    };
+    C c(a, a + sizeof(a) / sizeof(a[0]));
+    assert(c.erase(5) == 0);
+    assert(c.size() == 4);
+    assert(c.at(1) == "one");
+    assert(c.at(2) == "two");
+    assert(c.at(3) == "three");
+    assert(c.at(4) == "four");
+
+    assert(c.erase(2) == 1);
+    assert(c.size() == 3);
+    assert(c.at(1) == "one");
+    assert(c.at(3) == "three");
+    assert(c.at(4) == "four");
+
+    assert(c.erase(2) == 0);
+    assert(c.size() == 3);
+    assert(c.at(1) == "one");
+    assert(c.at(3) == "three");
+    assert(c.at(4) == "four");
+
+    assert(c.erase(4) == 1);
+    assert(c.size() == 2);
+    assert(c.at(1) == "one");
+    assert(c.at(3) == "three");
+
+    assert(c.erase(4) == 0);
+    assert(c.size() == 2);
+    assert(c.at(1) == "one");
+    assert(c.at(3) == "three");
+
+    assert(c.erase(1) == 1);
+    assert(c.size() == 1);
+    assert(c.at(3) == "three");
+
+    assert(c.erase(1) == 0);
+    assert(c.size() == 1);
+    assert(c.at(3) == "three");
+
+    assert(c.erase(3) == 1);
+    assert(c.size() == 0);
+
+    assert(c.erase(3) == 0);
+    assert(c.size() == 0);
+  }
+  {
     typedef std::unordered_map<int, int> C;
     C m, m2;
-    for ( int i = 0; i < 10; ++i ) {
-        m[i] = i;
-        m2[i] = i;
-        }
+    for (int i = 0; i < 10; ++i) {
+      m[i]  = i;
+      m2[i] = i;
+    }
 
     C::iterator i = m2.begin();
-    int ctr = 0;
+    int ctr       = 0;
     while (i != m2.end()) {
-        if (ctr++ % 2 == 0)
-            m2.erase(i++);
-        else
-            ++i;
-        }
-
-    assert (only_deletions (m, m2));
+      if (ctr++ % 2 == 0)
+        m2.erase(i++);
+      else
+        ++i;
     }
+
+    assert(only_deletions(m, m2));
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/unord/unord.map/unord.map.modifiers/erase_range.pass.cpp b/libcxx/test/std/containers/unord/unord.map/unord.map.modifiers/erase_range.pass.cpp
index f6baeec9ff1c4..532413437f6be 100644
--- a/libcxx/test/std/containers/unord/unord.map/unord.map.modifiers/erase_range.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.map/unord.map.modifiers/erase_range.pass.cpp
@@ -22,80 +22,81 @@
 #include "test_macros.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef std::unordered_map<int, std::string> C;
-        typedef std::pair<int, std::string> P;
-        P a[] =
-        {
-            P(1, "one"),
-            P(2, "two"),
-            P(3, "three"),
-            P(4, "four"),
-            P(1, "four"),
-            P(2, "four"),
-        };
-        C c(a, a + sizeof(a)/sizeof(a[0]));
-        C::const_iterator i = c.find(2);
-        C::const_iterator j = std::next(i, 1);
-        C::iterator k = c.erase(i, i);
-        assert(k == i);
-        assert(c.size() == 4);
-        assert(c.at(1) == "one");
-        assert(c.at(2) == "two");
-        assert(c.at(3) == "three");
-        assert(c.at(4) == "four");
+int main(int, char**) {
+  {
+    typedef std::unordered_map<int, std::string> C;
+    typedef std::pair<int, std::string> P;
+    P a[] = {
+        P(1, "one"),
+        P(2, "two"),
+        P(3, "three"),
+        P(4, "four"),
+        P(1, "four"),
+        P(2, "four"),
+    };
+    C c(a, a + sizeof(a) / sizeof(a[0]));
+    C::const_iterator i = c.find(2);
+    C::const_iterator j = std::next(i, 1);
+    C::iterator k       = c.erase(i, i);
+    assert(k == i);
+    assert(c.size() == 4);
+    assert(c.at(1) == "one");
+    assert(c.at(2) == "two");
+    assert(c.at(3) == "three");
+    assert(c.at(4) == "four");
 
-        k = c.erase(i, j);
-        assert(c.size() == 3);
-        assert(k == j);
-        assert(c.at(1) == "one");
-        assert(c.at(3) == "three");
-        assert(c.at(4) == "four");
+    k = c.erase(i, j);
+    assert(c.size() == 3);
+    assert(k == j);
+    assert(c.at(1) == "one");
+    assert(c.at(3) == "three");
+    assert(c.at(4) == "four");
 
-        k = c.erase(c.cbegin(), c.cend());
-        assert(k == c.cend());
-        assert(c.size() == 0);
-        assert(k == c.end());
-    }
+    k = c.erase(c.cbegin(), c.cend());
+    assert(k == c.cend());
+    assert(c.size() == 0);
+    assert(k == c.end());
+  }
 #if TEST_STD_VER >= 11
-    {
-        typedef std::unordered_map<int, std::string, std::hash<int>, std::equal_to<int>,
-                            min_allocator<std::pair<const int, std::string>>> C;
-        typedef std::pair<int, std::string> P;
-        P a[] =
-        {
-            P(1, "one"),
-            P(2, "two"),
-            P(3, "three"),
-            P(4, "four"),
-            P(1, "four"),
-            P(2, "four"),
-        };
-        C c(a, a + sizeof(a)/sizeof(a[0]));
-        C::const_iterator i = c.find(2);
-        C::const_iterator j = std::next(i, 1);
-        C::iterator k = c.erase(i, i);
-        assert(k == i);
-        assert(c.size() == 4);
-        assert(c.at(1) == "one");
-        assert(c.at(2) == "two");
-        assert(c.at(3) == "three");
-        assert(c.at(4) == "four");
+  {
+    typedef std::unordered_map<int,
+                               std::string,
+                               std::hash<int>,
+                               std::equal_to<int>,
+                               min_allocator<std::pair<const int, std::string>>>
+        C;
+    typedef std::pair<int, std::string> P;
+    P a[] = {
+        P(1, "one"),
+        P(2, "two"),
+        P(3, "three"),
+        P(4, "four"),
+        P(1, "four"),
+        P(2, "four"),
+    };
+    C c(a, a + sizeof(a) / sizeof(a[0]));
+    C::const_iterator i = c.find(2);
+    C::const_iterator j = std::next(i, 1);
+    C::iterator k       = c.erase(i, i);
+    assert(k == i);
+    assert(c.size() == 4);
+    assert(c.at(1) == "one");
+    assert(c.at(2) == "two");
+    assert(c.at(3) == "three");
+    assert(c.at(4) == "four");
 
-        k = c.erase(i, j);
-        assert(c.size() == 3);
-        assert(k == j);
-        assert(c.at(1) == "one");
-        assert(c.at(3) == "three");
-        assert(c.at(4) == "four");
+    k = c.erase(i, j);
+    assert(c.size() == 3);
+    assert(k == j);
+    assert(c.at(1) == "one");
+    assert(c.at(3) == "three");
+    assert(c.at(4) == "four");
 
-        k = c.erase(c.cbegin(), c.cend());
-        assert(k == c.cend());
-        assert(c.size() == 0);
-        assert(k == c.end());
-    }
+    k = c.erase(c.cbegin(), c.cend());
+    assert(k == c.cend());
+    assert(c.size() == 0);
+    assert(k == c.end());
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/unord/unord.map/unord.map.modifiers/extract_iterator.pass.cpp b/libcxx/test/std/containers/unord/unord.map/unord.map.modifiers/extract_iterator.pass.cpp
index ac241c07d972d..811d3eb085ffc 100644
--- a/libcxx/test/std/containers/unord/unord.map/unord.map.modifiers/extract_iterator.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.map/unord.map.modifiers/extract_iterator.pass.cpp
@@ -20,50 +20,45 @@
 #include "Counter.h"
 
 template <class Container>
-void test(Container& c)
-{
-    std::size_t sz = c.size();
+void test(Container& c) {
+  std::size_t sz = c.size();
 
-    auto some_key = c.cbegin()->first;
+  auto some_key = c.cbegin()->first;
 
-    for (auto first = c.cbegin(); first != c.cend();)
-    {
-        auto key_value = first->first;
-        typename Container::node_type t = c.extract(first++);
-        --sz;
-        assert(t.key() == key_value);
-        t.key() = some_key;
-        assert(t.key() == some_key);
-        assert(t.get_allocator() == c.get_allocator());
-        assert(sz == c.size());
-    }
+  for (auto first = c.cbegin(); first != c.cend();) {
+    auto key_value                  = first->first;
+    typename Container::node_type t = c.extract(first++);
+    --sz;
+    assert(t.key() == key_value);
+    t.key() = some_key;
+    assert(t.key() == some_key);
+    assert(t.get_allocator() == c.get_allocator());
+    assert(sz == c.size());
+  }
 
-    assert(c.size() == 0);
+  assert(c.size() == 0);
 }
 
-int main(int, char**)
-{
-    {
-        using map_type = std::unordered_map<int, int>;
-        map_type m = {{1,1}, {2,2}, {3,3}, {4,4}, {5,5}, {6,6}};
-        test(m);
-    }
+int main(int, char**) {
+  {
+    using map_type = std::unordered_map<int, int>;
+    map_type m     = {{1, 1}, {2, 2}, {3, 3}, {4, 4}, {5, 5}, {6, 6}};
+    test(m);
+  }
 
-    {
-        std::unordered_map<Counter<int>, Counter<int>> m =
-            {{1,1}, {2,2}, {3,3}, {4,4}, {5,5}, {6,6}};
-        assert(Counter_base::gConstructed == 12);
-        test(m);
-        assert(Counter_base::gConstructed == 0);
-    }
+  {
+    std::unordered_map<Counter<int>, Counter<int>> m = {{1, 1}, {2, 2}, {3, 3}, {4, 4}, {5, 5}, {6, 6}};
+    assert(Counter_base::gConstructed == 12);
+    test(m);
+    assert(Counter_base::gConstructed == 0);
+  }
 
-    {
-        using min_alloc_map =
-            std::unordered_map<int, int, std::hash<int>, std::equal_to<int>,
-                               min_allocator<std::pair<const int, int>>>;
-        min_alloc_map m = {{1, 1}, {2, 2}, {3, 3}, {4, 4}, {5, 5}, {6, 6}};
-        test(m);
-    }
+  {
+    using min_alloc_map =
+        std::unordered_map<int, int, std::hash<int>, std::equal_to<int>, min_allocator<std::pair<const int, int>>>;
+    min_alloc_map m = {{1, 1}, {2, 2}, {3, 3}, {4, 4}, {5, 5}, {6, 6}};
+    test(m);
+  }
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/unord/unord.map/unord.map.modifiers/extract_key.pass.cpp b/libcxx/test/std/containers/unord/unord.map/unord.map.modifiers/extract_key.pass.cpp
index 78721db16e0be..60f3310797b3d 100644
--- a/libcxx/test/std/containers/unord/unord.map/unord.map.modifiers/extract_key.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.map/unord.map.modifiers/extract_key.pass.cpp
@@ -22,59 +22,53 @@
 #include "Counter.h"
 
 template <class Container, class KeyTypeIter>
-void test(Container& c, KeyTypeIter first, KeyTypeIter last)
-{
-    std::size_t sz = c.size();
-    assert((std::size_t)std::distance(first, last) == sz);
+void test(Container& c, KeyTypeIter first, KeyTypeIter last) {
+  std::size_t sz = c.size();
+  assert((std::size_t)std::distance(first, last) == sz);
 
-    for (KeyTypeIter copy = first; copy != last; ++copy)
-    {
-        typename Container::node_type t = c.extract(*copy);
-        assert(!t.empty());
-        --sz;
-        assert(t.key() == *copy);
-        t.key() = *first; // We should be able to mutate key.
-        assert(t.key() == *first);
-        assert(t.get_allocator() == c.get_allocator());
-        assert(sz == c.size());
-    }
+  for (KeyTypeIter copy = first; copy != last; ++copy) {
+    typename Container::node_type t = c.extract(*copy);
+    assert(!t.empty());
+    --sz;
+    assert(t.key() == *copy);
+    t.key() = *first; // We should be able to mutate key.
+    assert(t.key() == *first);
+    assert(t.get_allocator() == c.get_allocator());
+    assert(sz == c.size());
+  }
 
-    assert(c.size() == 0);
+  assert(c.size() == 0);
 
-    for (KeyTypeIter copy = first; copy != last; ++copy)
-    {
-        typename Container::node_type t = c.extract(*copy);
-        assert(t.empty());
-    }
+  for (KeyTypeIter copy = first; copy != last; ++copy) {
+    typename Container::node_type t = c.extract(*copy);
+    assert(t.empty());
+  }
 }
 
-int main(int, char**)
-{
-    {
-        std::unordered_map<int, int> m = {{1,1}, {2,2}, {3,3}, {4,4}, {5,5}, {6,6}};
-        int keys[] = {1, 2, 3, 4, 5, 6};
-        test(m, std::begin(keys), std::end(keys));
-    }
+int main(int, char**) {
+  {
+    std::unordered_map<int, int> m = {{1, 1}, {2, 2}, {3, 3}, {4, 4}, {5, 5}, {6, 6}};
+    int keys[]                     = {1, 2, 3, 4, 5, 6};
+    test(m, std::begin(keys), std::end(keys));
+  }
 
+  {
+    std::unordered_map<Counter<int>, Counter<int>> m = {{1, 1}, {2, 2}, {3, 3}, {4, 4}, {5, 5}, {6, 6}};
     {
-        std::unordered_map<Counter<int>, Counter<int>> m =
-            {{1,1}, {2,2}, {3,3}, {4,4}, {5,5}, {6,6}};
-        {
-            Counter<int> keys[] = {1, 2, 3, 4, 5, 6};
-            assert(Counter_base::gConstructed == 12+6);
-            test(m, std::begin(keys), std::end(keys));
-        }
-        assert(Counter_base::gConstructed == 0);
+      Counter<int> keys[] = {1, 2, 3, 4, 5, 6};
+      assert(Counter_base::gConstructed == 12 + 6);
+      test(m, std::begin(keys), std::end(keys));
     }
+    assert(Counter_base::gConstructed == 0);
+  }
 
-    {
-        using min_alloc_map =
-            std::unordered_map<int, int, std::hash<int>, std::equal_to<int>,
-                               min_allocator<std::pair<const int, int>>>;
-        min_alloc_map m = {{1, 1}, {2, 2}, {3, 3}, {4, 4}, {5, 5}, {6, 6}};
-        int keys[] = {1, 2, 3, 4, 5, 6};
-        test(m, std::begin(keys), std::end(keys));
-    }
+  {
+    using min_alloc_map =
+        std::unordered_map<int, int, std::hash<int>, std::equal_to<int>, min_allocator<std::pair<const int, int>>>;
+    min_alloc_map m = {{1, 1}, {2, 2}, {3, 3}, {4, 4}, {5, 5}, {6, 6}};
+    int keys[]      = {1, 2, 3, 4, 5, 6};
+    test(m, std::begin(keys), std::end(keys));
+  }
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/unord/unord.map/unord.map.modifiers/insert_and_emplace_allocator_requirements.pass.cpp b/libcxx/test/std/containers/unord/unord.map/unord.map.modifiers/insert_and_emplace_allocator_requirements.pass.cpp
index a28a866b4309b..a504d953a26dd 100644
--- a/libcxx/test/std/containers/unord/unord.map/unord.map.modifiers/insert_and_emplace_allocator_requirements.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.map/unord.map.modifiers/insert_and_emplace_allocator_requirements.pass.cpp
@@ -15,15 +15,13 @@
 
 // UNSUPPORTED: c++03
 
-
 #include <unordered_map>
 
 #include "test_macros.h"
 #include "container_test_types.h"
 #include "../../../map_allocator_requirement_test_templates.h"
 
-int main(int, char**)
-{
+int main(int, char**) {
   testMapInsert<TCT::unordered_map<> >();
   testMapInsertHint<TCT::unordered_map<> >();
   testMapEmplace<TCT::unordered_map<> >();
diff --git a/libcxx/test/std/containers/unord/unord.map/unord.map.modifiers/insert_const_lvalue.pass.cpp b/libcxx/test/std/containers/unord/unord.map/unord.map.modifiers/insert_const_lvalue.pass.cpp
index eb505218fdd1d..03f214f239e22 100644
--- a/libcxx/test/std/containers/unord/unord.map/unord.map.modifiers/insert_const_lvalue.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.map/unord.map.modifiers/insert_const_lvalue.pass.cpp
@@ -20,63 +20,64 @@
 #include "test_macros.h"
 #include "min_allocator.h"
 
-
 template <class Container>
-void do_insert_cv_test()
-{
-    typedef Container M;
-    typedef std::pair<typename M::iterator, bool> R;
-    typedef typename M::value_type VT;
-    M m;
+void do_insert_cv_test() {
+  typedef Container M;
+  typedef std::pair<typename M::iterator, bool> R;
+  typedef typename M::value_type VT;
+  M m;
 
-    const VT v1(2.5, 2);
-    R r = m.insert(v1);
-    assert(r.second);
-    assert(m.size() == 1);
-    assert(r.first->first == 2.5);
-    assert(r.first->second == 2);
+  const VT v1(2.5, 2);
+  R r = m.insert(v1);
+  assert(r.second);
+  assert(m.size() == 1);
+  assert(r.first->first == 2.5);
+  assert(r.first->second == 2);
 
-    const VT v2(2.5, 3);
-    r = m.insert(v2);
-    assert(!r.second);
-    assert(m.size() == 1);
-    assert(r.first->first == 2.5);
-    assert(r.first->second == 2);
+  const VT v2(2.5, 3);
+  r = m.insert(v2);
+  assert(!r.second);
+  assert(m.size() == 1);
+  assert(r.first->first == 2.5);
+  assert(r.first->second == 2);
 
-    const VT v3(1.5, 1);
-    r = m.insert(v3);
-    assert(r.second);
-    assert(m.size() == 2);
-    assert(r.first->first == 1.5);
-    assert(r.first->second == 1);
+  const VT v3(1.5, 1);
+  r = m.insert(v3);
+  assert(r.second);
+  assert(m.size() == 2);
+  assert(r.first->first == 1.5);
+  assert(r.first->second == 1);
 
-    const VT v4(3.5, 3);
-    r = m.insert(v4);
-    assert(r.second);
-    assert(m.size() == 3);
-    assert(r.first->first == 3.5);
-    assert(r.first->second == 3);
+  const VT v4(3.5, 3);
+  r = m.insert(v4);
+  assert(r.second);
+  assert(m.size() == 3);
+  assert(r.first->first == 3.5);
+  assert(r.first->second == 3);
 
-    const VT v5(3.5, 4);
-    r = m.insert(v5);
-    assert(!r.second);
-    assert(m.size() == 3);
-    assert(r.first->first == 3.5);
-    assert(r.first->second == 3);
+  const VT v5(3.5, 4);
+  r = m.insert(v5);
+  assert(!r.second);
+  assert(m.size() == 3);
+  assert(r.first->first == 3.5);
+  assert(r.first->second == 3);
 }
 
-int main(int, char**)
-{
-    {
-        typedef std::unordered_map<double, int> M;
-        do_insert_cv_test<M>();
-    }
+int main(int, char**) {
+  {
+    typedef std::unordered_map<double, int> M;
+    do_insert_cv_test<M>();
+  }
 #if TEST_STD_VER >= 11
-    {
-        typedef std::unordered_map<double, int, std::hash<double>, std::equal_to<double>,
-                            min_allocator<std::pair<const double, int>>> M;
-        do_insert_cv_test<M>();
-    }
+  {
+    typedef std::unordered_map<double,
+                               int,
+                               std::hash<double>,
+                               std::equal_to<double>,
+                               min_allocator<std::pair<const double, int>>>
+        M;
+    do_insert_cv_test<M>();
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/unord/unord.map/unord.map.modifiers/insert_hint_const_lvalue.pass.cpp b/libcxx/test/std/containers/unord/unord.map/unord.map.modifiers/insert_hint_const_lvalue.pass.cpp
index 208d4827d72e3..d2280d7daed76 100644
--- a/libcxx/test/std/containers/unord/unord.map/unord.map.modifiers/insert_hint_const_lvalue.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.map/unord.map.modifiers/insert_hint_const_lvalue.pass.cpp
@@ -20,50 +20,52 @@
 #include "test_macros.h"
 #include "min_allocator.h"
 
-template<class Container>
-void do_insert_hint_const_lvalue_test()
-{
-    typedef Container C;
-    typedef typename C::iterator R;
-    typedef typename C::value_type VT;
-    C c;
-    typename C::const_iterator e = c.end();
-    const VT v1(3.5, 3);
-    R r = c.insert(e, v1);
-    assert(c.size() == 1);
-    assert(r->first == 3.5);
-    assert(r->second == 3);
+template <class Container>
+void do_insert_hint_const_lvalue_test() {
+  typedef Container C;
+  typedef typename C::iterator R;
+  typedef typename C::value_type VT;
+  C c;
+  typename C::const_iterator e = c.end();
+  const VT v1(3.5, 3);
+  R r = c.insert(e, v1);
+  assert(c.size() == 1);
+  assert(r->first == 3.5);
+  assert(r->second == 3);
 
-    const VT v2(3.5, 4);
-    r = c.insert(r, v2);
-    assert(c.size() == 1);
-    assert(r->first == 3.5);
-    assert(r->second == 3);
+  const VT v2(3.5, 4);
+  r = c.insert(r, v2);
+  assert(c.size() == 1);
+  assert(r->first == 3.5);
+  assert(r->second == 3);
 
-    const VT v3(4.5, 4);
-    r = c.insert(c.end(), v3);
-    assert(c.size() == 2);
-    assert(r->first == 4.5);
-    assert(r->second == 4);
+  const VT v3(4.5, 4);
+  r = c.insert(c.end(), v3);
+  assert(c.size() == 2);
+  assert(r->first == 4.5);
+  assert(r->second == 4);
 
-    const VT v4(5.5, 4);
-    r = c.insert(c.end(), v4);
-    assert(c.size() == 3);
-    assert(r->first == 5.5);
-    assert(r->second == 4);
+  const VT v4(5.5, 4);
+  r = c.insert(c.end(), v4);
+  assert(c.size() == 3);
+  assert(r->first == 5.5);
+  assert(r->second == 4);
 }
 
-int main(int, char**)
-{
-    do_insert_hint_const_lvalue_test<std::unordered_map<double, int> >();
+int main(int, char**) {
+  do_insert_hint_const_lvalue_test<std::unordered_map<double, int> >();
 #if TEST_STD_VER >= 11
-    {
-        typedef std::unordered_map<double, int, std::hash<double>, std::equal_to<double>,
-                            min_allocator<std::pair<const double, int>>> C;
+  {
+    typedef std::unordered_map<double,
+                               int,
+                               std::hash<double>,
+                               std::equal_to<double>,
+                               min_allocator<std::pair<const double, int>>>
+        C;
 
-        do_insert_hint_const_lvalue_test<C>();
-    }
+    do_insert_hint_const_lvalue_test<C>();
+  }
 #endif
 
-    return 0;
+  return 0;
 }
diff --git a/libcxx/test/std/containers/unord/unord.map/unord.map.modifiers/insert_hint_rvalue.pass.cpp b/libcxx/test/std/containers/unord/unord.map/unord.map.modifiers/insert_hint_rvalue.pass.cpp
index 44ed22791ffbb..30b6bf6bdc9ef 100644
--- a/libcxx/test/std/containers/unord/unord.map/unord.map.modifiers/insert_hint_rvalue.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.map/unord.map.modifiers/insert_hint_rvalue.pass.cpp
@@ -25,139 +25,146 @@
 #include "MoveOnly.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef std::unordered_map<double, int> C;
-        typedef C::iterator R;
-        typedef std::pair<double, short> P;
-        C c;
-        C::const_iterator e = c.end();
-        R r = c.insert(e, P(3.5, static_cast<short>(3)));
-        assert(c.size() == 1);
-        assert(r->first == 3.5);
-        assert(r->second == 3);
-
-        r = c.insert(r, P(3.5, static_cast<short>(4)));
-        assert(c.size() == 1);
-        assert(r->first == 3.5);
-        assert(r->second == 3);
-
-        r = c.insert(c.end(), P(4.5, static_cast<short>(4)));
-        assert(c.size() == 2);
-        assert(r->first == 4.5);
-        assert(r->second == 4);
-
-        r = c.insert(c.end(), P(5.5, static_cast<short>(4)));
-        assert(c.size() == 3);
-        assert(r->first == 5.5);
-        assert(r->second == 4);
-    }
-    {
-        typedef std::unordered_map<MoveOnly, MoveOnly> C;
-        typedef C::iterator R;
-        typedef std::pair<MoveOnly, MoveOnly> P;
-        C c;
-        C::const_iterator e = c.end();
-        R r = c.insert(e, P(3, 3));
-        assert(c.size() == 1);
-        assert(r->first == 3);
-        assert(r->second == 3);
-
-        r = c.insert(r, P(3, 4));
-        assert(c.size() == 1);
-        assert(r->first == 3);
-        assert(r->second == 3);
-
-        r = c.insert(c.end(), P(4, 4));
-        assert(c.size() == 2);
-        assert(r->first == 4);
-        assert(r->second == 4);
-
-        r = c.insert(c.end(), P(5, 4));
-        assert(c.size() == 3);
-        assert(r->first == 5);
-        assert(r->second == 4);
-    }
-    {
-        typedef std::unordered_map<double, int, std::hash<double>, std::equal_to<double>,
-                            min_allocator<std::pair<const double, int>>> C;
-        typedef C::iterator R;
-        typedef std::pair<double, short> P;
-        C c;
-        C::const_iterator e = c.end();
-        R r = c.insert(e, P(3.5, static_cast<short>(3)));
-        assert(c.size() == 1);
-        assert(r->first == 3.5);
-        assert(r->second == 3);
-
-        r = c.insert(r, P(3.5, static_cast<short>(4)));
-        assert(c.size() == 1);
-        assert(r->first == 3.5);
-        assert(r->second == 3);
-
-        r = c.insert(c.end(), P(4.5, static_cast<short>(4)));
-        assert(c.size() == 2);
-        assert(r->first == 4.5);
-        assert(r->second == 4);
-
-        r = c.insert(c.end(), P(5.5, static_cast<short>(4)));
-        assert(c.size() == 3);
-        assert(r->first == 5.5);
-        assert(r->second == 4);
-    }
-    {
-        typedef std::unordered_map<MoveOnly, MoveOnly, std::hash<MoveOnly>, std::equal_to<MoveOnly>,
-                            min_allocator<std::pair<const MoveOnly, MoveOnly>>> C;
-        typedef C::iterator R;
-        typedef std::pair<MoveOnly, MoveOnly> P;
-        C c;
-        C::const_iterator e = c.end();
-        R r = c.insert(e, P(3, 3));
-        assert(c.size() == 1);
-        assert(r->first == 3);
-        assert(r->second == 3);
-
-        r = c.insert(r, P(3, 4));
-        assert(c.size() == 1);
-        assert(r->first == 3);
-        assert(r->second == 3);
-
-        r = c.insert(c.end(), P(4, 4));
-        assert(c.size() == 2);
-        assert(r->first == 4);
-        assert(r->second == 4);
-
-        r = c.insert(c.end(), P(5, 4));
-        assert(c.size() == 3);
-        assert(r->first == 5);
-        assert(r->second == 4);
-    }
-    {
-        typedef std::unordered_map<double, MoveOnly> C;
-        typedef C::iterator R;
-        C c;
-        C::const_iterator e = c.end();
-        R r = c.insert(e, {3.5, 3});
-        assert(c.size() == 1);
-        assert(r->first == 3.5);
-        assert(r->second == 3);
-
-        r = c.insert(r, {3.5, 4});
-        assert(c.size() == 1);
-        assert(r->first == 3.5);
-        assert(r->second == 3);
-
-        r = c.insert(c.end(), {4.5, 4});
-        assert(c.size() == 2);
-        assert(r->first == 4.5);
-        assert(r->second == 4);
-
-        r = c.insert(c.end(), {5.5, 4});
-        assert(c.size() == 3);
-        assert(r->first == 5.5);
-        assert(r->second == 4);
-    }
-
-    return 0;
+int main(int, char**) {
+  {
+    typedef std::unordered_map<double, int> C;
+    typedef C::iterator R;
+    typedef std::pair<double, short> P;
+    C c;
+    C::const_iterator e = c.end();
+    R r                 = c.insert(e, P(3.5, static_cast<short>(3)));
+    assert(c.size() == 1);
+    assert(r->first == 3.5);
+    assert(r->second == 3);
+
+    r = c.insert(r, P(3.5, static_cast<short>(4)));
+    assert(c.size() == 1);
+    assert(r->first == 3.5);
+    assert(r->second == 3);
+
+    r = c.insert(c.end(), P(4.5, static_cast<short>(4)));
+    assert(c.size() == 2);
+    assert(r->first == 4.5);
+    assert(r->second == 4);
+
+    r = c.insert(c.end(), P(5.5, static_cast<short>(4)));
+    assert(c.size() == 3);
+    assert(r->first == 5.5);
+    assert(r->second == 4);
+  }
+  {
+    typedef std::unordered_map<MoveOnly, MoveOnly> C;
+    typedef C::iterator R;
+    typedef std::pair<MoveOnly, MoveOnly> P;
+    C c;
+    C::const_iterator e = c.end();
+    R r                 = c.insert(e, P(3, 3));
+    assert(c.size() == 1);
+    assert(r->first == 3);
+    assert(r->second == 3);
+
+    r = c.insert(r, P(3, 4));
+    assert(c.size() == 1);
+    assert(r->first == 3);
+    assert(r->second == 3);
+
+    r = c.insert(c.end(), P(4, 4));
+    assert(c.size() == 2);
+    assert(r->first == 4);
+    assert(r->second == 4);
+
+    r = c.insert(c.end(), P(5, 4));
+    assert(c.size() == 3);
+    assert(r->first == 5);
+    assert(r->second == 4);
+  }
+  {
+    typedef std::unordered_map<double,
+                               int,
+                               std::hash<double>,
+                               std::equal_to<double>,
+                               min_allocator<std::pair<const double, int>>>
+        C;
+    typedef C::iterator R;
+    typedef std::pair<double, short> P;
+    C c;
+    C::const_iterator e = c.end();
+    R r                 = c.insert(e, P(3.5, static_cast<short>(3)));
+    assert(c.size() == 1);
+    assert(r->first == 3.5);
+    assert(r->second == 3);
+
+    r = c.insert(r, P(3.5, static_cast<short>(4)));
+    assert(c.size() == 1);
+    assert(r->first == 3.5);
+    assert(r->second == 3);
+
+    r = c.insert(c.end(), P(4.5, static_cast<short>(4)));
+    assert(c.size() == 2);
+    assert(r->first == 4.5);
+    assert(r->second == 4);
+
+    r = c.insert(c.end(), P(5.5, static_cast<short>(4)));
+    assert(c.size() == 3);
+    assert(r->first == 5.5);
+    assert(r->second == 4);
+  }
+  {
+    typedef std::unordered_map<MoveOnly,
+                               MoveOnly,
+                               std::hash<MoveOnly>,
+                               std::equal_to<MoveOnly>,
+                               min_allocator<std::pair<const MoveOnly, MoveOnly>>>
+        C;
+    typedef C::iterator R;
+    typedef std::pair<MoveOnly, MoveOnly> P;
+    C c;
+    C::const_iterator e = c.end();
+    R r                 = c.insert(e, P(3, 3));
+    assert(c.size() == 1);
+    assert(r->first == 3);
+    assert(r->second == 3);
+
+    r = c.insert(r, P(3, 4));
+    assert(c.size() == 1);
+    assert(r->first == 3);
+    assert(r->second == 3);
+
+    r = c.insert(c.end(), P(4, 4));
+    assert(c.size() == 2);
+    assert(r->first == 4);
+    assert(r->second == 4);
+
+    r = c.insert(c.end(), P(5, 4));
+    assert(c.size() == 3);
+    assert(r->first == 5);
+    assert(r->second == 4);
+  }
+  {
+    typedef std::unordered_map<double, MoveOnly> C;
+    typedef C::iterator R;
+    C c;
+    C::const_iterator e = c.end();
+    R r                 = c.insert(e, {3.5, 3});
+    assert(c.size() == 1);
+    assert(r->first == 3.5);
+    assert(r->second == 3);
+
+    r = c.insert(r, {3.5, 4});
+    assert(c.size() == 1);
+    assert(r->first == 3.5);
+    assert(r->second == 3);
+
+    r = c.insert(c.end(), {4.5, 4});
+    assert(c.size() == 2);
+    assert(r->first == 4.5);
+    assert(r->second == 4);
+
+    r = c.insert(c.end(), {5.5, 4});
+    assert(c.size() == 3);
+    assert(r->first == 5.5);
+    assert(r->second == 4);
+  }
+
+  return 0;
 }
diff --git a/libcxx/test/std/containers/unord/unord.map/unord.map.modifiers/insert_init.pass.cpp b/libcxx/test/std/containers/unord/unord.map/unord.map.modifiers/insert_init.pass.cpp
index dcdad5605e9c7..9699df4b74d26 100644
--- a/libcxx/test/std/containers/unord/unord.map/unord.map.modifiers/insert_init.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.map/unord.map.modifiers/insert_init.pass.cpp
@@ -24,49 +24,48 @@
 #include "test_iterators.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef std::unordered_map<int, std::string> C;
-        typedef std::pair<int, std::string> P;
-        C c;
-        c.insert(
-                    {
-                        P(1, "one"),
-                        P(2, "two"),
-                        P(3, "three"),
-                        P(4, "four"),
-                        P(1, "four"),
-                        P(2, "four"),
-                    }
-                );
-        assert(c.size() == 4);
-        assert(c.at(1) == "one");
-        assert(c.at(2) == "two");
-        assert(c.at(3) == "three");
-        assert(c.at(4) == "four");
-    }
-    {
-        typedef std::unordered_map<int, std::string, std::hash<int>, std::equal_to<int>,
-                            min_allocator<std::pair<const int, std::string>>> C;
-        typedef std::pair<int, std::string> P;
-        C c;
-        c.insert(
-                    {
-                        P(1, "one"),
-                        P(2, "two"),
-                        P(3, "three"),
-                        P(4, "four"),
-                        P(1, "four"),
-                        P(2, "four"),
-                    }
-                );
-        assert(c.size() == 4);
-        assert(c.at(1) == "one");
-        assert(c.at(2) == "two");
-        assert(c.at(3) == "three");
-        assert(c.at(4) == "four");
-    }
+int main(int, char**) {
+  {
+    typedef std::unordered_map<int, std::string> C;
+    typedef std::pair<int, std::string> P;
+    C c;
+    c.insert({
+        P(1, "one"),
+        P(2, "two"),
+        P(3, "three"),
+        P(4, "four"),
+        P(1, "four"),
+        P(2, "four"),
+    });
+    assert(c.size() == 4);
+    assert(c.at(1) == "one");
+    assert(c.at(2) == "two");
+    assert(c.at(3) == "three");
+    assert(c.at(4) == "four");
+  }
+  {
+    typedef std::unordered_map<int,
+                               std::string,
+                               std::hash<int>,
+                               std::equal_to<int>,
+                               min_allocator<std::pair<const int, std::string>>>
+        C;
+    typedef std::pair<int, std::string> P;
+    C c;
+    c.insert({
+        P(1, "one"),
+        P(2, "two"),
+        P(3, "three"),
+        P(4, "four"),
+        P(1, "four"),
+        P(2, "four"),
+    });
+    assert(c.size() == 4);
+    assert(c.at(1) == "one");
+    assert(c.at(2) == "two");
+    assert(c.at(3) == "three");
+    assert(c.at(4) == "four");
+  }
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/unord/unord.map/unord.map.modifiers/insert_iter_iter.pass.cpp b/libcxx/test/std/containers/unord/unord.map/unord.map.modifiers/insert_iter_iter.pass.cpp
index 342ff584f6450..414631c8b7638 100644
--- a/libcxx/test/std/containers/unord/unord.map/unord.map.modifiers/insert_iter_iter.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.map/unord.map.modifiers/insert_iter_iter.pass.cpp
@@ -23,50 +23,51 @@
 #include "test_iterators.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef std::unordered_map<int, std::string> C;
-        typedef std::pair<int, std::string> P;
-        P a[] =
-        {
-            P(1, "one"),
-            P(2, "two"),
-            P(3, "three"),
-            P(4, "four"),
-            P(1, "four"),
-            P(2, "four"),
-        };
-        C c;
-        c.insert(cpp17_input_iterator<P*>(a), cpp17_input_iterator<P*>(a + sizeof(a)/sizeof(a[0])));
-        assert(c.size() == 4);
-        assert(c.at(1) == "one");
-        assert(c.at(2) == "two");
-        assert(c.at(3) == "three");
-        assert(c.at(4) == "four");
-    }
+int main(int, char**) {
+  {
+    typedef std::unordered_map<int, std::string> C;
+    typedef std::pair<int, std::string> P;
+    P a[] = {
+        P(1, "one"),
+        P(2, "two"),
+        P(3, "three"),
+        P(4, "four"),
+        P(1, "four"),
+        P(2, "four"),
+    };
+    C c;
+    c.insert(cpp17_input_iterator<P*>(a), cpp17_input_iterator<P*>(a + sizeof(a) / sizeof(a[0])));
+    assert(c.size() == 4);
+    assert(c.at(1) == "one");
+    assert(c.at(2) == "two");
+    assert(c.at(3) == "three");
+    assert(c.at(4) == "four");
+  }
 #if TEST_STD_VER >= 11
-    {
-        typedef std::unordered_map<int, std::string, std::hash<int>, std::equal_to<int>,
-                            min_allocator<std::pair<const int, std::string>>> C;
-        typedef std::pair<int, std::string> P;
-        P a[] =
-        {
-            P(1, "one"),
-            P(2, "two"),
-            P(3, "three"),
-            P(4, "four"),
-            P(1, "four"),
-            P(2, "four"),
-        };
-        C c;
-        c.insert(cpp17_input_iterator<P*>(a), cpp17_input_iterator<P*>(a + sizeof(a)/sizeof(a[0])));
-        assert(c.size() == 4);
-        assert(c.at(1) == "one");
-        assert(c.at(2) == "two");
-        assert(c.at(3) == "three");
-        assert(c.at(4) == "four");
-    }
+  {
+    typedef std::unordered_map<int,
+                               std::string,
+                               std::hash<int>,
+                               std::equal_to<int>,
+                               min_allocator<std::pair<const int, std::string>>>
+        C;
+    typedef std::pair<int, std::string> P;
+    P a[] = {
+        P(1, "one"),
+        P(2, "two"),
+        P(3, "three"),
+        P(4, "four"),
+        P(1, "four"),
+        P(2, "four"),
+    };
+    C c;
+    c.insert(cpp17_input_iterator<P*>(a), cpp17_input_iterator<P*>(a + sizeof(a) / sizeof(a[0])));
+    assert(c.size() == 4);
+    assert(c.at(1) == "one");
+    assert(c.at(2) == "two");
+    assert(c.at(3) == "three");
+    assert(c.at(4) == "four");
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/unord/unord.map/unord.map.modifiers/insert_node_type.pass.cpp b/libcxx/test/std/containers/unord/unord.map/unord.map.modifiers/insert_node_type.pass.cpp
index 7188774466969..c2920128e5b96 100644
--- a/libcxx/test/std/containers/unord/unord.map/unord.map.modifiers/insert_node_type.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.map/unord.map.modifiers/insert_node_type.pass.cpp
@@ -42,70 +42,64 @@ void verify_insert_return_type(T&& t) {
 
 template <class Container>
 typename Container::node_type
-node_factory(typename Container::key_type const& key,
-             typename Container::mapped_type const& mapped)
-{
-    static Container c;
-    auto p = c.insert({key, mapped});
-    assert(p.second);
-    return c.extract(p.first);
+node_factory(typename Container::key_type const& key, typename Container::mapped_type const& mapped) {
+  static Container c;
+  auto p = c.insert({key, mapped});
+  assert(p.second);
+  return c.extract(p.first);
 }
 
 template <class Container>
-void test(Container& c)
-{
-    auto* nf = &node_factory<Container>;
-
-    for (int i = 0; i != 10; ++i)
-    {
-        typename Container::node_type node = nf(i, i + 1);
-        assert(!node.empty());
-        typename Container::insert_return_type irt = c.insert(std::move(node));
-        assert(node.empty());
-        assert(irt.inserted);
-        assert(irt.node.empty());
-        assert(irt.position->first == i && irt.position->second == i + 1);
-        verify_insert_return_type<Container>(irt);
-    }
-
-    assert(c.size() == 10);
-
-    { // Insert empty node.
-        typename Container::node_type def;
-        auto irt = c.insert(std::move(def));
-        assert(def.empty());
-        assert(!irt.inserted);
-        assert(irt.node.empty());
-        assert(irt.position == c.end());
-        verify_insert_return_type<Container>(irt);
-    }
-
-    { // Insert duplicate node.
-        typename Container::node_type dupl = nf(0, 42);
-        auto irt = c.insert(std::move(dupl));
-        assert(dupl.empty());
-        assert(!irt.inserted);
-        assert(!irt.node.empty());
-        assert(irt.position == c.find(0));
-        assert(irt.node.key() == 0 && irt.node.mapped() == 42);
-        verify_insert_return_type<Container>(irt);
-    }
-
-    assert(c.size() == 10);
-
-    for (int i = 0; i != 10; ++i)
-    {
-        assert(c.count(i) == 1);
-        assert(c[i] == i + 1);
-    }
+void test(Container& c) {
+  auto* nf = &node_factory<Container>;
+
+  for (int i = 0; i != 10; ++i) {
+    typename Container::node_type node = nf(i, i + 1);
+    assert(!node.empty());
+    typename Container::insert_return_type irt = c.insert(std::move(node));
+    assert(node.empty());
+    assert(irt.inserted);
+    assert(irt.node.empty());
+    assert(irt.position->first == i && irt.position->second == i + 1);
+    verify_insert_return_type<Container>(irt);
+  }
+
+  assert(c.size() == 10);
+
+  { // Insert empty node.
+    typename Container::node_type def;
+    auto irt = c.insert(std::move(def));
+    assert(def.empty());
+    assert(!irt.inserted);
+    assert(irt.node.empty());
+    assert(irt.position == c.end());
+    verify_insert_return_type<Container>(irt);
+  }
+
+  { // Insert duplicate node.
+    typename Container::node_type dupl = nf(0, 42);
+    auto irt                           = c.insert(std::move(dupl));
+    assert(dupl.empty());
+    assert(!irt.inserted);
+    assert(!irt.node.empty());
+    assert(irt.position == c.find(0));
+    assert(irt.node.key() == 0 && irt.node.mapped() == 42);
+    verify_insert_return_type<Container>(irt);
+  }
+
+  assert(c.size() == 10);
+
+  for (int i = 0; i != 10; ++i) {
+    assert(c.count(i) == 1);
+    assert(c[i] == i + 1);
+  }
 }
 
-int main(int, char**)
-{
-    std::unordered_map<int, int> m;
-    test(m);
-    std::unordered_map<int, int, std::hash<int>, std::equal_to<int>, min_allocator<std::pair<const int, int>>> m2;
-    test(m2);
+int main(int, char**) {
+  std::unordered_map<int, int> m;
+  test(m);
+  std::unordered_map<int, int, std::hash<int>, std::equal_to<int>, min_allocator<std::pair<const int, int>>> m2;
+  test(m2);
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/unord/unord.map/unord.map.modifiers/insert_node_type_hint.pass.cpp b/libcxx/test/std/containers/unord/unord.map/unord.map.modifiers/insert_node_type_hint.pass.cpp
index 8677810187fb0..16f664ac89ff6 100644
--- a/libcxx/test/std/containers/unord/unord.map/unord.map.modifiers/insert_node_type_hint.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.map/unord.map.modifiers/insert_node_type_hint.pass.cpp
@@ -20,47 +20,41 @@
 
 template <class Container>
 typename Container::node_type
-node_factory(typename Container::key_type const& key,
-             typename Container::mapped_type const& mapped)
-{
-    static Container c;
-    auto p = c.insert({key, mapped});
-    assert(p.second);
-    return c.extract(p.first);
+node_factory(typename Container::key_type const& key, typename Container::mapped_type const& mapped) {
+  static Container c;
+  auto p = c.insert({key, mapped});
+  assert(p.second);
+  return c.extract(p.first);
 }
 
 template <class Container>
-void test(Container& c)
-{
-    auto* nf = &node_factory<Container>;
-
-    for (int i = 0; i != 10; ++i)
-    {
-        typename Container::node_type node = nf(i, i + 1);
-        assert(!node.empty());
-        std::size_t prev = c.size();
-        auto it = c.insert(c.end(), std::move(node));
-        assert(node.empty());
-        assert(prev + 1 == c.size());
-        assert(it->first == i);
-        assert(it->second == i + 1);
-    }
-
-    assert(c.size() == 10);
-
-    for (int i = 0; i != 10; ++i)
-    {
-        assert(c.count(i) == 1);
-        assert(c[i] == i + 1);
-    }
+void test(Container& c) {
+  auto* nf = &node_factory<Container>;
+
+  for (int i = 0; i != 10; ++i) {
+    typename Container::node_type node = nf(i, i + 1);
+    assert(!node.empty());
+    std::size_t prev = c.size();
+    auto it          = c.insert(c.end(), std::move(node));
+    assert(node.empty());
+    assert(prev + 1 == c.size());
+    assert(it->first == i);
+    assert(it->second == i + 1);
+  }
+
+  assert(c.size() == 10);
+
+  for (int i = 0; i != 10; ++i) {
+    assert(c.count(i) == 1);
+    assert(c[i] == i + 1);
+  }
 }
 
-int main(int, char**)
-{
-    std::unordered_map<int, int> m;
-    test(m);
-    std::unordered_map<int, int, std::hash<int>, std::equal_to<int>, min_allocator<std::pair<const int, int>>> m2;
-    test(m2);
+int main(int, char**) {
+  std::unordered_map<int, int> m;
+  test(m);
+  std::unordered_map<int, int, std::hash<int>, std::equal_to<int>, min_allocator<std::pair<const int, int>>> m2;
+  test(m2);
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/unord/unord.map/unord.map.modifiers/insert_or_assign.pass.cpp b/libcxx/test/std/containers/unord/unord.map/unord.map.modifiers/insert_or_assign.pass.cpp
index 6ba28c00675bc..b8bfd6cbc0f3c 100644
--- a/libcxx/test/std/containers/unord/unord.map/unord.map.modifiers/insert_or_assign.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.map/unord.map.modifiers/insert_or_assign.pass.cpp
@@ -28,34 +28,34 @@
 
 #include "test_macros.h"
 
+class Moveable {
+  Moveable(const Moveable&);
+  Moveable& operator=(const Moveable&);
 
-class Moveable
-{
-    Moveable(const Moveable&);
-    Moveable& operator=(const Moveable&);
+  int int_;
+  double double_;
 
-    int int_;
-    double double_;
 public:
-    Moveable() : int_(0), double_(0) {}
-    Moveable(int i, double d) : int_(i), double_(d) {}
-    Moveable(Moveable&& x)
-        : int_(x.int_), double_(x.double_)
-            {x.int_ = -1; x.double_ = -1;}
-    Moveable& operator=(Moveable&& x)
-        {int_ = x.int_; x.int_ = -1;
-         double_ = x.double_; x.double_ = -1;
-         return *this;
-        }
-
-    bool operator==(const Moveable& x) const
-        {return int_ == x.int_ && double_ == x.double_;}
-    bool operator<(const Moveable& x) const
-        {return int_ < x.int_ || (int_ == x.int_ && double_ < x.double_);}
-    std::size_t hash () const { return std::hash<int>()(int_) + std::hash<double>()(double_); }
-
-    int get() const {return int_;}
-    bool moved() const {return int_ == -1;}
+  Moveable() : int_(0), double_(0) {}
+  Moveable(int i, double d) : int_(i), double_(d) {}
+  Moveable(Moveable&& x) : int_(x.int_), double_(x.double_) {
+    x.int_    = -1;
+    x.double_ = -1;
+  }
+  Moveable& operator=(Moveable&& x) {
+    int_      = x.int_;
+    x.int_    = -1;
+    double_   = x.double_;
+    x.double_ = -1;
+    return *this;
+  }
+
+  bool operator==(const Moveable& x) const { return int_ == x.int_ && double_ == x.double_; }
+  bool operator<(const Moveable& x) const { return int_ < x.int_ || (int_ == x.int_ && double_ < x.double_); }
+  std::size_t hash() const { return std::hash<int>()(int_) + std::hash<double>()(double_); }
+
+  int get() const { return int_; }
+  bool moved() const { return int_ == -1; }
 };
 
 template <>
@@ -63,133 +63,129 @@ struct std::hash<Moveable> {
   std::size_t operator()(const Moveable& m) const { return m.hash(); }
 };
 
-int main(int, char**)
-{
-
-    { // pair<iterator, bool> insert_or_assign(const key_type& k, M&& obj);
-        typedef std::unordered_map<int, Moveable> M;
-        typedef std::pair<M::iterator, bool> R;
-        M m;
-        R r;
-        for (int i = 0; i < 20; i += 2)
-            m.emplace ( i, Moveable(i, (double) i));
-        assert(m.size() == 10);
-
-        for (int i=0; i < 20; i += 2)
-        {
-            Moveable mv(i+1, i+1);
-            r = m.insert_or_assign(i, std::move(mv));
-            assert(m.size() == 10);
-            assert(!r.second);                    // was not inserted
-            assert(mv.moved());                   // was moved from
-            assert(r.first->first == i);          // key
-            assert(r.first->second.get() == i+1); // value
-        }
-
-        Moveable mv1(5, 5.0);
-        r = m.insert_or_assign(-1, std::move(mv1));
-        assert(m.size() == 11);
-        assert(r.second);                    // was inserted
-        assert(mv1.moved());                 // was moved from
-        assert(r.first->first        == -1); // key
-        assert(r.first->second.get() == 5);  // value
-
-        Moveable mv2(9, 9.0);
-        r = m.insert_or_assign(3, std::move(mv2));
-        assert(m.size() == 12);
-        assert(r.second);                   // was inserted
-        assert(mv2.moved());                // was moved from
-        assert(r.first->first        == 3); // key
-        assert(r.first->second.get() == 9); // value
-
-        Moveable mv3(-1, 5.0);
-        r = m.insert_or_assign(117, std::move(mv3));
-        assert(m.size() == 13);
-        assert(r.second);                     // was inserted
-        assert(mv3.moved());                  // was moved from
-        assert(r.first->first        == 117); // key
-        assert(r.first->second.get() == -1);  // value
-    }
-    { // pair<iterator, bool> insert_or_assign(key_type&& k, M&& obj);
-        typedef std::unordered_map<Moveable, Moveable> M;
-        typedef std::pair<M::iterator, bool> R;
-        M m;
-        R r;
-        for (int i = 0; i < 20; i += 2)
-            m.emplace ( Moveable(i, (double) i), Moveable(i+1, (double) i+1));
-        assert(m.size() == 10);
-
-        Moveable mvkey1(2, 2.0);
-        Moveable mv1(4, 4.0);
-        r = m.insert_or_assign(std::move(mvkey1), std::move(mv1));
-        assert(m.size() == 10);
-        assert(!r.second);                  // was not inserted
-        assert(!mvkey1.moved());            // was not moved from
-        assert(mv1.moved());                // was moved from
-        assert(r.first->first == mvkey1);   // key
-        assert(r.first->second.get() == 4); // value
-
-        Moveable mvkey2(3, 3.0);
-        Moveable mv2(5, 5.0);
-        r = m.try_emplace(std::move(mvkey2), std::move(mv2));
-        assert(m.size() == 11);
-        assert(r.second);                   // was inserted
-        assert(mv2.moved());                // was moved from
-        assert(mvkey2.moved());             // was moved from
-        assert(r.first->first.get()  == 3); // key
-        assert(r.first->second.get() == 5); // value
-    }
-    { // iterator insert_or_assign(const_iterator hint, const key_type& k, M&& obj);
-        typedef std::unordered_map<int, Moveable> M;
-        M m;
-        M::iterator r;
-        for (int i = 0; i < 20; i += 2)
-            m.emplace ( i, Moveable(i, (double) i));
-        assert(m.size() == 10);
-        M::const_iterator it = m.find(2);
-
-        Moveable mv1(3, 3.0);
-        r = m.insert_or_assign(it, 2, std::move(mv1));
-        assert(m.size() == 10);
-        assert(mv1.moved());           // was moved from
-        assert(r->first        == 2);  // key
-        assert(r->second.get() == 3);  // value
-
-        Moveable mv2(5, 5.0);
-        r = m.insert_or_assign(it, 3, std::move(mv2));
-        assert(m.size() == 11);
-        assert(mv2.moved());           // was moved from
-        assert(r->first        == 3);  // key
-        assert(r->second.get() == 5);  // value
-    }
-    { // iterator insert_or_assign(const_iterator hint, key_type&& k, M&& obj);
-        typedef std::unordered_map<Moveable, Moveable> M;
-        M m;
-        M::iterator r;
-        for (int i = 0; i < 20; i += 2)
-            m.emplace ( Moveable(i, (double) i), Moveable(i+1, (double) i+1));
-        assert(m.size() == 10);
-        M::const_iterator it = std::next(m.cbegin());
-
-        Moveable mvkey1(2, 2.0);
-        Moveable mv1(4, 4.0);
-        r = m.insert_or_assign(it, std::move(mvkey1), std::move(mv1));
-        assert(m.size() == 10);
-        assert(mv1.moved());          // was moved from
-        assert(!mvkey1.moved());      // was not moved from
-        assert(r->first == mvkey1);   // key
-        assert(r->second.get() == 4); // value
-
-        Moveable mvkey2(3, 3.0);
-        Moveable mv2(5, 5.0);
-        r = m.insert_or_assign(it, std::move(mvkey2), std::move(mv2));
-        assert(m.size() == 11);
-        assert(mv2.moved());           // was moved from
-        assert(mvkey2.moved());        // was moved from
-        assert(r->first.get()  == 3);  // key
-        assert(r->second.get() == 5);  // value
+int main(int, char**) {
+  { // pair<iterator, bool> insert_or_assign(const key_type& k, M&& obj);
+    typedef std::unordered_map<int, Moveable> M;
+    typedef std::pair<M::iterator, bool> R;
+    M m;
+    R r;
+    for (int i = 0; i < 20; i += 2)
+      m.emplace(i, Moveable(i, (double)i));
+    assert(m.size() == 10);
+
+    for (int i = 0; i < 20; i += 2) {
+      Moveable mv(i + 1, i + 1);
+      r = m.insert_or_assign(i, std::move(mv));
+      assert(m.size() == 10);
+      assert(!r.second);                      // was not inserted
+      assert(mv.moved());                     // was moved from
+      assert(r.first->first == i);            // key
+      assert(r.first->second.get() == i + 1); // value
     }
 
+    Moveable mv1(5, 5.0);
+    r = m.insert_or_assign(-1, std::move(mv1));
+    assert(m.size() == 11);
+    assert(r.second);                   // was inserted
+    assert(mv1.moved());                // was moved from
+    assert(r.first->first == -1);       // key
+    assert(r.first->second.get() == 5); // value
+
+    Moveable mv2(9, 9.0);
+    r = m.insert_or_assign(3, std::move(mv2));
+    assert(m.size() == 12);
+    assert(r.second);                   // was inserted
+    assert(mv2.moved());                // was moved from
+    assert(r.first->first == 3);        // key
+    assert(r.first->second.get() == 9); // value
+
+    Moveable mv3(-1, 5.0);
+    r = m.insert_or_assign(117, std::move(mv3));
+    assert(m.size() == 13);
+    assert(r.second);                    // was inserted
+    assert(mv3.moved());                 // was moved from
+    assert(r.first->first == 117);       // key
+    assert(r.first->second.get() == -1); // value
+  }
+  { // pair<iterator, bool> insert_or_assign(key_type&& k, M&& obj);
+    typedef std::unordered_map<Moveable, Moveable> M;
+    typedef std::pair<M::iterator, bool> R;
+    M m;
+    R r;
+    for (int i = 0; i < 20; i += 2)
+      m.emplace(Moveable(i, (double)i), Moveable(i + 1, (double)i + 1));
+    assert(m.size() == 10);
+
+    Moveable mvkey1(2, 2.0);
+    Moveable mv1(4, 4.0);
+    r = m.insert_or_assign(std::move(mvkey1), std::move(mv1));
+    assert(m.size() == 10);
+    assert(!r.second);                  // was not inserted
+    assert(!mvkey1.moved());            // was not moved from
+    assert(mv1.moved());                // was moved from
+    assert(r.first->first == mvkey1);   // key
+    assert(r.first->second.get() == 4); // value
+
+    Moveable mvkey2(3, 3.0);
+    Moveable mv2(5, 5.0);
+    r = m.try_emplace(std::move(mvkey2), std::move(mv2));
+    assert(m.size() == 11);
+    assert(r.second);                   // was inserted
+    assert(mv2.moved());                // was moved from
+    assert(mvkey2.moved());             // was moved from
+    assert(r.first->first.get() == 3);  // key
+    assert(r.first->second.get() == 5); // value
+  }
+  { // iterator insert_or_assign(const_iterator hint, const key_type& k, M&& obj);
+    typedef std::unordered_map<int, Moveable> M;
+    M m;
+    M::iterator r;
+    for (int i = 0; i < 20; i += 2)
+      m.emplace(i, Moveable(i, (double)i));
+    assert(m.size() == 10);
+    M::const_iterator it = m.find(2);
+
+    Moveable mv1(3, 3.0);
+    r = m.insert_or_assign(it, 2, std::move(mv1));
+    assert(m.size() == 10);
+    assert(mv1.moved());          // was moved from
+    assert(r->first == 2);        // key
+    assert(r->second.get() == 3); // value
+
+    Moveable mv2(5, 5.0);
+    r = m.insert_or_assign(it, 3, std::move(mv2));
+    assert(m.size() == 11);
+    assert(mv2.moved());          // was moved from
+    assert(r->first == 3);        // key
+    assert(r->second.get() == 5); // value
+  }
+  { // iterator insert_or_assign(const_iterator hint, key_type&& k, M&& obj);
+    typedef std::unordered_map<Moveable, Moveable> M;
+    M m;
+    M::iterator r;
+    for (int i = 0; i < 20; i += 2)
+      m.emplace(Moveable(i, (double)i), Moveable(i + 1, (double)i + 1));
+    assert(m.size() == 10);
+    M::const_iterator it = std::next(m.cbegin());
+
+    Moveable mvkey1(2, 2.0);
+    Moveable mv1(4, 4.0);
+    r = m.insert_or_assign(it, std::move(mvkey1), std::move(mv1));
+    assert(m.size() == 10);
+    assert(mv1.moved());          // was moved from
+    assert(!mvkey1.moved());      // was not moved from
+    assert(r->first == mvkey1);   // key
+    assert(r->second.get() == 4); // value
+
+    Moveable mvkey2(3, 3.0);
+    Moveable mv2(5, 5.0);
+    r = m.insert_or_assign(it, std::move(mvkey2), std::move(mv2));
+    assert(m.size() == 11);
+    assert(mv2.moved());          // was moved from
+    assert(mvkey2.moved());       // was moved from
+    assert(r->first.get() == 3);  // key
+    assert(r->second.get() == 5); // value
+  }
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/unord/unord.map/unord.map.modifiers/insert_range.pass.cpp b/libcxx/test/std/containers/unord/unord.map/unord.map.modifiers/insert_range.pass.cpp
index d1c90c4ed4853..98c8e78a54394 100644
--- a/libcxx/test/std/containers/unord/unord.map/unord.map.modifiers/insert_range.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.map/unord.map.modifiers/insert_range.pass.cpp
@@ -23,10 +23,13 @@
 int main(int, char**) {
   // Note: we want to use a pair with non-const elements for input (an assignable type is a lot more convenient) but
   // have to use the exact `value_type` of the map (that is, `pair<const K, V>`) for the allocator.
-  using Pair = std::pair<int, char>;
+  using Pair      = std::pair<int, char>;
   using ConstPair = std::pair<const int, char>;
   for_all_iterators_and_allocators<ConstPair, const Pair*>([]<class Iter, class Sent, class Alloc>() {
-    test_map_set_insert_range<std::unordered_map<int, char, test_hash<int>, test_equal_to<int>, Alloc>, Pair, Iter, Sent>();
+    test_map_set_insert_range<std::unordered_map<int, char, test_hash<int>, test_equal_to<int>, Alloc>,
+                              Pair,
+                              Iter,
+                              Sent>();
   });
 
   static_assert(test_map_constraints_insert_range<std::unordered_map, int, int, char, double>());
diff --git a/libcxx/test/std/containers/unord/unord.map/unord.map.modifiers/insert_rvalue.pass.cpp b/libcxx/test/std/containers/unord/unord.map/unord.map.modifiers/insert_rvalue.pass.cpp
index 94f0d9b914027..b9659b00937a5 100644
--- a/libcxx/test/std/containers/unord/unord.map/unord.map.modifiers/insert_rvalue.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.map/unord.map.modifiers/insert_rvalue.pass.cpp
@@ -25,154 +25,161 @@
 #include "MoveOnly.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef std::unordered_map<double, int> C;
-        typedef std::pair<C::iterator, bool> R;
-        typedef std::pair<double, short> P;
-        C c;
-        R r = c.insert(P(3.5, static_cast<short>(3)));
-        assert(r.second);
-        assert(c.size() == 1);
-        assert(r.first->first == 3.5);
-        assert(r.first->second == 3);
-
-        r = c.insert(P(3.5, static_cast<short>(4)));
-        assert(!r.second);
-        assert(c.size() == 1);
-        assert(r.first->first == 3.5);
-        assert(r.first->second == 3);
-
-        r = c.insert(P(4.5, static_cast<short>(4)));
-        assert(r.second);
-        assert(c.size() == 2);
-        assert(r.first->first == 4.5);
-        assert(r.first->second == 4);
-
-        r = c.insert(P(5.5, static_cast<short>(4)));
-        assert(r.second);
-        assert(c.size() == 3);
-        assert(r.first->first == 5.5);
-        assert(r.first->second == 4);
-    }
-    {
-        typedef std::unordered_map<MoveOnly, MoveOnly> C;
-        typedef std::pair<C::iterator, bool> R;
-        typedef std::pair<MoveOnly, MoveOnly> P;
-        C c;
-        R r = c.insert(P(3, 3));
-        assert(r.second);
-        assert(c.size() == 1);
-        assert(r.first->first == 3);
-        assert(r.first->second == 3);
-
-        r = c.insert(P(3, 4));
-        assert(!r.second);
-        assert(c.size() == 1);
-        assert(r.first->first == 3);
-        assert(r.first->second == 3);
-
-        r = c.insert(P(4, 4));
-        assert(r.second);
-        assert(c.size() == 2);
-        assert(r.first->first == 4);
-        assert(r.first->second == 4);
-
-        r = c.insert(P(5, 4));
-        assert(r.second);
-        assert(c.size() == 3);
-        assert(r.first->first == 5);
-        assert(r.first->second == 4);
-    }
-    {
-        typedef std::unordered_map<double, int, std::hash<double>, std::equal_to<double>,
-                            min_allocator<std::pair<const double, int>>> C;
-        typedef std::pair<C::iterator, bool> R;
-        typedef std::pair<double, short> P;
-        C c;
-        R r = c.insert(P(3.5, static_cast<short>(3)));
-        assert(r.second);
-        assert(c.size() == 1);
-        assert(r.first->first == 3.5);
-        assert(r.first->second == 3);
-
-        r = c.insert(P(3.5, static_cast<short>(4)));
-        assert(!r.second);
-        assert(c.size() == 1);
-        assert(r.first->first == 3.5);
-        assert(r.first->second == 3);
-
-        r = c.insert(P(4.5, static_cast<short>(4)));
-        assert(r.second);
-        assert(c.size() == 2);
-        assert(r.first->first == 4.5);
-        assert(r.first->second == 4);
-
-        r = c.insert(P(5.5, static_cast<short>(4)));
-        assert(r.second);
-        assert(c.size() == 3);
-        assert(r.first->first == 5.5);
-        assert(r.first->second == 4);
-    }
-    {
-        typedef std::unordered_map<MoveOnly, MoveOnly, std::hash<MoveOnly>, std::equal_to<MoveOnly>,
-                            min_allocator<std::pair<const MoveOnly, MoveOnly>>> C;
-        typedef std::pair<C::iterator, bool> R;
-        typedef std::pair<MoveOnly, MoveOnly> P;
-        C c;
-        R r = c.insert(P(3, 3));
-        assert(r.second);
-        assert(c.size() == 1);
-        assert(r.first->first == 3);
-        assert(r.first->second == 3);
-
-        r = c.insert(P(3, 4));
-        assert(!r.second);
-        assert(c.size() == 1);
-        assert(r.first->first == 3);
-        assert(r.first->second == 3);
-
-        r = c.insert(P(4, 4));
-        assert(r.second);
-        assert(c.size() == 2);
-        assert(r.first->first == 4);
-        assert(r.first->second == 4);
-
-        r = c.insert(P(5, 4));
-        assert(r.second);
-        assert(c.size() == 3);
-        assert(r.first->first == 5);
-        assert(r.first->second == 4);
-    }
-    {
-        typedef std::unordered_map<double, MoveOnly> C;
-        typedef std::pair<C::iterator, bool> R;
-        C c;
-        R r = c.insert({3.5, 3});
-        assert(r.second);
-        assert(c.size() == 1);
-        assert(r.first->first == 3.5);
-        assert(r.first->second == 3);
-
-        r = c.insert({3.5, 4});
-        assert(!r.second);
-        assert(c.size() == 1);
-        assert(r.first->first == 3.5);
-        assert(r.first->second == 3);
-
-        r = c.insert({4.5, 4});
-        assert(r.second);
-        assert(c.size() == 2);
-        assert(r.first->first == 4.5);
-        assert(r.first->second == 4);
-
-        r = c.insert({5.5, 4});
-        assert(r.second);
-        assert(c.size() == 3);
-        assert(r.first->first == 5.5);
-        assert(r.first->second == 4);
-    }
+int main(int, char**) {
+  {
+    typedef std::unordered_map<double, int> C;
+    typedef std::pair<C::iterator, bool> R;
+    typedef std::pair<double, short> P;
+    C c;
+    R r = c.insert(P(3.5, static_cast<short>(3)));
+    assert(r.second);
+    assert(c.size() == 1);
+    assert(r.first->first == 3.5);
+    assert(r.first->second == 3);
+
+    r = c.insert(P(3.5, static_cast<short>(4)));
+    assert(!r.second);
+    assert(c.size() == 1);
+    assert(r.first->first == 3.5);
+    assert(r.first->second == 3);
+
+    r = c.insert(P(4.5, static_cast<short>(4)));
+    assert(r.second);
+    assert(c.size() == 2);
+    assert(r.first->first == 4.5);
+    assert(r.first->second == 4);
+
+    r = c.insert(P(5.5, static_cast<short>(4)));
+    assert(r.second);
+    assert(c.size() == 3);
+    assert(r.first->first == 5.5);
+    assert(r.first->second == 4);
+  }
+  {
+    typedef std::unordered_map<MoveOnly, MoveOnly> C;
+    typedef std::pair<C::iterator, bool> R;
+    typedef std::pair<MoveOnly, MoveOnly> P;
+    C c;
+    R r = c.insert(P(3, 3));
+    assert(r.second);
+    assert(c.size() == 1);
+    assert(r.first->first == 3);
+    assert(r.first->second == 3);
+
+    r = c.insert(P(3, 4));
+    assert(!r.second);
+    assert(c.size() == 1);
+    assert(r.first->first == 3);
+    assert(r.first->second == 3);
+
+    r = c.insert(P(4, 4));
+    assert(r.second);
+    assert(c.size() == 2);
+    assert(r.first->first == 4);
+    assert(r.first->second == 4);
+
+    r = c.insert(P(5, 4));
+    assert(r.second);
+    assert(c.size() == 3);
+    assert(r.first->first == 5);
+    assert(r.first->second == 4);
+  }
+  {
+    typedef std::unordered_map<double,
+                               int,
+                               std::hash<double>,
+                               std::equal_to<double>,
+                               min_allocator<std::pair<const double, int>>>
+        C;
+    typedef std::pair<C::iterator, bool> R;
+    typedef std::pair<double, short> P;
+    C c;
+    R r = c.insert(P(3.5, static_cast<short>(3)));
+    assert(r.second);
+    assert(c.size() == 1);
+    assert(r.first->first == 3.5);
+    assert(r.first->second == 3);
+
+    r = c.insert(P(3.5, static_cast<short>(4)));
+    assert(!r.second);
+    assert(c.size() == 1);
+    assert(r.first->first == 3.5);
+    assert(r.first->second == 3);
+
+    r = c.insert(P(4.5, static_cast<short>(4)));
+    assert(r.second);
+    assert(c.size() == 2);
+    assert(r.first->first == 4.5);
+    assert(r.first->second == 4);
+
+    r = c.insert(P(5.5, static_cast<short>(4)));
+    assert(r.second);
+    assert(c.size() == 3);
+    assert(r.first->first == 5.5);
+    assert(r.first->second == 4);
+  }
+  {
+    typedef std::unordered_map<MoveOnly,
+                               MoveOnly,
+                               std::hash<MoveOnly>,
+                               std::equal_to<MoveOnly>,
+                               min_allocator<std::pair<const MoveOnly, MoveOnly>>>
+        C;
+    typedef std::pair<C::iterator, bool> R;
+    typedef std::pair<MoveOnly, MoveOnly> P;
+    C c;
+    R r = c.insert(P(3, 3));
+    assert(r.second);
+    assert(c.size() == 1);
+    assert(r.first->first == 3);
+    assert(r.first->second == 3);
+
+    r = c.insert(P(3, 4));
+    assert(!r.second);
+    assert(c.size() == 1);
+    assert(r.first->first == 3);
+    assert(r.first->second == 3);
+
+    r = c.insert(P(4, 4));
+    assert(r.second);
+    assert(c.size() == 2);
+    assert(r.first->first == 4);
+    assert(r.first->second == 4);
+
+    r = c.insert(P(5, 4));
+    assert(r.second);
+    assert(c.size() == 3);
+    assert(r.first->first == 5);
+    assert(r.first->second == 4);
+  }
+  {
+    typedef std::unordered_map<double, MoveOnly> C;
+    typedef std::pair<C::iterator, bool> R;
+    C c;
+    R r = c.insert({3.5, 3});
+    assert(r.second);
+    assert(c.size() == 1);
+    assert(r.first->first == 3.5);
+    assert(r.first->second == 3);
+
+    r = c.insert({3.5, 4});
+    assert(!r.second);
+    assert(c.size() == 1);
+    assert(r.first->first == 3.5);
+    assert(r.first->second == 3);
+
+    r = c.insert({4.5, 4});
+    assert(r.second);
+    assert(c.size() == 2);
+    assert(r.first->first == 4.5);
+    assert(r.first->second == 4);
+
+    r = c.insert({5.5, 4});
+    assert(r.second);
+    assert(c.size() == 3);
+    assert(r.first->first == 5.5);
+    assert(r.first->second == 4);
+  }
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/unord/unord.map/unord.map.modifiers/merge.pass.cpp b/libcxx/test/std/containers/unord/unord.map/unord.map.modifiers/merge.pass.cpp
index e5c29b13dcf12..12095bffe40a7 100644
--- a/libcxx/test/std/containers/unord/unord.map/unord.map.modifiers/merge.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.map/unord.map.modifiers/merge.pass.cpp
@@ -27,131 +27,116 @@
 #include "Counter.h"
 
 template <class Map>
-bool map_equal(const Map& map, Map other)
-{
-    return map == other;
+bool map_equal(const Map& map, Map other) {
+  return map == other;
 }
 
 #ifndef TEST_HAS_NO_EXCEPTIONS
 template <class T>
-struct throw_hasher
-{
-    bool& should_throw_;
+struct throw_hasher {
+  bool& should_throw_;
 
-    throw_hasher(bool& should_throw) : should_throw_(should_throw) {}
+  throw_hasher(bool& should_throw) : should_throw_(should_throw) {}
 
-    std::size_t operator()(const T& p) const
-    {
-        if (should_throw_)
-            throw 0;
-        return std::hash<T>()(p);
-    }
+  std::size_t operator()(const T& p) const {
+    if (should_throw_)
+      throw 0;
+    return std::hash<T>()(p);
+  }
 };
 #endif
 
-int main(int, char**)
-{
-    {
-        std::unordered_map<int, int> src{{1, 0}, {3, 0}, {5, 0}};
-        std::unordered_map<int, int> dst{{2, 0}, {4, 0}, {5, 0}};
-        dst.merge(src);
-        assert(map_equal(src, {{5,0}}));
-        assert(map_equal(dst, {{1, 0}, {2, 0}, {3, 0}, {4, 0}, {5, 0}}));
-    }
+int main(int, char**) {
+  {
+    std::unordered_map<int, int> src{{1, 0}, {3, 0}, {5, 0}};
+    std::unordered_map<int, int> dst{{2, 0}, {4, 0}, {5, 0}};
+    dst.merge(src);
+    assert(map_equal(src, {{5, 0}}));
+    assert(map_equal(dst, {{1, 0}, {2, 0}, {3, 0}, {4, 0}, {5, 0}}));
+  }
 
 #ifndef TEST_HAS_NO_EXCEPTIONS
-    {
-        bool do_throw = false;
-        typedef std::unordered_map<Counter<int>, int, throw_hasher<Counter<int>>> map_type;
-        map_type src({{1, 0}, {3, 0}, {5, 0}}, 0, throw_hasher<Counter<int>>(do_throw));
-        map_type dst({{2, 0}, {4, 0}, {5, 0}}, 0, throw_hasher<Counter<int>>(do_throw));
-
-        assert(Counter_base::gConstructed == 6);
-
-        do_throw = true;
-        try
-        {
-            dst.merge(src);
-        }
-        catch (int)
-        {
-            do_throw = false;
-        }
-        assert(!do_throw);
-        assert(map_equal(src, map_type({{1, 0}, {3, 0}, {5, 0}}, 0, throw_hasher<Counter<int>>(do_throw))));
-        assert(map_equal(dst, map_type({{2, 0}, {4, 0}, {5, 0}}, 0, throw_hasher<Counter<int>>(do_throw))));
+  {
+    bool do_throw = false;
+    typedef std::unordered_map<Counter<int>, int, throw_hasher<Counter<int>>> map_type;
+    map_type src({{1, 0}, {3, 0}, {5, 0}}, 0, throw_hasher<Counter<int>>(do_throw));
+    map_type dst({{2, 0}, {4, 0}, {5, 0}}, 0, throw_hasher<Counter<int>>(do_throw));
+
+    assert(Counter_base::gConstructed == 6);
+
+    do_throw = true;
+    try {
+      dst.merge(src);
+    } catch (int) {
+      do_throw = false;
     }
+    assert(!do_throw);
+    assert(map_equal(src, map_type({{1, 0}, {3, 0}, {5, 0}}, 0, throw_hasher<Counter<int>>(do_throw))));
+    assert(map_equal(dst, map_type({{2, 0}, {4, 0}, {5, 0}}, 0, throw_hasher<Counter<int>>(do_throw))));
+  }
 #endif
-    assert(Counter_base::gConstructed == 0);
-    struct equal
-    {
-        equal() = default;
-
-        bool operator()(const Counter<int>& lhs, const Counter<int>& rhs) const
-        {
-            return lhs == rhs;
-        }
-    };
-    struct hasher
-    {
-        hasher() = default;
-        std::size_t operator()(const Counter<int>& p) const
-        {
-            return std::hash<Counter<int>>()(p);
-        }
-    };
-    {
-        typedef std::unordered_map<Counter<int>, int, std::hash<Counter<int>>, std::equal_to<Counter<int>>> first_map_type;
-        typedef std::unordered_map<Counter<int>, int, hasher, equal> second_map_type;
-        typedef std::unordered_multimap<Counter<int>, int, hasher, equal> third_map_type;
+  assert(Counter_base::gConstructed == 0);
+  struct equal {
+    equal() = default;
+
+    bool operator()(const Counter<int>& lhs, const Counter<int>& rhs) const { return lhs == rhs; }
+  };
+  struct hasher {
+    hasher() = default;
+    std::size_t operator()(const Counter<int>& p) const { return std::hash<Counter<int>>()(p); }
+  };
+  {
+    typedef std::unordered_map<Counter<int>, int, std::hash<Counter<int>>, std::equal_to<Counter<int>>> first_map_type;
+    typedef std::unordered_map<Counter<int>, int, hasher, equal> second_map_type;
+    typedef std::unordered_multimap<Counter<int>, int, hasher, equal> third_map_type;
 
-        {
-            first_map_type first{{1, 0}, {2, 0}, {3, 0}};
-            second_map_type second{{2, 0}, {3, 0}, {4, 0}};
-            third_map_type third{{1, 0}, {3, 0}};
+    {
+      first_map_type first{{1, 0}, {2, 0}, {3, 0}};
+      second_map_type second{{2, 0}, {3, 0}, {4, 0}};
+      third_map_type third{{1, 0}, {3, 0}};
 
-            assert(Counter_base::gConstructed == 8);
+      assert(Counter_base::gConstructed == 8);
 
-            first.merge(second);
-            first.merge(third);
+      first.merge(second);
+      first.merge(third);
 
-            assert(map_equal(first, {{1, 0}, {2, 0}, {3, 0}, {4, 0}}));
-            assert(map_equal(second, {{2, 0}, {3, 0}}));
-            assert(map_equal(third, {{1, 0}, {3, 0}}));
+      assert(map_equal(first, {{1, 0}, {2, 0}, {3, 0}, {4, 0}}));
+      assert(map_equal(second, {{2, 0}, {3, 0}}));
+      assert(map_equal(third, {{1, 0}, {3, 0}}));
 
-            assert(Counter_base::gConstructed == 8);
-        }
-        assert(Counter_base::gConstructed == 0);
-        {
-            first_map_type first{{1, 0}, {2, 0}, {3, 0}};
-            second_map_type second{{2, 0}, {3, 0}, {4, 0}};
-            third_map_type third{{1, 0}, {3, 0}};
+      assert(Counter_base::gConstructed == 8);
+    }
+    assert(Counter_base::gConstructed == 0);
+    {
+      first_map_type first{{1, 0}, {2, 0}, {3, 0}};
+      second_map_type second{{2, 0}, {3, 0}, {4, 0}};
+      third_map_type third{{1, 0}, {3, 0}};
 
-            assert(Counter_base::gConstructed == 8);
+      assert(Counter_base::gConstructed == 8);
 
-            first.merge(std::move(second));
-            first.merge(std::move(third));
+      first.merge(std::move(second));
+      first.merge(std::move(third));
 
-            assert(map_equal(first, {{1, 0}, {2, 0}, {3, 0}, {4, 0}}));
-            assert(map_equal(second, {{2, 0}, {3, 0}}));
-            assert(map_equal(third, {{1, 0}, {3, 0}}));
+      assert(map_equal(first, {{1, 0}, {2, 0}, {3, 0}, {4, 0}}));
+      assert(map_equal(second, {{2, 0}, {3, 0}}));
+      assert(map_equal(third, {{1, 0}, {3, 0}}));
 
-            assert(Counter_base::gConstructed == 8);
-        }
-        assert(Counter_base::gConstructed == 0);
+      assert(Counter_base::gConstructed == 8);
+    }
+    assert(Counter_base::gConstructed == 0);
+  }
+  {
+    std::unordered_map<int, int> first;
+    {
+      std::unordered_map<int, int> second;
+      first.merge(second);
+      first.merge(std::move(second));
     }
     {
-        std::unordered_map<int, int> first;
-        {
-            std::unordered_map<int, int> second;
-            first.merge(second);
-            first.merge(std::move(second));
-        }
-        {
-            std::unordered_multimap<int, int> second;
-            first.merge(second);
-            first.merge(std::move(second));
-        }
+      std::unordered_multimap<int, int> second;
+      first.merge(second);
+      first.merge(std::move(second));
     }
-    return 0;
+  }
+  return 0;
 }
diff --git a/libcxx/test/std/containers/unord/unord.map/unord.map.modifiers/try.emplace.pass.cpp b/libcxx/test/std/containers/unord/unord.map/unord.map.modifiers/try.emplace.pass.cpp
index 15cf8bc59eba9..a7d76d310e5be 100644
--- a/libcxx/test/std/containers/unord/unord.map/unord.map.modifiers/try.emplace.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.map/unord.map.modifiers/try.emplace.pass.cpp
@@ -28,33 +28,34 @@
 
 #include "test_macros.h"
 
-class Moveable
-{
-    Moveable(const Moveable&);
-    Moveable& operator=(const Moveable&);
+class Moveable {
+  Moveable(const Moveable&);
+  Moveable& operator=(const Moveable&);
+
+  int int_;
+  double double_;
 
-    int int_;
-    double double_;
 public:
-    Moveable() : int_(0), double_(0) {}
-    Moveable(int i, double d) : int_(i), double_(d) {}
-    Moveable(Moveable&& x)
-        : int_(x.int_), double_(x.double_)
-            {x.int_ = -1; x.double_ = -1;}
-    Moveable& operator=(Moveable&& x)
-        {int_ = x.int_; x.int_ = -1;
-         double_ = x.double_; x.double_ = -1;
-         return *this;
-        }
-
-    bool operator==(const Moveable& x) const
-        {return int_ == x.int_ && double_ == x.double_;}
-    bool operator<(const Moveable& x) const
-        {return int_ < x.int_ || (int_ == x.int_ && double_ < x.double_);}
-    std::size_t hash () const { return std::hash<int>()(int_) + std::hash<double>()(double_); }
-
-    int get() const {return int_;}
-    bool moved() const {return int_ == -1;}
+  Moveable() : int_(0), double_(0) {}
+  Moveable(int i, double d) : int_(i), double_(d) {}
+  Moveable(Moveable&& x) : int_(x.int_), double_(x.double_) {
+    x.int_    = -1;
+    x.double_ = -1;
+  }
+  Moveable& operator=(Moveable&& x) {
+    int_      = x.int_;
+    x.int_    = -1;
+    double_   = x.double_;
+    x.double_ = -1;
+    return *this;
+  }
+
+  bool operator==(const Moveable& x) const { return int_ == x.int_ && double_ == x.double_; }
+  bool operator<(const Moveable& x) const { return int_ < x.int_ || (int_ == x.int_ && double_ < x.double_); }
+  std::size_t hash() const { return std::hash<int>()(int_) + std::hash<double>()(double_); }
+
+  int get() const { return int_; }
+  bool moved() const { return int_ == -1; }
 };
 
 template <>
@@ -62,131 +63,127 @@ struct std::hash<Moveable> {
   std::size_t operator()(const Moveable& m) const { return m.hash(); }
 };
 
-int main(int, char**)
-{
-
-    { // pair<iterator, bool> try_emplace(const key_type& k, Args&&... args);
-        typedef std::unordered_map<int, Moveable> M;
-        typedef std::pair<M::iterator, bool> R;
-        M m;
-        R r;
-        for (int i = 0; i < 20; i += 2)
-            m.emplace (i, Moveable(i, (double) i));
-        assert(m.size() == 10);
-
-        Moveable mv1(3, 3.0);
-        for (int i=0; i < 20; i += 2)
-        {
-            r = m.try_emplace(i, std::move(mv1));
-            assert(m.size() == 10);
-            assert(!r.second);              // was not inserted
-            assert(!mv1.moved());           // was not moved from
-            assert(r.first->first == i);    // key
-        }
-
-        r = m.try_emplace(-1, std::move(mv1));
-        assert(m.size() == 11);
-        assert(r.second);                   // was inserted
-        assert(mv1.moved());                // was moved from
-        assert(r.first->first == -1);       // key
-        assert(r.first->second.get() == 3); // value
-
-        Moveable mv2(5, 3.0);
-        r = m.try_emplace(5, std::move(mv2));
-        assert(m.size() == 12);
-        assert(r.second);                   // was inserted
-        assert(mv2.moved());                // was moved from
-        assert(r.first->first == 5);        // key
-        assert(r.first->second.get() == 5); // value
-
-        Moveable mv3(-1, 3.0);
-        r = m.try_emplace(117, std::move(mv2));
-        assert(m.size() == 13);
-        assert(r.second);                    // was inserted
-        assert(mv2.moved());                 // was moved from
-        assert(r.first->first == 117);       // key
-        assert(r.first->second.get() == -1); // value
-    }
-
-    {  // pair<iterator, bool> try_emplace(key_type&& k, Args&&... args);
-        typedef std::unordered_map<Moveable, Moveable> M;
-        typedef std::pair<M::iterator, bool> R;
-        M m;
-        R r;
-        for (int i = 0; i < 20; i += 2)
-            m.emplace ( Moveable(i, (double) i), Moveable(i+1, (double) i+1));
-        assert(m.size() == 10);
-
-        Moveable mvkey1(2, 2.0);
-        Moveable mv1(4, 4.0);
-        r = m.try_emplace(std::move(mvkey1), std::move(mv1));
-        assert(m.size() == 10);
-        assert(!r.second);                 // was not inserted
-        assert(!mv1.moved());              // was not moved from
-        assert(!mvkey1.moved());           // was not moved from
-        assert(r.first->first == mvkey1);  // key
-
-        Moveable mvkey2(3, 3.0);
-        r = m.try_emplace(std::move(mvkey2), std::move(mv1));
-        assert(m.size() == 11);
-        assert(r.second);                   // was inserted
-        assert(mv1.moved());                // was moved from
-        assert(mvkey2.moved());             // was moved from
-        assert(r.first->first.get()  == 3); // key
-        assert(r.first->second.get() == 4); // value
+int main(int, char**) {
+  { // pair<iterator, bool> try_emplace(const key_type& k, Args&&... args);
+    typedef std::unordered_map<int, Moveable> M;
+    typedef std::pair<M::iterator, bool> R;
+    M m;
+    R r;
+    for (int i = 0; i < 20; i += 2)
+      m.emplace(i, Moveable(i, (double)i));
+    assert(m.size() == 10);
+
+    Moveable mv1(3, 3.0);
+    for (int i = 0; i < 20; i += 2) {
+      r = m.try_emplace(i, std::move(mv1));
+      assert(m.size() == 10);
+      assert(!r.second);           // was not inserted
+      assert(!mv1.moved());        // was not moved from
+      assert(r.first->first == i); // key
     }
 
-    {  // iterator try_emplace(const_iterator hint, const key_type& k, Args&&... args);
-        typedef std::unordered_map<int, Moveable> M;
-        M m;
-        M::iterator r;
-        for (int i = 0; i < 20; i += 2)
-            m.try_emplace ( i, Moveable(i, (double) i));
-        assert(m.size() == 10);
-        M::const_iterator it = m.find(2);
-
-        Moveable mv1(3, 3.0);
-        for (int i=0; i < 20; i += 2)
-        {
-            r = m.try_emplace(it, i, std::move(mv1));
-            assert(m.size() == 10);
-            assert(!mv1.moved());         // was not moved from
-            assert(r->first == i);        // key
-            assert(r->second.get() == i); // value
-        }
-
-        r = m.try_emplace(it, 3, std::move(mv1));
-        assert(m.size() == 11);
-        assert(mv1.moved());          // was moved from
-        assert(r->first == 3);        // key
-        assert(r->second.get() == 3); // value
+    r = m.try_emplace(-1, std::move(mv1));
+    assert(m.size() == 11);
+    assert(r.second);                   // was inserted
+    assert(mv1.moved());                // was moved from
+    assert(r.first->first == -1);       // key
+    assert(r.first->second.get() == 3); // value
+
+    Moveable mv2(5, 3.0);
+    r = m.try_emplace(5, std::move(mv2));
+    assert(m.size() == 12);
+    assert(r.second);                   // was inserted
+    assert(mv2.moved());                // was moved from
+    assert(r.first->first == 5);        // key
+    assert(r.first->second.get() == 5); // value
+
+    Moveable mv3(-1, 3.0);
+    r = m.try_emplace(117, std::move(mv2));
+    assert(m.size() == 13);
+    assert(r.second);                    // was inserted
+    assert(mv2.moved());                 // was moved from
+    assert(r.first->first == 117);       // key
+    assert(r.first->second.get() == -1); // value
+  }
+
+  { // pair<iterator, bool> try_emplace(key_type&& k, Args&&... args);
+    typedef std::unordered_map<Moveable, Moveable> M;
+    typedef std::pair<M::iterator, bool> R;
+    M m;
+    R r;
+    for (int i = 0; i < 20; i += 2)
+      m.emplace(Moveable(i, (double)i), Moveable(i + 1, (double)i + 1));
+    assert(m.size() == 10);
+
+    Moveable mvkey1(2, 2.0);
+    Moveable mv1(4, 4.0);
+    r = m.try_emplace(std::move(mvkey1), std::move(mv1));
+    assert(m.size() == 10);
+    assert(!r.second);                // was not inserted
+    assert(!mv1.moved());             // was not moved from
+    assert(!mvkey1.moved());          // was not moved from
+    assert(r.first->first == mvkey1); // key
+
+    Moveable mvkey2(3, 3.0);
+    r = m.try_emplace(std::move(mvkey2), std::move(mv1));
+    assert(m.size() == 11);
+    assert(r.second);                   // was inserted
+    assert(mv1.moved());                // was moved from
+    assert(mvkey2.moved());             // was moved from
+    assert(r.first->first.get() == 3);  // key
+    assert(r.first->second.get() == 4); // value
+  }
+
+  { // iterator try_emplace(const_iterator hint, const key_type& k, Args&&... args);
+    typedef std::unordered_map<int, Moveable> M;
+    M m;
+    M::iterator r;
+    for (int i = 0; i < 20; i += 2)
+      m.try_emplace(i, Moveable(i, (double)i));
+    assert(m.size() == 10);
+    M::const_iterator it = m.find(2);
+
+    Moveable mv1(3, 3.0);
+    for (int i = 0; i < 20; i += 2) {
+      r = m.try_emplace(it, i, std::move(mv1));
+      assert(m.size() == 10);
+      assert(!mv1.moved());         // was not moved from
+      assert(r->first == i);        // key
+      assert(r->second.get() == i); // value
     }
 
-    {  // iterator try_emplace(const_iterator hint, key_type&& k, Args&&... args);
-        typedef std::unordered_map<Moveable, Moveable> M;
-        M m;
-        M::iterator r;
-        for ( int i = 0; i < 20; i += 2 )
-            m.emplace ( Moveable(i, (double) i), Moveable(i+1, (double) i+1));
-        assert(m.size() == 10);
-        M::const_iterator it = std::next(m.cbegin());
-
-        Moveable mvkey1(2, 2.0);
-        Moveable mv1(4, 4.0);
-        r = m.try_emplace(it, std::move(mvkey1), std::move(mv1));
-        assert(m.size() == 10);
-        assert(!mv1.moved());        // was not moved from
-        assert(!mvkey1.moved());     // was not moved from
-        assert(r->first == mvkey1);  // key
-
-        Moveable mvkey2(3, 3.0);
-        r = m.try_emplace(it, std::move(mvkey2), std::move(mv1));
-        assert(m.size() == 11);
-        assert(mv1.moved());          // was moved from
-        assert(mvkey2.moved());       // was moved from
-        assert(r->first.get()  == 3); // key
-        assert(r->second.get() == 4); // value
-    }
+    r = m.try_emplace(it, 3, std::move(mv1));
+    assert(m.size() == 11);
+    assert(mv1.moved());          // was moved from
+    assert(r->first == 3);        // key
+    assert(r->second.get() == 3); // value
+  }
+
+  { // iterator try_emplace(const_iterator hint, key_type&& k, Args&&... args);
+    typedef std::unordered_map<Moveable, Moveable> M;
+    M m;
+    M::iterator r;
+    for (int i = 0; i < 20; i += 2)
+      m.emplace(Moveable(i, (double)i), Moveable(i + 1, (double)i + 1));
+    assert(m.size() == 10);
+    M::const_iterator it = std::next(m.cbegin());
+
+    Moveable mvkey1(2, 2.0);
+    Moveable mv1(4, 4.0);
+    r = m.try_emplace(it, std::move(mvkey1), std::move(mv1));
+    assert(m.size() == 10);
+    assert(!mv1.moved());       // was not moved from
+    assert(!mvkey1.moved());    // was not moved from
+    assert(r->first == mvkey1); // key
+
+    Moveable mvkey2(3, 3.0);
+    r = m.try_emplace(it, std::move(mvkey2), std::move(mv1));
+    assert(m.size() == 11);
+    assert(mv1.moved());          // was moved from
+    assert(mvkey2.moved());       // was moved from
+    assert(r->first.get() == 3);  // key
+    assert(r->second.get() == 4); // value
+  }
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/unord/unord.map/unord.map.swap/swap_noexcept.pass.cpp b/libcxx/test/std/containers/unord/unord.map/unord.map.swap/swap_noexcept.pass.cpp
index 44f8cf57703ae..4dc9a5d37d50b 100644
--- a/libcxx/test/std/containers/unord/unord.map/unord.map.swap/swap_noexcept.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.map/unord.map.swap/swap_noexcept.pass.cpp
@@ -33,23 +33,21 @@
 #include "test_allocator.h"
 
 template <class T>
-struct some_comp
-{
-    typedef T value_type;
+struct some_comp {
+  typedef T value_type;
 
-    some_comp() {}
-    some_comp(const some_comp&) {}
-    bool operator()(const T&, const T&) const { return false; }
+  some_comp() {}
+  some_comp(const some_comp&) {}
+  bool operator()(const T&, const T&) const { return false; }
 };
 
 template <class T>
-struct some_comp2
-{
-    typedef T value_type;
+struct some_comp2 {
+  typedef T value_type;
 
-    some_comp2() {}
-    some_comp2(const some_comp2&) {}
-    bool operator()(const T&, const T&) const { return false; }
+  some_comp2() {}
+  some_comp2(const some_comp2&) {}
+  bool operator()(const T&, const T&) const { return false; }
 };
 
 #if TEST_STD_VER >= 14
@@ -58,21 +56,19 @@ void swap(some_comp2<T>&, some_comp2<T>&) noexcept {}
 #endif
 
 template <class T>
-struct some_hash
-{
-    typedef T value_type;
-    some_hash() {}
-    some_hash(const some_hash&);
-    std::size_t operator()(T const&) const;
+struct some_hash {
+  typedef T value_type;
+  some_hash() {}
+  some_hash(const some_hash&);
+  std::size_t operator()(T const&) const;
 };
 
 template <class T>
-struct some_hash2
-{
-    typedef T value_type;
-    some_hash2() {}
-    some_hash2(const some_hash2&);
-    std::size_t operator()(T const&) const;
+struct some_hash2 {
+  typedef T value_type;
+  some_hash2() {}
+  some_hash2(const some_hash2&);
+  std::size_t operator()(T const&) const;
 };
 
 #if TEST_STD_VER >= 14
@@ -81,112 +77,108 @@ void swap(some_hash2<T>&, some_hash2<T>&) noexcept {}
 #endif
 
 template <class T>
-struct some_alloc
-{
-    typedef T value_type;
+struct some_alloc {
+  typedef T value_type;
 
-    some_alloc() {}
-    some_alloc(const some_alloc&);
-    void deallocate(void*, unsigned) {}
+  some_alloc() {}
+  some_alloc(const some_alloc&);
+  void deallocate(void*, unsigned) {}
 
-    typedef std::true_type propagate_on_container_swap;
+  typedef std::true_type propagate_on_container_swap;
 };
 
 template <class T>
-struct some_alloc2
-{
-    typedef T value_type;
+struct some_alloc2 {
+  typedef T value_type;
 
-    some_alloc2() {}
-    some_alloc2(const some_alloc2&);
-    void deallocate(void*, unsigned) {}
+  some_alloc2() {}
+  some_alloc2(const some_alloc2&);
+  void deallocate(void*, unsigned) {}
 
-    typedef std::false_type propagate_on_container_swap;
-    typedef std::true_type is_always_equal;
+  typedef std::false_type propagate_on_container_swap;
+  typedef std::true_type is_always_equal;
 };
 
 template <class T>
-struct some_alloc3
-{
-    typedef T value_type;
+struct some_alloc3 {
+  typedef T value_type;
 
-    some_alloc3() {}
-    some_alloc3(const some_alloc3&);
-    void deallocate(void*, unsigned) {}
+  some_alloc3() {}
+  some_alloc3(const some_alloc3&);
+  void deallocate(void*, unsigned) {}
 
-    typedef std::false_type propagate_on_container_swap;
-    typedef std::false_type is_always_equal;
+  typedef std::false_type propagate_on_container_swap;
+  typedef std::false_type is_always_equal;
 };
 
-
-int main(int, char**)
-{
-    typedef std::pair<const MoveOnly, MoveOnly> MapType;
-    {
-        typedef std::unordered_map<MoveOnly, MoveOnly> C;
-        static_assert(noexcept(swap(std::declval<C&>(), std::declval<C&>())), "");
-    }
+int main(int, char**) {
+  typedef std::pair<const MoveOnly, MoveOnly> MapType;
+  {
+    typedef std::unordered_map<MoveOnly, MoveOnly> C;
+    static_assert(noexcept(swap(std::declval<C&>(), std::declval<C&>())), "");
+  }
 #if defined(_LIBCPP_VERSION)
-    {
-        typedef std::unordered_map<MoveOnly, MoveOnly, std::hash<MoveOnly>,
-                           std::equal_to<MoveOnly>, test_allocator<MapType>> C;
-        static_assert(noexcept(swap(std::declval<C&>(), std::declval<C&>())), "");
-    }
-    {
-        typedef std::unordered_map<MoveOnly, MoveOnly, std::hash<MoveOnly>,
-                          std::equal_to<MoveOnly>, other_allocator<MapType>> C;
-        static_assert(noexcept(swap(std::declval<C&>(), std::declval<C&>())), "");
-    }
+  {
+    typedef std::
+        unordered_map<MoveOnly, MoveOnly, std::hash<MoveOnly>, std::equal_to<MoveOnly>, test_allocator<MapType>>
+            C;
+    static_assert(noexcept(swap(std::declval<C&>(), std::declval<C&>())), "");
+  }
+  {
+    typedef std::
+        unordered_map<MoveOnly, MoveOnly, std::hash<MoveOnly>, std::equal_to<MoveOnly>, other_allocator<MapType>>
+            C;
+    static_assert(noexcept(swap(std::declval<C&>(), std::declval<C&>())), "");
+  }
 #endif // _LIBCPP_VERSION
-    {
-        typedef std::unordered_map<MoveOnly, MoveOnly, some_hash<MoveOnly>> C;
-        static_assert(!noexcept(swap(std::declval<C&>(), std::declval<C&>())), "");
-    }
-    {
-        typedef std::unordered_map<MoveOnly, MoveOnly, std::hash<MoveOnly>,
-                                                         some_comp<MoveOnly>> C;
-        static_assert(!noexcept(swap(std::declval<C&>(), std::declval<C&>())), "");
-    }
+  {
+    typedef std::unordered_map<MoveOnly, MoveOnly, some_hash<MoveOnly>> C;
+    static_assert(!noexcept(swap(std::declval<C&>(), std::declval<C&>())), "");
+  }
+  {
+    typedef std::unordered_map<MoveOnly, MoveOnly, std::hash<MoveOnly>, some_comp<MoveOnly>> C;
+    static_assert(!noexcept(swap(std::declval<C&>(), std::declval<C&>())), "");
+  }
 
 #if TEST_STD_VER >= 14
-    { // POCS allocator, throwable swap for hash, throwable swap for comp
-    typedef std::unordered_map<MoveOnly, MoveOnly, some_hash<MoveOnly>, some_comp <MoveOnly>, some_alloc <MapType>> C;
+  { // POCS allocator, throwable swap for hash, throwable swap for comp
+    typedef std::unordered_map<MoveOnly, MoveOnly, some_hash<MoveOnly>, some_comp<MoveOnly>, some_alloc<MapType>> C;
     static_assert(!noexcept(swap(std::declval<C&>(), std::declval<C&>())), "");
-    }
-    { // always equal allocator, throwable swap for hash, throwable swap for comp
-    typedef std::unordered_map<MoveOnly, MoveOnly, some_hash<MoveOnly>, some_comp <MoveOnly>, some_alloc2<MapType>> C;
+  }
+  { // always equal allocator, throwable swap for hash, throwable swap for comp
+    typedef std::unordered_map<MoveOnly, MoveOnly, some_hash<MoveOnly>, some_comp<MoveOnly>, some_alloc2<MapType>> C;
     static_assert(!noexcept(swap(std::declval<C&>(), std::declval<C&>())), "");
-    }
-    { // POCS allocator, throwable swap for hash, nothrow swap for comp
-    typedef std::unordered_map<MoveOnly, MoveOnly, some_hash<MoveOnly>, some_comp2<MoveOnly>, some_alloc <MapType>> C;
+  }
+  { // POCS allocator, throwable swap for hash, nothrow swap for comp
+    typedef std::unordered_map<MoveOnly, MoveOnly, some_hash<MoveOnly>, some_comp2<MoveOnly>, some_alloc<MapType>> C;
     static_assert(!noexcept(swap(std::declval<C&>(), std::declval<C&>())), "");
-    }
-    { // always equal allocator, throwable swap for hash, nothrow swap for comp
+  }
+  { // always equal allocator, throwable swap for hash, nothrow swap for comp
     typedef std::unordered_map<MoveOnly, MoveOnly, some_hash<MoveOnly>, some_comp2<MoveOnly>, some_alloc2<MapType>> C;
     static_assert(!noexcept(swap(std::declval<C&>(), std::declval<C&>())), "");
-    }
-    { // POCS allocator, nothrow swap for hash, throwable swap for comp
-    typedef std::unordered_map<MoveOnly, MoveOnly, some_hash2<MoveOnly>, some_comp <MoveOnly>, some_alloc <MapType>> C;
+  }
+  { // POCS allocator, nothrow swap for hash, throwable swap for comp
+    typedef std::unordered_map<MoveOnly, MoveOnly, some_hash2<MoveOnly>, some_comp<MoveOnly>, some_alloc<MapType>> C;
     static_assert(!noexcept(swap(std::declval<C&>(), std::declval<C&>())), "");
-    }
-    { // always equal allocator, nothrow swap for hash, throwable swap for comp
-    typedef std::unordered_map<MoveOnly, MoveOnly, some_hash2<MoveOnly>, some_comp <MoveOnly>, some_alloc2<MapType>> C;
+  }
+  { // always equal allocator, nothrow swap for hash, throwable swap for comp
+    typedef std::unordered_map<MoveOnly, MoveOnly, some_hash2<MoveOnly>, some_comp<MoveOnly>, some_alloc2<MapType>> C;
     static_assert(!noexcept(swap(std::declval<C&>(), std::declval<C&>())), "");
-    }
-    { // POCS allocator, nothrow swap for hash, nothrow swap for comp
-    typedef std::unordered_map<MoveOnly, MoveOnly, some_hash2<MoveOnly>, some_comp2<MoveOnly>, some_alloc <MapType>> C;
-    static_assert( noexcept(swap(std::declval<C&>(), std::declval<C&>())), "");
-    }
-    { // always equal allocator, nothrow swap for hash, nothrow swap for comp
+  }
+  { // POCS allocator, nothrow swap for hash, nothrow swap for comp
+    typedef std::unordered_map<MoveOnly, MoveOnly, some_hash2<MoveOnly>, some_comp2<MoveOnly>, some_alloc<MapType>> C;
+    static_assert(noexcept(swap(std::declval<C&>(), std::declval<C&>())), "");
+  }
+  { // always equal allocator, nothrow swap for hash, nothrow swap for comp
     typedef std::unordered_map<MoveOnly, MoveOnly, some_hash2<MoveOnly>, some_comp2<MoveOnly>, some_alloc2<MapType>> C;
-    static_assert( noexcept(swap(std::declval<C&>(), std::declval<C&>())), "");
-    }
-#if defined(_LIBCPP_VERSION)
-    { // NOT always equal allocator, nothrow swap for hash, nothrow swap for comp
+    static_assert(noexcept(swap(std::declval<C&>(), std::declval<C&>())), "");
+  }
+#  if defined(_LIBCPP_VERSION)
+  { // NOT always equal allocator, nothrow swap for hash, nothrow swap for comp
     typedef std::unordered_map<MoveOnly, MoveOnly, some_hash2<MoveOnly>, some_comp2<MoveOnly>, some_alloc3<MapType>> C;
-    static_assert( noexcept(swap(std::declval<C&>(), std::declval<C&>())), "");
-    }
-#endif // _LIBCPP_VERSION
+    static_assert(noexcept(swap(std::declval<C&>(), std::declval<C&>())), "");
+  }
+#  endif // _LIBCPP_VERSION
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/unord/unord.map/unord.map.swap/swap_non_member.pass.cpp b/libcxx/test/std/containers/unord/unord.map/unord.map.swap/swap_non_member.pass.cpp
index 2b83a7462b75a..149a0b040a2df 100644
--- a/libcxx/test/std/containers/unord/unord.map/unord.map.swap/swap_non_member.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.map/unord.map.swap/swap_non_member.pass.cpp
@@ -25,555 +25,542 @@
 #include "test_allocator.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef test_hash<int> Hash;
-        typedef test_equal_to<int> Compare;
-        typedef test_allocator<std::pair<const int, std::string> > Alloc;
-        typedef std::unordered_map<int, std::string, Hash, Compare, Alloc> C;
-        C c1(0, Hash(1), Compare(1), Alloc(1, 1));
-        C c2(0, Hash(2), Compare(2), Alloc(1, 2));
-        c2.max_load_factor(2);
-        swap(c1, c2);
-
-        LIBCPP_ASSERT(c1.bucket_count() == 0);
-        assert(c1.size() == 0);
-        assert(c1.hash_function() == Hash(2));
-        assert(c1.key_eq() == Compare(2));
-        assert(c1.get_allocator().get_id() == 1);
-        assert(static_cast<std::size_t>(std::distance(c1.begin(), c1.end())) == c1.size());
-        assert(static_cast<std::size_t>(std::distance(c1.cbegin(), c1.cend())) == c1.size());
-        assert(c1.max_load_factor() == 2);
-
-        LIBCPP_ASSERT(c2.bucket_count() == 0);
-        assert(c2.size() == 0);
-        assert(c2.hash_function() == Hash(1));
-        assert(c2.key_eq() == Compare(1));
-        assert(c2.get_allocator().get_id() == 2);
-        assert(static_cast<std::size_t>(std::distance(c2.begin(), c2.end())) == c2.size());
-        assert(static_cast<std::size_t>(std::distance(c2.cbegin(), c2.cend())) == c2.size());
-        assert(c2.max_load_factor() == 1);
-    }
-    {
-        typedef test_hash<int> Hash;
-        typedef test_equal_to<int> Compare;
-        typedef test_allocator<std::pair<const int, std::string> > Alloc;
-        typedef std::unordered_map<int, std::string, Hash, Compare, Alloc> C;
-        typedef std::pair<int, std::string> P;
-        P a2[] =
-        {
-            P(10, "ten"),
-            P(20, "twenty"),
-            P(30, "thirty"),
-            P(40, "forty"),
-            P(50, "fifty"),
-            P(60, "sixty"),
-            P(70, "seventy"),
-            P(80, "eighty"),
-        };
-        C c1(0, Hash(1), Compare(1), Alloc(1, 1));
-        C c2(std::begin(a2), std::end(a2), 0, Hash(2), Compare(2), Alloc(1, 2));
-        c2.max_load_factor(2);
-        C::iterator it2 = c2.begin();
-        swap(c1, c2);
-
-        assert(c1.bucket_count() >= 8);
-        assert(c1.size() == 8);
-        assert(c1.at(10) == "ten");
-        assert(c1.at(20) == "twenty");
-        assert(c1.at(30) == "thirty");
-        assert(c1.at(40) == "forty");
-        assert(c1.at(50) == "fifty");
-        assert(c1.at(60) == "sixty");
-        assert(c1.at(70) == "seventy");
-        assert(c1.at(80) == "eighty");
-        assert(c1.hash_function() == Hash(2));
-        assert(c1.key_eq() == Compare(2));
-        assert(c1.get_allocator().get_id() == 1);
-        assert(static_cast<std::size_t>(std::distance(c1.begin(), c1.end())) == c1.size());
-        assert(static_cast<std::size_t>(std::distance(c1.cbegin(), c1.cend())) == c1.size());
-        assert(c1.max_load_factor() == 2);
-        assert(it2 == c1.begin()); // Iterators are not invalidated
-
-        LIBCPP_ASSERT(c2.bucket_count() == 0);
-        assert(c2.size() == 0);
-        assert(c2.hash_function() == Hash(1));
-        assert(c2.key_eq() == Compare(1));
-        assert(c2.get_allocator().get_id() == 2);
-        assert(static_cast<std::size_t>(std::distance(c2.begin(), c2.end())) == c2.size());
-        assert(static_cast<std::size_t>(std::distance(c2.cbegin(), c2.cend())) == c2.size());
-        assert(c2.max_load_factor() == 1);
-    }
-    {
-        typedef test_hash<int> Hash;
-        typedef test_equal_to<int> Compare;
-        typedef test_allocator<std::pair<const int, std::string> > Alloc;
-        typedef std::unordered_map<int, std::string, Hash, Compare, Alloc> C;
-        typedef std::pair<int, std::string> P;
-        P a1[] =
-        {
-            P(1, "one"),
-            P(2, "two"),
-            P(3, "three"),
-            P(4, "four"),
-            P(1, "four"),
-            P(2, "four"),
-        };
-        C c1(std::begin(a1), std::end(a1), 0, Hash(1), Compare(1), Alloc(1, 1));
-        C c2(0, Hash(2), Compare(2), Alloc(1, 2));
-        c2.max_load_factor(2);
-        C::iterator it1 = c1.begin();
-        swap(c1, c2);
-
-        LIBCPP_ASSERT(c1.bucket_count() == 0);
-        assert(c1.size() == 0);
-        assert(c1.hash_function() == Hash(2));
-        assert(c1.key_eq() == Compare(2));
-        assert(c1.get_allocator().get_id() == 1);
-        assert(static_cast<std::size_t>(std::distance(c1.begin(), c1.end())) == c1.size());
-        assert(static_cast<std::size_t>(std::distance(c1.cbegin(), c1.cend())) == c1.size());
-        assert(c1.max_load_factor() == 2);
-
-        assert(c2.bucket_count() >= 4);
-        assert(c2.size() == 4);
-        assert(c2.at(1) == "one");
-        assert(c2.at(2) == "two");
-        assert(c2.at(3) == "three");
-        assert(c2.at(4) == "four");
-        assert(c2.hash_function() == Hash(1));
-        assert(c2.key_eq() == Compare(1));
-        assert(c2.get_allocator().get_id() == 2);
-        assert(static_cast<std::size_t>(std::distance(c2.begin(), c2.end())) == c2.size());
-        assert(static_cast<std::size_t>(std::distance(c2.cbegin(), c2.cend())) == c2.size());
-        assert(c2.max_load_factor() == 1);
-        assert(it1 == c2.begin()); // Iterators are not invalidated
-    }
-    {
-        typedef test_hash<int> Hash;
-        typedef test_equal_to<int> Compare;
-        typedef test_allocator<std::pair<const int, std::string> > Alloc;
-        typedef std::unordered_map<int, std::string, Hash, Compare, Alloc> C;
-        typedef std::pair<int, std::string> P;
-        P a1[] =
-        {
-            P(1, "one"),
-            P(2, "two"),
-            P(3, "three"),
-            P(4, "four"),
-            P(1, "four"),
-            P(2, "four"),
-        };
-        P a2[] =
-        {
-            P(10, "ten"),
-            P(20, "twenty"),
-            P(30, "thirty"),
-            P(40, "forty"),
-            P(50, "fifty"),
-            P(60, "sixty"),
-            P(70, "seventy"),
-            P(80, "eighty"),
-        };
-        C c1(std::begin(a1), std::end(a1), 0, Hash(1), Compare(1), Alloc(1, 1));
-        C c2(std::begin(a2), std::end(a2), 0, Hash(2), Compare(2), Alloc(1, 2));
-        c2.max_load_factor(2);
-        C::iterator it1 = c1.begin();
-        C::iterator it2 = c2.begin();
-        swap(c1, c2);
-
-        assert(c1.bucket_count() >= 8);
-        assert(c1.size() == 8);
-        assert(c1.at(10) == "ten");
-        assert(c1.at(20) == "twenty");
-        assert(c1.at(30) == "thirty");
-        assert(c1.at(40) == "forty");
-        assert(c1.at(50) == "fifty");
-        assert(c1.at(60) == "sixty");
-        assert(c1.at(70) == "seventy");
-        assert(c1.at(80) == "eighty");
-        assert(c1.hash_function() == Hash(2));
-        assert(c1.key_eq() == Compare(2));
-        assert(c1.get_allocator().get_id() == 1);
-        assert(static_cast<std::size_t>(std::distance(c1.begin(), c1.end())) == c1.size());
-        assert(static_cast<std::size_t>(std::distance(c1.cbegin(), c1.cend())) == c1.size());
-        assert(c1.max_load_factor() == 2);
-        assert(it2 == c1.begin()); // Iterators are not invalidated
-
-        assert(c2.bucket_count() >= 4);
-        assert(c2.size() == 4);
-        assert(c2.at(1) == "one");
-        assert(c2.at(2) == "two");
-        assert(c2.at(3) == "three");
-        assert(c2.at(4) == "four");
-        assert(c2.hash_function() == Hash(1));
-        assert(c2.key_eq() == Compare(1));
-        assert(c2.get_allocator().get_id() == 2);
-        assert(static_cast<std::size_t>(std::distance(c2.begin(), c2.end())) == c2.size());
-        assert(static_cast<std::size_t>(std::distance(c2.cbegin(), c2.cend())) == c2.size());
-        assert(c2.max_load_factor() == 1);
-        assert(it1 == c2.begin()); // Iterators are not invalidated
-    }
-
-    {
-        typedef test_hash<int> Hash;
-        typedef test_equal_to<int> Compare;
-        typedef other_allocator<std::pair<const int, std::string> > Alloc;
-        typedef std::unordered_map<int, std::string, Hash, Compare, Alloc> C;
-        C c1(0, Hash(1), Compare(1), Alloc(1));
-        C c2(0, Hash(2), Compare(2), Alloc(2));
-        c2.max_load_factor(2);
-        swap(c1, c2);
-
-        LIBCPP_ASSERT(c1.bucket_count() == 0);
-        assert(c1.size() == 0);
-        assert(c1.hash_function() == Hash(2));
-        assert(c1.key_eq() == Compare(2));
-        assert(c1.get_allocator() == Alloc(2));
-        assert(static_cast<std::size_t>(std::distance(c1.begin(), c1.end())) == c1.size());
-        assert(static_cast<std::size_t>(std::distance(c1.cbegin(), c1.cend())) == c1.size());
-        assert(c1.max_load_factor() == 2);
-
-        LIBCPP_ASSERT(c2.bucket_count() == 0);
-        assert(c2.size() == 0);
-        assert(c2.hash_function() == Hash(1));
-        assert(c2.key_eq() == Compare(1));
-        assert(c2.get_allocator() == Alloc(1));
-        assert(static_cast<std::size_t>(std::distance(c2.begin(), c2.end())) == c2.size());
-        assert(static_cast<std::size_t>(std::distance(c2.cbegin(), c2.cend())) == c2.size());
-        assert(c2.max_load_factor() == 1);
-    }
-    {
-        typedef test_hash<int> Hash;
-        typedef test_equal_to<int> Compare;
-        typedef other_allocator<std::pair<const int, std::string> > Alloc;
-        typedef std::unordered_map<int, std::string, Hash, Compare, Alloc> C;
-        typedef std::pair<int, std::string> P;
-        P a2[] =
-        {
-            P(10, "ten"),
-            P(20, "twenty"),
-            P(30, "thirty"),
-            P(40, "forty"),
-            P(50, "fifty"),
-            P(60, "sixty"),
-            P(70, "seventy"),
-            P(80, "eighty"),
-        };
-        C c1(0, Hash(1), Compare(1), Alloc(1));
-        C c2(std::begin(a2), std::end(a2), 0, Hash(2), Compare(2), Alloc(2));
-        c2.max_load_factor(2);
-        swap(c1, c2);
-
-        assert(c1.bucket_count() >= 8);
-        assert(c1.size() == 8);
-        assert(c1.at(10) == "ten");
-        assert(c1.at(20) == "twenty");
-        assert(c1.at(30) == "thirty");
-        assert(c1.at(40) == "forty");
-        assert(c1.at(50) == "fifty");
-        assert(c1.at(60) == "sixty");
-        assert(c1.at(70) == "seventy");
-        assert(c1.at(80) == "eighty");
-        assert(c1.hash_function() == Hash(2));
-        assert(c1.key_eq() == Compare(2));
-        assert(c1.get_allocator() == Alloc(2));
-        assert(static_cast<std::size_t>(std::distance(c1.begin(), c1.end())) == c1.size());
-        assert(static_cast<std::size_t>(std::distance(c1.cbegin(), c1.cend())) == c1.size());
-        assert(c1.max_load_factor() == 2);
-
-        LIBCPP_ASSERT(c2.bucket_count() == 0);
-        assert(c2.size() == 0);
-        assert(c2.hash_function() == Hash(1));
-        assert(c2.key_eq() == Compare(1));
-        assert(c2.get_allocator() == Alloc(1));
-        assert(static_cast<std::size_t>(std::distance(c2.begin(), c2.end())) == c2.size());
-        assert(static_cast<std::size_t>(std::distance(c2.cbegin(), c2.cend())) == c2.size());
-        assert(c2.max_load_factor() == 1);
-    }
-    {
-        typedef test_hash<int> Hash;
-        typedef test_equal_to<int> Compare;
-        typedef other_allocator<std::pair<const int, std::string> > Alloc;
-        typedef std::unordered_map<int, std::string, Hash, Compare, Alloc> C;
-        typedef std::pair<int, std::string> P;
-        P a1[] =
-        {
-            P(1, "one"),
-            P(2, "two"),
-            P(3, "three"),
-            P(4, "four"),
-            P(1, "four"),
-            P(2, "four"),
-        };
-        C c1(std::begin(a1), std::end(a1), 0, Hash(1), Compare(1), Alloc(1));
-        C c2(0, Hash(2), Compare(2), Alloc(2));
-        c2.max_load_factor(2);
-        swap(c1, c2);
-
-        LIBCPP_ASSERT(c1.bucket_count() == 0);
-        assert(c1.size() == 0);
-        assert(c1.hash_function() == Hash(2));
-        assert(c1.key_eq() == Compare(2));
-        assert(c1.get_allocator() == Alloc(2));
-        assert(static_cast<std::size_t>(std::distance(c1.begin(), c1.end())) == c1.size());
-        assert(static_cast<std::size_t>(std::distance(c1.cbegin(), c1.cend())) == c1.size());
-        assert(c1.max_load_factor() == 2);
-
-        assert(c2.bucket_count() >= 4);
-        assert(c2.size() == 4);
-        assert(c2.at(1) == "one");
-        assert(c2.at(2) == "two");
-        assert(c2.at(3) == "three");
-        assert(c2.at(4) == "four");
-        assert(c2.hash_function() == Hash(1));
-        assert(c2.key_eq() == Compare(1));
-        assert(c2.get_allocator() == Alloc(1));
-        assert(static_cast<std::size_t>(std::distance(c2.begin(), c2.end())) == c2.size());
-        assert(static_cast<std::size_t>(std::distance(c2.cbegin(), c2.cend())) == c2.size());
-        assert(c2.max_load_factor() == 1);
-    }
-    {
-        typedef test_hash<int> Hash;
-        typedef test_equal_to<int> Compare;
-        typedef other_allocator<std::pair<const int, std::string> > Alloc;
-        typedef std::unordered_map<int, std::string, Hash, Compare, Alloc> C;
-        typedef std::pair<int, std::string> P;
-        P a1[] =
-        {
-            P(1, "one"),
-            P(2, "two"),
-            P(3, "three"),
-            P(4, "four"),
-            P(1, "four"),
-            P(2, "four"),
-        };
-        P a2[] =
-        {
-            P(10, "ten"),
-            P(20, "twenty"),
-            P(30, "thirty"),
-            P(40, "forty"),
-            P(50, "fifty"),
-            P(60, "sixty"),
-            P(70, "seventy"),
-            P(80, "eighty"),
-        };
-        C c1(std::begin(a1), std::end(a1), 0, Hash(1), Compare(1), Alloc(1));
-        C c2(std::begin(a2), std::end(a2), 0, Hash(2), Compare(2), Alloc(2));
-        c2.max_load_factor(2);
-        swap(c1, c2);
-
-        assert(c1.bucket_count() >= 8);
-        assert(c1.size() == 8);
-        assert(c1.at(10) == "ten");
-        assert(c1.at(20) == "twenty");
-        assert(c1.at(30) == "thirty");
-        assert(c1.at(40) == "forty");
-        assert(c1.at(50) == "fifty");
-        assert(c1.at(60) == "sixty");
-        assert(c1.at(70) == "seventy");
-        assert(c1.at(80) == "eighty");
-        assert(c1.hash_function() == Hash(2));
-        assert(c1.key_eq() == Compare(2));
-        assert(c1.get_allocator() == Alloc(2));
-        assert(static_cast<std::size_t>(std::distance(c1.begin(), c1.end())) == c1.size());
-        assert(static_cast<std::size_t>(std::distance(c1.cbegin(), c1.cend())) == c1.size());
-        assert(c1.max_load_factor() == 2);
-
-        assert(c2.bucket_count() >= 4);
-        assert(c2.size() == 4);
-        assert(c2.at(1) == "one");
-        assert(c2.at(2) == "two");
-        assert(c2.at(3) == "three");
-        assert(c2.at(4) == "four");
-        assert(c2.hash_function() == Hash(1));
-        assert(c2.key_eq() == Compare(1));
-        assert(c2.get_allocator() == Alloc(1));
-        assert(static_cast<std::size_t>(std::distance(c2.begin(), c2.end())) == c2.size());
-        assert(static_cast<std::size_t>(std::distance(c2.cbegin(), c2.cend())) == c2.size());
-        assert(c2.max_load_factor() == 1);
-    }
+int main(int, char**) {
+  {
+    typedef test_hash<int> Hash;
+    typedef test_equal_to<int> Compare;
+    typedef test_allocator<std::pair<const int, std::string> > Alloc;
+    typedef std::unordered_map<int, std::string, Hash, Compare, Alloc> C;
+    C c1(0, Hash(1), Compare(1), Alloc(1, 1));
+    C c2(0, Hash(2), Compare(2), Alloc(1, 2));
+    c2.max_load_factor(2);
+    swap(c1, c2);
+
+    LIBCPP_ASSERT(c1.bucket_count() == 0);
+    assert(c1.size() == 0);
+    assert(c1.hash_function() == Hash(2));
+    assert(c1.key_eq() == Compare(2));
+    assert(c1.get_allocator().get_id() == 1);
+    assert(static_cast<std::size_t>(std::distance(c1.begin(), c1.end())) == c1.size());
+    assert(static_cast<std::size_t>(std::distance(c1.cbegin(), c1.cend())) == c1.size());
+    assert(c1.max_load_factor() == 2);
+
+    LIBCPP_ASSERT(c2.bucket_count() == 0);
+    assert(c2.size() == 0);
+    assert(c2.hash_function() == Hash(1));
+    assert(c2.key_eq() == Compare(1));
+    assert(c2.get_allocator().get_id() == 2);
+    assert(static_cast<std::size_t>(std::distance(c2.begin(), c2.end())) == c2.size());
+    assert(static_cast<std::size_t>(std::distance(c2.cbegin(), c2.cend())) == c2.size());
+    assert(c2.max_load_factor() == 1);
+  }
+  {
+    typedef test_hash<int> Hash;
+    typedef test_equal_to<int> Compare;
+    typedef test_allocator<std::pair<const int, std::string> > Alloc;
+    typedef std::unordered_map<int, std::string, Hash, Compare, Alloc> C;
+    typedef std::pair<int, std::string> P;
+    P a2[] = {
+        P(10, "ten"),
+        P(20, "twenty"),
+        P(30, "thirty"),
+        P(40, "forty"),
+        P(50, "fifty"),
+        P(60, "sixty"),
+        P(70, "seventy"),
+        P(80, "eighty"),
+    };
+    C c1(0, Hash(1), Compare(1), Alloc(1, 1));
+    C c2(std::begin(a2), std::end(a2), 0, Hash(2), Compare(2), Alloc(1, 2));
+    c2.max_load_factor(2);
+    C::iterator it2 = c2.begin();
+    swap(c1, c2);
+
+    assert(c1.bucket_count() >= 8);
+    assert(c1.size() == 8);
+    assert(c1.at(10) == "ten");
+    assert(c1.at(20) == "twenty");
+    assert(c1.at(30) == "thirty");
+    assert(c1.at(40) == "forty");
+    assert(c1.at(50) == "fifty");
+    assert(c1.at(60) == "sixty");
+    assert(c1.at(70) == "seventy");
+    assert(c1.at(80) == "eighty");
+    assert(c1.hash_function() == Hash(2));
+    assert(c1.key_eq() == Compare(2));
+    assert(c1.get_allocator().get_id() == 1);
+    assert(static_cast<std::size_t>(std::distance(c1.begin(), c1.end())) == c1.size());
+    assert(static_cast<std::size_t>(std::distance(c1.cbegin(), c1.cend())) == c1.size());
+    assert(c1.max_load_factor() == 2);
+    assert(it2 == c1.begin()); // Iterators are not invalidated
+
+    LIBCPP_ASSERT(c2.bucket_count() == 0);
+    assert(c2.size() == 0);
+    assert(c2.hash_function() == Hash(1));
+    assert(c2.key_eq() == Compare(1));
+    assert(c2.get_allocator().get_id() == 2);
+    assert(static_cast<std::size_t>(std::distance(c2.begin(), c2.end())) == c2.size());
+    assert(static_cast<std::size_t>(std::distance(c2.cbegin(), c2.cend())) == c2.size());
+    assert(c2.max_load_factor() == 1);
+  }
+  {
+    typedef test_hash<int> Hash;
+    typedef test_equal_to<int> Compare;
+    typedef test_allocator<std::pair<const int, std::string> > Alloc;
+    typedef std::unordered_map<int, std::string, Hash, Compare, Alloc> C;
+    typedef std::pair<int, std::string> P;
+    P a1[] = {
+        P(1, "one"),
+        P(2, "two"),
+        P(3, "three"),
+        P(4, "four"),
+        P(1, "four"),
+        P(2, "four"),
+    };
+    C c1(std::begin(a1), std::end(a1), 0, Hash(1), Compare(1), Alloc(1, 1));
+    C c2(0, Hash(2), Compare(2), Alloc(1, 2));
+    c2.max_load_factor(2);
+    C::iterator it1 = c1.begin();
+    swap(c1, c2);
+
+    LIBCPP_ASSERT(c1.bucket_count() == 0);
+    assert(c1.size() == 0);
+    assert(c1.hash_function() == Hash(2));
+    assert(c1.key_eq() == Compare(2));
+    assert(c1.get_allocator().get_id() == 1);
+    assert(static_cast<std::size_t>(std::distance(c1.begin(), c1.end())) == c1.size());
+    assert(static_cast<std::size_t>(std::distance(c1.cbegin(), c1.cend())) == c1.size());
+    assert(c1.max_load_factor() == 2);
+
+    assert(c2.bucket_count() >= 4);
+    assert(c2.size() == 4);
+    assert(c2.at(1) == "one");
+    assert(c2.at(2) == "two");
+    assert(c2.at(3) == "three");
+    assert(c2.at(4) == "four");
+    assert(c2.hash_function() == Hash(1));
+    assert(c2.key_eq() == Compare(1));
+    assert(c2.get_allocator().get_id() == 2);
+    assert(static_cast<std::size_t>(std::distance(c2.begin(), c2.end())) == c2.size());
+    assert(static_cast<std::size_t>(std::distance(c2.cbegin(), c2.cend())) == c2.size());
+    assert(c2.max_load_factor() == 1);
+    assert(it1 == c2.begin()); // Iterators are not invalidated
+  }
+  {
+    typedef test_hash<int> Hash;
+    typedef test_equal_to<int> Compare;
+    typedef test_allocator<std::pair<const int, std::string> > Alloc;
+    typedef std::unordered_map<int, std::string, Hash, Compare, Alloc> C;
+    typedef std::pair<int, std::string> P;
+    P a1[] = {
+        P(1, "one"),
+        P(2, "two"),
+        P(3, "three"),
+        P(4, "four"),
+        P(1, "four"),
+        P(2, "four"),
+    };
+    P a2[] = {
+        P(10, "ten"),
+        P(20, "twenty"),
+        P(30, "thirty"),
+        P(40, "forty"),
+        P(50, "fifty"),
+        P(60, "sixty"),
+        P(70, "seventy"),
+        P(80, "eighty"),
+    };
+    C c1(std::begin(a1), std::end(a1), 0, Hash(1), Compare(1), Alloc(1, 1));
+    C c2(std::begin(a2), std::end(a2), 0, Hash(2), Compare(2), Alloc(1, 2));
+    c2.max_load_factor(2);
+    C::iterator it1 = c1.begin();
+    C::iterator it2 = c2.begin();
+    swap(c1, c2);
+
+    assert(c1.bucket_count() >= 8);
+    assert(c1.size() == 8);
+    assert(c1.at(10) == "ten");
+    assert(c1.at(20) == "twenty");
+    assert(c1.at(30) == "thirty");
+    assert(c1.at(40) == "forty");
+    assert(c1.at(50) == "fifty");
+    assert(c1.at(60) == "sixty");
+    assert(c1.at(70) == "seventy");
+    assert(c1.at(80) == "eighty");
+    assert(c1.hash_function() == Hash(2));
+    assert(c1.key_eq() == Compare(2));
+    assert(c1.get_allocator().get_id() == 1);
+    assert(static_cast<std::size_t>(std::distance(c1.begin(), c1.end())) == c1.size());
+    assert(static_cast<std::size_t>(std::distance(c1.cbegin(), c1.cend())) == c1.size());
+    assert(c1.max_load_factor() == 2);
+    assert(it2 == c1.begin()); // Iterators are not invalidated
+
+    assert(c2.bucket_count() >= 4);
+    assert(c2.size() == 4);
+    assert(c2.at(1) == "one");
+    assert(c2.at(2) == "two");
+    assert(c2.at(3) == "three");
+    assert(c2.at(4) == "four");
+    assert(c2.hash_function() == Hash(1));
+    assert(c2.key_eq() == Compare(1));
+    assert(c2.get_allocator().get_id() == 2);
+    assert(static_cast<std::size_t>(std::distance(c2.begin(), c2.end())) == c2.size());
+    assert(static_cast<std::size_t>(std::distance(c2.cbegin(), c2.cend())) == c2.size());
+    assert(c2.max_load_factor() == 1);
+    assert(it1 == c2.begin()); // Iterators are not invalidated
+  }
+
+  {
+    typedef test_hash<int> Hash;
+    typedef test_equal_to<int> Compare;
+    typedef other_allocator<std::pair<const int, std::string> > Alloc;
+    typedef std::unordered_map<int, std::string, Hash, Compare, Alloc> C;
+    C c1(0, Hash(1), Compare(1), Alloc(1));
+    C c2(0, Hash(2), Compare(2), Alloc(2));
+    c2.max_load_factor(2);
+    swap(c1, c2);
+
+    LIBCPP_ASSERT(c1.bucket_count() == 0);
+    assert(c1.size() == 0);
+    assert(c1.hash_function() == Hash(2));
+    assert(c1.key_eq() == Compare(2));
+    assert(c1.get_allocator() == Alloc(2));
+    assert(static_cast<std::size_t>(std::distance(c1.begin(), c1.end())) == c1.size());
+    assert(static_cast<std::size_t>(std::distance(c1.cbegin(), c1.cend())) == c1.size());
+    assert(c1.max_load_factor() == 2);
+
+    LIBCPP_ASSERT(c2.bucket_count() == 0);
+    assert(c2.size() == 0);
+    assert(c2.hash_function() == Hash(1));
+    assert(c2.key_eq() == Compare(1));
+    assert(c2.get_allocator() == Alloc(1));
+    assert(static_cast<std::size_t>(std::distance(c2.begin(), c2.end())) == c2.size());
+    assert(static_cast<std::size_t>(std::distance(c2.cbegin(), c2.cend())) == c2.size());
+    assert(c2.max_load_factor() == 1);
+  }
+  {
+    typedef test_hash<int> Hash;
+    typedef test_equal_to<int> Compare;
+    typedef other_allocator<std::pair<const int, std::string> > Alloc;
+    typedef std::unordered_map<int, std::string, Hash, Compare, Alloc> C;
+    typedef std::pair<int, std::string> P;
+    P a2[] = {
+        P(10, "ten"),
+        P(20, "twenty"),
+        P(30, "thirty"),
+        P(40, "forty"),
+        P(50, "fifty"),
+        P(60, "sixty"),
+        P(70, "seventy"),
+        P(80, "eighty"),
+    };
+    C c1(0, Hash(1), Compare(1), Alloc(1));
+    C c2(std::begin(a2), std::end(a2), 0, Hash(2), Compare(2), Alloc(2));
+    c2.max_load_factor(2);
+    swap(c1, c2);
+
+    assert(c1.bucket_count() >= 8);
+    assert(c1.size() == 8);
+    assert(c1.at(10) == "ten");
+    assert(c1.at(20) == "twenty");
+    assert(c1.at(30) == "thirty");
+    assert(c1.at(40) == "forty");
+    assert(c1.at(50) == "fifty");
+    assert(c1.at(60) == "sixty");
+    assert(c1.at(70) == "seventy");
+    assert(c1.at(80) == "eighty");
+    assert(c1.hash_function() == Hash(2));
+    assert(c1.key_eq() == Compare(2));
+    assert(c1.get_allocator() == Alloc(2));
+    assert(static_cast<std::size_t>(std::distance(c1.begin(), c1.end())) == c1.size());
+    assert(static_cast<std::size_t>(std::distance(c1.cbegin(), c1.cend())) == c1.size());
+    assert(c1.max_load_factor() == 2);
+
+    LIBCPP_ASSERT(c2.bucket_count() == 0);
+    assert(c2.size() == 0);
+    assert(c2.hash_function() == Hash(1));
+    assert(c2.key_eq() == Compare(1));
+    assert(c2.get_allocator() == Alloc(1));
+    assert(static_cast<std::size_t>(std::distance(c2.begin(), c2.end())) == c2.size());
+    assert(static_cast<std::size_t>(std::distance(c2.cbegin(), c2.cend())) == c2.size());
+    assert(c2.max_load_factor() == 1);
+  }
+  {
+    typedef test_hash<int> Hash;
+    typedef test_equal_to<int> Compare;
+    typedef other_allocator<std::pair<const int, std::string> > Alloc;
+    typedef std::unordered_map<int, std::string, Hash, Compare, Alloc> C;
+    typedef std::pair<int, std::string> P;
+    P a1[] = {
+        P(1, "one"),
+        P(2, "two"),
+        P(3, "three"),
+        P(4, "four"),
+        P(1, "four"),
+        P(2, "four"),
+    };
+    C c1(std::begin(a1), std::end(a1), 0, Hash(1), Compare(1), Alloc(1));
+    C c2(0, Hash(2), Compare(2), Alloc(2));
+    c2.max_load_factor(2);
+    swap(c1, c2);
+
+    LIBCPP_ASSERT(c1.bucket_count() == 0);
+    assert(c1.size() == 0);
+    assert(c1.hash_function() == Hash(2));
+    assert(c1.key_eq() == Compare(2));
+    assert(c1.get_allocator() == Alloc(2));
+    assert(static_cast<std::size_t>(std::distance(c1.begin(), c1.end())) == c1.size());
+    assert(static_cast<std::size_t>(std::distance(c1.cbegin(), c1.cend())) == c1.size());
+    assert(c1.max_load_factor() == 2);
+
+    assert(c2.bucket_count() >= 4);
+    assert(c2.size() == 4);
+    assert(c2.at(1) == "one");
+    assert(c2.at(2) == "two");
+    assert(c2.at(3) == "three");
+    assert(c2.at(4) == "four");
+    assert(c2.hash_function() == Hash(1));
+    assert(c2.key_eq() == Compare(1));
+    assert(c2.get_allocator() == Alloc(1));
+    assert(static_cast<std::size_t>(std::distance(c2.begin(), c2.end())) == c2.size());
+    assert(static_cast<std::size_t>(std::distance(c2.cbegin(), c2.cend())) == c2.size());
+    assert(c2.max_load_factor() == 1);
+  }
+  {
+    typedef test_hash<int> Hash;
+    typedef test_equal_to<int> Compare;
+    typedef other_allocator<std::pair<const int, std::string> > Alloc;
+    typedef std::unordered_map<int, std::string, Hash, Compare, Alloc> C;
+    typedef std::pair<int, std::string> P;
+    P a1[] = {
+        P(1, "one"),
+        P(2, "two"),
+        P(3, "three"),
+        P(4, "four"),
+        P(1, "four"),
+        P(2, "four"),
+    };
+    P a2[] = {
+        P(10, "ten"),
+        P(20, "twenty"),
+        P(30, "thirty"),
+        P(40, "forty"),
+        P(50, "fifty"),
+        P(60, "sixty"),
+        P(70, "seventy"),
+        P(80, "eighty"),
+    };
+    C c1(std::begin(a1), std::end(a1), 0, Hash(1), Compare(1), Alloc(1));
+    C c2(std::begin(a2), std::end(a2), 0, Hash(2), Compare(2), Alloc(2));
+    c2.max_load_factor(2);
+    swap(c1, c2);
+
+    assert(c1.bucket_count() >= 8);
+    assert(c1.size() == 8);
+    assert(c1.at(10) == "ten");
+    assert(c1.at(20) == "twenty");
+    assert(c1.at(30) == "thirty");
+    assert(c1.at(40) == "forty");
+    assert(c1.at(50) == "fifty");
+    assert(c1.at(60) == "sixty");
+    assert(c1.at(70) == "seventy");
+    assert(c1.at(80) == "eighty");
+    assert(c1.hash_function() == Hash(2));
+    assert(c1.key_eq() == Compare(2));
+    assert(c1.get_allocator() == Alloc(2));
+    assert(static_cast<std::size_t>(std::distance(c1.begin(), c1.end())) == c1.size());
+    assert(static_cast<std::size_t>(std::distance(c1.cbegin(), c1.cend())) == c1.size());
+    assert(c1.max_load_factor() == 2);
+
+    assert(c2.bucket_count() >= 4);
+    assert(c2.size() == 4);
+    assert(c2.at(1) == "one");
+    assert(c2.at(2) == "two");
+    assert(c2.at(3) == "three");
+    assert(c2.at(4) == "four");
+    assert(c2.hash_function() == Hash(1));
+    assert(c2.key_eq() == Compare(1));
+    assert(c2.get_allocator() == Alloc(1));
+    assert(static_cast<std::size_t>(std::distance(c2.begin(), c2.end())) == c2.size());
+    assert(static_cast<std::size_t>(std::distance(c2.cbegin(), c2.cend())) == c2.size());
+    assert(c2.max_load_factor() == 1);
+  }
 #if TEST_STD_VER >= 11
-    {
-        typedef test_hash<int> Hash;
-        typedef test_equal_to<int> Compare;
-        typedef min_allocator<std::pair<const int, std::string> > Alloc;
-        typedef std::unordered_map<int, std::string, Hash, Compare, Alloc> C;
-        C c1(0, Hash(1), Compare(1), Alloc());
-        C c2(0, Hash(2), Compare(2), Alloc());
-        c2.max_load_factor(2);
-        swap(c1, c2);
-
-        LIBCPP_ASSERT(c1.bucket_count() == 0);
-        assert(c1.size() == 0);
-        assert(c1.hash_function() == Hash(2));
-        assert(c1.key_eq() == Compare(2));
-        assert(c1.get_allocator() == Alloc());
-        assert(static_cast<std::size_t>(std::distance(c1.begin(), c1.end())) == c1.size());
-        assert(static_cast<std::size_t>(std::distance(c1.cbegin(), c1.cend())) == c1.size());
-        assert(c1.max_load_factor() == 2);
-
-        LIBCPP_ASSERT(c2.bucket_count() == 0);
-        assert(c2.size() == 0);
-        assert(c2.hash_function() == Hash(1));
-        assert(c2.key_eq() == Compare(1));
-        assert(c2.get_allocator() == Alloc());
-        assert(static_cast<std::size_t>(std::distance(c2.begin(), c2.end())) == c2.size());
-        assert(static_cast<std::size_t>(std::distance(c2.cbegin(), c2.cend())) == c2.size());
-        assert(c2.max_load_factor() == 1);
-    }
-    {
-        typedef test_hash<int> Hash;
-        typedef test_equal_to<int> Compare;
-        typedef min_allocator<std::pair<const int, std::string> > Alloc;
-        typedef std::unordered_map<int, std::string, Hash, Compare, Alloc> C;
-        typedef std::pair<int, std::string> P;
-        P a2[] =
-        {
-            P(10, "ten"),
-            P(20, "twenty"),
-            P(30, "thirty"),
-            P(40, "forty"),
-            P(50, "fifty"),
-            P(60, "sixty"),
-            P(70, "seventy"),
-            P(80, "eighty"),
-        };
-        C c1(0, Hash(1), Compare(1), Alloc());
-        C c2(std::begin(a2), std::end(a2), 0, Hash(2), Compare(2), Alloc());
-        c2.max_load_factor(2);
-        swap(c1, c2);
-
-        assert(c1.bucket_count() >= 8);
-        assert(c1.size() == 8);
-        assert(c1.at(10) == "ten");
-        assert(c1.at(20) == "twenty");
-        assert(c1.at(30) == "thirty");
-        assert(c1.at(40) == "forty");
-        assert(c1.at(50) == "fifty");
-        assert(c1.at(60) == "sixty");
-        assert(c1.at(70) == "seventy");
-        assert(c1.at(80) == "eighty");
-        assert(c1.hash_function() == Hash(2));
-        assert(c1.key_eq() == Compare(2));
-        assert(c1.get_allocator() == Alloc());
-        assert(static_cast<std::size_t>(std::distance(c1.begin(), c1.end())) == c1.size());
-        assert(static_cast<std::size_t>(std::distance(c1.cbegin(), c1.cend())) == c1.size());
-        assert(c1.max_load_factor() == 2);
-
-        LIBCPP_ASSERT(c2.bucket_count() == 0);
-        assert(c2.size() == 0);
-        assert(c2.hash_function() == Hash(1));
-        assert(c2.key_eq() == Compare(1));
-        assert(c2.get_allocator() == Alloc());
-        assert(static_cast<std::size_t>(std::distance(c2.begin(), c2.end())) == c2.size());
-        assert(static_cast<std::size_t>(std::distance(c2.cbegin(), c2.cend())) == c2.size());
-        assert(c2.max_load_factor() == 1);
-    }
-    {
-        typedef test_hash<int> Hash;
-        typedef test_equal_to<int> Compare;
-        typedef min_allocator<std::pair<const int, std::string> > Alloc;
-        typedef std::unordered_map<int, std::string, Hash, Compare, Alloc> C;
-        typedef std::pair<int, std::string> P;
-        P a1[] =
-        {
-            P(1, "one"),
-            P(2, "two"),
-            P(3, "three"),
-            P(4, "four"),
-            P(1, "four"),
-            P(2, "four"),
-        };
-        C c1(std::begin(a1), std::end(a1), 0, Hash(1), Compare(1), Alloc());
-        C c2(0, Hash(2), Compare(2), Alloc());
-        c2.max_load_factor(2);
-        swap(c1, c2);
-
-        LIBCPP_ASSERT(c1.bucket_count() == 0);
-        assert(c1.size() == 0);
-        assert(c1.hash_function() == Hash(2));
-        assert(c1.key_eq() == Compare(2));
-        assert(c1.get_allocator() == Alloc());
-        assert(static_cast<std::size_t>(std::distance(c1.begin(), c1.end())) == c1.size());
-        assert(static_cast<std::size_t>(std::distance(c1.cbegin(), c1.cend())) == c1.size());
-        assert(c1.max_load_factor() == 2);
-
-        assert(c2.bucket_count() >= 4);
-        assert(c2.size() == 4);
-        assert(c2.at(1) == "one");
-        assert(c2.at(2) == "two");
-        assert(c2.at(3) == "three");
-        assert(c2.at(4) == "four");
-        assert(c2.hash_function() == Hash(1));
-        assert(c2.key_eq() == Compare(1));
-        assert(c2.get_allocator() == Alloc());
-        assert(static_cast<std::size_t>(std::distance(c2.begin(), c2.end())) == c2.size());
-        assert(static_cast<std::size_t>(std::distance(c2.cbegin(), c2.cend())) == c2.size());
-        assert(c2.max_load_factor() == 1);
-    }
-    {
-        typedef test_hash<int> Hash;
-        typedef test_equal_to<int> Compare;
-        typedef min_allocator<std::pair<const int, std::string> > Alloc;
-        typedef std::unordered_map<int, std::string, Hash, Compare, Alloc> C;
-        typedef std::pair<int, std::string> P;
-        P a1[] =
-        {
-            P(1, "one"),
-            P(2, "two"),
-            P(3, "three"),
-            P(4, "four"),
-            P(1, "four"),
-            P(2, "four"),
-        };
-        P a2[] =
-        {
-            P(10, "ten"),
-            P(20, "twenty"),
-            P(30, "thirty"),
-            P(40, "forty"),
-            P(50, "fifty"),
-            P(60, "sixty"),
-            P(70, "seventy"),
-            P(80, "eighty"),
-        };
-        C c1(std::begin(a1), std::end(a1), 0, Hash(1), Compare(1), Alloc());
-        C c2(std::begin(a2), std::end(a2), 0, Hash(2), Compare(2), Alloc());
-        c2.max_load_factor(2);
-        swap(c1, c2);
-
-        assert(c1.bucket_count() >= 8);
-        assert(c1.size() == 8);
-        assert(c1.at(10) == "ten");
-        assert(c1.at(20) == "twenty");
-        assert(c1.at(30) == "thirty");
-        assert(c1.at(40) == "forty");
-        assert(c1.at(50) == "fifty");
-        assert(c1.at(60) == "sixty");
-        assert(c1.at(70) == "seventy");
-        assert(c1.at(80) == "eighty");
-        assert(c1.hash_function() == Hash(2));
-        assert(c1.key_eq() == Compare(2));
-        assert(c1.get_allocator() == Alloc());
-        assert(static_cast<std::size_t>(std::distance(c1.begin(), c1.end())) == c1.size());
-        assert(static_cast<std::size_t>(std::distance(c1.cbegin(), c1.cend())) == c1.size());
-        assert(c1.max_load_factor() == 2);
-
-        assert(c2.bucket_count() >= 4);
-        assert(c2.size() == 4);
-        assert(c2.at(1) == "one");
-        assert(c2.at(2) == "two");
-        assert(c2.at(3) == "three");
-        assert(c2.at(4) == "four");
-        assert(c2.hash_function() == Hash(1));
-        assert(c2.key_eq() == Compare(1));
-        assert(c2.get_allocator() == Alloc());
-        assert(static_cast<std::size_t>(std::distance(c2.begin(), c2.end())) == c2.size());
-        assert(static_cast<std::size_t>(std::distance(c2.cbegin(), c2.cend())) == c2.size());
-        assert(c2.max_load_factor() == 1);
-    }
+  {
+    typedef test_hash<int> Hash;
+    typedef test_equal_to<int> Compare;
+    typedef min_allocator<std::pair<const int, std::string> > Alloc;
+    typedef std::unordered_map<int, std::string, Hash, Compare, Alloc> C;
+    C c1(0, Hash(1), Compare(1), Alloc());
+    C c2(0, Hash(2), Compare(2), Alloc());
+    c2.max_load_factor(2);
+    swap(c1, c2);
+
+    LIBCPP_ASSERT(c1.bucket_count() == 0);
+    assert(c1.size() == 0);
+    assert(c1.hash_function() == Hash(2));
+    assert(c1.key_eq() == Compare(2));
+    assert(c1.get_allocator() == Alloc());
+    assert(static_cast<std::size_t>(std::distance(c1.begin(), c1.end())) == c1.size());
+    assert(static_cast<std::size_t>(std::distance(c1.cbegin(), c1.cend())) == c1.size());
+    assert(c1.max_load_factor() == 2);
+
+    LIBCPP_ASSERT(c2.bucket_count() == 0);
+    assert(c2.size() == 0);
+    assert(c2.hash_function() == Hash(1));
+    assert(c2.key_eq() == Compare(1));
+    assert(c2.get_allocator() == Alloc());
+    assert(static_cast<std::size_t>(std::distance(c2.begin(), c2.end())) == c2.size());
+    assert(static_cast<std::size_t>(std::distance(c2.cbegin(), c2.cend())) == c2.size());
+    assert(c2.max_load_factor() == 1);
+  }
+  {
+    typedef test_hash<int> Hash;
+    typedef test_equal_to<int> Compare;
+    typedef min_allocator<std::pair<const int, std::string> > Alloc;
+    typedef std::unordered_map<int, std::string, Hash, Compare, Alloc> C;
+    typedef std::pair<int, std::string> P;
+    P a2[] = {
+        P(10, "ten"),
+        P(20, "twenty"),
+        P(30, "thirty"),
+        P(40, "forty"),
+        P(50, "fifty"),
+        P(60, "sixty"),
+        P(70, "seventy"),
+        P(80, "eighty"),
+    };
+    C c1(0, Hash(1), Compare(1), Alloc());
+    C c2(std::begin(a2), std::end(a2), 0, Hash(2), Compare(2), Alloc());
+    c2.max_load_factor(2);
+    swap(c1, c2);
+
+    assert(c1.bucket_count() >= 8);
+    assert(c1.size() == 8);
+    assert(c1.at(10) == "ten");
+    assert(c1.at(20) == "twenty");
+    assert(c1.at(30) == "thirty");
+    assert(c1.at(40) == "forty");
+    assert(c1.at(50) == "fifty");
+    assert(c1.at(60) == "sixty");
+    assert(c1.at(70) == "seventy");
+    assert(c1.at(80) == "eighty");
+    assert(c1.hash_function() == Hash(2));
+    assert(c1.key_eq() == Compare(2));
+    assert(c1.get_allocator() == Alloc());
+    assert(static_cast<std::size_t>(std::distance(c1.begin(), c1.end())) == c1.size());
+    assert(static_cast<std::size_t>(std::distance(c1.cbegin(), c1.cend())) == c1.size());
+    assert(c1.max_load_factor() == 2);
+
+    LIBCPP_ASSERT(c2.bucket_count() == 0);
+    assert(c2.size() == 0);
+    assert(c2.hash_function() == Hash(1));
+    assert(c2.key_eq() == Compare(1));
+    assert(c2.get_allocator() == Alloc());
+    assert(static_cast<std::size_t>(std::distance(c2.begin(), c2.end())) == c2.size());
+    assert(static_cast<std::size_t>(std::distance(c2.cbegin(), c2.cend())) == c2.size());
+    assert(c2.max_load_factor() == 1);
+  }
+  {
+    typedef test_hash<int> Hash;
+    typedef test_equal_to<int> Compare;
+    typedef min_allocator<std::pair<const int, std::string> > Alloc;
+    typedef std::unordered_map<int, std::string, Hash, Compare, Alloc> C;
+    typedef std::pair<int, std::string> P;
+    P a1[] = {
+        P(1, "one"),
+        P(2, "two"),
+        P(3, "three"),
+        P(4, "four"),
+        P(1, "four"),
+        P(2, "four"),
+    };
+    C c1(std::begin(a1), std::end(a1), 0, Hash(1), Compare(1), Alloc());
+    C c2(0, Hash(2), Compare(2), Alloc());
+    c2.max_load_factor(2);
+    swap(c1, c2);
+
+    LIBCPP_ASSERT(c1.bucket_count() == 0);
+    assert(c1.size() == 0);
+    assert(c1.hash_function() == Hash(2));
+    assert(c1.key_eq() == Compare(2));
+    assert(c1.get_allocator() == Alloc());
+    assert(static_cast<std::size_t>(std::distance(c1.begin(), c1.end())) == c1.size());
+    assert(static_cast<std::size_t>(std::distance(c1.cbegin(), c1.cend())) == c1.size());
+    assert(c1.max_load_factor() == 2);
+
+    assert(c2.bucket_count() >= 4);
+    assert(c2.size() == 4);
+    assert(c2.at(1) == "one");
+    assert(c2.at(2) == "two");
+    assert(c2.at(3) == "three");
+    assert(c2.at(4) == "four");
+    assert(c2.hash_function() == Hash(1));
+    assert(c2.key_eq() == Compare(1));
+    assert(c2.get_allocator() == Alloc());
+    assert(static_cast<std::size_t>(std::distance(c2.begin(), c2.end())) == c2.size());
+    assert(static_cast<std::size_t>(std::distance(c2.cbegin(), c2.cend())) == c2.size());
+    assert(c2.max_load_factor() == 1);
+  }
+  {
+    typedef test_hash<int> Hash;
+    typedef test_equal_to<int> Compare;
+    typedef min_allocator<std::pair<const int, std::string> > Alloc;
+    typedef std::unordered_map<int, std::string, Hash, Compare, Alloc> C;
+    typedef std::pair<int, std::string> P;
+    P a1[] = {
+        P(1, "one"),
+        P(2, "two"),
+        P(3, "three"),
+        P(4, "four"),
+        P(1, "four"),
+        P(2, "four"),
+    };
+    P a2[] = {
+        P(10, "ten"),
+        P(20, "twenty"),
+        P(30, "thirty"),
+        P(40, "forty"),
+        P(50, "fifty"),
+        P(60, "sixty"),
+        P(70, "seventy"),
+        P(80, "eighty"),
+    };
+    C c1(std::begin(a1), std::end(a1), 0, Hash(1), Compare(1), Alloc());
+    C c2(std::begin(a2), std::end(a2), 0, Hash(2), Compare(2), Alloc());
+    c2.max_load_factor(2);
+    swap(c1, c2);
+
+    assert(c1.bucket_count() >= 8);
+    assert(c1.size() == 8);
+    assert(c1.at(10) == "ten");
+    assert(c1.at(20) == "twenty");
+    assert(c1.at(30) == "thirty");
+    assert(c1.at(40) == "forty");
+    assert(c1.at(50) == "fifty");
+    assert(c1.at(60) == "sixty");
+    assert(c1.at(70) == "seventy");
+    assert(c1.at(80) == "eighty");
+    assert(c1.hash_function() == Hash(2));
+    assert(c1.key_eq() == Compare(2));
+    assert(c1.get_allocator() == Alloc());
+    assert(static_cast<std::size_t>(std::distance(c1.begin(), c1.end())) == c1.size());
+    assert(static_cast<std::size_t>(std::distance(c1.cbegin(), c1.cend())) == c1.size());
+    assert(c1.max_load_factor() == 2);
+
+    assert(c2.bucket_count() >= 4);
+    assert(c2.size() == 4);
+    assert(c2.at(1) == "one");
+    assert(c2.at(2) == "two");
+    assert(c2.at(3) == "three");
+    assert(c2.at(4) == "four");
+    assert(c2.hash_function() == Hash(1));
+    assert(c2.key_eq() == Compare(1));
+    assert(c2.get_allocator() == Alloc());
+    assert(static_cast<std::size_t>(std::distance(c2.begin(), c2.end())) == c2.size());
+    assert(static_cast<std::size_t>(std::distance(c2.cbegin(), c2.cend())) == c2.size());
+    assert(c2.max_load_factor() == 1);
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/unord/unord.multimap/allocator_mismatch.verify.cpp b/libcxx/test/std/containers/unord/unord.multimap/allocator_mismatch.verify.cpp
index 546ddc3dd1108..d4bf7b6ca2a03 100644
--- a/libcxx/test/std/containers/unord/unord.multimap/allocator_mismatch.verify.cpp
+++ b/libcxx/test/std/containers/unord/unord.multimap/allocator_mismatch.verify.cpp
@@ -12,4 +12,4 @@
 #include <unordered_map>
 
 std::unordered_multimap<int, int, std::hash<int>, std::less<int>, std::allocator<long> > m;
-  // expected-error-re@*:* {{static assertion failed{{.*}}Allocator::value_type must be same type as value_type}}
+// expected-error-re@*:* {{static assertion failed{{.*}}Allocator::value_type must be same type as value_type}}
diff --git a/libcxx/test/std/containers/unord/unord.multimap/bucket.pass.cpp b/libcxx/test/std/containers/unord/unord.multimap/bucket.pass.cpp
index 034408740315a..b4a640da52953 100644
--- a/libcxx/test/std/containers/unord/unord.multimap/bucket.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.multimap/bucket.pass.cpp
@@ -22,47 +22,48 @@
 #include "test_macros.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef std::unordered_multimap<int, std::string> C;
-        typedef std::pair<int, std::string> P;
-        P a[] =
-        {
-            P(1, "one"),
-            P(2, "two"),
-            P(3, "three"),
-            P(4, "four"),
-            P(1, "four"),
-            P(2, "four"),
-        };
-        const C c(std::begin(a), std::end(a));
-        std::size_t bc = c.bucket_count();
-        assert(bc >= 7);
-        for (std::size_t i = 0; i < 13; ++i)
-            LIBCPP_ASSERT(c.bucket(i) == i % bc);
-    }
+int main(int, char**) {
+  {
+    typedef std::unordered_multimap<int, std::string> C;
+    typedef std::pair<int, std::string> P;
+    P a[] = {
+        P(1, "one"),
+        P(2, "two"),
+        P(3, "three"),
+        P(4, "four"),
+        P(1, "four"),
+        P(2, "four"),
+    };
+    const C c(std::begin(a), std::end(a));
+    std::size_t bc = c.bucket_count();
+    assert(bc >= 7);
+    for (std::size_t i = 0; i < 13; ++i)
+      LIBCPP_ASSERT(c.bucket(i) == i % bc);
+  }
 #if TEST_STD_VER >= 11
-    {
-        typedef std::unordered_multimap<int, std::string, std::hash<int>, std::equal_to<int>,
-                            min_allocator<std::pair<const int, std::string>>> C;
-        typedef std::pair<int, std::string> P;
-        P a[] =
-        {
-            P(1, "one"),
-            P(2, "two"),
-            P(3, "three"),
-            P(4, "four"),
-            P(1, "four"),
-            P(2, "four"),
-        };
-        const C c(std::begin(a), std::end(a));
-        std::size_t bc = c.bucket_count();
-        assert(bc >= 7);
-        for (std::size_t i = 0; i < 13; ++i)
-            LIBCPP_ASSERT(c.bucket(i) == i % bc);
-    }
+  {
+    typedef std::unordered_multimap<int,
+                                    std::string,
+                                    std::hash<int>,
+                                    std::equal_to<int>,
+                                    min_allocator<std::pair<const int, std::string>>>
+        C;
+    typedef std::pair<int, std::string> P;
+    P a[] = {
+        P(1, "one"),
+        P(2, "two"),
+        P(3, "three"),
+        P(4, "four"),
+        P(1, "four"),
+        P(2, "four"),
+    };
+    const C c(std::begin(a), std::end(a));
+    std::size_t bc = c.bucket_count();
+    assert(bc >= 7);
+    for (std::size_t i = 0; i < 13; ++i)
+      LIBCPP_ASSERT(c.bucket(i) == i % bc);
+  }
 #endif
 
-    return 0;
+  return 0;
 }
diff --git a/libcxx/test/std/containers/unord/unord.multimap/bucket_count.pass.cpp b/libcxx/test/std/containers/unord/unord.multimap/bucket_count.pass.cpp
index 3515dbc93ca45..8368214d5375a 100644
--- a/libcxx/test/std/containers/unord/unord.multimap/bucket_count.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.multimap/bucket_count.pass.cpp
@@ -21,30 +21,28 @@
 
 #include "test_macros.h"
 
-int main(int, char**)
-{
-    {
-        typedef std::unordered_multimap<int, std::string> C;
-        const C c;
-        LIBCPP_ASSERT(c.bucket_count() == 0);
-    }
-    {
-        typedef std::unordered_multimap<int, std::string> C;
-        typedef std::pair<int, std::string> P;
-        P a[] =
-        {
-            P(10, "ten"),
-            P(20, "twenty"),
-            P(30, "thirty"),
-            P(40, "forty"),
-            P(50, "fifty"),
-            P(60, "sixty"),
-            P(70, "seventy"),
-            P(80, "eighty"),
-        };
-        const C c(std::begin(a), std::end(a));
-        assert(c.bucket_count() >= 8);
-    }
+int main(int, char**) {
+  {
+    typedef std::unordered_multimap<int, std::string> C;
+    const C c;
+    LIBCPP_ASSERT(c.bucket_count() == 0);
+  }
+  {
+    typedef std::unordered_multimap<int, std::string> C;
+    typedef std::pair<int, std::string> P;
+    P a[] = {
+        P(10, "ten"),
+        P(20, "twenty"),
+        P(30, "thirty"),
+        P(40, "forty"),
+        P(50, "fifty"),
+        P(60, "sixty"),
+        P(70, "seventy"),
+        P(80, "eighty"),
+    };
+    const C c(std::begin(a), std::end(a));
+    assert(c.bucket_count() >= 8);
+  }
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/unord/unord.multimap/bucket_size.pass.cpp b/libcxx/test/std/containers/unord/unord.multimap/bucket_size.pass.cpp
index 1659fef9d462c..ee80e34660b2e 100644
--- a/libcxx/test/std/containers/unord/unord.multimap/bucket_size.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.multimap/bucket_size.pass.cpp
@@ -22,55 +22,56 @@
 #include "test_macros.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef std::unordered_multimap<int, std::string> C;
-        typedef std::pair<int, std::string> P;
-        P a[] =
-        {
-            P(1, "one"),
-            P(2, "two"),
-            P(3, "three"),
-            P(4, "four"),
-            P(1, "four"),
-            P(2, "four"),
-        };
-        const C c(std::begin(a), std::end(a));
-        assert(c.bucket_count() >= 7);
-        LIBCPP_ASSERT(c.bucket_size(0) == 0);
-        LIBCPP_ASSERT(c.bucket_size(1) == 2);
-        LIBCPP_ASSERT(c.bucket_size(2) == 2);
-        LIBCPP_ASSERT(c.bucket_size(3) == 1);
-        LIBCPP_ASSERT(c.bucket_size(4) == 1);
-        LIBCPP_ASSERT(c.bucket_size(5) == 0);
-        LIBCPP_ASSERT(c.bucket_size(6) == 0);
-    }
+int main(int, char**) {
+  {
+    typedef std::unordered_multimap<int, std::string> C;
+    typedef std::pair<int, std::string> P;
+    P a[] = {
+        P(1, "one"),
+        P(2, "two"),
+        P(3, "three"),
+        P(4, "four"),
+        P(1, "four"),
+        P(2, "four"),
+    };
+    const C c(std::begin(a), std::end(a));
+    assert(c.bucket_count() >= 7);
+    LIBCPP_ASSERT(c.bucket_size(0) == 0);
+    LIBCPP_ASSERT(c.bucket_size(1) == 2);
+    LIBCPP_ASSERT(c.bucket_size(2) == 2);
+    LIBCPP_ASSERT(c.bucket_size(3) == 1);
+    LIBCPP_ASSERT(c.bucket_size(4) == 1);
+    LIBCPP_ASSERT(c.bucket_size(5) == 0);
+    LIBCPP_ASSERT(c.bucket_size(6) == 0);
+  }
 #if TEST_STD_VER >= 11
-    {
-        typedef std::unordered_multimap<int, std::string, std::hash<int>, std::equal_to<int>,
-                            min_allocator<std::pair<const int, std::string>>> C;
-        typedef std::pair<int, std::string> P;
-        P a[] =
-        {
-            P(1, "one"),
-            P(2, "two"),
-            P(3, "three"),
-            P(4, "four"),
-            P(1, "four"),
-            P(2, "four"),
-        };
-        const C c(std::begin(a), std::end(a));
-        assert(c.bucket_count() >= 7);
-        LIBCPP_ASSERT(c.bucket_size(0) == 0);
-        LIBCPP_ASSERT(c.bucket_size(1) == 2);
-        LIBCPP_ASSERT(c.bucket_size(2) == 2);
-        LIBCPP_ASSERT(c.bucket_size(3) == 1);
-        LIBCPP_ASSERT(c.bucket_size(4) == 1);
-        LIBCPP_ASSERT(c.bucket_size(5) == 0);
-        LIBCPP_ASSERT(c.bucket_size(6) == 0);
-    }
+  {
+    typedef std::unordered_multimap<int,
+                                    std::string,
+                                    std::hash<int>,
+                                    std::equal_to<int>,
+                                    min_allocator<std::pair<const int, std::string>>>
+        C;
+    typedef std::pair<int, std::string> P;
+    P a[] = {
+        P(1, "one"),
+        P(2, "two"),
+        P(3, "three"),
+        P(4, "four"),
+        P(1, "four"),
+        P(2, "four"),
+    };
+    const C c(std::begin(a), std::end(a));
+    assert(c.bucket_count() >= 7);
+    LIBCPP_ASSERT(c.bucket_size(0) == 0);
+    LIBCPP_ASSERT(c.bucket_size(1) == 2);
+    LIBCPP_ASSERT(c.bucket_size(2) == 2);
+    LIBCPP_ASSERT(c.bucket_size(3) == 1);
+    LIBCPP_ASSERT(c.bucket_size(4) == 1);
+    LIBCPP_ASSERT(c.bucket_size(5) == 0);
+    LIBCPP_ASSERT(c.bucket_size(6) == 0);
+  }
 #endif
 
-    return 0;
+  return 0;
 }
diff --git a/libcxx/test/std/containers/unord/unord.multimap/contains.transparent.pass.cpp b/libcxx/test/std/containers/unord/unord.multimap/contains.transparent.pass.cpp
index 9ebf4f2aa5df9..7132bf79ba24d 100644
--- a/libcxx/test/std/containers/unord/unord.multimap/contains.transparent.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.multimap/contains.transparent.pass.cpp
@@ -20,44 +20,43 @@
 #include <unordered_map>
 #include "test_transparent_unordered.h"
 
-int main(int, char**)
-{
-    using key_type = StoredType<int>;
-
-    {
-      // Make sure conversions don't happen for transparent non-final hasher and key_equal
-      using M = unord_map_type<std::unordered_multimap, transparent_hash, std::equal_to<>>;
-      test_transparent_contains<M>({{1, 2}, {1, 3}, {2, 3}});
-      test_transparent_contains<const M>({{1, 2}, {1, 3}, {2, 3}});
-    }
-
-    {
-      // Make sure conversions don't happen for transparent final hasher and key_equal
-      using M = unord_map_type<std::unordered_multimap, transparent_hash_final, transparent_equal_final>;
-      test_transparent_contains<M>({{1, 2}, {1, 3}, {2, 3}});
-      test_transparent_contains<const M>({{1, 2}, {1, 3}, {2, 3}});
-    }
-
-    {
-      // Make sure conversions do happen for non-transparent hasher
-      using M = unord_map_type<std::unordered_multimap, non_transparent_hash, std::equal_to<>>;
-      test_non_transparent_contains<M>({{1, 2}, {1, 3}, {2, 3}});
-      test_non_transparent_contains<const M>({{1, 2}, {1, 3}, {2, 3}});
-    }
-
-    {
-      // Make sure conversions do happen for non-transparent key_equal
-      using M = unord_map_type<std::unordered_multimap, transparent_hash, std::equal_to<key_type>>;
-      test_non_transparent_contains<M>({{1, 2}, {1, 3}, {2, 3}});
-      test_non_transparent_contains<const M>({{1, 2}, {1, 3}, {2, 3}});
-    }
-
-    {
-      // Make sure conversions do happen for both non-transparent hasher and key_equal
-      using M = unord_map_type<std::unordered_multimap, non_transparent_hash, std::equal_to<key_type>>;
-      test_non_transparent_contains<M>({{1, 2}, {1, 3}, {2, 3}});
-      test_non_transparent_contains<const M>({{1, 2}, {1, 3}, {2, 3}});
-    }
-
-    return 0;
+int main(int, char**) {
+  using key_type = StoredType<int>;
+
+  {
+    // Make sure conversions don't happen for transparent non-final hasher and key_equal
+    using M = unord_map_type<std::unordered_multimap, transparent_hash, std::equal_to<>>;
+    test_transparent_contains<M>({{1, 2}, {1, 3}, {2, 3}});
+    test_transparent_contains<const M>({{1, 2}, {1, 3}, {2, 3}});
+  }
+
+  {
+    // Make sure conversions don't happen for transparent final hasher and key_equal
+    using M = unord_map_type<std::unordered_multimap, transparent_hash_final, transparent_equal_final>;
+    test_transparent_contains<M>({{1, 2}, {1, 3}, {2, 3}});
+    test_transparent_contains<const M>({{1, 2}, {1, 3}, {2, 3}});
+  }
+
+  {
+    // Make sure conversions do happen for non-transparent hasher
+    using M = unord_map_type<std::unordered_multimap, non_transparent_hash, std::equal_to<>>;
+    test_non_transparent_contains<M>({{1, 2}, {1, 3}, {2, 3}});
+    test_non_transparent_contains<const M>({{1, 2}, {1, 3}, {2, 3}});
+  }
+
+  {
+    // Make sure conversions do happen for non-transparent key_equal
+    using M = unord_map_type<std::unordered_multimap, transparent_hash, std::equal_to<key_type>>;
+    test_non_transparent_contains<M>({{1, 2}, {1, 3}, {2, 3}});
+    test_non_transparent_contains<const M>({{1, 2}, {1, 3}, {2, 3}});
+  }
+
+  {
+    // Make sure conversions do happen for both non-transparent hasher and key_equal
+    using M = unord_map_type<std::unordered_multimap, non_transparent_hash, std::equal_to<key_type>>;
+    test_non_transparent_contains<M>({{1, 2}, {1, 3}, {2, 3}});
+    test_non_transparent_contains<const M>({{1, 2}, {1, 3}, {2, 3}});
+  }
+
+  return 0;
 }
diff --git a/libcxx/test/std/containers/unord/unord.multimap/count.pass.cpp b/libcxx/test/std/containers/unord/unord.multimap/count.pass.cpp
index 4ece256652955..e2eead5093293 100644
--- a/libcxx/test/std/containers/unord/unord.multimap/count.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.multimap/count.pass.cpp
@@ -22,52 +22,53 @@
 #include "test_macros.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef std::unordered_multimap<int, std::string> C;
-        typedef std::pair<int, std::string> P;
-        P a[] =
-        {
-            P(10, "ten"),
-            P(20, "twenty"),
-            P(30, "thirty"),
-            P(40, "forty"),
-            P(50, "fifty"),
-            P(50, "fiftyA"),
-            P(50, "fiftyB"),
-            P(60, "sixty"),
-            P(70, "seventy"),
-            P(80, "eighty"),
-        };
-        const C c(std::begin(a), std::end(a));
-        assert(c.count(30) == 1);
-        assert(c.count(50) == 3);
-        assert(c.count(5) == 0);
-    }
+int main(int, char**) {
+  {
+    typedef std::unordered_multimap<int, std::string> C;
+    typedef std::pair<int, std::string> P;
+    P a[] = {
+        P(10, "ten"),
+        P(20, "twenty"),
+        P(30, "thirty"),
+        P(40, "forty"),
+        P(50, "fifty"),
+        P(50, "fiftyA"),
+        P(50, "fiftyB"),
+        P(60, "sixty"),
+        P(70, "seventy"),
+        P(80, "eighty"),
+    };
+    const C c(std::begin(a), std::end(a));
+    assert(c.count(30) == 1);
+    assert(c.count(50) == 3);
+    assert(c.count(5) == 0);
+  }
 #if TEST_STD_VER >= 11
-    {
-        typedef std::unordered_multimap<int, std::string, std::hash<int>, std::equal_to<int>,
-                            min_allocator<std::pair<const int, std::string>>> C;
-        typedef std::pair<int, std::string> P;
-        P a[] =
-        {
-            P(10, "ten"),
-            P(20, "twenty"),
-            P(30, "thirty"),
-            P(40, "forty"),
-            P(50, "fifty"),
-            P(50, "fiftyA"),
-            P(50, "fiftyB"),
-            P(60, "sixty"),
-            P(70, "seventy"),
-            P(80, "eighty"),
-        };
-        const C c(std::begin(a), std::end(a));
-        assert(c.count(30) == 1);
-        assert(c.count(50) == 3);
-        assert(c.count(5) == 0);
-    }
+  {
+    typedef std::unordered_multimap<int,
+                                    std::string,
+                                    std::hash<int>,
+                                    std::equal_to<int>,
+                                    min_allocator<std::pair<const int, std::string>>>
+        C;
+    typedef std::pair<int, std::string> P;
+    P a[] = {
+        P(10, "ten"),
+        P(20, "twenty"),
+        P(30, "thirty"),
+        P(40, "forty"),
+        P(50, "fifty"),
+        P(50, "fiftyA"),
+        P(50, "fiftyB"),
+        P(60, "sixty"),
+        P(70, "seventy"),
+        P(80, "eighty"),
+    };
+    const C c(std::begin(a), std::end(a));
+    assert(c.count(30) == 1);
+    assert(c.count(50) == 3);
+    assert(c.count(5) == 0);
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/unord/unord.multimap/count.transparent.pass.cpp b/libcxx/test/std/containers/unord/unord.multimap/count.transparent.pass.cpp
index b9755cfdeaaf2..9e394c6b08e75 100644
--- a/libcxx/test/std/containers/unord/unord.multimap/count.transparent.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.multimap/count.transparent.pass.cpp
@@ -21,44 +21,43 @@
 
 #include "test_transparent_unordered.h"
 
-int main(int, char**)
-{
-    using key_type = StoredType<int>;
-
-    {
-      // Make sure conversions don't happen for transparent non-final hasher and key_equal
-      using M = unord_map_type<std::unordered_multimap, transparent_hash, std::equal_to<>>;
-      test_transparent_count<M>({{1, 2}, {1, 3}, {2, 3}});
-      test_transparent_count<const M>({{1, 2}, {1, 3}, {2, 3}});
-    }
-
-    {
-      // Make sure conversions don't happen for transparent final hasher and key_equal
-      using M = unord_map_type<std::unordered_multimap, transparent_hash_final, transparent_equal_final>;
-      test_transparent_count<M>({{1, 2}, {1, 3}, {2, 3}});
-      test_transparent_count<const M>({{1, 2}, {1, 3}, {2, 3}});
-    }
-
-    {
-      // Make sure conversions do happen for non-transparent hasher
-      using M = unord_map_type<std::unordered_multimap, non_transparent_hash, std::equal_to<>>;
-      test_non_transparent_count<M>({{1, 2}, {1, 3}, {2, 3}});
-      test_non_transparent_count<const M>({{1, 2}, {1, 3}, {2, 3}});
-    }
-
-    {
-      // Make sure conversions do happen for non-transparent key_equal
-      using M = unord_map_type<std::unordered_multimap, transparent_hash, std::equal_to<key_type>>;
-      test_non_transparent_count<M>({{1, 2}, {1, 3}, {2, 3}});
-      test_non_transparent_count<const M>({{1, 2}, {1, 3}, {2, 3}});
-    }
-
-    {
-      // Make sure conversions do happen for both non-transparent hasher and key_equal
-      using M = unord_map_type<std::unordered_multimap, non_transparent_hash, std::equal_to<key_type>>;
-      test_non_transparent_count<M>({{1, 2}, {1, 3}, {2, 3}});
-      test_non_transparent_count<const M>({{1, 2}, {1, 3}, {2, 3}});
-    }
-
-    return 0;
+int main(int, char**) {
+  using key_type = StoredType<int>;
+
+  {
+    // Make sure conversions don't happen for transparent non-final hasher and key_equal
+    using M = unord_map_type<std::unordered_multimap, transparent_hash, std::equal_to<>>;
+    test_transparent_count<M>({{1, 2}, {1, 3}, {2, 3}});
+    test_transparent_count<const M>({{1, 2}, {1, 3}, {2, 3}});
+  }
+
+  {
+    // Make sure conversions don't happen for transparent final hasher and key_equal
+    using M = unord_map_type<std::unordered_multimap, transparent_hash_final, transparent_equal_final>;
+    test_transparent_count<M>({{1, 2}, {1, 3}, {2, 3}});
+    test_transparent_count<const M>({{1, 2}, {1, 3}, {2, 3}});
+  }
+
+  {
+    // Make sure conversions do happen for non-transparent hasher
+    using M = unord_map_type<std::unordered_multimap, non_transparent_hash, std::equal_to<>>;
+    test_non_transparent_count<M>({{1, 2}, {1, 3}, {2, 3}});
+    test_non_transparent_count<const M>({{1, 2}, {1, 3}, {2, 3}});
+  }
+
+  {
+    // Make sure conversions do happen for non-transparent key_equal
+    using M = unord_map_type<std::unordered_multimap, transparent_hash, std::equal_to<key_type>>;
+    test_non_transparent_count<M>({{1, 2}, {1, 3}, {2, 3}});
+    test_non_transparent_count<const M>({{1, 2}, {1, 3}, {2, 3}});
+  }
+
+  {
+    // Make sure conversions do happen for both non-transparent hasher and key_equal
+    using M = unord_map_type<std::unordered_multimap, non_transparent_hash, std::equal_to<key_type>>;
+    test_non_transparent_count<M>({{1, 2}, {1, 3}, {2, 3}});
+    test_non_transparent_count<const M>({{1, 2}, {1, 3}, {2, 3}});
+  }
+
+  return 0;
 }
diff --git a/libcxx/test/std/containers/unord/unord.multimap/empty.pass.cpp b/libcxx/test/std/containers/unord/unord.multimap/empty.pass.cpp
index 6189b7f0e3ded..8769d11dcaa81 100644
--- a/libcxx/test/std/containers/unord/unord.multimap/empty.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.multimap/empty.pass.cpp
@@ -18,9 +18,8 @@
 #include "test_macros.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
+int main(int, char**) {
+  {
     typedef std::unordered_multimap<int, double> M;
     M m;
     ASSERT_NOEXCEPT(m.empty());
@@ -29,10 +28,12 @@ int main(int, char**)
     assert(!m.empty());
     m.clear();
     assert(m.empty());
-    }
+  }
 #if TEST_STD_VER >= 11
-    {
-    typedef std::unordered_multimap<int, double, std::hash<int>, std::equal_to<int>, min_allocator<std::pair<const int, double>>> M;
+  {
+    typedef std::
+        unordered_multimap<int, double, std::hash<int>, std::equal_to<int>, min_allocator<std::pair<const int, double>>>
+            M;
     M m;
     ASSERT_NOEXCEPT(m.empty());
     assert(m.empty());
@@ -40,7 +41,7 @@ int main(int, char**)
     assert(!m.empty());
     m.clear();
     assert(m.empty());
-    }
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/unord/unord.multimap/empty.verify.cpp b/libcxx/test/std/containers/unord/unord.multimap/empty.verify.cpp
index 999d43fb63bc0..21775acd5acda 100644
--- a/libcxx/test/std/containers/unord/unord.multimap/empty.verify.cpp
+++ b/libcxx/test/std/containers/unord/unord.multimap/empty.verify.cpp
@@ -17,6 +17,6 @@
 #include <unordered_map>
 
 void f() {
-    std::unordered_multimap<int, int> c;
-    c.empty(); // expected-warning {{ignoring return value of function declared with 'nodiscard' attribute}}
+  std::unordered_multimap<int, int> c;
+  c.empty(); // expected-warning {{ignoring return value of function declared with 'nodiscard' attribute}}
 }
diff --git a/libcxx/test/std/containers/unord/unord.multimap/eq.pass.cpp b/libcxx/test/std/containers/unord/unord.multimap/eq.pass.cpp
index 083fff13b3812..06d86b5e02662 100644
--- a/libcxx/test/std/containers/unord/unord.multimap/eq.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.multimap/eq.pass.cpp
@@ -28,223 +28,230 @@
 
 #include "test_comparisons.h"
 
-int main(int, char**)
-{
-    {
-        typedef std::unordered_multimap<int, std::string> C;
-        typedef std::pair<int, std::string> P;
-        P a[] =
-        {
-            P(10, "ten"),
-            P(20, "twenty"),
-            P(20, "twenty 2"),
-            P(30, "thirty"),
-            P(40, "forty"),
-            P(50, "fifty"),
-            P(50, "fifty 2"),
-            P(50, "fifty 3"),
-            P(60, "sixty"),
-            P(70, "seventy"),
-            P(80, "eighty"),
-        };
-        const C c1(std::begin(a), std::end(a));
-        const C c2;
-        assert(testEquality(c1, c2, false));
-    }
-    {
-        typedef std::unordered_multimap<int, std::string> C;
-        typedef std::pair<int, std::string> P;
-        P a[] =
-        {
-            P(10, "ten"),
-            P(20, "twenty"),
-            P(20, "twenty 2"),
-            P(30, "thirty"),
-            P(40, "forty"),
-            P(50, "fifty"),
-            P(50, "fifty 2"),
-            P(50, "fifty 3"),
-            P(60, "sixty"),
-            P(70, "seventy"),
-            P(80, "eighty"),
-        };
-        const C c1(std::begin(a), std::end(a));
-        const C c2 = c1;
-        assert(testEquality(c1, c2, true));
-    }
-    {
-        typedef std::unordered_multimap<int, std::string> C;
-        typedef std::pair<int, std::string> P;
-        P a[] =
-        {
-            P(10, "ten"),
-            P(20, "twenty"),
-            P(20, "twenty 2"),
-            P(30, "thirty"),
-            P(40, "forty"),
-            P(50, "fifty"),
-            P(50, "fifty 2"),
-            P(50, "fifty 3"),
-            P(60, "sixty"),
-            P(70, "seventy"),
-            P(80, "eighty"),
-        };
-        C c1(std::begin(a), std::end(a));
-        C c2 = c1;
-        c2.rehash(30);
-        assert(testEquality(c1, c2, true));
-        c2.insert(P(90, "ninety"));
-        assert(testEquality(c1, c2, false));
-        c1.insert(P(90, "ninety"));
-        assert(testEquality(c1, c2, true));
-    }
-    {
-        typedef std::unordered_multimap<int, std::string> C;
-        typedef std::pair<int, std::string> P;
-        P a[] =
-        {
-            P(10, "ten"),
-            P(20, "twenty"),
-            P(20, "twenty 2"),
-            P(30, "thirty"),
-            P(40, "forty"),
-            P(50, "fifty"),
-            P(50, "fifty 2"),
-            P(50, "fifty 3"),
-            P(60, "sixty"),
-            P(70, "seventy"),
-            P(80, "eighty"),
-        };
-        C c1(std::begin(a), std::end(a));
-        C c2 = c1;
-        assert(testEquality(c1, c2, true));
-        c1.insert(P(70, "seventy 2"));
-        c2.insert(P(80, "eighty 2"));
-        assert(testEquality(c1, c2, false));
-    }
+int main(int, char**) {
+  {
+    typedef std::unordered_multimap<int, std::string> C;
+    typedef std::pair<int, std::string> P;
+    P a[] = {
+        P(10, "ten"),
+        P(20, "twenty"),
+        P(20, "twenty 2"),
+        P(30, "thirty"),
+        P(40, "forty"),
+        P(50, "fifty"),
+        P(50, "fifty 2"),
+        P(50, "fifty 3"),
+        P(60, "sixty"),
+        P(70, "seventy"),
+        P(80, "eighty"),
+    };
+    const C c1(std::begin(a), std::end(a));
+    const C c2;
+    assert(testEquality(c1, c2, false));
+  }
+  {
+    typedef std::unordered_multimap<int, std::string> C;
+    typedef std::pair<int, std::string> P;
+    P a[] = {
+        P(10, "ten"),
+        P(20, "twenty"),
+        P(20, "twenty 2"),
+        P(30, "thirty"),
+        P(40, "forty"),
+        P(50, "fifty"),
+        P(50, "fifty 2"),
+        P(50, "fifty 3"),
+        P(60, "sixty"),
+        P(70, "seventy"),
+        P(80, "eighty"),
+    };
+    const C c1(std::begin(a), std::end(a));
+    const C c2 = c1;
+    assert(testEquality(c1, c2, true));
+  }
+  {
+    typedef std::unordered_multimap<int, std::string> C;
+    typedef std::pair<int, std::string> P;
+    P a[] = {
+        P(10, "ten"),
+        P(20, "twenty"),
+        P(20, "twenty 2"),
+        P(30, "thirty"),
+        P(40, "forty"),
+        P(50, "fifty"),
+        P(50, "fifty 2"),
+        P(50, "fifty 3"),
+        P(60, "sixty"),
+        P(70, "seventy"),
+        P(80, "eighty"),
+    };
+    C c1(std::begin(a), std::end(a));
+    C c2 = c1;
+    c2.rehash(30);
+    assert(testEquality(c1, c2, true));
+    c2.insert(P(90, "ninety"));
+    assert(testEquality(c1, c2, false));
+    c1.insert(P(90, "ninety"));
+    assert(testEquality(c1, c2, true));
+  }
+  {
+    typedef std::unordered_multimap<int, std::string> C;
+    typedef std::pair<int, std::string> P;
+    P a[] = {
+        P(10, "ten"),
+        P(20, "twenty"),
+        P(20, "twenty 2"),
+        P(30, "thirty"),
+        P(40, "forty"),
+        P(50, "fifty"),
+        P(50, "fifty 2"),
+        P(50, "fifty 3"),
+        P(60, "sixty"),
+        P(70, "seventy"),
+        P(80, "eighty"),
+    };
+    C c1(std::begin(a), std::end(a));
+    C c2 = c1;
+    assert(testEquality(c1, c2, true));
+    c1.insert(P(70, "seventy 2"));
+    c2.insert(P(80, "eighty 2"));
+    assert(testEquality(c1, c2, false));
+  }
 #if TEST_STD_VER >= 11
-    {
-        typedef std::unordered_multimap<int, std::string, std::hash<int>, std::equal_to<int>,
-                            min_allocator<std::pair<const int, std::string>>> C;
-        typedef std::pair<int, std::string> P;
-        P a[] =
-        {
-            P(10, "ten"),
-            P(20, "twenty"),
-            P(20, "twenty 2"),
-            P(30, "thirty"),
-            P(40, "forty"),
-            P(50, "fifty"),
-            P(50, "fifty 2"),
-            P(50, "fifty 3"),
-            P(60, "sixty"),
-            P(70, "seventy"),
-            P(80, "eighty"),
-        };
-        const C c1(std::begin(a), std::end(a));
-        const C c2;
-        assert(testEquality(c1, c2, false));
-    }
-    {
-        typedef std::unordered_multimap<int, std::string, std::hash<int>, std::equal_to<int>,
-                            min_allocator<std::pair<const int, std::string>>> C;
-        typedef std::pair<int, std::string> P;
-        P a[] =
-        {
-            P(10, "ten"),
-            P(20, "twenty"),
-            P(20, "twenty 2"),
-            P(30, "thirty"),
-            P(40, "forty"),
-            P(50, "fifty"),
-            P(50, "fifty 2"),
-            P(50, "fifty 3"),
-            P(60, "sixty"),
-            P(70, "seventy"),
-            P(80, "eighty"),
-        };
-        const C c1(std::begin(a), std::end(a));
-        const C c2 = c1;
-        assert(testEquality(c1, c2, true));
-    }
-    {
-        typedef std::unordered_multimap<int, std::string, std::hash<int>, std::equal_to<int>,
-                            min_allocator<std::pair<const int, std::string>>> C;
-        typedef std::pair<int, std::string> P;
-        P a[] =
-        {
-            P(10, "ten"),
-            P(20, "twenty"),
-            P(20, "twenty 2"),
-            P(30, "thirty"),
-            P(40, "forty"),
-            P(50, "fifty"),
-            P(50, "fifty 2"),
-            P(50, "fifty 3"),
-            P(60, "sixty"),
-            P(70, "seventy"),
-            P(80, "eighty"),
-        };
-        C c1(std::begin(a), std::end(a));
-        C c2 = c1;
-        c2.rehash(30);
-        assert(testEquality(c1, c2, true));
-        c2.insert(P(90, "ninety"));
-        assert(testEquality(c1, c2, false));
-        c1.insert(P(90, "ninety"));
-        assert(testEquality(c1, c2, true));
-    }
-    {
-        typedef std::unordered_multimap<int, std::string, std::hash<int>, std::equal_to<int>,
-                            min_allocator<std::pair<const int, std::string>>> C;
-        typedef std::pair<int, std::string> P;
-        P a[] =
-        {
-            P(10, "ten"),
-            P(20, "twenty"),
-            P(20, "twenty 2"),
-            P(30, "thirty"),
-            P(40, "forty"),
-            P(50, "fifty"),
-            P(50, "fifty 2"),
-            P(50, "fifty 3"),
-            P(60, "sixty"),
-            P(70, "seventy"),
-            P(80, "eighty"),
-        };
-        C c1(std::begin(a), std::end(a));
-        C c2 = c1;
-        assert(testEquality(c1, c2, true));
-        c1.insert(P(70, "seventy 2"));
-        c2.insert(P(80, "eighty 2"));
-        assert(testEquality(c1, c2, false));
-    }
+  {
+    typedef std::unordered_multimap<int,
+                                    std::string,
+                                    std::hash<int>,
+                                    std::equal_to<int>,
+                                    min_allocator<std::pair<const int, std::string>>>
+        C;
+    typedef std::pair<int, std::string> P;
+    P a[] = {
+        P(10, "ten"),
+        P(20, "twenty"),
+        P(20, "twenty 2"),
+        P(30, "thirty"),
+        P(40, "forty"),
+        P(50, "fifty"),
+        P(50, "fifty 2"),
+        P(50, "fifty 3"),
+        P(60, "sixty"),
+        P(70, "seventy"),
+        P(80, "eighty"),
+    };
+    const C c1(std::begin(a), std::end(a));
+    const C c2;
+    assert(testEquality(c1, c2, false));
+  }
+  {
+    typedef std::unordered_multimap<int,
+                                    std::string,
+                                    std::hash<int>,
+                                    std::equal_to<int>,
+                                    min_allocator<std::pair<const int, std::string>>>
+        C;
+    typedef std::pair<int, std::string> P;
+    P a[] = {
+        P(10, "ten"),
+        P(20, "twenty"),
+        P(20, "twenty 2"),
+        P(30, "thirty"),
+        P(40, "forty"),
+        P(50, "fifty"),
+        P(50, "fifty 2"),
+        P(50, "fifty 3"),
+        P(60, "sixty"),
+        P(70, "seventy"),
+        P(80, "eighty"),
+    };
+    const C c1(std::begin(a), std::end(a));
+    const C c2 = c1;
+    assert(testEquality(c1, c2, true));
+  }
+  {
+    typedef std::unordered_multimap<int,
+                                    std::string,
+                                    std::hash<int>,
+                                    std::equal_to<int>,
+                                    min_allocator<std::pair<const int, std::string>>>
+        C;
+    typedef std::pair<int, std::string> P;
+    P a[] = {
+        P(10, "ten"),
+        P(20, "twenty"),
+        P(20, "twenty 2"),
+        P(30, "thirty"),
+        P(40, "forty"),
+        P(50, "fifty"),
+        P(50, "fifty 2"),
+        P(50, "fifty 3"),
+        P(60, "sixty"),
+        P(70, "seventy"),
+        P(80, "eighty"),
+    };
+    C c1(std::begin(a), std::end(a));
+    C c2 = c1;
+    c2.rehash(30);
+    assert(testEquality(c1, c2, true));
+    c2.insert(P(90, "ninety"));
+    assert(testEquality(c1, c2, false));
+    c1.insert(P(90, "ninety"));
+    assert(testEquality(c1, c2, true));
+  }
+  {
+    typedef std::unordered_multimap<int,
+                                    std::string,
+                                    std::hash<int>,
+                                    std::equal_to<int>,
+                                    min_allocator<std::pair<const int, std::string>>>
+        C;
+    typedef std::pair<int, std::string> P;
+    P a[] = {
+        P(10, "ten"),
+        P(20, "twenty"),
+        P(20, "twenty 2"),
+        P(30, "thirty"),
+        P(40, "forty"),
+        P(50, "fifty"),
+        P(50, "fifty 2"),
+        P(50, "fifty 3"),
+        P(60, "sixty"),
+        P(70, "seventy"),
+        P(80, "eighty"),
+    };
+    C c1(std::begin(a), std::end(a));
+    C c2 = c1;
+    assert(testEquality(c1, c2, true));
+    c1.insert(P(70, "seventy 2"));
+    c2.insert(P(80, "eighty 2"));
+    assert(testEquality(c1, c2, false));
+  }
 #endif
 
-    // Make sure we take into account the number of times that a key repeats into equality.
-    {
-        typedef std::pair<int, char> P;
-        P a[] = {P(1, 'a'), P(1, 'b'),            P(1, 'd'), P(2, 'b')};
-        P b[] = {P(1, 'a'), P(1, 'b'), P(1, 'b'), P(1, 'd'), P(2, 'b')};
+  // Make sure we take into account the number of times that a key repeats into equality.
+  {
+    typedef std::pair<int, char> P;
+    P a[] = {P(1, 'a'), P(1, 'b'), P(1, 'd'), P(2, 'b')};
+    P b[] = {P(1, 'a'), P(1, 'b'), P(1, 'b'), P(1, 'd'), P(2, 'b')};
 
-        std::unordered_multimap<int, char> c1(std::begin(a), std::end(a));
-        std::unordered_multimap<int, char> c2(std::begin(b), std::end(b));
-        assert(testEquality(c1, c2, false));
-    }
+    std::unordered_multimap<int, char> c1(std::begin(a), std::end(a));
+    std::unordered_multimap<int, char> c2(std::begin(b), std::end(b));
+    assert(testEquality(c1, c2, false));
+  }
 
-    // Make sure we incorporate the values into the equality of the maps.
-    // If we were to compare only the keys (including how many time each key repeats),
-    // the following test would fail cause only the values differ.
-    {
-        typedef std::pair<int, char> P;
-        P a[] = {P(1, 'a'), P(1, 'b'), P(1, 'd'), P(2, 'b')};
-        P b[] = {P(1, 'a'), P(1, 'b'), P(1, 'E'), P(2, 'b')};
-        //                                   ^ different here
+  // Make sure we incorporate the values into the equality of the maps.
+  // If we were to compare only the keys (including how many time each key repeats),
+  // the following test would fail cause only the values differ.
+  {
+    typedef std::pair<int, char> P;
+    P a[] = {P(1, 'a'), P(1, 'b'), P(1, 'd'), P(2, 'b')};
+    P b[] = {P(1, 'a'), P(1, 'b'), P(1, 'E'), P(2, 'b')};
+    //                                   ^ different here
 
-        std::unordered_multimap<int, char> c1(std::begin(a), std::end(a));
-        std::unordered_multimap<int, char> c2(std::begin(b), std::end(b));
-        assert(testEquality(c1, c2, false));
-    }
+    std::unordered_multimap<int, char> c1(std::begin(a), std::end(a));
+    std::unordered_multimap<int, char> c2(std::begin(b), std::end(b));
+    assert(testEquality(c1, c2, false));
+  }
 
-    return 0;
+  return 0;
 }
diff --git a/libcxx/test/std/containers/unord/unord.multimap/equal_range.transparent.pass.cpp b/libcxx/test/std/containers/unord/unord.multimap/equal_range.transparent.pass.cpp
index 85248e59d780e..2f47e8d625817 100644
--- a/libcxx/test/std/containers/unord/unord.multimap/equal_range.transparent.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.multimap/equal_range.transparent.pass.cpp
@@ -21,44 +21,43 @@
 
 #include "test_transparent_unordered.h"
 
-int main(int, char**)
-{
-    using key_type = StoredType<int>;
-
-    {
-      // Make sure conversions don't happen for transparent non-final hasher and key_equal
-      using M = unord_map_type<std::unordered_multimap, transparent_hash, std::equal_to<>>;
-      test_transparent_equal_range<M>({{1, 2}, {1, 3}, {2, 3}});
-      test_transparent_equal_range<const M>({{1, 2}, {1, 3}, {2, 3}});
-    }
-
-    {
-      // Make sure conversions don't happen for transparent final hasher and key_equal
-      using M = unord_map_type<std::unordered_multimap, transparent_hash_final, transparent_equal_final>;
-      test_transparent_equal_range<M>({{1, 2}, {1, 3}, {2, 3}});
-      test_transparent_equal_range<const M>({{1, 2}, {1, 3}, {2, 3}});
-    }
-
-    {
-      // Make sure conversions do happen for non-transparent hasher
-      using M = unord_map_type<std::unordered_multimap, non_transparent_hash, std::equal_to<>>;
-      test_non_transparent_equal_range<M>({{1, 2}, {1, 3}, {2, 3}});
-      test_non_transparent_equal_range<const M>({{1, 2}, {1, 3}, {2, 3}});
-    }
-
-    {
-      // Make sure conversions do happen for non-transparent key_equal
-      using M = unord_map_type<std::unordered_multimap, transparent_hash, std::equal_to<key_type>>;
-      test_non_transparent_equal_range<M>({{1, 2}, {1, 3}, {2, 3}});
-      test_non_transparent_equal_range<const M>({{1, 2}, {1, 3}, {2, 3}});
-    }
-
-    {
-      // Make sure conversions do happen for both non-transparent hasher and key_equal
-      using M = unord_map_type<std::unordered_multimap, non_transparent_hash, std::equal_to<key_type>>;
-      test_non_transparent_equal_range<M>({{1, 2}, {1, 3}, {2, 3}});
-      test_non_transparent_equal_range<const M>({{1, 2}, {1, 3}, {2, 3}});
-    }
-
-    return 0;
+int main(int, char**) {
+  using key_type = StoredType<int>;
+
+  {
+    // Make sure conversions don't happen for transparent non-final hasher and key_equal
+    using M = unord_map_type<std::unordered_multimap, transparent_hash, std::equal_to<>>;
+    test_transparent_equal_range<M>({{1, 2}, {1, 3}, {2, 3}});
+    test_transparent_equal_range<const M>({{1, 2}, {1, 3}, {2, 3}});
+  }
+
+  {
+    // Make sure conversions don't happen for transparent final hasher and key_equal
+    using M = unord_map_type<std::unordered_multimap, transparent_hash_final, transparent_equal_final>;
+    test_transparent_equal_range<M>({{1, 2}, {1, 3}, {2, 3}});
+    test_transparent_equal_range<const M>({{1, 2}, {1, 3}, {2, 3}});
+  }
+
+  {
+    // Make sure conversions do happen for non-transparent hasher
+    using M = unord_map_type<std::unordered_multimap, non_transparent_hash, std::equal_to<>>;
+    test_non_transparent_equal_range<M>({{1, 2}, {1, 3}, {2, 3}});
+    test_non_transparent_equal_range<const M>({{1, 2}, {1, 3}, {2, 3}});
+  }
+
+  {
+    // Make sure conversions do happen for non-transparent key_equal
+    using M = unord_map_type<std::unordered_multimap, transparent_hash, std::equal_to<key_type>>;
+    test_non_transparent_equal_range<M>({{1, 2}, {1, 3}, {2, 3}});
+    test_non_transparent_equal_range<const M>({{1, 2}, {1, 3}, {2, 3}});
+  }
+
+  {
+    // Make sure conversions do happen for both non-transparent hasher and key_equal
+    using M = unord_map_type<std::unordered_multimap, non_transparent_hash, std::equal_to<key_type>>;
+    test_non_transparent_equal_range<M>({{1, 2}, {1, 3}, {2, 3}});
+    test_non_transparent_equal_range<const M>({{1, 2}, {1, 3}, {2, 3}});
+  }
+
+  return 0;
 }
diff --git a/libcxx/test/std/containers/unord/unord.multimap/equal_range_const.pass.cpp b/libcxx/test/std/containers/unord/unord.multimap/equal_range_const.pass.cpp
index c827ae5009a67..1c5c5c1d1c876 100644
--- a/libcxx/test/std/containers/unord/unord.multimap/equal_range_const.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.multimap/equal_range_const.pass.cpp
@@ -23,84 +23,83 @@
 #include "test_macros.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef std::unordered_multimap<int, std::string> C;
-        typedef C::const_iterator I;
-        typedef std::pair<int, std::string> P;
-        P a[] =
-        {
-            P(10, "ten"),
-            P(20, "twenty"),
-            P(30, "thirty"),
-            P(40, "forty"),
-            P(50, "fifty"),
-            P(50, "fiftyA"),
-            P(50, "fiftyB"),
-            P(60, "sixty"),
-            P(70, "seventy"),
-            P(80, "eighty"),
-        };
-        const C c(std::begin(a), std::end(a));
-        std::pair<I, I> r = c.equal_range(30);
-        assert(std::distance(r.first, r.second) == 1);
-        assert(r.first->first == 30);
-        assert(r.first->second == "thirty");
-        r = c.equal_range(5);
-        assert(std::distance(r.first, r.second) == 0);
-        r = c.equal_range(50);
-        std::set<std::string> s;
-        s.insert("fifty");
-        s.insert("fiftyA");
-        s.insert("fiftyB");
-        for ( int i = 0; i < 3; ++i )
-        {
-            assert(r.first->first == 50);
-            assert(s.find(r.first->second) != s.end());
-            s.erase(s.find(r.first->second));
-            ++r.first;
-        }
+int main(int, char**) {
+  {
+    typedef std::unordered_multimap<int, std::string> C;
+    typedef C::const_iterator I;
+    typedef std::pair<int, std::string> P;
+    P a[] = {
+        P(10, "ten"),
+        P(20, "twenty"),
+        P(30, "thirty"),
+        P(40, "forty"),
+        P(50, "fifty"),
+        P(50, "fiftyA"),
+        P(50, "fiftyB"),
+        P(60, "sixty"),
+        P(70, "seventy"),
+        P(80, "eighty"),
+    };
+    const C c(std::begin(a), std::end(a));
+    std::pair<I, I> r = c.equal_range(30);
+    assert(std::distance(r.first, r.second) == 1);
+    assert(r.first->first == 30);
+    assert(r.first->second == "thirty");
+    r = c.equal_range(5);
+    assert(std::distance(r.first, r.second) == 0);
+    r = c.equal_range(50);
+    std::set<std::string> s;
+    s.insert("fifty");
+    s.insert("fiftyA");
+    s.insert("fiftyB");
+    for (int i = 0; i < 3; ++i) {
+      assert(r.first->first == 50);
+      assert(s.find(r.first->second) != s.end());
+      s.erase(s.find(r.first->second));
+      ++r.first;
     }
+  }
 #if TEST_STD_VER >= 11
-    {
-        typedef std::unordered_multimap<int, std::string, std::hash<int>, std::equal_to<int>,
-                            min_allocator<std::pair<const int, std::string>>> C;
-        typedef C::const_iterator I;
-        typedef std::pair<int, std::string> P;
-        P a[] =
-        {
-            P(10, "ten"),
-            P(20, "twenty"),
-            P(30, "thirty"),
-            P(40, "forty"),
-            P(50, "fifty"),
-            P(50, "fiftyA"),
-            P(50, "fiftyB"),
-            P(60, "sixty"),
-            P(70, "seventy"),
-            P(80, "eighty"),
-        };
-        const C c(std::begin(a), std::end(a));
-        std::pair<I, I> r = c.equal_range(30);
-        assert(std::distance(r.first, r.second) == 1);
-        assert(r.first->first == 30);
-        assert(r.first->second == "thirty");
-        r = c.equal_range(5);
-        assert(std::distance(r.first, r.second) == 0);
-        r = c.equal_range(50);
-        std::set<std::string> s;
-        s.insert("fifty");
-        s.insert("fiftyA");
-        s.insert("fiftyB");
-        for ( int i = 0; i < 3; ++i )
-        {
-            assert(r.first->first == 50);
-            assert(s.find(r.first->second) != s.end());
-            s.erase(s.find(r.first->second));
-            ++r.first;
-        }
+  {
+    typedef std::unordered_multimap<int,
+                                    std::string,
+                                    std::hash<int>,
+                                    std::equal_to<int>,
+                                    min_allocator<std::pair<const int, std::string>>>
+        C;
+    typedef C::const_iterator I;
+    typedef std::pair<int, std::string> P;
+    P a[] = {
+        P(10, "ten"),
+        P(20, "twenty"),
+        P(30, "thirty"),
+        P(40, "forty"),
+        P(50, "fifty"),
+        P(50, "fiftyA"),
+        P(50, "fiftyB"),
+        P(60, "sixty"),
+        P(70, "seventy"),
+        P(80, "eighty"),
+    };
+    const C c(std::begin(a), std::end(a));
+    std::pair<I, I> r = c.equal_range(30);
+    assert(std::distance(r.first, r.second) == 1);
+    assert(r.first->first == 30);
+    assert(r.first->second == "thirty");
+    r = c.equal_range(5);
+    assert(std::distance(r.first, r.second) == 0);
+    r = c.equal_range(50);
+    std::set<std::string> s;
+    s.insert("fifty");
+    s.insert("fiftyA");
+    s.insert("fiftyB");
+    for (int i = 0; i < 3; ++i) {
+      assert(r.first->first == 50);
+      assert(s.find(r.first->second) != s.end());
+      s.erase(s.find(r.first->second));
+      ++r.first;
     }
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/unord/unord.multimap/equal_range_non_const.pass.cpp b/libcxx/test/std/containers/unord/unord.multimap/equal_range_non_const.pass.cpp
index 6cfa586d5a17e..af2277221c17d 100644
--- a/libcxx/test/std/containers/unord/unord.multimap/equal_range_non_const.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.multimap/equal_range_non_const.pass.cpp
@@ -23,84 +23,83 @@
 #include "test_macros.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef std::unordered_multimap<int, std::string> C;
-        typedef C::iterator I;
-        typedef std::pair<int, std::string> P;
-        P a[] =
-        {
-            P(10, "ten"),
-            P(20, "twenty"),
-            P(30, "thirty"),
-            P(40, "forty"),
-            P(50, "fifty"),
-            P(50, "fiftyA"),
-            P(50, "fiftyB"),
-            P(60, "sixty"),
-            P(70, "seventy"),
-            P(80, "eighty"),
-        };
-        C c(std::begin(a), std::end(a));
-        std::pair<I, I> r = c.equal_range(30);
-        assert(std::distance(r.first, r.second) == 1);
-        assert(r.first->first == 30);
-        assert(r.first->second == "thirty");
-        r = c.equal_range(5);
-        assert(std::distance(r.first, r.second) == 0);
-        r = c.equal_range(50);
-        std::set<std::string> s;
-        s.insert("fifty");
-        s.insert("fiftyA");
-        s.insert("fiftyB");
-        for ( int i = 0; i < 3; ++i )
-        {
-            assert(r.first->first == 50);
-            assert(s.find(r.first->second) != s.end());
-            s.erase(s.find(r.first->second));
-            ++r.first;
-        }
+int main(int, char**) {
+  {
+    typedef std::unordered_multimap<int, std::string> C;
+    typedef C::iterator I;
+    typedef std::pair<int, std::string> P;
+    P a[] = {
+        P(10, "ten"),
+        P(20, "twenty"),
+        P(30, "thirty"),
+        P(40, "forty"),
+        P(50, "fifty"),
+        P(50, "fiftyA"),
+        P(50, "fiftyB"),
+        P(60, "sixty"),
+        P(70, "seventy"),
+        P(80, "eighty"),
+    };
+    C c(std::begin(a), std::end(a));
+    std::pair<I, I> r = c.equal_range(30);
+    assert(std::distance(r.first, r.second) == 1);
+    assert(r.first->first == 30);
+    assert(r.first->second == "thirty");
+    r = c.equal_range(5);
+    assert(std::distance(r.first, r.second) == 0);
+    r = c.equal_range(50);
+    std::set<std::string> s;
+    s.insert("fifty");
+    s.insert("fiftyA");
+    s.insert("fiftyB");
+    for (int i = 0; i < 3; ++i) {
+      assert(r.first->first == 50);
+      assert(s.find(r.first->second) != s.end());
+      s.erase(s.find(r.first->second));
+      ++r.first;
     }
+  }
 #if TEST_STD_VER >= 11
-    {
-        typedef std::unordered_multimap<int, std::string, std::hash<int>, std::equal_to<int>,
-                            min_allocator<std::pair<const int, std::string>>> C;
-        typedef C::iterator I;
-        typedef std::pair<int, std::string> P;
-        P a[] =
-        {
-            P(10, "ten"),
-            P(20, "twenty"),
-            P(30, "thirty"),
-            P(40, "forty"),
-            P(50, "fifty"),
-            P(50, "fiftyA"),
-            P(50, "fiftyB"),
-            P(60, "sixty"),
-            P(70, "seventy"),
-            P(80, "eighty"),
-        };
-        C c(std::begin(a), std::end(a));
-        std::pair<I, I> r = c.equal_range(30);
-        assert(std::distance(r.first, r.second) == 1);
-        assert(r.first->first == 30);
-        assert(r.first->second == "thirty");
-        r = c.equal_range(5);
-        assert(std::distance(r.first, r.second) == 0);
-        r = c.equal_range(50);
-        std::set<std::string> s;
-        s.insert("fifty");
-        s.insert("fiftyA");
-        s.insert("fiftyB");
-        for ( int i = 0; i < 3; ++i )
-        {
-            assert(r.first->first == 50);
-            assert(s.find(r.first->second) != s.end());
-            s.erase(s.find(r.first->second));
-            ++r.first;
-        }
+  {
+    typedef std::unordered_multimap<int,
+                                    std::string,
+                                    std::hash<int>,
+                                    std::equal_to<int>,
+                                    min_allocator<std::pair<const int, std::string>>>
+        C;
+    typedef C::iterator I;
+    typedef std::pair<int, std::string> P;
+    P a[] = {
+        P(10, "ten"),
+        P(20, "twenty"),
+        P(30, "thirty"),
+        P(40, "forty"),
+        P(50, "fifty"),
+        P(50, "fiftyA"),
+        P(50, "fiftyB"),
+        P(60, "sixty"),
+        P(70, "seventy"),
+        P(80, "eighty"),
+    };
+    C c(std::begin(a), std::end(a));
+    std::pair<I, I> r = c.equal_range(30);
+    assert(std::distance(r.first, r.second) == 1);
+    assert(r.first->first == 30);
+    assert(r.first->second == "thirty");
+    r = c.equal_range(5);
+    assert(std::distance(r.first, r.second) == 0);
+    r = c.equal_range(50);
+    std::set<std::string> s;
+    s.insert("fifty");
+    s.insert("fiftyA");
+    s.insert("fiftyB");
+    for (int i = 0; i < 3; ++i) {
+      assert(r.first->first == 50);
+      assert(s.find(r.first->second) != s.end());
+      s.erase(s.find(r.first->second));
+      ++r.first;
     }
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/unord/unord.multimap/erase_if.pass.cpp b/libcxx/test/std/containers/unord/unord.multimap/erase_if.pass.cpp
index 90a6cae088c4f..2f4ef2c010e4c 100644
--- a/libcxx/test/std/containers/unord/unord.multimap/erase_if.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.multimap/erase_if.pass.cpp
@@ -22,12 +22,11 @@
 
 using Init = std::initializer_list<int>;
 template <typename M>
-M make (Init vals)
-{
-    M ret;
-    for (int v : vals)
-        ret.emplace(static_cast<typename M::key_type>(v), static_cast<typename M::mapped_type>(v + 10));
-    return ret;
+M make(Init vals) {
+  M ret;
+  for (int v : vals)
+    ret.emplace(static_cast<typename M::key_type>(v), static_cast<typename M::mapped_type>(v + 10));
+  return ret;
 }
 
 template <typename M, typename Pred>
@@ -40,52 +39,56 @@ void test0(Init vals, Pred p, Init expected, std::size_t expected_erased_count)
 }
 
 template <typename S>
-void test()
-{
-    auto is1 = [](auto v) { return v.first == 1;};
-    auto is2 = [](auto v) { return v.first == 2;};
-    auto is3 = [](auto v) { return v.first == 3;};
-    auto is4 = [](auto v) { return v.first == 4;};
-    auto True  = [](auto) { return true; };
-    auto False = [](auto) { return false; };
-
-    test0<S>({}, is1, {}, 0);
-
-    test0<S>({1}, is1, {}, 1);
-    test0<S>({1}, is2, {1}, 0);
-
-    test0<S>({1, 2}, is1, {2}, 1);
-    test0<S>({1, 2}, is2, {1}, 1);
-    test0<S>({1, 2}, is3, {1, 2}, 0);
-    test0<S>({1, 1}, is1, {}, 2);
-    test0<S>({1, 1}, is3, {1, 1}, 0);
-
-    test0<S>({1, 2, 3}, is1, {2, 3}, 1);
-    test0<S>({1, 2, 3}, is2, {1, 3}, 1);
-    test0<S>({1, 2, 3}, is3, {1, 2}, 1);
-    test0<S>({1, 2, 3}, is4, {1, 2, 3}, 0);
-
-    test0<S>({1, 1, 1}, is1, {}, 3);
-    test0<S>({1, 1, 1}, is2, {1, 1, 1}, 0);
-    test0<S>({1, 1, 2}, is1, {2}, 2);
-    test0<S>({1, 1, 2}, is2, {1, 1}, 1);
-    test0<S>({1, 1, 2}, is3, {1, 1, 2}, 0);
-    test0<S>({1, 2, 2}, is1, {2, 2}, 1);
-    test0<S>({1, 2, 2}, is2, {1}, 2);
-    test0<S>({1, 2, 2}, is3, {1, 2, 2}, 0);
-
-    test0<S>({1, 2, 3}, True, {}, 3);
-    test0<S>({1, 2, 3}, False, {1, 2, 3}, 0);
+void test() {
+  auto is1   = [](auto v) { return v.first == 1; };
+  auto is2   = [](auto v) { return v.first == 2; };
+  auto is3   = [](auto v) { return v.first == 3; };
+  auto is4   = [](auto v) { return v.first == 4; };
+  auto True  = [](auto) { return true; };
+  auto False = [](auto) { return false; };
+
+  test0<S>({}, is1, {}, 0);
+
+  test0<S>({1}, is1, {}, 1);
+  test0<S>({1}, is2, {1}, 0);
+
+  test0<S>({1, 2}, is1, {2}, 1);
+  test0<S>({1, 2}, is2, {1}, 1);
+  test0<S>({1, 2}, is3, {1, 2}, 0);
+  test0<S>({1, 1}, is1, {}, 2);
+  test0<S>({1, 1}, is3, {1, 1}, 0);
+
+  test0<S>({1, 2, 3}, is1, {2, 3}, 1);
+  test0<S>({1, 2, 3}, is2, {1, 3}, 1);
+  test0<S>({1, 2, 3}, is3, {1, 2}, 1);
+  test0<S>({1, 2, 3}, is4, {1, 2, 3}, 0);
+
+  test0<S>({1, 1, 1}, is1, {}, 3);
+  test0<S>({1, 1, 1}, is2, {1, 1, 1}, 0);
+  test0<S>({1, 1, 2}, is1, {2}, 2);
+  test0<S>({1, 1, 2}, is2, {1, 1}, 1);
+  test0<S>({1, 1, 2}, is3, {1, 1, 2}, 0);
+  test0<S>({1, 2, 2}, is1, {2, 2}, 1);
+  test0<S>({1, 2, 2}, is2, {1}, 2);
+  test0<S>({1, 2, 2}, is3, {1, 2, 2}, 0);
+
+  test0<S>({1, 2, 3}, True, {}, 3);
+  test0<S>({1, 2, 3}, False, {1, 2, 3}, 0);
 }
 
-int main(int, char**)
-{
-    test<std::unordered_multimap<int, int>>();
-    test<std::unordered_multimap<int, int, std::hash<int>, std::equal_to<int>, min_allocator<std::pair<const int, int>>>> ();
-    test<std::unordered_multimap<int, int, std::hash<int>, std::equal_to<int>, test_allocator<std::pair<const int, int>>>> ();
-
-    test<std::unordered_multimap<long, short>>();
-    test<std::unordered_multimap<short, double>>();
+int main(int, char**) {
+  test<std::unordered_multimap<int, int>>();
+  test<
+      std::
+          unordered_multimap<int, int, std::hash<int>, std::equal_to<int>, min_allocator<std::pair<const int, int>>>>();
+  test<std::unordered_multimap<int,
+                               int,
+                               std::hash<int>,
+                               std::equal_to<int>,
+                               test_allocator<std::pair<const int, int>>>>();
+
+  test<std::unordered_multimap<long, short>>();
+  test<std::unordered_multimap<short, double>>();
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/unord/unord.multimap/find.transparent.pass.cpp b/libcxx/test/std/containers/unord/unord.multimap/find.transparent.pass.cpp
index 4a32248b13ad6..18351e398d389 100644
--- a/libcxx/test/std/containers/unord/unord.multimap/find.transparent.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.multimap/find.transparent.pass.cpp
@@ -21,44 +21,43 @@
 
 #include "test_transparent_unordered.h"
 
-int main(int, char**)
-{
-    using key_type = StoredType<int>;
-
-    {
-      // Make sure conversions don't happen for transparent non-final hasher and key_equal
-      using M = unord_map_type<std::unordered_multimap, transparent_hash, std::equal_to<>>;
-      test_transparent_find<M>({{1, 2}, {1, 3}, {2, 3}});
-      test_transparent_find<const M>({{1, 2}, {1, 3}, {2, 3}});
-    }
-
-    {
-      // Make sure conversions don't happen for transparent final hasher and key_equal
-      using M = unord_map_type<std::unordered_multimap, transparent_hash_final, transparent_equal_final>;
-      test_transparent_find<M>({{1, 2}, {1, 3}, {2, 3}});
-      test_transparent_find<const M>({{1, 2}, {1, 3}, {2, 3}});
-    }
-
-    {
-      // Make sure conversions do happen for non-transparent hasher
-      using M = unord_map_type<std::unordered_multimap, non_transparent_hash, std::equal_to<>>;
-      test_non_transparent_find<M>({{1, 2}, {1, 3}, {2, 3}});
-      test_non_transparent_find<const M>({{1, 2}, {1, 3}, {2, 3}});
-    }
-
-    {
-      // Make sure conversions do happen for non-transparent key_equal
-      using M = unord_map_type<std::unordered_multimap, transparent_hash, std::equal_to<key_type>>;
-      test_non_transparent_find<M>({{1, 2}, {1, 3}, {2, 3}});
-      test_non_transparent_find<const M>({{1, 2}, {1, 3}, {2, 3}});
-    }
-
-    {
-      // Make sure conversions do happen for both non-transparent hasher and key_equal
-      using M = unord_map_type<std::unordered_multimap, non_transparent_hash, std::equal_to<key_type>>;
-      test_non_transparent_find<M>({{1, 2}, {1, 3}, {2, 3}});
-      test_non_transparent_find<const M>({{1, 2}, {1, 3}, {2, 3}});
-    }
-
-    return 0;
+int main(int, char**) {
+  using key_type = StoredType<int>;
+
+  {
+    // Make sure conversions don't happen for transparent non-final hasher and key_equal
+    using M = unord_map_type<std::unordered_multimap, transparent_hash, std::equal_to<>>;
+    test_transparent_find<M>({{1, 2}, {1, 3}, {2, 3}});
+    test_transparent_find<const M>({{1, 2}, {1, 3}, {2, 3}});
+  }
+
+  {
+    // Make sure conversions don't happen for transparent final hasher and key_equal
+    using M = unord_map_type<std::unordered_multimap, transparent_hash_final, transparent_equal_final>;
+    test_transparent_find<M>({{1, 2}, {1, 3}, {2, 3}});
+    test_transparent_find<const M>({{1, 2}, {1, 3}, {2, 3}});
+  }
+
+  {
+    // Make sure conversions do happen for non-transparent hasher
+    using M = unord_map_type<std::unordered_multimap, non_transparent_hash, std::equal_to<>>;
+    test_non_transparent_find<M>({{1, 2}, {1, 3}, {2, 3}});
+    test_non_transparent_find<const M>({{1, 2}, {1, 3}, {2, 3}});
+  }
+
+  {
+    // Make sure conversions do happen for non-transparent key_equal
+    using M = unord_map_type<std::unordered_multimap, transparent_hash, std::equal_to<key_type>>;
+    test_non_transparent_find<M>({{1, 2}, {1, 3}, {2, 3}});
+    test_non_transparent_find<const M>({{1, 2}, {1, 3}, {2, 3}});
+  }
+
+  {
+    // Make sure conversions do happen for both non-transparent hasher and key_equal
+    using M = unord_map_type<std::unordered_multimap, non_transparent_hash, std::equal_to<key_type>>;
+    test_non_transparent_find<M>({{1, 2}, {1, 3}, {2, 3}});
+    test_non_transparent_find<const M>({{1, 2}, {1, 3}, {2, 3}});
+  }
+
+  return 0;
 }
diff --git a/libcxx/test/std/containers/unord/unord.multimap/find_const.pass.cpp b/libcxx/test/std/containers/unord/unord.multimap/find_const.pass.cpp
index d9ccd868896c7..0da245bc2a943 100644
--- a/libcxx/test/std/containers/unord/unord.multimap/find_const.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.multimap/find_const.pass.cpp
@@ -22,52 +22,53 @@
 #include "test_macros.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef std::unordered_multimap<int, std::string> C;
-        typedef std::pair<int, std::string> P;
-        P a[] =
-        {
-            P(10, "ten"),
-            P(20, "twenty"),
-            P(30, "thirty"),
-            P(40, "forty"),
-            P(50, "fifty"),
-            P(60, "sixty"),
-            P(70, "seventy"),
-            P(80, "eighty"),
-        };
-        const C c(std::begin(a), std::end(a));
-        C::const_iterator i = c.find(30);
-        assert(i->first == 30);
-        assert(i->second == "thirty");
-        i = c.find(5);
-        assert(i == c.cend());
-    }
+int main(int, char**) {
+  {
+    typedef std::unordered_multimap<int, std::string> C;
+    typedef std::pair<int, std::string> P;
+    P a[] = {
+        P(10, "ten"),
+        P(20, "twenty"),
+        P(30, "thirty"),
+        P(40, "forty"),
+        P(50, "fifty"),
+        P(60, "sixty"),
+        P(70, "seventy"),
+        P(80, "eighty"),
+    };
+    const C c(std::begin(a), std::end(a));
+    C::const_iterator i = c.find(30);
+    assert(i->first == 30);
+    assert(i->second == "thirty");
+    i = c.find(5);
+    assert(i == c.cend());
+  }
 #if TEST_STD_VER >= 11
-    {
-        typedef std::unordered_multimap<int, std::string, std::hash<int>, std::equal_to<int>,
-                            min_allocator<std::pair<const int, std::string>>> C;
-        typedef std::pair<int, std::string> P;
-        P a[] =
-        {
-            P(10, "ten"),
-            P(20, "twenty"),
-            P(30, "thirty"),
-            P(40, "forty"),
-            P(50, "fifty"),
-            P(60, "sixty"),
-            P(70, "seventy"),
-            P(80, "eighty"),
-        };
-        const C c(std::begin(a), std::end(a));
-        C::const_iterator i = c.find(30);
-        assert(i->first == 30);
-        assert(i->second == "thirty");
-        i = c.find(5);
-        assert(i == c.cend());
-    }
+  {
+    typedef std::unordered_multimap<int,
+                                    std::string,
+                                    std::hash<int>,
+                                    std::equal_to<int>,
+                                    min_allocator<std::pair<const int, std::string>>>
+        C;
+    typedef std::pair<int, std::string> P;
+    P a[] = {
+        P(10, "ten"),
+        P(20, "twenty"),
+        P(30, "thirty"),
+        P(40, "forty"),
+        P(50, "fifty"),
+        P(60, "sixty"),
+        P(70, "seventy"),
+        P(80, "eighty"),
+    };
+    const C c(std::begin(a), std::end(a));
+    C::const_iterator i = c.find(30);
+    assert(i->first == 30);
+    assert(i->second == "thirty");
+    i = c.find(5);
+    assert(i == c.cend());
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/unord/unord.multimap/find_non_const.pass.cpp b/libcxx/test/std/containers/unord/unord.multimap/find_non_const.pass.cpp
index 5f9b60fdfdc84..9a73008c4e15e 100644
--- a/libcxx/test/std/containers/unord/unord.multimap/find_non_const.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.multimap/find_non_const.pass.cpp
@@ -22,52 +22,53 @@
 #include "test_macros.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef std::unordered_multimap<int, std::string> C;
-        typedef std::pair<int, std::string> P;
-        P a[] =
-        {
-            P(10, "ten"),
-            P(20, "twenty"),
-            P(30, "thirty"),
-            P(40, "forty"),
-            P(50, "fifty"),
-            P(60, "sixty"),
-            P(70, "seventy"),
-            P(80, "eighty"),
-        };
-        C c(std::begin(a), std::end(a));
-        C::iterator i = c.find(30);
-        assert(i->first == 30);
-        assert(i->second == "thirty");
-        i = c.find(5);
-        assert(i == c.end());
-    }
+int main(int, char**) {
+  {
+    typedef std::unordered_multimap<int, std::string> C;
+    typedef std::pair<int, std::string> P;
+    P a[] = {
+        P(10, "ten"),
+        P(20, "twenty"),
+        P(30, "thirty"),
+        P(40, "forty"),
+        P(50, "fifty"),
+        P(60, "sixty"),
+        P(70, "seventy"),
+        P(80, "eighty"),
+    };
+    C c(std::begin(a), std::end(a));
+    C::iterator i = c.find(30);
+    assert(i->first == 30);
+    assert(i->second == "thirty");
+    i = c.find(5);
+    assert(i == c.end());
+  }
 #if TEST_STD_VER >= 11
-    {
-        typedef std::unordered_multimap<int, std::string, std::hash<int>, std::equal_to<int>,
-                            min_allocator<std::pair<const int, std::string>>> C;
-        typedef std::pair<int, std::string> P;
-        P a[] =
-        {
-            P(10, "ten"),
-            P(20, "twenty"),
-            P(30, "thirty"),
-            P(40, "forty"),
-            P(50, "fifty"),
-            P(60, "sixty"),
-            P(70, "seventy"),
-            P(80, "eighty"),
-        };
-        C c(std::begin(a), std::end(a));
-        C::iterator i = c.find(30);
-        assert(i->first == 30);
-        assert(i->second == "thirty");
-        i = c.find(5);
-        assert(i == c.end());
-    }
+  {
+    typedef std::unordered_multimap<int,
+                                    std::string,
+                                    std::hash<int>,
+                                    std::equal_to<int>,
+                                    min_allocator<std::pair<const int, std::string>>>
+        C;
+    typedef std::pair<int, std::string> P;
+    P a[] = {
+        P(10, "ten"),
+        P(20, "twenty"),
+        P(30, "thirty"),
+        P(40, "forty"),
+        P(50, "fifty"),
+        P(60, "sixty"),
+        P(70, "seventy"),
+        P(80, "eighty"),
+    };
+    C c(std::begin(a), std::end(a));
+    C::iterator i = c.find(30);
+    assert(i->first == 30);
+    assert(i->second == "thirty");
+    i = c.find(5);
+    assert(i == c.end());
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/unord/unord.multimap/get_allocator.pass.cpp b/libcxx/test/std/containers/unord/unord.multimap/get_allocator.pass.cpp
index 2cca558949026..0a8d6c4d098eb 100644
--- a/libcxx/test/std/containers/unord/unord.multimap/get_allocator.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.multimap/get_allocator.pass.cpp
@@ -20,19 +20,18 @@
 #include "test_macros.h"
 
 int main(int, char**) {
-    typedef std::pair<const int, std::string> ValueType;
-    {
-        std::allocator<ValueType> alloc;
-        const std::unordered_multimap<int, std::string> m(alloc);
-        assert(m.get_allocator() == alloc);
-    }
-    {
-        other_allocator<ValueType> alloc(1);
-        const std::unordered_multimap<int, std::string, std::hash<int>,
-                                      std::equal_to<int>,
-                                      other_allocator<ValueType> > m(alloc);
-        assert(m.get_allocator() == alloc);
-    }
+  typedef std::pair<const int, std::string> ValueType;
+  {
+    std::allocator<ValueType> alloc;
+    const std::unordered_multimap<int, std::string> m(alloc);
+    assert(m.get_allocator() == alloc);
+  }
+  {
+    other_allocator<ValueType> alloc(1);
+    const std::unordered_multimap<int, std::string, std::hash<int>, std::equal_to<int>, other_allocator<ValueType> > m(
+        alloc);
+    assert(m.get_allocator() == alloc);
+  }
 
-    return 0;
+  return 0;
 }
diff --git a/libcxx/test/std/containers/unord/unord.multimap/incomplete.pass.cpp b/libcxx/test/std/containers/unord/unord.multimap/incomplete.pass.cpp
index 5d76e6f83e989..2cc33030b552c 100644
--- a/libcxx/test/std/containers/unord/unord.multimap/incomplete.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.multimap/incomplete.pass.cpp
@@ -19,22 +19,22 @@
 template <class Tp>
 struct MyHash {
   MyHash() {}
-  std::size_t operator()(Tp const&) const {return 42;}
+  std::size_t operator()(Tp const&) const { return 42; }
 };
 
 struct A {
-    typedef std::unordered_multimap<A, A, MyHash<A> > Map;
-    Map m;
-    Map::iterator it;
-    Map::const_iterator cit;
-    Map::local_iterator lit;
-    Map::const_local_iterator clit;
+  typedef std::unordered_multimap<A, A, MyHash<A> > Map;
+  Map m;
+  Map::iterator it;
+  Map::const_iterator cit;
+  Map::local_iterator lit;
+  Map::const_local_iterator clit;
 };
 
 inline bool operator==(A const& L, A const& R) { return &L == &R; }
 
 int main(int, char**) {
-    A a;
+  A a;
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/unord/unord.multimap/iterator_concept_conformance.compile.pass.cpp b/libcxx/test/std/containers/unord/unord.multimap/iterator_concept_conformance.compile.pass.cpp
index 5b1b25a855ede..b1a969d843ee5 100644
--- a/libcxx/test/std/containers/unord/unord.multimap/iterator_concept_conformance.compile.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.multimap/iterator_concept_conformance.compile.pass.cpp
@@ -16,11 +16,11 @@
 
 #include "test_macros.h"
 
-using iterator = std::unordered_multimap<int, int>::iterator;
-using const_iterator = std::unordered_multimap<int, int>::const_iterator;
-using local_iterator = std::unordered_multimap<int, int>::local_iterator;
+using iterator             = std::unordered_multimap<int, int>::iterator;
+using const_iterator       = std::unordered_multimap<int, int>::const_iterator;
+using local_iterator       = std::unordered_multimap<int, int>::local_iterator;
 using const_local_iterator = std::unordered_multimap<int, int>::const_local_iterator;
-using value_type = std::pair<const int, int>;
+using value_type           = std::pair<const int, int>;
 
 static_assert(std::forward_iterator<iterator>);
 LIBCPP_STATIC_ASSERT(!std::bidirectional_iterator<iterator>);
diff --git a/libcxx/test/std/containers/unord/unord.multimap/iterators.compile.fail.cpp b/libcxx/test/std/containers/unord/unord.multimap/iterators.compile.fail.cpp
index 0c1b50fac059c..9b9c25e39f355 100644
--- a/libcxx/test/std/containers/unord/unord.multimap/iterators.compile.fail.cpp
+++ b/libcxx/test/std/containers/unord/unord.multimap/iterators.compile.fail.cpp
@@ -25,48 +25,45 @@
 
 #include "test_macros.h"
 
-int main(int, char**)
-{
-    {
-        typedef std::unordered_multimap<int, std::string> C;
-        typedef std::pair<int, std::string> P;
-        P a[] =
-        {
-            P(1, "one"),
-            P(2, "two"),
-            P(3, "three"),
-            P(4, "four"),
-            P(1, "four"),
-            P(2, "four"),
-        };
-        C c(a, a + sizeof(a)/sizeof(a[0]));
-        LIBCPP_ASSERT(c.bucket_count() == 7);
-        assert(c.size() == 6);
-        assert(std::distance(c.begin(), c.end()) == c.size());
-        assert(std::distance(c.cbegin(), c.cend()) == c.size());
-        C::iterator i = c.begin();
-        i->second = "ONE";
-        assert(i->second == "ONE");
-        i->first = 2;
-    }
-    {
-        typedef std::unordered_multimap<int, std::string> C;
-        typedef std::pair<int, std::string> P;
-        P a[] =
-        {
-            P(1, "one"),
-            P(2, "two"),
-            P(3, "three"),
-            P(4, "four"),
-            P(1, "four"),
-            P(2, "four"),
-        };
-        const C c(a, a + sizeof(a)/sizeof(a[0]));
-        LIBCPP_ASSERT(c.bucket_count() == 7);
-        assert(c.size() == 6);
-        assert(std::distance(c.begin(), c.end()) == c.size());
-        assert(std::distance(c.cbegin(), c.cend()) == c.size());
-    }
+int main(int, char**) {
+  {
+    typedef std::unordered_multimap<int, std::string> C;
+    typedef std::pair<int, std::string> P;
+    P a[] = {
+        P(1, "one"),
+        P(2, "two"),
+        P(3, "three"),
+        P(4, "four"),
+        P(1, "four"),
+        P(2, "four"),
+    };
+    C c(a, a + sizeof(a) / sizeof(a[0]));
+    LIBCPP_ASSERT(c.bucket_count() == 7);
+    assert(c.size() == 6);
+    assert(std::distance(c.begin(), c.end()) == c.size());
+    assert(std::distance(c.cbegin(), c.cend()) == c.size());
+    C::iterator i = c.begin();
+    i->second     = "ONE";
+    assert(i->second == "ONE");
+    i->first = 2;
+  }
+  {
+    typedef std::unordered_multimap<int, std::string> C;
+    typedef std::pair<int, std::string> P;
+    P a[] = {
+        P(1, "one"),
+        P(2, "two"),
+        P(3, "three"),
+        P(4, "four"),
+        P(1, "four"),
+        P(2, "four"),
+    };
+    const C c(a, a + sizeof(a) / sizeof(a[0]));
+    LIBCPP_ASSERT(c.bucket_count() == 7);
+    assert(c.size() == 6);
+    assert(std::distance(c.begin(), c.end()) == c.size());
+    assert(std::distance(c.cbegin(), c.cend()) == c.size());
+  }
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/unord/unord.multimap/iterators.pass.cpp b/libcxx/test/std/containers/unord/unord.multimap/iterators.pass.cpp
index 5fd52beebbc74..d8ff8e72ad90e 100644
--- a/libcxx/test/std/containers/unord/unord.multimap/iterators.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.multimap/iterators.pass.cpp
@@ -27,110 +27,113 @@
 #include "test_macros.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef std::unordered_multimap<int, std::string> C;
-        typedef std::pair<int, std::string> P;
-        P a[] =
-        {
-            P(1, "one"),
-            P(2, "two"),
-            P(3, "three"),
-            P(4, "four"),
-            P(1, "four"),
-            P(2, "four"),
-        };
-        C c(a, a + sizeof(a)/sizeof(a[0]));
-        assert(c.bucket_count() >= 7);
-        assert(c.size() == 6);
-        assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-        assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
-        C::iterator i;
-        i = c.begin();
-        i->second = "ONE";
-        assert(i->second == "ONE");
-    }
-    {
-        typedef std::unordered_multimap<int, std::string> C;
-        typedef std::pair<int, std::string> P;
-        P a[] =
-        {
-            P(1, "one"),
-            P(2, "two"),
-            P(3, "three"),
-            P(4, "four"),
-            P(1, "four"),
-            P(2, "four"),
-        };
-        const C c(a, a + sizeof(a)/sizeof(a[0]));
-        assert(c.bucket_count() >= 7);
-        assert(c.size() == 6);
-        assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-        assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
-        C::const_iterator i;
-    }
+int main(int, char**) {
+  {
+    typedef std::unordered_multimap<int, std::string> C;
+    typedef std::pair<int, std::string> P;
+    P a[] = {
+        P(1, "one"),
+        P(2, "two"),
+        P(3, "three"),
+        P(4, "four"),
+        P(1, "four"),
+        P(2, "four"),
+    };
+    C c(a, a + sizeof(a) / sizeof(a[0]));
+    assert(c.bucket_count() >= 7);
+    assert(c.size() == 6);
+    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
+    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
+    C::iterator i;
+    i         = c.begin();
+    i->second = "ONE";
+    assert(i->second == "ONE");
+  }
+  {
+    typedef std::unordered_multimap<int, std::string> C;
+    typedef std::pair<int, std::string> P;
+    P a[] = {
+        P(1, "one"),
+        P(2, "two"),
+        P(3, "three"),
+        P(4, "four"),
+        P(1, "four"),
+        P(2, "four"),
+    };
+    const C c(a, a + sizeof(a) / sizeof(a[0]));
+    assert(c.bucket_count() >= 7);
+    assert(c.size() == 6);
+    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
+    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
+    C::const_iterator i;
+  }
 #if TEST_STD_VER >= 11
-    {
-        typedef std::unordered_multimap<int, std::string, std::hash<int>, std::equal_to<int>,
-                            min_allocator<std::pair<const int, std::string>>> C;
-        typedef std::pair<int, std::string> P;
-        P a[] =
-        {
-            P(1, "one"),
-            P(2, "two"),
-            P(3, "three"),
-            P(4, "four"),
-            P(1, "four"),
-            P(2, "four"),
-        };
-        C c(a, a + sizeof(a)/sizeof(a[0]));
-        assert(c.bucket_count() >= 7);
-        assert(c.size() == 6);
-        assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-        assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
-        C::iterator i;
-        i = c.begin();
-        i->second = "ONE";
-        assert(i->second == "ONE");
-    }
-    {
-        typedef std::unordered_multimap<int, std::string, std::hash<int>, std::equal_to<int>,
-                            min_allocator<std::pair<const int, std::string>>> C;
-        typedef std::pair<int, std::string> P;
-        P a[] =
-        {
-            P(1, "one"),
-            P(2, "two"),
-            P(3, "three"),
-            P(4, "four"),
-            P(1, "four"),
-            P(2, "four"),
-        };
-        const C c(a, a + sizeof(a)/sizeof(a[0]));
-        assert(c.bucket_count() >= 7);
-        assert(c.size() == 6);
-        assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-        assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
-        C::const_iterator i;
-    }
+  {
+    typedef std::unordered_multimap<int,
+                                    std::string,
+                                    std::hash<int>,
+                                    std::equal_to<int>,
+                                    min_allocator<std::pair<const int, std::string>>>
+        C;
+    typedef std::pair<int, std::string> P;
+    P a[] = {
+        P(1, "one"),
+        P(2, "two"),
+        P(3, "three"),
+        P(4, "four"),
+        P(1, "four"),
+        P(2, "four"),
+    };
+    C c(a, a + sizeof(a) / sizeof(a[0]));
+    assert(c.bucket_count() >= 7);
+    assert(c.size() == 6);
+    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
+    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
+    C::iterator i;
+    i         = c.begin();
+    i->second = "ONE";
+    assert(i->second == "ONE");
+  }
+  {
+    typedef std::unordered_multimap<int,
+                                    std::string,
+                                    std::hash<int>,
+                                    std::equal_to<int>,
+                                    min_allocator<std::pair<const int, std::string>>>
+        C;
+    typedef std::pair<int, std::string> P;
+    P a[] = {
+        P(1, "one"),
+        P(2, "two"),
+        P(3, "three"),
+        P(4, "four"),
+        P(1, "four"),
+        P(2, "four"),
+    };
+    const C c(a, a + sizeof(a) / sizeof(a[0]));
+    assert(c.bucket_count() >= 7);
+    assert(c.size() == 6);
+    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
+    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
+    C::const_iterator i;
+  }
 #endif
 #if TEST_STD_VER > 11
-    { // N3644 testing
-        typedef std::unordered_multimap<int,double> C;
-        C::iterator ii1{}, ii2{};
-        C::iterator ii4 = ii1;
-        C::const_iterator cii{};
-        assert ( ii1 == ii2 );
-        assert ( ii1 == ii4 );
+  { // N3644 testing
+    typedef std::unordered_multimap<int, double> C;
+    C::iterator ii1{}, ii2{};
+    C::iterator ii4 = ii1;
+    C::const_iterator cii{};
+    assert(ii1 == ii2);
+    assert(ii1 == ii4);
 
-        assert (!(ii1 != ii2 ));
+    assert(!(ii1 != ii2));
 
-        assert ( (ii1 == cii ));
-        assert ( (cii == ii1 ));
-        assert (!(ii1 != cii ));
-        assert (!(cii != ii1 ));
-    }
+    assert((ii1 == cii));
+    assert((cii == ii1));
+    assert(!(ii1 != cii));
+    assert(!(cii != ii1));
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/unord/unord.multimap/load_factor.pass.cpp b/libcxx/test/std/containers/unord/unord.multimap/load_factor.pass.cpp
index 81284edab8d49..6cd2e28b07229 100644
--- a/libcxx/test/std/containers/unord/unord.multimap/load_factor.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.multimap/load_factor.pass.cpp
@@ -24,55 +24,60 @@
 #include "test_macros.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef std::unordered_multimap<int, std::string> C;
-        typedef std::pair<int, std::string> P;
-        P a[] =
-        {
-            P(10, "ten"),
-            P(20, "twenty"),
-            P(30, "thirty"),
-            P(40, "forty"),
-            P(50, "fifty"),
-            P(60, "sixty"),
-            P(70, "seventy"),
-            P(80, "eighty"),
-        };
-        const C c(std::begin(a), std::end(a));
-        assert(std::fabs(c.load_factor() - (float)c.size()/c.bucket_count()) < FLT_EPSILON);
-    }
-    {
-        typedef std::unordered_multimap<int, std::string> C;
-        const C c;
-        assert(c.load_factor() == 0);
-    }
+int main(int, char**) {
+  {
+    typedef std::unordered_multimap<int, std::string> C;
+    typedef std::pair<int, std::string> P;
+    P a[] = {
+        P(10, "ten"),
+        P(20, "twenty"),
+        P(30, "thirty"),
+        P(40, "forty"),
+        P(50, "fifty"),
+        P(60, "sixty"),
+        P(70, "seventy"),
+        P(80, "eighty"),
+    };
+    const C c(std::begin(a), std::end(a));
+    assert(std::fabs(c.load_factor() - (float)c.size() / c.bucket_count()) < FLT_EPSILON);
+  }
+  {
+    typedef std::unordered_multimap<int, std::string> C;
+    const C c;
+    assert(c.load_factor() == 0);
+  }
 #if TEST_STD_VER >= 11
-    {
-        typedef std::unordered_multimap<int, std::string, std::hash<int>, std::equal_to<int>,
-                            min_allocator<std::pair<const int, std::string>>> C;
-        typedef std::pair<int, std::string> P;
-        P a[] =
-        {
-            P(10, "ten"),
-            P(20, "twenty"),
-            P(30, "thirty"),
-            P(40, "forty"),
-            P(50, "fifty"),
-            P(60, "sixty"),
-            P(70, "seventy"),
-            P(80, "eighty"),
-        };
-        const C c(std::begin(a), std::end(a));
-        assert(std::fabs(c.load_factor() - (float)c.size()/c.bucket_count()) < FLT_EPSILON);
-    }
-    {
-        typedef std::unordered_multimap<int, std::string, std::hash<int>, std::equal_to<int>,
-                            min_allocator<std::pair<const int, std::string>>> C;
-        const C c;
-        assert(c.load_factor() == 0);
-    }
+  {
+    typedef std::unordered_multimap<int,
+                                    std::string,
+                                    std::hash<int>,
+                                    std::equal_to<int>,
+                                    min_allocator<std::pair<const int, std::string>>>
+        C;
+    typedef std::pair<int, std::string> P;
+    P a[] = {
+        P(10, "ten"),
+        P(20, "twenty"),
+        P(30, "thirty"),
+        P(40, "forty"),
+        P(50, "fifty"),
+        P(60, "sixty"),
+        P(70, "seventy"),
+        P(80, "eighty"),
+    };
+    const C c(std::begin(a), std::end(a));
+    assert(std::fabs(c.load_factor() - (float)c.size() / c.bucket_count()) < FLT_EPSILON);
+  }
+  {
+    typedef std::unordered_multimap<int,
+                                    std::string,
+                                    std::hash<int>,
+                                    std::equal_to<int>,
+                                    min_allocator<std::pair<const int, std::string>>>
+        C;
+    const C c;
+    assert(c.load_factor() == 0);
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/unord/unord.multimap/local_iterators.compile.fail.cpp b/libcxx/test/std/containers/unord/unord.multimap/local_iterators.compile.fail.cpp
index f5af791804bdd..bcf307a0e2ef1 100644
--- a/libcxx/test/std/containers/unord/unord.multimap/local_iterators.compile.fail.cpp
+++ b/libcxx/test/std/containers/unord/unord.multimap/local_iterators.compile.fail.cpp
@@ -25,265 +25,260 @@
 
 #include "test_macros.h"
 
-int main(int, char**)
-{
-    {
-        typedef std::unordered_multimap<int, std::string> C;
-        typedef std::pair<int, std::string> P;
-        typedef C::local_iterator I;
-        P a[] =
-        {
-            P(1, "one"),
-            P(2, "two"),
-            P(3, "three"),
-            P(4, "four"),
-            P(1, "four"),
-            P(2, "four"),
-        };
-        C c(a, a + sizeof(a)/sizeof(a[0]));
-        LIBCPP_ASSERT(c.bucket_count() == 7);
-        C::size_type b = c.bucket(0);
-        I i = c.begin(b);
-        I j = c.end(b);
-        assert(std::distance(i, j) == 0);
+int main(int, char**) {
+  {
+    typedef std::unordered_multimap<int, std::string> C;
+    typedef std::pair<int, std::string> P;
+    typedef C::local_iterator I;
+    P a[] = {
+        P(1, "one"),
+        P(2, "two"),
+        P(3, "three"),
+        P(4, "four"),
+        P(1, "four"),
+        P(2, "four"),
+    };
+    C c(a, a + sizeof(a) / sizeof(a[0]));
+    LIBCPP_ASSERT(c.bucket_count() == 7);
+    C::size_type b = c.bucket(0);
+    I i            = c.begin(b);
+    I j            = c.end(b);
+    assert(std::distance(i, j) == 0);
 
-        b = c.bucket(1);
-        i = c.begin(b);
-        j = c.end(b);
-        assert(std::distance(i, j) == 2);
-        assert(i->first == 1);
-        assert(i->second == "one");
-        ++i;
-        assert(i->first == 1);
-        assert(i->second == "four");
-        i->first = 2;
+    b = c.bucket(1);
+    i = c.begin(b);
+    j = c.end(b);
+    assert(std::distance(i, j) == 2);
+    assert(i->first == 1);
+    assert(i->second == "one");
+    ++i;
+    assert(i->first == 1);
+    assert(i->second == "four");
+    i->first = 2;
 
-        b = c.bucket(2);
-        i = c.begin(b);
-        j = c.end(b);
-        assert(std::distance(i, j) == 2);
-        assert(i->first == 2);
-        assert(i->second == "two");
-        ++i;
-        assert(i->first == 2);
-        assert(i->second == "four");
+    b = c.bucket(2);
+    i = c.begin(b);
+    j = c.end(b);
+    assert(std::distance(i, j) == 2);
+    assert(i->first == 2);
+    assert(i->second == "two");
+    ++i;
+    assert(i->first == 2);
+    assert(i->second == "four");
 
-        b = c.bucket(3);
-        i = c.begin(b);
-        j = c.end(b);
-        assert(std::distance(i, j) == 1);
-        assert(i->first == 3);
-        assert(i->second == "three");
+    b = c.bucket(3);
+    i = c.begin(b);
+    j = c.end(b);
+    assert(std::distance(i, j) == 1);
+    assert(i->first == 3);
+    assert(i->second == "three");
 
-        b = c.bucket(4);
-        i = c.begin(b);
-        j = c.end(b);
-        assert(std::distance(i, j) == 1);
-        assert(i->first == 4);
-        assert(i->second == "four");
+    b = c.bucket(4);
+    i = c.begin(b);
+    j = c.end(b);
+    assert(std::distance(i, j) == 1);
+    assert(i->first == 4);
+    assert(i->second == "four");
 
-        b = c.bucket(5);
-        i = c.begin(b);
-        j = c.end(b);
-        assert(std::distance(i, j) == 0);
+    b = c.bucket(5);
+    i = c.begin(b);
+    j = c.end(b);
+    assert(std::distance(i, j) == 0);
 
-        b = c.bucket(6);
-        i = c.begin(b);
-        j = c.end(b);
-        assert(std::distance(i, j) == 0);
-    }
-    {
-        typedef std::unordered_multimap<int, std::string> C;
-        typedef std::pair<int, std::string> P;
-        typedef C::const_local_iterator I;
-        P a[] =
-        {
-            P(1, "one"),
-            P(2, "two"),
-            P(3, "three"),
-            P(4, "four"),
-            P(1, "four"),
-            P(2, "four"),
-        };
-        const C c(a, a + sizeof(a)/sizeof(a[0]));
-        LIBCPP_ASSERT(c.bucket_count() == 7);
-        C::size_type b = c.bucket(0);
-        I i = c.begin(b);
-        I j = c.end(b);
-        assert(std::distance(i, j) == 0);
+    b = c.bucket(6);
+    i = c.begin(b);
+    j = c.end(b);
+    assert(std::distance(i, j) == 0);
+  }
+  {
+    typedef std::unordered_multimap<int, std::string> C;
+    typedef std::pair<int, std::string> P;
+    typedef C::const_local_iterator I;
+    P a[] = {
+        P(1, "one"),
+        P(2, "two"),
+        P(3, "three"),
+        P(4, "four"),
+        P(1, "four"),
+        P(2, "four"),
+    };
+    const C c(a, a + sizeof(a) / sizeof(a[0]));
+    LIBCPP_ASSERT(c.bucket_count() == 7);
+    C::size_type b = c.bucket(0);
+    I i            = c.begin(b);
+    I j            = c.end(b);
+    assert(std::distance(i, j) == 0);
 
-        b = c.bucket(1);
-        i = c.begin(b);
-        j = c.end(b);
-        assert(std::distance(i, j) == 2);
-        assert(i->first == 1);
-        assert(i->second == "one");
-        ++i;
-        assert(i->first == 1);
-        assert(i->second == "four");
+    b = c.bucket(1);
+    i = c.begin(b);
+    j = c.end(b);
+    assert(std::distance(i, j) == 2);
+    assert(i->first == 1);
+    assert(i->second == "one");
+    ++i;
+    assert(i->first == 1);
+    assert(i->second == "four");
 
-        b = c.bucket(2);
-        i = c.begin(b);
-        j = c.end(b);
-        assert(std::distance(i, j) == 2);
-        assert(i->first == 2);
-        assert(i->second == "two");
-        ++i;
-        assert(i->first == 2);
-        assert(i->second == "four");
+    b = c.bucket(2);
+    i = c.begin(b);
+    j = c.end(b);
+    assert(std::distance(i, j) == 2);
+    assert(i->first == 2);
+    assert(i->second == "two");
+    ++i;
+    assert(i->first == 2);
+    assert(i->second == "four");
 
-        b = c.bucket(3);
-        i = c.begin(b);
-        j = c.end(b);
-        assert(std::distance(i, j) == 1);
-        assert(i->first == 3);
-        assert(i->second == "three");
+    b = c.bucket(3);
+    i = c.begin(b);
+    j = c.end(b);
+    assert(std::distance(i, j) == 1);
+    assert(i->first == 3);
+    assert(i->second == "three");
 
-        b = c.bucket(4);
-        i = c.begin(b);
-        j = c.end(b);
-        assert(std::distance(i, j) == 1);
-        assert(i->first == 4);
-        assert(i->second == "four");
+    b = c.bucket(4);
+    i = c.begin(b);
+    j = c.end(b);
+    assert(std::distance(i, j) == 1);
+    assert(i->first == 4);
+    assert(i->second == "four");
 
-        b = c.bucket(5);
-        i = c.begin(b);
-        j = c.end(b);
-        assert(std::distance(i, j) == 0);
+    b = c.bucket(5);
+    i = c.begin(b);
+    j = c.end(b);
+    assert(std::distance(i, j) == 0);
 
-        b = c.bucket(6);
-        i = c.begin(b);
-        j = c.end(b);
-        assert(std::distance(i, j) == 0);
-    }
-    {
-        typedef std::unordered_multimap<int, std::string> C;
-        typedef std::pair<int, std::string> P;
-        typedef C::const_local_iterator I;
-        P a[] =
-        {
-            P(1, "one"),
-            P(2, "two"),
-            P(3, "three"),
-            P(4, "four"),
-            P(1, "four"),
-            P(2, "four"),
-        };
-        C c(a, a + sizeof(a)/sizeof(a[0]));
-        LIBCPP_ASSERT(c.bucket_count() == 7);
-        C::size_type b = c.bucket(0);
-        I i = c.cbegin(b);
-        I j = c.cend(b);
-        assert(std::distance(i, j) == 0);
+    b = c.bucket(6);
+    i = c.begin(b);
+    j = c.end(b);
+    assert(std::distance(i, j) == 0);
+  }
+  {
+    typedef std::unordered_multimap<int, std::string> C;
+    typedef std::pair<int, std::string> P;
+    typedef C::const_local_iterator I;
+    P a[] = {
+        P(1, "one"),
+        P(2, "two"),
+        P(3, "three"),
+        P(4, "four"),
+        P(1, "four"),
+        P(2, "four"),
+    };
+    C c(a, a + sizeof(a) / sizeof(a[0]));
+    LIBCPP_ASSERT(c.bucket_count() == 7);
+    C::size_type b = c.bucket(0);
+    I i            = c.cbegin(b);
+    I j            = c.cend(b);
+    assert(std::distance(i, j) == 0);
 
-        b = c.bucket(1);
-        i = c.cbegin(b);
-        j = c.cend(b);
-        assert(std::distance(i, j) == 2);
-        assert(i->first == 1);
-        assert(i->second == "one");
-        ++i;
-        assert(i->first == 1);
-        assert(i->second == "four");
+    b = c.bucket(1);
+    i = c.cbegin(b);
+    j = c.cend(b);
+    assert(std::distance(i, j) == 2);
+    assert(i->first == 1);
+    assert(i->second == "one");
+    ++i;
+    assert(i->first == 1);
+    assert(i->second == "four");
 
-        b = c.bucket(2);
-        i = c.cbegin(b);
-        j = c.cend(b);
-        assert(std::distance(i, j) == 2);
-        assert(i->first == 2);
-        assert(i->second == "two");
-        ++i;
-        assert(i->first == 2);
-        assert(i->second == "four");
+    b = c.bucket(2);
+    i = c.cbegin(b);
+    j = c.cend(b);
+    assert(std::distance(i, j) == 2);
+    assert(i->first == 2);
+    assert(i->second == "two");
+    ++i;
+    assert(i->first == 2);
+    assert(i->second == "four");
 
-        b = c.bucket(3);
-        i = c.cbegin(b);
-        j = c.cend(b);
-        assert(std::distance(i, j) == 1);
-        assert(i->first == 3);
-        assert(i->second == "three");
+    b = c.bucket(3);
+    i = c.cbegin(b);
+    j = c.cend(b);
+    assert(std::distance(i, j) == 1);
+    assert(i->first == 3);
+    assert(i->second == "three");
 
-        b = c.bucket(4);
-        i = c.cbegin(b);
-        j = c.cend(b);
-        assert(std::distance(i, j) == 1);
-        assert(i->first == 4);
-        assert(i->second == "four");
+    b = c.bucket(4);
+    i = c.cbegin(b);
+    j = c.cend(b);
+    assert(std::distance(i, j) == 1);
+    assert(i->first == 4);
+    assert(i->second == "four");
 
-        b = c.bucket(5);
-        i = c.cbegin(b);
-        j = c.cend(b);
-        assert(std::distance(i, j) == 0);
+    b = c.bucket(5);
+    i = c.cbegin(b);
+    j = c.cend(b);
+    assert(std::distance(i, j) == 0);
 
-        b = c.bucket(6);
-        i = c.cbegin(b);
-        j = c.cend(b);
-        assert(std::distance(i, j) == 0);
-    }
-    {
-        typedef std::unordered_multimap<int, std::string> C;
-        typedef std::pair<int, std::string> P;
-        typedef C::const_local_iterator I;
-        P a[] =
-        {
-            P(1, "one"),
-            P(2, "two"),
-            P(3, "three"),
-            P(4, "four"),
-            P(1, "four"),
-            P(2, "four"),
-        };
-        const C c(a, a + sizeof(a)/sizeof(a[0]));
-        LIBCPP_ASSERT(c.bucket_count() == 7);
-        C::size_type b = c.bucket(0);
-        I i = c.cbegin(b);
-        I j = c.cend(b);
-        assert(std::distance(i, j) == 0);
+    b = c.bucket(6);
+    i = c.cbegin(b);
+    j = c.cend(b);
+    assert(std::distance(i, j) == 0);
+  }
+  {
+    typedef std::unordered_multimap<int, std::string> C;
+    typedef std::pair<int, std::string> P;
+    typedef C::const_local_iterator I;
+    P a[] = {
+        P(1, "one"),
+        P(2, "two"),
+        P(3, "three"),
+        P(4, "four"),
+        P(1, "four"),
+        P(2, "four"),
+    };
+    const C c(a, a + sizeof(a) / sizeof(a[0]));
+    LIBCPP_ASSERT(c.bucket_count() == 7);
+    C::size_type b = c.bucket(0);
+    I i            = c.cbegin(b);
+    I j            = c.cend(b);
+    assert(std::distance(i, j) == 0);
 
-        b = c.bucket(1);
-        i = c.cbegin(b);
-        j = c.cend(b);
-        assert(std::distance(i, j) == 2);
-        assert(i->first == 1);
-        assert(i->second == "one");
-        ++i;
-        assert(i->first == 1);
-        assert(i->second == "four");
+    b = c.bucket(1);
+    i = c.cbegin(b);
+    j = c.cend(b);
+    assert(std::distance(i, j) == 2);
+    assert(i->first == 1);
+    assert(i->second == "one");
+    ++i;
+    assert(i->first == 1);
+    assert(i->second == "four");
 
-        b = c.bucket(2);
-        i = c.cbegin(b);
-        j = c.cend(b);
-        assert(std::distance(i, j) == 2);
-        assert(i->first == 2);
-        assert(i->second == "two");
-        ++i;
-        assert(i->first == 2);
-        assert(i->second == "four");
+    b = c.bucket(2);
+    i = c.cbegin(b);
+    j = c.cend(b);
+    assert(std::distance(i, j) == 2);
+    assert(i->first == 2);
+    assert(i->second == "two");
+    ++i;
+    assert(i->first == 2);
+    assert(i->second == "four");
 
-        b = c.bucket(3);
-        i = c.cbegin(b);
-        j = c.cend(b);
-        assert(std::distance(i, j) == 1);
-        assert(i->first == 3);
-        assert(i->second == "three");
+    b = c.bucket(3);
+    i = c.cbegin(b);
+    j = c.cend(b);
+    assert(std::distance(i, j) == 1);
+    assert(i->first == 3);
+    assert(i->second == "three");
 
-        b = c.bucket(4);
-        i = c.cbegin(b);
-        j = c.cend(b);
-        assert(std::distance(i, j) == 1);
-        assert(i->first == 4);
-        assert(i->second == "four");
+    b = c.bucket(4);
+    i = c.cbegin(b);
+    j = c.cend(b);
+    assert(std::distance(i, j) == 1);
+    assert(i->first == 4);
+    assert(i->second == "four");
 
-        b = c.bucket(5);
-        i = c.cbegin(b);
-        j = c.cend(b);
-        assert(std::distance(i, j) == 0);
+    b = c.bucket(5);
+    i = c.cbegin(b);
+    j = c.cend(b);
+    assert(std::distance(i, j) == 0);
 
-        b = c.bucket(6);
-        i = c.cbegin(b);
-        j = c.cend(b);
-        assert(std::distance(i, j) == 0);
-    }
+    b = c.bucket(6);
+    i = c.cbegin(b);
+    j = c.cend(b);
+    assert(std::distance(i, j) == 0);
+  }
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/unord/unord.multimap/local_iterators.pass.cpp b/libcxx/test/std/containers/unord/unord.multimap/local_iterators.pass.cpp
index 12b11aac8face..487c525d370c5 100644
--- a/libcxx/test/std/containers/unord/unord.multimap/local_iterators.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.multimap/local_iterators.pass.cpp
@@ -27,637 +27,628 @@
 #include "test_macros.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
+int main(int, char**) {
+  {
+    typedef std::unordered_multimap<int, std::string> C;
+    typedef std::pair<int, std::string> P;
+    typedef C::local_iterator I;
+    P a[] = {
+        P(1, "one"),
+        P(2, "two"),
+        P(3, "three"),
+        P(4, "four"),
+        P(1, "four"),
+        P(2, "four"),
+    };
+    C c(a, a + sizeof(a) / sizeof(a[0]));
+    assert(c.bucket_count() >= 7);
+    C::size_type b = c.bucket(0);
+    I i            = c.begin(b);
+    I j            = c.end(b);
+    assert(std::distance(i, j) == 0);
+
+    b = c.bucket(1);
+    i = c.begin(b);
+    j = c.end(b);
+    assert(std::distance(i, j) == 2);
     {
-        typedef std::unordered_multimap<int, std::string> C;
-        typedef std::pair<int, std::string> P;
-        typedef C::local_iterator I;
-        P a[] =
-        {
-            P(1, "one"),
-            P(2, "two"),
-            P(3, "three"),
-            P(4, "four"),
-            P(1, "four"),
-            P(2, "four"),
-        };
-        C c(a, a + sizeof(a)/sizeof(a[0]));
-        assert(c.bucket_count() >= 7);
-        C::size_type b = c.bucket(0);
-        I i = c.begin(b);
-        I j = c.end(b);
-        assert(std::distance(i, j) == 0);
-
-        b = c.bucket(1);
-        i = c.begin(b);
-        j = c.end(b);
-        assert(std::distance(i, j) == 2);
-        {
-            std::set<std::string> s;
-            s.insert("one");
-            s.insert("four");
-            for ( int n = 0; n < 2; ++n )
-            {
-                assert(i->first == 1);
-                assert(s.find(i->second) != s.end());
-                s.erase(s.find(i->second));
-                ++i;
-            }
-        }
-
-        b = c.bucket(2);
-        i = c.begin(b);
-        j = c.end(b);
-        assert(std::distance(i, j) == 2);
-        {
-            std::set<std::string> s;
-            s.insert("two");
-            s.insert("four");
-            for ( int n = 0; n < 2; ++n )
-            {
-                assert(i->first == 2);
-                assert(s.find(i->second) != s.end());
-                s.erase(s.find(i->second));
-                ++i;
-            }
-        }
-
-        b = c.bucket(3);
-        i = c.begin(b);
-        j = c.end(b);
-        assert(std::distance(i, j) == 1);
-        assert(i->first == 3);
-        assert(i->second == "three");
-
-        b = c.bucket(4);
-        i = c.begin(b);
-        j = c.end(b);
-        assert(std::distance(i, j) == 1);
-        assert(i->first == 4);
-        assert(i->second == "four");
-
-        b = c.bucket(5);
-        i = c.begin(b);
-        j = c.end(b);
-        assert(std::distance(i, j) == 0);
-
-        b = c.bucket(6);
-        i = c.begin(b);
-        j = c.end(b);
-        assert(std::distance(i, j) == 0);
+      std::set<std::string> s;
+      s.insert("one");
+      s.insert("four");
+      for (int n = 0; n < 2; ++n) {
+        assert(i->first == 1);
+        assert(s.find(i->second) != s.end());
+        s.erase(s.find(i->second));
+        ++i;
+      }
     }
+
+    b = c.bucket(2);
+    i = c.begin(b);
+    j = c.end(b);
+    assert(std::distance(i, j) == 2);
+    {
+      std::set<std::string> s;
+      s.insert("two");
+      s.insert("four");
+      for (int n = 0; n < 2; ++n) {
+        assert(i->first == 2);
+        assert(s.find(i->second) != s.end());
+        s.erase(s.find(i->second));
+        ++i;
+      }
+    }
+
+    b = c.bucket(3);
+    i = c.begin(b);
+    j = c.end(b);
+    assert(std::distance(i, j) == 1);
+    assert(i->first == 3);
+    assert(i->second == "three");
+
+    b = c.bucket(4);
+    i = c.begin(b);
+    j = c.end(b);
+    assert(std::distance(i, j) == 1);
+    assert(i->first == 4);
+    assert(i->second == "four");
+
+    b = c.bucket(5);
+    i = c.begin(b);
+    j = c.end(b);
+    assert(std::distance(i, j) == 0);
+
+    b = c.bucket(6);
+    i = c.begin(b);
+    j = c.end(b);
+    assert(std::distance(i, j) == 0);
+  }
+  {
+    typedef std::unordered_multimap<int, std::string> C;
+    typedef std::pair<int, std::string> P;
+    typedef C::const_local_iterator I;
+    P a[] = {
+        P(1, "one"),
+        P(2, "two"),
+        P(3, "three"),
+        P(4, "four"),
+        P(1, "four"),
+        P(2, "four"),
+    };
+    const C c(a, a + sizeof(a) / sizeof(a[0]));
+    assert(c.bucket_count() >= 7);
+    C::size_type b = c.bucket(0);
+    I i            = c.begin(b);
+    I j            = c.end(b);
+    assert(std::distance(i, j) == 0);
+
+    b = c.bucket(1);
+    i = c.begin(b);
+    j = c.end(b);
+    assert(std::distance(i, j) == 2);
+    {
+      std::set<std::string> s;
+      s.insert("one");
+      s.insert("four");
+      for (int n = 0; n < 2; ++n) {
+        assert(i->first == 1);
+        assert(s.find(i->second) != s.end());
+        s.erase(s.find(i->second));
+        ++i;
+      }
+    }
+
+    b = c.bucket(2);
+    i = c.begin(b);
+    j = c.end(b);
+    assert(std::distance(i, j) == 2);
+    {
+      std::set<std::string> s;
+      s.insert("two");
+      s.insert("four");
+      for (int n = 0; n < 2; ++n) {
+        assert(i->first == 2);
+        assert(s.find(i->second) != s.end());
+        s.erase(s.find(i->second));
+        ++i;
+      }
+    }
+
+    b = c.bucket(3);
+    i = c.begin(b);
+    j = c.end(b);
+    assert(std::distance(i, j) == 1);
+    assert(i->first == 3);
+    assert(i->second == "three");
+
+    b = c.bucket(4);
+    i = c.begin(b);
+    j = c.end(b);
+    assert(std::distance(i, j) == 1);
+    assert(i->first == 4);
+    assert(i->second == "four");
+
+    b = c.bucket(5);
+    i = c.begin(b);
+    j = c.end(b);
+    assert(std::distance(i, j) == 0);
+
+    b = c.bucket(6);
+    i = c.begin(b);
+    j = c.end(b);
+    assert(std::distance(i, j) == 0);
+  }
+  {
+    typedef std::unordered_multimap<int, std::string> C;
+    typedef std::pair<int, std::string> P;
+    typedef C::const_local_iterator I;
+    P a[] = {
+        P(1, "one"),
+        P(2, "two"),
+        P(3, "three"),
+        P(4, "four"),
+        P(1, "four"),
+        P(2, "four"),
+    };
+    C c(a, a + sizeof(a) / sizeof(a[0]));
+    assert(c.bucket_count() >= 7);
+    C::size_type b = c.bucket(0);
+    I i            = c.cbegin(b);
+    I j            = c.cend(b);
+    assert(std::distance(i, j) == 0);
+
+    b = c.bucket(1);
+    i = c.cbegin(b);
+    j = c.cend(b);
+    assert(std::distance(i, j) == 2);
+    {
+      std::set<std::string> s;
+      s.insert("one");
+      s.insert("four");
+      for (int n = 0; n < 2; ++n) {
+        assert(i->first == 1);
+        assert(s.find(i->second) != s.end());
+        s.erase(s.find(i->second));
+        ++i;
+      }
+    }
+
+    b = c.bucket(2);
+    i = c.cbegin(b);
+    j = c.cend(b);
+    assert(std::distance(i, j) == 2);
     {
-        typedef std::unordered_multimap<int, std::string> C;
-        typedef std::pair<int, std::string> P;
-        typedef C::const_local_iterator I;
-        P a[] =
-        {
-            P(1, "one"),
-            P(2, "two"),
-            P(3, "three"),
-            P(4, "four"),
-            P(1, "four"),
-            P(2, "four"),
-        };
-        const C c(a, a + sizeof(a)/sizeof(a[0]));
-        assert(c.bucket_count() >= 7);
-        C::size_type b = c.bucket(0);
-        I i = c.begin(b);
-        I j = c.end(b);
-        assert(std::distance(i, j) == 0);
-
-        b = c.bucket(1);
-        i = c.begin(b);
-        j = c.end(b);
-        assert(std::distance(i, j) == 2);
-        {
-            std::set<std::string> s;
-            s.insert("one");
-            s.insert("four");
-            for ( int n = 0; n < 2; ++n )
-            {
-                assert(i->first == 1);
-                assert(s.find(i->second) != s.end());
-                s.erase(s.find(i->second));
-                ++i;
-            }
-        }
-
-        b = c.bucket(2);
-        i = c.begin(b);
-        j = c.end(b);
-        assert(std::distance(i, j) == 2);
-        {
-            std::set<std::string> s;
-            s.insert("two");
-            s.insert("four");
-            for ( int n = 0; n < 2; ++n )
-            {
-                assert(i->first == 2);
-                assert(s.find(i->second) != s.end());
-                s.erase(s.find(i->second));
-                ++i;
-            }
-        }
-
-        b = c.bucket(3);
-        i = c.begin(b);
-        j = c.end(b);
-        assert(std::distance(i, j) == 1);
-        assert(i->first == 3);
-        assert(i->second == "three");
-
-        b = c.bucket(4);
-        i = c.begin(b);
-        j = c.end(b);
-        assert(std::distance(i, j) == 1);
-        assert(i->first == 4);
-        assert(i->second == "four");
-
-        b = c.bucket(5);
-        i = c.begin(b);
-        j = c.end(b);
-        assert(std::distance(i, j) == 0);
-
-        b = c.bucket(6);
-        i = c.begin(b);
-        j = c.end(b);
-        assert(std::distance(i, j) == 0);
+      std::set<std::string> s;
+      s.insert("two");
+      s.insert("four");
+      for (int n = 0; n < 2; ++n) {
+        assert(i->first == 2);
+        assert(s.find(i->second) != s.end());
+        s.erase(s.find(i->second));
+        ++i;
+      }
     }
+
+    b = c.bucket(3);
+    i = c.cbegin(b);
+    j = c.cend(b);
+    assert(std::distance(i, j) == 1);
+    assert(i->first == 3);
+    assert(i->second == "three");
+
+    b = c.bucket(4);
+    i = c.cbegin(b);
+    j = c.cend(b);
+    assert(std::distance(i, j) == 1);
+    assert(i->first == 4);
+    assert(i->second == "four");
+
+    b = c.bucket(5);
+    i = c.cbegin(b);
+    j = c.cend(b);
+    assert(std::distance(i, j) == 0);
+
+    b = c.bucket(6);
+    i = c.cbegin(b);
+    j = c.cend(b);
+    assert(std::distance(i, j) == 0);
+  }
+  {
+    typedef std::unordered_multimap<int, std::string> C;
+    typedef std::pair<int, std::string> P;
+    typedef C::const_local_iterator I;
+    P a[] = {
+        P(1, "one"),
+        P(2, "two"),
+        P(3, "three"),
+        P(4, "four"),
+        P(1, "four"),
+        P(2, "four"),
+    };
+    const C c(a, a + sizeof(a) / sizeof(a[0]));
+    assert(c.bucket_count() >= 7);
+    C::size_type b = c.bucket(0);
+    I i            = c.cbegin(b);
+    I j            = c.cend(b);
+    assert(std::distance(i, j) == 0);
+
+    b = c.bucket(1);
+    i = c.cbegin(b);
+    j = c.cend(b);
+    assert(std::distance(i, j) == 2);
     {
-        typedef std::unordered_multimap<int, std::string> C;
-        typedef std::pair<int, std::string> P;
-        typedef C::const_local_iterator I;
-        P a[] =
-        {
-            P(1, "one"),
-            P(2, "two"),
-            P(3, "three"),
-            P(4, "four"),
-            P(1, "four"),
-            P(2, "four"),
-        };
-        C c(a, a + sizeof(a)/sizeof(a[0]));
-        assert(c.bucket_count() >= 7);
-        C::size_type b = c.bucket(0);
-        I i = c.cbegin(b);
-        I j = c.cend(b);
-        assert(std::distance(i, j) == 0);
-
-        b = c.bucket(1);
-        i = c.cbegin(b);
-        j = c.cend(b);
-        assert(std::distance(i, j) == 2);
-        {
-            std::set<std::string> s;
-            s.insert("one");
-            s.insert("four");
-            for ( int n = 0; n < 2; ++n )
-            {
-                assert(i->first == 1);
-                assert(s.find(i->second) != s.end());
-                s.erase(s.find(i->second));
-                ++i;
-            }
-        }
-
-        b = c.bucket(2);
-        i = c.cbegin(b);
-        j = c.cend(b);
-        assert(std::distance(i, j) == 2);
-        {
-            std::set<std::string> s;
-            s.insert("two");
-            s.insert("four");
-            for ( int n = 0; n < 2; ++n )
-            {
-                assert(i->first == 2);
-                assert(s.find(i->second) != s.end());
-                s.erase(s.find(i->second));
-                ++i;
-            }
-        }
-
-        b = c.bucket(3);
-        i = c.cbegin(b);
-        j = c.cend(b);
-        assert(std::distance(i, j) == 1);
-        assert(i->first == 3);
-        assert(i->second == "three");
-
-        b = c.bucket(4);
-        i = c.cbegin(b);
-        j = c.cend(b);
-        assert(std::distance(i, j) == 1);
-        assert(i->first == 4);
-        assert(i->second == "four");
-
-        b = c.bucket(5);
-        i = c.cbegin(b);
-        j = c.cend(b);
-        assert(std::distance(i, j) == 0);
-
-        b = c.bucket(6);
-        i = c.cbegin(b);
-        j = c.cend(b);
-        assert(std::distance(i, j) == 0);
+      std::set<std::string> s;
+      s.insert("one");
+      s.insert("four");
+      for (int n = 0; n < 2; ++n) {
+        assert(i->first == 1);
+        assert(s.find(i->second) != s.end());
+        s.erase(s.find(i->second));
+        ++i;
+      }
     }
+
+    b = c.bucket(2);
+    i = c.cbegin(b);
+    j = c.cend(b);
+    assert(std::distance(i, j) == 2);
     {
-        typedef std::unordered_multimap<int, std::string> C;
-        typedef std::pair<int, std::string> P;
-        typedef C::const_local_iterator I;
-        P a[] =
-        {
-            P(1, "one"),
-            P(2, "two"),
-            P(3, "three"),
-            P(4, "four"),
-            P(1, "four"),
-            P(2, "four"),
-        };
-        const C c(a, a + sizeof(a)/sizeof(a[0]));
-        assert(c.bucket_count() >= 7);
-        C::size_type b = c.bucket(0);
-        I i = c.cbegin(b);
-        I j = c.cend(b);
-        assert(std::distance(i, j) == 0);
-
-        b = c.bucket(1);
-        i = c.cbegin(b);
-        j = c.cend(b);
-        assert(std::distance(i, j) == 2);
-        {
-            std::set<std::string> s;
-            s.insert("one");
-            s.insert("four");
-            for ( int n = 0; n < 2; ++n )
-            {
-                assert(i->first == 1);
-                assert(s.find(i->second) != s.end());
-                s.erase(s.find(i->second));
-                ++i;
-            }
-        }
-
-        b = c.bucket(2);
-        i = c.cbegin(b);
-        j = c.cend(b);
-        assert(std::distance(i, j) == 2);
-        {
-            std::set<std::string> s;
-            s.insert("two");
-            s.insert("four");
-            for ( int n = 0; n < 2; ++n )
-            {
-                assert(i->first == 2);
-                assert(s.find(i->second) != s.end());
-                s.erase(s.find(i->second));
-                ++i;
-            }
-        }
-
-        b = c.bucket(3);
-        i = c.cbegin(b);
-        j = c.cend(b);
-        assert(std::distance(i, j) == 1);
-        assert(i->first == 3);
-        assert(i->second == "three");
-
-        b = c.bucket(4);
-        i = c.cbegin(b);
-        j = c.cend(b);
-        assert(std::distance(i, j) == 1);
-        assert(i->first == 4);
-        assert(i->second == "four");
-
-        b = c.bucket(5);
-        i = c.cbegin(b);
-        j = c.cend(b);
-        assert(std::distance(i, j) == 0);
-
-        b = c.bucket(6);
-        i = c.cbegin(b);
-        j = c.cend(b);
-        assert(std::distance(i, j) == 0);
+      std::set<std::string> s;
+      s.insert("two");
+      s.insert("four");
+      for (int n = 0; n < 2; ++n) {
+        assert(i->first == 2);
+        assert(s.find(i->second) != s.end());
+        s.erase(s.find(i->second));
+        ++i;
+      }
     }
+
+    b = c.bucket(3);
+    i = c.cbegin(b);
+    j = c.cend(b);
+    assert(std::distance(i, j) == 1);
+    assert(i->first == 3);
+    assert(i->second == "three");
+
+    b = c.bucket(4);
+    i = c.cbegin(b);
+    j = c.cend(b);
+    assert(std::distance(i, j) == 1);
+    assert(i->first == 4);
+    assert(i->second == "four");
+
+    b = c.bucket(5);
+    i = c.cbegin(b);
+    j = c.cend(b);
+    assert(std::distance(i, j) == 0);
+
+    b = c.bucket(6);
+    i = c.cbegin(b);
+    j = c.cend(b);
+    assert(std::distance(i, j) == 0);
+  }
 #if TEST_STD_VER >= 11
+  {
+    typedef std::unordered_multimap<int,
+                                    std::string,
+                                    std::hash<int>,
+                                    std::equal_to<int>,
+                                    min_allocator<std::pair<const int, std::string>>>
+        C;
+    typedef std::pair<int, std::string> P;
+    typedef C::local_iterator I;
+    P a[] = {
+        P(1, "one"),
+        P(2, "two"),
+        P(3, "three"),
+        P(4, "four"),
+        P(1, "four"),
+        P(2, "four"),
+    };
+    C c(a, a + sizeof(a) / sizeof(a[0]));
+    assert(c.bucket_count() >= 7);
+    C::size_type b = c.bucket(0);
+    I i            = c.begin(b);
+    I j            = c.end(b);
+    assert(std::distance(i, j) == 0);
+
+    b = c.bucket(1);
+    i = c.begin(b);
+    j = c.end(b);
+    assert(std::distance(i, j) == 2);
     {
-        typedef std::unordered_multimap<int, std::string, std::hash<int>, std::equal_to<int>,
-                            min_allocator<std::pair<const int, std::string>>> C;
-        typedef std::pair<int, std::string> P;
-        typedef C::local_iterator I;
-        P a[] =
-        {
-            P(1, "one"),
-            P(2, "two"),
-            P(3, "three"),
-            P(4, "four"),
-            P(1, "four"),
-            P(2, "four"),
-        };
-        C c(a, a + sizeof(a)/sizeof(a[0]));
-        assert(c.bucket_count() >= 7);
-        C::size_type b = c.bucket(0);
-        I i = c.begin(b);
-        I j = c.end(b);
-        assert(std::distance(i, j) == 0);
-
-        b = c.bucket(1);
-        i = c.begin(b);
-        j = c.end(b);
-        assert(std::distance(i, j) == 2);
-        {
-            std::set<std::string> s;
-            s.insert("one");
-            s.insert("four");
-            for ( int n = 0; n < 2; ++n )
-            {
-                assert(i->first == 1);
-                assert(s.find(i->second) != s.end());
-                s.erase(s.find(i->second));
-                ++i;
-            }
-        }
-
-        b = c.bucket(2);
-        i = c.begin(b);
-        j = c.end(b);
-        assert(std::distance(i, j) == 2);
-        {
-            std::set<std::string> s;
-            s.insert("two");
-            s.insert("four");
-            for ( int n = 0; n < 2; ++n )
-            {
-                assert(i->first == 2);
-                assert(s.find(i->second) != s.end());
-                s.erase(s.find(i->second));
-                ++i;
-            }
-        }
-
-        b = c.bucket(3);
-        i = c.begin(b);
-        j = c.end(b);
-        assert(std::distance(i, j) == 1);
-        assert(i->first == 3);
-        assert(i->second == "three");
-
-        b = c.bucket(4);
-        i = c.begin(b);
-        j = c.end(b);
-        assert(std::distance(i, j) == 1);
-        assert(i->first == 4);
-        assert(i->second == "four");
-
-        b = c.bucket(5);
-        i = c.begin(b);
-        j = c.end(b);
-        assert(std::distance(i, j) == 0);
-
-        b = c.bucket(6);
-        i = c.begin(b);
-        j = c.end(b);
-        assert(std::distance(i, j) == 0);
+      std::set<std::string> s;
+      s.insert("one");
+      s.insert("four");
+      for (int n = 0; n < 2; ++n) {
+        assert(i->first == 1);
+        assert(s.find(i->second) != s.end());
+        s.erase(s.find(i->second));
+        ++i;
+      }
     }
+
+    b = c.bucket(2);
+    i = c.begin(b);
+    j = c.end(b);
+    assert(std::distance(i, j) == 2);
+    {
+      std::set<std::string> s;
+      s.insert("two");
+      s.insert("four");
+      for (int n = 0; n < 2; ++n) {
+        assert(i->first == 2);
+        assert(s.find(i->second) != s.end());
+        s.erase(s.find(i->second));
+        ++i;
+      }
+    }
+
+    b = c.bucket(3);
+    i = c.begin(b);
+    j = c.end(b);
+    assert(std::distance(i, j) == 1);
+    assert(i->first == 3);
+    assert(i->second == "three");
+
+    b = c.bucket(4);
+    i = c.begin(b);
+    j = c.end(b);
+    assert(std::distance(i, j) == 1);
+    assert(i->first == 4);
+    assert(i->second == "four");
+
+    b = c.bucket(5);
+    i = c.begin(b);
+    j = c.end(b);
+    assert(std::distance(i, j) == 0);
+
+    b = c.bucket(6);
+    i = c.begin(b);
+    j = c.end(b);
+    assert(std::distance(i, j) == 0);
+  }
+  {
+    typedef std::unordered_multimap<int,
+                                    std::string,
+                                    std::hash<int>,
+                                    std::equal_to<int>,
+                                    min_allocator<std::pair<const int, std::string>>>
+        C;
+    typedef std::pair<int, std::string> P;
+    typedef C::const_local_iterator I;
+    P a[] = {
+        P(1, "one"),
+        P(2, "two"),
+        P(3, "three"),
+        P(4, "four"),
+        P(1, "four"),
+        P(2, "four"),
+    };
+    const C c(a, a + sizeof(a) / sizeof(a[0]));
+    assert(c.bucket_count() >= 7);
+    C::size_type b = c.bucket(0);
+    I i            = c.begin(b);
+    I j            = c.end(b);
+    assert(std::distance(i, j) == 0);
+
+    b = c.bucket(1);
+    i = c.begin(b);
+    j = c.end(b);
+    assert(std::distance(i, j) == 2);
+    {
+      std::set<std::string> s;
+      s.insert("one");
+      s.insert("four");
+      for (int n = 0; n < 2; ++n) {
+        assert(i->first == 1);
+        assert(s.find(i->second) != s.end());
+        s.erase(s.find(i->second));
+        ++i;
+      }
+    }
+
+    b = c.bucket(2);
+    i = c.begin(b);
+    j = c.end(b);
+    assert(std::distance(i, j) == 2);
+    {
+      std::set<std::string> s;
+      s.insert("two");
+      s.insert("four");
+      for (int n = 0; n < 2; ++n) {
+        assert(i->first == 2);
+        assert(s.find(i->second) != s.end());
+        s.erase(s.find(i->second));
+        ++i;
+      }
+    }
+
+    b = c.bucket(3);
+    i = c.begin(b);
+    j = c.end(b);
+    assert(std::distance(i, j) == 1);
+    assert(i->first == 3);
+    assert(i->second == "three");
+
+    b = c.bucket(4);
+    i = c.begin(b);
+    j = c.end(b);
+    assert(std::distance(i, j) == 1);
+    assert(i->first == 4);
+    assert(i->second == "four");
+
+    b = c.bucket(5);
+    i = c.begin(b);
+    j = c.end(b);
+    assert(std::distance(i, j) == 0);
+
+    b = c.bucket(6);
+    i = c.begin(b);
+    j = c.end(b);
+    assert(std::distance(i, j) == 0);
+  }
+  {
+    typedef std::unordered_multimap<int,
+                                    std::string,
+                                    std::hash<int>,
+                                    std::equal_to<int>,
+                                    min_allocator<std::pair<const int, std::string>>>
+        C;
+    typedef std::pair<int, std::string> P;
+    typedef C::const_local_iterator I;
+    P a[] = {
+        P(1, "one"),
+        P(2, "two"),
+        P(3, "three"),
+        P(4, "four"),
+        P(1, "four"),
+        P(2, "four"),
+    };
+    C c(a, a + sizeof(a) / sizeof(a[0]));
+    assert(c.bucket_count() >= 7);
+    C::size_type b = c.bucket(0);
+    I i            = c.cbegin(b);
+    I j            = c.cend(b);
+    assert(std::distance(i, j) == 0);
+
+    b = c.bucket(1);
+    i = c.cbegin(b);
+    j = c.cend(b);
+    assert(std::distance(i, j) == 2);
+    {
+      std::set<std::string> s;
+      s.insert("one");
+      s.insert("four");
+      for (int n = 0; n < 2; ++n) {
+        assert(i->first == 1);
+        assert(s.find(i->second) != s.end());
+        s.erase(s.find(i->second));
+        ++i;
+      }
+    }
+
+    b = c.bucket(2);
+    i = c.cbegin(b);
+    j = c.cend(b);
+    assert(std::distance(i, j) == 2);
     {
-        typedef std::unordered_multimap<int, std::string, std::hash<int>, std::equal_to<int>,
-                            min_allocator<std::pair<const int, std::string>>> C;
-        typedef std::pair<int, std::string> P;
-        typedef C::const_local_iterator I;
-        P a[] =
-        {
-            P(1, "one"),
-            P(2, "two"),
-            P(3, "three"),
-            P(4, "four"),
-            P(1, "four"),
-            P(2, "four"),
-        };
-        const C c(a, a + sizeof(a)/sizeof(a[0]));
-        assert(c.bucket_count() >= 7);
-        C::size_type b = c.bucket(0);
-        I i = c.begin(b);
-        I j = c.end(b);
-        assert(std::distance(i, j) == 0);
-
-        b = c.bucket(1);
-        i = c.begin(b);
-        j = c.end(b);
-        assert(std::distance(i, j) == 2);
-        {
-            std::set<std::string> s;
-            s.insert("one");
-            s.insert("four");
-            for ( int n = 0; n < 2; ++n )
-            {
-                assert(i->first == 1);
-                assert(s.find(i->second) != s.end());
-                s.erase(s.find(i->second));
-                ++i;
-            }
-        }
-
-        b = c.bucket(2);
-        i = c.begin(b);
-        j = c.end(b);
-        assert(std::distance(i, j) == 2);
-        {
-            std::set<std::string> s;
-            s.insert("two");
-            s.insert("four");
-            for ( int n = 0; n < 2; ++n )
-            {
-                assert(i->first == 2);
-                assert(s.find(i->second) != s.end());
-                s.erase(s.find(i->second));
-                ++i;
-            }
-        }
-
-        b = c.bucket(3);
-        i = c.begin(b);
-        j = c.end(b);
-        assert(std::distance(i, j) == 1);
-        assert(i->first == 3);
-        assert(i->second == "three");
-
-        b = c.bucket(4);
-        i = c.begin(b);
-        j = c.end(b);
-        assert(std::distance(i, j) == 1);
-        assert(i->first == 4);
-        assert(i->second == "four");
-
-        b = c.bucket(5);
-        i = c.begin(b);
-        j = c.end(b);
-        assert(std::distance(i, j) == 0);
-
-        b = c.bucket(6);
-        i = c.begin(b);
-        j = c.end(b);
-        assert(std::distance(i, j) == 0);
+      std::set<std::string> s;
+      s.insert("two");
+      s.insert("four");
+      for (int n = 0; n < 2; ++n) {
+        assert(i->first == 2);
+        assert(s.find(i->second) != s.end());
+        s.erase(s.find(i->second));
+        ++i;
+      }
     }
+
+    b = c.bucket(3);
+    i = c.cbegin(b);
+    j = c.cend(b);
+    assert(std::distance(i, j) == 1);
+    assert(i->first == 3);
+    assert(i->second == "three");
+
+    b = c.bucket(4);
+    i = c.cbegin(b);
+    j = c.cend(b);
+    assert(std::distance(i, j) == 1);
+    assert(i->first == 4);
+    assert(i->second == "four");
+
+    b = c.bucket(5);
+    i = c.cbegin(b);
+    j = c.cend(b);
+    assert(std::distance(i, j) == 0);
+
+    b = c.bucket(6);
+    i = c.cbegin(b);
+    j = c.cend(b);
+    assert(std::distance(i, j) == 0);
+  }
+  {
+    typedef std::unordered_multimap<int,
+                                    std::string,
+                                    std::hash<int>,
+                                    std::equal_to<int>,
+                                    min_allocator<std::pair<const int, std::string>>>
+        C;
+    typedef std::pair<int, std::string> P;
+    typedef C::const_local_iterator I;
+    P a[] = {
+        P(1, "one"),
+        P(2, "two"),
+        P(3, "three"),
+        P(4, "four"),
+        P(1, "four"),
+        P(2, "four"),
+    };
+    const C c(a, a + sizeof(a) / sizeof(a[0]));
+    assert(c.bucket_count() >= 7);
+    C::size_type b = c.bucket(0);
+    I i            = c.cbegin(b);
+    I j            = c.cend(b);
+    assert(std::distance(i, j) == 0);
+
+    b = c.bucket(1);
+    i = c.cbegin(b);
+    j = c.cend(b);
+    assert(std::distance(i, j) == 2);
     {
-        typedef std::unordered_multimap<int, std::string, std::hash<int>, std::equal_to<int>,
-                            min_allocator<std::pair<const int, std::string>>> C;
-        typedef std::pair<int, std::string> P;
-        typedef C::const_local_iterator I;
-        P a[] =
-        {
-            P(1, "one"),
-            P(2, "two"),
-            P(3, "three"),
-            P(4, "four"),
-            P(1, "four"),
-            P(2, "four"),
-        };
-        C c(a, a + sizeof(a)/sizeof(a[0]));
-        assert(c.bucket_count() >= 7);
-        C::size_type b = c.bucket(0);
-        I i = c.cbegin(b);
-        I j = c.cend(b);
-        assert(std::distance(i, j) == 0);
-
-        b = c.bucket(1);
-        i = c.cbegin(b);
-        j = c.cend(b);
-        assert(std::distance(i, j) == 2);
-        {
-            std::set<std::string> s;
-            s.insert("one");
-            s.insert("four");
-            for ( int n = 0; n < 2; ++n )
-            {
-                assert(i->first == 1);
-                assert(s.find(i->second) != s.end());
-                s.erase(s.find(i->second));
-                ++i;
-            }
-        }
-
-        b = c.bucket(2);
-        i = c.cbegin(b);
-        j = c.cend(b);
-        assert(std::distance(i, j) == 2);
-        {
-            std::set<std::string> s;
-            s.insert("two");
-            s.insert("four");
-            for ( int n = 0; n < 2; ++n )
-            {
-                assert(i->first == 2);
-                assert(s.find(i->second) != s.end());
-                s.erase(s.find(i->second));
-                ++i;
-            }
-        }
-
-        b = c.bucket(3);
-        i = c.cbegin(b);
-        j = c.cend(b);
-        assert(std::distance(i, j) == 1);
-        assert(i->first == 3);
-        assert(i->second == "three");
-
-        b = c.bucket(4);
-        i = c.cbegin(b);
-        j = c.cend(b);
-        assert(std::distance(i, j) == 1);
-        assert(i->first == 4);
-        assert(i->second == "four");
-
-        b = c.bucket(5);
-        i = c.cbegin(b);
-        j = c.cend(b);
-        assert(std::distance(i, j) == 0);
-
-        b = c.bucket(6);
-        i = c.cbegin(b);
-        j = c.cend(b);
-        assert(std::distance(i, j) == 0);
+      std::set<std::string> s;
+      s.insert("one");
+      s.insert("four");
+      for (int n = 0; n < 2; ++n) {
+        assert(i->first == 1);
+        assert(s.find(i->second) != s.end());
+        s.erase(s.find(i->second));
+        ++i;
+      }
     }
+
+    b = c.bucket(2);
+    i = c.cbegin(b);
+    j = c.cend(b);
+    assert(std::distance(i, j) == 2);
     {
-        typedef std::unordered_multimap<int, std::string, std::hash<int>, std::equal_to<int>,
-                            min_allocator<std::pair<const int, std::string>>> C;
-        typedef std::pair<int, std::string> P;
-        typedef C::const_local_iterator I;
-        P a[] =
-        {
-            P(1, "one"),
-            P(2, "two"),
-            P(3, "three"),
-            P(4, "four"),
-            P(1, "four"),
-            P(2, "four"),
-        };
-        const C c(a, a + sizeof(a)/sizeof(a[0]));
-        assert(c.bucket_count() >= 7);
-        C::size_type b = c.bucket(0);
-        I i = c.cbegin(b);
-        I j = c.cend(b);
-        assert(std::distance(i, j) == 0);
-
-        b = c.bucket(1);
-        i = c.cbegin(b);
-        j = c.cend(b);
-        assert(std::distance(i, j) == 2);
-        {
-            std::set<std::string> s;
-            s.insert("one");
-            s.insert("four");
-            for ( int n = 0; n < 2; ++n )
-            {
-                assert(i->first == 1);
-                assert(s.find(i->second) != s.end());
-                s.erase(s.find(i->second));
-                ++i;
-            }
-        }
-
-        b = c.bucket(2);
-        i = c.cbegin(b);
-        j = c.cend(b);
-        assert(std::distance(i, j) == 2);
-        {
-            std::set<std::string> s;
-            s.insert("two");
-            s.insert("four");
-            for ( int n = 0; n < 2; ++n )
-            {
-                assert(i->first == 2);
-                assert(s.find(i->second) != s.end());
-                s.erase(s.find(i->second));
-                ++i;
-            }
-        }
-
-        b = c.bucket(3);
-        i = c.cbegin(b);
-        j = c.cend(b);
-        assert(std::distance(i, j) == 1);
-        assert(i->first == 3);
-        assert(i->second == "three");
-
-        b = c.bucket(4);
-        i = c.cbegin(b);
-        j = c.cend(b);
-        assert(std::distance(i, j) == 1);
-        assert(i->first == 4);
-        assert(i->second == "four");
-
-        b = c.bucket(5);
-        i = c.cbegin(b);
-        j = c.cend(b);
-        assert(std::distance(i, j) == 0);
-
-        b = c.bucket(6);
-        i = c.cbegin(b);
-        j = c.cend(b);
-        assert(std::distance(i, j) == 0);
+      std::set<std::string> s;
+      s.insert("two");
+      s.insert("four");
+      for (int n = 0; n < 2; ++n) {
+        assert(i->first == 2);
+        assert(s.find(i->second) != s.end());
+        s.erase(s.find(i->second));
+        ++i;
+      }
     }
+
+    b = c.bucket(3);
+    i = c.cbegin(b);
+    j = c.cend(b);
+    assert(std::distance(i, j) == 1);
+    assert(i->first == 3);
+    assert(i->second == "three");
+
+    b = c.bucket(4);
+    i = c.cbegin(b);
+    j = c.cend(b);
+    assert(std::distance(i, j) == 1);
+    assert(i->first == 4);
+    assert(i->second == "four");
+
+    b = c.bucket(5);
+    i = c.cbegin(b);
+    j = c.cend(b);
+    assert(std::distance(i, j) == 0);
+
+    b = c.bucket(6);
+    i = c.cbegin(b);
+    j = c.cend(b);
+    assert(std::distance(i, j) == 0);
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/unord/unord.multimap/max_bucket_count.pass.cpp b/libcxx/test/std/containers/unord/unord.multimap/max_bucket_count.pass.cpp
index 1dac9024cbedd..6416539c4fa59 100644
--- a/libcxx/test/std/containers/unord/unord.multimap/max_bucket_count.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.multimap/max_bucket_count.pass.cpp
@@ -21,20 +21,23 @@
 #include "test_macros.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef std::unordered_multimap<int, std::string> C;
-        const C c;
-        assert(c.max_bucket_count() > 0);
-    }
+int main(int, char**) {
+  {
+    typedef std::unordered_multimap<int, std::string> C;
+    const C c;
+    assert(c.max_bucket_count() > 0);
+  }
 #if TEST_STD_VER >= 11
-    {
-        typedef std::unordered_multimap<int, std::string, std::hash<int>, std::equal_to<int>,
-                            min_allocator<std::pair<const int, std::string>>> C;
-        const C c;
-        assert(c.max_bucket_count() > 0);
-    }
+  {
+    typedef std::unordered_multimap<int,
+                                    std::string,
+                                    std::hash<int>,
+                                    std::equal_to<int>,
+                                    min_allocator<std::pair<const int, std::string>>>
+        C;
+    const C c;
+    assert(c.max_bucket_count() > 0);
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/unord/unord.multimap/max_load_factor.pass.cpp b/libcxx/test/std/containers/unord/unord.multimap/max_load_factor.pass.cpp
index ab08808783df7..cd4d06248bd52 100644
--- a/libcxx/test/std/containers/unord/unord.multimap/max_load_factor.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.multimap/max_load_factor.pass.cpp
@@ -22,36 +22,43 @@
 #include "test_macros.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef std::unordered_multimap<int, std::string> C;
-        const C c;
-        assert(c.max_load_factor() == 1);
-    }
-    {
-        typedef std::unordered_multimap<int, std::string> C;
-        C c;
-        assert(c.max_load_factor() == 1);
-        c.max_load_factor(2.5);
-        assert(c.max_load_factor() == 2.5);
-    }
+int main(int, char**) {
+  {
+    typedef std::unordered_multimap<int, std::string> C;
+    const C c;
+    assert(c.max_load_factor() == 1);
+  }
+  {
+    typedef std::unordered_multimap<int, std::string> C;
+    C c;
+    assert(c.max_load_factor() == 1);
+    c.max_load_factor(2.5);
+    assert(c.max_load_factor() == 2.5);
+  }
 #if TEST_STD_VER >= 11
-    {
-        typedef std::unordered_multimap<int, std::string, std::hash<int>, std::equal_to<int>,
-                            min_allocator<std::pair<const int, std::string>>> C;
-        const C c;
-        assert(c.max_load_factor() == 1);
-    }
-    {
-        typedef std::unordered_multimap<int, std::string, std::hash<int>, std::equal_to<int>,
-                            min_allocator<std::pair<const int, std::string>>> C;
-        C c;
-        assert(c.max_load_factor() == 1);
-        c.max_load_factor(2.5);
-        assert(c.max_load_factor() == 2.5);
-    }
+  {
+    typedef std::unordered_multimap<int,
+                                    std::string,
+                                    std::hash<int>,
+                                    std::equal_to<int>,
+                                    min_allocator<std::pair<const int, std::string>>>
+        C;
+    const C c;
+    assert(c.max_load_factor() == 1);
+  }
+  {
+    typedef std::unordered_multimap<int,
+                                    std::string,
+                                    std::hash<int>,
+                                    std::equal_to<int>,
+                                    min_allocator<std::pair<const int, std::string>>>
+        C;
+    C c;
+    assert(c.max_load_factor() == 1);
+    c.max_load_factor(2.5);
+    assert(c.max_load_factor() == 2.5);
+  }
 #endif
 
-    return 0;
+  return 0;
 }
diff --git a/libcxx/test/std/containers/unord/unord.multimap/max_size.pass.cpp b/libcxx/test/std/containers/unord/unord.multimap/max_size.pass.cpp
index 182f5a3693b03..4f8706760a6b1 100644
--- a/libcxx/test/std/containers/unord/unord.multimap/max_size.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.multimap/max_size.pass.cpp
@@ -20,37 +20,30 @@
 #include "test_allocator.h"
 #include "test_macros.h"
 
-int main(int, char**)
-{
+int main(int, char**) {
   typedef std::pair<const int, int> KV;
   {
     typedef limited_allocator<KV, 10> A;
-    typedef std::unordered_multimap<int, int, std::hash<int>,
-                                    std::equal_to<int>, A>
-        C;
+    typedef std::unordered_multimap<int, int, std::hash<int>, std::equal_to<int>, A> C;
     C c;
     assert(c.max_size() <= 10);
     LIBCPP_ASSERT(c.max_size() == 10);
   }
   {
     typedef limited_allocator<KV, (std::size_t)-1> A;
-    typedef std::unordered_multimap<int, int, std::hash<int>,
-                                    std::equal_to<int>, A>
-        C;
-    const C::size_type max_dist =
-        static_cast<C::size_type>(std::numeric_limits<C::difference_type>::max());
+    typedef std::unordered_multimap<int, int, std::hash<int>, std::equal_to<int>, A> C;
+    const C::size_type max_dist = static_cast<C::size_type>(std::numeric_limits<C::difference_type>::max());
     C c;
     assert(c.max_size() <= max_dist);
     LIBCPP_ASSERT(c.max_size() == max_dist);
-    }
-    {
-      typedef std::unordered_multimap<char, int> C;
-    const C::size_type max_dist =
-        static_cast<C::size_type>(std::numeric_limits<C::difference_type>::max());
-      C c;
-      assert(c.max_size() <= max_dist);
-      assert(c.max_size() <= alloc_max_size(c.get_allocator()));
-    }
+  }
+  {
+    typedef std::unordered_multimap<char, int> C;
+    const C::size_type max_dist = static_cast<C::size_type>(std::numeric_limits<C::difference_type>::max());
+    C c;
+    assert(c.max_size() <= max_dist);
+    assert(c.max_size() <= alloc_max_size(c.get_allocator()));
+  }
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/unord/unord.multimap/range_concept_conformance.compile.pass.cpp b/libcxx/test/std/containers/unord/unord.multimap/range_concept_conformance.compile.pass.cpp
index 31c7df7032205..f90fa470f7b8b 100644
--- a/libcxx/test/std/containers/unord/unord.multimap/range_concept_conformance.compile.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.multimap/range_concept_conformance.compile.pass.cpp
@@ -19,7 +19,6 @@
 
 using range = std::unordered_multimap<int, int>;
 
-
 static_assert(std::same_as<std::ranges::iterator_t<range>, range::iterator>);
 static_assert(std::ranges::common_range<range>);
 static_assert(std::ranges::forward_range<range>);
diff --git a/libcxx/test/std/containers/unord/unord.multimap/rehash.pass.cpp b/libcxx/test/std/containers/unord/unord.multimap/rehash.pass.cpp
index 33d2bc0ed107b..f496a5e8c2e15 100644
--- a/libcxx/test/std/containers/unord/unord.multimap/rehash.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.multimap/rehash.pass.cpp
@@ -26,123 +26,120 @@
 #include "min_allocator.h"
 
 template <class C>
-void rehash_postcondition(const C& c, std::size_t n)
-{
-    assert(c.bucket_count() >= c.size() / c.max_load_factor() && c.bucket_count() >= n);
+void rehash_postcondition(const C& c, std::size_t n) {
+  assert(c.bucket_count() >= c.size() / c.max_load_factor() && c.bucket_count() >= n);
 }
 
 template <class C>
-void test(const C& c)
-{
-    assert(c.size() == 6);
-    typedef std::pair<typename C::const_iterator, typename C::const_iterator> Eq;
-    Eq eq = c.equal_range(1);
-    assert(std::distance(eq.first, eq.second) == 2);
-    typename C::const_iterator i = eq.first;
-    {
-        std::set<std::string> s;
-        s.insert("one");
-        s.insert("four");
-        for ( int n = 0; n < 2; ++n )
-        {
-            assert(i->first == 1);
-            assert(s.find(i->second) != s.end());
-            s.erase(s.find(i->second));
-            ++i;
-        }
+void test(const C& c) {
+  assert(c.size() == 6);
+  typedef std::pair<typename C::const_iterator, typename C::const_iterator> Eq;
+  Eq eq = c.equal_range(1);
+  assert(std::distance(eq.first, eq.second) == 2);
+  typename C::const_iterator i = eq.first;
+  {
+    std::set<std::string> s;
+    s.insert("one");
+    s.insert("four");
+    for (int n = 0; n < 2; ++n) {
+      assert(i->first == 1);
+      assert(s.find(i->second) != s.end());
+      s.erase(s.find(i->second));
+      ++i;
     }
-    eq = c.equal_range(2);
-    assert(std::distance(eq.first, eq.second) == 2);
-    i = eq.first;
-    {
-        std::set<std::string> s;
-        s.insert("two");
-        s.insert("four");
-        for ( int n = 0; n < 2; ++n )
-        {
-            assert(i->first == 2);
-            assert(s.find(i->second) != s.end());
-            s.erase(s.find(i->second));
-            ++i;
-        }
+  }
+  eq = c.equal_range(2);
+  assert(std::distance(eq.first, eq.second) == 2);
+  i = eq.first;
+  {
+    std::set<std::string> s;
+    s.insert("two");
+    s.insert("four");
+    for (int n = 0; n < 2; ++n) {
+      assert(i->first == 2);
+      assert(s.find(i->second) != s.end());
+      s.erase(s.find(i->second));
+      ++i;
     }
-    eq = c.equal_range(3);
-    assert(std::distance(eq.first, eq.second) == 1);
-    i = eq.first;
-    assert(i->first == 3);
-    assert(i->second == "three");
-    eq = c.equal_range(4);
-    assert(std::distance(eq.first, eq.second) == 1);
-    i = eq.first;
-    assert(i->first == 4);
-    assert(i->second == "four");
-    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
-    assert(std::fabs(c.load_factor() - (float)c.size()/c.bucket_count()) < FLT_EPSILON);
+  }
+  eq = c.equal_range(3);
+  assert(std::distance(eq.first, eq.second) == 1);
+  i = eq.first;
+  assert(i->first == 3);
+  assert(i->second == "three");
+  eq = c.equal_range(4);
+  assert(std::distance(eq.first, eq.second) == 1);
+  i = eq.first;
+  assert(i->first == 4);
+  assert(i->second == "four");
+  assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
+  assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
+  assert(std::fabs(c.load_factor() - (float)c.size() / c.bucket_count()) < FLT_EPSILON);
 }
 
-int main(int, char**)
-{
-    {
-        typedef std::unordered_multimap<int, std::string> C;
-        typedef std::pair<int, std::string> P;
-        P a[] =
-        {
-            P(1, "one"),
-            P(2, "two"),
-            P(3, "three"),
-            P(4, "four"),
-            P(1, "four"),
-            P(2, "four"),
-        };
-        C c(a, a + sizeof(a)/sizeof(a[0]));
-        test(c);
-        assert(c.bucket_count() >= 7);
-        c.rehash(3);
-        rehash_postcondition(c, 3);
-        LIBCPP_ASSERT(c.bucket_count() == 7);
-        test(c);
-        c.max_load_factor(2);
-        c.rehash(3);
-        rehash_postcondition(c, 3);
-        LIBCPP_ASSERT(c.bucket_count() == 3);
-        test(c);
-        c.rehash(31);
-        rehash_postcondition(c, 31);
-        LIBCPP_ASSERT(c.bucket_count() == 31);
-        test(c);
-    }
+int main(int, char**) {
+  {
+    typedef std::unordered_multimap<int, std::string> C;
+    typedef std::pair<int, std::string> P;
+    P a[] = {
+        P(1, "one"),
+        P(2, "two"),
+        P(3, "three"),
+        P(4, "four"),
+        P(1, "four"),
+        P(2, "four"),
+    };
+    C c(a, a + sizeof(a) / sizeof(a[0]));
+    test(c);
+    assert(c.bucket_count() >= 7);
+    c.rehash(3);
+    rehash_postcondition(c, 3);
+    LIBCPP_ASSERT(c.bucket_count() == 7);
+    test(c);
+    c.max_load_factor(2);
+    c.rehash(3);
+    rehash_postcondition(c, 3);
+    LIBCPP_ASSERT(c.bucket_count() == 3);
+    test(c);
+    c.rehash(31);
+    rehash_postcondition(c, 31);
+    LIBCPP_ASSERT(c.bucket_count() == 31);
+    test(c);
+  }
 #if TEST_STD_VER >= 11
-    {
-        typedef std::unordered_multimap<int, std::string, std::hash<int>, std::equal_to<int>,
-                            min_allocator<std::pair<const int, std::string>>> C;
-        typedef std::pair<int, std::string> P;
-        P a[] =
-        {
-            P(1, "one"),
-            P(2, "two"),
-            P(3, "three"),
-            P(4, "four"),
-            P(1, "four"),
-            P(2, "four"),
-        };
-        C c(a, a + sizeof(a)/sizeof(a[0]));
-        test(c);
-        assert(c.bucket_count() >= 7);
-        c.rehash(3);
-        rehash_postcondition(c, 3);
-        LIBCPP_ASSERT(c.bucket_count() == 7);
-        test(c);
-        c.max_load_factor(2);
-        c.rehash(3);
-        rehash_postcondition(c, 3);
-        LIBCPP_ASSERT(c.bucket_count() == 3);
-        test(c);
-        c.rehash(31);
-        rehash_postcondition(c, 31);
-        LIBCPP_ASSERT(c.bucket_count() == 31);
-        test(c);
-    }
+  {
+    typedef std::unordered_multimap<int,
+                                    std::string,
+                                    std::hash<int>,
+                                    std::equal_to<int>,
+                                    min_allocator<std::pair<const int, std::string>>>
+        C;
+    typedef std::pair<int, std::string> P;
+    P a[] = {
+        P(1, "one"),
+        P(2, "two"),
+        P(3, "three"),
+        P(4, "four"),
+        P(1, "four"),
+        P(2, "four"),
+    };
+    C c(a, a + sizeof(a) / sizeof(a[0]));
+    test(c);
+    assert(c.bucket_count() >= 7);
+    c.rehash(3);
+    rehash_postcondition(c, 3);
+    LIBCPP_ASSERT(c.bucket_count() == 7);
+    test(c);
+    c.max_load_factor(2);
+    c.rehash(3);
+    rehash_postcondition(c, 3);
+    LIBCPP_ASSERT(c.bucket_count() == 3);
+    test(c);
+    c.rehash(31);
+    rehash_postcondition(c, 31);
+    LIBCPP_ASSERT(c.bucket_count() == 31);
+    test(c);
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/unord/unord.multimap/reserve.pass.cpp b/libcxx/test/std/containers/unord/unord.multimap/reserve.pass.cpp
index b658539c401e5..6063853ef4674 100644
--- a/libcxx/test/std/containers/unord/unord.multimap/reserve.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.multimap/reserve.pass.cpp
@@ -23,102 +23,100 @@
 #include "min_allocator.h"
 
 template <class C>
-void test(const C& c)
-{
-    assert(c.size() == 6);
-    {
-        std::set<std::string> s;
-        s.insert("one");
-        s.insert("four");
-        assert(s.find(c.find(1)->second) != s.end());
-        s.erase(s.find(c.find(1)->second));
-        assert(s.find(std::next(c.find(1))->second) != s.end());
-    }
-    {
-        std::set<std::string> s;
-        s.insert("two");
-        s.insert("four");
-        assert(s.find(c.find(2)->second) != s.end());
-        s.erase(s.find(c.find(2)->second));
-        assert(s.find(std::next(c.find(2))->second) != s.end());
-    }
-    assert(c.find(3)->second == "three");
-    assert(c.find(4)->second == "four");
+void test(const C& c) {
+  assert(c.size() == 6);
+  {
+    std::set<std::string> s;
+    s.insert("one");
+    s.insert("four");
+    assert(s.find(c.find(1)->second) != s.end());
+    s.erase(s.find(c.find(1)->second));
+    assert(s.find(std::next(c.find(1))->second) != s.end());
+  }
+  {
+    std::set<std::string> s;
+    s.insert("two");
+    s.insert("four");
+    assert(s.find(c.find(2)->second) != s.end());
+    s.erase(s.find(c.find(2)->second));
+    assert(s.find(std::next(c.find(2))->second) != s.end());
+  }
+  assert(c.find(3)->second == "three");
+  assert(c.find(4)->second == "four");
 }
 
 void reserve_invariant(std::size_t n) // LWG #2156
 {
-    for (std::size_t i = 0; i < n; ++i)
-    {
-        std::unordered_multimap<std::size_t, size_t> c;
-        c.reserve(n);
-        std::size_t buckets = c.bucket_count();
-        for (std::size_t j = 0; j < i; ++j)
-        {
-            c.insert(std::unordered_multimap<std::size_t, size_t>::value_type(i,i));
-            assert(buckets == c.bucket_count());
-        }
+  for (std::size_t i = 0; i < n; ++i) {
+    std::unordered_multimap<std::size_t, size_t> c;
+    c.reserve(n);
+    std::size_t buckets = c.bucket_count();
+    for (std::size_t j = 0; j < i; ++j) {
+      c.insert(std::unordered_multimap<std::size_t, size_t>::value_type(i, i));
+      assert(buckets == c.bucket_count());
     }
+  }
 }
 
-int main(int, char**)
-{
-    {
-        typedef std::unordered_multimap<int, std::string> C;
-        typedef std::pair<int, std::string> P;
-        P a[] =
-        {
-            P(1, "one"),
-            P(2, "two"),
-            P(3, "three"),
-            P(4, "four"),
-            P(1, "four"),
-            P(2, "four"),
-        };
-        C c(a, a + sizeof(a)/sizeof(a[0]));
-        test(c);
-        assert(c.bucket_count() >= 7);
-        c.reserve(3);
-        LIBCPP_ASSERT(c.bucket_count() == 7);
-        test(c);
-        c.max_load_factor(2);
-        c.reserve(3);
-        LIBCPP_ASSERT(c.bucket_count() == 3);
-        test(c);
-        c.reserve(31);
-        assert(c.bucket_count() >= 16);
-        test(c);
-    }
+int main(int, char**) {
+  {
+    typedef std::unordered_multimap<int, std::string> C;
+    typedef std::pair<int, std::string> P;
+    P a[] = {
+        P(1, "one"),
+        P(2, "two"),
+        P(3, "three"),
+        P(4, "four"),
+        P(1, "four"),
+        P(2, "four"),
+    };
+    C c(a, a + sizeof(a) / sizeof(a[0]));
+    test(c);
+    assert(c.bucket_count() >= 7);
+    c.reserve(3);
+    LIBCPP_ASSERT(c.bucket_count() == 7);
+    test(c);
+    c.max_load_factor(2);
+    c.reserve(3);
+    LIBCPP_ASSERT(c.bucket_count() == 3);
+    test(c);
+    c.reserve(31);
+    assert(c.bucket_count() >= 16);
+    test(c);
+  }
 #if TEST_STD_VER >= 11
-    {
-        typedef std::unordered_multimap<int, std::string, std::hash<int>, std::equal_to<int>,
-                            min_allocator<std::pair<const int, std::string>>> C;
-        typedef std::pair<int, std::string> P;
-        P a[] =
-        {
-            P(1, "one"),
-            P(2, "two"),
-            P(3, "three"),
-            P(4, "four"),
-            P(1, "four"),
-            P(2, "four"),
-        };
-        C c(a, a + sizeof(a)/sizeof(a[0]));
-        test(c);
-        assert(c.bucket_count() >= 7);
-        c.reserve(3);
-        LIBCPP_ASSERT(c.bucket_count() == 7);
-        test(c);
-        c.max_load_factor(2);
-        c.reserve(3);
-        LIBCPP_ASSERT(c.bucket_count() == 3);
-        test(c);
-        c.reserve(31);
-        assert(c.bucket_count() >= 16);
-        test(c);
-    }
+  {
+    typedef std::unordered_multimap<int,
+                                    std::string,
+                                    std::hash<int>,
+                                    std::equal_to<int>,
+                                    min_allocator<std::pair<const int, std::string>>>
+        C;
+    typedef std::pair<int, std::string> P;
+    P a[] = {
+        P(1, "one"),
+        P(2, "two"),
+        P(3, "three"),
+        P(4, "four"),
+        P(1, "four"),
+        P(2, "four"),
+    };
+    C c(a, a + sizeof(a) / sizeof(a[0]));
+    test(c);
+    assert(c.bucket_count() >= 7);
+    c.reserve(3);
+    LIBCPP_ASSERT(c.bucket_count() == 7);
+    test(c);
+    c.max_load_factor(2);
+    c.reserve(3);
+    LIBCPP_ASSERT(c.bucket_count() == 3);
+    test(c);
+    c.reserve(31);
+    assert(c.bucket_count() >= 16);
+    test(c);
+  }
 #endif
-    reserve_invariant(20);
+  reserve_invariant(20);
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/unord/unord.multimap/scary.pass.cpp b/libcxx/test/std/containers/unord/unord.multimap/scary.pass.cpp
index b306da2f818ed..59ade49434ca9 100644
--- a/libcxx/test/std/containers/unord/unord.multimap/scary.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.multimap/scary.pass.cpp
@@ -16,13 +16,12 @@
 
 #include "test_macros.h"
 
-int main(int, char**)
-{
-    typedef std::unordered_map<int, int> M1;
-    typedef std::unordered_multimap<int, int> M2;
-    M2::iterator i;
-    M1::iterator j = i;
-    ((void)j);
+int main(int, char**) {
+  typedef std::unordered_map<int, int> M1;
+  typedef std::unordered_multimap<int, int> M2;
+  M2::iterator i;
+  M1::iterator j = i;
+  ((void)j);
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/unord/unord.multimap/size.pass.cpp b/libcxx/test/std/containers/unord/unord.multimap/size.pass.cpp
index 493b8d757cfdd..a577419112fc5 100644
--- a/libcxx/test/std/containers/unord/unord.multimap/size.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.multimap/size.pass.cpp
@@ -18,9 +18,8 @@
 #include "test_macros.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
+int main(int, char**) {
+  {
     typedef std::unordered_multimap<int, double> M;
     M m;
     ASSERT_NOEXCEPT(m.size());
@@ -37,10 +36,12 @@ int main(int, char**)
     assert(m.size() == 1);
     m.erase(m.begin());
     assert(m.size() == 0);
-    }
+  }
 #if TEST_STD_VER >= 11
-    {
-    typedef std::unordered_multimap<int, double, std::hash<int>, std::equal_to<int>, min_allocator<std::pair<const int, double>>> M;
+  {
+    typedef std::
+        unordered_multimap<int, double, std::hash<int>, std::equal_to<int>, min_allocator<std::pair<const int, double>>>
+            M;
     M m;
     ASSERT_NOEXCEPT(m.size());
     assert(m.size() == 0);
@@ -56,7 +57,7 @@ int main(int, char**)
     assert(m.size() == 1);
     m.erase(m.begin());
     assert(m.size() == 0);
-    }
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/unord/unord.multimap/swap_member.pass.cpp b/libcxx/test/std/containers/unord/unord.multimap/swap_member.pass.cpp
index 257938b0628f1..af6cebdc9d7a4 100644
--- a/libcxx/test/std/containers/unord/unord.multimap/swap_member.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.multimap/swap_member.pass.cpp
@@ -28,631 +28,618 @@
 
 #include "min_allocator.h"
 
-int main(int, char**)
-{
+int main(int, char**) {
+  {
+    typedef test_hash<int> Hash;
+    typedef test_equal_to<int> Compare;
+    typedef test_allocator<std::pair<const int, std::string> > Alloc;
+    typedef std::unordered_multimap<int, std::string, Hash, Compare, Alloc> C;
+    C c1(0, Hash(1), Compare(1), Alloc(1, 1));
+    C c2(0, Hash(2), Compare(2), Alloc(1, 2));
+    c2.max_load_factor(2);
+    c1.swap(c2);
+
+    LIBCPP_ASSERT(c1.bucket_count() == 0);
+    assert(c1.size() == 0);
+    assert(c1.hash_function() == Hash(2));
+    assert(c1.key_eq() == Compare(2));
+    assert(c1.get_allocator().get_id() == 1);
+    assert(static_cast<std::size_t>(std::distance(c1.begin(), c1.end())) == c1.size());
+    assert(static_cast<std::size_t>(std::distance(c1.cbegin(), c1.cend())) == c1.size());
+    assert(c1.max_load_factor() == 2);
+
+    LIBCPP_ASSERT(c2.bucket_count() == 0);
+    assert(c2.size() == 0);
+    assert(c2.hash_function() == Hash(1));
+    assert(c2.key_eq() == Compare(1));
+    assert(c2.get_allocator().get_id() == 2);
+    assert(static_cast<std::size_t>(std::distance(c2.begin(), c2.end())) == c2.size());
+    assert(static_cast<std::size_t>(std::distance(c2.cbegin(), c2.cend())) == c2.size());
+    assert(c2.max_load_factor() == 1);
+  }
+  {
+    typedef test_hash<int> Hash;
+    typedef test_equal_to<int> Compare;
+    typedef test_allocator<std::pair<const int, std::string> > Alloc;
+    typedef std::unordered_multimap<int, std::string, Hash, Compare, Alloc> C;
+    typedef std::pair<int, std::string> P;
+    P a2[] = {
+        P(10, "ten"),
+        P(20, "twenty"),
+        P(30, "thirty"),
+        P(40, "forty"),
+        P(50, "fifty"),
+        P(60, "sixty"),
+        P(70, "seventy"),
+        P(80, "eighty"),
+    };
+    C c1(0, Hash(1), Compare(1), Alloc(1, 1));
+    C c2(std::begin(a2), std::end(a2), 0, Hash(2), Compare(2), Alloc(1, 2));
+    c2.max_load_factor(2);
+    c1.swap(c2);
+
+    assert(c1.bucket_count() >= 8);
+    assert(c1.size() == 8);
+    assert(c1.find(10)->second == "ten");
+    assert(c1.find(20)->second == "twenty");
+    assert(c1.find(30)->second == "thirty");
+    assert(c1.find(40)->second == "forty");
+    assert(c1.find(50)->second == "fifty");
+    assert(c1.find(60)->second == "sixty");
+    assert(c1.find(70)->second == "seventy");
+    assert(c1.find(80)->second == "eighty");
+    assert(c1.hash_function() == Hash(2));
+    assert(c1.key_eq() == Compare(2));
+    assert(c1.get_allocator().get_id() == 1);
+    assert(static_cast<std::size_t>(std::distance(c1.begin(), c1.end())) == c1.size());
+    assert(static_cast<std::size_t>(std::distance(c1.cbegin(), c1.cend())) == c1.size());
+    assert(c1.max_load_factor() == 2);
+
+    LIBCPP_ASSERT(c2.bucket_count() == 0);
+    assert(c2.size() == 0);
+    assert(c2.hash_function() == Hash(1));
+    assert(c2.key_eq() == Compare(1));
+    assert(c2.get_allocator().get_id() == 2);
+    assert(static_cast<std::size_t>(std::distance(c2.begin(), c2.end())) == c2.size());
+    assert(static_cast<std::size_t>(std::distance(c2.cbegin(), c2.cend())) == c2.size());
+    assert(c2.max_load_factor() == 1);
+  }
+  {
+    typedef test_hash<int> Hash;
+    typedef test_equal_to<int> Compare;
+    typedef test_allocator<std::pair<const int, std::string> > Alloc;
+    typedef std::unordered_multimap<int, std::string, Hash, Compare, Alloc> C;
+    typedef std::pair<int, std::string> P;
+    P a1[] = {
+        P(1, "one"),
+        P(2, "two"),
+        P(3, "three"),
+        P(4, "four"),
+        P(1, "four"),
+        P(2, "four"),
+    };
+    C c1(std::begin(a1), std::end(a1), 0, Hash(1), Compare(1), Alloc(1, 1));
+    C c2(0, Hash(2), Compare(2), Alloc(1, 2));
+    c2.max_load_factor(2);
+    c1.swap(c2);
+
+    LIBCPP_ASSERT(c1.bucket_count() == 0);
+    assert(c1.size() == 0);
+    assert(c1.hash_function() == Hash(2));
+    assert(c1.key_eq() == Compare(2));
+    assert(c1.get_allocator().get_id() == 1);
+    assert(static_cast<std::size_t>(std::distance(c1.begin(), c1.end())) == c1.size());
+    assert(static_cast<std::size_t>(std::distance(c1.cbegin(), c1.cend())) == c1.size());
+    assert(c1.max_load_factor() == 2);
+
+    assert(c2.bucket_count() >= 6);
+    assert(c2.size() == 6);
     {
-        typedef test_hash<int> Hash;
-        typedef test_equal_to<int> Compare;
-        typedef test_allocator<std::pair<const int, std::string> > Alloc;
-        typedef std::unordered_multimap<int, std::string, Hash, Compare, Alloc> C;
-        C c1(0, Hash(1), Compare(1), Alloc(1, 1));
-        C c2(0, Hash(2), Compare(2), Alloc(1, 2));
-        c2.max_load_factor(2);
-        c1.swap(c2);
-
-        LIBCPP_ASSERT(c1.bucket_count() == 0);
-        assert(c1.size() == 0);
-        assert(c1.hash_function() == Hash(2));
-        assert(c1.key_eq() == Compare(2));
-        assert(c1.get_allocator().get_id() == 1);
-        assert(static_cast<std::size_t>(std::distance(c1.begin(), c1.end())) == c1.size());
-        assert(static_cast<std::size_t>(std::distance(c1.cbegin(), c1.cend())) == c1.size());
-        assert(c1.max_load_factor() == 2);
-
-        LIBCPP_ASSERT(c2.bucket_count() == 0);
-        assert(c2.size() == 0);
-        assert(c2.hash_function() == Hash(1));
-        assert(c2.key_eq() == Compare(1));
-        assert(c2.get_allocator().get_id() == 2);
-        assert(static_cast<std::size_t>(std::distance(c2.begin(), c2.end())) == c2.size());
-        assert(static_cast<std::size_t>(std::distance(c2.cbegin(), c2.cend())) == c2.size());
-        assert(c2.max_load_factor() == 1);
+      std::set<std::string> s;
+      s.insert("one");
+      s.insert("four");
+      assert(s.find(c2.find(1)->second) != s.end());
+      s.erase(s.find(c2.find(1)->second));
+      assert(s.find(std::next(c2.find(1))->second) != s.end());
     }
     {
-        typedef test_hash<int> Hash;
-        typedef test_equal_to<int> Compare;
-        typedef test_allocator<std::pair<const int, std::string> > Alloc;
-        typedef std::unordered_multimap<int, std::string, Hash, Compare, Alloc> C;
-        typedef std::pair<int, std::string> P;
-        P a2[] =
-        {
-            P(10, "ten"),
-            P(20, "twenty"),
-            P(30, "thirty"),
-            P(40, "forty"),
-            P(50, "fifty"),
-            P(60, "sixty"),
-            P(70, "seventy"),
-            P(80, "eighty"),
-        };
-        C c1(0, Hash(1), Compare(1), Alloc(1, 1));
-        C c2(std::begin(a2), std::end(a2), 0, Hash(2), Compare(2), Alloc(1, 2));
-        c2.max_load_factor(2);
-        c1.swap(c2);
-
-        assert(c1.bucket_count() >= 8);
-        assert(c1.size() == 8);
-        assert(c1.find(10)->second == "ten");
-        assert(c1.find(20)->second == "twenty");
-        assert(c1.find(30)->second == "thirty");
-        assert(c1.find(40)->second == "forty");
-        assert(c1.find(50)->second == "fifty");
-        assert(c1.find(60)->second == "sixty");
-        assert(c1.find(70)->second == "seventy");
-        assert(c1.find(80)->second == "eighty");
-        assert(c1.hash_function() == Hash(2));
-        assert(c1.key_eq() == Compare(2));
-        assert(c1.get_allocator().get_id() == 1);
-        assert(static_cast<std::size_t>(std::distance(c1.begin(), c1.end())) == c1.size());
-        assert(static_cast<std::size_t>(std::distance(c1.cbegin(), c1.cend())) == c1.size());
-        assert(c1.max_load_factor() == 2);
-
-        LIBCPP_ASSERT(c2.bucket_count() == 0);
-        assert(c2.size() == 0);
-        assert(c2.hash_function() == Hash(1));
-        assert(c2.key_eq() == Compare(1));
-        assert(c2.get_allocator().get_id() == 2);
-        assert(static_cast<std::size_t>(std::distance(c2.begin(), c2.end())) == c2.size());
-        assert(static_cast<std::size_t>(std::distance(c2.cbegin(), c2.cend())) == c2.size());
-        assert(c2.max_load_factor() == 1);
+      std::set<std::string> s;
+      s.insert("two");
+      s.insert("four");
+      assert(s.find(c2.find(2)->second) != s.end());
+      s.erase(s.find(c2.find(2)->second));
+      assert(s.find(std::next(c2.find(2))->second) != s.end());
     }
+    assert(c2.find(3)->second == "three");
+    assert(c2.find(4)->second == "four");
+    assert(c2.hash_function() == Hash(1));
+    assert(c2.key_eq() == Compare(1));
+    assert(c2.get_allocator().get_id() == 2);
+    assert(static_cast<std::size_t>(std::distance(c2.begin(), c2.end())) == c2.size());
+    assert(static_cast<std::size_t>(std::distance(c2.cbegin(), c2.cend())) == c2.size());
+    assert(c2.max_load_factor() == 1);
+  }
+  {
+    typedef test_hash<int> Hash;
+    typedef test_equal_to<int> Compare;
+    typedef test_allocator<std::pair<const int, std::string> > Alloc;
+    typedef std::unordered_multimap<int, std::string, Hash, Compare, Alloc> C;
+    typedef std::pair<int, std::string> P;
+    P a1[] = {
+        P(1, "one"),
+        P(2, "two"),
+        P(3, "three"),
+        P(4, "four"),
+        P(1, "four"),
+        P(2, "four"),
+    };
+    P a2[] = {
+        P(10, "ten"),
+        P(20, "twenty"),
+        P(30, "thirty"),
+        P(40, "forty"),
+        P(50, "fifty"),
+        P(60, "sixty"),
+        P(70, "seventy"),
+        P(80, "eighty"),
+    };
+    C c1(std::begin(a1), std::end(a1), 0, Hash(1), Compare(1), Alloc(1, 1));
+    C c2(std::begin(a2), std::end(a2), 0, Hash(2), Compare(2), Alloc(1, 2));
+    c2.max_load_factor(2);
+    c1.swap(c2);
+
+    assert(c1.bucket_count() >= 8);
+    assert(c1.size() == 8);
+    assert(c1.find(10)->second == "ten");
+    assert(c1.find(20)->second == "twenty");
+    assert(c1.find(30)->second == "thirty");
+    assert(c1.find(40)->second == "forty");
+    assert(c1.find(50)->second == "fifty");
+    assert(c1.find(60)->second == "sixty");
+    assert(c1.find(70)->second == "seventy");
+    assert(c1.find(80)->second == "eighty");
+    assert(c1.hash_function() == Hash(2));
+    assert(c1.key_eq() == Compare(2));
+    assert(c1.get_allocator().get_id() == 1);
+    assert(static_cast<std::size_t>(std::distance(c1.begin(), c1.end())) == c1.size());
+    assert(static_cast<std::size_t>(std::distance(c1.cbegin(), c1.cend())) == c1.size());
+    assert(c1.max_load_factor() == 2);
+
+    assert(c2.bucket_count() >= 6);
+    assert(c2.size() == 6);
     {
-        typedef test_hash<int> Hash;
-        typedef test_equal_to<int> Compare;
-        typedef test_allocator<std::pair<const int, std::string> > Alloc;
-        typedef std::unordered_multimap<int, std::string, Hash, Compare, Alloc> C;
-        typedef std::pair<int, std::string> P;
-        P a1[] =
-        {
-            P(1, "one"),
-            P(2, "two"),
-            P(3, "three"),
-            P(4, "four"),
-            P(1, "four"),
-            P(2, "four"),
-        };
-        C c1(std::begin(a1), std::end(a1), 0, Hash(1), Compare(1), Alloc(1, 1));
-        C c2(0, Hash(2), Compare(2), Alloc(1, 2));
-        c2.max_load_factor(2);
-        c1.swap(c2);
-
-        LIBCPP_ASSERT(c1.bucket_count() == 0);
-        assert(c1.size() == 0);
-        assert(c1.hash_function() == Hash(2));
-        assert(c1.key_eq() == Compare(2));
-        assert(c1.get_allocator().get_id() == 1);
-        assert(static_cast<std::size_t>(std::distance(c1.begin(), c1.end())) == c1.size());
-        assert(static_cast<std::size_t>(std::distance(c1.cbegin(), c1.cend())) == c1.size());
-        assert(c1.max_load_factor() == 2);
-
-        assert(c2.bucket_count() >= 6);
-        assert(c2.size() == 6);
-        {
-            std::set<std::string> s;
-            s.insert("one");
-            s.insert("four");
-            assert(s.find(c2.find(1)->second) != s.end());
-            s.erase(s.find(c2.find(1)->second));
-            assert(s.find(std::next(c2.find(1))->second) != s.end());
-        }
-        {
-            std::set<std::string> s;
-            s.insert("two");
-            s.insert("four");
-            assert(s.find(c2.find(2)->second) != s.end());
-            s.erase(s.find(c2.find(2)->second));
-            assert(s.find(std::next(c2.find(2))->second) != s.end());
-        }
-        assert(c2.find(3)->second == "three");
-        assert(c2.find(4)->second == "four");
-        assert(c2.hash_function() == Hash(1));
-        assert(c2.key_eq() == Compare(1));
-        assert(c2.get_allocator().get_id() == 2);
-        assert(static_cast<std::size_t>(std::distance(c2.begin(), c2.end())) == c2.size());
-        assert(static_cast<std::size_t>(std::distance(c2.cbegin(), c2.cend())) == c2.size());
-        assert(c2.max_load_factor() == 1);
+      std::set<std::string> s;
+      s.insert("one");
+      s.insert("four");
+      assert(s.find(c2.find(1)->second) != s.end());
+      s.erase(s.find(c2.find(1)->second));
+      assert(s.find(std::next(c2.find(1))->second) != s.end());
     }
     {
-        typedef test_hash<int> Hash;
-        typedef test_equal_to<int> Compare;
-        typedef test_allocator<std::pair<const int, std::string> > Alloc;
-        typedef std::unordered_multimap<int, std::string, Hash, Compare, Alloc> C;
-        typedef std::pair<int, std::string> P;
-        P a1[] =
-        {
-            P(1, "one"),
-            P(2, "two"),
-            P(3, "three"),
-            P(4, "four"),
-            P(1, "four"),
-            P(2, "four"),
-        };
-        P a2[] =
-        {
-            P(10, "ten"),
-            P(20, "twenty"),
-            P(30, "thirty"),
-            P(40, "forty"),
-            P(50, "fifty"),
-            P(60, "sixty"),
-            P(70, "seventy"),
-            P(80, "eighty"),
-        };
-        C c1(std::begin(a1), std::end(a1), 0, Hash(1), Compare(1), Alloc(1, 1));
-        C c2(std::begin(a2), std::end(a2), 0, Hash(2), Compare(2), Alloc(1, 2));
-        c2.max_load_factor(2);
-        c1.swap(c2);
-
-        assert(c1.bucket_count() >= 8);
-        assert(c1.size() == 8);
-        assert(c1.find(10)->second == "ten");
-        assert(c1.find(20)->second == "twenty");
-        assert(c1.find(30)->second == "thirty");
-        assert(c1.find(40)->second == "forty");
-        assert(c1.find(50)->second == "fifty");
-        assert(c1.find(60)->second == "sixty");
-        assert(c1.find(70)->second == "seventy");
-        assert(c1.find(80)->second == "eighty");
-        assert(c1.hash_function() == Hash(2));
-        assert(c1.key_eq() == Compare(2));
-        assert(c1.get_allocator().get_id() == 1);
-        assert(static_cast<std::size_t>(std::distance(c1.begin(), c1.end())) == c1.size());
-        assert(static_cast<std::size_t>(std::distance(c1.cbegin(), c1.cend())) == c1.size());
-        assert(c1.max_load_factor() == 2);
-
-        assert(c2.bucket_count() >= 6);
-        assert(c2.size() == 6);
-        {
-            std::set<std::string> s;
-            s.insert("one");
-            s.insert("four");
-            assert(s.find(c2.find(1)->second) != s.end());
-            s.erase(s.find(c2.find(1)->second));
-            assert(s.find(std::next(c2.find(1))->second) != s.end());
-        }
-        {
-            std::set<std::string> s;
-            s.insert("two");
-            s.insert("four");
-            assert(s.find(c2.find(2)->second) != s.end());
-            s.erase(s.find(c2.find(2)->second));
-            assert(s.find(std::next(c2.find(2))->second) != s.end());
-        }
-        assert(c2.find(3)->second == "three");
-        assert(c2.find(4)->second == "four");
-        assert(c2.hash_function() == Hash(1));
-        assert(c2.key_eq() == Compare(1));
-        assert(c2.get_allocator().get_id() == 2);
-        assert(static_cast<std::size_t>(std::distance(c2.begin(), c2.end())) == c2.size());
-        assert(static_cast<std::size_t>(std::distance(c2.cbegin(), c2.cend())) == c2.size());
-        assert(c2.max_load_factor() == 1);
+      std::set<std::string> s;
+      s.insert("two");
+      s.insert("four");
+      assert(s.find(c2.find(2)->second) != s.end());
+      s.erase(s.find(c2.find(2)->second));
+      assert(s.find(std::next(c2.find(2))->second) != s.end());
     }
-
+    assert(c2.find(3)->second == "three");
+    assert(c2.find(4)->second == "four");
+    assert(c2.hash_function() == Hash(1));
+    assert(c2.key_eq() == Compare(1));
+    assert(c2.get_allocator().get_id() == 2);
+    assert(static_cast<std::size_t>(std::distance(c2.begin(), c2.end())) == c2.size());
+    assert(static_cast<std::size_t>(std::distance(c2.cbegin(), c2.cend())) == c2.size());
+    assert(c2.max_load_factor() == 1);
+  }
+
+  {
+    typedef test_hash<int> Hash;
+    typedef test_equal_to<int> Compare;
+    typedef other_allocator<std::pair<const int, std::string> > Alloc;
+    typedef std::unordered_multimap<int, std::string, Hash, Compare, Alloc> C;
+    C c1(0, Hash(1), Compare(1), Alloc(1));
+    C c2(0, Hash(2), Compare(2), Alloc(2));
+    c2.max_load_factor(2);
+    c1.swap(c2);
+
+    LIBCPP_ASSERT(c1.bucket_count() == 0);
+    assert(c1.size() == 0);
+    assert(c1.hash_function() == Hash(2));
+    assert(c1.key_eq() == Compare(2));
+    assert(c1.get_allocator() == Alloc(2));
+    assert(static_cast<std::size_t>(std::distance(c1.begin(), c1.end())) == c1.size());
+    assert(static_cast<std::size_t>(std::distance(c1.cbegin(), c1.cend())) == c1.size());
+    assert(c1.max_load_factor() == 2);
+
+    LIBCPP_ASSERT(c2.bucket_count() == 0);
+    assert(c2.size() == 0);
+    assert(c2.hash_function() == Hash(1));
+    assert(c2.key_eq() == Compare(1));
+    assert(c2.get_allocator() == Alloc(1));
+    assert(static_cast<std::size_t>(std::distance(c2.begin(), c2.end())) == c2.size());
+    assert(static_cast<std::size_t>(std::distance(c2.cbegin(), c2.cend())) == c2.size());
+    assert(c2.max_load_factor() == 1);
+  }
+  {
+    typedef test_hash<int> Hash;
+    typedef test_equal_to<int> Compare;
+    typedef other_allocator<std::pair<const int, std::string> > Alloc;
+    typedef std::unordered_multimap<int, std::string, Hash, Compare, Alloc> C;
+    typedef std::pair<int, std::string> P;
+    P a2[] = {
+        P(10, "ten"),
+        P(20, "twenty"),
+        P(30, "thirty"),
+        P(40, "forty"),
+        P(50, "fifty"),
+        P(60, "sixty"),
+        P(70, "seventy"),
+        P(80, "eighty"),
+    };
+    C c1(0, Hash(1), Compare(1), Alloc(1));
+    C c2(std::begin(a2), std::end(a2), 0, Hash(2), Compare(2), Alloc(2));
+    c2.max_load_factor(2);
+    c1.swap(c2);
+
+    assert(c1.bucket_count() >= 8);
+    assert(c1.size() == 8);
+    assert(c1.find(10)->second == "ten");
+    assert(c1.find(20)->second == "twenty");
+    assert(c1.find(30)->second == "thirty");
+    assert(c1.find(40)->second == "forty");
+    assert(c1.find(50)->second == "fifty");
+    assert(c1.find(60)->second == "sixty");
+    assert(c1.find(70)->second == "seventy");
+    assert(c1.find(80)->second == "eighty");
+    assert(c1.hash_function() == Hash(2));
+    assert(c1.key_eq() == Compare(2));
+    assert(c1.get_allocator() == Alloc(2));
+    assert(static_cast<std::size_t>(std::distance(c1.begin(), c1.end())) == c1.size());
+    assert(static_cast<std::size_t>(std::distance(c1.cbegin(), c1.cend())) == c1.size());
+    assert(c1.max_load_factor() == 2);
+
+    LIBCPP_ASSERT(c2.bucket_count() == 0);
+    assert(c2.size() == 0);
+    assert(c2.hash_function() == Hash(1));
+    assert(c2.key_eq() == Compare(1));
+    assert(c2.get_allocator() == Alloc(1));
+    assert(static_cast<std::size_t>(std::distance(c2.begin(), c2.end())) == c2.size());
+    assert(static_cast<std::size_t>(std::distance(c2.cbegin(), c2.cend())) == c2.size());
+    assert(c2.max_load_factor() == 1);
+  }
+  {
+    typedef test_hash<int> Hash;
+    typedef test_equal_to<int> Compare;
+    typedef other_allocator<std::pair<const int, std::string> > Alloc;
+    typedef std::unordered_multimap<int, std::string, Hash, Compare, Alloc> C;
+    typedef std::pair<int, std::string> P;
+    P a1[] = {
+        P(1, "one"),
+        P(2, "two"),
+        P(3, "three"),
+        P(4, "four"),
+        P(1, "four"),
+        P(2, "four"),
+    };
+    C c1(std::begin(a1), std::end(a1), 0, Hash(1), Compare(1), Alloc(1));
+    C c2(0, Hash(2), Compare(2), Alloc(2));
+    c2.max_load_factor(2);
+    c1.swap(c2);
+
+    LIBCPP_ASSERT(c1.bucket_count() == 0);
+    assert(c1.size() == 0);
+    assert(c1.hash_function() == Hash(2));
+    assert(c1.key_eq() == Compare(2));
+    assert(c1.get_allocator() == Alloc(2));
+    assert(static_cast<std::size_t>(std::distance(c1.begin(), c1.end())) == c1.size());
+    assert(static_cast<std::size_t>(std::distance(c1.cbegin(), c1.cend())) == c1.size());
+    assert(c1.max_load_factor() == 2);
+
+    assert(c2.bucket_count() >= 6);
+    assert(c2.size() == 6);
     {
-        typedef test_hash<int> Hash;
-        typedef test_equal_to<int> Compare;
-        typedef other_allocator<std::pair<const int, std::string> > Alloc;
-        typedef std::unordered_multimap<int, std::string, Hash, Compare, Alloc> C;
-        C c1(0, Hash(1), Compare(1), Alloc(1));
-        C c2(0, Hash(2), Compare(2), Alloc(2));
-        c2.max_load_factor(2);
-        c1.swap(c2);
-
-        LIBCPP_ASSERT(c1.bucket_count() == 0);
-        assert(c1.size() == 0);
-        assert(c1.hash_function() == Hash(2));
-        assert(c1.key_eq() == Compare(2));
-        assert(c1.get_allocator() == Alloc(2));
-        assert(static_cast<std::size_t>(std::distance(c1.begin(), c1.end())) == c1.size());
-        assert(static_cast<std::size_t>(std::distance(c1.cbegin(), c1.cend())) == c1.size());
-        assert(c1.max_load_factor() == 2);
-
-        LIBCPP_ASSERT(c2.bucket_count() == 0);
-        assert(c2.size() == 0);
-        assert(c2.hash_function() == Hash(1));
-        assert(c2.key_eq() == Compare(1));
-        assert(c2.get_allocator() == Alloc(1));
-        assert(static_cast<std::size_t>(std::distance(c2.begin(), c2.end())) == c2.size());
-        assert(static_cast<std::size_t>(std::distance(c2.cbegin(), c2.cend())) == c2.size());
-        assert(c2.max_load_factor() == 1);
+      std::set<std::string> s;
+      s.insert("one");
+      s.insert("four");
+      assert(s.find(c2.find(1)->second) != s.end());
+      s.erase(s.find(c2.find(1)->second));
+      assert(s.find(std::next(c2.find(1))->second) != s.end());
     }
     {
-        typedef test_hash<int> Hash;
-        typedef test_equal_to<int> Compare;
-        typedef other_allocator<std::pair<const int, std::string> > Alloc;
-        typedef std::unordered_multimap<int, std::string, Hash, Compare, Alloc> C;
-        typedef std::pair<int, std::string> P;
-        P a2[] =
-        {
-            P(10, "ten"),
-            P(20, "twenty"),
-            P(30, "thirty"),
-            P(40, "forty"),
-            P(50, "fifty"),
-            P(60, "sixty"),
-            P(70, "seventy"),
-            P(80, "eighty"),
-        };
-        C c1(0, Hash(1), Compare(1), Alloc(1));
-        C c2(std::begin(a2), std::end(a2), 0, Hash(2), Compare(2), Alloc(2));
-        c2.max_load_factor(2);
-        c1.swap(c2);
-
-        assert(c1.bucket_count() >= 8);
-        assert(c1.size() == 8);
-        assert(c1.find(10)->second == "ten");
-        assert(c1.find(20)->second == "twenty");
-        assert(c1.find(30)->second == "thirty");
-        assert(c1.find(40)->second == "forty");
-        assert(c1.find(50)->second == "fifty");
-        assert(c1.find(60)->second == "sixty");
-        assert(c1.find(70)->second == "seventy");
-        assert(c1.find(80)->second == "eighty");
-        assert(c1.hash_function() == Hash(2));
-        assert(c1.key_eq() == Compare(2));
-        assert(c1.get_allocator() == Alloc(2));
-        assert(static_cast<std::size_t>(std::distance(c1.begin(), c1.end())) == c1.size());
-        assert(static_cast<std::size_t>(std::distance(c1.cbegin(), c1.cend())) == c1.size());
-        assert(c1.max_load_factor() == 2);
-
-        LIBCPP_ASSERT(c2.bucket_count() == 0);
-        assert(c2.size() == 0);
-        assert(c2.hash_function() == Hash(1));
-        assert(c2.key_eq() == Compare(1));
-        assert(c2.get_allocator() == Alloc(1));
-        assert(static_cast<std::size_t>(std::distance(c2.begin(), c2.end())) == c2.size());
-        assert(static_cast<std::size_t>(std::distance(c2.cbegin(), c2.cend())) == c2.size());
-        assert(c2.max_load_factor() == 1);
+      std::set<std::string> s;
+      s.insert("two");
+      s.insert("four");
+      assert(s.find(c2.find(2)->second) != s.end());
+      s.erase(s.find(c2.find(2)->second));
+      assert(s.find(std::next(c2.find(2))->second) != s.end());
     }
+    assert(c2.find(3)->second == "three");
+    assert(c2.find(4)->second == "four");
+    assert(c2.hash_function() == Hash(1));
+    assert(c2.key_eq() == Compare(1));
+    assert(c2.get_allocator() == Alloc(1));
+    assert(static_cast<std::size_t>(std::distance(c2.begin(), c2.end())) == c2.size());
+    assert(static_cast<std::size_t>(std::distance(c2.cbegin(), c2.cend())) == c2.size());
+    assert(c2.max_load_factor() == 1);
+  }
+  {
+    typedef test_hash<int> Hash;
+    typedef test_equal_to<int> Compare;
+    typedef other_allocator<std::pair<const int, std::string> > Alloc;
+    typedef std::unordered_multimap<int, std::string, Hash, Compare, Alloc> C;
+    typedef std::pair<int, std::string> P;
+    P a1[] = {
+        P(1, "one"),
+        P(2, "two"),
+        P(3, "three"),
+        P(4, "four"),
+        P(1, "four"),
+        P(2, "four"),
+    };
+    P a2[] = {
+        P(10, "ten"),
+        P(20, "twenty"),
+        P(30, "thirty"),
+        P(40, "forty"),
+        P(50, "fifty"),
+        P(60, "sixty"),
+        P(70, "seventy"),
+        P(80, "eighty"),
+    };
+    C c1(std::begin(a1), std::end(a1), 0, Hash(1), Compare(1), Alloc(1));
+    C c2(std::begin(a2), std::end(a2), 0, Hash(2), Compare(2), Alloc(2));
+    c2.max_load_factor(2);
+    c1.swap(c2);
+
+    assert(c1.bucket_count() >= 8);
+    assert(c1.size() == 8);
+    assert(c1.find(10)->second == "ten");
+    assert(c1.find(20)->second == "twenty");
+    assert(c1.find(30)->second == "thirty");
+    assert(c1.find(40)->second == "forty");
+    assert(c1.find(50)->second == "fifty");
+    assert(c1.find(60)->second == "sixty");
+    assert(c1.find(70)->second == "seventy");
+    assert(c1.find(80)->second == "eighty");
+    assert(c1.hash_function() == Hash(2));
+    assert(c1.key_eq() == Compare(2));
+    assert(c1.get_allocator() == Alloc(2));
+    assert(static_cast<std::size_t>(std::distance(c1.begin(), c1.end())) == c1.size());
+    assert(static_cast<std::size_t>(std::distance(c1.cbegin(), c1.cend())) == c1.size());
+    assert(c1.max_load_factor() == 2);
+
+    assert(c2.bucket_count() >= 6);
+    assert(c2.size() == 6);
     {
-        typedef test_hash<int> Hash;
-        typedef test_equal_to<int> Compare;
-        typedef other_allocator<std::pair<const int, std::string> > Alloc;
-        typedef std::unordered_multimap<int, std::string, Hash, Compare, Alloc> C;
-        typedef std::pair<int, std::string> P;
-        P a1[] =
-        {
-            P(1, "one"),
-            P(2, "two"),
-            P(3, "three"),
-            P(4, "four"),
-            P(1, "four"),
-            P(2, "four"),
-        };
-        C c1(std::begin(a1), std::end(a1), 0, Hash(1), Compare(1), Alloc(1));
-        C c2(0, Hash(2), Compare(2), Alloc(2));
-        c2.max_load_factor(2);
-        c1.swap(c2);
-
-        LIBCPP_ASSERT(c1.bucket_count() == 0);
-        assert(c1.size() == 0);
-        assert(c1.hash_function() == Hash(2));
-        assert(c1.key_eq() == Compare(2));
-        assert(c1.get_allocator() == Alloc(2));
-        assert(static_cast<std::size_t>(std::distance(c1.begin(), c1.end())) == c1.size());
-        assert(static_cast<std::size_t>(std::distance(c1.cbegin(), c1.cend())) == c1.size());
-        assert(c1.max_load_factor() == 2);
-
-        assert(c2.bucket_count() >= 6);
-        assert(c2.size() == 6);
-        {
-            std::set<std::string> s;
-            s.insert("one");
-            s.insert("four");
-            assert(s.find(c2.find(1)->second) != s.end());
-            s.erase(s.find(c2.find(1)->second));
-            assert(s.find(std::next(c2.find(1))->second) != s.end());
-        }
-        {
-            std::set<std::string> s;
-            s.insert("two");
-            s.insert("four");
-            assert(s.find(c2.find(2)->second) != s.end());
-            s.erase(s.find(c2.find(2)->second));
-            assert(s.find(std::next(c2.find(2))->second) != s.end());
-        }
-        assert(c2.find(3)->second == "three");
-        assert(c2.find(4)->second == "four");
-        assert(c2.hash_function() == Hash(1));
-        assert(c2.key_eq() == Compare(1));
-        assert(c2.get_allocator() == Alloc(1));
-        assert(static_cast<std::size_t>(std::distance(c2.begin(), c2.end())) == c2.size());
-        assert(static_cast<std::size_t>(std::distance(c2.cbegin(), c2.cend())) == c2.size());
-        assert(c2.max_load_factor() == 1);
+      std::set<std::string> s;
+      s.insert("one");
+      s.insert("four");
+      assert(s.find(c2.find(1)->second) != s.end());
+      s.erase(s.find(c2.find(1)->second));
+      assert(s.find(std::next(c2.find(1))->second) != s.end());
     }
     {
-        typedef test_hash<int> Hash;
-        typedef test_equal_to<int> Compare;
-        typedef other_allocator<std::pair<const int, std::string> > Alloc;
-        typedef std::unordered_multimap<int, std::string, Hash, Compare, Alloc> C;
-        typedef std::pair<int, std::string> P;
-        P a1[] =
-        {
-            P(1, "one"),
-            P(2, "two"),
-            P(3, "three"),
-            P(4, "four"),
-            P(1, "four"),
-            P(2, "four"),
-        };
-        P a2[] =
-        {
-            P(10, "ten"),
-            P(20, "twenty"),
-            P(30, "thirty"),
-            P(40, "forty"),
-            P(50, "fifty"),
-            P(60, "sixty"),
-            P(70, "seventy"),
-            P(80, "eighty"),
-        };
-        C c1(std::begin(a1), std::end(a1), 0, Hash(1), Compare(1), Alloc(1));
-        C c2(std::begin(a2), std::end(a2), 0, Hash(2), Compare(2), Alloc(2));
-        c2.max_load_factor(2);
-        c1.swap(c2);
-
-        assert(c1.bucket_count() >= 8);
-        assert(c1.size() == 8);
-        assert(c1.find(10)->second == "ten");
-        assert(c1.find(20)->second == "twenty");
-        assert(c1.find(30)->second == "thirty");
-        assert(c1.find(40)->second == "forty");
-        assert(c1.find(50)->second == "fifty");
-        assert(c1.find(60)->second == "sixty");
-        assert(c1.find(70)->second == "seventy");
-        assert(c1.find(80)->second == "eighty");
-        assert(c1.hash_function() == Hash(2));
-        assert(c1.key_eq() == Compare(2));
-        assert(c1.get_allocator() == Alloc(2));
-        assert(static_cast<std::size_t>(std::distance(c1.begin(), c1.end())) == c1.size());
-        assert(static_cast<std::size_t>(std::distance(c1.cbegin(), c1.cend())) == c1.size());
-        assert(c1.max_load_factor() == 2);
-
-        assert(c2.bucket_count() >= 6);
-        assert(c2.size() == 6);
-        {
-            std::set<std::string> s;
-            s.insert("one");
-            s.insert("four");
-            assert(s.find(c2.find(1)->second) != s.end());
-            s.erase(s.find(c2.find(1)->second));
-            assert(s.find(std::next(c2.find(1))->second) != s.end());
-        }
-        {
-            std::set<std::string> s;
-            s.insert("two");
-            s.insert("four");
-            assert(s.find(c2.find(2)->second) != s.end());
-            s.erase(s.find(c2.find(2)->second));
-            assert(s.find(std::next(c2.find(2))->second) != s.end());
-        }
-        assert(c2.find(3)->second == "three");
-        assert(c2.find(4)->second == "four");
-        assert(c2.hash_function() == Hash(1));
-        assert(c2.key_eq() == Compare(1));
-        assert(c2.get_allocator() == Alloc(1));
-        assert(static_cast<std::size_t>(std::distance(c2.begin(), c2.end())) == c2.size());
-        assert(static_cast<std::size_t>(std::distance(c2.cbegin(), c2.cend())) == c2.size());
-        assert(c2.max_load_factor() == 1);
+      std::set<std::string> s;
+      s.insert("two");
+      s.insert("four");
+      assert(s.find(c2.find(2)->second) != s.end());
+      s.erase(s.find(c2.find(2)->second));
+      assert(s.find(std::next(c2.find(2))->second) != s.end());
     }
+    assert(c2.find(3)->second == "three");
+    assert(c2.find(4)->second == "four");
+    assert(c2.hash_function() == Hash(1));
+    assert(c2.key_eq() == Compare(1));
+    assert(c2.get_allocator() == Alloc(1));
+    assert(static_cast<std::size_t>(std::distance(c2.begin(), c2.end())) == c2.size());
+    assert(static_cast<std::size_t>(std::distance(c2.cbegin(), c2.cend())) == c2.size());
+    assert(c2.max_load_factor() == 1);
+  }
 #if TEST_STD_VER >= 11
+  {
+    typedef test_hash<int> Hash;
+    typedef test_equal_to<int> Compare;
+    typedef min_allocator<std::pair<const int, std::string> > Alloc;
+    typedef std::unordered_multimap<int, std::string, Hash, Compare, Alloc> C;
+    C c1(0, Hash(1), Compare(1), Alloc());
+    C c2(0, Hash(2), Compare(2), Alloc());
+    c2.max_load_factor(2);
+    c1.swap(c2);
+
+    LIBCPP_ASSERT(c1.bucket_count() == 0);
+    assert(c1.size() == 0);
+    assert(c1.hash_function() == Hash(2));
+    assert(c1.key_eq() == Compare(2));
+    assert(c1.get_allocator() == Alloc());
+    assert(static_cast<std::size_t>(std::distance(c1.begin(), c1.end())) == c1.size());
+    assert(static_cast<std::size_t>(std::distance(c1.cbegin(), c1.cend())) == c1.size());
+    assert(c1.max_load_factor() == 2);
+
+    LIBCPP_ASSERT(c2.bucket_count() == 0);
+    assert(c2.size() == 0);
+    assert(c2.hash_function() == Hash(1));
+    assert(c2.key_eq() == Compare(1));
+    assert(c2.get_allocator() == Alloc());
+    assert(static_cast<std::size_t>(std::distance(c2.begin(), c2.end())) == c2.size());
+    assert(static_cast<std::size_t>(std::distance(c2.cbegin(), c2.cend())) == c2.size());
+    assert(c2.max_load_factor() == 1);
+  }
+  {
+    typedef test_hash<int> Hash;
+    typedef test_equal_to<int> Compare;
+    typedef min_allocator<std::pair<const int, std::string> > Alloc;
+    typedef std::unordered_multimap<int, std::string, Hash, Compare, Alloc> C;
+    typedef std::pair<int, std::string> P;
+    P a2[] = {
+        P(10, "ten"),
+        P(20, "twenty"),
+        P(30, "thirty"),
+        P(40, "forty"),
+        P(50, "fifty"),
+        P(60, "sixty"),
+        P(70, "seventy"),
+        P(80, "eighty"),
+    };
+    C c1(0, Hash(1), Compare(1), Alloc());
+    C c2(std::begin(a2), std::end(a2), 0, Hash(2), Compare(2), Alloc());
+    c2.max_load_factor(2);
+    c1.swap(c2);
+
+    assert(c1.bucket_count() >= 8);
+    assert(c1.size() == 8);
+    assert(c1.find(10)->second == "ten");
+    assert(c1.find(20)->second == "twenty");
+    assert(c1.find(30)->second == "thirty");
+    assert(c1.find(40)->second == "forty");
+    assert(c1.find(50)->second == "fifty");
+    assert(c1.find(60)->second == "sixty");
+    assert(c1.find(70)->second == "seventy");
+    assert(c1.find(80)->second == "eighty");
+    assert(c1.hash_function() == Hash(2));
+    assert(c1.key_eq() == Compare(2));
+    assert(c1.get_allocator() == Alloc());
+    assert(static_cast<std::size_t>(std::distance(c1.begin(), c1.end())) == c1.size());
+    assert(static_cast<std::size_t>(std::distance(c1.cbegin(), c1.cend())) == c1.size());
+    assert(c1.max_load_factor() == 2);
+
+    LIBCPP_ASSERT(c2.bucket_count() == 0);
+    assert(c2.size() == 0);
+    assert(c2.hash_function() == Hash(1));
+    assert(c2.key_eq() == Compare(1));
+    assert(c2.get_allocator() == Alloc());
+    assert(static_cast<std::size_t>(std::distance(c2.begin(), c2.end())) == c2.size());
+    assert(static_cast<std::size_t>(std::distance(c2.cbegin(), c2.cend())) == c2.size());
+    assert(c2.max_load_factor() == 1);
+  }
+  {
+    typedef test_hash<int> Hash;
+    typedef test_equal_to<int> Compare;
+    typedef min_allocator<std::pair<const int, std::string> > Alloc;
+    typedef std::unordered_multimap<int, std::string, Hash, Compare, Alloc> C;
+    typedef std::pair<int, std::string> P;
+    P a1[] = {
+        P(1, "one"),
+        P(2, "two"),
+        P(3, "three"),
+        P(4, "four"),
+        P(1, "four"),
+        P(2, "four"),
+    };
+    C c1(std::begin(a1), std::end(a1), 0, Hash(1), Compare(1), Alloc());
+    C c2(0, Hash(2), Compare(2), Alloc());
+    c2.max_load_factor(2);
+    c1.swap(c2);
+
+    LIBCPP_ASSERT(c1.bucket_count() == 0);
+    assert(c1.size() == 0);
+    assert(c1.hash_function() == Hash(2));
+    assert(c1.key_eq() == Compare(2));
+    assert(c1.get_allocator() == Alloc());
+    assert(static_cast<std::size_t>(std::distance(c1.begin(), c1.end())) == c1.size());
+    assert(static_cast<std::size_t>(std::distance(c1.cbegin(), c1.cend())) == c1.size());
+    assert(c1.max_load_factor() == 2);
+
+    assert(c2.bucket_count() >= 6);
+    assert(c2.size() == 6);
     {
-        typedef test_hash<int> Hash;
-        typedef test_equal_to<int> Compare;
-        typedef min_allocator<std::pair<const int, std::string> > Alloc;
-        typedef std::unordered_multimap<int, std::string, Hash, Compare, Alloc> C;
-        C c1(0, Hash(1), Compare(1), Alloc());
-        C c2(0, Hash(2), Compare(2), Alloc());
-        c2.max_load_factor(2);
-        c1.swap(c2);
-
-        LIBCPP_ASSERT(c1.bucket_count() == 0);
-        assert(c1.size() == 0);
-        assert(c1.hash_function() == Hash(2));
-        assert(c1.key_eq() == Compare(2));
-        assert(c1.get_allocator() == Alloc());
-        assert(static_cast<std::size_t>(std::distance(c1.begin(), c1.end())) == c1.size());
-        assert(static_cast<std::size_t>(std::distance(c1.cbegin(), c1.cend())) == c1.size());
-        assert(c1.max_load_factor() == 2);
-
-        LIBCPP_ASSERT(c2.bucket_count() == 0);
-        assert(c2.size() == 0);
-        assert(c2.hash_function() == Hash(1));
-        assert(c2.key_eq() == Compare(1));
-        assert(c2.get_allocator() == Alloc());
-        assert(static_cast<std::size_t>(std::distance(c2.begin(), c2.end())) == c2.size());
-        assert(static_cast<std::size_t>(std::distance(c2.cbegin(), c2.cend())) == c2.size());
-        assert(c2.max_load_factor() == 1);
+      std::set<std::string> s;
+      s.insert("one");
+      s.insert("four");
+      assert(s.find(c2.find(1)->second) != s.end());
+      s.erase(s.find(c2.find(1)->second));
+      assert(s.find(std::next(c2.find(1))->second) != s.end());
     }
     {
-        typedef test_hash<int> Hash;
-        typedef test_equal_to<int> Compare;
-        typedef min_allocator<std::pair<const int, std::string> > Alloc;
-        typedef std::unordered_multimap<int, std::string, Hash, Compare, Alloc> C;
-        typedef std::pair<int, std::string> P;
-        P a2[] =
-        {
-            P(10, "ten"),
-            P(20, "twenty"),
-            P(30, "thirty"),
-            P(40, "forty"),
-            P(50, "fifty"),
-            P(60, "sixty"),
-            P(70, "seventy"),
-            P(80, "eighty"),
-        };
-        C c1(0, Hash(1), Compare(1), Alloc());
-        C c2(std::begin(a2), std::end(a2), 0, Hash(2), Compare(2), Alloc());
-        c2.max_load_factor(2);
-        c1.swap(c2);
-
-        assert(c1.bucket_count() >= 8);
-        assert(c1.size() == 8);
-        assert(c1.find(10)->second == "ten");
-        assert(c1.find(20)->second == "twenty");
-        assert(c1.find(30)->second == "thirty");
-        assert(c1.find(40)->second == "forty");
-        assert(c1.find(50)->second == "fifty");
-        assert(c1.find(60)->second == "sixty");
-        assert(c1.find(70)->second == "seventy");
-        assert(c1.find(80)->second == "eighty");
-        assert(c1.hash_function() == Hash(2));
-        assert(c1.key_eq() == Compare(2));
-        assert(c1.get_allocator() == Alloc());
-        assert(static_cast<std::size_t>(std::distance(c1.begin(), c1.end())) == c1.size());
-        assert(static_cast<std::size_t>(std::distance(c1.cbegin(), c1.cend())) == c1.size());
-        assert(c1.max_load_factor() == 2);
-
-        LIBCPP_ASSERT(c2.bucket_count() == 0);
-        assert(c2.size() == 0);
-        assert(c2.hash_function() == Hash(1));
-        assert(c2.key_eq() == Compare(1));
-        assert(c2.get_allocator() == Alloc());
-        assert(static_cast<std::size_t>(std::distance(c2.begin(), c2.end())) == c2.size());
-        assert(static_cast<std::size_t>(std::distance(c2.cbegin(), c2.cend())) == c2.size());
-        assert(c2.max_load_factor() == 1);
+      std::set<std::string> s;
+      s.insert("two");
+      s.insert("four");
+      assert(s.find(c2.find(2)->second) != s.end());
+      s.erase(s.find(c2.find(2)->second));
+      assert(s.find(std::next(c2.find(2))->second) != s.end());
     }
+    assert(c2.find(3)->second == "three");
+    assert(c2.find(4)->second == "four");
+    assert(c2.hash_function() == Hash(1));
+    assert(c2.key_eq() == Compare(1));
+    assert(c2.get_allocator() == Alloc());
+    assert(static_cast<std::size_t>(std::distance(c2.begin(), c2.end())) == c2.size());
+    assert(static_cast<std::size_t>(std::distance(c2.cbegin(), c2.cend())) == c2.size());
+    assert(c2.max_load_factor() == 1);
+  }
+  {
+    typedef test_hash<int> Hash;
+    typedef test_equal_to<int> Compare;
+    typedef min_allocator<std::pair<const int, std::string> > Alloc;
+    typedef std::unordered_multimap<int, std::string, Hash, Compare, Alloc> C;
+    typedef std::pair<int, std::string> P;
+    P a1[] = {
+        P(1, "one"),
+        P(2, "two"),
+        P(3, "three"),
+        P(4, "four"),
+        P(1, "four"),
+        P(2, "four"),
+    };
+    P a2[] = {
+        P(10, "ten"),
+        P(20, "twenty"),
+        P(30, "thirty"),
+        P(40, "forty"),
+        P(50, "fifty"),
+        P(60, "sixty"),
+        P(70, "seventy"),
+        P(80, "eighty"),
+    };
+    C c1(std::begin(a1), std::end(a1), 0, Hash(1), Compare(1), Alloc());
+    C c2(std::begin(a2), std::end(a2), 0, Hash(2), Compare(2), Alloc());
+    c2.max_load_factor(2);
+    c1.swap(c2);
+
+    assert(c1.bucket_count() >= 8);
+    assert(c1.size() == 8);
+    assert(c1.find(10)->second == "ten");
+    assert(c1.find(20)->second == "twenty");
+    assert(c1.find(30)->second == "thirty");
+    assert(c1.find(40)->second == "forty");
+    assert(c1.find(50)->second == "fifty");
+    assert(c1.find(60)->second == "sixty");
+    assert(c1.find(70)->second == "seventy");
+    assert(c1.find(80)->second == "eighty");
+    assert(c1.hash_function() == Hash(2));
+    assert(c1.key_eq() == Compare(2));
+    assert(c1.get_allocator() == Alloc());
+    assert(static_cast<std::size_t>(std::distance(c1.begin(), c1.end())) == c1.size());
+    assert(static_cast<std::size_t>(std::distance(c1.cbegin(), c1.cend())) == c1.size());
+    assert(c1.max_load_factor() == 2);
+
+    assert(c2.bucket_count() >= 6);
+    assert(c2.size() == 6);
     {
-        typedef test_hash<int> Hash;
-        typedef test_equal_to<int> Compare;
-        typedef min_allocator<std::pair<const int, std::string> > Alloc;
-        typedef std::unordered_multimap<int, std::string, Hash, Compare, Alloc> C;
-        typedef std::pair<int, std::string> P;
-        P a1[] =
-        {
-            P(1, "one"),
-            P(2, "two"),
-            P(3, "three"),
-            P(4, "four"),
-            P(1, "four"),
-            P(2, "four"),
-        };
-        C c1(std::begin(a1), std::end(a1), 0, Hash(1), Compare(1), Alloc());
-        C c2(0, Hash(2), Compare(2), Alloc());
-        c2.max_load_factor(2);
-        c1.swap(c2);
-
-        LIBCPP_ASSERT(c1.bucket_count() == 0);
-        assert(c1.size() == 0);
-        assert(c1.hash_function() == Hash(2));
-        assert(c1.key_eq() == Compare(2));
-        assert(c1.get_allocator() == Alloc());
-        assert(static_cast<std::size_t>(std::distance(c1.begin(), c1.end())) == c1.size());
-        assert(static_cast<std::size_t>(std::distance(c1.cbegin(), c1.cend())) == c1.size());
-        assert(c1.max_load_factor() == 2);
-
-        assert(c2.bucket_count() >= 6);
-        assert(c2.size() == 6);
-        {
-            std::set<std::string> s;
-            s.insert("one");
-            s.insert("four");
-            assert(s.find(c2.find(1)->second) != s.end());
-            s.erase(s.find(c2.find(1)->second));
-            assert(s.find(std::next(c2.find(1))->second) != s.end());
-        }
-        {
-            std::set<std::string> s;
-            s.insert("two");
-            s.insert("four");
-            assert(s.find(c2.find(2)->second) != s.end());
-            s.erase(s.find(c2.find(2)->second));
-            assert(s.find(std::next(c2.find(2))->second) != s.end());
-        }
-        assert(c2.find(3)->second == "three");
-        assert(c2.find(4)->second == "four");
-        assert(c2.hash_function() == Hash(1));
-        assert(c2.key_eq() == Compare(1));
-        assert(c2.get_allocator() == Alloc());
-        assert(static_cast<std::size_t>(std::distance(c2.begin(), c2.end())) == c2.size());
-        assert(static_cast<std::size_t>(std::distance(c2.cbegin(), c2.cend())) == c2.size());
-        assert(c2.max_load_factor() == 1);
+      std::set<std::string> s;
+      s.insert("one");
+      s.insert("four");
+      assert(s.find(c2.find(1)->second) != s.end());
+      s.erase(s.find(c2.find(1)->second));
+      assert(s.find(std::next(c2.find(1))->second) != s.end());
     }
     {
-        typedef test_hash<int> Hash;
-        typedef test_equal_to<int> Compare;
-        typedef min_allocator<std::pair<const int, std::string> > Alloc;
-        typedef std::unordered_multimap<int, std::string, Hash, Compare, Alloc> C;
-        typedef std::pair<int, std::string> P;
-        P a1[] =
-        {
-            P(1, "one"),
-            P(2, "two"),
-            P(3, "three"),
-            P(4, "four"),
-            P(1, "four"),
-            P(2, "four"),
-        };
-        P a2[] =
-        {
-            P(10, "ten"),
-            P(20, "twenty"),
-            P(30, "thirty"),
-            P(40, "forty"),
-            P(50, "fifty"),
-            P(60, "sixty"),
-            P(70, "seventy"),
-            P(80, "eighty"),
-        };
-        C c1(std::begin(a1), std::end(a1), 0, Hash(1), Compare(1), Alloc());
-        C c2(std::begin(a2), std::end(a2), 0, Hash(2), Compare(2), Alloc());
-        c2.max_load_factor(2);
-        c1.swap(c2);
-
-        assert(c1.bucket_count() >= 8);
-        assert(c1.size() == 8);
-        assert(c1.find(10)->second == "ten");
-        assert(c1.find(20)->second == "twenty");
-        assert(c1.find(30)->second == "thirty");
-        assert(c1.find(40)->second == "forty");
-        assert(c1.find(50)->second == "fifty");
-        assert(c1.find(60)->second == "sixty");
-        assert(c1.find(70)->second == "seventy");
-        assert(c1.find(80)->second == "eighty");
-        assert(c1.hash_function() == Hash(2));
-        assert(c1.key_eq() == Compare(2));
-        assert(c1.get_allocator() == Alloc());
-        assert(static_cast<std::size_t>(std::distance(c1.begin(), c1.end())) == c1.size());
-        assert(static_cast<std::size_t>(std::distance(c1.cbegin(), c1.cend())) == c1.size());
-        assert(c1.max_load_factor() == 2);
-
-        assert(c2.bucket_count() >= 6);
-        assert(c2.size() == 6);
-        {
-            std::set<std::string> s;
-            s.insert("one");
-            s.insert("four");
-            assert(s.find(c2.find(1)->second) != s.end());
-            s.erase(s.find(c2.find(1)->second));
-            assert(s.find(std::next(c2.find(1))->second) != s.end());
-        }
-        {
-            std::set<std::string> s;
-            s.insert("two");
-            s.insert("four");
-            assert(s.find(c2.find(2)->second) != s.end());
-            s.erase(s.find(c2.find(2)->second));
-            assert(s.find(std::next(c2.find(2))->second) != s.end());
-        }
-        assert(c2.find(3)->second == "three");
-        assert(c2.find(4)->second == "four");
-        assert(c2.hash_function() == Hash(1));
-        assert(c2.key_eq() == Compare(1));
-        assert(c2.get_allocator() == Alloc());
-        assert(static_cast<std::size_t>(std::distance(c2.begin(), c2.end())) == c2.size());
-        assert(static_cast<std::size_t>(std::distance(c2.cbegin(), c2.cend())) == c2.size());
-        assert(c2.max_load_factor() == 1);
+      std::set<std::string> s;
+      s.insert("two");
+      s.insert("four");
+      assert(s.find(c2.find(2)->second) != s.end());
+      s.erase(s.find(c2.find(2)->second));
+      assert(s.find(std::next(c2.find(2))->second) != s.end());
     }
+    assert(c2.find(3)->second == "three");
+    assert(c2.find(4)->second == "four");
+    assert(c2.hash_function() == Hash(1));
+    assert(c2.key_eq() == Compare(1));
+    assert(c2.get_allocator() == Alloc());
+    assert(static_cast<std::size_t>(std::distance(c2.begin(), c2.end())) == c2.size());
+    assert(static_cast<std::size_t>(std::distance(c2.cbegin(), c2.cend())) == c2.size());
+    assert(c2.max_load_factor() == 1);
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/unord/unord.multimap/types.pass.cpp b/libcxx/test/std/containers/unord/unord.multimap/types.pass.cpp
index 5af2f68b61686..e9f5878646dc8 100644
--- a/libcxx/test/std/containers/unord/unord.multimap/types.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.multimap/types.pass.cpp
@@ -33,41 +33,44 @@
 #include "test_macros.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef std::unordered_multimap<char, short> C;
-        static_assert((std::is_same<C::key_type, char>::value), "");
-        static_assert((std::is_same<C::mapped_type, short>::value), "");
-        static_assert((std::is_same<C::hasher, std::hash<C::key_type> >::value), "");
-        static_assert((std::is_same<C::key_equal, std::equal_to<C::key_type> >::value), "");
-        static_assert((std::is_same<C::allocator_type, std::allocator<C::value_type> >::value), "");
-        static_assert((std::is_same<C::value_type, std::pair<const C::key_type, C::mapped_type> >::value), "");
-        static_assert((std::is_same<C::reference, C::value_type&>::value), "");
-        static_assert((std::is_same<C::const_reference, const C::value_type&>::value), "");
-        static_assert((std::is_same<C::pointer, C::value_type*>::value), "");
-        static_assert((std::is_same<C::const_pointer, const C::value_type*>::value), "");
-        static_assert((std::is_same<C::size_type, std::size_t>::value), "");
-        static_assert((std::is_same<C::difference_type, std::ptrdiff_t>::value), "");
-    }
+int main(int, char**) {
+  {
+    typedef std::unordered_multimap<char, short> C;
+    static_assert((std::is_same<C::key_type, char>::value), "");
+    static_assert((std::is_same<C::mapped_type, short>::value), "");
+    static_assert((std::is_same<C::hasher, std::hash<C::key_type> >::value), "");
+    static_assert((std::is_same<C::key_equal, std::equal_to<C::key_type> >::value), "");
+    static_assert((std::is_same<C::allocator_type, std::allocator<C::value_type> >::value), "");
+    static_assert((std::is_same<C::value_type, std::pair<const C::key_type, C::mapped_type> >::value), "");
+    static_assert((std::is_same<C::reference, C::value_type&>::value), "");
+    static_assert((std::is_same<C::const_reference, const C::value_type&>::value), "");
+    static_assert((std::is_same<C::pointer, C::value_type*>::value), "");
+    static_assert((std::is_same<C::const_pointer, const C::value_type*>::value), "");
+    static_assert((std::is_same<C::size_type, std::size_t>::value), "");
+    static_assert((std::is_same<C::difference_type, std::ptrdiff_t>::value), "");
+  }
 #if TEST_STD_VER >= 11
-    {
-        typedef std::unordered_multimap<char, short, std::hash<char>, std::equal_to<char>,
-                            min_allocator<std::pair<const char, short>>> C;
-        static_assert((std::is_same<C::key_type, char>::value), "");
-        static_assert((std::is_same<C::mapped_type, short>::value), "");
-        static_assert((std::is_same<C::hasher, std::hash<C::key_type> >::value), "");
-        static_assert((std::is_same<C::key_equal, std::equal_to<C::key_type> >::value), "");
-        static_assert((std::is_same<C::allocator_type, min_allocator<C::value_type> >::value), "");
-        static_assert((std::is_same<C::value_type, std::pair<const C::key_type, C::mapped_type> >::value), "");
-        static_assert((std::is_same<C::reference, C::value_type&>::value), "");
-        static_assert((std::is_same<C::const_reference, const C::value_type&>::value), "");
-        static_assert((std::is_same<C::pointer, min_pointer<C::value_type>>::value), "");
-        static_assert((std::is_same<C::const_pointer, min_pointer<const C::value_type>>::value), "");
+  {
+    typedef std::unordered_multimap<char,
+                                    short,
+                                    std::hash<char>,
+                                    std::equal_to<char>,
+                                    min_allocator<std::pair<const char, short>>>
+        C;
+    static_assert((std::is_same<C::key_type, char>::value), "");
+    static_assert((std::is_same<C::mapped_type, short>::value), "");
+    static_assert((std::is_same<C::hasher, std::hash<C::key_type> >::value), "");
+    static_assert((std::is_same<C::key_equal, std::equal_to<C::key_type> >::value), "");
+    static_assert((std::is_same<C::allocator_type, min_allocator<C::value_type> >::value), "");
+    static_assert((std::is_same<C::value_type, std::pair<const C::key_type, C::mapped_type> >::value), "");
+    static_assert((std::is_same<C::reference, C::value_type&>::value), "");
+    static_assert((std::is_same<C::const_reference, const C::value_type&>::value), "");
+    static_assert((std::is_same<C::pointer, min_pointer<C::value_type>>::value), "");
+    static_assert((std::is_same<C::const_pointer, min_pointer<const C::value_type>>::value), "");
     //  min_allocator doesn't have a size_type, so one gets synthesized
-        static_assert((std::is_same<C::size_type, std::make_unsigned<C::difference_type>::type>::value), "");
-        static_assert((std::is_same<C::difference_type, std::ptrdiff_t>::value), "");
-    }
+    static_assert((std::is_same<C::size_type, std::make_unsigned<C::difference_type>::type>::value), "");
+    static_assert((std::is_same<C::difference_type, std::ptrdiff_t>::value), "");
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.cnstr/allocator.pass.cpp b/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.cnstr/allocator.pass.cpp
index de69e6743067d..9610408f25351 100644
--- a/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.cnstr/allocator.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.cnstr/allocator.pass.cpp
@@ -24,107 +24,105 @@
 #include "test_allocator.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef std::unordered_multimap<NotConstructible, NotConstructible,
-                                   test_hash<NotConstructible>,
-                                   test_equal_to<NotConstructible>,
-                                   test_allocator<std::pair<const NotConstructible,
-                                                                  NotConstructible> >
-                                   > C;
-        C c(test_allocator<std::pair<const NotConstructible, NotConstructible> >(10));
-        LIBCPP_ASSERT(c.bucket_count() == 0);
-        assert(c.hash_function() == test_hash<NotConstructible>());
-        assert(c.key_eq() == test_equal_to<NotConstructible>());
-        assert(c.get_allocator() ==
-               (test_allocator<std::pair<const NotConstructible, NotConstructible> >(10)));
-        assert(c.size() == 0);
-        assert(c.empty());
-        assert(std::distance(c.begin(), c.end()) == 0);
-        assert(c.load_factor() == 0);
-        assert(c.max_load_factor() == 1);
-    }
+int main(int, char**) {
+  {
+    typedef std::unordered_multimap<NotConstructible,
+                                    NotConstructible,
+                                    test_hash<NotConstructible>,
+                                    test_equal_to<NotConstructible>,
+                                    test_allocator<std::pair<const NotConstructible, NotConstructible> > >
+        C;
+    C c(test_allocator<std::pair<const NotConstructible, NotConstructible> >(10));
+    LIBCPP_ASSERT(c.bucket_count() == 0);
+    assert(c.hash_function() == test_hash<NotConstructible>());
+    assert(c.key_eq() == test_equal_to<NotConstructible>());
+    assert(c.get_allocator() == (test_allocator<std::pair<const NotConstructible, NotConstructible> >(10)));
+    assert(c.size() == 0);
+    assert(c.empty());
+    assert(std::distance(c.begin(), c.end()) == 0);
+    assert(c.load_factor() == 0);
+    assert(c.max_load_factor() == 1);
+  }
 #if TEST_STD_VER >= 11
-    {
-        typedef std::unordered_multimap<NotConstructible, NotConstructible,
-                                   test_hash<NotConstructible>,
-                                   test_equal_to<NotConstructible>,
-                                   min_allocator<std::pair<const NotConstructible,
-                                                                  NotConstructible> >
-                                   > C;
-        C c(min_allocator<std::pair<const NotConstructible, NotConstructible> >{});
-        LIBCPP_ASSERT(c.bucket_count() == 0);
-        assert(c.hash_function() == test_hash<NotConstructible>());
-        assert(c.key_eq() == test_equal_to<NotConstructible>());
-        assert(c.get_allocator() ==
-               (min_allocator<std::pair<const NotConstructible, NotConstructible> >()));
-        assert(c.size() == 0);
-        assert(c.empty());
-        assert(std::distance(c.begin(), c.end()) == 0);
-        assert(c.load_factor() == 0);
-        assert(c.max_load_factor() == 1);
-    }
-    {
-        typedef explicit_allocator<std::pair<const NotConstructible, NotConstructible>> A;
-        typedef std::unordered_multimap<NotConstructible, NotConstructible,
-                                   test_hash<NotConstructible>,
-                                   test_equal_to<NotConstructible>,
-                                   A
-                                   > C;
-        C c(A{});
-        LIBCPP_ASSERT(c.bucket_count() == 0);
-        assert(c.hash_function() == test_hash<NotConstructible>());
-        assert(c.key_eq() == test_equal_to<NotConstructible>());
-        assert(c.get_allocator() == A{});
-        assert(c.size() == 0);
-        assert(c.empty());
-        assert(std::distance(c.begin(), c.end()) == 0);
-        assert(c.load_factor() == 0);
-        assert(c.max_load_factor() == 1);
-    }
-#if TEST_STD_VER > 11
-    {
-        typedef NotConstructible T;
-        typedef test_allocator<std::pair<const T, T>> A;
-        typedef test_hash<T> HF;
-        typedef test_equal_to<T> Comp;
-        typedef std::unordered_multimap<T, T, HF, Comp, A> C;
+  {
+    typedef std::unordered_multimap<NotConstructible,
+                                    NotConstructible,
+                                    test_hash<NotConstructible>,
+                                    test_equal_to<NotConstructible>,
+                                    min_allocator<std::pair<const NotConstructible, NotConstructible> > >
+        C;
+    C c(min_allocator<std::pair<const NotConstructible, NotConstructible> >{});
+    LIBCPP_ASSERT(c.bucket_count() == 0);
+    assert(c.hash_function() == test_hash<NotConstructible>());
+    assert(c.key_eq() == test_equal_to<NotConstructible>());
+    assert(c.get_allocator() == (min_allocator<std::pair<const NotConstructible, NotConstructible> >()));
+    assert(c.size() == 0);
+    assert(c.empty());
+    assert(std::distance(c.begin(), c.end()) == 0);
+    assert(c.load_factor() == 0);
+    assert(c.max_load_factor() == 1);
+  }
+  {
+    typedef explicit_allocator<std::pair<const NotConstructible, NotConstructible>> A;
+    typedef std::unordered_multimap<NotConstructible,
+                                    NotConstructible,
+                                    test_hash<NotConstructible>,
+                                    test_equal_to<NotConstructible>,
+                                    A >
+        C;
+    C c(A{});
+    LIBCPP_ASSERT(c.bucket_count() == 0);
+    assert(c.hash_function() == test_hash<NotConstructible>());
+    assert(c.key_eq() == test_equal_to<NotConstructible>());
+    assert(c.get_allocator() == A{});
+    assert(c.size() == 0);
+    assert(c.empty());
+    assert(std::distance(c.begin(), c.end()) == 0);
+    assert(c.load_factor() == 0);
+    assert(c.max_load_factor() == 1);
+  }
+#  if TEST_STD_VER > 11
+  {
+    typedef NotConstructible T;
+    typedef test_allocator<std::pair<const T, T>> A;
+    typedef test_hash<T> HF;
+    typedef test_equal_to<T> Comp;
+    typedef std::unordered_multimap<T, T, HF, Comp, A> C;
 
-        A a(10);
-        C c(2, a);
-        LIBCPP_ASSERT(c.bucket_count() == 2);
-        assert(c.hash_function() == HF());
-        assert(c.key_eq() == Comp());
-        assert(c.get_allocator() == a);
-        assert(c.size() == 0);
-        assert(c.empty());
-        assert(std::distance(c.begin(), c.end()) == 0);
-        assert(c.load_factor() == 0);
-        assert(c.max_load_factor() == 1);
-    }
-    {
-        typedef NotConstructible T;
-        typedef test_allocator<std::pair<const T, T>> A;
-        typedef test_hash<T> HF;
-        typedef test_equal_to<T> Comp;
-        typedef std::unordered_multimap<T, T, HF, Comp, A> C;
+    A a(10);
+    C c(2, a);
+    LIBCPP_ASSERT(c.bucket_count() == 2);
+    assert(c.hash_function() == HF());
+    assert(c.key_eq() == Comp());
+    assert(c.get_allocator() == a);
+    assert(c.size() == 0);
+    assert(c.empty());
+    assert(std::distance(c.begin(), c.end()) == 0);
+    assert(c.load_factor() == 0);
+    assert(c.max_load_factor() == 1);
+  }
+  {
+    typedef NotConstructible T;
+    typedef test_allocator<std::pair<const T, T>> A;
+    typedef test_hash<T> HF;
+    typedef test_equal_to<T> Comp;
+    typedef std::unordered_multimap<T, T, HF, Comp, A> C;
 
-        A a(10);
-        HF hf(12);
-        C c(2, hf, a);
-        LIBCPP_ASSERT(c.bucket_count() == 2);
-        assert(c.hash_function() == hf);
-        assert(!(c.hash_function() == HF()));
-        assert(c.key_eq() == Comp());
-        assert(c.get_allocator() == a);
-        assert(c.size() == 0);
-        assert(c.empty());
-        assert(std::distance(c.begin(), c.end()) == 0);
-        assert(c.load_factor() == 0);
-        assert(c.max_load_factor() == 1);
-    }
-#endif
+    A a(10);
+    HF hf(12);
+    C c(2, hf, a);
+    LIBCPP_ASSERT(c.bucket_count() == 2);
+    assert(c.hash_function() == hf);
+    assert(!(c.hash_function() == HF()));
+    assert(c.key_eq() == Comp());
+    assert(c.get_allocator() == a);
+    assert(c.size() == 0);
+    assert(c.empty());
+    assert(std::distance(c.begin(), c.end()) == 0);
+    assert(c.load_factor() == 0);
+    assert(c.max_load_factor() == 1);
+  }
+#  endif
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.cnstr/assign_copy.pass.cpp b/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.cnstr/assign_copy.pass.cpp
index d35d00d4de1a0..8897b6abd1439 100644
--- a/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.cnstr/assign_copy.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.cnstr/assign_copy.pass.cpp
@@ -30,183 +30,136 @@
 #include "test_allocator.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef test_allocator<std::pair<const int, std::string> > A;
-        typedef std::unordered_multimap<int, std::string,
-                                   test_hash<int>,
-                                   test_equal_to<int>,
-                                   A
-                                   > C;
-        typedef std::pair<int, std::string> P;
-        P a[] =
-        {
-            P(1, "one"),
-            P(2, "two"),
-            P(3, "three"),
-            P(4, "four"),
-            P(1, "four"),
-            P(2, "four"),
-        };
-        C c0(a, a + sizeof(a)/sizeof(a[0]),
-            7,
-            test_hash<int>(8),
-            test_equal_to<int>(9),
-            A(10)
-           );
-        C c(a, a + 2,
-            7,
-            test_hash<int>(2),
-            test_equal_to<int>(3),
-            A(4)
-           );
-        c = c0;
-        LIBCPP_ASSERT(c.bucket_count() == 7);
-        assert(c.size() == 6);
-        std::multiset<std::string> s;
-        s.insert("one");
-        s.insert("four");
-        CheckConsecutiveKeys<C::const_iterator>(c.find(1), c.end(), 1, s);
-        s.insert("two");
-        s.insert("four");
-        CheckConsecutiveKeys<C::const_iterator>(c.find(2), c.end(), 2, s);
-        s.insert("three");
-        CheckConsecutiveKeys<C::const_iterator>(c.find(3), c.end(), 3, s);
-        s.insert("four");
-        CheckConsecutiveKeys<C::const_iterator>(c.find(4), c.end(), 4, s);
-        assert(c.hash_function() == test_hash<int>(8));
-        assert(c.key_eq() == test_equal_to<int>(9));
-        assert(c.get_allocator() == A(4));
-        assert(!c.empty());
-        assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-        assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
-        assert(fabs(c.load_factor() - (float)c.size()/c.bucket_count()) < FLT_EPSILON);
-        assert(c.max_load_factor() == 1);
-    }
-    {
-        typedef std::unordered_multimap<int, std::string> C;
-        typedef std::pair<const int, std::string> P;
-        const P a[] =
-        {
-            P(1, "one"),
-            P(2, "two"),
-            P(3, "three"),
-            P(4, "four"),
-            P(1, "four"),
-            P(2, "four"),
-        };
-        C c(a, a+sizeof(a)/sizeof(a[0]));
-        C *p = &c;
-        c = *p;
-        assert(c.size() == 6);
-        assert(std::is_permutation(c.begin(), c.end(), a));
-    }
-    {
-        typedef other_allocator<std::pair<const int, std::string> > A;
-        typedef std::unordered_multimap<int, std::string,
-                                   test_hash<int>,
-                                   test_equal_to<int>,
-                                   A
-                                   > C;
-        typedef std::pair<int, std::string> P;
-        P a[] =
-        {
-            P(1, "one"),
-            P(2, "two"),
-            P(3, "three"),
-            P(4, "four"),
-            P(1, "four"),
-            P(2, "four"),
-        };
-        C c0(a, a + sizeof(a)/sizeof(a[0]),
-            7,
-            test_hash<int>(8),
-            test_equal_to<int>(9),
-            A(10)
-           );
-        C c(a, a + 2,
-            7,
-            test_hash<int>(2),
-            test_equal_to<int>(3),
-            A(4)
-           );
-        c = c0;
-        assert(c.bucket_count() >= 7);
-        assert(c.size() == 6);
-        std::multiset<std::string> s;
-        s.insert("one");
-        s.insert("four");
-        CheckConsecutiveKeys<C::const_iterator>(c.find(1), c.end(), 1, s);
-        s.insert("two");
-        s.insert("four");
-        CheckConsecutiveKeys<C::const_iterator>(c.find(2), c.end(), 2, s);
-        s.insert("three");
-        CheckConsecutiveKeys<C::const_iterator>(c.find(3), c.end(), 3, s);
-        s.insert("four");
-        CheckConsecutiveKeys<C::const_iterator>(c.find(4), c.end(), 4, s);
-        assert(c.hash_function() == test_hash<int>(8));
-        assert(c.key_eq() == test_equal_to<int>(9));
-        assert(c.get_allocator() == A(10));
-        assert(!c.empty());
-        assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-        assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
-        assert(fabs(c.load_factor() - (float)c.size()/c.bucket_count()) < FLT_EPSILON);
-        assert(c.max_load_factor() == 1);
-    }
+int main(int, char**) {
+  {
+    typedef test_allocator<std::pair<const int, std::string> > A;
+    typedef std::unordered_multimap<int, std::string, test_hash<int>, test_equal_to<int>, A > C;
+    typedef std::pair<int, std::string> P;
+    P a[] = {
+        P(1, "one"),
+        P(2, "two"),
+        P(3, "three"),
+        P(4, "four"),
+        P(1, "four"),
+        P(2, "four"),
+    };
+    C c0(a, a + sizeof(a) / sizeof(a[0]), 7, test_hash<int>(8), test_equal_to<int>(9), A(10));
+    C c(a, a + 2, 7, test_hash<int>(2), test_equal_to<int>(3), A(4));
+    c = c0;
+    LIBCPP_ASSERT(c.bucket_count() == 7);
+    assert(c.size() == 6);
+    std::multiset<std::string> s;
+    s.insert("one");
+    s.insert("four");
+    CheckConsecutiveKeys<C::const_iterator>(c.find(1), c.end(), 1, s);
+    s.insert("two");
+    s.insert("four");
+    CheckConsecutiveKeys<C::const_iterator>(c.find(2), c.end(), 2, s);
+    s.insert("three");
+    CheckConsecutiveKeys<C::const_iterator>(c.find(3), c.end(), 3, s);
+    s.insert("four");
+    CheckConsecutiveKeys<C::const_iterator>(c.find(4), c.end(), 4, s);
+    assert(c.hash_function() == test_hash<int>(8));
+    assert(c.key_eq() == test_equal_to<int>(9));
+    assert(c.get_allocator() == A(4));
+    assert(!c.empty());
+    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
+    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
+    assert(fabs(c.load_factor() - (float)c.size() / c.bucket_count()) < FLT_EPSILON);
+    assert(c.max_load_factor() == 1);
+  }
+  {
+    typedef std::unordered_multimap<int, std::string> C;
+    typedef std::pair<const int, std::string> P;
+    const P a[] = {
+        P(1, "one"),
+        P(2, "two"),
+        P(3, "three"),
+        P(4, "four"),
+        P(1, "four"),
+        P(2, "four"),
+    };
+    C c(a, a + sizeof(a) / sizeof(a[0]));
+    C* p = &c;
+    c    = *p;
+    assert(c.size() == 6);
+    assert(std::is_permutation(c.begin(), c.end(), a));
+  }
+  {
+    typedef other_allocator<std::pair<const int, std::string> > A;
+    typedef std::unordered_multimap<int, std::string, test_hash<int>, test_equal_to<int>, A > C;
+    typedef std::pair<int, std::string> P;
+    P a[] = {
+        P(1, "one"),
+        P(2, "two"),
+        P(3, "three"),
+        P(4, "four"),
+        P(1, "four"),
+        P(2, "four"),
+    };
+    C c0(a, a + sizeof(a) / sizeof(a[0]), 7, test_hash<int>(8), test_equal_to<int>(9), A(10));
+    C c(a, a + 2, 7, test_hash<int>(2), test_equal_to<int>(3), A(4));
+    c = c0;
+    assert(c.bucket_count() >= 7);
+    assert(c.size() == 6);
+    std::multiset<std::string> s;
+    s.insert("one");
+    s.insert("four");
+    CheckConsecutiveKeys<C::const_iterator>(c.find(1), c.end(), 1, s);
+    s.insert("two");
+    s.insert("four");
+    CheckConsecutiveKeys<C::const_iterator>(c.find(2), c.end(), 2, s);
+    s.insert("three");
+    CheckConsecutiveKeys<C::const_iterator>(c.find(3), c.end(), 3, s);
+    s.insert("four");
+    CheckConsecutiveKeys<C::const_iterator>(c.find(4), c.end(), 4, s);
+    assert(c.hash_function() == test_hash<int>(8));
+    assert(c.key_eq() == test_equal_to<int>(9));
+    assert(c.get_allocator() == A(10));
+    assert(!c.empty());
+    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
+    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
+    assert(fabs(c.load_factor() - (float)c.size() / c.bucket_count()) < FLT_EPSILON);
+    assert(c.max_load_factor() == 1);
+  }
 #if TEST_STD_VER >= 11
-    {
-        typedef min_allocator<std::pair<const int, std::string> > A;
-        typedef std::unordered_multimap<int, std::string,
-                                   test_hash<int>,
-                                   test_equal_to<int>,
-                                   A
-                                   > C;
-        typedef std::pair<int, std::string> P;
-        P a[] =
-        {
-            P(1, "one"),
-            P(2, "two"),
-            P(3, "three"),
-            P(4, "four"),
-            P(1, "four"),
-            P(2, "four"),
-        };
-        C c0(a, a + sizeof(a)/sizeof(a[0]),
-            7,
-            test_hash<int>(8),
-            test_equal_to<int>(9),
-            A()
-           );
-        C c(a, a + 2,
-            7,
-            test_hash<int>(2),
-            test_equal_to<int>(3),
-            A()
-           );
-        c = c0;
-        LIBCPP_ASSERT(c.bucket_count() == 7);
-        assert(c.size() == 6);
-        std::multiset<std::string> s;
-        s.insert("one");
-        s.insert("four");
-        CheckConsecutiveKeys<C::const_iterator>(c.find(1), c.end(), 1, s);
-        s.insert("two");
-        s.insert("four");
-        CheckConsecutiveKeys<C::const_iterator>(c.find(2), c.end(), 2, s);
-        s.insert("three");
-        CheckConsecutiveKeys<C::const_iterator>(c.find(3), c.end(), 3, s);
-        s.insert("four");
-        CheckConsecutiveKeys<C::const_iterator>(c.find(4), c.end(), 4, s);
-        assert(c.hash_function() == test_hash<int>(8));
-        assert(c.key_eq() == test_equal_to<int>(9));
-        assert(c.get_allocator() == A());
-        assert(!c.empty());
-        assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-        assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
-        assert(fabs(c.load_factor() - (float)c.size()/c.bucket_count()) < FLT_EPSILON);
-        assert(c.max_load_factor() == 1);
-    }
+  {
+    typedef min_allocator<std::pair<const int, std::string> > A;
+    typedef std::unordered_multimap<int, std::string, test_hash<int>, test_equal_to<int>, A > C;
+    typedef std::pair<int, std::string> P;
+    P a[] = {
+        P(1, "one"),
+        P(2, "two"),
+        P(3, "three"),
+        P(4, "four"),
+        P(1, "four"),
+        P(2, "four"),
+    };
+    C c0(a, a + sizeof(a) / sizeof(a[0]), 7, test_hash<int>(8), test_equal_to<int>(9), A());
+    C c(a, a + 2, 7, test_hash<int>(2), test_equal_to<int>(3), A());
+    c = c0;
+    LIBCPP_ASSERT(c.bucket_count() == 7);
+    assert(c.size() == 6);
+    std::multiset<std::string> s;
+    s.insert("one");
+    s.insert("four");
+    CheckConsecutiveKeys<C::const_iterator>(c.find(1), c.end(), 1, s);
+    s.insert("two");
+    s.insert("four");
+    CheckConsecutiveKeys<C::const_iterator>(c.find(2), c.end(), 2, s);
+    s.insert("three");
+    CheckConsecutiveKeys<C::const_iterator>(c.find(3), c.end(), 3, s);
+    s.insert("four");
+    CheckConsecutiveKeys<C::const_iterator>(c.find(4), c.end(), 4, s);
+    assert(c.hash_function() == test_hash<int>(8));
+    assert(c.key_eq() == test_equal_to<int>(9));
+    assert(c.get_allocator() == A());
+    assert(!c.empty());
+    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
+    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
+    assert(fabs(c.load_factor() - (float)c.size() / c.bucket_count()) < FLT_EPSILON);
+    assert(c.max_load_factor() == 1);
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.cnstr/assign_init.pass.cpp b/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.cnstr/assign_init.pass.cpp
index 9eb3fa3b13b80..785fbe7019c0b 100644
--- a/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.cnstr/assign_init.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.cnstr/assign_init.pass.cpp
@@ -31,110 +31,101 @@
 #include "test_allocator.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef test_allocator<std::pair<const int, std::string> > A;
-        typedef std::unordered_multimap<int, std::string,
-                                   test_hash<int>,
-                                   test_equal_to<int>,
-                                   A
-                                   > C;
-        typedef std::pair<int, std::string> P;
-        C c =   {
-                    P(4, "four"),
-                    P(1, "four"),
-                    P(2, "four"),
-                };
-        c =     {
-                    P(1, "one"),
-                    P(2, "two"),
-                    P(3, "three"),
-                    P(4, "four"),
-                    P(1, "four"),
-                    P(2, "four"),
-                };
-        assert(c.bucket_count() >= 7);
-        assert(c.size() == 6);
-        typedef std::pair<C::const_iterator, C::const_iterator> Eq;
-        Eq eq = c.equal_range(1);
-        assert(std::distance(eq.first, eq.second) == 2);
-        std::multiset<std::string> s;
-        s.insert("one");
-        s.insert("four");
-        CheckConsecutiveKeys<C::const_iterator>(c.find(1), c.end(), 1, s);
-        eq = c.equal_range(2);
-        assert(std::distance(eq.first, eq.second) == 2);
-        s.insert("two");
-        s.insert("four");
-        CheckConsecutiveKeys<C::const_iterator>(c.find(2), c.end(), 2, s);
+int main(int, char**) {
+  {
+    typedef test_allocator<std::pair<const int, std::string> > A;
+    typedef std::unordered_multimap<int, std::string, test_hash<int>, test_equal_to<int>, A > C;
+    typedef std::pair<int, std::string> P;
+    C c = {
+        P(4, "four"),
+        P(1, "four"),
+        P(2, "four"),
+    };
+    c = {
+        P(1, "one"),
+        P(2, "two"),
+        P(3, "three"),
+        P(4, "four"),
+        P(1, "four"),
+        P(2, "four"),
+    };
+    assert(c.bucket_count() >= 7);
+    assert(c.size() == 6);
+    typedef std::pair<C::const_iterator, C::const_iterator> Eq;
+    Eq eq = c.equal_range(1);
+    assert(std::distance(eq.first, eq.second) == 2);
+    std::multiset<std::string> s;
+    s.insert("one");
+    s.insert("four");
+    CheckConsecutiveKeys<C::const_iterator>(c.find(1), c.end(), 1, s);
+    eq = c.equal_range(2);
+    assert(std::distance(eq.first, eq.second) == 2);
+    s.insert("two");
+    s.insert("four");
+    CheckConsecutiveKeys<C::const_iterator>(c.find(2), c.end(), 2, s);
 
-        eq = c.equal_range(3);
-        assert(std::distance(eq.first, eq.second) == 1);
-        C::const_iterator i = eq.first;
-        assert(i->first == 3);
-        assert(i->second == "three");
-        eq = c.equal_range(4);
-        assert(std::distance(eq.first, eq.second) == 1);
-        i = eq.first;
-        assert(i->first == 4);
-        assert(i->second == "four");
-        assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-        assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
-        assert(fabs(c.load_factor() - (float)c.size()/c.bucket_count()) < FLT_EPSILON);
-        assert(c.max_load_factor() == 1);
-    }
-    {
-        typedef min_allocator<std::pair<const int, std::string> > A;
-        typedef std::unordered_multimap<int, std::string,
-                                   test_hash<int>,
-                                   test_equal_to<int>,
-                                   A
-                                   > C;
-        typedef std::pair<int, std::string> P;
-        C c =   {
-                    P(4, "four"),
-                    P(1, "four"),
-                    P(2, "four"),
-                };
-        c =     {
-                    P(1, "one"),
-                    P(2, "two"),
-                    P(3, "three"),
-                    P(4, "four"),
-                    P(1, "four"),
-                    P(2, "four"),
-                };
-        assert(c.bucket_count() >= 7);
-        assert(c.size() == 6);
-        typedef std::pair<C::const_iterator, C::const_iterator> Eq;
-        Eq eq = c.equal_range(1);
-        assert(std::distance(eq.first, eq.second) == 2);
-        std::multiset<std::string> s;
-        s.insert("one");
-        s.insert("four");
-        CheckConsecutiveKeys<C::const_iterator>(c.find(1), c.end(), 1, s);
-        eq = c.equal_range(2);
-        assert(std::distance(eq.first, eq.second) == 2);
-        s.insert("two");
-        s.insert("four");
-        CheckConsecutiveKeys<C::const_iterator>(c.find(2), c.end(), 2, s);
+    eq = c.equal_range(3);
+    assert(std::distance(eq.first, eq.second) == 1);
+    C::const_iterator i = eq.first;
+    assert(i->first == 3);
+    assert(i->second == "three");
+    eq = c.equal_range(4);
+    assert(std::distance(eq.first, eq.second) == 1);
+    i = eq.first;
+    assert(i->first == 4);
+    assert(i->second == "four");
+    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
+    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
+    assert(fabs(c.load_factor() - (float)c.size() / c.bucket_count()) < FLT_EPSILON);
+    assert(c.max_load_factor() == 1);
+  }
+  {
+    typedef min_allocator<std::pair<const int, std::string> > A;
+    typedef std::unordered_multimap<int, std::string, test_hash<int>, test_equal_to<int>, A > C;
+    typedef std::pair<int, std::string> P;
+    C c = {
+        P(4, "four"),
+        P(1, "four"),
+        P(2, "four"),
+    };
+    c = {
+        P(1, "one"),
+        P(2, "two"),
+        P(3, "three"),
+        P(4, "four"),
+        P(1, "four"),
+        P(2, "four"),
+    };
+    assert(c.bucket_count() >= 7);
+    assert(c.size() == 6);
+    typedef std::pair<C::const_iterator, C::const_iterator> Eq;
+    Eq eq = c.equal_range(1);
+    assert(std::distance(eq.first, eq.second) == 2);
+    std::multiset<std::string> s;
+    s.insert("one");
+    s.insert("four");
+    CheckConsecutiveKeys<C::const_iterator>(c.find(1), c.end(), 1, s);
+    eq = c.equal_range(2);
+    assert(std::distance(eq.first, eq.second) == 2);
+    s.insert("two");
+    s.insert("four");
+    CheckConsecutiveKeys<C::const_iterator>(c.find(2), c.end(), 2, s);
 
-        eq = c.equal_range(3);
-        assert(std::distance(eq.first, eq.second) == 1);
-        C::const_iterator i = eq.first;
-        assert(i->first == 3);
-        assert(i->second == "three");
-        eq = c.equal_range(4);
-        assert(std::distance(eq.first, eq.second) == 1);
-        i = eq.first;
-        assert(i->first == 4);
-        assert(i->second == "four");
-        assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-        assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
-        assert(fabs(c.load_factor() - (float)c.size()/c.bucket_count()) < FLT_EPSILON);
-        assert(c.max_load_factor() == 1);
-    }
+    eq = c.equal_range(3);
+    assert(std::distance(eq.first, eq.second) == 1);
+    C::const_iterator i = eq.first;
+    assert(i->first == 3);
+    assert(i->second == "three");
+    eq = c.equal_range(4);
+    assert(std::distance(eq.first, eq.second) == 1);
+    i = eq.first;
+    assert(i->first == 4);
+    assert(i->second == "four");
+    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
+    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
+    assert(fabs(c.load_factor() - (float)c.size() / c.bucket_count()) < FLT_EPSILON);
+    assert(c.max_load_factor() == 1);
+  }
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.cnstr/assign_move.pass.cpp b/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.cnstr/assign_move.pass.cpp
index c4ba56781e55d..799278e7a4aae 100644
--- a/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.cnstr/assign_move.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.cnstr/assign_move.pass.cpp
@@ -31,254 +31,193 @@
 #include "test_allocator.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef test_allocator<std::pair<const int, std::string> > A;
-        typedef std::unordered_multimap<int, std::string,
-                                   test_hash<int>,
-                                   test_equal_to<int>,
-                                   A
-                                   > C;
-        typedef std::pair<int, std::string> P;
-        P a[] =
-        {
-            P(1, "one"),
-            P(2, "two"),
-            P(3, "three"),
-            P(4, "four"),
-            P(1, "four"),
-            P(2, "four"),
-        };
-        C c0(a, a + sizeof(a)/sizeof(a[0]),
-            7,
-            test_hash<int>(8),
-            test_equal_to<int>(9),
-            A(10)
-           );
-        C c(a, a + 2,
-            7,
-            test_hash<int>(2),
-            test_equal_to<int>(3),
-            A(4)
-           );
-        c = std::move(c0);
-        LIBCPP_ASSERT(c.bucket_count() == 7);
-        assert(c.size() == 6);
-        typedef std::pair<C::const_iterator, C::const_iterator> Eq;
-        Eq eq = c.equal_range(1);
-        assert(std::distance(eq.first, eq.second) == 2);
-        std::multiset<std::string> s;
-        s.insert("one");
-        s.insert("four");
-        CheckConsecutiveKeys<C::const_iterator>(c.find(1), c.end(), 1, s);
-        eq = c.equal_range(2);
-        assert(std::distance(eq.first, eq.second) == 2);
-        s.insert("two");
-        s.insert("four");
-        CheckConsecutiveKeys<C::const_iterator>(c.find(2), c.end(), 2, s);
+int main(int, char**) {
+  {
+    typedef test_allocator<std::pair<const int, std::string> > A;
+    typedef std::unordered_multimap<int, std::string, test_hash<int>, test_equal_to<int>, A > C;
+    typedef std::pair<int, std::string> P;
+    P a[] = {
+        P(1, "one"),
+        P(2, "two"),
+        P(3, "three"),
+        P(4, "four"),
+        P(1, "four"),
+        P(2, "four"),
+    };
+    C c0(a, a + sizeof(a) / sizeof(a[0]), 7, test_hash<int>(8), test_equal_to<int>(9), A(10));
+    C c(a, a + 2, 7, test_hash<int>(2), test_equal_to<int>(3), A(4));
+    c = std::move(c0);
+    LIBCPP_ASSERT(c.bucket_count() == 7);
+    assert(c.size() == 6);
+    typedef std::pair<C::const_iterator, C::const_iterator> Eq;
+    Eq eq = c.equal_range(1);
+    assert(std::distance(eq.first, eq.second) == 2);
+    std::multiset<std::string> s;
+    s.insert("one");
+    s.insert("four");
+    CheckConsecutiveKeys<C::const_iterator>(c.find(1), c.end(), 1, s);
+    eq = c.equal_range(2);
+    assert(std::distance(eq.first, eq.second) == 2);
+    s.insert("two");
+    s.insert("four");
+    CheckConsecutiveKeys<C::const_iterator>(c.find(2), c.end(), 2, s);
 
-        eq = c.equal_range(3);
-        assert(std::distance(eq.first, eq.second) == 1);
-        C::const_iterator i = eq.first;
-        assert(i->first == 3);
-        assert(i->second == "three");
-        eq = c.equal_range(4);
-        assert(std::distance(eq.first, eq.second) == 1);
-        i = eq.first;
-        assert(i->first == 4);
-        assert(i->second == "four");
-        assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-        assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
-        assert(fabs(c.load_factor() - (float)c.size()/c.bucket_count()) < FLT_EPSILON);
-        assert(c.max_load_factor() == 1);
-    }
-    {
-        typedef test_allocator<std::pair<const int, std::string> > A;
-        typedef std::unordered_multimap<int, std::string,
-                                   test_hash<int>,
-                                   test_equal_to<int>,
-                                   A
-                                   > C;
-        typedef std::pair<int, std::string> P;
-        P a[] =
-        {
-            P(1, "one"),
-            P(2, "two"),
-            P(3, "three"),
-            P(4, "four"),
-            P(1, "four"),
-            P(2, "four"),
-        };
-        C c0(a, a + sizeof(a)/sizeof(a[0]),
-            7,
-            test_hash<int>(8),
-            test_equal_to<int>(9),
-            A(10)
-           );
-        C c(a, a + 2,
-            7,
-            test_hash<int>(2),
-            test_equal_to<int>(3),
-            A(10)
-           );
-        C::iterator it0 = c0.begin();
-        c = std::move(c0);
-        assert(it0 == c.begin()); // Iterators remain valid
-        LIBCPP_ASSERT(c.bucket_count() == 7);
-        assert(c.size() == 6);
-        typedef std::pair<C::const_iterator, C::const_iterator> Eq;
-        Eq eq = c.equal_range(1);
-        assert(std::distance(eq.first, eq.second) == 2);
-        std::multiset<std::string> s;
-        s.insert("one");
-        s.insert("four");
-        CheckConsecutiveKeys<C::const_iterator>(c.find(1), c.end(), 1, s);
-        eq = c.equal_range(2);
-        assert(std::distance(eq.first, eq.second) == 2);
-        s.insert("two");
-        s.insert("four");
-        CheckConsecutiveKeys<C::const_iterator>(c.find(2), c.end(), 2, s);
+    eq = c.equal_range(3);
+    assert(std::distance(eq.first, eq.second) == 1);
+    C::const_iterator i = eq.first;
+    assert(i->first == 3);
+    assert(i->second == "three");
+    eq = c.equal_range(4);
+    assert(std::distance(eq.first, eq.second) == 1);
+    i = eq.first;
+    assert(i->first == 4);
+    assert(i->second == "four");
+    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
+    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
+    assert(fabs(c.load_factor() - (float)c.size() / c.bucket_count()) < FLT_EPSILON);
+    assert(c.max_load_factor() == 1);
+  }
+  {
+    typedef test_allocator<std::pair<const int, std::string> > A;
+    typedef std::unordered_multimap<int, std::string, test_hash<int>, test_equal_to<int>, A > C;
+    typedef std::pair<int, std::string> P;
+    P a[] = {
+        P(1, "one"),
+        P(2, "two"),
+        P(3, "three"),
+        P(4, "four"),
+        P(1, "four"),
+        P(2, "four"),
+    };
+    C c0(a, a + sizeof(a) / sizeof(a[0]), 7, test_hash<int>(8), test_equal_to<int>(9), A(10));
+    C c(a, a + 2, 7, test_hash<int>(2), test_equal_to<int>(3), A(10));
+    C::iterator it0 = c0.begin();
+    c               = std::move(c0);
+    assert(it0 == c.begin()); // Iterators remain valid
+    LIBCPP_ASSERT(c.bucket_count() == 7);
+    assert(c.size() == 6);
+    typedef std::pair<C::const_iterator, C::const_iterator> Eq;
+    Eq eq = c.equal_range(1);
+    assert(std::distance(eq.first, eq.second) == 2);
+    std::multiset<std::string> s;
+    s.insert("one");
+    s.insert("four");
+    CheckConsecutiveKeys<C::const_iterator>(c.find(1), c.end(), 1, s);
+    eq = c.equal_range(2);
+    assert(std::distance(eq.first, eq.second) == 2);
+    s.insert("two");
+    s.insert("four");
+    CheckConsecutiveKeys<C::const_iterator>(c.find(2), c.end(), 2, s);
 
-        eq = c.equal_range(3);
-        assert(std::distance(eq.first, eq.second) == 1);
-        C::const_iterator i = eq.first;
-        assert(i->first == 3);
-        assert(i->second == "three");
-        eq = c.equal_range(4);
-        assert(std::distance(eq.first, eq.second) == 1);
-        i = eq.first;
-        assert(i->first == 4);
-        assert(i->second == "four");
-        assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-        assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
-        assert(fabs(c.load_factor() - (float)c.size()/c.bucket_count()) < FLT_EPSILON);
-        assert(c.max_load_factor() == 1);
-    }
-    {
-        typedef other_allocator<std::pair<const int, std::string> > A;
-        typedef std::unordered_multimap<int, std::string,
-                                   test_hash<int>,
-                                   test_equal_to<int>,
-                                   A
-                                   > C;
-        typedef std::pair<int, std::string> P;
-        P a[] =
-        {
-            P(1, "one"),
-            P(2, "two"),
-            P(3, "three"),
-            P(4, "four"),
-            P(1, "four"),
-            P(2, "four"),
-        };
-        C c0(a, a + sizeof(a)/sizeof(a[0]),
-            7,
-            test_hash<int>(8),
-            test_equal_to<int>(9),
-            A(10)
-           );
-        C c(a, a + 2,
-            7,
-            test_hash<int>(2),
-            test_equal_to<int>(3),
-            A(4)
-           );
-        C::iterator it0 = c0.begin();
-        c = std::move(c0);
-        assert(it0 == c.begin()); // Iterators remain valid
-        LIBCPP_ASSERT(c.bucket_count() == 7);
-        assert(c.size() == 6);
-        typedef std::pair<C::const_iterator, C::const_iterator> Eq;
-        Eq eq = c.equal_range(1);
-        assert(std::distance(eq.first, eq.second) == 2);
-        std::multiset<std::string> s;
-        s.insert("one");
-        s.insert("four");
-        CheckConsecutiveKeys<C::const_iterator>(c.find(1), c.end(), 1, s);
-        eq = c.equal_range(2);
-        assert(std::distance(eq.first, eq.second) == 2);
-        s.insert("two");
-        s.insert("four");
-        CheckConsecutiveKeys<C::const_iterator>(c.find(2), c.end(), 2, s);
+    eq = c.equal_range(3);
+    assert(std::distance(eq.first, eq.second) == 1);
+    C::const_iterator i = eq.first;
+    assert(i->first == 3);
+    assert(i->second == "three");
+    eq = c.equal_range(4);
+    assert(std::distance(eq.first, eq.second) == 1);
+    i = eq.first;
+    assert(i->first == 4);
+    assert(i->second == "four");
+    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
+    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
+    assert(fabs(c.load_factor() - (float)c.size() / c.bucket_count()) < FLT_EPSILON);
+    assert(c.max_load_factor() == 1);
+  }
+  {
+    typedef other_allocator<std::pair<const int, std::string> > A;
+    typedef std::unordered_multimap<int, std::string, test_hash<int>, test_equal_to<int>, A > C;
+    typedef std::pair<int, std::string> P;
+    P a[] = {
+        P(1, "one"),
+        P(2, "two"),
+        P(3, "three"),
+        P(4, "four"),
+        P(1, "four"),
+        P(2, "four"),
+    };
+    C c0(a, a + sizeof(a) / sizeof(a[0]), 7, test_hash<int>(8), test_equal_to<int>(9), A(10));
+    C c(a, a + 2, 7, test_hash<int>(2), test_equal_to<int>(3), A(4));
+    C::iterator it0 = c0.begin();
+    c               = std::move(c0);
+    assert(it0 == c.begin()); // Iterators remain valid
+    LIBCPP_ASSERT(c.bucket_count() == 7);
+    assert(c.size() == 6);
+    typedef std::pair<C::const_iterator, C::const_iterator> Eq;
+    Eq eq = c.equal_range(1);
+    assert(std::distance(eq.first, eq.second) == 2);
+    std::multiset<std::string> s;
+    s.insert("one");
+    s.insert("four");
+    CheckConsecutiveKeys<C::const_iterator>(c.find(1), c.end(), 1, s);
+    eq = c.equal_range(2);
+    assert(std::distance(eq.first, eq.second) == 2);
+    s.insert("two");
+    s.insert("four");
+    CheckConsecutiveKeys<C::const_iterator>(c.find(2), c.end(), 2, s);
 
-        eq = c.equal_range(3);
-        assert(std::distance(eq.first, eq.second) == 1);
-        C::const_iterator i = eq.first;
-        assert(i->first == 3);
-        assert(i->second == "three");
-        eq = c.equal_range(4);
-        assert(std::distance(eq.first, eq.second) == 1);
-        i = eq.first;
-        assert(i->first == 4);
-        assert(i->second == "four");
-        assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-        assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
-        assert(fabs(c.load_factor() - (float)c.size()/c.bucket_count()) < FLT_EPSILON);
-        assert(c.max_load_factor() == 1);
-    }
-    {
-        typedef min_allocator<std::pair<const int, std::string> > A;
-        typedef std::unordered_multimap<int, std::string,
-                                   test_hash<int>,
-                                   test_equal_to<int>,
-                                   A
-                                   > C;
-        typedef std::pair<int, std::string> P;
-        P a[] =
-        {
-            P(1, "one"),
-            P(2, "two"),
-            P(3, "three"),
-            P(4, "four"),
-            P(1, "four"),
-            P(2, "four"),
-        };
-        C c0(a, a + sizeof(a)/sizeof(a[0]),
-            7,
-            test_hash<int>(8),
-            test_equal_to<int>(9),
-            A()
-           );
-        C c(a, a + 2,
-            7,
-            test_hash<int>(2),
-            test_equal_to<int>(3),
-            A()
-           );
-        C::iterator it0 = c0.begin();
-        c = std::move(c0);
-        assert(it0 == c.begin()); // Iterators remain valid
-        LIBCPP_ASSERT(c.bucket_count() == 7);
-        assert(c.size() == 6);
-        typedef std::pair<C::const_iterator, C::const_iterator> Eq;
-        Eq eq = c.equal_range(1);
-        assert(std::distance(eq.first, eq.second) == 2);
-        std::multiset<std::string> s;
-        s.insert("one");
-        s.insert("four");
-        CheckConsecutiveKeys<C::const_iterator>(c.find(1), c.end(), 1, s);
-        eq = c.equal_range(2);
-        assert(std::distance(eq.first, eq.second) == 2);
-        s.insert("two");
-        s.insert("four");
-        CheckConsecutiveKeys<C::const_iterator>(c.find(2), c.end(), 2, s);
+    eq = c.equal_range(3);
+    assert(std::distance(eq.first, eq.second) == 1);
+    C::const_iterator i = eq.first;
+    assert(i->first == 3);
+    assert(i->second == "three");
+    eq = c.equal_range(4);
+    assert(std::distance(eq.first, eq.second) == 1);
+    i = eq.first;
+    assert(i->first == 4);
+    assert(i->second == "four");
+    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
+    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
+    assert(fabs(c.load_factor() - (float)c.size() / c.bucket_count()) < FLT_EPSILON);
+    assert(c.max_load_factor() == 1);
+  }
+  {
+    typedef min_allocator<std::pair<const int, std::string> > A;
+    typedef std::unordered_multimap<int, std::string, test_hash<int>, test_equal_to<int>, A > C;
+    typedef std::pair<int, std::string> P;
+    P a[] = {
+        P(1, "one"),
+        P(2, "two"),
+        P(3, "three"),
+        P(4, "four"),
+        P(1, "four"),
+        P(2, "four"),
+    };
+    C c0(a, a + sizeof(a) / sizeof(a[0]), 7, test_hash<int>(8), test_equal_to<int>(9), A());
+    C c(a, a + 2, 7, test_hash<int>(2), test_equal_to<int>(3), A());
+    C::iterator it0 = c0.begin();
+    c               = std::move(c0);
+    assert(it0 == c.begin()); // Iterators remain valid
+    LIBCPP_ASSERT(c.bucket_count() == 7);
+    assert(c.size() == 6);
+    typedef std::pair<C::const_iterator, C::const_iterator> Eq;
+    Eq eq = c.equal_range(1);
+    assert(std::distance(eq.first, eq.second) == 2);
+    std::multiset<std::string> s;
+    s.insert("one");
+    s.insert("four");
+    CheckConsecutiveKeys<C::const_iterator>(c.find(1), c.end(), 1, s);
+    eq = c.equal_range(2);
+    assert(std::distance(eq.first, eq.second) == 2);
+    s.insert("two");
+    s.insert("four");
+    CheckConsecutiveKeys<C::const_iterator>(c.find(2), c.end(), 2, s);
 
-        eq = c.equal_range(3);
-        assert(std::distance(eq.first, eq.second) == 1);
-        C::const_iterator i = eq.first;
-        assert(i->first == 3);
-        assert(i->second == "three");
-        eq = c.equal_range(4);
-        assert(std::distance(eq.first, eq.second) == 1);
-        i = eq.first;
-        assert(i->first == 4);
-        assert(i->second == "four");
-        assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-        assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
-        assert(fabs(c.load_factor() - (float)c.size()/c.bucket_count()) < FLT_EPSILON);
-        assert(c.max_load_factor() == 1);
-    }
+    eq = c.equal_range(3);
+    assert(std::distance(eq.first, eq.second) == 1);
+    C::const_iterator i = eq.first;
+    assert(i->first == 3);
+    assert(i->second == "three");
+    eq = c.equal_range(4);
+    assert(std::distance(eq.first, eq.second) == 1);
+    i = eq.first;
+    assert(i->first == 4);
+    assert(i->second == "four");
+    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
+    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
+    assert(fabs(c.load_factor() - (float)c.size() / c.bucket_count()) < FLT_EPSILON);
+    assert(c.max_load_factor() == 1);
+  }
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.cnstr/compare_copy_constructible.compile.fail.cpp b/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.cnstr/compare_copy_constructible.compile.fail.cpp
index 501ef7593c839..31e6ded016e8b 100644
--- a/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.cnstr/compare_copy_constructible.compile.fail.cpp
+++ b/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.cnstr/compare_copy_constructible.compile.fail.cpp
@@ -18,16 +18,16 @@
 
 template <class T>
 struct Comp {
-    bool operator () (const T& lhs, const T& rhs) const { return lhs == rhs; }
+  bool operator()(const T& lhs, const T& rhs) const { return lhs == rhs; }
 
-    Comp () {}
-private:
-    Comp (const Comp &); // declared but not defined
-    };
+  Comp() {}
 
+private:
+  Comp(const Comp&); // declared but not defined
+};
 
 int main(int, char**) {
-    std::unordered_multimap<int, int, std::hash<int>, Comp<int> > m;
+  std::unordered_multimap<int, int, std::hash<int>, Comp<int> > m;
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.cnstr/copy.pass.cpp b/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.cnstr/copy.pass.cpp
index 60c6341e00050..e369ce2ac268e 100644
--- a/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.cnstr/copy.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.cnstr/copy.pass.cpp
@@ -29,147 +29,143 @@
 #include "test_allocator.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef std::unordered_multimap<int, std::string,
-                                   test_hash<int>,
-                                   test_equal_to<int>,
-                                   test_allocator<std::pair<const int, std::string> >
-                                   > C;
-        typedef std::pair<int, std::string> P;
-        P a[] =
-        {
-            P(1, "one"),
-            P(2, "two"),
-            P(3, "three"),
-            P(4, "four"),
-            P(1, "four"),
-            P(2, "four"),
-        };
-        C c0(a, a + sizeof(a)/sizeof(a[0]),
-            7,
-            test_hash<int>(8),
-            test_equal_to<int>(9),
-            test_allocator<std::pair<const int, std::string> >(10)
-           );
-        C c = c0;
-        LIBCPP_ASSERT(c.bucket_count() == 7);
-        assert(c.size() == 6);
-        std::multiset<std::string> s;
-        s.insert("one");
-        s.insert("four");
-        CheckConsecutiveKeys<C::const_iterator>(c.find(1), c.end(), 1, s);
-        s.insert("two");
-        s.insert("four");
-        CheckConsecutiveKeys<C::const_iterator>(c.find(2), c.end(), 2, s);
-        s.insert("three");
-        CheckConsecutiveKeys<C::const_iterator>(c.find(3), c.end(), 3, s);
-        s.insert("four");
-        CheckConsecutiveKeys<C::const_iterator>(c.find(4), c.end(), 4, s);
-        assert(c.hash_function() == test_hash<int>(8));
-        assert(c.key_eq() == test_equal_to<int>(9));
-        assert(c.get_allocator() ==
-               (test_allocator<std::pair<const int, std::string> >(10)));
-        assert(!c.empty());
-        assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-        assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
-        assert(std::fabs(c.load_factor() - (float)c.size()/c.bucket_count()) < FLT_EPSILON);
-        assert(c.max_load_factor() == 1);
-    }
+int main(int, char**) {
+  {
+    typedef std::unordered_multimap<int,
+                                    std::string,
+                                    test_hash<int>,
+                                    test_equal_to<int>,
+                                    test_allocator<std::pair<const int, std::string> > >
+        C;
+    typedef std::pair<int, std::string> P;
+    P a[] = {
+        P(1, "one"),
+        P(2, "two"),
+        P(3, "three"),
+        P(4, "four"),
+        P(1, "four"),
+        P(2, "four"),
+    };
+    C c0(a,
+         a + sizeof(a) / sizeof(a[0]),
+         7,
+         test_hash<int>(8),
+         test_equal_to<int>(9),
+         test_allocator<std::pair<const int, std::string> >(10));
+    C c = c0;
+    LIBCPP_ASSERT(c.bucket_count() == 7);
+    assert(c.size() == 6);
+    std::multiset<std::string> s;
+    s.insert("one");
+    s.insert("four");
+    CheckConsecutiveKeys<C::const_iterator>(c.find(1), c.end(), 1, s);
+    s.insert("two");
+    s.insert("four");
+    CheckConsecutiveKeys<C::const_iterator>(c.find(2), c.end(), 2, s);
+    s.insert("three");
+    CheckConsecutiveKeys<C::const_iterator>(c.find(3), c.end(), 3, s);
+    s.insert("four");
+    CheckConsecutiveKeys<C::const_iterator>(c.find(4), c.end(), 4, s);
+    assert(c.hash_function() == test_hash<int>(8));
+    assert(c.key_eq() == test_equal_to<int>(9));
+    assert(c.get_allocator() == (test_allocator<std::pair<const int, std::string> >(10)));
+    assert(!c.empty());
+    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
+    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
+    assert(std::fabs(c.load_factor() - (float)c.size() / c.bucket_count()) < FLT_EPSILON);
+    assert(c.max_load_factor() == 1);
+  }
 #if TEST_STD_VER >= 11
-    {
-        typedef std::unordered_multimap<int, std::string,
-                                   test_hash<int>,
-                                   test_equal_to<int>,
-                                   other_allocator<std::pair<const int, std::string> >
-                                   > C;
-        typedef std::pair<int, std::string> P;
-        P a[] =
-        {
-            P(1, "one"),
-            P(2, "two"),
-            P(3, "three"),
-            P(4, "four"),
-            P(1, "four"),
-            P(2, "four"),
-        };
-        C c0(a, a + sizeof(a)/sizeof(a[0]),
-            7,
-            test_hash<int>(8),
-            test_equal_to<int>(9),
-            other_allocator<std::pair<const int, std::string> >(10)
-           );
-        C c = c0;
-        LIBCPP_ASSERT(c.bucket_count() == 7);
-        assert(c.size() == 6);
-        std::multiset<std::string> s;
-        s.insert("one");
-        s.insert("four");
-        CheckConsecutiveKeys<C::const_iterator>(c.find(1), c.end(), 1, s);
-        s.insert("two");
-        s.insert("four");
-        CheckConsecutiveKeys<C::const_iterator>(c.find(2), c.end(), 2, s);
-        s.insert("three");
-        CheckConsecutiveKeys<C::const_iterator>(c.find(3), c.end(), 3, s);
-        s.insert("four");
-        CheckConsecutiveKeys<C::const_iterator>(c.find(4), c.end(), 4, s);
-        assert(c.hash_function() == test_hash<int>(8));
-        assert(c.key_eq() == test_equal_to<int>(9));
-        assert(c.get_allocator() ==
-               (other_allocator<std::pair<const int, std::string> >(-2)));
-        assert(!c.empty());
-        assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-        assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
-        assert(std::fabs(c.load_factor() - (float)c.size()/c.bucket_count()) < FLT_EPSILON);
-        assert(c.max_load_factor() == 1);
-    }
-    {
-        typedef std::unordered_multimap<int, std::string,
-                                   test_hash<int>,
-                                   test_equal_to<int>,
-                                   min_allocator<std::pair<const int, std::string> >
-                                   > C;
-        typedef std::pair<int, std::string> P;
-        P a[] =
-        {
-            P(1, "one"),
-            P(2, "two"),
-            P(3, "three"),
-            P(4, "four"),
-            P(1, "four"),
-            P(2, "four"),
-        };
-        C c0(a, a + sizeof(a)/sizeof(a[0]),
-            7,
-            test_hash<int>(8),
-            test_equal_to<int>(9),
-            min_allocator<std::pair<const int, std::string> >()
-           );
-        C c = c0;
-        LIBCPP_ASSERT(c.bucket_count() == 7);
-        assert(c.size() == 6);
-        std::multiset<std::string> s;
-        s.insert("one");
-        s.insert("four");
-        CheckConsecutiveKeys<C::const_iterator>(c.find(1), c.end(), 1, s);
-        s.insert("two");
-        s.insert("four");
-        CheckConsecutiveKeys<C::const_iterator>(c.find(2), c.end(), 2, s);
-        s.insert("three");
-        CheckConsecutiveKeys<C::const_iterator>(c.find(3), c.end(), 3, s);
-        s.insert("four");
-        CheckConsecutiveKeys<C::const_iterator>(c.find(4), c.end(), 4, s);
-        assert(c.hash_function() == test_hash<int>(8));
-        assert(c.key_eq() == test_equal_to<int>(9));
-        assert(c.get_allocator() ==
-               (min_allocator<std::pair<const int, std::string> >()));
-        assert(!c.empty());
-        assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-        assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
-        assert(std::fabs(c.load_factor() - (float)c.size()/c.bucket_count()) < FLT_EPSILON);
-        assert(c.max_load_factor() == 1);
-    }
+  {
+    typedef std::unordered_multimap<int,
+                                    std::string,
+                                    test_hash<int>,
+                                    test_equal_to<int>,
+                                    other_allocator<std::pair<const int, std::string> > >
+        C;
+    typedef std::pair<int, std::string> P;
+    P a[] = {
+        P(1, "one"),
+        P(2, "two"),
+        P(3, "three"),
+        P(4, "four"),
+        P(1, "four"),
+        P(2, "four"),
+    };
+    C c0(a,
+         a + sizeof(a) / sizeof(a[0]),
+         7,
+         test_hash<int>(8),
+         test_equal_to<int>(9),
+         other_allocator<std::pair<const int, std::string> >(10));
+    C c = c0;
+    LIBCPP_ASSERT(c.bucket_count() == 7);
+    assert(c.size() == 6);
+    std::multiset<std::string> s;
+    s.insert("one");
+    s.insert("four");
+    CheckConsecutiveKeys<C::const_iterator>(c.find(1), c.end(), 1, s);
+    s.insert("two");
+    s.insert("four");
+    CheckConsecutiveKeys<C::const_iterator>(c.find(2), c.end(), 2, s);
+    s.insert("three");
+    CheckConsecutiveKeys<C::const_iterator>(c.find(3), c.end(), 3, s);
+    s.insert("four");
+    CheckConsecutiveKeys<C::const_iterator>(c.find(4), c.end(), 4, s);
+    assert(c.hash_function() == test_hash<int>(8));
+    assert(c.key_eq() == test_equal_to<int>(9));
+    assert(c.get_allocator() == (other_allocator<std::pair<const int, std::string> >(-2)));
+    assert(!c.empty());
+    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
+    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
+    assert(std::fabs(c.load_factor() - (float)c.size() / c.bucket_count()) < FLT_EPSILON);
+    assert(c.max_load_factor() == 1);
+  }
+  {
+    typedef std::unordered_multimap<int,
+                                    std::string,
+                                    test_hash<int>,
+                                    test_equal_to<int>,
+                                    min_allocator<std::pair<const int, std::string> > >
+        C;
+    typedef std::pair<int, std::string> P;
+    P a[] = {
+        P(1, "one"),
+        P(2, "two"),
+        P(3, "three"),
+        P(4, "four"),
+        P(1, "four"),
+        P(2, "four"),
+    };
+    C c0(a,
+         a + sizeof(a) / sizeof(a[0]),
+         7,
+         test_hash<int>(8),
+         test_equal_to<int>(9),
+         min_allocator<std::pair<const int, std::string> >());
+    C c = c0;
+    LIBCPP_ASSERT(c.bucket_count() == 7);
+    assert(c.size() == 6);
+    std::multiset<std::string> s;
+    s.insert("one");
+    s.insert("four");
+    CheckConsecutiveKeys<C::const_iterator>(c.find(1), c.end(), 1, s);
+    s.insert("two");
+    s.insert("four");
+    CheckConsecutiveKeys<C::const_iterator>(c.find(2), c.end(), 2, s);
+    s.insert("three");
+    CheckConsecutiveKeys<C::const_iterator>(c.find(3), c.end(), 3, s);
+    s.insert("four");
+    CheckConsecutiveKeys<C::const_iterator>(c.find(4), c.end(), 4, s);
+    assert(c.hash_function() == test_hash<int>(8));
+    assert(c.key_eq() == test_equal_to<int>(9));
+    assert(c.get_allocator() == (min_allocator<std::pair<const int, std::string> >()));
+    assert(!c.empty());
+    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
+    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
+    assert(std::fabs(c.load_factor() - (float)c.size() / c.bucket_count()) < FLT_EPSILON);
+    assert(c.max_load_factor() == 1);
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.cnstr/copy_alloc.pass.cpp b/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.cnstr/copy_alloc.pass.cpp
index d9ec0ad901a35..25348786a3607 100644
--- a/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.cnstr/copy_alloc.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.cnstr/copy_alloc.pass.cpp
@@ -29,147 +29,134 @@
 #include "test_allocator.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef std::unordered_multimap<int, std::string,
-                                   test_hash<int>,
-                                   test_equal_to<int>,
-                                   test_allocator<std::pair<const int, std::string> >
-                                   > C;
-        typedef std::pair<int, std::string> P;
-        P a[] =
-        {
-            P(1, "one"),
-            P(2, "two"),
-            P(3, "three"),
-            P(4, "four"),
-            P(1, "four"),
-            P(2, "four"),
-        };
-        C c0(a, a + sizeof(a)/sizeof(a[0]),
-            7,
-            test_hash<int>(8),
-            test_equal_to<int>(9),
-            test_allocator<std::pair<const int, std::string> >(10)
-           );
-        C c(c0, test_allocator<std::pair<const int, std::string> >(5));
-        LIBCPP_ASSERT(c.bucket_count() == 7);
-        assert(c.size() == 6);
-        std::multiset<std::string> s;
-        s.insert("one");
-        s.insert("four");
-        CheckConsecutiveKeys<C::const_iterator>(c.find(1), c.end(), 1, s);
-        s.insert("two");
-        s.insert("four");
-        CheckConsecutiveKeys<C::const_iterator>(c.find(2), c.end(), 2, s);
-        s.insert("three");
-        CheckConsecutiveKeys<C::const_iterator>(c.find(3), c.end(), 3, s);
-        s.insert("four");
-        CheckConsecutiveKeys<C::const_iterator>(c.find(4), c.end(), 4, s);
-        assert(c.hash_function() == test_hash<int>(8));
-        assert(c.key_eq() == test_equal_to<int>(9));
-        assert(c.get_allocator() ==
-               (test_allocator<std::pair<const int, std::string> >(5)));
-        assert(!c.empty());
-        assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-        assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
-        assert(std::fabs(c.load_factor() - (float)c.size()/c.bucket_count()) < FLT_EPSILON);
-        assert(c.max_load_factor() == 1);
-    }
+int main(int, char**) {
+  {
+    typedef std::unordered_multimap<int,
+                                    std::string,
+                                    test_hash<int>,
+                                    test_equal_to<int>,
+                                    test_allocator<std::pair<const int, std::string> > >
+        C;
+    typedef std::pair<int, std::string> P;
+    P a[] = {
+        P(1, "one"),
+        P(2, "two"),
+        P(3, "three"),
+        P(4, "four"),
+        P(1, "four"),
+        P(2, "four"),
+    };
+    C c0(a,
+         a + sizeof(a) / sizeof(a[0]),
+         7,
+         test_hash<int>(8),
+         test_equal_to<int>(9),
+         test_allocator<std::pair<const int, std::string> >(10));
+    C c(c0, test_allocator<std::pair<const int, std::string> >(5));
+    LIBCPP_ASSERT(c.bucket_count() == 7);
+    assert(c.size() == 6);
+    std::multiset<std::string> s;
+    s.insert("one");
+    s.insert("four");
+    CheckConsecutiveKeys<C::const_iterator>(c.find(1), c.end(), 1, s);
+    s.insert("two");
+    s.insert("four");
+    CheckConsecutiveKeys<C::const_iterator>(c.find(2), c.end(), 2, s);
+    s.insert("three");
+    CheckConsecutiveKeys<C::const_iterator>(c.find(3), c.end(), 3, s);
+    s.insert("four");
+    CheckConsecutiveKeys<C::const_iterator>(c.find(4), c.end(), 4, s);
+    assert(c.hash_function() == test_hash<int>(8));
+    assert(c.key_eq() == test_equal_to<int>(9));
+    assert(c.get_allocator() == (test_allocator<std::pair<const int, std::string> >(5)));
+    assert(!c.empty());
+    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
+    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
+    assert(std::fabs(c.load_factor() - (float)c.size() / c.bucket_count()) < FLT_EPSILON);
+    assert(c.max_load_factor() == 1);
+  }
 #if TEST_STD_VER >= 11
-    {
-        typedef std::unordered_multimap<int, std::string,
-                                   test_hash<int>,
-                                   test_equal_to<int>,
-                                   min_allocator<std::pair<const int, std::string> >
-                                   > C;
-        typedef std::pair<int, std::string> P;
-        P a[] =
-        {
-            P(1, "one"),
-            P(2, "two"),
-            P(3, "three"),
-            P(4, "four"),
-            P(1, "four"),
-            P(2, "four"),
-        };
-        C c0(a, a + sizeof(a)/sizeof(a[0]),
-            7,
-            test_hash<int>(8),
-            test_equal_to<int>(9),
-            min_allocator<std::pair<const int, std::string> >()
-           );
-        C c(c0, min_allocator<std::pair<const int, std::string> >());
-        LIBCPP_ASSERT(c.bucket_count() == 7);
-        assert(c.size() == 6);
-        std::multiset<std::string> s;
-        s.insert("one");
-        s.insert("four");
-        CheckConsecutiveKeys<C::const_iterator>(c.find(1), c.end(), 1, s);
-        s.insert("two");
-        s.insert("four");
-        CheckConsecutiveKeys<C::const_iterator>(c.find(2), c.end(), 2, s);
-        s.insert("three");
-        CheckConsecutiveKeys<C::const_iterator>(c.find(3), c.end(), 3, s);
-        s.insert("four");
-        CheckConsecutiveKeys<C::const_iterator>(c.find(4), c.end(), 4, s);
-        assert(c.hash_function() == test_hash<int>(8));
-        assert(c.key_eq() == test_equal_to<int>(9));
-        assert(c.get_allocator() ==
-               (min_allocator<std::pair<const int, std::string> >()));
-        assert(!c.empty());
-        assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-        assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
-        assert(std::fabs(c.load_factor() - (float)c.size()/c.bucket_count()) < FLT_EPSILON);
-        assert(c.max_load_factor() == 1);
-    }
-    {
-        typedef explicit_allocator<std::pair<const int, std::string>> A;
-        typedef std::unordered_multimap<int, std::string,
-                                   test_hash<int>,
-                                   test_equal_to<int>,
-                                   A
-                                   > C;
-        typedef std::pair<int, std::string> P;
-        P a[] =
-        {
-            P(1, "one"),
-            P(2, "two"),
-            P(3, "three"),
-            P(4, "four"),
-            P(1, "four"),
-            P(2, "four"),
-        };
-        C c0(a, a + sizeof(a)/sizeof(a[0]),
-            7,
-            test_hash<int>(8),
-            test_equal_to<int>(9),
-            A{}
-           );
-        C c(c0, A{});
-        LIBCPP_ASSERT(c.bucket_count() == 7);
-        assert(c.size() == 6);
-        std::multiset<std::string> s;
-        s.insert("one");
-        s.insert("four");
-        CheckConsecutiveKeys<C::const_iterator>(c.find(1), c.end(), 1, s);
-        s.insert("two");
-        s.insert("four");
-        CheckConsecutiveKeys<C::const_iterator>(c.find(2), c.end(), 2, s);
-        s.insert("three");
-        CheckConsecutiveKeys<C::const_iterator>(c.find(3), c.end(), 3, s);
-        s.insert("four");
-        CheckConsecutiveKeys<C::const_iterator>(c.find(4), c.end(), 4, s);
-        assert(c.hash_function() == test_hash<int>(8));
-        assert(c.key_eq() == test_equal_to<int>(9));
-        assert(c.get_allocator() == A{});
-        assert(!c.empty());
-        assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-        assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
-        assert(std::fabs(c.load_factor() - (float)c.size()/c.bucket_count()) < FLT_EPSILON);
-        assert(c.max_load_factor() == 1);
-    }
+  {
+    typedef std::unordered_multimap<int,
+                                    std::string,
+                                    test_hash<int>,
+                                    test_equal_to<int>,
+                                    min_allocator<std::pair<const int, std::string> > >
+        C;
+    typedef std::pair<int, std::string> P;
+    P a[] = {
+        P(1, "one"),
+        P(2, "two"),
+        P(3, "three"),
+        P(4, "four"),
+        P(1, "four"),
+        P(2, "four"),
+    };
+    C c0(a,
+         a + sizeof(a) / sizeof(a[0]),
+         7,
+         test_hash<int>(8),
+         test_equal_to<int>(9),
+         min_allocator<std::pair<const int, std::string> >());
+    C c(c0, min_allocator<std::pair<const int, std::string> >());
+    LIBCPP_ASSERT(c.bucket_count() == 7);
+    assert(c.size() == 6);
+    std::multiset<std::string> s;
+    s.insert("one");
+    s.insert("four");
+    CheckConsecutiveKeys<C::const_iterator>(c.find(1), c.end(), 1, s);
+    s.insert("two");
+    s.insert("four");
+    CheckConsecutiveKeys<C::const_iterator>(c.find(2), c.end(), 2, s);
+    s.insert("three");
+    CheckConsecutiveKeys<C::const_iterator>(c.find(3), c.end(), 3, s);
+    s.insert("four");
+    CheckConsecutiveKeys<C::const_iterator>(c.find(4), c.end(), 4, s);
+    assert(c.hash_function() == test_hash<int>(8));
+    assert(c.key_eq() == test_equal_to<int>(9));
+    assert(c.get_allocator() == (min_allocator<std::pair<const int, std::string> >()));
+    assert(!c.empty());
+    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
+    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
+    assert(std::fabs(c.load_factor() - (float)c.size() / c.bucket_count()) < FLT_EPSILON);
+    assert(c.max_load_factor() == 1);
+  }
+  {
+    typedef explicit_allocator<std::pair<const int, std::string>> A;
+    typedef std::unordered_multimap<int, std::string, test_hash<int>, test_equal_to<int>, A > C;
+    typedef std::pair<int, std::string> P;
+    P a[] = {
+        P(1, "one"),
+        P(2, "two"),
+        P(3, "three"),
+        P(4, "four"),
+        P(1, "four"),
+        P(2, "four"),
+    };
+    C c0(a, a + sizeof(a) / sizeof(a[0]), 7, test_hash<int>(8), test_equal_to<int>(9), A{});
+    C c(c0, A{});
+    LIBCPP_ASSERT(c.bucket_count() == 7);
+    assert(c.size() == 6);
+    std::multiset<std::string> s;
+    s.insert("one");
+    s.insert("four");
+    CheckConsecutiveKeys<C::const_iterator>(c.find(1), c.end(), 1, s);
+    s.insert("two");
+    s.insert("four");
+    CheckConsecutiveKeys<C::const_iterator>(c.find(2), c.end(), 2, s);
+    s.insert("three");
+    CheckConsecutiveKeys<C::const_iterator>(c.find(3), c.end(), 3, s);
+    s.insert("four");
+    CheckConsecutiveKeys<C::const_iterator>(c.find(4), c.end(), 4, s);
+    assert(c.hash_function() == test_hash<int>(8));
+    assert(c.key_eq() == test_equal_to<int>(9));
+    assert(c.get_allocator() == A{});
+    assert(!c.empty());
+    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
+    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
+    assert(std::fabs(c.load_factor() - (float)c.size() / c.bucket_count()) < FLT_EPSILON);
+    assert(c.max_load_factor() == 1);
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.cnstr/deduct.pass.cpp b/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.cnstr/deduct.pass.cpp
index 7e93032f0175b..5d625a1deaab5 100644
--- a/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.cnstr/deduct.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.cnstr/deduct.pass.cpp
@@ -89,152 +89,165 @@
 #include "deduction_guides_sfinae_checks.h"
 #include "test_allocator.h"
 
-using P = std::pair<int, long>;
+using P  = std::pair<int, long>;
 using PC = std::pair<const int, long>;
 
-int main(int, char**)
-{
-    const PC expected_m[] = { {1,1}, {1,1}, {2,2}, {3,1}, {INT_MAX,1} };
+int main(int, char**) {
+  const PC expected_m[] = {{1, 1}, {1, 1}, {2, 2}, {3, 1}, {INT_MAX, 1}};
 
-    {
-    const P arr[] = { {1,1}, {2,2}, {1,1}, {INT_MAX,1}, {3,1} };
+  {
+    const P arr[] = {{1, 1}, {2, 2}, {1, 1}, {INT_MAX, 1}, {3, 1}};
     std::unordered_multimap m(std::begin(arr), std::end(arr));
     ASSERT_SAME_TYPE(decltype(m), std::unordered_multimap<int, long>);
     assert(std::is_permutation(m.begin(), m.end(), std::begin(expected_m), std::end(expected_m)));
-    }
+  }
 
-    {
-    const P arr[] = { {1,1}, {2,2}, {1,1}, {INT_MAX,1}, {3,1} };
+  {
+    const P arr[] = {{1, 1}, {2, 2}, {1, 1}, {INT_MAX, 1}, {3, 1}};
     std::unordered_multimap m(std::begin(arr), std::end(arr), 42);
     ASSERT_SAME_TYPE(decltype(m), std::unordered_multimap<int, long>);
     assert(std::is_permutation(m.begin(), m.end(), std::begin(expected_m), std::end(expected_m)));
-    }
+  }
 
-    {
-    const P arr[] = { {1,1}, {2,2}, {1,1}, {INT_MAX,1}, {3,1} };
+  {
+    const P arr[] = {{1, 1}, {2, 2}, {1, 1}, {INT_MAX, 1}, {3, 1}};
     std::unordered_multimap m(std::begin(arr), std::end(arr), 42, std::hash<short>());
     ASSERT_SAME_TYPE(decltype(m), std::unordered_multimap<int, long, std::hash<short>, std::equal_to<int>>);
     assert(std::is_permutation(m.begin(), m.end(), std::begin(expected_m), std::end(expected_m)));
-    }
+  }
 
-    {
-    const P arr[] = { {1,1}, {2,2}, {1,1}, {INT_MAX,1}, {3,1} };
+  {
+    const P arr[] = {{1, 1}, {2, 2}, {1, 1}, {INT_MAX, 1}, {3, 1}};
     std::unordered_multimap m(std::begin(arr), std::end(arr), 42, std::hash<short>(), std::equal_to<>());
     ASSERT_SAME_TYPE(decltype(m), std::unordered_multimap<int, long, std::hash<short>, std::equal_to<>>);
     assert(std::is_permutation(m.begin(), m.end(), std::begin(expected_m), std::end(expected_m)));
-    }
-
-    {
-    const P arr[] = { {1,1}, {2,2}, {1,1}, {INT_MAX,1}, {3,1} };
-    std::unordered_multimap m(std::begin(arr), std::end(arr), 42, std::hash<short>(), std::equal_to<>(), test_allocator<PC>(0, 41));
-    ASSERT_SAME_TYPE(decltype(m), std::unordered_multimap<int, long, std::hash<short>, std::equal_to<>, test_allocator<PC>>);
+  }
+
+  {
+    const P arr[] = {{1, 1}, {2, 2}, {1, 1}, {INT_MAX, 1}, {3, 1}};
+    std::unordered_multimap m(
+        std::begin(arr), std::end(arr), 42, std::hash<short>(), std::equal_to<>(), test_allocator<PC>(0, 41));
+    ASSERT_SAME_TYPE(
+        decltype(m), std::unordered_multimap<int, long, std::hash<short>, std::equal_to<>, test_allocator<PC>>);
     assert(std::is_permutation(m.begin(), m.end(), std::begin(expected_m), std::end(expected_m)));
     assert(m.get_allocator().get_id() == 41);
-    }
+  }
 
-    {
+  {
     std::unordered_multimap<int, long> source;
     std::unordered_multimap m(source);
     ASSERT_SAME_TYPE(decltype(m), decltype(source));
     assert(m.size() == 0);
-    }
+  }
 
-    {
+  {
     std::unordered_multimap<int, long> source;
-    std::unordered_multimap m{source};  // braces instead of parens
+    std::unordered_multimap m{source}; // braces instead of parens
     ASSERT_SAME_TYPE(decltype(m), decltype(source));
     assert(m.size() == 0);
-    }
+  }
 
-    {
+  {
     std::unordered_multimap<int, long, std::hash<short>, std::equal_to<>, test_allocator<PC>> source;
     test_allocator<PC> a(0, 42);
     std::unordered_multimap m(source, a);
     ASSERT_SAME_TYPE(decltype(m), decltype(source));
     assert(m.get_allocator().get_id() == 42);
     assert(m.size() == 0);
-    }
+  }
 
-    {
+  {
     std::unordered_multimap<int, long, std::hash<short>, std::equal_to<>, test_allocator<PC>> source;
     test_allocator<PC> a(0, 43);
-    std::unordered_multimap m{source, a};  // braces instead of parens
+    std::unordered_multimap m{source, a}; // braces instead of parens
     ASSERT_SAME_TYPE(decltype(m), decltype(source));
     assert(m.get_allocator().get_id() == 43);
     assert(m.size() == 0);
-    }
+  }
 
-    {
-    std::unordered_multimap m { P{1,1L}, P{2,2L}, P{1,1L}, P{INT_MAX,1L}, P{3,1L} };
+  {
+    std::unordered_multimap m{P{1, 1L}, P{2, 2L}, P{1, 1L}, P{INT_MAX, 1L}, P{3, 1L}};
     ASSERT_SAME_TYPE(decltype(m), std::unordered_multimap<int, long>);
     assert(std::is_permutation(m.begin(), m.end(), std::begin(expected_m), std::end(expected_m)));
-    }
+  }
 
-    {
-    std::unordered_multimap m({ P{1,1L}, P{2,2L}, P{1,1L}, P{INT_MAX,1L}, P{3,1L} }, 42);
+  {
+    std::unordered_multimap m({P{1, 1L}, P{2, 2L}, P{1, 1L}, P{INT_MAX, 1L}, P{3, 1L}}, 42);
     ASSERT_SAME_TYPE(decltype(m), std::unordered_multimap<int, long>);
     assert(std::is_permutation(m.begin(), m.end(), std::begin(expected_m), std::end(expected_m)));
-    }
+  }
 
-    {
-    std::unordered_multimap m({ P{1,1L}, P{2,2L}, P{1,1L}, P{INT_MAX,1L}, P{3,1L} }, 42, std::hash<short>());
+  {
+    std::unordered_multimap m({P{1, 1L}, P{2, 2L}, P{1, 1L}, P{INT_MAX, 1L}, P{3, 1L}}, 42, std::hash<short>());
     ASSERT_SAME_TYPE(decltype(m), std::unordered_multimap<int, long, std::hash<short>>);
     assert(std::is_permutation(m.begin(), m.end(), std::begin(expected_m), std::end(expected_m)));
-    }
+  }
 
-    {
-    std::unordered_multimap m({ P{1,1L}, P{2,2L}, P{1,1L}, P{INT_MAX,1L}, P{3,1L} }, 42, std::hash<short>(), std::equal_to<>());
+  {
+    std::unordered_multimap m(
+        {P{1, 1L}, P{2, 2L}, P{1, 1L}, P{INT_MAX, 1L}, P{3, 1L}}, 42, std::hash<short>(), std::equal_to<>());
     ASSERT_SAME_TYPE(decltype(m), std::unordered_multimap<int, long, std::hash<short>, std::equal_to<>>);
     assert(std::is_permutation(m.begin(), m.end(), std::begin(expected_m), std::end(expected_m)));
-    }
-
-    {
-    std::unordered_multimap m({ P{1,1L}, P{2,2L}, P{1,1L}, P{INT_MAX,1L}, P{3,1L} }, 42, std::hash<short>(), std::equal_to<>(), test_allocator<PC>(0, 44));
-    ASSERT_SAME_TYPE(decltype(m), std::unordered_multimap<int, long, std::hash<short>, std::equal_to<>, test_allocator<PC>>);
+  }
+
+  {
+    std::unordered_multimap m(
+        {P{1, 1L}, P{2, 2L}, P{1, 1L}, P{INT_MAX, 1L}, P{3, 1L}},
+        42,
+        std::hash<short>(),
+        std::equal_to<>(),
+        test_allocator<PC>(0, 44));
+    ASSERT_SAME_TYPE(
+        decltype(m), std::unordered_multimap<int, long, std::hash<short>, std::equal_to<>, test_allocator<PC>>);
     assert(std::is_permutation(m.begin(), m.end(), std::begin(expected_m), std::end(expected_m)));
     assert(m.get_allocator().get_id() == 44);
-    }
+  }
 
-    {
-    const P arr[] = { {1,1}, {2,2}, {1,1}, {INT_MAX,1}, {3,1} };
+  {
+    const P arr[] = {{1, 1}, {2, 2}, {1, 1}, {INT_MAX, 1}, {3, 1}};
     std::unordered_multimap m(std::begin(arr), std::end(arr), 42, test_allocator<PC>(0, 45));
-    ASSERT_SAME_TYPE(decltype(m), std::unordered_multimap<int, long, std::hash<int>, std::equal_to<int>, test_allocator<PC>>);
+    ASSERT_SAME_TYPE(
+        decltype(m), std::unordered_multimap<int, long, std::hash<int>, std::equal_to<int>, test_allocator<PC>>);
     assert(std::is_permutation(m.begin(), m.end(), std::begin(expected_m), std::end(expected_m)));
     assert(m.get_allocator().get_id() == 45);
-    }
+  }
 
-    {
-    const P arr[] = { {1,1}, {2,2}, {1,1}, {INT_MAX,1}, {3,1} };
+  {
+    const P arr[] = {{1, 1}, {2, 2}, {1, 1}, {INT_MAX, 1}, {3, 1}};
     std::unordered_multimap m(std::begin(arr), std::end(arr), 42, std::hash<short>(), test_allocator<PC>(0, 46));
-    ASSERT_SAME_TYPE(decltype(m), std::unordered_multimap<int, long, std::hash<short>, std::equal_to<int>, test_allocator<PC>>);
+    ASSERT_SAME_TYPE(
+        decltype(m), std::unordered_multimap<int, long, std::hash<short>, std::equal_to<int>, test_allocator<PC>>);
     assert(std::is_permutation(m.begin(), m.end(), std::begin(expected_m), std::end(expected_m)));
     assert(m.get_allocator().get_id() == 46);
-    }
+  }
 
-    {
-    std::unordered_multimap m({ P{1,1L}, P{2,2L}, P{1,1L}, P{INT_MAX,1L}, P{3,1L} }, 42, test_allocator<PC>(0, 47));
-    ASSERT_SAME_TYPE(decltype(m), std::unordered_multimap<int, long, std::hash<int>, std::equal_to<int>, test_allocator<PC>>);
+  {
+    std::unordered_multimap m({P{1, 1L}, P{2, 2L}, P{1, 1L}, P{INT_MAX, 1L}, P{3, 1L}}, 42, test_allocator<PC>(0, 47));
+    ASSERT_SAME_TYPE(
+        decltype(m), std::unordered_multimap<int, long, std::hash<int>, std::equal_to<int>, test_allocator<PC>>);
     assert(std::is_permutation(m.begin(), m.end(), std::begin(expected_m), std::end(expected_m)));
     assert(m.get_allocator().get_id() == 47);
-    }
+  }
 
-    {
-    std::unordered_multimap m({ P{1,1L}, P{2,2L}, P{1,1L}, P{INT_MAX,1L}, P{3,1L} }, 42, std::hash<short>(), test_allocator<PC>(0, 48));
-    ASSERT_SAME_TYPE(decltype(m), std::unordered_multimap<int, long, std::hash<short>, std::equal_to<int>, test_allocator<PC>>);
+  {
+    std::unordered_multimap m(
+        {P{1, 1L}, P{2, 2L}, P{1, 1L}, P{INT_MAX, 1L}, P{3, 1L}}, 42, std::hash<short>(), test_allocator<PC>(0, 48));
+    ASSERT_SAME_TYPE(
+        decltype(m), std::unordered_multimap<int, long, std::hash<short>, std::equal_to<int>, test_allocator<PC>>);
     assert(std::is_permutation(m.begin(), m.end(), std::begin(expected_m), std::end(expected_m)));
     assert(m.get_allocator().get_id() == 48);
-    }
+  }
 
-    {
+  {
     // Examples from LWG3025
     std::unordered_multimap m{std::pair{1, 1}, {2, 2}, {3, 3}};
     ASSERT_SAME_TYPE(decltype(m), std::unordered_multimap<int, int>);
 
     std::unordered_multimap m2{m.begin(), m.end()};
     ASSERT_SAME_TYPE(decltype(m2), std::unordered_multimap<int, int>);
-    }
+  }
 
-    {
+  {
     // Examples from LWG3531
     std::unordered_multimap m1{{std::pair{1, 2}, {3, 4}}, 0};
     ASSERT_SAME_TYPE(decltype(m1), std::unordered_multimap<int, int>);
@@ -242,61 +255,61 @@ int main(int, char**)
     using value_type = std::pair<const int, int>;
     std::unordered_multimap m2{{value_type{1, 2}, {3, 4}}, 0};
     ASSERT_SAME_TYPE(decltype(m2), std::unordered_multimap<int, int>);
-    }
+  }
 
 #if TEST_STD_VER >= 23
-    {
-      using Range = std::array<P, 0>;
-      using Pred = test_equal_to<int>;
-      using DefaultPred = std::equal_to<int>;
-      using Hash = test_hash<int>;
-      using DefaultHash = std::hash<int>;
-      using Alloc = test_allocator<PC>;
-
-      { // (from_range, range)
-        std::unordered_multimap c(std::from_range, Range());
-        static_assert(std::is_same_v<decltype(c), std::unordered_multimap<int, long>>);
-      }
-
-      { // (from_range, range, n)
-        std::unordered_multimap c(std::from_range, Range(), std::size_t());
-        static_assert(std::is_same_v<decltype(c), std::unordered_multimap<int, long>>);
-      }
-
-      { // (from_range, range, n, hash)
-        std::unordered_multimap c(std::from_range, Range(), std::size_t(), Hash());
-        static_assert(std::is_same_v<decltype(c), std::unordered_multimap<int, long, Hash>>);
-      }
-
-      { // (from_range, range, n, hash, pred)
-        std::unordered_multimap c(std::from_range, Range(), std::size_t(), Hash(), Pred());
-        static_assert(std::is_same_v<decltype(c), std::unordered_multimap<int, long, Hash, Pred>>);
-      }
-
-      { // (from_range, range, n, hash, pred, alloc)
-        std::unordered_multimap c(std::from_range, Range(), std::size_t(), Hash(), Pred(), Alloc());
-        static_assert(std::is_same_v<decltype(c), std::unordered_multimap<int, long, Hash, Pred, Alloc>>);
-      }
-
-      { // (from_range, range, n, alloc)
-        std::unordered_multimap c(std::from_range, Range(), std::size_t(), Alloc());
-        static_assert(std::is_same_v<decltype(c), std::unordered_multimap<int, long, DefaultHash, DefaultPred, Alloc>>);
-      }
-
-      // TODO(LWG 2713): uncomment this test once the constructor is added.
-      { // (from_range, range, alloc)
-        //std::unordered_multimap c(std::from_range, Range(), Alloc());
-        //static_assert(std::is_same_v<decltype(c), std::unordered_multimap<int, long, DefaultHash, DefaultPred, Alloc>>);
-      }
-
-      { // (from_range, range, n, hash, alloc)
-        std::unordered_multimap c(std::from_range, Range(), std::size_t(), Hash(), Alloc());
-        static_assert(std::is_same_v<decltype(c), std::unordered_multimap<int, long, Hash, DefaultPred, Alloc>>);
-      }
+  {
+    using Range       = std::array<P, 0>;
+    using Pred        = test_equal_to<int>;
+    using DefaultPred = std::equal_to<int>;
+    using Hash        = test_hash<int>;
+    using DefaultHash = std::hash<int>;
+    using Alloc       = test_allocator<PC>;
+
+    { // (from_range, range)
+      std::unordered_multimap c(std::from_range, Range());
+      static_assert(std::is_same_v<decltype(c), std::unordered_multimap<int, long>>);
+    }
+
+    { // (from_range, range, n)
+      std::unordered_multimap c(std::from_range, Range(), std::size_t());
+      static_assert(std::is_same_v<decltype(c), std::unordered_multimap<int, long>>);
+    }
+
+    { // (from_range, range, n, hash)
+      std::unordered_multimap c(std::from_range, Range(), std::size_t(), Hash());
+      static_assert(std::is_same_v<decltype(c), std::unordered_multimap<int, long, Hash>>);
+    }
+
+    { // (from_range, range, n, hash, pred)
+      std::unordered_multimap c(std::from_range, Range(), std::size_t(), Hash(), Pred());
+      static_assert(std::is_same_v<decltype(c), std::unordered_multimap<int, long, Hash, Pred>>);
+    }
+
+    { // (from_range, range, n, hash, pred, alloc)
+      std::unordered_multimap c(std::from_range, Range(), std::size_t(), Hash(), Pred(), Alloc());
+      static_assert(std::is_same_v<decltype(c), std::unordered_multimap<int, long, Hash, Pred, Alloc>>);
+    }
+
+    { // (from_range, range, n, alloc)
+      std::unordered_multimap c(std::from_range, Range(), std::size_t(), Alloc());
+      static_assert(std::is_same_v<decltype(c), std::unordered_multimap<int, long, DefaultHash, DefaultPred, Alloc>>);
+    }
+
+    // TODO(LWG 2713): uncomment this test once the constructor is added.
+    { // (from_range, range, alloc)
+      //std::unordered_multimap c(std::from_range, Range(), Alloc());
+      //static_assert(std::is_same_v<decltype(c), std::unordered_multimap<int, long, DefaultHash, DefaultPred, Alloc>>);
+    }
+
+    { // (from_range, range, n, hash, alloc)
+      std::unordered_multimap c(std::from_range, Range(), std::size_t(), Hash(), Alloc());
+      static_assert(std::is_same_v<decltype(c), std::unordered_multimap<int, long, Hash, DefaultPred, Alloc>>);
     }
+  }
 #endif
 
-    UnorderedContainerDeductionGuidesSfinaeAway<std::unordered_multimap, std::unordered_multimap<int, long>>();
+  UnorderedContainerDeductionGuidesSfinaeAway<std::unordered_multimap, std::unordered_multimap<int, long>>();
 
-    return 0;
+  return 0;
 }
diff --git a/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.cnstr/deduct.verify.cpp b/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.cnstr/deduct.verify.cpp
index efcbbbce0e302..11410ef2da9a4 100644
--- a/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.cnstr/deduct.verify.cpp
+++ b/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.cnstr/deduct.verify.cpp
@@ -59,47 +59,48 @@
 #include <functional>
 #include <unordered_map>
 
-int main(int, char**)
-{
-    using P = std::pair<const int, int>;
-    {
-        // cannot deduce Key from nothing
-        std::unordered_multimap m; // expected-error-re{{no viable constructor or deduction guide for deduction of template arguments of '{{(std::)?}}unordered_multimap'}}
-    }
-    {
-        // cannot deduce Key from just (Size)
-        std::unordered_multimap m(42); // expected-error-re{{no viable constructor or deduction guide for deduction of template arguments of '{{(std::)?}}unordered_multimap'}}
-    }
-    {
-        // cannot deduce Key from just (Size, Hash)
-        std::unordered_multimap m(42, std::hash<int>());
-            // expected-error-re@-1{{no viable constructor or deduction guide for deduction of template arguments of '{{(std::)?}}unordered_multimap'}}
-    }
-    {
-        // cannot deduce Key from just (Size, Hash, Pred)
-        std::unordered_multimap m(42, std::hash<int>(), std::equal_to<int>());
-            // expected-error-re@-1{{no viable constructor or deduction guide for deduction of template arguments of '{{(std::)?}}unordered_multimap'}}
-    }
-    {
-        // cannot deduce Key from just (Size, Hash, Pred, Allocator)
-        std::unordered_multimap m(42, std::hash<int>(), std::equal_to<int>(), std::allocator<P>());
-            // expected-error-re@-1{{no viable constructor or deduction guide for deduction of template arguments of '{{(std::)?}}unordered_multimap'}}
-    }
-    {
-        // cannot deduce Key from just (Allocator)
-        std::unordered_multimap m(std::allocator<P>{});
-            // expected-error-re@-1{{no viable constructor or deduction guide for deduction of template arguments of '{{(std::)?}}unordered_multimap'}}
-    }
-    {
-        // cannot deduce Key from just (Size, Allocator)
-        std::unordered_multimap m(42, std::allocator<P>());
-            // expected-error-re@-1{{no viable constructor or deduction guide for deduction of template arguments of '{{(std::)?}}unordered_multimap'}}
-    }
-    {
-        // cannot deduce Key from just (Size, Hash, Allocator)
-        std::unordered_multimap m(42, std::hash<int>(), std::allocator<P>());
-            // expected-error-re@-1{{no viable constructor or deduction guide for deduction of template arguments of '{{(std::)?}}unordered_multimap'}}
-    }
+int main(int, char**) {
+  using P = std::pair<const int, int>;
+  {
+    // cannot deduce Key from nothing
+    std::unordered_multimap m;
+    // expected-error-re@-1{{no viable constructor or deduction guide for deduction of template arguments of '{{(std::)?}}unordered_multimap'}}
+  }
+  {
+    // cannot deduce Key from just (Size)
+    std::unordered_multimap m(42);
+    // expected-error-re@-1{{no viable constructor or deduction guide for deduction of template arguments of '{{(std::)?}}unordered_multimap'}}
+  }
+  {
+    // cannot deduce Key from just (Size, Hash)
+    std::unordered_multimap m(42, std::hash<int>());
+    // expected-error-re@-1{{no viable constructor or deduction guide for deduction of template arguments of '{{(std::)?}}unordered_multimap'}}
+  }
+  {
+    // cannot deduce Key from just (Size, Hash, Pred)
+    std::unordered_multimap m(42, std::hash<int>(), std::equal_to<int>());
+    // expected-error-re@-1{{no viable constructor or deduction guide for deduction of template arguments of '{{(std::)?}}unordered_multimap'}}
+  }
+  {
+    // cannot deduce Key from just (Size, Hash, Pred, Allocator)
+    std::unordered_multimap m(42, std::hash<int>(), std::equal_to<int>(), std::allocator<P>());
+    // expected-error-re@-1{{no viable constructor or deduction guide for deduction of template arguments of '{{(std::)?}}unordered_multimap'}}
+  }
+  {
+    // cannot deduce Key from just (Allocator)
+    std::unordered_multimap m(std::allocator<P>{});
+    // expected-error-re@-1{{no viable constructor or deduction guide for deduction of template arguments of '{{(std::)?}}unordered_multimap'}}
+  }
+  {
+    // cannot deduce Key from just (Size, Allocator)
+    std::unordered_multimap m(42, std::allocator<P>());
+    // expected-error-re@-1{{no viable constructor or deduction guide for deduction of template arguments of '{{(std::)?}}unordered_multimap'}}
+  }
+  {
+    // cannot deduce Key from just (Size, Hash, Allocator)
+    std::unordered_multimap m(42, std::hash<int>(), std::allocator<P>());
+    // expected-error-re@-1{{no viable constructor or deduction guide for deduction of template arguments of '{{(std::)?}}unordered_multimap'}}
+  }
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.cnstr/deduct_const.pass.cpp b/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.cnstr/deduct_const.pass.cpp
index 5a98a013a0933..5d6930a2bdfc4 100644
--- a/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.cnstr/deduct_const.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.cnstr/deduct_const.pass.cpp
@@ -65,109 +65,126 @@
 
 #include "test_allocator.h"
 
-using P = std::pair<int, long>;
+using P  = std::pair<int, long>;
 using PC = std::pair<const int, long>;
 
-int main(int, char**)
-{
-    const PC expected_m[] = { {1,1}, {1,1}, {2,2}, {3,1}, {INT_MAX,1} };
+int main(int, char**) {
+  const PC expected_m[] = {{1, 1}, {1, 1}, {2, 2}, {3, 1}, {INT_MAX, 1}};
 
-    {
-    const PC arr[] = { {1,1}, {2,2}, {1,1}, {INT_MAX,1}, {3,1} };
+  {
+    const PC arr[] = {{1, 1}, {2, 2}, {1, 1}, {INT_MAX, 1}, {3, 1}};
     std::unordered_multimap m(std::begin(arr), std::end(arr));
     ASSERT_SAME_TYPE(decltype(m), std::unordered_multimap<int, long>);
     assert(std::is_permutation(m.begin(), m.end(), std::begin(expected_m), std::end(expected_m)));
-    }
+  }
 
-    {
-    const PC arr[] = { {1,1}, {2,2}, {1,1}, {INT_MAX,1}, {3,1} };
+  {
+    const PC arr[] = {{1, 1}, {2, 2}, {1, 1}, {INT_MAX, 1}, {3, 1}};
     std::unordered_multimap m(std::begin(arr), std::end(arr), 42);
     ASSERT_SAME_TYPE(decltype(m), std::unordered_multimap<int, long>);
     assert(std::is_permutation(m.begin(), m.end(), std::begin(expected_m), std::end(expected_m)));
-    }
+  }
 
-    {
-    const PC arr[] = { {1,1}, {2,2}, {1,1}, {INT_MAX,1}, {3,1} };
+  {
+    const PC arr[] = {{1, 1}, {2, 2}, {1, 1}, {INT_MAX, 1}, {3, 1}};
     std::unordered_multimap m(std::begin(arr), std::end(arr), 42, std::hash<short>());
     ASSERT_SAME_TYPE(decltype(m), std::unordered_multimap<int, long, std::hash<short>, std::equal_to<int>>);
     assert(std::is_permutation(m.begin(), m.end(), std::begin(expected_m), std::end(expected_m)));
-    }
+  }
 
-    {
-    const PC arr[] = { {1,1}, {2,2}, {1,1}, {INT_MAX,1}, {3,1} };
+  {
+    const PC arr[] = {{1, 1}, {2, 2}, {1, 1}, {INT_MAX, 1}, {3, 1}};
     std::unordered_multimap m(std::begin(arr), std::end(arr), 42, std::hash<short>(), std::equal_to<>());
     ASSERT_SAME_TYPE(decltype(m), std::unordered_multimap<int, long, std::hash<short>, std::equal_to<>>);
     assert(std::is_permutation(m.begin(), m.end(), std::begin(expected_m), std::end(expected_m)));
-    }
-
-    {
-    const PC arr[] = { {1,1}, {2,2}, {1,1}, {INT_MAX,1}, {3,1} };
-    std::unordered_multimap m(std::begin(arr), std::end(arr), 42, std::hash<short>(), std::equal_to<>(), test_allocator<PC>(0, 41));
-    ASSERT_SAME_TYPE(decltype(m), std::unordered_multimap<int, long, std::hash<short>, std::equal_to<>, test_allocator<PC>>);
+  }
+
+  {
+    const PC arr[] = {{1, 1}, {2, 2}, {1, 1}, {INT_MAX, 1}, {3, 1}};
+    std::unordered_multimap m(
+        std::begin(arr), std::end(arr), 42, std::hash<short>(), std::equal_to<>(), test_allocator<PC>(0, 41));
+    ASSERT_SAME_TYPE(
+        decltype(m), std::unordered_multimap<int, long, std::hash<short>, std::equal_to<>, test_allocator<PC>>);
     assert(std::is_permutation(m.begin(), m.end(), std::begin(expected_m), std::end(expected_m)));
     assert(m.get_allocator().get_id() == 41);
-    }
+  }
 
-    {
-    std::unordered_multimap m { PC{1,1L}, PC{2,2L}, PC{1,1L}, PC{INT_MAX,1L}, PC{3,1L} };
+  {
+    std::unordered_multimap m{PC{1, 1L}, PC{2, 2L}, PC{1, 1L}, PC{INT_MAX, 1L}, PC{3, 1L}};
     ASSERT_SAME_TYPE(decltype(m), std::unordered_multimap<int, long>);
     assert(std::is_permutation(m.begin(), m.end(), std::begin(expected_m), std::end(expected_m)));
-    }
+  }
 
-    {
-    std::unordered_multimap m({ PC{1,1L}, PC{2,2L}, PC{1,1L}, PC{INT_MAX,1L}, PC{3,1L} }, 42);
+  {
+    std::unordered_multimap m({PC{1, 1L}, PC{2, 2L}, PC{1, 1L}, PC{INT_MAX, 1L}, PC{3, 1L}}, 42);
     ASSERT_SAME_TYPE(decltype(m), std::unordered_multimap<int, long>);
     assert(std::is_permutation(m.begin(), m.end(), std::begin(expected_m), std::end(expected_m)));
-    }
+  }
 
-    {
-    std::unordered_multimap m({ PC{1,1L}, PC{2,2L}, PC{1,1L}, PC{INT_MAX,1L}, PC{3,1L} }, 42, std::hash<short>());
+  {
+    std::unordered_multimap m({PC{1, 1L}, PC{2, 2L}, PC{1, 1L}, PC{INT_MAX, 1L}, PC{3, 1L}}, 42, std::hash<short>());
     ASSERT_SAME_TYPE(decltype(m), std::unordered_multimap<int, long, std::hash<short>>);
     assert(std::is_permutation(m.begin(), m.end(), std::begin(expected_m), std::end(expected_m)));
-    }
+  }
 
-    {
-    std::unordered_multimap m({ PC{1,1L}, PC{2,2L}, PC{1,1L}, PC{INT_MAX,1L}, PC{3,1L} }, 42, std::hash<short>(), std::equal_to<>());
+  {
+    std::unordered_multimap m(
+        {PC{1, 1L}, PC{2, 2L}, PC{1, 1L}, PC{INT_MAX, 1L}, PC{3, 1L}}, 42, std::hash<short>(), std::equal_to<>());
     ASSERT_SAME_TYPE(decltype(m), std::unordered_multimap<int, long, std::hash<short>, std::equal_to<>>);
     assert(std::is_permutation(m.begin(), m.end(), std::begin(expected_m), std::end(expected_m)));
-    }
-
-    {
-    std::unordered_multimap m({ PC{1,1L}, PC{2,2L}, PC{1,1L}, PC{INT_MAX,1L}, PC{3,1L} }, 42, std::hash<short>(), std::equal_to<>(), test_allocator<PC>(0, 44));
-    ASSERT_SAME_TYPE(decltype(m), std::unordered_multimap<int, long, std::hash<short>, std::equal_to<>, test_allocator<PC>>);
+  }
+
+  {
+    std::unordered_multimap m(
+        {PC{1, 1L}, PC{2, 2L}, PC{1, 1L}, PC{INT_MAX, 1L}, PC{3, 1L}},
+        42,
+        std::hash<short>(),
+        std::equal_to<>(),
+        test_allocator<PC>(0, 44));
+    ASSERT_SAME_TYPE(
+        decltype(m), std::unordered_multimap<int, long, std::hash<short>, std::equal_to<>, test_allocator<PC>>);
     assert(std::is_permutation(m.begin(), m.end(), std::begin(expected_m), std::end(expected_m)));
     assert(m.get_allocator().get_id() == 44);
-    }
+  }
 
-    {
-    const PC arr[] = { {1,1}, {2,2}, {1,1}, {INT_MAX,1}, {3,1} };
+  {
+    const PC arr[] = {{1, 1}, {2, 2}, {1, 1}, {INT_MAX, 1}, {3, 1}};
     std::unordered_multimap m(std::begin(arr), std::end(arr), 42, test_allocator<PC>(0, 45));
-    ASSERT_SAME_TYPE(decltype(m), std::unordered_multimap<int, long, std::hash<int>, std::equal_to<int>, test_allocator<PC>>);
+    ASSERT_SAME_TYPE(
+        decltype(m), std::unordered_multimap<int, long, std::hash<int>, std::equal_to<int>, test_allocator<PC>>);
     assert(std::is_permutation(m.begin(), m.end(), std::begin(expected_m), std::end(expected_m)));
     assert(m.get_allocator().get_id() == 45);
-    }
+  }
 
-    {
-    const PC arr[] = { {1,1}, {2,2}, {1,1}, {INT_MAX,1}, {3,1} };
+  {
+    const PC arr[] = {{1, 1}, {2, 2}, {1, 1}, {INT_MAX, 1}, {3, 1}};
     std::unordered_multimap m(std::begin(arr), std::end(arr), 42, std::hash<short>(), test_allocator<PC>(0, 46));
-    ASSERT_SAME_TYPE(decltype(m), std::unordered_multimap<int, long, std::hash<short>, std::equal_to<int>, test_allocator<PC>>);
+    ASSERT_SAME_TYPE(
+        decltype(m), std::unordered_multimap<int, long, std::hash<short>, std::equal_to<int>, test_allocator<PC>>);
     assert(std::is_permutation(m.begin(), m.end(), std::begin(expected_m), std::end(expected_m)));
     assert(m.get_allocator().get_id() == 46);
-    }
+  }
 
-    {
-    std::unordered_multimap m({ PC{1,1L}, PC{2,2L}, PC{1,1L}, PC{INT_MAX,1L}, PC{3,1L} }, 42, test_allocator<PC>(0, 47));
-    ASSERT_SAME_TYPE(decltype(m), std::unordered_multimap<int, long, std::hash<int>, std::equal_to<int>, test_allocator<PC>>);
+  {
+    std::unordered_multimap m(
+        {PC{1, 1L}, PC{2, 2L}, PC{1, 1L}, PC{INT_MAX, 1L}, PC{3, 1L}}, 42, test_allocator<PC>(0, 47));
+    ASSERT_SAME_TYPE(
+        decltype(m), std::unordered_multimap<int, long, std::hash<int>, std::equal_to<int>, test_allocator<PC>>);
     assert(std::is_permutation(m.begin(), m.end(), std::begin(expected_m), std::end(expected_m)));
     assert(m.get_allocator().get_id() == 47);
-    }
-
-    {
-    std::unordered_multimap m({ PC{1,1L}, PC{2,2L}, PC{1,1L}, PC{INT_MAX,1L}, PC{3,1L} }, 42, std::hash<short>(), test_allocator<PC>(0, 48));
-    ASSERT_SAME_TYPE(decltype(m), std::unordered_multimap<int, long, std::hash<short>, std::equal_to<int>, test_allocator<PC>>);
+  }
+
+  {
+    std::unordered_multimap m(
+        {PC{1, 1L}, PC{2, 2L}, PC{1, 1L}, PC{INT_MAX, 1L}, PC{3, 1L}},
+        42,
+        std::hash<short>(),
+        test_allocator<PC>(0, 48));
+    ASSERT_SAME_TYPE(
+        decltype(m), std::unordered_multimap<int, long, std::hash<short>, std::equal_to<int>, test_allocator<PC>>);
     assert(std::is_permutation(m.begin(), m.end(), std::begin(expected_m), std::end(expected_m)));
     assert(m.get_allocator().get_id() == 48);
-    }
+  }
 
-    return 0;
+  return 0;
 }
diff --git a/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.cnstr/default.pass.cpp b/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.cnstr/default.pass.cpp
index deea354543093..4e66768db1b22 100644
--- a/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.cnstr/default.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.cnstr/default.pass.cpp
@@ -24,89 +24,87 @@
 #include "test_allocator.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef std::unordered_multimap<NotConstructible, NotConstructible,
-                                   test_hash<NotConstructible>,
-                                   test_equal_to<NotConstructible>,
-                                   test_allocator<std::pair<const NotConstructible,
-                                                                  NotConstructible> >
-                                   > C;
-        C c;
-        LIBCPP_ASSERT(c.bucket_count() == 0);
-        assert(c.hash_function() == test_hash<NotConstructible>());
-        assert(c.key_eq() == test_equal_to<NotConstructible>());
-        assert(c.get_allocator() ==
-               (test_allocator<std::pair<const NotConstructible, NotConstructible> >()));
-        assert(c.size() == 0);
-        assert(c.empty());
-        assert(std::distance(c.begin(), c.end()) == 0);
-        assert(c.load_factor() == 0);
-        assert(c.max_load_factor() == 1);
-    }
+int main(int, char**) {
+  {
+    typedef std::unordered_multimap<NotConstructible,
+                                    NotConstructible,
+                                    test_hash<NotConstructible>,
+                                    test_equal_to<NotConstructible>,
+                                    test_allocator<std::pair<const NotConstructible, NotConstructible> > >
+        C;
+    C c;
+    LIBCPP_ASSERT(c.bucket_count() == 0);
+    assert(c.hash_function() == test_hash<NotConstructible>());
+    assert(c.key_eq() == test_equal_to<NotConstructible>());
+    assert(c.get_allocator() == (test_allocator<std::pair<const NotConstructible, NotConstructible> >()));
+    assert(c.size() == 0);
+    assert(c.empty());
+    assert(std::distance(c.begin(), c.end()) == 0);
+    assert(c.load_factor() == 0);
+    assert(c.max_load_factor() == 1);
+  }
 #if TEST_STD_VER >= 11
+  {
+    typedef std::unordered_multimap<NotConstructible,
+                                    NotConstructible,
+                                    test_hash<NotConstructible>,
+                                    test_equal_to<NotConstructible>,
+                                    min_allocator<std::pair<const NotConstructible, NotConstructible> > >
+        C;
+    C c;
+    LIBCPP_ASSERT(c.bucket_count() == 0);
+    assert(c.hash_function() == test_hash<NotConstructible>());
+    assert(c.key_eq() == test_equal_to<NotConstructible>());
+    assert(c.get_allocator() == (min_allocator<std::pair<const NotConstructible, NotConstructible> >()));
+    assert(c.size() == 0);
+    assert(c.empty());
+    assert(std::distance(c.begin(), c.end()) == 0);
+    assert(c.load_factor() == 0);
+    assert(c.max_load_factor() == 1);
+  }
+  {
+    typedef explicit_allocator<std::pair<const NotConstructible, NotConstructible>> A;
+    typedef std::unordered_multimap<NotConstructible,
+                                    NotConstructible,
+                                    test_hash<NotConstructible>,
+                                    test_equal_to<NotConstructible>,
+                                    A >
+        C;
     {
-        typedef std::unordered_multimap<NotConstructible, NotConstructible,
-                                   test_hash<NotConstructible>,
-                                   test_equal_to<NotConstructible>,
-                                   min_allocator<std::pair<const NotConstructible,
-                                                                 NotConstructible> >
-                                   > C;
-        C c;
-        LIBCPP_ASSERT(c.bucket_count() == 0);
-        assert(c.hash_function() == test_hash<NotConstructible>());
-        assert(c.key_eq() == test_equal_to<NotConstructible>());
-        assert(c.get_allocator() ==
-               (min_allocator<std::pair<const NotConstructible, NotConstructible> >()));
-        assert(c.size() == 0);
-        assert(c.empty());
-        assert(std::distance(c.begin(), c.end()) == 0);
-        assert(c.load_factor() == 0);
-        assert(c.max_load_factor() == 1);
-    }
-    {
-        typedef explicit_allocator<std::pair<const NotConstructible, NotConstructible>> A;
-        typedef std::unordered_multimap<NotConstructible, NotConstructible,
-                                   test_hash<NotConstructible>,
-                                   test_equal_to<NotConstructible>,
-                                   A
-                                   > C;
-        {
-        C c;
-        LIBCPP_ASSERT(c.bucket_count() == 0);
-        assert(c.hash_function() == test_hash<NotConstructible>());
-        assert(c.key_eq() == test_equal_to<NotConstructible>());
-        assert(c.get_allocator() == A());
-        assert(c.size() == 0);
-        assert(c.empty());
-        assert(std::distance(c.begin(), c.end()) == 0);
-        assert(c.load_factor() == 0);
-        assert(c.max_load_factor() == 1);
-        }
-        {
-        A a;
-        C c(a);
-        LIBCPP_ASSERT(c.bucket_count() == 0);
-        assert(c.hash_function() == test_hash<NotConstructible>());
-        assert(c.key_eq() == test_equal_to<NotConstructible>());
-        assert(c.get_allocator() == a);
-        assert(c.size() == 0);
-        assert(c.empty());
-        assert(std::distance(c.begin(), c.end()) == 0);
-        assert(c.load_factor() == 0);
-        assert(c.max_load_factor() == 1);
-        }
+      C c;
+      LIBCPP_ASSERT(c.bucket_count() == 0);
+      assert(c.hash_function() == test_hash<NotConstructible>());
+      assert(c.key_eq() == test_equal_to<NotConstructible>());
+      assert(c.get_allocator() == A());
+      assert(c.size() == 0);
+      assert(c.empty());
+      assert(std::distance(c.begin(), c.end()) == 0);
+      assert(c.load_factor() == 0);
+      assert(c.max_load_factor() == 1);
     }
     {
-        std::unordered_multimap<int, int> c = {};
-        LIBCPP_ASSERT(c.bucket_count() == 0);
-        assert(c.size() == 0);
-        assert(c.empty());
-        assert(std::distance(c.begin(), c.end()) == 0);
-        assert(c.load_factor() == 0);
-        assert(c.max_load_factor() == 1);
+      A a;
+      C c(a);
+      LIBCPP_ASSERT(c.bucket_count() == 0);
+      assert(c.hash_function() == test_hash<NotConstructible>());
+      assert(c.key_eq() == test_equal_to<NotConstructible>());
+      assert(c.get_allocator() == a);
+      assert(c.size() == 0);
+      assert(c.empty());
+      assert(std::distance(c.begin(), c.end()) == 0);
+      assert(c.load_factor() == 0);
+      assert(c.max_load_factor() == 1);
     }
+  }
+  {
+    std::unordered_multimap<int, int> c = {};
+    LIBCPP_ASSERT(c.bucket_count() == 0);
+    assert(c.size() == 0);
+    assert(c.empty());
+    assert(std::distance(c.begin(), c.end()) == 0);
+    assert(c.load_factor() == 0);
+    assert(c.max_load_factor() == 1);
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.cnstr/default_noexcept.pass.cpp b/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.cnstr/default_noexcept.pass.cpp
index 394600a43b388..5e8f92d8ebdd3 100644
--- a/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.cnstr/default_noexcept.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.cnstr/default_noexcept.pass.cpp
@@ -27,50 +27,54 @@
 #include "../../../test_hash.h"
 
 template <class T>
-struct some_comp
-{
-    typedef T value_type;
-    some_comp();
-    some_comp(const some_comp&);
-    bool operator()(const T&, const T&) const { return false; }
+struct some_comp {
+  typedef T value_type;
+  some_comp();
+  some_comp(const some_comp&);
+  bool operator()(const T&, const T&) const { return false; }
 };
 
 template <class T>
-struct some_hash
-{
-    typedef T value_type;
-    some_hash();
-    some_hash(const some_hash&);
-    std::size_t operator()(T const&) const;
+struct some_hash {
+  typedef T value_type;
+  some_hash();
+  some_hash(const some_hash&);
+  std::size_t operator()(T const&) const;
 };
 
-int main(int, char**)
-{
+int main(int, char**) {
 #if defined(_LIBCPP_VERSION)
-    {
-        typedef std::unordered_multimap<MoveOnly, MoveOnly> C;
-        static_assert(std::is_nothrow_default_constructible<C>::value, "");
-    }
-    {
-        typedef std::unordered_multimap<MoveOnly, MoveOnly, std::hash<MoveOnly>,
-                           std::equal_to<MoveOnly>, test_allocator<std::pair<const MoveOnly, MoveOnly>>> C;
-        static_assert(std::is_nothrow_default_constructible<C>::value, "");
-    }
+  {
+    typedef std::unordered_multimap<MoveOnly, MoveOnly> C;
+    static_assert(std::is_nothrow_default_constructible<C>::value, "");
+  }
+  {
+    typedef std::unordered_multimap<MoveOnly,
+                                    MoveOnly,
+                                    std::hash<MoveOnly>,
+                                    std::equal_to<MoveOnly>,
+                                    test_allocator<std::pair<const MoveOnly, MoveOnly>>>
+        C;
+    static_assert(std::is_nothrow_default_constructible<C>::value, "");
+  }
 #endif // _LIBCPP_VERSION
-    {
-        typedef std::unordered_multimap<MoveOnly, MoveOnly, std::hash<MoveOnly>,
-                          std::equal_to<MoveOnly>, other_allocator<std::pair<const MoveOnly, MoveOnly>>> C;
-        static_assert(!std::is_nothrow_default_constructible<C>::value, "");
-    }
-    {
-        typedef std::unordered_multimap<MoveOnly, MoveOnly, some_hash<MoveOnly>> C;
-        static_assert(!std::is_nothrow_default_constructible<C>::value, "");
-    }
-    {
-        typedef std::unordered_multimap<MoveOnly, MoveOnly, std::hash<MoveOnly>,
-                                                         some_comp<MoveOnly>> C;
-        static_assert(!std::is_nothrow_default_constructible<C>::value, "");
-    }
+  {
+    typedef std::unordered_multimap<MoveOnly,
+                                    MoveOnly,
+                                    std::hash<MoveOnly>,
+                                    std::equal_to<MoveOnly>,
+                                    other_allocator<std::pair<const MoveOnly, MoveOnly>>>
+        C;
+    static_assert(!std::is_nothrow_default_constructible<C>::value, "");
+  }
+  {
+    typedef std::unordered_multimap<MoveOnly, MoveOnly, some_hash<MoveOnly>> C;
+    static_assert(!std::is_nothrow_default_constructible<C>::value, "");
+  }
+  {
+    typedef std::unordered_multimap<MoveOnly, MoveOnly, std::hash<MoveOnly>, some_comp<MoveOnly>> C;
+    static_assert(!std::is_nothrow_default_constructible<C>::value, "");
+  }
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.cnstr/dtor_noexcept.pass.cpp b/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.cnstr/dtor_noexcept.pass.cpp
index e8160e160549b..6fdc9e3eb7f5e 100644
--- a/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.cnstr/dtor_noexcept.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.cnstr/dtor_noexcept.pass.cpp
@@ -20,49 +20,53 @@
 #include "test_allocator.h"
 
 template <class T>
-struct some_comp
-{
-    typedef T value_type;
-    ~some_comp() noexcept(false);
-    bool operator()(const T&, const T&) const { return false; }
+struct some_comp {
+  typedef T value_type;
+  ~some_comp() noexcept(false);
+  bool operator()(const T&, const T&) const { return false; }
 };
 
 template <class T>
-struct some_hash
-{
-    typedef T value_type;
-    some_hash();
-    some_hash(const some_hash&);
-    ~some_hash() noexcept(false);
-    std::size_t operator()(T const&) const;
+struct some_hash {
+  typedef T value_type;
+  some_hash();
+  some_hash(const some_hash&);
+  ~some_hash() noexcept(false);
+  std::size_t operator()(T const&) const;
 };
 
-int main(int, char**)
-{
-    {
-        typedef std::unordered_multimap<MoveOnly, MoveOnly> C;
-        static_assert(std::is_nothrow_destructible<C>::value, "");
-    }
-    {
-        typedef std::unordered_multimap<MoveOnly, MoveOnly, std::hash<MoveOnly>,
-                           std::equal_to<MoveOnly>, test_allocator<std::pair<const MoveOnly, MoveOnly>>> C;
-        static_assert(std::is_nothrow_destructible<C>::value, "");
-    }
-    {
-        typedef std::unordered_multimap<MoveOnly, MoveOnly, std::hash<MoveOnly>,
-                          std::equal_to<MoveOnly>, other_allocator<std::pair<const MoveOnly, MoveOnly>>> C;
-        static_assert(std::is_nothrow_destructible<C>::value, "");
-    }
+int main(int, char**) {
+  {
+    typedef std::unordered_multimap<MoveOnly, MoveOnly> C;
+    static_assert(std::is_nothrow_destructible<C>::value, "");
+  }
+  {
+    typedef std::unordered_multimap<MoveOnly,
+                                    MoveOnly,
+                                    std::hash<MoveOnly>,
+                                    std::equal_to<MoveOnly>,
+                                    test_allocator<std::pair<const MoveOnly, MoveOnly>>>
+        C;
+    static_assert(std::is_nothrow_destructible<C>::value, "");
+  }
+  {
+    typedef std::unordered_multimap<MoveOnly,
+                                    MoveOnly,
+                                    std::hash<MoveOnly>,
+                                    std::equal_to<MoveOnly>,
+                                    other_allocator<std::pair<const MoveOnly, MoveOnly>>>
+        C;
+    static_assert(std::is_nothrow_destructible<C>::value, "");
+  }
 #if defined(_LIBCPP_VERSION)
-    {
-        typedef std::unordered_multimap<MoveOnly, MoveOnly, some_hash<MoveOnly>> C;
-        static_assert(!std::is_nothrow_destructible<C>::value, "");
-    }
-    {
-        typedef std::unordered_multimap<MoveOnly, MoveOnly, std::hash<MoveOnly>,
-                                                         some_comp<MoveOnly>> C;
-        static_assert(!std::is_nothrow_destructible<C>::value, "");
-    }
+  {
+    typedef std::unordered_multimap<MoveOnly, MoveOnly, some_hash<MoveOnly>> C;
+    static_assert(!std::is_nothrow_destructible<C>::value, "");
+  }
+  {
+    typedef std::unordered_multimap<MoveOnly, MoveOnly, std::hash<MoveOnly>, some_comp<MoveOnly>> C;
+    static_assert(!std::is_nothrow_destructible<C>::value, "");
+  }
 #endif // _LIBCPP_VERSION
 
   return 0;
diff --git a/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.cnstr/from_range.pass.cpp b/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.cnstr/from_range.pass.cpp
index 4fe95da010394..df8b3c54ff1bc 100644
--- a/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.cnstr/from_range.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.cnstr/from_range.pass.cpp
@@ -28,11 +28,9 @@
 #include "test_macros.h"
 
 void test_duplicates() {
-  using T = std::pair<const int, char>;
-  std::array input = {
-    T{1, 'a'}, T{2, 'a'}, T{3, 'a'}, T{3, 'b'}, T{3, 'c'}, T{2, 'b'}, T{4, 'a'}
-  };
-  auto c = std::unordered_multimap<int, char>(std::from_range, input);
+  using T          = std::pair<const int, char>;
+  std::array input = {T{1, 'a'}, T{2, 'a'}, T{3, 'a'}, T{3, 'b'}, T{3, 'c'}, T{2, 'b'}, T{4, 'a'}};
+  auto c           = std::unordered_multimap<int, char>(std::from_range, input);
   assert(std::ranges::is_permutation(input, c));
 }
 
diff --git a/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.cnstr/hash_copy_constructible.compile.fail.cpp b/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.cnstr/hash_copy_constructible.compile.fail.cpp
index 77faad2e76542..5949334c5a3fb 100644
--- a/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.cnstr/hash_copy_constructible.compile.fail.cpp
+++ b/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.cnstr/hash_copy_constructible.compile.fail.cpp
@@ -18,16 +18,16 @@
 
 template <class T>
 struct Hash {
-    std::size_t operator () (const T& lhs) const { return 0; }
+  std::size_t operator()(const T& lhs) const { return 0; }
 
-    Hash () {}
-private:
-    Hash (const Hash &); // declared but not defined
-    };
+  Hash() {}
 
+private:
+  Hash(const Hash&); // declared but not defined
+};
 
 int main(int, char**) {
-    std::unordered_multimap<int, int, Hash<int> > m;
+  std::unordered_multimap<int, int, Hash<int> > m;
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.cnstr/init.pass.cpp b/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.cnstr/init.pass.cpp
index 8486062dbc250..a1c0be2a3c64e 100644
--- a/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.cnstr/init.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.cnstr/init.pass.cpp
@@ -31,207 +31,215 @@
 #include "test_allocator.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef std::unordered_multimap<int, std::string,
-                                   test_hash<int>,
-                                   test_equal_to<int>,
-                                   test_allocator<std::pair<const int, std::string> >
-                                   > C;
-        typedef std::pair<int, std::string> P;
-        C c = {
-                P(1, "one"),
-                P(2, "two"),
-                P(3, "three"),
-                P(4, "four"),
-                P(1, "four"),
-                P(2, "four"),
-              };
-        assert(c.bucket_count() >= 7);
-        assert(c.size() == 6);
-        typedef std::pair<C::const_iterator, C::const_iterator> Eq;
-        Eq eq = c.equal_range(1);
-        assert(std::distance(eq.first, eq.second) == 2);
-        std::multiset<std::string> s;
-        s.insert("one");
-        s.insert("four");
-        CheckConsecutiveKeys<C::const_iterator>(c.find(1), c.end(), 1, s);
-        eq = c.equal_range(2);
-        assert(std::distance(eq.first, eq.second) == 2);
-        s.insert("two");
-        s.insert("four");
-        CheckConsecutiveKeys<C::const_iterator>(c.find(2), c.end(), 2, s);
+int main(int, char**) {
+  {
+    typedef std::unordered_multimap<int,
+                                    std::string,
+                                    test_hash<int>,
+                                    test_equal_to<int>,
+                                    test_allocator<std::pair<const int, std::string> > >
+        C;
+    typedef std::pair<int, std::string> P;
+    C c = {
+        P(1, "one"),
+        P(2, "two"),
+        P(3, "three"),
+        P(4, "four"),
+        P(1, "four"),
+        P(2, "four"),
+    };
+    assert(c.bucket_count() >= 7);
+    assert(c.size() == 6);
+    typedef std::pair<C::const_iterator, C::const_iterator> Eq;
+    Eq eq = c.equal_range(1);
+    assert(std::distance(eq.first, eq.second) == 2);
+    std::multiset<std::string> s;
+    s.insert("one");
+    s.insert("four");
+    CheckConsecutiveKeys<C::const_iterator>(c.find(1), c.end(), 1, s);
+    eq = c.equal_range(2);
+    assert(std::distance(eq.first, eq.second) == 2);
+    s.insert("two");
+    s.insert("four");
+    CheckConsecutiveKeys<C::const_iterator>(c.find(2), c.end(), 2, s);
 
-        eq = c.equal_range(3);
-        assert(std::distance(eq.first, eq.second) == 1);
-        C::const_iterator i = eq.first;
-        assert(i->first == 3);
-        assert(i->second == "three");
-        eq = c.equal_range(4);
-        assert(std::distance(eq.first, eq.second) == 1);
-        i = eq.first;
-        assert(i->first == 4);
-        assert(i->second == "four");
-        assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-        assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
-        assert(fabs(c.load_factor() - (float)c.size()/c.bucket_count()) < FLT_EPSILON);
-        assert(c.max_load_factor() == 1);
-        assert(c.hash_function() == test_hash<int>());
-        assert(c.key_eq() == test_equal_to<int>());
-        assert((c.get_allocator() == test_allocator<std::pair<const int, std::string> >()));
-    }
-    {
-        typedef std::unordered_multimap<int, std::string,
-                                   test_hash<int>,
-                                   test_equal_to<int>,
-                                   min_allocator<std::pair<const int, std::string> >
-                                   > C;
-        typedef std::pair<int, std::string> P;
-        C c = {
-                P(1, "one"),
-                P(2, "two"),
-                P(3, "three"),
-                P(4, "four"),
-                P(1, "four"),
-                P(2, "four"),
-              };
-        assert(c.bucket_count() >= 7);
-        assert(c.size() == 6);
-        typedef std::pair<C::const_iterator, C::const_iterator> Eq;
-        Eq eq = c.equal_range(1);
-        assert(std::distance(eq.first, eq.second) == 2);
-        std::multiset<std::string> s;
-        s.insert("one");
-        s.insert("four");
-        CheckConsecutiveKeys<C::const_iterator>(c.find(1), c.end(), 1, s);
-        eq = c.equal_range(2);
-        assert(std::distance(eq.first, eq.second) == 2);
-        s.insert("two");
-        s.insert("four");
-        CheckConsecutiveKeys<C::const_iterator>(c.find(2), c.end(), 2, s);
+    eq = c.equal_range(3);
+    assert(std::distance(eq.first, eq.second) == 1);
+    C::const_iterator i = eq.first;
+    assert(i->first == 3);
+    assert(i->second == "three");
+    eq = c.equal_range(4);
+    assert(std::distance(eq.first, eq.second) == 1);
+    i = eq.first;
+    assert(i->first == 4);
+    assert(i->second == "four");
+    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
+    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
+    assert(fabs(c.load_factor() - (float)c.size() / c.bucket_count()) < FLT_EPSILON);
+    assert(c.max_load_factor() == 1);
+    assert(c.hash_function() == test_hash<int>());
+    assert(c.key_eq() == test_equal_to<int>());
+    assert((c.get_allocator() == test_allocator<std::pair<const int, std::string> >()));
+  }
+  {
+    typedef std::unordered_multimap<int,
+                                    std::string,
+                                    test_hash<int>,
+                                    test_equal_to<int>,
+                                    min_allocator<std::pair<const int, std::string> > >
+        C;
+    typedef std::pair<int, std::string> P;
+    C c = {
+        P(1, "one"),
+        P(2, "two"),
+        P(3, "three"),
+        P(4, "four"),
+        P(1, "four"),
+        P(2, "four"),
+    };
+    assert(c.bucket_count() >= 7);
+    assert(c.size() == 6);
+    typedef std::pair<C::const_iterator, C::const_iterator> Eq;
+    Eq eq = c.equal_range(1);
+    assert(std::distance(eq.first, eq.second) == 2);
+    std::multiset<std::string> s;
+    s.insert("one");
+    s.insert("four");
+    CheckConsecutiveKeys<C::const_iterator>(c.find(1), c.end(), 1, s);
+    eq = c.equal_range(2);
+    assert(std::distance(eq.first, eq.second) == 2);
+    s.insert("two");
+    s.insert("four");
+    CheckConsecutiveKeys<C::const_iterator>(c.find(2), c.end(), 2, s);
 
-        eq = c.equal_range(3);
-        assert(std::distance(eq.first, eq.second) == 1);
-        C::const_iterator i = eq.first;
-        assert(i->first == 3);
-        assert(i->second == "three");
-        eq = c.equal_range(4);
-        assert(std::distance(eq.first, eq.second) == 1);
-        i = eq.first;
-        assert(i->first == 4);
-        assert(i->second == "four");
-        assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-        assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
-        assert(fabs(c.load_factor() - (float)c.size()/c.bucket_count()) < FLT_EPSILON);
-        assert(c.max_load_factor() == 1);
-        assert(c.hash_function() == test_hash<int>());
-        assert(c.key_eq() == test_equal_to<int>());
-        assert((c.get_allocator() == min_allocator<std::pair<const int, std::string> >()));
-    }
+    eq = c.equal_range(3);
+    assert(std::distance(eq.first, eq.second) == 1);
+    C::const_iterator i = eq.first;
+    assert(i->first == 3);
+    assert(i->second == "three");
+    eq = c.equal_range(4);
+    assert(std::distance(eq.first, eq.second) == 1);
+    i = eq.first;
+    assert(i->first == 4);
+    assert(i->second == "four");
+    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
+    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
+    assert(fabs(c.load_factor() - (float)c.size() / c.bucket_count()) < FLT_EPSILON);
+    assert(c.max_load_factor() == 1);
+    assert(c.hash_function() == test_hash<int>());
+    assert(c.key_eq() == test_equal_to<int>());
+    assert((c.get_allocator() == min_allocator<std::pair<const int, std::string> >()));
+  }
 #if TEST_STD_VER > 11
-    {
-        typedef std::pair<int, std::string> P;
-        typedef test_allocator<std::pair<const int, std::string>> A;
-        typedef test_hash<int> HF;
-        typedef test_equal_to<int> Comp;
-        typedef std::unordered_multimap<int, std::string, HF, Comp, A> C;
+  {
+    typedef std::pair<int, std::string> P;
+    typedef test_allocator<std::pair<const int, std::string>> A;
+    typedef test_hash<int> HF;
+    typedef test_equal_to<int> Comp;
+    typedef std::unordered_multimap<int, std::string, HF, Comp, A> C;
 
-        A a(42);
-        C c ({
-                P(1, "one"),
-                P(2, "two"),
-                P(3, "three"),
-                P(4, "four"),
-                P(1, "four"),
-                P(2, "four"),
-              }, 12, a );
-        assert(c.bucket_count() >= 12);
-        assert(c.size() == 6);
-        typedef std::pair<C::const_iterator, C::const_iterator> Eq;
-        Eq eq = c.equal_range(1);
-        assert(std::distance(eq.first, eq.second) == 2);
-        std::multiset<std::string> s;
-        s.insert("one");
-        s.insert("four");
-        CheckConsecutiveKeys<C::const_iterator>(c.find(1), c.end(), 1, s);
-        eq = c.equal_range(2);
-        assert(std::distance(eq.first, eq.second) == 2);
-        s.insert("two");
-        s.insert("four");
-        CheckConsecutiveKeys<C::const_iterator>(c.find(2), c.end(), 2, s);
+    A a(42);
+    C c(
+        {
+            P(1, "one"),
+            P(2, "two"),
+            P(3, "three"),
+            P(4, "four"),
+            P(1, "four"),
+            P(2, "four"),
+        },
+        12,
+        a);
+    assert(c.bucket_count() >= 12);
+    assert(c.size() == 6);
+    typedef std::pair<C::const_iterator, C::const_iterator> Eq;
+    Eq eq = c.equal_range(1);
+    assert(std::distance(eq.first, eq.second) == 2);
+    std::multiset<std::string> s;
+    s.insert("one");
+    s.insert("four");
+    CheckConsecutiveKeys<C::const_iterator>(c.find(1), c.end(), 1, s);
+    eq = c.equal_range(2);
+    assert(std::distance(eq.first, eq.second) == 2);
+    s.insert("two");
+    s.insert("four");
+    CheckConsecutiveKeys<C::const_iterator>(c.find(2), c.end(), 2, s);
 
-        eq = c.equal_range(3);
-        assert(std::distance(eq.first, eq.second) == 1);
-        C::const_iterator i = eq.first;
-        assert(i->first == 3);
-        assert(i->second == "three");
-        eq = c.equal_range(4);
-        assert(std::distance(eq.first, eq.second) == 1);
-        i = eq.first;
-        assert(i->first == 4);
-        assert(i->second == "four");
-        assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-        assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
-        assert(fabs(c.load_factor() - (float)c.size()/c.bucket_count()) < FLT_EPSILON);
-        assert(c.max_load_factor() == 1);
-        assert(c.hash_function() == HF());
-        assert(c.key_eq() == Comp());
-        assert(c.get_allocator() == a);
-        assert(!(c.get_allocator() == A()));
-    }
-    {
-        typedef std::pair<int, std::string> P;
-        typedef test_allocator<std::pair<const int, std::string>> A;
-        typedef test_hash<int> HF;
-        typedef test_equal_to<int> Comp;
-        typedef std::unordered_multimap<int, std::string, HF, Comp, A> C;
+    eq = c.equal_range(3);
+    assert(std::distance(eq.first, eq.second) == 1);
+    C::const_iterator i = eq.first;
+    assert(i->first == 3);
+    assert(i->second == "three");
+    eq = c.equal_range(4);
+    assert(std::distance(eq.first, eq.second) == 1);
+    i = eq.first;
+    assert(i->first == 4);
+    assert(i->second == "four");
+    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
+    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
+    assert(fabs(c.load_factor() - (float)c.size() / c.bucket_count()) < FLT_EPSILON);
+    assert(c.max_load_factor() == 1);
+    assert(c.hash_function() == HF());
+    assert(c.key_eq() == Comp());
+    assert(c.get_allocator() == a);
+    assert(!(c.get_allocator() == A()));
+  }
+  {
+    typedef std::pair<int, std::string> P;
+    typedef test_allocator<std::pair<const int, std::string>> A;
+    typedef test_hash<int> HF;
+    typedef test_equal_to<int> Comp;
+    typedef std::unordered_multimap<int, std::string, HF, Comp, A> C;
 
-        HF hf(42);
-        A a(43);
-        C c ({
-                P(1, "one"),
-                P(2, "two"),
-                P(3, "three"),
-                P(4, "four"),
-                P(1, "four"),
-                P(2, "four"),
-              }, 12, hf, a );
-        assert(c.bucket_count() >= 12);
-        assert(c.size() == 6);
-        typedef std::pair<C::const_iterator, C::const_iterator> Eq;
-        Eq eq = c.equal_range(1);
-        assert(std::distance(eq.first, eq.second) == 2);
-        std::multiset<std::string> s;
-        s.insert("one");
-        s.insert("four");
-        CheckConsecutiveKeys<C::const_iterator>(c.find(1), c.end(), 1, s);
-        eq = c.equal_range(2);
-        assert(std::distance(eq.first, eq.second) == 2);
-        s.insert("two");
-        s.insert("four");
-        CheckConsecutiveKeys<C::const_iterator>(c.find(2), c.end(), 2, s);
+    HF hf(42);
+    A a(43);
+    C c(
+        {
+            P(1, "one"),
+            P(2, "two"),
+            P(3, "three"),
+            P(4, "four"),
+            P(1, "four"),
+            P(2, "four"),
+        },
+        12,
+        hf,
+        a);
+    assert(c.bucket_count() >= 12);
+    assert(c.size() == 6);
+    typedef std::pair<C::const_iterator, C::const_iterator> Eq;
+    Eq eq = c.equal_range(1);
+    assert(std::distance(eq.first, eq.second) == 2);
+    std::multiset<std::string> s;
+    s.insert("one");
+    s.insert("four");
+    CheckConsecutiveKeys<C::const_iterator>(c.find(1), c.end(), 1, s);
+    eq = c.equal_range(2);
+    assert(std::distance(eq.first, eq.second) == 2);
+    s.insert("two");
+    s.insert("four");
+    CheckConsecutiveKeys<C::const_iterator>(c.find(2), c.end(), 2, s);
 
-        eq = c.equal_range(3);
-        assert(std::distance(eq.first, eq.second) == 1);
-        C::const_iterator i = eq.first;
-        assert(i->first == 3);
-        assert(i->second == "three");
-        eq = c.equal_range(4);
-        assert(std::distance(eq.first, eq.second) == 1);
-        i = eq.first;
-        assert(i->first == 4);
-        assert(i->second == "four");
-        assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-        assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
-        assert(fabs(c.load_factor() - (float)c.size()/c.bucket_count()) < FLT_EPSILON);
-        assert(c.max_load_factor() == 1);
-        assert(c.hash_function() == hf);
-        assert(!(c.hash_function() == HF()));
-        assert(c.key_eq() == Comp());
-        assert(c.get_allocator() == a);
-        assert(!(c.get_allocator() == A()));
-    }
+    eq = c.equal_range(3);
+    assert(std::distance(eq.first, eq.second) == 1);
+    C::const_iterator i = eq.first;
+    assert(i->first == 3);
+    assert(i->second == "three");
+    eq = c.equal_range(4);
+    assert(std::distance(eq.first, eq.second) == 1);
+    i = eq.first;
+    assert(i->first == 4);
+    assert(i->second == "four");
+    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
+    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
+    assert(fabs(c.load_factor() - (float)c.size() / c.bucket_count()) < FLT_EPSILON);
+    assert(c.max_load_factor() == 1);
+    assert(c.hash_function() == hf);
+    assert(!(c.hash_function() == HF()));
+    assert(c.key_eq() == Comp());
+    assert(c.get_allocator() == a);
+    assert(!(c.get_allocator() == A()));
+  }
 #endif // TEST_STD_VER > 11
 
   return 0;
diff --git a/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.cnstr/init_size.pass.cpp b/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.cnstr/init_size.pass.cpp
index c5cb706269e21..6231a02be9ff1 100644
--- a/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.cnstr/init_size.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.cnstr/init_size.pass.cpp
@@ -31,108 +31,109 @@
 #include "test_allocator.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef std::unordered_multimap<int, std::string,
-                                   test_hash<int>,
-                                   test_equal_to<int>,
-                                   test_allocator<std::pair<const int, std::string> >
-                                   > C;
-        typedef std::pair<int, std::string> P;
-        C c({
-                P(1, "one"),
-                P(2, "two"),
-                P(3, "three"),
-                P(4, "four"),
-                P(1, "four"),
-                P(2, "four"),
-            },
-            7
-           );
-        LIBCPP_ASSERT(c.bucket_count() == 7);
-        assert(c.size() == 6);
-        typedef std::pair<C::const_iterator, C::const_iterator> Eq;
-        Eq eq = c.equal_range(1);
-        assert(std::distance(eq.first, eq.second) == 2);
-        std::multiset<std::string> s;
-        s.insert("one");
-        s.insert("four");
-        CheckConsecutiveKeys<C::const_iterator>(c.find(1), c.end(), 1, s);
-        eq = c.equal_range(2);
-        assert(std::distance(eq.first, eq.second) == 2);
-        s.insert("two");
-        s.insert("four");
-        CheckConsecutiveKeys<C::const_iterator>(c.find(2), c.end(), 2, s);
+int main(int, char**) {
+  {
+    typedef std::unordered_multimap<int,
+                                    std::string,
+                                    test_hash<int>,
+                                    test_equal_to<int>,
+                                    test_allocator<std::pair<const int, std::string> > >
+        C;
+    typedef std::pair<int, std::string> P;
+    C c(
+        {
+            P(1, "one"),
+            P(2, "two"),
+            P(3, "three"),
+            P(4, "four"),
+            P(1, "four"),
+            P(2, "four"),
+        },
+        7);
+    LIBCPP_ASSERT(c.bucket_count() == 7);
+    assert(c.size() == 6);
+    typedef std::pair<C::const_iterator, C::const_iterator> Eq;
+    Eq eq = c.equal_range(1);
+    assert(std::distance(eq.first, eq.second) == 2);
+    std::multiset<std::string> s;
+    s.insert("one");
+    s.insert("four");
+    CheckConsecutiveKeys<C::const_iterator>(c.find(1), c.end(), 1, s);
+    eq = c.equal_range(2);
+    assert(std::distance(eq.first, eq.second) == 2);
+    s.insert("two");
+    s.insert("four");
+    CheckConsecutiveKeys<C::const_iterator>(c.find(2), c.end(), 2, s);
 
-        eq = c.equal_range(3);
-        assert(std::distance(eq.first, eq.second) == 1);
-        C::const_iterator i = eq.first;
-        assert(i->first == 3);
-        assert(i->second == "three");
-        eq = c.equal_range(4);
-        assert(std::distance(eq.first, eq.second) == 1);
-        i = eq.first;
-        assert(i->first == 4);
-        assert(i->second == "four");
-        assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-        assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
-        assert(std::fabs(c.load_factor() - (float)c.size()/c.bucket_count()) < FLT_EPSILON);
-        assert(c.max_load_factor() == 1);
-        assert(c.hash_function() == test_hash<int>());
-        assert(c.key_eq() == test_equal_to<int>());
-        assert((c.get_allocator() == test_allocator<std::pair<const int, std::string> >()));
-    }
-    {
-        typedef std::unordered_multimap<int, std::string,
-                                   test_hash<int>,
-                                   test_equal_to<int>,
-                                   min_allocator<std::pair<const int, std::string> >
-                                   > C;
-        typedef std::pair<int, std::string> P;
-        C c({
-                P(1, "one"),
-                P(2, "two"),
-                P(3, "three"),
-                P(4, "four"),
-                P(1, "four"),
-                P(2, "four"),
-            },
-            7
-           );
-        LIBCPP_ASSERT(c.bucket_count() == 7);
-        assert(c.size() == 6);
-        typedef std::pair<C::const_iterator, C::const_iterator> Eq;
-        Eq eq = c.equal_range(1);
-        assert(std::distance(eq.first, eq.second) == 2);
-        std::multiset<std::string> s;
-        s.insert("one");
-        s.insert("four");
-        CheckConsecutiveKeys<C::const_iterator>(c.find(1), c.end(), 1, s);
-        eq = c.equal_range(2);
-        assert(std::distance(eq.first, eq.second) == 2);
-        s.insert("two");
-        s.insert("four");
-        CheckConsecutiveKeys<C::const_iterator>(c.find(2), c.end(), 2, s);
+    eq = c.equal_range(3);
+    assert(std::distance(eq.first, eq.second) == 1);
+    C::const_iterator i = eq.first;
+    assert(i->first == 3);
+    assert(i->second == "three");
+    eq = c.equal_range(4);
+    assert(std::distance(eq.first, eq.second) == 1);
+    i = eq.first;
+    assert(i->first == 4);
+    assert(i->second == "four");
+    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
+    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
+    assert(std::fabs(c.load_factor() - (float)c.size() / c.bucket_count()) < FLT_EPSILON);
+    assert(c.max_load_factor() == 1);
+    assert(c.hash_function() == test_hash<int>());
+    assert(c.key_eq() == test_equal_to<int>());
+    assert((c.get_allocator() == test_allocator<std::pair<const int, std::string> >()));
+  }
+  {
+    typedef std::unordered_multimap<int,
+                                    std::string,
+                                    test_hash<int>,
+                                    test_equal_to<int>,
+                                    min_allocator<std::pair<const int, std::string> > >
+        C;
+    typedef std::pair<int, std::string> P;
+    C c(
+        {
+            P(1, "one"),
+            P(2, "two"),
+            P(3, "three"),
+            P(4, "four"),
+            P(1, "four"),
+            P(2, "four"),
+        },
+        7);
+    LIBCPP_ASSERT(c.bucket_count() == 7);
+    assert(c.size() == 6);
+    typedef std::pair<C::const_iterator, C::const_iterator> Eq;
+    Eq eq = c.equal_range(1);
+    assert(std::distance(eq.first, eq.second) == 2);
+    std::multiset<std::string> s;
+    s.insert("one");
+    s.insert("four");
+    CheckConsecutiveKeys<C::const_iterator>(c.find(1), c.end(), 1, s);
+    eq = c.equal_range(2);
+    assert(std::distance(eq.first, eq.second) == 2);
+    s.insert("two");
+    s.insert("four");
+    CheckConsecutiveKeys<C::const_iterator>(c.find(2), c.end(), 2, s);
 
-        eq = c.equal_range(3);
-        assert(std::distance(eq.first, eq.second) == 1);
-        C::const_iterator i = eq.first;
-        assert(i->first == 3);
-        assert(i->second == "three");
-        eq = c.equal_range(4);
-        assert(std::distance(eq.first, eq.second) == 1);
-        i = eq.first;
-        assert(i->first == 4);
-        assert(i->second == "four");
-        assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-        assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
-        assert(std::fabs(c.load_factor() - (float)c.size()/c.bucket_count()) < FLT_EPSILON);
-        assert(c.max_load_factor() == 1);
-        assert(c.hash_function() == test_hash<int>());
-        assert(c.key_eq() == test_equal_to<int>());
-        assert((c.get_allocator() == min_allocator<std::pair<const int, std::string> >()));
-    }
+    eq = c.equal_range(3);
+    assert(std::distance(eq.first, eq.second) == 1);
+    C::const_iterator i = eq.first;
+    assert(i->first == 3);
+    assert(i->second == "three");
+    eq = c.equal_range(4);
+    assert(std::distance(eq.first, eq.second) == 1);
+    i = eq.first;
+    assert(i->first == 4);
+    assert(i->second == "four");
+    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
+    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
+    assert(std::fabs(c.load_factor() - (float)c.size() / c.bucket_count()) < FLT_EPSILON);
+    assert(c.max_load_factor() == 1);
+    assert(c.hash_function() == test_hash<int>());
+    assert(c.key_eq() == test_equal_to<int>());
+    assert((c.get_allocator() == min_allocator<std::pair<const int, std::string> >()));
+  }
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.cnstr/init_size_allocator.pass.cpp b/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.cnstr/init_size_allocator.pass.cpp
index 523977190c217..7ec2dd2efb895 100644
--- a/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.cnstr/init_size_allocator.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.cnstr/init_size_allocator.pass.cpp
@@ -34,67 +34,53 @@
 #include "min_allocator.h"
 
 template <class Allocator>
-void test(const Allocator& alloc)
-{
-    typedef std::unordered_multimap<int, std::string,
-                                    test_hash<int>,
-                                    test_equal_to<int>,
-                                    Allocator> C;
-    typedef std::pair<int, std::string> P;
-    C c({
-            P(1, "one"),
-            P(2, "two"),
-            P(3, "three"),
-            P(4, "four"),
-            P(1, "four"),
-            P(2, "four")
-        },
-        7,
-        alloc);
+void test(const Allocator& alloc) {
+  typedef std::unordered_multimap<int, std::string, test_hash<int>, test_equal_to<int>, Allocator> C;
+  typedef std::pair<int, std::string> P;
+  C c({P(1, "one"), P(2, "two"), P(3, "three"), P(4, "four"), P(1, "four"), P(2, "four")}, 7, alloc);
 
-    LIBCPP_ASSERT(c.bucket_count() == 7);
-    assert(c.size() == 6);
+  LIBCPP_ASSERT(c.bucket_count() == 7);
+  assert(c.size() == 6);
 
-    typedef std::pair<typename C::const_iterator, typename C::const_iterator> Eq;
-    Eq eq = c.equal_range(1);
-    assert(std::distance(eq.first, eq.second) == 2);
-    std::multiset<std::string> s;
-    s.insert("one");
-    s.insert("four");
-    CheckConsecutiveKeys<typename C::const_iterator>(c.find(1), c.end(), 1, s);
+  typedef std::pair<typename C::const_iterator, typename C::const_iterator> Eq;
+  Eq eq = c.equal_range(1);
+  assert(std::distance(eq.first, eq.second) == 2);
+  std::multiset<std::string> s;
+  s.insert("one");
+  s.insert("four");
+  CheckConsecutiveKeys<typename C::const_iterator>(c.find(1), c.end(), 1, s);
 
-    eq = c.equal_range(2);
-    assert(std::distance(eq.first, eq.second) == 2);
-    s.insert("two");
-    s.insert("four");
-    CheckConsecutiveKeys<typename C::const_iterator>(c.find(2), c.end(), 2, s);
+  eq = c.equal_range(2);
+  assert(std::distance(eq.first, eq.second) == 2);
+  s.insert("two");
+  s.insert("four");
+  CheckConsecutiveKeys<typename C::const_iterator>(c.find(2), c.end(), 2, s);
 
-    eq = c.equal_range(3);
-    assert(std::distance(eq.first, eq.second) == 1);
-    typename C::const_iterator i = eq.first;
-    assert(i->first == 3);
-    assert(i->second == "three");
+  eq = c.equal_range(3);
+  assert(std::distance(eq.first, eq.second) == 1);
+  typename C::const_iterator i = eq.first;
+  assert(i->first == 3);
+  assert(i->second == "three");
 
-    eq = c.equal_range(4);
-    assert(std::distance(eq.first, eq.second) == 1);
-    i = eq.first;
-    assert(i->first == 4);
-    assert(i->second == "four");
+  eq = c.equal_range(4);
+  assert(std::distance(eq.first, eq.second) == 1);
+  i = eq.first;
+  assert(i->first == 4);
+  assert(i->second == "four");
 
-    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-    assert(std::fabs(c.load_factor() - (float)c.size() / c.bucket_count()) < FLT_EPSILON);
-    assert(c.max_load_factor() == 1);
-    assert(c.hash_function() == test_hash<int>());
-    assert(c.key_eq() == test_equal_to<int>());
-    assert(c.get_allocator() == alloc);
+  assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
+  assert(std::fabs(c.load_factor() - (float)c.size() / c.bucket_count()) < FLT_EPSILON);
+  assert(c.max_load_factor() == 1);
+  assert(c.hash_function() == test_hash<int>());
+  assert(c.key_eq() == test_equal_to<int>());
+  assert(c.get_allocator() == alloc);
 }
 
-int main(int, char**)
-{
-    typedef std::pair<const int, std::string> V;
-    test(test_allocator<V>(10));
-    test(min_allocator<V>());
-    test(explicit_allocator<V>());
+int main(int, char**) {
+  typedef std::pair<const int, std::string> V;
+  test(test_allocator<V>(10));
+  test(min_allocator<V>());
+  test(explicit_allocator<V>());
 
-    return 0;
+  return 0;
 }
diff --git a/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.cnstr/init_size_hash.pass.cpp b/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.cnstr/init_size_hash.pass.cpp
index 785df175094d1..7550d3781f4c0 100644
--- a/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.cnstr/init_size_hash.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.cnstr/init_size_hash.pass.cpp
@@ -31,110 +31,111 @@
 #include "test_allocator.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef std::unordered_multimap<int, std::string,
-                                   test_hash<int>,
-                                   test_equal_to<int>,
-                                   test_allocator<std::pair<const int, std::string> >
-                                   > C;
-        typedef std::pair<int, std::string> P;
-        C c({
-                P(1, "one"),
-                P(2, "two"),
-                P(3, "three"),
-                P(4, "four"),
-                P(1, "four"),
-                P(2, "four"),
-            },
-            7,
-            test_hash<int>(8)
-           );
-        LIBCPP_ASSERT(c.bucket_count() == 7);
-        assert(c.size() == 6);
-        typedef std::pair<C::const_iterator, C::const_iterator> Eq;
-        Eq eq = c.equal_range(1);
-        assert(std::distance(eq.first, eq.second) == 2);
-        std::multiset<std::string> s;
-        s.insert("one");
-        s.insert("four");
-        CheckConsecutiveKeys<C::const_iterator>(c.find(1), c.end(), 1, s);
-        eq = c.equal_range(2);
-        assert(std::distance(eq.first, eq.second) == 2);
-        s.insert("two");
-        s.insert("four");
-        CheckConsecutiveKeys<C::const_iterator>(c.find(2), c.end(), 2, s);
+int main(int, char**) {
+  {
+    typedef std::unordered_multimap<int,
+                                    std::string,
+                                    test_hash<int>,
+                                    test_equal_to<int>,
+                                    test_allocator<std::pair<const int, std::string> > >
+        C;
+    typedef std::pair<int, std::string> P;
+    C c(
+        {
+            P(1, "one"),
+            P(2, "two"),
+            P(3, "three"),
+            P(4, "four"),
+            P(1, "four"),
+            P(2, "four"),
+        },
+        7,
+        test_hash<int>(8));
+    LIBCPP_ASSERT(c.bucket_count() == 7);
+    assert(c.size() == 6);
+    typedef std::pair<C::const_iterator, C::const_iterator> Eq;
+    Eq eq = c.equal_range(1);
+    assert(std::distance(eq.first, eq.second) == 2);
+    std::multiset<std::string> s;
+    s.insert("one");
+    s.insert("four");
+    CheckConsecutiveKeys<C::const_iterator>(c.find(1), c.end(), 1, s);
+    eq = c.equal_range(2);
+    assert(std::distance(eq.first, eq.second) == 2);
+    s.insert("two");
+    s.insert("four");
+    CheckConsecutiveKeys<C::const_iterator>(c.find(2), c.end(), 2, s);
 
-        eq = c.equal_range(3);
-        assert(std::distance(eq.first, eq.second) == 1);
-        C::const_iterator i = eq.first;
-        assert(i->first == 3);
-        assert(i->second == "three");
-        eq = c.equal_range(4);
-        assert(std::distance(eq.first, eq.second) == 1);
-        i = eq.first;
-        assert(i->first == 4);
-        assert(i->second == "four");
-        assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-        assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
-        assert(std::fabs(c.load_factor() - (float)c.size()/c.bucket_count()) < FLT_EPSILON);
-        assert(c.max_load_factor() == 1);
-        assert(c.hash_function() == test_hash<int>(8));
-        assert(c.key_eq() == test_equal_to<int>());
-        assert((c.get_allocator() == test_allocator<std::pair<const int, std::string> >()));
-    }
-    {
-        typedef std::unordered_multimap<int, std::string,
-                                   test_hash<int>,
-                                   test_equal_to<int>,
-                                   min_allocator<std::pair<const int, std::string> >
-                                   > C;
-        typedef std::pair<int, std::string> P;
-        C c({
-                P(1, "one"),
-                P(2, "two"),
-                P(3, "three"),
-                P(4, "four"),
-                P(1, "four"),
-                P(2, "four"),
-            },
-            7,
-            test_hash<int>(8)
-           );
-        LIBCPP_ASSERT(c.bucket_count() == 7);
-        assert(c.size() == 6);
-        typedef std::pair<C::const_iterator, C::const_iterator> Eq;
-        Eq eq = c.equal_range(1);
-        assert(std::distance(eq.first, eq.second) == 2);
-        std::multiset<std::string> s;
-        s.insert("one");
-        s.insert("four");
-        CheckConsecutiveKeys<C::const_iterator>(c.find(1), c.end(), 1, s);
-        eq = c.equal_range(2);
-        assert(std::distance(eq.first, eq.second) == 2);
-        s.insert("two");
-        s.insert("four");
-        CheckConsecutiveKeys<C::const_iterator>(c.find(2), c.end(), 2, s);
+    eq = c.equal_range(3);
+    assert(std::distance(eq.first, eq.second) == 1);
+    C::const_iterator i = eq.first;
+    assert(i->first == 3);
+    assert(i->second == "three");
+    eq = c.equal_range(4);
+    assert(std::distance(eq.first, eq.second) == 1);
+    i = eq.first;
+    assert(i->first == 4);
+    assert(i->second == "four");
+    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
+    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
+    assert(std::fabs(c.load_factor() - (float)c.size() / c.bucket_count()) < FLT_EPSILON);
+    assert(c.max_load_factor() == 1);
+    assert(c.hash_function() == test_hash<int>(8));
+    assert(c.key_eq() == test_equal_to<int>());
+    assert((c.get_allocator() == test_allocator<std::pair<const int, std::string> >()));
+  }
+  {
+    typedef std::unordered_multimap<int,
+                                    std::string,
+                                    test_hash<int>,
+                                    test_equal_to<int>,
+                                    min_allocator<std::pair<const int, std::string> > >
+        C;
+    typedef std::pair<int, std::string> P;
+    C c(
+        {
+            P(1, "one"),
+            P(2, "two"),
+            P(3, "three"),
+            P(4, "four"),
+            P(1, "four"),
+            P(2, "four"),
+        },
+        7,
+        test_hash<int>(8));
+    LIBCPP_ASSERT(c.bucket_count() == 7);
+    assert(c.size() == 6);
+    typedef std::pair<C::const_iterator, C::const_iterator> Eq;
+    Eq eq = c.equal_range(1);
+    assert(std::distance(eq.first, eq.second) == 2);
+    std::multiset<std::string> s;
+    s.insert("one");
+    s.insert("four");
+    CheckConsecutiveKeys<C::const_iterator>(c.find(1), c.end(), 1, s);
+    eq = c.equal_range(2);
+    assert(std::distance(eq.first, eq.second) == 2);
+    s.insert("two");
+    s.insert("four");
+    CheckConsecutiveKeys<C::const_iterator>(c.find(2), c.end(), 2, s);
 
-        eq = c.equal_range(3);
-        assert(std::distance(eq.first, eq.second) == 1);
-        C::const_iterator i = eq.first;
-        assert(i->first == 3);
-        assert(i->second == "three");
-        eq = c.equal_range(4);
-        assert(std::distance(eq.first, eq.second) == 1);
-        i = eq.first;
-        assert(i->first == 4);
-        assert(i->second == "four");
-        assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-        assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
-        assert(std::fabs(c.load_factor() - (float)c.size()/c.bucket_count()) < FLT_EPSILON);
-        assert(c.max_load_factor() == 1);
-        assert(c.hash_function() == test_hash<int>(8));
-        assert(c.key_eq() == test_equal_to<int>());
-        assert((c.get_allocator() == min_allocator<std::pair<const int, std::string> >()));
-    }
+    eq = c.equal_range(3);
+    assert(std::distance(eq.first, eq.second) == 1);
+    C::const_iterator i = eq.first;
+    assert(i->first == 3);
+    assert(i->second == "three");
+    eq = c.equal_range(4);
+    assert(std::distance(eq.first, eq.second) == 1);
+    i = eq.first;
+    assert(i->first == 4);
+    assert(i->second == "four");
+    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
+    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
+    assert(std::fabs(c.load_factor() - (float)c.size() / c.bucket_count()) < FLT_EPSILON);
+    assert(c.max_load_factor() == 1);
+    assert(c.hash_function() == test_hash<int>(8));
+    assert(c.key_eq() == test_equal_to<int>());
+    assert((c.get_allocator() == min_allocator<std::pair<const int, std::string> >()));
+  }
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.cnstr/init_size_hash_allocator.pass.cpp b/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.cnstr/init_size_hash_allocator.pass.cpp
index 6d0f4e5404afb..eab5d60fbf6f4 100644
--- a/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.cnstr/init_size_hash_allocator.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.cnstr/init_size_hash_allocator.pass.cpp
@@ -34,68 +34,53 @@
 #include "min_allocator.h"
 
 template <class Allocator>
-void test(const Allocator& alloc)
-{
-    typedef std::unordered_multimap<int, std::string,
-                                    test_hash<int>,
-                                    test_equal_to<int>,
-                                    Allocator> C;
-    typedef std::pair<int, std::string> P;
-    C c({
-            P(1, "one"),
-            P(2, "two"),
-            P(3, "three"),
-            P(4, "four"),
-            P(1, "four"),
-            P(2, "four")
-        },
-        7,
-        test_hash<int>(5),
-        alloc);
+void test(const Allocator& alloc) {
+  typedef std::unordered_multimap<int, std::string, test_hash<int>, test_equal_to<int>, Allocator> C;
+  typedef std::pair<int, std::string> P;
+  C c({P(1, "one"), P(2, "two"), P(3, "three"), P(4, "four"), P(1, "four"), P(2, "four")}, 7, test_hash<int>(5), alloc);
 
-    LIBCPP_ASSERT(c.bucket_count() == 7);
-    assert(c.size() == 6);
+  LIBCPP_ASSERT(c.bucket_count() == 7);
+  assert(c.size() == 6);
 
-    typedef std::pair<typename C::const_iterator, typename C::const_iterator> Eq;
-    Eq eq = c.equal_range(1);
-    assert(std::distance(eq.first, eq.second) == 2);
-    std::multiset<std::string> s;
-    s.insert("one");
-    s.insert("four");
-    CheckConsecutiveKeys<typename C::const_iterator>(c.find(1), c.end(), 1, s);
+  typedef std::pair<typename C::const_iterator, typename C::const_iterator> Eq;
+  Eq eq = c.equal_range(1);
+  assert(std::distance(eq.first, eq.second) == 2);
+  std::multiset<std::string> s;
+  s.insert("one");
+  s.insert("four");
+  CheckConsecutiveKeys<typename C::const_iterator>(c.find(1), c.end(), 1, s);
 
-    eq = c.equal_range(2);
-    assert(std::distance(eq.first, eq.second) == 2);
-    s.insert("two");
-    s.insert("four");
-    CheckConsecutiveKeys<typename C::const_iterator>(c.find(2), c.end(), 2, s);
+  eq = c.equal_range(2);
+  assert(std::distance(eq.first, eq.second) == 2);
+  s.insert("two");
+  s.insert("four");
+  CheckConsecutiveKeys<typename C::const_iterator>(c.find(2), c.end(), 2, s);
 
-    eq = c.equal_range(3);
-    assert(std::distance(eq.first, eq.second) == 1);
-    typename C::const_iterator i = eq.first;
-    assert(i->first == 3);
-    assert(i->second == "three");
+  eq = c.equal_range(3);
+  assert(std::distance(eq.first, eq.second) == 1);
+  typename C::const_iterator i = eq.first;
+  assert(i->first == 3);
+  assert(i->second == "three");
 
-    eq = c.equal_range(4);
-    assert(std::distance(eq.first, eq.second) == 1);
-    i = eq.first;
-    assert(i->first == 4);
-    assert(i->second == "four");
+  eq = c.equal_range(4);
+  assert(std::distance(eq.first, eq.second) == 1);
+  i = eq.first;
+  assert(i->first == 4);
+  assert(i->second == "four");
 
-    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-    assert(std::fabs(c.load_factor() - (float)c.size() / c.bucket_count()) < FLT_EPSILON);
-    assert(c.max_load_factor() == 1);
-    assert(c.hash_function() == test_hash<int>(5));
-    assert(c.key_eq() == test_equal_to<int>());
-    assert(c.get_allocator() == alloc);
+  assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
+  assert(std::fabs(c.load_factor() - (float)c.size() / c.bucket_count()) < FLT_EPSILON);
+  assert(c.max_load_factor() == 1);
+  assert(c.hash_function() == test_hash<int>(5));
+  assert(c.key_eq() == test_equal_to<int>());
+  assert(c.get_allocator() == alloc);
 }
 
-int main(int, char**)
-{
-    typedef std::pair<const int, std::string> V;
-    test(test_allocator<V>(10));
-    test(min_allocator<V>());
-    test(explicit_allocator<V>());
+int main(int, char**) {
+  typedef std::pair<const int, std::string> V;
+  test(test_allocator<V>(10));
+  test(min_allocator<V>());
+  test(explicit_allocator<V>());
 
-    return 0;
+  return 0;
 }
diff --git a/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.cnstr/init_size_hash_equal.pass.cpp b/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.cnstr/init_size_hash_equal.pass.cpp
index 42d3aa2ba659a..88408926b0b5e 100644
--- a/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.cnstr/init_size_hash_equal.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.cnstr/init_size_hash_equal.pass.cpp
@@ -32,112 +32,113 @@
 #include "test_allocator.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef std::unordered_multimap<int, std::string,
-                                   test_hash<int>,
-                                   test_equal_to<int>,
-                                   test_allocator<std::pair<const int, std::string> >
-                                   > C;
-        typedef std::pair<int, std::string> P;
-        C c({
-                P(1, "one"),
-                P(2, "two"),
-                P(3, "three"),
-                P(4, "four"),
-                P(1, "four"),
-                P(2, "four"),
-            },
-            7,
-            test_hash<int>(8),
-            test_equal_to<int>(9)
-           );
-        LIBCPP_ASSERT(c.bucket_count() == 7);
-        assert(c.size() == 6);
-        typedef std::pair<C::const_iterator, C::const_iterator> Eq;
-        Eq eq = c.equal_range(1);
-        assert(std::distance(eq.first, eq.second) == 2);
-        std::multiset<std::string> s;
-        s.insert("one");
-        s.insert("four");
-        CheckConsecutiveKeys<C::const_iterator>(c.find(1), c.end(), 1, s);
-        eq = c.equal_range(2);
-        assert(std::distance(eq.first, eq.second) == 2);
-        s.insert("two");
-        s.insert("four");
-        CheckConsecutiveKeys<C::const_iterator>(c.find(2), c.end(), 2, s);
+int main(int, char**) {
+  {
+    typedef std::unordered_multimap<int,
+                                    std::string,
+                                    test_hash<int>,
+                                    test_equal_to<int>,
+                                    test_allocator<std::pair<const int, std::string> > >
+        C;
+    typedef std::pair<int, std::string> P;
+    C c(
+        {
+            P(1, "one"),
+            P(2, "two"),
+            P(3, "three"),
+            P(4, "four"),
+            P(1, "four"),
+            P(2, "four"),
+        },
+        7,
+        test_hash<int>(8),
+        test_equal_to<int>(9));
+    LIBCPP_ASSERT(c.bucket_count() == 7);
+    assert(c.size() == 6);
+    typedef std::pair<C::const_iterator, C::const_iterator> Eq;
+    Eq eq = c.equal_range(1);
+    assert(std::distance(eq.first, eq.second) == 2);
+    std::multiset<std::string> s;
+    s.insert("one");
+    s.insert("four");
+    CheckConsecutiveKeys<C::const_iterator>(c.find(1), c.end(), 1, s);
+    eq = c.equal_range(2);
+    assert(std::distance(eq.first, eq.second) == 2);
+    s.insert("two");
+    s.insert("four");
+    CheckConsecutiveKeys<C::const_iterator>(c.find(2), c.end(), 2, s);
 
-        eq = c.equal_range(3);
-        assert(std::distance(eq.first, eq.second) == 1);
-        C::const_iterator i = eq.first;
-        assert(i->first == 3);
-        assert(i->second == "three");
-        eq = c.equal_range(4);
-        assert(std::distance(eq.first, eq.second) == 1);
-        i = eq.first;
-        assert(i->first == 4);
-        assert(i->second == "four");
-        assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-        assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
-        assert(std::fabs(c.load_factor() - (float)c.size()/c.bucket_count()) < FLT_EPSILON);
-        assert(c.max_load_factor() == 1);
-        assert(c.hash_function() == test_hash<int>(8));
-        assert(c.key_eq() == test_equal_to<int>(9));
-        assert((c.get_allocator() == test_allocator<std::pair<const int, std::string> >()));
-    }
-    {
-        typedef std::unordered_multimap<int, std::string,
-                                   test_hash<int>,
-                                   test_equal_to<int>,
-                                   min_allocator<std::pair<const int, std::string> >
-                                   > C;
-        typedef std::pair<int, std::string> P;
-        C c({
-                P(1, "one"),
-                P(2, "two"),
-                P(3, "three"),
-                P(4, "four"),
-                P(1, "four"),
-                P(2, "four"),
-            },
-            7,
-            test_hash<int>(8),
-            test_equal_to<int>(9)
-           );
-        LIBCPP_ASSERT(c.bucket_count() == 7);
-        assert(c.size() == 6);
-        typedef std::pair<C::const_iterator, C::const_iterator> Eq;
-        Eq eq = c.equal_range(1);
-        assert(std::distance(eq.first, eq.second) == 2);
-        std::multiset<std::string> s;
-        s.insert("one");
-        s.insert("four");
-        CheckConsecutiveKeys<C::const_iterator>(c.find(1), c.end(), 1, s);
-        eq = c.equal_range(2);
-        assert(std::distance(eq.first, eq.second) == 2);
-        s.insert("two");
-        s.insert("four");
-        CheckConsecutiveKeys<C::const_iterator>(c.find(2), c.end(), 2, s);
+    eq = c.equal_range(3);
+    assert(std::distance(eq.first, eq.second) == 1);
+    C::const_iterator i = eq.first;
+    assert(i->first == 3);
+    assert(i->second == "three");
+    eq = c.equal_range(4);
+    assert(std::distance(eq.first, eq.second) == 1);
+    i = eq.first;
+    assert(i->first == 4);
+    assert(i->second == "four");
+    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
+    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
+    assert(std::fabs(c.load_factor() - (float)c.size() / c.bucket_count()) < FLT_EPSILON);
+    assert(c.max_load_factor() == 1);
+    assert(c.hash_function() == test_hash<int>(8));
+    assert(c.key_eq() == test_equal_to<int>(9));
+    assert((c.get_allocator() == test_allocator<std::pair<const int, std::string> >()));
+  }
+  {
+    typedef std::unordered_multimap<int,
+                                    std::string,
+                                    test_hash<int>,
+                                    test_equal_to<int>,
+                                    min_allocator<std::pair<const int, std::string> > >
+        C;
+    typedef std::pair<int, std::string> P;
+    C c(
+        {
+            P(1, "one"),
+            P(2, "two"),
+            P(3, "three"),
+            P(4, "four"),
+            P(1, "four"),
+            P(2, "four"),
+        },
+        7,
+        test_hash<int>(8),
+        test_equal_to<int>(9));
+    LIBCPP_ASSERT(c.bucket_count() == 7);
+    assert(c.size() == 6);
+    typedef std::pair<C::const_iterator, C::const_iterator> Eq;
+    Eq eq = c.equal_range(1);
+    assert(std::distance(eq.first, eq.second) == 2);
+    std::multiset<std::string> s;
+    s.insert("one");
+    s.insert("four");
+    CheckConsecutiveKeys<C::const_iterator>(c.find(1), c.end(), 1, s);
+    eq = c.equal_range(2);
+    assert(std::distance(eq.first, eq.second) == 2);
+    s.insert("two");
+    s.insert("four");
+    CheckConsecutiveKeys<C::const_iterator>(c.find(2), c.end(), 2, s);
 
-        eq = c.equal_range(3);
-        assert(std::distance(eq.first, eq.second) == 1);
-        C::const_iterator i = eq.first;
-        assert(i->first == 3);
-        assert(i->second == "three");
-        eq = c.equal_range(4);
-        assert(std::distance(eq.first, eq.second) == 1);
-        i = eq.first;
-        assert(i->first == 4);
-        assert(i->second == "four");
-        assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-        assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
-        assert(std::fabs(c.load_factor() - (float)c.size()/c.bucket_count()) < FLT_EPSILON);
-        assert(c.max_load_factor() == 1);
-        assert(c.hash_function() == test_hash<int>(8));
-        assert(c.key_eq() == test_equal_to<int>(9));
-        assert((c.get_allocator() == min_allocator<std::pair<const int, std::string> >()));
-    }
+    eq = c.equal_range(3);
+    assert(std::distance(eq.first, eq.second) == 1);
+    C::const_iterator i = eq.first;
+    assert(i->first == 3);
+    assert(i->second == "three");
+    eq = c.equal_range(4);
+    assert(std::distance(eq.first, eq.second) == 1);
+    i = eq.first;
+    assert(i->first == 4);
+    assert(i->second == "four");
+    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
+    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
+    assert(std::fabs(c.load_factor() - (float)c.size() / c.bucket_count()) < FLT_EPSILON);
+    assert(c.max_load_factor() == 1);
+    assert(c.hash_function() == test_hash<int>(8));
+    assert(c.key_eq() == test_equal_to<int>(9));
+    assert((c.get_allocator() == min_allocator<std::pair<const int, std::string> >()));
+  }
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.cnstr/init_size_hash_equal_allocator.pass.cpp b/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.cnstr/init_size_hash_equal_allocator.pass.cpp
index beeb5b355ec51..493d6a7089154 100644
--- a/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.cnstr/init_size_hash_equal_allocator.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.cnstr/init_size_hash_equal_allocator.pass.cpp
@@ -32,168 +32,165 @@
 #include "test_allocator.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef std::unordered_multimap<int, std::string,
-                                   test_hash<int>,
-                                   test_equal_to<int>,
-                                   test_allocator<std::pair<const int, std::string> >
-                                   > C;
-        typedef std::pair<int, std::string> P;
-        C c({
-                P(1, "one"),
-                P(2, "two"),
-                P(3, "three"),
-                P(4, "four"),
-                P(1, "four"),
-                P(2, "four"),
-            },
-            7,
-            test_hash<int>(8),
-            test_equal_to<int>(9),
-            test_allocator<std::pair<const int, std::string> >(10)
-           );
-        LIBCPP_ASSERT(c.bucket_count() == 7);
-        assert(c.size() == 6);
-        typedef std::pair<C::const_iterator, C::const_iterator> Eq;
-        Eq eq = c.equal_range(1);
-        assert(std::distance(eq.first, eq.second) == 2);
-        std::multiset<std::string> s;
-        s.insert("one");
-        s.insert("four");
-        CheckConsecutiveKeys<C::const_iterator>(c.find(1), c.end(), 1, s);
-        eq = c.equal_range(2);
-        assert(std::distance(eq.first, eq.second) == 2);
-        s.insert("two");
-        s.insert("four");
-        CheckConsecutiveKeys<C::const_iterator>(c.find(2), c.end(), 2, s);
+int main(int, char**) {
+  {
+    typedef std::unordered_multimap<int,
+                                    std::string,
+                                    test_hash<int>,
+                                    test_equal_to<int>,
+                                    test_allocator<std::pair<const int, std::string> > >
+        C;
+    typedef std::pair<int, std::string> P;
+    C c(
+        {
+            P(1, "one"),
+            P(2, "two"),
+            P(3, "three"),
+            P(4, "four"),
+            P(1, "four"),
+            P(2, "four"),
+        },
+        7,
+        test_hash<int>(8),
+        test_equal_to<int>(9),
+        test_allocator<std::pair<const int, std::string> >(10));
+    LIBCPP_ASSERT(c.bucket_count() == 7);
+    assert(c.size() == 6);
+    typedef std::pair<C::const_iterator, C::const_iterator> Eq;
+    Eq eq = c.equal_range(1);
+    assert(std::distance(eq.first, eq.second) == 2);
+    std::multiset<std::string> s;
+    s.insert("one");
+    s.insert("four");
+    CheckConsecutiveKeys<C::const_iterator>(c.find(1), c.end(), 1, s);
+    eq = c.equal_range(2);
+    assert(std::distance(eq.first, eq.second) == 2);
+    s.insert("two");
+    s.insert("four");
+    CheckConsecutiveKeys<C::const_iterator>(c.find(2), c.end(), 2, s);
 
-        eq = c.equal_range(3);
-        assert(std::distance(eq.first, eq.second) == 1);
-        C::const_iterator i = eq.first;
-        assert(i->first == 3);
-        assert(i->second == "three");
-        eq = c.equal_range(4);
-        assert(std::distance(eq.first, eq.second) == 1);
-        i = eq.first;
-        assert(i->first == 4);
-        assert(i->second == "four");
-        assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-        assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
-        assert(std::fabs(c.load_factor() - (float)c.size()/c.bucket_count()) < FLT_EPSILON);
-        assert(c.max_load_factor() == 1);
-        assert(c.hash_function() == test_hash<int>(8));
-        assert(c.key_eq() == test_equal_to<int>(9));
-        assert((c.get_allocator() == test_allocator<std::pair<const int, std::string> >(10)));
-    }
-    {
-        typedef std::unordered_multimap<int, std::string,
-                                   test_hash<int>,
-                                   test_equal_to<int>,
-                                   min_allocator<std::pair<const int, std::string> >
-                                   > C;
-        typedef std::pair<int, std::string> P;
-        C c({
-                P(1, "one"),
-                P(2, "two"),
-                P(3, "three"),
-                P(4, "four"),
-                P(1, "four"),
-                P(2, "four"),
-            },
-            7,
-            test_hash<int>(8),
-            test_equal_to<int>(9),
-            min_allocator<std::pair<const int, std::string> >()
-           );
-        LIBCPP_ASSERT(c.bucket_count() == 7);
-        assert(c.size() == 6);
-        typedef std::pair<C::const_iterator, C::const_iterator> Eq;
-        Eq eq = c.equal_range(1);
-        assert(std::distance(eq.first, eq.second) == 2);
-        std::multiset<std::string> s;
-        s.insert("one");
-        s.insert("four");
-        CheckConsecutiveKeys<C::const_iterator>(c.find(1), c.end(), 1, s);
-        eq = c.equal_range(2);
-        assert(std::distance(eq.first, eq.second) == 2);
-        s.insert("two");
-        s.insert("four");
-        CheckConsecutiveKeys<C::const_iterator>(c.find(2), c.end(), 2, s);
+    eq = c.equal_range(3);
+    assert(std::distance(eq.first, eq.second) == 1);
+    C::const_iterator i = eq.first;
+    assert(i->first == 3);
+    assert(i->second == "three");
+    eq = c.equal_range(4);
+    assert(std::distance(eq.first, eq.second) == 1);
+    i = eq.first;
+    assert(i->first == 4);
+    assert(i->second == "four");
+    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
+    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
+    assert(std::fabs(c.load_factor() - (float)c.size() / c.bucket_count()) < FLT_EPSILON);
+    assert(c.max_load_factor() == 1);
+    assert(c.hash_function() == test_hash<int>(8));
+    assert(c.key_eq() == test_equal_to<int>(9));
+    assert((c.get_allocator() == test_allocator<std::pair<const int, std::string> >(10)));
+  }
+  {
+    typedef std::unordered_multimap<int,
+                                    std::string,
+                                    test_hash<int>,
+                                    test_equal_to<int>,
+                                    min_allocator<std::pair<const int, std::string> > >
+        C;
+    typedef std::pair<int, std::string> P;
+    C c(
+        {
+            P(1, "one"),
+            P(2, "two"),
+            P(3, "three"),
+            P(4, "four"),
+            P(1, "four"),
+            P(2, "four"),
+        },
+        7,
+        test_hash<int>(8),
+        test_equal_to<int>(9),
+        min_allocator<std::pair<const int, std::string> >());
+    LIBCPP_ASSERT(c.bucket_count() == 7);
+    assert(c.size() == 6);
+    typedef std::pair<C::const_iterator, C::const_iterator> Eq;
+    Eq eq = c.equal_range(1);
+    assert(std::distance(eq.first, eq.second) == 2);
+    std::multiset<std::string> s;
+    s.insert("one");
+    s.insert("four");
+    CheckConsecutiveKeys<C::const_iterator>(c.find(1), c.end(), 1, s);
+    eq = c.equal_range(2);
+    assert(std::distance(eq.first, eq.second) == 2);
+    s.insert("two");
+    s.insert("four");
+    CheckConsecutiveKeys<C::const_iterator>(c.find(2), c.end(), 2, s);
 
-        eq = c.equal_range(3);
-        assert(std::distance(eq.first, eq.second) == 1);
-        C::const_iterator i = eq.first;
-        assert(i->first == 3);
-        assert(i->second == "three");
-        eq = c.equal_range(4);
-        assert(std::distance(eq.first, eq.second) == 1);
-        i = eq.first;
-        assert(i->first == 4);
-        assert(i->second == "four");
-        assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-        assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
-        assert(std::fabs(c.load_factor() - (float)c.size()/c.bucket_count()) < FLT_EPSILON);
-        assert(c.max_load_factor() == 1);
-        assert(c.hash_function() == test_hash<int>(8));
-        assert(c.key_eq() == test_equal_to<int>(9));
-        assert((c.get_allocator() == min_allocator<std::pair<const int, std::string> >()));
-    }
-    {
-        typedef explicit_allocator<std::pair<const int, std::string>> A;
-        typedef std::unordered_multimap<int, std::string,
-                                   test_hash<int>,
-                                   test_equal_to<int>,
-                                   A
-                                   > C;
-        typedef std::pair<int, std::string> P;
-        C c({
-                P(1, "one"),
-                P(2, "two"),
-                P(3, "three"),
-                P(4, "four"),
-                P(1, "four"),
-                P(2, "four"),
-            },
-            7,
-            test_hash<int>(8),
-            test_equal_to<int>(9),
-            A{}
-           );
-        LIBCPP_ASSERT(c.bucket_count() == 7);
-        assert(c.size() == 6);
-        typedef std::pair<C::const_iterator, C::const_iterator> Eq;
-        Eq eq = c.equal_range(1);
-        assert(std::distance(eq.first, eq.second) == 2);
-        std::multiset<std::string> s;
-        s.insert("one");
-        s.insert("four");
-        CheckConsecutiveKeys<C::const_iterator>(c.find(1), c.end(), 1, s);
-        eq = c.equal_range(2);
-        assert(std::distance(eq.first, eq.second) == 2);
-        s.insert("two");
-        s.insert("four");
-        CheckConsecutiveKeys<C::const_iterator>(c.find(2), c.end(), 2, s);
+    eq = c.equal_range(3);
+    assert(std::distance(eq.first, eq.second) == 1);
+    C::const_iterator i = eq.first;
+    assert(i->first == 3);
+    assert(i->second == "three");
+    eq = c.equal_range(4);
+    assert(std::distance(eq.first, eq.second) == 1);
+    i = eq.first;
+    assert(i->first == 4);
+    assert(i->second == "four");
+    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
+    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
+    assert(std::fabs(c.load_factor() - (float)c.size() / c.bucket_count()) < FLT_EPSILON);
+    assert(c.max_load_factor() == 1);
+    assert(c.hash_function() == test_hash<int>(8));
+    assert(c.key_eq() == test_equal_to<int>(9));
+    assert((c.get_allocator() == min_allocator<std::pair<const int, std::string> >()));
+  }
+  {
+    typedef explicit_allocator<std::pair<const int, std::string>> A;
+    typedef std::unordered_multimap<int, std::string, test_hash<int>, test_equal_to<int>, A > C;
+    typedef std::pair<int, std::string> P;
+    C c(
+        {
+            P(1, "one"),
+            P(2, "two"),
+            P(3, "three"),
+            P(4, "four"),
+            P(1, "four"),
+            P(2, "four"),
+        },
+        7,
+        test_hash<int>(8),
+        test_equal_to<int>(9),
+        A{});
+    LIBCPP_ASSERT(c.bucket_count() == 7);
+    assert(c.size() == 6);
+    typedef std::pair<C::const_iterator, C::const_iterator> Eq;
+    Eq eq = c.equal_range(1);
+    assert(std::distance(eq.first, eq.second) == 2);
+    std::multiset<std::string> s;
+    s.insert("one");
+    s.insert("four");
+    CheckConsecutiveKeys<C::const_iterator>(c.find(1), c.end(), 1, s);
+    eq = c.equal_range(2);
+    assert(std::distance(eq.first, eq.second) == 2);
+    s.insert("two");
+    s.insert("four");
+    CheckConsecutiveKeys<C::const_iterator>(c.find(2), c.end(), 2, s);
 
-        eq = c.equal_range(3);
-        assert(std::distance(eq.first, eq.second) == 1);
-        C::const_iterator i = eq.first;
-        assert(i->first == 3);
-        assert(i->second == "three");
-        eq = c.equal_range(4);
-        assert(std::distance(eq.first, eq.second) == 1);
-        i = eq.first;
-        assert(i->first == 4);
-        assert(i->second == "four");
-        assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-        assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
-        assert(std::fabs(c.load_factor() - (float)c.size()/c.bucket_count()) < FLT_EPSILON);
-        assert(c.max_load_factor() == 1);
-        assert(c.hash_function() == test_hash<int>(8));
-        assert(c.key_eq() == test_equal_to<int>(9));
-        assert(c.get_allocator() == A{});
-    }
+    eq = c.equal_range(3);
+    assert(std::distance(eq.first, eq.second) == 1);
+    C::const_iterator i = eq.first;
+    assert(i->first == 3);
+    assert(i->second == "three");
+    eq = c.equal_range(4);
+    assert(std::distance(eq.first, eq.second) == 1);
+    i = eq.first;
+    assert(i->first == 4);
+    assert(i->second == "four");
+    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
+    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
+    assert(std::fabs(c.load_factor() - (float)c.size() / c.bucket_count()) < FLT_EPSILON);
+    assert(c.max_load_factor() == 1);
+    assert(c.hash_function() == test_hash<int>(8));
+    assert(c.key_eq() == test_equal_to<int>(9));
+    assert(c.get_allocator() == A{});
+  }
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.cnstr/iter_iter.pass.cpp b/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.cnstr/iter_iter.pass.cpp
index b3974e08a3d67..f370d762a459e 100644
--- a/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.cnstr/iter_iter.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.cnstr/iter_iter.pass.cpp
@@ -32,217 +32,214 @@
 #include "test_allocator.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef std::unordered_multimap<int, std::string,
-                                   test_hash<int>,
-                                   test_equal_to<int>,
-                                   test_allocator<std::pair<const int, std::string> >
-                                   > C;
-        typedef std::pair<int, std::string> P;
-        P a[] =
-        {
-            P(1, "one"),
-            P(2, "two"),
-            P(3, "three"),
-            P(4, "four"),
-            P(1, "four"),
-            P(2, "four"),
-        };
-        C c(cpp17_input_iterator<P*>(a), cpp17_input_iterator<P*>(a + sizeof(a)/sizeof(a[0])));
-        assert(c.bucket_count() >= 7);
-        assert(c.size() == 6);
-        typedef std::pair<C::const_iterator, C::const_iterator> Eq;
-        Eq eq = c.equal_range(1);
-        assert(std::distance(eq.first, eq.second) == 2);
-        std::multiset<std::string> s;
-        s.insert("one");
-        s.insert("four");
-        CheckConsecutiveKeys<C::const_iterator>(c.find(1), c.end(), 1, s);
-        eq = c.equal_range(2);
-        assert(std::distance(eq.first, eq.second) == 2);
-        s.insert("two");
-        s.insert("four");
-        CheckConsecutiveKeys<C::const_iterator>(c.find(2), c.end(), 2, s);
+int main(int, char**) {
+  {
+    typedef std::unordered_multimap<int,
+                                    std::string,
+                                    test_hash<int>,
+                                    test_equal_to<int>,
+                                    test_allocator<std::pair<const int, std::string> > >
+        C;
+    typedef std::pair<int, std::string> P;
+    P a[] = {
+        P(1, "one"),
+        P(2, "two"),
+        P(3, "three"),
+        P(4, "four"),
+        P(1, "four"),
+        P(2, "four"),
+    };
+    C c(cpp17_input_iterator<P*>(a), cpp17_input_iterator<P*>(a + sizeof(a) / sizeof(a[0])));
+    assert(c.bucket_count() >= 7);
+    assert(c.size() == 6);
+    typedef std::pair<C::const_iterator, C::const_iterator> Eq;
+    Eq eq = c.equal_range(1);
+    assert(std::distance(eq.first, eq.second) == 2);
+    std::multiset<std::string> s;
+    s.insert("one");
+    s.insert("four");
+    CheckConsecutiveKeys<C::const_iterator>(c.find(1), c.end(), 1, s);
+    eq = c.equal_range(2);
+    assert(std::distance(eq.first, eq.second) == 2);
+    s.insert("two");
+    s.insert("four");
+    CheckConsecutiveKeys<C::const_iterator>(c.find(2), c.end(), 2, s);
 
-        eq = c.equal_range(3);
-        assert(std::distance(eq.first, eq.second) == 1);
-        C::const_iterator i = eq.first;
-        assert(i->first == 3);
-        assert(i->second == "three");
-        eq = c.equal_range(4);
-        assert(std::distance(eq.first, eq.second) == 1);
-        i = eq.first;
-        assert(i->first == 4);
-        assert(i->second == "four");
-        assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-        assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
-        assert(fabs(c.load_factor() - (float)c.size()/c.bucket_count()) < FLT_EPSILON);
-        assert(c.max_load_factor() == 1);
-        assert(c.hash_function() == test_hash<int>());
-        assert(c.key_eq() == test_equal_to<int>());
-        assert((c.get_allocator() == test_allocator<std::pair<const int, std::string> >()));
-    }
+    eq = c.equal_range(3);
+    assert(std::distance(eq.first, eq.second) == 1);
+    C::const_iterator i = eq.first;
+    assert(i->first == 3);
+    assert(i->second == "three");
+    eq = c.equal_range(4);
+    assert(std::distance(eq.first, eq.second) == 1);
+    i = eq.first;
+    assert(i->first == 4);
+    assert(i->second == "four");
+    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
+    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
+    assert(fabs(c.load_factor() - (float)c.size() / c.bucket_count()) < FLT_EPSILON);
+    assert(c.max_load_factor() == 1);
+    assert(c.hash_function() == test_hash<int>());
+    assert(c.key_eq() == test_equal_to<int>());
+    assert((c.get_allocator() == test_allocator<std::pair<const int, std::string> >()));
+  }
 #if TEST_STD_VER >= 11
-    {
-        typedef std::unordered_multimap<int, std::string,
-                                   test_hash<int>,
-                                   test_equal_to<int>,
-                                   min_allocator<std::pair<const int, std::string> >
-                                   > C;
-        typedef std::pair<int, std::string> P;
-        P a[] =
-        {
-            P(1, "one"),
-            P(2, "two"),
-            P(3, "three"),
-            P(4, "four"),
-            P(1, "four"),
-            P(2, "four"),
-        };
-        C c(cpp17_input_iterator<P*>(a), cpp17_input_iterator<P*>(a + sizeof(a)/sizeof(a[0])));
-        assert(c.bucket_count() >= 7);
-        assert(c.size() == 6);
-        typedef std::pair<C::const_iterator, C::const_iterator> Eq;
-        Eq eq = c.equal_range(1);
-        assert(std::distance(eq.first, eq.second) == 2);
-        std::multiset<std::string> s;
-        s.insert("one");
-        s.insert("four");
-        CheckConsecutiveKeys<C::const_iterator>(c.find(1), c.end(), 1, s);
-        eq = c.equal_range(2);
-        assert(std::distance(eq.first, eq.second) == 2);
-        s.insert("two");
-        s.insert("four");
-        CheckConsecutiveKeys<C::const_iterator>(c.find(2), c.end(), 2, s);
+  {
+    typedef std::unordered_multimap<int,
+                                    std::string,
+                                    test_hash<int>,
+                                    test_equal_to<int>,
+                                    min_allocator<std::pair<const int, std::string> > >
+        C;
+    typedef std::pair<int, std::string> P;
+    P a[] = {
+        P(1, "one"),
+        P(2, "two"),
+        P(3, "three"),
+        P(4, "four"),
+        P(1, "four"),
+        P(2, "four"),
+    };
+    C c(cpp17_input_iterator<P*>(a), cpp17_input_iterator<P*>(a + sizeof(a) / sizeof(a[0])));
+    assert(c.bucket_count() >= 7);
+    assert(c.size() == 6);
+    typedef std::pair<C::const_iterator, C::const_iterator> Eq;
+    Eq eq = c.equal_range(1);
+    assert(std::distance(eq.first, eq.second) == 2);
+    std::multiset<std::string> s;
+    s.insert("one");
+    s.insert("four");
+    CheckConsecutiveKeys<C::const_iterator>(c.find(1), c.end(), 1, s);
+    eq = c.equal_range(2);
+    assert(std::distance(eq.first, eq.second) == 2);
+    s.insert("two");
+    s.insert("four");
+    CheckConsecutiveKeys<C::const_iterator>(c.find(2), c.end(), 2, s);
 
-        eq = c.equal_range(3);
-        assert(std::distance(eq.first, eq.second) == 1);
-        C::const_iterator i = eq.first;
-        assert(i->first == 3);
-        assert(i->second == "three");
-        eq = c.equal_range(4);
-        assert(std::distance(eq.first, eq.second) == 1);
-        i = eq.first;
-        assert(i->first == 4);
-        assert(i->second == "four");
-        assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-        assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
-        assert(fabs(c.load_factor() - (float)c.size()/c.bucket_count()) < FLT_EPSILON);
-        assert(c.max_load_factor() == 1);
-        assert(c.hash_function() == test_hash<int>());
-        assert(c.key_eq() == test_equal_to<int>());
-        assert((c.get_allocator() == min_allocator<std::pair<const int, std::string> >()));
-    }
-#if TEST_STD_VER > 11
-    {
-        typedef std::pair<int, std::string> P;
-        typedef test_allocator<std::pair<const int, std::string>> A;
-        typedef test_hash<int> HF;
-        typedef test_equal_to<int> Comp;
-        typedef std::unordered_multimap<int, std::string, HF, Comp, A> C;
+    eq = c.equal_range(3);
+    assert(std::distance(eq.first, eq.second) == 1);
+    C::const_iterator i = eq.first;
+    assert(i->first == 3);
+    assert(i->second == "three");
+    eq = c.equal_range(4);
+    assert(std::distance(eq.first, eq.second) == 1);
+    i = eq.first;
+    assert(i->first == 4);
+    assert(i->second == "four");
+    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
+    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
+    assert(fabs(c.load_factor() - (float)c.size() / c.bucket_count()) < FLT_EPSILON);
+    assert(c.max_load_factor() == 1);
+    assert(c.hash_function() == test_hash<int>());
+    assert(c.key_eq() == test_equal_to<int>());
+    assert((c.get_allocator() == min_allocator<std::pair<const int, std::string> >()));
+  }
+#  if TEST_STD_VER > 11
+  {
+    typedef std::pair<int, std::string> P;
+    typedef test_allocator<std::pair<const int, std::string>> A;
+    typedef test_hash<int> HF;
+    typedef test_equal_to<int> Comp;
+    typedef std::unordered_multimap<int, std::string, HF, Comp, A> C;
 
-        P arr[] =
-        {
-            P(1, "one"),
-            P(2, "two"),
-            P(3, "three"),
-            P(4, "four"),
-            P(1, "four"),
-            P(2, "four"),
-        };
-         A a(42);
-       C c(cpp17_input_iterator<P*>(arr), cpp17_input_iterator<P*>(arr + sizeof(arr)/sizeof(arr[0])), 14, a);
-        assert(c.bucket_count() >= 14);
-        assert(c.size() == 6);
-        typedef std::pair<C::const_iterator, C::const_iterator> Eq;
-        Eq eq = c.equal_range(1);
-        assert(std::distance(eq.first, eq.second) == 2);
-        std::multiset<std::string> s;
-        s.insert("one");
-        s.insert("four");
-        CheckConsecutiveKeys<C::const_iterator>(c.find(1), c.end(), 1, s);
-        eq = c.equal_range(2);
-        assert(std::distance(eq.first, eq.second) == 2);
-        s.insert("two");
-        s.insert("four");
-        CheckConsecutiveKeys<C::const_iterator>(c.find(2), c.end(), 2, s);
+    P arr[] = {
+        P(1, "one"),
+        P(2, "two"),
+        P(3, "three"),
+        P(4, "four"),
+        P(1, "four"),
+        P(2, "four"),
+    };
+    A a(42);
+    C c(cpp17_input_iterator<P*>(arr), cpp17_input_iterator<P*>(arr + sizeof(arr) / sizeof(arr[0])), 14, a);
+    assert(c.bucket_count() >= 14);
+    assert(c.size() == 6);
+    typedef std::pair<C::const_iterator, C::const_iterator> Eq;
+    Eq eq = c.equal_range(1);
+    assert(std::distance(eq.first, eq.second) == 2);
+    std::multiset<std::string> s;
+    s.insert("one");
+    s.insert("four");
+    CheckConsecutiveKeys<C::const_iterator>(c.find(1), c.end(), 1, s);
+    eq = c.equal_range(2);
+    assert(std::distance(eq.first, eq.second) == 2);
+    s.insert("two");
+    s.insert("four");
+    CheckConsecutiveKeys<C::const_iterator>(c.find(2), c.end(), 2, s);
 
-        eq = c.equal_range(3);
-        assert(std::distance(eq.first, eq.second) == 1);
-        C::const_iterator i = eq.first;
-        assert(i->first == 3);
-        assert(i->second == "three");
-        eq = c.equal_range(4);
-        assert(std::distance(eq.first, eq.second) == 1);
-        i = eq.first;
-        assert(i->first == 4);
-        assert(i->second == "four");
-        assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-        assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
-        assert(fabs(c.load_factor() - (float)c.size()/c.bucket_count()) < FLT_EPSILON);
-        assert(c.max_load_factor() == 1);
-        assert(c.hash_function() == HF());
-        assert(c.key_eq() == Comp());
-        assert(c.get_allocator() == a);
-        assert(!(c.get_allocator() == A()));
-    }
-    {
-        typedef std::pair<int, std::string> P;
-        typedef test_allocator<std::pair<const int, std::string>> A;
-        typedef test_hash<int> HF;
-        typedef test_equal_to<int> Comp;
-        typedef std::unordered_multimap<int, std::string, HF, Comp, A> C;
+    eq = c.equal_range(3);
+    assert(std::distance(eq.first, eq.second) == 1);
+    C::const_iterator i = eq.first;
+    assert(i->first == 3);
+    assert(i->second == "three");
+    eq = c.equal_range(4);
+    assert(std::distance(eq.first, eq.second) == 1);
+    i = eq.first;
+    assert(i->first == 4);
+    assert(i->second == "four");
+    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
+    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
+    assert(fabs(c.load_factor() - (float)c.size() / c.bucket_count()) < FLT_EPSILON);
+    assert(c.max_load_factor() == 1);
+    assert(c.hash_function() == HF());
+    assert(c.key_eq() == Comp());
+    assert(c.get_allocator() == a);
+    assert(!(c.get_allocator() == A()));
+  }
+  {
+    typedef std::pair<int, std::string> P;
+    typedef test_allocator<std::pair<const int, std::string>> A;
+    typedef test_hash<int> HF;
+    typedef test_equal_to<int> Comp;
+    typedef std::unordered_multimap<int, std::string, HF, Comp, A> C;
 
-        P arr[] =
-        {
-            P(1, "one"),
-            P(2, "two"),
-            P(3, "three"),
-            P(4, "four"),
-            P(1, "four"),
-            P(2, "four"),
-        };
-        A a(42);
-        HF hf (43);
-        C c(cpp17_input_iterator<P*>(arr), cpp17_input_iterator<P*>(arr + sizeof(arr)/sizeof(arr[0])), 12, hf, a );
-        assert(c.bucket_count() >= 12);
-        assert(c.size() == 6);
-        typedef std::pair<C::const_iterator, C::const_iterator> Eq;
-        Eq eq = c.equal_range(1);
-        assert(std::distance(eq.first, eq.second) == 2);
-        std::multiset<std::string> s;
-        s.insert("one");
-        s.insert("four");
-        CheckConsecutiveKeys<C::const_iterator>(c.find(1), c.end(), 1, s);
-        eq = c.equal_range(2);
-        assert(std::distance(eq.first, eq.second) == 2);
-        s.insert("two");
-        s.insert("four");
-        CheckConsecutiveKeys<C::const_iterator>(c.find(2), c.end(), 2, s);
+    P arr[] = {
+        P(1, "one"),
+        P(2, "two"),
+        P(3, "three"),
+        P(4, "four"),
+        P(1, "four"),
+        P(2, "four"),
+    };
+    A a(42);
+    HF hf(43);
+    C c(cpp17_input_iterator<P*>(arr), cpp17_input_iterator<P*>(arr + sizeof(arr) / sizeof(arr[0])), 12, hf, a);
+    assert(c.bucket_count() >= 12);
+    assert(c.size() == 6);
+    typedef std::pair<C::const_iterator, C::const_iterator> Eq;
+    Eq eq = c.equal_range(1);
+    assert(std::distance(eq.first, eq.second) == 2);
+    std::multiset<std::string> s;
+    s.insert("one");
+    s.insert("four");
+    CheckConsecutiveKeys<C::const_iterator>(c.find(1), c.end(), 1, s);
+    eq = c.equal_range(2);
+    assert(std::distance(eq.first, eq.second) == 2);
+    s.insert("two");
+    s.insert("four");
+    CheckConsecutiveKeys<C::const_iterator>(c.find(2), c.end(), 2, s);
 
-        eq = c.equal_range(3);
-        assert(std::distance(eq.first, eq.second) == 1);
-        C::const_iterator i = eq.first;
-        assert(i->first == 3);
-        assert(i->second == "three");
-        eq = c.equal_range(4);
-        assert(std::distance(eq.first, eq.second) == 1);
-        i = eq.first;
-        assert(i->first == 4);
-        assert(i->second == "four");
-        assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-        assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
-        assert(fabs(c.load_factor() - (float)c.size()/c.bucket_count()) < FLT_EPSILON);
-        assert(c.max_load_factor() == 1);
-        assert(c.hash_function() == hf);
-        assert(!(c.hash_function() == HF()));
-        assert(c.key_eq() == Comp());
-        assert(c.get_allocator() == a);
-        assert(!(c.get_allocator() == A()));
-    }
-#endif
+    eq = c.equal_range(3);
+    assert(std::distance(eq.first, eq.second) == 1);
+    C::const_iterator i = eq.first;
+    assert(i->first == 3);
+    assert(i->second == "three");
+    eq = c.equal_range(4);
+    assert(std::distance(eq.first, eq.second) == 1);
+    i = eq.first;
+    assert(i->first == 4);
+    assert(i->second == "four");
+    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
+    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
+    assert(fabs(c.load_factor() - (float)c.size() / c.bucket_count()) < FLT_EPSILON);
+    assert(c.max_load_factor() == 1);
+    assert(c.hash_function() == hf);
+    assert(!(c.hash_function() == HF()));
+    assert(c.key_eq() == Comp());
+    assert(c.get_allocator() == a);
+    assert(!(c.get_allocator() == A()));
+  }
+#  endif
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.cnstr/iter_iter_size.pass.cpp b/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.cnstr/iter_iter_size.pass.cpp
index 374e95eaf8423..9f42bff9f4175 100644
--- a/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.cnstr/iter_iter_size.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.cnstr/iter_iter_size.pass.cpp
@@ -32,113 +32,108 @@
 #include "test_allocator.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef std::unordered_multimap<int, std::string,
-                                   test_hash<int>,
-                                   test_equal_to<int>,
-                                   test_allocator<std::pair<const int, std::string> >
-                                   > C;
-        typedef std::pair<int, std::string> P;
-        P a[] =
-        {
-            P(1, "one"),
-            P(2, "two"),
-            P(3, "three"),
-            P(4, "four"),
-            P(1, "four"),
-            P(2, "four"),
-        };
-        C c(cpp17_input_iterator<P*>(a), cpp17_input_iterator<P*>(a + sizeof(a)/sizeof(a[0])),
-            10
-           );
-        LIBCPP_ASSERT(c.bucket_count() == 11);
-        assert(c.size() == 6);
-        typedef std::pair<C::const_iterator, C::const_iterator> Eq;
-        Eq eq = c.equal_range(1);
-        assert(std::distance(eq.first, eq.second) == 2);
-        std::multiset<std::string> s;
-        s.insert("one");
-        s.insert("four");
-        CheckConsecutiveKeys<C::const_iterator>(c.find(1), c.end(), 1, s);
-        eq = c.equal_range(2);
-        assert(std::distance(eq.first, eq.second) == 2);
-        s.insert("two");
-        s.insert("four");
-        CheckConsecutiveKeys<C::const_iterator>(c.find(2), c.end(), 2, s);
+int main(int, char**) {
+  {
+    typedef std::unordered_multimap<int,
+                                    std::string,
+                                    test_hash<int>,
+                                    test_equal_to<int>,
+                                    test_allocator<std::pair<const int, std::string> > >
+        C;
+    typedef std::pair<int, std::string> P;
+    P a[] = {
+        P(1, "one"),
+        P(2, "two"),
+        P(3, "three"),
+        P(4, "four"),
+        P(1, "four"),
+        P(2, "four"),
+    };
+    C c(cpp17_input_iterator<P*>(a), cpp17_input_iterator<P*>(a + sizeof(a) / sizeof(a[0])), 10);
+    LIBCPP_ASSERT(c.bucket_count() == 11);
+    assert(c.size() == 6);
+    typedef std::pair<C::const_iterator, C::const_iterator> Eq;
+    Eq eq = c.equal_range(1);
+    assert(std::distance(eq.first, eq.second) == 2);
+    std::multiset<std::string> s;
+    s.insert("one");
+    s.insert("four");
+    CheckConsecutiveKeys<C::const_iterator>(c.find(1), c.end(), 1, s);
+    eq = c.equal_range(2);
+    assert(std::distance(eq.first, eq.second) == 2);
+    s.insert("two");
+    s.insert("four");
+    CheckConsecutiveKeys<C::const_iterator>(c.find(2), c.end(), 2, s);
 
-        eq = c.equal_range(3);
-        assert(std::distance(eq.first, eq.second) == 1);
-        C::const_iterator i = eq.first;
-        assert(i->first == 3);
-        assert(i->second == "three");
-        eq = c.equal_range(4);
-        assert(std::distance(eq.first, eq.second) == 1);
-        i = eq.first;
-        assert(i->first == 4);
-        assert(i->second == "four");
-        assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-        assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
-        assert(std::fabs(c.load_factor() - (float)c.size()/c.bucket_count()) < FLT_EPSILON);
-        assert(c.max_load_factor() == 1);
-        assert(c.hash_function() == test_hash<int>());
-        assert(c.key_eq() == test_equal_to<int>());
-        assert((c.get_allocator() == test_allocator<std::pair<const int, std::string> >()));
-    }
+    eq = c.equal_range(3);
+    assert(std::distance(eq.first, eq.second) == 1);
+    C::const_iterator i = eq.first;
+    assert(i->first == 3);
+    assert(i->second == "three");
+    eq = c.equal_range(4);
+    assert(std::distance(eq.first, eq.second) == 1);
+    i = eq.first;
+    assert(i->first == 4);
+    assert(i->second == "four");
+    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
+    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
+    assert(std::fabs(c.load_factor() - (float)c.size() / c.bucket_count()) < FLT_EPSILON);
+    assert(c.max_load_factor() == 1);
+    assert(c.hash_function() == test_hash<int>());
+    assert(c.key_eq() == test_equal_to<int>());
+    assert((c.get_allocator() == test_allocator<std::pair<const int, std::string> >()));
+  }
 #if TEST_STD_VER >= 11
-    {
-        typedef std::unordered_multimap<int, std::string,
-                                   test_hash<int>,
-                                   test_equal_to<int>,
-                                   min_allocator<std::pair<const int, std::string> >
-                                   > C;
-        typedef std::pair<int, std::string> P;
-        P a[] =
-        {
-            P(1, "one"),
-            P(2, "two"),
-            P(3, "three"),
-            P(4, "four"),
-            P(1, "four"),
-            P(2, "four"),
-        };
-        C c(cpp17_input_iterator<P*>(a), cpp17_input_iterator<P*>(a + sizeof(a)/sizeof(a[0])),
-            10
-           );
-        LIBCPP_ASSERT(c.bucket_count() == 11);
-        assert(c.size() == 6);
-        typedef std::pair<C::const_iterator, C::const_iterator> Eq;
-        Eq eq = c.equal_range(1);
-        assert(std::distance(eq.first, eq.second) == 2);
-        std::multiset<std::string> s;
-        s.insert("one");
-        s.insert("four");
-        CheckConsecutiveKeys<C::const_iterator>(c.find(1), c.end(), 1, s);
-        eq = c.equal_range(2);
-        assert(std::distance(eq.first, eq.second) == 2);
-        s.insert("two");
-        s.insert("four");
-        CheckConsecutiveKeys<C::const_iterator>(c.find(2), c.end(), 2, s);
+  {
+    typedef std::unordered_multimap<int,
+                                    std::string,
+                                    test_hash<int>,
+                                    test_equal_to<int>,
+                                    min_allocator<std::pair<const int, std::string> > >
+        C;
+    typedef std::pair<int, std::string> P;
+    P a[] = {
+        P(1, "one"),
+        P(2, "two"),
+        P(3, "three"),
+        P(4, "four"),
+        P(1, "four"),
+        P(2, "four"),
+    };
+    C c(cpp17_input_iterator<P*>(a), cpp17_input_iterator<P*>(a + sizeof(a) / sizeof(a[0])), 10);
+    LIBCPP_ASSERT(c.bucket_count() == 11);
+    assert(c.size() == 6);
+    typedef std::pair<C::const_iterator, C::const_iterator> Eq;
+    Eq eq = c.equal_range(1);
+    assert(std::distance(eq.first, eq.second) == 2);
+    std::multiset<std::string> s;
+    s.insert("one");
+    s.insert("four");
+    CheckConsecutiveKeys<C::const_iterator>(c.find(1), c.end(), 1, s);
+    eq = c.equal_range(2);
+    assert(std::distance(eq.first, eq.second) == 2);
+    s.insert("two");
+    s.insert("four");
+    CheckConsecutiveKeys<C::const_iterator>(c.find(2), c.end(), 2, s);
 
-        eq = c.equal_range(3);
-        assert(std::distance(eq.first, eq.second) == 1);
-        C::const_iterator i = eq.first;
-        assert(i->first == 3);
-        assert(i->second == "three");
-        eq = c.equal_range(4);
-        assert(std::distance(eq.first, eq.second) == 1);
-        i = eq.first;
-        assert(i->first == 4);
-        assert(i->second == "four");
-        assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-        assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
-        assert(std::fabs(c.load_factor() - (float)c.size()/c.bucket_count()) < FLT_EPSILON);
-        assert(c.max_load_factor() == 1);
-        assert(c.hash_function() == test_hash<int>());
-        assert(c.key_eq() == test_equal_to<int>());
-        assert((c.get_allocator() == min_allocator<std::pair<const int, std::string> >()));
-    }
+    eq = c.equal_range(3);
+    assert(std::distance(eq.first, eq.second) == 1);
+    C::const_iterator i = eq.first;
+    assert(i->first == 3);
+    assert(i->second == "three");
+    eq = c.equal_range(4);
+    assert(std::distance(eq.first, eq.second) == 1);
+    i = eq.first;
+    assert(i->first == 4);
+    assert(i->second == "four");
+    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
+    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
+    assert(std::fabs(c.load_factor() - (float)c.size() / c.bucket_count()) < FLT_EPSILON);
+    assert(c.max_load_factor() == 1);
+    assert(c.hash_function() == test_hash<int>());
+    assert(c.key_eq() == test_equal_to<int>());
+    assert((c.get_allocator() == min_allocator<std::pair<const int, std::string> >()));
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.cnstr/iter_iter_size_allocator.pass.cpp b/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.cnstr/iter_iter_size_allocator.pass.cpp
index 5c1f4b6e39f58..ad6ad9d1ad1ef 100644
--- a/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.cnstr/iter_iter_size_allocator.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.cnstr/iter_iter_size_allocator.pass.cpp
@@ -36,70 +36,55 @@
 #include "min_allocator.h"
 
 template <class Allocator>
-void test(const Allocator& alloc)
-{
-    typedef std::unordered_multimap<int, std::string,
-                                    test_hash<int>,
-                                    test_equal_to<int>,
-                                    Allocator> C;
-    typedef std::pair<int, std::string> P;
-    P a[] =
-    {
-        P(1, "one"),
-        P(2, "two"),
-        P(3, "three"),
-        P(4, "four"),
-        P(1, "four"),
-        P(2, "four")
-    };
-
-    C c(cpp17_input_iterator<P*>(a), cpp17_input_iterator<P*>(a + sizeof(a) / sizeof(a[0])),
-        7,
-        alloc);
-
-    LIBCPP_ASSERT(c.bucket_count() == 7);
-    assert(c.size() == 6);
-
-    typedef std::pair<typename C::const_iterator, typename C::const_iterator> Eq;
-    Eq eq = c.equal_range(1);
-    assert(std::distance(eq.first, eq.second) == 2);
-    std::multiset<std::string> s;
-    s.insert("one");
-    s.insert("four");
-    CheckConsecutiveKeys<typename C::const_iterator>(c.find(1), c.end(), 1, s);
-
-    eq = c.equal_range(2);
-    assert(std::distance(eq.first, eq.second) == 2);
-    s.insert("two");
-    s.insert("four");
-    CheckConsecutiveKeys<typename C::const_iterator>(c.find(2), c.end(), 2, s);
-
-    eq = c.equal_range(3);
-    assert(std::distance(eq.first, eq.second) == 1);
-    typename C::const_iterator i = eq.first;
-    assert(i->first == 3);
-    assert(i->second == "three");
-
-    eq = c.equal_range(4);
-    assert(std::distance(eq.first, eq.second) == 1);
-    i = eq.first;
-    assert(i->first == 4);
-    assert(i->second == "four");
-
-    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-    assert(std::fabs(c.load_factor() - (float)c.size() / c.bucket_count()) < FLT_EPSILON);
-    assert(c.max_load_factor() == 1);
-    assert(c.hash_function() == test_hash<int>());
-    assert(c.key_eq() == test_equal_to<int>());
-    assert(c.get_allocator() == alloc);
+void test(const Allocator& alloc) {
+  typedef std::unordered_multimap<int, std::string, test_hash<int>, test_equal_to<int>, Allocator> C;
+  typedef std::pair<int, std::string> P;
+  P a[] = {P(1, "one"), P(2, "two"), P(3, "three"), P(4, "four"), P(1, "four"), P(2, "four")};
+
+  C c(cpp17_input_iterator<P*>(a), cpp17_input_iterator<P*>(a + sizeof(a) / sizeof(a[0])), 7, alloc);
+
+  LIBCPP_ASSERT(c.bucket_count() == 7);
+  assert(c.size() == 6);
+
+  typedef std::pair<typename C::const_iterator, typename C::const_iterator> Eq;
+  Eq eq = c.equal_range(1);
+  assert(std::distance(eq.first, eq.second) == 2);
+  std::multiset<std::string> s;
+  s.insert("one");
+  s.insert("four");
+  CheckConsecutiveKeys<typename C::const_iterator>(c.find(1), c.end(), 1, s);
+
+  eq = c.equal_range(2);
+  assert(std::distance(eq.first, eq.second) == 2);
+  s.insert("two");
+  s.insert("four");
+  CheckConsecutiveKeys<typename C::const_iterator>(c.find(2), c.end(), 2, s);
+
+  eq = c.equal_range(3);
+  assert(std::distance(eq.first, eq.second) == 1);
+  typename C::const_iterator i = eq.first;
+  assert(i->first == 3);
+  assert(i->second == "three");
+
+  eq = c.equal_range(4);
+  assert(std::distance(eq.first, eq.second) == 1);
+  i = eq.first;
+  assert(i->first == 4);
+  assert(i->second == "four");
+
+  assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
+  assert(std::fabs(c.load_factor() - (float)c.size() / c.bucket_count()) < FLT_EPSILON);
+  assert(c.max_load_factor() == 1);
+  assert(c.hash_function() == test_hash<int>());
+  assert(c.key_eq() == test_equal_to<int>());
+  assert(c.get_allocator() == alloc);
 }
 
-int main(int, char**)
-{
-    typedef std::pair<const int, std::string> V;
-    test(test_allocator<V>(10));
-    test(min_allocator<V>());
-    test(explicit_allocator<V>());
+int main(int, char**) {
+  typedef std::pair<const int, std::string> V;
+  test(test_allocator<V>(10));
+  test(min_allocator<V>());
+  test(explicit_allocator<V>());
 
-    return 0;
+  return 0;
 }
diff --git a/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.cnstr/iter_iter_size_hash.pass.cpp b/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.cnstr/iter_iter_size_hash.pass.cpp
index ce99c918acfea..dc6a567fa8849 100644
--- a/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.cnstr/iter_iter_size_hash.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.cnstr/iter_iter_size_hash.pass.cpp
@@ -33,115 +33,108 @@
 #include "test_allocator.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef std::unordered_multimap<int, std::string,
-                                   test_hash<int>,
-                                   test_equal_to<int>,
-                                   test_allocator<std::pair<const int, std::string> >
-                                   > C;
-        typedef std::pair<int, std::string> P;
-        P a[] =
-        {
-            P(1, "one"),
-            P(2, "two"),
-            P(3, "three"),
-            P(4, "four"),
-            P(1, "four"),
-            P(2, "four"),
-        };
-        C c(cpp17_input_iterator<P*>(a), cpp17_input_iterator<P*>(a + sizeof(a)/sizeof(a[0])),
-            7,
-            test_hash<int>(8)
-           );
-        LIBCPP_ASSERT(c.bucket_count() == 7);
-        assert(c.size() == 6);
-        typedef std::pair<C::const_iterator, C::const_iterator> Eq;
-        Eq eq = c.equal_range(1);
-        assert(std::distance(eq.first, eq.second) == 2);
-        std::multiset<std::string> s;
-        s.insert("one");
-        s.insert("four");
-        CheckConsecutiveKeys<C::const_iterator>(c.find(1), c.end(), 1, s);
-        eq = c.equal_range(2);
-        assert(std::distance(eq.first, eq.second) == 2);
-        s.insert("two");
-        s.insert("four");
-        CheckConsecutiveKeys<C::const_iterator>(c.find(2), c.end(), 2, s);
+int main(int, char**) {
+  {
+    typedef std::unordered_multimap<int,
+                                    std::string,
+                                    test_hash<int>,
+                                    test_equal_to<int>,
+                                    test_allocator<std::pair<const int, std::string> > >
+        C;
+    typedef std::pair<int, std::string> P;
+    P a[] = {
+        P(1, "one"),
+        P(2, "two"),
+        P(3, "three"),
+        P(4, "four"),
+        P(1, "four"),
+        P(2, "four"),
+    };
+    C c(cpp17_input_iterator<P*>(a), cpp17_input_iterator<P*>(a + sizeof(a) / sizeof(a[0])), 7, test_hash<int>(8));
+    LIBCPP_ASSERT(c.bucket_count() == 7);
+    assert(c.size() == 6);
+    typedef std::pair<C::const_iterator, C::const_iterator> Eq;
+    Eq eq = c.equal_range(1);
+    assert(std::distance(eq.first, eq.second) == 2);
+    std::multiset<std::string> s;
+    s.insert("one");
+    s.insert("four");
+    CheckConsecutiveKeys<C::const_iterator>(c.find(1), c.end(), 1, s);
+    eq = c.equal_range(2);
+    assert(std::distance(eq.first, eq.second) == 2);
+    s.insert("two");
+    s.insert("four");
+    CheckConsecutiveKeys<C::const_iterator>(c.find(2), c.end(), 2, s);
 
-        eq = c.equal_range(3);
-        assert(std::distance(eq.first, eq.second) == 1);
-        C::const_iterator i = eq.first;
-        assert(i->first == 3);
-        assert(i->second == "three");
-        eq = c.equal_range(4);
-        assert(std::distance(eq.first, eq.second) == 1);
-        i = eq.first;
-        assert(i->first == 4);
-        assert(i->second == "four");
-        assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-        assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
-        assert(std::fabs(c.load_factor() - (float)c.size()/c.bucket_count()) < FLT_EPSILON);
-        assert(c.max_load_factor() == 1);
-        assert(c.hash_function() == test_hash<int>(8));
-        assert(c.key_eq() == test_equal_to<int>());
-        assert((c.get_allocator() == test_allocator<std::pair<const int, std::string> >()));
-    }
+    eq = c.equal_range(3);
+    assert(std::distance(eq.first, eq.second) == 1);
+    C::const_iterator i = eq.first;
+    assert(i->first == 3);
+    assert(i->second == "three");
+    eq = c.equal_range(4);
+    assert(std::distance(eq.first, eq.second) == 1);
+    i = eq.first;
+    assert(i->first == 4);
+    assert(i->second == "four");
+    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
+    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
+    assert(std::fabs(c.load_factor() - (float)c.size() / c.bucket_count()) < FLT_EPSILON);
+    assert(c.max_load_factor() == 1);
+    assert(c.hash_function() == test_hash<int>(8));
+    assert(c.key_eq() == test_equal_to<int>());
+    assert((c.get_allocator() == test_allocator<std::pair<const int, std::string> >()));
+  }
 #if TEST_STD_VER >= 11
-    {
-        typedef std::unordered_multimap<int, std::string,
-                                   test_hash<int>,
-                                   test_equal_to<int>,
-                                   min_allocator<std::pair<const int, std::string> >
-                                   > C;
-        typedef std::pair<int, std::string> P;
-        P a[] =
-        {
-            P(1, "one"),
-            P(2, "two"),
-            P(3, "three"),
-            P(4, "four"),
-            P(1, "four"),
-            P(2, "four"),
-        };
-        C c(cpp17_input_iterator<P*>(a), cpp17_input_iterator<P*>(a + sizeof(a)/sizeof(a[0])),
-            7,
-            test_hash<int>(8)
-           );
-        LIBCPP_ASSERT(c.bucket_count() == 7);
-        assert(c.size() == 6);
-        typedef std::pair<C::const_iterator, C::const_iterator> Eq;
-        Eq eq = c.equal_range(1);
-        assert(std::distance(eq.first, eq.second) == 2);
-        std::multiset<std::string> s;
-        s.insert("one");
-        s.insert("four");
-        CheckConsecutiveKeys<C::const_iterator>(c.find(1), c.end(), 1, s);
-        eq = c.equal_range(2);
-        assert(std::distance(eq.first, eq.second) == 2);
-        s.insert("two");
-        s.insert("four");
-        CheckConsecutiveKeys<C::const_iterator>(c.find(2), c.end(), 2, s);
+  {
+    typedef std::unordered_multimap<int,
+                                    std::string,
+                                    test_hash<int>,
+                                    test_equal_to<int>,
+                                    min_allocator<std::pair<const int, std::string> > >
+        C;
+    typedef std::pair<int, std::string> P;
+    P a[] = {
+        P(1, "one"),
+        P(2, "two"),
+        P(3, "three"),
+        P(4, "four"),
+        P(1, "four"),
+        P(2, "four"),
+    };
+    C c(cpp17_input_iterator<P*>(a), cpp17_input_iterator<P*>(a + sizeof(a) / sizeof(a[0])), 7, test_hash<int>(8));
+    LIBCPP_ASSERT(c.bucket_count() == 7);
+    assert(c.size() == 6);
+    typedef std::pair<C::const_iterator, C::const_iterator> Eq;
+    Eq eq = c.equal_range(1);
+    assert(std::distance(eq.first, eq.second) == 2);
+    std::multiset<std::string> s;
+    s.insert("one");
+    s.insert("four");
+    CheckConsecutiveKeys<C::const_iterator>(c.find(1), c.end(), 1, s);
+    eq = c.equal_range(2);
+    assert(std::distance(eq.first, eq.second) == 2);
+    s.insert("two");
+    s.insert("four");
+    CheckConsecutiveKeys<C::const_iterator>(c.find(2), c.end(), 2, s);
 
-        eq = c.equal_range(3);
-        assert(std::distance(eq.first, eq.second) == 1);
-        C::const_iterator i = eq.first;
-        assert(i->first == 3);
-        assert(i->second == "three");
-        eq = c.equal_range(4);
-        assert(std::distance(eq.first, eq.second) == 1);
-        i = eq.first;
-        assert(i->first == 4);
-        assert(i->second == "four");
-        assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-        assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
-        assert(std::fabs(c.load_factor() - (float)c.size()/c.bucket_count()) < FLT_EPSILON);
-        assert(c.max_load_factor() == 1);
-        assert(c.hash_function() == test_hash<int>(8));
-        assert(c.key_eq() == test_equal_to<int>());
-        assert((c.get_allocator() == min_allocator<std::pair<const int, std::string> >()));
-    }
+    eq = c.equal_range(3);
+    assert(std::distance(eq.first, eq.second) == 1);
+    C::const_iterator i = eq.first;
+    assert(i->first == 3);
+    assert(i->second == "three");
+    eq = c.equal_range(4);
+    assert(std::distance(eq.first, eq.second) == 1);
+    i = eq.first;
+    assert(i->first == 4);
+    assert(i->second == "four");
+    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
+    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
+    assert(std::fabs(c.load_factor() - (float)c.size() / c.bucket_count()) < FLT_EPSILON);
+    assert(c.max_load_factor() == 1);
+    assert(c.hash_function() == test_hash<int>(8));
+    assert(c.key_eq() == test_equal_to<int>());
+    assert((c.get_allocator() == min_allocator<std::pair<const int, std::string> >()));
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.cnstr/iter_iter_size_hash_allocator.pass.cpp b/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.cnstr/iter_iter_size_hash_allocator.pass.cpp
index 82cb9fc443f2a..ee35b36cf7519 100644
--- a/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.cnstr/iter_iter_size_hash_allocator.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.cnstr/iter_iter_size_hash_allocator.pass.cpp
@@ -36,71 +36,55 @@
 #include "min_allocator.h"
 
 template <class Allocator>
-void test(const Allocator& alloc)
-{
-    typedef std::unordered_multimap<int, std::string,
-                                    test_hash<int>,
-                                    test_equal_to<int>,
-                                    Allocator> C;
-    typedef std::pair<int, std::string> P;
-    P a[] =
-    {
-        P(1, "one"),
-        P(2, "two"),
-        P(3, "three"),
-        P(4, "four"),
-        P(1, "four"),
-        P(2, "four")
-    };
-
-    C c(cpp17_input_iterator<P*>(a), cpp17_input_iterator<P*>(a + sizeof(a) / sizeof(a[0])),
-        7,
-        test_hash<int>(5),
-        alloc);
-
-    LIBCPP_ASSERT(c.bucket_count() == 7);
-    assert(c.size() == 6);
-
-    typedef std::pair<typename C::const_iterator, typename C::const_iterator> Eq;
-    Eq eq = c.equal_range(1);
-    assert(std::distance(eq.first, eq.second) == 2);
-    std::multiset<std::string> s;
-    s.insert("one");
-    s.insert("four");
-    CheckConsecutiveKeys<typename C::const_iterator>(c.find(1), c.end(), 1, s);
-
-    eq = c.equal_range(2);
-    assert(std::distance(eq.first, eq.second) == 2);
-    s.insert("two");
-    s.insert("four");
-    CheckConsecutiveKeys<typename C::const_iterator>(c.find(2), c.end(), 2, s);
-
-    eq = c.equal_range(3);
-    assert(std::distance(eq.first, eq.second) == 1);
-    typename C::const_iterator i = eq.first;
-    assert(i->first == 3);
-    assert(i->second == "three");
-
-    eq = c.equal_range(4);
-    assert(std::distance(eq.first, eq.second) == 1);
-    i = eq.first;
-    assert(i->first == 4);
-    assert(i->second == "four");
-
-    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-    assert(std::fabs(c.load_factor() - (float)c.size() / c.bucket_count()) < FLT_EPSILON);
-    assert(c.max_load_factor() == 1);
-    assert(c.hash_function() == test_hash<int>(5));
-    assert(c.key_eq() == test_equal_to<int>());
-    assert(c.get_allocator() == alloc);
+void test(const Allocator& alloc) {
+  typedef std::unordered_multimap<int, std::string, test_hash<int>, test_equal_to<int>, Allocator> C;
+  typedef std::pair<int, std::string> P;
+  P a[] = {P(1, "one"), P(2, "two"), P(3, "three"), P(4, "four"), P(1, "four"), P(2, "four")};
+
+  C c(cpp17_input_iterator<P*>(a), cpp17_input_iterator<P*>(a + sizeof(a) / sizeof(a[0])), 7, test_hash<int>(5), alloc);
+
+  LIBCPP_ASSERT(c.bucket_count() == 7);
+  assert(c.size() == 6);
+
+  typedef std::pair<typename C::const_iterator, typename C::const_iterator> Eq;
+  Eq eq = c.equal_range(1);
+  assert(std::distance(eq.first, eq.second) == 2);
+  std::multiset<std::string> s;
+  s.insert("one");
+  s.insert("four");
+  CheckConsecutiveKeys<typename C::const_iterator>(c.find(1), c.end(), 1, s);
+
+  eq = c.equal_range(2);
+  assert(std::distance(eq.first, eq.second) == 2);
+  s.insert("two");
+  s.insert("four");
+  CheckConsecutiveKeys<typename C::const_iterator>(c.find(2), c.end(), 2, s);
+
+  eq = c.equal_range(3);
+  assert(std::distance(eq.first, eq.second) == 1);
+  typename C::const_iterator i = eq.first;
+  assert(i->first == 3);
+  assert(i->second == "three");
+
+  eq = c.equal_range(4);
+  assert(std::distance(eq.first, eq.second) == 1);
+  i = eq.first;
+  assert(i->first == 4);
+  assert(i->second == "four");
+
+  assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
+  assert(std::fabs(c.load_factor() - (float)c.size() / c.bucket_count()) < FLT_EPSILON);
+  assert(c.max_load_factor() == 1);
+  assert(c.hash_function() == test_hash<int>(5));
+  assert(c.key_eq() == test_equal_to<int>());
+  assert(c.get_allocator() == alloc);
 }
 
-int main(int, char**)
-{
-    typedef std::pair<const int, std::string> V;
-    test(test_allocator<V>(10));
-    test(min_allocator<V>());
-    test(explicit_allocator<V>());
+int main(int, char**) {
+  typedef std::pair<const int, std::string> V;
+  test(test_allocator<V>(10));
+  test(min_allocator<V>());
+  test(explicit_allocator<V>());
 
-    return 0;
+  return 0;
 }
diff --git a/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.cnstr/iter_iter_size_hash_equal.pass.cpp b/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.cnstr/iter_iter_size_hash_equal.pass.cpp
index 0a27a4c37f62a..d071f182f95bc 100644
--- a/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.cnstr/iter_iter_size_hash_equal.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.cnstr/iter_iter_size_hash_equal.pass.cpp
@@ -33,117 +33,116 @@
 #include "test_allocator.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef std::unordered_multimap<int, std::string,
-                                   test_hash<int>,
-                                   test_equal_to<int>,
-                                   test_allocator<std::pair<const int, std::string> >
-                                   > C;
-        typedef std::pair<int, std::string> P;
-        P a[] =
-        {
-            P(1, "one"),
-            P(2, "two"),
-            P(3, "three"),
-            P(4, "four"),
-            P(1, "four"),
-            P(2, "four"),
-        };
-        C c(cpp17_input_iterator<P*>(a), cpp17_input_iterator<P*>(a + sizeof(a)/sizeof(a[0])),
-            7,
-            test_hash<int>(8),
-            test_equal_to<int>(9)
-           );
-        LIBCPP_ASSERT(c.bucket_count() == 7);
-        assert(c.size() == 6);
-        typedef std::pair<C::const_iterator, C::const_iterator> Eq;
-        Eq eq = c.equal_range(1);
-        assert(std::distance(eq.first, eq.second) == 2);
-        std::multiset<std::string> s;
-        s.insert("one");
-        s.insert("four");
-        CheckConsecutiveKeys<C::const_iterator>(c.find(1), c.end(), 1, s);
-        eq = c.equal_range(2);
-        assert(std::distance(eq.first, eq.second) == 2);
-        s.insert("two");
-        s.insert("four");
-        CheckConsecutiveKeys<C::const_iterator>(c.find(2), c.end(), 2, s);
+int main(int, char**) {
+  {
+    typedef std::unordered_multimap<int,
+                                    std::string,
+                                    test_hash<int>,
+                                    test_equal_to<int>,
+                                    test_allocator<std::pair<const int, std::string> > >
+        C;
+    typedef std::pair<int, std::string> P;
+    P a[] = {
+        P(1, "one"),
+        P(2, "two"),
+        P(3, "three"),
+        P(4, "four"),
+        P(1, "four"),
+        P(2, "four"),
+    };
+    C c(cpp17_input_iterator<P*>(a),
+        cpp17_input_iterator<P*>(a + sizeof(a) / sizeof(a[0])),
+        7,
+        test_hash<int>(8),
+        test_equal_to<int>(9));
+    LIBCPP_ASSERT(c.bucket_count() == 7);
+    assert(c.size() == 6);
+    typedef std::pair<C::const_iterator, C::const_iterator> Eq;
+    Eq eq = c.equal_range(1);
+    assert(std::distance(eq.first, eq.second) == 2);
+    std::multiset<std::string> s;
+    s.insert("one");
+    s.insert("four");
+    CheckConsecutiveKeys<C::const_iterator>(c.find(1), c.end(), 1, s);
+    eq = c.equal_range(2);
+    assert(std::distance(eq.first, eq.second) == 2);
+    s.insert("two");
+    s.insert("four");
+    CheckConsecutiveKeys<C::const_iterator>(c.find(2), c.end(), 2, s);
 
-        eq = c.equal_range(3);
-        assert(std::distance(eq.first, eq.second) == 1);
-        C::const_iterator i = eq.first;
-        assert(i->first == 3);
-        assert(i->second == "three");
-        eq = c.equal_range(4);
-        assert(std::distance(eq.first, eq.second) == 1);
-        i = eq.first;
-        assert(i->first == 4);
-        assert(i->second == "four");
-        assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-        assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
-        assert(std::fabs(c.load_factor() - (float)c.size()/c.bucket_count()) < FLT_EPSILON);
-        assert(c.max_load_factor() == 1);
-        assert(c.hash_function() == test_hash<int>(8));
-        assert(c.key_eq() == test_equal_to<int>(9));
-        assert((c.get_allocator() == test_allocator<std::pair<const int, std::string> >()));
-    }
+    eq = c.equal_range(3);
+    assert(std::distance(eq.first, eq.second) == 1);
+    C::const_iterator i = eq.first;
+    assert(i->first == 3);
+    assert(i->second == "three");
+    eq = c.equal_range(4);
+    assert(std::distance(eq.first, eq.second) == 1);
+    i = eq.first;
+    assert(i->first == 4);
+    assert(i->second == "four");
+    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
+    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
+    assert(std::fabs(c.load_factor() - (float)c.size() / c.bucket_count()) < FLT_EPSILON);
+    assert(c.max_load_factor() == 1);
+    assert(c.hash_function() == test_hash<int>(8));
+    assert(c.key_eq() == test_equal_to<int>(9));
+    assert((c.get_allocator() == test_allocator<std::pair<const int, std::string> >()));
+  }
 #if TEST_STD_VER >= 11
-    {
-        typedef std::unordered_multimap<int, std::string,
-                                   test_hash<int>,
-                                   test_equal_to<int>,
-                                   min_allocator<std::pair<const int, std::string> >
-                                   > C;
-        typedef std::pair<int, std::string> P;
-        P a[] =
-        {
-            P(1, "one"),
-            P(2, "two"),
-            P(3, "three"),
-            P(4, "four"),
-            P(1, "four"),
-            P(2, "four"),
-        };
-        C c(cpp17_input_iterator<P*>(a), cpp17_input_iterator<P*>(a + sizeof(a)/sizeof(a[0])),
-            7,
-            test_hash<int>(8),
-            test_equal_to<int>(9)
-           );
-        LIBCPP_ASSERT(c.bucket_count() == 7);
-        assert(c.size() == 6);
-        typedef std::pair<C::const_iterator, C::const_iterator> Eq;
-        Eq eq = c.equal_range(1);
-        assert(std::distance(eq.first, eq.second) == 2);
-        std::multiset<std::string> s;
-        s.insert("one");
-        s.insert("four");
-        CheckConsecutiveKeys<C::const_iterator>(c.find(1), c.end(), 1, s);
-        eq = c.equal_range(2);
-        assert(std::distance(eq.first, eq.second) == 2);
-        s.insert("two");
-        s.insert("four");
-        CheckConsecutiveKeys<C::const_iterator>(c.find(2), c.end(), 2, s);
+  {
+    typedef std::unordered_multimap<int,
+                                    std::string,
+                                    test_hash<int>,
+                                    test_equal_to<int>,
+                                    min_allocator<std::pair<const int, std::string> > >
+        C;
+    typedef std::pair<int, std::string> P;
+    P a[] = {
+        P(1, "one"),
+        P(2, "two"),
+        P(3, "three"),
+        P(4, "four"),
+        P(1, "four"),
+        P(2, "four"),
+    };
+    C c(cpp17_input_iterator<P*>(a),
+        cpp17_input_iterator<P*>(a + sizeof(a) / sizeof(a[0])),
+        7,
+        test_hash<int>(8),
+        test_equal_to<int>(9));
+    LIBCPP_ASSERT(c.bucket_count() == 7);
+    assert(c.size() == 6);
+    typedef std::pair<C::const_iterator, C::const_iterator> Eq;
+    Eq eq = c.equal_range(1);
+    assert(std::distance(eq.first, eq.second) == 2);
+    std::multiset<std::string> s;
+    s.insert("one");
+    s.insert("four");
+    CheckConsecutiveKeys<C::const_iterator>(c.find(1), c.end(), 1, s);
+    eq = c.equal_range(2);
+    assert(std::distance(eq.first, eq.second) == 2);
+    s.insert("two");
+    s.insert("four");
+    CheckConsecutiveKeys<C::const_iterator>(c.find(2), c.end(), 2, s);
 
-        eq = c.equal_range(3);
-        assert(std::distance(eq.first, eq.second) == 1);
-        C::const_iterator i = eq.first;
-        assert(i->first == 3);
-        assert(i->second == "three");
-        eq = c.equal_range(4);
-        assert(std::distance(eq.first, eq.second) == 1);
-        i = eq.first;
-        assert(i->first == 4);
-        assert(i->second == "four");
-        assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-        assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
-        assert(std::fabs(c.load_factor() - (float)c.size()/c.bucket_count()) < FLT_EPSILON);
-        assert(c.max_load_factor() == 1);
-        assert(c.hash_function() == test_hash<int>(8));
-        assert(c.key_eq() == test_equal_to<int>(9));
-        assert((c.get_allocator() == min_allocator<std::pair<const int, std::string> >()));
-    }
+    eq = c.equal_range(3);
+    assert(std::distance(eq.first, eq.second) == 1);
+    C::const_iterator i = eq.first;
+    assert(i->first == 3);
+    assert(i->second == "three");
+    eq = c.equal_range(4);
+    assert(std::distance(eq.first, eq.second) == 1);
+    i = eq.first;
+    assert(i->first == 4);
+    assert(i->second == "four");
+    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
+    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
+    assert(std::fabs(c.load_factor() - (float)c.size() / c.bucket_count()) < FLT_EPSILON);
+    assert(c.max_load_factor() == 1);
+    assert(c.hash_function() == test_hash<int>(8));
+    assert(c.key_eq() == test_equal_to<int>(9));
+    assert((c.get_allocator() == min_allocator<std::pair<const int, std::string> >()));
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.cnstr/iter_iter_size_hash_equal_allocator.pass.cpp b/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.cnstr/iter_iter_size_hash_equal_allocator.pass.cpp
index 0dd157ae5b11a..4eee3451c610f 100644
--- a/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.cnstr/iter_iter_size_hash_equal_allocator.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.cnstr/iter_iter_size_hash_equal_allocator.pass.cpp
@@ -34,175 +34,169 @@
 #include "test_allocator.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef std::unordered_multimap<int, std::string,
-                                   test_hash<int>,
-                                   test_equal_to<int>,
-                                   test_allocator<std::pair<const int, std::string> >
-                                   > C;
-        typedef std::pair<int, std::string> P;
-        P a[] =
-        {
-            P(1, "one"),
-            P(2, "two"),
-            P(3, "three"),
-            P(4, "four"),
-            P(1, "four"),
-            P(2, "four"),
-        };
-        C c(cpp17_input_iterator<P*>(a), cpp17_input_iterator<P*>(a + sizeof(a)/sizeof(a[0])),
-            7,
-            test_hash<int>(8),
-            test_equal_to<int>(9),
-            test_allocator<std::pair<const int, std::string> >(10)
-           );
-        LIBCPP_ASSERT(c.bucket_count() == 7);
-        assert(c.size() == 6);
-        typedef std::pair<C::const_iterator, C::const_iterator> Eq;
-        Eq eq = c.equal_range(1);
-        assert(std::distance(eq.first, eq.second) == 2);
-        std::multiset<std::string> s;
-        s.insert("one");
-        s.insert("four");
-        CheckConsecutiveKeys<C::const_iterator>(c.find(1), c.end(), 1, s);
-        eq = c.equal_range(2);
-        assert(std::distance(eq.first, eq.second) == 2);
-        s.insert("two");
-        s.insert("four");
-        CheckConsecutiveKeys<C::const_iterator>(c.find(2), c.end(), 2, s);
+int main(int, char**) {
+  {
+    typedef std::unordered_multimap<int,
+                                    std::string,
+                                    test_hash<int>,
+                                    test_equal_to<int>,
+                                    test_allocator<std::pair<const int, std::string> > >
+        C;
+    typedef std::pair<int, std::string> P;
+    P a[] = {
+        P(1, "one"),
+        P(2, "two"),
+        P(3, "three"),
+        P(4, "four"),
+        P(1, "four"),
+        P(2, "four"),
+    };
+    C c(cpp17_input_iterator<P*>(a),
+        cpp17_input_iterator<P*>(a + sizeof(a) / sizeof(a[0])),
+        7,
+        test_hash<int>(8),
+        test_equal_to<int>(9),
+        test_allocator<std::pair<const int, std::string> >(10));
+    LIBCPP_ASSERT(c.bucket_count() == 7);
+    assert(c.size() == 6);
+    typedef std::pair<C::const_iterator, C::const_iterator> Eq;
+    Eq eq = c.equal_range(1);
+    assert(std::distance(eq.first, eq.second) == 2);
+    std::multiset<std::string> s;
+    s.insert("one");
+    s.insert("four");
+    CheckConsecutiveKeys<C::const_iterator>(c.find(1), c.end(), 1, s);
+    eq = c.equal_range(2);
+    assert(std::distance(eq.first, eq.second) == 2);
+    s.insert("two");
+    s.insert("four");
+    CheckConsecutiveKeys<C::const_iterator>(c.find(2), c.end(), 2, s);
 
-        eq = c.equal_range(3);
-        assert(std::distance(eq.first, eq.second) == 1);
-        C::const_iterator i = eq.first;
-        assert(i->first == 3);
-        assert(i->second == "three");
-        eq = c.equal_range(4);
-        assert(std::distance(eq.first, eq.second) == 1);
-        i = eq.first;
-        assert(i->first == 4);
-        assert(i->second == "four");
-        assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-        assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
-        assert(std::fabs(c.load_factor() - (float)c.size()/c.bucket_count()) < FLT_EPSILON);
-        assert(c.max_load_factor() == 1);
-        assert(c.hash_function() == test_hash<int>(8));
-        assert(c.key_eq() == test_equal_to<int>(9));
-        assert((c.get_allocator() == test_allocator<std::pair<const int, std::string> >(10)));
-    }
+    eq = c.equal_range(3);
+    assert(std::distance(eq.first, eq.second) == 1);
+    C::const_iterator i = eq.first;
+    assert(i->first == 3);
+    assert(i->second == "three");
+    eq = c.equal_range(4);
+    assert(std::distance(eq.first, eq.second) == 1);
+    i = eq.first;
+    assert(i->first == 4);
+    assert(i->second == "four");
+    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
+    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
+    assert(std::fabs(c.load_factor() - (float)c.size() / c.bucket_count()) < FLT_EPSILON);
+    assert(c.max_load_factor() == 1);
+    assert(c.hash_function() == test_hash<int>(8));
+    assert(c.key_eq() == test_equal_to<int>(9));
+    assert((c.get_allocator() == test_allocator<std::pair<const int, std::string> >(10)));
+  }
 #if TEST_STD_VER >= 11
-    {
-        typedef std::unordered_multimap<int, std::string,
-                                   test_hash<int>,
-                                   test_equal_to<int>,
-                                   min_allocator<std::pair<const int, std::string> >
-                                   > C;
-        typedef std::pair<int, std::string> P;
-        P a[] =
-        {
-            P(1, "one"),
-            P(2, "two"),
-            P(3, "three"),
-            P(4, "four"),
-            P(1, "four"),
-            P(2, "four"),
-        };
-        C c(cpp17_input_iterator<P*>(a), cpp17_input_iterator<P*>(a + sizeof(a)/sizeof(a[0])),
-            7,
-            test_hash<int>(8),
-            test_equal_to<int>(9),
-            min_allocator<std::pair<const int, std::string> >()
-           );
-        LIBCPP_ASSERT(c.bucket_count() == 7);
-        assert(c.size() == 6);
-        typedef std::pair<C::const_iterator, C::const_iterator> Eq;
-        Eq eq = c.equal_range(1);
-        assert(std::distance(eq.first, eq.second) == 2);
-        std::multiset<std::string> s;
-        s.insert("one");
-        s.insert("four");
-        CheckConsecutiveKeys<C::const_iterator>(c.find(1), c.end(), 1, s);
-        eq = c.equal_range(2);
-        assert(std::distance(eq.first, eq.second) == 2);
-        s.insert("two");
-        s.insert("four");
-        CheckConsecutiveKeys<C::const_iterator>(c.find(2), c.end(), 2, s);
+  {
+    typedef std::unordered_multimap<int,
+                                    std::string,
+                                    test_hash<int>,
+                                    test_equal_to<int>,
+                                    min_allocator<std::pair<const int, std::string> > >
+        C;
+    typedef std::pair<int, std::string> P;
+    P a[] = {
+        P(1, "one"),
+        P(2, "two"),
+        P(3, "three"),
+        P(4, "four"),
+        P(1, "four"),
+        P(2, "four"),
+    };
+    C c(cpp17_input_iterator<P*>(a),
+        cpp17_input_iterator<P*>(a + sizeof(a) / sizeof(a[0])),
+        7,
+        test_hash<int>(8),
+        test_equal_to<int>(9),
+        min_allocator<std::pair<const int, std::string> >());
+    LIBCPP_ASSERT(c.bucket_count() == 7);
+    assert(c.size() == 6);
+    typedef std::pair<C::const_iterator, C::const_iterator> Eq;
+    Eq eq = c.equal_range(1);
+    assert(std::distance(eq.first, eq.second) == 2);
+    std::multiset<std::string> s;
+    s.insert("one");
+    s.insert("four");
+    CheckConsecutiveKeys<C::const_iterator>(c.find(1), c.end(), 1, s);
+    eq = c.equal_range(2);
+    assert(std::distance(eq.first, eq.second) == 2);
+    s.insert("two");
+    s.insert("four");
+    CheckConsecutiveKeys<C::const_iterator>(c.find(2), c.end(), 2, s);
 
-        eq = c.equal_range(3);
-        assert(std::distance(eq.first, eq.second) == 1);
-        C::const_iterator i = eq.first;
-        assert(i->first == 3);
-        assert(i->second == "three");
-        eq = c.equal_range(4);
-        assert(std::distance(eq.first, eq.second) == 1);
-        i = eq.first;
-        assert(i->first == 4);
-        assert(i->second == "four");
-        assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-        assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
-        assert(std::fabs(c.load_factor() - (float)c.size()/c.bucket_count()) < FLT_EPSILON);
-        assert(c.max_load_factor() == 1);
-        assert(c.hash_function() == test_hash<int>(8));
-        assert(c.key_eq() == test_equal_to<int>(9));
-        assert((c.get_allocator() == min_allocator<std::pair<const int, std::string> >()));
-    }
-    {
-        typedef explicit_allocator<std::pair<const int, std::string>> A;
-        typedef std::unordered_multimap<int, std::string,
-                                   test_hash<int>,
-                                   test_equal_to<int>,
-                                   A
-                                   > C;
-        typedef std::pair<int, std::string> P;
-        P a[] =
-        {
-            P(1, "one"),
-            P(2, "two"),
-            P(3, "three"),
-            P(4, "four"),
-            P(1, "four"),
-            P(2, "four"),
-        };
-        C c(cpp17_input_iterator<P*>(a), cpp17_input_iterator<P*>(a + sizeof(a)/sizeof(a[0])),
-            7,
-            test_hash<int>(8),
-            test_equal_to<int>(9),
-            A{}
-           );
-        LIBCPP_ASSERT(c.bucket_count() == 7);
-        assert(c.size() == 6);
-        typedef std::pair<C::const_iterator, C::const_iterator> Eq;
-        Eq eq = c.equal_range(1);
-        assert(std::distance(eq.first, eq.second) == 2);
-        std::multiset<std::string> s;
-        s.insert("one");
-        s.insert("four");
-        CheckConsecutiveKeys<C::const_iterator>(c.find(1), c.end(), 1, s);
-        eq = c.equal_range(2);
-        assert(std::distance(eq.first, eq.second) == 2);
-        s.insert("two");
-        s.insert("four");
-        CheckConsecutiveKeys<C::const_iterator>(c.find(2), c.end(), 2, s);
+    eq = c.equal_range(3);
+    assert(std::distance(eq.first, eq.second) == 1);
+    C::const_iterator i = eq.first;
+    assert(i->first == 3);
+    assert(i->second == "three");
+    eq = c.equal_range(4);
+    assert(std::distance(eq.first, eq.second) == 1);
+    i = eq.first;
+    assert(i->first == 4);
+    assert(i->second == "four");
+    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
+    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
+    assert(std::fabs(c.load_factor() - (float)c.size() / c.bucket_count()) < FLT_EPSILON);
+    assert(c.max_load_factor() == 1);
+    assert(c.hash_function() == test_hash<int>(8));
+    assert(c.key_eq() == test_equal_to<int>(9));
+    assert((c.get_allocator() == min_allocator<std::pair<const int, std::string> >()));
+  }
+  {
+    typedef explicit_allocator<std::pair<const int, std::string>> A;
+    typedef std::unordered_multimap<int, std::string, test_hash<int>, test_equal_to<int>, A > C;
+    typedef std::pair<int, std::string> P;
+    P a[] = {
+        P(1, "one"),
+        P(2, "two"),
+        P(3, "three"),
+        P(4, "four"),
+        P(1, "four"),
+        P(2, "four"),
+    };
+    C c(cpp17_input_iterator<P*>(a),
+        cpp17_input_iterator<P*>(a + sizeof(a) / sizeof(a[0])),
+        7,
+        test_hash<int>(8),
+        test_equal_to<int>(9),
+        A{});
+    LIBCPP_ASSERT(c.bucket_count() == 7);
+    assert(c.size() == 6);
+    typedef std::pair<C::const_iterator, C::const_iterator> Eq;
+    Eq eq = c.equal_range(1);
+    assert(std::distance(eq.first, eq.second) == 2);
+    std::multiset<std::string> s;
+    s.insert("one");
+    s.insert("four");
+    CheckConsecutiveKeys<C::const_iterator>(c.find(1), c.end(), 1, s);
+    eq = c.equal_range(2);
+    assert(std::distance(eq.first, eq.second) == 2);
+    s.insert("two");
+    s.insert("four");
+    CheckConsecutiveKeys<C::const_iterator>(c.find(2), c.end(), 2, s);
 
-        eq = c.equal_range(3);
-        assert(std::distance(eq.first, eq.second) == 1);
-        C::const_iterator i = eq.first;
-        assert(i->first == 3);
-        assert(i->second == "three");
-        eq = c.equal_range(4);
-        assert(std::distance(eq.first, eq.second) == 1);
-        i = eq.first;
-        assert(i->first == 4);
-        assert(i->second == "four");
-        assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-        assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
-        assert(std::fabs(c.load_factor() - (float)c.size()/c.bucket_count()) < FLT_EPSILON);
-        assert(c.max_load_factor() == 1);
-        assert(c.hash_function() == test_hash<int>(8));
-        assert(c.key_eq() == test_equal_to<int>(9));
-        assert(c.get_allocator() == A{});
-    }
+    eq = c.equal_range(3);
+    assert(std::distance(eq.first, eq.second) == 1);
+    C::const_iterator i = eq.first;
+    assert(i->first == 3);
+    assert(i->second == "three");
+    eq = c.equal_range(4);
+    assert(std::distance(eq.first, eq.second) == 1);
+    i = eq.first;
+    assert(i->first == 4);
+    assert(i->second == "four");
+    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
+    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
+    assert(std::fabs(c.load_factor() - (float)c.size() / c.bucket_count()) < FLT_EPSILON);
+    assert(c.max_load_factor() == 1);
+    assert(c.hash_function() == test_hash<int>(8));
+    assert(c.key_eq() == test_equal_to<int>(9));
+    assert(c.get_allocator() == A{});
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.cnstr/move.pass.cpp b/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.cnstr/move.pass.cpp
index afe88a27c7c51..12e6d07708d87 100644
--- a/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.cnstr/move.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.cnstr/move.pass.cpp
@@ -31,181 +31,172 @@
 #include "test_allocator.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef std::unordered_multimap<int, std::string,
-                                   test_hash<int>,
-                                   test_equal_to<int>,
-                                   test_allocator<std::pair<const int, std::string> >
-                                   > C;
+int main(int, char**) {
+  {
+    typedef std::unordered_multimap<int,
+                                    std::string,
+                                    test_hash<int>,
+                                    test_equal_to<int>,
+                                    test_allocator<std::pair<const int, std::string> > >
+        C;
 
-        C c0(7,
-            test_hash<int>(8),
-            test_equal_to<int>(9),
-            test_allocator<std::pair<const int, std::string> >(10)
-           );
-        C c = std::move(c0);
-        LIBCPP_ASSERT(c.bucket_count() == 7);
-        assert(c.size() == 0);
-        assert(c.hash_function() == test_hash<int>(8));
-        assert(c.key_eq() == test_equal_to<int>(9));
-        assert(c.get_allocator() ==
-               (test_allocator<std::pair<const int, std::string> >(10)));
-        assert(c.empty());
-        assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-        assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
-        assert(c.load_factor() == 0);
-        assert(c.max_load_factor() == 1);
+    C c0(7, test_hash<int>(8), test_equal_to<int>(9), test_allocator<std::pair<const int, std::string> >(10));
+    C c = std::move(c0);
+    LIBCPP_ASSERT(c.bucket_count() == 7);
+    assert(c.size() == 0);
+    assert(c.hash_function() == test_hash<int>(8));
+    assert(c.key_eq() == test_equal_to<int>(9));
+    assert(c.get_allocator() == (test_allocator<std::pair<const int, std::string> >(10)));
+    assert(c.empty());
+    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
+    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
+    assert(c.load_factor() == 0);
+    assert(c.max_load_factor() == 1);
 
-        assert(c0.empty());
-    }
-    {
-        typedef std::unordered_multimap<int, std::string,
-                                   test_hash<int>,
-                                   test_equal_to<int>,
-                                   test_allocator<std::pair<const int, std::string> >
-                                   > C;
-        typedef std::pair<int, std::string> P;
-        P a[] =
-        {
-            P(1, "one"),
-            P(2, "two"),
-            P(3, "three"),
-            P(4, "four"),
-            P(1, "four"),
-            P(2, "four"),
-        };
-        C c0(a, a + sizeof(a)/sizeof(a[0]),
-            7,
-            test_hash<int>(8),
-            test_equal_to<int>(9),
-            test_allocator<std::pair<const int, std::string> >(10)
-           );
-        C::iterator it0 = c0.begin();
-        C c = std::move(c0);
-        assert(it0 == c.begin()); // Iterators remain valid
-        LIBCPP_ASSERT(c.bucket_count() == 7);
-        assert(c.size() == 6);
-        typedef std::pair<C::const_iterator, C::const_iterator> Eq;
-        Eq eq = c.equal_range(1);
-        assert(std::distance(eq.first, eq.second) == 2);
-        std::multiset<std::string> s;
-        s.insert("one");
-        s.insert("four");
-        CheckConsecutiveKeys<C::const_iterator>(c.find(1), c.end(), 1, s);
-        eq = c.equal_range(2);
-        assert(std::distance(eq.first, eq.second) == 2);
-        s.insert("two");
-        s.insert("four");
-        CheckConsecutiveKeys<C::const_iterator>(c.find(2), c.end(), 2, s);
+    assert(c0.empty());
+  }
+  {
+    typedef std::unordered_multimap<int,
+                                    std::string,
+                                    test_hash<int>,
+                                    test_equal_to<int>,
+                                    test_allocator<std::pair<const int, std::string> > >
+        C;
+    typedef std::pair<int, std::string> P;
+    P a[] = {
+        P(1, "one"),
+        P(2, "two"),
+        P(3, "three"),
+        P(4, "four"),
+        P(1, "four"),
+        P(2, "four"),
+    };
+    C c0(a,
+         a + sizeof(a) / sizeof(a[0]),
+         7,
+         test_hash<int>(8),
+         test_equal_to<int>(9),
+         test_allocator<std::pair<const int, std::string> >(10));
+    C::iterator it0 = c0.begin();
+    C c             = std::move(c0);
+    assert(it0 == c.begin()); // Iterators remain valid
+    LIBCPP_ASSERT(c.bucket_count() == 7);
+    assert(c.size() == 6);
+    typedef std::pair<C::const_iterator, C::const_iterator> Eq;
+    Eq eq = c.equal_range(1);
+    assert(std::distance(eq.first, eq.second) == 2);
+    std::multiset<std::string> s;
+    s.insert("one");
+    s.insert("four");
+    CheckConsecutiveKeys<C::const_iterator>(c.find(1), c.end(), 1, s);
+    eq = c.equal_range(2);
+    assert(std::distance(eq.first, eq.second) == 2);
+    s.insert("two");
+    s.insert("four");
+    CheckConsecutiveKeys<C::const_iterator>(c.find(2), c.end(), 2, s);
 
-        eq = c.equal_range(3);
-        assert(std::distance(eq.first, eq.second) == 1);
-        C::const_iterator i = eq.first;
-        assert(i->first == 3);
-        assert(i->second == "three");
-        eq = c.equal_range(4);
-        assert(std::distance(eq.first, eq.second) == 1);
-        i = eq.first;
-        assert(i->first == 4);
-        assert(i->second == "four");
-        assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-        assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
-        assert(std::fabs(c.load_factor() - (float)c.size()/c.bucket_count()) < FLT_EPSILON);
-        assert(c.max_load_factor() == 1);
-        assert(c.hash_function() == test_hash<int>(8));
-        assert(c.key_eq() == test_equal_to<int>(9));
-        assert((c.get_allocator() == test_allocator<std::pair<const int, std::string> >(10)));
+    eq = c.equal_range(3);
+    assert(std::distance(eq.first, eq.second) == 1);
+    C::const_iterator i = eq.first;
+    assert(i->first == 3);
+    assert(i->second == "three");
+    eq = c.equal_range(4);
+    assert(std::distance(eq.first, eq.second) == 1);
+    i = eq.first;
+    assert(i->first == 4);
+    assert(i->second == "four");
+    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
+    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
+    assert(std::fabs(c.load_factor() - (float)c.size() / c.bucket_count()) < FLT_EPSILON);
+    assert(c.max_load_factor() == 1);
+    assert(c.hash_function() == test_hash<int>(8));
+    assert(c.key_eq() == test_equal_to<int>(9));
+    assert((c.get_allocator() == test_allocator<std::pair<const int, std::string> >(10)));
 
-        assert(c0.empty());
-    }
-    {
-        typedef std::unordered_multimap<int, std::string,
-                                   test_hash<int>,
-                                   test_equal_to<int>,
-                                   min_allocator<std::pair<const int, std::string> >
-                                   > C;
-        C c0(7,
-            test_hash<int>(8),
-            test_equal_to<int>(9),
-            min_allocator<std::pair<const int, std::string> >()
-           );
-        C c = std::move(c0);
-        LIBCPP_ASSERT(c.bucket_count() == 7);
-        assert(c.size() == 0);
-        assert(c.hash_function() == test_hash<int>(8));
-        assert(c.key_eq() == test_equal_to<int>(9));
-        assert(c.get_allocator() ==
-               (min_allocator<std::pair<const int, std::string> >()));
-        assert(c.empty());
-        assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-        assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
-        assert(c.load_factor() == 0);
-        assert(c.max_load_factor() == 1);
+    assert(c0.empty());
+  }
+  {
+    typedef std::unordered_multimap<int,
+                                    std::string,
+                                    test_hash<int>,
+                                    test_equal_to<int>,
+                                    min_allocator<std::pair<const int, std::string> > >
+        C;
+    C c0(7, test_hash<int>(8), test_equal_to<int>(9), min_allocator<std::pair<const int, std::string> >());
+    C c = std::move(c0);
+    LIBCPP_ASSERT(c.bucket_count() == 7);
+    assert(c.size() == 0);
+    assert(c.hash_function() == test_hash<int>(8));
+    assert(c.key_eq() == test_equal_to<int>(9));
+    assert(c.get_allocator() == (min_allocator<std::pair<const int, std::string> >()));
+    assert(c.empty());
+    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
+    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
+    assert(c.load_factor() == 0);
+    assert(c.max_load_factor() == 1);
 
-        assert(c0.empty());
-    }
-    {
-        typedef std::unordered_multimap<int, std::string,
-                                   test_hash<int>,
-                                   test_equal_to<int>,
-                                   min_allocator<std::pair<const int, std::string> >
-                                   > C;
-        typedef std::pair<int, std::string> P;
-        P a[] =
-        {
-            P(1, "one"),
-            P(2, "two"),
-            P(3, "three"),
-            P(4, "four"),
-            P(1, "four"),
-            P(2, "four"),
-        };
-        C c0(a, a + sizeof(a)/sizeof(a[0]),
-            7,
-            test_hash<int>(8),
-            test_equal_to<int>(9),
-            min_allocator<std::pair<const int, std::string> >()
-           );
-        C::iterator it0 = c0.begin();
-        C c = std::move(c0);
-        assert(it0 == c.begin()); // Iterators remain valid
-        LIBCPP_ASSERT(c.bucket_count() == 7);
-        assert(c.size() == 6);
-        typedef std::pair<C::const_iterator, C::const_iterator> Eq;
-        Eq eq = c.equal_range(1);
-        assert(std::distance(eq.first, eq.second) == 2);
-        std::multiset<std::string> s;
-        s.insert("one");
-        s.insert("four");
-        CheckConsecutiveKeys<C::const_iterator>(c.find(1), c.end(), 1, s);
-        eq = c.equal_range(2);
-        assert(std::distance(eq.first, eq.second) == 2);
-        s.insert("two");
-        s.insert("four");
-        CheckConsecutiveKeys<C::const_iterator>(c.find(2), c.end(), 2, s);
+    assert(c0.empty());
+  }
+  {
+    typedef std::unordered_multimap<int,
+                                    std::string,
+                                    test_hash<int>,
+                                    test_equal_to<int>,
+                                    min_allocator<std::pair<const int, std::string> > >
+        C;
+    typedef std::pair<int, std::string> P;
+    P a[] = {
+        P(1, "one"),
+        P(2, "two"),
+        P(3, "three"),
+        P(4, "four"),
+        P(1, "four"),
+        P(2, "four"),
+    };
+    C c0(a,
+         a + sizeof(a) / sizeof(a[0]),
+         7,
+         test_hash<int>(8),
+         test_equal_to<int>(9),
+         min_allocator<std::pair<const int, std::string> >());
+    C::iterator it0 = c0.begin();
+    C c             = std::move(c0);
+    assert(it0 == c.begin()); // Iterators remain valid
+    LIBCPP_ASSERT(c.bucket_count() == 7);
+    assert(c.size() == 6);
+    typedef std::pair<C::const_iterator, C::const_iterator> Eq;
+    Eq eq = c.equal_range(1);
+    assert(std::distance(eq.first, eq.second) == 2);
+    std::multiset<std::string> s;
+    s.insert("one");
+    s.insert("four");
+    CheckConsecutiveKeys<C::const_iterator>(c.find(1), c.end(), 1, s);
+    eq = c.equal_range(2);
+    assert(std::distance(eq.first, eq.second) == 2);
+    s.insert("two");
+    s.insert("four");
+    CheckConsecutiveKeys<C::const_iterator>(c.find(2), c.end(), 2, s);
 
-        eq = c.equal_range(3);
-        assert(std::distance(eq.first, eq.second) == 1);
-        C::const_iterator i = eq.first;
-        assert(i->first == 3);
-        assert(i->second == "three");
-        eq = c.equal_range(4);
-        assert(std::distance(eq.first, eq.second) == 1);
-        i = eq.first;
-        assert(i->first == 4);
-        assert(i->second == "four");
-        assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-        assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
-        assert(std::fabs(c.load_factor() - (float)c.size()/c.bucket_count()) < FLT_EPSILON);
-        assert(c.max_load_factor() == 1);
-        assert(c.hash_function() == test_hash<int>(8));
-        assert(c.key_eq() == test_equal_to<int>(9));
-        assert((c.get_allocator() == min_allocator<std::pair<const int, std::string> >()));
+    eq = c.equal_range(3);
+    assert(std::distance(eq.first, eq.second) == 1);
+    C::const_iterator i = eq.first;
+    assert(i->first == 3);
+    assert(i->second == "three");
+    eq = c.equal_range(4);
+    assert(std::distance(eq.first, eq.second) == 1);
+    i = eq.first;
+    assert(i->first == 4);
+    assert(i->second == "four");
+    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
+    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
+    assert(std::fabs(c.load_factor() - (float)c.size() / c.bucket_count()) < FLT_EPSILON);
+    assert(c.max_load_factor() == 1);
+    assert(c.hash_function() == test_hash<int>(8));
+    assert(c.key_eq() == test_equal_to<int>(9));
+    assert((c.get_allocator() == min_allocator<std::pair<const int, std::string> >()));
 
-        assert(c0.empty());
-    }
+    assert(c0.empty());
+  }
 
-    return 0;
+  return 0;
 }
diff --git a/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.cnstr/move_alloc.addressof.compile.pass.cpp b/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.cnstr/move_alloc.addressof.compile.pass.cpp
index 6419a03666d65..53c5179242949 100644
--- a/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.cnstr/move_alloc.addressof.compile.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.cnstr/move_alloc.addressof.compile.pass.cpp
@@ -28,8 +28,12 @@
 
 void test() {
   using A = test_allocator<std::pair<const operator_hijacker, operator_hijacker>>;
-  using C = std::unordered_multimap<operator_hijacker, operator_hijacker, std::hash<operator_hijacker>,
-                                    std::equal_to<operator_hijacker>, A>;
+  using C =
+      std::unordered_multimap<operator_hijacker,
+                              operator_hijacker,
+                              std::hash<operator_hijacker>,
+                              std::equal_to<operator_hijacker>,
+                              A>;
 
   C mo;
   C m(std::move(mo), A());
diff --git a/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.cnstr/move_alloc.pass.cpp b/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.cnstr/move_alloc.pass.cpp
index 01260c4662701..4a038dc4a6589 100644
--- a/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.cnstr/move_alloc.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.cnstr/move_alloc.pass.cpp
@@ -31,244 +31,203 @@
 #include "test_allocator.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef std::pair<int, std::string> P;
-        typedef test_allocator<std::pair<const int, std::string>> A;
-        typedef std::unordered_multimap<int, std::string,
-                                   test_hash<int>,
-                                   test_equal_to<int>,
-                                   A
-                                   > C;
-        P a[] =
-        {
-            P(1, "one"),
-            P(2, "two"),
-            P(3, "three"),
-            P(4, "four"),
-            P(1, "four"),
-            P(2, "four"),
-        };
-        C c0(a, a + sizeof(a)/sizeof(a[0]),
-            7,
-            test_hash<int>(8),
-            test_equal_to<int>(9),
-            A(10)
-           );
-        C c(std::move(c0), A(12));
-        assert(c.bucket_count() >= 7);
-        assert(c.size() == 6);
-        typedef std::pair<C::const_iterator, C::const_iterator> Eq;
-        Eq eq = c.equal_range(1);
-        assert(std::distance(eq.first, eq.second) == 2);
-        std::multiset<std::string> s;
-        s.insert("one");
-        s.insert("four");
-        CheckConsecutiveKeys<C::const_iterator>(c.find(1), c.end(), 1, s);
-        eq = c.equal_range(2);
-        assert(std::distance(eq.first, eq.second) == 2);
-        s.insert("two");
-        s.insert("four");
-        CheckConsecutiveKeys<C::const_iterator>(c.find(2), c.end(), 2, s);
-
-        eq = c.equal_range(3);
-        assert(std::distance(eq.first, eq.second) == 1);
-        C::const_iterator i = eq.first;
-        assert(i->first == 3);
-        assert(i->second == "three");
-        eq = c.equal_range(4);
-        assert(std::distance(eq.first, eq.second) == 1);
-        i = eq.first;
-        assert(i->first == 4);
-        assert(i->second == "four");
-        assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-        assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
-        assert(std::fabs(c.load_factor() - (float)c.size()/c.bucket_count()) < FLT_EPSILON);
-        assert(c.max_load_factor() == 1);
-        assert(c.hash_function() == test_hash<int>(8));
-        assert(c.key_eq() == test_equal_to<int>(9));
-        assert((c.get_allocator() == test_allocator<std::pair<const int, std::string> >(12)));
-
-        assert(c0.empty());
-    }
-    {
-        typedef std::pair<int, std::string> P;
-        typedef test_allocator<std::pair<const int, std::string>> A;
-        typedef std::unordered_multimap<int, std::string,
-                                   test_hash<int>,
-                                   test_equal_to<int>,
-                                   A
-                                   > C;
-        P a[] =
-        {
-            P(1, "one"),
-            P(2, "two"),
-            P(3, "three"),
-            P(4, "four"),
-            P(1, "four"),
-            P(2, "four"),
-        };
-        C c0(a, a + sizeof(a)/sizeof(a[0]),
-            7,
-            test_hash<int>(8),
-            test_equal_to<int>(9),
-            A(10)
-           );
-        C c(std::move(c0), A(10));
-        LIBCPP_ASSERT(c.bucket_count() == 7);
-        assert(c.size() == 6);
-        typedef std::pair<C::const_iterator, C::const_iterator> Eq;
-        Eq eq = c.equal_range(1);
-        assert(std::distance(eq.first, eq.second) == 2);
-        std::multiset<std::string> s;
-        s.insert("one");
-        s.insert("four");
-        CheckConsecutiveKeys<C::const_iterator>(c.find(1), c.end(), 1, s);
-        eq = c.equal_range(2);
-        assert(std::distance(eq.first, eq.second) == 2);
-        s.insert("two");
-        s.insert("four");
-        CheckConsecutiveKeys<C::const_iterator>(c.find(2), c.end(), 2, s);
-
-        eq = c.equal_range(3);
-        assert(std::distance(eq.first, eq.second) == 1);
-        C::const_iterator i = eq.first;
-        assert(i->first == 3);
-        assert(i->second == "three");
-        eq = c.equal_range(4);
-        assert(std::distance(eq.first, eq.second) == 1);
-        i = eq.first;
-        assert(i->first == 4);
-        assert(i->second == "four");
-        assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-        assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
-        assert(std::fabs(c.load_factor() - (float)c.size()/c.bucket_count()) < FLT_EPSILON);
-        assert(c.max_load_factor() == 1);
-        assert(c.hash_function() == test_hash<int>(8));
-        assert(c.key_eq() == test_equal_to<int>(9));
-        assert((c.get_allocator() == test_allocator<std::pair<const int, std::string> >(10)));
-
-        assert(c0.empty());
-    }
-    {
-        typedef std::pair<int, std::string> P;
-        typedef min_allocator<std::pair<const int, std::string>> A;
-        typedef std::unordered_multimap<int, std::string,
-                                   test_hash<int>,
-                                   test_equal_to<int>,
-                                   A
-                                   > C;
-        P a[] =
-        {
-            P(1, "one"),
-            P(2, "two"),
-            P(3, "three"),
-            P(4, "four"),
-            P(1, "four"),
-            P(2, "four"),
-        };
-        C c0(a, a + sizeof(a)/sizeof(a[0]),
-            7,
-            test_hash<int>(8),
-            test_equal_to<int>(9),
-            A()
-           );
-        C c(std::move(c0), A());
-        LIBCPP_ASSERT(c.bucket_count() == 7);
-        assert(c.size() == 6);
-        typedef std::pair<C::const_iterator, C::const_iterator> Eq;
-        Eq eq = c.equal_range(1);
-        assert(std::distance(eq.first, eq.second) == 2);
-        std::multiset<std::string> s;
-        s.insert("one");
-        s.insert("four");
-        CheckConsecutiveKeys<C::const_iterator>(c.find(1), c.end(), 1, s);
-        eq = c.equal_range(2);
-        assert(std::distance(eq.first, eq.second) == 2);
-        s.insert("two");
-        s.insert("four");
-        CheckConsecutiveKeys<C::const_iterator>(c.find(2), c.end(), 2, s);
-
-        eq = c.equal_range(3);
-        assert(std::distance(eq.first, eq.second) == 1);
-        C::const_iterator i = eq.first;
-        assert(i->first == 3);
-        assert(i->second == "three");
-        eq = c.equal_range(4);
-        assert(std::distance(eq.first, eq.second) == 1);
-        i = eq.first;
-        assert(i->first == 4);
-        assert(i->second == "four");
-        assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-        assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
-        assert(std::fabs(c.load_factor() - (float)c.size()/c.bucket_count()) < FLT_EPSILON);
-        assert(c.max_load_factor() == 1);
-        assert(c.hash_function() == test_hash<int>(8));
-        assert(c.key_eq() == test_equal_to<int>(9));
-        assert((c.get_allocator() == min_allocator<std::pair<const int, std::string> >()));
-
-        assert(c0.empty());
-    }
-    {
-        typedef std::pair<int, std::string> P;
-        typedef explicit_allocator<std::pair<const int, std::string>> A;
-        typedef std::unordered_multimap<int, std::string,
-                                   test_hash<int>,
-                                   test_equal_to<int>,
-                                   A
-                                   > C;
-        P a[] =
-        {
-            P(1, "one"),
-            P(2, "two"),
-            P(3, "three"),
-            P(4, "four"),
-            P(1, "four"),
-            P(2, "four"),
-        };
-        C c0(a, a + sizeof(a)/sizeof(a[0]),
-            7,
-            test_hash<int>(8),
-            test_equal_to<int>(9),
-            A{}
-           );
-        C c(std::move(c0), A());
-        LIBCPP_ASSERT(c.bucket_count() == 7);
-        assert(c.size() == 6);
-        typedef std::pair<C::const_iterator, C::const_iterator> Eq;
-        Eq eq = c.equal_range(1);
-        assert(std::distance(eq.first, eq.second) == 2);
-        std::multiset<std::string> s;
-        s.insert("one");
-        s.insert("four");
-        CheckConsecutiveKeys<C::const_iterator>(c.find(1), c.end(), 1, s);
-        eq = c.equal_range(2);
-        assert(std::distance(eq.first, eq.second) == 2);
-        s.insert("two");
-        s.insert("four");
-        CheckConsecutiveKeys<C::const_iterator>(c.find(2), c.end(), 2, s);
-
-        eq = c.equal_range(3);
-        assert(std::distance(eq.first, eq.second) == 1);
-        C::const_iterator i = eq.first;
-        assert(i->first == 3);
-        assert(i->second == "three");
-        eq = c.equal_range(4);
-        assert(std::distance(eq.first, eq.second) == 1);
-        i = eq.first;
-        assert(i->first == 4);
-        assert(i->second == "four");
-        assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-        assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
-        assert(std::fabs(c.load_factor() - (float)c.size()/c.bucket_count()) < FLT_EPSILON);
-        assert(c.max_load_factor() == 1);
-        assert(c.hash_function() == test_hash<int>(8));
-        assert(c.key_eq() == test_equal_to<int>(9));
-        assert(c.get_allocator() == A{});
-
-        assert(c0.empty());
-    }
-
-    return 0;
+int main(int, char**) {
+  {
+    typedef std::pair<int, std::string> P;
+    typedef test_allocator<std::pair<const int, std::string>> A;
+    typedef std::unordered_multimap<int, std::string, test_hash<int>, test_equal_to<int>, A > C;
+    P a[] = {
+        P(1, "one"),
+        P(2, "two"),
+        P(3, "three"),
+        P(4, "four"),
+        P(1, "four"),
+        P(2, "four"),
+    };
+    C c0(a, a + sizeof(a) / sizeof(a[0]), 7, test_hash<int>(8), test_equal_to<int>(9), A(10));
+    C c(std::move(c0), A(12));
+    assert(c.bucket_count() >= 7);
+    assert(c.size() == 6);
+    typedef std::pair<C::const_iterator, C::const_iterator> Eq;
+    Eq eq = c.equal_range(1);
+    assert(std::distance(eq.first, eq.second) == 2);
+    std::multiset<std::string> s;
+    s.insert("one");
+    s.insert("four");
+    CheckConsecutiveKeys<C::const_iterator>(c.find(1), c.end(), 1, s);
+    eq = c.equal_range(2);
+    assert(std::distance(eq.first, eq.second) == 2);
+    s.insert("two");
+    s.insert("four");
+    CheckConsecutiveKeys<C::const_iterator>(c.find(2), c.end(), 2, s);
+
+    eq = c.equal_range(3);
+    assert(std::distance(eq.first, eq.second) == 1);
+    C::const_iterator i = eq.first;
+    assert(i->first == 3);
+    assert(i->second == "three");
+    eq = c.equal_range(4);
+    assert(std::distance(eq.first, eq.second) == 1);
+    i = eq.first;
+    assert(i->first == 4);
+    assert(i->second == "four");
+    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
+    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
+    assert(std::fabs(c.load_factor() - (float)c.size() / c.bucket_count()) < FLT_EPSILON);
+    assert(c.max_load_factor() == 1);
+    assert(c.hash_function() == test_hash<int>(8));
+    assert(c.key_eq() == test_equal_to<int>(9));
+    assert((c.get_allocator() == test_allocator<std::pair<const int, std::string> >(12)));
+
+    assert(c0.empty());
+  }
+  {
+    typedef std::pair<int, std::string> P;
+    typedef test_allocator<std::pair<const int, std::string>> A;
+    typedef std::unordered_multimap<int, std::string, test_hash<int>, test_equal_to<int>, A > C;
+    P a[] = {
+        P(1, "one"),
+        P(2, "two"),
+        P(3, "three"),
+        P(4, "four"),
+        P(1, "four"),
+        P(2, "four"),
+    };
+    C c0(a, a + sizeof(a) / sizeof(a[0]), 7, test_hash<int>(8), test_equal_to<int>(9), A(10));
+    C c(std::move(c0), A(10));
+    LIBCPP_ASSERT(c.bucket_count() == 7);
+    assert(c.size() == 6);
+    typedef std::pair<C::const_iterator, C::const_iterator> Eq;
+    Eq eq = c.equal_range(1);
+    assert(std::distance(eq.first, eq.second) == 2);
+    std::multiset<std::string> s;
+    s.insert("one");
+    s.insert("four");
+    CheckConsecutiveKeys<C::const_iterator>(c.find(1), c.end(), 1, s);
+    eq = c.equal_range(2);
+    assert(std::distance(eq.first, eq.second) == 2);
+    s.insert("two");
+    s.insert("four");
+    CheckConsecutiveKeys<C::const_iterator>(c.find(2), c.end(), 2, s);
+
+    eq = c.equal_range(3);
+    assert(std::distance(eq.first, eq.second) == 1);
+    C::const_iterator i = eq.first;
+    assert(i->first == 3);
+    assert(i->second == "three");
+    eq = c.equal_range(4);
+    assert(std::distance(eq.first, eq.second) == 1);
+    i = eq.first;
+    assert(i->first == 4);
+    assert(i->second == "four");
+    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
+    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
+    assert(std::fabs(c.load_factor() - (float)c.size() / c.bucket_count()) < FLT_EPSILON);
+    assert(c.max_load_factor() == 1);
+    assert(c.hash_function() == test_hash<int>(8));
+    assert(c.key_eq() == test_equal_to<int>(9));
+    assert((c.get_allocator() == test_allocator<std::pair<const int, std::string> >(10)));
+
+    assert(c0.empty());
+  }
+  {
+    typedef std::pair<int, std::string> P;
+    typedef min_allocator<std::pair<const int, std::string>> A;
+    typedef std::unordered_multimap<int, std::string, test_hash<int>, test_equal_to<int>, A > C;
+    P a[] = {
+        P(1, "one"),
+        P(2, "two"),
+        P(3, "three"),
+        P(4, "four"),
+        P(1, "four"),
+        P(2, "four"),
+    };
+    C c0(a, a + sizeof(a) / sizeof(a[0]), 7, test_hash<int>(8), test_equal_to<int>(9), A());
+    C c(std::move(c0), A());
+    LIBCPP_ASSERT(c.bucket_count() == 7);
+    assert(c.size() == 6);
+    typedef std::pair<C::const_iterator, C::const_iterator> Eq;
+    Eq eq = c.equal_range(1);
+    assert(std::distance(eq.first, eq.second) == 2);
+    std::multiset<std::string> s;
+    s.insert("one");
+    s.insert("four");
+    CheckConsecutiveKeys<C::const_iterator>(c.find(1), c.end(), 1, s);
+    eq = c.equal_range(2);
+    assert(std::distance(eq.first, eq.second) == 2);
+    s.insert("two");
+    s.insert("four");
+    CheckConsecutiveKeys<C::const_iterator>(c.find(2), c.end(), 2, s);
+
+    eq = c.equal_range(3);
+    assert(std::distance(eq.first, eq.second) == 1);
+    C::const_iterator i = eq.first;
+    assert(i->first == 3);
+    assert(i->second == "three");
+    eq = c.equal_range(4);
+    assert(std::distance(eq.first, eq.second) == 1);
+    i = eq.first;
+    assert(i->first == 4);
+    assert(i->second == "four");
+    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
+    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
+    assert(std::fabs(c.load_factor() - (float)c.size() / c.bucket_count()) < FLT_EPSILON);
+    assert(c.max_load_factor() == 1);
+    assert(c.hash_function() == test_hash<int>(8));
+    assert(c.key_eq() == test_equal_to<int>(9));
+    assert((c.get_allocator() == min_allocator<std::pair<const int, std::string> >()));
+
+    assert(c0.empty());
+  }
+  {
+    typedef std::pair<int, std::string> P;
+    typedef explicit_allocator<std::pair<const int, std::string>> A;
+    typedef std::unordered_multimap<int, std::string, test_hash<int>, test_equal_to<int>, A > C;
+    P a[] = {
+        P(1, "one"),
+        P(2, "two"),
+        P(3, "three"),
+        P(4, "four"),
+        P(1, "four"),
+        P(2, "four"),
+    };
+    C c0(a, a + sizeof(a) / sizeof(a[0]), 7, test_hash<int>(8), test_equal_to<int>(9), A{});
+    C c(std::move(c0), A());
+    LIBCPP_ASSERT(c.bucket_count() == 7);
+    assert(c.size() == 6);
+    typedef std::pair<C::const_iterator, C::const_iterator> Eq;
+    Eq eq = c.equal_range(1);
+    assert(std::distance(eq.first, eq.second) == 2);
+    std::multiset<std::string> s;
+    s.insert("one");
+    s.insert("four");
+    CheckConsecutiveKeys<C::const_iterator>(c.find(1), c.end(), 1, s);
+    eq = c.equal_range(2);
+    assert(std::distance(eq.first, eq.second) == 2);
+    s.insert("two");
+    s.insert("four");
+    CheckConsecutiveKeys<C::const_iterator>(c.find(2), c.end(), 2, s);
+
+    eq = c.equal_range(3);
+    assert(std::distance(eq.first, eq.second) == 1);
+    C::const_iterator i = eq.first;
+    assert(i->first == 3);
+    assert(i->second == "three");
+    eq = c.equal_range(4);
+    assert(std::distance(eq.first, eq.second) == 1);
+    i = eq.first;
+    assert(i->first == 4);
+    assert(i->second == "four");
+    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
+    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
+    assert(std::fabs(c.load_factor() - (float)c.size() / c.bucket_count()) < FLT_EPSILON);
+    assert(c.max_load_factor() == 1);
+    assert(c.hash_function() == test_hash<int>(8));
+    assert(c.key_eq() == test_equal_to<int>(9));
+    assert(c.get_allocator() == A{});
+
+    assert(c0.empty());
+  }
+
+  return 0;
 }
diff --git a/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.cnstr/move_assign_noexcept.compile.pass.cpp b/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.cnstr/move_assign_noexcept.compile.pass.cpp
new file mode 100644
index 0000000000000..1420250fad61c
--- /dev/null
+++ b/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.cnstr/move_assign_noexcept.compile.pass.cpp
@@ -0,0 +1,78 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// <unordered_map>
+
+// unordered_multimap& operator=(unordered_multimap&& c)
+//     noexcept(
+//          allocator_type::propagate_on_container_move_assignment::value &&
+//          is_nothrow_move_assignable<allocator_type>::value &&
+//          is_nothrow_move_assignable<key_compare>::value);
+
+// This tests a conforming extension
+
+// UNSUPPORTED: c++03
+
+#include <unordered_map>
+
+#include "test_macros.h"
+#include "MoveOnly.h"
+#include "test_allocator.h"
+
+template <class T>
+struct some_hash {
+  using value_type = T;
+  some_hash();
+  some_hash(const some_hash&);
+  some_hash& operator=(const some_hash&);
+
+  std::size_t operator()(T const&) const;
+};
+
+template <class T>
+struct some_comp {
+  using value_type = T;
+  some_comp& operator=(const some_comp&);
+  bool operator()(const T&, const T&) const { return false; }
+};
+
+template <class T>
+struct always_equal_alloc {
+  using value_type = T;
+  always_equal_alloc(const always_equal_alloc&);
+  void allocate(std::size_t);
+};
+
+template <class T>
+struct not_always_equal_alloc {
+  int i;
+  using value_type = T;
+  not_always_equal_alloc(const not_always_equal_alloc&);
+  void allocate(std::size_t);
+};
+
+template <template <class> class Alloc>
+using unordered_set_alloc =
+    std::unordered_multimap<MoveOnly,
+                            MoveOnly,
+                            std::hash<MoveOnly>,
+                            std::equal_to<MoveOnly>,
+                            Alloc<std::pair<const MoveOnly, MoveOnly>>>;
+
+static_assert(std::is_nothrow_move_assignable<unordered_set_alloc<std::allocator>>::value, "");
+static_assert(!std::is_nothrow_move_assignable<unordered_set_alloc<test_allocator>>::value, "");
+#if TEST_STD_VER >= 17
+static_assert(std::is_nothrow_move_assignable<unordered_set_alloc<always_equal_alloc>>::value, "");
+#endif
+static_assert(!std::is_nothrow_move_assignable<unordered_set_alloc<not_always_equal_alloc>>::value, "");
+#if defined(_LIBCPP_VERSION)
+static_assert(std::is_nothrow_move_assignable<unordered_set_alloc<other_allocator>>::value, "");
+#endif // _LIBCPP_VERSION
+static_assert(!std::is_nothrow_move_assignable<std::unordered_map<int, int, some_hash<int>>>::value, "");
+static_assert(!std::is_nothrow_move_assignable<std::unordered_map<int, int, std::hash<int>, some_comp<int>>>::value,
+              "");
diff --git a/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.cnstr/move_assign_noexcept.pass.cpp b/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.cnstr/move_assign_noexcept.pass.cpp
deleted file mode 100644
index 2cd99546e5bd2..0000000000000
--- a/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.cnstr/move_assign_noexcept.pass.cpp
+++ /dev/null
@@ -1,75 +0,0 @@
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-// <unordered_map>
-
-// unordered_multimap& operator=(unordered_multimap&& c)
-//     noexcept(
-//          allocator_type::propagate_on_container_move_assignment::value &&
-//          is_nothrow_move_assignable<allocator_type>::value &&
-//          is_nothrow_move_assignable<key_compare>::value);
-
-// This tests a conforming extension
-
-// UNSUPPORTED: c++03
-
-#include <unordered_map>
-#include <cassert>
-
-#include "test_macros.h"
-#include "MoveOnly.h"
-#include "test_allocator.h"
-
-template <class T>
-struct some_comp
-{
-    typedef T value_type;
-    some_comp& operator=(const some_comp&);
-    bool operator()(const T&, const T&) const { return false; }
-};
-
-template <class T>
-struct some_hash
-{
-    typedef T value_type;
-    some_hash();
-    some_hash(const some_hash&);
-    some_hash& operator=(const some_hash&);
-    std::size_t operator()(T const&) const;
-};
-
-int main(int, char**)
-{
-    {
-        typedef std::unordered_multimap<MoveOnly, MoveOnly> C;
-        static_assert(std::is_nothrow_move_assignable<C>::value, "");
-    }
-    {
-        typedef std::unordered_multimap<MoveOnly, MoveOnly, std::hash<MoveOnly>,
-                           std::equal_to<MoveOnly>, test_allocator<std::pair<const MoveOnly, MoveOnly>>> C;
-        static_assert(!std::is_nothrow_move_assignable<C>::value, "");
-    }
-#if defined(_LIBCPP_VERSION)
-    {
-        typedef std::unordered_multimap<MoveOnly, MoveOnly, std::hash<MoveOnly>,
-                          std::equal_to<MoveOnly>, other_allocator<std::pair<const MoveOnly, MoveOnly>>> C;
-        static_assert(std::is_nothrow_move_assignable<C>::value, "");
-    }
-#endif // _LIBCPP_VERSION
-    {
-        typedef std::unordered_multimap<MoveOnly, MoveOnly, some_hash<MoveOnly>> C;
-        static_assert(!std::is_nothrow_move_assignable<C>::value, "");
-    }
-    {
-        typedef std::unordered_multimap<MoveOnly, MoveOnly, std::hash<MoveOnly>,
-                                                         some_comp<MoveOnly>> C;
-        static_assert(!std::is_nothrow_move_assignable<C>::value, "");
-    }
-
-  return 0;
-}
diff --git a/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.cnstr/move_noexcept.pass.cpp b/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.cnstr/move_noexcept.pass.cpp
index ae62e66d4071a..229ec7121cf3e 100644
--- a/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.cnstr/move_noexcept.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.cnstr/move_noexcept.pass.cpp
@@ -24,49 +24,53 @@
 #include "test_allocator.h"
 
 template <class T>
-struct some_comp
-{
-    typedef T value_type;
-    some_comp(const some_comp&);
-    bool operator()(const T&, const T&) const { return false; }
+struct some_comp {
+  typedef T value_type;
+  some_comp(const some_comp&);
+  bool operator()(const T&, const T&) const { return false; }
 };
 
 template <class T>
-struct some_hash
-{
-    typedef T value_type;
-    some_hash();
-    some_hash(const some_hash&);
-    std::size_t operator()(T const&) const;
+struct some_hash {
+  typedef T value_type;
+  some_hash();
+  some_hash(const some_hash&);
+  std::size_t operator()(T const&) const;
 };
 
-int main(int, char**)
-{
+int main(int, char**) {
 #if defined(_LIBCPP_VERSION)
-    {
-        typedef std::unordered_multimap<MoveOnly, MoveOnly> C;
-        static_assert(std::is_nothrow_move_constructible<C>::value, "");
-    }
-    {
-        typedef std::unordered_multimap<MoveOnly, MoveOnly, std::hash<MoveOnly>,
-                           std::equal_to<MoveOnly>, test_allocator<std::pair<const MoveOnly, MoveOnly>>> C;
-        static_assert(std::is_nothrow_move_constructible<C>::value, "");
-    }
-    {
-        typedef std::unordered_multimap<MoveOnly, MoveOnly, std::hash<MoveOnly>,
-                          std::equal_to<MoveOnly>, other_allocator<std::pair<const MoveOnly, MoveOnly>>> C;
-        static_assert(std::is_nothrow_move_constructible<C>::value, "");
-    }
+  {
+    typedef std::unordered_multimap<MoveOnly, MoveOnly> C;
+    static_assert(std::is_nothrow_move_constructible<C>::value, "");
+  }
+  {
+    typedef std::unordered_multimap<MoveOnly,
+                                    MoveOnly,
+                                    std::hash<MoveOnly>,
+                                    std::equal_to<MoveOnly>,
+                                    test_allocator<std::pair<const MoveOnly, MoveOnly>>>
+        C;
+    static_assert(std::is_nothrow_move_constructible<C>::value, "");
+  }
+  {
+    typedef std::unordered_multimap<MoveOnly,
+                                    MoveOnly,
+                                    std::hash<MoveOnly>,
+                                    std::equal_to<MoveOnly>,
+                                    other_allocator<std::pair<const MoveOnly, MoveOnly>>>
+        C;
+    static_assert(std::is_nothrow_move_constructible<C>::value, "");
+  }
 #endif // _LIBCPP_VERSION
-    {
-        typedef std::unordered_multimap<MoveOnly, MoveOnly, some_hash<MoveOnly>> C;
-        static_assert(!std::is_nothrow_move_constructible<C>::value, "");
-    }
-    {
-        typedef std::unordered_multimap<MoveOnly, MoveOnly, std::hash<MoveOnly>,
-                                                         some_comp<MoveOnly>> C;
-        static_assert(!std::is_nothrow_move_constructible<C>::value, "");
-    }
+  {
+    typedef std::unordered_multimap<MoveOnly, MoveOnly, some_hash<MoveOnly>> C;
+    static_assert(!std::is_nothrow_move_constructible<C>::value, "");
+  }
+  {
+    typedef std::unordered_multimap<MoveOnly, MoveOnly, std::hash<MoveOnly>, some_comp<MoveOnly>> C;
+    static_assert(!std::is_nothrow_move_constructible<C>::value, "");
+  }
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.cnstr/size.compile.fail.cpp b/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.cnstr/size.compile.fail.cpp
index 810f6a51c72c2..a550d23d55b38 100644
--- a/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.cnstr/size.compile.fail.cpp
+++ b/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.cnstr/size.compile.fail.cpp
@@ -24,47 +24,44 @@
 #include "test_allocator.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef std::unordered_multimap<NotConstructible, NotConstructible,
-                                   test_hash<NotConstructible>,
-                                   test_equal_to<NotConstructible>,
-                                   test_allocator<std::pair<const NotConstructible,
-                                                                  NotConstructible> >
-                                   > C;
-        C c = 7;
-        LIBCPP_ASSERT(c.bucket_count() == 7);
-        assert(c.hash_function() == test_hash<NotConstructible>());
-        assert(c.key_eq() == test_equal_to<NotConstructible>());
-        assert(c.get_allocator() ==
-               (test_allocator<std::pair<const NotConstructible, NotConstructible> >()));
-        assert(c.size() == 0);
-        assert(c.empty());
-        assert(std::distance(c.begin(), c.end()) == 0);
-        assert(c.load_factor() == 0);
-        assert(c.max_load_factor() == 1);
-    }
+int main(int, char**) {
+  {
+    typedef std::unordered_multimap<NotConstructible,
+                                    NotConstructible,
+                                    test_hash<NotConstructible>,
+                                    test_equal_to<NotConstructible>,
+                                    test_allocator<std::pair<const NotConstructible, NotConstructible> > >
+        C;
+    C c = 7;
+    LIBCPP_ASSERT(c.bucket_count() == 7);
+    assert(c.hash_function() == test_hash<NotConstructible>());
+    assert(c.key_eq() == test_equal_to<NotConstructible>());
+    assert(c.get_allocator() == (test_allocator<std::pair<const NotConstructible, NotConstructible> >()));
+    assert(c.size() == 0);
+    assert(c.empty());
+    assert(std::distance(c.begin(), c.end()) == 0);
+    assert(c.load_factor() == 0);
+    assert(c.max_load_factor() == 1);
+  }
 #if TEST_STD_VER >= 11
-    {
-        typedef std::unordered_multimap<NotConstructible, NotConstructible,
-                                   test_hash<NotConstructible>,
-                                   test_equal_to<NotConstructible>,
-                                   min_allocator<std::pair<const NotConstructible,
-                                                                 NotConstructible> >
-                                   > C;
-        C c = 7;
-        LIBCPP_ASSERT(c.bucket_count() == 7);
-        assert(c.hash_function() == test_hash<NotConstructible>());
-        assert(c.key_eq() == test_equal_to<NotConstructible>());
-        assert(c.get_allocator() ==
-               (min_allocator<std::pair<const NotConstructible, NotConstructible> >()));
-        assert(c.size() == 0);
-        assert(c.empty());
-        assert(std::distance(c.begin(), c.end()) == 0);
-        assert(c.load_factor() == 0);
-        assert(c.max_load_factor() == 1);
-    }
+  {
+    typedef std::unordered_multimap<NotConstructible,
+                                    NotConstructible,
+                                    test_hash<NotConstructible>,
+                                    test_equal_to<NotConstructible>,
+                                    min_allocator<std::pair<const NotConstructible, NotConstructible> > >
+        C;
+    C c = 7;
+    LIBCPP_ASSERT(c.bucket_count() == 7);
+    assert(c.hash_function() == test_hash<NotConstructible>());
+    assert(c.key_eq() == test_equal_to<NotConstructible>());
+    assert(c.get_allocator() == (min_allocator<std::pair<const NotConstructible, NotConstructible> >()));
+    assert(c.size() == 0);
+    assert(c.empty());
+    assert(std::distance(c.begin(), c.end()) == 0);
+    assert(c.load_factor() == 0);
+    assert(c.max_load_factor() == 1);
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.cnstr/size.pass.cpp b/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.cnstr/size.pass.cpp
index fa87144f1692f..20f09424489ad 100644
--- a/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.cnstr/size.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.cnstr/size.pass.cpp
@@ -24,47 +24,44 @@
 #include "test_allocator.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef std::unordered_multimap<NotConstructible, NotConstructible,
-                                   test_hash<NotConstructible>,
-                                   test_equal_to<NotConstructible>,
-                                   test_allocator<std::pair<const NotConstructible,
-                                                                  NotConstructible> >
-                                   > C;
-        C c(7);
-        LIBCPP_ASSERT(c.bucket_count() == 7);
-        assert(c.hash_function() == test_hash<NotConstructible>());
-        assert(c.key_eq() == test_equal_to<NotConstructible>());
-        assert(c.get_allocator() ==
-               (test_allocator<std::pair<const NotConstructible, NotConstructible> >()));
-        assert(c.size() == 0);
-        assert(c.empty());
-        assert(std::distance(c.begin(), c.end()) == 0);
-        assert(c.load_factor() == 0);
-        assert(c.max_load_factor() == 1);
-    }
+int main(int, char**) {
+  {
+    typedef std::unordered_multimap<NotConstructible,
+                                    NotConstructible,
+                                    test_hash<NotConstructible>,
+                                    test_equal_to<NotConstructible>,
+                                    test_allocator<std::pair<const NotConstructible, NotConstructible> > >
+        C;
+    C c(7);
+    LIBCPP_ASSERT(c.bucket_count() == 7);
+    assert(c.hash_function() == test_hash<NotConstructible>());
+    assert(c.key_eq() == test_equal_to<NotConstructible>());
+    assert(c.get_allocator() == (test_allocator<std::pair<const NotConstructible, NotConstructible> >()));
+    assert(c.size() == 0);
+    assert(c.empty());
+    assert(std::distance(c.begin(), c.end()) == 0);
+    assert(c.load_factor() == 0);
+    assert(c.max_load_factor() == 1);
+  }
 #if TEST_STD_VER >= 11
-    {
-        typedef std::unordered_multimap<NotConstructible, NotConstructible,
-                                   test_hash<NotConstructible>,
-                                   test_equal_to<NotConstructible>,
-                                   min_allocator<std::pair<const NotConstructible,
-                                                                 NotConstructible> >
-                                   > C;
-        C c(7);
-        LIBCPP_ASSERT(c.bucket_count() == 7);
-        assert(c.hash_function() == test_hash<NotConstructible>());
-        assert(c.key_eq() == test_equal_to<NotConstructible>());
-        assert(c.get_allocator() ==
-               (min_allocator<std::pair<const NotConstructible, NotConstructible> >()));
-        assert(c.size() == 0);
-        assert(c.empty());
-        assert(std::distance(c.begin(), c.end()) == 0);
-        assert(c.load_factor() == 0);
-        assert(c.max_load_factor() == 1);
-    }
+  {
+    typedef std::unordered_multimap<NotConstructible,
+                                    NotConstructible,
+                                    test_hash<NotConstructible>,
+                                    test_equal_to<NotConstructible>,
+                                    min_allocator<std::pair<const NotConstructible, NotConstructible> > >
+        C;
+    C c(7);
+    LIBCPP_ASSERT(c.bucket_count() == 7);
+    assert(c.hash_function() == test_hash<NotConstructible>());
+    assert(c.key_eq() == test_equal_to<NotConstructible>());
+    assert(c.get_allocator() == (min_allocator<std::pair<const NotConstructible, NotConstructible> >()));
+    assert(c.size() == 0);
+    assert(c.empty());
+    assert(std::distance(c.begin(), c.end()) == 0);
+    assert(c.load_factor() == 0);
+    assert(c.max_load_factor() == 1);
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.cnstr/size_allocator.pass.cpp b/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.cnstr/size_allocator.pass.cpp
index b90f6db7913b8..d49be5518922a 100644
--- a/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.cnstr/size_allocator.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.cnstr/size_allocator.pass.cpp
@@ -28,31 +28,30 @@
 #include "min_allocator.h"
 
 template <class Allocator>
-void test(const Allocator& alloc)
-{
-    typedef std::unordered_multimap<NotConstructible, NotConstructible,
-                                    test_hash<NotConstructible>,
-                                    test_equal_to<NotConstructible>,
-                                    Allocator> C;
-    C c(7,
-        alloc);
-    LIBCPP_ASSERT(c.bucket_count() == 7);
-    assert(c.hash_function() == test_hash<NotConstructible>());
-    assert(c.key_eq() == test_equal_to<NotConstructible>());
-    assert(c.get_allocator() == alloc);
-    assert(c.size() == 0);
-    assert(c.empty());
-    assert(std::distance(c.begin(), c.end()) == 0);
-    assert(c.load_factor() == 0);
-    assert(c.max_load_factor() == 1);
+void test(const Allocator& alloc) {
+  typedef std::unordered_multimap<NotConstructible,
+                                  NotConstructible,
+                                  test_hash<NotConstructible>,
+                                  test_equal_to<NotConstructible>,
+                                  Allocator>
+      C;
+  C c(7, alloc);
+  LIBCPP_ASSERT(c.bucket_count() == 7);
+  assert(c.hash_function() == test_hash<NotConstructible>());
+  assert(c.key_eq() == test_equal_to<NotConstructible>());
+  assert(c.get_allocator() == alloc);
+  assert(c.size() == 0);
+  assert(c.empty());
+  assert(std::distance(c.begin(), c.end()) == 0);
+  assert(c.load_factor() == 0);
+  assert(c.max_load_factor() == 1);
 }
 
-int main(int, char**)
-{
-    typedef std::pair<const NotConstructible, NotConstructible> V;
-    test(test_allocator<V>(10));
-    test(min_allocator<V>());
-    test(explicit_allocator<V>());
+int main(int, char**) {
+  typedef std::pair<const NotConstructible, NotConstructible> V;
+  test(test_allocator<V>(10));
+  test(min_allocator<V>());
+  test(explicit_allocator<V>());
 
-    return 0;
+  return 0;
 }
diff --git a/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.cnstr/size_hash.pass.cpp b/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.cnstr/size_hash.pass.cpp
index 70bab4197449b..99575cc8cadf9 100644
--- a/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.cnstr/size_hash.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.cnstr/size_hash.pass.cpp
@@ -24,51 +24,44 @@
 #include "test_allocator.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef std::unordered_multimap<NotConstructible, NotConstructible,
-                                   test_hash<NotConstructible>,
-                                   test_equal_to<NotConstructible>,
-                                   test_allocator<std::pair<const NotConstructible,
-                                                                  NotConstructible> >
-                                   > C;
-        C c(7,
-            test_hash<NotConstructible>(8)
-           );
-        LIBCPP_ASSERT(c.bucket_count() == 7);
-        assert(c.hash_function() == test_hash<NotConstructible>(8));
-        assert(c.key_eq() == test_equal_to<NotConstructible>());
-        assert(c.get_allocator() ==
-               (test_allocator<std::pair<const NotConstructible, NotConstructible> >()));
-        assert(c.size() == 0);
-        assert(c.empty());
-        assert(std::distance(c.begin(), c.end()) == 0);
-        assert(c.load_factor() == 0);
-        assert(c.max_load_factor() == 1);
-    }
+int main(int, char**) {
+  {
+    typedef std::unordered_multimap<NotConstructible,
+                                    NotConstructible,
+                                    test_hash<NotConstructible>,
+                                    test_equal_to<NotConstructible>,
+                                    test_allocator<std::pair<const NotConstructible, NotConstructible> > >
+        C;
+    C c(7, test_hash<NotConstructible>(8));
+    LIBCPP_ASSERT(c.bucket_count() == 7);
+    assert(c.hash_function() == test_hash<NotConstructible>(8));
+    assert(c.key_eq() == test_equal_to<NotConstructible>());
+    assert(c.get_allocator() == (test_allocator<std::pair<const NotConstructible, NotConstructible> >()));
+    assert(c.size() == 0);
+    assert(c.empty());
+    assert(std::distance(c.begin(), c.end()) == 0);
+    assert(c.load_factor() == 0);
+    assert(c.max_load_factor() == 1);
+  }
 #if TEST_STD_VER >= 11
-    {
-        typedef std::unordered_multimap<NotConstructible, NotConstructible,
-                                   test_hash<NotConstructible>,
-                                   test_equal_to<NotConstructible>,
-                                   min_allocator<std::pair<const NotConstructible,
-                                                                 NotConstructible> >
-                                   > C;
-        C c(7,
-            test_hash<NotConstructible>(8)
-           );
-        LIBCPP_ASSERT(c.bucket_count() == 7);
-        assert(c.hash_function() == test_hash<NotConstructible>(8));
-        assert(c.key_eq() == test_equal_to<NotConstructible>());
-        assert(c.get_allocator() ==
-               (min_allocator<std::pair<const NotConstructible, NotConstructible> >()));
-        assert(c.size() == 0);
-        assert(c.empty());
-        assert(std::distance(c.begin(), c.end()) == 0);
-        assert(c.load_factor() == 0);
-        assert(c.max_load_factor() == 1);
-    }
+  {
+    typedef std::unordered_multimap<NotConstructible,
+                                    NotConstructible,
+                                    test_hash<NotConstructible>,
+                                    test_equal_to<NotConstructible>,
+                                    min_allocator<std::pair<const NotConstructible, NotConstructible> > >
+        C;
+    C c(7, test_hash<NotConstructible>(8));
+    LIBCPP_ASSERT(c.bucket_count() == 7);
+    assert(c.hash_function() == test_hash<NotConstructible>(8));
+    assert(c.key_eq() == test_equal_to<NotConstructible>());
+    assert(c.get_allocator() == (min_allocator<std::pair<const NotConstructible, NotConstructible> >()));
+    assert(c.size() == 0);
+    assert(c.empty());
+    assert(std::distance(c.begin(), c.end()) == 0);
+    assert(c.load_factor() == 0);
+    assert(c.max_load_factor() == 1);
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.cnstr/size_hash_allocator.pass.cpp b/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.cnstr/size_hash_allocator.pass.cpp
index ad98e99558d48..f26fae2a2a81d 100644
--- a/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.cnstr/size_hash_allocator.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.cnstr/size_hash_allocator.pass.cpp
@@ -28,32 +28,30 @@
 #include "min_allocator.h"
 
 template <class Allocator>
-void test(const Allocator& alloc)
-{
-    typedef std::unordered_multimap<NotConstructible, NotConstructible,
-                                    test_hash<NotConstructible>,
-                                    test_equal_to<NotConstructible>,
-                                    Allocator> C;
-    C c(7,
-        test_hash<NotConstructible>(8),
-        alloc);
-    LIBCPP_ASSERT(c.bucket_count() == 7);
-    assert(c.hash_function() == test_hash<NotConstructible>(8));
-    assert(c.key_eq() == test_equal_to<NotConstructible>());
-    assert(c.get_allocator() == alloc);
-    assert(c.size() == 0);
-    assert(c.empty());
-    assert(std::distance(c.begin(), c.end()) == 0);
-    assert(c.load_factor() == 0);
-    assert(c.max_load_factor() == 1);
+void test(const Allocator& alloc) {
+  typedef std::unordered_multimap<NotConstructible,
+                                  NotConstructible,
+                                  test_hash<NotConstructible>,
+                                  test_equal_to<NotConstructible>,
+                                  Allocator>
+      C;
+  C c(7, test_hash<NotConstructible>(8), alloc);
+  LIBCPP_ASSERT(c.bucket_count() == 7);
+  assert(c.hash_function() == test_hash<NotConstructible>(8));
+  assert(c.key_eq() == test_equal_to<NotConstructible>());
+  assert(c.get_allocator() == alloc);
+  assert(c.size() == 0);
+  assert(c.empty());
+  assert(std::distance(c.begin(), c.end()) == 0);
+  assert(c.load_factor() == 0);
+  assert(c.max_load_factor() == 1);
 }
 
-int main(int, char**)
-{
-    typedef std::pair<const NotConstructible, NotConstructible> V;
-    test(test_allocator<V>(10));
-    test(min_allocator<V>());
-    test(explicit_allocator<V>());
+int main(int, char**) {
+  typedef std::pair<const NotConstructible, NotConstructible> V;
+  test(test_allocator<V>(10));
+  test(min_allocator<V>());
+  test(explicit_allocator<V>());
 
-    return 0;
+  return 0;
 }
diff --git a/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.cnstr/size_hash_equal.pass.cpp b/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.cnstr/size_hash_equal.pass.cpp
index 7709dfe9452f7..8cd804d1b401c 100644
--- a/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.cnstr/size_hash_equal.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.cnstr/size_hash_equal.pass.cpp
@@ -24,53 +24,44 @@
 #include "test_allocator.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef std::unordered_multimap<NotConstructible, NotConstructible,
-                                   test_hash<NotConstructible>,
-                                   test_equal_to<NotConstructible>,
-                                   test_allocator<std::pair<const NotConstructible,
-                                                                  NotConstructible> >
-                                   > C;
-        C c(7,
-            test_hash<NotConstructible>(8),
-            test_equal_to<NotConstructible>(9)
-           );
-        LIBCPP_ASSERT(c.bucket_count() == 7);
-        assert(c.hash_function() == test_hash<NotConstructible>(8));
-        assert(c.key_eq() == test_equal_to<NotConstructible>(9));
-        assert(c.get_allocator() ==
-               (test_allocator<std::pair<const NotConstructible, NotConstructible> >()));
-        assert(c.size() == 0);
-        assert(c.empty());
-        assert(std::distance(c.begin(), c.end()) == 0);
-        assert(c.load_factor() == 0);
-        assert(c.max_load_factor() == 1);
-    }
+int main(int, char**) {
+  {
+    typedef std::unordered_multimap<NotConstructible,
+                                    NotConstructible,
+                                    test_hash<NotConstructible>,
+                                    test_equal_to<NotConstructible>,
+                                    test_allocator<std::pair<const NotConstructible, NotConstructible> > >
+        C;
+    C c(7, test_hash<NotConstructible>(8), test_equal_to<NotConstructible>(9));
+    LIBCPP_ASSERT(c.bucket_count() == 7);
+    assert(c.hash_function() == test_hash<NotConstructible>(8));
+    assert(c.key_eq() == test_equal_to<NotConstructible>(9));
+    assert(c.get_allocator() == (test_allocator<std::pair<const NotConstructible, NotConstructible> >()));
+    assert(c.size() == 0);
+    assert(c.empty());
+    assert(std::distance(c.begin(), c.end()) == 0);
+    assert(c.load_factor() == 0);
+    assert(c.max_load_factor() == 1);
+  }
 #if TEST_STD_VER >= 11
-    {
-        typedef std::unordered_multimap<NotConstructible, NotConstructible,
-                                   test_hash<NotConstructible>,
-                                   test_equal_to<NotConstructible>,
-                                   min_allocator<std::pair<const NotConstructible,
-                                                                 NotConstructible> >
-                                   > C;
-        C c(7,
-            test_hash<NotConstructible>(8),
-            test_equal_to<NotConstructible>(9)
-           );
-        LIBCPP_ASSERT(c.bucket_count() == 7);
-        assert(c.hash_function() == test_hash<NotConstructible>(8));
-        assert(c.key_eq() == test_equal_to<NotConstructible>(9));
-        assert(c.get_allocator() ==
-               (min_allocator<std::pair<const NotConstructible, NotConstructible> >()));
-        assert(c.size() == 0);
-        assert(c.empty());
-        assert(std::distance(c.begin(), c.end()) == 0);
-        assert(c.load_factor() == 0);
-        assert(c.max_load_factor() == 1);
-    }
+  {
+    typedef std::unordered_multimap<NotConstructible,
+                                    NotConstructible,
+                                    test_hash<NotConstructible>,
+                                    test_equal_to<NotConstructible>,
+                                    min_allocator<std::pair<const NotConstructible, NotConstructible> > >
+        C;
+    C c(7, test_hash<NotConstructible>(8), test_equal_to<NotConstructible>(9));
+    LIBCPP_ASSERT(c.bucket_count() == 7);
+    assert(c.hash_function() == test_hash<NotConstructible>(8));
+    assert(c.key_eq() == test_equal_to<NotConstructible>(9));
+    assert(c.get_allocator() == (min_allocator<std::pair<const NotConstructible, NotConstructible> >()));
+    assert(c.size() == 0);
+    assert(c.empty());
+    assert(std::distance(c.begin(), c.end()) == 0);
+    assert(c.load_factor() == 0);
+    assert(c.max_load_factor() == 1);
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.cnstr/size_hash_equal_allocator.pass.cpp b/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.cnstr/size_hash_equal_allocator.pass.cpp
index 1fd4bb60a5dbe..64298cd610c14 100644
--- a/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.cnstr/size_hash_equal_allocator.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.cnstr/size_hash_equal_allocator.pass.cpp
@@ -24,77 +24,69 @@
 #include "test_allocator.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef std::unordered_multimap<NotConstructible, NotConstructible,
-                                   test_hash<NotConstructible>,
-                                   test_equal_to<NotConstructible>,
-                                   test_allocator<std::pair<const NotConstructible,
-                                                                  NotConstructible> >
-                                   > C;
-        C c(7,
-            test_hash<NotConstructible>(8),
-            test_equal_to<NotConstructible>(9),
-            test_allocator<std::pair<const NotConstructible, NotConstructible> >(10)
-           );
-        LIBCPP_ASSERT(c.bucket_count() == 7);
-        assert(c.hash_function() == test_hash<NotConstructible>(8));
-        assert(c.key_eq() == test_equal_to<NotConstructible>(9));
-        assert(c.get_allocator() ==
-               (test_allocator<std::pair<const NotConstructible, NotConstructible> >(10)));
-        assert(c.size() == 0);
-        assert(c.empty());
-        assert(std::distance(c.begin(), c.end()) == 0);
-        assert(c.load_factor() == 0);
-        assert(c.max_load_factor() == 1);
-    }
+int main(int, char**) {
+  {
+    typedef std::unordered_multimap<NotConstructible,
+                                    NotConstructible,
+                                    test_hash<NotConstructible>,
+                                    test_equal_to<NotConstructible>,
+                                    test_allocator<std::pair<const NotConstructible, NotConstructible> > >
+        C;
+    C c(7,
+        test_hash<NotConstructible>(8),
+        test_equal_to<NotConstructible>(9),
+        test_allocator<std::pair<const NotConstructible, NotConstructible> >(10));
+    LIBCPP_ASSERT(c.bucket_count() == 7);
+    assert(c.hash_function() == test_hash<NotConstructible>(8));
+    assert(c.key_eq() == test_equal_to<NotConstructible>(9));
+    assert(c.get_allocator() == (test_allocator<std::pair<const NotConstructible, NotConstructible> >(10)));
+    assert(c.size() == 0);
+    assert(c.empty());
+    assert(std::distance(c.begin(), c.end()) == 0);
+    assert(c.load_factor() == 0);
+    assert(c.max_load_factor() == 1);
+  }
 #if TEST_STD_VER >= 11
-    {
-        typedef std::unordered_multimap<NotConstructible, NotConstructible,
-                                   test_hash<NotConstructible>,
-                                   test_equal_to<NotConstructible>,
-                                   min_allocator<std::pair<const NotConstructible,
-                                                                 NotConstructible> >
-                                   > C;
-        C c(7,
-            test_hash<NotConstructible>(8),
-            test_equal_to<NotConstructible>(9),
-            min_allocator<std::pair<const NotConstructible, NotConstructible> >()
-           );
-        LIBCPP_ASSERT(c.bucket_count() == 7);
-        assert(c.hash_function() == test_hash<NotConstructible>(8));
-        assert(c.key_eq() == test_equal_to<NotConstructible>(9));
-        assert(c.get_allocator() ==
-               (min_allocator<std::pair<const NotConstructible, NotConstructible> >()));
-        assert(c.size() == 0);
-        assert(c.empty());
-        assert(std::distance(c.begin(), c.end()) == 0);
-        assert(c.load_factor() == 0);
-        assert(c.max_load_factor() == 1);
-    }
-    {
-        typedef explicit_allocator<std::pair<const NotConstructible, NotConstructible> > A;
-        typedef std::unordered_multimap<NotConstructible, NotConstructible,
-                                   test_hash<NotConstructible>,
-                                   test_equal_to<NotConstructible>,
-                                   A
-                                   > C;
-        C c(7,
-            test_hash<NotConstructible>(8),
-            test_equal_to<NotConstructible>(9),
-            A{}
-           );
-        LIBCPP_ASSERT(c.bucket_count() == 7);
-        assert(c.hash_function() == test_hash<NotConstructible>(8));
-        assert(c.key_eq() == test_equal_to<NotConstructible>(9));
-        assert(c.get_allocator() == A{});
-        assert(c.size() == 0);
-        assert(c.empty());
-        assert(std::distance(c.begin(), c.end()) == 0);
-        assert(c.load_factor() == 0);
-        assert(c.max_load_factor() == 1);
-    }
+  {
+    typedef std::unordered_multimap<NotConstructible,
+                                    NotConstructible,
+                                    test_hash<NotConstructible>,
+                                    test_equal_to<NotConstructible>,
+                                    min_allocator<std::pair<const NotConstructible, NotConstructible> > >
+        C;
+    C c(7,
+        test_hash<NotConstructible>(8),
+        test_equal_to<NotConstructible>(9),
+        min_allocator<std::pair<const NotConstructible, NotConstructible> >());
+    LIBCPP_ASSERT(c.bucket_count() == 7);
+    assert(c.hash_function() == test_hash<NotConstructible>(8));
+    assert(c.key_eq() == test_equal_to<NotConstructible>(9));
+    assert(c.get_allocator() == (min_allocator<std::pair<const NotConstructible, NotConstructible> >()));
+    assert(c.size() == 0);
+    assert(c.empty());
+    assert(std::distance(c.begin(), c.end()) == 0);
+    assert(c.load_factor() == 0);
+    assert(c.max_load_factor() == 1);
+  }
+  {
+    typedef explicit_allocator<std::pair<const NotConstructible, NotConstructible> > A;
+    typedef std::unordered_multimap<NotConstructible,
+                                    NotConstructible,
+                                    test_hash<NotConstructible>,
+                                    test_equal_to<NotConstructible>,
+                                    A >
+        C;
+    C c(7, test_hash<NotConstructible>(8), test_equal_to<NotConstructible>(9), A{});
+    LIBCPP_ASSERT(c.bucket_count() == 7);
+    assert(c.hash_function() == test_hash<NotConstructible>(8));
+    assert(c.key_eq() == test_equal_to<NotConstructible>(9));
+    assert(c.get_allocator() == A{});
+    assert(c.size() == 0);
+    assert(c.empty());
+    assert(std::distance(c.begin(), c.end()) == 0);
+    assert(c.load_factor() == 0);
+    assert(c.max_load_factor() == 1);
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.modifiers/clear.pass.cpp b/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.modifiers/clear.pass.cpp
index 71c3c57c215fe..4e69b19cf2c10 100644
--- a/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.modifiers/clear.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.modifiers/clear.pass.cpp
@@ -21,44 +21,45 @@
 #include "test_macros.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef std::unordered_multimap<int, std::string> C;
-        typedef std::pair<int, std::string> P;
-        P a[] =
-        {
-            P(1, "one"),
-            P(2, "two"),
-            P(3, "three"),
-            P(4, "four"),
-            P(1, "four"),
-            P(2, "four"),
-        };
-        C c(a, a + sizeof(a)/sizeof(a[0]));
-        ASSERT_NOEXCEPT(c.clear());
-        c.clear();
-        assert(c.size() == 0);
-    }
+int main(int, char**) {
+  {
+    typedef std::unordered_multimap<int, std::string> C;
+    typedef std::pair<int, std::string> P;
+    P a[] = {
+        P(1, "one"),
+        P(2, "two"),
+        P(3, "three"),
+        P(4, "four"),
+        P(1, "four"),
+        P(2, "four"),
+    };
+    C c(a, a + sizeof(a) / sizeof(a[0]));
+    ASSERT_NOEXCEPT(c.clear());
+    c.clear();
+    assert(c.size() == 0);
+  }
 #if TEST_STD_VER >= 11
-    {
-        typedef std::unordered_multimap<int, std::string, std::hash<int>, std::equal_to<int>,
-                            min_allocator<std::pair<const int, std::string>>> C;
-        typedef std::pair<int, std::string> P;
-        P a[] =
-        {
-            P(1, "one"),
-            P(2, "two"),
-            P(3, "three"),
-            P(4, "four"),
-            P(1, "four"),
-            P(2, "four"),
-        };
-        C c(a, a + sizeof(a)/sizeof(a[0]));
-        ASSERT_NOEXCEPT(c.clear());
-        c.clear();
-        assert(c.size() == 0);
-    }
+  {
+    typedef std::unordered_multimap<int,
+                                    std::string,
+                                    std::hash<int>,
+                                    std::equal_to<int>,
+                                    min_allocator<std::pair<const int, std::string>>>
+        C;
+    typedef std::pair<int, std::string> P;
+    P a[] = {
+        P(1, "one"),
+        P(2, "two"),
+        P(3, "three"),
+        P(4, "four"),
+        P(1, "four"),
+        P(2, "four"),
+    };
+    C c(a, a + sizeof(a) / sizeof(a[0]));
+    ASSERT_NOEXCEPT(c.clear());
+    c.clear();
+    assert(c.size() == 0);
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.modifiers/emplace.pass.cpp b/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.modifiers/emplace.pass.cpp
index 1f32893f47add..e3c1b6e2450e4 100644
--- a/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.modifiers/emplace.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.modifiers/emplace.pass.cpp
@@ -24,51 +24,50 @@
 #include "../../../Emplaceable.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef std::unordered_multimap<int, Emplaceable> C;
-        typedef C::iterator R;
-        C c;
-        R r = c.emplace(std::piecewise_construct, std::forward_as_tuple(3),
-                                                  std::forward_as_tuple());
-        assert(c.size() == 1);
-        assert(r->first == 3);
-        assert(r->second == Emplaceable());
+int main(int, char**) {
+  {
+    typedef std::unordered_multimap<int, Emplaceable> C;
+    typedef C::iterator R;
+    C c;
+    R r = c.emplace(std::piecewise_construct, std::forward_as_tuple(3), std::forward_as_tuple());
+    assert(c.size() == 1);
+    assert(r->first == 3);
+    assert(r->second == Emplaceable());
 
-        r = c.emplace(std::pair<const int, Emplaceable>(4, Emplaceable(5, 6)));
-        assert(c.size() == 2);
-        assert(r->first == 4);
-        assert(r->second == Emplaceable(5, 6));
+    r = c.emplace(std::pair<const int, Emplaceable>(4, Emplaceable(5, 6)));
+    assert(c.size() == 2);
+    assert(r->first == 4);
+    assert(r->second == Emplaceable(5, 6));
 
-        r = c.emplace(std::piecewise_construct, std::forward_as_tuple(5),
-                                                std::forward_as_tuple(6, 7));
-        assert(c.size() == 3);
-        assert(r->first == 5);
-        assert(r->second == Emplaceable(6, 7));
-    }
-    {
-        typedef std::unordered_multimap<int, Emplaceable, std::hash<int>, std::equal_to<int>,
-                            min_allocator<std::pair<const int, Emplaceable>>> C;
-        typedef C::iterator R;
-        C c;
-        R r = c.emplace(std::piecewise_construct, std::forward_as_tuple(3),
-                                                  std::forward_as_tuple());
-        assert(c.size() == 1);
-        assert(r->first == 3);
-        assert(r->second == Emplaceable());
+    r = c.emplace(std::piecewise_construct, std::forward_as_tuple(5), std::forward_as_tuple(6, 7));
+    assert(c.size() == 3);
+    assert(r->first == 5);
+    assert(r->second == Emplaceable(6, 7));
+  }
+  {
+    typedef std::unordered_multimap<int,
+                                    Emplaceable,
+                                    std::hash<int>,
+                                    std::equal_to<int>,
+                                    min_allocator<std::pair<const int, Emplaceable>>>
+        C;
+    typedef C::iterator R;
+    C c;
+    R r = c.emplace(std::piecewise_construct, std::forward_as_tuple(3), std::forward_as_tuple());
+    assert(c.size() == 1);
+    assert(r->first == 3);
+    assert(r->second == Emplaceable());
 
-        r = c.emplace(std::pair<const int, Emplaceable>(4, Emplaceable(5, 6)));
-        assert(c.size() == 2);
-        assert(r->first == 4);
-        assert(r->second == Emplaceable(5, 6));
+    r = c.emplace(std::pair<const int, Emplaceable>(4, Emplaceable(5, 6)));
+    assert(c.size() == 2);
+    assert(r->first == 4);
+    assert(r->second == Emplaceable(5, 6));
 
-        r = c.emplace(std::piecewise_construct, std::forward_as_tuple(5),
-                                                std::forward_as_tuple(6, 7));
-        assert(c.size() == 3);
-        assert(r->first == 5);
-        assert(r->second == Emplaceable(6, 7));
-    }
+    r = c.emplace(std::piecewise_construct, std::forward_as_tuple(5), std::forward_as_tuple(6, 7));
+    assert(c.size() == 3);
+    assert(r->first == 5);
+    assert(r->second == Emplaceable(6, 7));
+  }
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.modifiers/emplace_hint.pass.cpp b/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.modifiers/emplace_hint.pass.cpp
index 2a6e3f60ca6b5..1d0014275bf8c 100644
--- a/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.modifiers/emplace_hint.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.modifiers/emplace_hint.pass.cpp
@@ -17,7 +17,6 @@
 // template <class... Args>
 //     iterator emplace_hint(const_iterator p, Args&&... args);
 
-
 #include <unordered_map>
 #include <cassert>
 #include <iterator>
@@ -26,69 +25,68 @@
 #include "min_allocator.h"
 #include "test_macros.h"
 
-int main(int, char**)
-{
-    {
-        typedef std::unordered_multimap<int, Emplaceable> C;
-        typedef C::iterator R;
-        C c;
-        C::const_iterator e = c.end();
-        R r = c.emplace_hint(e, std::piecewise_construct, std::forward_as_tuple(3),
-                                                          std::forward_as_tuple());
-        assert(c.size() == 1);
-        assert(r->first == 3);
-        assert(r->second == Emplaceable());
+int main(int, char**) {
+  {
+    typedef std::unordered_multimap<int, Emplaceable> C;
+    typedef C::iterator R;
+    C c;
+    C::const_iterator e = c.end();
+    R r = c.emplace_hint(e, std::piecewise_construct, std::forward_as_tuple(3), std::forward_as_tuple());
+    assert(c.size() == 1);
+    assert(r->first == 3);
+    assert(r->second == Emplaceable());
 
-        r = c.emplace_hint(c.end(), std::pair<const int, Emplaceable>(3, Emplaceable(5, 6)));
-        assert(c.size() == 2);
-        assert(r->first == 3);
-        assert(r->second == Emplaceable(5, 6));
-        LIBCPP_ASSERT(r == std::next(c.begin()));
+    r = c.emplace_hint(c.end(), std::pair<const int, Emplaceable>(3, Emplaceable(5, 6)));
+    assert(c.size() == 2);
+    assert(r->first == 3);
+    assert(r->second == Emplaceable(5, 6));
+    LIBCPP_ASSERT(r == std::next(c.begin()));
 
-        r = c.emplace_hint(r, std::piecewise_construct, std::forward_as_tuple(3),
-                                                        std::forward_as_tuple(6, 7));
-        assert(c.size() == 3);
-        assert(r->first == 3);
-        assert(r->second == Emplaceable(6, 7));
-        LIBCPP_ASSERT(r == std::next(c.begin()));
-        r = c.begin();
-        assert(r->first == 3);
-        LIBCPP_ASSERT(r->second == Emplaceable());
-        r = std::next(r, 2);
-        assert(r->first == 3);
-        LIBCPP_ASSERT(r->second == Emplaceable(5, 6));
-    }
-    {
-        typedef std::unordered_multimap<int, Emplaceable, std::hash<int>, std::equal_to<int>,
-                            min_allocator<std::pair<const int, Emplaceable>>> C;
-        typedef C::iterator R;
-        C c;
-        C::const_iterator e = c.end();
-        R r = c.emplace_hint(e, std::piecewise_construct, std::forward_as_tuple(3),
-                                                          std::forward_as_tuple());
-        assert(c.size() == 1);
-        assert(r->first == 3);
-        assert(r->second == Emplaceable());
+    r = c.emplace_hint(r, std::piecewise_construct, std::forward_as_tuple(3), std::forward_as_tuple(6, 7));
+    assert(c.size() == 3);
+    assert(r->first == 3);
+    assert(r->second == Emplaceable(6, 7));
+    LIBCPP_ASSERT(r == std::next(c.begin()));
+    r = c.begin();
+    assert(r->first == 3);
+    LIBCPP_ASSERT(r->second == Emplaceable());
+    r = std::next(r, 2);
+    assert(r->first == 3);
+    LIBCPP_ASSERT(r->second == Emplaceable(5, 6));
+  }
+  {
+    typedef std::unordered_multimap<int,
+                                    Emplaceable,
+                                    std::hash<int>,
+                                    std::equal_to<int>,
+                                    min_allocator<std::pair<const int, Emplaceable>>>
+        C;
+    typedef C::iterator R;
+    C c;
+    C::const_iterator e = c.end();
+    R r = c.emplace_hint(e, std::piecewise_construct, std::forward_as_tuple(3), std::forward_as_tuple());
+    assert(c.size() == 1);
+    assert(r->first == 3);
+    assert(r->second == Emplaceable());
 
-        r = c.emplace_hint(c.end(), std::pair<const int, Emplaceable>(3, Emplaceable(5, 6)));
-        assert(c.size() == 2);
-        assert(r->first == 3);
-        assert(r->second == Emplaceable(5, 6));
-        LIBCPP_ASSERT(r == std::next(c.begin()));
+    r = c.emplace_hint(c.end(), std::pair<const int, Emplaceable>(3, Emplaceable(5, 6)));
+    assert(c.size() == 2);
+    assert(r->first == 3);
+    assert(r->second == Emplaceable(5, 6));
+    LIBCPP_ASSERT(r == std::next(c.begin()));
 
-        r = c.emplace_hint(r, std::piecewise_construct, std::forward_as_tuple(3),
-                                                        std::forward_as_tuple(6, 7));
-        assert(c.size() == 3);
-        assert(r->first == 3);
-        assert(r->second == Emplaceable(6, 7));
-        LIBCPP_ASSERT(r == std::next(c.begin()));
-        r = c.begin();
-        assert(r->first == 3);
-        LIBCPP_ASSERT(r->second == Emplaceable());
-        r = std::next(r, 2);
-        assert(r->first == 3);
-        LIBCPP_ASSERT(r->second == Emplaceable(5, 6));
-    }
+    r = c.emplace_hint(r, std::piecewise_construct, std::forward_as_tuple(3), std::forward_as_tuple(6, 7));
+    assert(c.size() == 3);
+    assert(r->first == 3);
+    assert(r->second == Emplaceable(6, 7));
+    LIBCPP_ASSERT(r == std::next(c.begin()));
+    r = c.begin();
+    assert(r->first == 3);
+    LIBCPP_ASSERT(r->second == Emplaceable());
+    r = std::next(r, 2);
+    assert(r->first == 3);
+    LIBCPP_ASSERT(r->second == Emplaceable(5, 6));
+  }
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.modifiers/erase_const_iter.pass.cpp b/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.modifiers/erase_const_iter.pass.cpp
index 24e9249bd78b2..5e55183f52e74 100644
--- a/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.modifiers/erase_const_iter.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.modifiers/erase_const_iter.pass.cpp
@@ -24,125 +24,127 @@
 #include "../../../check_consecutive.h"
 #include "min_allocator.h"
 
-struct TemplateConstructor
-{
-    template<typename T>
-    TemplateConstructor (const T&) {}
+struct TemplateConstructor {
+  template <typename T>
+  TemplateConstructor(const T&) {}
 };
 
 bool operator==(const TemplateConstructor&, const TemplateConstructor&) { return false; }
-struct Hash { std::size_t operator() (const TemplateConstructor &) const { return 0; } };
+struct Hash {
+  std::size_t operator()(const TemplateConstructor&) const { return 0; }
+};
 
-int main(int, char**)
-{
-    {
-        typedef std::unordered_multimap<int, std::string> C;
-        typedef std::pair<int, std::string> P;
-        P a[] =
-        {
-            P(1, "one"),
-            P(2, "two"),
-            P(3, "three"),
-            P(4, "four"),
-            P(1, "four"),
-            P(2, "four"),
-        };
-        C c(a, a + sizeof(a)/sizeof(a[0]));
-        C::const_iterator i = c.find(2);
-        C::const_iterator i_next = i;
-        ++i_next;
-        std::string es = i->second;
-        C::iterator j = c.erase(i);
-        assert(j == i_next);
+int main(int, char**) {
+  {
+    typedef std::unordered_multimap<int, std::string> C;
+    typedef std::pair<int, std::string> P;
+    P a[] = {
+        P(1, "one"),
+        P(2, "two"),
+        P(3, "three"),
+        P(4, "four"),
+        P(1, "four"),
+        P(2, "four"),
+    };
+    C c(a, a + sizeof(a) / sizeof(a[0]));
+    C::const_iterator i      = c.find(2);
+    C::const_iterator i_next = i;
+    ++i_next;
+    std::string es = i->second;
+    C::iterator j  = c.erase(i);
+    assert(j == i_next);
 
-        assert(c.size() == 5);
-        typedef std::pair<C::const_iterator, C::const_iterator> Eq;
-        Eq eq = c.equal_range(1);
-        assert(std::distance(eq.first, eq.second) == 2);
-        std::multiset<std::string> s;
-        s.insert("one");
-        s.insert("four");
-        CheckConsecutiveKeys<C::const_iterator>(c.find(1), c.end(), 1, s);
-        eq = c.equal_range(2);
-        assert(std::distance(eq.first, eq.second) == 1);
-        C::const_iterator k = eq.first;
-        assert(k->first == 2);
-        assert(k->second == (es == "two" ? "four" : "two"));
-        eq = c.equal_range(3);
-        assert(std::distance(eq.first, eq.second) == 1);
-        k = eq.first;
-        assert(k->first == 3);
-        assert(k->second == "three");
-        eq = c.equal_range(4);
-        assert(std::distance(eq.first, eq.second) == 1);
-        k = eq.first;
-        assert(k->first == 4);
-        assert(k->second == "four");
-        assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-        assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
-    }
+    assert(c.size() == 5);
+    typedef std::pair<C::const_iterator, C::const_iterator> Eq;
+    Eq eq = c.equal_range(1);
+    assert(std::distance(eq.first, eq.second) == 2);
+    std::multiset<std::string> s;
+    s.insert("one");
+    s.insert("four");
+    CheckConsecutiveKeys<C::const_iterator>(c.find(1), c.end(), 1, s);
+    eq = c.equal_range(2);
+    assert(std::distance(eq.first, eq.second) == 1);
+    C::const_iterator k = eq.first;
+    assert(k->first == 2);
+    assert(k->second == (es == "two" ? "four" : "two"));
+    eq = c.equal_range(3);
+    assert(std::distance(eq.first, eq.second) == 1);
+    k = eq.first;
+    assert(k->first == 3);
+    assert(k->second == "three");
+    eq = c.equal_range(4);
+    assert(std::distance(eq.first, eq.second) == 1);
+    k = eq.first;
+    assert(k->first == 4);
+    assert(k->second == "four");
+    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
+    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
+  }
 #if TEST_STD_VER >= 11
-    {
-        typedef std::unordered_multimap<int, std::string, std::hash<int>, std::equal_to<int>,
-                            min_allocator<std::pair<const int, std::string>>> C;
-        typedef std::pair<int, std::string> P;
-        P a[] =
-        {
-            P(1, "one"),
-            P(2, "two"),
-            P(3, "three"),
-            P(4, "four"),
-            P(1, "four"),
-            P(2, "four"),
-        };
-        C c(a, a + sizeof(a)/sizeof(a[0]));
-        C::const_iterator i = c.find(2);
-        C::const_iterator i_next = i;
-        ++i_next;
-        std::string es = i->second;
-        C::iterator j = c.erase(i);
-        assert(j == i_next);
+  {
+    typedef std::unordered_multimap<int,
+                                    std::string,
+                                    std::hash<int>,
+                                    std::equal_to<int>,
+                                    min_allocator<std::pair<const int, std::string>>>
+        C;
+    typedef std::pair<int, std::string> P;
+    P a[] = {
+        P(1, "one"),
+        P(2, "two"),
+        P(3, "three"),
+        P(4, "four"),
+        P(1, "four"),
+        P(2, "four"),
+    };
+    C c(a, a + sizeof(a) / sizeof(a[0]));
+    C::const_iterator i      = c.find(2);
+    C::const_iterator i_next = i;
+    ++i_next;
+    std::string es = i->second;
+    C::iterator j  = c.erase(i);
+    assert(j == i_next);
 
-        assert(c.size() == 5);
-        typedef std::pair<C::const_iterator, C::const_iterator> Eq;
-        Eq eq = c.equal_range(1);
-        assert(std::distance(eq.first, eq.second) == 2);
-        std::multiset<std::string> s;
-        s.insert("one");
-        s.insert("four");
-        CheckConsecutiveKeys<C::const_iterator>(c.find(1), c.end(), 1, s);
-        eq = c.equal_range(2);
-        assert(std::distance(eq.first, eq.second) == 1);
-        C::const_iterator k = eq.first;
-        assert(k->first == 2);
-        assert(k->second == (es == "two" ? "four" : "two"));
-        eq = c.equal_range(3);
-        assert(std::distance(eq.first, eq.second) == 1);
-        k = eq.first;
-        assert(k->first == 3);
-        assert(k->second == "three");
-        eq = c.equal_range(4);
-        assert(std::distance(eq.first, eq.second) == 1);
-        k = eq.first;
-        assert(k->first == 4);
-        assert(k->second == "four");
-        assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-        assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
-    }
+    assert(c.size() == 5);
+    typedef std::pair<C::const_iterator, C::const_iterator> Eq;
+    Eq eq = c.equal_range(1);
+    assert(std::distance(eq.first, eq.second) == 2);
+    std::multiset<std::string> s;
+    s.insert("one");
+    s.insert("four");
+    CheckConsecutiveKeys<C::const_iterator>(c.find(1), c.end(), 1, s);
+    eq = c.equal_range(2);
+    assert(std::distance(eq.first, eq.second) == 1);
+    C::const_iterator k = eq.first;
+    assert(k->first == 2);
+    assert(k->second == (es == "two" ? "four" : "two"));
+    eq = c.equal_range(3);
+    assert(std::distance(eq.first, eq.second) == 1);
+    k = eq.first;
+    assert(k->first == 3);
+    assert(k->second == "three");
+    eq = c.equal_range(4);
+    assert(std::distance(eq.first, eq.second) == 1);
+    k = eq.first;
+    assert(k->first == 4);
+    assert(k->second == "four");
+    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
+    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
+  }
 #endif
 #if TEST_STD_VER >= 14
-    {
+  {
     //  This is LWG #2059
-        typedef TemplateConstructor T;
-        typedef std::unordered_multimap<T, int, Hash> C;
-        typedef C::iterator I;
+    typedef TemplateConstructor T;
+    typedef std::unordered_multimap<T, int, Hash> C;
+    typedef C::iterator I;
 
-        C m;
-        T a{0};
-        I it = m.find(a);
-        if (it != m.end())
-            m.erase(it);
-    }
+    C m;
+    T a{0};
+    I it = m.find(a);
+    if (it != m.end())
+      m.erase(it);
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.modifiers/erase_key.pass.cpp b/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.modifiers/erase_key.pass.cpp
index 25464c2c3495a..d988e2a31e58f 100644
--- a/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.modifiers/erase_key.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.modifiers/erase_key.pass.cpp
@@ -26,333 +26,334 @@
 
 #if TEST_STD_VER >= 11
 template <typename Unordered>
-bool only_deletions ( const Unordered &whole, const Unordered &part ) {
-    typename Unordered::const_iterator w = whole.begin();
-    typename Unordered::const_iterator p = part.begin();
+bool only_deletions(const Unordered& whole, const Unordered& part) {
+  typename Unordered::const_iterator w = whole.begin();
+  typename Unordered::const_iterator p = part.begin();
 
-    while ( w != whole.end () && p != part.end()) {
-        if ( *w == *p )
-            p++;
-        w++;
-        }
+  while (w != whole.end() && p != part.end()) {
+    if (*w == *p)
+      p++;
+    w++;
+  }
 
-    return p == part.end();
+  return p == part.end();
 }
 #endif
 
-int main(int, char**)
-{
-    {
-        typedef std::unordered_multimap<int, std::string> C;
-        typedef std::pair<int, std::string> P;
-        P a[] =
-        {
-            P(1, "one"),
-            P(2, "two"),
-            P(3, "three"),
-            P(4, "four"),
-            P(1, "four"),
-            P(2, "four"),
-        };
-        C c(a, a + sizeof(a)/sizeof(a[0]));
-        assert(c.erase(5) == 0);
-        assert(c.size() == 6);
-        typedef std::pair<C::const_iterator, C::const_iterator> Eq;
-        Eq eq = c.equal_range(1);
-        assert(std::distance(eq.first, eq.second) == 2);
-        std::multiset<std::string> s;
-        s.insert("one");
-        s.insert("four");
-        CheckConsecutiveKeys<C::const_iterator>(c.find(1), c.end(), 1, s);
-        eq = c.equal_range(2);
-        assert(std::distance(eq.first, eq.second) == 2);
-        s.insert("two");
-        s.insert("four");
-        CheckConsecutiveKeys<C::const_iterator>(c.find(2), c.end(), 2, s);
-        eq = c.equal_range(3);
-        assert(std::distance(eq.first, eq.second) == 1);
-        C::const_iterator k = eq.first;
-        assert(k->first == 3);
-        assert(k->second == "three");
-        eq = c.equal_range(4);
-        assert(std::distance(eq.first, eq.second) == 1);
-        k = eq.first;
-        assert(k->first == 4);
-        assert(k->second == "four");
-        assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-        assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
+int main(int, char**) {
+  {
+    typedef std::unordered_multimap<int, std::string> C;
+    typedef std::pair<int, std::string> P;
+    P a[] = {
+        P(1, "one"),
+        P(2, "two"),
+        P(3, "three"),
+        P(4, "four"),
+        P(1, "four"),
+        P(2, "four"),
+    };
+    C c(a, a + sizeof(a) / sizeof(a[0]));
+    assert(c.erase(5) == 0);
+    assert(c.size() == 6);
+    typedef std::pair<C::const_iterator, C::const_iterator> Eq;
+    Eq eq = c.equal_range(1);
+    assert(std::distance(eq.first, eq.second) == 2);
+    std::multiset<std::string> s;
+    s.insert("one");
+    s.insert("four");
+    CheckConsecutiveKeys<C::const_iterator>(c.find(1), c.end(), 1, s);
+    eq = c.equal_range(2);
+    assert(std::distance(eq.first, eq.second) == 2);
+    s.insert("two");
+    s.insert("four");
+    CheckConsecutiveKeys<C::const_iterator>(c.find(2), c.end(), 2, s);
+    eq = c.equal_range(3);
+    assert(std::distance(eq.first, eq.second) == 1);
+    C::const_iterator k = eq.first;
+    assert(k->first == 3);
+    assert(k->second == "three");
+    eq = c.equal_range(4);
+    assert(std::distance(eq.first, eq.second) == 1);
+    k = eq.first;
+    assert(k->first == 4);
+    assert(k->second == "four");
+    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
+    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
 
-        assert(c.erase(2) == 2);
-        assert(c.size() == 4);
-        eq = c.equal_range(1);
-        assert(std::distance(eq.first, eq.second) == 2);
-        s.insert("one");
-        s.insert("four");
-        CheckConsecutiveKeys<C::const_iterator>(c.find(1), c.end(), 1, s);
-        eq = c.equal_range(3);
-        assert(std::distance(eq.first, eq.second) == 1);
-        k = eq.first;
-        assert(k->first == 3);
-        assert(k->second == "three");
-        eq = c.equal_range(4);
-        assert(std::distance(eq.first, eq.second) == 1);
-        k = eq.first;
-        assert(k->first == 4);
-        assert(k->second == "four");
-        assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-        assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
+    assert(c.erase(2) == 2);
+    assert(c.size() == 4);
+    eq = c.equal_range(1);
+    assert(std::distance(eq.first, eq.second) == 2);
+    s.insert("one");
+    s.insert("four");
+    CheckConsecutiveKeys<C::const_iterator>(c.find(1), c.end(), 1, s);
+    eq = c.equal_range(3);
+    assert(std::distance(eq.first, eq.second) == 1);
+    k = eq.first;
+    assert(k->first == 3);
+    assert(k->second == "three");
+    eq = c.equal_range(4);
+    assert(std::distance(eq.first, eq.second) == 1);
+    k = eq.first;
+    assert(k->first == 4);
+    assert(k->second == "four");
+    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
+    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
 
-        assert(c.erase(2) == 0);
-        assert(c.size() == 4);
-        eq = c.equal_range(1);
-        assert(std::distance(eq.first, eq.second) == 2);
-        s.insert("one");
-        s.insert("four");
-        CheckConsecutiveKeys<C::const_iterator>(c.find(1), c.end(), 1, s);
-        eq = c.equal_range(3);
-        assert(std::distance(eq.first, eq.second) == 1);
-        k = eq.first;
-        assert(k->first == 3);
-        assert(k->second == "three");
-        eq = c.equal_range(4);
-        assert(std::distance(eq.first, eq.second) == 1);
-        k = eq.first;
-        assert(k->first == 4);
-        assert(k->second == "four");
-        assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-        assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
+    assert(c.erase(2) == 0);
+    assert(c.size() == 4);
+    eq = c.equal_range(1);
+    assert(std::distance(eq.first, eq.second) == 2);
+    s.insert("one");
+    s.insert("four");
+    CheckConsecutiveKeys<C::const_iterator>(c.find(1), c.end(), 1, s);
+    eq = c.equal_range(3);
+    assert(std::distance(eq.first, eq.second) == 1);
+    k = eq.first;
+    assert(k->first == 3);
+    assert(k->second == "three");
+    eq = c.equal_range(4);
+    assert(std::distance(eq.first, eq.second) == 1);
+    k = eq.first;
+    assert(k->first == 4);
+    assert(k->second == "four");
+    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
+    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
 
-        assert(c.erase(4) == 1);
-        assert(c.size() == 3);
-        eq = c.equal_range(1);
-        assert(std::distance(eq.first, eq.second) == 2);
-        s.insert("one");
-        s.insert("four");
-        CheckConsecutiveKeys<C::const_iterator>(c.find(1), c.end(), 1, s);
-        eq = c.equal_range(3);
-        assert(std::distance(eq.first, eq.second) == 1);
-        k = eq.first;
-        assert(k->first == 3);
-        assert(k->second == "three");
-        assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-        assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
+    assert(c.erase(4) == 1);
+    assert(c.size() == 3);
+    eq = c.equal_range(1);
+    assert(std::distance(eq.first, eq.second) == 2);
+    s.insert("one");
+    s.insert("four");
+    CheckConsecutiveKeys<C::const_iterator>(c.find(1), c.end(), 1, s);
+    eq = c.equal_range(3);
+    assert(std::distance(eq.first, eq.second) == 1);
+    k = eq.first;
+    assert(k->first == 3);
+    assert(k->second == "three");
+    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
+    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
 
-        assert(c.erase(4) == 0);
-        assert(c.size() == 3);
-        eq = c.equal_range(1);
-        assert(std::distance(eq.first, eq.second) == 2);
-        s.insert("one");
-        s.insert("four");
-        CheckConsecutiveKeys<C::const_iterator>(c.find(1), c.end(), 1, s);
-        eq = c.equal_range(3);
-        assert(std::distance(eq.first, eq.second) == 1);
-        k = eq.first;
-        assert(k->first == 3);
-        assert(k->second == "three");
-        assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-        assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
+    assert(c.erase(4) == 0);
+    assert(c.size() == 3);
+    eq = c.equal_range(1);
+    assert(std::distance(eq.first, eq.second) == 2);
+    s.insert("one");
+    s.insert("four");
+    CheckConsecutiveKeys<C::const_iterator>(c.find(1), c.end(), 1, s);
+    eq = c.equal_range(3);
+    assert(std::distance(eq.first, eq.second) == 1);
+    k = eq.first;
+    assert(k->first == 3);
+    assert(k->second == "three");
+    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
+    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
 
-        assert(c.erase(1) == 2);
-        assert(c.size() == 1);
-        eq = c.equal_range(3);
-        assert(std::distance(eq.first, eq.second) == 1);
-        k = eq.first;
-        assert(k->first == 3);
-        assert(k->second == "three");
-        assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-        assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
+    assert(c.erase(1) == 2);
+    assert(c.size() == 1);
+    eq = c.equal_range(3);
+    assert(std::distance(eq.first, eq.second) == 1);
+    k = eq.first;
+    assert(k->first == 3);
+    assert(k->second == "three");
+    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
+    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
 
-        assert(c.erase(1) == 0);
-        assert(c.size() == 1);
-        eq = c.equal_range(3);
-        assert(std::distance(eq.first, eq.second) == 1);
-        k = eq.first;
-        assert(k->first == 3);
-        assert(k->second == "three");
-        assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-        assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
+    assert(c.erase(1) == 0);
+    assert(c.size() == 1);
+    eq = c.equal_range(3);
+    assert(std::distance(eq.first, eq.second) == 1);
+    k = eq.first;
+    assert(k->first == 3);
+    assert(k->second == "three");
+    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
+    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
 
-        assert(c.erase(3) == 1);
-        assert(c.size() == 0);
-        eq = c.equal_range(3);
-        assert(std::distance(eq.first, eq.second) == 0);
-        assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-        assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
+    assert(c.erase(3) == 1);
+    assert(c.size() == 0);
+    eq = c.equal_range(3);
+    assert(std::distance(eq.first, eq.second) == 0);
+    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
+    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
 
-        assert(c.erase(3) == 0);
-        assert(c.size() == 0);
-        eq = c.equal_range(3);
-        assert(std::distance(eq.first, eq.second) == 0);
-        assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-        assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
-    }
+    assert(c.erase(3) == 0);
+    assert(c.size() == 0);
+    eq = c.equal_range(3);
+    assert(std::distance(eq.first, eq.second) == 0);
+    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
+    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
+  }
 #if TEST_STD_VER >= 11
-    {
-        typedef std::unordered_multimap<int, std::string, std::hash<int>, std::equal_to<int>,
-                            min_allocator<std::pair<const int, std::string>>> C;
-        typedef std::pair<int, std::string> P;
-        P a[] =
-        {
-            P(1, "one"),
-            P(2, "two"),
-            P(3, "three"),
-            P(4, "four"),
-            P(1, "four"),
-            P(2, "four"),
-        };
-        C c(a, a + sizeof(a)/sizeof(a[0]));
-        assert(c.erase(5) == 0);
-        assert(c.size() == 6);
-        typedef std::pair<C::const_iterator, C::const_iterator> Eq;
-        Eq eq = c.equal_range(1);
-        assert(std::distance(eq.first, eq.second) == 2);
-        std::multiset<std::string> s;
-        s.insert("one");
-        s.insert("four");
-        CheckConsecutiveKeys<C::const_iterator>(c.find(1), c.end(), 1, s);
-        eq = c.equal_range(2);
-        assert(std::distance(eq.first, eq.second) == 2);
-        s.insert("two");
-        s.insert("four");
-        CheckConsecutiveKeys<C::const_iterator>(c.find(2), c.end(), 2, s);
-        eq = c.equal_range(3);
-        assert(std::distance(eq.first, eq.second) == 1);
-        C::const_iterator k = eq.first;
-        assert(k->first == 3);
-        assert(k->second == "three");
-        eq = c.equal_range(4);
-        assert(std::distance(eq.first, eq.second) == 1);
-        k = eq.first;
-        assert(k->first == 4);
-        assert(k->second == "four");
-        assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-        assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
+  {
+    typedef std::unordered_multimap<int,
+                                    std::string,
+                                    std::hash<int>,
+                                    std::equal_to<int>,
+                                    min_allocator<std::pair<const int, std::string>>>
+        C;
+    typedef std::pair<int, std::string> P;
+    P a[] = {
+        P(1, "one"),
+        P(2, "two"),
+        P(3, "three"),
+        P(4, "four"),
+        P(1, "four"),
+        P(2, "four"),
+    };
+    C c(a, a + sizeof(a) / sizeof(a[0]));
+    assert(c.erase(5) == 0);
+    assert(c.size() == 6);
+    typedef std::pair<C::const_iterator, C::const_iterator> Eq;
+    Eq eq = c.equal_range(1);
+    assert(std::distance(eq.first, eq.second) == 2);
+    std::multiset<std::string> s;
+    s.insert("one");
+    s.insert("four");
+    CheckConsecutiveKeys<C::const_iterator>(c.find(1), c.end(), 1, s);
+    eq = c.equal_range(2);
+    assert(std::distance(eq.first, eq.second) == 2);
+    s.insert("two");
+    s.insert("four");
+    CheckConsecutiveKeys<C::const_iterator>(c.find(2), c.end(), 2, s);
+    eq = c.equal_range(3);
+    assert(std::distance(eq.first, eq.second) == 1);
+    C::const_iterator k = eq.first;
+    assert(k->first == 3);
+    assert(k->second == "three");
+    eq = c.equal_range(4);
+    assert(std::distance(eq.first, eq.second) == 1);
+    k = eq.first;
+    assert(k->first == 4);
+    assert(k->second == "four");
+    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
+    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
 
-        assert(c.erase(2) == 2);
-        assert(c.size() == 4);
-        eq = c.equal_range(1);
-        assert(std::distance(eq.first, eq.second) == 2);
-        s.insert("one");
-        s.insert("four");
-        CheckConsecutiveKeys<C::const_iterator>(c.find(1), c.end(), 1, s);
-        eq = c.equal_range(3);
-        assert(std::distance(eq.first, eq.second) == 1);
-        k = eq.first;
-        assert(k->first == 3);
-        assert(k->second == "three");
-        eq = c.equal_range(4);
-        assert(std::distance(eq.first, eq.second) == 1);
-        k = eq.first;
-        assert(k->first == 4);
-        assert(k->second == "four");
-        assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-        assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
+    assert(c.erase(2) == 2);
+    assert(c.size() == 4);
+    eq = c.equal_range(1);
+    assert(std::distance(eq.first, eq.second) == 2);
+    s.insert("one");
+    s.insert("four");
+    CheckConsecutiveKeys<C::const_iterator>(c.find(1), c.end(), 1, s);
+    eq = c.equal_range(3);
+    assert(std::distance(eq.first, eq.second) == 1);
+    k = eq.first;
+    assert(k->first == 3);
+    assert(k->second == "three");
+    eq = c.equal_range(4);
+    assert(std::distance(eq.first, eq.second) == 1);
+    k = eq.first;
+    assert(k->first == 4);
+    assert(k->second == "four");
+    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
+    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
 
-        assert(c.erase(2) == 0);
-        assert(c.size() == 4);
-        eq = c.equal_range(1);
-        assert(std::distance(eq.first, eq.second) == 2);
-        s.insert("one");
-        s.insert("four");
-        CheckConsecutiveKeys<C::const_iterator>(c.find(1), c.end(), 1, s);
-        eq = c.equal_range(3);
-        assert(std::distance(eq.first, eq.second) == 1);
-        k = eq.first;
-        assert(k->first == 3);
-        assert(k->second == "three");
-        eq = c.equal_range(4);
-        assert(std::distance(eq.first, eq.second) == 1);
-        k = eq.first;
-        assert(k->first == 4);
-        assert(k->second == "four");
-        assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-        assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
+    assert(c.erase(2) == 0);
+    assert(c.size() == 4);
+    eq = c.equal_range(1);
+    assert(std::distance(eq.first, eq.second) == 2);
+    s.insert("one");
+    s.insert("four");
+    CheckConsecutiveKeys<C::const_iterator>(c.find(1), c.end(), 1, s);
+    eq = c.equal_range(3);
+    assert(std::distance(eq.first, eq.second) == 1);
+    k = eq.first;
+    assert(k->first == 3);
+    assert(k->second == "three");
+    eq = c.equal_range(4);
+    assert(std::distance(eq.first, eq.second) == 1);
+    k = eq.first;
+    assert(k->first == 4);
+    assert(k->second == "four");
+    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
+    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
 
-        assert(c.erase(4) == 1);
-        assert(c.size() == 3);
-        eq = c.equal_range(1);
-        assert(std::distance(eq.first, eq.second) == 2);
-        s.insert("one");
-        s.insert("four");
-        CheckConsecutiveKeys<C::const_iterator>(c.find(1), c.end(), 1, s);
-        eq = c.equal_range(3);
-        assert(std::distance(eq.first, eq.second) == 1);
-        k = eq.first;
-        assert(k->first == 3);
-        assert(k->second == "three");
-        assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-        assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
+    assert(c.erase(4) == 1);
+    assert(c.size() == 3);
+    eq = c.equal_range(1);
+    assert(std::distance(eq.first, eq.second) == 2);
+    s.insert("one");
+    s.insert("four");
+    CheckConsecutiveKeys<C::const_iterator>(c.find(1), c.end(), 1, s);
+    eq = c.equal_range(3);
+    assert(std::distance(eq.first, eq.second) == 1);
+    k = eq.first;
+    assert(k->first == 3);
+    assert(k->second == "three");
+    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
+    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
 
-        assert(c.erase(4) == 0);
-        assert(c.size() == 3);
-        eq = c.equal_range(1);
-        assert(std::distance(eq.first, eq.second) == 2);
-        s.insert("one");
-        s.insert("four");
-        CheckConsecutiveKeys<C::const_iterator>(c.find(1), c.end(), 1, s);
-        eq = c.equal_range(3);
-        assert(std::distance(eq.first, eq.second) == 1);
-        k = eq.first;
-        assert(k->first == 3);
-        assert(k->second == "three");
-        assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-        assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
+    assert(c.erase(4) == 0);
+    assert(c.size() == 3);
+    eq = c.equal_range(1);
+    assert(std::distance(eq.first, eq.second) == 2);
+    s.insert("one");
+    s.insert("four");
+    CheckConsecutiveKeys<C::const_iterator>(c.find(1), c.end(), 1, s);
+    eq = c.equal_range(3);
+    assert(std::distance(eq.first, eq.second) == 1);
+    k = eq.first;
+    assert(k->first == 3);
+    assert(k->second == "three");
+    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
+    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
 
-        assert(c.erase(1) == 2);
-        assert(c.size() == 1);
-        eq = c.equal_range(3);
-        assert(std::distance(eq.first, eq.second) == 1);
-        k = eq.first;
-        assert(k->first == 3);
-        assert(k->second == "three");
-        assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-        assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
+    assert(c.erase(1) == 2);
+    assert(c.size() == 1);
+    eq = c.equal_range(3);
+    assert(std::distance(eq.first, eq.second) == 1);
+    k = eq.first;
+    assert(k->first == 3);
+    assert(k->second == "three");
+    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
+    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
 
-        assert(c.erase(1) == 0);
-        assert(c.size() == 1);
-        eq = c.equal_range(3);
-        assert(std::distance(eq.first, eq.second) == 1);
-        k = eq.first;
-        assert(k->first == 3);
-        assert(k->second == "three");
-        assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-        assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
+    assert(c.erase(1) == 0);
+    assert(c.size() == 1);
+    eq = c.equal_range(3);
+    assert(std::distance(eq.first, eq.second) == 1);
+    k = eq.first;
+    assert(k->first == 3);
+    assert(k->second == "three");
+    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
+    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
 
-        assert(c.erase(3) == 1);
-        assert(c.size() == 0);
-        eq = c.equal_range(3);
-        assert(std::distance(eq.first, eq.second) == 0);
-        assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-        assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
+    assert(c.erase(3) == 1);
+    assert(c.size() == 0);
+    eq = c.equal_range(3);
+    assert(std::distance(eq.first, eq.second) == 0);
+    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
+    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
 
-        assert(c.erase(3) == 0);
-        assert(c.size() == 0);
-        eq = c.equal_range(3);
-        assert(std::distance(eq.first, eq.second) == 0);
-        assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-        assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
-    }
-    {
+    assert(c.erase(3) == 0);
+    assert(c.size() == 0);
+    eq = c.equal_range(3);
+    assert(std::distance(eq.first, eq.second) == 0);
+    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
+    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
+  }
+  {
     typedef std::unordered_multimap<int, int> C;
     C m, m2;
-    for ( int i = 0; i < 10; ++i ) {
-        for (int j = 0; j < 2; ++j ) {
-            m.insert (std::make_pair(i,j));
-            m2.insert(std::make_pair(i,j));
-            }
-        }
+    for (int i = 0; i < 10; ++i) {
+      for (int j = 0; j < 2; ++j) {
+        m.insert(std::make_pair(i, j));
+        m2.insert(std::make_pair(i, j));
+      }
+    }
 
     C::iterator i = m2.begin();
-    int ctr = 0;
+    int ctr       = 0;
     while (i != m2.end()) {
-        if (ctr++ % 2 == 0)
-            m2.erase(i++);
-        else
-            ++i;
-        }
-
-    assert (only_deletions (m, m2));
+      if (ctr++ % 2 == 0)
+        m2.erase(i++);
+      else
+        ++i;
     }
+
+    assert(only_deletions(m, m2));
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.modifiers/erase_range.pass.cpp b/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.modifiers/erase_range.pass.cpp
index cab3702b1273f..f8a2bdd3fee73 100644
--- a/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.modifiers/erase_range.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.modifiers/erase_range.pass.cpp
@@ -24,144 +24,145 @@
 #include "../../../check_consecutive.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef std::unordered_multimap<int, std::string> C;
-        typedef std::pair<int, std::string> P;
-        P a[] =
-        {
-            P(1, "one"),
-            P(2, "two"),
-            P(3, "three"),
-            P(4, "four"),
-            P(1, "four"),
-            P(2, "four"),
-        };
-        C c(a, a + sizeof(a)/sizeof(a[0]));
-        C::const_iterator i = c.find(2);
-        C::const_iterator j = std::next(i, 2);
-        C::iterator k = c.erase(i, i);
-        assert(k == i);
-        assert(c.size() == 6);
-        typedef std::pair<C::iterator, C::iterator> Eq;
-        Eq eq = c.equal_range(1);
-        assert(std::distance(eq.first, eq.second) == 2);
-        std::multiset<std::string> s;
-        s.insert("one");
-        s.insert("four");
-        CheckConsecutiveKeys<C::const_iterator>(c.find(1), c.end(), 1, s);
-        eq = c.equal_range(2);
-        assert(std::distance(eq.first, eq.second) == 2);
-        s.insert("two");
-        s.insert("four");
-        CheckConsecutiveKeys<C::const_iterator>(c.find(2), c.end(), 2, s);
-        eq = c.equal_range(3);
-        assert(std::distance(eq.first, eq.second) == 1);
-        k = eq.first;
-        assert(k->first == 3);
-        assert(k->second == "three");
-        eq = c.equal_range(4);
-        assert(std::distance(eq.first, eq.second) == 1);
-        k = eq.first;
-        assert(k->first == 4);
-        assert(k->second == "four");
-        assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-        assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
+int main(int, char**) {
+  {
+    typedef std::unordered_multimap<int, std::string> C;
+    typedef std::pair<int, std::string> P;
+    P a[] = {
+        P(1, "one"),
+        P(2, "two"),
+        P(3, "three"),
+        P(4, "four"),
+        P(1, "four"),
+        P(2, "four"),
+    };
+    C c(a, a + sizeof(a) / sizeof(a[0]));
+    C::const_iterator i = c.find(2);
+    C::const_iterator j = std::next(i, 2);
+    C::iterator k       = c.erase(i, i);
+    assert(k == i);
+    assert(c.size() == 6);
+    typedef std::pair<C::iterator, C::iterator> Eq;
+    Eq eq = c.equal_range(1);
+    assert(std::distance(eq.first, eq.second) == 2);
+    std::multiset<std::string> s;
+    s.insert("one");
+    s.insert("four");
+    CheckConsecutiveKeys<C::const_iterator>(c.find(1), c.end(), 1, s);
+    eq = c.equal_range(2);
+    assert(std::distance(eq.first, eq.second) == 2);
+    s.insert("two");
+    s.insert("four");
+    CheckConsecutiveKeys<C::const_iterator>(c.find(2), c.end(), 2, s);
+    eq = c.equal_range(3);
+    assert(std::distance(eq.first, eq.second) == 1);
+    k = eq.first;
+    assert(k->first == 3);
+    assert(k->second == "three");
+    eq = c.equal_range(4);
+    assert(std::distance(eq.first, eq.second) == 1);
+    k = eq.first;
+    assert(k->first == 4);
+    assert(k->second == "four");
+    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
+    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
 
-        k = c.erase(i, j);
-        assert(c.size() == 4);
-        eq = c.equal_range(1);
-        assert(std::distance(eq.first, eq.second) == 2);
-        s.insert("one");
-        s.insert("four");
-        CheckConsecutiveKeys<C::const_iterator>(c.find(1), c.end(), 1, s);
-        eq = c.equal_range(3);
-        assert(std::distance(eq.first, eq.second) == 1);
-        k = eq.first;
-        assert(k->first == 3);
-        assert(k->second == "three");
-        eq = c.equal_range(4);
-        assert(std::distance(eq.first, eq.second) == 1);
-        k = eq.first;
-        assert(k->first == 4);
-        assert(k->second == "four");
-        assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-        assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
+    k = c.erase(i, j);
+    assert(c.size() == 4);
+    eq = c.equal_range(1);
+    assert(std::distance(eq.first, eq.second) == 2);
+    s.insert("one");
+    s.insert("four");
+    CheckConsecutiveKeys<C::const_iterator>(c.find(1), c.end(), 1, s);
+    eq = c.equal_range(3);
+    assert(std::distance(eq.first, eq.second) == 1);
+    k = eq.first;
+    assert(k->first == 3);
+    assert(k->second == "three");
+    eq = c.equal_range(4);
+    assert(std::distance(eq.first, eq.second) == 1);
+    k = eq.first;
+    assert(k->first == 4);
+    assert(k->second == "four");
+    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
+    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
 
-        k = c.erase(c.cbegin(), c.cend());
-        assert(c.size() == 0);
-        assert(k == c.end());
-    }
+    k = c.erase(c.cbegin(), c.cend());
+    assert(c.size() == 0);
+    assert(k == c.end());
+  }
 #if TEST_STD_VER >= 11
-    {
-        typedef std::unordered_multimap<int, std::string, std::hash<int>, std::equal_to<int>,
-                            min_allocator<std::pair<const int, std::string>>> C;
-        typedef std::pair<int, std::string> P;
-        P a[] =
-        {
-            P(1, "one"),
-            P(2, "two"),
-            P(3, "three"),
-            P(4, "four"),
-            P(1, "four"),
-            P(2, "four"),
-        };
-        C c(a, a + sizeof(a)/sizeof(a[0]));
-        C::const_iterator i = c.find(2);
-        C::const_iterator j = std::next(i, 2);
-        C::iterator k = c.erase(i, i);
-        assert(k == i);
-        assert(c.size() == 6);
-        typedef std::pair<C::iterator, C::iterator> Eq;
-        Eq eq = c.equal_range(1);
-        assert(std::distance(eq.first, eq.second) == 2);
-        std::multiset<std::string> s;
-        s.insert("one");
-        s.insert("four");
-        CheckConsecutiveKeys<C::const_iterator>(c.find(1), c.end(), 1, s);
-        eq = c.equal_range(2);
-        assert(std::distance(eq.first, eq.second) == 2);
-        s.insert("two");
-        s.insert("four");
-        CheckConsecutiveKeys<C::const_iterator>(c.find(2), c.end(), 2, s);
-        eq = c.equal_range(3);
-        assert(std::distance(eq.first, eq.second) == 1);
-        k = eq.first;
-        assert(k->first == 3);
-        assert(k->second == "three");
-        eq = c.equal_range(4);
-        assert(std::distance(eq.first, eq.second) == 1);
-        k = eq.first;
-        assert(k->first == 4);
-        assert(k->second == "four");
-        assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-        assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
+  {
+    typedef std::unordered_multimap<int,
+                                    std::string,
+                                    std::hash<int>,
+                                    std::equal_to<int>,
+                                    min_allocator<std::pair<const int, std::string>>>
+        C;
+    typedef std::pair<int, std::string> P;
+    P a[] = {
+        P(1, "one"),
+        P(2, "two"),
+        P(3, "three"),
+        P(4, "four"),
+        P(1, "four"),
+        P(2, "four"),
+    };
+    C c(a, a + sizeof(a) / sizeof(a[0]));
+    C::const_iterator i = c.find(2);
+    C::const_iterator j = std::next(i, 2);
+    C::iterator k       = c.erase(i, i);
+    assert(k == i);
+    assert(c.size() == 6);
+    typedef std::pair<C::iterator, C::iterator> Eq;
+    Eq eq = c.equal_range(1);
+    assert(std::distance(eq.first, eq.second) == 2);
+    std::multiset<std::string> s;
+    s.insert("one");
+    s.insert("four");
+    CheckConsecutiveKeys<C::const_iterator>(c.find(1), c.end(), 1, s);
+    eq = c.equal_range(2);
+    assert(std::distance(eq.first, eq.second) == 2);
+    s.insert("two");
+    s.insert("four");
+    CheckConsecutiveKeys<C::const_iterator>(c.find(2), c.end(), 2, s);
+    eq = c.equal_range(3);
+    assert(std::distance(eq.first, eq.second) == 1);
+    k = eq.first;
+    assert(k->first == 3);
+    assert(k->second == "three");
+    eq = c.equal_range(4);
+    assert(std::distance(eq.first, eq.second) == 1);
+    k = eq.first;
+    assert(k->first == 4);
+    assert(k->second == "four");
+    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
+    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
 
-        k = c.erase(i, j);
-        assert(c.size() == 4);
-        eq = c.equal_range(1);
-        assert(std::distance(eq.first, eq.second) == 2);
-        s.insert("one");
-        s.insert("four");
-        CheckConsecutiveKeys<C::const_iterator>(c.find(1), c.end(), 1, s);
-        eq = c.equal_range(3);
-        assert(std::distance(eq.first, eq.second) == 1);
-        k = eq.first;
-        assert(k->first == 3);
-        assert(k->second == "three");
-        eq = c.equal_range(4);
-        assert(std::distance(eq.first, eq.second) == 1);
-        k = eq.first;
-        assert(k->first == 4);
-        assert(k->second == "four");
-        assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-        assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
+    k = c.erase(i, j);
+    assert(c.size() == 4);
+    eq = c.equal_range(1);
+    assert(std::distance(eq.first, eq.second) == 2);
+    s.insert("one");
+    s.insert("four");
+    CheckConsecutiveKeys<C::const_iterator>(c.find(1), c.end(), 1, s);
+    eq = c.equal_range(3);
+    assert(std::distance(eq.first, eq.second) == 1);
+    k = eq.first;
+    assert(k->first == 3);
+    assert(k->second == "three");
+    eq = c.equal_range(4);
+    assert(std::distance(eq.first, eq.second) == 1);
+    k = eq.first;
+    assert(k->first == 4);
+    assert(k->second == "four");
+    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
+    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
 
-        k = c.erase(c.cbegin(), c.cend());
-        assert(c.size() == 0);
-        assert(k == c.end());
-    }
+    k = c.erase(c.cbegin(), c.cend());
+    assert(c.size() == 0);
+    assert(k == c.end());
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.modifiers/extract_iterator.pass.cpp b/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.modifiers/extract_iterator.pass.cpp
index 8905dc2c119c4..d8a73cd3d2b3b 100644
--- a/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.modifiers/extract_iterator.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.modifiers/extract_iterator.pass.cpp
@@ -20,50 +20,45 @@
 #include "Counter.h"
 
 template <class Container>
-void test(Container& c)
-{
-    std::size_t sz = c.size();
+void test(Container& c) {
+  std::size_t sz = c.size();
 
-    auto some_key = c.cbegin()->first;
+  auto some_key = c.cbegin()->first;
 
-    for (auto first = c.cbegin(); first != c.cend();)
-    {
-        auto key_value = first->first;
-        typename Container::node_type t = c.extract(first++);
-        --sz;
-        assert(t.key() == key_value);
-        t.key() = some_key;
-        assert(t.key() == some_key);
-        assert(t.get_allocator() == c.get_allocator());
-        assert(sz == c.size());
-    }
+  for (auto first = c.cbegin(); first != c.cend();) {
+    auto key_value                  = first->first;
+    typename Container::node_type t = c.extract(first++);
+    --sz;
+    assert(t.key() == key_value);
+    t.key() = some_key;
+    assert(t.key() == some_key);
+    assert(t.get_allocator() == c.get_allocator());
+    assert(sz == c.size());
+  }
 
-    assert(c.size() == 0);
+  assert(c.size() == 0);
 }
 
-int main(int, char**)
-{
-    {
-        using map_type = std::unordered_multimap<int, int>;
-        map_type m = {{1,1}, {2,2}, {3,3}, {4,4}, {5,5}, {6,6}};
-        test(m);
-    }
+int main(int, char**) {
+  {
+    using map_type = std::unordered_multimap<int, int>;
+    map_type m     = {{1, 1}, {2, 2}, {3, 3}, {4, 4}, {5, 5}, {6, 6}};
+    test(m);
+  }
 
-    {
-        std::unordered_multimap<Counter<int>, Counter<int>> m =
-            {{1,1}, {2,2}, {3,3}, {4,4}, {5,5}, {6,6}};
-        assert(Counter_base::gConstructed == 12);
-        test(m);
-        assert(Counter_base::gConstructed == 0);
-    }
+  {
+    std::unordered_multimap<Counter<int>, Counter<int>> m = {{1, 1}, {2, 2}, {3, 3}, {4, 4}, {5, 5}, {6, 6}};
+    assert(Counter_base::gConstructed == 12);
+    test(m);
+    assert(Counter_base::gConstructed == 0);
+  }
 
-    {
-        using min_alloc_map =
-            std::unordered_multimap<int, int, std::hash<int>, std::equal_to<int>,
-                                    min_allocator<std::pair<const int, int>>>;
-        min_alloc_map m = {{1, 1}, {2, 2}, {3, 3}, {4, 4}, {5, 5}, {6, 6}};
-        test(m);
-    }
+  {
+    using min_alloc_map =
+        std::unordered_multimap<int, int, std::hash<int>, std::equal_to<int>, min_allocator<std::pair<const int, int>>>;
+    min_alloc_map m = {{1, 1}, {2, 2}, {3, 3}, {4, 4}, {5, 5}, {6, 6}};
+    test(m);
+  }
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.modifiers/extract_key.pass.cpp b/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.modifiers/extract_key.pass.cpp
index 3f2f69f031b7b..28716f13fbd6c 100644
--- a/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.modifiers/extract_key.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.modifiers/extract_key.pass.cpp
@@ -20,60 +20,53 @@
 #include "Counter.h"
 
 template <class Container, class KeyTypeIter>
-void test(Container& c, KeyTypeIter first, KeyTypeIter last)
-{
-    std::size_t sz = c.size();
-    assert((std::size_t)std::distance(first, last) == sz);
+void test(Container& c, KeyTypeIter first, KeyTypeIter last) {
+  std::size_t sz = c.size();
+  assert((std::size_t)std::distance(first, last) == sz);
 
-    for (KeyTypeIter copy = first; copy != last; ++copy)
-    {
-        typename Container::node_type t = c.extract(*copy);
-        assert(!t.empty());
-        --sz;
-        assert(t.key() == *copy);
-        t.key() = *first; // We should be able to mutate key.
-        assert(t.key() == *first);
-        assert(t.get_allocator() == c.get_allocator());
-        assert(sz == c.size());
-    }
+  for (KeyTypeIter copy = first; copy != last; ++copy) {
+    typename Container::node_type t = c.extract(*copy);
+    assert(!t.empty());
+    --sz;
+    assert(t.key() == *copy);
+    t.key() = *first; // We should be able to mutate key.
+    assert(t.key() == *first);
+    assert(t.get_allocator() == c.get_allocator());
+    assert(sz == c.size());
+  }
 
-    assert(c.size() == 0);
+  assert(c.size() == 0);
 
-    for (KeyTypeIter copy = first; copy != last; ++copy)
-    {
-        typename Container::node_type t = c.extract(*copy);
-        assert(t.empty());
-    }
+  for (KeyTypeIter copy = first; copy != last; ++copy) {
+    typename Container::node_type t = c.extract(*copy);
+    assert(t.empty());
+  }
 }
 
-int main(int, char**)
-{
-    {
-        std::unordered_multimap<int, int> m =
-            {{1,1}, {2,2}, {3,3}, {4,4}, {5,5}, {6,6}};
-        int keys[] = {1, 2, 3, 4, 5, 6};
-        test(m, std::begin(keys), std::end(keys));
-    }
+int main(int, char**) {
+  {
+    std::unordered_multimap<int, int> m = {{1, 1}, {2, 2}, {3, 3}, {4, 4}, {5, 5}, {6, 6}};
+    int keys[]                          = {1, 2, 3, 4, 5, 6};
+    test(m, std::begin(keys), std::end(keys));
+  }
 
+  {
+    std::unordered_multimap<Counter<int>, Counter<int>> m = {{1, 1}, {2, 2}, {3, 3}, {4, 4}, {5, 5}, {6, 6}};
     {
-        std::unordered_multimap<Counter<int>, Counter<int>> m =
-            {{1,1}, {2,2}, {3,3}, {4,4}, {5,5}, {6,6}};
-        {
-            Counter<int> keys[] = {1, 2, 3, 4, 5, 6};
-            assert(Counter_base::gConstructed == 12+6);
-            test(m, std::begin(keys), std::end(keys));
-        }
-        assert(Counter_base::gConstructed == 0);
+      Counter<int> keys[] = {1, 2, 3, 4, 5, 6};
+      assert(Counter_base::gConstructed == 12 + 6);
+      test(m, std::begin(keys), std::end(keys));
     }
+    assert(Counter_base::gConstructed == 0);
+  }
 
-    {
-        using min_alloc_map =
-            std::unordered_multimap<int, int, std::hash<int>, std::equal_to<int>,
-                                    min_allocator<std::pair<const int, int>>>;
-        min_alloc_map m = {{1, 1}, {2, 2}, {3, 3}, {4, 4}, {5, 5}, {6, 6}};
-        int keys[] = {1, 2, 3, 4, 5, 6};
-        test(m, std::begin(keys), std::end(keys));
-    }
+  {
+    using min_alloc_map =
+        std::unordered_multimap<int, int, std::hash<int>, std::equal_to<int>, min_allocator<std::pair<const int, int>>>;
+    min_alloc_map m = {{1, 1}, {2, 2}, {3, 3}, {4, 4}, {5, 5}, {6, 6}};
+    int keys[]      = {1, 2, 3, 4, 5, 6};
+    test(m, std::begin(keys), std::end(keys));
+  }
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.modifiers/insert_allocator_requirements.pass.cpp b/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.modifiers/insert_allocator_requirements.pass.cpp
index d9d338c357823..261a7b68f2194 100644
--- a/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.modifiers/insert_allocator_requirements.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.modifiers/insert_allocator_requirements.pass.cpp
@@ -20,8 +20,7 @@
 #include "container_test_types.h"
 #include "../../../map_allocator_requirement_test_templates.h"
 
-int main(int, char**)
-{
+int main(int, char**) {
   testMultimapInsert<TCT::unordered_multimap<> >();
   testMultimapInsertHint<TCT::unordered_multimap<> >();
 
diff --git a/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.modifiers/insert_const_lvalue.pass.cpp b/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.modifiers/insert_const_lvalue.pass.cpp
index fb3d390aa74ac..ef63a00f5b526 100644
--- a/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.modifiers/insert_const_lvalue.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.modifiers/insert_const_lvalue.pass.cpp
@@ -20,47 +20,49 @@
 #include "test_macros.h"
 #include "min_allocator.h"
 
-template<class Container>
-void do_insert_const_lvalue_test()
-{
-    typedef Container C;
-    typedef typename C::iterator R;
-    typedef typename C::value_type VT;
-    C c;
-    const VT v1(3.5, 3);
-    R r = c.insert(v1);
-    assert(c.size() == 1);
-    assert(r->first == 3.5);
-    assert(r->second == 3);
+template <class Container>
+void do_insert_const_lvalue_test() {
+  typedef Container C;
+  typedef typename C::iterator R;
+  typedef typename C::value_type VT;
+  C c;
+  const VT v1(3.5, 3);
+  R r = c.insert(v1);
+  assert(c.size() == 1);
+  assert(r->first == 3.5);
+  assert(r->second == 3);
 
-    const VT v2(3.5, 4);
-    r = c.insert(v2);
-    assert(c.size() == 2);
-    assert(r->first == 3.5);
-    assert(r->second == 4);
+  const VT v2(3.5, 4);
+  r = c.insert(v2);
+  assert(c.size() == 2);
+  assert(r->first == 3.5);
+  assert(r->second == 4);
 
-    const VT v3(4.5, 4);
-    r = c.insert(v3);
-    assert(c.size() == 3);
-    assert(r->first == 4.5);
-    assert(r->second == 4);
+  const VT v3(4.5, 4);
+  r = c.insert(v3);
+  assert(c.size() == 3);
+  assert(r->first == 4.5);
+  assert(r->second == 4);
 
-    const VT v4(5.5, 4);
-    r = c.insert(v4);
-    assert(c.size() == 4);
-    assert(r->first == 5.5);
-    assert(r->second == 4);
+  const VT v4(5.5, 4);
+  r = c.insert(v4);
+  assert(c.size() == 4);
+  assert(r->first == 5.5);
+  assert(r->second == 4);
 }
 
-int main(int, char**)
-{
-    do_insert_const_lvalue_test<std::unordered_multimap<double, int> >();
+int main(int, char**) {
+  do_insert_const_lvalue_test<std::unordered_multimap<double, int> >();
 #if TEST_STD_VER >= 11
-    {
-        typedef std::unordered_multimap<double, int, std::hash<double>, std::equal_to<double>,
-                            min_allocator<std::pair<const double, int>>> C;
-        do_insert_const_lvalue_test<C>();
-    }
+  {
+    typedef std::unordered_multimap<double,
+                                    int,
+                                    std::hash<double>,
+                                    std::equal_to<double>,
+                                    min_allocator<std::pair<const double, int>>>
+        C;
+    do_insert_const_lvalue_test<C>();
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.modifiers/insert_hint_const_lvalue.pass.cpp b/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.modifiers/insert_hint_const_lvalue.pass.cpp
index b3a5487c07860..1682ec5ba90ca 100644
--- a/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.modifiers/insert_hint_const_lvalue.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.modifiers/insert_hint_const_lvalue.pass.cpp
@@ -20,49 +20,51 @@
 #include "test_macros.h"
 #include "min_allocator.h"
 
-template<class Container>
-void do_insert_const_lvalue_test()
-{
-    typedef Container C;
-    typedef typename C::iterator R;
-    typedef typename C::value_type VT;
-    C c;
-    typename C::const_iterator e = c.end();
-    const VT v1(3.5, 3);
-    R r = c.insert(e, v1);
-    assert(c.size() == 1);
-    assert(r->first == 3.5);
-    assert(r->second == 3);
+template <class Container>
+void do_insert_const_lvalue_test() {
+  typedef Container C;
+  typedef typename C::iterator R;
+  typedef typename C::value_type VT;
+  C c;
+  typename C::const_iterator e = c.end();
+  const VT v1(3.5, 3);
+  R r = c.insert(e, v1);
+  assert(c.size() == 1);
+  assert(r->first == 3.5);
+  assert(r->second == 3);
 
-    const VT v2(3.5, 4);
-    r = c.insert(r, v2);
-    assert(c.size() == 2);
-    assert(r->first == 3.5);
-    assert(r->second == 4);
+  const VT v2(3.5, 4);
+  r = c.insert(r, v2);
+  assert(c.size() == 2);
+  assert(r->first == 3.5);
+  assert(r->second == 4);
 
-    const VT v3(4.5, 4);
-    r = c.insert(c.end(), v3);
-    assert(c.size() == 3);
-    assert(r->first == 4.5);
-    assert(r->second == 4);
+  const VT v3(4.5, 4);
+  r = c.insert(c.end(), v3);
+  assert(c.size() == 3);
+  assert(r->first == 4.5);
+  assert(r->second == 4);
 
-    const VT v4(5.5, 4);
-    r = c.insert(c.end(), v4);
-    assert(c.size() == 4);
-    assert(r->first == 5.5);
-    assert(r->second == 4);
+  const VT v4(5.5, 4);
+  r = c.insert(c.end(), v4);
+  assert(c.size() == 4);
+  assert(r->first == 5.5);
+  assert(r->second == 4);
 }
 
-int main(int, char**)
-{
-    do_insert_const_lvalue_test<std::unordered_multimap<double, int> >();
+int main(int, char**) {
+  do_insert_const_lvalue_test<std::unordered_multimap<double, int> >();
 #if TEST_STD_VER >= 11
-    {
-        typedef std::unordered_multimap<double, int, std::hash<double>, std::equal_to<double>,
-                            min_allocator<std::pair<const double, int>>> C;
-        do_insert_const_lvalue_test<C>();
-    }
+  {
+    typedef std::unordered_multimap<double,
+                                    int,
+                                    std::hash<double>,
+                                    std::equal_to<double>,
+                                    min_allocator<std::pair<const double, int>>>
+        C;
+    do_insert_const_lvalue_test<C>();
+  }
 #endif
 
-    return 0;
+  return 0;
 }
diff --git a/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.modifiers/insert_hint_rvalue.pass.cpp b/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.modifiers/insert_hint_rvalue.pass.cpp
index 6fa3b45fab779..63a3624b17437 100644
--- a/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.modifiers/insert_hint_rvalue.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.modifiers/insert_hint_rvalue.pass.cpp
@@ -25,139 +25,146 @@
 #include "MoveOnly.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef std::unordered_multimap<double, int> C;
-        typedef C::iterator R;
-        typedef std::pair<double, short> P;
-        C c;
-        C::const_iterator e = c.end();
-        R r = c.insert(e, P(3.5, static_cast<short>(3)));
-        assert(c.size() == 1);
-        assert(r->first == 3.5);
-        assert(r->second == 3);
-
-        r = c.insert(r, P(3.5, static_cast<short>(4)));
-        assert(c.size() == 2);
-        assert(r->first == 3.5);
-        assert(r->second == 4);
-
-        r = c.insert(c.end(), P(4.5, static_cast<short>(4)));
-        assert(c.size() == 3);
-        assert(r->first == 4.5);
-        assert(r->second == 4);
-
-        r = c.insert(c.end(), P(5.5, static_cast<short>(4)));
-        assert(c.size() == 4);
-        assert(r->first == 5.5);
-        assert(r->second == 4);
-    }
-    {
-        typedef std::unordered_multimap<MoveOnly, MoveOnly> C;
-        typedef C::iterator R;
-        typedef std::pair<MoveOnly, MoveOnly> P;
-        C c;
-        C::const_iterator e = c.end();
-        R r = c.insert(e, P(3, 3));
-        assert(c.size() == 1);
-        assert(r->first == 3);
-        assert(r->second == 3);
-
-        r = c.insert(r, P(3, 4));
-        assert(c.size() == 2);
-        assert(r->first == 3);
-        assert(r->second == 4);
-
-        r = c.insert(c.end(), P(4, 4));
-        assert(c.size() == 3);
-        assert(r->first == 4);
-        assert(r->second == 4);
-
-        r = c.insert(c.end(), P(5, 4));
-        assert(c.size() == 4);
-        assert(r->first == 5);
-        assert(r->second == 4);
-    }
-    {
-        typedef std::unordered_multimap<double, int, std::hash<double>, std::equal_to<double>,
-                            min_allocator<std::pair<const double, int>>> C;
-        typedef C::iterator R;
-        typedef std::pair<double, short> P;
-        C c;
-        C::const_iterator e = c.end();
-        R r = c.insert(e, P(3.5, static_cast<short>(3)));
-        assert(c.size() == 1);
-        assert(r->first == 3.5);
-        assert(r->second == 3);
-
-        r = c.insert(r, P(3.5, static_cast<short>(4)));
-        assert(c.size() == 2);
-        assert(r->first == 3.5);
-        assert(r->second == 4);
-
-        r = c.insert(c.end(), P(4.5, static_cast<short>(4)));
-        assert(c.size() == 3);
-        assert(r->first == 4.5);
-        assert(r->second == 4);
-
-        r = c.insert(c.end(), P(5.5, static_cast<short>(4)));
-        assert(c.size() == 4);
-        assert(r->first == 5.5);
-        assert(r->second == 4);
-    }
-    {
-        typedef std::unordered_multimap<MoveOnly, MoveOnly, std::hash<MoveOnly>, std::equal_to<MoveOnly>,
-                            min_allocator<std::pair<const MoveOnly, MoveOnly>>> C;
-        typedef C::iterator R;
-        typedef std::pair<MoveOnly, MoveOnly> P;
-        C c;
-        C::const_iterator e = c.end();
-        R r = c.insert(e, P(3, 3));
-        assert(c.size() == 1);
-        assert(r->first == 3);
-        assert(r->second == 3);
-
-        r = c.insert(r, P(3, 4));
-        assert(c.size() == 2);
-        assert(r->first == 3);
-        assert(r->second == 4);
-
-        r = c.insert(c.end(), P(4, 4));
-        assert(c.size() == 3);
-        assert(r->first == 4);
-        assert(r->second == 4);
-
-        r = c.insert(c.end(), P(5, 4));
-        assert(c.size() == 4);
-        assert(r->first == 5);
-        assert(r->second == 4);
-    }
-    {
-        typedef std::unordered_multimap<double, int> C;
-        typedef C::iterator R;
-        C c;
-        C::const_iterator e = c.end();
-        R r = c.insert(e, {3.5, 3});
-        assert(c.size() == 1);
-        assert(r->first == 3.5);
-        assert(r->second == 3);
-
-        r = c.insert(r, {3.5, 4});
-        assert(c.size() == 2);
-        assert(r->first == 3.5);
-        assert(r->second == 4);
-
-        r = c.insert(c.end(), {4.5, 4});
-        assert(c.size() == 3);
-        assert(r->first == 4.5);
-        assert(r->second == 4);
-
-        r = c.insert(c.end(), {5.5, 4});
-        assert(c.size() == 4);
-        assert(r->first == 5.5);
-        assert(r->second == 4);
-    }
-
-    return 0;
+int main(int, char**) {
+  {
+    typedef std::unordered_multimap<double, int> C;
+    typedef C::iterator R;
+    typedef std::pair<double, short> P;
+    C c;
+    C::const_iterator e = c.end();
+    R r                 = c.insert(e, P(3.5, static_cast<short>(3)));
+    assert(c.size() == 1);
+    assert(r->first == 3.5);
+    assert(r->second == 3);
+
+    r = c.insert(r, P(3.5, static_cast<short>(4)));
+    assert(c.size() == 2);
+    assert(r->first == 3.5);
+    assert(r->second == 4);
+
+    r = c.insert(c.end(), P(4.5, static_cast<short>(4)));
+    assert(c.size() == 3);
+    assert(r->first == 4.5);
+    assert(r->second == 4);
+
+    r = c.insert(c.end(), P(5.5, static_cast<short>(4)));
+    assert(c.size() == 4);
+    assert(r->first == 5.5);
+    assert(r->second == 4);
+  }
+  {
+    typedef std::unordered_multimap<MoveOnly, MoveOnly> C;
+    typedef C::iterator R;
+    typedef std::pair<MoveOnly, MoveOnly> P;
+    C c;
+    C::const_iterator e = c.end();
+    R r                 = c.insert(e, P(3, 3));
+    assert(c.size() == 1);
+    assert(r->first == 3);
+    assert(r->second == 3);
+
+    r = c.insert(r, P(3, 4));
+    assert(c.size() == 2);
+    assert(r->first == 3);
+    assert(r->second == 4);
+
+    r = c.insert(c.end(), P(4, 4));
+    assert(c.size() == 3);
+    assert(r->first == 4);
+    assert(r->second == 4);
+
+    r = c.insert(c.end(), P(5, 4));
+    assert(c.size() == 4);
+    assert(r->first == 5);
+    assert(r->second == 4);
+  }
+  {
+    typedef std::unordered_multimap<double,
+                                    int,
+                                    std::hash<double>,
+                                    std::equal_to<double>,
+                                    min_allocator<std::pair<const double, int>>>
+        C;
+    typedef C::iterator R;
+    typedef std::pair<double, short> P;
+    C c;
+    C::const_iterator e = c.end();
+    R r                 = c.insert(e, P(3.5, static_cast<short>(3)));
+    assert(c.size() == 1);
+    assert(r->first == 3.5);
+    assert(r->second == 3);
+
+    r = c.insert(r, P(3.5, static_cast<short>(4)));
+    assert(c.size() == 2);
+    assert(r->first == 3.5);
+    assert(r->second == 4);
+
+    r = c.insert(c.end(), P(4.5, static_cast<short>(4)));
+    assert(c.size() == 3);
+    assert(r->first == 4.5);
+    assert(r->second == 4);
+
+    r = c.insert(c.end(), P(5.5, static_cast<short>(4)));
+    assert(c.size() == 4);
+    assert(r->first == 5.5);
+    assert(r->second == 4);
+  }
+  {
+    typedef std::unordered_multimap<MoveOnly,
+                                    MoveOnly,
+                                    std::hash<MoveOnly>,
+                                    std::equal_to<MoveOnly>,
+                                    min_allocator<std::pair<const MoveOnly, MoveOnly>>>
+        C;
+    typedef C::iterator R;
+    typedef std::pair<MoveOnly, MoveOnly> P;
+    C c;
+    C::const_iterator e = c.end();
+    R r                 = c.insert(e, P(3, 3));
+    assert(c.size() == 1);
+    assert(r->first == 3);
+    assert(r->second == 3);
+
+    r = c.insert(r, P(3, 4));
+    assert(c.size() == 2);
+    assert(r->first == 3);
+    assert(r->second == 4);
+
+    r = c.insert(c.end(), P(4, 4));
+    assert(c.size() == 3);
+    assert(r->first == 4);
+    assert(r->second == 4);
+
+    r = c.insert(c.end(), P(5, 4));
+    assert(c.size() == 4);
+    assert(r->first == 5);
+    assert(r->second == 4);
+  }
+  {
+    typedef std::unordered_multimap<double, int> C;
+    typedef C::iterator R;
+    C c;
+    C::const_iterator e = c.end();
+    R r                 = c.insert(e, {3.5, 3});
+    assert(c.size() == 1);
+    assert(r->first == 3.5);
+    assert(r->second == 3);
+
+    r = c.insert(r, {3.5, 4});
+    assert(c.size() == 2);
+    assert(r->first == 3.5);
+    assert(r->second == 4);
+
+    r = c.insert(c.end(), {4.5, 4});
+    assert(c.size() == 3);
+    assert(r->first == 4.5);
+    assert(r->second == 4);
+
+    r = c.insert(c.end(), {5.5, 4});
+    assert(c.size() == 4);
+    assert(r->first == 5.5);
+    assert(r->second == 4);
+  }
+
+  return 0;
 }
diff --git a/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.modifiers/insert_init.pass.cpp b/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.modifiers/insert_init.pass.cpp
index 3f7af5e5eebde..18d42854d1325 100644
--- a/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.modifiers/insert_init.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.modifiers/insert_init.pass.cpp
@@ -27,89 +27,88 @@
 #include "test_iterators.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef std::unordered_multimap<int, std::string> C;
-        typedef std::pair<int, std::string> P;
-        C c;
-        c.insert(
-                    {
-                        P(1, "one"),
-                        P(2, "two"),
-                        P(3, "three"),
-                        P(4, "four"),
-                        P(1, "four"),
-                        P(2, "four"),
-                    }
-                );
-        assert(c.size() == 6);
-        typedef std::pair<C::iterator, C::iterator> Eq;
-        Eq eq = c.equal_range(1);
-        assert(std::distance(eq.first, eq.second) == 2);
-        std::multiset<std::string> s;
-        s.insert("one");
-        s.insert("four");
-        CheckConsecutiveKeys<C::const_iterator>(c.find(1), c.end(), 1, s);
-        eq = c.equal_range(2);
-        assert(std::distance(eq.first, eq.second) == 2);
-        s.insert("two");
-        s.insert("four");
-        CheckConsecutiveKeys<C::const_iterator>(c.find(2), c.end(), 2, s);
-        eq = c.equal_range(3);
-        assert(std::distance(eq.first, eq.second) == 1);
-        C::iterator k = eq.first;
-        assert(k->first == 3);
-        assert(k->second == "three");
-        eq = c.equal_range(4);
-        assert(std::distance(eq.first, eq.second) == 1);
-        k = eq.first;
-        assert(k->first == 4);
-        assert(k->second == "four");
-        assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-        assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
-    }
-    {
-        typedef std::unordered_multimap<int, std::string, std::hash<int>, std::equal_to<int>,
-                            min_allocator<std::pair<const int, std::string>>> C;
-        typedef std::pair<int, std::string> P;
-        C c;
-        c.insert(
-                    {
-                        P(1, "one"),
-                        P(2, "two"),
-                        P(3, "three"),
-                        P(4, "four"),
-                        P(1, "four"),
-                        P(2, "four"),
-                    }
-                );
-        assert(c.size() == 6);
-        typedef std::pair<C::iterator, C::iterator> Eq;
-        Eq eq = c.equal_range(1);
-        assert(std::distance(eq.first, eq.second) == 2);
-        std::multiset<std::string> s;
-        s.insert("one");
-        s.insert("four");
-        CheckConsecutiveKeys<C::const_iterator>(c.find(1), c.end(), 1, s);
-        eq = c.equal_range(2);
-        assert(std::distance(eq.first, eq.second) == 2);
-        s.insert("two");
-        s.insert("four");
-        CheckConsecutiveKeys<C::const_iterator>(c.find(2), c.end(), 2, s);
-        eq = c.equal_range(3);
-        assert(std::distance(eq.first, eq.second) == 1);
-        C::iterator k = eq.first;
-        assert(k->first == 3);
-        assert(k->second == "three");
-        eq = c.equal_range(4);
-        assert(std::distance(eq.first, eq.second) == 1);
-        k = eq.first;
-        assert(k->first == 4);
-        assert(k->second == "four");
-        assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-        assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
-    }
+int main(int, char**) {
+  {
+    typedef std::unordered_multimap<int, std::string> C;
+    typedef std::pair<int, std::string> P;
+    C c;
+    c.insert({
+        P(1, "one"),
+        P(2, "two"),
+        P(3, "three"),
+        P(4, "four"),
+        P(1, "four"),
+        P(2, "four"),
+    });
+    assert(c.size() == 6);
+    typedef std::pair<C::iterator, C::iterator> Eq;
+    Eq eq = c.equal_range(1);
+    assert(std::distance(eq.first, eq.second) == 2);
+    std::multiset<std::string> s;
+    s.insert("one");
+    s.insert("four");
+    CheckConsecutiveKeys<C::const_iterator>(c.find(1), c.end(), 1, s);
+    eq = c.equal_range(2);
+    assert(std::distance(eq.first, eq.second) == 2);
+    s.insert("two");
+    s.insert("four");
+    CheckConsecutiveKeys<C::const_iterator>(c.find(2), c.end(), 2, s);
+    eq = c.equal_range(3);
+    assert(std::distance(eq.first, eq.second) == 1);
+    C::iterator k = eq.first;
+    assert(k->first == 3);
+    assert(k->second == "three");
+    eq = c.equal_range(4);
+    assert(std::distance(eq.first, eq.second) == 1);
+    k = eq.first;
+    assert(k->first == 4);
+    assert(k->second == "four");
+    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
+    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
+  }
+  {
+    typedef std::unordered_multimap<int,
+                                    std::string,
+                                    std::hash<int>,
+                                    std::equal_to<int>,
+                                    min_allocator<std::pair<const int, std::string>>>
+        C;
+    typedef std::pair<int, std::string> P;
+    C c;
+    c.insert({
+        P(1, "one"),
+        P(2, "two"),
+        P(3, "three"),
+        P(4, "four"),
+        P(1, "four"),
+        P(2, "four"),
+    });
+    assert(c.size() == 6);
+    typedef std::pair<C::iterator, C::iterator> Eq;
+    Eq eq = c.equal_range(1);
+    assert(std::distance(eq.first, eq.second) == 2);
+    std::multiset<std::string> s;
+    s.insert("one");
+    s.insert("four");
+    CheckConsecutiveKeys<C::const_iterator>(c.find(1), c.end(), 1, s);
+    eq = c.equal_range(2);
+    assert(std::distance(eq.first, eq.second) == 2);
+    s.insert("two");
+    s.insert("four");
+    CheckConsecutiveKeys<C::const_iterator>(c.find(2), c.end(), 2, s);
+    eq = c.equal_range(3);
+    assert(std::distance(eq.first, eq.second) == 1);
+    C::iterator k = eq.first;
+    assert(k->first == 3);
+    assert(k->second == "three");
+    eq = c.equal_range(4);
+    assert(std::distance(eq.first, eq.second) == 1);
+    k = eq.first;
+    assert(k->first == 4);
+    assert(k->second == "four");
+    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
+    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
+  }
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.modifiers/insert_iter_iter.pass.cpp b/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.modifiers/insert_iter_iter.pass.cpp
index 519ed7a7bf20e..13a8470316222 100644
--- a/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.modifiers/insert_iter_iter.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.modifiers/insert_iter_iter.pass.cpp
@@ -26,90 +26,91 @@
 #include "test_iterators.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef std::unordered_multimap<int, std::string> C;
-        typedef std::pair<int, std::string> P;
-        P a[] =
-        {
-            P(1, "one"),
-            P(2, "two"),
-            P(3, "three"),
-            P(4, "four"),
-            P(1, "four"),
-            P(2, "four"),
-        };
-        C c;
-        c.insert(cpp17_input_iterator<P*>(a), cpp17_input_iterator<P*>(a + sizeof(a)/sizeof(a[0])));
-        assert(c.size() == 6);
-        typedef std::pair<C::iterator, C::iterator> Eq;
-        Eq eq = c.equal_range(1);
-        assert(std::distance(eq.first, eq.second) == 2);
-        std::multiset<std::string> s;
-        s.insert("one");
-        s.insert("four");
-        CheckConsecutiveKeys<C::const_iterator>(c.find(1), c.end(), 1, s);
-        eq = c.equal_range(2);
-        assert(std::distance(eq.first, eq.second) == 2);
-        s.insert("two");
-        s.insert("four");
-        CheckConsecutiveKeys<C::const_iterator>(c.find(2), c.end(), 2, s);
-        eq = c.equal_range(3);
-        assert(std::distance(eq.first, eq.second) == 1);
-        C::iterator k = eq.first;
-        assert(k->first == 3);
-        assert(k->second == "three");
-        eq = c.equal_range(4);
-        assert(std::distance(eq.first, eq.second) == 1);
-        k = eq.first;
-        assert(k->first == 4);
-        assert(k->second == "four");
-        assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-        assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
-    }
+int main(int, char**) {
+  {
+    typedef std::unordered_multimap<int, std::string> C;
+    typedef std::pair<int, std::string> P;
+    P a[] = {
+        P(1, "one"),
+        P(2, "two"),
+        P(3, "three"),
+        P(4, "four"),
+        P(1, "four"),
+        P(2, "four"),
+    };
+    C c;
+    c.insert(cpp17_input_iterator<P*>(a), cpp17_input_iterator<P*>(a + sizeof(a) / sizeof(a[0])));
+    assert(c.size() == 6);
+    typedef std::pair<C::iterator, C::iterator> Eq;
+    Eq eq = c.equal_range(1);
+    assert(std::distance(eq.first, eq.second) == 2);
+    std::multiset<std::string> s;
+    s.insert("one");
+    s.insert("four");
+    CheckConsecutiveKeys<C::const_iterator>(c.find(1), c.end(), 1, s);
+    eq = c.equal_range(2);
+    assert(std::distance(eq.first, eq.second) == 2);
+    s.insert("two");
+    s.insert("four");
+    CheckConsecutiveKeys<C::const_iterator>(c.find(2), c.end(), 2, s);
+    eq = c.equal_range(3);
+    assert(std::distance(eq.first, eq.second) == 1);
+    C::iterator k = eq.first;
+    assert(k->first == 3);
+    assert(k->second == "three");
+    eq = c.equal_range(4);
+    assert(std::distance(eq.first, eq.second) == 1);
+    k = eq.first;
+    assert(k->first == 4);
+    assert(k->second == "four");
+    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
+    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
+  }
 #if TEST_STD_VER >= 11
-    {
-        typedef std::unordered_multimap<int, std::string, std::hash<int>, std::equal_to<int>,
-                            min_allocator<std::pair<const int, std::string>>> C;
-        typedef std::pair<int, std::string> P;
-        P a[] =
-        {
-            P(1, "one"),
-            P(2, "two"),
-            P(3, "three"),
-            P(4, "four"),
-            P(1, "four"),
-            P(2, "four"),
-        };
-        C c;
-        c.insert(cpp17_input_iterator<P*>(a), cpp17_input_iterator<P*>(a + sizeof(a)/sizeof(a[0])));
-        assert(c.size() == 6);
-        typedef std::pair<C::iterator, C::iterator> Eq;
-        Eq eq = c.equal_range(1);
-        assert(std::distance(eq.first, eq.second) == 2);
-        std::multiset<std::string> s;
-        s.insert("one");
-        s.insert("four");
-        CheckConsecutiveKeys<C::const_iterator>(c.find(1), c.end(), 1, s);
-        eq = c.equal_range(2);
-        assert(std::distance(eq.first, eq.second) == 2);
-        s.insert("two");
-        s.insert("four");
-        CheckConsecutiveKeys<C::const_iterator>(c.find(2), c.end(), 2, s);
-        eq = c.equal_range(3);
-        assert(std::distance(eq.first, eq.second) == 1);
-        C::iterator k = eq.first;
-        assert(k->first == 3);
-        assert(k->second == "three");
-        eq = c.equal_range(4);
-        assert(std::distance(eq.first, eq.second) == 1);
-        k = eq.first;
-        assert(k->first == 4);
-        assert(k->second == "four");
-        assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-        assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
-    }
+  {
+    typedef std::unordered_multimap<int,
+                                    std::string,
+                                    std::hash<int>,
+                                    std::equal_to<int>,
+                                    min_allocator<std::pair<const int, std::string>>>
+        C;
+    typedef std::pair<int, std::string> P;
+    P a[] = {
+        P(1, "one"),
+        P(2, "two"),
+        P(3, "three"),
+        P(4, "four"),
+        P(1, "four"),
+        P(2, "four"),
+    };
+    C c;
+    c.insert(cpp17_input_iterator<P*>(a), cpp17_input_iterator<P*>(a + sizeof(a) / sizeof(a[0])));
+    assert(c.size() == 6);
+    typedef std::pair<C::iterator, C::iterator> Eq;
+    Eq eq = c.equal_range(1);
+    assert(std::distance(eq.first, eq.second) == 2);
+    std::multiset<std::string> s;
+    s.insert("one");
+    s.insert("four");
+    CheckConsecutiveKeys<C::const_iterator>(c.find(1), c.end(), 1, s);
+    eq = c.equal_range(2);
+    assert(std::distance(eq.first, eq.second) == 2);
+    s.insert("two");
+    s.insert("four");
+    CheckConsecutiveKeys<C::const_iterator>(c.find(2), c.end(), 2, s);
+    eq = c.equal_range(3);
+    assert(std::distance(eq.first, eq.second) == 1);
+    C::iterator k = eq.first;
+    assert(k->first == 3);
+    assert(k->second == "three");
+    eq = c.equal_range(4);
+    assert(std::distance(eq.first, eq.second) == 1);
+    k = eq.first;
+    assert(k->first == 4);
+    assert(k->second == "four");
+    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
+    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.modifiers/insert_node_type.pass.cpp b/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.modifiers/insert_node_type.pass.cpp
index d9131c1f68f68..a9fdc89688ad9 100644
--- a/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.modifiers/insert_node_type.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.modifiers/insert_node_type.pass.cpp
@@ -20,60 +20,54 @@
 
 template <class Container>
 typename Container::node_type
-node_factory(typename Container::key_type const& key,
-             typename Container::mapped_type const& mapped)
-{
-    static Container c;
-    auto it = c.insert({key, mapped});
-    return c.extract(it);
+node_factory(typename Container::key_type const& key, typename Container::mapped_type const& mapped) {
+  static Container c;
+  auto it = c.insert({key, mapped});
+  return c.extract(it);
 }
 
 template <class Container>
-void test(Container& c)
-{
-    auto* nf = &node_factory<Container>;
+void test(Container& c) {
+  auto* nf = &node_factory<Container>;
 
-    for (int i = 0; i != 10; ++i)
-    {
-        typename Container::node_type node = nf(i, i + 1);
-        assert(!node.empty());
-        typename Container::iterator it = c.insert(std::move(node));
-        assert(node.empty());
-        assert(it == c.find(i) && it != c.end());
-        assert(it->first == i && it->second == i + 1);
-    }
+  for (int i = 0; i != 10; ++i) {
+    typename Container::node_type node = nf(i, i + 1);
+    assert(!node.empty());
+    typename Container::iterator it = c.insert(std::move(node));
+    assert(node.empty());
+    assert(it == c.find(i) && it != c.end());
+    assert(it->first == i && it->second == i + 1);
+  }
 
-    assert(c.size() == 10);
+  assert(c.size() == 10);
 
-    { // Insert empty node.
-        typename Container::node_type def;
-        auto it = c.insert(std::move(def));
-        assert(def.empty());
-        assert(it == c.end());
-    }
+  { // Insert empty node.
+    typename Container::node_type def;
+    auto it = c.insert(std::move(def));
+    assert(def.empty());
+    assert(it == c.end());
+  }
 
-    { // Insert duplicate node.
-        typename Container::node_type dupl = nf(0, 42);
-        auto it = c.insert(std::move(dupl));
-        assert(dupl.empty());
-        assert(it != c.end() && it->second == 42);
-    }
+  { // Insert duplicate node.
+    typename Container::node_type dupl = nf(0, 42);
+    auto it                            = c.insert(std::move(dupl));
+    assert(dupl.empty());
+    assert(it != c.end() && it->second == 42);
+  }
 
-    assert(c.size() == 11);
-    assert(c.count(0) == 2);
-    for (int i = 1; i != 10; ++i)
-    {
-        assert(c.count(i) == 1);
-        assert(c.find(i)->second == i + 1);
-    }
+  assert(c.size() == 11);
+  assert(c.count(0) == 2);
+  for (int i = 1; i != 10; ++i) {
+    assert(c.count(i) == 1);
+    assert(c.find(i)->second == i + 1);
+  }
 }
 
-int main(int, char**)
-{
-    std::unordered_multimap<int, int> m;
-    test(m);
-    std::unordered_multimap<int, int, std::hash<int>, std::equal_to<int>, min_allocator<std::pair<const int, int>>> m2;
-    test(m2);
+int main(int, char**) {
+  std::unordered_multimap<int, int> m;
+  test(m);
+  std::unordered_multimap<int, int, std::hash<int>, std::equal_to<int>, min_allocator<std::pair<const int, int>>> m2;
+  test(m2);
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.modifiers/insert_node_type_hint.pass.cpp b/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.modifiers/insert_node_type_hint.pass.cpp
index 461f7a33d35d0..c329001a086ae 100644
--- a/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.modifiers/insert_node_type_hint.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.modifiers/insert_node_type_hint.pass.cpp
@@ -20,46 +20,40 @@
 
 template <class Container>
 typename Container::node_type
-node_factory(typename Container::key_type const& key,
-             typename Container::mapped_type const& mapped)
-{
-    static Container c;
-    auto it = c.insert({key, mapped});
-    return c.extract(it);
+node_factory(typename Container::key_type const& key, typename Container::mapped_type const& mapped) {
+  static Container c;
+  auto it = c.insert({key, mapped});
+  return c.extract(it);
 }
 
 template <class Container>
-void test(Container& c)
-{
-    auto* nf = &node_factory<Container>;
-
-    for (int i = 0; i != 10; ++i)
-    {
-        typename Container::node_type node = nf(i, i + 1);
-        assert(!node.empty());
-        std::size_t prev = c.size();
-        auto it = c.insert(c.end(), std::move(node));
-        assert(node.empty());
-        assert(prev + 1 == c.size());
-        assert(it->first == i);
-        assert(it->second == i + 1);
-    }
-
-    assert(c.size() == 10);
-
-    for (int i = 0; i != 10; ++i)
-    {
-        assert(c.count(i) == 1);
-        assert(c.find(i)->second == i + 1);
-    }
+void test(Container& c) {
+  auto* nf = &node_factory<Container>;
+
+  for (int i = 0; i != 10; ++i) {
+    typename Container::node_type node = nf(i, i + 1);
+    assert(!node.empty());
+    std::size_t prev = c.size();
+    auto it          = c.insert(c.end(), std::move(node));
+    assert(node.empty());
+    assert(prev + 1 == c.size());
+    assert(it->first == i);
+    assert(it->second == i + 1);
+  }
+
+  assert(c.size() == 10);
+
+  for (int i = 0; i != 10; ++i) {
+    assert(c.count(i) == 1);
+    assert(c.find(i)->second == i + 1);
+  }
 }
 
-int main(int, char**)
-{
-    std::unordered_multimap<int, int> m;
-    test(m);
-    std::unordered_multimap<int, int, std::hash<int>, std::equal_to<int>, min_allocator<std::pair<const int, int>>> m2;
-    test(m2);
+int main(int, char**) {
+  std::unordered_multimap<int, int> m;
+  test(m);
+  std::unordered_multimap<int, int, std::hash<int>, std::equal_to<int>, min_allocator<std::pair<const int, int>>> m2;
+  test(m2);
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.modifiers/insert_range.pass.cpp b/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.modifiers/insert_range.pass.cpp
index e6924ad52f6a7..8f269a7dd0151 100644
--- a/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.modifiers/insert_range.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.modifiers/insert_range.pass.cpp
@@ -23,10 +23,13 @@
 int main(int, char**) {
   // Note: we want to use a pair with non-const elements for input (an assignable type is a lot more convenient) but
   // have to use the exact `value_type` of the map (that is, `pair<const K, V>`) for the allocator.
-  using Pair = std::pair<int, char>;
+  using Pair      = std::pair<int, char>;
   using ConstPair = std::pair<const int, char>;
   for_all_iterators_and_allocators<ConstPair, const Pair*>([]<class Iter, class Sent, class Alloc>() {
-    test_map_set_insert_range<std::unordered_multimap<int, char, test_hash<int>, test_equal_to<int>, Alloc>, Pair, Iter, Sent>(/*allow_duplicates=*/true);
+    test_map_set_insert_range<std::unordered_multimap<int, char, test_hash<int>, test_equal_to<int>, Alloc>,
+                              Pair,
+                              Iter,
+                              Sent>(/*allow_duplicates=*/true);
   });
 
   static_assert(test_map_constraints_insert_range<std::unordered_multimap, int, int, char, double>());
@@ -38,4 +41,3 @@ int main(int, char**) {
 
   return 0;
 }
-
diff --git a/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.modifiers/insert_rvalue.pass.cpp b/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.modifiers/insert_rvalue.pass.cpp
index 89d842f9b9f6f..96754ff9bcc61 100644
--- a/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.modifiers/insert_rvalue.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.modifiers/insert_rvalue.pass.cpp
@@ -25,134 +25,141 @@
 #include "MoveOnly.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef std::unordered_multimap<double, int> C;
-        typedef C::iterator R;
-        typedef std::pair<double, short> P;
-        C c;
-        R r = c.insert(P(3.5, static_cast<short>(3)));
-        assert(c.size() == 1);
-        assert(r->first == 3.5);
-        assert(r->second == 3);
-
-        r = c.insert(P(3.5, static_cast<short>(4)));
-        assert(c.size() == 2);
-        assert(r->first == 3.5);
-        assert(r->second == 4);
-
-        r = c.insert(P(4.5, static_cast<short>(4)));
-        assert(c.size() == 3);
-        assert(r->first == 4.5);
-        assert(r->second == 4);
-
-        r = c.insert(P(5.5, static_cast<short>(4)));
-        assert(c.size() == 4);
-        assert(r->first == 5.5);
-        assert(r->second == 4);
-    }
-    {
-        typedef std::unordered_multimap<MoveOnly, MoveOnly> C;
-        typedef C::iterator R;
-        typedef std::pair<MoveOnly, MoveOnly> P;
-        C c;
-        R r = c.insert(P(3, 3));
-        assert(c.size() == 1);
-        assert(r->first == 3);
-        assert(r->second == 3);
-
-        r = c.insert(P(3, 4));
-        assert(c.size() == 2);
-        assert(r->first == 3);
-        assert(r->second == 4);
-
-        r = c.insert(P(4, 4));
-        assert(c.size() == 3);
-        assert(r->first == 4);
-        assert(r->second == 4);
-
-        r = c.insert(P(5, 4));
-        assert(c.size() == 4);
-        assert(r->first == 5);
-        assert(r->second == 4);
-    }
-    {
-        typedef std::unordered_multimap<double, int, std::hash<double>, std::equal_to<double>,
-                            min_allocator<std::pair<const double, int>>> C;
-        typedef C::iterator R;
-        typedef std::pair<double, short> P;
-        C c;
-        R r = c.insert(P(3.5, static_cast<short>(3)));
-        assert(c.size() == 1);
-        assert(r->first == 3.5);
-        assert(r->second == 3);
-
-        r = c.insert(P(3.5, static_cast<short>(4)));
-        assert(c.size() == 2);
-        assert(r->first == 3.5);
-        assert(r->second == 4);
-
-        r = c.insert(P(4.5, static_cast<short>(4)));
-        assert(c.size() == 3);
-        assert(r->first == 4.5);
-        assert(r->second == 4);
-
-        r = c.insert(P(5.5, static_cast<short>(4)));
-        assert(c.size() == 4);
-        assert(r->first == 5.5);
-        assert(r->second == 4);
-    }
-    {
-        typedef std::unordered_multimap<MoveOnly, MoveOnly, std::hash<MoveOnly>, std::equal_to<MoveOnly>,
-                            min_allocator<std::pair<const MoveOnly, MoveOnly>>> C;
-        typedef C::iterator R;
-        typedef std::pair<MoveOnly, MoveOnly> P;
-        C c;
-        R r = c.insert(P(3, 3));
-        assert(c.size() == 1);
-        assert(r->first == 3);
-        assert(r->second == 3);
-
-        r = c.insert(P(3, 4));
-        assert(c.size() == 2);
-        assert(r->first == 3);
-        assert(r->second == 4);
-
-        r = c.insert(P(4, 4));
-        assert(c.size() == 3);
-        assert(r->first == 4);
-        assert(r->second == 4);
-
-        r = c.insert(P(5, 4));
-        assert(c.size() == 4);
-        assert(r->first == 5);
-        assert(r->second == 4);
-    }
-    {
-        typedef std::unordered_multimap<double, MoveOnly> C;
-        typedef C::iterator R;
-        C c;
-        R r = c.insert({3.5, 3});
-        assert(c.size() == 1);
-        assert(r->first == 3.5);
-        assert(r->second == 3);
-
-        r = c.insert({3.5, 4});
-        assert(c.size() == 2);
-        assert(r->first == 3.5);
-        assert(r->second == 4);
-
-        r = c.insert({4.5, 4});
-        assert(c.size() == 3);
-        assert(r->first == 4.5);
-        assert(r->second == 4);
-
-        r = c.insert({5.5, 4});
-        assert(c.size() == 4);
-        assert(r->first == 5.5);
-        assert(r->second == 4);
-    }
+int main(int, char**) {
+  {
+    typedef std::unordered_multimap<double, int> C;
+    typedef C::iterator R;
+    typedef std::pair<double, short> P;
+    C c;
+    R r = c.insert(P(3.5, static_cast<short>(3)));
+    assert(c.size() == 1);
+    assert(r->first == 3.5);
+    assert(r->second == 3);
+
+    r = c.insert(P(3.5, static_cast<short>(4)));
+    assert(c.size() == 2);
+    assert(r->first == 3.5);
+    assert(r->second == 4);
+
+    r = c.insert(P(4.5, static_cast<short>(4)));
+    assert(c.size() == 3);
+    assert(r->first == 4.5);
+    assert(r->second == 4);
+
+    r = c.insert(P(5.5, static_cast<short>(4)));
+    assert(c.size() == 4);
+    assert(r->first == 5.5);
+    assert(r->second == 4);
+  }
+  {
+    typedef std::unordered_multimap<MoveOnly, MoveOnly> C;
+    typedef C::iterator R;
+    typedef std::pair<MoveOnly, MoveOnly> P;
+    C c;
+    R r = c.insert(P(3, 3));
+    assert(c.size() == 1);
+    assert(r->first == 3);
+    assert(r->second == 3);
+
+    r = c.insert(P(3, 4));
+    assert(c.size() == 2);
+    assert(r->first == 3);
+    assert(r->second == 4);
+
+    r = c.insert(P(4, 4));
+    assert(c.size() == 3);
+    assert(r->first == 4);
+    assert(r->second == 4);
+
+    r = c.insert(P(5, 4));
+    assert(c.size() == 4);
+    assert(r->first == 5);
+    assert(r->second == 4);
+  }
+  {
+    typedef std::unordered_multimap<double,
+                                    int,
+                                    std::hash<double>,
+                                    std::equal_to<double>,
+                                    min_allocator<std::pair<const double, int>>>
+        C;
+    typedef C::iterator R;
+    typedef std::pair<double, short> P;
+    C c;
+    R r = c.insert(P(3.5, static_cast<short>(3)));
+    assert(c.size() == 1);
+    assert(r->first == 3.5);
+    assert(r->second == 3);
+
+    r = c.insert(P(3.5, static_cast<short>(4)));
+    assert(c.size() == 2);
+    assert(r->first == 3.5);
+    assert(r->second == 4);
+
+    r = c.insert(P(4.5, static_cast<short>(4)));
+    assert(c.size() == 3);
+    assert(r->first == 4.5);
+    assert(r->second == 4);
+
+    r = c.insert(P(5.5, static_cast<short>(4)));
+    assert(c.size() == 4);
+    assert(r->first == 5.5);
+    assert(r->second == 4);
+  }
+  {
+    typedef std::unordered_multimap<MoveOnly,
+                                    MoveOnly,
+                                    std::hash<MoveOnly>,
+                                    std::equal_to<MoveOnly>,
+                                    min_allocator<std::pair<const MoveOnly, MoveOnly>>>
+        C;
+    typedef C::iterator R;
+    typedef std::pair<MoveOnly, MoveOnly> P;
+    C c;
+    R r = c.insert(P(3, 3));
+    assert(c.size() == 1);
+    assert(r->first == 3);
+    assert(r->second == 3);
+
+    r = c.insert(P(3, 4));
+    assert(c.size() == 2);
+    assert(r->first == 3);
+    assert(r->second == 4);
+
+    r = c.insert(P(4, 4));
+    assert(c.size() == 3);
+    assert(r->first == 4);
+    assert(r->second == 4);
+
+    r = c.insert(P(5, 4));
+    assert(c.size() == 4);
+    assert(r->first == 5);
+    assert(r->second == 4);
+  }
+  {
+    typedef std::unordered_multimap<double, MoveOnly> C;
+    typedef C::iterator R;
+    C c;
+    R r = c.insert({3.5, 3});
+    assert(c.size() == 1);
+    assert(r->first == 3.5);
+    assert(r->second == 3);
+
+    r = c.insert({3.5, 4});
+    assert(c.size() == 2);
+    assert(r->first == 3.5);
+    assert(r->second == 4);
+
+    r = c.insert({4.5, 4});
+    assert(c.size() == 3);
+    assert(r->first == 4.5);
+    assert(r->second == 4);
+
+    r = c.insert({5.5, 4});
+    assert(c.size() == 4);
+    assert(r->first == 5.5);
+    assert(r->second == 4);
+  }
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.modifiers/merge.pass.cpp b/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.modifiers/merge.pass.cpp
index e9c0c3fb75a2a..cbe8b0240fbef 100644
--- a/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.modifiers/merge.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.modifiers/merge.pass.cpp
@@ -27,131 +27,117 @@
 #include "Counter.h"
 
 template <class Map>
-bool map_equal(const Map& map, Map other)
-{
-    return map == other;
+bool map_equal(const Map& map, Map other) {
+  return map == other;
 }
 
 #ifndef TEST_HAS_NO_EXCEPTIONS
 template <class T>
-struct throw_hasher
-{
-    bool& should_throw_;
+struct throw_hasher {
+  bool& should_throw_;
 
-    throw_hasher(bool& should_throw) : should_throw_(should_throw) {}
+  throw_hasher(bool& should_throw) : should_throw_(should_throw) {}
 
-    std::size_t operator()(const T& p) const
-    {
-        if (should_throw_)
-            throw 0;
-        return std::hash<T>()(p);
-    }
+  std::size_t operator()(const T& p) const {
+    if (should_throw_)
+      throw 0;
+    return std::hash<T>()(p);
+  }
 };
 #endif
 
-int main(int, char**)
-{
-    {
-        std::unordered_multimap<int, int> src{{1, 0}, {3, 0}, {5, 0}};
-        std::unordered_multimap<int, int> dst{{2, 0}, {4, 0}, {5, 0}};
-        dst.merge(src);
-        assert(map_equal(src, {}));
-        assert(map_equal(dst, {{1, 0}, {2, 0}, {3, 0}, {4, 0}, {5, 0}, {5, 0}}));
-    }
+int main(int, char**) {
+  {
+    std::unordered_multimap<int, int> src{{1, 0}, {3, 0}, {5, 0}};
+    std::unordered_multimap<int, int> dst{{2, 0}, {4, 0}, {5, 0}};
+    dst.merge(src);
+    assert(map_equal(src, {}));
+    assert(map_equal(dst, {{1, 0}, {2, 0}, {3, 0}, {4, 0}, {5, 0}, {5, 0}}));
+  }
 
 #ifndef TEST_HAS_NO_EXCEPTIONS
-    {
-        bool do_throw = false;
-        typedef std::unordered_multimap<Counter<int>, int, throw_hasher<Counter<int>>> map_type;
-        map_type src({{1, 0}, {3, 0}, {5, 0}}, 0, throw_hasher<Counter<int>>(do_throw));
-        map_type dst({{2, 0}, {4, 0}, {5, 0}}, 0, throw_hasher<Counter<int>>(do_throw));
-
-        assert(Counter_base::gConstructed == 6);
-
-        do_throw = true;
-        try
-        {
-            dst.merge(src);
-        }
-        catch (int)
-        {
-            do_throw = false;
-        }
-        assert(!do_throw);
-        assert(map_equal(src, map_type({{1, 0}, {3, 0}, {5, 0}}, 0, throw_hasher<Counter<int>>(do_throw))));
-        assert(map_equal(dst, map_type({{2, 0}, {4, 0}, {5, 0}}, 0, throw_hasher<Counter<int>>(do_throw))));
+  {
+    bool do_throw = false;
+    typedef std::unordered_multimap<Counter<int>, int, throw_hasher<Counter<int>>> map_type;
+    map_type src({{1, 0}, {3, 0}, {5, 0}}, 0, throw_hasher<Counter<int>>(do_throw));
+    map_type dst({{2, 0}, {4, 0}, {5, 0}}, 0, throw_hasher<Counter<int>>(do_throw));
+
+    assert(Counter_base::gConstructed == 6);
+
+    do_throw = true;
+    try {
+      dst.merge(src);
+    } catch (int) {
+      do_throw = false;
     }
+    assert(!do_throw);
+    assert(map_equal(src, map_type({{1, 0}, {3, 0}, {5, 0}}, 0, throw_hasher<Counter<int>>(do_throw))));
+    assert(map_equal(dst, map_type({{2, 0}, {4, 0}, {5, 0}}, 0, throw_hasher<Counter<int>>(do_throw))));
+  }
 #endif
-    assert(Counter_base::gConstructed == 0);
-    struct equal
-    {
-        equal() = default;
-
-        bool operator()(const Counter<int>& lhs, const Counter<int>& rhs) const
-        {
-            return lhs == rhs;
-        }
-    };
-    struct hasher
-    {
-        hasher() = default;
-        std::size_t operator()(const Counter<int>& p) const
-        {
-            return std::hash<Counter<int>>()(p);
-        }
-    };
-    {
-        typedef std::unordered_multimap<Counter<int>, int, std::hash<Counter<int>>, std::equal_to<Counter<int>>> first_map_type;
-        typedef std::unordered_multimap<Counter<int>, int, hasher, equal> second_map_type;
-        typedef std::unordered_map<Counter<int>, int, hasher, equal> third_map_type;
+  assert(Counter_base::gConstructed == 0);
+  struct equal {
+    equal() = default;
+
+    bool operator()(const Counter<int>& lhs, const Counter<int>& rhs) const { return lhs == rhs; }
+  };
+  struct hasher {
+    hasher() = default;
+    std::size_t operator()(const Counter<int>& p) const { return std::hash<Counter<int>>()(p); }
+  };
+  {
+    typedef std::unordered_multimap<Counter<int>, int, std::hash<Counter<int>>, std::equal_to<Counter<int>>>
+        first_map_type;
+    typedef std::unordered_multimap<Counter<int>, int, hasher, equal> second_map_type;
+    typedef std::unordered_map<Counter<int>, int, hasher, equal> third_map_type;
 
-        {
-            first_map_type first{{1, 0}, {2, 0}, {3, 0}};
-            second_map_type second{{2, 0}, {3, 0}, {4, 0}};
-            third_map_type third{{1, 0}, {3, 0}};
+    {
+      first_map_type first{{1, 0}, {2, 0}, {3, 0}};
+      second_map_type second{{2, 0}, {3, 0}, {4, 0}};
+      third_map_type third{{1, 0}, {3, 0}};
 
-            assert(Counter_base::gConstructed == 8);
+      assert(Counter_base::gConstructed == 8);
 
-            first.merge(second);
-            first.merge(third);
+      first.merge(second);
+      first.merge(third);
 
-            assert(map_equal(first, {{1, 0}, {2, 0}, {3, 0}, {4, 0}, {2, 0}, {3, 0}, {1, 0}, {3, 0}}));
-            assert(map_equal(second, {}));
-            assert(map_equal(third, {}));
+      assert(map_equal(first, {{1, 0}, {2, 0}, {3, 0}, {4, 0}, {2, 0}, {3, 0}, {1, 0}, {3, 0}}));
+      assert(map_equal(second, {}));
+      assert(map_equal(third, {}));
 
-            assert(Counter_base::gConstructed == 8);
-        }
-        assert(Counter_base::gConstructed == 0);
-        {
-            first_map_type first{{1, 0}, {2, 0}, {3, 0}};
-            second_map_type second{{2, 0}, {3, 0}, {4, 0}};
-            third_map_type third{{1, 0}, {3, 0}};
+      assert(Counter_base::gConstructed == 8);
+    }
+    assert(Counter_base::gConstructed == 0);
+    {
+      first_map_type first{{1, 0}, {2, 0}, {3, 0}};
+      second_map_type second{{2, 0}, {3, 0}, {4, 0}};
+      third_map_type third{{1, 0}, {3, 0}};
 
-            assert(Counter_base::gConstructed == 8);
+      assert(Counter_base::gConstructed == 8);
 
-            first.merge(std::move(second));
-            first.merge(std::move(third));
+      first.merge(std::move(second));
+      first.merge(std::move(third));
 
-            assert(map_equal(first, {{1, 0}, {2, 0}, {3, 0}, {4, 0}, {2, 0}, {3, 0}, {1, 0}, {3, 0}}));
-            assert(map_equal(second, {}));
-            assert(map_equal(third, {}));
+      assert(map_equal(first, {{1, 0}, {2, 0}, {3, 0}, {4, 0}, {2, 0}, {3, 0}, {1, 0}, {3, 0}}));
+      assert(map_equal(second, {}));
+      assert(map_equal(third, {}));
 
-            assert(Counter_base::gConstructed == 8);
-        }
-        assert(Counter_base::gConstructed == 0);
+      assert(Counter_base::gConstructed == 8);
+    }
+    assert(Counter_base::gConstructed == 0);
+  }
+  {
+    std::unordered_multimap<int, int> first;
+    {
+      std::unordered_multimap<int, int> second;
+      first.merge(second);
+      first.merge(std::move(second));
     }
     {
-        std::unordered_multimap<int, int> first;
-        {
-            std::unordered_multimap<int, int> second;
-            first.merge(second);
-            first.merge(std::move(second));
-        }
-        {
-            std::unordered_map<int, int> second;
-            first.merge(second);
-            first.merge(std::move(second));
-        }
+      std::unordered_map<int, int> second;
+      first.merge(second);
+      first.merge(std::move(second));
     }
-    return 0;
+  }
+  return 0;
 }
diff --git a/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.swap/swap_noexcept.pass.cpp b/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.swap/swap_noexcept.pass.cpp
index 6220f55a3e869..b60cd4cafc18d 100644
--- a/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.swap/swap_noexcept.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.swap/swap_noexcept.pass.cpp
@@ -33,23 +33,21 @@
 #include "test_allocator.h"
 
 template <class T>
-struct some_comp
-{
-    typedef T value_type;
+struct some_comp {
+  typedef T value_type;
 
-    some_comp() {}
-    some_comp(const some_comp&) {}
-    bool operator()(const T&, const T&) const { return false; }
+  some_comp() {}
+  some_comp(const some_comp&) {}
+  bool operator()(const T&, const T&) const { return false; }
 };
 
 template <class T>
-struct some_comp2
-{
-    typedef T value_type;
+struct some_comp2 {
+  typedef T value_type;
 
-    some_comp2() {}
-    some_comp2(const some_comp2&) {}
-    bool operator()(const T&, const T&) const { return false; }
+  some_comp2() {}
+  some_comp2(const some_comp2&) {}
+  bool operator()(const T&, const T&) const { return false; }
 };
 
 #if TEST_STD_VER >= 14
@@ -58,21 +56,19 @@ void swap(some_comp2<T>&, some_comp2<T>&) noexcept {}
 #endif
 
 template <class T>
-struct some_hash
-{
-    typedef T value_type;
-    some_hash() {}
-    some_hash(const some_hash&);
-    std::size_t operator()(T const&) const;
+struct some_hash {
+  typedef T value_type;
+  some_hash() {}
+  some_hash(const some_hash&);
+  std::size_t operator()(T const&) const;
 };
 
 template <class T>
-struct some_hash2
-{
-    typedef T value_type;
-    some_hash2() {}
-    some_hash2(const some_hash2&);
-    std::size_t operator()(T const&) const;
+struct some_hash2 {
+  typedef T value_type;
+  some_hash2() {}
+  some_hash2(const some_hash2&);
+  std::size_t operator()(T const&) const;
 };
 
 #if TEST_STD_VER >= 14
@@ -81,111 +77,107 @@ void swap(some_hash2<T>&, some_hash2<T>&) noexcept {}
 #endif
 
 template <class T>
-struct some_alloc
-{
-    typedef T value_type;
+struct some_alloc {
+  typedef T value_type;
 
-    some_alloc() {}
-    some_alloc(const some_alloc&);
-    void deallocate(void*, unsigned) {}
+  some_alloc() {}
+  some_alloc(const some_alloc&);
+  void deallocate(void*, unsigned) {}
 
-    typedef std::true_type propagate_on_container_swap;
+  typedef std::true_type propagate_on_container_swap;
 };
 
 template <class T>
-struct some_alloc2
-{
-    typedef T value_type;
+struct some_alloc2 {
+  typedef T value_type;
 
-    some_alloc2() {}
-    some_alloc2(const some_alloc2&);
-    void deallocate(void*, unsigned) {}
+  some_alloc2() {}
+  some_alloc2(const some_alloc2&);
+  void deallocate(void*, unsigned) {}
 
-    typedef std::false_type propagate_on_container_swap;
-    typedef std::true_type is_always_equal;
+  typedef std::false_type propagate_on_container_swap;
+  typedef std::true_type is_always_equal;
 };
 
 template <class T>
-struct some_alloc3
-{
-    typedef T value_type;
+struct some_alloc3 {
+  typedef T value_type;
 
-    some_alloc3() {}
-    some_alloc3(const some_alloc3&);
-    void deallocate(void*, unsigned) {}
+  some_alloc3() {}
+  some_alloc3(const some_alloc3&);
+  void deallocate(void*, unsigned) {}
 
-    typedef std::false_type propagate_on_container_swap;
-    typedef std::false_type is_always_equal;
+  typedef std::false_type propagate_on_container_swap;
+  typedef std::false_type is_always_equal;
 };
 
-int main(int, char**)
-{
-    typedef std::pair<const MoveOnly, MoveOnly> V;
-    {
-        typedef std::unordered_multimap<MoveOnly, MoveOnly> C;
-        static_assert(noexcept(swap(std::declval<C&>(), std::declval<C&>())), "");
-    }
+int main(int, char**) {
+  typedef std::pair<const MoveOnly, MoveOnly> V;
+  {
+    typedef std::unordered_multimap<MoveOnly, MoveOnly> C;
+    static_assert(noexcept(swap(std::declval<C&>(), std::declval<C&>())), "");
+  }
 #if defined(_LIBCPP_VERSION)
-    {
-        typedef std::unordered_multimap<MoveOnly, MoveOnly, std::hash<MoveOnly>,
-                           std::equal_to<MoveOnly>, test_allocator<V>> C;
-        static_assert(noexcept(swap(std::declval<C&>(), std::declval<C&>())), "");
-    }
-    {
-        typedef std::unordered_multimap<MoveOnly, MoveOnly, std::hash<MoveOnly>,
-                          std::equal_to<MoveOnly>, other_allocator<V>> C;
-        static_assert(noexcept(swap(std::declval<C&>(), std::declval<C&>())), "");
-    }
+  {
+    typedef std::unordered_multimap<MoveOnly, MoveOnly, std::hash<MoveOnly>, std::equal_to<MoveOnly>, test_allocator<V>>
+        C;
+    static_assert(noexcept(swap(std::declval<C&>(), std::declval<C&>())), "");
+  }
+  {
+    typedef std::
+        unordered_multimap<MoveOnly, MoveOnly, std::hash<MoveOnly>, std::equal_to<MoveOnly>, other_allocator<V>>
+            C;
+    static_assert(noexcept(swap(std::declval<C&>(), std::declval<C&>())), "");
+  }
 #endif // _LIBCPP_VERSION
-    {
-        typedef std::unordered_multimap<MoveOnly, MoveOnly, some_hash<MoveOnly>> C;
-        static_assert(!noexcept(swap(std::declval<C&>(), std::declval<C&>())), "");
-    }
-    {
-        typedef std::unordered_multimap<MoveOnly, MoveOnly, std::hash<MoveOnly>,
-                                                         some_comp<MoveOnly>> C;
-        static_assert(!noexcept(swap(std::declval<C&>(), std::declval<C&>())), "");
-    }
+  {
+    typedef std::unordered_multimap<MoveOnly, MoveOnly, some_hash<MoveOnly>> C;
+    static_assert(!noexcept(swap(std::declval<C&>(), std::declval<C&>())), "");
+  }
+  {
+    typedef std::unordered_multimap<MoveOnly, MoveOnly, std::hash<MoveOnly>, some_comp<MoveOnly>> C;
+    static_assert(!noexcept(swap(std::declval<C&>(), std::declval<C&>())), "");
+  }
 
 #if TEST_STD_VER >= 14
-    { // POCS allocator, throwable swap for hash, throwable swap for comp
-    typedef std::unordered_multimap<MoveOnly, MoveOnly, some_hash<MoveOnly>, some_comp <MoveOnly>, some_alloc <V>> C;
+  { // POCS allocator, throwable swap for hash, throwable swap for comp
+    typedef std::unordered_multimap<MoveOnly, MoveOnly, some_hash<MoveOnly>, some_comp<MoveOnly>, some_alloc<V>> C;
     static_assert(!noexcept(swap(std::declval<C&>(), std::declval<C&>())), "");
-    }
-    { // always equal allocator, throwable swap for hash, throwable swap for comp
-    typedef std::unordered_multimap<MoveOnly, MoveOnly, some_hash<MoveOnly>, some_comp <MoveOnly>, some_alloc2<V>> C;
+  }
+  { // always equal allocator, throwable swap for hash, throwable swap for comp
+    typedef std::unordered_multimap<MoveOnly, MoveOnly, some_hash<MoveOnly>, some_comp<MoveOnly>, some_alloc2<V>> C;
     static_assert(!noexcept(swap(std::declval<C&>(), std::declval<C&>())), "");
-    }
-    { // POCS allocator, throwable swap for hash, nothrow swap for comp
-    typedef std::unordered_multimap<MoveOnly, MoveOnly, some_hash<MoveOnly>, some_comp2<MoveOnly>, some_alloc <V>> C;
+  }
+  { // POCS allocator, throwable swap for hash, nothrow swap for comp
+    typedef std::unordered_multimap<MoveOnly, MoveOnly, some_hash<MoveOnly>, some_comp2<MoveOnly>, some_alloc<V>> C;
     static_assert(!noexcept(swap(std::declval<C&>(), std::declval<C&>())), "");
-    }
-    { // always equal allocator, throwable swap for hash, nothrow swap for comp
+  }
+  { // always equal allocator, throwable swap for hash, nothrow swap for comp
     typedef std::unordered_multimap<MoveOnly, MoveOnly, some_hash<MoveOnly>, some_comp2<MoveOnly>, some_alloc2<V>> C;
     static_assert(!noexcept(swap(std::declval<C&>(), std::declval<C&>())), "");
-    }
-    { // POCS allocator, nothrow swap for hash, throwable swap for comp
-    typedef std::unordered_multimap<MoveOnly, MoveOnly, some_hash2<MoveOnly>, some_comp <MoveOnly>, some_alloc <V>> C;
+  }
+  { // POCS allocator, nothrow swap for hash, throwable swap for comp
+    typedef std::unordered_multimap<MoveOnly, MoveOnly, some_hash2<MoveOnly>, some_comp<MoveOnly>, some_alloc<V>> C;
     static_assert(!noexcept(swap(std::declval<C&>(), std::declval<C&>())), "");
-    }
-    { // always equal allocator, nothrow swap for hash, throwable swap for comp
-    typedef std::unordered_multimap<MoveOnly, MoveOnly, some_hash2<MoveOnly>, some_comp <MoveOnly>, some_alloc2<V>> C;
+  }
+  { // always equal allocator, nothrow swap for hash, throwable swap for comp
+    typedef std::unordered_multimap<MoveOnly, MoveOnly, some_hash2<MoveOnly>, some_comp<MoveOnly>, some_alloc2<V>> C;
     static_assert(!noexcept(swap(std::declval<C&>(), std::declval<C&>())), "");
-    }
-    { // POCS allocator, nothrow swap for hash, nothrow swap for comp
-    typedef std::unordered_multimap<MoveOnly, MoveOnly, some_hash2<MoveOnly>, some_comp2<MoveOnly>, some_alloc <V>> C;
-    static_assert( noexcept(swap(std::declval<C&>(), std::declval<C&>())), "");
-    }
-    { // always equal allocator, nothrow swap for hash, nothrow swap for comp
+  }
+  { // POCS allocator, nothrow swap for hash, nothrow swap for comp
+    typedef std::unordered_multimap<MoveOnly, MoveOnly, some_hash2<MoveOnly>, some_comp2<MoveOnly>, some_alloc<V>> C;
+    static_assert(noexcept(swap(std::declval<C&>(), std::declval<C&>())), "");
+  }
+  { // always equal allocator, nothrow swap for hash, nothrow swap for comp
     typedef std::unordered_multimap<MoveOnly, MoveOnly, some_hash2<MoveOnly>, some_comp2<MoveOnly>, some_alloc2<V>> C;
-    static_assert( noexcept(swap(std::declval<C&>(), std::declval<C&>())), "");
-    }
-#if defined(_LIBCPP_VERSION)
-    { // NOT always equal allocator, nothrow swap for hash, nothrow swap for comp
+    static_assert(noexcept(swap(std::declval<C&>(), std::declval<C&>())), "");
+  }
+#  if defined(_LIBCPP_VERSION)
+  { // NOT always equal allocator, nothrow swap for hash, nothrow swap for comp
     typedef std::unordered_multimap<MoveOnly, MoveOnly, some_hash2<MoveOnly>, some_comp2<MoveOnly>, some_alloc3<V>> C;
-    static_assert( noexcept(swap(std::declval<C&>(), std::declval<C&>())), "");
-    }
-#endif // _LIBCPP_VERSION
+    static_assert(noexcept(swap(std::declval<C&>(), std::declval<C&>())), "");
+  }
+#  endif // _LIBCPP_VERSION
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.swap/swap_non_member.pass.cpp b/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.swap/swap_non_member.pass.cpp
index 8aafbdd46b12b..fb2bdd2f79ef9 100644
--- a/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.swap/swap_non_member.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.swap/swap_non_member.pass.cpp
@@ -28,585 +28,572 @@
 #include "test_allocator.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef test_hash<int> Hash;
-        typedef test_equal_to<int> Compare;
-        typedef test_allocator<std::pair<const int, std::string> > Alloc;
-        typedef std::unordered_multimap<int, std::string, Hash, Compare, Alloc> C;
-        C c1(0, Hash(1), Compare(1), Alloc(1, 1));
-        C c2(0, Hash(2), Compare(2), Alloc(1, 2));
-        c2.max_load_factor(2);
-        swap(c1, c2);
-
-        LIBCPP_ASSERT(c1.bucket_count() == 0);
-        assert(c1.size() == 0);
-        assert(c1.hash_function() == Hash(2));
-        assert(c1.key_eq() == Compare(2));
-        assert(c1.get_allocator().get_id() == 1);
-        assert(static_cast<std::size_t>(std::distance(c1.begin(), c1.end())) == c1.size());
-        assert(static_cast<std::size_t>(std::distance(c1.cbegin(), c1.cend())) == c1.size());
-        assert(c1.max_load_factor() == 2);
-
-        LIBCPP_ASSERT(c2.bucket_count() == 0);
-        assert(c2.size() == 0);
-        assert(c2.hash_function() == Hash(1));
-        assert(c2.key_eq() == Compare(1));
-        assert(c2.get_allocator().get_id() == 2);
-        assert(static_cast<std::size_t>(std::distance(c2.begin(), c2.end())) == c2.size());
-        assert(static_cast<std::size_t>(std::distance(c2.cbegin(), c2.cend())) == c2.size());
-        assert(c2.max_load_factor() == 1);
-    }
-    {
-        typedef test_hash<int> Hash;
-        typedef test_equal_to<int> Compare;
-        typedef test_allocator<std::pair<const int, std::string> > Alloc;
-        typedef std::unordered_multimap<int, std::string, Hash, Compare, Alloc> C;
-        typedef std::pair<int, std::string> P;
-        P a2[] =
-        {
-            P(10, "ten"),
-            P(20, "twenty"),
-            P(30, "thirty"),
-            P(40, "forty"),
-            P(50, "fifty"),
-            P(60, "sixty"),
-            P(70, "seventy"),
-            P(80, "eighty"),
-        };
-        C c1(0, Hash(1), Compare(1), Alloc(1, 1));
-        C c2(std::begin(a2), std::end(a2), 0, Hash(2), Compare(2), Alloc(1, 2));
-        c2.max_load_factor(2);
-        C::iterator it2 = c2.begin();
-        swap(c1, c2);
-
-        assert(c1.bucket_count() >= 8);
-        assert(c1.size() == 8);
-        assert(c1.find(10)->second == "ten");
-        assert(c1.find(20)->second == "twenty");
-        assert(c1.find(30)->second == "thirty");
-        assert(c1.find(40)->second == "forty");
-        assert(c1.find(50)->second == "fifty");
-        assert(c1.find(60)->second == "sixty");
-        assert(c1.find(70)->second == "seventy");
-        assert(c1.find(80)->second == "eighty");
-        assert(c1.hash_function() == Hash(2));
-        assert(c1.key_eq() == Compare(2));
-        assert(c1.get_allocator().get_id() == 1);
-        assert(static_cast<std::size_t>(std::distance(c1.begin(), c1.end())) == c1.size());
-        assert(static_cast<std::size_t>(std::distance(c1.cbegin(), c1.cend())) == c1.size());
-        assert(c1.max_load_factor() == 2);
-        assert(it2 == c1.begin()); // Iterators are not invalidated
-
-        LIBCPP_ASSERT(c2.bucket_count() == 0);
-        assert(c2.size() == 0);
-        assert(c2.hash_function() == Hash(1));
-        assert(c2.key_eq() == Compare(1));
-        assert(c2.get_allocator().get_id() == 2);
-        assert(static_cast<std::size_t>(std::distance(c2.begin(), c2.end())) == c2.size());
-        assert(static_cast<std::size_t>(std::distance(c2.cbegin(), c2.cend())) == c2.size());
-        assert(c2.max_load_factor() == 1);
-    }
-    {
-        typedef test_hash<int> Hash;
-        typedef test_equal_to<int> Compare;
-        typedef test_allocator<std::pair<const int, std::string> > Alloc;
-        typedef std::unordered_multimap<int, std::string, Hash, Compare, Alloc> C;
-        typedef std::pair<int, std::string> P;
-        P a1[] =
-        {
-            P(1, "one"),
-            P(2, "two"),
-            P(3, "three"),
-            P(4, "four"),
-            P(1, "four"),
-            P(2, "four"),
-        };
-        C c1(std::begin(a1), std::end(a1), 0, Hash(1), Compare(1), Alloc(1, 1));
-        C c2(0, Hash(2), Compare(2), Alloc(1, 2));
-        c2.max_load_factor(2);
-        C::iterator it1 = c1.begin();
-        swap(c1, c2);
-
-        LIBCPP_ASSERT(c1.bucket_count() == 0);
-        assert(c1.size() == 0);
-        assert(c1.hash_function() == Hash(2));
-        assert(c1.key_eq() == Compare(2));
-        assert(c1.get_allocator().get_id() == 1);
-        assert(static_cast<std::size_t>(std::distance(c1.begin(), c1.end())) == c1.size());
-        assert(static_cast<std::size_t>(std::distance(c1.cbegin(), c1.cend())) == c1.size());
-        assert(c1.max_load_factor() == 2);
-
-        assert(c2.bucket_count() >= 6);
-        assert(c2.size() == 6);
-        std::multiset<std::string> s;
-        s.insert("one");
-        s.insert("four");
-        CheckConsecutiveKeys<C::const_iterator>(c2.find(1), c2.end(), 1, s);
-        s.insert("two");
-        s.insert("four");
-        CheckConsecutiveKeys<C::const_iterator>(c2.find(2), c2.end(), 2, s);
-        assert(c2.find(3)->second == "three");
-        assert(c2.find(4)->second == "four");
-        assert(c2.hash_function() == Hash(1));
-        assert(c2.key_eq() == Compare(1));
-        assert(c2.get_allocator().get_id() == 2);
-        assert(static_cast<std::size_t>(std::distance(c2.begin(), c2.end())) == c2.size());
-        assert(static_cast<std::size_t>(std::distance(c2.cbegin(), c2.cend())) == c2.size());
-        assert(c2.max_load_factor() == 1);
-        assert(it1 == c2.begin()); // Iterators are not invalidated
-    }
-    {
-        typedef test_hash<int> Hash;
-        typedef test_equal_to<int> Compare;
-        typedef test_allocator<std::pair<const int, std::string> > Alloc;
-        typedef std::unordered_multimap<int, std::string, Hash, Compare, Alloc> C;
-        typedef std::pair<int, std::string> P;
-        P a1[] =
-        {
-            P(1, "one"),
-            P(2, "two"),
-            P(3, "three"),
-            P(4, "four"),
-            P(1, "four"),
-            P(2, "four"),
-        };
-        P a2[] =
-        {
-            P(10, "ten"),
-            P(20, "twenty"),
-            P(30, "thirty"),
-            P(40, "forty"),
-            P(50, "fifty"),
-            P(60, "sixty"),
-            P(70, "seventy"),
-            P(80, "eighty"),
-        };
-        C c1(std::begin(a1), std::end(a1), 0, Hash(1), Compare(1), Alloc(1, 1));
-        C c2(std::begin(a2), std::end(a2), 0, Hash(2), Compare(2), Alloc(1, 2));
-        c2.max_load_factor(2);
-        C::iterator it1 = c1.begin();
-        C::iterator it2 = c2.begin();
-        swap(c1, c2);
-
-        assert(c1.bucket_count() >= 8);
-        assert(c1.size() == 8);
-        assert(c1.find(10)->second == "ten");
-        assert(c1.find(20)->second == "twenty");
-        assert(c1.find(30)->second == "thirty");
-        assert(c1.find(40)->second == "forty");
-        assert(c1.find(50)->second == "fifty");
-        assert(c1.find(60)->second == "sixty");
-        assert(c1.find(70)->second == "seventy");
-        assert(c1.find(80)->second == "eighty");
-        assert(c1.hash_function() == Hash(2));
-        assert(c1.key_eq() == Compare(2));
-        assert(c1.get_allocator().get_id() == 1);
-        assert(static_cast<std::size_t>(std::distance(c1.begin(), c1.end())) == c1.size());
-        assert(static_cast<std::size_t>(std::distance(c1.cbegin(), c1.cend())) == c1.size());
-        assert(c1.max_load_factor() == 2);
-        assert(it2 == c1.begin()); // Iterators are not invalidated
-
-        assert(c2.bucket_count() >= 6);
-        assert(c2.size() == 6);
-        std::multiset<std::string> s;
-        s.insert("one");
-        s.insert("four");
-        CheckConsecutiveKeys<C::const_iterator>(c2.find(1), c2.end(), 1, s);
-        s.insert("two");
-        s.insert("four");
-        CheckConsecutiveKeys<C::const_iterator>(c2.find(2), c2.end(), 2, s);
-        assert(c2.find(3)->second == "three");
-        assert(c2.find(4)->second == "four");
-        assert(c2.hash_function() == Hash(1));
-        assert(c2.key_eq() == Compare(1));
-        assert(c2.get_allocator().get_id() == 2);
-        assert(static_cast<std::size_t>(std::distance(c2.begin(), c2.end())) == c2.size());
-        assert(static_cast<std::size_t>(std::distance(c2.cbegin(), c2.cend())) == c2.size());
-        assert(c2.max_load_factor() == 1);
-        assert(it1 == c2.begin()); // Iterators are not invalidated
-    }
-
-    {
-        typedef test_hash<int> Hash;
-        typedef test_equal_to<int> Compare;
-        typedef other_allocator<std::pair<const int, std::string> > Alloc;
-        typedef std::unordered_multimap<int, std::string, Hash, Compare, Alloc> C;
-        C c1(0, Hash(1), Compare(1), Alloc(1));
-        C c2(0, Hash(2), Compare(2), Alloc(2));
-        c2.max_load_factor(2);
-        swap(c1, c2);
-
-        LIBCPP_ASSERT(c1.bucket_count() == 0);
-        assert(c1.size() == 0);
-        assert(c1.hash_function() == Hash(2));
-        assert(c1.key_eq() == Compare(2));
-        assert(c1.get_allocator() == Alloc(2));
-        assert(static_cast<std::size_t>(std::distance(c1.begin(), c1.end())) == c1.size());
-        assert(static_cast<std::size_t>(std::distance(c1.cbegin(), c1.cend())) == c1.size());
-        assert(c1.max_load_factor() == 2);
-
-        LIBCPP_ASSERT(c2.bucket_count() == 0);
-        assert(c2.size() == 0);
-        assert(c2.hash_function() == Hash(1));
-        assert(c2.key_eq() == Compare(1));
-        assert(c2.get_allocator() == Alloc(1));
-        assert(static_cast<std::size_t>(std::distance(c2.begin(), c2.end())) == c2.size());
-        assert(static_cast<std::size_t>(std::distance(c2.cbegin(), c2.cend())) == c2.size());
-        assert(c2.max_load_factor() == 1);
-    }
-    {
-        typedef test_hash<int> Hash;
-        typedef test_equal_to<int> Compare;
-        typedef other_allocator<std::pair<const int, std::string> > Alloc;
-        typedef std::unordered_multimap<int, std::string, Hash, Compare, Alloc> C;
-        typedef std::pair<int, std::string> P;
-        P a2[] =
-        {
-            P(10, "ten"),
-            P(20, "twenty"),
-            P(30, "thirty"),
-            P(40, "forty"),
-            P(50, "fifty"),
-            P(60, "sixty"),
-            P(70, "seventy"),
-            P(80, "eighty"),
-        };
-        C c1(0, Hash(1), Compare(1), Alloc(1));
-        C c2(std::begin(a2), std::end(a2), 0, Hash(2), Compare(2), Alloc(2));
-        c2.max_load_factor(2);
-        swap(c1, c2);
-
-        assert(c1.bucket_count() >= 8);
-        assert(c1.size() == 8);
-        assert(c1.find(10)->second == "ten");
-        assert(c1.find(20)->second == "twenty");
-        assert(c1.find(30)->second == "thirty");
-        assert(c1.find(40)->second == "forty");
-        assert(c1.find(50)->second == "fifty");
-        assert(c1.find(60)->second == "sixty");
-        assert(c1.find(70)->second == "seventy");
-        assert(c1.find(80)->second == "eighty");
-        assert(c1.hash_function() == Hash(2));
-        assert(c1.key_eq() == Compare(2));
-        assert(c1.get_allocator() == Alloc(2));
-        assert(static_cast<std::size_t>(std::distance(c1.begin(), c1.end())) == c1.size());
-        assert(static_cast<std::size_t>(std::distance(c1.cbegin(), c1.cend())) == c1.size());
-        assert(c1.max_load_factor() == 2);
-
-        LIBCPP_ASSERT(c2.bucket_count() == 0);
-        assert(c2.size() == 0);
-        assert(c2.hash_function() == Hash(1));
-        assert(c2.key_eq() == Compare(1));
-        assert(c2.get_allocator() == Alloc(1));
-        assert(static_cast<std::size_t>(std::distance(c2.begin(), c2.end())) == c2.size());
-        assert(static_cast<std::size_t>(std::distance(c2.cbegin(), c2.cend())) == c2.size());
-        assert(c2.max_load_factor() == 1);
-    }
-    {
-        typedef test_hash<int> Hash;
-        typedef test_equal_to<int> Compare;
-        typedef other_allocator<std::pair<const int, std::string> > Alloc;
-        typedef std::unordered_multimap<int, std::string, Hash, Compare, Alloc> C;
-        typedef std::pair<int, std::string> P;
-        P a1[] =
-        {
-            P(1, "one"),
-            P(2, "two"),
-            P(3, "three"),
-            P(4, "four"),
-            P(1, "four"),
-            P(2, "four"),
-        };
-        C c1(std::begin(a1), std::end(a1), 0, Hash(1), Compare(1), Alloc(1));
-        C c2(0, Hash(2), Compare(2), Alloc(2));
-        c2.max_load_factor(2);
-        swap(c1, c2);
-
-        LIBCPP_ASSERT(c1.bucket_count() == 0);
-        assert(c1.size() == 0);
-        assert(c1.hash_function() == Hash(2));
-        assert(c1.key_eq() == Compare(2));
-        assert(c1.get_allocator() == Alloc(2));
-        assert(static_cast<std::size_t>(std::distance(c1.begin(), c1.end())) == c1.size());
-        assert(static_cast<std::size_t>(std::distance(c1.cbegin(), c1.cend())) == c1.size());
-        assert(c1.max_load_factor() == 2);
-
-        assert(c2.bucket_count() >= 6);
-        assert(c2.size() == 6);
-        std::multiset<std::string> s;
-        s.insert("one");
-        s.insert("four");
-        CheckConsecutiveKeys<C::const_iterator>(c2.find(1), c2.end(), 1, s);
-        s.insert("two");
-        s.insert("four");
-        CheckConsecutiveKeys<C::const_iterator>(c2.find(2), c2.end(), 2, s);
-        assert(c2.find(3)->second == "three");
-        assert(c2.find(4)->second == "four");
-        assert(c2.hash_function() == Hash(1));
-        assert(c2.key_eq() == Compare(1));
-        assert(c2.get_allocator() == Alloc(1));
-        assert(static_cast<std::size_t>(std::distance(c2.begin(), c2.end())) == c2.size());
-        assert(static_cast<std::size_t>(std::distance(c2.cbegin(), c2.cend())) == c2.size());
-        assert(c2.max_load_factor() == 1);
-    }
-    {
-        typedef test_hash<int> Hash;
-        typedef test_equal_to<int> Compare;
-        typedef other_allocator<std::pair<const int, std::string> > Alloc;
-        typedef std::unordered_multimap<int, std::string, Hash, Compare, Alloc> C;
-        typedef std::pair<int, std::string> P;
-        P a1[] =
-        {
-            P(1, "one"),
-            P(2, "two"),
-            P(3, "three"),
-            P(4, "four"),
-            P(1, "four"),
-            P(2, "four"),
-        };
-        P a2[] =
-        {
-            P(10, "ten"),
-            P(20, "twenty"),
-            P(30, "thirty"),
-            P(40, "forty"),
-            P(50, "fifty"),
-            P(60, "sixty"),
-            P(70, "seventy"),
-            P(80, "eighty"),
-        };
-        C c1(std::begin(a1), std::end(a1), 0, Hash(1), Compare(1), Alloc(1));
-        C c2(std::begin(a2), std::end(a2), 0, Hash(2), Compare(2), Alloc(2));
-        c2.max_load_factor(2);
-        swap(c1, c2);
-
-        assert(c1.bucket_count() >= 8);
-        assert(c1.size() == 8);
-        assert(c1.find(10)->second == "ten");
-        assert(c1.find(20)->second == "twenty");
-        assert(c1.find(30)->second == "thirty");
-        assert(c1.find(40)->second == "forty");
-        assert(c1.find(50)->second == "fifty");
-        assert(c1.find(60)->second == "sixty");
-        assert(c1.find(70)->second == "seventy");
-        assert(c1.find(80)->second == "eighty");
-        assert(c1.hash_function() == Hash(2));
-        assert(c1.key_eq() == Compare(2));
-        assert(c1.get_allocator() == Alloc(2));
-        assert(static_cast<std::size_t>(std::distance(c1.begin(), c1.end())) == c1.size());
-        assert(static_cast<std::size_t>(std::distance(c1.cbegin(), c1.cend())) == c1.size());
-        assert(c1.max_load_factor() == 2);
-
-        assert(c2.bucket_count() >= 6);
-        assert(c2.size() == 6);
-        std::multiset<std::string> s;
-        s.insert("one");
-        s.insert("four");
-        CheckConsecutiveKeys<C::const_iterator>(c2.find(1), c2.end(), 1, s);
-        s.insert("two");
-        s.insert("four");
-        CheckConsecutiveKeys<C::const_iterator>(c2.find(2), c2.end(), 2, s);
-        assert(c2.find(3)->second == "three");
-        assert(c2.find(4)->second == "four");
-        assert(c2.hash_function() == Hash(1));
-        assert(c2.key_eq() == Compare(1));
-        assert(c2.get_allocator() == Alloc(1));
-        assert(static_cast<std::size_t>(std::distance(c2.begin(), c2.end())) == c2.size());
-        assert(static_cast<std::size_t>(std::distance(c2.cbegin(), c2.cend())) == c2.size());
-        assert(c2.max_load_factor() == 1);
-    }
+int main(int, char**) {
+  {
+    typedef test_hash<int> Hash;
+    typedef test_equal_to<int> Compare;
+    typedef test_allocator<std::pair<const int, std::string> > Alloc;
+    typedef std::unordered_multimap<int, std::string, Hash, Compare, Alloc> C;
+    C c1(0, Hash(1), Compare(1), Alloc(1, 1));
+    C c2(0, Hash(2), Compare(2), Alloc(1, 2));
+    c2.max_load_factor(2);
+    swap(c1, c2);
+
+    LIBCPP_ASSERT(c1.bucket_count() == 0);
+    assert(c1.size() == 0);
+    assert(c1.hash_function() == Hash(2));
+    assert(c1.key_eq() == Compare(2));
+    assert(c1.get_allocator().get_id() == 1);
+    assert(static_cast<std::size_t>(std::distance(c1.begin(), c1.end())) == c1.size());
+    assert(static_cast<std::size_t>(std::distance(c1.cbegin(), c1.cend())) == c1.size());
+    assert(c1.max_load_factor() == 2);
+
+    LIBCPP_ASSERT(c2.bucket_count() == 0);
+    assert(c2.size() == 0);
+    assert(c2.hash_function() == Hash(1));
+    assert(c2.key_eq() == Compare(1));
+    assert(c2.get_allocator().get_id() == 2);
+    assert(static_cast<std::size_t>(std::distance(c2.begin(), c2.end())) == c2.size());
+    assert(static_cast<std::size_t>(std::distance(c2.cbegin(), c2.cend())) == c2.size());
+    assert(c2.max_load_factor() == 1);
+  }
+  {
+    typedef test_hash<int> Hash;
+    typedef test_equal_to<int> Compare;
+    typedef test_allocator<std::pair<const int, std::string> > Alloc;
+    typedef std::unordered_multimap<int, std::string, Hash, Compare, Alloc> C;
+    typedef std::pair<int, std::string> P;
+    P a2[] = {
+        P(10, "ten"),
+        P(20, "twenty"),
+        P(30, "thirty"),
+        P(40, "forty"),
+        P(50, "fifty"),
+        P(60, "sixty"),
+        P(70, "seventy"),
+        P(80, "eighty"),
+    };
+    C c1(0, Hash(1), Compare(1), Alloc(1, 1));
+    C c2(std::begin(a2), std::end(a2), 0, Hash(2), Compare(2), Alloc(1, 2));
+    c2.max_load_factor(2);
+    C::iterator it2 = c2.begin();
+    swap(c1, c2);
+
+    assert(c1.bucket_count() >= 8);
+    assert(c1.size() == 8);
+    assert(c1.find(10)->second == "ten");
+    assert(c1.find(20)->second == "twenty");
+    assert(c1.find(30)->second == "thirty");
+    assert(c1.find(40)->second == "forty");
+    assert(c1.find(50)->second == "fifty");
+    assert(c1.find(60)->second == "sixty");
+    assert(c1.find(70)->second == "seventy");
+    assert(c1.find(80)->second == "eighty");
+    assert(c1.hash_function() == Hash(2));
+    assert(c1.key_eq() == Compare(2));
+    assert(c1.get_allocator().get_id() == 1);
+    assert(static_cast<std::size_t>(std::distance(c1.begin(), c1.end())) == c1.size());
+    assert(static_cast<std::size_t>(std::distance(c1.cbegin(), c1.cend())) == c1.size());
+    assert(c1.max_load_factor() == 2);
+    assert(it2 == c1.begin()); // Iterators are not invalidated
+
+    LIBCPP_ASSERT(c2.bucket_count() == 0);
+    assert(c2.size() == 0);
+    assert(c2.hash_function() == Hash(1));
+    assert(c2.key_eq() == Compare(1));
+    assert(c2.get_allocator().get_id() == 2);
+    assert(static_cast<std::size_t>(std::distance(c2.begin(), c2.end())) == c2.size());
+    assert(static_cast<std::size_t>(std::distance(c2.cbegin(), c2.cend())) == c2.size());
+    assert(c2.max_load_factor() == 1);
+  }
+  {
+    typedef test_hash<int> Hash;
+    typedef test_equal_to<int> Compare;
+    typedef test_allocator<std::pair<const int, std::string> > Alloc;
+    typedef std::unordered_multimap<int, std::string, Hash, Compare, Alloc> C;
+    typedef std::pair<int, std::string> P;
+    P a1[] = {
+        P(1, "one"),
+        P(2, "two"),
+        P(3, "three"),
+        P(4, "four"),
+        P(1, "four"),
+        P(2, "four"),
+    };
+    C c1(std::begin(a1), std::end(a1), 0, Hash(1), Compare(1), Alloc(1, 1));
+    C c2(0, Hash(2), Compare(2), Alloc(1, 2));
+    c2.max_load_factor(2);
+    C::iterator it1 = c1.begin();
+    swap(c1, c2);
+
+    LIBCPP_ASSERT(c1.bucket_count() == 0);
+    assert(c1.size() == 0);
+    assert(c1.hash_function() == Hash(2));
+    assert(c1.key_eq() == Compare(2));
+    assert(c1.get_allocator().get_id() == 1);
+    assert(static_cast<std::size_t>(std::distance(c1.begin(), c1.end())) == c1.size());
+    assert(static_cast<std::size_t>(std::distance(c1.cbegin(), c1.cend())) == c1.size());
+    assert(c1.max_load_factor() == 2);
+
+    assert(c2.bucket_count() >= 6);
+    assert(c2.size() == 6);
+    std::multiset<std::string> s;
+    s.insert("one");
+    s.insert("four");
+    CheckConsecutiveKeys<C::const_iterator>(c2.find(1), c2.end(), 1, s);
+    s.insert("two");
+    s.insert("four");
+    CheckConsecutiveKeys<C::const_iterator>(c2.find(2), c2.end(), 2, s);
+    assert(c2.find(3)->second == "three");
+    assert(c2.find(4)->second == "four");
+    assert(c2.hash_function() == Hash(1));
+    assert(c2.key_eq() == Compare(1));
+    assert(c2.get_allocator().get_id() == 2);
+    assert(static_cast<std::size_t>(std::distance(c2.begin(), c2.end())) == c2.size());
+    assert(static_cast<std::size_t>(std::distance(c2.cbegin(), c2.cend())) == c2.size());
+    assert(c2.max_load_factor() == 1);
+    assert(it1 == c2.begin()); // Iterators are not invalidated
+  }
+  {
+    typedef test_hash<int> Hash;
+    typedef test_equal_to<int> Compare;
+    typedef test_allocator<std::pair<const int, std::string> > Alloc;
+    typedef std::unordered_multimap<int, std::string, Hash, Compare, Alloc> C;
+    typedef std::pair<int, std::string> P;
+    P a1[] = {
+        P(1, "one"),
+        P(2, "two"),
+        P(3, "three"),
+        P(4, "four"),
+        P(1, "four"),
+        P(2, "four"),
+    };
+    P a2[] = {
+        P(10, "ten"),
+        P(20, "twenty"),
+        P(30, "thirty"),
+        P(40, "forty"),
+        P(50, "fifty"),
+        P(60, "sixty"),
+        P(70, "seventy"),
+        P(80, "eighty"),
+    };
+    C c1(std::begin(a1), std::end(a1), 0, Hash(1), Compare(1), Alloc(1, 1));
+    C c2(std::begin(a2), std::end(a2), 0, Hash(2), Compare(2), Alloc(1, 2));
+    c2.max_load_factor(2);
+    C::iterator it1 = c1.begin();
+    C::iterator it2 = c2.begin();
+    swap(c1, c2);
+
+    assert(c1.bucket_count() >= 8);
+    assert(c1.size() == 8);
+    assert(c1.find(10)->second == "ten");
+    assert(c1.find(20)->second == "twenty");
+    assert(c1.find(30)->second == "thirty");
+    assert(c1.find(40)->second == "forty");
+    assert(c1.find(50)->second == "fifty");
+    assert(c1.find(60)->second == "sixty");
+    assert(c1.find(70)->second == "seventy");
+    assert(c1.find(80)->second == "eighty");
+    assert(c1.hash_function() == Hash(2));
+    assert(c1.key_eq() == Compare(2));
+    assert(c1.get_allocator().get_id() == 1);
+    assert(static_cast<std::size_t>(std::distance(c1.begin(), c1.end())) == c1.size());
+    assert(static_cast<std::size_t>(std::distance(c1.cbegin(), c1.cend())) == c1.size());
+    assert(c1.max_load_factor() == 2);
+    assert(it2 == c1.begin()); // Iterators are not invalidated
+
+    assert(c2.bucket_count() >= 6);
+    assert(c2.size() == 6);
+    std::multiset<std::string> s;
+    s.insert("one");
+    s.insert("four");
+    CheckConsecutiveKeys<C::const_iterator>(c2.find(1), c2.end(), 1, s);
+    s.insert("two");
+    s.insert("four");
+    CheckConsecutiveKeys<C::const_iterator>(c2.find(2), c2.end(), 2, s);
+    assert(c2.find(3)->second == "three");
+    assert(c2.find(4)->second == "four");
+    assert(c2.hash_function() == Hash(1));
+    assert(c2.key_eq() == Compare(1));
+    assert(c2.get_allocator().get_id() == 2);
+    assert(static_cast<std::size_t>(std::distance(c2.begin(), c2.end())) == c2.size());
+    assert(static_cast<std::size_t>(std::distance(c2.cbegin(), c2.cend())) == c2.size());
+    assert(c2.max_load_factor() == 1);
+    assert(it1 == c2.begin()); // Iterators are not invalidated
+  }
+
+  {
+    typedef test_hash<int> Hash;
+    typedef test_equal_to<int> Compare;
+    typedef other_allocator<std::pair<const int, std::string> > Alloc;
+    typedef std::unordered_multimap<int, std::string, Hash, Compare, Alloc> C;
+    C c1(0, Hash(1), Compare(1), Alloc(1));
+    C c2(0, Hash(2), Compare(2), Alloc(2));
+    c2.max_load_factor(2);
+    swap(c1, c2);
+
+    LIBCPP_ASSERT(c1.bucket_count() == 0);
+    assert(c1.size() == 0);
+    assert(c1.hash_function() == Hash(2));
+    assert(c1.key_eq() == Compare(2));
+    assert(c1.get_allocator() == Alloc(2));
+    assert(static_cast<std::size_t>(std::distance(c1.begin(), c1.end())) == c1.size());
+    assert(static_cast<std::size_t>(std::distance(c1.cbegin(), c1.cend())) == c1.size());
+    assert(c1.max_load_factor() == 2);
+
+    LIBCPP_ASSERT(c2.bucket_count() == 0);
+    assert(c2.size() == 0);
+    assert(c2.hash_function() == Hash(1));
+    assert(c2.key_eq() == Compare(1));
+    assert(c2.get_allocator() == Alloc(1));
+    assert(static_cast<std::size_t>(std::distance(c2.begin(), c2.end())) == c2.size());
+    assert(static_cast<std::size_t>(std::distance(c2.cbegin(), c2.cend())) == c2.size());
+    assert(c2.max_load_factor() == 1);
+  }
+  {
+    typedef test_hash<int> Hash;
+    typedef test_equal_to<int> Compare;
+    typedef other_allocator<std::pair<const int, std::string> > Alloc;
+    typedef std::unordered_multimap<int, std::string, Hash, Compare, Alloc> C;
+    typedef std::pair<int, std::string> P;
+    P a2[] = {
+        P(10, "ten"),
+        P(20, "twenty"),
+        P(30, "thirty"),
+        P(40, "forty"),
+        P(50, "fifty"),
+        P(60, "sixty"),
+        P(70, "seventy"),
+        P(80, "eighty"),
+    };
+    C c1(0, Hash(1), Compare(1), Alloc(1));
+    C c2(std::begin(a2), std::end(a2), 0, Hash(2), Compare(2), Alloc(2));
+    c2.max_load_factor(2);
+    swap(c1, c2);
+
+    assert(c1.bucket_count() >= 8);
+    assert(c1.size() == 8);
+    assert(c1.find(10)->second == "ten");
+    assert(c1.find(20)->second == "twenty");
+    assert(c1.find(30)->second == "thirty");
+    assert(c1.find(40)->second == "forty");
+    assert(c1.find(50)->second == "fifty");
+    assert(c1.find(60)->second == "sixty");
+    assert(c1.find(70)->second == "seventy");
+    assert(c1.find(80)->second == "eighty");
+    assert(c1.hash_function() == Hash(2));
+    assert(c1.key_eq() == Compare(2));
+    assert(c1.get_allocator() == Alloc(2));
+    assert(static_cast<std::size_t>(std::distance(c1.begin(), c1.end())) == c1.size());
+    assert(static_cast<std::size_t>(std::distance(c1.cbegin(), c1.cend())) == c1.size());
+    assert(c1.max_load_factor() == 2);
+
+    LIBCPP_ASSERT(c2.bucket_count() == 0);
+    assert(c2.size() == 0);
+    assert(c2.hash_function() == Hash(1));
+    assert(c2.key_eq() == Compare(1));
+    assert(c2.get_allocator() == Alloc(1));
+    assert(static_cast<std::size_t>(std::distance(c2.begin(), c2.end())) == c2.size());
+    assert(static_cast<std::size_t>(std::distance(c2.cbegin(), c2.cend())) == c2.size());
+    assert(c2.max_load_factor() == 1);
+  }
+  {
+    typedef test_hash<int> Hash;
+    typedef test_equal_to<int> Compare;
+    typedef other_allocator<std::pair<const int, std::string> > Alloc;
+    typedef std::unordered_multimap<int, std::string, Hash, Compare, Alloc> C;
+    typedef std::pair<int, std::string> P;
+    P a1[] = {
+        P(1, "one"),
+        P(2, "two"),
+        P(3, "three"),
+        P(4, "four"),
+        P(1, "four"),
+        P(2, "four"),
+    };
+    C c1(std::begin(a1), std::end(a1), 0, Hash(1), Compare(1), Alloc(1));
+    C c2(0, Hash(2), Compare(2), Alloc(2));
+    c2.max_load_factor(2);
+    swap(c1, c2);
+
+    LIBCPP_ASSERT(c1.bucket_count() == 0);
+    assert(c1.size() == 0);
+    assert(c1.hash_function() == Hash(2));
+    assert(c1.key_eq() == Compare(2));
+    assert(c1.get_allocator() == Alloc(2));
+    assert(static_cast<std::size_t>(std::distance(c1.begin(), c1.end())) == c1.size());
+    assert(static_cast<std::size_t>(std::distance(c1.cbegin(), c1.cend())) == c1.size());
+    assert(c1.max_load_factor() == 2);
+
+    assert(c2.bucket_count() >= 6);
+    assert(c2.size() == 6);
+    std::multiset<std::string> s;
+    s.insert("one");
+    s.insert("four");
+    CheckConsecutiveKeys<C::const_iterator>(c2.find(1), c2.end(), 1, s);
+    s.insert("two");
+    s.insert("four");
+    CheckConsecutiveKeys<C::const_iterator>(c2.find(2), c2.end(), 2, s);
+    assert(c2.find(3)->second == "three");
+    assert(c2.find(4)->second == "four");
+    assert(c2.hash_function() == Hash(1));
+    assert(c2.key_eq() == Compare(1));
+    assert(c2.get_allocator() == Alloc(1));
+    assert(static_cast<std::size_t>(std::distance(c2.begin(), c2.end())) == c2.size());
+    assert(static_cast<std::size_t>(std::distance(c2.cbegin(), c2.cend())) == c2.size());
+    assert(c2.max_load_factor() == 1);
+  }
+  {
+    typedef test_hash<int> Hash;
+    typedef test_equal_to<int> Compare;
+    typedef other_allocator<std::pair<const int, std::string> > Alloc;
+    typedef std::unordered_multimap<int, std::string, Hash, Compare, Alloc> C;
+    typedef std::pair<int, std::string> P;
+    P a1[] = {
+        P(1, "one"),
+        P(2, "two"),
+        P(3, "three"),
+        P(4, "four"),
+        P(1, "four"),
+        P(2, "four"),
+    };
+    P a2[] = {
+        P(10, "ten"),
+        P(20, "twenty"),
+        P(30, "thirty"),
+        P(40, "forty"),
+        P(50, "fifty"),
+        P(60, "sixty"),
+        P(70, "seventy"),
+        P(80, "eighty"),
+    };
+    C c1(std::begin(a1), std::end(a1), 0, Hash(1), Compare(1), Alloc(1));
+    C c2(std::begin(a2), std::end(a2), 0, Hash(2), Compare(2), Alloc(2));
+    c2.max_load_factor(2);
+    swap(c1, c2);
+
+    assert(c1.bucket_count() >= 8);
+    assert(c1.size() == 8);
+    assert(c1.find(10)->second == "ten");
+    assert(c1.find(20)->second == "twenty");
+    assert(c1.find(30)->second == "thirty");
+    assert(c1.find(40)->second == "forty");
+    assert(c1.find(50)->second == "fifty");
+    assert(c1.find(60)->second == "sixty");
+    assert(c1.find(70)->second == "seventy");
+    assert(c1.find(80)->second == "eighty");
+    assert(c1.hash_function() == Hash(2));
+    assert(c1.key_eq() == Compare(2));
+    assert(c1.get_allocator() == Alloc(2));
+    assert(static_cast<std::size_t>(std::distance(c1.begin(), c1.end())) == c1.size());
+    assert(static_cast<std::size_t>(std::distance(c1.cbegin(), c1.cend())) == c1.size());
+    assert(c1.max_load_factor() == 2);
+
+    assert(c2.bucket_count() >= 6);
+    assert(c2.size() == 6);
+    std::multiset<std::string> s;
+    s.insert("one");
+    s.insert("four");
+    CheckConsecutiveKeys<C::const_iterator>(c2.find(1), c2.end(), 1, s);
+    s.insert("two");
+    s.insert("four");
+    CheckConsecutiveKeys<C::const_iterator>(c2.find(2), c2.end(), 2, s);
+    assert(c2.find(3)->second == "three");
+    assert(c2.find(4)->second == "four");
+    assert(c2.hash_function() == Hash(1));
+    assert(c2.key_eq() == Compare(1));
+    assert(c2.get_allocator() == Alloc(1));
+    assert(static_cast<std::size_t>(std::distance(c2.begin(), c2.end())) == c2.size());
+    assert(static_cast<std::size_t>(std::distance(c2.cbegin(), c2.cend())) == c2.size());
+    assert(c2.max_load_factor() == 1);
+  }
 #if TEST_STD_VER >= 11
-    {
-        typedef test_hash<int> Hash;
-        typedef test_equal_to<int> Compare;
-        typedef min_allocator<std::pair<const int, std::string> > Alloc;
-        typedef std::unordered_multimap<int, std::string, Hash, Compare, Alloc> C;
-        C c1(0, Hash(1), Compare(1), Alloc());
-        C c2(0, Hash(2), Compare(2), Alloc());
-        c2.max_load_factor(2);
-        swap(c1, c2);
-
-        LIBCPP_ASSERT(c1.bucket_count() == 0);
-        assert(c1.size() == 0);
-        assert(c1.hash_function() == Hash(2));
-        assert(c1.key_eq() == Compare(2));
-        assert(c1.get_allocator() == Alloc());
-        assert(static_cast<std::size_t>(std::distance(c1.begin(), c1.end())) == c1.size());
-        assert(static_cast<std::size_t>(std::distance(c1.cbegin(), c1.cend())) == c1.size());
-        assert(c1.max_load_factor() == 2);
-
-        LIBCPP_ASSERT(c2.bucket_count() == 0);
-        assert(c2.size() == 0);
-        assert(c2.hash_function() == Hash(1));
-        assert(c2.key_eq() == Compare(1));
-        assert(c2.get_allocator() == Alloc());
-        assert(static_cast<std::size_t>(std::distance(c2.begin(), c2.end())) == c2.size());
-        assert(static_cast<std::size_t>(std::distance(c2.cbegin(), c2.cend())) == c2.size());
-        assert(c2.max_load_factor() == 1);
-    }
-    {
-        typedef test_hash<int> Hash;
-        typedef test_equal_to<int> Compare;
-        typedef min_allocator<std::pair<const int, std::string> > Alloc;
-        typedef std::unordered_multimap<int, std::string, Hash, Compare, Alloc> C;
-        typedef std::pair<int, std::string> P;
-        P a2[] =
-        {
-            P(10, "ten"),
-            P(20, "twenty"),
-            P(30, "thirty"),
-            P(40, "forty"),
-            P(50, "fifty"),
-            P(60, "sixty"),
-            P(70, "seventy"),
-            P(80, "eighty"),
-        };
-        C c1(0, Hash(1), Compare(1), Alloc());
-        C c2(std::begin(a2), std::end(a2), 0, Hash(2), Compare(2), Alloc());
-        c2.max_load_factor(2);
-        swap(c1, c2);
-
-        assert(c1.bucket_count() >= 8);
-        assert(c1.size() == 8);
-        assert(c1.find(10)->second == "ten");
-        assert(c1.find(20)->second == "twenty");
-        assert(c1.find(30)->second == "thirty");
-        assert(c1.find(40)->second == "forty");
-        assert(c1.find(50)->second == "fifty");
-        assert(c1.find(60)->second == "sixty");
-        assert(c1.find(70)->second == "seventy");
-        assert(c1.find(80)->second == "eighty");
-        assert(c1.hash_function() == Hash(2));
-        assert(c1.key_eq() == Compare(2));
-        assert(c1.get_allocator() == Alloc());
-        assert(static_cast<std::size_t>(std::distance(c1.begin(), c1.end())) == c1.size());
-        assert(static_cast<std::size_t>(std::distance(c1.cbegin(), c1.cend())) == c1.size());
-        assert(c1.max_load_factor() == 2);
-
-        LIBCPP_ASSERT(c2.bucket_count() == 0);
-        assert(c2.size() == 0);
-        assert(c2.hash_function() == Hash(1));
-        assert(c2.key_eq() == Compare(1));
-        assert(c2.get_allocator() == Alloc());
-        assert(static_cast<std::size_t>(std::distance(c2.begin(), c2.end())) == c2.size());
-        assert(static_cast<std::size_t>(std::distance(c2.cbegin(), c2.cend())) == c2.size());
-        assert(c2.max_load_factor() == 1);
-    }
-    {
-        typedef test_hash<int> Hash;
-        typedef test_equal_to<int> Compare;
-        typedef min_allocator<std::pair<const int, std::string> > Alloc;
-        typedef std::unordered_multimap<int, std::string, Hash, Compare, Alloc> C;
-        typedef std::pair<int, std::string> P;
-        P a1[] =
-        {
-            P(1, "one"),
-            P(2, "two"),
-            P(3, "three"),
-            P(4, "four"),
-            P(1, "four"),
-            P(2, "four"),
-        };
-        C c1(std::begin(a1), std::end(a1), 0, Hash(1), Compare(1), Alloc());
-        C c2(0, Hash(2), Compare(2), Alloc());
-        c2.max_load_factor(2);
-        swap(c1, c2);
-
-        LIBCPP_ASSERT(c1.bucket_count() == 0);
-        assert(c1.size() == 0);
-        assert(c1.hash_function() == Hash(2));
-        assert(c1.key_eq() == Compare(2));
-        assert(c1.get_allocator() == Alloc());
-        assert(static_cast<std::size_t>(std::distance(c1.begin(), c1.end())) == c1.size());
-        assert(static_cast<std::size_t>(std::distance(c1.cbegin(), c1.cend())) == c1.size());
-        assert(c1.max_load_factor() == 2);
-
-        assert(c2.bucket_count() >= 6);
-        assert(c2.size() == 6);
-        std::multiset<std::string> s;
-        s.insert("one");
-        s.insert("four");
-        CheckConsecutiveKeys<C::const_iterator>(c2.find(1), c2.end(), 1, s);
-        s.insert("two");
-        s.insert("four");
-        CheckConsecutiveKeys<C::const_iterator>(c2.find(2), c2.end(), 2, s);
-        assert(c2.find(3)->second == "three");
-        assert(c2.find(4)->second == "four");
-        assert(c2.hash_function() == Hash(1));
-        assert(c2.key_eq() == Compare(1));
-        assert(c2.get_allocator() == Alloc());
-        assert(static_cast<std::size_t>(std::distance(c2.begin(), c2.end())) == c2.size());
-        assert(static_cast<std::size_t>(std::distance(c2.cbegin(), c2.cend())) == c2.size());
-        assert(c2.max_load_factor() == 1);
-    }
-    {
-        typedef test_hash<int> Hash;
-        typedef test_equal_to<int> Compare;
-        typedef min_allocator<std::pair<const int, std::string> > Alloc;
-        typedef std::unordered_multimap<int, std::string, Hash, Compare, Alloc> C;
-        typedef std::pair<int, std::string> P;
-        P a1[] =
-        {
-            P(1, "one"),
-            P(2, "two"),
-            P(3, "three"),
-            P(4, "four"),
-            P(1, "four"),
-            P(2, "four"),
-        };
-        P a2[] =
-        {
-            P(10, "ten"),
-            P(20, "twenty"),
-            P(30, "thirty"),
-            P(40, "forty"),
-            P(50, "fifty"),
-            P(60, "sixty"),
-            P(70, "seventy"),
-            P(80, "eighty"),
-        };
-        C c1(std::begin(a1), std::end(a1), 0, Hash(1), Compare(1), Alloc());
-        C c2(std::begin(a2), std::end(a2), 0, Hash(2), Compare(2), Alloc());
-        c2.max_load_factor(2);
-        swap(c1, c2);
-
-        assert(c1.bucket_count() >= 8);
-        assert(c1.size() == 8);
-        assert(c1.find(10)->second == "ten");
-        assert(c1.find(20)->second == "twenty");
-        assert(c1.find(30)->second == "thirty");
-        assert(c1.find(40)->second == "forty");
-        assert(c1.find(50)->second == "fifty");
-        assert(c1.find(60)->second == "sixty");
-        assert(c1.find(70)->second == "seventy");
-        assert(c1.find(80)->second == "eighty");
-        assert(c1.hash_function() == Hash(2));
-        assert(c1.key_eq() == Compare(2));
-        assert(c1.get_allocator() == Alloc());
-        assert(static_cast<std::size_t>(std::distance(c1.begin(), c1.end())) == c1.size());
-        assert(static_cast<std::size_t>(std::distance(c1.cbegin(), c1.cend())) == c1.size());
-        assert(c1.max_load_factor() == 2);
-
-        assert(c2.bucket_count() >= 6);
-        assert(c2.size() == 6);
-        std::multiset<std::string> s;
-        s.insert("one");
-        s.insert("four");
-        CheckConsecutiveKeys<C::const_iterator>(c2.find(1), c2.end(), 1, s);
-        s.insert("two");
-        s.insert("four");
-        CheckConsecutiveKeys<C::const_iterator>(c2.find(2), c2.end(), 2, s);
-        assert(c2.find(3)->second == "three");
-        assert(c2.find(4)->second == "four");
-        assert(c2.hash_function() == Hash(1));
-        assert(c2.key_eq() == Compare(1));
-        assert(c2.get_allocator() == Alloc());
-        assert(static_cast<std::size_t>(std::distance(c2.begin(), c2.end())) == c2.size());
-        assert(static_cast<std::size_t>(std::distance(c2.cbegin(), c2.cend())) == c2.size());
-        assert(c2.max_load_factor() == 1);
-    }
+  {
+    typedef test_hash<int> Hash;
+    typedef test_equal_to<int> Compare;
+    typedef min_allocator<std::pair<const int, std::string> > Alloc;
+    typedef std::unordered_multimap<int, std::string, Hash, Compare, Alloc> C;
+    C c1(0, Hash(1), Compare(1), Alloc());
+    C c2(0, Hash(2), Compare(2), Alloc());
+    c2.max_load_factor(2);
+    swap(c1, c2);
+
+    LIBCPP_ASSERT(c1.bucket_count() == 0);
+    assert(c1.size() == 0);
+    assert(c1.hash_function() == Hash(2));
+    assert(c1.key_eq() == Compare(2));
+    assert(c1.get_allocator() == Alloc());
+    assert(static_cast<std::size_t>(std::distance(c1.begin(), c1.end())) == c1.size());
+    assert(static_cast<std::size_t>(std::distance(c1.cbegin(), c1.cend())) == c1.size());
+    assert(c1.max_load_factor() == 2);
+
+    LIBCPP_ASSERT(c2.bucket_count() == 0);
+    assert(c2.size() == 0);
+    assert(c2.hash_function() == Hash(1));
+    assert(c2.key_eq() == Compare(1));
+    assert(c2.get_allocator() == Alloc());
+    assert(static_cast<std::size_t>(std::distance(c2.begin(), c2.end())) == c2.size());
+    assert(static_cast<std::size_t>(std::distance(c2.cbegin(), c2.cend())) == c2.size());
+    assert(c2.max_load_factor() == 1);
+  }
+  {
+    typedef test_hash<int> Hash;
+    typedef test_equal_to<int> Compare;
+    typedef min_allocator<std::pair<const int, std::string> > Alloc;
+    typedef std::unordered_multimap<int, std::string, Hash, Compare, Alloc> C;
+    typedef std::pair<int, std::string> P;
+    P a2[] = {
+        P(10, "ten"),
+        P(20, "twenty"),
+        P(30, "thirty"),
+        P(40, "forty"),
+        P(50, "fifty"),
+        P(60, "sixty"),
+        P(70, "seventy"),
+        P(80, "eighty"),
+    };
+    C c1(0, Hash(1), Compare(1), Alloc());
+    C c2(std::begin(a2), std::end(a2), 0, Hash(2), Compare(2), Alloc());
+    c2.max_load_factor(2);
+    swap(c1, c2);
+
+    assert(c1.bucket_count() >= 8);
+    assert(c1.size() == 8);
+    assert(c1.find(10)->second == "ten");
+    assert(c1.find(20)->second == "twenty");
+    assert(c1.find(30)->second == "thirty");
+    assert(c1.find(40)->second == "forty");
+    assert(c1.find(50)->second == "fifty");
+    assert(c1.find(60)->second == "sixty");
+    assert(c1.find(70)->second == "seventy");
+    assert(c1.find(80)->second == "eighty");
+    assert(c1.hash_function() == Hash(2));
+    assert(c1.key_eq() == Compare(2));
+    assert(c1.get_allocator() == Alloc());
+    assert(static_cast<std::size_t>(std::distance(c1.begin(), c1.end())) == c1.size());
+    assert(static_cast<std::size_t>(std::distance(c1.cbegin(), c1.cend())) == c1.size());
+    assert(c1.max_load_factor() == 2);
+
+    LIBCPP_ASSERT(c2.bucket_count() == 0);
+    assert(c2.size() == 0);
+    assert(c2.hash_function() == Hash(1));
+    assert(c2.key_eq() == Compare(1));
+    assert(c2.get_allocator() == Alloc());
+    assert(static_cast<std::size_t>(std::distance(c2.begin(), c2.end())) == c2.size());
+    assert(static_cast<std::size_t>(std::distance(c2.cbegin(), c2.cend())) == c2.size());
+    assert(c2.max_load_factor() == 1);
+  }
+  {
+    typedef test_hash<int> Hash;
+    typedef test_equal_to<int> Compare;
+    typedef min_allocator<std::pair<const int, std::string> > Alloc;
+    typedef std::unordered_multimap<int, std::string, Hash, Compare, Alloc> C;
+    typedef std::pair<int, std::string> P;
+    P a1[] = {
+        P(1, "one"),
+        P(2, "two"),
+        P(3, "three"),
+        P(4, "four"),
+        P(1, "four"),
+        P(2, "four"),
+    };
+    C c1(std::begin(a1), std::end(a1), 0, Hash(1), Compare(1), Alloc());
+    C c2(0, Hash(2), Compare(2), Alloc());
+    c2.max_load_factor(2);
+    swap(c1, c2);
+
+    LIBCPP_ASSERT(c1.bucket_count() == 0);
+    assert(c1.size() == 0);
+    assert(c1.hash_function() == Hash(2));
+    assert(c1.key_eq() == Compare(2));
+    assert(c1.get_allocator() == Alloc());
+    assert(static_cast<std::size_t>(std::distance(c1.begin(), c1.end())) == c1.size());
+    assert(static_cast<std::size_t>(std::distance(c1.cbegin(), c1.cend())) == c1.size());
+    assert(c1.max_load_factor() == 2);
+
+    assert(c2.bucket_count() >= 6);
+    assert(c2.size() == 6);
+    std::multiset<std::string> s;
+    s.insert("one");
+    s.insert("four");
+    CheckConsecutiveKeys<C::const_iterator>(c2.find(1), c2.end(), 1, s);
+    s.insert("two");
+    s.insert("four");
+    CheckConsecutiveKeys<C::const_iterator>(c2.find(2), c2.end(), 2, s);
+    assert(c2.find(3)->second == "three");
+    assert(c2.find(4)->second == "four");
+    assert(c2.hash_function() == Hash(1));
+    assert(c2.key_eq() == Compare(1));
+    assert(c2.get_allocator() == Alloc());
+    assert(static_cast<std::size_t>(std::distance(c2.begin(), c2.end())) == c2.size());
+    assert(static_cast<std::size_t>(std::distance(c2.cbegin(), c2.cend())) == c2.size());
+    assert(c2.max_load_factor() == 1);
+  }
+  {
+    typedef test_hash<int> Hash;
+    typedef test_equal_to<int> Compare;
+    typedef min_allocator<std::pair<const int, std::string> > Alloc;
+    typedef std::unordered_multimap<int, std::string, Hash, Compare, Alloc> C;
+    typedef std::pair<int, std::string> P;
+    P a1[] = {
+        P(1, "one"),
+        P(2, "two"),
+        P(3, "three"),
+        P(4, "four"),
+        P(1, "four"),
+        P(2, "four"),
+    };
+    P a2[] = {
+        P(10, "ten"),
+        P(20, "twenty"),
+        P(30, "thirty"),
+        P(40, "forty"),
+        P(50, "fifty"),
+        P(60, "sixty"),
+        P(70, "seventy"),
+        P(80, "eighty"),
+    };
+    C c1(std::begin(a1), std::end(a1), 0, Hash(1), Compare(1), Alloc());
+    C c2(std::begin(a2), std::end(a2), 0, Hash(2), Compare(2), Alloc());
+    c2.max_load_factor(2);
+    swap(c1, c2);
+
+    assert(c1.bucket_count() >= 8);
+    assert(c1.size() == 8);
+    assert(c1.find(10)->second == "ten");
+    assert(c1.find(20)->second == "twenty");
+    assert(c1.find(30)->second == "thirty");
+    assert(c1.find(40)->second == "forty");
+    assert(c1.find(50)->second == "fifty");
+    assert(c1.find(60)->second == "sixty");
+    assert(c1.find(70)->second == "seventy");
+    assert(c1.find(80)->second == "eighty");
+    assert(c1.hash_function() == Hash(2));
+    assert(c1.key_eq() == Compare(2));
+    assert(c1.get_allocator() == Alloc());
+    assert(static_cast<std::size_t>(std::distance(c1.begin(), c1.end())) == c1.size());
+    assert(static_cast<std::size_t>(std::distance(c1.cbegin(), c1.cend())) == c1.size());
+    assert(c1.max_load_factor() == 2);
+
+    assert(c2.bucket_count() >= 6);
+    assert(c2.size() == 6);
+    std::multiset<std::string> s;
+    s.insert("one");
+    s.insert("four");
+    CheckConsecutiveKeys<C::const_iterator>(c2.find(1), c2.end(), 1, s);
+    s.insert("two");
+    s.insert("four");
+    CheckConsecutiveKeys<C::const_iterator>(c2.find(2), c2.end(), 2, s);
+    assert(c2.find(3)->second == "three");
+    assert(c2.find(4)->second == "four");
+    assert(c2.hash_function() == Hash(1));
+    assert(c2.key_eq() == Compare(1));
+    assert(c2.get_allocator() == Alloc());
+    assert(static_cast<std::size_t>(std::distance(c2.begin(), c2.end())) == c2.size());
+    assert(static_cast<std::size_t>(std::distance(c2.cbegin(), c2.cend())) == c2.size());
+    assert(c2.max_load_factor() == 1);
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/unord/unord.multiset/allocator_mismatch.verify.cpp b/libcxx/test/std/containers/unord/unord.multiset/allocator_mismatch.verify.cpp
index 39ec7077511a9..0ed0c37191a70 100644
--- a/libcxx/test/std/containers/unord/unord.multiset/allocator_mismatch.verify.cpp
+++ b/libcxx/test/std/containers/unord/unord.multiset/allocator_mismatch.verify.cpp
@@ -12,4 +12,4 @@
 #include <unordered_set>
 
 std::unordered_multiset<int, std::hash<int>, std::less<int>, std::allocator<long> > v;
-  // expected-error-re@*:* {{static assertion failed{{.*}}Allocator::value_type must be same type as value_type}}
+// expected-error-re@*:* {{static assertion failed{{.*}}Allocator::value_type must be same type as value_type}}
diff --git a/libcxx/test/std/containers/unord/unord.multiset/bucket.pass.cpp b/libcxx/test/std/containers/unord/unord.multiset/bucket.pass.cpp
index dd487106c4358..27d3a27555a0b 100644
--- a/libcxx/test/std/containers/unord/unord.multiset/bucket.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.multiset/bucket.pass.cpp
@@ -20,47 +20,29 @@
 #include "test_macros.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef std::unordered_multiset<int> C;
-        typedef int P;
-        P a[] =
-        {
-            P(1),
-            P(2),
-            P(3),
-            P(4),
-            P(1),
-            P(2)
-        };
-        const C c(std::begin(a), std::end(a));
-        std::size_t bc = c.bucket_count();
-        assert(bc >= 7);
-        for (std::size_t i = 0; i < 13; ++i)
-            LIBCPP_ASSERT(c.bucket(i) == i % bc);
-    }
+int main(int, char**) {
+  {
+    typedef std::unordered_multiset<int> C;
+    typedef int P;
+    P a[] = {P(1), P(2), P(3), P(4), P(1), P(2)};
+    const C c(std::begin(a), std::end(a));
+    std::size_t bc = c.bucket_count();
+    assert(bc >= 7);
+    for (std::size_t i = 0; i < 13; ++i)
+      LIBCPP_ASSERT(c.bucket(i) == i % bc);
+  }
 #if TEST_STD_VER >= 11
-    {
-        typedef std::unordered_multiset<int, std::hash<int>,
-                                      std::equal_to<int>, min_allocator<int>> C;
-        typedef int P;
-        P a[] =
-        {
-            P(1),
-            P(2),
-            P(3),
-            P(4),
-            P(1),
-            P(2)
-        };
-        const C c(std::begin(a), std::end(a));
-        std::size_t bc = c.bucket_count();
-        assert(bc >= 7);
-        for (std::size_t i = 0; i < 13; ++i)
-            LIBCPP_ASSERT(c.bucket(i) == i % bc);
-    }
+  {
+    typedef std::unordered_multiset<int, std::hash<int>, std::equal_to<int>, min_allocator<int>> C;
+    typedef int P;
+    P a[] = {P(1), P(2), P(3), P(4), P(1), P(2)};
+    const C c(std::begin(a), std::end(a));
+    std::size_t bc = c.bucket_count();
+    assert(bc >= 7);
+    for (std::size_t i = 0; i < 13; ++i)
+      LIBCPP_ASSERT(c.bucket(i) == i % bc);
+  }
 #endif
 
-    return 0;
+  return 0;
 }
diff --git a/libcxx/test/std/containers/unord/unord.multiset/bucket_count.pass.cpp b/libcxx/test/std/containers/unord/unord.multiset/bucket_count.pass.cpp
index 8f389ebc206ba..9df863bba120f 100644
--- a/libcxx/test/std/containers/unord/unord.multiset/bucket_count.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.multiset/bucket_count.pass.cpp
@@ -20,55 +20,32 @@
 #include "test_macros.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef std::unordered_multiset<int> C;
-        const C c;
-        LIBCPP_ASSERT(c.bucket_count() == 0);
-    }
-    {
-        typedef std::unordered_multiset<int> C;
-        typedef int P;
-        P a[] =
-        {
-            P(10),
-            P(20),
-            P(30),
-            P(40),
-            P(50),
-            P(60),
-            P(70),
-            P(80)
-        };
-        const C c(std::begin(a), std::end(a));
-        assert(c.bucket_count() >= 8);
-    }
+int main(int, char**) {
+  {
+    typedef std::unordered_multiset<int> C;
+    const C c;
+    LIBCPP_ASSERT(c.bucket_count() == 0);
+  }
+  {
+    typedef std::unordered_multiset<int> C;
+    typedef int P;
+    P a[] = {P(10), P(20), P(30), P(40), P(50), P(60), P(70), P(80)};
+    const C c(std::begin(a), std::end(a));
+    assert(c.bucket_count() >= 8);
+  }
 #if TEST_STD_VER >= 11
-    {
-        typedef std::unordered_multiset<int, std::hash<int>,
-                                      std::equal_to<int>, min_allocator<int>> C;
-        const C c;
-        LIBCPP_ASSERT(c.bucket_count() == 0);
-    }
-    {
-        typedef std::unordered_multiset<int, std::hash<int>,
-                                      std::equal_to<int>, min_allocator<int>> C;
-        typedef int P;
-        P a[] =
-        {
-            P(10),
-            P(20),
-            P(30),
-            P(40),
-            P(50),
-            P(60),
-            P(70),
-            P(80)
-        };
-        const C c(std::begin(a), std::end(a));
-        assert(c.bucket_count() >= 8);
-    }
+  {
+    typedef std::unordered_multiset<int, std::hash<int>, std::equal_to<int>, min_allocator<int>> C;
+    const C c;
+    LIBCPP_ASSERT(c.bucket_count() == 0);
+  }
+  {
+    typedef std::unordered_multiset<int, std::hash<int>, std::equal_to<int>, min_allocator<int>> C;
+    typedef int P;
+    P a[] = {P(10), P(20), P(30), P(40), P(50), P(60), P(70), P(80)};
+    const C c(std::begin(a), std::end(a));
+    assert(c.bucket_count() >= 8);
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/unord/unord.multiset/bucket_size.pass.cpp b/libcxx/test/std/containers/unord/unord.multiset/bucket_size.pass.cpp
index be7a651416e21..306cadd87e9d1 100644
--- a/libcxx/test/std/containers/unord/unord.multiset/bucket_size.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.multiset/bucket_size.pass.cpp
@@ -20,54 +20,36 @@
 #include "test_macros.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef std::unordered_multiset<int> C;
-        typedef int P;
-        P a[] =
-        {
-            P(1),
-            P(2),
-            P(3),
-            P(4),
-            P(1),
-            P(2)
-        };
-        const C c(std::begin(a), std::end(a));
-        assert(c.bucket_count() >= 7);
-        LIBCPP_ASSERT(c.bucket_size(0) == 0);
-        LIBCPP_ASSERT(c.bucket_size(1) == 2);
-        LIBCPP_ASSERT(c.bucket_size(2) == 2);
-        LIBCPP_ASSERT(c.bucket_size(3) == 1);
-        LIBCPP_ASSERT(c.bucket_size(4) == 1);
-        LIBCPP_ASSERT(c.bucket_size(5) == 0);
-        LIBCPP_ASSERT(c.bucket_size(6) == 0);
-    }
+int main(int, char**) {
+  {
+    typedef std::unordered_multiset<int> C;
+    typedef int P;
+    P a[] = {P(1), P(2), P(3), P(4), P(1), P(2)};
+    const C c(std::begin(a), std::end(a));
+    assert(c.bucket_count() >= 7);
+    LIBCPP_ASSERT(c.bucket_size(0) == 0);
+    LIBCPP_ASSERT(c.bucket_size(1) == 2);
+    LIBCPP_ASSERT(c.bucket_size(2) == 2);
+    LIBCPP_ASSERT(c.bucket_size(3) == 1);
+    LIBCPP_ASSERT(c.bucket_size(4) == 1);
+    LIBCPP_ASSERT(c.bucket_size(5) == 0);
+    LIBCPP_ASSERT(c.bucket_size(6) == 0);
+  }
 #if TEST_STD_VER >= 11
-    {
-        typedef std::unordered_multiset<int, std::hash<int>,
-                                      std::equal_to<int>, min_allocator<int>> C;
-        typedef int P;
-        P a[] =
-        {
-            P(1),
-            P(2),
-            P(3),
-            P(4),
-            P(1),
-            P(2)
-        };
-        const C c(std::begin(a), std::end(a));
-        assert(c.bucket_count() >= 7);
-        LIBCPP_ASSERT(c.bucket_size(0) == 0);
-        LIBCPP_ASSERT(c.bucket_size(1) == 2);
-        LIBCPP_ASSERT(c.bucket_size(2) == 2);
-        LIBCPP_ASSERT(c.bucket_size(3) == 1);
-        LIBCPP_ASSERT(c.bucket_size(4) == 1);
-        LIBCPP_ASSERT(c.bucket_size(5) == 0);
-        LIBCPP_ASSERT(c.bucket_size(6) == 0);
-    }
+  {
+    typedef std::unordered_multiset<int, std::hash<int>, std::equal_to<int>, min_allocator<int>> C;
+    typedef int P;
+    P a[] = {P(1), P(2), P(3), P(4), P(1), P(2)};
+    const C c(std::begin(a), std::end(a));
+    assert(c.bucket_count() >= 7);
+    LIBCPP_ASSERT(c.bucket_size(0) == 0);
+    LIBCPP_ASSERT(c.bucket_size(1) == 2);
+    LIBCPP_ASSERT(c.bucket_size(2) == 2);
+    LIBCPP_ASSERT(c.bucket_size(3) == 1);
+    LIBCPP_ASSERT(c.bucket_size(4) == 1);
+    LIBCPP_ASSERT(c.bucket_size(5) == 0);
+    LIBCPP_ASSERT(c.bucket_size(6) == 0);
+  }
 #endif
-    return 0;
+  return 0;
 }
diff --git a/libcxx/test/std/containers/unord/unord.multiset/clear.pass.cpp b/libcxx/test/std/containers/unord/unord.multiset/clear.pass.cpp
index 01ff045938397..2471c2a42ec37 100644
--- a/libcxx/test/std/containers/unord/unord.multiset/clear.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.multiset/clear.pass.cpp
@@ -20,44 +20,26 @@
 #include "test_macros.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef std::unordered_multiset<int> C;
-        typedef int P;
-        P a[] =
-        {
-            P(1),
-            P(2),
-            P(3),
-            P(4),
-            P(1),
-            P(2)
-        };
-        C c(a, a + sizeof(a)/sizeof(a[0]));
-        ASSERT_NOEXCEPT(c.clear());
-        c.clear();
-        assert(c.size() == 0);
-    }
+int main(int, char**) {
+  {
+    typedef std::unordered_multiset<int> C;
+    typedef int P;
+    P a[] = {P(1), P(2), P(3), P(4), P(1), P(2)};
+    C c(a, a + sizeof(a) / sizeof(a[0]));
+    ASSERT_NOEXCEPT(c.clear());
+    c.clear();
+    assert(c.size() == 0);
+  }
 #if TEST_STD_VER >= 11
-    {
-        typedef std::unordered_multiset<int, std::hash<int>,
-                                      std::equal_to<int>, min_allocator<int>> C;
-        typedef int P;
-        P a[] =
-        {
-            P(1),
-            P(2),
-            P(3),
-            P(4),
-            P(1),
-            P(2)
-        };
-        C c(a, a + sizeof(a)/sizeof(a[0]));
-        ASSERT_NOEXCEPT(c.clear());
-        c.clear();
-        assert(c.size() == 0);
-    }
+  {
+    typedef std::unordered_multiset<int, std::hash<int>, std::equal_to<int>, min_allocator<int>> C;
+    typedef int P;
+    P a[] = {P(1), P(2), P(3), P(4), P(1), P(2)};
+    C c(a, a + sizeof(a) / sizeof(a[0]));
+    ASSERT_NOEXCEPT(c.clear());
+    c.clear();
+    assert(c.size() == 0);
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/unord/unord.multiset/contains.transparent.pass.cpp b/libcxx/test/std/containers/unord/unord.multiset/contains.transparent.pass.cpp
index b14021c6b8c8c..58a4fe72c78e8 100644
--- a/libcxx/test/std/containers/unord/unord.multiset/contains.transparent.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.multiset/contains.transparent.pass.cpp
@@ -20,44 +20,43 @@
 #include <unordered_set>
 #include "test_transparent_unordered.h"
 
-int main(int, char**)
-{
-    using key_type = StoredType<int>;
-
-    {
-      // Make sure conversions don't happen for transparent non-final hasher and key_equal
-      using S = unord_set_type<std::unordered_multiset, transparent_hash, std::equal_to<>>;
-      test_transparent_contains<S>({1, 1, 2});
-      test_transparent_contains<const S>({1, 1, 2});
-    }
-
-    {
-      // Make sure conversions don't happen for transparent final hasher and key_equal
-      using S = unord_set_type<std::unordered_multiset, transparent_hash_final, transparent_equal_final>;
-      test_transparent_contains<S>({1, 1, 2});
-      test_transparent_contains<const S>({1, 1, 2});
-    }
-
-    {
-      // Make sure conversions do happen for non-transparent hasher
-      using S = unord_set_type<std::unordered_multiset, non_transparent_hash, std::equal_to<>>;
-      test_non_transparent_contains<S>({1, 1, 2});
-      test_non_transparent_contains<const S>({1, 1, 2});
-    }
-
-    {
-      // Make sure conversions do happen for non-transparent key_equal
-      using S = unord_set_type<std::unordered_multiset, transparent_hash, std::equal_to<key_type>>;
-      test_non_transparent_contains<S>({1, 1, 2});
-      test_non_transparent_contains<const S>({1, 1, 2});
-    }
-
-    {
-      // Make sure conversions do happen for both non-transparent hasher and key_equal
-      using S = unord_set_type<std::unordered_multiset, non_transparent_hash, std::equal_to<key_type>>;
-      test_non_transparent_contains<S>({1, 1, 2});
-      test_non_transparent_contains<const S>({1, 1, 2});
-    }
-
-    return 0;
+int main(int, char**) {
+  using key_type = StoredType<int>;
+
+  {
+    // Make sure conversions don't happen for transparent non-final hasher and key_equal
+    using S = unord_set_type<std::unordered_multiset, transparent_hash, std::equal_to<>>;
+    test_transparent_contains<S>({1, 1, 2});
+    test_transparent_contains<const S>({1, 1, 2});
+  }
+
+  {
+    // Make sure conversions don't happen for transparent final hasher and key_equal
+    using S = unord_set_type<std::unordered_multiset, transparent_hash_final, transparent_equal_final>;
+    test_transparent_contains<S>({1, 1, 2});
+    test_transparent_contains<const S>({1, 1, 2});
+  }
+
+  {
+    // Make sure conversions do happen for non-transparent hasher
+    using S = unord_set_type<std::unordered_multiset, non_transparent_hash, std::equal_to<>>;
+    test_non_transparent_contains<S>({1, 1, 2});
+    test_non_transparent_contains<const S>({1, 1, 2});
+  }
+
+  {
+    // Make sure conversions do happen for non-transparent key_equal
+    using S = unord_set_type<std::unordered_multiset, transparent_hash, std::equal_to<key_type>>;
+    test_non_transparent_contains<S>({1, 1, 2});
+    test_non_transparent_contains<const S>({1, 1, 2});
+  }
+
+  {
+    // Make sure conversions do happen for both non-transparent hasher and key_equal
+    using S = unord_set_type<std::unordered_multiset, non_transparent_hash, std::equal_to<key_type>>;
+    test_non_transparent_contains<S>({1, 1, 2});
+    test_non_transparent_contains<const S>({1, 1, 2});
+  }
+
+  return 0;
 }
diff --git a/libcxx/test/std/containers/unord/unord.multiset/count.pass.cpp b/libcxx/test/std/containers/unord/unord.multiset/count.pass.cpp
index 9feb51c789259..be4514c4d6938 100644
--- a/libcxx/test/std/containers/unord/unord.multiset/count.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.multiset/count.pass.cpp
@@ -20,52 +20,26 @@
 #include "test_macros.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef std::unordered_multiset<int> C;
-        typedef int P;
-        P a[] =
-        {
-            P(10),
-            P(20),
-            P(30),
-            P(40),
-            P(50),
-            P(50),
-            P(50),
-            P(60),
-            P(70),
-            P(80)
-        };
-        const C c(std::begin(a), std::end(a));
-        assert(c.count(30) == 1);
-        assert(c.count(50) == 3);
-        assert(c.count(5) == 0);
-    }
+int main(int, char**) {
+  {
+    typedef std::unordered_multiset<int> C;
+    typedef int P;
+    P a[] = {P(10), P(20), P(30), P(40), P(50), P(50), P(50), P(60), P(70), P(80)};
+    const C c(std::begin(a), std::end(a));
+    assert(c.count(30) == 1);
+    assert(c.count(50) == 3);
+    assert(c.count(5) == 0);
+  }
 #if TEST_STD_VER >= 11
-    {
-        typedef std::unordered_multiset<int, std::hash<int>,
-                                      std::equal_to<int>, min_allocator<int>> C;
-        typedef int P;
-        P a[] =
-        {
-            P(10),
-            P(20),
-            P(30),
-            P(40),
-            P(50),
-            P(50),
-            P(50),
-            P(60),
-            P(70),
-            P(80)
-        };
-        const C c(std::begin(a), std::end(a));
-        assert(c.count(30) == 1);
-        assert(c.count(50) == 3);
-        assert(c.count(5) == 0);
-    }
+  {
+    typedef std::unordered_multiset<int, std::hash<int>, std::equal_to<int>, min_allocator<int>> C;
+    typedef int P;
+    P a[] = {P(10), P(20), P(30), P(40), P(50), P(50), P(50), P(60), P(70), P(80)};
+    const C c(std::begin(a), std::end(a));
+    assert(c.count(30) == 1);
+    assert(c.count(50) == 3);
+    assert(c.count(5) == 0);
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/unord/unord.multiset/count.transparent.pass.cpp b/libcxx/test/std/containers/unord/unord.multiset/count.transparent.pass.cpp
index 64f9eddbddae0..0c0ca58558d39 100644
--- a/libcxx/test/std/containers/unord/unord.multiset/count.transparent.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.multiset/count.transparent.pass.cpp
@@ -21,44 +21,43 @@
 
 #include "test_transparent_unordered.h"
 
-int main(int, char**)
-{
-    using key_type = StoredType<int>;
-
-    {
-      // Make sure conversions don't happen for transparent non-final hasher and key_equal
-      using S = unord_set_type<std::unordered_multiset, transparent_hash, std::equal_to<>>;
-      test_transparent_count<S>({1, 1, 2});
-      test_transparent_count<const S>({1, 1, 2});
-    }
-
-    {
-      // Make sure conversions don't happen for transparent final hasher and key_equal
-      using S = unord_set_type<std::unordered_multiset, transparent_hash_final, transparent_equal_final>;
-      test_transparent_count<S>({1, 1, 2});
-      test_transparent_count<const S>({1, 1, 2});
-    }
-
-    {
-      // Make sure conversions do happen for non-transparent hasher
-      using S = unord_set_type<std::unordered_multiset, non_transparent_hash, std::equal_to<>>;
-      test_non_transparent_count<S>({1, 1, 2});
-      test_non_transparent_count<const S>({1, 1, 2});
-    }
-
-    {
-      // Make sure conversions do happen for non-transparent key_equal
-      using S = unord_set_type<std::unordered_multiset, transparent_hash, std::equal_to<key_type>>;
-      test_non_transparent_count<S>({1, 1, 2});
-      test_non_transparent_count<const S>({1, 1, 2});
-    }
-
-    {
-      // Make sure conversions do happen for both non-transparent hasher and key_equal
-      using S = unord_set_type<std::unordered_multiset, non_transparent_hash, std::equal_to<key_type>>;
-      test_non_transparent_count<S>({1, 1, 2});
-      test_non_transparent_count<const S>({1, 1, 2});
-    }
-
-    return 0;
+int main(int, char**) {
+  using key_type = StoredType<int>;
+
+  {
+    // Make sure conversions don't happen for transparent non-final hasher and key_equal
+    using S = unord_set_type<std::unordered_multiset, transparent_hash, std::equal_to<>>;
+    test_transparent_count<S>({1, 1, 2});
+    test_transparent_count<const S>({1, 1, 2});
+  }
+
+  {
+    // Make sure conversions don't happen for transparent final hasher and key_equal
+    using S = unord_set_type<std::unordered_multiset, transparent_hash_final, transparent_equal_final>;
+    test_transparent_count<S>({1, 1, 2});
+    test_transparent_count<const S>({1, 1, 2});
+  }
+
+  {
+    // Make sure conversions do happen for non-transparent hasher
+    using S = unord_set_type<std::unordered_multiset, non_transparent_hash, std::equal_to<>>;
+    test_non_transparent_count<S>({1, 1, 2});
+    test_non_transparent_count<const S>({1, 1, 2});
+  }
+
+  {
+    // Make sure conversions do happen for non-transparent key_equal
+    using S = unord_set_type<std::unordered_multiset, transparent_hash, std::equal_to<key_type>>;
+    test_non_transparent_count<S>({1, 1, 2});
+    test_non_transparent_count<const S>({1, 1, 2});
+  }
+
+  {
+    // Make sure conversions do happen for both non-transparent hasher and key_equal
+    using S = unord_set_type<std::unordered_multiset, non_transparent_hash, std::equal_to<key_type>>;
+    test_non_transparent_count<S>({1, 1, 2});
+    test_non_transparent_count<const S>({1, 1, 2});
+  }
+
+  return 0;
 }
diff --git a/libcxx/test/std/containers/unord/unord.multiset/emplace.pass.cpp b/libcxx/test/std/containers/unord/unord.multiset/emplace.pass.cpp
index 820b4cb0c97f8..886ddd74efd45 100644
--- a/libcxx/test/std/containers/unord/unord.multiset/emplace.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.multiset/emplace.pass.cpp
@@ -24,41 +24,41 @@
 #include "../../Emplaceable.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef std::unordered_multiset<Emplaceable> C;
-        typedef C::iterator R;
-        C c;
-        R r = c.emplace();
-        assert(c.size() == 1);
-        assert(*r == Emplaceable());
+int main(int, char**) {
+  {
+    typedef std::unordered_multiset<Emplaceable> C;
+    typedef C::iterator R;
+    C c;
+    R r = c.emplace();
+    assert(c.size() == 1);
+    assert(*r == Emplaceable());
 
-        r = c.emplace(Emplaceable(5, 6));
-        assert(c.size() == 2);
-        assert(*r == Emplaceable(5, 6));
+    r = c.emplace(Emplaceable(5, 6));
+    assert(c.size() == 2);
+    assert(*r == Emplaceable(5, 6));
 
-        r = c.emplace(5, 6);
-        assert(c.size() == 3);
-        assert(*r == Emplaceable(5, 6));
-    }
-    {
-        typedef std::unordered_multiset<Emplaceable, std::hash<Emplaceable>,
-                      std::equal_to<Emplaceable>, min_allocator<Emplaceable>> C;
-        typedef C::iterator R;
-        C c;
-        R r = c.emplace();
-        assert(c.size() == 1);
-        assert(*r == Emplaceable());
+    r = c.emplace(5, 6);
+    assert(c.size() == 3);
+    assert(*r == Emplaceable(5, 6));
+  }
+  {
+    typedef std::
+        unordered_multiset<Emplaceable, std::hash<Emplaceable>, std::equal_to<Emplaceable>, min_allocator<Emplaceable>>
+            C;
+    typedef C::iterator R;
+    C c;
+    R r = c.emplace();
+    assert(c.size() == 1);
+    assert(*r == Emplaceable());
 
-        r = c.emplace(Emplaceable(5, 6));
-        assert(c.size() == 2);
-        assert(*r == Emplaceable(5, 6));
+    r = c.emplace(Emplaceable(5, 6));
+    assert(c.size() == 2);
+    assert(*r == Emplaceable(5, 6));
 
-        r = c.emplace(5, 6);
-        assert(c.size() == 3);
-        assert(*r == Emplaceable(5, 6));
-    }
+    r = c.emplace(5, 6);
+    assert(c.size() == 3);
+    assert(*r == Emplaceable(5, 6));
+  }
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/unord/unord.multiset/emplace_hint.pass.cpp b/libcxx/test/std/containers/unord/unord.multiset/emplace_hint.pass.cpp
index 285a13db3b32a..6eb5fca80dc89 100644
--- a/libcxx/test/std/containers/unord/unord.multiset/emplace_hint.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.multiset/emplace_hint.pass.cpp
@@ -17,7 +17,6 @@
 // template <class... Args>
 //     iterator emplace_hint(const_iterator p, Args&&... args);
 
-
 #include <unordered_set>
 #include <cassert>
 
@@ -25,43 +24,43 @@
 #include "../../Emplaceable.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef std::unordered_multiset<Emplaceable> C;
-        typedef C::iterator R;
-        C c;
-        C::const_iterator e = c.end();
-        R r = c.emplace_hint(e);
-        assert(c.size() == 1);
-        assert(*r == Emplaceable());
+int main(int, char**) {
+  {
+    typedef std::unordered_multiset<Emplaceable> C;
+    typedef C::iterator R;
+    C c;
+    C::const_iterator e = c.end();
+    R r                 = c.emplace_hint(e);
+    assert(c.size() == 1);
+    assert(*r == Emplaceable());
 
-        r = c.emplace_hint(c.end(), Emplaceable(5, 6));
-        assert(c.size() == 2);
-        assert(*r == Emplaceable(5, 6));
+    r = c.emplace_hint(c.end(), Emplaceable(5, 6));
+    assert(c.size() == 2);
+    assert(*r == Emplaceable(5, 6));
 
-        r = c.emplace_hint(r, 5, 6);
-        assert(c.size() == 3);
-        assert(*r == Emplaceable(5, 6));
-    }
-    {
-        typedef std::unordered_multiset<Emplaceable, std::hash<Emplaceable>,
-                      std::equal_to<Emplaceable>, min_allocator<Emplaceable>> C;
-        typedef C::iterator R;
-        C c;
-        C::const_iterator e = c.end();
-        R r = c.emplace_hint(e);
-        assert(c.size() == 1);
-        assert(*r == Emplaceable());
+    r = c.emplace_hint(r, 5, 6);
+    assert(c.size() == 3);
+    assert(*r == Emplaceable(5, 6));
+  }
+  {
+    typedef std::
+        unordered_multiset<Emplaceable, std::hash<Emplaceable>, std::equal_to<Emplaceable>, min_allocator<Emplaceable>>
+            C;
+    typedef C::iterator R;
+    C c;
+    C::const_iterator e = c.end();
+    R r                 = c.emplace_hint(e);
+    assert(c.size() == 1);
+    assert(*r == Emplaceable());
 
-        r = c.emplace_hint(c.end(), Emplaceable(5, 6));
-        assert(c.size() == 2);
-        assert(*r == Emplaceable(5, 6));
+    r = c.emplace_hint(c.end(), Emplaceable(5, 6));
+    assert(c.size() == 2);
+    assert(*r == Emplaceable(5, 6));
 
-        r = c.emplace_hint(r, 5, 6);
-        assert(c.size() == 3);
-        assert(*r == Emplaceable(5, 6));
-    }
+    r = c.emplace_hint(r, 5, 6);
+    assert(c.size() == 3);
+    assert(*r == Emplaceable(5, 6));
+  }
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/unord/unord.multiset/empty.pass.cpp b/libcxx/test/std/containers/unord/unord.multiset/empty.pass.cpp
index 078f95f564b68..63ef2a858880e 100644
--- a/libcxx/test/std/containers/unord/unord.multiset/empty.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.multiset/empty.pass.cpp
@@ -18,9 +18,8 @@
 #include "test_macros.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
+int main(int, char**) {
+  {
     typedef std::unordered_multiset<int> M;
     M m;
     ASSERT_NOEXCEPT(m.empty());
@@ -29,9 +28,9 @@ int main(int, char**)
     assert(!m.empty());
     m.clear();
     assert(m.empty());
-    }
+  }
 #if TEST_STD_VER >= 11
-    {
+  {
     typedef std::unordered_multiset<int, std::hash<int>, std::equal_to<int>, min_allocator<int>> M;
     M m;
     ASSERT_NOEXCEPT(m.empty());
@@ -40,7 +39,7 @@ int main(int, char**)
     assert(!m.empty());
     m.clear();
     assert(m.empty());
-    }
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/unord/unord.multiset/empty.verify.cpp b/libcxx/test/std/containers/unord/unord.multiset/empty.verify.cpp
index 8f5fa55c3aefb..8df5b993f31da 100644
--- a/libcxx/test/std/containers/unord/unord.multiset/empty.verify.cpp
+++ b/libcxx/test/std/containers/unord/unord.multiset/empty.verify.cpp
@@ -17,6 +17,6 @@
 #include <unordered_set>
 
 void f() {
-    std::unordered_multiset<int> c;
-    c.empty(); // expected-warning {{ignoring return value of function declared with 'nodiscard' attribute}}
+  std::unordered_multiset<int> c;
+  c.empty(); // expected-warning {{ignoring return value of function declared with 'nodiscard' attribute}}
 }
diff --git a/libcxx/test/std/containers/unord/unord.multiset/eq.pass.cpp b/libcxx/test/std/containers/unord/unord.multiset/eq.pass.cpp
index 7c4cfd63c7bb6..d05bc05a89695 100644
--- a/libcxx/test/std/containers/unord/unord.multiset/eq.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.multiset/eq.pass.cpp
@@ -27,189 +27,107 @@
 
 #include "test_comparisons.h"
 
-int main(int, char**)
-{
-    {
-        typedef std::unordered_multiset<int> C;
-        typedef int P;
-        P a[] =
-        {
-            P(10),
-            P(20),
-            P(20),
-            P(30),
-            P(40),
-            P(50),
-            P(50),
-            P(50),
-            P(60),
-            P(70),
-            P(80)
-        };
-        const C c1(std::begin(a), std::end(a));
-        const C c2;
-        assert(!(c1 == c2));
-        assert( (c1 != c2));
-    }
-    {
-        typedef std::unordered_multiset<int> C;
-        typedef int P;
-        P a[] =
-        {
-            P(10),
-            P(20),
-            P(20),
-            P(30),
-            P(40),
-            P(50),
-            P(50),
-            P(50),
-            P(60),
-            P(70),
-            P(80)
-        };
-        const C c1(std::begin(a), std::end(a));
-        const C c2 = c1;
-        assert( (c1 == c2));
-        assert(!(c1 != c2));
-    }
-    {
-        typedef std::unordered_multiset<int> C;
-        typedef int P;
-        P a[] =
-        {
-            P(10),
-            P(20),
-            P(20),
-            P(30),
-            P(40),
-            P(50),
-            P(50),
-            P(50),
-            P(60),
-            P(70),
-            P(80)
-        };
-        C c1(std::begin(a), std::end(a));
-        C c2 = c1;
-        c2.rehash(30);
-        assert( (c1 == c2));
-        assert(!(c1 != c2));
-        c2.insert(P(90));
-        assert(!(c1 == c2));
-        assert( (c1 != c2));
-        c1.insert(P(90));
-        assert( (c1 == c2));
-        assert(!(c1 != c2));
-    }
+int main(int, char**) {
+  {
+    typedef std::unordered_multiset<int> C;
+    typedef int P;
+    P a[] = {P(10), P(20), P(20), P(30), P(40), P(50), P(50), P(50), P(60), P(70), P(80)};
+    const C c1(std::begin(a), std::end(a));
+    const C c2;
+    assert(!(c1 == c2));
+    assert((c1 != c2));
+  }
+  {
+    typedef std::unordered_multiset<int> C;
+    typedef int P;
+    P a[] = {P(10), P(20), P(20), P(30), P(40), P(50), P(50), P(50), P(60), P(70), P(80)};
+    const C c1(std::begin(a), std::end(a));
+    const C c2 = c1;
+    assert((c1 == c2));
+    assert(!(c1 != c2));
+  }
+  {
+    typedef std::unordered_multiset<int> C;
+    typedef int P;
+    P a[] = {P(10), P(20), P(20), P(30), P(40), P(50), P(50), P(50), P(60), P(70), P(80)};
+    C c1(std::begin(a), std::end(a));
+    C c2 = c1;
+    c2.rehash(30);
+    assert((c1 == c2));
+    assert(!(c1 != c2));
+    c2.insert(P(90));
+    assert(!(c1 == c2));
+    assert((c1 != c2));
+    c1.insert(P(90));
+    assert((c1 == c2));
+    assert(!(c1 != c2));
+  }
 #if TEST_STD_VER >= 11
-    {
-        typedef std::unordered_multiset<int, std::hash<int>,
-                                      std::equal_to<int>, min_allocator<int>> C;
-        typedef int P;
-        P a[] =
-        {
-            P(10),
-            P(20),
-            P(20),
-            P(30),
-            P(40),
-            P(50),
-            P(50),
-            P(50),
-            P(60),
-            P(70),
-            P(80)
-        };
-        const C c1(std::begin(a), std::end(a));
-        const C c2;
-        assert(!(c1 == c2));
-        assert( (c1 != c2));
-    }
-    {
-        typedef std::unordered_multiset<int, std::hash<int>,
-                                      std::equal_to<int>, min_allocator<int>> C;
-        typedef int P;
-        P a[] =
-        {
-            P(10),
-            P(20),
-            P(20),
-            P(30),
-            P(40),
-            P(50),
-            P(50),
-            P(50),
-            P(60),
-            P(70),
-            P(80)
-        };
-        const C c1(std::begin(a), std::end(a));
-        const C c2 = c1;
-        assert( (c1 == c2));
-        assert(!(c1 != c2));
-    }
-    {
-        typedef std::unordered_multiset<int, std::hash<int>,
-                                      std::equal_to<int>, min_allocator<int>> C;
-        typedef int P;
-        P a[] =
-        {
-            P(10),
-            P(20),
-            P(20),
-            P(30),
-            P(40),
-            P(50),
-            P(50),
-            P(50),
-            P(60),
-            P(70),
-            P(80)
-        };
-        C c1(std::begin(a), std::end(a));
-        C c2 = c1;
-        c2.rehash(30);
-        assert( (c1 == c2));
-        assert(!(c1 != c2));
-        c2.insert(P(90));
-        assert(!(c1 == c2));
-        assert( (c1 != c2));
-        c1.insert(P(90));
-        assert( (c1 == c2));
-        assert(!(c1 != c2));
-    }
+  {
+    typedef std::unordered_multiset<int, std::hash<int>, std::equal_to<int>, min_allocator<int>> C;
+    typedef int P;
+    P a[] = {P(10), P(20), P(20), P(30), P(40), P(50), P(50), P(50), P(60), P(70), P(80)};
+    const C c1(std::begin(a), std::end(a));
+    const C c2;
+    assert(!(c1 == c2));
+    assert((c1 != c2));
+  }
+  {
+    typedef std::unordered_multiset<int, std::hash<int>, std::equal_to<int>, min_allocator<int>> C;
+    typedef int P;
+    P a[] = {P(10), P(20), P(20), P(30), P(40), P(50), P(50), P(50), P(60), P(70), P(80)};
+    const C c1(std::begin(a), std::end(a));
+    const C c2 = c1;
+    assert((c1 == c2));
+    assert(!(c1 != c2));
+  }
+  {
+    typedef std::unordered_multiset<int, std::hash<int>, std::equal_to<int>, min_allocator<int>> C;
+    typedef int P;
+    P a[] = {P(10), P(20), P(20), P(30), P(40), P(50), P(50), P(50), P(60), P(70), P(80)};
+    C c1(std::begin(a), std::end(a));
+    C c2 = c1;
+    c2.rehash(30);
+    assert((c1 == c2));
+    assert(!(c1 != c2));
+    c2.insert(P(90));
+    assert(!(c1 == c2));
+    assert((c1 != c2));
+    c1.insert(P(90));
+    assert((c1 == c2));
+    assert(!(c1 != c2));
+  }
 #endif
 
-    // Make sure we take into account the number of times that a key repeats into equality.
-    {
-        int a[] = {1, 1, 1, 2};
-        int b[] = {1, 1, 1, 1, 2};
+  // Make sure we take into account the number of times that a key repeats into equality.
+  {
+    int a[] = {1, 1, 1, 2};
+    int b[] = {1, 1, 1, 1, 2};
 
-        std::unordered_multiset<int> c1(std::begin(a), std::end(a));
-        std::unordered_multiset<int> c2(std::begin(b), std::end(b));
-        assert(testEquality(c1, c2, false));
-    }
+    std::unordered_multiset<int> c1(std::begin(a), std::end(a));
+    std::unordered_multiset<int> c2(std::begin(b), std::end(b));
+    assert(testEquality(c1, c2, false));
+  }
 
-    // Make sure we behave properly when a custom key predicate is provided.
-    {
-        int a[] = {1, 3};
-        int b[] = {1, 1};
-        // A very poor hash
-        struct HashModuloOddness {
-            std::size_t operator()(int x) const { return std::hash<int>()(x % 2); }
-        };
-        // A very poor hash
-        struct CompareModuloOddness {
-            bool operator()(int x, int y) const { return (x % 2) == (y % 2); }
-        };
+  // Make sure we behave properly when a custom key predicate is provided.
+  {
+    int a[] = {1, 3};
+    int b[] = {1, 1};
+    // A very poor hash
+    struct HashModuloOddness {
+      std::size_t operator()(int x) const { return std::hash<int>()(x % 2); }
+    };
+    // A very poor hash
+    struct CompareModuloOddness {
+      bool operator()(int x, int y) const { return (x % 2) == (y % 2); }
+    };
 
-        using Set = std::unordered_multiset<int, HashModuloOddness, CompareModuloOddness>;
-        Set c1(std::begin(a), std::end(a));
-        Set c2(std::begin(b), std::end(b));
+    using Set = std::unordered_multiset<int, HashModuloOddness, CompareModuloOddness>;
+    Set c1(std::begin(a), std::end(a));
+    Set c2(std::begin(b), std::end(b));
 
-        assert(testEquality(c1, c2, false));
-    }
+    assert(testEquality(c1, c2, false));
+  }
 
-    return 0;
+  return 0;
 }
diff --git a/libcxx/test/std/containers/unord/unord.multiset/equal_range.transparent.pass.cpp b/libcxx/test/std/containers/unord/unord.multiset/equal_range.transparent.pass.cpp
index 741058d8d2514..4d3a1baa47d23 100644
--- a/libcxx/test/std/containers/unord/unord.multiset/equal_range.transparent.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.multiset/equal_range.transparent.pass.cpp
@@ -21,44 +21,43 @@
 
 #include "test_transparent_unordered.h"
 
-int main(int, char**)
-{
-    using key_type = StoredType<int>;
-
-    {
-      // Make sure conversions don't happen for transparent non-final hasher and key_equal
-      using S = unord_set_type<std::unordered_multiset, transparent_hash, std::equal_to<>>;
-      test_transparent_equal_range<S>({1, 1, 2});
-      test_transparent_equal_range<const S>({1, 1, 2});
-    }
-
-    {
-      // Make sure conversions don't happen for transparent final hasher and key_equal
-      using S = unord_set_type<std::unordered_multiset, transparent_hash_final, transparent_equal_final>;
-      test_transparent_equal_range<S>({1, 1, 2});
-      test_transparent_equal_range<const S>({1, 1, 2});
-    }
-
-    {
-      // Make sure conversions do happen for non-transparent hasher
-      using S = unord_set_type<std::unordered_multiset, non_transparent_hash, std::equal_to<>>;
-      test_non_transparent_equal_range<S>({1, 1, 2});
-      test_non_transparent_equal_range<const S>({1, 1, 2});
-    }
-
-    {
-      // Make sure conversions do happen for non-transparent key_equal
-      using S = unord_set_type<std::unordered_multiset, transparent_hash, std::equal_to<key_type>>;
-      test_non_transparent_equal_range<S>({1, 1, 2});
-      test_non_transparent_equal_range<const S>({1, 1, 2});
-    }
-
-    {
-      // Make sure conversions do happen for both non-transparent hasher and key_equal
-      using S = unord_set_type<std::unordered_multiset, non_transparent_hash, std::equal_to<key_type>>;
-      test_non_transparent_equal_range<S>({1, 1, 2});
-      test_non_transparent_equal_range<const S>({1, 1, 2});
-    }
-
-    return 0;
+int main(int, char**) {
+  using key_type = StoredType<int>;
+
+  {
+    // Make sure conversions don't happen for transparent non-final hasher and key_equal
+    using S = unord_set_type<std::unordered_multiset, transparent_hash, std::equal_to<>>;
+    test_transparent_equal_range<S>({1, 1, 2});
+    test_transparent_equal_range<const S>({1, 1, 2});
+  }
+
+  {
+    // Make sure conversions don't happen for transparent final hasher and key_equal
+    using S = unord_set_type<std::unordered_multiset, transparent_hash_final, transparent_equal_final>;
+    test_transparent_equal_range<S>({1, 1, 2});
+    test_transparent_equal_range<const S>({1, 1, 2});
+  }
+
+  {
+    // Make sure conversions do happen for non-transparent hasher
+    using S = unord_set_type<std::unordered_multiset, non_transparent_hash, std::equal_to<>>;
+    test_non_transparent_equal_range<S>({1, 1, 2});
+    test_non_transparent_equal_range<const S>({1, 1, 2});
+  }
+
+  {
+    // Make sure conversions do happen for non-transparent key_equal
+    using S = unord_set_type<std::unordered_multiset, transparent_hash, std::equal_to<key_type>>;
+    test_non_transparent_equal_range<S>({1, 1, 2});
+    test_non_transparent_equal_range<const S>({1, 1, 2});
+  }
+
+  {
+    // Make sure conversions do happen for both non-transparent hasher and key_equal
+    using S = unord_set_type<std::unordered_multiset, non_transparent_hash, std::equal_to<key_type>>;
+    test_non_transparent_equal_range<S>({1, 1, 2});
+    test_non_transparent_equal_range<const S>({1, 1, 2});
+  }
+
+  return 0;
 }
diff --git a/libcxx/test/std/containers/unord/unord.multiset/equal_range_const.pass.cpp b/libcxx/test/std/containers/unord/unord.multiset/equal_range_const.pass.cpp
index 6ef53875fcbcb..f5ec5165c885a 100644
--- a/libcxx/test/std/containers/unord/unord.multiset/equal_range_const.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.multiset/equal_range_const.pass.cpp
@@ -20,72 +20,46 @@
 #include "test_macros.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef std::unordered_multiset<int> C;
-        typedef C::const_iterator I;
-        typedef int P;
-        P a[] =
-        {
-            P(10),
-            P(20),
-            P(30),
-            P(40),
-            P(50),
-            P(50),
-            P(50),
-            P(60),
-            P(70),
-            P(80)
-        };
-        const C c(std::begin(a), std::end(a));
-        std::pair<I, I> r = c.equal_range(30);
-        assert(std::distance(r.first, r.second) == 1);
-        assert(*r.first == 30);
-        r = c.equal_range(5);
-        assert(std::distance(r.first, r.second) == 0);
-        r = c.equal_range(50);
-        assert(std::distance(r.first, r.second) == 3);
-        assert(*r.first == 50);
-        ++r.first;
-        assert(*r.first == 50);
-        ++r.first;
-        assert(*r.first == 50);
-    }
+int main(int, char**) {
+  {
+    typedef std::unordered_multiset<int> C;
+    typedef C::const_iterator I;
+    typedef int P;
+    P a[] = {P(10), P(20), P(30), P(40), P(50), P(50), P(50), P(60), P(70), P(80)};
+    const C c(std::begin(a), std::end(a));
+    std::pair<I, I> r = c.equal_range(30);
+    assert(std::distance(r.first, r.second) == 1);
+    assert(*r.first == 30);
+    r = c.equal_range(5);
+    assert(std::distance(r.first, r.second) == 0);
+    r = c.equal_range(50);
+    assert(std::distance(r.first, r.second) == 3);
+    assert(*r.first == 50);
+    ++r.first;
+    assert(*r.first == 50);
+    ++r.first;
+    assert(*r.first == 50);
+  }
 #if TEST_STD_VER >= 11
-    {
-        typedef std::unordered_multiset<int, std::hash<int>,
-                                      std::equal_to<int>, min_allocator<int>> C;
-        typedef C::const_iterator I;
-        typedef int P;
-        P a[] =
-        {
-            P(10),
-            P(20),
-            P(30),
-            P(40),
-            P(50),
-            P(50),
-            P(50),
-            P(60),
-            P(70),
-            P(80)
-        };
-        const C c(std::begin(a), std::end(a));
-        std::pair<I, I> r = c.equal_range(30);
-        assert(std::distance(r.first, r.second) == 1);
-        assert(*r.first == 30);
-        r = c.equal_range(5);
-        assert(std::distance(r.first, r.second) == 0);
-        r = c.equal_range(50);
-        assert(std::distance(r.first, r.second) == 3);
-        assert(*r.first == 50);
-        ++r.first;
-        assert(*r.first == 50);
-        ++r.first;
-        assert(*r.first == 50);
-    }
+  {
+    typedef std::unordered_multiset<int, std::hash<int>, std::equal_to<int>, min_allocator<int>> C;
+    typedef C::const_iterator I;
+    typedef int P;
+    P a[] = {P(10), P(20), P(30), P(40), P(50), P(50), P(50), P(60), P(70), P(80)};
+    const C c(std::begin(a), std::end(a));
+    std::pair<I, I> r = c.equal_range(30);
+    assert(std::distance(r.first, r.second) == 1);
+    assert(*r.first == 30);
+    r = c.equal_range(5);
+    assert(std::distance(r.first, r.second) == 0);
+    r = c.equal_range(50);
+    assert(std::distance(r.first, r.second) == 3);
+    assert(*r.first == 50);
+    ++r.first;
+    assert(*r.first == 50);
+    ++r.first;
+    assert(*r.first == 50);
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/unord/unord.multiset/equal_range_non_const.pass.cpp b/libcxx/test/std/containers/unord/unord.multiset/equal_range_non_const.pass.cpp
index 3c70d8c89cf9b..8d3d11cabfd1e 100644
--- a/libcxx/test/std/containers/unord/unord.multiset/equal_range_non_const.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.multiset/equal_range_non_const.pass.cpp
@@ -20,72 +20,46 @@
 #include "test_macros.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef std::unordered_multiset<int> C;
-        typedef C::iterator I;
-        typedef int P;
-        P a[] =
-        {
-            P(10),
-            P(20),
-            P(30),
-            P(40),
-            P(50),
-            P(50),
-            P(50),
-            P(60),
-            P(70),
-            P(80)
-        };
-        C c(std::begin(a), std::end(a));
-        std::pair<I, I> r = c.equal_range(30);
-        assert(std::distance(r.first, r.second) == 1);
-        assert(*r.first == 30);
-        r = c.equal_range(5);
-        assert(std::distance(r.first, r.second) == 0);
-        r = c.equal_range(50);
-        assert(std::distance(r.first, r.second) == 3);
-        assert(*r.first == 50);
-        ++r.first;
-        assert(*r.first == 50);
-        ++r.first;
-        assert(*r.first == 50);
-    }
+int main(int, char**) {
+  {
+    typedef std::unordered_multiset<int> C;
+    typedef C::iterator I;
+    typedef int P;
+    P a[] = {P(10), P(20), P(30), P(40), P(50), P(50), P(50), P(60), P(70), P(80)};
+    C c(std::begin(a), std::end(a));
+    std::pair<I, I> r = c.equal_range(30);
+    assert(std::distance(r.first, r.second) == 1);
+    assert(*r.first == 30);
+    r = c.equal_range(5);
+    assert(std::distance(r.first, r.second) == 0);
+    r = c.equal_range(50);
+    assert(std::distance(r.first, r.second) == 3);
+    assert(*r.first == 50);
+    ++r.first;
+    assert(*r.first == 50);
+    ++r.first;
+    assert(*r.first == 50);
+  }
 #if TEST_STD_VER >= 11
-    {
-        typedef std::unordered_multiset<int, std::hash<int>,
-                                      std::equal_to<int>, min_allocator<int>> C;
-        typedef C::iterator I;
-        typedef int P;
-        P a[] =
-        {
-            P(10),
-            P(20),
-            P(30),
-            P(40),
-            P(50),
-            P(50),
-            P(50),
-            P(60),
-            P(70),
-            P(80)
-        };
-        C c(std::begin(a), std::end(a));
-        std::pair<I, I> r = c.equal_range(30);
-        assert(std::distance(r.first, r.second) == 1);
-        assert(*r.first == 30);
-        r = c.equal_range(5);
-        assert(std::distance(r.first, r.second) == 0);
-        r = c.equal_range(50);
-        assert(std::distance(r.first, r.second) == 3);
-        assert(*r.first == 50);
-        ++r.first;
-        assert(*r.first == 50);
-        ++r.first;
-        assert(*r.first == 50);
-    }
+  {
+    typedef std::unordered_multiset<int, std::hash<int>, std::equal_to<int>, min_allocator<int>> C;
+    typedef C::iterator I;
+    typedef int P;
+    P a[] = {P(10), P(20), P(30), P(40), P(50), P(50), P(50), P(60), P(70), P(80)};
+    C c(std::begin(a), std::end(a));
+    std::pair<I, I> r = c.equal_range(30);
+    assert(std::distance(r.first, r.second) == 1);
+    assert(*r.first == 30);
+    r = c.equal_range(5);
+    assert(std::distance(r.first, r.second) == 0);
+    r = c.equal_range(50);
+    assert(std::distance(r.first, r.second) == 3);
+    assert(*r.first == 50);
+    ++r.first;
+    assert(*r.first == 50);
+    ++r.first;
+    assert(*r.first == 50);
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/unord/unord.multiset/erase_const_iter.pass.cpp b/libcxx/test/std/containers/unord/unord.multiset/erase_const_iter.pass.cpp
index ffdcde80d14d3..d30a3d4b202ce 100644
--- a/libcxx/test/std/containers/unord/unord.multiset/erase_const_iter.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.multiset/erase_const_iter.pass.cpp
@@ -20,82 +20,65 @@
 #include "test_macros.h"
 #include "min_allocator.h"
 
-struct TemplateConstructor
-{
-    template<typename T>
-    TemplateConstructor (const T&) {}
+struct TemplateConstructor {
+  template <typename T>
+  TemplateConstructor(const T&) {}
 };
 
 bool operator==(const TemplateConstructor&, const TemplateConstructor&) { return false; }
-struct Hash { std::size_t operator() (const TemplateConstructor &) const { return 0; } };
+struct Hash {
+  std::size_t operator()(const TemplateConstructor&) const { return 0; }
+};
 
-int main(int, char**)
-{
-    {
-        typedef std::unordered_multiset<int> C;
-        typedef int P;
-        P a[] =
-        {
-            P(1),
-            P(2),
-            P(3),
-            P(4),
-            P(1),
-            P(2)
-        };
-        C c(a, a + sizeof(a)/sizeof(a[0]));
-        C::const_iterator i = c.find(2);
-        C::const_iterator i_next = i;
-        ++i_next;
-        C::iterator j = c.erase(i);
-        assert(j == i_next);
+int main(int, char**) {
+  {
+    typedef std::unordered_multiset<int> C;
+    typedef int P;
+    P a[] = {P(1), P(2), P(3), P(4), P(1), P(2)};
+    C c(a, a + sizeof(a) / sizeof(a[0]));
+    C::const_iterator i      = c.find(2);
+    C::const_iterator i_next = i;
+    ++i_next;
+    C::iterator j = c.erase(i);
+    assert(j == i_next);
 
-        assert(c.size() == 5);
-        assert(c.count(1) == 2);
-        assert(c.count(2) == 1);
-        assert(c.count(3) == 1);
-        assert(c.count(4) == 1);
-    }
+    assert(c.size() == 5);
+    assert(c.count(1) == 2);
+    assert(c.count(2) == 1);
+    assert(c.count(3) == 1);
+    assert(c.count(4) == 1);
+  }
 #if TEST_STD_VER >= 11
-    {
-        typedef std::unordered_multiset<int, std::hash<int>,
-                                      std::equal_to<int>, min_allocator<int>> C;
-        typedef int P;
-        P a[] =
-        {
-            P(1),
-            P(2),
-            P(3),
-            P(4),
-            P(1),
-            P(2)
-        };
-        C c(a, a + sizeof(a)/sizeof(a[0]));
-        C::const_iterator i = c.find(2);
-        C::const_iterator i_next = i;
-        ++i_next;
-        C::iterator j = c.erase(i);
-        assert(j == i_next);
-        assert(c.size() == 5);
-        assert(c.count(1) == 2);
-        assert(c.count(2) == 1);
-        assert(c.count(3) == 1);
-        assert(c.count(4) == 1);
-    }
+  {
+    typedef std::unordered_multiset<int, std::hash<int>, std::equal_to<int>, min_allocator<int>> C;
+    typedef int P;
+    P a[] = {P(1), P(2), P(3), P(4), P(1), P(2)};
+    C c(a, a + sizeof(a) / sizeof(a[0]));
+    C::const_iterator i      = c.find(2);
+    C::const_iterator i_next = i;
+    ++i_next;
+    C::iterator j = c.erase(i);
+    assert(j == i_next);
+    assert(c.size() == 5);
+    assert(c.count(1) == 2);
+    assert(c.count(2) == 1);
+    assert(c.count(3) == 1);
+    assert(c.count(4) == 1);
+  }
 #endif
 #if TEST_STD_VER >= 14
-    {
+  {
     //  This is LWG #2059
-        typedef TemplateConstructor T;
-        typedef std::unordered_set<T, Hash> C;
-        typedef C::iterator I;
+    typedef TemplateConstructor T;
+    typedef std::unordered_set<T, Hash> C;
+    typedef C::iterator I;
 
-        C m;
-        T a{0};
-        I it = m.find(a);
-        if (it != m.end())
-            m.erase(it);
-    }
+    C m;
+    T a{0};
+    I it = m.find(a);
+    if (it != m.end())
+      m.erase(it);
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/unord/unord.multiset/erase_if.pass.cpp b/libcxx/test/std/containers/unord/unord.multiset/erase_if.pass.cpp
index 7bdc4f871571f..23980d7e1bac5 100644
--- a/libcxx/test/std/containers/unord/unord.multiset/erase_if.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.multiset/erase_if.pass.cpp
@@ -23,12 +23,11 @@
 using Init = std::initializer_list<int>;
 
 template <typename M>
-M make (Init vals)
-{
-    M ret;
-    for (int v : vals)
-        ret.insert(v);
-    return ret;
+M make(Init vals) {
+  M ret;
+  for (int v : vals)
+    ret.insert(v);
+  return ret;
 }
 
 template <typename M, typename Pred>
@@ -41,52 +40,50 @@ void test0(Init vals, Pred p, Init expected, std::size_t expected_erased_count)
 }
 
 template <typename S>
-void test()
-{
-    auto is1 = [](auto v) { return v == 1;};
-    auto is2 = [](auto v) { return v == 2;};
-    auto is3 = [](auto v) { return v == 3;};
-    auto is4 = [](auto v) { return v == 4;};
-    auto True  = [](auto) { return true; };
-    auto False = [](auto) { return false; };
-
-    test0<S>({}, is1, {}, 0);
-
-    test0<S>({1}, is1, {}, 1);
-    test0<S>({1}, is2, {1}, 0);
-
-    test0<S>({1, 2}, is1, {2}, 1);
-    test0<S>({1, 2}, is2, {1}, 1);
-    test0<S>({1, 2}, is3, {1, 2}, 0);
-    test0<S>({1, 1}, is1, {}, 2);
-    test0<S>({1, 1}, is3, {1, 1}, 0);
-
-    test0<S>({1, 2, 3}, is1, {2, 3}, 1);
-    test0<S>({1, 2, 3}, is2, {1, 3}, 1);
-    test0<S>({1, 2, 3}, is3, {1, 2}, 1);
-    test0<S>({1, 2, 3}, is4, {1, 2, 3}, 0);
-
-    test0<S>({1, 1, 1}, is1, {}, 3);
-    test0<S>({1, 1, 1}, is2, {1, 1, 1}, 0);
-    test0<S>({1, 1, 2}, is1, {2}, 2);
-    test0<S>({1, 1, 2}, is2, {1, 1}, 1);
-    test0<S>({1, 1, 2}, is3, {1, 1, 2}, 0);
-    test0<S>({1, 2, 2}, is1, {2, 2}, 1);
-    test0<S>({1, 2, 2}, is2, {1}, 2);
-    test0<S>({1, 2, 2}, is3, {1, 2, 2}, 0);
-
-    test0<S>({1, 2, 3}, True, {}, 3);
-    test0<S>({1, 2, 3}, False, {1, 2, 3}, 0);
+void test() {
+  auto is1   = [](auto v) { return v == 1; };
+  auto is2   = [](auto v) { return v == 2; };
+  auto is3   = [](auto v) { return v == 3; };
+  auto is4   = [](auto v) { return v == 4; };
+  auto True  = [](auto) { return true; };
+  auto False = [](auto) { return false; };
+
+  test0<S>({}, is1, {}, 0);
+
+  test0<S>({1}, is1, {}, 1);
+  test0<S>({1}, is2, {1}, 0);
+
+  test0<S>({1, 2}, is1, {2}, 1);
+  test0<S>({1, 2}, is2, {1}, 1);
+  test0<S>({1, 2}, is3, {1, 2}, 0);
+  test0<S>({1, 1}, is1, {}, 2);
+  test0<S>({1, 1}, is3, {1, 1}, 0);
+
+  test0<S>({1, 2, 3}, is1, {2, 3}, 1);
+  test0<S>({1, 2, 3}, is2, {1, 3}, 1);
+  test0<S>({1, 2, 3}, is3, {1, 2}, 1);
+  test0<S>({1, 2, 3}, is4, {1, 2, 3}, 0);
+
+  test0<S>({1, 1, 1}, is1, {}, 3);
+  test0<S>({1, 1, 1}, is2, {1, 1, 1}, 0);
+  test0<S>({1, 1, 2}, is1, {2}, 2);
+  test0<S>({1, 1, 2}, is2, {1, 1}, 1);
+  test0<S>({1, 1, 2}, is3, {1, 1, 2}, 0);
+  test0<S>({1, 2, 2}, is1, {2, 2}, 1);
+  test0<S>({1, 2, 2}, is2, {1}, 2);
+  test0<S>({1, 2, 2}, is3, {1, 2, 2}, 0);
+
+  test0<S>({1, 2, 3}, True, {}, 3);
+  test0<S>({1, 2, 3}, False, {1, 2, 3}, 0);
 }
 
-int main(int, char**)
-{
-    test<std::unordered_multiset<int>>();
-    test<std::unordered_multiset<int, std::hash<int>, std::equal_to<int>, min_allocator<int>>> ();
-    test<std::unordered_multiset<int, std::hash<int>, std::equal_to<int>, test_allocator<int>>> ();
+int main(int, char**) {
+  test<std::unordered_multiset<int>>();
+  test<std::unordered_multiset<int, std::hash<int>, std::equal_to<int>, min_allocator<int>>>();
+  test<std::unordered_multiset<int, std::hash<int>, std::equal_to<int>, test_allocator<int>>>();
 
-    test<std::unordered_multiset<long>>();
-    test<std::unordered_multiset<double>>();
+  test<std::unordered_multiset<long>>();
+  test<std::unordered_multiset<double>>();
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/unord/unord.multiset/erase_key.pass.cpp b/libcxx/test/std/containers/unord/unord.multiset/erase_key.pass.cpp
index 522b6de2b87b1..93c3b23aaff3c 100644
--- a/libcxx/test/std/containers/unord/unord.multiset/erase_key.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.multiset/erase_key.pass.cpp
@@ -23,155 +23,139 @@
 
 #if TEST_STD_VER >= 11
 template <typename Unordered>
-bool only_deletions ( const Unordered &whole, const Unordered &part ) {
-    typename Unordered::const_iterator w = whole.begin();
-    typename Unordered::const_iterator p = part.begin();
+bool only_deletions(const Unordered& whole, const Unordered& part) {
+  typename Unordered::const_iterator w = whole.begin();
+  typename Unordered::const_iterator p = part.begin();
 
-    while ( w != whole.end () && p != part.end()) {
-        if ( *w == *p )
-            p++;
-        w++;
-        }
+  while (w != whole.end() && p != part.end()) {
+    if (*w == *p)
+      p++;
+    w++;
+  }
 
-    return p == part.end();
+  return p == part.end();
 }
 #endif
 
-int main(int, char**)
-{
-    {
-        typedef std::unordered_multiset<int> C;
-        typedef int P;
-        P a[] =
-        {
-            P(1),
-            P(2),
-            P(3),
-            P(4),
-            P(1),
-            P(2)
-        };
-        C c(a, a + sizeof(a)/sizeof(a[0]));
-        assert(c.erase(5) == 0);
-        assert(c.size() == 6);
-        assert(c.count(1) == 2);
-        assert(c.count(2) == 2);
-        assert(c.count(3) == 1);
-        assert(c.count(4) == 1);
-
-        assert(c.erase(2) == 2);
-        assert(c.size() == 4);
-        assert(c.count(1) == 2);
-        assert(c.count(3) == 1);
-        assert(c.count(4) == 1);
-
-        assert(c.erase(2) == 0);
-        assert(c.size() == 4);
-        assert(c.count(1) == 2);
-        assert(c.count(3) == 1);
-        assert(c.count(4) == 1);
-
-        assert(c.erase(4) == 1);
-        assert(c.size() == 3);
-        assert(c.count(1) == 2);
-        assert(c.count(3) == 1);
-
-        assert(c.erase(4) == 0);
-        assert(c.size() == 3);
-        assert(c.count(1) == 2);
-        assert(c.count(3) == 1);
-
-        assert(c.erase(1) == 2);
-        assert(c.size() == 1);
-        assert(c.count(3) == 1);
-
-        assert(c.erase(1) == 0);
-        assert(c.size() == 1);
-        assert(c.count(3) == 1);
-
-        assert(c.erase(3) == 1);
-        assert(c.size() == 0);
-
-        assert(c.erase(3) == 0);
-        assert(c.size() == 0);
-    }
+int main(int, char**) {
+  {
+    typedef std::unordered_multiset<int> C;
+    typedef int P;
+    P a[] = {P(1), P(2), P(3), P(4), P(1), P(2)};
+    C c(a, a + sizeof(a) / sizeof(a[0]));
+    assert(c.erase(5) == 0);
+    assert(c.size() == 6);
+    assert(c.count(1) == 2);
+    assert(c.count(2) == 2);
+    assert(c.count(3) == 1);
+    assert(c.count(4) == 1);
+
+    assert(c.erase(2) == 2);
+    assert(c.size() == 4);
+    assert(c.count(1) == 2);
+    assert(c.count(3) == 1);
+    assert(c.count(4) == 1);
+
+    assert(c.erase(2) == 0);
+    assert(c.size() == 4);
+    assert(c.count(1) == 2);
+    assert(c.count(3) == 1);
+    assert(c.count(4) == 1);
+
+    assert(c.erase(4) == 1);
+    assert(c.size() == 3);
+    assert(c.count(1) == 2);
+    assert(c.count(3) == 1);
+
+    assert(c.erase(4) == 0);
+    assert(c.size() == 3);
+    assert(c.count(1) == 2);
+    assert(c.count(3) == 1);
+
+    assert(c.erase(1) == 2);
+    assert(c.size() == 1);
+    assert(c.count(3) == 1);
+
+    assert(c.erase(1) == 0);
+    assert(c.size() == 1);
+    assert(c.count(3) == 1);
+
+    assert(c.erase(3) == 1);
+    assert(c.size() == 0);
+
+    assert(c.erase(3) == 0);
+    assert(c.size() == 0);
+  }
 #if TEST_STD_VER >= 11
-    {
-        typedef std::unordered_multiset<int, std::hash<int>,
-                                      std::equal_to<int>, min_allocator<int>> C;
-        typedef int P;
-        P a[] =
-        {
-            P(1),
-            P(2),
-            P(3),
-            P(4),
-            P(1),
-            P(2)
-        };
-        C c(a, a + sizeof(a)/sizeof(a[0]));
-        assert(c.erase(5) == 0);
-        assert(c.size() == 6);
-        assert(c.count(1) == 2);
-        assert(c.count(2) == 2);
-        assert(c.count(3) == 1);
-        assert(c.count(4) == 1);
-
-        assert(c.erase(2) == 2);
-        assert(c.size() == 4);
-        assert(c.count(1) == 2);
-        assert(c.count(3) == 1);
-        assert(c.count(4) == 1);
-
-        assert(c.erase(2) == 0);
-        assert(c.size() == 4);
-        assert(c.count(1) == 2);
-        assert(c.count(3) == 1);
-        assert(c.count(4) == 1);
-
-        assert(c.erase(4) == 1);
-        assert(c.size() == 3);
-        assert(c.count(1) == 2);
-        assert(c.count(3) == 1);
-
-        assert(c.erase(4) == 0);
-        assert(c.size() == 3);
-        assert(c.count(1) == 2);
-        assert(c.count(3) == 1);
-
-        assert(c.erase(1) == 2);
-        assert(c.size() == 1);
-        assert(c.count(3) == 1);
-
-        assert(c.erase(1) == 0);
-        assert(c.size() == 1);
-        assert(c.count(3) == 1);
-
-        assert(c.erase(3) == 1);
-        assert(c.size() == 0);
-
-        assert(c.erase(3) == 0);
-        assert(c.size() == 0);
-    }
-    {
+  {
+    typedef std::unordered_multiset<int, std::hash<int>, std::equal_to<int>, min_allocator<int>> C;
+    typedef int P;
+    P a[] = {P(1), P(2), P(3), P(4), P(1), P(2)};
+    C c(a, a + sizeof(a) / sizeof(a[0]));
+    assert(c.erase(5) == 0);
+    assert(c.size() == 6);
+    assert(c.count(1) == 2);
+    assert(c.count(2) == 2);
+    assert(c.count(3) == 1);
+    assert(c.count(4) == 1);
+
+    assert(c.erase(2) == 2);
+    assert(c.size() == 4);
+    assert(c.count(1) == 2);
+    assert(c.count(3) == 1);
+    assert(c.count(4) == 1);
+
+    assert(c.erase(2) == 0);
+    assert(c.size() == 4);
+    assert(c.count(1) == 2);
+    assert(c.count(3) == 1);
+    assert(c.count(4) == 1);
+
+    assert(c.erase(4) == 1);
+    assert(c.size() == 3);
+    assert(c.count(1) == 2);
+    assert(c.count(3) == 1);
+
+    assert(c.erase(4) == 0);
+    assert(c.size() == 3);
+    assert(c.count(1) == 2);
+    assert(c.count(3) == 1);
+
+    assert(c.erase(1) == 2);
+    assert(c.size() == 1);
+    assert(c.count(3) == 1);
+
+    assert(c.erase(1) == 0);
+    assert(c.size() == 1);
+    assert(c.count(3) == 1);
+
+    assert(c.erase(3) == 1);
+    assert(c.size() == 0);
+
+    assert(c.erase(3) == 0);
+    assert(c.size() == 0);
+  }
+  {
     typedef std::unordered_multiset<int> C;
     C m, m2;
-    for ( int i = 0; i < 10; ++i ) {
-        m.insert(i);  m.insert(i);
-        m2.insert(i); m2.insert(i);
-        }
+    for (int i = 0; i < 10; ++i) {
+      m.insert(i);
+      m.insert(i);
+      m2.insert(i);
+      m2.insert(i);
+    }
 
     C::iterator i = m2.begin();
-    int ctr = 0;
+    int ctr       = 0;
     while (i != m2.end()) {
-        if (ctr++ % 2 == 0)
-            m2.erase(i++);
-        else
-            ++i;
-        }
-
-    assert (only_deletions (m, m2));
+      if (ctr++ % 2 == 0)
+        m2.erase(i++);
+      else
+        ++i;
     }
+
+    assert(only_deletions(m, m2));
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/unord/unord.multiset/erase_range.pass.cpp b/libcxx/test/std/containers/unord/unord.multiset/erase_range.pass.cpp
index 954d3bc1f9ebc..62724fdbeead8 100644
--- a/libcxx/test/std/containers/unord/unord.multiset/erase_range.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.multiset/erase_range.pass.cpp
@@ -21,76 +21,58 @@
 #include "test_macros.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef std::unordered_multiset<int> C;
-        typedef int P;
-        P a[] =
-        {
-            P(1),
-            P(2),
-            P(3),
-            P(4),
-            P(1),
-            P(2)
-        };
-        C c(a, a + sizeof(a)/sizeof(a[0]));
-        C::const_iterator i = c.find(2);
-        C::const_iterator j = std::next(i, 2);
-        C::iterator k = c.erase(i, i);
-        assert(k == i);
-        assert(c.size() == 6);
-        assert(c.count(1) == 2);
-        assert(c.count(2) == 2);
-        assert(c.count(3) == 1);
-        assert(c.count(4) == 1);
+int main(int, char**) {
+  {
+    typedef std::unordered_multiset<int> C;
+    typedef int P;
+    P a[] = {P(1), P(2), P(3), P(4), P(1), P(2)};
+    C c(a, a + sizeof(a) / sizeof(a[0]));
+    C::const_iterator i = c.find(2);
+    C::const_iterator j = std::next(i, 2);
+    C::iterator k       = c.erase(i, i);
+    assert(k == i);
+    assert(c.size() == 6);
+    assert(c.count(1) == 2);
+    assert(c.count(2) == 2);
+    assert(c.count(3) == 1);
+    assert(c.count(4) == 1);
 
-        k = c.erase(i, j);
-        assert(c.size() == 4);
-        assert(c.count(1) == 2);
-        assert(c.count(3) == 1);
-        assert(c.count(4) == 1);
+    k = c.erase(i, j);
+    assert(c.size() == 4);
+    assert(c.count(1) == 2);
+    assert(c.count(3) == 1);
+    assert(c.count(4) == 1);
 
-        k = c.erase(c.cbegin(), c.cend());
-        assert(c.size() == 0);
-        assert(k == c.end());
-    }
+    k = c.erase(c.cbegin(), c.cend());
+    assert(c.size() == 0);
+    assert(k == c.end());
+  }
 #if TEST_STD_VER >= 11
-    {
-        typedef std::unordered_multiset<int, std::hash<int>,
-                                      std::equal_to<int>, min_allocator<int>> C;
-        typedef int P;
-        P a[] =
-        {
-            P(1),
-            P(2),
-            P(3),
-            P(4),
-            P(1),
-            P(2)
-        };
-        C c(a, a + sizeof(a)/sizeof(a[0]));
-        C::const_iterator i = c.find(2);
-        C::const_iterator j = std::next(i, 2);
-        C::iterator k = c.erase(i, i);
-        assert(k == i);
-        assert(c.size() == 6);
-        assert(c.count(1) == 2);
-        assert(c.count(2) == 2);
-        assert(c.count(3) == 1);
-        assert(c.count(4) == 1);
+  {
+    typedef std::unordered_multiset<int, std::hash<int>, std::equal_to<int>, min_allocator<int>> C;
+    typedef int P;
+    P a[] = {P(1), P(2), P(3), P(4), P(1), P(2)};
+    C c(a, a + sizeof(a) / sizeof(a[0]));
+    C::const_iterator i = c.find(2);
+    C::const_iterator j = std::next(i, 2);
+    C::iterator k       = c.erase(i, i);
+    assert(k == i);
+    assert(c.size() == 6);
+    assert(c.count(1) == 2);
+    assert(c.count(2) == 2);
+    assert(c.count(3) == 1);
+    assert(c.count(4) == 1);
 
-        k = c.erase(i, j);
-        assert(c.size() == 4);
-        assert(c.count(1) == 2);
-        assert(c.count(3) == 1);
-        assert(c.count(4) == 1);
+    k = c.erase(i, j);
+    assert(c.size() == 4);
+    assert(c.count(1) == 2);
+    assert(c.count(3) == 1);
+    assert(c.count(4) == 1);
 
-        k = c.erase(c.cbegin(), c.cend());
-        assert(c.size() == 0);
-        assert(k == c.end());
-    }
+    k = c.erase(c.cbegin(), c.cend());
+    assert(c.size() == 0);
+    assert(k == c.end());
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/unord/unord.multiset/extract_iterator.pass.cpp b/libcxx/test/std/containers/unord/unord.multiset/extract_iterator.pass.cpp
index d88e33bddd70d..c0a40bf0b332e 100644
--- a/libcxx/test/std/containers/unord/unord.multiset/extract_iterator.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.multiset/extract_iterator.pass.cpp
@@ -20,43 +20,40 @@
 #include "Counter.h"
 
 template <class Container>
-void test(Container& c)
-{
-    std::size_t sz = c.size();
-
-    for (auto first = c.cbegin(); first != c.cend();)
-    {
-        auto key_value = *first;
-        typename Container::node_type t = c.extract(first++);
-        --sz;
-        assert(t.value() == key_value);
-        assert(t.get_allocator() == c.get_allocator());
-        assert(sz == c.size());
-    }
-
-    assert(c.size() == 0);
+void test(Container& c) {
+  std::size_t sz = c.size();
+
+  for (auto first = c.cbegin(); first != c.cend();) {
+    auto key_value                  = *first;
+    typename Container::node_type t = c.extract(first++);
+    --sz;
+    assert(t.value() == key_value);
+    assert(t.get_allocator() == c.get_allocator());
+    assert(sz == c.size());
+  }
+
+  assert(c.size() == 0);
 }
 
-int main(int, char**)
-{
-    {
-        using set_type = std::unordered_multiset<int>;
-        set_type m = {1, 2, 3, 4, 5, 6};
-        test(m);
-    }
-
-    {
-        std::unordered_multiset<Counter<int>> m = {1, 2, 3, 4, 5, 6};
-        assert(Counter_base::gConstructed == 6);
-        test(m);
-        assert(Counter_base::gConstructed == 0);
-    }
-
-    {
-        using min_alloc_set = std::unordered_multiset<int, std::hash<int>, std::equal_to<int>, min_allocator<int>>;
-        min_alloc_set m = {1, 2, 3, 4, 5, 6};
-        test(m);
-    }
+int main(int, char**) {
+  {
+    using set_type = std::unordered_multiset<int>;
+    set_type m     = {1, 2, 3, 4, 5, 6};
+    test(m);
+  }
+
+  {
+    std::unordered_multiset<Counter<int>> m = {1, 2, 3, 4, 5, 6};
+    assert(Counter_base::gConstructed == 6);
+    test(m);
+    assert(Counter_base::gConstructed == 0);
+  }
+
+  {
+    using min_alloc_set = std::unordered_multiset<int, std::hash<int>, std::equal_to<int>, min_allocator<int>>;
+    min_alloc_set m     = {1, 2, 3, 4, 5, 6};
+    test(m);
+  }
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/unord/unord.multiset/extract_key.pass.cpp b/libcxx/test/std/containers/unord/unord.multiset/extract_key.pass.cpp
index ec1ace86b5661..bd83b45672d2f 100644
--- a/libcxx/test/std/containers/unord/unord.multiset/extract_key.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.multiset/extract_key.pass.cpp
@@ -20,54 +20,50 @@
 #include "Counter.h"
 
 template <class Container, class KeyTypeIter>
-void test(Container& c, KeyTypeIter first, KeyTypeIter last)
-{
-    std::size_t sz = c.size();
-    assert((std::size_t)std::distance(first, last) == sz);
+void test(Container& c, KeyTypeIter first, KeyTypeIter last) {
+  std::size_t sz = c.size();
+  assert((std::size_t)std::distance(first, last) == sz);
 
-    for (KeyTypeIter copy = first; copy != last; ++copy)
-    {
-        typename Container::node_type t = c.extract(*copy);
-        assert(!t.empty());
-        --sz;
-        assert(t.value() == *copy);
-        assert(t.get_allocator() == c.get_allocator());
-        assert(sz == c.size());
-    }
+  for (KeyTypeIter copy = first; copy != last; ++copy) {
+    typename Container::node_type t = c.extract(*copy);
+    assert(!t.empty());
+    --sz;
+    assert(t.value() == *copy);
+    assert(t.get_allocator() == c.get_allocator());
+    assert(sz == c.size());
+  }
 
-    assert(c.size() == 0);
+  assert(c.size() == 0);
 
-    for (KeyTypeIter copy = first; copy != last; ++copy)
-    {
-        typename Container::node_type t = c.extract(*copy);
-        assert(t.empty());
-    }
+  for (KeyTypeIter copy = first; copy != last; ++copy) {
+    typename Container::node_type t = c.extract(*copy);
+    assert(t.empty());
+  }
 }
 
-int main(int, char**)
-{
-    {
-        std::unordered_multiset<int> m = {1, 2, 3, 4, 5, 6};
-        int keys[] = {1, 2, 3, 4, 5, 6};
-        test(m, std::begin(keys), std::end(keys));
-    }
+int main(int, char**) {
+  {
+    std::unordered_multiset<int> m = {1, 2, 3, 4, 5, 6};
+    int keys[]                     = {1, 2, 3, 4, 5, 6};
+    test(m, std::begin(keys), std::end(keys));
+  }
 
+  {
+    std::unordered_multiset<Counter<int>> m = {1, 2, 3, 4, 5, 6};
     {
-        std::unordered_multiset<Counter<int>> m = {1, 2, 3, 4, 5, 6};
-        {
-            Counter<int> keys[] = {1, 2, 3, 4, 5, 6};
-            assert(Counter_base::gConstructed == 6+6);
-            test(m, std::begin(keys), std::end(keys));
-        }
-        assert(Counter_base::gConstructed == 0);
+      Counter<int> keys[] = {1, 2, 3, 4, 5, 6};
+      assert(Counter_base::gConstructed == 6 + 6);
+      test(m, std::begin(keys), std::end(keys));
     }
+    assert(Counter_base::gConstructed == 0);
+  }
 
-    {
-        using min_alloc_set = std::unordered_multiset<int, std::hash<int>, std::equal_to<int>, min_allocator<int>>;
-        min_alloc_set m = {1, 2, 3, 4, 5, 6};
-        int keys[] = {1, 2, 3, 4, 5, 6};
-        test(m, std::begin(keys), std::end(keys));
-    }
+  {
+    using min_alloc_set = std::unordered_multiset<int, std::hash<int>, std::equal_to<int>, min_allocator<int>>;
+    min_alloc_set m     = {1, 2, 3, 4, 5, 6};
+    int keys[]          = {1, 2, 3, 4, 5, 6};
+    test(m, std::begin(keys), std::end(keys));
+  }
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/unord/unord.multiset/find.transparent.pass.cpp b/libcxx/test/std/containers/unord/unord.multiset/find.transparent.pass.cpp
index 27abb1569e419..3e67484a1dc29 100644
--- a/libcxx/test/std/containers/unord/unord.multiset/find.transparent.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.multiset/find.transparent.pass.cpp
@@ -21,44 +21,43 @@
 
 #include "test_transparent_unordered.h"
 
-int main(int, char**)
-{
-    using key_type = StoredType<int>;
-
-    {
-      // Make sure conversions don't happen for transparent non-final hasher and key_equal
-      using S = unord_set_type<std::unordered_multiset, transparent_hash, std::equal_to<>>;
-      test_transparent_find<S>({1, 1, 2});
-      test_transparent_find<const S>({1, 1, 2});
-    }
-
-    {
-      // Make sure conversions don't happen for transparent final hasher and key_equal
-      using S = unord_set_type<std::unordered_multiset, transparent_hash_final, transparent_equal_final>;
-      test_transparent_find<S>({1, 1, 2});
-      test_transparent_find<const S>({1, 1, 2});
-    }
-
-    {
-      // Make sure conversions do happen for non-transparent hasher
-      using S = unord_set_type<std::unordered_multiset, non_transparent_hash, std::equal_to<>>;
-      test_non_transparent_find<S>({1, 1, 2});
-      test_non_transparent_find<const S>({1, 1, 2});
-    }
-
-    {
-      // Make sure conversions do happen for non-transparent key_equal
-      using S = unord_set_type<std::unordered_multiset, transparent_hash, std::equal_to<key_type>>;
-      test_non_transparent_find<S>({1, 1, 2});
-      test_non_transparent_find<const S>({1, 1, 2});
-    }
-
-    {
-      // Make sure conversions do happen for both non-transparent hasher and key_equal
-      using S = unord_set_type<std::unordered_multiset, non_transparent_hash, std::equal_to<key_type>>;
-      test_non_transparent_find<S>({1, 1, 2});
-      test_non_transparent_find<const S>({1, 1, 2});
-    }
-
-    return 0;
+int main(int, char**) {
+  using key_type = StoredType<int>;
+
+  {
+    // Make sure conversions don't happen for transparent non-final hasher and key_equal
+    using S = unord_set_type<std::unordered_multiset, transparent_hash, std::equal_to<>>;
+    test_transparent_find<S>({1, 1, 2});
+    test_transparent_find<const S>({1, 1, 2});
+  }
+
+  {
+    // Make sure conversions don't happen for transparent final hasher and key_equal
+    using S = unord_set_type<std::unordered_multiset, transparent_hash_final, transparent_equal_final>;
+    test_transparent_find<S>({1, 1, 2});
+    test_transparent_find<const S>({1, 1, 2});
+  }
+
+  {
+    // Make sure conversions do happen for non-transparent hasher
+    using S = unord_set_type<std::unordered_multiset, non_transparent_hash, std::equal_to<>>;
+    test_non_transparent_find<S>({1, 1, 2});
+    test_non_transparent_find<const S>({1, 1, 2});
+  }
+
+  {
+    // Make sure conversions do happen for non-transparent key_equal
+    using S = unord_set_type<std::unordered_multiset, transparent_hash, std::equal_to<key_type>>;
+    test_non_transparent_find<S>({1, 1, 2});
+    test_non_transparent_find<const S>({1, 1, 2});
+  }
+
+  {
+    // Make sure conversions do happen for both non-transparent hasher and key_equal
+    using S = unord_set_type<std::unordered_multiset, non_transparent_hash, std::equal_to<key_type>>;
+    test_non_transparent_find<S>({1, 1, 2});
+    test_non_transparent_find<const S>({1, 1, 2});
+  }
+
+  return 0;
 }
diff --git a/libcxx/test/std/containers/unord/unord.multiset/find_const.pass.cpp b/libcxx/test/std/containers/unord/unord.multiset/find_const.pass.cpp
index 65570dcb51e93..5d751ff57d9c9 100644
--- a/libcxx/test/std/containers/unord/unord.multiset/find_const.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.multiset/find_const.pass.cpp
@@ -20,50 +20,28 @@
 #include "test_macros.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef std::unordered_multiset<int> C;
-        typedef int P;
-        P a[] =
-        {
-            P(10),
-            P(20),
-            P(30),
-            P(40),
-            P(50),
-            P(60),
-            P(70),
-            P(80)
-        };
-        const C c(std::begin(a), std::end(a));
-        C::const_iterator i = c.find(30);
-        assert(*i == 30);
-        i = c.find(5);
-        assert(i == c.cend());
-    }
+int main(int, char**) {
+  {
+    typedef std::unordered_multiset<int> C;
+    typedef int P;
+    P a[] = {P(10), P(20), P(30), P(40), P(50), P(60), P(70), P(80)};
+    const C c(std::begin(a), std::end(a));
+    C::const_iterator i = c.find(30);
+    assert(*i == 30);
+    i = c.find(5);
+    assert(i == c.cend());
+  }
 #if TEST_STD_VER >= 11
-    {
-        typedef std::unordered_multiset<int, std::hash<int>,
-                                      std::equal_to<int>, min_allocator<int>> C;
-        typedef int P;
-        P a[] =
-        {
-            P(10),
-            P(20),
-            P(30),
-            P(40),
-            P(50),
-            P(60),
-            P(70),
-            P(80)
-        };
-        const C c(std::begin(a), std::end(a));
-        C::const_iterator i = c.find(30);
-        assert(*i == 30);
-        i = c.find(5);
-        assert(i == c.cend());
-    }
+  {
+    typedef std::unordered_multiset<int, std::hash<int>, std::equal_to<int>, min_allocator<int>> C;
+    typedef int P;
+    P a[] = {P(10), P(20), P(30), P(40), P(50), P(60), P(70), P(80)};
+    const C c(std::begin(a), std::end(a));
+    C::const_iterator i = c.find(30);
+    assert(*i == 30);
+    i = c.find(5);
+    assert(i == c.cend());
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/unord/unord.multiset/find_non_const.pass.cpp b/libcxx/test/std/containers/unord/unord.multiset/find_non_const.pass.cpp
index a70109bb94448..b15b6443e343c 100644
--- a/libcxx/test/std/containers/unord/unord.multiset/find_non_const.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.multiset/find_non_const.pass.cpp
@@ -20,50 +20,28 @@
 #include "test_macros.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef std::unordered_multiset<int> C;
-        typedef int P;
-        P a[] =
-        {
-            P(10),
-            P(20),
-            P(30),
-            P(40),
-            P(50),
-            P(60),
-            P(70),
-            P(80)
-        };
-        C c(std::begin(a), std::end(a));
-        C::iterator i = c.find(30);
-        assert(*i == 30);
-        i = c.find(5);
-        assert(i == c.cend());
-    }
+int main(int, char**) {
+  {
+    typedef std::unordered_multiset<int> C;
+    typedef int P;
+    P a[] = {P(10), P(20), P(30), P(40), P(50), P(60), P(70), P(80)};
+    C c(std::begin(a), std::end(a));
+    C::iterator i = c.find(30);
+    assert(*i == 30);
+    i = c.find(5);
+    assert(i == c.cend());
+  }
 #if TEST_STD_VER >= 11
-    {
-        typedef std::unordered_multiset<int, std::hash<int>,
-                                      std::equal_to<int>, min_allocator<int>> C;
-        typedef int P;
-        P a[] =
-        {
-            P(10),
-            P(20),
-            P(30),
-            P(40),
-            P(50),
-            P(60),
-            P(70),
-            P(80)
-        };
-        C c(std::begin(a), std::end(a));
-        C::iterator i = c.find(30);
-        assert(*i == 30);
-        i = c.find(5);
-        assert(i == c.cend());
-    }
+  {
+    typedef std::unordered_multiset<int, std::hash<int>, std::equal_to<int>, min_allocator<int>> C;
+    typedef int P;
+    P a[] = {P(10), P(20), P(30), P(40), P(50), P(60), P(70), P(80)};
+    C c(std::begin(a), std::end(a));
+    C::iterator i = c.find(30);
+    assert(*i == 30);
+    i = c.find(5);
+    assert(i == c.cend());
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/unord/unord.multiset/get_allocator.pass.cpp b/libcxx/test/std/containers/unord/unord.multiset/get_allocator.pass.cpp
index 699565cc0378a..4005ff50a1a95 100644
--- a/libcxx/test/std/containers/unord/unord.multiset/get_allocator.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.multiset/get_allocator.pass.cpp
@@ -19,18 +19,16 @@
 #include "test_macros.h"
 
 int main(int, char**) {
-    {
-        std::allocator<int> alloc;
-        const std::unordered_multiset<int> s(alloc);
-        assert(s.get_allocator() == alloc);
-    }
-    {
-        other_allocator<int> alloc(1);
-        const std::unordered_multiset<int, std::hash<int>,
-                                      std::equal_to<int>,
-                                      other_allocator<int> > s(alloc);
-        assert(s.get_allocator() == alloc);
-    }
+  {
+    std::allocator<int> alloc;
+    const std::unordered_multiset<int> s(alloc);
+    assert(s.get_allocator() == alloc);
+  }
+  {
+    other_allocator<int> alloc(1);
+    const std::unordered_multiset<int, std::hash<int>, std::equal_to<int>, other_allocator<int> > s(alloc);
+    assert(s.get_allocator() == alloc);
+  }
 
-    return 0;
+  return 0;
 }
diff --git a/libcxx/test/std/containers/unord/unord.multiset/incomplete.pass.cpp b/libcxx/test/std/containers/unord/unord.multiset/incomplete.pass.cpp
index a90aa9a7e1975..5d78db2feb89b 100644
--- a/libcxx/test/std/containers/unord/unord.multiset/incomplete.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.multiset/incomplete.pass.cpp
@@ -20,22 +20,22 @@
 template <class Tp>
 struct MyHash {
   MyHash() {}
-  std::size_t operator()(Tp const&) const {return 42;}
+  std::size_t operator()(Tp const&) const { return 42; }
 };
 
 struct A {
-    typedef std::unordered_multiset<A, MyHash<A> > Map;
-    Map m;
-    Map::iterator it;
-    Map::const_iterator cit;
-    Map::local_iterator lit;
-    Map::const_local_iterator clit;
+  typedef std::unordered_multiset<A, MyHash<A> > Map;
+  Map m;
+  Map::iterator it;
+  Map::const_iterator cit;
+  Map::local_iterator lit;
+  Map::const_local_iterator clit;
 };
 
 inline bool operator==(A const& L, A const& R) { return &L == &R; }
 
 int main(int, char**) {
-    A a;
+  A a;
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/unord/unord.multiset/insert_const_lvalue.pass.cpp b/libcxx/test/std/containers/unord/unord.multiset/insert_const_lvalue.pass.cpp
index 6f7c05bea2bab..6c458be54c412 100644
--- a/libcxx/test/std/containers/unord/unord.multiset/insert_const_lvalue.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.multiset/insert_const_lvalue.pass.cpp
@@ -20,42 +20,39 @@
 #include "test_macros.h"
 #include "min_allocator.h"
 
-template<class Container>
-void do_insert_const_lvalue_test()
-{
-    typedef Container C;
-    typedef typename C::iterator R;
-    typedef typename C::value_type VT;
-    C c;
-    const VT v1(3.5);
-    R r = c.insert(v1);
-    assert(c.size() == 1);
-    assert(*r == 3.5);
-
-    r = c.insert(v1);
-    assert(c.size() == 2);
-    assert(*r == 3.5);
-
-    const VT v2(4.5);
-    r = c.insert(v2);
-    assert(c.size() == 3);
-    assert(*r == 4.5);
-
-    const VT v3(5.5);
-    r = c.insert(v3);
-    assert(c.size() == 4);
-    assert(*r == 5.5);
+template <class Container>
+void do_insert_const_lvalue_test() {
+  typedef Container C;
+  typedef typename C::iterator R;
+  typedef typename C::value_type VT;
+  C c;
+  const VT v1(3.5);
+  R r = c.insert(v1);
+  assert(c.size() == 1);
+  assert(*r == 3.5);
+
+  r = c.insert(v1);
+  assert(c.size() == 2);
+  assert(*r == 3.5);
+
+  const VT v2(4.5);
+  r = c.insert(v2);
+  assert(c.size() == 3);
+  assert(*r == 4.5);
+
+  const VT v3(5.5);
+  r = c.insert(v3);
+  assert(c.size() == 4);
+  assert(*r == 5.5);
 }
 
-int main(int, char**)
-{
-    do_insert_const_lvalue_test<std::unordered_multiset<double> >();
+int main(int, char**) {
+  do_insert_const_lvalue_test<std::unordered_multiset<double> >();
 #if TEST_STD_VER >= 11
-    {
-        typedef std::unordered_multiset<double, std::hash<double>,
-            std::equal_to<double>, min_allocator<double>> C;
-        do_insert_const_lvalue_test<C>();
-    }
+  {
+    typedef std::unordered_multiset<double, std::hash<double>, std::equal_to<double>, min_allocator<double>> C;
+    do_insert_const_lvalue_test<C>();
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/unord/unord.multiset/insert_emplace_allocator_requirements.pass.cpp b/libcxx/test/std/containers/unord/unord.multiset/insert_emplace_allocator_requirements.pass.cpp
index c94f27c00a54b..751744892fe90 100644
--- a/libcxx/test/std/containers/unord/unord.multiset/insert_emplace_allocator_requirements.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.multiset/insert_emplace_allocator_requirements.pass.cpp
@@ -19,8 +19,7 @@
 #include "container_test_types.h"
 #include "../../set_allocator_requirement_test_templates.h"
 
-int main(int, char**)
-{
+int main(int, char**) {
   testMultisetInsert<TCT::unordered_multiset<> >();
   testMultisetEmplace<TCT::unordered_multiset<> >();
 
diff --git a/libcxx/test/std/containers/unord/unord.multiset/insert_hint_const_lvalue.pass.cpp b/libcxx/test/std/containers/unord/unord.multiset/insert_hint_const_lvalue.pass.cpp
index a33cada53684c..60a3b18f22322 100644
--- a/libcxx/test/std/containers/unord/unord.multiset/insert_hint_const_lvalue.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.multiset/insert_hint_const_lvalue.pass.cpp
@@ -20,44 +20,41 @@
 #include "test_macros.h"
 #include "min_allocator.h"
 
-template<class Container>
-void do_insert_hint_const_lvalue_test()
-{
-    typedef Container C;
-    typedef typename C::iterator R;
-    typedef typename C::value_type VT;
-    C c;
-    typename C::const_iterator e = c.end();
-    const VT v1(3.5);
-    R r = c.insert(e, v1);
-    assert(c.size() == 1);
-    assert(*r == 3.5);
-
-    r = c.insert(r, v1);
-    assert(c.size() == 2);
-    assert(*r == 3.5);
-
-    const VT v2(4.5);
-    r = c.insert(c.end(), v2);
-    assert(c.size() == 3);
-    assert(*r == 4.5);
-
-    const VT v3(5.5);
-    r = c.insert(c.end(), v3);
-    assert(c.size() == 4);
-    assert(*r == 5.5);
+template <class Container>
+void do_insert_hint_const_lvalue_test() {
+  typedef Container C;
+  typedef typename C::iterator R;
+  typedef typename C::value_type VT;
+  C c;
+  typename C::const_iterator e = c.end();
+  const VT v1(3.5);
+  R r = c.insert(e, v1);
+  assert(c.size() == 1);
+  assert(*r == 3.5);
+
+  r = c.insert(r, v1);
+  assert(c.size() == 2);
+  assert(*r == 3.5);
+
+  const VT v2(4.5);
+  r = c.insert(c.end(), v2);
+  assert(c.size() == 3);
+  assert(*r == 4.5);
+
+  const VT v3(5.5);
+  r = c.insert(c.end(), v3);
+  assert(c.size() == 4);
+  assert(*r == 5.5);
 }
 
-int main(int, char**)
-{
-    do_insert_hint_const_lvalue_test<std::unordered_multiset<double> >();
+int main(int, char**) {
+  do_insert_hint_const_lvalue_test<std::unordered_multiset<double> >();
 #if TEST_STD_VER >= 11
-    {
-        typedef std::unordered_multiset<double, std::hash<double>,
-            std::equal_to<double>, min_allocator<double>> C;
-        do_insert_hint_const_lvalue_test<C>();
-    }
+  {
+    typedef std::unordered_multiset<double, std::hash<double>, std::equal_to<double>, min_allocator<double>> C;
+    do_insert_hint_const_lvalue_test<C>();
+  }
 #endif
 
-    return 0;
+  return 0;
 }
diff --git a/libcxx/test/std/containers/unord/unord.multiset/insert_hint_rvalue.pass.cpp b/libcxx/test/std/containers/unord/unord.multiset/insert_hint_rvalue.pass.cpp
index ab5302462e97c..e87bfbdae4b1c 100644
--- a/libcxx/test/std/containers/unord/unord.multiset/insert_hint_rvalue.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.multiset/insert_hint_rvalue.pass.cpp
@@ -21,99 +21,96 @@
 #include "MoveOnly.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef std::unordered_multiset<double> C;
-        typedef C::iterator R;
-        typedef double P;
-        C c;
-        C::const_iterator e = c.end();
-        R r = c.insert(e, P(3.5));
-        assert(c.size() == 1);
-        assert(*r == 3.5);
-
-        r = c.insert(r, P(3.5));
-        assert(c.size() == 2);
-        assert(*r == 3.5);
-
-        r = c.insert(c.end(), P(4.5));
-        assert(c.size() == 3);
-        assert(*r == 4.5);
-
-        r = c.insert(c.end(), P(5.5));
-        assert(c.size() == 4);
-        assert(*r == 5.5);
-    }
+int main(int, char**) {
+  {
+    typedef std::unordered_multiset<double> C;
+    typedef C::iterator R;
+    typedef double P;
+    C c;
+    C::const_iterator e = c.end();
+    R r                 = c.insert(e, P(3.5));
+    assert(c.size() == 1);
+    assert(*r == 3.5);
+
+    r = c.insert(r, P(3.5));
+    assert(c.size() == 2);
+    assert(*r == 3.5);
+
+    r = c.insert(c.end(), P(4.5));
+    assert(c.size() == 3);
+    assert(*r == 4.5);
+
+    r = c.insert(c.end(), P(5.5));
+    assert(c.size() == 4);
+    assert(*r == 5.5);
+  }
 #if TEST_STD_VER >= 11
-    {
-        typedef std::unordered_multiset<MoveOnly> C;
-        typedef C::iterator R;
-        typedef MoveOnly P;
-        C c;
-        C::const_iterator e = c.end();
-        R r = c.insert(e, P(3));
-        assert(c.size() == 1);
-        assert(*r == 3);
-
-        r = c.insert(r, P(3));
-        assert(c.size() == 2);
-        assert(*r == 3);
-
-        r = c.insert(c.end(), P(4));
-        assert(c.size() == 3);
-        assert(*r == 4);
-
-        r = c.insert(c.end(), P(5));
-        assert(c.size() == 4);
-        assert(*r == 5);
-    }
-    {
-        typedef std::unordered_multiset<double, std::hash<double>,
-                                std::equal_to<double>, min_allocator<double>> C;
-        typedef C::iterator R;
-        typedef double P;
-        C c;
-        C::const_iterator e = c.end();
-        R r = c.insert(e, P(3.5));
-        assert(c.size() == 1);
-        assert(*r == 3.5);
-
-        r = c.insert(r, P(3.5));
-        assert(c.size() == 2);
-        assert(*r == 3.5);
-
-        r = c.insert(c.end(), P(4.5));
-        assert(c.size() == 3);
-        assert(*r == 4.5);
-
-        r = c.insert(c.end(), P(5.5));
-        assert(c.size() == 4);
-        assert(*r == 5.5);
-    }
-    {
-        typedef std::unordered_multiset<MoveOnly, std::hash<MoveOnly>,
-                            std::equal_to<MoveOnly>, min_allocator<MoveOnly>> C;
-        typedef C::iterator R;
-        typedef MoveOnly P;
-        C c;
-        C::const_iterator e = c.end();
-        R r = c.insert(e, P(3));
-        assert(c.size() == 1);
-        assert(*r == 3);
-
-        r = c.insert(r, P(3));
-        assert(c.size() == 2);
-        assert(*r == 3);
-
-        r = c.insert(c.end(), P(4));
-        assert(c.size() == 3);
-        assert(*r == 4);
-
-        r = c.insert(c.end(), P(5));
-        assert(c.size() == 4);
-        assert(*r == 5);
-    }
+  {
+    typedef std::unordered_multiset<MoveOnly> C;
+    typedef C::iterator R;
+    typedef MoveOnly P;
+    C c;
+    C::const_iterator e = c.end();
+    R r                 = c.insert(e, P(3));
+    assert(c.size() == 1);
+    assert(*r == 3);
+
+    r = c.insert(r, P(3));
+    assert(c.size() == 2);
+    assert(*r == 3);
+
+    r = c.insert(c.end(), P(4));
+    assert(c.size() == 3);
+    assert(*r == 4);
+
+    r = c.insert(c.end(), P(5));
+    assert(c.size() == 4);
+    assert(*r == 5);
+  }
+  {
+    typedef std::unordered_multiset<double, std::hash<double>, std::equal_to<double>, min_allocator<double>> C;
+    typedef C::iterator R;
+    typedef double P;
+    C c;
+    C::const_iterator e = c.end();
+    R r                 = c.insert(e, P(3.5));
+    assert(c.size() == 1);
+    assert(*r == 3.5);
+
+    r = c.insert(r, P(3.5));
+    assert(c.size() == 2);
+    assert(*r == 3.5);
+
+    r = c.insert(c.end(), P(4.5));
+    assert(c.size() == 3);
+    assert(*r == 4.5);
+
+    r = c.insert(c.end(), P(5.5));
+    assert(c.size() == 4);
+    assert(*r == 5.5);
+  }
+  {
+    typedef std::unordered_multiset<MoveOnly, std::hash<MoveOnly>, std::equal_to<MoveOnly>, min_allocator<MoveOnly>> C;
+    typedef C::iterator R;
+    typedef MoveOnly P;
+    C c;
+    C::const_iterator e = c.end();
+    R r                 = c.insert(e, P(3));
+    assert(c.size() == 1);
+    assert(*r == 3);
+
+    r = c.insert(r, P(3));
+    assert(c.size() == 2);
+    assert(*r == 3);
+
+    r = c.insert(c.end(), P(4));
+    assert(c.size() == 3);
+    assert(*r == 4);
+
+    r = c.insert(c.end(), P(5));
+    assert(c.size() == 4);
+    assert(*r == 5);
+  }
 #endif // TEST_STD_VER >= 11
 
   return 0;
diff --git a/libcxx/test/std/containers/unord/unord.multiset/insert_init.pass.cpp b/libcxx/test/std/containers/unord/unord.multiset/insert_init.pass.cpp
index d27d0a9b65a58..b58df799042a5 100644
--- a/libcxx/test/std/containers/unord/unord.multiset/insert_init.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.multiset/insert_init.pass.cpp
@@ -23,49 +23,29 @@
 #include "test_iterators.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef std::unordered_multiset<int> C;
-        typedef int P;
-        C c;
-        c.insert(
-                    {
-                        P(1),
-                        P(2),
-                        P(3),
-                        P(4),
-                        P(1),
-                        P(2)
-                    }
-                );
-        assert(c.size() == 6);
-        assert(c.count(1) == 2);
-        assert(c.count(2) == 2);
-        assert(c.count(3) == 1);
-        assert(c.count(4) == 1);
-    }
-    {
-        typedef std::unordered_multiset<int, std::hash<int>,
-                                      std::equal_to<int>, min_allocator<int>> C;
-        typedef int P;
-        C c;
-        c.insert(
-                    {
-                        P(1),
-                        P(2),
-                        P(3),
-                        P(4),
-                        P(1),
-                        P(2)
-                    }
-                );
-        assert(c.size() == 6);
-        assert(c.count(1) == 2);
-        assert(c.count(2) == 2);
-        assert(c.count(3) == 1);
-        assert(c.count(4) == 1);
-    }
+int main(int, char**) {
+  {
+    typedef std::unordered_multiset<int> C;
+    typedef int P;
+    C c;
+    c.insert({P(1), P(2), P(3), P(4), P(1), P(2)});
+    assert(c.size() == 6);
+    assert(c.count(1) == 2);
+    assert(c.count(2) == 2);
+    assert(c.count(3) == 1);
+    assert(c.count(4) == 1);
+  }
+  {
+    typedef std::unordered_multiset<int, std::hash<int>, std::equal_to<int>, min_allocator<int>> C;
+    typedef int P;
+    C c;
+    c.insert({P(1), P(2), P(3), P(4), P(1), P(2)});
+    assert(c.size() == 6);
+    assert(c.count(1) == 2);
+    assert(c.count(2) == 2);
+    assert(c.count(3) == 1);
+    assert(c.count(4) == 1);
+  }
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/unord/unord.multiset/insert_iter_iter.pass.cpp b/libcxx/test/std/containers/unord/unord.multiset/insert_iter_iter.pass.cpp
index f893a9dfd0912..e449ab51e5709 100644
--- a/libcxx/test/std/containers/unord/unord.multiset/insert_iter_iter.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.multiset/insert_iter_iter.pass.cpp
@@ -22,50 +22,32 @@
 #include "test_iterators.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef std::unordered_multiset<int> C;
-        typedef int P;
-        P a[] =
-        {
-            P(1),
-            P(2),
-            P(3),
-            P(4),
-            P(1),
-            P(2)
-        };
-        C c;
-        c.insert(cpp17_input_iterator<P*>(a), cpp17_input_iterator<P*>(a + sizeof(a)/sizeof(a[0])));
-        assert(c.size() == 6);
-        assert(c.count(1) == 2);
-        assert(c.count(2) == 2);
-        assert(c.count(3) == 1);
-        assert(c.count(4) == 1);
-    }
+int main(int, char**) {
+  {
+    typedef std::unordered_multiset<int> C;
+    typedef int P;
+    P a[] = {P(1), P(2), P(3), P(4), P(1), P(2)};
+    C c;
+    c.insert(cpp17_input_iterator<P*>(a), cpp17_input_iterator<P*>(a + sizeof(a) / sizeof(a[0])));
+    assert(c.size() == 6);
+    assert(c.count(1) == 2);
+    assert(c.count(2) == 2);
+    assert(c.count(3) == 1);
+    assert(c.count(4) == 1);
+  }
 #if TEST_STD_VER >= 11
-    {
-        typedef std::unordered_multiset<int, std::hash<int>,
-                                      std::equal_to<int>, min_allocator<int>> C;
-        typedef int P;
-        P a[] =
-        {
-            P(1),
-            P(2),
-            P(3),
-            P(4),
-            P(1),
-            P(2)
-        };
-        C c;
-        c.insert(cpp17_input_iterator<P*>(a), cpp17_input_iterator<P*>(a + sizeof(a)/sizeof(a[0])));
-        assert(c.size() == 6);
-        assert(c.count(1) == 2);
-        assert(c.count(2) == 2);
-        assert(c.count(3) == 1);
-        assert(c.count(4) == 1);
-    }
+  {
+    typedef std::unordered_multiset<int, std::hash<int>, std::equal_to<int>, min_allocator<int>> C;
+    typedef int P;
+    P a[] = {P(1), P(2), P(3), P(4), P(1), P(2)};
+    C c;
+    c.insert(cpp17_input_iterator<P*>(a), cpp17_input_iterator<P*>(a + sizeof(a) / sizeof(a[0])));
+    assert(c.size() == 6);
+    assert(c.count(1) == 2);
+    assert(c.count(2) == 2);
+    assert(c.count(3) == 1);
+    assert(c.count(4) == 1);
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/unord/unord.multiset/insert_node_type.pass.cpp b/libcxx/test/std/containers/unord/unord.multiset/insert_node_type.pass.cpp
index d57427724dcaf..8006a254f9379 100644
--- a/libcxx/test/std/containers/unord/unord.multiset/insert_node_type.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.multiset/insert_node_type.pass.cpp
@@ -20,59 +20,53 @@
 #include "min_allocator.h"
 
 template <class Container>
-typename Container::node_type
-node_factory(typename Container::key_type const& key)
-{
-    static Container c;
-    auto it = c.insert(key);
-    return c.extract(it);
+typename Container::node_type node_factory(typename Container::key_type const& key) {
+  static Container c;
+  auto it = c.insert(key);
+  return c.extract(it);
 }
 
 template <class Container>
-void test(Container& c)
-{
-    auto* nf = &node_factory<Container>;
+void test(Container& c) {
+  auto* nf = &node_factory<Container>;
 
-    for (int i = 0; i != 10; ++i)
-    {
-        typename Container::node_type node = nf(i);
-        assert(!node.empty());
-        typename Container::iterator it = c.insert(std::move(node));
-        assert(node.empty());
-        assert(it == c.find(i) && it != c.end());
-        assert(*it == i);
-        assert(node.empty());
-    }
+  for (int i = 0; i != 10; ++i) {
+    typename Container::node_type node = nf(i);
+    assert(!node.empty());
+    typename Container::iterator it = c.insert(std::move(node));
+    assert(node.empty());
+    assert(it == c.find(i) && it != c.end());
+    assert(*it == i);
+    assert(node.empty());
+  }
 
-    assert(c.size() == 10);
+  assert(c.size() == 10);
 
-    { // Insert empty node.
-        typename Container::node_type def;
-        auto it = c.insert(std::move(def));
-        assert(def.empty());
-        assert(it == c.end());
-    }
+  { // Insert empty node.
+    typename Container::node_type def;
+    auto it = c.insert(std::move(def));
+    assert(def.empty());
+    assert(it == c.end());
+  }
 
-    { // Insert duplicate node.
-        typename Container::node_type dupl = nf(0);
-        auto it = c.insert(std::move(dupl));
-        assert(*it == 0);
-    }
+  { // Insert duplicate node.
+    typename Container::node_type dupl = nf(0);
+    auto it                            = c.insert(std::move(dupl));
+    assert(*it == 0);
+  }
 
-    assert(c.size() == 11);
-    assert(c.count(0) == 2);
-    for (int i = 1; i != 10; ++i)
-    {
-        assert(c.count(i) == 1);
-    }
+  assert(c.size() == 11);
+  assert(c.count(0) == 2);
+  for (int i = 1; i != 10; ++i) {
+    assert(c.count(i) == 1);
+  }
 }
 
-int main(int, char**)
-{
-    std::unordered_multiset<int> m;
-    test(m);
-    std::unordered_multiset<int, std::hash<int>, std::equal_to<int>, min_allocator<int>> m2;
-    test(m2);
+int main(int, char**) {
+  std::unordered_multiset<int> m;
+  test(m);
+  std::unordered_multiset<int, std::hash<int>, std::equal_to<int>, min_allocator<int>> m2;
+  test(m2);
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/unord/unord.multiset/insert_node_type_hint.pass.cpp b/libcxx/test/std/containers/unord/unord.multiset/insert_node_type_hint.pass.cpp
index 2706899b19a95..7509cf746043f 100644
--- a/libcxx/test/std/containers/unord/unord.multiset/insert_node_type_hint.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.multiset/insert_node_type_hint.pass.cpp
@@ -19,43 +19,37 @@
 #include "min_allocator.h"
 
 template <class Container>
-typename Container::node_type
-node_factory(typename Container::key_type const& key)
-{
-    static Container c;
-    auto it = c.insert(key);
-    return c.extract(it);
+typename Container::node_type node_factory(typename Container::key_type const& key) {
+  static Container c;
+  auto it = c.insert(key);
+  return c.extract(it);
 }
 
 template <class Container>
-void test(Container& c)
-{
-    auto* nf = &node_factory<Container>;
-
-    for (int i = 0; i != 10; ++i)
-    {
-        typename Container::node_type node = nf(i);
-        assert(!node.empty());
-        std::size_t prev = c.size();
-        auto it = c.insert(c.end(), std::move(node));
-        assert(prev + 1 == c.size());
-        assert(*it == i);
-    }
-
-    assert(c.size() == 10);
-
-    for (int i = 0; i != 10; ++i)
-    {
-        assert(c.count(i) == 1);
-    }
+void test(Container& c) {
+  auto* nf = &node_factory<Container>;
+
+  for (int i = 0; i != 10; ++i) {
+    typename Container::node_type node = nf(i);
+    assert(!node.empty());
+    std::size_t prev = c.size();
+    auto it          = c.insert(c.end(), std::move(node));
+    assert(prev + 1 == c.size());
+    assert(*it == i);
+  }
+
+  assert(c.size() == 10);
+
+  for (int i = 0; i != 10; ++i) {
+    assert(c.count(i) == 1);
+  }
 }
 
-int main(int, char**)
-{
-    std::unordered_multiset<int> m;
-    test(m);
-    std::unordered_multiset<int, std::hash<int>, std::equal_to<int>, min_allocator<int>> m2;
-    test(m2);
+int main(int, char**) {
+  std::unordered_multiset<int> m;
+  test(m);
+  std::unordered_multiset<int, std::hash<int>, std::equal_to<int>, min_allocator<int>> m2;
+  test(m2);
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/unord/unord.multiset/insert_range.pass.cpp b/libcxx/test/std/containers/unord/unord.multiset/insert_range.pass.cpp
index c2d5ed3278332..6aeb463685d72 100644
--- a/libcxx/test/std/containers/unord/unord.multiset/insert_range.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.multiset/insert_range.pass.cpp
@@ -22,7 +22,8 @@
 
 int main(int, char**) {
   for_all_iterators_and_allocators<int, const int*>([]<class Iter, class Sent, class Alloc>() {
-    test_map_set_insert_range<std::unordered_multiset<int, test_hash<int>, test_equal_to<int>, Alloc>, int, Iter, Sent>(/*allow_duplicates=*/true);
+    test_map_set_insert_range<std::unordered_multiset<int, test_hash<int>, test_equal_to<int>, Alloc>, int, Iter, Sent>(
+        /*allow_duplicates=*/true);
   });
 
   static_assert(test_set_constraints_insert_range<std::unordered_multiset, int, double>());
@@ -34,4 +35,3 @@ int main(int, char**) {
 
   return 0;
 }
-
diff --git a/libcxx/test/std/containers/unord/unord.multiset/insert_rvalue.pass.cpp b/libcxx/test/std/containers/unord/unord.multiset/insert_rvalue.pass.cpp
index 1696f52b066e0..53ac84eccc2f1 100644
--- a/libcxx/test/std/containers/unord/unord.multiset/insert_rvalue.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.multiset/insert_rvalue.pass.cpp
@@ -21,95 +21,92 @@
 #include "MoveOnly.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef std::unordered_multiset<double> C;
-        typedef C::iterator R;
-        typedef double P;
-        C c;
-        R r = c.insert(P(3.5));
-        assert(c.size() == 1);
-        assert(*r == 3.5);
-
-        r = c.insert(P(3.5));
-        assert(c.size() == 2);
-        assert(*r == 3.5);
-
-        r = c.insert(P(4.5));
-        assert(c.size() == 3);
-        assert(*r == 4.5);
-
-        r = c.insert(P(5.5));
-        assert(c.size() == 4);
-        assert(*r == 5.5);
-    }
+int main(int, char**) {
+  {
+    typedef std::unordered_multiset<double> C;
+    typedef C::iterator R;
+    typedef double P;
+    C c;
+    R r = c.insert(P(3.5));
+    assert(c.size() == 1);
+    assert(*r == 3.5);
+
+    r = c.insert(P(3.5));
+    assert(c.size() == 2);
+    assert(*r == 3.5);
+
+    r = c.insert(P(4.5));
+    assert(c.size() == 3);
+    assert(*r == 4.5);
+
+    r = c.insert(P(5.5));
+    assert(c.size() == 4);
+    assert(*r == 5.5);
+  }
 #if TEST_STD_VER >= 11
-    {
-        typedef std::unordered_multiset<MoveOnly> C;
-        typedef C::iterator R;
-        typedef MoveOnly P;
-        C c;
-        R r = c.insert(P(3));
-        assert(c.size() == 1);
-        assert(*r == 3);
-
-        r = c.insert(P(3));
-        assert(c.size() == 2);
-        assert(*r == 3);
-
-        r = c.insert(P(4));
-        assert(c.size() == 3);
-        assert(*r == 4);
-
-        r = c.insert(P(5));
-        assert(c.size() == 4);
-        assert(*r == 5);
-    }
-    {
-        typedef std::unordered_multiset<double, std::hash<double>,
-                                std::equal_to<double>, min_allocator<double>> C;
-        typedef C::iterator R;
-        typedef double P;
-        C c;
-        R r = c.insert(P(3.5));
-        assert(c.size() == 1);
-        assert(*r == 3.5);
-
-        r = c.insert(P(3.5));
-        assert(c.size() == 2);
-        assert(*r == 3.5);
-
-        r = c.insert(P(4.5));
-        assert(c.size() == 3);
-        assert(*r == 4.5);
-
-        r = c.insert(P(5.5));
-        assert(c.size() == 4);
-        assert(*r == 5.5);
-    }
-    {
-        typedef std::unordered_multiset<MoveOnly, std::hash<MoveOnly>,
-                            std::equal_to<MoveOnly>, min_allocator<MoveOnly>> C;
-        typedef C::iterator R;
-        typedef MoveOnly P;
-        C c;
-        R r = c.insert(P(3));
-        assert(c.size() == 1);
-        assert(*r == 3);
-
-        r = c.insert(P(3));
-        assert(c.size() == 2);
-        assert(*r == 3);
-
-        r = c.insert(P(4));
-        assert(c.size() == 3);
-        assert(*r == 4);
-
-        r = c.insert(P(5));
-        assert(c.size() == 4);
-        assert(*r == 5);
-    }
+  {
+    typedef std::unordered_multiset<MoveOnly> C;
+    typedef C::iterator R;
+    typedef MoveOnly P;
+    C c;
+    R r = c.insert(P(3));
+    assert(c.size() == 1);
+    assert(*r == 3);
+
+    r = c.insert(P(3));
+    assert(c.size() == 2);
+    assert(*r == 3);
+
+    r = c.insert(P(4));
+    assert(c.size() == 3);
+    assert(*r == 4);
+
+    r = c.insert(P(5));
+    assert(c.size() == 4);
+    assert(*r == 5);
+  }
+  {
+    typedef std::unordered_multiset<double, std::hash<double>, std::equal_to<double>, min_allocator<double>> C;
+    typedef C::iterator R;
+    typedef double P;
+    C c;
+    R r = c.insert(P(3.5));
+    assert(c.size() == 1);
+    assert(*r == 3.5);
+
+    r = c.insert(P(3.5));
+    assert(c.size() == 2);
+    assert(*r == 3.5);
+
+    r = c.insert(P(4.5));
+    assert(c.size() == 3);
+    assert(*r == 4.5);
+
+    r = c.insert(P(5.5));
+    assert(c.size() == 4);
+    assert(*r == 5.5);
+  }
+  {
+    typedef std::unordered_multiset<MoveOnly, std::hash<MoveOnly>, std::equal_to<MoveOnly>, min_allocator<MoveOnly>> C;
+    typedef C::iterator R;
+    typedef MoveOnly P;
+    C c;
+    R r = c.insert(P(3));
+    assert(c.size() == 1);
+    assert(*r == 3);
+
+    r = c.insert(P(3));
+    assert(c.size() == 2);
+    assert(*r == 3);
+
+    r = c.insert(P(4));
+    assert(c.size() == 3);
+    assert(*r == 4);
+
+    r = c.insert(P(5));
+    assert(c.size() == 4);
+    assert(*r == 5);
+  }
 #endif // TEST_STD_VER >= 11
 
   return 0;
diff --git a/libcxx/test/std/containers/unord/unord.multiset/iterator_concept_conformance.compile.pass.cpp b/libcxx/test/std/containers/unord/unord.multiset/iterator_concept_conformance.compile.pass.cpp
index 816baa143f3d3..6e209ba892efb 100644
--- a/libcxx/test/std/containers/unord/unord.multiset/iterator_concept_conformance.compile.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.multiset/iterator_concept_conformance.compile.pass.cpp
@@ -16,11 +16,11 @@
 
 #include "test_macros.h"
 
-using iterator = std::unordered_multiset<int>::iterator;
-using const_iterator = std::unordered_multiset<int>::const_iterator;
-using local_iterator = std::unordered_multiset<int>::local_iterator;
+using iterator             = std::unordered_multiset<int>::iterator;
+using const_iterator       = std::unordered_multiset<int>::const_iterator;
+using local_iterator       = std::unordered_multiset<int>::local_iterator;
 using const_local_iterator = std::unordered_multiset<int>::const_local_iterator;
-using value_type = int;
+using value_type           = int;
 
 static_assert(std::forward_iterator<iterator>);
 LIBCPP_STATIC_ASSERT(!std::bidirectional_iterator<iterator>);
diff --git a/libcxx/test/std/containers/unord/unord.multiset/iterators.compile.fail.cpp b/libcxx/test/std/containers/unord/unord.multiset/iterators.compile.fail.cpp
index 2c282fdbaab4b..08950f732a044 100644
--- a/libcxx/test/std/containers/unord/unord.multiset/iterators.compile.fail.cpp
+++ b/libcxx/test/std/containers/unord/unord.multiset/iterators.compile.fail.cpp
@@ -24,47 +24,30 @@
 
 #include "test_macros.h"
 
-int main(int, char**)
-{
-    {
-        typedef std::unordered_multiset<int> C;
-        typedef int P;
-        P a[] =
-        {
-            P(1),
-            P(2),
-            P(3),
-            P(4),
-            P(1),
-            P(2)
-        };
-        C c(a, a + sizeof(a)/sizeof(a[0]));
-        LIBCPP_ASSERT(c.bucket_count() == 7);
-        assert(c.size() == 6);
-        assert(std::distance(c.begin(), c.end()) == c.size());
-        assert(std::distance(c.cbegin(), c.cend()) == c.size());
-        C::iterator i = c.begin();
-        assert(*i == 1);
-        *i = 2;
-    }
-    {
-        typedef std::unordered_multiset<int> C;
-        typedef int P;
-        P a[] =
-        {
-            P(1),
-            P(2),
-            P(3),
-            P(4),
-            P(1),
-            P(2)
-        };
-        const C c(a, a + sizeof(a)/sizeof(a[0]));
-        LIBCPP_ASSERT(c.bucket_count() == 7);
-        assert(c.size() == 6);
-        assert(std::distance(c.begin(), c.end()) == c.size());
-        assert(std::distance(c.cbegin(), c.cend()) == c.size());
-    }
+int main(int, char**) {
+  {
+    typedef std::unordered_multiset<int> C;
+    typedef int P;
+    P a[] = {P(1), P(2), P(3), P(4), P(1), P(2)};
+    C c(a, a + sizeof(a) / sizeof(a[0]));
+    LIBCPP_ASSERT(c.bucket_count() == 7);
+    assert(c.size() == 6);
+    assert(std::distance(c.begin(), c.end()) == c.size());
+    assert(std::distance(c.cbegin(), c.cend()) == c.size());
+    C::iterator i = c.begin();
+    assert(*i == 1);
+    *i = 2;
+  }
+  {
+    typedef std::unordered_multiset<int> C;
+    typedef int P;
+    P a[] = {P(1), P(2), P(3), P(4), P(1), P(2)};
+    const C c(a, a + sizeof(a) / sizeof(a[0]));
+    LIBCPP_ASSERT(c.bucket_count() == 7);
+    assert(c.size() == 6);
+    assert(std::distance(c.begin(), c.end()) == c.size());
+    assert(std::distance(c.cbegin(), c.cend()) == c.size());
+  }
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/unord/unord.multiset/iterators.pass.cpp b/libcxx/test/std/containers/unord/unord.multiset/iterators.pass.cpp
index 2c0cd496ddb24..c75be73850656 100644
--- a/libcxx/test/std/containers/unord/unord.multiset/iterators.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.multiset/iterators.pass.cpp
@@ -26,104 +26,69 @@
 #include "test_macros.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef std::unordered_multiset<int> C;
-        typedef int P;
-        P a[] =
-        {
-            P(1),
-            P(2),
-            P(3),
-            P(4),
-            P(1),
-            P(2)
-        };
-        C c(a, a + sizeof(a)/sizeof(a[0]));
-        assert(c.bucket_count() >= 7);
-        assert(c.size() == 6);
-        assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-        assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
-        C::iterator i;
-    }
-    {
-        typedef std::unordered_multiset<int> C;
-        typedef int P;
-        P a[] =
-        {
-            P(1),
-            P(2),
-            P(3),
-            P(4),
-            P(1),
-            P(2)
-        };
-        const C c(a, a + sizeof(a)/sizeof(a[0]));
-        assert(c.bucket_count() >= 7);
-        assert(c.size() == 6);
-        assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-        assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
-        C::const_iterator i;
-    }
+int main(int, char**) {
+  {
+    typedef std::unordered_multiset<int> C;
+    typedef int P;
+    P a[] = {P(1), P(2), P(3), P(4), P(1), P(2)};
+    C c(a, a + sizeof(a) / sizeof(a[0]));
+    assert(c.bucket_count() >= 7);
+    assert(c.size() == 6);
+    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
+    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
+    C::iterator i;
+  }
+  {
+    typedef std::unordered_multiset<int> C;
+    typedef int P;
+    P a[] = {P(1), P(2), P(3), P(4), P(1), P(2)};
+    const C c(a, a + sizeof(a) / sizeof(a[0]));
+    assert(c.bucket_count() >= 7);
+    assert(c.size() == 6);
+    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
+    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
+    C::const_iterator i;
+  }
 #if TEST_STD_VER >= 11
-    {
-        typedef std::unordered_multiset<int, std::hash<int>,
-                                      std::equal_to<int>, min_allocator<int>> C;
-        typedef int P;
-        P a[] =
-        {
-            P(1),
-            P(2),
-            P(3),
-            P(4),
-            P(1),
-            P(2)
-        };
-        C c(a, a + sizeof(a)/sizeof(a[0]));
-        assert(c.bucket_count() >= 7);
-        assert(c.size() == 6);
-        assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-        assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
-        C::iterator i;
-    }
-    {
-        typedef std::unordered_multiset<int, std::hash<int>,
-                                      std::equal_to<int>, min_allocator<int>> C;
-        typedef int P;
-        P a[] =
-        {
-            P(1),
-            P(2),
-            P(3),
-            P(4),
-            P(1),
-            P(2)
-        };
-        const C c(a, a + sizeof(a)/sizeof(a[0]));
-        assert(c.bucket_count() >= 7);
-        assert(c.size() == 6);
-        assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-        assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
-        C::const_iterator i;
-    }
+  {
+    typedef std::unordered_multiset<int, std::hash<int>, std::equal_to<int>, min_allocator<int>> C;
+    typedef int P;
+    P a[] = {P(1), P(2), P(3), P(4), P(1), P(2)};
+    C c(a, a + sizeof(a) / sizeof(a[0]));
+    assert(c.bucket_count() >= 7);
+    assert(c.size() == 6);
+    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
+    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
+    C::iterator i;
+  }
+  {
+    typedef std::unordered_multiset<int, std::hash<int>, std::equal_to<int>, min_allocator<int>> C;
+    typedef int P;
+    P a[] = {P(1), P(2), P(3), P(4), P(1), P(2)};
+    const C c(a, a + sizeof(a) / sizeof(a[0]));
+    assert(c.bucket_count() >= 7);
+    assert(c.size() == 6);
+    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
+    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
+    C::const_iterator i;
+  }
 #endif
 #if TEST_STD_VER > 11
-    { // N3644 testing
-        typedef std::unordered_multiset<int> C;
-        C::iterator ii1{}, ii2{};
-        C::iterator ii4 = ii1;
-        C::const_iterator cii{};
-        assert ( ii1 == ii2 );
-        assert ( ii1 == ii4 );
+  { // N3644 testing
+    typedef std::unordered_multiset<int> C;
+    C::iterator ii1{}, ii2{};
+    C::iterator ii4 = ii1;
+    C::const_iterator cii{};
+    assert(ii1 == ii2);
+    assert(ii1 == ii4);
 
-        assert (!(ii1 != ii2 ));
+    assert(!(ii1 != ii2));
 
-        assert ( (ii1 == cii ));
-        assert ( (cii == ii1 ));
-        assert (!(ii1 != cii ));
-        assert (!(cii != ii1 ));
-    }
+    assert((ii1 == cii));
+    assert((cii == ii1));
+    assert(!(ii1 != cii));
+    assert(!(cii != ii1));
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/unord/unord.multiset/load_factor.pass.cpp b/libcxx/test/std/containers/unord/unord.multiset/load_factor.pass.cpp
index 76c8132efe408..d383ae74948d7 100644
--- a/libcxx/test/std/containers/unord/unord.multiset/load_factor.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.multiset/load_factor.pass.cpp
@@ -22,55 +22,32 @@
 #include "test_macros.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef std::unordered_multiset<int> C;
-        typedef int P;
-        P a[] =
-        {
-            P(10),
-            P(20),
-            P(30),
-            P(40),
-            P(50),
-            P(60),
-            P(70),
-            P(80)
-        };
-        const C c(std::begin(a), std::end(a));
-        assert(std::fabs(c.load_factor() - (float)c.size()/c.bucket_count()) < FLT_EPSILON);
-    }
-    {
-        typedef std::unordered_multiset<int> C;
-        const C c;
-        assert(c.load_factor() == 0);
-    }
+int main(int, char**) {
+  {
+    typedef std::unordered_multiset<int> C;
+    typedef int P;
+    P a[] = {P(10), P(20), P(30), P(40), P(50), P(60), P(70), P(80)};
+    const C c(std::begin(a), std::end(a));
+    assert(std::fabs(c.load_factor() - (float)c.size() / c.bucket_count()) < FLT_EPSILON);
+  }
+  {
+    typedef std::unordered_multiset<int> C;
+    const C c;
+    assert(c.load_factor() == 0);
+  }
 #if TEST_STD_VER >= 11
-    {
-        typedef std::unordered_multiset<int, std::hash<int>,
-                                      std::equal_to<int>, min_allocator<int>> C;
-        typedef int P;
-        P a[] =
-        {
-            P(10),
-            P(20),
-            P(30),
-            P(40),
-            P(50),
-            P(60),
-            P(70),
-            P(80)
-        };
-        const C c(std::begin(a), std::end(a));
-        assert(std::fabs(c.load_factor() - (float)c.size()/c.bucket_count()) < FLT_EPSILON);
-    }
-    {
-        typedef std::unordered_multiset<int, std::hash<int>,
-                                      std::equal_to<int>, min_allocator<int>> C;
-        const C c;
-        assert(c.load_factor() == 0);
-    }
+  {
+    typedef std::unordered_multiset<int, std::hash<int>, std::equal_to<int>, min_allocator<int>> C;
+    typedef int P;
+    P a[] = {P(10), P(20), P(30), P(40), P(50), P(60), P(70), P(80)};
+    const C c(std::begin(a), std::end(a));
+    assert(std::fabs(c.load_factor() - (float)c.size() / c.bucket_count()) < FLT_EPSILON);
+  }
+  {
+    typedef std::unordered_multiset<int, std::hash<int>, std::equal_to<int>, min_allocator<int>> C;
+    const C c;
+    assert(c.load_factor() == 0);
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/unord/unord.multiset/local_iterators.compile.fail.cpp b/libcxx/test/std/containers/unord/unord.multiset/local_iterators.compile.fail.cpp
index d6f1a55fd49cb..b5b424f488738 100644
--- a/libcxx/test/std/containers/unord/unord.multiset/local_iterators.compile.fail.cpp
+++ b/libcxx/test/std/containers/unord/unord.multiset/local_iterators.compile.fail.cpp
@@ -24,241 +24,208 @@
 
 #include "test_macros.h"
 
-int main(int, char**)
-{
-    {
-        typedef std::unordered_multiset<int> C;
-        typedef int P;
-        typedef C::local_iterator I;
-        P a[] =
-        {
-            P(1),
-            P(2),
-            P(3),
-            P(4),
-            P(1),
-            P(2)
-        };
-        C c(a, a + sizeof(a)/sizeof(a[0]));
-        LIBCPP_ASSERT(c.bucket_count() == 7);
-        C::size_type b = c.bucket(0);
-        I i = c.begin(b);
-        I j = c.end(b);
-        assert(std::distance(i, j) == 0);
-
-        b = c.bucket(1);
-        i = c.begin(b);
-        j = c.end(b);
-        assert(std::distance(i, j) == 2);
-        assert(*i == 1);
-        ++i;
-        assert(*i == 1);
-        *i = 2;
-
-        b = c.bucket(2);
-        i = c.begin(b);
-        j = c.end(b);
-        assert(std::distance(i, j) == 2);
-        assert(*i == 2);
-        ++i;
-        assert(*i == 2);
-
-        b = c.bucket(3);
-        i = c.begin(b);
-        j = c.end(b);
-        assert(std::distance(i, j) == 1);
-        assert(*i == 3);
-
-        b = c.bucket(4);
-        i = c.begin(b);
-        j = c.end(b);
-        assert(std::distance(i, j) == 1);
-        assert(*i == 4);
-
-        b = c.bucket(5);
-        i = c.begin(b);
-        j = c.end(b);
-        assert(std::distance(i, j) == 0);
-
-        b = c.bucket(6);
-        i = c.begin(b);
-        j = c.end(b);
-        assert(std::distance(i, j) == 0);
-    }
-    {
-        typedef std::unordered_multiset<int> C;
-        typedef int P;
-        typedef C::const_local_iterator I;
-        P a[] =
-        {
-            P(1),
-            P(2),
-            P(3),
-            P(4),
-            P(1),
-            P(2)
-        };
-        const C c(a, a + sizeof(a)/sizeof(a[0]));
-        LIBCPP_ASSERT(c.bucket_count() == 7);
-        C::size_type b = c.bucket(0);
-        I i = c.begin(b);
-        I j = c.end(b);
-        assert(std::distance(i, j) == 0);
-
-        b = c.bucket(1);
-        i = c.begin(b);
-        j = c.end(b);
-        assert(std::distance(i, j) == 2);
-        assert(*i == 1);
-        ++i;
-        assert(*i == 1);
-
-        b = c.bucket(2);
-        i = c.begin(b);
-        j = c.end(b);
-        assert(std::distance(i, j) == 2);
-        assert(*i == 2);
-        ++i;
-        assert(*i == 2);
-
-        b = c.bucket(3);
-        i = c.begin(b);
-        j = c.end(b);
-        assert(std::distance(i, j) == 1);
-        assert(*i == 3);
-
-        b = c.bucket(4);
-        i = c.begin(b);
-        j = c.end(b);
-        assert(std::distance(i, j) == 1);
-        assert(*i == 4);
-
-        b = c.bucket(5);
-        i = c.begin(b);
-        j = c.end(b);
-        assert(std::distance(i, j) == 0);
-
-        b = c.bucket(6);
-        i = c.begin(b);
-        j = c.end(b);
-        assert(std::distance(i, j) == 0);
-    }
-    {
-        typedef std::unordered_multiset<int> C;
-        typedef int P;
-        typedef C::const_local_iterator I;
-        P a[] =
-        {
-            P(1),
-            P(2),
-            P(3),
-            P(4),
-            P(1),
-            P(2)
-        };
-        C c(a, a + sizeof(a)/sizeof(a[0]));
-        LIBCPP_ASSERT(c.bucket_count() == 7);
-        C::size_type b = c.bucket(0);
-        I i = c.cbegin(b);
-        I j = c.cend(b);
-        assert(std::distance(i, j) == 0);
-
-        b = c.bucket(1);
-        i = c.cbegin(b);
-        j = c.cend(b);
-        assert(std::distance(i, j) == 2);
-        assert(*i == 1);
-        ++i;
-        assert(*i == 1);
-
-        b = c.bucket(2);
-        i = c.cbegin(b);
-        j = c.cend(b);
-        assert(std::distance(i, j) == 2);
-        assert(*i == 2);
-        ++i;
-        assert(*i == 2);
-
-        b = c.bucket(3);
-        i = c.cbegin(b);
-        j = c.cend(b);
-        assert(std::distance(i, j) == 1);
-        assert(*i == 3);
-
-        b = c.bucket(4);
-        i = c.cbegin(b);
-        j = c.cend(b);
-        assert(std::distance(i, j) == 1);
-        assert(*i == 4);
-
-        b = c.bucket(5);
-        i = c.cbegin(b);
-        j = c.cend(b);
-        assert(std::distance(i, j) == 0);
-
-        b = c.bucket(6);
-        i = c.cbegin(b);
-        j = c.cend(b);
-        assert(std::distance(i, j) == 0);
-    }
-    {
-        typedef std::unordered_multiset<int> C;
-        typedef int P;
-        typedef C::const_local_iterator I;
-        P a[] =
-        {
-            P(1),
-            P(2),
-            P(3),
-            P(4),
-            P(1),
-            P(2)
-        };
-        const C c(a, a + sizeof(a)/sizeof(a[0]));
-        LIBCPP_ASSERT(c.bucket_count() == 7);
-        C::size_type b = c.bucket(0);
-        I i = c.cbegin(b);
-        I j = c.cend(b);
-        assert(std::distance(i, j) == 0);
-
-        b = c.bucket(1);
-        i = c.cbegin(b);
-        j = c.cend(b);
-        assert(std::distance(i, j) == 2);
-        assert(*i == 1);
-        ++i;
-        assert(*i == 1);
-
-        b = c.bucket(2);
-        i = c.cbegin(b);
-        j = c.cend(b);
-        assert(std::distance(i, j) == 2);
-        assert(*i == 2);
-        ++i;
-        assert(*i == 2);
-
-        b = c.bucket(3);
-        i = c.cbegin(b);
-        j = c.cend(b);
-        assert(std::distance(i, j) == 1);
-        assert(*i == 3);
-
-        b = c.bucket(4);
-        i = c.cbegin(b);
-        j = c.cend(b);
-        assert(std::distance(i, j) == 1);
-        assert(*i == 4);
-
-        b = c.bucket(5);
-        i = c.cbegin(b);
-        j = c.cend(b);
-        assert(std::distance(i, j) == 0);
-
-        b = c.bucket(6);
-        i = c.cbegin(b);
-        j = c.cend(b);
-        assert(std::distance(i, j) == 0);
-    }
+int main(int, char**) {
+  {
+    typedef std::unordered_multiset<int> C;
+    typedef int P;
+    typedef C::local_iterator I;
+    P a[] = {P(1), P(2), P(3), P(4), P(1), P(2)};
+    C c(a, a + sizeof(a) / sizeof(a[0]));
+    LIBCPP_ASSERT(c.bucket_count() == 7);
+    C::size_type b = c.bucket(0);
+    I i            = c.begin(b);
+    I j            = c.end(b);
+    assert(std::distance(i, j) == 0);
+
+    b = c.bucket(1);
+    i = c.begin(b);
+    j = c.end(b);
+    assert(std::distance(i, j) == 2);
+    assert(*i == 1);
+    ++i;
+    assert(*i == 1);
+    *i = 2;
+
+    b = c.bucket(2);
+    i = c.begin(b);
+    j = c.end(b);
+    assert(std::distance(i, j) == 2);
+    assert(*i == 2);
+    ++i;
+    assert(*i == 2);
+
+    b = c.bucket(3);
+    i = c.begin(b);
+    j = c.end(b);
+    assert(std::distance(i, j) == 1);
+    assert(*i == 3);
+
+    b = c.bucket(4);
+    i = c.begin(b);
+    j = c.end(b);
+    assert(std::distance(i, j) == 1);
+    assert(*i == 4);
+
+    b = c.bucket(5);
+    i = c.begin(b);
+    j = c.end(b);
+    assert(std::distance(i, j) == 0);
+
+    b = c.bucket(6);
+    i = c.begin(b);
+    j = c.end(b);
+    assert(std::distance(i, j) == 0);
+  }
+  {
+    typedef std::unordered_multiset<int> C;
+    typedef int P;
+    typedef C::const_local_iterator I;
+    P a[] = {P(1), P(2), P(3), P(4), P(1), P(2)};
+    const C c(a, a + sizeof(a) / sizeof(a[0]));
+    LIBCPP_ASSERT(c.bucket_count() == 7);
+    C::size_type b = c.bucket(0);
+    I i            = c.begin(b);
+    I j            = c.end(b);
+    assert(std::distance(i, j) == 0);
+
+    b = c.bucket(1);
+    i = c.begin(b);
+    j = c.end(b);
+    assert(std::distance(i, j) == 2);
+    assert(*i == 1);
+    ++i;
+    assert(*i == 1);
+
+    b = c.bucket(2);
+    i = c.begin(b);
+    j = c.end(b);
+    assert(std::distance(i, j) == 2);
+    assert(*i == 2);
+    ++i;
+    assert(*i == 2);
+
+    b = c.bucket(3);
+    i = c.begin(b);
+    j = c.end(b);
+    assert(std::distance(i, j) == 1);
+    assert(*i == 3);
+
+    b = c.bucket(4);
+    i = c.begin(b);
+    j = c.end(b);
+    assert(std::distance(i, j) == 1);
+    assert(*i == 4);
+
+    b = c.bucket(5);
+    i = c.begin(b);
+    j = c.end(b);
+    assert(std::distance(i, j) == 0);
+
+    b = c.bucket(6);
+    i = c.begin(b);
+    j = c.end(b);
+    assert(std::distance(i, j) == 0);
+  }
+  {
+    typedef std::unordered_multiset<int> C;
+    typedef int P;
+    typedef C::const_local_iterator I;
+    P a[] = {P(1), P(2), P(3), P(4), P(1), P(2)};
+    C c(a, a + sizeof(a) / sizeof(a[0]));
+    LIBCPP_ASSERT(c.bucket_count() == 7);
+    C::size_type b = c.bucket(0);
+    I i            = c.cbegin(b);
+    I j            = c.cend(b);
+    assert(std::distance(i, j) == 0);
+
+    b = c.bucket(1);
+    i = c.cbegin(b);
+    j = c.cend(b);
+    assert(std::distance(i, j) == 2);
+    assert(*i == 1);
+    ++i;
+    assert(*i == 1);
+
+    b = c.bucket(2);
+    i = c.cbegin(b);
+    j = c.cend(b);
+    assert(std::distance(i, j) == 2);
+    assert(*i == 2);
+    ++i;
+    assert(*i == 2);
+
+    b = c.bucket(3);
+    i = c.cbegin(b);
+    j = c.cend(b);
+    assert(std::distance(i, j) == 1);
+    assert(*i == 3);
+
+    b = c.bucket(4);
+    i = c.cbegin(b);
+    j = c.cend(b);
+    assert(std::distance(i, j) == 1);
+    assert(*i == 4);
+
+    b = c.bucket(5);
+    i = c.cbegin(b);
+    j = c.cend(b);
+    assert(std::distance(i, j) == 0);
+
+    b = c.bucket(6);
+    i = c.cbegin(b);
+    j = c.cend(b);
+    assert(std::distance(i, j) == 0);
+  }
+  {
+    typedef std::unordered_multiset<int> C;
+    typedef int P;
+    typedef C::const_local_iterator I;
+    P a[] = {P(1), P(2), P(3), P(4), P(1), P(2)};
+    const C c(a, a + sizeof(a) / sizeof(a[0]));
+    LIBCPP_ASSERT(c.bucket_count() == 7);
+    C::size_type b = c.bucket(0);
+    I i            = c.cbegin(b);
+    I j            = c.cend(b);
+    assert(std::distance(i, j) == 0);
+
+    b = c.bucket(1);
+    i = c.cbegin(b);
+    j = c.cend(b);
+    assert(std::distance(i, j) == 2);
+    assert(*i == 1);
+    ++i;
+    assert(*i == 1);
+
+    b = c.bucket(2);
+    i = c.cbegin(b);
+    j = c.cend(b);
+    assert(std::distance(i, j) == 2);
+    assert(*i == 2);
+    ++i;
+    assert(*i == 2);
+
+    b = c.bucket(3);
+    i = c.cbegin(b);
+    j = c.cend(b);
+    assert(std::distance(i, j) == 1);
+    assert(*i == 3);
+
+    b = c.bucket(4);
+    i = c.cbegin(b);
+    j = c.cend(b);
+    assert(std::distance(i, j) == 1);
+    assert(*i == 4);
+
+    b = c.bucket(5);
+    i = c.cbegin(b);
+    j = c.cend(b);
+    assert(std::distance(i, j) == 0);
+
+    b = c.bucket(6);
+    i = c.cbegin(b);
+    j = c.cend(b);
+    assert(std::distance(i, j) == 0);
+  }
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/unord/unord.multiset/local_iterators.pass.cpp b/libcxx/test/std/containers/unord/unord.multiset/local_iterators.pass.cpp
index 5d4053272d176..180b2deb56647 100644
--- a/libcxx/test/std/containers/unord/unord.multiset/local_iterators.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.multiset/local_iterators.pass.cpp
@@ -25,477 +25,408 @@
 #include "test_macros.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef std::unordered_multiset<int> C;
-        typedef int P;
-        typedef C::local_iterator I;
-        P a[] =
-        {
-            P(1),
-            P(2),
-            P(3),
-            P(4),
-            P(1),
-            P(2)
-        };
-        C c(a, a + sizeof(a)/sizeof(a[0]));
-        assert(c.bucket_count() >= 7);
-        C::size_type b = c.bucket(0);
-        I i = c.begin(b);
-        I j = c.end(b);
-        assert(std::distance(i, j) == 0);
-
-        b = c.bucket(1);
-        i = c.begin(b);
-        j = c.end(b);
-        assert(std::distance(i, j) == 2);
-        assert(*i == 1);
-        ++i;
-        assert(*i == 1);
-
-        b = c.bucket(2);
-        i = c.begin(b);
-        j = c.end(b);
-        assert(std::distance(i, j) == 2);
-        assert(*i == 2);
-        ++i;
-        assert(*i == 2);
-
-        b = c.bucket(3);
-        i = c.begin(b);
-        j = c.end(b);
-        assert(std::distance(i, j) == 1);
-        assert(*i == 3);
-
-        b = c.bucket(4);
-        i = c.begin(b);
-        j = c.end(b);
-        assert(std::distance(i, j) == 1);
-        assert(*i == 4);
-
-        b = c.bucket(5);
-        i = c.begin(b);
-        j = c.end(b);
-        assert(std::distance(i, j) == 0);
-
-        b = c.bucket(6);
-        i = c.begin(b);
-        j = c.end(b);
-        assert(std::distance(i, j) == 0);
-    }
-    {
-        typedef std::unordered_multiset<int> C;
-        typedef int P;
-        typedef C::const_local_iterator I;
-        P a[] =
-        {
-            P(1),
-            P(2),
-            P(3),
-            P(4),
-            P(1),
-            P(2)
-        };
-        const C c(a, a + sizeof(a)/sizeof(a[0]));
-        assert(c.bucket_count() >= 7);
-        C::size_type b = c.bucket(0);
-        I i = c.begin(b);
-        I j = c.end(b);
-        assert(std::distance(i, j) == 0);
-
-        b = c.bucket(1);
-        i = c.begin(b);
-        j = c.end(b);
-        assert(std::distance(i, j) == 2);
-        assert(*i == 1);
-        ++i;
-        assert(*i == 1);
-
-        b = c.bucket(2);
-        i = c.begin(b);
-        j = c.end(b);
-        assert(std::distance(i, j) == 2);
-        assert(*i == 2);
-        ++i;
-        assert(*i == 2);
-
-        b = c.bucket(3);
-        i = c.begin(b);
-        j = c.end(b);
-        assert(std::distance(i, j) == 1);
-        assert(*i == 3);
-
-        b = c.bucket(4);
-        i = c.begin(b);
-        j = c.end(b);
-        assert(std::distance(i, j) == 1);
-        assert(*i == 4);
-
-        b = c.bucket(5);
-        i = c.begin(b);
-        j = c.end(b);
-        assert(std::distance(i, j) == 0);
-
-        b = c.bucket(6);
-        i = c.begin(b);
-        j = c.end(b);
-        assert(std::distance(i, j) == 0);
-    }
-    {
-        typedef std::unordered_multiset<int> C;
-        typedef int P;
-        typedef C::const_local_iterator I;
-        P a[] =
-        {
-            P(1),
-            P(2),
-            P(3),
-            P(4),
-            P(1),
-            P(2)
-        };
-        C c(a, a + sizeof(a)/sizeof(a[0]));
-        assert(c.bucket_count() >= 7);
-        C::size_type b = c.bucket(0);
-        I i = c.cbegin(b);
-        I j = c.cend(b);
-        assert(std::distance(i, j) == 0);
-
-        b = c.bucket(1);
-        i = c.cbegin(b);
-        j = c.cend(b);
-        assert(std::distance(i, j) == 2);
-        assert(*i == 1);
-        ++i;
-        assert(*i == 1);
-
-        b = c.bucket(2);
-        i = c.cbegin(b);
-        j = c.cend(b);
-        assert(std::distance(i, j) == 2);
-        assert(*i == 2);
-        ++i;
-        assert(*i == 2);
-
-        b = c.bucket(3);
-        i = c.cbegin(b);
-        j = c.cend(b);
-        assert(std::distance(i, j) == 1);
-        assert(*i == 3);
-
-        b = c.bucket(4);
-        i = c.cbegin(b);
-        j = c.cend(b);
-        assert(std::distance(i, j) == 1);
-        assert(*i == 4);
-
-        b = c.bucket(5);
-        i = c.cbegin(b);
-        j = c.cend(b);
-        assert(std::distance(i, j) == 0);
-
-        b = c.bucket(6);
-        i = c.cbegin(b);
-        j = c.cend(b);
-        assert(std::distance(i, j) == 0);
-    }
-    {
-        typedef std::unordered_multiset<int> C;
-        typedef int P;
-        typedef C::const_local_iterator I;
-        P a[] =
-        {
-            P(1),
-            P(2),
-            P(3),
-            P(4),
-            P(1),
-            P(2)
-        };
-        const C c(a, a + sizeof(a)/sizeof(a[0]));
-        assert(c.bucket_count() >= 7);
-        C::size_type b = c.bucket(0);
-        I i = c.cbegin(b);
-        I j = c.cend(b);
-        assert(std::distance(i, j) == 0);
-
-        b = c.bucket(1);
-        i = c.cbegin(b);
-        j = c.cend(b);
-        assert(std::distance(i, j) == 2);
-        assert(*i == 1);
-        ++i;
-        assert(*i == 1);
-
-        b = c.bucket(2);
-        i = c.cbegin(b);
-        j = c.cend(b);
-        assert(std::distance(i, j) == 2);
-        assert(*i == 2);
-        ++i;
-        assert(*i == 2);
-
-        b = c.bucket(3);
-        i = c.cbegin(b);
-        j = c.cend(b);
-        assert(std::distance(i, j) == 1);
-        assert(*i == 3);
-
-        b = c.bucket(4);
-        i = c.cbegin(b);
-        j = c.cend(b);
-        assert(std::distance(i, j) == 1);
-        assert(*i == 4);
-
-        b = c.bucket(5);
-        i = c.cbegin(b);
-        j = c.cend(b);
-        assert(std::distance(i, j) == 0);
-
-        b = c.bucket(6);
-        i = c.cbegin(b);
-        j = c.cend(b);
-        assert(std::distance(i, j) == 0);
-    }
+int main(int, char**) {
+  {
+    typedef std::unordered_multiset<int> C;
+    typedef int P;
+    typedef C::local_iterator I;
+    P a[] = {P(1), P(2), P(3), P(4), P(1), P(2)};
+    C c(a, a + sizeof(a) / sizeof(a[0]));
+    assert(c.bucket_count() >= 7);
+    C::size_type b = c.bucket(0);
+    I i            = c.begin(b);
+    I j            = c.end(b);
+    assert(std::distance(i, j) == 0);
+
+    b = c.bucket(1);
+    i = c.begin(b);
+    j = c.end(b);
+    assert(std::distance(i, j) == 2);
+    assert(*i == 1);
+    ++i;
+    assert(*i == 1);
+
+    b = c.bucket(2);
+    i = c.begin(b);
+    j = c.end(b);
+    assert(std::distance(i, j) == 2);
+    assert(*i == 2);
+    ++i;
+    assert(*i == 2);
+
+    b = c.bucket(3);
+    i = c.begin(b);
+    j = c.end(b);
+    assert(std::distance(i, j) == 1);
+    assert(*i == 3);
+
+    b = c.bucket(4);
+    i = c.begin(b);
+    j = c.end(b);
+    assert(std::distance(i, j) == 1);
+    assert(*i == 4);
+
+    b = c.bucket(5);
+    i = c.begin(b);
+    j = c.end(b);
+    assert(std::distance(i, j) == 0);
+
+    b = c.bucket(6);
+    i = c.begin(b);
+    j = c.end(b);
+    assert(std::distance(i, j) == 0);
+  }
+  {
+    typedef std::unordered_multiset<int> C;
+    typedef int P;
+    typedef C::const_local_iterator I;
+    P a[] = {P(1), P(2), P(3), P(4), P(1), P(2)};
+    const C c(a, a + sizeof(a) / sizeof(a[0]));
+    assert(c.bucket_count() >= 7);
+    C::size_type b = c.bucket(0);
+    I i            = c.begin(b);
+    I j            = c.end(b);
+    assert(std::distance(i, j) == 0);
+
+    b = c.bucket(1);
+    i = c.begin(b);
+    j = c.end(b);
+    assert(std::distance(i, j) == 2);
+    assert(*i == 1);
+    ++i;
+    assert(*i == 1);
+
+    b = c.bucket(2);
+    i = c.begin(b);
+    j = c.end(b);
+    assert(std::distance(i, j) == 2);
+    assert(*i == 2);
+    ++i;
+    assert(*i == 2);
+
+    b = c.bucket(3);
+    i = c.begin(b);
+    j = c.end(b);
+    assert(std::distance(i, j) == 1);
+    assert(*i == 3);
+
+    b = c.bucket(4);
+    i = c.begin(b);
+    j = c.end(b);
+    assert(std::distance(i, j) == 1);
+    assert(*i == 4);
+
+    b = c.bucket(5);
+    i = c.begin(b);
+    j = c.end(b);
+    assert(std::distance(i, j) == 0);
+
+    b = c.bucket(6);
+    i = c.begin(b);
+    j = c.end(b);
+    assert(std::distance(i, j) == 0);
+  }
+  {
+    typedef std::unordered_multiset<int> C;
+    typedef int P;
+    typedef C::const_local_iterator I;
+    P a[] = {P(1), P(2), P(3), P(4), P(1), P(2)};
+    C c(a, a + sizeof(a) / sizeof(a[0]));
+    assert(c.bucket_count() >= 7);
+    C::size_type b = c.bucket(0);
+    I i            = c.cbegin(b);
+    I j            = c.cend(b);
+    assert(std::distance(i, j) == 0);
+
+    b = c.bucket(1);
+    i = c.cbegin(b);
+    j = c.cend(b);
+    assert(std::distance(i, j) == 2);
+    assert(*i == 1);
+    ++i;
+    assert(*i == 1);
+
+    b = c.bucket(2);
+    i = c.cbegin(b);
+    j = c.cend(b);
+    assert(std::distance(i, j) == 2);
+    assert(*i == 2);
+    ++i;
+    assert(*i == 2);
+
+    b = c.bucket(3);
+    i = c.cbegin(b);
+    j = c.cend(b);
+    assert(std::distance(i, j) == 1);
+    assert(*i == 3);
+
+    b = c.bucket(4);
+    i = c.cbegin(b);
+    j = c.cend(b);
+    assert(std::distance(i, j) == 1);
+    assert(*i == 4);
+
+    b = c.bucket(5);
+    i = c.cbegin(b);
+    j = c.cend(b);
+    assert(std::distance(i, j) == 0);
+
+    b = c.bucket(6);
+    i = c.cbegin(b);
+    j = c.cend(b);
+    assert(std::distance(i, j) == 0);
+  }
+  {
+    typedef std::unordered_multiset<int> C;
+    typedef int P;
+    typedef C::const_local_iterator I;
+    P a[] = {P(1), P(2), P(3), P(4), P(1), P(2)};
+    const C c(a, a + sizeof(a) / sizeof(a[0]));
+    assert(c.bucket_count() >= 7);
+    C::size_type b = c.bucket(0);
+    I i            = c.cbegin(b);
+    I j            = c.cend(b);
+    assert(std::distance(i, j) == 0);
+
+    b = c.bucket(1);
+    i = c.cbegin(b);
+    j = c.cend(b);
+    assert(std::distance(i, j) == 2);
+    assert(*i == 1);
+    ++i;
+    assert(*i == 1);
+
+    b = c.bucket(2);
+    i = c.cbegin(b);
+    j = c.cend(b);
+    assert(std::distance(i, j) == 2);
+    assert(*i == 2);
+    ++i;
+    assert(*i == 2);
+
+    b = c.bucket(3);
+    i = c.cbegin(b);
+    j = c.cend(b);
+    assert(std::distance(i, j) == 1);
+    assert(*i == 3);
+
+    b = c.bucket(4);
+    i = c.cbegin(b);
+    j = c.cend(b);
+    assert(std::distance(i, j) == 1);
+    assert(*i == 4);
+
+    b = c.bucket(5);
+    i = c.cbegin(b);
+    j = c.cend(b);
+    assert(std::distance(i, j) == 0);
+
+    b = c.bucket(6);
+    i = c.cbegin(b);
+    j = c.cend(b);
+    assert(std::distance(i, j) == 0);
+  }
 #if TEST_STD_VER >= 11
-    {
-        typedef std::unordered_multiset<int, std::hash<int>,
-                                      std::equal_to<int>, min_allocator<int>> C;
-        typedef int P;
-        typedef C::local_iterator I;
-        P a[] =
-        {
-            P(1),
-            P(2),
-            P(3),
-            P(4),
-            P(1),
-            P(2)
-        };
-        C c(a, a + sizeof(a)/sizeof(a[0]));
-        assert(c.bucket_count() >= 7);
-        C::size_type b = c.bucket(0);
-        I i = c.begin(b);
-        I j = c.end(b);
-        assert(std::distance(i, j) == 0);
-
-        b = c.bucket(1);
-        i = c.begin(b);
-        j = c.end(b);
-        assert(std::distance(i, j) == 2);
-        assert(*i == 1);
-        ++i;
-        assert(*i == 1);
-
-        b = c.bucket(2);
-        i = c.begin(b);
-        j = c.end(b);
-        assert(std::distance(i, j) == 2);
-        assert(*i == 2);
-        ++i;
-        assert(*i == 2);
-
-        b = c.bucket(3);
-        i = c.begin(b);
-        j = c.end(b);
-        assert(std::distance(i, j) == 1);
-        assert(*i == 3);
-
-        b = c.bucket(4);
-        i = c.begin(b);
-        j = c.end(b);
-        assert(std::distance(i, j) == 1);
-        assert(*i == 4);
-
-        b = c.bucket(5);
-        i = c.begin(b);
-        j = c.end(b);
-        assert(std::distance(i, j) == 0);
-
-        b = c.bucket(6);
-        i = c.begin(b);
-        j = c.end(b);
-        assert(std::distance(i, j) == 0);
-    }
-    {
-        typedef std::unordered_multiset<int, std::hash<int>,
-                                      std::equal_to<int>, min_allocator<int>> C;
-        typedef int P;
-        typedef C::const_local_iterator I;
-        P a[] =
-        {
-            P(1),
-            P(2),
-            P(3),
-            P(4),
-            P(1),
-            P(2)
-        };
-        const C c(a, a + sizeof(a)/sizeof(a[0]));
-        assert(c.bucket_count() >= 7);
-        C::size_type b = c.bucket(0);
-        I i = c.begin(b);
-        I j = c.end(b);
-        assert(std::distance(i, j) == 0);
-
-        b = c.bucket(1);
-        i = c.begin(b);
-        j = c.end(b);
-        assert(std::distance(i, j) == 2);
-        assert(*i == 1);
-        ++i;
-        assert(*i == 1);
-
-        b = c.bucket(2);
-        i = c.begin(b);
-        j = c.end(b);
-        assert(std::distance(i, j) == 2);
-        assert(*i == 2);
-        ++i;
-        assert(*i == 2);
-
-        b = c.bucket(3);
-        i = c.begin(b);
-        j = c.end(b);
-        assert(std::distance(i, j) == 1);
-        assert(*i == 3);
-
-        b = c.bucket(4);
-        i = c.begin(b);
-        j = c.end(b);
-        assert(std::distance(i, j) == 1);
-        assert(*i == 4);
-
-        b = c.bucket(5);
-        i = c.begin(b);
-        j = c.end(b);
-        assert(std::distance(i, j) == 0);
-
-        b = c.bucket(6);
-        i = c.begin(b);
-        j = c.end(b);
-        assert(std::distance(i, j) == 0);
-    }
-    {
-        typedef std::unordered_multiset<int, std::hash<int>,
-                                      std::equal_to<int>, min_allocator<int>> C;
-        typedef int P;
-        typedef C::const_local_iterator I;
-        P a[] =
-        {
-            P(1),
-            P(2),
-            P(3),
-            P(4),
-            P(1),
-            P(2)
-        };
-        C c(a, a + sizeof(a)/sizeof(a[0]));
-        assert(c.bucket_count() >= 7);
-        C::size_type b = c.bucket(0);
-        I i = c.cbegin(b);
-        I j = c.cend(b);
-        assert(std::distance(i, j) == 0);
-
-        b = c.bucket(1);
-        i = c.cbegin(b);
-        j = c.cend(b);
-        assert(std::distance(i, j) == 2);
-        assert(*i == 1);
-        ++i;
-        assert(*i == 1);
-
-        b = c.bucket(2);
-        i = c.cbegin(b);
-        j = c.cend(b);
-        assert(std::distance(i, j) == 2);
-        assert(*i == 2);
-        ++i;
-        assert(*i == 2);
-
-        b = c.bucket(3);
-        i = c.cbegin(b);
-        j = c.cend(b);
-        assert(std::distance(i, j) == 1);
-        assert(*i == 3);
-
-        b = c.bucket(4);
-        i = c.cbegin(b);
-        j = c.cend(b);
-        assert(std::distance(i, j) == 1);
-        assert(*i == 4);
-
-        b = c.bucket(5);
-        i = c.cbegin(b);
-        j = c.cend(b);
-        assert(std::distance(i, j) == 0);
-
-        b = c.bucket(6);
-        i = c.cbegin(b);
-        j = c.cend(b);
-        assert(std::distance(i, j) == 0);
-    }
-    {
-        typedef std::unordered_multiset<int, std::hash<int>,
-                                      std::equal_to<int>, min_allocator<int>> C;
-        typedef int P;
-        typedef C::const_local_iterator I;
-        P a[] =
-        {
-            P(1),
-            P(2),
-            P(3),
-            P(4),
-            P(1),
-            P(2)
-        };
-        const C c(a, a + sizeof(a)/sizeof(a[0]));
-        assert(c.bucket_count() >= 7);
-        C::size_type b = c.bucket(0);
-        I i = c.cbegin(b);
-        I j = c.cend(b);
-        assert(std::distance(i, j) == 0);
-
-        b = c.bucket(1);
-        i = c.cbegin(b);
-        j = c.cend(b);
-        assert(std::distance(i, j) == 2);
-        assert(*i == 1);
-        ++i;
-        assert(*i == 1);
-
-        b = c.bucket(2);
-        i = c.cbegin(b);
-        j = c.cend(b);
-        assert(std::distance(i, j) == 2);
-        assert(*i == 2);
-        ++i;
-        assert(*i == 2);
-
-        b = c.bucket(3);
-        i = c.cbegin(b);
-        j = c.cend(b);
-        assert(std::distance(i, j) == 1);
-        assert(*i == 3);
-
-        b = c.bucket(4);
-        i = c.cbegin(b);
-        j = c.cend(b);
-        assert(std::distance(i, j) == 1);
-        assert(*i == 4);
-
-        b = c.bucket(5);
-        i = c.cbegin(b);
-        j = c.cend(b);
-        assert(std::distance(i, j) == 0);
-
-        b = c.bucket(6);
-        i = c.cbegin(b);
-        j = c.cend(b);
-        assert(std::distance(i, j) == 0);
-    }
+  {
+    typedef std::unordered_multiset<int, std::hash<int>, std::equal_to<int>, min_allocator<int>> C;
+    typedef int P;
+    typedef C::local_iterator I;
+    P a[] = {P(1), P(2), P(3), P(4), P(1), P(2)};
+    C c(a, a + sizeof(a) / sizeof(a[0]));
+    assert(c.bucket_count() >= 7);
+    C::size_type b = c.bucket(0);
+    I i            = c.begin(b);
+    I j            = c.end(b);
+    assert(std::distance(i, j) == 0);
+
+    b = c.bucket(1);
+    i = c.begin(b);
+    j = c.end(b);
+    assert(std::distance(i, j) == 2);
+    assert(*i == 1);
+    ++i;
+    assert(*i == 1);
+
+    b = c.bucket(2);
+    i = c.begin(b);
+    j = c.end(b);
+    assert(std::distance(i, j) == 2);
+    assert(*i == 2);
+    ++i;
+    assert(*i == 2);
+
+    b = c.bucket(3);
+    i = c.begin(b);
+    j = c.end(b);
+    assert(std::distance(i, j) == 1);
+    assert(*i == 3);
+
+    b = c.bucket(4);
+    i = c.begin(b);
+    j = c.end(b);
+    assert(std::distance(i, j) == 1);
+    assert(*i == 4);
+
+    b = c.bucket(5);
+    i = c.begin(b);
+    j = c.end(b);
+    assert(std::distance(i, j) == 0);
+
+    b = c.bucket(6);
+    i = c.begin(b);
+    j = c.end(b);
+    assert(std::distance(i, j) == 0);
+  }
+  {
+    typedef std::unordered_multiset<int, std::hash<int>, std::equal_to<int>, min_allocator<int>> C;
+    typedef int P;
+    typedef C::const_local_iterator I;
+    P a[] = {P(1), P(2), P(3), P(4), P(1), P(2)};
+    const C c(a, a + sizeof(a) / sizeof(a[0]));
+    assert(c.bucket_count() >= 7);
+    C::size_type b = c.bucket(0);
+    I i            = c.begin(b);
+    I j            = c.end(b);
+    assert(std::distance(i, j) == 0);
+
+    b = c.bucket(1);
+    i = c.begin(b);
+    j = c.end(b);
+    assert(std::distance(i, j) == 2);
+    assert(*i == 1);
+    ++i;
+    assert(*i == 1);
+
+    b = c.bucket(2);
+    i = c.begin(b);
+    j = c.end(b);
+    assert(std::distance(i, j) == 2);
+    assert(*i == 2);
+    ++i;
+    assert(*i == 2);
+
+    b = c.bucket(3);
+    i = c.begin(b);
+    j = c.end(b);
+    assert(std::distance(i, j) == 1);
+    assert(*i == 3);
+
+    b = c.bucket(4);
+    i = c.begin(b);
+    j = c.end(b);
+    assert(std::distance(i, j) == 1);
+    assert(*i == 4);
+
+    b = c.bucket(5);
+    i = c.begin(b);
+    j = c.end(b);
+    assert(std::distance(i, j) == 0);
+
+    b = c.bucket(6);
+    i = c.begin(b);
+    j = c.end(b);
+    assert(std::distance(i, j) == 0);
+  }
+  {
+    typedef std::unordered_multiset<int, std::hash<int>, std::equal_to<int>, min_allocator<int>> C;
+    typedef int P;
+    typedef C::const_local_iterator I;
+    P a[] = {P(1), P(2), P(3), P(4), P(1), P(2)};
+    C c(a, a + sizeof(a) / sizeof(a[0]));
+    assert(c.bucket_count() >= 7);
+    C::size_type b = c.bucket(0);
+    I i            = c.cbegin(b);
+    I j            = c.cend(b);
+    assert(std::distance(i, j) == 0);
+
+    b = c.bucket(1);
+    i = c.cbegin(b);
+    j = c.cend(b);
+    assert(std::distance(i, j) == 2);
+    assert(*i == 1);
+    ++i;
+    assert(*i == 1);
+
+    b = c.bucket(2);
+    i = c.cbegin(b);
+    j = c.cend(b);
+    assert(std::distance(i, j) == 2);
+    assert(*i == 2);
+    ++i;
+    assert(*i == 2);
+
+    b = c.bucket(3);
+    i = c.cbegin(b);
+    j = c.cend(b);
+    assert(std::distance(i, j) == 1);
+    assert(*i == 3);
+
+    b = c.bucket(4);
+    i = c.cbegin(b);
+    j = c.cend(b);
+    assert(std::distance(i, j) == 1);
+    assert(*i == 4);
+
+    b = c.bucket(5);
+    i = c.cbegin(b);
+    j = c.cend(b);
+    assert(std::distance(i, j) == 0);
+
+    b = c.bucket(6);
+    i = c.cbegin(b);
+    j = c.cend(b);
+    assert(std::distance(i, j) == 0);
+  }
+  {
+    typedef std::unordered_multiset<int, std::hash<int>, std::equal_to<int>, min_allocator<int>> C;
+    typedef int P;
+    typedef C::const_local_iterator I;
+    P a[] = {P(1), P(2), P(3), P(4), P(1), P(2)};
+    const C c(a, a + sizeof(a) / sizeof(a[0]));
+    assert(c.bucket_count() >= 7);
+    C::size_type b = c.bucket(0);
+    I i            = c.cbegin(b);
+    I j            = c.cend(b);
+    assert(std::distance(i, j) == 0);
+
+    b = c.bucket(1);
+    i = c.cbegin(b);
+    j = c.cend(b);
+    assert(std::distance(i, j) == 2);
+    assert(*i == 1);
+    ++i;
+    assert(*i == 1);
+
+    b = c.bucket(2);
+    i = c.cbegin(b);
+    j = c.cend(b);
+    assert(std::distance(i, j) == 2);
+    assert(*i == 2);
+    ++i;
+    assert(*i == 2);
+
+    b = c.bucket(3);
+    i = c.cbegin(b);
+    j = c.cend(b);
+    assert(std::distance(i, j) == 1);
+    assert(*i == 3);
+
+    b = c.bucket(4);
+    i = c.cbegin(b);
+    j = c.cend(b);
+    assert(std::distance(i, j) == 1);
+    assert(*i == 4);
+
+    b = c.bucket(5);
+    i = c.cbegin(b);
+    j = c.cend(b);
+    assert(std::distance(i, j) == 0);
+
+    b = c.bucket(6);
+    i = c.cbegin(b);
+    j = c.cend(b);
+    assert(std::distance(i, j) == 0);
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/unord/unord.multiset/max_bucket_count.pass.cpp b/libcxx/test/std/containers/unord/unord.multiset/max_bucket_count.pass.cpp
index 675101b4d33af..fd70bd9c247b5 100644
--- a/libcxx/test/std/containers/unord/unord.multiset/max_bucket_count.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.multiset/max_bucket_count.pass.cpp
@@ -20,20 +20,18 @@
 #include "test_macros.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef std::unordered_multiset<int> C;
-        const C c;
-        assert(c.max_bucket_count() > 0);
-    }
+int main(int, char**) {
+  {
+    typedef std::unordered_multiset<int> C;
+    const C c;
+    assert(c.max_bucket_count() > 0);
+  }
 #if TEST_STD_VER >= 11
-    {
-        typedef std::unordered_multiset<int, std::hash<int>,
-                                      std::equal_to<int>, min_allocator<int>> C;
-        const C c;
-        assert(c.max_bucket_count() > 0);
-    }
+  {
+    typedef std::unordered_multiset<int, std::hash<int>, std::equal_to<int>, min_allocator<int>> C;
+    const C c;
+    assert(c.max_bucket_count() > 0);
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/unord/unord.multiset/max_load_factor.pass.cpp b/libcxx/test/std/containers/unord/unord.multiset/max_load_factor.pass.cpp
index 4d6738ba4e8d1..1391df7d9f52a 100644
--- a/libcxx/test/std/containers/unord/unord.multiset/max_load_factor.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.multiset/max_load_factor.pass.cpp
@@ -21,36 +21,33 @@
 #include "test_macros.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef std::unordered_multiset<int> C;
-        const C c;
-        assert(c.max_load_factor() == 1);
-    }
-    {
-        typedef std::unordered_multiset<int> C;
-        C c;
-        assert(c.max_load_factor() == 1);
-        c.max_load_factor(2.5);
-        assert(c.max_load_factor() == 2.5);
-    }
+int main(int, char**) {
+  {
+    typedef std::unordered_multiset<int> C;
+    const C c;
+    assert(c.max_load_factor() == 1);
+  }
+  {
+    typedef std::unordered_multiset<int> C;
+    C c;
+    assert(c.max_load_factor() == 1);
+    c.max_load_factor(2.5);
+    assert(c.max_load_factor() == 2.5);
+  }
 #if TEST_STD_VER >= 11
-    {
-        typedef std::unordered_multiset<int, std::hash<int>,
-                                      std::equal_to<int>, min_allocator<int>> C;
-        const C c;
-        assert(c.max_load_factor() == 1);
-    }
-    {
-        typedef std::unordered_multiset<int, std::hash<int>,
-                                      std::equal_to<int>, min_allocator<int>> C;
-        C c;
-        assert(c.max_load_factor() == 1);
-        c.max_load_factor(2.5);
-        assert(c.max_load_factor() == 2.5);
-    }
+  {
+    typedef std::unordered_multiset<int, std::hash<int>, std::equal_to<int>, min_allocator<int>> C;
+    const C c;
+    assert(c.max_load_factor() == 1);
+  }
+  {
+    typedef std::unordered_multiset<int, std::hash<int>, std::equal_to<int>, min_allocator<int>> C;
+    C c;
+    assert(c.max_load_factor() == 1);
+    c.max_load_factor(2.5);
+    assert(c.max_load_factor() == 2.5);
+  }
 #endif
 
-    return 0;
+  return 0;
 }
diff --git a/libcxx/test/std/containers/unord/unord.multiset/max_size.pass.cpp b/libcxx/test/std/containers/unord/unord.multiset/max_size.pass.cpp
index 47b165f230007..745dca9eb95a0 100644
--- a/libcxx/test/std/containers/unord/unord.multiset/max_size.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.multiset/max_size.pass.cpp
@@ -20,36 +20,29 @@
 #include "test_allocator.h"
 #include "test_macros.h"
 
-int main(int, char**)
-{
-    {
-      typedef limited_allocator<int, 10> A;
-      typedef std::unordered_multiset<int, std::hash<int>, std::equal_to<int>,
-                                      A>
-          C;
-      C c;
-      assert(c.max_size() <= 10);
-      LIBCPP_ASSERT(c.max_size() == 10);
-    }
-    {
-      typedef limited_allocator<int, (std::size_t)-1> A;
-      typedef std::unordered_multiset<int, std::hash<int>, std::equal_to<int>,
-                                      A>
-          C;
-      const C::size_type max_dist =
-          static_cast<C::size_type>(std::numeric_limits<C::difference_type>::max());
-      C c;
-      assert(c.max_size() <= max_dist);
-      LIBCPP_ASSERT(c.max_size() == max_dist);
-    }
-    {
-      typedef std::unordered_multiset<char> C;
-      const C::size_type max_dist =
-          static_cast<C::size_type>(std::numeric_limits<C::difference_type>::max());
-      C c;
-      assert(c.max_size() <= max_dist);
-      assert(c.max_size() <= alloc_max_size(c.get_allocator()));
-    }
+int main(int, char**) {
+  {
+    typedef limited_allocator<int, 10> A;
+    typedef std::unordered_multiset<int, std::hash<int>, std::equal_to<int>, A> C;
+    C c;
+    assert(c.max_size() <= 10);
+    LIBCPP_ASSERT(c.max_size() == 10);
+  }
+  {
+    typedef limited_allocator<int, (std::size_t)-1> A;
+    typedef std::unordered_multiset<int, std::hash<int>, std::equal_to<int>, A> C;
+    const C::size_type max_dist = static_cast<C::size_type>(std::numeric_limits<C::difference_type>::max());
+    C c;
+    assert(c.max_size() <= max_dist);
+    LIBCPP_ASSERT(c.max_size() == max_dist);
+  }
+  {
+    typedef std::unordered_multiset<char> C;
+    const C::size_type max_dist = static_cast<C::size_type>(std::numeric_limits<C::difference_type>::max());
+    C c;
+    assert(c.max_size() <= max_dist);
+    assert(c.max_size() <= alloc_max_size(c.get_allocator()));
+  }
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/unord/unord.multiset/merge.pass.cpp b/libcxx/test/std/containers/unord/unord.multiset/merge.pass.cpp
index 00435d8df6a06..ce69111f9544f 100644
--- a/libcxx/test/std/containers/unord/unord.multiset/merge.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.multiset/merge.pass.cpp
@@ -27,128 +27,116 @@
 #include "Counter.h"
 
 template <class Set>
-bool set_equal(const Set& set, Set other)
-{
-    return set == other;
+bool set_equal(const Set& set, Set other) {
+  return set == other;
 }
 
 #ifndef TEST_HAS_NO_EXCEPTIONS
 template <class T>
-struct throw_hasher
-{
-    bool& should_throw_;
+struct throw_hasher {
+  bool& should_throw_;
 
-    throw_hasher(bool& should_throw) : should_throw_(should_throw) {}
+  throw_hasher(bool& should_throw) : should_throw_(should_throw) {}
 
-    std::size_t operator()(const T& p) const
-    {
-        if (should_throw_)
-            throw 0;
-        return std::hash<T>()(p);
-    }
+  std::size_t operator()(const T& p) const {
+    if (should_throw_)
+      throw 0;
+    return std::hash<T>()(p);
+  }
 };
 #endif
 
-int main(int, char**)
-{
-    {
-        std::unordered_multiset<int> src{1, 3, 5};
-        std::unordered_multiset<int> dst{2, 4, 5};
-        dst.merge(src);
-        assert(set_equal(src, {}));
-        assert(set_equal(dst, {1, 2, 3, 4, 5, 5}));
-    }
+int main(int, char**) {
+  {
+    std::unordered_multiset<int> src{1, 3, 5};
+    std::unordered_multiset<int> dst{2, 4, 5};
+    dst.merge(src);
+    assert(set_equal(src, {}));
+    assert(set_equal(dst, {1, 2, 3, 4, 5, 5}));
+  }
 
 #ifndef TEST_HAS_NO_EXCEPTIONS
-    {
-        bool do_throw = false;
-        typedef std::unordered_multiset<Counter<int>, throw_hasher<Counter<int>>> set_type;
-        set_type src({1, 3, 5}, 0, throw_hasher<Counter<int>>(do_throw));
-        set_type dst({2, 4, 5}, 0, throw_hasher<Counter<int>>(do_throw));
-
-        assert(Counter_base::gConstructed == 6);
-
-        do_throw = true;
-        try
-        {
-            dst.merge(src);
-        }
-        catch (int)
-        {
-            do_throw = false;
-        }
-        assert(!do_throw);
-        assert(set_equal(src, set_type({1, 3, 5}, 0, throw_hasher<Counter<int>>(do_throw))));
-        assert(set_equal(dst, set_type({2, 4, 5}, 0, throw_hasher<Counter<int>>(do_throw))));
+  {
+    bool do_throw = false;
+    typedef std::unordered_multiset<Counter<int>, throw_hasher<Counter<int>>> set_type;
+    set_type src({1, 3, 5}, 0, throw_hasher<Counter<int>>(do_throw));
+    set_type dst({2, 4, 5}, 0, throw_hasher<Counter<int>>(do_throw));
+
+    assert(Counter_base::gConstructed == 6);
+
+    do_throw = true;
+    try {
+      dst.merge(src);
+    } catch (int) {
+      do_throw = false;
     }
+    assert(!do_throw);
+    assert(set_equal(src, set_type({1, 3, 5}, 0, throw_hasher<Counter<int>>(do_throw))));
+    assert(set_equal(dst, set_type({2, 4, 5}, 0, throw_hasher<Counter<int>>(do_throw))));
+  }
 #endif
-    assert(Counter_base::gConstructed == 0);
-    struct equal
-    {
-        equal() = default;
-
-        bool operator()(const Counter<int>& lhs, const Counter<int>& rhs) const
-        {
-            return lhs == rhs;
-        }
-    };
-    struct hasher
-    {
-        hasher() = default;
-        std::size_t operator()(const Counter<int>& p) const { return std::hash<Counter<int>>()(p); }
-    };
-    {
-        typedef std::unordered_multiset<Counter<int>, std::hash<Counter<int>>, std::equal_to<Counter<int>>> first_set_type;
-        typedef std::unordered_multiset<Counter<int>, hasher, equal> second_set_type;
-        typedef std::unordered_set<Counter<int>, hasher, equal> third_set_type;
+  assert(Counter_base::gConstructed == 0);
+  struct equal {
+    equal() = default;
+
+    bool operator()(const Counter<int>& lhs, const Counter<int>& rhs) const { return lhs == rhs; }
+  };
+  struct hasher {
+    hasher() = default;
+    std::size_t operator()(const Counter<int>& p) const { return std::hash<Counter<int>>()(p); }
+  };
+  {
+    typedef std::unordered_multiset<Counter<int>, std::hash<Counter<int>>, std::equal_to<Counter<int>>> first_set_type;
+    typedef std::unordered_multiset<Counter<int>, hasher, equal> second_set_type;
+    typedef std::unordered_set<Counter<int>, hasher, equal> third_set_type;
 
-        {
-            first_set_type first{1, 2, 3};
-            second_set_type second{2, 3, 4};
-            third_set_type third{1, 3};
+    {
+      first_set_type first{1, 2, 3};
+      second_set_type second{2, 3, 4};
+      third_set_type third{1, 3};
 
-            assert(Counter_base::gConstructed == 8);
+      assert(Counter_base::gConstructed == 8);
 
-            first.merge(second);
-            first.merge(third);
+      first.merge(second);
+      first.merge(third);
 
-            assert(set_equal(first, {1, 2, 3, 4, 2, 3, 1, 3}));
-            assert(set_equal(second, {}));
-            assert(set_equal(third, {}));
+      assert(set_equal(first, {1, 2, 3, 4, 2, 3, 1, 3}));
+      assert(set_equal(second, {}));
+      assert(set_equal(third, {}));
 
-            assert(Counter_base::gConstructed == 8);
-        }
-        assert(Counter_base::gConstructed == 0);
-        {
-            first_set_type first{1, 2, 3};
-            second_set_type second{2, 3, 4};
-            third_set_type third{1, 3};
+      assert(Counter_base::gConstructed == 8);
+    }
+    assert(Counter_base::gConstructed == 0);
+    {
+      first_set_type first{1, 2, 3};
+      second_set_type second{2, 3, 4};
+      third_set_type third{1, 3};
 
-            assert(Counter_base::gConstructed == 8);
+      assert(Counter_base::gConstructed == 8);
 
-            first.merge(std::move(second));
-            first.merge(std::move(third));
+      first.merge(std::move(second));
+      first.merge(std::move(third));
 
-            assert(set_equal(first, {1, 2, 3, 4, 2, 3, 1, 3}));
-            assert(set_equal(second, {}));
-            assert(set_equal(third, {}));
+      assert(set_equal(first, {1, 2, 3, 4, 2, 3, 1, 3}));
+      assert(set_equal(second, {}));
+      assert(set_equal(third, {}));
 
-            assert(Counter_base::gConstructed == 8);
-        }
-        assert(Counter_base::gConstructed == 0);
+      assert(Counter_base::gConstructed == 8);
+    }
+    assert(Counter_base::gConstructed == 0);
+  }
+  {
+    std::unordered_multiset<int> first;
+    {
+      std::unordered_multiset<int> second;
+      first.merge(second);
+      first.merge(std::move(second));
     }
     {
-        std::unordered_multiset<int> first;
-        {
-            std::unordered_multiset<int> second;
-            first.merge(second);
-            first.merge(std::move(second));
-        }
-        {
-            std::unordered_set<int> second;
-            first.merge(second);
-            first.merge(std::move(second));
-        }
+      std::unordered_set<int> second;
+      first.merge(second);
+      first.merge(std::move(second));
     }
-    return 0;
+  }
+  return 0;
 }
diff --git a/libcxx/test/std/containers/unord/unord.multiset/range_concept_conformance.compile.pass.cpp b/libcxx/test/std/containers/unord/unord.multiset/range_concept_conformance.compile.pass.cpp
index 1a87404c835e1..1cb91e9e6b83c 100644
--- a/libcxx/test/std/containers/unord/unord.multiset/range_concept_conformance.compile.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.multiset/range_concept_conformance.compile.pass.cpp
@@ -19,7 +19,6 @@
 
 using range = std::unordered_multiset<int>;
 
-
 static_assert(std::same_as<std::ranges::iterator_t<range>, range::iterator>);
 static_assert(std::ranges::common_range<range>);
 static_assert(std::ranges::forward_range<range>);
diff --git a/libcxx/test/std/containers/unord/unord.multiset/rehash.pass.cpp b/libcxx/test/std/containers/unord/unord.multiset/rehash.pass.cpp
index 1ef3bdcb99d0b..e6012eceae10b 100644
--- a/libcxx/test/std/containers/unord/unord.multiset/rehash.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.multiset/rehash.pass.cpp
@@ -21,83 +21,63 @@
 #include "min_allocator.h"
 
 template <class C>
-void rehash_postcondition(const C& c, std::size_t n)
-{
-    assert(c.bucket_count() >= c.size() / c.max_load_factor() && c.bucket_count() >= n);
+void rehash_postcondition(const C& c, std::size_t n) {
+  assert(c.bucket_count() >= c.size() / c.max_load_factor() && c.bucket_count() >= n);
 }
 
 template <class C>
-void test(const C& c)
-{
-    assert(c.size() == 6);
-    assert(c.count(1) == 2);
-    assert(c.count(2) == 2);
-    assert(c.count(3) == 1);
-    assert(c.count(4) == 1);
+void test(const C& c) {
+  assert(c.size() == 6);
+  assert(c.count(1) == 2);
+  assert(c.count(2) == 2);
+  assert(c.count(3) == 1);
+  assert(c.count(4) == 1);
 }
 
-int main(int, char**)
-{
-    {
-        typedef std::unordered_multiset<int> C;
-        typedef int P;
-        P a[] =
-        {
-            P(1),
-            P(2),
-            P(3),
-            P(4),
-            P(1),
-            P(2)
-        };
-        C c(a, a + sizeof(a)/sizeof(a[0]));
-        test(c);
-        assert(c.bucket_count() >= 7);
-        c.rehash(3);
-        rehash_postcondition(c, 3);
-        LIBCPP_ASSERT(c.bucket_count() == 7);
-        test(c);
-        c.max_load_factor(2);
-        c.rehash(3);
-        rehash_postcondition(c, 3);
-        LIBCPP_ASSERT(c.bucket_count() == 3);
-        test(c);
-        c.rehash(31);
-        rehash_postcondition(c, 31);
-        LIBCPP_ASSERT(c.bucket_count() == 31);
-        test(c);
-    }
+int main(int, char**) {
+  {
+    typedef std::unordered_multiset<int> C;
+    typedef int P;
+    P a[] = {P(1), P(2), P(3), P(4), P(1), P(2)};
+    C c(a, a + sizeof(a) / sizeof(a[0]));
+    test(c);
+    assert(c.bucket_count() >= 7);
+    c.rehash(3);
+    rehash_postcondition(c, 3);
+    LIBCPP_ASSERT(c.bucket_count() == 7);
+    test(c);
+    c.max_load_factor(2);
+    c.rehash(3);
+    rehash_postcondition(c, 3);
+    LIBCPP_ASSERT(c.bucket_count() == 3);
+    test(c);
+    c.rehash(31);
+    rehash_postcondition(c, 31);
+    LIBCPP_ASSERT(c.bucket_count() == 31);
+    test(c);
+  }
 #if TEST_STD_VER >= 11
-    {
-        typedef std::unordered_multiset<int, std::hash<int>,
-                                      std::equal_to<int>, min_allocator<int>> C;
-        typedef int P;
-        P a[] =
-        {
-            P(1),
-            P(2),
-            P(3),
-            P(4),
-            P(1),
-            P(2)
-        };
-        C c(a, a + sizeof(a)/sizeof(a[0]));
-        test(c);
-        assert(c.bucket_count() >= 7);
-        c.rehash(3);
-        rehash_postcondition(c, 3);
-        LIBCPP_ASSERT(c.bucket_count() == 7);
-        test(c);
-        c.max_load_factor(2);
-        c.rehash(3);
-        rehash_postcondition(c, 3);
-        LIBCPP_ASSERT(c.bucket_count() == 3);
-        test(c);
-        c.rehash(31);
-        rehash_postcondition(c, 31);
-        LIBCPP_ASSERT(c.bucket_count() == 31);
-        test(c);
-    }
+  {
+    typedef std::unordered_multiset<int, std::hash<int>, std::equal_to<int>, min_allocator<int>> C;
+    typedef int P;
+    P a[] = {P(1), P(2), P(3), P(4), P(1), P(2)};
+    C c(a, a + sizeof(a) / sizeof(a[0]));
+    test(c);
+    assert(c.bucket_count() >= 7);
+    c.rehash(3);
+    rehash_postcondition(c, 3);
+    LIBCPP_ASSERT(c.bucket_count() == 7);
+    test(c);
+    c.max_load_factor(2);
+    c.rehash(3);
+    rehash_postcondition(c, 3);
+    LIBCPP_ASSERT(c.bucket_count() == 3);
+    test(c);
+    c.rehash(31);
+    rehash_postcondition(c, 31);
+    LIBCPP_ASSERT(c.bucket_count() == 31);
+    test(c);
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/unord/unord.multiset/reserve.pass.cpp b/libcxx/test/std/containers/unord/unord.multiset/reserve.pass.cpp
index 557a9241b4de1..4862ed2ec730d 100644
--- a/libcxx/test/std/containers/unord/unord.multiset/reserve.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.multiset/reserve.pass.cpp
@@ -21,88 +21,67 @@
 #include "min_allocator.h"
 
 template <class C>
-void test(const C& c)
-{
-    assert(c.size() == 6);
-    assert(c.count(1) == 2);
-    assert(c.count(2) == 2);
-    assert(c.count(3) == 1);
-    assert(c.count(4) == 1);
+void test(const C& c) {
+  assert(c.size() == 6);
+  assert(c.count(1) == 2);
+  assert(c.count(2) == 2);
+  assert(c.count(3) == 1);
+  assert(c.count(4) == 1);
 }
 
 void reserve_invariant(std::size_t n) // LWG #2156
 {
-    for (std::size_t i = 0; i < n; ++i)
-    {
-        std::unordered_multiset<std::size_t> c;
-        c.reserve(n);
-        std::size_t buckets = c.bucket_count();
-        for (std::size_t j = 0; j < i; ++j)
-        {
-            c.insert(i);
-            assert(buckets == c.bucket_count());
-        }
+  for (std::size_t i = 0; i < n; ++i) {
+    std::unordered_multiset<std::size_t> c;
+    c.reserve(n);
+    std::size_t buckets = c.bucket_count();
+    for (std::size_t j = 0; j < i; ++j) {
+      c.insert(i);
+      assert(buckets == c.bucket_count());
     }
+  }
 }
 
-int main(int, char**)
-{
-    {
-        typedef std::unordered_multiset<int> C;
-        typedef int P;
-        P a[] =
-        {
-            P(1),
-            P(2),
-            P(3),
-            P(4),
-            P(1),
-            P(2)
-        };
-        C c(a, a + sizeof(a)/sizeof(a[0]));
-        test(c);
-        assert(c.bucket_count() >= 7);
-        c.reserve(3);
-        LIBCPP_ASSERT(c.bucket_count() == 7);
-        test(c);
-        c.max_load_factor(2);
-        c.reserve(3);
-        LIBCPP_ASSERT(c.bucket_count() == 3);
-        test(c);
-        c.reserve(31);
-        assert(c.bucket_count() >= 16);
-        test(c);
-    }
+int main(int, char**) {
+  {
+    typedef std::unordered_multiset<int> C;
+    typedef int P;
+    P a[] = {P(1), P(2), P(3), P(4), P(1), P(2)};
+    C c(a, a + sizeof(a) / sizeof(a[0]));
+    test(c);
+    assert(c.bucket_count() >= 7);
+    c.reserve(3);
+    LIBCPP_ASSERT(c.bucket_count() == 7);
+    test(c);
+    c.max_load_factor(2);
+    c.reserve(3);
+    LIBCPP_ASSERT(c.bucket_count() == 3);
+    test(c);
+    c.reserve(31);
+    assert(c.bucket_count() >= 16);
+    test(c);
+  }
 #if TEST_STD_VER >= 11
-    {
-        typedef std::unordered_multiset<int, std::hash<int>,
-                                      std::equal_to<int>, min_allocator<int>> C;
-        typedef int P;
-        P a[] =
-        {
-            P(1),
-            P(2),
-            P(3),
-            P(4),
-            P(1),
-            P(2)
-        };
-        C c(a, a + sizeof(a)/sizeof(a[0]));
-        test(c);
-        assert(c.bucket_count() >= 7);
-        c.reserve(3);
-        LIBCPP_ASSERT(c.bucket_count() == 7);
-        test(c);
-        c.max_load_factor(2);
-        c.reserve(3);
-        LIBCPP_ASSERT(c.bucket_count() == 3);
-        test(c);
-        c.reserve(31);
-        assert(c.bucket_count() >= 16);
-        test(c);
-    }
+  {
+    typedef std::unordered_multiset<int, std::hash<int>, std::equal_to<int>, min_allocator<int>> C;
+    typedef int P;
+    P a[] = {P(1), P(2), P(3), P(4), P(1), P(2)};
+    C c(a, a + sizeof(a) / sizeof(a[0]));
+    test(c);
+    assert(c.bucket_count() >= 7);
+    c.reserve(3);
+    LIBCPP_ASSERT(c.bucket_count() == 7);
+    test(c);
+    c.max_load_factor(2);
+    c.reserve(3);
+    LIBCPP_ASSERT(c.bucket_count() == 3);
+    test(c);
+    c.reserve(31);
+    assert(c.bucket_count() >= 16);
+    test(c);
+  }
 #endif
-    reserve_invariant(20);
+  reserve_invariant(20);
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/unord/unord.multiset/scary.pass.cpp b/libcxx/test/std/containers/unord/unord.multiset/scary.pass.cpp
index 9c9a16c7b043c..89f575bc2d790 100644
--- a/libcxx/test/std/containers/unord/unord.multiset/scary.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.multiset/scary.pass.cpp
@@ -16,13 +16,12 @@
 
 #include "test_macros.h"
 
-int main(int, char**)
-{
-    typedef std::unordered_set<int> M1;
-    typedef std::unordered_multiset<int> M2;
-    M2::iterator i;
-    M1::iterator j = i;
-    ((void)j);
+int main(int, char**) {
+  typedef std::unordered_set<int> M1;
+  typedef std::unordered_multiset<int> M2;
+  M2::iterator i;
+  M1::iterator j = i;
+  ((void)j);
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/unord/unord.multiset/size.pass.cpp b/libcxx/test/std/containers/unord/unord.multiset/size.pass.cpp
index 12a4733ccb192..59473fe085e08 100644
--- a/libcxx/test/std/containers/unord/unord.multiset/size.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.multiset/size.pass.cpp
@@ -18,9 +18,8 @@
 #include "test_macros.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
+int main(int, char**) {
+  {
     typedef std::unordered_multiset<int> M;
     M m;
     ASSERT_NOEXCEPT(m.size());
@@ -37,9 +36,9 @@ int main(int, char**)
     assert(m.size() == 1);
     m.erase(m.begin());
     assert(m.size() == 0);
-    }
+  }
 #if TEST_STD_VER >= 11
-    {
+  {
     typedef std::unordered_multiset<int, std::hash<int>, std::equal_to<int>, min_allocator<int>> M;
     M m;
     ASSERT_NOEXCEPT(m.size());
@@ -56,7 +55,7 @@ int main(int, char**)
     assert(m.size() == 1);
     m.erase(m.begin());
     assert(m.size() == 0);
-    }
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/unord/unord.multiset/swap_member.pass.cpp b/libcxx/test/std/containers/unord/unord.multiset/swap_member.pass.cpp
index 59acb07d7cf17..770cbfd21e400 100644
--- a/libcxx/test/std/containers/unord/unord.multiset/swap_member.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.multiset/swap_member.pass.cpp
@@ -24,547 +24,438 @@
 #include "test_allocator.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef test_hash<int> Hash;
-        typedef test_equal_to<int> Compare;
-        typedef test_allocator<int> Alloc;
-        typedef std::unordered_multiset<int, Hash, Compare, Alloc> C;
-        C c1(0, Hash(1), Compare(1), Alloc(1, 1));
-        C c2(0, Hash(2), Compare(2), Alloc(1, 2));
-        c2.max_load_factor(2);
-        c1.swap(c2);
-
-        LIBCPP_ASSERT(c1.bucket_count() == 0);
-        assert(c1.size() == 0);
-        assert(c1.hash_function() == Hash(2));
-        assert(c1.key_eq() == Compare(2));
-        assert(c1.get_allocator().get_id() == 1);
-        assert(static_cast<std::size_t>(std::distance(c1.begin(), c1.end())) == c1.size());
-        assert(static_cast<std::size_t>(std::distance(c1.cbegin(), c1.cend())) == c1.size());
-        assert(c1.max_load_factor() == 2);
-
-        LIBCPP_ASSERT(c2.bucket_count() == 0);
-        assert(c2.size() == 0);
-        assert(c2.hash_function() == Hash(1));
-        assert(c2.key_eq() == Compare(1));
-        assert(c2.get_allocator().get_id() == 2);
-        assert(static_cast<std::size_t>(std::distance(c2.begin(), c2.end())) == c2.size());
-        assert(static_cast<std::size_t>(std::distance(c2.cbegin(), c2.cend())) == c2.size());
-        assert(c2.max_load_factor() == 1);
-    }
-    {
-        typedef test_hash<int> Hash;
-        typedef test_equal_to<int> Compare;
-        typedef test_allocator<int> Alloc;
-        typedef std::unordered_multiset<int, Hash, Compare, Alloc> C;
-        typedef int P;
-        P a2[] =
-        {
-            P(10),
-            P(20),
-            P(30),
-            P(40),
-            P(50),
-            P(60),
-            P(70),
-            P(80)
-        };
-        C c1(0, Hash(1), Compare(1), Alloc(1, 1));
-        C c2(std::begin(a2), std::end(a2), 0, Hash(2), Compare(2), Alloc(1, 2));
-        c2.max_load_factor(2);
-        c1.swap(c2);
-
-        assert(c1.bucket_count() >= 8);
-        assert(c1.size() == 8);
-        assert(*c1.find(10) == 10);
-        assert(*c1.find(20) == 20);
-        assert(*c1.find(30) == 30);
-        assert(*c1.find(40) == 40);
-        assert(*c1.find(50) == 50);
-        assert(*c1.find(60) == 60);
-        assert(*c1.find(70) == 70);
-        assert(*c1.find(80) == 80);
-        assert(c1.hash_function() == Hash(2));
-        assert(c1.key_eq() == Compare(2));
-        assert(c1.get_allocator().get_id() == 1);
-        assert(static_cast<std::size_t>(std::distance(c1.begin(), c1.end())) == c1.size());
-        assert(static_cast<std::size_t>(std::distance(c1.cbegin(), c1.cend())) == c1.size());
-        assert(c1.max_load_factor() == 2);
-
-        LIBCPP_ASSERT(c2.bucket_count() == 0);
-        assert(c2.size() == 0);
-        assert(c2.hash_function() == Hash(1));
-        assert(c2.key_eq() == Compare(1));
-        assert(c2.get_allocator().get_id() == 2);
-        assert(static_cast<std::size_t>(std::distance(c2.begin(), c2.end())) == c2.size());
-        assert(static_cast<std::size_t>(std::distance(c2.cbegin(), c2.cend())) == c2.size());
-        assert(c2.max_load_factor() == 1);
-    }
-    {
-        typedef test_hash<int> Hash;
-        typedef test_equal_to<int> Compare;
-        typedef test_allocator<int> Alloc;
-        typedef std::unordered_multiset<int, Hash, Compare, Alloc> C;
-        typedef int P;
-        P a1[] =
-        {
-            P(1),
-            P(2),
-            P(3),
-            P(4),
-            P(1),
-            P(2)
-        };
-        C c1(std::begin(a1), std::end(a1), 0, Hash(1), Compare(1), Alloc(1, 1));
-        C c2(0, Hash(2), Compare(2), Alloc(1, 2));
-        c2.max_load_factor(2);
-        c1.swap(c2);
-
-        LIBCPP_ASSERT(c1.bucket_count() == 0);
-        assert(c1.size() == 0);
-        assert(c1.hash_function() == Hash(2));
-        assert(c1.key_eq() == Compare(2));
-        assert(c1.get_allocator().get_id() == 1);
-        assert(static_cast<std::size_t>(std::distance(c1.begin(), c1.end())) == c1.size());
-        assert(static_cast<std::size_t>(std::distance(c1.cbegin(), c1.cend())) == c1.size());
-        assert(c1.max_load_factor() == 2);
-
-        assert(c2.bucket_count() >= 6);
-        assert(c2.size() == 6);
-        assert(c2.count(1) == 2);
-        assert(c2.count(2) == 2);
-        assert(c2.count(3) == 1);
-        assert(c2.count(4) == 1);
-        assert(c2.hash_function() == Hash(1));
-        assert(c2.key_eq() == Compare(1));
-        assert(c2.get_allocator().get_id() == 2);
-        assert(static_cast<std::size_t>(std::distance(c2.begin(), c2.end())) == c2.size());
-        assert(static_cast<std::size_t>(std::distance(c2.cbegin(), c2.cend())) == c2.size());
-        assert(c2.max_load_factor() == 1);
-    }
-    {
-        typedef test_hash<int> Hash;
-        typedef test_equal_to<int> Compare;
-        typedef test_allocator<int> Alloc;
-        typedef std::unordered_multiset<int, Hash, Compare, Alloc> C;
-        typedef int P;
-        P a1[] =
-        {
-            P(1),
-            P(2),
-            P(3),
-            P(4),
-            P(1),
-            P(2)
-        };
-        P a2[] =
-        {
-            P(10),
-            P(20),
-            P(30),
-            P(40),
-            P(50),
-            P(60),
-            P(70),
-            P(80)
-        };
-        C c1(std::begin(a1), std::end(a1), 0, Hash(1), Compare(1), Alloc(1, 1));
-        C c2(std::begin(a2), std::end(a2), 0, Hash(2), Compare(2), Alloc(1, 2));
-        c2.max_load_factor(2);
-        c1.swap(c2);
-
-        assert(c1.bucket_count() >= 8);
-        assert(c1.size() == 8);
-        assert(*c1.find(10) == 10);
-        assert(*c1.find(20) == 20);
-        assert(*c1.find(30) == 30);
-        assert(*c1.find(40) == 40);
-        assert(*c1.find(50) == 50);
-        assert(*c1.find(60) == 60);
-        assert(*c1.find(70) == 70);
-        assert(*c1.find(80) == 80);
-        assert(c1.hash_function() == Hash(2));
-        assert(c1.key_eq() == Compare(2));
-        assert(c1.get_allocator().get_id() == 1);
-        assert(static_cast<std::size_t>(std::distance(c1.begin(), c1.end())) == c1.size());
-        assert(static_cast<std::size_t>(std::distance(c1.cbegin(), c1.cend())) == c1.size());
-        assert(c1.max_load_factor() == 2);
-
-        assert(c2.bucket_count() >= 6);
-        assert(c2.size() == 6);
-        assert(c2.count(1) == 2);
-        assert(c2.count(2) == 2);
-        assert(c2.count(3) == 1);
-        assert(c2.count(4) == 1);
-        assert(c2.hash_function() == Hash(1));
-        assert(c2.key_eq() == Compare(1));
-        assert(c2.get_allocator().get_id() == 2);
-        assert(static_cast<std::size_t>(std::distance(c2.begin(), c2.end())) == c2.size());
-        assert(static_cast<std::size_t>(std::distance(c2.cbegin(), c2.cend())) == c2.size());
-        assert(c2.max_load_factor() == 1);
-    }
-
-    {
-        typedef test_hash<int> Hash;
-        typedef test_equal_to<int> Compare;
-        typedef other_allocator<int> Alloc;
-        typedef std::unordered_multiset<int, Hash, Compare, Alloc> C;
-        C c1(0, Hash(1), Compare(1), Alloc(1));
-        C c2(0, Hash(2), Compare(2), Alloc(2));
-        c2.max_load_factor(2);
-        c1.swap(c2);
-
-        LIBCPP_ASSERT(c1.bucket_count() == 0);
-        assert(c1.size() == 0);
-        assert(c1.hash_function() == Hash(2));
-        assert(c1.key_eq() == Compare(2));
-        assert(c1.get_allocator() == Alloc(2));
-        assert(static_cast<std::size_t>(std::distance(c1.begin(), c1.end())) == c1.size());
-        assert(static_cast<std::size_t>(std::distance(c1.cbegin(), c1.cend())) == c1.size());
-        assert(c1.max_load_factor() == 2);
-
-        LIBCPP_ASSERT(c2.bucket_count() == 0);
-        assert(c2.size() == 0);
-        assert(c2.hash_function() == Hash(1));
-        assert(c2.key_eq() == Compare(1));
-        assert(c2.get_allocator() == Alloc(1));
-        assert(static_cast<std::size_t>(std::distance(c2.begin(), c2.end())) == c2.size());
-        assert(static_cast<std::size_t>(std::distance(c2.cbegin(), c2.cend())) == c2.size());
-        assert(c2.max_load_factor() == 1);
-    }
-    {
-        typedef test_hash<int> Hash;
-        typedef test_equal_to<int> Compare;
-        typedef other_allocator<int> Alloc;
-        typedef std::unordered_multiset<int, Hash, Compare, Alloc> C;
-        typedef int P;
-        P a2[] =
-        {
-            P(10),
-            P(20),
-            P(30),
-            P(40),
-            P(50),
-            P(60),
-            P(70),
-            P(80)
-        };
-        C c1(0, Hash(1), Compare(1), Alloc(1));
-        C c2(std::begin(a2), std::end(a2), 0, Hash(2), Compare(2), Alloc(2));
-        c2.max_load_factor(2);
-        c1.swap(c2);
-
-        assert(c1.bucket_count() >= 8);
-        assert(c1.size() == 8);
-        assert(*c1.find(10) == 10);
-        assert(*c1.find(20) == 20);
-        assert(*c1.find(30) == 30);
-        assert(*c1.find(40) == 40);
-        assert(*c1.find(50) == 50);
-        assert(*c1.find(60) == 60);
-        assert(*c1.find(70) == 70);
-        assert(*c1.find(80) == 80);
-        assert(c1.hash_function() == Hash(2));
-        assert(c1.key_eq() == Compare(2));
-        assert(c1.get_allocator() == Alloc(2));
-        assert(static_cast<std::size_t>(std::distance(c1.begin(), c1.end())) == c1.size());
-        assert(static_cast<std::size_t>(std::distance(c1.cbegin(), c1.cend())) == c1.size());
-        assert(c1.max_load_factor() == 2);
-
-        LIBCPP_ASSERT(c2.bucket_count() == 0);
-        assert(c2.size() == 0);
-        assert(c2.hash_function() == Hash(1));
-        assert(c2.key_eq() == Compare(1));
-        assert(c2.get_allocator() == Alloc(1));
-        assert(static_cast<std::size_t>(std::distance(c2.begin(), c2.end())) == c2.size());
-        assert(static_cast<std::size_t>(std::distance(c2.cbegin(), c2.cend())) == c2.size());
-        assert(c2.max_load_factor() == 1);
-    }
-    {
-        typedef test_hash<int> Hash;
-        typedef test_equal_to<int> Compare;
-        typedef other_allocator<int> Alloc;
-        typedef std::unordered_multiset<int, Hash, Compare, Alloc> C;
-        typedef int P;
-        P a1[] =
-        {
-            P(1),
-            P(2),
-            P(3),
-            P(4),
-            P(1),
-            P(2)
-        };
-        C c1(std::begin(a1), std::end(a1), 0, Hash(1), Compare(1), Alloc(1));
-        C c2(0, Hash(2), Compare(2), Alloc(2));
-        c2.max_load_factor(2);
-        c1.swap(c2);
-
-        LIBCPP_ASSERT(c1.bucket_count() == 0);
-        assert(c1.size() == 0);
-        assert(c1.hash_function() == Hash(2));
-        assert(c1.key_eq() == Compare(2));
-        assert(c1.get_allocator() == Alloc(2));
-        assert(static_cast<std::size_t>(std::distance(c1.begin(), c1.end())) == c1.size());
-        assert(static_cast<std::size_t>(std::distance(c1.cbegin(), c1.cend())) == c1.size());
-        assert(c1.max_load_factor() == 2);
-
-        assert(c2.bucket_count() >= 6);
-        assert(c2.size() == 6);
-        assert(c2.count(1) == 2);
-        assert(c2.count(2) == 2);
-        assert(c2.count(3) == 1);
-        assert(c2.count(4) == 1);
-        assert(c2.hash_function() == Hash(1));
-        assert(c2.key_eq() == Compare(1));
-        assert(c2.get_allocator() == Alloc(1));
-        assert(static_cast<std::size_t>(std::distance(c2.begin(), c2.end())) == c2.size());
-        assert(static_cast<std::size_t>(std::distance(c2.cbegin(), c2.cend())) == c2.size());
-        assert(c2.max_load_factor() == 1);
-    }
-    {
-        typedef test_hash<int> Hash;
-        typedef test_equal_to<int> Compare;
-        typedef other_allocator<int> Alloc;
-        typedef std::unordered_multiset<int, Hash, Compare, Alloc> C;
-        typedef int P;
-        P a1[] =
-        {
-            P(1),
-            P(2),
-            P(3),
-            P(4),
-            P(1),
-            P(2)
-        };
-        P a2[] =
-        {
-            P(10),
-            P(20),
-            P(30),
-            P(40),
-            P(50),
-            P(60),
-            P(70),
-            P(80)
-        };
-        C c1(std::begin(a1), std::end(a1), 0, Hash(1), Compare(1), Alloc(1));
-        C c2(std::begin(a2), std::end(a2), 0, Hash(2), Compare(2), Alloc(2));
-        c2.max_load_factor(2);
-        c1.swap(c2);
-
-        assert(c1.bucket_count() >= 8);
-        assert(c1.size() == 8);
-        assert(*c1.find(10) == 10);
-        assert(*c1.find(20) == 20);
-        assert(*c1.find(30) == 30);
-        assert(*c1.find(40) == 40);
-        assert(*c1.find(50) == 50);
-        assert(*c1.find(60) == 60);
-        assert(*c1.find(70) == 70);
-        assert(*c1.find(80) == 80);
-        assert(c1.hash_function() == Hash(2));
-        assert(c1.key_eq() == Compare(2));
-        assert(c1.get_allocator() == Alloc(2));
-        assert(static_cast<std::size_t>(std::distance(c1.begin(), c1.end())) == c1.size());
-        assert(static_cast<std::size_t>(std::distance(c1.cbegin(), c1.cend())) == c1.size());
-        assert(c1.max_load_factor() == 2);
-
-        assert(c2.bucket_count() >= 6);
-        assert(c2.size() == 6);
-        assert(c2.count(1) == 2);
-        assert(c2.count(2) == 2);
-        assert(c2.count(3) == 1);
-        assert(c2.count(4) == 1);
-        assert(c2.hash_function() == Hash(1));
-        assert(c2.key_eq() == Compare(1));
-        assert(c2.get_allocator() == Alloc(1));
-        assert(static_cast<std::size_t>(std::distance(c2.begin(), c2.end())) == c2.size());
-        assert(static_cast<std::size_t>(std::distance(c2.cbegin(), c2.cend())) == c2.size());
-        assert(c2.max_load_factor() == 1);
-    }
+int main(int, char**) {
+  {
+    typedef test_hash<int> Hash;
+    typedef test_equal_to<int> Compare;
+    typedef test_allocator<int> Alloc;
+    typedef std::unordered_multiset<int, Hash, Compare, Alloc> C;
+    C c1(0, Hash(1), Compare(1), Alloc(1, 1));
+    C c2(0, Hash(2), Compare(2), Alloc(1, 2));
+    c2.max_load_factor(2);
+    c1.swap(c2);
+
+    LIBCPP_ASSERT(c1.bucket_count() == 0);
+    assert(c1.size() == 0);
+    assert(c1.hash_function() == Hash(2));
+    assert(c1.key_eq() == Compare(2));
+    assert(c1.get_allocator().get_id() == 1);
+    assert(static_cast<std::size_t>(std::distance(c1.begin(), c1.end())) == c1.size());
+    assert(static_cast<std::size_t>(std::distance(c1.cbegin(), c1.cend())) == c1.size());
+    assert(c1.max_load_factor() == 2);
+
+    LIBCPP_ASSERT(c2.bucket_count() == 0);
+    assert(c2.size() == 0);
+    assert(c2.hash_function() == Hash(1));
+    assert(c2.key_eq() == Compare(1));
+    assert(c2.get_allocator().get_id() == 2);
+    assert(static_cast<std::size_t>(std::distance(c2.begin(), c2.end())) == c2.size());
+    assert(static_cast<std::size_t>(std::distance(c2.cbegin(), c2.cend())) == c2.size());
+    assert(c2.max_load_factor() == 1);
+  }
+  {
+    typedef test_hash<int> Hash;
+    typedef test_equal_to<int> Compare;
+    typedef test_allocator<int> Alloc;
+    typedef std::unordered_multiset<int, Hash, Compare, Alloc> C;
+    typedef int P;
+    P a2[] = {P(10), P(20), P(30), P(40), P(50), P(60), P(70), P(80)};
+    C c1(0, Hash(1), Compare(1), Alloc(1, 1));
+    C c2(std::begin(a2), std::end(a2), 0, Hash(2), Compare(2), Alloc(1, 2));
+    c2.max_load_factor(2);
+    c1.swap(c2);
+
+    assert(c1.bucket_count() >= 8);
+    assert(c1.size() == 8);
+    assert(*c1.find(10) == 10);
+    assert(*c1.find(20) == 20);
+    assert(*c1.find(30) == 30);
+    assert(*c1.find(40) == 40);
+    assert(*c1.find(50) == 50);
+    assert(*c1.find(60) == 60);
+    assert(*c1.find(70) == 70);
+    assert(*c1.find(80) == 80);
+    assert(c1.hash_function() == Hash(2));
+    assert(c1.key_eq() == Compare(2));
+    assert(c1.get_allocator().get_id() == 1);
+    assert(static_cast<std::size_t>(std::distance(c1.begin(), c1.end())) == c1.size());
+    assert(static_cast<std::size_t>(std::distance(c1.cbegin(), c1.cend())) == c1.size());
+    assert(c1.max_load_factor() == 2);
+
+    LIBCPP_ASSERT(c2.bucket_count() == 0);
+    assert(c2.size() == 0);
+    assert(c2.hash_function() == Hash(1));
+    assert(c2.key_eq() == Compare(1));
+    assert(c2.get_allocator().get_id() == 2);
+    assert(static_cast<std::size_t>(std::distance(c2.begin(), c2.end())) == c2.size());
+    assert(static_cast<std::size_t>(std::distance(c2.cbegin(), c2.cend())) == c2.size());
+    assert(c2.max_load_factor() == 1);
+  }
+  {
+    typedef test_hash<int> Hash;
+    typedef test_equal_to<int> Compare;
+    typedef test_allocator<int> Alloc;
+    typedef std::unordered_multiset<int, Hash, Compare, Alloc> C;
+    typedef int P;
+    P a1[] = {P(1), P(2), P(3), P(4), P(1), P(2)};
+    C c1(std::begin(a1), std::end(a1), 0, Hash(1), Compare(1), Alloc(1, 1));
+    C c2(0, Hash(2), Compare(2), Alloc(1, 2));
+    c2.max_load_factor(2);
+    c1.swap(c2);
+
+    LIBCPP_ASSERT(c1.bucket_count() == 0);
+    assert(c1.size() == 0);
+    assert(c1.hash_function() == Hash(2));
+    assert(c1.key_eq() == Compare(2));
+    assert(c1.get_allocator().get_id() == 1);
+    assert(static_cast<std::size_t>(std::distance(c1.begin(), c1.end())) == c1.size());
+    assert(static_cast<std::size_t>(std::distance(c1.cbegin(), c1.cend())) == c1.size());
+    assert(c1.max_load_factor() == 2);
+
+    assert(c2.bucket_count() >= 6);
+    assert(c2.size() == 6);
+    assert(c2.count(1) == 2);
+    assert(c2.count(2) == 2);
+    assert(c2.count(3) == 1);
+    assert(c2.count(4) == 1);
+    assert(c2.hash_function() == Hash(1));
+    assert(c2.key_eq() == Compare(1));
+    assert(c2.get_allocator().get_id() == 2);
+    assert(static_cast<std::size_t>(std::distance(c2.begin(), c2.end())) == c2.size());
+    assert(static_cast<std::size_t>(std::distance(c2.cbegin(), c2.cend())) == c2.size());
+    assert(c2.max_load_factor() == 1);
+  }
+  {
+    typedef test_hash<int> Hash;
+    typedef test_equal_to<int> Compare;
+    typedef test_allocator<int> Alloc;
+    typedef std::unordered_multiset<int, Hash, Compare, Alloc> C;
+    typedef int P;
+    P a1[] = {P(1), P(2), P(3), P(4), P(1), P(2)};
+    P a2[] = {P(10), P(20), P(30), P(40), P(50), P(60), P(70), P(80)};
+    C c1(std::begin(a1), std::end(a1), 0, Hash(1), Compare(1), Alloc(1, 1));
+    C c2(std::begin(a2), std::end(a2), 0, Hash(2), Compare(2), Alloc(1, 2));
+    c2.max_load_factor(2);
+    c1.swap(c2);
+
+    assert(c1.bucket_count() >= 8);
+    assert(c1.size() == 8);
+    assert(*c1.find(10) == 10);
+    assert(*c1.find(20) == 20);
+    assert(*c1.find(30) == 30);
+    assert(*c1.find(40) == 40);
+    assert(*c1.find(50) == 50);
+    assert(*c1.find(60) == 60);
+    assert(*c1.find(70) == 70);
+    assert(*c1.find(80) == 80);
+    assert(c1.hash_function() == Hash(2));
+    assert(c1.key_eq() == Compare(2));
+    assert(c1.get_allocator().get_id() == 1);
+    assert(static_cast<std::size_t>(std::distance(c1.begin(), c1.end())) == c1.size());
+    assert(static_cast<std::size_t>(std::distance(c1.cbegin(), c1.cend())) == c1.size());
+    assert(c1.max_load_factor() == 2);
+
+    assert(c2.bucket_count() >= 6);
+    assert(c2.size() == 6);
+    assert(c2.count(1) == 2);
+    assert(c2.count(2) == 2);
+    assert(c2.count(3) == 1);
+    assert(c2.count(4) == 1);
+    assert(c2.hash_function() == Hash(1));
+    assert(c2.key_eq() == Compare(1));
+    assert(c2.get_allocator().get_id() == 2);
+    assert(static_cast<std::size_t>(std::distance(c2.begin(), c2.end())) == c2.size());
+    assert(static_cast<std::size_t>(std::distance(c2.cbegin(), c2.cend())) == c2.size());
+    assert(c2.max_load_factor() == 1);
+  }
+
+  {
+    typedef test_hash<int> Hash;
+    typedef test_equal_to<int> Compare;
+    typedef other_allocator<int> Alloc;
+    typedef std::unordered_multiset<int, Hash, Compare, Alloc> C;
+    C c1(0, Hash(1), Compare(1), Alloc(1));
+    C c2(0, Hash(2), Compare(2), Alloc(2));
+    c2.max_load_factor(2);
+    c1.swap(c2);
+
+    LIBCPP_ASSERT(c1.bucket_count() == 0);
+    assert(c1.size() == 0);
+    assert(c1.hash_function() == Hash(2));
+    assert(c1.key_eq() == Compare(2));
+    assert(c1.get_allocator() == Alloc(2));
+    assert(static_cast<std::size_t>(std::distance(c1.begin(), c1.end())) == c1.size());
+    assert(static_cast<std::size_t>(std::distance(c1.cbegin(), c1.cend())) == c1.size());
+    assert(c1.max_load_factor() == 2);
+
+    LIBCPP_ASSERT(c2.bucket_count() == 0);
+    assert(c2.size() == 0);
+    assert(c2.hash_function() == Hash(1));
+    assert(c2.key_eq() == Compare(1));
+    assert(c2.get_allocator() == Alloc(1));
+    assert(static_cast<std::size_t>(std::distance(c2.begin(), c2.end())) == c2.size());
+    assert(static_cast<std::size_t>(std::distance(c2.cbegin(), c2.cend())) == c2.size());
+    assert(c2.max_load_factor() == 1);
+  }
+  {
+    typedef test_hash<int> Hash;
+    typedef test_equal_to<int> Compare;
+    typedef other_allocator<int> Alloc;
+    typedef std::unordered_multiset<int, Hash, Compare, Alloc> C;
+    typedef int P;
+    P a2[] = {P(10), P(20), P(30), P(40), P(50), P(60), P(70), P(80)};
+    C c1(0, Hash(1), Compare(1), Alloc(1));
+    C c2(std::begin(a2), std::end(a2), 0, Hash(2), Compare(2), Alloc(2));
+    c2.max_load_factor(2);
+    c1.swap(c2);
+
+    assert(c1.bucket_count() >= 8);
+    assert(c1.size() == 8);
+    assert(*c1.find(10) == 10);
+    assert(*c1.find(20) == 20);
+    assert(*c1.find(30) == 30);
+    assert(*c1.find(40) == 40);
+    assert(*c1.find(50) == 50);
+    assert(*c1.find(60) == 60);
+    assert(*c1.find(70) == 70);
+    assert(*c1.find(80) == 80);
+    assert(c1.hash_function() == Hash(2));
+    assert(c1.key_eq() == Compare(2));
+    assert(c1.get_allocator() == Alloc(2));
+    assert(static_cast<std::size_t>(std::distance(c1.begin(), c1.end())) == c1.size());
+    assert(static_cast<std::size_t>(std::distance(c1.cbegin(), c1.cend())) == c1.size());
+    assert(c1.max_load_factor() == 2);
+
+    LIBCPP_ASSERT(c2.bucket_count() == 0);
+    assert(c2.size() == 0);
+    assert(c2.hash_function() == Hash(1));
+    assert(c2.key_eq() == Compare(1));
+    assert(c2.get_allocator() == Alloc(1));
+    assert(static_cast<std::size_t>(std::distance(c2.begin(), c2.end())) == c2.size());
+    assert(static_cast<std::size_t>(std::distance(c2.cbegin(), c2.cend())) == c2.size());
+    assert(c2.max_load_factor() == 1);
+  }
+  {
+    typedef test_hash<int> Hash;
+    typedef test_equal_to<int> Compare;
+    typedef other_allocator<int> Alloc;
+    typedef std::unordered_multiset<int, Hash, Compare, Alloc> C;
+    typedef int P;
+    P a1[] = {P(1), P(2), P(3), P(4), P(1), P(2)};
+    C c1(std::begin(a1), std::end(a1), 0, Hash(1), Compare(1), Alloc(1));
+    C c2(0, Hash(2), Compare(2), Alloc(2));
+    c2.max_load_factor(2);
+    c1.swap(c2);
+
+    LIBCPP_ASSERT(c1.bucket_count() == 0);
+    assert(c1.size() == 0);
+    assert(c1.hash_function() == Hash(2));
+    assert(c1.key_eq() == Compare(2));
+    assert(c1.get_allocator() == Alloc(2));
+    assert(static_cast<std::size_t>(std::distance(c1.begin(), c1.end())) == c1.size());
+    assert(static_cast<std::size_t>(std::distance(c1.cbegin(), c1.cend())) == c1.size());
+    assert(c1.max_load_factor() == 2);
+
+    assert(c2.bucket_count() >= 6);
+    assert(c2.size() == 6);
+    assert(c2.count(1) == 2);
+    assert(c2.count(2) == 2);
+    assert(c2.count(3) == 1);
+    assert(c2.count(4) == 1);
+    assert(c2.hash_function() == Hash(1));
+    assert(c2.key_eq() == Compare(1));
+    assert(c2.get_allocator() == Alloc(1));
+    assert(static_cast<std::size_t>(std::distance(c2.begin(), c2.end())) == c2.size());
+    assert(static_cast<std::size_t>(std::distance(c2.cbegin(), c2.cend())) == c2.size());
+    assert(c2.max_load_factor() == 1);
+  }
+  {
+    typedef test_hash<int> Hash;
+    typedef test_equal_to<int> Compare;
+    typedef other_allocator<int> Alloc;
+    typedef std::unordered_multiset<int, Hash, Compare, Alloc> C;
+    typedef int P;
+    P a1[] = {P(1), P(2), P(3), P(4), P(1), P(2)};
+    P a2[] = {P(10), P(20), P(30), P(40), P(50), P(60), P(70), P(80)};
+    C c1(std::begin(a1), std::end(a1), 0, Hash(1), Compare(1), Alloc(1));
+    C c2(std::begin(a2), std::end(a2), 0, Hash(2), Compare(2), Alloc(2));
+    c2.max_load_factor(2);
+    c1.swap(c2);
+
+    assert(c1.bucket_count() >= 8);
+    assert(c1.size() == 8);
+    assert(*c1.find(10) == 10);
+    assert(*c1.find(20) == 20);
+    assert(*c1.find(30) == 30);
+    assert(*c1.find(40) == 40);
+    assert(*c1.find(50) == 50);
+    assert(*c1.find(60) == 60);
+    assert(*c1.find(70) == 70);
+    assert(*c1.find(80) == 80);
+    assert(c1.hash_function() == Hash(2));
+    assert(c1.key_eq() == Compare(2));
+    assert(c1.get_allocator() == Alloc(2));
+    assert(static_cast<std::size_t>(std::distance(c1.begin(), c1.end())) == c1.size());
+    assert(static_cast<std::size_t>(std::distance(c1.cbegin(), c1.cend())) == c1.size());
+    assert(c1.max_load_factor() == 2);
+
+    assert(c2.bucket_count() >= 6);
+    assert(c2.size() == 6);
+    assert(c2.count(1) == 2);
+    assert(c2.count(2) == 2);
+    assert(c2.count(3) == 1);
+    assert(c2.count(4) == 1);
+    assert(c2.hash_function() == Hash(1));
+    assert(c2.key_eq() == Compare(1));
+    assert(c2.get_allocator() == Alloc(1));
+    assert(static_cast<std::size_t>(std::distance(c2.begin(), c2.end())) == c2.size());
+    assert(static_cast<std::size_t>(std::distance(c2.cbegin(), c2.cend())) == c2.size());
+    assert(c2.max_load_factor() == 1);
+  }
 #if TEST_STD_VER >= 11
-    {
-        typedef test_hash<int> Hash;
-        typedef test_equal_to<int> Compare;
-        typedef min_allocator<int> Alloc;
-        typedef std::unordered_multiset<int, Hash, Compare, Alloc> C;
-        C c1(0, Hash(1), Compare(1), Alloc());
-        C c2(0, Hash(2), Compare(2), Alloc());
-        c2.max_load_factor(2);
-        c1.swap(c2);
-
-        LIBCPP_ASSERT(c1.bucket_count() == 0);
-        assert(c1.size() == 0);
-        assert(c1.hash_function() == Hash(2));
-        assert(c1.key_eq() == Compare(2));
-        assert(c1.get_allocator() == Alloc());
-        assert(static_cast<std::size_t>(std::distance(c1.begin(), c1.end())) == c1.size());
-        assert(static_cast<std::size_t>(std::distance(c1.cbegin(), c1.cend())) == c1.size());
-        assert(c1.max_load_factor() == 2);
-
-        LIBCPP_ASSERT(c2.bucket_count() == 0);
-        assert(c2.size() == 0);
-        assert(c2.hash_function() == Hash(1));
-        assert(c2.key_eq() == Compare(1));
-        assert(c2.get_allocator() == Alloc());
-        assert(static_cast<std::size_t>(std::distance(c2.begin(), c2.end())) == c2.size());
-        assert(static_cast<std::size_t>(std::distance(c2.cbegin(), c2.cend())) == c2.size());
-        assert(c2.max_load_factor() == 1);
-    }
-    {
-        typedef test_hash<int> Hash;
-        typedef test_equal_to<int> Compare;
-        typedef min_allocator<int> Alloc;
-        typedef std::unordered_multiset<int, Hash, Compare, Alloc> C;
-        typedef int P;
-        P a2[] =
-        {
-            P(10),
-            P(20),
-            P(30),
-            P(40),
-            P(50),
-            P(60),
-            P(70),
-            P(80)
-        };
-        C c1(0, Hash(1), Compare(1), Alloc());
-        C c2(std::begin(a2), std::end(a2), 0, Hash(2), Compare(2), Alloc());
-        c2.max_load_factor(2);
-        c1.swap(c2);
-
-        assert(c1.bucket_count() >= 8);
-        assert(c1.size() == 8);
-        assert(*c1.find(10) == 10);
-        assert(*c1.find(20) == 20);
-        assert(*c1.find(30) == 30);
-        assert(*c1.find(40) == 40);
-        assert(*c1.find(50) == 50);
-        assert(*c1.find(60) == 60);
-        assert(*c1.find(70) == 70);
-        assert(*c1.find(80) == 80);
-        assert(c1.hash_function() == Hash(2));
-        assert(c1.key_eq() == Compare(2));
-        assert(c1.get_allocator() == Alloc());
-        assert(static_cast<std::size_t>(std::distance(c1.begin(), c1.end())) == c1.size());
-        assert(static_cast<std::size_t>(std::distance(c1.cbegin(), c1.cend())) == c1.size());
-        assert(c1.max_load_factor() == 2);
-
-        LIBCPP_ASSERT(c2.bucket_count() == 0);
-        assert(c2.size() == 0);
-        assert(c2.hash_function() == Hash(1));
-        assert(c2.key_eq() == Compare(1));
-        assert(c2.get_allocator() == Alloc());
-        assert(static_cast<std::size_t>(std::distance(c2.begin(), c2.end())) == c2.size());
-        assert(static_cast<std::size_t>(std::distance(c2.cbegin(), c2.cend())) == c2.size());
-        assert(c2.max_load_factor() == 1);
-    }
-    {
-        typedef test_hash<int> Hash;
-        typedef test_equal_to<int> Compare;
-        typedef min_allocator<int> Alloc;
-        typedef std::unordered_multiset<int, Hash, Compare, Alloc> C;
-        typedef int P;
-        P a1[] =
-        {
-            P(1),
-            P(2),
-            P(3),
-            P(4),
-            P(1),
-            P(2)
-        };
-        C c1(std::begin(a1), std::end(a1), 0, Hash(1), Compare(1), Alloc());
-        C c2(0, Hash(2), Compare(2), Alloc());
-        c2.max_load_factor(2);
-        c1.swap(c2);
-
-        LIBCPP_ASSERT(c1.bucket_count() == 0);
-        assert(c1.size() == 0);
-        assert(c1.hash_function() == Hash(2));
-        assert(c1.key_eq() == Compare(2));
-        assert(c1.get_allocator() == Alloc());
-        assert(static_cast<std::size_t>(std::distance(c1.begin(), c1.end())) == c1.size());
-        assert(static_cast<std::size_t>(std::distance(c1.cbegin(), c1.cend())) == c1.size());
-        assert(c1.max_load_factor() == 2);
-
-        assert(c2.bucket_count() >= 6);
-        assert(c2.size() == 6);
-        assert(c2.count(1) == 2);
-        assert(c2.count(2) == 2);
-        assert(c2.count(3) == 1);
-        assert(c2.count(4) == 1);
-        assert(c2.hash_function() == Hash(1));
-        assert(c2.key_eq() == Compare(1));
-        assert(c2.get_allocator() == Alloc());
-        assert(static_cast<std::size_t>(std::distance(c2.begin(), c2.end())) == c2.size());
-        assert(static_cast<std::size_t>(std::distance(c2.cbegin(), c2.cend())) == c2.size());
-        assert(c2.max_load_factor() == 1);
-    }
-    {
-        typedef test_hash<int> Hash;
-        typedef test_equal_to<int> Compare;
-        typedef min_allocator<int> Alloc;
-        typedef std::unordered_multiset<int, Hash, Compare, Alloc> C;
-        typedef int P;
-        P a1[] =
-        {
-            P(1),
-            P(2),
-            P(3),
-            P(4),
-            P(1),
-            P(2)
-        };
-        P a2[] =
-        {
-            P(10),
-            P(20),
-            P(30),
-            P(40),
-            P(50),
-            P(60),
-            P(70),
-            P(80)
-        };
-        C c1(std::begin(a1), std::end(a1), 0, Hash(1), Compare(1), Alloc());
-        C c2(std::begin(a2), std::end(a2), 0, Hash(2), Compare(2), Alloc());
-        c2.max_load_factor(2);
-        c1.swap(c2);
-
-        assert(c1.bucket_count() >= 8);
-        assert(c1.size() == 8);
-        assert(*c1.find(10) == 10);
-        assert(*c1.find(20) == 20);
-        assert(*c1.find(30) == 30);
-        assert(*c1.find(40) == 40);
-        assert(*c1.find(50) == 50);
-        assert(*c1.find(60) == 60);
-        assert(*c1.find(70) == 70);
-        assert(*c1.find(80) == 80);
-        assert(c1.hash_function() == Hash(2));
-        assert(c1.key_eq() == Compare(2));
-        assert(c1.get_allocator() == Alloc());
-        assert(static_cast<std::size_t>(std::distance(c1.begin(), c1.end())) == c1.size());
-        assert(static_cast<std::size_t>(std::distance(c1.cbegin(), c1.cend())) == c1.size());
-        assert(c1.max_load_factor() == 2);
-
-        assert(c2.bucket_count() >= 6);
-        assert(c2.size() == 6);
-        assert(c2.count(1) == 2);
-        assert(c2.count(2) == 2);
-        assert(c2.count(3) == 1);
-        assert(c2.count(4) == 1);
-        assert(c2.hash_function() == Hash(1));
-        assert(c2.key_eq() == Compare(1));
-        assert(c2.get_allocator() == Alloc());
-        assert(static_cast<std::size_t>(std::distance(c2.begin(), c2.end())) == c2.size());
-        assert(static_cast<std::size_t>(std::distance(c2.cbegin(), c2.cend())) == c2.size());
-        assert(c2.max_load_factor() == 1);
-    }
+  {
+    typedef test_hash<int> Hash;
+    typedef test_equal_to<int> Compare;
+    typedef min_allocator<int> Alloc;
+    typedef std::unordered_multiset<int, Hash, Compare, Alloc> C;
+    C c1(0, Hash(1), Compare(1), Alloc());
+    C c2(0, Hash(2), Compare(2), Alloc());
+    c2.max_load_factor(2);
+    c1.swap(c2);
+
+    LIBCPP_ASSERT(c1.bucket_count() == 0);
+    assert(c1.size() == 0);
+    assert(c1.hash_function() == Hash(2));
+    assert(c1.key_eq() == Compare(2));
+    assert(c1.get_allocator() == Alloc());
+    assert(static_cast<std::size_t>(std::distance(c1.begin(), c1.end())) == c1.size());
+    assert(static_cast<std::size_t>(std::distance(c1.cbegin(), c1.cend())) == c1.size());
+    assert(c1.max_load_factor() == 2);
+
+    LIBCPP_ASSERT(c2.bucket_count() == 0);
+    assert(c2.size() == 0);
+    assert(c2.hash_function() == Hash(1));
+    assert(c2.key_eq() == Compare(1));
+    assert(c2.get_allocator() == Alloc());
+    assert(static_cast<std::size_t>(std::distance(c2.begin(), c2.end())) == c2.size());
+    assert(static_cast<std::size_t>(std::distance(c2.cbegin(), c2.cend())) == c2.size());
+    assert(c2.max_load_factor() == 1);
+  }
+  {
+    typedef test_hash<int> Hash;
+    typedef test_equal_to<int> Compare;
+    typedef min_allocator<int> Alloc;
+    typedef std::unordered_multiset<int, Hash, Compare, Alloc> C;
+    typedef int P;
+    P a2[] = {P(10), P(20), P(30), P(40), P(50), P(60), P(70), P(80)};
+    C c1(0, Hash(1), Compare(1), Alloc());
+    C c2(std::begin(a2), std::end(a2), 0, Hash(2), Compare(2), Alloc());
+    c2.max_load_factor(2);
+    c1.swap(c2);
+
+    assert(c1.bucket_count() >= 8);
+    assert(c1.size() == 8);
+    assert(*c1.find(10) == 10);
+    assert(*c1.find(20) == 20);
+    assert(*c1.find(30) == 30);
+    assert(*c1.find(40) == 40);
+    assert(*c1.find(50) == 50);
+    assert(*c1.find(60) == 60);
+    assert(*c1.find(70) == 70);
+    assert(*c1.find(80) == 80);
+    assert(c1.hash_function() == Hash(2));
+    assert(c1.key_eq() == Compare(2));
+    assert(c1.get_allocator() == Alloc());
+    assert(static_cast<std::size_t>(std::distance(c1.begin(), c1.end())) == c1.size());
+    assert(static_cast<std::size_t>(std::distance(c1.cbegin(), c1.cend())) == c1.size());
+    assert(c1.max_load_factor() == 2);
+
+    LIBCPP_ASSERT(c2.bucket_count() == 0);
+    assert(c2.size() == 0);
+    assert(c2.hash_function() == Hash(1));
+    assert(c2.key_eq() == Compare(1));
+    assert(c2.get_allocator() == Alloc());
+    assert(static_cast<std::size_t>(std::distance(c2.begin(), c2.end())) == c2.size());
+    assert(static_cast<std::size_t>(std::distance(c2.cbegin(), c2.cend())) == c2.size());
+    assert(c2.max_load_factor() == 1);
+  }
+  {
+    typedef test_hash<int> Hash;
+    typedef test_equal_to<int> Compare;
+    typedef min_allocator<int> Alloc;
+    typedef std::unordered_multiset<int, Hash, Compare, Alloc> C;
+    typedef int P;
+    P a1[] = {P(1), P(2), P(3), P(4), P(1), P(2)};
+    C c1(std::begin(a1), std::end(a1), 0, Hash(1), Compare(1), Alloc());
+    C c2(0, Hash(2), Compare(2), Alloc());
+    c2.max_load_factor(2);
+    c1.swap(c2);
+
+    LIBCPP_ASSERT(c1.bucket_count() == 0);
+    assert(c1.size() == 0);
+    assert(c1.hash_function() == Hash(2));
+    assert(c1.key_eq() == Compare(2));
+    assert(c1.get_allocator() == Alloc());
+    assert(static_cast<std::size_t>(std::distance(c1.begin(), c1.end())) == c1.size());
+    assert(static_cast<std::size_t>(std::distance(c1.cbegin(), c1.cend())) == c1.size());
+    assert(c1.max_load_factor() == 2);
+
+    assert(c2.bucket_count() >= 6);
+    assert(c2.size() == 6);
+    assert(c2.count(1) == 2);
+    assert(c2.count(2) == 2);
+    assert(c2.count(3) == 1);
+    assert(c2.count(4) == 1);
+    assert(c2.hash_function() == Hash(1));
+    assert(c2.key_eq() == Compare(1));
+    assert(c2.get_allocator() == Alloc());
+    assert(static_cast<std::size_t>(std::distance(c2.begin(), c2.end())) == c2.size());
+    assert(static_cast<std::size_t>(std::distance(c2.cbegin(), c2.cend())) == c2.size());
+    assert(c2.max_load_factor() == 1);
+  }
+  {
+    typedef test_hash<int> Hash;
+    typedef test_equal_to<int> Compare;
+    typedef min_allocator<int> Alloc;
+    typedef std::unordered_multiset<int, Hash, Compare, Alloc> C;
+    typedef int P;
+    P a1[] = {P(1), P(2), P(3), P(4), P(1), P(2)};
+    P a2[] = {P(10), P(20), P(30), P(40), P(50), P(60), P(70), P(80)};
+    C c1(std::begin(a1), std::end(a1), 0, Hash(1), Compare(1), Alloc());
+    C c2(std::begin(a2), std::end(a2), 0, Hash(2), Compare(2), Alloc());
+    c2.max_load_factor(2);
+    c1.swap(c2);
+
+    assert(c1.bucket_count() >= 8);
+    assert(c1.size() == 8);
+    assert(*c1.find(10) == 10);
+    assert(*c1.find(20) == 20);
+    assert(*c1.find(30) == 30);
+    assert(*c1.find(40) == 40);
+    assert(*c1.find(50) == 50);
+    assert(*c1.find(60) == 60);
+    assert(*c1.find(70) == 70);
+    assert(*c1.find(80) == 80);
+    assert(c1.hash_function() == Hash(2));
+    assert(c1.key_eq() == Compare(2));
+    assert(c1.get_allocator() == Alloc());
+    assert(static_cast<std::size_t>(std::distance(c1.begin(), c1.end())) == c1.size());
+    assert(static_cast<std::size_t>(std::distance(c1.cbegin(), c1.cend())) == c1.size());
+    assert(c1.max_load_factor() == 2);
+
+    assert(c2.bucket_count() >= 6);
+    assert(c2.size() == 6);
+    assert(c2.count(1) == 2);
+    assert(c2.count(2) == 2);
+    assert(c2.count(3) == 1);
+    assert(c2.count(4) == 1);
+    assert(c2.hash_function() == Hash(1));
+    assert(c2.key_eq() == Compare(1));
+    assert(c2.get_allocator() == Alloc());
+    assert(static_cast<std::size_t>(std::distance(c2.begin(), c2.end())) == c2.size());
+    assert(static_cast<std::size_t>(std::distance(c2.cbegin(), c2.cend())) == c2.size());
+    assert(c2.max_load_factor() == 1);
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/unord/unord.multiset/types.pass.cpp b/libcxx/test/std/containers/unord/unord.multiset/types.pass.cpp
index 2c229a8680d50..4e5b0e0ce6981 100644
--- a/libcxx/test/std/containers/unord/unord.multiset/types.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.multiset/types.pass.cpp
@@ -32,39 +32,37 @@
 #include "test_macros.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef std::unordered_multiset<short> C;
-        static_assert((std::is_same<C::value_type, short>::value), "");
-        static_assert((std::is_same<C::key_type, short>::value), "");
-        static_assert((std::is_same<C::hasher, std::hash<C::key_type> >::value), "");
-        static_assert((std::is_same<C::key_equal, std::equal_to<C::key_type> >::value), "");
-        static_assert((std::is_same<C::allocator_type, std::allocator<C::value_type> >::value), "");
-        static_assert((std::is_same<C::reference, C::value_type&>::value), "");
-        static_assert((std::is_same<C::const_reference, const C::value_type&>::value), "");
-        static_assert((std::is_same<C::pointer, C::value_type*>::value), "");
-        static_assert((std::is_same<C::const_pointer, const C::value_type*>::value), "");
-        static_assert((std::is_same<C::size_type, std::size_t>::value), "");
-        static_assert((std::is_same<C::difference_type, std::ptrdiff_t>::value), "");
-    }
+int main(int, char**) {
+  {
+    typedef std::unordered_multiset<short> C;
+    static_assert((std::is_same<C::value_type, short>::value), "");
+    static_assert((std::is_same<C::key_type, short>::value), "");
+    static_assert((std::is_same<C::hasher, std::hash<C::key_type> >::value), "");
+    static_assert((std::is_same<C::key_equal, std::equal_to<C::key_type> >::value), "");
+    static_assert((std::is_same<C::allocator_type, std::allocator<C::value_type> >::value), "");
+    static_assert((std::is_same<C::reference, C::value_type&>::value), "");
+    static_assert((std::is_same<C::const_reference, const C::value_type&>::value), "");
+    static_assert((std::is_same<C::pointer, C::value_type*>::value), "");
+    static_assert((std::is_same<C::const_pointer, const C::value_type*>::value), "");
+    static_assert((std::is_same<C::size_type, std::size_t>::value), "");
+    static_assert((std::is_same<C::difference_type, std::ptrdiff_t>::value), "");
+  }
 #if TEST_STD_VER >= 11
-    {
-        typedef std::unordered_multiset<short, std::hash<short>,
-                                  std::equal_to<short>, min_allocator<short>> C;
-        static_assert((std::is_same<C::value_type, short>::value), "");
-        static_assert((std::is_same<C::key_type, short>::value), "");
-        static_assert((std::is_same<C::hasher, std::hash<C::key_type> >::value), "");
-        static_assert((std::is_same<C::key_equal, std::equal_to<C::key_type> >::value), "");
-        static_assert((std::is_same<C::allocator_type, min_allocator<C::value_type> >::value), "");
-        static_assert((std::is_same<C::reference, C::value_type&>::value), "");
-        static_assert((std::is_same<C::const_reference, const C::value_type&>::value), "");
-        static_assert((std::is_same<C::pointer, min_pointer<C::value_type>>::value), "");
-        static_assert((std::is_same<C::const_pointer, min_pointer<const C::value_type>>::value), "");
+  {
+    typedef std::unordered_multiset<short, std::hash<short>, std::equal_to<short>, min_allocator<short>> C;
+    static_assert((std::is_same<C::value_type, short>::value), "");
+    static_assert((std::is_same<C::key_type, short>::value), "");
+    static_assert((std::is_same<C::hasher, std::hash<C::key_type> >::value), "");
+    static_assert((std::is_same<C::key_equal, std::equal_to<C::key_type> >::value), "");
+    static_assert((std::is_same<C::allocator_type, min_allocator<C::value_type> >::value), "");
+    static_assert((std::is_same<C::reference, C::value_type&>::value), "");
+    static_assert((std::is_same<C::const_reference, const C::value_type&>::value), "");
+    static_assert((std::is_same<C::pointer, min_pointer<C::value_type>>::value), "");
+    static_assert((std::is_same<C::const_pointer, min_pointer<const C::value_type>>::value), "");
     //  min_allocator doesn't have a size_type, so one gets synthesized
-        static_assert((std::is_same<C::size_type, std::make_unsigned<C::difference_type>::type>::value), "");
-        static_assert((std::is_same<C::difference_type, std::ptrdiff_t>::value), "");
-    }
+    static_assert((std::is_same<C::size_type, std::make_unsigned<C::difference_type>::type>::value), "");
+    static_assert((std::is_same<C::difference_type, std::ptrdiff_t>::value), "");
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/unord/unord.multiset/unord.multiset.cnstr/allocator.pass.cpp b/libcxx/test/std/containers/unord/unord.multiset/unord.multiset.cnstr/allocator.pass.cpp
index bb2a9bf1a58e1..73b22c0a81f0e 100644
--- a/libcxx/test/std/containers/unord/unord.multiset/unord.multiset.cnstr/allocator.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.multiset/unord.multiset.cnstr/allocator.pass.cpp
@@ -24,87 +24,86 @@
 #include "test_allocator.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef std::unordered_multiset<NotConstructible,
-                                   test_hash<NotConstructible>,
-                                   test_equal_to<NotConstructible>,
-                                   test_allocator<NotConstructible>
-                                   > C;
-        C c(test_allocator<NotConstructible>(10));
-        LIBCPP_ASSERT(c.bucket_count() == 0);
-        assert(c.hash_function() == test_hash<NotConstructible>());
-        assert(c.key_eq() == test_equal_to<NotConstructible>());
-        assert(c.get_allocator() == test_allocator<NotConstructible>(10));
-        assert(c.size() == 0);
-        assert(c.empty());
-        assert(std::distance(c.begin(), c.end()) == 0);
-        assert(c.load_factor() == 0);
-        assert(c.max_load_factor() == 1);
-    }
+int main(int, char**) {
+  {
+    typedef std::unordered_multiset<NotConstructible,
+                                    test_hash<NotConstructible>,
+                                    test_equal_to<NotConstructible>,
+                                    test_allocator<NotConstructible> >
+        C;
+    C c(test_allocator<NotConstructible>(10));
+    LIBCPP_ASSERT(c.bucket_count() == 0);
+    assert(c.hash_function() == test_hash<NotConstructible>());
+    assert(c.key_eq() == test_equal_to<NotConstructible>());
+    assert(c.get_allocator() == test_allocator<NotConstructible>(10));
+    assert(c.size() == 0);
+    assert(c.empty());
+    assert(std::distance(c.begin(), c.end()) == 0);
+    assert(c.load_factor() == 0);
+    assert(c.max_load_factor() == 1);
+  }
 #if TEST_STD_VER >= 11
-    {
-        typedef std::unordered_multiset<NotConstructible,
-                                   test_hash<NotConstructible>,
-                                   test_equal_to<NotConstructible>,
-                                   min_allocator<NotConstructible>
-                                   > C;
-        C c(min_allocator<NotConstructible>{});
-        LIBCPP_ASSERT(c.bucket_count() == 0);
-        assert(c.hash_function() == test_hash<NotConstructible>());
-        assert(c.key_eq() == test_equal_to<NotConstructible>());
-        assert(c.get_allocator() == min_allocator<NotConstructible>());
-        assert(c.size() == 0);
-        assert(c.empty());
-        assert(std::distance(c.begin(), c.end()) == 0);
-        assert(c.load_factor() == 0);
-        assert(c.max_load_factor() == 1);
-    }
-#if TEST_STD_VER > 11
-    {
-        typedef NotConstructible T;
-        typedef test_hash<T> HF;
-        typedef test_equal_to<T> Comp;
-        typedef test_allocator<T> A;
-        typedef std::unordered_multiset<T, HF, Comp, A> C;
+  {
+    typedef std::unordered_multiset<NotConstructible,
+                                    test_hash<NotConstructible>,
+                                    test_equal_to<NotConstructible>,
+                                    min_allocator<NotConstructible> >
+        C;
+    C c(min_allocator<NotConstructible>{});
+    LIBCPP_ASSERT(c.bucket_count() == 0);
+    assert(c.hash_function() == test_hash<NotConstructible>());
+    assert(c.key_eq() == test_equal_to<NotConstructible>());
+    assert(c.get_allocator() == min_allocator<NotConstructible>());
+    assert(c.size() == 0);
+    assert(c.empty());
+    assert(std::distance(c.begin(), c.end()) == 0);
+    assert(c.load_factor() == 0);
+    assert(c.max_load_factor() == 1);
+  }
+#  if TEST_STD_VER > 11
+  {
+    typedef NotConstructible T;
+    typedef test_hash<T> HF;
+    typedef test_equal_to<T> Comp;
+    typedef test_allocator<T> A;
+    typedef std::unordered_multiset<T, HF, Comp, A> C;
 
-        A a(43);
-        C c(3, a);
-        LIBCPP_ASSERT(c.bucket_count() == 3);
-        assert(c.hash_function() == HF());
-        assert(c.key_eq() == Comp ());
-        assert(c.get_allocator() == a);
-        assert(!(c.get_allocator() == A()));
-        assert(c.size() == 0);
-        assert(c.empty());
-        assert(std::distance(c.begin(), c.end()) == 0);
-        assert(c.load_factor() == 0);
-        assert(c.max_load_factor() == 1);
-    }
-    {
-        typedef NotConstructible T;
-        typedef test_hash<T> HF;
-        typedef test_equal_to<T> Comp;
-        typedef test_allocator<T> A;
-        typedef std::unordered_multiset<T, HF, Comp, A> C;
+    A a(43);
+    C c(3, a);
+    LIBCPP_ASSERT(c.bucket_count() == 3);
+    assert(c.hash_function() == HF());
+    assert(c.key_eq() == Comp());
+    assert(c.get_allocator() == a);
+    assert(!(c.get_allocator() == A()));
+    assert(c.size() == 0);
+    assert(c.empty());
+    assert(std::distance(c.begin(), c.end()) == 0);
+    assert(c.load_factor() == 0);
+    assert(c.max_load_factor() == 1);
+  }
+  {
+    typedef NotConstructible T;
+    typedef test_hash<T> HF;
+    typedef test_equal_to<T> Comp;
+    typedef test_allocator<T> A;
+    typedef std::unordered_multiset<T, HF, Comp, A> C;
 
-        HF hf(42);
-        A a(43);
-        C c(4, hf, a);
-        LIBCPP_ASSERT(c.bucket_count() == 4);
-        assert(c.hash_function() == hf);
-        assert(!(c.hash_function() == HF()));
-        assert(c.key_eq() == Comp ());
-        assert(c.get_allocator() == a);
-        assert(!(c.get_allocator() == A()));
-        assert(c.size() == 0);
-        assert(c.empty());
-        assert(std::distance(c.begin(), c.end()) == 0);
-        assert(c.load_factor() == 0);
-        assert(c.max_load_factor() == 1);
-    }
-#endif
+    HF hf(42);
+    A a(43);
+    C c(4, hf, a);
+    LIBCPP_ASSERT(c.bucket_count() == 4);
+    assert(c.hash_function() == hf);
+    assert(!(c.hash_function() == HF()));
+    assert(c.key_eq() == Comp());
+    assert(c.get_allocator() == a);
+    assert(!(c.get_allocator() == A()));
+    assert(c.size() == 0);
+    assert(c.empty());
+    assert(std::distance(c.begin(), c.end()) == 0);
+    assert(c.load_factor() == 0);
+    assert(c.max_load_factor() == 1);
+  }
+#  endif
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/unord/unord.multiset/unord.multiset.cnstr/assign_copy.pass.cpp b/libcxx/test/std/containers/unord/unord.multiset/unord.multiset.cnstr/assign_copy.pass.cpp
index 91aa0f4d390cd..248d446b92eb7 100644
--- a/libcxx/test/std/containers/unord/unord.multiset/unord.multiset.cnstr/assign_copy.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.multiset/unord.multiset.cnstr/assign_copy.pass.cpp
@@ -28,163 +28,88 @@
 #include "test_allocator.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef test_allocator<int> A;
-        typedef std::unordered_multiset<int,
-                                   test_hash<int>,
-                                   test_equal_to<int>,
-                                   A
-                                   > C;
-        typedef int P;
-        P a[] =
-        {
-            P(1),
-            P(2),
-            P(3),
-            P(4),
-            P(1),
-            P(2)
-        };
-        C c0(a, a + sizeof(a)/sizeof(a[0]),
-            7,
-            test_hash<int>(8),
-            test_equal_to<int>(9),
-            A(10)
-           );
-        C c(a, a + 2,
-            7,
-            test_hash<int>(2),
-            test_equal_to<int>(3),
-            A(4)
-           );
-        c = c0;
-        LIBCPP_ASSERT(c.bucket_count() == 7);
-        assert(c.size() == 6);
-        CheckConsecutiveValues<C::const_iterator>(c.find(1), c.end(), 1, 2);
-        CheckConsecutiveValues<C::const_iterator>(c.find(2), c.end(), 2, 2);
-        CheckConsecutiveValues<C::const_iterator>(c.find(3), c.end(), 3, 1);
-        CheckConsecutiveValues<C::const_iterator>(c.find(4), c.end(), 4, 1);
-        assert(c.hash_function() == test_hash<int>(8));
-        assert(c.key_eq() == test_equal_to<int>(9));
-        assert(c.get_allocator() == A(4));
-        assert(!c.empty());
-        assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-        assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
-        assert(fabs(c.load_factor() - (float)c.size()/c.bucket_count()) < FLT_EPSILON);
-        assert(c.max_load_factor() == 1);
-    }
-    {
-        typedef std::unordered_multiset<int> C;
-        typedef int P;
-        P a[] =
-        {
-            P(1),
-            P(2),
-            P(3),
-            P(4),
-            P(1),
-            P(2)
-        };
-        C c(a, a + sizeof(a)/sizeof(a[0]));
-        C *p = &c;
-        c = *p;
+int main(int, char**) {
+  {
+    typedef test_allocator<int> A;
+    typedef std::unordered_multiset<int, test_hash<int>, test_equal_to<int>, A > C;
+    typedef int P;
+    P a[] = {P(1), P(2), P(3), P(4), P(1), P(2)};
+    C c0(a, a + sizeof(a) / sizeof(a[0]), 7, test_hash<int>(8), test_equal_to<int>(9), A(10));
+    C c(a, a + 2, 7, test_hash<int>(2), test_equal_to<int>(3), A(4));
+    c = c0;
+    LIBCPP_ASSERT(c.bucket_count() == 7);
+    assert(c.size() == 6);
+    CheckConsecutiveValues<C::const_iterator>(c.find(1), c.end(), 1, 2);
+    CheckConsecutiveValues<C::const_iterator>(c.find(2), c.end(), 2, 2);
+    CheckConsecutiveValues<C::const_iterator>(c.find(3), c.end(), 3, 1);
+    CheckConsecutiveValues<C::const_iterator>(c.find(4), c.end(), 4, 1);
+    assert(c.hash_function() == test_hash<int>(8));
+    assert(c.key_eq() == test_equal_to<int>(9));
+    assert(c.get_allocator() == A(4));
+    assert(!c.empty());
+    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
+    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
+    assert(fabs(c.load_factor() - (float)c.size() / c.bucket_count()) < FLT_EPSILON);
+    assert(c.max_load_factor() == 1);
+  }
+  {
+    typedef std::unordered_multiset<int> C;
+    typedef int P;
+    P a[] = {P(1), P(2), P(3), P(4), P(1), P(2)};
+    C c(a, a + sizeof(a) / sizeof(a[0]));
+    C* p = &c;
+    c    = *p;
 
-        assert(c.size() == 6);
-        assert(std::is_permutation(c.begin(), c.end(), a));
-    }
-    {
-        typedef other_allocator<int> A;
-        typedef std::unordered_multiset<int,
-                                   test_hash<int>,
-                                   test_equal_to<int>,
-                                   A
-                                   > C;
-        typedef int P;
-        P a[] =
-        {
-            P(1),
-            P(2),
-            P(3),
-            P(4),
-            P(1),
-            P(2)
-        };
-        C c0(a, a + sizeof(a)/sizeof(a[0]),
-            7,
-            test_hash<int>(8),
-            test_equal_to<int>(9),
-            A(10)
-           );
-        C c(a, a + 2,
-            7,
-            test_hash<int>(2),
-            test_equal_to<int>(3),
-            A(4)
-           );
-        c = c0;
-        assert(c.bucket_count() >= 7);
-        assert(c.size() == 6);
-        CheckConsecutiveValues<C::const_iterator>(c.find(1), c.end(), 1, 2);
-        CheckConsecutiveValues<C::const_iterator>(c.find(2), c.end(), 2, 2);
-        CheckConsecutiveValues<C::const_iterator>(c.find(3), c.end(), 3, 1);
-        CheckConsecutiveValues<C::const_iterator>(c.find(4), c.end(), 4, 1);
-        assert(c.hash_function() == test_hash<int>(8));
-        assert(c.key_eq() == test_equal_to<int>(9));
-        assert(c.get_allocator() == A(10));
-        assert(!c.empty());
-        assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-        assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
-        assert(fabs(c.load_factor() - (float)c.size()/c.bucket_count()) < FLT_EPSILON);
-        assert(c.max_load_factor() == 1);
-    }
+    assert(c.size() == 6);
+    assert(std::is_permutation(c.begin(), c.end(), a));
+  }
+  {
+    typedef other_allocator<int> A;
+    typedef std::unordered_multiset<int, test_hash<int>, test_equal_to<int>, A > C;
+    typedef int P;
+    P a[] = {P(1), P(2), P(3), P(4), P(1), P(2)};
+    C c0(a, a + sizeof(a) / sizeof(a[0]), 7, test_hash<int>(8), test_equal_to<int>(9), A(10));
+    C c(a, a + 2, 7, test_hash<int>(2), test_equal_to<int>(3), A(4));
+    c = c0;
+    assert(c.bucket_count() >= 7);
+    assert(c.size() == 6);
+    CheckConsecutiveValues<C::const_iterator>(c.find(1), c.end(), 1, 2);
+    CheckConsecutiveValues<C::const_iterator>(c.find(2), c.end(), 2, 2);
+    CheckConsecutiveValues<C::const_iterator>(c.find(3), c.end(), 3, 1);
+    CheckConsecutiveValues<C::const_iterator>(c.find(4), c.end(), 4, 1);
+    assert(c.hash_function() == test_hash<int>(8));
+    assert(c.key_eq() == test_equal_to<int>(9));
+    assert(c.get_allocator() == A(10));
+    assert(!c.empty());
+    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
+    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
+    assert(fabs(c.load_factor() - (float)c.size() / c.bucket_count()) < FLT_EPSILON);
+    assert(c.max_load_factor() == 1);
+  }
 #if TEST_STD_VER >= 11
-    {
-        typedef min_allocator<int> A;
-        typedef std::unordered_multiset<int,
-                                   test_hash<int>,
-                                   test_equal_to<int>,
-                                   A
-                                   > C;
-        typedef int P;
-        P a[] =
-        {
-            P(1),
-            P(2),
-            P(3),
-            P(4),
-            P(1),
-            P(2)
-        };
-        C c0(a, a + sizeof(a)/sizeof(a[0]),
-            7,
-            test_hash<int>(8),
-            test_equal_to<int>(9),
-            A()
-           );
-        C c(a, a + 2,
-            7,
-            test_hash<int>(2),
-            test_equal_to<int>(3),
-            A()
-           );
-        c = c0;
-        LIBCPP_ASSERT(c.bucket_count() == 7);
-        assert(c.size() == 6);
-        CheckConsecutiveValues<C::const_iterator>(c.find(1), c.end(), 1, 2);
-        CheckConsecutiveValues<C::const_iterator>(c.find(2), c.end(), 2, 2);
-        CheckConsecutiveValues<C::const_iterator>(c.find(3), c.end(), 3, 1);
-        CheckConsecutiveValues<C::const_iterator>(c.find(4), c.end(), 4, 1);
-        assert(c.hash_function() == test_hash<int>(8));
-        assert(c.key_eq() == test_equal_to<int>(9));
-        assert(c.get_allocator() == A());
-        assert(!c.empty());
-        assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-        assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
-        assert(fabs(c.load_factor() - (float)c.size()/c.bucket_count()) < FLT_EPSILON);
-        assert(c.max_load_factor() == 1);
-    }
+  {
+    typedef min_allocator<int> A;
+    typedef std::unordered_multiset<int, test_hash<int>, test_equal_to<int>, A > C;
+    typedef int P;
+    P a[] = {P(1), P(2), P(3), P(4), P(1), P(2)};
+    C c0(a, a + sizeof(a) / sizeof(a[0]), 7, test_hash<int>(8), test_equal_to<int>(9), A());
+    C c(a, a + 2, 7, test_hash<int>(2), test_equal_to<int>(3), A());
+    c = c0;
+    LIBCPP_ASSERT(c.bucket_count() == 7);
+    assert(c.size() == 6);
+    CheckConsecutiveValues<C::const_iterator>(c.find(1), c.end(), 1, 2);
+    CheckConsecutiveValues<C::const_iterator>(c.find(2), c.end(), 2, 2);
+    CheckConsecutiveValues<C::const_iterator>(c.find(3), c.end(), 3, 1);
+    CheckConsecutiveValues<C::const_iterator>(c.find(4), c.end(), 4, 1);
+    assert(c.hash_function() == test_hash<int>(8));
+    assert(c.key_eq() == test_equal_to<int>(9));
+    assert(c.get_allocator() == A());
+    assert(!c.empty());
+    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
+    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
+    assert(fabs(c.load_factor() - (float)c.size() / c.bucket_count()) < FLT_EPSILON);
+    assert(c.max_load_factor() == 1);
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/unord/unord.multiset/unord.multiset.cnstr/assign_init.pass.cpp b/libcxx/test/std/containers/unord/unord.multiset/unord.multiset.cnstr/assign_init.pass.cpp
index b27e535cf16ce..840991a6095b1 100644
--- a/libcxx/test/std/containers/unord/unord.multiset/unord.multiset.cnstr/assign_init.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.multiset/unord.multiset.cnstr/assign_init.pass.cpp
@@ -28,72 +28,41 @@
 #include "test_allocator.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef test_allocator<int> A;
-        typedef std::unordered_multiset<int,
-                                   test_hash<int>,
-                                   test_equal_to<int>,
-                                   A
-                                   > C;
-        typedef int P;
-        C c =   {
-                    P(4),
-                    P(1),
-                    P(2)
-                };
-        c =     {
-                    P(1),
-                    P(2),
-                    P(3),
-                    P(4),
-                    P(1),
-                    P(2)
-                };
-        assert(c.bucket_count() >= 7);
-        assert(c.size() == 6);
-        assert(c.count(1) == 2);
-        assert(c.count(2) == 2);
-        assert(c.count(3) == 1);
-        assert(c.count(4) == 1);
-        assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-        assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
-        assert(fabs(c.load_factor() - (float)c.size()/c.bucket_count()) < FLT_EPSILON);
-        assert(c.max_load_factor() == 1);
-    }
-    {
-        typedef min_allocator<int> A;
-        typedef std::unordered_multiset<int,
-                                   test_hash<int>,
-                                   test_equal_to<int>,
-                                   A
-                                   > C;
-        typedef int P;
-        C c =   {
-                    P(4),
-                    P(1),
-                    P(2)
-                };
-        c =     {
-                    P(1),
-                    P(2),
-                    P(3),
-                    P(4),
-                    P(1),
-                    P(2)
-                };
-        assert(c.bucket_count() >= 7);
-        assert(c.size() == 6);
-        assert(c.count(1) == 2);
-        assert(c.count(2) == 2);
-        assert(c.count(3) == 1);
-        assert(c.count(4) == 1);
-        assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-        assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
-        assert(fabs(c.load_factor() - (float)c.size()/c.bucket_count()) < FLT_EPSILON);
-        assert(c.max_load_factor() == 1);
-    }
+int main(int, char**) {
+  {
+    typedef test_allocator<int> A;
+    typedef std::unordered_multiset<int, test_hash<int>, test_equal_to<int>, A > C;
+    typedef int P;
+    C c = {P(4), P(1), P(2)};
+    c   = {P(1), P(2), P(3), P(4), P(1), P(2)};
+    assert(c.bucket_count() >= 7);
+    assert(c.size() == 6);
+    assert(c.count(1) == 2);
+    assert(c.count(2) == 2);
+    assert(c.count(3) == 1);
+    assert(c.count(4) == 1);
+    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
+    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
+    assert(fabs(c.load_factor() - (float)c.size() / c.bucket_count()) < FLT_EPSILON);
+    assert(c.max_load_factor() == 1);
+  }
+  {
+    typedef min_allocator<int> A;
+    typedef std::unordered_multiset<int, test_hash<int>, test_equal_to<int>, A > C;
+    typedef int P;
+    C c = {P(4), P(1), P(2)};
+    c   = {P(1), P(2), P(3), P(4), P(1), P(2)};
+    assert(c.bucket_count() >= 7);
+    assert(c.size() == 6);
+    assert(c.count(1) == 2);
+    assert(c.count(2) == 2);
+    assert(c.count(3) == 1);
+    assert(c.count(4) == 1);
+    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
+    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
+    assert(fabs(c.load_factor() - (float)c.size() / c.bucket_count()) < FLT_EPSILON);
+    assert(c.max_load_factor() == 1);
+  }
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/unord/unord.multiset/unord.multiset.cnstr/assign_move.pass.cpp b/libcxx/test/std/containers/unord/unord.multiset/unord.multiset.cnstr/assign_move.pass.cpp
index 32575f4eec23e..c07d02ebb067a 100644
--- a/libcxx/test/std/containers/unord/unord.multiset/unord.multiset.cnstr/assign_move.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.multiset/unord.multiset.cnstr/assign_move.pass.cpp
@@ -29,241 +29,130 @@
 #include "test_allocator.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef test_allocator<int> A;
-        typedef std::unordered_multiset<int,
-                                   test_hash<int>,
-                                   test_equal_to<int>,
-                                   A
-                                   > C;
-        typedef int P;
-        P a[] =
-        {
-            P(1),
-            P(2),
-            P(3),
-            P(4),
-            P(1),
-            P(2)
-        };
-        C c0(a, a + sizeof(a)/sizeof(a[0]),
-            7,
-            test_hash<int>(8),
-            test_equal_to<int>(9),
-            A(10)
-           );
-        C c(a, a + 2,
-            7,
-            test_hash<int>(2),
-            test_equal_to<int>(3),
-            A(4)
-           );
-        c = std::move(c0);
-        LIBCPP_ASSERT(c.bucket_count() == 7);
-        assert(c.size() == 6);
-        CheckConsecutiveValues<C::const_iterator>(c.find(1), c.end(), 1, 2);
-        CheckConsecutiveValues<C::const_iterator>(c.find(2), c.end(), 2, 2);
-        CheckConsecutiveValues<C::const_iterator>(c.find(3), c.end(), 3, 1);
-        CheckConsecutiveValues<C::const_iterator>(c.find(4), c.end(), 4, 1);
-        assert(c.hash_function() == test_hash<int>(8));
-        assert(c.key_eq() == test_equal_to<int>(9));
-        assert(c.get_allocator() == A(4));
-        assert(!c.empty());
-        assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-        assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
-        assert(fabs(c.load_factor() - (float)c.size()/c.bucket_count()) < FLT_EPSILON);
-        assert(c.max_load_factor() == 1);
-    }
-    {
-        typedef test_allocator<int> A;
-        typedef std::unordered_multiset<int,
-                                   test_hash<int>,
-                                   test_equal_to<int>,
-                                   A
-                                   > C;
-        typedef int P;
-        P a[] =
-        {
-            P(1),
-            P(2),
-            P(3),
-            P(4),
-            P(1),
-            P(2)
-        };
-        C c0(a, a + sizeof(a)/sizeof(a[0]),
-            7,
-            test_hash<int>(8),
-            test_equal_to<int>(9),
-            A(10)
-           );
-        C c(a, a + 2,
-            7,
-            test_hash<int>(2),
-            test_equal_to<int>(3),
-            A(10)
-           );
-        C::iterator it0 = c0.begin();
-        c = std::move(c0);
-        LIBCPP_ASSERT(c.bucket_count() == 7);
-        assert(c.size() == 6);
-        assert(c.count(1) == 2);
-        assert(c.count(2) == 2);
-        assert(c.count(3) == 1);
-        assert(c.count(4) == 1);
-        assert(c.hash_function() == test_hash<int>(8));
-        assert(c.key_eq() == test_equal_to<int>(9));
-        assert(c.get_allocator() == A(10));
-        assert(!c.empty());
-        assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-        assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
-        assert(fabs(c.load_factor() - (float)c.size()/c.bucket_count()) < FLT_EPSILON);
-        assert(c.max_load_factor() == 1);
-        assert(it0 == c.begin()); // Iterators remain valid
-    }
-    {
-        typedef other_allocator<int> A;
-        typedef std::unordered_multiset<int,
-                                   test_hash<int>,
-                                   test_equal_to<int>,
-                                   A
-                                   > C;
-        typedef int P;
-        P a[] =
-        {
-            P(1),
-            P(2),
-            P(3),
-            P(4),
-            P(1),
-            P(2)
-        };
-        C c0(a, a + sizeof(a)/sizeof(a[0]),
-            7,
-            test_hash<int>(8),
-            test_equal_to<int>(9),
-            A(10)
-           );
-        C c(a, a + 2,
-            7,
-            test_hash<int>(2),
-            test_equal_to<int>(3),
-            A(4)
-           );
-        C::iterator it0 = c0.begin();
-        c = std::move(c0);
-        LIBCPP_ASSERT(c.bucket_count() == 7);
-        assert(c.size() == 6);
-        assert(c.count(1) == 2);
-        assert(c.count(2) == 2);
-        assert(c.count(3) == 1);
-        assert(c.count(4) == 1);
-        assert(c.hash_function() == test_hash<int>(8));
-        assert(c.key_eq() == test_equal_to<int>(9));
-        assert(c.get_allocator() == A(10));
-        assert(!c.empty());
-        assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-        assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
-        assert(fabs(c.load_factor() - (float)c.size()/c.bucket_count()) < FLT_EPSILON);
-        assert(c.max_load_factor() == 1);
-        assert(it0 == c.begin()); // Iterators remain valid
-    }
-    {
-        typedef test_allocator<int> A;
-        typedef std::unordered_multiset<int,
-                                   test_hash<int>,
-                                   test_equal_to<int>,
-                                   A
-                                   > C;
-        typedef int P;
-        P a[] =
-        {
-            P(1),
-            P(2),
-            P(3),
-            P(4),
-            P(1),
-            P(2)
-        };
-        C c0(a, a + sizeof(a)/sizeof(a[0]),
-            7,
-            test_hash<int>(8),
-            test_equal_to<int>(9),
-            A()
-           );
-        C c(a, a + 2,
-            7,
-            test_hash<int>(2),
-            test_equal_to<int>(3),
-            A()
-           );
-        C::iterator it0 = c0.begin();
-        c = std::move(c0);
-        LIBCPP_ASSERT(c.bucket_count() == 7);
-        assert(c.size() == 6);
-        assert(c.count(1) == 2);
-        assert(c.count(2) == 2);
-        assert(c.count(3) == 1);
-        assert(c.count(4) == 1);
-        assert(c.hash_function() == test_hash<int>(8));
-        assert(c.key_eq() == test_equal_to<int>(9));
-        assert(c.get_allocator() == A());
-        assert(!c.empty());
-        assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-        assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
-        assert(fabs(c.load_factor() - (float)c.size()/c.bucket_count()) < FLT_EPSILON);
-        assert(c.max_load_factor() == 1);
-        assert(it0 == c.begin()); // Iterators remain valid
-    }
-    {
-        typedef min_allocator<int> A;
-        typedef std::unordered_multiset<int,
-                                   test_hash<int>,
-                                   test_equal_to<int>,
-                                   A
-                                   > C;
-        typedef int P;
-        P a[] =
-        {
-            P(1),
-            P(2),
-            P(3),
-            P(4),
-            P(1),
-            P(2)
-        };
-        C c0(a, a + sizeof(a)/sizeof(a[0]),
-            7,
-            test_hash<int>(8),
-            test_equal_to<int>(9),
-            A()
-           );
-        C c(a, a + 2,
-            7,
-            test_hash<int>(2),
-            test_equal_to<int>(3),
-            A()
-           );
-        C::iterator it0 = c0.begin();
-        c = std::move(c0);
-        LIBCPP_ASSERT(c.bucket_count() == 7);
-        assert(c.size() == 6);
-        assert(c.count(1) == 2);
-        assert(c.count(2) == 2);
-        assert(c.count(3) == 1);
-        assert(c.count(4) == 1);
-        assert(c.hash_function() == test_hash<int>(8));
-        assert(c.key_eq() == test_equal_to<int>(9));
-        assert(c.get_allocator() == A());
-        assert(!c.empty());
-        assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-        assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
-        assert(fabs(c.load_factor() - (float)c.size()/c.bucket_count()) < FLT_EPSILON);
-        assert(c.max_load_factor() == 1);
-        assert(it0 == c.begin()); // Iterators remain valid
-    }
+int main(int, char**) {
+  {
+    typedef test_allocator<int> A;
+    typedef std::unordered_multiset<int, test_hash<int>, test_equal_to<int>, A > C;
+    typedef int P;
+    P a[] = {P(1), P(2), P(3), P(4), P(1), P(2)};
+    C c0(a, a + sizeof(a) / sizeof(a[0]), 7, test_hash<int>(8), test_equal_to<int>(9), A(10));
+    C c(a, a + 2, 7, test_hash<int>(2), test_equal_to<int>(3), A(4));
+    c = std::move(c0);
+    LIBCPP_ASSERT(c.bucket_count() == 7);
+    assert(c.size() == 6);
+    CheckConsecutiveValues<C::const_iterator>(c.find(1), c.end(), 1, 2);
+    CheckConsecutiveValues<C::const_iterator>(c.find(2), c.end(), 2, 2);
+    CheckConsecutiveValues<C::const_iterator>(c.find(3), c.end(), 3, 1);
+    CheckConsecutiveValues<C::const_iterator>(c.find(4), c.end(), 4, 1);
+    assert(c.hash_function() == test_hash<int>(8));
+    assert(c.key_eq() == test_equal_to<int>(9));
+    assert(c.get_allocator() == A(4));
+    assert(!c.empty());
+    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
+    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
+    assert(fabs(c.load_factor() - (float)c.size() / c.bucket_count()) < FLT_EPSILON);
+    assert(c.max_load_factor() == 1);
+  }
+  {
+    typedef test_allocator<int> A;
+    typedef std::unordered_multiset<int, test_hash<int>, test_equal_to<int>, A > C;
+    typedef int P;
+    P a[] = {P(1), P(2), P(3), P(4), P(1), P(2)};
+    C c0(a, a + sizeof(a) / sizeof(a[0]), 7, test_hash<int>(8), test_equal_to<int>(9), A(10));
+    C c(a, a + 2, 7, test_hash<int>(2), test_equal_to<int>(3), A(10));
+    C::iterator it0 = c0.begin();
+    c               = std::move(c0);
+    LIBCPP_ASSERT(c.bucket_count() == 7);
+    assert(c.size() == 6);
+    assert(c.count(1) == 2);
+    assert(c.count(2) == 2);
+    assert(c.count(3) == 1);
+    assert(c.count(4) == 1);
+    assert(c.hash_function() == test_hash<int>(8));
+    assert(c.key_eq() == test_equal_to<int>(9));
+    assert(c.get_allocator() == A(10));
+    assert(!c.empty());
+    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
+    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
+    assert(fabs(c.load_factor() - (float)c.size() / c.bucket_count()) < FLT_EPSILON);
+    assert(c.max_load_factor() == 1);
+    assert(it0 == c.begin()); // Iterators remain valid
+  }
+  {
+    typedef other_allocator<int> A;
+    typedef std::unordered_multiset<int, test_hash<int>, test_equal_to<int>, A > C;
+    typedef int P;
+    P a[] = {P(1), P(2), P(3), P(4), P(1), P(2)};
+    C c0(a, a + sizeof(a) / sizeof(a[0]), 7, test_hash<int>(8), test_equal_to<int>(9), A(10));
+    C c(a, a + 2, 7, test_hash<int>(2), test_equal_to<int>(3), A(4));
+    C::iterator it0 = c0.begin();
+    c               = std::move(c0);
+    LIBCPP_ASSERT(c.bucket_count() == 7);
+    assert(c.size() == 6);
+    assert(c.count(1) == 2);
+    assert(c.count(2) == 2);
+    assert(c.count(3) == 1);
+    assert(c.count(4) == 1);
+    assert(c.hash_function() == test_hash<int>(8));
+    assert(c.key_eq() == test_equal_to<int>(9));
+    assert(c.get_allocator() == A(10));
+    assert(!c.empty());
+    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
+    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
+    assert(fabs(c.load_factor() - (float)c.size() / c.bucket_count()) < FLT_EPSILON);
+    assert(c.max_load_factor() == 1);
+    assert(it0 == c.begin()); // Iterators remain valid
+  }
+  {
+    typedef test_allocator<int> A;
+    typedef std::unordered_multiset<int, test_hash<int>, test_equal_to<int>, A > C;
+    typedef int P;
+    P a[] = {P(1), P(2), P(3), P(4), P(1), P(2)};
+    C c0(a, a + sizeof(a) / sizeof(a[0]), 7, test_hash<int>(8), test_equal_to<int>(9), A());
+    C c(a, a + 2, 7, test_hash<int>(2), test_equal_to<int>(3), A());
+    C::iterator it0 = c0.begin();
+    c               = std::move(c0);
+    LIBCPP_ASSERT(c.bucket_count() == 7);
+    assert(c.size() == 6);
+    assert(c.count(1) == 2);
+    assert(c.count(2) == 2);
+    assert(c.count(3) == 1);
+    assert(c.count(4) == 1);
+    assert(c.hash_function() == test_hash<int>(8));
+    assert(c.key_eq() == test_equal_to<int>(9));
+    assert(c.get_allocator() == A());
+    assert(!c.empty());
+    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
+    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
+    assert(fabs(c.load_factor() - (float)c.size() / c.bucket_count()) < FLT_EPSILON);
+    assert(c.max_load_factor() == 1);
+    assert(it0 == c.begin()); // Iterators remain valid
+  }
+  {
+    typedef min_allocator<int> A;
+    typedef std::unordered_multiset<int, test_hash<int>, test_equal_to<int>, A > C;
+    typedef int P;
+    P a[] = {P(1), P(2), P(3), P(4), P(1), P(2)};
+    C c0(a, a + sizeof(a) / sizeof(a[0]), 7, test_hash<int>(8), test_equal_to<int>(9), A());
+    C c(a, a + 2, 7, test_hash<int>(2), test_equal_to<int>(3), A());
+    C::iterator it0 = c0.begin();
+    c               = std::move(c0);
+    LIBCPP_ASSERT(c.bucket_count() == 7);
+    assert(c.size() == 6);
+    assert(c.count(1) == 2);
+    assert(c.count(2) == 2);
+    assert(c.count(3) == 1);
+    assert(c.count(4) == 1);
+    assert(c.hash_function() == test_hash<int>(8));
+    assert(c.key_eq() == test_equal_to<int>(9));
+    assert(c.get_allocator() == A());
+    assert(!c.empty());
+    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
+    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
+    assert(fabs(c.load_factor() - (float)c.size() / c.bucket_count()) < FLT_EPSILON);
+    assert(c.max_load_factor() == 1);
+    assert(it0 == c.begin()); // Iterators remain valid
+  }
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/unord/unord.multiset/unord.multiset.cnstr/compare_copy_constructible.compile.fail.cpp b/libcxx/test/std/containers/unord/unord.multiset/unord.multiset.cnstr/compare_copy_constructible.compile.fail.cpp
index 24816ab09e25a..b70892459485b 100644
--- a/libcxx/test/std/containers/unord/unord.multiset/unord.multiset.cnstr/compare_copy_constructible.compile.fail.cpp
+++ b/libcxx/test/std/containers/unord/unord.multiset/unord.multiset.cnstr/compare_copy_constructible.compile.fail.cpp
@@ -6,7 +6,6 @@
 //
 //===----------------------------------------------------------------------===//
 
-
 // <unordered_set>
 
 // Check that std::unordered_set fails to instantiate if the comparison predicate is
@@ -16,16 +15,16 @@
 
 template <class T>
 struct Comp {
-    bool operator () (const T& lhs, const T& rhs) const { return lhs == rhs; }
+  bool operator()(const T& lhs, const T& rhs) const { return lhs == rhs; }
 
-    Comp () {}
-private:
-    Comp (const Comp &); // declared but not defined
-    };
+  Comp() {}
 
+private:
+  Comp(const Comp&); // declared but not defined
+};
 
 int main(int, char**) {
-    std::unordered_multiset<int, std::hash<int>, Comp<int> > m;
+  std::unordered_multiset<int, std::hash<int>, Comp<int> > m;
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/unord/unord.multiset/unord.multiset.cnstr/copy.pass.cpp b/libcxx/test/std/containers/unord/unord.multiset/unord.multiset.cnstr/copy.pass.cpp
index 8ae0be01eb5b5..fc577f323e43b 100644
--- a/libcxx/test/std/containers/unord/unord.multiset/unord.multiset.cnstr/copy.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.multiset/unord.multiset.cnstr/copy.pass.cpp
@@ -27,123 +27,71 @@
 #include "test_allocator.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef std::unordered_multiset<int,
-                                   test_hash<int>,
-                                   test_equal_to<int>,
-                                   test_allocator<int>
-                                   > C;
-        typedef int P;
-        P a[] =
-        {
-            P(1),
-            P(2),
-            P(3),
-            P(4),
-            P(1),
-            P(2)
-        };
-        C c0(a, a + sizeof(a)/sizeof(a[0]),
-            7,
-            test_hash<int>(8),
-            test_equal_to<int>(9),
-            test_allocator<int>(10)
-           );
-        C c = c0;
-        LIBCPP_ASSERT(c.bucket_count() == 7);
-        assert(c.size() == 6);
-        CheckConsecutiveValues<C::const_iterator>(c.find(1), c.end(), 1, 2);
-        CheckConsecutiveValues<C::const_iterator>(c.find(2), c.end(), 2, 2);
-        CheckConsecutiveValues<C::const_iterator>(c.find(3), c.end(), 3, 1);
-        CheckConsecutiveValues<C::const_iterator>(c.find(4), c.end(), 4, 1);
-        assert(c.hash_function() == test_hash<int>(8));
-        assert(c.key_eq() == test_equal_to<int>(9));
-        assert(c.get_allocator() == test_allocator<int>(10));
-        assert(!c.empty());
-        assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-        assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
-        assert(std::fabs(c.load_factor() - (float)c.size()/c.bucket_count()) < FLT_EPSILON);
-        assert(c.max_load_factor() == 1);
-    }
+int main(int, char**) {
+  {
+    typedef std::unordered_multiset<int, test_hash<int>, test_equal_to<int>, test_allocator<int> > C;
+    typedef int P;
+    P a[] = {P(1), P(2), P(3), P(4), P(1), P(2)};
+    C c0(a, a + sizeof(a) / sizeof(a[0]), 7, test_hash<int>(8), test_equal_to<int>(9), test_allocator<int>(10));
+    C c = c0;
+    LIBCPP_ASSERT(c.bucket_count() == 7);
+    assert(c.size() == 6);
+    CheckConsecutiveValues<C::const_iterator>(c.find(1), c.end(), 1, 2);
+    CheckConsecutiveValues<C::const_iterator>(c.find(2), c.end(), 2, 2);
+    CheckConsecutiveValues<C::const_iterator>(c.find(3), c.end(), 3, 1);
+    CheckConsecutiveValues<C::const_iterator>(c.find(4), c.end(), 4, 1);
+    assert(c.hash_function() == test_hash<int>(8));
+    assert(c.key_eq() == test_equal_to<int>(9));
+    assert(c.get_allocator() == test_allocator<int>(10));
+    assert(!c.empty());
+    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
+    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
+    assert(std::fabs(c.load_factor() - (float)c.size() / c.bucket_count()) < FLT_EPSILON);
+    assert(c.max_load_factor() == 1);
+  }
 #if TEST_STD_VER >= 11
-    {
-        typedef std::unordered_multiset<int,
-                                   test_hash<int>,
-                                   test_equal_to<int>,
-                                   other_allocator<int>
-                                   > C;
-        typedef int P;
-        P a[] =
-        {
-            P(1),
-            P(2),
-            P(3),
-            P(4),
-            P(1),
-            P(2)
-        };
-        C c0(a, a + sizeof(a)/sizeof(a[0]),
-            7,
-            test_hash<int>(8),
-            test_equal_to<int>(9),
-            other_allocator<int>(10)
-           );
-        C c = c0;
-        LIBCPP_ASSERT(c.bucket_count() == 7);
-        assert(c.size() == 6);
-        CheckConsecutiveValues<C::const_iterator>(c.find(1), c.end(), 1, 2);
-        CheckConsecutiveValues<C::const_iterator>(c.find(2), c.end(), 2, 2);
-        CheckConsecutiveValues<C::const_iterator>(c.find(3), c.end(), 3, 1);
-        CheckConsecutiveValues<C::const_iterator>(c.find(4), c.end(), 4, 1);
-        assert(c.hash_function() == test_hash<int>(8));
-        assert(c.key_eq() == test_equal_to<int>(9));
-        assert(c.get_allocator() == other_allocator<int>(-2));
-        assert(!c.empty());
-        assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-        assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
-        assert(std::fabs(c.load_factor() - (float)c.size()/c.bucket_count()) < FLT_EPSILON);
-        assert(c.max_load_factor() == 1);
-    }
-    {
-        typedef std::unordered_multiset<int,
-                                   test_hash<int>,
-                                   test_equal_to<int>,
-                                   min_allocator<int>
-                                   > C;
-        typedef int P;
-        P a[] =
-        {
-            P(1),
-            P(2),
-            P(3),
-            P(4),
-            P(1),
-            P(2)
-        };
-        C c0(a, a + sizeof(a)/sizeof(a[0]),
-            7,
-            test_hash<int>(8),
-            test_equal_to<int>(9),
-            min_allocator<int>()
-           );
-        C c = c0;
-        LIBCPP_ASSERT(c.bucket_count() == 7);
-        assert(c.size() == 6);
-        CheckConsecutiveValues<C::const_iterator>(c.find(1), c.end(), 1, 2);
-        CheckConsecutiveValues<C::const_iterator>(c.find(2), c.end(), 2, 2);
-        CheckConsecutiveValues<C::const_iterator>(c.find(3), c.end(), 3, 1);
-        CheckConsecutiveValues<C::const_iterator>(c.find(4), c.end(), 4, 1);
-        assert(c.hash_function() == test_hash<int>(8));
-        assert(c.key_eq() == test_equal_to<int>(9));
-        assert(c.get_allocator() == min_allocator<int>());
-        assert(!c.empty());
-        assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-        assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
-        assert(std::fabs(c.load_factor() - (float)c.size()/c.bucket_count()) < FLT_EPSILON);
-        assert(c.max_load_factor() == 1);
-    }
+  {
+    typedef std::unordered_multiset<int, test_hash<int>, test_equal_to<int>, other_allocator<int> > C;
+    typedef int P;
+    P a[] = {P(1), P(2), P(3), P(4), P(1), P(2)};
+    C c0(a, a + sizeof(a) / sizeof(a[0]), 7, test_hash<int>(8), test_equal_to<int>(9), other_allocator<int>(10));
+    C c = c0;
+    LIBCPP_ASSERT(c.bucket_count() == 7);
+    assert(c.size() == 6);
+    CheckConsecutiveValues<C::const_iterator>(c.find(1), c.end(), 1, 2);
+    CheckConsecutiveValues<C::const_iterator>(c.find(2), c.end(), 2, 2);
+    CheckConsecutiveValues<C::const_iterator>(c.find(3), c.end(), 3, 1);
+    CheckConsecutiveValues<C::const_iterator>(c.find(4), c.end(), 4, 1);
+    assert(c.hash_function() == test_hash<int>(8));
+    assert(c.key_eq() == test_equal_to<int>(9));
+    assert(c.get_allocator() == other_allocator<int>(-2));
+    assert(!c.empty());
+    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
+    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
+    assert(std::fabs(c.load_factor() - (float)c.size() / c.bucket_count()) < FLT_EPSILON);
+    assert(c.max_load_factor() == 1);
+  }
+  {
+    typedef std::unordered_multiset<int, test_hash<int>, test_equal_to<int>, min_allocator<int> > C;
+    typedef int P;
+    P a[] = {P(1), P(2), P(3), P(4), P(1), P(2)};
+    C c0(a, a + sizeof(a) / sizeof(a[0]), 7, test_hash<int>(8), test_equal_to<int>(9), min_allocator<int>());
+    C c = c0;
+    LIBCPP_ASSERT(c.bucket_count() == 7);
+    assert(c.size() == 6);
+    CheckConsecutiveValues<C::const_iterator>(c.find(1), c.end(), 1, 2);
+    CheckConsecutiveValues<C::const_iterator>(c.find(2), c.end(), 2, 2);
+    CheckConsecutiveValues<C::const_iterator>(c.find(3), c.end(), 3, 1);
+    CheckConsecutiveValues<C::const_iterator>(c.find(4), c.end(), 4, 1);
+    assert(c.hash_function() == test_hash<int>(8));
+    assert(c.key_eq() == test_equal_to<int>(9));
+    assert(c.get_allocator() == min_allocator<int>());
+    assert(!c.empty());
+    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
+    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
+    assert(std::fabs(c.load_factor() - (float)c.size() / c.bucket_count()) < FLT_EPSILON);
+    assert(c.max_load_factor() == 1);
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/unord/unord.multiset/unord.multiset.cnstr/copy_alloc.pass.cpp b/libcxx/test/std/containers/unord/unord.multiset/unord.multiset.cnstr/copy_alloc.pass.cpp
index 5c920b91f747c..7f45a3f2e4bd4 100644
--- a/libcxx/test/std/containers/unord/unord.multiset/unord.multiset.cnstr/copy_alloc.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.multiset/unord.multiset.cnstr/copy_alloc.pass.cpp
@@ -27,85 +27,50 @@
 #include "test_allocator.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef std::unordered_multiset<int,
-                                   test_hash<int>,
-                                   test_equal_to<int>,
-                                   test_allocator<int>
-                                   > C;
-        typedef int P;
-        P a[] =
-        {
-            P(1),
-            P(2),
-            P(3),
-            P(4),
-            P(1),
-            P(2)
-        };
-        C c0(a, a + sizeof(a)/sizeof(a[0]),
-            7,
-            test_hash<int>(8),
-            test_equal_to<int>(9),
-            test_allocator<int>(10)
-           );
-        C c(c0, test_allocator<int>(5));
-        LIBCPP_ASSERT(c.bucket_count() == 7);
-        assert(c.size() == 6);
-        CheckConsecutiveValues<C::const_iterator>(c.find(1), c.end(), 1, 2);
-        CheckConsecutiveValues<C::const_iterator>(c.find(2), c.end(), 2, 2);
-        CheckConsecutiveValues<C::const_iterator>(c.find(3), c.end(), 3, 1);
-        CheckConsecutiveValues<C::const_iterator>(c.find(4), c.end(), 4, 1);
-        assert(c.hash_function() == test_hash<int>(8));
-        assert(c.key_eq() == test_equal_to<int>(9));
-        assert(c.get_allocator() == test_allocator<int>(5));
-        assert(!c.empty());
-        assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-        assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
-        assert(std::fabs(c.load_factor() - (float)c.size()/c.bucket_count()) < FLT_EPSILON);
-        assert(c.max_load_factor() == 1);
-    }
+int main(int, char**) {
+  {
+    typedef std::unordered_multiset<int, test_hash<int>, test_equal_to<int>, test_allocator<int> > C;
+    typedef int P;
+    P a[] = {P(1), P(2), P(3), P(4), P(1), P(2)};
+    C c0(a, a + sizeof(a) / sizeof(a[0]), 7, test_hash<int>(8), test_equal_to<int>(9), test_allocator<int>(10));
+    C c(c0, test_allocator<int>(5));
+    LIBCPP_ASSERT(c.bucket_count() == 7);
+    assert(c.size() == 6);
+    CheckConsecutiveValues<C::const_iterator>(c.find(1), c.end(), 1, 2);
+    CheckConsecutiveValues<C::const_iterator>(c.find(2), c.end(), 2, 2);
+    CheckConsecutiveValues<C::const_iterator>(c.find(3), c.end(), 3, 1);
+    CheckConsecutiveValues<C::const_iterator>(c.find(4), c.end(), 4, 1);
+    assert(c.hash_function() == test_hash<int>(8));
+    assert(c.key_eq() == test_equal_to<int>(9));
+    assert(c.get_allocator() == test_allocator<int>(5));
+    assert(!c.empty());
+    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
+    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
+    assert(std::fabs(c.load_factor() - (float)c.size() / c.bucket_count()) < FLT_EPSILON);
+    assert(c.max_load_factor() == 1);
+  }
 #if TEST_STD_VER >= 11
-    {
-        typedef std::unordered_multiset<int,
-                                   test_hash<int>,
-                                   test_equal_to<int>,
-                                   min_allocator<int>
-                                   > C;
-        typedef int P;
-        P a[] =
-        {
-            P(1),
-            P(2),
-            P(3),
-            P(4),
-            P(1),
-            P(2)
-        };
-        C c0(a, a + sizeof(a)/sizeof(a[0]),
-            7,
-            test_hash<int>(8),
-            test_equal_to<int>(9),
-            min_allocator<int>()
-           );
-        C c(c0, min_allocator<int>());
-        LIBCPP_ASSERT(c.bucket_count() == 7);
-        assert(c.size() == 6);
-        CheckConsecutiveValues<C::const_iterator>(c.find(1), c.end(), 1, 2);
-        CheckConsecutiveValues<C::const_iterator>(c.find(2), c.end(), 2, 2);
-        CheckConsecutiveValues<C::const_iterator>(c.find(3), c.end(), 3, 1);
-        CheckConsecutiveValues<C::const_iterator>(c.find(4), c.end(), 4, 1);
-        assert(c.hash_function() == test_hash<int>(8));
-        assert(c.key_eq() == test_equal_to<int>(9));
-        assert(c.get_allocator() == min_allocator<int>());
-        assert(!c.empty());
-        assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-        assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
-        assert(std::fabs(c.load_factor() - (float)c.size()/c.bucket_count()) < FLT_EPSILON);
-        assert(c.max_load_factor() == 1);
-    }
+  {
+    typedef std::unordered_multiset<int, test_hash<int>, test_equal_to<int>, min_allocator<int> > C;
+    typedef int P;
+    P a[] = {P(1), P(2), P(3), P(4), P(1), P(2)};
+    C c0(a, a + sizeof(a) / sizeof(a[0]), 7, test_hash<int>(8), test_equal_to<int>(9), min_allocator<int>());
+    C c(c0, min_allocator<int>());
+    LIBCPP_ASSERT(c.bucket_count() == 7);
+    assert(c.size() == 6);
+    CheckConsecutiveValues<C::const_iterator>(c.find(1), c.end(), 1, 2);
+    CheckConsecutiveValues<C::const_iterator>(c.find(2), c.end(), 2, 2);
+    CheckConsecutiveValues<C::const_iterator>(c.find(3), c.end(), 3, 1);
+    CheckConsecutiveValues<C::const_iterator>(c.find(4), c.end(), 4, 1);
+    assert(c.hash_function() == test_hash<int>(8));
+    assert(c.key_eq() == test_equal_to<int>(9));
+    assert(c.get_allocator() == min_allocator<int>());
+    assert(!c.empty());
+    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
+    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
+    assert(std::fabs(c.load_factor() - (float)c.size() / c.bucket_count()) < FLT_EPSILON);
+    assert(c.max_load_factor() == 1);
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/unord/unord.multiset/unord.multiset.cnstr/deduct.pass.cpp b/libcxx/test/std/containers/unord/unord.multiset/unord.multiset.cnstr/deduct.pass.cpp
index 72448875cda75..b5520bd21890b 100644
--- a/libcxx/test/std/containers/unord/unord.multiset/unord.multiset.cnstr/deduct.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.multiset/unord.multiset.cnstr/deduct.pass.cpp
@@ -81,195 +81,201 @@
 #include "deduction_guides_sfinae_checks.h"
 #include "test_allocator.h"
 
-int main(int, char**)
-{
-    const int expected_s[] = {1, 1, 2, 3, INT_MAX};
+int main(int, char**) {
+  const int expected_s[] = {1, 1, 2, 3, INT_MAX};
 
-    {
-    const int arr[] = { 1, 2, 1, INT_MAX, 3 };
+  {
+    const int arr[] = {1, 2, 1, INT_MAX, 3};
     std::unordered_multiset s(std::begin(arr), std::end(arr));
 
     ASSERT_SAME_TYPE(decltype(s), std::unordered_multiset<int>);
     assert(std::is_permutation(s.begin(), s.end(), std::begin(expected_s), std::end(expected_s)));
-    }
+  }
 
-    {
-    const int arr[] = { 1, 2, 1, INT_MAX, 3 };
+  {
+    const int arr[] = {1, 2, 1, INT_MAX, 3};
     std::unordered_multiset s(std::begin(arr), std::end(arr), 42);
 
     ASSERT_SAME_TYPE(decltype(s), std::unordered_multiset<int>);
     assert(std::is_permutation(s.begin(), s.end(), std::begin(expected_s), std::end(expected_s)));
-    }
+  }
 
-    {
-    const int arr[] = { 1, 2, 1, INT_MAX, 3 };
+  {
+    const int arr[] = {1, 2, 1, INT_MAX, 3};
     std::unordered_multiset s(std::begin(arr), std::end(arr), 42, std::hash<long long>());
 
     ASSERT_SAME_TYPE(decltype(s), std::unordered_multiset<int, std::hash<long long>>);
     assert(std::is_permutation(s.begin(), s.end(), std::begin(expected_s), std::end(expected_s)));
-    }
+  }
 
-    {
-    const int arr[] = { 1, 2, 1, INT_MAX, 3 };
+  {
+    const int arr[] = {1, 2, 1, INT_MAX, 3};
     std::unordered_multiset s(std::begin(arr), std::end(arr), 42, std::hash<long long>(), test_allocator<int>(0, 40));
 
-    ASSERT_SAME_TYPE(decltype(s), std::unordered_multiset<int, std::hash<long long>, std::equal_to<int>, test_allocator<int>>);
+    ASSERT_SAME_TYPE(
+        decltype(s), std::unordered_multiset<int, std::hash<long long>, std::equal_to<int>, test_allocator<int>>);
     assert(std::is_permutation(s.begin(), s.end(), std::begin(expected_s), std::end(expected_s)));
     assert(s.get_allocator().get_id() == 40);
-    }
+  }
 
-    {
+  {
     std::unordered_multiset<int, std::hash<long long>, std::equal_to<>, test_allocator<int>> source;
     std::unordered_multiset s(source);
     ASSERT_SAME_TYPE(decltype(s), decltype(source));
     assert(s.size() == 0);
-    }
+  }
 
-    {
+  {
     std::unordered_multiset<int, std::hash<long long>, std::equal_to<>, test_allocator<int>> source;
-    std::unordered_multiset s{source};  // braces instead of parens
+    std::unordered_multiset s{source}; // braces instead of parens
     ASSERT_SAME_TYPE(decltype(s), decltype(source));
     assert(s.size() == 0);
-    }
+  }
 
-    {
+  {
     std::unordered_multiset<int, std::hash<long long>, std::equal_to<>, test_allocator<int>> source;
     std::unordered_multiset s(source, test_allocator<int>(0, 41));
     ASSERT_SAME_TYPE(decltype(s), decltype(source));
     assert(s.size() == 0);
     assert(s.get_allocator().get_id() == 41);
-    }
+  }
 
-    {
+  {
     std::unordered_multiset<int, std::hash<long long>, std::equal_to<>, test_allocator<int>> source;
-    std::unordered_multiset s{source, test_allocator<int>(0, 42)};  // braces instead of parens
+    std::unordered_multiset s{source, test_allocator<int>(0, 42)}; // braces instead of parens
     ASSERT_SAME_TYPE(decltype(s), decltype(source));
     assert(s.size() == 0);
     assert(s.get_allocator().get_id() == 42);
-    }
+  }
 
-    {
-    std::unordered_multiset s{ 1, 2, 1, INT_MAX, 3 };
+  {
+    std::unordered_multiset s{1, 2, 1, INT_MAX, 3};
 
     ASSERT_SAME_TYPE(decltype(s), std::unordered_multiset<int>);
     assert(std::is_permutation(s.begin(), s.end(), std::begin(expected_s), std::end(expected_s)));
-    }
+  }
 
-    {
-    std::unordered_multiset s({ 1, 2, 1, INT_MAX, 3 }, 42);
+  {
+    std::unordered_multiset s({1, 2, 1, INT_MAX, 3}, 42);
 
     ASSERT_SAME_TYPE(decltype(s), std::unordered_multiset<int>);
     assert(std::is_permutation(s.begin(), s.end(), std::begin(expected_s), std::end(expected_s)));
-    }
+  }
 
-    {
-    std::unordered_multiset s({ 1, 2, 1, INT_MAX, 3 }, 42, std::hash<long long>());
+  {
+    std::unordered_multiset s({1, 2, 1, INT_MAX, 3}, 42, std::hash<long long>());
 
     ASSERT_SAME_TYPE(decltype(s), std::unordered_multiset<int, std::hash<long long>>);
     assert(std::is_permutation(s.begin(), s.end(), std::begin(expected_s), std::end(expected_s)));
-    }
+  }
 
-    {
-    std::unordered_multiset s({ 1, 2, 1, INT_MAX, 3 }, 42, std::hash<long long>(), std::equal_to<>());
+  {
+    std::unordered_multiset s({1, 2, 1, INT_MAX, 3}, 42, std::hash<long long>(), std::equal_to<>());
 
     ASSERT_SAME_TYPE(decltype(s), std::unordered_multiset<int, std::hash<long long>, std::equal_to<>>);
     assert(std::is_permutation(s.begin(), s.end(), std::begin(expected_s), std::end(expected_s)));
-    }
+  }
 
-    {
-    std::unordered_multiset s({ 1, 2, 1, INT_MAX, 3 }, 42, std::hash<long long>(), std::equal_to<>(), test_allocator<int>(0, 43));
+  {
+    std::unordered_multiset s(
+        {1, 2, 1, INT_MAX, 3}, 42, std::hash<long long>(), std::equal_to<>(), test_allocator<int>(0, 43));
 
-    ASSERT_SAME_TYPE(decltype(s), std::unordered_multiset<int, std::hash<long long>, std::equal_to<>, test_allocator<int>>);
+    ASSERT_SAME_TYPE(
+        decltype(s), std::unordered_multiset<int, std::hash<long long>, std::equal_to<>, test_allocator<int>>);
     assert(std::is_permutation(s.begin(), s.end(), std::begin(expected_s), std::end(expected_s)));
     assert(s.get_allocator().get_id() == 43);
-    }
+  }
 
-    {
-    const int arr[] = { 1, 2, 1, INT_MAX, 3 };
+  {
+    const int arr[] = {1, 2, 1, INT_MAX, 3};
     std::unordered_multiset s(std::begin(arr), std::end(arr), 42, test_allocator<int>(0, 44));
 
-    ASSERT_SAME_TYPE(decltype(s), std::unordered_multiset<int, std::hash<int>, std::equal_to<int>, test_allocator<int>>);
+    ASSERT_SAME_TYPE(
+        decltype(s), std::unordered_multiset<int, std::hash<int>, std::equal_to<int>, test_allocator<int>>);
     assert(std::is_permutation(s.begin(), s.end(), std::begin(expected_s), std::end(expected_s)));
     assert(s.get_allocator().get_id() == 44);
-    }
+  }
 
-    {
-    const int arr[] = { 1, 2, 1, INT_MAX, 3 };
+  {
+    const int arr[] = {1, 2, 1, INT_MAX, 3};
     std::unordered_multiset s(std::begin(arr), std::end(arr), 42, std::hash<long long>(), test_allocator<int>(0, 44));
 
-    ASSERT_SAME_TYPE(decltype(s), std::unordered_multiset<int, std::hash<long long>, std::equal_to<int>, test_allocator<int>>);
+    ASSERT_SAME_TYPE(
+        decltype(s), std::unordered_multiset<int, std::hash<long long>, std::equal_to<int>, test_allocator<int>>);
     assert(std::is_permutation(s.begin(), s.end(), std::begin(expected_s), std::end(expected_s)));
     assert(s.get_allocator().get_id() == 44);
-    }
+  }
 
-    {
-    std::unordered_multiset s({ 1, 2, 1, INT_MAX, 3 }, 42, test_allocator<int>(0, 43));
+  {
+    std::unordered_multiset s({1, 2, 1, INT_MAX, 3}, 42, test_allocator<int>(0, 43));
 
-    ASSERT_SAME_TYPE(decltype(s), std::unordered_multiset<int, std::hash<int>, std::equal_to<int>, test_allocator<int>>);
+    ASSERT_SAME_TYPE(
+        decltype(s), std::unordered_multiset<int, std::hash<int>, std::equal_to<int>, test_allocator<int>>);
     assert(std::is_permutation(s.begin(), s.end(), std::begin(expected_s), std::end(expected_s)));
     assert(s.get_allocator().get_id() == 43);
-    }
+  }
 
-    {
-    std::unordered_multiset s({ 1, 2, 1, INT_MAX, 3 }, 42, std::hash<long long>(), test_allocator<int>(0, 42));
+  {
+    std::unordered_multiset s({1, 2, 1, INT_MAX, 3}, 42, std::hash<long long>(), test_allocator<int>(0, 42));
 
-    ASSERT_SAME_TYPE(decltype(s), std::unordered_multiset<int, std::hash<long long>, std::equal_to<int>, test_allocator<int>>);
+    ASSERT_SAME_TYPE(
+        decltype(s), std::unordered_multiset<int, std::hash<long long>, std::equal_to<int>, test_allocator<int>>);
     assert(std::is_permutation(s.begin(), s.end(), std::begin(expected_s), std::end(expected_s)));
     assert(s.get_allocator().get_id() == 42);
-    }
+  }
 
 #if TEST_STD_VER >= 23
-    {
-      using Range = std::array<int, 0>;
-      using Pred = test_equal_to<int>;
-      using DefaultPred = std::equal_to<int>;
-      using Hash = test_hash<int>;
-      using DefaultHash = std::hash<int>;
-      using Alloc = test_allocator<int>;
-
-      { // (from_range, range)
-        std::unordered_multiset c(std::from_range, Range());
-        static_assert(std::is_same_v<decltype(c), std::unordered_multiset<int>>);
-      }
-
-      { // (from_range, range, n)
-        std::unordered_multiset c(std::from_range, Range(), std::size_t());
-        static_assert(std::is_same_v<decltype(c), std::unordered_multiset<int>>);
-      }
-
-      { // (from_range, range, n, hash)
-        std::unordered_multiset c(std::from_range, Range(), std::size_t(), Hash());
-        static_assert(std::is_same_v<decltype(c), std::unordered_multiset<int, Hash>>);
-      }
-
-      { // (from_range, range, n, hash, pred)
-        std::unordered_multiset c(std::from_range, Range(), std::size_t(), Hash(), Pred());
-        static_assert(std::is_same_v<decltype(c), std::unordered_multiset<int, Hash, Pred>>);
-      }
-
-      { // (from_range, range, n, hash, pred, alloc)
-        std::unordered_multiset c(std::from_range, Range(), std::size_t(), Hash(), Pred(), Alloc());
-        static_assert(std::is_same_v<decltype(c), std::unordered_multiset<int, Hash, Pred, Alloc>>);
-      }
-
-      { // (from_range, range, n, alloc)
-        std::unordered_multiset c(std::from_range, Range(), std::size_t(), Alloc());
-        static_assert(std::is_same_v<decltype(c), std::unordered_multiset<int, DefaultHash, DefaultPred, Alloc>>);
-      }
-
-      // TODO(LWG 2713): uncomment this test once the constructor is added.
-      { // (from_range, range, alloc)
-        //std::unordered_multiset c(std::from_range, Range(), Alloc());
-        //static_assert(std::is_same_v<decltype(c), std::unordered_multiset<int, DefaultHash, DefaultPred, Alloc>>);
-      }
-
-      { // (from_range, range, n, hash, alloc)
-        std::unordered_multiset c(std::from_range, Range(), std::size_t(), Hash(), Alloc());
-        static_assert(std::is_same_v<decltype(c), std::unordered_multiset<int, Hash, DefaultPred, Alloc>>);
-      }
+  {
+    using Range       = std::array<int, 0>;
+    using Pred        = test_equal_to<int>;
+    using DefaultPred = std::equal_to<int>;
+    using Hash        = test_hash<int>;
+    using DefaultHash = std::hash<int>;
+    using Alloc       = test_allocator<int>;
+
+    { // (from_range, range)
+      std::unordered_multiset c(std::from_range, Range());
+      static_assert(std::is_same_v<decltype(c), std::unordered_multiset<int>>);
+    }
+
+    { // (from_range, range, n)
+      std::unordered_multiset c(std::from_range, Range(), std::size_t());
+      static_assert(std::is_same_v<decltype(c), std::unordered_multiset<int>>);
+    }
+
+    { // (from_range, range, n, hash)
+      std::unordered_multiset c(std::from_range, Range(), std::size_t(), Hash());
+      static_assert(std::is_same_v<decltype(c), std::unordered_multiset<int, Hash>>);
+    }
+
+    { // (from_range, range, n, hash, pred)
+      std::unordered_multiset c(std::from_range, Range(), std::size_t(), Hash(), Pred());
+      static_assert(std::is_same_v<decltype(c), std::unordered_multiset<int, Hash, Pred>>);
+    }
+
+    { // (from_range, range, n, hash, pred, alloc)
+      std::unordered_multiset c(std::from_range, Range(), std::size_t(), Hash(), Pred(), Alloc());
+      static_assert(std::is_same_v<decltype(c), std::unordered_multiset<int, Hash, Pred, Alloc>>);
+    }
+
+    { // (from_range, range, n, alloc)
+      std::unordered_multiset c(std::from_range, Range(), std::size_t(), Alloc());
+      static_assert(std::is_same_v<decltype(c), std::unordered_multiset<int, DefaultHash, DefaultPred, Alloc>>);
+    }
+
+    // TODO(LWG 2713): uncomment this test once the constructor is added.
+    { // (from_range, range, alloc)
+      //std::unordered_multiset c(std::from_range, Range(), Alloc());
+      //static_assert(std::is_same_v<decltype(c), std::unordered_multiset<int, DefaultHash, DefaultPred, Alloc>>);
+    }
+
+    { // (from_range, range, n, hash, alloc)
+      std::unordered_multiset c(std::from_range, Range(), std::size_t(), Hash(), Alloc());
+      static_assert(std::is_same_v<decltype(c), std::unordered_multiset<int, Hash, DefaultPred, Alloc>>);
     }
+  }
 #endif
-    UnorderedContainerDeductionGuidesSfinaeAway<std::unordered_multiset, std::unordered_multiset<int>>();
+  UnorderedContainerDeductionGuidesSfinaeAway<std::unordered_multiset, std::unordered_multiset<int>>();
 
-    return 0;
+  return 0;
 }
diff --git a/libcxx/test/std/containers/unord/unord.multiset/unord.multiset.cnstr/deduct.verify.cpp b/libcxx/test/std/containers/unord/unord.multiset/unord.multiset.cnstr/deduct.verify.cpp
index 8fd7d1d6c20ef..9a1ec2c0241ea 100644
--- a/libcxx/test/std/containers/unord/unord.multiset/unord.multiset.cnstr/deduct.verify.cpp
+++ b/libcxx/test/std/containers/unord/unord.multiset/unord.multiset.cnstr/deduct.verify.cpp
@@ -50,48 +50,47 @@
 #include <functional>
 #include <unordered_set>
 
-int main(int, char**)
-{
-    {
-        // cannot deduce Key from nothing
-        std::unordered_multiset s;
-            // expected-error-re@-1{{no viable constructor or deduction guide for deduction of template arguments of '{{(std::)?}}unordered_multiset'}}
-    }
-    {
-        // cannot deduce Key from just (Size)
-        std::unordered_multiset s(42);
-            // expected-error-re@-1{{no viable constructor or deduction guide for deduction of template arguments of '{{(std::)?}}unordered_multiset'}}
-    }
-    {
-        // cannot deduce Key from just (Size, Hash)
-        std::unordered_multiset s(42, std::hash<int>());
-            // expected-error-re@-1{{no viable constructor or deduction guide for deduction of template arguments of '{{(std::)?}}unordered_multiset'}}
-    }
-    {
-        // cannot deduce Key from just (Size, Hash, Pred)
-        std::unordered_multiset s(42, std::hash<int>(), std::equal_to<>());
-            // expected-error-re@-1{{no viable constructor or deduction guide for deduction of template arguments of '{{(std::)?}}unordered_multiset'}}
-    }
-    {
-        // cannot deduce Key from just (Size, Hash, Pred, Allocator)
-        std::unordered_multiset s(42, std::hash<int>(), std::equal_to<>(), std::allocator<int>());
-            // expected-error-re@-1{{no viable constructor or deduction guide for deduction of template arguments of '{{(std::)?}}unordered_multiset'}}
-    }
-    {
-        // cannot deduce Key from just (Allocator)
-        std::unordered_multiset s(std::allocator<int>{});
-            // expected-error-re@-1{{no viable constructor or deduction guide for deduction of template arguments of '{{(std::)?}}unordered_multiset'}}
-    }
-    {
-        // cannot deduce Key from just (Size, Allocator)
-        std::unordered_multiset s(42, std::allocator<int>());
-            // expected-error-re@-1{{no viable constructor or deduction guide for deduction of template arguments of '{{(std::)?}}unordered_multiset'}}
-    }
-    {
-        // cannot deduce Key from just (Size, Hash, Allocator)
-        std::unordered_multiset s(42, std::hash<short>(), std::allocator<int>());
-            // expected-error-re@-1{{no viable constructor or deduction guide for deduction of template arguments of '{{(std::)?}}unordered_multiset'}}
-    }
+int main(int, char**) {
+  {
+    // cannot deduce Key from nothing
+    std::unordered_multiset s;
+    // expected-error-re@-1{{no viable constructor or deduction guide for deduction of template arguments of '{{(std::)?}}unordered_multiset'}}
+  }
+  {
+    // cannot deduce Key from just (Size)
+    std::unordered_multiset s(42);
+    // expected-error-re@-1{{no viable constructor or deduction guide for deduction of template arguments of '{{(std::)?}}unordered_multiset'}}
+  }
+  {
+    // cannot deduce Key from just (Size, Hash)
+    std::unordered_multiset s(42, std::hash<int>());
+    // expected-error-re@-1{{no viable constructor or deduction guide for deduction of template arguments of '{{(std::)?}}unordered_multiset'}}
+  }
+  {
+    // cannot deduce Key from just (Size, Hash, Pred)
+    std::unordered_multiset s(42, std::hash<int>(), std::equal_to<>());
+    // expected-error-re@-1{{no viable constructor or deduction guide for deduction of template arguments of '{{(std::)?}}unordered_multiset'}}
+  }
+  {
+    // cannot deduce Key from just (Size, Hash, Pred, Allocator)
+    std::unordered_multiset s(42, std::hash<int>(), std::equal_to<>(), std::allocator<int>());
+    // expected-error-re@-1{{no viable constructor or deduction guide for deduction of template arguments of '{{(std::)?}}unordered_multiset'}}
+  }
+  {
+    // cannot deduce Key from just (Allocator)
+    std::unordered_multiset s(std::allocator<int>{});
+    // expected-error-re@-1{{no viable constructor or deduction guide for deduction of template arguments of '{{(std::)?}}unordered_multiset'}}
+  }
+  {
+    // cannot deduce Key from just (Size, Allocator)
+    std::unordered_multiset s(42, std::allocator<int>());
+    // expected-error-re@-1{{no viable constructor or deduction guide for deduction of template arguments of '{{(std::)?}}unordered_multiset'}}
+  }
+  {
+    // cannot deduce Key from just (Size, Hash, Allocator)
+    std::unordered_multiset s(42, std::hash<short>(), std::allocator<int>());
+    // expected-error-re@-1{{no viable constructor or deduction guide for deduction of template arguments of '{{(std::)?}}unordered_multiset'}}
+  }
 
-    return 0;
+  return 0;
 }
diff --git a/libcxx/test/std/containers/unord/unord.multiset/unord.multiset.cnstr/default.pass.cpp b/libcxx/test/std/containers/unord/unord.multiset/unord.multiset.cnstr/default.pass.cpp
index 00d8c6f59eeba..4e182180f9486 100644
--- a/libcxx/test/std/containers/unord/unord.multiset/unord.multiset.cnstr/default.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.multiset/unord.multiset.cnstr/default.pass.cpp
@@ -24,85 +24,81 @@
 #include "test_allocator.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef std::unordered_multiset<NotConstructible,
-                                   test_hash<NotConstructible>,
-                                   test_equal_to<NotConstructible>,
-                                   test_allocator<NotConstructible>
-                                   > C;
-        C c;
-        LIBCPP_ASSERT(c.bucket_count() == 0);
-        assert(c.hash_function() == test_hash<NotConstructible>());
-        assert(c.key_eq() == test_equal_to<NotConstructible>());
-        assert(c.get_allocator() == (test_allocator<NotConstructible>()));
-        assert(c.size() == 0);
-        assert(c.empty());
-        assert(std::distance(c.begin(), c.end()) == 0);
-        assert(c.load_factor() == 0);
-        assert(c.max_load_factor() == 1);
-    }
+int main(int, char**) {
+  {
+    typedef std::unordered_multiset<NotConstructible,
+                                    test_hash<NotConstructible>,
+                                    test_equal_to<NotConstructible>,
+                                    test_allocator<NotConstructible> >
+        C;
+    C c;
+    LIBCPP_ASSERT(c.bucket_count() == 0);
+    assert(c.hash_function() == test_hash<NotConstructible>());
+    assert(c.key_eq() == test_equal_to<NotConstructible>());
+    assert(c.get_allocator() == (test_allocator<NotConstructible>()));
+    assert(c.size() == 0);
+    assert(c.empty());
+    assert(std::distance(c.begin(), c.end()) == 0);
+    assert(c.load_factor() == 0);
+    assert(c.max_load_factor() == 1);
+  }
 #if TEST_STD_VER >= 11
+  {
+    typedef std::unordered_multiset<NotConstructible,
+                                    test_hash<NotConstructible>,
+                                    test_equal_to<NotConstructible>,
+                                    min_allocator<NotConstructible> >
+        C;
+    C c;
+    LIBCPP_ASSERT(c.bucket_count() == 0);
+    assert(c.hash_function() == test_hash<NotConstructible>());
+    assert(c.key_eq() == test_equal_to<NotConstructible>());
+    assert(c.get_allocator() == (min_allocator<NotConstructible>()));
+    assert(c.size() == 0);
+    assert(c.empty());
+    assert(std::distance(c.begin(), c.end()) == 0);
+    assert(c.load_factor() == 0);
+    assert(c.max_load_factor() == 1);
+  }
+  {
+    typedef explicit_allocator<NotConstructible> A;
+    typedef std::unordered_multiset<NotConstructible, test_hash<NotConstructible>, test_equal_to<NotConstructible>, A >
+        C;
     {
-        typedef std::unordered_multiset<NotConstructible,
-                                   test_hash<NotConstructible>,
-                                   test_equal_to<NotConstructible>,
-                                   min_allocator<NotConstructible>
-                                   > C;
-        C c;
-        LIBCPP_ASSERT(c.bucket_count() == 0);
-        assert(c.hash_function() == test_hash<NotConstructible>());
-        assert(c.key_eq() == test_equal_to<NotConstructible>());
-        assert(c.get_allocator() == (min_allocator<NotConstructible>()));
-        assert(c.size() == 0);
-        assert(c.empty());
-        assert(std::distance(c.begin(), c.end()) == 0);
-        assert(c.load_factor() == 0);
-        assert(c.max_load_factor() == 1);
-    }
-    {
-        typedef explicit_allocator<NotConstructible> A;
-        typedef std::unordered_multiset<NotConstructible,
-                                   test_hash<NotConstructible>,
-                                   test_equal_to<NotConstructible>,
-                                   A
-                                   > C;
-        {
-        C c;
-        LIBCPP_ASSERT(c.bucket_count() == 0);
-        assert(c.hash_function() == test_hash<NotConstructible>());
-        assert(c.key_eq() == test_equal_to<NotConstructible>());
-        assert(c.get_allocator() == A());
-        assert(c.size() == 0);
-        assert(c.empty());
-        assert(std::distance(c.begin(), c.end()) == 0);
-        assert(c.load_factor() == 0);
-        assert(c.max_load_factor() == 1);
-        }
-        {
-        A a;
-        C c(a);
-        LIBCPP_ASSERT(c.bucket_count() == 0);
-        assert(c.hash_function() == test_hash<NotConstructible>());
-        assert(c.key_eq() == test_equal_to<NotConstructible>());
-        assert(c.get_allocator() == a);
-        assert(c.size() == 0);
-        assert(c.empty());
-        assert(std::distance(c.begin(), c.end()) == 0);
-        assert(c.load_factor() == 0);
-        assert(c.max_load_factor() == 1);
-        }
+      C c;
+      LIBCPP_ASSERT(c.bucket_count() == 0);
+      assert(c.hash_function() == test_hash<NotConstructible>());
+      assert(c.key_eq() == test_equal_to<NotConstructible>());
+      assert(c.get_allocator() == A());
+      assert(c.size() == 0);
+      assert(c.empty());
+      assert(std::distance(c.begin(), c.end()) == 0);
+      assert(c.load_factor() == 0);
+      assert(c.max_load_factor() == 1);
     }
     {
-        std::unordered_multiset<int> c = {};
-        LIBCPP_ASSERT(c.bucket_count() == 0);
-        assert(c.size() == 0);
-        assert(c.empty());
-        assert(std::distance(c.begin(), c.end()) == 0);
-        assert(c.load_factor() == 0);
-        assert(c.max_load_factor() == 1);
+      A a;
+      C c(a);
+      LIBCPP_ASSERT(c.bucket_count() == 0);
+      assert(c.hash_function() == test_hash<NotConstructible>());
+      assert(c.key_eq() == test_equal_to<NotConstructible>());
+      assert(c.get_allocator() == a);
+      assert(c.size() == 0);
+      assert(c.empty());
+      assert(std::distance(c.begin(), c.end()) == 0);
+      assert(c.load_factor() == 0);
+      assert(c.max_load_factor() == 1);
     }
+  }
+  {
+    std::unordered_multiset<int> c = {};
+    LIBCPP_ASSERT(c.bucket_count() == 0);
+    assert(c.size() == 0);
+    assert(c.empty());
+    assert(std::distance(c.begin(), c.end()) == 0);
+    assert(c.load_factor() == 0);
+    assert(c.max_load_factor() == 1);
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/unord/unord.multiset/unord.multiset.cnstr/default_noexcept.pass.cpp b/libcxx/test/std/containers/unord/unord.multiset/unord.multiset.cnstr/default_noexcept.pass.cpp
index 1c64692d85653..6bf666371ddb7 100644
--- a/libcxx/test/std/containers/unord/unord.multiset/unord.multiset.cnstr/default_noexcept.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.multiset/unord.multiset.cnstr/default_noexcept.pass.cpp
@@ -27,50 +27,45 @@
 #include "../../../test_hash.h"
 
 template <class T>
-struct some_comp
-{
-    typedef T value_type;
-    some_comp();
-    some_comp(const some_comp&);
-    bool operator()(const T&, const T&) const { return false; }
+struct some_comp {
+  typedef T value_type;
+  some_comp();
+  some_comp(const some_comp&);
+  bool operator()(const T&, const T&) const { return false; }
 };
 
 template <class T>
-struct some_hash
-{
-    typedef T value_type;
-    some_hash();
-    some_hash(const some_hash&);
-    std::size_t operator()(T const&) const;
+struct some_hash {
+  typedef T value_type;
+  some_hash();
+  some_hash(const some_hash&);
+  std::size_t operator()(T const&) const;
 };
 
-int main(int, char**)
-{
+int main(int, char**) {
 #if defined(_LIBCPP_VERSION)
-    {
-        typedef std::unordered_multiset<MoveOnly> C;
-        static_assert(std::is_nothrow_default_constructible<C>::value, "");
-    }
-    {
-        typedef std::unordered_multiset<MoveOnly, std::hash<MoveOnly>,
-                           std::equal_to<MoveOnly>, test_allocator<MoveOnly>> C;
-        static_assert(std::is_nothrow_default_constructible<C>::value, "");
-    }
+  {
+    typedef std::unordered_multiset<MoveOnly> C;
+    static_assert(std::is_nothrow_default_constructible<C>::value, "");
+  }
+  {
+    typedef std::unordered_multiset<MoveOnly, std::hash<MoveOnly>, std::equal_to<MoveOnly>, test_allocator<MoveOnly>> C;
+    static_assert(std::is_nothrow_default_constructible<C>::value, "");
+  }
 #endif // _LIBCPP_VERSION
-    {
-        typedef std::unordered_multiset<MoveOnly, std::hash<MoveOnly>,
-                          std::equal_to<MoveOnly>, other_allocator<MoveOnly>> C;
-        static_assert(!std::is_nothrow_default_constructible<C>::value, "");
-    }
-    {
-        typedef std::unordered_multiset<MoveOnly, some_hash<MoveOnly>> C;
-        static_assert(!std::is_nothrow_default_constructible<C>::value, "");
-    }
-    {
-        typedef std::unordered_multiset<MoveOnly, std::hash<MoveOnly>,
-                                                         some_comp<MoveOnly>> C;
-        static_assert(!std::is_nothrow_default_constructible<C>::value, "");
-    }
+  {
+    typedef std::unordered_multiset<MoveOnly, std::hash<MoveOnly>, std::equal_to<MoveOnly>, other_allocator<MoveOnly>>
+        C;
+    static_assert(!std::is_nothrow_default_constructible<C>::value, "");
+  }
+  {
+    typedef std::unordered_multiset<MoveOnly, some_hash<MoveOnly>> C;
+    static_assert(!std::is_nothrow_default_constructible<C>::value, "");
+  }
+  {
+    typedef std::unordered_multiset<MoveOnly, std::hash<MoveOnly>, some_comp<MoveOnly>> C;
+    static_assert(!std::is_nothrow_default_constructible<C>::value, "");
+  }
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/unord/unord.multiset/unord.multiset.cnstr/dtor_noexcept.pass.cpp b/libcxx/test/std/containers/unord/unord.multiset/unord.multiset.cnstr/dtor_noexcept.pass.cpp
index 41f68c6c47b59..32c757e16d344 100644
--- a/libcxx/test/std/containers/unord/unord.multiset/unord.multiset.cnstr/dtor_noexcept.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.multiset/unord.multiset.cnstr/dtor_noexcept.pass.cpp
@@ -20,49 +20,44 @@
 #include "test_allocator.h"
 
 template <class T>
-struct some_comp
-{
-    typedef T value_type;
-    ~some_comp() noexcept(false);
-    bool operator()(const T&, const T&) const { return false; }
+struct some_comp {
+  typedef T value_type;
+  ~some_comp() noexcept(false);
+  bool operator()(const T&, const T&) const { return false; }
 };
 
 template <class T>
-struct some_hash
-{
-    typedef T value_type;
-    some_hash();
-    some_hash(const some_hash&);
-    ~some_hash() noexcept(false);
-    std::size_t operator()(T const&) const;
+struct some_hash {
+  typedef T value_type;
+  some_hash();
+  some_hash(const some_hash&);
+  ~some_hash() noexcept(false);
+  std::size_t operator()(T const&) const;
 };
 
-int main(int, char**)
-{
-    {
-        typedef std::unordered_multiset<MoveOnly> C;
-        static_assert(std::is_nothrow_destructible<C>::value, "");
-    }
-    {
-        typedef std::unordered_multiset<MoveOnly, std::hash<MoveOnly>,
-                           std::equal_to<MoveOnly>, test_allocator<MoveOnly>> C;
-        static_assert(std::is_nothrow_destructible<C>::value, "");
-    }
-    {
-        typedef std::unordered_multiset<MoveOnly, std::hash<MoveOnly>,
-                          std::equal_to<MoveOnly>, other_allocator<MoveOnly>> C;
-        static_assert(std::is_nothrow_destructible<C>::value, "");
-    }
+int main(int, char**) {
+  {
+    typedef std::unordered_multiset<MoveOnly> C;
+    static_assert(std::is_nothrow_destructible<C>::value, "");
+  }
+  {
+    typedef std::unordered_multiset<MoveOnly, std::hash<MoveOnly>, std::equal_to<MoveOnly>, test_allocator<MoveOnly>> C;
+    static_assert(std::is_nothrow_destructible<C>::value, "");
+  }
+  {
+    typedef std::unordered_multiset<MoveOnly, std::hash<MoveOnly>, std::equal_to<MoveOnly>, other_allocator<MoveOnly>>
+        C;
+    static_assert(std::is_nothrow_destructible<C>::value, "");
+  }
 #if defined(_LIBCPP_VERSION)
-    {
-        typedef std::unordered_multiset<MoveOnly, some_hash<MoveOnly>> C;
-        static_assert(!std::is_nothrow_destructible<C>::value, "");
-    }
-    {
-        typedef std::unordered_multiset<MoveOnly, std::hash<MoveOnly>,
-                                                         some_comp<MoveOnly>> C;
-        static_assert(!std::is_nothrow_destructible<C>::value, "");
-    }
+  {
+    typedef std::unordered_multiset<MoveOnly, some_hash<MoveOnly>> C;
+    static_assert(!std::is_nothrow_destructible<C>::value, "");
+  }
+  {
+    typedef std::unordered_multiset<MoveOnly, std::hash<MoveOnly>, some_comp<MoveOnly>> C;
+    static_assert(!std::is_nothrow_destructible<C>::value, "");
+  }
 #endif // _LIBCPP_VERSION
 
   return 0;
diff --git a/libcxx/test/std/containers/unord/unord.multiset/unord.multiset.cnstr/from_range.pass.cpp b/libcxx/test/std/containers/unord/unord.multiset/unord.multiset.cnstr/from_range.pass.cpp
index 918e7d85e99eb..e5672e577f95c 100644
--- a/libcxx/test/std/containers/unord/unord.multiset/unord.multiset.cnstr/from_range.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.multiset/unord.multiset.cnstr/from_range.pass.cpp
@@ -29,7 +29,7 @@
 
 void test_duplicates() {
   std::array input = {1, 2, 3, 3, 3, 4, 2, 1, 2};
-  auto c = std::unordered_multiset<int>(std::from_range, input);
+  auto c           = std::unordered_multiset<int>(std::from_range, input);
   assert(std::ranges::is_permutation(input, c));
 }
 
diff --git a/libcxx/test/std/containers/unord/unord.multiset/unord.multiset.cnstr/hash_copy_constructible.compile.fail.cpp b/libcxx/test/std/containers/unord/unord.multiset/unord.multiset.cnstr/hash_copy_constructible.compile.fail.cpp
index 1f2ee87afd46f..d0db3f10cc3d0 100644
--- a/libcxx/test/std/containers/unord/unord.multiset/unord.multiset.cnstr/hash_copy_constructible.compile.fail.cpp
+++ b/libcxx/test/std/containers/unord/unord.multiset/unord.multiset.cnstr/hash_copy_constructible.compile.fail.cpp
@@ -6,7 +6,6 @@
 //
 //===----------------------------------------------------------------------===//
 
-
 // <unordered_set>
 
 // Check that std::unordered_multiset fails to instantiate if the hash function is
@@ -16,16 +15,16 @@
 
 template <class T>
 struct Hash {
-    std::size_t operator () (const T& lhs) const { return 0; }
+  std::size_t operator()(const T& lhs) const { return 0; }
 
-    Hash () {}
-private:
-    Hash (const Hash &); // declared but not defined
-    };
+  Hash() {}
 
+private:
+  Hash(const Hash&); // declared but not defined
+};
 
 int main(int, char**) {
-    std::unordered_multiset<int, Hash<int> > m;
+  std::unordered_multiset<int, Hash<int> > m;
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/unord/unord.multiset/unord.multiset.cnstr/init.pass.cpp b/libcxx/test/std/containers/unord/unord.multiset/unord.multiset.cnstr/init.pass.cpp
index bd5cef7a4931e..d0ebce70cde22 100644
--- a/libcxx/test/std/containers/unord/unord.multiset/unord.multiset.cnstr/init.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.multiset/unord.multiset.cnstr/init.pass.cpp
@@ -28,137 +28,100 @@
 #include "test_allocator.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef std::unordered_multiset<int,
-                                   test_hash<int>,
-                                   test_equal_to<int>,
-                                   test_allocator<int>
-                                   > C;
-        typedef int P;
-        C c = {
-                P(1),
-                P(2),
-                P(3),
-                P(4),
-                P(1),
-                P(2)
-            };
-        assert(c.bucket_count() >= 7);
-        assert(c.size() == 6);
-        assert(c.count(1) == 2);
-        assert(c.count(2) == 2);
-        assert(c.count(3) == 1);
-        assert(c.count(4) == 1);
-        assert(c.hash_function() == test_hash<int>());
-        assert(c.key_eq() == test_equal_to<int>());
-        assert(c.get_allocator() == test_allocator<int>());
-        assert(!c.empty());
-        assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-        assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
-        assert(fabs(c.load_factor() - (float)c.size()/c.bucket_count()) < FLT_EPSILON);
-        assert(c.max_load_factor() == 1);
-    }
-    {
-        typedef std::unordered_multiset<int,
-                                   test_hash<int>,
-                                   test_equal_to<int>,
-                                   min_allocator<int>
-                                   > C;
-        typedef int P;
-        C c = {
-                P(1),
-                P(2),
-                P(3),
-                P(4),
-                P(1),
-                P(2)
-            };
-        assert(c.bucket_count() >= 7);
-        assert(c.size() == 6);
-        assert(c.count(1) == 2);
-        assert(c.count(2) == 2);
-        assert(c.count(3) == 1);
-        assert(c.count(4) == 1);
-        assert(c.hash_function() == test_hash<int>());
-        assert(c.key_eq() == test_equal_to<int>());
-        assert(c.get_allocator() == min_allocator<int>());
-        assert(!c.empty());
-        assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-        assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
-        assert(fabs(c.load_factor() - (float)c.size()/c.bucket_count()) < FLT_EPSILON);
-        assert(c.max_load_factor() == 1);
-    }
+int main(int, char**) {
+  {
+    typedef std::unordered_multiset<int, test_hash<int>, test_equal_to<int>, test_allocator<int> > C;
+    typedef int P;
+    C c = {P(1), P(2), P(3), P(4), P(1), P(2)};
+    assert(c.bucket_count() >= 7);
+    assert(c.size() == 6);
+    assert(c.count(1) == 2);
+    assert(c.count(2) == 2);
+    assert(c.count(3) == 1);
+    assert(c.count(4) == 1);
+    assert(c.hash_function() == test_hash<int>());
+    assert(c.key_eq() == test_equal_to<int>());
+    assert(c.get_allocator() == test_allocator<int>());
+    assert(!c.empty());
+    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
+    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
+    assert(fabs(c.load_factor() - (float)c.size() / c.bucket_count()) < FLT_EPSILON);
+    assert(c.max_load_factor() == 1);
+  }
+  {
+    typedef std::unordered_multiset<int, test_hash<int>, test_equal_to<int>, min_allocator<int> > C;
+    typedef int P;
+    C c = {P(1), P(2), P(3), P(4), P(1), P(2)};
+    assert(c.bucket_count() >= 7);
+    assert(c.size() == 6);
+    assert(c.count(1) == 2);
+    assert(c.count(2) == 2);
+    assert(c.count(3) == 1);
+    assert(c.count(4) == 1);
+    assert(c.hash_function() == test_hash<int>());
+    assert(c.key_eq() == test_equal_to<int>());
+    assert(c.get_allocator() == min_allocator<int>());
+    assert(!c.empty());
+    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
+    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
+    assert(fabs(c.load_factor() - (float)c.size() / c.bucket_count()) < FLT_EPSILON);
+    assert(c.max_load_factor() == 1);
+  }
 #if TEST_STD_VER > 11
-    {
-        typedef int T;
-        typedef test_hash<T> HF;
-        typedef test_equal_to<T> Comp;
-        typedef test_allocator<T> A;
-        typedef std::unordered_multiset<T, HF, Comp, A> C;
+  {
+    typedef int T;
+    typedef test_hash<T> HF;
+    typedef test_equal_to<T> Comp;
+    typedef test_allocator<T> A;
+    typedef std::unordered_multiset<T, HF, Comp, A> C;
 
-        A a(42);
-        C c({
-                T(1),
-                T(2),
-                T(3),
-                T(4),
-                T(1),
-                T(2)
-            }, 12, a);
+    A a(42);
+    C c({T(1), T(2), T(3), T(4), T(1), T(2)}, 12, a);
 
-        assert(c.bucket_count() >= 12);
-        assert(c.size() == 6);
-        assert(c.count(1) == 2);
-        assert(c.count(2) == 2);
-        assert(c.count(3) == 1);
-        assert(c.count(4) == 1);
-        assert(c.hash_function() == HF());
-        assert(c.key_eq() == Comp());
-        assert(c.get_allocator() == a);
-        assert(!(c.get_allocator() == A()));
-        assert(!c.empty());
-        assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-        assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
-        assert(fabs(c.load_factor() - (float)c.size()/c.bucket_count()) < FLT_EPSILON);
-        assert(c.max_load_factor() == 1);
-    }
-    {
-        typedef int T;
-        typedef test_hash<T> HF;
-        typedef test_equal_to<T> Comp;
-        typedef test_allocator<T> A;
-        typedef std::unordered_multiset<T, HF, Comp, A> C;
+    assert(c.bucket_count() >= 12);
+    assert(c.size() == 6);
+    assert(c.count(1) == 2);
+    assert(c.count(2) == 2);
+    assert(c.count(3) == 1);
+    assert(c.count(4) == 1);
+    assert(c.hash_function() == HF());
+    assert(c.key_eq() == Comp());
+    assert(c.get_allocator() == a);
+    assert(!(c.get_allocator() == A()));
+    assert(!c.empty());
+    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
+    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
+    assert(fabs(c.load_factor() - (float)c.size() / c.bucket_count()) < FLT_EPSILON);
+    assert(c.max_load_factor() == 1);
+  }
+  {
+    typedef int T;
+    typedef test_hash<T> HF;
+    typedef test_equal_to<T> Comp;
+    typedef test_allocator<T> A;
+    typedef std::unordered_multiset<T, HF, Comp, A> C;
 
-        A a(42);
-        HF hf(43);
-        C c({
-                T(1),
-                T(2),
-                T(3),
-                T(4),
-                T(1),
-                T(2)
-            }, 12, hf, a);
+    A a(42);
+    HF hf(43);
+    C c({T(1), T(2), T(3), T(4), T(1), T(2)}, 12, hf, a);
 
-        assert(c.bucket_count() >= 12);
-        assert(c.size() == 6);
-        assert(c.count(1) == 2);
-        assert(c.count(2) == 2);
-        assert(c.count(3) == 1);
-        assert(c.count(4) == 1);
-        assert(c.hash_function() == hf);
-        assert(!(c.hash_function() == HF()));
-        assert(c.key_eq() == Comp());
-        assert(c.get_allocator() == a);
-        assert(!(c.get_allocator() == A()));
-        assert(!c.empty());
-        assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-        assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
-        assert(fabs(c.load_factor() - (float)c.size()/c.bucket_count()) < FLT_EPSILON);
-        assert(c.max_load_factor() == 1);
-    }
+    assert(c.bucket_count() >= 12);
+    assert(c.size() == 6);
+    assert(c.count(1) == 2);
+    assert(c.count(2) == 2);
+    assert(c.count(3) == 1);
+    assert(c.count(4) == 1);
+    assert(c.hash_function() == hf);
+    assert(!(c.hash_function() == HF()));
+    assert(c.key_eq() == Comp());
+    assert(c.get_allocator() == a);
+    assert(!(c.get_allocator() == A()));
+    assert(!c.empty());
+    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
+    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
+    assert(fabs(c.load_factor() - (float)c.size() / c.bucket_count()) < FLT_EPSILON);
+    assert(c.max_load_factor() == 1);
+  }
 #endif // TEST_STD_VER > 11
 
   return 0;
diff --git a/libcxx/test/std/containers/unord/unord.multiset/unord.multiset.cnstr/init_size.pass.cpp b/libcxx/test/std/containers/unord/unord.multiset/unord.multiset.cnstr/init_size.pass.cpp
index 80123b159b9ab..3b358c123bcfd 100644
--- a/libcxx/test/std/containers/unord/unord.multiset/unord.multiset.cnstr/init_size.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.multiset/unord.multiset.cnstr/init_size.pass.cpp
@@ -28,72 +28,45 @@
 #include "test_allocator.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef std::unordered_multiset<int,
-                                   test_hash<int>,
-                                   test_equal_to<int>,
-                                   test_allocator<int>
-                                   > C;
-        typedef int P;
-        C c({
-                P(1),
-                P(2),
-                P(3),
-                P(4),
-                P(1),
-                P(2)
-            },
-            7
-           );
-        LIBCPP_ASSERT(c.bucket_count() == 7);
-        assert(c.size() == 6);
-        assert(c.count(1) == 2);
-        assert(c.count(2) == 2);
-        assert(c.count(3) == 1);
-        assert(c.count(4) == 1);
-        assert(c.hash_function() == test_hash<int>());
-        assert(c.key_eq() == test_equal_to<int>());
-        assert(c.get_allocator() == test_allocator<int>());
-        assert(!c.empty());
-        assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-        assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
-        assert(std::fabs(c.load_factor() - (float)c.size()/c.bucket_count()) < FLT_EPSILON);
-        assert(c.max_load_factor() == 1);
-    }
-    {
-        typedef std::unordered_multiset<int,
-                                   test_hash<int>,
-                                   test_equal_to<int>,
-                                   min_allocator<int>
-                                   > C;
-        typedef int P;
-        C c({
-                P(1),
-                P(2),
-                P(3),
-                P(4),
-                P(1),
-                P(2)
-            },
-            7
-           );
-        LIBCPP_ASSERT(c.bucket_count() == 7);
-        assert(c.size() == 6);
-        assert(c.count(1) == 2);
-        assert(c.count(2) == 2);
-        assert(c.count(3) == 1);
-        assert(c.count(4) == 1);
-        assert(c.hash_function() == test_hash<int>());
-        assert(c.key_eq() == test_equal_to<int>());
-        assert(c.get_allocator() == min_allocator<int>());
-        assert(!c.empty());
-        assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-        assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
-        assert(std::fabs(c.load_factor() - (float)c.size()/c.bucket_count()) < FLT_EPSILON);
-        assert(c.max_load_factor() == 1);
-    }
+int main(int, char**) {
+  {
+    typedef std::unordered_multiset<int, test_hash<int>, test_equal_to<int>, test_allocator<int> > C;
+    typedef int P;
+    C c({P(1), P(2), P(3), P(4), P(1), P(2)}, 7);
+    LIBCPP_ASSERT(c.bucket_count() == 7);
+    assert(c.size() == 6);
+    assert(c.count(1) == 2);
+    assert(c.count(2) == 2);
+    assert(c.count(3) == 1);
+    assert(c.count(4) == 1);
+    assert(c.hash_function() == test_hash<int>());
+    assert(c.key_eq() == test_equal_to<int>());
+    assert(c.get_allocator() == test_allocator<int>());
+    assert(!c.empty());
+    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
+    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
+    assert(std::fabs(c.load_factor() - (float)c.size() / c.bucket_count()) < FLT_EPSILON);
+    assert(c.max_load_factor() == 1);
+  }
+  {
+    typedef std::unordered_multiset<int, test_hash<int>, test_equal_to<int>, min_allocator<int> > C;
+    typedef int P;
+    C c({P(1), P(2), P(3), P(4), P(1), P(2)}, 7);
+    LIBCPP_ASSERT(c.bucket_count() == 7);
+    assert(c.size() == 6);
+    assert(c.count(1) == 2);
+    assert(c.count(2) == 2);
+    assert(c.count(3) == 1);
+    assert(c.count(4) == 1);
+    assert(c.hash_function() == test_hash<int>());
+    assert(c.key_eq() == test_equal_to<int>());
+    assert(c.get_allocator() == min_allocator<int>());
+    assert(!c.empty());
+    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
+    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
+    assert(std::fabs(c.load_factor() - (float)c.size() / c.bucket_count()) < FLT_EPSILON);
+    assert(c.max_load_factor() == 1);
+  }
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/unord/unord.multiset/unord.multiset.cnstr/init_size_allocator.pass.cpp b/libcxx/test/std/containers/unord/unord.multiset/unord.multiset.cnstr/init_size_allocator.pass.cpp
index 4551de8a931fb..bb0cc2441bfb7 100644
--- a/libcxx/test/std/containers/unord/unord.multiset/unord.multiset.cnstr/init_size_allocator.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.multiset/unord.multiset.cnstr/init_size_allocator.pass.cpp
@@ -30,43 +30,29 @@
 #include "min_allocator.h"
 
 template <class Allocator>
-void test(const Allocator& alloc)
-{
-    typedef std::unordered_multiset<int,
-                                    test_hash<int>,
-                                    test_equal_to<int>,
-                                    Allocator> C;
-    C c({
-            1,
-            2,
-            3,
-            4,
-            1,
-            2
-        },
-        7,
-        alloc);
-
-    LIBCPP_ASSERT(c.bucket_count() == 7);
-    assert(c.size() == 6);
-    assert(c.count(1) == 2);
-    assert(c.count(2) == 2);
-    assert(c.count(3) == 1);
-    assert(c.count(4) == 1);
-    assert(c.hash_function() == test_hash<int>());
-    assert(c.key_eq() == test_equal_to<int>());
-    assert(c.get_allocator() == alloc);
-    assert(!c.empty());
-    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-    assert(std::fabs(c.load_factor() - (float)c.size()/c.bucket_count()) < FLT_EPSILON);
-    assert(c.max_load_factor() == 1);
+void test(const Allocator& alloc) {
+  typedef std::unordered_multiset<int, test_hash<int>, test_equal_to<int>, Allocator> C;
+  C c({1, 2, 3, 4, 1, 2}, 7, alloc);
+
+  LIBCPP_ASSERT(c.bucket_count() == 7);
+  assert(c.size() == 6);
+  assert(c.count(1) == 2);
+  assert(c.count(2) == 2);
+  assert(c.count(3) == 1);
+  assert(c.count(4) == 1);
+  assert(c.hash_function() == test_hash<int>());
+  assert(c.key_eq() == test_equal_to<int>());
+  assert(c.get_allocator() == alloc);
+  assert(!c.empty());
+  assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
+  assert(std::fabs(c.load_factor() - (float)c.size() / c.bucket_count()) < FLT_EPSILON);
+  assert(c.max_load_factor() == 1);
 }
 
-int main(int, char**)
-{
-    test(test_allocator<int>(10));
-    test(min_allocator<int>());
-    test(explicit_allocator<int>());
+int main(int, char**) {
+  test(test_allocator<int>(10));
+  test(min_allocator<int>());
+  test(explicit_allocator<int>());
 
-    return 0;
+  return 0;
 }
diff --git a/libcxx/test/std/containers/unord/unord.multiset/unord.multiset.cnstr/init_size_hash.pass.cpp b/libcxx/test/std/containers/unord/unord.multiset/unord.multiset.cnstr/init_size_hash.pass.cpp
index ae8ef8d813aa6..cea5247c31f76 100644
--- a/libcxx/test/std/containers/unord/unord.multiset/unord.multiset.cnstr/init_size_hash.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.multiset/unord.multiset.cnstr/init_size_hash.pass.cpp
@@ -29,74 +29,45 @@
 #include "test_allocator.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef std::unordered_multiset<int,
-                                   test_hash<int>,
-                                   test_equal_to<int>,
-                                   test_allocator<int>
-                                   > C;
-        typedef int P;
-        C c({
-                P(1),
-                P(2),
-                P(3),
-                P(4),
-                P(1),
-                P(2)
-            },
-            7,
-            test_hash<int>(8)
-           );
-        LIBCPP_ASSERT(c.bucket_count() == 7);
-        assert(c.size() == 6);
-        assert(c.count(1) == 2);
-        assert(c.count(2) == 2);
-        assert(c.count(3) == 1);
-        assert(c.count(4) == 1);
-        assert(c.hash_function() == test_hash<int>(8));
-        assert(c.key_eq() == test_equal_to<int>());
-        assert(c.get_allocator() == test_allocator<int>());
-        assert(!c.empty());
-        assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-        assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
-        assert(std::fabs(c.load_factor() - (float)c.size()/c.bucket_count()) < FLT_EPSILON);
-        assert(c.max_load_factor() == 1);
-    }
-    {
-        typedef std::unordered_multiset<int,
-                                   test_hash<int>,
-                                   test_equal_to<int>,
-                                   min_allocator<int>
-                                   > C;
-        typedef int P;
-        C c({
-                P(1),
-                P(2),
-                P(3),
-                P(4),
-                P(1),
-                P(2)
-            },
-            7,
-            test_hash<int>(8)
-           );
-        LIBCPP_ASSERT(c.bucket_count() == 7);
-        assert(c.size() == 6);
-        assert(c.count(1) == 2);
-        assert(c.count(2) == 2);
-        assert(c.count(3) == 1);
-        assert(c.count(4) == 1);
-        assert(c.hash_function() == test_hash<int>(8));
-        assert(c.key_eq() == test_equal_to<int>());
-        assert(c.get_allocator() == min_allocator<int>());
-        assert(!c.empty());
-        assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-        assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
-        assert(std::fabs(c.load_factor() - (float)c.size()/c.bucket_count()) < FLT_EPSILON);
-        assert(c.max_load_factor() == 1);
-    }
+int main(int, char**) {
+  {
+    typedef std::unordered_multiset<int, test_hash<int>, test_equal_to<int>, test_allocator<int> > C;
+    typedef int P;
+    C c({P(1), P(2), P(3), P(4), P(1), P(2)}, 7, test_hash<int>(8));
+    LIBCPP_ASSERT(c.bucket_count() == 7);
+    assert(c.size() == 6);
+    assert(c.count(1) == 2);
+    assert(c.count(2) == 2);
+    assert(c.count(3) == 1);
+    assert(c.count(4) == 1);
+    assert(c.hash_function() == test_hash<int>(8));
+    assert(c.key_eq() == test_equal_to<int>());
+    assert(c.get_allocator() == test_allocator<int>());
+    assert(!c.empty());
+    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
+    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
+    assert(std::fabs(c.load_factor() - (float)c.size() / c.bucket_count()) < FLT_EPSILON);
+    assert(c.max_load_factor() == 1);
+  }
+  {
+    typedef std::unordered_multiset<int, test_hash<int>, test_equal_to<int>, min_allocator<int> > C;
+    typedef int P;
+    C c({P(1), P(2), P(3), P(4), P(1), P(2)}, 7, test_hash<int>(8));
+    LIBCPP_ASSERT(c.bucket_count() == 7);
+    assert(c.size() == 6);
+    assert(c.count(1) == 2);
+    assert(c.count(2) == 2);
+    assert(c.count(3) == 1);
+    assert(c.count(4) == 1);
+    assert(c.hash_function() == test_hash<int>(8));
+    assert(c.key_eq() == test_equal_to<int>());
+    assert(c.get_allocator() == min_allocator<int>());
+    assert(!c.empty());
+    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
+    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
+    assert(std::fabs(c.load_factor() - (float)c.size() / c.bucket_count()) < FLT_EPSILON);
+    assert(c.max_load_factor() == 1);
+  }
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/unord/unord.multiset/unord.multiset.cnstr/init_size_hash_allocator.pass.cpp b/libcxx/test/std/containers/unord/unord.multiset/unord.multiset.cnstr/init_size_hash_allocator.pass.cpp
index 01e51b1951aae..24b802f4fcca2 100644
--- a/libcxx/test/std/containers/unord/unord.multiset/unord.multiset.cnstr/init_size_hash_allocator.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.multiset/unord.multiset.cnstr/init_size_hash_allocator.pass.cpp
@@ -31,44 +31,29 @@
 #include "min_allocator.h"
 
 template <class Allocator>
-void test(const Allocator& alloc)
-{
-    typedef std::unordered_multiset<int,
-                                    test_hash<int>,
-                                    test_equal_to<int>,
-                                    Allocator> C;
-    C c({
-            1,
-            2,
-            3,
-            4,
-            1,
-            2
-        },
-        7,
-        test_hash<int>(5),
-        alloc);
-
-    LIBCPP_ASSERT(c.bucket_count() == 7);
-    assert(c.size() == 6);
-    assert(c.count(1) == 2);
-    assert(c.count(2) == 2);
-    assert(c.count(3) == 1);
-    assert(c.count(4) == 1);
-    assert(c.hash_function() == test_hash<int>(5));
-    assert(c.key_eq() == test_equal_to<int>());
-    assert(c.get_allocator() == alloc);
-    assert(!c.empty());
-    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-    assert(std::fabs(c.load_factor() - (float)c.size()/c.bucket_count()) < FLT_EPSILON);
-    assert(c.max_load_factor() == 1);
+void test(const Allocator& alloc) {
+  typedef std::unordered_multiset<int, test_hash<int>, test_equal_to<int>, Allocator> C;
+  C c({1, 2, 3, 4, 1, 2}, 7, test_hash<int>(5), alloc);
+
+  LIBCPP_ASSERT(c.bucket_count() == 7);
+  assert(c.size() == 6);
+  assert(c.count(1) == 2);
+  assert(c.count(2) == 2);
+  assert(c.count(3) == 1);
+  assert(c.count(4) == 1);
+  assert(c.hash_function() == test_hash<int>(5));
+  assert(c.key_eq() == test_equal_to<int>());
+  assert(c.get_allocator() == alloc);
+  assert(!c.empty());
+  assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
+  assert(std::fabs(c.load_factor() - (float)c.size() / c.bucket_count()) < FLT_EPSILON);
+  assert(c.max_load_factor() == 1);
 }
 
-int main(int, char**)
-{
-    test(test_allocator<int>(10));
-    test(min_allocator<int>());
-    test(explicit_allocator<int>());
+int main(int, char**) {
+  test(test_allocator<int>(10));
+  test(min_allocator<int>());
+  test(explicit_allocator<int>());
 
-    return 0;
+  return 0;
 }
diff --git a/libcxx/test/std/containers/unord/unord.multiset/unord.multiset.cnstr/init_size_hash_equal.pass.cpp b/libcxx/test/std/containers/unord/unord.multiset/unord.multiset.cnstr/init_size_hash_equal.pass.cpp
index ee10f31fd7756..d4f227b3b5dd9 100644
--- a/libcxx/test/std/containers/unord/unord.multiset/unord.multiset.cnstr/init_size_hash_equal.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.multiset/unord.multiset.cnstr/init_size_hash_equal.pass.cpp
@@ -29,76 +29,45 @@
 #include "test_allocator.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef std::unordered_multiset<int,
-                                   test_hash<int>,
-                                   test_equal_to<int>,
-                                   test_allocator<int>
-                                   > C;
-        typedef int P;
-        C c({
-                P(1),
-                P(2),
-                P(3),
-                P(4),
-                P(1),
-                P(2)
-            },
-            7,
-            test_hash<int>(8),
-            test_equal_to<int>(9)
-           );
-        LIBCPP_ASSERT(c.bucket_count() == 7);
-        assert(c.size() == 6);
-        assert(c.count(1) == 2);
-        assert(c.count(2) == 2);
-        assert(c.count(3) == 1);
-        assert(c.count(4) == 1);
-        assert(c.hash_function() == test_hash<int>(8));
-        assert(c.key_eq() == test_equal_to<int>(9));
-        assert(c.get_allocator() == test_allocator<int>());
-        assert(!c.empty());
-        assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-        assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
-        assert(std::fabs(c.load_factor() - (float)c.size()/c.bucket_count()) < FLT_EPSILON);
-        assert(c.max_load_factor() == 1);
-    }
-    {
-        typedef std::unordered_multiset<int,
-                                   test_hash<int>,
-                                   test_equal_to<int>,
-                                   min_allocator<int>
-                                   > C;
-        typedef int P;
-        C c({
-                P(1),
-                P(2),
-                P(3),
-                P(4),
-                P(1),
-                P(2)
-            },
-            7,
-            test_hash<int>(8),
-            test_equal_to<int>(9)
-           );
-        LIBCPP_ASSERT(c.bucket_count() == 7);
-        assert(c.size() == 6);
-        assert(c.count(1) == 2);
-        assert(c.count(2) == 2);
-        assert(c.count(3) == 1);
-        assert(c.count(4) == 1);
-        assert(c.hash_function() == test_hash<int>(8));
-        assert(c.key_eq() == test_equal_to<int>(9));
-        assert(c.get_allocator() == min_allocator<int>());
-        assert(!c.empty());
-        assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-        assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
-        assert(std::fabs(c.load_factor() - (float)c.size()/c.bucket_count()) < FLT_EPSILON);
-        assert(c.max_load_factor() == 1);
-    }
+int main(int, char**) {
+  {
+    typedef std::unordered_multiset<int, test_hash<int>, test_equal_to<int>, test_allocator<int> > C;
+    typedef int P;
+    C c({P(1), P(2), P(3), P(4), P(1), P(2)}, 7, test_hash<int>(8), test_equal_to<int>(9));
+    LIBCPP_ASSERT(c.bucket_count() == 7);
+    assert(c.size() == 6);
+    assert(c.count(1) == 2);
+    assert(c.count(2) == 2);
+    assert(c.count(3) == 1);
+    assert(c.count(4) == 1);
+    assert(c.hash_function() == test_hash<int>(8));
+    assert(c.key_eq() == test_equal_to<int>(9));
+    assert(c.get_allocator() == test_allocator<int>());
+    assert(!c.empty());
+    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
+    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
+    assert(std::fabs(c.load_factor() - (float)c.size() / c.bucket_count()) < FLT_EPSILON);
+    assert(c.max_load_factor() == 1);
+  }
+  {
+    typedef std::unordered_multiset<int, test_hash<int>, test_equal_to<int>, min_allocator<int> > C;
+    typedef int P;
+    C c({P(1), P(2), P(3), P(4), P(1), P(2)}, 7, test_hash<int>(8), test_equal_to<int>(9));
+    LIBCPP_ASSERT(c.bucket_count() == 7);
+    assert(c.size() == 6);
+    assert(c.count(1) == 2);
+    assert(c.count(2) == 2);
+    assert(c.count(3) == 1);
+    assert(c.count(4) == 1);
+    assert(c.hash_function() == test_hash<int>(8));
+    assert(c.key_eq() == test_equal_to<int>(9));
+    assert(c.get_allocator() == min_allocator<int>());
+    assert(!c.empty());
+    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
+    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
+    assert(std::fabs(c.load_factor() - (float)c.size() / c.bucket_count()) < FLT_EPSILON);
+    assert(c.max_load_factor() == 1);
+  }
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/unord/unord.multiset/unord.multiset.cnstr/init_size_hash_equal_allocator.pass.cpp b/libcxx/test/std/containers/unord/unord.multiset/unord.multiset.cnstr/init_size_hash_equal_allocator.pass.cpp
index 9ca37e318cfaa..16096977795c0 100644
--- a/libcxx/test/std/containers/unord/unord.multiset/unord.multiset.cnstr/init_size_hash_equal_allocator.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.multiset/unord.multiset.cnstr/init_size_hash_equal_allocator.pass.cpp
@@ -29,78 +29,45 @@
 #include "test_allocator.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef std::unordered_multiset<int,
-                                   test_hash<int>,
-                                   test_equal_to<int>,
-                                   test_allocator<int>
-                                   > C;
-        typedef int P;
-        C c({
-                P(1),
-                P(2),
-                P(3),
-                P(4),
-                P(1),
-                P(2)
-            },
-            7,
-            test_hash<int>(8),
-            test_equal_to<int>(9),
-            test_allocator<int>(10)
-           );
-        LIBCPP_ASSERT(c.bucket_count() == 7);
-        assert(c.size() == 6);
-        assert(c.count(1) == 2);
-        assert(c.count(2) == 2);
-        assert(c.count(3) == 1);
-        assert(c.count(4) == 1);
-        assert(c.hash_function() == test_hash<int>(8));
-        assert(c.key_eq() == test_equal_to<int>(9));
-        assert(c.get_allocator() == test_allocator<int>(10));
-        assert(!c.empty());
-        assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-        assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
-        assert(std::fabs(c.load_factor() - (float)c.size()/c.bucket_count()) < FLT_EPSILON);
-        assert(c.max_load_factor() == 1);
-    }
-    {
-        typedef std::unordered_multiset<int,
-                                   test_hash<int>,
-                                   test_equal_to<int>,
-                                   min_allocator<int>
-                                   > C;
-        typedef int P;
-        C c({
-                P(1),
-                P(2),
-                P(3),
-                P(4),
-                P(1),
-                P(2)
-            },
-            7,
-            test_hash<int>(8),
-            test_equal_to<int>(9),
-            min_allocator<int>()
-           );
-        LIBCPP_ASSERT(c.bucket_count() == 7);
-        assert(c.size() == 6);
-        assert(c.count(1) == 2);
-        assert(c.count(2) == 2);
-        assert(c.count(3) == 1);
-        assert(c.count(4) == 1);
-        assert(c.hash_function() == test_hash<int>(8));
-        assert(c.key_eq() == test_equal_to<int>(9));
-        assert(c.get_allocator() == min_allocator<int>());
-        assert(!c.empty());
-        assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-        assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
-        assert(std::fabs(c.load_factor() - (float)c.size()/c.bucket_count()) < FLT_EPSILON);
-        assert(c.max_load_factor() == 1);
-    }
+int main(int, char**) {
+  {
+    typedef std::unordered_multiset<int, test_hash<int>, test_equal_to<int>, test_allocator<int> > C;
+    typedef int P;
+    C c({P(1), P(2), P(3), P(4), P(1), P(2)}, 7, test_hash<int>(8), test_equal_to<int>(9), test_allocator<int>(10));
+    LIBCPP_ASSERT(c.bucket_count() == 7);
+    assert(c.size() == 6);
+    assert(c.count(1) == 2);
+    assert(c.count(2) == 2);
+    assert(c.count(3) == 1);
+    assert(c.count(4) == 1);
+    assert(c.hash_function() == test_hash<int>(8));
+    assert(c.key_eq() == test_equal_to<int>(9));
+    assert(c.get_allocator() == test_allocator<int>(10));
+    assert(!c.empty());
+    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
+    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
+    assert(std::fabs(c.load_factor() - (float)c.size() / c.bucket_count()) < FLT_EPSILON);
+    assert(c.max_load_factor() == 1);
+  }
+  {
+    typedef std::unordered_multiset<int, test_hash<int>, test_equal_to<int>, min_allocator<int> > C;
+    typedef int P;
+    C c({P(1), P(2), P(3), P(4), P(1), P(2)}, 7, test_hash<int>(8), test_equal_to<int>(9), min_allocator<int>());
+    LIBCPP_ASSERT(c.bucket_count() == 7);
+    assert(c.size() == 6);
+    assert(c.count(1) == 2);
+    assert(c.count(2) == 2);
+    assert(c.count(3) == 1);
+    assert(c.count(4) == 1);
+    assert(c.hash_function() == test_hash<int>(8));
+    assert(c.key_eq() == test_equal_to<int>(9));
+    assert(c.get_allocator() == min_allocator<int>());
+    assert(!c.empty());
+    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
+    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
+    assert(std::fabs(c.load_factor() - (float)c.size() / c.bucket_count()) < FLT_EPSILON);
+    assert(c.max_load_factor() == 1);
+  }
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/unord/unord.multiset/unord.multiset.cnstr/iter_iter.pass.cpp b/libcxx/test/std/containers/unord/unord.multiset/unord.multiset.cnstr/iter_iter.pass.cpp
index 5fe3a02e4af58..e01ab17e34dff 100644
--- a/libcxx/test/std/containers/unord/unord.multiset/unord.multiset.cnstr/iter_iter.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.multiset/unord.multiset.cnstr/iter_iter.pass.cpp
@@ -28,143 +28,102 @@
 #include "test_allocator.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef std::unordered_multiset<int,
-                                   test_hash<int>,
-                                   test_equal_to<int>,
-                                   test_allocator<int>
-                                   > C;
-        typedef int P;
-        P a[] =
-        {
-            P(1),
-            P(2),
-            P(3),
-            P(4),
-            P(1),
-            P(2)
-        };
-        C c(cpp17_input_iterator<P*>(a), cpp17_input_iterator<P*>(a + sizeof(a)/sizeof(a[0])));
-        assert(c.bucket_count() >= 7);
-        assert(c.size() == 6);
-        assert(c.count(1) == 2);
-        assert(c.count(2) == 2);
-        assert(c.count(3) == 1);
-        assert(c.count(4) == 1);
-        assert(c.hash_function() == test_hash<int>());
-        assert(c.key_eq() == test_equal_to<int>());
-        assert(c.get_allocator() == test_allocator<int>());
-        assert(!c.empty());
-        assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-        assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
-        assert(fabs(c.load_factor() - (float)c.size()/c.bucket_count()) < FLT_EPSILON);
-        assert(c.max_load_factor() == 1);
-    }
+int main(int, char**) {
+  {
+    typedef std::unordered_multiset<int, test_hash<int>, test_equal_to<int>, test_allocator<int> > C;
+    typedef int P;
+    P a[] = {P(1), P(2), P(3), P(4), P(1), P(2)};
+    C c(cpp17_input_iterator<P*>(a), cpp17_input_iterator<P*>(a + sizeof(a) / sizeof(a[0])));
+    assert(c.bucket_count() >= 7);
+    assert(c.size() == 6);
+    assert(c.count(1) == 2);
+    assert(c.count(2) == 2);
+    assert(c.count(3) == 1);
+    assert(c.count(4) == 1);
+    assert(c.hash_function() == test_hash<int>());
+    assert(c.key_eq() == test_equal_to<int>());
+    assert(c.get_allocator() == test_allocator<int>());
+    assert(!c.empty());
+    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
+    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
+    assert(fabs(c.load_factor() - (float)c.size() / c.bucket_count()) < FLT_EPSILON);
+    assert(c.max_load_factor() == 1);
+  }
 #if TEST_STD_VER >= 11
-    {
-        typedef std::unordered_multiset<int,
-                                   test_hash<int>,
-                                   test_equal_to<int>,
-                                   min_allocator<int>
-                                   > C;
-        typedef int P;
-        P a[] =
-        {
-            P(1),
-            P(2),
-            P(3),
-            P(4),
-            P(1),
-            P(2)
-        };
-        C c(cpp17_input_iterator<P*>(a), cpp17_input_iterator<P*>(a + sizeof(a)/sizeof(a[0])));
-        assert(c.bucket_count() >= 7);
-        assert(c.size() == 6);
-        assert(c.count(1) == 2);
-        assert(c.count(2) == 2);
-        assert(c.count(3) == 1);
-        assert(c.count(4) == 1);
-        assert(c.hash_function() == test_hash<int>());
-        assert(c.key_eq() == test_equal_to<int>());
-        assert(c.get_allocator() == min_allocator<int>());
-        assert(!c.empty());
-        assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-        assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
-        assert(fabs(c.load_factor() - (float)c.size()/c.bucket_count()) < FLT_EPSILON);
-        assert(c.max_load_factor() == 1);
-    }
-#if TEST_STD_VER > 11
-    {
-        typedef int T;
-        typedef test_hash<T> HF;
-        typedef test_equal_to<T> Comp;
-        typedef test_allocator<T> A;
-        typedef std::unordered_multiset<T, HF, Comp, A> C;
-        T arr[] =
-        {
-            T(1),
-            T(2),
-            T(3),
-            T(4),
-            T(1),
-            T(2)
-        };
-        A a(42);
-        C c(cpp17_input_iterator<T*>(arr), cpp17_input_iterator<T*>(arr + sizeof(arr)/sizeof(arr[0])), 12, a);
-        assert(c.bucket_count() >= 12);
-        assert(c.size() == 6);
-        assert(c.count(1) == 2);
-        assert(c.count(2) == 2);
-        assert(c.count(3) == 1);
-        assert(c.count(4) == 1);
-        assert(c.hash_function() == HF());
-        assert(c.key_eq() == Comp());
-        assert(c.get_allocator() == a);
-        assert(!(c.get_allocator() == A()));
-        assert(!c.empty());
-        assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-        assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
-        assert(fabs(c.load_factor() - (float)c.size()/c.bucket_count()) < FLT_EPSILON);
-        assert(c.max_load_factor() == 1);
-    }
-    {
-        typedef int T;
-        typedef test_hash<T> HF;
-        typedef test_equal_to<T> Comp;
-        typedef test_allocator<T> A;
-        typedef std::unordered_multiset<T, HF, Comp, A> C;
-        T arr[] =
-        {
-            T(1),
-            T(2),
-            T(3),
-            T(4),
-            T(1),
-            T(2)
-        };
-        HF hf(43);
-        A a(42);
-        C c(cpp17_input_iterator<T*>(arr), cpp17_input_iterator<T*>(arr + sizeof(arr)/sizeof(arr[0])), 16, hf, a);
-        assert(c.bucket_count() >= 16);
-        assert(c.size() == 6);
-        assert(c.count(1) == 2);
-        assert(c.count(2) == 2);
-        assert(c.count(3) == 1);
-        assert(c.count(4) == 1);
-        assert(c.hash_function() == hf);
-        assert(!(c.hash_function() == HF()));
-        assert(c.key_eq() == Comp());
-        assert(c.get_allocator() == a);
-        assert(!(c.get_allocator() == A()));
-        assert(!c.empty());
-        assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-        assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
-        assert(fabs(c.load_factor() - (float)c.size()/c.bucket_count()) < FLT_EPSILON);
-        assert(c.max_load_factor() == 1);
-    }
-#endif
+  {
+    typedef std::unordered_multiset<int, test_hash<int>, test_equal_to<int>, min_allocator<int> > C;
+    typedef int P;
+    P a[] = {P(1), P(2), P(3), P(4), P(1), P(2)};
+    C c(cpp17_input_iterator<P*>(a), cpp17_input_iterator<P*>(a + sizeof(a) / sizeof(a[0])));
+    assert(c.bucket_count() >= 7);
+    assert(c.size() == 6);
+    assert(c.count(1) == 2);
+    assert(c.count(2) == 2);
+    assert(c.count(3) == 1);
+    assert(c.count(4) == 1);
+    assert(c.hash_function() == test_hash<int>());
+    assert(c.key_eq() == test_equal_to<int>());
+    assert(c.get_allocator() == min_allocator<int>());
+    assert(!c.empty());
+    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
+    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
+    assert(fabs(c.load_factor() - (float)c.size() / c.bucket_count()) < FLT_EPSILON);
+    assert(c.max_load_factor() == 1);
+  }
+#  if TEST_STD_VER > 11
+  {
+    typedef int T;
+    typedef test_hash<T> HF;
+    typedef test_equal_to<T> Comp;
+    typedef test_allocator<T> A;
+    typedef std::unordered_multiset<T, HF, Comp, A> C;
+    T arr[] = {T(1), T(2), T(3), T(4), T(1), T(2)};
+    A a(42);
+    C c(cpp17_input_iterator<T*>(arr), cpp17_input_iterator<T*>(arr + sizeof(arr) / sizeof(arr[0])), 12, a);
+    assert(c.bucket_count() >= 12);
+    assert(c.size() == 6);
+    assert(c.count(1) == 2);
+    assert(c.count(2) == 2);
+    assert(c.count(3) == 1);
+    assert(c.count(4) == 1);
+    assert(c.hash_function() == HF());
+    assert(c.key_eq() == Comp());
+    assert(c.get_allocator() == a);
+    assert(!(c.get_allocator() == A()));
+    assert(!c.empty());
+    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
+    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
+    assert(fabs(c.load_factor() - (float)c.size() / c.bucket_count()) < FLT_EPSILON);
+    assert(c.max_load_factor() == 1);
+  }
+  {
+    typedef int T;
+    typedef test_hash<T> HF;
+    typedef test_equal_to<T> Comp;
+    typedef test_allocator<T> A;
+    typedef std::unordered_multiset<T, HF, Comp, A> C;
+    T arr[] = {T(1), T(2), T(3), T(4), T(1), T(2)};
+    HF hf(43);
+    A a(42);
+    C c(cpp17_input_iterator<T*>(arr), cpp17_input_iterator<T*>(arr + sizeof(arr) / sizeof(arr[0])), 16, hf, a);
+    assert(c.bucket_count() >= 16);
+    assert(c.size() == 6);
+    assert(c.count(1) == 2);
+    assert(c.count(2) == 2);
+    assert(c.count(3) == 1);
+    assert(c.count(4) == 1);
+    assert(c.hash_function() == hf);
+    assert(!(c.hash_function() == HF()));
+    assert(c.key_eq() == Comp());
+    assert(c.get_allocator() == a);
+    assert(!(c.get_allocator() == A()));
+    assert(!c.empty());
+    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
+    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
+    assert(fabs(c.load_factor() - (float)c.size() / c.bucket_count()) < FLT_EPSILON);
+    assert(c.max_load_factor() == 1);
+  }
+#  endif
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/unord/unord.multiset/unord.multiset.cnstr/iter_iter_size.pass.cpp b/libcxx/test/std/containers/unord/unord.multiset/unord.multiset.cnstr/iter_iter_size.pass.cpp
index 4ce70a4b475d1..a232d4c4a2415 100644
--- a/libcxx/test/std/containers/unord/unord.multiset/unord.multiset.cnstr/iter_iter_size.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.multiset/unord.multiset.cnstr/iter_iter_size.pass.cpp
@@ -28,77 +28,48 @@
 #include "test_allocator.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef std::unordered_multiset<int,
-                                   test_hash<int>,
-                                   test_equal_to<int>,
-                                   test_allocator<int>
-                                   > C;
-        typedef int P;
-        P a[] =
-        {
-            P(1),
-            P(2),
-            P(3),
-            P(4),
-            P(1),
-            P(2)
-        };
-        C c(cpp17_input_iterator<P*>(a), cpp17_input_iterator<P*>(a + sizeof(a)/sizeof(a[0])),
-            7
-           );
-        LIBCPP_ASSERT(c.bucket_count() == 7);
-        assert(c.size() == 6);
-        assert(c.count(1) == 2);
-        assert(c.count(2) == 2);
-        assert(c.count(3) == 1);
-        assert(c.count(4) == 1);
-        assert(c.hash_function() == test_hash<int>());
-        assert(c.key_eq() == test_equal_to<int>());
-        assert(c.get_allocator() == test_allocator<int>());
-        assert(!c.empty());
-        assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-        assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
-        assert(std::fabs(c.load_factor() - (float)c.size()/c.bucket_count()) < FLT_EPSILON);
-        assert(c.max_load_factor() == 1);
-    }
+int main(int, char**) {
+  {
+    typedef std::unordered_multiset<int, test_hash<int>, test_equal_to<int>, test_allocator<int> > C;
+    typedef int P;
+    P a[] = {P(1), P(2), P(3), P(4), P(1), P(2)};
+    C c(cpp17_input_iterator<P*>(a), cpp17_input_iterator<P*>(a + sizeof(a) / sizeof(a[0])), 7);
+    LIBCPP_ASSERT(c.bucket_count() == 7);
+    assert(c.size() == 6);
+    assert(c.count(1) == 2);
+    assert(c.count(2) == 2);
+    assert(c.count(3) == 1);
+    assert(c.count(4) == 1);
+    assert(c.hash_function() == test_hash<int>());
+    assert(c.key_eq() == test_equal_to<int>());
+    assert(c.get_allocator() == test_allocator<int>());
+    assert(!c.empty());
+    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
+    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
+    assert(std::fabs(c.load_factor() - (float)c.size() / c.bucket_count()) < FLT_EPSILON);
+    assert(c.max_load_factor() == 1);
+  }
 #if TEST_STD_VER >= 11
-    {
-        typedef std::unordered_multiset<int,
-                                   test_hash<int>,
-                                   test_equal_to<int>,
-                                   min_allocator<int>
-                                   > C;
-        typedef int P;
-        P a[] =
-        {
-            P(1),
-            P(2),
-            P(3),
-            P(4),
-            P(1),
-            P(2)
-        };
-        C c(cpp17_input_iterator<P*>(a), cpp17_input_iterator<P*>(a + sizeof(a)/sizeof(a[0])),
-            7
-           );
-        LIBCPP_ASSERT(c.bucket_count() == 7);
-        assert(c.size() == 6);
-        assert(c.count(1) == 2);
-        assert(c.count(2) == 2);
-        assert(c.count(3) == 1);
-        assert(c.count(4) == 1);
-        assert(c.hash_function() == test_hash<int>());
-        assert(c.key_eq() == test_equal_to<int>());
-        assert(c.get_allocator() == min_allocator<int>());
-        assert(!c.empty());
-        assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-        assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
-        assert(std::fabs(c.load_factor() - (float)c.size()/c.bucket_count()) < FLT_EPSILON);
-        assert(c.max_load_factor() == 1);
-    }
+  {
+    typedef std::unordered_multiset<int, test_hash<int>, test_equal_to<int>, min_allocator<int> > C;
+    typedef int P;
+    P a[] = {P(1), P(2), P(3), P(4), P(1), P(2)};
+    C c(cpp17_input_iterator<P*>(a), cpp17_input_iterator<P*>(a + sizeof(a) / sizeof(a[0])), 7);
+    LIBCPP_ASSERT(c.bucket_count() == 7);
+    assert(c.size() == 6);
+    assert(c.count(1) == 2);
+    assert(c.count(2) == 2);
+    assert(c.count(3) == 1);
+    assert(c.count(4) == 1);
+    assert(c.hash_function() == test_hash<int>());
+    assert(c.key_eq() == test_equal_to<int>());
+    assert(c.get_allocator() == min_allocator<int>());
+    assert(!c.empty());
+    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
+    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
+    assert(std::fabs(c.load_factor() - (float)c.size() / c.bucket_count()) < FLT_EPSILON);
+    assert(c.max_load_factor() == 1);
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/unord/unord.multiset/unord.multiset.cnstr/iter_iter_size_allocator.pass.cpp b/libcxx/test/std/containers/unord/unord.multiset/unord.multiset.cnstr/iter_iter_size_allocator.pass.cpp
index 3d9509e76417d..994ec058fae94 100644
--- a/libcxx/test/std/containers/unord/unord.multiset/unord.multiset.cnstr/iter_iter_size_allocator.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.multiset/unord.multiset.cnstr/iter_iter_size_allocator.pass.cpp
@@ -33,45 +33,29 @@
 #include "min_allocator.h"
 
 template <class Allocator>
-void test(const Allocator& alloc)
-{
-    typedef std::unordered_multiset<int,
-                                    test_hash<int>,
-                                    test_equal_to<int>,
-                                    Allocator> C;
-    int a[] =
-    {
-        1,
-        2,
-        3,
-        4,
-        1,
-        2
-    };
-    C c(cpp17_input_iterator<int*>(a), cpp17_input_iterator<int*>(a + sizeof(a)/sizeof(a[0])),
-        7,
-        alloc
-        );
-    LIBCPP_ASSERT(c.bucket_count() == 7);
-    assert(c.size() == 6);
-    assert(c.count(1) == 2);
-    assert(c.count(2) == 2);
-    assert(c.count(3) == 1);
-    assert(c.count(4) == 1);
-    assert(c.hash_function() == test_hash<int>());
-    assert(c.key_eq() == test_equal_to<int>());
-    assert(c.get_allocator() == alloc);
-    assert(!c.empty());
-    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-    assert(std::fabs(c.load_factor() - (float)c.size()/c.bucket_count()) < FLT_EPSILON);
-    assert(c.max_load_factor() == 1);
+void test(const Allocator& alloc) {
+  typedef std::unordered_multiset<int, test_hash<int>, test_equal_to<int>, Allocator> C;
+  int a[] = {1, 2, 3, 4, 1, 2};
+  C c(cpp17_input_iterator<int*>(a), cpp17_input_iterator<int*>(a + sizeof(a) / sizeof(a[0])), 7, alloc);
+  LIBCPP_ASSERT(c.bucket_count() == 7);
+  assert(c.size() == 6);
+  assert(c.count(1) == 2);
+  assert(c.count(2) == 2);
+  assert(c.count(3) == 1);
+  assert(c.count(4) == 1);
+  assert(c.hash_function() == test_hash<int>());
+  assert(c.key_eq() == test_equal_to<int>());
+  assert(c.get_allocator() == alloc);
+  assert(!c.empty());
+  assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
+  assert(std::fabs(c.load_factor() - (float)c.size() / c.bucket_count()) < FLT_EPSILON);
+  assert(c.max_load_factor() == 1);
 }
 
-int main(int, char**)
-{
-    test(test_allocator<int>(10));
-    test(min_allocator<int>());
-    test(explicit_allocator<int>());
+int main(int, char**) {
+  test(test_allocator<int>(10));
+  test(min_allocator<int>());
+  test(explicit_allocator<int>());
 
-    return 0;
+  return 0;
 }
diff --git a/libcxx/test/std/containers/unord/unord.multiset/unord.multiset.cnstr/iter_iter_size_hash.pass.cpp b/libcxx/test/std/containers/unord/unord.multiset/unord.multiset.cnstr/iter_iter_size_hash.pass.cpp
index 7db30cf91dc73..667b026e23259 100644
--- a/libcxx/test/std/containers/unord/unord.multiset/unord.multiset.cnstr/iter_iter_size_hash.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.multiset/unord.multiset.cnstr/iter_iter_size_hash.pass.cpp
@@ -29,79 +29,48 @@
 #include "test_allocator.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef std::unordered_multiset<int,
-                                   test_hash<int>,
-                                   test_equal_to<int>,
-                                   test_allocator<int>
-                                   > C;
-        typedef int P;
-        P a[] =
-        {
-            P(1),
-            P(2),
-            P(3),
-            P(4),
-            P(1),
-            P(2)
-        };
-        C c(cpp17_input_iterator<P*>(a), cpp17_input_iterator<P*>(a + sizeof(a)/sizeof(a[0])),
-            7,
-            test_hash<int>(8)
-           );
-        LIBCPP_ASSERT(c.bucket_count() == 7);
-        assert(c.size() == 6);
-        assert(c.count(1) == 2);
-        assert(c.count(2) == 2);
-        assert(c.count(3) == 1);
-        assert(c.count(4) == 1);
-        assert(c.hash_function() == test_hash<int>(8));
-        assert(c.key_eq() == test_equal_to<int>());
-        assert(c.get_allocator() == test_allocator<int>());
-        assert(!c.empty());
-        assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-        assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
-        assert(std::fabs(c.load_factor() - (float)c.size()/c.bucket_count()) < FLT_EPSILON);
-        assert(c.max_load_factor() == 1);
-    }
+int main(int, char**) {
+  {
+    typedef std::unordered_multiset<int, test_hash<int>, test_equal_to<int>, test_allocator<int> > C;
+    typedef int P;
+    P a[] = {P(1), P(2), P(3), P(4), P(1), P(2)};
+    C c(cpp17_input_iterator<P*>(a), cpp17_input_iterator<P*>(a + sizeof(a) / sizeof(a[0])), 7, test_hash<int>(8));
+    LIBCPP_ASSERT(c.bucket_count() == 7);
+    assert(c.size() == 6);
+    assert(c.count(1) == 2);
+    assert(c.count(2) == 2);
+    assert(c.count(3) == 1);
+    assert(c.count(4) == 1);
+    assert(c.hash_function() == test_hash<int>(8));
+    assert(c.key_eq() == test_equal_to<int>());
+    assert(c.get_allocator() == test_allocator<int>());
+    assert(!c.empty());
+    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
+    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
+    assert(std::fabs(c.load_factor() - (float)c.size() / c.bucket_count()) < FLT_EPSILON);
+    assert(c.max_load_factor() == 1);
+  }
 #if TEST_STD_VER >= 11
-    {
-        typedef std::unordered_multiset<int,
-                                   test_hash<int>,
-                                   test_equal_to<int>,
-                                   min_allocator<int>
-                                   > C;
-        typedef int P;
-        P a[] =
-        {
-            P(1),
-            P(2),
-            P(3),
-            P(4),
-            P(1),
-            P(2)
-        };
-        C c(cpp17_input_iterator<P*>(a), cpp17_input_iterator<P*>(a + sizeof(a)/sizeof(a[0])),
-            7,
-            test_hash<int>(8)
-           );
-        LIBCPP_ASSERT(c.bucket_count() == 7);
-        assert(c.size() == 6);
-        assert(c.count(1) == 2);
-        assert(c.count(2) == 2);
-        assert(c.count(3) == 1);
-        assert(c.count(4) == 1);
-        assert(c.hash_function() == test_hash<int>(8));
-        assert(c.key_eq() == test_equal_to<int>());
-        assert(c.get_allocator() == min_allocator<int>());
-        assert(!c.empty());
-        assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-        assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
-        assert(std::fabs(c.load_factor() - (float)c.size()/c.bucket_count()) < FLT_EPSILON);
-        assert(c.max_load_factor() == 1);
-    }
+  {
+    typedef std::unordered_multiset<int, test_hash<int>, test_equal_to<int>, min_allocator<int> > C;
+    typedef int P;
+    P a[] = {P(1), P(2), P(3), P(4), P(1), P(2)};
+    C c(cpp17_input_iterator<P*>(a), cpp17_input_iterator<P*>(a + sizeof(a) / sizeof(a[0])), 7, test_hash<int>(8));
+    LIBCPP_ASSERT(c.bucket_count() == 7);
+    assert(c.size() == 6);
+    assert(c.count(1) == 2);
+    assert(c.count(2) == 2);
+    assert(c.count(3) == 1);
+    assert(c.count(4) == 1);
+    assert(c.hash_function() == test_hash<int>(8));
+    assert(c.key_eq() == test_equal_to<int>());
+    assert(c.get_allocator() == min_allocator<int>());
+    assert(!c.empty());
+    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
+    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
+    assert(std::fabs(c.load_factor() - (float)c.size() / c.bucket_count()) < FLT_EPSILON);
+    assert(c.max_load_factor() == 1);
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/unord/unord.multiset/unord.multiset.cnstr/iter_iter_size_hash_allocator.pass.cpp b/libcxx/test/std/containers/unord/unord.multiset/unord.multiset.cnstr/iter_iter_size_hash_allocator.pass.cpp
index 34341b26a79a6..138af889d5978 100644
--- a/libcxx/test/std/containers/unord/unord.multiset/unord.multiset.cnstr/iter_iter_size_hash_allocator.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.multiset/unord.multiset.cnstr/iter_iter_size_hash_allocator.pass.cpp
@@ -33,46 +33,33 @@
 #include "min_allocator.h"
 
 template <class Allocator>
-void test(const Allocator& alloc)
-{
-    typedef std::unordered_multiset<int,
-                                    test_hash<int>,
-                                    test_equal_to<int>,
-                                    Allocator> C;
-    int a[] =
-    {
-        1,
-        2,
-        3,
-        4,
-        1,
-        2
-    };
-    C c(cpp17_input_iterator<int*>(a), cpp17_input_iterator<int*>(a + sizeof(a)/sizeof(a[0])),
-        7,
-        test_hash<int>(5),
-        alloc
-        );
-    LIBCPP_ASSERT(c.bucket_count() == 7);
-    assert(c.size() == 6);
-    assert(c.count(1) == 2);
-    assert(c.count(2) == 2);
-    assert(c.count(3) == 1);
-    assert(c.count(4) == 1);
-    assert(c.hash_function() == test_hash<int>(5));
-    assert(c.key_eq() == test_equal_to<int>());
-    assert(c.get_allocator() == alloc);
-    assert(!c.empty());
-    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-    assert(std::fabs(c.load_factor() - (float)c.size()/c.bucket_count()) < FLT_EPSILON);
-    assert(c.max_load_factor() == 1);
+void test(const Allocator& alloc) {
+  typedef std::unordered_multiset<int, test_hash<int>, test_equal_to<int>, Allocator> C;
+  int a[] = {1, 2, 3, 4, 1, 2};
+  C c(cpp17_input_iterator<int*>(a),
+      cpp17_input_iterator<int*>(a + sizeof(a) / sizeof(a[0])),
+      7,
+      test_hash<int>(5),
+      alloc);
+  LIBCPP_ASSERT(c.bucket_count() == 7);
+  assert(c.size() == 6);
+  assert(c.count(1) == 2);
+  assert(c.count(2) == 2);
+  assert(c.count(3) == 1);
+  assert(c.count(4) == 1);
+  assert(c.hash_function() == test_hash<int>(5));
+  assert(c.key_eq() == test_equal_to<int>());
+  assert(c.get_allocator() == alloc);
+  assert(!c.empty());
+  assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
+  assert(std::fabs(c.load_factor() - (float)c.size() / c.bucket_count()) < FLT_EPSILON);
+  assert(c.max_load_factor() == 1);
 }
 
-int main(int, char**)
-{
-    test(test_allocator<int>(10));
-    test(min_allocator<int>());
-    test(explicit_allocator<int>());
+int main(int, char**) {
+  test(test_allocator<int>(10));
+  test(min_allocator<int>());
+  test(explicit_allocator<int>());
 
-    return 0;
+  return 0;
 }
diff --git a/libcxx/test/std/containers/unord/unord.multiset/unord.multiset.cnstr/iter_iter_size_hash_equal.pass.cpp b/libcxx/test/std/containers/unord/unord.multiset/unord.multiset.cnstr/iter_iter_size_hash_equal.pass.cpp
index dd2c09a7fcf9c..5de1ee93eb866 100644
--- a/libcxx/test/std/containers/unord/unord.multiset/unord.multiset.cnstr/iter_iter_size_hash_equal.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.multiset/unord.multiset.cnstr/iter_iter_size_hash_equal.pass.cpp
@@ -29,81 +29,56 @@
 #include "test_allocator.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef std::unordered_multiset<int,
-                                   test_hash<int>,
-                                   test_equal_to<int>,
-                                   test_allocator<int>
-                                   > C;
-        typedef int P;
-        P a[] =
-        {
-            P(1),
-            P(2),
-            P(3),
-            P(4),
-            P(1),
-            P(2)
-        };
-        C c(cpp17_input_iterator<P*>(a), cpp17_input_iterator<P*>(a + sizeof(a)/sizeof(a[0])),
-            7,
-            test_hash<int>(8),
-            test_equal_to<int>(9)
-           );
-        LIBCPP_ASSERT(c.bucket_count() == 7);
-        assert(c.size() == 6);
-        assert(c.count(1) == 2);
-        assert(c.count(2) == 2);
-        assert(c.count(3) == 1);
-        assert(c.count(4) == 1);
-        assert(c.hash_function() == test_hash<int>(8));
-        assert(c.key_eq() == test_equal_to<int>(9));
-        assert(c.get_allocator() == test_allocator<int>());
-        assert(!c.empty());
-        assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-        assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
-        assert(std::fabs(c.load_factor() - (float)c.size()/c.bucket_count()) < FLT_EPSILON);
-        assert(c.max_load_factor() == 1);
-    }
+int main(int, char**) {
+  {
+    typedef std::unordered_multiset<int, test_hash<int>, test_equal_to<int>, test_allocator<int> > C;
+    typedef int P;
+    P a[] = {P(1), P(2), P(3), P(4), P(1), P(2)};
+    C c(cpp17_input_iterator<P*>(a),
+        cpp17_input_iterator<P*>(a + sizeof(a) / sizeof(a[0])),
+        7,
+        test_hash<int>(8),
+        test_equal_to<int>(9));
+    LIBCPP_ASSERT(c.bucket_count() == 7);
+    assert(c.size() == 6);
+    assert(c.count(1) == 2);
+    assert(c.count(2) == 2);
+    assert(c.count(3) == 1);
+    assert(c.count(4) == 1);
+    assert(c.hash_function() == test_hash<int>(8));
+    assert(c.key_eq() == test_equal_to<int>(9));
+    assert(c.get_allocator() == test_allocator<int>());
+    assert(!c.empty());
+    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
+    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
+    assert(std::fabs(c.load_factor() - (float)c.size() / c.bucket_count()) < FLT_EPSILON);
+    assert(c.max_load_factor() == 1);
+  }
 #if TEST_STD_VER >= 11
-    {
-        typedef std::unordered_multiset<int,
-                                   test_hash<int>,
-                                   test_equal_to<int>,
-                                   min_allocator<int>
-                                   > C;
-        typedef int P;
-        P a[] =
-        {
-            P(1),
-            P(2),
-            P(3),
-            P(4),
-            P(1),
-            P(2)
-        };
-        C c(cpp17_input_iterator<P*>(a), cpp17_input_iterator<P*>(a + sizeof(a)/sizeof(a[0])),
-            7,
-            test_hash<int>(8),
-            test_equal_to<int>(9)
-           );
-        LIBCPP_ASSERT(c.bucket_count() == 7);
-        assert(c.size() == 6);
-        assert(c.count(1) == 2);
-        assert(c.count(2) == 2);
-        assert(c.count(3) == 1);
-        assert(c.count(4) == 1);
-        assert(c.hash_function() == test_hash<int>(8));
-        assert(c.key_eq() == test_equal_to<int>(9));
-        assert(c.get_allocator() == min_allocator<int>());
-        assert(!c.empty());
-        assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-        assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
-        assert(std::fabs(c.load_factor() - (float)c.size()/c.bucket_count()) < FLT_EPSILON);
-        assert(c.max_load_factor() == 1);
-    }
+  {
+    typedef std::unordered_multiset<int, test_hash<int>, test_equal_to<int>, min_allocator<int> > C;
+    typedef int P;
+    P a[] = {P(1), P(2), P(3), P(4), P(1), P(2)};
+    C c(cpp17_input_iterator<P*>(a),
+        cpp17_input_iterator<P*>(a + sizeof(a) / sizeof(a[0])),
+        7,
+        test_hash<int>(8),
+        test_equal_to<int>(9));
+    LIBCPP_ASSERT(c.bucket_count() == 7);
+    assert(c.size() == 6);
+    assert(c.count(1) == 2);
+    assert(c.count(2) == 2);
+    assert(c.count(3) == 1);
+    assert(c.count(4) == 1);
+    assert(c.hash_function() == test_hash<int>(8));
+    assert(c.key_eq() == test_equal_to<int>(9));
+    assert(c.get_allocator() == min_allocator<int>());
+    assert(!c.empty());
+    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
+    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
+    assert(std::fabs(c.load_factor() - (float)c.size() / c.bucket_count()) < FLT_EPSILON);
+    assert(c.max_load_factor() == 1);
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/unord/unord.multiset/unord.multiset.cnstr/iter_iter_size_hash_equal_allocator.pass.cpp b/libcxx/test/std/containers/unord/unord.multiset/unord.multiset.cnstr/iter_iter_size_hash_equal_allocator.pass.cpp
index 996ebb873ce2e..2beb5125eb427 100644
--- a/libcxx/test/std/containers/unord/unord.multiset/unord.multiset.cnstr/iter_iter_size_hash_equal_allocator.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.multiset/unord.multiset.cnstr/iter_iter_size_hash_equal_allocator.pass.cpp
@@ -30,83 +30,58 @@
 #include "test_allocator.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef std::unordered_multiset<int,
-                                   test_hash<int>,
-                                   test_equal_to<int>,
-                                   test_allocator<int>
-                                   > C;
-        typedef int P;
-        P a[] =
-        {
-            P(1),
-            P(2),
-            P(3),
-            P(4),
-            P(1),
-            P(2)
-        };
-        C c(cpp17_input_iterator<P*>(a), cpp17_input_iterator<P*>(a + sizeof(a)/sizeof(a[0])),
-            7,
-            test_hash<int>(8),
-            test_equal_to<int>(9),
-            test_allocator<int>(10)
-           );
-        LIBCPP_ASSERT(c.bucket_count() == 7);
-        assert(c.size() == 6);
-        assert(c.count(1) == 2);
-        assert(c.count(2) == 2);
-        assert(c.count(3) == 1);
-        assert(c.count(4) == 1);
-        assert(c.hash_function() == test_hash<int>(8));
-        assert(c.key_eq() == test_equal_to<int>(9));
-        assert(c.get_allocator() == test_allocator<int>(10));
-        assert(!c.empty());
-        assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-        assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
-        assert(std::fabs(c.load_factor() - (float)c.size()/c.bucket_count()) < FLT_EPSILON);
-        assert(c.max_load_factor() == 1);
-    }
+int main(int, char**) {
+  {
+    typedef std::unordered_multiset<int, test_hash<int>, test_equal_to<int>, test_allocator<int> > C;
+    typedef int P;
+    P a[] = {P(1), P(2), P(3), P(4), P(1), P(2)};
+    C c(cpp17_input_iterator<P*>(a),
+        cpp17_input_iterator<P*>(a + sizeof(a) / sizeof(a[0])),
+        7,
+        test_hash<int>(8),
+        test_equal_to<int>(9),
+        test_allocator<int>(10));
+    LIBCPP_ASSERT(c.bucket_count() == 7);
+    assert(c.size() == 6);
+    assert(c.count(1) == 2);
+    assert(c.count(2) == 2);
+    assert(c.count(3) == 1);
+    assert(c.count(4) == 1);
+    assert(c.hash_function() == test_hash<int>(8));
+    assert(c.key_eq() == test_equal_to<int>(9));
+    assert(c.get_allocator() == test_allocator<int>(10));
+    assert(!c.empty());
+    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
+    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
+    assert(std::fabs(c.load_factor() - (float)c.size() / c.bucket_count()) < FLT_EPSILON);
+    assert(c.max_load_factor() == 1);
+  }
 #if TEST_STD_VER >= 11
-    {
-        typedef std::unordered_multiset<int,
-                                   test_hash<int>,
-                                   test_equal_to<int>,
-                                   min_allocator<int>
-                                   > C;
-        typedef int P;
-        P a[] =
-        {
-            P(1),
-            P(2),
-            P(3),
-            P(4),
-            P(1),
-            P(2)
-        };
-        C c(cpp17_input_iterator<P*>(a), cpp17_input_iterator<P*>(a + sizeof(a)/sizeof(a[0])),
-            7,
-            test_hash<int>(8),
-            test_equal_to<int>(9),
-            min_allocator<int>()
-           );
-        LIBCPP_ASSERT(c.bucket_count() == 7);
-        assert(c.size() == 6);
-        assert(c.count(1) == 2);
-        assert(c.count(2) == 2);
-        assert(c.count(3) == 1);
-        assert(c.count(4) == 1);
-        assert(c.hash_function() == test_hash<int>(8));
-        assert(c.key_eq() == test_equal_to<int>(9));
-        assert(c.get_allocator() == min_allocator<int>());
-        assert(!c.empty());
-        assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-        assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
-        assert(std::fabs(c.load_factor() - (float)c.size()/c.bucket_count()) < FLT_EPSILON);
-        assert(c.max_load_factor() == 1);
-    }
+  {
+    typedef std::unordered_multiset<int, test_hash<int>, test_equal_to<int>, min_allocator<int> > C;
+    typedef int P;
+    P a[] = {P(1), P(2), P(3), P(4), P(1), P(2)};
+    C c(cpp17_input_iterator<P*>(a),
+        cpp17_input_iterator<P*>(a + sizeof(a) / sizeof(a[0])),
+        7,
+        test_hash<int>(8),
+        test_equal_to<int>(9),
+        min_allocator<int>());
+    LIBCPP_ASSERT(c.bucket_count() == 7);
+    assert(c.size() == 6);
+    assert(c.count(1) == 2);
+    assert(c.count(2) == 2);
+    assert(c.count(3) == 1);
+    assert(c.count(4) == 1);
+    assert(c.hash_function() == test_hash<int>(8));
+    assert(c.key_eq() == test_equal_to<int>(9));
+    assert(c.get_allocator() == min_allocator<int>());
+    assert(!c.empty());
+    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
+    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
+    assert(std::fabs(c.load_factor() - (float)c.size() / c.bucket_count()) < FLT_EPSILON);
+    assert(c.max_load_factor() == 1);
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/unord/unord.multiset/unord.multiset.cnstr/move.pass.cpp b/libcxx/test/std/containers/unord/unord.multiset/unord.multiset.cnstr/move.pass.cpp
index 4d4881b61f349..09a9b424389f8 100644
--- a/libcxx/test/std/containers/unord/unord.multiset/unord.multiset.cnstr/move.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.multiset/unord.multiset.cnstr/move.pass.cpp
@@ -28,142 +28,91 @@
 #include "test_allocator.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef std::unordered_multiset<int,
-                                   test_hash<int>,
-                                   test_equal_to<int>,
-                                   test_allocator<int>
-                                   > C;
-        C c0(7,
-            test_hash<int>(8),
-            test_equal_to<int>(9),
-            test_allocator<int>(10)
-           );
-        C c = std::move(c0);
-        LIBCPP_ASSERT(c.bucket_count() == 7);
-        assert(c.size() == 0);
-        assert(c.hash_function() == test_hash<int>(8));
-        assert(c.key_eq() == test_equal_to<int>(9));
-        assert(c.get_allocator() == test_allocator<int>(10));
-        assert(c.empty());
-        assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-        assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
-        assert(c.load_factor() == 0);
-        assert(c.max_load_factor() == 1);
+int main(int, char**) {
+  {
+    typedef std::unordered_multiset<int, test_hash<int>, test_equal_to<int>, test_allocator<int> > C;
+    C c0(7, test_hash<int>(8), test_equal_to<int>(9), test_allocator<int>(10));
+    C c = std::move(c0);
+    LIBCPP_ASSERT(c.bucket_count() == 7);
+    assert(c.size() == 0);
+    assert(c.hash_function() == test_hash<int>(8));
+    assert(c.key_eq() == test_equal_to<int>(9));
+    assert(c.get_allocator() == test_allocator<int>(10));
+    assert(c.empty());
+    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
+    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
+    assert(c.load_factor() == 0);
+    assert(c.max_load_factor() == 1);
 
-        assert(c0.empty());
-    }
-    {
-        typedef std::unordered_multiset<int,
-                                   test_hash<int>,
-                                   test_equal_to<int>,
-                                   test_allocator<int>
-                                   > C;
-        typedef int P;
-        P a[] =
-        {
-            P(1),
-            P(2),
-            P(3),
-            P(4),
-            P(1),
-            P(2)
-        };
-        C c0(a, a + sizeof(a)/sizeof(a[0]),
-            7,
-            test_hash<int>(8),
-            test_equal_to<int>(9),
-            test_allocator<int>(10)
-           );
-        C::iterator it0 = c0.begin();
-        C c = std::move(c0);
-        assert(it0 == c.begin()); // Iterators remain valid
-        LIBCPP_ASSERT(c.bucket_count() == 7);
-        assert(c.size() == 6);
-        assert(c.count(1) == 2);
-        assert(c.count(2) == 2);
-        assert(c.count(3) == 1);
-        assert(c.count(4) == 1);
-        assert(c.hash_function() == test_hash<int>(8));
-        assert(c.key_eq() == test_equal_to<int>(9));
-        assert(c.get_allocator() == test_allocator<int>(10));
-        assert(!c.empty());
-        assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-        assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
-        assert(std::fabs(c.load_factor() - (float)c.size()/c.bucket_count()) < FLT_EPSILON);
-        assert(c.max_load_factor() == 1);
+    assert(c0.empty());
+  }
+  {
+    typedef std::unordered_multiset<int, test_hash<int>, test_equal_to<int>, test_allocator<int> > C;
+    typedef int P;
+    P a[] = {P(1), P(2), P(3), P(4), P(1), P(2)};
+    C c0(a, a + sizeof(a) / sizeof(a[0]), 7, test_hash<int>(8), test_equal_to<int>(9), test_allocator<int>(10));
+    C::iterator it0 = c0.begin();
+    C c             = std::move(c0);
+    assert(it0 == c.begin()); // Iterators remain valid
+    LIBCPP_ASSERT(c.bucket_count() == 7);
+    assert(c.size() == 6);
+    assert(c.count(1) == 2);
+    assert(c.count(2) == 2);
+    assert(c.count(3) == 1);
+    assert(c.count(4) == 1);
+    assert(c.hash_function() == test_hash<int>(8));
+    assert(c.key_eq() == test_equal_to<int>(9));
+    assert(c.get_allocator() == test_allocator<int>(10));
+    assert(!c.empty());
+    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
+    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
+    assert(std::fabs(c.load_factor() - (float)c.size() / c.bucket_count()) < FLT_EPSILON);
+    assert(c.max_load_factor() == 1);
 
-        assert(c0.empty());
-    }
-    {
-        typedef std::unordered_multiset<int,
-                                   test_hash<int>,
-                                   test_equal_to<int>,
-                                   min_allocator<int>
-                                   > C;
-        C c0(7,
-            test_hash<int>(8),
-            test_equal_to<int>(9),
-            min_allocator<int>()
-           );
-        C c = std::move(c0);
-        LIBCPP_ASSERT(c.bucket_count() == 7);
-        assert(c.size() == 0);
-        assert(c.hash_function() == test_hash<int>(8));
-        assert(c.key_eq() == test_equal_to<int>(9));
-        assert(c.get_allocator() == min_allocator<int>());
-        assert(c.empty());
-        assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-        assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
-        assert(c.load_factor() == 0);
-        assert(c.max_load_factor() == 1);
+    assert(c0.empty());
+  }
+  {
+    typedef std::unordered_multiset<int, test_hash<int>, test_equal_to<int>, min_allocator<int> > C;
+    C c0(7, test_hash<int>(8), test_equal_to<int>(9), min_allocator<int>());
+    C c = std::move(c0);
+    LIBCPP_ASSERT(c.bucket_count() == 7);
+    assert(c.size() == 0);
+    assert(c.hash_function() == test_hash<int>(8));
+    assert(c.key_eq() == test_equal_to<int>(9));
+    assert(c.get_allocator() == min_allocator<int>());
+    assert(c.empty());
+    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
+    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
+    assert(c.load_factor() == 0);
+    assert(c.max_load_factor() == 1);
 
-        assert(c0.empty());
-    }
-    {
-        typedef std::unordered_multiset<int,
-                                   test_hash<int>,
-                                   test_equal_to<int>,
-                                   min_allocator<int>
-                                   > C;
-        typedef int P;
-        P a[] =
-        {
-            P(1),
-            P(2),
-            P(3),
-            P(4),
-            P(1),
-            P(2)
-        };
-        C c0(a, a + sizeof(a)/sizeof(a[0]),
-            7,
-            test_hash<int>(8),
-            test_equal_to<int>(9),
-            min_allocator<int>()
-           );
-        C::iterator it0 = c0.begin();
-        C c = std::move(c0);
-        assert(it0 == c.begin()); // Iterators remain valid
-        LIBCPP_ASSERT(c.bucket_count() == 7);
-        assert(c.size() == 6);
-        assert(c.count(1) == 2);
-        assert(c.count(2) == 2);
-        assert(c.count(3) == 1);
-        assert(c.count(4) == 1);
-        assert(c.hash_function() == test_hash<int>(8));
-        assert(c.key_eq() == test_equal_to<int>(9));
-        assert(c.get_allocator() == min_allocator<int>());
-        assert(!c.empty());
-        assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-        assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
-        assert(std::fabs(c.load_factor() - (float)c.size()/c.bucket_count()) < FLT_EPSILON);
-        assert(c.max_load_factor() == 1);
+    assert(c0.empty());
+  }
+  {
+    typedef std::unordered_multiset<int, test_hash<int>, test_equal_to<int>, min_allocator<int> > C;
+    typedef int P;
+    P a[] = {P(1), P(2), P(3), P(4), P(1), P(2)};
+    C c0(a, a + sizeof(a) / sizeof(a[0]), 7, test_hash<int>(8), test_equal_to<int>(9), min_allocator<int>());
+    C::iterator it0 = c0.begin();
+    C c             = std::move(c0);
+    assert(it0 == c.begin()); // Iterators remain valid
+    LIBCPP_ASSERT(c.bucket_count() == 7);
+    assert(c.size() == 6);
+    assert(c.count(1) == 2);
+    assert(c.count(2) == 2);
+    assert(c.count(3) == 1);
+    assert(c.count(4) == 1);
+    assert(c.hash_function() == test_hash<int>(8));
+    assert(c.key_eq() == test_equal_to<int>(9));
+    assert(c.get_allocator() == min_allocator<int>());
+    assert(!c.empty());
+    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
+    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
+    assert(std::fabs(c.load_factor() - (float)c.size() / c.bucket_count()) < FLT_EPSILON);
+    assert(c.max_load_factor() == 1);
 
-        assert(c0.empty());
-    }
+    assert(c0.empty());
+  }
 
-    return 0;
+  return 0;
 }
diff --git a/libcxx/test/std/containers/unord/unord.multiset/unord.multiset.cnstr/move_alloc.pass.cpp b/libcxx/test/std/containers/unord/unord.multiset/unord.multiset.cnstr/move_alloc.pass.cpp
index a2b6c3f7bb1c6..10143d8745017 100644
--- a/libcxx/test/std/containers/unord/unord.multiset/unord.multiset.cnstr/move_alloc.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.multiset/unord.multiset.cnstr/move_alloc.pass.cpp
@@ -29,172 +29,103 @@
 #include "test_allocator.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef int P;
-        typedef test_allocator<int> A;
-        typedef std::unordered_multiset<int,
-                                   test_hash<int>,
-                                   test_equal_to<int>,
-                                   A
-                                   > C;
-        P a[] =
-        {
-            P(1),
-            P(2),
-            P(3),
-            P(4),
-            P(1),
-            P(2)
-        };
-        C c0(a, a + sizeof(a)/sizeof(a[0]),
-            7,
-            test_hash<int>(8),
-            test_equal_to<int>(9),
-            A(10)
-           );
-        C c(std::move(c0), A(12));
-        assert(c.bucket_count() >= 7);
-        assert(c.size() == 6);
-        CheckConsecutiveValues<C::const_iterator>(c.find(1), c.end(), 1, 2);
-        CheckConsecutiveValues<C::const_iterator>(c.find(2), c.end(), 2, 2);
-        CheckConsecutiveValues<C::const_iterator>(c.find(3), c.end(), 3, 1);
-        CheckConsecutiveValues<C::const_iterator>(c.find(4), c.end(), 4, 1);
-        assert(c.hash_function() == test_hash<int>(8));
-        assert(c.key_eq() == test_equal_to<int>(9));
-        assert(c.get_allocator() == A(12));
-        assert(!c.empty());
-        assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-        assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
-        assert(std::fabs(c.load_factor() - (float)c.size()/c.bucket_count()) < FLT_EPSILON);
-        assert(c.max_load_factor() == 1);
+int main(int, char**) {
+  {
+    typedef int P;
+    typedef test_allocator<int> A;
+    typedef std::unordered_multiset<int, test_hash<int>, test_equal_to<int>, A > C;
+    P a[] = {P(1), P(2), P(3), P(4), P(1), P(2)};
+    C c0(a, a + sizeof(a) / sizeof(a[0]), 7, test_hash<int>(8), test_equal_to<int>(9), A(10));
+    C c(std::move(c0), A(12));
+    assert(c.bucket_count() >= 7);
+    assert(c.size() == 6);
+    CheckConsecutiveValues<C::const_iterator>(c.find(1), c.end(), 1, 2);
+    CheckConsecutiveValues<C::const_iterator>(c.find(2), c.end(), 2, 2);
+    CheckConsecutiveValues<C::const_iterator>(c.find(3), c.end(), 3, 1);
+    CheckConsecutiveValues<C::const_iterator>(c.find(4), c.end(), 4, 1);
+    assert(c.hash_function() == test_hash<int>(8));
+    assert(c.key_eq() == test_equal_to<int>(9));
+    assert(c.get_allocator() == A(12));
+    assert(!c.empty());
+    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
+    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
+    assert(std::fabs(c.load_factor() - (float)c.size() / c.bucket_count()) < FLT_EPSILON);
+    assert(c.max_load_factor() == 1);
 
-        assert(c0.empty());
-    }
-    {
-        typedef int P;
-        typedef test_allocator<int> A;
-        typedef std::unordered_multiset<int,
-                                   test_hash<int>,
-                                   test_equal_to<int>,
-                                   A
-                                   > C;
-        P a[] =
-        {
-            P(1),
-            P(2),
-            P(3),
-            P(4),
-            P(1),
-            P(2)
-        };
-        C c0(a, a + sizeof(a)/sizeof(a[0]),
-            7,
-            test_hash<int>(8),
-            test_equal_to<int>(9),
-            A(10)
-           );
-        C c(std::move(c0), A(10));
-        LIBCPP_ASSERT(c.bucket_count() == 7);
-        assert(c.size() == 6);
-        assert(c.count(1) == 2);
-        assert(c.count(2) == 2);
-        assert(c.count(3) == 1);
-        assert(c.count(4) == 1);
-        assert(c.hash_function() == test_hash<int>(8));
-        assert(c.key_eq() == test_equal_to<int>(9));
-        assert(c.get_allocator() == A(10));
-        assert(!c.empty());
-        assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-        assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
-        assert(std::fabs(c.load_factor() - (float)c.size()/c.bucket_count()) < FLT_EPSILON);
-        assert(c.max_load_factor() == 1);
+    assert(c0.empty());
+  }
+  {
+    typedef int P;
+    typedef test_allocator<int> A;
+    typedef std::unordered_multiset<int, test_hash<int>, test_equal_to<int>, A > C;
+    P a[] = {P(1), P(2), P(3), P(4), P(1), P(2)};
+    C c0(a, a + sizeof(a) / sizeof(a[0]), 7, test_hash<int>(8), test_equal_to<int>(9), A(10));
+    C c(std::move(c0), A(10));
+    LIBCPP_ASSERT(c.bucket_count() == 7);
+    assert(c.size() == 6);
+    assert(c.count(1) == 2);
+    assert(c.count(2) == 2);
+    assert(c.count(3) == 1);
+    assert(c.count(4) == 1);
+    assert(c.hash_function() == test_hash<int>(8));
+    assert(c.key_eq() == test_equal_to<int>(9));
+    assert(c.get_allocator() == A(10));
+    assert(!c.empty());
+    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
+    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
+    assert(std::fabs(c.load_factor() - (float)c.size() / c.bucket_count()) < FLT_EPSILON);
+    assert(c.max_load_factor() == 1);
 
-        assert(c0.empty());
-    }
-    {
-        typedef int P;
-        typedef min_allocator<int> A;
-        typedef std::unordered_multiset<int,
-                                   test_hash<int>,
-                                   test_equal_to<int>,
-                                   A
-                                   > C;
-        P a[] =
-        {
-            P(1),
-            P(2),
-            P(3),
-            P(4),
-            P(1),
-            P(2)
-        };
-        C c0(a, a + sizeof(a)/sizeof(a[0]),
-            7,
-            test_hash<int>(8),
-            test_equal_to<int>(9),
-            A()
-           );
-        C c(std::move(c0), A());
-        assert(c.bucket_count() >= 7);
-        assert(c.size() == 6);
-        assert(c.count(1) == 2);
-        assert(c.count(2) == 2);
-        assert(c.count(3) == 1);
-        assert(c.count(4) == 1);
-        assert(c.hash_function() == test_hash<int>(8));
-        assert(c.key_eq() == test_equal_to<int>(9));
-        assert(c.get_allocator() == A());
-        assert(!c.empty());
-        assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-        assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
-        assert(std::fabs(c.load_factor() - (float)c.size()/c.bucket_count()) < FLT_EPSILON);
-        assert(c.max_load_factor() == 1);
+    assert(c0.empty());
+  }
+  {
+    typedef int P;
+    typedef min_allocator<int> A;
+    typedef std::unordered_multiset<int, test_hash<int>, test_equal_to<int>, A > C;
+    P a[] = {P(1), P(2), P(3), P(4), P(1), P(2)};
+    C c0(a, a + sizeof(a) / sizeof(a[0]), 7, test_hash<int>(8), test_equal_to<int>(9), A());
+    C c(std::move(c0), A());
+    assert(c.bucket_count() >= 7);
+    assert(c.size() == 6);
+    assert(c.count(1) == 2);
+    assert(c.count(2) == 2);
+    assert(c.count(3) == 1);
+    assert(c.count(4) == 1);
+    assert(c.hash_function() == test_hash<int>(8));
+    assert(c.key_eq() == test_equal_to<int>(9));
+    assert(c.get_allocator() == A());
+    assert(!c.empty());
+    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
+    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
+    assert(std::fabs(c.load_factor() - (float)c.size() / c.bucket_count()) < FLT_EPSILON);
+    assert(c.max_load_factor() == 1);
 
-        assert(c0.empty());
-    }
-    {
-        typedef int P;
-        typedef min_allocator<int> A;
-        typedef std::unordered_multiset<int,
-                                   test_hash<int>,
-                                   test_equal_to<int>,
-                                   A
-                                   > C;
-        P a[] =
-        {
-            P(1),
-            P(2),
-            P(3),
-            P(4),
-            P(1),
-            P(2)
-        };
-        C c0(a, a + sizeof(a)/sizeof(a[0]),
-            7,
-            test_hash<int>(8),
-            test_equal_to<int>(9),
-            A()
-           );
-        C c(std::move(c0), A());
-        LIBCPP_ASSERT(c.bucket_count() == 7);
-        assert(c.size() == 6);
-        assert(c.count(1) == 2);
-        assert(c.count(2) == 2);
-        assert(c.count(3) == 1);
-        assert(c.count(4) == 1);
-        assert(c.hash_function() == test_hash<int>(8));
-        assert(c.key_eq() == test_equal_to<int>(9));
-        assert(c.get_allocator() == A());
-        assert(!c.empty());
-        assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-        assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
-        assert(std::fabs(c.load_factor() - (float)c.size()/c.bucket_count()) < FLT_EPSILON);
-        assert(c.max_load_factor() == 1);
+    assert(c0.empty());
+  }
+  {
+    typedef int P;
+    typedef min_allocator<int> A;
+    typedef std::unordered_multiset<int, test_hash<int>, test_equal_to<int>, A > C;
+    P a[] = {P(1), P(2), P(3), P(4), P(1), P(2)};
+    C c0(a, a + sizeof(a) / sizeof(a[0]), 7, test_hash<int>(8), test_equal_to<int>(9), A());
+    C c(std::move(c0), A());
+    LIBCPP_ASSERT(c.bucket_count() == 7);
+    assert(c.size() == 6);
+    assert(c.count(1) == 2);
+    assert(c.count(2) == 2);
+    assert(c.count(3) == 1);
+    assert(c.count(4) == 1);
+    assert(c.hash_function() == test_hash<int>(8));
+    assert(c.key_eq() == test_equal_to<int>(9));
+    assert(c.get_allocator() == A());
+    assert(!c.empty());
+    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
+    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
+    assert(std::fabs(c.load_factor() - (float)c.size() / c.bucket_count()) < FLT_EPSILON);
+    assert(c.max_load_factor() == 1);
 
-        assert(c0.empty());
-    }
+    assert(c0.empty());
+  }
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/unord/unord.multiset/unord.multiset.cnstr/move_assign_noexcept.compile.pass.cpp b/libcxx/test/std/containers/unord/unord.multiset/unord.multiset.cnstr/move_assign_noexcept.compile.pass.cpp
new file mode 100644
index 0000000000000..814810d341e2b
--- /dev/null
+++ b/libcxx/test/std/containers/unord/unord.multiset/unord.multiset.cnstr/move_assign_noexcept.compile.pass.cpp
@@ -0,0 +1,74 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// <unordered_set>
+
+// unordered_multiset& operator=(unordered_multiset&& c)
+//     noexcept(
+//          allocator_type::propagate_on_container_move_assignment::value &&
+//          is_nothrow_move_assignable<allocator_type>::value &&
+//          is_nothrow_move_assignable<key_compare>::value);
+
+// This tests a conforming extension
+
+// UNSUPPORTED: c++03
+
+#include <unordered_set>
+
+#include "test_macros.h"
+#include "MoveOnly.h"
+#include "test_allocator.h"
+
+template <class T>
+struct some_hash {
+  using value_type = T;
+  some_hash();
+  some_hash(const some_hash&);
+  some_hash& operator=(const some_hash&);
+
+  std::size_t operator()(T const&) const;
+};
+
+template <class T>
+struct some_comp {
+  using value_type = T;
+  some_comp& operator=(const some_comp&);
+  bool operator()(const T&, const T&) const { return false; }
+};
+
+template <class T>
+struct always_equal_alloc {
+  using value_type = T;
+  always_equal_alloc(const always_equal_alloc&);
+  void allocate(std::size_t);
+};
+
+template <class T>
+struct not_always_equal_alloc {
+  int i;
+  using value_type = T;
+  not_always_equal_alloc(const not_always_equal_alloc&);
+  void allocate(std::size_t);
+};
+
+template <template <class> class Alloc>
+using unordered_set_alloc =
+    std::unordered_multiset<MoveOnly, std::hash<MoveOnly>, std::equal_to<MoveOnly>, Alloc<MoveOnly>>;
+
+static_assert(std::is_nothrow_move_assignable<unordered_set_alloc<std::allocator>>::value, "");
+static_assert(!std::is_nothrow_move_assignable<unordered_set_alloc<test_allocator>>::value, "");
+#if TEST_STD_VER > 17
+static_assert(std::is_nothrow_move_assignable<unordered_set_alloc<always_equal_alloc>>::value, "");
+#endif
+static_assert(!std::is_nothrow_move_assignable<unordered_set_alloc<not_always_equal_alloc>>::value, "");
+#if defined(_LIBCPP_VERSION)
+static_assert(std::is_nothrow_move_assignable<unordered_set_alloc<other_allocator>>::value, "");
+#endif // _LIBCPP_VERSION
+static_assert(!std::is_nothrow_move_assignable<std::unordered_multiset<int, some_hash<int>>>::value, "");
+static_assert(!std::is_nothrow_move_assignable<std::unordered_multiset<int, std::hash<int>, some_comp<int>>>::value,
+              "");
diff --git a/libcxx/test/std/containers/unord/unord.multiset/unord.multiset.cnstr/move_assign_noexcept.pass.cpp b/libcxx/test/std/containers/unord/unord.multiset/unord.multiset.cnstr/move_assign_noexcept.pass.cpp
deleted file mode 100644
index 9dbca6e3e8674..0000000000000
--- a/libcxx/test/std/containers/unord/unord.multiset/unord.multiset.cnstr/move_assign_noexcept.pass.cpp
+++ /dev/null
@@ -1,75 +0,0 @@
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-// <unordered_set>
-
-// unordered_multiset& operator=(unordered_multiset&& c)
-//     noexcept(
-//          allocator_type::propagate_on_container_move_assignment::value &&
-//          is_nothrow_move_assignable<allocator_type>::value &&
-//          is_nothrow_move_assignable<key_compare>::value);
-
-// This tests a conforming extension
-
-// UNSUPPORTED: c++03
-
-#include <unordered_set>
-#include <cassert>
-
-#include "test_macros.h"
-#include "MoveOnly.h"
-#include "test_allocator.h"
-
-template <class T>
-struct some_comp
-{
-    typedef T value_type;
-    some_comp& operator=(const some_comp&);
-    bool operator()(const T&, const T&) const { return false; }
-};
-
-template <class T>
-struct some_hash
-{
-    typedef T value_type;
-    some_hash();
-    some_hash(const some_hash&);
-    some_hash& operator=(const some_hash&);
-    std::size_t operator()(T const&) const;
-};
-
-int main(int, char**)
-{
-    {
-        typedef std::unordered_multiset<MoveOnly> C;
-        static_assert(std::is_nothrow_move_assignable<C>::value, "");
-    }
-    {
-        typedef std::unordered_multiset<MoveOnly, std::hash<MoveOnly>,
-                           std::equal_to<MoveOnly>, test_allocator<MoveOnly>> C;
-        static_assert(!std::is_nothrow_move_assignable<C>::value, "");
-    }
-#if defined(_LIBCPP_VERSION)
-    {
-        typedef std::unordered_multiset<MoveOnly, std::hash<MoveOnly>,
-                          std::equal_to<MoveOnly>, other_allocator<MoveOnly>> C;
-        static_assert(std::is_nothrow_move_assignable<C>::value, "");
-    }
-#endif // _LIBCPP_VERSION
-    {
-        typedef std::unordered_multiset<MoveOnly, some_hash<MoveOnly>> C;
-        static_assert(!std::is_nothrow_move_assignable<C>::value, "");
-    }
-    {
-        typedef std::unordered_multiset<MoveOnly, std::hash<MoveOnly>,
-                                                         some_comp<MoveOnly>> C;
-        static_assert(!std::is_nothrow_move_assignable<C>::value, "");
-    }
-
-  return 0;
-}
diff --git a/libcxx/test/std/containers/unord/unord.multiset/unord.multiset.cnstr/move_noexcept.pass.cpp b/libcxx/test/std/containers/unord/unord.multiset/unord.multiset.cnstr/move_noexcept.pass.cpp
index 8e6e4643c88b3..4024fd68622c4 100644
--- a/libcxx/test/std/containers/unord/unord.multiset/unord.multiset.cnstr/move_noexcept.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.multiset/unord.multiset.cnstr/move_noexcept.pass.cpp
@@ -24,49 +24,44 @@
 #include "test_allocator.h"
 
 template <class T>
-struct some_comp
-{
-    typedef T value_type;
-    some_comp(const some_comp&);
-    bool operator()(const T&, const T&) const { return false; }
+struct some_comp {
+  typedef T value_type;
+  some_comp(const some_comp&);
+  bool operator()(const T&, const T&) const { return false; }
 };
 
 template <class T>
-struct some_hash
-{
-    typedef T value_type;
-    some_hash();
-    some_hash(const some_hash&);
-    std::size_t operator()(T const&) const;
+struct some_hash {
+  typedef T value_type;
+  some_hash();
+  some_hash(const some_hash&);
+  std::size_t operator()(T const&) const;
 };
 
-int main(int, char**)
-{
+int main(int, char**) {
 #if defined(_LIBCPP_VERSION)
-    {
-        typedef std::unordered_multiset<MoveOnly> C;
-        static_assert(std::is_nothrow_move_constructible<C>::value, "");
-    }
-    {
-        typedef std::unordered_multiset<MoveOnly, std::hash<MoveOnly>,
-                           std::equal_to<MoveOnly>, test_allocator<MoveOnly>> C;
-        static_assert(std::is_nothrow_move_constructible<C>::value, "");
-    }
-    {
-        typedef std::unordered_multiset<MoveOnly, std::hash<MoveOnly>,
-                          std::equal_to<MoveOnly>, other_allocator<MoveOnly>> C;
-        static_assert(std::is_nothrow_move_constructible<C>::value, "");
-    }
+  {
+    typedef std::unordered_multiset<MoveOnly> C;
+    static_assert(std::is_nothrow_move_constructible<C>::value, "");
+  }
+  {
+    typedef std::unordered_multiset<MoveOnly, std::hash<MoveOnly>, std::equal_to<MoveOnly>, test_allocator<MoveOnly>> C;
+    static_assert(std::is_nothrow_move_constructible<C>::value, "");
+  }
+  {
+    typedef std::unordered_multiset<MoveOnly, std::hash<MoveOnly>, std::equal_to<MoveOnly>, other_allocator<MoveOnly>>
+        C;
+    static_assert(std::is_nothrow_move_constructible<C>::value, "");
+  }
 #endif // _LIBCPP_VERSION
-    {
-        typedef std::unordered_multiset<MoveOnly, some_hash<MoveOnly>> C;
-        static_assert(!std::is_nothrow_move_constructible<C>::value, "");
-    }
-    {
-        typedef std::unordered_multiset<MoveOnly, std::hash<MoveOnly>,
-                                                         some_comp<MoveOnly>> C;
-        static_assert(!std::is_nothrow_move_constructible<C>::value, "");
-    }
+  {
+    typedef std::unordered_multiset<MoveOnly, some_hash<MoveOnly>> C;
+    static_assert(!std::is_nothrow_move_constructible<C>::value, "");
+  }
+  {
+    typedef std::unordered_multiset<MoveOnly, std::hash<MoveOnly>, some_comp<MoveOnly>> C;
+    static_assert(!std::is_nothrow_move_constructible<C>::value, "");
+  }
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/unord/unord.multiset/unord.multiset.cnstr/size.compile.fail.cpp b/libcxx/test/std/containers/unord/unord.multiset/unord.multiset.cnstr/size.compile.fail.cpp
index bd41c6d975435..09448fe6be7e4 100644
--- a/libcxx/test/std/containers/unord/unord.multiset/unord.multiset.cnstr/size.compile.fail.cpp
+++ b/libcxx/test/std/containers/unord/unord.multiset/unord.multiset.cnstr/size.compile.fail.cpp
@@ -24,43 +24,42 @@
 #include "test_allocator.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef std::unordered_multiset<NotConstructible,
-                                   test_hash<NotConstructible>,
-                                   test_equal_to<NotConstructible>,
-                                   test_allocator<NotConstructible>
-                                   > C;
-        C c = 7;
-        LIBCPP_ASSERT(c.bucket_count() == 7);
-        assert(c.hash_function() == test_hash<NotConstructible>());
-        assert(c.key_eq() == test_equal_to<NotConstructible>());
-        assert(c.get_allocator() == (test_allocator<NotConstructible>()));
-        assert(c.size() == 0);
-        assert(c.empty());
-        assert(std::distance(c.begin(), c.end()) == 0);
-        assert(c.load_factor() == 0);
-        assert(c.max_load_factor() == 1);
-    }
+int main(int, char**) {
+  {
+    typedef std::unordered_multiset<NotConstructible,
+                                    test_hash<NotConstructible>,
+                                    test_equal_to<NotConstructible>,
+                                    test_allocator<NotConstructible> >
+        C;
+    C c = 7;
+    LIBCPP_ASSERT(c.bucket_count() == 7);
+    assert(c.hash_function() == test_hash<NotConstructible>());
+    assert(c.key_eq() == test_equal_to<NotConstructible>());
+    assert(c.get_allocator() == (test_allocator<NotConstructible>()));
+    assert(c.size() == 0);
+    assert(c.empty());
+    assert(std::distance(c.begin(), c.end()) == 0);
+    assert(c.load_factor() == 0);
+    assert(c.max_load_factor() == 1);
+  }
 #if TEST_STD_VER >= 11
-    {
-        typedef std::unordered_multiset<NotConstructible,
-                                   test_hash<NotConstructible>,
-                                   test_equal_to<NotConstructible>,
-                                   min_allocator<NotConstructible>
-                                   > C;
-        C c = 7;
-        LIBCPP_ASSERT(c.bucket_count() == 7);
-        assert(c.hash_function() == test_hash<NotConstructible>());
-        assert(c.key_eq() == test_equal_to<NotConstructible>());
-        assert(c.get_allocator() == (min_allocator<NotConstructible>()));
-        assert(c.size() == 0);
-        assert(c.empty());
-        assert(std::distance(c.begin(), c.end()) == 0);
-        assert(c.load_factor() == 0);
-        assert(c.max_load_factor() == 1);
-    }
+  {
+    typedef std::unordered_multiset<NotConstructible,
+                                    test_hash<NotConstructible>,
+                                    test_equal_to<NotConstructible>,
+                                    min_allocator<NotConstructible> >
+        C;
+    C c = 7;
+    LIBCPP_ASSERT(c.bucket_count() == 7);
+    assert(c.hash_function() == test_hash<NotConstructible>());
+    assert(c.key_eq() == test_equal_to<NotConstructible>());
+    assert(c.get_allocator() == (min_allocator<NotConstructible>()));
+    assert(c.size() == 0);
+    assert(c.empty());
+    assert(std::distance(c.begin(), c.end()) == 0);
+    assert(c.load_factor() == 0);
+    assert(c.max_load_factor() == 1);
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/unord/unord.multiset/unord.multiset.cnstr/size.pass.cpp b/libcxx/test/std/containers/unord/unord.multiset/unord.multiset.cnstr/size.pass.cpp
index 8d40f478cfeae..870a0bce2ad73 100644
--- a/libcxx/test/std/containers/unord/unord.multiset/unord.multiset.cnstr/size.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.multiset/unord.multiset.cnstr/size.pass.cpp
@@ -24,43 +24,42 @@
 #include "test_allocator.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef std::unordered_multiset<NotConstructible,
-                                   test_hash<NotConstructible>,
-                                   test_equal_to<NotConstructible>,
-                                   test_allocator<NotConstructible>
-                                   > C;
-        C c(7);
-        LIBCPP_ASSERT(c.bucket_count() == 7);
-        assert(c.hash_function() == test_hash<NotConstructible>());
-        assert(c.key_eq() == test_equal_to<NotConstructible>());
-        assert(c.get_allocator() == (test_allocator<NotConstructible>()));
-        assert(c.size() == 0);
-        assert(c.empty());
-        assert(std::distance(c.begin(), c.end()) == 0);
-        assert(c.load_factor() == 0);
-        assert(c.max_load_factor() == 1);
-    }
+int main(int, char**) {
+  {
+    typedef std::unordered_multiset<NotConstructible,
+                                    test_hash<NotConstructible>,
+                                    test_equal_to<NotConstructible>,
+                                    test_allocator<NotConstructible> >
+        C;
+    C c(7);
+    LIBCPP_ASSERT(c.bucket_count() == 7);
+    assert(c.hash_function() == test_hash<NotConstructible>());
+    assert(c.key_eq() == test_equal_to<NotConstructible>());
+    assert(c.get_allocator() == (test_allocator<NotConstructible>()));
+    assert(c.size() == 0);
+    assert(c.empty());
+    assert(std::distance(c.begin(), c.end()) == 0);
+    assert(c.load_factor() == 0);
+    assert(c.max_load_factor() == 1);
+  }
 #if TEST_STD_VER >= 11
-    {
-        typedef std::unordered_multiset<NotConstructible,
-                                   test_hash<NotConstructible>,
-                                   test_equal_to<NotConstructible>,
-                                   min_allocator<NotConstructible>
-                                   > C;
-        C c(7);
-        LIBCPP_ASSERT(c.bucket_count() == 7);
-        assert(c.hash_function() == test_hash<NotConstructible>());
-        assert(c.key_eq() == test_equal_to<NotConstructible>());
-        assert(c.get_allocator() == (min_allocator<NotConstructible>()));
-        assert(c.size() == 0);
-        assert(c.empty());
-        assert(std::distance(c.begin(), c.end()) == 0);
-        assert(c.load_factor() == 0);
-        assert(c.max_load_factor() == 1);
-    }
+  {
+    typedef std::unordered_multiset<NotConstructible,
+                                    test_hash<NotConstructible>,
+                                    test_equal_to<NotConstructible>,
+                                    min_allocator<NotConstructible> >
+        C;
+    C c(7);
+    LIBCPP_ASSERT(c.bucket_count() == 7);
+    assert(c.hash_function() == test_hash<NotConstructible>());
+    assert(c.key_eq() == test_equal_to<NotConstructible>());
+    assert(c.get_allocator() == (min_allocator<NotConstructible>()));
+    assert(c.size() == 0);
+    assert(c.empty());
+    assert(std::distance(c.begin(), c.end()) == 0);
+    assert(c.load_factor() == 0);
+    assert(c.max_load_factor() == 1);
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/unord/unord.multiset/unord.multiset.cnstr/size_allocator.pass.cpp b/libcxx/test/std/containers/unord/unord.multiset/unord.multiset.cnstr/size_allocator.pass.cpp
index 2855487d32062..e536644470122 100644
--- a/libcxx/test/std/containers/unord/unord.multiset/unord.multiset.cnstr/size_allocator.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.multiset/unord.multiset.cnstr/size_allocator.pass.cpp
@@ -28,29 +28,26 @@
 #include "min_allocator.h"
 
 template <class Allocator>
-void test(const Allocator& alloc)
-{
-    typedef std::unordered_multiset<NotConstructible,
-                                    test_hash<NotConstructible>,
-                                    test_equal_to<NotConstructible>,
-                                    Allocator> C;
-    C c(7, alloc);
-    LIBCPP_ASSERT(c.bucket_count() == 7);
-    assert(c.hash_function() == test_hash<NotConstructible>());
-    assert(c.key_eq() == test_equal_to<NotConstructible>());
-    assert(c.get_allocator() == alloc);
-    assert(c.size() == 0);
-    assert(c.empty());
-    assert(std::distance(c.begin(), c.end()) == 0);
-    assert(c.load_factor() == 0);
-    assert(c.max_load_factor() == 1);
+void test(const Allocator& alloc) {
+  typedef std::
+      unordered_multiset<NotConstructible, test_hash<NotConstructible>, test_equal_to<NotConstructible>, Allocator>
+          C;
+  C c(7, alloc);
+  LIBCPP_ASSERT(c.bucket_count() == 7);
+  assert(c.hash_function() == test_hash<NotConstructible>());
+  assert(c.key_eq() == test_equal_to<NotConstructible>());
+  assert(c.get_allocator() == alloc);
+  assert(c.size() == 0);
+  assert(c.empty());
+  assert(std::distance(c.begin(), c.end()) == 0);
+  assert(c.load_factor() == 0);
+  assert(c.max_load_factor() == 1);
 }
 
-int main(int, char**)
-{
-    test(test_allocator<NotConstructible>(10));
-    test(min_allocator<NotConstructible>());
-    test(explicit_allocator<NotConstructible>());
+int main(int, char**) {
+  test(test_allocator<NotConstructible>(10));
+  test(min_allocator<NotConstructible>());
+  test(explicit_allocator<NotConstructible>());
 
-    return 0;
+  return 0;
 }
diff --git a/libcxx/test/std/containers/unord/unord.multiset/unord.multiset.cnstr/size_hash.pass.cpp b/libcxx/test/std/containers/unord/unord.multiset/unord.multiset.cnstr/size_hash.pass.cpp
index 58f84dfd02b61..def11f4a0cde7 100644
--- a/libcxx/test/std/containers/unord/unord.multiset/unord.multiset.cnstr/size_hash.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.multiset/unord.multiset.cnstr/size_hash.pass.cpp
@@ -24,47 +24,42 @@
 #include "test_allocator.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef std::unordered_multiset<NotConstructible,
-                                   test_hash<NotConstructible>,
-                                   test_equal_to<NotConstructible>,
-                                   test_allocator<NotConstructible>
-                                   > C;
-        C c(7,
-            test_hash<NotConstructible>(8)
-           );
-        LIBCPP_ASSERT(c.bucket_count() == 7);
-        assert(c.hash_function() == test_hash<NotConstructible>(8));
-        assert(c.key_eq() == test_equal_to<NotConstructible>());
-        assert(c.get_allocator() == (test_allocator<NotConstructible>()));
-        assert(c.size() == 0);
-        assert(c.empty());
-        assert(std::distance(c.begin(), c.end()) == 0);
-        assert(c.load_factor() == 0);
-        assert(c.max_load_factor() == 1);
-    }
+int main(int, char**) {
+  {
+    typedef std::unordered_multiset<NotConstructible,
+                                    test_hash<NotConstructible>,
+                                    test_equal_to<NotConstructible>,
+                                    test_allocator<NotConstructible> >
+        C;
+    C c(7, test_hash<NotConstructible>(8));
+    LIBCPP_ASSERT(c.bucket_count() == 7);
+    assert(c.hash_function() == test_hash<NotConstructible>(8));
+    assert(c.key_eq() == test_equal_to<NotConstructible>());
+    assert(c.get_allocator() == (test_allocator<NotConstructible>()));
+    assert(c.size() == 0);
+    assert(c.empty());
+    assert(std::distance(c.begin(), c.end()) == 0);
+    assert(c.load_factor() == 0);
+    assert(c.max_load_factor() == 1);
+  }
 #if TEST_STD_VER >= 11
-    {
-        typedef std::unordered_multiset<NotConstructible,
-                                   test_hash<NotConstructible>,
-                                   test_equal_to<NotConstructible>,
-                                   min_allocator<NotConstructible>
-                                   > C;
-        C c(7,
-            test_hash<NotConstructible>(8)
-           );
-        LIBCPP_ASSERT(c.bucket_count() == 7);
-        assert(c.hash_function() == test_hash<NotConstructible>(8));
-        assert(c.key_eq() == test_equal_to<NotConstructible>());
-        assert(c.get_allocator() == (min_allocator<NotConstructible>()));
-        assert(c.size() == 0);
-        assert(c.empty());
-        assert(std::distance(c.begin(), c.end()) == 0);
-        assert(c.load_factor() == 0);
-        assert(c.max_load_factor() == 1);
-    }
+  {
+    typedef std::unordered_multiset<NotConstructible,
+                                    test_hash<NotConstructible>,
+                                    test_equal_to<NotConstructible>,
+                                    min_allocator<NotConstructible> >
+        C;
+    C c(7, test_hash<NotConstructible>(8));
+    LIBCPP_ASSERT(c.bucket_count() == 7);
+    assert(c.hash_function() == test_hash<NotConstructible>(8));
+    assert(c.key_eq() == test_equal_to<NotConstructible>());
+    assert(c.get_allocator() == (min_allocator<NotConstructible>()));
+    assert(c.size() == 0);
+    assert(c.empty());
+    assert(std::distance(c.begin(), c.end()) == 0);
+    assert(c.load_factor() == 0);
+    assert(c.max_load_factor() == 1);
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/unord/unord.multiset/unord.multiset.cnstr/size_hash_allocator.pass.cpp b/libcxx/test/std/containers/unord/unord.multiset/unord.multiset.cnstr/size_hash_allocator.pass.cpp
index 2c20093579b15..6ea22dab1aac9 100644
--- a/libcxx/test/std/containers/unord/unord.multiset/unord.multiset.cnstr/size_hash_allocator.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.multiset/unord.multiset.cnstr/size_hash_allocator.pass.cpp
@@ -28,29 +28,26 @@
 #include "min_allocator.h"
 
 template <class Allocator>
-void test(const Allocator& alloc)
-{
-    typedef std::unordered_multiset<NotConstructible,
-                                    test_hash<NotConstructible>,
-                                    test_equal_to<NotConstructible>,
-                                    Allocator> C;
-    C c(7, test_hash<NotConstructible>(5), alloc);
-    LIBCPP_ASSERT(c.bucket_count() == 7);
-    assert(c.hash_function() == test_hash<NotConstructible>(5));
-    assert(c.key_eq() == test_equal_to<NotConstructible>());
-    assert(c.get_allocator() == alloc);
-    assert(c.size() == 0);
-    assert(c.empty());
-    assert(std::distance(c.begin(), c.end()) == 0);
-    assert(c.load_factor() == 0);
-    assert(c.max_load_factor() == 1);
+void test(const Allocator& alloc) {
+  typedef std::
+      unordered_multiset<NotConstructible, test_hash<NotConstructible>, test_equal_to<NotConstructible>, Allocator>
+          C;
+  C c(7, test_hash<NotConstructible>(5), alloc);
+  LIBCPP_ASSERT(c.bucket_count() == 7);
+  assert(c.hash_function() == test_hash<NotConstructible>(5));
+  assert(c.key_eq() == test_equal_to<NotConstructible>());
+  assert(c.get_allocator() == alloc);
+  assert(c.size() == 0);
+  assert(c.empty());
+  assert(std::distance(c.begin(), c.end()) == 0);
+  assert(c.load_factor() == 0);
+  assert(c.max_load_factor() == 1);
 }
 
-int main(int, char**)
-{
-    test(test_allocator<NotConstructible>(10));
-    test(min_allocator<NotConstructible>());
-    test(explicit_allocator<NotConstructible>());
+int main(int, char**) {
+  test(test_allocator<NotConstructible>(10));
+  test(min_allocator<NotConstructible>());
+  test(explicit_allocator<NotConstructible>());
 
-    return 0;
+  return 0;
 }
diff --git a/libcxx/test/std/containers/unord/unord.multiset/unord.multiset.cnstr/size_hash_equal.pass.cpp b/libcxx/test/std/containers/unord/unord.multiset/unord.multiset.cnstr/size_hash_equal.pass.cpp
index 3e6130aff2cee..1d3b6903e4d42 100644
--- a/libcxx/test/std/containers/unord/unord.multiset/unord.multiset.cnstr/size_hash_equal.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.multiset/unord.multiset.cnstr/size_hash_equal.pass.cpp
@@ -24,49 +24,42 @@
 #include "test_allocator.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef std::unordered_multiset<NotConstructible,
-                                   test_hash<NotConstructible>,
-                                   test_equal_to<NotConstructible>,
-                                   test_allocator<NotConstructible>
-                                   > C;
-        C c(7,
-            test_hash<NotConstructible>(8),
-            test_equal_to<NotConstructible>(9)
-           );
-        LIBCPP_ASSERT(c.bucket_count() == 7);
-        assert(c.hash_function() == test_hash<NotConstructible>(8));
-        assert(c.key_eq() == test_equal_to<NotConstructible>(9));
-        assert(c.get_allocator() == (test_allocator<NotConstructible>()));
-        assert(c.size() == 0);
-        assert(c.empty());
-        assert(std::distance(c.begin(), c.end()) == 0);
-        assert(c.load_factor() == 0);
-        assert(c.max_load_factor() == 1);
-    }
+int main(int, char**) {
+  {
+    typedef std::unordered_multiset<NotConstructible,
+                                    test_hash<NotConstructible>,
+                                    test_equal_to<NotConstructible>,
+                                    test_allocator<NotConstructible> >
+        C;
+    C c(7, test_hash<NotConstructible>(8), test_equal_to<NotConstructible>(9));
+    LIBCPP_ASSERT(c.bucket_count() == 7);
+    assert(c.hash_function() == test_hash<NotConstructible>(8));
+    assert(c.key_eq() == test_equal_to<NotConstructible>(9));
+    assert(c.get_allocator() == (test_allocator<NotConstructible>()));
+    assert(c.size() == 0);
+    assert(c.empty());
+    assert(std::distance(c.begin(), c.end()) == 0);
+    assert(c.load_factor() == 0);
+    assert(c.max_load_factor() == 1);
+  }
 #if TEST_STD_VER >= 11
-    {
-        typedef std::unordered_multiset<NotConstructible,
-                                   test_hash<NotConstructible>,
-                                   test_equal_to<NotConstructible>,
-                                   min_allocator<NotConstructible>
-                                   > C;
-        C c(7,
-            test_hash<NotConstructible>(8),
-            test_equal_to<NotConstructible>(9)
-           );
-        LIBCPP_ASSERT(c.bucket_count() == 7);
-        assert(c.hash_function() == test_hash<NotConstructible>(8));
-        assert(c.key_eq() == test_equal_to<NotConstructible>(9));
-        assert(c.get_allocator() == (min_allocator<NotConstructible>()));
-        assert(c.size() == 0);
-        assert(c.empty());
-        assert(std::distance(c.begin(), c.end()) == 0);
-        assert(c.load_factor() == 0);
-        assert(c.max_load_factor() == 1);
-    }
+  {
+    typedef std::unordered_multiset<NotConstructible,
+                                    test_hash<NotConstructible>,
+                                    test_equal_to<NotConstructible>,
+                                    min_allocator<NotConstructible> >
+        C;
+    C c(7, test_hash<NotConstructible>(8), test_equal_to<NotConstructible>(9));
+    LIBCPP_ASSERT(c.bucket_count() == 7);
+    assert(c.hash_function() == test_hash<NotConstructible>(8));
+    assert(c.key_eq() == test_equal_to<NotConstructible>(9));
+    assert(c.get_allocator() == (min_allocator<NotConstructible>()));
+    assert(c.size() == 0);
+    assert(c.empty());
+    assert(std::distance(c.begin(), c.end()) == 0);
+    assert(c.load_factor() == 0);
+    assert(c.max_load_factor() == 1);
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/unord/unord.multiset/unord.multiset.cnstr/size_hash_equal_allocator.pass.cpp b/libcxx/test/std/containers/unord/unord.multiset/unord.multiset.cnstr/size_hash_equal_allocator.pass.cpp
index c64fb8d33a0b3..ebe6df57f480e 100644
--- a/libcxx/test/std/containers/unord/unord.multiset/unord.multiset.cnstr/size_hash_equal_allocator.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.multiset/unord.multiset.cnstr/size_hash_equal_allocator.pass.cpp
@@ -24,51 +24,48 @@
 #include "test_allocator.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef std::unordered_multiset<NotConstructible,
-                                   test_hash<NotConstructible>,
-                                   test_equal_to<NotConstructible>,
-                                   test_allocator<NotConstructible>
-                                   > C;
-        C c(7,
-            test_hash<NotConstructible>(8),
-            test_equal_to<NotConstructible>(9),
-            test_allocator<std::pair<const NotConstructible, NotConstructible> >(10)
-           );
-        LIBCPP_ASSERT(c.bucket_count() == 7);
-        assert(c.hash_function() == test_hash<NotConstructible>(8));
-        assert(c.key_eq() == test_equal_to<NotConstructible>(9));
-        assert(c.get_allocator() == (test_allocator<NotConstructible>(10)));
-        assert(c.size() == 0);
-        assert(c.empty());
-        assert(std::distance(c.begin(), c.end()) == 0);
-        assert(c.load_factor() == 0);
-        assert(c.max_load_factor() == 1);
-    }
+int main(int, char**) {
+  {
+    typedef std::unordered_multiset<NotConstructible,
+                                    test_hash<NotConstructible>,
+                                    test_equal_to<NotConstructible>,
+                                    test_allocator<NotConstructible> >
+        C;
+    C c(7,
+        test_hash<NotConstructible>(8),
+        test_equal_to<NotConstructible>(9),
+        test_allocator<std::pair<const NotConstructible, NotConstructible> >(10));
+    LIBCPP_ASSERT(c.bucket_count() == 7);
+    assert(c.hash_function() == test_hash<NotConstructible>(8));
+    assert(c.key_eq() == test_equal_to<NotConstructible>(9));
+    assert(c.get_allocator() == (test_allocator<NotConstructible>(10)));
+    assert(c.size() == 0);
+    assert(c.empty());
+    assert(std::distance(c.begin(), c.end()) == 0);
+    assert(c.load_factor() == 0);
+    assert(c.max_load_factor() == 1);
+  }
 #if TEST_STD_VER >= 11
-    {
-        typedef std::unordered_multiset<NotConstructible,
-                                   test_hash<NotConstructible>,
-                                   test_equal_to<NotConstructible>,
-                                   min_allocator<NotConstructible>
-                                   > C;
-        C c(7,
-            test_hash<NotConstructible>(8),
-            test_equal_to<NotConstructible>(9),
-            min_allocator<std::pair<const NotConstructible, NotConstructible> >()
-           );
-        LIBCPP_ASSERT(c.bucket_count() == 7);
-        assert(c.hash_function() == test_hash<NotConstructible>(8));
-        assert(c.key_eq() == test_equal_to<NotConstructible>(9));
-        assert(c.get_allocator() == (min_allocator<NotConstructible>()));
-        assert(c.size() == 0);
-        assert(c.empty());
-        assert(std::distance(c.begin(), c.end()) == 0);
-        assert(c.load_factor() == 0);
-        assert(c.max_load_factor() == 1);
-    }
+  {
+    typedef std::unordered_multiset<NotConstructible,
+                                    test_hash<NotConstructible>,
+                                    test_equal_to<NotConstructible>,
+                                    min_allocator<NotConstructible> >
+        C;
+    C c(7,
+        test_hash<NotConstructible>(8),
+        test_equal_to<NotConstructible>(9),
+        min_allocator<std::pair<const NotConstructible, NotConstructible> >());
+    LIBCPP_ASSERT(c.bucket_count() == 7);
+    assert(c.hash_function() == test_hash<NotConstructible>(8));
+    assert(c.key_eq() == test_equal_to<NotConstructible>(9));
+    assert(c.get_allocator() == (min_allocator<NotConstructible>()));
+    assert(c.size() == 0);
+    assert(c.empty());
+    assert(std::distance(c.begin(), c.end()) == 0);
+    assert(c.load_factor() == 0);
+    assert(c.max_load_factor() == 1);
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/unord/unord.multiset/unord.multiset.swap/swap_noexcept.pass.cpp b/libcxx/test/std/containers/unord/unord.multiset/unord.multiset.swap/swap_noexcept.pass.cpp
index 057e5bb32ace7..c510d70f090c2 100644
--- a/libcxx/test/std/containers/unord/unord.multiset/unord.multiset.swap/swap_noexcept.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.multiset/unord.multiset.swap/swap_noexcept.pass.cpp
@@ -33,23 +33,21 @@
 #include "test_allocator.h"
 
 template <class T>
-struct some_comp
-{
-    typedef T value_type;
+struct some_comp {
+  typedef T value_type;
 
-    some_comp() {}
-    some_comp(const some_comp&) {}
-    bool operator()(const T&, const T&) const { return false; }
+  some_comp() {}
+  some_comp(const some_comp&) {}
+  bool operator()(const T&, const T&) const { return false; }
 };
 
 template <class T>
-struct some_comp2
-{
-    typedef T value_type;
+struct some_comp2 {
+  typedef T value_type;
 
-    some_comp2() {}
-    some_comp2(const some_comp2&) {}
-    bool operator()(const T&, const T&) const { return false; }
+  some_comp2() {}
+  some_comp2(const some_comp2&) {}
+  bool operator()(const T&, const T&) const { return false; }
 };
 
 #if TEST_STD_VER >= 14
@@ -58,21 +56,19 @@ void swap(some_comp2<T>&, some_comp2<T>&) noexcept {}
 #endif
 
 template <class T>
-struct some_hash
-{
-    typedef T value_type;
-    some_hash() {}
-    some_hash(const some_hash&);
-    std::size_t operator()(const T&) const { return 0; }
+struct some_hash {
+  typedef T value_type;
+  some_hash() {}
+  some_hash(const some_hash&);
+  std::size_t operator()(const T&) const { return 0; }
 };
 
 template <class T>
-struct some_hash2
-{
-    typedef T value_type;
-    some_hash2() {}
-    some_hash2(const some_hash2&);
-    std::size_t operator()(const T&) const { return 0; }
+struct some_hash2 {
+  typedef T value_type;
+  some_hash2() {}
+  some_hash2(const some_hash2&);
+  std::size_t operator()(const T&) const { return 0; }
 };
 
 #if TEST_STD_VER >= 14
@@ -81,110 +77,104 @@ void swap(some_hash2<T>&, some_hash2<T>&) noexcept {}
 #endif
 
 template <class T>
-struct some_alloc
-{
-    typedef T value_type;
+struct some_alloc {
+  typedef T value_type;
 
-    some_alloc() {}
-    some_alloc(const some_alloc&);
-    void deallocate(void*, unsigned) {}
+  some_alloc() {}
+  some_alloc(const some_alloc&);
+  void deallocate(void*, unsigned) {}
 
-    typedef std::true_type propagate_on_container_swap;
+  typedef std::true_type propagate_on_container_swap;
 };
 
 template <class T>
-struct some_alloc2
-{
-    typedef T value_type;
+struct some_alloc2 {
+  typedef T value_type;
 
-    some_alloc2() {}
-    some_alloc2(const some_alloc2&);
-    void deallocate(void*, unsigned) {}
+  some_alloc2() {}
+  some_alloc2(const some_alloc2&);
+  void deallocate(void*, unsigned) {}
 
-    typedef std::false_type propagate_on_container_swap;
-    typedef std::true_type is_always_equal;
+  typedef std::false_type propagate_on_container_swap;
+  typedef std::true_type is_always_equal;
 };
 
 template <class T>
-struct some_alloc3
-{
-    typedef T value_type;
+struct some_alloc3 {
+  typedef T value_type;
 
-    some_alloc3() {}
-    some_alloc3(const some_alloc3&);
-    void deallocate(void*, unsigned) {}
+  some_alloc3() {}
+  some_alloc3(const some_alloc3&);
+  void deallocate(void*, unsigned) {}
 
-    typedef std::false_type propagate_on_container_swap;
-    typedef std::false_type is_always_equal;
+  typedef std::false_type propagate_on_container_swap;
+  typedef std::false_type is_always_equal;
 };
 
-int main(int, char**)
-{
-    {
-        typedef std::unordered_multiset<MoveOnly> C;
-        static_assert(noexcept(swap(std::declval<C&>(), std::declval<C&>())), "");
-    }
+int main(int, char**) {
+  {
+    typedef std::unordered_multiset<MoveOnly> C;
+    static_assert(noexcept(swap(std::declval<C&>(), std::declval<C&>())), "");
+  }
 #if defined(_LIBCPP_VERSION)
-    {
-        typedef std::unordered_multiset<MoveOnly, std::hash<MoveOnly>,
-                           std::equal_to<MoveOnly>, test_allocator<MoveOnly>> C;
-        static_assert(noexcept(swap(std::declval<C&>(), std::declval<C&>())), "");
-    }
-    {
-        typedef std::unordered_multiset<MoveOnly, std::hash<MoveOnly>,
-                          std::equal_to<MoveOnly>, other_allocator<MoveOnly>> C;
-        static_assert(noexcept(swap(std::declval<C&>(), std::declval<C&>())), "");
-    }
+  {
+    typedef std::unordered_multiset<MoveOnly, std::hash<MoveOnly>, std::equal_to<MoveOnly>, test_allocator<MoveOnly>> C;
+    static_assert(noexcept(swap(std::declval<C&>(), std::declval<C&>())), "");
+  }
+  {
+    typedef std::unordered_multiset<MoveOnly, std::hash<MoveOnly>, std::equal_to<MoveOnly>, other_allocator<MoveOnly>>
+        C;
+    static_assert(noexcept(swap(std::declval<C&>(), std::declval<C&>())), "");
+  }
 #endif // _LIBCPP_VERSION
-    {
-        typedef std::unordered_multiset<MoveOnly, some_hash<MoveOnly>> C;
-        static_assert(!noexcept(swap(std::declval<C&>(), std::declval<C&>())), "");
-    }
-    {
-        typedef std::unordered_multiset<MoveOnly, std::hash<MoveOnly>,
-                                                         some_comp<MoveOnly>> C;
-        static_assert(!noexcept(swap(std::declval<C&>(), std::declval<C&>())), "");
-    }
+  {
+    typedef std::unordered_multiset<MoveOnly, some_hash<MoveOnly>> C;
+    static_assert(!noexcept(swap(std::declval<C&>(), std::declval<C&>())), "");
+  }
+  {
+    typedef std::unordered_multiset<MoveOnly, std::hash<MoveOnly>, some_comp<MoveOnly>> C;
+    static_assert(!noexcept(swap(std::declval<C&>(), std::declval<C&>())), "");
+  }
 
 #if TEST_STD_VER >= 14
-    { // POCS allocator, throwable swap for hash, throwable swap for comp
-    typedef std::unordered_multiset<MoveOnly, some_hash<MoveOnly>, some_comp <MoveOnly>, some_alloc <MoveOnly>> C;
+  { // POCS allocator, throwable swap for hash, throwable swap for comp
+    typedef std::unordered_multiset<MoveOnly, some_hash<MoveOnly>, some_comp<MoveOnly>, some_alloc<MoveOnly>> C;
     static_assert(!noexcept(swap(std::declval<C&>(), std::declval<C&>())), "");
-    }
-    { // always equal allocator, throwable swap for hash, throwable swap for comp
-    typedef std::unordered_multiset<MoveOnly, some_hash<MoveOnly>, some_comp <MoveOnly>, some_alloc2<MoveOnly>> C;
+  }
+  { // always equal allocator, throwable swap for hash, throwable swap for comp
+    typedef std::unordered_multiset<MoveOnly, some_hash<MoveOnly>, some_comp<MoveOnly>, some_alloc2<MoveOnly>> C;
     static_assert(!noexcept(swap(std::declval<C&>(), std::declval<C&>())), "");
-    }
-    { // POCS allocator, throwable swap for hash, nothrow swap for comp
-    typedef std::unordered_multiset<MoveOnly, some_hash<MoveOnly>, some_comp2<MoveOnly>, some_alloc <MoveOnly>> C;
+  }
+  { // POCS allocator, throwable swap for hash, nothrow swap for comp
+    typedef std::unordered_multiset<MoveOnly, some_hash<MoveOnly>, some_comp2<MoveOnly>, some_alloc<MoveOnly>> C;
     static_assert(!noexcept(swap(std::declval<C&>(), std::declval<C&>())), "");
-    }
-    { // always equal allocator, throwable swap for hash, nothrow swap for comp
+  }
+  { // always equal allocator, throwable swap for hash, nothrow swap for comp
     typedef std::unordered_multiset<MoveOnly, some_hash<MoveOnly>, some_comp2<MoveOnly>, some_alloc2<MoveOnly>> C;
     static_assert(!noexcept(swap(std::declval<C&>(), std::declval<C&>())), "");
-    }
-    { // POCS allocator, nothrow swap for hash, throwable swap for comp
-    typedef std::unordered_multiset<MoveOnly, some_hash2<MoveOnly>, some_comp <MoveOnly>, some_alloc <MoveOnly>> C;
+  }
+  { // POCS allocator, nothrow swap for hash, throwable swap for comp
+    typedef std::unordered_multiset<MoveOnly, some_hash2<MoveOnly>, some_comp<MoveOnly>, some_alloc<MoveOnly>> C;
     static_assert(!noexcept(swap(std::declval<C&>(), std::declval<C&>())), "");
-    }
-    { // always equal allocator, nothrow swap for hash, throwable swap for comp
-    typedef std::unordered_multiset<MoveOnly, some_hash2<MoveOnly>, some_comp <MoveOnly>, some_alloc2<MoveOnly>> C;
+  }
+  { // always equal allocator, nothrow swap for hash, throwable swap for comp
+    typedef std::unordered_multiset<MoveOnly, some_hash2<MoveOnly>, some_comp<MoveOnly>, some_alloc2<MoveOnly>> C;
     static_assert(!noexcept(swap(std::declval<C&>(), std::declval<C&>())), "");
-    }
-    { // POCS allocator, nothrow swap for hash, nothrow swap for comp
-    typedef std::unordered_multiset<MoveOnly, some_hash2<MoveOnly>, some_comp2<MoveOnly>, some_alloc <MoveOnly>> C;
-    static_assert( noexcept(swap(std::declval<C&>(), std::declval<C&>())), "");
-    }
-    { // always equal allocator, nothrow swap for hash, nothrow swap for comp
+  }
+  { // POCS allocator, nothrow swap for hash, nothrow swap for comp
+    typedef std::unordered_multiset<MoveOnly, some_hash2<MoveOnly>, some_comp2<MoveOnly>, some_alloc<MoveOnly>> C;
+    static_assert(noexcept(swap(std::declval<C&>(), std::declval<C&>())), "");
+  }
+  { // always equal allocator, nothrow swap for hash, nothrow swap for comp
     typedef std::unordered_multiset<MoveOnly, some_hash2<MoveOnly>, some_comp2<MoveOnly>, some_alloc2<MoveOnly>> C;
-    static_assert( noexcept(swap(std::declval<C&>(), std::declval<C&>())), "");
-    }
-#if defined(_LIBCPP_VERSION)
-    { // NOT always equal allocator, nothrow swap for hash, nothrow swap for comp
+    static_assert(noexcept(swap(std::declval<C&>(), std::declval<C&>())), "");
+  }
+#  if defined(_LIBCPP_VERSION)
+  { // NOT always equal allocator, nothrow swap for hash, nothrow swap for comp
     typedef std::unordered_multiset<MoveOnly, some_hash2<MoveOnly>, some_comp2<MoveOnly>, some_alloc3<MoveOnly>> C;
-    static_assert( noexcept(swap(std::declval<C&>(), std::declval<C&>())), "");
-    }
-#endif // _LIBCPP_VERSION
+    static_assert(noexcept(swap(std::declval<C&>(), std::declval<C&>())), "");
+  }
+#  endif // _LIBCPP_VERSION
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/unord/unord.multiset/unord.multiset.swap/swap_non_member.pass.cpp b/libcxx/test/std/containers/unord/unord.multiset/unord.multiset.swap/swap_non_member.pass.cpp
index 3b8aad272256c..0665ad7dbfd14 100644
--- a/libcxx/test/std/containers/unord/unord.multiset/unord.multiset.swap/swap_non_member.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.multiset/unord.multiset.swap/swap_non_member.pass.cpp
@@ -24,555 +24,446 @@
 #include "test_allocator.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef test_hash<int> Hash;
-        typedef test_equal_to<int> Compare;
-        typedef test_allocator<int> Alloc;
-        typedef std::unordered_multiset<int, Hash, Compare, Alloc> C;
-        C c1(0, Hash(1), Compare(1), Alloc(1, 1));
-        C c2(0, Hash(2), Compare(2), Alloc(1, 2));
-        c2.max_load_factor(2);
-        swap(c1, c2);
-
-        LIBCPP_ASSERT(c1.bucket_count() == 0);
-        assert(c1.size() == 0);
-        assert(c1.hash_function() == Hash(2));
-        assert(c1.key_eq() == Compare(2));
-        assert(c1.get_allocator().get_id() == 1);
-        assert(static_cast<std::size_t>(std::distance(c1.begin(), c1.end())) == c1.size());
-        assert(static_cast<std::size_t>(std::distance(c1.cbegin(), c1.cend())) == c1.size());
-        assert(c1.max_load_factor() == 2);
-
-        LIBCPP_ASSERT(c2.bucket_count() == 0);
-        assert(c2.size() == 0);
-        assert(c2.hash_function() == Hash(1));
-        assert(c2.key_eq() == Compare(1));
-        assert(c2.get_allocator().get_id() == 2);
-        assert(static_cast<std::size_t>(std::distance(c2.begin(), c2.end())) == c2.size());
-        assert(static_cast<std::size_t>(std::distance(c2.cbegin(), c2.cend())) == c2.size());
-        assert(c2.max_load_factor() == 1);
-    }
-    {
-        typedef test_hash<int> Hash;
-        typedef test_equal_to<int> Compare;
-        typedef test_allocator<int> Alloc;
-        typedef std::unordered_multiset<int, Hash, Compare, Alloc> C;
-        typedef int P;
-        P a2[] =
-        {
-            P(10),
-            P(20),
-            P(30),
-            P(40),
-            P(50),
-            P(60),
-            P(70),
-            P(80)
-        };
-        C c1(0, Hash(1), Compare(1), Alloc(1, 1));
-        C c2(std::begin(a2), std::end(a2), 0, Hash(2), Compare(2), Alloc(1, 2));
-        c2.max_load_factor(2);
-        C::iterator it2 = c2.begin();
-        swap(c1, c2);
-
-        assert(c1.bucket_count() >= 8);
-        assert(c1.size() == 8);
-        assert(*c1.find(10) == 10);
-        assert(*c1.find(20) == 20);
-        assert(*c1.find(30) == 30);
-        assert(*c1.find(40) == 40);
-        assert(*c1.find(50) == 50);
-        assert(*c1.find(60) == 60);
-        assert(*c1.find(70) == 70);
-        assert(*c1.find(80) == 80);
-        assert(c1.hash_function() == Hash(2));
-        assert(c1.key_eq() == Compare(2));
-        assert(c1.get_allocator().get_id() == 1);
-        assert(static_cast<std::size_t>(std::distance(c1.begin(), c1.end())) == c1.size());
-        assert(static_cast<std::size_t>(std::distance(c1.cbegin(), c1.cend())) == c1.size());
-        assert(c1.max_load_factor() == 2);
-        assert(it2 == c1.begin()); // Iterators are not invalidated
-
-        LIBCPP_ASSERT(c2.bucket_count() == 0);
-        assert(c2.size() == 0);
-        assert(c2.hash_function() == Hash(1));
-        assert(c2.key_eq() == Compare(1));
-        assert(c2.get_allocator().get_id() == 2);
-        assert(static_cast<std::size_t>(std::distance(c2.begin(), c2.end())) == c2.size());
-        assert(static_cast<std::size_t>(std::distance(c2.cbegin(), c2.cend())) == c2.size());
-        assert(c2.max_load_factor() == 1);
-    }
-    {
-        typedef test_hash<int> Hash;
-        typedef test_equal_to<int> Compare;
-        typedef test_allocator<int> Alloc;
-        typedef std::unordered_multiset<int, Hash, Compare, Alloc> C;
-        typedef int P;
-        P a1[] =
-        {
-            P(1),
-            P(2),
-            P(3),
-            P(4),
-            P(1),
-            P(2)
-        };
-        C c1(std::begin(a1), std::end(a1), 0, Hash(1), Compare(1), Alloc(1, 1));
-        C c2(0, Hash(2), Compare(2), Alloc(1, 2));
-        c2.max_load_factor(2);
-        C::iterator it1 = c1.begin();
-        swap(c1, c2);
-
-        LIBCPP_ASSERT(c1.bucket_count() == 0);
-        assert(c1.size() == 0);
-        assert(c1.hash_function() == Hash(2));
-        assert(c1.key_eq() == Compare(2));
-        assert(c1.get_allocator().get_id() == 1);
-        assert(static_cast<std::size_t>(std::distance(c1.begin(), c1.end())) == c1.size());
-        assert(static_cast<std::size_t>(std::distance(c1.cbegin(), c1.cend())) == c1.size());
-        assert(c1.max_load_factor() == 2);
-
-        assert(c2.bucket_count() >= 6);
-        assert(c2.size() == 6);
-        assert(c2.count(1) == 2);
-        assert(c2.count(2) == 2);
-        assert(c2.count(3) == 1);
-        assert(c2.count(4) == 1);
-        assert(c2.hash_function() == Hash(1));
-        assert(c2.key_eq() == Compare(1));
-        assert(c2.get_allocator().get_id() == 2);
-        assert(static_cast<std::size_t>(std::distance(c2.begin(), c2.end())) == c2.size());
-        assert(static_cast<std::size_t>(std::distance(c2.cbegin(), c2.cend())) == c2.size());
-        assert(c2.max_load_factor() == 1);
-        assert(it1 == c2.begin()); // Iterators are not invalidated
-    }
-    {
-        typedef test_hash<int> Hash;
-        typedef test_equal_to<int> Compare;
-        typedef test_allocator<int> Alloc;
-        typedef std::unordered_multiset<int, Hash, Compare, Alloc> C;
-        typedef int P;
-        P a1[] =
-        {
-            P(1),
-            P(2),
-            P(3),
-            P(4),
-            P(1),
-            P(2)
-        };
-        P a2[] =
-        {
-            P(10),
-            P(20),
-            P(30),
-            P(40),
-            P(50),
-            P(60),
-            P(70),
-            P(80)
-        };
-        C c1(std::begin(a1), std::end(a1), 0, Hash(1), Compare(1), Alloc(1, 1));
-        C c2(std::begin(a2), std::end(a2), 0, Hash(2), Compare(2), Alloc(1, 2));
-        c2.max_load_factor(2);
-        C::iterator it1 = c1.begin();
-        C::iterator it2 = c2.begin();
-        swap(c1, c2);
-
-        assert(c1.bucket_count() >= 8);
-        assert(c1.size() == 8);
-        assert(*c1.find(10) == 10);
-        assert(*c1.find(20) == 20);
-        assert(*c1.find(30) == 30);
-        assert(*c1.find(40) == 40);
-        assert(*c1.find(50) == 50);
-        assert(*c1.find(60) == 60);
-        assert(*c1.find(70) == 70);
-        assert(*c1.find(80) == 80);
-        assert(c1.hash_function() == Hash(2));
-        assert(c1.key_eq() == Compare(2));
-        assert(c1.get_allocator().get_id() == 1);
-        assert(static_cast<std::size_t>(std::distance(c1.begin(), c1.end())) == c1.size());
-        assert(static_cast<std::size_t>(std::distance(c1.cbegin(), c1.cend())) == c1.size());
-        assert(c1.max_load_factor() == 2);
-        assert(it2 == c1.begin()); // Iterators are not invalidated
-
-        assert(c2.bucket_count() >= 6);
-        assert(c2.size() == 6);
-        assert(c2.count(1) == 2);
-        assert(c2.count(2) == 2);
-        assert(c2.count(3) == 1);
-        assert(c2.count(4) == 1);
-        assert(c2.hash_function() == Hash(1));
-        assert(c2.key_eq() == Compare(1));
-        assert(c2.get_allocator().get_id() == 2);
-        assert(static_cast<std::size_t>(std::distance(c2.begin(), c2.end())) == c2.size());
-        assert(static_cast<std::size_t>(std::distance(c2.cbegin(), c2.cend())) == c2.size());
-        assert(c2.max_load_factor() == 1);
-        assert(it1 == c2.begin()); // Iterators are not invalidated
-    }
-
-    {
-        typedef test_hash<int> Hash;
-        typedef test_equal_to<int> Compare;
-        typedef other_allocator<int> Alloc;
-        typedef std::unordered_multiset<int, Hash, Compare, Alloc> C;
-        C c1(0, Hash(1), Compare(1), Alloc(1));
-        C c2(0, Hash(2), Compare(2), Alloc(2));
-        c2.max_load_factor(2);
-        swap(c1, c2);
-
-        LIBCPP_ASSERT(c1.bucket_count() == 0);
-        assert(c1.size() == 0);
-        assert(c1.hash_function() == Hash(2));
-        assert(c1.key_eq() == Compare(2));
-        assert(c1.get_allocator() == Alloc(2));
-        assert(static_cast<std::size_t>(std::distance(c1.begin(), c1.end())) == c1.size());
-        assert(static_cast<std::size_t>(std::distance(c1.cbegin(), c1.cend())) == c1.size());
-        assert(c1.max_load_factor() == 2);
-
-        LIBCPP_ASSERT(c2.bucket_count() == 0);
-        assert(c2.size() == 0);
-        assert(c2.hash_function() == Hash(1));
-        assert(c2.key_eq() == Compare(1));
-        assert(c2.get_allocator() == Alloc(1));
-        assert(static_cast<std::size_t>(std::distance(c2.begin(), c2.end())) == c2.size());
-        assert(static_cast<std::size_t>(std::distance(c2.cbegin(), c2.cend())) == c2.size());
-        assert(c2.max_load_factor() == 1);
-    }
-    {
-        typedef test_hash<int> Hash;
-        typedef test_equal_to<int> Compare;
-        typedef other_allocator<int> Alloc;
-        typedef std::unordered_multiset<int, Hash, Compare, Alloc> C;
-        typedef int P;
-        P a2[] =
-        {
-            P(10),
-            P(20),
-            P(30),
-            P(40),
-            P(50),
-            P(60),
-            P(70),
-            P(80)
-        };
-        C c1(0, Hash(1), Compare(1), Alloc(1));
-        C c2(std::begin(a2), std::end(a2), 0, Hash(2), Compare(2), Alloc(2));
-        c2.max_load_factor(2);
-        swap(c1, c2);
-
-        assert(c1.bucket_count() >= 8);
-        assert(c1.size() == 8);
-        assert(*c1.find(10) == 10);
-        assert(*c1.find(20) == 20);
-        assert(*c1.find(30) == 30);
-        assert(*c1.find(40) == 40);
-        assert(*c1.find(50) == 50);
-        assert(*c1.find(60) == 60);
-        assert(*c1.find(70) == 70);
-        assert(*c1.find(80) == 80);
-        assert(c1.hash_function() == Hash(2));
-        assert(c1.key_eq() == Compare(2));
-        assert(c1.get_allocator() == Alloc(2));
-        assert(static_cast<std::size_t>(std::distance(c1.begin(), c1.end())) == c1.size());
-        assert(static_cast<std::size_t>(std::distance(c1.cbegin(), c1.cend())) == c1.size());
-        assert(c1.max_load_factor() == 2);
-
-        LIBCPP_ASSERT(c2.bucket_count() == 0);
-        assert(c2.size() == 0);
-        assert(c2.hash_function() == Hash(1));
-        assert(c2.key_eq() == Compare(1));
-        assert(c2.get_allocator() == Alloc(1));
-        assert(static_cast<std::size_t>(std::distance(c2.begin(), c2.end())) == c2.size());
-        assert(static_cast<std::size_t>(std::distance(c2.cbegin(), c2.cend())) == c2.size());
-        assert(c2.max_load_factor() == 1);
-    }
-    {
-        typedef test_hash<int> Hash;
-        typedef test_equal_to<int> Compare;
-        typedef other_allocator<int> Alloc;
-        typedef std::unordered_multiset<int, Hash, Compare, Alloc> C;
-        typedef int P;
-        P a1[] =
-        {
-            P(1),
-            P(2),
-            P(3),
-            P(4),
-            P(1),
-            P(2)
-        };
-        C c1(std::begin(a1), std::end(a1), 0, Hash(1), Compare(1), Alloc(1));
-        C c2(0, Hash(2), Compare(2), Alloc(2));
-        c2.max_load_factor(2);
-        swap(c1, c2);
-
-        LIBCPP_ASSERT(c1.bucket_count() == 0);
-        assert(c1.size() == 0);
-        assert(c1.hash_function() == Hash(2));
-        assert(c1.key_eq() == Compare(2));
-        assert(c1.get_allocator() == Alloc(2));
-        assert(static_cast<std::size_t>(std::distance(c1.begin(), c1.end())) == c1.size());
-        assert(static_cast<std::size_t>(std::distance(c1.cbegin(), c1.cend())) == c1.size());
-        assert(c1.max_load_factor() == 2);
-
-        assert(c2.bucket_count() >= 6);
-        assert(c2.size() == 6);
-        assert(c2.count(1) == 2);
-        assert(c2.count(2) == 2);
-        assert(c2.count(3) == 1);
-        assert(c2.count(4) == 1);
-        assert(c2.hash_function() == Hash(1));
-        assert(c2.key_eq() == Compare(1));
-        assert(c2.get_allocator() == Alloc(1));
-        assert(static_cast<std::size_t>(std::distance(c2.begin(), c2.end())) == c2.size());
-        assert(static_cast<std::size_t>(std::distance(c2.cbegin(), c2.cend())) == c2.size());
-        assert(c2.max_load_factor() == 1);
-    }
-    {
-        typedef test_hash<int> Hash;
-        typedef test_equal_to<int> Compare;
-        typedef other_allocator<int> Alloc;
-        typedef std::unordered_multiset<int, Hash, Compare, Alloc> C;
-        typedef int P;
-        P a1[] =
-        {
-            P(1),
-            P(2),
-            P(3),
-            P(4),
-            P(1),
-            P(2)
-        };
-        P a2[] =
-        {
-            P(10),
-            P(20),
-            P(30),
-            P(40),
-            P(50),
-            P(60),
-            P(70),
-            P(80)
-        };
-        C c1(std::begin(a1), std::end(a1), 0, Hash(1), Compare(1), Alloc(1));
-        C c2(std::begin(a2), std::end(a2), 0, Hash(2), Compare(2), Alloc(2));
-        c2.max_load_factor(2);
-        swap(c1, c2);
-
-        assert(c1.bucket_count() >= 8);
-        assert(c1.size() == 8);
-        assert(*c1.find(10) == 10);
-        assert(*c1.find(20) == 20);
-        assert(*c1.find(30) == 30);
-        assert(*c1.find(40) == 40);
-        assert(*c1.find(50) == 50);
-        assert(*c1.find(60) == 60);
-        assert(*c1.find(70) == 70);
-        assert(*c1.find(80) == 80);
-        assert(c1.hash_function() == Hash(2));
-        assert(c1.key_eq() == Compare(2));
-        assert(c1.get_allocator() == Alloc(2));
-        assert(static_cast<std::size_t>(std::distance(c1.begin(), c1.end())) == c1.size());
-        assert(static_cast<std::size_t>(std::distance(c1.cbegin(), c1.cend())) == c1.size());
-        assert(c1.max_load_factor() == 2);
-
-        assert(c2.bucket_count() >= 6);
-        assert(c2.size() == 6);
-        assert(c2.count(1) == 2);
-        assert(c2.count(2) == 2);
-        assert(c2.count(3) == 1);
-        assert(c2.count(4) == 1);
-        assert(c2.hash_function() == Hash(1));
-        assert(c2.key_eq() == Compare(1));
-        assert(c2.get_allocator() == Alloc(1));
-        assert(static_cast<std::size_t>(std::distance(c2.begin(), c2.end())) == c2.size());
-        assert(static_cast<std::size_t>(std::distance(c2.cbegin(), c2.cend())) == c2.size());
-        assert(c2.max_load_factor() == 1);
-    }
+int main(int, char**) {
+  {
+    typedef test_hash<int> Hash;
+    typedef test_equal_to<int> Compare;
+    typedef test_allocator<int> Alloc;
+    typedef std::unordered_multiset<int, Hash, Compare, Alloc> C;
+    C c1(0, Hash(1), Compare(1), Alloc(1, 1));
+    C c2(0, Hash(2), Compare(2), Alloc(1, 2));
+    c2.max_load_factor(2);
+    swap(c1, c2);
+
+    LIBCPP_ASSERT(c1.bucket_count() == 0);
+    assert(c1.size() == 0);
+    assert(c1.hash_function() == Hash(2));
+    assert(c1.key_eq() == Compare(2));
+    assert(c1.get_allocator().get_id() == 1);
+    assert(static_cast<std::size_t>(std::distance(c1.begin(), c1.end())) == c1.size());
+    assert(static_cast<std::size_t>(std::distance(c1.cbegin(), c1.cend())) == c1.size());
+    assert(c1.max_load_factor() == 2);
+
+    LIBCPP_ASSERT(c2.bucket_count() == 0);
+    assert(c2.size() == 0);
+    assert(c2.hash_function() == Hash(1));
+    assert(c2.key_eq() == Compare(1));
+    assert(c2.get_allocator().get_id() == 2);
+    assert(static_cast<std::size_t>(std::distance(c2.begin(), c2.end())) == c2.size());
+    assert(static_cast<std::size_t>(std::distance(c2.cbegin(), c2.cend())) == c2.size());
+    assert(c2.max_load_factor() == 1);
+  }
+  {
+    typedef test_hash<int> Hash;
+    typedef test_equal_to<int> Compare;
+    typedef test_allocator<int> Alloc;
+    typedef std::unordered_multiset<int, Hash, Compare, Alloc> C;
+    typedef int P;
+    P a2[] = {P(10), P(20), P(30), P(40), P(50), P(60), P(70), P(80)};
+    C c1(0, Hash(1), Compare(1), Alloc(1, 1));
+    C c2(std::begin(a2), std::end(a2), 0, Hash(2), Compare(2), Alloc(1, 2));
+    c2.max_load_factor(2);
+    C::iterator it2 = c2.begin();
+    swap(c1, c2);
+
+    assert(c1.bucket_count() >= 8);
+    assert(c1.size() == 8);
+    assert(*c1.find(10) == 10);
+    assert(*c1.find(20) == 20);
+    assert(*c1.find(30) == 30);
+    assert(*c1.find(40) == 40);
+    assert(*c1.find(50) == 50);
+    assert(*c1.find(60) == 60);
+    assert(*c1.find(70) == 70);
+    assert(*c1.find(80) == 80);
+    assert(c1.hash_function() == Hash(2));
+    assert(c1.key_eq() == Compare(2));
+    assert(c1.get_allocator().get_id() == 1);
+    assert(static_cast<std::size_t>(std::distance(c1.begin(), c1.end())) == c1.size());
+    assert(static_cast<std::size_t>(std::distance(c1.cbegin(), c1.cend())) == c1.size());
+    assert(c1.max_load_factor() == 2);
+    assert(it2 == c1.begin()); // Iterators are not invalidated
+
+    LIBCPP_ASSERT(c2.bucket_count() == 0);
+    assert(c2.size() == 0);
+    assert(c2.hash_function() == Hash(1));
+    assert(c2.key_eq() == Compare(1));
+    assert(c2.get_allocator().get_id() == 2);
+    assert(static_cast<std::size_t>(std::distance(c2.begin(), c2.end())) == c2.size());
+    assert(static_cast<std::size_t>(std::distance(c2.cbegin(), c2.cend())) == c2.size());
+    assert(c2.max_load_factor() == 1);
+  }
+  {
+    typedef test_hash<int> Hash;
+    typedef test_equal_to<int> Compare;
+    typedef test_allocator<int> Alloc;
+    typedef std::unordered_multiset<int, Hash, Compare, Alloc> C;
+    typedef int P;
+    P a1[] = {P(1), P(2), P(3), P(4), P(1), P(2)};
+    C c1(std::begin(a1), std::end(a1), 0, Hash(1), Compare(1), Alloc(1, 1));
+    C c2(0, Hash(2), Compare(2), Alloc(1, 2));
+    c2.max_load_factor(2);
+    C::iterator it1 = c1.begin();
+    swap(c1, c2);
+
+    LIBCPP_ASSERT(c1.bucket_count() == 0);
+    assert(c1.size() == 0);
+    assert(c1.hash_function() == Hash(2));
+    assert(c1.key_eq() == Compare(2));
+    assert(c1.get_allocator().get_id() == 1);
+    assert(static_cast<std::size_t>(std::distance(c1.begin(), c1.end())) == c1.size());
+    assert(static_cast<std::size_t>(std::distance(c1.cbegin(), c1.cend())) == c1.size());
+    assert(c1.max_load_factor() == 2);
+
+    assert(c2.bucket_count() >= 6);
+    assert(c2.size() == 6);
+    assert(c2.count(1) == 2);
+    assert(c2.count(2) == 2);
+    assert(c2.count(3) == 1);
+    assert(c2.count(4) == 1);
+    assert(c2.hash_function() == Hash(1));
+    assert(c2.key_eq() == Compare(1));
+    assert(c2.get_allocator().get_id() == 2);
+    assert(static_cast<std::size_t>(std::distance(c2.begin(), c2.end())) == c2.size());
+    assert(static_cast<std::size_t>(std::distance(c2.cbegin(), c2.cend())) == c2.size());
+    assert(c2.max_load_factor() == 1);
+    assert(it1 == c2.begin()); // Iterators are not invalidated
+  }
+  {
+    typedef test_hash<int> Hash;
+    typedef test_equal_to<int> Compare;
+    typedef test_allocator<int> Alloc;
+    typedef std::unordered_multiset<int, Hash, Compare, Alloc> C;
+    typedef int P;
+    P a1[] = {P(1), P(2), P(3), P(4), P(1), P(2)};
+    P a2[] = {P(10), P(20), P(30), P(40), P(50), P(60), P(70), P(80)};
+    C c1(std::begin(a1), std::end(a1), 0, Hash(1), Compare(1), Alloc(1, 1));
+    C c2(std::begin(a2), std::end(a2), 0, Hash(2), Compare(2), Alloc(1, 2));
+    c2.max_load_factor(2);
+    C::iterator it1 = c1.begin();
+    C::iterator it2 = c2.begin();
+    swap(c1, c2);
+
+    assert(c1.bucket_count() >= 8);
+    assert(c1.size() == 8);
+    assert(*c1.find(10) == 10);
+    assert(*c1.find(20) == 20);
+    assert(*c1.find(30) == 30);
+    assert(*c1.find(40) == 40);
+    assert(*c1.find(50) == 50);
+    assert(*c1.find(60) == 60);
+    assert(*c1.find(70) == 70);
+    assert(*c1.find(80) == 80);
+    assert(c1.hash_function() == Hash(2));
+    assert(c1.key_eq() == Compare(2));
+    assert(c1.get_allocator().get_id() == 1);
+    assert(static_cast<std::size_t>(std::distance(c1.begin(), c1.end())) == c1.size());
+    assert(static_cast<std::size_t>(std::distance(c1.cbegin(), c1.cend())) == c1.size());
+    assert(c1.max_load_factor() == 2);
+    assert(it2 == c1.begin()); // Iterators are not invalidated
+
+    assert(c2.bucket_count() >= 6);
+    assert(c2.size() == 6);
+    assert(c2.count(1) == 2);
+    assert(c2.count(2) == 2);
+    assert(c2.count(3) == 1);
+    assert(c2.count(4) == 1);
+    assert(c2.hash_function() == Hash(1));
+    assert(c2.key_eq() == Compare(1));
+    assert(c2.get_allocator().get_id() == 2);
+    assert(static_cast<std::size_t>(std::distance(c2.begin(), c2.end())) == c2.size());
+    assert(static_cast<std::size_t>(std::distance(c2.cbegin(), c2.cend())) == c2.size());
+    assert(c2.max_load_factor() == 1);
+    assert(it1 == c2.begin()); // Iterators are not invalidated
+  }
+
+  {
+    typedef test_hash<int> Hash;
+    typedef test_equal_to<int> Compare;
+    typedef other_allocator<int> Alloc;
+    typedef std::unordered_multiset<int, Hash, Compare, Alloc> C;
+    C c1(0, Hash(1), Compare(1), Alloc(1));
+    C c2(0, Hash(2), Compare(2), Alloc(2));
+    c2.max_load_factor(2);
+    swap(c1, c2);
+
+    LIBCPP_ASSERT(c1.bucket_count() == 0);
+    assert(c1.size() == 0);
+    assert(c1.hash_function() == Hash(2));
+    assert(c1.key_eq() == Compare(2));
+    assert(c1.get_allocator() == Alloc(2));
+    assert(static_cast<std::size_t>(std::distance(c1.begin(), c1.end())) == c1.size());
+    assert(static_cast<std::size_t>(std::distance(c1.cbegin(), c1.cend())) == c1.size());
+    assert(c1.max_load_factor() == 2);
+
+    LIBCPP_ASSERT(c2.bucket_count() == 0);
+    assert(c2.size() == 0);
+    assert(c2.hash_function() == Hash(1));
+    assert(c2.key_eq() == Compare(1));
+    assert(c2.get_allocator() == Alloc(1));
+    assert(static_cast<std::size_t>(std::distance(c2.begin(), c2.end())) == c2.size());
+    assert(static_cast<std::size_t>(std::distance(c2.cbegin(), c2.cend())) == c2.size());
+    assert(c2.max_load_factor() == 1);
+  }
+  {
+    typedef test_hash<int> Hash;
+    typedef test_equal_to<int> Compare;
+    typedef other_allocator<int> Alloc;
+    typedef std::unordered_multiset<int, Hash, Compare, Alloc> C;
+    typedef int P;
+    P a2[] = {P(10), P(20), P(30), P(40), P(50), P(60), P(70), P(80)};
+    C c1(0, Hash(1), Compare(1), Alloc(1));
+    C c2(std::begin(a2), std::end(a2), 0, Hash(2), Compare(2), Alloc(2));
+    c2.max_load_factor(2);
+    swap(c1, c2);
+
+    assert(c1.bucket_count() >= 8);
+    assert(c1.size() == 8);
+    assert(*c1.find(10) == 10);
+    assert(*c1.find(20) == 20);
+    assert(*c1.find(30) == 30);
+    assert(*c1.find(40) == 40);
+    assert(*c1.find(50) == 50);
+    assert(*c1.find(60) == 60);
+    assert(*c1.find(70) == 70);
+    assert(*c1.find(80) == 80);
+    assert(c1.hash_function() == Hash(2));
+    assert(c1.key_eq() == Compare(2));
+    assert(c1.get_allocator() == Alloc(2));
+    assert(static_cast<std::size_t>(std::distance(c1.begin(), c1.end())) == c1.size());
+    assert(static_cast<std::size_t>(std::distance(c1.cbegin(), c1.cend())) == c1.size());
+    assert(c1.max_load_factor() == 2);
+
+    LIBCPP_ASSERT(c2.bucket_count() == 0);
+    assert(c2.size() == 0);
+    assert(c2.hash_function() == Hash(1));
+    assert(c2.key_eq() == Compare(1));
+    assert(c2.get_allocator() == Alloc(1));
+    assert(static_cast<std::size_t>(std::distance(c2.begin(), c2.end())) == c2.size());
+    assert(static_cast<std::size_t>(std::distance(c2.cbegin(), c2.cend())) == c2.size());
+    assert(c2.max_load_factor() == 1);
+  }
+  {
+    typedef test_hash<int> Hash;
+    typedef test_equal_to<int> Compare;
+    typedef other_allocator<int> Alloc;
+    typedef std::unordered_multiset<int, Hash, Compare, Alloc> C;
+    typedef int P;
+    P a1[] = {P(1), P(2), P(3), P(4), P(1), P(2)};
+    C c1(std::begin(a1), std::end(a1), 0, Hash(1), Compare(1), Alloc(1));
+    C c2(0, Hash(2), Compare(2), Alloc(2));
+    c2.max_load_factor(2);
+    swap(c1, c2);
+
+    LIBCPP_ASSERT(c1.bucket_count() == 0);
+    assert(c1.size() == 0);
+    assert(c1.hash_function() == Hash(2));
+    assert(c1.key_eq() == Compare(2));
+    assert(c1.get_allocator() == Alloc(2));
+    assert(static_cast<std::size_t>(std::distance(c1.begin(), c1.end())) == c1.size());
+    assert(static_cast<std::size_t>(std::distance(c1.cbegin(), c1.cend())) == c1.size());
+    assert(c1.max_load_factor() == 2);
+
+    assert(c2.bucket_count() >= 6);
+    assert(c2.size() == 6);
+    assert(c2.count(1) == 2);
+    assert(c2.count(2) == 2);
+    assert(c2.count(3) == 1);
+    assert(c2.count(4) == 1);
+    assert(c2.hash_function() == Hash(1));
+    assert(c2.key_eq() == Compare(1));
+    assert(c2.get_allocator() == Alloc(1));
+    assert(static_cast<std::size_t>(std::distance(c2.begin(), c2.end())) == c2.size());
+    assert(static_cast<std::size_t>(std::distance(c2.cbegin(), c2.cend())) == c2.size());
+    assert(c2.max_load_factor() == 1);
+  }
+  {
+    typedef test_hash<int> Hash;
+    typedef test_equal_to<int> Compare;
+    typedef other_allocator<int> Alloc;
+    typedef std::unordered_multiset<int, Hash, Compare, Alloc> C;
+    typedef int P;
+    P a1[] = {P(1), P(2), P(3), P(4), P(1), P(2)};
+    P a2[] = {P(10), P(20), P(30), P(40), P(50), P(60), P(70), P(80)};
+    C c1(std::begin(a1), std::end(a1), 0, Hash(1), Compare(1), Alloc(1));
+    C c2(std::begin(a2), std::end(a2), 0, Hash(2), Compare(2), Alloc(2));
+    c2.max_load_factor(2);
+    swap(c1, c2);
+
+    assert(c1.bucket_count() >= 8);
+    assert(c1.size() == 8);
+    assert(*c1.find(10) == 10);
+    assert(*c1.find(20) == 20);
+    assert(*c1.find(30) == 30);
+    assert(*c1.find(40) == 40);
+    assert(*c1.find(50) == 50);
+    assert(*c1.find(60) == 60);
+    assert(*c1.find(70) == 70);
+    assert(*c1.find(80) == 80);
+    assert(c1.hash_function() == Hash(2));
+    assert(c1.key_eq() == Compare(2));
+    assert(c1.get_allocator() == Alloc(2));
+    assert(static_cast<std::size_t>(std::distance(c1.begin(), c1.end())) == c1.size());
+    assert(static_cast<std::size_t>(std::distance(c1.cbegin(), c1.cend())) == c1.size());
+    assert(c1.max_load_factor() == 2);
+
+    assert(c2.bucket_count() >= 6);
+    assert(c2.size() == 6);
+    assert(c2.count(1) == 2);
+    assert(c2.count(2) == 2);
+    assert(c2.count(3) == 1);
+    assert(c2.count(4) == 1);
+    assert(c2.hash_function() == Hash(1));
+    assert(c2.key_eq() == Compare(1));
+    assert(c2.get_allocator() == Alloc(1));
+    assert(static_cast<std::size_t>(std::distance(c2.begin(), c2.end())) == c2.size());
+    assert(static_cast<std::size_t>(std::distance(c2.cbegin(), c2.cend())) == c2.size());
+    assert(c2.max_load_factor() == 1);
+  }
 #if TEST_STD_VER >= 11
-    {
-        typedef test_hash<int> Hash;
-        typedef test_equal_to<int> Compare;
-        typedef min_allocator<int> Alloc;
-        typedef std::unordered_multiset<int, Hash, Compare, Alloc> C;
-        C c1(0, Hash(1), Compare(1), Alloc());
-        C c2(0, Hash(2), Compare(2), Alloc());
-        c2.max_load_factor(2);
-        swap(c1, c2);
-
-        LIBCPP_ASSERT(c1.bucket_count() == 0);
-        assert(c1.size() == 0);
-        assert(c1.hash_function() == Hash(2));
-        assert(c1.key_eq() == Compare(2));
-        assert(c1.get_allocator() == Alloc());
-        assert(static_cast<std::size_t>(std::distance(c1.begin(), c1.end())) == c1.size());
-        assert(static_cast<std::size_t>(std::distance(c1.cbegin(), c1.cend())) == c1.size());
-        assert(c1.max_load_factor() == 2);
-
-        LIBCPP_ASSERT(c2.bucket_count() == 0);
-        assert(c2.size() == 0);
-        assert(c2.hash_function() == Hash(1));
-        assert(c2.key_eq() == Compare(1));
-        assert(c2.get_allocator() == Alloc());
-        assert(static_cast<std::size_t>(std::distance(c2.begin(), c2.end())) == c2.size());
-        assert(static_cast<std::size_t>(std::distance(c2.cbegin(), c2.cend())) == c2.size());
-        assert(c2.max_load_factor() == 1);
-    }
-    {
-        typedef test_hash<int> Hash;
-        typedef test_equal_to<int> Compare;
-        typedef min_allocator<int> Alloc;
-        typedef std::unordered_multiset<int, Hash, Compare, Alloc> C;
-        typedef int P;
-        P a2[] =
-        {
-            P(10),
-            P(20),
-            P(30),
-            P(40),
-            P(50),
-            P(60),
-            P(70),
-            P(80)
-        };
-        C c1(0, Hash(1), Compare(1), Alloc());
-        C c2(std::begin(a2), std::end(a2), 0, Hash(2), Compare(2), Alloc());
-        c2.max_load_factor(2);
-        swap(c1, c2);
-
-        assert(c1.bucket_count() >= 8);
-        assert(c1.size() == 8);
-        assert(*c1.find(10) == 10);
-        assert(*c1.find(20) == 20);
-        assert(*c1.find(30) == 30);
-        assert(*c1.find(40) == 40);
-        assert(*c1.find(50) == 50);
-        assert(*c1.find(60) == 60);
-        assert(*c1.find(70) == 70);
-        assert(*c1.find(80) == 80);
-        assert(c1.hash_function() == Hash(2));
-        assert(c1.key_eq() == Compare(2));
-        assert(c1.get_allocator() == Alloc());
-        assert(static_cast<std::size_t>(std::distance(c1.begin(), c1.end())) == c1.size());
-        assert(static_cast<std::size_t>(std::distance(c1.cbegin(), c1.cend())) == c1.size());
-        assert(c1.max_load_factor() == 2);
-
-        LIBCPP_ASSERT(c2.bucket_count() == 0);
-        assert(c2.size() == 0);
-        assert(c2.hash_function() == Hash(1));
-        assert(c2.key_eq() == Compare(1));
-        assert(c2.get_allocator() == Alloc());
-        assert(static_cast<std::size_t>(std::distance(c2.begin(), c2.end())) == c2.size());
-        assert(static_cast<std::size_t>(std::distance(c2.cbegin(), c2.cend())) == c2.size());
-        assert(c2.max_load_factor() == 1);
-    }
-    {
-        typedef test_hash<int> Hash;
-        typedef test_equal_to<int> Compare;
-        typedef min_allocator<int> Alloc;
-        typedef std::unordered_multiset<int, Hash, Compare, Alloc> C;
-        typedef int P;
-        P a1[] =
-        {
-            P(1),
-            P(2),
-            P(3),
-            P(4),
-            P(1),
-            P(2)
-        };
-        C c1(std::begin(a1), std::end(a1), 0, Hash(1), Compare(1), Alloc());
-        C c2(0, Hash(2), Compare(2), Alloc());
-        c2.max_load_factor(2);
-        swap(c1, c2);
-
-        LIBCPP_ASSERT(c1.bucket_count() == 0);
-        assert(c1.size() == 0);
-        assert(c1.hash_function() == Hash(2));
-        assert(c1.key_eq() == Compare(2));
-        assert(c1.get_allocator() == Alloc());
-        assert(static_cast<std::size_t>(std::distance(c1.begin(), c1.end())) == c1.size());
-        assert(static_cast<std::size_t>(std::distance(c1.cbegin(), c1.cend())) == c1.size());
-        assert(c1.max_load_factor() == 2);
-
-        assert(c2.bucket_count() >= 6);
-        assert(c2.size() == 6);
-        assert(c2.count(1) == 2);
-        assert(c2.count(2) == 2);
-        assert(c2.count(3) == 1);
-        assert(c2.count(4) == 1);
-        assert(c2.hash_function() == Hash(1));
-        assert(c2.key_eq() == Compare(1));
-        assert(c2.get_allocator() == Alloc());
-        assert(static_cast<std::size_t>(std::distance(c2.begin(), c2.end())) == c2.size());
-        assert(static_cast<std::size_t>(std::distance(c2.cbegin(), c2.cend())) == c2.size());
-        assert(c2.max_load_factor() == 1);
-    }
-    {
-        typedef test_hash<int> Hash;
-        typedef test_equal_to<int> Compare;
-        typedef min_allocator<int> Alloc;
-        typedef std::unordered_multiset<int, Hash, Compare, Alloc> C;
-        typedef int P;
-        P a1[] =
-        {
-            P(1),
-            P(2),
-            P(3),
-            P(4),
-            P(1),
-            P(2)
-        };
-        P a2[] =
-        {
-            P(10),
-            P(20),
-            P(30),
-            P(40),
-            P(50),
-            P(60),
-            P(70),
-            P(80)
-        };
-        C c1(std::begin(a1), std::end(a1), 0, Hash(1), Compare(1), Alloc());
-        C c2(std::begin(a2), std::end(a2), 0, Hash(2), Compare(2), Alloc());
-        c2.max_load_factor(2);
-        swap(c1, c2);
-
-        assert(c1.bucket_count() >= 8);
-        assert(c1.size() == 8);
-        assert(*c1.find(10) == 10);
-        assert(*c1.find(20) == 20);
-        assert(*c1.find(30) == 30);
-        assert(*c1.find(40) == 40);
-        assert(*c1.find(50) == 50);
-        assert(*c1.find(60) == 60);
-        assert(*c1.find(70) == 70);
-        assert(*c1.find(80) == 80);
-        assert(c1.hash_function() == Hash(2));
-        assert(c1.key_eq() == Compare(2));
-        assert(c1.get_allocator() == Alloc());
-        assert(static_cast<std::size_t>(std::distance(c1.begin(), c1.end())) == c1.size());
-        assert(static_cast<std::size_t>(std::distance(c1.cbegin(), c1.cend())) == c1.size());
-        assert(c1.max_load_factor() == 2);
-
-        assert(c2.bucket_count() >= 6);
-        assert(c2.size() == 6);
-        assert(c2.count(1) == 2);
-        assert(c2.count(2) == 2);
-        assert(c2.count(3) == 1);
-        assert(c2.count(4) == 1);
-        assert(c2.hash_function() == Hash(1));
-        assert(c2.key_eq() == Compare(1));
-        assert(c2.get_allocator() == Alloc());
-        assert(static_cast<std::size_t>(std::distance(c2.begin(), c2.end())) == c2.size());
-        assert(static_cast<std::size_t>(std::distance(c2.cbegin(), c2.cend())) == c2.size());
-        assert(c2.max_load_factor() == 1);
-    }
+  {
+    typedef test_hash<int> Hash;
+    typedef test_equal_to<int> Compare;
+    typedef min_allocator<int> Alloc;
+    typedef std::unordered_multiset<int, Hash, Compare, Alloc> C;
+    C c1(0, Hash(1), Compare(1), Alloc());
+    C c2(0, Hash(2), Compare(2), Alloc());
+    c2.max_load_factor(2);
+    swap(c1, c2);
+
+    LIBCPP_ASSERT(c1.bucket_count() == 0);
+    assert(c1.size() == 0);
+    assert(c1.hash_function() == Hash(2));
+    assert(c1.key_eq() == Compare(2));
+    assert(c1.get_allocator() == Alloc());
+    assert(static_cast<std::size_t>(std::distance(c1.begin(), c1.end())) == c1.size());
+    assert(static_cast<std::size_t>(std::distance(c1.cbegin(), c1.cend())) == c1.size());
+    assert(c1.max_load_factor() == 2);
+
+    LIBCPP_ASSERT(c2.bucket_count() == 0);
+    assert(c2.size() == 0);
+    assert(c2.hash_function() == Hash(1));
+    assert(c2.key_eq() == Compare(1));
+    assert(c2.get_allocator() == Alloc());
+    assert(static_cast<std::size_t>(std::distance(c2.begin(), c2.end())) == c2.size());
+    assert(static_cast<std::size_t>(std::distance(c2.cbegin(), c2.cend())) == c2.size());
+    assert(c2.max_load_factor() == 1);
+  }
+  {
+    typedef test_hash<int> Hash;
+    typedef test_equal_to<int> Compare;
+    typedef min_allocator<int> Alloc;
+    typedef std::unordered_multiset<int, Hash, Compare, Alloc> C;
+    typedef int P;
+    P a2[] = {P(10), P(20), P(30), P(40), P(50), P(60), P(70), P(80)};
+    C c1(0, Hash(1), Compare(1), Alloc());
+    C c2(std::begin(a2), std::end(a2), 0, Hash(2), Compare(2), Alloc());
+    c2.max_load_factor(2);
+    swap(c1, c2);
+
+    assert(c1.bucket_count() >= 8);
+    assert(c1.size() == 8);
+    assert(*c1.find(10) == 10);
+    assert(*c1.find(20) == 20);
+    assert(*c1.find(30) == 30);
+    assert(*c1.find(40) == 40);
+    assert(*c1.find(50) == 50);
+    assert(*c1.find(60) == 60);
+    assert(*c1.find(70) == 70);
+    assert(*c1.find(80) == 80);
+    assert(c1.hash_function() == Hash(2));
+    assert(c1.key_eq() == Compare(2));
+    assert(c1.get_allocator() == Alloc());
+    assert(static_cast<std::size_t>(std::distance(c1.begin(), c1.end())) == c1.size());
+    assert(static_cast<std::size_t>(std::distance(c1.cbegin(), c1.cend())) == c1.size());
+    assert(c1.max_load_factor() == 2);
+
+    LIBCPP_ASSERT(c2.bucket_count() == 0);
+    assert(c2.size() == 0);
+    assert(c2.hash_function() == Hash(1));
+    assert(c2.key_eq() == Compare(1));
+    assert(c2.get_allocator() == Alloc());
+    assert(static_cast<std::size_t>(std::distance(c2.begin(), c2.end())) == c2.size());
+    assert(static_cast<std::size_t>(std::distance(c2.cbegin(), c2.cend())) == c2.size());
+    assert(c2.max_load_factor() == 1);
+  }
+  {
+    typedef test_hash<int> Hash;
+    typedef test_equal_to<int> Compare;
+    typedef min_allocator<int> Alloc;
+    typedef std::unordered_multiset<int, Hash, Compare, Alloc> C;
+    typedef int P;
+    P a1[] = {P(1), P(2), P(3), P(4), P(1), P(2)};
+    C c1(std::begin(a1), std::end(a1), 0, Hash(1), Compare(1), Alloc());
+    C c2(0, Hash(2), Compare(2), Alloc());
+    c2.max_load_factor(2);
+    swap(c1, c2);
+
+    LIBCPP_ASSERT(c1.bucket_count() == 0);
+    assert(c1.size() == 0);
+    assert(c1.hash_function() == Hash(2));
+    assert(c1.key_eq() == Compare(2));
+    assert(c1.get_allocator() == Alloc());
+    assert(static_cast<std::size_t>(std::distance(c1.begin(), c1.end())) == c1.size());
+    assert(static_cast<std::size_t>(std::distance(c1.cbegin(), c1.cend())) == c1.size());
+    assert(c1.max_load_factor() == 2);
+
+    assert(c2.bucket_count() >= 6);
+    assert(c2.size() == 6);
+    assert(c2.count(1) == 2);
+    assert(c2.count(2) == 2);
+    assert(c2.count(3) == 1);
+    assert(c2.count(4) == 1);
+    assert(c2.hash_function() == Hash(1));
+    assert(c2.key_eq() == Compare(1));
+    assert(c2.get_allocator() == Alloc());
+    assert(static_cast<std::size_t>(std::distance(c2.begin(), c2.end())) == c2.size());
+    assert(static_cast<std::size_t>(std::distance(c2.cbegin(), c2.cend())) == c2.size());
+    assert(c2.max_load_factor() == 1);
+  }
+  {
+    typedef test_hash<int> Hash;
+    typedef test_equal_to<int> Compare;
+    typedef min_allocator<int> Alloc;
+    typedef std::unordered_multiset<int, Hash, Compare, Alloc> C;
+    typedef int P;
+    P a1[] = {P(1), P(2), P(3), P(4), P(1), P(2)};
+    P a2[] = {P(10), P(20), P(30), P(40), P(50), P(60), P(70), P(80)};
+    C c1(std::begin(a1), std::end(a1), 0, Hash(1), Compare(1), Alloc());
+    C c2(std::begin(a2), std::end(a2), 0, Hash(2), Compare(2), Alloc());
+    c2.max_load_factor(2);
+    swap(c1, c2);
+
+    assert(c1.bucket_count() >= 8);
+    assert(c1.size() == 8);
+    assert(*c1.find(10) == 10);
+    assert(*c1.find(20) == 20);
+    assert(*c1.find(30) == 30);
+    assert(*c1.find(40) == 40);
+    assert(*c1.find(50) == 50);
+    assert(*c1.find(60) == 60);
+    assert(*c1.find(70) == 70);
+    assert(*c1.find(80) == 80);
+    assert(c1.hash_function() == Hash(2));
+    assert(c1.key_eq() == Compare(2));
+    assert(c1.get_allocator() == Alloc());
+    assert(static_cast<std::size_t>(std::distance(c1.begin(), c1.end())) == c1.size());
+    assert(static_cast<std::size_t>(std::distance(c1.cbegin(), c1.cend())) == c1.size());
+    assert(c1.max_load_factor() == 2);
+
+    assert(c2.bucket_count() >= 6);
+    assert(c2.size() == 6);
+    assert(c2.count(1) == 2);
+    assert(c2.count(2) == 2);
+    assert(c2.count(3) == 1);
+    assert(c2.count(4) == 1);
+    assert(c2.hash_function() == Hash(1));
+    assert(c2.key_eq() == Compare(1));
+    assert(c2.get_allocator() == Alloc());
+    assert(static_cast<std::size_t>(std::distance(c2.begin(), c2.end())) == c2.size());
+    assert(static_cast<std::size_t>(std::distance(c2.cbegin(), c2.cend())) == c2.size());
+    assert(c2.max_load_factor() == 1);
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/unord/unord.set/allocator_mismatch.verify.cpp b/libcxx/test/std/containers/unord/unord.set/allocator_mismatch.verify.cpp
index 81469059a2271..8b6cc8e93c8bc 100644
--- a/libcxx/test/std/containers/unord/unord.set/allocator_mismatch.verify.cpp
+++ b/libcxx/test/std/containers/unord/unord.set/allocator_mismatch.verify.cpp
@@ -12,4 +12,4 @@
 #include <unordered_set>
 
 std::unordered_set<int, std::hash<int>, std::less<int>, std::allocator<long> > v;
-  // expected-error-re@*:* {{static assertion failed{{.*}}Allocator::value_type must be same type as value_type}}
+// expected-error-re@*:* {{static assertion failed{{.*}}Allocator::value_type must be same type as value_type}}
diff --git a/libcxx/test/std/containers/unord/unord.set/bucket.pass.cpp b/libcxx/test/std/containers/unord/unord.set/bucket.pass.cpp
index 969c3f6b93e74..b7bb0153f1930 100644
--- a/libcxx/test/std/containers/unord/unord.set/bucket.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.set/bucket.pass.cpp
@@ -20,46 +20,29 @@
 #include "test_macros.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef std::unordered_set<int> C;
-        typedef int P;
-        P a[] =
-        {
-            P(1),
-            P(2),
-            P(3),
-            P(4),
-            P(1),
-            P(2)
-        };
-        const C c(std::begin(a), std::end(a));
-        std::size_t bc = c.bucket_count();
-        assert(bc >= 5);
-        for (std::size_t i = 0; i < 13; ++i)
-            LIBCPP_ASSERT(c.bucket(i) == i % bc);
-    }
+int main(int, char**) {
+  {
+    typedef std::unordered_set<int> C;
+    typedef int P;
+    P a[] = {P(1), P(2), P(3), P(4), P(1), P(2)};
+    const C c(std::begin(a), std::end(a));
+    std::size_t bc = c.bucket_count();
+    assert(bc >= 5);
+    for (std::size_t i = 0; i < 13; ++i)
+      LIBCPP_ASSERT(c.bucket(i) == i % bc);
+  }
 #if TEST_STD_VER >= 11
-    {
-        typedef std::unordered_set<int, std::hash<int>, std::equal_to<int>, min_allocator<int>> C;
-        typedef int P;
-        P a[] =
-        {
-            P(1),
-            P(2),
-            P(3),
-            P(4),
-            P(1),
-            P(2)
-        };
-        const C c(std::begin(a), std::end(a));
-        std::size_t bc = c.bucket_count();
-        assert(bc >= 5);
-        for (std::size_t i = 0; i < 13; ++i)
-            LIBCPP_ASSERT(c.bucket(i) == i % bc);
-    }
+  {
+    typedef std::unordered_set<int, std::hash<int>, std::equal_to<int>, min_allocator<int>> C;
+    typedef int P;
+    P a[] = {P(1), P(2), P(3), P(4), P(1), P(2)};
+    const C c(std::begin(a), std::end(a));
+    std::size_t bc = c.bucket_count();
+    assert(bc >= 5);
+    for (std::size_t i = 0; i < 13; ++i)
+      LIBCPP_ASSERT(c.bucket(i) == i % bc);
+  }
 #endif
 
-    return 0;
+  return 0;
 }
diff --git a/libcxx/test/std/containers/unord/unord.set/bucket_count.pass.cpp b/libcxx/test/std/containers/unord/unord.set/bucket_count.pass.cpp
index 3a1a78cb3d80a..4d03857581bf2 100644
--- a/libcxx/test/std/containers/unord/unord.set/bucket_count.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.set/bucket_count.pass.cpp
@@ -20,53 +20,32 @@
 #include "test_macros.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef std::unordered_set<int> C;
-        const C c;
-        LIBCPP_ASSERT(c.bucket_count() == 0);
-    }
-    {
-        typedef std::unordered_set<int> C;
-        typedef int P;
-        P a[] =
-        {
-            P(10),
-            P(20),
-            P(30),
-            P(40),
-            P(50),
-            P(60),
-            P(70),
-            P(80)
-        };
-        const C c(std::begin(a), std::end(a));
-        assert(c.bucket_count() >= 8);
-    }
+int main(int, char**) {
+  {
+    typedef std::unordered_set<int> C;
+    const C c;
+    LIBCPP_ASSERT(c.bucket_count() == 0);
+  }
+  {
+    typedef std::unordered_set<int> C;
+    typedef int P;
+    P a[] = {P(10), P(20), P(30), P(40), P(50), P(60), P(70), P(80)};
+    const C c(std::begin(a), std::end(a));
+    assert(c.bucket_count() >= 8);
+  }
 #if TEST_STD_VER >= 11
-    {
-        typedef std::unordered_set<int, std::hash<int>, std::equal_to<int>, min_allocator<int>> C;
-        const C c;
-        LIBCPP_ASSERT(c.bucket_count() == 0);
-    }
-    {
-        typedef std::unordered_set<int, std::hash<int>, std::equal_to<int>, min_allocator<int>> C;
-        typedef int P;
-        P a[] =
-        {
-            P(10),
-            P(20),
-            P(30),
-            P(40),
-            P(50),
-            P(60),
-            P(70),
-            P(80)
-        };
-        const C c(std::begin(a), std::end(a));
-        assert(c.bucket_count() >= 8);
-    }
+  {
+    typedef std::unordered_set<int, std::hash<int>, std::equal_to<int>, min_allocator<int>> C;
+    const C c;
+    LIBCPP_ASSERT(c.bucket_count() == 0);
+  }
+  {
+    typedef std::unordered_set<int, std::hash<int>, std::equal_to<int>, min_allocator<int>> C;
+    typedef int P;
+    P a[] = {P(10), P(20), P(30), P(40), P(50), P(60), P(70), P(80)};
+    const C c(std::begin(a), std::end(a));
+    assert(c.bucket_count() >= 8);
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/unord/unord.set/bucket_size.pass.cpp b/libcxx/test/std/containers/unord/unord.set/bucket_size.pass.cpp
index 1016c499d6745..d44943fcdb272 100644
--- a/libcxx/test/std/containers/unord/unord.set/bucket_size.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.set/bucket_size.pass.cpp
@@ -20,50 +20,33 @@
 #include "test_macros.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef std::unordered_set<int> C;
-        typedef int P;
-        P a[] =
-        {
-            P(1),
-            P(2),
-            P(3),
-            P(4),
-            P(1),
-            P(2)
-        };
-        const C c(std::begin(a), std::end(a));
-        assert(c.bucket_count() >= 5);
-        LIBCPP_ASSERT(c.bucket_size(0) == 0);
-        LIBCPP_ASSERT(c.bucket_size(1) == 1);
-        LIBCPP_ASSERT(c.bucket_size(2) == 1);
-        LIBCPP_ASSERT(c.bucket_size(3) == 1);
-        LIBCPP_ASSERT(c.bucket_size(4) == 1);
-    }
+int main(int, char**) {
+  {
+    typedef std::unordered_set<int> C;
+    typedef int P;
+    P a[] = {P(1), P(2), P(3), P(4), P(1), P(2)};
+    const C c(std::begin(a), std::end(a));
+    assert(c.bucket_count() >= 5);
+    LIBCPP_ASSERT(c.bucket_size(0) == 0);
+    LIBCPP_ASSERT(c.bucket_size(1) == 1);
+    LIBCPP_ASSERT(c.bucket_size(2) == 1);
+    LIBCPP_ASSERT(c.bucket_size(3) == 1);
+    LIBCPP_ASSERT(c.bucket_size(4) == 1);
+  }
 #if TEST_STD_VER >= 11
-    {
-        typedef std::unordered_set<int, std::hash<int>, std::equal_to<int>, min_allocator<int>> C;
-        typedef int P;
-        P a[] =
-        {
-            P(1),
-            P(2),
-            P(3),
-            P(4),
-            P(1),
-            P(2)
-        };
-        const C c(std::begin(a), std::end(a));
-        assert(c.bucket_count() >= 5);
-        LIBCPP_ASSERT(c.bucket_size(0) == 0);
-        LIBCPP_ASSERT(c.bucket_size(1) == 1);
-        LIBCPP_ASSERT(c.bucket_size(2) == 1);
-        LIBCPP_ASSERT(c.bucket_size(3) == 1);
-        LIBCPP_ASSERT(c.bucket_size(4) == 1);
-    }
+  {
+    typedef std::unordered_set<int, std::hash<int>, std::equal_to<int>, min_allocator<int>> C;
+    typedef int P;
+    P a[] = {P(1), P(2), P(3), P(4), P(1), P(2)};
+    const C c(std::begin(a), std::end(a));
+    assert(c.bucket_count() >= 5);
+    LIBCPP_ASSERT(c.bucket_size(0) == 0);
+    LIBCPP_ASSERT(c.bucket_size(1) == 1);
+    LIBCPP_ASSERT(c.bucket_size(2) == 1);
+    LIBCPP_ASSERT(c.bucket_size(3) == 1);
+    LIBCPP_ASSERT(c.bucket_size(4) == 1);
+  }
 #endif
 
-    return 0;
+  return 0;
 }
diff --git a/libcxx/test/std/containers/unord/unord.set/clear.pass.cpp b/libcxx/test/std/containers/unord/unord.set/clear.pass.cpp
index ab04cdd910599..8687904287070 100644
--- a/libcxx/test/std/containers/unord/unord.set/clear.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.set/clear.pass.cpp
@@ -20,43 +20,26 @@
 #include "test_macros.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef std::unordered_set<int> C;
-        typedef int P;
-        P a[] =
-        {
-            P(1),
-            P(2),
-            P(3),
-            P(4),
-            P(1),
-            P(2)
-        };
-        C c(a, a + sizeof(a)/sizeof(a[0]));
-        ASSERT_NOEXCEPT(c.clear());
-        c.clear();
-        assert(c.size() == 0);
-    }
+int main(int, char**) {
+  {
+    typedef std::unordered_set<int> C;
+    typedef int P;
+    P a[] = {P(1), P(2), P(3), P(4), P(1), P(2)};
+    C c(a, a + sizeof(a) / sizeof(a[0]));
+    ASSERT_NOEXCEPT(c.clear());
+    c.clear();
+    assert(c.size() == 0);
+  }
 #if TEST_STD_VER >= 11
-    {
-        typedef std::unordered_set<int, std::hash<int>, std::equal_to<int>, min_allocator<int>> C;
-        typedef int P;
-        P a[] =
-        {
-            P(1),
-            P(2),
-            P(3),
-            P(4),
-            P(1),
-            P(2)
-        };
-        C c(a, a + sizeof(a)/sizeof(a[0]));
-        ASSERT_NOEXCEPT(c.clear());
-        c.clear();
-        assert(c.size() == 0);
-    }
+  {
+    typedef std::unordered_set<int, std::hash<int>, std::equal_to<int>, min_allocator<int>> C;
+    typedef int P;
+    P a[] = {P(1), P(2), P(3), P(4), P(1), P(2)};
+    C c(a, a + sizeof(a) / sizeof(a[0]));
+    ASSERT_NOEXCEPT(c.clear());
+    c.clear();
+    assert(c.size() == 0);
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/unord/unord.set/contains.pass.cpp b/libcxx/test/std/containers/unord/unord.set/contains.pass.cpp
index c9b78368431df..7a4532f10f132 100644
--- a/libcxx/test/std/containers/unord/unord.set/contains.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.set/contains.pass.cpp
@@ -17,27 +17,32 @@
 
 template <typename T, typename V, typename B, typename... Vals>
 void test(B bad, Vals... args) {
-    T set;
-    V vals[] = {args...};
+  T set;
+  V vals[] = {args...};
 
-    for (auto& v : vals) set.insert(v);
-    for (auto& v : vals) assert(set.contains(v));
+  for (auto& v : vals)
+    set.insert(v);
+  for (auto& v : vals)
+    assert(set.contains(v));
 
-    assert(!set.contains(bad));
+  assert(!set.contains(bad));
 }
 
-struct E { int a = 1; double b = 1; char c = 1; };
-
-int main(int, char**)
-{
-    {
-        test<std::unordered_set<int>, int>(14, 10, 11, 12, 13);
-        test<std::unordered_set<char>, char>('e', 'a', 'b', 'c', 'd');
-    }
-    {
-        test<std::unordered_multiset<int>, int>(14, 10, 11, 12, 13);
-        test<std::unordered_multiset<char>, char>('e', 'a', 'b', 'c', 'd');
-    }
-
-    return 0;
+struct E {
+  int a    = 1;
+  double b = 1;
+  char c   = 1;
+};
+
+int main(int, char**) {
+  {
+    test<std::unordered_set<int>, int>(14, 10, 11, 12, 13);
+    test<std::unordered_set<char>, char>('e', 'a', 'b', 'c', 'd');
+  }
+  {
+    test<std::unordered_multiset<int>, int>(14, 10, 11, 12, 13);
+    test<std::unordered_multiset<char>, char>('e', 'a', 'b', 'c', 'd');
+  }
+
+  return 0;
 }
diff --git a/libcxx/test/std/containers/unord/unord.set/contains.transparent.pass.cpp b/libcxx/test/std/containers/unord/unord.set/contains.transparent.pass.cpp
index 78908053aa2f5..5b685dad1c376 100644
--- a/libcxx/test/std/containers/unord/unord.set/contains.transparent.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.set/contains.transparent.pass.cpp
@@ -20,44 +20,43 @@
 #include <unordered_set>
 #include "test_transparent_unordered.h"
 
-int main(int, char**)
-{
-    using key_type = StoredType<int>;
-
-    {
-      // Make sure conversions don't happen for transparent non-final hasher and key_equal
-      using S = unord_set_type<std::unordered_set, transparent_hash, std::equal_to<>>;
-      test_transparent_contains<S>({1, 2});
-      test_transparent_contains<const S>({1, 2});
-    }
-
-    {
-      // Make sure conversions don't happen for transparent final hasher and key_equal
-      using S = unord_set_type<std::unordered_set, transparent_hash_final, transparent_equal_final>;
-      test_transparent_contains<S>({1, 2});
-      test_transparent_contains<const S>({1, 2});
-    }
-
-    {
-      // Make sure conversions do happen for non-transparent hasher
-      using S = unord_set_type<std::unordered_set, non_transparent_hash, std::equal_to<>>;
-      test_non_transparent_contains<S>({1, 2});
-      test_non_transparent_contains<const S>({1, 2});
-    }
-
-    {
-      // Make sure conversions do happen for non-transparent key_equal
-      using S = unord_set_type<std::unordered_set, transparent_hash, std::equal_to<key_type>>;
-      test_non_transparent_contains<S>({1, 2});
-      test_non_transparent_contains<const S>({1, 2});
-    }
-
-    {
-      // Make sure conversions do happen for both non-transparent hasher and key_equal
-      using S = unord_set_type<std::unordered_set, non_transparent_hash, std::equal_to<key_type>>;
-      test_non_transparent_contains<S>({1, 2});
-      test_non_transparent_contains<const S>({1, 2});
-    }
-
-    return 0;
+int main(int, char**) {
+  using key_type = StoredType<int>;
+
+  {
+    // Make sure conversions don't happen for transparent non-final hasher and key_equal
+    using S = unord_set_type<std::unordered_set, transparent_hash, std::equal_to<>>;
+    test_transparent_contains<S>({1, 2});
+    test_transparent_contains<const S>({1, 2});
+  }
+
+  {
+    // Make sure conversions don't happen for transparent final hasher and key_equal
+    using S = unord_set_type<std::unordered_set, transparent_hash_final, transparent_equal_final>;
+    test_transparent_contains<S>({1, 2});
+    test_transparent_contains<const S>({1, 2});
+  }
+
+  {
+    // Make sure conversions do happen for non-transparent hasher
+    using S = unord_set_type<std::unordered_set, non_transparent_hash, std::equal_to<>>;
+    test_non_transparent_contains<S>({1, 2});
+    test_non_transparent_contains<const S>({1, 2});
+  }
+
+  {
+    // Make sure conversions do happen for non-transparent key_equal
+    using S = unord_set_type<std::unordered_set, transparent_hash, std::equal_to<key_type>>;
+    test_non_transparent_contains<S>({1, 2});
+    test_non_transparent_contains<const S>({1, 2});
+  }
+
+  {
+    // Make sure conversions do happen for both non-transparent hasher and key_equal
+    using S = unord_set_type<std::unordered_set, non_transparent_hash, std::equal_to<key_type>>;
+    test_non_transparent_contains<S>({1, 2});
+    test_non_transparent_contains<const S>({1, 2});
+  }
+
+  return 0;
 }
diff --git a/libcxx/test/std/containers/unord/unord.set/count.pass.cpp b/libcxx/test/std/containers/unord/unord.set/count.pass.cpp
index fc9fce5991d13..56df9a1693f54 100644
--- a/libcxx/test/std/containers/unord/unord.set/count.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.set/count.pass.cpp
@@ -20,51 +20,26 @@
 #include "test_macros.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef std::unordered_set<int> C;
-        typedef int P;
-        P a[] =
-        {
-            P(10),
-            P(20),
-            P(30),
-            P(40),
-            P(50),
-            P(50),
-            P(50),
-            P(60),
-            P(70),
-            P(80)
-        };
-        const C c(std::begin(a), std::end(a));
-        assert(c.count(30) == 1);
-        assert(c.count(50) == 1);
-        assert(c.count(5) == 0);
-    }
+int main(int, char**) {
+  {
+    typedef std::unordered_set<int> C;
+    typedef int P;
+    P a[] = {P(10), P(20), P(30), P(40), P(50), P(50), P(50), P(60), P(70), P(80)};
+    const C c(std::begin(a), std::end(a));
+    assert(c.count(30) == 1);
+    assert(c.count(50) == 1);
+    assert(c.count(5) == 0);
+  }
 #if TEST_STD_VER >= 11
-    {
-        typedef std::unordered_set<int, std::hash<int>, std::equal_to<int>, min_allocator<int>> C;
-        typedef int P;
-        P a[] =
-        {
-            P(10),
-            P(20),
-            P(30),
-            P(40),
-            P(50),
-            P(50),
-            P(50),
-            P(60),
-            P(70),
-            P(80)
-        };
-        const C c(std::begin(a), std::end(a));
-        assert(c.count(30) == 1);
-        assert(c.count(50) == 1);
-        assert(c.count(5) == 0);
-    }
+  {
+    typedef std::unordered_set<int, std::hash<int>, std::equal_to<int>, min_allocator<int>> C;
+    typedef int P;
+    P a[] = {P(10), P(20), P(30), P(40), P(50), P(50), P(50), P(60), P(70), P(80)};
+    const C c(std::begin(a), std::end(a));
+    assert(c.count(30) == 1);
+    assert(c.count(50) == 1);
+    assert(c.count(5) == 0);
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/unord/unord.set/count.transparent.pass.cpp b/libcxx/test/std/containers/unord/unord.set/count.transparent.pass.cpp
index 32af0a54cc5b5..95362a45e2891 100644
--- a/libcxx/test/std/containers/unord/unord.set/count.transparent.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.set/count.transparent.pass.cpp
@@ -21,44 +21,43 @@
 
 #include "test_transparent_unordered.h"
 
-int main(int, char**)
-{
-    using key_type = StoredType<int>;
-
-    {
-      // Make sure conversions don't happen for transparent non-final hasher and key_equal
-      using S = unord_set_type<std::unordered_set, transparent_hash, std::equal_to<>>;
-      test_transparent_count<S>({1, 2});
-      test_transparent_count<const S>({1, 2});
-    }
-
-    {
-      // Make sure conversions don't happen for transparent final hasher and key_equal
-      using S = unord_set_type<std::unordered_set, transparent_hash_final, transparent_equal_final>;
-      test_transparent_count<S>({1, 2});
-      test_transparent_count<const S>({1, 2});
-    }
-
-    {
-      // Make sure conversions do happen for non-transparent hasher
-      using S = unord_set_type<std::unordered_set, non_transparent_hash, std::equal_to<>>;
-      test_non_transparent_count<S>({1, 2});
-      test_non_transparent_count<const S>({1, 2});
-    }
-
-    {
-      // Make sure conversions do happen for non-transparent key_equal
-      using S = unord_set_type<std::unordered_set, transparent_hash, std::equal_to<key_type>>;
-      test_non_transparent_count<S>({1, 2});
-      test_non_transparent_count<const S>({1, 2});
-    }
-
-    {
-      // Make sure conversions do happen for both non-transparent hasher and key_equal
-      using S = unord_set_type<std::unordered_set, non_transparent_hash, std::equal_to<key_type>>;
-      test_non_transparent_count<S>({1, 2});
-      test_non_transparent_count<const S>({1, 2});
-    }
-
-    return 0;
+int main(int, char**) {
+  using key_type = StoredType<int>;
+
+  {
+    // Make sure conversions don't happen for transparent non-final hasher and key_equal
+    using S = unord_set_type<std::unordered_set, transparent_hash, std::equal_to<>>;
+    test_transparent_count<S>({1, 2});
+    test_transparent_count<const S>({1, 2});
+  }
+
+  {
+    // Make sure conversions don't happen for transparent final hasher and key_equal
+    using S = unord_set_type<std::unordered_set, transparent_hash_final, transparent_equal_final>;
+    test_transparent_count<S>({1, 2});
+    test_transparent_count<const S>({1, 2});
+  }
+
+  {
+    // Make sure conversions do happen for non-transparent hasher
+    using S = unord_set_type<std::unordered_set, non_transparent_hash, std::equal_to<>>;
+    test_non_transparent_count<S>({1, 2});
+    test_non_transparent_count<const S>({1, 2});
+  }
+
+  {
+    // Make sure conversions do happen for non-transparent key_equal
+    using S = unord_set_type<std::unordered_set, transparent_hash, std::equal_to<key_type>>;
+    test_non_transparent_count<S>({1, 2});
+    test_non_transparent_count<const S>({1, 2});
+  }
+
+  {
+    // Make sure conversions do happen for both non-transparent hasher and key_equal
+    using S = unord_set_type<std::unordered_set, non_transparent_hash, std::equal_to<key_type>>;
+    test_non_transparent_count<S>({1, 2});
+    test_non_transparent_count<const S>({1, 2});
+  }
+
+  return 0;
 }
diff --git a/libcxx/test/std/containers/unord/unord.set/emplace.pass.cpp b/libcxx/test/std/containers/unord/unord.set/emplace.pass.cpp
index c247b0b914110..8972f03f2d2a8 100644
--- a/libcxx/test/std/containers/unord/unord.set/emplace.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.set/emplace.pass.cpp
@@ -24,47 +24,47 @@
 #include "../../Emplaceable.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef std::unordered_set<Emplaceable> C;
-        typedef std::pair<C::iterator, bool> R;
-        C c;
-        R r = c.emplace();
-        assert(c.size() == 1);
-        assert(*r.first == Emplaceable());
-        assert(r.second);
+int main(int, char**) {
+  {
+    typedef std::unordered_set<Emplaceable> C;
+    typedef std::pair<C::iterator, bool> R;
+    C c;
+    R r = c.emplace();
+    assert(c.size() == 1);
+    assert(*r.first == Emplaceable());
+    assert(r.second);
 
-        r = c.emplace(Emplaceable(5, 6));
-        assert(c.size() == 2);
-        assert(*r.first == Emplaceable(5, 6));
-        assert(r.second);
+    r = c.emplace(Emplaceable(5, 6));
+    assert(c.size() == 2);
+    assert(*r.first == Emplaceable(5, 6));
+    assert(r.second);
 
-        r = c.emplace(5, 6);
-        assert(c.size() == 2);
-        assert(*r.first == Emplaceable(5, 6));
-        assert(!r.second);
-    }
-    {
-        typedef std::unordered_set<Emplaceable, std::hash<Emplaceable>,
-                      std::equal_to<Emplaceable>, min_allocator<Emplaceable>> C;
-        typedef std::pair<C::iterator, bool> R;
-        C c;
-        R r = c.emplace();
-        assert(c.size() == 1);
-        assert(*r.first == Emplaceable());
-        assert(r.second);
+    r = c.emplace(5, 6);
+    assert(c.size() == 2);
+    assert(*r.first == Emplaceable(5, 6));
+    assert(!r.second);
+  }
+  {
+    typedef std::
+        unordered_set<Emplaceable, std::hash<Emplaceable>, std::equal_to<Emplaceable>, min_allocator<Emplaceable>>
+            C;
+    typedef std::pair<C::iterator, bool> R;
+    C c;
+    R r = c.emplace();
+    assert(c.size() == 1);
+    assert(*r.first == Emplaceable());
+    assert(r.second);
 
-        r = c.emplace(Emplaceable(5, 6));
-        assert(c.size() == 2);
-        assert(*r.first == Emplaceable(5, 6));
-        assert(r.second);
+    r = c.emplace(Emplaceable(5, 6));
+    assert(c.size() == 2);
+    assert(*r.first == Emplaceable(5, 6));
+    assert(r.second);
 
-        r = c.emplace(5, 6);
-        assert(c.size() == 2);
-        assert(*r.first == Emplaceable(5, 6));
-        assert(!r.second);
-    }
+    r = c.emplace(5, 6);
+    assert(c.size() == 2);
+    assert(*r.first == Emplaceable(5, 6));
+    assert(!r.second);
+  }
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/unord/unord.set/emplace_hint.pass.cpp b/libcxx/test/std/containers/unord/unord.set/emplace_hint.pass.cpp
index a078a5fb62005..0abf522d38b4c 100644
--- a/libcxx/test/std/containers/unord/unord.set/emplace_hint.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.set/emplace_hint.pass.cpp
@@ -17,7 +17,6 @@
 // template <class... Args>
 //     iterator emplace_hint(const_iterator p, Args&&... args);
 
-
 #include <unordered_set>
 #include <cassert>
 
@@ -25,41 +24,41 @@
 #include "../../Emplaceable.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef std::unordered_set<Emplaceable> C;
-        typedef C::iterator R;
-        C c;
-        R r = c.emplace_hint(c.end());
-        assert(c.size() == 1);
-        assert(*r == Emplaceable());
+int main(int, char**) {
+  {
+    typedef std::unordered_set<Emplaceable> C;
+    typedef C::iterator R;
+    C c;
+    R r = c.emplace_hint(c.end());
+    assert(c.size() == 1);
+    assert(*r == Emplaceable());
 
-        r = c.emplace_hint(c.end(), Emplaceable(5, 6));
-        assert(c.size() == 2);
-        assert(*r == Emplaceable(5, 6));
+    r = c.emplace_hint(c.end(), Emplaceable(5, 6));
+    assert(c.size() == 2);
+    assert(*r == Emplaceable(5, 6));
 
-        r = c.emplace_hint(r, 5, 6);
-        assert(c.size() == 2);
-        assert(*r == Emplaceable(5, 6));
-    }
-    {
-        typedef std::unordered_set<Emplaceable, std::hash<Emplaceable>,
-                      std::equal_to<Emplaceable>, min_allocator<Emplaceable>> C;
-        typedef C::iterator R;
-        C c;
-        R r = c.emplace_hint(c.end());
-        assert(c.size() == 1);
-        assert(*r == Emplaceable());
+    r = c.emplace_hint(r, 5, 6);
+    assert(c.size() == 2);
+    assert(*r == Emplaceable(5, 6));
+  }
+  {
+    typedef std::
+        unordered_set<Emplaceable, std::hash<Emplaceable>, std::equal_to<Emplaceable>, min_allocator<Emplaceable>>
+            C;
+    typedef C::iterator R;
+    C c;
+    R r = c.emplace_hint(c.end());
+    assert(c.size() == 1);
+    assert(*r == Emplaceable());
 
-        r = c.emplace_hint(c.end(), Emplaceable(5, 6));
-        assert(c.size() == 2);
-        assert(*r == Emplaceable(5, 6));
+    r = c.emplace_hint(c.end(), Emplaceable(5, 6));
+    assert(c.size() == 2);
+    assert(*r == Emplaceable(5, 6));
 
-        r = c.emplace_hint(r, 5, 6);
-        assert(c.size() == 2);
-        assert(*r == Emplaceable(5, 6));
-    }
+    r = c.emplace_hint(r, 5, 6);
+    assert(c.size() == 2);
+    assert(*r == Emplaceable(5, 6));
+  }
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/unord/unord.set/empty.pass.cpp b/libcxx/test/std/containers/unord/unord.set/empty.pass.cpp
index ce9cfdcf5f479..51414ec9a6f2f 100644
--- a/libcxx/test/std/containers/unord/unord.set/empty.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.set/empty.pass.cpp
@@ -18,9 +18,8 @@
 #include "test_macros.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
+int main(int, char**) {
+  {
     typedef std::unordered_set<int> M;
     M m;
     ASSERT_NOEXCEPT(m.empty());
@@ -29,9 +28,9 @@ int main(int, char**)
     assert(!m.empty());
     m.clear();
     assert(m.empty());
-    }
+  }
 #if TEST_STD_VER >= 11
-    {
+  {
     typedef std::unordered_set<int, std::hash<int>, std::equal_to<int>, min_allocator<int>> M;
     M m;
     ASSERT_NOEXCEPT(m.empty());
@@ -40,7 +39,7 @@ int main(int, char**)
     assert(!m.empty());
     m.clear();
     assert(m.empty());
-    }
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/unord/unord.set/empty.verify.cpp b/libcxx/test/std/containers/unord/unord.set/empty.verify.cpp
index e2164415aa182..5d8e2cf62d163 100644
--- a/libcxx/test/std/containers/unord/unord.set/empty.verify.cpp
+++ b/libcxx/test/std/containers/unord/unord.set/empty.verify.cpp
@@ -17,6 +17,6 @@
 #include <unordered_set>
 
 void f() {
-    std::unordered_set<int> c;
-    c.empty(); // expected-warning {{ignoring return value of function declared with 'nodiscard' attribute}}
+  std::unordered_set<int> c;
+  c.empty(); // expected-warning {{ignoring return value of function declared with 'nodiscard' attribute}}
 }
diff --git a/libcxx/test/std/containers/unord/unord.set/eq.pass.cpp b/libcxx/test/std/containers/unord/unord.set/eq.pass.cpp
index a0ea30fae0705..5d40042a91d20 100644
--- a/libcxx/test/std/containers/unord/unord.set/eq.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.set/eq.pass.cpp
@@ -24,137 +24,76 @@
 #include "test_macros.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef std::unordered_set<int> C;
-        typedef int P;
-        P a[] =
-        {
-            P(10),
-            P(20),
-            P(30),
-            P(40),
-            P(50),
-            P(60),
-            P(70),
-            P(80)
-        };
-        const C c1(std::begin(a), std::end(a));
-        const C c2;
-        assert(!(c1 == c2));
-        assert( (c1 != c2));
-    }
-    {
-        typedef std::unordered_set<int> C;
-        typedef int P;
-        P a[] =
-        {
-            P(10),
-            P(20),
-            P(30),
-            P(40),
-            P(50),
-            P(60),
-            P(70),
-            P(80)
-        };
-        const C c1(std::begin(a), std::end(a));
-        const C c2 = c1;
-        assert( (c1 == c2));
-        assert(!(c1 != c2));
-    }
-    {
-        typedef std::unordered_set<int> C;
-        typedef int P;
-        P a[] =
-        {
-            P(10),
-            P(20),
-            P(30),
-            P(40),
-            P(50),
-            P(60),
-            P(70),
-            P(80)
-        };
-        C c1(std::begin(a), std::end(a));
-        C c2 = c1;
-        c2.rehash(30);
-        assert( (c1 == c2));
-        assert(!(c1 != c2));
-        c2.insert(P(90));
-        assert(!(c1 == c2));
-        assert( (c1 != c2));
-        c1.insert(P(90));
-        assert( (c1 == c2));
-        assert(!(c1 != c2));
-    }
+int main(int, char**) {
+  {
+    typedef std::unordered_set<int> C;
+    typedef int P;
+    P a[] = {P(10), P(20), P(30), P(40), P(50), P(60), P(70), P(80)};
+    const C c1(std::begin(a), std::end(a));
+    const C c2;
+    assert(!(c1 == c2));
+    assert((c1 != c2));
+  }
+  {
+    typedef std::unordered_set<int> C;
+    typedef int P;
+    P a[] = {P(10), P(20), P(30), P(40), P(50), P(60), P(70), P(80)};
+    const C c1(std::begin(a), std::end(a));
+    const C c2 = c1;
+    assert((c1 == c2));
+    assert(!(c1 != c2));
+  }
+  {
+    typedef std::unordered_set<int> C;
+    typedef int P;
+    P a[] = {P(10), P(20), P(30), P(40), P(50), P(60), P(70), P(80)};
+    C c1(std::begin(a), std::end(a));
+    C c2 = c1;
+    c2.rehash(30);
+    assert((c1 == c2));
+    assert(!(c1 != c2));
+    c2.insert(P(90));
+    assert(!(c1 == c2));
+    assert((c1 != c2));
+    c1.insert(P(90));
+    assert((c1 == c2));
+    assert(!(c1 != c2));
+  }
 #if TEST_STD_VER >= 11
-    {
-        typedef std::unordered_set<int, std::hash<int>, std::equal_to<int>, min_allocator<int>> C;
-        typedef int P;
-        P a[] =
-        {
-            P(10),
-            P(20),
-            P(30),
-            P(40),
-            P(50),
-            P(60),
-            P(70),
-            P(80)
-        };
-        const C c1(std::begin(a), std::end(a));
-        const C c2;
-        assert(!(c1 == c2));
-        assert( (c1 != c2));
-    }
-    {
-        typedef std::unordered_set<int, std::hash<int>, std::equal_to<int>, min_allocator<int>> C;
-        typedef int P;
-        P a[] =
-        {
-            P(10),
-            P(20),
-            P(30),
-            P(40),
-            P(50),
-            P(60),
-            P(70),
-            P(80)
-        };
-        const C c1(std::begin(a), std::end(a));
-        const C c2 = c1;
-        assert( (c1 == c2));
-        assert(!(c1 != c2));
-    }
-    {
-        typedef std::unordered_set<int, std::hash<int>, std::equal_to<int>, min_allocator<int>> C;
-        typedef int P;
-        P a[] =
-        {
-            P(10),
-            P(20),
-            P(30),
-            P(40),
-            P(50),
-            P(60),
-            P(70),
-            P(80)
-        };
-        C c1(std::begin(a), std::end(a));
-        C c2 = c1;
-        c2.rehash(30);
-        assert( (c1 == c2));
-        assert(!(c1 != c2));
-        c2.insert(P(90));
-        assert(!(c1 == c2));
-        assert( (c1 != c2));
-        c1.insert(P(90));
-        assert( (c1 == c2));
-        assert(!(c1 != c2));
-    }
+  {
+    typedef std::unordered_set<int, std::hash<int>, std::equal_to<int>, min_allocator<int>> C;
+    typedef int P;
+    P a[] = {P(10), P(20), P(30), P(40), P(50), P(60), P(70), P(80)};
+    const C c1(std::begin(a), std::end(a));
+    const C c2;
+    assert(!(c1 == c2));
+    assert((c1 != c2));
+  }
+  {
+    typedef std::unordered_set<int, std::hash<int>, std::equal_to<int>, min_allocator<int>> C;
+    typedef int P;
+    P a[] = {P(10), P(20), P(30), P(40), P(50), P(60), P(70), P(80)};
+    const C c1(std::begin(a), std::end(a));
+    const C c2 = c1;
+    assert((c1 == c2));
+    assert(!(c1 != c2));
+  }
+  {
+    typedef std::unordered_set<int, std::hash<int>, std::equal_to<int>, min_allocator<int>> C;
+    typedef int P;
+    P a[] = {P(10), P(20), P(30), P(40), P(50), P(60), P(70), P(80)};
+    C c1(std::begin(a), std::end(a));
+    C c2 = c1;
+    c2.rehash(30);
+    assert((c1 == c2));
+    assert(!(c1 != c2));
+    c2.insert(P(90));
+    assert(!(c1 == c2));
+    assert((c1 != c2));
+    c1.insert(P(90));
+    assert((c1 == c2));
+    assert(!(c1 != c2));
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/unord/unord.set/equal_range.transparent.pass.cpp b/libcxx/test/std/containers/unord/unord.set/equal_range.transparent.pass.cpp
index 9844d83cda465..c1a1353b88790 100644
--- a/libcxx/test/std/containers/unord/unord.set/equal_range.transparent.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.set/equal_range.transparent.pass.cpp
@@ -21,44 +21,43 @@
 
 #include "test_transparent_unordered.h"
 
-int main(int, char**)
-{
-    using key_type = StoredType<int>;
-
-    {
-      // Make sure conversions don't happen for transparent non-final hasher and key_equal
-      using S = unord_set_type<std::unordered_set, transparent_hash, std::equal_to<>>;
-      test_transparent_equal_range<S>({1, 2});
-      test_transparent_equal_range<const S>({1, 2});
-    }
-
-    {
-      // Make sure conversions don't happen for transparent final hasher and key_equal
-      using S = unord_set_type<std::unordered_set, transparent_hash_final, transparent_equal_final>;
-      test_transparent_equal_range<S>({1, 2});
-      test_transparent_equal_range<const S>({1, 2});
-    }
-
-    {
-      // Make sure conversions do happen for non-transparent hasher
-      using S = unord_set_type<std::unordered_set, non_transparent_hash, std::equal_to<>>;
-      test_non_transparent_equal_range<S>({1, 2});
-      test_non_transparent_equal_range<const S>({1, 2});
-    }
-
-    {
-      // Make sure conversions do happen for non-transparent key_equal
-      using S = unord_set_type<std::unordered_set, transparent_hash, std::equal_to<key_type>>;
-      test_non_transparent_equal_range<S>({1, 2});
-      test_non_transparent_equal_range<const S>({1, 2});
-    }
-
-    {
-      // Make sure conversions do happen for both non-transparent hasher and key_equal
-      using S = unord_set_type<std::unordered_set, non_transparent_hash, std::equal_to<key_type>>;
-      test_non_transparent_equal_range<S>({1, 2});
-      test_non_transparent_equal_range<const S>({1, 2});
-    }
-
-    return 0;
+int main(int, char**) {
+  using key_type = StoredType<int>;
+
+  {
+    // Make sure conversions don't happen for transparent non-final hasher and key_equal
+    using S = unord_set_type<std::unordered_set, transparent_hash, std::equal_to<>>;
+    test_transparent_equal_range<S>({1, 2});
+    test_transparent_equal_range<const S>({1, 2});
+  }
+
+  {
+    // Make sure conversions don't happen for transparent final hasher and key_equal
+    using S = unord_set_type<std::unordered_set, transparent_hash_final, transparent_equal_final>;
+    test_transparent_equal_range<S>({1, 2});
+    test_transparent_equal_range<const S>({1, 2});
+  }
+
+  {
+    // Make sure conversions do happen for non-transparent hasher
+    using S = unord_set_type<std::unordered_set, non_transparent_hash, std::equal_to<>>;
+    test_non_transparent_equal_range<S>({1, 2});
+    test_non_transparent_equal_range<const S>({1, 2});
+  }
+
+  {
+    // Make sure conversions do happen for non-transparent key_equal
+    using S = unord_set_type<std::unordered_set, transparent_hash, std::equal_to<key_type>>;
+    test_non_transparent_equal_range<S>({1, 2});
+    test_non_transparent_equal_range<const S>({1, 2});
+  }
+
+  {
+    // Make sure conversions do happen for both non-transparent hasher and key_equal
+    using S = unord_set_type<std::unordered_set, non_transparent_hash, std::equal_to<key_type>>;
+    test_non_transparent_equal_range<S>({1, 2});
+    test_non_transparent_equal_range<const S>({1, 2});
+  }
+
+  return 0;
 }
diff --git a/libcxx/test/std/containers/unord/unord.set/equal_range_const.pass.cpp b/libcxx/test/std/containers/unord/unord.set/equal_range_const.pass.cpp
index 707b3721cb235..1689b83060747 100644
--- a/libcxx/test/std/containers/unord/unord.set/equal_range_const.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.set/equal_range_const.pass.cpp
@@ -20,63 +20,38 @@
 #include "test_macros.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef std::unordered_set<int> C;
-        typedef C::const_iterator I;
-        typedef int P;
-        P a[] =
-        {
-            P(10),
-            P(20),
-            P(30),
-            P(40),
-            P(50),
-            P(50),
-            P(50),
-            P(60),
-            P(70),
-            P(80)
-        };
-        const C c(std::begin(a), std::end(a));
-        std::pair<I, I> r = c.equal_range(30);
-        assert(std::distance(r.first, r.second) == 1);
-        assert(*r.first == 30);
-        r = c.equal_range(5);
-        assert(std::distance(r.first, r.second) == 0);
-        r = c.equal_range(50);
-        assert(std::distance(r.first, r.second) == 1);
-        assert(*r.first == 50);
-    }
+int main(int, char**) {
+  {
+    typedef std::unordered_set<int> C;
+    typedef C::const_iterator I;
+    typedef int P;
+    P a[] = {P(10), P(20), P(30), P(40), P(50), P(50), P(50), P(60), P(70), P(80)};
+    const C c(std::begin(a), std::end(a));
+    std::pair<I, I> r = c.equal_range(30);
+    assert(std::distance(r.first, r.second) == 1);
+    assert(*r.first == 30);
+    r = c.equal_range(5);
+    assert(std::distance(r.first, r.second) == 0);
+    r = c.equal_range(50);
+    assert(std::distance(r.first, r.second) == 1);
+    assert(*r.first == 50);
+  }
 #if TEST_STD_VER >= 11
-    {
-        typedef std::unordered_set<int, std::hash<int>, std::equal_to<int>, min_allocator<int>> C;
-        typedef C::const_iterator I;
-        typedef int P;
-        P a[] =
-        {
-            P(10),
-            P(20),
-            P(30),
-            P(40),
-            P(50),
-            P(50),
-            P(50),
-            P(60),
-            P(70),
-            P(80)
-        };
-        const C c(std::begin(a), std::end(a));
-        std::pair<I, I> r = c.equal_range(30);
-        assert(std::distance(r.first, r.second) == 1);
-        assert(*r.first == 30);
-        r = c.equal_range(5);
-        assert(std::distance(r.first, r.second) == 0);
-        r = c.equal_range(50);
-        assert(std::distance(r.first, r.second) == 1);
-        assert(*r.first == 50);
-    }
+  {
+    typedef std::unordered_set<int, std::hash<int>, std::equal_to<int>, min_allocator<int>> C;
+    typedef C::const_iterator I;
+    typedef int P;
+    P a[] = {P(10), P(20), P(30), P(40), P(50), P(50), P(50), P(60), P(70), P(80)};
+    const C c(std::begin(a), std::end(a));
+    std::pair<I, I> r = c.equal_range(30);
+    assert(std::distance(r.first, r.second) == 1);
+    assert(*r.first == 30);
+    r = c.equal_range(5);
+    assert(std::distance(r.first, r.second) == 0);
+    r = c.equal_range(50);
+    assert(std::distance(r.first, r.second) == 1);
+    assert(*r.first == 50);
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/unord/unord.set/equal_range_non_const.pass.cpp b/libcxx/test/std/containers/unord/unord.set/equal_range_non_const.pass.cpp
index a2872fedd2ce8..2b7acce993c3e 100644
--- a/libcxx/test/std/containers/unord/unord.set/equal_range_non_const.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.set/equal_range_non_const.pass.cpp
@@ -20,63 +20,38 @@
 #include "test_macros.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef std::unordered_set<int> C;
-        typedef C::iterator I;
-        typedef int P;
-        P a[] =
-        {
-            P(10),
-            P(20),
-            P(30),
-            P(40),
-            P(50),
-            P(50),
-            P(50),
-            P(60),
-            P(70),
-            P(80)
-        };
-        C c(std::begin(a), std::end(a));
-        std::pair<I, I> r = c.equal_range(30);
-        assert(std::distance(r.first, r.second) == 1);
-        assert(*r.first == 30);
-        r = c.equal_range(5);
-        assert(std::distance(r.first, r.second) == 0);
-        r = c.equal_range(50);
-        assert(std::distance(r.first, r.second) == 1);
-        assert(*r.first == 50);
-    }
+int main(int, char**) {
+  {
+    typedef std::unordered_set<int> C;
+    typedef C::iterator I;
+    typedef int P;
+    P a[] = {P(10), P(20), P(30), P(40), P(50), P(50), P(50), P(60), P(70), P(80)};
+    C c(std::begin(a), std::end(a));
+    std::pair<I, I> r = c.equal_range(30);
+    assert(std::distance(r.first, r.second) == 1);
+    assert(*r.first == 30);
+    r = c.equal_range(5);
+    assert(std::distance(r.first, r.second) == 0);
+    r = c.equal_range(50);
+    assert(std::distance(r.first, r.second) == 1);
+    assert(*r.first == 50);
+  }
 #if TEST_STD_VER >= 11
-    {
-        typedef std::unordered_set<int, std::hash<int>, std::equal_to<int>, min_allocator<int>> C;
-        typedef C::iterator I;
-        typedef int P;
-        P a[] =
-        {
-            P(10),
-            P(20),
-            P(30),
-            P(40),
-            P(50),
-            P(50),
-            P(50),
-            P(60),
-            P(70),
-            P(80)
-        };
-        C c(std::begin(a), std::end(a));
-        std::pair<I, I> r = c.equal_range(30);
-        assert(std::distance(r.first, r.second) == 1);
-        assert(*r.first == 30);
-        r = c.equal_range(5);
-        assert(std::distance(r.first, r.second) == 0);
-        r = c.equal_range(50);
-        assert(std::distance(r.first, r.second) == 1);
-        assert(*r.first == 50);
-    }
+  {
+    typedef std::unordered_set<int, std::hash<int>, std::equal_to<int>, min_allocator<int>> C;
+    typedef C::iterator I;
+    typedef int P;
+    P a[] = {P(10), P(20), P(30), P(40), P(50), P(50), P(50), P(60), P(70), P(80)};
+    C c(std::begin(a), std::end(a));
+    std::pair<I, I> r = c.equal_range(30);
+    assert(std::distance(r.first, r.second) == 1);
+    assert(*r.first == 30);
+    r = c.equal_range(5);
+    assert(std::distance(r.first, r.second) == 0);
+    r = c.equal_range(50);
+    assert(std::distance(r.first, r.second) == 1);
+    assert(*r.first == 50);
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/unord/unord.set/erase_const_iter.pass.cpp b/libcxx/test/std/containers/unord/unord.set/erase_const_iter.pass.cpp
index 4a66329d57177..46cdda9fb7696 100644
--- a/libcxx/test/std/containers/unord/unord.set/erase_const_iter.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.set/erase_const_iter.pass.cpp
@@ -20,80 +20,64 @@
 #include "test_macros.h"
 #include "min_allocator.h"
 
-struct TemplateConstructor
-{
-    template<typename T>
-    TemplateConstructor (const T&) {}
+struct TemplateConstructor {
+  template <typename T>
+  TemplateConstructor(const T&) {}
 };
 
 bool operator==(const TemplateConstructor&, const TemplateConstructor&) { return false; }
-struct Hash { std::size_t operator() (const TemplateConstructor &) const { return 0; } };
+struct Hash {
+  std::size_t operator()(const TemplateConstructor&) const { return 0; }
+};
 
-int main(int, char**)
-{
-    {
-        typedef std::unordered_set<int> C;
-        typedef int P;
-        P a[] =
-        {
-            P(1),
-            P(2),
-            P(3),
-            P(4),
-            P(1),
-            P(2)
-        };
-        C c(a, a + sizeof(a)/sizeof(a[0]));
-        C::const_iterator i = c.find(2);
-        C::const_iterator i_next = i;
-        ++i_next;
-        C::iterator j = c.erase(i);
-        assert(j == i_next);
+int main(int, char**) {
+  {
+    typedef std::unordered_set<int> C;
+    typedef int P;
+    P a[] = {P(1), P(2), P(3), P(4), P(1), P(2)};
+    C c(a, a + sizeof(a) / sizeof(a[0]));
+    C::const_iterator i      = c.find(2);
+    C::const_iterator i_next = i;
+    ++i_next;
+    C::iterator j = c.erase(i);
+    assert(j == i_next);
 
-        assert(c.size() == 3);
-        assert(c.count(1) == 1);
-        assert(c.count(3) == 1);
-        assert(c.count(4) == 1);
-    }
+    assert(c.size() == 3);
+    assert(c.count(1) == 1);
+    assert(c.count(3) == 1);
+    assert(c.count(4) == 1);
+  }
 #if TEST_STD_VER >= 11
-    {
-        typedef std::unordered_set<int, std::hash<int>, std::equal_to<int>, min_allocator<int>> C;
-        typedef int P;
-        P a[] =
-        {
-            P(1),
-            P(2),
-            P(3),
-            P(4),
-            P(1),
-            P(2)
-        };
-        C c(a, a + sizeof(a)/sizeof(a[0]));
-        C::const_iterator i = c.find(2);
-        C::const_iterator i_next = i;
-        ++i_next;
-        C::iterator j = c.erase(i);
-        assert(j == i_next);
+  {
+    typedef std::unordered_set<int, std::hash<int>, std::equal_to<int>, min_allocator<int>> C;
+    typedef int P;
+    P a[] = {P(1), P(2), P(3), P(4), P(1), P(2)};
+    C c(a, a + sizeof(a) / sizeof(a[0]));
+    C::const_iterator i      = c.find(2);
+    C::const_iterator i_next = i;
+    ++i_next;
+    C::iterator j = c.erase(i);
+    assert(j == i_next);
 
-        assert(c.size() == 3);
-        assert(c.count(1) == 1);
-        assert(c.count(3) == 1);
-        assert(c.count(4) == 1);
-    }
+    assert(c.size() == 3);
+    assert(c.count(1) == 1);
+    assert(c.count(3) == 1);
+    assert(c.count(4) == 1);
+  }
 #endif
 #if TEST_STD_VER >= 14
-    {
+  {
     //  This is LWG #2059
-        typedef TemplateConstructor T;
-        typedef std::unordered_set<T, Hash> C;
-        typedef C::iterator I;
+    typedef TemplateConstructor T;
+    typedef std::unordered_set<T, Hash> C;
+    typedef C::iterator I;
 
-        C m;
-        T a{0};
-        I it = m.find(a);
-        if (it != m.end())
-            m.erase(it);
-    }
+    C m;
+    T a{0};
+    I it = m.find(a);
+    if (it != m.end())
+      m.erase(it);
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/unord/unord.set/erase_if.pass.cpp b/libcxx/test/std/containers/unord/unord.set/erase_if.pass.cpp
index ba5f3bfe181fc..103d41972029d 100644
--- a/libcxx/test/std/containers/unord/unord.set/erase_if.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.set/erase_if.pass.cpp
@@ -23,12 +23,11 @@
 using Init = std::initializer_list<int>;
 
 template <typename M>
-M make (Init vals)
-{
-    M ret;
-    for (int v : vals)
-        ret.insert(v);
-    return ret;
+M make(Init vals) {
+  M ret;
+  for (int v : vals)
+    ret.insert(v);
+  return ret;
 }
 
 template <typename M, typename Pred>
@@ -41,41 +40,39 @@ void test0(Init vals, Pred p, Init expected, std::size_t expected_erased_count)
 }
 
 template <typename S>
-void test()
-{
-    auto is1 = [](auto v) { return v == 1;};
-    auto is2 = [](auto v) { return v == 2;};
-    auto is3 = [](auto v) { return v == 3;};
-    auto is4 = [](auto v) { return v == 4;};
-    auto True  = [](auto) { return true; };
-    auto False = [](auto) { return false; };
-
-    test0<S>({}, is1, {}, 0);
-
-    test0<S>({1}, is1, {}, 1);
-    test0<S>({1}, is2, {1}, 0);
-
-    test0<S>({1, 2}, is1, {2}, 1);
-    test0<S>({1, 2}, is2, {1}, 1);
-    test0<S>({1, 2}, is3, {1, 2}, 0);
-
-    test0<S>({1, 2, 3}, is1, {2, 3}, 1);
-    test0<S>({1, 2, 3}, is2, {1, 3}, 1);
-    test0<S>({1, 2, 3}, is3, {1, 2}, 1);
-    test0<S>({1, 2, 3}, is4, {1, 2, 3}, 0);
-
-    test0<S>({1, 2, 3}, True, {}, 3);
-    test0<S>({1, 2, 3}, False, {1, 2, 3}, 0);
+void test() {
+  auto is1   = [](auto v) { return v == 1; };
+  auto is2   = [](auto v) { return v == 2; };
+  auto is3   = [](auto v) { return v == 3; };
+  auto is4   = [](auto v) { return v == 4; };
+  auto True  = [](auto) { return true; };
+  auto False = [](auto) { return false; };
+
+  test0<S>({}, is1, {}, 0);
+
+  test0<S>({1}, is1, {}, 1);
+  test0<S>({1}, is2, {1}, 0);
+
+  test0<S>({1, 2}, is1, {2}, 1);
+  test0<S>({1, 2}, is2, {1}, 1);
+  test0<S>({1, 2}, is3, {1, 2}, 0);
+
+  test0<S>({1, 2, 3}, is1, {2, 3}, 1);
+  test0<S>({1, 2, 3}, is2, {1, 3}, 1);
+  test0<S>({1, 2, 3}, is3, {1, 2}, 1);
+  test0<S>({1, 2, 3}, is4, {1, 2, 3}, 0);
+
+  test0<S>({1, 2, 3}, True, {}, 3);
+  test0<S>({1, 2, 3}, False, {1, 2, 3}, 0);
 }
 
-int main(int, char**)
-{
-    test<std::unordered_set<int>>();
-    test<std::unordered_set<int, std::hash<int>, std::equal_to<int>, min_allocator<int>>> ();
-    test<std::unordered_set<int, std::hash<int>, std::equal_to<int>, test_allocator<int>>> ();
+int main(int, char**) {
+  test<std::unordered_set<int>>();
+  test<std::unordered_set<int, std::hash<int>, std::equal_to<int>, min_allocator<int>>>();
+  test<std::unordered_set<int, std::hash<int>, std::equal_to<int>, test_allocator<int>>>();
 
-    test<std::unordered_set<long>>();
-    test<std::unordered_set<double>>();
+  test<std::unordered_set<long>>();
+  test<std::unordered_set<double>>();
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/unord/unord.set/erase_key.pass.cpp b/libcxx/test/std/containers/unord/unord.set/erase_key.pass.cpp
index a50f56b0a3232..80a24ba8b77f8 100644
--- a/libcxx/test/std/containers/unord/unord.set/erase_key.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.set/erase_key.pass.cpp
@@ -23,154 +23,137 @@
 
 #if TEST_STD_VER >= 11
 template <typename Unordered>
-bool only_deletions ( const Unordered &whole, const Unordered &part ) {
-    typename Unordered::const_iterator w = whole.begin();
-    typename Unordered::const_iterator p = part.begin();
+bool only_deletions(const Unordered& whole, const Unordered& part) {
+  typename Unordered::const_iterator w = whole.begin();
+  typename Unordered::const_iterator p = part.begin();
 
-    while ( w != whole.end () && p != part.end()) {
-        if ( *w == *p )
-            p++;
-        w++;
-        }
+  while (w != whole.end() && p != part.end()) {
+    if (*w == *p)
+      p++;
+    w++;
+  }
 
-    return p == part.end();
+  return p == part.end();
 }
 #endif
 
-int main(int, char**)
-{
-    {
-        typedef std::unordered_set<int> C;
-        typedef int P;
-        P a[] =
-        {
-            P(1),
-            P(2),
-            P(3),
-            P(4),
-            P(1),
-            P(2)
-        };
-        C c(a, a + sizeof(a)/sizeof(a[0]));
-        assert(c.erase(5) == 0);
-        assert(c.size() == 4);
-        assert(c.count(1) == 1);
-        assert(c.count(2) == 1);
-        assert(c.count(3) == 1);
-        assert(c.count(4) == 1);
-
-        assert(c.erase(2) == 1);
-        assert(c.size() == 3);
-        assert(c.count(1) == 1);
-        assert(c.count(3) == 1);
-        assert(c.count(4) == 1);
-
-        assert(c.erase(2) == 0);
-        assert(c.size() == 3);
-        assert(c.count(1) == 1);
-        assert(c.count(3) == 1);
-        assert(c.count(4) == 1);
-
-        assert(c.erase(4) == 1);
-        assert(c.size() == 2);
-        assert(c.count(1) == 1);
-        assert(c.count(3) == 1);
-
-        assert(c.erase(4) == 0);
-        assert(c.size() == 2);
-        assert(c.count(1) == 1);
-        assert(c.count(3) == 1);
-
-        assert(c.erase(1) == 1);
-        assert(c.size() == 1);
-        assert(c.count(3) == 1);
-
-        assert(c.erase(1) == 0);
-        assert(c.size() == 1);
-        assert(c.count(3) == 1);
-
-        assert(c.erase(3) == 1);
-        assert(c.size() == 0);
-
-        assert(c.erase(3) == 0);
-        assert(c.size() == 0);
-    }
+int main(int, char**) {
+  {
+    typedef std::unordered_set<int> C;
+    typedef int P;
+    P a[] = {P(1), P(2), P(3), P(4), P(1), P(2)};
+    C c(a, a + sizeof(a) / sizeof(a[0]));
+    assert(c.erase(5) == 0);
+    assert(c.size() == 4);
+    assert(c.count(1) == 1);
+    assert(c.count(2) == 1);
+    assert(c.count(3) == 1);
+    assert(c.count(4) == 1);
+
+    assert(c.erase(2) == 1);
+    assert(c.size() == 3);
+    assert(c.count(1) == 1);
+    assert(c.count(3) == 1);
+    assert(c.count(4) == 1);
+
+    assert(c.erase(2) == 0);
+    assert(c.size() == 3);
+    assert(c.count(1) == 1);
+    assert(c.count(3) == 1);
+    assert(c.count(4) == 1);
+
+    assert(c.erase(4) == 1);
+    assert(c.size() == 2);
+    assert(c.count(1) == 1);
+    assert(c.count(3) == 1);
+
+    assert(c.erase(4) == 0);
+    assert(c.size() == 2);
+    assert(c.count(1) == 1);
+    assert(c.count(3) == 1);
+
+    assert(c.erase(1) == 1);
+    assert(c.size() == 1);
+    assert(c.count(3) == 1);
+
+    assert(c.erase(1) == 0);
+    assert(c.size() == 1);
+    assert(c.count(3) == 1);
+
+    assert(c.erase(3) == 1);
+    assert(c.size() == 0);
+
+    assert(c.erase(3) == 0);
+    assert(c.size() == 0);
+  }
 #if TEST_STD_VER >= 11
-    {
-        typedef std::unordered_set<int, std::hash<int>, std::equal_to<int>, min_allocator<int>> C;
-        typedef int P;
-        P a[] =
-        {
-            P(1),
-            P(2),
-            P(3),
-            P(4),
-            P(1),
-            P(2)
-        };
-        C c(a, a + sizeof(a)/sizeof(a[0]));
-        assert(c.erase(5) == 0);
-        assert(c.size() == 4);
-        assert(c.count(1) == 1);
-        assert(c.count(2) == 1);
-        assert(c.count(3) == 1);
-        assert(c.count(4) == 1);
-
-        assert(c.erase(2) == 1);
-        assert(c.size() == 3);
-        assert(c.count(1) == 1);
-        assert(c.count(3) == 1);
-        assert(c.count(4) == 1);
-
-        assert(c.erase(2) == 0);
-        assert(c.size() == 3);
-        assert(c.count(1) == 1);
-        assert(c.count(3) == 1);
-        assert(c.count(4) == 1);
-
-        assert(c.erase(4) == 1);
-        assert(c.size() == 2);
-        assert(c.count(1) == 1);
-        assert(c.count(3) == 1);
-
-        assert(c.erase(4) == 0);
-        assert(c.size() == 2);
-        assert(c.count(1) == 1);
-        assert(c.count(3) == 1);
-
-        assert(c.erase(1) == 1);
-        assert(c.size() == 1);
-        assert(c.count(3) == 1);
-
-        assert(c.erase(1) == 0);
-        assert(c.size() == 1);
-        assert(c.count(3) == 1);
-
-        assert(c.erase(3) == 1);
-        assert(c.size() == 0);
-
-        assert(c.erase(3) == 0);
-        assert(c.size() == 0);
-    }
-    {
+  {
+    typedef std::unordered_set<int, std::hash<int>, std::equal_to<int>, min_allocator<int>> C;
+    typedef int P;
+    P a[] = {P(1), P(2), P(3), P(4), P(1), P(2)};
+    C c(a, a + sizeof(a) / sizeof(a[0]));
+    assert(c.erase(5) == 0);
+    assert(c.size() == 4);
+    assert(c.count(1) == 1);
+    assert(c.count(2) == 1);
+    assert(c.count(3) == 1);
+    assert(c.count(4) == 1);
+
+    assert(c.erase(2) == 1);
+    assert(c.size() == 3);
+    assert(c.count(1) == 1);
+    assert(c.count(3) == 1);
+    assert(c.count(4) == 1);
+
+    assert(c.erase(2) == 0);
+    assert(c.size() == 3);
+    assert(c.count(1) == 1);
+    assert(c.count(3) == 1);
+    assert(c.count(4) == 1);
+
+    assert(c.erase(4) == 1);
+    assert(c.size() == 2);
+    assert(c.count(1) == 1);
+    assert(c.count(3) == 1);
+
+    assert(c.erase(4) == 0);
+    assert(c.size() == 2);
+    assert(c.count(1) == 1);
+    assert(c.count(3) == 1);
+
+    assert(c.erase(1) == 1);
+    assert(c.size() == 1);
+    assert(c.count(3) == 1);
+
+    assert(c.erase(1) == 0);
+    assert(c.size() == 1);
+    assert(c.count(3) == 1);
+
+    assert(c.erase(3) == 1);
+    assert(c.size() == 0);
+
+    assert(c.erase(3) == 0);
+    assert(c.size() == 0);
+  }
+  {
     typedef std::unordered_set<int> C;
     C m, m2;
-    for ( int i = 0; i < 10; ++i ) {
-        m.insert(i);
-        m2.insert(i);
-        }
+    for (int i = 0; i < 10; ++i) {
+      m.insert(i);
+      m2.insert(i);
+    }
 
     C::iterator i = m2.begin();
-    int ctr = 0;
+    int ctr       = 0;
     while (i != m2.end()) {
-        if (ctr++ % 2 == 0)
-            m2.erase(i++);
-        else
-            ++i;
-        }
-
-    assert (only_deletions (m, m2));
+      if (ctr++ % 2 == 0)
+        m2.erase(i++);
+      else
+        ++i;
     }
+
+    assert(only_deletions(m, m2));
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/unord/unord.set/erase_range.pass.cpp b/libcxx/test/std/containers/unord/unord.set/erase_range.pass.cpp
index 8e0110a8b263e..5fa6e4199f756 100644
--- a/libcxx/test/std/containers/unord/unord.set/erase_range.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.set/erase_range.pass.cpp
@@ -21,75 +21,58 @@
 #include "test_macros.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef std::unordered_set<int> C;
-        typedef int P;
-        P a[] =
-        {
-            P(1),
-            P(2),
-            P(3),
-            P(4),
-            P(1),
-            P(2)
-        };
-        C c(a, a + sizeof(a)/sizeof(a[0]));
-        C::const_iterator i = c.find(2);
-        C::const_iterator j = std::next(i);
-        C::iterator k = c.erase(i, i);
-        assert(k == i);
-        assert(c.size() == 4);
-        assert(c.count(1) == 1);
-        assert(c.count(2) == 1);
-        assert(c.count(3) == 1);
-        assert(c.count(4) == 1);
+int main(int, char**) {
+  {
+    typedef std::unordered_set<int> C;
+    typedef int P;
+    P a[] = {P(1), P(2), P(3), P(4), P(1), P(2)};
+    C c(a, a + sizeof(a) / sizeof(a[0]));
+    C::const_iterator i = c.find(2);
+    C::const_iterator j = std::next(i);
+    C::iterator k       = c.erase(i, i);
+    assert(k == i);
+    assert(c.size() == 4);
+    assert(c.count(1) == 1);
+    assert(c.count(2) == 1);
+    assert(c.count(3) == 1);
+    assert(c.count(4) == 1);
 
-        k = c.erase(i, j);
-        assert(c.size() == 3);
-        assert(c.count(1) == 1);
-        assert(c.count(3) == 1);
-        assert(c.count(4) == 1);
+    k = c.erase(i, j);
+    assert(c.size() == 3);
+    assert(c.count(1) == 1);
+    assert(c.count(3) == 1);
+    assert(c.count(4) == 1);
 
-        k = c.erase(c.cbegin(), c.cend());
-        assert(c.size() == 0);
-        assert(k == c.end());
-    }
+    k = c.erase(c.cbegin(), c.cend());
+    assert(c.size() == 0);
+    assert(k == c.end());
+  }
 #if TEST_STD_VER >= 11
-    {
-        typedef std::unordered_set<int, std::hash<int>, std::equal_to<int>, min_allocator<int>> C;
-        typedef int P;
-        P a[] =
-        {
-            P(1),
-            P(2),
-            P(3),
-            P(4),
-            P(1),
-            P(2)
-        };
-        C c(a, a + sizeof(a)/sizeof(a[0]));
-        C::const_iterator i = c.find(2);
-        C::const_iterator j = std::next(i);
-        C::iterator k = c.erase(i, i);
-        assert(k == i);
-        assert(c.size() == 4);
-        assert(c.count(1) == 1);
-        assert(c.count(2) == 1);
-        assert(c.count(3) == 1);
-        assert(c.count(4) == 1);
+  {
+    typedef std::unordered_set<int, std::hash<int>, std::equal_to<int>, min_allocator<int>> C;
+    typedef int P;
+    P a[] = {P(1), P(2), P(3), P(4), P(1), P(2)};
+    C c(a, a + sizeof(a) / sizeof(a[0]));
+    C::const_iterator i = c.find(2);
+    C::const_iterator j = std::next(i);
+    C::iterator k       = c.erase(i, i);
+    assert(k == i);
+    assert(c.size() == 4);
+    assert(c.count(1) == 1);
+    assert(c.count(2) == 1);
+    assert(c.count(3) == 1);
+    assert(c.count(4) == 1);
 
-        k = c.erase(i, j);
-        assert(c.size() == 3);
-        assert(c.count(1) == 1);
-        assert(c.count(3) == 1);
-        assert(c.count(4) == 1);
+    k = c.erase(i, j);
+    assert(c.size() == 3);
+    assert(c.count(1) == 1);
+    assert(c.count(3) == 1);
+    assert(c.count(4) == 1);
 
-        k = c.erase(c.cbegin(), c.cend());
-        assert(c.size() == 0);
-        assert(k == c.end());
-    }
+    k = c.erase(c.cbegin(), c.cend());
+    assert(c.size() == 0);
+    assert(k == c.end());
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/unord/unord.set/extract_iterator.pass.cpp b/libcxx/test/std/containers/unord/unord.set/extract_iterator.pass.cpp
index f6382502e6354..7f5a34d9cb64c 100644
--- a/libcxx/test/std/containers/unord/unord.set/extract_iterator.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.set/extract_iterator.pass.cpp
@@ -20,43 +20,40 @@
 #include "Counter.h"
 
 template <class Container>
-void test(Container& c)
-{
-    std::size_t sz = c.size();
-
-    for (auto first = c.cbegin(); first != c.cend();)
-    {
-        auto key_value = *first;
-        typename Container::node_type t = c.extract(first++);
-        --sz;
-        assert(t.value() == key_value);
-        assert(t.get_allocator() == c.get_allocator());
-        assert(sz == c.size());
-    }
-
-    assert(c.size() == 0);
+void test(Container& c) {
+  std::size_t sz = c.size();
+
+  for (auto first = c.cbegin(); first != c.cend();) {
+    auto key_value                  = *first;
+    typename Container::node_type t = c.extract(first++);
+    --sz;
+    assert(t.value() == key_value);
+    assert(t.get_allocator() == c.get_allocator());
+    assert(sz == c.size());
+  }
+
+  assert(c.size() == 0);
 }
 
-int main(int, char**)
-{
-    {
-        using set_type = std::unordered_set<int>;
-        set_type m = {1, 2, 3, 4, 5, 6};
-        test(m);
-    }
-
-    {
-        std::unordered_set<Counter<int>> m = {1, 2, 3, 4, 5, 6};
-        assert(Counter_base::gConstructed == 6);
-        test(m);
-        assert(Counter_base::gConstructed == 0);
-    }
-
-    {
-        using min_alloc_set = std::unordered_set<int, std::hash<int>, std::equal_to<int>, min_allocator<int>>;
-        min_alloc_set m = {1, 2, 3, 4, 5, 6};
-        test(m);
-    }
+int main(int, char**) {
+  {
+    using set_type = std::unordered_set<int>;
+    set_type m     = {1, 2, 3, 4, 5, 6};
+    test(m);
+  }
+
+  {
+    std::unordered_set<Counter<int>> m = {1, 2, 3, 4, 5, 6};
+    assert(Counter_base::gConstructed == 6);
+    test(m);
+    assert(Counter_base::gConstructed == 0);
+  }
+
+  {
+    using min_alloc_set = std::unordered_set<int, std::hash<int>, std::equal_to<int>, min_allocator<int>>;
+    min_alloc_set m     = {1, 2, 3, 4, 5, 6};
+    test(m);
+  }
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/unord/unord.set/extract_key.pass.cpp b/libcxx/test/std/containers/unord/unord.set/extract_key.pass.cpp
index 3bfb6c358c8b6..dac309787b75b 100644
--- a/libcxx/test/std/containers/unord/unord.set/extract_key.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.set/extract_key.pass.cpp
@@ -20,54 +20,50 @@
 #include "Counter.h"
 
 template <class Container, class KeyTypeIter>
-void test(Container& c, KeyTypeIter first, KeyTypeIter last)
-{
-    std::size_t sz = c.size();
-    assert((std::size_t)std::distance(first, last) == sz);
+void test(Container& c, KeyTypeIter first, KeyTypeIter last) {
+  std::size_t sz = c.size();
+  assert((std::size_t)std::distance(first, last) == sz);
 
-    for (KeyTypeIter copy = first; copy != last; ++copy)
-    {
-        typename Container::node_type t = c.extract(*copy);
-        assert(!t.empty());
-        --sz;
-        assert(t.value() == *copy);
-        assert(t.get_allocator() == c.get_allocator());
-        assert(sz == c.size());
-    }
+  for (KeyTypeIter copy = first; copy != last; ++copy) {
+    typename Container::node_type t = c.extract(*copy);
+    assert(!t.empty());
+    --sz;
+    assert(t.value() == *copy);
+    assert(t.get_allocator() == c.get_allocator());
+    assert(sz == c.size());
+  }
 
-    assert(c.size() == 0);
+  assert(c.size() == 0);
 
-    for (KeyTypeIter copy = first; copy != last; ++copy)
-    {
-        typename Container::node_type t = c.extract(*copy);
-        assert(t.empty());
-    }
+  for (KeyTypeIter copy = first; copy != last; ++copy) {
+    typename Container::node_type t = c.extract(*copy);
+    assert(t.empty());
+  }
 }
 
-int main(int, char**)
-{
-    {
-        std::unordered_set<int> m = {1, 2, 3, 4, 5, 6};
-        int keys[] = {1, 2, 3, 4, 5, 6};
-        test(m, std::begin(keys), std::end(keys));
-    }
+int main(int, char**) {
+  {
+    std::unordered_set<int> m = {1, 2, 3, 4, 5, 6};
+    int keys[]                = {1, 2, 3, 4, 5, 6};
+    test(m, std::begin(keys), std::end(keys));
+  }
 
+  {
+    std::unordered_set<Counter<int>> m = {1, 2, 3, 4, 5, 6};
     {
-        std::unordered_set<Counter<int>> m = {1, 2, 3, 4, 5, 6};
-        {
-            Counter<int> keys[] = {1, 2, 3, 4, 5, 6};
-            assert(Counter_base::gConstructed == 6+6);
-            test(m, std::begin(keys), std::end(keys));
-        }
-        assert(Counter_base::gConstructed == 0);
+      Counter<int> keys[] = {1, 2, 3, 4, 5, 6};
+      assert(Counter_base::gConstructed == 6 + 6);
+      test(m, std::begin(keys), std::end(keys));
     }
+    assert(Counter_base::gConstructed == 0);
+  }
 
-    {
-        using min_alloc_set = std::unordered_set<int, std::hash<int>, std::equal_to<int>, min_allocator<int>>;
-        min_alloc_set m = {1, 2, 3, 4, 5, 6};
-        int keys[] = {1, 2, 3, 4, 5, 6};
-        test(m, std::begin(keys), std::end(keys));
-    }
+  {
+    using min_alloc_set = std::unordered_set<int, std::hash<int>, std::equal_to<int>, min_allocator<int>>;
+    min_alloc_set m     = {1, 2, 3, 4, 5, 6};
+    int keys[]          = {1, 2, 3, 4, 5, 6};
+    test(m, std::begin(keys), std::end(keys));
+  }
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/unord/unord.set/find.transparent.pass.cpp b/libcxx/test/std/containers/unord/unord.set/find.transparent.pass.cpp
index 7510dd3514e65..503c2d07cd8f0 100644
--- a/libcxx/test/std/containers/unord/unord.set/find.transparent.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.set/find.transparent.pass.cpp
@@ -21,44 +21,43 @@
 
 #include "test_transparent_unordered.h"
 
-int main(int, char**)
-{
-    using key_type = StoredType<int>;
-
-    {
-      // Make sure conversions don't happen for transparent non-final hasher and key_equal
-      using S = unord_set_type<std::unordered_set, transparent_hash, std::equal_to<>>;
-      test_transparent_find<S>({1, 2});
-      test_transparent_find<const S>({1, 2});
-    }
-
-    {
-      // Make sure conversions don't happen for transparent final hasher and key_equal
-      using S = unord_set_type<std::unordered_set, transparent_hash_final, transparent_equal_final>;
-      test_transparent_find<S>({1, 2});
-      test_transparent_find<const S>({1, 2});
-    }
-
-    {
-      // Make sure conversions do happen for non-transparent hasher
-      using S = unord_set_type<std::unordered_set, non_transparent_hash, std::equal_to<>>;
-      test_non_transparent_find<S>({1, 2});
-      test_non_transparent_find<const S>({1, 2});
-    }
-
-    {
-      // Make sure conversions do happen for non-transparent key_equal
-      using S = unord_set_type<std::unordered_set, transparent_hash, std::equal_to<key_type>>;
-      test_non_transparent_find<S>({1, 2});
-      test_non_transparent_find<const S>({1, 2});
-    }
-
-    {
-      // Make sure conversions do happen for both non-transparent hasher and key_equal
-      using S = unord_set_type<std::unordered_set, non_transparent_hash, std::equal_to<key_type>>;
-      test_non_transparent_find<S>({1, 2});
-      test_non_transparent_find<const S>({1, 2});
-    }
-
-    return 0;
+int main(int, char**) {
+  using key_type = StoredType<int>;
+
+  {
+    // Make sure conversions don't happen for transparent non-final hasher and key_equal
+    using S = unord_set_type<std::unordered_set, transparent_hash, std::equal_to<>>;
+    test_transparent_find<S>({1, 2});
+    test_transparent_find<const S>({1, 2});
+  }
+
+  {
+    // Make sure conversions don't happen for transparent final hasher and key_equal
+    using S = unord_set_type<std::unordered_set, transparent_hash_final, transparent_equal_final>;
+    test_transparent_find<S>({1, 2});
+    test_transparent_find<const S>({1, 2});
+  }
+
+  {
+    // Make sure conversions do happen for non-transparent hasher
+    using S = unord_set_type<std::unordered_set, non_transparent_hash, std::equal_to<>>;
+    test_non_transparent_find<S>({1, 2});
+    test_non_transparent_find<const S>({1, 2});
+  }
+
+  {
+    // Make sure conversions do happen for non-transparent key_equal
+    using S = unord_set_type<std::unordered_set, transparent_hash, std::equal_to<key_type>>;
+    test_non_transparent_find<S>({1, 2});
+    test_non_transparent_find<const S>({1, 2});
+  }
+
+  {
+    // Make sure conversions do happen for both non-transparent hasher and key_equal
+    using S = unord_set_type<std::unordered_set, non_transparent_hash, std::equal_to<key_type>>;
+    test_non_transparent_find<S>({1, 2});
+    test_non_transparent_find<const S>({1, 2});
+  }
+
+  return 0;
 }
diff --git a/libcxx/test/std/containers/unord/unord.set/find_const.pass.cpp b/libcxx/test/std/containers/unord/unord.set/find_const.pass.cpp
index 96d17873ba697..16805b26edaa5 100644
--- a/libcxx/test/std/containers/unord/unord.set/find_const.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.set/find_const.pass.cpp
@@ -20,49 +20,28 @@
 #include "test_macros.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef std::unordered_set<int> C;
-        typedef int P;
-        P a[] =
-        {
-            P(10),
-            P(20),
-            P(30),
-            P(40),
-            P(50),
-            P(60),
-            P(70),
-            P(80)
-        };
-        const C c(std::begin(a), std::end(a));
-        C::const_iterator i = c.find(30);
-        assert(*i == 30);
-        i = c.find(5);
-        assert(i == c.cend());
-    }
+int main(int, char**) {
+  {
+    typedef std::unordered_set<int> C;
+    typedef int P;
+    P a[] = {P(10), P(20), P(30), P(40), P(50), P(60), P(70), P(80)};
+    const C c(std::begin(a), std::end(a));
+    C::const_iterator i = c.find(30);
+    assert(*i == 30);
+    i = c.find(5);
+    assert(i == c.cend());
+  }
 #if TEST_STD_VER >= 11
-    {
-        typedef std::unordered_set<int, std::hash<int>, std::equal_to<int>, min_allocator<int>> C;
-        typedef int P;
-        P a[] =
-        {
-            P(10),
-            P(20),
-            P(30),
-            P(40),
-            P(50),
-            P(60),
-            P(70),
-            P(80)
-        };
-        const C c(std::begin(a), std::end(a));
-        C::const_iterator i = c.find(30);
-        assert(*i == 30);
-        i = c.find(5);
-        assert(i == c.cend());
-    }
+  {
+    typedef std::unordered_set<int, std::hash<int>, std::equal_to<int>, min_allocator<int>> C;
+    typedef int P;
+    P a[] = {P(10), P(20), P(30), P(40), P(50), P(60), P(70), P(80)};
+    const C c(std::begin(a), std::end(a));
+    C::const_iterator i = c.find(30);
+    assert(*i == 30);
+    i = c.find(5);
+    assert(i == c.cend());
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/unord/unord.set/find_non_const.pass.cpp b/libcxx/test/std/containers/unord/unord.set/find_non_const.pass.cpp
index d49efd7eea414..40f41ab40732d 100644
--- a/libcxx/test/std/containers/unord/unord.set/find_non_const.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.set/find_non_const.pass.cpp
@@ -20,49 +20,28 @@
 #include "test_macros.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef std::unordered_set<int> C;
-        typedef int P;
-        P a[] =
-        {
-            P(10),
-            P(20),
-            P(30),
-            P(40),
-            P(50),
-            P(60),
-            P(70),
-            P(80)
-        };
-        C c(std::begin(a), std::end(a));
-        C::iterator i = c.find(30);
-        assert(*i == 30);
-        i = c.find(5);
-        assert(i == c.cend());
-    }
+int main(int, char**) {
+  {
+    typedef std::unordered_set<int> C;
+    typedef int P;
+    P a[] = {P(10), P(20), P(30), P(40), P(50), P(60), P(70), P(80)};
+    C c(std::begin(a), std::end(a));
+    C::iterator i = c.find(30);
+    assert(*i == 30);
+    i = c.find(5);
+    assert(i == c.cend());
+  }
 #if TEST_STD_VER >= 11
-    {
-        typedef std::unordered_set<int, std::hash<int>, std::equal_to<int>, min_allocator<int>> C;
-        typedef int P;
-        P a[] =
-        {
-            P(10),
-            P(20),
-            P(30),
-            P(40),
-            P(50),
-            P(60),
-            P(70),
-            P(80)
-        };
-        C c(std::begin(a), std::end(a));
-        C::iterator i = c.find(30);
-        assert(*i == 30);
-        i = c.find(5);
-        assert(i == c.cend());
-    }
+  {
+    typedef std::unordered_set<int, std::hash<int>, std::equal_to<int>, min_allocator<int>> C;
+    typedef int P;
+    P a[] = {P(10), P(20), P(30), P(40), P(50), P(60), P(70), P(80)};
+    C c(std::begin(a), std::end(a));
+    C::iterator i = c.find(30);
+    assert(*i == 30);
+    i = c.find(5);
+    assert(i == c.cend());
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/unord/unord.set/get_allocator.pass.cpp b/libcxx/test/std/containers/unord/unord.set/get_allocator.pass.cpp
index 1b0a6520da043..bd3bd00872801 100644
--- a/libcxx/test/std/containers/unord/unord.set/get_allocator.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.set/get_allocator.pass.cpp
@@ -19,18 +19,16 @@
 #include "test_macros.h"
 
 int main(int, char**) {
-    {
-        std::allocator<int> alloc;
-        const std::unordered_set<int> s(alloc);
-        assert(s.get_allocator() == alloc);
-    }
-    {
-        other_allocator<int> alloc(1);
-        const std::unordered_set<int, std::hash<int>,
-                                 std::equal_to<int>,
-                                 other_allocator<int> > s(alloc);
-        assert(s.get_allocator() == alloc);
-    }
+  {
+    std::allocator<int> alloc;
+    const std::unordered_set<int> s(alloc);
+    assert(s.get_allocator() == alloc);
+  }
+  {
+    other_allocator<int> alloc(1);
+    const std::unordered_set<int, std::hash<int>, std::equal_to<int>, other_allocator<int> > s(alloc);
+    assert(s.get_allocator() == alloc);
+  }
 
-    return 0;
+  return 0;
 }
diff --git a/libcxx/test/std/containers/unord/unord.set/incomplete.pass.cpp b/libcxx/test/std/containers/unord/unord.set/incomplete.pass.cpp
index d87d080555e23..cdd8bb3542abf 100644
--- a/libcxx/test/std/containers/unord/unord.set/incomplete.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.set/incomplete.pass.cpp
@@ -20,22 +20,22 @@
 template <class Tp>
 struct MyHash {
   MyHash() {}
-  std::size_t operator()(Tp const&) const {return 42;}
+  std::size_t operator()(Tp const&) const { return 42; }
 };
 
 struct A {
-    typedef std::unordered_set<A, MyHash<A> > Map;
-    Map m;
-    Map::iterator it;
-    Map::const_iterator cit;
-    Map::local_iterator lit;
-    Map::const_local_iterator clit;
+  typedef std::unordered_set<A, MyHash<A> > Map;
+  Map m;
+  Map::iterator it;
+  Map::const_iterator cit;
+  Map::local_iterator lit;
+  Map::const_local_iterator clit;
 };
 
 inline bool operator==(A const& L, A const& R) { return &L == &R; }
 
 int main(int, char**) {
-    A a;
+  A a;
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/unord/unord.set/insert_and_emplace_allocator_requirements.pass.cpp b/libcxx/test/std/containers/unord/unord.set/insert_and_emplace_allocator_requirements.pass.cpp
index 0ca0e935965f6..41eed705abaae 100644
--- a/libcxx/test/std/containers/unord/unord.set/insert_and_emplace_allocator_requirements.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.set/insert_and_emplace_allocator_requirements.pass.cpp
@@ -21,9 +21,7 @@
 #include "container_test_types.h"
 #include "../../set_allocator_requirement_test_templates.h"
 
-
-int main(int, char**)
-{
+int main(int, char**) {
   testSetInsert<TCT::unordered_set<> >();
   testSetEmplace<TCT::unordered_set<> >();
 
diff --git a/libcxx/test/std/containers/unord/unord.set/insert_const_lvalue.pass.cpp b/libcxx/test/std/containers/unord/unord.set/insert_const_lvalue.pass.cpp
index a0f5ed6c98cd9..15c617acfbe88 100644
--- a/libcxx/test/std/containers/unord/unord.set/insert_const_lvalue.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.set/insert_const_lvalue.pass.cpp
@@ -20,46 +20,43 @@
 #include "test_macros.h"
 #include "min_allocator.h"
 
-template<class Container>
-void do_insert_const_lvalue_test()
-{
-    typedef Container C;
-    typedef std::pair<typename C::iterator, bool> R;
-    typedef typename C::value_type VT;
-    C c;
-    const VT v1(3.5);
-    R r = c.insert(v1);
-    assert(c.size() == 1);
-    assert(*r.first == 3.5);
-    assert(r.second);
-
-    r = c.insert(v1);
-    assert(c.size() == 1);
-    assert(*r.first == 3.5);
-    assert(!r.second);
-
-    const VT v2(4.5);
-    r = c.insert(v2);
-    assert(c.size() == 2);
-    assert(*r.first == 4.5);
-    assert(r.second);
-
-    const VT v3(5.5);
-    r = c.insert(v3);
-    assert(c.size() == 3);
-    assert(*r.first == 5.5);
-    assert(r.second);
+template <class Container>
+void do_insert_const_lvalue_test() {
+  typedef Container C;
+  typedef std::pair<typename C::iterator, bool> R;
+  typedef typename C::value_type VT;
+  C c;
+  const VT v1(3.5);
+  R r = c.insert(v1);
+  assert(c.size() == 1);
+  assert(*r.first == 3.5);
+  assert(r.second);
+
+  r = c.insert(v1);
+  assert(c.size() == 1);
+  assert(*r.first == 3.5);
+  assert(!r.second);
+
+  const VT v2(4.5);
+  r = c.insert(v2);
+  assert(c.size() == 2);
+  assert(*r.first == 4.5);
+  assert(r.second);
+
+  const VT v3(5.5);
+  r = c.insert(v3);
+  assert(c.size() == 3);
+  assert(*r.first == 5.5);
+  assert(r.second);
 }
 
-int main(int, char**)
-{
-    do_insert_const_lvalue_test<std::unordered_set<double> >();
+int main(int, char**) {
+  do_insert_const_lvalue_test<std::unordered_set<double> >();
 #if TEST_STD_VER >= 11
-    {
-        typedef std::unordered_set<double, std::hash<double>,
-                                std::equal_to<double>, min_allocator<double>> C;
-        do_insert_const_lvalue_test<C>();
-    }
+  {
+    typedef std::unordered_set<double, std::hash<double>, std::equal_to<double>, min_allocator<double>> C;
+    do_insert_const_lvalue_test<C>();
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/unord/unord.set/insert_hint_const_lvalue.pass.cpp b/libcxx/test/std/containers/unord/unord.set/insert_hint_const_lvalue.pass.cpp
index 16a9766e5f6e0..45569c7d7a522 100644
--- a/libcxx/test/std/containers/unord/unord.set/insert_hint_const_lvalue.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.set/insert_hint_const_lvalue.pass.cpp
@@ -20,44 +20,41 @@
 #include "test_macros.h"
 #include "min_allocator.h"
 
-template<class Container>
-void do_insert_hint_const_lvalue_test()
-{
-    typedef Container C;
-    typedef typename C::iterator R;
-    typedef typename C::value_type VT;
-    C c;
-    typename C::const_iterator e = c.end();
-    const VT v1(3.5);
-    R r = c.insert(e, v1);
-    assert(c.size() == 1);
-    assert(*r == 3.5);
-
-    r = c.insert(r, v1);
-    assert(c.size() == 1);
-    assert(*r == 3.5);
-
-    const VT v2(4.5);
-    r = c.insert(c.end(), v2);
-    assert(c.size() == 2);
-    assert(*r == 4.5);
-
-    const VT v3(5.5);
-    r = c.insert(c.end(), v3);
-    assert(c.size() == 3);
-    assert(*r == 5.5);
+template <class Container>
+void do_insert_hint_const_lvalue_test() {
+  typedef Container C;
+  typedef typename C::iterator R;
+  typedef typename C::value_type VT;
+  C c;
+  typename C::const_iterator e = c.end();
+  const VT v1(3.5);
+  R r = c.insert(e, v1);
+  assert(c.size() == 1);
+  assert(*r == 3.5);
+
+  r = c.insert(r, v1);
+  assert(c.size() == 1);
+  assert(*r == 3.5);
+
+  const VT v2(4.5);
+  r = c.insert(c.end(), v2);
+  assert(c.size() == 2);
+  assert(*r == 4.5);
+
+  const VT v3(5.5);
+  r = c.insert(c.end(), v3);
+  assert(c.size() == 3);
+  assert(*r == 5.5);
 }
 
-int main(int, char**)
-{
-    do_insert_hint_const_lvalue_test<std::unordered_set<double> >();
+int main(int, char**) {
+  do_insert_hint_const_lvalue_test<std::unordered_set<double> >();
 #if TEST_STD_VER >= 11
-    {
-        typedef std::unordered_set<double, std::hash<double>,
-                                std::equal_to<double>, min_allocator<double>> C;
-        do_insert_hint_const_lvalue_test<C>();
-    }
+  {
+    typedef std::unordered_set<double, std::hash<double>, std::equal_to<double>, min_allocator<double>> C;
+    do_insert_hint_const_lvalue_test<C>();
+  }
 #endif
 
-    return 0;
+  return 0;
 }
diff --git a/libcxx/test/std/containers/unord/unord.set/insert_hint_rvalue.pass.cpp b/libcxx/test/std/containers/unord/unord.set/insert_hint_rvalue.pass.cpp
index de67a912dc92c..c8098c420eb84 100644
--- a/libcxx/test/std/containers/unord/unord.set/insert_hint_rvalue.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.set/insert_hint_rvalue.pass.cpp
@@ -21,99 +21,96 @@
 #include "MoveOnly.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef std::unordered_set<double> C;
-        typedef C::iterator R;
-        typedef double P;
-        C c;
-        typename C::const_iterator e = c.end();
-        R r = c.insert(e, P(3.5));
-        assert(c.size() == 1);
-        assert(*r == 3.5);
-
-        r = c.insert(r, P(3.5));
-        assert(c.size() == 1);
-        assert(*r == 3.5);
-
-        r = c.insert(c.end(), P(4.5));
-        assert(c.size() == 2);
-        assert(*r == 4.5);
-
-        r = c.insert(c.end(), P(5.5));
-        assert(c.size() == 3);
-        assert(*r == 5.5);
-    }
+int main(int, char**) {
+  {
+    typedef std::unordered_set<double> C;
+    typedef C::iterator R;
+    typedef double P;
+    C c;
+    typename C::const_iterator e = c.end();
+    R r                          = c.insert(e, P(3.5));
+    assert(c.size() == 1);
+    assert(*r == 3.5);
+
+    r = c.insert(r, P(3.5));
+    assert(c.size() == 1);
+    assert(*r == 3.5);
+
+    r = c.insert(c.end(), P(4.5));
+    assert(c.size() == 2);
+    assert(*r == 4.5);
+
+    r = c.insert(c.end(), P(5.5));
+    assert(c.size() == 3);
+    assert(*r == 5.5);
+  }
 #if TEST_STD_VER >= 11
-    {
-        typedef std::unordered_set<MoveOnly> C;
-        typedef C::iterator R;
-        typedef MoveOnly P;
-        C c;
-        typename C::const_iterator e = c.end();
-        R r = c.insert(e, P(3));
-        assert(c.size() == 1);
-        assert(*r == 3);
-
-        r = c.insert(r, P(3));
-        assert(c.size() == 1);
-        assert(*r == 3);
-
-        r = c.insert(c.end(), P(4));
-        assert(c.size() == 2);
-        assert(*r == 4);
-
-        r = c.insert(c.end(), P(5));
-        assert(c.size() == 3);
-        assert(*r == 5);
-    }
-    {
-        typedef std::unordered_set<double, std::hash<double>,
-                                std::equal_to<double>, min_allocator<double>> C;
-        typedef C::iterator R;
-        typedef double P;
-        C c;
-        typename C::const_iterator e = c.end();
-        R r = c.insert(e, P(3.5));
-        assert(c.size() == 1);
-        assert(*r == 3.5);
-
-        r = c.insert(r, P(3.5));
-        assert(c.size() == 1);
-        assert(*r == 3.5);
-
-        r = c.insert(c.end(), P(4.5));
-        assert(c.size() == 2);
-        assert(*r == 4.5);
-
-        r = c.insert(c.end(), P(5.5));
-        assert(c.size() == 3);
-        assert(*r == 5.5);
-    }
-    {
-        typedef std::unordered_set<MoveOnly, std::hash<MoveOnly>,
-                            std::equal_to<MoveOnly>, min_allocator<MoveOnly>> C;
-        typedef C::iterator R;
-        typedef MoveOnly P;
-        C c;
-        typename C::const_iterator e = c.end();
-        R r = c.insert(e, P(3));
-        assert(c.size() == 1);
-        assert(*r == 3);
-
-        r = c.insert(r, P(3));
-        assert(c.size() == 1);
-        assert(*r == 3);
-
-        r = c.insert(c.end(), P(4));
-        assert(c.size() == 2);
-        assert(*r == 4);
-
-        r = c.insert(c.end(), P(5));
-        assert(c.size() == 3);
-        assert(*r == 5);
-    }
+  {
+    typedef std::unordered_set<MoveOnly> C;
+    typedef C::iterator R;
+    typedef MoveOnly P;
+    C c;
+    typename C::const_iterator e = c.end();
+    R r                          = c.insert(e, P(3));
+    assert(c.size() == 1);
+    assert(*r == 3);
+
+    r = c.insert(r, P(3));
+    assert(c.size() == 1);
+    assert(*r == 3);
+
+    r = c.insert(c.end(), P(4));
+    assert(c.size() == 2);
+    assert(*r == 4);
+
+    r = c.insert(c.end(), P(5));
+    assert(c.size() == 3);
+    assert(*r == 5);
+  }
+  {
+    typedef std::unordered_set<double, std::hash<double>, std::equal_to<double>, min_allocator<double>> C;
+    typedef C::iterator R;
+    typedef double P;
+    C c;
+    typename C::const_iterator e = c.end();
+    R r                          = c.insert(e, P(3.5));
+    assert(c.size() == 1);
+    assert(*r == 3.5);
+
+    r = c.insert(r, P(3.5));
+    assert(c.size() == 1);
+    assert(*r == 3.5);
+
+    r = c.insert(c.end(), P(4.5));
+    assert(c.size() == 2);
+    assert(*r == 4.5);
+
+    r = c.insert(c.end(), P(5.5));
+    assert(c.size() == 3);
+    assert(*r == 5.5);
+  }
+  {
+    typedef std::unordered_set<MoveOnly, std::hash<MoveOnly>, std::equal_to<MoveOnly>, min_allocator<MoveOnly>> C;
+    typedef C::iterator R;
+    typedef MoveOnly P;
+    C c;
+    typename C::const_iterator e = c.end();
+    R r                          = c.insert(e, P(3));
+    assert(c.size() == 1);
+    assert(*r == 3);
+
+    r = c.insert(r, P(3));
+    assert(c.size() == 1);
+    assert(*r == 3);
+
+    r = c.insert(c.end(), P(4));
+    assert(c.size() == 2);
+    assert(*r == 4);
+
+    r = c.insert(c.end(), P(5));
+    assert(c.size() == 3);
+    assert(*r == 5);
+  }
 #endif // TEST_STD_VER >= 11
 
   return 0;
diff --git a/libcxx/test/std/containers/unord/unord.set/insert_init.pass.cpp b/libcxx/test/std/containers/unord/unord.set/insert_init.pass.cpp
index 63331efe387dd..56036caa9c445 100644
--- a/libcxx/test/std/containers/unord/unord.set/insert_init.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.set/insert_init.pass.cpp
@@ -23,49 +23,29 @@
 #include "test_iterators.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef std::unordered_set<int> C;
-        typedef int P;
-        C c;
-        c.insert(
-                    {
-                        P(1),
-                        P(2),
-                        P(3),
-                        P(4),
-                        P(1),
-                        P(2)
-                    }
-                );
-        assert(c.size() == 4);
-        assert(c.count(1) == 1);
-        assert(c.count(2) == 1);
-        assert(c.count(3) == 1);
-        assert(c.count(4) == 1);
-    }
-    {
-        typedef std::unordered_set<int, std::hash<int>,
-                                      std::equal_to<int>, min_allocator<int>> C;
-        typedef int P;
-        C c;
-        c.insert(
-                    {
-                        P(1),
-                        P(2),
-                        P(3),
-                        P(4),
-                        P(1),
-                        P(2)
-                    }
-                );
-        assert(c.size() == 4);
-        assert(c.count(1) == 1);
-        assert(c.count(2) == 1);
-        assert(c.count(3) == 1);
-        assert(c.count(4) == 1);
-    }
+int main(int, char**) {
+  {
+    typedef std::unordered_set<int> C;
+    typedef int P;
+    C c;
+    c.insert({P(1), P(2), P(3), P(4), P(1), P(2)});
+    assert(c.size() == 4);
+    assert(c.count(1) == 1);
+    assert(c.count(2) == 1);
+    assert(c.count(3) == 1);
+    assert(c.count(4) == 1);
+  }
+  {
+    typedef std::unordered_set<int, std::hash<int>, std::equal_to<int>, min_allocator<int>> C;
+    typedef int P;
+    C c;
+    c.insert({P(1), P(2), P(3), P(4), P(1), P(2)});
+    assert(c.size() == 4);
+    assert(c.count(1) == 1);
+    assert(c.count(2) == 1);
+    assert(c.count(3) == 1);
+    assert(c.count(4) == 1);
+  }
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/unord/unord.set/insert_iter_iter.pass.cpp b/libcxx/test/std/containers/unord/unord.set/insert_iter_iter.pass.cpp
index 451ee6767100a..a0592b2d4b510 100644
--- a/libcxx/test/std/containers/unord/unord.set/insert_iter_iter.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.set/insert_iter_iter.pass.cpp
@@ -22,50 +22,32 @@
 #include "test_iterators.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef std::unordered_set<int> C;
-        typedef int P;
-        P a[] =
-        {
-            P(1),
-            P(2),
-            P(3),
-            P(4),
-            P(1),
-            P(2)
-        };
-        C c;
-        c.insert(cpp17_input_iterator<P*>(a), cpp17_input_iterator<P*>(a + sizeof(a)/sizeof(a[0])));
-        assert(c.size() == 4);
-        assert(c.count(1) == 1);
-        assert(c.count(2) == 1);
-        assert(c.count(3) == 1);
-        assert(c.count(4) == 1);
-    }
+int main(int, char**) {
+  {
+    typedef std::unordered_set<int> C;
+    typedef int P;
+    P a[] = {P(1), P(2), P(3), P(4), P(1), P(2)};
+    C c;
+    c.insert(cpp17_input_iterator<P*>(a), cpp17_input_iterator<P*>(a + sizeof(a) / sizeof(a[0])));
+    assert(c.size() == 4);
+    assert(c.count(1) == 1);
+    assert(c.count(2) == 1);
+    assert(c.count(3) == 1);
+    assert(c.count(4) == 1);
+  }
 #if TEST_STD_VER >= 11
-    {
-        typedef std::unordered_set<int, std::hash<int>,
-                                      std::equal_to<int>, min_allocator<int>> C;
-        typedef int P;
-        P a[] =
-        {
-            P(1),
-            P(2),
-            P(3),
-            P(4),
-            P(1),
-            P(2)
-        };
-        C c;
-        c.insert(cpp17_input_iterator<P*>(a), cpp17_input_iterator<P*>(a + sizeof(a)/sizeof(a[0])));
-        assert(c.size() == 4);
-        assert(c.count(1) == 1);
-        assert(c.count(2) == 1);
-        assert(c.count(3) == 1);
-        assert(c.count(4) == 1);
-    }
+  {
+    typedef std::unordered_set<int, std::hash<int>, std::equal_to<int>, min_allocator<int>> C;
+    typedef int P;
+    P a[] = {P(1), P(2), P(3), P(4), P(1), P(2)};
+    C c;
+    c.insert(cpp17_input_iterator<P*>(a), cpp17_input_iterator<P*>(a + sizeof(a) / sizeof(a[0])));
+    assert(c.size() == 4);
+    assert(c.count(1) == 1);
+    assert(c.count(2) == 1);
+    assert(c.count(3) == 1);
+    assert(c.count(4) == 1);
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/unord/unord.set/insert_node_type.pass.cpp b/libcxx/test/std/containers/unord/unord.set/insert_node_type.pass.cpp
index 16f52f456fbff..5db9bcb8bd59c 100644
--- a/libcxx/test/std/containers/unord/unord.set/insert_node_type.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.set/insert_node_type.pass.cpp
@@ -42,69 +42,63 @@ void verify_insert_return_type(T&& t) {
 }
 
 template <class Container>
-typename Container::node_type
-node_factory(typename Container::key_type const& key)
-{
-    static Container c;
-    auto p = c.insert(key);
-    assert(p.second);
-    return c.extract(p.first);
+typename Container::node_type node_factory(typename Container::key_type const& key) {
+  static Container c;
+  auto p = c.insert(key);
+  assert(p.second);
+  return c.extract(p.first);
 }
 
 template <class Container>
-void test(Container& c)
-{
-    auto* nf = &node_factory<Container>;
-
-    for (int i = 0; i != 10; ++i)
-    {
-        typename Container::node_type node = nf(i);
-        assert(!node.empty());
-        typename Container::insert_return_type irt = c.insert(std::move(node));
-        assert(node.empty());
-        assert(irt.inserted);
-        assert(irt.node.empty());
-        assert(*irt.position == i);
-        verify_insert_return_type<Container>(irt);
-    }
-
-    assert(c.size() == 10);
-
-    { // Insert empty node.
-        typename Container::node_type def;
-        auto irt = c.insert(std::move(def));
-        assert(def.empty());
-        assert(!irt.inserted);
-        assert(irt.node.empty());
-        assert(irt.position == c.end());
-        verify_insert_return_type<Container>(irt);
-    }
-
-    { // Insert duplicate node.
-        typename Container::node_type dupl = nf(0);
-        auto irt = c.insert(std::move(dupl));
-        assert(dupl.empty());
-        assert(!irt.inserted);
-        assert(!irt.node.empty());
-        assert(irt.position == c.find(0));
-        assert(irt.node.value() == 0);
-        verify_insert_return_type<Container>(irt);
-    }
-
-    assert(c.size() == 10);
-
-    for (int i = 0; i != 10; ++i)
-    {
-        assert(c.count(i) == 1);
-    }
+void test(Container& c) {
+  auto* nf = &node_factory<Container>;
+
+  for (int i = 0; i != 10; ++i) {
+    typename Container::node_type node = nf(i);
+    assert(!node.empty());
+    typename Container::insert_return_type irt = c.insert(std::move(node));
+    assert(node.empty());
+    assert(irt.inserted);
+    assert(irt.node.empty());
+    assert(*irt.position == i);
+    verify_insert_return_type<Container>(irt);
+  }
+
+  assert(c.size() == 10);
+
+  { // Insert empty node.
+    typename Container::node_type def;
+    auto irt = c.insert(std::move(def));
+    assert(def.empty());
+    assert(!irt.inserted);
+    assert(irt.node.empty());
+    assert(irt.position == c.end());
+    verify_insert_return_type<Container>(irt);
+  }
+
+  { // Insert duplicate node.
+    typename Container::node_type dupl = nf(0);
+    auto irt                           = c.insert(std::move(dupl));
+    assert(dupl.empty());
+    assert(!irt.inserted);
+    assert(!irt.node.empty());
+    assert(irt.position == c.find(0));
+    assert(irt.node.value() == 0);
+    verify_insert_return_type<Container>(irt);
+  }
+
+  assert(c.size() == 10);
+
+  for (int i = 0; i != 10; ++i) {
+    assert(c.count(i) == 1);
+  }
 }
 
-int main(int, char**)
-{
-    std::unordered_set<int> m;
-    test(m);
-    std::unordered_set<int, std::hash<int>, std::equal_to<int>, min_allocator<int>> m2;
-    test(m2);
+int main(int, char**) {
+  std::unordered_set<int> m;
+  test(m);
+  std::unordered_set<int, std::hash<int>, std::equal_to<int>, min_allocator<int>> m2;
+  test(m2);
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/unord/unord.set/insert_node_type_hint.pass.cpp b/libcxx/test/std/containers/unord/unord.set/insert_node_type_hint.pass.cpp
index b4b3d8822592c..851aa7a932727 100644
--- a/libcxx/test/std/containers/unord/unord.set/insert_node_type_hint.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.set/insert_node_type_hint.pass.cpp
@@ -19,45 +19,39 @@
 #include "min_allocator.h"
 
 template <class Container>
-typename Container::node_type
-node_factory(typename Container::key_type const& key)
-{
-    static Container c;
-    auto p = c.insert(key);
-    assert(p.second);
-    return c.extract(p.first);
+typename Container::node_type node_factory(typename Container::key_type const& key) {
+  static Container c;
+  auto p = c.insert(key);
+  assert(p.second);
+  return c.extract(p.first);
 }
 
 template <class Container>
-void test(Container& c)
-{
-    auto* nf = &node_factory<Container>;
-
-    for (int i = 0; i != 10; ++i)
-    {
-        typename Container::node_type node = nf(i);
-        assert(!node.empty());
-        std::size_t prev = c.size();
-        auto it = c.insert(c.end(), std::move(node));
-        assert(node.empty());
-        assert(prev + 1 == c.size());
-        assert(*it == i);
-    }
-
-    assert(c.size() == 10);
-
-    for (int i = 0; i != 10; ++i)
-    {
-        assert(c.count(i) == 1);
-    }
+void test(Container& c) {
+  auto* nf = &node_factory<Container>;
+
+  for (int i = 0; i != 10; ++i) {
+    typename Container::node_type node = nf(i);
+    assert(!node.empty());
+    std::size_t prev = c.size();
+    auto it          = c.insert(c.end(), std::move(node));
+    assert(node.empty());
+    assert(prev + 1 == c.size());
+    assert(*it == i);
+  }
+
+  assert(c.size() == 10);
+
+  for (int i = 0; i != 10; ++i) {
+    assert(c.count(i) == 1);
+  }
 }
 
-int main(int, char**)
-{
-    std::unordered_set<int> m;
-    test(m);
-    std::unordered_set<int, std::hash<int>, std::equal_to<int>, min_allocator<int>> m2;
-    test(m2);
+int main(int, char**) {
+  std::unordered_set<int> m;
+  test(m);
+  std::unordered_set<int, std::hash<int>, std::equal_to<int>, min_allocator<int>> m2;
+  test(m2);
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/unord/unord.set/insert_rvalue.pass.cpp b/libcxx/test/std/containers/unord/unord.set/insert_rvalue.pass.cpp
index 9edab27404155..34e70f2701b33 100644
--- a/libcxx/test/std/containers/unord/unord.set/insert_rvalue.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.set/insert_rvalue.pass.cpp
@@ -21,111 +21,108 @@
 #include "MoveOnly.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef std::unordered_set<double> C;
-        typedef std::pair<C::iterator, bool> R;
-        typedef double P;
-        C c;
-        R r = c.insert(P(3.5));
-        assert(c.size() == 1);
-        assert(*r.first == 3.5);
-        assert(r.second);
-
-        r = c.insert(P(3.5));
-        assert(c.size() == 1);
-        assert(*r.first == 3.5);
-        assert(!r.second);
-
-        r = c.insert(P(4.5));
-        assert(c.size() == 2);
-        assert(*r.first == 4.5);
-        assert(r.second);
-
-        r = c.insert(P(5.5));
-        assert(c.size() == 3);
-        assert(*r.first == 5.5);
-        assert(r.second);
-    }
+int main(int, char**) {
+  {
+    typedef std::unordered_set<double> C;
+    typedef std::pair<C::iterator, bool> R;
+    typedef double P;
+    C c;
+    R r = c.insert(P(3.5));
+    assert(c.size() == 1);
+    assert(*r.first == 3.5);
+    assert(r.second);
+
+    r = c.insert(P(3.5));
+    assert(c.size() == 1);
+    assert(*r.first == 3.5);
+    assert(!r.second);
+
+    r = c.insert(P(4.5));
+    assert(c.size() == 2);
+    assert(*r.first == 4.5);
+    assert(r.second);
+
+    r = c.insert(P(5.5));
+    assert(c.size() == 3);
+    assert(*r.first == 5.5);
+    assert(r.second);
+  }
 #if TEST_STD_VER >= 11
-    {
-        typedef std::unordered_set<MoveOnly> C;
-        typedef std::pair<C::iterator, bool> R;
-        typedef MoveOnly P;
-        C c;
-        R r = c.insert(P(3));
-        assert(c.size() == 1);
-        assert(*r.first == 3);
-        assert(r.second);
-
-        r = c.insert(P(3));
-        assert(c.size() == 1);
-        assert(*r.first == 3);
-        assert(!r.second);
-
-        r = c.insert(P(4));
-        assert(c.size() == 2);
-        assert(*r.first == 4);
-        assert(r.second);
-
-        r = c.insert(P(5));
-        assert(c.size() == 3);
-        assert(*r.first == 5);
-        assert(r.second);
-    }
-    {
-        typedef std::unordered_set<double, std::hash<double>,
-                                std::equal_to<double>, min_allocator<double>> C;
-        typedef std::pair<C::iterator, bool> R;
-        typedef double P;
-        C c;
-        R r = c.insert(P(3.5));
-        assert(c.size() == 1);
-        assert(*r.first == 3.5);
-        assert(r.second);
-
-        r = c.insert(P(3.5));
-        assert(c.size() == 1);
-        assert(*r.first == 3.5);
-        assert(!r.second);
-
-        r = c.insert(P(4.5));
-        assert(c.size() == 2);
-        assert(*r.first == 4.5);
-        assert(r.second);
-
-        r = c.insert(P(5.5));
-        assert(c.size() == 3);
-        assert(*r.first == 5.5);
-        assert(r.second);
-    }
-    {
-        typedef std::unordered_set<MoveOnly, std::hash<MoveOnly>,
-                            std::equal_to<MoveOnly>, min_allocator<MoveOnly>> C;
-        typedef std::pair<C::iterator, bool> R;
-        typedef MoveOnly P;
-        C c;
-        R r = c.insert(P(3));
-        assert(c.size() == 1);
-        assert(*r.first == 3);
-        assert(r.second);
-
-        r = c.insert(P(3));
-        assert(c.size() == 1);
-        assert(*r.first == 3);
-        assert(!r.second);
-
-        r = c.insert(P(4));
-        assert(c.size() == 2);
-        assert(*r.first == 4);
-        assert(r.second);
-
-        r = c.insert(P(5));
-        assert(c.size() == 3);
-        assert(*r.first == 5);
-        assert(r.second);
-    }
+  {
+    typedef std::unordered_set<MoveOnly> C;
+    typedef std::pair<C::iterator, bool> R;
+    typedef MoveOnly P;
+    C c;
+    R r = c.insert(P(3));
+    assert(c.size() == 1);
+    assert(*r.first == 3);
+    assert(r.second);
+
+    r = c.insert(P(3));
+    assert(c.size() == 1);
+    assert(*r.first == 3);
+    assert(!r.second);
+
+    r = c.insert(P(4));
+    assert(c.size() == 2);
+    assert(*r.first == 4);
+    assert(r.second);
+
+    r = c.insert(P(5));
+    assert(c.size() == 3);
+    assert(*r.first == 5);
+    assert(r.second);
+  }
+  {
+    typedef std::unordered_set<double, std::hash<double>, std::equal_to<double>, min_allocator<double>> C;
+    typedef std::pair<C::iterator, bool> R;
+    typedef double P;
+    C c;
+    R r = c.insert(P(3.5));
+    assert(c.size() == 1);
+    assert(*r.first == 3.5);
+    assert(r.second);
+
+    r = c.insert(P(3.5));
+    assert(c.size() == 1);
+    assert(*r.first == 3.5);
+    assert(!r.second);
+
+    r = c.insert(P(4.5));
+    assert(c.size() == 2);
+    assert(*r.first == 4.5);
+    assert(r.second);
+
+    r = c.insert(P(5.5));
+    assert(c.size() == 3);
+    assert(*r.first == 5.5);
+    assert(r.second);
+  }
+  {
+    typedef std::unordered_set<MoveOnly, std::hash<MoveOnly>, std::equal_to<MoveOnly>, min_allocator<MoveOnly>> C;
+    typedef std::pair<C::iterator, bool> R;
+    typedef MoveOnly P;
+    C c;
+    R r = c.insert(P(3));
+    assert(c.size() == 1);
+    assert(*r.first == 3);
+    assert(r.second);
+
+    r = c.insert(P(3));
+    assert(c.size() == 1);
+    assert(*r.first == 3);
+    assert(!r.second);
+
+    r = c.insert(P(4));
+    assert(c.size() == 2);
+    assert(*r.first == 4);
+    assert(r.second);
+
+    r = c.insert(P(5));
+    assert(c.size() == 3);
+    assert(*r.first == 5);
+    assert(r.second);
+  }
 #endif // TEST_STD_VER >= 11
 
   return 0;
diff --git a/libcxx/test/std/containers/unord/unord.set/iterator.operators.addressof.compile.pass.cpp b/libcxx/test/std/containers/unord/unord.set/iterator.operators.addressof.compile.pass.cpp
index 457172d11f6bd..759077ec6ff39 100644
--- a/libcxx/test/std/containers/unord/unord.set/iterator.operators.addressof.compile.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.set/iterator.operators.addressof.compile.pass.cpp
@@ -31,14 +31,14 @@ void test() {
 
 void test() {
   {
-    using I = std::unordered_set<operator_hijacker>::iterator;
+    using I  = std::unordered_set<operator_hijacker>::iterator;
     using CI = std::unordered_set<operator_hijacker>::const_iterator;
     test<I, I>();
     test<CI, I>();
     test<CI, CI>();
   }
   {
-    using IL = std::unordered_set<operator_hijacker>::local_iterator;
+    using IL  = std::unordered_set<operator_hijacker>::local_iterator;
     using CIL = std::unordered_set<operator_hijacker>::const_local_iterator;
     test<IL, IL>();
     test<CIL, IL>();
diff --git a/libcxx/test/std/containers/unord/unord.set/iterator_concept_conformance.compile.pass.cpp b/libcxx/test/std/containers/unord/unord.set/iterator_concept_conformance.compile.pass.cpp
index 70490c828a11a..216b59655a826 100644
--- a/libcxx/test/std/containers/unord/unord.set/iterator_concept_conformance.compile.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.set/iterator_concept_conformance.compile.pass.cpp
@@ -16,11 +16,11 @@
 
 #include "test_macros.h"
 
-using iterator = std::unordered_set<int>::iterator;
-using const_iterator = std::unordered_set<int>::const_iterator;
-using local_iterator = std::unordered_set<int>::local_iterator;
+using iterator             = std::unordered_set<int>::iterator;
+using const_iterator       = std::unordered_set<int>::const_iterator;
+using local_iterator       = std::unordered_set<int>::local_iterator;
 using const_local_iterator = std::unordered_set<int>::const_local_iterator;
-using value_type = int;
+using value_type           = int;
 
 static_assert(std::forward_iterator<iterator>);
 LIBCPP_STATIC_ASSERT(!std::bidirectional_iterator<iterator>);
diff --git a/libcxx/test/std/containers/unord/unord.set/iterators.compile.fail.cpp b/libcxx/test/std/containers/unord/unord.set/iterators.compile.fail.cpp
index de5f88e792fed..bf4eee2640cfc 100644
--- a/libcxx/test/std/containers/unord/unord.set/iterators.compile.fail.cpp
+++ b/libcxx/test/std/containers/unord/unord.set/iterators.compile.fail.cpp
@@ -22,47 +22,30 @@
 #include <unordered_set>
 #include <cassert>
 
-int main(int, char**)
-{
-    {
-        typedef std::unordered_set<int> C;
-        typedef int P;
-        P a[] =
-        {
-            P(1),
-            P(2),
-            P(3),
-            P(4),
-            P(1),
-            P(2)
-        };
-        C c(a, a + sizeof(a)/sizeof(a[0]));
-        assert(c.bucket_count() >= 5);
-        assert(c.size() == 6);
-        assert(std::distance(c.begin(), c.end()) == c.size());
-        assert(std::distance(c.cbegin(), c.cend()) == c.size());
-        C::iterator i = c.begin();
-        assert(*i == 1);
-        *i = 2;
-    }
-    {
-        typedef std::unordered_set<int> C;
-        typedef int P;
-        P a[] =
-        {
-            P(1),
-            P(2),
-            P(3),
-            P(4),
-            P(1),
-            P(2)
-        };
-        const C c(a, a + sizeof(a)/sizeof(a[0]));
-        assert(c.bucket_count() >= 5);
-        assert(c.size() == 6);
-        assert(std::distance(c.begin(), c.end()) == c.size());
-        assert(std::distance(c.cbegin(), c.cend()) == c.size());
-    }
+int main(int, char**) {
+  {
+    typedef std::unordered_set<int> C;
+    typedef int P;
+    P a[] = {P(1), P(2), P(3), P(4), P(1), P(2)};
+    C c(a, a + sizeof(a) / sizeof(a[0]));
+    assert(c.bucket_count() >= 5);
+    assert(c.size() == 6);
+    assert(std::distance(c.begin(), c.end()) == c.size());
+    assert(std::distance(c.cbegin(), c.cend()) == c.size());
+    C::iterator i = c.begin();
+    assert(*i == 1);
+    *i = 2;
+  }
+  {
+    typedef std::unordered_set<int> C;
+    typedef int P;
+    P a[] = {P(1), P(2), P(3), P(4), P(1), P(2)};
+    const C c(a, a + sizeof(a) / sizeof(a[0]));
+    assert(c.bucket_count() >= 5);
+    assert(c.size() == 6);
+    assert(std::distance(c.begin(), c.end()) == c.size());
+    assert(std::distance(c.cbegin(), c.cend()) == c.size());
+  }
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/unord/unord.set/iterators.pass.cpp b/libcxx/test/std/containers/unord/unord.set/iterators.pass.cpp
index eb8459101b2a2..abf20f1946350 100644
--- a/libcxx/test/std/containers/unord/unord.set/iterators.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.set/iterators.pass.cpp
@@ -26,104 +26,69 @@
 #include "test_macros.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef std::unordered_set<int> C;
-        typedef int P;
-        P a[] =
-        {
-            P(1),
-            P(2),
-            P(3),
-            P(4),
-            P(1),
-            P(2)
-        };
-        C c(a, a + sizeof(a)/sizeof(a[0]));
-        assert(c.bucket_count() >= 5);
-        assert(c.size() == 4);
-        assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-        assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
-        C::iterator i;
-    }
-    {
-        typedef std::unordered_set<int> C;
-        typedef int P;
-        P a[] =
-        {
-            P(1),
-            P(2),
-            P(3),
-            P(4),
-            P(1),
-            P(2)
-        };
-        const C c(a, a + sizeof(a)/sizeof(a[0]));
-        assert(c.bucket_count() >= 5);
-        assert(c.size() == 4);
-        assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-        assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
-        C::const_iterator i;
-    }
+int main(int, char**) {
+  {
+    typedef std::unordered_set<int> C;
+    typedef int P;
+    P a[] = {P(1), P(2), P(3), P(4), P(1), P(2)};
+    C c(a, a + sizeof(a) / sizeof(a[0]));
+    assert(c.bucket_count() >= 5);
+    assert(c.size() == 4);
+    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
+    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
+    C::iterator i;
+  }
+  {
+    typedef std::unordered_set<int> C;
+    typedef int P;
+    P a[] = {P(1), P(2), P(3), P(4), P(1), P(2)};
+    const C c(a, a + sizeof(a) / sizeof(a[0]));
+    assert(c.bucket_count() >= 5);
+    assert(c.size() == 4);
+    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
+    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
+    C::const_iterator i;
+  }
 #if TEST_STD_VER >= 11
-    {
-        typedef std::unordered_set<int, std::hash<int>,
-                                      std::equal_to<int>, min_allocator<int>> C;
-        typedef int P;
-        P a[] =
-        {
-            P(1),
-            P(2),
-            P(3),
-            P(4),
-            P(1),
-            P(2)
-        };
-        C c(a, a + sizeof(a)/sizeof(a[0]));
-        assert(c.bucket_count() >= 5);
-        assert(c.size() == 4);
-        assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-        assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
-        C::iterator i;
-    }
-    {
-        typedef std::unordered_set<int, std::hash<int>,
-                                      std::equal_to<int>, min_allocator<int>> C;
-        typedef int P;
-        P a[] =
-        {
-            P(1),
-            P(2),
-            P(3),
-            P(4),
-            P(1),
-            P(2)
-        };
-        const C c(a, a + sizeof(a)/sizeof(a[0]));
-        assert(c.bucket_count() >= 5);
-        assert(c.size() == 4);
-        assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-        assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
-        C::const_iterator i;
-    }
+  {
+    typedef std::unordered_set<int, std::hash<int>, std::equal_to<int>, min_allocator<int>> C;
+    typedef int P;
+    P a[] = {P(1), P(2), P(3), P(4), P(1), P(2)};
+    C c(a, a + sizeof(a) / sizeof(a[0]));
+    assert(c.bucket_count() >= 5);
+    assert(c.size() == 4);
+    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
+    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
+    C::iterator i;
+  }
+  {
+    typedef std::unordered_set<int, std::hash<int>, std::equal_to<int>, min_allocator<int>> C;
+    typedef int P;
+    P a[] = {P(1), P(2), P(3), P(4), P(1), P(2)};
+    const C c(a, a + sizeof(a) / sizeof(a[0]));
+    assert(c.bucket_count() >= 5);
+    assert(c.size() == 4);
+    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
+    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
+    C::const_iterator i;
+  }
 #endif
 #if TEST_STD_VER > 11
-    { // N3644 testing
-        typedef std::unordered_set<int> C;
-        C::iterator ii1{}, ii2{};
-        C::iterator ii4 = ii1;
-        C::const_iterator cii{};
-        assert ( ii1 == ii2 );
-        assert ( ii1 == ii4 );
+  { // N3644 testing
+    typedef std::unordered_set<int> C;
+    C::iterator ii1{}, ii2{};
+    C::iterator ii4 = ii1;
+    C::const_iterator cii{};
+    assert(ii1 == ii2);
+    assert(ii1 == ii4);
 
-        assert (!(ii1 != ii2 ));
+    assert(!(ii1 != ii2));
 
-        assert ( (ii1 == cii ));
-        assert ( (cii == ii1 ));
-        assert (!(ii1 != cii ));
-        assert (!(cii != ii1 ));
-    }
+    assert((ii1 == cii));
+    assert((cii == ii1));
+    assert(!(ii1 != cii));
+    assert(!(cii != ii1));
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/unord/unord.set/load_factor.pass.cpp b/libcxx/test/std/containers/unord/unord.set/load_factor.pass.cpp
index 0952a4c3f2134..0e4294311237d 100644
--- a/libcxx/test/std/containers/unord/unord.set/load_factor.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.set/load_factor.pass.cpp
@@ -22,55 +22,32 @@
 #include "test_macros.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef std::unordered_set<int> C;
-        typedef int P;
-        P a[] =
-        {
-            P(10),
-            P(20),
-            P(30),
-            P(40),
-            P(50),
-            P(60),
-            P(70),
-            P(80)
-        };
-        const C c(std::begin(a), std::end(a));
-        assert(std::fabs(c.load_factor() - (float)c.size()/c.bucket_count()) < FLT_EPSILON);
-    }
-    {
-        typedef std::unordered_set<int> C;
-        const C c;
-        assert(c.load_factor() == 0);
-    }
+int main(int, char**) {
+  {
+    typedef std::unordered_set<int> C;
+    typedef int P;
+    P a[] = {P(10), P(20), P(30), P(40), P(50), P(60), P(70), P(80)};
+    const C c(std::begin(a), std::end(a));
+    assert(std::fabs(c.load_factor() - (float)c.size() / c.bucket_count()) < FLT_EPSILON);
+  }
+  {
+    typedef std::unordered_set<int> C;
+    const C c;
+    assert(c.load_factor() == 0);
+  }
 #if TEST_STD_VER >= 11
-    {
-        typedef std::unordered_set<int, std::hash<int>,
-                                      std::equal_to<int>, min_allocator<int>> C;
-        typedef int P;
-        P a[] =
-        {
-            P(10),
-            P(20),
-            P(30),
-            P(40),
-            P(50),
-            P(60),
-            P(70),
-            P(80)
-        };
-        const C c(std::begin(a), std::end(a));
-        assert(std::fabs(c.load_factor() - (float)c.size()/c.bucket_count()) < FLT_EPSILON);
-    }
-    {
-        typedef std::unordered_set<int, std::hash<int>,
-                                      std::equal_to<int>, min_allocator<int>> C;
-        const C c;
-        assert(c.load_factor() == 0);
-    }
+  {
+    typedef std::unordered_set<int, std::hash<int>, std::equal_to<int>, min_allocator<int>> C;
+    typedef int P;
+    P a[] = {P(10), P(20), P(30), P(40), P(50), P(60), P(70), P(80)};
+    const C c(std::begin(a), std::end(a));
+    assert(std::fabs(c.load_factor() - (float)c.size() / c.bucket_count()) < FLT_EPSILON);
+  }
+  {
+    typedef std::unordered_set<int, std::hash<int>, std::equal_to<int>, min_allocator<int>> C;
+    const C c;
+    assert(c.load_factor() == 0);
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/unord/unord.set/local_iterators.compile.fail.cpp b/libcxx/test/std/containers/unord/unord.set/local_iterators.compile.fail.cpp
index 7bacd2f6f7653..9f7aca15c3343 100644
--- a/libcxx/test/std/containers/unord/unord.set/local_iterators.compile.fail.cpp
+++ b/libcxx/test/std/containers/unord/unord.set/local_iterators.compile.fail.cpp
@@ -22,241 +22,208 @@
 #include <unordered_set>
 #include <cassert>
 
-int main(int, char**)
-{
-    {
-        typedef std::unordered_set<int> C;
-        typedef int P;
-        typedef C::local_iterator I;
-        P a[] =
-        {
-            P(1),
-            P(2),
-            P(3),
-            P(4),
-            P(1),
-            P(2)
-        };
-        C c(a, a + sizeof(a)/sizeof(a[0]));
-        assert(c.bucket_count() >= 5);
-        C::size_type b = c.bucket(0);
-        I i = c.begin(b);
-        I j = c.end(b);
-        assert(std::distance(i, j) == 0);
-
-        b = c.bucket(1);
-        i = c.begin(b);
-        j = c.end(b);
-        assert(std::distance(i, j) == 2);
-        assert(*i == 1);
-        ++i;
-        assert(*i == 1);
-        *i = 2;
-
-        b = c.bucket(2);
-        i = c.begin(b);
-        j = c.end(b);
-        assert(std::distance(i, j) == 2);
-        assert(*i == 2);
-        ++i;
-        assert(*i == 2);
-
-        b = c.bucket(3);
-        i = c.begin(b);
-        j = c.end(b);
-        assert(std::distance(i, j) == 1);
-        assert(*i == 3);
-
-        b = c.bucket(4);
-        i = c.begin(b);
-        j = c.end(b);
-        assert(std::distance(i, j) == 1);
-        assert(*i == 4);
-
-        b = c.bucket(5);
-        i = c.begin(b);
-        j = c.end(b);
-        assert(std::distance(i, j) == 0);
-
-        b = c.bucket(6);
-        i = c.begin(b);
-        j = c.end(b);
-        assert(std::distance(i, j) == 0);
-    }
-    {
-        typedef std::unordered_set<int> C;
-        typedef int P;
-        typedef C::const_local_iterator I;
-        P a[] =
-        {
-            P(1),
-            P(2),
-            P(3),
-            P(4),
-            P(1),
-            P(2)
-        };
-        const C c(a, a + sizeof(a)/sizeof(a[0]));
-        assert(c.bucket_count() >= 5);
-        C::size_type b = c.bucket(0);
-        I i = c.begin(b);
-        I j = c.end(b);
-        assert(std::distance(i, j) == 0);
-
-        b = c.bucket(1);
-        i = c.begin(b);
-        j = c.end(b);
-        assert(std::distance(i, j) == 2);
-        assert(*i == 1);
-        ++i;
-        assert(*i == 1);
-
-        b = c.bucket(2);
-        i = c.begin(b);
-        j = c.end(b);
-        assert(std::distance(i, j) == 2);
-        assert(*i == 2);
-        ++i;
-        assert(*i == 2);
-
-        b = c.bucket(3);
-        i = c.begin(b);
-        j = c.end(b);
-        assert(std::distance(i, j) == 1);
-        assert(*i == 3);
-
-        b = c.bucket(4);
-        i = c.begin(b);
-        j = c.end(b);
-        assert(std::distance(i, j) == 1);
-        assert(*i == 4);
-
-        b = c.bucket(5);
-        i = c.begin(b);
-        j = c.end(b);
-        assert(std::distance(i, j) == 0);
-
-        b = c.bucket(6);
-        i = c.begin(b);
-        j = c.end(b);
-        assert(std::distance(i, j) == 0);
-    }
-    {
-        typedef std::unordered_set<int> C;
-        typedef int P;
-        typedef C::const_local_iterator I;
-        P a[] =
-        {
-            P(1),
-            P(2),
-            P(3),
-            P(4),
-            P(1),
-            P(2)
-        };
-        C c(a, a + sizeof(a)/sizeof(a[0]));
-        assert(c.bucket_count() >= 5);
-        C::size_type b = c.bucket(0);
-        I i = c.cbegin(b);
-        I j = c.cend(b);
-        assert(std::distance(i, j) == 0);
-
-        b = c.bucket(1);
-        i = c.cbegin(b);
-        j = c.cend(b);
-        assert(std::distance(i, j) == 2);
-        assert(*i == 1);
-        ++i;
-        assert(*i == 1);
-
-        b = c.bucket(2);
-        i = c.cbegin(b);
-        j = c.cend(b);
-        assert(std::distance(i, j) == 2);
-        assert(*i == 2);
-        ++i;
-        assert(*i == 2);
-
-        b = c.bucket(3);
-        i = c.cbegin(b);
-        j = c.cend(b);
-        assert(std::distance(i, j) == 1);
-        assert(*i == 3);
-
-        b = c.bucket(4);
-        i = c.cbegin(b);
-        j = c.cend(b);
-        assert(std::distance(i, j) == 1);
-        assert(*i == 4);
-
-        b = c.bucket(5);
-        i = c.cbegin(b);
-        j = c.cend(b);
-        assert(std::distance(i, j) == 0);
-
-        b = c.bucket(6);
-        i = c.cbegin(b);
-        j = c.cend(b);
-        assert(std::distance(i, j) == 0);
-    }
-    {
-        typedef std::unordered_set<int> C;
-        typedef int P;
-        typedef C::const_local_iterator I;
-        P a[] =
-        {
-            P(1),
-            P(2),
-            P(3),
-            P(4),
-            P(1),
-            P(2)
-        };
-        const C c(a, a + sizeof(a)/sizeof(a[0]));
-        assert(c.bucket_count() >= 5);
-        C::size_type b = c.bucket(0);
-        I i = c.cbegin(b);
-        I j = c.cend(b);
-        assert(std::distance(i, j) == 0);
-
-        b = c.bucket(1);
-        i = c.cbegin(b);
-        j = c.cend(b);
-        assert(std::distance(i, j) == 2);
-        assert(*i == 1);
-        ++i;
-        assert(*i == 1);
-
-        b = c.bucket(2);
-        i = c.cbegin(b);
-        j = c.cend(b);
-        assert(std::distance(i, j) == 2);
-        assert(*i == 2);
-        ++i;
-        assert(*i == 2);
-
-        b = c.bucket(3);
-        i = c.cbegin(b);
-        j = c.cend(b);
-        assert(std::distance(i, j) == 1);
-        assert(*i == 3);
-
-        b = c.bucket(4);
-        i = c.cbegin(b);
-        j = c.cend(b);
-        assert(std::distance(i, j) == 1);
-        assert(*i == 4);
-
-        b = c.bucket(5);
-        i = c.cbegin(b);
-        j = c.cend(b);
-        assert(std::distance(i, j) == 0);
-
-        b = c.bucket(6);
-        i = c.cbegin(b);
-        j = c.cend(b);
-        assert(std::distance(i, j) == 0);
-    }
+int main(int, char**) {
+  {
+    typedef std::unordered_set<int> C;
+    typedef int P;
+    typedef C::local_iterator I;
+    P a[] = {P(1), P(2), P(3), P(4), P(1), P(2)};
+    C c(a, a + sizeof(a) / sizeof(a[0]));
+    assert(c.bucket_count() >= 5);
+    C::size_type b = c.bucket(0);
+    I i            = c.begin(b);
+    I j            = c.end(b);
+    assert(std::distance(i, j) == 0);
+
+    b = c.bucket(1);
+    i = c.begin(b);
+    j = c.end(b);
+    assert(std::distance(i, j) == 2);
+    assert(*i == 1);
+    ++i;
+    assert(*i == 1);
+    *i = 2;
+
+    b = c.bucket(2);
+    i = c.begin(b);
+    j = c.end(b);
+    assert(std::distance(i, j) == 2);
+    assert(*i == 2);
+    ++i;
+    assert(*i == 2);
+
+    b = c.bucket(3);
+    i = c.begin(b);
+    j = c.end(b);
+    assert(std::distance(i, j) == 1);
+    assert(*i == 3);
+
+    b = c.bucket(4);
+    i = c.begin(b);
+    j = c.end(b);
+    assert(std::distance(i, j) == 1);
+    assert(*i == 4);
+
+    b = c.bucket(5);
+    i = c.begin(b);
+    j = c.end(b);
+    assert(std::distance(i, j) == 0);
+
+    b = c.bucket(6);
+    i = c.begin(b);
+    j = c.end(b);
+    assert(std::distance(i, j) == 0);
+  }
+  {
+    typedef std::unordered_set<int> C;
+    typedef int P;
+    typedef C::const_local_iterator I;
+    P a[] = {P(1), P(2), P(3), P(4), P(1), P(2)};
+    const C c(a, a + sizeof(a) / sizeof(a[0]));
+    assert(c.bucket_count() >= 5);
+    C::size_type b = c.bucket(0);
+    I i            = c.begin(b);
+    I j            = c.end(b);
+    assert(std::distance(i, j) == 0);
+
+    b = c.bucket(1);
+    i = c.begin(b);
+    j = c.end(b);
+    assert(std::distance(i, j) == 2);
+    assert(*i == 1);
+    ++i;
+    assert(*i == 1);
+
+    b = c.bucket(2);
+    i = c.begin(b);
+    j = c.end(b);
+    assert(std::distance(i, j) == 2);
+    assert(*i == 2);
+    ++i;
+    assert(*i == 2);
+
+    b = c.bucket(3);
+    i = c.begin(b);
+    j = c.end(b);
+    assert(std::distance(i, j) == 1);
+    assert(*i == 3);
+
+    b = c.bucket(4);
+    i = c.begin(b);
+    j = c.end(b);
+    assert(std::distance(i, j) == 1);
+    assert(*i == 4);
+
+    b = c.bucket(5);
+    i = c.begin(b);
+    j = c.end(b);
+    assert(std::distance(i, j) == 0);
+
+    b = c.bucket(6);
+    i = c.begin(b);
+    j = c.end(b);
+    assert(std::distance(i, j) == 0);
+  }
+  {
+    typedef std::unordered_set<int> C;
+    typedef int P;
+    typedef C::const_local_iterator I;
+    P a[] = {P(1), P(2), P(3), P(4), P(1), P(2)};
+    C c(a, a + sizeof(a) / sizeof(a[0]));
+    assert(c.bucket_count() >= 5);
+    C::size_type b = c.bucket(0);
+    I i            = c.cbegin(b);
+    I j            = c.cend(b);
+    assert(std::distance(i, j) == 0);
+
+    b = c.bucket(1);
+    i = c.cbegin(b);
+    j = c.cend(b);
+    assert(std::distance(i, j) == 2);
+    assert(*i == 1);
+    ++i;
+    assert(*i == 1);
+
+    b = c.bucket(2);
+    i = c.cbegin(b);
+    j = c.cend(b);
+    assert(std::distance(i, j) == 2);
+    assert(*i == 2);
+    ++i;
+    assert(*i == 2);
+
+    b = c.bucket(3);
+    i = c.cbegin(b);
+    j = c.cend(b);
+    assert(std::distance(i, j) == 1);
+    assert(*i == 3);
+
+    b = c.bucket(4);
+    i = c.cbegin(b);
+    j = c.cend(b);
+    assert(std::distance(i, j) == 1);
+    assert(*i == 4);
+
+    b = c.bucket(5);
+    i = c.cbegin(b);
+    j = c.cend(b);
+    assert(std::distance(i, j) == 0);
+
+    b = c.bucket(6);
+    i = c.cbegin(b);
+    j = c.cend(b);
+    assert(std::distance(i, j) == 0);
+  }
+  {
+    typedef std::unordered_set<int> C;
+    typedef int P;
+    typedef C::const_local_iterator I;
+    P a[] = {P(1), P(2), P(3), P(4), P(1), P(2)};
+    const C c(a, a + sizeof(a) / sizeof(a[0]));
+    assert(c.bucket_count() >= 5);
+    C::size_type b = c.bucket(0);
+    I i            = c.cbegin(b);
+    I j            = c.cend(b);
+    assert(std::distance(i, j) == 0);
+
+    b = c.bucket(1);
+    i = c.cbegin(b);
+    j = c.cend(b);
+    assert(std::distance(i, j) == 2);
+    assert(*i == 1);
+    ++i;
+    assert(*i == 1);
+
+    b = c.bucket(2);
+    i = c.cbegin(b);
+    j = c.cend(b);
+    assert(std::distance(i, j) == 2);
+    assert(*i == 2);
+    ++i;
+    assert(*i == 2);
+
+    b = c.bucket(3);
+    i = c.cbegin(b);
+    j = c.cend(b);
+    assert(std::distance(i, j) == 1);
+    assert(*i == 3);
+
+    b = c.bucket(4);
+    i = c.cbegin(b);
+    j = c.cend(b);
+    assert(std::distance(i, j) == 1);
+    assert(*i == 4);
+
+    b = c.bucket(5);
+    i = c.cbegin(b);
+    j = c.cend(b);
+    assert(std::distance(i, j) == 0);
+
+    b = c.bucket(6);
+    i = c.cbegin(b);
+    j = c.cend(b);
+    assert(std::distance(i, j) == 0);
+  }
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/unord/unord.set/local_iterators.pass.cpp b/libcxx/test/std/containers/unord/unord.set/local_iterators.pass.cpp
index adc0164947bb9..8c9b5722efa0c 100644
--- a/libcxx/test/std/containers/unord/unord.set/local_iterators.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.set/local_iterators.pass.cpp
@@ -25,365 +25,296 @@
 #include "test_macros.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef std::unordered_set<int> C;
-        typedef int P;
-        typedef C::local_iterator I;
-        P a[] =
-        {
-            P(1),
-            P(2),
-            P(3),
-            P(4),
-            P(1),
-            P(2)
-        };
-        C c(a, a + sizeof(a)/sizeof(a[0]));
-        assert(c.bucket_count() >= 5);
-        C::size_type b = c.bucket(0);
-        I i = c.begin(b);
-        I j = c.end(b);
-        assert(std::distance(i, j) == 0);
-
-        b = c.bucket(1);
-        i = c.begin(b);
-        j = c.end(b);
-        assert(std::distance(i, j) == 1);
-        assert(*i == 1);
-
-        b = c.bucket(2);
-        i = c.begin(b);
-        j = c.end(b);
-        assert(std::distance(i, j) == 1);
-        assert(*i == 2);
-
-        b = c.bucket(3);
-        i = c.begin(b);
-        j = c.end(b);
-        assert(std::distance(i, j) == 1);
-        assert(*i == 3);
-
-        b = c.bucket(4);
-        i = c.begin(b);
-        j = c.end(b);
-        assert(std::distance(i, j) == 1);
-        assert(*i == 4);
-    }
-    {
-        typedef std::unordered_set<int> C;
-        typedef int P;
-        typedef C::const_local_iterator I;
-        P a[] =
-        {
-            P(1),
-            P(2),
-            P(3),
-            P(4),
-            P(1),
-            P(2)
-        };
-        const C c(a, a + sizeof(a)/sizeof(a[0]));
-        assert(c.bucket_count() >= 5);
-        C::size_type b = c.bucket(0);
-        I i = c.begin(b);
-        I j = c.end(b);
-        assert(std::distance(i, j) == 0);
-
-        b = c.bucket(1);
-        i = c.begin(b);
-        j = c.end(b);
-        assert(std::distance(i, j) == 1);
-        assert(*i == 1);
-
-        b = c.bucket(2);
-        i = c.begin(b);
-        j = c.end(b);
-        assert(std::distance(i, j) == 1);
-        assert(*i == 2);
-
-        b = c.bucket(3);
-        i = c.begin(b);
-        j = c.end(b);
-        assert(std::distance(i, j) == 1);
-        assert(*i == 3);
-
-        b = c.bucket(4);
-        i = c.begin(b);
-        j = c.end(b);
-        assert(std::distance(i, j) == 1);
-        assert(*i == 4);
-    }
-    {
-        typedef std::unordered_set<int> C;
-        typedef int P;
-        typedef C::const_local_iterator I;
-        P a[] =
-        {
-            P(1),
-            P(2),
-            P(3),
-            P(4),
-            P(1),
-            P(2)
-        };
-        C c(a, a + sizeof(a)/sizeof(a[0]));
-        assert(c.bucket_count() >= 5);
-        C::size_type b = c.bucket(0);
-        I i = c.cbegin(b);
-        I j = c.cend(b);
-        assert(std::distance(i, j) == 0);
-
-        b = c.bucket(1);
-        i = c.cbegin(b);
-        j = c.cend(b);
-        assert(std::distance(i, j) == 1);
-        assert(*i == 1);
-
-        b = c.bucket(2);
-        i = c.cbegin(b);
-        j = c.cend(b);
-        assert(std::distance(i, j) == 1);
-        assert(*i == 2);
-
-        b = c.bucket(3);
-        i = c.cbegin(b);
-        j = c.cend(b);
-        assert(std::distance(i, j) == 1);
-        assert(*i == 3);
-
-        b = c.bucket(4);
-        i = c.cbegin(b);
-        j = c.cend(b);
-        assert(std::distance(i, j) == 1);
-        assert(*i == 4);
-    }
-    {
-        typedef std::unordered_set<int> C;
-        typedef int P;
-        typedef C::const_local_iterator I;
-        P a[] =
-        {
-            P(1),
-            P(2),
-            P(3),
-            P(4),
-            P(1),
-            P(2)
-        };
-        const C c(a, a + sizeof(a)/sizeof(a[0]));
-        assert(c.bucket_count() >= 5);
-        C::size_type b = c.bucket(0);
-        I i = c.cbegin(b);
-        I j = c.cend(b);
-        assert(std::distance(i, j) == 0);
-
-        b = c.bucket(1);
-        i = c.cbegin(b);
-        j = c.cend(b);
-        assert(std::distance(i, j) == 1);
-        assert(*i == 1);
-
-        b = c.bucket(2);
-        i = c.cbegin(b);
-        j = c.cend(b);
-        assert(std::distance(i, j) == 1);
-        assert(*i == 2);
-
-        b = c.bucket(3);
-        i = c.cbegin(b);
-        j = c.cend(b);
-        assert(std::distance(i, j) == 1);
-        assert(*i == 3);
-
-        b = c.bucket(4);
-        i = c.cbegin(b);
-        j = c.cend(b);
-        assert(std::distance(i, j) == 1);
-        assert(*i == 4);
-    }
+int main(int, char**) {
+  {
+    typedef std::unordered_set<int> C;
+    typedef int P;
+    typedef C::local_iterator I;
+    P a[] = {P(1), P(2), P(3), P(4), P(1), P(2)};
+    C c(a, a + sizeof(a) / sizeof(a[0]));
+    assert(c.bucket_count() >= 5);
+    C::size_type b = c.bucket(0);
+    I i            = c.begin(b);
+    I j            = c.end(b);
+    assert(std::distance(i, j) == 0);
+
+    b = c.bucket(1);
+    i = c.begin(b);
+    j = c.end(b);
+    assert(std::distance(i, j) == 1);
+    assert(*i == 1);
+
+    b = c.bucket(2);
+    i = c.begin(b);
+    j = c.end(b);
+    assert(std::distance(i, j) == 1);
+    assert(*i == 2);
+
+    b = c.bucket(3);
+    i = c.begin(b);
+    j = c.end(b);
+    assert(std::distance(i, j) == 1);
+    assert(*i == 3);
+
+    b = c.bucket(4);
+    i = c.begin(b);
+    j = c.end(b);
+    assert(std::distance(i, j) == 1);
+    assert(*i == 4);
+  }
+  {
+    typedef std::unordered_set<int> C;
+    typedef int P;
+    typedef C::const_local_iterator I;
+    P a[] = {P(1), P(2), P(3), P(4), P(1), P(2)};
+    const C c(a, a + sizeof(a) / sizeof(a[0]));
+    assert(c.bucket_count() >= 5);
+    C::size_type b = c.bucket(0);
+    I i            = c.begin(b);
+    I j            = c.end(b);
+    assert(std::distance(i, j) == 0);
+
+    b = c.bucket(1);
+    i = c.begin(b);
+    j = c.end(b);
+    assert(std::distance(i, j) == 1);
+    assert(*i == 1);
+
+    b = c.bucket(2);
+    i = c.begin(b);
+    j = c.end(b);
+    assert(std::distance(i, j) == 1);
+    assert(*i == 2);
+
+    b = c.bucket(3);
+    i = c.begin(b);
+    j = c.end(b);
+    assert(std::distance(i, j) == 1);
+    assert(*i == 3);
+
+    b = c.bucket(4);
+    i = c.begin(b);
+    j = c.end(b);
+    assert(std::distance(i, j) == 1);
+    assert(*i == 4);
+  }
+  {
+    typedef std::unordered_set<int> C;
+    typedef int P;
+    typedef C::const_local_iterator I;
+    P a[] = {P(1), P(2), P(3), P(4), P(1), P(2)};
+    C c(a, a + sizeof(a) / sizeof(a[0]));
+    assert(c.bucket_count() >= 5);
+    C::size_type b = c.bucket(0);
+    I i            = c.cbegin(b);
+    I j            = c.cend(b);
+    assert(std::distance(i, j) == 0);
+
+    b = c.bucket(1);
+    i = c.cbegin(b);
+    j = c.cend(b);
+    assert(std::distance(i, j) == 1);
+    assert(*i == 1);
+
+    b = c.bucket(2);
+    i = c.cbegin(b);
+    j = c.cend(b);
+    assert(std::distance(i, j) == 1);
+    assert(*i == 2);
+
+    b = c.bucket(3);
+    i = c.cbegin(b);
+    j = c.cend(b);
+    assert(std::distance(i, j) == 1);
+    assert(*i == 3);
+
+    b = c.bucket(4);
+    i = c.cbegin(b);
+    j = c.cend(b);
+    assert(std::distance(i, j) == 1);
+    assert(*i == 4);
+  }
+  {
+    typedef std::unordered_set<int> C;
+    typedef int P;
+    typedef C::const_local_iterator I;
+    P a[] = {P(1), P(2), P(3), P(4), P(1), P(2)};
+    const C c(a, a + sizeof(a) / sizeof(a[0]));
+    assert(c.bucket_count() >= 5);
+    C::size_type b = c.bucket(0);
+    I i            = c.cbegin(b);
+    I j            = c.cend(b);
+    assert(std::distance(i, j) == 0);
+
+    b = c.bucket(1);
+    i = c.cbegin(b);
+    j = c.cend(b);
+    assert(std::distance(i, j) == 1);
+    assert(*i == 1);
+
+    b = c.bucket(2);
+    i = c.cbegin(b);
+    j = c.cend(b);
+    assert(std::distance(i, j) == 1);
+    assert(*i == 2);
+
+    b = c.bucket(3);
+    i = c.cbegin(b);
+    j = c.cend(b);
+    assert(std::distance(i, j) == 1);
+    assert(*i == 3);
+
+    b = c.bucket(4);
+    i = c.cbegin(b);
+    j = c.cend(b);
+    assert(std::distance(i, j) == 1);
+    assert(*i == 4);
+  }
 #if TEST_STD_VER >= 11
-    {
-        typedef std::unordered_set<int, std::hash<int>,
-                                      std::equal_to<int>, min_allocator<int>> C;
-        typedef int P;
-        typedef C::local_iterator I;
-        P a[] =
-        {
-            P(1),
-            P(2),
-            P(3),
-            P(4),
-            P(1),
-            P(2)
-        };
-        C c(a, a + sizeof(a)/sizeof(a[0]));
-        assert(c.bucket_count() >= 5);
-        C::size_type b = c.bucket(0);
-        I i = c.begin(b);
-        I j = c.end(b);
-        assert(std::distance(i, j) == 0);
-
-        b = c.bucket(1);
-        i = c.begin(b);
-        j = c.end(b);
-        assert(std::distance(i, j) == 1);
-        assert(*i == 1);
-
-        b = c.bucket(2);
-        i = c.begin(b);
-        j = c.end(b);
-        assert(std::distance(i, j) == 1);
-        assert(*i == 2);
-
-        b = c.bucket(3);
-        i = c.begin(b);
-        j = c.end(b);
-        assert(std::distance(i, j) == 1);
-        assert(*i == 3);
-
-        b = c.bucket(4);
-        i = c.begin(b);
-        j = c.end(b);
-        assert(std::distance(i, j) == 1);
-        assert(*i == 4);
-    }
-    {
-        typedef std::unordered_set<int, std::hash<int>,
-                                      std::equal_to<int>, min_allocator<int>> C;
-        typedef int P;
-        typedef C::const_local_iterator I;
-        P a[] =
-        {
-            P(1),
-            P(2),
-            P(3),
-            P(4),
-            P(1),
-            P(2)
-        };
-        const C c(a, a + sizeof(a)/sizeof(a[0]));
-        assert(c.bucket_count() >= 5);
-        C::size_type b = c.bucket(0);
-        I i = c.begin(b);
-        I j = c.end(b);
-        assert(std::distance(i, j) == 0);
-
-        b = c.bucket(1);
-        i = c.begin(b);
-        j = c.end(b);
-        assert(std::distance(i, j) == 1);
-        assert(*i == 1);
-
-        b = c.bucket(2);
-        i = c.begin(b);
-        j = c.end(b);
-        assert(std::distance(i, j) == 1);
-        assert(*i == 2);
-
-        b = c.bucket(3);
-        i = c.begin(b);
-        j = c.end(b);
-        assert(std::distance(i, j) == 1);
-        assert(*i == 3);
-
-        b = c.bucket(4);
-        i = c.begin(b);
-        j = c.end(b);
-        assert(std::distance(i, j) == 1);
-        assert(*i == 4);
-    }
-    {
-        typedef std::unordered_set<int, std::hash<int>,
-                                      std::equal_to<int>, min_allocator<int>> C;
-        typedef int P;
-        typedef C::const_local_iterator I;
-        P a[] =
-        {
-            P(1),
-            P(2),
-            P(3),
-            P(4),
-            P(1),
-            P(2)
-        };
-        C c(a, a + sizeof(a)/sizeof(a[0]));
-        assert(c.bucket_count() >= 5);
-        C::size_type b = c.bucket(0);
-        I i = c.cbegin(b);
-        I j = c.cend(b);
-        assert(std::distance(i, j) == 0);
-
-        b = c.bucket(1);
-        i = c.cbegin(b);
-        j = c.cend(b);
-        assert(std::distance(i, j) == 1);
-        assert(*i == 1);
-
-        b = c.bucket(2);
-        i = c.cbegin(b);
-        j = c.cend(b);
-        assert(std::distance(i, j) == 1);
-        assert(*i == 2);
-
-        b = c.bucket(3);
-        i = c.cbegin(b);
-        j = c.cend(b);
-        assert(std::distance(i, j) == 1);
-        assert(*i == 3);
-
-        b = c.bucket(4);
-        i = c.cbegin(b);
-        j = c.cend(b);
-        assert(std::distance(i, j) == 1);
-        assert(*i == 4);
-    }
-    {
-        typedef std::unordered_set<int, std::hash<int>,
-                                      std::equal_to<int>, min_allocator<int>> C;
-        typedef int P;
-        typedef C::const_local_iterator I;
-        P a[] =
-        {
-            P(1),
-            P(2),
-            P(3),
-            P(4),
-            P(1),
-            P(2)
-        };
-        const C c(a, a + sizeof(a)/sizeof(a[0]));
-        assert(c.bucket_count() >= 5);
-        C::size_type b = c.bucket(0);
-        I i = c.cbegin(b);
-        I j = c.cend(b);
-        assert(std::distance(i, j) == 0);
-
-        b = c.bucket(1);
-        i = c.cbegin(b);
-        j = c.cend(b);
-        assert(std::distance(i, j) == 1);
-        assert(*i == 1);
-
-        b = c.bucket(2);
-        i = c.cbegin(b);
-        j = c.cend(b);
-        assert(std::distance(i, j) == 1);
-        assert(*i == 2);
-
-        b = c.bucket(3);
-        i = c.cbegin(b);
-        j = c.cend(b);
-        assert(std::distance(i, j) == 1);
-        assert(*i == 3);
-
-        b = c.bucket(4);
-        i = c.cbegin(b);
-        j = c.cend(b);
-        assert(std::distance(i, j) == 1);
-        assert(*i == 4);
-    }
+  {
+    typedef std::unordered_set<int, std::hash<int>, std::equal_to<int>, min_allocator<int>> C;
+    typedef int P;
+    typedef C::local_iterator I;
+    P a[] = {P(1), P(2), P(3), P(4), P(1), P(2)};
+    C c(a, a + sizeof(a) / sizeof(a[0]));
+    assert(c.bucket_count() >= 5);
+    C::size_type b = c.bucket(0);
+    I i            = c.begin(b);
+    I j            = c.end(b);
+    assert(std::distance(i, j) == 0);
+
+    b = c.bucket(1);
+    i = c.begin(b);
+    j = c.end(b);
+    assert(std::distance(i, j) == 1);
+    assert(*i == 1);
+
+    b = c.bucket(2);
+    i = c.begin(b);
+    j = c.end(b);
+    assert(std::distance(i, j) == 1);
+    assert(*i == 2);
+
+    b = c.bucket(3);
+    i = c.begin(b);
+    j = c.end(b);
+    assert(std::distance(i, j) == 1);
+    assert(*i == 3);
+
+    b = c.bucket(4);
+    i = c.begin(b);
+    j = c.end(b);
+    assert(std::distance(i, j) == 1);
+    assert(*i == 4);
+  }
+  {
+    typedef std::unordered_set<int, std::hash<int>, std::equal_to<int>, min_allocator<int>> C;
+    typedef int P;
+    typedef C::const_local_iterator I;
+    P a[] = {P(1), P(2), P(3), P(4), P(1), P(2)};
+    const C c(a, a + sizeof(a) / sizeof(a[0]));
+    assert(c.bucket_count() >= 5);
+    C::size_type b = c.bucket(0);
+    I i            = c.begin(b);
+    I j            = c.end(b);
+    assert(std::distance(i, j) == 0);
+
+    b = c.bucket(1);
+    i = c.begin(b);
+    j = c.end(b);
+    assert(std::distance(i, j) == 1);
+    assert(*i == 1);
+
+    b = c.bucket(2);
+    i = c.begin(b);
+    j = c.end(b);
+    assert(std::distance(i, j) == 1);
+    assert(*i == 2);
+
+    b = c.bucket(3);
+    i = c.begin(b);
+    j = c.end(b);
+    assert(std::distance(i, j) == 1);
+    assert(*i == 3);
+
+    b = c.bucket(4);
+    i = c.begin(b);
+    j = c.end(b);
+    assert(std::distance(i, j) == 1);
+    assert(*i == 4);
+  }
+  {
+    typedef std::unordered_set<int, std::hash<int>, std::equal_to<int>, min_allocator<int>> C;
+    typedef int P;
+    typedef C::const_local_iterator I;
+    P a[] = {P(1), P(2), P(3), P(4), P(1), P(2)};
+    C c(a, a + sizeof(a) / sizeof(a[0]));
+    assert(c.bucket_count() >= 5);
+    C::size_type b = c.bucket(0);
+    I i            = c.cbegin(b);
+    I j            = c.cend(b);
+    assert(std::distance(i, j) == 0);
+
+    b = c.bucket(1);
+    i = c.cbegin(b);
+    j = c.cend(b);
+    assert(std::distance(i, j) == 1);
+    assert(*i == 1);
+
+    b = c.bucket(2);
+    i = c.cbegin(b);
+    j = c.cend(b);
+    assert(std::distance(i, j) == 1);
+    assert(*i == 2);
+
+    b = c.bucket(3);
+    i = c.cbegin(b);
+    j = c.cend(b);
+    assert(std::distance(i, j) == 1);
+    assert(*i == 3);
+
+    b = c.bucket(4);
+    i = c.cbegin(b);
+    j = c.cend(b);
+    assert(std::distance(i, j) == 1);
+    assert(*i == 4);
+  }
+  {
+    typedef std::unordered_set<int, std::hash<int>, std::equal_to<int>, min_allocator<int>> C;
+    typedef int P;
+    typedef C::const_local_iterator I;
+    P a[] = {P(1), P(2), P(3), P(4), P(1), P(2)};
+    const C c(a, a + sizeof(a) / sizeof(a[0]));
+    assert(c.bucket_count() >= 5);
+    C::size_type b = c.bucket(0);
+    I i            = c.cbegin(b);
+    I j            = c.cend(b);
+    assert(std::distance(i, j) == 0);
+
+    b = c.bucket(1);
+    i = c.cbegin(b);
+    j = c.cend(b);
+    assert(std::distance(i, j) == 1);
+    assert(*i == 1);
+
+    b = c.bucket(2);
+    i = c.cbegin(b);
+    j = c.cend(b);
+    assert(std::distance(i, j) == 1);
+    assert(*i == 2);
+
+    b = c.bucket(3);
+    i = c.cbegin(b);
+    j = c.cend(b);
+    assert(std::distance(i, j) == 1);
+    assert(*i == 3);
+
+    b = c.bucket(4);
+    i = c.cbegin(b);
+    j = c.cend(b);
+    assert(std::distance(i, j) == 1);
+    assert(*i == 4);
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/unord/unord.set/max_bucket_count.pass.cpp b/libcxx/test/std/containers/unord/unord.set/max_bucket_count.pass.cpp
index c830f444bbaee..2ac65b06f8b29 100644
--- a/libcxx/test/std/containers/unord/unord.set/max_bucket_count.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.set/max_bucket_count.pass.cpp
@@ -20,20 +20,18 @@
 #include "test_macros.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef std::unordered_set<int> C;
-        const C c;
-        assert(c.max_bucket_count() > 0);
-    }
+int main(int, char**) {
+  {
+    typedef std::unordered_set<int> C;
+    const C c;
+    assert(c.max_bucket_count() > 0);
+  }
 #if TEST_STD_VER >= 11
-    {
-        typedef std::unordered_set<int, std::hash<int>,
-                                      std::equal_to<int>, min_allocator<int>> C;
-        const C c;
-        assert(c.max_bucket_count() > 0);
-    }
+  {
+    typedef std::unordered_set<int, std::hash<int>, std::equal_to<int>, min_allocator<int>> C;
+    const C c;
+    assert(c.max_bucket_count() > 0);
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/unord/unord.set/max_load_factor.pass.cpp b/libcxx/test/std/containers/unord/unord.set/max_load_factor.pass.cpp
index 9951956da66d7..ba8d9aa6619bb 100644
--- a/libcxx/test/std/containers/unord/unord.set/max_load_factor.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.set/max_load_factor.pass.cpp
@@ -21,36 +21,33 @@
 #include "test_macros.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef std::unordered_set<int> C;
-        const C c;
-        assert(c.max_load_factor() == 1);
-    }
-    {
-        typedef std::unordered_set<int> C;
-        C c;
-        assert(c.max_load_factor() == 1);
-        c.max_load_factor(2.5);
-        assert(c.max_load_factor() == 2.5);
-    }
+int main(int, char**) {
+  {
+    typedef std::unordered_set<int> C;
+    const C c;
+    assert(c.max_load_factor() == 1);
+  }
+  {
+    typedef std::unordered_set<int> C;
+    C c;
+    assert(c.max_load_factor() == 1);
+    c.max_load_factor(2.5);
+    assert(c.max_load_factor() == 2.5);
+  }
 #if TEST_STD_VER >= 11
-    {
-        typedef std::unordered_set<int, std::hash<int>,
-                                      std::equal_to<int>, min_allocator<int>> C;
-        const C c;
-        assert(c.max_load_factor() == 1);
-    }
-    {
-        typedef std::unordered_set<int, std::hash<int>,
-                                      std::equal_to<int>, min_allocator<int>> C;
-        C c;
-        assert(c.max_load_factor() == 1);
-        c.max_load_factor(2.5);
-        assert(c.max_load_factor() == 2.5);
-    }
+  {
+    typedef std::unordered_set<int, std::hash<int>, std::equal_to<int>, min_allocator<int>> C;
+    const C c;
+    assert(c.max_load_factor() == 1);
+  }
+  {
+    typedef std::unordered_set<int, std::hash<int>, std::equal_to<int>, min_allocator<int>> C;
+    C c;
+    assert(c.max_load_factor() == 1);
+    c.max_load_factor(2.5);
+    assert(c.max_load_factor() == 2.5);
+  }
 #endif
 
-    return 0;
+  return 0;
 }
diff --git a/libcxx/test/std/containers/unord/unord.set/max_size.pass.cpp b/libcxx/test/std/containers/unord/unord.set/max_size.pass.cpp
index 289c4d3231749..978a6ac6ba601 100644
--- a/libcxx/test/std/containers/unord/unord.set/max_size.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.set/max_size.pass.cpp
@@ -20,32 +20,29 @@
 #include "test_allocator.h"
 #include "test_macros.h"
 
-int main(int, char**)
-{
-    {
-      typedef limited_allocator<int, 10> A;
-      typedef std::unordered_set<int, std::hash<int>, std::equal_to<int>, A> C;
-      C c;
-      assert(c.max_size() <= 10);
-      LIBCPP_ASSERT(c.max_size() == 10);
-    }
-    {
-      typedef limited_allocator<int, (std::size_t)-1> A;
-      typedef std::unordered_set<int, std::hash<int>, std::equal_to<int>, A> C;
-      const C::size_type max_dist =
-          static_cast<C::size_type>(std::numeric_limits<C::difference_type>::max());
-      C c;
-      assert(c.max_size() <= max_dist);
-      LIBCPP_ASSERT(c.max_size() == max_dist);
-    }
-    {
-      typedef std::unordered_set<char> C;
-      const C::size_type max_dist =
-          static_cast<C::size_type>(std::numeric_limits<C::difference_type>::max());
-      C c;
-      assert(c.max_size() <= max_dist);
-      assert(c.max_size() <= alloc_max_size(c.get_allocator()));
-    }
+int main(int, char**) {
+  {
+    typedef limited_allocator<int, 10> A;
+    typedef std::unordered_set<int, std::hash<int>, std::equal_to<int>, A> C;
+    C c;
+    assert(c.max_size() <= 10);
+    LIBCPP_ASSERT(c.max_size() == 10);
+  }
+  {
+    typedef limited_allocator<int, (std::size_t)-1> A;
+    typedef std::unordered_set<int, std::hash<int>, std::equal_to<int>, A> C;
+    const C::size_type max_dist = static_cast<C::size_type>(std::numeric_limits<C::difference_type>::max());
+    C c;
+    assert(c.max_size() <= max_dist);
+    LIBCPP_ASSERT(c.max_size() == max_dist);
+  }
+  {
+    typedef std::unordered_set<char> C;
+    const C::size_type max_dist = static_cast<C::size_type>(std::numeric_limits<C::difference_type>::max());
+    C c;
+    assert(c.max_size() <= max_dist);
+    assert(c.max_size() <= alloc_max_size(c.get_allocator()));
+  }
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/unord/unord.set/merge.pass.cpp b/libcxx/test/std/containers/unord/unord.set/merge.pass.cpp
index c1b0e174cc0c1..966875b3e6c1a 100644
--- a/libcxx/test/std/containers/unord/unord.set/merge.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.set/merge.pass.cpp
@@ -27,128 +27,116 @@
 #include "Counter.h"
 
 template <class Set>
-bool set_equal(const Set& set, Set other)
-{
-    return set == other;
+bool set_equal(const Set& set, Set other) {
+  return set == other;
 }
 
 #ifndef TEST_HAS_NO_EXCEPTIONS
 template <class T>
-struct throw_hasher
-{
-    bool& should_throw_;
+struct throw_hasher {
+  bool& should_throw_;
 
-    throw_hasher(bool& should_throw) : should_throw_(should_throw) {}
+  throw_hasher(bool& should_throw) : should_throw_(should_throw) {}
 
-    std::size_t operator()(const T& p) const
-    {
-        if (should_throw_)
-            throw 0;
-        return std::hash<T>()(p);
-    }
+  std::size_t operator()(const T& p) const {
+    if (should_throw_)
+      throw 0;
+    return std::hash<T>()(p);
+  }
 };
 #endif
 
-int main(int, char**)
-{
-    {
-        std::unordered_set<int> src{1, 3, 5};
-        std::unordered_set<int> dst{2, 4, 5};
-        dst.merge(src);
-        assert(set_equal(src, {5}));
-        assert(set_equal(dst, {1, 2, 3, 4, 5}));
-    }
+int main(int, char**) {
+  {
+    std::unordered_set<int> src{1, 3, 5};
+    std::unordered_set<int> dst{2, 4, 5};
+    dst.merge(src);
+    assert(set_equal(src, {5}));
+    assert(set_equal(dst, {1, 2, 3, 4, 5}));
+  }
 
 #ifndef TEST_HAS_NO_EXCEPTIONS
-    {
-        bool do_throw = false;
-        typedef std::unordered_set<Counter<int>, throw_hasher<Counter<int>>> set_type;
-        set_type src({1, 3, 5}, 0, throw_hasher<Counter<int>>(do_throw));
-        set_type dst({2, 4, 5}, 0, throw_hasher<Counter<int>>(do_throw));
-
-        assert(Counter_base::gConstructed == 6);
-
-        do_throw = true;
-        try
-        {
-            dst.merge(src);
-        }
-        catch (int)
-        {
-            do_throw = false;
-        }
-        assert(!do_throw);
-        assert(set_equal(src, set_type({1, 3, 5}, 0, throw_hasher<Counter<int>>(do_throw))));
-        assert(set_equal(dst, set_type({2, 4, 5}, 0, throw_hasher<Counter<int>>(do_throw))));
+  {
+    bool do_throw = false;
+    typedef std::unordered_set<Counter<int>, throw_hasher<Counter<int>>> set_type;
+    set_type src({1, 3, 5}, 0, throw_hasher<Counter<int>>(do_throw));
+    set_type dst({2, 4, 5}, 0, throw_hasher<Counter<int>>(do_throw));
+
+    assert(Counter_base::gConstructed == 6);
+
+    do_throw = true;
+    try {
+      dst.merge(src);
+    } catch (int) {
+      do_throw = false;
     }
+    assert(!do_throw);
+    assert(set_equal(src, set_type({1, 3, 5}, 0, throw_hasher<Counter<int>>(do_throw))));
+    assert(set_equal(dst, set_type({2, 4, 5}, 0, throw_hasher<Counter<int>>(do_throw))));
+  }
 #endif
-    assert(Counter_base::gConstructed == 0);
-    struct equal
-    {
-        equal() = default;
-
-        bool operator()(const Counter<int>& lhs, const Counter<int>& rhs) const
-        {
-            return lhs == rhs;
-        }
-    };
-    struct hasher
-    {
-        hasher() = default;
-        std::size_t operator()(const Counter<int>& p) const { return std::hash<Counter<int>>()(p); }
-    };
-    {
-        typedef std::unordered_set<Counter<int>, std::hash<Counter<int>>, std::equal_to<Counter<int>>> first_set_type;
-        typedef std::unordered_set<Counter<int>, hasher, equal> second_set_type;
-        typedef std::unordered_multiset<Counter<int>, hasher, equal> third_set_type;
+  assert(Counter_base::gConstructed == 0);
+  struct equal {
+    equal() = default;
+
+    bool operator()(const Counter<int>& lhs, const Counter<int>& rhs) const { return lhs == rhs; }
+  };
+  struct hasher {
+    hasher() = default;
+    std::size_t operator()(const Counter<int>& p) const { return std::hash<Counter<int>>()(p); }
+  };
+  {
+    typedef std::unordered_set<Counter<int>, std::hash<Counter<int>>, std::equal_to<Counter<int>>> first_set_type;
+    typedef std::unordered_set<Counter<int>, hasher, equal> second_set_type;
+    typedef std::unordered_multiset<Counter<int>, hasher, equal> third_set_type;
 
-        {
-            first_set_type first{1, 2, 3};
-            second_set_type second{2, 3, 4};
-            third_set_type third{1, 3};
+    {
+      first_set_type first{1, 2, 3};
+      second_set_type second{2, 3, 4};
+      third_set_type third{1, 3};
 
-            assert(Counter_base::gConstructed == 8);
+      assert(Counter_base::gConstructed == 8);
 
-            first.merge(second);
-            first.merge(third);
+      first.merge(second);
+      first.merge(third);
 
-            assert(set_equal(first, {1, 2, 3, 4}));
-            assert(set_equal(second, {2, 3}));
-            assert(set_equal(third, {1, 3}));
+      assert(set_equal(first, {1, 2, 3, 4}));
+      assert(set_equal(second, {2, 3}));
+      assert(set_equal(third, {1, 3}));
 
-            assert(Counter_base::gConstructed == 8);
-        }
-        assert(Counter_base::gConstructed == 0);
-        {
-            first_set_type first{1, 2, 3};
-            second_set_type second{2, 3, 4};
-            third_set_type third{1, 3};
+      assert(Counter_base::gConstructed == 8);
+    }
+    assert(Counter_base::gConstructed == 0);
+    {
+      first_set_type first{1, 2, 3};
+      second_set_type second{2, 3, 4};
+      third_set_type third{1, 3};
 
-            assert(Counter_base::gConstructed == 8);
+      assert(Counter_base::gConstructed == 8);
 
-            first.merge(std::move(second));
-            first.merge(std::move(third));
+      first.merge(std::move(second));
+      first.merge(std::move(third));
 
-            assert(set_equal(first, {1, 2, 3, 4}));
-            assert(set_equal(second, {2, 3}));
-            assert(set_equal(third, {1, 3}));
+      assert(set_equal(first, {1, 2, 3, 4}));
+      assert(set_equal(second, {2, 3}));
+      assert(set_equal(third, {1, 3}));
 
-            assert(Counter_base::gConstructed == 8);
-        }
-        assert(Counter_base::gConstructed == 0);
+      assert(Counter_base::gConstructed == 8);
+    }
+    assert(Counter_base::gConstructed == 0);
+  }
+  {
+    std::unordered_set<int> first;
+    {
+      std::unordered_set<int> second;
+      first.merge(second);
+      first.merge(std::move(second));
     }
     {
-        std::unordered_set<int> first;
-        {
-            std::unordered_set<int> second;
-            first.merge(second);
-            first.merge(std::move(second));
-        }
-        {
-            std::unordered_multiset<int> second;
-            first.merge(second);
-            first.merge(std::move(second));
-        }
+      std::unordered_multiset<int> second;
+      first.merge(second);
+      first.merge(std::move(second));
     }
-    return 0;
+  }
+  return 0;
 }
diff --git a/libcxx/test/std/containers/unord/unord.set/range_concept_conformance.compile.pass.cpp b/libcxx/test/std/containers/unord/unord.set/range_concept_conformance.compile.pass.cpp
index 1094e5ae67ee1..428b5abefc6cc 100644
--- a/libcxx/test/std/containers/unord/unord.set/range_concept_conformance.compile.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.set/range_concept_conformance.compile.pass.cpp
@@ -19,7 +19,6 @@
 
 using range = std::unordered_set<int>;
 
-
 static_assert(std::same_as<std::ranges::iterator_t<range>, range::iterator>);
 static_assert(std::ranges::common_range<range>);
 static_assert(std::ranges::forward_range<range>);
diff --git a/libcxx/test/std/containers/unord/unord.set/rehash.pass.cpp b/libcxx/test/std/containers/unord/unord.set/rehash.pass.cpp
index a0482fc2db706..cba4aeb33e585 100644
--- a/libcxx/test/std/containers/unord/unord.set/rehash.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.set/rehash.pass.cpp
@@ -21,83 +21,63 @@
 #include "min_allocator.h"
 
 template <class C>
-void rehash_postcondition(const C& c, std::size_t n)
-{
-    assert(c.bucket_count() >= c.size() / c.max_load_factor() && c.bucket_count() >= n);
+void rehash_postcondition(const C& c, std::size_t n) {
+  assert(c.bucket_count() >= c.size() / c.max_load_factor() && c.bucket_count() >= n);
 }
 
 template <class C>
-void test(const C& c)
-{
-    assert(c.size() == 4);
-    assert(c.count(1) == 1);
-    assert(c.count(2) == 1);
-    assert(c.count(3) == 1);
-    assert(c.count(4) == 1);
+void test(const C& c) {
+  assert(c.size() == 4);
+  assert(c.count(1) == 1);
+  assert(c.count(2) == 1);
+  assert(c.count(3) == 1);
+  assert(c.count(4) == 1);
 }
 
-int main(int, char**)
-{
-    {
-        typedef std::unordered_set<int> C;
-        typedef int P;
-        P a[] =
-        {
-            P(1),
-            P(2),
-            P(3),
-            P(4),
-            P(1),
-            P(2)
-        };
-        C c(a, a + sizeof(a)/sizeof(a[0]));
-        test(c);
-        assert(c.bucket_count() >= 5);
-        c.rehash(3);
-        rehash_postcondition(c, 3);
-        LIBCPP_ASSERT(c.bucket_count() == 5);
-        test(c);
-        c.max_load_factor(2);
-        c.rehash(3);
-        rehash_postcondition(c, 3);
-        LIBCPP_ASSERT(c.bucket_count() == 3);
-        test(c);
-        c.rehash(31);
-        rehash_postcondition(c, 31);
-        LIBCPP_ASSERT(c.bucket_count() == 31);
-        test(c);
-    }
+int main(int, char**) {
+  {
+    typedef std::unordered_set<int> C;
+    typedef int P;
+    P a[] = {P(1), P(2), P(3), P(4), P(1), P(2)};
+    C c(a, a + sizeof(a) / sizeof(a[0]));
+    test(c);
+    assert(c.bucket_count() >= 5);
+    c.rehash(3);
+    rehash_postcondition(c, 3);
+    LIBCPP_ASSERT(c.bucket_count() == 5);
+    test(c);
+    c.max_load_factor(2);
+    c.rehash(3);
+    rehash_postcondition(c, 3);
+    LIBCPP_ASSERT(c.bucket_count() == 3);
+    test(c);
+    c.rehash(31);
+    rehash_postcondition(c, 31);
+    LIBCPP_ASSERT(c.bucket_count() == 31);
+    test(c);
+  }
 #if TEST_STD_VER >= 11
-    {
-        typedef std::unordered_set<int, std::hash<int>,
-                                      std::equal_to<int>, min_allocator<int>> C;
-        typedef int P;
-        P a[] =
-        {
-            P(1),
-            P(2),
-            P(3),
-            P(4),
-            P(1),
-            P(2)
-        };
-        C c(a, a + sizeof(a)/sizeof(a[0]));
-        test(c);
-        assert(c.bucket_count() >= 5);
-        c.rehash(3);
-        rehash_postcondition(c, 3);
-        LIBCPP_ASSERT(c.bucket_count() == 5);
-        test(c);
-        c.max_load_factor(2);
-        c.rehash(3);
-        rehash_postcondition(c, 3);
-        LIBCPP_ASSERT(c.bucket_count() == 3);
-        test(c);
-        c.rehash(31);
-        rehash_postcondition(c, 31);
-        LIBCPP_ASSERT(c.bucket_count() == 31);
-        test(c);
-    }
+  {
+    typedef std::unordered_set<int, std::hash<int>, std::equal_to<int>, min_allocator<int>> C;
+    typedef int P;
+    P a[] = {P(1), P(2), P(3), P(4), P(1), P(2)};
+    C c(a, a + sizeof(a) / sizeof(a[0]));
+    test(c);
+    assert(c.bucket_count() >= 5);
+    c.rehash(3);
+    rehash_postcondition(c, 3);
+    LIBCPP_ASSERT(c.bucket_count() == 5);
+    test(c);
+    c.max_load_factor(2);
+    c.rehash(3);
+    rehash_postcondition(c, 3);
+    LIBCPP_ASSERT(c.bucket_count() == 3);
+    test(c);
+    c.rehash(31);
+    rehash_postcondition(c, 31);
+    LIBCPP_ASSERT(c.bucket_count() == 31);
+    test(c);
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/unord/unord.set/reserve.pass.cpp b/libcxx/test/std/containers/unord/unord.set/reserve.pass.cpp
index afcccf9e03795..cd4b670a9c797 100644
--- a/libcxx/test/std/containers/unord/unord.set/reserve.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.set/reserve.pass.cpp
@@ -21,88 +21,67 @@
 #include "min_allocator.h"
 
 template <class C>
-void test(const C& c)
-{
-    assert(c.size() == 4);
-    assert(c.count(1) == 1);
-    assert(c.count(2) == 1);
-    assert(c.count(3) == 1);
-    assert(c.count(4) == 1);
+void test(const C& c) {
+  assert(c.size() == 4);
+  assert(c.count(1) == 1);
+  assert(c.count(2) == 1);
+  assert(c.count(3) == 1);
+  assert(c.count(4) == 1);
 }
 
 void reserve_invariant(std::size_t n) // LWG #2156
 {
-    for (std::size_t i = 0; i < n; ++i)
-    {
-        std::unordered_set<std::size_t> c;
-        c.reserve(n);
-        std::size_t buckets = c.bucket_count();
-        for (std::size_t j = 0; j < i; ++j)
-        {
-            c.insert(i);
-            assert(buckets == c.bucket_count());
-        }
+  for (std::size_t i = 0; i < n; ++i) {
+    std::unordered_set<std::size_t> c;
+    c.reserve(n);
+    std::size_t buckets = c.bucket_count();
+    for (std::size_t j = 0; j < i; ++j) {
+      c.insert(i);
+      assert(buckets == c.bucket_count());
     }
+  }
 }
 
-int main(int, char**)
-{
-    {
-        typedef std::unordered_set<int> C;
-        typedef int P;
-        P a[] =
-        {
-            P(1),
-            P(2),
-            P(3),
-            P(4),
-            P(1),
-            P(2)
-        };
-        C c(a, a + sizeof(a)/sizeof(a[0]));
-        test(c);
-        assert(c.bucket_count() >= 5);
-        c.reserve(3);
-        LIBCPP_ASSERT(c.bucket_count() == 5);
-        test(c);
-        c.max_load_factor(2);
-        c.reserve(3);
-        assert(c.bucket_count() >= 2);
-        test(c);
-        c.reserve(31);
-        assert(c.bucket_count() >= 16);
-        test(c);
-    }
+int main(int, char**) {
+  {
+    typedef std::unordered_set<int> C;
+    typedef int P;
+    P a[] = {P(1), P(2), P(3), P(4), P(1), P(2)};
+    C c(a, a + sizeof(a) / sizeof(a[0]));
+    test(c);
+    assert(c.bucket_count() >= 5);
+    c.reserve(3);
+    LIBCPP_ASSERT(c.bucket_count() == 5);
+    test(c);
+    c.max_load_factor(2);
+    c.reserve(3);
+    assert(c.bucket_count() >= 2);
+    test(c);
+    c.reserve(31);
+    assert(c.bucket_count() >= 16);
+    test(c);
+  }
 #if TEST_STD_VER >= 11
-    {
-        typedef std::unordered_set<int, std::hash<int>,
-                                      std::equal_to<int>, min_allocator<int>> C;
-        typedef int P;
-        P a[] =
-        {
-            P(1),
-            P(2),
-            P(3),
-            P(4),
-            P(1),
-            P(2)
-        };
-        C c(a, a + sizeof(a)/sizeof(a[0]));
-        test(c);
-        assert(c.bucket_count() >= 5);
-        c.reserve(3);
-        LIBCPP_ASSERT(c.bucket_count() == 5);
-        test(c);
-        c.max_load_factor(2);
-        c.reserve(3);
-        assert(c.bucket_count() >= 2);
-        test(c);
-        c.reserve(31);
-        assert(c.bucket_count() >= 16);
-        test(c);
-    }
+  {
+    typedef std::unordered_set<int, std::hash<int>, std::equal_to<int>, min_allocator<int>> C;
+    typedef int P;
+    P a[] = {P(1), P(2), P(3), P(4), P(1), P(2)};
+    C c(a, a + sizeof(a) / sizeof(a[0]));
+    test(c);
+    assert(c.bucket_count() >= 5);
+    c.reserve(3);
+    LIBCPP_ASSERT(c.bucket_count() == 5);
+    test(c);
+    c.max_load_factor(2);
+    c.reserve(3);
+    assert(c.bucket_count() >= 2);
+    test(c);
+    c.reserve(31);
+    assert(c.bucket_count() >= 16);
+    test(c);
+  }
 #endif
-    reserve_invariant(20);
+  reserve_invariant(20);
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/unord/unord.set/size.pass.cpp b/libcxx/test/std/containers/unord/unord.set/size.pass.cpp
index f7967fcd5f7a5..4b2e8f3814d86 100644
--- a/libcxx/test/std/containers/unord/unord.set/size.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.set/size.pass.cpp
@@ -18,9 +18,8 @@
 #include "test_macros.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
+int main(int, char**) {
+  {
     typedef std::unordered_set<int> M;
     M m;
     ASSERT_NOEXCEPT(m.size());
@@ -37,9 +36,9 @@ int main(int, char**)
     assert(m.size() == 1);
     m.erase(m.begin());
     assert(m.size() == 0);
-    }
+  }
 #if TEST_STD_VER >= 11
-    {
+  {
     typedef std::unordered_set<int, std::hash<int>, std::equal_to<int>, min_allocator<int>> M;
     M m;
     ASSERT_NOEXCEPT(m.size());
@@ -56,7 +55,7 @@ int main(int, char**)
     assert(m.size() == 1);
     m.erase(m.begin());
     assert(m.size() == 0);
-    }
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/unord/unord.set/swap_member.pass.cpp b/libcxx/test/std/containers/unord/unord.set/swap_member.pass.cpp
index d5adfbb2462cf..be8a96f0d4569 100644
--- a/libcxx/test/std/containers/unord/unord.set/swap_member.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.set/swap_member.pass.cpp
@@ -24,547 +24,438 @@
 #include "test_allocator.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef test_hash<int> Hash;
-        typedef test_equal_to<int> Compare;
-        typedef test_allocator<int> Alloc;
-        typedef std::unordered_set<int, Hash, Compare, Alloc> C;
-        C c1(0, Hash(1), Compare(1), Alloc(1, 1));
-        C c2(0, Hash(2), Compare(2), Alloc(1, 2));
-        c2.max_load_factor(2);
-        c1.swap(c2);
-
-        LIBCPP_ASSERT(c1.bucket_count() == 0);
-        assert(c1.size() == 0);
-        assert(c1.hash_function() == Hash(2));
-        assert(c1.key_eq() == Compare(2));
-        assert(c1.get_allocator().get_id() == 1);
-        assert(static_cast<std::size_t>(std::distance(c1.begin(), c1.end())) == c1.size());
-        assert(static_cast<std::size_t>(std::distance(c1.cbegin(), c1.cend())) == c1.size());
-        assert(c1.max_load_factor() == 2);
-
-        LIBCPP_ASSERT(c2.bucket_count() == 0);
-        assert(c2.size() == 0);
-        assert(c2.hash_function() == Hash(1));
-        assert(c2.key_eq() == Compare(1));
-        assert(c2.get_allocator().get_id() == 2);
-        assert(static_cast<std::size_t>(std::distance(c2.begin(), c2.end())) == c2.size());
-        assert(static_cast<std::size_t>(std::distance(c2.cbegin(), c2.cend())) == c2.size());
-        assert(c2.max_load_factor() == 1);
-    }
-    {
-        typedef test_hash<int> Hash;
-        typedef test_equal_to<int> Compare;
-        typedef test_allocator<int> Alloc;
-        typedef std::unordered_set<int, Hash, Compare, Alloc> C;
-        typedef int P;
-        P a2[] =
-        {
-            P(10),
-            P(20),
-            P(30),
-            P(40),
-            P(50),
-            P(60),
-            P(70),
-            P(80)
-        };
-        C c1(0, Hash(1), Compare(1), Alloc(1, 1));
-        C c2(std::begin(a2), std::end(a2), 0, Hash(2), Compare(2), Alloc(1, 2));
-        c2.max_load_factor(2);
-        c1.swap(c2);
-
-        assert(c1.bucket_count() >= 8);
-        assert(c1.size() == 8);
-        assert(*c1.find(10) == 10);
-        assert(*c1.find(20) == 20);
-        assert(*c1.find(30) == 30);
-        assert(*c1.find(40) == 40);
-        assert(*c1.find(50) == 50);
-        assert(*c1.find(60) == 60);
-        assert(*c1.find(70) == 70);
-        assert(*c1.find(80) == 80);
-        assert(c1.hash_function() == Hash(2));
-        assert(c1.key_eq() == Compare(2));
-        assert(c1.get_allocator().get_id() == 1);
-        assert(static_cast<std::size_t>(std::distance(c1.begin(), c1.end())) == c1.size());
-        assert(static_cast<std::size_t>(std::distance(c1.cbegin(), c1.cend())) == c1.size());
-        assert(c1.max_load_factor() == 2);
-
-        LIBCPP_ASSERT(c2.bucket_count() == 0);
-        assert(c2.size() == 0);
-        assert(c2.hash_function() == Hash(1));
-        assert(c2.key_eq() == Compare(1));
-        assert(c2.get_allocator().get_id() == 2);
-        assert(static_cast<std::size_t>(std::distance(c2.begin(), c2.end())) == c2.size());
-        assert(static_cast<std::size_t>(std::distance(c2.cbegin(), c2.cend())) == c2.size());
-        assert(c2.max_load_factor() == 1);
-    }
-    {
-        typedef test_hash<int> Hash;
-        typedef test_equal_to<int> Compare;
-        typedef test_allocator<int> Alloc;
-        typedef std::unordered_set<int, Hash, Compare, Alloc> C;
-        typedef int P;
-        P a1[] =
-        {
-            P(1),
-            P(2),
-            P(3),
-            P(4),
-            P(1),
-            P(2)
-        };
-        C c1(std::begin(a1), std::end(a1), 0, Hash(1), Compare(1), Alloc(1, 1));
-        C c2(0, Hash(2), Compare(2), Alloc(1, 2));
-        c2.max_load_factor(2);
-        c1.swap(c2);
-
-        LIBCPP_ASSERT(c1.bucket_count() == 0);
-        assert(c1.size() == 0);
-        assert(c1.hash_function() == Hash(2));
-        assert(c1.key_eq() == Compare(2));
-        assert(c1.get_allocator().get_id() == 1);
-        assert(static_cast<std::size_t>(std::distance(c1.begin(), c1.end())) == c1.size());
-        assert(static_cast<std::size_t>(std::distance(c1.cbegin(), c1.cend())) == c1.size());
-        assert(c1.max_load_factor() == 2);
-
-        assert(c2.bucket_count() >= 4);
-        assert(c2.size() == 4);
-        assert(c2.count(1) == 1);
-        assert(c2.count(2) == 1);
-        assert(c2.count(3) == 1);
-        assert(c2.count(4) == 1);
-        assert(c2.hash_function() == Hash(1));
-        assert(c2.key_eq() == Compare(1));
-        assert(c2.get_allocator().get_id() == 2);
-        assert(static_cast<std::size_t>(std::distance(c2.begin(), c2.end())) == c2.size());
-        assert(static_cast<std::size_t>(std::distance(c2.cbegin(), c2.cend())) == c2.size());
-        assert(c2.max_load_factor() == 1);
-    }
-    {
-        typedef test_hash<int> Hash;
-        typedef test_equal_to<int> Compare;
-        typedef test_allocator<int> Alloc;
-        typedef std::unordered_set<int, Hash, Compare, Alloc> C;
-        typedef int P;
-        P a1[] =
-        {
-            P(1),
-            P(2),
-            P(3),
-            P(4),
-            P(1),
-            P(2)
-        };
-        P a2[] =
-        {
-            P(10),
-            P(20),
-            P(30),
-            P(40),
-            P(50),
-            P(60),
-            P(70),
-            P(80)
-        };
-        C c1(std::begin(a1), std::end(a1), 0, Hash(1), Compare(1), Alloc(1, 1));
-        C c2(std::begin(a2), std::end(a2), 0, Hash(2), Compare(2), Alloc(1, 2));
-        c2.max_load_factor(2);
-        c1.swap(c2);
-
-        assert(c1.bucket_count() >= 8);
-        assert(c1.size() == 8);
-        assert(*c1.find(10) == 10);
-        assert(*c1.find(20) == 20);
-        assert(*c1.find(30) == 30);
-        assert(*c1.find(40) == 40);
-        assert(*c1.find(50) == 50);
-        assert(*c1.find(60) == 60);
-        assert(*c1.find(70) == 70);
-        assert(*c1.find(80) == 80);
-        assert(c1.hash_function() == Hash(2));
-        assert(c1.key_eq() == Compare(2));
-        assert(c1.get_allocator().get_id() == 1);
-        assert(static_cast<std::size_t>(std::distance(c1.begin(), c1.end())) == c1.size());
-        assert(static_cast<std::size_t>(std::distance(c1.cbegin(), c1.cend())) == c1.size());
-        assert(c1.max_load_factor() == 2);
-
-        assert(c2.bucket_count() >= 4);
-        assert(c2.size() == 4);
-        assert(c2.count(1) == 1);
-        assert(c2.count(2) == 1);
-        assert(c2.count(3) == 1);
-        assert(c2.count(4) == 1);
-        assert(c2.hash_function() == Hash(1));
-        assert(c2.key_eq() == Compare(1));
-        assert(c2.get_allocator().get_id() == 2);
-        assert(static_cast<std::size_t>(std::distance(c2.begin(), c2.end())) == c2.size());
-        assert(static_cast<std::size_t>(std::distance(c2.cbegin(), c2.cend())) == c2.size());
-        assert(c2.max_load_factor() == 1);
-    }
-
-    {
-        typedef test_hash<int> Hash;
-        typedef test_equal_to<int> Compare;
-        typedef other_allocator<int> Alloc;
-        typedef std::unordered_set<int, Hash, Compare, Alloc> C;
-        C c1(0, Hash(1), Compare(1), Alloc(1));
-        C c2(0, Hash(2), Compare(2), Alloc(2));
-        c2.max_load_factor(2);
-        c1.swap(c2);
-
-        LIBCPP_ASSERT(c1.bucket_count() == 0);
-        assert(c1.size() == 0);
-        assert(c1.hash_function() == Hash(2));
-        assert(c1.key_eq() == Compare(2));
-        assert(c1.get_allocator() == Alloc(2));
-        assert(static_cast<std::size_t>(std::distance(c1.begin(), c1.end())) == c1.size());
-        assert(static_cast<std::size_t>(std::distance(c1.cbegin(), c1.cend())) == c1.size());
-        assert(c1.max_load_factor() == 2);
-
-        LIBCPP_ASSERT(c2.bucket_count() == 0);
-        assert(c2.size() == 0);
-        assert(c2.hash_function() == Hash(1));
-        assert(c2.key_eq() == Compare(1));
-        assert(c2.get_allocator() == Alloc(1));
-        assert(static_cast<std::size_t>(std::distance(c2.begin(), c2.end())) == c2.size());
-        assert(static_cast<std::size_t>(std::distance(c2.cbegin(), c2.cend())) == c2.size());
-        assert(c2.max_load_factor() == 1);
-    }
-    {
-        typedef test_hash<int> Hash;
-        typedef test_equal_to<int> Compare;
-        typedef other_allocator<int> Alloc;
-        typedef std::unordered_set<int, Hash, Compare, Alloc> C;
-        typedef int P;
-        P a2[] =
-        {
-            P(10),
-            P(20),
-            P(30),
-            P(40),
-            P(50),
-            P(60),
-            P(70),
-            P(80)
-        };
-        C c1(0, Hash(1), Compare(1), Alloc(1));
-        C c2(std::begin(a2), std::end(a2), 0, Hash(2), Compare(2), Alloc(2));
-        c2.max_load_factor(2);
-        c1.swap(c2);
-
-        assert(c1.bucket_count() >= 8);
-        assert(c1.size() == 8);
-        assert(*c1.find(10) == 10);
-        assert(*c1.find(20) == 20);
-        assert(*c1.find(30) == 30);
-        assert(*c1.find(40) == 40);
-        assert(*c1.find(50) == 50);
-        assert(*c1.find(60) == 60);
-        assert(*c1.find(70) == 70);
-        assert(*c1.find(80) == 80);
-        assert(c1.hash_function() == Hash(2));
-        assert(c1.key_eq() == Compare(2));
-        assert(c1.get_allocator() == Alloc(2));
-        assert(static_cast<std::size_t>(std::distance(c1.begin(), c1.end())) == c1.size());
-        assert(static_cast<std::size_t>(std::distance(c1.cbegin(), c1.cend())) == c1.size());
-        assert(c1.max_load_factor() == 2);
-
-        LIBCPP_ASSERT(c2.bucket_count() == 0);
-        assert(c2.size() == 0);
-        assert(c2.hash_function() == Hash(1));
-        assert(c2.key_eq() == Compare(1));
-        assert(c2.get_allocator() == Alloc(1));
-        assert(static_cast<std::size_t>(std::distance(c2.begin(), c2.end())) == c2.size());
-        assert(static_cast<std::size_t>(std::distance(c2.cbegin(), c2.cend())) == c2.size());
-        assert(c2.max_load_factor() == 1);
-    }
-    {
-        typedef test_hash<int> Hash;
-        typedef test_equal_to<int> Compare;
-        typedef other_allocator<int> Alloc;
-        typedef std::unordered_set<int, Hash, Compare, Alloc> C;
-        typedef int P;
-        P a1[] =
-        {
-            P(1),
-            P(2),
-            P(3),
-            P(4),
-            P(1),
-            P(2)
-        };
-        C c1(std::begin(a1), std::end(a1), 0, Hash(1), Compare(1), Alloc(1));
-        C c2(0, Hash(2), Compare(2), Alloc(2));
-        c2.max_load_factor(2);
-        c1.swap(c2);
-
-        LIBCPP_ASSERT(c1.bucket_count() == 0);
-        assert(c1.size() == 0);
-        assert(c1.hash_function() == Hash(2));
-        assert(c1.key_eq() == Compare(2));
-        assert(c1.get_allocator() == Alloc(2));
-        assert(static_cast<std::size_t>(std::distance(c1.begin(), c1.end())) == c1.size());
-        assert(static_cast<std::size_t>(std::distance(c1.cbegin(), c1.cend())) == c1.size());
-        assert(c1.max_load_factor() == 2);
-
-        assert(c2.bucket_count() >= 4);
-        assert(c2.size() == 4);
-        assert(c2.count(1) == 1);
-        assert(c2.count(2) == 1);
-        assert(c2.count(3) == 1);
-        assert(c2.count(4) == 1);
-        assert(c2.hash_function() == Hash(1));
-        assert(c2.key_eq() == Compare(1));
-        assert(c2.get_allocator() == Alloc(1));
-        assert(static_cast<std::size_t>(std::distance(c2.begin(), c2.end())) == c2.size());
-        assert(static_cast<std::size_t>(std::distance(c2.cbegin(), c2.cend())) == c2.size());
-        assert(c2.max_load_factor() == 1);
-    }
-    {
-        typedef test_hash<int> Hash;
-        typedef test_equal_to<int> Compare;
-        typedef other_allocator<int> Alloc;
-        typedef std::unordered_set<int, Hash, Compare, Alloc> C;
-        typedef int P;
-        P a1[] =
-        {
-            P(1),
-            P(2),
-            P(3),
-            P(4),
-            P(1),
-            P(2)
-        };
-        P a2[] =
-        {
-            P(10),
-            P(20),
-            P(30),
-            P(40),
-            P(50),
-            P(60),
-            P(70),
-            P(80)
-        };
-        C c1(std::begin(a1), std::end(a1), 0, Hash(1), Compare(1), Alloc(1));
-        C c2(std::begin(a2), std::end(a2), 0, Hash(2), Compare(2), Alloc(2));
-        c2.max_load_factor(2);
-        c1.swap(c2);
-
-        assert(c1.bucket_count() >= 8);
-        assert(c1.size() == 8);
-        assert(*c1.find(10) == 10);
-        assert(*c1.find(20) == 20);
-        assert(*c1.find(30) == 30);
-        assert(*c1.find(40) == 40);
-        assert(*c1.find(50) == 50);
-        assert(*c1.find(60) == 60);
-        assert(*c1.find(70) == 70);
-        assert(*c1.find(80) == 80);
-        assert(c1.hash_function() == Hash(2));
-        assert(c1.key_eq() == Compare(2));
-        assert(c1.get_allocator() == Alloc(2));
-        assert(static_cast<std::size_t>(std::distance(c1.begin(), c1.end())) == c1.size());
-        assert(static_cast<std::size_t>(std::distance(c1.cbegin(), c1.cend())) == c1.size());
-        assert(c1.max_load_factor() == 2);
-
-        assert(c2.bucket_count() >= 4);
-        assert(c2.size() == 4);
-        assert(c2.count(1) == 1);
-        assert(c2.count(2) == 1);
-        assert(c2.count(3) == 1);
-        assert(c2.count(4) == 1);
-        assert(c2.hash_function() == Hash(1));
-        assert(c2.key_eq() == Compare(1));
-        assert(c2.get_allocator() == Alloc(1));
-        assert(static_cast<std::size_t>(std::distance(c2.begin(), c2.end())) == c2.size());
-        assert(static_cast<std::size_t>(std::distance(c2.cbegin(), c2.cend())) == c2.size());
-        assert(c2.max_load_factor() == 1);
-    }
+int main(int, char**) {
+  {
+    typedef test_hash<int> Hash;
+    typedef test_equal_to<int> Compare;
+    typedef test_allocator<int> Alloc;
+    typedef std::unordered_set<int, Hash, Compare, Alloc> C;
+    C c1(0, Hash(1), Compare(1), Alloc(1, 1));
+    C c2(0, Hash(2), Compare(2), Alloc(1, 2));
+    c2.max_load_factor(2);
+    c1.swap(c2);
+
+    LIBCPP_ASSERT(c1.bucket_count() == 0);
+    assert(c1.size() == 0);
+    assert(c1.hash_function() == Hash(2));
+    assert(c1.key_eq() == Compare(2));
+    assert(c1.get_allocator().get_id() == 1);
+    assert(static_cast<std::size_t>(std::distance(c1.begin(), c1.end())) == c1.size());
+    assert(static_cast<std::size_t>(std::distance(c1.cbegin(), c1.cend())) == c1.size());
+    assert(c1.max_load_factor() == 2);
+
+    LIBCPP_ASSERT(c2.bucket_count() == 0);
+    assert(c2.size() == 0);
+    assert(c2.hash_function() == Hash(1));
+    assert(c2.key_eq() == Compare(1));
+    assert(c2.get_allocator().get_id() == 2);
+    assert(static_cast<std::size_t>(std::distance(c2.begin(), c2.end())) == c2.size());
+    assert(static_cast<std::size_t>(std::distance(c2.cbegin(), c2.cend())) == c2.size());
+    assert(c2.max_load_factor() == 1);
+  }
+  {
+    typedef test_hash<int> Hash;
+    typedef test_equal_to<int> Compare;
+    typedef test_allocator<int> Alloc;
+    typedef std::unordered_set<int, Hash, Compare, Alloc> C;
+    typedef int P;
+    P a2[] = {P(10), P(20), P(30), P(40), P(50), P(60), P(70), P(80)};
+    C c1(0, Hash(1), Compare(1), Alloc(1, 1));
+    C c2(std::begin(a2), std::end(a2), 0, Hash(2), Compare(2), Alloc(1, 2));
+    c2.max_load_factor(2);
+    c1.swap(c2);
+
+    assert(c1.bucket_count() >= 8);
+    assert(c1.size() == 8);
+    assert(*c1.find(10) == 10);
+    assert(*c1.find(20) == 20);
+    assert(*c1.find(30) == 30);
+    assert(*c1.find(40) == 40);
+    assert(*c1.find(50) == 50);
+    assert(*c1.find(60) == 60);
+    assert(*c1.find(70) == 70);
+    assert(*c1.find(80) == 80);
+    assert(c1.hash_function() == Hash(2));
+    assert(c1.key_eq() == Compare(2));
+    assert(c1.get_allocator().get_id() == 1);
+    assert(static_cast<std::size_t>(std::distance(c1.begin(), c1.end())) == c1.size());
+    assert(static_cast<std::size_t>(std::distance(c1.cbegin(), c1.cend())) == c1.size());
+    assert(c1.max_load_factor() == 2);
+
+    LIBCPP_ASSERT(c2.bucket_count() == 0);
+    assert(c2.size() == 0);
+    assert(c2.hash_function() == Hash(1));
+    assert(c2.key_eq() == Compare(1));
+    assert(c2.get_allocator().get_id() == 2);
+    assert(static_cast<std::size_t>(std::distance(c2.begin(), c2.end())) == c2.size());
+    assert(static_cast<std::size_t>(std::distance(c2.cbegin(), c2.cend())) == c2.size());
+    assert(c2.max_load_factor() == 1);
+  }
+  {
+    typedef test_hash<int> Hash;
+    typedef test_equal_to<int> Compare;
+    typedef test_allocator<int> Alloc;
+    typedef std::unordered_set<int, Hash, Compare, Alloc> C;
+    typedef int P;
+    P a1[] = {P(1), P(2), P(3), P(4), P(1), P(2)};
+    C c1(std::begin(a1), std::end(a1), 0, Hash(1), Compare(1), Alloc(1, 1));
+    C c2(0, Hash(2), Compare(2), Alloc(1, 2));
+    c2.max_load_factor(2);
+    c1.swap(c2);
+
+    LIBCPP_ASSERT(c1.bucket_count() == 0);
+    assert(c1.size() == 0);
+    assert(c1.hash_function() == Hash(2));
+    assert(c1.key_eq() == Compare(2));
+    assert(c1.get_allocator().get_id() == 1);
+    assert(static_cast<std::size_t>(std::distance(c1.begin(), c1.end())) == c1.size());
+    assert(static_cast<std::size_t>(std::distance(c1.cbegin(), c1.cend())) == c1.size());
+    assert(c1.max_load_factor() == 2);
+
+    assert(c2.bucket_count() >= 4);
+    assert(c2.size() == 4);
+    assert(c2.count(1) == 1);
+    assert(c2.count(2) == 1);
+    assert(c2.count(3) == 1);
+    assert(c2.count(4) == 1);
+    assert(c2.hash_function() == Hash(1));
+    assert(c2.key_eq() == Compare(1));
+    assert(c2.get_allocator().get_id() == 2);
+    assert(static_cast<std::size_t>(std::distance(c2.begin(), c2.end())) == c2.size());
+    assert(static_cast<std::size_t>(std::distance(c2.cbegin(), c2.cend())) == c2.size());
+    assert(c2.max_load_factor() == 1);
+  }
+  {
+    typedef test_hash<int> Hash;
+    typedef test_equal_to<int> Compare;
+    typedef test_allocator<int> Alloc;
+    typedef std::unordered_set<int, Hash, Compare, Alloc> C;
+    typedef int P;
+    P a1[] = {P(1), P(2), P(3), P(4), P(1), P(2)};
+    P a2[] = {P(10), P(20), P(30), P(40), P(50), P(60), P(70), P(80)};
+    C c1(std::begin(a1), std::end(a1), 0, Hash(1), Compare(1), Alloc(1, 1));
+    C c2(std::begin(a2), std::end(a2), 0, Hash(2), Compare(2), Alloc(1, 2));
+    c2.max_load_factor(2);
+    c1.swap(c2);
+
+    assert(c1.bucket_count() >= 8);
+    assert(c1.size() == 8);
+    assert(*c1.find(10) == 10);
+    assert(*c1.find(20) == 20);
+    assert(*c1.find(30) == 30);
+    assert(*c1.find(40) == 40);
+    assert(*c1.find(50) == 50);
+    assert(*c1.find(60) == 60);
+    assert(*c1.find(70) == 70);
+    assert(*c1.find(80) == 80);
+    assert(c1.hash_function() == Hash(2));
+    assert(c1.key_eq() == Compare(2));
+    assert(c1.get_allocator().get_id() == 1);
+    assert(static_cast<std::size_t>(std::distance(c1.begin(), c1.end())) == c1.size());
+    assert(static_cast<std::size_t>(std::distance(c1.cbegin(), c1.cend())) == c1.size());
+    assert(c1.max_load_factor() == 2);
+
+    assert(c2.bucket_count() >= 4);
+    assert(c2.size() == 4);
+    assert(c2.count(1) == 1);
+    assert(c2.count(2) == 1);
+    assert(c2.count(3) == 1);
+    assert(c2.count(4) == 1);
+    assert(c2.hash_function() == Hash(1));
+    assert(c2.key_eq() == Compare(1));
+    assert(c2.get_allocator().get_id() == 2);
+    assert(static_cast<std::size_t>(std::distance(c2.begin(), c2.end())) == c2.size());
+    assert(static_cast<std::size_t>(std::distance(c2.cbegin(), c2.cend())) == c2.size());
+    assert(c2.max_load_factor() == 1);
+  }
+
+  {
+    typedef test_hash<int> Hash;
+    typedef test_equal_to<int> Compare;
+    typedef other_allocator<int> Alloc;
+    typedef std::unordered_set<int, Hash, Compare, Alloc> C;
+    C c1(0, Hash(1), Compare(1), Alloc(1));
+    C c2(0, Hash(2), Compare(2), Alloc(2));
+    c2.max_load_factor(2);
+    c1.swap(c2);
+
+    LIBCPP_ASSERT(c1.bucket_count() == 0);
+    assert(c1.size() == 0);
+    assert(c1.hash_function() == Hash(2));
+    assert(c1.key_eq() == Compare(2));
+    assert(c1.get_allocator() == Alloc(2));
+    assert(static_cast<std::size_t>(std::distance(c1.begin(), c1.end())) == c1.size());
+    assert(static_cast<std::size_t>(std::distance(c1.cbegin(), c1.cend())) == c1.size());
+    assert(c1.max_load_factor() == 2);
+
+    LIBCPP_ASSERT(c2.bucket_count() == 0);
+    assert(c2.size() == 0);
+    assert(c2.hash_function() == Hash(1));
+    assert(c2.key_eq() == Compare(1));
+    assert(c2.get_allocator() == Alloc(1));
+    assert(static_cast<std::size_t>(std::distance(c2.begin(), c2.end())) == c2.size());
+    assert(static_cast<std::size_t>(std::distance(c2.cbegin(), c2.cend())) == c2.size());
+    assert(c2.max_load_factor() == 1);
+  }
+  {
+    typedef test_hash<int> Hash;
+    typedef test_equal_to<int> Compare;
+    typedef other_allocator<int> Alloc;
+    typedef std::unordered_set<int, Hash, Compare, Alloc> C;
+    typedef int P;
+    P a2[] = {P(10), P(20), P(30), P(40), P(50), P(60), P(70), P(80)};
+    C c1(0, Hash(1), Compare(1), Alloc(1));
+    C c2(std::begin(a2), std::end(a2), 0, Hash(2), Compare(2), Alloc(2));
+    c2.max_load_factor(2);
+    c1.swap(c2);
+
+    assert(c1.bucket_count() >= 8);
+    assert(c1.size() == 8);
+    assert(*c1.find(10) == 10);
+    assert(*c1.find(20) == 20);
+    assert(*c1.find(30) == 30);
+    assert(*c1.find(40) == 40);
+    assert(*c1.find(50) == 50);
+    assert(*c1.find(60) == 60);
+    assert(*c1.find(70) == 70);
+    assert(*c1.find(80) == 80);
+    assert(c1.hash_function() == Hash(2));
+    assert(c1.key_eq() == Compare(2));
+    assert(c1.get_allocator() == Alloc(2));
+    assert(static_cast<std::size_t>(std::distance(c1.begin(), c1.end())) == c1.size());
+    assert(static_cast<std::size_t>(std::distance(c1.cbegin(), c1.cend())) == c1.size());
+    assert(c1.max_load_factor() == 2);
+
+    LIBCPP_ASSERT(c2.bucket_count() == 0);
+    assert(c2.size() == 0);
+    assert(c2.hash_function() == Hash(1));
+    assert(c2.key_eq() == Compare(1));
+    assert(c2.get_allocator() == Alloc(1));
+    assert(static_cast<std::size_t>(std::distance(c2.begin(), c2.end())) == c2.size());
+    assert(static_cast<std::size_t>(std::distance(c2.cbegin(), c2.cend())) == c2.size());
+    assert(c2.max_load_factor() == 1);
+  }
+  {
+    typedef test_hash<int> Hash;
+    typedef test_equal_to<int> Compare;
+    typedef other_allocator<int> Alloc;
+    typedef std::unordered_set<int, Hash, Compare, Alloc> C;
+    typedef int P;
+    P a1[] = {P(1), P(2), P(3), P(4), P(1), P(2)};
+    C c1(std::begin(a1), std::end(a1), 0, Hash(1), Compare(1), Alloc(1));
+    C c2(0, Hash(2), Compare(2), Alloc(2));
+    c2.max_load_factor(2);
+    c1.swap(c2);
+
+    LIBCPP_ASSERT(c1.bucket_count() == 0);
+    assert(c1.size() == 0);
+    assert(c1.hash_function() == Hash(2));
+    assert(c1.key_eq() == Compare(2));
+    assert(c1.get_allocator() == Alloc(2));
+    assert(static_cast<std::size_t>(std::distance(c1.begin(), c1.end())) == c1.size());
+    assert(static_cast<std::size_t>(std::distance(c1.cbegin(), c1.cend())) == c1.size());
+    assert(c1.max_load_factor() == 2);
+
+    assert(c2.bucket_count() >= 4);
+    assert(c2.size() == 4);
+    assert(c2.count(1) == 1);
+    assert(c2.count(2) == 1);
+    assert(c2.count(3) == 1);
+    assert(c2.count(4) == 1);
+    assert(c2.hash_function() == Hash(1));
+    assert(c2.key_eq() == Compare(1));
+    assert(c2.get_allocator() == Alloc(1));
+    assert(static_cast<std::size_t>(std::distance(c2.begin(), c2.end())) == c2.size());
+    assert(static_cast<std::size_t>(std::distance(c2.cbegin(), c2.cend())) == c2.size());
+    assert(c2.max_load_factor() == 1);
+  }
+  {
+    typedef test_hash<int> Hash;
+    typedef test_equal_to<int> Compare;
+    typedef other_allocator<int> Alloc;
+    typedef std::unordered_set<int, Hash, Compare, Alloc> C;
+    typedef int P;
+    P a1[] = {P(1), P(2), P(3), P(4), P(1), P(2)};
+    P a2[] = {P(10), P(20), P(30), P(40), P(50), P(60), P(70), P(80)};
+    C c1(std::begin(a1), std::end(a1), 0, Hash(1), Compare(1), Alloc(1));
+    C c2(std::begin(a2), std::end(a2), 0, Hash(2), Compare(2), Alloc(2));
+    c2.max_load_factor(2);
+    c1.swap(c2);
+
+    assert(c1.bucket_count() >= 8);
+    assert(c1.size() == 8);
+    assert(*c1.find(10) == 10);
+    assert(*c1.find(20) == 20);
+    assert(*c1.find(30) == 30);
+    assert(*c1.find(40) == 40);
+    assert(*c1.find(50) == 50);
+    assert(*c1.find(60) == 60);
+    assert(*c1.find(70) == 70);
+    assert(*c1.find(80) == 80);
+    assert(c1.hash_function() == Hash(2));
+    assert(c1.key_eq() == Compare(2));
+    assert(c1.get_allocator() == Alloc(2));
+    assert(static_cast<std::size_t>(std::distance(c1.begin(), c1.end())) == c1.size());
+    assert(static_cast<std::size_t>(std::distance(c1.cbegin(), c1.cend())) == c1.size());
+    assert(c1.max_load_factor() == 2);
+
+    assert(c2.bucket_count() >= 4);
+    assert(c2.size() == 4);
+    assert(c2.count(1) == 1);
+    assert(c2.count(2) == 1);
+    assert(c2.count(3) == 1);
+    assert(c2.count(4) == 1);
+    assert(c2.hash_function() == Hash(1));
+    assert(c2.key_eq() == Compare(1));
+    assert(c2.get_allocator() == Alloc(1));
+    assert(static_cast<std::size_t>(std::distance(c2.begin(), c2.end())) == c2.size());
+    assert(static_cast<std::size_t>(std::distance(c2.cbegin(), c2.cend())) == c2.size());
+    assert(c2.max_load_factor() == 1);
+  }
 #if TEST_STD_VER >= 11
-    {
-        typedef test_hash<int> Hash;
-        typedef test_equal_to<int> Compare;
-        typedef min_allocator<int> Alloc;
-        typedef std::unordered_set<int, Hash, Compare, Alloc> C;
-        C c1(0, Hash(1), Compare(1), Alloc());
-        C c2(0, Hash(2), Compare(2), Alloc());
-        c2.max_load_factor(2);
-        c1.swap(c2);
-
-        LIBCPP_ASSERT(c1.bucket_count() == 0);
-        assert(c1.size() == 0);
-        assert(c1.hash_function() == Hash(2));
-        assert(c1.key_eq() == Compare(2));
-        assert(c1.get_allocator() == Alloc());
-        assert(static_cast<std::size_t>(std::distance(c1.begin(), c1.end())) == c1.size());
-        assert(static_cast<std::size_t>(std::distance(c1.cbegin(), c1.cend())) == c1.size());
-        assert(c1.max_load_factor() == 2);
-
-        LIBCPP_ASSERT(c2.bucket_count() == 0);
-        assert(c2.size() == 0);
-        assert(c2.hash_function() == Hash(1));
-        assert(c2.key_eq() == Compare(1));
-        assert(c2.get_allocator() == Alloc());
-        assert(static_cast<std::size_t>(std::distance(c2.begin(), c2.end())) == c2.size());
-        assert(static_cast<std::size_t>(std::distance(c2.cbegin(), c2.cend())) == c2.size());
-        assert(c2.max_load_factor() == 1);
-    }
-    {
-        typedef test_hash<int> Hash;
-        typedef test_equal_to<int> Compare;
-        typedef min_allocator<int> Alloc;
-        typedef std::unordered_set<int, Hash, Compare, Alloc> C;
-        typedef int P;
-        P a2[] =
-        {
-            P(10),
-            P(20),
-            P(30),
-            P(40),
-            P(50),
-            P(60),
-            P(70),
-            P(80)
-        };
-        C c1(0, Hash(1), Compare(1), Alloc());
-        C c2(std::begin(a2), std::end(a2), 0, Hash(2), Compare(2), Alloc());
-        c2.max_load_factor(2);
-        c1.swap(c2);
-
-        assert(c1.bucket_count() >= 8);
-        assert(c1.size() == 8);
-        assert(*c1.find(10) == 10);
-        assert(*c1.find(20) == 20);
-        assert(*c1.find(30) == 30);
-        assert(*c1.find(40) == 40);
-        assert(*c1.find(50) == 50);
-        assert(*c1.find(60) == 60);
-        assert(*c1.find(70) == 70);
-        assert(*c1.find(80) == 80);
-        assert(c1.hash_function() == Hash(2));
-        assert(c1.key_eq() == Compare(2));
-        assert(c1.get_allocator() == Alloc());
-        assert(static_cast<std::size_t>(std::distance(c1.begin(), c1.end())) == c1.size());
-        assert(static_cast<std::size_t>(std::distance(c1.cbegin(), c1.cend())) == c1.size());
-        assert(c1.max_load_factor() == 2);
-
-        LIBCPP_ASSERT(c2.bucket_count() == 0);
-        assert(c2.size() == 0);
-        assert(c2.hash_function() == Hash(1));
-        assert(c2.key_eq() == Compare(1));
-        assert(c2.get_allocator() == Alloc());
-        assert(static_cast<std::size_t>(std::distance(c2.begin(), c2.end())) == c2.size());
-        assert(static_cast<std::size_t>(std::distance(c2.cbegin(), c2.cend())) == c2.size());
-        assert(c2.max_load_factor() == 1);
-    }
-    {
-        typedef test_hash<int> Hash;
-        typedef test_equal_to<int> Compare;
-        typedef min_allocator<int> Alloc;
-        typedef std::unordered_set<int, Hash, Compare, Alloc> C;
-        typedef int P;
-        P a1[] =
-        {
-            P(1),
-            P(2),
-            P(3),
-            P(4),
-            P(1),
-            P(2)
-        };
-        C c1(std::begin(a1), std::end(a1), 0, Hash(1), Compare(1), Alloc());
-        C c2(0, Hash(2), Compare(2), Alloc());
-        c2.max_load_factor(2);
-        c1.swap(c2);
-
-        LIBCPP_ASSERT(c1.bucket_count() == 0);
-        assert(c1.size() == 0);
-        assert(c1.hash_function() == Hash(2));
-        assert(c1.key_eq() == Compare(2));
-        assert(c1.get_allocator() == Alloc());
-        assert(static_cast<std::size_t>(std::distance(c1.begin(), c1.end())) == c1.size());
-        assert(static_cast<std::size_t>(std::distance(c1.cbegin(), c1.cend())) == c1.size());
-        assert(c1.max_load_factor() == 2);
-
-        assert(c2.bucket_count() >= 4);
-        assert(c2.size() == 4);
-        assert(c2.count(1) == 1);
-        assert(c2.count(2) == 1);
-        assert(c2.count(3) == 1);
-        assert(c2.count(4) == 1);
-        assert(c2.hash_function() == Hash(1));
-        assert(c2.key_eq() == Compare(1));
-        assert(c2.get_allocator() == Alloc());
-        assert(static_cast<std::size_t>(std::distance(c2.begin(), c2.end())) == c2.size());
-        assert(static_cast<std::size_t>(std::distance(c2.cbegin(), c2.cend())) == c2.size());
-        assert(c2.max_load_factor() == 1);
-    }
-    {
-        typedef test_hash<int> Hash;
-        typedef test_equal_to<int> Compare;
-        typedef min_allocator<int> Alloc;
-        typedef std::unordered_set<int, Hash, Compare, Alloc> C;
-        typedef int P;
-        P a1[] =
-        {
-            P(1),
-            P(2),
-            P(3),
-            P(4),
-            P(1),
-            P(2)
-        };
-        P a2[] =
-        {
-            P(10),
-            P(20),
-            P(30),
-            P(40),
-            P(50),
-            P(60),
-            P(70),
-            P(80)
-        };
-        C c1(std::begin(a1), std::end(a1), 0, Hash(1), Compare(1), Alloc());
-        C c2(std::begin(a2), std::end(a2), 0, Hash(2), Compare(2), Alloc());
-        c2.max_load_factor(2);
-        c1.swap(c2);
-
-        assert(c1.bucket_count() >= 8);
-        assert(c1.size() == 8);
-        assert(*c1.find(10) == 10);
-        assert(*c1.find(20) == 20);
-        assert(*c1.find(30) == 30);
-        assert(*c1.find(40) == 40);
-        assert(*c1.find(50) == 50);
-        assert(*c1.find(60) == 60);
-        assert(*c1.find(70) == 70);
-        assert(*c1.find(80) == 80);
-        assert(c1.hash_function() == Hash(2));
-        assert(c1.key_eq() == Compare(2));
-        assert(c1.get_allocator() == Alloc());
-        assert(static_cast<std::size_t>(std::distance(c1.begin(), c1.end())) == c1.size());
-        assert(static_cast<std::size_t>(std::distance(c1.cbegin(), c1.cend())) == c1.size());
-        assert(c1.max_load_factor() == 2);
-
-        assert(c2.bucket_count() >= 4);
-        assert(c2.size() == 4);
-        assert(c2.count(1) == 1);
-        assert(c2.count(2) == 1);
-        assert(c2.count(3) == 1);
-        assert(c2.count(4) == 1);
-        assert(c2.hash_function() == Hash(1));
-        assert(c2.key_eq() == Compare(1));
-        assert(c2.get_allocator() == Alloc());
-        assert(static_cast<std::size_t>(std::distance(c2.begin(), c2.end())) == c2.size());
-        assert(static_cast<std::size_t>(std::distance(c2.cbegin(), c2.cend())) == c2.size());
-        assert(c2.max_load_factor() == 1);
-    }
+  {
+    typedef test_hash<int> Hash;
+    typedef test_equal_to<int> Compare;
+    typedef min_allocator<int> Alloc;
+    typedef std::unordered_set<int, Hash, Compare, Alloc> C;
+    C c1(0, Hash(1), Compare(1), Alloc());
+    C c2(0, Hash(2), Compare(2), Alloc());
+    c2.max_load_factor(2);
+    c1.swap(c2);
+
+    LIBCPP_ASSERT(c1.bucket_count() == 0);
+    assert(c1.size() == 0);
+    assert(c1.hash_function() == Hash(2));
+    assert(c1.key_eq() == Compare(2));
+    assert(c1.get_allocator() == Alloc());
+    assert(static_cast<std::size_t>(std::distance(c1.begin(), c1.end())) == c1.size());
+    assert(static_cast<std::size_t>(std::distance(c1.cbegin(), c1.cend())) == c1.size());
+    assert(c1.max_load_factor() == 2);
+
+    LIBCPP_ASSERT(c2.bucket_count() == 0);
+    assert(c2.size() == 0);
+    assert(c2.hash_function() == Hash(1));
+    assert(c2.key_eq() == Compare(1));
+    assert(c2.get_allocator() == Alloc());
+    assert(static_cast<std::size_t>(std::distance(c2.begin(), c2.end())) == c2.size());
+    assert(static_cast<std::size_t>(std::distance(c2.cbegin(), c2.cend())) == c2.size());
+    assert(c2.max_load_factor() == 1);
+  }
+  {
+    typedef test_hash<int> Hash;
+    typedef test_equal_to<int> Compare;
+    typedef min_allocator<int> Alloc;
+    typedef std::unordered_set<int, Hash, Compare, Alloc> C;
+    typedef int P;
+    P a2[] = {P(10), P(20), P(30), P(40), P(50), P(60), P(70), P(80)};
+    C c1(0, Hash(1), Compare(1), Alloc());
+    C c2(std::begin(a2), std::end(a2), 0, Hash(2), Compare(2), Alloc());
+    c2.max_load_factor(2);
+    c1.swap(c2);
+
+    assert(c1.bucket_count() >= 8);
+    assert(c1.size() == 8);
+    assert(*c1.find(10) == 10);
+    assert(*c1.find(20) == 20);
+    assert(*c1.find(30) == 30);
+    assert(*c1.find(40) == 40);
+    assert(*c1.find(50) == 50);
+    assert(*c1.find(60) == 60);
+    assert(*c1.find(70) == 70);
+    assert(*c1.find(80) == 80);
+    assert(c1.hash_function() == Hash(2));
+    assert(c1.key_eq() == Compare(2));
+    assert(c1.get_allocator() == Alloc());
+    assert(static_cast<std::size_t>(std::distance(c1.begin(), c1.end())) == c1.size());
+    assert(static_cast<std::size_t>(std::distance(c1.cbegin(), c1.cend())) == c1.size());
+    assert(c1.max_load_factor() == 2);
+
+    LIBCPP_ASSERT(c2.bucket_count() == 0);
+    assert(c2.size() == 0);
+    assert(c2.hash_function() == Hash(1));
+    assert(c2.key_eq() == Compare(1));
+    assert(c2.get_allocator() == Alloc());
+    assert(static_cast<std::size_t>(std::distance(c2.begin(), c2.end())) == c2.size());
+    assert(static_cast<std::size_t>(std::distance(c2.cbegin(), c2.cend())) == c2.size());
+    assert(c2.max_load_factor() == 1);
+  }
+  {
+    typedef test_hash<int> Hash;
+    typedef test_equal_to<int> Compare;
+    typedef min_allocator<int> Alloc;
+    typedef std::unordered_set<int, Hash, Compare, Alloc> C;
+    typedef int P;
+    P a1[] = {P(1), P(2), P(3), P(4), P(1), P(2)};
+    C c1(std::begin(a1), std::end(a1), 0, Hash(1), Compare(1), Alloc());
+    C c2(0, Hash(2), Compare(2), Alloc());
+    c2.max_load_factor(2);
+    c1.swap(c2);
+
+    LIBCPP_ASSERT(c1.bucket_count() == 0);
+    assert(c1.size() == 0);
+    assert(c1.hash_function() == Hash(2));
+    assert(c1.key_eq() == Compare(2));
+    assert(c1.get_allocator() == Alloc());
+    assert(static_cast<std::size_t>(std::distance(c1.begin(), c1.end())) == c1.size());
+    assert(static_cast<std::size_t>(std::distance(c1.cbegin(), c1.cend())) == c1.size());
+    assert(c1.max_load_factor() == 2);
+
+    assert(c2.bucket_count() >= 4);
+    assert(c2.size() == 4);
+    assert(c2.count(1) == 1);
+    assert(c2.count(2) == 1);
+    assert(c2.count(3) == 1);
+    assert(c2.count(4) == 1);
+    assert(c2.hash_function() == Hash(1));
+    assert(c2.key_eq() == Compare(1));
+    assert(c2.get_allocator() == Alloc());
+    assert(static_cast<std::size_t>(std::distance(c2.begin(), c2.end())) == c2.size());
+    assert(static_cast<std::size_t>(std::distance(c2.cbegin(), c2.cend())) == c2.size());
+    assert(c2.max_load_factor() == 1);
+  }
+  {
+    typedef test_hash<int> Hash;
+    typedef test_equal_to<int> Compare;
+    typedef min_allocator<int> Alloc;
+    typedef std::unordered_set<int, Hash, Compare, Alloc> C;
+    typedef int P;
+    P a1[] = {P(1), P(2), P(3), P(4), P(1), P(2)};
+    P a2[] = {P(10), P(20), P(30), P(40), P(50), P(60), P(70), P(80)};
+    C c1(std::begin(a1), std::end(a1), 0, Hash(1), Compare(1), Alloc());
+    C c2(std::begin(a2), std::end(a2), 0, Hash(2), Compare(2), Alloc());
+    c2.max_load_factor(2);
+    c1.swap(c2);
+
+    assert(c1.bucket_count() >= 8);
+    assert(c1.size() == 8);
+    assert(*c1.find(10) == 10);
+    assert(*c1.find(20) == 20);
+    assert(*c1.find(30) == 30);
+    assert(*c1.find(40) == 40);
+    assert(*c1.find(50) == 50);
+    assert(*c1.find(60) == 60);
+    assert(*c1.find(70) == 70);
+    assert(*c1.find(80) == 80);
+    assert(c1.hash_function() == Hash(2));
+    assert(c1.key_eq() == Compare(2));
+    assert(c1.get_allocator() == Alloc());
+    assert(static_cast<std::size_t>(std::distance(c1.begin(), c1.end())) == c1.size());
+    assert(static_cast<std::size_t>(std::distance(c1.cbegin(), c1.cend())) == c1.size());
+    assert(c1.max_load_factor() == 2);
+
+    assert(c2.bucket_count() >= 4);
+    assert(c2.size() == 4);
+    assert(c2.count(1) == 1);
+    assert(c2.count(2) == 1);
+    assert(c2.count(3) == 1);
+    assert(c2.count(4) == 1);
+    assert(c2.hash_function() == Hash(1));
+    assert(c2.key_eq() == Compare(1));
+    assert(c2.get_allocator() == Alloc());
+    assert(static_cast<std::size_t>(std::distance(c2.begin(), c2.end())) == c2.size());
+    assert(static_cast<std::size_t>(std::distance(c2.cbegin(), c2.cend())) == c2.size());
+    assert(c2.max_load_factor() == 1);
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/unord/unord.set/types.pass.cpp b/libcxx/test/std/containers/unord/unord.set/types.pass.cpp
index 8d5a22fdb24cf..623da69aba239 100644
--- a/libcxx/test/std/containers/unord/unord.set/types.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.set/types.pass.cpp
@@ -32,39 +32,37 @@
 #include "test_macros.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef std::unordered_set<short> C;
-        static_assert((std::is_same<C::value_type, short>::value), "");
-        static_assert((std::is_same<C::key_type, short>::value), "");
-        static_assert((std::is_same<C::hasher, std::hash<C::key_type> >::value), "");
-        static_assert((std::is_same<C::key_equal, std::equal_to<C::key_type> >::value), "");
-        static_assert((std::is_same<C::allocator_type, std::allocator<C::value_type> >::value), "");
-        static_assert((std::is_same<C::reference, C::value_type&>::value), "");
-        static_assert((std::is_same<C::const_reference, const C::value_type&>::value), "");
-        static_assert((std::is_same<C::pointer, C::value_type*>::value), "");
-        static_assert((std::is_same<C::const_pointer, const C::value_type*>::value), "");
-        static_assert((std::is_same<C::size_type, std::size_t>::value), "");
-        static_assert((std::is_same<C::difference_type, std::ptrdiff_t>::value), "");
-    }
+int main(int, char**) {
+  {
+    typedef std::unordered_set<short> C;
+    static_assert((std::is_same<C::value_type, short>::value), "");
+    static_assert((std::is_same<C::key_type, short>::value), "");
+    static_assert((std::is_same<C::hasher, std::hash<C::key_type> >::value), "");
+    static_assert((std::is_same<C::key_equal, std::equal_to<C::key_type> >::value), "");
+    static_assert((std::is_same<C::allocator_type, std::allocator<C::value_type> >::value), "");
+    static_assert((std::is_same<C::reference, C::value_type&>::value), "");
+    static_assert((std::is_same<C::const_reference, const C::value_type&>::value), "");
+    static_assert((std::is_same<C::pointer, C::value_type*>::value), "");
+    static_assert((std::is_same<C::const_pointer, const C::value_type*>::value), "");
+    static_assert((std::is_same<C::size_type, std::size_t>::value), "");
+    static_assert((std::is_same<C::difference_type, std::ptrdiff_t>::value), "");
+  }
 #if TEST_STD_VER >= 11
-    {
-        typedef std::unordered_set<short, std::hash<short>,
-                                  std::equal_to<short>, min_allocator<short>> C;
-        static_assert((std::is_same<C::value_type, short>::value), "");
-        static_assert((std::is_same<C::key_type, short>::value), "");
-        static_assert((std::is_same<C::hasher, std::hash<C::key_type> >::value), "");
-        static_assert((std::is_same<C::key_equal, std::equal_to<C::key_type> >::value), "");
-        static_assert((std::is_same<C::allocator_type, min_allocator<C::value_type> >::value), "");
-        static_assert((std::is_same<C::reference, C::value_type&>::value), "");
-        static_assert((std::is_same<C::const_reference, const C::value_type&>::value), "");
-        static_assert((std::is_same<C::pointer, min_pointer<C::value_type>>::value), "");
-        static_assert((std::is_same<C::const_pointer, min_pointer<const C::value_type>>::value), "");
+  {
+    typedef std::unordered_set<short, std::hash<short>, std::equal_to<short>, min_allocator<short>> C;
+    static_assert((std::is_same<C::value_type, short>::value), "");
+    static_assert((std::is_same<C::key_type, short>::value), "");
+    static_assert((std::is_same<C::hasher, std::hash<C::key_type> >::value), "");
+    static_assert((std::is_same<C::key_equal, std::equal_to<C::key_type> >::value), "");
+    static_assert((std::is_same<C::allocator_type, min_allocator<C::value_type> >::value), "");
+    static_assert((std::is_same<C::reference, C::value_type&>::value), "");
+    static_assert((std::is_same<C::const_reference, const C::value_type&>::value), "");
+    static_assert((std::is_same<C::pointer, min_pointer<C::value_type>>::value), "");
+    static_assert((std::is_same<C::const_pointer, min_pointer<const C::value_type>>::value), "");
     //  min_allocator doesn't have a size_type, so one gets synthesized
-        static_assert((std::is_same<C::size_type, std::make_unsigned<C::difference_type>::type>::value), "");
-        static_assert((std::is_same<C::difference_type, std::ptrdiff_t>::value), "");
-    }
+    static_assert((std::is_same<C::size_type, std::make_unsigned<C::difference_type>::type>::value), "");
+    static_assert((std::is_same<C::difference_type, std::ptrdiff_t>::value), "");
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/unord/unord.set/unord.set.cnstr/allocator.pass.cpp b/libcxx/test/std/containers/unord/unord.set/unord.set.cnstr/allocator.pass.cpp
index 7c7ce3d077bb8..e3d5979f7d13b 100644
--- a/libcxx/test/std/containers/unord/unord.set/unord.set.cnstr/allocator.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.set/unord.set.cnstr/allocator.pass.cpp
@@ -24,87 +24,86 @@
 #include "test_allocator.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef std::unordered_set<NotConstructible,
-                                   test_hash<NotConstructible>,
-                                   test_equal_to<NotConstructible>,
-                                   test_allocator<NotConstructible>
-                                   > C;
-        C c(test_allocator<NotConstructible>(10));
-        LIBCPP_ASSERT(c.bucket_count() == 0);
-        assert(c.hash_function() == test_hash<NotConstructible>());
-        assert(c.key_eq() == test_equal_to<NotConstructible>());
-        assert(c.get_allocator() == test_allocator<NotConstructible>(10));
-        assert(c.size() == 0);
-        assert(c.empty());
-        assert(std::distance(c.begin(), c.end()) == 0);
-        assert(c.load_factor() == 0);
-        assert(c.max_load_factor() == 1);
-    }
+int main(int, char**) {
+  {
+    typedef std::unordered_set<NotConstructible,
+                               test_hash<NotConstructible>,
+                               test_equal_to<NotConstructible>,
+                               test_allocator<NotConstructible> >
+        C;
+    C c(test_allocator<NotConstructible>(10));
+    LIBCPP_ASSERT(c.bucket_count() == 0);
+    assert(c.hash_function() == test_hash<NotConstructible>());
+    assert(c.key_eq() == test_equal_to<NotConstructible>());
+    assert(c.get_allocator() == test_allocator<NotConstructible>(10));
+    assert(c.size() == 0);
+    assert(c.empty());
+    assert(std::distance(c.begin(), c.end()) == 0);
+    assert(c.load_factor() == 0);
+    assert(c.max_load_factor() == 1);
+  }
 #if TEST_STD_VER >= 11
-    {
-        typedef std::unordered_set<NotConstructible,
-                                   test_hash<NotConstructible>,
-                                   test_equal_to<NotConstructible>,
-                                   min_allocator<NotConstructible>
-                                   > C;
-        C c(min_allocator<NotConstructible>{});
-        LIBCPP_ASSERT(c.bucket_count() == 0);
-        assert(c.hash_function() == test_hash<NotConstructible>());
-        assert(c.key_eq() == test_equal_to<NotConstructible>());
-        assert(c.get_allocator() == min_allocator<NotConstructible>());
-        assert(c.size() == 0);
-        assert(c.empty());
-        assert(std::distance(c.begin(), c.end()) == 0);
-        assert(c.load_factor() == 0);
-        assert(c.max_load_factor() == 1);
-    }
-#if TEST_STD_VER > 11
-    {
-        typedef NotConstructible T;
-        typedef test_hash<T> HF;
-        typedef test_equal_to<T> Comp;
-        typedef test_allocator<T> A;
-        typedef std::unordered_set<T, HF, Comp, A> C;
+  {
+    typedef std::unordered_set<NotConstructible,
+                               test_hash<NotConstructible>,
+                               test_equal_to<NotConstructible>,
+                               min_allocator<NotConstructible> >
+        C;
+    C c(min_allocator<NotConstructible>{});
+    LIBCPP_ASSERT(c.bucket_count() == 0);
+    assert(c.hash_function() == test_hash<NotConstructible>());
+    assert(c.key_eq() == test_equal_to<NotConstructible>());
+    assert(c.get_allocator() == min_allocator<NotConstructible>());
+    assert(c.size() == 0);
+    assert(c.empty());
+    assert(std::distance(c.begin(), c.end()) == 0);
+    assert(c.load_factor() == 0);
+    assert(c.max_load_factor() == 1);
+  }
+#  if TEST_STD_VER > 11
+  {
+    typedef NotConstructible T;
+    typedef test_hash<T> HF;
+    typedef test_equal_to<T> Comp;
+    typedef test_allocator<T> A;
+    typedef std::unordered_set<T, HF, Comp, A> C;
 
-        A a(43);
-        C c(3, a);
-        LIBCPP_ASSERT(c.bucket_count() == 3);
-        assert(c.hash_function() == HF());
-        assert(c.key_eq() == Comp ());
-        assert(c.get_allocator() == a);
-        assert(!(c.get_allocator() == A()));
-        assert(c.size() == 0);
-        assert(c.empty());
-        assert(std::distance(c.begin(), c.end()) == 0);
-        assert(c.load_factor() == 0);
-        assert(c.max_load_factor() == 1);
-    }
-    {
-        typedef NotConstructible T;
-        typedef test_hash<T> HF;
-        typedef test_equal_to<T> Comp;
-        typedef test_allocator<T> A;
-        typedef std::unordered_set<T, HF, Comp, A> C;
+    A a(43);
+    C c(3, a);
+    LIBCPP_ASSERT(c.bucket_count() == 3);
+    assert(c.hash_function() == HF());
+    assert(c.key_eq() == Comp());
+    assert(c.get_allocator() == a);
+    assert(!(c.get_allocator() == A()));
+    assert(c.size() == 0);
+    assert(c.empty());
+    assert(std::distance(c.begin(), c.end()) == 0);
+    assert(c.load_factor() == 0);
+    assert(c.max_load_factor() == 1);
+  }
+  {
+    typedef NotConstructible T;
+    typedef test_hash<T> HF;
+    typedef test_equal_to<T> Comp;
+    typedef test_allocator<T> A;
+    typedef std::unordered_set<T, HF, Comp, A> C;
 
-        HF hf(42);
-        A a(43);
-        C c(4, hf, a);
-        LIBCPP_ASSERT(c.bucket_count() == 4);
-        assert(c.hash_function() == hf);
-        assert(!(c.hash_function() == HF()));
-        assert(c.key_eq() == Comp ());
-        assert(c.get_allocator() == a);
-        assert(!(c.get_allocator() == A()));
-        assert(c.size() == 0);
-        assert(c.empty());
-        assert(std::distance(c.begin(), c.end()) == 0);
-        assert(c.load_factor() == 0);
-        assert(c.max_load_factor() == 1);
-    }
-#endif
+    HF hf(42);
+    A a(43);
+    C c(4, hf, a);
+    LIBCPP_ASSERT(c.bucket_count() == 4);
+    assert(c.hash_function() == hf);
+    assert(!(c.hash_function() == HF()));
+    assert(c.key_eq() == Comp());
+    assert(c.get_allocator() == a);
+    assert(!(c.get_allocator() == A()));
+    assert(c.size() == 0);
+    assert(c.empty());
+    assert(std::distance(c.begin(), c.end()) == 0);
+    assert(c.load_factor() == 0);
+    assert(c.max_load_factor() == 1);
+  }
+#  endif
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/unord/unord.set/unord.set.cnstr/assign_copy.pass.cpp b/libcxx/test/std/containers/unord/unord.set/unord.set.cnstr/assign_copy.pass.cpp
index 74b3b0744f988..25851e941a7d1 100644
--- a/libcxx/test/std/containers/unord/unord.set/unord.set.cnstr/assign_copy.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.set/unord.set.cnstr/assign_copy.pass.cpp
@@ -27,162 +27,87 @@
 #include "test_allocator.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef test_allocator<int> A;
-        typedef std::unordered_set<int,
-                                   test_hash<int>,
-                                   test_equal_to<int>,
-                                   A
-                                   > C;
-        typedef int P;
-        P a[] =
-        {
-            P(1),
-            P(2),
-            P(3),
-            P(4),
-            P(1),
-            P(2)
-        };
-        C c0(a, a + sizeof(a)/sizeof(a[0]),
-            7,
-            test_hash<int>(8),
-            test_equal_to<int>(9),
-            A(10)
-           );
-        C c(a, a + 2,
-            7,
-            test_hash<int>(2),
-            test_equal_to<int>(3),
-            A(4)
-           );
-        c = c0;
-        LIBCPP_ASSERT(c.bucket_count() == 7);
-        assert(c.size() == 4);
-        assert(c.count(1) == 1);
-        assert(c.count(2) == 1);
-        assert(c.count(3) == 1);
-        assert(c.count(4) == 1);
-        assert(c.hash_function() == test_hash<int>(8));
-        assert(c.key_eq() == test_equal_to<int>(9));
-        assert(c.get_allocator() == A(4));
-        assert(!c.empty());
-        assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-        assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
-        assert(fabs(c.load_factor() - (float)c.size()/c.bucket_count()) < FLT_EPSILON);
-        assert(c.max_load_factor() == 1);
-    }
-    {
-        typedef std::unordered_set<int> C;
-        typedef int P;
-        P a[] =
-        {
-            P(1),
-            P(2),
-            P(3),
-            P(4),
-            P(1),
-            P(2)
-        };
-        C c(a, a + sizeof(a)/sizeof(a[0]));
-        C *p = &c;
-        c = *p;
-        assert(c.size() == 4);
-        assert(std::is_permutation(c.begin(), c.end(), a));
-    }
-    {
-        typedef other_allocator<int> A;
-        typedef std::unordered_set<int,
-                                   test_hash<int>,
-                                   test_equal_to<int>,
-                                   A
-                                   > C;
-        typedef int P;
-        P a[] =
-        {
-            P(1),
-            P(2),
-            P(3),
-            P(4),
-            P(1),
-            P(2)
-        };
-        C c0(a, a + sizeof(a)/sizeof(a[0]),
-            7,
-            test_hash<int>(8),
-            test_equal_to<int>(9),
-            A(10)
-           );
-        C c(a, a + 2,
-            7,
-            test_hash<int>(2),
-            test_equal_to<int>(3),
-            A(4)
-           );
-        c = c0;
-        assert(c.bucket_count() >= 5);
-        assert(c.size() == 4);
-        assert(c.count(1) == 1);
-        assert(c.count(2) == 1);
-        assert(c.count(3) == 1);
-        assert(c.count(4) == 1);
-        assert(c.hash_function() == test_hash<int>(8));
-        assert(c.key_eq() == test_equal_to<int>(9));
-        assert(c.get_allocator() == A(10));
-        assert(!c.empty());
-        assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-        assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
-        assert(fabs(c.load_factor() - (float)c.size()/c.bucket_count()) < FLT_EPSILON);
-        assert(c.max_load_factor() == 1);
-    }
+int main(int, char**) {
+  {
+    typedef test_allocator<int> A;
+    typedef std::unordered_set<int, test_hash<int>, test_equal_to<int>, A > C;
+    typedef int P;
+    P a[] = {P(1), P(2), P(3), P(4), P(1), P(2)};
+    C c0(a, a + sizeof(a) / sizeof(a[0]), 7, test_hash<int>(8), test_equal_to<int>(9), A(10));
+    C c(a, a + 2, 7, test_hash<int>(2), test_equal_to<int>(3), A(4));
+    c = c0;
+    LIBCPP_ASSERT(c.bucket_count() == 7);
+    assert(c.size() == 4);
+    assert(c.count(1) == 1);
+    assert(c.count(2) == 1);
+    assert(c.count(3) == 1);
+    assert(c.count(4) == 1);
+    assert(c.hash_function() == test_hash<int>(8));
+    assert(c.key_eq() == test_equal_to<int>(9));
+    assert(c.get_allocator() == A(4));
+    assert(!c.empty());
+    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
+    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
+    assert(fabs(c.load_factor() - (float)c.size() / c.bucket_count()) < FLT_EPSILON);
+    assert(c.max_load_factor() == 1);
+  }
+  {
+    typedef std::unordered_set<int> C;
+    typedef int P;
+    P a[] = {P(1), P(2), P(3), P(4), P(1), P(2)};
+    C c(a, a + sizeof(a) / sizeof(a[0]));
+    C* p = &c;
+    c    = *p;
+    assert(c.size() == 4);
+    assert(std::is_permutation(c.begin(), c.end(), a));
+  }
+  {
+    typedef other_allocator<int> A;
+    typedef std::unordered_set<int, test_hash<int>, test_equal_to<int>, A > C;
+    typedef int P;
+    P a[] = {P(1), P(2), P(3), P(4), P(1), P(2)};
+    C c0(a, a + sizeof(a) / sizeof(a[0]), 7, test_hash<int>(8), test_equal_to<int>(9), A(10));
+    C c(a, a + 2, 7, test_hash<int>(2), test_equal_to<int>(3), A(4));
+    c = c0;
+    assert(c.bucket_count() >= 5);
+    assert(c.size() == 4);
+    assert(c.count(1) == 1);
+    assert(c.count(2) == 1);
+    assert(c.count(3) == 1);
+    assert(c.count(4) == 1);
+    assert(c.hash_function() == test_hash<int>(8));
+    assert(c.key_eq() == test_equal_to<int>(9));
+    assert(c.get_allocator() == A(10));
+    assert(!c.empty());
+    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
+    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
+    assert(fabs(c.load_factor() - (float)c.size() / c.bucket_count()) < FLT_EPSILON);
+    assert(c.max_load_factor() == 1);
+  }
 #if TEST_STD_VER >= 11
-    {
-        typedef min_allocator<int> A;
-        typedef std::unordered_set<int,
-                                   test_hash<int>,
-                                   test_equal_to<int>,
-                                   A
-                                   > C;
-        typedef int P;
-        P a[] =
-        {
-            P(1),
-            P(2),
-            P(3),
-            P(4),
-            P(1),
-            P(2)
-        };
-        C c0(a, a + sizeof(a)/sizeof(a[0]),
-            7,
-            test_hash<int>(8),
-            test_equal_to<int>(9),
-            A()
-           );
-        C c(a, a + 2,
-            7,
-            test_hash<int>(2),
-            test_equal_to<int>(3),
-            A()
-           );
-        c = c0;
-        LIBCPP_ASSERT(c.bucket_count() == 7);
-        assert(c.size() == 4);
-        assert(c.count(1) == 1);
-        assert(c.count(2) == 1);
-        assert(c.count(3) == 1);
-        assert(c.count(4) == 1);
-        assert(c.hash_function() == test_hash<int>(8));
-        assert(c.key_eq() == test_equal_to<int>(9));
-        assert(c.get_allocator() == A());
-        assert(!c.empty());
-        assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-        assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
-        assert(fabs(c.load_factor() - (float)c.size()/c.bucket_count()) < FLT_EPSILON);
-        assert(c.max_load_factor() == 1);
-    }
+  {
+    typedef min_allocator<int> A;
+    typedef std::unordered_set<int, test_hash<int>, test_equal_to<int>, A > C;
+    typedef int P;
+    P a[] = {P(1), P(2), P(3), P(4), P(1), P(2)};
+    C c0(a, a + sizeof(a) / sizeof(a[0]), 7, test_hash<int>(8), test_equal_to<int>(9), A());
+    C c(a, a + 2, 7, test_hash<int>(2), test_equal_to<int>(3), A());
+    c = c0;
+    LIBCPP_ASSERT(c.bucket_count() == 7);
+    assert(c.size() == 4);
+    assert(c.count(1) == 1);
+    assert(c.count(2) == 1);
+    assert(c.count(3) == 1);
+    assert(c.count(4) == 1);
+    assert(c.hash_function() == test_hash<int>(8));
+    assert(c.key_eq() == test_equal_to<int>(9));
+    assert(c.get_allocator() == A());
+    assert(!c.empty());
+    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
+    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
+    assert(fabs(c.load_factor() - (float)c.size() / c.bucket_count()) < FLT_EPSILON);
+    assert(c.max_load_factor() == 1);
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/unord/unord.set/unord.set.cnstr/assign_init.pass.cpp b/libcxx/test/std/containers/unord/unord.set/unord.set.cnstr/assign_init.pass.cpp
index cc35d51ef9d7b..c2850b8e000b9 100644
--- a/libcxx/test/std/containers/unord/unord.set/unord.set.cnstr/assign_init.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.set/unord.set.cnstr/assign_init.pass.cpp
@@ -28,72 +28,41 @@
 #include "test_allocator.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef test_allocator<int> A;
-        typedef std::unordered_set<int,
-                                   test_hash<int>,
-                                   test_equal_to<int>,
-                                   A
-                                   > C;
-        typedef int P;
-        C c =   {
-                    P(4),
-                    P(1),
-                    P(2)
-                };
-        c =     {
-                    P(1),
-                    P(2),
-                    P(3),
-                    P(4),
-                    P(1),
-                    P(2)
-                };
-        assert(c.bucket_count() >= 5);
-        assert(c.size() == 4);
-        assert(c.count(1) == 1);
-        assert(c.count(2) == 1);
-        assert(c.count(3) == 1);
-        assert(c.count(4) == 1);
-        assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-        assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
-        assert(fabs(c.load_factor() - (float)c.size()/c.bucket_count()) < FLT_EPSILON);
-        assert(c.max_load_factor() == 1);
-    }
-    {
-        typedef min_allocator<int> A;
-        typedef std::unordered_set<int,
-                                   test_hash<int>,
-                                   test_equal_to<int>,
-                                   A
-                                   > C;
-        typedef int P;
-        C c =   {
-                    P(4),
-                    P(1),
-                    P(2)
-                };
-        c =     {
-                    P(1),
-                    P(2),
-                    P(3),
-                    P(4),
-                    P(1),
-                    P(2)
-                };
-        assert(c.bucket_count() >= 5);
-        assert(c.size() == 4);
-        assert(c.count(1) == 1);
-        assert(c.count(2) == 1);
-        assert(c.count(3) == 1);
-        assert(c.count(4) == 1);
-        assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-        assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
-        assert(fabs(c.load_factor() - (float)c.size()/c.bucket_count()) < FLT_EPSILON);
-        assert(c.max_load_factor() == 1);
-    }
+int main(int, char**) {
+  {
+    typedef test_allocator<int> A;
+    typedef std::unordered_set<int, test_hash<int>, test_equal_to<int>, A > C;
+    typedef int P;
+    C c = {P(4), P(1), P(2)};
+    c   = {P(1), P(2), P(3), P(4), P(1), P(2)};
+    assert(c.bucket_count() >= 5);
+    assert(c.size() == 4);
+    assert(c.count(1) == 1);
+    assert(c.count(2) == 1);
+    assert(c.count(3) == 1);
+    assert(c.count(4) == 1);
+    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
+    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
+    assert(fabs(c.load_factor() - (float)c.size() / c.bucket_count()) < FLT_EPSILON);
+    assert(c.max_load_factor() == 1);
+  }
+  {
+    typedef min_allocator<int> A;
+    typedef std::unordered_set<int, test_hash<int>, test_equal_to<int>, A > C;
+    typedef int P;
+    C c = {P(4), P(1), P(2)};
+    c   = {P(1), P(2), P(3), P(4), P(1), P(2)};
+    assert(c.bucket_count() >= 5);
+    assert(c.size() == 4);
+    assert(c.count(1) == 1);
+    assert(c.count(2) == 1);
+    assert(c.count(3) == 1);
+    assert(c.count(4) == 1);
+    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
+    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
+    assert(fabs(c.load_factor() - (float)c.size() / c.bucket_count()) < FLT_EPSILON);
+    assert(c.max_load_factor() == 1);
+  }
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/unord/unord.set/unord.set.cnstr/assign_move.pass.cpp b/libcxx/test/std/containers/unord/unord.set/unord.set.cnstr/assign_move.pass.cpp
index b1ec876e704f5..8d5ae4831e358 100644
--- a/libcxx/test/std/containers/unord/unord.set/unord.set.cnstr/assign_move.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.set/unord.set.cnstr/assign_move.pass.cpp
@@ -28,194 +28,105 @@
 #include "test_allocator.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef test_allocator<int> A;
-        typedef std::unordered_set<int,
-                                   test_hash<int>,
-                                   test_equal_to<int>,
-                                   A
-                                   > C;
-        typedef int P;
-        P a[] =
-        {
-            P(1),
-            P(2),
-            P(3),
-            P(4),
-            P(1),
-            P(2)
-        };
-        C c0(a, a + sizeof(a)/sizeof(a[0]),
-            7,
-            test_hash<int>(8),
-            test_equal_to<int>(9),
-            A(10)
-           );
-        C c(a, a + 2,
-            7,
-            test_hash<int>(2),
-            test_equal_to<int>(3),
-            A(4)
-           );
-        c = std::move(c0);
-        LIBCPP_ASSERT(c.bucket_count() == 7);
-        assert(c.size() == 4);
-        assert(c.count(1) == 1);
-        assert(c.count(2) == 1);
-        assert(c.count(3) == 1);
-        assert(c.count(4) == 1);
-        assert(c.hash_function() == test_hash<int>(8));
-        assert(c.key_eq() == test_equal_to<int>(9));
-        assert(c.get_allocator() == A(4));
-        assert(!c.empty());
-        assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-        assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
-        assert(fabs(c.load_factor() - (float)c.size()/c.bucket_count()) < FLT_EPSILON);
-        assert(c.max_load_factor() == 1);
-    }
-    {
-        typedef test_allocator<int> A;
-        typedef std::unordered_set<int,
-                                   test_hash<int>,
-                                   test_equal_to<int>,
-                                   A
-                                   > C;
-        typedef int P;
-        P a[] =
-        {
-            P(1),
-            P(2),
-            P(3),
-            P(4),
-            P(1),
-            P(2)
-        };
-        C c0(a, a + sizeof(a)/sizeof(a[0]),
-            7,
-            test_hash<int>(8),
-            test_equal_to<int>(9),
-            A(10)
-           );
-        C c(a, a + 2,
-            7,
-            test_hash<int>(2),
-            test_equal_to<int>(3),
-            A(10)
-           );
-        C::iterator it0 = c0.begin();
-        c = std::move(c0);
-        LIBCPP_ASSERT(c.bucket_count() == 7);
-        assert(c.size() == 4);
-        assert(c.count(1) == 1);
-        assert(c.count(2) == 1);
-        assert(c.count(3) == 1);
-        assert(c.count(4) == 1);
-        assert(c.hash_function() == test_hash<int>(8));
-        assert(c.key_eq() == test_equal_to<int>(9));
-        assert(c.get_allocator() == A(10));
-        assert(!c.empty());
-        assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-        assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
-        assert(fabs(c.load_factor() - (float)c.size()/c.bucket_count()) < FLT_EPSILON);
-        assert(c.max_load_factor() == 1);
-        assert(it0 == c.begin()); // Iterators remain valid
-    }
-    {
-        typedef other_allocator<int> A;
-        typedef std::unordered_set<int,
-                                   test_hash<int>,
-                                   test_equal_to<int>,
-                                   A
-                                   > C;
-        typedef int P;
-        P a[] =
-        {
-            P(1),
-            P(2),
-            P(3),
-            P(4),
-            P(1),
-            P(2)
-        };
-        C c0(a, a + sizeof(a)/sizeof(a[0]),
-            7,
-            test_hash<int>(8),
-            test_equal_to<int>(9),
-            A(10)
-           );
-        C c(a, a + 2,
-            7,
-            test_hash<int>(2),
-            test_equal_to<int>(3),
-            A(4)
-           );
-        C::iterator it0 = c0.begin();
-        c = std::move(c0);
-        LIBCPP_ASSERT(c.bucket_count() == 7);
-        assert(c.size() == 4);
-        assert(c.count(1) == 1);
-        assert(c.count(2) == 1);
-        assert(c.count(3) == 1);
-        assert(c.count(4) == 1);
-        assert(c.hash_function() == test_hash<int>(8));
-        assert(c.key_eq() == test_equal_to<int>(9));
-        assert(c.get_allocator() == A(10));
-        assert(!c.empty());
-        assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-        assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
-        assert(fabs(c.load_factor() - (float)c.size()/c.bucket_count()) < FLT_EPSILON);
-        assert(c.max_load_factor() == 1);
-        assert(it0 == c.begin()); // Iterators remain valid
-    }
-    {
-        typedef min_allocator<int> A;
-        typedef std::unordered_set<int,
-                                   test_hash<int>,
-                                   test_equal_to<int>,
-                                   A
-                                   > C;
-        typedef int P;
-        P a[] =
-        {
-            P(1),
-            P(2),
-            P(3),
-            P(4),
-            P(1),
-            P(2)
-        };
-        C c0(a, a + sizeof(a)/sizeof(a[0]),
-            7,
-            test_hash<int>(8),
-            test_equal_to<int>(9),
-            A()
-           );
-        C c(a, a + 2,
-            7,
-            test_hash<int>(2),
-            test_equal_to<int>(3),
-            A()
-           );
-        C::iterator it0 = c0.begin();
-        c = std::move(c0);
-        LIBCPP_ASSERT(c.bucket_count() == 7);
-        assert(c.size() == 4);
-        assert(c.count(1) == 1);
-        assert(c.count(2) == 1);
-        assert(c.count(3) == 1);
-        assert(c.count(4) == 1);
-        assert(c.hash_function() == test_hash<int>(8));
-        assert(c.key_eq() == test_equal_to<int>(9));
-        assert(c.get_allocator() == A());
-        assert(!c.empty());
-        assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-        assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
-        assert(fabs(c.load_factor() - (float)c.size()/c.bucket_count()) < FLT_EPSILON);
-        assert(c.max_load_factor() == 1);
-        assert(it0 == c.begin()); // Iterators remain valid
-    }
+int main(int, char**) {
+  {
+    typedef test_allocator<int> A;
+    typedef std::unordered_set<int, test_hash<int>, test_equal_to<int>, A > C;
+    typedef int P;
+    P a[] = {P(1), P(2), P(3), P(4), P(1), P(2)};
+    C c0(a, a + sizeof(a) / sizeof(a[0]), 7, test_hash<int>(8), test_equal_to<int>(9), A(10));
+    C c(a, a + 2, 7, test_hash<int>(2), test_equal_to<int>(3), A(4));
+    c = std::move(c0);
+    LIBCPP_ASSERT(c.bucket_count() == 7);
+    assert(c.size() == 4);
+    assert(c.count(1) == 1);
+    assert(c.count(2) == 1);
+    assert(c.count(3) == 1);
+    assert(c.count(4) == 1);
+    assert(c.hash_function() == test_hash<int>(8));
+    assert(c.key_eq() == test_equal_to<int>(9));
+    assert(c.get_allocator() == A(4));
+    assert(!c.empty());
+    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
+    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
+    assert(fabs(c.load_factor() - (float)c.size() / c.bucket_count()) < FLT_EPSILON);
+    assert(c.max_load_factor() == 1);
+  }
+  {
+    typedef test_allocator<int> A;
+    typedef std::unordered_set<int, test_hash<int>, test_equal_to<int>, A > C;
+    typedef int P;
+    P a[] = {P(1), P(2), P(3), P(4), P(1), P(2)};
+    C c0(a, a + sizeof(a) / sizeof(a[0]), 7, test_hash<int>(8), test_equal_to<int>(9), A(10));
+    C c(a, a + 2, 7, test_hash<int>(2), test_equal_to<int>(3), A(10));
+    C::iterator it0 = c0.begin();
+    c               = std::move(c0);
+    LIBCPP_ASSERT(c.bucket_count() == 7);
+    assert(c.size() == 4);
+    assert(c.count(1) == 1);
+    assert(c.count(2) == 1);
+    assert(c.count(3) == 1);
+    assert(c.count(4) == 1);
+    assert(c.hash_function() == test_hash<int>(8));
+    assert(c.key_eq() == test_equal_to<int>(9));
+    assert(c.get_allocator() == A(10));
+    assert(!c.empty());
+    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
+    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
+    assert(fabs(c.load_factor() - (float)c.size() / c.bucket_count()) < FLT_EPSILON);
+    assert(c.max_load_factor() == 1);
+    assert(it0 == c.begin()); // Iterators remain valid
+  }
+  {
+    typedef other_allocator<int> A;
+    typedef std::unordered_set<int, test_hash<int>, test_equal_to<int>, A > C;
+    typedef int P;
+    P a[] = {P(1), P(2), P(3), P(4), P(1), P(2)};
+    C c0(a, a + sizeof(a) / sizeof(a[0]), 7, test_hash<int>(8), test_equal_to<int>(9), A(10));
+    C c(a, a + 2, 7, test_hash<int>(2), test_equal_to<int>(3), A(4));
+    C::iterator it0 = c0.begin();
+    c               = std::move(c0);
+    LIBCPP_ASSERT(c.bucket_count() == 7);
+    assert(c.size() == 4);
+    assert(c.count(1) == 1);
+    assert(c.count(2) == 1);
+    assert(c.count(3) == 1);
+    assert(c.count(4) == 1);
+    assert(c.hash_function() == test_hash<int>(8));
+    assert(c.key_eq() == test_equal_to<int>(9));
+    assert(c.get_allocator() == A(10));
+    assert(!c.empty());
+    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
+    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
+    assert(fabs(c.load_factor() - (float)c.size() / c.bucket_count()) < FLT_EPSILON);
+    assert(c.max_load_factor() == 1);
+    assert(it0 == c.begin()); // Iterators remain valid
+  }
+  {
+    typedef min_allocator<int> A;
+    typedef std::unordered_set<int, test_hash<int>, test_equal_to<int>, A > C;
+    typedef int P;
+    P a[] = {P(1), P(2), P(3), P(4), P(1), P(2)};
+    C c0(a, a + sizeof(a) / sizeof(a[0]), 7, test_hash<int>(8), test_equal_to<int>(9), A());
+    C c(a, a + 2, 7, test_hash<int>(2), test_equal_to<int>(3), A());
+    C::iterator it0 = c0.begin();
+    c               = std::move(c0);
+    LIBCPP_ASSERT(c.bucket_count() == 7);
+    assert(c.size() == 4);
+    assert(c.count(1) == 1);
+    assert(c.count(2) == 1);
+    assert(c.count(3) == 1);
+    assert(c.count(4) == 1);
+    assert(c.hash_function() == test_hash<int>(8));
+    assert(c.key_eq() == test_equal_to<int>(9));
+    assert(c.get_allocator() == A());
+    assert(!c.empty());
+    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
+    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
+    assert(fabs(c.load_factor() - (float)c.size() / c.bucket_count()) < FLT_EPSILON);
+    assert(c.max_load_factor() == 1);
+    assert(it0 == c.begin()); // Iterators remain valid
+  }
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/unord/unord.set/unord.set.cnstr/compare_copy_constructible.compile.fail.cpp b/libcxx/test/std/containers/unord/unord.set/unord.set.cnstr/compare_copy_constructible.compile.fail.cpp
index 03258628ee268..09df0edd00f51 100644
--- a/libcxx/test/std/containers/unord/unord.set/unord.set.cnstr/compare_copy_constructible.compile.fail.cpp
+++ b/libcxx/test/std/containers/unord/unord.set/unord.set.cnstr/compare_copy_constructible.compile.fail.cpp
@@ -6,7 +6,6 @@
 //
 //===----------------------------------------------------------------------===//
 
-
 // <unordered_set>
 
 // Check that std::unordered_set fails to instantiate if the comparison predicate is
@@ -16,16 +15,16 @@
 
 template <class T>
 struct Comp {
-    bool operator () (const T& lhs, const T& rhs) const { return lhs == rhs; }
+  bool operator()(const T& lhs, const T& rhs) const { return lhs == rhs; }
 
-    Comp () {}
-private:
-    Comp (const Comp &); // declared but not defined
-    };
+  Comp() {}
 
+private:
+  Comp(const Comp&); // declared but not defined
+};
 
 int main(int, char**) {
-    std::unordered_set<int, std::hash<int>, Comp<int> > m;
+  std::unordered_set<int, std::hash<int>, Comp<int> > m;
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/unord/unord.set/unord.set.cnstr/copy.pass.cpp b/libcxx/test/std/containers/unord/unord.set/unord.set.cnstr/copy.pass.cpp
index 8a675950d9baa..e795d810e9fa1 100644
--- a/libcxx/test/std/containers/unord/unord.set/unord.set.cnstr/copy.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.set/unord.set.cnstr/copy.pass.cpp
@@ -26,123 +26,71 @@
 #include "test_allocator.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef std::unordered_set<int,
-                                   test_hash<int>,
-                                   test_equal_to<int>,
-                                   test_allocator<int>
-                                   > C;
-        typedef int P;
-        P a[] =
-        {
-            P(1),
-            P(2),
-            P(3),
-            P(4),
-            P(1),
-            P(2)
-        };
-        C c0(a, a + sizeof(a)/sizeof(a[0]),
-            7,
-            test_hash<int>(8),
-            test_equal_to<int>(9),
-            test_allocator<int>(10)
-           );
-        C c = c0;
-        LIBCPP_ASSERT(c.bucket_count() == 7);
-        assert(c.size() == 4);
-        assert(c.count(1) == 1);
-        assert(c.count(2) == 1);
-        assert(c.count(3) == 1);
-        assert(c.count(4) == 1);
-        assert(c.hash_function() == test_hash<int>(8));
-        assert(c.key_eq() == test_equal_to<int>(9));
-        assert(c.get_allocator() == test_allocator<int>(10));
-        assert(!c.empty());
-        assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-        assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
-        assert(std::fabs(c.load_factor() - (float)c.size()/c.bucket_count()) < FLT_EPSILON);
-        assert(c.max_load_factor() == 1);
-    }
+int main(int, char**) {
+  {
+    typedef std::unordered_set<int, test_hash<int>, test_equal_to<int>, test_allocator<int> > C;
+    typedef int P;
+    P a[] = {P(1), P(2), P(3), P(4), P(1), P(2)};
+    C c0(a, a + sizeof(a) / sizeof(a[0]), 7, test_hash<int>(8), test_equal_to<int>(9), test_allocator<int>(10));
+    C c = c0;
+    LIBCPP_ASSERT(c.bucket_count() == 7);
+    assert(c.size() == 4);
+    assert(c.count(1) == 1);
+    assert(c.count(2) == 1);
+    assert(c.count(3) == 1);
+    assert(c.count(4) == 1);
+    assert(c.hash_function() == test_hash<int>(8));
+    assert(c.key_eq() == test_equal_to<int>(9));
+    assert(c.get_allocator() == test_allocator<int>(10));
+    assert(!c.empty());
+    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
+    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
+    assert(std::fabs(c.load_factor() - (float)c.size() / c.bucket_count()) < FLT_EPSILON);
+    assert(c.max_load_factor() == 1);
+  }
 #if TEST_STD_VER >= 11
-    {
-        typedef std::unordered_set<int,
-                                   test_hash<int>,
-                                   test_equal_to<int>,
-                                   other_allocator<int>
-                                   > C;
-        typedef int P;
-        P a[] =
-        {
-            P(1),
-            P(2),
-            P(3),
-            P(4),
-            P(1),
-            P(2)
-        };
-        C c0(a, a + sizeof(a)/sizeof(a[0]),
-            7,
-            test_hash<int>(8),
-            test_equal_to<int>(9),
-            other_allocator<int>(10)
-           );
-        C c = c0;
-        LIBCPP_ASSERT(c.bucket_count() == 7);
-        assert(c.size() == 4);
-        assert(c.count(1) == 1);
-        assert(c.count(2) == 1);
-        assert(c.count(3) == 1);
-        assert(c.count(4) == 1);
-        assert(c.hash_function() == test_hash<int>(8));
-        assert(c.key_eq() == test_equal_to<int>(9));
-        assert(c.get_allocator() == other_allocator<int>(-2));
-        assert(!c.empty());
-        assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-        assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
-        assert(std::fabs(c.load_factor() - (float)c.size()/c.bucket_count()) < FLT_EPSILON);
-        assert(c.max_load_factor() == 1);
-    }
-    {
-        typedef std::unordered_set<int,
-                                   test_hash<int>,
-                                   test_equal_to<int>,
-                                   min_allocator<int>
-                                   > C;
-        typedef int P;
-        P a[] =
-        {
-            P(1),
-            P(2),
-            P(3),
-            P(4),
-            P(1),
-            P(2)
-        };
-        C c0(a, a + sizeof(a)/sizeof(a[0]),
-            7,
-            test_hash<int>(8),
-            test_equal_to<int>(9),
-            min_allocator<int>()
-           );
-        C c = c0;
-        LIBCPP_ASSERT(c.bucket_count() == 7);
-        assert(c.size() == 4);
-        assert(c.count(1) == 1);
-        assert(c.count(2) == 1);
-        assert(c.count(3) == 1);
-        assert(c.count(4) == 1);
-        assert(c.hash_function() == test_hash<int>(8));
-        assert(c.key_eq() == test_equal_to<int>(9));
-        assert(c.get_allocator() == min_allocator<int>());
-        assert(!c.empty());
-        assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-        assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
-        assert(std::fabs(c.load_factor() - (float)c.size()/c.bucket_count()) < FLT_EPSILON);
-        assert(c.max_load_factor() == 1);
-    }
+  {
+    typedef std::unordered_set<int, test_hash<int>, test_equal_to<int>, other_allocator<int> > C;
+    typedef int P;
+    P a[] = {P(1), P(2), P(3), P(4), P(1), P(2)};
+    C c0(a, a + sizeof(a) / sizeof(a[0]), 7, test_hash<int>(8), test_equal_to<int>(9), other_allocator<int>(10));
+    C c = c0;
+    LIBCPP_ASSERT(c.bucket_count() == 7);
+    assert(c.size() == 4);
+    assert(c.count(1) == 1);
+    assert(c.count(2) == 1);
+    assert(c.count(3) == 1);
+    assert(c.count(4) == 1);
+    assert(c.hash_function() == test_hash<int>(8));
+    assert(c.key_eq() == test_equal_to<int>(9));
+    assert(c.get_allocator() == other_allocator<int>(-2));
+    assert(!c.empty());
+    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
+    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
+    assert(std::fabs(c.load_factor() - (float)c.size() / c.bucket_count()) < FLT_EPSILON);
+    assert(c.max_load_factor() == 1);
+  }
+  {
+    typedef std::unordered_set<int, test_hash<int>, test_equal_to<int>, min_allocator<int> > C;
+    typedef int P;
+    P a[] = {P(1), P(2), P(3), P(4), P(1), P(2)};
+    C c0(a, a + sizeof(a) / sizeof(a[0]), 7, test_hash<int>(8), test_equal_to<int>(9), min_allocator<int>());
+    C c = c0;
+    LIBCPP_ASSERT(c.bucket_count() == 7);
+    assert(c.size() == 4);
+    assert(c.count(1) == 1);
+    assert(c.count(2) == 1);
+    assert(c.count(3) == 1);
+    assert(c.count(4) == 1);
+    assert(c.hash_function() == test_hash<int>(8));
+    assert(c.key_eq() == test_equal_to<int>(9));
+    assert(c.get_allocator() == min_allocator<int>());
+    assert(!c.empty());
+    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
+    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
+    assert(std::fabs(c.load_factor() - (float)c.size() / c.bucket_count()) < FLT_EPSILON);
+    assert(c.max_load_factor() == 1);
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/unord/unord.set/unord.set.cnstr/copy_alloc.pass.cpp b/libcxx/test/std/containers/unord/unord.set/unord.set.cnstr/copy_alloc.pass.cpp
index 09d22de4ed0d9..49a5f9406e1e1 100644
--- a/libcxx/test/std/containers/unord/unord.set/unord.set.cnstr/copy_alloc.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.set/unord.set.cnstr/copy_alloc.pass.cpp
@@ -26,85 +26,50 @@
 #include "test_allocator.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef std::unordered_set<int,
-                                   test_hash<int>,
-                                   test_equal_to<int>,
-                                   test_allocator<int>
-                                   > C;
-        typedef int P;
-        P a[] =
-        {
-            P(1),
-            P(2),
-            P(3),
-            P(4),
-            P(1),
-            P(2)
-        };
-        C c0(a, a + sizeof(a)/sizeof(a[0]),
-            7,
-            test_hash<int>(8),
-            test_equal_to<int>(9),
-            test_allocator<int>(10)
-           );
-        C c(c0, test_allocator<int>(5));
-        LIBCPP_ASSERT(c.bucket_count() == 7);
-        assert(c.size() == 4);
-        assert(c.count(1) == 1);
-        assert(c.count(2) == 1);
-        assert(c.count(3) == 1);
-        assert(c.count(4) == 1);
-        assert(c.hash_function() == test_hash<int>(8));
-        assert(c.key_eq() == test_equal_to<int>(9));
-        assert(c.get_allocator() == test_allocator<int>(5));
-        assert(!c.empty());
-        assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-        assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
-        assert(std::fabs(c.load_factor() - (float)c.size()/c.bucket_count()) < FLT_EPSILON);
-        assert(c.max_load_factor() == 1);
-    }
+int main(int, char**) {
+  {
+    typedef std::unordered_set<int, test_hash<int>, test_equal_to<int>, test_allocator<int> > C;
+    typedef int P;
+    P a[] = {P(1), P(2), P(3), P(4), P(1), P(2)};
+    C c0(a, a + sizeof(a) / sizeof(a[0]), 7, test_hash<int>(8), test_equal_to<int>(9), test_allocator<int>(10));
+    C c(c0, test_allocator<int>(5));
+    LIBCPP_ASSERT(c.bucket_count() == 7);
+    assert(c.size() == 4);
+    assert(c.count(1) == 1);
+    assert(c.count(2) == 1);
+    assert(c.count(3) == 1);
+    assert(c.count(4) == 1);
+    assert(c.hash_function() == test_hash<int>(8));
+    assert(c.key_eq() == test_equal_to<int>(9));
+    assert(c.get_allocator() == test_allocator<int>(5));
+    assert(!c.empty());
+    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
+    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
+    assert(std::fabs(c.load_factor() - (float)c.size() / c.bucket_count()) < FLT_EPSILON);
+    assert(c.max_load_factor() == 1);
+  }
 #if TEST_STD_VER >= 11
-    {
-        typedef std::unordered_set<int,
-                                   test_hash<int>,
-                                   test_equal_to<int>,
-                                   min_allocator<int>
-                                   > C;
-        typedef int P;
-        P a[] =
-        {
-            P(1),
-            P(2),
-            P(3),
-            P(4),
-            P(1),
-            P(2)
-        };
-        C c0(a, a + sizeof(a)/sizeof(a[0]),
-            7,
-            test_hash<int>(8),
-            test_equal_to<int>(9),
-            min_allocator<int>()
-           );
-        C c(c0, min_allocator<int>());
-        LIBCPP_ASSERT(c.bucket_count() == 7);
-        assert(c.size() == 4);
-        assert(c.count(1) == 1);
-        assert(c.count(2) == 1);
-        assert(c.count(3) == 1);
-        assert(c.count(4) == 1);
-        assert(c.hash_function() == test_hash<int>(8));
-        assert(c.key_eq() == test_equal_to<int>(9));
-        assert(c.get_allocator() == min_allocator<int>());
-        assert(!c.empty());
-        assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-        assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
-        assert(std::fabs(c.load_factor() - (float)c.size()/c.bucket_count()) < FLT_EPSILON);
-        assert(c.max_load_factor() == 1);
-    }
+  {
+    typedef std::unordered_set<int, test_hash<int>, test_equal_to<int>, min_allocator<int> > C;
+    typedef int P;
+    P a[] = {P(1), P(2), P(3), P(4), P(1), P(2)};
+    C c0(a, a + sizeof(a) / sizeof(a[0]), 7, test_hash<int>(8), test_equal_to<int>(9), min_allocator<int>());
+    C c(c0, min_allocator<int>());
+    LIBCPP_ASSERT(c.bucket_count() == 7);
+    assert(c.size() == 4);
+    assert(c.count(1) == 1);
+    assert(c.count(2) == 1);
+    assert(c.count(3) == 1);
+    assert(c.count(4) == 1);
+    assert(c.hash_function() == test_hash<int>(8));
+    assert(c.key_eq() == test_equal_to<int>(9));
+    assert(c.get_allocator() == min_allocator<int>());
+    assert(!c.empty());
+    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
+    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
+    assert(std::fabs(c.load_factor() - (float)c.size() / c.bucket_count()) < FLT_EPSILON);
+    assert(c.max_load_factor() == 1);
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/unord/unord.set/unord.set.cnstr/deduct.pass.cpp b/libcxx/test/std/containers/unord/unord.set/unord.set.cnstr/deduct.pass.cpp
index 13e42f37fd21f..c748986883eee 100644
--- a/libcxx/test/std/containers/unord/unord.set/unord.set.cnstr/deduct.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.set/unord.set.cnstr/deduct.pass.cpp
@@ -81,196 +81,199 @@
 #include "deduction_guides_sfinae_checks.h"
 #include "test_allocator.h"
 
-int main(int, char**)
-{
-    const int expected_s[] = {1, 2, 3, INT_MAX};
+int main(int, char**) {
+  const int expected_s[] = {1, 2, 3, INT_MAX};
 
-    {
-    const int arr[] = { 1, 2, 1, INT_MAX, 3 };
+  {
+    const int arr[] = {1, 2, 1, INT_MAX, 3};
     std::unordered_set s(std::begin(arr), std::end(arr));
 
     ASSERT_SAME_TYPE(decltype(s), std::unordered_set<int>);
     assert(std::is_permutation(s.begin(), s.end(), std::begin(expected_s), std::end(expected_s)));
-    }
+  }
 
-    {
-    const int arr[] = { 1, 2, 1, INT_MAX, 3 };
+  {
+    const int arr[] = {1, 2, 1, INT_MAX, 3};
     std::unordered_set s(std::begin(arr), std::end(arr), 42);
 
     ASSERT_SAME_TYPE(decltype(s), std::unordered_set<int>);
     assert(std::is_permutation(s.begin(), s.end(), std::begin(expected_s), std::end(expected_s)));
-    }
+  }
 
-    {
-    const int arr[] = { 1, 2, 1, INT_MAX, 3 };
+  {
+    const int arr[] = {1, 2, 1, INT_MAX, 3};
     std::unordered_set s(std::begin(arr), std::end(arr), 42, std::hash<long long>());
 
     ASSERT_SAME_TYPE(decltype(s), std::unordered_set<int, std::hash<long long>>);
     assert(std::is_permutation(s.begin(), s.end(), std::begin(expected_s), std::end(expected_s)));
-    }
+  }
 
-    {
-    const int arr[] = { 1, 2, 1, INT_MAX, 3 };
+  {
+    const int arr[] = {1, 2, 1, INT_MAX, 3};
     std::unordered_set s(std::begin(arr), std::end(arr), 42, std::hash<long long>(), test_allocator<int>(0, 40));
 
-    ASSERT_SAME_TYPE(decltype(s), std::unordered_set<int, std::hash<long long>, std::equal_to<int>, test_allocator<int>>);
+    ASSERT_SAME_TYPE(
+        decltype(s), std::unordered_set<int, std::hash<long long>, std::equal_to<int>, test_allocator<int>>);
     assert(std::is_permutation(s.begin(), s.end(), std::begin(expected_s), std::end(expected_s)));
     assert(s.get_allocator().get_id() == 40);
-    }
+  }
 
-    {
+  {
     std::unordered_set<int, std::hash<long long>, std::equal_to<>, test_allocator<int>> source;
     std::unordered_set s(source);
     ASSERT_SAME_TYPE(decltype(s), decltype(source));
     assert(s.size() == 0);
-    }
+  }
 
-    {
+  {
     std::unordered_set<int, std::hash<long long>, std::equal_to<>, test_allocator<int>> source;
-    std::unordered_set s{source};  // braces instead of parens
+    std::unordered_set s{source}; // braces instead of parens
     ASSERT_SAME_TYPE(decltype(s), decltype(source));
     assert(s.size() == 0);
-    }
+  }
 
-    {
+  {
     std::unordered_set<int, std::hash<long long>, std::equal_to<>, test_allocator<int>> source;
     std::unordered_set s(source, test_allocator<int>(0, 41));
     ASSERT_SAME_TYPE(decltype(s), decltype(source));
     assert(s.size() == 0);
     assert(s.get_allocator().get_id() == 41);
-    }
+  }
 
-    {
+  {
     std::unordered_set<int, std::hash<long long>, std::equal_to<>, test_allocator<int>> source;
-    std::unordered_set s{source, test_allocator<int>(0, 42)};  // braces instead of parens
+    std::unordered_set s{source, test_allocator<int>(0, 42)}; // braces instead of parens
     ASSERT_SAME_TYPE(decltype(s), decltype(source));
     assert(s.size() == 0);
     assert(s.get_allocator().get_id() == 42);
-    }
+  }
 
-    {
-    std::unordered_set s{ 1, 2, 1, INT_MAX, 3 };
+  {
+    std::unordered_set s{1, 2, 1, INT_MAX, 3};
 
     ASSERT_SAME_TYPE(decltype(s), std::unordered_set<int>);
     assert(std::is_permutation(s.begin(), s.end(), std::begin(expected_s), std::end(expected_s)));
-    }
+  }
 
-    {
-    std::unordered_set s({ 1, 2, 1, INT_MAX, 3 }, 42);
+  {
+    std::unordered_set s({1, 2, 1, INT_MAX, 3}, 42);
 
     ASSERT_SAME_TYPE(decltype(s), std::unordered_set<int>);
     assert(std::is_permutation(s.begin(), s.end(), std::begin(expected_s), std::end(expected_s)));
-    }
+  }
 
-    {
-    std::unordered_set s({ 1, 2, 1, INT_MAX, 3 }, 42, std::hash<long long>());
+  {
+    std::unordered_set s({1, 2, 1, INT_MAX, 3}, 42, std::hash<long long>());
 
     ASSERT_SAME_TYPE(decltype(s), std::unordered_set<int, std::hash<long long>>);
     assert(std::is_permutation(s.begin(), s.end(), std::begin(expected_s), std::end(expected_s)));
-    }
+  }
 
-    {
-    std::unordered_set s({ 1, 2, 1, INT_MAX, 3 }, 42, std::hash<long long>(), std::equal_to<>());
+  {
+    std::unordered_set s({1, 2, 1, INT_MAX, 3}, 42, std::hash<long long>(), std::equal_to<>());
 
     ASSERT_SAME_TYPE(decltype(s), std::unordered_set<int, std::hash<long long>, std::equal_to<>>);
     assert(std::is_permutation(s.begin(), s.end(), std::begin(expected_s), std::end(expected_s)));
-    }
+  }
 
-    {
-    std::unordered_set s({ 1, 2, 1, INT_MAX, 3 }, 42, std::hash<long long>(), std::equal_to<>(), test_allocator<int>(0, 43));
+  {
+    std::unordered_set s(
+        {1, 2, 1, INT_MAX, 3}, 42, std::hash<long long>(), std::equal_to<>(), test_allocator<int>(0, 43));
 
     ASSERT_SAME_TYPE(decltype(s), std::unordered_set<int, std::hash<long long>, std::equal_to<>, test_allocator<int>>);
     assert(std::is_permutation(s.begin(), s.end(), std::begin(expected_s), std::end(expected_s)));
     assert(s.get_allocator().get_id() == 43);
-    }
+  }
 
-    {
-    const int arr[] = { 1, 2, 1, INT_MAX, 3 };
+  {
+    const int arr[] = {1, 2, 1, INT_MAX, 3};
     std::unordered_set s(std::begin(arr), std::end(arr), 42, test_allocator<int>(0, 44));
 
     ASSERT_SAME_TYPE(decltype(s), std::unordered_set<int, std::hash<int>, std::equal_to<int>, test_allocator<int>>);
     assert(std::is_permutation(s.begin(), s.end(), std::begin(expected_s), std::end(expected_s)));
     assert(s.get_allocator().get_id() == 44);
-    }
+  }
 
-    {
-    const int arr[] = { 1, 2, 1, INT_MAX, 3 };
+  {
+    const int arr[] = {1, 2, 1, INT_MAX, 3};
     std::unordered_set s(std::begin(arr), std::end(arr), 42, std::hash<long long>(), test_allocator<int>(0, 44));
 
-    ASSERT_SAME_TYPE(decltype(s), std::unordered_set<int, std::hash<long long>, std::equal_to<int>, test_allocator<int>>);
+    ASSERT_SAME_TYPE(
+        decltype(s), std::unordered_set<int, std::hash<long long>, std::equal_to<int>, test_allocator<int>>);
     assert(std::is_permutation(s.begin(), s.end(), std::begin(expected_s), std::end(expected_s)));
     assert(s.get_allocator().get_id() == 44);
-    }
+  }
 
-    {
-    std::unordered_set s({ 1, 2, 1, INT_MAX, 3 }, 42, test_allocator<int>(0, 43));
+  {
+    std::unordered_set s({1, 2, 1, INT_MAX, 3}, 42, test_allocator<int>(0, 43));
 
     ASSERT_SAME_TYPE(decltype(s), std::unordered_set<int, std::hash<int>, std::equal_to<int>, test_allocator<int>>);
     assert(std::is_permutation(s.begin(), s.end(), std::begin(expected_s), std::end(expected_s)));
     assert(s.get_allocator().get_id() == 43);
-    }
+  }
 
-    {
-    std::unordered_set s({ 1, 2, 1, INT_MAX, 3 }, 42, std::hash<long long>(), test_allocator<int>(0, 42));
+  {
+    std::unordered_set s({1, 2, 1, INT_MAX, 3}, 42, std::hash<long long>(), test_allocator<int>(0, 42));
 
-    ASSERT_SAME_TYPE(decltype(s), std::unordered_set<int, std::hash<long long>, std::equal_to<int>, test_allocator<int>>);
+    ASSERT_SAME_TYPE(
+        decltype(s), std::unordered_set<int, std::hash<long long>, std::equal_to<int>, test_allocator<int>>);
     assert(std::is_permutation(s.begin(), s.end(), std::begin(expected_s), std::end(expected_s)));
     assert(s.get_allocator().get_id() == 42);
-    }
+  }
 
 #if TEST_STD_VER >= 23
-    {
-      using Range = std::array<int, 0>;
-      using Pred = test_equal_to<int>;
-      using DefaultPred = std::equal_to<int>;
-      using Hash = test_hash<int>;
-      using DefaultHash = std::hash<int>;
-      using Alloc = test_allocator<int>;
-
-      { // (from_range, range)
-        std::unordered_set c(std::from_range, Range());
-        static_assert(std::is_same_v<decltype(c), std::unordered_set<int>>);
-      }
-
-      { // (from_range, range, n)
-        std::unordered_set c(std::from_range, Range(), std::size_t());
-        static_assert(std::is_same_v<decltype(c), std::unordered_set<int>>);
-      }
-
-      { // (from_range, range, n, hash)
-        std::unordered_set c(std::from_range, Range(), std::size_t(), Hash());
-        static_assert(std::is_same_v<decltype(c), std::unordered_set<int, Hash>>);
-      }
-
-      { // (from_range, range, n, hash, pred)
-        std::unordered_set c(std::from_range, Range(), std::size_t(), Hash(), Pred());
-        static_assert(std::is_same_v<decltype(c), std::unordered_set<int, Hash, Pred>>);
-      }
-
-      { // (from_range, range, n, hash, pred, alloc)
-        std::unordered_set c(std::from_range, Range(), std::size_t(), Hash(), Pred(), Alloc());
-        static_assert(std::is_same_v<decltype(c), std::unordered_set<int, Hash, Pred, Alloc>>);
-      }
-
-      { // (from_range, range, n, alloc)
-        std::unordered_set c(std::from_range, Range(), std::size_t(), Alloc());
-        static_assert(std::is_same_v<decltype(c), std::unordered_set<int, DefaultHash, DefaultPred, Alloc>>);
-      }
-
-      // TODO(LWG 2713): uncomment this test once the constructor is added.
-      { // (from_range, range, alloc)
-        //std::unordered_set c(std::from_range, Range(), Alloc());
-        //static_assert(std::is_same_v<decltype(c), std::unordered_set<int, DefaultHash, DefaultPred, Alloc>>);
-      }
-
-      { // (from_range, range, n, hash, alloc)
-        std::unordered_set c(std::from_range, Range(), std::size_t(), Hash(), Alloc());
-        static_assert(std::is_same_v<decltype(c), std::unordered_set<int, Hash, DefaultPred, Alloc>>);
-      }
+  {
+    using Range       = std::array<int, 0>;
+    using Pred        = test_equal_to<int>;
+    using DefaultPred = std::equal_to<int>;
+    using Hash        = test_hash<int>;
+    using DefaultHash = std::hash<int>;
+    using Alloc       = test_allocator<int>;
+
+    { // (from_range, range)
+      std::unordered_set c(std::from_range, Range());
+      static_assert(std::is_same_v<decltype(c), std::unordered_set<int>>);
+    }
+
+    { // (from_range, range, n)
+      std::unordered_set c(std::from_range, Range(), std::size_t());
+      static_assert(std::is_same_v<decltype(c), std::unordered_set<int>>);
+    }
+
+    { // (from_range, range, n, hash)
+      std::unordered_set c(std::from_range, Range(), std::size_t(), Hash());
+      static_assert(std::is_same_v<decltype(c), std::unordered_set<int, Hash>>);
+    }
+
+    { // (from_range, range, n, hash, pred)
+      std::unordered_set c(std::from_range, Range(), std::size_t(), Hash(), Pred());
+      static_assert(std::is_same_v<decltype(c), std::unordered_set<int, Hash, Pred>>);
+    }
+
+    { // (from_range, range, n, hash, pred, alloc)
+      std::unordered_set c(std::from_range, Range(), std::size_t(), Hash(), Pred(), Alloc());
+      static_assert(std::is_same_v<decltype(c), std::unordered_set<int, Hash, Pred, Alloc>>);
+    }
+
+    { // (from_range, range, n, alloc)
+      std::unordered_set c(std::from_range, Range(), std::size_t(), Alloc());
+      static_assert(std::is_same_v<decltype(c), std::unordered_set<int, DefaultHash, DefaultPred, Alloc>>);
+    }
+
+    // TODO(LWG 2713): uncomment this test once the constructor is added.
+    { // (from_range, range, alloc)
+      //std::unordered_set c(std::from_range, Range(), Alloc());
+      //static_assert(std::is_same_v<decltype(c), std::unordered_set<int, DefaultHash, DefaultPred, Alloc>>);
+    }
+
+    { // (from_range, range, n, hash, alloc)
+      std::unordered_set c(std::from_range, Range(), std::size_t(), Hash(), Alloc());
+      static_assert(std::is_same_v<decltype(c), std::unordered_set<int, Hash, DefaultPred, Alloc>>);
     }
+  }
 #endif
 
-    UnorderedContainerDeductionGuidesSfinaeAway<std::unordered_set, std::unordered_set<int>>();
+  UnorderedContainerDeductionGuidesSfinaeAway<std::unordered_set, std::unordered_set<int>>();
 
-    return 0;
+  return 0;
 }
diff --git a/libcxx/test/std/containers/unord/unord.set/unord.set.cnstr/deduct.verify.cpp b/libcxx/test/std/containers/unord/unord.set/unord.set.cnstr/deduct.verify.cpp
index 26e5f9ae6ce43..147b1f53ceafa 100644
--- a/libcxx/test/std/containers/unord/unord.set/unord.set.cnstr/deduct.verify.cpp
+++ b/libcxx/test/std/containers/unord/unord.set/unord.set.cnstr/deduct.verify.cpp
@@ -50,48 +50,47 @@
 #include <functional>
 #include <unordered_set>
 
-int main(int, char**)
-{
-    {
-        // cannot deduce Key from nothing
-        std::unordered_set s;
-            // expected-error-re@-1{{no viable constructor or deduction guide for deduction of template arguments of '{{(std::)?}}unordered_set'}}
-    }
-    {
-        // cannot deduce Key from just (Size)
-        std::unordered_set s(42);
-            // expected-error-re@-1{{no viable constructor or deduction guide for deduction of template arguments of '{{(std::)?}}unordered_set'}}
-    }
-    {
-        // cannot deduce Key from just (Size, Hash)
-        std::unordered_set s(42, std::hash<int>());
-            // expected-error-re@-1{{no viable constructor or deduction guide for deduction of template arguments of '{{(std::)?}}unordered_set'}}
-    }
-    {
-        // cannot deduce Key from just (Size, Hash, Pred)
-        std::unordered_set s(42, std::hash<int>(), std::equal_to<>());
-            // expected-error-re@-1{{no viable constructor or deduction guide for deduction of template arguments of '{{(std::)?}}unordered_set'}}
-    }
-    {
-        // cannot deduce Key from just (Size, Hash, Pred, Allocator)
-        std::unordered_set s(42, std::hash<int>(), std::equal_to<>(), std::allocator<int>());
-            // expected-error-re@-1{{no viable constructor or deduction guide for deduction of template arguments of '{{(std::)?}}unordered_set'}}
-    }
-    {
-        // cannot deduce Key from just (Allocator)
-        std::unordered_set s(std::allocator<int>{});
-            // expected-error-re@-1{{no viable constructor or deduction guide for deduction of template arguments of '{{(std::)?}}unordered_set'}}
-    }
-    {
-        // cannot deduce Key from just (Size, Allocator)
-        std::unordered_set s(42, std::allocator<int>());
-            // expected-error-re@-1{{no viable constructor or deduction guide for deduction of template arguments of '{{(std::)?}}unordered_set'}}
-    }
-    {
-        // cannot deduce Key from just (Size, Hash, Allocator)
-        std::unordered_set s(42, std::hash<short>(), std::allocator<int>());
-            // expected-error-re@-1{{no viable constructor or deduction guide for deduction of template arguments of '{{(std::)?}}unordered_set'}}
-    }
+int main(int, char**) {
+  {
+    // cannot deduce Key from nothing
+    std::unordered_set s;
+    // expected-error-re@-1{{no viable constructor or deduction guide for deduction of template arguments of '{{(std::)?}}unordered_set'}}
+  }
+  {
+    // cannot deduce Key from just (Size)
+    std::unordered_set s(42);
+    // expected-error-re@-1{{no viable constructor or deduction guide for deduction of template arguments of '{{(std::)?}}unordered_set'}}
+  }
+  {
+    // cannot deduce Key from just (Size, Hash)
+    std::unordered_set s(42, std::hash<int>());
+    // expected-error-re@-1{{no viable constructor or deduction guide for deduction of template arguments of '{{(std::)?}}unordered_set'}}
+  }
+  {
+    // cannot deduce Key from just (Size, Hash, Pred)
+    std::unordered_set s(42, std::hash<int>(), std::equal_to<>());
+    // expected-error-re@-1{{no viable constructor or deduction guide for deduction of template arguments of '{{(std::)?}}unordered_set'}}
+  }
+  {
+    // cannot deduce Key from just (Size, Hash, Pred, Allocator)
+    std::unordered_set s(42, std::hash<int>(), std::equal_to<>(), std::allocator<int>());
+    // expected-error-re@-1{{no viable constructor or deduction guide for deduction of template arguments of '{{(std::)?}}unordered_set'}}
+  }
+  {
+    // cannot deduce Key from just (Allocator)
+    std::unordered_set s(std::allocator<int>{});
+    // expected-error-re@-1{{no viable constructor or deduction guide for deduction of template arguments of '{{(std::)?}}unordered_set'}}
+  }
+  {
+    // cannot deduce Key from just (Size, Allocator)
+    std::unordered_set s(42, std::allocator<int>());
+    // expected-error-re@-1{{no viable constructor or deduction guide for deduction of template arguments of '{{(std::)?}}unordered_set'}}
+  }
+  {
+    // cannot deduce Key from just (Size, Hash, Allocator)
+    std::unordered_set s(42, std::hash<short>(), std::allocator<int>());
+    // expected-error-re@-1{{no viable constructor or deduction guide for deduction of template arguments of '{{(std::)?}}unordered_set'}}
+  }
 
-    return 0;
+  return 0;
 }
diff --git a/libcxx/test/std/containers/unord/unord.set/unord.set.cnstr/default.pass.cpp b/libcxx/test/std/containers/unord/unord.set/unord.set.cnstr/default.pass.cpp
index f61d46d5fc3b6..6784883f2d470 100644
--- a/libcxx/test/std/containers/unord/unord.set/unord.set.cnstr/default.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.set/unord.set.cnstr/default.pass.cpp
@@ -24,85 +24,80 @@
 #include "test_allocator.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef std::unordered_set<NotConstructible,
-                                   test_hash<NotConstructible>,
-                                   test_equal_to<NotConstructible>,
-                                   test_allocator<NotConstructible>
-                                   > C;
-        C c;
-        LIBCPP_ASSERT(c.bucket_count() == 0);
-        assert(c.hash_function() == test_hash<NotConstructible>());
-        assert(c.key_eq() == test_equal_to<NotConstructible>());
-        assert(c.get_allocator() == (test_allocator<NotConstructible>()));
-        assert(c.size() == 0);
-        assert(c.empty());
-        assert(std::distance(c.begin(), c.end()) == 0);
-        assert(c.load_factor() == 0);
-        assert(c.max_load_factor() == 1);
-    }
+int main(int, char**) {
+  {
+    typedef std::unordered_set<NotConstructible,
+                               test_hash<NotConstructible>,
+                               test_equal_to<NotConstructible>,
+                               test_allocator<NotConstructible> >
+        C;
+    C c;
+    LIBCPP_ASSERT(c.bucket_count() == 0);
+    assert(c.hash_function() == test_hash<NotConstructible>());
+    assert(c.key_eq() == test_equal_to<NotConstructible>());
+    assert(c.get_allocator() == (test_allocator<NotConstructible>()));
+    assert(c.size() == 0);
+    assert(c.empty());
+    assert(std::distance(c.begin(), c.end()) == 0);
+    assert(c.load_factor() == 0);
+    assert(c.max_load_factor() == 1);
+  }
 #if TEST_STD_VER >= 11
+  {
+    typedef std::unordered_set<NotConstructible,
+                               test_hash<NotConstructible>,
+                               test_equal_to<NotConstructible>,
+                               min_allocator<NotConstructible> >
+        C;
+    C c;
+    LIBCPP_ASSERT(c.bucket_count() == 0);
+    assert(c.hash_function() == test_hash<NotConstructible>());
+    assert(c.key_eq() == test_equal_to<NotConstructible>());
+    assert(c.get_allocator() == (min_allocator<NotConstructible>()));
+    assert(c.size() == 0);
+    assert(c.empty());
+    assert(std::distance(c.begin(), c.end()) == 0);
+    assert(c.load_factor() == 0);
+    assert(c.max_load_factor() == 1);
+  }
+  {
+    typedef explicit_allocator<NotConstructible> A;
+    typedef std::unordered_set<NotConstructible, test_hash<NotConstructible>, test_equal_to<NotConstructible>, A > C;
     {
-        typedef std::unordered_set<NotConstructible,
-                                   test_hash<NotConstructible>,
-                                   test_equal_to<NotConstructible>,
-                                   min_allocator<NotConstructible>
-                                   > C;
-        C c;
-        LIBCPP_ASSERT(c.bucket_count() == 0);
-        assert(c.hash_function() == test_hash<NotConstructible>());
-        assert(c.key_eq() == test_equal_to<NotConstructible>());
-        assert(c.get_allocator() == (min_allocator<NotConstructible>()));
-        assert(c.size() == 0);
-        assert(c.empty());
-        assert(std::distance(c.begin(), c.end()) == 0);
-        assert(c.load_factor() == 0);
-        assert(c.max_load_factor() == 1);
-    }
-    {
-        typedef explicit_allocator<NotConstructible> A;
-        typedef std::unordered_set<NotConstructible,
-                                   test_hash<NotConstructible>,
-                                   test_equal_to<NotConstructible>,
-                                   A
-                                   > C;
-        {
-        C c;
-        LIBCPP_ASSERT(c.bucket_count() == 0);
-        assert(c.hash_function() == test_hash<NotConstructible>());
-        assert(c.key_eq() == test_equal_to<NotConstructible>());
-        assert(c.get_allocator() == A());
-        assert(c.size() == 0);
-        assert(c.empty());
-        assert(std::distance(c.begin(), c.end()) == 0);
-        assert(c.load_factor() == 0);
-        assert(c.max_load_factor() == 1);
-        }
-        {
-        A a;
-        C c(a);
-        LIBCPP_ASSERT(c.bucket_count() == 0);
-        assert(c.hash_function() == test_hash<NotConstructible>());
-        assert(c.key_eq() == test_equal_to<NotConstructible>());
-        assert(c.get_allocator() == a);
-        assert(c.size() == 0);
-        assert(c.empty());
-        assert(std::distance(c.begin(), c.end()) == 0);
-        assert(c.load_factor() == 0);
-        assert(c.max_load_factor() == 1);
-        }
+      C c;
+      LIBCPP_ASSERT(c.bucket_count() == 0);
+      assert(c.hash_function() == test_hash<NotConstructible>());
+      assert(c.key_eq() == test_equal_to<NotConstructible>());
+      assert(c.get_allocator() == A());
+      assert(c.size() == 0);
+      assert(c.empty());
+      assert(std::distance(c.begin(), c.end()) == 0);
+      assert(c.load_factor() == 0);
+      assert(c.max_load_factor() == 1);
     }
     {
-        std::unordered_set<int> c = {};
-        LIBCPP_ASSERT(c.bucket_count() == 0);
-        assert(c.size() == 0);
-        assert(c.empty());
-        assert(std::distance(c.begin(), c.end()) == 0);
-        assert(c.load_factor() == 0);
-        assert(c.max_load_factor() == 1);
+      A a;
+      C c(a);
+      LIBCPP_ASSERT(c.bucket_count() == 0);
+      assert(c.hash_function() == test_hash<NotConstructible>());
+      assert(c.key_eq() == test_equal_to<NotConstructible>());
+      assert(c.get_allocator() == a);
+      assert(c.size() == 0);
+      assert(c.empty());
+      assert(std::distance(c.begin(), c.end()) == 0);
+      assert(c.load_factor() == 0);
+      assert(c.max_load_factor() == 1);
     }
+  }
+  {
+    std::unordered_set<int> c = {};
+    LIBCPP_ASSERT(c.bucket_count() == 0);
+    assert(c.size() == 0);
+    assert(c.empty());
+    assert(std::distance(c.begin(), c.end()) == 0);
+    assert(c.load_factor() == 0);
+    assert(c.max_load_factor() == 1);
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/unord/unord.set/unord.set.cnstr/default_noexcept.pass.cpp b/libcxx/test/std/containers/unord/unord.set/unord.set.cnstr/default_noexcept.pass.cpp
index b867231fb70a9..2bd1e10b5413b 100644
--- a/libcxx/test/std/containers/unord/unord.set/unord.set.cnstr/default_noexcept.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.set/unord.set.cnstr/default_noexcept.pass.cpp
@@ -27,50 +27,44 @@
 #include "../../../test_hash.h"
 
 template <class T>
-struct some_comp
-{
-    typedef T value_type;
-    some_comp();
-    some_comp(const some_comp&);
-    bool operator()(const T&, const T&) const { return false; }
+struct some_comp {
+  typedef T value_type;
+  some_comp();
+  some_comp(const some_comp&);
+  bool operator()(const T&, const T&) const { return false; }
 };
 
 template <class T>
-struct some_hash
-{
-    typedef T value_type;
-    some_hash();
-    some_hash(const some_hash&);
-    std::size_t operator()(T const&) const;
+struct some_hash {
+  typedef T value_type;
+  some_hash();
+  some_hash(const some_hash&);
+  std::size_t operator()(T const&) const;
 };
 
-int main(int, char**)
-{
+int main(int, char**) {
 #if defined(_LIBCPP_VERSION)
-    {
-        typedef std::unordered_set<MoveOnly> C;
-        static_assert(std::is_nothrow_default_constructible<C>::value, "");
-    }
-    {
-        typedef std::unordered_set<MoveOnly, std::hash<MoveOnly>,
-                           std::equal_to<MoveOnly>, test_allocator<MoveOnly>> C;
-        static_assert(std::is_nothrow_default_constructible<C>::value, "");
-    }
+  {
+    typedef std::unordered_set<MoveOnly> C;
+    static_assert(std::is_nothrow_default_constructible<C>::value, "");
+  }
+  {
+    typedef std::unordered_set<MoveOnly, std::hash<MoveOnly>, std::equal_to<MoveOnly>, test_allocator<MoveOnly>> C;
+    static_assert(std::is_nothrow_default_constructible<C>::value, "");
+  }
 #endif // _LIBCPP_VERSION
-    {
-        typedef std::unordered_set<MoveOnly, std::hash<MoveOnly>,
-                          std::equal_to<MoveOnly>, other_allocator<MoveOnly>> C;
-        static_assert(!std::is_nothrow_default_constructible<C>::value, "");
-    }
-    {
-        typedef std::unordered_set<MoveOnly, some_hash<MoveOnly>> C;
-        static_assert(!std::is_nothrow_default_constructible<C>::value, "");
-    }
-    {
-        typedef std::unordered_set<MoveOnly, std::hash<MoveOnly>,
-                                                         some_comp<MoveOnly>> C;
-        static_assert(!std::is_nothrow_default_constructible<C>::value, "");
-    }
+  {
+    typedef std::unordered_set<MoveOnly, std::hash<MoveOnly>, std::equal_to<MoveOnly>, other_allocator<MoveOnly>> C;
+    static_assert(!std::is_nothrow_default_constructible<C>::value, "");
+  }
+  {
+    typedef std::unordered_set<MoveOnly, some_hash<MoveOnly>> C;
+    static_assert(!std::is_nothrow_default_constructible<C>::value, "");
+  }
+  {
+    typedef std::unordered_set<MoveOnly, std::hash<MoveOnly>, some_comp<MoveOnly>> C;
+    static_assert(!std::is_nothrow_default_constructible<C>::value, "");
+  }
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/unord/unord.set/unord.set.cnstr/dtor_noexcept.pass.cpp b/libcxx/test/std/containers/unord/unord.set/unord.set.cnstr/dtor_noexcept.pass.cpp
index f0b7e28699f36..17cfae0f989c7 100644
--- a/libcxx/test/std/containers/unord/unord.set/unord.set.cnstr/dtor_noexcept.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.set/unord.set.cnstr/dtor_noexcept.pass.cpp
@@ -20,49 +20,43 @@
 #include "test_allocator.h"
 
 template <class T>
-struct some_comp
-{
-    typedef T value_type;
-    ~some_comp() noexcept(false);
-    bool operator()(const T&, const T&) const { return false; }
+struct some_comp {
+  typedef T value_type;
+  ~some_comp() noexcept(false);
+  bool operator()(const T&, const T&) const { return false; }
 };
 
 template <class T>
-struct some_hash
-{
-    typedef T value_type;
-    some_hash();
-    some_hash(const some_hash&);
-    ~some_hash() noexcept(false);
-    std::size_t operator()(T const&) const;
+struct some_hash {
+  typedef T value_type;
+  some_hash();
+  some_hash(const some_hash&);
+  ~some_hash() noexcept(false);
+  std::size_t operator()(T const&) const;
 };
 
-int main(int, char**)
-{
-    {
-        typedef std::unordered_set<MoveOnly> C;
-        static_assert(std::is_nothrow_destructible<C>::value, "");
-    }
-    {
-        typedef std::unordered_set<MoveOnly, std::hash<MoveOnly>,
-                           std::equal_to<MoveOnly>, test_allocator<MoveOnly>> C;
-        static_assert(std::is_nothrow_destructible<C>::value, "");
-    }
-    {
-        typedef std::unordered_set<MoveOnly, std::hash<MoveOnly>,
-                          std::equal_to<MoveOnly>, other_allocator<MoveOnly>> C;
-        static_assert(std::is_nothrow_destructible<C>::value, "");
-    }
+int main(int, char**) {
+  {
+    typedef std::unordered_set<MoveOnly> C;
+    static_assert(std::is_nothrow_destructible<C>::value, "");
+  }
+  {
+    typedef std::unordered_set<MoveOnly, std::hash<MoveOnly>, std::equal_to<MoveOnly>, test_allocator<MoveOnly>> C;
+    static_assert(std::is_nothrow_destructible<C>::value, "");
+  }
+  {
+    typedef std::unordered_set<MoveOnly, std::hash<MoveOnly>, std::equal_to<MoveOnly>, other_allocator<MoveOnly>> C;
+    static_assert(std::is_nothrow_destructible<C>::value, "");
+  }
 #if defined(_LIBCPP_VERSION)
-    {
-        typedef std::unordered_set<MoveOnly, some_hash<MoveOnly>> C;
-        static_assert(!std::is_nothrow_destructible<C>::value, "");
-    }
-    {
-        typedef std::unordered_set<MoveOnly, std::hash<MoveOnly>,
-                                                         some_comp<MoveOnly>> C;
-        static_assert(!std::is_nothrow_destructible<C>::value, "");
-    }
+  {
+    typedef std::unordered_set<MoveOnly, some_hash<MoveOnly>> C;
+    static_assert(!std::is_nothrow_destructible<C>::value, "");
+  }
+  {
+    typedef std::unordered_set<MoveOnly, std::hash<MoveOnly>, some_comp<MoveOnly>> C;
+    static_assert(!std::is_nothrow_destructible<C>::value, "");
+  }
 #endif // _LIBCPP_VERSION
 
   return 0;
diff --git a/libcxx/test/std/containers/unord/unord.set/unord.set.cnstr/from_range.pass.cpp b/libcxx/test/std/containers/unord/unord.set/unord.set.cnstr/from_range.pass.cpp
index 581781949365e..c4ce08e39176f 100644
--- a/libcxx/test/std/containers/unord/unord.set/unord.set.cnstr/from_range.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.set/unord.set.cnstr/from_range.pass.cpp
@@ -30,13 +30,9 @@
 void test_duplicates() {
   using T = KeyValue;
 
-  std::array input = {
-    T{1, 'a'}, T{2, 'a'}, T{3, 'a'}, T{3, 'b'}, T{3, 'c'}, T{2, 'b'}, T{4, 'a'}
-  };
-  std::array expected = {
-    T{1, 'a'}, T{2, 'b'}, T{3, 'c'}, T{4, 'a'}
-  };
-  auto c = std::unordered_set<T>(std::from_range, input);
+  std::array input    = {T{1, 'a'}, T{2, 'a'}, T{3, 'a'}, T{3, 'b'}, T{3, 'c'}, T{2, 'b'}, T{4, 'a'}};
+  std::array expected = {T{1, 'a'}, T{2, 'b'}, T{3, 'c'}, T{4, 'a'}};
+  auto c              = std::unordered_set<T>(std::from_range, input);
   assert(std::ranges::is_permutation(expected, c));
 }
 
diff --git a/libcxx/test/std/containers/unord/unord.set/unord.set.cnstr/hash_copy_constructible.compile.fail.cpp b/libcxx/test/std/containers/unord/unord.set/unord.set.cnstr/hash_copy_constructible.compile.fail.cpp
index 4857f80f45489..3e0ab51727db9 100644
--- a/libcxx/test/std/containers/unord/unord.set/unord.set.cnstr/hash_copy_constructible.compile.fail.cpp
+++ b/libcxx/test/std/containers/unord/unord.set/unord.set.cnstr/hash_copy_constructible.compile.fail.cpp
@@ -6,7 +6,6 @@
 //
 //===----------------------------------------------------------------------===//
 
-
 // <unordered_map>
 
 // Check that std::unordered_set fails to instantiate if the hash function is
@@ -16,16 +15,16 @@
 
 template <class T>
 struct Hash {
-    std::size_t operator () (const T& lhs) const { return 0; }
+  std::size_t operator()(const T& lhs) const { return 0; }
 
-    Hash () {}
-private:
-    Hash (const Hash &); // declared but not defined
-    };
+  Hash() {}
 
+private:
+  Hash(const Hash&); // declared but not defined
+};
 
 int main(int, char**) {
-    std::unordered_set<int, Hash<int> > m;
+  std::unordered_set<int, Hash<int> > m;
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/unord/unord.set/unord.set.cnstr/init.pass.cpp b/libcxx/test/std/containers/unord/unord.set/unord.set.cnstr/init.pass.cpp
index 28122424dfb4a..7e42decbffd6c 100644
--- a/libcxx/test/std/containers/unord/unord.set/unord.set.cnstr/init.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.set/unord.set.cnstr/init.pass.cpp
@@ -28,137 +28,100 @@
 #include "test_allocator.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
+int main(int, char**) {
   {
-        typedef std::unordered_set<int,
-                                   test_hash<int>,
-                                   test_equal_to<int>,
-                                   test_allocator<int>
-                                   > C;
-        typedef int P;
-        C c = {
-                P(1),
-                P(2),
-                P(3),
-                P(4),
-                P(1),
-                P(2)
-            };
-        assert(c.bucket_count() >= 5);
-        assert(c.size() == 4);
-        assert(c.count(1) == 1);
-        assert(c.count(2) == 1);
-        assert(c.count(3) == 1);
-        assert(c.count(4) == 1);
-        assert(c.hash_function() == test_hash<int>());
-        assert(c.key_eq() == test_equal_to<int>());
-        assert(c.get_allocator() == test_allocator<int>());
-        assert(!c.empty());
-        assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-        assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
-        assert(fabs(c.load_factor() - (float)c.size()/c.bucket_count()) < FLT_EPSILON);
-        assert(c.max_load_factor() == 1);
-    }
-    {
-        typedef std::unordered_set<int,
-                                   test_hash<int>,
-                                   test_equal_to<int>,
-                                   min_allocator<int>
-                                   > C;
-        typedef int P;
-        C c = {
-                P(1),
-                P(2),
-                P(3),
-                P(4),
-                P(1),
-                P(2)
-            };
-        assert(c.bucket_count() >= 5);
-        assert(c.size() == 4);
-        assert(c.count(1) == 1);
-        assert(c.count(2) == 1);
-        assert(c.count(3) == 1);
-        assert(c.count(4) == 1);
-        assert(c.hash_function() == test_hash<int>());
-        assert(c.key_eq() == test_equal_to<int>());
-        assert(c.get_allocator() == min_allocator<int>());
-        assert(!c.empty());
-        assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-        assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
-        assert(fabs(c.load_factor() - (float)c.size()/c.bucket_count()) < FLT_EPSILON);
-        assert(c.max_load_factor() == 1);
-    }
+    typedef std::unordered_set<int, test_hash<int>, test_equal_to<int>, test_allocator<int> > C;
+    typedef int P;
+    C c = {P(1), P(2), P(3), P(4), P(1), P(2)};
+    assert(c.bucket_count() >= 5);
+    assert(c.size() == 4);
+    assert(c.count(1) == 1);
+    assert(c.count(2) == 1);
+    assert(c.count(3) == 1);
+    assert(c.count(4) == 1);
+    assert(c.hash_function() == test_hash<int>());
+    assert(c.key_eq() == test_equal_to<int>());
+    assert(c.get_allocator() == test_allocator<int>());
+    assert(!c.empty());
+    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
+    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
+    assert(fabs(c.load_factor() - (float)c.size() / c.bucket_count()) < FLT_EPSILON);
+    assert(c.max_load_factor() == 1);
+  }
+  {
+    typedef std::unordered_set<int, test_hash<int>, test_equal_to<int>, min_allocator<int> > C;
+    typedef int P;
+    C c = {P(1), P(2), P(3), P(4), P(1), P(2)};
+    assert(c.bucket_count() >= 5);
+    assert(c.size() == 4);
+    assert(c.count(1) == 1);
+    assert(c.count(2) == 1);
+    assert(c.count(3) == 1);
+    assert(c.count(4) == 1);
+    assert(c.hash_function() == test_hash<int>());
+    assert(c.key_eq() == test_equal_to<int>());
+    assert(c.get_allocator() == min_allocator<int>());
+    assert(!c.empty());
+    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
+    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
+    assert(fabs(c.load_factor() - (float)c.size() / c.bucket_count()) < FLT_EPSILON);
+    assert(c.max_load_factor() == 1);
+  }
 #if TEST_STD_VER > 11
-    {
-        typedef int T;
-        typedef test_hash<T> HF;
-        typedef test_equal_to<T> Comp;
-        typedef test_allocator<T> A;
-        typedef std::unordered_set<T, HF, Comp, A> C;
+  {
+    typedef int T;
+    typedef test_hash<T> HF;
+    typedef test_equal_to<T> Comp;
+    typedef test_allocator<T> A;
+    typedef std::unordered_set<T, HF, Comp, A> C;
 
-        A a(42);
-        C c({
-                T(1),
-                T(2),
-                T(3),
-                T(4),
-                T(1),
-                T(2)
-            }, 12, a);
+    A a(42);
+    C c({T(1), T(2), T(3), T(4), T(1), T(2)}, 12, a);
 
-        assert(c.bucket_count() >= 12);
-        assert(c.size() == 4);
-        assert(c.count(1) == 1);
-        assert(c.count(2) == 1);
-        assert(c.count(3) == 1);
-        assert(c.count(4) == 1);
-        assert(c.hash_function() == HF());
-        assert(c.key_eq() == Comp());
-        assert(c.get_allocator() == a);
-        assert(!(c.get_allocator() == A()));
-        assert(!c.empty());
-        assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-        assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
-        assert(fabs(c.load_factor() - (float)c.size()/c.bucket_count()) < FLT_EPSILON);
-        assert(c.max_load_factor() == 1);
-    }
-    {
-        typedef int T;
-        typedef test_hash<T> HF;
-        typedef test_equal_to<T> Comp;
-        typedef test_allocator<T> A;
-        typedef std::unordered_set<T, HF, Comp, A> C;
+    assert(c.bucket_count() >= 12);
+    assert(c.size() == 4);
+    assert(c.count(1) == 1);
+    assert(c.count(2) == 1);
+    assert(c.count(3) == 1);
+    assert(c.count(4) == 1);
+    assert(c.hash_function() == HF());
+    assert(c.key_eq() == Comp());
+    assert(c.get_allocator() == a);
+    assert(!(c.get_allocator() == A()));
+    assert(!c.empty());
+    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
+    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
+    assert(fabs(c.load_factor() - (float)c.size() / c.bucket_count()) < FLT_EPSILON);
+    assert(c.max_load_factor() == 1);
+  }
+  {
+    typedef int T;
+    typedef test_hash<T> HF;
+    typedef test_equal_to<T> Comp;
+    typedef test_allocator<T> A;
+    typedef std::unordered_set<T, HF, Comp, A> C;
 
-        A a(42);
-        HF hf(43);
-        C c({
-                T(1),
-                T(2),
-                T(3),
-                T(4),
-                T(1),
-                T(2)
-            }, 12, hf, a);
+    A a(42);
+    HF hf(43);
+    C c({T(1), T(2), T(3), T(4), T(1), T(2)}, 12, hf, a);
 
-        assert(c.bucket_count() >= 12);
-        assert(c.size() == 4);
-        assert(c.count(1) == 1);
-        assert(c.count(2) == 1);
-        assert(c.count(3) == 1);
-        assert(c.count(4) == 1);
-        assert(c.hash_function() == hf);
-        assert(!(c.hash_function() == HF()));
-        assert(c.key_eq() == Comp());
-        assert(c.get_allocator() == a);
-        assert(!(c.get_allocator() == A()));
-        assert(!c.empty());
-        assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-        assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
-        assert(fabs(c.load_factor() - (float)c.size()/c.bucket_count()) < FLT_EPSILON);
-        assert(c.max_load_factor() == 1);
-    }
+    assert(c.bucket_count() >= 12);
+    assert(c.size() == 4);
+    assert(c.count(1) == 1);
+    assert(c.count(2) == 1);
+    assert(c.count(3) == 1);
+    assert(c.count(4) == 1);
+    assert(c.hash_function() == hf);
+    assert(!(c.hash_function() == HF()));
+    assert(c.key_eq() == Comp());
+    assert(c.get_allocator() == a);
+    assert(!(c.get_allocator() == A()));
+    assert(!c.empty());
+    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
+    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
+    assert(fabs(c.load_factor() - (float)c.size() / c.bucket_count()) < FLT_EPSILON);
+    assert(c.max_load_factor() == 1);
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/unord/unord.set/unord.set.cnstr/init_size.pass.cpp b/libcxx/test/std/containers/unord/unord.set/unord.set.cnstr/init_size.pass.cpp
index ff516a757c1cd..1d13b0215aa30 100644
--- a/libcxx/test/std/containers/unord/unord.set/unord.set.cnstr/init_size.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.set/unord.set.cnstr/init_size.pass.cpp
@@ -28,72 +28,45 @@
 #include "test_allocator.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef std::unordered_set<int,
-                                   test_hash<int>,
-                                   test_equal_to<int>,
-                                   test_allocator<int>
-                                   > C;
-        typedef int P;
-        C c({
-                P(1),
-                P(2),
-                P(3),
-                P(4),
-                P(1),
-                P(2)
-            },
-            7
-           );
-        LIBCPP_ASSERT(c.bucket_count() == 7);
-        assert(c.size() == 4);
-        assert(c.count(1) == 1);
-        assert(c.count(2) == 1);
-        assert(c.count(3) == 1);
-        assert(c.count(4) == 1);
-        assert(c.hash_function() == test_hash<int>());
-        assert(c.key_eq() == test_equal_to<int>());
-        assert(c.get_allocator() == test_allocator<int>());
-        assert(!c.empty());
-        assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-        assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
-        assert(std::fabs(c.load_factor() - (float)c.size()/c.bucket_count()) < FLT_EPSILON);
-        assert(c.max_load_factor() == 1);
-    }
-    {
-        typedef std::unordered_set<int,
-                                   test_hash<int>,
-                                   test_equal_to<int>,
-                                   min_allocator<int>
-                                   > C;
-        typedef int P;
-        C c({
-                P(1),
-                P(2),
-                P(3),
-                P(4),
-                P(1),
-                P(2)
-            },
-            7
-           );
-        LIBCPP_ASSERT(c.bucket_count() == 7);
-        assert(c.size() == 4);
-        assert(c.count(1) == 1);
-        assert(c.count(2) == 1);
-        assert(c.count(3) == 1);
-        assert(c.count(4) == 1);
-        assert(c.hash_function() == test_hash<int>());
-        assert(c.key_eq() == test_equal_to<int>());
-        assert(c.get_allocator() == min_allocator<int>());
-        assert(!c.empty());
-        assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-        assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
-        assert(std::fabs(c.load_factor() - (float)c.size()/c.bucket_count()) < FLT_EPSILON);
-        assert(c.max_load_factor() == 1);
-    }
+int main(int, char**) {
+  {
+    typedef std::unordered_set<int, test_hash<int>, test_equal_to<int>, test_allocator<int> > C;
+    typedef int P;
+    C c({P(1), P(2), P(3), P(4), P(1), P(2)}, 7);
+    LIBCPP_ASSERT(c.bucket_count() == 7);
+    assert(c.size() == 4);
+    assert(c.count(1) == 1);
+    assert(c.count(2) == 1);
+    assert(c.count(3) == 1);
+    assert(c.count(4) == 1);
+    assert(c.hash_function() == test_hash<int>());
+    assert(c.key_eq() == test_equal_to<int>());
+    assert(c.get_allocator() == test_allocator<int>());
+    assert(!c.empty());
+    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
+    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
+    assert(std::fabs(c.load_factor() - (float)c.size() / c.bucket_count()) < FLT_EPSILON);
+    assert(c.max_load_factor() == 1);
+  }
+  {
+    typedef std::unordered_set<int, test_hash<int>, test_equal_to<int>, min_allocator<int> > C;
+    typedef int P;
+    C c({P(1), P(2), P(3), P(4), P(1), P(2)}, 7);
+    LIBCPP_ASSERT(c.bucket_count() == 7);
+    assert(c.size() == 4);
+    assert(c.count(1) == 1);
+    assert(c.count(2) == 1);
+    assert(c.count(3) == 1);
+    assert(c.count(4) == 1);
+    assert(c.hash_function() == test_hash<int>());
+    assert(c.key_eq() == test_equal_to<int>());
+    assert(c.get_allocator() == min_allocator<int>());
+    assert(!c.empty());
+    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
+    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
+    assert(std::fabs(c.load_factor() - (float)c.size() / c.bucket_count()) < FLT_EPSILON);
+    assert(c.max_load_factor() == 1);
+  }
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/unord/unord.set/unord.set.cnstr/init_size_allocator.pass.cpp b/libcxx/test/std/containers/unord/unord.set/unord.set.cnstr/init_size_allocator.pass.cpp
index 340dbba9b49a9..7346a951b1976 100644
--- a/libcxx/test/std/containers/unord/unord.set/unord.set.cnstr/init_size_allocator.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.set/unord.set.cnstr/init_size_allocator.pass.cpp
@@ -30,43 +30,29 @@
 #include "min_allocator.h"
 
 template <class Allocator>
-void test(const Allocator& alloc)
-{
-    typedef std::unordered_set<int,
-                               test_hash<int>,
-                               test_equal_to<int>,
-                               Allocator> C;
-    C c({
-            1,
-            2,
-            3,
-            4,
-            1,
-            2
-        },
-        7,
-        alloc);
-
-    LIBCPP_ASSERT(c.bucket_count() == 7);
-    assert(c.size() == 4);
-    assert(c.count(1) == 1);
-    assert(c.count(2) == 1);
-    assert(c.count(3) == 1);
-    assert(c.count(4) == 1);
-    assert(c.hash_function() == test_hash<int>());
-    assert(c.key_eq() == test_equal_to<int>());
-    assert(c.get_allocator() == alloc);
-    assert(!c.empty());
-    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-    assert(std::fabs(c.load_factor() - (float)c.size()/c.bucket_count()) < FLT_EPSILON);
-    assert(c.max_load_factor() == 1);
+void test(const Allocator& alloc) {
+  typedef std::unordered_set<int, test_hash<int>, test_equal_to<int>, Allocator> C;
+  C c({1, 2, 3, 4, 1, 2}, 7, alloc);
+
+  LIBCPP_ASSERT(c.bucket_count() == 7);
+  assert(c.size() == 4);
+  assert(c.count(1) == 1);
+  assert(c.count(2) == 1);
+  assert(c.count(3) == 1);
+  assert(c.count(4) == 1);
+  assert(c.hash_function() == test_hash<int>());
+  assert(c.key_eq() == test_equal_to<int>());
+  assert(c.get_allocator() == alloc);
+  assert(!c.empty());
+  assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
+  assert(std::fabs(c.load_factor() - (float)c.size() / c.bucket_count()) < FLT_EPSILON);
+  assert(c.max_load_factor() == 1);
 }
 
-int main(int, char**)
-{
-    test(test_allocator<int>(10));
-    test(min_allocator<int>());
-    test(explicit_allocator<int>());
+int main(int, char**) {
+  test(test_allocator<int>(10));
+  test(min_allocator<int>());
+  test(explicit_allocator<int>());
 
-    return 0;
+  return 0;
 }
diff --git a/libcxx/test/std/containers/unord/unord.set/unord.set.cnstr/init_size_hash.pass.cpp b/libcxx/test/std/containers/unord/unord.set/unord.set.cnstr/init_size_hash.pass.cpp
index 628b3277764c3..5e3d869805f6b 100644
--- a/libcxx/test/std/containers/unord/unord.set/unord.set.cnstr/init_size_hash.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.set/unord.set.cnstr/init_size_hash.pass.cpp
@@ -29,74 +29,45 @@
 #include "test_allocator.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef std::unordered_set<int,
-                                   test_hash<int>,
-                                   test_equal_to<int>,
-                                   test_allocator<int>
-                                   > C;
-        typedef int P;
-        C c({
-                P(1),
-                P(2),
-                P(3),
-                P(4),
-                P(1),
-                P(2)
-            },
-            7,
-            test_hash<int>(8)
-           );
-        LIBCPP_ASSERT(c.bucket_count() == 7);
-        assert(c.size() == 4);
-        assert(c.count(1) == 1);
-        assert(c.count(2) == 1);
-        assert(c.count(3) == 1);
-        assert(c.count(4) == 1);
-        assert(c.hash_function() == test_hash<int>(8));
-        assert(c.key_eq() == test_equal_to<int>());
-        assert(c.get_allocator() == test_allocator<int>());
-        assert(!c.empty());
-        assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-        assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
-        assert(std::fabs(c.load_factor() - (float)c.size()/c.bucket_count()) < FLT_EPSILON);
-        assert(c.max_load_factor() == 1);
-    }
-    {
-        typedef std::unordered_set<int,
-                                   test_hash<int>,
-                                   test_equal_to<int>,
-                                   min_allocator<int>
-                                   > C;
-        typedef int P;
-        C c({
-                P(1),
-                P(2),
-                P(3),
-                P(4),
-                P(1),
-                P(2)
-            },
-            7,
-            test_hash<int>(8)
-           );
-        LIBCPP_ASSERT(c.bucket_count() == 7);
-        assert(c.size() == 4);
-        assert(c.count(1) == 1);
-        assert(c.count(2) == 1);
-        assert(c.count(3) == 1);
-        assert(c.count(4) == 1);
-        assert(c.hash_function() == test_hash<int>(8));
-        assert(c.key_eq() == test_equal_to<int>());
-        assert(c.get_allocator() == min_allocator<int>());
-        assert(!c.empty());
-        assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-        assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
-        assert(std::fabs(c.load_factor() - (float)c.size()/c.bucket_count()) < FLT_EPSILON);
-        assert(c.max_load_factor() == 1);
-    }
+int main(int, char**) {
+  {
+    typedef std::unordered_set<int, test_hash<int>, test_equal_to<int>, test_allocator<int> > C;
+    typedef int P;
+    C c({P(1), P(2), P(3), P(4), P(1), P(2)}, 7, test_hash<int>(8));
+    LIBCPP_ASSERT(c.bucket_count() == 7);
+    assert(c.size() == 4);
+    assert(c.count(1) == 1);
+    assert(c.count(2) == 1);
+    assert(c.count(3) == 1);
+    assert(c.count(4) == 1);
+    assert(c.hash_function() == test_hash<int>(8));
+    assert(c.key_eq() == test_equal_to<int>());
+    assert(c.get_allocator() == test_allocator<int>());
+    assert(!c.empty());
+    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
+    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
+    assert(std::fabs(c.load_factor() - (float)c.size() / c.bucket_count()) < FLT_EPSILON);
+    assert(c.max_load_factor() == 1);
+  }
+  {
+    typedef std::unordered_set<int, test_hash<int>, test_equal_to<int>, min_allocator<int> > C;
+    typedef int P;
+    C c({P(1), P(2), P(3), P(4), P(1), P(2)}, 7, test_hash<int>(8));
+    LIBCPP_ASSERT(c.bucket_count() == 7);
+    assert(c.size() == 4);
+    assert(c.count(1) == 1);
+    assert(c.count(2) == 1);
+    assert(c.count(3) == 1);
+    assert(c.count(4) == 1);
+    assert(c.hash_function() == test_hash<int>(8));
+    assert(c.key_eq() == test_equal_to<int>());
+    assert(c.get_allocator() == min_allocator<int>());
+    assert(!c.empty());
+    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
+    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
+    assert(std::fabs(c.load_factor() - (float)c.size() / c.bucket_count()) < FLT_EPSILON);
+    assert(c.max_load_factor() == 1);
+  }
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/unord/unord.set/unord.set.cnstr/init_size_hash_allocator.pass.cpp b/libcxx/test/std/containers/unord/unord.set/unord.set.cnstr/init_size_hash_allocator.pass.cpp
index ab5f7d9f9b504..704fff0f51cca 100644
--- a/libcxx/test/std/containers/unord/unord.set/unord.set.cnstr/init_size_hash_allocator.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.set/unord.set.cnstr/init_size_hash_allocator.pass.cpp
@@ -31,44 +31,29 @@
 #include "min_allocator.h"
 
 template <class Allocator>
-void test(const Allocator& alloc)
-{
-    typedef std::unordered_set<int,
-                               test_hash<int>,
-                               test_equal_to<int>,
-                               Allocator> C;
-    C c({
-            1,
-            2,
-            3,
-            4,
-            1,
-            2
-        },
-        7,
-        test_hash<int>(5),
-        alloc);
-
-    LIBCPP_ASSERT(c.bucket_count() == 7);
-    assert(c.size() == 4);
-    assert(c.count(1) == 1);
-    assert(c.count(2) == 1);
-    assert(c.count(3) == 1);
-    assert(c.count(4) == 1);
-    assert(c.hash_function() == test_hash<int>(5));
-    assert(c.key_eq() == test_equal_to<int>());
-    assert(c.get_allocator() == alloc);
-    assert(!c.empty());
-    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-    assert(std::fabs(c.load_factor() - (float)c.size()/c.bucket_count()) < FLT_EPSILON);
-    assert(c.max_load_factor() == 1);
+void test(const Allocator& alloc) {
+  typedef std::unordered_set<int, test_hash<int>, test_equal_to<int>, Allocator> C;
+  C c({1, 2, 3, 4, 1, 2}, 7, test_hash<int>(5), alloc);
+
+  LIBCPP_ASSERT(c.bucket_count() == 7);
+  assert(c.size() == 4);
+  assert(c.count(1) == 1);
+  assert(c.count(2) == 1);
+  assert(c.count(3) == 1);
+  assert(c.count(4) == 1);
+  assert(c.hash_function() == test_hash<int>(5));
+  assert(c.key_eq() == test_equal_to<int>());
+  assert(c.get_allocator() == alloc);
+  assert(!c.empty());
+  assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
+  assert(std::fabs(c.load_factor() - (float)c.size() / c.bucket_count()) < FLT_EPSILON);
+  assert(c.max_load_factor() == 1);
 }
 
-int main(int, char**)
-{
-    test(test_allocator<int>(10));
-    test(min_allocator<int>());
-    test(explicit_allocator<int>());
+int main(int, char**) {
+  test(test_allocator<int>(10));
+  test(min_allocator<int>());
+  test(explicit_allocator<int>());
 
-    return 0;
+  return 0;
 }
diff --git a/libcxx/test/std/containers/unord/unord.set/unord.set.cnstr/init_size_hash_equal.pass.cpp b/libcxx/test/std/containers/unord/unord.set/unord.set.cnstr/init_size_hash_equal.pass.cpp
index af5a371bd458b..44d3d9b3fb200 100644
--- a/libcxx/test/std/containers/unord/unord.set/unord.set.cnstr/init_size_hash_equal.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.set/unord.set.cnstr/init_size_hash_equal.pass.cpp
@@ -29,76 +29,45 @@
 #include "test_allocator.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef std::unordered_set<int,
-                                   test_hash<int>,
-                                   test_equal_to<int>,
-                                   test_allocator<int>
-                                   > C;
-        typedef int P;
-        C c({
-                P(1),
-                P(2),
-                P(3),
-                P(4),
-                P(1),
-                P(2)
-            },
-            7,
-            test_hash<int>(8),
-            test_equal_to<int>(9)
-           );
-        LIBCPP_ASSERT(c.bucket_count() == 7);
-        assert(c.size() == 4);
-        assert(c.count(1) == 1);
-        assert(c.count(2) == 1);
-        assert(c.count(3) == 1);
-        assert(c.count(4) == 1);
-        assert(c.hash_function() == test_hash<int>(8));
-        assert(c.key_eq() == test_equal_to<int>(9));
-        assert(c.get_allocator() == test_allocator<int>());
-        assert(!c.empty());
-        assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-        assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
-        assert(std::fabs(c.load_factor() - (float)c.size()/c.bucket_count()) < FLT_EPSILON);
-        assert(c.max_load_factor() == 1);
-    }
-    {
-        typedef std::unordered_set<int,
-                                   test_hash<int>,
-                                   test_equal_to<int>,
-                                   min_allocator<int>
-                                   > C;
-        typedef int P;
-        C c({
-                P(1),
-                P(2),
-                P(3),
-                P(4),
-                P(1),
-                P(2)
-            },
-            7,
-            test_hash<int>(8),
-            test_equal_to<int>(9)
-           );
-        LIBCPP_ASSERT(c.bucket_count() == 7);
-        assert(c.size() == 4);
-        assert(c.count(1) == 1);
-        assert(c.count(2) == 1);
-        assert(c.count(3) == 1);
-        assert(c.count(4) == 1);
-        assert(c.hash_function() == test_hash<int>(8));
-        assert(c.key_eq() == test_equal_to<int>(9));
-        assert(c.get_allocator() == min_allocator<int>());
-        assert(!c.empty());
-        assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-        assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
-        assert(std::fabs(c.load_factor() - (float)c.size()/c.bucket_count()) < FLT_EPSILON);
-        assert(c.max_load_factor() == 1);
-    }
+int main(int, char**) {
+  {
+    typedef std::unordered_set<int, test_hash<int>, test_equal_to<int>, test_allocator<int> > C;
+    typedef int P;
+    C c({P(1), P(2), P(3), P(4), P(1), P(2)}, 7, test_hash<int>(8), test_equal_to<int>(9));
+    LIBCPP_ASSERT(c.bucket_count() == 7);
+    assert(c.size() == 4);
+    assert(c.count(1) == 1);
+    assert(c.count(2) == 1);
+    assert(c.count(3) == 1);
+    assert(c.count(4) == 1);
+    assert(c.hash_function() == test_hash<int>(8));
+    assert(c.key_eq() == test_equal_to<int>(9));
+    assert(c.get_allocator() == test_allocator<int>());
+    assert(!c.empty());
+    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
+    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
+    assert(std::fabs(c.load_factor() - (float)c.size() / c.bucket_count()) < FLT_EPSILON);
+    assert(c.max_load_factor() == 1);
+  }
+  {
+    typedef std::unordered_set<int, test_hash<int>, test_equal_to<int>, min_allocator<int> > C;
+    typedef int P;
+    C c({P(1), P(2), P(3), P(4), P(1), P(2)}, 7, test_hash<int>(8), test_equal_to<int>(9));
+    LIBCPP_ASSERT(c.bucket_count() == 7);
+    assert(c.size() == 4);
+    assert(c.count(1) == 1);
+    assert(c.count(2) == 1);
+    assert(c.count(3) == 1);
+    assert(c.count(4) == 1);
+    assert(c.hash_function() == test_hash<int>(8));
+    assert(c.key_eq() == test_equal_to<int>(9));
+    assert(c.get_allocator() == min_allocator<int>());
+    assert(!c.empty());
+    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
+    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
+    assert(std::fabs(c.load_factor() - (float)c.size() / c.bucket_count()) < FLT_EPSILON);
+    assert(c.max_load_factor() == 1);
+  }
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/unord/unord.set/unord.set.cnstr/init_size_hash_equal_allocator.pass.cpp b/libcxx/test/std/containers/unord/unord.set/unord.set.cnstr/init_size_hash_equal_allocator.pass.cpp
index bf8c9561e76ee..bd21eeb163516 100644
--- a/libcxx/test/std/containers/unord/unord.set/unord.set.cnstr/init_size_hash_equal_allocator.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.set/unord.set.cnstr/init_size_hash_equal_allocator.pass.cpp
@@ -29,78 +29,45 @@
 #include "test_allocator.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef std::unordered_set<int,
-                                   test_hash<int>,
-                                   test_equal_to<int>,
-                                   test_allocator<int>
-                                   > C;
-        typedef int P;
-        C c({
-                P(1),
-                P(2),
-                P(3),
-                P(4),
-                P(1),
-                P(2)
-            },
-            7,
-            test_hash<int>(8),
-            test_equal_to<int>(9),
-            test_allocator<int>(10)
-           );
-        LIBCPP_ASSERT(c.bucket_count() == 7);
-        assert(c.size() == 4);
-        assert(c.count(1) == 1);
-        assert(c.count(2) == 1);
-        assert(c.count(3) == 1);
-        assert(c.count(4) == 1);
-        assert(c.hash_function() == test_hash<int>(8));
-        assert(c.key_eq() == test_equal_to<int>(9));
-        assert(c.get_allocator() == test_allocator<int>(10));
-        assert(!c.empty());
-        assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-        assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
-        assert(std::fabs(c.load_factor() - (float)c.size()/c.bucket_count()) < FLT_EPSILON);
-        assert(c.max_load_factor() == 1);
-    }
-    {
-        typedef std::unordered_set<int,
-                                   test_hash<int>,
-                                   test_equal_to<int>,
-                                   min_allocator<int>
-                                   > C;
-        typedef int P;
-        C c({
-                P(1),
-                P(2),
-                P(3),
-                P(4),
-                P(1),
-                P(2)
-            },
-            7,
-            test_hash<int>(8),
-            test_equal_to<int>(9),
-            min_allocator<int>()
-           );
-        LIBCPP_ASSERT(c.bucket_count() == 7);
-        assert(c.size() == 4);
-        assert(c.count(1) == 1);
-        assert(c.count(2) == 1);
-        assert(c.count(3) == 1);
-        assert(c.count(4) == 1);
-        assert(c.hash_function() == test_hash<int>(8));
-        assert(c.key_eq() == test_equal_to<int>(9));
-        assert(c.get_allocator() == min_allocator<int>());
-        assert(!c.empty());
-        assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-        assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
-        assert(std::fabs(c.load_factor() - (float)c.size()/c.bucket_count()) < FLT_EPSILON);
-        assert(c.max_load_factor() == 1);
-    }
+int main(int, char**) {
+  {
+    typedef std::unordered_set<int, test_hash<int>, test_equal_to<int>, test_allocator<int> > C;
+    typedef int P;
+    C c({P(1), P(2), P(3), P(4), P(1), P(2)}, 7, test_hash<int>(8), test_equal_to<int>(9), test_allocator<int>(10));
+    LIBCPP_ASSERT(c.bucket_count() == 7);
+    assert(c.size() == 4);
+    assert(c.count(1) == 1);
+    assert(c.count(2) == 1);
+    assert(c.count(3) == 1);
+    assert(c.count(4) == 1);
+    assert(c.hash_function() == test_hash<int>(8));
+    assert(c.key_eq() == test_equal_to<int>(9));
+    assert(c.get_allocator() == test_allocator<int>(10));
+    assert(!c.empty());
+    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
+    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
+    assert(std::fabs(c.load_factor() - (float)c.size() / c.bucket_count()) < FLT_EPSILON);
+    assert(c.max_load_factor() == 1);
+  }
+  {
+    typedef std::unordered_set<int, test_hash<int>, test_equal_to<int>, min_allocator<int> > C;
+    typedef int P;
+    C c({P(1), P(2), P(3), P(4), P(1), P(2)}, 7, test_hash<int>(8), test_equal_to<int>(9), min_allocator<int>());
+    LIBCPP_ASSERT(c.bucket_count() == 7);
+    assert(c.size() == 4);
+    assert(c.count(1) == 1);
+    assert(c.count(2) == 1);
+    assert(c.count(3) == 1);
+    assert(c.count(4) == 1);
+    assert(c.hash_function() == test_hash<int>(8));
+    assert(c.key_eq() == test_equal_to<int>(9));
+    assert(c.get_allocator() == min_allocator<int>());
+    assert(!c.empty());
+    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
+    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
+    assert(std::fabs(c.load_factor() - (float)c.size() / c.bucket_count()) < FLT_EPSILON);
+    assert(c.max_load_factor() == 1);
+  }
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/unord/unord.set/unord.set.cnstr/iter_iter.pass.cpp b/libcxx/test/std/containers/unord/unord.set/unord.set.cnstr/iter_iter.pass.cpp
index 8934490b6c711..76e2cee74df71 100644
--- a/libcxx/test/std/containers/unord/unord.set/unord.set.cnstr/iter_iter.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.set/unord.set.cnstr/iter_iter.pass.cpp
@@ -28,144 +28,103 @@
 #include "test_allocator.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef std::unordered_set<int,
-                                   test_hash<int>,
-                                   test_equal_to<int>,
-                                   test_allocator<int>
-                                   > C;
-        typedef int P;
-        P a[] =
-        {
-            P(1),
-            P(2),
-            P(3),
-            P(4),
-            P(1),
-            P(2)
-        };
-        C c(cpp17_input_iterator<P*>(a), cpp17_input_iterator<P*>(a + sizeof(a)/sizeof(a[0])));
-        assert(c.bucket_count() >= 5);
-        assert(c.size() == 4);
-        assert(c.count(1) == 1);
-        assert(c.count(2) == 1);
-        assert(c.count(3) == 1);
-        assert(c.count(4) == 1);
-        assert(c.hash_function() == test_hash<int>());
-        assert(c.key_eq() == test_equal_to<int>());
-        assert(c.get_allocator() == test_allocator<int>());
-        assert(!c.empty());
-        assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-        assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
-        assert(fabs(c.load_factor() - (float)c.size()/c.bucket_count()) < FLT_EPSILON);
-        assert(c.max_load_factor() == 1);
-    }
+int main(int, char**) {
+  {
+    typedef std::unordered_set<int, test_hash<int>, test_equal_to<int>, test_allocator<int> > C;
+    typedef int P;
+    P a[] = {P(1), P(2), P(3), P(4), P(1), P(2)};
+    C c(cpp17_input_iterator<P*>(a), cpp17_input_iterator<P*>(a + sizeof(a) / sizeof(a[0])));
+    assert(c.bucket_count() >= 5);
+    assert(c.size() == 4);
+    assert(c.count(1) == 1);
+    assert(c.count(2) == 1);
+    assert(c.count(3) == 1);
+    assert(c.count(4) == 1);
+    assert(c.hash_function() == test_hash<int>());
+    assert(c.key_eq() == test_equal_to<int>());
+    assert(c.get_allocator() == test_allocator<int>());
+    assert(!c.empty());
+    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
+    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
+    assert(fabs(c.load_factor() - (float)c.size() / c.bucket_count()) < FLT_EPSILON);
+    assert(c.max_load_factor() == 1);
+  }
 #if TEST_STD_VER >= 11
-    {
-        typedef std::unordered_set<int,
-                                   test_hash<int>,
-                                   test_equal_to<int>,
-                                   min_allocator<int>
-                                   > C;
-        typedef int P;
-        P a[] =
-        {
-            P(1),
-            P(2),
-            P(3),
-            P(4),
-            P(1),
-            P(2)
-        };
-        C c(cpp17_input_iterator<P*>(a), cpp17_input_iterator<P*>(a + sizeof(a)/sizeof(a[0])));
-        assert(c.bucket_count() >= 5);
-        assert(c.size() == 4);
-        assert(c.count(1) == 1);
-        assert(c.count(2) == 1);
-        assert(c.count(3) == 1);
-        assert(c.count(4) == 1);
-        assert(c.hash_function() == test_hash<int>());
-        assert(c.key_eq() == test_equal_to<int>());
-        assert(c.get_allocator() == min_allocator<int>());
-        assert(!c.empty());
-        assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-        assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
-        assert(fabs(c.load_factor() - (float)c.size()/c.bucket_count()) < FLT_EPSILON);
-        assert(c.max_load_factor() == 1);
-    }
-#if TEST_STD_VER > 11
-    {
-        typedef int T;
-        typedef test_hash<T> HF;
-        typedef test_equal_to<T> Comp;
-        typedef test_allocator<T> A;
-        typedef std::unordered_set<T, HF, Comp, A> C;
-        T arr[] =
-        {
-            T(1),
-            T(2),
-            T(3),
-            T(4),
-            T(1),
-            T(2)
-        };
-        A a(42);
-        C c(cpp17_input_iterator<T*>(arr), cpp17_input_iterator<T*>(arr + sizeof(arr)/sizeof(arr[0])), 12, a);
-        assert(c.bucket_count() >= 12);
-        assert(c.size() == 4);
-        assert(c.count(1) == 1);
-        assert(c.count(2) == 1);
-        assert(c.count(3) == 1);
-        assert(c.count(4) == 1);
-        assert(c.hash_function() == HF());
-        assert(c.key_eq() == Comp());
-        assert(c.get_allocator() == a);
-        assert(!(c.get_allocator() == A()));
-        assert(!c.empty());
-        assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-        assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
-        assert(fabs(c.load_factor() - (float)c.size()/c.bucket_count()) < FLT_EPSILON);
-        assert(c.max_load_factor() == 1);
-    }
-    {
-        typedef int T;
-        typedef test_hash<T> HF;
-        typedef test_equal_to<T> Comp;
-        typedef test_allocator<T> A;
-        typedef std::unordered_set<T, HF, Comp, A> C;
-        T arr[] =
-        {
-            T(1),
-            T(2),
-            T(3),
-            T(4),
-            T(1),
-            T(2)
-        };
-        HF hf(43);
-        A a(42);
-        C c(cpp17_input_iterator<T*>(arr), cpp17_input_iterator<T*>(arr + sizeof(arr)/sizeof(arr[0])), 16, hf, a);
-        assert(c.bucket_count() >= 16);
-        assert(c.size() == 4);
-        assert(c.count(1) == 1);
-        assert(c.count(2) == 1);
-        assert(c.count(3) == 1);
-        assert(c.count(4) == 1);
-        assert(c.hash_function() == hf);
-        assert(!(c.hash_function() == HF()));
-        assert(c.key_eq() == Comp());
-        assert(c.get_allocator() == a);
-        assert(!(c.get_allocator() == A()));
-        assert(!c.empty());
-        assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-        assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
-        assert(fabs(c.load_factor() - (float)c.size()/c.bucket_count()) < FLT_EPSILON);
-        assert(c.max_load_factor() == 1);
-    }
+  {
+    typedef std::unordered_set<int, test_hash<int>, test_equal_to<int>, min_allocator<int> > C;
+    typedef int P;
+    P a[] = {P(1), P(2), P(3), P(4), P(1), P(2)};
+    C c(cpp17_input_iterator<P*>(a), cpp17_input_iterator<P*>(a + sizeof(a) / sizeof(a[0])));
+    assert(c.bucket_count() >= 5);
+    assert(c.size() == 4);
+    assert(c.count(1) == 1);
+    assert(c.count(2) == 1);
+    assert(c.count(3) == 1);
+    assert(c.count(4) == 1);
+    assert(c.hash_function() == test_hash<int>());
+    assert(c.key_eq() == test_equal_to<int>());
+    assert(c.get_allocator() == min_allocator<int>());
+    assert(!c.empty());
+    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
+    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
+    assert(fabs(c.load_factor() - (float)c.size() / c.bucket_count()) < FLT_EPSILON);
+    assert(c.max_load_factor() == 1);
+  }
+#  if TEST_STD_VER > 11
+  {
+    typedef int T;
+    typedef test_hash<T> HF;
+    typedef test_equal_to<T> Comp;
+    typedef test_allocator<T> A;
+    typedef std::unordered_set<T, HF, Comp, A> C;
+    T arr[] = {T(1), T(2), T(3), T(4), T(1), T(2)};
+    A a(42);
+    C c(cpp17_input_iterator<T*>(arr), cpp17_input_iterator<T*>(arr + sizeof(arr) / sizeof(arr[0])), 12, a);
+    assert(c.bucket_count() >= 12);
+    assert(c.size() == 4);
+    assert(c.count(1) == 1);
+    assert(c.count(2) == 1);
+    assert(c.count(3) == 1);
+    assert(c.count(4) == 1);
+    assert(c.hash_function() == HF());
+    assert(c.key_eq() == Comp());
+    assert(c.get_allocator() == a);
+    assert(!(c.get_allocator() == A()));
+    assert(!c.empty());
+    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
+    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
+    assert(fabs(c.load_factor() - (float)c.size() / c.bucket_count()) < FLT_EPSILON);
+    assert(c.max_load_factor() == 1);
+  }
+  {
+    typedef int T;
+    typedef test_hash<T> HF;
+    typedef test_equal_to<T> Comp;
+    typedef test_allocator<T> A;
+    typedef std::unordered_set<T, HF, Comp, A> C;
+    T arr[] = {T(1), T(2), T(3), T(4), T(1), T(2)};
+    HF hf(43);
+    A a(42);
+    C c(cpp17_input_iterator<T*>(arr), cpp17_input_iterator<T*>(arr + sizeof(arr) / sizeof(arr[0])), 16, hf, a);
+    assert(c.bucket_count() >= 16);
+    assert(c.size() == 4);
+    assert(c.count(1) == 1);
+    assert(c.count(2) == 1);
+    assert(c.count(3) == 1);
+    assert(c.count(4) == 1);
+    assert(c.hash_function() == hf);
+    assert(!(c.hash_function() == HF()));
+    assert(c.key_eq() == Comp());
+    assert(c.get_allocator() == a);
+    assert(!(c.get_allocator() == A()));
+    assert(!c.empty());
+    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
+    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
+    assert(fabs(c.load_factor() - (float)c.size() / c.bucket_count()) < FLT_EPSILON);
+    assert(c.max_load_factor() == 1);
+  }
 
-#endif
+#  endif
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/unord/unord.set/unord.set.cnstr/iter_iter_size.pass.cpp b/libcxx/test/std/containers/unord/unord.set/unord.set.cnstr/iter_iter_size.pass.cpp
index 8b6149ac8f1f9..257df08e00f57 100644
--- a/libcxx/test/std/containers/unord/unord.set/unord.set.cnstr/iter_iter_size.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.set/unord.set.cnstr/iter_iter_size.pass.cpp
@@ -28,77 +28,48 @@
 #include "test_allocator.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef std::unordered_set<int,
-                                   test_hash<int>,
-                                   test_equal_to<int>,
-                                   test_allocator<int>
-                                   > C;
-        typedef int P;
-        P a[] =
-        {
-            P(1),
-            P(2),
-            P(3),
-            P(4),
-            P(1),
-            P(2)
-        };
-        C c(cpp17_input_iterator<P*>(a), cpp17_input_iterator<P*>(a + sizeof(a)/sizeof(a[0])),
-            7
-           );
-        LIBCPP_ASSERT(c.bucket_count() == 7);
-        assert(c.size() == 4);
-        assert(c.count(1) == 1);
-        assert(c.count(2) == 1);
-        assert(c.count(3) == 1);
-        assert(c.count(4) == 1);
-        assert(c.hash_function() == test_hash<int>());
-        assert(c.key_eq() == test_equal_to<int>());
-        assert(c.get_allocator() == test_allocator<int>());
-        assert(!c.empty());
-        assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-        assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
-        assert(std::fabs(c.load_factor() - (float)c.size()/c.bucket_count()) < FLT_EPSILON);
-        assert(c.max_load_factor() == 1);
-    }
+int main(int, char**) {
+  {
+    typedef std::unordered_set<int, test_hash<int>, test_equal_to<int>, test_allocator<int> > C;
+    typedef int P;
+    P a[] = {P(1), P(2), P(3), P(4), P(1), P(2)};
+    C c(cpp17_input_iterator<P*>(a), cpp17_input_iterator<P*>(a + sizeof(a) / sizeof(a[0])), 7);
+    LIBCPP_ASSERT(c.bucket_count() == 7);
+    assert(c.size() == 4);
+    assert(c.count(1) == 1);
+    assert(c.count(2) == 1);
+    assert(c.count(3) == 1);
+    assert(c.count(4) == 1);
+    assert(c.hash_function() == test_hash<int>());
+    assert(c.key_eq() == test_equal_to<int>());
+    assert(c.get_allocator() == test_allocator<int>());
+    assert(!c.empty());
+    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
+    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
+    assert(std::fabs(c.load_factor() - (float)c.size() / c.bucket_count()) < FLT_EPSILON);
+    assert(c.max_load_factor() == 1);
+  }
 #if TEST_STD_VER >= 11
-    {
-        typedef std::unordered_set<int,
-                                   test_hash<int>,
-                                   test_equal_to<int>,
-                                   min_allocator<int>
-                                   > C;
-        typedef int P;
-        P a[] =
-        {
-            P(1),
-            P(2),
-            P(3),
-            P(4),
-            P(1),
-            P(2)
-        };
-        C c(cpp17_input_iterator<P*>(a), cpp17_input_iterator<P*>(a + sizeof(a)/sizeof(a[0])),
-            7
-           );
-        LIBCPP_ASSERT(c.bucket_count() == 7);
-        assert(c.size() == 4);
-        assert(c.count(1) == 1);
-        assert(c.count(2) == 1);
-        assert(c.count(3) == 1);
-        assert(c.count(4) == 1);
-        assert(c.hash_function() == test_hash<int>());
-        assert(c.key_eq() == test_equal_to<int>());
-        assert(c.get_allocator() == min_allocator<int>());
-        assert(!c.empty());
-        assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-        assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
-        assert(std::fabs(c.load_factor() - (float)c.size()/c.bucket_count()) < FLT_EPSILON);
-        assert(c.max_load_factor() == 1);
-    }
+  {
+    typedef std::unordered_set<int, test_hash<int>, test_equal_to<int>, min_allocator<int> > C;
+    typedef int P;
+    P a[] = {P(1), P(2), P(3), P(4), P(1), P(2)};
+    C c(cpp17_input_iterator<P*>(a), cpp17_input_iterator<P*>(a + sizeof(a) / sizeof(a[0])), 7);
+    LIBCPP_ASSERT(c.bucket_count() == 7);
+    assert(c.size() == 4);
+    assert(c.count(1) == 1);
+    assert(c.count(2) == 1);
+    assert(c.count(3) == 1);
+    assert(c.count(4) == 1);
+    assert(c.hash_function() == test_hash<int>());
+    assert(c.key_eq() == test_equal_to<int>());
+    assert(c.get_allocator() == min_allocator<int>());
+    assert(!c.empty());
+    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
+    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
+    assert(std::fabs(c.load_factor() - (float)c.size() / c.bucket_count()) < FLT_EPSILON);
+    assert(c.max_load_factor() == 1);
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/unord/unord.set/unord.set.cnstr/iter_iter_size_allocator.pass.cpp b/libcxx/test/std/containers/unord/unord.set/unord.set.cnstr/iter_iter_size_allocator.pass.cpp
index cfed6dfbb36c1..414485466d71e 100644
--- a/libcxx/test/std/containers/unord/unord.set/unord.set.cnstr/iter_iter_size_allocator.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.set/unord.set.cnstr/iter_iter_size_allocator.pass.cpp
@@ -33,45 +33,29 @@
 #include "min_allocator.h"
 
 template <class Allocator>
-void test(const Allocator& alloc)
-{
-    typedef std::unordered_set<int,
-                               test_hash<int>,
-                               test_equal_to<int>,
-                               Allocator> C;
-    int a[] =
-    {
-        1,
-        2,
-        3,
-        4,
-        1,
-        2
-    };
-    C c(cpp17_input_iterator<int*>(a), cpp17_input_iterator<int*>(a + sizeof(a)/sizeof(a[0])),
-        7,
-        alloc
-        );
-    LIBCPP_ASSERT(c.bucket_count() == 7);
-    assert(c.size() == 4);
-    assert(c.count(1) == 1);
-    assert(c.count(2) == 1);
-    assert(c.count(3) == 1);
-    assert(c.count(4) == 1);
-    assert(c.hash_function() == test_hash<int>());
-    assert(c.key_eq() == test_equal_to<int>());
-    assert(c.get_allocator() == alloc);
-    assert(!c.empty());
-    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-    assert(std::fabs(c.load_factor() - (float)c.size()/c.bucket_count()) < FLT_EPSILON);
-    assert(c.max_load_factor() == 1);
+void test(const Allocator& alloc) {
+  typedef std::unordered_set<int, test_hash<int>, test_equal_to<int>, Allocator> C;
+  int a[] = {1, 2, 3, 4, 1, 2};
+  C c(cpp17_input_iterator<int*>(a), cpp17_input_iterator<int*>(a + sizeof(a) / sizeof(a[0])), 7, alloc);
+  LIBCPP_ASSERT(c.bucket_count() == 7);
+  assert(c.size() == 4);
+  assert(c.count(1) == 1);
+  assert(c.count(2) == 1);
+  assert(c.count(3) == 1);
+  assert(c.count(4) == 1);
+  assert(c.hash_function() == test_hash<int>());
+  assert(c.key_eq() == test_equal_to<int>());
+  assert(c.get_allocator() == alloc);
+  assert(!c.empty());
+  assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
+  assert(std::fabs(c.load_factor() - (float)c.size() / c.bucket_count()) < FLT_EPSILON);
+  assert(c.max_load_factor() == 1);
 }
 
-int main(int, char**)
-{
-    test(test_allocator<int>(10));
-    test(min_allocator<int>());
-    test(explicit_allocator<int>());
+int main(int, char**) {
+  test(test_allocator<int>(10));
+  test(min_allocator<int>());
+  test(explicit_allocator<int>());
 
-    return 0;
+  return 0;
 }
diff --git a/libcxx/test/std/containers/unord/unord.set/unord.set.cnstr/iter_iter_size_hash.pass.cpp b/libcxx/test/std/containers/unord/unord.set/unord.set.cnstr/iter_iter_size_hash.pass.cpp
index 93ff746b3e039..92a26608dd6ec 100644
--- a/libcxx/test/std/containers/unord/unord.set/unord.set.cnstr/iter_iter_size_hash.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.set/unord.set.cnstr/iter_iter_size_hash.pass.cpp
@@ -29,79 +29,48 @@
 #include "test_allocator.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef std::unordered_set<int,
-                                   test_hash<int>,
-                                   test_equal_to<int>,
-                                   test_allocator<int>
-                                   > C;
-        typedef int P;
-        P a[] =
-        {
-            P(1),
-            P(2),
-            P(3),
-            P(4),
-            P(1),
-            P(2)
-        };
-        C c(cpp17_input_iterator<P*>(a), cpp17_input_iterator<P*>(a + sizeof(a)/sizeof(a[0])),
-            7,
-            test_hash<int>(8)
-           );
-        LIBCPP_ASSERT(c.bucket_count() == 7);
-        assert(c.size() == 4);
-        assert(c.count(1) == 1);
-        assert(c.count(2) == 1);
-        assert(c.count(3) == 1);
-        assert(c.count(4) == 1);
-        assert(c.hash_function() == test_hash<int>(8));
-        assert(c.key_eq() == test_equal_to<int>());
-        assert(c.get_allocator() == test_allocator<int>());
-        assert(!c.empty());
-        assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-        assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
-        assert(std::fabs(c.load_factor() - (float)c.size()/c.bucket_count()) < FLT_EPSILON);
-        assert(c.max_load_factor() == 1);
-    }
+int main(int, char**) {
+  {
+    typedef std::unordered_set<int, test_hash<int>, test_equal_to<int>, test_allocator<int> > C;
+    typedef int P;
+    P a[] = {P(1), P(2), P(3), P(4), P(1), P(2)};
+    C c(cpp17_input_iterator<P*>(a), cpp17_input_iterator<P*>(a + sizeof(a) / sizeof(a[0])), 7, test_hash<int>(8));
+    LIBCPP_ASSERT(c.bucket_count() == 7);
+    assert(c.size() == 4);
+    assert(c.count(1) == 1);
+    assert(c.count(2) == 1);
+    assert(c.count(3) == 1);
+    assert(c.count(4) == 1);
+    assert(c.hash_function() == test_hash<int>(8));
+    assert(c.key_eq() == test_equal_to<int>());
+    assert(c.get_allocator() == test_allocator<int>());
+    assert(!c.empty());
+    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
+    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
+    assert(std::fabs(c.load_factor() - (float)c.size() / c.bucket_count()) < FLT_EPSILON);
+    assert(c.max_load_factor() == 1);
+  }
 #if TEST_STD_VER >= 11
-    {
-        typedef std::unordered_set<int,
-                                   test_hash<int>,
-                                   test_equal_to<int>,
-                                   min_allocator<int>
-                                   > C;
-        typedef int P;
-        P a[] =
-        {
-            P(1),
-            P(2),
-            P(3),
-            P(4),
-            P(1),
-            P(2)
-        };
-        C c(cpp17_input_iterator<P*>(a), cpp17_input_iterator<P*>(a + sizeof(a)/sizeof(a[0])),
-            7,
-            test_hash<int>(8)
-           );
-        LIBCPP_ASSERT(c.bucket_count() == 7);
-        assert(c.size() == 4);
-        assert(c.count(1) == 1);
-        assert(c.count(2) == 1);
-        assert(c.count(3) == 1);
-        assert(c.count(4) == 1);
-        assert(c.hash_function() == test_hash<int>(8));
-        assert(c.key_eq() == test_equal_to<int>());
-        assert(c.get_allocator() == min_allocator<int>());
-        assert(!c.empty());
-        assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-        assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
-        assert(std::fabs(c.load_factor() - (float)c.size()/c.bucket_count()) < FLT_EPSILON);
-        assert(c.max_load_factor() == 1);
-    }
+  {
+    typedef std::unordered_set<int, test_hash<int>, test_equal_to<int>, min_allocator<int> > C;
+    typedef int P;
+    P a[] = {P(1), P(2), P(3), P(4), P(1), P(2)};
+    C c(cpp17_input_iterator<P*>(a), cpp17_input_iterator<P*>(a + sizeof(a) / sizeof(a[0])), 7, test_hash<int>(8));
+    LIBCPP_ASSERT(c.bucket_count() == 7);
+    assert(c.size() == 4);
+    assert(c.count(1) == 1);
+    assert(c.count(2) == 1);
+    assert(c.count(3) == 1);
+    assert(c.count(4) == 1);
+    assert(c.hash_function() == test_hash<int>(8));
+    assert(c.key_eq() == test_equal_to<int>());
+    assert(c.get_allocator() == min_allocator<int>());
+    assert(!c.empty());
+    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
+    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
+    assert(std::fabs(c.load_factor() - (float)c.size() / c.bucket_count()) < FLT_EPSILON);
+    assert(c.max_load_factor() == 1);
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/unord/unord.set/unord.set.cnstr/iter_iter_size_hash_allocator.pass.cpp b/libcxx/test/std/containers/unord/unord.set/unord.set.cnstr/iter_iter_size_hash_allocator.pass.cpp
index 2c44cba3b7520..ba986eba5f71c 100644
--- a/libcxx/test/std/containers/unord/unord.set/unord.set.cnstr/iter_iter_size_hash_allocator.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.set/unord.set.cnstr/iter_iter_size_hash_allocator.pass.cpp
@@ -33,46 +33,33 @@
 #include "min_allocator.h"
 
 template <class Allocator>
-void test(const Allocator& alloc)
-{
-    typedef std::unordered_set<int,
-                               test_hash<int>,
-                               test_equal_to<int>,
-                               Allocator> C;
-    int a[] =
-    {
-        1,
-        2,
-        3,
-        4,
-        1,
-        2
-    };
-    C c(cpp17_input_iterator<int*>(a), cpp17_input_iterator<int*>(a + sizeof(a)/sizeof(a[0])),
-        7,
-        test_hash<int>(5),
-        alloc
-        );
-    LIBCPP_ASSERT(c.bucket_count() == 7);
-    assert(c.size() == 4);
-    assert(c.count(1) == 1);
-    assert(c.count(2) == 1);
-    assert(c.count(3) == 1);
-    assert(c.count(4) == 1);
-    assert(c.hash_function() == test_hash<int>(5));
-    assert(c.key_eq() == test_equal_to<int>());
-    assert(c.get_allocator() == alloc);
-    assert(!c.empty());
-    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-    assert(std::fabs(c.load_factor() - (float)c.size()/c.bucket_count()) < FLT_EPSILON);
-    assert(c.max_load_factor() == 1);
+void test(const Allocator& alloc) {
+  typedef std::unordered_set<int, test_hash<int>, test_equal_to<int>, Allocator> C;
+  int a[] = {1, 2, 3, 4, 1, 2};
+  C c(cpp17_input_iterator<int*>(a),
+      cpp17_input_iterator<int*>(a + sizeof(a) / sizeof(a[0])),
+      7,
+      test_hash<int>(5),
+      alloc);
+  LIBCPP_ASSERT(c.bucket_count() == 7);
+  assert(c.size() == 4);
+  assert(c.count(1) == 1);
+  assert(c.count(2) == 1);
+  assert(c.count(3) == 1);
+  assert(c.count(4) == 1);
+  assert(c.hash_function() == test_hash<int>(5));
+  assert(c.key_eq() == test_equal_to<int>());
+  assert(c.get_allocator() == alloc);
+  assert(!c.empty());
+  assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
+  assert(std::fabs(c.load_factor() - (float)c.size() / c.bucket_count()) < FLT_EPSILON);
+  assert(c.max_load_factor() == 1);
 }
 
-int main(int, char**)
-{
-    test(test_allocator<int>(10));
-    test(min_allocator<int>());
-    test(explicit_allocator<int>());
+int main(int, char**) {
+  test(test_allocator<int>(10));
+  test(min_allocator<int>());
+  test(explicit_allocator<int>());
 
-    return 0;
+  return 0;
 }
diff --git a/libcxx/test/std/containers/unord/unord.set/unord.set.cnstr/iter_iter_size_hash_equal.pass.cpp b/libcxx/test/std/containers/unord/unord.set/unord.set.cnstr/iter_iter_size_hash_equal.pass.cpp
index 272c08058353d..eb0c18bf78ca2 100644
--- a/libcxx/test/std/containers/unord/unord.set/unord.set.cnstr/iter_iter_size_hash_equal.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.set/unord.set.cnstr/iter_iter_size_hash_equal.pass.cpp
@@ -29,81 +29,56 @@
 #include "test_allocator.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef std::unordered_set<int,
-                                   test_hash<int>,
-                                   test_equal_to<int>,
-                                   test_allocator<int>
-                                   > C;
-        typedef int P;
-        P a[] =
-        {
-            P(1),
-            P(2),
-            P(3),
-            P(4),
-            P(1),
-            P(2)
-        };
-        C c(cpp17_input_iterator<P*>(a), cpp17_input_iterator<P*>(a + sizeof(a)/sizeof(a[0])),
-            7,
-            test_hash<int>(8),
-            test_equal_to<int>(9)
-           );
-        LIBCPP_ASSERT(c.bucket_count() == 7);
-        assert(c.size() == 4);
-        assert(c.count(1) == 1);
-        assert(c.count(2) == 1);
-        assert(c.count(3) == 1);
-        assert(c.count(4) == 1);
-        assert(c.hash_function() == test_hash<int>(8));
-        assert(c.key_eq() == test_equal_to<int>(9));
-        assert(c.get_allocator() == test_allocator<int>());
-        assert(!c.empty());
-        assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-        assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
-        assert(std::fabs(c.load_factor() - (float)c.size()/c.bucket_count()) < FLT_EPSILON);
-        assert(c.max_load_factor() == 1);
-    }
+int main(int, char**) {
+  {
+    typedef std::unordered_set<int, test_hash<int>, test_equal_to<int>, test_allocator<int> > C;
+    typedef int P;
+    P a[] = {P(1), P(2), P(3), P(4), P(1), P(2)};
+    C c(cpp17_input_iterator<P*>(a),
+        cpp17_input_iterator<P*>(a + sizeof(a) / sizeof(a[0])),
+        7,
+        test_hash<int>(8),
+        test_equal_to<int>(9));
+    LIBCPP_ASSERT(c.bucket_count() == 7);
+    assert(c.size() == 4);
+    assert(c.count(1) == 1);
+    assert(c.count(2) == 1);
+    assert(c.count(3) == 1);
+    assert(c.count(4) == 1);
+    assert(c.hash_function() == test_hash<int>(8));
+    assert(c.key_eq() == test_equal_to<int>(9));
+    assert(c.get_allocator() == test_allocator<int>());
+    assert(!c.empty());
+    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
+    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
+    assert(std::fabs(c.load_factor() - (float)c.size() / c.bucket_count()) < FLT_EPSILON);
+    assert(c.max_load_factor() == 1);
+  }
 #if TEST_STD_VER >= 11
-    {
-        typedef std::unordered_set<int,
-                                   test_hash<int>,
-                                   test_equal_to<int>,
-                                   min_allocator<int>
-                                   > C;
-        typedef int P;
-        P a[] =
-        {
-            P(1),
-            P(2),
-            P(3),
-            P(4),
-            P(1),
-            P(2)
-        };
-        C c(cpp17_input_iterator<P*>(a), cpp17_input_iterator<P*>(a + sizeof(a)/sizeof(a[0])),
-            7,
-            test_hash<int>(8),
-            test_equal_to<int>(9)
-           );
-        LIBCPP_ASSERT(c.bucket_count() == 7);
-        assert(c.size() == 4);
-        assert(c.count(1) == 1);
-        assert(c.count(2) == 1);
-        assert(c.count(3) == 1);
-        assert(c.count(4) == 1);
-        assert(c.hash_function() == test_hash<int>(8));
-        assert(c.key_eq() == test_equal_to<int>(9));
-        assert(c.get_allocator() == min_allocator<int>());
-        assert(!c.empty());
-        assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-        assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
-        assert(std::fabs(c.load_factor() - (float)c.size()/c.bucket_count()) < FLT_EPSILON);
-        assert(c.max_load_factor() == 1);
-    }
+  {
+    typedef std::unordered_set<int, test_hash<int>, test_equal_to<int>, min_allocator<int> > C;
+    typedef int P;
+    P a[] = {P(1), P(2), P(3), P(4), P(1), P(2)};
+    C c(cpp17_input_iterator<P*>(a),
+        cpp17_input_iterator<P*>(a + sizeof(a) / sizeof(a[0])),
+        7,
+        test_hash<int>(8),
+        test_equal_to<int>(9));
+    LIBCPP_ASSERT(c.bucket_count() == 7);
+    assert(c.size() == 4);
+    assert(c.count(1) == 1);
+    assert(c.count(2) == 1);
+    assert(c.count(3) == 1);
+    assert(c.count(4) == 1);
+    assert(c.hash_function() == test_hash<int>(8));
+    assert(c.key_eq() == test_equal_to<int>(9));
+    assert(c.get_allocator() == min_allocator<int>());
+    assert(!c.empty());
+    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
+    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
+    assert(std::fabs(c.load_factor() - (float)c.size() / c.bucket_count()) < FLT_EPSILON);
+    assert(c.max_load_factor() == 1);
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/unord/unord.set/unord.set.cnstr/iter_iter_size_hash_equal_allocator.pass.cpp b/libcxx/test/std/containers/unord/unord.set/unord.set.cnstr/iter_iter_size_hash_equal_allocator.pass.cpp
index 024dc4a47721b..6eb7a26e19070 100644
--- a/libcxx/test/std/containers/unord/unord.set/unord.set.cnstr/iter_iter_size_hash_equal_allocator.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.set/unord.set.cnstr/iter_iter_size_hash_equal_allocator.pass.cpp
@@ -30,83 +30,58 @@
 #include "test_allocator.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef std::unordered_set<int,
-                                   test_hash<int>,
-                                   test_equal_to<int>,
-                                   test_allocator<int>
-                                   > C;
-        typedef int P;
-        P a[] =
-        {
-            P(1),
-            P(2),
-            P(3),
-            P(4),
-            P(1),
-            P(2)
-        };
-        C c(cpp17_input_iterator<P*>(a), cpp17_input_iterator<P*>(a + sizeof(a)/sizeof(a[0])),
-            7,
-            test_hash<int>(8),
-            test_equal_to<int>(9),
-            test_allocator<int>(10)
-           );
-        LIBCPP_ASSERT(c.bucket_count() == 7);
-        assert(c.size() == 4);
-        assert(c.count(1) == 1);
-        assert(c.count(2) == 1);
-        assert(c.count(3) == 1);
-        assert(c.count(4) == 1);
-        assert(c.hash_function() == test_hash<int>(8));
-        assert(c.key_eq() == test_equal_to<int>(9));
-        assert(c.get_allocator() == test_allocator<int>(10));
-        assert(!c.empty());
-        assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-        assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
-        assert(std::fabs(c.load_factor() - (float)c.size()/c.bucket_count()) < FLT_EPSILON);
-        assert(c.max_load_factor() == 1);
-    }
+int main(int, char**) {
+  {
+    typedef std::unordered_set<int, test_hash<int>, test_equal_to<int>, test_allocator<int> > C;
+    typedef int P;
+    P a[] = {P(1), P(2), P(3), P(4), P(1), P(2)};
+    C c(cpp17_input_iterator<P*>(a),
+        cpp17_input_iterator<P*>(a + sizeof(a) / sizeof(a[0])),
+        7,
+        test_hash<int>(8),
+        test_equal_to<int>(9),
+        test_allocator<int>(10));
+    LIBCPP_ASSERT(c.bucket_count() == 7);
+    assert(c.size() == 4);
+    assert(c.count(1) == 1);
+    assert(c.count(2) == 1);
+    assert(c.count(3) == 1);
+    assert(c.count(4) == 1);
+    assert(c.hash_function() == test_hash<int>(8));
+    assert(c.key_eq() == test_equal_to<int>(9));
+    assert(c.get_allocator() == test_allocator<int>(10));
+    assert(!c.empty());
+    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
+    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
+    assert(std::fabs(c.load_factor() - (float)c.size() / c.bucket_count()) < FLT_EPSILON);
+    assert(c.max_load_factor() == 1);
+  }
 #if TEST_STD_VER >= 11
-    {
-        typedef std::unordered_set<int,
-                                   test_hash<int>,
-                                   test_equal_to<int>,
-                                   min_allocator<int>
-                                   > C;
-        typedef int P;
-        P a[] =
-        {
-            P(1),
-            P(2),
-            P(3),
-            P(4),
-            P(1),
-            P(2)
-        };
-        C c(cpp17_input_iterator<P*>(a), cpp17_input_iterator<P*>(a + sizeof(a)/sizeof(a[0])),
-            7,
-            test_hash<int>(8),
-            test_equal_to<int>(9),
-            min_allocator<int>()
-           );
-        LIBCPP_ASSERT(c.bucket_count() == 7);
-        assert(c.size() == 4);
-        assert(c.count(1) == 1);
-        assert(c.count(2) == 1);
-        assert(c.count(3) == 1);
-        assert(c.count(4) == 1);
-        assert(c.hash_function() == test_hash<int>(8));
-        assert(c.key_eq() == test_equal_to<int>(9));
-        assert(c.get_allocator() == min_allocator<int>());
-        assert(!c.empty());
-        assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-        assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
-        assert(std::fabs(c.load_factor() - (float)c.size()/c.bucket_count()) < FLT_EPSILON);
-        assert(c.max_load_factor() == 1);
-    }
+  {
+    typedef std::unordered_set<int, test_hash<int>, test_equal_to<int>, min_allocator<int> > C;
+    typedef int P;
+    P a[] = {P(1), P(2), P(3), P(4), P(1), P(2)};
+    C c(cpp17_input_iterator<P*>(a),
+        cpp17_input_iterator<P*>(a + sizeof(a) / sizeof(a[0])),
+        7,
+        test_hash<int>(8),
+        test_equal_to<int>(9),
+        min_allocator<int>());
+    LIBCPP_ASSERT(c.bucket_count() == 7);
+    assert(c.size() == 4);
+    assert(c.count(1) == 1);
+    assert(c.count(2) == 1);
+    assert(c.count(3) == 1);
+    assert(c.count(4) == 1);
+    assert(c.hash_function() == test_hash<int>(8));
+    assert(c.key_eq() == test_equal_to<int>(9));
+    assert(c.get_allocator() == min_allocator<int>());
+    assert(!c.empty());
+    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
+    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
+    assert(std::fabs(c.load_factor() - (float)c.size() / c.bucket_count()) < FLT_EPSILON);
+    assert(c.max_load_factor() == 1);
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/unord/unord.set/unord.set.cnstr/move.pass.cpp b/libcxx/test/std/containers/unord/unord.set/unord.set.cnstr/move.pass.cpp
index 80131be690d8b..ce05ff552470d 100644
--- a/libcxx/test/std/containers/unord/unord.set/unord.set.cnstr/move.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.set/unord.set.cnstr/move.pass.cpp
@@ -28,142 +28,91 @@
 #include "test_allocator.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef std::unordered_set<int,
-                                   test_hash<int>,
-                                   test_equal_to<int>,
-                                   test_allocator<int>
-                                   > C;
-        C c0(7,
-            test_hash<int>(8),
-            test_equal_to<int>(9),
-            test_allocator<int>(10)
-           );
-        C c = std::move(c0);
-        LIBCPP_ASSERT(c.bucket_count() == 7);
-        assert(c.size() == 0);
-        assert(c.hash_function() == test_hash<int>(8));
-        assert(c.key_eq() == test_equal_to<int>(9));
-        assert(c.get_allocator() == test_allocator<int>(10));
-        assert(c.empty());
-        assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-        assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
-        assert(c.load_factor() == 0);
-        assert(c.max_load_factor() == 1);
+int main(int, char**) {
+  {
+    typedef std::unordered_set<int, test_hash<int>, test_equal_to<int>, test_allocator<int> > C;
+    C c0(7, test_hash<int>(8), test_equal_to<int>(9), test_allocator<int>(10));
+    C c = std::move(c0);
+    LIBCPP_ASSERT(c.bucket_count() == 7);
+    assert(c.size() == 0);
+    assert(c.hash_function() == test_hash<int>(8));
+    assert(c.key_eq() == test_equal_to<int>(9));
+    assert(c.get_allocator() == test_allocator<int>(10));
+    assert(c.empty());
+    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
+    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
+    assert(c.load_factor() == 0);
+    assert(c.max_load_factor() == 1);
 
-        assert(c0.empty());
-    }
-    {
-        typedef std::unordered_set<int,
-                                   test_hash<int>,
-                                   test_equal_to<int>,
-                                   test_allocator<int>
-                                   > C;
-        typedef int P;
-        P a[] =
-        {
-            P(1),
-            P(2),
-            P(3),
-            P(4),
-            P(1),
-            P(2)
-        };
-        C c0(a, a + sizeof(a)/sizeof(a[0]),
-            7,
-            test_hash<int>(8),
-            test_equal_to<int>(9),
-            test_allocator<int>(10)
-           );
-        C::iterator it0 = c0.begin();
-        C c = std::move(c0);
-        assert(it0 == c.begin()); // Iterators remain valid
-        LIBCPP_ASSERT(c.bucket_count() == 7);
-        assert(c.size() == 4);
-        assert(c.count(1) == 1);
-        assert(c.count(2) == 1);
-        assert(c.count(3) == 1);
-        assert(c.count(4) == 1);
-        assert(c.hash_function() == test_hash<int>(8));
-        assert(c.key_eq() == test_equal_to<int>(9));
-        assert(c.get_allocator() == test_allocator<int>(10));
-        assert(!c.empty());
-        assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-        assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
-        assert(std::fabs(c.load_factor() - (float)c.size()/c.bucket_count()) < FLT_EPSILON);
-        assert(c.max_load_factor() == 1);
+    assert(c0.empty());
+  }
+  {
+    typedef std::unordered_set<int, test_hash<int>, test_equal_to<int>, test_allocator<int> > C;
+    typedef int P;
+    P a[] = {P(1), P(2), P(3), P(4), P(1), P(2)};
+    C c0(a, a + sizeof(a) / sizeof(a[0]), 7, test_hash<int>(8), test_equal_to<int>(9), test_allocator<int>(10));
+    C::iterator it0 = c0.begin();
+    C c             = std::move(c0);
+    assert(it0 == c.begin()); // Iterators remain valid
+    LIBCPP_ASSERT(c.bucket_count() == 7);
+    assert(c.size() == 4);
+    assert(c.count(1) == 1);
+    assert(c.count(2) == 1);
+    assert(c.count(3) == 1);
+    assert(c.count(4) == 1);
+    assert(c.hash_function() == test_hash<int>(8));
+    assert(c.key_eq() == test_equal_to<int>(9));
+    assert(c.get_allocator() == test_allocator<int>(10));
+    assert(!c.empty());
+    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
+    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
+    assert(std::fabs(c.load_factor() - (float)c.size() / c.bucket_count()) < FLT_EPSILON);
+    assert(c.max_load_factor() == 1);
 
-        assert(c0.empty());
-    }
-    {
-        typedef std::unordered_set<int,
-                                   test_hash<int>,
-                                   test_equal_to<int>,
-                                   min_allocator<int>
-                                   > C;
-        C c0(7,
-            test_hash<int>(8),
-            test_equal_to<int>(9),
-            min_allocator<int>()
-           );
-        C c = std::move(c0);
-        LIBCPP_ASSERT(c.bucket_count() == 7);
-        assert(c.size() == 0);
-        assert(c.hash_function() == test_hash<int>(8));
-        assert(c.key_eq() == test_equal_to<int>(9));
-        assert(c.get_allocator() == min_allocator<int>());
-        assert(c.empty());
-        assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-        assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
-        assert(c.load_factor() == 0);
-        assert(c.max_load_factor() == 1);
+    assert(c0.empty());
+  }
+  {
+    typedef std::unordered_set<int, test_hash<int>, test_equal_to<int>, min_allocator<int> > C;
+    C c0(7, test_hash<int>(8), test_equal_to<int>(9), min_allocator<int>());
+    C c = std::move(c0);
+    LIBCPP_ASSERT(c.bucket_count() == 7);
+    assert(c.size() == 0);
+    assert(c.hash_function() == test_hash<int>(8));
+    assert(c.key_eq() == test_equal_to<int>(9));
+    assert(c.get_allocator() == min_allocator<int>());
+    assert(c.empty());
+    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
+    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
+    assert(c.load_factor() == 0);
+    assert(c.max_load_factor() == 1);
 
-        assert(c0.empty());
-    }
-    {
-        typedef std::unordered_set<int,
-                                   test_hash<int>,
-                                   test_equal_to<int>,
-                                   min_allocator<int>
-                                   > C;
-        typedef int P;
-        P a[] =
-        {
-            P(1),
-            P(2),
-            P(3),
-            P(4),
-            P(1),
-            P(2)
-        };
-        C c0(a, a + sizeof(a)/sizeof(a[0]),
-            7,
-            test_hash<int>(8),
-            test_equal_to<int>(9),
-            min_allocator<int>()
-           );
-        C::iterator it0 = c0.begin();
-        C c = std::move(c0);
-        assert(it0 == c.begin()); // Iterators remain valid
-        LIBCPP_ASSERT(c.bucket_count() == 7);
-        assert(c.size() == 4);
-        assert(c.count(1) == 1);
-        assert(c.count(2) == 1);
-        assert(c.count(3) == 1);
-        assert(c.count(4) == 1);
-        assert(c.hash_function() == test_hash<int>(8));
-        assert(c.key_eq() == test_equal_to<int>(9));
-        assert(c.get_allocator() == min_allocator<int>());
-        assert(!c.empty());
-        assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-        assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
-        assert(std::fabs(c.load_factor() - (float)c.size()/c.bucket_count()) < FLT_EPSILON);
-        assert(c.max_load_factor() == 1);
+    assert(c0.empty());
+  }
+  {
+    typedef std::unordered_set<int, test_hash<int>, test_equal_to<int>, min_allocator<int> > C;
+    typedef int P;
+    P a[] = {P(1), P(2), P(3), P(4), P(1), P(2)};
+    C c0(a, a + sizeof(a) / sizeof(a[0]), 7, test_hash<int>(8), test_equal_to<int>(9), min_allocator<int>());
+    C::iterator it0 = c0.begin();
+    C c             = std::move(c0);
+    assert(it0 == c.begin()); // Iterators remain valid
+    LIBCPP_ASSERT(c.bucket_count() == 7);
+    assert(c.size() == 4);
+    assert(c.count(1) == 1);
+    assert(c.count(2) == 1);
+    assert(c.count(3) == 1);
+    assert(c.count(4) == 1);
+    assert(c.hash_function() == test_hash<int>(8));
+    assert(c.key_eq() == test_equal_to<int>(9));
+    assert(c.get_allocator() == min_allocator<int>());
+    assert(!c.empty());
+    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
+    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
+    assert(std::fabs(c.load_factor() - (float)c.size() / c.bucket_count()) < FLT_EPSILON);
+    assert(c.max_load_factor() == 1);
 
-        assert(c0.empty());
-    }
+    assert(c0.empty());
+  }
 
-    return 0;
+  return 0;
 }
diff --git a/libcxx/test/std/containers/unord/unord.set/unord.set.cnstr/move_alloc.pass.cpp b/libcxx/test/std/containers/unord/unord.set/unord.set.cnstr/move_alloc.pass.cpp
index 41bd56ba43549..7bcd679d012e2 100644
--- a/libcxx/test/std/containers/unord/unord.set/unord.set.cnstr/move_alloc.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.set/unord.set.cnstr/move_alloc.pass.cpp
@@ -28,131 +28,79 @@
 #include "test_allocator.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef int P;
-        typedef test_allocator<int> A;
-        typedef std::unordered_set<int,
-                                   test_hash<int>,
-                                   test_equal_to<int>,
-                                   A
-                                   > C;
-        P a[] =
-        {
-            P(1),
-            P(2),
-            P(3),
-            P(4),
-            P(1),
-            P(2)
-        };
-        C c0(a, a + sizeof(a)/sizeof(a[0]),
-            7,
-            test_hash<int>(8),
-            test_equal_to<int>(9),
-            A(10)
-           );
-        C c(std::move(c0), A(12));
-        assert(c.bucket_count() >= 5);
-        assert(c.size() == 4);
-        assert(c.count(1) == 1);
-        assert(c.count(2) == 1);
-        assert(c.count(3) == 1);
-        assert(c.count(4) == 1);
-        assert(c.hash_function() == test_hash<int>(8));
-        assert(c.key_eq() == test_equal_to<int>(9));
-        assert(c.get_allocator() == A(12));
-        assert(!c.empty());
-        assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-        assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
-        assert(std::fabs(c.load_factor() - (float)c.size()/c.bucket_count()) < FLT_EPSILON);
-        assert(c.max_load_factor() == 1);
+int main(int, char**) {
+  {
+    typedef int P;
+    typedef test_allocator<int> A;
+    typedef std::unordered_set<int, test_hash<int>, test_equal_to<int>, A > C;
+    P a[] = {P(1), P(2), P(3), P(4), P(1), P(2)};
+    C c0(a, a + sizeof(a) / sizeof(a[0]), 7, test_hash<int>(8), test_equal_to<int>(9), A(10));
+    C c(std::move(c0), A(12));
+    assert(c.bucket_count() >= 5);
+    assert(c.size() == 4);
+    assert(c.count(1) == 1);
+    assert(c.count(2) == 1);
+    assert(c.count(3) == 1);
+    assert(c.count(4) == 1);
+    assert(c.hash_function() == test_hash<int>(8));
+    assert(c.key_eq() == test_equal_to<int>(9));
+    assert(c.get_allocator() == A(12));
+    assert(!c.empty());
+    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
+    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
+    assert(std::fabs(c.load_factor() - (float)c.size() / c.bucket_count()) < FLT_EPSILON);
+    assert(c.max_load_factor() == 1);
 
-        assert(c0.empty());
-    }
-    {
-        typedef int P;
-        typedef test_allocator<int> A;
-        typedef std::unordered_set<int,
-                                   test_hash<int>,
-                                   test_equal_to<int>,
-                                   A
-                                   > C;
-        P a[] =
-        {
-            P(1),
-            P(2),
-            P(3),
-            P(4),
-            P(1),
-            P(2)
-        };
-        C c0(a, a + sizeof(a)/sizeof(a[0]),
-            7,
-            test_hash<int>(8),
-            test_equal_to<int>(9),
-            A(10)
-           );
-        C c(std::move(c0), A(10));
-        assert(c.bucket_count() >= 5);
-        assert(c.size() == 4);
-        assert(c.count(1) == 1);
-        assert(c.count(2) == 1);
-        assert(c.count(3) == 1);
-        assert(c.count(4) == 1);
-        assert(c.hash_function() == test_hash<int>(8));
-        assert(c.key_eq() == test_equal_to<int>(9));
-        assert(c.get_allocator() == A(10));
-        assert(!c.empty());
-        assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-        assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
-        assert(std::fabs(c.load_factor() - (float)c.size()/c.bucket_count()) < FLT_EPSILON);
-        assert(c.max_load_factor() == 1);
+    assert(c0.empty());
+  }
+  {
+    typedef int P;
+    typedef test_allocator<int> A;
+    typedef std::unordered_set<int, test_hash<int>, test_equal_to<int>, A > C;
+    P a[] = {P(1), P(2), P(3), P(4), P(1), P(2)};
+    C c0(a, a + sizeof(a) / sizeof(a[0]), 7, test_hash<int>(8), test_equal_to<int>(9), A(10));
+    C c(std::move(c0), A(10));
+    assert(c.bucket_count() >= 5);
+    assert(c.size() == 4);
+    assert(c.count(1) == 1);
+    assert(c.count(2) == 1);
+    assert(c.count(3) == 1);
+    assert(c.count(4) == 1);
+    assert(c.hash_function() == test_hash<int>(8));
+    assert(c.key_eq() == test_equal_to<int>(9));
+    assert(c.get_allocator() == A(10));
+    assert(!c.empty());
+    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
+    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
+    assert(std::fabs(c.load_factor() - (float)c.size() / c.bucket_count()) < FLT_EPSILON);
+    assert(c.max_load_factor() == 1);
 
-        assert(c0.empty());
-    }
-    {
-        typedef int P;
-        typedef min_allocator<int> A;
-        typedef std::unordered_set<int,
-                                   test_hash<int>,
-                                   test_equal_to<int>,
-                                   A
-                                   > C;
-        P a[] =
-        {
-            P(1),
-            P(2),
-            P(3),
-            P(4),
-            P(1),
-            P(2)
-        };
-        C c0(a, a + sizeof(a)/sizeof(a[0]),
-            7,
-            test_hash<int>(8),
-            test_equal_to<int>(9),
-            A()
-           );
-        C c(std::move(c0), A());
-        assert(c.bucket_count() >= 5);
-        assert(c.size() == 4);
-        assert(c.count(1) == 1);
-        assert(c.count(2) == 1);
-        assert(c.count(3) == 1);
-        assert(c.count(4) == 1);
-        assert(c.hash_function() == test_hash<int>(8));
-        assert(c.key_eq() == test_equal_to<int>(9));
-        assert(c.get_allocator() == A());
-        assert(!c.empty());
-        assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-        assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
-        assert(std::fabs(c.load_factor() - (float)c.size()/c.bucket_count()) < FLT_EPSILON);
-        assert(c.max_load_factor() == 1);
+    assert(c0.empty());
+  }
+  {
+    typedef int P;
+    typedef min_allocator<int> A;
+    typedef std::unordered_set<int, test_hash<int>, test_equal_to<int>, A > C;
+    P a[] = {P(1), P(2), P(3), P(4), P(1), P(2)};
+    C c0(a, a + sizeof(a) / sizeof(a[0]), 7, test_hash<int>(8), test_equal_to<int>(9), A());
+    C c(std::move(c0), A());
+    assert(c.bucket_count() >= 5);
+    assert(c.size() == 4);
+    assert(c.count(1) == 1);
+    assert(c.count(2) == 1);
+    assert(c.count(3) == 1);
+    assert(c.count(4) == 1);
+    assert(c.hash_function() == test_hash<int>(8));
+    assert(c.key_eq() == test_equal_to<int>(9));
+    assert(c.get_allocator() == A());
+    assert(!c.empty());
+    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
+    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
+    assert(std::fabs(c.load_factor() - (float)c.size() / c.bucket_count()) < FLT_EPSILON);
+    assert(c.max_load_factor() == 1);
 
-        assert(c0.empty());
-    }
+    assert(c0.empty());
+  }
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/unord/unord.set/unord.set.cnstr/move_assign_noexcept.compile.pass.cpp b/libcxx/test/std/containers/unord/unord.set/unord.set.cnstr/move_assign_noexcept.compile.pass.cpp
new file mode 100644
index 0000000000000..1b4df4e3f23b5
--- /dev/null
+++ b/libcxx/test/std/containers/unord/unord.set/unord.set.cnstr/move_assign_noexcept.compile.pass.cpp
@@ -0,0 +1,72 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// <unordered_set>
+
+// unordered_set& operator=(unordered_set&& c)
+//     noexcept(
+//          allocator_type::propagate_on_container_move_assignment::value &&
+//          is_nothrow_move_assignable<allocator_type>::value &&
+//          is_nothrow_move_assignable<key_compare>::value);
+
+// This tests a conforming extension
+
+// UNSUPPORTED: c++03
+
+#include <unordered_set>
+
+#include "test_macros.h"
+#include "MoveOnly.h"
+#include "test_allocator.h"
+
+template <class T>
+struct some_hash {
+  using value_type = T;
+  some_hash();
+  some_hash(const some_hash&);
+  some_hash& operator=(const some_hash&);
+
+  std::size_t operator()(T const&) const;
+};
+
+template <class T>
+struct some_comp {
+  using value_type = T;
+  some_comp& operator=(const some_comp&);
+  bool operator()(const T&, const T&) const { return false; }
+};
+
+template <class T>
+struct always_equal_alloc {
+  using value_type = T;
+  always_equal_alloc(const always_equal_alloc&);
+  void allocate(std::size_t);
+};
+
+template <class T>
+struct not_always_equal_alloc {
+  int i;
+  using value_type = T;
+  not_always_equal_alloc(const not_always_equal_alloc&);
+  void allocate(std::size_t);
+};
+
+template <template <class> class Alloc>
+using unordered_set_alloc = std::unordered_set<MoveOnly, std::hash<MoveOnly>, std::equal_to<MoveOnly>, Alloc<MoveOnly>>;
+
+static_assert(std::is_nothrow_move_assignable<unordered_set_alloc<std::allocator>>::value, "");
+static_assert(!std::is_nothrow_move_assignable<unordered_set_alloc<test_allocator>>::value, "");
+#if TEST_STD_VER >= 17
+static_assert(std::is_nothrow_move_assignable<unordered_set_alloc<always_equal_alloc>>::value, "");
+#endif
+static_assert(!std::is_nothrow_move_assignable<unordered_set_alloc<not_always_equal_alloc>>::value, "");
+#if defined(_LIBCPP_VERSION)
+static_assert(std::is_nothrow_move_assignable<unordered_set_alloc<other_allocator>>::value, "");
+#endif // _LIBCPP_VERSION
+static_assert(!std::is_nothrow_move_assignable<std::unordered_set<int, some_hash<int>>>::value, "");
+static_assert(!std::is_nothrow_move_assignable<std::unordered_set<int, std::hash<int>, some_comp<int>>>::value, "");
diff --git a/libcxx/test/std/containers/unord/unord.set/unord.set.cnstr/move_assign_noexcept.pass.cpp b/libcxx/test/std/containers/unord/unord.set/unord.set.cnstr/move_assign_noexcept.pass.cpp
deleted file mode 100644
index 1ff2a7b471a1d..0000000000000
--- a/libcxx/test/std/containers/unord/unord.set/unord.set.cnstr/move_assign_noexcept.pass.cpp
+++ /dev/null
@@ -1,75 +0,0 @@
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-// <unordered_set>
-
-// unordered_set& operator=(unordered_set&& c)
-//     noexcept(
-//          allocator_type::propagate_on_container_move_assignment::value &&
-//          is_nothrow_move_assignable<allocator_type>::value &&
-//          is_nothrow_move_assignable<key_compare>::value);
-
-// This tests a conforming extension
-
-// UNSUPPORTED: c++03
-
-#include <unordered_set>
-#include <cassert>
-
-#include "test_macros.h"
-#include "MoveOnly.h"
-#include "test_allocator.h"
-
-template <class T>
-struct some_comp
-{
-    typedef T value_type;
-    some_comp& operator=(const some_comp&);
-    bool operator()(const T&, const T&) const { return false; }
-};
-
-template <class T>
-struct some_hash
-{
-    typedef T value_type;
-    some_hash();
-    some_hash(const some_hash&);
-    some_hash& operator=(const some_hash&);
-    std::size_t operator()(T const&) const;
-};
-
-int main(int, char**)
-{
-    {
-        typedef std::unordered_set<MoveOnly> C;
-        static_assert(std::is_nothrow_move_assignable<C>::value, "");
-    }
-    {
-        typedef std::unordered_set<MoveOnly, std::hash<MoveOnly>,
-                           std::equal_to<MoveOnly>, test_allocator<MoveOnly>> C;
-        static_assert(!std::is_nothrow_move_assignable<C>::value, "");
-    }
-#if defined(_LIBCPP_VERSION)
-    {
-        typedef std::unordered_set<MoveOnly, std::hash<MoveOnly>,
-                          std::equal_to<MoveOnly>, other_allocator<MoveOnly>> C;
-        static_assert(std::is_nothrow_move_assignable<C>::value, "");
-    }
-#endif // _LIBCPP_VERSION
-    {
-        typedef std::unordered_set<MoveOnly, some_hash<MoveOnly>> C;
-        static_assert(!std::is_nothrow_move_assignable<C>::value, "");
-    }
-    {
-        typedef std::unordered_set<MoveOnly, std::hash<MoveOnly>,
-                                                         some_comp<MoveOnly>> C;
-        static_assert(!std::is_nothrow_move_assignable<C>::value, "");
-    }
-
-  return 0;
-}
diff --git a/libcxx/test/std/containers/unord/unord.set/unord.set.cnstr/move_noexcept.pass.cpp b/libcxx/test/std/containers/unord/unord.set/unord.set.cnstr/move_noexcept.pass.cpp
index a31108ab936eb..e9aae18781280 100644
--- a/libcxx/test/std/containers/unord/unord.set/unord.set.cnstr/move_noexcept.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.set/unord.set.cnstr/move_noexcept.pass.cpp
@@ -24,49 +24,43 @@
 #include "test_allocator.h"
 
 template <class T>
-struct some_comp
-{
-    typedef T value_type;
-    some_comp(const some_comp&);
-    bool operator()(const T&, const T&) const { return false; }
+struct some_comp {
+  typedef T value_type;
+  some_comp(const some_comp&);
+  bool operator()(const T&, const T&) const { return false; }
 };
 
 template <class T>
-struct some_hash
-{
-    typedef T value_type;
-    some_hash();
-    some_hash(const some_hash&);
-    std::size_t operator()(T const&) const;
+struct some_hash {
+  typedef T value_type;
+  some_hash();
+  some_hash(const some_hash&);
+  std::size_t operator()(T const&) const;
 };
 
-int main(int, char**)
-{
+int main(int, char**) {
 #if defined(_LIBCPP_VERSION)
-    {
-        typedef std::unordered_set<MoveOnly> C;
-        static_assert(std::is_nothrow_move_constructible<C>::value, "");
-    }
-    {
-        typedef std::unordered_set<MoveOnly, std::hash<MoveOnly>,
-                           std::equal_to<MoveOnly>, test_allocator<MoveOnly>> C;
-        static_assert(std::is_nothrow_move_constructible<C>::value, "");
-    }
-    {
-        typedef std::unordered_set<MoveOnly, std::hash<MoveOnly>,
-                          std::equal_to<MoveOnly>, other_allocator<MoveOnly>> C;
-        static_assert(std::is_nothrow_move_constructible<C>::value, "");
-    }
+  {
+    typedef std::unordered_set<MoveOnly> C;
+    static_assert(std::is_nothrow_move_constructible<C>::value, "");
+  }
+  {
+    typedef std::unordered_set<MoveOnly, std::hash<MoveOnly>, std::equal_to<MoveOnly>, test_allocator<MoveOnly>> C;
+    static_assert(std::is_nothrow_move_constructible<C>::value, "");
+  }
+  {
+    typedef std::unordered_set<MoveOnly, std::hash<MoveOnly>, std::equal_to<MoveOnly>, other_allocator<MoveOnly>> C;
+    static_assert(std::is_nothrow_move_constructible<C>::value, "");
+  }
 #endif // _LIBCPP_VERSION
-    {
-        typedef std::unordered_set<MoveOnly, some_hash<MoveOnly>> C;
-        static_assert(!std::is_nothrow_move_constructible<C>::value, "");
-    }
-    {
-        typedef std::unordered_set<MoveOnly, std::hash<MoveOnly>,
-                                                         some_comp<MoveOnly>> C;
-        static_assert(!std::is_nothrow_move_constructible<C>::value, "");
-    }
+  {
+    typedef std::unordered_set<MoveOnly, some_hash<MoveOnly>> C;
+    static_assert(!std::is_nothrow_move_constructible<C>::value, "");
+  }
+  {
+    typedef std::unordered_set<MoveOnly, std::hash<MoveOnly>, some_comp<MoveOnly>> C;
+    static_assert(!std::is_nothrow_move_constructible<C>::value, "");
+  }
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/unord/unord.set/unord.set.cnstr/size.compile.fail.cpp b/libcxx/test/std/containers/unord/unord.set/unord.set.cnstr/size.compile.fail.cpp
index f9bb88f371245..692ca98de1a80 100644
--- a/libcxx/test/std/containers/unord/unord.set/unord.set.cnstr/size.compile.fail.cpp
+++ b/libcxx/test/std/containers/unord/unord.set/unord.set.cnstr/size.compile.fail.cpp
@@ -23,25 +23,24 @@
 #include "../../../test_hash.h"
 #include "test_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef std::unordered_set<NotConstructible,
-                                   test_hash<NotConstructible>,
-                                   test_equal_to<NotConstructible>,
-                                   test_allocator<NotConstructible>
-                                   > C;
-        C c = 7;
-        LIBCPP_ASSERT(c.bucket_count() == 7);
-        assert(c.hash_function() == test_hash<NotConstructible>());
-        assert(c.key_eq() == test_equal_to<NotConstructible>());
-        assert(c.get_allocator() == (test_allocator<NotConstructible>()));
-        assert(c.size() == 0);
-        assert(c.empty());
-        assert(std::distance(c.begin(), c.end()) == 0);
-        assert(c.load_factor() == 0);
-        assert(c.max_load_factor() == 1);
-    }
+int main(int, char**) {
+  {
+    typedef std::unordered_set<NotConstructible,
+                               test_hash<NotConstructible>,
+                               test_equal_to<NotConstructible>,
+                               test_allocator<NotConstructible> >
+        C;
+    C c = 7;
+    LIBCPP_ASSERT(c.bucket_count() == 7);
+    assert(c.hash_function() == test_hash<NotConstructible>());
+    assert(c.key_eq() == test_equal_to<NotConstructible>());
+    assert(c.get_allocator() == (test_allocator<NotConstructible>()));
+    assert(c.size() == 0);
+    assert(c.empty());
+    assert(std::distance(c.begin(), c.end()) == 0);
+    assert(c.load_factor() == 0);
+    assert(c.max_load_factor() == 1);
+  }
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/unord/unord.set/unord.set.cnstr/size.pass.cpp b/libcxx/test/std/containers/unord/unord.set/unord.set.cnstr/size.pass.cpp
index 5c43179b0b1e6..3add76761d8b1 100644
--- a/libcxx/test/std/containers/unord/unord.set/unord.set.cnstr/size.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.set/unord.set.cnstr/size.pass.cpp
@@ -24,43 +24,42 @@
 #include "test_allocator.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef std::unordered_set<NotConstructible,
-                                   test_hash<NotConstructible>,
-                                   test_equal_to<NotConstructible>,
-                                   test_allocator<NotConstructible>
-                                   > C;
-        C c(7);
-        LIBCPP_ASSERT(c.bucket_count() == 7);
-        assert(c.hash_function() == test_hash<NotConstructible>());
-        assert(c.key_eq() == test_equal_to<NotConstructible>());
-        assert(c.get_allocator() == (test_allocator<NotConstructible>()));
-        assert(c.size() == 0);
-        assert(c.empty());
-        assert(std::distance(c.begin(), c.end()) == 0);
-        assert(c.load_factor() == 0);
-        assert(c.max_load_factor() == 1);
-    }
+int main(int, char**) {
+  {
+    typedef std::unordered_set<NotConstructible,
+                               test_hash<NotConstructible>,
+                               test_equal_to<NotConstructible>,
+                               test_allocator<NotConstructible> >
+        C;
+    C c(7);
+    LIBCPP_ASSERT(c.bucket_count() == 7);
+    assert(c.hash_function() == test_hash<NotConstructible>());
+    assert(c.key_eq() == test_equal_to<NotConstructible>());
+    assert(c.get_allocator() == (test_allocator<NotConstructible>()));
+    assert(c.size() == 0);
+    assert(c.empty());
+    assert(std::distance(c.begin(), c.end()) == 0);
+    assert(c.load_factor() == 0);
+    assert(c.max_load_factor() == 1);
+  }
 #if TEST_STD_VER >= 11
-    {
-        typedef std::unordered_set<NotConstructible,
-                                   test_hash<NotConstructible>,
-                                   test_equal_to<NotConstructible>,
-                                   min_allocator<NotConstructible>
-                                   > C;
-        C c(7);
-        LIBCPP_ASSERT(c.bucket_count() == 7);
-        assert(c.hash_function() == test_hash<NotConstructible>());
-        assert(c.key_eq() == test_equal_to<NotConstructible>());
-        assert(c.get_allocator() == (min_allocator<NotConstructible>()));
-        assert(c.size() == 0);
-        assert(c.empty());
-        assert(std::distance(c.begin(), c.end()) == 0);
-        assert(c.load_factor() == 0);
-        assert(c.max_load_factor() == 1);
-    }
+  {
+    typedef std::unordered_set<NotConstructible,
+                               test_hash<NotConstructible>,
+                               test_equal_to<NotConstructible>,
+                               min_allocator<NotConstructible> >
+        C;
+    C c(7);
+    LIBCPP_ASSERT(c.bucket_count() == 7);
+    assert(c.hash_function() == test_hash<NotConstructible>());
+    assert(c.key_eq() == test_equal_to<NotConstructible>());
+    assert(c.get_allocator() == (min_allocator<NotConstructible>()));
+    assert(c.size() == 0);
+    assert(c.empty());
+    assert(std::distance(c.begin(), c.end()) == 0);
+    assert(c.load_factor() == 0);
+    assert(c.max_load_factor() == 1);
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/unord/unord.set/unord.set.cnstr/size_allocator.pass.cpp b/libcxx/test/std/containers/unord/unord.set/unord.set.cnstr/size_allocator.pass.cpp
index d547ea2b4bda3..99abd12745eca 100644
--- a/libcxx/test/std/containers/unord/unord.set/unord.set.cnstr/size_allocator.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.set/unord.set.cnstr/size_allocator.pass.cpp
@@ -28,29 +28,25 @@
 #include "min_allocator.h"
 
 template <class Allocator>
-void test(const Allocator& alloc)
-{
-    typedef std::unordered_set<NotConstructible,
-                               test_hash<NotConstructible>,
-                               test_equal_to<NotConstructible>,
-                               Allocator> C;
-    C c(7, alloc);
-    LIBCPP_ASSERT(c.bucket_count() == 7);
-    assert(c.hash_function() == test_hash<NotConstructible>());
-    assert(c.key_eq() == test_equal_to<NotConstructible>());
-    assert(c.get_allocator() == alloc);
-    assert(c.size() == 0);
-    assert(c.empty());
-    assert(std::distance(c.begin(), c.end()) == 0);
-    assert(c.load_factor() == 0);
-    assert(c.max_load_factor() == 1);
+void test(const Allocator& alloc) {
+  typedef std::unordered_set<NotConstructible, test_hash<NotConstructible>, test_equal_to<NotConstructible>, Allocator>
+      C;
+  C c(7, alloc);
+  LIBCPP_ASSERT(c.bucket_count() == 7);
+  assert(c.hash_function() == test_hash<NotConstructible>());
+  assert(c.key_eq() == test_equal_to<NotConstructible>());
+  assert(c.get_allocator() == alloc);
+  assert(c.size() == 0);
+  assert(c.empty());
+  assert(std::distance(c.begin(), c.end()) == 0);
+  assert(c.load_factor() == 0);
+  assert(c.max_load_factor() == 1);
 }
 
-int main(int, char**)
-{
-    test(test_allocator<NotConstructible>(10));
-    test(min_allocator<NotConstructible>());
-    test(explicit_allocator<NotConstructible>());
+int main(int, char**) {
+  test(test_allocator<NotConstructible>(10));
+  test(min_allocator<NotConstructible>());
+  test(explicit_allocator<NotConstructible>());
 
-    return 0;
+  return 0;
 }
diff --git a/libcxx/test/std/containers/unord/unord.set/unord.set.cnstr/size_hash.pass.cpp b/libcxx/test/std/containers/unord/unord.set/unord.set.cnstr/size_hash.pass.cpp
index 6ee0f5c99f69c..a7d4f4d798a0b 100644
--- a/libcxx/test/std/containers/unord/unord.set/unord.set.cnstr/size_hash.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.set/unord.set.cnstr/size_hash.pass.cpp
@@ -24,47 +24,42 @@
 #include "test_allocator.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef std::unordered_set<NotConstructible,
-                                   test_hash<NotConstructible>,
-                                   test_equal_to<NotConstructible>,
-                                   test_allocator<NotConstructible>
-                                   > C;
-        C c(7,
-            test_hash<NotConstructible>(8)
-           );
-        LIBCPP_ASSERT(c.bucket_count() == 7);
-        assert(c.hash_function() == test_hash<NotConstructible>(8));
-        assert(c.key_eq() == test_equal_to<NotConstructible>());
-        assert(c.get_allocator() == (test_allocator<NotConstructible>()));
-        assert(c.size() == 0);
-        assert(c.empty());
-        assert(std::distance(c.begin(), c.end()) == 0);
-        assert(c.load_factor() == 0);
-        assert(c.max_load_factor() == 1);
-    }
+int main(int, char**) {
+  {
+    typedef std::unordered_set<NotConstructible,
+                               test_hash<NotConstructible>,
+                               test_equal_to<NotConstructible>,
+                               test_allocator<NotConstructible> >
+        C;
+    C c(7, test_hash<NotConstructible>(8));
+    LIBCPP_ASSERT(c.bucket_count() == 7);
+    assert(c.hash_function() == test_hash<NotConstructible>(8));
+    assert(c.key_eq() == test_equal_to<NotConstructible>());
+    assert(c.get_allocator() == (test_allocator<NotConstructible>()));
+    assert(c.size() == 0);
+    assert(c.empty());
+    assert(std::distance(c.begin(), c.end()) == 0);
+    assert(c.load_factor() == 0);
+    assert(c.max_load_factor() == 1);
+  }
 #if TEST_STD_VER >= 11
-    {
-        typedef std::unordered_set<NotConstructible,
-                                   test_hash<NotConstructible>,
-                                   test_equal_to<NotConstructible>,
-                                   min_allocator<NotConstructible>
-                                   > C;
-        C c(7,
-            test_hash<NotConstructible>(8)
-           );
-        LIBCPP_ASSERT(c.bucket_count() == 7);
-        assert(c.hash_function() == test_hash<NotConstructible>(8));
-        assert(c.key_eq() == test_equal_to<NotConstructible>());
-        assert(c.get_allocator() == (min_allocator<NotConstructible>()));
-        assert(c.size() == 0);
-        assert(c.empty());
-        assert(std::distance(c.begin(), c.end()) == 0);
-        assert(c.load_factor() == 0);
-        assert(c.max_load_factor() == 1);
-    }
+  {
+    typedef std::unordered_set<NotConstructible,
+                               test_hash<NotConstructible>,
+                               test_equal_to<NotConstructible>,
+                               min_allocator<NotConstructible> >
+        C;
+    C c(7, test_hash<NotConstructible>(8));
+    LIBCPP_ASSERT(c.bucket_count() == 7);
+    assert(c.hash_function() == test_hash<NotConstructible>(8));
+    assert(c.key_eq() == test_equal_to<NotConstructible>());
+    assert(c.get_allocator() == (min_allocator<NotConstructible>()));
+    assert(c.size() == 0);
+    assert(c.empty());
+    assert(std::distance(c.begin(), c.end()) == 0);
+    assert(c.load_factor() == 0);
+    assert(c.max_load_factor() == 1);
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/unord/unord.set/unord.set.cnstr/size_hash_allocator.pass.cpp b/libcxx/test/std/containers/unord/unord.set/unord.set.cnstr/size_hash_allocator.pass.cpp
index 972c0db3149d7..e44854eb0a132 100644
--- a/libcxx/test/std/containers/unord/unord.set/unord.set.cnstr/size_hash_allocator.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.set/unord.set.cnstr/size_hash_allocator.pass.cpp
@@ -28,29 +28,25 @@
 #include "min_allocator.h"
 
 template <class Allocator>
-void test(const Allocator& alloc)
-{
-    typedef std::unordered_set<NotConstructible,
-                               test_hash<NotConstructible>,
-                               test_equal_to<NotConstructible>,
-                               Allocator> C;
-    C c(7, test_hash<NotConstructible>(5), alloc);
-    LIBCPP_ASSERT(c.bucket_count() == 7);
-    assert(c.hash_function() == test_hash<NotConstructible>(5));
-    assert(c.key_eq() == test_equal_to<NotConstructible>());
-    assert(c.get_allocator() == alloc);
-    assert(c.size() == 0);
-    assert(c.empty());
-    assert(std::distance(c.begin(), c.end()) == 0);
-    assert(c.load_factor() == 0);
-    assert(c.max_load_factor() == 1);
+void test(const Allocator& alloc) {
+  typedef std::unordered_set<NotConstructible, test_hash<NotConstructible>, test_equal_to<NotConstructible>, Allocator>
+      C;
+  C c(7, test_hash<NotConstructible>(5), alloc);
+  LIBCPP_ASSERT(c.bucket_count() == 7);
+  assert(c.hash_function() == test_hash<NotConstructible>(5));
+  assert(c.key_eq() == test_equal_to<NotConstructible>());
+  assert(c.get_allocator() == alloc);
+  assert(c.size() == 0);
+  assert(c.empty());
+  assert(std::distance(c.begin(), c.end()) == 0);
+  assert(c.load_factor() == 0);
+  assert(c.max_load_factor() == 1);
 }
 
-int main(int, char**)
-{
-    test(test_allocator<NotConstructible>(10));
-    test(min_allocator<NotConstructible>());
-    test(explicit_allocator<NotConstructible>());
+int main(int, char**) {
+  test(test_allocator<NotConstructible>(10));
+  test(min_allocator<NotConstructible>());
+  test(explicit_allocator<NotConstructible>());
 
-    return 0;
+  return 0;
 }
diff --git a/libcxx/test/std/containers/unord/unord.set/unord.set.cnstr/size_hash_equal.pass.cpp b/libcxx/test/std/containers/unord/unord.set/unord.set.cnstr/size_hash_equal.pass.cpp
index a54ddaf4627f6..cbaa0ae995115 100644
--- a/libcxx/test/std/containers/unord/unord.set/unord.set.cnstr/size_hash_equal.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.set/unord.set.cnstr/size_hash_equal.pass.cpp
@@ -24,49 +24,42 @@
 #include "test_allocator.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef std::unordered_set<NotConstructible,
-                                   test_hash<NotConstructible>,
-                                   test_equal_to<NotConstructible>,
-                                   test_allocator<NotConstructible>
-                                   > C;
-        C c(7,
-            test_hash<NotConstructible>(8),
-            test_equal_to<NotConstructible>(9)
-           );
-        LIBCPP_ASSERT(c.bucket_count() == 7);
-        assert(c.hash_function() == test_hash<NotConstructible>(8));
-        assert(c.key_eq() == test_equal_to<NotConstructible>(9));
-        assert(c.get_allocator() == (test_allocator<NotConstructible>()));
-        assert(c.size() == 0);
-        assert(c.empty());
-        assert(std::distance(c.begin(), c.end()) == 0);
-        assert(c.load_factor() == 0);
-        assert(c.max_load_factor() == 1);
-    }
+int main(int, char**) {
+  {
+    typedef std::unordered_set<NotConstructible,
+                               test_hash<NotConstructible>,
+                               test_equal_to<NotConstructible>,
+                               test_allocator<NotConstructible> >
+        C;
+    C c(7, test_hash<NotConstructible>(8), test_equal_to<NotConstructible>(9));
+    LIBCPP_ASSERT(c.bucket_count() == 7);
+    assert(c.hash_function() == test_hash<NotConstructible>(8));
+    assert(c.key_eq() == test_equal_to<NotConstructible>(9));
+    assert(c.get_allocator() == (test_allocator<NotConstructible>()));
+    assert(c.size() == 0);
+    assert(c.empty());
+    assert(std::distance(c.begin(), c.end()) == 0);
+    assert(c.load_factor() == 0);
+    assert(c.max_load_factor() == 1);
+  }
 #if TEST_STD_VER >= 11
-    {
-        typedef std::unordered_set<NotConstructible,
-                                   test_hash<NotConstructible>,
-                                   test_equal_to<NotConstructible>,
-                                   min_allocator<NotConstructible>
-                                   > C;
-        C c(7,
-            test_hash<NotConstructible>(8),
-            test_equal_to<NotConstructible>(9)
-           );
-        LIBCPP_ASSERT(c.bucket_count() == 7);
-        assert(c.hash_function() == test_hash<NotConstructible>(8));
-        assert(c.key_eq() == test_equal_to<NotConstructible>(9));
-        assert(c.get_allocator() == (min_allocator<NotConstructible>()));
-        assert(c.size() == 0);
-        assert(c.empty());
-        assert(std::distance(c.begin(), c.end()) == 0);
-        assert(c.load_factor() == 0);
-        assert(c.max_load_factor() == 1);
-    }
+  {
+    typedef std::unordered_set<NotConstructible,
+                               test_hash<NotConstructible>,
+                               test_equal_to<NotConstructible>,
+                               min_allocator<NotConstructible> >
+        C;
+    C c(7, test_hash<NotConstructible>(8), test_equal_to<NotConstructible>(9));
+    LIBCPP_ASSERT(c.bucket_count() == 7);
+    assert(c.hash_function() == test_hash<NotConstructible>(8));
+    assert(c.key_eq() == test_equal_to<NotConstructible>(9));
+    assert(c.get_allocator() == (min_allocator<NotConstructible>()));
+    assert(c.size() == 0);
+    assert(c.empty());
+    assert(std::distance(c.begin(), c.end()) == 0);
+    assert(c.load_factor() == 0);
+    assert(c.max_load_factor() == 1);
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/unord/unord.set/unord.set.cnstr/size_hash_equal_allocator.pass.cpp b/libcxx/test/std/containers/unord/unord.set/unord.set.cnstr/size_hash_equal_allocator.pass.cpp
index c3d97c579177e..f6d208f3b59b5 100644
--- a/libcxx/test/std/containers/unord/unord.set/unord.set.cnstr/size_hash_equal_allocator.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.set/unord.set.cnstr/size_hash_equal_allocator.pass.cpp
@@ -24,51 +24,42 @@
 #include "test_allocator.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef std::unordered_set<NotConstructible,
-                                   test_hash<NotConstructible>,
-                                   test_equal_to<NotConstructible>,
-                                   test_allocator<NotConstructible>
-                                   > C;
-        C c(7,
-            test_hash<NotConstructible>(8),
-            test_equal_to<NotConstructible>(9),
-            test_allocator<NotConstructible>(10)
-           );
-        LIBCPP_ASSERT(c.bucket_count() == 7);
-        assert(c.hash_function() == test_hash<NotConstructible>(8));
-        assert(c.key_eq() == test_equal_to<NotConstructible>(9));
-        assert(c.get_allocator() == (test_allocator<NotConstructible>(10)));
-        assert(c.size() == 0);
-        assert(c.empty());
-        assert(std::distance(c.begin(), c.end()) == 0);
-        assert(c.load_factor() == 0);
-        assert(c.max_load_factor() == 1);
-    }
+int main(int, char**) {
+  {
+    typedef std::unordered_set<NotConstructible,
+                               test_hash<NotConstructible>,
+                               test_equal_to<NotConstructible>,
+                               test_allocator<NotConstructible> >
+        C;
+    C c(7, test_hash<NotConstructible>(8), test_equal_to<NotConstructible>(9), test_allocator<NotConstructible>(10));
+    LIBCPP_ASSERT(c.bucket_count() == 7);
+    assert(c.hash_function() == test_hash<NotConstructible>(8));
+    assert(c.key_eq() == test_equal_to<NotConstructible>(9));
+    assert(c.get_allocator() == (test_allocator<NotConstructible>(10)));
+    assert(c.size() == 0);
+    assert(c.empty());
+    assert(std::distance(c.begin(), c.end()) == 0);
+    assert(c.load_factor() == 0);
+    assert(c.max_load_factor() == 1);
+  }
 #if TEST_STD_VER >= 11
-    {
-        typedef std::unordered_set<NotConstructible,
-                                   test_hash<NotConstructible>,
-                                   test_equal_to<NotConstructible>,
-                                   min_allocator<NotConstructible>
-                                   > C;
-        C c(7,
-            test_hash<NotConstructible>(8),
-            test_equal_to<NotConstructible>(9),
-            min_allocator<NotConstructible>()
-           );
-        LIBCPP_ASSERT(c.bucket_count() == 7);
-        assert(c.hash_function() == test_hash<NotConstructible>(8));
-        assert(c.key_eq() == test_equal_to<NotConstructible>(9));
-        assert(c.get_allocator() == (min_allocator<NotConstructible>()));
-        assert(c.size() == 0);
-        assert(c.empty());
-        assert(std::distance(c.begin(), c.end()) == 0);
-        assert(c.load_factor() == 0);
-        assert(c.max_load_factor() == 1);
-    }
+  {
+    typedef std::unordered_set<NotConstructible,
+                               test_hash<NotConstructible>,
+                               test_equal_to<NotConstructible>,
+                               min_allocator<NotConstructible> >
+        C;
+    C c(7, test_hash<NotConstructible>(8), test_equal_to<NotConstructible>(9), min_allocator<NotConstructible>());
+    LIBCPP_ASSERT(c.bucket_count() == 7);
+    assert(c.hash_function() == test_hash<NotConstructible>(8));
+    assert(c.key_eq() == test_equal_to<NotConstructible>(9));
+    assert(c.get_allocator() == (min_allocator<NotConstructible>()));
+    assert(c.size() == 0);
+    assert(c.empty());
+    assert(std::distance(c.begin(), c.end()) == 0);
+    assert(c.load_factor() == 0);
+    assert(c.max_load_factor() == 1);
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/unord/unord.set/unord.set.swap/swap_noexcept.pass.cpp b/libcxx/test/std/containers/unord/unord.set/unord.set.swap/swap_noexcept.pass.cpp
index e4d4861283f16..81219fe7d6dc5 100644
--- a/libcxx/test/std/containers/unord/unord.set/unord.set.swap/swap_noexcept.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.set/unord.set.swap/swap_noexcept.pass.cpp
@@ -33,23 +33,21 @@
 #include "test_allocator.h"
 
 template <class T>
-struct some_comp
-{
-    typedef T value_type;
+struct some_comp {
+  typedef T value_type;
 
-    some_comp() {}
-    some_comp(const some_comp&) {}
-    bool operator()(const T&, const T&) const { return false; }
+  some_comp() {}
+  some_comp(const some_comp&) {}
+  bool operator()(const T&, const T&) const { return false; }
 };
 
 template <class T>
-struct some_comp2
-{
-    typedef T value_type;
+struct some_comp2 {
+  typedef T value_type;
 
-    some_comp2() {}
-    some_comp2(const some_comp2&) {}
-    bool operator()(const T&, const T&) const { return false; }
+  some_comp2() {}
+  some_comp2(const some_comp2&) {}
+  bool operator()(const T&, const T&) const { return false; }
 };
 
 #if TEST_STD_VER >= 14
@@ -58,21 +56,19 @@ void swap(some_comp2<T>&, some_comp2<T>&) noexcept {}
 #endif
 
 template <class T>
-struct some_hash
-{
-    typedef T value_type;
-    some_hash() {}
-    some_hash(const some_hash&);
-    std::size_t operator()(const T&) const { return 0; }
+struct some_hash {
+  typedef T value_type;
+  some_hash() {}
+  some_hash(const some_hash&);
+  std::size_t operator()(const T&) const { return 0; }
 };
 
 template <class T>
-struct some_hash2
-{
-    typedef T value_type;
-    some_hash2() {}
-    some_hash2(const some_hash2&);
-    std::size_t operator()(const T&) const { return 0; }
+struct some_hash2 {
+  typedef T value_type;
+  some_hash2() {}
+  some_hash2(const some_hash2&);
+  std::size_t operator()(const T&) const { return 0; }
 };
 
 #if TEST_STD_VER >= 14
@@ -81,110 +77,103 @@ void swap(some_hash2<T>&, some_hash2<T>&) noexcept {}
 #endif
 
 template <class T>
-struct some_alloc
-{
-    typedef T value_type;
+struct some_alloc {
+  typedef T value_type;
 
-    some_alloc() {}
-    some_alloc(const some_alloc&);
-    void deallocate(void*, unsigned) {}
+  some_alloc() {}
+  some_alloc(const some_alloc&);
+  void deallocate(void*, unsigned) {}
 
-    typedef std::true_type propagate_on_container_swap;
+  typedef std::true_type propagate_on_container_swap;
 };
 
 template <class T>
-struct some_alloc2
-{
-    typedef T value_type;
+struct some_alloc2 {
+  typedef T value_type;
 
-    some_alloc2() {}
-    some_alloc2(const some_alloc2&);
-    void deallocate(void*, unsigned) {}
+  some_alloc2() {}
+  some_alloc2(const some_alloc2&);
+  void deallocate(void*, unsigned) {}
 
-    typedef std::false_type propagate_on_container_swap;
-    typedef std::true_type is_always_equal;
+  typedef std::false_type propagate_on_container_swap;
+  typedef std::true_type is_always_equal;
 };
 
 template <class T>
-struct some_alloc3
-{
-    typedef T value_type;
+struct some_alloc3 {
+  typedef T value_type;
 
-    some_alloc3() {}
-    some_alloc3(const some_alloc3&);
-    void deallocate(void*, unsigned) {}
+  some_alloc3() {}
+  some_alloc3(const some_alloc3&);
+  void deallocate(void*, unsigned) {}
 
-    typedef std::false_type propagate_on_container_swap;
-    typedef std::false_type is_always_equal;
+  typedef std::false_type propagate_on_container_swap;
+  typedef std::false_type is_always_equal;
 };
 
-int main(int, char**)
-{
-    {
-        typedef std::unordered_set<MoveOnly> C;
-        static_assert(noexcept(swap(std::declval<C&>(), std::declval<C&>())), "");
-    }
+int main(int, char**) {
+  {
+    typedef std::unordered_set<MoveOnly> C;
+    static_assert(noexcept(swap(std::declval<C&>(), std::declval<C&>())), "");
+  }
 #if defined(_LIBCPP_VERSION)
-    {
-        typedef std::unordered_set<MoveOnly, std::hash<MoveOnly>,
-                           std::equal_to<MoveOnly>, test_allocator<MoveOnly>> C;
-        static_assert(noexcept(swap(std::declval<C&>(), std::declval<C&>())), "");
-    }
-    {
-        typedef std::unordered_set<MoveOnly, std::hash<MoveOnly>,
-                          std::equal_to<MoveOnly>, other_allocator<MoveOnly>> C;
-        static_assert(noexcept(swap(std::declval<C&>(), std::declval<C&>())), "");
-    }
+  {
+    typedef std::unordered_set<MoveOnly, std::hash<MoveOnly>, std::equal_to<MoveOnly>, test_allocator<MoveOnly>> C;
+    static_assert(noexcept(swap(std::declval<C&>(), std::declval<C&>())), "");
+  }
+  {
+    typedef std::unordered_set<MoveOnly, std::hash<MoveOnly>, std::equal_to<MoveOnly>, other_allocator<MoveOnly>> C;
+    static_assert(noexcept(swap(std::declval<C&>(), std::declval<C&>())), "");
+  }
 #endif // _LIBCPP_VERSION
-    {
-        typedef std::unordered_set<MoveOnly, some_hash<MoveOnly>> C;
-        static_assert(!noexcept(swap(std::declval<C&>(), std::declval<C&>())), "");
-    }
-    {
-        typedef std::unordered_set<MoveOnly, std::hash<MoveOnly>,
-                                                         some_comp<MoveOnly>> C;
-        static_assert(!noexcept(swap(std::declval<C&>(), std::declval<C&>())), "");
-    }
+  {
+    typedef std::unordered_set<MoveOnly, some_hash<MoveOnly>> C;
+    static_assert(!noexcept(swap(std::declval<C&>(), std::declval<C&>())), "");
+  }
+  {
+    typedef std::unordered_set<MoveOnly, std::hash<MoveOnly>, some_comp<MoveOnly>> C;
+    static_assert(!noexcept(swap(std::declval<C&>(), std::declval<C&>())), "");
+  }
 
 #if TEST_STD_VER >= 14
-    { // POCS allocator, throwable swap for hash, throwable swap for comp
-    typedef std::unordered_set<MoveOnly, some_hash<MoveOnly>, some_comp <MoveOnly>, some_alloc <MoveOnly>> C;
+  { // POCS allocator, throwable swap for hash, throwable swap for comp
+    typedef std::unordered_set<MoveOnly, some_hash<MoveOnly>, some_comp<MoveOnly>, some_alloc<MoveOnly>> C;
     static_assert(!noexcept(swap(std::declval<C&>(), std::declval<C&>())), "");
-    }
-    { // always equal allocator, throwable swap for hash, throwable swap for comp
-    typedef std::unordered_set<MoveOnly, some_hash<MoveOnly>, some_comp <MoveOnly>, some_alloc2<MoveOnly>> C;
+  }
+  { // always equal allocator, throwable swap for hash, throwable swap for comp
+    typedef std::unordered_set<MoveOnly, some_hash<MoveOnly>, some_comp<MoveOnly>, some_alloc2<MoveOnly>> C;
     static_assert(!noexcept(swap(std::declval<C&>(), std::declval<C&>())), "");
-    }
-    { // POCS allocator, throwable swap for hash, nothrow swap for comp
-    typedef std::unordered_set<MoveOnly, some_hash<MoveOnly>, some_comp2<MoveOnly>, some_alloc <MoveOnly>> C;
+  }
+  { // POCS allocator, throwable swap for hash, nothrow swap for comp
+    typedef std::unordered_set<MoveOnly, some_hash<MoveOnly>, some_comp2<MoveOnly>, some_alloc<MoveOnly>> C;
     static_assert(!noexcept(swap(std::declval<C&>(), std::declval<C&>())), "");
-    }
-    { // always equal allocator, throwable swap for hash, nothrow swap for comp
+  }
+  { // always equal allocator, throwable swap for hash, nothrow swap for comp
     typedef std::unordered_set<MoveOnly, some_hash<MoveOnly>, some_comp2<MoveOnly>, some_alloc2<MoveOnly>> C;
     static_assert(!noexcept(swap(std::declval<C&>(), std::declval<C&>())), "");
-    }
-    { // POCS allocator, nothrow swap for hash, throwable swap for comp
-    typedef std::unordered_set<MoveOnly, some_hash2<MoveOnly>, some_comp <MoveOnly>, some_alloc <MoveOnly>> C;
+  }
+  { // POCS allocator, nothrow swap for hash, throwable swap for comp
+    typedef std::unordered_set<MoveOnly, some_hash2<MoveOnly>, some_comp<MoveOnly>, some_alloc<MoveOnly>> C;
     static_assert(!noexcept(swap(std::declval<C&>(), std::declval<C&>())), "");
-    }
-    { // always equal allocator, nothrow swap for hash, throwable swap for comp
-    typedef std::unordered_set<MoveOnly, some_hash2<MoveOnly>, some_comp <MoveOnly>, some_alloc2<MoveOnly>> C;
+  }
+  { // always equal allocator, nothrow swap for hash, throwable swap for comp
+    typedef std::unordered_set<MoveOnly, some_hash2<MoveOnly>, some_comp<MoveOnly>, some_alloc2<MoveOnly>> C;
     static_assert(!noexcept(swap(std::declval<C&>(), std::declval<C&>())), "");
-    }
-    { // POCS allocator, nothrow swap for hash, nothrow swap for comp
-    typedef std::unordered_set<MoveOnly, some_hash2<MoveOnly>, some_comp2<MoveOnly>, some_alloc <MoveOnly>> C;
-    static_assert( noexcept(swap(std::declval<C&>(), std::declval<C&>())), "");
-    }
-    { // always equal allocator, nothrow swap for hash, nothrow swap for comp
+  }
+  { // POCS allocator, nothrow swap for hash, nothrow swap for comp
+    typedef std::unordered_set<MoveOnly, some_hash2<MoveOnly>, some_comp2<MoveOnly>, some_alloc<MoveOnly>> C;
+    static_assert(noexcept(swap(std::declval<C&>(), std::declval<C&>())), "");
+  }
+  { // always equal allocator, nothrow swap for hash, nothrow swap for comp
     typedef std::unordered_set<MoveOnly, some_hash2<MoveOnly>, some_comp2<MoveOnly>, some_alloc2<MoveOnly>> C;
-    static_assert( noexcept(swap(std::declval<C&>(), std::declval<C&>())), "");
-    }
-#if defined(_LIBCPP_VERSION)
-    { // NOT always equal allocator, nothrow swap for hash, nothrow swap for comp
+    static_assert(noexcept(swap(std::declval<C&>(), std::declval<C&>())), "");
+  }
+#  if defined(_LIBCPP_VERSION)
+  { // NOT always equal allocator, nothrow swap for hash, nothrow swap for comp
     typedef std::unordered_set<MoveOnly, some_hash2<MoveOnly>, some_comp2<MoveOnly>, some_alloc3<MoveOnly>> C;
-    static_assert( noexcept(swap(std::declval<C&>(), std::declval<C&>())), "");
-    }
-#endif // _LIBCPP_VERSION
+    static_assert(noexcept(swap(std::declval<C&>(), std::declval<C&>())), "");
+  }
+#  endif // _LIBCPP_VERSION
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/unord/unord.set/unord.set.swap/swap_non_member.pass.cpp b/libcxx/test/std/containers/unord/unord.set/unord.set.swap/swap_non_member.pass.cpp
index 49c45f2ecb25b..22184df129c77 100644
--- a/libcxx/test/std/containers/unord/unord.set/unord.set.swap/swap_non_member.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.set/unord.set.swap/swap_non_member.pass.cpp
@@ -24,555 +24,446 @@
 #include "test_allocator.h"
 #include "min_allocator.h"
 
-int main(int, char**)
-{
-    {
-        typedef test_hash<int> Hash;
-        typedef test_equal_to<int> Compare;
-        typedef test_allocator<int> Alloc;
-        typedef std::unordered_set<int, Hash, Compare, Alloc> C;
-        C c1(0, Hash(1), Compare(1), Alloc(1, 1));
-        C c2(0, Hash(2), Compare(2), Alloc(1, 2));
-        c2.max_load_factor(2);
-        swap(c1, c2);
-
-        LIBCPP_ASSERT(c1.bucket_count() == 0);
-        assert(c1.size() == 0);
-        assert(c1.hash_function() == Hash(2));
-        assert(c1.key_eq() == Compare(2));
-        assert(c1.get_allocator().get_id() == 1);
-        assert(static_cast<std::size_t>(std::distance(c1.begin(), c1.end())) == c1.size());
-        assert(static_cast<std::size_t>(std::distance(c1.cbegin(), c1.cend())) == c1.size());
-        assert(c1.max_load_factor() == 2);
-
-        LIBCPP_ASSERT(c2.bucket_count() == 0);
-        assert(c2.size() == 0);
-        assert(c2.hash_function() == Hash(1));
-        assert(c2.key_eq() == Compare(1));
-        assert(c2.get_allocator().get_id() == 2);
-        assert(static_cast<std::size_t>(std::distance(c2.begin(), c2.end())) == c2.size());
-        assert(static_cast<std::size_t>(std::distance(c2.cbegin(), c2.cend())) == c2.size());
-        assert(c2.max_load_factor() == 1);
-    }
-    {
-        typedef test_hash<int> Hash;
-        typedef test_equal_to<int> Compare;
-        typedef test_allocator<int> Alloc;
-        typedef std::unordered_set<int, Hash, Compare, Alloc> C;
-        typedef int P;
-        P a2[] =
-        {
-            P(10),
-            P(20),
-            P(30),
-            P(40),
-            P(50),
-            P(60),
-            P(70),
-            P(80)
-        };
-        C c1(0, Hash(1), Compare(1), Alloc(1, 1));
-        C c2(std::begin(a2), std::end(a2), 0, Hash(2), Compare(2), Alloc(1, 2));
-        c2.max_load_factor(2);
-        C::iterator it2 = c2.begin();
-        swap(c1, c2);
-
-        assert(c1.bucket_count() >= 8);
-        assert(c1.size() == 8);
-        assert(*c1.find(10) == 10);
-        assert(*c1.find(20) == 20);
-        assert(*c1.find(30) == 30);
-        assert(*c1.find(40) == 40);
-        assert(*c1.find(50) == 50);
-        assert(*c1.find(60) == 60);
-        assert(*c1.find(70) == 70);
-        assert(*c1.find(80) == 80);
-        assert(c1.hash_function() == Hash(2));
-        assert(c1.key_eq() == Compare(2));
-        assert(c1.get_allocator().get_id() == 1);
-        assert(static_cast<std::size_t>(std::distance(c1.begin(), c1.end())) == c1.size());
-        assert(static_cast<std::size_t>(std::distance(c1.cbegin(), c1.cend())) == c1.size());
-        assert(c1.max_load_factor() == 2);
-        assert(it2 == c1.begin()); // Iterators are not invalidated
-
-        LIBCPP_ASSERT(c2.bucket_count() == 0);
-        assert(c2.size() == 0);
-        assert(c2.hash_function() == Hash(1));
-        assert(c2.key_eq() == Compare(1));
-        assert(c2.get_allocator().get_id() == 2);
-        assert(static_cast<std::size_t>(std::distance(c2.begin(), c2.end())) == c2.size());
-        assert(static_cast<std::size_t>(std::distance(c2.cbegin(), c2.cend())) == c2.size());
-        assert(c2.max_load_factor() == 1);
-    }
-    {
-        typedef test_hash<int> Hash;
-        typedef test_equal_to<int> Compare;
-        typedef test_allocator<int> Alloc;
-        typedef std::unordered_set<int, Hash, Compare, Alloc> C;
-        typedef int P;
-        P a1[] =
-        {
-            P(1),
-            P(2),
-            P(3),
-            P(4),
-            P(1),
-            P(2)
-        };
-        C c1(std::begin(a1), std::end(a1), 0, Hash(1), Compare(1), Alloc(1, 1));
-        C c2(0, Hash(2), Compare(2), Alloc(1, 2));
-        c2.max_load_factor(2);
-        C::iterator it1 = c1.begin();
-        swap(c1, c2);
-
-        LIBCPP_ASSERT(c1.bucket_count() == 0);
-        assert(c1.size() == 0);
-        assert(c1.hash_function() == Hash(2));
-        assert(c1.key_eq() == Compare(2));
-        assert(c1.get_allocator().get_id() == 1);
-        assert(static_cast<std::size_t>(std::distance(c1.begin(), c1.end())) == c1.size());
-        assert(static_cast<std::size_t>(std::distance(c1.cbegin(), c1.cend())) == c1.size());
-        assert(c1.max_load_factor() == 2);
-
-        assert(c2.bucket_count() >= 4);
-        assert(c2.size() == 4);
-        assert(c2.count(1) == 1);
-        assert(c2.count(2) == 1);
-        assert(c2.count(3) == 1);
-        assert(c2.count(4) == 1);
-        assert(c2.hash_function() == Hash(1));
-        assert(c2.key_eq() == Compare(1));
-        assert(c2.get_allocator().get_id() == 2);
-        assert(static_cast<std::size_t>(std::distance(c2.begin(), c2.end())) == c2.size());
-        assert(static_cast<std::size_t>(std::distance(c2.cbegin(), c2.cend())) == c2.size());
-        assert(c2.max_load_factor() == 1);
-        assert(it1 == c2.begin()); // Iterators are not invalidated
-    }
-    {
-        typedef test_hash<int> Hash;
-        typedef test_equal_to<int> Compare;
-        typedef test_allocator<int> Alloc;
-        typedef std::unordered_set<int, Hash, Compare, Alloc> C;
-        typedef int P;
-        P a1[] =
-        {
-            P(1),
-            P(2),
-            P(3),
-            P(4),
-            P(1),
-            P(2)
-        };
-        P a2[] =
-        {
-            P(10),
-            P(20),
-            P(30),
-            P(40),
-            P(50),
-            P(60),
-            P(70),
-            P(80)
-        };
-        C c1(std::begin(a1), std::end(a1), 0, Hash(1), Compare(1), Alloc(1, 1));
-        C c2(std::begin(a2), std::end(a2), 0, Hash(2), Compare(2), Alloc(1, 2));
-        c2.max_load_factor(2);
-        C::iterator it1 = c1.begin();
-        C::iterator it2 = c2.begin();
-        swap(c1, c2);
-
-        assert(c1.bucket_count() >= 8);
-        assert(c1.size() == 8);
-        assert(*c1.find(10) == 10);
-        assert(*c1.find(20) == 20);
-        assert(*c1.find(30) == 30);
-        assert(*c1.find(40) == 40);
-        assert(*c1.find(50) == 50);
-        assert(*c1.find(60) == 60);
-        assert(*c1.find(70) == 70);
-        assert(*c1.find(80) == 80);
-        assert(c1.hash_function() == Hash(2));
-        assert(c1.key_eq() == Compare(2));
-        assert(c1.get_allocator().get_id() == 1);
-        assert(static_cast<std::size_t>(std::distance(c1.begin(), c1.end())) == c1.size());
-        assert(static_cast<std::size_t>(std::distance(c1.cbegin(), c1.cend())) == c1.size());
-        assert(c1.max_load_factor() == 2);
-        assert(it2 == c1.begin()); // Iterators are not invalidated
-
-        assert(c2.bucket_count() >= 4);
-        assert(c2.size() == 4);
-        assert(c2.count(1) == 1);
-        assert(c2.count(2) == 1);
-        assert(c2.count(3) == 1);
-        assert(c2.count(4) == 1);
-        assert(c2.hash_function() == Hash(1));
-        assert(c2.key_eq() == Compare(1));
-        assert(c2.get_allocator().get_id() == 2);
-        assert(static_cast<std::size_t>(std::distance(c2.begin(), c2.end())) == c2.size());
-        assert(static_cast<std::size_t>(std::distance(c2.cbegin(), c2.cend())) == c2.size());
-        assert(c2.max_load_factor() == 1);
-        assert(it1 == c2.begin()); // Iterators are not invalidated
-    }
-
-    {
-        typedef test_hash<int> Hash;
-        typedef test_equal_to<int> Compare;
-        typedef other_allocator<int> Alloc;
-        typedef std::unordered_set<int, Hash, Compare, Alloc> C;
-        C c1(0, Hash(1), Compare(1), Alloc(1));
-        C c2(0, Hash(2), Compare(2), Alloc(2));
-        c2.max_load_factor(2);
-        swap(c1, c2);
-
-        LIBCPP_ASSERT(c1.bucket_count() == 0);
-        assert(c1.size() == 0);
-        assert(c1.hash_function() == Hash(2));
-        assert(c1.key_eq() == Compare(2));
-        assert(c1.get_allocator() == Alloc(2));
-        assert(static_cast<std::size_t>(std::distance(c1.begin(), c1.end())) == c1.size());
-        assert(static_cast<std::size_t>(std::distance(c1.cbegin(), c1.cend())) == c1.size());
-        assert(c1.max_load_factor() == 2);
-
-        LIBCPP_ASSERT(c2.bucket_count() == 0);
-        assert(c2.size() == 0);
-        assert(c2.hash_function() == Hash(1));
-        assert(c2.key_eq() == Compare(1));
-        assert(c2.get_allocator() == Alloc(1));
-        assert(static_cast<std::size_t>(std::distance(c2.begin(), c2.end())) == c2.size());
-        assert(static_cast<std::size_t>(std::distance(c2.cbegin(), c2.cend())) == c2.size());
-        assert(c2.max_load_factor() == 1);
-    }
-    {
-        typedef test_hash<int> Hash;
-        typedef test_equal_to<int> Compare;
-        typedef other_allocator<int> Alloc;
-        typedef std::unordered_set<int, Hash, Compare, Alloc> C;
-        typedef int P;
-        P a2[] =
-        {
-            P(10),
-            P(20),
-            P(30),
-            P(40),
-            P(50),
-            P(60),
-            P(70),
-            P(80)
-        };
-        C c1(0, Hash(1), Compare(1), Alloc(1));
-        C c2(std::begin(a2), std::end(a2), 0, Hash(2), Compare(2), Alloc(2));
-        c2.max_load_factor(2);
-        swap(c1, c2);
-
-        assert(c1.bucket_count() >= 8);
-        assert(c1.size() == 8);
-        assert(*c1.find(10) == 10);
-        assert(*c1.find(20) == 20);
-        assert(*c1.find(30) == 30);
-        assert(*c1.find(40) == 40);
-        assert(*c1.find(50) == 50);
-        assert(*c1.find(60) == 60);
-        assert(*c1.find(70) == 70);
-        assert(*c1.find(80) == 80);
-        assert(c1.hash_function() == Hash(2));
-        assert(c1.key_eq() == Compare(2));
-        assert(c1.get_allocator() == Alloc(2));
-        assert(static_cast<std::size_t>(std::distance(c1.begin(), c1.end())) == c1.size());
-        assert(static_cast<std::size_t>(std::distance(c1.cbegin(), c1.cend())) == c1.size());
-        assert(c1.max_load_factor() == 2);
-
-        LIBCPP_ASSERT(c2.bucket_count() == 0);
-        assert(c2.size() == 0);
-        assert(c2.hash_function() == Hash(1));
-        assert(c2.key_eq() == Compare(1));
-        assert(c2.get_allocator() == Alloc(1));
-        assert(static_cast<std::size_t>(std::distance(c2.begin(), c2.end())) == c2.size());
-        assert(static_cast<std::size_t>(std::distance(c2.cbegin(), c2.cend())) == c2.size());
-        assert(c2.max_load_factor() == 1);
-    }
-    {
-        typedef test_hash<int> Hash;
-        typedef test_equal_to<int> Compare;
-        typedef other_allocator<int> Alloc;
-        typedef std::unordered_set<int, Hash, Compare, Alloc> C;
-        typedef int P;
-        P a1[] =
-        {
-            P(1),
-            P(2),
-            P(3),
-            P(4),
-            P(1),
-            P(2)
-        };
-        C c1(std::begin(a1), std::end(a1), 0, Hash(1), Compare(1), Alloc(1));
-        C c2(0, Hash(2), Compare(2), Alloc(2));
-        c2.max_load_factor(2);
-        swap(c1, c2);
-
-        LIBCPP_ASSERT(c1.bucket_count() == 0);
-        assert(c1.size() == 0);
-        assert(c1.hash_function() == Hash(2));
-        assert(c1.key_eq() == Compare(2));
-        assert(c1.get_allocator() == Alloc(2));
-        assert(static_cast<std::size_t>(std::distance(c1.begin(), c1.end())) == c1.size());
-        assert(static_cast<std::size_t>(std::distance(c1.cbegin(), c1.cend())) == c1.size());
-        assert(c1.max_load_factor() == 2);
-
-        assert(c2.bucket_count() >= 4);
-        assert(c2.size() == 4);
-        assert(c2.count(1) == 1);
-        assert(c2.count(2) == 1);
-        assert(c2.count(3) == 1);
-        assert(c2.count(4) == 1);
-        assert(c2.hash_function() == Hash(1));
-        assert(c2.key_eq() == Compare(1));
-        assert(c2.get_allocator() == Alloc(1));
-        assert(static_cast<std::size_t>(std::distance(c2.begin(), c2.end())) == c2.size());
-        assert(static_cast<std::size_t>(std::distance(c2.cbegin(), c2.cend())) == c2.size());
-        assert(c2.max_load_factor() == 1);
-    }
-    {
-        typedef test_hash<int> Hash;
-        typedef test_equal_to<int> Compare;
-        typedef other_allocator<int> Alloc;
-        typedef std::unordered_set<int, Hash, Compare, Alloc> C;
-        typedef int P;
-        P a1[] =
-        {
-            P(1),
-            P(2),
-            P(3),
-            P(4),
-            P(1),
-            P(2)
-        };
-        P a2[] =
-        {
-            P(10),
-            P(20),
-            P(30),
-            P(40),
-            P(50),
-            P(60),
-            P(70),
-            P(80)
-        };
-        C c1(std::begin(a1), std::end(a1), 0, Hash(1), Compare(1), Alloc(1));
-        C c2(std::begin(a2), std::end(a2), 0, Hash(2), Compare(2), Alloc(2));
-        c2.max_load_factor(2);
-        swap(c1, c2);
-
-        assert(c1.bucket_count() >= 8);
-        assert(c1.size() == 8);
-        assert(*c1.find(10) == 10);
-        assert(*c1.find(20) == 20);
-        assert(*c1.find(30) == 30);
-        assert(*c1.find(40) == 40);
-        assert(*c1.find(50) == 50);
-        assert(*c1.find(60) == 60);
-        assert(*c1.find(70) == 70);
-        assert(*c1.find(80) == 80);
-        assert(c1.hash_function() == Hash(2));
-        assert(c1.key_eq() == Compare(2));
-        assert(c1.get_allocator() == Alloc(2));
-        assert(static_cast<std::size_t>(std::distance(c1.begin(), c1.end())) == c1.size());
-        assert(static_cast<std::size_t>(std::distance(c1.cbegin(), c1.cend())) == c1.size());
-        assert(c1.max_load_factor() == 2);
-
-        assert(c2.bucket_count() >= 4);
-        assert(c2.size() == 4);
-        assert(c2.count(1) == 1);
-        assert(c2.count(2) == 1);
-        assert(c2.count(3) == 1);
-        assert(c2.count(4) == 1);
-        assert(c2.hash_function() == Hash(1));
-        assert(c2.key_eq() == Compare(1));
-        assert(c2.get_allocator() == Alloc(1));
-        assert(static_cast<std::size_t>(std::distance(c2.begin(), c2.end())) == c2.size());
-        assert(static_cast<std::size_t>(std::distance(c2.cbegin(), c2.cend())) == c2.size());
-        assert(c2.max_load_factor() == 1);
-    }
+int main(int, char**) {
+  {
+    typedef test_hash<int> Hash;
+    typedef test_equal_to<int> Compare;
+    typedef test_allocator<int> Alloc;
+    typedef std::unordered_set<int, Hash, Compare, Alloc> C;
+    C c1(0, Hash(1), Compare(1), Alloc(1, 1));
+    C c2(0, Hash(2), Compare(2), Alloc(1, 2));
+    c2.max_load_factor(2);
+    swap(c1, c2);
+
+    LIBCPP_ASSERT(c1.bucket_count() == 0);
+    assert(c1.size() == 0);
+    assert(c1.hash_function() == Hash(2));
+    assert(c1.key_eq() == Compare(2));
+    assert(c1.get_allocator().get_id() == 1);
+    assert(static_cast<std::size_t>(std::distance(c1.begin(), c1.end())) == c1.size());
+    assert(static_cast<std::size_t>(std::distance(c1.cbegin(), c1.cend())) == c1.size());
+    assert(c1.max_load_factor() == 2);
+
+    LIBCPP_ASSERT(c2.bucket_count() == 0);
+    assert(c2.size() == 0);
+    assert(c2.hash_function() == Hash(1));
+    assert(c2.key_eq() == Compare(1));
+    assert(c2.get_allocator().get_id() == 2);
+    assert(static_cast<std::size_t>(std::distance(c2.begin(), c2.end())) == c2.size());
+    assert(static_cast<std::size_t>(std::distance(c2.cbegin(), c2.cend())) == c2.size());
+    assert(c2.max_load_factor() == 1);
+  }
+  {
+    typedef test_hash<int> Hash;
+    typedef test_equal_to<int> Compare;
+    typedef test_allocator<int> Alloc;
+    typedef std::unordered_set<int, Hash, Compare, Alloc> C;
+    typedef int P;
+    P a2[] = {P(10), P(20), P(30), P(40), P(50), P(60), P(70), P(80)};
+    C c1(0, Hash(1), Compare(1), Alloc(1, 1));
+    C c2(std::begin(a2), std::end(a2), 0, Hash(2), Compare(2), Alloc(1, 2));
+    c2.max_load_factor(2);
+    C::iterator it2 = c2.begin();
+    swap(c1, c2);
+
+    assert(c1.bucket_count() >= 8);
+    assert(c1.size() == 8);
+    assert(*c1.find(10) == 10);
+    assert(*c1.find(20) == 20);
+    assert(*c1.find(30) == 30);
+    assert(*c1.find(40) == 40);
+    assert(*c1.find(50) == 50);
+    assert(*c1.find(60) == 60);
+    assert(*c1.find(70) == 70);
+    assert(*c1.find(80) == 80);
+    assert(c1.hash_function() == Hash(2));
+    assert(c1.key_eq() == Compare(2));
+    assert(c1.get_allocator().get_id() == 1);
+    assert(static_cast<std::size_t>(std::distance(c1.begin(), c1.end())) == c1.size());
+    assert(static_cast<std::size_t>(std::distance(c1.cbegin(), c1.cend())) == c1.size());
+    assert(c1.max_load_factor() == 2);
+    assert(it2 == c1.begin()); // Iterators are not invalidated
+
+    LIBCPP_ASSERT(c2.bucket_count() == 0);
+    assert(c2.size() == 0);
+    assert(c2.hash_function() == Hash(1));
+    assert(c2.key_eq() == Compare(1));
+    assert(c2.get_allocator().get_id() == 2);
+    assert(static_cast<std::size_t>(std::distance(c2.begin(), c2.end())) == c2.size());
+    assert(static_cast<std::size_t>(std::distance(c2.cbegin(), c2.cend())) == c2.size());
+    assert(c2.max_load_factor() == 1);
+  }
+  {
+    typedef test_hash<int> Hash;
+    typedef test_equal_to<int> Compare;
+    typedef test_allocator<int> Alloc;
+    typedef std::unordered_set<int, Hash, Compare, Alloc> C;
+    typedef int P;
+    P a1[] = {P(1), P(2), P(3), P(4), P(1), P(2)};
+    C c1(std::begin(a1), std::end(a1), 0, Hash(1), Compare(1), Alloc(1, 1));
+    C c2(0, Hash(2), Compare(2), Alloc(1, 2));
+    c2.max_load_factor(2);
+    C::iterator it1 = c1.begin();
+    swap(c1, c2);
+
+    LIBCPP_ASSERT(c1.bucket_count() == 0);
+    assert(c1.size() == 0);
+    assert(c1.hash_function() == Hash(2));
+    assert(c1.key_eq() == Compare(2));
+    assert(c1.get_allocator().get_id() == 1);
+    assert(static_cast<std::size_t>(std::distance(c1.begin(), c1.end())) == c1.size());
+    assert(static_cast<std::size_t>(std::distance(c1.cbegin(), c1.cend())) == c1.size());
+    assert(c1.max_load_factor() == 2);
+
+    assert(c2.bucket_count() >= 4);
+    assert(c2.size() == 4);
+    assert(c2.count(1) == 1);
+    assert(c2.count(2) == 1);
+    assert(c2.count(3) == 1);
+    assert(c2.count(4) == 1);
+    assert(c2.hash_function() == Hash(1));
+    assert(c2.key_eq() == Compare(1));
+    assert(c2.get_allocator().get_id() == 2);
+    assert(static_cast<std::size_t>(std::distance(c2.begin(), c2.end())) == c2.size());
+    assert(static_cast<std::size_t>(std::distance(c2.cbegin(), c2.cend())) == c2.size());
+    assert(c2.max_load_factor() == 1);
+    assert(it1 == c2.begin()); // Iterators are not invalidated
+  }
+  {
+    typedef test_hash<int> Hash;
+    typedef test_equal_to<int> Compare;
+    typedef test_allocator<int> Alloc;
+    typedef std::unordered_set<int, Hash, Compare, Alloc> C;
+    typedef int P;
+    P a1[] = {P(1), P(2), P(3), P(4), P(1), P(2)};
+    P a2[] = {P(10), P(20), P(30), P(40), P(50), P(60), P(70), P(80)};
+    C c1(std::begin(a1), std::end(a1), 0, Hash(1), Compare(1), Alloc(1, 1));
+    C c2(std::begin(a2), std::end(a2), 0, Hash(2), Compare(2), Alloc(1, 2));
+    c2.max_load_factor(2);
+    C::iterator it1 = c1.begin();
+    C::iterator it2 = c2.begin();
+    swap(c1, c2);
+
+    assert(c1.bucket_count() >= 8);
+    assert(c1.size() == 8);
+    assert(*c1.find(10) == 10);
+    assert(*c1.find(20) == 20);
+    assert(*c1.find(30) == 30);
+    assert(*c1.find(40) == 40);
+    assert(*c1.find(50) == 50);
+    assert(*c1.find(60) == 60);
+    assert(*c1.find(70) == 70);
+    assert(*c1.find(80) == 80);
+    assert(c1.hash_function() == Hash(2));
+    assert(c1.key_eq() == Compare(2));
+    assert(c1.get_allocator().get_id() == 1);
+    assert(static_cast<std::size_t>(std::distance(c1.begin(), c1.end())) == c1.size());
+    assert(static_cast<std::size_t>(std::distance(c1.cbegin(), c1.cend())) == c1.size());
+    assert(c1.max_load_factor() == 2);
+    assert(it2 == c1.begin()); // Iterators are not invalidated
+
+    assert(c2.bucket_count() >= 4);
+    assert(c2.size() == 4);
+    assert(c2.count(1) == 1);
+    assert(c2.count(2) == 1);
+    assert(c2.count(3) == 1);
+    assert(c2.count(4) == 1);
+    assert(c2.hash_function() == Hash(1));
+    assert(c2.key_eq() == Compare(1));
+    assert(c2.get_allocator().get_id() == 2);
+    assert(static_cast<std::size_t>(std::distance(c2.begin(), c2.end())) == c2.size());
+    assert(static_cast<std::size_t>(std::distance(c2.cbegin(), c2.cend())) == c2.size());
+    assert(c2.max_load_factor() == 1);
+    assert(it1 == c2.begin()); // Iterators are not invalidated
+  }
+
+  {
+    typedef test_hash<int> Hash;
+    typedef test_equal_to<int> Compare;
+    typedef other_allocator<int> Alloc;
+    typedef std::unordered_set<int, Hash, Compare, Alloc> C;
+    C c1(0, Hash(1), Compare(1), Alloc(1));
+    C c2(0, Hash(2), Compare(2), Alloc(2));
+    c2.max_load_factor(2);
+    swap(c1, c2);
+
+    LIBCPP_ASSERT(c1.bucket_count() == 0);
+    assert(c1.size() == 0);
+    assert(c1.hash_function() == Hash(2));
+    assert(c1.key_eq() == Compare(2));
+    assert(c1.get_allocator() == Alloc(2));
+    assert(static_cast<std::size_t>(std::distance(c1.begin(), c1.end())) == c1.size());
+    assert(static_cast<std::size_t>(std::distance(c1.cbegin(), c1.cend())) == c1.size());
+    assert(c1.max_load_factor() == 2);
+
+    LIBCPP_ASSERT(c2.bucket_count() == 0);
+    assert(c2.size() == 0);
+    assert(c2.hash_function() == Hash(1));
+    assert(c2.key_eq() == Compare(1));
+    assert(c2.get_allocator() == Alloc(1));
+    assert(static_cast<std::size_t>(std::distance(c2.begin(), c2.end())) == c2.size());
+    assert(static_cast<std::size_t>(std::distance(c2.cbegin(), c2.cend())) == c2.size());
+    assert(c2.max_load_factor() == 1);
+  }
+  {
+    typedef test_hash<int> Hash;
+    typedef test_equal_to<int> Compare;
+    typedef other_allocator<int> Alloc;
+    typedef std::unordered_set<int, Hash, Compare, Alloc> C;
+    typedef int P;
+    P a2[] = {P(10), P(20), P(30), P(40), P(50), P(60), P(70), P(80)};
+    C c1(0, Hash(1), Compare(1), Alloc(1));
+    C c2(std::begin(a2), std::end(a2), 0, Hash(2), Compare(2), Alloc(2));
+    c2.max_load_factor(2);
+    swap(c1, c2);
+
+    assert(c1.bucket_count() >= 8);
+    assert(c1.size() == 8);
+    assert(*c1.find(10) == 10);
+    assert(*c1.find(20) == 20);
+    assert(*c1.find(30) == 30);
+    assert(*c1.find(40) == 40);
+    assert(*c1.find(50) == 50);
+    assert(*c1.find(60) == 60);
+    assert(*c1.find(70) == 70);
+    assert(*c1.find(80) == 80);
+    assert(c1.hash_function() == Hash(2));
+    assert(c1.key_eq() == Compare(2));
+    assert(c1.get_allocator() == Alloc(2));
+    assert(static_cast<std::size_t>(std::distance(c1.begin(), c1.end())) == c1.size());
+    assert(static_cast<std::size_t>(std::distance(c1.cbegin(), c1.cend())) == c1.size());
+    assert(c1.max_load_factor() == 2);
+
+    LIBCPP_ASSERT(c2.bucket_count() == 0);
+    assert(c2.size() == 0);
+    assert(c2.hash_function() == Hash(1));
+    assert(c2.key_eq() == Compare(1));
+    assert(c2.get_allocator() == Alloc(1));
+    assert(static_cast<std::size_t>(std::distance(c2.begin(), c2.end())) == c2.size());
+    assert(static_cast<std::size_t>(std::distance(c2.cbegin(), c2.cend())) == c2.size());
+    assert(c2.max_load_factor() == 1);
+  }
+  {
+    typedef test_hash<int> Hash;
+    typedef test_equal_to<int> Compare;
+    typedef other_allocator<int> Alloc;
+    typedef std::unordered_set<int, Hash, Compare, Alloc> C;
+    typedef int P;
+    P a1[] = {P(1), P(2), P(3), P(4), P(1), P(2)};
+    C c1(std::begin(a1), std::end(a1), 0, Hash(1), Compare(1), Alloc(1));
+    C c2(0, Hash(2), Compare(2), Alloc(2));
+    c2.max_load_factor(2);
+    swap(c1, c2);
+
+    LIBCPP_ASSERT(c1.bucket_count() == 0);
+    assert(c1.size() == 0);
+    assert(c1.hash_function() == Hash(2));
+    assert(c1.key_eq() == Compare(2));
+    assert(c1.get_allocator() == Alloc(2));
+    assert(static_cast<std::size_t>(std::distance(c1.begin(), c1.end())) == c1.size());
+    assert(static_cast<std::size_t>(std::distance(c1.cbegin(), c1.cend())) == c1.size());
+    assert(c1.max_load_factor() == 2);
+
+    assert(c2.bucket_count() >= 4);
+    assert(c2.size() == 4);
+    assert(c2.count(1) == 1);
+    assert(c2.count(2) == 1);
+    assert(c2.count(3) == 1);
+    assert(c2.count(4) == 1);
+    assert(c2.hash_function() == Hash(1));
+    assert(c2.key_eq() == Compare(1));
+    assert(c2.get_allocator() == Alloc(1));
+    assert(static_cast<std::size_t>(std::distance(c2.begin(), c2.end())) == c2.size());
+    assert(static_cast<std::size_t>(std::distance(c2.cbegin(), c2.cend())) == c2.size());
+    assert(c2.max_load_factor() == 1);
+  }
+  {
+    typedef test_hash<int> Hash;
+    typedef test_equal_to<int> Compare;
+    typedef other_allocator<int> Alloc;
+    typedef std::unordered_set<int, Hash, Compare, Alloc> C;
+    typedef int P;
+    P a1[] = {P(1), P(2), P(3), P(4), P(1), P(2)};
+    P a2[] = {P(10), P(20), P(30), P(40), P(50), P(60), P(70), P(80)};
+    C c1(std::begin(a1), std::end(a1), 0, Hash(1), Compare(1), Alloc(1));
+    C c2(std::begin(a2), std::end(a2), 0, Hash(2), Compare(2), Alloc(2));
+    c2.max_load_factor(2);
+    swap(c1, c2);
+
+    assert(c1.bucket_count() >= 8);
+    assert(c1.size() == 8);
+    assert(*c1.find(10) == 10);
+    assert(*c1.find(20) == 20);
+    assert(*c1.find(30) == 30);
+    assert(*c1.find(40) == 40);
+    assert(*c1.find(50) == 50);
+    assert(*c1.find(60) == 60);
+    assert(*c1.find(70) == 70);
+    assert(*c1.find(80) == 80);
+    assert(c1.hash_function() == Hash(2));
+    assert(c1.key_eq() == Compare(2));
+    assert(c1.get_allocator() == Alloc(2));
+    assert(static_cast<std::size_t>(std::distance(c1.begin(), c1.end())) == c1.size());
+    assert(static_cast<std::size_t>(std::distance(c1.cbegin(), c1.cend())) == c1.size());
+    assert(c1.max_load_factor() == 2);
+
+    assert(c2.bucket_count() >= 4);
+    assert(c2.size() == 4);
+    assert(c2.count(1) == 1);
+    assert(c2.count(2) == 1);
+    assert(c2.count(3) == 1);
+    assert(c2.count(4) == 1);
+    assert(c2.hash_function() == Hash(1));
+    assert(c2.key_eq() == Compare(1));
+    assert(c2.get_allocator() == Alloc(1));
+    assert(static_cast<std::size_t>(std::distance(c2.begin(), c2.end())) == c2.size());
+    assert(static_cast<std::size_t>(std::distance(c2.cbegin(), c2.cend())) == c2.size());
+    assert(c2.max_load_factor() == 1);
+  }
 #if TEST_STD_VER >= 11
-    {
-        typedef test_hash<int> Hash;
-        typedef test_equal_to<int> Compare;
-        typedef min_allocator<int> Alloc;
-        typedef std::unordered_set<int, Hash, Compare, Alloc> C;
-        C c1(0, Hash(1), Compare(1), Alloc());
-        C c2(0, Hash(2), Compare(2), Alloc());
-        c2.max_load_factor(2);
-        swap(c1, c2);
-
-        LIBCPP_ASSERT(c1.bucket_count() == 0);
-        assert(c1.size() == 0);
-        assert(c1.hash_function() == Hash(2));
-        assert(c1.key_eq() == Compare(2));
-        assert(c1.get_allocator() == Alloc());
-        assert(static_cast<std::size_t>(std::distance(c1.begin(), c1.end())) == c1.size());
-        assert(static_cast<std::size_t>(std::distance(c1.cbegin(), c1.cend())) == c1.size());
-        assert(c1.max_load_factor() == 2);
-
-        LIBCPP_ASSERT(c2.bucket_count() == 0);
-        assert(c2.size() == 0);
-        assert(c2.hash_function() == Hash(1));
-        assert(c2.key_eq() == Compare(1));
-        assert(c2.get_allocator() == Alloc());
-        assert(static_cast<std::size_t>(std::distance(c2.begin(), c2.end())) == c2.size());
-        assert(static_cast<std::size_t>(std::distance(c2.cbegin(), c2.cend())) == c2.size());
-        assert(c2.max_load_factor() == 1);
-    }
-    {
-        typedef test_hash<int> Hash;
-        typedef test_equal_to<int> Compare;
-        typedef min_allocator<int> Alloc;
-        typedef std::unordered_set<int, Hash, Compare, Alloc> C;
-        typedef int P;
-        P a2[] =
-        {
-            P(10),
-            P(20),
-            P(30),
-            P(40),
-            P(50),
-            P(60),
-            P(70),
-            P(80)
-        };
-        C c1(0, Hash(1), Compare(1), Alloc());
-        C c2(std::begin(a2), std::end(a2), 0, Hash(2), Compare(2), Alloc());
-        c2.max_load_factor(2);
-        swap(c1, c2);
-
-        assert(c1.bucket_count() >= 8);
-        assert(c1.size() == 8);
-        assert(*c1.find(10) == 10);
-        assert(*c1.find(20) == 20);
-        assert(*c1.find(30) == 30);
-        assert(*c1.find(40) == 40);
-        assert(*c1.find(50) == 50);
-        assert(*c1.find(60) == 60);
-        assert(*c1.find(70) == 70);
-        assert(*c1.find(80) == 80);
-        assert(c1.hash_function() == Hash(2));
-        assert(c1.key_eq() == Compare(2));
-        assert(c1.get_allocator() == Alloc());
-        assert(static_cast<std::size_t>(std::distance(c1.begin(), c1.end())) == c1.size());
-        assert(static_cast<std::size_t>(std::distance(c1.cbegin(), c1.cend())) == c1.size());
-        assert(c1.max_load_factor() == 2);
-
-        LIBCPP_ASSERT(c2.bucket_count() == 0);
-        assert(c2.size() == 0);
-        assert(c2.hash_function() == Hash(1));
-        assert(c2.key_eq() == Compare(1));
-        assert(c2.get_allocator() == Alloc());
-        assert(static_cast<std::size_t>(std::distance(c2.begin(), c2.end())) == c2.size());
-        assert(static_cast<std::size_t>(std::distance(c2.cbegin(), c2.cend())) == c2.size());
-        assert(c2.max_load_factor() == 1);
-    }
-    {
-        typedef test_hash<int> Hash;
-        typedef test_equal_to<int> Compare;
-        typedef min_allocator<int> Alloc;
-        typedef std::unordered_set<int, Hash, Compare, Alloc> C;
-        typedef int P;
-        P a1[] =
-        {
-            P(1),
-            P(2),
-            P(3),
-            P(4),
-            P(1),
-            P(2)
-        };
-        C c1(std::begin(a1), std::end(a1), 0, Hash(1), Compare(1), Alloc());
-        C c2(0, Hash(2), Compare(2), Alloc());
-        c2.max_load_factor(2);
-        swap(c1, c2);
-
-        LIBCPP_ASSERT(c1.bucket_count() == 0);
-        assert(c1.size() == 0);
-        assert(c1.hash_function() == Hash(2));
-        assert(c1.key_eq() == Compare(2));
-        assert(c1.get_allocator() == Alloc());
-        assert(static_cast<std::size_t>(std::distance(c1.begin(), c1.end())) == c1.size());
-        assert(static_cast<std::size_t>(std::distance(c1.cbegin(), c1.cend())) == c1.size());
-        assert(c1.max_load_factor() == 2);
-
-        assert(c2.bucket_count() >= 4);
-        assert(c2.size() == 4);
-        assert(c2.count(1) == 1);
-        assert(c2.count(2) == 1);
-        assert(c2.count(3) == 1);
-        assert(c2.count(4) == 1);
-        assert(c2.hash_function() == Hash(1));
-        assert(c2.key_eq() == Compare(1));
-        assert(c2.get_allocator() == Alloc());
-        assert(static_cast<std::size_t>(std::distance(c2.begin(), c2.end())) == c2.size());
-        assert(static_cast<std::size_t>(std::distance(c2.cbegin(), c2.cend())) == c2.size());
-        assert(c2.max_load_factor() == 1);
-    }
-    {
-        typedef test_hash<int> Hash;
-        typedef test_equal_to<int> Compare;
-        typedef min_allocator<int> Alloc;
-        typedef std::unordered_set<int, Hash, Compare, Alloc> C;
-        typedef int P;
-        P a1[] =
-        {
-            P(1),
-            P(2),
-            P(3),
-            P(4),
-            P(1),
-            P(2)
-        };
-        P a2[] =
-        {
-            P(10),
-            P(20),
-            P(30),
-            P(40),
-            P(50),
-            P(60),
-            P(70),
-            P(80)
-        };
-        C c1(std::begin(a1), std::end(a1), 0, Hash(1), Compare(1), Alloc());
-        C c2(std::begin(a2), std::end(a2), 0, Hash(2), Compare(2), Alloc());
-        c2.max_load_factor(2);
-        swap(c1, c2);
-
-        assert(c1.bucket_count() >= 8);
-        assert(c1.size() == 8);
-        assert(*c1.find(10) == 10);
-        assert(*c1.find(20) == 20);
-        assert(*c1.find(30) == 30);
-        assert(*c1.find(40) == 40);
-        assert(*c1.find(50) == 50);
-        assert(*c1.find(60) == 60);
-        assert(*c1.find(70) == 70);
-        assert(*c1.find(80) == 80);
-        assert(c1.hash_function() == Hash(2));
-        assert(c1.key_eq() == Compare(2));
-        assert(c1.get_allocator() == Alloc());
-        assert(static_cast<std::size_t>(std::distance(c1.begin(), c1.end())) == c1.size());
-        assert(static_cast<std::size_t>(std::distance(c1.cbegin(), c1.cend())) == c1.size());
-        assert(c1.max_load_factor() == 2);
-
-        assert(c2.bucket_count() >= 4);
-        assert(c2.size() == 4);
-        assert(c2.count(1) == 1);
-        assert(c2.count(2) == 1);
-        assert(c2.count(3) == 1);
-        assert(c2.count(4) == 1);
-        assert(c2.hash_function() == Hash(1));
-        assert(c2.key_eq() == Compare(1));
-        assert(c2.get_allocator() == Alloc());
-        assert(static_cast<std::size_t>(std::distance(c2.begin(), c2.end())) == c2.size());
-        assert(static_cast<std::size_t>(std::distance(c2.cbegin(), c2.cend())) == c2.size());
-        assert(c2.max_load_factor() == 1);
-    }
+  {
+    typedef test_hash<int> Hash;
+    typedef test_equal_to<int> Compare;
+    typedef min_allocator<int> Alloc;
+    typedef std::unordered_set<int, Hash, Compare, Alloc> C;
+    C c1(0, Hash(1), Compare(1), Alloc());
+    C c2(0, Hash(2), Compare(2), Alloc());
+    c2.max_load_factor(2);
+    swap(c1, c2);
+
+    LIBCPP_ASSERT(c1.bucket_count() == 0);
+    assert(c1.size() == 0);
+    assert(c1.hash_function() == Hash(2));
+    assert(c1.key_eq() == Compare(2));
+    assert(c1.get_allocator() == Alloc());
+    assert(static_cast<std::size_t>(std::distance(c1.begin(), c1.end())) == c1.size());
+    assert(static_cast<std::size_t>(std::distance(c1.cbegin(), c1.cend())) == c1.size());
+    assert(c1.max_load_factor() == 2);
+
+    LIBCPP_ASSERT(c2.bucket_count() == 0);
+    assert(c2.size() == 0);
+    assert(c2.hash_function() == Hash(1));
+    assert(c2.key_eq() == Compare(1));
+    assert(c2.get_allocator() == Alloc());
+    assert(static_cast<std::size_t>(std::distance(c2.begin(), c2.end())) == c2.size());
+    assert(static_cast<std::size_t>(std::distance(c2.cbegin(), c2.cend())) == c2.size());
+    assert(c2.max_load_factor() == 1);
+  }
+  {
+    typedef test_hash<int> Hash;
+    typedef test_equal_to<int> Compare;
+    typedef min_allocator<int> Alloc;
+    typedef std::unordered_set<int, Hash, Compare, Alloc> C;
+    typedef int P;
+    P a2[] = {P(10), P(20), P(30), P(40), P(50), P(60), P(70), P(80)};
+    C c1(0, Hash(1), Compare(1), Alloc());
+    C c2(std::begin(a2), std::end(a2), 0, Hash(2), Compare(2), Alloc());
+    c2.max_load_factor(2);
+    swap(c1, c2);
+
+    assert(c1.bucket_count() >= 8);
+    assert(c1.size() == 8);
+    assert(*c1.find(10) == 10);
+    assert(*c1.find(20) == 20);
+    assert(*c1.find(30) == 30);
+    assert(*c1.find(40) == 40);
+    assert(*c1.find(50) == 50);
+    assert(*c1.find(60) == 60);
+    assert(*c1.find(70) == 70);
+    assert(*c1.find(80) == 80);
+    assert(c1.hash_function() == Hash(2));
+    assert(c1.key_eq() == Compare(2));
+    assert(c1.get_allocator() == Alloc());
+    assert(static_cast<std::size_t>(std::distance(c1.begin(), c1.end())) == c1.size());
+    assert(static_cast<std::size_t>(std::distance(c1.cbegin(), c1.cend())) == c1.size());
+    assert(c1.max_load_factor() == 2);
+
+    LIBCPP_ASSERT(c2.bucket_count() == 0);
+    assert(c2.size() == 0);
+    assert(c2.hash_function() == Hash(1));
+    assert(c2.key_eq() == Compare(1));
+    assert(c2.get_allocator() == Alloc());
+    assert(static_cast<std::size_t>(std::distance(c2.begin(), c2.end())) == c2.size());
+    assert(static_cast<std::size_t>(std::distance(c2.cbegin(), c2.cend())) == c2.size());
+    assert(c2.max_load_factor() == 1);
+  }
+  {
+    typedef test_hash<int> Hash;
+    typedef test_equal_to<int> Compare;
+    typedef min_allocator<int> Alloc;
+    typedef std::unordered_set<int, Hash, Compare, Alloc> C;
+    typedef int P;
+    P a1[] = {P(1), P(2), P(3), P(4), P(1), P(2)};
+    C c1(std::begin(a1), std::end(a1), 0, Hash(1), Compare(1), Alloc());
+    C c2(0, Hash(2), Compare(2), Alloc());
+    c2.max_load_factor(2);
+    swap(c1, c2);
+
+    LIBCPP_ASSERT(c1.bucket_count() == 0);
+    assert(c1.size() == 0);
+    assert(c1.hash_function() == Hash(2));
+    assert(c1.key_eq() == Compare(2));
+    assert(c1.get_allocator() == Alloc());
+    assert(static_cast<std::size_t>(std::distance(c1.begin(), c1.end())) == c1.size());
+    assert(static_cast<std::size_t>(std::distance(c1.cbegin(), c1.cend())) == c1.size());
+    assert(c1.max_load_factor() == 2);
+
+    assert(c2.bucket_count() >= 4);
+    assert(c2.size() == 4);
+    assert(c2.count(1) == 1);
+    assert(c2.count(2) == 1);
+    assert(c2.count(3) == 1);
+    assert(c2.count(4) == 1);
+    assert(c2.hash_function() == Hash(1));
+    assert(c2.key_eq() == Compare(1));
+    assert(c2.get_allocator() == Alloc());
+    assert(static_cast<std::size_t>(std::distance(c2.begin(), c2.end())) == c2.size());
+    assert(static_cast<std::size_t>(std::distance(c2.cbegin(), c2.cend())) == c2.size());
+    assert(c2.max_load_factor() == 1);
+  }
+  {
+    typedef test_hash<int> Hash;
+    typedef test_equal_to<int> Compare;
+    typedef min_allocator<int> Alloc;
+    typedef std::unordered_set<int, Hash, Compare, Alloc> C;
+    typedef int P;
+    P a1[] = {P(1), P(2), P(3), P(4), P(1), P(2)};
+    P a2[] = {P(10), P(20), P(30), P(40), P(50), P(60), P(70), P(80)};
+    C c1(std::begin(a1), std::end(a1), 0, Hash(1), Compare(1), Alloc());
+    C c2(std::begin(a2), std::end(a2), 0, Hash(2), Compare(2), Alloc());
+    c2.max_load_factor(2);
+    swap(c1, c2);
+
+    assert(c1.bucket_count() >= 8);
+    assert(c1.size() == 8);
+    assert(*c1.find(10) == 10);
+    assert(*c1.find(20) == 20);
+    assert(*c1.find(30) == 30);
+    assert(*c1.find(40) == 40);
+    assert(*c1.find(50) == 50);
+    assert(*c1.find(60) == 60);
+    assert(*c1.find(70) == 70);
+    assert(*c1.find(80) == 80);
+    assert(c1.hash_function() == Hash(2));
+    assert(c1.key_eq() == Compare(2));
+    assert(c1.get_allocator() == Alloc());
+    assert(static_cast<std::size_t>(std::distance(c1.begin(), c1.end())) == c1.size());
+    assert(static_cast<std::size_t>(std::distance(c1.cbegin(), c1.cend())) == c1.size());
+    assert(c1.max_load_factor() == 2);
+
+    assert(c2.bucket_count() >= 4);
+    assert(c2.size() == 4);
+    assert(c2.count(1) == 1);
+    assert(c2.count(2) == 1);
+    assert(c2.count(3) == 1);
+    assert(c2.count(4) == 1);
+    assert(c2.hash_function() == Hash(1));
+    assert(c2.key_eq() == Compare(1));
+    assert(c2.get_allocator() == Alloc());
+    assert(static_cast<std::size_t>(std::distance(c2.begin(), c2.end())) == c2.size());
+    assert(static_cast<std::size_t>(std::distance(c2.cbegin(), c2.cend())) == c2.size());
+    assert(c2.max_load_factor() == 1);
+  }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/views/mdspan/ConvertibleToIntegral.h b/libcxx/test/std/containers/views/mdspan/ConvertibleToIntegral.h
index 0ca5c330a390e..7768f3c5000e3 100644
--- a/libcxx/test/std/containers/views/mdspan/ConvertibleToIntegral.h
+++ b/libcxx/test/std/containers/views/mdspan/ConvertibleToIntegral.h
@@ -33,12 +33,20 @@ struct IntTypeNC {
 };
 
 // weird configurability of convertibility to int
-template<bool conv_c, bool conv_nc, bool ctor_nt_c, bool ctor_nt_nc>
+template <bool conv_c, bool conv_nc, bool ctor_nt_c, bool ctor_nt_nc>
 struct IntConfig {
   int val;
-  constexpr explicit IntConfig(int val_):val(val_){}
-  constexpr operator int() noexcept(ctor_nt_nc) requires(conv_nc) { return val; }
-  constexpr operator int() const noexcept(ctor_nt_c) requires(conv_c) { return val; }
+  constexpr explicit IntConfig(int val_) : val(val_) {}
+  constexpr operator int() noexcept(ctor_nt_nc)
+    requires(conv_nc)
+  {
+    return val;
+  }
+  constexpr operator int() const noexcept(ctor_nt_c)
+    requires(conv_c)
+  {
+    return val;
+  }
 };
 
 #endif // TEST_STD_CONTAINERS_VIEWS_MDSPAN_CONVERTIBLE_TO_INTEGRAL_H
diff --git a/libcxx/test/std/containers/views/mdspan/MinimalElementType.h b/libcxx/test/std/containers/views/mdspan/MinimalElementType.h
index 1d1a2c3752102..c39e502deda67 100644
--- a/libcxx/test/std/containers/views/mdspan/MinimalElementType.h
+++ b/libcxx/test/std/containers/views/mdspan/MinimalElementType.h
@@ -17,14 +17,14 @@
 // Make sure we don't assume copyable, default constructible, movable etc.
 struct MinimalElementType {
   int val;
-  constexpr MinimalElementType() = delete;
+  constexpr MinimalElementType()                          = delete;
   constexpr MinimalElementType(const MinimalElementType&) = delete;
-  constexpr explicit MinimalElementType(int v) noexcept : val(v){}
+  constexpr explicit MinimalElementType(int v) noexcept : val(v) {}
   constexpr MinimalElementType& operator=(const MinimalElementType&) = delete;
 };
 
 // Helper class to create pointer to MinimalElementType
-template<class T, size_t N>
+template <class T, size_t N>
 struct ElementPool {
   constexpr ElementPool() {
     ptr_ = std::allocator<std::remove_const_t<T>>().allocate(N);
diff --git a/libcxx/test/std/containers/views/mdspan/default_accessor/access.pass.cpp b/libcxx/test/std/containers/views/mdspan/default_accessor/access.pass.cpp
index a15ad6895254e..869c3b4c25361 100644
--- a/libcxx/test/std/containers/views/mdspan/default_accessor/access.pass.cpp
+++ b/libcxx/test/std/containers/views/mdspan/default_accessor/access.pass.cpp
@@ -27,7 +27,7 @@ constexpr void test_access() {
   ElementPool<std::remove_const_t<T>, 10> data;
   T* ptr = data.get_ptr();
   std::default_accessor<T> acc;
-  for(int i = 0; i < 10; i++) {
+  for (int i = 0; i < 10; i++) {
     static_assert(std::is_same_v<decltype(acc.access(ptr, i)), typename std::default_accessor<T>::reference>);
     ASSERT_NOEXCEPT(acc.access(ptr, i));
     assert(&acc.access(ptr, i) == ptr + i);
diff --git a/libcxx/test/std/containers/views/mdspan/default_accessor/ctor.conversion.pass.cpp b/libcxx/test/std/containers/views/mdspan/default_accessor/ctor.conversion.pass.cpp
index 299d98f0b2d01..b1b5806446c00 100644
--- a/libcxx/test/std/containers/views/mdspan/default_accessor/ctor.conversion.pass.cpp
+++ b/libcxx/test/std/containers/views/mdspan/default_accessor/ctor.conversion.pass.cpp
@@ -27,7 +27,7 @@
 #include "../MinimalElementType.h"
 
 struct Base {};
-struct Derived: public Base {};
+struct Derived : public Base {};
 
 template <class FromT, class ToT>
 constexpr void test_conversion() {
@@ -52,7 +52,8 @@ constexpr bool test() {
   // MinimalElementType is constructible from int, but accessors should not be convertible
   static_assert(!std::is_constructible_v<std::default_accessor<MinimalElementType>, std::default_accessor<int>>);
   // don't allow conversion from const elements to non-const
-  static_assert(!std::is_constructible_v<std::default_accessor<MinimalElementType>, std::default_accessor<const MinimalElementType>>);
+  static_assert(!std::is_constructible_v<std::default_accessor<MinimalElementType>,
+                                         std::default_accessor<const MinimalElementType>>);
   // don't allow conversion from Base to Derived
   static_assert(!std::is_constructible_v<std::default_accessor<Derived>, std::default_accessor<Base>>);
   // don't allow conversion from Derived to Base
diff --git a/libcxx/test/std/containers/views/mdspan/default_accessor/element_type.verify.cpp b/libcxx/test/std/containers/views/mdspan/default_accessor/element_type.verify.cpp
index 0ed870d9f541b..4c0dffe84ed24 100644
--- a/libcxx/test/std/containers/views/mdspan/default_accessor/element_type.verify.cpp
+++ b/libcxx/test/std/containers/views/mdspan/default_accessor/element_type.verify.cpp
@@ -30,4 +30,3 @@ void not_array_type() {
   // expected-error-re@*:* {{static assertion failed {{.*}}default_accessor: template argument may not be an array type}}
   [[maybe_unused]] std::default_accessor<int[5]> acc;
 }
-
diff --git a/libcxx/test/std/containers/views/mdspan/default_accessor/offset.pass.cpp b/libcxx/test/std/containers/views/mdspan/default_accessor/offset.pass.cpp
index 5842ff9179634..b342d1a70c62a 100644
--- a/libcxx/test/std/containers/views/mdspan/default_accessor/offset.pass.cpp
+++ b/libcxx/test/std/containers/views/mdspan/default_accessor/offset.pass.cpp
@@ -27,7 +27,7 @@ constexpr void test_offset() {
   ElementPool<std::remove_const_t<T>, 10> data;
   T* ptr = data.get_ptr();
   std::default_accessor<T> acc;
-  for(int i = 0; i < 10; i++) {
+  for (int i = 0; i < 10; i++) {
     static_assert(std::is_same_v<decltype(acc.offset(ptr, i)), typename std::default_accessor<T>::data_handle_type>);
     ASSERT_NOEXCEPT(acc.offset(ptr, i));
     assert(acc.offset(ptr, i) == ptr + i);
diff --git a/libcxx/test/std/containers/views/mdspan/default_accessor/types.pass.cpp b/libcxx/test/std/containers/views/mdspan/default_accessor/types.pass.cpp
index 025b992f97aef..ca65eecee8c5b 100644
--- a/libcxx/test/std/containers/views/mdspan/default_accessor/types.pass.cpp
+++ b/libcxx/test/std/containers/views/mdspan/default_accessor/types.pass.cpp
@@ -21,7 +21,6 @@
 //
 //  Each specialization of default_accessor is a trivially copyable type that models semiregular.
 
-
 #include <mdspan>
 #include <type_traits>
 #include <concepts>
diff --git a/libcxx/test/std/containers/views/mdspan/layout_left/index_operator.pass.cpp b/libcxx/test/std/containers/views/mdspan/layout_left/index_operator.pass.cpp
index 84c3ef45c69b6..75fd37055ad60 100644
--- a/libcxx/test/std/containers/views/mdspan/layout_left/index_operator.pass.cpp
+++ b/libcxx/test/std/containers/views/mdspan/layout_left/index_operator.pass.cpp
@@ -34,22 +34,20 @@
 
 #include "../ConvertibleToIntegral.h"
 
-template<class Mapping, class ... Indices>
-concept operator_constraints = requires(Mapping m, Indices ... idxs) {
-  {std::is_same_v<decltype(m(idxs...)), typename Mapping::index_type>};
+template <class Mapping, class... Indices>
+concept operator_constraints = requires(Mapping m, Indices... idxs) {
+  { std::is_same_v<decltype(m(idxs...)), typename Mapping::index_type> };
 };
 
-template<class Mapping, class ... Indices>
-  requires(
-    operator_constraints<Mapping, Indices...>
-  )
-constexpr bool check_operator_constraints(Mapping m, Indices ... idxs) {
-  (void) m(idxs...);
+template <class Mapping, class... Indices>
+  requires(operator_constraints<Mapping, Indices...>)
+constexpr bool check_operator_constraints(Mapping m, Indices... idxs) {
+  (void)m(idxs...);
   return true;
 }
 
-template<class Mapping, class ... Indices>
-constexpr bool check_operator_constraints(Mapping, Indices ...) {
+template <class Mapping, class... Indices>
+constexpr bool check_operator_constraints(Mapping, Indices...) {
   return false;
 }
 
@@ -86,15 +84,20 @@ constexpr bool test() {
   test_iteration<std::extents<signed char, D, D, D, D>>(1, 1, 1, 1);
 
   // Check operator constraint for number of arguments
-  static_assert(check_operator_constraints(std::layout_left::mapping<std::extents<int, D>>(std::extents<int, D>(1)), 0));
-  static_assert(!check_operator_constraints(std::layout_left::mapping<std::extents<int, D>>(std::extents<int, D>(1)), 0, 0));
+  static_assert(
+      check_operator_constraints(std::layout_left::mapping<std::extents<int, D>>(std::extents<int, D>(1)), 0));
+  static_assert(
+      !check_operator_constraints(std::layout_left::mapping<std::extents<int, D>>(std::extents<int, D>(1)), 0, 0));
 
   // Check operator constraint for convertibility of arguments to index_type
-  static_assert(check_operator_constraints(std::layout_left::mapping<std::extents<int, D>>(std::extents<int, D>(1)), IntType(0)));
-  static_assert(!check_operator_constraints(std::layout_left::mapping<std::extents<unsigned, D>>(std::extents<unsigned, D>(1)), IntType(0)));
+  static_assert(
+      check_operator_constraints(std::layout_left::mapping<std::extents<int, D>>(std::extents<int, D>(1)), IntType(0)));
+  static_assert(!check_operator_constraints(
+      std::layout_left::mapping<std::extents<unsigned, D>>(std::extents<unsigned, D>(1)), IntType(0)));
 
   // Check operator constraint for no-throw-constructibility of index_type from arguments
-  static_assert(!check_operator_constraints(std::layout_left::mapping<std::extents<unsigned char, D>>(std::extents<unsigned char, D>(1)), IntType(0)));
+  static_assert(!check_operator_constraints(
+      std::layout_left::mapping<std::extents<unsigned char, D>>(std::extents<unsigned char, D>(1)), IntType(0)));
 
   return true;
 }
diff --git a/libcxx/test/std/containers/views/mdspan/layout_right/index_operator.pass.cpp b/libcxx/test/std/containers/views/mdspan/layout_right/index_operator.pass.cpp
index 42d4e9a2d24b6..e9362ed50df24 100644
--- a/libcxx/test/std/containers/views/mdspan/layout_right/index_operator.pass.cpp
+++ b/libcxx/test/std/containers/views/mdspan/layout_right/index_operator.pass.cpp
@@ -34,22 +34,20 @@
 
 #include "../ConvertibleToIntegral.h"
 
-template<class Mapping, class ... Indices>
-concept operator_constraints = requires(Mapping m, Indices ... idxs) {
-  {std::is_same_v<decltype(m(idxs...)), typename Mapping::index_type>};
+template <class Mapping, class... Indices>
+concept operator_constraints = requires(Mapping m, Indices... idxs) {
+  { std::is_same_v<decltype(m(idxs...)), typename Mapping::index_type> };
 };
 
-template<class Mapping, class ... Indices>
-  requires(
-    operator_constraints<Mapping, Indices...>
-  )
-constexpr bool check_operator_constraints(Mapping m, Indices ... idxs) {
-  (void) m(idxs...);
+template <class Mapping, class... Indices>
+  requires(operator_constraints<Mapping, Indices...>)
+constexpr bool check_operator_constraints(Mapping m, Indices... idxs) {
+  (void)m(idxs...);
   return true;
 }
 
-template<class Mapping, class ... Indices>
-constexpr bool check_operator_constraints(Mapping, Indices ...) {
+template <class Mapping, class... Indices>
+constexpr bool check_operator_constraints(Mapping, Indices...) {
   return false;
 }
 
@@ -86,15 +84,20 @@ constexpr bool test() {
   test_iteration<std::extents<signed char, D, D, D, D>>(1, 1, 1, 1);
 
   // Check operator constraint for number of arguments
-  static_assert(check_operator_constraints(std::layout_right::mapping<std::extents<int, D>>(std::extents<int, D>(1)), 0));
-  static_assert(!check_operator_constraints(std::layout_right::mapping<std::extents<int, D>>(std::extents<int, D>(1)), 0, 0));
+  static_assert(
+      check_operator_constraints(std::layout_right::mapping<std::extents<int, D>>(std::extents<int, D>(1)), 0));
+  static_assert(
+      !check_operator_constraints(std::layout_right::mapping<std::extents<int, D>>(std::extents<int, D>(1)), 0, 0));
 
   // Check operator constraint for convertibility of arguments to index_type
-  static_assert(check_operator_constraints(std::layout_right::mapping<std::extents<int, D>>(std::extents<int, D>(1)), IntType(0)));
-  static_assert(!check_operator_constraints(std::layout_right::mapping<std::extents<unsigned, D>>(std::extents<unsigned, D>(1)), IntType(0)));
+  static_assert(check_operator_constraints(
+      std::layout_right::mapping<std::extents<int, D>>(std::extents<int, D>(1)), IntType(0)));
+  static_assert(!check_operator_constraints(
+      std::layout_right::mapping<std::extents<unsigned, D>>(std::extents<unsigned, D>(1)), IntType(0)));
 
   // Check operator constraint for no-throw-constructibility of index_type from arguments
-  static_assert(!check_operator_constraints(std::layout_right::mapping<std::extents<unsigned char, D>>(std::extents<unsigned char, D>(1)), IntType(0)));
+  static_assert(!check_operator_constraints(
+      std::layout_right::mapping<std::extents<unsigned char, D>>(std::extents<unsigned char, D>(1)), IntType(0)));
 
   return true;
 }
diff --git a/libcxx/test/std/containers/views/mdspan/mdspan/assign.pass.cpp b/libcxx/test/std/containers/views/mdspan/mdspan/assign.pass.cpp
index 4c59b5b61a5a4..e0577aeb2826d 100644
--- a/libcxx/test/std/containers/views/mdspan/mdspan/assign.pass.cpp
+++ b/libcxx/test/std/containers/views/mdspan/mdspan/assign.pass.cpp
@@ -49,9 +49,8 @@ constexpr void test_mdspan_types(const H& handle, const M& map, const A& acc) {
     assert(m.accessor() == acc);
 
   static_assert(std::is_trivially_assignable_v<MDS, const MDS&> ==
-                ((!std::is_class_v<H> ||
-                  std::is_trivially_assignable_v<H, const H&>)&&std::is_trivially_assignable_v<M, const M&> &&
-                 std::is_trivially_assignable_v<A, const A&>));
+                ((!std::is_class_v<H> || std::is_trivially_assignable_v<H, const H&>) &&
+                 std::is_trivially_assignable_v<M, const M&> && std::is_trivially_assignable_v<A, const A&>));
 }
 
 template <class H, class L, class A>
diff --git a/libcxx/test/std/containers/views/mdspan/mdspan/conversion.pass.cpp b/libcxx/test/std/containers/views/mdspan/mdspan/conversion.pass.cpp
index 5fe14c1562f4d..1adab199b0ca6 100644
--- a/libcxx/test/std/containers/views/mdspan/mdspan/conversion.pass.cpp
+++ b/libcxx/test/std/containers/views/mdspan/mdspan/conversion.pass.cpp
@@ -231,13 +231,13 @@ int main(int, char**) {
   test<t, o, t, o, t, t, t, t, checked_accessor<const MinimalElementType>>(checked_accessor<MinimalElementType>(1024));
   test<t, o, t, o, t, t, t, t, conv_test_accessor_c<int, t, t, t, t>>(conv_test_accessor_nc<int, t, t>());
   test<t, o, t, t, t, t, t, t, conv_test_accessor_c<int, t, t, o, o>>(conv_test_accessor_nc<int, t, o>());
-  // FIXME: these tests trigger what appears to be a compiler bug on MINGW32 with --target=x86_64-w64-windows-gnu
-  // https://godbolt.org/z/KK8aj5bs7
-  // Bug report: https://github.com/llvm/llvm-project/issues/64077
-  #ifndef __MINGW32__
+// FIXME: these tests trigger what appears to be a compiler bug on MINGW32 with --target=x86_64-w64-windows-gnu
+// https://godbolt.org/z/KK8aj5bs7
+// Bug report: https://github.com/llvm/llvm-project/issues/64077
+#ifndef __MINGW32__
   test<t, t, t, o, t, t, t, t, conv_test_accessor_c<int, o, t, t, t>>(conv_test_accessor_nc<int, t, t>());
   test<t, t, t, t, t, t, t, t, conv_test_accessor_c<int, o, o, o, o>>(conv_test_accessor_nc<int, t, o>());
-  #endif
+#endif
 
   // ElementType convertible, but accessor not constructible
   test<o, o, o, o, o, o, o, o, std::default_accessor<float>>(std::default_accessor<int>());
diff --git a/libcxx/test/std/containers/views/mdspan/mdspan/conversion.verify.cpp b/libcxx/test/std/containers/views/mdspan/mdspan/conversion.verify.cpp
index e3f03ebd07e5f..d46ba6ffbe5ec 100644
--- a/libcxx/test/std/containers/views/mdspan/mdspan/conversion.verify.cpp
+++ b/libcxx/test/std/containers/views/mdspan/mdspan/conversion.verify.cpp
@@ -45,8 +45,7 @@ void cant_construct_data_handle_type() {
   // expected-error-re@*:* {{{{.*}}no matching constructor for initialization of {{.*}} (aka 'not_const_convertible_handle<const int>')}}
   // expected-error-re@*:* {{static assertion failed {{.*}}mdspan: incompatible data_handle_type for mdspan construction}}
   [[maybe_unused]] std::
-      mdspan<const int, std::extents<int>, std::layout_right, convertible_accessor_but_not_handle<const int>>
-          m_c(m_nc);
+      mdspan<const int, std::extents<int>, std::layout_right, convertible_accessor_but_not_handle<const int>> m_c(m_nc);
 }
 
 void mapping_constructible_despite_extents_compatibility() {
diff --git a/libcxx/test/std/containers/views/mdspan/mdspan/ctor.copy.pass.cpp b/libcxx/test/std/containers/views/mdspan/mdspan/ctor.copy.pass.cpp
index 9540c37cfc4b9..b35a4fb832a1a 100644
--- a/libcxx/test/std/containers/views/mdspan/mdspan/ctor.copy.pass.cpp
+++ b/libcxx/test/std/containers/views/mdspan/mdspan/ctor.copy.pass.cpp
@@ -31,7 +31,7 @@ constexpr void test_mdspan_types(const H& handle, const M& map, const A& acc) {
 
   MDS m_org(handle, map, acc);
   MDS m(m_org);
-  static_assert(noexcept(MDS(m_org)) == (noexcept(H(handle))&& noexcept(M(map))&& noexcept(A(acc))));
+  static_assert(noexcept(MDS(m_org)) == (noexcept(H(handle)) && noexcept(M(map)) && noexcept(A(acc))));
   static_assert(
       std::is_trivially_copyable_v<MDS> ==
       (std::is_trivially_copyable_v<H> && std::is_trivially_copyable_v<M> && std::is_trivially_copyable_v<A>));
diff --git a/libcxx/test/std/containers/views/mdspan/mdspan/ctor.default.pass.cpp b/libcxx/test/std/containers/views/mdspan/mdspan/ctor.default.pass.cpp
index 5b84973bc43ef..f77dbc46285a7 100644
--- a/libcxx/test/std/containers/views/mdspan/mdspan/ctor.default.pass.cpp
+++ b/libcxx/test/std/containers/views/mdspan/mdspan/ctor.default.pass.cpp
@@ -43,7 +43,7 @@ constexpr void test_mdspan_types(const H&, const M&, const A&) {
 
   if constexpr (MDS::rank_dynamic() > 0 && hc && mc && ac) {
     MDS m;
-    static_assert(noexcept(MDS()) == (noexcept(H())&& noexcept(M())&& noexcept(A())));
+    static_assert(noexcept(MDS()) == (noexcept(H()) && noexcept(M()) && noexcept(A())));
     assert(m.extents() == typename MDS::extents_type());
     if constexpr (std::equality_comparable<H>)
       assert(m.data_handle() == H());
diff --git a/libcxx/test/std/containers/views/mdspan/mdspan/ctor.dh_array.pass.cpp b/libcxx/test/std/containers/views/mdspan/mdspan/ctor.dh_array.pass.cpp
index 9ca4fe240495c..f9540fd99c991 100644
--- a/libcxx/test/std/containers/views/mdspan/mdspan/ctor.dh_array.pass.cpp
+++ b/libcxx/test/std/containers/views/mdspan/mdspan/ctor.dh_array.pass.cpp
@@ -48,9 +48,8 @@ constexpr auto array_from_extents(const Extents& exts, std::index_sequence<Idxs.
 }
 
 template <class MDS, class Exts>
-concept check_mdspan_ctor_implicit = requires(MDS m, typename MDS::data_handle_type h, const Exts& exts) {
-  m = {h, exts};
-};
+concept check_mdspan_ctor_implicit =
+    requires(MDS m, typename MDS::data_handle_type h, const Exts& exts) { m = {h, exts}; };
 
 template <class H, class M, class A, size_t N>
 constexpr void
diff --git a/libcxx/test/std/containers/views/mdspan/mdspan/ctor.dh_span.pass.cpp b/libcxx/test/std/containers/views/mdspan/mdspan/ctor.dh_span.pass.cpp
index 2240dbe0801be..383a490082d73 100644
--- a/libcxx/test/std/containers/views/mdspan/mdspan/ctor.dh_span.pass.cpp
+++ b/libcxx/test/std/containers/views/mdspan/mdspan/ctor.dh_span.pass.cpp
@@ -48,9 +48,8 @@ constexpr auto array_from_extents(const Extents& exts, std::index_sequence<Idxs.
 }
 
 template <class MDS, class Exts>
-concept check_mdspan_ctor_implicit = requires(MDS m, typename MDS::data_handle_type h, const Exts& exts) {
-  m = {h, exts};
-};
+concept check_mdspan_ctor_implicit =
+    requires(MDS m, typename MDS::data_handle_type h, const Exts& exts) { m = {h, exts}; };
 
 template <class H, class M, class A, size_t N>
 constexpr void
diff --git a/libcxx/test/std/containers/views/mdspan/mdspan/index_operator.pass.cpp b/libcxx/test/std/containers/views/mdspan/mdspan/index_operator.pass.cpp
index 22020b1f64881..9124bd2314806 100644
--- a/libcxx/test/std/containers/views/mdspan/mdspan/index_operator.pass.cpp
+++ b/libcxx/test/std/containers/views/mdspan/mdspan/index_operator.pass.cpp
@@ -39,8 +39,8 @@
 #include "../ConvertibleToIntegral.h"
 #include "../CustomTestLayouts.h"
 
-// Clang 16 does not support argument packs as input to operator []
-#if defined(__clang_major__) && __clang_major__ < 17
+// Apple Clang does not support argument packs as input to operator []
+#ifdef TEST_COMPILER_APPLE_CLANG
 template <class MDS>
 constexpr auto& access(MDS mds) {
   return mds[];
@@ -84,7 +84,7 @@ template <class MDS, class... Args>
 constexpr void iterate(MDS mds, Args... args) {
   constexpr int r = static_cast<int>(MDS::extents_type::rank()) - 1 - static_cast<int>(sizeof...(Args));
   if constexpr (-1 == r) {
-#if defined(__clang_major__) && __clang_major__ < 17
+#ifdef TEST_COMPILER_APPLE_CLANG
     int* ptr1 = &access(mds, args...);
 #else
     int* ptr1 = &mds[args...];
diff --git a/libcxx/test/std/containers/views/mdspan/mdspan/types.pass.cpp b/libcxx/test/std/containers/views/mdspan/mdspan/types.pass.cpp
index 934e861e78d78..69bce593bd363 100644
--- a/libcxx/test/std/containers/views/mdspan/mdspan/types.pass.cpp
+++ b/libcxx/test/std/containers/views/mdspan/mdspan/types.pass.cpp
@@ -41,29 +41,29 @@
 
 // Calculated expected size of an mdspan
 // Note this expects that only default_accessor is empty
-template<class MDS>
+template <class MDS>
 constexpr size_t expected_size() {
   size_t sizeof_dht = sizeof(typename MDS::data_handle_type);
-  size_t result = sizeof_dht;
-  if(MDS::rank_dynamic() > 0) {
+  size_t result     = sizeof_dht;
+  if (MDS::rank_dynamic() > 0) {
     size_t alignof_idx = alignof(typename MDS::index_type);
-    size_t sizeof_idx = sizeof(typename MDS::index_type);
+    size_t sizeof_idx  = sizeof(typename MDS::index_type);
     // add alignment if necessary
-    result += sizeof_dht%alignof_idx == 0?0:alignof_idx - (sizeof_dht%alignof_idx);
+    result += sizeof_dht % alignof_idx == 0 ? 0 : alignof_idx - (sizeof_dht % alignof_idx);
     // add sizeof stored extents
     result += MDS::rank_dynamic() * sizeof_idx;
   }
   using A = typename MDS::accessor_type;
-  if(!std::is_same_v<A, std::default_accessor<typename MDS::element_type>>) {
+  if (!std::is_same_v<A, std::default_accessor<typename MDS::element_type>>) {
     size_t alignof_acc = alignof(A);
-    size_t sizeof_acc = sizeof(A);
+    size_t sizeof_acc  = sizeof(A);
     // add alignment if necessary
-    result += result%alignof_acc == 0?0:alignof_acc - (result%alignof_acc);
+    result += result % alignof_acc == 0 ? 0 : alignof_acc - (result % alignof_acc);
     // add sizeof stored accessor
     result += sizeof_acc;
   }
   // add alignment of the mdspan itself
-  result += result%alignof(MDS) == 0?0:alignof(MDS) - (result%alignof(MDS));
+  result += result % alignof(MDS) == 0 ? 0 : alignof(MDS) - (result % alignof(MDS));
   return result;
 }
 
diff --git a/libcxx/test/std/containers/views/views.span/range_concept_conformance.compile.pass.cpp b/libcxx/test/std/containers/views/views.span/range_concept_conformance.compile.pass.cpp
index fe28aeea3802b..52f0a76dfd2ce 100644
--- a/libcxx/test/std/containers/views/views.span/range_concept_conformance.compile.pass.cpp
+++ b/libcxx/test/std/containers/views/views.span/range_concept_conformance.compile.pass.cpp
@@ -17,7 +17,6 @@
 
 using range = std::span<int>;
 
-
 static_assert(std::same_as<std::ranges::iterator_t<range>, range::iterator>);
 static_assert(std::ranges::common_range<range>);
 static_assert(std::ranges::random_access_range<range>);
diff --git a/libcxx/test/std/containers/views/views.span/span.cons/array.pass.cpp b/libcxx/test/std/containers/views/views.span/span.cons/array.pass.cpp
index c02f42400b6e1..ad9f44aa474c6 100644
--- a/libcxx/test/std/containers/views/views.span/span.cons/array.pass.cpp
+++ b/libcxx/test/std/containers/views/views.span/span.cons/array.pass.cpp
@@ -24,139 +24,136 @@
 
 #include "test_macros.h"
 
-
-void checkCV()
-{
-                   int   arr[] = {1,2,3};
-    const          int  carr[] = {4,5,6};
-          volatile int  varr[] = {7,8,9};
-    const volatile int cvarr[] = {1,3,5};
-
-//  Types the same (dynamic sized)
-    {
-    std::span<               int> s1{  arr};    // a span<               int> pointing at int.
-    std::span<const          int> s2{ carr};    // a span<const          int> pointing at const int.
-    std::span<      volatile int> s3{ varr};    // a span<      volatile int> pointing at volatile int.
-    std::span<const volatile int> s4{cvarr};    // a span<const volatile int> pointing at const volatile int.
+void checkCV() {
+  int arr[]                  = {1, 2, 3};
+  const int carr[]           = {4, 5, 6};
+  volatile int varr[]        = {7, 8, 9};
+  const volatile int cvarr[] = {1, 3, 5};
+
+  //  Types the same (dynamic sized)
+  {
+    std::span< int> s1{arr};                 // a span<               int> pointing at int.
+    std::span<const int> s2{carr};           // a span<const          int> pointing at const int.
+    std::span< volatile int> s3{varr};       // a span<      volatile int> pointing at volatile int.
+    std::span<const volatile int> s4{cvarr}; // a span<const volatile int> pointing at const volatile int.
     assert(s1.size() + s2.size() + s3.size() + s4.size() == 12);
-    }
-
-//  Types the same (static sized)
-    {
-    std::span<               int,3> s1{  arr};  // a span<               int> pointing at int.
-    std::span<const          int,3> s2{ carr};  // a span<const          int> pointing at const int.
-    std::span<      volatile int,3> s3{ varr};  // a span<      volatile int> pointing at volatile int.
-    std::span<const volatile int,3> s4{cvarr};  // a span<const volatile int> pointing at const volatile int.
+  }
+
+  //  Types the same (static sized)
+  {
+    std::span< int, 3> s1{arr};                 // a span<               int> pointing at int.
+    std::span<const int, 3> s2{carr};           // a span<const          int> pointing at const int.
+    std::span< volatile int, 3> s3{varr};       // a span<      volatile int> pointing at volatile int.
+    std::span<const volatile int, 3> s4{cvarr}; // a span<const volatile int> pointing at const volatile int.
     assert(s1.size() + s2.size() + s3.size() + s4.size() == 12);
-    }
-
-
-//  types different (dynamic sized)
-    {
-    std::span<const          int> s1{ arr};     // a span<const          int> pointing at int.
-    std::span<      volatile int> s2{ arr};     // a span<      volatile int> pointing at int.
-    std::span<      volatile int> s3{ arr};     // a span<      volatile int> pointing at const int.
-    std::span<const volatile int> s4{ arr};     // a span<const volatile int> pointing at int.
-    std::span<const volatile int> s5{carr};     // a span<const volatile int> pointing at const int.
-    std::span<const volatile int> s6{varr};     // a span<const volatile int> pointing at volatile int.
+  }
+
+  //  types different (dynamic sized)
+  {
+    std::span<const int> s1{arr};           // a span<const          int> pointing at int.
+    std::span< volatile int> s2{arr};       // a span<      volatile int> pointing at int.
+    std::span< volatile int> s3{arr};       // a span<      volatile int> pointing at const int.
+    std::span<const volatile int> s4{arr};  // a span<const volatile int> pointing at int.
+    std::span<const volatile int> s5{carr}; // a span<const volatile int> pointing at const int.
+    std::span<const volatile int> s6{varr}; // a span<const volatile int> pointing at volatile int.
     assert(s1.size() + s2.size() + s3.size() + s4.size() + s5.size() + s6.size() == 18);
-    }
-
-//  types different (static sized)
-    {
-    std::span<const          int,3> s1{ arr};   // a span<const          int> pointing at int.
-    std::span<      volatile int,3> s2{ arr};   // a span<      volatile int> pointing at int.
-    std::span<      volatile int,3> s3{ arr};   // a span<      volatile int> pointing at const int.
-    std::span<const volatile int,3> s4{ arr};   // a span<const volatile int> pointing at int.
-    std::span<const volatile int,3> s5{carr};   // a span<const volatile int> pointing at const int.
-    std::span<const volatile int,3> s6{varr};   // a span<const volatile int> pointing at volatile int.
+  }
+
+  //  types different (static sized)
+  {
+    std::span<const int, 3> s1{arr};           // a span<const          int> pointing at int.
+    std::span< volatile int, 3> s2{arr};       // a span<      volatile int> pointing at int.
+    std::span< volatile int, 3> s3{arr};       // a span<      volatile int> pointing at const int.
+    std::span<const volatile int, 3> s4{arr};  // a span<const volatile int> pointing at int.
+    std::span<const volatile int, 3> s5{carr}; // a span<const volatile int> pointing at const int.
+    std::span<const volatile int, 3> s6{varr}; // a span<const volatile int> pointing at volatile int.
     assert(s1.size() + s2.size() + s3.size() + s4.size() + s5.size() + s6.size() == 18);
-    }
+  }
 }
 
-template<class T>
-constexpr bool testSpan()
-{
-    T val[2] = {};
-
-    ASSERT_NOEXCEPT(std::span<T>{val});
-    ASSERT_NOEXCEPT(std::span<T, 2>{val});
-    ASSERT_NOEXCEPT(std::span<const T>{val});
-    ASSERT_NOEXCEPT(std::span<const T, 2>{val});
-
-    std::span<T> s1 = val;
-    std::span<T, 2> s2 = val;
-    std::span<const T> s3 = val;
-    std::span<const T, 2> s4 = val;
-    assert(s1.data() == val && s1.size() == 2);
-    assert(s2.data() == val && s2.size() == 2);
-    assert(s3.data() == val && s3.size() == 2);
-    assert(s4.data() == val && s4.size() == 2);
-
-    std::span<const int> s5 = {{1,2}};
+template <class T>
+constexpr bool testSpan() {
+  T val[2] = {};
+
+  ASSERT_NOEXCEPT(std::span<T>{val});
+  ASSERT_NOEXCEPT(std::span<T, 2>{val});
+  ASSERT_NOEXCEPT(std::span<const T>{val});
+  ASSERT_NOEXCEPT(std::span<const T, 2>{val});
+
+  std::span<T> s1          = val;
+  std::span<T, 2> s2       = val;
+  std::span<const T> s3    = val;
+  std::span<const T, 2> s4 = val;
+  assert(s1.data() == val && s1.size() == 2);
+  assert(s2.data() == val && s2.size() == 2);
+  assert(s3.data() == val && s3.size() == 2);
+  assert(s4.data() == val && s4.size() == 2);
+
+  TEST_DIAGNOSTIC_PUSH
+  TEST_CLANG_DIAGNOSTIC_IGNORED("-Wdangling")
+  std::span<const int> s5 = {{1, 2}};
 #if TEST_STD_VER >= 26
-    std::span<const int, 2> s6({1, 2});
+  std::span<const int, 2> s6({1, 2});
 #else
-    std::span<const int, 2> s6 = {{1,2}};
+  std::span<const int, 2> s6 = {{1, 2}};
 #endif
-    assert(s5.size() == 2);  // and it dangles
-    assert(s6.size() == 2);  // and it dangles
+  assert(s5.size() == 2); // and it dangles
+  assert(s6.size() == 2); // and it dangles
+  TEST_DIAGNOSTIC_POP
 
-    return true;
+  return true;
 }
 
-
 struct A {};
 
-int main(int, char**)
-{
-    testSpan<int>();
-    testSpan<double>();
-    testSpan<A>();
-    testSpan<std::string>();
-
-    static_assert(testSpan<int>());
-    static_assert(testSpan<double>());
-    static_assert(testSpan<A>());
-
-    checkCV();
-
-    // Size wrong
-    {
-        static_assert(!std::is_constructible<std::span<int, 2>, int (&)[3]>::value, "");
-    }
-
-    // Type wrong
-    {
-        static_assert(!std::is_constructible<std::span<float>,    int (&)[3]>::value, "");
-        static_assert(!std::is_constructible<std::span<float, 3>, int (&)[3]>::value, "");
-    }
-
-    // CV wrong (dynamically sized)
-    {
-        static_assert(!std::is_constructible<std::span<int>, const int (&)[3]>::value, "");
-        static_assert(!std::is_constructible<std::span<int>, volatile int (&)[3]>::value, "");
-        static_assert(!std::is_constructible<std::span<int>, const volatile int (&)[3]>::value, "");
-
-        static_assert(!std::is_constructible<std::span<const int>, volatile int (&)[3]>::value, "");
-        static_assert(!std::is_constructible<std::span<const int>, const volatile int (&)[3]>::value, "");
-
-        static_assert(!std::is_constructible<std::span<volatile int>, const int (&)[3]>::value, "");
-        static_assert(!std::is_constructible<std::span<volatile int>, const volatile int (&)[3]>::value, "");
-    }
-
-    // CV wrong (statically sized)
-    {
-        static_assert(!std::is_constructible<std::span<int, 3>, const int (&)[3]>::value, "");
-        static_assert(!std::is_constructible<std::span<int, 3>, volatile int (&)[3]>::value, "");
-        static_assert(!std::is_constructible<std::span<int, 3>, const volatile int (&)[3]>::value, "");
-
-        static_assert(!std::is_constructible<std::span<const int, 3>, volatile int (&)[3]>::value, "");
-        static_assert(!std::is_constructible<std::span<const int, 3>, const volatile int (&)[3]>::value, "");
-
-        static_assert(!std::is_constructible<std::span<volatile int, 3>, const int (&)[3]>::value, "");
-        static_assert(!std::is_constructible<std::span<volatile int, 3>, const volatile int (&)[3]>::value, "");
-    }
-
-    return 0;
+int main(int, char**) {
+  testSpan<int>();
+  testSpan<double>();
+  testSpan<A>();
+  testSpan<std::string>();
+
+  static_assert(testSpan<int>());
+  static_assert(testSpan<double>());
+  static_assert(testSpan<A>());
+
+  checkCV();
+
+  // Size wrong
+  {
+    static_assert(!std::is_constructible<std::span<int, 2>, int(&)[3]>::value, "");
+  }
+
+  // Type wrong
+  {
+    static_assert(!std::is_constructible<std::span<float>, int(&)[3]>::value, "");
+    static_assert(!std::is_constructible<std::span<float, 3>, int(&)[3]>::value, "");
+  }
+
+  // CV wrong (dynamically sized)
+  {
+    static_assert(!std::is_constructible<std::span<int>, const int(&)[3]>::value, "");
+    static_assert(!std::is_constructible<std::span<int>, volatile int(&)[3]>::value, "");
+    static_assert(!std::is_constructible<std::span<int>, const volatile int(&)[3]>::value, "");
+
+    static_assert(!std::is_constructible<std::span<const int>, volatile int(&)[3]>::value, "");
+    static_assert(!std::is_constructible<std::span<const int>, const volatile int(&)[3]>::value, "");
+
+    static_assert(!std::is_constructible<std::span<volatile int>, const int(&)[3]>::value, "");
+    static_assert(!std::is_constructible<std::span<volatile int>, const volatile int(&)[3]>::value, "");
+  }
+
+  // CV wrong (statically sized)
+  {
+    static_assert(!std::is_constructible<std::span<int, 3>, const int(&)[3]>::value, "");
+    static_assert(!std::is_constructible<std::span<int, 3>, volatile int(&)[3]>::value, "");
+    static_assert(!std::is_constructible<std::span<int, 3>, const volatile int(&)[3]>::value, "");
+
+    static_assert(!std::is_constructible<std::span<const int, 3>, volatile int(&)[3]>::value, "");
+    static_assert(!std::is_constructible<std::span<const int, 3>, const volatile int(&)[3]>::value, "");
+
+    static_assert(!std::is_constructible<std::span<volatile int, 3>, const int(&)[3]>::value, "");
+    static_assert(!std::is_constructible<std::span<volatile int, 3>, const volatile int(&)[3]>::value, "");
+  }
+
+  return 0;
 }
diff --git a/libcxx/test/std/containers/views/views.span/span.cons/assign.pass.cpp b/libcxx/test/std/containers/views/views.span/span.cons/assign.pass.cpp
index 3f358116b8c6f..dcbf0fb36d858 100644
--- a/libcxx/test/std/containers/views/views.span/span.cons/assign.pass.cpp
+++ b/libcxx/test/std/containers/views/views.span/span.cons/assign.pass.cpp
@@ -20,278 +20,257 @@
 #include "test_macros.h"
 
 template <typename T>
-constexpr bool doAssign(T lhs, T rhs)
-{
-    ASSERT_NOEXCEPT(std::declval<T&>() = rhs);
-    lhs = rhs;
-    return lhs.data() == rhs.data()
-     &&    lhs.size() == rhs.size();
+constexpr bool doAssign(T lhs, T rhs) {
+  ASSERT_NOEXCEPT(std::declval<T&>() = rhs);
+  lhs = rhs;
+  return lhs.data() == rhs.data() && lhs.size() == rhs.size();
 }
 
-struct A{};
-
-constexpr int carr1[] = {1,2,3,4};
-constexpr int carr2[] = {3,4,5};
-constexpr int carr3[] = {7,8};
-          int   arr[] = {5,6,7,9};
-std::string strs[] = {"ABC", "DEF", "GHI"};
-
-
-int main(int, char**)
-{
-
-//  constexpr dynamically sized assignment
-    {
-//  On systems where 'ptrdiff_t' is a synonym for 'int',
-//  the call span(ptr, 0) selects the (pointer, size_type) constructor.
-//  On systems where 'ptrdiff_t' is NOT a synonym for 'int',
-//  it is ambiguous, because of 0 also being convertible to a null pointer
-//  and so the compiler can't choose between:
-//      span(pointer, size_type)
-//  and span(pointer, pointer)
-//  We cast zero to std::ptrdiff_t to remove that ambiguity.
-//  Example:
-//      On darwin x86_64, ptrdiff_t is the same as long int.
-//      On darwin i386, ptrdiff_t is the same as int.
-        constexpr std::span<const int> spans[] = {
-            {},
-            {carr1, static_cast<std::size_t>(0)},
-            {carr1,     1U},
-            {carr1,     2U},
-            {carr1,     3U},
-            {carr1,     4U},
-            {carr2, static_cast<std::size_t>(0)},
-            {carr2,     1U},
-            {carr2,     2U},
-            {carr2,     3U},
-            {carr3, static_cast<std::size_t>(0)},
-            {carr3,     1U},
-            {carr3,     2U}
-            };
-
-        static_assert(std::size(spans) == 13, "" );
-
-//  No for loops in constexpr land :-(
-        static_assert(doAssign(spans[0], spans[0]), "");
-        static_assert(doAssign(spans[0], spans[1]), "");
-        static_assert(doAssign(spans[0], spans[2]), "");
-        static_assert(doAssign(spans[0], spans[3]), "");
-        static_assert(doAssign(spans[0], spans[4]), "");
-        static_assert(doAssign(spans[0], spans[5]), "");
-        static_assert(doAssign(spans[0], spans[6]), "");
-        static_assert(doAssign(spans[0], spans[7]), "");
-        static_assert(doAssign(spans[0], spans[8]), "");
-        static_assert(doAssign(spans[0], spans[9]), "");
-        static_assert(doAssign(spans[0], spans[10]), "");
-        static_assert(doAssign(spans[0], spans[11]), "");
-        static_assert(doAssign(spans[0], spans[12]), "");
-
-        static_assert(doAssign(spans[1], spans[1]), "");
-        static_assert(doAssign(spans[1], spans[2]), "");
-        static_assert(doAssign(spans[1], spans[3]), "");
-        static_assert(doAssign(spans[1], spans[4]), "");
-        static_assert(doAssign(spans[1], spans[5]), "");
-        static_assert(doAssign(spans[1], spans[6]), "");
-        static_assert(doAssign(spans[1], spans[7]), "");
-        static_assert(doAssign(spans[1], spans[8]), "");
-        static_assert(doAssign(spans[1], spans[9]), "");
-        static_assert(doAssign(spans[1], spans[10]), "");
-        static_assert(doAssign(spans[1], spans[11]), "");
-        static_assert(doAssign(spans[1], spans[12]), "");
-
-        static_assert(doAssign(spans[2], spans[2]), "");
-        static_assert(doAssign(spans[2], spans[3]), "");
-        static_assert(doAssign(spans[2], spans[4]), "");
-        static_assert(doAssign(spans[2], spans[5]), "");
-        static_assert(doAssign(spans[2], spans[6]), "");
-        static_assert(doAssign(spans[2], spans[7]), "");
-        static_assert(doAssign(spans[2], spans[8]), "");
-        static_assert(doAssign(spans[2], spans[9]), "");
-        static_assert(doAssign(spans[2], spans[10]), "");
-        static_assert(doAssign(spans[2], spans[11]), "");
-        static_assert(doAssign(spans[2], spans[12]), "");
-
-        static_assert(doAssign(spans[3], spans[3]), "");
-        static_assert(doAssign(spans[3], spans[4]), "");
-        static_assert(doAssign(spans[3], spans[4]), "");
-        static_assert(doAssign(spans[3], spans[4]), "");
-        static_assert(doAssign(spans[3], spans[4]), "");
-        static_assert(doAssign(spans[3], spans[4]), "");
-        static_assert(doAssign(spans[3], spans[4]), "");
-        static_assert(doAssign(spans[3], spans[4]), "");
-        static_assert(doAssign(spans[3], spans[4]), "");
-        static_assert(doAssign(spans[3], spans[10]), "");
-        static_assert(doAssign(spans[3], spans[11]), "");
-        static_assert(doAssign(spans[3], spans[12]), "");
-
-        static_assert(doAssign(spans[4], spans[4]), "");
-        static_assert(doAssign(spans[4], spans[5]), "");
-        static_assert(doAssign(spans[4], spans[6]), "");
-        static_assert(doAssign(spans[4], spans[7]), "");
-        static_assert(doAssign(spans[4], spans[8]), "");
-        static_assert(doAssign(spans[4], spans[9]), "");
-        static_assert(doAssign(spans[4], spans[10]), "");
-        static_assert(doAssign(spans[4], spans[11]), "");
-        static_assert(doAssign(spans[4], spans[12]), "");
-
-        static_assert(doAssign(spans[5], spans[5]), "");
-        static_assert(doAssign(spans[5], spans[6]), "");
-        static_assert(doAssign(spans[5], spans[7]), "");
-        static_assert(doAssign(spans[5], spans[8]), "");
-        static_assert(doAssign(spans[5], spans[9]), "");
-        static_assert(doAssign(spans[5], spans[10]), "");
-        static_assert(doAssign(spans[5], spans[11]), "");
-        static_assert(doAssign(spans[5], spans[12]), "");
-
-        static_assert(doAssign(spans[6], spans[6]), "");
-        static_assert(doAssign(spans[6], spans[7]), "");
-        static_assert(doAssign(spans[6], spans[8]), "");
-        static_assert(doAssign(spans[6], spans[9]), "");
-        static_assert(doAssign(spans[6], spans[10]), "");
-        static_assert(doAssign(spans[6], spans[11]), "");
-        static_assert(doAssign(spans[6], spans[12]), "");
-
-        static_assert(doAssign(spans[7], spans[7]), "");
-        static_assert(doAssign(spans[7], spans[8]), "");
-        static_assert(doAssign(spans[7], spans[9]), "");
-        static_assert(doAssign(spans[7], spans[10]), "");
-        static_assert(doAssign(spans[7], spans[11]), "");
-        static_assert(doAssign(spans[7], spans[12]), "");
-
-        static_assert(doAssign(spans[8], spans[8]), "");
-        static_assert(doAssign(spans[8], spans[9]), "");
-        static_assert(doAssign(spans[8], spans[10]), "");
-        static_assert(doAssign(spans[8], spans[11]), "");
-        static_assert(doAssign(spans[8], spans[12]), "");
-
-        static_assert(doAssign(spans[9], spans[9]), "");
-        static_assert(doAssign(spans[9], spans[10]), "");
-        static_assert(doAssign(spans[9], spans[11]), "");
-        static_assert(doAssign(spans[9], spans[12]), "");
-
-        static_assert(doAssign(spans[10], spans[10]), "");
-        static_assert(doAssign(spans[10], spans[11]), "");
-        static_assert(doAssign(spans[10], spans[12]), "");
-
-        static_assert(doAssign(spans[11], spans[11]), "");
-        static_assert(doAssign(spans[11], spans[12]), "");
-
-        static_assert(doAssign(spans[12], spans[12]), "");
-
-//      for (size_t i = 0; i < std::size(spans); ++i)
-//          for (size_t j = i; j < std::size(spans); ++j)
-//              static_assert(doAssign(spans[i], spans[j]), "");
-    }
-
-//  constexpr statically sized assignment
-    {
-        using spanType = std::span<const int,2>;
-        constexpr spanType spans[] = {
-            spanType{carr1, 2},
-            spanType{carr1 + 1, 2},
-            spanType{carr1 + 2, 2},
-            spanType{carr2, 2},
-            spanType{carr2 + 1, 2},
-            spanType{carr3, 2}
-            };
-
-        static_assert(std::size(spans) == 6, "" );
-
-//  No for loops in constexpr land :-(
-        static_assert(doAssign(spans[0], spans[0]), "");
-        static_assert(doAssign(spans[0], spans[1]), "");
-        static_assert(doAssign(spans[0], spans[2]), "");
-        static_assert(doAssign(spans[0], spans[3]), "");
-        static_assert(doAssign(spans[0], spans[4]), "");
-        static_assert(doAssign(spans[0], spans[5]), "");
-
-        static_assert(doAssign(spans[1], spans[1]), "");
-        static_assert(doAssign(spans[1], spans[2]), "");
-        static_assert(doAssign(spans[1], spans[3]), "");
-        static_assert(doAssign(spans[1], spans[4]), "");
-        static_assert(doAssign(spans[1], spans[5]), "");
-
-        static_assert(doAssign(spans[2], spans[2]), "");
-        static_assert(doAssign(spans[2], spans[3]), "");
-        static_assert(doAssign(spans[2], spans[4]), "");
-        static_assert(doAssign(spans[2], spans[5]), "");
-
-        static_assert(doAssign(spans[3], spans[3]), "");
-        static_assert(doAssign(spans[3], spans[4]), "");
-        static_assert(doAssign(spans[3], spans[5]), "");
-
-        static_assert(doAssign(spans[4], spans[4]), "");
-        static_assert(doAssign(spans[4], spans[5]), "");
-
-        static_assert(doAssign(spans[5], spans[5]), "");
-
-//      for (size_t i = 0; i < std::size(spans); ++i)
-//          for (size_t j = i; j < std::size(spans); ++j)
-//              static_assert(doAssign(spans[i], spans[j]), "");
-    }
-
-
-//  dynamically sized assignment
-    {
-        std::span<int> spans[] = {
-            {},
-            {arr,     arr + 1},
-            {arr,     arr + 2},
-            {arr,     arr + 3},
-            {arr + 1, arr + 3} // same size as s2
-            };
-
-        for (std::size_t i = 0; i < std::size(spans); ++i)
-            for (std::size_t j = i; j < std::size(spans); ++j)
-                assert((doAssign(spans[i], spans[j])));
-    }
-
-//  statically sized assignment
-    {
-        using spanType = std::span<int,2>;
-        spanType spans[] = {
-            spanType{arr,     arr + 2},
-            spanType{arr + 1, arr + 3},
-            spanType{arr + 2, arr + 4}
-            };
-
-        for (std::size_t i = 0; i < std::size(spans); ++i)
-            for (std::size_t j = i; j < std::size(spans); ++j)
-                assert((doAssign(spans[i], spans[j])));
-    }
-
-//  dynamically sized assignment
-    {
+struct A {};
+
+constexpr int carr1[] = {1, 2, 3, 4};
+constexpr int carr2[] = {3, 4, 5};
+constexpr int carr3[] = {7, 8};
+int arr[]             = {5, 6, 7, 9};
+std::string strs[]    = {"ABC", "DEF", "GHI"};
+
+int main(int, char**) {
+  //  constexpr dynamically sized assignment
+  {
+    //  On systems where 'ptrdiff_t' is a synonym for 'int',
+    //  the call span(ptr, 0) selects the (pointer, size_type) constructor.
+    //  On systems where 'ptrdiff_t' is NOT a synonym for 'int',
+    //  it is ambiguous, because of 0 also being convertible to a null pointer
+    //  and so the compiler can't choose between:
+    //      span(pointer, size_type)
+    //  and span(pointer, pointer)
+    //  We cast zero to std::ptrdiff_t to remove that ambiguity.
+    //  Example:
+    //      On darwin x86_64, ptrdiff_t is the same as long int.
+    //      On darwin i386, ptrdiff_t is the same as int.
+    constexpr std::span<const int> spans[] = {
+        {},
+        {carr1, static_cast<std::size_t>(0)},
+        {carr1, 1U},
+        {carr1, 2U},
+        {carr1, 3U},
+        {carr1, 4U},
+        {carr2, static_cast<std::size_t>(0)},
+        {carr2, 1U},
+        {carr2, 2U},
+        {carr2, 3U},
+        {carr3, static_cast<std::size_t>(0)},
+        {carr3, 1U},
+        {carr3, 2U}};
+
+    static_assert(std::size(spans) == 13, "");
+
+    //  No for loops in constexpr land :-(
+    static_assert(doAssign(spans[0], spans[0]), "");
+    static_assert(doAssign(spans[0], spans[1]), "");
+    static_assert(doAssign(spans[0], spans[2]), "");
+    static_assert(doAssign(spans[0], spans[3]), "");
+    static_assert(doAssign(spans[0], spans[4]), "");
+    static_assert(doAssign(spans[0], spans[5]), "");
+    static_assert(doAssign(spans[0], spans[6]), "");
+    static_assert(doAssign(spans[0], spans[7]), "");
+    static_assert(doAssign(spans[0], spans[8]), "");
+    static_assert(doAssign(spans[0], spans[9]), "");
+    static_assert(doAssign(spans[0], spans[10]), "");
+    static_assert(doAssign(spans[0], spans[11]), "");
+    static_assert(doAssign(spans[0], spans[12]), "");
+
+    static_assert(doAssign(spans[1], spans[1]), "");
+    static_assert(doAssign(spans[1], spans[2]), "");
+    static_assert(doAssign(spans[1], spans[3]), "");
+    static_assert(doAssign(spans[1], spans[4]), "");
+    static_assert(doAssign(spans[1], spans[5]), "");
+    static_assert(doAssign(spans[1], spans[6]), "");
+    static_assert(doAssign(spans[1], spans[7]), "");
+    static_assert(doAssign(spans[1], spans[8]), "");
+    static_assert(doAssign(spans[1], spans[9]), "");
+    static_assert(doAssign(spans[1], spans[10]), "");
+    static_assert(doAssign(spans[1], spans[11]), "");
+    static_assert(doAssign(spans[1], spans[12]), "");
+
+    static_assert(doAssign(spans[2], spans[2]), "");
+    static_assert(doAssign(spans[2], spans[3]), "");
+    static_assert(doAssign(spans[2], spans[4]), "");
+    static_assert(doAssign(spans[2], spans[5]), "");
+    static_assert(doAssign(spans[2], spans[6]), "");
+    static_assert(doAssign(spans[2], spans[7]), "");
+    static_assert(doAssign(spans[2], spans[8]), "");
+    static_assert(doAssign(spans[2], spans[9]), "");
+    static_assert(doAssign(spans[2], spans[10]), "");
+    static_assert(doAssign(spans[2], spans[11]), "");
+    static_assert(doAssign(spans[2], spans[12]), "");
+
+    static_assert(doAssign(spans[3], spans[3]), "");
+    static_assert(doAssign(spans[3], spans[4]), "");
+    static_assert(doAssign(spans[3], spans[4]), "");
+    static_assert(doAssign(spans[3], spans[4]), "");
+    static_assert(doAssign(spans[3], spans[4]), "");
+    static_assert(doAssign(spans[3], spans[4]), "");
+    static_assert(doAssign(spans[3], spans[4]), "");
+    static_assert(doAssign(spans[3], spans[4]), "");
+    static_assert(doAssign(spans[3], spans[4]), "");
+    static_assert(doAssign(spans[3], spans[10]), "");
+    static_assert(doAssign(spans[3], spans[11]), "");
+    static_assert(doAssign(spans[3], spans[12]), "");
+
+    static_assert(doAssign(spans[4], spans[4]), "");
+    static_assert(doAssign(spans[4], spans[5]), "");
+    static_assert(doAssign(spans[4], spans[6]), "");
+    static_assert(doAssign(spans[4], spans[7]), "");
+    static_assert(doAssign(spans[4], spans[8]), "");
+    static_assert(doAssign(spans[4], spans[9]), "");
+    static_assert(doAssign(spans[4], spans[10]), "");
+    static_assert(doAssign(spans[4], spans[11]), "");
+    static_assert(doAssign(spans[4], spans[12]), "");
+
+    static_assert(doAssign(spans[5], spans[5]), "");
+    static_assert(doAssign(spans[5], spans[6]), "");
+    static_assert(doAssign(spans[5], spans[7]), "");
+    static_assert(doAssign(spans[5], spans[8]), "");
+    static_assert(doAssign(spans[5], spans[9]), "");
+    static_assert(doAssign(spans[5], spans[10]), "");
+    static_assert(doAssign(spans[5], spans[11]), "");
+    static_assert(doAssign(spans[5], spans[12]), "");
+
+    static_assert(doAssign(spans[6], spans[6]), "");
+    static_assert(doAssign(spans[6], spans[7]), "");
+    static_assert(doAssign(spans[6], spans[8]), "");
+    static_assert(doAssign(spans[6], spans[9]), "");
+    static_assert(doAssign(spans[6], spans[10]), "");
+    static_assert(doAssign(spans[6], spans[11]), "");
+    static_assert(doAssign(spans[6], spans[12]), "");
+
+    static_assert(doAssign(spans[7], spans[7]), "");
+    static_assert(doAssign(spans[7], spans[8]), "");
+    static_assert(doAssign(spans[7], spans[9]), "");
+    static_assert(doAssign(spans[7], spans[10]), "");
+    static_assert(doAssign(spans[7], spans[11]), "");
+    static_assert(doAssign(spans[7], spans[12]), "");
+
+    static_assert(doAssign(spans[8], spans[8]), "");
+    static_assert(doAssign(spans[8], spans[9]), "");
+    static_assert(doAssign(spans[8], spans[10]), "");
+    static_assert(doAssign(spans[8], spans[11]), "");
+    static_assert(doAssign(spans[8], spans[12]), "");
+
+    static_assert(doAssign(spans[9], spans[9]), "");
+    static_assert(doAssign(spans[9], spans[10]), "");
+    static_assert(doAssign(spans[9], spans[11]), "");
+    static_assert(doAssign(spans[9], spans[12]), "");
+
+    static_assert(doAssign(spans[10], spans[10]), "");
+    static_assert(doAssign(spans[10], spans[11]), "");
+    static_assert(doAssign(spans[10], spans[12]), "");
+
+    static_assert(doAssign(spans[11], spans[11]), "");
+    static_assert(doAssign(spans[11], spans[12]), "");
+
+    static_assert(doAssign(spans[12], spans[12]), "");
+
+    //      for (size_t i = 0; i < std::size(spans); ++i)
+    //          for (size_t j = i; j < std::size(spans); ++j)
+    //              static_assert(doAssign(spans[i], spans[j]), "");
+  }
+
+  //  constexpr statically sized assignment
+  {
+    using spanType             = std::span<const int, 2>;
+    constexpr spanType spans[] = {
+        spanType{carr1, 2},
+        spanType{carr1 + 1, 2},
+        spanType{carr1 + 2, 2},
+        spanType{carr2, 2},
+        spanType{carr2 + 1, 2},
+        spanType{carr3, 2}};
+
+    static_assert(std::size(spans) == 6, "");
+
+    //  No for loops in constexpr land :-(
+    static_assert(doAssign(spans[0], spans[0]), "");
+    static_assert(doAssign(spans[0], spans[1]), "");
+    static_assert(doAssign(spans[0], spans[2]), "");
+    static_assert(doAssign(spans[0], spans[3]), "");
+    static_assert(doAssign(spans[0], spans[4]), "");
+    static_assert(doAssign(spans[0], spans[5]), "");
+
+    static_assert(doAssign(spans[1], spans[1]), "");
+    static_assert(doAssign(spans[1], spans[2]), "");
+    static_assert(doAssign(spans[1], spans[3]), "");
+    static_assert(doAssign(spans[1], spans[4]), "");
+    static_assert(doAssign(spans[1], spans[5]), "");
+
+    static_assert(doAssign(spans[2], spans[2]), "");
+    static_assert(doAssign(spans[2], spans[3]), "");
+    static_assert(doAssign(spans[2], spans[4]), "");
+    static_assert(doAssign(spans[2], spans[5]), "");
+
+    static_assert(doAssign(spans[3], spans[3]), "");
+    static_assert(doAssign(spans[3], spans[4]), "");
+    static_assert(doAssign(spans[3], spans[5]), "");
+
+    static_assert(doAssign(spans[4], spans[4]), "");
+    static_assert(doAssign(spans[4], spans[5]), "");
+
+    static_assert(doAssign(spans[5], spans[5]), "");
+
+    //      for (size_t i = 0; i < std::size(spans); ++i)
+    //          for (size_t j = i; j < std::size(spans); ++j)
+    //              static_assert(doAssign(spans[i], spans[j]), "");
+  }
+
+  //  dynamically sized assignment
+  {
+    std::span<int> spans[] = {
+        {}, {arr, arr + 1}, {arr, arr + 2}, {arr, arr + 3}, {arr + 1, arr + 3} // same size as s2
+    };
+
+    for (std::size_t i = 0; i < std::size(spans); ++i)
+      for (std::size_t j = i; j < std::size(spans); ++j)
+        assert((doAssign(spans[i], spans[j])));
+  }
+
+  //  statically sized assignment
+  {
+    using spanType   = std::span<int, 2>;
+    spanType spans[] = {spanType{arr, arr + 2}, spanType{arr + 1, arr + 3}, spanType{arr + 2, arr + 4}};
+
+    for (std::size_t i = 0; i < std::size(spans); ++i)
+      for (std::size_t j = i; j < std::size(spans); ++j)
+        assert((doAssign(spans[i], spans[j])));
+  }
+
+  //  dynamically sized assignment
+  {
     std::span<std::string> spans[] = {
-            {strs,     strs},
-            {strs,     strs + 1},
-            {strs,     strs + 2},
-            {strs,     strs + 3},
-            {strs + 1, strs + 1},
-            {strs + 1, strs + 2},
-            {strs + 1, strs + 3},
-            {strs + 2, strs + 2},
-            {strs + 2, strs + 3},
-            {strs + 3, strs + 3}
-            };
-
-        for (std::size_t i = 0; i < std::size(spans); ++i)
-            for (std::size_t j = i; j < std::size(spans); ++j)
-                assert((doAssign(spans[i], spans[j])));
-    }
-
-    {
-    using spanType = std::span<std::string, 1>;
-    spanType spans[] = {
-            spanType{strs,     strs + 1},
-            spanType{strs + 1, strs + 2},
-            spanType{strs + 2, strs + 3}
-            };
-
-        for (std::size_t i = 0; i < std::size(spans); ++i)
-            for (std::size_t j = i; j < std::size(spans); ++j)
-                assert((doAssign(spans[i], spans[j])));
-    }
+        {strs, strs},
+        {strs, strs + 1},
+        {strs, strs + 2},
+        {strs, strs + 3},
+        {strs + 1, strs + 1},
+        {strs + 1, strs + 2},
+        {strs + 1, strs + 3},
+        {strs + 2, strs + 2},
+        {strs + 2, strs + 3},
+        {strs + 3, strs + 3}};
+
+    for (std::size_t i = 0; i < std::size(spans); ++i)
+      for (std::size_t j = i; j < std::size(spans); ++j)
+        assert((doAssign(spans[i], spans[j])));
+  }
+
+  {
+    using spanType   = std::span<std::string, 1>;
+    spanType spans[] = {spanType{strs, strs + 1}, spanType{strs + 1, strs + 2}, spanType{strs + 2, strs + 3}};
+
+    for (std::size_t i = 0; i < std::size(spans); ++i)
+      for (std::size_t j = i; j < std::size(spans); ++j)
+        assert((doAssign(spans[i], spans[j])));
+  }
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/views/views.span/span.cons/deduct.pass.cpp b/libcxx/test/std/containers/views/views.span/span.cons/deduct.pass.cpp
index 064d5c3a3fac4..28dda866371f5 100644
--- a/libcxx/test/std/containers/views/views.span/span.cons/deduct.pass.cpp
+++ b/libcxx/test/std/containers/views/views.span/span.cons/deduct.pass.cpp
@@ -24,7 +24,6 @@
 //   template<class R>
 //     span(R&&) -> span<remove_reference_t<ranges::range_reference_t<R>>>;
 
-
 #include <span>
 #include <array>
 #include <cassert>
@@ -38,16 +37,16 @@
 void test_iterator_sentinel() {
   int arr[] = {1, 2, 3};
   {
-  std::span s{std::begin(arr), std::end(arr)};
-  ASSERT_SAME_TYPE(decltype(s), std::span<int>);
-  assert(s.size() == std::size(arr));
-  assert(s.data() == std::data(arr));
+    std::span s{std::begin(arr), std::end(arr)};
+    ASSERT_SAME_TYPE(decltype(s), std::span<int>);
+    assert(s.size() == std::size(arr));
+    assert(s.data() == std::data(arr));
   }
   {
-  std::span s{std::begin(arr), 3};
-  ASSERT_SAME_TYPE(decltype(s), std::span<int>);
-  assert(s.size() == std::size(arr));
-  assert(s.data() == std::data(arr));
+    std::span s{std::begin(arr), 3};
+    ASSERT_SAME_TYPE(decltype(s), std::span<int>);
+    assert(s.size() == std::size(arr));
+    assert(s.data() == std::data(arr));
   }
 
 #if TEST_STD_VER >= 26
@@ -62,61 +61,60 @@ void test_iterator_sentinel() {
 }
 
 void test_c_array() {
-    {
+  {
     int arr[] = {1, 2, 3};
     std::span s{arr};
     ASSERT_SAME_TYPE(decltype(s), std::span<int, 3>);
     assert(s.size() == std::size(arr));
     assert(s.data() == std::data(arr));
-    }
+  }
 
-    {
-    const int arr[] = {1,2,3};
+  {
+    const int arr[] = {1, 2, 3};
     std::span s{arr};
     ASSERT_SAME_TYPE(decltype(s), std::span<const int, 3>);
     assert(s.size() == std::size(arr));
     assert(s.data() == std::data(arr));
-    }
+  }
 }
 
 void test_std_array() {
-    {
+  {
     std::array<double, 4> arr = {1.0, 2.0, 3.0, 4.0};
     std::span s{arr};
     ASSERT_SAME_TYPE(decltype(s), std::span<double, 4>);
     assert(s.size() == arr.size());
     assert(s.data() == arr.data());
-    }
+  }
 
-    {
+  {
     const std::array<long, 5> arr = {4, 5, 6, 7, 8};
     std::span s{arr};
     ASSERT_SAME_TYPE(decltype(s), std::span<const long, 5>);
     assert(s.size() == arr.size());
     assert(s.data() == arr.data());
-    }
+  }
 }
 
 void test_range_std_container() {
-    {
+  {
     std::string str{"ABCDE"};
     std::span s{str};
     ASSERT_SAME_TYPE(decltype(s), std::span<char>);
     assert(s.size() == str.size());
     assert(s.data() == str.data());
-    }
+  }
 
-    {
+  {
     const std::string str{"QWERTYUIOP"};
     std::span s{str};
     ASSERT_SAME_TYPE(decltype(s), std::span<const char>);
     assert(s.size() == str.size());
     assert(s.data() == str.data());
-    }
+  }
 }
 
-int main(int, char**)
-{
+int main(int, char**) {
   test_iterator_sentinel();
   test_c_array();
   test_std_array();
diff --git a/libcxx/test/std/containers/views/views.span/span.cons/default.pass.cpp b/libcxx/test/std/containers/views/views.span/span.cons/default.pass.cpp
index 7ae49c8c3237b..02fc48e145b77 100644
--- a/libcxx/test/std/containers/views/views.span/span.cons/default.pass.cpp
+++ b/libcxx/test/std/containers/views/views.span/span.cons/default.pass.cpp
@@ -18,70 +18,61 @@
 
 #include "test_macros.h"
 
-void checkCV()
-{
-//  Types the same (dynamic sized)
-    {
-    std::span<               int> s1;
-    std::span<const          int> s2;
-    std::span<      volatile int> s3;
+void checkCV() {
+  //  Types the same (dynamic sized)
+  {
+    std::span< int> s1;
+    std::span<const int> s2;
+    std::span< volatile int> s3;
     std::span<const volatile int> s4;
     assert(s1.size() + s2.size() + s3.size() + s4.size() == 0);
-    }
-
-//  Types the same (static sized)
-    {
-    std::span<               int,0> s1;
-    std::span<const          int,0> s2;
-    std::span<      volatile int,0> s3;
-    std::span<const volatile int,0> s4;
+  }
+
+  //  Types the same (static sized)
+  {
+    std::span< int, 0> s1;
+    std::span<const int, 0> s2;
+    std::span< volatile int, 0> s3;
+    std::span<const volatile int, 0> s4;
     assert(s1.size() + s2.size() + s3.size() + s4.size() == 0);
-    }
+  }
 }
 
-
 template <typename T>
-constexpr bool testConstexprSpan()
-{
-    std::span<const T>    s1;
-    std::span<const T, 0> s2;
-    return
-        s1.data() == nullptr && s1.size() == 0
-    &&  s2.data() == nullptr && s2.size() == 0;
+constexpr bool testConstexprSpan() {
+  std::span<const T> s1;
+  std::span<const T, 0> s2;
+  return s1.data() == nullptr && s1.size() == 0 && s2.data() == nullptr && s2.size() == 0;
 }
 
-
 template <typename T>
-void testRuntimeSpan()
-{
-    ASSERT_NOEXCEPT(T{});
-    std::span<const T>    s1;
-    std::span<const T, 0> s2;
-    assert(s1.data() == nullptr && s1.size() == 0);
-    assert(s2.data() == nullptr && s2.size() == 0);
+void testRuntimeSpan() {
+  ASSERT_NOEXCEPT(T{});
+  std::span<const T> s1;
+  std::span<const T, 0> s2;
+  assert(s1.data() == nullptr && s1.size() == 0);
+  assert(s2.data() == nullptr && s2.size() == 0);
 }
 
+struct A {};
 
-struct A{};
-
-int main(int, char**)
-{
-    static_assert(testConstexprSpan<int>(),    "");
-    static_assert(testConstexprSpan<long>(),   "");
-    static_assert(testConstexprSpan<double>(), "");
-    static_assert(testConstexprSpan<A>(),      "");
+int main(int, char**) {
+  static_assert(testConstexprSpan<int>(), "");
+  static_assert(testConstexprSpan<long>(), "");
+  static_assert(testConstexprSpan<double>(), "");
+  static_assert(testConstexprSpan<A>(), "");
 
-    testRuntimeSpan<int>();
-    testRuntimeSpan<long>();
-    testRuntimeSpan<double>();
-    testRuntimeSpan<std::string>();
-    testRuntimeSpan<A>();
+  testRuntimeSpan<int>();
+  testRuntimeSpan<long>();
+  testRuntimeSpan<double>();
+  testRuntimeSpan<std::string>();
+  testRuntimeSpan<A>();
 
-    checkCV();
+  checkCV();
 
-    static_assert( std::is_default_constructible_v<std::span<int, std::dynamic_extent>>, "");
-    static_assert( std::is_default_constructible_v<std::span<int, 0>>, "");
-    static_assert(!std::is_default_constructible_v<std::span<int, 2>>, "");
+  static_assert(std::is_default_constructible_v<std::span<int, std::dynamic_extent>>, "");
+  static_assert(std::is_default_constructible_v<std::span<int, 0>>, "");
+  static_assert(!std::is_default_constructible_v<std::span<int, 2>>, "");
 
-    return 0;
+  return 0;
 }
diff --git a/libcxx/test/std/containers/views/views.span/span.cons/iterator_len.pass.cpp b/libcxx/test/std/containers/views/views.span/span.cons/iterator_len.pass.cpp
index fcc2625ee6596..3d95733e9d6c5 100644
--- a/libcxx/test/std/containers/views/views.span/span.cons/iterator_len.pass.cpp
+++ b/libcxx/test/std/containers/views/views.span/span.cons/iterator_len.pass.cpp
@@ -42,8 +42,8 @@ constexpr void test_constructibility() {
 template <class T>
 constexpr bool test_ctor() {
   T val[2] = {};
-  auto s1 = std::span<T>(val, 2);
-  auto s2 = std::span<T, 2>(val, 2);
+  auto s1  = std::span<T>(val, 2);
+  auto s2  = std::span<T, 2>(val, 2);
   assert(s1.data() == std::data(val) && s1.size() == std::size(val));
   assert(s2.data() == std::data(val) && s2.size() == std::size(val));
   return true;
diff --git a/libcxx/test/std/containers/views/views.span/span.cons/iterator_sentinel.verify.cpp b/libcxx/test/std/containers/views/views.span/span.cons/iterator_sentinel.verify.cpp
index 937d8d921ea0b..b97c1a51572cb 100644
--- a/libcxx/test/std/containers/views/views.span/span.cons/iterator_sentinel.verify.cpp
+++ b/libcxx/test/std/containers/views/views.span/span.cons/iterator_sentinel.verify.cpp
@@ -19,9 +19,9 @@
 #include <iterator>
 #include <span>
 
-template<class T, std::size_t Extent>
+template <class T, std::size_t Extent>
 std::span<T, Extent> createImplicitSpan(T* first, T* last) {
-    return {first, last}; // expected-error {{chosen constructor is explicit in copy-initialization}}
+  return {first, last}; // expected-error {{chosen constructor is explicit in copy-initialization}}
 }
 
 void f() {
diff --git a/libcxx/test/std/containers/views/views.span/span.cons/range.pass.cpp b/libcxx/test/std/containers/views/views.span/span.cons/range.pass.cpp
index 71fee430f8f42..e9f8d26ee747a 100644
--- a/libcxx/test/std/containers/views/views.span/span.cons/range.pass.cpp
+++ b/libcxx/test/std/containers/views/views.span/span.cons/range.pass.cpp
@@ -12,7 +12,6 @@
 //  template<class R>
 //    constexpr explicit(Extent != dynamic_extent) span(R&& r);
 
-
 #include <span>
 #include <cassert>
 #include <ranges>
@@ -56,10 +55,16 @@ static_assert(std::is_constructible_v<std::span<const int, 3>, std::vector<int>>
 static_assert(!std::is_constructible_v<std::span<int>, std::vector<int>&&>);              // non-borrowed rvalue
 static_assert(!std::is_constructible_v<std::span<int, 3>, std::vector<int>&&>);           // non-borrowed rvalue
 
-static_assert(std::is_constructible_v<std::span<int>, std::ranges::subrange<contiguous_iterator<int*>>>);         // contiguous borrowed rvalue
-static_assert(std::is_constructible_v<std::span<int, 3>, std::ranges::subrange<contiguous_iterator<int*>>>);      // contiguous borrowed rvalue
-static_assert(!std::is_constructible_v<std::span<int>, std::ranges::subrange<random_access_iterator<int*>>>);     // non-contiguous borrowed rvalue
-static_assert(!std::is_constructible_v<std::span<int, 3>, std::ranges::subrange<random_access_iterator<int*>>>);  // non-contiguous borrowed rvalue
+static_assert(std::is_constructible_v<std::span<int>,
+                                      std::ranges::subrange<contiguous_iterator<int*>>>); // contiguous borrowed rvalue
+static_assert(std::is_constructible_v<std::span<int, 3>,
+                                      std::ranges::subrange<contiguous_iterator<int*>>>); // contiguous borrowed rvalue
+static_assert(
+    !std::is_constructible_v<std::span<int>,
+                             std::ranges::subrange<random_access_iterator<int*>>>); // non-contiguous borrowed rvalue
+static_assert(
+    !std::is_constructible_v<std::span<int, 3>,
+                             std::ranges::subrange<random_access_iterator<int*>>>); // non-contiguous borrowed rvalue
 
 using BorrowedContiguousSizedRange = std::string_view;
 static_assert(std::is_constructible_v<std::span<const char>, BorrowedContiguousSizedRange>);
diff --git a/libcxx/test/std/containers/views/views.span/span.cons/span.verify.cpp b/libcxx/test/std/containers/views/views.span/span.cons/span.verify.cpp
index 784530ebcb106..0b9a88d3dc656 100644
--- a/libcxx/test/std/containers/views/views.span/span.cons/span.verify.cpp
+++ b/libcxx/test/std/containers/views/views.span/span.cons/span.verify.cpp
@@ -16,99 +16,113 @@
 //      Extent == dynamic_extent || Extent == OtherExtent is true, and
 //      OtherElementType(*)[] is convertible to ElementType(*)[].
 
-
 #include <span>
 #include <cassert>
 #include <string>
 
 #include "test_macros.h"
 
-template<class T, std::size_t extent, size_t otherExtent>
+template <class T, std::size_t extent, size_t otherExtent>
 std::span<T, extent> createImplicitSpan(std::span<T, otherExtent> s) {
-    return {s}; // expected-error {{chosen constructor is explicit in copy-initialization}}
+  return {s}; // expected-error {{chosen constructor is explicit in copy-initialization}}
 }
 
-void checkCV ()
-{
-//  std::span<               int>   sp;
-    std::span<const          int>  csp;
-    std::span<      volatile int>  vsp;
-    std::span<const volatile int> cvsp;
-
-//  std::span<               int, 0>   sp0;
-    std::span<const          int, 0>  csp0;
-    std::span<      volatile int, 0>  vsp0;
-    std::span<const volatile int, 0> cvsp0;
-
-//  Try to remove const and/or volatile (dynamic -> dynamic)
-    {
-    std::span<               int> s1{ csp}; // expected-error {{no matching constructor for initialization of 'std::span<int>'}}
-    std::span<               int> s2{ vsp}; // expected-error {{no matching constructor for initialization of 'std::span<int>'}}
-    std::span<               int> s3{cvsp}; // expected-error {{no matching constructor for initialization of 'std::span<int>'}}
-
-    std::span<const          int> s4{ vsp}; // expected-error {{no matching constructor for initialization of 'std::span<const int>'}}
-    std::span<const          int> s5{cvsp}; // expected-error {{no matching constructor for initialization of 'std::span<const int>'}}
-
-    std::span<      volatile int> s6{ csp}; // expected-error {{no matching constructor for initialization of 'std::span<volatile int>'}}
-    std::span<      volatile int> s7{cvsp}; // expected-error {{no matching constructor for initialization of 'std::span<volatile int>'}}
-    }
-
-//  Try to remove const and/or volatile (static -> static)
-    {
-    std::span<               int, 0> s1{ csp0}; // expected-error {{no matching constructor for initialization of 'std::span<int, 0>'}}
-    std::span<               int, 0> s2{ vsp0}; // expected-error {{no matching constructor for initialization of 'std::span<int, 0>'}}
-    std::span<               int, 0> s3{cvsp0}; // expected-error {{no matching constructor for initialization of 'std::span<int, 0>'}}
-
-    std::span<const          int, 0> s4{ vsp0}; // expected-error {{no matching constructor for initialization of 'std::span<const int, 0>'}}
-    std::span<const          int, 0> s5{cvsp0}; // expected-error {{no matching constructor for initialization of 'std::span<const int, 0>'}}
-
-    std::span<      volatile int, 0> s6{ csp0}; // expected-error {{no matching constructor for initialization of 'std::span<volatile int, 0>'}}
-    std::span<      volatile int, 0> s7{cvsp0}; // expected-error {{no matching constructor for initialization of 'std::span<volatile int, 0>'}}
-    }
-
-//  Try to remove const and/or volatile (static -> dynamic)
-    {
-    std::span<               int> s1{ csp0}; // expected-error {{no matching constructor for initialization of 'std::span<int>'}}
-    std::span<               int> s2{ vsp0}; // expected-error {{no matching constructor for initialization of 'std::span<int>'}}
-    std::span<               int> s3{cvsp0}; // expected-error {{no matching constructor for initialization of 'std::span<int>'}}
-
-    std::span<const          int> s4{ vsp0}; // expected-error {{no matching constructor for initialization of 'std::span<const int>'}}
-    std::span<const          int> s5{cvsp0}; // expected-error {{no matching constructor for initialization of 'std::span<const int>'}}
-
-    std::span<      volatile int> s6{ csp0}; // expected-error {{no matching constructor for initialization of 'std::span<volatile int>'}}
-    std::span<      volatile int> s7{cvsp0}; // expected-error {{no matching constructor for initialization of 'std::span<volatile int>'}}
-    }
-
-//  Try to remove const and/or volatile (static -> static)
-    {
-    std::span<               int, 0> s1{ csp}; // expected-error {{no matching constructor for initialization of 'std::span<int, 0>'}}
-    std::span<               int, 0> s2{ vsp}; // expected-error {{no matching constructor for initialization of 'std::span<int, 0>'}}
-    std::span<               int, 0> s3{cvsp}; // expected-error {{no matching constructor for initialization of 'std::span<int, 0>'}}
-
-    std::span<const          int, 0> s4{ vsp}; // expected-error {{no matching constructor for initialization of 'std::span<const int, 0>'}}
-    std::span<const          int, 0> s5{cvsp}; // expected-error {{no matching constructor for initialization of 'std::span<const int, 0>'}}
-
-    std::span<      volatile int, 0> s6{ csp}; // expected-error {{no matching constructor for initialization of 'std::span<volatile int, 0>'}}
-    std::span<      volatile int, 0> s7{cvsp}; // expected-error {{no matching constructor for initialization of 'std::span<volatile int, 0>'}}
-    }
+void checkCV() {
+  //  std::span<               int>   sp;
+  std::span<const int> csp;
+  std::span< volatile int> vsp;
+  std::span<const volatile int> cvsp;
+
+  //  std::span<               int, 0>   sp0;
+  std::span<const int, 0> csp0;
+  std::span< volatile int, 0> vsp0;
+  std::span<const volatile int, 0> cvsp0;
+
+  //  Try to remove const and/or volatile (dynamic -> dynamic)
+  {
+    std::span< int> s1{csp};  // expected-error {{no matching constructor for initialization of 'std::span<int>'}}
+    std::span< int> s2{vsp};  // expected-error {{no matching constructor for initialization of 'std::span<int>'}}
+    std::span< int> s3{cvsp}; // expected-error {{no matching constructor for initialization of 'std::span<int>'}}
+
+    std::span<const int> s4{vsp};
+    // expected-error@-1 {{no matching constructor for initialization of 'std::span<const int>'}}
+    std::span<const int> s5{cvsp};
+    // expected-error@-1 {{no matching constructor for initialization of 'std::span<const int>'}}
+
+    std::span< volatile int> s6{csp};
+    // expected-error@-1 {{no matching constructor for initialization of 'std::span<volatile int>'}}
+    std::span< volatile int> s7{cvsp};
+    // expected-error@-1 {{no matching constructor for initialization of 'std::span<volatile int>'}}
+  }
+
+  //  Try to remove const and/or volatile (static -> static)
+  {
+    std::span< int, 0> s1{csp0}; // expected-error {{no matching constructor for initialization of 'std::span<int, 0>'}}
+    std::span< int, 0> s2{vsp0}; // expected-error {{no matching constructor for initialization of 'std::span<int, 0>'}}
+    std::span< int, 0> s3{cvsp0};
+    // expected-error@-1 {{no matching constructor for initialization of 'std::span<int, 0>'}}
+
+    std::span<const int, 0> s4{vsp0};
+    // expected-error@-1 {{no matching constructor for initialization of 'std::span<const int, 0>'}}
+    std::span<const int, 0> s5{cvsp0};
+    // expected-error@-1 {{no matching constructor for initialization of 'std::span<const int, 0>'}}
+
+    std::span< volatile int, 0> s6{csp0};
+    // expected-error@-1 {{no matching constructor for initialization of 'std::span<volatile int, 0>'}}
+    std::span< volatile int, 0> s7{cvsp0};
+    // expected-error@-1 {{no matching constructor for initialization of 'std::span<volatile int, 0>'}}
+  }
+
+  //  Try to remove const and/or volatile (static -> dynamic)
+  {
+    std::span< int> s1{csp0};  // expected-error {{no matching constructor for initialization of 'std::span<int>'}}
+    std::span< int> s2{vsp0};  // expected-error {{no matching constructor for initialization of 'std::span<int>'}}
+    std::span< int> s3{cvsp0}; // expected-error {{no matching constructor for initialization of 'std::span<int>'}}
+
+    std::span<const int> s4{vsp0};
+    // expected-error@-1 {{no matching constructor for initialization of 'std::span<const int>'}}
+    std::span<const int> s5{cvsp0};
+    // expected-error@-1 {{no matching constructor for initialization of 'std::span<const int>'}}
+
+    std::span< volatile int> s6{csp0};
+    // expected-error@-1 {{no matching constructor for initialization of 'std::span<volatile int>'}}
+    std::span< volatile int> s7{cvsp0};
+    // expected-error@-1 {{no matching constructor for initialization of 'std::span<volatile int>'}}
+  }
+
+  //  Try to remove const and/or volatile (static -> static)
+  {
+    std::span< int, 0> s1{csp};  // expected-error {{no matching constructor for initialization of 'std::span<int, 0>'}}
+    std::span< int, 0> s2{vsp};  // expected-error {{no matching constructor for initialization of 'std::span<int, 0>'}}
+    std::span< int, 0> s3{cvsp}; // expected-error {{no matching constructor for initialization of 'std::span<int, 0>'}}
+
+    std::span<const int, 0> s4{vsp};
+    // expected-error@-1 {{no matching constructor for initialization of 'std::span<const int, 0>'}}
+    std::span<const int, 0> s5{cvsp};
+    // expected-error@-1 {{no matching constructor for initialization of 'std::span<const int, 0>'}}
+
+    std::span< volatile int, 0> s6{csp};
+    // expected-error@-1 {{no matching constructor for initialization of 'std::span<volatile int, 0>'}}
+    std::span< volatile int, 0> s7{cvsp};
+    // expected-error@-1 {{no matching constructor for initialization of 'std::span<volatile int, 0>'}}
+  }
 }
 
-int main(int, char**)
-{
-    std::span<int>      sp;
-    std::span<int, 0>   sp0;
+int main(int, char**) {
+  std::span<int> sp;
+  std::span<int, 0> sp0;
 
-    std::span<float> s1{sp};    // expected-error {{no matching constructor for initialization of 'std::span<float>'}}
-    std::span<float> s2{sp0};   // expected-error {{no matching constructor for initialization of 'std::span<float>'}}
-    std::span<float, 0> s3{sp}; // expected-error {{no matching constructor for initialization of 'std::span<float, 0>'}}
-    std::span<float, 0> s4{sp0};    // expected-error {{no matching constructor for initialization of 'std::span<float, 0>'}}
+  std::span<float> s1{sp};     // expected-error {{no matching constructor for initialization of 'std::span<float>'}}
+  std::span<float> s2{sp0};    // expected-error {{no matching constructor for initialization of 'std::span<float>'}}
+  std::span<float, 0> s3{sp};  // expected-error {{no matching constructor for initialization of 'std::span<float, 0>'}}
+  std::span<float, 0> s4{sp0}; // expected-error {{no matching constructor for initialization of 'std::span<float, 0>'}}
 
-    checkCV();
+  checkCV();
 
-    // explicit constructor necessary
-    {
+  // explicit constructor necessary
+  {
     createImplicitSpan<int, 1>(sp);
-    }
+  }
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/views/views.span/span.cons/stdarray.pass.cpp b/libcxx/test/std/containers/views/views.span/span.cons/stdarray.pass.cpp
index 97aa349fdcbc6..1c261f4f38f74 100644
--- a/libcxx/test/std/containers/views/views.span/span.cons/stdarray.pass.cpp
+++ b/libcxx/test/std/containers/views/views.span/span.cons/stdarray.pass.cpp
@@ -19,7 +19,6 @@
 //   - remove_pointer_t<decltype(data(arr))>(*)[] is convertible to ElementType(*)[].
 //
 
-
 #include <span>
 #include <array>
 #include <cassert>
@@ -27,40 +26,38 @@
 
 #include "test_macros.h"
 
-void checkCV()
-{
-    std::array<int, 3> arr  = {1,2,3};
-//  STL says these are not cromulent
-//  std::array<const int,3> carr = {4,5,6};
-//  std::array<volatile int, 3> varr = {7,8,9};
-//  std::array<const volatile int, 3> cvarr = {1,3,5};
-
-//  Types the same (dynamic sized)
-    {
-    std::span<               int> s1{  arr};    // a span<               int> pointing at int.
-    }
-
-//  Types the same (static sized)
-    {
-    std::span<               int,3> s1{  arr};  // a span<               int> pointing at int.
-    }
-
-
-//  types different (dynamic sized)
-    {
-    std::span<const          int> s1{ arr};     // a span<const          int> pointing at int.
-    std::span<      volatile int> s2{ arr};     // a span<      volatile int> pointing at int.
-    std::span<      volatile int> s3{ arr};     // a span<      volatile int> pointing at const int.
-    std::span<const volatile int> s4{ arr};     // a span<const volatile int> pointing at int.
-    }
-
-//  types different (static sized)
-    {
-    std::span<const          int,3> s1{ arr};   // a span<const          int> pointing at int.
-    std::span<      volatile int,3> s2{ arr};   // a span<      volatile int> pointing at int.
-    std::span<      volatile int,3> s3{ arr};   // a span<      volatile int> pointing at const int.
-    std::span<const volatile int,3> s4{ arr};   // a span<const volatile int> pointing at int.
-    }
+void checkCV() {
+  std::array<int, 3> arr = {1, 2, 3};
+  //  STL says these are not cromulent
+  //  std::array<const int,3> carr = {4,5,6};
+  //  std::array<volatile int, 3> varr = {7,8,9};
+  //  std::array<const volatile int, 3> cvarr = {1,3,5};
+
+  //  Types the same (dynamic sized)
+  {
+    std::span< int> s1{arr}; // a span<               int> pointing at int.
+  }
+
+  //  Types the same (static sized)
+  {
+    std::span< int, 3> s1{arr}; // a span<               int> pointing at int.
+  }
+
+  //  types different (dynamic sized)
+  {
+    std::span<const int> s1{arr};          // a span<const          int> pointing at int.
+    std::span< volatile int> s2{arr};      // a span<      volatile int> pointing at int.
+    std::span< volatile int> s3{arr};      // a span<      volatile int> pointing at const int.
+    std::span<const volatile int> s4{arr}; // a span<const volatile int> pointing at int.
+  }
+
+  //  types different (static sized)
+  {
+    std::span<const int, 3> s1{arr};          // a span<const          int> pointing at int.
+    std::span< volatile int, 3> s2{arr};      // a span<      volatile int> pointing at int.
+    std::span< volatile int, 3> s3{arr};      // a span<      volatile int> pointing at const int.
+    std::span<const volatile int, 3> s4{arr}; // a span<const volatile int> pointing at int.
+  }
 }
 
 template <typename T, typename U>
@@ -70,8 +67,7 @@ constexpr bool testConstructorArray() {
   ASSERT_NOEXCEPT(std::span<T, 2>{val});
   std::span<T> s1{val};
   std::span<T, 2> s2{val};
-  return s1.data() == &val[0] && s1.size() == 2 && s2.data() == &val[0] &&
-         s2.size() == 2;
+  return s1.data() == &val[0] && s1.size() == 2 && s2.data() == &val[0] && s2.size() == 2;
 }
 
 template <typename T, typename U>
@@ -81,8 +77,7 @@ constexpr bool testConstructorConstArray() {
   ASSERT_NOEXCEPT(std::span<const T, 2>{val});
   std::span<const T> s1{val};
   std::span<const T, 2> s2{val};
-  return s1.data() == &val[0] && s1.size() == 2 && s2.data() == &val[0] &&
-         s2.size() == 2;
+  return s1.data() == &val[0] && s1.size() == 2 && s2.data() == &val[0] && s2.size() == 2;
 }
 
 template <typename T>
@@ -94,27 +89,23 @@ constexpr bool testConstructors() {
   static_assert(testConstructorConstArray<const T, const T>(), "");
   static_assert(testConstructorConstArray<const T, T>(), "");
 
-  return testConstructorArray<T, T>() &&
-         testConstructorArray<const T, const T>() &&
-         testConstructorArray<const T, T>() &&
-         testConstructorConstArray<T, T>() &&
-         testConstructorConstArray<const T, const T>() &&
-         testConstructorConstArray<const T, T>();
+  return testConstructorArray<T, T>() && testConstructorArray<const T, const T>() &&
+         testConstructorArray<const T, T>() && testConstructorConstArray<T, T>() &&
+         testConstructorConstArray<const T, const T>() && testConstructorConstArray<const T, T>();
 }
 
-struct A{};
+struct A {};
 
-int main(int, char**)
-{
-    assert(testConstructors<int>());
-    assert(testConstructors<long>());
-    assert(testConstructors<double>());
-    assert(testConstructors<A>());
+int main(int, char**) {
+  assert(testConstructors<int>());
+  assert(testConstructors<long>());
+  assert(testConstructors<double>());
+  assert(testConstructors<A>());
 
-    assert(testConstructors<int*>());
-    assert(testConstructors<const int*>());
+  assert(testConstructors<int*>());
+  assert(testConstructors<const int*>());
 
-    checkCV();
+  checkCV();
 
-    return 0;
+  return 0;
 }
diff --git a/libcxx/test/std/containers/views/views.span/span.elem/back.pass.cpp b/libcxx/test/std/containers/views/views.span/span.elem/back.pass.cpp
index dfc01d509210d..c5086d076798d 100644
--- a/libcxx/test/std/containers/views/views.span/span.elem/back.pass.cpp
+++ b/libcxx/test/std/containers/views/views.span/span.elem/back.pass.cpp
@@ -14,69 +14,61 @@
 //   Effects: Equivalent to: return *(data() + (size() - 1));
 //
 
-
 #include <span>
 #include <cassert>
 #include <string>
 
 #include "test_macros.h"
 
-
 template <typename Span>
-constexpr bool testConstexprSpan(Span sp)
-{
-    LIBCPP_ASSERT(noexcept(sp.back()));
-    return std::addressof(sp.back()) == sp.data() + sp.size() - 1;
+constexpr bool testConstexprSpan(Span sp) {
+  LIBCPP_ASSERT(noexcept(sp.back()));
+  return std::addressof(sp.back()) == sp.data() + sp.size() - 1;
 }
 
 template <typename Span>
-void testRuntimeSpan(Span sp)
-{
-    LIBCPP_ASSERT(noexcept(sp.back()));
-    assert(std::addressof(sp.back()) == sp.data() + sp.size() - 1);
+void testRuntimeSpan(Span sp) {
+  LIBCPP_ASSERT(noexcept(sp.back()));
+  assert(std::addressof(sp.back()) == sp.data() + sp.size() - 1);
 }
 
 template <typename Span>
-void testEmptySpan(Span sp)
-{
-    if (!sp.empty())
-        [[maybe_unused]] auto res = sp.back();
+void testEmptySpan(Span sp) {
+  if (!sp.empty()) [[maybe_unused]]
+    auto res = sp.back();
 }
 
-struct A{};
-constexpr int iArr1[] = { 0,  1,  2,  3,  4,  5,  6,  7,  8,  9};
-          int iArr2[] = {10, 11, 12, 13, 14, 15, 16, 17, 18, 19};
-
-int main(int, char**)
-{
-    static_assert(testConstexprSpan(std::span<const int>(iArr1, 1)), "");
-    static_assert(testConstexprSpan(std::span<const int>(iArr1, 2)), "");
-    static_assert(testConstexprSpan(std::span<const int>(iArr1, 3)), "");
-    static_assert(testConstexprSpan(std::span<const int>(iArr1, 4)), "");
-
-    static_assert(testConstexprSpan(std::span<const int, 1>(iArr1, 1)), "");
-    static_assert(testConstexprSpan(std::span<const int, 2>(iArr1, 2)), "");
-    static_assert(testConstexprSpan(std::span<const int, 3>(iArr1, 3)), "");
-    static_assert(testConstexprSpan(std::span<const int, 4>(iArr1, 4)), "");
+struct A {};
+constexpr int iArr1[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9};
+int iArr2[]           = {10, 11, 12, 13, 14, 15, 16, 17, 18, 19};
 
+int main(int, char**) {
+  static_assert(testConstexprSpan(std::span<const int>(iArr1, 1)), "");
+  static_assert(testConstexprSpan(std::span<const int>(iArr1, 2)), "");
+  static_assert(testConstexprSpan(std::span<const int>(iArr1, 3)), "");
+  static_assert(testConstexprSpan(std::span<const int>(iArr1, 4)), "");
 
-    testRuntimeSpan(std::span<int>(iArr2, 1));
-    testRuntimeSpan(std::span<int>(iArr2, 2));
-    testRuntimeSpan(std::span<int>(iArr2, 3));
-    testRuntimeSpan(std::span<int>(iArr2, 4));
+  static_assert(testConstexprSpan(std::span<const int, 1>(iArr1, 1)), "");
+  static_assert(testConstexprSpan(std::span<const int, 2>(iArr1, 2)), "");
+  static_assert(testConstexprSpan(std::span<const int, 3>(iArr1, 3)), "");
+  static_assert(testConstexprSpan(std::span<const int, 4>(iArr1, 4)), "");
 
+  testRuntimeSpan(std::span<int>(iArr2, 1));
+  testRuntimeSpan(std::span<int>(iArr2, 2));
+  testRuntimeSpan(std::span<int>(iArr2, 3));
+  testRuntimeSpan(std::span<int>(iArr2, 4));
 
-    testRuntimeSpan(std::span<int, 1>(iArr2, 1));
-    testRuntimeSpan(std::span<int, 2>(iArr2, 2));
-    testRuntimeSpan(std::span<int, 3>(iArr2, 3));
-    testRuntimeSpan(std::span<int, 4>(iArr2, 4));
+  testRuntimeSpan(std::span<int, 1>(iArr2, 1));
+  testRuntimeSpan(std::span<int, 2>(iArr2, 2));
+  testRuntimeSpan(std::span<int, 3>(iArr2, 3));
+  testRuntimeSpan(std::span<int, 4>(iArr2, 4));
 
-    std::string s;
-    testRuntimeSpan(std::span<std::string>   (&s, 1));
-    testRuntimeSpan(std::span<std::string, 1>(&s, 1));
+  std::string s;
+  testRuntimeSpan(std::span<std::string>(&s, 1));
+  testRuntimeSpan(std::span<std::string, 1>(&s, 1));
 
-    std::span<int, 0> sp;
-    testEmptySpan(sp);
+  std::span<int, 0> sp;
+  testEmptySpan(sp);
 
-    return 0;
+  return 0;
 }
diff --git a/libcxx/test/std/containers/views/views.span/span.elem/data.pass.cpp b/libcxx/test/std/containers/views/views.span/span.elem/data.pass.cpp
index 1bf63f95de987..a95b9772615ba 100644
--- a/libcxx/test/std/containers/views/views.span/span.elem/data.pass.cpp
+++ b/libcxx/test/std/containers/views/views.span/span.elem/data.pass.cpp
@@ -12,110 +12,100 @@
 // constexpr pointer data() const noexcept;
 //
 
-
 #include <span>
 #include <cassert>
 #include <string>
 
 #include "test_macros.h"
 
-
 template <typename Span>
-constexpr bool testConstexprSpan(Span sp, typename Span::pointer ptr)
-{
-    ASSERT_NOEXCEPT(sp.data());
-    return sp.data() == ptr;
+constexpr bool testConstexprSpan(Span sp, typename Span::pointer ptr) {
+  ASSERT_NOEXCEPT(sp.data());
+  return sp.data() == ptr;
 }
 
-
 template <typename Span>
-void testRuntimeSpan(Span sp, typename Span::pointer ptr)
-{
-    ASSERT_NOEXCEPT(sp.data());
-    assert(sp.data() == ptr);
+void testRuntimeSpan(Span sp, typename Span::pointer ptr) {
+  ASSERT_NOEXCEPT(sp.data());
+  assert(sp.data() == ptr);
 }
 
-struct A{};
-constexpr int iArr1[] = { 0,  1,  2,  3,  4,  5,  6,  7,  8,  9};
-          int iArr2[] = {10, 11, 12, 13, 14, 15, 16, 17, 18, 19};
-
-int main(int, char**)
-{
-
-//  dynamic size
-    static_assert(testConstexprSpan(std::span<int>(), nullptr),         "");
-    static_assert(testConstexprSpan(std::span<long>(), nullptr),        "");
-    static_assert(testConstexprSpan(std::span<double>(), nullptr),      "");
-    static_assert(testConstexprSpan(std::span<A>(), nullptr),           "");
-    static_assert(testConstexprSpan(std::span<std::string>(), nullptr), "");
-
-    static_assert(testConstexprSpan(std::span<const int>(iArr1, 1), iArr1), "");
-    static_assert(testConstexprSpan(std::span<const int>(iArr1, 2), iArr1), "");
-    static_assert(testConstexprSpan(std::span<const int>(iArr1, 3), iArr1), "");
-    static_assert(testConstexprSpan(std::span<const int>(iArr1, 4), iArr1), "");
-
-    static_assert(testConstexprSpan(std::span<const int>(iArr1 + 1, 1), iArr1 + 1), "");
-    static_assert(testConstexprSpan(std::span<const int>(iArr1 + 2, 2), iArr1 + 2), "");
-    static_assert(testConstexprSpan(std::span<const int>(iArr1 + 3, 3), iArr1 + 3), "");
-    static_assert(testConstexprSpan(std::span<const int>(iArr1 + 4, 4), iArr1 + 4), "");
-
-//  static size
-    static_assert(testConstexprSpan(std::span<int, 0>(), nullptr),         "");
-    static_assert(testConstexprSpan(std::span<long, 0>(), nullptr),        "");
-    static_assert(testConstexprSpan(std::span<double, 0>(), nullptr),      "");
-    static_assert(testConstexprSpan(std::span<A, 0>(), nullptr),           "");
-    static_assert(testConstexprSpan(std::span<std::string, 0>(), nullptr), "");
-
-    static_assert(testConstexprSpan(std::span<const int, 1>(iArr1, 1), iArr1), "");
-    static_assert(testConstexprSpan(std::span<const int, 2>(iArr1, 2), iArr1), "");
-    static_assert(testConstexprSpan(std::span<const int, 3>(iArr1, 3), iArr1), "");
-    static_assert(testConstexprSpan(std::span<const int, 4>(iArr1, 4), iArr1), "");
-
-    static_assert(testConstexprSpan(std::span<const int, 1>(iArr1 + 1, 1), iArr1 + 1), "");
-    static_assert(testConstexprSpan(std::span<const int, 2>(iArr1 + 2, 2), iArr1 + 2), "");
-    static_assert(testConstexprSpan(std::span<const int, 3>(iArr1 + 3, 3), iArr1 + 3), "");
-    static_assert(testConstexprSpan(std::span<const int, 4>(iArr1 + 4, 4), iArr1 + 4), "");
-
-
-//  dynamic size
-    testRuntimeSpan(std::span<int>(), nullptr);
-    testRuntimeSpan(std::span<long>(), nullptr);
-    testRuntimeSpan(std::span<double>(), nullptr);
-    testRuntimeSpan(std::span<A>(), nullptr);
-    testRuntimeSpan(std::span<std::string>(), nullptr);
-
-    testRuntimeSpan(std::span<int>(iArr2, 1), iArr2);
-    testRuntimeSpan(std::span<int>(iArr2, 2), iArr2);
-    testRuntimeSpan(std::span<int>(iArr2, 3), iArr2);
-    testRuntimeSpan(std::span<int>(iArr2, 4), iArr2);
-
-    testRuntimeSpan(std::span<int>(iArr2 + 1, 1), iArr2 + 1);
-    testRuntimeSpan(std::span<int>(iArr2 + 2, 2), iArr2 + 2);
-    testRuntimeSpan(std::span<int>(iArr2 + 3, 3), iArr2 + 3);
-    testRuntimeSpan(std::span<int>(iArr2 + 4, 4), iArr2 + 4);
-
-//  static size
-    testRuntimeSpan(std::span<int, 0>(), nullptr);
-    testRuntimeSpan(std::span<long, 0>(), nullptr);
-    testRuntimeSpan(std::span<double, 0>(), nullptr);
-    testRuntimeSpan(std::span<A, 0>(), nullptr);
-    testRuntimeSpan(std::span<std::string, 0>(), nullptr);
-
-    testRuntimeSpan(std::span<int, 1>(iArr2, 1), iArr2);
-    testRuntimeSpan(std::span<int, 2>(iArr2, 2), iArr2);
-    testRuntimeSpan(std::span<int, 3>(iArr2, 3), iArr2);
-    testRuntimeSpan(std::span<int, 4>(iArr2, 4), iArr2);
-
-    testRuntimeSpan(std::span<int, 1>(iArr2 + 1, 1), iArr2 + 1);
-    testRuntimeSpan(std::span<int, 2>(iArr2 + 2, 2), iArr2 + 2);
-    testRuntimeSpan(std::span<int, 3>(iArr2 + 3, 3), iArr2 + 3);
-    testRuntimeSpan(std::span<int, 4>(iArr2 + 4, 4), iArr2 + 4);
-
-
-    std::string s;
-    testRuntimeSpan(std::span<std::string>(&s, 1), &s);
-    testRuntimeSpan(std::span<std::string, 1>(&s, 1), &s);
-
+struct A {};
+constexpr int iArr1[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9};
+int iArr2[]           = {10, 11, 12, 13, 14, 15, 16, 17, 18, 19};
+
+int main(int, char**) {
+  //  dynamic size
+  static_assert(testConstexprSpan(std::span<int>(), nullptr), "");
+  static_assert(testConstexprSpan(std::span<long>(), nullptr), "");
+  static_assert(testConstexprSpan(std::span<double>(), nullptr), "");
+  static_assert(testConstexprSpan(std::span<A>(), nullptr), "");
+  static_assert(testConstexprSpan(std::span<std::string>(), nullptr), "");
+
+  static_assert(testConstexprSpan(std::span<const int>(iArr1, 1), iArr1), "");
+  static_assert(testConstexprSpan(std::span<const int>(iArr1, 2), iArr1), "");
+  static_assert(testConstexprSpan(std::span<const int>(iArr1, 3), iArr1), "");
+  static_assert(testConstexprSpan(std::span<const int>(iArr1, 4), iArr1), "");
+
+  static_assert(testConstexprSpan(std::span<const int>(iArr1 + 1, 1), iArr1 + 1), "");
+  static_assert(testConstexprSpan(std::span<const int>(iArr1 + 2, 2), iArr1 + 2), "");
+  static_assert(testConstexprSpan(std::span<const int>(iArr1 + 3, 3), iArr1 + 3), "");
+  static_assert(testConstexprSpan(std::span<const int>(iArr1 + 4, 4), iArr1 + 4), "");
+
+  //  static size
+  static_assert(testConstexprSpan(std::span<int, 0>(), nullptr), "");
+  static_assert(testConstexprSpan(std::span<long, 0>(), nullptr), "");
+  static_assert(testConstexprSpan(std::span<double, 0>(), nullptr), "");
+  static_assert(testConstexprSpan(std::span<A, 0>(), nullptr), "");
+  static_assert(testConstexprSpan(std::span<std::string, 0>(), nullptr), "");
+
+  static_assert(testConstexprSpan(std::span<const int, 1>(iArr1, 1), iArr1), "");
+  static_assert(testConstexprSpan(std::span<const int, 2>(iArr1, 2), iArr1), "");
+  static_assert(testConstexprSpan(std::span<const int, 3>(iArr1, 3), iArr1), "");
+  static_assert(testConstexprSpan(std::span<const int, 4>(iArr1, 4), iArr1), "");
+
+  static_assert(testConstexprSpan(std::span<const int, 1>(iArr1 + 1, 1), iArr1 + 1), "");
+  static_assert(testConstexprSpan(std::span<const int, 2>(iArr1 + 2, 2), iArr1 + 2), "");
+  static_assert(testConstexprSpan(std::span<const int, 3>(iArr1 + 3, 3), iArr1 + 3), "");
+  static_assert(testConstexprSpan(std::span<const int, 4>(iArr1 + 4, 4), iArr1 + 4), "");
+
+  //  dynamic size
+  testRuntimeSpan(std::span<int>(), nullptr);
+  testRuntimeSpan(std::span<long>(), nullptr);
+  testRuntimeSpan(std::span<double>(), nullptr);
+  testRuntimeSpan(std::span<A>(), nullptr);
+  testRuntimeSpan(std::span<std::string>(), nullptr);
+
+  testRuntimeSpan(std::span<int>(iArr2, 1), iArr2);
+  testRuntimeSpan(std::span<int>(iArr2, 2), iArr2);
+  testRuntimeSpan(std::span<int>(iArr2, 3), iArr2);
+  testRuntimeSpan(std::span<int>(iArr2, 4), iArr2);
+
+  testRuntimeSpan(std::span<int>(iArr2 + 1, 1), iArr2 + 1);
+  testRuntimeSpan(std::span<int>(iArr2 + 2, 2), iArr2 + 2);
+  testRuntimeSpan(std::span<int>(iArr2 + 3, 3), iArr2 + 3);
+  testRuntimeSpan(std::span<int>(iArr2 + 4, 4), iArr2 + 4);
+
+  //  static size
+  testRuntimeSpan(std::span<int, 0>(), nullptr);
+  testRuntimeSpan(std::span<long, 0>(), nullptr);
+  testRuntimeSpan(std::span<double, 0>(), nullptr);
+  testRuntimeSpan(std::span<A, 0>(), nullptr);
+  testRuntimeSpan(std::span<std::string, 0>(), nullptr);
+
+  testRuntimeSpan(std::span<int, 1>(iArr2, 1), iArr2);
+  testRuntimeSpan(std::span<int, 2>(iArr2, 2), iArr2);
+  testRuntimeSpan(std::span<int, 3>(iArr2, 3), iArr2);
+  testRuntimeSpan(std::span<int, 4>(iArr2, 4), iArr2);
+
+  testRuntimeSpan(std::span<int, 1>(iArr2 + 1, 1), iArr2 + 1);
+  testRuntimeSpan(std::span<int, 2>(iArr2 + 2, 2), iArr2 + 2);
+  testRuntimeSpan(std::span<int, 3>(iArr2 + 3, 3), iArr2 + 3);
+  testRuntimeSpan(std::span<int, 4>(iArr2 + 4, 4), iArr2 + 4);
+
+  std::string s;
+  testRuntimeSpan(std::span<std::string>(&s, 1), &s);
+  testRuntimeSpan(std::span<std::string, 1>(&s, 1), &s);
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/views/views.span/span.elem/front.pass.cpp b/libcxx/test/std/containers/views/views.span/span.elem/front.pass.cpp
index e1c9a0c984a23..0a5cb0641df21 100644
--- a/libcxx/test/std/containers/views/views.span/span.elem/front.pass.cpp
+++ b/libcxx/test/std/containers/views/views.span/span.elem/front.pass.cpp
@@ -14,70 +14,61 @@
 //   Effects: Equivalent to: return *data();
 //
 
-
 #include <span>
 #include <cassert>
 #include <string>
 
 #include "test_macros.h"
 
-
 template <typename Span>
-constexpr bool testConstexprSpan(Span sp)
-{
-    LIBCPP_ASSERT(noexcept(sp.front()));
-    return std::addressof(sp.front()) == sp.data();
+constexpr bool testConstexprSpan(Span sp) {
+  LIBCPP_ASSERT(noexcept(sp.front()));
+  return std::addressof(sp.front()) == sp.data();
 }
 
-
 template <typename Span>
-void testRuntimeSpan(Span sp)
-{
-    LIBCPP_ASSERT(noexcept(sp.front()));
-    assert(std::addressof(sp.front()) == sp.data());
+void testRuntimeSpan(Span sp) {
+  LIBCPP_ASSERT(noexcept(sp.front()));
+  assert(std::addressof(sp.front()) == sp.data());
 }
 
 template <typename Span>
-void testEmptySpan(Span sp)
-{
-    if (!sp.empty())
-        [[maybe_unused]] auto res = sp.front();
+void testEmptySpan(Span sp) {
+  if (!sp.empty()) [[maybe_unused]]
+    auto res = sp.front();
 }
 
-struct A{};
-constexpr int iArr1[] = { 0,  1,  2,  3,  4,  5,  6,  7,  8,  9};
-          int iArr2[] = {10, 11, 12, 13, 14, 15, 16, 17, 18, 19};
-
-int main(int, char**)
-{
-    static_assert(testConstexprSpan(std::span<const int>(iArr1, 1)), "");
-    static_assert(testConstexprSpan(std::span<const int>(iArr1, 2)), "");
-    static_assert(testConstexprSpan(std::span<const int>(iArr1, 3)), "");
-    static_assert(testConstexprSpan(std::span<const int>(iArr1, 4)), "");
-
-    static_assert(testConstexprSpan(std::span<const int, 1>(iArr1, 1)), "");
-    static_assert(testConstexprSpan(std::span<const int, 2>(iArr1, 2)), "");
-    static_assert(testConstexprSpan(std::span<const int, 3>(iArr1, 3)), "");
-    static_assert(testConstexprSpan(std::span<const int, 4>(iArr1, 4)), "");
+struct A {};
+constexpr int iArr1[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9};
+int iArr2[]           = {10, 11, 12, 13, 14, 15, 16, 17, 18, 19};
 
+int main(int, char**) {
+  static_assert(testConstexprSpan(std::span<const int>(iArr1, 1)), "");
+  static_assert(testConstexprSpan(std::span<const int>(iArr1, 2)), "");
+  static_assert(testConstexprSpan(std::span<const int>(iArr1, 3)), "");
+  static_assert(testConstexprSpan(std::span<const int>(iArr1, 4)), "");
 
-    testRuntimeSpan(std::span<int>(iArr2, 1));
-    testRuntimeSpan(std::span<int>(iArr2, 2));
-    testRuntimeSpan(std::span<int>(iArr2, 3));
-    testRuntimeSpan(std::span<int>(iArr2, 4));
+  static_assert(testConstexprSpan(std::span<const int, 1>(iArr1, 1)), "");
+  static_assert(testConstexprSpan(std::span<const int, 2>(iArr1, 2)), "");
+  static_assert(testConstexprSpan(std::span<const int, 3>(iArr1, 3)), "");
+  static_assert(testConstexprSpan(std::span<const int, 4>(iArr1, 4)), "");
 
+  testRuntimeSpan(std::span<int>(iArr2, 1));
+  testRuntimeSpan(std::span<int>(iArr2, 2));
+  testRuntimeSpan(std::span<int>(iArr2, 3));
+  testRuntimeSpan(std::span<int>(iArr2, 4));
 
-    testRuntimeSpan(std::span<int, 1>(iArr2, 1));
-    testRuntimeSpan(std::span<int, 2>(iArr2, 2));
-    testRuntimeSpan(std::span<int, 3>(iArr2, 3));
-    testRuntimeSpan(std::span<int, 4>(iArr2, 4));
+  testRuntimeSpan(std::span<int, 1>(iArr2, 1));
+  testRuntimeSpan(std::span<int, 2>(iArr2, 2));
+  testRuntimeSpan(std::span<int, 3>(iArr2, 3));
+  testRuntimeSpan(std::span<int, 4>(iArr2, 4));
 
-    std::string s;
-    testRuntimeSpan(std::span<std::string>   (&s, 1));
-    testRuntimeSpan(std::span<std::string, 1>(&s, 1));
+  std::string s;
+  testRuntimeSpan(std::span<std::string>(&s, 1));
+  testRuntimeSpan(std::span<std::string, 1>(&s, 1));
 
-    std::span<int, 0> sp;
-    testEmptySpan(sp);
+  std::span<int, 0> sp;
+  testEmptySpan(sp);
 
-    return 0;
+  return 0;
 }
diff --git a/libcxx/test/std/containers/views/views.span/span.elem/op_idx.pass.cpp b/libcxx/test/std/containers/views/views.span/span.elem/op_idx.pass.cpp
index b7f36c5758588..4ba2896553027 100644
--- a/libcxx/test/std/containers/views/views.span/span.elem/op_idx.pass.cpp
+++ b/libcxx/test/std/containers/views/views.span/span.elem/op_idx.pass.cpp
@@ -12,102 +12,93 @@
 // constexpr reference operator[](size_type idx) const;
 //
 
-
 #include <span>
 #include <cassert>
 #include <string>
 
 #include "test_macros.h"
 
-
 template <typename Span>
-constexpr bool testConstexprSpan(Span sp, std::size_t idx)
-{
-    LIBCPP_ASSERT(noexcept(sp[idx]));
+constexpr bool testConstexprSpan(Span sp, std::size_t idx) {
+  LIBCPP_ASSERT(noexcept(sp[idx]));
 
-    typename Span::reference r1 = sp[idx];
-    typename Span::reference r2 = *(sp.data() + idx);
-    return r1 == r2;
+  typename Span::reference r1 = sp[idx];
+  typename Span::reference r2 = *(sp.data() + idx);
+  return r1 == r2;
 }
 
-
 template <typename Span>
-void testRuntimeSpan(Span sp, std::size_t idx)
-{
-    LIBCPP_ASSERT(noexcept(sp[idx]));
+void testRuntimeSpan(Span sp, std::size_t idx) {
+  LIBCPP_ASSERT(noexcept(sp[idx]));
 
-    typename Span::reference r1 = sp[idx];
-    typename Span::reference r2 = *(sp.data() + idx);
-    assert(r1 == r2);
+  typename Span::reference r1 = sp[idx];
+  typename Span::reference r2 = *(sp.data() + idx);
+  assert(r1 == r2);
 }
 
-constexpr int iArr1[] = { 0,  1,  2,  3,  4,  5,  6,  7,  8,  9};
-          int iArr2[] = {10, 11, 12, 13, 14, 15, 16, 17, 18, 19};
-
-int main(int, char**)
-{
-    static_assert(testConstexprSpan(std::span<const int>(iArr1, 1), 0), "");
-
-    static_assert(testConstexprSpan(std::span<const int>(iArr1, 2), 0), "");
-    static_assert(testConstexprSpan(std::span<const int>(iArr1, 2), 1), "");
-
-    static_assert(testConstexprSpan(std::span<const int>(iArr1, 3), 0), "");
-    static_assert(testConstexprSpan(std::span<const int>(iArr1, 3), 1), "");
-    static_assert(testConstexprSpan(std::span<const int>(iArr1, 3), 2), "");
+constexpr int iArr1[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9};
+int iArr2[]           = {10, 11, 12, 13, 14, 15, 16, 17, 18, 19};
 
-    static_assert(testConstexprSpan(std::span<const int>(iArr1, 4), 0), "");
-    static_assert(testConstexprSpan(std::span<const int>(iArr1, 4), 1), "");
-    static_assert(testConstexprSpan(std::span<const int>(iArr1, 4), 2), "");
-    static_assert(testConstexprSpan(std::span<const int>(iArr1, 4), 3), "");
+int main(int, char**) {
+  static_assert(testConstexprSpan(std::span<const int>(iArr1, 1), 0), "");
 
+  static_assert(testConstexprSpan(std::span<const int>(iArr1, 2), 0), "");
+  static_assert(testConstexprSpan(std::span<const int>(iArr1, 2), 1), "");
 
-    static_assert(testConstexprSpan(std::span<const int, 1>(iArr1, 1), 0), "");
+  static_assert(testConstexprSpan(std::span<const int>(iArr1, 3), 0), "");
+  static_assert(testConstexprSpan(std::span<const int>(iArr1, 3), 1), "");
+  static_assert(testConstexprSpan(std::span<const int>(iArr1, 3), 2), "");
 
-    static_assert(testConstexprSpan(std::span<const int, 2>(iArr1, 2), 0), "");
-    static_assert(testConstexprSpan(std::span<const int, 2>(iArr1, 2), 1), "");
+  static_assert(testConstexprSpan(std::span<const int>(iArr1, 4), 0), "");
+  static_assert(testConstexprSpan(std::span<const int>(iArr1, 4), 1), "");
+  static_assert(testConstexprSpan(std::span<const int>(iArr1, 4), 2), "");
+  static_assert(testConstexprSpan(std::span<const int>(iArr1, 4), 3), "");
 
-    static_assert(testConstexprSpan(std::span<const int, 3>(iArr1, 3), 0), "");
-    static_assert(testConstexprSpan(std::span<const int, 3>(iArr1, 3), 1), "");
-    static_assert(testConstexprSpan(std::span<const int, 3>(iArr1, 3), 2), "");
+  static_assert(testConstexprSpan(std::span<const int, 1>(iArr1, 1), 0), "");
 
-    static_assert(testConstexprSpan(std::span<const int, 4>(iArr1, 4), 0), "");
-    static_assert(testConstexprSpan(std::span<const int, 4>(iArr1, 4), 1), "");
-    static_assert(testConstexprSpan(std::span<const int, 4>(iArr1, 4), 2), "");
-    static_assert(testConstexprSpan(std::span<const int, 4>(iArr1, 4), 3), "");
+  static_assert(testConstexprSpan(std::span<const int, 2>(iArr1, 2), 0), "");
+  static_assert(testConstexprSpan(std::span<const int, 2>(iArr1, 2), 1), "");
 
+  static_assert(testConstexprSpan(std::span<const int, 3>(iArr1, 3), 0), "");
+  static_assert(testConstexprSpan(std::span<const int, 3>(iArr1, 3), 1), "");
+  static_assert(testConstexprSpan(std::span<const int, 3>(iArr1, 3), 2), "");
 
-    testRuntimeSpan(std::span<int>(iArr2, 1), 0);
+  static_assert(testConstexprSpan(std::span<const int, 4>(iArr1, 4), 0), "");
+  static_assert(testConstexprSpan(std::span<const int, 4>(iArr1, 4), 1), "");
+  static_assert(testConstexprSpan(std::span<const int, 4>(iArr1, 4), 2), "");
+  static_assert(testConstexprSpan(std::span<const int, 4>(iArr1, 4), 3), "");
 
-    testRuntimeSpan(std::span<int>(iArr2, 2), 0);
-    testRuntimeSpan(std::span<int>(iArr2, 2), 1);
+  testRuntimeSpan(std::span<int>(iArr2, 1), 0);
 
-    testRuntimeSpan(std::span<int>(iArr2, 3), 0);
-    testRuntimeSpan(std::span<int>(iArr2, 3), 1);
-    testRuntimeSpan(std::span<int>(iArr2, 3), 2);
+  testRuntimeSpan(std::span<int>(iArr2, 2), 0);
+  testRuntimeSpan(std::span<int>(iArr2, 2), 1);
 
-    testRuntimeSpan(std::span<int>(iArr2, 4), 0);
-    testRuntimeSpan(std::span<int>(iArr2, 4), 1);
-    testRuntimeSpan(std::span<int>(iArr2, 4), 2);
-    testRuntimeSpan(std::span<int>(iArr2, 4), 3);
+  testRuntimeSpan(std::span<int>(iArr2, 3), 0);
+  testRuntimeSpan(std::span<int>(iArr2, 3), 1);
+  testRuntimeSpan(std::span<int>(iArr2, 3), 2);
 
+  testRuntimeSpan(std::span<int>(iArr2, 4), 0);
+  testRuntimeSpan(std::span<int>(iArr2, 4), 1);
+  testRuntimeSpan(std::span<int>(iArr2, 4), 2);
+  testRuntimeSpan(std::span<int>(iArr2, 4), 3);
 
-    testRuntimeSpan(std::span<int, 1>(iArr2, 1), 0);
+  testRuntimeSpan(std::span<int, 1>(iArr2, 1), 0);
 
-    testRuntimeSpan(std::span<int, 2>(iArr2, 2), 0);
-    testRuntimeSpan(std::span<int, 2>(iArr2, 2), 1);
+  testRuntimeSpan(std::span<int, 2>(iArr2, 2), 0);
+  testRuntimeSpan(std::span<int, 2>(iArr2, 2), 1);
 
-    testRuntimeSpan(std::span<int, 3>(iArr2, 3), 0);
-    testRuntimeSpan(std::span<int, 3>(iArr2, 3), 1);
-    testRuntimeSpan(std::span<int, 3>(iArr2, 3), 2);
+  testRuntimeSpan(std::span<int, 3>(iArr2, 3), 0);
+  testRuntimeSpan(std::span<int, 3>(iArr2, 3), 1);
+  testRuntimeSpan(std::span<int, 3>(iArr2, 3), 2);
 
-    testRuntimeSpan(std::span<int, 4>(iArr2, 4), 0);
-    testRuntimeSpan(std::span<int, 4>(iArr2, 4), 1);
-    testRuntimeSpan(std::span<int, 4>(iArr2, 4), 2);
-    testRuntimeSpan(std::span<int, 4>(iArr2, 4), 3);
+  testRuntimeSpan(std::span<int, 4>(iArr2, 4), 0);
+  testRuntimeSpan(std::span<int, 4>(iArr2, 4), 1);
+  testRuntimeSpan(std::span<int, 4>(iArr2, 4), 2);
+  testRuntimeSpan(std::span<int, 4>(iArr2, 4), 3);
 
-    std::string s;
-    testRuntimeSpan(std::span<std::string>   (&s, 1), 0);
-    testRuntimeSpan(std::span<std::string, 1>(&s, 1), 0);
+  std::string s;
+  testRuntimeSpan(std::span<std::string>(&s, 1), 0);
+  testRuntimeSpan(std::span<std::string, 1>(&s, 1), 0);
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/views/views.span/span.iterators/begin.pass.cpp b/libcxx/test/std/containers/views/views.span/span.iterators/begin.pass.cpp
index f6b24017ffc1f..4792b756e42dc 100644
--- a/libcxx/test/std/containers/views/views.span/span.iterators/begin.pass.cpp
+++ b/libcxx/test/std/containers/views/views.span/span.iterators/begin.pass.cpp
@@ -19,89 +19,77 @@
 #include "test_macros.h"
 
 template <class Span>
-constexpr bool testConstexprSpan(Span s)
-{
-    bool ret = true;
-    typename Span::iterator b = s.begin();
-
-    if (s.empty())
-    {
-        ret = ret && (b == s.end());
-    }
-    else
-    {
-        ret = ret && ( *b ==  s[0]);
-        ret = ret && (&*b == &s[0]);
-    }
-    return ret;
+constexpr bool testConstexprSpan(Span s) {
+  bool ret                  = true;
+  typename Span::iterator b = s.begin();
+
+  if (s.empty()) {
+    ret = ret && (b == s.end());
+  } else {
+    ret = ret && (*b == s[0]);
+    ret = ret && (&*b == &s[0]);
+  }
+  return ret;
 }
 
-
 template <class Span>
-void testRuntimeSpan(Span s)
-{
-    typename Span::iterator b = s.begin();
-
-    if (s.empty())
-    {
-        assert(b == s.end());
-    }
-    else
-    {
-        assert( *b ==  s[0]);
-        assert(&*b == &s[0]);
-    }
+void testRuntimeSpan(Span s) {
+  typename Span::iterator b = s.begin();
+
+  if (s.empty()) {
+    assert(b == s.end());
+  } else {
+    assert(*b == s[0]);
+    assert(&*b == &s[0]);
+  }
 }
 
-struct A{};
-bool operator==(A, A) {return true;}
-
-constexpr int iArr1[] = { 0,  1,  2,  3,  4,  5,  6,  7,  8,  9};
-          int iArr2[] = {10, 11, 12, 13, 14, 15, 16, 17, 18, 19};
-
-
-int main(int, char**)
-{
-    static_assert(testConstexprSpan(std::span<int>()),            "");
-    static_assert(testConstexprSpan(std::span<long>()),           "");
-    static_assert(testConstexprSpan(std::span<double>()),         "");
-    static_assert(testConstexprSpan(std::span<A>()),              "");
-    static_assert(testConstexprSpan(std::span<std::string>()),    "");
-
-    static_assert(testConstexprSpan(std::span<int, 0>()),         "");
-    static_assert(testConstexprSpan(std::span<long, 0>()),        "");
-    static_assert(testConstexprSpan(std::span<double, 0>()),      "");
-    static_assert(testConstexprSpan(std::span<A, 0>()),           "");
-    static_assert(testConstexprSpan(std::span<std::string, 0>()), "");
-
-    static_assert(testConstexprSpan(std::span<const int>(iArr1, 1)),    "");
-    static_assert(testConstexprSpan(std::span<const int>(iArr1, 2)),    "");
-    static_assert(testConstexprSpan(std::span<const int>(iArr1, 3)),    "");
-    static_assert(testConstexprSpan(std::span<const int>(iArr1, 4)),    "");
-    static_assert(testConstexprSpan(std::span<const int>(iArr1, 5)),    "");
-
-
-    testRuntimeSpan(std::span<int>        ());
-    testRuntimeSpan(std::span<long>       ());
-    testRuntimeSpan(std::span<double>     ());
-    testRuntimeSpan(std::span<A>          ());
-    testRuntimeSpan(std::span<std::string>());
-
-    testRuntimeSpan(std::span<int, 0>        ());
-    testRuntimeSpan(std::span<long, 0>       ());
-    testRuntimeSpan(std::span<double, 0>     ());
-    testRuntimeSpan(std::span<A, 0>          ());
-    testRuntimeSpan(std::span<std::string, 0>());
-
-    testRuntimeSpan(std::span<int>(iArr2, 1));
-    testRuntimeSpan(std::span<int>(iArr2, 2));
-    testRuntimeSpan(std::span<int>(iArr2, 3));
-    testRuntimeSpan(std::span<int>(iArr2, 4));
-    testRuntimeSpan(std::span<int>(iArr2, 5));
-
-    std::string s;
-    testRuntimeSpan(std::span<std::string>(&s, (std::size_t) 0));
-    testRuntimeSpan(std::span<std::string>(&s, 1));
+struct A {};
+bool operator==(A, A) { return true; }
+
+constexpr int iArr1[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9};
+int iArr2[]           = {10, 11, 12, 13, 14, 15, 16, 17, 18, 19};
+
+int main(int, char**) {
+  static_assert(testConstexprSpan(std::span<int>()), "");
+  static_assert(testConstexprSpan(std::span<long>()), "");
+  static_assert(testConstexprSpan(std::span<double>()), "");
+  static_assert(testConstexprSpan(std::span<A>()), "");
+  static_assert(testConstexprSpan(std::span<std::string>()), "");
+
+  static_assert(testConstexprSpan(std::span<int, 0>()), "");
+  static_assert(testConstexprSpan(std::span<long, 0>()), "");
+  static_assert(testConstexprSpan(std::span<double, 0>()), "");
+  static_assert(testConstexprSpan(std::span<A, 0>()), "");
+  static_assert(testConstexprSpan(std::span<std::string, 0>()), "");
+
+  static_assert(testConstexprSpan(std::span<const int>(iArr1, 1)), "");
+  static_assert(testConstexprSpan(std::span<const int>(iArr1, 2)), "");
+  static_assert(testConstexprSpan(std::span<const int>(iArr1, 3)), "");
+  static_assert(testConstexprSpan(std::span<const int>(iArr1, 4)), "");
+  static_assert(testConstexprSpan(std::span<const int>(iArr1, 5)), "");
+
+  testRuntimeSpan(std::span<int>());
+  testRuntimeSpan(std::span<long>());
+  testRuntimeSpan(std::span<double>());
+  testRuntimeSpan(std::span<A>());
+  testRuntimeSpan(std::span<std::string>());
+
+  testRuntimeSpan(std::span<int, 0>());
+  testRuntimeSpan(std::span<long, 0>());
+  testRuntimeSpan(std::span<double, 0>());
+  testRuntimeSpan(std::span<A, 0>());
+  testRuntimeSpan(std::span<std::string, 0>());
+
+  testRuntimeSpan(std::span<int>(iArr2, 1));
+  testRuntimeSpan(std::span<int>(iArr2, 2));
+  testRuntimeSpan(std::span<int>(iArr2, 3));
+  testRuntimeSpan(std::span<int>(iArr2, 4));
+  testRuntimeSpan(std::span<int>(iArr2, 5));
+
+  std::string s;
+  testRuntimeSpan(std::span<std::string>(&s, (std::size_t)0));
+  testRuntimeSpan(std::span<std::string>(&s, 1));
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/views/views.span/span.iterators/end.pass.cpp b/libcxx/test/std/containers/views/views.span/span.iterators/end.pass.cpp
index d6aaf74d1058c..0daa55bf0542c 100644
--- a/libcxx/test/std/containers/views/views.span/span.iterators/end.pass.cpp
+++ b/libcxx/test/std/containers/views/views.span/span.iterators/end.pass.cpp
@@ -19,93 +19,81 @@
 #include "test_macros.h"
 
 template <class Span>
-constexpr bool testConstexprSpan(Span s)
-{
-    bool ret = true;
-    typename Span::iterator e = s.end();
-    if (s.empty())
-    {
-        ret = ret &&  (e == s.begin());
-    }
-    else
-    {
-        typename Span::const_pointer last = &*(s.begin() + s.size() - 1);
-        ret = ret &&  (e != s.begin());
-        ret = ret &&  (&*( e-1) == last);
-    }
-
-    ret = ret &&  (static_cast<std::size_t>(e - s.begin()) == s.size());
-    return ret;
+constexpr bool testConstexprSpan(Span s) {
+  bool ret                  = true;
+  typename Span::iterator e = s.end();
+  if (s.empty()) {
+    ret = ret && (e == s.begin());
+  } else {
+    typename Span::const_pointer last = &*(s.begin() + s.size() - 1);
+    ret                               = ret && (e != s.begin());
+    ret                               = ret && (&*(e - 1) == last);
+  }
+
+  ret = ret && (static_cast<std::size_t>(e - s.begin()) == s.size());
+  return ret;
 }
 
 template <class Span>
-void testRuntimeSpan(Span s)
-{
-    typename Span::iterator e = s.end();
-    if (s.empty())
-    {
-        assert(e == s.begin());
-    }
-    else
-    {
-        typename Span::const_pointer last = &*(s.begin() + s.size() - 1);
-        assert(e != s.begin());
-        assert(&*( e-1) == last);
-    }
-
-    assert(static_cast<std::size_t>(e - s.begin()) == s.size());
+void testRuntimeSpan(Span s) {
+  typename Span::iterator e = s.end();
+  if (s.empty()) {
+    assert(e == s.begin());
+  } else {
+    typename Span::const_pointer last = &*(s.begin() + s.size() - 1);
+    assert(e != s.begin());
+    assert(&*(e - 1) == last);
+  }
+
+  assert(static_cast<std::size_t>(e - s.begin()) == s.size());
 }
 
-
-struct A{};
-bool operator==(A, A) {return true;}
-
-constexpr int iArr1[] = { 0,  1,  2,  3,  4,  5,  6,  7,  8,  9};
-          int iArr2[] = {10, 11, 12, 13, 14, 15, 16, 17, 18, 19};
-
-
-int main(int, char**)
-{
-    static_assert(testConstexprSpan(std::span<int>()),            "");
-    static_assert(testConstexprSpan(std::span<long>()),           "");
-    static_assert(testConstexprSpan(std::span<double>()),         "");
-    static_assert(testConstexprSpan(std::span<A>()),              "");
-    static_assert(testConstexprSpan(std::span<std::string>()),    "");
-
-    static_assert(testConstexprSpan(std::span<int, 0>()),         "");
-    static_assert(testConstexprSpan(std::span<long, 0>()),        "");
-    static_assert(testConstexprSpan(std::span<double, 0>()),      "");
-    static_assert(testConstexprSpan(std::span<A, 0>()),           "");
-    static_assert(testConstexprSpan(std::span<std::string, 0>()), "");
-
-    static_assert(testConstexprSpan(std::span<const int>(iArr1, 1)),    "");
-    static_assert(testConstexprSpan(std::span<const int>(iArr1, 2)),    "");
-    static_assert(testConstexprSpan(std::span<const int>(iArr1, 3)),    "");
-    static_assert(testConstexprSpan(std::span<const int>(iArr1, 4)),    "");
-    static_assert(testConstexprSpan(std::span<const int>(iArr1, 5)),    "");
-
-
-    testRuntimeSpan(std::span<int>        ());
-    testRuntimeSpan(std::span<long>       ());
-    testRuntimeSpan(std::span<double>     ());
-    testRuntimeSpan(std::span<A>          ());
-    testRuntimeSpan(std::span<std::string>());
-
-    testRuntimeSpan(std::span<int, 0>        ());
-    testRuntimeSpan(std::span<long, 0>       ());
-    testRuntimeSpan(std::span<double, 0>     ());
-    testRuntimeSpan(std::span<A, 0>          ());
-    testRuntimeSpan(std::span<std::string, 0>());
-
-    testRuntimeSpan(std::span<int>(iArr2, 1));
-    testRuntimeSpan(std::span<int>(iArr2, 2));
-    testRuntimeSpan(std::span<int>(iArr2, 3));
-    testRuntimeSpan(std::span<int>(iArr2, 4));
-    testRuntimeSpan(std::span<int>(iArr2, 5));
-
-    std::string s;
-    testRuntimeSpan(std::span<std::string>(&s, (std::size_t) 0));
-    testRuntimeSpan(std::span<std::string>(&s, 1));
+struct A {};
+bool operator==(A, A) { return true; }
+
+constexpr int iArr1[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9};
+int iArr2[]           = {10, 11, 12, 13, 14, 15, 16, 17, 18, 19};
+
+int main(int, char**) {
+  static_assert(testConstexprSpan(std::span<int>()), "");
+  static_assert(testConstexprSpan(std::span<long>()), "");
+  static_assert(testConstexprSpan(std::span<double>()), "");
+  static_assert(testConstexprSpan(std::span<A>()), "");
+  static_assert(testConstexprSpan(std::span<std::string>()), "");
+
+  static_assert(testConstexprSpan(std::span<int, 0>()), "");
+  static_assert(testConstexprSpan(std::span<long, 0>()), "");
+  static_assert(testConstexprSpan(std::span<double, 0>()), "");
+  static_assert(testConstexprSpan(std::span<A, 0>()), "");
+  static_assert(testConstexprSpan(std::span<std::string, 0>()), "");
+
+  static_assert(testConstexprSpan(std::span<const int>(iArr1, 1)), "");
+  static_assert(testConstexprSpan(std::span<const int>(iArr1, 2)), "");
+  static_assert(testConstexprSpan(std::span<const int>(iArr1, 3)), "");
+  static_assert(testConstexprSpan(std::span<const int>(iArr1, 4)), "");
+  static_assert(testConstexprSpan(std::span<const int>(iArr1, 5)), "");
+
+  testRuntimeSpan(std::span<int>());
+  testRuntimeSpan(std::span<long>());
+  testRuntimeSpan(std::span<double>());
+  testRuntimeSpan(std::span<A>());
+  testRuntimeSpan(std::span<std::string>());
+
+  testRuntimeSpan(std::span<int, 0>());
+  testRuntimeSpan(std::span<long, 0>());
+  testRuntimeSpan(std::span<double, 0>());
+  testRuntimeSpan(std::span<A, 0>());
+  testRuntimeSpan(std::span<std::string, 0>());
+
+  testRuntimeSpan(std::span<int>(iArr2, 1));
+  testRuntimeSpan(std::span<int>(iArr2, 2));
+  testRuntimeSpan(std::span<int>(iArr2, 3));
+  testRuntimeSpan(std::span<int>(iArr2, 4));
+  testRuntimeSpan(std::span<int>(iArr2, 5));
+
+  std::string s;
+  testRuntimeSpan(std::span<std::string>(&s, (std::size_t)0));
+  testRuntimeSpan(std::span<std::string>(&s, 1));
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/views/views.span/span.iterators/iterator_concept_conformance.compile.pass.cpp b/libcxx/test/std/containers/views/views.span/span.iterators/iterator_concept_conformance.compile.pass.cpp
index 1d9ea600bd2a0..e07be3efb4a36 100644
--- a/libcxx/test/std/containers/views/views.span/span.iterators/iterator_concept_conformance.compile.pass.cpp
+++ b/libcxx/test/std/containers/views/views.span/span.iterators/iterator_concept_conformance.compile.pass.cpp
@@ -15,9 +15,9 @@
 #include <iterator>
 #include "test_macros.h"
 
-using iterator = std::span<int>::iterator;
+using iterator         = std::span<int>::iterator;
 using reverse_iterator = std::span<int>::reverse_iterator;
-using value_type = int;
+using value_type       = int;
 
 static_assert(std::contiguous_iterator<iterator>);
 LIBCPP_STATIC_ASSERT(std::__has_random_access_iterator_category<iterator>::value);
diff --git a/libcxx/test/std/containers/views/views.span/span.iterators/rbegin.pass.cpp b/libcxx/test/std/containers/views/views.span/span.iterators/rbegin.pass.cpp
index 26e4389840857..1030dec7f6e40 100644
--- a/libcxx/test/std/containers/views/views.span/span.iterators/rbegin.pass.cpp
+++ b/libcxx/test/std/containers/views/views.span/span.iterators/rbegin.pass.cpp
@@ -19,90 +19,77 @@
 #include "test_macros.h"
 
 template <class Span>
-constexpr bool testConstexprSpan(Span s)
-{
-    bool ret = true;
-    typename Span::reverse_iterator b = s.rbegin();
-    if (s.empty())
-    {
-        ret = ret &&  ( b ==  s.rend());
-    }
-    else
-    {
-        const typename Span::size_type last = s.size() - 1;
-        ret = ret && ( *b ==  s[last]);
-        ret = ret && (&*b == &s[last]);
-    }
-    return ret;
+constexpr bool testConstexprSpan(Span s) {
+  bool ret                          = true;
+  typename Span::reverse_iterator b = s.rbegin();
+  if (s.empty()) {
+    ret = ret && (b == s.rend());
+  } else {
+    const typename Span::size_type last = s.size() - 1;
+    ret                                 = ret && (*b == s[last]);
+    ret                                 = ret && (&*b == &s[last]);
+  }
+  return ret;
 }
 
-
 template <class Span>
-void testRuntimeSpan(Span s)
-{
-    typename Span::reverse_iterator b = s.rbegin();
-    if (s.empty())
-    {
-        assert(b == s.rend());
-    }
-    else
-    {
-        const typename Span::size_type last = s.size() - 1;
-        assert( *b ==  s[last]);
-        assert(&*b == &s[last]);
-    }
+void testRuntimeSpan(Span s) {
+  typename Span::reverse_iterator b = s.rbegin();
+  if (s.empty()) {
+    assert(b == s.rend());
+  } else {
+    const typename Span::size_type last = s.size() - 1;
+    assert(*b == s[last]);
+    assert(&*b == &s[last]);
+  }
 }
 
-
-struct A{};
-bool operator==(A, A) {return true;}
-
-constexpr int iArr1[] = { 0,  1,  2,  3,  4,  5,  6,  7,  8,  9};
-          int iArr2[] = {10, 11, 12, 13, 14, 15, 16, 17, 18, 19};
-
-
-int main(int, char**)
-{
-    static_assert(testConstexprSpan(std::span<int>()),            "");
-    static_assert(testConstexprSpan(std::span<long>()),           "");
-    static_assert(testConstexprSpan(std::span<double>()),         "");
-    static_assert(testConstexprSpan(std::span<A>()),              "");
-    static_assert(testConstexprSpan(std::span<std::string>()),    "");
-
-    static_assert(testConstexprSpan(std::span<int, 0>()),         "");
-    static_assert(testConstexprSpan(std::span<long, 0>()),        "");
-    static_assert(testConstexprSpan(std::span<double, 0>()),      "");
-    static_assert(testConstexprSpan(std::span<A, 0>()),           "");
-    static_assert(testConstexprSpan(std::span<std::string, 0>()), "");
-
-    static_assert(testConstexprSpan(std::span<const int>(iArr1, 1)),    "");
-    static_assert(testConstexprSpan(std::span<const int>(iArr1, 2)),    "");
-    static_assert(testConstexprSpan(std::span<const int>(iArr1, 3)),    "");
-    static_assert(testConstexprSpan(std::span<const int>(iArr1, 4)),    "");
-    static_assert(testConstexprSpan(std::span<const int>(iArr1, 5)),    "");
-
-
-    testRuntimeSpan(std::span<int>        ());
-    testRuntimeSpan(std::span<long>       ());
-    testRuntimeSpan(std::span<double>     ());
-    testRuntimeSpan(std::span<A>          ());
-    testRuntimeSpan(std::span<std::string>());
-
-    testRuntimeSpan(std::span<int, 0>        ());
-    testRuntimeSpan(std::span<long, 0>       ());
-    testRuntimeSpan(std::span<double, 0>     ());
-    testRuntimeSpan(std::span<A, 0>          ());
-    testRuntimeSpan(std::span<std::string, 0>());
-
-    testRuntimeSpan(std::span<int>(iArr2, 1));
-    testRuntimeSpan(std::span<int>(iArr2, 2));
-    testRuntimeSpan(std::span<int>(iArr2, 3));
-    testRuntimeSpan(std::span<int>(iArr2, 4));
-    testRuntimeSpan(std::span<int>(iArr2, 5));
-
-    std::string s;
-    testRuntimeSpan(std::span<std::string>(&s, static_cast<std::size_t>(0)));
-    testRuntimeSpan(std::span<std::string>(&s, 1));
+struct A {};
+bool operator==(A, A) { return true; }
+
+constexpr int iArr1[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9};
+int iArr2[]           = {10, 11, 12, 13, 14, 15, 16, 17, 18, 19};
+
+int main(int, char**) {
+  static_assert(testConstexprSpan(std::span<int>()), "");
+  static_assert(testConstexprSpan(std::span<long>()), "");
+  static_assert(testConstexprSpan(std::span<double>()), "");
+  static_assert(testConstexprSpan(std::span<A>()), "");
+  static_assert(testConstexprSpan(std::span<std::string>()), "");
+
+  static_assert(testConstexprSpan(std::span<int, 0>()), "");
+  static_assert(testConstexprSpan(std::span<long, 0>()), "");
+  static_assert(testConstexprSpan(std::span<double, 0>()), "");
+  static_assert(testConstexprSpan(std::span<A, 0>()), "");
+  static_assert(testConstexprSpan(std::span<std::string, 0>()), "");
+
+  static_assert(testConstexprSpan(std::span<const int>(iArr1, 1)), "");
+  static_assert(testConstexprSpan(std::span<const int>(iArr1, 2)), "");
+  static_assert(testConstexprSpan(std::span<const int>(iArr1, 3)), "");
+  static_assert(testConstexprSpan(std::span<const int>(iArr1, 4)), "");
+  static_assert(testConstexprSpan(std::span<const int>(iArr1, 5)), "");
+
+  testRuntimeSpan(std::span<int>());
+  testRuntimeSpan(std::span<long>());
+  testRuntimeSpan(std::span<double>());
+  testRuntimeSpan(std::span<A>());
+  testRuntimeSpan(std::span<std::string>());
+
+  testRuntimeSpan(std::span<int, 0>());
+  testRuntimeSpan(std::span<long, 0>());
+  testRuntimeSpan(std::span<double, 0>());
+  testRuntimeSpan(std::span<A, 0>());
+  testRuntimeSpan(std::span<std::string, 0>());
+
+  testRuntimeSpan(std::span<int>(iArr2, 1));
+  testRuntimeSpan(std::span<int>(iArr2, 2));
+  testRuntimeSpan(std::span<int>(iArr2, 3));
+  testRuntimeSpan(std::span<int>(iArr2, 4));
+  testRuntimeSpan(std::span<int>(iArr2, 5));
+
+  std::string s;
+  testRuntimeSpan(std::span<std::string>(&s, static_cast<std::size_t>(0)));
+  testRuntimeSpan(std::span<std::string>(&s, 1));
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/views/views.span/span.iterators/rend.pass.cpp b/libcxx/test/std/containers/views/views.span/span.iterators/rend.pass.cpp
index ffec15c6b2492..ca2b781aaf614 100644
--- a/libcxx/test/std/containers/views/views.span/span.iterators/rend.pass.cpp
+++ b/libcxx/test/std/containers/views/views.span/span.iterators/rend.pass.cpp
@@ -19,89 +19,77 @@
 #include "test_macros.h"
 
 template <class Span>
-constexpr bool testConstexprSpan(Span s)
-{
-    bool ret = true;
-    typename Span::reverse_iterator e = s.rend();
-    if (s.empty())
-    {
-        ret = ret &&  (e == s.rbegin());
-    }
-    else
-    {
-        ret = ret &&  (e != s.rbegin());
-    }
-
-    ret = ret &&  (static_cast<std::size_t>(e - s.rbegin()) == s.size());
-    return ret;
+constexpr bool testConstexprSpan(Span s) {
+  bool ret                          = true;
+  typename Span::reverse_iterator e = s.rend();
+  if (s.empty()) {
+    ret = ret && (e == s.rbegin());
+  } else {
+    ret = ret && (e != s.rbegin());
+  }
+
+  ret = ret && (static_cast<std::size_t>(e - s.rbegin()) == s.size());
+  return ret;
 }
 
 template <class Span>
-void testRuntimeSpan(Span s)
-{
-    typename Span::reverse_iterator e = s.rend();
-    if (s.empty())
-    {
-        assert(e == s.rbegin());
-    }
-    else
-    {
-        assert(e != s.rbegin());
-    }
-
-    assert(static_cast<std::size_t>(e - s.rbegin()) == s.size());
+void testRuntimeSpan(Span s) {
+  typename Span::reverse_iterator e = s.rend();
+  if (s.empty()) {
+    assert(e == s.rbegin());
+  } else {
+    assert(e != s.rbegin());
+  }
+
+  assert(static_cast<std::size_t>(e - s.rbegin()) == s.size());
 }
 
-
-struct A{};
-bool operator==(A, A) {return true;}
-
-constexpr int iArr1[] = { 0,  1,  2,  3,  4,  5,  6,  7,  8,  9};
-          int iArr2[] = {10, 11, 12, 13, 14, 15, 16, 17, 18, 19};
-
-
-int main(int, char**)
-{
-    static_assert(testConstexprSpan(std::span<int>()),            "");
-    static_assert(testConstexprSpan(std::span<long>()),           "");
-    static_assert(testConstexprSpan(std::span<double>()),         "");
-    static_assert(testConstexprSpan(std::span<A>()),              "");
-    static_assert(testConstexprSpan(std::span<std::string>()),    "");
-
-    static_assert(testConstexprSpan(std::span<int, 0>()),         "");
-    static_assert(testConstexprSpan(std::span<long, 0>()),        "");
-    static_assert(testConstexprSpan(std::span<double, 0>()),      "");
-    static_assert(testConstexprSpan(std::span<A, 0>()),           "");
-    static_assert(testConstexprSpan(std::span<std::string, 0>()), "");
-
-    static_assert(testConstexprSpan(std::span<const int>(iArr1, 1)),    "");
-    static_assert(testConstexprSpan(std::span<const int>(iArr1, 2)),    "");
-    static_assert(testConstexprSpan(std::span<const int>(iArr1, 3)),    "");
-    static_assert(testConstexprSpan(std::span<const int>(iArr1, 4)),    "");
-    static_assert(testConstexprSpan(std::span<const int>(iArr1, 5)),    "");
-
-
-    testRuntimeSpan(std::span<int>        ());
-    testRuntimeSpan(std::span<long>       ());
-    testRuntimeSpan(std::span<double>     ());
-    testRuntimeSpan(std::span<A>          ());
-    testRuntimeSpan(std::span<std::string>());
-
-    testRuntimeSpan(std::span<int, 0>        ());
-    testRuntimeSpan(std::span<long, 0>       ());
-    testRuntimeSpan(std::span<double, 0>     ());
-    testRuntimeSpan(std::span<A, 0>          ());
-    testRuntimeSpan(std::span<std::string, 0>());
-
-    testRuntimeSpan(std::span<int>(iArr2, 1));
-    testRuntimeSpan(std::span<int>(iArr2, 2));
-    testRuntimeSpan(std::span<int>(iArr2, 3));
-    testRuntimeSpan(std::span<int>(iArr2, 4));
-    testRuntimeSpan(std::span<int>(iArr2, 5));
-
-    std::string s;
-    testRuntimeSpan(std::span<std::string>(&s, (std::size_t) 0));
-    testRuntimeSpan(std::span<std::string>(&s, 1));
+struct A {};
+bool operator==(A, A) { return true; }
+
+constexpr int iArr1[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9};
+int iArr2[]           = {10, 11, 12, 13, 14, 15, 16, 17, 18, 19};
+
+int main(int, char**) {
+  static_assert(testConstexprSpan(std::span<int>()), "");
+  static_assert(testConstexprSpan(std::span<long>()), "");
+  static_assert(testConstexprSpan(std::span<double>()), "");
+  static_assert(testConstexprSpan(std::span<A>()), "");
+  static_assert(testConstexprSpan(std::span<std::string>()), "");
+
+  static_assert(testConstexprSpan(std::span<int, 0>()), "");
+  static_assert(testConstexprSpan(std::span<long, 0>()), "");
+  static_assert(testConstexprSpan(std::span<double, 0>()), "");
+  static_assert(testConstexprSpan(std::span<A, 0>()), "");
+  static_assert(testConstexprSpan(std::span<std::string, 0>()), "");
+
+  static_assert(testConstexprSpan(std::span<const int>(iArr1, 1)), "");
+  static_assert(testConstexprSpan(std::span<const int>(iArr1, 2)), "");
+  static_assert(testConstexprSpan(std::span<const int>(iArr1, 3)), "");
+  static_assert(testConstexprSpan(std::span<const int>(iArr1, 4)), "");
+  static_assert(testConstexprSpan(std::span<const int>(iArr1, 5)), "");
+
+  testRuntimeSpan(std::span<int>());
+  testRuntimeSpan(std::span<long>());
+  testRuntimeSpan(std::span<double>());
+  testRuntimeSpan(std::span<A>());
+  testRuntimeSpan(std::span<std::string>());
+
+  testRuntimeSpan(std::span<int, 0>());
+  testRuntimeSpan(std::span<long, 0>());
+  testRuntimeSpan(std::span<double, 0>());
+  testRuntimeSpan(std::span<A, 0>());
+  testRuntimeSpan(std::span<std::string, 0>());
+
+  testRuntimeSpan(std::span<int>(iArr2, 1));
+  testRuntimeSpan(std::span<int>(iArr2, 2));
+  testRuntimeSpan(std::span<int>(iArr2, 3));
+  testRuntimeSpan(std::span<int>(iArr2, 4));
+  testRuntimeSpan(std::span<int>(iArr2, 5));
+
+  std::string s;
+  testRuntimeSpan(std::span<std::string>(&s, (std::size_t)0));
+  testRuntimeSpan(std::span<std::string>(&s, 1));
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/views/views.span/span.objectrep/as_bytes.pass.cpp b/libcxx/test/std/containers/views/views.span/span.objectrep/as_bytes.pass.cpp
index 44b658fb83759..1b62fb94e9ab2 100644
--- a/libcxx/test/std/containers/views/views.span/span.objectrep/as_bytes.pass.cpp
+++ b/libcxx/test/std/containers/views/views.span/span.objectrep/as_bytes.pass.cpp
@@ -23,56 +23,54 @@
 
 #include "test_macros.h"
 
-template<typename Span>
-void testRuntimeSpan(Span sp)
-{
-    ASSERT_NOEXCEPT(std::as_bytes(sp));
+template <typename Span>
+void testRuntimeSpan(Span sp) {
+  ASSERT_NOEXCEPT(std::as_bytes(sp));
 
-    auto spBytes = std::as_bytes(sp);
-    using SB = decltype(spBytes);
-    ASSERT_SAME_TYPE(const std::byte, typename SB::element_type);
+  auto spBytes = std::as_bytes(sp);
+  using SB     = decltype(spBytes);
+  ASSERT_SAME_TYPE(const std::byte, typename SB::element_type);
 
-    if constexpr (sp.extent == std::dynamic_extent)
-        assert(spBytes.extent == std::dynamic_extent);
-    else
-        assert(spBytes.extent == sizeof(typename Span::element_type) * sp.extent);
+  if constexpr (sp.extent == std::dynamic_extent)
+    assert(spBytes.extent == std::dynamic_extent);
+  else
+    assert(spBytes.extent == sizeof(typename Span::element_type) * sp.extent);
 
-    assert((void *) spBytes.data() == (void *) sp.data());
-    assert(spBytes.size() == sp.size_bytes());
+  assert((void*)spBytes.data() == (void*)sp.data());
+  assert(spBytes.size() == sp.size_bytes());
 }
 
-struct A{};
-int iArr2[] = { 0,  1,  2,  3,  4,  5,  6,  7,  8,  9};
+struct A {};
+int iArr2[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9};
 
-int main(int, char**)
-{
-    testRuntimeSpan(std::span<int>        ());
-    testRuntimeSpan(std::span<long>       ());
-    testRuntimeSpan(std::span<double>     ());
-    testRuntimeSpan(std::span<A>          ());
-    testRuntimeSpan(std::span<std::string>());
+int main(int, char**) {
+  testRuntimeSpan(std::span<int>());
+  testRuntimeSpan(std::span<long>());
+  testRuntimeSpan(std::span<double>());
+  testRuntimeSpan(std::span<A>());
+  testRuntimeSpan(std::span<std::string>());
 
-    testRuntimeSpan(std::span<int, 0>        ());
-    testRuntimeSpan(std::span<long, 0>       ());
-    testRuntimeSpan(std::span<double, 0>     ());
-    testRuntimeSpan(std::span<A, 0>          ());
-    testRuntimeSpan(std::span<std::string, 0>());
+  testRuntimeSpan(std::span<int, 0>());
+  testRuntimeSpan(std::span<long, 0>());
+  testRuntimeSpan(std::span<double, 0>());
+  testRuntimeSpan(std::span<A, 0>());
+  testRuntimeSpan(std::span<std::string, 0>());
 
-    testRuntimeSpan(std::span<int>(iArr2, 1));
-    testRuntimeSpan(std::span<int>(iArr2, 2));
-    testRuntimeSpan(std::span<int>(iArr2, 3));
-    testRuntimeSpan(std::span<int>(iArr2, 4));
-    testRuntimeSpan(std::span<int>(iArr2, 5));
+  testRuntimeSpan(std::span<int>(iArr2, 1));
+  testRuntimeSpan(std::span<int>(iArr2, 2));
+  testRuntimeSpan(std::span<int>(iArr2, 3));
+  testRuntimeSpan(std::span<int>(iArr2, 4));
+  testRuntimeSpan(std::span<int>(iArr2, 5));
 
-    testRuntimeSpan(std::span<int, 1>(iArr2 + 5, 1));
-    testRuntimeSpan(std::span<int, 2>(iArr2 + 4, 2));
-    testRuntimeSpan(std::span<int, 3>(iArr2 + 3, 3));
-    testRuntimeSpan(std::span<int, 4>(iArr2 + 2, 4));
-    testRuntimeSpan(std::span<int, 5>(iArr2 + 1, 5));
+  testRuntimeSpan(std::span<int, 1>(iArr2 + 5, 1));
+  testRuntimeSpan(std::span<int, 2>(iArr2 + 4, 2));
+  testRuntimeSpan(std::span<int, 3>(iArr2 + 3, 3));
+  testRuntimeSpan(std::span<int, 4>(iArr2 + 2, 4));
+  testRuntimeSpan(std::span<int, 5>(iArr2 + 1, 5));
 
-    std::string s;
-    testRuntimeSpan(std::span<std::string>(&s, (std::size_t) 0));
-    testRuntimeSpan(std::span<std::string>(&s, 1));
+  std::string s;
+  testRuntimeSpan(std::span<std::string>(&s, (std::size_t)0));
+  testRuntimeSpan(std::span<std::string>(&s, 1));
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/views/views.span/span.objectrep/as_writable_bytes.pass.cpp b/libcxx/test/std/containers/views/views.span/span.objectrep/as_writable_bytes.pass.cpp
index d38d69d9fee1d..40c6a581e9acf 100644
--- a/libcxx/test/std/containers/views/views.span/span.objectrep/as_writable_bytes.pass.cpp
+++ b/libcxx/test/std/containers/views/views.span/span.objectrep/as_writable_bytes.pass.cpp
@@ -23,56 +23,54 @@
 
 #include "test_macros.h"
 
-template<typename Span>
-void testRuntimeSpan(Span sp)
-{
-    ASSERT_NOEXCEPT(std::as_writable_bytes(sp));
+template <typename Span>
+void testRuntimeSpan(Span sp) {
+  ASSERT_NOEXCEPT(std::as_writable_bytes(sp));
 
-    auto spBytes = std::as_writable_bytes(sp);
-    using SB = decltype(spBytes);
-    ASSERT_SAME_TYPE(std::byte, typename SB::element_type);
+  auto spBytes = std::as_writable_bytes(sp);
+  using SB     = decltype(spBytes);
+  ASSERT_SAME_TYPE(std::byte, typename SB::element_type);
 
-    if constexpr (sp.extent == std::dynamic_extent)
-        assert(spBytes.extent == std::dynamic_extent);
-    else
-        assert(spBytes.extent == sizeof(typename Span::element_type) * sp.extent);
+  if constexpr (sp.extent == std::dynamic_extent)
+    assert(spBytes.extent == std::dynamic_extent);
+  else
+    assert(spBytes.extent == sizeof(typename Span::element_type) * sp.extent);
 
-    assert(static_cast<void*>(spBytes.data()) == static_cast<void*>(sp.data()));
-    assert(spBytes.size() == sp.size_bytes());
+  assert(static_cast<void*>(spBytes.data()) == static_cast<void*>(sp.data()));
+  assert(spBytes.size() == sp.size_bytes());
 }
 
-struct A{};
-int iArr2[] = { 0,  1,  2,  3,  4,  5,  6,  7,  8,  9};
+struct A {};
+int iArr2[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9};
 
-int main(int, char**)
-{
-    testRuntimeSpan(std::span<int>        ());
-    testRuntimeSpan(std::span<long>       ());
-    testRuntimeSpan(std::span<double>     ());
-    testRuntimeSpan(std::span<A>          ());
-    testRuntimeSpan(std::span<std::string>());
+int main(int, char**) {
+  testRuntimeSpan(std::span<int>());
+  testRuntimeSpan(std::span<long>());
+  testRuntimeSpan(std::span<double>());
+  testRuntimeSpan(std::span<A>());
+  testRuntimeSpan(std::span<std::string>());
 
-    testRuntimeSpan(std::span<int, 0>        ());
-    testRuntimeSpan(std::span<long, 0>       ());
-    testRuntimeSpan(std::span<double, 0>     ());
-    testRuntimeSpan(std::span<A, 0>          ());
-    testRuntimeSpan(std::span<std::string, 0>());
+  testRuntimeSpan(std::span<int, 0>());
+  testRuntimeSpan(std::span<long, 0>());
+  testRuntimeSpan(std::span<double, 0>());
+  testRuntimeSpan(std::span<A, 0>());
+  testRuntimeSpan(std::span<std::string, 0>());
 
-    testRuntimeSpan(std::span<int>(iArr2, 1));
-    testRuntimeSpan(std::span<int>(iArr2, 2));
-    testRuntimeSpan(std::span<int>(iArr2, 3));
-    testRuntimeSpan(std::span<int>(iArr2, 4));
-    testRuntimeSpan(std::span<int>(iArr2, 5));
+  testRuntimeSpan(std::span<int>(iArr2, 1));
+  testRuntimeSpan(std::span<int>(iArr2, 2));
+  testRuntimeSpan(std::span<int>(iArr2, 3));
+  testRuntimeSpan(std::span<int>(iArr2, 4));
+  testRuntimeSpan(std::span<int>(iArr2, 5));
 
-    testRuntimeSpan(std::span<int, 1>(iArr2 + 5, 1));
-    testRuntimeSpan(std::span<int, 2>(iArr2 + 4, 2));
-    testRuntimeSpan(std::span<int, 3>(iArr2 + 3, 3));
-    testRuntimeSpan(std::span<int, 4>(iArr2 + 2, 4));
-    testRuntimeSpan(std::span<int, 5>(iArr2 + 1, 5));
+  testRuntimeSpan(std::span<int, 1>(iArr2 + 5, 1));
+  testRuntimeSpan(std::span<int, 2>(iArr2 + 4, 2));
+  testRuntimeSpan(std::span<int, 3>(iArr2 + 3, 3));
+  testRuntimeSpan(std::span<int, 4>(iArr2 + 2, 4));
+  testRuntimeSpan(std::span<int, 5>(iArr2 + 1, 5));
 
-    std::string s;
-    testRuntimeSpan(std::span<std::string>(&s, (std::size_t) 0));
-    testRuntimeSpan(std::span<std::string>(&s, 1));
+  std::string s;
+  testRuntimeSpan(std::span<std::string>(&s, (std::size_t)0));
+  testRuntimeSpan(std::span<std::string>(&s, 1));
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/views/views.span/span.objectrep/as_writable_bytes.verify.cpp b/libcxx/test/std/containers/views/views.span/span.objectrep/as_writable_bytes.verify.cpp
index 45485b339752c..9d22641ab2687 100644
--- a/libcxx/test/std/containers/views/views.span/span.objectrep/as_writable_bytes.verify.cpp
+++ b/libcxx/test/std/containers/views/views.span/span.objectrep/as_writable_bytes.verify.cpp
@@ -21,23 +21,35 @@
 
 #include "test_macros.h"
 
-const int iArr2[] = { 0,  1,  2,  3,  4,  5,  6,  7,  8,  9};
+const int iArr2[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9};
 
 struct A {};
 
 void f() {
-  std::as_writable_bytes(std::span<const int>());            // expected-error {{no matching function for call to 'as_writable_bytes'}}
-  std::as_writable_bytes(std::span<const long>());           // expected-error {{no matching function for call to 'as_writable_bytes'}}
-  std::as_writable_bytes(std::span<const double>());         // expected-error {{no matching function for call to 'as_writable_bytes'}}
-  std::as_writable_bytes(std::span<const A>());              // expected-error {{no matching function for call to 'as_writable_bytes'}}
-  std::as_writable_bytes(std::span<const std::string>());    // expected-error {{no matching function for call to 'as_writable_bytes'}}
-
-  std::as_writable_bytes(std::span<const int, 0>());         // expected-error {{no matching function for call to 'as_writable_bytes'}}
-  std::as_writable_bytes(std::span<const long, 0>());        // expected-error {{no matching function for call to 'as_writable_bytes'}}
-  std::as_writable_bytes(std::span<const double, 0>());      // expected-error {{no matching function for call to 'as_writable_bytes'}}
-  std::as_writable_bytes(std::span<const A, 0>());           // expected-error {{no matching function for call to 'as_writable_bytes'}}
-  std::as_writable_bytes(std::span<const std::string, (std::size_t)0>()); // expected-error {{no matching function for call to 'as_writable_bytes'}}
-
-  std::as_writable_bytes(std::span<const int>   (iArr2, 1));     // expected-error {{no matching function for call to 'as_writable_bytes'}}
-  std::as_writable_bytes(std::span<const int, 1>(iArr2 + 5, 1)); // expected-error {{no matching function for call to 'as_writable_bytes'}}
+  std::as_writable_bytes(std::span<const int>());
+  // expected-error@-1 {{no matching function for call to 'as_writable_bytes'}}
+  std::as_writable_bytes(std::span<const long>());
+  // expected-error@-1 {{no matching function for call to 'as_writable_bytes'}}
+  std::as_writable_bytes(std::span<const double>());
+  // expected-error@-1 {{no matching function for call to 'as_writable_bytes'}}
+  std::as_writable_bytes(std::span<const A>());
+  // expected-error@-1 {{no matching function for call to 'as_writable_bytes'}}
+  std::as_writable_bytes(std::span<const std::string>());
+  // expected-error@-1 {{no matching function for call to 'as_writable_bytes'}}
+
+  std::as_writable_bytes(std::span<const int, 0>());
+  // expected-error@-1 {{no matching function for call to 'as_writable_bytes'}}
+  std::as_writable_bytes(std::span<const long, 0>());
+  // expected-error@-1 {{no matching function for call to 'as_writable_bytes'}}
+  std::as_writable_bytes(std::span<const double, 0>());
+  // expected-error@-1 {{no matching function for call to 'as_writable_bytes'}}
+  std::as_writable_bytes(std::span<const A, 0>());
+  // expected-error@-1 {{no matching function for call to 'as_writable_bytes'}}
+  std::as_writable_bytes(std::span<const std::string, (std::size_t)0>());
+  // expected-error@-1 {{no matching function for call to 'as_writable_bytes'}}
+
+  std::as_writable_bytes(std::span<const int>(iArr2, 1));
+  // expected-error@-1 {{no matching function for call to 'as_writable_bytes'}}
+  std::as_writable_bytes(std::span<const int, 1>(iArr2 + 5, 1));
+  // expected-error@-1 {{no matching function for call to 'as_writable_bytes'}}
 }
diff --git a/libcxx/test/std/containers/views/views.span/span.obs/empty.pass.cpp b/libcxx/test/std/containers/views/views.span/span.obs/empty.pass.cpp
index 3aa99b0854f36..c7b66884abc8c 100644
--- a/libcxx/test/std/containers/views/views.span/span.obs/empty.pass.cpp
+++ b/libcxx/test/std/containers/views/views.span/span.obs/empty.pass.cpp
@@ -12,62 +12,59 @@
 // [[nodiscard]] constexpr bool empty() const noexcept;
 //
 
-
 #include <span>
 #include <cassert>
 #include <string>
 
 #include "test_macros.h"
 
-struct A{};
-constexpr int iArr1[] = { 0,  1,  2,  3,  4,  5,  6,  7,  8,  9};
-          int iArr2[] = {10, 11, 12, 13, 14, 15, 16, 17, 18, 19};
-
-int main(int, char**)
-{
-    static_assert( noexcept(std::span<int>   ().empty()), "");
-    static_assert( noexcept(std::span<int, 0>().empty()), "");
-
-
-    static_assert( std::span<int>().empty(),            "");
-    static_assert( std::span<long>().empty(),           "");
-    static_assert( std::span<double>().empty(),         "");
-    static_assert( std::span<A>().empty(),              "");
-    static_assert( std::span<std::string>().empty(),    "");
-
-    static_assert( std::span<int, 0>().empty(),         "");
-    static_assert( std::span<long, 0>().empty(),        "");
-    static_assert( std::span<double, 0>().empty(),      "");
-    static_assert( std::span<A, 0>().empty(),           "");
-    static_assert( std::span<std::string, 0>().empty(), "");
-
-    static_assert(!std::span<const int>(iArr1, 1).empty(), "");
-    static_assert(!std::span<const int>(iArr1, 2).empty(), "");
-    static_assert(!std::span<const int>(iArr1, 3).empty(), "");
-    static_assert(!std::span<const int>(iArr1, 4).empty(), "");
-    static_assert(!std::span<const int>(iArr1, 5).empty(), "");
-
-    assert( (std::span<int>().empty()           ));
-    assert( (std::span<long>().empty()          ));
-    assert( (std::span<double>().empty()        ));
-    assert( (std::span<A>().empty()             ));
-    assert( (std::span<std::string>().empty()   ));
-
-    assert( (std::span<int, 0>().empty()        ));
-    assert( (std::span<long, 0>().empty()       ));
-    assert( (std::span<double, 0>().empty()     ));
-    assert( (std::span<A, 0>().empty()          ));
-    assert( (std::span<std::string, 0>().empty()));
-
-    assert(!(std::span<int, 1>(iArr2, 1).empty()));
-    assert(!(std::span<int, 2>(iArr2, 2).empty()));
-    assert(!(std::span<int, 3>(iArr2, 3).empty()));
-    assert(!(std::span<int, 4>(iArr2, 4).empty()));
-    assert(!(std::span<int, 5>(iArr2, 5).empty()));
-
-    std::string s;
-    assert( ((std::span<std::string>(&s, (std::size_t) 0)).empty()));
-    assert(!((std::span<std::string>(&s, 1).empty())));
+struct A {};
+constexpr int iArr1[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9};
+int iArr2[]           = {10, 11, 12, 13, 14, 15, 16, 17, 18, 19};
+
+int main(int, char**) {
+  static_assert(noexcept(std::span<int>().empty()), "");
+  static_assert(noexcept(std::span<int, 0>().empty()), "");
+
+  static_assert(std::span<int>().empty(), "");
+  static_assert(std::span<long>().empty(), "");
+  static_assert(std::span<double>().empty(), "");
+  static_assert(std::span<A>().empty(), "");
+  static_assert(std::span<std::string>().empty(), "");
+
+  static_assert(std::span<int, 0>().empty(), "");
+  static_assert(std::span<long, 0>().empty(), "");
+  static_assert(std::span<double, 0>().empty(), "");
+  static_assert(std::span<A, 0>().empty(), "");
+  static_assert(std::span<std::string, 0>().empty(), "");
+
+  static_assert(!std::span<const int>(iArr1, 1).empty(), "");
+  static_assert(!std::span<const int>(iArr1, 2).empty(), "");
+  static_assert(!std::span<const int>(iArr1, 3).empty(), "");
+  static_assert(!std::span<const int>(iArr1, 4).empty(), "");
+  static_assert(!std::span<const int>(iArr1, 5).empty(), "");
+
+  assert((std::span<int>().empty()));
+  assert((std::span<long>().empty()));
+  assert((std::span<double>().empty()));
+  assert((std::span<A>().empty()));
+  assert((std::span<std::string>().empty()));
+
+  assert((std::span<int, 0>().empty()));
+  assert((std::span<long, 0>().empty()));
+  assert((std::span<double, 0>().empty()));
+  assert((std::span<A, 0>().empty()));
+  assert((std::span<std::string, 0>().empty()));
+
+  assert(!(std::span<int, 1>(iArr2, 1).empty()));
+  assert(!(std::span<int, 2>(iArr2, 2).empty()));
+  assert(!(std::span<int, 3>(iArr2, 3).empty()));
+  assert(!(std::span<int, 4>(iArr2, 4).empty()));
+  assert(!(std::span<int, 5>(iArr2, 5).empty()));
+
+  std::string s;
+  assert(((std::span<std::string>(&s, (std::size_t)0)).empty()));
+  assert(!((std::span<std::string>(&s, 1).empty())));
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/views/views.span/span.obs/size.pass.cpp b/libcxx/test/std/containers/views/views.span/span.obs/size.pass.cpp
index fa41d7803e97e..9dde3d4610b24 100644
--- a/libcxx/test/std/containers/views/views.span/span.obs/size.pass.cpp
+++ b/libcxx/test/std/containers/views/views.span/span.obs/size.pass.cpp
@@ -12,80 +12,74 @@
 // constexpr size_type size() const noexcept;
 //
 
-
 #include <span>
 #include <cassert>
 #include <string>
 
 #include "test_macros.h"
 
-
 template <typename Span>
-constexpr bool testConstexprSpan(Span sp, std::size_t sz)
-{
-    ASSERT_NOEXCEPT(sp.size());
-    return sp.size() == sz;
+constexpr bool testConstexprSpan(Span sp, std::size_t sz) {
+  ASSERT_NOEXCEPT(sp.size());
+  return sp.size() == sz;
 }
 
-
 template <typename Span>
-void testRuntimeSpan(Span sp, std::size_t sz)
-{
-    ASSERT_NOEXCEPT(sp.size());
-    assert(sp.size() == sz);
+void testRuntimeSpan(Span sp, std::size_t sz) {
+  ASSERT_NOEXCEPT(sp.size());
+  assert(sp.size() == sz);
 }
 
-struct A{};
-constexpr int iArr1[] = { 0,  1,  2,  3,  4,  5,  6,  7,  8,  9};
-          int iArr2[] = {10, 11, 12, 13, 14, 15, 16, 17, 18, 19};
-
-int main(int, char**)
-{
-    static_assert(testConstexprSpan(std::span<int>(), 0),            "");
-    static_assert(testConstexprSpan(std::span<long>(), 0),           "");
-    static_assert(testConstexprSpan(std::span<double>(), 0),         "");
-    static_assert(testConstexprSpan(std::span<A>(), 0),              "");
-    static_assert(testConstexprSpan(std::span<std::string>(), 0),    "");
-
-    static_assert(testConstexprSpan(std::span<int, 0>(), 0),         "");
-    static_assert(testConstexprSpan(std::span<long, 0>(), 0),        "");
-    static_assert(testConstexprSpan(std::span<double, 0>(), 0),      "");
-    static_assert(testConstexprSpan(std::span<A, 0>(), 0),           "");
-    static_assert(testConstexprSpan(std::span<std::string, 0>(), 0), "");
-
-    static_assert(testConstexprSpan(std::span<const int>(iArr1, 1), 1),    "");
-    static_assert(testConstexprSpan(std::span<const int>(iArr1, 2), 2),    "");
-    static_assert(testConstexprSpan(std::span<const int>(iArr1, 3), 3),    "");
-    static_assert(testConstexprSpan(std::span<const int>(iArr1, 4), 4),    "");
-    static_assert(testConstexprSpan(std::span<const int>(iArr1, 5), 5),    "");
-
-    testRuntimeSpan(std::span<int>        (), 0);
-    testRuntimeSpan(std::span<long>       (), 0);
-    testRuntimeSpan(std::span<double>     (), 0);
-    testRuntimeSpan(std::span<A>          (), 0);
-    testRuntimeSpan(std::span<std::string>(), 0);
-
-    testRuntimeSpan(std::span<int, 0>        (), 0);
-    testRuntimeSpan(std::span<long, 0>       (), 0);
-    testRuntimeSpan(std::span<double, 0>     (), 0);
-    testRuntimeSpan(std::span<A, 0>          (), 0);
-    testRuntimeSpan(std::span<std::string, 0>(), 0);
-
-    testRuntimeSpan(std::span<int>(iArr2, 1), 1);
-    testRuntimeSpan(std::span<int>(iArr2, 2), 2);
-    testRuntimeSpan(std::span<int>(iArr2, 3), 3);
-    testRuntimeSpan(std::span<int>(iArr2, 4), 4);
-    testRuntimeSpan(std::span<int>(iArr2, 5), 5);
-
-    testRuntimeSpan(std::span<int, 1>(iArr2 + 5, 1), 1);
-    testRuntimeSpan(std::span<int, 2>(iArr2 + 4, 2), 2);
-    testRuntimeSpan(std::span<int, 3>(iArr2 + 3, 3), 3);
-    testRuntimeSpan(std::span<int, 4>(iArr2 + 2, 4), 4);
-    testRuntimeSpan(std::span<int, 5>(iArr2 + 1, 5), 5);
-
-    std::string s;
-    testRuntimeSpan(std::span<std::string>(&s, (std::size_t) 0), 0);
-    testRuntimeSpan(std::span<std::string>(&s, 1), 1);
+struct A {};
+constexpr int iArr1[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9};
+int iArr2[]           = {10, 11, 12, 13, 14, 15, 16, 17, 18, 19};
+
+int main(int, char**) {
+  static_assert(testConstexprSpan(std::span<int>(), 0), "");
+  static_assert(testConstexprSpan(std::span<long>(), 0), "");
+  static_assert(testConstexprSpan(std::span<double>(), 0), "");
+  static_assert(testConstexprSpan(std::span<A>(), 0), "");
+  static_assert(testConstexprSpan(std::span<std::string>(), 0), "");
+
+  static_assert(testConstexprSpan(std::span<int, 0>(), 0), "");
+  static_assert(testConstexprSpan(std::span<long, 0>(), 0), "");
+  static_assert(testConstexprSpan(std::span<double, 0>(), 0), "");
+  static_assert(testConstexprSpan(std::span<A, 0>(), 0), "");
+  static_assert(testConstexprSpan(std::span<std::string, 0>(), 0), "");
+
+  static_assert(testConstexprSpan(std::span<const int>(iArr1, 1), 1), "");
+  static_assert(testConstexprSpan(std::span<const int>(iArr1, 2), 2), "");
+  static_assert(testConstexprSpan(std::span<const int>(iArr1, 3), 3), "");
+  static_assert(testConstexprSpan(std::span<const int>(iArr1, 4), 4), "");
+  static_assert(testConstexprSpan(std::span<const int>(iArr1, 5), 5), "");
+
+  testRuntimeSpan(std::span<int>(), 0);
+  testRuntimeSpan(std::span<long>(), 0);
+  testRuntimeSpan(std::span<double>(), 0);
+  testRuntimeSpan(std::span<A>(), 0);
+  testRuntimeSpan(std::span<std::string>(), 0);
+
+  testRuntimeSpan(std::span<int, 0>(), 0);
+  testRuntimeSpan(std::span<long, 0>(), 0);
+  testRuntimeSpan(std::span<double, 0>(), 0);
+  testRuntimeSpan(std::span<A, 0>(), 0);
+  testRuntimeSpan(std::span<std::string, 0>(), 0);
+
+  testRuntimeSpan(std::span<int>(iArr2, 1), 1);
+  testRuntimeSpan(std::span<int>(iArr2, 2), 2);
+  testRuntimeSpan(std::span<int>(iArr2, 3), 3);
+  testRuntimeSpan(std::span<int>(iArr2, 4), 4);
+  testRuntimeSpan(std::span<int>(iArr2, 5), 5);
+
+  testRuntimeSpan(std::span<int, 1>(iArr2 + 5, 1), 1);
+  testRuntimeSpan(std::span<int, 2>(iArr2 + 4, 2), 2);
+  testRuntimeSpan(std::span<int, 3>(iArr2 + 3, 3), 3);
+  testRuntimeSpan(std::span<int, 4>(iArr2 + 2, 4), 4);
+  testRuntimeSpan(std::span<int, 5>(iArr2 + 1, 5), 5);
+
+  std::string s;
+  testRuntimeSpan(std::span<std::string>(&s, (std::size_t)0), 0);
+  testRuntimeSpan(std::span<std::string>(&s, 1), 1);
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/views/views.span/span.obs/size_bytes.pass.cpp b/libcxx/test/std/containers/views/views.span/span.obs/size_bytes.pass.cpp
index c25eaaa2017bf..3d6e4209315d7 100644
--- a/libcxx/test/std/containers/views/views.span/span.obs/size_bytes.pass.cpp
+++ b/libcxx/test/std/containers/views/views.span/span.obs/size_bytes.pass.cpp
@@ -13,80 +13,74 @@
 //
 //  Effects: Equivalent to: return size() * sizeof(element_type);
 
-
 #include <span>
 #include <cassert>
 #include <string>
 
 #include "test_macros.h"
 
-
 template <typename Span>
-constexpr bool testConstexprSpan(Span sp, std::size_t sz)
-{
-    ASSERT_NOEXCEPT(sp.size_bytes());
-    return (std::size_t) sp.size_bytes() == sz * sizeof(typename Span::element_type);
+constexpr bool testConstexprSpan(Span sp, std::size_t sz) {
+  ASSERT_NOEXCEPT(sp.size_bytes());
+  return (std::size_t)sp.size_bytes() == sz * sizeof(typename Span::element_type);
 }
 
-
 template <typename Span>
-void testRuntimeSpan(Span sp, std::size_t sz)
-{
-    ASSERT_NOEXCEPT(sp.size_bytes());
-    assert((std::size_t) sp.size_bytes() == sz * sizeof(typename Span::element_type));
+void testRuntimeSpan(Span sp, std::size_t sz) {
+  ASSERT_NOEXCEPT(sp.size_bytes());
+  assert((std::size_t)sp.size_bytes() == sz * sizeof(typename Span::element_type));
 }
 
-struct A{};
-constexpr int iArr1[] = { 0,  1,  2,  3,  4,  5,  6,  7,  8,  9};
-          int iArr2[] = {10, 11, 12, 13, 14, 15, 16, 17, 18, 19};
-
-int main(int, char**)
-{
-    static_assert(testConstexprSpan(std::span<int>(), 0),            "");
-    static_assert(testConstexprSpan(std::span<long>(), 0),           "");
-    static_assert(testConstexprSpan(std::span<double>(), 0),         "");
-    static_assert(testConstexprSpan(std::span<A>(), 0),              "");
-    static_assert(testConstexprSpan(std::span<std::string>(), 0),    "");
-
-    static_assert(testConstexprSpan(std::span<int, 0>(), 0),         "");
-    static_assert(testConstexprSpan(std::span<long, 0>(), 0),        "");
-    static_assert(testConstexprSpan(std::span<double, 0>(), 0),      "");
-    static_assert(testConstexprSpan(std::span<A, 0>(), 0),           "");
-    static_assert(testConstexprSpan(std::span<std::string, 0>(), 0), "");
-
-    static_assert(testConstexprSpan(std::span<const int>(iArr1, 1), 1),    "");
-    static_assert(testConstexprSpan(std::span<const int>(iArr1, 2), 2),    "");
-    static_assert(testConstexprSpan(std::span<const int>(iArr1, 3), 3),    "");
-    static_assert(testConstexprSpan(std::span<const int>(iArr1, 4), 4),    "");
-    static_assert(testConstexprSpan(std::span<const int>(iArr1, 5), 5),    "");
-
-    testRuntimeSpan(std::span<int>        (), 0);
-    testRuntimeSpan(std::span<long>       (), 0);
-    testRuntimeSpan(std::span<double>     (), 0);
-    testRuntimeSpan(std::span<A>          (), 0);
-    testRuntimeSpan(std::span<std::string>(), 0);
-
-    testRuntimeSpan(std::span<int, 0>        (), 0);
-    testRuntimeSpan(std::span<long, 0>       (), 0);
-    testRuntimeSpan(std::span<double, 0>     (), 0);
-    testRuntimeSpan(std::span<A, 0>          (), 0);
-    testRuntimeSpan(std::span<std::string, 0>(), 0);
-
-    testRuntimeSpan(std::span<int>(iArr2, 1), 1);
-    testRuntimeSpan(std::span<int>(iArr2, 2), 2);
-    testRuntimeSpan(std::span<int>(iArr2, 3), 3);
-    testRuntimeSpan(std::span<int>(iArr2, 4), 4);
-    testRuntimeSpan(std::span<int>(iArr2, 5), 5);
-
-    testRuntimeSpan(std::span<int, 1>(iArr2 + 5, 1), 1);
-    testRuntimeSpan(std::span<int, 2>(iArr2 + 4, 2), 2);
-    testRuntimeSpan(std::span<int, 3>(iArr2 + 3, 3), 3);
-    testRuntimeSpan(std::span<int, 4>(iArr2 + 2, 4), 4);
-    testRuntimeSpan(std::span<int, 5>(iArr2 + 1, 5), 5);
-
-    std::string s;
-    testRuntimeSpan(std::span<std::string>(&s, (std::size_t) 0), 0);
-    testRuntimeSpan(std::span<std::string>(&s, 1), 1);
+struct A {};
+constexpr int iArr1[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9};
+int iArr2[]           = {10, 11, 12, 13, 14, 15, 16, 17, 18, 19};
+
+int main(int, char**) {
+  static_assert(testConstexprSpan(std::span<int>(), 0), "");
+  static_assert(testConstexprSpan(std::span<long>(), 0), "");
+  static_assert(testConstexprSpan(std::span<double>(), 0), "");
+  static_assert(testConstexprSpan(std::span<A>(), 0), "");
+  static_assert(testConstexprSpan(std::span<std::string>(), 0), "");
+
+  static_assert(testConstexprSpan(std::span<int, 0>(), 0), "");
+  static_assert(testConstexprSpan(std::span<long, 0>(), 0), "");
+  static_assert(testConstexprSpan(std::span<double, 0>(), 0), "");
+  static_assert(testConstexprSpan(std::span<A, 0>(), 0), "");
+  static_assert(testConstexprSpan(std::span<std::string, 0>(), 0), "");
+
+  static_assert(testConstexprSpan(std::span<const int>(iArr1, 1), 1), "");
+  static_assert(testConstexprSpan(std::span<const int>(iArr1, 2), 2), "");
+  static_assert(testConstexprSpan(std::span<const int>(iArr1, 3), 3), "");
+  static_assert(testConstexprSpan(std::span<const int>(iArr1, 4), 4), "");
+  static_assert(testConstexprSpan(std::span<const int>(iArr1, 5), 5), "");
+
+  testRuntimeSpan(std::span<int>(), 0);
+  testRuntimeSpan(std::span<long>(), 0);
+  testRuntimeSpan(std::span<double>(), 0);
+  testRuntimeSpan(std::span<A>(), 0);
+  testRuntimeSpan(std::span<std::string>(), 0);
+
+  testRuntimeSpan(std::span<int, 0>(), 0);
+  testRuntimeSpan(std::span<long, 0>(), 0);
+  testRuntimeSpan(std::span<double, 0>(), 0);
+  testRuntimeSpan(std::span<A, 0>(), 0);
+  testRuntimeSpan(std::span<std::string, 0>(), 0);
+
+  testRuntimeSpan(std::span<int>(iArr2, 1), 1);
+  testRuntimeSpan(std::span<int>(iArr2, 2), 2);
+  testRuntimeSpan(std::span<int>(iArr2, 3), 3);
+  testRuntimeSpan(std::span<int>(iArr2, 4), 4);
+  testRuntimeSpan(std::span<int>(iArr2, 5), 5);
+
+  testRuntimeSpan(std::span<int, 1>(iArr2 + 5, 1), 1);
+  testRuntimeSpan(std::span<int, 2>(iArr2 + 4, 2), 2);
+  testRuntimeSpan(std::span<int, 3>(iArr2 + 3, 3), 3);
+  testRuntimeSpan(std::span<int, 4>(iArr2 + 2, 4), 4);
+  testRuntimeSpan(std::span<int, 5>(iArr2 + 1, 5), 5);
+
+  std::string s;
+  testRuntimeSpan(std::span<std::string>(&s, (std::size_t)0), 0);
+  testRuntimeSpan(std::span<std::string>(&s, 1), 1);
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/views/views.span/span.sub/first.pass.cpp b/libcxx/test/std/containers/views/views.span/span.sub/first.pass.cpp
index a354ade2247ae..a9a697c0c4d21 100644
--- a/libcxx/test/std/containers/views/views.span/span.sub/first.pass.cpp
+++ b/libcxx/test/std/containers/views/views.span/span.sub/first.pass.cpp
@@ -24,51 +24,43 @@
 #include "test_macros.h"
 
 template <typename Span, std::size_t Count>
-constexpr bool testConstexprSpan(Span sp)
-{
-    LIBCPP_ASSERT((noexcept(sp.template first<Count>())));
-    LIBCPP_ASSERT((noexcept(sp.first(Count))));
-    auto s1 = sp.template first<Count>();
-    auto s2 = sp.first(Count);
-    using S1 = decltype(s1);
-    using S2 = decltype(s2);
-    ASSERT_SAME_TYPE(typename Span::value_type, typename S1::value_type);
-    ASSERT_SAME_TYPE(typename Span::value_type, typename S2::value_type);
-    static_assert(S1::extent == Count, "");
-    static_assert(S2::extent == std::dynamic_extent, "");
-    return
-        s1.data() == s2.data()
-     && s1.size() == s2.size()
-     && std::equal(s1.begin(), s1.end(), sp.begin());
+constexpr bool testConstexprSpan(Span sp) {
+  LIBCPP_ASSERT((noexcept(sp.template first<Count>())));
+  LIBCPP_ASSERT((noexcept(sp.first(Count))));
+  auto s1  = sp.template first<Count>();
+  auto s2  = sp.first(Count);
+  using S1 = decltype(s1);
+  using S2 = decltype(s2);
+  ASSERT_SAME_TYPE(typename Span::value_type, typename S1::value_type);
+  ASSERT_SAME_TYPE(typename Span::value_type, typename S2::value_type);
+  static_assert(S1::extent == Count, "");
+  static_assert(S2::extent == std::dynamic_extent, "");
+  return s1.data() == s2.data() && s1.size() == s2.size() && std::equal(s1.begin(), s1.end(), sp.begin());
 }
 
-
 template <typename Span, std::size_t Count>
-void testRuntimeSpan(Span sp)
-{
-    LIBCPP_ASSERT((noexcept(sp.template first<Count>())));
-    LIBCPP_ASSERT((noexcept(sp.first(Count))));
-    auto s1 = sp.template first<Count>();
-    auto s2 = sp.first(Count);
-    using S1 = decltype(s1);
-    using S2 = decltype(s2);
-    ASSERT_SAME_TYPE(typename Span::value_type, typename S1::value_type);
-    ASSERT_SAME_TYPE(typename Span::value_type, typename S2::value_type);
-    static_assert(S1::extent == Count, "");
-    static_assert(S2::extent == std::dynamic_extent, "");
-    assert(s1.data() == s2.data());
-    assert(s1.size() == s2.size());
-    assert(std::equal(s1.begin(), s1.end(), sp.begin()));
+void testRuntimeSpan(Span sp) {
+  LIBCPP_ASSERT((noexcept(sp.template first<Count>())));
+  LIBCPP_ASSERT((noexcept(sp.first(Count))));
+  auto s1  = sp.template first<Count>();
+  auto s2  = sp.first(Count);
+  using S1 = decltype(s1);
+  using S2 = decltype(s2);
+  ASSERT_SAME_TYPE(typename Span::value_type, typename S1::value_type);
+  ASSERT_SAME_TYPE(typename Span::value_type, typename S2::value_type);
+  static_assert(S1::extent == Count, "");
+  static_assert(S2::extent == std::dynamic_extent, "");
+  assert(s1.data() == s2.data());
+  assert(s1.size() == s2.size());
+  assert(std::equal(s1.begin(), s1.end(), sp.begin()));
 }
 
+constexpr int carr1[] = {1, 2, 3, 4};
+int arr[]             = {5, 6, 7};
+std::string sarr[]    = {"ABC", "DEF", "GHI", "JKL", "MNO"};
 
-constexpr int carr1[] = {1,2,3,4};
-          int   arr[] = {5,6,7};
-std::string   sarr [] = { "ABC", "DEF", "GHI", "JKL", "MNO"};
-
-int main(int, char**)
-{
-    {
+int main(int, char**) {
+  {
     using Sp = std::span<const int>;
     static_assert(testConstexprSpan<Sp, 0>(Sp{}), "");
 
@@ -77,9 +69,9 @@ int main(int, char**)
     static_assert(testConstexprSpan<Sp, 2>(Sp{carr1}), "");
     static_assert(testConstexprSpan<Sp, 3>(Sp{carr1}), "");
     static_assert(testConstexprSpan<Sp, 4>(Sp{carr1}), "");
-    }
+  }
 
-    {
+  {
     using Sp = std::span<const int, 4>;
 
     static_assert(testConstexprSpan<Sp, 0>(Sp{carr1}), "");
@@ -87,9 +79,9 @@ int main(int, char**)
     static_assert(testConstexprSpan<Sp, 2>(Sp{carr1}), "");
     static_assert(testConstexprSpan<Sp, 3>(Sp{carr1}), "");
     static_assert(testConstexprSpan<Sp, 4>(Sp{carr1}), "");
-    }
+  }
 
-    {
+  {
     using Sp = std::span<int>;
     testRuntimeSpan<Sp, 0>(Sp{});
 
@@ -97,18 +89,18 @@ int main(int, char**)
     testRuntimeSpan<Sp, 1>(Sp{arr});
     testRuntimeSpan<Sp, 2>(Sp{arr});
     testRuntimeSpan<Sp, 3>(Sp{arr});
-    }
+  }
 
-    {
+  {
     using Sp = std::span<int, 3>;
 
     testRuntimeSpan<Sp, 0>(Sp{arr});
     testRuntimeSpan<Sp, 1>(Sp{arr});
     testRuntimeSpan<Sp, 2>(Sp{arr});
     testRuntimeSpan<Sp, 3>(Sp{arr});
-    }
+  }
 
-    {
+  {
     using Sp = std::span<std::string>;
     testConstexprSpan<Sp, 0>(Sp{});
 
@@ -118,9 +110,9 @@ int main(int, char**)
     testRuntimeSpan<Sp, 3>(Sp{sarr});
     testRuntimeSpan<Sp, 4>(Sp{sarr});
     testRuntimeSpan<Sp, 5>(Sp{sarr});
-    }
+  }
 
-    {
+  {
     using Sp = std::span<std::string, 5>;
 
     testRuntimeSpan<Sp, 0>(Sp{sarr});
@@ -129,7 +121,7 @@ int main(int, char**)
     testRuntimeSpan<Sp, 3>(Sp{sarr});
     testRuntimeSpan<Sp, 4>(Sp{sarr});
     testRuntimeSpan<Sp, 5>(Sp{sarr});
-    }
+  }
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/views/views.span/span.sub/first.verify.cpp b/libcxx/test/std/containers/views/views.span/span.sub/first.verify.cpp
index 39d04b0e12253..8cf28c25e3b91 100644
--- a/libcxx/test/std/containers/views/views.span/span.sub/first.verify.cpp
+++ b/libcxx/test/std/containers/views/views.span/span.sub/first.verify.cpp
@@ -25,5 +25,6 @@ void f() {
   [[maybe_unused]] auto s1 = sp.first<5>(); // expected-error@span:* {{span<T, N>::first<Count>(): Count out of range}}
 
   //  Count numeric_limits
-  [[maybe_unused]] auto s2 = sp.first<std::size_t(-1)>(); // expected-error@span:* {{span<T, N>::first<Count>(): Count out of range}}
+  [[maybe_unused]] auto s2 =
+      sp.first<std::size_t(-1)>(); // expected-error@span:* {{span<T, N>::first<Count>(): Count out of range}}
 }
diff --git a/libcxx/test/std/containers/views/views.span/span.sub/last.pass.cpp b/libcxx/test/std/containers/views/views.span/span.sub/last.pass.cpp
index 017c805af3ed5..29b4e94b7c5df 100644
--- a/libcxx/test/std/containers/views/views.span/span.sub/last.pass.cpp
+++ b/libcxx/test/std/containers/views/views.span/span.sub/last.pass.cpp
@@ -24,51 +24,43 @@
 #include "test_macros.h"
 
 template <typename Span, std::size_t Count>
-constexpr bool testConstexprSpan(Span sp)
-{
-    LIBCPP_ASSERT((noexcept(sp.template last<Count>())));
-    LIBCPP_ASSERT((noexcept(sp.last(Count))));
-    auto s1 = sp.template last<Count>();
-    auto s2 = sp.last(Count);
-    using S1 = decltype(s1);
-    using S2 = decltype(s2);
-    ASSERT_SAME_TYPE(typename Span::value_type, typename S1::value_type);
-    ASSERT_SAME_TYPE(typename Span::value_type, typename S2::value_type);
-    static_assert(S1::extent == Count, "");
-    static_assert(S2::extent == std::dynamic_extent, "");
-    return
-        s1.data() == s2.data()
-     && s1.size() == s2.size()
-     && std::equal(s1.begin(), s1.end(), sp.end() - Count);
+constexpr bool testConstexprSpan(Span sp) {
+  LIBCPP_ASSERT((noexcept(sp.template last<Count>())));
+  LIBCPP_ASSERT((noexcept(sp.last(Count))));
+  auto s1  = sp.template last<Count>();
+  auto s2  = sp.last(Count);
+  using S1 = decltype(s1);
+  using S2 = decltype(s2);
+  ASSERT_SAME_TYPE(typename Span::value_type, typename S1::value_type);
+  ASSERT_SAME_TYPE(typename Span::value_type, typename S2::value_type);
+  static_assert(S1::extent == Count, "");
+  static_assert(S2::extent == std::dynamic_extent, "");
+  return s1.data() == s2.data() && s1.size() == s2.size() && std::equal(s1.begin(), s1.end(), sp.end() - Count);
 }
 
-
 template <typename Span, std::size_t Count>
-void testRuntimeSpan(Span sp)
-{
-    LIBCPP_ASSERT((noexcept(sp.template last<Count>())));
-    LIBCPP_ASSERT((noexcept(sp.last(Count))));
-    auto s1 = sp.template last<Count>();
-    auto s2 = sp.last(Count);
-    using S1 = decltype(s1);
-    using S2 = decltype(s2);
-    ASSERT_SAME_TYPE(typename Span::value_type, typename S1::value_type);
-    ASSERT_SAME_TYPE(typename Span::value_type, typename S2::value_type);
-    static_assert(S1::extent == Count, "");
-    static_assert(S2::extent == std::dynamic_extent, "");
-    assert(s1.data() == s2.data());
-    assert(s1.size() == s2.size());
-    assert(std::equal(s1.begin(), s1.end(), sp.end() - Count));
+void testRuntimeSpan(Span sp) {
+  LIBCPP_ASSERT((noexcept(sp.template last<Count>())));
+  LIBCPP_ASSERT((noexcept(sp.last(Count))));
+  auto s1  = sp.template last<Count>();
+  auto s2  = sp.last(Count);
+  using S1 = decltype(s1);
+  using S2 = decltype(s2);
+  ASSERT_SAME_TYPE(typename Span::value_type, typename S1::value_type);
+  ASSERT_SAME_TYPE(typename Span::value_type, typename S2::value_type);
+  static_assert(S1::extent == Count, "");
+  static_assert(S2::extent == std::dynamic_extent, "");
+  assert(s1.data() == s2.data());
+  assert(s1.size() == s2.size());
+  assert(std::equal(s1.begin(), s1.end(), sp.end() - Count));
 }
 
+constexpr int carr1[] = {1, 2, 3, 4};
+int arr[]             = {5, 6, 7};
+std::string sarr[]    = {"ABC", "DEF", "GHI", "JKL", "MNO"};
 
-constexpr int carr1[] = {1,2,3,4};
-          int   arr[] = {5,6,7};
-std::string   sarr [] = { "ABC", "DEF", "GHI", "JKL", "MNO"};
-
-int main(int, char**)
-{
-    {
+int main(int, char**) {
+  {
     using Sp = std::span<const int>;
     static_assert(testConstexprSpan<Sp, 0>(Sp{}), "");
 
@@ -77,9 +69,9 @@ int main(int, char**)
     static_assert(testConstexprSpan<Sp, 2>(Sp{carr1}), "");
     static_assert(testConstexprSpan<Sp, 3>(Sp{carr1}), "");
     static_assert(testConstexprSpan<Sp, 4>(Sp{carr1}), "");
-    }
+  }
 
-    {
+  {
     using Sp = std::span<const int, 4>;
 
     static_assert(testConstexprSpan<Sp, 0>(Sp{carr1}), "");
@@ -87,9 +79,9 @@ int main(int, char**)
     static_assert(testConstexprSpan<Sp, 2>(Sp{carr1}), "");
     static_assert(testConstexprSpan<Sp, 3>(Sp{carr1}), "");
     static_assert(testConstexprSpan<Sp, 4>(Sp{carr1}), "");
-    }
+  }
 
-    {
+  {
     using Sp = std::span<int>;
     testRuntimeSpan<Sp, 0>(Sp{});
 
@@ -97,18 +89,18 @@ int main(int, char**)
     testRuntimeSpan<Sp, 1>(Sp{arr});
     testRuntimeSpan<Sp, 2>(Sp{arr});
     testRuntimeSpan<Sp, 3>(Sp{arr});
-    }
+  }
 
-    {
+  {
     using Sp = std::span<int, 3>;
 
     testRuntimeSpan<Sp, 0>(Sp{arr});
     testRuntimeSpan<Sp, 1>(Sp{arr});
     testRuntimeSpan<Sp, 2>(Sp{arr});
     testRuntimeSpan<Sp, 3>(Sp{arr});
-    }
+  }
 
-    {
+  {
     using Sp = std::span<std::string>;
     testConstexprSpan<Sp, 0>(Sp{});
 
@@ -118,9 +110,9 @@ int main(int, char**)
     testRuntimeSpan<Sp, 3>(Sp{sarr});
     testRuntimeSpan<Sp, 4>(Sp{sarr});
     testRuntimeSpan<Sp, 5>(Sp{sarr});
-    }
+  }
 
-    {
+  {
     using Sp = std::span<std::string, 5>;
 
     testRuntimeSpan<Sp, 0>(Sp{sarr});
@@ -129,7 +121,7 @@ int main(int, char**)
     testRuntimeSpan<Sp, 3>(Sp{sarr});
     testRuntimeSpan<Sp, 4>(Sp{sarr});
     testRuntimeSpan<Sp, 5>(Sp{sarr});
-    }
+  }
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/views/views.span/span.sub/last.verify.cpp b/libcxx/test/std/containers/views/views.span/span.sub/last.verify.cpp
index 9fc6deb87a0fa..d96bfac6d98de 100644
--- a/libcxx/test/std/containers/views/views.span/span.sub/last.verify.cpp
+++ b/libcxx/test/std/containers/views/views.span/span.sub/last.verify.cpp
@@ -25,5 +25,6 @@ void f() {
   [[maybe_unused]] auto s1 = sp.last<5>(); // expected-error@span:* {{span<T, N>::last<Count>(): Count out of range}}
 
   //  Count numeric_limits
-  [[maybe_unused]] auto s2 = sp.last<std::size_t(-1)>(); // expected-error@span:* {{span<T, N>::last<Count>(): Count out of range}}
+  [[maybe_unused]] auto s2 =
+      sp.last<std::size_t(-1)>(); // expected-error@span:* {{span<T, N>::last<Count>(): Count out of range}}
 }
diff --git a/libcxx/test/std/containers/views/views.span/span.sub/subspan.pass.cpp b/libcxx/test/std/containers/views/views.span/span.sub/subspan.pass.cpp
index 401363090e6e5..50e43c5e5582f 100644
--- a/libcxx/test/std/containers/views/views.span/span.sub/subspan.pass.cpp
+++ b/libcxx/test/std/containers/views/views.span/span.sub/subspan.pass.cpp
@@ -25,88 +25,75 @@
 #include "test_macros.h"
 
 template <typename Span, std::size_t Offset, size_t Count>
-constexpr bool testConstexprSpan(Span sp)
-{
-    LIBCPP_ASSERT((noexcept(sp.template subspan<Offset, Count>())));
-    LIBCPP_ASSERT((noexcept(sp.subspan(Offset, Count))));
-    auto s1 = sp.template subspan<Offset, Count>();
-    auto s2 = sp.subspan(Offset, Count);
-    using S1 = decltype(s1);
-    using S2 = decltype(s2);
-    ASSERT_SAME_TYPE(typename Span::value_type, typename S1::value_type);
-    ASSERT_SAME_TYPE(typename Span::value_type, typename S2::value_type);
-    static_assert(S1::extent == Count);
-    static_assert(S2::extent == std::dynamic_extent, "");
-    return
-        s1.data() == s2.data()
-     && s1.size() == s2.size()
-     && std::equal(s1.begin(), s1.end(), sp.begin() + Offset);
+constexpr bool testConstexprSpan(Span sp) {
+  LIBCPP_ASSERT((noexcept(sp.template subspan<Offset, Count>())));
+  LIBCPP_ASSERT((noexcept(sp.subspan(Offset, Count))));
+  auto s1  = sp.template subspan<Offset, Count>();
+  auto s2  = sp.subspan(Offset, Count);
+  using S1 = decltype(s1);
+  using S2 = decltype(s2);
+  ASSERT_SAME_TYPE(typename Span::value_type, typename S1::value_type);
+  ASSERT_SAME_TYPE(typename Span::value_type, typename S2::value_type);
+  static_assert(S1::extent == Count);
+  static_assert(S2::extent == std::dynamic_extent, "");
+  return s1.data() == s2.data() && s1.size() == s2.size() && std::equal(s1.begin(), s1.end(), sp.begin() + Offset);
 }
 
 template <typename Span, std::size_t Offset>
-constexpr bool testConstexprSpan(Span sp)
-{
-    LIBCPP_ASSERT((noexcept(sp.template subspan<Offset>())));
-    LIBCPP_ASSERT((noexcept(sp.subspan(Offset))));
-    auto s1 = sp.template subspan<Offset>();
-    auto s2 = sp.subspan(Offset);
-    using S1 = decltype(s1);
-    using S2 = decltype(s2);
-    ASSERT_SAME_TYPE(typename Span::value_type, typename S1::value_type);
-    ASSERT_SAME_TYPE(typename Span::value_type, typename S2::value_type);
-    static_assert(S1::extent == (Span::extent == std::dynamic_extent ? std::dynamic_extent : Span::extent - Offset), "");
-    static_assert(S2::extent == std::dynamic_extent, "");
-    return
-        s1.data() == s2.data()
-     && s1.size() == s2.size()
-     && std::equal(s1.begin(), s1.end(), sp.begin() + Offset, sp.end());
+constexpr bool testConstexprSpan(Span sp) {
+  LIBCPP_ASSERT((noexcept(sp.template subspan<Offset>())));
+  LIBCPP_ASSERT((noexcept(sp.subspan(Offset))));
+  auto s1  = sp.template subspan<Offset>();
+  auto s2  = sp.subspan(Offset);
+  using S1 = decltype(s1);
+  using S2 = decltype(s2);
+  ASSERT_SAME_TYPE(typename Span::value_type, typename S1::value_type);
+  ASSERT_SAME_TYPE(typename Span::value_type, typename S2::value_type);
+  static_assert(S1::extent == (Span::extent == std::dynamic_extent ? std::dynamic_extent : Span::extent - Offset), "");
+  static_assert(S2::extent == std::dynamic_extent, "");
+  return s1.data() == s2.data() && s1.size() == s2.size() &&
+         std::equal(s1.begin(), s1.end(), sp.begin() + Offset, sp.end());
 }
 
-
 template <typename Span, std::size_t Offset, size_t Count>
-void testRuntimeSpan(Span sp)
-{
-    LIBCPP_ASSERT((noexcept(sp.template subspan<Offset, Count>())));
-    LIBCPP_ASSERT((noexcept(sp.subspan(Offset, Count))));
-    auto s1 = sp.template subspan<Offset, Count>();
-    auto s2 = sp.subspan(Offset, Count);
-    using S1 = decltype(s1);
-    using S2 = decltype(s2);
-    ASSERT_SAME_TYPE(typename Span::value_type, typename S1::value_type);
-    ASSERT_SAME_TYPE(typename Span::value_type, typename S2::value_type);
-    static_assert(S1::extent == Count);
-    static_assert(S2::extent == std::dynamic_extent, "");
-    assert(s1.data() == s2.data());
-    assert(s1.size() == s2.size());
-    assert(std::equal(s1.begin(), s1.end(), sp.begin() + Offset));
+void testRuntimeSpan(Span sp) {
+  LIBCPP_ASSERT((noexcept(sp.template subspan<Offset, Count>())));
+  LIBCPP_ASSERT((noexcept(sp.subspan(Offset, Count))));
+  auto s1  = sp.template subspan<Offset, Count>();
+  auto s2  = sp.subspan(Offset, Count);
+  using S1 = decltype(s1);
+  using S2 = decltype(s2);
+  ASSERT_SAME_TYPE(typename Span::value_type, typename S1::value_type);
+  ASSERT_SAME_TYPE(typename Span::value_type, typename S2::value_type);
+  static_assert(S1::extent == Count);
+  static_assert(S2::extent == std::dynamic_extent, "");
+  assert(s1.data() == s2.data());
+  assert(s1.size() == s2.size());
+  assert(std::equal(s1.begin(), s1.end(), sp.begin() + Offset));
 }
 
-
 template <typename Span, std::size_t Offset>
-void testRuntimeSpan(Span sp)
-{
-    LIBCPP_ASSERT((noexcept(sp.template subspan<Offset>())));
-    LIBCPP_ASSERT((noexcept(sp.subspan(Offset))));
-    auto s1 = sp.template subspan<Offset>();
-    auto s2 = sp.subspan(Offset);
-    using S1 = decltype(s1);
-    using S2 = decltype(s2);
-    ASSERT_SAME_TYPE(typename Span::value_type, typename S1::value_type);
-    ASSERT_SAME_TYPE(typename Span::value_type, typename S2::value_type);
-    static_assert(S1::extent == (Span::extent == std::dynamic_extent ? std::dynamic_extent : Span::extent - Offset), "");
-    static_assert(S2::extent == std::dynamic_extent, "");
-    assert(s1.data() == s2.data());
-    assert(s1.size() == s2.size());
-    assert(std::equal(s1.begin(), s1.end(), sp.begin() + Offset, sp.end()));
+void testRuntimeSpan(Span sp) {
+  LIBCPP_ASSERT((noexcept(sp.template subspan<Offset>())));
+  LIBCPP_ASSERT((noexcept(sp.subspan(Offset))));
+  auto s1  = sp.template subspan<Offset>();
+  auto s2  = sp.subspan(Offset);
+  using S1 = decltype(s1);
+  using S2 = decltype(s2);
+  ASSERT_SAME_TYPE(typename Span::value_type, typename S1::value_type);
+  ASSERT_SAME_TYPE(typename Span::value_type, typename S2::value_type);
+  static_assert(S1::extent == (Span::extent == std::dynamic_extent ? std::dynamic_extent : Span::extent - Offset), "");
+  static_assert(S2::extent == std::dynamic_extent, "");
+  assert(s1.data() == s2.data());
+  assert(s1.size() == s2.size());
+  assert(std::equal(s1.begin(), s1.end(), sp.begin() + Offset, sp.end()));
 }
 
+constexpr int carr1[] = {1, 2, 3, 4};
+int arr1[]            = {5, 6, 7};
 
-constexpr int carr1[] = {1,2,3,4};
-          int  arr1[] = {5,6,7};
-
-int main(int, char**)
-{
-    {
+int main(int, char**) {
+  {
     using Sp = std::span<const int>;
     static_assert(testConstexprSpan<Sp, 0>(Sp{}), "");
 
@@ -120,9 +107,9 @@ int main(int, char**)
     static_assert(testConstexprSpan<Sp, 2, 2>(Sp{carr1}), "");
     static_assert(testConstexprSpan<Sp, 3, 1>(Sp{carr1}), "");
     static_assert(testConstexprSpan<Sp, 4, 0>(Sp{carr1}), "");
-    }
+  }
 
-    {
+  {
     using Sp = std::span<const int, 4>;
 
     static_assert(testConstexprSpan<Sp, 0, 4>(Sp{carr1}), "");
@@ -135,9 +122,9 @@ int main(int, char**)
     static_assert(testConstexprSpan<Sp, 2, 2>(Sp{carr1}), "");
     static_assert(testConstexprSpan<Sp, 3, 1>(Sp{carr1}), "");
     static_assert(testConstexprSpan<Sp, 4, 0>(Sp{carr1}), "");
-    }
+  }
 
-    {
+  {
     using Sp = std::span<const int>;
     static_assert(testConstexprSpan<Sp, 0>(Sp{}), "");
 
@@ -146,9 +133,9 @@ int main(int, char**)
     static_assert(testConstexprSpan<Sp, 2>(Sp{carr1}), "");
     static_assert(testConstexprSpan<Sp, 3>(Sp{carr1}), "");
     static_assert(testConstexprSpan<Sp, 4>(Sp{carr1}), "");
-    }
+  }
 
-    {
+  {
     using Sp = std::span<const int, 4>;
 
     static_assert(testConstexprSpan<Sp, 0>(Sp{carr1}), "");
@@ -157,9 +144,9 @@ int main(int, char**)
     static_assert(testConstexprSpan<Sp, 2>(Sp{carr1}), "");
     static_assert(testConstexprSpan<Sp, 3>(Sp{carr1}), "");
     static_assert(testConstexprSpan<Sp, 4>(Sp{carr1}), "");
-    }
+  }
 
-    {
+  {
     using Sp = std::span<int>;
     testRuntimeSpan<Sp, 0>(Sp{});
 
@@ -171,9 +158,9 @@ int main(int, char**)
     testRuntimeSpan<Sp, 1, 2>(Sp{arr1});
     testRuntimeSpan<Sp, 2, 1>(Sp{arr1});
     testRuntimeSpan<Sp, 3, 0>(Sp{arr1});
-    }
+  }
 
-    {
+  {
     using Sp = std::span<int, 3>;
 
     testRuntimeSpan<Sp, 0, 3>(Sp{arr1});
@@ -184,9 +171,9 @@ int main(int, char**)
     testRuntimeSpan<Sp, 1, 2>(Sp{arr1});
     testRuntimeSpan<Sp, 2, 1>(Sp{arr1});
     testRuntimeSpan<Sp, 3, 0>(Sp{arr1});
-    }
+  }
 
-    {
+  {
     using Sp = std::span<int>;
     testRuntimeSpan<Sp, 0>(Sp{});
 
@@ -194,16 +181,16 @@ int main(int, char**)
     testRuntimeSpan<Sp, 1>(Sp{arr1});
     testRuntimeSpan<Sp, 2>(Sp{arr1});
     testRuntimeSpan<Sp, 3>(Sp{arr1});
-    }
+  }
 
-    {
+  {
     using Sp = std::span<int, 3>;
 
     testRuntimeSpan<Sp, 0>(Sp{arr1});
     testRuntimeSpan<Sp, 1>(Sp{arr1});
     testRuntimeSpan<Sp, 2>(Sp{arr1});
     testRuntimeSpan<Sp, 3>(Sp{arr1});
-    }
+  }
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/views/views.span/span.sub/subspan.verify.cpp b/libcxx/test/std/containers/views/views.span/span.sub/subspan.verify.cpp
index d6bc21197098a..3c8b81a0d4734 100644
--- a/libcxx/test/std/containers/views/views.span/span.sub/subspan.verify.cpp
+++ b/libcxx/test/std/containers/views/views.span/span.sub/subspan.verify.cpp
@@ -26,14 +26,20 @@ void f() {
   std::span<const int, 4> sp(array);
 
   //  Offset too large templatized
-  [[maybe_unused]] auto s1 = sp.subspan<5>(); // expected-error@span:* {{span<T, N>::subspan<Offset, Count>(): Offset out of range}}
+  [[maybe_unused]] auto s1 =
+      sp.subspan<5>(); // expected-error@span:* {{span<T, N>::subspan<Offset, Count>(): Offset out of range}}
 
   //  Count too large templatized
-  [[maybe_unused]] auto s2 = sp.subspan<0, 5>(); // expected-error@span:* {{span<T, N>::subspan<Offset, Count>(): Offset + Count out of range}}
+  [[maybe_unused]] auto s2 =
+      sp.subspan<0, 5>(); // expected-error@span:* {{span<T, N>::subspan<Offset, Count>(): Offset + Count out of range}}
 
   //  Offset + Count too large templatized
-  [[maybe_unused]] auto s3 = sp.subspan<2, 3>(); // expected-error@span:* {{span<T, N>::subspan<Offset, Count>(): Offset + Count out of range}}
+  [[maybe_unused]] auto s3 =
+      sp.subspan<2, 3>(); // expected-error@span:* {{span<T, N>::subspan<Offset, Count>(): Offset + Count out of range}}
 
   //  Offset + Count overflow templatized
-  [[maybe_unused]] auto s4 = sp.subspan<3, std::size_t(-2)>(); // expected-error@span:* {{span<T, N>::subspan<Offset, Count>(): Offset + Count out of range}}, expected-error-re@span:* {{array is too large{{(.* elements)}}}}
+  [[maybe_unused]] auto s4 = sp.subspan<
+      3,
+      std::size_t(
+          -2)>(); // expected-error@span:* {{span<T, N>::subspan<Offset, Count>(): Offset + Count out of range}}, expected-error-re@span:* {{array is too large{{(.* elements)}}}}
 }
diff --git a/libcxx/test/std/containers/views/views.span/types.pass.cpp b/libcxx/test/std/containers/views/views.span/types.pass.cpp
index be17884dae295..14f5de823f353 100644
--- a/libcxx/test/std/containers/views/views.span/types.pass.cpp
+++ b/libcxx/test/std/containers/views/views.span/types.pass.cpp
@@ -37,59 +37,54 @@
 #include "test_macros.h"
 
 template <typename S, typename Iter>
-void testIterator()
-{
-    typedef std::iterator_traits<Iter> ItT;
+void testIterator() {
+  typedef std::iterator_traits<Iter> ItT;
 
-    ASSERT_SAME_TYPE(typename ItT::iterator_category, std::random_access_iterator_tag);
-    ASSERT_SAME_TYPE(typename ItT::value_type,        typename S::value_type);
-    ASSERT_SAME_TYPE(typename ItT::reference,         typename S::reference);
-    ASSERT_SAME_TYPE(typename ItT::pointer,           typename S::pointer);
-    ASSERT_SAME_TYPE(typename ItT::difference_type,   typename S::difference_type);
+  ASSERT_SAME_TYPE(typename ItT::iterator_category, std::random_access_iterator_tag);
+  ASSERT_SAME_TYPE(typename ItT::value_type, typename S::value_type);
+  ASSERT_SAME_TYPE(typename ItT::reference, typename S::reference);
+  ASSERT_SAME_TYPE(typename ItT::pointer, typename S::pointer);
+  ASSERT_SAME_TYPE(typename ItT::difference_type, typename S::difference_type);
 }
 
 template <typename S, typename ElementType, std::size_t Size>
-void testSpan()
-{
-    ASSERT_SAME_TYPE(typename S::element_type,    ElementType);
-    ASSERT_SAME_TYPE(typename S::value_type,      std::remove_cv_t<ElementType>);
-    ASSERT_SAME_TYPE(typename S::size_type,       std::size_t);
-    ASSERT_SAME_TYPE(typename S::difference_type, std::ptrdiff_t);
-    ASSERT_SAME_TYPE(typename S::pointer,         ElementType *);
-    ASSERT_SAME_TYPE(typename S::const_pointer,   const ElementType *);
-    ASSERT_SAME_TYPE(typename S::reference,       ElementType &);
-    ASSERT_SAME_TYPE(typename S::const_reference, const ElementType &);
+void testSpan() {
+  ASSERT_SAME_TYPE(typename S::element_type, ElementType);
+  ASSERT_SAME_TYPE(typename S::value_type, std::remove_cv_t<ElementType>);
+  ASSERT_SAME_TYPE(typename S::size_type, std::size_t);
+  ASSERT_SAME_TYPE(typename S::difference_type, std::ptrdiff_t);
+  ASSERT_SAME_TYPE(typename S::pointer, ElementType*);
+  ASSERT_SAME_TYPE(typename S::const_pointer, const ElementType*);
+  ASSERT_SAME_TYPE(typename S::reference, ElementType&);
+  ASSERT_SAME_TYPE(typename S::const_reference, const ElementType&);
 
-    static_assert(S::extent == Size); // check that it exists
+  static_assert(S::extent == Size); // check that it exists
 
-    testIterator<S, typename S::iterator>();
-    testIterator<S, typename S::reverse_iterator>();
+  testIterator<S, typename S::iterator>();
+  testIterator<S, typename S::reverse_iterator>();
 }
 
-
 template <typename T>
-void test()
-{
-    testSpan<std::span<               T>,                T, std::dynamic_extent>();
-    testSpan<std::span<const          T>, const          T, std::dynamic_extent>();
-    testSpan<std::span<      volatile T>,       volatile T, std::dynamic_extent>();
-    testSpan<std::span<const volatile T>, const volatile T, std::dynamic_extent>();
+void test() {
+  testSpan<std::span< T>, T, std::dynamic_extent>();
+  testSpan<std::span<const T>, const T, std::dynamic_extent>();
+  testSpan<std::span< volatile T>, volatile T, std::dynamic_extent>();
+  testSpan<std::span<const volatile T>, const volatile T, std::dynamic_extent>();
 
-    testSpan<std::span<               T, 5>,                T, 5>();
-    testSpan<std::span<const          T, 5>, const          T, 5>();
-    testSpan<std::span<      volatile T, 5>,       volatile T, 5>();
-    testSpan<std::span<const volatile T, 5>, const volatile T, 5>();
+  testSpan<std::span< T, 5>, T, 5>();
+  testSpan<std::span<const T, 5>, const T, 5>();
+  testSpan<std::span< volatile T, 5>, volatile T, 5>();
+  testSpan<std::span<const volatile T, 5>, const volatile T, 5>();
 }
 
-struct A{};
+struct A {};
 
-int main(int, char**)
-{
-    test<int>();
-    test<long>();
-    test<double>();
-    test<std::string>();
-    test<A>();
+int main(int, char**) {
+  test<int>();
+  test<long>();
+  test<double>();
+  test<std::string>();
+  test<A>();
 
   return 0;
 }
diff --git a/libcxx/test/std/depr/depr.c.headers/math_h.pass.cpp b/libcxx/test/std/depr/depr.c.headers/math_h.pass.cpp
index 189e9d890e778..aebef0b2e1363 100644
--- a/libcxx/test/std/depr/depr.c.headers/math_h.pass.cpp
+++ b/libcxx/test/std/depr/depr.c.headers/math_h.pass.cpp
@@ -8,7 +8,7 @@
 
 // This test fails because Clang no longer enables -fdelayed-template-parsing
 // by default on Windows with C++20 (#69431).
-// XFAIL: msvc && (clang-18 || clang-19 || clang-20)
+// XFAIL: msvc && (clang-18 || clang-19 || clang-20 || clang-21)
 
 // <math.h>
 
diff --git a/libcxx/test/std/experimental/simd/simd.class/simd_copy.pass.cpp b/libcxx/test/std/experimental/simd/simd.class/simd_copy.pass.cpp
index 7d91ca0eada1d..6929831eca361 100644
--- a/libcxx/test/std/experimental/simd/simd.class/simd_copy.pass.cpp
+++ b/libcxx/test/std/experimental/simd/simd.class/simd_copy.pass.cpp
@@ -10,7 +10,7 @@
 
 // Older versions of clang may encounter a backend error (see 0295c2ad):
 //   Pass-by-value arguments with alignment greater than register width are not supported.
-// XFAIL: target=powerpc{{.*}}-ibm-{{.*}} && (clang-17 || clang-18)
+// XFAIL: target=powerpc{{.*}}-ibm-{{.*}} && clang-18
 
 // <experimental/simd>
 //
diff --git a/libcxx/test/std/experimental/simd/simd.class/simd_unary.pass.cpp b/libcxx/test/std/experimental/simd/simd.class/simd_unary.pass.cpp
index 17ecfd3b50d25..eb88d90ca18bd 100644
--- a/libcxx/test/std/experimental/simd/simd.class/simd_unary.pass.cpp
+++ b/libcxx/test/std/experimental/simd/simd.class/simd_unary.pass.cpp
@@ -10,7 +10,7 @@
 
 // Older versions of clang may encounter a backend error (see 0295c2ad):
 //   Pass-by-value arguments with alignment greater than register width are not supported.
-// XFAIL: target=powerpc{{.*}}-ibm-{{.*}} && (clang-17 || clang-18)
+// XFAIL: target=powerpc{{.*}}-ibm-{{.*}} && clang-18
 
 // This test crashes AppleClang 15 but not later versions.
 // UNSUPPORTED: apple-clang-15
diff --git a/libcxx/test/std/input.output/file.streams/fstreams/ifstream.members/offset_range.pass.cpp b/libcxx/test/std/input.output/file.streams/fstreams/ifstream.members/offset_range.pass.cpp
index 6ffe750564c2c..c6e07d045e145 100644
--- a/libcxx/test/std/input.output/file.streams/fstreams/ifstream.members/offset_range.pass.cpp
+++ b/libcxx/test/std/input.output/file.streams/fstreams/ifstream.members/offset_range.pass.cpp
@@ -11,12 +11,6 @@
 // Test that we can seek using offsets larger than 32 bit, and that we can
 // retrieve file offsets larger than 32 bit.
 
-// On MSVC targets, we only use the 32 bit fseek/ftell functions. For MinGW
-// targets, we use fseeko/ftello, but the user needs to define
-// _FILE_OFFSET_BITS=64 to make them 64 bit.
-//
-// XFAIL: target={{.*}}-windows{{.*}}
-
 // On 32 bit Android platforms, off_t is 32 bit by default. By defining
 // _FILE_OFFSET_BITS=64, one gets a 64 bit off_t, but the corresponding
 // 64 bit ftello/fseeko functions are only available since Android API 24 (7.0).
diff --git a/libcxx/test/std/input.output/iostream.format/print.fun/includes.compile.pass.cpp b/libcxx/test/std/input.output/iostream.format/print.fun/includes.compile.pass.cpp
index 9b9b0e404e6b7..38e4e4d3fb9ef 100644
--- a/libcxx/test/std/input.output/iostream.format/print.fun/includes.compile.pass.cpp
+++ b/libcxx/test/std/input.output/iostream.format/print.fun/includes.compile.pass.cpp
@@ -6,7 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
+// REQUIRES: std-at-least-c++23
 // UNSUPPORTED: no-filesystem
 // UNSUPPORTED: GCC-ALWAYS_INLINE-FIXME
 
diff --git a/libcxx/test/std/input.output/iostream.format/print.fun/no_file_description.pass.cpp b/libcxx/test/std/input.output/iostream.format/print.fun/no_file_description.pass.cpp
index d3e4463fe0bc8..5561a1a8b3334 100644
--- a/libcxx/test/std/input.output/iostream.format/print.fun/no_file_description.pass.cpp
+++ b/libcxx/test/std/input.output/iostream.format/print.fun/no_file_description.pass.cpp
@@ -6,7 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
+// REQUIRES: std-at-least-c++23
 // UNSUPPORTED: no-filesystem
 // UNSUPPORTED: GCC-ALWAYS_INLINE-FIXME
 
diff --git a/libcxx/test/std/language.support/support.dynamic/new.delete/new.delete.array/sized_delete_array.pass.cpp b/libcxx/test/std/language.support/support.dynamic/new.delete/new.delete.array/sized_delete_array.pass.cpp
index f0ad2c0e67df3..01387feed67b6 100644
--- a/libcxx/test/std/language.support/support.dynamic/new.delete/new.delete.array/sized_delete_array.pass.cpp
+++ b/libcxx/test/std/language.support/support.dynamic/new.delete/new.delete.array/sized_delete_array.pass.cpp
@@ -11,10 +11,10 @@
 // UNSUPPORTED: c++03, c++11
 
 // These compiler versions and platforms don't enable sized deallocation by default.
-// ADDITIONAL_COMPILE_FLAGS(clang-17): -fsized-deallocation
 // ADDITIONAL_COMPILE_FLAGS(clang-18): -fsized-deallocation
 // ADDITIONAL_COMPILE_FLAGS(apple-clang-15): -fsized-deallocation
 // ADDITIONAL_COMPILE_FLAGS(apple-clang-16): -fsized-deallocation
+// ADDITIONAL_COMPILE_FLAGS(apple-clang-17): -fsized-deallocation
 // ADDITIONAL_COMPILE_FLAGS(target=x86_64-w64-windows-gnu): -fsized-deallocation
 // ADDITIONAL_COMPILE_FLAGS(target=i686-w64-windows-gnu): -fsized-deallocation
 
diff --git a/libcxx/test/std/language.support/support.dynamic/new.delete/new.delete.single/sized_delete.pass.cpp b/libcxx/test/std/language.support/support.dynamic/new.delete/new.delete.single/sized_delete.pass.cpp
index fd52df451afc8..06d3b0e5b3c35 100644
--- a/libcxx/test/std/language.support/support.dynamic/new.delete/new.delete.single/sized_delete.pass.cpp
+++ b/libcxx/test/std/language.support/support.dynamic/new.delete/new.delete.single/sized_delete.pass.cpp
@@ -11,10 +11,10 @@
 // UNSUPPORTED: c++03, c++11
 
 // These compiler versions and platforms don't enable sized deallocation by default.
-// ADDITIONAL_COMPILE_FLAGS(clang-17): -fsized-deallocation
 // ADDITIONAL_COMPILE_FLAGS(clang-18): -fsized-deallocation
 // ADDITIONAL_COMPILE_FLAGS(apple-clang-15): -fsized-deallocation
 // ADDITIONAL_COMPILE_FLAGS(apple-clang-16): -fsized-deallocation
+// ADDITIONAL_COMPILE_FLAGS(apple-clang-17): -fsized-deallocation
 // ADDITIONAL_COMPILE_FLAGS(target=x86_64-w64-windows-gnu): -fsized-deallocation
 // ADDITIONAL_COMPILE_FLAGS(target=i686-w64-windows-gnu): -fsized-deallocation
 
diff --git a/libcxx/test/std/language.support/support.limits/support.limits.general/flat_map.version.compile.pass.cpp b/libcxx/test/std/language.support/support.limits/support.limits.general/flat_map.version.compile.pass.cpp
new file mode 100644
index 0000000000000..0add849312d5e
--- /dev/null
+++ b/libcxx/test/std/language.support/support.limits/support.limits.general/flat_map.version.compile.pass.cpp
@@ -0,0 +1,68 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// WARNING: This test was generated by generate_feature_test_macro_components.py
+// and should not be edited manually.
+//
+// clang-format off
+
+// <flat_map>
+
+// Test the feature test macros defined by <flat_map>
+
+/*  Constant              Value
+    __cpp_lib_flat_map    202207L [C++23]
+*/
+
+#include <flat_map>
+#include "test_macros.h"
+
+#if TEST_STD_VER < 14
+
+# ifdef __cpp_lib_flat_map
+#   error "__cpp_lib_flat_map should not be defined before c++23"
+# endif
+
+#elif TEST_STD_VER == 14
+
+# ifdef __cpp_lib_flat_map
+#   error "__cpp_lib_flat_map should not be defined before c++23"
+# endif
+
+#elif TEST_STD_VER == 17
+
+# ifdef __cpp_lib_flat_map
+#   error "__cpp_lib_flat_map should not be defined before c++23"
+# endif
+
+#elif TEST_STD_VER == 20
+
+# ifdef __cpp_lib_flat_map
+#   error "__cpp_lib_flat_map should not be defined before c++23"
+# endif
+
+#elif TEST_STD_VER == 23
+
+# ifndef __cpp_lib_flat_map
+#   error "__cpp_lib_flat_map should be defined in c++23"
+# endif
+# if __cpp_lib_flat_map != 202207L
+#   error "__cpp_lib_flat_map should have the value 202207L in c++23"
+# endif
+
+#elif TEST_STD_VER > 23
+
+# ifndef __cpp_lib_flat_map
+#   error "__cpp_lib_flat_map should be defined in c++26"
+# endif
+# if __cpp_lib_flat_map != 202207L
+#   error "__cpp_lib_flat_map should have the value 202207L in c++26"
+# endif
+
+#endif // TEST_STD_VER > 23
+
diff --git a/libcxx/test/std/language.support/support.limits/support.limits.general/version.version.compile.pass.cpp b/libcxx/test/std/language.support/support.limits/support.limits.general/version.version.compile.pass.cpp
index 605788f559d3c..8f5788d2bed20 100644
--- a/libcxx/test/std/language.support/support.limits/support.limits.general/version.version.compile.pass.cpp
+++ b/libcxx/test/std/language.support/support.limits/support.limits.general/version.version.compile.pass.cpp
@@ -88,6 +88,8 @@
                                                             201902L [C++20]
     __cpp_lib_expected                                      202211L [C++23]
     __cpp_lib_filesystem                                    201703L [C++17]
+    __cpp_lib_flat_map                                      202207L [C++23]
+    __cpp_lib_flat_set                                      202207L [C++23]
     __cpp_lib_format                                        202110L [C++20]
     __cpp_lib_format_path                                   202403L [C++26]
     __cpp_lib_format_ranges                                 202207L [C++23]
@@ -528,6 +530,14 @@
 #   error "__cpp_lib_filesystem should not be defined before c++17"
 # endif
 
+# ifdef __cpp_lib_flat_map
+#   error "__cpp_lib_flat_map should not be defined before c++23"
+# endif
+
+# ifdef __cpp_lib_flat_set
+#   error "__cpp_lib_flat_set should not be defined before c++23"
+# endif
+
 # ifdef __cpp_lib_format
 #   error "__cpp_lib_format should not be defined before c++20"
 # endif
@@ -1399,6 +1409,14 @@
 #   error "__cpp_lib_filesystem should not be defined before c++17"
 # endif
 
+# ifdef __cpp_lib_flat_map
+#   error "__cpp_lib_flat_map should not be defined before c++23"
+# endif
+
+# ifdef __cpp_lib_flat_set
+#   error "__cpp_lib_flat_set should not be defined before c++23"
+# endif
+
 # ifdef __cpp_lib_format
 #   error "__cpp_lib_format should not be defined before c++20"
 # endif
@@ -2390,6 +2408,14 @@
 #   endif
 # endif
 
+# ifdef __cpp_lib_flat_map
+#   error "__cpp_lib_flat_map should not be defined before c++23"
+# endif
+
+# ifdef __cpp_lib_flat_set
+#   error "__cpp_lib_flat_set should not be defined before c++23"
+# endif
+
 # ifdef __cpp_lib_format
 #   error "__cpp_lib_format should not be defined before c++20"
 # endif
@@ -3651,6 +3677,14 @@
 #   endif
 # endif
 
+# ifdef __cpp_lib_flat_map
+#   error "__cpp_lib_flat_map should not be defined before c++23"
+# endif
+
+# ifdef __cpp_lib_flat_set
+#   error "__cpp_lib_flat_set should not be defined before c++23"
+# endif
+
 # ifndef __cpp_lib_format
 #   error "__cpp_lib_format should be defined in c++20"
 # endif
@@ -5092,6 +5126,26 @@
 #   endif
 # endif
 
+# ifndef __cpp_lib_flat_map
+#   error "__cpp_lib_flat_map should be defined in c++23"
+# endif
+# if __cpp_lib_flat_map != 202207L
+#   error "__cpp_lib_flat_map should have the value 202207L in c++23"
+# endif
+
+# if !defined(_LIBCPP_VERSION)
+#   ifndef __cpp_lib_flat_set
+#     error "__cpp_lib_flat_set should be defined in c++23"
+#   endif
+#   if __cpp_lib_flat_set != 202207L
+#     error "__cpp_lib_flat_set should have the value 202207L in c++23"
+#   endif
+# else // _LIBCPP_VERSION
+#   ifdef __cpp_lib_flat_set
+#     error "__cpp_lib_flat_set should not be defined because it is unimplemented in libc++!"
+#   endif
+# endif
+
 # ifndef __cpp_lib_format
 #   error "__cpp_lib_format should be defined in c++23"
 # endif
@@ -6779,6 +6833,26 @@
 #   endif
 # endif
 
+# ifndef __cpp_lib_flat_map
+#   error "__cpp_lib_flat_map should be defined in c++26"
+# endif
+# if __cpp_lib_flat_map != 202207L
+#   error "__cpp_lib_flat_map should have the value 202207L in c++26"
+# endif
+
+# if !defined(_LIBCPP_VERSION)
+#   ifndef __cpp_lib_flat_set
+#     error "__cpp_lib_flat_set should be defined in c++26"
+#   endif
+#   if __cpp_lib_flat_set != 202207L
+#     error "__cpp_lib_flat_set should have the value 202207L in c++26"
+#   endif
+# else // _LIBCPP_VERSION
+#   ifdef __cpp_lib_flat_set
+#     error "__cpp_lib_flat_set should not be defined because it is unimplemented in libc++!"
+#   endif
+# endif
+
 # ifndef __cpp_lib_format
 #   error "__cpp_lib_format should be defined in c++26"
 # endif
diff --git a/libcxx/test/std/localization/locale.stdcvt/depr.verify.cpp b/libcxx/test/std/localization/locale.stdcvt/depr.verify.cpp
index b3c6fc8674f8a..7bdcaa5190bd0 100644
--- a/libcxx/test/std/localization/locale.stdcvt/depr.verify.cpp
+++ b/libcxx/test/std/localization/locale.stdcvt/depr.verify.cpp
@@ -6,7 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-// UNSUPPORTED: c++03, c++11, c++14, c++26
+// REQUIRES: c++17 || c++20 || c++23
 // UNSUPPORTED: no-wide-characters
 
 // <codecvt>
diff --git a/libcxx/test/std/localization/locales/locale.convenience/conversions/conversions.buffer/depr.verify.cpp b/libcxx/test/std/localization/locales/locale.convenience/conversions/conversions.buffer/depr.verify.cpp
index cb067e99a4764..dcab5cef3a550 100644
--- a/libcxx/test/std/localization/locales/locale.convenience/conversions/conversions.buffer/depr.verify.cpp
+++ b/libcxx/test/std/localization/locales/locale.convenience/conversions/conversions.buffer/depr.verify.cpp
@@ -6,7 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-// UNSUPPORTED: c++03, c++11, c++14, c++26
+// REQUIRES: c++17 || c++20 || c++23
 
 // XFAIL: no-wide-characters
 
diff --git a/libcxx/test/std/localization/locales/locale.convenience/conversions/conversions.string/depr.verify.cpp b/libcxx/test/std/localization/locales/locale.convenience/conversions/conversions.string/depr.verify.cpp
index f8bd156bdd5f6..6eab4a5dd9223 100644
--- a/libcxx/test/std/localization/locales/locale.convenience/conversions/conversions.string/depr.verify.cpp
+++ b/libcxx/test/std/localization/locales/locale.convenience/conversions/conversions.string/depr.verify.cpp
@@ -8,7 +8,7 @@
 
 // ADDITIONAL_COMPILE_FLAGS: -D_LIBCPP_ENABLE_CXX26_REMOVED_WSTRING_CONVERT
 
-// UNSUPPORTED: c++03, c++11, c++14, c++26
+// REQUIRES: c++17 || c++20 || c++23
 // UNSUPPORTED: no-wide-characters
 
 // <codecvt>
diff --git a/libcxx/test/std/numerics/c.math/cmath.pass.cpp b/libcxx/test/std/numerics/c.math/cmath.pass.cpp
index fc3dcfd7b4834..48c2918802fc3 100644
--- a/libcxx/test/std/numerics/c.math/cmath.pass.cpp
+++ b/libcxx/test/std/numerics/c.math/cmath.pass.cpp
@@ -8,7 +8,7 @@
 
 // This test fails because Clang no longer enables -fdelayed-template-parsing
 // by default on Windows with C++20 (#69431).
-// XFAIL: msvc && (clang-18 || clang-19 || clang-20)
+// XFAIL: msvc && (clang-18 || clang-19 || clang-20 || clang-21)
 
 // <cmath>
 
diff --git a/libcxx/test/std/numerics/c.math/signbit.pass.cpp b/libcxx/test/std/numerics/c.math/signbit.pass.cpp
index cbad968a8aa20..b5e63dedf136e 100644
--- a/libcxx/test/std/numerics/c.math/signbit.pass.cpp
+++ b/libcxx/test/std/numerics/c.math/signbit.pass.cpp
@@ -12,7 +12,7 @@
 // UNSUPPORTED: windows
 
 // These compilers don't support constexpr `__builtin_signbit` yet.
-// UNSUPPORTED: clang-17, clang-18, clang-19, apple-clang-15, apple-clang-16
+// UNSUPPORTED: clang-18, clang-19, apple-clang-15, apple-clang-16, apple-clang-17
 
 // XFAIL: FROZEN-CXX03-HEADERS-FIXME
 
diff --git a/libcxx/test/std/strings/basic.string/string.capacity/reserve.deprecated_in_cxx20.verify.cpp b/libcxx/test/std/strings/basic.string/string.capacity/reserve.deprecated_in_cxx20.verify.cpp
index 81edd9b83d184..87b56c06b9512 100644
--- a/libcxx/test/std/strings/basic.string/string.capacity/reserve.deprecated_in_cxx20.verify.cpp
+++ b/libcxx/test/std/strings/basic.string/string.capacity/reserve.deprecated_in_cxx20.verify.cpp
@@ -10,7 +10,7 @@
 
 // void reserve(); // Deprecated in C++20
 
-// UNSUPPORTED: c++03, c++11, c++14, c++17, c++26
+// REQUIRES: c++20 || c++23
 
 #include <string>
 
diff --git a/libcxx/test/std/strings/basic.string/string.capacity/reserve_size.pass.cpp b/libcxx/test/std/strings/basic.string/string.capacity/reserve_size.pass.cpp
index 6196fd6836e7d..cb1e6b18ed450 100644
--- a/libcxx/test/std/strings/basic.string/string.capacity/reserve_size.pass.cpp
+++ b/libcxx/test/std/strings/basic.string/string.capacity/reserve_size.pass.cpp
@@ -75,6 +75,15 @@ TEST_CONSTEXPR_CXX20 void test_string() {
     test<S>(100, 50, 1000);
     test<S>(100, 50, S::npos);
   }
+
+  { // Check that growing twice works as expected
+    S str;
+    str.reserve(50);
+    assert(str.capacity() >= 50);
+    size_t old_cap = str.capacity();
+    str.reserve(str.capacity() + 1);
+    assert(str.capacity() > old_cap);
+  }
 }
 
 TEST_CONSTEXPR_CXX20 bool test() {
diff --git a/libcxx/test/std/strings/basic.string/string.nonmembers/string_op+/char_string.pass.cpp b/libcxx/test/std/strings/basic.string/string.nonmembers/string_op+/char_string.pass.cpp
index 2ca5788e6d8a3..68a250c593318 100644
--- a/libcxx/test/std/strings/basic.string/string.nonmembers/string_op+/char_string.pass.cpp
+++ b/libcxx/test/std/strings/basic.string/string.nonmembers/string_op+/char_string.pass.cpp
@@ -16,39 +16,32 @@
 //   basic_string<charT,traits,Allocator>&&
 //   operator+(charT lhs, basic_string<charT,traits,Allocator>&& rhs); // constexpr since C++20
 
+#include <cassert>
 #include <string>
 #include <utility>
-#include <cassert>
 
 #include "test_macros.h"
 #include "min_allocator.h"
 #include "asan_testing.h"
 
-template <class S>
-TEST_CONSTEXPR_CXX20 void test0(typename S::value_type lhs, const S& rhs, const S& x) {
-  assert(lhs + rhs == x);
-  LIBCPP_ASSERT(is_string_asan_correct(lhs + rhs));
-}
-
-#if TEST_STD_VER >= 11
-template <class S>
-TEST_CONSTEXPR_CXX20 void test1(typename S::value_type lhs, S&& rhs, const S& x) {
-  assert(lhs + std::move(rhs) == x);
-}
-#endif
-
 template <class S>
 TEST_CONSTEXPR_CXX20 void test_string() {
-  test0('a', S(""), S("a"));
-  test0('a', S("12345"), S("a12345"));
-  test0('a', S("1234567890"), S("a1234567890"));
-  test0('a', S("12345678901234567890"), S("a12345678901234567890"));
+  const char* test_data[] = {"", "12345", "1234567890", "12345678901234567890"};
+  const char* results[]   = {"a", "a12345", "a1234567890", "a12345678901234567890"};
+
+  for (size_t i = 0; i != 4; ++i) {
+    { // operator+(value_type, const string&);
+      const S str(test_data[i]);
+      assert('a' + str == results[i]);
+      LIBCPP_ASSERT(is_string_asan_correct('a' + str));
+    }
 #if TEST_STD_VER >= 11
-  test1('a', S(""), S("a"));
-  test1('a', S("12345"), S("a12345"));
-  test1('a', S("1234567890"), S("a1234567890"));
-  test1('a', S("12345678901234567890"), S("a12345678901234567890"));
+    { // operator+(value_type, string&&);
+      S str(test_data[i]);
+      assert('a' + std::move(str) == results[i]);
+    }
 #endif
+  }
 }
 
 TEST_CONSTEXPR_CXX20 bool test() {
@@ -63,7 +56,7 @@ TEST_CONSTEXPR_CXX20 bool test() {
 
 int main(int, char**) {
   test();
-#if TEST_STD_VER > 17
+#if TEST_STD_VER >= 20
   static_assert(test());
 #endif
 
diff --git a/libcxx/test/std/strings/basic.string/string.nonmembers/string_op+/pointer_string.pass.cpp b/libcxx/test/std/strings/basic.string/string.nonmembers/string_op+/pointer_string.pass.cpp
index fa14f657d8269..434f7292c149c 100644
--- a/libcxx/test/std/strings/basic.string/string.nonmembers/string_op+/pointer_string.pass.cpp
+++ b/libcxx/test/std/strings/basic.string/string.nonmembers/string_op+/pointer_string.pass.cpp
@@ -16,68 +16,49 @@
 //   basic_string<charT,traits,Allocator>&&
 //   operator+(const charT* lhs, basic_string<charT,traits,Allocator>&& rhs); // constexpr since C++20
 
+#include <cassert>
 #include <string>
 #include <utility>
-#include <cassert>
 
 #include "test_macros.h"
 #include "min_allocator.h"
 
-template <class S>
-TEST_CONSTEXPR_CXX20 void test0(const typename S::value_type* lhs, const S& rhs, const S& x) {
-  assert(lhs + rhs == x);
-}
-
-#if TEST_STD_VER >= 11
-template <class S>
-TEST_CONSTEXPR_CXX20 void test1(const typename S::value_type* lhs, S&& rhs, const S& x) {
-  assert(lhs + std::move(rhs) == x);
-}
-#endif
-
 template <class S>
 TEST_CONSTEXPR_CXX20 void test_string() {
-  test0("", S(""), S(""));
-  test0("", S("12345"), S("12345"));
-  test0("", S("1234567890"), S("1234567890"));
-  test0("", S("12345678901234567890"), S("12345678901234567890"));
-  test0("abcde", S(""), S("abcde"));
-  test0("abcde", S("12345"), S("abcde12345"));
-  test0("abcde", S("1234567890"), S("abcde1234567890"));
-  test0("abcde", S("12345678901234567890"), S("abcde12345678901234567890"));
-  test0("abcdefghij", S(""), S("abcdefghij"));
-  test0("abcdefghij", S("12345"), S("abcdefghij12345"));
-  test0("abcdefghij", S("1234567890"), S("abcdefghij1234567890"));
-  test0("abcdefghij", S("12345678901234567890"), S("abcdefghij12345678901234567890"));
-  test0("abcdefghijklmnopqrst", S(""), S("abcdefghijklmnopqrst"));
-  test0("abcdefghijklmnopqrst", S("12345"), S("abcdefghijklmnopqrst12345"));
-  test0("abcdefghijklmnopqrst", S("1234567890"), S("abcdefghijklmnopqrst1234567890"));
-  test0("abcdefghijklmnopqrst", S("12345678901234567890"), S("abcdefghijklmnopqrst12345678901234567890"));
+  const char* test_data[2][4] = {
+      {"", "abcde", "abcdefghij", "abcdefghijklmnopqrst"}, {"", "12345", "1234567890", "12345678901234567890"}};
+
+  const char* results[4][4] = {
+      {"", "12345", "1234567890", "12345678901234567890"},
+      {"abcde", "abcde12345", "abcde1234567890", "abcde12345678901234567890"},
+      {"abcdefghij", "abcdefghij12345", "abcdefghij1234567890", "abcdefghij12345678901234567890"},
+      {"abcdefghijklmnopqrst",
+       "abcdefghijklmnopqrst12345",
+       "abcdefghijklmnopqrst1234567890",
+       "abcdefghijklmnopqrst12345678901234567890"}};
 
+  for (size_t i = 0; i != 4; ++i) {
+    for (size_t k = 0; k != 4; ++k) {
+      { // operator+(const value_type*, const string&);
+        const char* lhs = test_data[0][i];
+        const S rhs(test_data[1][k]);
+        assert(lhs + rhs == results[i][k]);
+      }
 #if TEST_STD_VER >= 11
-  test1("", S(""), S(""));
-  test1("", S("12345"), S("12345"));
-  test1("", S("1234567890"), S("1234567890"));
-  test1("", S("12345678901234567890"), S("12345678901234567890"));
-  test1("abcde", S(""), S("abcde"));
-  test1("abcde", S("12345"), S("abcde12345"));
-  test1("abcde", S("1234567890"), S("abcde1234567890"));
-  test1("abcde", S("12345678901234567890"), S("abcde12345678901234567890"));
-  test1("abcdefghij", S(""), S("abcdefghij"));
-  test1("abcdefghij", S("12345"), S("abcdefghij12345"));
-  test1("abcdefghij", S("1234567890"), S("abcdefghij1234567890"));
-  test1("abcdefghij", S("12345678901234567890"), S("abcdefghij12345678901234567890"));
-  test1("abcdefghijklmnopqrst", S(""), S("abcdefghijklmnopqrst"));
-  test1("abcdefghijklmnopqrst", S("12345"), S("abcdefghijklmnopqrst12345"));
-  test1("abcdefghijklmnopqrst", S("1234567890"), S("abcdefghijklmnopqrst1234567890"));
-  test1("abcdefghijklmnopqrst", S("12345678901234567890"), S("abcdefghijklmnopqrst12345678901234567890"));
+      { // operator+(const value_type*, string&&);
+        const char* lhs = test_data[0][i];
+        S rhs(test_data[1][k]);
+        assert(lhs + std::move(rhs) == results[i][k]);
+      }
 #endif
+    }
+  }
 }
 
 TEST_CONSTEXPR_CXX20 bool test() {
   test_string<std::string>();
 #if TEST_STD_VER >= 11
-  test_string<std::basic_string<char, std::char_traits<char>, min_allocator<char> > >();
+  test_string<std::basic_string<char, std::char_traits<char>, min_allocator<char>>>();
 #endif
 
   return true;
@@ -85,7 +66,7 @@ TEST_CONSTEXPR_CXX20 bool test() {
 
 int main(int, char**) {
   test();
-#if TEST_STD_VER > 17
+#if TEST_STD_VER >= 20
   static_assert(test());
 #endif
 
diff --git a/libcxx/test/std/strings/basic.string/string.nonmembers/string_op+/string_char.pass.cpp b/libcxx/test/std/strings/basic.string/string.nonmembers/string_op+/string_char.pass.cpp
index 9c0fb56e1f1c9..5c3989ae304e9 100644
--- a/libcxx/test/std/strings/basic.string/string.nonmembers/string_op+/string_char.pass.cpp
+++ b/libcxx/test/std/strings/basic.string/string.nonmembers/string_op+/string_char.pass.cpp
@@ -16,41 +16,32 @@
 //   basic_string<charT,traits,Allocator>&&
 //   operator+(basic_string<charT,traits,Allocator>&& lhs, charT rhs); // constexpr since C++20
 
+#include <cassert>
 #include <string>
 #include <utility>
-#include <cassert>
 
-#include "test_macros.h"
-#include "min_allocator.h"
 #include "asan_testing.h"
-
-template <class S>
-TEST_CONSTEXPR_CXX20 void test0(const S& lhs, typename S::value_type rhs, const S& x) {
-  assert(lhs + rhs == x);
-  LIBCPP_ASSERT(is_string_asan_correct(lhs + rhs));
-}
-
-#if TEST_STD_VER >= 11
-template <class S>
-TEST_CONSTEXPR_CXX20 void test1(S&& lhs, typename S::value_type rhs, const S& x) {
-  assert(std::move(lhs) + rhs == x);
-}
-#endif
+#include "min_allocator.h"
+#include "test_macros.h"
 
 template <class S>
 TEST_CONSTEXPR_CXX20 void test_string() {
-  test0(S(""), '1', S("1"));
-  test0(S(""), '1', S("1"));
-  test0(S("abcde"), '1', S("abcde1"));
-  test0(S("abcdefghij"), '1', S("abcdefghij1"));
-  test0(S("abcdefghijklmnopqrst"), '1', S("abcdefghijklmnopqrst1"));
+  const char* test_data[] = {"", "12345", "1234567890", "12345678901234567890"};
+  const char* results[]   = {"a", "12345a", "1234567890a", "12345678901234567890a"};
 
+  for (size_t i = 0; i != 4; ++i) {
+    { // operator+(const string&, value_type);
+      const S str(test_data[i]);
+      assert(str + 'a' == results[i]);
+      LIBCPP_ASSERT(is_string_asan_correct(str + 'a'));
+    }
 #if TEST_STD_VER >= 11
-  test1(S(""), '1', S("1"));
-  test1(S("abcde"), '1', S("abcde1"));
-  test1(S("abcdefghij"), '1', S("abcdefghij1"));
-  test1(S("abcdefghijklmnopqrst"), '1', S("abcdefghijklmnopqrst1"));
+    { // operator+(string&&, value_type);
+      S str(test_data[i]);
+      assert(std::move(str) + 'a' == results[i]);
+    }
 #endif
+  }
 }
 
 TEST_CONSTEXPR_CXX20 bool test() {
@@ -65,7 +56,7 @@ TEST_CONSTEXPR_CXX20 bool test() {
 
 int main(int, char**) {
   test();
-#if TEST_STD_VER > 17
+#if TEST_STD_VER >= 20
   static_assert(test());
 #endif
 
diff --git a/libcxx/test/std/strings/basic.string/string.nonmembers/string_op+/string_pointer.pass.cpp b/libcxx/test/std/strings/basic.string/string.nonmembers/string_op+/string_pointer.pass.cpp
index 3e1aaebf112eb..589c737348a11 100644
--- a/libcxx/test/std/strings/basic.string/string.nonmembers/string_op+/string_pointer.pass.cpp
+++ b/libcxx/test/std/strings/basic.string/string.nonmembers/string_op+/string_pointer.pass.cpp
@@ -16,13 +16,13 @@
 //   basic_string<charT,traits,Allocator>&&
 //   operator+(basic_string<charT,traits,Allocator>&& lhs, const charT* rhs); // constexpr since C++20
 
+#include <cassert>
 #include <string>
 #include <utility>
-#include <cassert>
 
-#include "test_macros.h"
-#include "min_allocator.h"
 #include "asan_testing.h"
+#include "min_allocator.h"
+#include "test_macros.h"
 
 template <class S>
 TEST_CONSTEXPR_CXX20 void test0(const S& lhs, const typename S::value_type* rhs, const S& x) {
@@ -39,40 +39,34 @@ TEST_CONSTEXPR_CXX20 void test1(S&& lhs, const typename S::value_type* rhs, cons
 
 template <class S>
 TEST_CONSTEXPR_CXX20 void test_string() {
-  test0(S(""), "", S(""));
-  test0(S(""), "12345", S("12345"));
-  test0(S(""), "1234567890", S("1234567890"));
-  test0(S(""), "12345678901234567890", S("12345678901234567890"));
-  test0(S("abcde"), "", S("abcde"));
-  test0(S("abcde"), "12345", S("abcde12345"));
-  test0(S("abcde"), "1234567890", S("abcde1234567890"));
-  test0(S("abcde"), "12345678901234567890", S("abcde12345678901234567890"));
-  test0(S("abcdefghij"), "", S("abcdefghij"));
-  test0(S("abcdefghij"), "12345", S("abcdefghij12345"));
-  test0(S("abcdefghij"), "1234567890", S("abcdefghij1234567890"));
-  test0(S("abcdefghij"), "12345678901234567890", S("abcdefghij12345678901234567890"));
-  test0(S("abcdefghijklmnopqrst"), "", S("abcdefghijklmnopqrst"));
-  test0(S("abcdefghijklmnopqrst"), "12345", S("abcdefghijklmnopqrst12345"));
-  test0(S("abcdefghijklmnopqrst"), "1234567890", S("abcdefghijklmnopqrst1234567890"));
-  test0(S("abcdefghijklmnopqrst"), "12345678901234567890", S("abcdefghijklmnopqrst12345678901234567890"));
+  const char* test_data[2][4] = {
+      {"", "abcde", "abcdefghij", "abcdefghijklmnopqrst"}, {"", "12345", "1234567890", "12345678901234567890"}};
+
+  const char* results[4][4] = {
+      {"", "12345", "1234567890", "12345678901234567890"},
+      {"abcde", "abcde12345", "abcde1234567890", "abcde12345678901234567890"},
+      {"abcdefghij", "abcdefghij12345", "abcdefghij1234567890", "abcdefghij12345678901234567890"},
+      {"abcdefghijklmnopqrst",
+       "abcdefghijklmnopqrst12345",
+       "abcdefghijklmnopqrst1234567890",
+       "abcdefghijklmnopqrst12345678901234567890"}};
+
+  for (size_t i = 0; i != 4; ++i) {
+    for (size_t k = 0; k != 4; ++k) {
+      { // operator+(const value_type*, const string&);
+        const S lhs(test_data[0][i]);
+        const char* rhs = test_data[1][k];
+        assert(lhs + rhs == results[i][k]);
+      }
 #if TEST_STD_VER >= 11
-  test1(S(""), "", S(""));
-  test1(S(""), "12345", S("12345"));
-  test1(S(""), "1234567890", S("1234567890"));
-  test1(S(""), "12345678901234567890", S("12345678901234567890"));
-  test1(S("abcde"), "", S("abcde"));
-  test1(S("abcde"), "12345", S("abcde12345"));
-  test1(S("abcde"), "1234567890", S("abcde1234567890"));
-  test1(S("abcde"), "12345678901234567890", S("abcde12345678901234567890"));
-  test1(S("abcdefghij"), "", S("abcdefghij"));
-  test1(S("abcdefghij"), "12345", S("abcdefghij12345"));
-  test1(S("abcdefghij"), "1234567890", S("abcdefghij1234567890"));
-  test1(S("abcdefghij"), "12345678901234567890", S("abcdefghij12345678901234567890"));
-  test1(S("abcdefghijklmnopqrst"), "", S("abcdefghijklmnopqrst"));
-  test1(S("abcdefghijklmnopqrst"), "12345", S("abcdefghijklmnopqrst12345"));
-  test1(S("abcdefghijklmnopqrst"), "1234567890", S("abcdefghijklmnopqrst1234567890"));
-  test1(S("abcdefghijklmnopqrst"), "12345678901234567890", S("abcdefghijklmnopqrst12345678901234567890"));
+      { // operator+(const value_type*, string&&);
+        S lhs(test_data[0][i]);
+        const char* rhs = test_data[1][k];
+        assert(std::move(lhs) + rhs == results[i][k]);
+      }
 #endif
+    }
+  }
 }
 
 TEST_CONSTEXPR_CXX20 bool test() {
@@ -87,7 +81,7 @@ TEST_CONSTEXPR_CXX20 bool test() {
 
 int main(int, char**) {
   test();
-#if TEST_STD_VER > 17
+#if TEST_STD_VER >= 20
   static_assert(test());
 #endif
 
diff --git a/libcxx/test/std/strings/basic.string/string.nonmembers/string_op+/string_string.pass.cpp b/libcxx/test/std/strings/basic.string/string.nonmembers/string_op+/string_string.pass.cpp
index fcc8ceab87071..2badb51753395 100644
--- a/libcxx/test/std/strings/basic.string/string.nonmembers/string_op+/string_string.pass.cpp
+++ b/libcxx/test/std/strings/basic.string/string.nonmembers/string_op+/string_string.pass.cpp
@@ -28,114 +28,62 @@
 //   operator+(const basic_string<charT,traits,Allocator>&& lhs,
 //             const basic_string<charT,traits,Allocator>&& rhs); // constexpr since C++20
 
+#include <cassert>
 #include <string>
 #include <utility>
-#include <cassert>
 
-#include "test_macros.h"
-#include "min_allocator.h"
 #include "asan_testing.h"
-
-template <class S>
-TEST_CONSTEXPR_CXX20 void test0(const S& lhs, const S& rhs, const S& x) {
-  assert(lhs + rhs == x);
-  LIBCPP_ASSERT(is_string_asan_correct(lhs + rhs));
-}
-
-#if TEST_STD_VER >= 11
-template <class S>
-TEST_CONSTEXPR_CXX20 void test1(S&& lhs, const S& rhs, const S& x) {
-  assert(std::move(lhs) + rhs == x);
-}
-
-template <class S>
-TEST_CONSTEXPR_CXX20 void test2(const S& lhs, S&& rhs, const S& x) {
-  assert(lhs + std::move(rhs) == x);
-}
-
-template <class S>
-TEST_CONSTEXPR_CXX20 void test3(S&& lhs, S&& rhs, const S& x) {
-  assert(std::move(lhs) + std::move(rhs) == x);
-}
-#endif
+#include "min_allocator.h"
+#include "test_macros.h"
 
 template <class S>
 TEST_CONSTEXPR_CXX20 void test_string() {
-  test0(S(""), S(""), S(""));
-  test0(S(""), S("12345"), S("12345"));
-  test0(S(""), S("1234567890"), S("1234567890"));
-  test0(S(""), S("12345678901234567890"), S("12345678901234567890"));
-  test0(S("abcde"), S(""), S("abcde"));
-  test0(S("abcde"), S("12345"), S("abcde12345"));
-  test0(S("abcde"), S("1234567890"), S("abcde1234567890"));
-  test0(S("abcde"), S("12345678901234567890"), S("abcde12345678901234567890"));
-  test0(S("abcdefghij"), S(""), S("abcdefghij"));
-  test0(S("abcdefghij"), S("12345"), S("abcdefghij12345"));
-  test0(S("abcdefghij"), S("1234567890"), S("abcdefghij1234567890"));
-  test0(S("abcdefghij"), S("12345678901234567890"), S("abcdefghij12345678901234567890"));
-  test0(S("abcdefghijklmnopqrst"), S(""), S("abcdefghijklmnopqrst"));
-  test0(S("abcdefghijklmnopqrst"), S("12345"), S("abcdefghijklmnopqrst12345"));
-  test0(S("abcdefghijklmnopqrst"), S("1234567890"), S("abcdefghijklmnopqrst1234567890"));
-  test0(S("abcdefghijklmnopqrst"), S("12345678901234567890"), S("abcdefghijklmnopqrst12345678901234567890"));
+  const char* test_data[2][4] = {
+      {"", "abcde", "abcdefghij", "abcdefghijklmnopqrst"}, {"", "12345", "1234567890", "12345678901234567890"}};
+
+  const char* results[4][4] = {
+      {"", "12345", "1234567890", "12345678901234567890"},
+      {"abcde", "abcde12345", "abcde1234567890", "abcde12345678901234567890"},
+      {"abcdefghij", "abcdefghij12345", "abcdefghij1234567890", "abcdefghij12345678901234567890"},
+      {"abcdefghijklmnopqrst",
+       "abcdefghijklmnopqrst12345",
+       "abcdefghijklmnopqrst1234567890",
+       "abcdefghijklmnopqrst12345678901234567890"}};
+
+  for (size_t i = 0; i != 4; ++i) {
+    for (size_t k = 0; k != 4; ++k) {
+      { // operator+(const string&, const string&);
+        const S lhs(test_data[0][i]);
+        const S rhs(test_data[1][k]);
+        assert(lhs + rhs == results[i][k]);
+        LIBCPP_ASSERT(is_string_asan_correct(lhs + rhs));
+      }
 #if TEST_STD_VER >= 11
-  test1(S(""), S(""), S(""));
-  test1(S(""), S("12345"), S("12345"));
-  test1(S(""), S("1234567890"), S("1234567890"));
-  test1(S(""), S("12345678901234567890"), S("12345678901234567890"));
-  test1(S("abcde"), S(""), S("abcde"));
-  test1(S("abcde"), S("12345"), S("abcde12345"));
-  test1(S("abcde"), S("1234567890"), S("abcde1234567890"));
-  test1(S("abcde"), S("12345678901234567890"), S("abcde12345678901234567890"));
-  test1(S("abcdefghij"), S(""), S("abcdefghij"));
-  test1(S("abcdefghij"), S("12345"), S("abcdefghij12345"));
-  test1(S("abcdefghij"), S("1234567890"), S("abcdefghij1234567890"));
-  test1(S("abcdefghij"), S("12345678901234567890"), S("abcdefghij12345678901234567890"));
-  test1(S("abcdefghijklmnopqrst"), S(""), S("abcdefghijklmnopqrst"));
-  test1(S("abcdefghijklmnopqrst"), S("12345"), S("abcdefghijklmnopqrst12345"));
-  test1(S("abcdefghijklmnopqrst"), S("1234567890"), S("abcdefghijklmnopqrst1234567890"));
-  test1(S("abcdefghijklmnopqrst"), S("12345678901234567890"), S("abcdefghijklmnopqrst12345678901234567890"));
-
-  test2(S(""), S(""), S(""));
-  test2(S(""), S("12345"), S("12345"));
-  test2(S(""), S("1234567890"), S("1234567890"));
-  test2(S(""), S("12345678901234567890"), S("12345678901234567890"));
-  test2(S("abcde"), S(""), S("abcde"));
-  test2(S("abcde"), S("12345"), S("abcde12345"));
-  test2(S("abcde"), S("1234567890"), S("abcde1234567890"));
-  test2(S("abcde"), S("12345678901234567890"), S("abcde12345678901234567890"));
-  test2(S("abcdefghij"), S(""), S("abcdefghij"));
-  test2(S("abcdefghij"), S("12345"), S("abcdefghij12345"));
-  test2(S("abcdefghij"), S("1234567890"), S("abcdefghij1234567890"));
-  test2(S("abcdefghij"), S("12345678901234567890"), S("abcdefghij12345678901234567890"));
-  test2(S("abcdefghijklmnopqrst"), S(""), S("abcdefghijklmnopqrst"));
-  test2(S("abcdefghijklmnopqrst"), S("12345"), S("abcdefghijklmnopqrst12345"));
-  test2(S("abcdefghijklmnopqrst"), S("1234567890"), S("abcdefghijklmnopqrst1234567890"));
-  test2(S("abcdefghijklmnopqrst"), S("12345678901234567890"), S("abcdefghijklmnopqrst12345678901234567890"));
-
-  test3(S(""), S(""), S(""));
-  test3(S(""), S("12345"), S("12345"));
-  test3(S(""), S("1234567890"), S("1234567890"));
-  test3(S(""), S("12345678901234567890"), S("12345678901234567890"));
-  test3(S("abcde"), S(""), S("abcde"));
-  test3(S("abcde"), S("12345"), S("abcde12345"));
-  test3(S("abcde"), S("1234567890"), S("abcde1234567890"));
-  test3(S("abcde"), S("12345678901234567890"), S("abcde12345678901234567890"));
-  test3(S("abcdefghij"), S(""), S("abcdefghij"));
-  test3(S("abcdefghij"), S("12345"), S("abcdefghij12345"));
-  test3(S("abcdefghij"), S("1234567890"), S("abcdefghij1234567890"));
-  test3(S("abcdefghij"), S("12345678901234567890"), S("abcdefghij12345678901234567890"));
-  test3(S("abcdefghijklmnopqrst"), S(""), S("abcdefghijklmnopqrst"));
-  test3(S("abcdefghijklmnopqrst"), S("12345"), S("abcdefghijklmnopqrst12345"));
-  test3(S("abcdefghijklmnopqrst"), S("1234567890"), S("abcdefghijklmnopqrst1234567890"));
-  test3(S("abcdefghijklmnopqrst"), S("12345678901234567890"), S("abcdefghijklmnopqrst12345678901234567890"));
+      { // operator+(string&&, const string&);
+        S lhs(test_data[0][i]);
+        const S rhs(test_data[1][k]);
+        assert(std::move(lhs) + rhs == results[i][k]);
+      }
+      { // operator+(const string&, string&&);
+        const S lhs(test_data[0][i]);
+        S rhs(test_data[1][k]);
+        assert(lhs + std::move(rhs) == results[i][k]);
+      }
+      { // operator+(string&&, string&&);
+        S lhs(test_data[0][i]);
+        S rhs(test_data[1][k]);
+        assert(std::move(lhs) + std::move(rhs) == results[i][k]);
+      }
 #endif
+    }
+  }
 }
 
 TEST_CONSTEXPR_CXX20 bool test() {
   test_string<std::string>();
 #if TEST_STD_VER >= 11
-  test_string<std::basic_string<char, std::char_traits<char>, min_allocator<char> > >();
-  test_string<std::basic_string<char, std::char_traits<char>, safe_allocator<char> > >();
+  test_string<std::basic_string<char, std::char_traits<char>, min_allocator<char>>>();
+  test_string<std::basic_string<char, std::char_traits<char>, safe_allocator<char>>>();
 #endif
 
   return true;
@@ -143,7 +91,7 @@ TEST_CONSTEXPR_CXX20 bool test() {
 
 int main(int, char**) {
   test();
-#if TEST_STD_VER > 17
+#if TEST_STD_VER >= 20
   static_assert(test());
 #endif
 
diff --git a/libcxx/test/std/time/time.clock/time.clock.tai/tai_time.ostream.pass.cpp b/libcxx/test/std/time/time.clock/time.clock.tai/tai_time.ostream.pass.cpp
new file mode 100644
index 0000000000000..e4f953118fd43
--- /dev/null
+++ b/libcxx/test/std/time/time.clock/time.clock.tai/tai_time.ostream.pass.cpp
@@ -0,0 +1,164 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// REQUIRES: std-at-least-c++20
+// UNSUPPORTED: no-filesystem, no-localization, no-tzdb
+// UNSUPPORTED: GCC-ALWAYS_INLINE-FIXME
+
+// TODO FMT This test should not require std::to_chars(floating-point)
+// XFAIL: availability-fp_to_chars-missing
+
+// XFAIL: libcpp-has-no-experimental-tzdb
+// XFAIL: availability-tzdb-missing
+
+// REQUIRES: locale.fr_FR.UTF-8
+// REQUIRES: locale.ja_JP.UTF-8
+
+// <chrono>
+
+// class tai_clock;
+
+// template<class charT, class traits, class Duration>
+//   basic_ostream<charT, traits>&
+//     operator<<(basic_ostream<charT, traits>& os, const tai_time<Duration>& tp);
+
+#include <chrono>
+#include <cassert>
+#include <ratio>
+#include <sstream>
+
+#include "make_string.h"
+#include "platform_support.h" // locale name macros
+#include "test_macros.h"
+
+#define SV(S) MAKE_STRING_VIEW(CharT, S)
+
+template <class CharT, class Duration>
+static std::basic_string<CharT> stream_c_locale(std::chrono::tai_time<Duration> time_point) {
+  std::basic_stringstream<CharT> sstr;
+  sstr << std::fixed << time_point;
+  return sstr.str();
+}
+
+template <class CharT, class Duration>
+static std::basic_string<CharT> stream_fr_FR_locale(std::chrono::tai_time<Duration> time_point) {
+  std::basic_stringstream<CharT> sstr;
+  const std::locale locale(LOCALE_fr_FR_UTF_8);
+  sstr.imbue(locale);
+  sstr << std::fixed << time_point;
+  return sstr.str();
+}
+
+template <class CharT, class Duration>
+static std::basic_string<CharT> stream_ja_JP_locale(std::chrono::tai_time<Duration> time_point) {
+  std::basic_stringstream<CharT> sstr;
+  const std::locale locale(LOCALE_ja_JP_UTF_8);
+  sstr.imbue(locale);
+  sstr << std::fixed << time_point;
+  return sstr.str();
+}
+
+template <class CharT>
+static void test_c() {
+  using namespace std::literals::chrono_literals;
+  namespace cr = std::chrono;
+
+  assert(stream_c_locale<CharT>(cr::tai_time<cr::nanoseconds>{946'688'523'123'456'789ns}) ==
+         SV("1988-01-01 01:02:03.123456789"));
+  assert(stream_c_locale<CharT>(cr::tai_time<cr::microseconds>{946'688'523'123'456us}) ==
+         SV("1988-01-01 01:02:03.123456"));
+
+  assert(stream_c_locale<CharT>(cr::tai_time<cr::milliseconds>{946'684'822'123ms}) == SV("1988-01-01 00:00:22.123"));
+  assert(stream_c_locale<CharT>(cr::tai_seconds{1'234'567'890s}) == SV("1997-02-13 23:31:30"));
+  assert(stream_c_locale<CharT>(cr::tai_time<cr::minutes>{20'576'131min}) == SV("1997-02-13 23:31:00"));
+  assert(stream_c_locale<CharT>(cr::tai_time<cr::hours>{342'935h}) == SV("1997-02-13 23:00:00"));
+
+  assert(stream_c_locale<CharT>(cr::tai_time<cr::duration<signed char, std::ratio<2, 1>>>{
+             cr::duration<signed char, std::ratio<2, 1>>{60}}) == SV("1958-01-01 00:02:00"));
+  assert(stream_c_locale<CharT>(cr::tai_time<cr::duration<short, std::ratio<1, 2>>>{
+             cr::duration<short, std::ratio<1, 2>>{3600}}) == SV("1958-01-01 00:30:00.0"));
+  assert(stream_c_locale<CharT>(cr::tai_time<cr::duration<int, std::ratio<1, 4>>>{
+             cr::duration<int, std::ratio<1, 4>>{3600}}) == SV("1958-01-01 00:15:00.00"));
+  assert(stream_c_locale<CharT>(cr::tai_time<cr::duration<long, std::ratio<1, 10>>>{
+             cr::duration<long, std::ratio<1, 10>>{36611}}) == SV("1958-01-01 01:01:01.1"));
+  assert(stream_c_locale<CharT>(cr::tai_time<cr::duration<long long, std::ratio<1, 100>>>{
+             cr::duration<long long, std::ratio<1, 100>>{12'345'678'9010}}) == SV("1997-02-13 23:31:30.10"));
+}
+
+template <class CharT>
+static void test_fr_FR() {
+  using namespace std::literals::chrono_literals;
+  namespace cr = std::chrono;
+
+  assert(stream_fr_FR_locale<CharT>(cr::tai_time<cr::nanoseconds>{946'688'523'123'456'789ns}) ==
+         SV("1988-01-01 01:02:03,123456789"));
+  assert(stream_fr_FR_locale<CharT>(cr::tai_time<cr::microseconds>{946'688'523'123'456us}) ==
+         SV("1988-01-01 01:02:03,123456"));
+
+  assert(stream_fr_FR_locale<CharT>(cr::tai_time<cr::milliseconds>{946'684'822'123ms}) ==
+         SV("1988-01-01 00:00:22,123"));
+  assert(stream_fr_FR_locale<CharT>(cr::tai_seconds{1'234'567'890s}) == SV("1997-02-13 23:31:30"));
+  assert(stream_fr_FR_locale<CharT>(cr::tai_time<cr::minutes>{20'576'131min}) == SV("1997-02-13 23:31:00"));
+  assert(stream_fr_FR_locale<CharT>(cr::tai_time<cr::hours>{342'935h}) == SV("1997-02-13 23:00:00"));
+
+  assert(stream_fr_FR_locale<CharT>(cr::tai_time<cr::duration<signed char, std::ratio<2, 1>>>{
+             cr::duration<signed char, std::ratio<2, 1>>{60}}) == SV("1958-01-01 00:02:00"));
+  assert(stream_fr_FR_locale<CharT>(cr::tai_time<cr::duration<short, std::ratio<1, 2>>>{
+             cr::duration<short, std::ratio<1, 2>>{3600}}) == SV("1958-01-01 00:30:00,0"));
+  assert(stream_fr_FR_locale<CharT>(cr::tai_time<cr::duration<int, std::ratio<1, 4>>>{
+             cr::duration<int, std::ratio<1, 4>>{3600}}) == SV("1958-01-01 00:15:00,00"));
+  assert(stream_fr_FR_locale<CharT>(cr::tai_time<cr::duration<long, std::ratio<1, 10>>>{
+             cr::duration<long, std::ratio<1, 10>>{36611}}) == SV("1958-01-01 01:01:01,1"));
+  assert(stream_fr_FR_locale<CharT>(cr::tai_time<cr::duration<long long, std::ratio<1, 100>>>{
+             cr::duration<long long, std::ratio<1, 100>>{12'345'678'9010}}) == SV("1997-02-13 23:31:30,10"));
+}
+
+template <class CharT>
+static void test_ja_JP() {
+  using namespace std::literals::chrono_literals;
+  namespace cr = std::chrono;
+
+  assert(stream_ja_JP_locale<CharT>(cr::tai_time<cr::nanoseconds>{946'688'523'123'456'789ns}) ==
+         SV("1988-01-01 01:02:03.123456789"));
+  assert(stream_ja_JP_locale<CharT>(cr::tai_time<cr::microseconds>{946'688'523'123'456us}) ==
+         SV("1988-01-01 01:02:03.123456"));
+
+  assert(stream_ja_JP_locale<CharT>(cr::tai_time<cr::milliseconds>{946'684'822'123ms}) ==
+         SV("1988-01-01 00:00:22.123"));
+  assert(stream_ja_JP_locale<CharT>(cr::tai_seconds{1'234'567'890s}) == SV("1997-02-13 23:31:30"));
+  assert(stream_ja_JP_locale<CharT>(cr::tai_time<cr::minutes>{20'576'131min}) == SV("1997-02-13 23:31:00"));
+  assert(stream_ja_JP_locale<CharT>(cr::tai_time<cr::hours>{342'935h}) == SV("1997-02-13 23:00:00"));
+
+  assert(stream_ja_JP_locale<CharT>(cr::tai_time<cr::duration<signed char, std::ratio<2, 1>>>{
+             cr::duration<signed char, std::ratio<2, 1>>{60}}) == SV("1958-01-01 00:02:00"));
+  assert(stream_ja_JP_locale<CharT>(cr::tai_time<cr::duration<short, std::ratio<1, 2>>>{
+             cr::duration<short, std::ratio<1, 2>>{3600}}) == SV("1958-01-01 00:30:00.0"));
+  assert(stream_ja_JP_locale<CharT>(cr::tai_time<cr::duration<int, std::ratio<1, 4>>>{
+             cr::duration<int, std::ratio<1, 4>>{3600}}) == SV("1958-01-01 00:15:00.00"));
+  assert(stream_ja_JP_locale<CharT>(cr::tai_time<cr::duration<long, std::ratio<1, 10>>>{
+             cr::duration<long, std::ratio<1, 10>>{36611}}) == SV("1958-01-01 01:01:01.1"));
+  assert(stream_ja_JP_locale<CharT>(cr::tai_time<cr::duration<long long, std::ratio<1, 100>>>{
+             cr::duration<long long, std::ratio<1, 100>>{12'345'678'9010}}) == SV("1997-02-13 23:31:30.10"));
+}
+
+template <class CharT>
+static void test() {
+  test_c<CharT>();
+  test_fr_FR<CharT>();
+  test_ja_JP<CharT>();
+}
+
+int main(int, char**) {
+  test<char>();
+
+#ifndef TEST_HAS_NO_WIDE_CHARACTERS
+  test<wchar_t>();
+#endif
+
+  return 0;
+}
diff --git a/libcxx/test/std/time/time.clock/time.clock.tai/time.clock.tai.members/from_utc.pass.cpp b/libcxx/test/std/time/time.clock/time.clock.tai/time.clock.tai.members/from_utc.pass.cpp
new file mode 100644
index 0000000000000..0d2d7bec16301
--- /dev/null
+++ b/libcxx/test/std/time/time.clock/time.clock.tai/time.clock.tai.members/from_utc.pass.cpp
@@ -0,0 +1,161 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// REQUIRES: std-at-least-c++20
+// UNSUPPORTED: no-filesystem, no-localization, no-tzdb
+
+// XFAIL: libcpp-has-no-experimental-tzdb
+// XFAIL: availability-tzdb-missing
+
+// <chrono>
+//
+// class tai_clock;
+
+// static tai_time<common_type_t<_Duration, seconds>>
+// from_utc(const utc<_Duration>& __time) noexcept;
+
+#include <chrono>
+#include <cassert>
+#include <source_location>
+
+#include "test_macros.h"
+#include "assert_macros.h"
+#include "concat_macros.h"
+
+static void test_known_values() {
+  namespace cr = std::chrono;
+  using namespace std::literals::chrono_literals;
+  constexpr auto unix_to_tai_epoch_offset = cr::sys_days{cr::January / 1 / 1970} - cr::sys_days{cr::January / 1 / 1958};
+
+  // [time.clock.tai.overview]/1
+  // ... 1958-01-01 00:00:00 TAI is equivalent to 1957-12-31 23:59:50 UTC
+  // ... 2000-01-01 00:00:00 UTC is equivalent to 2000-01-01 00:00:32 TAI
+
+  assert(cr::tai_clock::from_utc(cr::utc_clock::from_sys(cr::sys_days{cr::January / 1 / 1958} - 10s)) ==
+         cr::tai_seconds{0s});
+
+  assert(cr::tai_clock::from_utc(cr::utc_clock::from_sys(cr::sys_days{cr::January / 1 / 2000})) ==
+         cr::tai_seconds{(cr::sys_days{cr::January / 1 / 2000} + unix_to_tai_epoch_offset).time_since_epoch()} + 32s);
+}
+
+template <class Duration>
+static void test_leap_seconds(std::chrono::utc_time<Duration> utc,
+                              std::chrono::tai_time<Duration> expected,
+                              std::source_location loc = std::source_location::current()) {
+  auto tai = std::chrono::tai_clock::from_utc(utc);
+  TEST_REQUIRE(tai == expected,
+               TEST_WRITE_CONCATENATED(loc, "\nExpected output ", expected, "\nActual output   ", tai, '\n'));
+}
+
+// Tests set if existing database entries at the time of writing.
+static void test_transitions() {
+  using namespace std::literals::chrono_literals;
+  namespace cr = std::chrono;
+
+  // "sys" is the time of the transition to the next leap second.
+  // "elapsed" is the number of leap seconds before the transition.
+  auto test_transition = [](cr::sys_days sys, cr::seconds elapsed) {
+    constexpr auto unix_to_tai_epoch_offset =
+        cr::sys_days{cr::January / 1 / 1970} - cr::sys_days{cr::January / 1 / 1958};
+    cr::tai_seconds tai{sys.time_since_epoch() + unix_to_tai_epoch_offset + elapsed};
+
+    test_leap_seconds(cr::utc_clock::from_sys(sys - 1ns), tai - 1ns);
+    test_leap_seconds(cr::utc_clock::from_sys(sys), tai + 1s);
+    test_leap_seconds(cr::utc_clock::from_sys(sys) + 1ns, tai + 1s + 1ns);
+  };
+
+  // Transitions from the start of UTC.
+  test_transition(cr::sys_days{cr::July / 1 / 1972}, 10s);
+  test_transition(cr::sys_days{cr::January / 1 / 1973}, 11s);
+  test_transition(cr::sys_days{cr::January / 1 / 1974}, 12s);
+  test_transition(cr::sys_days{cr::January / 1 / 1975}, 13s);
+  test_transition(cr::sys_days{cr::January / 1 / 1976}, 14s);
+  test_transition(cr::sys_days{cr::January / 1 / 1977}, 15s);
+  test_transition(cr::sys_days{cr::January / 1 / 1978}, 16s);
+  test_transition(cr::sys_days{cr::January / 1 / 1979}, 17s);
+  test_transition(cr::sys_days{cr::January / 1 / 1980}, 18s);
+  test_transition(cr::sys_days{cr::July / 1 / 1981}, 19s);
+  test_transition(cr::sys_days{cr::July / 1 / 1982}, 20s);
+  test_transition(cr::sys_days{cr::July / 1 / 1983}, 21s);
+  test_transition(cr::sys_days{cr::July / 1 / 1985}, 22s);
+  test_transition(cr::sys_days{cr::January / 1 / 1988}, 23s);
+  test_transition(cr::sys_days{cr::January / 1 / 1990}, 24s);
+  test_transition(cr::sys_days{cr::January / 1 / 1991}, 25s);
+  test_transition(cr::sys_days{cr::July / 1 / 1992}, 26s);
+  test_transition(cr::sys_days{cr::July / 1 / 1993}, 27s);
+  test_transition(cr::sys_days{cr::July / 1 / 1994}, 28s);
+  test_transition(cr::sys_days{cr::January / 1 / 1996}, 29s);
+  test_transition(cr::sys_days{cr::July / 1 / 1997}, 30s);
+  test_transition(cr::sys_days{cr::January / 1 / 1999}, 31s);
+  test_transition(cr::sys_days{cr::January / 1 / 2006}, 32s);
+  test_transition(cr::sys_days{cr::January / 1 / 2009}, 33s);
+  test_transition(cr::sys_days{cr::July / 1 / 2012}, 34s);
+  test_transition(cr::sys_days{cr::July / 1 / 2015}, 35s);
+  test_transition(cr::sys_days{cr::January / 1 / 2017}, 36s);
+}
+
+// Tests whether the return type is the expected type.
+static void test_return_type() {
+  using namespace std::literals::chrono_literals;
+  namespace cr = std::chrono;
+
+  {
+    [[maybe_unused]] std::same_as<cr::tai_time<cr::nanoseconds>> decltype(auto) _ =
+        cr::tai_clock::from_utc(cr::utc_time<cr::nanoseconds>{0ns});
+  }
+  {
+    [[maybe_unused]] std::same_as<cr::tai_time<cr::microseconds>> decltype(auto) _ =
+        cr::tai_clock::from_utc(cr::utc_time<cr::microseconds>{0us});
+  }
+  {
+    [[maybe_unused]] std::same_as<cr::tai_time<cr::milliseconds>> decltype(auto) _ =
+        cr::tai_clock::from_utc(cr::utc_time<cr::milliseconds>{0ms});
+  }
+
+  {
+    [[maybe_unused]] std::same_as<cr::tai_time<cr::seconds>> decltype(auto) _ =
+        cr::tai_clock::from_utc(cr::utc_time<cr::seconds>{cr::seconds{0}});
+  }
+
+  {
+    [[maybe_unused]] std::same_as<cr::tai_time<cr::seconds>> decltype(auto) _ =
+        cr::tai_clock::from_utc(cr::utc_time<cr::minutes>{cr::minutes{0}});
+  }
+  {
+    [[maybe_unused]] std::same_as<cr::tai_time<cr::seconds>> decltype(auto) _ =
+        cr::tai_clock::from_utc(cr::utc_time<cr::hours>{cr::hours{0}});
+  }
+  {
+    [[maybe_unused]] std::same_as<cr::tai_time<cr::seconds>> decltype(auto) _ =
+        cr::tai_clock::from_utc(cr::utc_time<cr::days>{cr::days{0}});
+  }
+  {
+    [[maybe_unused]] std::same_as<cr::tai_time<cr::seconds>> decltype(auto) _ =
+        cr::tai_clock::from_utc(cr::utc_time<cr::weeks>{cr::weeks{0}});
+  }
+  {
+    [[maybe_unused]] std::same_as<cr::tai_time<cr::seconds>> decltype(auto) _ =
+        cr::tai_clock::from_utc(cr::utc_time<cr::months>{cr::months{0}});
+  }
+  {
+    [[maybe_unused]] std::same_as<cr::tai_time<cr::seconds>> decltype(auto) _ =
+        cr::tai_clock::from_utc(cr::utc_time<cr::years>{cr::years{0}});
+  }
+}
+
+int main(int, const char**) {
+  using namespace std::literals::chrono_literals;
+  std::chrono::utc_seconds time = std::chrono::utc_seconds{0s};
+  static_assert(noexcept(std::chrono::tai_clock::from_utc(time)));
+
+  test_known_values();
+  test_transitions();
+  test_return_type();
+
+  return 0;
+}
diff --git a/libcxx/test/std/time/time.clock/time.clock.tai/time.clock.tai.members/now.pass.cpp b/libcxx/test/std/time/time.clock/time.clock.tai/time.clock.tai.members/now.pass.cpp
new file mode 100644
index 0000000000000..2baef34277023
--- /dev/null
+++ b/libcxx/test/std/time/time.clock/time.clock.tai/time.clock.tai.members/now.pass.cpp
@@ -0,0 +1,33 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// REQUIRES: std-at-least-c++20
+// UNSUPPORTED: no-filesystem, no-localization, no-tzdb
+
+// XFAIL: libcpp-has-no-experimental-tzdb
+// XFAIL: availability-tzdb-missing
+
+// <chrono>
+//
+// class tai_clock;
+
+// static time_point now();
+
+#include <chrono>
+#include <concepts>
+#include <cassert>
+
+int main(int, const char**) {
+  using clock                                      = std::chrono::tai_clock;
+  std::same_as<clock::time_point> decltype(auto) t = clock::now();
+
+  assert(t >= clock::time_point::min());
+  assert(t <= clock::time_point::max());
+
+  return 0;
+}
diff --git a/libcxx/test/std/time/time.clock/time.clock.tai/time.clock.tai.members/to_utc.pass.cpp b/libcxx/test/std/time/time.clock/time.clock.tai/time.clock.tai.members/to_utc.pass.cpp
new file mode 100644
index 0000000000000..38b50bf275871
--- /dev/null
+++ b/libcxx/test/std/time/time.clock/time.clock.tai/time.clock.tai.members/to_utc.pass.cpp
@@ -0,0 +1,163 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// REQUIRES: std-at-least-c++20
+// UNSUPPORTED: no-filesystem, no-localization, no-tzdb
+
+// XFAIL: libcpp-has-no-experimental-tzdb
+// XFAIL: availability-tzdb-missing
+
+// <chrono>
+//
+// class tai_clock;
+
+// static utc_time<common_type_t<_Duration, seconds>>
+// to_utc(const tai_time<_Duration>& __time) noexcept;
+
+#include <chrono>
+#include <cassert>
+#include <source_location>
+
+#include "test_macros.h"
+#include "assert_macros.h"
+#include "concat_macros.h"
+
+static void test_known_values() {
+  namespace cr = std::chrono;
+  using namespace std::literals::chrono_literals;
+  constexpr auto unix_to_tai_epoch_offset = cr::sys_days{cr::January / 1 / 1970} - cr::sys_days{cr::January / 1 / 1958};
+
+  // [time.clock.tai.overview]/1
+  // ... 1958-01-01 00:00:00 TAI is equivalent to 1957-12-31 23:59:50 UTC
+  // ... 2000-01-01 00:00:00 UTC is equivalent to 2000-01-01 00:00:32 TAI
+
+  assert(cr::tai_clock::to_utc(cr::tai_seconds{0s}) ==
+         cr::utc_clock::from_sys(cr::sys_days{cr::January / 1 / 1958} - 10s));
+
+  assert(cr::tai_clock::to_utc(
+             cr::tai_seconds{(cr::sys_days{cr::January / 1 / 2000} + unix_to_tai_epoch_offset).time_since_epoch()} +
+             32s) == cr::utc_clock::from_sys(cr::sys_days{cr::January / 1 / 2000}));
+}
+
+template <class Duration>
+static void test_leap_seconds(std::chrono::tai_time<Duration> tai,
+                              std::chrono::utc_time<Duration> expected,
+                              std::source_location loc = std::source_location::current()) {
+  auto utc = std::chrono::tai_clock::to_utc(tai);
+  TEST_REQUIRE(utc == expected,
+               TEST_WRITE_CONCATENATED(loc, "\nExpected output ", expected, "\nActual output   ", utc, '\n'));
+}
+
+// Tests set if existing database entries at the time of writing.
+static void test_transitions() {
+  using namespace std::literals::chrono_literals;
+  namespace cr = std::chrono;
+
+  // "sys" is the time of the transition to the next leap second.
+  // "elapsed" is the number of leap seconds before the transition.
+  auto test_transition = [](cr::sys_days sys, cr::seconds elapsed) {
+    constexpr auto unix_to_tai_epoch_offset =
+        cr::sys_days{cr::January / 1 / 1970} - cr::sys_days{cr::January / 1 / 1958};
+    cr::tai_seconds tai{sys.time_since_epoch() + unix_to_tai_epoch_offset + elapsed};
+
+    test_leap_seconds(tai - 1ns, cr::utc_clock::from_sys(sys - 1ns));
+    test_leap_seconds(tai + 1s, cr::utc_clock::from_sys(sys));
+    test_leap_seconds(tai + 1s + 1ns, cr::utc_clock::from_sys(sys + 1ns));
+  };
+
+  // Transitions from the start of UTC.
+  test_transition(cr::sys_days{cr::July / 1 / 1972}, 10s);
+  test_transition(cr::sys_days{cr::January / 1 / 1973}, 11s);
+  test_transition(cr::sys_days{cr::January / 1 / 1974}, 12s);
+  test_transition(cr::sys_days{cr::January / 1 / 1975}, 13s);
+  test_transition(cr::sys_days{cr::January / 1 / 1976}, 14s);
+  test_transition(cr::sys_days{cr::January / 1 / 1977}, 15s);
+  test_transition(cr::sys_days{cr::January / 1 / 1978}, 16s);
+  test_transition(cr::sys_days{cr::January / 1 / 1979}, 17s);
+  test_transition(cr::sys_days{cr::January / 1 / 1980}, 18s);
+  test_transition(cr::sys_days{cr::July / 1 / 1981}, 19s);
+  test_transition(cr::sys_days{cr::July / 1 / 1982}, 20s);
+  test_transition(cr::sys_days{cr::July / 1 / 1983}, 21s);
+  test_transition(cr::sys_days{cr::July / 1 / 1985}, 22s);
+  test_transition(cr::sys_days{cr::January / 1 / 1988}, 23s);
+  test_transition(cr::sys_days{cr::January / 1 / 1990}, 24s);
+  test_transition(cr::sys_days{cr::January / 1 / 1991}, 25s);
+  test_transition(cr::sys_days{cr::July / 1 / 1992}, 26s);
+  test_transition(cr::sys_days{cr::July / 1 / 1993}, 27s);
+  test_transition(cr::sys_days{cr::July / 1 / 1994}, 28s);
+  test_transition(cr::sys_days{cr::January / 1 / 1996}, 29s);
+  test_transition(cr::sys_days{cr::July / 1 / 1997}, 30s);
+  test_transition(cr::sys_days{cr::January / 1 / 1999}, 31s);
+  test_transition(cr::sys_days{cr::January / 1 / 2006}, 32s);
+  test_transition(cr::sys_days{cr::January / 1 / 2009}, 33s);
+  test_transition(cr::sys_days{cr::July / 1 / 2012}, 34s);
+  test_transition(cr::sys_days{cr::July / 1 / 2015}, 35s);
+  test_transition(cr::sys_days{cr::January / 1 / 2017}, 36s);
+}
+
+// Tests whether the return type is the expected type.
+static void test_return_type() {
+  using namespace std::literals::chrono_literals;
+  namespace cr = std::chrono;
+
+  {
+    [[maybe_unused]] std::same_as<cr::utc_time<cr::nanoseconds>> decltype(auto) _ =
+        cr::tai_clock::to_utc(cr::tai_time<cr::nanoseconds>{0ns});
+  }
+  {
+    [[maybe_unused]] std::same_as<cr::utc_time<cr::microseconds>> decltype(auto) _ =
+        cr::tai_clock::to_utc(cr::tai_time<cr::microseconds>{0us});
+  }
+  {
+    [[maybe_unused]] std::same_as<cr::utc_time<cr::milliseconds>> decltype(auto) _ =
+        cr::tai_clock::to_utc(cr::tai_time<cr::milliseconds>{0ms});
+  }
+
+  {
+    [[maybe_unused]] std::same_as<cr::utc_time<cr::seconds>> decltype(auto) _ =
+        cr::tai_clock::to_utc(cr::tai_time<cr::seconds>{cr::seconds{0}});
+  }
+
+  {
+    [[maybe_unused]] std::same_as<cr::utc_time<cr::seconds>> decltype(auto) _ =
+        cr::tai_clock::to_utc(cr::tai_time<cr::minutes>{cr::minutes{0}});
+  }
+  {
+    [[maybe_unused]] std::same_as<cr::utc_time<cr::seconds>> decltype(auto) _ =
+        cr::tai_clock::to_utc(cr::tai_time<cr::hours>{cr::hours{0}});
+  }
+  {
+    [[maybe_unused]] std::same_as<cr::utc_time<cr::seconds>> decltype(auto) _ =
+        cr::tai_clock::to_utc(cr::tai_time<cr::days>{cr::days{0}});
+  }
+  {
+    [[maybe_unused]] std::same_as<cr::utc_time<cr::seconds>> decltype(auto) _ =
+        cr::tai_clock::to_utc(cr::tai_time<cr::weeks>{cr::weeks{0}});
+  }
+  {
+    [[maybe_unused]] std::same_as<cr::utc_time<cr::seconds>> decltype(auto) _ =
+        cr::tai_clock::to_utc(cr::tai_time<cr::months>{cr::months{0}});
+  }
+  {
+    [[maybe_unused]] std::same_as<cr::utc_time<cr::seconds>> decltype(auto) _ =
+        cr::tai_clock::to_utc(cr::tai_time<cr::years>{cr::years{0}});
+  }
+}
+
+int main(int, const char**) {
+  using namespace std::literals::chrono_literals;
+
+  std::chrono::tai_seconds time = std::chrono::tai_seconds{0s};
+  static_assert(noexcept(std::chrono::tai_clock::to_utc(time)));
+
+  test_known_values();
+  test_transitions();
+  test_return_type();
+
+  return 0;
+}
diff --git a/libcxx/test/std/time/time.clock/time.clock.tai/types.compile.pass.cpp b/libcxx/test/std/time/time.clock/time.clock.tai/types.compile.pass.cpp
new file mode 100644
index 0000000000000..ddb057333f49a
--- /dev/null
+++ b/libcxx/test/std/time/time.clock/time.clock.tai/types.compile.pass.cpp
@@ -0,0 +1,60 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// REQUIRES: std-at-least-c++20
+// UNSUPPORTED: no-filesystem, no-localization, no-tzdb
+
+// XFAIL: libcpp-has-no-experimental-tzdb
+// XFAIL: availability-tzdb-missing
+
+// <chrono>
+
+// class tai_clock {
+// public:
+//     using rep                       = a signed arithmetic type;
+//     using period                    = ratio<unspecified, unspecified>;
+//     using duration                  = chrono::duration<rep, period>;
+//     using time_point                = chrono::time_point<tai_clock>;
+//     static constexpr bool is_steady = unspecified;
+//
+//     ...
+// };
+//
+// template<class Duration>
+// using tai_time  = time_point<tai_clock, Duration>;
+// using tai_seconds = tai_time<seconds>;
+
+#include <chrono>
+#include <concepts>
+#include <ratio>
+
+#include "test_macros.h"
+
+// class tai_clock
+using rep                = std::chrono::tai_clock::rep;
+using period             = std::chrono::tai_clock::period;
+using duration           = std::chrono::tai_clock::duration;
+using time_point         = std::chrono::tai_clock::time_point;
+constexpr bool is_steady = std::chrono::tai_clock::is_steady;
+
+// Tests the values. part of them are implementation defined.
+LIBCPP_STATIC_ASSERT(std::same_as<rep, std::chrono::utc_clock::rep>);
+static_assert(std::is_arithmetic_v<rep>);
+static_assert(std::is_signed_v<rep>);
+
+LIBCPP_STATIC_ASSERT(std::same_as<period, std::chrono::utc_clock::period>);
+static_assert(std::same_as<period, std::ratio<period::num, period::den>>);
+
+static_assert(std::same_as<duration, std::chrono::duration<rep, period>>);
+static_assert(std::same_as<time_point, std::chrono::time_point<std::chrono::tai_clock>>);
+LIBCPP_STATIC_ASSERT(is_steady == false);
+
+// typedefs
+static_assert(std::same_as<std::chrono::tai_time<int>, std::chrono::time_point<std::chrono::tai_clock, int>>);
+static_assert(std::same_as<std::chrono::tai_time<long>, std::chrono::time_point<std::chrono::tai_clock, long>>);
+static_assert(std::same_as<std::chrono::tai_seconds, std::chrono::tai_time<std::chrono::seconds>>);
diff --git a/libcxx/test/std/time/time.syn/formatter.tai_time.pass.cpp b/libcxx/test/std/time/time.syn/formatter.tai_time.pass.cpp
new file mode 100644
index 0000000000000..7ca088cc6e8f4
--- /dev/null
+++ b/libcxx/test/std/time/time.syn/formatter.tai_time.pass.cpp
@@ -0,0 +1,998 @@
+//===----------------------------------------------------------------------===//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// REQUIRES: std-at-least-c++20
+// UNSUPPORTED: no-filesystem, no-localization, no-tzdb
+// UNSUPPORTED: GCC-ALWAYS_INLINE-FIXME
+
+// TODO FMT This test should not require std::to_chars(floating-point)
+// XFAIL: availability-fp_to_chars-missing
+
+// XFAIL: libcpp-has-no-experimental-tzdb
+// XFAIL: availability-tzdb-missing
+
+// REQUIRES: locale.fr_FR.UTF-8
+// REQUIRES: locale.ja_JP.UTF-8
+
+// <chrono>
+
+// template<class Duration, class charT>
+//   struct formatter<chrono::tai_time<Duration>, charT>;
+
+#include <chrono>
+#include <format>
+
+#include <cassert>
+#include <concepts>
+#include <locale>
+#include <iostream>
+#include <type_traits>
+
+#include "formatter_tests.h"
+#include "make_string.h"
+#include "platform_support.h" // locale name macros
+#include "test_macros.h"
+
+template <class CharT>
+static void test_no_chrono_specs() {
+  using namespace std::literals::chrono_literals;
+  namespace cr = std::chrono;
+
+  std::locale::global(std::locale(LOCALE_fr_FR_UTF_8));
+
+  // Non localized output
+
+  // [time.syn]
+  //   using nanoseconds  = duration<signed integer type of at least 64 bits, nano>;
+  //   using microseconds = duration<signed integer type of at least 55 bits, micro>;
+  //   using milliseconds = duration<signed integer type of at least 45 bits, milli>;
+  //   using seconds      = duration<signed integer type of at least 35 bits>;
+  //   using minutes      = duration<signed integer type of at least 29 bits, ratio<  60>>;
+  //   using hours        = duration<signed integer type of at least 23 bits, ratio<3600>>;
+  check(SV("1413-08-04 22:06:56"), SV("{}"), cr::tai_seconds(-17'179'869'184s)); // Minimum value for 35 bits.
+  check(SV("1889-12-12 20:45:52"), SV("{}"), cr::tai_seconds(-2'147'483'648s));
+
+  check(SV("1957-12-31 00:00:00"), SV("{}"), cr::tai_seconds(-24h));
+  check(SV("1957-12-31 06:00:00"), SV("{}"), cr::tai_seconds(-18h));
+  check(SV("1957-12-31 12:00:00"), SV("{}"), cr::tai_seconds(-12h));
+  check(SV("1957-12-31 18:00:00"), SV("{}"), cr::tai_seconds(-6h));
+  check(SV("1957-12-31 23:59:59"), SV("{}"), cr::tai_seconds(-1s));
+
+  check(SV("1958-01-01 00:00:00"), SV("{}"), cr::tai_seconds(0s));
+  check(SV("1988-01-01 00:00:00"), SV("{}"), cr::tai_seconds(946'684'800s));
+  check(SV("1988-01-01 01:02:03"), SV("{}"), cr::tai_seconds(946'688'523s));
+
+  check(SV("2026-01-19 03:14:07"), SV("{}"), cr::tai_seconds(2'147'483'647s));
+  check(SV("2502-05-30 01:53:03"), SV("{}"), cr::tai_seconds(17'179'869'183s)); // Maximum value for 35 bits.
+
+  check(SV("1988-01-01 01:02:03.123"), SV("{}"), cr::tai_time<cr::milliseconds>(946'688'523'123ms));
+
+  std::locale::global(std::locale::classic());
+}
+
+template <class CharT>
+static void test_valid_values_year() {
+  using namespace std::literals::chrono_literals;
+  namespace cr = std::chrono;
+
+  constexpr std::basic_string_view<CharT> fmt =
+      SV("{:%%C='%C'%t%%EC='%EC'%t%%y='%y'%t%%Oy='%Oy'%t%%Ey='%Ey'%t%%Y='%Y'%t%%EY='%EY'%n}");
+  constexpr std::basic_string_view<CharT> lfmt =
+      SV("{:L%%C='%C'%t%%EC='%EC'%t%%y='%y'%t%%Oy='%Oy'%t%%Ey='%Ey'%t%%Y='%Y'%t%%EY='%EY'%n}");
+
+  const std::locale loc(LOCALE_ja_JP_UTF_8);
+  std::locale::global(std::locale(LOCALE_fr_FR_UTF_8));
+
+  // Non localized output using C-locale
+  check(SV("%C='19'\t%EC='19'\t%y='58'\t%Oy='58'\t%Ey='58'\t%Y='1958'\t%EY='1958'\n"),
+        fmt,
+        cr::tai_seconds(0s)); // 00:00:00 TAI Wednesday, 1 January 1958
+
+  check(SV("%C='20'\t%EC='20'\t%y='09'\t%Oy='09'\t%Ey='09'\t%Y='2009'\t%EY='2009'\n"),
+        fmt,
+        cr::tai_seconds(1'613'259'090s)); // 23:31:30 TAI Friday, 13 February 2009
+
+  // Use the global locale (fr_FR)
+  check(SV("%C='19'\t%EC='19'\t%y='58'\t%Oy='58'\t%Ey='58'\t%Y='1958'\t%EY='1958'\n"),
+        lfmt,
+        cr::tai_seconds(0s)); // 00:00:00 TAI Wednesday, 1 January 1958
+
+  check(SV("%C='20'\t%EC='20'\t%y='09'\t%Oy='09'\t%Ey='09'\t%Y='2009'\t%EY='2009'\n"),
+        lfmt,
+        cr::tai_seconds(1'613'259'090s)); // 23:31:30 TAI Friday, 13 February 2009
+
+  // Use supplied locale (ja_JP). This locale has a different alternate.
+#if defined(_WIN32) || defined(__APPLE__) || defined(_AIX) || defined(__FreeBSD__)
+  check(loc,
+        SV("%C='19'\t%EC='19'\t%y='58'\t%Oy='58'\t%Ey='58'\t%Y='1958'\t%EY='1958'\n"),
+        lfmt,
+        cr::tai_seconds(0s)); // 00:00:00 TAI Wednesday, 1 January 1958
+
+  check(loc,
+        SV("%C='20'\t%EC='20'\t%y='09'\t%Oy='09'\t%Ey='09'\t%Y='2009'\t%EY='2009'\n"),
+        lfmt,
+        cr::tai_seconds(1'613'259'090s)); // 23:31:30 TAI Friday, 13 February 2009
+#else  // defined(_WIN32) || defined(__APPLE__) || defined(_AIX)||defined(__FreeBSD__)
+  check(loc,
+        SV("%C='19'\t%EC='昭和'\t%y='58'\t%Oy='五十八'\t%Ey='33'\t%Y='1958'\t%EY='昭和33年'\n"),
+        lfmt,
+        cr::tai_seconds(0s)); // 00:00:00 TAI Wednesday, 1 January 1958
+
+  check(loc,
+        SV("%C='20'\t%EC='平成'\t%y='09'\t%Oy='九'\t%Ey='21'\t%Y='2009'\t%EY='平成21年'\n"),
+        lfmt,
+        cr::tai_seconds(1'613'259'090s)); // 23:31:30 TAI Friday, 13 February 2009
+#endif // defined(_WIN32) || defined(__APPLE__) || defined(_AIX)||defined(__FreeBSD__)
+
+  std::locale::global(std::locale::classic());
+}
+
+template <class CharT>
+static void test_valid_values_month() {
+  using namespace std::literals::chrono_literals;
+  namespace cr = std::chrono;
+
+  constexpr std::basic_string_view<CharT> fmt  = SV("{:%%b='%b'%t%%h='%h'%t%%B='%B'%t%%m='%m'%t%%Om='%Om'%n}");
+  constexpr std::basic_string_view<CharT> lfmt = SV("{:L%%b='%b'%t%%h='%h'%t%%B='%B'%t%%m='%m'%t%%Om='%Om'%n}");
+
+  const std::locale loc(LOCALE_ja_JP_UTF_8);
+  std::locale::global(std::locale(LOCALE_fr_FR_UTF_8));
+
+  // Non localized output using C-locale
+  check(SV("%b='Jan'\t%h='Jan'\t%B='January'\t%m='01'\t%Om='01'\n"),
+        fmt,
+        cr::tai_seconds(0s)); // 00:00:00 TAI Wednesday, 1 January 1958
+
+  check(SV("%b='May'\t%h='May'\t%B='May'\t%m='05'\t%Om='05'\n"),
+        fmt,
+        cr::tai_seconds(2'378'691'200s)); // 03:33:20 TAI on Wednesday, 18 May 2033
+
+  // Use the global locale (fr_FR)
+#if defined(__APPLE__)
+  check(SV("%b='jan'\t%h='jan'\t%B='janvier'\t%m='01'\t%Om='01'\n"),
+        lfmt,
+        cr::tai_seconds(0s)); // 00:00:00 TAI Wednesday, 1 January 1958
+#else
+  check(SV("%b='janv.'\t%h='janv.'\t%B='janvier'\t%m='01'\t%Om='01'\n"),
+        lfmt,
+        cr::tai_seconds(0s)); // 00:00:00 TAI Wednesday, 1 January 1958
+#endif
+
+  check(SV("%b='mai'\t%h='mai'\t%B='mai'\t%m='05'\t%Om='05'\n"),
+        lfmt,
+        cr::tai_seconds(2'378'691'200s)); // 03:33:20 TAI on Wednesday, 18 May 2033
+
+  // Use supplied locale (ja_JP). This locale has a different alternate.
+#ifdef _WIN32
+  check(loc,
+        SV("%b='1'\t%h='1'\t%B='1月'\t%m='01'\t%Om='01'\n"),
+        lfmt,
+        cr::tai_seconds(0s)); // 00:00:00 TAI Wednesday, 1 January 1958
+
+  check(loc,
+        SV("%b='5'\t%h='5'\t%B='5月'\t%m='05'\t%Om='05'\n"),
+        lfmt,
+        cr::tai_seconds(2'378'691'200s)); // 03:33:20 TAI Wednesday, 18 May 2033
+#elif defined(_AIX)                       // _WIN32
+  check(loc,
+        SV("%b='1月'\t%h='1月'\t%B='1月'\t%m='01'\t%Om='01'\n"),
+        lfmt,
+        cr::tai_seconds(0s)); // 00:00:00 TAI Wednesday, 1 January 1958
+
+  check(loc,
+        SV("%b='5月'\t%h='5月'\t%B='5月'\t%m='05'\t%Om='05'\n"),
+        lfmt,
+        cr::tai_seconds(2'378'691'200s)); // 03:33:20 TAI Wednesday, 18 May 2033
+#elif defined(__APPLE__)                  // _WIN32
+  check(loc,
+        SV("%b=' 1'\t%h=' 1'\t%B='1月'\t%m='01'\t%Om='01'\n"),
+        lfmt,
+        cr::tai_seconds(0s)); // 00:00:00 TAI Wednesday, 1 January 1958
+
+  check(loc,
+        SV("%b=' 5'\t%h=' 5'\t%B='5月'\t%m='05'\t%Om='05'\n"),
+        lfmt,
+        cr::tai_seconds(2'378'691'200s)); // 03:33:20 TAI Wednesday, 18 May 2033
+#elif defined(__FreeBSD__)                // _WIN32
+  check(loc,
+        SV("%b=' 1月'\t%h=' 1月'\t%B='1月'\t%m='01'\t%Om='01'\n"),
+        lfmt,
+        cr::tai_seconds(0s)); // 00:00:00 TAI Wednesday, 1 January 1958
+
+  check(loc,
+        SV("%b=' 5月'\t%h=' 5月'\t%B='5月'\t%m='05'\t%Om='05'\n"),
+        lfmt,
+        cr::tai_seconds(2'378'691'200s)); // 03:33:20 TAI Wednesday, 18 May 2033
+#else                                     // _WIN32
+  check(loc,
+        SV("%b=' 1月'\t%h=' 1月'\t%B='1月'\t%m='01'\t%Om='一'\n"),
+        lfmt,
+        cr::tai_seconds(0s)); // 00:00:00 TAI Wednesday, 1 January 1958
+
+  check(loc,
+        SV("%b=' 5月'\t%h=' 5月'\t%B='5月'\t%m='05'\t%Om='五'\n"),
+        lfmt,
+        cr::tai_seconds(2'378'691'200s)); // 03:33:20 TAI Wednesday, 18 May 2033
+#endif                                    // _WIN32
+
+  std::locale::global(std::locale::classic());
+}
+
+template <class CharT>
+static void test_valid_values_day() {
+  using namespace std::literals::chrono_literals;
+  namespace cr = std::chrono;
+
+  constexpr std::basic_string_view<CharT> fmt  = SV("{:%%d='%d'%t%%Od='%Od'%t%%e='%e'%t%%Oe='%Oe'%n}");
+  constexpr std::basic_string_view<CharT> lfmt = SV("{:L%%d='%d'%t%%Od='%Od'%t%%e='%e'%t%%Oe='%Oe'%n}");
+
+  const std::locale loc(LOCALE_ja_JP_UTF_8);
+  std::locale::global(std::locale(LOCALE_fr_FR_UTF_8));
+
+  // Non localized output using C-locale
+  check(SV("%d='01'\t%Od='01'\t%e=' 1'\t%Oe=' 1'\n"),
+        fmt,
+        cr::tai_seconds(0s)); // 00:00:00 TAI Wednesday, 1 January 1958
+
+  check(SV("%d='13'\t%Od='13'\t%e='13'\t%Oe='13'\n"),
+        fmt,
+        cr::tai_seconds(1'613'259'090s)); // 23:31:30 TAI Friday, 13 February 2009
+
+  // Use the global locale (fr_FR)
+  check(SV("%d='01'\t%Od='01'\t%e=' 1'\t%Oe=' 1'\n"),
+        lfmt,
+        cr::tai_seconds(0s)); // 00:00:00 TAI Wednesday, 1 January 1958
+
+  check(SV("%d='13'\t%Od='13'\t%e='13'\t%Oe='13'\n"),
+        lfmt,
+        cr::tai_seconds(1'613'259'090s)); // 23:31:30 TAI Friday, 13 February 2009
+
+  // Use the global locale (fr_FR)
+  check(SV("%d='01'\t%Od='01'\t%e=' 1'\t%Oe=' 1'\n"),
+        lfmt,
+        cr::tai_seconds(0s)); // 00:00:00 TAI Wednesday, 1 January 1958
+
+  // Use supplied locale (ja_JP). This locale has a different alternate.
+#if defined(_WIN32) || defined(__APPLE__) || defined(_AIX) || defined(__FreeBSD__)
+  check(loc,
+        SV("%d='01'\t%Od='01'\t%e=' 1'\t%Oe=' 1'\n"),
+        lfmt,
+        cr::tai_seconds(0s)); // 00:00:00 TAI Wednesday, 1 January 1958
+
+  check(loc,
+        SV("%d='13'\t%Od='13'\t%e='13'\t%Oe='13'\n"),
+        lfmt,
+        cr::tai_seconds(1'613'259'090s)); // 23:31:30 TAI Friday, 13 February 2009
+
+  // Use the global locale (fr_FR)
+  check(SV("%d='01'\t%Od='01'\t%e=' 1'\t%Oe=' 1'\n"),
+        lfmt,
+#else // defined(_WIN32) || defined(__APPLE__) || defined(_AIX) || defined(__FreeBSD__)
+  check(loc,
+        SV("%d='01'\t%Od='一'\t%e=' 1'\t%Oe='一'\n"),
+        lfmt,
+        cr::tai_seconds(0s)); // 00:00:00 TAI Wednesday, 1 January 1958
+
+  check(loc,
+        SV("%d='13'\t%Od='十三'\t%e='13'\t%Oe='十三'\n"),
+        lfmt,
+        cr::tai_seconds(1'613'259'090s)); // 23:31:30 TAI Friday, 13 February 2009
+
+#endif // defined(_WIN32) || defined(__APPLE__) || defined(_AIX) || defined(__FreeBSD__)
+
+  std::locale::global(std::locale::classic());
+}
+
+template <class CharT>
+static void test_valid_values_weekday() {
+  using namespace std::literals::chrono_literals;
+  namespace cr = std::chrono;
+
+  constexpr std::basic_string_view<CharT> fmt =
+      SV("{:%%a='%a'%t%%A='%A'%t%%u='%u'%t%%Ou='%Ou'%t%%w='%w'%t%%Ow='%Ow'%n}");
+  constexpr std::basic_string_view<CharT> lfmt =
+      SV("{:L%%a='%a'%t%%A='%A'%t%%u='%u'%t%%Ou='%Ou'%t%%w='%w'%t%%Ow='%Ow'%n}");
+
+  const std::locale loc(LOCALE_ja_JP_UTF_8);
+  std::locale::global(std::locale(LOCALE_fr_FR_UTF_8));
+
+  // Non localized output using C-locale
+  check(SV("%a='Wed'\t%A='Wednesday'\t%u='3'\t%Ou='3'\t%w='3'\t%Ow='3'\n"),
+        fmt,
+        cr::tai_seconds(0s)); // 00:00:00 TAI Wednesday, 1 January 1958
+
+  check(SV("%a='Sun'\t%A='Sunday'\t%u='7'\t%Ou='7'\t%w='0'\t%Ow='0'\n"),
+        fmt,
+        cr::tai_seconds(4'673'658'495s)); // 06:28:15 TAI on Sunday, 7 February 2106
+
+  // Use the global locale (fr_FR)
+#if defined(__APPLE__)
+  check(SV("%a='mer'\t%A='Mercredi'\t%u='3'\t%Ou='3'\t%w='3'\t%Ow='3'\n"),
+        lfmt,
+        cr::tai_seconds(0s)); // 00:00:00 TAI Wednesday, 1 January 1958
+
+  check(SV("%a='Dim'\t%A='Dimanche'\t%u='7'\t%Ou='7'\t%w='0'\t%Ow='0'\n"),
+        lfmt,
+        cr::tai_seconds(4'673'658'495s)); // 06:28:15 TAI on Sunday, 7 February 2106
+#else
+  check(SV("%a='mer.'\t%A='mercredi'\t%u='3'\t%Ou='3'\t%w='3'\t%Ow='3'\n"),
+        lfmt,
+        cr::tai_seconds(0s)); // 00:00:00 TAI Wednesday, 1 January 1958
+
+  check(SV("%a='dim.'\t%A='dimanche'\t%u='7'\t%Ou='7'\t%w='0'\t%Ow='0'\n"),
+        lfmt,
+        cr::tai_seconds(4'673'658'495s)); // 06:28:15 TAI on Sunday, 7 February 2106
+#endif
+
+  // Use supplied locale (ja_JP).
+  // This locale has a different alternate, but not on all platforms
+#if defined(_WIN32) || defined(__APPLE__) || defined(_AIX) || defined(__FreeBSD__)
+  check(loc,
+        SV("%a='水'\t%A='水曜日'\t%u='3'\t%Ou='3'\t%w='3'\t%Ow='3'\n"),
+        lfmt,
+        cr::tai_seconds(0s)); // 00:00:00 TAI Wednesday, 1 January 1958
+
+  check(loc,
+        SV("%a='日'\t%A='日曜日'\t%u='7'\t%Ou='7'\t%w='0'\t%Ow='0'\n"),
+        lfmt,
+        cr::tai_seconds(4'673'658'495s)); // 06:28:15 TAI on Sunday, 7 February 2106
+#else  // defined(_WIN32) || defined(__APPLE__) || defined(_AIX) || defined(__FreeBSD__)
+  check(loc,
+        SV("%a='水'\t%A='水曜日'\t%u='3'\t%Ou='三'\t%w='3'\t%Ow='三'\n"),
+        lfmt,
+        cr::tai_seconds(0s)); // 00:00:00 TAI Wednesday, 1 January 1958
+
+  check(loc,
+        SV("%a='日'\t%A='日曜日'\t%u='7'\t%Ou='七'\t%w='0'\t%Ow='〇'\n"),
+        lfmt,
+        cr::tai_seconds(4'673'658'495s)); // 06:28:15 TAI on Sunday, 7 February 2106
+#endif // defined(_WIN32) || defined(__APPLE__) || defined(_AIX) || defined(__FreeBSD__)
+
+  std::locale::global(std::locale::classic());
+}
+
+template <class CharT>
+static void test_valid_values_day_of_year() {
+  using namespace std::literals::chrono_literals;
+  namespace cr = std::chrono;
+
+  constexpr std::basic_string_view<CharT> fmt  = SV("{:%%j='%j'%n}");
+  constexpr std::basic_string_view<CharT> lfmt = SV("{:L%%j='%j'%n}");
+
+  const std::locale loc(LOCALE_ja_JP_UTF_8);
+  std::locale::global(std::locale(LOCALE_fr_FR_UTF_8));
+
+  // Non localized output using C-locale
+  check(SV("%j='001'\n"), fmt, cr::tai_seconds(0s));             // 00:00:00 TAI Wednesday, 1 January 1958
+  check(SV("%j='138'\n"), fmt, cr::tai_seconds(2'378'691'200s)); // 03:33:20 TAI Wednesday, 18 May 2033
+
+  // Use the global locale (fr_FR)
+  check(SV("%j='001'\n"), lfmt, cr::tai_seconds(0s));             // 00:00:00 TAI Wednesday, 1 January 1958
+  check(SV("%j='138'\n"), lfmt, cr::tai_seconds(2'378'691'200s)); // 03:33:20 TAI Wednesday, 18 May 2033
+
+  // Use supplied locale (ja_JP). This locale has a different alternate.
+  check(loc, SV("%j='001'\n"), lfmt, cr::tai_seconds(0s));             // 00:00:00 TAI Wednesday, 1 January 1958
+  check(loc, SV("%j='138'\n"), lfmt, cr::tai_seconds(2'378'691'200s)); // 03:33:20 TAI Wednesday, 18 May 2033
+
+  std::locale::global(std::locale::classic());
+}
+
+template <class CharT>
+static void test_valid_values_week() {
+  using namespace std::literals::chrono_literals;
+  namespace cr = std::chrono;
+
+  constexpr std::basic_string_view<CharT> fmt  = SV("{:%%U='%U'%t%%OU='%OU'%t%%W='%W'%t%%OW='%OW'%n}");
+  constexpr std::basic_string_view<CharT> lfmt = SV("{:L%%U='%U'%t%%OU='%OU'%t%%W='%W'%t%%OW='%OW'%n}");
+
+  const std::locale loc(LOCALE_ja_JP_UTF_8);
+  std::locale::global(std::locale(LOCALE_fr_FR_UTF_8));
+
+  // Non localized output using C-locale
+  check(SV("%U='00'\t%OU='00'\t%W='00'\t%OW='00'\n"),
+        fmt,
+        cr::tai_seconds(0s)); // 00:00:00 TAI Wednesday, 1 January 1958
+
+  check(SV("%U='20'\t%OU='20'\t%W='20'\t%OW='20'\n"),
+        fmt,
+        cr::tai_seconds(2'378'691'200s)); // 03:33:20 TAI Wednesday, 18 May 2033
+
+  // Use the global locale (fr_FR)
+  check(SV("%U='00'\t%OU='00'\t%W='00'\t%OW='00'\n"),
+        lfmt,
+        cr::tai_seconds(0s)); // 00:00:00 TAI Wednesday, 1 January 1958
+
+  check(SV("%U='20'\t%OU='20'\t%W='20'\t%OW='20'\n"),
+        lfmt,
+        cr::tai_seconds(2'378'691'200s)); // 03:33:20 TAI Wednesday, 18 May 2033
+
+  // Use supplied locale (ja_JP). This locale has a different alternate.
+#if defined(_WIN32) || defined(__APPLE__) || defined(_AIX) || defined(__FreeBSD__)
+  check(loc,
+        SV("%U='00'\t%OU='00'\t%W='00'\t%OW='00'\n"),
+        lfmt,
+        cr::tai_seconds(0s)); // 00:00:00 TAI Wednesday, 1 January 1958
+
+  check(loc,
+        SV("%U='20'\t%OU='20'\t%W='20'\t%OW='20'\n"),
+        lfmt,
+        cr::tai_seconds(2'378'691'200s)); // 03:33:20 TAI Wednesday, 18 May 2033
+#else  // defined(_WIN32) || defined(__APPLE__) || defined(_AIX) || defined(__FreeBSD__)
+  check(loc,
+        SV("%U='00'\t%OU='〇'\t%W='00'\t%OW='〇'\n"),
+        lfmt,
+        cr::tai_seconds(0s)); // 00:00:00 TAI Wednesday, 1 January 1958
+
+  check(loc,
+        SV("%U='20'\t%OU='二十'\t%W='20'\t%OW='二十'\n"),
+        lfmt,
+        cr::tai_seconds(2'378'691'200s)); // 03:33:20 TAI Wednesday, 18 May 2033
+#endif // defined(_WIN32) || defined(__APPLE__) || defined(_AIX) || defined(__FreeBSD__)
+  std::locale::global(std::locale::classic());
+}
+
+template <class CharT>
+static void test_valid_values_iso_8601_week() {
+  using namespace std::literals::chrono_literals;
+  namespace cr = std::chrono;
+
+  constexpr std::basic_string_view<CharT> fmt  = SV("{:%%g='%g'%t%%G='%G'%t%%V='%V'%t%%OV='%OV'%n}");
+  constexpr std::basic_string_view<CharT> lfmt = SV("{:L%%g='%g'%t%%G='%G'%t%%V='%V'%t%%OV='%OV'%n}");
+
+  const std::locale loc(LOCALE_ja_JP_UTF_8);
+  std::locale::global(std::locale(LOCALE_fr_FR_UTF_8));
+
+  // Non localized output using C-locale
+  check(SV("%g='58'\t%G='1958'\t%V='01'\t%OV='01'\n"),
+        fmt,
+        cr::tai_seconds(0s)); // 00:00:00 TAI Wednesday, 1 January 1958
+
+  check(SV("%g='09'\t%G='2009'\t%V='07'\t%OV='07'\n"),
+        fmt,
+        cr::tai_seconds(1'613'259'090s)); // 23:31:30 TAI Friday, 13 February 2009
+
+  // Use the global locale (fr_FR)
+  check(SV("%g='58'\t%G='1958'\t%V='01'\t%OV='01'\n"),
+        lfmt,
+        cr::tai_seconds(0s)); // 00:00:00 TAI Wednesday, 1 January 1958
+
+  check(SV("%g='09'\t%G='2009'\t%V='07'\t%OV='07'\n"),
+        lfmt,
+        cr::tai_seconds(1'613'259'090s)); // 23:31:30 TAI Friday, 13 February 2009
+
+  // Use supplied locale (ja_JP). This locale has a different alternate.
+#if defined(_WIN32) || defined(__APPLE__) || defined(_AIX) || defined(__FreeBSD__)
+  check(loc,
+        SV("%g='58'\t%G='1958'\t%V='01'\t%OV='01'\n"),
+        lfmt,
+        cr::tai_seconds(0s)); // 00:00:00 TAI Wednesday, 1 January 1958
+
+  check(loc,
+        SV("%g='09'\t%G='2009'\t%V='07'\t%OV='07'\n"),
+        lfmt,
+        cr::tai_seconds(1'613'259'090s)); // 23:31:30 TAI Friday, 13 February 2009
+#else  // defined(_WIN32) || defined(__APPLE__) || defined(_AIX) || defined(__FreeBSD__)
+  check(loc,
+        SV("%g='58'\t%G='1958'\t%V='01'\t%OV='一'\n"),
+        lfmt,
+        cr::tai_seconds(0s)); // 00:00:00 TAI Wednesday, 1 January 1958
+
+  check(loc,
+        SV("%g='09'\t%G='2009'\t%V='07'\t%OV='七'\n"),
+        lfmt,
+        cr::tai_seconds(1'613'259'090s)); // 23:31:30 TAI Friday, 13 February 2009
+#endif // defined(_WIN32) || defined(__APPLE__) || defined(_AIX) || defined(__FreeBSD__)
+
+  std::locale::global(std::locale::classic());
+}
+
+template <class CharT>
+static void test_valid_values_date() {
+  using namespace std::literals::chrono_literals;
+  namespace cr = std::chrono;
+
+  constexpr std::basic_string_view<CharT> fmt  = SV("{:%%D='%D'%t%%F='%F'%t%%x='%x'%t%%Ex='%Ex'%n}");
+  constexpr std::basic_string_view<CharT> lfmt = SV("{:L%%D='%D'%t%%F='%F'%t%%x='%x'%t%%Ex='%Ex'%n}");
+
+  const std::locale loc(LOCALE_ja_JP_UTF_8);
+  std::locale::global(std::locale(LOCALE_fr_FR_UTF_8));
+
+  // Non localized output using C-locale
+  check(SV("%D='01/01/58'\t%F='1958-01-01'\t%x='01/01/58'\t%Ex='01/01/58'\n"),
+        fmt,
+        cr::tai_seconds(0s)); // 00:00:00 TAI Wednesday, 1 January 1958
+
+  check(SV("%D='02/13/09'\t%F='2009-02-13'\t%x='02/13/09'\t%Ex='02/13/09'\n"),
+        fmt,
+        cr::tai_seconds(1'613'259'090s)); // 23:31:30 TAI Friday, 13 February 2009
+
+  // Use the global locale (fr_FR)
+#if defined(__APPLE__) || defined(__FreeBSD__)
+  check(SV("%D='01/01/58'\t%F='1958-01-01'\t%x='01.01.1958'\t%Ex='01.01.1958'\n"),
+        lfmt,
+        cr::tai_seconds(0s)); // 00:00:00 TAI Wednesday, 1 January 1958
+
+  check(SV("%D='02/13/09'\t%F='2009-02-13'\t%x='13.02.2009'\t%Ex='13.02.2009'\n"),
+        lfmt,
+        cr::tai_seconds(1'613'259'090s)); // 23:31:30 TAI Friday, 13 February 2009
+#else
+  check(SV("%D='01/01/58'\t%F='1958-01-01'\t%x='01/01/1958'\t%Ex='01/01/1958'\n"),
+        lfmt,
+        cr::tai_seconds(0s)); // 00:00:00 TAI Wednesday, 1 January 1958
+
+  check(SV("%D='02/13/09'\t%F='2009-02-13'\t%x='13/02/2009'\t%Ex='13/02/2009'\n"),
+        lfmt,
+        cr::tai_seconds(1'613'259'090s)); // 23:31:30 TAI Friday, 13 February 2009
+#endif
+
+  // Use supplied locale (ja_JP). This locale has a different alternate.
+#if defined(_WIN32) || defined(__APPLE__) || defined(_AIX) || defined(__FreeBSD__)
+  check(loc,
+        SV("%D='01/01/58'\t%F='1958-01-01'\t%x='1958/01/01'\t%Ex='1958/01/01'\n"),
+        lfmt,
+        cr::tai_seconds(0s)); // 00:00:00 TAI Wednesday, 1 January 1958
+
+  check(loc,
+        SV("%D='02/13/09'\t%F='2009-02-13'\t%x='2009/02/13'\t%Ex='2009/02/13'\n"),
+        lfmt,
+        cr::tai_seconds(1'613'259'090s)); // 23:31:30 TAI Friday, 13 February 2009
+#else  // defined(_WIN32) || defined(__APPLE__) || defined(_AIX) || defined(__FreeBSD__)
+  check(loc,
+        SV("%D='01/01/58'\t%F='1958-01-01'\t%x='1958年01月01日'\t%Ex='昭和33年01月01日'\n"),
+        lfmt,
+        cr::tai_seconds(0s)); // 00:00:00 TAI Wednesday, 1 January 1958
+
+  check(loc,
+        SV("%D='02/13/09'\t%F='2009-02-13'\t%x='2009年02月13日'\t%Ex='平成21年02月13日'\n"),
+        lfmt,
+        cr::tai_seconds(1'613'259'090s)); // 23:31:30 TAI Friday, 13 February 2009
+#endif // defined(_WIN32) || defined(__APPLE__) || defined(_AIX) || defined(__FreeBSD__)
+
+  std::locale::global(std::locale::classic());
+}
+
+template <class CharT>
+static void test_valid_values_time() {
+  using namespace std::literals::chrono_literals;
+  namespace cr = std::chrono;
+
+  constexpr std::basic_string_view<CharT> fmt = SV(
+      "{:"
+      "%%H='%H'%t"
+      "%%OH='%OH'%t"
+      "%%I='%I'%t"
+      "%%OI='%OI'%t"
+      "%%M='%M'%t"
+      "%%OM='%OM'%t"
+      "%%S='%S'%t"
+      "%%OS='%OS'%t"
+      "%%p='%p'%t"
+      "%%R='%R'%t"
+      "%%T='%T'%t"
+      "%%r='%r'%t"
+      "%%X='%X'%t"
+      "%%EX='%EX'%t"
+      "%n}");
+  constexpr std::basic_string_view<CharT> lfmt = SV(
+      "{:L"
+      "%%H='%H'%t"
+      "%%OH='%OH'%t"
+      "%%I='%I'%t"
+      "%%OI='%OI'%t"
+      "%%M='%M'%t"
+      "%%OM='%OM'%t"
+      "%%S='%S'%t"
+      "%%OS='%OS'%t"
+      "%%p='%p'%t"
+      "%%R='%R'%t"
+      "%%T='%T'%t"
+      "%%r='%r'%t"
+      "%%X='%X'%t"
+      "%%EX='%EX'%t"
+      "%n}");
+
+  const std::locale loc(LOCALE_ja_JP_UTF_8);
+  std::locale::global(std::locale(LOCALE_fr_FR_UTF_8));
+
+  // Non localized output using C-locale
+  check(SV("%H='00'\t"
+           "%OH='00'\t"
+           "%I='12'\t"
+           "%OI='12'\t"
+           "%M='00'\t"
+           "%OM='00'\t"
+           "%S='00'\t"
+           "%OS='00'\t"
+           "%p='AM'\t"
+           "%R='00:00'\t"
+           "%T='00:00:00'\t"
+           "%r='12:00:00 AM'\t"
+           "%X='00:00:00'\t"
+           "%EX='00:00:00'\t"
+           "\n"),
+        fmt,
+        cr::tai_seconds(0s)); // 00:00:00 TAI Wednesday, 1 January 1958
+
+  check(SV("%H='23'\t"
+           "%OH='23'\t"
+           "%I='11'\t"
+           "%OI='11'\t"
+           "%M='31'\t"
+           "%OM='31'\t"
+           "%S='30.123'\t"
+           "%OS='30.123'\t"
+           "%p='PM'\t"
+           "%R='23:31'\t"
+           "%T='23:31:30.123'\t"
+           "%r='11:31:30 PM'\t"
+           "%X='23:31:30'\t"
+           "%EX='23:31:30'\t"
+           "\n"),
+        fmt,
+        cr::tai_time<cr::milliseconds>(1'613'259'090'123ms)); // 23:31:30 TAI Friday, 13 February 2009
+  // Use the global locale (fr_FR)
+  check(SV("%H='00'\t"
+           "%OH='00'\t"
+           "%I='12'\t"
+           "%OI='12'\t"
+           "%M='00'\t"
+           "%OM='00'\t"
+           "%S='00'\t"
+           "%OS='00'\t"
+#if defined(_AIX)
+           "%p='AM'\t"
+#else
+           "%p=''\t"
+#endif
+           "%R='00:00'\t"
+           "%T='00:00:00'\t"
+#ifdef _WIN32
+           "%r='00:00:00'\t"
+#elif defined(_AIX)
+           "%r='12:00:00 AM'\t"
+#elif defined(__APPLE__) || defined(__FreeBSD__)
+           "%r=''\t"
+#else
+           "%r='12:00:00 '\t"
+#endif
+           "%X='00:00:00'\t"
+           "%EX='00:00:00'\t"
+           "\n"),
+        lfmt,
+        cr::tai_seconds(0s)); // 00:00:00 TAI Wednesday, 1 January 1958
+
+  check(SV("%H='23'\t"
+           "%OH='23'\t"
+           "%I='11'\t"
+           "%OI='11'\t"
+           "%M='31'\t"
+           "%OM='31'\t"
+           "%S='30,123'\t"
+           "%OS='30,123'\t"
+#if defined(_AIX)
+           "%p='PM'\t"
+#else
+           "%p=''\t"
+#endif
+           "%R='23:31'\t"
+           "%T='23:31:30,123'\t"
+#ifdef _WIN32
+           "%r='23:31:30'\t"
+#elif defined(_AIX)
+           "%r='11:31:30 PM'\t"
+#elif defined(__APPLE__) || defined(__FreeBSD__)
+           "%r=''\t"
+#else
+           "%r='11:31:30 '\t"
+#endif
+           "%X='23:31:30'\t"
+           "%EX='23:31:30'\t"
+           "\n"),
+        lfmt,
+        cr::tai_time<cr::milliseconds>(1'613'259'090'123ms)); // 23:31:30 TAI Friday, 13 February 2009
+
+  // Use supplied locale (ja_JP). This locale has a different alternate.
+#if defined(__APPLE__) || defined(_AIX) || defined(_WIN32) || defined(__FreeBSD__)
+  check(loc,
+        SV("%H='00'\t"
+           "%OH='00'\t"
+           "%I='12'\t"
+           "%OI='12'\t"
+           "%M='00'\t"
+           "%OM='00'\t"
+           "%S='00'\t"
+           "%OS='00'\t"
+#  if defined(__APPLE__)
+           "%p='AM'\t"
+#  else
+           "%p='午前'\t"
+#  endif
+           "%R='00:00'\t"
+           "%T='00:00:00'\t"
+#  if defined(__APPLE__) || defined(__FreeBSD__)
+#    if defined(__APPLE__)
+           "%r='12:00:00 AM'\t"
+#    else
+           "%r='12:00:00 午前'\t"
+#    endif
+           "%X='00時00分00秒'\t"
+           "%EX='00時00分00秒'\t"
+#  elif defined(_WIN32)
+           "%r='0:00:00'\t"
+           "%X='0:00:00'\t"
+           "%EX='0:00:00'\t"
+#  else
+           "%r='午前12:00:00'\t"
+           "%X='00:00:00'\t"
+           "%EX='00:00:00'\t"
+#  endif
+           "\n"),
+        lfmt,
+        cr::hh_mm_ss(0s));
+
+  check(loc,
+        SV("%H='23'\t"
+           "%OH='23'\t"
+           "%I='11'\t"
+           "%OI='11'\t"
+           "%M='31'\t"
+           "%OM='31'\t"
+           "%S='30.123'\t"
+           "%OS='30.123'\t"
+#  if defined(__APPLE__)
+           "%p='PM'\t"
+#  else
+           "%p='午後'\t"
+#  endif
+           "%R='23:31'\t"
+           "%T='23:31:30.123'\t"
+#  if defined(__APPLE__) || defined(__FreeBSD__)
+#    if defined(__APPLE__)
+           "%r='11:31:30 PM'\t"
+#    else
+           "%r='11:31:30 午後'\t"
+#    endif
+           "%X='23時31分30秒'\t"
+           "%EX='23時31分30秒'\t"
+#  elif defined(_WIN32)
+           "%r='23:31:30'\t"
+           "%X='23:31:30'\t"
+           "%EX='23:31:30'\t"
+#  else
+           "%r='午後11:31:30'\t"
+           "%X='23:31:30'\t"
+           "%EX='23:31:30'\t"
+#  endif
+           "\n"),
+        lfmt,
+        cr::hh_mm_ss(23h + 31min + 30s + 123ms));
+#else  // defined(__APPLE__) || defined(_AIX) || defined(_WIN32) || defined(__FreeBSD__)
+  check(loc,
+        SV("%H='00'\t"
+           "%OH='〇'\t"
+           "%I='12'\t"
+           "%OI='十二'\t"
+           "%M='00'\t"
+           "%OM='〇'\t"
+           "%S='00'\t"
+           "%OS='〇'\t"
+           "%p='午前'\t"
+           "%R='00:00'\t"
+           "%T='00:00:00'\t"
+           "%r='午前12時00分00秒'\t"
+           "%X='00時00分00秒'\t"
+           "%EX='00時00分00秒'\t"
+           "\n"),
+        lfmt,
+        cr::tai_seconds(0s)); // 00:00:00 TAI Wednesday, 1 January 1958
+
+  check(loc,
+        SV("%H='23'\t"
+           "%OH='二十三'\t"
+           "%I='11'\t"
+           "%OI='十一'\t"
+           "%M='31'\t"
+           "%OM='三十一'\t"
+           "%S='30.123'\t"
+           "%OS='三十.123'\t"
+           "%p='午後'\t"
+           "%R='23:31'\t"
+           "%T='23:31:30.123'\t"
+           "%r='午後11時31分30秒'\t"
+           "%X='23時31分30秒'\t"
+           "%EX='23時31分30秒'\t"
+           "\n"),
+        lfmt,
+        cr::tai_time<cr::milliseconds>(1'613'259'090'123ms)); // 23:31:30 TAI Friday, 13 February 2009
+#endif // defined(__APPLE__) || defined(_AIX) || defined(_WIN32) || defined(__FreeBSD__)
+
+  std::locale::global(std::locale::classic());
+}
+
+template <class CharT>
+static void test_valid_values_date_time() {
+  using namespace std::literals::chrono_literals;
+  namespace cr = std::chrono;
+
+  constexpr std::basic_string_view<CharT> fmt  = SV("{:%%c='%c'%t%%Ec='%Ec'%n}");
+  constexpr std::basic_string_view<CharT> lfmt = SV("{:L%%c='%c'%t%%Ec='%Ec'%n}");
+
+  const std::locale loc(LOCALE_ja_JP_UTF_8);
+  std::locale::global(std::locale(LOCALE_fr_FR_UTF_8));
+
+  // Non localized output using C-locale
+  check(SV("%c='Wed Jan  1 00:00:00 1958'\t%Ec='Wed Jan  1 00:00:00 1958'\n"),
+        fmt,
+        cr::tai_seconds(0s)); // 00:00:00 TAI Wednesday, 1 January 1958
+
+  check(SV("%c='Fri Feb 13 23:31:30 2009'\t%Ec='Fri Feb 13 23:31:30 2009'\n"),
+        fmt,
+        cr::tai_seconds(1'613'259'090s)); // 23:31:30 TAI Friday, 13 February 2009
+
+  // Use the global locale (fr_FR)
+  check(
+// https://sourceware.org/bugzilla/show_bug.cgi?id=24054
+#if defined(__powerpc__) && defined(__linux__)
+      SV("%c='mer. 01 janv. 1958 00:00:00 TAI'\t%Ec='mer. 01 janv. 1958 00:00:00 TAI'\n"),
+#elif defined(__GLIBC__) && __GLIBC__ <= 2 && __GLIBC_MINOR__ < 29
+      SV("%c='mer. 01 janv. 1958 00:00:00 GMT'\t%Ec='mer. 01 janv. 1958 00:00:00 GMT'\n"),
+#elif defined(_AIX)
+      SV("%c=' 1 janvier 1958 à 00:00:00 TAI'\t%Ec=' 1 janvier 1958 à 00:00:00 TAI'\n"),
+#elif defined(__APPLE__)
+      SV("%c='Jeu  1 jan 00:00:00 1958'\t%Ec='Jeu  1 jan 00:00:00 1958'\n"),
+#elif defined(_WIN32)
+      SV("%c='01/01/1958 00:00:00'\t%Ec='01/01/1958 00:00:00'\n"),
+#elif defined(__FreeBSD__)
+      SV("%c='mer.  1 janv. 00:00:00 1958'\t%Ec='mer.  1 janv. 00:00:00 1958'\n"),
+#else
+      SV("%c='mer. 01 janv. 1958 00:00:00'\t%Ec='mer. 01 janv. 1958 00:00:00'\n"),
+#endif
+      lfmt,
+      cr::tai_seconds(0s)); // 00:00:00 TAI Wednesday, 1 January 1958
+
+  check(
+// https://sourceware.org/bugzilla/show_bug.cgi?id=24054
+#if defined(__powerpc__) && defined(__linux__)
+      SV("%c='ven. 13 févr. 2009 23:31:30 TAI'\t%Ec='ven. 13 févr. 2009 23:31:30 TAI'\n"),
+#elif defined(__GLIBC__) && __GLIBC__ <= 2 && __GLIBC_MINOR__ < 29
+      SV("%c='ven. 13 févr. 2009 23:31:30 GMT'\t%Ec='ven. 13 févr. 2009 23:31:30 GMT'\n"),
+#elif defined(_AIX)
+      SV("%c='13 février 2009 à 23:31:30 TAI'\t%Ec='13 février 2009 à 23:31:30 TAI'\n"),
+#elif defined(__APPLE__)
+      SV("%c='Ven 13 fév 23:31:30 2009'\t%Ec='Ven 13 fév 23:31:30 2009'\n"),
+#elif defined(_WIN32)
+      SV("%c='13/02/2009 23:31:30'\t%Ec='13/02/2009 23:31:30'\n"),
+#elif defined(__FreeBSD__)
+      SV("%c='ven. 13 févr. 23:31:30 2009'\t%Ec='ven. 13 févr. 23:31:30 2009'\n"),
+#else
+      SV("%c='ven. 13 févr. 2009 23:31:30'\t%Ec='ven. 13 févr. 2009 23:31:30'\n"),
+#endif
+      lfmt,
+      cr::tai_seconds(1'613'259'090s)); // 23:31:30 TAI Friday, 13 February 2009
+
+  // Use supplied locale (ja_JP). This locale has a different alternate.a
+#if defined(__APPLE__) || defined(__FreeBSD__)
+  check(loc,
+        SV("%c='水  1/ 1 00:00:00 1958'\t%Ec='水  1/ 1 00:00:00 1958'\n"),
+        lfmt,
+        cr::tai_seconds(0s)); // 00:00:00 TAI Wednesday, 1 January 1958
+  check(loc,
+        SV("%c='金  2/13 23:31:30 2009'\t%Ec='金  2/13 23:31:30 2009'\n"),
+        lfmt,
+        cr::tai_seconds(1'613'259'090s)); // 23:31:30 TAI Friday, 13 February 2009
+#elif defined(_AIX)                       // defined(__APPLE__)|| defined(__FreeBSD__)
+  check(loc,
+        SV("%c='1958年01月 1日 00:00:00 TAI'\t%Ec='1958年01月 1日 00:00:00 TAI'\n"),
+        lfmt,
+        cr::tai_seconds(0s)); // 00:00:00 TAI Wednesday, 1 January 1958
+  check(loc,
+        SV("%c='2009年02月13日 23:31:30 TAI'\t%Ec='2009年02月13日 23:31:30 TAI'\n"),
+        lfmt,
+        cr::tai_seconds(1'613'259'090s)); // 23:31:30 TAI Friday, 13 February 2009
+#elif defined(_WIN32)                     // defined(__APPLE__)|| defined(__FreeBSD__)
+  check(loc,
+        SV("%c='1958/01/01 0:00:00'\t%Ec='1958/01/01 0:00:00'\n"),
+        lfmt,
+        cr::tai_seconds(0s)); // 00:00:00 TAI Wednesday, 1 January 1958
+  check(loc,
+        SV("%c='2009/02/13 23:31:30'\t%Ec='2009/02/13 23:31:30'\n"),
+        lfmt,
+        cr::tai_seconds(1'613'259'090s)); // 23:31:30 TAI Friday, 13 February 2009
+#else                                     // defined(__APPLE__)|| defined(__FreeBSD__)
+  check(loc,
+        SV("%c='1958年01月01日 00時00分00秒'\t%Ec='昭和33年01月01日 00時00分00秒'\n"),
+        lfmt,
+        cr::tai_seconds(0s)); // 00:00:00 TAI Wednesday, 1 January 1958
+
+  check(loc,
+        SV("%c='2009年02月13日 23時31分30秒'\t%Ec='平成21年02月13日 23時31分30秒'\n"),
+        lfmt,
+        cr::tai_seconds(1'613'259'090s)); // 23:31:30 TAI Friday, 13 February 2009
+#endif                                    // defined(__APPLE__)|| defined(__FreeBSD__)
+
+  std::locale::global(std::locale::classic());
+}
+
+template <class CharT>
+static void test_valid_values_time_zone() {
+  using namespace std::literals::chrono_literals;
+  namespace cr = std::chrono;
+
+  constexpr std::basic_string_view<CharT> fmt  = SV("{:%%z='%z'%t%%Ez='%Ez'%t%%Oz='%Oz'%t%%Z='%Z'%n}");
+  constexpr std::basic_string_view<CharT> lfmt = SV("{:L%%z='%z'%t%%Ez='%Ez'%t%%Oz='%Oz'%t%%Z='%Z'%n}");
+
+  const std::locale loc(LOCALE_ja_JP_UTF_8);
+  std::locale::global(std::locale(LOCALE_fr_FR_UTF_8));
+
+  // Non localized output using C-locale
+  check(SV("%z='+0000'\t%Ez='+00:00'\t%Oz='+00:00'\t%Z='TAI'\n"),
+        fmt,
+        cr::tai_seconds(0s)); // 00:00:00 TAI Wednesday, 1 January 1958
+
+  // Use the global locale (fr_FR)
+  check(SV("%z='+0000'\t%Ez='+00:00'\t%Oz='+00:00'\t%Z='TAI'\n"),
+        lfmt,
+        cr::tai_seconds(0s)); // 00:00:00 TAI Wednesday, 1 January 1958
+
+  // Use supplied locale (ja_JP).
+  check(loc,
+        SV("%z='+0000'\t%Ez='+00:00'\t%Oz='+00:00'\t%Z='TAI'\n"),
+        lfmt,
+        cr::tai_seconds(0s)); // 00:00:00 TAI Wednesday, 1 January 1958
+
+  std::locale::global(std::locale::classic());
+}
+
+template <class CharT>
+static void test_valid_values() {
+  test_valid_values_year<CharT>();
+  test_valid_values_month<CharT>();
+  test_valid_values_day<CharT>();
+  test_valid_values_weekday<CharT>();
+  test_valid_values_day_of_year<CharT>();
+  test_valid_values_week<CharT>();
+  test_valid_values_iso_8601_week<CharT>();
+  test_valid_values_date<CharT>();
+  test_valid_values_time<CharT>();
+  test_valid_values_date_time<CharT>();
+  test_valid_values_time_zone<CharT>();
+}
+
+template <class CharT>
+static void test() {
+  using namespace std::literals::chrono_literals;
+  namespace cr = std::chrono;
+
+  test_no_chrono_specs<CharT>();
+  test_valid_values<CharT>();
+  check_invalid_types<CharT>(
+      {SV("a"),  SV("A"),  SV("b"),  SV("B"),  SV("c"),  SV("C"),  SV("d"),  SV("D"),  SV("e"),  SV("F"),  SV("g"),
+       SV("G"),  SV("h"),  SV("H"),  SV("I"),  SV("j"),  SV("m"),  SV("M"),  SV("p"),  SV("r"),  SV("R"),  SV("S"),
+       SV("T"),  SV("u"),  SV("U"),  SV("V"),  SV("w"),  SV("W"),  SV("x"),  SV("X"),  SV("y"),  SV("Y"),  SV("z"),
+       SV("Z"),  SV("Ec"), SV("EC"), SV("Ex"), SV("EX"), SV("Ey"), SV("EY"), SV("Ez"), SV("Od"), SV("Oe"), SV("OH"),
+       SV("OI"), SV("Om"), SV("OM"), SV("OS"), SV("Ou"), SV("OU"), SV("OV"), SV("Ow"), SV("OW"), SV("Oy"), SV("Oz")},
+      cr::tai_seconds(0s));
+
+  check_exception("The format specifier expects a '%' or a '}'", SV("{:A"), cr::tai_seconds(0s));
+  check_exception("The chrono specifiers contain a '{'", SV("{:%%{"), cr::tai_seconds(0s));
+  check_exception("End of input while parsing a conversion specifier", SV("{:%"), cr::tai_seconds(0s));
+  check_exception("End of input while parsing the modifier E", SV("{:%E"), cr::tai_seconds(0s));
+  check_exception("End of input while parsing the modifier O", SV("{:%O"), cr::tai_seconds(0s));
+
+  // Precision not allowed
+  check_exception("The format specifier expects a '%' or a '}'", SV("{:.3}"), cr::tai_seconds(0s));
+}
+
+int main(int, char**) {
+  test<char>();
+
+#ifndef TEST_HAS_NO_WIDE_CHARACTERS
+  test<wchar_t>();
+#endif
+
+  return 0;
+}
diff --git a/libcxx/test/std/utilities/expected/expected.expected/ctor/ctor.copy.pass.cpp b/libcxx/test/std/utilities/expected/expected.expected/ctor/ctor.copy.pass.cpp
index 9e78596929fb6..028655412c921 100644
--- a/libcxx/test/std/utilities/expected/expected.expected/ctor/ctor.copy.pass.cpp
+++ b/libcxx/test/std/utilities/expected/expected.expected/ctor/ctor.copy.pass.cpp
@@ -121,8 +121,8 @@ constexpr bool test() {
   }
 
   {
-    // TODO(LLVM 20): Remove once we drop support for Clang 17
-#if defined(TEST_CLANG_VER) && TEST_CLANG_VER >= 1800
+    // TODO: Drop this once AppleClang is upgraded
+#ifndef TEST_COMPILER_APPLE_CLANG
     // https://github.com/llvm/llvm-project/issues/92676
     std::expected<Any, int> e1;
     auto e2 = e1;
diff --git a/libcxx/test/std/utilities/format/format.arguments/format.arg/visit.pass.cpp b/libcxx/test/std/utilities/format/format.arguments/format.arg/visit.pass.cpp
index 829b74121b9c6..20e0a5ed66bd0 100644
--- a/libcxx/test/std/utilities/format/format.arguments/format.arg/visit.pass.cpp
+++ b/libcxx/test/std/utilities/format/format.arguments/format.arg/visit.pass.cpp
@@ -9,7 +9,6 @@
 // UNSUPPORTED: c++03, c++11, c++14, c++17, c++20, c++23
 // UNSUPPORTED: GCC-ALWAYS_INLINE-FIXME
 // The tested functionality needs deducing this.
-// UNSUPPORTED: clang-17
 // XFAIL: apple-clang
 
 // <format>
diff --git a/libcxx/test/std/utilities/format/format.arguments/format.arg/visit.return_type.pass.cpp b/libcxx/test/std/utilities/format/format.arguments/format.arg/visit.return_type.pass.cpp
index 874d609432f22..8a79dd4d50f20 100644
--- a/libcxx/test/std/utilities/format/format.arguments/format.arg/visit.return_type.pass.cpp
+++ b/libcxx/test/std/utilities/format/format.arguments/format.arg/visit.return_type.pass.cpp
@@ -9,7 +9,6 @@
 // UNSUPPORTED: c++03, c++11, c++14, c++17, c++20, c++23
 // UNSUPPORTED: GCC-ALWAYS_INLINE-FIXME
 // The tested functionality needs deducing this.
-// UNSUPPORTED: clang-17
 // XFAIL: apple-clang
 
 // <format>
diff --git a/libcxx/test/std/utilities/format/format.arguments/format.arg/visit_format_arg.deprecated.verify.cpp b/libcxx/test/std/utilities/format/format.arguments/format.arg/visit_format_arg.deprecated.verify.cpp
index e3e3e9a19e122..146ceba58872e 100644
--- a/libcxx/test/std/utilities/format/format.arguments/format.arg/visit_format_arg.deprecated.verify.cpp
+++ b/libcxx/test/std/utilities/format/format.arguments/format.arg/visit_format_arg.deprecated.verify.cpp
@@ -8,7 +8,6 @@
 
 // UNSUPPORTED: c++03, c++11, c++14, c++17, c++20, c++23
 // UNSUPPORTED: GCC-ALWAYS_INLINE-FIXME
-// UNSUPPORTED: clang-17
 // XFAIL: apple-clang
 
 // <format>
diff --git a/libcxx/test/std/utilities/format/format.formattable/concept.formattable.compile.pass.cpp b/libcxx/test/std/utilities/format/format.formattable/concept.formattable.compile.pass.cpp
index 88a485a256c80..ec27acd75e110 100644
--- a/libcxx/test/std/utilities/format/format.formattable/concept.formattable.compile.pass.cpp
+++ b/libcxx/test/std/utilities/format/format.formattable/concept.formattable.compile.pass.cpp
@@ -24,6 +24,7 @@
 #include <concepts>
 #include <deque>
 #include <filesystem>
+#include <flat_map>
 #include <format>
 #include <forward_list>
 #include <list>
@@ -242,6 +243,13 @@ void test_P2286() {
   assert_is_formattable<std::multiset<int>, CharT>();
   assert_is_formattable<std::multimap<int, int>, CharT>();
 
+#if TEST_STD_VER >= 23
+  // assert_is_formattable<std::flat_set<int>, CharT>();
+  assert_is_formattable<std::flat_map<int, int>, CharT>();
+  // assert_is_formattable<std::flat_multiset<int>, CharT>();
+  assert_is_formattable<std::flat_multimap<int, int>, CharT>();
+#endif // TEST_STD_VER >= 2
+
   assert_is_formattable<std::unordered_set<int>, CharT>();
   assert_is_formattable<std::unordered_map<int, int>, CharT>();
   assert_is_formattable<std::unordered_multiset<int>, CharT>();
diff --git a/libcxx/test/std/utilities/format/format.functions/format_tests.h b/libcxx/test/std/utilities/format/format.functions/format_tests.h
index b2ed6775fe8a1..3969b341cb146 100644
--- a/libcxx/test/std/utilities/format/format.functions/format_tests.h
+++ b/libcxx/test/std/utilities/format/format.functions/format_tests.h
@@ -3038,7 +3038,7 @@ void format_test_buffer_optimizations(TestFunction check) {
   // Used to validate our test sets are the proper size.
   // To test the chunked operations it needs to be larger than the internal
   // buffer. Picked a nice looking number.
-  constexpr int minimum = 3 * std::__format::__internal_storage<CharT>::__buffer_size;
+  constexpr int minimum = 3 * 256;
 #else
   constexpr int minimum = 1;
 #endif
diff --git a/libcxx/test/std/utilities/format/format.range/format.range.fmtmap/format.functions.tests.h b/libcxx/test/std/utilities/format/format.range/format.range.fmtmap/format.functions.tests.h
index fc638da9c840d..3eba50d5aff1a 100644
--- a/libcxx/test/std/utilities/format/format.range/format.range.fmtmap/format.functions.tests.h
+++ b/libcxx/test/std/utilities/format/format.range/format.range.fmtmap/format.functions.tests.h
@@ -10,6 +10,8 @@
 #define TEST_STD_UTILITIES_FORMAT_FORMAT_RANGE_FORMAT_RANGE_FMTMAP_FORMAT_FUNCTIONS_TESTS_H
 
 #include <algorithm>
+#include <deque>
+#include <flat_map>
 #include <format>
 #include <map>
 #include <unordered_map>
@@ -241,10 +243,7 @@ void test_char_to_wchar(TestFunction check, ExceptionTest check_exception) {
 // Bool
 //
 template <class CharT, class TestFunction, class ExceptionTest>
-void test_bool(TestFunction check, ExceptionTest check_exception) {
-  // duplicates are stored in order of insertion
-  std::multimap<bool, int> input{{true, 42}, {false, 0}, {true, 1}};
-
+void test_bool(TestFunction check, ExceptionTest check_exception, auto&& input) {
   check(SV("{false: 0, true: 42, true: 1}"), SV("{}"), input);
   check(SV("{false: 0, true: 42, true: 1}^42"), SV("{}^42"), input);
   check(SV("{false: 0, true: 42, true: 1}^42"), SV("{:}^42"), input);
@@ -339,6 +338,17 @@ void test_bool(TestFunction check, ExceptionTest check_exception) {
       "The argument index value is too large for the number of arguments supplied", SV("{:^^{}:#>{}}"), input, 41);
 }
 
+template <class CharT, class TestFunction, class ExceptionTest>
+void test_bool(TestFunction check, ExceptionTest check_exception) {
+  // duplicates are stored in order of insertion
+  test_bool<CharT>(check, check_exception, std::multimap<bool, int>{{true, 42}, {false, 0}, {true, 1}});
+#if TEST_STD_VER >= 23
+  test_bool<CharT>(check,
+                   check_exception,
+                   std::flat_multimap<bool, int, std::less<bool>, std::deque<bool>>{{true, 42}, {false, 0}, {true, 1}});
+#endif
+}
+
 //
 // Integral
 //
@@ -442,6 +452,9 @@ void test_int(TestFunction check, ExceptionTest check_exception, auto&& input) {
 template <class CharT, class TestFunction, class ExceptionTest>
 void test_int(TestFunction check, ExceptionTest check_exception) {
   test_int<CharT>(check, check_exception, std::map<int, int>{{1, -1}, {42, -42}, {-42, 42}});
+#if TEST_STD_VER >= 23
+  test_int<CharT>(check, check_exception, std::flat_map<int, int>{{1, -1}, {42, -42}, {-42, 42}});
+#endif
 }
 
 //
diff --git a/libcxx/test/std/utilities/meta/meta.rel/is_virtual_base_of.pass.cpp b/libcxx/test/std/utilities/meta/meta.rel/is_virtual_base_of.pass.cpp
index 6b34d56e2c6f4..bcffa5812d04e 100644
--- a/libcxx/test/std/utilities/meta/meta.rel/is_virtual_base_of.pass.cpp
+++ b/libcxx/test/std/utilities/meta/meta.rel/is_virtual_base_of.pass.cpp
@@ -9,7 +9,7 @@
 // UNSUPPORTED: c++03, c++11, c++14, c++17, c++20, c++23
 
 // These compilers don't support __builtin_is_virtual_base_of yet.
-// UNSUPPORTED: clang-17, clang-18, clang-19, gcc-14, apple-clang-16, apple-clang-17
+// UNSUPPORTED: clang-18, clang-19, gcc-14, apple-clang-16
 
 // <type_traits>
 
diff --git a/libcxx/test/std/utilities/meta/meta.unary/meta.unary.prop/is_implicit_lifetime.pass.cpp b/libcxx/test/std/utilities/meta/meta.unary/meta.unary.prop/is_implicit_lifetime.pass.cpp
index a6ab77158aae1..681ad13a07dfd 100644
--- a/libcxx/test/std/utilities/meta/meta.unary/meta.unary.prop/is_implicit_lifetime.pass.cpp
+++ b/libcxx/test/std/utilities/meta/meta.unary/meta.unary.prop/is_implicit_lifetime.pass.cpp
@@ -9,7 +9,7 @@
 // UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
 
 // These compilers don't support __builtin_is_implicit_lifetime yet.
-// UNSUPPORTED: clang-17, clang-18, clang-19, gcc-14, apple-clang-15, apple-clang-16
+// UNSUPPORTED: clang-18, clang-19, gcc-14, apple-clang-15, apple-clang-16, apple-clang-17
 
 // <type_traits>
 
diff --git a/libcxx/test/std/utilities/meta/meta.unary/meta.unary.prop/is_implicit_lifetime.verify.cpp b/libcxx/test/std/utilities/meta/meta.unary/meta.unary.prop/is_implicit_lifetime.verify.cpp
index 25bba30da612e..34462f9bf0ec6 100644
--- a/libcxx/test/std/utilities/meta/meta.unary/meta.unary.prop/is_implicit_lifetime.verify.cpp
+++ b/libcxx/test/std/utilities/meta/meta.unary/meta.unary.prop/is_implicit_lifetime.verify.cpp
@@ -9,7 +9,7 @@
 // UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
 
 // These compilers don't support __builtin_is_implicit_lifetime yet.
-// UNSUPPORTED: clang-17, clang-18, clang-19, gcc-14, apple-clang-15, apple-clang-16
+// UNSUPPORTED: clang-18, clang-19, gcc-14, apple-clang-15, apple-clang-16, apple-clang-17
 
 // <type_traits>
 
diff --git a/libcxx/test/std/utilities/utility/pairs/pairs.pair/nttp.equivalence.compile.pass.cpp b/libcxx/test/std/utilities/utility/pairs/pairs.pair/nttp.equivalence.compile.pass.cpp
index db45a56feb88a..f5fd5a674882b 100644
--- a/libcxx/test/std/utilities/utility/pairs/pairs.pair/nttp.equivalence.compile.pass.cpp
+++ b/libcxx/test/std/utilities/utility/pairs/pairs.pair/nttp.equivalence.compile.pass.cpp
@@ -7,7 +7,6 @@
 //===----------------------------------------------------------------------===//
 
 // UNSUPPORTED: c++03, c++11, c++14, c++17
-// UNSUPPORTED: clang-17
 
 // <utility>
 
diff --git a/libcxx/test/std/utilities/utility/pairs/pairs.pair/nttp.verify.cpp b/libcxx/test/std/utilities/utility/pairs/pairs.pair/nttp.verify.cpp
index ac081495a6205..499ba6b243bed 100644
--- a/libcxx/test/std/utilities/utility/pairs/pairs.pair/nttp.verify.cpp
+++ b/libcxx/test/std/utilities/utility/pairs/pairs.pair/nttp.verify.cpp
@@ -7,7 +7,6 @@
 //===----------------------------------------------------------------------===//
 
 // UNSUPPORTED: c++03, c++11, c++14, c++17
-// UNSUPPORTED: clang-17
 
 // <utility>
 
diff --git a/libcxx/test/std/utilities/variant/variant.visit.member/robust_against_adl.pass.cpp b/libcxx/test/std/utilities/variant/variant.visit.member/robust_against_adl.pass.cpp
index bea6d949924bd..7be7c7ff9122b 100644
--- a/libcxx/test/std/utilities/variant/variant.visit.member/robust_against_adl.pass.cpp
+++ b/libcxx/test/std/utilities/variant/variant.visit.member/robust_against_adl.pass.cpp
@@ -7,8 +7,6 @@
 //===----------------------------------------------------------------------===//
 
 // UNSUPPORTED: c++03, c++11, c++14, c++17, c++20, c++23
-// The tested functionality needs deducing this.
-// UNSUPPORTED: clang-17
 // XFAIL: apple-clang
 
 // <variant>
diff --git a/libcxx/test/std/utilities/variant/variant.visit.member/visit.pass.cpp b/libcxx/test/std/utilities/variant/variant.visit.member/visit.pass.cpp
index 0da23fd58ccaa..f68112d30fc35 100644
--- a/libcxx/test/std/utilities/variant/variant.visit.member/visit.pass.cpp
+++ b/libcxx/test/std/utilities/variant/variant.visit.member/visit.pass.cpp
@@ -8,7 +8,6 @@
 
 // UNSUPPORTED: c++03, c++11, c++14, c++17, c++20, c++23
 // The tested functionality needs deducing this.
-// UNSUPPORTED: clang-17
 // XFAIL: apple-clang
 
 // <variant>
diff --git a/libcxx/test/std/utilities/variant/variant.visit.member/visit_return_type.pass.cpp b/libcxx/test/std/utilities/variant/variant.visit.member/visit_return_type.pass.cpp
index 7429cdf80faca..8093af0aba587 100644
--- a/libcxx/test/std/utilities/variant/variant.visit.member/visit_return_type.pass.cpp
+++ b/libcxx/test/std/utilities/variant/variant.visit.member/visit_return_type.pass.cpp
@@ -8,7 +8,6 @@
 
 // UNSUPPORTED: c++03, c++11, c++14, c++17, c++20, c++23
 // The tested functionality needs deducing this.
-// UNSUPPORTED: clang-17
 // XFAIL: apple-clang
 
 // <variant>
diff --git a/libcxx/test/support/MinSequenceContainer.h b/libcxx/test/support/MinSequenceContainer.h
index d0e29ae40c400..7fee4dd0fbdc1 100644
--- a/libcxx/test/support/MinSequenceContainer.h
+++ b/libcxx/test/support/MinSequenceContainer.h
@@ -31,7 +31,7 @@ struct MinSequenceContainer {
   const_iterator cbegin() const { return const_iterator(data_.data()); }
   iterator end() { return begin() + size(); }
   const_iterator end() const { return begin() + size(); }
-  size_type size() const { return data_.size(); }
+  size_type size() const { return static_cast<size_type>(data_.size()); }
   bool empty() const { return data_.empty(); }
 
   void clear() { data_.clear(); }
diff --git a/libcxx/test/support/concat_macros.h b/libcxx/test/support/concat_macros.h
index d7340b8faf6e5..50b6562197392 100644
--- a/libcxx/test/support/concat_macros.h
+++ b/libcxx/test/support/concat_macros.h
@@ -11,6 +11,7 @@
 
 #include <cstdio>
 #include <string>
+#include <source_location>
 
 #include "assert_macros.h"
 #include "test_macros.h"
@@ -134,6 +135,10 @@ OutIt test_transcode(InIt first, InIt last, OutIt out_it) {
   return out_it;
 }
 
+std::ostream& operator<<(std::ostream& os, const std::source_location& loc) {
+  return os << loc.file_name() << ':' << loc.line() << ':' << loc.column();
+}
+
 template <class T>
 concept test_streamable = requires(std::stringstream& stream, T&& value) { stream << value; };
 
diff --git a/libcxx/test/support/min_allocator.h b/libcxx/test/support/min_allocator.h
index 18f51f8072640..d3ee27a23bc89 100644
--- a/libcxx/test/support/min_allocator.h
+++ b/libcxx/test/support/min_allocator.h
@@ -394,15 +394,9 @@ class min_allocator
     template <class U>
     TEST_CONSTEXPR_CXX20 min_allocator(min_allocator<U>) {}
 
-    TEST_CONSTEXPR_CXX20 pointer allocate(std::ptrdiff_t n)
-    {
-        return pointer(std::allocator<T>().allocate(n));
-    }
+    TEST_CONSTEXPR_CXX20 pointer allocate(std::size_t n) { return pointer(std::allocator<T>().allocate(n)); }
 
-    TEST_CONSTEXPR_CXX20 void deallocate(pointer p, std::ptrdiff_t n)
-    {
-        std::allocator<T>().deallocate(p.ptr_, n);
-    }
+    TEST_CONSTEXPR_CXX20 void deallocate(pointer p, std::size_t n) { std::allocator<T>().deallocate(p.ptr_, n); }
 
     TEST_CONSTEXPR_CXX20 friend bool operator==(min_allocator, min_allocator) {return true;}
     TEST_CONSTEXPR_CXX20 friend bool operator!=(min_allocator x, min_allocator y) {return !(x == y);}
diff --git a/libcxx/test/support/sized_allocator.h b/libcxx/test/support/sized_allocator.h
new file mode 100644
index 0000000000000..8d52f5bf252c7
--- /dev/null
+++ b/libcxx/test/support/sized_allocator.h
@@ -0,0 +1,60 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef TEST_SUPPORT_SIZED_ALLOCATOR_H
+#define TEST_SUPPORT_SIZED_ALLOCATOR_H
+
+#include <cstddef>
+#include <limits>
+#include <memory>
+#include <new>
+
+#include "test_macros.h"
+
+// Allocator with a provided size_type and difference_type, used to test corner cases
+// like arithmetic on Allocator::size_type in generic code.
+template <typename T, typename Size = std::size_t, typename Difference = std::ptrdiff_t>
+class sized_allocator {
+  template <typename U, typename Sz, typename Diff>
+  friend class sized_allocator;
+
+public:
+  using value_type                  = T;
+  using size_type                   = Size;
+  using difference_type             = Difference;
+  using propagate_on_container_swap = std::true_type;
+
+  TEST_CONSTEXPR_CXX20 explicit sized_allocator(int d = 0) : data_(d) {}
+
+  template <typename U, typename Sz, typename Diff>
+  TEST_CONSTEXPR_CXX20 sized_allocator(const sized_allocator<U, Sz, Diff>& a) TEST_NOEXCEPT : data_(a.data_) {}
+
+  TEST_CONSTEXPR_CXX20 T* allocate(size_type n) {
+    if (n > max_size())
+      TEST_THROW(std::bad_array_new_length());
+    return std::allocator<T>().allocate(n);
+  }
+
+  TEST_CONSTEXPR_CXX20 void deallocate(T* p, size_type n) TEST_NOEXCEPT { std::allocator<T>().deallocate(p, n); }
+
+  TEST_CONSTEXPR size_type max_size() const TEST_NOEXCEPT {
+    return std::numeric_limits<size_type>::max() / sizeof(value_type);
+  }
+
+private:
+  int data_;
+
+  TEST_CONSTEXPR friend bool operator==(const sized_allocator& a, const sized_allocator& b) {
+    return a.data_ == b.data_;
+  }
+  TEST_CONSTEXPR friend bool operator!=(const sized_allocator& a, const sized_allocator& b) {
+    return a.data_ != b.data_;
+  }
+};
+
+#endif // TEST_SUPPORT_SIZED_ALLOCATOR_H
diff --git a/libcxx/utils/ci/Dockerfile b/libcxx/utils/ci/Dockerfile
index d0144908f098b..bd41f288686a4 100644
--- a/libcxx/utils/ci/Dockerfile
+++ b/libcxx/utils/ci/Dockerfile
@@ -7,18 +7,17 @@
 # ===----------------------------------------------------------------------===##
 #
 # This file defines the buildkite and github actions builder images.
-# You can build both images using:
+# This images are tagged with <tag>. You can build both images using:
 #
-#   docker compose build
+#   TAG=<tag> docker compose build
 #
 # Or you can select a single image to build
 #
-#  docker compose build buildkite-builder
+#  TAG=test docker compose build actions-builder
 #
 # The final images can be found at
 #
-#  ghcr.io/libcxx/buildkite-builder
-#  ghcr.io/libcxx/actions-builder
+#  ghcr.io/libcxx/libcxx-linux-builder
 #  ghcr.io/libcxx/android-buildkite-builder
 #
 # Members of the github.com/libcxx/ organizations can push new images to the CI.
diff --git a/libcxx/utils/ci/docker-compose.yml b/libcxx/utils/ci/docker-compose.yml
index 1e5d35a7fecdb..16db1b0e3acb3 100644
--- a/libcxx/utils/ci/docker-compose.yml
+++ b/libcxx/utils/ci/docker-compose.yml
@@ -1,6 +1,6 @@
 x-versions: &compiler_versions
   GCC_LATEST_VERSION: 14
-  LLVM_HEAD_VERSION: 20
+  LLVM_HEAD_VERSION: 21
 
 services:
   actions-builder:
diff --git a/libcxx/utils/data/unicode/DerivedCoreProperties.txt b/libcxx/utils/data/unicode/DerivedCoreProperties.txt
index 220c55685d4b0..1075638f1a654 100644
--- a/libcxx/utils/data/unicode/DerivedCoreProperties.txt
+++ b/libcxx/utils/data/unicode/DerivedCoreProperties.txt
@@ -1,8 +1,8 @@
-# DerivedCoreProperties-15.1.0.txt
-# Date: 2023-08-07, 15:21:24 GMT
-# © 2023 Unicode®, Inc.
+# DerivedCoreProperties-16.0.0.txt
+# Date: 2024-05-31, 18:09:32 GMT
+# © 2024 Unicode®, Inc.
 # Unicode and the Unicode Logo are registered trademarks of Unicode, Inc. in the U.S. and other countries.
-# For terms of use, see https://www.unicode.org/terms_of_use.html
+# For terms of use and license, see https://www.unicode.org/terms_of_use.html
 #
 # Unicode Character Database
 #   For documentation, see https://www.unicode.org/reports/tr44/
@@ -177,6 +177,7 @@ FF5C          ; Math # Sm       FULLWIDTH VERTICAL LINE
 FF5E          ; Math # Sm       FULLWIDTH TILDE
 FFE2          ; Math # Sm       FULLWIDTH NOT SIGN
 FFE9..FFEC    ; Math # Sm   [4] HALFWIDTH LEFTWARDS ARROW..HALFWIDTH DOWNWARDS ARROW
+10D8E..10D8F  ; Math # Sm   [2] GARAY PLUS SIGN..GARAY MINUS SIGN
 1D400..1D454  ; Math # L&  [85] MATHEMATICAL BOLD CAPITAL A..MATHEMATICAL ITALIC SMALL G
 1D456..1D49C  ; Math # L&  [71] MATHEMATICAL ITALIC SMALL I..MATHEMATICAL SCRIPT CAPITAL A
 1D49E..1D49F  ; Math # L&   [2] MATHEMATICAL SCRIPT CAPITAL C..MATHEMATICAL SCRIPT CAPITAL D
@@ -253,7 +254,7 @@ FFE9..FFEC    ; Math # Sm   [4] HALFWIDTH LEFTWARDS ARROW..HALFWIDTH DOWNWARDS A
 1EEAB..1EEBB  ; Math # Lo  [17] ARABIC MATHEMATICAL DOUBLE-STRUCK LAM..ARABIC MATHEMATICAL DOUBLE-STRUCK GHAIN
 1EEF0..1EEF1  ; Math # Sm   [2] ARABIC MATHEMATICAL OPERATOR MEEM WITH HAH WITH TATWEEL..ARABIC MATHEMATICAL OPERATOR HAH WITH DAL
 
-# Total code points: 2310
+# Total code points: 2312
 
 # ================================================
 
@@ -280,6 +281,7 @@ FFE9..FFEC    ; Math # Sm   [4] HALFWIDTH LEFTWARDS ARROW..HALFWIDTH DOWNWARDS A
 02EC          ; Alphabetic # Lm       MODIFIER LETTER VOICING
 02EE          ; Alphabetic # Lm       MODIFIER LETTER DOUBLE APOSTROPHE
 0345          ; Alphabetic # Mn       COMBINING GREEK YPOGEGRAMMENI
+0363..036F    ; Alphabetic # Mn  [13] COMBINING LATIN SMALL LETTER A..COMBINING LATIN SMALL LETTER X
 0370..0373    ; Alphabetic # L&   [4] GREEK CAPITAL LETTER HETA..GREEK SMALL LETTER ARCHAIC SAMPI
 0374          ; Alphabetic # Lm       GREEK NUMERAL SIGN
 0376..0377    ; Alphabetic # L&   [2] GREEK CAPITAL LETTER PAMPHYLIAN DIGAMMA..GREEK SMALL LETTER PAMPHYLIAN DIGAMMA
@@ -343,6 +345,7 @@ FFE9..FFEC    ; Math # Sm   [4] HALFWIDTH LEFTWARDS ARROW..HALFWIDTH DOWNWARDS A
 0860..086A    ; Alphabetic # Lo  [11] SYRIAC LETTER MALAYALAM NGA..SYRIAC LETTER MALAYALAM SSA
 0870..0887    ; Alphabetic # Lo  [24] ARABIC LETTER ALEF WITH ATTACHED FATHA..ARABIC BASELINE ROUND DOT
 0889..088E    ; Alphabetic # Lo   [6] ARABIC LETTER NOON WITH INVERTED SMALL V..ARABIC VERTICAL TAIL
+0897          ; Alphabetic # Mn       ARABIC PEPET
 08A0..08C8    ; Alphabetic # Lo  [41] ARABIC LETTER BEH WITH SMALL V BELOW..ARABIC LETTER GRAF
 08C9          ; Alphabetic # Lm       ARABIC SMALL FARSI YEH
 08D4..08DF    ; Alphabetic # Mn  [12] ARABIC SMALL HIGH WORD AR-RUB..ARABIC SMALL HIGH WORD WAQFA
@@ -710,7 +713,7 @@ FFE9..FFEC    ; Math # Sm   [4] HALFWIDTH LEFTWARDS ARROW..HALFWIDTH DOWNWARDS A
 1C4D..1C4F    ; Alphabetic # Lo   [3] LEPCHA LETTER TTA..LEPCHA LETTER DDA
 1C5A..1C77    ; Alphabetic # Lo  [30] OL CHIKI LETTER LA..OL CHIKI LETTER OH
 1C78..1C7D    ; Alphabetic # Lm   [6] OL CHIKI MU TTUDDAG..OL CHIKI AHAD
-1C80..1C88    ; Alphabetic # L&   [9] CYRILLIC SMALL LETTER ROUNDED VE..CYRILLIC SMALL LETTER UNBLENDED UK
+1C80..1C8A    ; Alphabetic # L&  [11] CYRILLIC SMALL LETTER ROUNDED VE..CYRILLIC SMALL LETTER TJE
 1C90..1CBA    ; Alphabetic # L&  [43] GEORGIAN MTAVRULI CAPITAL LETTER AN..GEORGIAN MTAVRULI CAPITAL LETTER AIN
 1CBD..1CBF    ; Alphabetic # L&   [3] GEORGIAN MTAVRULI CAPITAL LETTER AEN..GEORGIAN MTAVRULI CAPITAL LETTER LABIAL SIGN
 1CE9..1CEC    ; Alphabetic # Lo   [4] VEDIC SIGN ANUSVARA ANTARGOMUKHA..VEDIC SIGN ANUSVARA VAMAGOMUKHA WITH TAIL
@@ -723,7 +726,7 @@ FFE9..FFEC    ; Math # Sm   [4] HALFWIDTH LEFTWARDS ARROW..HALFWIDTH DOWNWARDS A
 1D78          ; Alphabetic # Lm       MODIFIER LETTER CYRILLIC EN
 1D79..1D9A    ; Alphabetic # L&  [34] LATIN SMALL LETTER INSULAR G..LATIN SMALL LETTER EZH WITH RETROFLEX HOOK
 1D9B..1DBF    ; Alphabetic # Lm  [37] MODIFIER LETTER SMALL TURNED ALPHA..MODIFIER LETTER SMALL THETA
-1DE7..1DF4    ; Alphabetic # Mn  [14] COMBINING LATIN SMALL LETTER ALPHA..COMBINING LATIN SMALL LETTER U WITH DIAERESIS
+1DD3..1DF4    ; Alphabetic # Mn  [34] COMBINING LATIN SMALL LETTER FLATTENED OPEN A ABOVE..COMBINING LATIN SMALL LETTER U WITH DIAERESIS
 1E00..1F15    ; Alphabetic # L& [278] LATIN CAPITAL LETTER A WITH RING BELOW..GREEK SMALL LETTER EPSILON WITH DASIA AND OXIA
 1F18..1F1D    ; Alphabetic # L&   [6] GREEK CAPITAL LETTER EPSILON WITH PSILI..GREEK CAPITAL LETTER EPSILON WITH DASIA AND OXIA
 1F20..1F45    ; Alphabetic # L&  [38] GREEK SMALL LETTER ETA WITH PSILI..GREEK SMALL LETTER OMICRON WITH DASIA AND OXIA
@@ -830,10 +833,10 @@ A771..A787    ; Alphabetic # L&  [23] LATIN SMALL LETTER DUM..LATIN SMALL LETTER
 A788          ; Alphabetic # Lm       MODIFIER LETTER LOW CIRCUMFLEX ACCENT
 A78B..A78E    ; Alphabetic # L&   [4] LATIN CAPITAL LETTER SALTILLO..LATIN SMALL LETTER L WITH RETROFLEX HOOK AND BELT
 A78F          ; Alphabetic # Lo       LATIN LETTER SINOLOGICAL DOT
-A790..A7CA    ; Alphabetic # L&  [59] LATIN CAPITAL LETTER N WITH DESCENDER..LATIN SMALL LETTER S WITH SHORT STROKE OVERLAY
+A790..A7CD    ; Alphabetic # L&  [62] LATIN CAPITAL LETTER N WITH DESCENDER..LATIN SMALL LETTER S WITH DIAGONAL STROKE
 A7D0..A7D1    ; Alphabetic # L&   [2] LATIN CAPITAL LETTER CLOSED INSULAR G..LATIN SMALL LETTER CLOSED INSULAR G
 A7D3          ; Alphabetic # L&       LATIN SMALL LETTER DOUBLE THORN
-A7D5..A7D9    ; Alphabetic # L&   [5] LATIN SMALL LETTER DOUBLE WYNN..LATIN SMALL LETTER SIGMOID S
+A7D5..A7DC    ; Alphabetic # L&   [8] LATIN SMALL LETTER DOUBLE WYNN..LATIN CAPITAL LETTER LAMBDA WITH STROKE
 A7F2..A7F4    ; Alphabetic # Lm   [3] MODIFIER LETTER CAPITAL C..MODIFIER LETTER CAPITAL Q
 A7F5..A7F6    ; Alphabetic # L&   [2] LATIN CAPITAL LETTER REVERSED HALF H..LATIN SMALL LETTER REVERSED HALF H
 A7F7          ; Alphabetic # Lo       LATIN EPIGRAPHIC LETTER SIDEWAYS I
@@ -998,6 +1001,7 @@ FFDA..FFDC    ; Alphabetic # Lo   [3] HALFWIDTH HANGUL LETTER EU..HALFWIDTH HANG
 105A3..105B1  ; Alphabetic # L&  [15] VITHKUQI SMALL LETTER HA..VITHKUQI SMALL LETTER RE
 105B3..105B9  ; Alphabetic # L&   [7] VITHKUQI SMALL LETTER SE..VITHKUQI SMALL LETTER XE
 105BB..105BC  ; Alphabetic # L&   [2] VITHKUQI SMALL LETTER Y..VITHKUQI SMALL LETTER ZE
+105C0..105F3  ; Alphabetic # Lo  [52] TODHRI LETTER A..TODHRI LETTER OO
 10600..10736  ; Alphabetic # Lo [311] LINEAR A SIGN AB001..LINEAR A SIGN A664
 10740..10755  ; Alphabetic # Lo  [22] LINEAR A SIGN A701 A..LINEAR A SIGN A732 JE
 10760..10767  ; Alphabetic # Lo   [8] LINEAR A SIGN A800..LINEAR A SIGN A807
@@ -1038,9 +1042,18 @@ FFDA..FFDC    ; Alphabetic # Lo   [3] HALFWIDTH HANGUL LETTER EU..HALFWIDTH HANG
 10CC0..10CF2  ; Alphabetic # L&  [51] OLD HUNGARIAN SMALL LETTER A..OLD HUNGARIAN SMALL LETTER US
 10D00..10D23  ; Alphabetic # Lo  [36] HANIFI ROHINGYA LETTER A..HANIFI ROHINGYA MARK NA KHONNA
 10D24..10D27  ; Alphabetic # Mn   [4] HANIFI ROHINGYA SIGN HARBAHAY..HANIFI ROHINGYA SIGN TASSI
+10D4A..10D4D  ; Alphabetic # Lo   [4] GARAY VOWEL SIGN A..GARAY VOWEL SIGN EE
+10D4E         ; Alphabetic # Lm       GARAY VOWEL LENGTH MARK
+10D4F         ; Alphabetic # Lo       GARAY SUKUN
+10D50..10D65  ; Alphabetic # L&  [22] GARAY CAPITAL LETTER A..GARAY CAPITAL LETTER OLD NA
+10D69         ; Alphabetic # Mn       GARAY VOWEL SIGN E
+10D6F         ; Alphabetic # Lm       GARAY REDUPLICATION MARK
+10D70..10D85  ; Alphabetic # L&  [22] GARAY SMALL LETTER A..GARAY SMALL LETTER OLD NA
 10E80..10EA9  ; Alphabetic # Lo  [42] YEZIDI LETTER ELIF..YEZIDI LETTER ET
 10EAB..10EAC  ; Alphabetic # Mn   [2] YEZIDI COMBINING HAMZA MARK..YEZIDI COMBINING MADDA MARK
 10EB0..10EB1  ; Alphabetic # Lo   [2] YEZIDI LETTER LAM WITH DOT ABOVE..YEZIDI LETTER YOT WITH CIRCUMFLEX ABOVE
+10EC2..10EC4  ; Alphabetic # Lo   [3] ARABIC LETTER DAL WITH TWO DOTS VERTICALLY BELOW..ARABIC LETTER KAF WITH TWO DOTS VERTICALLY BELOW
+10EFC         ; Alphabetic # Mn       ARABIC COMBINING ALEF OVERLAY
 10F00..10F1C  ; Alphabetic # Lo  [29] OLD SOGDIAN LETTER ALEPH..OLD SOGDIAN LETTER FINAL TAW WITH VERTICAL TAIL
 10F27         ; Alphabetic # Lo       OLD SOGDIAN LIGATURE AYIN-DALETH
 10F30..10F45  ; Alphabetic # Lo  [22] SOGDIAN LETTER ALEPH..SOGDIAN INDEPENDENT SHIN
@@ -1121,6 +1134,19 @@ FFDA..FFDC    ; Alphabetic # Lo   [3] HALFWIDTH HANGUL LETTER EU..HALFWIDTH HANG
 11357         ; Alphabetic # Mc       GRANTHA AU LENGTH MARK
 1135D..11361  ; Alphabetic # Lo   [5] GRANTHA SIGN PLUTA..GRANTHA LETTER VOCALIC LL
 11362..11363  ; Alphabetic # Mc   [2] GRANTHA VOWEL SIGN VOCALIC L..GRANTHA VOWEL SIGN VOCALIC LL
+11380..11389  ; Alphabetic # Lo  [10] TULU-TIGALARI LETTER A..TULU-TIGALARI LETTER VOCALIC LL
+1138B         ; Alphabetic # Lo       TULU-TIGALARI LETTER EE
+1138E         ; Alphabetic # Lo       TULU-TIGALARI LETTER AI
+11390..113B5  ; Alphabetic # Lo  [38] TULU-TIGALARI LETTER OO..TULU-TIGALARI LETTER LLLA
+113B7         ; Alphabetic # Lo       TULU-TIGALARI SIGN AVAGRAHA
+113B8..113BA  ; Alphabetic # Mc   [3] TULU-TIGALARI VOWEL SIGN AA..TULU-TIGALARI VOWEL SIGN II
+113BB..113C0  ; Alphabetic # Mn   [6] TULU-TIGALARI VOWEL SIGN U..TULU-TIGALARI VOWEL SIGN VOCALIC LL
+113C2         ; Alphabetic # Mc       TULU-TIGALARI VOWEL SIGN EE
+113C5         ; Alphabetic # Mc       TULU-TIGALARI VOWEL SIGN AI
+113C7..113CA  ; Alphabetic # Mc   [4] TULU-TIGALARI VOWEL SIGN OO..TULU-TIGALARI SIGN CANDRA ANUNASIKA
+113CC..113CD  ; Alphabetic # Mc   [2] TULU-TIGALARI SIGN ANUSVARA..TULU-TIGALARI SIGN VISARGA
+113D1         ; Alphabetic # Lo       TULU-TIGALARI REPHA
+113D3         ; Alphabetic # Lo       TULU-TIGALARI SIGN PLUTA
 11400..11434  ; Alphabetic # Lo  [53] NEWA LETTER A..NEWA LETTER HA
 11435..11437  ; Alphabetic # Mc   [3] NEWA VOWEL SIGN AA..NEWA VOWEL SIGN II
 11438..1143F  ; Alphabetic # Mn   [8] NEWA VOWEL SIGN U..NEWA VOWEL SIGN AI
@@ -1163,7 +1189,9 @@ FFDA..FFDC    ; Alphabetic # Lo   [3] HALFWIDTH HANGUL LETTER EU..HALFWIDTH HANG
 116B0..116B5  ; Alphabetic # Mn   [6] TAKRI VOWEL SIGN U..TAKRI VOWEL SIGN AU
 116B8         ; Alphabetic # Lo       TAKRI LETTER ARCHAIC KHA
 11700..1171A  ; Alphabetic # Lo  [27] AHOM LETTER KA..AHOM LETTER ALTERNATE BA
-1171D..1171F  ; Alphabetic # Mn   [3] AHOM CONSONANT SIGN MEDIAL LA..AHOM CONSONANT SIGN MEDIAL LIGATING RA
+1171D         ; Alphabetic # Mn       AHOM CONSONANT SIGN MEDIAL LA
+1171E         ; Alphabetic # Mc       AHOM CONSONANT SIGN MEDIAL RA
+1171F         ; Alphabetic # Mn       AHOM CONSONANT SIGN MEDIAL LIGATING RA
 11720..11721  ; Alphabetic # Mc   [2] AHOM VOWEL SIGN A..AHOM VOWEL SIGN AA
 11722..11725  ; Alphabetic # Mn   [4] AHOM VOWEL SIGN I..AHOM VOWEL SIGN UU
 11726         ; Alphabetic # Mc       AHOM VOWEL SIGN E
@@ -1211,6 +1239,7 @@ FFDA..FFDC    ; Alphabetic # Lo   [3] HALFWIDTH HANGUL LETTER EU..HALFWIDTH HANG
 11A97         ; Alphabetic # Mc       SOYOMBO SIGN VISARGA
 11A9D         ; Alphabetic # Lo       SOYOMBO MARK PLUTA
 11AB0..11AF8  ; Alphabetic # Lo  [73] CANADIAN SYLLABICS NATTILIK HI..PAU CIN HAU GLOTTAL STOP FINAL
+11BC0..11BE0  ; Alphabetic # Lo  [33] SUNUWAR LETTER DEVI..SUNUWAR LETTER KLOKO
 11C00..11C08  ; Alphabetic # Lo   [9] BHAIKSUKI LETTER A..BHAIKSUKI LETTER VOCALIC L
 11C0A..11C2E  ; Alphabetic # Lo  [37] BHAIKSUKI LETTER E..BHAIKSUKI LETTER HA
 11C2F         ; Alphabetic # Mc       BHAIKSUKI VOWEL SIGN AA
@@ -1264,7 +1293,12 @@ FFDA..FFDC    ; Alphabetic # Lo   [3] HALFWIDTH HANGUL LETTER EU..HALFWIDTH HANG
 12F90..12FF0  ; Alphabetic # Lo  [97] CYPRO-MINOAN SIGN CM001..CYPRO-MINOAN SIGN CM114
 13000..1342F  ; Alphabetic # Lo [1072] EGYPTIAN HIEROGLYPH A001..EGYPTIAN HIEROGLYPH V011D
 13441..13446  ; Alphabetic # Lo   [6] EGYPTIAN HIEROGLYPH FULL BLANK..EGYPTIAN HIEROGLYPH WIDE LOST SIGN
+13460..143FA  ; Alphabetic # Lo [3995] EGYPTIAN HIEROGLYPH-13460..EGYPTIAN HIEROGLYPH-143FA
 14400..14646  ; Alphabetic # Lo [583] ANATOLIAN HIEROGLYPH A001..ANATOLIAN HIEROGLYPH A530
+16100..1611D  ; Alphabetic # Lo  [30] GURUNG KHEMA LETTER A..GURUNG KHEMA LETTER SA
+1611E..16129  ; Alphabetic # Mn  [12] GURUNG KHEMA VOWEL SIGN AA..GURUNG KHEMA VOWEL LENGTH MARK
+1612A..1612C  ; Alphabetic # Mc   [3] GURUNG KHEMA CONSONANT SIGN MEDIAL YA..GURUNG KHEMA CONSONANT SIGN MEDIAL HA
+1612D..1612E  ; Alphabetic # Mn   [2] GURUNG KHEMA SIGN ANUSVARA..GURUNG KHEMA CONSONANT SIGN MEDIAL RA
 16800..16A38  ; Alphabetic # Lo [569] BAMUM LETTER PHASE-A NGKUE MFON..BAMUM LETTER PHASE-F VUEQ
 16A40..16A5E  ; Alphabetic # Lo  [31] MRO LETTER TA..MRO LETTER TEK
 16A70..16ABE  ; Alphabetic # Lo  [79] TANGSA LETTER OZ..TANGSA LETTER ZA
@@ -1273,6 +1307,9 @@ FFDA..FFDC    ; Alphabetic # Lo   [3] HALFWIDTH HANGUL LETTER EU..HALFWIDTH HANG
 16B40..16B43  ; Alphabetic # Lm   [4] PAHAWH HMONG SIGN VOS SEEV..PAHAWH HMONG SIGN IB YAM
 16B63..16B77  ; Alphabetic # Lo  [21] PAHAWH HMONG SIGN VOS LUB..PAHAWH HMONG SIGN CIM NRES TOS
 16B7D..16B8F  ; Alphabetic # Lo  [19] PAHAWH HMONG CLAN SIGN TSHEEJ..PAHAWH HMONG CLAN SIGN VWJ
+16D40..16D42  ; Alphabetic # Lm   [3] KIRAT RAI SIGN ANUSVARA..KIRAT RAI SIGN VISARGA
+16D43..16D6A  ; Alphabetic # Lo  [40] KIRAT RAI LETTER A..KIRAT RAI VOWEL SIGN AU
+16D6B..16D6C  ; Alphabetic # Lm   [2] KIRAT RAI SIGN VIRAMA..KIRAT RAI SIGN SAAT
 16E40..16E7F  ; Alphabetic # L&  [64] MEDEFAIDRIN CAPITAL LETTER M..MEDEFAIDRIN SMALL LETTER Y
 16F00..16F4A  ; Alphabetic # Lo  [75] MIAO LETTER PA..MIAO LETTER RTE
 16F4F         ; Alphabetic # Mn       MIAO SIGN CONSONANT MODIFIER BAR
@@ -1285,7 +1322,7 @@ FFDA..FFDC    ; Alphabetic # Lo   [3] HALFWIDTH HANGUL LETTER EU..HALFWIDTH HANG
 16FF0..16FF1  ; Alphabetic # Mc   [2] VIETNAMESE ALTERNATE READING MARK CA..VIETNAMESE ALTERNATE READING MARK NHAY
 17000..187F7  ; Alphabetic # Lo [6136] TANGUT IDEOGRAPH-17000..TANGUT IDEOGRAPH-187F7
 18800..18CD5  ; Alphabetic # Lo [1238] TANGUT COMPONENT-001..KHITAN SMALL SCRIPT CHARACTER-18CD5
-18D00..18D08  ; Alphabetic # Lo   [9] TANGUT IDEOGRAPH-18D00..TANGUT IDEOGRAPH-18D08
+18CFF..18D08  ; Alphabetic # Lo  [10] KHITAN SMALL SCRIPT CHARACTER-18CFF..TANGUT IDEOGRAPH-18D08
 1AFF0..1AFF3  ; Alphabetic # Lm   [4] KATAKANA LETTER MINNAN TONE-2..KATAKANA LETTER MINNAN TONE-5
 1AFF5..1AFFB  ; Alphabetic # Lm   [7] KATAKANA LETTER MINNAN TONE-7..KATAKANA LETTER MINNAN NASALIZED TONE-5
 1AFFD..1AFFE  ; Alphabetic # Lm   [2] KATAKANA LETTER MINNAN NASALIZED TONE-7..KATAKANA LETTER MINNAN NASALIZED TONE-8
@@ -1348,6 +1385,8 @@ FFDA..FFDC    ; Alphabetic # Lo   [3] HALFWIDTH HANGUL LETTER EU..HALFWIDTH HANG
 1E2C0..1E2EB  ; Alphabetic # Lo  [44] WANCHO LETTER AA..WANCHO LETTER YIH
 1E4D0..1E4EA  ; Alphabetic # Lo  [27] NAG MUNDARI LETTER O..NAG MUNDARI LETTER ELL
 1E4EB         ; Alphabetic # Lm       NAG MUNDARI SIGN OJOD
+1E5D0..1E5ED  ; Alphabetic # Lo  [30] OL ONAL LETTER O..OL ONAL LETTER EG
+1E5F0         ; Alphabetic # Lo       OL ONAL SIGN HODDOND
 1E7E0..1E7E6  ; Alphabetic # Lo   [7] ETHIOPIC SYLLABLE HHYA..ETHIOPIC SYLLABLE HHYO
 1E7E8..1E7EB  ; Alphabetic # Lo   [4] ETHIOPIC SYLLABLE GURAGE HHWA..ETHIOPIC SYLLABLE HHWE
 1E7ED..1E7EE  ; Alphabetic # Lo   [2] ETHIOPIC SYLLABLE GURAGE MWI..ETHIOPIC SYLLABLE GURAGE MWEE
@@ -1402,7 +1441,7 @@ FFDA..FFDC    ; Alphabetic # Lo   [3] HALFWIDTH HANGUL LETTER EU..HALFWIDTH HANG
 30000..3134A  ; Alphabetic # Lo [4939] CJK UNIFIED IDEOGRAPH-30000..CJK UNIFIED IDEOGRAPH-3134A
 31350..323AF  ; Alphabetic # Lo [4192] CJK UNIFIED IDEOGRAPH-31350..CJK UNIFIED IDEOGRAPH-323AF
 
-# Total code points: 138387
+# Total code points: 142759
 
 # ================================================
 
@@ -1691,6 +1730,7 @@ FFDA..FFDC    ; Alphabetic # Lo   [3] HALFWIDTH HANGUL LETTER EU..HALFWIDTH HANG
 10FD..10FF    ; Lowercase # L&   [3] GEORGIAN LETTER AEN..GEORGIAN LETTER LABIAL SIGN
 13F8..13FD    ; Lowercase # L&   [6] CHEROKEE SMALL LETTER YE..CHEROKEE SMALL LETTER MV
 1C80..1C88    ; Lowercase # L&   [9] CYRILLIC SMALL LETTER ROUNDED VE..CYRILLIC SMALL LETTER UNBLENDED UK
+1C8A          ; Lowercase # L&       CYRILLIC SMALL LETTER TJE
 1D00..1D2B    ; Lowercase # L&  [44] LATIN LETTER SMALL CAPITAL A..CYRILLIC LETTER SMALL CAPITAL EL
 1D2C..1D6A    ; Lowercase # Lm  [63] MODIFIER LETTER CAPITAL A..GREEK SUBSCRIPT SMALL LETTER CHI
 1D6B..1D77    ; Lowercase # L&  [13] LATIN SMALL LETTER UE..LATIN SMALL LETTER TURNED G
@@ -2032,11 +2072,13 @@ A7C1          ; Lowercase # L&       LATIN SMALL LETTER OLD POLISH O
 A7C3          ; Lowercase # L&       LATIN SMALL LETTER ANGLICANA W
 A7C8          ; Lowercase # L&       LATIN SMALL LETTER D WITH SHORT STROKE OVERLAY
 A7CA          ; Lowercase # L&       LATIN SMALL LETTER S WITH SHORT STROKE OVERLAY
+A7CD          ; Lowercase # L&       LATIN SMALL LETTER S WITH DIAGONAL STROKE
 A7D1          ; Lowercase # L&       LATIN SMALL LETTER CLOSED INSULAR G
 A7D3          ; Lowercase # L&       LATIN SMALL LETTER DOUBLE THORN
 A7D5          ; Lowercase # L&       LATIN SMALL LETTER DOUBLE WYNN
 A7D7          ; Lowercase # L&       LATIN SMALL LETTER MIDDLE SCOTS S
 A7D9          ; Lowercase # L&       LATIN SMALL LETTER SIGMOID S
+A7DB          ; Lowercase # L&       LATIN SMALL LETTER LAMBDA
 A7F2..A7F4    ; Lowercase # Lm   [3] MODIFIER LETTER CAPITAL C..MODIFIER LETTER CAPITAL Q
 A7F6          ; Lowercase # L&       LATIN SMALL LETTER REVERSED HALF H
 A7F8..A7F9    ; Lowercase # Lm   [2] MODIFIER LETTER CAPITAL H WITH STROKE..MODIFIER LETTER SMALL LIGATURE OE
@@ -2060,6 +2102,7 @@ FF41..FF5A    ; Lowercase # L&  [26] FULLWIDTH LATIN SMALL LETTER A..FULLWIDTH L
 10787..107B0  ; Lowercase # Lm  [42] MODIFIER LETTER SMALL DZ DIGRAPH..MODIFIER LETTER SMALL V WITH RIGHT HOOK
 107B2..107BA  ; Lowercase # Lm   [9] MODIFIER LETTER SMALL CAPITAL Y..MODIFIER LETTER SMALL S WITH CURL
 10CC0..10CF2  ; Lowercase # L&  [51] OLD HUNGARIAN SMALL LETTER A..OLD HUNGARIAN SMALL LETTER US
+10D70..10D85  ; Lowercase # L&  [22] GARAY SMALL LETTER A..GARAY SMALL LETTER OLD NA
 118C0..118DF  ; Lowercase # L&  [32] WARANG CITI SMALL LETTER NGAA..WARANG CITI SMALL LETTER VIYO
 16E60..16E7F  ; Lowercase # L&  [32] MEDEFAIDRIN SMALL LETTER M..MEDEFAIDRIN SMALL LETTER Y
 1D41A..1D433  ; Lowercase # L&  [26] MATHEMATICAL BOLD SMALL A..MATHEMATICAL BOLD SMALL Z
@@ -2096,7 +2139,7 @@ FF41..FF5A    ; Lowercase # L&  [26] FULLWIDTH LATIN SMALL LETTER A..FULLWIDTH L
 1E030..1E06D  ; Lowercase # Lm  [62] MODIFIER LETTER CYRILLIC SMALL A..MODIFIER LETTER CYRILLIC SMALL STRAIGHT U WITH STROKE
 1E922..1E943  ; Lowercase # L&  [34] ADLAM SMALL LETTER ALIF..ADLAM SMALL LETTER SHA
 
-# Total code points: 2544
+# Total code points: 2569
 
 # ================================================
 
@@ -2379,6 +2422,7 @@ FF41..FF5A    ; Lowercase # L&  [26] FULLWIDTH LATIN SMALL LETTER A..FULLWIDTH L
 10C7          ; Uppercase # L&       GEORGIAN CAPITAL LETTER YN
 10CD          ; Uppercase # L&       GEORGIAN CAPITAL LETTER AEN
 13A0..13F5    ; Uppercase # L&  [86] CHEROKEE LETTER A..CHEROKEE LETTER MV
+1C89          ; Uppercase # L&       CYRILLIC CAPITAL LETTER TJE
 1C90..1CBA    ; Uppercase # L&  [43] GEORGIAN MTAVRULI CAPITAL LETTER AN..GEORGIAN MTAVRULI CAPITAL LETTER AIN
 1CBD..1CBF    ; Uppercase # L&   [3] GEORGIAN MTAVRULI CAPITAL LETTER AEN..GEORGIAN MTAVRULI CAPITAL LETTER LABIAL SIGN
 1E00          ; Uppercase # L&       LATIN CAPITAL LETTER A WITH RING BELOW
@@ -2705,9 +2749,12 @@ A7C0          ; Uppercase # L&       LATIN CAPITAL LETTER OLD POLISH O
 A7C2          ; Uppercase # L&       LATIN CAPITAL LETTER ANGLICANA W
 A7C4..A7C7    ; Uppercase # L&   [4] LATIN CAPITAL LETTER C WITH PALATAL HOOK..LATIN CAPITAL LETTER D WITH SHORT STROKE OVERLAY
 A7C9          ; Uppercase # L&       LATIN CAPITAL LETTER S WITH SHORT STROKE OVERLAY
+A7CB..A7CC    ; Uppercase # L&   [2] LATIN CAPITAL LETTER RAMS HORN..LATIN CAPITAL LETTER S WITH DIAGONAL STROKE
 A7D0          ; Uppercase # L&       LATIN CAPITAL LETTER CLOSED INSULAR G
 A7D6          ; Uppercase # L&       LATIN CAPITAL LETTER MIDDLE SCOTS S
 A7D8          ; Uppercase # L&       LATIN CAPITAL LETTER SIGMOID S
+A7DA          ; Uppercase # L&       LATIN CAPITAL LETTER LAMBDA
+A7DC          ; Uppercase # L&       LATIN CAPITAL LETTER LAMBDA WITH STROKE
 A7F5          ; Uppercase # L&       LATIN CAPITAL LETTER REVERSED HALF H
 FF21..FF3A    ; Uppercase # L&  [26] FULLWIDTH LATIN CAPITAL LETTER A..FULLWIDTH LATIN CAPITAL LETTER Z
 10400..10427  ; Uppercase # L&  [40] DESERET CAPITAL LETTER LONG I..DESERET CAPITAL LETTER EW
@@ -2717,6 +2764,7 @@ FF21..FF3A    ; Uppercase # L&  [26] FULLWIDTH LATIN CAPITAL LETTER A..FULLWIDTH
 1058C..10592  ; Uppercase # L&   [7] VITHKUQI CAPITAL LETTER SE..VITHKUQI CAPITAL LETTER XE
 10594..10595  ; Uppercase # L&   [2] VITHKUQI CAPITAL LETTER Y..VITHKUQI CAPITAL LETTER ZE
 10C80..10CB2  ; Uppercase # L&  [51] OLD HUNGARIAN CAPITAL LETTER A..OLD HUNGARIAN CAPITAL LETTER US
+10D50..10D65  ; Uppercase # L&  [22] GARAY CAPITAL LETTER A..GARAY CAPITAL LETTER OLD NA
 118A0..118BF  ; Uppercase # L&  [32] WARANG CITI CAPITAL LETTER NGAA..WARANG CITI CAPITAL LETTER VIYO
 16E40..16E5F  ; Uppercase # L&  [32] MEDEFAIDRIN CAPITAL LETTER M..MEDEFAIDRIN CAPITAL LETTER Y
 1D400..1D419  ; Uppercase # L&  [26] MATHEMATICAL BOLD CAPITAL A..MATHEMATICAL BOLD CAPITAL Z
@@ -2755,7 +2803,7 @@ FF21..FF3A    ; Uppercase # L&  [26] FULLWIDTH LATIN CAPITAL LETTER A..FULLWIDTH
 1F150..1F169  ; Uppercase # So  [26] NEGATIVE CIRCLED LATIN CAPITAL LETTER A..NEGATIVE CIRCLED LATIN CAPITAL LETTER Z
 1F170..1F189  ; Uppercase # So  [26] NEGATIVE SQUARED LATIN CAPITAL LETTER A..NEGATIVE SQUARED LATIN CAPITAL LETTER Z
 
-# Total code points: 1951
+# Total code points: 1978
 
 # ================================================
 
@@ -2800,7 +2848,7 @@ FF21..FF3A    ; Uppercase # L&  [26] FULLWIDTH LATIN CAPITAL LETTER A..FULLWIDTH
 10FD..10FF    ; Cased # L&   [3] GEORGIAN LETTER AEN..GEORGIAN LETTER LABIAL SIGN
 13A0..13F5    ; Cased # L&  [86] CHEROKEE LETTER A..CHEROKEE LETTER MV
 13F8..13FD    ; Cased # L&   [6] CHEROKEE SMALL LETTER YE..CHEROKEE SMALL LETTER MV
-1C80..1C88    ; Cased # L&   [9] CYRILLIC SMALL LETTER ROUNDED VE..CYRILLIC SMALL LETTER UNBLENDED UK
+1C80..1C8A    ; Cased # L&  [11] CYRILLIC SMALL LETTER ROUNDED VE..CYRILLIC SMALL LETTER TJE
 1C90..1CBA    ; Cased # L&  [43] GEORGIAN MTAVRULI CAPITAL LETTER AN..GEORGIAN MTAVRULI CAPITAL LETTER AIN
 1CBD..1CBF    ; Cased # L&   [3] GEORGIAN MTAVRULI CAPITAL LETTER AEN..GEORGIAN MTAVRULI CAPITAL LETTER LABIAL SIGN
 1D00..1D2B    ; Cased # L&  [44] LATIN LETTER SMALL CAPITAL A..CYRILLIC LETTER SMALL CAPITAL EL
@@ -2863,10 +2911,10 @@ A722..A76F    ; Cased # L&  [78] LATIN CAPITAL LETTER EGYPTOLOGICAL ALEF..LATIN
 A770          ; Cased # Lm       MODIFIER LETTER US
 A771..A787    ; Cased # L&  [23] LATIN SMALL LETTER DUM..LATIN SMALL LETTER INSULAR T
 A78B..A78E    ; Cased # L&   [4] LATIN CAPITAL LETTER SALTILLO..LATIN SMALL LETTER L WITH RETROFLEX HOOK AND BELT
-A790..A7CA    ; Cased # L&  [59] LATIN CAPITAL LETTER N WITH DESCENDER..LATIN SMALL LETTER S WITH SHORT STROKE OVERLAY
+A790..A7CD    ; Cased # L&  [62] LATIN CAPITAL LETTER N WITH DESCENDER..LATIN SMALL LETTER S WITH DIAGONAL STROKE
 A7D0..A7D1    ; Cased # L&   [2] LATIN CAPITAL LETTER CLOSED INSULAR G..LATIN SMALL LETTER CLOSED INSULAR G
 A7D3          ; Cased # L&       LATIN SMALL LETTER DOUBLE THORN
-A7D5..A7D9    ; Cased # L&   [5] LATIN SMALL LETTER DOUBLE WYNN..LATIN SMALL LETTER SIGMOID S
+A7D5..A7DC    ; Cased # L&   [8] LATIN SMALL LETTER DOUBLE WYNN..LATIN CAPITAL LETTER LAMBDA WITH STROKE
 A7F2..A7F4    ; Cased # Lm   [3] MODIFIER LETTER CAPITAL C..MODIFIER LETTER CAPITAL Q
 A7F5..A7F6    ; Cased # L&   [2] LATIN CAPITAL LETTER REVERSED HALF H..LATIN SMALL LETTER REVERSED HALF H
 A7F8..A7F9    ; Cased # Lm   [2] MODIFIER LETTER CAPITAL H WITH STROKE..MODIFIER LETTER SMALL LIGATURE OE
@@ -2897,6 +2945,8 @@ FF41..FF5A    ; Cased # L&  [26] FULLWIDTH LATIN SMALL LETTER A..FULLWIDTH LATIN
 107B2..107BA  ; Cased # Lm   [9] MODIFIER LETTER SMALL CAPITAL Y..MODIFIER LETTER SMALL S WITH CURL
 10C80..10CB2  ; Cased # L&  [51] OLD HUNGARIAN CAPITAL LETTER A..OLD HUNGARIAN CAPITAL LETTER US
 10CC0..10CF2  ; Cased # L&  [51] OLD HUNGARIAN SMALL LETTER A..OLD HUNGARIAN SMALL LETTER US
+10D50..10D65  ; Cased # L&  [22] GARAY CAPITAL LETTER A..GARAY CAPITAL LETTER OLD NA
+10D70..10D85  ; Cased # L&  [22] GARAY SMALL LETTER A..GARAY SMALL LETTER OLD NA
 118A0..118DF  ; Cased # L&  [64] WARANG CITI CAPITAL LETTER NGAA..WARANG CITI SMALL LETTER VIYO
 16E40..16E7F  ; Cased # L&  [64] MEDEFAIDRIN CAPITAL LETTER M..MEDEFAIDRIN SMALL LETTER Y
 1D400..1D454  ; Cased # L&  [85] MATHEMATICAL BOLD CAPITAL A..MATHEMATICAL ITALIC SMALL G
@@ -2938,7 +2988,7 @@ FF41..FF5A    ; Cased # L&  [26] FULLWIDTH LATIN SMALL LETTER A..FULLWIDTH LATIN
 1F150..1F169  ; Cased # So  [26] NEGATIVE CIRCLED LATIN CAPITAL LETTER A..NEGATIVE CIRCLED LATIN CAPITAL LETTER Z
 1F170..1F189  ; Cased # So  [26] NEGATIVE SQUARED LATIN CAPITAL LETTER A..NEGATIVE SQUARED LATIN CAPITAL LETTER Z
 
-# Total code points: 4526
+# Total code points: 4578
 
 # ================================================
 
@@ -3015,7 +3065,7 @@ FF41..FF5A    ; Cased # L&  [26] FULLWIDTH LATIN SMALL LETTER A..FULLWIDTH LATIN
 0859..085B    ; Case_Ignorable # Mn   [3] MANDAIC AFFRICATION MARK..MANDAIC GEMINATION MARK
 0888          ; Case_Ignorable # Sk       ARABIC RAISED ROUND DOT
 0890..0891    ; Case_Ignorable # Cf   [2] ARABIC POUND MARK ABOVE..ARABIC PIASTRE MARK ABOVE
-0898..089F    ; Case_Ignorable # Mn   [8] ARABIC SMALL HIGH WORD AL-JUZ..ARABIC HALF MADDA OVER MADDA
+0897..089F    ; Case_Ignorable # Mn   [9] ARABIC PEPET..ARABIC HALF MADDA OVER MADDA
 08C9          ; Case_Ignorable # Lm       ARABIC SMALL FARSI YEH
 08CA..08E1    ; Case_Ignorable # Mn  [24] ARABIC SMALL HIGH FARSI YEH..ARABIC SMALL HIGH SIGN SAFHA
 08E2          ; Case_Ignorable # Cf       ARABIC DISPUTED END OF AYAH
@@ -3296,8 +3346,11 @@ FFF9..FFFB    ; Case_Ignorable # Cf   [3] INTERLINEAR ANNOTATION ANCHOR..INTERLI
 10A3F         ; Case_Ignorable # Mn       KHAROSHTHI VIRAMA
 10AE5..10AE6  ; Case_Ignorable # Mn   [2] MANICHAEAN ABBREVIATION MARK ABOVE..MANICHAEAN ABBREVIATION MARK BELOW
 10D24..10D27  ; Case_Ignorable # Mn   [4] HANIFI ROHINGYA SIGN HARBAHAY..HANIFI ROHINGYA SIGN TASSI
+10D4E         ; Case_Ignorable # Lm       GARAY VOWEL LENGTH MARK
+10D69..10D6D  ; Case_Ignorable # Mn   [5] GARAY VOWEL SIGN E..GARAY CONSONANT NASALIZATION MARK
+10D6F         ; Case_Ignorable # Lm       GARAY REDUPLICATION MARK
 10EAB..10EAC  ; Case_Ignorable # Mn   [2] YEZIDI COMBINING HAMZA MARK..YEZIDI COMBINING MADDA MARK
-10EFD..10EFF  ; Case_Ignorable # Mn   [3] ARABIC SMALL LOW WORD SAKTA..ARABIC SMALL LOW WORD MADDA
+10EFC..10EFF  ; Case_Ignorable # Mn   [4] ARABIC COMBINING ALEF OVERLAY..ARABIC SMALL LOW WORD MADDA
 10F46..10F50  ; Case_Ignorable # Mn  [11] SOGDIAN COMBINING DOT BELOW..SOGDIAN COMBINING STROKE BELOW
 10F82..10F85  ; Case_Ignorable # Mn   [4] OLD UYGHUR COMBINING DOT ABOVE..OLD UYGHUR COMBINING TWO DOTS BELOW
 11001         ; Case_Ignorable # Mn       BRAHMI SIGN ANUSVARA
@@ -3330,6 +3383,11 @@ FFF9..FFFB    ; Case_Ignorable # Cf   [3] INTERLINEAR ANNOTATION ANCHOR..INTERLI
 11340         ; Case_Ignorable # Mn       GRANTHA VOWEL SIGN II
 11366..1136C  ; Case_Ignorable # Mn   [7] COMBINING GRANTHA DIGIT ZERO..COMBINING GRANTHA DIGIT SIX
 11370..11374  ; Case_Ignorable # Mn   [5] COMBINING GRANTHA LETTER A..COMBINING GRANTHA LETTER PA
+113BB..113C0  ; Case_Ignorable # Mn   [6] TULU-TIGALARI VOWEL SIGN U..TULU-TIGALARI VOWEL SIGN VOCALIC LL
+113CE         ; Case_Ignorable # Mn       TULU-TIGALARI SIGN VIRAMA
+113D0         ; Case_Ignorable # Mn       TULU-TIGALARI CONJOINER
+113D2         ; Case_Ignorable # Mn       TULU-TIGALARI GEMINATION MARK
+113E1..113E2  ; Case_Ignorable # Mn   [2] TULU-TIGALARI VEDIC TONE SVARITA..TULU-TIGALARI VEDIC TONE ANUDATTA
 11438..1143F  ; Case_Ignorable # Mn   [8] NEWA VOWEL SIGN U..NEWA VOWEL SIGN AI
 11442..11444  ; Case_Ignorable # Mn   [3] NEWA SIGN VIRAMA..NEWA SIGN ANUSVARA
 11446         ; Case_Ignorable # Mn       NEWA SIGN NUKTA
@@ -3349,7 +3407,8 @@ FFF9..FFFB    ; Case_Ignorable # Cf   [3] INTERLINEAR ANNOTATION ANCHOR..INTERLI
 116AD         ; Case_Ignorable # Mn       TAKRI VOWEL SIGN AA
 116B0..116B5  ; Case_Ignorable # Mn   [6] TAKRI VOWEL SIGN U..TAKRI VOWEL SIGN AU
 116B7         ; Case_Ignorable # Mn       TAKRI SIGN NUKTA
-1171D..1171F  ; Case_Ignorable # Mn   [3] AHOM CONSONANT SIGN MEDIAL LA..AHOM CONSONANT SIGN MEDIAL LIGATING RA
+1171D         ; Case_Ignorable # Mn       AHOM CONSONANT SIGN MEDIAL LA
+1171F         ; Case_Ignorable # Mn       AHOM CONSONANT SIGN MEDIAL LIGATING RA
 11722..11725  ; Case_Ignorable # Mn   [4] AHOM VOWEL SIGN I..AHOM VOWEL SIGN UU
 11727..1172B  ; Case_Ignorable # Mn   [5] AHOM VOWEL SIGN AW..AHOM SIGN KILLER
 1182F..11837  ; Case_Ignorable # Mn   [9] DOGRA VOWEL SIGN U..DOGRA SIGN ANUSVARA
@@ -3388,12 +3447,17 @@ FFF9..FFFB    ; Case_Ignorable # Cf   [3] INTERLINEAR ANNOTATION ANCHOR..INTERLI
 11F36..11F3A  ; Case_Ignorable # Mn   [5] KAWI VOWEL SIGN I..KAWI VOWEL SIGN VOCALIC R
 11F40         ; Case_Ignorable # Mn       KAWI VOWEL SIGN EU
 11F42         ; Case_Ignorable # Mn       KAWI CONJOINER
+11F5A         ; Case_Ignorable # Mn       KAWI SIGN NUKTA
 13430..1343F  ; Case_Ignorable # Cf  [16] EGYPTIAN HIEROGLYPH VERTICAL JOINER..EGYPTIAN HIEROGLYPH END WALLED ENCLOSURE
 13440         ; Case_Ignorable # Mn       EGYPTIAN HIEROGLYPH MIRROR HORIZONTALLY
 13447..13455  ; Case_Ignorable # Mn  [15] EGYPTIAN HIEROGLYPH MODIFIER DAMAGED AT TOP START..EGYPTIAN HIEROGLYPH MODIFIER DAMAGED
+1611E..16129  ; Case_Ignorable # Mn  [12] GURUNG KHEMA VOWEL SIGN AA..GURUNG KHEMA VOWEL LENGTH MARK
+1612D..1612F  ; Case_Ignorable # Mn   [3] GURUNG KHEMA SIGN ANUSVARA..GURUNG KHEMA SIGN THOLHOMA
 16AF0..16AF4  ; Case_Ignorable # Mn   [5] BASSA VAH COMBINING HIGH TONE..BASSA VAH COMBINING HIGH-LOW TONE
 16B30..16B36  ; Case_Ignorable # Mn   [7] PAHAWH HMONG MARK CIM TUB..PAHAWH HMONG MARK CIM TAUM
 16B40..16B43  ; Case_Ignorable # Lm   [4] PAHAWH HMONG SIGN VOS SEEV..PAHAWH HMONG SIGN IB YAM
+16D40..16D42  ; Case_Ignorable # Lm   [3] KIRAT RAI SIGN ANUSVARA..KIRAT RAI SIGN VISARGA
+16D6B..16D6C  ; Case_Ignorable # Lm   [2] KIRAT RAI SIGN VIRAMA..KIRAT RAI SIGN SAAT
 16F4F         ; Case_Ignorable # Mn       MIAO SIGN CONSONANT MODIFIER BAR
 16F8F..16F92  ; Case_Ignorable # Mn   [4] MIAO TONE RIGHT..MIAO TONE BELOW
 16F93..16F9F  ; Case_Ignorable # Lm  [13] MIAO LETTER TONE-2..MIAO LETTER REFORMED TONE-8
@@ -3432,6 +3496,7 @@ FFF9..FFFB    ; Case_Ignorable # Cf   [3] INTERLINEAR ANNOTATION ANCHOR..INTERLI
 1E2EC..1E2EF  ; Case_Ignorable # Mn   [4] WANCHO TONE TUP..WANCHO TONE KOINI
 1E4EB         ; Case_Ignorable # Lm       NAG MUNDARI SIGN OJOD
 1E4EC..1E4EF  ; Case_Ignorable # Mn   [4] NAG MUNDARI SIGN MUHOR..NAG MUNDARI SIGN SUTUH
+1E5EE..1E5EF  ; Case_Ignorable # Mn   [2] OL ONAL SIGN MU..OL ONAL SIGN IKIR
 1E8D0..1E8D6  ; Case_Ignorable # Mn   [7] MENDE KIKAKUI COMBINING NUMBER TEENS..MENDE KIKAKUI COMBINING NUMBER MILLIONS
 1E944..1E94A  ; Case_Ignorable # Mn   [7] ADLAM ALIF LENGTHENER..ADLAM NUKTA
 1E94B         ; Case_Ignorable # Lm       ADLAM NASALIZATION MARK
@@ -3440,7 +3505,7 @@ E0001         ; Case_Ignorable # Cf       LANGUAGE TAG
 E0020..E007F  ; Case_Ignorable # Cf  [96] TAG SPACE..CANCEL TAG
 E0100..E01EF  ; Case_Ignorable # Mn [240] VARIATION SELECTOR-17..VARIATION SELECTOR-256
 
-# Total code points: 2707
+# Total code points: 2749
 
 # ================================================
 
@@ -3724,6 +3789,7 @@ E0100..E01EF  ; Case_Ignorable # Mn [240] VARIATION SELECTOR-17..VARIATION SELEC
 10C7          ; Changes_When_Lowercased # L&       GEORGIAN CAPITAL LETTER YN
 10CD          ; Changes_When_Lowercased # L&       GEORGIAN CAPITAL LETTER AEN
 13A0..13F5    ; Changes_When_Lowercased # L&  [86] CHEROKEE LETTER A..CHEROKEE LETTER MV
+1C89          ; Changes_When_Lowercased # L&       CYRILLIC CAPITAL LETTER TJE
 1C90..1CBA    ; Changes_When_Lowercased # L&  [43] GEORGIAN MTAVRULI CAPITAL LETTER AN..GEORGIAN MTAVRULI CAPITAL LETTER AIN
 1CBD..1CBF    ; Changes_When_Lowercased # L&   [3] GEORGIAN MTAVRULI CAPITAL LETTER AEN..GEORGIAN MTAVRULI CAPITAL LETTER LABIAL SIGN
 1E00          ; Changes_When_Lowercased # L&       LATIN CAPITAL LETTER A WITH RING BELOW
@@ -4043,9 +4109,12 @@ A7C0          ; Changes_When_Lowercased # L&       LATIN CAPITAL LETTER OLD POLI
 A7C2          ; Changes_When_Lowercased # L&       LATIN CAPITAL LETTER ANGLICANA W
 A7C4..A7C7    ; Changes_When_Lowercased # L&   [4] LATIN CAPITAL LETTER C WITH PALATAL HOOK..LATIN CAPITAL LETTER D WITH SHORT STROKE OVERLAY
 A7C9          ; Changes_When_Lowercased # L&       LATIN CAPITAL LETTER S WITH SHORT STROKE OVERLAY
+A7CB..A7CC    ; Changes_When_Lowercased # L&   [2] LATIN CAPITAL LETTER RAMS HORN..LATIN CAPITAL LETTER S WITH DIAGONAL STROKE
 A7D0          ; Changes_When_Lowercased # L&       LATIN CAPITAL LETTER CLOSED INSULAR G
 A7D6          ; Changes_When_Lowercased # L&       LATIN CAPITAL LETTER MIDDLE SCOTS S
 A7D8          ; Changes_When_Lowercased # L&       LATIN CAPITAL LETTER SIGMOID S
+A7DA          ; Changes_When_Lowercased # L&       LATIN CAPITAL LETTER LAMBDA
+A7DC          ; Changes_When_Lowercased # L&       LATIN CAPITAL LETTER LAMBDA WITH STROKE
 A7F5          ; Changes_When_Lowercased # L&       LATIN CAPITAL LETTER REVERSED HALF H
 FF21..FF3A    ; Changes_When_Lowercased # L&  [26] FULLWIDTH LATIN CAPITAL LETTER A..FULLWIDTH LATIN CAPITAL LETTER Z
 10400..10427  ; Changes_When_Lowercased # L&  [40] DESERET CAPITAL LETTER LONG I..DESERET CAPITAL LETTER EW
@@ -4055,11 +4124,12 @@ FF21..FF3A    ; Changes_When_Lowercased # L&  [26] FULLWIDTH LATIN CAPITAL LETTE
 1058C..10592  ; Changes_When_Lowercased # L&   [7] VITHKUQI CAPITAL LETTER SE..VITHKUQI CAPITAL LETTER XE
 10594..10595  ; Changes_When_Lowercased # L&   [2] VITHKUQI CAPITAL LETTER Y..VITHKUQI CAPITAL LETTER ZE
 10C80..10CB2  ; Changes_When_Lowercased # L&  [51] OLD HUNGARIAN CAPITAL LETTER A..OLD HUNGARIAN CAPITAL LETTER US
+10D50..10D65  ; Changes_When_Lowercased # L&  [22] GARAY CAPITAL LETTER A..GARAY CAPITAL LETTER OLD NA
 118A0..118BF  ; Changes_When_Lowercased # L&  [32] WARANG CITI CAPITAL LETTER NGAA..WARANG CITI CAPITAL LETTER VIYO
 16E40..16E5F  ; Changes_When_Lowercased # L&  [32] MEDEFAIDRIN CAPITAL LETTER M..MEDEFAIDRIN CAPITAL LETTER Y
 1E900..1E921  ; Changes_When_Lowercased # L&  [34] ADLAM CAPITAL LETTER ALIF..ADLAM CAPITAL LETTER SHA
 
-# Total code points: 1433
+# Total code points: 1460
 
 # ================================================
 
@@ -4140,7 +4210,7 @@ FF21..FF3A    ; Changes_When_Lowercased # L&  [26] FULLWIDTH LATIN CAPITAL LETTE
 018C          ; Changes_When_Uppercased # L&       LATIN SMALL LETTER D WITH TOPBAR
 0192          ; Changes_When_Uppercased # L&       LATIN SMALL LETTER F WITH HOOK
 0195          ; Changes_When_Uppercased # L&       LATIN SMALL LETTER HV
-0199..019A    ; Changes_When_Uppercased # L&   [2] LATIN SMALL LETTER K WITH HOOK..LATIN SMALL LETTER L WITH BAR
+0199..019B    ; Changes_When_Uppercased # L&   [3] LATIN SMALL LETTER K WITH HOOK..LATIN SMALL LETTER LAMBDA WITH STROKE
 019E          ; Changes_When_Uppercased # L&       LATIN SMALL LETTER N WITH LONG RIGHT LEG
 01A1          ; Changes_When_Uppercased # L&       LATIN SMALL LETTER O WITH HORN
 01A3          ; Changes_When_Uppercased # L&       LATIN SMALL LETTER OI
@@ -4216,8 +4286,7 @@ FF21..FF3A    ; Changes_When_Lowercased # L&  [26] FULLWIDTH LATIN CAPITAL LETTE
 0259          ; Changes_When_Uppercased # L&       LATIN SMALL LETTER SCHWA
 025B..025C    ; Changes_When_Uppercased # L&   [2] LATIN SMALL LETTER OPEN E..LATIN SMALL LETTER REVERSED OPEN E
 0260..0261    ; Changes_When_Uppercased # L&   [2] LATIN SMALL LETTER G WITH HOOK..LATIN SMALL LETTER SCRIPT G
-0263          ; Changes_When_Uppercased # L&       LATIN SMALL LETTER GAMMA
-0265..0266    ; Changes_When_Uppercased # L&   [2] LATIN SMALL LETTER TURNED H..LATIN SMALL LETTER H WITH HOOK
+0263..0266    ; Changes_When_Uppercased # L&   [4] LATIN SMALL LETTER GAMMA..LATIN SMALL LETTER H WITH HOOK
 0268..026C    ; Changes_When_Uppercased # L&   [5] LATIN SMALL LETTER I WITH STROKE..LATIN SMALL LETTER L WITH BELT
 026F          ; Changes_When_Uppercased # L&       LATIN SMALL LETTER TURNED M
 0271..0272    ; Changes_When_Uppercased # L&   [2] LATIN SMALL LETTER M WITH HOOK..LATIN SMALL LETTER N WITH LEFT HOOK
@@ -4357,6 +4426,7 @@ FF21..FF3A    ; Changes_When_Lowercased # L&  [26] FULLWIDTH LATIN CAPITAL LETTE
 10FD..10FF    ; Changes_When_Uppercased # L&   [3] GEORGIAN LETTER AEN..GEORGIAN LETTER LABIAL SIGN
 13F8..13FD    ; Changes_When_Uppercased # L&   [6] CHEROKEE SMALL LETTER YE..CHEROKEE SMALL LETTER MV
 1C80..1C88    ; Changes_When_Uppercased # L&   [9] CYRILLIC SMALL LETTER ROUNDED VE..CYRILLIC SMALL LETTER UNBLENDED UK
+1C8A          ; Changes_When_Uppercased # L&       CYRILLIC SMALL LETTER TJE
 1D79          ; Changes_When_Uppercased # L&       LATIN SMALL LETTER INSULAR G
 1D7D          ; Changes_When_Uppercased # L&       LATIN SMALL LETTER P WITH STROKE
 1D8E          ; Changes_When_Uppercased # L&       LATIN SMALL LETTER Z WITH PALATAL HOOK
@@ -4676,9 +4746,11 @@ A7C1          ; Changes_When_Uppercased # L&       LATIN SMALL LETTER OLD POLISH
 A7C3          ; Changes_When_Uppercased # L&       LATIN SMALL LETTER ANGLICANA W
 A7C8          ; Changes_When_Uppercased # L&       LATIN SMALL LETTER D WITH SHORT STROKE OVERLAY
 A7CA          ; Changes_When_Uppercased # L&       LATIN SMALL LETTER S WITH SHORT STROKE OVERLAY
+A7CD          ; Changes_When_Uppercased # L&       LATIN SMALL LETTER S WITH DIAGONAL STROKE
 A7D1          ; Changes_When_Uppercased # L&       LATIN SMALL LETTER CLOSED INSULAR G
 A7D7          ; Changes_When_Uppercased # L&       LATIN SMALL LETTER MIDDLE SCOTS S
 A7D9          ; Changes_When_Uppercased # L&       LATIN SMALL LETTER SIGMOID S
+A7DB          ; Changes_When_Uppercased # L&       LATIN SMALL LETTER LAMBDA
 A7F6          ; Changes_When_Uppercased # L&       LATIN SMALL LETTER REVERSED HALF H
 AB53          ; Changes_When_Uppercased # L&       LATIN SMALL LETTER CHI
 AB70..ABBF    ; Changes_When_Uppercased # L&  [80] CHEROKEE SMALL LETTER A..CHEROKEE SMALL LETTER YA
@@ -4692,11 +4764,12 @@ FF41..FF5A    ; Changes_When_Uppercased # L&  [26] FULLWIDTH LATIN SMALL LETTER
 105B3..105B9  ; Changes_When_Uppercased # L&   [7] VITHKUQI SMALL LETTER SE..VITHKUQI SMALL LETTER XE
 105BB..105BC  ; Changes_When_Uppercased # L&   [2] VITHKUQI SMALL LETTER Y..VITHKUQI SMALL LETTER ZE
 10CC0..10CF2  ; Changes_When_Uppercased # L&  [51] OLD HUNGARIAN SMALL LETTER A..OLD HUNGARIAN SMALL LETTER US
+10D70..10D85  ; Changes_When_Uppercased # L&  [22] GARAY SMALL LETTER A..GARAY SMALL LETTER OLD NA
 118C0..118DF  ; Changes_When_Uppercased # L&  [32] WARANG CITI SMALL LETTER NGAA..WARANG CITI SMALL LETTER VIYO
 16E60..16E7F  ; Changes_When_Uppercased # L&  [32] MEDEFAIDRIN SMALL LETTER M..MEDEFAIDRIN SMALL LETTER Y
 1E922..1E943  ; Changes_When_Uppercased # L&  [34] ADLAM SMALL LETTER ALIF..ADLAM SMALL LETTER SHA
 
-# Total code points: 1525
+# Total code points: 1552
 
 # ================================================
 
@@ -4777,7 +4850,7 @@ FF41..FF5A    ; Changes_When_Uppercased # L&  [26] FULLWIDTH LATIN SMALL LETTER
 018C          ; Changes_When_Titlecased # L&       LATIN SMALL LETTER D WITH TOPBAR
 0192          ; Changes_When_Titlecased # L&       LATIN SMALL LETTER F WITH HOOK
 0195          ; Changes_When_Titlecased # L&       LATIN SMALL LETTER HV
-0199..019A    ; Changes_When_Titlecased # L&   [2] LATIN SMALL LETTER K WITH HOOK..LATIN SMALL LETTER L WITH BAR
+0199..019B    ; Changes_When_Titlecased # L&   [3] LATIN SMALL LETTER K WITH HOOK..LATIN SMALL LETTER LAMBDA WITH STROKE
 019E          ; Changes_When_Titlecased # L&       LATIN SMALL LETTER N WITH LONG RIGHT LEG
 01A1          ; Changes_When_Titlecased # L&       LATIN SMALL LETTER O WITH HORN
 01A3          ; Changes_When_Titlecased # L&       LATIN SMALL LETTER OI
@@ -4854,8 +4927,7 @@ FF41..FF5A    ; Changes_When_Uppercased # L&  [26] FULLWIDTH LATIN SMALL LETTER
 0259          ; Changes_When_Titlecased # L&       LATIN SMALL LETTER SCHWA
 025B..025C    ; Changes_When_Titlecased # L&   [2] LATIN SMALL LETTER OPEN E..LATIN SMALL LETTER REVERSED OPEN E
 0260..0261    ; Changes_When_Titlecased # L&   [2] LATIN SMALL LETTER G WITH HOOK..LATIN SMALL LETTER SCRIPT G
-0263          ; Changes_When_Titlecased # L&       LATIN SMALL LETTER GAMMA
-0265..0266    ; Changes_When_Titlecased # L&   [2] LATIN SMALL LETTER TURNED H..LATIN SMALL LETTER H WITH HOOK
+0263..0266    ; Changes_When_Titlecased # L&   [4] LATIN SMALL LETTER GAMMA..LATIN SMALL LETTER H WITH HOOK
 0268..026C    ; Changes_When_Titlecased # L&   [5] LATIN SMALL LETTER I WITH STROKE..LATIN SMALL LETTER L WITH BELT
 026F          ; Changes_When_Titlecased # L&       LATIN SMALL LETTER TURNED M
 0271..0272    ; Changes_When_Titlecased # L&   [2] LATIN SMALL LETTER M WITH HOOK..LATIN SMALL LETTER N WITH LEFT HOOK
@@ -4993,6 +5065,7 @@ FF41..FF5A    ; Changes_When_Uppercased # L&  [26] FULLWIDTH LATIN SMALL LETTER
 0561..0587    ; Changes_When_Titlecased # L&  [39] ARMENIAN SMALL LETTER AYB..ARMENIAN SMALL LIGATURE ECH YIWN
 13F8..13FD    ; Changes_When_Titlecased # L&   [6] CHEROKEE SMALL LETTER YE..CHEROKEE SMALL LETTER MV
 1C80..1C88    ; Changes_When_Titlecased # L&   [9] CYRILLIC SMALL LETTER ROUNDED VE..CYRILLIC SMALL LETTER UNBLENDED UK
+1C8A          ; Changes_When_Titlecased # L&       CYRILLIC SMALL LETTER TJE
 1D79          ; Changes_When_Titlecased # L&       LATIN SMALL LETTER INSULAR G
 1D7D          ; Changes_When_Titlecased # L&       LATIN SMALL LETTER P WITH STROKE
 1D8E          ; Changes_When_Titlecased # L&       LATIN SMALL LETTER Z WITH PALATAL HOOK
@@ -5312,9 +5385,11 @@ A7C1          ; Changes_When_Titlecased # L&       LATIN SMALL LETTER OLD POLISH
 A7C3          ; Changes_When_Titlecased # L&       LATIN SMALL LETTER ANGLICANA W
 A7C8          ; Changes_When_Titlecased # L&       LATIN SMALL LETTER D WITH SHORT STROKE OVERLAY
 A7CA          ; Changes_When_Titlecased # L&       LATIN SMALL LETTER S WITH SHORT STROKE OVERLAY
+A7CD          ; Changes_When_Titlecased # L&       LATIN SMALL LETTER S WITH DIAGONAL STROKE
 A7D1          ; Changes_When_Titlecased # L&       LATIN SMALL LETTER CLOSED INSULAR G
 A7D7          ; Changes_When_Titlecased # L&       LATIN SMALL LETTER MIDDLE SCOTS S
 A7D9          ; Changes_When_Titlecased # L&       LATIN SMALL LETTER SIGMOID S
+A7DB          ; Changes_When_Titlecased # L&       LATIN SMALL LETTER LAMBDA
 A7F6          ; Changes_When_Titlecased # L&       LATIN SMALL LETTER REVERSED HALF H
 AB53          ; Changes_When_Titlecased # L&       LATIN SMALL LETTER CHI
 AB70..ABBF    ; Changes_When_Titlecased # L&  [80] CHEROKEE SMALL LETTER A..CHEROKEE SMALL LETTER YA
@@ -5328,11 +5403,12 @@ FF41..FF5A    ; Changes_When_Titlecased # L&  [26] FULLWIDTH LATIN SMALL LETTER
 105B3..105B9  ; Changes_When_Titlecased # L&   [7] VITHKUQI SMALL LETTER SE..VITHKUQI SMALL LETTER XE
 105BB..105BC  ; Changes_When_Titlecased # L&   [2] VITHKUQI SMALL LETTER Y..VITHKUQI SMALL LETTER ZE
 10CC0..10CF2  ; Changes_When_Titlecased # L&  [51] OLD HUNGARIAN SMALL LETTER A..OLD HUNGARIAN SMALL LETTER US
+10D70..10D85  ; Changes_When_Titlecased # L&  [22] GARAY SMALL LETTER A..GARAY SMALL LETTER OLD NA
 118C0..118DF  ; Changes_When_Titlecased # L&  [32] WARANG CITI SMALL LETTER NGAA..WARANG CITI SMALL LETTER VIYO
 16E60..16E7F  ; Changes_When_Titlecased # L&  [32] MEDEFAIDRIN SMALL LETTER M..MEDEFAIDRIN SMALL LETTER Y
 1E922..1E943  ; Changes_When_Titlecased # L&  [34] ADLAM SMALL LETTER ALIF..ADLAM SMALL LETTER SHA
 
-# Total code points: 1452
+# Total code points: 1479
 
 # ================================================
 
@@ -5623,7 +5699,7 @@ FF41..FF5A    ; Changes_When_Titlecased # L&  [26] FULLWIDTH LATIN SMALL LETTER
 10C7          ; Changes_When_Casefolded # L&       GEORGIAN CAPITAL LETTER YN
 10CD          ; Changes_When_Casefolded # L&       GEORGIAN CAPITAL LETTER AEN
 13F8..13FD    ; Changes_When_Casefolded # L&   [6] CHEROKEE SMALL LETTER YE..CHEROKEE SMALL LETTER MV
-1C80..1C88    ; Changes_When_Casefolded # L&   [9] CYRILLIC SMALL LETTER ROUNDED VE..CYRILLIC SMALL LETTER UNBLENDED UK
+1C80..1C89    ; Changes_When_Casefolded # L&  [10] CYRILLIC SMALL LETTER ROUNDED VE..CYRILLIC CAPITAL LETTER TJE
 1C90..1CBA    ; Changes_When_Casefolded # L&  [43] GEORGIAN MTAVRULI CAPITAL LETTER AN..GEORGIAN MTAVRULI CAPITAL LETTER AIN
 1CBD..1CBF    ; Changes_When_Casefolded # L&   [3] GEORGIAN MTAVRULI CAPITAL LETTER AEN..GEORGIAN MTAVRULI CAPITAL LETTER LABIAL SIGN
 1E00          ; Changes_When_Casefolded # L&       LATIN CAPITAL LETTER A WITH RING BELOW
@@ -5945,9 +6021,12 @@ A7C0          ; Changes_When_Casefolded # L&       LATIN CAPITAL LETTER OLD POLI
 A7C2          ; Changes_When_Casefolded # L&       LATIN CAPITAL LETTER ANGLICANA W
 A7C4..A7C7    ; Changes_When_Casefolded # L&   [4] LATIN CAPITAL LETTER C WITH PALATAL HOOK..LATIN CAPITAL LETTER D WITH SHORT STROKE OVERLAY
 A7C9          ; Changes_When_Casefolded # L&       LATIN CAPITAL LETTER S WITH SHORT STROKE OVERLAY
+A7CB..A7CC    ; Changes_When_Casefolded # L&   [2] LATIN CAPITAL LETTER RAMS HORN..LATIN CAPITAL LETTER S WITH DIAGONAL STROKE
 A7D0          ; Changes_When_Casefolded # L&       LATIN CAPITAL LETTER CLOSED INSULAR G
 A7D6          ; Changes_When_Casefolded # L&       LATIN CAPITAL LETTER MIDDLE SCOTS S
 A7D8          ; Changes_When_Casefolded # L&       LATIN CAPITAL LETTER SIGMOID S
+A7DA          ; Changes_When_Casefolded # L&       LATIN CAPITAL LETTER LAMBDA
+A7DC          ; Changes_When_Casefolded # L&       LATIN CAPITAL LETTER LAMBDA WITH STROKE
 A7F5          ; Changes_When_Casefolded # L&       LATIN CAPITAL LETTER REVERSED HALF H
 AB70..ABBF    ; Changes_When_Casefolded # L&  [80] CHEROKEE SMALL LETTER A..CHEROKEE SMALL LETTER YA
 FB00..FB06    ; Changes_When_Casefolded # L&   [7] LATIN SMALL LIGATURE FF..LATIN SMALL LIGATURE ST
@@ -5960,11 +6039,12 @@ FF21..FF3A    ; Changes_When_Casefolded # L&  [26] FULLWIDTH LATIN CAPITAL LETTE
 1058C..10592  ; Changes_When_Casefolded # L&   [7] VITHKUQI CAPITAL LETTER SE..VITHKUQI CAPITAL LETTER XE
 10594..10595  ; Changes_When_Casefolded # L&   [2] VITHKUQI CAPITAL LETTER Y..VITHKUQI CAPITAL LETTER ZE
 10C80..10CB2  ; Changes_When_Casefolded # L&  [51] OLD HUNGARIAN CAPITAL LETTER A..OLD HUNGARIAN CAPITAL LETTER US
+10D50..10D65  ; Changes_When_Casefolded # L&  [22] GARAY CAPITAL LETTER A..GARAY CAPITAL LETTER OLD NA
 118A0..118BF  ; Changes_When_Casefolded # L&  [32] WARANG CITI CAPITAL LETTER NGAA..WARANG CITI CAPITAL LETTER VIYO
 16E40..16E5F  ; Changes_When_Casefolded # L&  [32] MEDEFAIDRIN CAPITAL LETTER M..MEDEFAIDRIN CAPITAL LETTER Y
 1E900..1E921  ; Changes_When_Casefolded # L&  [34] ADLAM CAPITAL LETTER ALIF..ADLAM CAPITAL LETTER SHA
 
-# Total code points: 1506
+# Total code points: 1533
 
 # ================================================
 
@@ -5980,8 +6060,7 @@ FF21..FF3A    ; Changes_When_Casefolded # L&  [26] FULLWIDTH LATIN CAPITAL LETTE
 00D8..00F6    ; Changes_When_Casemapped # L&  [31] LATIN CAPITAL LETTER O WITH STROKE..LATIN SMALL LETTER O WITH DIAERESIS
 00F8..0137    ; Changes_When_Casemapped # L&  [64] LATIN SMALL LETTER O WITH STROKE..LATIN SMALL LETTER K WITH CEDILLA
 0139..018C    ; Changes_When_Casemapped # L&  [84] LATIN CAPITAL LETTER L WITH ACUTE..LATIN SMALL LETTER D WITH TOPBAR
-018E..019A    ; Changes_When_Casemapped # L&  [13] LATIN CAPITAL LETTER REVERSED E..LATIN SMALL LETTER L WITH BAR
-019C..01A9    ; Changes_When_Casemapped # L&  [14] LATIN CAPITAL LETTER TURNED M..LATIN CAPITAL LETTER ESH
+018E..01A9    ; Changes_When_Casemapped # L&  [28] LATIN CAPITAL LETTER REVERSED E..LATIN CAPITAL LETTER ESH
 01AC..01B9    ; Changes_When_Casemapped # L&  [14] LATIN CAPITAL LETTER T WITH HOOK..LATIN SMALL LETTER EZH REVERSED
 01BC..01BD    ; Changes_When_Casemapped # L&   [2] LATIN CAPITAL LETTER TONE FIVE..LATIN SMALL LETTER TONE FIVE
 01BF          ; Changes_When_Casemapped # L&       LATIN LETTER WYNN
@@ -5992,8 +6071,7 @@ FF21..FF3A    ; Changes_When_Casefolded # L&  [26] FULLWIDTH LATIN CAPITAL LETTE
 0259          ; Changes_When_Casemapped # L&       LATIN SMALL LETTER SCHWA
 025B..025C    ; Changes_When_Casemapped # L&   [2] LATIN SMALL LETTER OPEN E..LATIN SMALL LETTER REVERSED OPEN E
 0260..0261    ; Changes_When_Casemapped # L&   [2] LATIN SMALL LETTER G WITH HOOK..LATIN SMALL LETTER SCRIPT G
-0263          ; Changes_When_Casemapped # L&       LATIN SMALL LETTER GAMMA
-0265..0266    ; Changes_When_Casemapped # L&   [2] LATIN SMALL LETTER TURNED H..LATIN SMALL LETTER H WITH HOOK
+0263..0266    ; Changes_When_Casemapped # L&   [4] LATIN SMALL LETTER GAMMA..LATIN SMALL LETTER H WITH HOOK
 0268..026C    ; Changes_When_Casemapped # L&   [5] LATIN SMALL LETTER I WITH STROKE..LATIN SMALL LETTER L WITH BELT
 026F          ; Changes_When_Casemapped # L&       LATIN SMALL LETTER TURNED M
 0271..0272    ; Changes_When_Casemapped # L&   [2] LATIN SMALL LETTER M WITH HOOK..LATIN SMALL LETTER N WITH LEFT HOOK
@@ -6027,7 +6105,7 @@ FF21..FF3A    ; Changes_When_Casefolded # L&  [26] FULLWIDTH LATIN CAPITAL LETTE
 10FD..10FF    ; Changes_When_Casemapped # L&   [3] GEORGIAN LETTER AEN..GEORGIAN LETTER LABIAL SIGN
 13A0..13F5    ; Changes_When_Casemapped # L&  [86] CHEROKEE LETTER A..CHEROKEE LETTER MV
 13F8..13FD    ; Changes_When_Casemapped # L&   [6] CHEROKEE SMALL LETTER YE..CHEROKEE SMALL LETTER MV
-1C80..1C88    ; Changes_When_Casemapped # L&   [9] CYRILLIC SMALL LETTER ROUNDED VE..CYRILLIC SMALL LETTER UNBLENDED UK
+1C80..1C8A    ; Changes_When_Casemapped # L&  [11] CYRILLIC SMALL LETTER ROUNDED VE..CYRILLIC SMALL LETTER TJE
 1C90..1CBA    ; Changes_When_Casemapped # L&  [43] GEORGIAN MTAVRULI CAPITAL LETTER AN..GEORGIAN MTAVRULI CAPITAL LETTER AIN
 1CBD..1CBF    ; Changes_When_Casemapped # L&   [3] GEORGIAN MTAVRULI CAPITAL LETTER AEN..GEORGIAN MTAVRULI CAPITAL LETTER LABIAL SIGN
 1D79          ; Changes_When_Casemapped # L&       LATIN SMALL LETTER INSULAR G
@@ -6078,9 +6156,9 @@ A779..A787    ; Changes_When_Casemapped # L&  [15] LATIN CAPITAL LETTER INSULAR
 A78B..A78D    ; Changes_When_Casemapped # L&   [3] LATIN CAPITAL LETTER SALTILLO..LATIN CAPITAL LETTER TURNED H
 A790..A794    ; Changes_When_Casemapped # L&   [5] LATIN CAPITAL LETTER N WITH DESCENDER..LATIN SMALL LETTER C WITH PALATAL HOOK
 A796..A7AE    ; Changes_When_Casemapped # L&  [25] LATIN CAPITAL LETTER B WITH FLOURISH..LATIN CAPITAL LETTER SMALL CAPITAL I
-A7B0..A7CA    ; Changes_When_Casemapped # L&  [27] LATIN CAPITAL LETTER TURNED K..LATIN SMALL LETTER S WITH SHORT STROKE OVERLAY
+A7B0..A7CD    ; Changes_When_Casemapped # L&  [30] LATIN CAPITAL LETTER TURNED K..LATIN SMALL LETTER S WITH DIAGONAL STROKE
 A7D0..A7D1    ; Changes_When_Casemapped # L&   [2] LATIN CAPITAL LETTER CLOSED INSULAR G..LATIN SMALL LETTER CLOSED INSULAR G
-A7D6..A7D9    ; Changes_When_Casemapped # L&   [4] LATIN CAPITAL LETTER MIDDLE SCOTS S..LATIN SMALL LETTER SIGMOID S
+A7D6..A7DC    ; Changes_When_Casemapped # L&   [7] LATIN CAPITAL LETTER MIDDLE SCOTS S..LATIN CAPITAL LETTER LAMBDA WITH STROKE
 A7F5..A7F6    ; Changes_When_Casemapped # L&   [2] LATIN CAPITAL LETTER REVERSED HALF H..LATIN SMALL LETTER REVERSED HALF H
 AB53          ; Changes_When_Casemapped # L&       LATIN SMALL LETTER CHI
 AB70..ABBF    ; Changes_When_Casemapped # L&  [80] CHEROKEE SMALL LETTER A..CHEROKEE SMALL LETTER YA
@@ -6101,11 +6179,13 @@ FF41..FF5A    ; Changes_When_Casemapped # L&  [26] FULLWIDTH LATIN SMALL LETTER
 105BB..105BC  ; Changes_When_Casemapped # L&   [2] VITHKUQI SMALL LETTER Y..VITHKUQI SMALL LETTER ZE
 10C80..10CB2  ; Changes_When_Casemapped # L&  [51] OLD HUNGARIAN CAPITAL LETTER A..OLD HUNGARIAN CAPITAL LETTER US
 10CC0..10CF2  ; Changes_When_Casemapped # L&  [51] OLD HUNGARIAN SMALL LETTER A..OLD HUNGARIAN SMALL LETTER US
+10D50..10D65  ; Changes_When_Casemapped # L&  [22] GARAY CAPITAL LETTER A..GARAY CAPITAL LETTER OLD NA
+10D70..10D85  ; Changes_When_Casemapped # L&  [22] GARAY SMALL LETTER A..GARAY SMALL LETTER OLD NA
 118A0..118DF  ; Changes_When_Casemapped # L&  [64] WARANG CITI CAPITAL LETTER NGAA..WARANG CITI SMALL LETTER VIYO
 16E40..16E7F  ; Changes_When_Casemapped # L&  [64] MEDEFAIDRIN CAPITAL LETTER M..MEDEFAIDRIN SMALL LETTER Y
 1E900..1E943  ; Changes_When_Casemapped # L&  [68] ADLAM CAPITAL LETTER ALIF..ADLAM SMALL LETTER SHA
 
-# Total code points: 2927
+# Total code points: 2981
 
 # ================================================
 
@@ -6364,7 +6444,7 @@ FF41..FF5A    ; Changes_When_Casemapped # L&  [26] FULLWIDTH LATIN SMALL LETTER
 1C4D..1C4F    ; ID_Start # Lo   [3] LEPCHA LETTER TTA..LEPCHA LETTER DDA
 1C5A..1C77    ; ID_Start # Lo  [30] OL CHIKI LETTER LA..OL CHIKI LETTER OH
 1C78..1C7D    ; ID_Start # Lm   [6] OL CHIKI MU TTUDDAG..OL CHIKI AHAD
-1C80..1C88    ; ID_Start # L&   [9] CYRILLIC SMALL LETTER ROUNDED VE..CYRILLIC SMALL LETTER UNBLENDED UK
+1C80..1C8A    ; ID_Start # L&  [11] CYRILLIC SMALL LETTER ROUNDED VE..CYRILLIC SMALL LETTER TJE
 1C90..1CBA    ; ID_Start # L&  [43] GEORGIAN MTAVRULI CAPITAL LETTER AN..GEORGIAN MTAVRULI CAPITAL LETTER AIN
 1CBD..1CBF    ; ID_Start # L&   [3] GEORGIAN MTAVRULI CAPITAL LETTER AEN..GEORGIAN MTAVRULI CAPITAL LETTER LABIAL SIGN
 1CE9..1CEC    ; ID_Start # Lo   [4] VEDIC SIGN ANUSVARA ANTARGOMUKHA..VEDIC SIGN ANUSVARA VAMAGOMUKHA WITH TAIL
@@ -6481,10 +6561,10 @@ A771..A787    ; ID_Start # L&  [23] LATIN SMALL LETTER DUM..LATIN SMALL LETTER I
 A788          ; ID_Start # Lm       MODIFIER LETTER LOW CIRCUMFLEX ACCENT
 A78B..A78E    ; ID_Start # L&   [4] LATIN CAPITAL LETTER SALTILLO..LATIN SMALL LETTER L WITH RETROFLEX HOOK AND BELT
 A78F          ; ID_Start # Lo       LATIN LETTER SINOLOGICAL DOT
-A790..A7CA    ; ID_Start # L&  [59] LATIN CAPITAL LETTER N WITH DESCENDER..LATIN SMALL LETTER S WITH SHORT STROKE OVERLAY
+A790..A7CD    ; ID_Start # L&  [62] LATIN CAPITAL LETTER N WITH DESCENDER..LATIN SMALL LETTER S WITH DIAGONAL STROKE
 A7D0..A7D1    ; ID_Start # L&   [2] LATIN CAPITAL LETTER CLOSED INSULAR G..LATIN SMALL LETTER CLOSED INSULAR G
 A7D3          ; ID_Start # L&       LATIN SMALL LETTER DOUBLE THORN
-A7D5..A7D9    ; ID_Start # L&   [5] LATIN SMALL LETTER DOUBLE WYNN..LATIN SMALL LETTER SIGMOID S
+A7D5..A7DC    ; ID_Start # L&   [8] LATIN SMALL LETTER DOUBLE WYNN..LATIN CAPITAL LETTER LAMBDA WITH STROKE
 A7F2..A7F4    ; ID_Start # Lm   [3] MODIFIER LETTER CAPITAL C..MODIFIER LETTER CAPITAL Q
 A7F5..A7F6    ; ID_Start # L&   [2] LATIN CAPITAL LETTER REVERSED HALF H..LATIN SMALL LETTER REVERSED HALF H
 A7F7          ; ID_Start # Lo       LATIN EPIGRAPHIC LETTER SIDEWAYS I
@@ -6603,6 +6683,7 @@ FFDA..FFDC    ; ID_Start # Lo   [3] HALFWIDTH HANGUL LETTER EU..HALFWIDTH HANGUL
 105A3..105B1  ; ID_Start # L&  [15] VITHKUQI SMALL LETTER HA..VITHKUQI SMALL LETTER RE
 105B3..105B9  ; ID_Start # L&   [7] VITHKUQI SMALL LETTER SE..VITHKUQI SMALL LETTER XE
 105BB..105BC  ; ID_Start # L&   [2] VITHKUQI SMALL LETTER Y..VITHKUQI SMALL LETTER ZE
+105C0..105F3  ; ID_Start # Lo  [52] TODHRI LETTER A..TODHRI LETTER OO
 10600..10736  ; ID_Start # Lo [311] LINEAR A SIGN AB001..LINEAR A SIGN A664
 10740..10755  ; ID_Start # Lo  [22] LINEAR A SIGN A701 A..LINEAR A SIGN A732 JE
 10760..10767  ; ID_Start # Lo   [8] LINEAR A SIGN A800..LINEAR A SIGN A807
@@ -6639,8 +6720,15 @@ FFDA..FFDC    ; ID_Start # Lo   [3] HALFWIDTH HANGUL LETTER EU..HALFWIDTH HANGUL
 10C80..10CB2  ; ID_Start # L&  [51] OLD HUNGARIAN CAPITAL LETTER A..OLD HUNGARIAN CAPITAL LETTER US
 10CC0..10CF2  ; ID_Start # L&  [51] OLD HUNGARIAN SMALL LETTER A..OLD HUNGARIAN SMALL LETTER US
 10D00..10D23  ; ID_Start # Lo  [36] HANIFI ROHINGYA LETTER A..HANIFI ROHINGYA MARK NA KHONNA
+10D4A..10D4D  ; ID_Start # Lo   [4] GARAY VOWEL SIGN A..GARAY VOWEL SIGN EE
+10D4E         ; ID_Start # Lm       GARAY VOWEL LENGTH MARK
+10D4F         ; ID_Start # Lo       GARAY SUKUN
+10D50..10D65  ; ID_Start # L&  [22] GARAY CAPITAL LETTER A..GARAY CAPITAL LETTER OLD NA
+10D6F         ; ID_Start # Lm       GARAY REDUPLICATION MARK
+10D70..10D85  ; ID_Start # L&  [22] GARAY SMALL LETTER A..GARAY SMALL LETTER OLD NA
 10E80..10EA9  ; ID_Start # Lo  [42] YEZIDI LETTER ELIF..YEZIDI LETTER ET
 10EB0..10EB1  ; ID_Start # Lo   [2] YEZIDI LETTER LAM WITH DOT ABOVE..YEZIDI LETTER YOT WITH CIRCUMFLEX ABOVE
+10EC2..10EC4  ; ID_Start # Lo   [3] ARABIC LETTER DAL WITH TWO DOTS VERTICALLY BELOW..ARABIC LETTER KAF WITH TWO DOTS VERTICALLY BELOW
 10F00..10F1C  ; ID_Start # Lo  [29] OLD SOGDIAN LETTER ALEPH..OLD SOGDIAN LETTER FINAL TAW WITH VERTICAL TAIL
 10F27         ; ID_Start # Lo       OLD SOGDIAN LIGATURE AYIN-DALETH
 10F30..10F45  ; ID_Start # Lo  [22] SOGDIAN LETTER ALEPH..SOGDIAN INDEPENDENT SHIN
@@ -6679,6 +6767,13 @@ FFDA..FFDC    ; ID_Start # Lo   [3] HALFWIDTH HANGUL LETTER EU..HALFWIDTH HANGUL
 1133D         ; ID_Start # Lo       GRANTHA SIGN AVAGRAHA
 11350         ; ID_Start # Lo       GRANTHA OM
 1135D..11361  ; ID_Start # Lo   [5] GRANTHA SIGN PLUTA..GRANTHA LETTER VOCALIC LL
+11380..11389  ; ID_Start # Lo  [10] TULU-TIGALARI LETTER A..TULU-TIGALARI LETTER VOCALIC LL
+1138B         ; ID_Start # Lo       TULU-TIGALARI LETTER EE
+1138E         ; ID_Start # Lo       TULU-TIGALARI LETTER AI
+11390..113B5  ; ID_Start # Lo  [38] TULU-TIGALARI LETTER OO..TULU-TIGALARI LETTER LLLA
+113B7         ; ID_Start # Lo       TULU-TIGALARI SIGN AVAGRAHA
+113D1         ; ID_Start # Lo       TULU-TIGALARI REPHA
+113D3         ; ID_Start # Lo       TULU-TIGALARI SIGN PLUTA
 11400..11434  ; ID_Start # Lo  [53] NEWA LETTER A..NEWA LETTER HA
 11447..1144A  ; ID_Start # Lo   [4] NEWA SIGN AVAGRAHA..NEWA SIDDHI
 1145F..11461  ; ID_Start # Lo   [3] NEWA LETTER VEDIC ANUSVARA..NEWA SIGN UPADHMANIYA
@@ -6713,6 +6808,7 @@ FFDA..FFDC    ; ID_Start # Lo   [3] HALFWIDTH HANGUL LETTER EU..HALFWIDTH HANGUL
 11A5C..11A89  ; ID_Start # Lo  [46] SOYOMBO LETTER KA..SOYOMBO CLUSTER-INITIAL LETTER SA
 11A9D         ; ID_Start # Lo       SOYOMBO MARK PLUTA
 11AB0..11AF8  ; ID_Start # Lo  [73] CANADIAN SYLLABICS NATTILIK HI..PAU CIN HAU GLOTTAL STOP FINAL
+11BC0..11BE0  ; ID_Start # Lo  [33] SUNUWAR LETTER DEVI..SUNUWAR LETTER KLOKO
 11C00..11C08  ; ID_Start # Lo   [9] BHAIKSUKI LETTER A..BHAIKSUKI LETTER VOCALIC L
 11C0A..11C2E  ; ID_Start # Lo  [37] BHAIKSUKI LETTER E..BHAIKSUKI LETTER HA
 11C40         ; ID_Start # Lo       BHAIKSUKI SIGN AVAGRAHA
@@ -6736,7 +6832,9 @@ FFDA..FFDC    ; ID_Start # Lo   [3] HALFWIDTH HANGUL LETTER EU..HALFWIDTH HANGUL
 12F90..12FF0  ; ID_Start # Lo  [97] CYPRO-MINOAN SIGN CM001..CYPRO-MINOAN SIGN CM114
 13000..1342F  ; ID_Start # Lo [1072] EGYPTIAN HIEROGLYPH A001..EGYPTIAN HIEROGLYPH V011D
 13441..13446  ; ID_Start # Lo   [6] EGYPTIAN HIEROGLYPH FULL BLANK..EGYPTIAN HIEROGLYPH WIDE LOST SIGN
+13460..143FA  ; ID_Start # Lo [3995] EGYPTIAN HIEROGLYPH-13460..EGYPTIAN HIEROGLYPH-143FA
 14400..14646  ; ID_Start # Lo [583] ANATOLIAN HIEROGLYPH A001..ANATOLIAN HIEROGLYPH A530
+16100..1611D  ; ID_Start # Lo  [30] GURUNG KHEMA LETTER A..GURUNG KHEMA LETTER SA
 16800..16A38  ; ID_Start # Lo [569] BAMUM LETTER PHASE-A NGKUE MFON..BAMUM LETTER PHASE-F VUEQ
 16A40..16A5E  ; ID_Start # Lo  [31] MRO LETTER TA..MRO LETTER TEK
 16A70..16ABE  ; ID_Start # Lo  [79] TANGSA LETTER OZ..TANGSA LETTER ZA
@@ -6745,6 +6843,9 @@ FFDA..FFDC    ; ID_Start # Lo   [3] HALFWIDTH HANGUL LETTER EU..HALFWIDTH HANGUL
 16B40..16B43  ; ID_Start # Lm   [4] PAHAWH HMONG SIGN VOS SEEV..PAHAWH HMONG SIGN IB YAM
 16B63..16B77  ; ID_Start # Lo  [21] PAHAWH HMONG SIGN VOS LUB..PAHAWH HMONG SIGN CIM NRES TOS
 16B7D..16B8F  ; ID_Start # Lo  [19] PAHAWH HMONG CLAN SIGN TSHEEJ..PAHAWH HMONG CLAN SIGN VWJ
+16D40..16D42  ; ID_Start # Lm   [3] KIRAT RAI SIGN ANUSVARA..KIRAT RAI SIGN VISARGA
+16D43..16D6A  ; ID_Start # Lo  [40] KIRAT RAI LETTER A..KIRAT RAI VOWEL SIGN AU
+16D6B..16D6C  ; ID_Start # Lm   [2] KIRAT RAI SIGN VIRAMA..KIRAT RAI SIGN SAAT
 16E40..16E7F  ; ID_Start # L&  [64] MEDEFAIDRIN CAPITAL LETTER M..MEDEFAIDRIN SMALL LETTER Y
 16F00..16F4A  ; ID_Start # Lo  [75] MIAO LETTER PA..MIAO LETTER RTE
 16F50         ; ID_Start # Lo       MIAO LETTER NASALIZATION
@@ -6753,7 +6854,7 @@ FFDA..FFDC    ; ID_Start # Lo   [3] HALFWIDTH HANGUL LETTER EU..HALFWIDTH HANGUL
 16FE3         ; ID_Start # Lm       OLD CHINESE ITERATION MARK
 17000..187F7  ; ID_Start # Lo [6136] TANGUT IDEOGRAPH-17000..TANGUT IDEOGRAPH-187F7
 18800..18CD5  ; ID_Start # Lo [1238] TANGUT COMPONENT-001..KHITAN SMALL SCRIPT CHARACTER-18CD5
-18D00..18D08  ; ID_Start # Lo   [9] TANGUT IDEOGRAPH-18D00..TANGUT IDEOGRAPH-18D08
+18CFF..18D08  ; ID_Start # Lo  [10] KHITAN SMALL SCRIPT CHARACTER-18CFF..TANGUT IDEOGRAPH-18D08
 1AFF0..1AFF3  ; ID_Start # Lm   [4] KATAKANA LETTER MINNAN TONE-2..KATAKANA LETTER MINNAN TONE-5
 1AFF5..1AFFB  ; ID_Start # Lm   [7] KATAKANA LETTER MINNAN TONE-7..KATAKANA LETTER MINNAN NASALIZED TONE-5
 1AFFD..1AFFE  ; ID_Start # Lm   [2] KATAKANA LETTER MINNAN NASALIZED TONE-7..KATAKANA LETTER MINNAN NASALIZED TONE-8
@@ -6809,6 +6910,8 @@ FFDA..FFDC    ; ID_Start # Lo   [3] HALFWIDTH HANGUL LETTER EU..HALFWIDTH HANGUL
 1E2C0..1E2EB  ; ID_Start # Lo  [44] WANCHO LETTER AA..WANCHO LETTER YIH
 1E4D0..1E4EA  ; ID_Start # Lo  [27] NAG MUNDARI LETTER O..NAG MUNDARI LETTER ELL
 1E4EB         ; ID_Start # Lm       NAG MUNDARI SIGN OJOD
+1E5D0..1E5ED  ; ID_Start # Lo  [30] OL ONAL LETTER O..OL ONAL LETTER EG
+1E5F0         ; ID_Start # Lo       OL ONAL SIGN HODDOND
 1E7E0..1E7E6  ; ID_Start # Lo   [7] ETHIOPIC SYLLABLE HHYA..ETHIOPIC SYLLABLE HHYO
 1E7E8..1E7EB  ; ID_Start # Lo   [4] ETHIOPIC SYLLABLE GURAGE HHWA..ETHIOPIC SYLLABLE HHWE
 1E7ED..1E7EE  ; ID_Start # Lo   [2] ETHIOPIC SYLLABLE GURAGE MWI..ETHIOPIC SYLLABLE GURAGE MWEE
@@ -6859,7 +6962,7 @@ FFDA..FFDC    ; ID_Start # Lo   [3] HALFWIDTH HANGUL LETTER EU..HALFWIDTH HANGUL
 30000..3134A  ; ID_Start # Lo [4939] CJK UNIFIED IDEOGRAPH-30000..CJK UNIFIED IDEOGRAPH-3134A
 31350..323AF  ; ID_Start # Lo [4192] CJK UNIFIED IDEOGRAPH-31350..CJK UNIFIED IDEOGRAPH-323AF
 
-# Total code points: 136967
+# Total code points: 141269
 
 # ================================================
 
@@ -6966,7 +7069,7 @@ FFDA..FFDC    ; ID_Start # Lo   [3] HALFWIDTH HANGUL LETTER EU..HALFWIDTH HANGUL
 0860..086A    ; ID_Continue # Lo  [11] SYRIAC LETTER MALAYALAM NGA..SYRIAC LETTER MALAYALAM SSA
 0870..0887    ; ID_Continue # Lo  [24] ARABIC LETTER ALEF WITH ATTACHED FATHA..ARABIC BASELINE ROUND DOT
 0889..088E    ; ID_Continue # Lo   [6] ARABIC LETTER NOON WITH INVERTED SMALL V..ARABIC VERTICAL TAIL
-0898..089F    ; ID_Continue # Mn   [8] ARABIC SMALL HIGH WORD AL-JUZ..ARABIC HALF MADDA OVER MADDA
+0897..089F    ; ID_Continue # Mn   [9] ARABIC PEPET..ARABIC HALF MADDA OVER MADDA
 08A0..08C8    ; ID_Continue # Lo  [41] ARABIC LETTER BEH WITH SMALL V BELOW..ARABIC LETTER GRAF
 08C9          ; ID_Continue # Lm       ARABIC SMALL FARSI YEH
 08CA..08E1    ; ID_Continue # Mn  [24] ARABIC SMALL HIGH FARSI YEH..ARABIC SMALL HIGH SIGN SAFHA
@@ -7399,7 +7502,7 @@ FFDA..FFDC    ; ID_Start # Lo   [3] HALFWIDTH HANGUL LETTER EU..HALFWIDTH HANGUL
 1C50..1C59    ; ID_Continue # Nd  [10] OL CHIKI DIGIT ZERO..OL CHIKI DIGIT NINE
 1C5A..1C77    ; ID_Continue # Lo  [30] OL CHIKI LETTER LA..OL CHIKI LETTER OH
 1C78..1C7D    ; ID_Continue # Lm   [6] OL CHIKI MU TTUDDAG..OL CHIKI AHAD
-1C80..1C88    ; ID_Continue # L&   [9] CYRILLIC SMALL LETTER ROUNDED VE..CYRILLIC SMALL LETTER UNBLENDED UK
+1C80..1C8A    ; ID_Continue # L&  [11] CYRILLIC SMALL LETTER ROUNDED VE..CYRILLIC SMALL LETTER TJE
 1C90..1CBA    ; ID_Continue # L&  [43] GEORGIAN MTAVRULI CAPITAL LETTER AN..GEORGIAN MTAVRULI CAPITAL LETTER AIN
 1CBD..1CBF    ; ID_Continue # L&   [3] GEORGIAN MTAVRULI CAPITAL LETTER AEN..GEORGIAN MTAVRULI CAPITAL LETTER LABIAL SIGN
 1CD0..1CD2    ; ID_Continue # Mn   [3] VEDIC TONE KARSHANA..VEDIC TONE PRENKHA
@@ -7543,10 +7646,10 @@ A771..A787    ; ID_Continue # L&  [23] LATIN SMALL LETTER DUM..LATIN SMALL LETTE
 A788          ; ID_Continue # Lm       MODIFIER LETTER LOW CIRCUMFLEX ACCENT
 A78B..A78E    ; ID_Continue # L&   [4] LATIN CAPITAL LETTER SALTILLO..LATIN SMALL LETTER L WITH RETROFLEX HOOK AND BELT
 A78F          ; ID_Continue # Lo       LATIN LETTER SINOLOGICAL DOT
-A790..A7CA    ; ID_Continue # L&  [59] LATIN CAPITAL LETTER N WITH DESCENDER..LATIN SMALL LETTER S WITH SHORT STROKE OVERLAY
+A790..A7CD    ; ID_Continue # L&  [62] LATIN CAPITAL LETTER N WITH DESCENDER..LATIN SMALL LETTER S WITH DIAGONAL STROKE
 A7D0..A7D1    ; ID_Continue # L&   [2] LATIN CAPITAL LETTER CLOSED INSULAR G..LATIN SMALL LETTER CLOSED INSULAR G
 A7D3          ; ID_Continue # L&       LATIN SMALL LETTER DOUBLE THORN
-A7D5..A7D9    ; ID_Continue # L&   [5] LATIN SMALL LETTER DOUBLE WYNN..LATIN SMALL LETTER SIGMOID S
+A7D5..A7DC    ; ID_Continue # L&   [8] LATIN SMALL LETTER DOUBLE WYNN..LATIN CAPITAL LETTER LAMBDA WITH STROKE
 A7F2..A7F4    ; ID_Continue # Lm   [3] MODIFIER LETTER CAPITAL C..MODIFIER LETTER CAPITAL Q
 A7F5..A7F6    ; ID_Continue # L&   [2] LATIN CAPITAL LETTER REVERSED HALF H..LATIN SMALL LETTER REVERSED HALF H
 A7F7          ; ID_Continue # Lo       LATIN EPIGRAPHIC LETTER SIDEWAYS I
@@ -7735,6 +7838,7 @@ FFDA..FFDC    ; ID_Continue # Lo   [3] HALFWIDTH HANGUL LETTER EU..HALFWIDTH HAN
 105A3..105B1  ; ID_Continue # L&  [15] VITHKUQI SMALL LETTER HA..VITHKUQI SMALL LETTER RE
 105B3..105B9  ; ID_Continue # L&   [7] VITHKUQI SMALL LETTER SE..VITHKUQI SMALL LETTER XE
 105BB..105BC  ; ID_Continue # L&   [2] VITHKUQI SMALL LETTER Y..VITHKUQI SMALL LETTER ZE
+105C0..105F3  ; ID_Continue # Lo  [52] TODHRI LETTER A..TODHRI LETTER OO
 10600..10736  ; ID_Continue # Lo [311] LINEAR A SIGN AB001..LINEAR A SIGN A664
 10740..10755  ; ID_Continue # Lo  [22] LINEAR A SIGN A701 A..LINEAR A SIGN A732 JE
 10760..10767  ; ID_Continue # Lo   [8] LINEAR A SIGN A800..LINEAR A SIGN A807
@@ -7779,10 +7883,19 @@ FFDA..FFDC    ; ID_Continue # Lo   [3] HALFWIDTH HANGUL LETTER EU..HALFWIDTH HAN
 10D00..10D23  ; ID_Continue # Lo  [36] HANIFI ROHINGYA LETTER A..HANIFI ROHINGYA MARK NA KHONNA
 10D24..10D27  ; ID_Continue # Mn   [4] HANIFI ROHINGYA SIGN HARBAHAY..HANIFI ROHINGYA SIGN TASSI
 10D30..10D39  ; ID_Continue # Nd  [10] HANIFI ROHINGYA DIGIT ZERO..HANIFI ROHINGYA DIGIT NINE
+10D40..10D49  ; ID_Continue # Nd  [10] GARAY DIGIT ZERO..GARAY DIGIT NINE
+10D4A..10D4D  ; ID_Continue # Lo   [4] GARAY VOWEL SIGN A..GARAY VOWEL SIGN EE
+10D4E         ; ID_Continue # Lm       GARAY VOWEL LENGTH MARK
+10D4F         ; ID_Continue # Lo       GARAY SUKUN
+10D50..10D65  ; ID_Continue # L&  [22] GARAY CAPITAL LETTER A..GARAY CAPITAL LETTER OLD NA
+10D69..10D6D  ; ID_Continue # Mn   [5] GARAY VOWEL SIGN E..GARAY CONSONANT NASALIZATION MARK
+10D6F         ; ID_Continue # Lm       GARAY REDUPLICATION MARK
+10D70..10D85  ; ID_Continue # L&  [22] GARAY SMALL LETTER A..GARAY SMALL LETTER OLD NA
 10E80..10EA9  ; ID_Continue # Lo  [42] YEZIDI LETTER ELIF..YEZIDI LETTER ET
 10EAB..10EAC  ; ID_Continue # Mn   [2] YEZIDI COMBINING HAMZA MARK..YEZIDI COMBINING MADDA MARK
 10EB0..10EB1  ; ID_Continue # Lo   [2] YEZIDI LETTER LAM WITH DOT ABOVE..YEZIDI LETTER YOT WITH CIRCUMFLEX ABOVE
-10EFD..10EFF  ; ID_Continue # Mn   [3] ARABIC SMALL LOW WORD SAKTA..ARABIC SMALL LOW WORD MADDA
+10EC2..10EC4  ; ID_Continue # Lo   [3] ARABIC LETTER DAL WITH TWO DOTS VERTICALLY BELOW..ARABIC LETTER KAF WITH TWO DOTS VERTICALLY BELOW
+10EFC..10EFF  ; ID_Continue # Mn   [4] ARABIC COMBINING ALEF OVERLAY..ARABIC SMALL LOW WORD MADDA
 10F00..10F1C  ; ID_Continue # Lo  [29] OLD SOGDIAN LETTER ALEPH..OLD SOGDIAN LETTER FINAL TAW WITH VERTICAL TAIL
 10F27         ; ID_Continue # Lo       OLD SOGDIAN LIGATURE AYIN-DALETH
 10F30..10F45  ; ID_Continue # Lo  [22] SOGDIAN LETTER ALEPH..SOGDIAN INDEPENDENT SHIN
@@ -7878,6 +7991,24 @@ FFDA..FFDC    ; ID_Continue # Lo   [3] HALFWIDTH HANGUL LETTER EU..HALFWIDTH HAN
 11362..11363  ; ID_Continue # Mc   [2] GRANTHA VOWEL SIGN VOCALIC L..GRANTHA VOWEL SIGN VOCALIC LL
 11366..1136C  ; ID_Continue # Mn   [7] COMBINING GRANTHA DIGIT ZERO..COMBINING GRANTHA DIGIT SIX
 11370..11374  ; ID_Continue # Mn   [5] COMBINING GRANTHA LETTER A..COMBINING GRANTHA LETTER PA
+11380..11389  ; ID_Continue # Lo  [10] TULU-TIGALARI LETTER A..TULU-TIGALARI LETTER VOCALIC LL
+1138B         ; ID_Continue # Lo       TULU-TIGALARI LETTER EE
+1138E         ; ID_Continue # Lo       TULU-TIGALARI LETTER AI
+11390..113B5  ; ID_Continue # Lo  [38] TULU-TIGALARI LETTER OO..TULU-TIGALARI LETTER LLLA
+113B7         ; ID_Continue # Lo       TULU-TIGALARI SIGN AVAGRAHA
+113B8..113BA  ; ID_Continue # Mc   [3] TULU-TIGALARI VOWEL SIGN AA..TULU-TIGALARI VOWEL SIGN II
+113BB..113C0  ; ID_Continue # Mn   [6] TULU-TIGALARI VOWEL SIGN U..TULU-TIGALARI VOWEL SIGN VOCALIC LL
+113C2         ; ID_Continue # Mc       TULU-TIGALARI VOWEL SIGN EE
+113C5         ; ID_Continue # Mc       TULU-TIGALARI VOWEL SIGN AI
+113C7..113CA  ; ID_Continue # Mc   [4] TULU-TIGALARI VOWEL SIGN OO..TULU-TIGALARI SIGN CANDRA ANUNASIKA
+113CC..113CD  ; ID_Continue # Mc   [2] TULU-TIGALARI SIGN ANUSVARA..TULU-TIGALARI SIGN VISARGA
+113CE         ; ID_Continue # Mn       TULU-TIGALARI SIGN VIRAMA
+113CF         ; ID_Continue # Mc       TULU-TIGALARI SIGN LOOPED VIRAMA
+113D0         ; ID_Continue # Mn       TULU-TIGALARI CONJOINER
+113D1         ; ID_Continue # Lo       TULU-TIGALARI REPHA
+113D2         ; ID_Continue # Mn       TULU-TIGALARI GEMINATION MARK
+113D3         ; ID_Continue # Lo       TULU-TIGALARI SIGN PLUTA
+113E1..113E2  ; ID_Continue # Mn   [2] TULU-TIGALARI VEDIC TONE SVARITA..TULU-TIGALARI VEDIC TONE ANUDATTA
 11400..11434  ; ID_Continue # Lo  [53] NEWA LETTER A..NEWA LETTER HA
 11435..11437  ; ID_Continue # Mc   [3] NEWA VOWEL SIGN AA..NEWA VOWEL SIGN II
 11438..1143F  ; ID_Continue # Mn   [8] NEWA VOWEL SIGN U..NEWA VOWEL SIGN AI
@@ -7929,8 +8060,11 @@ FFDA..FFDC    ; ID_Continue # Lo   [3] HALFWIDTH HANGUL LETTER EU..HALFWIDTH HAN
 116B7         ; ID_Continue # Mn       TAKRI SIGN NUKTA
 116B8         ; ID_Continue # Lo       TAKRI LETTER ARCHAIC KHA
 116C0..116C9  ; ID_Continue # Nd  [10] TAKRI DIGIT ZERO..TAKRI DIGIT NINE
+116D0..116E3  ; ID_Continue # Nd  [20] MYANMAR PAO DIGIT ZERO..MYANMAR EASTERN PWO KAREN DIGIT NINE
 11700..1171A  ; ID_Continue # Lo  [27] AHOM LETTER KA..AHOM LETTER ALTERNATE BA
-1171D..1171F  ; ID_Continue # Mn   [3] AHOM CONSONANT SIGN MEDIAL LA..AHOM CONSONANT SIGN MEDIAL LIGATING RA
+1171D         ; ID_Continue # Mn       AHOM CONSONANT SIGN MEDIAL LA
+1171E         ; ID_Continue # Mc       AHOM CONSONANT SIGN MEDIAL RA
+1171F         ; ID_Continue # Mn       AHOM CONSONANT SIGN MEDIAL LIGATING RA
 11720..11721  ; ID_Continue # Mc   [2] AHOM VOWEL SIGN A..AHOM VOWEL SIGN AA
 11722..11725  ; ID_Continue # Mn   [4] AHOM VOWEL SIGN I..AHOM VOWEL SIGN UU
 11726         ; ID_Continue # Mc       AHOM VOWEL SIGN E
@@ -7988,6 +8122,8 @@ FFDA..FFDC    ; ID_Continue # Lo   [3] HALFWIDTH HANGUL LETTER EU..HALFWIDTH HAN
 11A98..11A99  ; ID_Continue # Mn   [2] SOYOMBO GEMINATION MARK..SOYOMBO SUBJOINER
 11A9D         ; ID_Continue # Lo       SOYOMBO MARK PLUTA
 11AB0..11AF8  ; ID_Continue # Lo  [73] CANADIAN SYLLABICS NATTILIK HI..PAU CIN HAU GLOTTAL STOP FINAL
+11BC0..11BE0  ; ID_Continue # Lo  [33] SUNUWAR LETTER DEVI..SUNUWAR LETTER KLOKO
+11BF0..11BF9  ; ID_Continue # Nd  [10] SUNUWAR DIGIT ZERO..SUNUWAR DIGIT NINE
 11C00..11C08  ; ID_Continue # Lo   [9] BHAIKSUKI LETTER A..BHAIKSUKI LETTER VOCALIC L
 11C0A..11C2E  ; ID_Continue # Lo  [37] BHAIKSUKI LETTER E..BHAIKSUKI LETTER HA
 11C2F         ; ID_Continue # Mc       BHAIKSUKI VOWEL SIGN AA
@@ -8041,6 +8177,7 @@ FFDA..FFDC    ; ID_Continue # Lo   [3] HALFWIDTH HANGUL LETTER EU..HALFWIDTH HAN
 11F41         ; ID_Continue # Mc       KAWI SIGN KILLER
 11F42         ; ID_Continue # Mn       KAWI CONJOINER
 11F50..11F59  ; ID_Continue # Nd  [10] KAWI DIGIT ZERO..KAWI DIGIT NINE
+11F5A         ; ID_Continue # Mn       KAWI SIGN NUKTA
 11FB0         ; ID_Continue # Lo       LISU LETTER YHA
 12000..12399  ; ID_Continue # Lo [922] CUNEIFORM SIGN A..CUNEIFORM SIGN U U
 12400..1246E  ; ID_Continue # Nl [111] CUNEIFORM NUMERIC SIGN TWO ASH..CUNEIFORM NUMERIC SIGN NINE U VARIANT FORM
@@ -8050,7 +8187,13 @@ FFDA..FFDC    ; ID_Continue # Lo   [3] HALFWIDTH HANGUL LETTER EU..HALFWIDTH HAN
 13440         ; ID_Continue # Mn       EGYPTIAN HIEROGLYPH MIRROR HORIZONTALLY
 13441..13446  ; ID_Continue # Lo   [6] EGYPTIAN HIEROGLYPH FULL BLANK..EGYPTIAN HIEROGLYPH WIDE LOST SIGN
 13447..13455  ; ID_Continue # Mn  [15] EGYPTIAN HIEROGLYPH MODIFIER DAMAGED AT TOP START..EGYPTIAN HIEROGLYPH MODIFIER DAMAGED
+13460..143FA  ; ID_Continue # Lo [3995] EGYPTIAN HIEROGLYPH-13460..EGYPTIAN HIEROGLYPH-143FA
 14400..14646  ; ID_Continue # Lo [583] ANATOLIAN HIEROGLYPH A001..ANATOLIAN HIEROGLYPH A530
+16100..1611D  ; ID_Continue # Lo  [30] GURUNG KHEMA LETTER A..GURUNG KHEMA LETTER SA
+1611E..16129  ; ID_Continue # Mn  [12] GURUNG KHEMA VOWEL SIGN AA..GURUNG KHEMA VOWEL LENGTH MARK
+1612A..1612C  ; ID_Continue # Mc   [3] GURUNG KHEMA CONSONANT SIGN MEDIAL YA..GURUNG KHEMA CONSONANT SIGN MEDIAL HA
+1612D..1612F  ; ID_Continue # Mn   [3] GURUNG KHEMA SIGN ANUSVARA..GURUNG KHEMA SIGN THOLHOMA
+16130..16139  ; ID_Continue # Nd  [10] GURUNG KHEMA DIGIT ZERO..GURUNG KHEMA DIGIT NINE
 16800..16A38  ; ID_Continue # Lo [569] BAMUM LETTER PHASE-A NGKUE MFON..BAMUM LETTER PHASE-F VUEQ
 16A40..16A5E  ; ID_Continue # Lo  [31] MRO LETTER TA..MRO LETTER TEK
 16A60..16A69  ; ID_Continue # Nd  [10] MRO DIGIT ZERO..MRO DIGIT NINE
@@ -8064,6 +8207,10 @@ FFDA..FFDC    ; ID_Continue # Lo   [3] HALFWIDTH HANGUL LETTER EU..HALFWIDTH HAN
 16B50..16B59  ; ID_Continue # Nd  [10] PAHAWH HMONG DIGIT ZERO..PAHAWH HMONG DIGIT NINE
 16B63..16B77  ; ID_Continue # Lo  [21] PAHAWH HMONG SIGN VOS LUB..PAHAWH HMONG SIGN CIM NRES TOS
 16B7D..16B8F  ; ID_Continue # Lo  [19] PAHAWH HMONG CLAN SIGN TSHEEJ..PAHAWH HMONG CLAN SIGN VWJ
+16D40..16D42  ; ID_Continue # Lm   [3] KIRAT RAI SIGN ANUSVARA..KIRAT RAI SIGN VISARGA
+16D43..16D6A  ; ID_Continue # Lo  [40] KIRAT RAI LETTER A..KIRAT RAI VOWEL SIGN AU
+16D6B..16D6C  ; ID_Continue # Lm   [2] KIRAT RAI SIGN VIRAMA..KIRAT RAI SIGN SAAT
+16D70..16D79  ; ID_Continue # Nd  [10] KIRAT RAI DIGIT ZERO..KIRAT RAI DIGIT NINE
 16E40..16E7F  ; ID_Continue # L&  [64] MEDEFAIDRIN CAPITAL LETTER M..MEDEFAIDRIN SMALL LETTER Y
 16F00..16F4A  ; ID_Continue # Lo  [75] MIAO LETTER PA..MIAO LETTER RTE
 16F4F         ; ID_Continue # Mn       MIAO SIGN CONSONANT MODIFIER BAR
@@ -8077,7 +8224,7 @@ FFDA..FFDC    ; ID_Continue # Lo   [3] HALFWIDTH HANGUL LETTER EU..HALFWIDTH HAN
 16FF0..16FF1  ; ID_Continue # Mc   [2] VIETNAMESE ALTERNATE READING MARK CA..VIETNAMESE ALTERNATE READING MARK NHAY
 17000..187F7  ; ID_Continue # Lo [6136] TANGUT IDEOGRAPH-17000..TANGUT IDEOGRAPH-187F7
 18800..18CD5  ; ID_Continue # Lo [1238] TANGUT COMPONENT-001..KHITAN SMALL SCRIPT CHARACTER-18CD5
-18D00..18D08  ; ID_Continue # Lo   [9] TANGUT IDEOGRAPH-18D00..TANGUT IDEOGRAPH-18D08
+18CFF..18D08  ; ID_Continue # Lo  [10] KHITAN SMALL SCRIPT CHARACTER-18CFF..TANGUT IDEOGRAPH-18D08
 1AFF0..1AFF3  ; ID_Continue # Lm   [4] KATAKANA LETTER MINNAN TONE-2..KATAKANA LETTER MINNAN TONE-5
 1AFF5..1AFFB  ; ID_Continue # Lm   [7] KATAKANA LETTER MINNAN TONE-7..KATAKANA LETTER MINNAN NASALIZED TONE-5
 1AFFD..1AFFE  ; ID_Continue # Lm   [2] KATAKANA LETTER MINNAN NASALIZED TONE-7..KATAKANA LETTER MINNAN NASALIZED TONE-8
@@ -8092,6 +8239,7 @@ FFDA..FFDC    ; ID_Continue # Lo   [3] HALFWIDTH HANGUL LETTER EU..HALFWIDTH HAN
 1BC80..1BC88  ; ID_Continue # Lo   [9] DUPLOYAN AFFIX HIGH ACUTE..DUPLOYAN AFFIX HIGH VERTICAL
 1BC90..1BC99  ; ID_Continue # Lo  [10] DUPLOYAN AFFIX LOW ACUTE..DUPLOYAN AFFIX LOW ARROW
 1BC9D..1BC9E  ; ID_Continue # Mn   [2] DUPLOYAN THICK LETTER SELECTOR..DUPLOYAN DOUBLE MARK
+1CCF0..1CCF9  ; ID_Continue # Nd  [10] OUTLINED DIGIT ZERO..OUTLINED DIGIT NINE
 1CF00..1CF2D  ; ID_Continue # Mn  [46] ZNAMENNY COMBINING MARK GORAZDO NIZKO S KRYZHEM ON LEFT..ZNAMENNY COMBINING MARK KRYZH ON LEFT
 1CF30..1CF46  ; ID_Continue # Mn  [23] ZNAMENNY COMBINING TONAL RANGE MARK MRACHNO..ZNAMENNY PRIZNAK MODIFIER ROG
 1D165..1D166  ; ID_Continue # Mc   [2] MUSICAL SYMBOL COMBINING STEM..MUSICAL SYMBOL COMBINING SPRECHGESANG STEM
@@ -8163,6 +8311,10 @@ FFDA..FFDC    ; ID_Continue # Lo   [3] HALFWIDTH HANGUL LETTER EU..HALFWIDTH HAN
 1E4EB         ; ID_Continue # Lm       NAG MUNDARI SIGN OJOD
 1E4EC..1E4EF  ; ID_Continue # Mn   [4] NAG MUNDARI SIGN MUHOR..NAG MUNDARI SIGN SUTUH
 1E4F0..1E4F9  ; ID_Continue # Nd  [10] NAG MUNDARI DIGIT ZERO..NAG MUNDARI DIGIT NINE
+1E5D0..1E5ED  ; ID_Continue # Lo  [30] OL ONAL LETTER O..OL ONAL LETTER EG
+1E5EE..1E5EF  ; ID_Continue # Mn   [2] OL ONAL SIGN MU..OL ONAL SIGN IKIR
+1E5F0         ; ID_Continue # Lo       OL ONAL SIGN HODDOND
+1E5F1..1E5FA  ; ID_Continue # Nd  [10] OL ONAL DIGIT ZERO..OL ONAL DIGIT NINE
 1E7E0..1E7E6  ; ID_Continue # Lo   [7] ETHIOPIC SYLLABLE HHYA..ETHIOPIC SYLLABLE HHYO
 1E7E8..1E7EB  ; ID_Continue # Lo   [4] ETHIOPIC SYLLABLE GURAGE HHWA..ETHIOPIC SYLLABLE HHWE
 1E7ED..1E7EE  ; ID_Continue # Lo   [2] ETHIOPIC SYLLABLE GURAGE MWI..ETHIOPIC SYLLABLE GURAGE MWEE
@@ -8218,7 +8370,7 @@ FFDA..FFDC    ; ID_Continue # Lo   [3] HALFWIDTH HANGUL LETTER EU..HALFWIDTH HAN
 31350..323AF  ; ID_Continue # Lo [4192] CJK UNIFIED IDEOGRAPH-31350..CJK UNIFIED IDEOGRAPH-323AF
 E0100..E01EF  ; ID_Continue # Mn [240] VARIATION SELECTOR-17..VARIATION SELECTOR-256
 
-# Total code points: 140108
+# Total code points: 144541
 
 # ================================================
 
@@ -8474,7 +8626,7 @@ E0100..E01EF  ; ID_Continue # Mn [240] VARIATION SELECTOR-17..VARIATION SELECTOR
 1C4D..1C4F    ; XID_Start # Lo   [3] LEPCHA LETTER TTA..LEPCHA LETTER DDA
 1C5A..1C77    ; XID_Start # Lo  [30] OL CHIKI LETTER LA..OL CHIKI LETTER OH
 1C78..1C7D    ; XID_Start # Lm   [6] OL CHIKI MU TTUDDAG..OL CHIKI AHAD
-1C80..1C88    ; XID_Start # L&   [9] CYRILLIC SMALL LETTER ROUNDED VE..CYRILLIC SMALL LETTER UNBLENDED UK
+1C80..1C8A    ; XID_Start # L&  [11] CYRILLIC SMALL LETTER ROUNDED VE..CYRILLIC SMALL LETTER TJE
 1C90..1CBA    ; XID_Start # L&  [43] GEORGIAN MTAVRULI CAPITAL LETTER AN..GEORGIAN MTAVRULI CAPITAL LETTER AIN
 1CBD..1CBF    ; XID_Start # L&   [3] GEORGIAN MTAVRULI CAPITAL LETTER AEN..GEORGIAN MTAVRULI CAPITAL LETTER LABIAL SIGN
 1CE9..1CEC    ; XID_Start # Lo   [4] VEDIC SIGN ANUSVARA ANTARGOMUKHA..VEDIC SIGN ANUSVARA VAMAGOMUKHA WITH TAIL
@@ -8590,10 +8742,10 @@ A771..A787    ; XID_Start # L&  [23] LATIN SMALL LETTER DUM..LATIN SMALL LETTER
 A788          ; XID_Start # Lm       MODIFIER LETTER LOW CIRCUMFLEX ACCENT
 A78B..A78E    ; XID_Start # L&   [4] LATIN CAPITAL LETTER SALTILLO..LATIN SMALL LETTER L WITH RETROFLEX HOOK AND BELT
 A78F          ; XID_Start # Lo       LATIN LETTER SINOLOGICAL DOT
-A790..A7CA    ; XID_Start # L&  [59] LATIN CAPITAL LETTER N WITH DESCENDER..LATIN SMALL LETTER S WITH SHORT STROKE OVERLAY
+A790..A7CD    ; XID_Start # L&  [62] LATIN CAPITAL LETTER N WITH DESCENDER..LATIN SMALL LETTER S WITH DIAGONAL STROKE
 A7D0..A7D1    ; XID_Start # L&   [2] LATIN CAPITAL LETTER CLOSED INSULAR G..LATIN SMALL LETTER CLOSED INSULAR G
 A7D3          ; XID_Start # L&       LATIN SMALL LETTER DOUBLE THORN
-A7D5..A7D9    ; XID_Start # L&   [5] LATIN SMALL LETTER DOUBLE WYNN..LATIN SMALL LETTER SIGMOID S
+A7D5..A7DC    ; XID_Start # L&   [8] LATIN SMALL LETTER DOUBLE WYNN..LATIN CAPITAL LETTER LAMBDA WITH STROKE
 A7F2..A7F4    ; XID_Start # Lm   [3] MODIFIER LETTER CAPITAL C..MODIFIER LETTER CAPITAL Q
 A7F5..A7F6    ; XID_Start # L&   [2] LATIN CAPITAL LETTER REVERSED HALF H..LATIN SMALL LETTER REVERSED HALF H
 A7F7          ; XID_Start # Lo       LATIN EPIGRAPHIC LETTER SIDEWAYS I
@@ -8717,6 +8869,7 @@ FFDA..FFDC    ; XID_Start # Lo   [3] HALFWIDTH HANGUL LETTER EU..HALFWIDTH HANGU
 105A3..105B1  ; XID_Start # L&  [15] VITHKUQI SMALL LETTER HA..VITHKUQI SMALL LETTER RE
 105B3..105B9  ; XID_Start # L&   [7] VITHKUQI SMALL LETTER SE..VITHKUQI SMALL LETTER XE
 105BB..105BC  ; XID_Start # L&   [2] VITHKUQI SMALL LETTER Y..VITHKUQI SMALL LETTER ZE
+105C0..105F3  ; XID_Start # Lo  [52] TODHRI LETTER A..TODHRI LETTER OO
 10600..10736  ; XID_Start # Lo [311] LINEAR A SIGN AB001..LINEAR A SIGN A664
 10740..10755  ; XID_Start # Lo  [22] LINEAR A SIGN A701 A..LINEAR A SIGN A732 JE
 10760..10767  ; XID_Start # Lo   [8] LINEAR A SIGN A800..LINEAR A SIGN A807
@@ -8753,8 +8906,15 @@ FFDA..FFDC    ; XID_Start # Lo   [3] HALFWIDTH HANGUL LETTER EU..HALFWIDTH HANGU
 10C80..10CB2  ; XID_Start # L&  [51] OLD HUNGARIAN CAPITAL LETTER A..OLD HUNGARIAN CAPITAL LETTER US
 10CC0..10CF2  ; XID_Start # L&  [51] OLD HUNGARIAN SMALL LETTER A..OLD HUNGARIAN SMALL LETTER US
 10D00..10D23  ; XID_Start # Lo  [36] HANIFI ROHINGYA LETTER A..HANIFI ROHINGYA MARK NA KHONNA
+10D4A..10D4D  ; XID_Start # Lo   [4] GARAY VOWEL SIGN A..GARAY VOWEL SIGN EE
+10D4E         ; XID_Start # Lm       GARAY VOWEL LENGTH MARK
+10D4F         ; XID_Start # Lo       GARAY SUKUN
+10D50..10D65  ; XID_Start # L&  [22] GARAY CAPITAL LETTER A..GARAY CAPITAL LETTER OLD NA
+10D6F         ; XID_Start # Lm       GARAY REDUPLICATION MARK
+10D70..10D85  ; XID_Start # L&  [22] GARAY SMALL LETTER A..GARAY SMALL LETTER OLD NA
 10E80..10EA9  ; XID_Start # Lo  [42] YEZIDI LETTER ELIF..YEZIDI LETTER ET
 10EB0..10EB1  ; XID_Start # Lo   [2] YEZIDI LETTER LAM WITH DOT ABOVE..YEZIDI LETTER YOT WITH CIRCUMFLEX ABOVE
+10EC2..10EC4  ; XID_Start # Lo   [3] ARABIC LETTER DAL WITH TWO DOTS VERTICALLY BELOW..ARABIC LETTER KAF WITH TWO DOTS VERTICALLY BELOW
 10F00..10F1C  ; XID_Start # Lo  [29] OLD SOGDIAN LETTER ALEPH..OLD SOGDIAN LETTER FINAL TAW WITH VERTICAL TAIL
 10F27         ; XID_Start # Lo       OLD SOGDIAN LIGATURE AYIN-DALETH
 10F30..10F45  ; XID_Start # Lo  [22] SOGDIAN LETTER ALEPH..SOGDIAN INDEPENDENT SHIN
@@ -8793,6 +8953,13 @@ FFDA..FFDC    ; XID_Start # Lo   [3] HALFWIDTH HANGUL LETTER EU..HALFWIDTH HANGU
 1133D         ; XID_Start # Lo       GRANTHA SIGN AVAGRAHA
 11350         ; XID_Start # Lo       GRANTHA OM
 1135D..11361  ; XID_Start # Lo   [5] GRANTHA SIGN PLUTA..GRANTHA LETTER VOCALIC LL
+11380..11389  ; XID_Start # Lo  [10] TULU-TIGALARI LETTER A..TULU-TIGALARI LETTER VOCALIC LL
+1138B         ; XID_Start # Lo       TULU-TIGALARI LETTER EE
+1138E         ; XID_Start # Lo       TULU-TIGALARI LETTER AI
+11390..113B5  ; XID_Start # Lo  [38] TULU-TIGALARI LETTER OO..TULU-TIGALARI LETTER LLLA
+113B7         ; XID_Start # Lo       TULU-TIGALARI SIGN AVAGRAHA
+113D1         ; XID_Start # Lo       TULU-TIGALARI REPHA
+113D3         ; XID_Start # Lo       TULU-TIGALARI SIGN PLUTA
 11400..11434  ; XID_Start # Lo  [53] NEWA LETTER A..NEWA LETTER HA
 11447..1144A  ; XID_Start # Lo   [4] NEWA SIGN AVAGRAHA..NEWA SIDDHI
 1145F..11461  ; XID_Start # Lo   [3] NEWA LETTER VEDIC ANUSVARA..NEWA SIGN UPADHMANIYA
@@ -8827,6 +8994,7 @@ FFDA..FFDC    ; XID_Start # Lo   [3] HALFWIDTH HANGUL LETTER EU..HALFWIDTH HANGU
 11A5C..11A89  ; XID_Start # Lo  [46] SOYOMBO LETTER KA..SOYOMBO CLUSTER-INITIAL LETTER SA
 11A9D         ; XID_Start # Lo       SOYOMBO MARK PLUTA
 11AB0..11AF8  ; XID_Start # Lo  [73] CANADIAN SYLLABICS NATTILIK HI..PAU CIN HAU GLOTTAL STOP FINAL
+11BC0..11BE0  ; XID_Start # Lo  [33] SUNUWAR LETTER DEVI..SUNUWAR LETTER KLOKO
 11C00..11C08  ; XID_Start # Lo   [9] BHAIKSUKI LETTER A..BHAIKSUKI LETTER VOCALIC L
 11C0A..11C2E  ; XID_Start # Lo  [37] BHAIKSUKI LETTER E..BHAIKSUKI LETTER HA
 11C40         ; XID_Start # Lo       BHAIKSUKI SIGN AVAGRAHA
@@ -8850,7 +9018,9 @@ FFDA..FFDC    ; XID_Start # Lo   [3] HALFWIDTH HANGUL LETTER EU..HALFWIDTH HANGU
 12F90..12FF0  ; XID_Start # Lo  [97] CYPRO-MINOAN SIGN CM001..CYPRO-MINOAN SIGN CM114
 13000..1342F  ; XID_Start # Lo [1072] EGYPTIAN HIEROGLYPH A001..EGYPTIAN HIEROGLYPH V011D
 13441..13446  ; XID_Start # Lo   [6] EGYPTIAN HIEROGLYPH FULL BLANK..EGYPTIAN HIEROGLYPH WIDE LOST SIGN
+13460..143FA  ; XID_Start # Lo [3995] EGYPTIAN HIEROGLYPH-13460..EGYPTIAN HIEROGLYPH-143FA
 14400..14646  ; XID_Start # Lo [583] ANATOLIAN HIEROGLYPH A001..ANATOLIAN HIEROGLYPH A530
+16100..1611D  ; XID_Start # Lo  [30] GURUNG KHEMA LETTER A..GURUNG KHEMA LETTER SA
 16800..16A38  ; XID_Start # Lo [569] BAMUM LETTER PHASE-A NGKUE MFON..BAMUM LETTER PHASE-F VUEQ
 16A40..16A5E  ; XID_Start # Lo  [31] MRO LETTER TA..MRO LETTER TEK
 16A70..16ABE  ; XID_Start # Lo  [79] TANGSA LETTER OZ..TANGSA LETTER ZA
@@ -8859,6 +9029,9 @@ FFDA..FFDC    ; XID_Start # Lo   [3] HALFWIDTH HANGUL LETTER EU..HALFWIDTH HANGU
 16B40..16B43  ; XID_Start # Lm   [4] PAHAWH HMONG SIGN VOS SEEV..PAHAWH HMONG SIGN IB YAM
 16B63..16B77  ; XID_Start # Lo  [21] PAHAWH HMONG SIGN VOS LUB..PAHAWH HMONG SIGN CIM NRES TOS
 16B7D..16B8F  ; XID_Start # Lo  [19] PAHAWH HMONG CLAN SIGN TSHEEJ..PAHAWH HMONG CLAN SIGN VWJ
+16D40..16D42  ; XID_Start # Lm   [3] KIRAT RAI SIGN ANUSVARA..KIRAT RAI SIGN VISARGA
+16D43..16D6A  ; XID_Start # Lo  [40] KIRAT RAI LETTER A..KIRAT RAI VOWEL SIGN AU
+16D6B..16D6C  ; XID_Start # Lm   [2] KIRAT RAI SIGN VIRAMA..KIRAT RAI SIGN SAAT
 16E40..16E7F  ; XID_Start # L&  [64] MEDEFAIDRIN CAPITAL LETTER M..MEDEFAIDRIN SMALL LETTER Y
 16F00..16F4A  ; XID_Start # Lo  [75] MIAO LETTER PA..MIAO LETTER RTE
 16F50         ; XID_Start # Lo       MIAO LETTER NASALIZATION
@@ -8867,7 +9040,7 @@ FFDA..FFDC    ; XID_Start # Lo   [3] HALFWIDTH HANGUL LETTER EU..HALFWIDTH HANGU
 16FE3         ; XID_Start # Lm       OLD CHINESE ITERATION MARK
 17000..187F7  ; XID_Start # Lo [6136] TANGUT IDEOGRAPH-17000..TANGUT IDEOGRAPH-187F7
 18800..18CD5  ; XID_Start # Lo [1238] TANGUT COMPONENT-001..KHITAN SMALL SCRIPT CHARACTER-18CD5
-18D00..18D08  ; XID_Start # Lo   [9] TANGUT IDEOGRAPH-18D00..TANGUT IDEOGRAPH-18D08
+18CFF..18D08  ; XID_Start # Lo  [10] KHITAN SMALL SCRIPT CHARACTER-18CFF..TANGUT IDEOGRAPH-18D08
 1AFF0..1AFF3  ; XID_Start # Lm   [4] KATAKANA LETTER MINNAN TONE-2..KATAKANA LETTER MINNAN TONE-5
 1AFF5..1AFFB  ; XID_Start # Lm   [7] KATAKANA LETTER MINNAN TONE-7..KATAKANA LETTER MINNAN NASALIZED TONE-5
 1AFFD..1AFFE  ; XID_Start # Lm   [2] KATAKANA LETTER MINNAN NASALIZED TONE-7..KATAKANA LETTER MINNAN NASALIZED TONE-8
@@ -8923,6 +9096,8 @@ FFDA..FFDC    ; XID_Start # Lo   [3] HALFWIDTH HANGUL LETTER EU..HALFWIDTH HANGU
 1E2C0..1E2EB  ; XID_Start # Lo  [44] WANCHO LETTER AA..WANCHO LETTER YIH
 1E4D0..1E4EA  ; XID_Start # Lo  [27] NAG MUNDARI LETTER O..NAG MUNDARI LETTER ELL
 1E4EB         ; XID_Start # Lm       NAG MUNDARI SIGN OJOD
+1E5D0..1E5ED  ; XID_Start # Lo  [30] OL ONAL LETTER O..OL ONAL LETTER EG
+1E5F0         ; XID_Start # Lo       OL ONAL SIGN HODDOND
 1E7E0..1E7E6  ; XID_Start # Lo   [7] ETHIOPIC SYLLABLE HHYA..ETHIOPIC SYLLABLE HHYO
 1E7E8..1E7EB  ; XID_Start # Lo   [4] ETHIOPIC SYLLABLE GURAGE HHWA..ETHIOPIC SYLLABLE HHWE
 1E7ED..1E7EE  ; XID_Start # Lo   [2] ETHIOPIC SYLLABLE GURAGE MWI..ETHIOPIC SYLLABLE GURAGE MWEE
@@ -8973,7 +9148,7 @@ FFDA..FFDC    ; XID_Start # Lo   [3] HALFWIDTH HANGUL LETTER EU..HALFWIDTH HANGU
 30000..3134A  ; XID_Start # Lo [4939] CJK UNIFIED IDEOGRAPH-30000..CJK UNIFIED IDEOGRAPH-3134A
 31350..323AF  ; XID_Start # Lo [4192] CJK UNIFIED IDEOGRAPH-31350..CJK UNIFIED IDEOGRAPH-323AF
 
-# Total code points: 136944
+# Total code points: 141246
 
 # ================================================
 
@@ -9076,7 +9251,7 @@ FFDA..FFDC    ; XID_Start # Lo   [3] HALFWIDTH HANGUL LETTER EU..HALFWIDTH HANGU
 0860..086A    ; XID_Continue # Lo  [11] SYRIAC LETTER MALAYALAM NGA..SYRIAC LETTER MALAYALAM SSA
 0870..0887    ; XID_Continue # Lo  [24] ARABIC LETTER ALEF WITH ATTACHED FATHA..ARABIC BASELINE ROUND DOT
 0889..088E    ; XID_Continue # Lo   [6] ARABIC LETTER NOON WITH INVERTED SMALL V..ARABIC VERTICAL TAIL
-0898..089F    ; XID_Continue # Mn   [8] ARABIC SMALL HIGH WORD AL-JUZ..ARABIC HALF MADDA OVER MADDA
+0897..089F    ; XID_Continue # Mn   [9] ARABIC PEPET..ARABIC HALF MADDA OVER MADDA
 08A0..08C8    ; XID_Continue # Lo  [41] ARABIC LETTER BEH WITH SMALL V BELOW..ARABIC LETTER GRAF
 08C9          ; XID_Continue # Lm       ARABIC SMALL FARSI YEH
 08CA..08E1    ; XID_Continue # Mn  [24] ARABIC SMALL HIGH FARSI YEH..ARABIC SMALL HIGH SIGN SAFHA
@@ -9509,7 +9684,7 @@ FFDA..FFDC    ; XID_Start # Lo   [3] HALFWIDTH HANGUL LETTER EU..HALFWIDTH HANGU
 1C50..1C59    ; XID_Continue # Nd  [10] OL CHIKI DIGIT ZERO..OL CHIKI DIGIT NINE
 1C5A..1C77    ; XID_Continue # Lo  [30] OL CHIKI LETTER LA..OL CHIKI LETTER OH
 1C78..1C7D    ; XID_Continue # Lm   [6] OL CHIKI MU TTUDDAG..OL CHIKI AHAD
-1C80..1C88    ; XID_Continue # L&   [9] CYRILLIC SMALL LETTER ROUNDED VE..CYRILLIC SMALL LETTER UNBLENDED UK
+1C80..1C8A    ; XID_Continue # L&  [11] CYRILLIC SMALL LETTER ROUNDED VE..CYRILLIC SMALL LETTER TJE
 1C90..1CBA    ; XID_Continue # L&  [43] GEORGIAN MTAVRULI CAPITAL LETTER AN..GEORGIAN MTAVRULI CAPITAL LETTER AIN
 1CBD..1CBF    ; XID_Continue # L&   [3] GEORGIAN MTAVRULI CAPITAL LETTER AEN..GEORGIAN MTAVRULI CAPITAL LETTER LABIAL SIGN
 1CD0..1CD2    ; XID_Continue # Mn   [3] VEDIC TONE KARSHANA..VEDIC TONE PRENKHA
@@ -9652,10 +9827,10 @@ A771..A787    ; XID_Continue # L&  [23] LATIN SMALL LETTER DUM..LATIN SMALL LETT
 A788          ; XID_Continue # Lm       MODIFIER LETTER LOW CIRCUMFLEX ACCENT
 A78B..A78E    ; XID_Continue # L&   [4] LATIN CAPITAL LETTER SALTILLO..LATIN SMALL LETTER L WITH RETROFLEX HOOK AND BELT
 A78F          ; XID_Continue # Lo       LATIN LETTER SINOLOGICAL DOT
-A790..A7CA    ; XID_Continue # L&  [59] LATIN CAPITAL LETTER N WITH DESCENDER..LATIN SMALL LETTER S WITH SHORT STROKE OVERLAY
+A790..A7CD    ; XID_Continue # L&  [62] LATIN CAPITAL LETTER N WITH DESCENDER..LATIN SMALL LETTER S WITH DIAGONAL STROKE
 A7D0..A7D1    ; XID_Continue # L&   [2] LATIN CAPITAL LETTER CLOSED INSULAR G..LATIN SMALL LETTER CLOSED INSULAR G
 A7D3          ; XID_Continue # L&       LATIN SMALL LETTER DOUBLE THORN
-A7D5..A7D9    ; XID_Continue # L&   [5] LATIN SMALL LETTER DOUBLE WYNN..LATIN SMALL LETTER SIGMOID S
+A7D5..A7DC    ; XID_Continue # L&   [8] LATIN SMALL LETTER DOUBLE WYNN..LATIN CAPITAL LETTER LAMBDA WITH STROKE
 A7F2..A7F4    ; XID_Continue # Lm   [3] MODIFIER LETTER CAPITAL C..MODIFIER LETTER CAPITAL Q
 A7F5..A7F6    ; XID_Continue # L&   [2] LATIN CAPITAL LETTER REVERSED HALF H..LATIN SMALL LETTER REVERSED HALF H
 A7F7          ; XID_Continue # Lo       LATIN EPIGRAPHIC LETTER SIDEWAYS I
@@ -9850,6 +10025,7 @@ FFDA..FFDC    ; XID_Continue # Lo   [3] HALFWIDTH HANGUL LETTER EU..HALFWIDTH HA
 105A3..105B1  ; XID_Continue # L&  [15] VITHKUQI SMALL LETTER HA..VITHKUQI SMALL LETTER RE
 105B3..105B9  ; XID_Continue # L&   [7] VITHKUQI SMALL LETTER SE..VITHKUQI SMALL LETTER XE
 105BB..105BC  ; XID_Continue # L&   [2] VITHKUQI SMALL LETTER Y..VITHKUQI SMALL LETTER ZE
+105C0..105F3  ; XID_Continue # Lo  [52] TODHRI LETTER A..TODHRI LETTER OO
 10600..10736  ; XID_Continue # Lo [311] LINEAR A SIGN AB001..LINEAR A SIGN A664
 10740..10755  ; XID_Continue # Lo  [22] LINEAR A SIGN A701 A..LINEAR A SIGN A732 JE
 10760..10767  ; XID_Continue # Lo   [8] LINEAR A SIGN A800..LINEAR A SIGN A807
@@ -9894,10 +10070,19 @@ FFDA..FFDC    ; XID_Continue # Lo   [3] HALFWIDTH HANGUL LETTER EU..HALFWIDTH HA
 10D00..10D23  ; XID_Continue # Lo  [36] HANIFI ROHINGYA LETTER A..HANIFI ROHINGYA MARK NA KHONNA
 10D24..10D27  ; XID_Continue # Mn   [4] HANIFI ROHINGYA SIGN HARBAHAY..HANIFI ROHINGYA SIGN TASSI
 10D30..10D39  ; XID_Continue # Nd  [10] HANIFI ROHINGYA DIGIT ZERO..HANIFI ROHINGYA DIGIT NINE
+10D40..10D49  ; XID_Continue # Nd  [10] GARAY DIGIT ZERO..GARAY DIGIT NINE
+10D4A..10D4D  ; XID_Continue # Lo   [4] GARAY VOWEL SIGN A..GARAY VOWEL SIGN EE
+10D4E         ; XID_Continue # Lm       GARAY VOWEL LENGTH MARK
+10D4F         ; XID_Continue # Lo       GARAY SUKUN
+10D50..10D65  ; XID_Continue # L&  [22] GARAY CAPITAL LETTER A..GARAY CAPITAL LETTER OLD NA
+10D69..10D6D  ; XID_Continue # Mn   [5] GARAY VOWEL SIGN E..GARAY CONSONANT NASALIZATION MARK
+10D6F         ; XID_Continue # Lm       GARAY REDUPLICATION MARK
+10D70..10D85  ; XID_Continue # L&  [22] GARAY SMALL LETTER A..GARAY SMALL LETTER OLD NA
 10E80..10EA9  ; XID_Continue # Lo  [42] YEZIDI LETTER ELIF..YEZIDI LETTER ET
 10EAB..10EAC  ; XID_Continue # Mn   [2] YEZIDI COMBINING HAMZA MARK..YEZIDI COMBINING MADDA MARK
 10EB0..10EB1  ; XID_Continue # Lo   [2] YEZIDI LETTER LAM WITH DOT ABOVE..YEZIDI LETTER YOT WITH CIRCUMFLEX ABOVE
-10EFD..10EFF  ; XID_Continue # Mn   [3] ARABIC SMALL LOW WORD SAKTA..ARABIC SMALL LOW WORD MADDA
+10EC2..10EC4  ; XID_Continue # Lo   [3] ARABIC LETTER DAL WITH TWO DOTS VERTICALLY BELOW..ARABIC LETTER KAF WITH TWO DOTS VERTICALLY BELOW
+10EFC..10EFF  ; XID_Continue # Mn   [4] ARABIC COMBINING ALEF OVERLAY..ARABIC SMALL LOW WORD MADDA
 10F00..10F1C  ; XID_Continue # Lo  [29] OLD SOGDIAN LETTER ALEPH..OLD SOGDIAN LETTER FINAL TAW WITH VERTICAL TAIL
 10F27         ; XID_Continue # Lo       OLD SOGDIAN LIGATURE AYIN-DALETH
 10F30..10F45  ; XID_Continue # Lo  [22] SOGDIAN LETTER ALEPH..SOGDIAN INDEPENDENT SHIN
@@ -9993,6 +10178,24 @@ FFDA..FFDC    ; XID_Continue # Lo   [3] HALFWIDTH HANGUL LETTER EU..HALFWIDTH HA
 11362..11363  ; XID_Continue # Mc   [2] GRANTHA VOWEL SIGN VOCALIC L..GRANTHA VOWEL SIGN VOCALIC LL
 11366..1136C  ; XID_Continue # Mn   [7] COMBINING GRANTHA DIGIT ZERO..COMBINING GRANTHA DIGIT SIX
 11370..11374  ; XID_Continue # Mn   [5] COMBINING GRANTHA LETTER A..COMBINING GRANTHA LETTER PA
+11380..11389  ; XID_Continue # Lo  [10] TULU-TIGALARI LETTER A..TULU-TIGALARI LETTER VOCALIC LL
+1138B         ; XID_Continue # Lo       TULU-TIGALARI LETTER EE
+1138E         ; XID_Continue # Lo       TULU-TIGALARI LETTER AI
+11390..113B5  ; XID_Continue # Lo  [38] TULU-TIGALARI LETTER OO..TULU-TIGALARI LETTER LLLA
+113B7         ; XID_Continue # Lo       TULU-TIGALARI SIGN AVAGRAHA
+113B8..113BA  ; XID_Continue # Mc   [3] TULU-TIGALARI VOWEL SIGN AA..TULU-TIGALARI VOWEL SIGN II
+113BB..113C0  ; XID_Continue # Mn   [6] TULU-TIGALARI VOWEL SIGN U..TULU-TIGALARI VOWEL SIGN VOCALIC LL
+113C2         ; XID_Continue # Mc       TULU-TIGALARI VOWEL SIGN EE
+113C5         ; XID_Continue # Mc       TULU-TIGALARI VOWEL SIGN AI
+113C7..113CA  ; XID_Continue # Mc   [4] TULU-TIGALARI VOWEL SIGN OO..TULU-TIGALARI SIGN CANDRA ANUNASIKA
+113CC..113CD  ; XID_Continue # Mc   [2] TULU-TIGALARI SIGN ANUSVARA..TULU-TIGALARI SIGN VISARGA
+113CE         ; XID_Continue # Mn       TULU-TIGALARI SIGN VIRAMA
+113CF         ; XID_Continue # Mc       TULU-TIGALARI SIGN LOOPED VIRAMA
+113D0         ; XID_Continue # Mn       TULU-TIGALARI CONJOINER
+113D1         ; XID_Continue # Lo       TULU-TIGALARI REPHA
+113D2         ; XID_Continue # Mn       TULU-TIGALARI GEMINATION MARK
+113D3         ; XID_Continue # Lo       TULU-TIGALARI SIGN PLUTA
+113E1..113E2  ; XID_Continue # Mn   [2] TULU-TIGALARI VEDIC TONE SVARITA..TULU-TIGALARI VEDIC TONE ANUDATTA
 11400..11434  ; XID_Continue # Lo  [53] NEWA LETTER A..NEWA LETTER HA
 11435..11437  ; XID_Continue # Mc   [3] NEWA VOWEL SIGN AA..NEWA VOWEL SIGN II
 11438..1143F  ; XID_Continue # Mn   [8] NEWA VOWEL SIGN U..NEWA VOWEL SIGN AI
@@ -10044,8 +10247,11 @@ FFDA..FFDC    ; XID_Continue # Lo   [3] HALFWIDTH HANGUL LETTER EU..HALFWIDTH HA
 116B7         ; XID_Continue # Mn       TAKRI SIGN NUKTA
 116B8         ; XID_Continue # Lo       TAKRI LETTER ARCHAIC KHA
 116C0..116C9  ; XID_Continue # Nd  [10] TAKRI DIGIT ZERO..TAKRI DIGIT NINE
+116D0..116E3  ; XID_Continue # Nd  [20] MYANMAR PAO DIGIT ZERO..MYANMAR EASTERN PWO KAREN DIGIT NINE
 11700..1171A  ; XID_Continue # Lo  [27] AHOM LETTER KA..AHOM LETTER ALTERNATE BA
-1171D..1171F  ; XID_Continue # Mn   [3] AHOM CONSONANT SIGN MEDIAL LA..AHOM CONSONANT SIGN MEDIAL LIGATING RA
+1171D         ; XID_Continue # Mn       AHOM CONSONANT SIGN MEDIAL LA
+1171E         ; XID_Continue # Mc       AHOM CONSONANT SIGN MEDIAL RA
+1171F         ; XID_Continue # Mn       AHOM CONSONANT SIGN MEDIAL LIGATING RA
 11720..11721  ; XID_Continue # Mc   [2] AHOM VOWEL SIGN A..AHOM VOWEL SIGN AA
 11722..11725  ; XID_Continue # Mn   [4] AHOM VOWEL SIGN I..AHOM VOWEL SIGN UU
 11726         ; XID_Continue # Mc       AHOM VOWEL SIGN E
@@ -10103,6 +10309,8 @@ FFDA..FFDC    ; XID_Continue # Lo   [3] HALFWIDTH HANGUL LETTER EU..HALFWIDTH HA
 11A98..11A99  ; XID_Continue # Mn   [2] SOYOMBO GEMINATION MARK..SOYOMBO SUBJOINER
 11A9D         ; XID_Continue # Lo       SOYOMBO MARK PLUTA
 11AB0..11AF8  ; XID_Continue # Lo  [73] CANADIAN SYLLABICS NATTILIK HI..PAU CIN HAU GLOTTAL STOP FINAL
+11BC0..11BE0  ; XID_Continue # Lo  [33] SUNUWAR LETTER DEVI..SUNUWAR LETTER KLOKO
+11BF0..11BF9  ; XID_Continue # Nd  [10] SUNUWAR DIGIT ZERO..SUNUWAR DIGIT NINE
 11C00..11C08  ; XID_Continue # Lo   [9] BHAIKSUKI LETTER A..BHAIKSUKI LETTER VOCALIC L
 11C0A..11C2E  ; XID_Continue # Lo  [37] BHAIKSUKI LETTER E..BHAIKSUKI LETTER HA
 11C2F         ; XID_Continue # Mc       BHAIKSUKI VOWEL SIGN AA
@@ -10156,6 +10364,7 @@ FFDA..FFDC    ; XID_Continue # Lo   [3] HALFWIDTH HANGUL LETTER EU..HALFWIDTH HA
 11F41         ; XID_Continue # Mc       KAWI SIGN KILLER
 11F42         ; XID_Continue # Mn       KAWI CONJOINER
 11F50..11F59  ; XID_Continue # Nd  [10] KAWI DIGIT ZERO..KAWI DIGIT NINE
+11F5A         ; XID_Continue # Mn       KAWI SIGN NUKTA
 11FB0         ; XID_Continue # Lo       LISU LETTER YHA
 12000..12399  ; XID_Continue # Lo [922] CUNEIFORM SIGN A..CUNEIFORM SIGN U U
 12400..1246E  ; XID_Continue # Nl [111] CUNEIFORM NUMERIC SIGN TWO ASH..CUNEIFORM NUMERIC SIGN NINE U VARIANT FORM
@@ -10165,7 +10374,13 @@ FFDA..FFDC    ; XID_Continue # Lo   [3] HALFWIDTH HANGUL LETTER EU..HALFWIDTH HA
 13440         ; XID_Continue # Mn       EGYPTIAN HIEROGLYPH MIRROR HORIZONTALLY
 13441..13446  ; XID_Continue # Lo   [6] EGYPTIAN HIEROGLYPH FULL BLANK..EGYPTIAN HIEROGLYPH WIDE LOST SIGN
 13447..13455  ; XID_Continue # Mn  [15] EGYPTIAN HIEROGLYPH MODIFIER DAMAGED AT TOP START..EGYPTIAN HIEROGLYPH MODIFIER DAMAGED
+13460..143FA  ; XID_Continue # Lo [3995] EGYPTIAN HIEROGLYPH-13460..EGYPTIAN HIEROGLYPH-143FA
 14400..14646  ; XID_Continue # Lo [583] ANATOLIAN HIEROGLYPH A001..ANATOLIAN HIEROGLYPH A530
+16100..1611D  ; XID_Continue # Lo  [30] GURUNG KHEMA LETTER A..GURUNG KHEMA LETTER SA
+1611E..16129  ; XID_Continue # Mn  [12] GURUNG KHEMA VOWEL SIGN AA..GURUNG KHEMA VOWEL LENGTH MARK
+1612A..1612C  ; XID_Continue # Mc   [3] GURUNG KHEMA CONSONANT SIGN MEDIAL YA..GURUNG KHEMA CONSONANT SIGN MEDIAL HA
+1612D..1612F  ; XID_Continue # Mn   [3] GURUNG KHEMA SIGN ANUSVARA..GURUNG KHEMA SIGN THOLHOMA
+16130..16139  ; XID_Continue # Nd  [10] GURUNG KHEMA DIGIT ZERO..GURUNG KHEMA DIGIT NINE
 16800..16A38  ; XID_Continue # Lo [569] BAMUM LETTER PHASE-A NGKUE MFON..BAMUM LETTER PHASE-F VUEQ
 16A40..16A5E  ; XID_Continue # Lo  [31] MRO LETTER TA..MRO LETTER TEK
 16A60..16A69  ; XID_Continue # Nd  [10] MRO DIGIT ZERO..MRO DIGIT NINE
@@ -10179,6 +10394,10 @@ FFDA..FFDC    ; XID_Continue # Lo   [3] HALFWIDTH HANGUL LETTER EU..HALFWIDTH HA
 16B50..16B59  ; XID_Continue # Nd  [10] PAHAWH HMONG DIGIT ZERO..PAHAWH HMONG DIGIT NINE
 16B63..16B77  ; XID_Continue # Lo  [21] PAHAWH HMONG SIGN VOS LUB..PAHAWH HMONG SIGN CIM NRES TOS
 16B7D..16B8F  ; XID_Continue # Lo  [19] PAHAWH HMONG CLAN SIGN TSHEEJ..PAHAWH HMONG CLAN SIGN VWJ
+16D40..16D42  ; XID_Continue # Lm   [3] KIRAT RAI SIGN ANUSVARA..KIRAT RAI SIGN VISARGA
+16D43..16D6A  ; XID_Continue # Lo  [40] KIRAT RAI LETTER A..KIRAT RAI VOWEL SIGN AU
+16D6B..16D6C  ; XID_Continue # Lm   [2] KIRAT RAI SIGN VIRAMA..KIRAT RAI SIGN SAAT
+16D70..16D79  ; XID_Continue # Nd  [10] KIRAT RAI DIGIT ZERO..KIRAT RAI DIGIT NINE
 16E40..16E7F  ; XID_Continue # L&  [64] MEDEFAIDRIN CAPITAL LETTER M..MEDEFAIDRIN SMALL LETTER Y
 16F00..16F4A  ; XID_Continue # Lo  [75] MIAO LETTER PA..MIAO LETTER RTE
 16F4F         ; XID_Continue # Mn       MIAO SIGN CONSONANT MODIFIER BAR
@@ -10192,7 +10411,7 @@ FFDA..FFDC    ; XID_Continue # Lo   [3] HALFWIDTH HANGUL LETTER EU..HALFWIDTH HA
 16FF0..16FF1  ; XID_Continue # Mc   [2] VIETNAMESE ALTERNATE READING MARK CA..VIETNAMESE ALTERNATE READING MARK NHAY
 17000..187F7  ; XID_Continue # Lo [6136] TANGUT IDEOGRAPH-17000..TANGUT IDEOGRAPH-187F7
 18800..18CD5  ; XID_Continue # Lo [1238] TANGUT COMPONENT-001..KHITAN SMALL SCRIPT CHARACTER-18CD5
-18D00..18D08  ; XID_Continue # Lo   [9] TANGUT IDEOGRAPH-18D00..TANGUT IDEOGRAPH-18D08
+18CFF..18D08  ; XID_Continue # Lo  [10] KHITAN SMALL SCRIPT CHARACTER-18CFF..TANGUT IDEOGRAPH-18D08
 1AFF0..1AFF3  ; XID_Continue # Lm   [4] KATAKANA LETTER MINNAN TONE-2..KATAKANA LETTER MINNAN TONE-5
 1AFF5..1AFFB  ; XID_Continue # Lm   [7] KATAKANA LETTER MINNAN TONE-7..KATAKANA LETTER MINNAN NASALIZED TONE-5
 1AFFD..1AFFE  ; XID_Continue # Lm   [2] KATAKANA LETTER MINNAN NASALIZED TONE-7..KATAKANA LETTER MINNAN NASALIZED TONE-8
@@ -10207,6 +10426,7 @@ FFDA..FFDC    ; XID_Continue # Lo   [3] HALFWIDTH HANGUL LETTER EU..HALFWIDTH HA
 1BC80..1BC88  ; XID_Continue # Lo   [9] DUPLOYAN AFFIX HIGH ACUTE..DUPLOYAN AFFIX HIGH VERTICAL
 1BC90..1BC99  ; XID_Continue # Lo  [10] DUPLOYAN AFFIX LOW ACUTE..DUPLOYAN AFFIX LOW ARROW
 1BC9D..1BC9E  ; XID_Continue # Mn   [2] DUPLOYAN THICK LETTER SELECTOR..DUPLOYAN DOUBLE MARK
+1CCF0..1CCF9  ; XID_Continue # Nd  [10] OUTLINED DIGIT ZERO..OUTLINED DIGIT NINE
 1CF00..1CF2D  ; XID_Continue # Mn  [46] ZNAMENNY COMBINING MARK GORAZDO NIZKO S KRYZHEM ON LEFT..ZNAMENNY COMBINING MARK KRYZH ON LEFT
 1CF30..1CF46  ; XID_Continue # Mn  [23] ZNAMENNY COMBINING TONAL RANGE MARK MRACHNO..ZNAMENNY PRIZNAK MODIFIER ROG
 1D165..1D166  ; XID_Continue # Mc   [2] MUSICAL SYMBOL COMBINING STEM..MUSICAL SYMBOL COMBINING SPRECHGESANG STEM
@@ -10278,6 +10498,10 @@ FFDA..FFDC    ; XID_Continue # Lo   [3] HALFWIDTH HANGUL LETTER EU..HALFWIDTH HA
 1E4EB         ; XID_Continue # Lm       NAG MUNDARI SIGN OJOD
 1E4EC..1E4EF  ; XID_Continue # Mn   [4] NAG MUNDARI SIGN MUHOR..NAG MUNDARI SIGN SUTUH
 1E4F0..1E4F9  ; XID_Continue # Nd  [10] NAG MUNDARI DIGIT ZERO..NAG MUNDARI DIGIT NINE
+1E5D0..1E5ED  ; XID_Continue # Lo  [30] OL ONAL LETTER O..OL ONAL LETTER EG
+1E5EE..1E5EF  ; XID_Continue # Mn   [2] OL ONAL SIGN MU..OL ONAL SIGN IKIR
+1E5F0         ; XID_Continue # Lo       OL ONAL SIGN HODDOND
+1E5F1..1E5FA  ; XID_Continue # Nd  [10] OL ONAL DIGIT ZERO..OL ONAL DIGIT NINE
 1E7E0..1E7E6  ; XID_Continue # Lo   [7] ETHIOPIC SYLLABLE HHYA..ETHIOPIC SYLLABLE HHYO
 1E7E8..1E7EB  ; XID_Continue # Lo   [4] ETHIOPIC SYLLABLE GURAGE HHWA..ETHIOPIC SYLLABLE HHWE
 1E7ED..1E7EE  ; XID_Continue # Lo   [2] ETHIOPIC SYLLABLE GURAGE MWI..ETHIOPIC SYLLABLE GURAGE MWEE
@@ -10333,7 +10557,7 @@ FFDA..FFDC    ; XID_Continue # Lo   [3] HALFWIDTH HANGUL LETTER EU..HALFWIDTH HA
 31350..323AF  ; XID_Continue # Lo [4192] CJK UNIFIED IDEOGRAPH-31350..CJK UNIFIED IDEOGRAPH-323AF
 E0100..E01EF  ; XID_Continue # Mn [240] VARIATION SELECTOR-17..VARIATION SELECTOR-256
 
-# Total code points: 140089
+# Total code points: 144522
 
 # ================================================
 
@@ -10418,7 +10642,7 @@ E01F0..E0FFF  ; Default_Ignorable_Code_Point # Cn [3600] <reserved-E01F0>..<rese
 0825..0827    ; Grapheme_Extend # Mn   [3] SAMARITAN VOWEL SIGN SHORT A..SAMARITAN VOWEL SIGN U
 0829..082D    ; Grapheme_Extend # Mn   [5] SAMARITAN VOWEL SIGN LONG I..SAMARITAN MARK NEQUDAA
 0859..085B    ; Grapheme_Extend # Mn   [3] MANDAIC AFFRICATION MARK..MANDAIC GEMINATION MARK
-0898..089F    ; Grapheme_Extend # Mn   [8] ARABIC SMALL HIGH WORD AL-JUZ..ARABIC HALF MADDA OVER MADDA
+0897..089F    ; Grapheme_Extend # Mn   [9] ARABIC PEPET..ARABIC HALF MADDA OVER MADDA
 08CA..08E1    ; Grapheme_Extend # Mn  [24] ARABIC SMALL HIGH FARSI YEH..ARABIC SMALL HIGH SIGN SAFHA
 08E3..0902    ; Grapheme_Extend # Mn  [32] ARABIC TURNED DAMMA BELOW..DEVANAGARI SIGN ANUSVARA
 093A          ; Grapheme_Extend # Mn       DEVANAGARI VOWEL SIGN OE
@@ -10475,8 +10699,11 @@ E01F0..E0FFF  ; Default_Ignorable_Code_Point # Cn [3600] <reserved-E01F0>..<rese
 0C81          ; Grapheme_Extend # Mn       KANNADA SIGN CANDRABINDU
 0CBC          ; Grapheme_Extend # Mn       KANNADA SIGN NUKTA
 0CBF          ; Grapheme_Extend # Mn       KANNADA VOWEL SIGN I
+0CC0          ; Grapheme_Extend # Mc       KANNADA VOWEL SIGN II
 0CC2          ; Grapheme_Extend # Mc       KANNADA VOWEL SIGN UU
 0CC6          ; Grapheme_Extend # Mn       KANNADA VOWEL SIGN E
+0CC7..0CC8    ; Grapheme_Extend # Mc   [2] KANNADA VOWEL SIGN EE..KANNADA VOWEL SIGN AI
+0CCA..0CCB    ; Grapheme_Extend # Mc   [2] KANNADA VOWEL SIGN O..KANNADA VOWEL SIGN OO
 0CCC..0CCD    ; Grapheme_Extend # Mn   [2] KANNADA VOWEL SIGN AU..KANNADA SIGN VIRAMA
 0CD5..0CD6    ; Grapheme_Extend # Mc   [2] KANNADA LENGTH MARK..KANNADA AI LENGTH MARK
 0CE2..0CE3    ; Grapheme_Extend # Mn   [2] KANNADA VOWEL SIGN VOCALIC L..KANNADA VOWEL SIGN VOCALIC LL
@@ -10522,7 +10749,9 @@ E01F0..E0FFF  ; Default_Ignorable_Code_Point # Cn [3600] <reserved-E01F0>..<rese
 109D          ; Grapheme_Extend # Mn       MYANMAR VOWEL SIGN AITON AI
 135D..135F    ; Grapheme_Extend # Mn   [3] ETHIOPIC COMBINING GEMINATION AND VOWEL LENGTH MARK..ETHIOPIC COMBINING GEMINATION MARK
 1712..1714    ; Grapheme_Extend # Mn   [3] TAGALOG VOWEL SIGN I..TAGALOG SIGN VIRAMA
+1715          ; Grapheme_Extend # Mc       TAGALOG SIGN PAMUDPOD
 1732..1733    ; Grapheme_Extend # Mn   [2] HANUNOO VOWEL SIGN I..HANUNOO VOWEL SIGN U
+1734          ; Grapheme_Extend # Mc       HANUNOO SIGN PAMUDPOD
 1752..1753    ; Grapheme_Extend # Mn   [2] BUHID VOWEL SIGN I..BUHID VOWEL SIGN U
 1772..1773    ; Grapheme_Extend # Mn   [2] TAGBANWA VOWEL SIGN I..TAGBANWA VOWEL SIGN U
 17B4..17B5    ; Grapheme_Extend # Mn   [2] KHMER VOWEL INHERENT AQ..KHMER VOWEL INHERENT AA
@@ -10554,17 +10783,22 @@ E01F0..E0FFF  ; Default_Ignorable_Code_Point # Cn [3600] <reserved-E01F0>..<rese
 1B34          ; Grapheme_Extend # Mn       BALINESE SIGN REREKAN
 1B35          ; Grapheme_Extend # Mc       BALINESE VOWEL SIGN TEDUNG
 1B36..1B3A    ; Grapheme_Extend # Mn   [5] BALINESE VOWEL SIGN ULU..BALINESE VOWEL SIGN RA REPA
+1B3B          ; Grapheme_Extend # Mc       BALINESE VOWEL SIGN RA REPA TEDUNG
 1B3C          ; Grapheme_Extend # Mn       BALINESE VOWEL SIGN LA LENGA
+1B3D          ; Grapheme_Extend # Mc       BALINESE VOWEL SIGN LA LENGA TEDUNG
 1B42          ; Grapheme_Extend # Mn       BALINESE VOWEL SIGN PEPET
+1B43..1B44    ; Grapheme_Extend # Mc   [2] BALINESE VOWEL SIGN PEPET TEDUNG..BALINESE ADEG ADEG
 1B6B..1B73    ; Grapheme_Extend # Mn   [9] BALINESE MUSICAL SYMBOL COMBINING TEGEH..BALINESE MUSICAL SYMBOL COMBINING GONG
 1B80..1B81    ; Grapheme_Extend # Mn   [2] SUNDANESE SIGN PANYECEK..SUNDANESE SIGN PANGLAYAR
 1BA2..1BA5    ; Grapheme_Extend # Mn   [4] SUNDANESE CONSONANT SIGN PANYAKRA..SUNDANESE VOWEL SIGN PANYUKU
 1BA8..1BA9    ; Grapheme_Extend # Mn   [2] SUNDANESE VOWEL SIGN PAMEPET..SUNDANESE VOWEL SIGN PANEULEUNG
+1BAA          ; Grapheme_Extend # Mc       SUNDANESE SIGN PAMAAEH
 1BAB..1BAD    ; Grapheme_Extend # Mn   [3] SUNDANESE SIGN VIRAMA..SUNDANESE CONSONANT SIGN PASANGAN WA
 1BE6          ; Grapheme_Extend # Mn       BATAK SIGN TOMPI
 1BE8..1BE9    ; Grapheme_Extend # Mn   [2] BATAK VOWEL SIGN PAKPAK E..BATAK VOWEL SIGN EE
 1BED          ; Grapheme_Extend # Mn       BATAK VOWEL SIGN KARO O
 1BEF..1BF1    ; Grapheme_Extend # Mn   [3] BATAK VOWEL SIGN U FOR SIMALUNGUN SA..BATAK CONSONANT SIGN H
+1BF2..1BF3    ; Grapheme_Extend # Mc   [2] BATAK PANGOLAT..BATAK PANONGONAN
 1C2C..1C33    ; Grapheme_Extend # Mn   [8] LEPCHA VOWEL SIGN E..LEPCHA CONSONANT SIGN T
 1C36..1C37    ; Grapheme_Extend # Mn   [2] LEPCHA SIGN RAN..LEPCHA SIGN NUKTA
 1CD0..1CD2    ; Grapheme_Extend # Mn   [3] VEDIC TONE KARSHANA..VEDIC TONE PRENKHA
@@ -10601,10 +10835,12 @@ A8E0..A8F1    ; Grapheme_Extend # Mn  [18] COMBINING DEVANAGARI DIGIT ZERO..COMB
 A8FF          ; Grapheme_Extend # Mn       DEVANAGARI VOWEL SIGN AY
 A926..A92D    ; Grapheme_Extend # Mn   [8] KAYAH LI VOWEL UE..KAYAH LI TONE CALYA PLOPHU
 A947..A951    ; Grapheme_Extend # Mn  [11] REJANG VOWEL SIGN I..REJANG CONSONANT SIGN R
+A953          ; Grapheme_Extend # Mc       REJANG VIRAMA
 A980..A982    ; Grapheme_Extend # Mn   [3] JAVANESE SIGN PANYANGGA..JAVANESE SIGN LAYAR
 A9B3          ; Grapheme_Extend # Mn       JAVANESE SIGN CECAK TELU
 A9B6..A9B9    ; Grapheme_Extend # Mn   [4] JAVANESE VOWEL SIGN WULU..JAVANESE VOWEL SIGN SUKU MENDUT
 A9BC..A9BD    ; Grapheme_Extend # Mn   [2] JAVANESE VOWEL SIGN PEPET..JAVANESE CONSONANT SIGN KERET
+A9C0          ; Grapheme_Extend # Mc       JAVANESE PANGKON
 A9E5          ; Grapheme_Extend # Mn       MYANMAR SIGN SHAN SAW
 AA29..AA2E    ; Grapheme_Extend # Mn   [6] CHAM VOWEL SIGN AA..CHAM VOWEL SIGN OE
 AA31..AA32    ; Grapheme_Extend # Mn   [2] CHAM VOWEL SIGN AU..CHAM VOWEL SIGN UE
@@ -10636,8 +10872,9 @@ FF9E..FF9F    ; Grapheme_Extend # Lm   [2] HALFWIDTH KATAKANA VOICED SOUND MARK.
 10A3F         ; Grapheme_Extend # Mn       KHAROSHTHI VIRAMA
 10AE5..10AE6  ; Grapheme_Extend # Mn   [2] MANICHAEAN ABBREVIATION MARK ABOVE..MANICHAEAN ABBREVIATION MARK BELOW
 10D24..10D27  ; Grapheme_Extend # Mn   [4] HANIFI ROHINGYA SIGN HARBAHAY..HANIFI ROHINGYA SIGN TASSI
+10D69..10D6D  ; Grapheme_Extend # Mn   [5] GARAY VOWEL SIGN E..GARAY CONSONANT NASALIZATION MARK
 10EAB..10EAC  ; Grapheme_Extend # Mn   [2] YEZIDI COMBINING HAMZA MARK..YEZIDI COMBINING MADDA MARK
-10EFD..10EFF  ; Grapheme_Extend # Mn   [3] ARABIC SMALL LOW WORD SAKTA..ARABIC SMALL LOW WORD MADDA
+10EFC..10EFF  ; Grapheme_Extend # Mn   [4] ARABIC COMBINING ALEF OVERLAY..ARABIC SMALL LOW WORD MADDA
 10F46..10F50  ; Grapheme_Extend # Mn  [11] SOGDIAN COMBINING DOT BELOW..SOGDIAN COMBINING STROKE BELOW
 10F82..10F85  ; Grapheme_Extend # Mn   [4] OLD UYGHUR COMBINING DOT ABOVE..OLD UYGHUR COMBINING TWO DOTS BELOW
 11001         ; Grapheme_Extend # Mn       BRAHMI SIGN ANUSVARA
@@ -10654,10 +10891,12 @@ FF9E..FF9F    ; Grapheme_Extend # Lm   [2] HALFWIDTH KATAKANA VOICED SOUND MARK.
 11173         ; Grapheme_Extend # Mn       MAHAJANI SIGN NUKTA
 11180..11181  ; Grapheme_Extend # Mn   [2] SHARADA SIGN CANDRABINDU..SHARADA SIGN ANUSVARA
 111B6..111BE  ; Grapheme_Extend # Mn   [9] SHARADA VOWEL SIGN U..SHARADA VOWEL SIGN O
+111C0         ; Grapheme_Extend # Mc       SHARADA SIGN VIRAMA
 111C9..111CC  ; Grapheme_Extend # Mn   [4] SHARADA SANDHI MARK..SHARADA EXTRA SHORT VOWEL MARK
 111CF         ; Grapheme_Extend # Mn       SHARADA SIGN INVERTED CANDRABINDU
 1122F..11231  ; Grapheme_Extend # Mn   [3] KHOJKI VOWEL SIGN U..KHOJKI VOWEL SIGN AI
 11234         ; Grapheme_Extend # Mn       KHOJKI SIGN ANUSVARA
+11235         ; Grapheme_Extend # Mc       KHOJKI SIGN VIRAMA
 11236..11237  ; Grapheme_Extend # Mn   [2] KHOJKI SIGN NUKTA..KHOJKI SIGN SHADDA
 1123E         ; Grapheme_Extend # Mn       KHOJKI SIGN SUKUN
 11241         ; Grapheme_Extend # Mn       KHOJKI VOWEL SIGN VOCALIC R
@@ -10667,9 +10906,20 @@ FF9E..FF9F    ; Grapheme_Extend # Lm   [2] HALFWIDTH KATAKANA VOICED SOUND MARK.
 1133B..1133C  ; Grapheme_Extend # Mn   [2] COMBINING BINDU BELOW..GRANTHA SIGN NUKTA
 1133E         ; Grapheme_Extend # Mc       GRANTHA VOWEL SIGN AA
 11340         ; Grapheme_Extend # Mn       GRANTHA VOWEL SIGN II
+1134D         ; Grapheme_Extend # Mc       GRANTHA SIGN VIRAMA
 11357         ; Grapheme_Extend # Mc       GRANTHA AU LENGTH MARK
 11366..1136C  ; Grapheme_Extend # Mn   [7] COMBINING GRANTHA DIGIT ZERO..COMBINING GRANTHA DIGIT SIX
 11370..11374  ; Grapheme_Extend # Mn   [5] COMBINING GRANTHA LETTER A..COMBINING GRANTHA LETTER PA
+113B8         ; Grapheme_Extend # Mc       TULU-TIGALARI VOWEL SIGN AA
+113BB..113C0  ; Grapheme_Extend # Mn   [6] TULU-TIGALARI VOWEL SIGN U..TULU-TIGALARI VOWEL SIGN VOCALIC LL
+113C2         ; Grapheme_Extend # Mc       TULU-TIGALARI VOWEL SIGN EE
+113C5         ; Grapheme_Extend # Mc       TULU-TIGALARI VOWEL SIGN AI
+113C7..113C9  ; Grapheme_Extend # Mc   [3] TULU-TIGALARI VOWEL SIGN OO..TULU-TIGALARI AU LENGTH MARK
+113CE         ; Grapheme_Extend # Mn       TULU-TIGALARI SIGN VIRAMA
+113CF         ; Grapheme_Extend # Mc       TULU-TIGALARI SIGN LOOPED VIRAMA
+113D0         ; Grapheme_Extend # Mn       TULU-TIGALARI CONJOINER
+113D2         ; Grapheme_Extend # Mn       TULU-TIGALARI GEMINATION MARK
+113E1..113E2  ; Grapheme_Extend # Mn   [2] TULU-TIGALARI VEDIC TONE SVARITA..TULU-TIGALARI VEDIC TONE ANUDATTA
 11438..1143F  ; Grapheme_Extend # Mn   [8] NEWA VOWEL SIGN U..NEWA VOWEL SIGN AI
 11442..11444  ; Grapheme_Extend # Mn   [3] NEWA SIGN VIRAMA..NEWA SIGN ANUSVARA
 11446         ; Grapheme_Extend # Mn       NEWA SIGN NUKTA
@@ -10691,14 +10941,17 @@ FF9E..FF9F    ; Grapheme_Extend # Lm   [2] HALFWIDTH KATAKANA VOICED SOUND MARK.
 116AB         ; Grapheme_Extend # Mn       TAKRI SIGN ANUSVARA
 116AD         ; Grapheme_Extend # Mn       TAKRI VOWEL SIGN AA
 116B0..116B5  ; Grapheme_Extend # Mn   [6] TAKRI VOWEL SIGN U..TAKRI VOWEL SIGN AU
+116B6         ; Grapheme_Extend # Mc       TAKRI SIGN VIRAMA
 116B7         ; Grapheme_Extend # Mn       TAKRI SIGN NUKTA
-1171D..1171F  ; Grapheme_Extend # Mn   [3] AHOM CONSONANT SIGN MEDIAL LA..AHOM CONSONANT SIGN MEDIAL LIGATING RA
+1171D         ; Grapheme_Extend # Mn       AHOM CONSONANT SIGN MEDIAL LA
+1171F         ; Grapheme_Extend # Mn       AHOM CONSONANT SIGN MEDIAL LIGATING RA
 11722..11725  ; Grapheme_Extend # Mn   [4] AHOM VOWEL SIGN I..AHOM VOWEL SIGN UU
 11727..1172B  ; Grapheme_Extend # Mn   [5] AHOM VOWEL SIGN AW..AHOM SIGN KILLER
 1182F..11837  ; Grapheme_Extend # Mn   [9] DOGRA VOWEL SIGN U..DOGRA SIGN ANUSVARA
 11839..1183A  ; Grapheme_Extend # Mn   [2] DOGRA SIGN VIRAMA..DOGRA SIGN NUKTA
 11930         ; Grapheme_Extend # Mc       DIVES AKURU VOWEL SIGN AA
 1193B..1193C  ; Grapheme_Extend # Mn   [2] DIVES AKURU SIGN ANUSVARA..DIVES AKURU SIGN CANDRABINDU
+1193D         ; Grapheme_Extend # Mc       DIVES AKURU SIGN HALANTA
 1193E         ; Grapheme_Extend # Mn       DIVES AKURU VIRAMA
 11943         ; Grapheme_Extend # Mn       DIVES AKURU SIGN NUKTA
 119D4..119D7  ; Grapheme_Extend # Mn   [4] NANDINAGARI VOWEL SIGN U..NANDINAGARI VOWEL SIGN VOCALIC RR
@@ -10731,20 +10984,25 @@ FF9E..FF9F    ; Grapheme_Extend # Lm   [2] HALFWIDTH KATAKANA VOICED SOUND MARK.
 11F00..11F01  ; Grapheme_Extend # Mn   [2] KAWI SIGN CANDRABINDU..KAWI SIGN ANUSVARA
 11F36..11F3A  ; Grapheme_Extend # Mn   [5] KAWI VOWEL SIGN I..KAWI VOWEL SIGN VOCALIC R
 11F40         ; Grapheme_Extend # Mn       KAWI VOWEL SIGN EU
+11F41         ; Grapheme_Extend # Mc       KAWI SIGN KILLER
 11F42         ; Grapheme_Extend # Mn       KAWI CONJOINER
+11F5A         ; Grapheme_Extend # Mn       KAWI SIGN NUKTA
 13440         ; Grapheme_Extend # Mn       EGYPTIAN HIEROGLYPH MIRROR HORIZONTALLY
 13447..13455  ; Grapheme_Extend # Mn  [15] EGYPTIAN HIEROGLYPH MODIFIER DAMAGED AT TOP START..EGYPTIAN HIEROGLYPH MODIFIER DAMAGED
+1611E..16129  ; Grapheme_Extend # Mn  [12] GURUNG KHEMA VOWEL SIGN AA..GURUNG KHEMA VOWEL LENGTH MARK
+1612D..1612F  ; Grapheme_Extend # Mn   [3] GURUNG KHEMA SIGN ANUSVARA..GURUNG KHEMA SIGN THOLHOMA
 16AF0..16AF4  ; Grapheme_Extend # Mn   [5] BASSA VAH COMBINING HIGH TONE..BASSA VAH COMBINING HIGH-LOW TONE
 16B30..16B36  ; Grapheme_Extend # Mn   [7] PAHAWH HMONG MARK CIM TUB..PAHAWH HMONG MARK CIM TAUM
 16F4F         ; Grapheme_Extend # Mn       MIAO SIGN CONSONANT MODIFIER BAR
 16F8F..16F92  ; Grapheme_Extend # Mn   [4] MIAO TONE RIGHT..MIAO TONE BELOW
 16FE4         ; Grapheme_Extend # Mn       KHITAN SMALL SCRIPT FILLER
+16FF0..16FF1  ; Grapheme_Extend # Mc   [2] VIETNAMESE ALTERNATE READING MARK CA..VIETNAMESE ALTERNATE READING MARK NHAY
 1BC9D..1BC9E  ; Grapheme_Extend # Mn   [2] DUPLOYAN THICK LETTER SELECTOR..DUPLOYAN DOUBLE MARK
 1CF00..1CF2D  ; Grapheme_Extend # Mn  [46] ZNAMENNY COMBINING MARK GORAZDO NIZKO S KRYZHEM ON LEFT..ZNAMENNY COMBINING MARK KRYZH ON LEFT
 1CF30..1CF46  ; Grapheme_Extend # Mn  [23] ZNAMENNY COMBINING TONAL RANGE MARK MRACHNO..ZNAMENNY PRIZNAK MODIFIER ROG
-1D165         ; Grapheme_Extend # Mc       MUSICAL SYMBOL COMBINING STEM
+1D165..1D166  ; Grapheme_Extend # Mc   [2] MUSICAL SYMBOL COMBINING STEM..MUSICAL SYMBOL COMBINING SPRECHGESANG STEM
 1D167..1D169  ; Grapheme_Extend # Mn   [3] MUSICAL SYMBOL COMBINING TREMOLO-1..MUSICAL SYMBOL COMBINING TREMOLO-3
-1D16E..1D172  ; Grapheme_Extend # Mc   [5] MUSICAL SYMBOL COMBINING FLAG-1..MUSICAL SYMBOL COMBINING FLAG-5
+1D16D..1D172  ; Grapheme_Extend # Mc   [6] MUSICAL SYMBOL COMBINING AUGMENTATION DOT..MUSICAL SYMBOL COMBINING FLAG-5
 1D17B..1D182  ; Grapheme_Extend # Mn   [8] MUSICAL SYMBOL COMBINING ACCENT..MUSICAL SYMBOL COMBINING LOURE
 1D185..1D18B  ; Grapheme_Extend # Mn   [7] MUSICAL SYMBOL COMBINING DOIT..MUSICAL SYMBOL COMBINING TRIPLE TONGUE
 1D1AA..1D1AD  ; Grapheme_Extend # Mn   [4] MUSICAL SYMBOL COMBINING DOWN BOW..MUSICAL SYMBOL COMBINING SNAP PIZZICATO
@@ -10765,12 +11023,13 @@ FF9E..FF9F    ; Grapheme_Extend # Lm   [2] HALFWIDTH KATAKANA VOICED SOUND MARK.
 1E2AE         ; Grapheme_Extend # Mn       TOTO SIGN RISING TONE
 1E2EC..1E2EF  ; Grapheme_Extend # Mn   [4] WANCHO TONE TUP..WANCHO TONE KOINI
 1E4EC..1E4EF  ; Grapheme_Extend # Mn   [4] NAG MUNDARI SIGN MUHOR..NAG MUNDARI SIGN SUTUH
+1E5EE..1E5EF  ; Grapheme_Extend # Mn   [2] OL ONAL SIGN MU..OL ONAL SIGN IKIR
 1E8D0..1E8D6  ; Grapheme_Extend # Mn   [7] MENDE KIKAKUI COMBINING NUMBER TEENS..MENDE KIKAKUI COMBINING NUMBER MILLIONS
 1E944..1E94A  ; Grapheme_Extend # Mn   [7] ADLAM ALIF LENGTHENER..ADLAM NUKTA
 E0020..E007F  ; Grapheme_Extend # Cf  [96] TAG SPACE..CANCEL TAG
 E0100..E01EF  ; Grapheme_Extend # Mn [240] VARIATION SELECTOR-17..VARIATION SELECTOR-256
 
-# Total code points: 2125
+# Total code points: 2193
 
 # ================================================
 
@@ -11062,10 +11321,8 @@ E0100..E01EF  ; Grapheme_Extend # Mn [240] VARIATION SELECTOR-17..VARIATION SELE
 0CB5..0CB9    ; Grapheme_Base # Lo   [5] KANNADA LETTER VA..KANNADA LETTER HA
 0CBD          ; Grapheme_Base # Lo       KANNADA SIGN AVAGRAHA
 0CBE          ; Grapheme_Base # Mc       KANNADA VOWEL SIGN AA
-0CC0..0CC1    ; Grapheme_Base # Mc   [2] KANNADA VOWEL SIGN II..KANNADA VOWEL SIGN U
+0CC1          ; Grapheme_Base # Mc       KANNADA VOWEL SIGN U
 0CC3..0CC4    ; Grapheme_Base # Mc   [2] KANNADA VOWEL SIGN VOCALIC R..KANNADA VOWEL SIGN VOCALIC RR
-0CC7..0CC8    ; Grapheme_Base # Mc   [2] KANNADA VOWEL SIGN EE..KANNADA VOWEL SIGN AI
-0CCA..0CCB    ; Grapheme_Base # Mc   [2] KANNADA VOWEL SIGN O..KANNADA VOWEL SIGN OO
 0CDD..0CDE    ; Grapheme_Base # Lo   [2] KANNADA LETTER NAKAARA POLLU..KANNADA LETTER FA
 0CE0..0CE1    ; Grapheme_Base # Lo   [2] KANNADA LETTER VOCALIC RR..KANNADA LETTER VOCALIC LL
 0CE6..0CEF    ; Grapheme_Base # Nd  [10] KANNADA DIGIT ZERO..KANNADA DIGIT NINE
@@ -11214,9 +11471,7 @@ E0100..E01EF  ; Grapheme_Extend # Mn [240] VARIATION SELECTOR-17..VARIATION SELE
 16EE..16F0    ; Grapheme_Base # Nl   [3] RUNIC ARLAUG SYMBOL..RUNIC BELGTHOR SYMBOL
 16F1..16F8    ; Grapheme_Base # Lo   [8] RUNIC LETTER K..RUNIC LETTER FRANKS CASKET AESC
 1700..1711    ; Grapheme_Base # Lo  [18] TAGALOG LETTER A..TAGALOG LETTER HA
-1715          ; Grapheme_Base # Mc       TAGALOG SIGN PAMUDPOD
 171F..1731    ; Grapheme_Base # Lo  [19] TAGALOG LETTER ARCHAIC RA..HANUNOO LETTER HA
-1734          ; Grapheme_Base # Mc       HANUNOO SIGN PAMUDPOD
 1735..1736    ; Grapheme_Base # Po   [2] PHILIPPINE SINGLE PUNCTUATION..PHILIPPINE DOUBLE PUNCTUATION
 1740..1751    ; Grapheme_Base # Lo  [18] BUHID LETTER A..BUHID LETTER HA
 1760..176C    ; Grapheme_Base # Lo  [13] TAGBANWA LETTER A..TAGBANWA LETTER YA
@@ -11274,27 +11529,24 @@ E0100..E01EF  ; Grapheme_Extend # Mn [240] VARIATION SELECTOR-17..VARIATION SELE
 1AA8..1AAD    ; Grapheme_Base # Po   [6] TAI THAM SIGN KAAN..TAI THAM SIGN CAANG
 1B04          ; Grapheme_Base # Mc       BALINESE SIGN BISAH
 1B05..1B33    ; Grapheme_Base # Lo  [47] BALINESE LETTER AKARA..BALINESE LETTER HA
-1B3B          ; Grapheme_Base # Mc       BALINESE VOWEL SIGN RA REPA TEDUNG
-1B3D..1B41    ; Grapheme_Base # Mc   [5] BALINESE VOWEL SIGN LA LENGA TEDUNG..BALINESE VOWEL SIGN TALING REPA TEDUNG
-1B43..1B44    ; Grapheme_Base # Mc   [2] BALINESE VOWEL SIGN PEPET TEDUNG..BALINESE ADEG ADEG
+1B3E..1B41    ; Grapheme_Base # Mc   [4] BALINESE VOWEL SIGN TALING..BALINESE VOWEL SIGN TALING REPA TEDUNG
 1B45..1B4C    ; Grapheme_Base # Lo   [8] BALINESE LETTER KAF SASAK..BALINESE LETTER ARCHAIC JNYA
+1B4E..1B4F    ; Grapheme_Base # Po   [2] BALINESE INVERTED CARIK SIKI..BALINESE INVERTED CARIK PAREREN
 1B50..1B59    ; Grapheme_Base # Nd  [10] BALINESE DIGIT ZERO..BALINESE DIGIT NINE
 1B5A..1B60    ; Grapheme_Base # Po   [7] BALINESE PANTI..BALINESE PAMENENG
 1B61..1B6A    ; Grapheme_Base # So  [10] BALINESE MUSICAL SYMBOL DONG..BALINESE MUSICAL SYMBOL DANG GEDE
 1B74..1B7C    ; Grapheme_Base # So   [9] BALINESE MUSICAL SYMBOL RIGHT-HAND OPEN DUG..BALINESE MUSICAL SYMBOL LEFT-HAND OPEN PING
-1B7D..1B7E    ; Grapheme_Base # Po   [2] BALINESE PANTI LANTANG..BALINESE PAMADA LANTANG
+1B7D..1B7F    ; Grapheme_Base # Po   [3] BALINESE PANTI LANTANG..BALINESE PANTI BAWAK
 1B82          ; Grapheme_Base # Mc       SUNDANESE SIGN PANGWISAD
 1B83..1BA0    ; Grapheme_Base # Lo  [30] SUNDANESE LETTER A..SUNDANESE LETTER HA
 1BA1          ; Grapheme_Base # Mc       SUNDANESE CONSONANT SIGN PAMINGKAL
 1BA6..1BA7    ; Grapheme_Base # Mc   [2] SUNDANESE VOWEL SIGN PANAELAENG..SUNDANESE VOWEL SIGN PANOLONG
-1BAA          ; Grapheme_Base # Mc       SUNDANESE SIGN PAMAAEH
 1BAE..1BAF    ; Grapheme_Base # Lo   [2] SUNDANESE LETTER KHA..SUNDANESE LETTER SYA
 1BB0..1BB9    ; Grapheme_Base # Nd  [10] SUNDANESE DIGIT ZERO..SUNDANESE DIGIT NINE
 1BBA..1BE5    ; Grapheme_Base # Lo  [44] SUNDANESE AVAGRAHA..BATAK LETTER U
 1BE7          ; Grapheme_Base # Mc       BATAK VOWEL SIGN E
 1BEA..1BEC    ; Grapheme_Base # Mc   [3] BATAK VOWEL SIGN I..BATAK VOWEL SIGN O
 1BEE          ; Grapheme_Base # Mc       BATAK VOWEL SIGN U
-1BF2..1BF3    ; Grapheme_Base # Mc   [2] BATAK PANGOLAT..BATAK PANONGONAN
 1BFC..1BFF    ; Grapheme_Base # Po   [4] BATAK SYMBOL BINDU NA METEK..BATAK SYMBOL BINDU PANGOLAT
 1C00..1C23    ; Grapheme_Base # Lo  [36] LEPCHA LETTER KA..LEPCHA LETTER A
 1C24..1C2B    ; Grapheme_Base # Mc   [8] LEPCHA SUBJOINED LETTER YA..LEPCHA VOWEL SIGN UU
@@ -11306,7 +11558,7 @@ E0100..E01EF  ; Grapheme_Extend # Mn [240] VARIATION SELECTOR-17..VARIATION SELE
 1C5A..1C77    ; Grapheme_Base # Lo  [30] OL CHIKI LETTER LA..OL CHIKI LETTER OH
 1C78..1C7D    ; Grapheme_Base # Lm   [6] OL CHIKI MU TTUDDAG..OL CHIKI AHAD
 1C7E..1C7F    ; Grapheme_Base # Po   [2] OL CHIKI PUNCTUATION MUCAAD..OL CHIKI PUNCTUATION DOUBLE MUCAAD
-1C80..1C88    ; Grapheme_Base # L&   [9] CYRILLIC SMALL LETTER ROUNDED VE..CYRILLIC SMALL LETTER UNBLENDED UK
+1C80..1C8A    ; Grapheme_Base # L&  [11] CYRILLIC SMALL LETTER ROUNDED VE..CYRILLIC SMALL LETTER TJE
 1C90..1CBA    ; Grapheme_Base # L&  [43] GEORGIAN MTAVRULI CAPITAL LETTER AN..GEORGIAN MTAVRULI CAPITAL LETTER AIN
 1CBD..1CBF    ; Grapheme_Base # L&   [3] GEORGIAN MTAVRULI CAPITAL LETTER AEN..GEORGIAN MTAVRULI CAPITAL LETTER LABIAL SIGN
 1CC0..1CC7    ; Grapheme_Base # Po   [8] SUNDANESE PUNCTUATION BINDU SURYA..SUNDANESE PUNCTUATION BINDU BA SATANGA
@@ -11461,7 +11713,7 @@ E0100..E01EF  ; Grapheme_Extend # Mn [240] VARIATION SELECTOR-17..VARIATION SELE
 239B..23B3    ; Grapheme_Base # Sm  [25] LEFT PARENTHESIS UPPER HOOK..SUMMATION BOTTOM
 23B4..23DB    ; Grapheme_Base # So  [40] TOP SQUARE BRACKET..FUSE
 23DC..23E1    ; Grapheme_Base # Sm   [6] TOP PARENTHESIS..BOTTOM TORTOISE SHELL BRACKET
-23E2..2426    ; Grapheme_Base # So  [69] WHITE TRAPEZIUM..SYMBOL FOR SUBSTITUTE FORM TWO
+23E2..2429    ; Grapheme_Base # So  [72] WHITE TRAPEZIUM..SYMBOL FOR DELETE MEDIUM SHADE FORM
 2440..244A    ; Grapheme_Base # So  [11] OCR HOOK..OCR DOUBLE BACKSLASH
 2460..249B    ; Grapheme_Base # No  [60] CIRCLED DIGIT ONE..NUMBER TWENTY FULL STOP
 249C..24E9    ; Grapheme_Base # So  [78] PARENTHESIZED LATIN SMALL LETTER A..CIRCLED LATIN SMALL LETTER Z
@@ -11676,7 +11928,7 @@ E0100..E01EF  ; Grapheme_Extend # Mn [240] VARIATION SELECTOR-17..VARIATION SELE
 3192..3195    ; Grapheme_Base # No   [4] IDEOGRAPHIC ANNOTATION ONE MARK..IDEOGRAPHIC ANNOTATION FOUR MARK
 3196..319F    ; Grapheme_Base # So  [10] IDEOGRAPHIC ANNOTATION TOP MARK..IDEOGRAPHIC ANNOTATION MAN MARK
 31A0..31BF    ; Grapheme_Base # Lo  [32] BOPOMOFO LETTER BU..BOPOMOFO LETTER AH
-31C0..31E3    ; Grapheme_Base # So  [36] CJK STROKE T..CJK STROKE Q
+31C0..31E5    ; Grapheme_Base # So  [38] CJK STROKE T..CJK STROKE SZP
 31EF          ; Grapheme_Base # So       IDEOGRAPHIC DESCRIPTION CHARACTER SUBTRACTION
 31F0..31FF    ; Grapheme_Base # Lo  [16] KATAKANA LETTER SMALL KU..KATAKANA LETTER SMALL RO
 3200..321E    ; Grapheme_Base # So  [31] PARENTHESIZED HANGUL KIYEOK..PARENTHESIZED KOREAN CHARACTER O HU
@@ -11725,10 +11977,10 @@ A788          ; Grapheme_Base # Lm       MODIFIER LETTER LOW CIRCUMFLEX ACCENT
 A789..A78A    ; Grapheme_Base # Sk   [2] MODIFIER LETTER COLON..MODIFIER LETTER SHORT EQUALS SIGN
 A78B..A78E    ; Grapheme_Base # L&   [4] LATIN CAPITAL LETTER SALTILLO..LATIN SMALL LETTER L WITH RETROFLEX HOOK AND BELT
 A78F          ; Grapheme_Base # Lo       LATIN LETTER SINOLOGICAL DOT
-A790..A7CA    ; Grapheme_Base # L&  [59] LATIN CAPITAL LETTER N WITH DESCENDER..LATIN SMALL LETTER S WITH SHORT STROKE OVERLAY
+A790..A7CD    ; Grapheme_Base # L&  [62] LATIN CAPITAL LETTER N WITH DESCENDER..LATIN SMALL LETTER S WITH DIAGONAL STROKE
 A7D0..A7D1    ; Grapheme_Base # L&   [2] LATIN CAPITAL LETTER CLOSED INSULAR G..LATIN SMALL LETTER CLOSED INSULAR G
 A7D3          ; Grapheme_Base # L&       LATIN SMALL LETTER DOUBLE THORN
-A7D5..A7D9    ; Grapheme_Base # L&   [5] LATIN SMALL LETTER DOUBLE WYNN..LATIN SMALL LETTER SIGMOID S
+A7D5..A7DC    ; Grapheme_Base # L&   [8] LATIN SMALL LETTER DOUBLE WYNN..LATIN CAPITAL LETTER LAMBDA WITH STROKE
 A7F2..A7F4    ; Grapheme_Base # Lm   [3] MODIFIER LETTER CAPITAL C..MODIFIER LETTER CAPITAL Q
 A7F5..A7F6    ; Grapheme_Base # L&   [2] LATIN CAPITAL LETTER REVERSED HALF H..LATIN SMALL LETTER REVERSED HALF H
 A7F7          ; Grapheme_Base # Lo       LATIN EPIGRAPHIC LETTER SIDEWAYS I
@@ -11761,14 +12013,14 @@ A900..A909    ; Grapheme_Base # Nd  [10] KAYAH LI DIGIT ZERO..KAYAH LI DIGIT NIN
 A90A..A925    ; Grapheme_Base # Lo  [28] KAYAH LI LETTER KA..KAYAH LI LETTER OO
 A92E..A92F    ; Grapheme_Base # Po   [2] KAYAH LI SIGN CWI..KAYAH LI SIGN SHYA
 A930..A946    ; Grapheme_Base # Lo  [23] REJANG LETTER KA..REJANG LETTER A
-A952..A953    ; Grapheme_Base # Mc   [2] REJANG CONSONANT SIGN H..REJANG VIRAMA
+A952          ; Grapheme_Base # Mc       REJANG CONSONANT SIGN H
 A95F          ; Grapheme_Base # Po       REJANG SECTION MARK
 A960..A97C    ; Grapheme_Base # Lo  [29] HANGUL CHOSEONG TIKEUT-MIEUM..HANGUL CHOSEONG SSANGYEORINHIEUH
 A983          ; Grapheme_Base # Mc       JAVANESE SIGN WIGNYAN
 A984..A9B2    ; Grapheme_Base # Lo  [47] JAVANESE LETTER A..JAVANESE LETTER HA
 A9B4..A9B5    ; Grapheme_Base # Mc   [2] JAVANESE VOWEL SIGN TARUNG..JAVANESE VOWEL SIGN TOLONG
 A9BA..A9BB    ; Grapheme_Base # Mc   [2] JAVANESE VOWEL SIGN TALING..JAVANESE VOWEL SIGN DIRGA MURE
-A9BE..A9C0    ; Grapheme_Base # Mc   [3] JAVANESE CONSONANT SIGN PENGKAL..JAVANESE PANGKON
+A9BE..A9BF    ; Grapheme_Base # Mc   [2] JAVANESE CONSONANT SIGN PENGKAL..JAVANESE CONSONANT SIGN CAKRA
 A9C1..A9CD    ; Grapheme_Base # Po  [13] JAVANESE LEFT RERENGGAN..JAVANESE TURNED PADA PISELEH
 A9CF          ; Grapheme_Base # Lm       JAVANESE PANGRANGKEP
 A9D0..A9D9    ; Grapheme_Base # Nd  [10] JAVANESE DIGIT ZERO..JAVANESE DIGIT NINE
@@ -12000,6 +12252,7 @@ FFFC..FFFD    ; Grapheme_Base # So   [2] OBJECT REPLACEMENT CHARACTER..REPLACEME
 105A3..105B1  ; Grapheme_Base # L&  [15] VITHKUQI SMALL LETTER HA..VITHKUQI SMALL LETTER RE
 105B3..105B9  ; Grapheme_Base # L&   [7] VITHKUQI SMALL LETTER SE..VITHKUQI SMALL LETTER XE
 105BB..105BC  ; Grapheme_Base # L&   [2] VITHKUQI SMALL LETTER Y..VITHKUQI SMALL LETTER ZE
+105C0..105F3  ; Grapheme_Base # Lo  [52] TODHRI LETTER A..TODHRI LETTER OO
 10600..10736  ; Grapheme_Base # Lo [311] LINEAR A SIGN AB001..LINEAR A SIGN A664
 10740..10755  ; Grapheme_Base # Lo  [22] LINEAR A SIGN A701 A..LINEAR A SIGN A732 JE
 10760..10767  ; Grapheme_Base # Lo   [8] LINEAR A SIGN A800..LINEAR A SIGN A807
@@ -12063,10 +12316,20 @@ FFFC..FFFD    ; Grapheme_Base # So   [2] OBJECT REPLACEMENT CHARACTER..REPLACEME
 10CFA..10CFF  ; Grapheme_Base # No   [6] OLD HUNGARIAN NUMBER ONE..OLD HUNGARIAN NUMBER ONE THOUSAND
 10D00..10D23  ; Grapheme_Base # Lo  [36] HANIFI ROHINGYA LETTER A..HANIFI ROHINGYA MARK NA KHONNA
 10D30..10D39  ; Grapheme_Base # Nd  [10] HANIFI ROHINGYA DIGIT ZERO..HANIFI ROHINGYA DIGIT NINE
+10D40..10D49  ; Grapheme_Base # Nd  [10] GARAY DIGIT ZERO..GARAY DIGIT NINE
+10D4A..10D4D  ; Grapheme_Base # Lo   [4] GARAY VOWEL SIGN A..GARAY VOWEL SIGN EE
+10D4E         ; Grapheme_Base # Lm       GARAY VOWEL LENGTH MARK
+10D4F         ; Grapheme_Base # Lo       GARAY SUKUN
+10D50..10D65  ; Grapheme_Base # L&  [22] GARAY CAPITAL LETTER A..GARAY CAPITAL LETTER OLD NA
+10D6E         ; Grapheme_Base # Pd       GARAY HYPHEN
+10D6F         ; Grapheme_Base # Lm       GARAY REDUPLICATION MARK
+10D70..10D85  ; Grapheme_Base # L&  [22] GARAY SMALL LETTER A..GARAY SMALL LETTER OLD NA
+10D8E..10D8F  ; Grapheme_Base # Sm   [2] GARAY PLUS SIGN..GARAY MINUS SIGN
 10E60..10E7E  ; Grapheme_Base # No  [31] RUMI DIGIT ONE..RUMI FRACTION TWO THIRDS
 10E80..10EA9  ; Grapheme_Base # Lo  [42] YEZIDI LETTER ELIF..YEZIDI LETTER ET
 10EAD         ; Grapheme_Base # Pd       YEZIDI HYPHENATION MARK
 10EB0..10EB1  ; Grapheme_Base # Lo   [2] YEZIDI LETTER LAM WITH DOT ABOVE..YEZIDI LETTER YOT WITH CIRCUMFLEX ABOVE
+10EC2..10EC4  ; Grapheme_Base # Lo   [3] ARABIC LETTER DAL WITH TWO DOTS VERTICALLY BELOW..ARABIC LETTER KAF WITH TWO DOTS VERTICALLY BELOW
 10F00..10F1C  ; Grapheme_Base # Lo  [29] OLD SOGDIAN LETTER ALEPH..OLD SOGDIAN LETTER FINAL TAW WITH VERTICAL TAIL
 10F1D..10F26  ; Grapheme_Base # No  [10] OLD SOGDIAN NUMBER ONE..OLD SOGDIAN FRACTION ONE HALF
 10F27         ; Grapheme_Base # Lo       OLD SOGDIAN LIGATURE AYIN-DALETH
@@ -12107,7 +12370,7 @@ FFFC..FFFD    ; Grapheme_Base # So   [2] OBJECT REPLACEMENT CHARACTER..REPLACEME
 11182         ; Grapheme_Base # Mc       SHARADA SIGN VISARGA
 11183..111B2  ; Grapheme_Base # Lo  [48] SHARADA LETTER A..SHARADA LETTER HA
 111B3..111B5  ; Grapheme_Base # Mc   [3] SHARADA VOWEL SIGN AA..SHARADA VOWEL SIGN II
-111BF..111C0  ; Grapheme_Base # Mc   [2] SHARADA VOWEL SIGN AU..SHARADA SIGN VIRAMA
+111BF         ; Grapheme_Base # Mc       SHARADA VOWEL SIGN AU
 111C1..111C4  ; Grapheme_Base # Lo   [4] SHARADA SIGN AVAGRAHA..SHARADA OM
 111C5..111C8  ; Grapheme_Base # Po   [4] SHARADA DANDA..SHARADA SEPARATOR
 111CD         ; Grapheme_Base # Po       SHARADA SUTRA MARK
@@ -12122,7 +12385,6 @@ FFFC..FFFD    ; Grapheme_Base # So   [2] OBJECT REPLACEMENT CHARACTER..REPLACEME
 11213..1122B  ; Grapheme_Base # Lo  [25] KHOJKI LETTER NYA..KHOJKI LETTER LLA
 1122C..1122E  ; Grapheme_Base # Mc   [3] KHOJKI VOWEL SIGN AA..KHOJKI VOWEL SIGN II
 11232..11233  ; Grapheme_Base # Mc   [2] KHOJKI VOWEL SIGN O..KHOJKI VOWEL SIGN AU
-11235         ; Grapheme_Base # Mc       KHOJKI SIGN VIRAMA
 11238..1123D  ; Grapheme_Base # Po   [6] KHOJKI DANDA..KHOJKI ABBREVIATION SIGN
 1123F..11240  ; Grapheme_Base # Lo   [2] KHOJKI LETTER QA..KHOJKI LETTER SHORT I
 11280..11286  ; Grapheme_Base # Lo   [7] MULTANI LETTER A..MULTANI LETTER GA
@@ -12145,10 +12407,22 @@ FFFC..FFFD    ; Grapheme_Base # So   [2] OBJECT REPLACEMENT CHARACTER..REPLACEME
 1133F         ; Grapheme_Base # Mc       GRANTHA VOWEL SIGN I
 11341..11344  ; Grapheme_Base # Mc   [4] GRANTHA VOWEL SIGN U..GRANTHA VOWEL SIGN VOCALIC RR
 11347..11348  ; Grapheme_Base # Mc   [2] GRANTHA VOWEL SIGN EE..GRANTHA VOWEL SIGN AI
-1134B..1134D  ; Grapheme_Base # Mc   [3] GRANTHA VOWEL SIGN OO..GRANTHA SIGN VIRAMA
+1134B..1134C  ; Grapheme_Base # Mc   [2] GRANTHA VOWEL SIGN OO..GRANTHA VOWEL SIGN AU
 11350         ; Grapheme_Base # Lo       GRANTHA OM
 1135D..11361  ; Grapheme_Base # Lo   [5] GRANTHA SIGN PLUTA..GRANTHA LETTER VOCALIC LL
 11362..11363  ; Grapheme_Base # Mc   [2] GRANTHA VOWEL SIGN VOCALIC L..GRANTHA VOWEL SIGN VOCALIC LL
+11380..11389  ; Grapheme_Base # Lo  [10] TULU-TIGALARI LETTER A..TULU-TIGALARI LETTER VOCALIC LL
+1138B         ; Grapheme_Base # Lo       TULU-TIGALARI LETTER EE
+1138E         ; Grapheme_Base # Lo       TULU-TIGALARI LETTER AI
+11390..113B5  ; Grapheme_Base # Lo  [38] TULU-TIGALARI LETTER OO..TULU-TIGALARI LETTER LLLA
+113B7         ; Grapheme_Base # Lo       TULU-TIGALARI SIGN AVAGRAHA
+113B9..113BA  ; Grapheme_Base # Mc   [2] TULU-TIGALARI VOWEL SIGN I..TULU-TIGALARI VOWEL SIGN II
+113CA         ; Grapheme_Base # Mc       TULU-TIGALARI SIGN CANDRA ANUNASIKA
+113CC..113CD  ; Grapheme_Base # Mc   [2] TULU-TIGALARI SIGN ANUSVARA..TULU-TIGALARI SIGN VISARGA
+113D1         ; Grapheme_Base # Lo       TULU-TIGALARI REPHA
+113D3         ; Grapheme_Base # Lo       TULU-TIGALARI SIGN PLUTA
+113D4..113D5  ; Grapheme_Base # Po   [2] TULU-TIGALARI DANDA..TULU-TIGALARI DOUBLE DANDA
+113D7..113D8  ; Grapheme_Base # Po   [2] TULU-TIGALARI SIGN OM PUSHPIKA..TULU-TIGALARI SIGN SHRII PUSHPIKA
 11400..11434  ; Grapheme_Base # Lo  [53] NEWA LETTER A..NEWA LETTER HA
 11435..11437  ; Grapheme_Base # Mc   [3] NEWA VOWEL SIGN AA..NEWA VOWEL SIGN II
 11440..11441  ; Grapheme_Base # Mc   [2] NEWA VOWEL SIGN O..NEWA VOWEL SIGN AU
@@ -12186,11 +12460,12 @@ FFFC..FFFD    ; Grapheme_Base # So   [2] OBJECT REPLACEMENT CHARACTER..REPLACEME
 11680..116AA  ; Grapheme_Base # Lo  [43] TAKRI LETTER A..TAKRI LETTER RRA
 116AC         ; Grapheme_Base # Mc       TAKRI SIGN VISARGA
 116AE..116AF  ; Grapheme_Base # Mc   [2] TAKRI VOWEL SIGN I..TAKRI VOWEL SIGN II
-116B6         ; Grapheme_Base # Mc       TAKRI SIGN VIRAMA
 116B8         ; Grapheme_Base # Lo       TAKRI LETTER ARCHAIC KHA
 116B9         ; Grapheme_Base # Po       TAKRI ABBREVIATION SIGN
 116C0..116C9  ; Grapheme_Base # Nd  [10] TAKRI DIGIT ZERO..TAKRI DIGIT NINE
+116D0..116E3  ; Grapheme_Base # Nd  [20] MYANMAR PAO DIGIT ZERO..MYANMAR EASTERN PWO KAREN DIGIT NINE
 11700..1171A  ; Grapheme_Base # Lo  [27] AHOM LETTER KA..AHOM LETTER ALTERNATE BA
+1171E         ; Grapheme_Base # Mc       AHOM CONSONANT SIGN MEDIAL RA
 11720..11721  ; Grapheme_Base # Mc   [2] AHOM VOWEL SIGN A..AHOM VOWEL SIGN AA
 11726         ; Grapheme_Base # Mc       AHOM VOWEL SIGN E
 11730..11739  ; Grapheme_Base # Nd  [10] AHOM DIGIT ZERO..AHOM DIGIT NINE
@@ -12212,7 +12487,6 @@ FFFC..FFFD    ; Grapheme_Base # So   [2] OBJECT REPLACEMENT CHARACTER..REPLACEME
 11918..1192F  ; Grapheme_Base # Lo  [24] DIVES AKURU LETTER DDA..DIVES AKURU LETTER ZA
 11931..11935  ; Grapheme_Base # Mc   [5] DIVES AKURU VOWEL SIGN I..DIVES AKURU VOWEL SIGN E
 11937..11938  ; Grapheme_Base # Mc   [2] DIVES AKURU VOWEL SIGN AI..DIVES AKURU VOWEL SIGN O
-1193D         ; Grapheme_Base # Mc       DIVES AKURU SIGN HALANTA
 1193F         ; Grapheme_Base # Lo       DIVES AKURU PREFIXED NASAL SIGN
 11940         ; Grapheme_Base # Mc       DIVES AKURU MEDIAL YA
 11941         ; Grapheme_Base # Lo       DIVES AKURU INITIAL RA
@@ -12241,6 +12515,9 @@ FFFC..FFFD    ; Grapheme_Base # So   [2] OBJECT REPLACEMENT CHARACTER..REPLACEME
 11A9E..11AA2  ; Grapheme_Base # Po   [5] SOYOMBO HEAD MARK WITH MOON AND SUN AND TRIPLE FLAME..SOYOMBO TERMINAL MARK-2
 11AB0..11AF8  ; Grapheme_Base # Lo  [73] CANADIAN SYLLABICS NATTILIK HI..PAU CIN HAU GLOTTAL STOP FINAL
 11B00..11B09  ; Grapheme_Base # Po  [10] DEVANAGARI HEAD MARK..DEVANAGARI SIGN MINDU
+11BC0..11BE0  ; Grapheme_Base # Lo  [33] SUNUWAR LETTER DEVI..SUNUWAR LETTER KLOKO
+11BE1         ; Grapheme_Base # Po       SUNUWAR SIGN PVO
+11BF0..11BF9  ; Grapheme_Base # Nd  [10] SUNUWAR DIGIT ZERO..SUNUWAR DIGIT NINE
 11C00..11C08  ; Grapheme_Base # Lo   [9] BHAIKSUKI LETTER A..BHAIKSUKI LETTER VOCALIC L
 11C0A..11C2E  ; Grapheme_Base # Lo  [37] BHAIKSUKI LETTER E..BHAIKSUKI LETTER HA
 11C2F         ; Grapheme_Base # Mc       BHAIKSUKI VOWEL SIGN AA
@@ -12276,7 +12553,6 @@ FFFC..FFFD    ; Grapheme_Base # So   [2] OBJECT REPLACEMENT CHARACTER..REPLACEME
 11F12..11F33  ; Grapheme_Base # Lo  [34] KAWI LETTER KA..KAWI LETTER JNYA
 11F34..11F35  ; Grapheme_Base # Mc   [2] KAWI VOWEL SIGN AA..KAWI VOWEL SIGN ALTERNATE AA
 11F3E..11F3F  ; Grapheme_Base # Mc   [2] KAWI VOWEL SIGN E..KAWI VOWEL SIGN AI
-11F41         ; Grapheme_Base # Mc       KAWI SIGN KILLER
 11F43..11F4F  ; Grapheme_Base # Po  [13] KAWI DANDA..KAWI PUNCTUATION CLOSING SPIRAL
 11F50..11F59  ; Grapheme_Base # Nd  [10] KAWI DIGIT ZERO..KAWI DIGIT NINE
 11FB0         ; Grapheme_Base # Lo       LISU LETTER YHA
@@ -12293,7 +12569,11 @@ FFFC..FFFD    ; Grapheme_Base # So   [2] OBJECT REPLACEMENT CHARACTER..REPLACEME
 12FF1..12FF2  ; Grapheme_Base # Po   [2] CYPRO-MINOAN SIGN CM301..CYPRO-MINOAN SIGN CM302
 13000..1342F  ; Grapheme_Base # Lo [1072] EGYPTIAN HIEROGLYPH A001..EGYPTIAN HIEROGLYPH V011D
 13441..13446  ; Grapheme_Base # Lo   [6] EGYPTIAN HIEROGLYPH FULL BLANK..EGYPTIAN HIEROGLYPH WIDE LOST SIGN
+13460..143FA  ; Grapheme_Base # Lo [3995] EGYPTIAN HIEROGLYPH-13460..EGYPTIAN HIEROGLYPH-143FA
 14400..14646  ; Grapheme_Base # Lo [583] ANATOLIAN HIEROGLYPH A001..ANATOLIAN HIEROGLYPH A530
+16100..1611D  ; Grapheme_Base # Lo  [30] GURUNG KHEMA LETTER A..GURUNG KHEMA LETTER SA
+1612A..1612C  ; Grapheme_Base # Mc   [3] GURUNG KHEMA CONSONANT SIGN MEDIAL YA..GURUNG KHEMA CONSONANT SIGN MEDIAL HA
+16130..16139  ; Grapheme_Base # Nd  [10] GURUNG KHEMA DIGIT ZERO..GURUNG KHEMA DIGIT NINE
 16800..16A38  ; Grapheme_Base # Lo [569] BAMUM LETTER PHASE-A NGKUE MFON..BAMUM LETTER PHASE-F VUEQ
 16A40..16A5E  ; Grapheme_Base # Lo  [31] MRO LETTER TA..MRO LETTER TEK
 16A60..16A69  ; Grapheme_Base # Nd  [10] MRO DIGIT ZERO..MRO DIGIT NINE
@@ -12312,6 +12592,11 @@ FFFC..FFFD    ; Grapheme_Base # So   [2] OBJECT REPLACEMENT CHARACTER..REPLACEME
 16B5B..16B61  ; Grapheme_Base # No   [7] PAHAWH HMONG NUMBER TENS..PAHAWH HMONG NUMBER TRILLIONS
 16B63..16B77  ; Grapheme_Base # Lo  [21] PAHAWH HMONG SIGN VOS LUB..PAHAWH HMONG SIGN CIM NRES TOS
 16B7D..16B8F  ; Grapheme_Base # Lo  [19] PAHAWH HMONG CLAN SIGN TSHEEJ..PAHAWH HMONG CLAN SIGN VWJ
+16D40..16D42  ; Grapheme_Base # Lm   [3] KIRAT RAI SIGN ANUSVARA..KIRAT RAI SIGN VISARGA
+16D43..16D6A  ; Grapheme_Base # Lo  [40] KIRAT RAI LETTER A..KIRAT RAI VOWEL SIGN AU
+16D6B..16D6C  ; Grapheme_Base # Lm   [2] KIRAT RAI SIGN VIRAMA..KIRAT RAI SIGN SAAT
+16D6D..16D6F  ; Grapheme_Base # Po   [3] KIRAT RAI SIGN YUPI..KIRAT RAI DOUBLE DANDA
+16D70..16D79  ; Grapheme_Base # Nd  [10] KIRAT RAI DIGIT ZERO..KIRAT RAI DIGIT NINE
 16E40..16E7F  ; Grapheme_Base # L&  [64] MEDEFAIDRIN CAPITAL LETTER M..MEDEFAIDRIN SMALL LETTER Y
 16E80..16E96  ; Grapheme_Base # No  [23] MEDEFAIDRIN DIGIT ZERO..MEDEFAIDRIN DIGIT THREE ALTERNATE FORM
 16E97..16E9A  ; Grapheme_Base # Po   [4] MEDEFAIDRIN COMMA..MEDEFAIDRIN EXCLAMATION OH
@@ -12322,10 +12607,9 @@ FFFC..FFFD    ; Grapheme_Base # So   [2] OBJECT REPLACEMENT CHARACTER..REPLACEME
 16FE0..16FE1  ; Grapheme_Base # Lm   [2] TANGUT ITERATION MARK..NUSHU ITERATION MARK
 16FE2         ; Grapheme_Base # Po       OLD CHINESE HOOK MARK
 16FE3         ; Grapheme_Base # Lm       OLD CHINESE ITERATION MARK
-16FF0..16FF1  ; Grapheme_Base # Mc   [2] VIETNAMESE ALTERNATE READING MARK CA..VIETNAMESE ALTERNATE READING MARK NHAY
 17000..187F7  ; Grapheme_Base # Lo [6136] TANGUT IDEOGRAPH-17000..TANGUT IDEOGRAPH-187F7
 18800..18CD5  ; Grapheme_Base # Lo [1238] TANGUT COMPONENT-001..KHITAN SMALL SCRIPT CHARACTER-18CD5
-18D00..18D08  ; Grapheme_Base # Lo   [9] TANGUT IDEOGRAPH-18D00..TANGUT IDEOGRAPH-18D08
+18CFF..18D08  ; Grapheme_Base # Lo  [10] KHITAN SMALL SCRIPT CHARACTER-18CFF..TANGUT IDEOGRAPH-18D08
 1AFF0..1AFF3  ; Grapheme_Base # Lm   [4] KATAKANA LETTER MINNAN TONE-2..KATAKANA LETTER MINNAN TONE-5
 1AFF5..1AFFB  ; Grapheme_Base # Lm   [7] KATAKANA LETTER MINNAN TONE-7..KATAKANA LETTER MINNAN NASALIZED TONE-5
 1AFFD..1AFFE  ; Grapheme_Base # Lm   [2] KATAKANA LETTER MINNAN NASALIZED TONE-7..KATAKANA LETTER MINNAN NASALIZED TONE-8
@@ -12341,13 +12625,14 @@ FFFC..FFFD    ; Grapheme_Base # So   [2] OBJECT REPLACEMENT CHARACTER..REPLACEME
 1BC90..1BC99  ; Grapheme_Base # Lo  [10] DUPLOYAN AFFIX LOW ACUTE..DUPLOYAN AFFIX LOW ARROW
 1BC9C         ; Grapheme_Base # So       DUPLOYAN SIGN O WITH CROSS
 1BC9F         ; Grapheme_Base # Po       DUPLOYAN PUNCTUATION CHINOOK FULL STOP
+1CC00..1CCEF  ; Grapheme_Base # So [240] UP-POINTING GO-KART..OUTLINED LATIN CAPITAL LETTER Z
+1CCF0..1CCF9  ; Grapheme_Base # Nd  [10] OUTLINED DIGIT ZERO..OUTLINED DIGIT NINE
+1CD00..1CEB3  ; Grapheme_Base # So [436] BLOCK OCTANT-3..BLACK RIGHT TRIANGLE CARET
 1CF50..1CFC3  ; Grapheme_Base # So [116] ZNAMENNY NEUME KRYUK..ZNAMENNY NEUME PAUK
 1D000..1D0F5  ; Grapheme_Base # So [246] BYZANTINE MUSICAL SYMBOL PSILI..BYZANTINE MUSICAL SYMBOL GORGON NEO KATO
 1D100..1D126  ; Grapheme_Base # So  [39] MUSICAL SYMBOL SINGLE BARLINE..MUSICAL SYMBOL DRUM CLEF-2
 1D129..1D164  ; Grapheme_Base # So  [60] MUSICAL SYMBOL MULTIPLE MEASURE REST..MUSICAL SYMBOL ONE HUNDRED TWENTY-EIGHTH NOTE
-1D166         ; Grapheme_Base # Mc       MUSICAL SYMBOL COMBINING SPRECHGESANG STEM
 1D16A..1D16C  ; Grapheme_Base # So   [3] MUSICAL SYMBOL FINGERED TREMOLO-1..MUSICAL SYMBOL FINGERED TREMOLO-3
-1D16D         ; Grapheme_Base # Mc       MUSICAL SYMBOL COMBINING AUGMENTATION DOT
 1D183..1D184  ; Grapheme_Base # So   [2] MUSICAL SYMBOL ARPEGGIATO UP..MUSICAL SYMBOL ARPEGGIATO DOWN
 1D18C..1D1A9  ; Grapheme_Base # So  [30] MUSICAL SYMBOL RINFORZANDO..MUSICAL SYMBOL DEGREE SLASH
 1D1AE..1D1EA  ; Grapheme_Base # So  [61] MUSICAL SYMBOL PEDAL MARK..MUSICAL SYMBOL KORON
@@ -12421,6 +12706,10 @@ FFFC..FFFD    ; Grapheme_Base # So   [2] OBJECT REPLACEMENT CHARACTER..REPLACEME
 1E4D0..1E4EA  ; Grapheme_Base # Lo  [27] NAG MUNDARI LETTER O..NAG MUNDARI LETTER ELL
 1E4EB         ; Grapheme_Base # Lm       NAG MUNDARI SIGN OJOD
 1E4F0..1E4F9  ; Grapheme_Base # Nd  [10] NAG MUNDARI DIGIT ZERO..NAG MUNDARI DIGIT NINE
+1E5D0..1E5ED  ; Grapheme_Base # Lo  [30] OL ONAL LETTER O..OL ONAL LETTER EG
+1E5F0         ; Grapheme_Base # Lo       OL ONAL SIGN HODDOND
+1E5F1..1E5FA  ; Grapheme_Base # Nd  [10] OL ONAL DIGIT ZERO..OL ONAL DIGIT NINE
+1E5FF         ; Grapheme_Base # Po       OL ONAL ABBREVIATION SIGN
 1E7E0..1E7E6  ; Grapheme_Base # Lo   [7] ETHIOPIC SYLLABLE HHYA..ETHIOPIC SYLLABLE HHYO
 1E7E8..1E7EB  ; Grapheme_Base # Lo   [4] ETHIOPIC SYLLABLE GURAGE HHWA..ETHIOPIC SYLLABLE HHWE
 1E7ED..1E7EE  ; Grapheme_Base # Lo   [2] ETHIOPIC SYLLABLE GURAGE MWI..ETHIOPIC SYLLABLE GURAGE MWEE
@@ -12500,18 +12789,18 @@ FFFC..FFFD    ; Grapheme_Base # So   [2] OBJECT REPLACEMENT CHARACTER..REPLACEME
 1F850..1F859  ; Grapheme_Base # So  [10] LEFTWARDS SANS-SERIF ARROW..UP DOWN SANS-SERIF ARROW
 1F860..1F887  ; Grapheme_Base # So  [40] WIDE-HEADED LEFTWARDS LIGHT BARB ARROW..WIDE-HEADED SOUTH WEST VERY HEAVY BARB ARROW
 1F890..1F8AD  ; Grapheme_Base # So  [30] LEFTWARDS TRIANGLE ARROWHEAD..WHITE ARROW SHAFT WIDTH TWO THIRDS
-1F8B0..1F8B1  ; Grapheme_Base # So   [2] ARROW POINTING UPWARDS THEN NORTH WEST..ARROW POINTING RIGHTWARDS THEN CURVING SOUTH WEST
+1F8B0..1F8BB  ; Grapheme_Base # So  [12] ARROW POINTING UPWARDS THEN NORTH WEST..SOUTH WEST ARROW FROM BAR
+1F8C0..1F8C1  ; Grapheme_Base # So   [2] LEFTWARDS ARROW FROM DOWNWARDS ARROW..RIGHTWARDS ARROW FROM DOWNWARDS ARROW
 1F900..1FA53  ; Grapheme_Base # So [340] CIRCLED CROSS FORMEE WITH FOUR DOTS..BLACK CHESS KNIGHT-BISHOP
 1FA60..1FA6D  ; Grapheme_Base # So  [14] XIANGQI RED GENERAL..XIANGQI BLACK SOLDIER
 1FA70..1FA7C  ; Grapheme_Base # So  [13] BALLET SHOES..CRUTCH
-1FA80..1FA88  ; Grapheme_Base # So   [9] YO-YO..FLUTE
-1FA90..1FABD  ; Grapheme_Base # So  [46] RINGED PLANET..WING
-1FABF..1FAC5  ; Grapheme_Base # So   [7] GOOSE..PERSON WITH CROWN
-1FACE..1FADB  ; Grapheme_Base # So  [14] MOOSE..PEA POD
-1FAE0..1FAE8  ; Grapheme_Base # So   [9] MELTING FACE..SHAKING FACE
+1FA80..1FA89  ; Grapheme_Base # So  [10] YO-YO..HARP
+1FA8F..1FAC6  ; Grapheme_Base # So  [56] SHOVEL..FINGERPRINT
+1FACE..1FADC  ; Grapheme_Base # So  [15] MOOSE..ROOT VEGETABLE
+1FADF..1FAE9  ; Grapheme_Base # So  [11] SPLATTER..FACE WITH BAGS UNDER EYES
 1FAF0..1FAF8  ; Grapheme_Base # So   [9] HAND WITH INDEX FINGER AND THUMB CROSSED..RIGHTWARDS PUSHING HAND
 1FB00..1FB92  ; Grapheme_Base # So [147] BLOCK SEXTANT-1..UPPER HALF INVERSE MEDIUM SHADE AND LOWER HALF BLOCK
-1FB94..1FBCA  ; Grapheme_Base # So  [55] LEFT HALF INVERSE MEDIUM SHADE AND RIGHT HALF BLOCK..WHITE UP-POINTING CHEVRON
+1FB94..1FBEF  ; Grapheme_Base # So  [92] LEFT HALF INVERSE MEDIUM SHADE AND RIGHT HALF BLOCK..TOP LEFT JUSTIFIED LOWER RIGHT QUARTER BLACK CIRCLE
 1FBF0..1FBF9  ; Grapheme_Base # Nd  [10] SEGMENTED DIGIT ZERO..SEGMENTED DIGIT NINE
 20000..2A6DF  ; Grapheme_Base # Lo [42720] CJK UNIFIED IDEOGRAPH-20000..CJK UNIFIED IDEOGRAPH-2A6DF
 2A700..2B739  ; Grapheme_Base # Lo [4154] CJK UNIFIED IDEOGRAPH-2A700..CJK UNIFIED IDEOGRAPH-2B739
@@ -12523,7 +12812,7 @@ FFFC..FFFD    ; Grapheme_Base # So   [2] OBJECT REPLACEMENT CHARACTER..REPLACEME
 30000..3134A  ; Grapheme_Base # Lo [4939] CJK UNIFIED IDEOGRAPH-30000..CJK UNIFIED IDEOGRAPH-3134A
 31350..323AF  ; Grapheme_Base # Lo [4192] CJK UNIFIED IDEOGRAPH-31350..CJK UNIFIED IDEOGRAPH-323AF
 
-# Total code points: 147613
+# Total code points: 152730
 
 # ================================================
 
@@ -12573,6 +12862,9 @@ ABED          ; Grapheme_Link # Mn       MEETEI MAYEK APUN IYEK
 11235         ; Grapheme_Link # Mc       KHOJKI SIGN VIRAMA
 112EA         ; Grapheme_Link # Mn       KHUDAWADI SIGN VIRAMA
 1134D         ; Grapheme_Link # Mc       GRANTHA SIGN VIRAMA
+113CE         ; Grapheme_Link # Mn       TULU-TIGALARI SIGN VIRAMA
+113CF         ; Grapheme_Link # Mc       TULU-TIGALARI SIGN LOOPED VIRAMA
+113D0         ; Grapheme_Link # Mn       TULU-TIGALARI CONJOINER
 11442         ; Grapheme_Link # Mn       NEWA SIGN VIRAMA
 114C2         ; Grapheme_Link # Mn       TIRHUTA SIGN VIRAMA
 115BF         ; Grapheme_Link # Mn       SIDDHAM SIGN VIRAMA
@@ -12591,8 +12883,9 @@ ABED          ; Grapheme_Link # Mn       MEETEI MAYEK APUN IYEK
 11D97         ; Grapheme_Link # Mn       GUNJALA GONDI VIRAMA
 11F41         ; Grapheme_Link # Mc       KAWI SIGN KILLER
 11F42         ; Grapheme_Link # Mn       KAWI CONJOINER
+1612F         ; Grapheme_Link # Mn       GURUNG KHEMA SIGN THOLHOMA
 
-# Total code points: 65
+# Total code points: 69
 
 # ================================================
 
@@ -12656,9 +12949,9 @@ ABED          ; Grapheme_Link # Mn       MEETEI MAYEK APUN IYEK
 
 # Indic_Conjunct_Break=Extend
 
-0300..034E    ; InCB; Extend # Mn  [79] COMBINING GRAVE ACCENT..COMBINING UPWARDS ARROW BELOW
-0350..036F    ; InCB; Extend # Mn  [32] COMBINING RIGHT ARROWHEAD ABOVE..COMBINING LATIN SMALL LETTER X
+0300..036F    ; InCB; Extend # Mn [112] COMBINING GRAVE ACCENT..COMBINING LATIN SMALL LETTER X
 0483..0487    ; InCB; Extend # Mn   [5] COMBINING CYRILLIC TITLO..COMBINING CYRILLIC POKRYTIE
+0488..0489    ; InCB; Extend # Me   [2] COMBINING CYRILLIC HUNDRED THOUSANDS SIGN..COMBINING CYRILLIC MILLIONS SIGN
 0591..05BD    ; InCB; Extend # Mn  [45] HEBREW ACCENT ETNAHTA..HEBREW POINT METEG
 05BF          ; InCB; Extend # Mn       HEBREW POINT RAFE
 05C1..05C2    ; InCB; Extend # Mn   [2] HEBREW POINT SHIN DOT..HEBREW POINT SIN DOT
@@ -12673,6 +12966,7 @@ ABED          ; Grapheme_Link # Mn       MEETEI MAYEK APUN IYEK
 06EA..06ED    ; InCB; Extend # Mn   [4] ARABIC EMPTY CENTRE LOW STOP..ARABIC SMALL LOW MEEM
 0711          ; InCB; Extend # Mn       SYRIAC LETTER SUPERSCRIPT ALAPH
 0730..074A    ; InCB; Extend # Mn  [27] SYRIAC PTHAHA ABOVE..SYRIAC BARREKH
+07A6..07B0    ; InCB; Extend # Mn  [11] THAANA ABAFILI..THAANA SUKUN
 07EB..07F3    ; InCB; Extend # Mn   [9] NKO COMBINING SHORT HIGH TONE..NKO COMBINING DOUBLE DOT ABOVE
 07FD          ; InCB; Extend # Mn       NKO DANTAYALAN
 0816..0819    ; InCB; Extend # Mn   [4] SAMARITAN MARK IN..SAMARITAN MARK DAGESH
@@ -12680,55 +12974,160 @@ ABED          ; Grapheme_Link # Mn       MEETEI MAYEK APUN IYEK
 0825..0827    ; InCB; Extend # Mn   [3] SAMARITAN VOWEL SIGN SHORT A..SAMARITAN VOWEL SIGN U
 0829..082D    ; InCB; Extend # Mn   [5] SAMARITAN VOWEL SIGN LONG I..SAMARITAN MARK NEQUDAA
 0859..085B    ; InCB; Extend # Mn   [3] MANDAIC AFFRICATION MARK..MANDAIC GEMINATION MARK
-0898..089F    ; InCB; Extend # Mn   [8] ARABIC SMALL HIGH WORD AL-JUZ..ARABIC HALF MADDA OVER MADDA
+0897..089F    ; InCB; Extend # Mn   [9] ARABIC PEPET..ARABIC HALF MADDA OVER MADDA
 08CA..08E1    ; InCB; Extend # Mn  [24] ARABIC SMALL HIGH FARSI YEH..ARABIC SMALL HIGH SIGN SAFHA
-08E3..08FF    ; InCB; Extend # Mn  [29] ARABIC TURNED DAMMA BELOW..ARABIC MARK SIDEWAYS NOON GHUNNA
+08E3..0902    ; InCB; Extend # Mn  [32] ARABIC TURNED DAMMA BELOW..DEVANAGARI SIGN ANUSVARA
+093A          ; InCB; Extend # Mn       DEVANAGARI VOWEL SIGN OE
 093C          ; InCB; Extend # Mn       DEVANAGARI SIGN NUKTA
-0951..0954    ; InCB; Extend # Mn   [4] DEVANAGARI STRESS SIGN UDATTA..DEVANAGARI ACUTE ACCENT
+0941..0948    ; InCB; Extend # Mn   [8] DEVANAGARI VOWEL SIGN U..DEVANAGARI VOWEL SIGN AI
+0951..0957    ; InCB; Extend # Mn   [7] DEVANAGARI STRESS SIGN UDATTA..DEVANAGARI VOWEL SIGN UUE
+0962..0963    ; InCB; Extend # Mn   [2] DEVANAGARI VOWEL SIGN VOCALIC L..DEVANAGARI VOWEL SIGN VOCALIC LL
+0981          ; InCB; Extend # Mn       BENGALI SIGN CANDRABINDU
 09BC          ; InCB; Extend # Mn       BENGALI SIGN NUKTA
+09BE          ; InCB; Extend # Mc       BENGALI VOWEL SIGN AA
+09C1..09C4    ; InCB; Extend # Mn   [4] BENGALI VOWEL SIGN U..BENGALI VOWEL SIGN VOCALIC RR
+09D7          ; InCB; Extend # Mc       BENGALI AU LENGTH MARK
+09E2..09E3    ; InCB; Extend # Mn   [2] BENGALI VOWEL SIGN VOCALIC L..BENGALI VOWEL SIGN VOCALIC LL
 09FE          ; InCB; Extend # Mn       BENGALI SANDHI MARK
+0A01..0A02    ; InCB; Extend # Mn   [2] GURMUKHI SIGN ADAK BINDI..GURMUKHI SIGN BINDI
 0A3C          ; InCB; Extend # Mn       GURMUKHI SIGN NUKTA
+0A41..0A42    ; InCB; Extend # Mn   [2] GURMUKHI VOWEL SIGN U..GURMUKHI VOWEL SIGN UU
+0A47..0A48    ; InCB; Extend # Mn   [2] GURMUKHI VOWEL SIGN EE..GURMUKHI VOWEL SIGN AI
+0A4B..0A4D    ; InCB; Extend # Mn   [3] GURMUKHI VOWEL SIGN OO..GURMUKHI SIGN VIRAMA
+0A51          ; InCB; Extend # Mn       GURMUKHI SIGN UDAAT
+0A70..0A71    ; InCB; Extend # Mn   [2] GURMUKHI TIPPI..GURMUKHI ADDAK
+0A75          ; InCB; Extend # Mn       GURMUKHI SIGN YAKASH
+0A81..0A82    ; InCB; Extend # Mn   [2] GUJARATI SIGN CANDRABINDU..GUJARATI SIGN ANUSVARA
 0ABC          ; InCB; Extend # Mn       GUJARATI SIGN NUKTA
+0AC1..0AC5    ; InCB; Extend # Mn   [5] GUJARATI VOWEL SIGN U..GUJARATI VOWEL SIGN CANDRA E
+0AC7..0AC8    ; InCB; Extend # Mn   [2] GUJARATI VOWEL SIGN E..GUJARATI VOWEL SIGN AI
+0AE2..0AE3    ; InCB; Extend # Mn   [2] GUJARATI VOWEL SIGN VOCALIC L..GUJARATI VOWEL SIGN VOCALIC LL
+0AFA..0AFF    ; InCB; Extend # Mn   [6] GUJARATI SIGN SUKUN..GUJARATI SIGN TWO-CIRCLE NUKTA ABOVE
+0B01          ; InCB; Extend # Mn       ORIYA SIGN CANDRABINDU
 0B3C          ; InCB; Extend # Mn       ORIYA SIGN NUKTA
+0B3E          ; InCB; Extend # Mc       ORIYA VOWEL SIGN AA
+0B3F          ; InCB; Extend # Mn       ORIYA VOWEL SIGN I
+0B41..0B44    ; InCB; Extend # Mn   [4] ORIYA VOWEL SIGN U..ORIYA VOWEL SIGN VOCALIC RR
+0B55..0B56    ; InCB; Extend # Mn   [2] ORIYA SIGN OVERLINE..ORIYA AI LENGTH MARK
+0B57          ; InCB; Extend # Mc       ORIYA AU LENGTH MARK
+0B62..0B63    ; InCB; Extend # Mn   [2] ORIYA VOWEL SIGN VOCALIC L..ORIYA VOWEL SIGN VOCALIC LL
+0B82          ; InCB; Extend # Mn       TAMIL SIGN ANUSVARA
+0BBE          ; InCB; Extend # Mc       TAMIL VOWEL SIGN AA
+0BC0          ; InCB; Extend # Mn       TAMIL VOWEL SIGN II
+0BCD          ; InCB; Extend # Mn       TAMIL SIGN VIRAMA
+0BD7          ; InCB; Extend # Mc       TAMIL AU LENGTH MARK
+0C00          ; InCB; Extend # Mn       TELUGU SIGN COMBINING CANDRABINDU ABOVE
+0C04          ; InCB; Extend # Mn       TELUGU SIGN COMBINING ANUSVARA ABOVE
 0C3C          ; InCB; Extend # Mn       TELUGU SIGN NUKTA
+0C3E..0C40    ; InCB; Extend # Mn   [3] TELUGU VOWEL SIGN AA..TELUGU VOWEL SIGN II
+0C46..0C48    ; InCB; Extend # Mn   [3] TELUGU VOWEL SIGN E..TELUGU VOWEL SIGN AI
+0C4A..0C4C    ; InCB; Extend # Mn   [3] TELUGU VOWEL SIGN O..TELUGU VOWEL SIGN AU
 0C55..0C56    ; InCB; Extend # Mn   [2] TELUGU LENGTH MARK..TELUGU AI LENGTH MARK
+0C62..0C63    ; InCB; Extend # Mn   [2] TELUGU VOWEL SIGN VOCALIC L..TELUGU VOWEL SIGN VOCALIC LL
+0C81          ; InCB; Extend # Mn       KANNADA SIGN CANDRABINDU
 0CBC          ; InCB; Extend # Mn       KANNADA SIGN NUKTA
+0CBF          ; InCB; Extend # Mn       KANNADA VOWEL SIGN I
+0CC0          ; InCB; Extend # Mc       KANNADA VOWEL SIGN II
+0CC2          ; InCB; Extend # Mc       KANNADA VOWEL SIGN UU
+0CC6          ; InCB; Extend # Mn       KANNADA VOWEL SIGN E
+0CC7..0CC8    ; InCB; Extend # Mc   [2] KANNADA VOWEL SIGN EE..KANNADA VOWEL SIGN AI
+0CCA..0CCB    ; InCB; Extend # Mc   [2] KANNADA VOWEL SIGN O..KANNADA VOWEL SIGN OO
+0CCC..0CCD    ; InCB; Extend # Mn   [2] KANNADA VOWEL SIGN AU..KANNADA SIGN VIRAMA
+0CD5..0CD6    ; InCB; Extend # Mc   [2] KANNADA LENGTH MARK..KANNADA AI LENGTH MARK
+0CE2..0CE3    ; InCB; Extend # Mn   [2] KANNADA VOWEL SIGN VOCALIC L..KANNADA VOWEL SIGN VOCALIC LL
+0D00..0D01    ; InCB; Extend # Mn   [2] MALAYALAM SIGN COMBINING ANUSVARA ABOVE..MALAYALAM SIGN CANDRABINDU
 0D3B..0D3C    ; InCB; Extend # Mn   [2] MALAYALAM SIGN VERTICAL BAR VIRAMA..MALAYALAM SIGN CIRCULAR VIRAMA
-0E38..0E3A    ; InCB; Extend # Mn   [3] THAI CHARACTER SARA U..THAI CHARACTER PHINTHU
-0E48..0E4B    ; InCB; Extend # Mn   [4] THAI CHARACTER MAI EK..THAI CHARACTER MAI CHATTAWA
-0EB8..0EBA    ; InCB; Extend # Mn   [3] LAO VOWEL SIGN U..LAO SIGN PALI VIRAMA
-0EC8..0ECB    ; InCB; Extend # Mn   [4] LAO TONE MAI EK..LAO TONE MAI CATAWA
+0D3E          ; InCB; Extend # Mc       MALAYALAM VOWEL SIGN AA
+0D41..0D44    ; InCB; Extend # Mn   [4] MALAYALAM VOWEL SIGN U..MALAYALAM VOWEL SIGN VOCALIC RR
+0D57          ; InCB; Extend # Mc       MALAYALAM AU LENGTH MARK
+0D62..0D63    ; InCB; Extend # Mn   [2] MALAYALAM VOWEL SIGN VOCALIC L..MALAYALAM VOWEL SIGN VOCALIC LL
+0D81          ; InCB; Extend # Mn       SINHALA SIGN CANDRABINDU
+0DCA          ; InCB; Extend # Mn       SINHALA SIGN AL-LAKUNA
+0DCF          ; InCB; Extend # Mc       SINHALA VOWEL SIGN AELA-PILLA
+0DD2..0DD4    ; InCB; Extend # Mn   [3] SINHALA VOWEL SIGN KETTI IS-PILLA..SINHALA VOWEL SIGN KETTI PAA-PILLA
+0DD6          ; InCB; Extend # Mn       SINHALA VOWEL SIGN DIGA PAA-PILLA
+0DDF          ; InCB; Extend # Mc       SINHALA VOWEL SIGN GAYANUKITTA
+0E31          ; InCB; Extend # Mn       THAI CHARACTER MAI HAN-AKAT
+0E34..0E3A    ; InCB; Extend # Mn   [7] THAI CHARACTER SARA I..THAI CHARACTER PHINTHU
+0E47..0E4E    ; InCB; Extend # Mn   [8] THAI CHARACTER MAITAIKHU..THAI CHARACTER YAMAKKAN
+0EB1          ; InCB; Extend # Mn       LAO VOWEL SIGN MAI KAN
+0EB4..0EBC    ; InCB; Extend # Mn   [9] LAO VOWEL SIGN I..LAO SEMIVOWEL SIGN LO
+0EC8..0ECE    ; InCB; Extend # Mn   [7] LAO TONE MAI EK..LAO YAMAKKAN
 0F18..0F19    ; InCB; Extend # Mn   [2] TIBETAN ASTROLOGICAL SIGN -KHYUD PA..TIBETAN ASTROLOGICAL SIGN SDONG TSHUGS
 0F35          ; InCB; Extend # Mn       TIBETAN MARK NGAS BZUNG NYI ZLA
 0F37          ; InCB; Extend # Mn       TIBETAN MARK NGAS BZUNG SGOR RTAGS
 0F39          ; InCB; Extend # Mn       TIBETAN MARK TSA -PHRU
-0F71..0F72    ; InCB; Extend # Mn   [2] TIBETAN VOWEL SIGN AA..TIBETAN VOWEL SIGN I
-0F74          ; InCB; Extend # Mn       TIBETAN VOWEL SIGN U
-0F7A..0F7D    ; InCB; Extend # Mn   [4] TIBETAN VOWEL SIGN E..TIBETAN VOWEL SIGN OO
-0F80          ; InCB; Extend # Mn       TIBETAN VOWEL SIGN REVERSED I
-0F82..0F84    ; InCB; Extend # Mn   [3] TIBETAN SIGN NYI ZLA NAA DA..TIBETAN MARK HALANTA
+0F71..0F7E    ; InCB; Extend # Mn  [14] TIBETAN VOWEL SIGN AA..TIBETAN SIGN RJES SU NGA RO
+0F80..0F84    ; InCB; Extend # Mn   [5] TIBETAN VOWEL SIGN REVERSED I..TIBETAN MARK HALANTA
 0F86..0F87    ; InCB; Extend # Mn   [2] TIBETAN SIGN LCI RTAGS..TIBETAN SIGN YANG RTAGS
+0F8D..0F97    ; InCB; Extend # Mn  [11] TIBETAN SUBJOINED SIGN LCE TSA CAN..TIBETAN SUBJOINED LETTER JA
+0F99..0FBC    ; InCB; Extend # Mn  [36] TIBETAN SUBJOINED LETTER NYA..TIBETAN SUBJOINED LETTER FIXED-FORM RA
 0FC6          ; InCB; Extend # Mn       TIBETAN SYMBOL PADMA GDAN
-1037          ; InCB; Extend # Mn       MYANMAR SIGN DOT BELOW
+102D..1030    ; InCB; Extend # Mn   [4] MYANMAR VOWEL SIGN I..MYANMAR VOWEL SIGN UU
+1032..1037    ; InCB; Extend # Mn   [6] MYANMAR VOWEL SIGN AI..MYANMAR SIGN DOT BELOW
 1039..103A    ; InCB; Extend # Mn   [2] MYANMAR SIGN VIRAMA..MYANMAR SIGN ASAT
+103D..103E    ; InCB; Extend # Mn   [2] MYANMAR CONSONANT SIGN MEDIAL WA..MYANMAR CONSONANT SIGN MEDIAL HA
+1058..1059    ; InCB; Extend # Mn   [2] MYANMAR VOWEL SIGN VOCALIC L..MYANMAR VOWEL SIGN VOCALIC LL
+105E..1060    ; InCB; Extend # Mn   [3] MYANMAR CONSONANT SIGN MON MEDIAL NA..MYANMAR CONSONANT SIGN MON MEDIAL LA
+1071..1074    ; InCB; Extend # Mn   [4] MYANMAR VOWEL SIGN GEBA KAREN I..MYANMAR VOWEL SIGN KAYAH EE
+1082          ; InCB; Extend # Mn       MYANMAR CONSONANT SIGN SHAN MEDIAL WA
+1085..1086    ; InCB; Extend # Mn   [2] MYANMAR VOWEL SIGN SHAN E ABOVE..MYANMAR VOWEL SIGN SHAN FINAL Y
 108D          ; InCB; Extend # Mn       MYANMAR SIGN SHAN COUNCIL EMPHATIC TONE
+109D          ; InCB; Extend # Mn       MYANMAR VOWEL SIGN AITON AI
 135D..135F    ; InCB; Extend # Mn   [3] ETHIOPIC COMBINING GEMINATION AND VOWEL LENGTH MARK..ETHIOPIC COMBINING GEMINATION MARK
-1714          ; InCB; Extend # Mn       TAGALOG SIGN VIRAMA
-17D2          ; InCB; Extend # Mn       KHMER SIGN COENG
+1712..1714    ; InCB; Extend # Mn   [3] TAGALOG VOWEL SIGN I..TAGALOG SIGN VIRAMA
+1715          ; InCB; Extend # Mc       TAGALOG SIGN PAMUDPOD
+1732..1733    ; InCB; Extend # Mn   [2] HANUNOO VOWEL SIGN I..HANUNOO VOWEL SIGN U
+1734          ; InCB; Extend # Mc       HANUNOO SIGN PAMUDPOD
+1752..1753    ; InCB; Extend # Mn   [2] BUHID VOWEL SIGN I..BUHID VOWEL SIGN U
+1772..1773    ; InCB; Extend # Mn   [2] TAGBANWA VOWEL SIGN I..TAGBANWA VOWEL SIGN U
+17B4..17B5    ; InCB; Extend # Mn   [2] KHMER VOWEL INHERENT AQ..KHMER VOWEL INHERENT AA
+17B7..17BD    ; InCB; Extend # Mn   [7] KHMER VOWEL SIGN I..KHMER VOWEL SIGN UA
+17C6          ; InCB; Extend # Mn       KHMER SIGN NIKAHIT
+17C9..17D3    ; InCB; Extend # Mn  [11] KHMER SIGN MUUSIKATOAN..KHMER SIGN BATHAMASAT
 17DD          ; InCB; Extend # Mn       KHMER SIGN ATTHACAN
+180B..180D    ; InCB; Extend # Mn   [3] MONGOLIAN FREE VARIATION SELECTOR ONE..MONGOLIAN FREE VARIATION SELECTOR THREE
+180F          ; InCB; Extend # Mn       MONGOLIAN FREE VARIATION SELECTOR FOUR
+1885..1886    ; InCB; Extend # Mn   [2] MONGOLIAN LETTER ALI GALI BALUDA..MONGOLIAN LETTER ALI GALI THREE BALUDA
 18A9          ; InCB; Extend # Mn       MONGOLIAN LETTER ALI GALI DAGALGA
+1920..1922    ; InCB; Extend # Mn   [3] LIMBU VOWEL SIGN A..LIMBU VOWEL SIGN U
+1927..1928    ; InCB; Extend # Mn   [2] LIMBU VOWEL SIGN E..LIMBU VOWEL SIGN O
+1932          ; InCB; Extend # Mn       LIMBU SMALL LETTER ANUSVARA
 1939..193B    ; InCB; Extend # Mn   [3] LIMBU SIGN MUKPHRENG..LIMBU SIGN SA-I
 1A17..1A18    ; InCB; Extend # Mn   [2] BUGINESE VOWEL SIGN I..BUGINESE VOWEL SIGN U
+1A1B          ; InCB; Extend # Mn       BUGINESE VOWEL SIGN AE
+1A56          ; InCB; Extend # Mn       TAI THAM CONSONANT SIGN MEDIAL LA
+1A58..1A5E    ; InCB; Extend # Mn   [7] TAI THAM SIGN MAI KANG LAI..TAI THAM CONSONANT SIGN SA
 1A60          ; InCB; Extend # Mn       TAI THAM SIGN SAKOT
-1A75..1A7C    ; InCB; Extend # Mn   [8] TAI THAM SIGN TONE-1..TAI THAM SIGN KHUEN-LUE KARAN
+1A62          ; InCB; Extend # Mn       TAI THAM VOWEL SIGN MAI SAT
+1A65..1A6C    ; InCB; Extend # Mn   [8] TAI THAM VOWEL SIGN I..TAI THAM VOWEL SIGN OA BELOW
+1A73..1A7C    ; InCB; Extend # Mn  [10] TAI THAM VOWEL SIGN OA ABOVE..TAI THAM SIGN KHUEN-LUE KARAN
 1A7F          ; InCB; Extend # Mn       TAI THAM COMBINING CRYPTOGRAMMIC DOT
 1AB0..1ABD    ; InCB; Extend # Mn  [14] COMBINING DOUBLED CIRCUMFLEX ACCENT..COMBINING PARENTHESES BELOW
+1ABE          ; InCB; Extend # Me       COMBINING PARENTHESES OVERLAY
 1ABF..1ACE    ; InCB; Extend # Mn  [16] COMBINING LATIN SMALL LETTER W BELOW..COMBINING LATIN SMALL LETTER INSULAR T
+1B00..1B03    ; InCB; Extend # Mn   [4] BALINESE SIGN ULU RICEM..BALINESE SIGN SURANG
 1B34          ; InCB; Extend # Mn       BALINESE SIGN REREKAN
+1B35          ; InCB; Extend # Mc       BALINESE VOWEL SIGN TEDUNG
+1B36..1B3A    ; InCB; Extend # Mn   [5] BALINESE VOWEL SIGN ULU..BALINESE VOWEL SIGN RA REPA
+1B3B          ; InCB; Extend # Mc       BALINESE VOWEL SIGN RA REPA TEDUNG
+1B3C          ; InCB; Extend # Mn       BALINESE VOWEL SIGN LA LENGA
+1B3D          ; InCB; Extend # Mc       BALINESE VOWEL SIGN LA LENGA TEDUNG
+1B42          ; InCB; Extend # Mn       BALINESE VOWEL SIGN PEPET
+1B43..1B44    ; InCB; Extend # Mc   [2] BALINESE VOWEL SIGN PEPET TEDUNG..BALINESE ADEG ADEG
 1B6B..1B73    ; InCB; Extend # Mn   [9] BALINESE MUSICAL SYMBOL COMBINING TEGEH..BALINESE MUSICAL SYMBOL COMBINING GONG
-1BAB          ; InCB; Extend # Mn       SUNDANESE SIGN VIRAMA
+1B80..1B81    ; InCB; Extend # Mn   [2] SUNDANESE SIGN PANYECEK..SUNDANESE SIGN PANGLAYAR
+1BA2..1BA5    ; InCB; Extend # Mn   [4] SUNDANESE CONSONANT SIGN PANYAKRA..SUNDANESE VOWEL SIGN PANYUKU
+1BA8..1BA9    ; InCB; Extend # Mn   [2] SUNDANESE VOWEL SIGN PAMEPET..SUNDANESE VOWEL SIGN PANEULEUNG
+1BAA          ; InCB; Extend # Mc       SUNDANESE SIGN PAMAAEH
+1BAB..1BAD    ; InCB; Extend # Mn   [3] SUNDANESE SIGN VIRAMA..SUNDANESE CONSONANT SIGN PASANGAN WA
 1BE6          ; InCB; Extend # Mn       BATAK SIGN TOMPI
-1C37          ; InCB; Extend # Mn       LEPCHA SIGN NUKTA
+1BE8..1BE9    ; InCB; Extend # Mn   [2] BATAK VOWEL SIGN PAKPAK E..BATAK VOWEL SIGN EE
+1BED          ; InCB; Extend # Mn       BATAK VOWEL SIGN KARO O
+1BEF..1BF1    ; InCB; Extend # Mn   [3] BATAK VOWEL SIGN U FOR SIMALUNGUN SA..BATAK CONSONANT SIGN H
+1BF2..1BF3    ; InCB; Extend # Mc   [2] BATAK PANGOLAT..BATAK PANONGONAN
+1C2C..1C33    ; InCB; Extend # Mn   [8] LEPCHA VOWEL SIGN E..LEPCHA CONSONANT SIGN T
+1C36..1C37    ; InCB; Extend # Mn   [2] LEPCHA SIGN RAN..LEPCHA SIGN NUKTA
 1CD0..1CD2    ; InCB; Extend # Mn   [3] VEDIC TONE KARSHANA..VEDIC TONE PRENKHA
 1CD4..1CE0    ; InCB; Extend # Mn  [13] VEDIC SIGN YAJURVEDIC MIDLINE SVARITA..VEDIC TONE RIGVEDIC KASHMIRI INDEPENDENT SVARITA
 1CE2..1CE8    ; InCB; Extend # Mn   [7] VEDIC SIGN VISARGA SVARITA..VEDIC SIGN VISARGA ANUDATTA WITH TAIL
@@ -12738,7 +13137,9 @@ ABED          ; Grapheme_Link # Mn       MEETEI MAYEK APUN IYEK
 1DC0..1DFF    ; InCB; Extend # Mn  [64] COMBINING DOTTED GRAVE ACCENT..COMBINING RIGHT ARROWHEAD AND DOWN ARROWHEAD BELOW
 200D          ; InCB; Extend # Cf       ZERO WIDTH JOINER
 20D0..20DC    ; InCB; Extend # Mn  [13] COMBINING LEFT HARPOON ABOVE..COMBINING FOUR DOTS ABOVE
+20DD..20E0    ; InCB; Extend # Me   [4] COMBINING ENCLOSING CIRCLE..COMBINING ENCLOSING CIRCLE BACKSLASH
 20E1          ; InCB; Extend # Mn       COMBINING LEFT RIGHT ARROW ABOVE
+20E2..20E4    ; InCB; Extend # Me   [3] COMBINING ENCLOSING SCREEN..COMBINING ENCLOSING UPWARD POINTING TRIANGLE
 20E5..20F0    ; InCB; Extend # Mn  [12] COMBINING REVERSE SOLIDUS OVERLAY..COMBINING ASTERISK ABOVE
 2CEF..2CF1    ; InCB; Extend # Mn   [3] COPTIC COMBINING NI ABOVE..COPTIC COMBINING SPIRITUS LENIS
 2D7F          ; InCB; Extend # Mn       TIFINAGH CONSONANT JOINER
@@ -12747,73 +13148,198 @@ ABED          ; Grapheme_Link # Mn       MEETEI MAYEK APUN IYEK
 302E..302F    ; InCB; Extend # Mc   [2] HANGUL SINGLE DOT TONE MARK..HANGUL DOUBLE DOT TONE MARK
 3099..309A    ; InCB; Extend # Mn   [2] COMBINING KATAKANA-HIRAGANA VOICED SOUND MARK..COMBINING KATAKANA-HIRAGANA SEMI-VOICED SOUND MARK
 A66F          ; InCB; Extend # Mn       COMBINING CYRILLIC VZMET
+A670..A672    ; InCB; Extend # Me   [3] COMBINING CYRILLIC TEN MILLIONS SIGN..COMBINING CYRILLIC THOUSAND MILLIONS SIGN
 A674..A67D    ; InCB; Extend # Mn  [10] COMBINING CYRILLIC LETTER UKRAINIAN IE..COMBINING CYRILLIC PAYEROK
 A69E..A69F    ; InCB; Extend # Mn   [2] COMBINING CYRILLIC LETTER EF..COMBINING CYRILLIC LETTER IOTIFIED E
 A6F0..A6F1    ; InCB; Extend # Mn   [2] BAMUM COMBINING MARK KOQNDON..BAMUM COMBINING MARK TUKWENTIS
+A802          ; InCB; Extend # Mn       SYLOTI NAGRI SIGN DVISVARA
+A806          ; InCB; Extend # Mn       SYLOTI NAGRI SIGN HASANTA
+A80B          ; InCB; Extend # Mn       SYLOTI NAGRI SIGN ANUSVARA
+A825..A826    ; InCB; Extend # Mn   [2] SYLOTI NAGRI VOWEL SIGN U..SYLOTI NAGRI VOWEL SIGN E
 A82C          ; InCB; Extend # Mn       SYLOTI NAGRI SIGN ALTERNATE HASANTA
+A8C4..A8C5    ; InCB; Extend # Mn   [2] SAURASHTRA SIGN VIRAMA..SAURASHTRA SIGN CANDRABINDU
 A8E0..A8F1    ; InCB; Extend # Mn  [18] COMBINING DEVANAGARI DIGIT ZERO..COMBINING DEVANAGARI SIGN AVAGRAHA
-A92B..A92D    ; InCB; Extend # Mn   [3] KAYAH LI TONE PLOPHU..KAYAH LI TONE CALYA PLOPHU
+A8FF          ; InCB; Extend # Mn       DEVANAGARI VOWEL SIGN AY
+A926..A92D    ; InCB; Extend # Mn   [8] KAYAH LI VOWEL UE..KAYAH LI TONE CALYA PLOPHU
+A947..A951    ; InCB; Extend # Mn  [11] REJANG VOWEL SIGN I..REJANG CONSONANT SIGN R
+A953          ; InCB; Extend # Mc       REJANG VIRAMA
+A980..A982    ; InCB; Extend # Mn   [3] JAVANESE SIGN PANYANGGA..JAVANESE SIGN LAYAR
 A9B3          ; InCB; Extend # Mn       JAVANESE SIGN CECAK TELU
+A9B6..A9B9    ; InCB; Extend # Mn   [4] JAVANESE VOWEL SIGN WULU..JAVANESE VOWEL SIGN SUKU MENDUT
+A9BC..A9BD    ; InCB; Extend # Mn   [2] JAVANESE VOWEL SIGN PEPET..JAVANESE CONSONANT SIGN KERET
+A9C0          ; InCB; Extend # Mc       JAVANESE PANGKON
+A9E5          ; InCB; Extend # Mn       MYANMAR SIGN SHAN SAW
+AA29..AA2E    ; InCB; Extend # Mn   [6] CHAM VOWEL SIGN AA..CHAM VOWEL SIGN OE
+AA31..AA32    ; InCB; Extend # Mn   [2] CHAM VOWEL SIGN AU..CHAM VOWEL SIGN UE
+AA35..AA36    ; InCB; Extend # Mn   [2] CHAM CONSONANT SIGN LA..CHAM CONSONANT SIGN WA
+AA43          ; InCB; Extend # Mn       CHAM CONSONANT SIGN FINAL NG
+AA4C          ; InCB; Extend # Mn       CHAM CONSONANT SIGN FINAL M
+AA7C          ; InCB; Extend # Mn       MYANMAR SIGN TAI LAING TONE-2
 AAB0          ; InCB; Extend # Mn       TAI VIET MAI KANG
 AAB2..AAB4    ; InCB; Extend # Mn   [3] TAI VIET VOWEL I..TAI VIET VOWEL U
 AAB7..AAB8    ; InCB; Extend # Mn   [2] TAI VIET MAI KHIT..TAI VIET VOWEL IA
 AABE..AABF    ; InCB; Extend # Mn   [2] TAI VIET VOWEL AM..TAI VIET TONE MAI EK
 AAC1          ; InCB; Extend # Mn       TAI VIET TONE MAI THO
+AAEC..AAED    ; InCB; Extend # Mn   [2] MEETEI MAYEK VOWEL SIGN UU..MEETEI MAYEK VOWEL SIGN AAI
 AAF6          ; InCB; Extend # Mn       MEETEI MAYEK VIRAMA
+ABE5          ; InCB; Extend # Mn       MEETEI MAYEK VOWEL SIGN ANAP
+ABE8          ; InCB; Extend # Mn       MEETEI MAYEK VOWEL SIGN UNAP
 ABED          ; InCB; Extend # Mn       MEETEI MAYEK APUN IYEK
 FB1E          ; InCB; Extend # Mn       HEBREW POINT JUDEO-SPANISH VARIKA
+FE00..FE0F    ; InCB; Extend # Mn  [16] VARIATION SELECTOR-1..VARIATION SELECTOR-16
 FE20..FE2F    ; InCB; Extend # Mn  [16] COMBINING LIGATURE LEFT HALF..COMBINING CYRILLIC TITLO RIGHT HALF
+FF9E..FF9F    ; InCB; Extend # Lm   [2] HALFWIDTH KATAKANA VOICED SOUND MARK..HALFWIDTH KATAKANA SEMI-VOICED SOUND MARK
 101FD         ; InCB; Extend # Mn       PHAISTOS DISC SIGN COMBINING OBLIQUE STROKE
 102E0         ; InCB; Extend # Mn       COPTIC EPACT THOUSANDS MARK
 10376..1037A  ; InCB; Extend # Mn   [5] COMBINING OLD PERMIC LETTER AN..COMBINING OLD PERMIC LETTER SII
-10A0D         ; InCB; Extend # Mn       KHAROSHTHI SIGN DOUBLE RING BELOW
-10A0F         ; InCB; Extend # Mn       KHAROSHTHI SIGN VISARGA
+10A01..10A03  ; InCB; Extend # Mn   [3] KHAROSHTHI VOWEL SIGN I..KHAROSHTHI VOWEL SIGN VOCALIC R
+10A05..10A06  ; InCB; Extend # Mn   [2] KHAROSHTHI VOWEL SIGN E..KHAROSHTHI VOWEL SIGN O
+10A0C..10A0F  ; InCB; Extend # Mn   [4] KHAROSHTHI VOWEL LENGTH MARK..KHAROSHTHI SIGN VISARGA
 10A38..10A3A  ; InCB; Extend # Mn   [3] KHAROSHTHI SIGN BAR ABOVE..KHAROSHTHI SIGN DOT BELOW
 10A3F         ; InCB; Extend # Mn       KHAROSHTHI VIRAMA
 10AE5..10AE6  ; InCB; Extend # Mn   [2] MANICHAEAN ABBREVIATION MARK ABOVE..MANICHAEAN ABBREVIATION MARK BELOW
 10D24..10D27  ; InCB; Extend # Mn   [4] HANIFI ROHINGYA SIGN HARBAHAY..HANIFI ROHINGYA SIGN TASSI
+10D69..10D6D  ; InCB; Extend # Mn   [5] GARAY VOWEL SIGN E..GARAY CONSONANT NASALIZATION MARK
 10EAB..10EAC  ; InCB; Extend # Mn   [2] YEZIDI COMBINING HAMZA MARK..YEZIDI COMBINING MADDA MARK
-10EFD..10EFF  ; InCB; Extend # Mn   [3] ARABIC SMALL LOW WORD SAKTA..ARABIC SMALL LOW WORD MADDA
+10EFC..10EFF  ; InCB; Extend # Mn   [4] ARABIC COMBINING ALEF OVERLAY..ARABIC SMALL LOW WORD MADDA
 10F46..10F50  ; InCB; Extend # Mn  [11] SOGDIAN COMBINING DOT BELOW..SOGDIAN COMBINING STROKE BELOW
 10F82..10F85  ; InCB; Extend # Mn   [4] OLD UYGHUR COMBINING DOT ABOVE..OLD UYGHUR COMBINING TWO DOTS BELOW
+11001         ; InCB; Extend # Mn       BRAHMI SIGN ANUSVARA
+11038..11046  ; InCB; Extend # Mn  [15] BRAHMI VOWEL SIGN AA..BRAHMI VIRAMA
 11070         ; InCB; Extend # Mn       BRAHMI SIGN OLD TAMIL VIRAMA
-1107F         ; InCB; Extend # Mn       BRAHMI NUMBER JOINER
-110BA         ; InCB; Extend # Mn       KAITHI SIGN NUKTA
+11073..11074  ; InCB; Extend # Mn   [2] BRAHMI VOWEL SIGN OLD TAMIL SHORT E..BRAHMI VOWEL SIGN OLD TAMIL SHORT O
+1107F..11081  ; InCB; Extend # Mn   [3] BRAHMI NUMBER JOINER..KAITHI SIGN ANUSVARA
+110B3..110B6  ; InCB; Extend # Mn   [4] KAITHI VOWEL SIGN U..KAITHI VOWEL SIGN AI
+110B9..110BA  ; InCB; Extend # Mn   [2] KAITHI SIGN VIRAMA..KAITHI SIGN NUKTA
+110C2         ; InCB; Extend # Mn       KAITHI VOWEL SIGN VOCALIC R
 11100..11102  ; InCB; Extend # Mn   [3] CHAKMA SIGN CANDRABINDU..CHAKMA SIGN VISARGA
-11133..11134  ; InCB; Extend # Mn   [2] CHAKMA VIRAMA..CHAKMA MAAYYAA
+11127..1112B  ; InCB; Extend # Mn   [5] CHAKMA VOWEL SIGN A..CHAKMA VOWEL SIGN UU
+1112D..11134  ; InCB; Extend # Mn   [8] CHAKMA VOWEL SIGN AI..CHAKMA MAAYYAA
 11173         ; InCB; Extend # Mn       MAHAJANI SIGN NUKTA
-111CA         ; InCB; Extend # Mn       SHARADA SIGN NUKTA
-11236         ; InCB; Extend # Mn       KHOJKI SIGN NUKTA
-112E9..112EA  ; InCB; Extend # Mn   [2] KHUDAWADI SIGN NUKTA..KHUDAWADI SIGN VIRAMA
+11180..11181  ; InCB; Extend # Mn   [2] SHARADA SIGN CANDRABINDU..SHARADA SIGN ANUSVARA
+111B6..111BE  ; InCB; Extend # Mn   [9] SHARADA VOWEL SIGN U..SHARADA VOWEL SIGN O
+111C0         ; InCB; Extend # Mc       SHARADA SIGN VIRAMA
+111C9..111CC  ; InCB; Extend # Mn   [4] SHARADA SANDHI MARK..SHARADA EXTRA SHORT VOWEL MARK
+111CF         ; InCB; Extend # Mn       SHARADA SIGN INVERTED CANDRABINDU
+1122F..11231  ; InCB; Extend # Mn   [3] KHOJKI VOWEL SIGN U..KHOJKI VOWEL SIGN AI
+11234         ; InCB; Extend # Mn       KHOJKI SIGN ANUSVARA
+11235         ; InCB; Extend # Mc       KHOJKI SIGN VIRAMA
+11236..11237  ; InCB; Extend # Mn   [2] KHOJKI SIGN NUKTA..KHOJKI SIGN SHADDA
+1123E         ; InCB; Extend # Mn       KHOJKI SIGN SUKUN
+11241         ; InCB; Extend # Mn       KHOJKI VOWEL SIGN VOCALIC R
+112DF         ; InCB; Extend # Mn       KHUDAWADI SIGN ANUSVARA
+112E3..112EA  ; InCB; Extend # Mn   [8] KHUDAWADI VOWEL SIGN U..KHUDAWADI SIGN VIRAMA
+11300..11301  ; InCB; Extend # Mn   [2] GRANTHA SIGN COMBINING ANUSVARA ABOVE..GRANTHA SIGN CANDRABINDU
 1133B..1133C  ; InCB; Extend # Mn   [2] COMBINING BINDU BELOW..GRANTHA SIGN NUKTA
+1133E         ; InCB; Extend # Mc       GRANTHA VOWEL SIGN AA
+11340         ; InCB; Extend # Mn       GRANTHA VOWEL SIGN II
+1134D         ; InCB; Extend # Mc       GRANTHA SIGN VIRAMA
+11357         ; InCB; Extend # Mc       GRANTHA AU LENGTH MARK
 11366..1136C  ; InCB; Extend # Mn   [7] COMBINING GRANTHA DIGIT ZERO..COMBINING GRANTHA DIGIT SIX
 11370..11374  ; InCB; Extend # Mn   [5] COMBINING GRANTHA LETTER A..COMBINING GRANTHA LETTER PA
+113B8         ; InCB; Extend # Mc       TULU-TIGALARI VOWEL SIGN AA
+113BB..113C0  ; InCB; Extend # Mn   [6] TULU-TIGALARI VOWEL SIGN U..TULU-TIGALARI VOWEL SIGN VOCALIC LL
+113C2         ; InCB; Extend # Mc       TULU-TIGALARI VOWEL SIGN EE
+113C5         ; InCB; Extend # Mc       TULU-TIGALARI VOWEL SIGN AI
+113C7..113C9  ; InCB; Extend # Mc   [3] TULU-TIGALARI VOWEL SIGN OO..TULU-TIGALARI AU LENGTH MARK
+113CE         ; InCB; Extend # Mn       TULU-TIGALARI SIGN VIRAMA
+113CF         ; InCB; Extend # Mc       TULU-TIGALARI SIGN LOOPED VIRAMA
+113D0         ; InCB; Extend # Mn       TULU-TIGALARI CONJOINER
+113D2         ; InCB; Extend # Mn       TULU-TIGALARI GEMINATION MARK
+113E1..113E2  ; InCB; Extend # Mn   [2] TULU-TIGALARI VEDIC TONE SVARITA..TULU-TIGALARI VEDIC TONE ANUDATTA
+11438..1143F  ; InCB; Extend # Mn   [8] NEWA VOWEL SIGN U..NEWA VOWEL SIGN AI
+11442..11444  ; InCB; Extend # Mn   [3] NEWA SIGN VIRAMA..NEWA SIGN ANUSVARA
 11446         ; InCB; Extend # Mn       NEWA SIGN NUKTA
 1145E         ; InCB; Extend # Mn       NEWA SANDHI MARK
-114C3         ; InCB; Extend # Mn       TIRHUTA SIGN NUKTA
-115C0         ; InCB; Extend # Mn       SIDDHAM SIGN NUKTA
+114B0         ; InCB; Extend # Mc       TIRHUTA VOWEL SIGN AA
+114B3..114B8  ; InCB; Extend # Mn   [6] TIRHUTA VOWEL SIGN U..TIRHUTA VOWEL SIGN VOCALIC LL
+114BA         ; InCB; Extend # Mn       TIRHUTA VOWEL SIGN SHORT E
+114BD         ; InCB; Extend # Mc       TIRHUTA VOWEL SIGN SHORT O
+114BF..114C0  ; InCB; Extend # Mn   [2] TIRHUTA SIGN CANDRABINDU..TIRHUTA SIGN ANUSVARA
+114C2..114C3  ; InCB; Extend # Mn   [2] TIRHUTA SIGN VIRAMA..TIRHUTA SIGN NUKTA
+115AF         ; InCB; Extend # Mc       SIDDHAM VOWEL SIGN AA
+115B2..115B5  ; InCB; Extend # Mn   [4] SIDDHAM VOWEL SIGN U..SIDDHAM VOWEL SIGN VOCALIC RR
+115BC..115BD  ; InCB; Extend # Mn   [2] SIDDHAM SIGN CANDRABINDU..SIDDHAM SIGN ANUSVARA
+115BF..115C0  ; InCB; Extend # Mn   [2] SIDDHAM SIGN VIRAMA..SIDDHAM SIGN NUKTA
+115DC..115DD  ; InCB; Extend # Mn   [2] SIDDHAM VOWEL SIGN ALTERNATE U..SIDDHAM VOWEL SIGN ALTERNATE UU
+11633..1163A  ; InCB; Extend # Mn   [8] MODI VOWEL SIGN U..MODI VOWEL SIGN AI
+1163D         ; InCB; Extend # Mn       MODI SIGN ANUSVARA
+1163F..11640  ; InCB; Extend # Mn   [2] MODI SIGN VIRAMA..MODI SIGN ARDHACANDRA
+116AB         ; InCB; Extend # Mn       TAKRI SIGN ANUSVARA
+116AD         ; InCB; Extend # Mn       TAKRI VOWEL SIGN AA
+116B0..116B5  ; InCB; Extend # Mn   [6] TAKRI VOWEL SIGN U..TAKRI VOWEL SIGN AU
+116B6         ; InCB; Extend # Mc       TAKRI SIGN VIRAMA
 116B7         ; InCB; Extend # Mn       TAKRI SIGN NUKTA
-1172B         ; InCB; Extend # Mn       AHOM SIGN KILLER
-1183A         ; InCB; Extend # Mn       DOGRA SIGN NUKTA
+1171D         ; InCB; Extend # Mn       AHOM CONSONANT SIGN MEDIAL LA
+1171F         ; InCB; Extend # Mn       AHOM CONSONANT SIGN MEDIAL LIGATING RA
+11722..11725  ; InCB; Extend # Mn   [4] AHOM VOWEL SIGN I..AHOM VOWEL SIGN UU
+11727..1172B  ; InCB; Extend # Mn   [5] AHOM VOWEL SIGN AW..AHOM SIGN KILLER
+1182F..11837  ; InCB; Extend # Mn   [9] DOGRA VOWEL SIGN U..DOGRA SIGN ANUSVARA
+11839..1183A  ; InCB; Extend # Mn   [2] DOGRA SIGN VIRAMA..DOGRA SIGN NUKTA
+11930         ; InCB; Extend # Mc       DIVES AKURU VOWEL SIGN AA
+1193B..1193C  ; InCB; Extend # Mn   [2] DIVES AKURU SIGN ANUSVARA..DIVES AKURU SIGN CANDRABINDU
+1193D         ; InCB; Extend # Mc       DIVES AKURU SIGN HALANTA
 1193E         ; InCB; Extend # Mn       DIVES AKURU VIRAMA
 11943         ; InCB; Extend # Mn       DIVES AKURU SIGN NUKTA
-11A34         ; InCB; Extend # Mn       ZANABAZAR SQUARE SIGN VIRAMA
+119D4..119D7  ; InCB; Extend # Mn   [4] NANDINAGARI VOWEL SIGN U..NANDINAGARI VOWEL SIGN VOCALIC RR
+119DA..119DB  ; InCB; Extend # Mn   [2] NANDINAGARI VOWEL SIGN E..NANDINAGARI VOWEL SIGN AI
+119E0         ; InCB; Extend # Mn       NANDINAGARI SIGN VIRAMA
+11A01..11A0A  ; InCB; Extend # Mn  [10] ZANABAZAR SQUARE VOWEL SIGN I..ZANABAZAR SQUARE VOWEL LENGTH MARK
+11A33..11A38  ; InCB; Extend # Mn   [6] ZANABAZAR SQUARE FINAL CONSONANT MARK..ZANABAZAR SQUARE SIGN ANUSVARA
+11A3B..11A3E  ; InCB; Extend # Mn   [4] ZANABAZAR SQUARE CLUSTER-FINAL LETTER YA..ZANABAZAR SQUARE CLUSTER-FINAL LETTER VA
 11A47         ; InCB; Extend # Mn       ZANABAZAR SQUARE SUBJOINER
-11A99         ; InCB; Extend # Mn       SOYOMBO SUBJOINER
-11D42         ; InCB; Extend # Mn       MASARAM GONDI SIGN NUKTA
-11D44..11D45  ; InCB; Extend # Mn   [2] MASARAM GONDI SIGN HALANTA..MASARAM GONDI VIRAMA
+11A51..11A56  ; InCB; Extend # Mn   [6] SOYOMBO VOWEL SIGN I..SOYOMBO VOWEL SIGN OE
+11A59..11A5B  ; InCB; Extend # Mn   [3] SOYOMBO VOWEL SIGN VOCALIC R..SOYOMBO VOWEL LENGTH MARK
+11A8A..11A96  ; InCB; Extend # Mn  [13] SOYOMBO FINAL CONSONANT SIGN G..SOYOMBO SIGN ANUSVARA
+11A98..11A99  ; InCB; Extend # Mn   [2] SOYOMBO GEMINATION MARK..SOYOMBO SUBJOINER
+11C30..11C36  ; InCB; Extend # Mn   [7] BHAIKSUKI VOWEL SIGN I..BHAIKSUKI VOWEL SIGN VOCALIC L
+11C38..11C3D  ; InCB; Extend # Mn   [6] BHAIKSUKI VOWEL SIGN E..BHAIKSUKI SIGN ANUSVARA
+11C3F         ; InCB; Extend # Mn       BHAIKSUKI SIGN VIRAMA
+11C92..11CA7  ; InCB; Extend # Mn  [22] MARCHEN SUBJOINED LETTER KA..MARCHEN SUBJOINED LETTER ZA
+11CAA..11CB0  ; InCB; Extend # Mn   [7] MARCHEN SUBJOINED LETTER RA..MARCHEN VOWEL SIGN AA
+11CB2..11CB3  ; InCB; Extend # Mn   [2] MARCHEN VOWEL SIGN U..MARCHEN VOWEL SIGN E
+11CB5..11CB6  ; InCB; Extend # Mn   [2] MARCHEN SIGN ANUSVARA..MARCHEN SIGN CANDRABINDU
+11D31..11D36  ; InCB; Extend # Mn   [6] MASARAM GONDI VOWEL SIGN AA..MASARAM GONDI VOWEL SIGN VOCALIC R
+11D3A         ; InCB; Extend # Mn       MASARAM GONDI VOWEL SIGN E
+11D3C..11D3D  ; InCB; Extend # Mn   [2] MASARAM GONDI VOWEL SIGN AI..MASARAM GONDI VOWEL SIGN O
+11D3F..11D45  ; InCB; Extend # Mn   [7] MASARAM GONDI VOWEL SIGN AU..MASARAM GONDI VIRAMA
+11D47         ; InCB; Extend # Mn       MASARAM GONDI RA-KARA
+11D90..11D91  ; InCB; Extend # Mn   [2] GUNJALA GONDI VOWEL SIGN EE..GUNJALA GONDI VOWEL SIGN AI
+11D95         ; InCB; Extend # Mn       GUNJALA GONDI SIGN ANUSVARA
 11D97         ; InCB; Extend # Mn       GUNJALA GONDI VIRAMA
+11EF3..11EF4  ; InCB; Extend # Mn   [2] MAKASAR VOWEL SIGN I..MAKASAR VOWEL SIGN U
+11F00..11F01  ; InCB; Extend # Mn   [2] KAWI SIGN CANDRABINDU..KAWI SIGN ANUSVARA
+11F36..11F3A  ; InCB; Extend # Mn   [5] KAWI VOWEL SIGN I..KAWI VOWEL SIGN VOCALIC R
+11F40         ; InCB; Extend # Mn       KAWI VOWEL SIGN EU
+11F41         ; InCB; Extend # Mc       KAWI SIGN KILLER
 11F42         ; InCB; Extend # Mn       KAWI CONJOINER
+11F5A         ; InCB; Extend # Mn       KAWI SIGN NUKTA
+13440         ; InCB; Extend # Mn       EGYPTIAN HIEROGLYPH MIRROR HORIZONTALLY
+13447..13455  ; InCB; Extend # Mn  [15] EGYPTIAN HIEROGLYPH MODIFIER DAMAGED AT TOP START..EGYPTIAN HIEROGLYPH MODIFIER DAMAGED
+1611E..16129  ; InCB; Extend # Mn  [12] GURUNG KHEMA VOWEL SIGN AA..GURUNG KHEMA VOWEL LENGTH MARK
+1612D..1612F  ; InCB; Extend # Mn   [3] GURUNG KHEMA SIGN ANUSVARA..GURUNG KHEMA SIGN THOLHOMA
 16AF0..16AF4  ; InCB; Extend # Mn   [5] BASSA VAH COMBINING HIGH TONE..BASSA VAH COMBINING HIGH-LOW TONE
 16B30..16B36  ; InCB; Extend # Mn   [7] PAHAWH HMONG MARK CIM TUB..PAHAWH HMONG MARK CIM TAUM
-1BC9E         ; InCB; Extend # Mn       DUPLOYAN DOUBLE MARK
-1D165         ; InCB; Extend # Mc       MUSICAL SYMBOL COMBINING STEM
+16F4F         ; InCB; Extend # Mn       MIAO SIGN CONSONANT MODIFIER BAR
+16F8F..16F92  ; InCB; Extend # Mn   [4] MIAO TONE RIGHT..MIAO TONE BELOW
+16FE4         ; InCB; Extend # Mn       KHITAN SMALL SCRIPT FILLER
+16FF0..16FF1  ; InCB; Extend # Mc   [2] VIETNAMESE ALTERNATE READING MARK CA..VIETNAMESE ALTERNATE READING MARK NHAY
+1BC9D..1BC9E  ; InCB; Extend # Mn   [2] DUPLOYAN THICK LETTER SELECTOR..DUPLOYAN DOUBLE MARK
+1CF00..1CF2D  ; InCB; Extend # Mn  [46] ZNAMENNY COMBINING MARK GORAZDO NIZKO S KRYZHEM ON LEFT..ZNAMENNY COMBINING MARK KRYZH ON LEFT
+1CF30..1CF46  ; InCB; Extend # Mn  [23] ZNAMENNY COMBINING TONAL RANGE MARK MRACHNO..ZNAMENNY PRIZNAK MODIFIER ROG
+1D165..1D166  ; InCB; Extend # Mc   [2] MUSICAL SYMBOL COMBINING STEM..MUSICAL SYMBOL COMBINING SPRECHGESANG STEM
 1D167..1D169  ; InCB; Extend # Mn   [3] MUSICAL SYMBOL COMBINING TREMOLO-1..MUSICAL SYMBOL COMBINING TREMOLO-3
-1D16E..1D172  ; InCB; Extend # Mc   [5] MUSICAL SYMBOL COMBINING FLAG-1..MUSICAL SYMBOL COMBINING FLAG-5
+1D16D..1D172  ; InCB; Extend # Mc   [6] MUSICAL SYMBOL COMBINING AUGMENTATION DOT..MUSICAL SYMBOL COMBINING FLAG-5
 1D17B..1D182  ; InCB; Extend # Mn   [8] MUSICAL SYMBOL COMBINING ACCENT..MUSICAL SYMBOL COMBINING LOURE
 1D185..1D18B  ; InCB; Extend # Mn   [7] MUSICAL SYMBOL COMBINING DOIT..MUSICAL SYMBOL COMBINING TRIPLE TONGUE
 1D1AA..1D1AD  ; InCB; Extend # Mn   [4] MUSICAL SYMBOL COMBINING DOWN BOW..MUSICAL SYMBOL COMBINING SNAP PIZZICATO
 1D242..1D244  ; InCB; Extend # Mn   [3] COMBINING GREEK MUSICAL TRISEME..COMBINING GREEK MUSICAL PENTASEME
+1DA00..1DA36  ; InCB; Extend # Mn  [55] SIGNWRITING HEAD RIM..SIGNWRITING AIR SUCKING IN
+1DA3B..1DA6C  ; InCB; Extend # Mn  [50] SIGNWRITING MOUTH CLOSED NEUTRAL..SIGNWRITING EXCITEMENT
+1DA75         ; InCB; Extend # Mn       SIGNWRITING UPPER BODY TILTING FROM HIP JOINTS
+1DA84         ; InCB; Extend # Mn       SIGNWRITING LOCATION HEAD NECK
+1DA9B..1DA9F  ; InCB; Extend # Mn   [5] SIGNWRITING FILL MODIFIER-2..SIGNWRITING FILL MODIFIER-6
+1DAA1..1DAAF  ; InCB; Extend # Mn  [15] SIGNWRITING ROTATION MODIFIER-2..SIGNWRITING ROTATION MODIFIER-16
 1E000..1E006  ; InCB; Extend # Mn   [7] COMBINING GLAGOLITIC LETTER AZU..COMBINING GLAGOLITIC LETTER ZHIVETE
 1E008..1E018  ; InCB; Extend # Mn  [17] COMBINING GLAGOLITIC LETTER ZEMLJA..COMBINING GLAGOLITIC LETTER HERU
 1E01B..1E021  ; InCB; Extend # Mn   [7] COMBINING GLAGOLITIC LETTER SHTA..COMBINING GLAGOLITIC LETTER YATI
@@ -12824,9 +13350,13 @@ FE20..FE2F    ; InCB; Extend # Mn  [16] COMBINING LIGATURE LEFT HALF..COMBINING
 1E2AE         ; InCB; Extend # Mn       TOTO SIGN RISING TONE
 1E2EC..1E2EF  ; InCB; Extend # Mn   [4] WANCHO TONE TUP..WANCHO TONE KOINI
 1E4EC..1E4EF  ; InCB; Extend # Mn   [4] NAG MUNDARI SIGN MUHOR..NAG MUNDARI SIGN SUTUH
+1E5EE..1E5EF  ; InCB; Extend # Mn   [2] OL ONAL SIGN MU..OL ONAL SIGN IKIR
 1E8D0..1E8D6  ; InCB; Extend # Mn   [7] MENDE KIKAKUI COMBINING NUMBER TEENS..MENDE KIKAKUI COMBINING NUMBER MILLIONS
 1E944..1E94A  ; InCB; Extend # Mn   [7] ADLAM ALIF LENGTHENER..ADLAM NUKTA
+1F3FB..1F3FF  ; InCB; Extend # Sk   [5] EMOJI MODIFIER FITZPATRICK TYPE-1-2..EMOJI MODIFIER FITZPATRICK TYPE-6
+E0020..E007F  ; InCB; Extend # Cf  [96] TAG SPACE..CANCEL TAG
+E0100..E01EF  ; InCB; Extend # Mn [240] VARIATION SELECTOR-17..VARIATION SELECTOR-256
 
-# Total code points: 884
+# Total code points: 2192
 
 # EOF
diff --git a/libcxx/utils/data/unicode/DerivedGeneralCategory.txt b/libcxx/utils/data/unicode/DerivedGeneralCategory.txt
index 285ffa8fb83ac..07bf7bca93d93 100644
--- a/libcxx/utils/data/unicode/DerivedGeneralCategory.txt
+++ b/libcxx/utils/data/unicode/DerivedGeneralCategory.txt
@@ -1,8 +1,8 @@
-# DerivedGeneralCategory-15.1.0.txt
-# Date: 2023-07-28, 23:34:02 GMT
-# © 2023 Unicode®, Inc.
+# DerivedGeneralCategory-16.0.0.txt
+# Date: 2024-04-30, 21:48:17 GMT
+# © 2024 Unicode®, Inc.
 # Unicode and the Unicode Logo are registered trademarks of Unicode, Inc. in the U.S. and other countries.
-# For terms of use, see https://www.unicode.org/terms_of_use.html
+# For terms of use and license, see https://www.unicode.org/terms_of_use.html
 #
 # Unicode Character Database
 #   For documentation, see https://www.unicode.org/reports/tr44/
@@ -37,7 +37,7 @@
 085F          ; Cn #       <reserved-085F>
 086B..086F    ; Cn #   [5] <reserved-086B>..<reserved-086F>
 088F          ; Cn #       <reserved-088F>
-0892..0897    ; Cn #   [6] <reserved-0892>..<reserved-0897>
+0892..0896    ; Cn #   [5] <reserved-0892>..<reserved-0896>
 0984          ; Cn #       <reserved-0984>
 098D..098E    ; Cn #   [2] <reserved-098D>..<reserved-098E>
 0991..0992    ; Cn #   [2] <reserved-0991>..<reserved-0992>
@@ -229,12 +229,11 @@
 1A9A..1A9F    ; Cn #   [6] <reserved-1A9A>..<reserved-1A9F>
 1AAE..1AAF    ; Cn #   [2] <reserved-1AAE>..<reserved-1AAF>
 1ACF..1AFF    ; Cn #  [49] <reserved-1ACF>..<reserved-1AFF>
-1B4D..1B4F    ; Cn #   [3] <reserved-1B4D>..<reserved-1B4F>
-1B7F          ; Cn #       <reserved-1B7F>
+1B4D          ; Cn #       <reserved-1B4D>
 1BF4..1BFB    ; Cn #   [8] <reserved-1BF4>..<reserved-1BFB>
 1C38..1C3A    ; Cn #   [3] <reserved-1C38>..<reserved-1C3A>
 1C4A..1C4C    ; Cn #   [3] <reserved-1C4A>..<reserved-1C4C>
-1C89..1C8F    ; Cn #   [7] <reserved-1C89>..<reserved-1C8F>
+1C8B..1C8F    ; Cn #   [5] <reserved-1C8B>..<reserved-1C8F>
 1CBB..1CBC    ; Cn #   [2] <reserved-1CBB>..<reserved-1CBC>
 1CC8..1CCF    ; Cn #   [8] <reserved-1CC8>..<reserved-1CCF>
 1CFB..1CFF    ; Cn #   [5] <reserved-1CFB>..<reserved-1CFF>
@@ -261,7 +260,7 @@
 20C1..20CF    ; Cn #  [15] <reserved-20C1>..<reserved-20CF>
 20F1..20FF    ; Cn #  [15] <reserved-20F1>..<reserved-20FF>
 218C..218F    ; Cn #   [4] <reserved-218C>..<reserved-218F>
-2427..243F    ; Cn #  [25] <reserved-2427>..<reserved-243F>
+242A..243F    ; Cn #  [22] <reserved-242A>..<reserved-243F>
 244B..245F    ; Cn #  [21] <reserved-244B>..<reserved-245F>
 2B74..2B75    ; Cn #   [2] <reserved-2B74>..<reserved-2B75>
 2B96          ; Cn #       <reserved-2B96>
@@ -289,16 +288,16 @@
 3100..3104    ; Cn #   [5] <reserved-3100>..<reserved-3104>
 3130          ; Cn #       <reserved-3130>
 318F          ; Cn #       <reserved-318F>
-31E4..31EE    ; Cn #  [11] <reserved-31E4>..<reserved-31EE>
+31E6..31EE    ; Cn #   [9] <reserved-31E6>..<reserved-31EE>
 321F          ; Cn #       <reserved-321F>
 A48D..A48F    ; Cn #   [3] <reserved-A48D>..<reserved-A48F>
 A4C7..A4CF    ; Cn #   [9] <reserved-A4C7>..<reserved-A4CF>
 A62C..A63F    ; Cn #  [20] <reserved-A62C>..<reserved-A63F>
 A6F8..A6FF    ; Cn #   [8] <reserved-A6F8>..<reserved-A6FF>
-A7CB..A7CF    ; Cn #   [5] <reserved-A7CB>..<reserved-A7CF>
+A7CE..A7CF    ; Cn #   [2] <reserved-A7CE>..<reserved-A7CF>
 A7D2          ; Cn #       <reserved-A7D2>
 A7D4          ; Cn #       <reserved-A7D4>
-A7DA..A7F1    ; Cn #  [24] <reserved-A7DA>..<reserved-A7F1>
+A7DD..A7F1    ; Cn #  [21] <reserved-A7DD>..<reserved-A7F1>
 A82D..A82F    ; Cn #   [3] <reserved-A82D>..<reserved-A82F>
 A83A..A83F    ; Cn #   [6] <reserved-A83A>..<reserved-A83F>
 A878..A87F    ; Cn #   [8] <reserved-A878>..<reserved-A87F>
@@ -388,7 +387,8 @@ FFFE..FFFF    ; Cn #   [2] <noncharacter-FFFE>..<noncharacter-FFFF>
 105A2         ; Cn #       <reserved-105A2>
 105B2         ; Cn #       <reserved-105B2>
 105BA         ; Cn #       <reserved-105BA>
-105BD..105FF  ; Cn #  [67] <reserved-105BD>..<reserved-105FF>
+105BD..105BF  ; Cn #   [3] <reserved-105BD>..<reserved-105BF>
+105F4..105FF  ; Cn #  [12] <reserved-105F4>..<reserved-105FF>
 10737..1073F  ; Cn #   [9] <reserved-10737>..<reserved-1073F>
 10756..1075F  ; Cn #  [10] <reserved-10756>..<reserved-1075F>
 10768..1077F  ; Cn #  [24] <reserved-10768>..<reserved-1077F>
@@ -431,11 +431,15 @@ FFFE..FFFF    ; Cn #   [2] <noncharacter-FFFE>..<noncharacter-FFFF>
 10CB3..10CBF  ; Cn #  [13] <reserved-10CB3>..<reserved-10CBF>
 10CF3..10CF9  ; Cn #   [7] <reserved-10CF3>..<reserved-10CF9>
 10D28..10D2F  ; Cn #   [8] <reserved-10D28>..<reserved-10D2F>
-10D3A..10E5F  ; Cn # [294] <reserved-10D3A>..<reserved-10E5F>
+10D3A..10D3F  ; Cn #   [6] <reserved-10D3A>..<reserved-10D3F>
+10D66..10D68  ; Cn #   [3] <reserved-10D66>..<reserved-10D68>
+10D86..10D8D  ; Cn #   [8] <reserved-10D86>..<reserved-10D8D>
+10D90..10E5F  ; Cn # [208] <reserved-10D90>..<reserved-10E5F>
 10E7F         ; Cn #       <reserved-10E7F>
 10EAA         ; Cn #       <reserved-10EAA>
 10EAE..10EAF  ; Cn #   [2] <reserved-10EAE>..<reserved-10EAF>
-10EB2..10EFC  ; Cn #  [75] <reserved-10EB2>..<reserved-10EFC>
+10EB2..10EC1  ; Cn #  [16] <reserved-10EB2>..<reserved-10EC1>
+10EC5..10EFB  ; Cn #  [55] <reserved-10EC5>..<reserved-10EFB>
 10F28..10F2F  ; Cn #   [8] <reserved-10F28>..<reserved-10F2F>
 10F5A..10F6F  ; Cn #  [22] <reserved-10F5A>..<reserved-10F6F>
 10F8A..10FAF  ; Cn #  [38] <reserved-10F8A>..<reserved-10FAF>
@@ -475,7 +479,18 @@ FFFE..FFFF    ; Cn #   [2] <noncharacter-FFFE>..<noncharacter-FFFF>
 11358..1135C  ; Cn #   [5] <reserved-11358>..<reserved-1135C>
 11364..11365  ; Cn #   [2] <reserved-11364>..<reserved-11365>
 1136D..1136F  ; Cn #   [3] <reserved-1136D>..<reserved-1136F>
-11375..113FF  ; Cn # [139] <reserved-11375>..<reserved-113FF>
+11375..1137F  ; Cn #  [11] <reserved-11375>..<reserved-1137F>
+1138A         ; Cn #       <reserved-1138A>
+1138C..1138D  ; Cn #   [2] <reserved-1138C>..<reserved-1138D>
+1138F         ; Cn #       <reserved-1138F>
+113B6         ; Cn #       <reserved-113B6>
+113C1         ; Cn #       <reserved-113C1>
+113C3..113C4  ; Cn #   [2] <reserved-113C3>..<reserved-113C4>
+113C6         ; Cn #       <reserved-113C6>
+113CB         ; Cn #       <reserved-113CB>
+113D6         ; Cn #       <reserved-113D6>
+113D9..113E0  ; Cn #   [8] <reserved-113D9>..<reserved-113E0>
+113E3..113FF  ; Cn #  [29] <reserved-113E3>..<reserved-113FF>
 1145C         ; Cn #       <reserved-1145C>
 11462..1147F  ; Cn #  [30] <reserved-11462>..<reserved-1147F>
 114C8..114CF  ; Cn #   [8] <reserved-114C8>..<reserved-114CF>
@@ -486,7 +501,8 @@ FFFE..FFFF    ; Cn #   [2] <noncharacter-FFFE>..<noncharacter-FFFF>
 1165A..1165F  ; Cn #   [6] <reserved-1165A>..<reserved-1165F>
 1166D..1167F  ; Cn #  [19] <reserved-1166D>..<reserved-1167F>
 116BA..116BF  ; Cn #   [6] <reserved-116BA>..<reserved-116BF>
-116CA..116FF  ; Cn #  [54] <reserved-116CA>..<reserved-116FF>
+116CA..116CF  ; Cn #   [6] <reserved-116CA>..<reserved-116CF>
+116E4..116FF  ; Cn #  [28] <reserved-116E4>..<reserved-116FF>
 1171B..1171C  ; Cn #   [2] <reserved-1171B>..<reserved-1171C>
 1172C..1172F  ; Cn #   [4] <reserved-1172C>..<reserved-1172F>
 11747..117FF  ; Cn # [185] <reserved-11747>..<reserved-117FF>
@@ -506,7 +522,9 @@ FFFE..FFFF    ; Cn #   [2] <noncharacter-FFFE>..<noncharacter-FFFF>
 11A48..11A4F  ; Cn #   [8] <reserved-11A48>..<reserved-11A4F>
 11AA3..11AAF  ; Cn #  [13] <reserved-11AA3>..<reserved-11AAF>
 11AF9..11AFF  ; Cn #   [7] <reserved-11AF9>..<reserved-11AFF>
-11B0A..11BFF  ; Cn # [246] <reserved-11B0A>..<reserved-11BFF>
+11B0A..11BBF  ; Cn # [182] <reserved-11B0A>..<reserved-11BBF>
+11BE2..11BEF  ; Cn #  [14] <reserved-11BE2>..<reserved-11BEF>
+11BFA..11BFF  ; Cn #   [6] <reserved-11BFA>..<reserved-11BFF>
 11C09         ; Cn #       <reserved-11C09>
 11C37         ; Cn #       <reserved-11C37>
 11C46..11C4F  ; Cn #  [10] <reserved-11C46>..<reserved-11C4F>
@@ -530,7 +548,7 @@ FFFE..FFFF    ; Cn #   [2] <noncharacter-FFFE>..<noncharacter-FFFF>
 11EF9..11EFF  ; Cn #   [7] <reserved-11EF9>..<reserved-11EFF>
 11F11         ; Cn #       <reserved-11F11>
 11F3B..11F3D  ; Cn #   [3] <reserved-11F3B>..<reserved-11F3D>
-11F5A..11FAF  ; Cn #  [86] <reserved-11F5A>..<reserved-11FAF>
+11F5B..11FAF  ; Cn #  [85] <reserved-11F5B>..<reserved-11FAF>
 11FB1..11FBF  ; Cn #  [15] <reserved-11FB1>..<reserved-11FBF>
 11FF2..11FFE  ; Cn #  [13] <reserved-11FF2>..<reserved-11FFE>
 1239A..123FF  ; Cn # [102] <reserved-1239A>..<reserved-123FF>
@@ -538,8 +556,10 @@ FFFE..FFFF    ; Cn #   [2] <noncharacter-FFFE>..<noncharacter-FFFF>
 12475..1247F  ; Cn #  [11] <reserved-12475>..<reserved-1247F>
 12544..12F8F  ; Cn # [2636] <reserved-12544>..<reserved-12F8F>
 12FF3..12FFF  ; Cn #  [13] <reserved-12FF3>..<reserved-12FFF>
-13456..143FF  ; Cn # [4010] <reserved-13456>..<reserved-143FF>
-14647..167FF  ; Cn # [8633] <reserved-14647>..<reserved-167FF>
+13456..1345F  ; Cn #  [10] <reserved-13456>..<reserved-1345F>
+143FB..143FF  ; Cn #   [5] <reserved-143FB>..<reserved-143FF>
+14647..160FF  ; Cn # [6841] <reserved-14647>..<reserved-160FF>
+1613A..167FF  ; Cn # [1734] <reserved-1613A>..<reserved-167FF>
 16A39..16A3F  ; Cn #   [7] <reserved-16A39>..<reserved-16A3F>
 16A5F         ; Cn #       <reserved-16A5F>
 16A6A..16A6D  ; Cn #   [4] <reserved-16A6A>..<reserved-16A6D>
@@ -551,7 +571,8 @@ FFFE..FFFF    ; Cn #   [2] <noncharacter-FFFE>..<noncharacter-FFFF>
 16B5A         ; Cn #       <reserved-16B5A>
 16B62         ; Cn #       <reserved-16B62>
 16B78..16B7C  ; Cn #   [5] <reserved-16B78>..<reserved-16B7C>
-16B90..16E3F  ; Cn # [688] <reserved-16B90>..<reserved-16E3F>
+16B90..16D3F  ; Cn # [432] <reserved-16B90>..<reserved-16D3F>
+16D7A..16E3F  ; Cn # [198] <reserved-16D7A>..<reserved-16E3F>
 16E9B..16EFF  ; Cn # [101] <reserved-16E9B>..<reserved-16EFF>
 16F4B..16F4E  ; Cn #   [4] <reserved-16F4B>..<reserved-16F4E>
 16F88..16F8E  ; Cn #   [7] <reserved-16F88>..<reserved-16F8E>
@@ -559,7 +580,7 @@ FFFE..FFFF    ; Cn #   [2] <noncharacter-FFFE>..<noncharacter-FFFF>
 16FE5..16FEF  ; Cn #  [11] <reserved-16FE5>..<reserved-16FEF>
 16FF2..16FFF  ; Cn #  [14] <reserved-16FF2>..<reserved-16FFF>
 187F8..187FF  ; Cn #   [8] <reserved-187F8>..<reserved-187FF>
-18CD6..18CFF  ; Cn #  [42] <reserved-18CD6>..<reserved-18CFF>
+18CD6..18CFE  ; Cn #  [41] <reserved-18CD6>..<reserved-18CFE>
 18D09..1AFEF  ; Cn # [8935] <reserved-18D09>..<reserved-1AFEF>
 1AFF4         ; Cn #       <reserved-1AFF4>
 1AFFC         ; Cn #       <reserved-1AFFC>
@@ -574,7 +595,9 @@ FFFE..FFFF    ; Cn #   [2] <noncharacter-FFFE>..<noncharacter-FFFF>
 1BC7D..1BC7F  ; Cn #   [3] <reserved-1BC7D>..<reserved-1BC7F>
 1BC89..1BC8F  ; Cn #   [7] <reserved-1BC89>..<reserved-1BC8F>
 1BC9A..1BC9B  ; Cn #   [2] <reserved-1BC9A>..<reserved-1BC9B>
-1BCA4..1CEFF  ; Cn # [4700] <reserved-1BCA4>..<reserved-1CEFF>
+1BCA4..1CBFF  ; Cn # [3932] <reserved-1BCA4>..<reserved-1CBFF>
+1CCFA..1CCFF  ; Cn #   [6] <reserved-1CCFA>..<reserved-1CCFF>
+1CEB4..1CEFF  ; Cn #  [76] <reserved-1CEB4>..<reserved-1CEFF>
 1CF2E..1CF2F  ; Cn #   [2] <reserved-1CF2E>..<reserved-1CF2F>
 1CF47..1CF4F  ; Cn #   [9] <reserved-1CF47>..<reserved-1CF4F>
 1CFC4..1CFFF  ; Cn #  [60] <reserved-1CFC4>..<reserved-1CFFF>
@@ -625,7 +648,9 @@ FFFE..FFFF    ; Cn #   [2] <noncharacter-FFFE>..<noncharacter-FFFF>
 1E2AF..1E2BF  ; Cn #  [17] <reserved-1E2AF>..<reserved-1E2BF>
 1E2FA..1E2FE  ; Cn #   [5] <reserved-1E2FA>..<reserved-1E2FE>
 1E300..1E4CF  ; Cn # [464] <reserved-1E300>..<reserved-1E4CF>
-1E4FA..1E7DF  ; Cn # [742] <reserved-1E4FA>..<reserved-1E7DF>
+1E4FA..1E5CF  ; Cn # [214] <reserved-1E4FA>..<reserved-1E5CF>
+1E5FB..1E5FE  ; Cn #   [4] <reserved-1E5FB>..<reserved-1E5FE>
+1E600..1E7DF  ; Cn # [480] <reserved-1E600>..<reserved-1E7DF>
 1E7E7         ; Cn #       <reserved-1E7E7>
 1E7EC         ; Cn #       <reserved-1E7EC>
 1E7EF         ; Cn #       <reserved-1E7EF>
@@ -695,18 +720,17 @@ FFFE..FFFF    ; Cn #   [2] <noncharacter-FFFE>..<noncharacter-FFFF>
 1F85A..1F85F  ; Cn #   [6] <reserved-1F85A>..<reserved-1F85F>
 1F888..1F88F  ; Cn #   [8] <reserved-1F888>..<reserved-1F88F>
 1F8AE..1F8AF  ; Cn #   [2] <reserved-1F8AE>..<reserved-1F8AF>
-1F8B2..1F8FF  ; Cn #  [78] <reserved-1F8B2>..<reserved-1F8FF>
+1F8BC..1F8BF  ; Cn #   [4] <reserved-1F8BC>..<reserved-1F8BF>
+1F8C2..1F8FF  ; Cn #  [62] <reserved-1F8C2>..<reserved-1F8FF>
 1FA54..1FA5F  ; Cn #  [12] <reserved-1FA54>..<reserved-1FA5F>
 1FA6E..1FA6F  ; Cn #   [2] <reserved-1FA6E>..<reserved-1FA6F>
 1FA7D..1FA7F  ; Cn #   [3] <reserved-1FA7D>..<reserved-1FA7F>
-1FA89..1FA8F  ; Cn #   [7] <reserved-1FA89>..<reserved-1FA8F>
-1FABE         ; Cn #       <reserved-1FABE>
-1FAC6..1FACD  ; Cn #   [8] <reserved-1FAC6>..<reserved-1FACD>
-1FADC..1FADF  ; Cn #   [4] <reserved-1FADC>..<reserved-1FADF>
-1FAE9..1FAEF  ; Cn #   [7] <reserved-1FAE9>..<reserved-1FAEF>
+1FA8A..1FA8E  ; Cn #   [5] <reserved-1FA8A>..<reserved-1FA8E>
+1FAC7..1FACD  ; Cn #   [7] <reserved-1FAC7>..<reserved-1FACD>
+1FADD..1FADE  ; Cn #   [2] <reserved-1FADD>..<reserved-1FADE>
+1FAEA..1FAEF  ; Cn #   [6] <reserved-1FAEA>..<reserved-1FAEF>
 1FAF9..1FAFF  ; Cn #   [7] <reserved-1FAF9>..<reserved-1FAFF>
 1FB93         ; Cn #       <reserved-1FB93>
-1FBCB..1FBEF  ; Cn #  [37] <reserved-1FBCB>..<reserved-1FBEF>
 1FBFA..1FFFF  ; Cn # [1030] <reserved-1FBFA>..<noncharacter-1FFFF>
 2A6E0..2A6FF  ; Cn #  [32] <reserved-2A6E0>..<reserved-2A6FF>
 2B73A..2B73F  ; Cn #   [6] <reserved-2B73A>..<reserved-2B73F>
@@ -723,7 +747,7 @@ E01F0..EFFFF  ; Cn # [65040] <reserved-E01F0>..<noncharacter-EFFFF>
 FFFFE..FFFFF  ; Cn #   [2] <noncharacter-FFFFE>..<noncharacter-FFFFF>
 10FFFE..10FFFF; Cn #   [2] <noncharacter-10FFFE>..<noncharacter-10FFFF>
 
-# Total code points: 824718
+# Total code points: 819533
 
 # ================================================
 
@@ -1005,6 +1029,7 @@ FFFFE..FFFFF  ; Cn #   [2] <noncharacter-FFFFE>..<noncharacter-FFFFF>
 10C7          ; Lu #       GEORGIAN CAPITAL LETTER YN
 10CD          ; Lu #       GEORGIAN CAPITAL LETTER AEN
 13A0..13F5    ; Lu #  [86] CHEROKEE LETTER A..CHEROKEE LETTER MV
+1C89          ; Lu #       CYRILLIC CAPITAL LETTER TJE
 1C90..1CBA    ; Lu #  [43] GEORGIAN MTAVRULI CAPITAL LETTER AN..GEORGIAN MTAVRULI CAPITAL LETTER AIN
 1CBD..1CBF    ; Lu #   [3] GEORGIAN MTAVRULI CAPITAL LETTER AEN..GEORGIAN MTAVRULI CAPITAL LETTER LABIAL SIGN
 1E00          ; Lu #       LATIN CAPITAL LETTER A WITH RING BELOW
@@ -1329,9 +1354,12 @@ A7C0          ; Lu #       LATIN CAPITAL LETTER OLD POLISH O
 A7C2          ; Lu #       LATIN CAPITAL LETTER ANGLICANA W
 A7C4..A7C7    ; Lu #   [4] LATIN CAPITAL LETTER C WITH PALATAL HOOK..LATIN CAPITAL LETTER D WITH SHORT STROKE OVERLAY
 A7C9          ; Lu #       LATIN CAPITAL LETTER S WITH SHORT STROKE OVERLAY
+A7CB..A7CC    ; Lu #   [2] LATIN CAPITAL LETTER RAMS HORN..LATIN CAPITAL LETTER S WITH DIAGONAL STROKE
 A7D0          ; Lu #       LATIN CAPITAL LETTER CLOSED INSULAR G
 A7D6          ; Lu #       LATIN CAPITAL LETTER MIDDLE SCOTS S
 A7D8          ; Lu #       LATIN CAPITAL LETTER SIGMOID S
+A7DA          ; Lu #       LATIN CAPITAL LETTER LAMBDA
+A7DC          ; Lu #       LATIN CAPITAL LETTER LAMBDA WITH STROKE
 A7F5          ; Lu #       LATIN CAPITAL LETTER REVERSED HALF H
 FF21..FF3A    ; Lu #  [26] FULLWIDTH LATIN CAPITAL LETTER A..FULLWIDTH LATIN CAPITAL LETTER Z
 10400..10427  ; Lu #  [40] DESERET CAPITAL LETTER LONG I..DESERET CAPITAL LETTER EW
@@ -1341,6 +1369,7 @@ FF21..FF3A    ; Lu #  [26] FULLWIDTH LATIN CAPITAL LETTER A..FULLWIDTH LATIN CAP
 1058C..10592  ; Lu #   [7] VITHKUQI CAPITAL LETTER SE..VITHKUQI CAPITAL LETTER XE
 10594..10595  ; Lu #   [2] VITHKUQI CAPITAL LETTER Y..VITHKUQI CAPITAL LETTER ZE
 10C80..10CB2  ; Lu #  [51] OLD HUNGARIAN CAPITAL LETTER A..OLD HUNGARIAN CAPITAL LETTER US
+10D50..10D65  ; Lu #  [22] GARAY CAPITAL LETTER A..GARAY CAPITAL LETTER OLD NA
 118A0..118BF  ; Lu #  [32] WARANG CITI CAPITAL LETTER NGAA..WARANG CITI CAPITAL LETTER VIYO
 16E40..16E5F  ; Lu #  [32] MEDEFAIDRIN CAPITAL LETTER M..MEDEFAIDRIN CAPITAL LETTER Y
 1D400..1D419  ; Lu #  [26] MATHEMATICAL BOLD CAPITAL A..MATHEMATICAL BOLD CAPITAL Z
@@ -1376,7 +1405,7 @@ FF21..FF3A    ; Lu #  [26] FULLWIDTH LATIN CAPITAL LETTER A..FULLWIDTH LATIN CAP
 1D7CA         ; Lu #       MATHEMATICAL BOLD CAPITAL DIGAMMA
 1E900..1E921  ; Lu #  [34] ADLAM CAPITAL LETTER ALIF..ADLAM CAPITAL LETTER SHA
 
-# Total code points: 1831
+# Total code points: 1858
 
 # ================================================
 
@@ -1656,6 +1685,7 @@ FF21..FF3A    ; Lu #  [26] FULLWIDTH LATIN CAPITAL LETTER A..FULLWIDTH LATIN CAP
 10FD..10FF    ; Ll #   [3] GEORGIAN LETTER AEN..GEORGIAN LETTER LABIAL SIGN
 13F8..13FD    ; Ll #   [6] CHEROKEE SMALL LETTER YE..CHEROKEE SMALL LETTER MV
 1C80..1C88    ; Ll #   [9] CYRILLIC SMALL LETTER ROUNDED VE..CYRILLIC SMALL LETTER UNBLENDED UK
+1C8A          ; Ll #       CYRILLIC SMALL LETTER TJE
 1D00..1D2B    ; Ll #  [44] LATIN LETTER SMALL CAPITAL A..CYRILLIC LETTER SMALL CAPITAL EL
 1D6B..1D77    ; Ll #  [13] LATIN SMALL LETTER UE..LATIN SMALL LETTER TURNED G
 1D79..1D9A    ; Ll #  [34] LATIN SMALL LETTER INSULAR G..LATIN SMALL LETTER EZH WITH RETROFLEX HOOK
@@ -1986,11 +2016,13 @@ A7C1          ; Ll #       LATIN SMALL LETTER OLD POLISH O
 A7C3          ; Ll #       LATIN SMALL LETTER ANGLICANA W
 A7C8          ; Ll #       LATIN SMALL LETTER D WITH SHORT STROKE OVERLAY
 A7CA          ; Ll #       LATIN SMALL LETTER S WITH SHORT STROKE OVERLAY
+A7CD          ; Ll #       LATIN SMALL LETTER S WITH DIAGONAL STROKE
 A7D1          ; Ll #       LATIN SMALL LETTER CLOSED INSULAR G
 A7D3          ; Ll #       LATIN SMALL LETTER DOUBLE THORN
 A7D5          ; Ll #       LATIN SMALL LETTER DOUBLE WYNN
 A7D7          ; Ll #       LATIN SMALL LETTER MIDDLE SCOTS S
 A7D9          ; Ll #       LATIN SMALL LETTER SIGMOID S
+A7DB          ; Ll #       LATIN SMALL LETTER LAMBDA
 A7F6          ; Ll #       LATIN SMALL LETTER REVERSED HALF H
 A7FA          ; Ll #       LATIN LETTER SMALL CAPITAL TURNED M
 AB30..AB5A    ; Ll #  [43] LATIN SMALL LETTER BARRED ALPHA..LATIN SMALL LETTER Y WITH SHORT RIGHT LEG
@@ -2006,6 +2038,7 @@ FF41..FF5A    ; Ll #  [26] FULLWIDTH LATIN SMALL LETTER A..FULLWIDTH LATIN SMALL
 105B3..105B9  ; Ll #   [7] VITHKUQI SMALL LETTER SE..VITHKUQI SMALL LETTER XE
 105BB..105BC  ; Ll #   [2] VITHKUQI SMALL LETTER Y..VITHKUQI SMALL LETTER ZE
 10CC0..10CF2  ; Ll #  [51] OLD HUNGARIAN SMALL LETTER A..OLD HUNGARIAN SMALL LETTER US
+10D70..10D85  ; Ll #  [22] GARAY SMALL LETTER A..GARAY SMALL LETTER OLD NA
 118C0..118DF  ; Ll #  [32] WARANG CITI SMALL LETTER NGAA..WARANG CITI SMALL LETTER VIYO
 16E60..16E7F  ; Ll #  [32] MEDEFAIDRIN SMALL LETTER M..MEDEFAIDRIN SMALL LETTER Y
 1D41A..1D433  ; Ll #  [26] MATHEMATICAL BOLD SMALL A..MATHEMATICAL BOLD SMALL Z
@@ -2041,7 +2074,7 @@ FF41..FF5A    ; Ll #  [26] FULLWIDTH LATIN SMALL LETTER A..FULLWIDTH LATIN SMALL
 1DF25..1DF2A  ; Ll #   [6] LATIN SMALL LETTER D WITH MID-HEIGHT LEFT HOOK..LATIN SMALL LETTER T WITH MID-HEIGHT LEFT HOOK
 1E922..1E943  ; Ll #  [34] ADLAM SMALL LETTER ALIF..ADLAM SMALL LETTER SHA
 
-# Total code points: 2233
+# Total code points: 2258
 
 # ================================================
 
@@ -2124,7 +2157,11 @@ FF9E..FF9F    ; Lm #   [2] HALFWIDTH KATAKANA VOICED SOUND MARK..HALFWIDTH KATAK
 10780..10785  ; Lm #   [6] MODIFIER LETTER SMALL CAPITAL AA..MODIFIER LETTER SMALL B WITH HOOK
 10787..107B0  ; Lm #  [42] MODIFIER LETTER SMALL DZ DIGRAPH..MODIFIER LETTER SMALL V WITH RIGHT HOOK
 107B2..107BA  ; Lm #   [9] MODIFIER LETTER SMALL CAPITAL Y..MODIFIER LETTER SMALL S WITH CURL
+10D4E         ; Lm #       GARAY VOWEL LENGTH MARK
+10D6F         ; Lm #       GARAY REDUPLICATION MARK
 16B40..16B43  ; Lm #   [4] PAHAWH HMONG SIGN VOS SEEV..PAHAWH HMONG SIGN IB YAM
+16D40..16D42  ; Lm #   [3] KIRAT RAI SIGN ANUSVARA..KIRAT RAI SIGN VISARGA
+16D6B..16D6C  ; Lm #   [2] KIRAT RAI SIGN VIRAMA..KIRAT RAI SIGN SAAT
 16F93..16F9F  ; Lm #  [13] MIAO LETTER TONE-2..MIAO LETTER REFORMED TONE-8
 16FE0..16FE1  ; Lm #   [2] TANGUT ITERATION MARK..NUSHU ITERATION MARK
 16FE3         ; Lm #       OLD CHINESE ITERATION MARK
@@ -2136,7 +2173,7 @@ FF9E..FF9F    ; Lm #   [2] HALFWIDTH KATAKANA VOICED SOUND MARK..HALFWIDTH KATAK
 1E4EB         ; Lm #       NAG MUNDARI SIGN OJOD
 1E94B         ; Lm #       ADLAM NASALIZATION MARK
 
-# Total code points: 397
+# Total code points: 404
 
 # ================================================
 
@@ -2451,6 +2488,7 @@ FFDA..FFDC    ; Lo #   [3] HALFWIDTH HANGUL LETTER EU..HALFWIDTH HANGUL LETTER I
 10450..1049D  ; Lo #  [78] SHAVIAN LETTER PEEP..OSMANYA LETTER OO
 10500..10527  ; Lo #  [40] ELBASAN LETTER A..ELBASAN LETTER KHE
 10530..10563  ; Lo #  [52] CAUCASIAN ALBANIAN LETTER ALT..CAUCASIAN ALBANIAN LETTER KIW
+105C0..105F3  ; Lo #  [52] TODHRI LETTER A..TODHRI LETTER OO
 10600..10736  ; Lo # [311] LINEAR A SIGN AB001..LINEAR A SIGN A664
 10740..10755  ; Lo #  [22] LINEAR A SIGN A701 A..LINEAR A SIGN A732 JE
 10760..10767  ; Lo #   [8] LINEAR A SIGN A800..LINEAR A SIGN A807
@@ -2482,8 +2520,11 @@ FFDA..FFDC    ; Lo #   [3] HALFWIDTH HANGUL LETTER EU..HALFWIDTH HANGUL LETTER I
 10B80..10B91  ; Lo #  [18] PSALTER PAHLAVI LETTER ALEPH..PSALTER PAHLAVI LETTER TAW
 10C00..10C48  ; Lo #  [73] OLD TURKIC LETTER ORKHON A..OLD TURKIC LETTER ORKHON BASH
 10D00..10D23  ; Lo #  [36] HANIFI ROHINGYA LETTER A..HANIFI ROHINGYA MARK NA KHONNA
+10D4A..10D4D  ; Lo #   [4] GARAY VOWEL SIGN A..GARAY VOWEL SIGN EE
+10D4F         ; Lo #       GARAY SUKUN
 10E80..10EA9  ; Lo #  [42] YEZIDI LETTER ELIF..YEZIDI LETTER ET
 10EB0..10EB1  ; Lo #   [2] YEZIDI LETTER LAM WITH DOT ABOVE..YEZIDI LETTER YOT WITH CIRCUMFLEX ABOVE
+10EC2..10EC4  ; Lo #   [3] ARABIC LETTER DAL WITH TWO DOTS VERTICALLY BELOW..ARABIC LETTER KAF WITH TWO DOTS VERTICALLY BELOW
 10F00..10F1C  ; Lo #  [29] OLD SOGDIAN LETTER ALEPH..OLD SOGDIAN LETTER FINAL TAW WITH VERTICAL TAIL
 10F27         ; Lo #       OLD SOGDIAN LIGATURE AYIN-DALETH
 10F30..10F45  ; Lo #  [22] SOGDIAN LETTER ALEPH..SOGDIAN INDEPENDENT SHIN
@@ -2522,6 +2563,13 @@ FFDA..FFDC    ; Lo #   [3] HALFWIDTH HANGUL LETTER EU..HALFWIDTH HANGUL LETTER I
 1133D         ; Lo #       GRANTHA SIGN AVAGRAHA
 11350         ; Lo #       GRANTHA OM
 1135D..11361  ; Lo #   [5] GRANTHA SIGN PLUTA..GRANTHA LETTER VOCALIC LL
+11380..11389  ; Lo #  [10] TULU-TIGALARI LETTER A..TULU-TIGALARI LETTER VOCALIC LL
+1138B         ; Lo #       TULU-TIGALARI LETTER EE
+1138E         ; Lo #       TULU-TIGALARI LETTER AI
+11390..113B5  ; Lo #  [38] TULU-TIGALARI LETTER OO..TULU-TIGALARI LETTER LLLA
+113B7         ; Lo #       TULU-TIGALARI SIGN AVAGRAHA
+113D1         ; Lo #       TULU-TIGALARI REPHA
+113D3         ; Lo #       TULU-TIGALARI SIGN PLUTA
 11400..11434  ; Lo #  [53] NEWA LETTER A..NEWA LETTER HA
 11447..1144A  ; Lo #   [4] NEWA SIGN AVAGRAHA..NEWA SIDDHI
 1145F..11461  ; Lo #   [3] NEWA LETTER VEDIC ANUSVARA..NEWA SIGN UPADHMANIYA
@@ -2555,6 +2603,7 @@ FFDA..FFDC    ; Lo #   [3] HALFWIDTH HANGUL LETTER EU..HALFWIDTH HANGUL LETTER I
 11A5C..11A89  ; Lo #  [46] SOYOMBO LETTER KA..SOYOMBO CLUSTER-INITIAL LETTER SA
 11A9D         ; Lo #       SOYOMBO MARK PLUTA
 11AB0..11AF8  ; Lo #  [73] CANADIAN SYLLABICS NATTILIK HI..PAU CIN HAU GLOTTAL STOP FINAL
+11BC0..11BE0  ; Lo #  [33] SUNUWAR LETTER DEVI..SUNUWAR LETTER KLOKO
 11C00..11C08  ; Lo #   [9] BHAIKSUKI LETTER A..BHAIKSUKI LETTER VOCALIC L
 11C0A..11C2E  ; Lo #  [37] BHAIKSUKI LETTER E..BHAIKSUKI LETTER HA
 11C40         ; Lo #       BHAIKSUKI SIGN AVAGRAHA
@@ -2577,7 +2626,9 @@ FFDA..FFDC    ; Lo #   [3] HALFWIDTH HANGUL LETTER EU..HALFWIDTH HANGUL LETTER I
 12F90..12FF0  ; Lo #  [97] CYPRO-MINOAN SIGN CM001..CYPRO-MINOAN SIGN CM114
 13000..1342F  ; Lo # [1072] EGYPTIAN HIEROGLYPH A001..EGYPTIAN HIEROGLYPH V011D
 13441..13446  ; Lo #   [6] EGYPTIAN HIEROGLYPH FULL BLANK..EGYPTIAN HIEROGLYPH WIDE LOST SIGN
+13460..143FA  ; Lo # [3995] EGYPTIAN HIEROGLYPH-13460..EGYPTIAN HIEROGLYPH-143FA
 14400..14646  ; Lo # [583] ANATOLIAN HIEROGLYPH A001..ANATOLIAN HIEROGLYPH A530
+16100..1611D  ; Lo #  [30] GURUNG KHEMA LETTER A..GURUNG KHEMA LETTER SA
 16800..16A38  ; Lo # [569] BAMUM LETTER PHASE-A NGKUE MFON..BAMUM LETTER PHASE-F VUEQ
 16A40..16A5E  ; Lo #  [31] MRO LETTER TA..MRO LETTER TEK
 16A70..16ABE  ; Lo #  [79] TANGSA LETTER OZ..TANGSA LETTER ZA
@@ -2585,11 +2636,12 @@ FFDA..FFDC    ; Lo #   [3] HALFWIDTH HANGUL LETTER EU..HALFWIDTH HANGUL LETTER I
 16B00..16B2F  ; Lo #  [48] PAHAWH HMONG VOWEL KEEB..PAHAWH HMONG CONSONANT CAU
 16B63..16B77  ; Lo #  [21] PAHAWH HMONG SIGN VOS LUB..PAHAWH HMONG SIGN CIM NRES TOS
 16B7D..16B8F  ; Lo #  [19] PAHAWH HMONG CLAN SIGN TSHEEJ..PAHAWH HMONG CLAN SIGN VWJ
+16D43..16D6A  ; Lo #  [40] KIRAT RAI LETTER A..KIRAT RAI VOWEL SIGN AU
 16F00..16F4A  ; Lo #  [75] MIAO LETTER PA..MIAO LETTER RTE
 16F50         ; Lo #       MIAO LETTER NASALIZATION
 17000..187F7  ; Lo # [6136] TANGUT IDEOGRAPH-17000..TANGUT IDEOGRAPH-187F7
 18800..18CD5  ; Lo # [1238] TANGUT COMPONENT-001..KHITAN SMALL SCRIPT CHARACTER-18CD5
-18D00..18D08  ; Lo #   [9] TANGUT IDEOGRAPH-18D00..TANGUT IDEOGRAPH-18D08
+18CFF..18D08  ; Lo #  [10] KHITAN SMALL SCRIPT CHARACTER-18CFF..TANGUT IDEOGRAPH-18D08
 1B000..1B122  ; Lo # [291] KATAKANA LETTER ARCHAIC E..KATAKANA LETTER ARCHAIC WU
 1B132         ; Lo #       HIRAGANA LETTER SMALL KO
 1B150..1B152  ; Lo #   [3] HIRAGANA LETTER SMALL WI..HIRAGANA LETTER SMALL WO
@@ -2606,6 +2658,8 @@ FFDA..FFDC    ; Lo #   [3] HALFWIDTH HANGUL LETTER EU..HALFWIDTH HANGUL LETTER I
 1E290..1E2AD  ; Lo #  [30] TOTO LETTER PA..TOTO LETTER A
 1E2C0..1E2EB  ; Lo #  [44] WANCHO LETTER AA..WANCHO LETTER YIH
 1E4D0..1E4EA  ; Lo #  [27] NAG MUNDARI LETTER O..NAG MUNDARI LETTER ELL
+1E5D0..1E5ED  ; Lo #  [30] OL ONAL LETTER O..OL ONAL LETTER EG
+1E5F0         ; Lo #       OL ONAL SIGN HODDOND
 1E7E0..1E7E6  ; Lo #   [7] ETHIOPIC SYLLABLE HHYA..ETHIOPIC SYLLABLE HHYO
 1E7E8..1E7EB  ; Lo #   [4] ETHIOPIC SYLLABLE GURAGE HHWA..ETHIOPIC SYLLABLE HHWE
 1E7ED..1E7EE  ; Lo #   [2] ETHIOPIC SYLLABLE GURAGE MWI..ETHIOPIC SYLLABLE GURAGE MWEE
@@ -2654,7 +2708,7 @@ FFDA..FFDC    ; Lo #   [3] HALFWIDTH HANGUL LETTER EU..HALFWIDTH HANGUL LETTER I
 30000..3134A  ; Lo # [4939] CJK UNIFIED IDEOGRAPH-30000..CJK UNIFIED IDEOGRAPH-3134A
 31350..323AF  ; Lo # [4192] CJK UNIFIED IDEOGRAPH-31350..CJK UNIFIED IDEOGRAPH-323AF
 
-# Total code points: 132234
+# Total code points: 136477
 
 # ================================================
 
@@ -2684,7 +2738,7 @@ FFDA..FFDC    ; Lo #   [3] HALFWIDTH HANGUL LETTER EU..HALFWIDTH HANGUL LETTER I
 0825..0827    ; Mn #   [3] SAMARITAN VOWEL SIGN SHORT A..SAMARITAN VOWEL SIGN U
 0829..082D    ; Mn #   [5] SAMARITAN VOWEL SIGN LONG I..SAMARITAN MARK NEQUDAA
 0859..085B    ; Mn #   [3] MANDAIC AFFRICATION MARK..MANDAIC GEMINATION MARK
-0898..089F    ; Mn #   [8] ARABIC SMALL HIGH WORD AL-JUZ..ARABIC HALF MADDA OVER MADDA
+0897..089F    ; Mn #   [9] ARABIC PEPET..ARABIC HALF MADDA OVER MADDA
 08CA..08E1    ; Mn #  [24] ARABIC SMALL HIGH FARSI YEH..ARABIC SMALL HIGH SIGN SAFHA
 08E3..0902    ; Mn #  [32] ARABIC TURNED DAMMA BELOW..DEVANAGARI SIGN ANUSVARA
 093A          ; Mn #       DEVANAGARI VOWEL SIGN OE
@@ -2882,8 +2936,9 @@ FE20..FE2F    ; Mn #  [16] COMBINING LIGATURE LEFT HALF..COMBINING CYRILLIC TITL
 10A3F         ; Mn #       KHAROSHTHI VIRAMA
 10AE5..10AE6  ; Mn #   [2] MANICHAEAN ABBREVIATION MARK ABOVE..MANICHAEAN ABBREVIATION MARK BELOW
 10D24..10D27  ; Mn #   [4] HANIFI ROHINGYA SIGN HARBAHAY..HANIFI ROHINGYA SIGN TASSI
+10D69..10D6D  ; Mn #   [5] GARAY VOWEL SIGN E..GARAY CONSONANT NASALIZATION MARK
 10EAB..10EAC  ; Mn #   [2] YEZIDI COMBINING HAMZA MARK..YEZIDI COMBINING MADDA MARK
-10EFD..10EFF  ; Mn #   [3] ARABIC SMALL LOW WORD SAKTA..ARABIC SMALL LOW WORD MADDA
+10EFC..10EFF  ; Mn #   [4] ARABIC COMBINING ALEF OVERLAY..ARABIC SMALL LOW WORD MADDA
 10F46..10F50  ; Mn #  [11] SOGDIAN COMBINING DOT BELOW..SOGDIAN COMBINING STROKE BELOW
 10F82..10F85  ; Mn #   [4] OLD UYGHUR COMBINING DOT ABOVE..OLD UYGHUR COMBINING TWO DOTS BELOW
 11001         ; Mn #       BRAHMI SIGN ANUSVARA
@@ -2914,6 +2969,11 @@ FE20..FE2F    ; Mn #  [16] COMBINING LIGATURE LEFT HALF..COMBINING CYRILLIC TITL
 11340         ; Mn #       GRANTHA VOWEL SIGN II
 11366..1136C  ; Mn #   [7] COMBINING GRANTHA DIGIT ZERO..COMBINING GRANTHA DIGIT SIX
 11370..11374  ; Mn #   [5] COMBINING GRANTHA LETTER A..COMBINING GRANTHA LETTER PA
+113BB..113C0  ; Mn #   [6] TULU-TIGALARI VOWEL SIGN U..TULU-TIGALARI VOWEL SIGN VOCALIC LL
+113CE         ; Mn #       TULU-TIGALARI SIGN VIRAMA
+113D0         ; Mn #       TULU-TIGALARI CONJOINER
+113D2         ; Mn #       TULU-TIGALARI GEMINATION MARK
+113E1..113E2  ; Mn #   [2] TULU-TIGALARI VEDIC TONE SVARITA..TULU-TIGALARI VEDIC TONE ANUDATTA
 11438..1143F  ; Mn #   [8] NEWA VOWEL SIGN U..NEWA VOWEL SIGN AI
 11442..11444  ; Mn #   [3] NEWA SIGN VIRAMA..NEWA SIGN ANUSVARA
 11446         ; Mn #       NEWA SIGN NUKTA
@@ -2933,7 +2993,8 @@ FE20..FE2F    ; Mn #  [16] COMBINING LIGATURE LEFT HALF..COMBINING CYRILLIC TITL
 116AD         ; Mn #       TAKRI VOWEL SIGN AA
 116B0..116B5  ; Mn #   [6] TAKRI VOWEL SIGN U..TAKRI VOWEL SIGN AU
 116B7         ; Mn #       TAKRI SIGN NUKTA
-1171D..1171F  ; Mn #   [3] AHOM CONSONANT SIGN MEDIAL LA..AHOM CONSONANT SIGN MEDIAL LIGATING RA
+1171D         ; Mn #       AHOM CONSONANT SIGN MEDIAL LA
+1171F         ; Mn #       AHOM CONSONANT SIGN MEDIAL LIGATING RA
 11722..11725  ; Mn #   [4] AHOM VOWEL SIGN I..AHOM VOWEL SIGN UU
 11727..1172B  ; Mn #   [5] AHOM VOWEL SIGN AW..AHOM SIGN KILLER
 1182F..11837  ; Mn #   [9] DOGRA VOWEL SIGN U..DOGRA SIGN ANUSVARA
@@ -2972,8 +3033,11 @@ FE20..FE2F    ; Mn #  [16] COMBINING LIGATURE LEFT HALF..COMBINING CYRILLIC TITL
 11F36..11F3A  ; Mn #   [5] KAWI VOWEL SIGN I..KAWI VOWEL SIGN VOCALIC R
 11F40         ; Mn #       KAWI VOWEL SIGN EU
 11F42         ; Mn #       KAWI CONJOINER
+11F5A         ; Mn #       KAWI SIGN NUKTA
 13440         ; Mn #       EGYPTIAN HIEROGLYPH MIRROR HORIZONTALLY
 13447..13455  ; Mn #  [15] EGYPTIAN HIEROGLYPH MODIFIER DAMAGED AT TOP START..EGYPTIAN HIEROGLYPH MODIFIER DAMAGED
+1611E..16129  ; Mn #  [12] GURUNG KHEMA VOWEL SIGN AA..GURUNG KHEMA VOWEL LENGTH MARK
+1612D..1612F  ; Mn #   [3] GURUNG KHEMA SIGN ANUSVARA..GURUNG KHEMA SIGN THOLHOMA
 16AF0..16AF4  ; Mn #   [5] BASSA VAH COMBINING HIGH TONE..BASSA VAH COMBINING HIGH-LOW TONE
 16B30..16B36  ; Mn #   [7] PAHAWH HMONG MARK CIM TUB..PAHAWH HMONG MARK CIM TAUM
 16F4F         ; Mn #       MIAO SIGN CONSONANT MODIFIER BAR
@@ -3003,11 +3067,12 @@ FE20..FE2F    ; Mn #  [16] COMBINING LIGATURE LEFT HALF..COMBINING CYRILLIC TITL
 1E2AE         ; Mn #       TOTO SIGN RISING TONE
 1E2EC..1E2EF  ; Mn #   [4] WANCHO TONE TUP..WANCHO TONE KOINI
 1E4EC..1E4EF  ; Mn #   [4] NAG MUNDARI SIGN MUHOR..NAG MUNDARI SIGN SUTUH
+1E5EE..1E5EF  ; Mn #   [2] OL ONAL SIGN MU..OL ONAL SIGN IKIR
 1E8D0..1E8D6  ; Mn #   [7] MENDE KIKAKUI COMBINING NUMBER TEENS..MENDE KIKAKUI COMBINING NUMBER MILLIONS
 1E944..1E94A  ; Mn #   [7] ADLAM ALIF LENGTHENER..ADLAM NUKTA
 E0100..E01EF  ; Mn # [240] VARIATION SELECTOR-17..VARIATION SELECTOR-256
 
-# Total code points: 1985
+# Total code points: 2020
 
 # ================================================
 
@@ -3159,6 +3224,12 @@ ABEC          ; Mc #       MEETEI MAYEK LUM IYEK
 1134B..1134D  ; Mc #   [3] GRANTHA VOWEL SIGN OO..GRANTHA SIGN VIRAMA
 11357         ; Mc #       GRANTHA AU LENGTH MARK
 11362..11363  ; Mc #   [2] GRANTHA VOWEL SIGN VOCALIC L..GRANTHA VOWEL SIGN VOCALIC LL
+113B8..113BA  ; Mc #   [3] TULU-TIGALARI VOWEL SIGN AA..TULU-TIGALARI VOWEL SIGN II
+113C2         ; Mc #       TULU-TIGALARI VOWEL SIGN EE
+113C5         ; Mc #       TULU-TIGALARI VOWEL SIGN AI
+113C7..113CA  ; Mc #   [4] TULU-TIGALARI VOWEL SIGN OO..TULU-TIGALARI SIGN CANDRA ANUNASIKA
+113CC..113CD  ; Mc #   [2] TULU-TIGALARI SIGN ANUSVARA..TULU-TIGALARI SIGN VISARGA
+113CF         ; Mc #       TULU-TIGALARI SIGN LOOPED VIRAMA
 11435..11437  ; Mc #   [3] NEWA VOWEL SIGN AA..NEWA VOWEL SIGN II
 11440..11441  ; Mc #   [2] NEWA VOWEL SIGN O..NEWA VOWEL SIGN AU
 11445         ; Mc #       NEWA SIGN VISARGA
@@ -3175,6 +3246,7 @@ ABEC          ; Mc #       MEETEI MAYEK LUM IYEK
 116AC         ; Mc #       TAKRI SIGN VISARGA
 116AE..116AF  ; Mc #   [2] TAKRI VOWEL SIGN I..TAKRI VOWEL SIGN II
 116B6         ; Mc #       TAKRI SIGN VIRAMA
+1171E         ; Mc #       AHOM CONSONANT SIGN MEDIAL RA
 11720..11721  ; Mc #   [2] AHOM VOWEL SIGN A..AHOM VOWEL SIGN AA
 11726         ; Mc #       AHOM VOWEL SIGN E
 1182C..1182E  ; Mc #   [3] DOGRA VOWEL SIGN AA..DOGRA VOWEL SIGN II
@@ -3203,12 +3275,13 @@ ABEC          ; Mc #       MEETEI MAYEK LUM IYEK
 11F34..11F35  ; Mc #   [2] KAWI VOWEL SIGN AA..KAWI VOWEL SIGN ALTERNATE AA
 11F3E..11F3F  ; Mc #   [2] KAWI VOWEL SIGN E..KAWI VOWEL SIGN AI
 11F41         ; Mc #       KAWI SIGN KILLER
+1612A..1612C  ; Mc #   [3] GURUNG KHEMA CONSONANT SIGN MEDIAL YA..GURUNG KHEMA CONSONANT SIGN MEDIAL HA
 16F51..16F87  ; Mc #  [55] MIAO SIGN ASPIRATION..MIAO VOWEL SIGN UI
 16FF0..16FF1  ; Mc #   [2] VIETNAMESE ALTERNATE READING MARK CA..VIETNAMESE ALTERNATE READING MARK NHAY
 1D165..1D166  ; Mc #   [2] MUSICAL SYMBOL COMBINING STEM..MUSICAL SYMBOL COMBINING SPRECHGESANG STEM
 1D16D..1D172  ; Mc #   [6] MUSICAL SYMBOL COMBINING AUGMENTATION DOT..MUSICAL SYMBOL COMBINING FLAG-5
 
-# Total code points: 452
+# Total code points: 468
 
 # ================================================
 
@@ -3253,6 +3326,7 @@ ABF0..ABF9    ; Nd #  [10] MEETEI MAYEK DIGIT ZERO..MEETEI MAYEK DIGIT NINE
 FF10..FF19    ; Nd #  [10] FULLWIDTH DIGIT ZERO..FULLWIDTH DIGIT NINE
 104A0..104A9  ; Nd #  [10] OSMANYA DIGIT ZERO..OSMANYA DIGIT NINE
 10D30..10D39  ; Nd #  [10] HANIFI ROHINGYA DIGIT ZERO..HANIFI ROHINGYA DIGIT NINE
+10D40..10D49  ; Nd #  [10] GARAY DIGIT ZERO..GARAY DIGIT NINE
 11066..1106F  ; Nd #  [10] BRAHMI DIGIT ZERO..BRAHMI DIGIT NINE
 110F0..110F9  ; Nd #  [10] SORA SOMPENG DIGIT ZERO..SORA SOMPENG DIGIT NINE
 11136..1113F  ; Nd #  [10] CHAKMA DIGIT ZERO..CHAKMA DIGIT NINE
@@ -3262,24 +3336,30 @@ FF10..FF19    ; Nd #  [10] FULLWIDTH DIGIT ZERO..FULLWIDTH DIGIT NINE
 114D0..114D9  ; Nd #  [10] TIRHUTA DIGIT ZERO..TIRHUTA DIGIT NINE
 11650..11659  ; Nd #  [10] MODI DIGIT ZERO..MODI DIGIT NINE
 116C0..116C9  ; Nd #  [10] TAKRI DIGIT ZERO..TAKRI DIGIT NINE
+116D0..116E3  ; Nd #  [20] MYANMAR PAO DIGIT ZERO..MYANMAR EASTERN PWO KAREN DIGIT NINE
 11730..11739  ; Nd #  [10] AHOM DIGIT ZERO..AHOM DIGIT NINE
 118E0..118E9  ; Nd #  [10] WARANG CITI DIGIT ZERO..WARANG CITI DIGIT NINE
 11950..11959  ; Nd #  [10] DIVES AKURU DIGIT ZERO..DIVES AKURU DIGIT NINE
+11BF0..11BF9  ; Nd #  [10] SUNUWAR DIGIT ZERO..SUNUWAR DIGIT NINE
 11C50..11C59  ; Nd #  [10] BHAIKSUKI DIGIT ZERO..BHAIKSUKI DIGIT NINE
 11D50..11D59  ; Nd #  [10] MASARAM GONDI DIGIT ZERO..MASARAM GONDI DIGIT NINE
 11DA0..11DA9  ; Nd #  [10] GUNJALA GONDI DIGIT ZERO..GUNJALA GONDI DIGIT NINE
 11F50..11F59  ; Nd #  [10] KAWI DIGIT ZERO..KAWI DIGIT NINE
+16130..16139  ; Nd #  [10] GURUNG KHEMA DIGIT ZERO..GURUNG KHEMA DIGIT NINE
 16A60..16A69  ; Nd #  [10] MRO DIGIT ZERO..MRO DIGIT NINE
 16AC0..16AC9  ; Nd #  [10] TANGSA DIGIT ZERO..TANGSA DIGIT NINE
 16B50..16B59  ; Nd #  [10] PAHAWH HMONG DIGIT ZERO..PAHAWH HMONG DIGIT NINE
+16D70..16D79  ; Nd #  [10] KIRAT RAI DIGIT ZERO..KIRAT RAI DIGIT NINE
+1CCF0..1CCF9  ; Nd #  [10] OUTLINED DIGIT ZERO..OUTLINED DIGIT NINE
 1D7CE..1D7FF  ; Nd #  [50] MATHEMATICAL BOLD DIGIT ZERO..MATHEMATICAL MONOSPACE DIGIT NINE
 1E140..1E149  ; Nd #  [10] NYIAKENG PUACHUE HMONG DIGIT ZERO..NYIAKENG PUACHUE HMONG DIGIT NINE
 1E2F0..1E2F9  ; Nd #  [10] WANCHO DIGIT ZERO..WANCHO DIGIT NINE
 1E4F0..1E4F9  ; Nd #  [10] NAG MUNDARI DIGIT ZERO..NAG MUNDARI DIGIT NINE
+1E5F1..1E5FA  ; Nd #  [10] OL ONAL DIGIT ZERO..OL ONAL DIGIT NINE
 1E950..1E959  ; Nd #  [10] ADLAM DIGIT ZERO..ADLAM DIGIT NINE
 1FBF0..1FBF9  ; Nd #  [10] SEGMENTED DIGIT ZERO..SEGMENTED DIGIT NINE
 
-# Total code points: 680
+# Total code points: 760
 
 # ================================================
 
@@ -3486,9 +3566,10 @@ FE31..FE32    ; Pd #   [2] PRESENTATION FORM FOR VERTICAL EM DASH..PRESENTATION
 FE58          ; Pd #       SMALL EM DASH
 FE63          ; Pd #       SMALL HYPHEN-MINUS
 FF0D          ; Pd #       FULLWIDTH HYPHEN-MINUS
+10D6E         ; Pd #       GARAY HYPHEN
 10EAD         ; Pd #       YEZIDI HYPHENATION MARK
 
-# Total code points: 26
+# Total code points: 27
 
 # ================================================
 
@@ -3735,8 +3816,9 @@ FF3F          ; Pc #       FULLWIDTH LOW LINE
 1A1E..1A1F    ; Po #   [2] BUGINESE PALLAWA..BUGINESE END OF SECTION
 1AA0..1AA6    ; Po #   [7] TAI THAM SIGN WIANG..TAI THAM SIGN REVERSED ROTATED RANA
 1AA8..1AAD    ; Po #   [6] TAI THAM SIGN KAAN..TAI THAM SIGN CAANG
+1B4E..1B4F    ; Po #   [2] BALINESE INVERTED CARIK SIKI..BALINESE INVERTED CARIK PAREREN
 1B5A..1B60    ; Po #   [7] BALINESE PANTI..BALINESE PAMENENG
-1B7D..1B7E    ; Po #   [2] BALINESE PANTI LANTANG..BALINESE PAMADA LANTANG
+1B7D..1B7F    ; Po #   [3] BALINESE PANTI LANTANG..BALINESE PANTI BAWAK
 1BFC..1BFF    ; Po #   [4] BATAK SYMBOL BINDU NA METEK..BATAK SYMBOL BINDU PANGOLAT
 1C3B..1C3F    ; Po #   [5] LEPCHA PUNCTUATION TA-ROL..LEPCHA PUNCTUATION TSHOOK
 1C7E..1C7F    ; Po #   [2] OL CHIKI PUNCTUATION MUCAAD..OL CHIKI PUNCTUATION DOUBLE MUCAAD
@@ -3831,6 +3913,8 @@ FF64..FF65    ; Po #   [2] HALFWIDTH IDEOGRAPHIC COMMA..HALFWIDTH KATAKANA MIDDL
 111DD..111DF  ; Po #   [3] SHARADA CONTINUATION SIGN..SHARADA SECTION MARK-2
 11238..1123D  ; Po #   [6] KHOJKI DANDA..KHOJKI ABBREVIATION SIGN
 112A9         ; Po #       MULTANI SECTION MARK
+113D4..113D5  ; Po #   [2] TULU-TIGALARI DANDA..TULU-TIGALARI DOUBLE DANDA
+113D7..113D8  ; Po #   [2] TULU-TIGALARI SIGN OM PUSHPIKA..TULU-TIGALARI SIGN SHRII PUSHPIKA
 1144B..1144F  ; Po #   [5] NEWA DANDA..NEWA ABBREVIATION SIGN
 1145A..1145B  ; Po #   [2] NEWA DOUBLE COMMA..NEWA PLACEHOLDER MARK
 1145D         ; Po #       NEWA INSERTION SIGN
@@ -3847,6 +3931,7 @@ FF64..FF65    ; Po #   [2] HALFWIDTH IDEOGRAPHIC COMMA..HALFWIDTH KATAKANA MIDDL
 11A9A..11A9C  ; Po #   [3] SOYOMBO MARK TSHEG..SOYOMBO MARK DOUBLE SHAD
 11A9E..11AA2  ; Po #   [5] SOYOMBO HEAD MARK WITH MOON AND SUN AND TRIPLE FLAME..SOYOMBO TERMINAL MARK-2
 11B00..11B09  ; Po #  [10] DEVANAGARI HEAD MARK..DEVANAGARI SIGN MINDU
+11BE1         ; Po #       SUNUWAR SIGN PVO
 11C41..11C45  ; Po #   [5] BHAIKSUKI DANDA..BHAIKSUKI GAP FILLER-2
 11C70..11C71  ; Po #   [2] MARCHEN HEAD MARK..MARCHEN MARK SHAD
 11EF7..11EF8  ; Po #   [2] MAKASAR PASSIMBANG..MAKASAR END OF SECTION
@@ -3858,13 +3943,15 @@ FF64..FF65    ; Po #   [2] HALFWIDTH IDEOGRAPHIC COMMA..HALFWIDTH KATAKANA MIDDL
 16AF5         ; Po #       BASSA VAH FULL STOP
 16B37..16B3B  ; Po #   [5] PAHAWH HMONG SIGN VOS THOM..PAHAWH HMONG SIGN VOS FEEM
 16B44         ; Po #       PAHAWH HMONG SIGN XAUS
+16D6D..16D6F  ; Po #   [3] KIRAT RAI SIGN YUPI..KIRAT RAI DOUBLE DANDA
 16E97..16E9A  ; Po #   [4] MEDEFAIDRIN COMMA..MEDEFAIDRIN EXCLAMATION OH
 16FE2         ; Po #       OLD CHINESE HOOK MARK
 1BC9F         ; Po #       DUPLOYAN PUNCTUATION CHINOOK FULL STOP
 1DA87..1DA8B  ; Po #   [5] SIGNWRITING COMMA..SIGNWRITING PARENTHESIS
+1E5FF         ; Po #       OL ONAL ABBREVIATION SIGN
 1E95E..1E95F  ; Po #   [2] ADLAM INITIAL EXCLAMATION MARK..ADLAM INITIAL QUESTION MARK
 
-# Total code points: 628
+# Total code points: 640
 
 # ================================================
 
@@ -3923,6 +4010,7 @@ FF5C          ; Sm #       FULLWIDTH VERTICAL LINE
 FF5E          ; Sm #       FULLWIDTH TILDE
 FFE2          ; Sm #       FULLWIDTH NOT SIGN
 FFE9..FFEC    ; Sm #   [4] HALFWIDTH LEFTWARDS ARROW..HALFWIDTH DOWNWARDS ARROW
+10D8E..10D8F  ; Sm #   [2] GARAY PLUS SIGN..GARAY MINUS SIGN
 1D6C1         ; Sm #       MATHEMATICAL BOLD NABLA
 1D6DB         ; Sm #       MATHEMATICAL BOLD PARTIAL DIFFERENTIAL
 1D6FB         ; Sm #       MATHEMATICAL ITALIC NABLA
@@ -3935,7 +4023,7 @@ FFE9..FFEC    ; Sm #   [4] HALFWIDTH LEFTWARDS ARROW..HALFWIDTH DOWNWARDS ARROW
 1D7C3         ; Sm #       MATHEMATICAL SANS-SERIF BOLD ITALIC PARTIAL DIFFERENTIAL
 1EEF0..1EEF1  ; Sm #   [2] ARABIC MATHEMATICAL OPERATOR MEEM WITH HAH WITH TATWEEL..ARABIC MATHEMATICAL OPERATOR HAH WITH DAL
 
-# Total code points: 948
+# Total code points: 950
 
 # ================================================
 
@@ -4073,7 +4161,7 @@ FFE3          ; Sk #       FULLWIDTH MACRON
 232B..237B    ; So #  [81] ERASE TO THE LEFT..NOT CHECK MARK
 237D..239A    ; So #  [30] SHOULDERED OPEN BOX..CLEAR SCREEN SYMBOL
 23B4..23DB    ; So #  [40] TOP SQUARE BRACKET..FUSE
-23E2..2426    ; So #  [69] WHITE TRAPEZIUM..SYMBOL FOR SUBSTITUTE FORM TWO
+23E2..2429    ; So #  [72] WHITE TRAPEZIUM..SYMBOL FOR DELETE MEDIUM SHADE FORM
 2440..244A    ; So #  [11] OCR HOOK..OCR DOUBLE BACKSLASH
 249C..24E9    ; So #  [78] PARENTHESIZED LATIN SMALL LETTER A..CIRCLED LATIN SMALL LETTER Z
 2500..25B6    ; So # [183] BOX DRAWINGS LIGHT HORIZONTAL..BLACK RIGHT-POINTING TRIANGLE
@@ -4101,7 +4189,7 @@ FFE3          ; Sk #       FULLWIDTH MACRON
 303E..303F    ; So #   [2] IDEOGRAPHIC VARIATION INDICATOR..IDEOGRAPHIC HALF FILL SPACE
 3190..3191    ; So #   [2] IDEOGRAPHIC ANNOTATION LINKING MARK..IDEOGRAPHIC ANNOTATION REVERSE MARK
 3196..319F    ; So #  [10] IDEOGRAPHIC ANNOTATION TOP MARK..IDEOGRAPHIC ANNOTATION MAN MARK
-31C0..31E3    ; So #  [36] CJK STROKE T..CJK STROKE Q
+31C0..31E5    ; So #  [38] CJK STROKE T..CJK STROKE SZP
 31EF          ; So #       IDEOGRAPHIC DESCRIPTION CHARACTER SUBTRACTION
 3200..321E    ; So #  [31] PARENTHESIZED HANGUL KIYEOK..PARENTHESIZED KOREAN CHARACTER O HU
 322A..3247    ; So #  [30] PARENTHESIZED IDEOGRAPH MOON..CIRCLED IDEOGRAPH KOTO
@@ -4136,6 +4224,8 @@ FFFC..FFFD    ; So #   [2] OBJECT REPLACEMENT CHARACTER..REPLACEMENT CHARACTER
 16B3C..16B3F  ; So #   [4] PAHAWH HMONG SIGN XYEEM NTXIV..PAHAWH HMONG SIGN XYEEM FAIB
 16B45         ; So #       PAHAWH HMONG SIGN CIM TSOV ROG
 1BC9C         ; So #       DUPLOYAN SIGN O WITH CROSS
+1CC00..1CCEF  ; So # [240] UP-POINTING GO-KART..OUTLINED LATIN CAPITAL LETTER Z
+1CD00..1CEB3  ; So # [436] BLOCK OCTANT-3..BLACK RIGHT TRIANGLE CARET
 1CF50..1CFC3  ; So # [116] ZNAMENNY NEUME KRYUK..ZNAMENNY NEUME PAUK
 1D000..1D0F5  ; So # [246] BYZANTINE MUSICAL SYMBOL PSILI..BYZANTINE MUSICAL SYMBOL GORGON NEO KATO
 1D100..1D126  ; So #  [39] MUSICAL SYMBOL SINGLE BARLINE..MUSICAL SYMBOL DRUM CLEF-2
@@ -4180,20 +4270,20 @@ FFFC..FFFD    ; So #   [2] OBJECT REPLACEMENT CHARACTER..REPLACEMENT CHARACTER
 1F850..1F859  ; So #  [10] LEFTWARDS SANS-SERIF ARROW..UP DOWN SANS-SERIF ARROW
 1F860..1F887  ; So #  [40] WIDE-HEADED LEFTWARDS LIGHT BARB ARROW..WIDE-HEADED SOUTH WEST VERY HEAVY BARB ARROW
 1F890..1F8AD  ; So #  [30] LEFTWARDS TRIANGLE ARROWHEAD..WHITE ARROW SHAFT WIDTH TWO THIRDS
-1F8B0..1F8B1  ; So #   [2] ARROW POINTING UPWARDS THEN NORTH WEST..ARROW POINTING RIGHTWARDS THEN CURVING SOUTH WEST
+1F8B0..1F8BB  ; So #  [12] ARROW POINTING UPWARDS THEN NORTH WEST..SOUTH WEST ARROW FROM BAR
+1F8C0..1F8C1  ; So #   [2] LEFTWARDS ARROW FROM DOWNWARDS ARROW..RIGHTWARDS ARROW FROM DOWNWARDS ARROW
 1F900..1FA53  ; So # [340] CIRCLED CROSS FORMEE WITH FOUR DOTS..BLACK CHESS KNIGHT-BISHOP
 1FA60..1FA6D  ; So #  [14] XIANGQI RED GENERAL..XIANGQI BLACK SOLDIER
 1FA70..1FA7C  ; So #  [13] BALLET SHOES..CRUTCH
-1FA80..1FA88  ; So #   [9] YO-YO..FLUTE
-1FA90..1FABD  ; So #  [46] RINGED PLANET..WING
-1FABF..1FAC5  ; So #   [7] GOOSE..PERSON WITH CROWN
-1FACE..1FADB  ; So #  [14] MOOSE..PEA POD
-1FAE0..1FAE8  ; So #   [9] MELTING FACE..SHAKING FACE
+1FA80..1FA89  ; So #  [10] YO-YO..HARP
+1FA8F..1FAC6  ; So #  [56] SHOVEL..FINGERPRINT
+1FACE..1FADC  ; So #  [15] MOOSE..ROOT VEGETABLE
+1FADF..1FAE9  ; So #  [11] SPLATTER..FACE WITH BAGS UNDER EYES
 1FAF0..1FAF8  ; So #   [9] HAND WITH INDEX FINGER AND THUMB CROSSED..RIGHTWARDS PUSHING HAND
 1FB00..1FB92  ; So # [147] BLOCK SEXTANT-1..UPPER HALF INVERSE MEDIUM SHADE AND LOWER HALF BLOCK
-1FB94..1FBCA  ; So #  [55] LEFT HALF INVERSE MEDIUM SHADE AND RIGHT HALF BLOCK..WHITE UP-POINTING CHEVRON
+1FB94..1FBEF  ; So #  [92] LEFT HALF INVERSE MEDIUM SHADE AND RIGHT HALF BLOCK..TOP LEFT JUSTIFIED LOWER RIGHT QUARTER BLACK CIRCLE
 
-# Total code points: 6639
+# Total code points: 7376
 
 # ================================================
 
diff --git a/libcxx/utils/data/unicode/EastAsianWidth.txt b/libcxx/utils/data/unicode/EastAsianWidth.txt
index 02df4df475cbe..99f7a31ea5d8a 100644
--- a/libcxx/utils/data/unicode/EastAsianWidth.txt
+++ b/libcxx/utils/data/unicode/EastAsianWidth.txt
@@ -1,8 +1,8 @@
-# EastAsianWidth-15.1.0.txt
-# Date: 2023-07-28, 23:34:08 GMT
-# © 2023 Unicode®, Inc.
+# EastAsianWidth-16.0.0.txt
+# Date: 2024-04-30, 21:48:20 GMT
+# © 2024 Unicode®, Inc.
 # Unicode and the Unicode Logo are registered trademarks of Unicode, Inc. in the U.S. and other countries.
-# For terms of use, see https://www.unicode.org/terms_of_use.html
+# For terms of use and license, see https://www.unicode.org/terms_of_use.html
 #
 # Unicode Character Database
 #   For documentation, see https://www.unicode.org/reports/tr44/
@@ -334,7 +334,7 @@
 0888           ; N  # Sk         ARABIC RAISED ROUND DOT
 0889..088E     ; N  # Lo     [6] ARABIC LETTER NOON WITH INVERTED SMALL V..ARABIC VERTICAL TAIL
 0890..0891     ; N  # Cf     [2] ARABIC POUND MARK ABOVE..ARABIC PIASTRE MARK ABOVE
-0898..089F     ; N  # Mn     [8] ARABIC SMALL HIGH WORD AL-JUZ..ARABIC HALF MADDA OVER MADDA
+0897..089F     ; N  # Mn     [9] ARABIC PEPET..ARABIC HALF MADDA OVER MADDA
 08A0..08C8     ; N  # Lo    [41] ARABIC LETTER BEH WITH SMALL V BELOW..ARABIC LETTER GRAF
 08C9           ; N  # Lm         ARABIC SMALL FARSI YEH
 08CA..08E1     ; N  # Mn    [24] ARABIC SMALL HIGH FARSI YEH..ARABIC SMALL HIGH SIGN SAFHA
@@ -819,12 +819,13 @@
 1B42           ; N  # Mn         BALINESE VOWEL SIGN PEPET
 1B43..1B44     ; N  # Mc     [2] BALINESE VOWEL SIGN PEPET TEDUNG..BALINESE ADEG ADEG
 1B45..1B4C     ; N  # Lo     [8] BALINESE LETTER KAF SASAK..BALINESE LETTER ARCHAIC JNYA
+1B4E..1B4F     ; N  # Po     [2] BALINESE INVERTED CARIK SIKI..BALINESE INVERTED CARIK PAREREN
 1B50..1B59     ; N  # Nd    [10] BALINESE DIGIT ZERO..BALINESE DIGIT NINE
 1B5A..1B60     ; N  # Po     [7] BALINESE PANTI..BALINESE PAMENENG
 1B61..1B6A     ; N  # So    [10] BALINESE MUSICAL SYMBOL DONG..BALINESE MUSICAL SYMBOL DANG GEDE
 1B6B..1B73     ; N  # Mn     [9] BALINESE MUSICAL SYMBOL COMBINING TEGEH..BALINESE MUSICAL SYMBOL COMBINING GONG
 1B74..1B7C     ; N  # So     [9] BALINESE MUSICAL SYMBOL RIGHT-HAND OPEN DUG..BALINESE MUSICAL SYMBOL LEFT-HAND OPEN PING
-1B7D..1B7E     ; N  # Po     [2] BALINESE PANTI LANTANG..BALINESE PAMADA LANTANG
+1B7D..1B7F     ; N  # Po     [3] BALINESE PANTI LANTANG..BALINESE PANTI BAWAK
 1B80..1B81     ; N  # Mn     [2] SUNDANESE SIGN PANYECEK..SUNDANESE SIGN PANGLAYAR
 1B82           ; N  # Mc         SUNDANESE SIGN PANGWISAD
 1B83..1BA0     ; N  # Lo    [30] SUNDANESE LETTER A..SUNDANESE LETTER HA
@@ -859,7 +860,7 @@
 1C5A..1C77     ; N  # Lo    [30] OL CHIKI LETTER LA..OL CHIKI LETTER OH
 1C78..1C7D     ; N  # Lm     [6] OL CHIKI MU TTUDDAG..OL CHIKI AHAD
 1C7E..1C7F     ; N  # Po     [2] OL CHIKI PUNCTUATION MUCAAD..OL CHIKI PUNCTUATION DOUBLE MUCAAD
-1C80..1C88     ; N  # Ll     [9] CYRILLIC SMALL LETTER ROUNDED VE..CYRILLIC SMALL LETTER UNBLENDED UK
+1C80..1C8A     ; N  # L&    [11] CYRILLIC SMALL LETTER ROUNDED VE..CYRILLIC SMALL LETTER TJE
 1C90..1CBA     ; N  # Lu    [43] GEORGIAN MTAVRULI CAPITAL LETTER AN..GEORGIAN MTAVRULI CAPITAL LETTER AIN
 1CBD..1CBF     ; N  # Lu     [3] GEORGIAN MTAVRULI CAPITAL LETTER AEN..GEORGIAN MTAVRULI CAPITAL LETTER LABIAL SIGN
 1CC0..1CC7     ; N  # Po     [8] SUNDANESE PUNCTUATION BINDU SURYA..SUNDANESE PUNCTUATION BINDU BA SATANGA
@@ -1142,7 +1143,7 @@
 23F1..23F2     ; N  # So     [2] STOPWATCH..TIMER CLOCK
 23F3           ; W  # So         HOURGLASS WITH FLOWING SAND
 23F4..23FF     ; N  # So    [12] BLACK MEDIUM LEFT-POINTING TRIANGLE..OBSERVER EYE SYMBOL
-2400..2426     ; N  # So    [39] SYMBOL FOR NULL..SYMBOL FOR SUBSTITUTE FORM TWO
+2400..2429     ; N  # So    [42] SYMBOL FOR NULL..SYMBOL FOR DELETE MEDIUM SHADE FORM
 2440..244A     ; N  # So    [11] OCR HOOK..OCR DOUBLE BACKSLASH
 2460..249B     ; A  # No    [60] CIRCLED DIGIT ONE..NUMBER TWENTY FULL STOP
 249C..24E9     ; A  # So    [78] PARENTHESIZED LATIN SMALL LETTER A..CIRCLED LATIN SMALL LETTER Z
@@ -1195,7 +1196,9 @@
 261C           ; A  # So         WHITE LEFT POINTING INDEX
 261D           ; N  # So         WHITE UP POINTING INDEX
 261E           ; A  # So         WHITE RIGHT POINTING INDEX
-261F..263F     ; N  # So    [33] WHITE DOWN POINTING INDEX..MERCURY
+261F..262F     ; N  # So    [17] WHITE DOWN POINTING INDEX..YIN YANG
+2630..2637     ; W  # So     [8] TRIGRAM FOR HEAVEN..TRIGRAM FOR EARTH
+2638..263F     ; N  # So     [8] WHEEL OF DHARMA..MERCURY
 2640           ; A  # So         FEMALE SIGN
 2641           ; N  # So         EARTH
 2642           ; A  # So         MALE SIGN
@@ -1213,7 +1216,9 @@
 266F           ; A  # Sm         MUSIC SHARP SIGN
 2670..267E     ; N  # So    [15] WEST SYRIAC CROSS..PERMANENT PAPER SIGN
 267F           ; W  # So         WHEELCHAIR SYMBOL
-2680..2692     ; N  # So    [19] DIE FACE-1..HAMMER AND PICK
+2680..2689     ; N  # So    [10] DIE FACE-1..BLACK CIRCLE WITH TWO WHITE DOTS
+268A..268F     ; W  # So     [6] MONOGRAM FOR YANG..DIGRAM FOR GREATER YIN
+2690..2692     ; N  # So     [3] WHITE FLAG..HAMMER AND PICK
 2693           ; W  # So         ANCHOR
 2694..269D     ; N  # So    [10] CROSSED SWORDS..OUTLINED WHITE STAR
 269E..269F     ; A  # So     [2] THREE LINES CONVERGING RIGHT..THREE LINES CONVERGING LEFT
@@ -1487,7 +1492,7 @@
 3192..3195     ; W  # No     [4] IDEOGRAPHIC ANNOTATION ONE MARK..IDEOGRAPHIC ANNOTATION FOUR MARK
 3196..319F     ; W  # So    [10] IDEOGRAPHIC ANNOTATION TOP MARK..IDEOGRAPHIC ANNOTATION MAN MARK
 31A0..31BF     ; W  # Lo    [32] BOPOMOFO LETTER BU..BOPOMOFO LETTER AH
-31C0..31E3     ; W  # So    [36] CJK STROKE T..CJK STROKE Q
+31C0..31E5     ; W  # So    [38] CJK STROKE T..CJK STROKE SZP
 31EF           ; W  # So         IDEOGRAPHIC DESCRIPTION CHARACTER SUBTRACTION
 31F0..31FF     ; W  # Lo    [16] KATAKANA LETTER SMALL KU..KATAKANA LETTER SMALL RO
 3200..321E     ; W  # So    [31] PARENTHESIZED HANGUL KIYEOK..PARENTHESIZED KOREAN CHARACTER O HU
@@ -1503,7 +1508,7 @@
 32C0..32FF     ; W  # So    [64] IDEOGRAPHIC TELEGRAPH SYMBOL FOR JANUARY..SQUARE ERA NAME REIWA
 3300..33FF     ; W  # So   [256] SQUARE APAATO..SQUARE GAL
 3400..4DBF     ; W  # Lo  [6592] CJK UNIFIED IDEOGRAPH-3400..CJK UNIFIED IDEOGRAPH-4DBF
-4DC0..4DFF     ; N  # So    [64] HEXAGRAM FOR THE CREATIVE HEAVEN..HEXAGRAM FOR BEFORE COMPLETION
+4DC0..4DFF     ; W  # So    [64] HEXAGRAM FOR THE CREATIVE HEAVEN..HEXAGRAM FOR BEFORE COMPLETION
 4E00..9FFF     ; W  # Lo [20992] CJK UNIFIED IDEOGRAPH-4E00..CJK UNIFIED IDEOGRAPH-9FFF
 A000..A014     ; W  # Lo    [21] YI SYLLABLE IT..YI SYLLABLE E
 A015           ; W  # Lm         YI SYLLABLE WU
@@ -1543,10 +1548,10 @@ A788           ; N  # Lm         MODIFIER LETTER LOW CIRCUMFLEX ACCENT
 A789..A78A     ; N  # Sk     [2] MODIFIER LETTER COLON..MODIFIER LETTER SHORT EQUALS SIGN
 A78B..A78E     ; N  # L&     [4] LATIN CAPITAL LETTER SALTILLO..LATIN SMALL LETTER L WITH RETROFLEX HOOK AND BELT
 A78F           ; N  # Lo         LATIN LETTER SINOLOGICAL DOT
-A790..A7CA     ; N  # L&    [59] LATIN CAPITAL LETTER N WITH DESCENDER..LATIN SMALL LETTER S WITH SHORT STROKE OVERLAY
+A790..A7CD     ; N  # L&    [62] LATIN CAPITAL LETTER N WITH DESCENDER..LATIN SMALL LETTER S WITH DIAGONAL STROKE
 A7D0..A7D1     ; N  # L&     [2] LATIN CAPITAL LETTER CLOSED INSULAR G..LATIN SMALL LETTER CLOSED INSULAR G
 A7D3           ; N  # Ll         LATIN SMALL LETTER DOUBLE THORN
-A7D5..A7D9     ; N  # L&     [5] LATIN SMALL LETTER DOUBLE WYNN..LATIN SMALL LETTER SIGMOID S
+A7D5..A7DC     ; N  # L&     [8] LATIN SMALL LETTER DOUBLE WYNN..LATIN CAPITAL LETTER LAMBDA WITH STROKE
 A7F2..A7F4     ; N  # Lm     [3] MODIFIER LETTER CAPITAL C..MODIFIER LETTER CAPITAL Q
 A7F5..A7F6     ; N  # L&     [2] LATIN CAPITAL LETTER REVERSED HALF H..LATIN SMALL LETTER REVERSED HALF H
 A7F7           ; N  # Lo         LATIN EPIGRAPHIC LETTER SIDEWAYS I
@@ -1870,6 +1875,7 @@ FFFD           ; A  # So         REPLACEMENT CHARACTER
 105A3..105B1   ; N  # Ll    [15] VITHKUQI SMALL LETTER HA..VITHKUQI SMALL LETTER RE
 105B3..105B9   ; N  # Ll     [7] VITHKUQI SMALL LETTER SE..VITHKUQI SMALL LETTER XE
 105BB..105BC   ; N  # Ll     [2] VITHKUQI SMALL LETTER Y..VITHKUQI SMALL LETTER ZE
+105C0..105F3   ; N  # Lo    [52] TODHRI LETTER A..TODHRI LETTER OO
 10600..10736   ; N  # Lo   [311] LINEAR A SIGN AB001..LINEAR A SIGN A664
 10740..10755   ; N  # Lo    [22] LINEAR A SIGN A701 A..LINEAR A SIGN A732 JE
 10760..10767   ; N  # Lo     [8] LINEAR A SIGN A800..LINEAR A SIGN A807
@@ -1942,12 +1948,23 @@ FFFD           ; A  # So         REPLACEMENT CHARACTER
 10D00..10D23   ; N  # Lo    [36] HANIFI ROHINGYA LETTER A..HANIFI ROHINGYA MARK NA KHONNA
 10D24..10D27   ; N  # Mn     [4] HANIFI ROHINGYA SIGN HARBAHAY..HANIFI ROHINGYA SIGN TASSI
 10D30..10D39   ; N  # Nd    [10] HANIFI ROHINGYA DIGIT ZERO..HANIFI ROHINGYA DIGIT NINE
+10D40..10D49   ; N  # Nd    [10] GARAY DIGIT ZERO..GARAY DIGIT NINE
+10D4A..10D4D   ; N  # Lo     [4] GARAY VOWEL SIGN A..GARAY VOWEL SIGN EE
+10D4E          ; N  # Lm         GARAY VOWEL LENGTH MARK
+10D4F          ; N  # Lo         GARAY SUKUN
+10D50..10D65   ; N  # Lu    [22] GARAY CAPITAL LETTER A..GARAY CAPITAL LETTER OLD NA
+10D69..10D6D   ; N  # Mn     [5] GARAY VOWEL SIGN E..GARAY CONSONANT NASALIZATION MARK
+10D6E          ; N  # Pd         GARAY HYPHEN
+10D6F          ; N  # Lm         GARAY REDUPLICATION MARK
+10D70..10D85   ; N  # Ll    [22] GARAY SMALL LETTER A..GARAY SMALL LETTER OLD NA
+10D8E..10D8F   ; N  # Sm     [2] GARAY PLUS SIGN..GARAY MINUS SIGN
 10E60..10E7E   ; N  # No    [31] RUMI DIGIT ONE..RUMI FRACTION TWO THIRDS
 10E80..10EA9   ; N  # Lo    [42] YEZIDI LETTER ELIF..YEZIDI LETTER ET
 10EAB..10EAC   ; N  # Mn     [2] YEZIDI COMBINING HAMZA MARK..YEZIDI COMBINING MADDA MARK
 10EAD          ; N  # Pd         YEZIDI HYPHENATION MARK
 10EB0..10EB1   ; N  # Lo     [2] YEZIDI LETTER LAM WITH DOT ABOVE..YEZIDI LETTER YOT WITH CIRCUMFLEX ABOVE
-10EFD..10EFF   ; N  # Mn     [3] ARABIC SMALL LOW WORD SAKTA..ARABIC SMALL LOW WORD MADDA
+10EC2..10EC4   ; N  # Lo     [3] ARABIC LETTER DAL WITH TWO DOTS VERTICALLY BELOW..ARABIC LETTER KAF WITH TWO DOTS VERTICALLY BELOW
+10EFC..10EFF   ; N  # Mn     [4] ARABIC COMBINING ALEF OVERLAY..ARABIC SMALL LOW WORD MADDA
 10F00..10F1C   ; N  # Lo    [29] OLD SOGDIAN LETTER ALEPH..OLD SOGDIAN LETTER FINAL TAW WITH VERTICAL TAIL
 10F1D..10F26   ; N  # No    [10] OLD SOGDIAN NUMBER ONE..OLD SOGDIAN FRACTION ONE HALF
 10F27          ; N  # Lo         OLD SOGDIAN LIGATURE AYIN-DALETH
@@ -2064,6 +2081,26 @@ FFFD           ; A  # So         REPLACEMENT CHARACTER
 11362..11363   ; N  # Mc     [2] GRANTHA VOWEL SIGN VOCALIC L..GRANTHA VOWEL SIGN VOCALIC LL
 11366..1136C   ; N  # Mn     [7] COMBINING GRANTHA DIGIT ZERO..COMBINING GRANTHA DIGIT SIX
 11370..11374   ; N  # Mn     [5] COMBINING GRANTHA LETTER A..COMBINING GRANTHA LETTER PA
+11380..11389   ; N  # Lo    [10] TULU-TIGALARI LETTER A..TULU-TIGALARI LETTER VOCALIC LL
+1138B          ; N  # Lo         TULU-TIGALARI LETTER EE
+1138E          ; N  # Lo         TULU-TIGALARI LETTER AI
+11390..113B5   ; N  # Lo    [38] TULU-TIGALARI LETTER OO..TULU-TIGALARI LETTER LLLA
+113B7          ; N  # Lo         TULU-TIGALARI SIGN AVAGRAHA
+113B8..113BA   ; N  # Mc     [3] TULU-TIGALARI VOWEL SIGN AA..TULU-TIGALARI VOWEL SIGN II
+113BB..113C0   ; N  # Mn     [6] TULU-TIGALARI VOWEL SIGN U..TULU-TIGALARI VOWEL SIGN VOCALIC LL
+113C2          ; N  # Mc         TULU-TIGALARI VOWEL SIGN EE
+113C5          ; N  # Mc         TULU-TIGALARI VOWEL SIGN AI
+113C7..113CA   ; N  # Mc     [4] TULU-TIGALARI VOWEL SIGN OO..TULU-TIGALARI SIGN CANDRA ANUNASIKA
+113CC..113CD   ; N  # Mc     [2] TULU-TIGALARI SIGN ANUSVARA..TULU-TIGALARI SIGN VISARGA
+113CE          ; N  # Mn         TULU-TIGALARI SIGN VIRAMA
+113CF          ; N  # Mc         TULU-TIGALARI SIGN LOOPED VIRAMA
+113D0          ; N  # Mn         TULU-TIGALARI CONJOINER
+113D1          ; N  # Lo         TULU-TIGALARI REPHA
+113D2          ; N  # Mn         TULU-TIGALARI GEMINATION MARK
+113D3          ; N  # Lo         TULU-TIGALARI SIGN PLUTA
+113D4..113D5   ; N  # Po     [2] TULU-TIGALARI DANDA..TULU-TIGALARI DOUBLE DANDA
+113D7..113D8   ; N  # Po     [2] TULU-TIGALARI SIGN OM PUSHPIKA..TULU-TIGALARI SIGN SHRII PUSHPIKA
+113E1..113E2   ; N  # Mn     [2] TULU-TIGALARI VEDIC TONE SVARITA..TULU-TIGALARI VEDIC TONE ANUDATTA
 11400..11434   ; N  # Lo    [53] NEWA LETTER A..NEWA LETTER HA
 11435..11437   ; N  # Mc     [3] NEWA VOWEL SIGN AA..NEWA VOWEL SIGN II
 11438..1143F   ; N  # Mn     [8] NEWA VOWEL SIGN U..NEWA VOWEL SIGN AI
@@ -2123,8 +2160,11 @@ FFFD           ; A  # So         REPLACEMENT CHARACTER
 116B8          ; N  # Lo         TAKRI LETTER ARCHAIC KHA
 116B9          ; N  # Po         TAKRI ABBREVIATION SIGN
 116C0..116C9   ; N  # Nd    [10] TAKRI DIGIT ZERO..TAKRI DIGIT NINE
+116D0..116E3   ; N  # Nd    [20] MYANMAR PAO DIGIT ZERO..MYANMAR EASTERN PWO KAREN DIGIT NINE
 11700..1171A   ; N  # Lo    [27] AHOM LETTER KA..AHOM LETTER ALTERNATE BA
-1171D..1171F   ; N  # Mn     [3] AHOM CONSONANT SIGN MEDIAL LA..AHOM CONSONANT SIGN MEDIAL LIGATING RA
+1171D          ; N  # Mn         AHOM CONSONANT SIGN MEDIAL LA
+1171E          ; N  # Mc         AHOM CONSONANT SIGN MEDIAL RA
+1171F          ; N  # Mn         AHOM CONSONANT SIGN MEDIAL LIGATING RA
 11720..11721   ; N  # Mc     [2] AHOM VOWEL SIGN A..AHOM VOWEL SIGN AA
 11722..11725   ; N  # Mn     [4] AHOM VOWEL SIGN I..AHOM VOWEL SIGN UU
 11726          ; N  # Mc         AHOM VOWEL SIGN E
@@ -2195,6 +2235,9 @@ FFFD           ; A  # So         REPLACEMENT CHARACTER
 11AB0..11ABF   ; N  # Lo    [16] CANADIAN SYLLABICS NATTILIK HI..CANADIAN SYLLABICS SPA
 11AC0..11AF8   ; N  # Lo    [57] PAU CIN HAU LETTER PA..PAU CIN HAU GLOTTAL STOP FINAL
 11B00..11B09   ; N  # Po    [10] DEVANAGARI HEAD MARK..DEVANAGARI SIGN MINDU
+11BC0..11BE0   ; N  # Lo    [33] SUNUWAR LETTER DEVI..SUNUWAR LETTER KLOKO
+11BE1          ; N  # Po         SUNUWAR SIGN PVO
+11BF0..11BF9   ; N  # Nd    [10] SUNUWAR DIGIT ZERO..SUNUWAR DIGIT NINE
 11C00..11C08   ; N  # Lo     [9] BHAIKSUKI LETTER A..BHAIKSUKI LETTER VOCALIC L
 11C0A..11C2E   ; N  # Lo    [37] BHAIKSUKI LETTER E..BHAIKSUKI LETTER HA
 11C2F          ; N  # Mc         BHAIKSUKI VOWEL SIGN AA
@@ -2253,6 +2296,7 @@ FFFD           ; A  # So         REPLACEMENT CHARACTER
 11F42          ; N  # Mn         KAWI CONJOINER
 11F43..11F4F   ; N  # Po    [13] KAWI DANDA..KAWI PUNCTUATION CLOSING SPIRAL
 11F50..11F59   ; N  # Nd    [10] KAWI DIGIT ZERO..KAWI DIGIT NINE
+11F5A          ; N  # Mn         KAWI SIGN NUKTA
 11FB0          ; N  # Lo         LISU LETTER YHA
 11FC0..11FD4   ; N  # No    [21] TAMIL FRACTION ONE THREE-HUNDRED-AND-TWENTIETH..TAMIL FRACTION DOWNSCALING FACTOR KIIZH
 11FD5..11FDC   ; N  # So     [8] TAMIL SIGN NEL..TAMIL SIGN MUKKURUNI
@@ -2270,7 +2314,13 @@ FFFD           ; A  # So         REPLACEMENT CHARACTER
 13440          ; N  # Mn         EGYPTIAN HIEROGLYPH MIRROR HORIZONTALLY
 13441..13446   ; N  # Lo     [6] EGYPTIAN HIEROGLYPH FULL BLANK..EGYPTIAN HIEROGLYPH WIDE LOST SIGN
 13447..13455   ; N  # Mn    [15] EGYPTIAN HIEROGLYPH MODIFIER DAMAGED AT TOP START..EGYPTIAN HIEROGLYPH MODIFIER DAMAGED
+13460..143FA   ; N  # Lo  [3995] EGYPTIAN HIEROGLYPH-13460..EGYPTIAN HIEROGLYPH-143FA
 14400..14646   ; N  # Lo   [583] ANATOLIAN HIEROGLYPH A001..ANATOLIAN HIEROGLYPH A530
+16100..1611D   ; N  # Lo    [30] GURUNG KHEMA LETTER A..GURUNG KHEMA LETTER SA
+1611E..16129   ; N  # Mn    [12] GURUNG KHEMA VOWEL SIGN AA..GURUNG KHEMA VOWEL LENGTH MARK
+1612A..1612C   ; N  # Mc     [3] GURUNG KHEMA CONSONANT SIGN MEDIAL YA..GURUNG KHEMA CONSONANT SIGN MEDIAL HA
+1612D..1612F   ; N  # Mn     [3] GURUNG KHEMA SIGN ANUSVARA..GURUNG KHEMA SIGN THOLHOMA
+16130..16139   ; N  # Nd    [10] GURUNG KHEMA DIGIT ZERO..GURUNG KHEMA DIGIT NINE
 16800..16A38   ; N  # Lo   [569] BAMUM LETTER PHASE-A NGKUE MFON..BAMUM LETTER PHASE-F VUEQ
 16A40..16A5E   ; N  # Lo    [31] MRO LETTER TA..MRO LETTER TEK
 16A60..16A69   ; N  # Nd    [10] MRO DIGIT ZERO..MRO DIGIT NINE
@@ -2291,6 +2341,11 @@ FFFD           ; A  # So         REPLACEMENT CHARACTER
 16B5B..16B61   ; N  # No     [7] PAHAWH HMONG NUMBER TENS..PAHAWH HMONG NUMBER TRILLIONS
 16B63..16B77   ; N  # Lo    [21] PAHAWH HMONG SIGN VOS LUB..PAHAWH HMONG SIGN CIM NRES TOS
 16B7D..16B8F   ; N  # Lo    [19] PAHAWH HMONG CLAN SIGN TSHEEJ..PAHAWH HMONG CLAN SIGN VWJ
+16D40..16D42   ; N  # Lm     [3] KIRAT RAI SIGN ANUSVARA..KIRAT RAI SIGN VISARGA
+16D43..16D6A   ; N  # Lo    [40] KIRAT RAI LETTER A..KIRAT RAI VOWEL SIGN AU
+16D6B..16D6C   ; N  # Lm     [2] KIRAT RAI SIGN VIRAMA..KIRAT RAI SIGN SAAT
+16D6D..16D6F   ; N  # Po     [3] KIRAT RAI SIGN YUPI..KIRAT RAI DOUBLE DANDA
+16D70..16D79   ; N  # Nd    [10] KIRAT RAI DIGIT ZERO..KIRAT RAI DIGIT NINE
 16E40..16E7F   ; N  # L&    [64] MEDEFAIDRIN CAPITAL LETTER M..MEDEFAIDRIN SMALL LETTER Y
 16E80..16E96   ; N  # No    [23] MEDEFAIDRIN DIGIT ZERO..MEDEFAIDRIN DIGIT THREE ALTERNATE FORM
 16E97..16E9A   ; N  # Po     [4] MEDEFAIDRIN COMMA..MEDEFAIDRIN EXCLAMATION OH
@@ -2308,6 +2363,7 @@ FFFD           ; A  # So         REPLACEMENT CHARACTER
 17000..187F7   ; W  # Lo  [6136] TANGUT IDEOGRAPH-17000..TANGUT IDEOGRAPH-187F7
 18800..18AFF   ; W  # Lo   [768] TANGUT COMPONENT-001..TANGUT COMPONENT-768
 18B00..18CD5   ; W  # Lo   [470] KHITAN SMALL SCRIPT CHARACTER-18B00..KHITAN SMALL SCRIPT CHARACTER-18CD5
+18CFF          ; W  # Lo         KHITAN SMALL SCRIPT CHARACTER-18CFF
 18D00..18D08   ; W  # Lo     [9] TANGUT IDEOGRAPH-18D00..TANGUT IDEOGRAPH-18D08
 1AFF0..1AFF3   ; W  # Lm     [4] KATAKANA LETTER MINNAN TONE-2..KATAKANA LETTER MINNAN TONE-5
 1AFF5..1AFFB   ; W  # Lm     [7] KATAKANA LETTER MINNAN TONE-7..KATAKANA LETTER MINNAN NASALIZED TONE-5
@@ -2327,6 +2383,9 @@ FFFD           ; A  # So         REPLACEMENT CHARACTER
 1BC9D..1BC9E   ; N  # Mn     [2] DUPLOYAN THICK LETTER SELECTOR..DUPLOYAN DOUBLE MARK
 1BC9F          ; N  # Po         DUPLOYAN PUNCTUATION CHINOOK FULL STOP
 1BCA0..1BCA3   ; N  # Cf     [4] SHORTHAND FORMAT LETTER OVERLAP..SHORTHAND FORMAT UP STEP
+1CC00..1CCEF   ; N  # So   [240] UP-POINTING GO-KART..OUTLINED LATIN CAPITAL LETTER Z
+1CCF0..1CCF9   ; N  # Nd    [10] OUTLINED DIGIT ZERO..OUTLINED DIGIT NINE
+1CD00..1CEB3   ; N  # So   [436] BLOCK OCTANT-3..BLACK RIGHT TRIANGLE CARET
 1CF00..1CF2D   ; N  # Mn    [46] ZNAMENNY COMBINING MARK GORAZDO NIZKO S KRYZHEM ON LEFT..ZNAMENNY COMBINING MARK KRYZH ON LEFT
 1CF30..1CF46   ; N  # Mn    [23] ZNAMENNY COMBINING TONAL RANGE MARK MRACHNO..ZNAMENNY PRIZNAK MODIFIER ROG
 1CF50..1CFC3   ; N  # So   [116] ZNAMENNY NEUME KRYUK..ZNAMENNY NEUME PAUK
@@ -2349,8 +2408,9 @@ FFFD           ; A  # So         REPLACEMENT CHARACTER
 1D245          ; N  # So         GREEK MUSICAL LEIMMA
 1D2C0..1D2D3   ; N  # No    [20] KAKTOVIK NUMERAL ZERO..KAKTOVIK NUMERAL NINETEEN
 1D2E0..1D2F3   ; N  # No    [20] MAYAN NUMERAL ZERO..MAYAN NUMERAL NINETEEN
-1D300..1D356   ; N  # So    [87] MONOGRAM FOR EARTH..TETRAGRAM FOR FOSTERING
-1D360..1D378   ; N  # No    [25] COUNTING ROD UNIT DIGIT ONE..TALLY MARK FIVE
+1D300..1D356   ; W  # So    [87] MONOGRAM FOR EARTH..TETRAGRAM FOR FOSTERING
+1D360..1D376   ; W  # No    [23] COUNTING ROD UNIT DIGIT ONE..IDEOGRAPHIC TALLY MARK FIVE
+1D377..1D378   ; N  # No     [2] TALLY MARK ONE..TALLY MARK FIVE
 1D400..1D454   ; N  # L&    [85] MATHEMATICAL BOLD CAPITAL A..MATHEMATICAL ITALIC SMALL G
 1D456..1D49C   ; N  # L&    [71] MATHEMATICAL ITALIC SMALL I..MATHEMATICAL SCRIPT CAPITAL A
 1D49E..1D49F   ; N  # Lu     [2] MATHEMATICAL SCRIPT CAPITAL C..MATHEMATICAL SCRIPT CAPITAL D
@@ -2431,6 +2491,11 @@ FFFD           ; A  # So         REPLACEMENT CHARACTER
 1E4EB          ; N  # Lm         NAG MUNDARI SIGN OJOD
 1E4EC..1E4EF   ; N  # Mn     [4] NAG MUNDARI SIGN MUHOR..NAG MUNDARI SIGN SUTUH
 1E4F0..1E4F9   ; N  # Nd    [10] NAG MUNDARI DIGIT ZERO..NAG MUNDARI DIGIT NINE
+1E5D0..1E5ED   ; N  # Lo    [30] OL ONAL LETTER O..OL ONAL LETTER EG
+1E5EE..1E5EF   ; N  # Mn     [2] OL ONAL SIGN MU..OL ONAL SIGN IKIR
+1E5F0          ; N  # Lo         OL ONAL SIGN HODDOND
+1E5F1..1E5FA   ; N  # Nd    [10] OL ONAL DIGIT ZERO..OL ONAL DIGIT NINE
+1E5FF          ; N  # Po         OL ONAL ABBREVIATION SIGN
 1E7E0..1E7E6   ; N  # Lo     [7] ETHIOPIC SYLLABLE HHYA..ETHIOPIC SYLLABLE HHYO
 1E7E8..1E7EB   ; N  # Lo     [4] ETHIOPIC SYLLABLE GURAGE HHWA..ETHIOPIC SYLLABLE HHWE
 1E7ED..1E7EE   ; N  # Lo     [2] ETHIOPIC SYLLABLE GURAGE MWI..ETHIOPIC SYLLABLE GURAGE MWEE
@@ -2574,7 +2639,8 @@ FFFD           ; A  # So         REPLACEMENT CHARACTER
 1F850..1F859   ; N  # So    [10] LEFTWARDS SANS-SERIF ARROW..UP DOWN SANS-SERIF ARROW
 1F860..1F887   ; N  # So    [40] WIDE-HEADED LEFTWARDS LIGHT BARB ARROW..WIDE-HEADED SOUTH WEST VERY HEAVY BARB ARROW
 1F890..1F8AD   ; N  # So    [30] LEFTWARDS TRIANGLE ARROWHEAD..WHITE ARROW SHAFT WIDTH TWO THIRDS
-1F8B0..1F8B1   ; N  # So     [2] ARROW POINTING UPWARDS THEN NORTH WEST..ARROW POINTING RIGHTWARDS THEN CURVING SOUTH WEST
+1F8B0..1F8BB   ; N  # So    [12] ARROW POINTING UPWARDS THEN NORTH WEST..SOUTH WEST ARROW FROM BAR
+1F8C0..1F8C1   ; N  # So     [2] LEFTWARDS ARROW FROM DOWNWARDS ARROW..RIGHTWARDS ARROW FROM DOWNWARDS ARROW
 1F900..1F90B   ; N  # So    [12] CIRCLED CROSS FORMEE WITH FOUR DOTS..DOWNWARD FACING NOTCHED HOOK WITH DOT
 1F90C..1F93A   ; W  # So    [47] PINCHED FINGERS..FENCER
 1F93B          ; N  # So         MODERN PENTATHLON
@@ -2584,14 +2650,13 @@ FFFD           ; A  # So         REPLACEMENT CHARACTER
 1FA00..1FA53   ; N  # So    [84] NEUTRAL CHESS KING..BLACK CHESS KNIGHT-BISHOP
 1FA60..1FA6D   ; N  # So    [14] XIANGQI RED GENERAL..XIANGQI BLACK SOLDIER
 1FA70..1FA7C   ; W  # So    [13] BALLET SHOES..CRUTCH
-1FA80..1FA88   ; W  # So     [9] YO-YO..FLUTE
-1FA90..1FABD   ; W  # So    [46] RINGED PLANET..WING
-1FABF..1FAC5   ; W  # So     [7] GOOSE..PERSON WITH CROWN
-1FACE..1FADB   ; W  # So    [14] MOOSE..PEA POD
-1FAE0..1FAE8   ; W  # So     [9] MELTING FACE..SHAKING FACE
+1FA80..1FA89   ; W  # So    [10] YO-YO..HARP
+1FA8F..1FAC6   ; W  # So    [56] SHOVEL..FINGERPRINT
+1FACE..1FADC   ; W  # So    [15] MOOSE..ROOT VEGETABLE
+1FADF..1FAE9   ; W  # So    [11] SPLATTER..FACE WITH BAGS UNDER EYES
 1FAF0..1FAF8   ; W  # So     [9] HAND WITH INDEX FINGER AND THUMB CROSSED..RIGHTWARDS PUSHING HAND
 1FB00..1FB92   ; N  # So   [147] BLOCK SEXTANT-1..UPPER HALF INVERSE MEDIUM SHADE AND LOWER HALF BLOCK
-1FB94..1FBCA   ; N  # So    [55] LEFT HALF INVERSE MEDIUM SHADE AND RIGHT HALF BLOCK..WHITE UP-POINTING CHEVRON
+1FB94..1FBEF   ; N  # So    [92] LEFT HALF INVERSE MEDIUM SHADE AND RIGHT HALF BLOCK..TOP LEFT JUSTIFIED LOWER RIGHT QUARTER BLACK CIRCLE
 1FBF0..1FBF9   ; N  # Nd    [10] SEGMENTED DIGIT ZERO..SEGMENTED DIGIT NINE
 20000..2A6DF   ; W  # Lo [42720] CJK UNIFIED IDEOGRAPH-20000..CJK UNIFIED IDEOGRAPH-2A6DF
 2A6E0..2A6FF   ; W  # Cn    [32] <reserved-2A6E0>..<reserved-2A6FF>
diff --git a/libcxx/utils/data/unicode/GraphemeBreakProperty.txt b/libcxx/utils/data/unicode/GraphemeBreakProperty.txt
index 12453cbdb54a1..a863397ddabaf 100644
--- a/libcxx/utils/data/unicode/GraphemeBreakProperty.txt
+++ b/libcxx/utils/data/unicode/GraphemeBreakProperty.txt
@@ -1,8 +1,8 @@
-# GraphemeBreakProperty-15.1.0.txt
-# Date: 2023-01-05, 20:34:41 GMT
-# © 2023 Unicode®, Inc.
+# GraphemeBreakProperty-16.0.0.txt
+# Date: 2024-05-31, 18:09:38 GMT
+# © 2024 Unicode®, Inc.
 # Unicode and the Unicode Logo are registered trademarks of Unicode, Inc. in the U.S. and other countries.
-# For terms of use, see https://www.unicode.org/terms_of_use.html
+# For terms of use and license, see https://www.unicode.org/terms_of_use.html
 #
 # Unicode Character Database
 #   For documentation, see https://www.unicode.org/reports/tr44/
@@ -27,6 +27,7 @@
 110BD         ; Prepend # Cf       KAITHI NUMBER SIGN
 110CD         ; Prepend # Cf       KAITHI NUMBER SIGN ABOVE
 111C2..111C3  ; Prepend # Lo   [2] SHARADA SIGN JIHVAMULIYA..SHARADA SIGN UPADHMANIYA
+113D1         ; Prepend # Lo       TULU-TIGALARI REPHA
 1193F         ; Prepend # Lo       DIVES AKURU PREFIXED NASAL SIGN
 11941         ; Prepend # Lo       DIVES AKURU INITIAL RA
 11A3A         ; Prepend # Lo       ZANABAZAR SQUARE CLUSTER-INITIAL LETTER RA
@@ -34,7 +35,7 @@
 11D46         ; Prepend # Lo       MASARAM GONDI REPHA
 11F02         ; Prepend # Lo       KAWI SIGN REPHA
 
-# Total code points: 27
+# Total code points: 28
 
 # ================================================
 
@@ -106,7 +107,7 @@ E01F0..E0FFF  ; Control # Cn [3600] <reserved-E01F0>..<reserved-E0FFF>
 0825..0827    ; Extend # Mn   [3] SAMARITAN VOWEL SIGN SHORT A..SAMARITAN VOWEL SIGN U
 0829..082D    ; Extend # Mn   [5] SAMARITAN VOWEL SIGN LONG I..SAMARITAN MARK NEQUDAA
 0859..085B    ; Extend # Mn   [3] MANDAIC AFFRICATION MARK..MANDAIC GEMINATION MARK
-0898..089F    ; Extend # Mn   [8] ARABIC SMALL HIGH WORD AL-JUZ..ARABIC HALF MADDA OVER MADDA
+0897..089F    ; Extend # Mn   [9] ARABIC PEPET..ARABIC HALF MADDA OVER MADDA
 08CA..08E1    ; Extend # Mn  [24] ARABIC SMALL HIGH FARSI YEH..ARABIC SMALL HIGH SIGN SAFHA
 08E3..0902    ; Extend # Mn  [32] ARABIC TURNED DAMMA BELOW..DEVANAGARI SIGN ANUSVARA
 093A          ; Extend # Mn       DEVANAGARI VOWEL SIGN OE
@@ -163,8 +164,11 @@ E01F0..E0FFF  ; Control # Cn [3600] <reserved-E01F0>..<reserved-E0FFF>
 0C81          ; Extend # Mn       KANNADA SIGN CANDRABINDU
 0CBC          ; Extend # Mn       KANNADA SIGN NUKTA
 0CBF          ; Extend # Mn       KANNADA VOWEL SIGN I
+0CC0          ; Extend # Mc       KANNADA VOWEL SIGN II
 0CC2          ; Extend # Mc       KANNADA VOWEL SIGN UU
 0CC6          ; Extend # Mn       KANNADA VOWEL SIGN E
+0CC7..0CC8    ; Extend # Mc   [2] KANNADA VOWEL SIGN EE..KANNADA VOWEL SIGN AI
+0CCA..0CCB    ; Extend # Mc   [2] KANNADA VOWEL SIGN O..KANNADA VOWEL SIGN OO
 0CCC..0CCD    ; Extend # Mn   [2] KANNADA VOWEL SIGN AU..KANNADA SIGN VIRAMA
 0CD5..0CD6    ; Extend # Mc   [2] KANNADA LENGTH MARK..KANNADA AI LENGTH MARK
 0CE2..0CE3    ; Extend # Mn   [2] KANNADA VOWEL SIGN VOCALIC L..KANNADA VOWEL SIGN VOCALIC LL
@@ -210,7 +214,9 @@ E01F0..E0FFF  ; Control # Cn [3600] <reserved-E01F0>..<reserved-E0FFF>
 109D          ; Extend # Mn       MYANMAR VOWEL SIGN AITON AI
 135D..135F    ; Extend # Mn   [3] ETHIOPIC COMBINING GEMINATION AND VOWEL LENGTH MARK..ETHIOPIC COMBINING GEMINATION MARK
 1712..1714    ; Extend # Mn   [3] TAGALOG VOWEL SIGN I..TAGALOG SIGN VIRAMA
+1715          ; Extend # Mc       TAGALOG SIGN PAMUDPOD
 1732..1733    ; Extend # Mn   [2] HANUNOO VOWEL SIGN I..HANUNOO VOWEL SIGN U
+1734          ; Extend # Mc       HANUNOO SIGN PAMUDPOD
 1752..1753    ; Extend # Mn   [2] BUHID VOWEL SIGN I..BUHID VOWEL SIGN U
 1772..1773    ; Extend # Mn   [2] TAGBANWA VOWEL SIGN I..TAGBANWA VOWEL SIGN U
 17B4..17B5    ; Extend # Mn   [2] KHMER VOWEL INHERENT AQ..KHMER VOWEL INHERENT AA
@@ -242,17 +248,22 @@ E01F0..E0FFF  ; Control # Cn [3600] <reserved-E01F0>..<reserved-E0FFF>
 1B34          ; Extend # Mn       BALINESE SIGN REREKAN
 1B35          ; Extend # Mc       BALINESE VOWEL SIGN TEDUNG
 1B36..1B3A    ; Extend # Mn   [5] BALINESE VOWEL SIGN ULU..BALINESE VOWEL SIGN RA REPA
+1B3B          ; Extend # Mc       BALINESE VOWEL SIGN RA REPA TEDUNG
 1B3C          ; Extend # Mn       BALINESE VOWEL SIGN LA LENGA
+1B3D          ; Extend # Mc       BALINESE VOWEL SIGN LA LENGA TEDUNG
 1B42          ; Extend # Mn       BALINESE VOWEL SIGN PEPET
+1B43..1B44    ; Extend # Mc   [2] BALINESE VOWEL SIGN PEPET TEDUNG..BALINESE ADEG ADEG
 1B6B..1B73    ; Extend # Mn   [9] BALINESE MUSICAL SYMBOL COMBINING TEGEH..BALINESE MUSICAL SYMBOL COMBINING GONG
 1B80..1B81    ; Extend # Mn   [2] SUNDANESE SIGN PANYECEK..SUNDANESE SIGN PANGLAYAR
 1BA2..1BA5    ; Extend # Mn   [4] SUNDANESE CONSONANT SIGN PANYAKRA..SUNDANESE VOWEL SIGN PANYUKU
 1BA8..1BA9    ; Extend # Mn   [2] SUNDANESE VOWEL SIGN PAMEPET..SUNDANESE VOWEL SIGN PANEULEUNG
+1BAA          ; Extend # Mc       SUNDANESE SIGN PAMAAEH
 1BAB..1BAD    ; Extend # Mn   [3] SUNDANESE SIGN VIRAMA..SUNDANESE CONSONANT SIGN PASANGAN WA
 1BE6          ; Extend # Mn       BATAK SIGN TOMPI
 1BE8..1BE9    ; Extend # Mn   [2] BATAK VOWEL SIGN PAKPAK E..BATAK VOWEL SIGN EE
 1BED          ; Extend # Mn       BATAK VOWEL SIGN KARO O
 1BEF..1BF1    ; Extend # Mn   [3] BATAK VOWEL SIGN U FOR SIMALUNGUN SA..BATAK CONSONANT SIGN H
+1BF2..1BF3    ; Extend # Mc   [2] BATAK PANGOLAT..BATAK PANONGONAN
 1C2C..1C33    ; Extend # Mn   [8] LEPCHA VOWEL SIGN E..LEPCHA CONSONANT SIGN T
 1C36..1C37    ; Extend # Mn   [2] LEPCHA SIGN RAN..LEPCHA SIGN NUKTA
 1CD0..1CD2    ; Extend # Mn   [3] VEDIC TONE KARSHANA..VEDIC TONE PRENKHA
@@ -289,10 +300,12 @@ A8E0..A8F1    ; Extend # Mn  [18] COMBINING DEVANAGARI DIGIT ZERO..COMBINING DEV
 A8FF          ; Extend # Mn       DEVANAGARI VOWEL SIGN AY
 A926..A92D    ; Extend # Mn   [8] KAYAH LI VOWEL UE..KAYAH LI TONE CALYA PLOPHU
 A947..A951    ; Extend # Mn  [11] REJANG VOWEL SIGN I..REJANG CONSONANT SIGN R
+A953          ; Extend # Mc       REJANG VIRAMA
 A980..A982    ; Extend # Mn   [3] JAVANESE SIGN PANYANGGA..JAVANESE SIGN LAYAR
 A9B3          ; Extend # Mn       JAVANESE SIGN CECAK TELU
 A9B6..A9B9    ; Extend # Mn   [4] JAVANESE VOWEL SIGN WULU..JAVANESE VOWEL SIGN SUKU MENDUT
 A9BC..A9BD    ; Extend # Mn   [2] JAVANESE VOWEL SIGN PEPET..JAVANESE CONSONANT SIGN KERET
+A9C0          ; Extend # Mc       JAVANESE PANGKON
 A9E5          ; Extend # Mn       MYANMAR SIGN SHAN SAW
 AA29..AA2E    ; Extend # Mn   [6] CHAM VOWEL SIGN AA..CHAM VOWEL SIGN OE
 AA31..AA32    ; Extend # Mn   [2] CHAM VOWEL SIGN AU..CHAM VOWEL SIGN UE
@@ -324,8 +337,9 @@ FF9E..FF9F    ; Extend # Lm   [2] HALFWIDTH KATAKANA VOICED SOUND MARK..HALFWIDT
 10A3F         ; Extend # Mn       KHAROSHTHI VIRAMA
 10AE5..10AE6  ; Extend # Mn   [2] MANICHAEAN ABBREVIATION MARK ABOVE..MANICHAEAN ABBREVIATION MARK BELOW
 10D24..10D27  ; Extend # Mn   [4] HANIFI ROHINGYA SIGN HARBAHAY..HANIFI ROHINGYA SIGN TASSI
+10D69..10D6D  ; Extend # Mn   [5] GARAY VOWEL SIGN E..GARAY CONSONANT NASALIZATION MARK
 10EAB..10EAC  ; Extend # Mn   [2] YEZIDI COMBINING HAMZA MARK..YEZIDI COMBINING MADDA MARK
-10EFD..10EFF  ; Extend # Mn   [3] ARABIC SMALL LOW WORD SAKTA..ARABIC SMALL LOW WORD MADDA
+10EFC..10EFF  ; Extend # Mn   [4] ARABIC COMBINING ALEF OVERLAY..ARABIC SMALL LOW WORD MADDA
 10F46..10F50  ; Extend # Mn  [11] SOGDIAN COMBINING DOT BELOW..SOGDIAN COMBINING STROKE BELOW
 10F82..10F85  ; Extend # Mn   [4] OLD UYGHUR COMBINING DOT ABOVE..OLD UYGHUR COMBINING TWO DOTS BELOW
 11001         ; Extend # Mn       BRAHMI SIGN ANUSVARA
@@ -342,10 +356,12 @@ FF9E..FF9F    ; Extend # Lm   [2] HALFWIDTH KATAKANA VOICED SOUND MARK..HALFWIDT
 11173         ; Extend # Mn       MAHAJANI SIGN NUKTA
 11180..11181  ; Extend # Mn   [2] SHARADA SIGN CANDRABINDU..SHARADA SIGN ANUSVARA
 111B6..111BE  ; Extend # Mn   [9] SHARADA VOWEL SIGN U..SHARADA VOWEL SIGN O
+111C0         ; Extend # Mc       SHARADA SIGN VIRAMA
 111C9..111CC  ; Extend # Mn   [4] SHARADA SANDHI MARK..SHARADA EXTRA SHORT VOWEL MARK
 111CF         ; Extend # Mn       SHARADA SIGN INVERTED CANDRABINDU
 1122F..11231  ; Extend # Mn   [3] KHOJKI VOWEL SIGN U..KHOJKI VOWEL SIGN AI
 11234         ; Extend # Mn       KHOJKI SIGN ANUSVARA
+11235         ; Extend # Mc       KHOJKI SIGN VIRAMA
 11236..11237  ; Extend # Mn   [2] KHOJKI SIGN NUKTA..KHOJKI SIGN SHADDA
 1123E         ; Extend # Mn       KHOJKI SIGN SUKUN
 11241         ; Extend # Mn       KHOJKI VOWEL SIGN VOCALIC R
@@ -355,9 +371,20 @@ FF9E..FF9F    ; Extend # Lm   [2] HALFWIDTH KATAKANA VOICED SOUND MARK..HALFWIDT
 1133B..1133C  ; Extend # Mn   [2] COMBINING BINDU BELOW..GRANTHA SIGN NUKTA
 1133E         ; Extend # Mc       GRANTHA VOWEL SIGN AA
 11340         ; Extend # Mn       GRANTHA VOWEL SIGN II
+1134D         ; Extend # Mc       GRANTHA SIGN VIRAMA
 11357         ; Extend # Mc       GRANTHA AU LENGTH MARK
 11366..1136C  ; Extend # Mn   [7] COMBINING GRANTHA DIGIT ZERO..COMBINING GRANTHA DIGIT SIX
 11370..11374  ; Extend # Mn   [5] COMBINING GRANTHA LETTER A..COMBINING GRANTHA LETTER PA
+113B8         ; Extend # Mc       TULU-TIGALARI VOWEL SIGN AA
+113BB..113C0  ; Extend # Mn   [6] TULU-TIGALARI VOWEL SIGN U..TULU-TIGALARI VOWEL SIGN VOCALIC LL
+113C2         ; Extend # Mc       TULU-TIGALARI VOWEL SIGN EE
+113C5         ; Extend # Mc       TULU-TIGALARI VOWEL SIGN AI
+113C7..113C9  ; Extend # Mc   [3] TULU-TIGALARI VOWEL SIGN OO..TULU-TIGALARI AU LENGTH MARK
+113CE         ; Extend # Mn       TULU-TIGALARI SIGN VIRAMA
+113CF         ; Extend # Mc       TULU-TIGALARI SIGN LOOPED VIRAMA
+113D0         ; Extend # Mn       TULU-TIGALARI CONJOINER
+113D2         ; Extend # Mn       TULU-TIGALARI GEMINATION MARK
+113E1..113E2  ; Extend # Mn   [2] TULU-TIGALARI VEDIC TONE SVARITA..TULU-TIGALARI VEDIC TONE ANUDATTA
 11438..1143F  ; Extend # Mn   [8] NEWA VOWEL SIGN U..NEWA VOWEL SIGN AI
 11442..11444  ; Extend # Mn   [3] NEWA SIGN VIRAMA..NEWA SIGN ANUSVARA
 11446         ; Extend # Mn       NEWA SIGN NUKTA
@@ -379,14 +406,17 @@ FF9E..FF9F    ; Extend # Lm   [2] HALFWIDTH KATAKANA VOICED SOUND MARK..HALFWIDT
 116AB         ; Extend # Mn       TAKRI SIGN ANUSVARA
 116AD         ; Extend # Mn       TAKRI VOWEL SIGN AA
 116B0..116B5  ; Extend # Mn   [6] TAKRI VOWEL SIGN U..TAKRI VOWEL SIGN AU
+116B6         ; Extend # Mc       TAKRI SIGN VIRAMA
 116B7         ; Extend # Mn       TAKRI SIGN NUKTA
-1171D..1171F  ; Extend # Mn   [3] AHOM CONSONANT SIGN MEDIAL LA..AHOM CONSONANT SIGN MEDIAL LIGATING RA
+1171D         ; Extend # Mn       AHOM CONSONANT SIGN MEDIAL LA
+1171F         ; Extend # Mn       AHOM CONSONANT SIGN MEDIAL LIGATING RA
 11722..11725  ; Extend # Mn   [4] AHOM VOWEL SIGN I..AHOM VOWEL SIGN UU
 11727..1172B  ; Extend # Mn   [5] AHOM VOWEL SIGN AW..AHOM SIGN KILLER
 1182F..11837  ; Extend # Mn   [9] DOGRA VOWEL SIGN U..DOGRA SIGN ANUSVARA
 11839..1183A  ; Extend # Mn   [2] DOGRA SIGN VIRAMA..DOGRA SIGN NUKTA
 11930         ; Extend # Mc       DIVES AKURU VOWEL SIGN AA
 1193B..1193C  ; Extend # Mn   [2] DIVES AKURU SIGN ANUSVARA..DIVES AKURU SIGN CANDRABINDU
+1193D         ; Extend # Mc       DIVES AKURU SIGN HALANTA
 1193E         ; Extend # Mn       DIVES AKURU VIRAMA
 11943         ; Extend # Mn       DIVES AKURU SIGN NUKTA
 119D4..119D7  ; Extend # Mn   [4] NANDINAGARI VOWEL SIGN U..NANDINAGARI VOWEL SIGN VOCALIC RR
@@ -419,20 +449,25 @@ FF9E..FF9F    ; Extend # Lm   [2] HALFWIDTH KATAKANA VOICED SOUND MARK..HALFWIDT
 11F00..11F01  ; Extend # Mn   [2] KAWI SIGN CANDRABINDU..KAWI SIGN ANUSVARA
 11F36..11F3A  ; Extend # Mn   [5] KAWI VOWEL SIGN I..KAWI VOWEL SIGN VOCALIC R
 11F40         ; Extend # Mn       KAWI VOWEL SIGN EU
+11F41         ; Extend # Mc       KAWI SIGN KILLER
 11F42         ; Extend # Mn       KAWI CONJOINER
+11F5A         ; Extend # Mn       KAWI SIGN NUKTA
 13440         ; Extend # Mn       EGYPTIAN HIEROGLYPH MIRROR HORIZONTALLY
 13447..13455  ; Extend # Mn  [15] EGYPTIAN HIEROGLYPH MODIFIER DAMAGED AT TOP START..EGYPTIAN HIEROGLYPH MODIFIER DAMAGED
+1611E..16129  ; Extend # Mn  [12] GURUNG KHEMA VOWEL SIGN AA..GURUNG KHEMA VOWEL LENGTH MARK
+1612D..1612F  ; Extend # Mn   [3] GURUNG KHEMA SIGN ANUSVARA..GURUNG KHEMA SIGN THOLHOMA
 16AF0..16AF4  ; Extend # Mn   [5] BASSA VAH COMBINING HIGH TONE..BASSA VAH COMBINING HIGH-LOW TONE
 16B30..16B36  ; Extend # Mn   [7] PAHAWH HMONG MARK CIM TUB..PAHAWH HMONG MARK CIM TAUM
 16F4F         ; Extend # Mn       MIAO SIGN CONSONANT MODIFIER BAR
 16F8F..16F92  ; Extend # Mn   [4] MIAO TONE RIGHT..MIAO TONE BELOW
 16FE4         ; Extend # Mn       KHITAN SMALL SCRIPT FILLER
+16FF0..16FF1  ; Extend # Mc   [2] VIETNAMESE ALTERNATE READING MARK CA..VIETNAMESE ALTERNATE READING MARK NHAY
 1BC9D..1BC9E  ; Extend # Mn   [2] DUPLOYAN THICK LETTER SELECTOR..DUPLOYAN DOUBLE MARK
 1CF00..1CF2D  ; Extend # Mn  [46] ZNAMENNY COMBINING MARK GORAZDO NIZKO S KRYZHEM ON LEFT..ZNAMENNY COMBINING MARK KRYZH ON LEFT
 1CF30..1CF46  ; Extend # Mn  [23] ZNAMENNY COMBINING TONAL RANGE MARK MRACHNO..ZNAMENNY PRIZNAK MODIFIER ROG
-1D165         ; Extend # Mc       MUSICAL SYMBOL COMBINING STEM
+1D165..1D166  ; Extend # Mc   [2] MUSICAL SYMBOL COMBINING STEM..MUSICAL SYMBOL COMBINING SPRECHGESANG STEM
 1D167..1D169  ; Extend # Mn   [3] MUSICAL SYMBOL COMBINING TREMOLO-1..MUSICAL SYMBOL COMBINING TREMOLO-3
-1D16E..1D172  ; Extend # Mc   [5] MUSICAL SYMBOL COMBINING FLAG-1..MUSICAL SYMBOL COMBINING FLAG-5
+1D16D..1D172  ; Extend # Mc   [6] MUSICAL SYMBOL COMBINING AUGMENTATION DOT..MUSICAL SYMBOL COMBINING FLAG-5
 1D17B..1D182  ; Extend # Mn   [8] MUSICAL SYMBOL COMBINING ACCENT..MUSICAL SYMBOL COMBINING LOURE
 1D185..1D18B  ; Extend # Mn   [7] MUSICAL SYMBOL COMBINING DOIT..MUSICAL SYMBOL COMBINING TRIPLE TONGUE
 1D1AA..1D1AD  ; Extend # Mn   [4] MUSICAL SYMBOL COMBINING DOWN BOW..MUSICAL SYMBOL COMBINING SNAP PIZZICATO
@@ -453,13 +488,14 @@ FF9E..FF9F    ; Extend # Lm   [2] HALFWIDTH KATAKANA VOICED SOUND MARK..HALFWIDT
 1E2AE         ; Extend # Mn       TOTO SIGN RISING TONE
 1E2EC..1E2EF  ; Extend # Mn   [4] WANCHO TONE TUP..WANCHO TONE KOINI
 1E4EC..1E4EF  ; Extend # Mn   [4] NAG MUNDARI SIGN MUHOR..NAG MUNDARI SIGN SUTUH
+1E5EE..1E5EF  ; Extend # Mn   [2] OL ONAL SIGN MU..OL ONAL SIGN IKIR
 1E8D0..1E8D6  ; Extend # Mn   [7] MENDE KIKAKUI COMBINING NUMBER TEENS..MENDE KIKAKUI COMBINING NUMBER MILLIONS
 1E944..1E94A  ; Extend # Mn   [7] ADLAM ALIF LENGTHENER..ADLAM NUKTA
 1F3FB..1F3FF  ; Extend # Sk   [5] EMOJI MODIFIER FITZPATRICK TYPE-1-2..EMOJI MODIFIER FITZPATRICK TYPE-6
 E0020..E007F  ; Extend # Cf  [96] TAG SPACE..CANCEL TAG
 E0100..E01EF  ; Extend # Mn [240] VARIATION SELECTOR-17..VARIATION SELECTOR-256
 
-# Total code points: 2130
+# Total code points: 2198
 
 # ================================================
 
@@ -496,10 +532,8 @@ E0100..E01EF  ; Extend # Mn [240] VARIATION SELECTOR-17..VARIATION SELECTOR-256
 0C41..0C44    ; SpacingMark # Mc   [4] TELUGU VOWEL SIGN U..TELUGU VOWEL SIGN VOCALIC RR
 0C82..0C83    ; SpacingMark # Mc   [2] KANNADA SIGN ANUSVARA..KANNADA SIGN VISARGA
 0CBE          ; SpacingMark # Mc       KANNADA VOWEL SIGN AA
-0CC0..0CC1    ; SpacingMark # Mc   [2] KANNADA VOWEL SIGN II..KANNADA VOWEL SIGN U
+0CC1          ; SpacingMark # Mc       KANNADA VOWEL SIGN U
 0CC3..0CC4    ; SpacingMark # Mc   [2] KANNADA VOWEL SIGN VOCALIC R..KANNADA VOWEL SIGN VOCALIC RR
-0CC7..0CC8    ; SpacingMark # Mc   [2] KANNADA VOWEL SIGN EE..KANNADA VOWEL SIGN AI
-0CCA..0CCB    ; SpacingMark # Mc   [2] KANNADA VOWEL SIGN O..KANNADA VOWEL SIGN OO
 0CF3          ; SpacingMark # Mc       KANNADA SIGN COMBINING ANUSVARA ABOVE RIGHT
 0D02..0D03    ; SpacingMark # Mc   [2] MALAYALAM SIGN ANUSVARA..MALAYALAM SIGN VISARGA
 0D3F..0D40    ; SpacingMark # Mc   [2] MALAYALAM VOWEL SIGN I..MALAYALAM VOWEL SIGN II
@@ -517,8 +551,6 @@ E0100..E01EF  ; Extend # Mn [240] VARIATION SELECTOR-17..VARIATION SELECTOR-256
 103B..103C    ; SpacingMark # Mc   [2] MYANMAR CONSONANT SIGN MEDIAL YA..MYANMAR CONSONANT SIGN MEDIAL RA
 1056..1057    ; SpacingMark # Mc   [2] MYANMAR VOWEL SIGN VOCALIC R..MYANMAR VOWEL SIGN VOCALIC RR
 1084          ; SpacingMark # Mc       MYANMAR VOWEL SIGN SHAN E
-1715          ; SpacingMark # Mc       TAGALOG SIGN PAMUDPOD
-1734          ; SpacingMark # Mc       HANUNOO SIGN PAMUDPOD
 17B6          ; SpacingMark # Mc       KHMER VOWEL SIGN AA
 17BE..17C5    ; SpacingMark # Mc   [8] KHMER VOWEL SIGN OE..KHMER VOWEL SIGN AU
 17C7..17C8    ; SpacingMark # Mc   [2] KHMER SIGN REAHMUK..KHMER SIGN YUUKALEAPINTU
@@ -531,17 +563,13 @@ E0100..E01EF  ; Extend # Mn [240] VARIATION SELECTOR-17..VARIATION SELECTOR-256
 1A57          ; SpacingMark # Mc       TAI THAM CONSONANT SIGN LA TANG LAI
 1A6D..1A72    ; SpacingMark # Mc   [6] TAI THAM VOWEL SIGN OY..TAI THAM VOWEL SIGN THAM AI
 1B04          ; SpacingMark # Mc       BALINESE SIGN BISAH
-1B3B          ; SpacingMark # Mc       BALINESE VOWEL SIGN RA REPA TEDUNG
-1B3D..1B41    ; SpacingMark # Mc   [5] BALINESE VOWEL SIGN LA LENGA TEDUNG..BALINESE VOWEL SIGN TALING REPA TEDUNG
-1B43..1B44    ; SpacingMark # Mc   [2] BALINESE VOWEL SIGN PEPET TEDUNG..BALINESE ADEG ADEG
+1B3E..1B41    ; SpacingMark # Mc   [4] BALINESE VOWEL SIGN TALING..BALINESE VOWEL SIGN TALING REPA TEDUNG
 1B82          ; SpacingMark # Mc       SUNDANESE SIGN PANGWISAD
 1BA1          ; SpacingMark # Mc       SUNDANESE CONSONANT SIGN PAMINGKAL
 1BA6..1BA7    ; SpacingMark # Mc   [2] SUNDANESE VOWEL SIGN PANAELAENG..SUNDANESE VOWEL SIGN PANOLONG
-1BAA          ; SpacingMark # Mc       SUNDANESE SIGN PAMAAEH
 1BE7          ; SpacingMark # Mc       BATAK VOWEL SIGN E
 1BEA..1BEC    ; SpacingMark # Mc   [3] BATAK VOWEL SIGN I..BATAK VOWEL SIGN O
 1BEE          ; SpacingMark # Mc       BATAK VOWEL SIGN U
-1BF2..1BF3    ; SpacingMark # Mc   [2] BATAK PANGOLAT..BATAK PANONGONAN
 1C24..1C2B    ; SpacingMark # Mc   [8] LEPCHA SUBJOINED LETTER YA..LEPCHA VOWEL SIGN UU
 1C34..1C35    ; SpacingMark # Mc   [2] LEPCHA CONSONANT SIGN NYIN-DO..LEPCHA CONSONANT SIGN KANG
 1CE1          ; SpacingMark # Mc       VEDIC TONE ATHARVAVEDIC INDEPENDENT SVARITA
@@ -550,11 +578,11 @@ A823..A824    ; SpacingMark # Mc   [2] SYLOTI NAGRI VOWEL SIGN A..SYLOTI NAGRI V
 A827          ; SpacingMark # Mc       SYLOTI NAGRI VOWEL SIGN OO
 A880..A881    ; SpacingMark # Mc   [2] SAURASHTRA SIGN ANUSVARA..SAURASHTRA SIGN VISARGA
 A8B4..A8C3    ; SpacingMark # Mc  [16] SAURASHTRA CONSONANT SIGN HAARU..SAURASHTRA VOWEL SIGN AU
-A952..A953    ; SpacingMark # Mc   [2] REJANG CONSONANT SIGN H..REJANG VIRAMA
+A952          ; SpacingMark # Mc       REJANG CONSONANT SIGN H
 A983          ; SpacingMark # Mc       JAVANESE SIGN WIGNYAN
 A9B4..A9B5    ; SpacingMark # Mc   [2] JAVANESE VOWEL SIGN TARUNG..JAVANESE VOWEL SIGN TOLONG
 A9BA..A9BB    ; SpacingMark # Mc   [2] JAVANESE VOWEL SIGN TALING..JAVANESE VOWEL SIGN DIRGA MURE
-A9BE..A9C0    ; SpacingMark # Mc   [3] JAVANESE CONSONANT SIGN PENGKAL..JAVANESE PANGKON
+A9BE..A9BF    ; SpacingMark # Mc   [2] JAVANESE CONSONANT SIGN PENGKAL..JAVANESE CONSONANT SIGN CAKRA
 AA2F..AA30    ; SpacingMark # Mc   [2] CHAM VOWEL SIGN O..CHAM VOWEL SIGN AI
 AA33..AA34    ; SpacingMark # Mc   [2] CHAM CONSONANT SIGN YA..CHAM CONSONANT SIGN RA
 AA4D          ; SpacingMark # Mc       CHAM CONSONANT SIGN FINAL H
@@ -574,18 +602,20 @@ ABEC          ; SpacingMark # Mc       MEETEI MAYEK LUM IYEK
 11145..11146  ; SpacingMark # Mc   [2] CHAKMA VOWEL SIGN AA..CHAKMA VOWEL SIGN EI
 11182         ; SpacingMark # Mc       SHARADA SIGN VISARGA
 111B3..111B5  ; SpacingMark # Mc   [3] SHARADA VOWEL SIGN AA..SHARADA VOWEL SIGN II
-111BF..111C0  ; SpacingMark # Mc   [2] SHARADA VOWEL SIGN AU..SHARADA SIGN VIRAMA
+111BF         ; SpacingMark # Mc       SHARADA VOWEL SIGN AU
 111CE         ; SpacingMark # Mc       SHARADA VOWEL SIGN PRISHTHAMATRA E
 1122C..1122E  ; SpacingMark # Mc   [3] KHOJKI VOWEL SIGN AA..KHOJKI VOWEL SIGN II
 11232..11233  ; SpacingMark # Mc   [2] KHOJKI VOWEL SIGN O..KHOJKI VOWEL SIGN AU
-11235         ; SpacingMark # Mc       KHOJKI SIGN VIRAMA
 112E0..112E2  ; SpacingMark # Mc   [3] KHUDAWADI VOWEL SIGN AA..KHUDAWADI VOWEL SIGN II
 11302..11303  ; SpacingMark # Mc   [2] GRANTHA SIGN ANUSVARA..GRANTHA SIGN VISARGA
 1133F         ; SpacingMark # Mc       GRANTHA VOWEL SIGN I
 11341..11344  ; SpacingMark # Mc   [4] GRANTHA VOWEL SIGN U..GRANTHA VOWEL SIGN VOCALIC RR
 11347..11348  ; SpacingMark # Mc   [2] GRANTHA VOWEL SIGN EE..GRANTHA VOWEL SIGN AI
-1134B..1134D  ; SpacingMark # Mc   [3] GRANTHA VOWEL SIGN OO..GRANTHA SIGN VIRAMA
+1134B..1134C  ; SpacingMark # Mc   [2] GRANTHA VOWEL SIGN OO..GRANTHA VOWEL SIGN AU
 11362..11363  ; SpacingMark # Mc   [2] GRANTHA VOWEL SIGN VOCALIC L..GRANTHA VOWEL SIGN VOCALIC LL
+113B9..113BA  ; SpacingMark # Mc   [2] TULU-TIGALARI VOWEL SIGN I..TULU-TIGALARI VOWEL SIGN II
+113CA         ; SpacingMark # Mc       TULU-TIGALARI SIGN CANDRA ANUNASIKA
+113CC..113CD  ; SpacingMark # Mc   [2] TULU-TIGALARI SIGN ANUSVARA..TULU-TIGALARI SIGN VISARGA
 11435..11437  ; SpacingMark # Mc   [3] NEWA VOWEL SIGN AA..NEWA VOWEL SIGN II
 11440..11441  ; SpacingMark # Mc   [2] NEWA VOWEL SIGN O..NEWA VOWEL SIGN AU
 11445         ; SpacingMark # Mc       NEWA SIGN VISARGA
@@ -602,13 +632,12 @@ ABEC          ; SpacingMark # Mc       MEETEI MAYEK LUM IYEK
 1163E         ; SpacingMark # Mc       MODI SIGN VISARGA
 116AC         ; SpacingMark # Mc       TAKRI SIGN VISARGA
 116AE..116AF  ; SpacingMark # Mc   [2] TAKRI VOWEL SIGN I..TAKRI VOWEL SIGN II
-116B6         ; SpacingMark # Mc       TAKRI SIGN VIRAMA
+1171E         ; SpacingMark # Mc       AHOM CONSONANT SIGN MEDIAL RA
 11726         ; SpacingMark # Mc       AHOM VOWEL SIGN E
 1182C..1182E  ; SpacingMark # Mc   [3] DOGRA VOWEL SIGN AA..DOGRA VOWEL SIGN II
 11838         ; SpacingMark # Mc       DOGRA SIGN VISARGA
 11931..11935  ; SpacingMark # Mc   [5] DIVES AKURU VOWEL SIGN I..DIVES AKURU VOWEL SIGN E
 11937..11938  ; SpacingMark # Mc   [2] DIVES AKURU VOWEL SIGN AI..DIVES AKURU VOWEL SIGN O
-1193D         ; SpacingMark # Mc       DIVES AKURU SIGN HALANTA
 11940         ; SpacingMark # Mc       DIVES AKURU MEDIAL YA
 11942         ; SpacingMark # Mc       DIVES AKURU MEDIAL RA
 119D1..119D3  ; SpacingMark # Mc   [3] NANDINAGARI VOWEL SIGN AA..NANDINAGARI VOWEL SIGN II
@@ -629,13 +658,10 @@ ABEC          ; SpacingMark # Mc       MEETEI MAYEK LUM IYEK
 11F03         ; SpacingMark # Mc       KAWI SIGN VISARGA
 11F34..11F35  ; SpacingMark # Mc   [2] KAWI VOWEL SIGN AA..KAWI VOWEL SIGN ALTERNATE AA
 11F3E..11F3F  ; SpacingMark # Mc   [2] KAWI VOWEL SIGN E..KAWI VOWEL SIGN AI
-11F41         ; SpacingMark # Mc       KAWI SIGN KILLER
+1612A..1612C  ; SpacingMark # Mc   [3] GURUNG KHEMA CONSONANT SIGN MEDIAL YA..GURUNG KHEMA CONSONANT SIGN MEDIAL HA
 16F51..16F87  ; SpacingMark # Mc  [55] MIAO SIGN ASPIRATION..MIAO VOWEL SIGN UI
-16FF0..16FF1  ; SpacingMark # Mc   [2] VIETNAMESE ALTERNATE READING MARK CA..VIETNAMESE ALTERNATE READING MARK NHAY
-1D166         ; SpacingMark # Mc       MUSICAL SYMBOL COMBINING SPRECHGESANG STEM
-1D16D         ; SpacingMark # Mc       MUSICAL SYMBOL COMBINING AUGMENTATION DOT
 
-# Total code points: 395
+# Total code points: 378
 
 # ================================================
 
@@ -648,8 +674,10 @@ A960..A97C    ; L # Lo  [29] HANGUL CHOSEONG TIKEUT-MIEUM..HANGUL CHOSEONG SSANG
 
 1160..11A7    ; V # Lo  [72] HANGUL JUNGSEONG FILLER..HANGUL JUNGSEONG O-YAE
 D7B0..D7C6    ; V # Lo  [23] HANGUL JUNGSEONG O-YEO..HANGUL JUNGSEONG ARAEA-E
+16D63         ; V # Lo       KIRAT RAI VOWEL SIGN AA
+16D67..16D6A  ; V # Lo   [4] KIRAT RAI VOWEL SIGN E..KIRAT RAI VOWEL SIGN AU
 
-# Total code points: 95
+# Total code points: 100
 
 # ================================================
 
diff --git a/libcxx/utils/data/unicode/GraphemeBreakTest.txt b/libcxx/utils/data/unicode/GraphemeBreakTest.txt
index 4c1ed512e4510..d10c174b6896d 100644
--- a/libcxx/utils/data/unicode/GraphemeBreakTest.txt
+++ b/libcxx/utils/data/unicode/GraphemeBreakTest.txt
@@ -1,8 +1,8 @@
-# GraphemeBreakTest-15.1.0.txt
-# Date: 2023-08-07, 15:52:55 GMT
-# © 2023 Unicode®, Inc.
+# GraphemeBreakTest-16.0.0.txt
+# Date: 2024-05-02, 15:02:48 GMT
+# © 2024 Unicode®, Inc.
 # Unicode and the Unicode Logo are registered trademarks of Unicode, Inc. in the U.S. and other countries.
-# For terms of use, see https://www.unicode.org/terms_of_use.html
+# For terms of use and license, see https://www.unicode.org/terms_of_use.html
 #
 # Unicode Character Database
 #   For documentation, see https://www.unicode.org/reports/tr44/
@@ -30,8 +30,8 @@
 ÷ 0020 × 0308 ÷ 000A ÷	#  ÷ [0.2] SPACE (Other) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] <LINE FEED (LF)> (LF) ÷ [0.3]
 ÷ 0020 ÷ 0001 ÷	#  ÷ [0.2] SPACE (Other) ÷ [5.0] <START OF HEADING> (Control) ÷ [0.3]
 ÷ 0020 × 0308 ÷ 0001 ÷	#  ÷ [0.2] SPACE (Other) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] <START OF HEADING> (Control) ÷ [0.3]
-÷ 0020 × 034F ÷	#  ÷ [0.2] SPACE (Other) × [9.0] COMBINING GRAPHEME JOINER (Extend) ÷ [0.3]
-÷ 0020 × 0308 × 034F ÷	#  ÷ [0.2] SPACE (Other) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] COMBINING GRAPHEME JOINER (Extend) ÷ [0.3]
+÷ 0020 × 200C ÷	#  ÷ [0.2] SPACE (Other) × [9.0] ZERO WIDTH NON-JOINER (Extend) ÷ [0.3]
+÷ 0020 × 0308 × 200C ÷	#  ÷ [0.2] SPACE (Other) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] ZERO WIDTH NON-JOINER (Extend) ÷ [0.3]
 ÷ 0020 ÷ 1F1E6 ÷	#  ÷ [0.2] SPACE (Other) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [0.3]
 ÷ 0020 × 0308 ÷ 1F1E6 ÷	#  ÷ [0.2] SPACE (Other) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [0.3]
 ÷ 0020 ÷ 0600 ÷	#  ÷ [0.2] SPACE (Other) ÷ [999.0] ARABIC NUMBER SIGN (Prepend) ÷ [0.3]
@@ -48,8 +48,6 @@
 ÷ 0020 × 0308 ÷ AC00 ÷	#  ÷ [0.2] SPACE (Other) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL SYLLABLE GA (LV) ÷ [0.3]
 ÷ 0020 ÷ AC01 ÷	#  ÷ [0.2] SPACE (Other) ÷ [999.0] HANGUL SYLLABLE GAG (LVT) ÷ [0.3]
 ÷ 0020 × 0308 ÷ AC01 ÷	#  ÷ [0.2] SPACE (Other) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL SYLLABLE GAG (LVT) ÷ [0.3]
-÷ 0020 × 0900 ÷	#  ÷ [0.2] SPACE (Other) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) ÷ [0.3]
-÷ 0020 × 0308 × 0900 ÷	#  ÷ [0.2] SPACE (Other) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) ÷ [0.3]
 ÷ 0020 × 0903 ÷	#  ÷ [0.2] SPACE (Other) × [9.1] DEVANAGARI SIGN VISARGA (SpacingMark_ConjunctLinkingScripts) ÷ [0.3]
 ÷ 0020 × 0308 × 0903 ÷	#  ÷ [0.2] SPACE (Other) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.1] DEVANAGARI SIGN VISARGA (SpacingMark_ConjunctLinkingScripts) ÷ [0.3]
 ÷ 0020 ÷ 0904 ÷	#  ÷ [0.2] SPACE (Other) ÷ [999.0] DEVANAGARI LETTER SHORT A (ConjunctLinkingScripts) ÷ [0.3]
@@ -62,8 +60,8 @@
 ÷ 0020 × 0308 ÷ 231A ÷	#  ÷ [0.2] SPACE (Other) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] WATCH (ExtPict) ÷ [0.3]
 ÷ 0020 × 0300 ÷	#  ÷ [0.2] SPACE (Other) × [9.0] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) ÷ [0.3]
 ÷ 0020 × 0308 × 0300 ÷	#  ÷ [0.2] SPACE (Other) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) ÷ [0.3]
-÷ 0020 × 093C ÷	#  ÷ [0.2] SPACE (Other) × [9.0] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3]
-÷ 0020 × 0308 × 093C ÷	#  ÷ [0.2] SPACE (Other) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3]
+÷ 0020 × 0900 ÷	#  ÷ [0.2] SPACE (Other) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3]
+÷ 0020 × 0308 × 0900 ÷	#  ÷ [0.2] SPACE (Other) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3]
 ÷ 0020 × 094D ÷	#  ÷ [0.2] SPACE (Other) × [9.0] DEVANAGARI SIGN VIRAMA (Extend_ConjunctLinkingScripts_ConjunctLinker_ExtCccZwj) ÷ [0.3]
 ÷ 0020 × 0308 × 094D ÷	#  ÷ [0.2] SPACE (Other) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN VIRAMA (Extend_ConjunctLinkingScripts_ConjunctLinker_ExtCccZwj) ÷ [0.3]
 ÷ 0020 × 200D ÷	#  ÷ [0.2] SPACE (Other) × [9.0] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) ÷ [0.3]
@@ -78,8 +76,8 @@
 ÷ 000D ÷ 0308 ÷ 000A ÷	#  ÷ [0.2] <CARRIAGE RETURN (CR)> (CR) ÷ [4.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] <LINE FEED (LF)> (LF) ÷ [0.3]
 ÷ 000D ÷ 0001 ÷	#  ÷ [0.2] <CARRIAGE RETURN (CR)> (CR) ÷ [4.0] <START OF HEADING> (Control) ÷ [0.3]
 ÷ 000D ÷ 0308 ÷ 0001 ÷	#  ÷ [0.2] <CARRIAGE RETURN (CR)> (CR) ÷ [4.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] <START OF HEADING> (Control) ÷ [0.3]
-÷ 000D ÷ 034F ÷	#  ÷ [0.2] <CARRIAGE RETURN (CR)> (CR) ÷ [4.0] COMBINING GRAPHEME JOINER (Extend) ÷ [0.3]
-÷ 000D ÷ 0308 × 034F ÷	#  ÷ [0.2] <CARRIAGE RETURN (CR)> (CR) ÷ [4.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] COMBINING GRAPHEME JOINER (Extend) ÷ [0.3]
+÷ 000D ÷ 200C ÷	#  ÷ [0.2] <CARRIAGE RETURN (CR)> (CR) ÷ [4.0] ZERO WIDTH NON-JOINER (Extend) ÷ [0.3]
+÷ 000D ÷ 0308 × 200C ÷	#  ÷ [0.2] <CARRIAGE RETURN (CR)> (CR) ÷ [4.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] ZERO WIDTH NON-JOINER (Extend) ÷ [0.3]
 ÷ 000D ÷ 1F1E6 ÷	#  ÷ [0.2] <CARRIAGE RETURN (CR)> (CR) ÷ [4.0] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [0.3]
 ÷ 000D ÷ 0308 ÷ 1F1E6 ÷	#  ÷ [0.2] <CARRIAGE RETURN (CR)> (CR) ÷ [4.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [0.3]
 ÷ 000D ÷ 0600 ÷	#  ÷ [0.2] <CARRIAGE RETURN (CR)> (CR) ÷ [4.0] ARABIC NUMBER SIGN (Prepend) ÷ [0.3]
@@ -96,8 +94,6 @@
 ÷ 000D ÷ 0308 ÷ AC00 ÷	#  ÷ [0.2] <CARRIAGE RETURN (CR)> (CR) ÷ [4.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL SYLLABLE GA (LV) ÷ [0.3]
 ÷ 000D ÷ AC01 ÷	#  ÷ [0.2] <CARRIAGE RETURN (CR)> (CR) ÷ [4.0] HANGUL SYLLABLE GAG (LVT) ÷ [0.3]
 ÷ 000D ÷ 0308 ÷ AC01 ÷	#  ÷ [0.2] <CARRIAGE RETURN (CR)> (CR) ÷ [4.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL SYLLABLE GAG (LVT) ÷ [0.3]
-÷ 000D ÷ 0900 ÷	#  ÷ [0.2] <CARRIAGE RETURN (CR)> (CR) ÷ [4.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) ÷ [0.3]
-÷ 000D ÷ 0308 × 0900 ÷	#  ÷ [0.2] <CARRIAGE RETURN (CR)> (CR) ÷ [4.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) ÷ [0.3]
 ÷ 000D ÷ 0903 ÷	#  ÷ [0.2] <CARRIAGE RETURN (CR)> (CR) ÷ [4.0] DEVANAGARI SIGN VISARGA (SpacingMark_ConjunctLinkingScripts) ÷ [0.3]
 ÷ 000D ÷ 0308 × 0903 ÷	#  ÷ [0.2] <CARRIAGE RETURN (CR)> (CR) ÷ [4.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.1] DEVANAGARI SIGN VISARGA (SpacingMark_ConjunctLinkingScripts) ÷ [0.3]
 ÷ 000D ÷ 0904 ÷	#  ÷ [0.2] <CARRIAGE RETURN (CR)> (CR) ÷ [4.0] DEVANAGARI LETTER SHORT A (ConjunctLinkingScripts) ÷ [0.3]
@@ -110,8 +106,8 @@
 ÷ 000D ÷ 0308 ÷ 231A ÷	#  ÷ [0.2] <CARRIAGE RETURN (CR)> (CR) ÷ [4.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] WATCH (ExtPict) ÷ [0.3]
 ÷ 000D ÷ 0300 ÷	#  ÷ [0.2] <CARRIAGE RETURN (CR)> (CR) ÷ [4.0] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) ÷ [0.3]
 ÷ 000D ÷ 0308 × 0300 ÷	#  ÷ [0.2] <CARRIAGE RETURN (CR)> (CR) ÷ [4.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) ÷ [0.3]
-÷ 000D ÷ 093C ÷	#  ÷ [0.2] <CARRIAGE RETURN (CR)> (CR) ÷ [4.0] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3]
-÷ 000D ÷ 0308 × 093C ÷	#  ÷ [0.2] <CARRIAGE RETURN (CR)> (CR) ÷ [4.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3]
+÷ 000D ÷ 0900 ÷	#  ÷ [0.2] <CARRIAGE RETURN (CR)> (CR) ÷ [4.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3]
+÷ 000D ÷ 0308 × 0900 ÷	#  ÷ [0.2] <CARRIAGE RETURN (CR)> (CR) ÷ [4.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3]
 ÷ 000D ÷ 094D ÷	#  ÷ [0.2] <CARRIAGE RETURN (CR)> (CR) ÷ [4.0] DEVANAGARI SIGN VIRAMA (Extend_ConjunctLinkingScripts_ConjunctLinker_ExtCccZwj) ÷ [0.3]
 ÷ 000D ÷ 0308 × 094D ÷	#  ÷ [0.2] <CARRIAGE RETURN (CR)> (CR) ÷ [4.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN VIRAMA (Extend_ConjunctLinkingScripts_ConjunctLinker_ExtCccZwj) ÷ [0.3]
 ÷ 000D ÷ 200D ÷	#  ÷ [0.2] <CARRIAGE RETURN (CR)> (CR) ÷ [4.0] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) ÷ [0.3]
@@ -126,8 +122,8 @@
 ÷ 000A ÷ 0308 ÷ 000A ÷	#  ÷ [0.2] <LINE FEED (LF)> (LF) ÷ [4.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] <LINE FEED (LF)> (LF) ÷ [0.3]
 ÷ 000A ÷ 0001 ÷	#  ÷ [0.2] <LINE FEED (LF)> (LF) ÷ [4.0] <START OF HEADING> (Control) ÷ [0.3]
 ÷ 000A ÷ 0308 ÷ 0001 ÷	#  ÷ [0.2] <LINE FEED (LF)> (LF) ÷ [4.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] <START OF HEADING> (Control) ÷ [0.3]
-÷ 000A ÷ 034F ÷	#  ÷ [0.2] <LINE FEED (LF)> (LF) ÷ [4.0] COMBINING GRAPHEME JOINER (Extend) ÷ [0.3]
-÷ 000A ÷ 0308 × 034F ÷	#  ÷ [0.2] <LINE FEED (LF)> (LF) ÷ [4.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] COMBINING GRAPHEME JOINER (Extend) ÷ [0.3]
+÷ 000A ÷ 200C ÷	#  ÷ [0.2] <LINE FEED (LF)> (LF) ÷ [4.0] ZERO WIDTH NON-JOINER (Extend) ÷ [0.3]
+÷ 000A ÷ 0308 × 200C ÷	#  ÷ [0.2] <LINE FEED (LF)> (LF) ÷ [4.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] ZERO WIDTH NON-JOINER (Extend) ÷ [0.3]
 ÷ 000A ÷ 1F1E6 ÷	#  ÷ [0.2] <LINE FEED (LF)> (LF) ÷ [4.0] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [0.3]
 ÷ 000A ÷ 0308 ÷ 1F1E6 ÷	#  ÷ [0.2] <LINE FEED (LF)> (LF) ÷ [4.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [0.3]
 ÷ 000A ÷ 0600 ÷	#  ÷ [0.2] <LINE FEED (LF)> (LF) ÷ [4.0] ARABIC NUMBER SIGN (Prepend) ÷ [0.3]
@@ -144,8 +140,6 @@
 ÷ 000A ÷ 0308 ÷ AC00 ÷	#  ÷ [0.2] <LINE FEED (LF)> (LF) ÷ [4.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL SYLLABLE GA (LV) ÷ [0.3]
 ÷ 000A ÷ AC01 ÷	#  ÷ [0.2] <LINE FEED (LF)> (LF) ÷ [4.0] HANGUL SYLLABLE GAG (LVT) ÷ [0.3]
 ÷ 000A ÷ 0308 ÷ AC01 ÷	#  ÷ [0.2] <LINE FEED (LF)> (LF) ÷ [4.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL SYLLABLE GAG (LVT) ÷ [0.3]
-÷ 000A ÷ 0900 ÷	#  ÷ [0.2] <LINE FEED (LF)> (LF) ÷ [4.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) ÷ [0.3]
-÷ 000A ÷ 0308 × 0900 ÷	#  ÷ [0.2] <LINE FEED (LF)> (LF) ÷ [4.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) ÷ [0.3]
 ÷ 000A ÷ 0903 ÷	#  ÷ [0.2] <LINE FEED (LF)> (LF) ÷ [4.0] DEVANAGARI SIGN VISARGA (SpacingMark_ConjunctLinkingScripts) ÷ [0.3]
 ÷ 000A ÷ 0308 × 0903 ÷	#  ÷ [0.2] <LINE FEED (LF)> (LF) ÷ [4.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.1] DEVANAGARI SIGN VISARGA (SpacingMark_ConjunctLinkingScripts) ÷ [0.3]
 ÷ 000A ÷ 0904 ÷	#  ÷ [0.2] <LINE FEED (LF)> (LF) ÷ [4.0] DEVANAGARI LETTER SHORT A (ConjunctLinkingScripts) ÷ [0.3]
@@ -158,8 +152,8 @@
 ÷ 000A ÷ 0308 ÷ 231A ÷	#  ÷ [0.2] <LINE FEED (LF)> (LF) ÷ [4.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] WATCH (ExtPict) ÷ [0.3]
 ÷ 000A ÷ 0300 ÷	#  ÷ [0.2] <LINE FEED (LF)> (LF) ÷ [4.0] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) ÷ [0.3]
 ÷ 000A ÷ 0308 × 0300 ÷	#  ÷ [0.2] <LINE FEED (LF)> (LF) ÷ [4.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) ÷ [0.3]
-÷ 000A ÷ 093C ÷	#  ÷ [0.2] <LINE FEED (LF)> (LF) ÷ [4.0] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3]
-÷ 000A ÷ 0308 × 093C ÷	#  ÷ [0.2] <LINE FEED (LF)> (LF) ÷ [4.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3]
+÷ 000A ÷ 0900 ÷	#  ÷ [0.2] <LINE FEED (LF)> (LF) ÷ [4.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3]
+÷ 000A ÷ 0308 × 0900 ÷	#  ÷ [0.2] <LINE FEED (LF)> (LF) ÷ [4.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3]
 ÷ 000A ÷ 094D ÷	#  ÷ [0.2] <LINE FEED (LF)> (LF) ÷ [4.0] DEVANAGARI SIGN VIRAMA (Extend_ConjunctLinkingScripts_ConjunctLinker_ExtCccZwj) ÷ [0.3]
 ÷ 000A ÷ 0308 × 094D ÷	#  ÷ [0.2] <LINE FEED (LF)> (LF) ÷ [4.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN VIRAMA (Extend_ConjunctLinkingScripts_ConjunctLinker_ExtCccZwj) ÷ [0.3]
 ÷ 000A ÷ 200D ÷	#  ÷ [0.2] <LINE FEED (LF)> (LF) ÷ [4.0] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) ÷ [0.3]
@@ -174,8 +168,8 @@
 ÷ 0001 ÷ 0308 ÷ 000A ÷	#  ÷ [0.2] <START OF HEADING> (Control) ÷ [4.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] <LINE FEED (LF)> (LF) ÷ [0.3]
 ÷ 0001 ÷ 0001 ÷	#  ÷ [0.2] <START OF HEADING> (Control) ÷ [4.0] <START OF HEADING> (Control) ÷ [0.3]
 ÷ 0001 ÷ 0308 ÷ 0001 ÷	#  ÷ [0.2] <START OF HEADING> (Control) ÷ [4.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] <START OF HEADING> (Control) ÷ [0.3]
-÷ 0001 ÷ 034F ÷	#  ÷ [0.2] <START OF HEADING> (Control) ÷ [4.0] COMBINING GRAPHEME JOINER (Extend) ÷ [0.3]
-÷ 0001 ÷ 0308 × 034F ÷	#  ÷ [0.2] <START OF HEADING> (Control) ÷ [4.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] COMBINING GRAPHEME JOINER (Extend) ÷ [0.3]
+÷ 0001 ÷ 200C ÷	#  ÷ [0.2] <START OF HEADING> (Control) ÷ [4.0] ZERO WIDTH NON-JOINER (Extend) ÷ [0.3]
+÷ 0001 ÷ 0308 × 200C ÷	#  ÷ [0.2] <START OF HEADING> (Control) ÷ [4.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] ZERO WIDTH NON-JOINER (Extend) ÷ [0.3]
 ÷ 0001 ÷ 1F1E6 ÷	#  ÷ [0.2] <START OF HEADING> (Control) ÷ [4.0] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [0.3]
 ÷ 0001 ÷ 0308 ÷ 1F1E6 ÷	#  ÷ [0.2] <START OF HEADING> (Control) ÷ [4.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [0.3]
 ÷ 0001 ÷ 0600 ÷	#  ÷ [0.2] <START OF HEADING> (Control) ÷ [4.0] ARABIC NUMBER SIGN (Prepend) ÷ [0.3]
@@ -192,8 +186,6 @@
 ÷ 0001 ÷ 0308 ÷ AC00 ÷	#  ÷ [0.2] <START OF HEADING> (Control) ÷ [4.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL SYLLABLE GA (LV) ÷ [0.3]
 ÷ 0001 ÷ AC01 ÷	#  ÷ [0.2] <START OF HEADING> (Control) ÷ [4.0] HANGUL SYLLABLE GAG (LVT) ÷ [0.3]
 ÷ 0001 ÷ 0308 ÷ AC01 ÷	#  ÷ [0.2] <START OF HEADING> (Control) ÷ [4.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL SYLLABLE GAG (LVT) ÷ [0.3]
-÷ 0001 ÷ 0900 ÷	#  ÷ [0.2] <START OF HEADING> (Control) ÷ [4.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) ÷ [0.3]
-÷ 0001 ÷ 0308 × 0900 ÷	#  ÷ [0.2] <START OF HEADING> (Control) ÷ [4.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) ÷ [0.3]
 ÷ 0001 ÷ 0903 ÷	#  ÷ [0.2] <START OF HEADING> (Control) ÷ [4.0] DEVANAGARI SIGN VISARGA (SpacingMark_ConjunctLinkingScripts) ÷ [0.3]
 ÷ 0001 ÷ 0308 × 0903 ÷	#  ÷ [0.2] <START OF HEADING> (Control) ÷ [4.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.1] DEVANAGARI SIGN VISARGA (SpacingMark_ConjunctLinkingScripts) ÷ [0.3]
 ÷ 0001 ÷ 0904 ÷	#  ÷ [0.2] <START OF HEADING> (Control) ÷ [4.0] DEVANAGARI LETTER SHORT A (ConjunctLinkingScripts) ÷ [0.3]
@@ -206,62 +198,60 @@
 ÷ 0001 ÷ 0308 ÷ 231A ÷	#  ÷ [0.2] <START OF HEADING> (Control) ÷ [4.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] WATCH (ExtPict) ÷ [0.3]
 ÷ 0001 ÷ 0300 ÷	#  ÷ [0.2] <START OF HEADING> (Control) ÷ [4.0] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) ÷ [0.3]
 ÷ 0001 ÷ 0308 × 0300 ÷	#  ÷ [0.2] <START OF HEADING> (Control) ÷ [4.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) ÷ [0.3]
-÷ 0001 ÷ 093C ÷	#  ÷ [0.2] <START OF HEADING> (Control) ÷ [4.0] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3]
-÷ 0001 ÷ 0308 × 093C ÷	#  ÷ [0.2] <START OF HEADING> (Control) ÷ [4.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3]
+÷ 0001 ÷ 0900 ÷	#  ÷ [0.2] <START OF HEADING> (Control) ÷ [4.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3]
+÷ 0001 ÷ 0308 × 0900 ÷	#  ÷ [0.2] <START OF HEADING> (Control) ÷ [4.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3]
 ÷ 0001 ÷ 094D ÷	#  ÷ [0.2] <START OF HEADING> (Control) ÷ [4.0] DEVANAGARI SIGN VIRAMA (Extend_ConjunctLinkingScripts_ConjunctLinker_ExtCccZwj) ÷ [0.3]
 ÷ 0001 ÷ 0308 × 094D ÷	#  ÷ [0.2] <START OF HEADING> (Control) ÷ [4.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN VIRAMA (Extend_ConjunctLinkingScripts_ConjunctLinker_ExtCccZwj) ÷ [0.3]
 ÷ 0001 ÷ 200D ÷	#  ÷ [0.2] <START OF HEADING> (Control) ÷ [4.0] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) ÷ [0.3]
 ÷ 0001 ÷ 0308 × 200D ÷	#  ÷ [0.2] <START OF HEADING> (Control) ÷ [4.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) ÷ [0.3]
 ÷ 0001 ÷ 0378 ÷	#  ÷ [0.2] <START OF HEADING> (Control) ÷ [4.0] <reserved-0378> (Other) ÷ [0.3]
 ÷ 0001 ÷ 0308 ÷ 0378 ÷	#  ÷ [0.2] <START OF HEADING> (Control) ÷ [4.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] <reserved-0378> (Other) ÷ [0.3]
-÷ 034F ÷ 0020 ÷	#  ÷ [0.2] COMBINING GRAPHEME JOINER (Extend) ÷ [999.0] SPACE (Other) ÷ [0.3]
-÷ 034F × 0308 ÷ 0020 ÷	#  ÷ [0.2] COMBINING GRAPHEME JOINER (Extend) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] SPACE (Other) ÷ [0.3]
-÷ 034F ÷ 000D ÷	#  ÷ [0.2] COMBINING GRAPHEME JOINER (Extend) ÷ [5.0] <CARRIAGE RETURN (CR)> (CR) ÷ [0.3]
-÷ 034F × 0308 ÷ 000D ÷	#  ÷ [0.2] COMBINING GRAPHEME JOINER (Extend) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] <CARRIAGE RETURN (CR)> (CR) ÷ [0.3]
-÷ 034F ÷ 000A ÷	#  ÷ [0.2] COMBINING GRAPHEME JOINER (Extend) ÷ [5.0] <LINE FEED (LF)> (LF) ÷ [0.3]
-÷ 034F × 0308 ÷ 000A ÷	#  ÷ [0.2] COMBINING GRAPHEME JOINER (Extend) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] <LINE FEED (LF)> (LF) ÷ [0.3]
-÷ 034F ÷ 0001 ÷	#  ÷ [0.2] COMBINING GRAPHEME JOINER (Extend) ÷ [5.0] <START OF HEADING> (Control) ÷ [0.3]
-÷ 034F × 0308 ÷ 0001 ÷	#  ÷ [0.2] COMBINING GRAPHEME JOINER (Extend) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] <START OF HEADING> (Control) ÷ [0.3]
-÷ 034F × 034F ÷	#  ÷ [0.2] COMBINING GRAPHEME JOINER (Extend) × [9.0] COMBINING GRAPHEME JOINER (Extend) ÷ [0.3]
-÷ 034F × 0308 × 034F ÷	#  ÷ [0.2] COMBINING GRAPHEME JOINER (Extend) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] COMBINING GRAPHEME JOINER (Extend) ÷ [0.3]
-÷ 034F ÷ 1F1E6 ÷	#  ÷ [0.2] COMBINING GRAPHEME JOINER (Extend) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [0.3]
-÷ 034F × 0308 ÷ 1F1E6 ÷	#  ÷ [0.2] COMBINING GRAPHEME JOINER (Extend) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [0.3]
-÷ 034F ÷ 0600 ÷	#  ÷ [0.2] COMBINING GRAPHEME JOINER (Extend) ÷ [999.0] ARABIC NUMBER SIGN (Prepend) ÷ [0.3]
-÷ 034F × 0308 ÷ 0600 ÷	#  ÷ [0.2] COMBINING GRAPHEME JOINER (Extend) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] ARABIC NUMBER SIGN (Prepend) ÷ [0.3]
-÷ 034F × 0A03 ÷	#  ÷ [0.2] COMBINING GRAPHEME JOINER (Extend) × [9.1] GURMUKHI SIGN VISARGA (SpacingMark) ÷ [0.3]
-÷ 034F × 0308 × 0A03 ÷	#  ÷ [0.2] COMBINING GRAPHEME JOINER (Extend) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.1] GURMUKHI SIGN VISARGA (SpacingMark) ÷ [0.3]
-÷ 034F ÷ 1100 ÷	#  ÷ [0.2] COMBINING GRAPHEME JOINER (Extend) ÷ [999.0] HANGUL CHOSEONG KIYEOK (L) ÷ [0.3]
-÷ 034F × 0308 ÷ 1100 ÷	#  ÷ [0.2] COMBINING GRAPHEME JOINER (Extend) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL CHOSEONG KIYEOK (L) ÷ [0.3]
-÷ 034F ÷ 1160 ÷	#  ÷ [0.2] COMBINING GRAPHEME JOINER (Extend) ÷ [999.0] HANGUL JUNGSEONG FILLER (V) ÷ [0.3]
-÷ 034F × 0308 ÷ 1160 ÷	#  ÷ [0.2] COMBINING GRAPHEME JOINER (Extend) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL JUNGSEONG FILLER (V) ÷ [0.3]
-÷ 034F ÷ 11A8 ÷	#  ÷ [0.2] COMBINING GRAPHEME JOINER (Extend) ÷ [999.0] HANGUL JONGSEONG KIYEOK (T) ÷ [0.3]
-÷ 034F × 0308 ÷ 11A8 ÷	#  ÷ [0.2] COMBINING GRAPHEME JOINER (Extend) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL JONGSEONG KIYEOK (T) ÷ [0.3]
-÷ 034F ÷ AC00 ÷	#  ÷ [0.2] COMBINING GRAPHEME JOINER (Extend) ÷ [999.0] HANGUL SYLLABLE GA (LV) ÷ [0.3]
-÷ 034F × 0308 ÷ AC00 ÷	#  ÷ [0.2] COMBINING GRAPHEME JOINER (Extend) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL SYLLABLE GA (LV) ÷ [0.3]
-÷ 034F ÷ AC01 ÷	#  ÷ [0.2] COMBINING GRAPHEME JOINER (Extend) ÷ [999.0] HANGUL SYLLABLE GAG (LVT) ÷ [0.3]
-÷ 034F × 0308 ÷ AC01 ÷	#  ÷ [0.2] COMBINING GRAPHEME JOINER (Extend) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL SYLLABLE GAG (LVT) ÷ [0.3]
-÷ 034F × 0900 ÷	#  ÷ [0.2] COMBINING GRAPHEME JOINER (Extend) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) ÷ [0.3]
-÷ 034F × 0308 × 0900 ÷	#  ÷ [0.2] COMBINING GRAPHEME JOINER (Extend) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) ÷ [0.3]
-÷ 034F × 0903 ÷	#  ÷ [0.2] COMBINING GRAPHEME JOINER (Extend) × [9.1] DEVANAGARI SIGN VISARGA (SpacingMark_ConjunctLinkingScripts) ÷ [0.3]
-÷ 034F × 0308 × 0903 ÷	#  ÷ [0.2] COMBINING GRAPHEME JOINER (Extend) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.1] DEVANAGARI SIGN VISARGA (SpacingMark_ConjunctLinkingScripts) ÷ [0.3]
-÷ 034F ÷ 0904 ÷	#  ÷ [0.2] COMBINING GRAPHEME JOINER (Extend) ÷ [999.0] DEVANAGARI LETTER SHORT A (ConjunctLinkingScripts) ÷ [0.3]
-÷ 034F × 0308 ÷ 0904 ÷	#  ÷ [0.2] COMBINING GRAPHEME JOINER (Extend) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] DEVANAGARI LETTER SHORT A (ConjunctLinkingScripts) ÷ [0.3]
-÷ 034F ÷ 0D4E ÷	#  ÷ [0.2] COMBINING GRAPHEME JOINER (Extend) ÷ [999.0] MALAYALAM LETTER DOT REPH (Prepend_ConjunctLinkingScripts) ÷ [0.3]
-÷ 034F × 0308 ÷ 0D4E ÷	#  ÷ [0.2] COMBINING GRAPHEME JOINER (Extend) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] MALAYALAM LETTER DOT REPH (Prepend_ConjunctLinkingScripts) ÷ [0.3]
-÷ 034F ÷ 0915 ÷	#  ÷ [0.2] COMBINING GRAPHEME JOINER (Extend) ÷ [999.0] DEVANAGARI LETTER KA (ConjunctLinkingScripts_LinkingConsonant) ÷ [0.3]
-÷ 034F × 0308 ÷ 0915 ÷	#  ÷ [0.2] COMBINING GRAPHEME JOINER (Extend) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] DEVANAGARI LETTER KA (ConjunctLinkingScripts_LinkingConsonant) ÷ [0.3]
-÷ 034F ÷ 231A ÷	#  ÷ [0.2] COMBINING GRAPHEME JOINER (Extend) ÷ [999.0] WATCH (ExtPict) ÷ [0.3]
-÷ 034F × 0308 ÷ 231A ÷	#  ÷ [0.2] COMBINING GRAPHEME JOINER (Extend) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] WATCH (ExtPict) ÷ [0.3]
-÷ 034F × 0300 ÷	#  ÷ [0.2] COMBINING GRAPHEME JOINER (Extend) × [9.0] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) ÷ [0.3]
-÷ 034F × 0308 × 0300 ÷	#  ÷ [0.2] COMBINING GRAPHEME JOINER (Extend) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) ÷ [0.3]
-÷ 034F × 093C ÷	#  ÷ [0.2] COMBINING GRAPHEME JOINER (Extend) × [9.0] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3]
-÷ 034F × 0308 × 093C ÷	#  ÷ [0.2] COMBINING GRAPHEME JOINER (Extend) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3]
-÷ 034F × 094D ÷	#  ÷ [0.2] COMBINING GRAPHEME JOINER (Extend) × [9.0] DEVANAGARI SIGN VIRAMA (Extend_ConjunctLinkingScripts_ConjunctLinker_ExtCccZwj) ÷ [0.3]
-÷ 034F × 0308 × 094D ÷	#  ÷ [0.2] COMBINING GRAPHEME JOINER (Extend) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN VIRAMA (Extend_ConjunctLinkingScripts_ConjunctLinker_ExtCccZwj) ÷ [0.3]
-÷ 034F × 200D ÷	#  ÷ [0.2] COMBINING GRAPHEME JOINER (Extend) × [9.0] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) ÷ [0.3]
-÷ 034F × 0308 × 200D ÷	#  ÷ [0.2] COMBINING GRAPHEME JOINER (Extend) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) ÷ [0.3]
-÷ 034F ÷ 0378 ÷	#  ÷ [0.2] COMBINING GRAPHEME JOINER (Extend) ÷ [999.0] <reserved-0378> (Other) ÷ [0.3]
-÷ 034F × 0308 ÷ 0378 ÷	#  ÷ [0.2] COMBINING GRAPHEME JOINER (Extend) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] <reserved-0378> (Other) ÷ [0.3]
+÷ 200C ÷ 0020 ÷	#  ÷ [0.2] ZERO WIDTH NON-JOINER (Extend) ÷ [999.0] SPACE (Other) ÷ [0.3]
+÷ 200C × 0308 ÷ 0020 ÷	#  ÷ [0.2] ZERO WIDTH NON-JOINER (Extend) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] SPACE (Other) ÷ [0.3]
+÷ 200C ÷ 000D ÷	#  ÷ [0.2] ZERO WIDTH NON-JOINER (Extend) ÷ [5.0] <CARRIAGE RETURN (CR)> (CR) ÷ [0.3]
+÷ 200C × 0308 ÷ 000D ÷	#  ÷ [0.2] ZERO WIDTH NON-JOINER (Extend) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] <CARRIAGE RETURN (CR)> (CR) ÷ [0.3]
+÷ 200C ÷ 000A ÷	#  ÷ [0.2] ZERO WIDTH NON-JOINER (Extend) ÷ [5.0] <LINE FEED (LF)> (LF) ÷ [0.3]
+÷ 200C × 0308 ÷ 000A ÷	#  ÷ [0.2] ZERO WIDTH NON-JOINER (Extend) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] <LINE FEED (LF)> (LF) ÷ [0.3]
+÷ 200C ÷ 0001 ÷	#  ÷ [0.2] ZERO WIDTH NON-JOINER (Extend) ÷ [5.0] <START OF HEADING> (Control) ÷ [0.3]
+÷ 200C × 0308 ÷ 0001 ÷	#  ÷ [0.2] ZERO WIDTH NON-JOINER (Extend) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] <START OF HEADING> (Control) ÷ [0.3]
+÷ 200C × 200C ÷	#  ÷ [0.2] ZERO WIDTH NON-JOINER (Extend) × [9.0] ZERO WIDTH NON-JOINER (Extend) ÷ [0.3]
+÷ 200C × 0308 × 200C ÷	#  ÷ [0.2] ZERO WIDTH NON-JOINER (Extend) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] ZERO WIDTH NON-JOINER (Extend) ÷ [0.3]
+÷ 200C ÷ 1F1E6 ÷	#  ÷ [0.2] ZERO WIDTH NON-JOINER (Extend) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [0.3]
+÷ 200C × 0308 ÷ 1F1E6 ÷	#  ÷ [0.2] ZERO WIDTH NON-JOINER (Extend) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [0.3]
+÷ 200C ÷ 0600 ÷	#  ÷ [0.2] ZERO WIDTH NON-JOINER (Extend) ÷ [999.0] ARABIC NUMBER SIGN (Prepend) ÷ [0.3]
+÷ 200C × 0308 ÷ 0600 ÷	#  ÷ [0.2] ZERO WIDTH NON-JOINER (Extend) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] ARABIC NUMBER SIGN (Prepend) ÷ [0.3]
+÷ 200C × 0A03 ÷	#  ÷ [0.2] ZERO WIDTH NON-JOINER (Extend) × [9.1] GURMUKHI SIGN VISARGA (SpacingMark) ÷ [0.3]
+÷ 200C × 0308 × 0A03 ÷	#  ÷ [0.2] ZERO WIDTH NON-JOINER (Extend) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.1] GURMUKHI SIGN VISARGA (SpacingMark) ÷ [0.3]
+÷ 200C ÷ 1100 ÷	#  ÷ [0.2] ZERO WIDTH NON-JOINER (Extend) ÷ [999.0] HANGUL CHOSEONG KIYEOK (L) ÷ [0.3]
+÷ 200C × 0308 ÷ 1100 ÷	#  ÷ [0.2] ZERO WIDTH NON-JOINER (Extend) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL CHOSEONG KIYEOK (L) ÷ [0.3]
+÷ 200C ÷ 1160 ÷	#  ÷ [0.2] ZERO WIDTH NON-JOINER (Extend) ÷ [999.0] HANGUL JUNGSEONG FILLER (V) ÷ [0.3]
+÷ 200C × 0308 ÷ 1160 ÷	#  ÷ [0.2] ZERO WIDTH NON-JOINER (Extend) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL JUNGSEONG FILLER (V) ÷ [0.3]
+÷ 200C ÷ 11A8 ÷	#  ÷ [0.2] ZERO WIDTH NON-JOINER (Extend) ÷ [999.0] HANGUL JONGSEONG KIYEOK (T) ÷ [0.3]
+÷ 200C × 0308 ÷ 11A8 ÷	#  ÷ [0.2] ZERO WIDTH NON-JOINER (Extend) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL JONGSEONG KIYEOK (T) ÷ [0.3]
+÷ 200C ÷ AC00 ÷	#  ÷ [0.2] ZERO WIDTH NON-JOINER (Extend) ÷ [999.0] HANGUL SYLLABLE GA (LV) ÷ [0.3]
+÷ 200C × 0308 ÷ AC00 ÷	#  ÷ [0.2] ZERO WIDTH NON-JOINER (Extend) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL SYLLABLE GA (LV) ÷ [0.3]
+÷ 200C ÷ AC01 ÷	#  ÷ [0.2] ZERO WIDTH NON-JOINER (Extend) ÷ [999.0] HANGUL SYLLABLE GAG (LVT) ÷ [0.3]
+÷ 200C × 0308 ÷ AC01 ÷	#  ÷ [0.2] ZERO WIDTH NON-JOINER (Extend) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL SYLLABLE GAG (LVT) ÷ [0.3]
+÷ 200C × 0903 ÷	#  ÷ [0.2] ZERO WIDTH NON-JOINER (Extend) × [9.1] DEVANAGARI SIGN VISARGA (SpacingMark_ConjunctLinkingScripts) ÷ [0.3]
+÷ 200C × 0308 × 0903 ÷	#  ÷ [0.2] ZERO WIDTH NON-JOINER (Extend) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.1] DEVANAGARI SIGN VISARGA (SpacingMark_ConjunctLinkingScripts) ÷ [0.3]
+÷ 200C ÷ 0904 ÷	#  ÷ [0.2] ZERO WIDTH NON-JOINER (Extend) ÷ [999.0] DEVANAGARI LETTER SHORT A (ConjunctLinkingScripts) ÷ [0.3]
+÷ 200C × 0308 ÷ 0904 ÷	#  ÷ [0.2] ZERO WIDTH NON-JOINER (Extend) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] DEVANAGARI LETTER SHORT A (ConjunctLinkingScripts) ÷ [0.3]
+÷ 200C ÷ 0D4E ÷	#  ÷ [0.2] ZERO WIDTH NON-JOINER (Extend) ÷ [999.0] MALAYALAM LETTER DOT REPH (Prepend_ConjunctLinkingScripts) ÷ [0.3]
+÷ 200C × 0308 ÷ 0D4E ÷	#  ÷ [0.2] ZERO WIDTH NON-JOINER (Extend) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] MALAYALAM LETTER DOT REPH (Prepend_ConjunctLinkingScripts) ÷ [0.3]
+÷ 200C ÷ 0915 ÷	#  ÷ [0.2] ZERO WIDTH NON-JOINER (Extend) ÷ [999.0] DEVANAGARI LETTER KA (ConjunctLinkingScripts_LinkingConsonant) ÷ [0.3]
+÷ 200C × 0308 ÷ 0915 ÷	#  ÷ [0.2] ZERO WIDTH NON-JOINER (Extend) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] DEVANAGARI LETTER KA (ConjunctLinkingScripts_LinkingConsonant) ÷ [0.3]
+÷ 200C ÷ 231A ÷	#  ÷ [0.2] ZERO WIDTH NON-JOINER (Extend) ÷ [999.0] WATCH (ExtPict) ÷ [0.3]
+÷ 200C × 0308 ÷ 231A ÷	#  ÷ [0.2] ZERO WIDTH NON-JOINER (Extend) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] WATCH (ExtPict) ÷ [0.3]
+÷ 200C × 0300 ÷	#  ÷ [0.2] ZERO WIDTH NON-JOINER (Extend) × [9.0] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) ÷ [0.3]
+÷ 200C × 0308 × 0300 ÷	#  ÷ [0.2] ZERO WIDTH NON-JOINER (Extend) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) ÷ [0.3]
+÷ 200C × 0900 ÷	#  ÷ [0.2] ZERO WIDTH NON-JOINER (Extend) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3]
+÷ 200C × 0308 × 0900 ÷	#  ÷ [0.2] ZERO WIDTH NON-JOINER (Extend) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3]
+÷ 200C × 094D ÷	#  ÷ [0.2] ZERO WIDTH NON-JOINER (Extend) × [9.0] DEVANAGARI SIGN VIRAMA (Extend_ConjunctLinkingScripts_ConjunctLinker_ExtCccZwj) ÷ [0.3]
+÷ 200C × 0308 × 094D ÷	#  ÷ [0.2] ZERO WIDTH NON-JOINER (Extend) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN VIRAMA (Extend_ConjunctLinkingScripts_ConjunctLinker_ExtCccZwj) ÷ [0.3]
+÷ 200C × 200D ÷	#  ÷ [0.2] ZERO WIDTH NON-JOINER (Extend) × [9.0] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) ÷ [0.3]
+÷ 200C × 0308 × 200D ÷	#  ÷ [0.2] ZERO WIDTH NON-JOINER (Extend) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) ÷ [0.3]
+÷ 200C ÷ 0378 ÷	#  ÷ [0.2] ZERO WIDTH NON-JOINER (Extend) ÷ [999.0] <reserved-0378> (Other) ÷ [0.3]
+÷ 200C × 0308 ÷ 0378 ÷	#  ÷ [0.2] ZERO WIDTH NON-JOINER (Extend) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] <reserved-0378> (Other) ÷ [0.3]
 ÷ 1F1E6 ÷ 0020 ÷	#  ÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [999.0] SPACE (Other) ÷ [0.3]
 ÷ 1F1E6 × 0308 ÷ 0020 ÷	#  ÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (RI) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] SPACE (Other) ÷ [0.3]
 ÷ 1F1E6 ÷ 000D ÷	#  ÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [5.0] <CARRIAGE RETURN (CR)> (CR) ÷ [0.3]
@@ -270,8 +260,8 @@
 ÷ 1F1E6 × 0308 ÷ 000A ÷	#  ÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (RI) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] <LINE FEED (LF)> (LF) ÷ [0.3]
 ÷ 1F1E6 ÷ 0001 ÷	#  ÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [5.0] <START OF HEADING> (Control) ÷ [0.3]
 ÷ 1F1E6 × 0308 ÷ 0001 ÷	#  ÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (RI) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] <START OF HEADING> (Control) ÷ [0.3]
-÷ 1F1E6 × 034F ÷	#  ÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (RI) × [9.0] COMBINING GRAPHEME JOINER (Extend) ÷ [0.3]
-÷ 1F1E6 × 0308 × 034F ÷	#  ÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (RI) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] COMBINING GRAPHEME JOINER (Extend) ÷ [0.3]
+÷ 1F1E6 × 200C ÷	#  ÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (RI) × [9.0] ZERO WIDTH NON-JOINER (Extend) ÷ [0.3]
+÷ 1F1E6 × 0308 × 200C ÷	#  ÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (RI) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] ZERO WIDTH NON-JOINER (Extend) ÷ [0.3]
 ÷ 1F1E6 × 1F1E6 ÷	#  ÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (RI) × [12.0] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [0.3]
 ÷ 1F1E6 × 0308 ÷ 1F1E6 ÷	#  ÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (RI) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [0.3]
 ÷ 1F1E6 ÷ 0600 ÷	#  ÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [999.0] ARABIC NUMBER SIGN (Prepend) ÷ [0.3]
@@ -288,8 +278,6 @@
 ÷ 1F1E6 × 0308 ÷ AC00 ÷	#  ÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (RI) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL SYLLABLE GA (LV) ÷ [0.3]
 ÷ 1F1E6 ÷ AC01 ÷	#  ÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [999.0] HANGUL SYLLABLE GAG (LVT) ÷ [0.3]
 ÷ 1F1E6 × 0308 ÷ AC01 ÷	#  ÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (RI) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL SYLLABLE GAG (LVT) ÷ [0.3]
-÷ 1F1E6 × 0900 ÷	#  ÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (RI) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) ÷ [0.3]
-÷ 1F1E6 × 0308 × 0900 ÷	#  ÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (RI) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) ÷ [0.3]
 ÷ 1F1E6 × 0903 ÷	#  ÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (RI) × [9.1] DEVANAGARI SIGN VISARGA (SpacingMark_ConjunctLinkingScripts) ÷ [0.3]
 ÷ 1F1E6 × 0308 × 0903 ÷	#  ÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (RI) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.1] DEVANAGARI SIGN VISARGA (SpacingMark_ConjunctLinkingScripts) ÷ [0.3]
 ÷ 1F1E6 ÷ 0904 ÷	#  ÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [999.0] DEVANAGARI LETTER SHORT A (ConjunctLinkingScripts) ÷ [0.3]
@@ -302,8 +290,8 @@
 ÷ 1F1E6 × 0308 ÷ 231A ÷	#  ÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (RI) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] WATCH (ExtPict) ÷ [0.3]
 ÷ 1F1E6 × 0300 ÷	#  ÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (RI) × [9.0] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) ÷ [0.3]
 ÷ 1F1E6 × 0308 × 0300 ÷	#  ÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (RI) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) ÷ [0.3]
-÷ 1F1E6 × 093C ÷	#  ÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (RI) × [9.0] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3]
-÷ 1F1E6 × 0308 × 093C ÷	#  ÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (RI) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3]
+÷ 1F1E6 × 0900 ÷	#  ÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (RI) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3]
+÷ 1F1E6 × 0308 × 0900 ÷	#  ÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (RI) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3]
 ÷ 1F1E6 × 094D ÷	#  ÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (RI) × [9.0] DEVANAGARI SIGN VIRAMA (Extend_ConjunctLinkingScripts_ConjunctLinker_ExtCccZwj) ÷ [0.3]
 ÷ 1F1E6 × 0308 × 094D ÷	#  ÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (RI) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN VIRAMA (Extend_ConjunctLinkingScripts_ConjunctLinker_ExtCccZwj) ÷ [0.3]
 ÷ 1F1E6 × 200D ÷	#  ÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (RI) × [9.0] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) ÷ [0.3]
@@ -318,8 +306,8 @@
 ÷ 0600 × 0308 ÷ 000A ÷	#  ÷ [0.2] ARABIC NUMBER SIGN (Prepend) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] <LINE FEED (LF)> (LF) ÷ [0.3]
 ÷ 0600 ÷ 0001 ÷	#  ÷ [0.2] ARABIC NUMBER SIGN (Prepend) ÷ [5.0] <START OF HEADING> (Control) ÷ [0.3]
 ÷ 0600 × 0308 ÷ 0001 ÷	#  ÷ [0.2] ARABIC NUMBER SIGN (Prepend) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] <START OF HEADING> (Control) ÷ [0.3]
-÷ 0600 × 034F ÷	#  ÷ [0.2] ARABIC NUMBER SIGN (Prepend) × [9.0] COMBINING GRAPHEME JOINER (Extend) ÷ [0.3]
-÷ 0600 × 0308 × 034F ÷	#  ÷ [0.2] ARABIC NUMBER SIGN (Prepend) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] COMBINING GRAPHEME JOINER (Extend) ÷ [0.3]
+÷ 0600 × 200C ÷	#  ÷ [0.2] ARABIC NUMBER SIGN (Prepend) × [9.0] ZERO WIDTH NON-JOINER (Extend) ÷ [0.3]
+÷ 0600 × 0308 × 200C ÷	#  ÷ [0.2] ARABIC NUMBER SIGN (Prepend) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] ZERO WIDTH NON-JOINER (Extend) ÷ [0.3]
 ÷ 0600 × 1F1E6 ÷	#  ÷ [0.2] ARABIC NUMBER SIGN (Prepend) × [9.2] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [0.3]
 ÷ 0600 × 0308 ÷ 1F1E6 ÷	#  ÷ [0.2] ARABIC NUMBER SIGN (Prepend) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [0.3]
 ÷ 0600 × 0600 ÷	#  ÷ [0.2] ARABIC NUMBER SIGN (Prepend) × [9.2] ARABIC NUMBER SIGN (Prepend) ÷ [0.3]
@@ -336,8 +324,6 @@
 ÷ 0600 × 0308 ÷ AC00 ÷	#  ÷ [0.2] ARABIC NUMBER SIGN (Prepend) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL SYLLABLE GA (LV) ÷ [0.3]
 ÷ 0600 × AC01 ÷	#  ÷ [0.2] ARABIC NUMBER SIGN (Prepend) × [9.2] HANGUL SYLLABLE GAG (LVT) ÷ [0.3]
 ÷ 0600 × 0308 ÷ AC01 ÷	#  ÷ [0.2] ARABIC NUMBER SIGN (Prepend) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL SYLLABLE GAG (LVT) ÷ [0.3]
-÷ 0600 × 0900 ÷	#  ÷ [0.2] ARABIC NUMBER SIGN (Prepend) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) ÷ [0.3]
-÷ 0600 × 0308 × 0900 ÷	#  ÷ [0.2] ARABIC NUMBER SIGN (Prepend) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) ÷ [0.3]
 ÷ 0600 × 0903 ÷	#  ÷ [0.2] ARABIC NUMBER SIGN (Prepend) × [9.1] DEVANAGARI SIGN VISARGA (SpacingMark_ConjunctLinkingScripts) ÷ [0.3]
 ÷ 0600 × 0308 × 0903 ÷	#  ÷ [0.2] ARABIC NUMBER SIGN (Prepend) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.1] DEVANAGARI SIGN VISARGA (SpacingMark_ConjunctLinkingScripts) ÷ [0.3]
 ÷ 0600 × 0904 ÷	#  ÷ [0.2] ARABIC NUMBER SIGN (Prepend) × [9.2] DEVANAGARI LETTER SHORT A (ConjunctLinkingScripts) ÷ [0.3]
@@ -350,8 +336,8 @@
 ÷ 0600 × 0308 ÷ 231A ÷	#  ÷ [0.2] ARABIC NUMBER SIGN (Prepend) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] WATCH (ExtPict) ÷ [0.3]
 ÷ 0600 × 0300 ÷	#  ÷ [0.2] ARABIC NUMBER SIGN (Prepend) × [9.0] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) ÷ [0.3]
 ÷ 0600 × 0308 × 0300 ÷	#  ÷ [0.2] ARABIC NUMBER SIGN (Prepend) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) ÷ [0.3]
-÷ 0600 × 093C ÷	#  ÷ [0.2] ARABIC NUMBER SIGN (Prepend) × [9.0] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3]
-÷ 0600 × 0308 × 093C ÷	#  ÷ [0.2] ARABIC NUMBER SIGN (Prepend) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3]
+÷ 0600 × 0900 ÷	#  ÷ [0.2] ARABIC NUMBER SIGN (Prepend) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3]
+÷ 0600 × 0308 × 0900 ÷	#  ÷ [0.2] ARABIC NUMBER SIGN (Prepend) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3]
 ÷ 0600 × 094D ÷	#  ÷ [0.2] ARABIC NUMBER SIGN (Prepend) × [9.0] DEVANAGARI SIGN VIRAMA (Extend_ConjunctLinkingScripts_ConjunctLinker_ExtCccZwj) ÷ [0.3]
 ÷ 0600 × 0308 × 094D ÷	#  ÷ [0.2] ARABIC NUMBER SIGN (Prepend) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN VIRAMA (Extend_ConjunctLinkingScripts_ConjunctLinker_ExtCccZwj) ÷ [0.3]
 ÷ 0600 × 200D ÷	#  ÷ [0.2] ARABIC NUMBER SIGN (Prepend) × [9.0] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) ÷ [0.3]
@@ -366,8 +352,8 @@
 ÷ 0A03 × 0308 ÷ 000A ÷	#  ÷ [0.2] GURMUKHI SIGN VISARGA (SpacingMark) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] <LINE FEED (LF)> (LF) ÷ [0.3]
 ÷ 0A03 ÷ 0001 ÷	#  ÷ [0.2] GURMUKHI SIGN VISARGA (SpacingMark) ÷ [5.0] <START OF HEADING> (Control) ÷ [0.3]
 ÷ 0A03 × 0308 ÷ 0001 ÷	#  ÷ [0.2] GURMUKHI SIGN VISARGA (SpacingMark) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] <START OF HEADING> (Control) ÷ [0.3]
-÷ 0A03 × 034F ÷	#  ÷ [0.2] GURMUKHI SIGN VISARGA (SpacingMark) × [9.0] COMBINING GRAPHEME JOINER (Extend) ÷ [0.3]
-÷ 0A03 × 0308 × 034F ÷	#  ÷ [0.2] GURMUKHI SIGN VISARGA (SpacingMark) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] COMBINING GRAPHEME JOINER (Extend) ÷ [0.3]
+÷ 0A03 × 200C ÷	#  ÷ [0.2] GURMUKHI SIGN VISARGA (SpacingMark) × [9.0] ZERO WIDTH NON-JOINER (Extend) ÷ [0.3]
+÷ 0A03 × 0308 × 200C ÷	#  ÷ [0.2] GURMUKHI SIGN VISARGA (SpacingMark) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] ZERO WIDTH NON-JOINER (Extend) ÷ [0.3]
 ÷ 0A03 ÷ 1F1E6 ÷	#  ÷ [0.2] GURMUKHI SIGN VISARGA (SpacingMark) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [0.3]
 ÷ 0A03 × 0308 ÷ 1F1E6 ÷	#  ÷ [0.2] GURMUKHI SIGN VISARGA (SpacingMark) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [0.3]
 ÷ 0A03 ÷ 0600 ÷	#  ÷ [0.2] GURMUKHI SIGN VISARGA (SpacingMark) ÷ [999.0] ARABIC NUMBER SIGN (Prepend) ÷ [0.3]
@@ -384,8 +370,6 @@
 ÷ 0A03 × 0308 ÷ AC00 ÷	#  ÷ [0.2] GURMUKHI SIGN VISARGA (SpacingMark) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL SYLLABLE GA (LV) ÷ [0.3]
 ÷ 0A03 ÷ AC01 ÷	#  ÷ [0.2] GURMUKHI SIGN VISARGA (SpacingMark) ÷ [999.0] HANGUL SYLLABLE GAG (LVT) ÷ [0.3]
 ÷ 0A03 × 0308 ÷ AC01 ÷	#  ÷ [0.2] GURMUKHI SIGN VISARGA (SpacingMark) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL SYLLABLE GAG (LVT) ÷ [0.3]
-÷ 0A03 × 0900 ÷	#  ÷ [0.2] GURMUKHI SIGN VISARGA (SpacingMark) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) ÷ [0.3]
-÷ 0A03 × 0308 × 0900 ÷	#  ÷ [0.2] GURMUKHI SIGN VISARGA (SpacingMark) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) ÷ [0.3]
 ÷ 0A03 × 0903 ÷	#  ÷ [0.2] GURMUKHI SIGN VISARGA (SpacingMark) × [9.1] DEVANAGARI SIGN VISARGA (SpacingMark_ConjunctLinkingScripts) ÷ [0.3]
 ÷ 0A03 × 0308 × 0903 ÷	#  ÷ [0.2] GURMUKHI SIGN VISARGA (SpacingMark) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.1] DEVANAGARI SIGN VISARGA (SpacingMark_ConjunctLinkingScripts) ÷ [0.3]
 ÷ 0A03 ÷ 0904 ÷	#  ÷ [0.2] GURMUKHI SIGN VISARGA (SpacingMark) ÷ [999.0] DEVANAGARI LETTER SHORT A (ConjunctLinkingScripts) ÷ [0.3]
@@ -398,8 +382,8 @@
 ÷ 0A03 × 0308 ÷ 231A ÷	#  ÷ [0.2] GURMUKHI SIGN VISARGA (SpacingMark) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] WATCH (ExtPict) ÷ [0.3]
 ÷ 0A03 × 0300 ÷	#  ÷ [0.2] GURMUKHI SIGN VISARGA (SpacingMark) × [9.0] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) ÷ [0.3]
 ÷ 0A03 × 0308 × 0300 ÷	#  ÷ [0.2] GURMUKHI SIGN VISARGA (SpacingMark) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) ÷ [0.3]
-÷ 0A03 × 093C ÷	#  ÷ [0.2] GURMUKHI SIGN VISARGA (SpacingMark) × [9.0] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3]
-÷ 0A03 × 0308 × 093C ÷	#  ÷ [0.2] GURMUKHI SIGN VISARGA (SpacingMark) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3]
+÷ 0A03 × 0900 ÷	#  ÷ [0.2] GURMUKHI SIGN VISARGA (SpacingMark) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3]
+÷ 0A03 × 0308 × 0900 ÷	#  ÷ [0.2] GURMUKHI SIGN VISARGA (SpacingMark) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3]
 ÷ 0A03 × 094D ÷	#  ÷ [0.2] GURMUKHI SIGN VISARGA (SpacingMark) × [9.0] DEVANAGARI SIGN VIRAMA (Extend_ConjunctLinkingScripts_ConjunctLinker_ExtCccZwj) ÷ [0.3]
 ÷ 0A03 × 0308 × 094D ÷	#  ÷ [0.2] GURMUKHI SIGN VISARGA (SpacingMark) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN VIRAMA (Extend_ConjunctLinkingScripts_ConjunctLinker_ExtCccZwj) ÷ [0.3]
 ÷ 0A03 × 200D ÷	#  ÷ [0.2] GURMUKHI SIGN VISARGA (SpacingMark) × [9.0] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) ÷ [0.3]
@@ -414,8 +398,8 @@
 ÷ 1100 × 0308 ÷ 000A ÷	#  ÷ [0.2] HANGUL CHOSEONG KIYEOK (L) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] <LINE FEED (LF)> (LF) ÷ [0.3]
 ÷ 1100 ÷ 0001 ÷	#  ÷ [0.2] HANGUL CHOSEONG KIYEOK (L) ÷ [5.0] <START OF HEADING> (Control) ÷ [0.3]
 ÷ 1100 × 0308 ÷ 0001 ÷	#  ÷ [0.2] HANGUL CHOSEONG KIYEOK (L) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] <START OF HEADING> (Control) ÷ [0.3]
-÷ 1100 × 034F ÷	#  ÷ [0.2] HANGUL CHOSEONG KIYEOK (L) × [9.0] COMBINING GRAPHEME JOINER (Extend) ÷ [0.3]
-÷ 1100 × 0308 × 034F ÷	#  ÷ [0.2] HANGUL CHOSEONG KIYEOK (L) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] COMBINING GRAPHEME JOINER (Extend) ÷ [0.3]
+÷ 1100 × 200C ÷	#  ÷ [0.2] HANGUL CHOSEONG KIYEOK (L) × [9.0] ZERO WIDTH NON-JOINER (Extend) ÷ [0.3]
+÷ 1100 × 0308 × 200C ÷	#  ÷ [0.2] HANGUL CHOSEONG KIYEOK (L) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] ZERO WIDTH NON-JOINER (Extend) ÷ [0.3]
 ÷ 1100 ÷ 1F1E6 ÷	#  ÷ [0.2] HANGUL CHOSEONG KIYEOK (L) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [0.3]
 ÷ 1100 × 0308 ÷ 1F1E6 ÷	#  ÷ [0.2] HANGUL CHOSEONG KIYEOK (L) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [0.3]
 ÷ 1100 ÷ 0600 ÷	#  ÷ [0.2] HANGUL CHOSEONG KIYEOK (L) ÷ [999.0] ARABIC NUMBER SIGN (Prepend) ÷ [0.3]
@@ -432,8 +416,6 @@
 ÷ 1100 × 0308 ÷ AC00 ÷	#  ÷ [0.2] HANGUL CHOSEONG KIYEOK (L) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL SYLLABLE GA (LV) ÷ [0.3]
 ÷ 1100 × AC01 ÷	#  ÷ [0.2] HANGUL CHOSEONG KIYEOK (L) × [6.0] HANGUL SYLLABLE GAG (LVT) ÷ [0.3]
 ÷ 1100 × 0308 ÷ AC01 ÷	#  ÷ [0.2] HANGUL CHOSEONG KIYEOK (L) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL SYLLABLE GAG (LVT) ÷ [0.3]
-÷ 1100 × 0900 ÷	#  ÷ [0.2] HANGUL CHOSEONG KIYEOK (L) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) ÷ [0.3]
-÷ 1100 × 0308 × 0900 ÷	#  ÷ [0.2] HANGUL CHOSEONG KIYEOK (L) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) ÷ [0.3]
 ÷ 1100 × 0903 ÷	#  ÷ [0.2] HANGUL CHOSEONG KIYEOK (L) × [9.1] DEVANAGARI SIGN VISARGA (SpacingMark_ConjunctLinkingScripts) ÷ [0.3]
 ÷ 1100 × 0308 × 0903 ÷	#  ÷ [0.2] HANGUL CHOSEONG KIYEOK (L) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.1] DEVANAGARI SIGN VISARGA (SpacingMark_ConjunctLinkingScripts) ÷ [0.3]
 ÷ 1100 ÷ 0904 ÷	#  ÷ [0.2] HANGUL CHOSEONG KIYEOK (L) ÷ [999.0] DEVANAGARI LETTER SHORT A (ConjunctLinkingScripts) ÷ [0.3]
@@ -446,8 +428,8 @@
 ÷ 1100 × 0308 ÷ 231A ÷	#  ÷ [0.2] HANGUL CHOSEONG KIYEOK (L) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] WATCH (ExtPict) ÷ [0.3]
 ÷ 1100 × 0300 ÷	#  ÷ [0.2] HANGUL CHOSEONG KIYEOK (L) × [9.0] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) ÷ [0.3]
 ÷ 1100 × 0308 × 0300 ÷	#  ÷ [0.2] HANGUL CHOSEONG KIYEOK (L) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) ÷ [0.3]
-÷ 1100 × 093C ÷	#  ÷ [0.2] HANGUL CHOSEONG KIYEOK (L) × [9.0] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3]
-÷ 1100 × 0308 × 093C ÷	#  ÷ [0.2] HANGUL CHOSEONG KIYEOK (L) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3]
+÷ 1100 × 0900 ÷	#  ÷ [0.2] HANGUL CHOSEONG KIYEOK (L) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3]
+÷ 1100 × 0308 × 0900 ÷	#  ÷ [0.2] HANGUL CHOSEONG KIYEOK (L) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3]
 ÷ 1100 × 094D ÷	#  ÷ [0.2] HANGUL CHOSEONG KIYEOK (L) × [9.0] DEVANAGARI SIGN VIRAMA (Extend_ConjunctLinkingScripts_ConjunctLinker_ExtCccZwj) ÷ [0.3]
 ÷ 1100 × 0308 × 094D ÷	#  ÷ [0.2] HANGUL CHOSEONG KIYEOK (L) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN VIRAMA (Extend_ConjunctLinkingScripts_ConjunctLinker_ExtCccZwj) ÷ [0.3]
 ÷ 1100 × 200D ÷	#  ÷ [0.2] HANGUL CHOSEONG KIYEOK (L) × [9.0] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) ÷ [0.3]
@@ -462,8 +444,8 @@
 ÷ 1160 × 0308 ÷ 000A ÷	#  ÷ [0.2] HANGUL JUNGSEONG FILLER (V) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] <LINE FEED (LF)> (LF) ÷ [0.3]
 ÷ 1160 ÷ 0001 ÷	#  ÷ [0.2] HANGUL JUNGSEONG FILLER (V) ÷ [5.0] <START OF HEADING> (Control) ÷ [0.3]
 ÷ 1160 × 0308 ÷ 0001 ÷	#  ÷ [0.2] HANGUL JUNGSEONG FILLER (V) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] <START OF HEADING> (Control) ÷ [0.3]
-÷ 1160 × 034F ÷	#  ÷ [0.2] HANGUL JUNGSEONG FILLER (V) × [9.0] COMBINING GRAPHEME JOINER (Extend) ÷ [0.3]
-÷ 1160 × 0308 × 034F ÷	#  ÷ [0.2] HANGUL JUNGSEONG FILLER (V) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] COMBINING GRAPHEME JOINER (Extend) ÷ [0.3]
+÷ 1160 × 200C ÷	#  ÷ [0.2] HANGUL JUNGSEONG FILLER (V) × [9.0] ZERO WIDTH NON-JOINER (Extend) ÷ [0.3]
+÷ 1160 × 0308 × 200C ÷	#  ÷ [0.2] HANGUL JUNGSEONG FILLER (V) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] ZERO WIDTH NON-JOINER (Extend) ÷ [0.3]
 ÷ 1160 ÷ 1F1E6 ÷	#  ÷ [0.2] HANGUL JUNGSEONG FILLER (V) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [0.3]
 ÷ 1160 × 0308 ÷ 1F1E6 ÷	#  ÷ [0.2] HANGUL JUNGSEONG FILLER (V) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [0.3]
 ÷ 1160 ÷ 0600 ÷	#  ÷ [0.2] HANGUL JUNGSEONG FILLER (V) ÷ [999.0] ARABIC NUMBER SIGN (Prepend) ÷ [0.3]
@@ -480,8 +462,6 @@
 ÷ 1160 × 0308 ÷ AC00 ÷	#  ÷ [0.2] HANGUL JUNGSEONG FILLER (V) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL SYLLABLE GA (LV) ÷ [0.3]
 ÷ 1160 ÷ AC01 ÷	#  ÷ [0.2] HANGUL JUNGSEONG FILLER (V) ÷ [999.0] HANGUL SYLLABLE GAG (LVT) ÷ [0.3]
 ÷ 1160 × 0308 ÷ AC01 ÷	#  ÷ [0.2] HANGUL JUNGSEONG FILLER (V) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL SYLLABLE GAG (LVT) ÷ [0.3]
-÷ 1160 × 0900 ÷	#  ÷ [0.2] HANGUL JUNGSEONG FILLER (V) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) ÷ [0.3]
-÷ 1160 × 0308 × 0900 ÷	#  ÷ [0.2] HANGUL JUNGSEONG FILLER (V) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) ÷ [0.3]
 ÷ 1160 × 0903 ÷	#  ÷ [0.2] HANGUL JUNGSEONG FILLER (V) × [9.1] DEVANAGARI SIGN VISARGA (SpacingMark_ConjunctLinkingScripts) ÷ [0.3]
 ÷ 1160 × 0308 × 0903 ÷	#  ÷ [0.2] HANGUL JUNGSEONG FILLER (V) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.1] DEVANAGARI SIGN VISARGA (SpacingMark_ConjunctLinkingScripts) ÷ [0.3]
 ÷ 1160 ÷ 0904 ÷	#  ÷ [0.2] HANGUL JUNGSEONG FILLER (V) ÷ [999.0] DEVANAGARI LETTER SHORT A (ConjunctLinkingScripts) ÷ [0.3]
@@ -494,8 +474,8 @@
 ÷ 1160 × 0308 ÷ 231A ÷	#  ÷ [0.2] HANGUL JUNGSEONG FILLER (V) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] WATCH (ExtPict) ÷ [0.3]
 ÷ 1160 × 0300 ÷	#  ÷ [0.2] HANGUL JUNGSEONG FILLER (V) × [9.0] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) ÷ [0.3]
 ÷ 1160 × 0308 × 0300 ÷	#  ÷ [0.2] HANGUL JUNGSEONG FILLER (V) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) ÷ [0.3]
-÷ 1160 × 093C ÷	#  ÷ [0.2] HANGUL JUNGSEONG FILLER (V) × [9.0] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3]
-÷ 1160 × 0308 × 093C ÷	#  ÷ [0.2] HANGUL JUNGSEONG FILLER (V) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3]
+÷ 1160 × 0900 ÷	#  ÷ [0.2] HANGUL JUNGSEONG FILLER (V) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3]
+÷ 1160 × 0308 × 0900 ÷	#  ÷ [0.2] HANGUL JUNGSEONG FILLER (V) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3]
 ÷ 1160 × 094D ÷	#  ÷ [0.2] HANGUL JUNGSEONG FILLER (V) × [9.0] DEVANAGARI SIGN VIRAMA (Extend_ConjunctLinkingScripts_ConjunctLinker_ExtCccZwj) ÷ [0.3]
 ÷ 1160 × 0308 × 094D ÷	#  ÷ [0.2] HANGUL JUNGSEONG FILLER (V) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN VIRAMA (Extend_ConjunctLinkingScripts_ConjunctLinker_ExtCccZwj) ÷ [0.3]
 ÷ 1160 × 200D ÷	#  ÷ [0.2] HANGUL JUNGSEONG FILLER (V) × [9.0] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) ÷ [0.3]
@@ -510,8 +490,8 @@
 ÷ 11A8 × 0308 ÷ 000A ÷	#  ÷ [0.2] HANGUL JONGSEONG KIYEOK (T) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] <LINE FEED (LF)> (LF) ÷ [0.3]
 ÷ 11A8 ÷ 0001 ÷	#  ÷ [0.2] HANGUL JONGSEONG KIYEOK (T) ÷ [5.0] <START OF HEADING> (Control) ÷ [0.3]
 ÷ 11A8 × 0308 ÷ 0001 ÷	#  ÷ [0.2] HANGUL JONGSEONG KIYEOK (T) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] <START OF HEADING> (Control) ÷ [0.3]
-÷ 11A8 × 034F ÷	#  ÷ [0.2] HANGUL JONGSEONG KIYEOK (T) × [9.0] COMBINING GRAPHEME JOINER (Extend) ÷ [0.3]
-÷ 11A8 × 0308 × 034F ÷	#  ÷ [0.2] HANGUL JONGSEONG KIYEOK (T) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] COMBINING GRAPHEME JOINER (Extend) ÷ [0.3]
+÷ 11A8 × 200C ÷	#  ÷ [0.2] HANGUL JONGSEONG KIYEOK (T) × [9.0] ZERO WIDTH NON-JOINER (Extend) ÷ [0.3]
+÷ 11A8 × 0308 × 200C ÷	#  ÷ [0.2] HANGUL JONGSEONG KIYEOK (T) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] ZERO WIDTH NON-JOINER (Extend) ÷ [0.3]
 ÷ 11A8 ÷ 1F1E6 ÷	#  ÷ [0.2] HANGUL JONGSEONG KIYEOK (T) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [0.3]
 ÷ 11A8 × 0308 ÷ 1F1E6 ÷	#  ÷ [0.2] HANGUL JONGSEONG KIYEOK (T) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [0.3]
 ÷ 11A8 ÷ 0600 ÷	#  ÷ [0.2] HANGUL JONGSEONG KIYEOK (T) ÷ [999.0] ARABIC NUMBER SIGN (Prepend) ÷ [0.3]
@@ -528,8 +508,6 @@
 ÷ 11A8 × 0308 ÷ AC00 ÷	#  ÷ [0.2] HANGUL JONGSEONG KIYEOK (T) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL SYLLABLE GA (LV) ÷ [0.3]
 ÷ 11A8 ÷ AC01 ÷	#  ÷ [0.2] HANGUL JONGSEONG KIYEOK (T) ÷ [999.0] HANGUL SYLLABLE GAG (LVT) ÷ [0.3]
 ÷ 11A8 × 0308 ÷ AC01 ÷	#  ÷ [0.2] HANGUL JONGSEONG KIYEOK (T) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL SYLLABLE GAG (LVT) ÷ [0.3]
-÷ 11A8 × 0900 ÷	#  ÷ [0.2] HANGUL JONGSEONG KIYEOK (T) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) ÷ [0.3]
-÷ 11A8 × 0308 × 0900 ÷	#  ÷ [0.2] HANGUL JONGSEONG KIYEOK (T) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) ÷ [0.3]
 ÷ 11A8 × 0903 ÷	#  ÷ [0.2] HANGUL JONGSEONG KIYEOK (T) × [9.1] DEVANAGARI SIGN VISARGA (SpacingMark_ConjunctLinkingScripts) ÷ [0.3]
 ÷ 11A8 × 0308 × 0903 ÷	#  ÷ [0.2] HANGUL JONGSEONG KIYEOK (T) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.1] DEVANAGARI SIGN VISARGA (SpacingMark_ConjunctLinkingScripts) ÷ [0.3]
 ÷ 11A8 ÷ 0904 ÷	#  ÷ [0.2] HANGUL JONGSEONG KIYEOK (T) ÷ [999.0] DEVANAGARI LETTER SHORT A (ConjunctLinkingScripts) ÷ [0.3]
@@ -542,8 +520,8 @@
 ÷ 11A8 × 0308 ÷ 231A ÷	#  ÷ [0.2] HANGUL JONGSEONG KIYEOK (T) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] WATCH (ExtPict) ÷ [0.3]
 ÷ 11A8 × 0300 ÷	#  ÷ [0.2] HANGUL JONGSEONG KIYEOK (T) × [9.0] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) ÷ [0.3]
 ÷ 11A8 × 0308 × 0300 ÷	#  ÷ [0.2] HANGUL JONGSEONG KIYEOK (T) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) ÷ [0.3]
-÷ 11A8 × 093C ÷	#  ÷ [0.2] HANGUL JONGSEONG KIYEOK (T) × [9.0] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3]
-÷ 11A8 × 0308 × 093C ÷	#  ÷ [0.2] HANGUL JONGSEONG KIYEOK (T) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3]
+÷ 11A8 × 0900 ÷	#  ÷ [0.2] HANGUL JONGSEONG KIYEOK (T) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3]
+÷ 11A8 × 0308 × 0900 ÷	#  ÷ [0.2] HANGUL JONGSEONG KIYEOK (T) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3]
 ÷ 11A8 × 094D ÷	#  ÷ [0.2] HANGUL JONGSEONG KIYEOK (T) × [9.0] DEVANAGARI SIGN VIRAMA (Extend_ConjunctLinkingScripts_ConjunctLinker_ExtCccZwj) ÷ [0.3]
 ÷ 11A8 × 0308 × 094D ÷	#  ÷ [0.2] HANGUL JONGSEONG KIYEOK (T) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN VIRAMA (Extend_ConjunctLinkingScripts_ConjunctLinker_ExtCccZwj) ÷ [0.3]
 ÷ 11A8 × 200D ÷	#  ÷ [0.2] HANGUL JONGSEONG KIYEOK (T) × [9.0] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) ÷ [0.3]
@@ -558,8 +536,8 @@
 ÷ AC00 × 0308 ÷ 000A ÷	#  ÷ [0.2] HANGUL SYLLABLE GA (LV) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] <LINE FEED (LF)> (LF) ÷ [0.3]
 ÷ AC00 ÷ 0001 ÷	#  ÷ [0.2] HANGUL SYLLABLE GA (LV) ÷ [5.0] <START OF HEADING> (Control) ÷ [0.3]
 ÷ AC00 × 0308 ÷ 0001 ÷	#  ÷ [0.2] HANGUL SYLLABLE GA (LV) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] <START OF HEADING> (Control) ÷ [0.3]
-÷ AC00 × 034F ÷	#  ÷ [0.2] HANGUL SYLLABLE GA (LV) × [9.0] COMBINING GRAPHEME JOINER (Extend) ÷ [0.3]
-÷ AC00 × 0308 × 034F ÷	#  ÷ [0.2] HANGUL SYLLABLE GA (LV) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] COMBINING GRAPHEME JOINER (Extend) ÷ [0.3]
+÷ AC00 × 200C ÷	#  ÷ [0.2] HANGUL SYLLABLE GA (LV) × [9.0] ZERO WIDTH NON-JOINER (Extend) ÷ [0.3]
+÷ AC00 × 0308 × 200C ÷	#  ÷ [0.2] HANGUL SYLLABLE GA (LV) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] ZERO WIDTH NON-JOINER (Extend) ÷ [0.3]
 ÷ AC00 ÷ 1F1E6 ÷	#  ÷ [0.2] HANGUL SYLLABLE GA (LV) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [0.3]
 ÷ AC00 × 0308 ÷ 1F1E6 ÷	#  ÷ [0.2] HANGUL SYLLABLE GA (LV) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [0.3]
 ÷ AC00 ÷ 0600 ÷	#  ÷ [0.2] HANGUL SYLLABLE GA (LV) ÷ [999.0] ARABIC NUMBER SIGN (Prepend) ÷ [0.3]
@@ -576,8 +554,6 @@
 ÷ AC00 × 0308 ÷ AC00 ÷	#  ÷ [0.2] HANGUL SYLLABLE GA (LV) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL SYLLABLE GA (LV) ÷ [0.3]
 ÷ AC00 ÷ AC01 ÷	#  ÷ [0.2] HANGUL SYLLABLE GA (LV) ÷ [999.0] HANGUL SYLLABLE GAG (LVT) ÷ [0.3]
 ÷ AC00 × 0308 ÷ AC01 ÷	#  ÷ [0.2] HANGUL SYLLABLE GA (LV) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL SYLLABLE GAG (LVT) ÷ [0.3]
-÷ AC00 × 0900 ÷	#  ÷ [0.2] HANGUL SYLLABLE GA (LV) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) ÷ [0.3]
-÷ AC00 × 0308 × 0900 ÷	#  ÷ [0.2] HANGUL SYLLABLE GA (LV) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) ÷ [0.3]
 ÷ AC00 × 0903 ÷	#  ÷ [0.2] HANGUL SYLLABLE GA (LV) × [9.1] DEVANAGARI SIGN VISARGA (SpacingMark_ConjunctLinkingScripts) ÷ [0.3]
 ÷ AC00 × 0308 × 0903 ÷	#  ÷ [0.2] HANGUL SYLLABLE GA (LV) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.1] DEVANAGARI SIGN VISARGA (SpacingMark_ConjunctLinkingScripts) ÷ [0.3]
 ÷ AC00 ÷ 0904 ÷	#  ÷ [0.2] HANGUL SYLLABLE GA (LV) ÷ [999.0] DEVANAGARI LETTER SHORT A (ConjunctLinkingScripts) ÷ [0.3]
@@ -590,8 +566,8 @@
 ÷ AC00 × 0308 ÷ 231A ÷	#  ÷ [0.2] HANGUL SYLLABLE GA (LV) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] WATCH (ExtPict) ÷ [0.3]
 ÷ AC00 × 0300 ÷	#  ÷ [0.2] HANGUL SYLLABLE GA (LV) × [9.0] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) ÷ [0.3]
 ÷ AC00 × 0308 × 0300 ÷	#  ÷ [0.2] HANGUL SYLLABLE GA (LV) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) ÷ [0.3]
-÷ AC00 × 093C ÷	#  ÷ [0.2] HANGUL SYLLABLE GA (LV) × [9.0] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3]
-÷ AC00 × 0308 × 093C ÷	#  ÷ [0.2] HANGUL SYLLABLE GA (LV) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3]
+÷ AC00 × 0900 ÷	#  ÷ [0.2] HANGUL SYLLABLE GA (LV) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3]
+÷ AC00 × 0308 × 0900 ÷	#  ÷ [0.2] HANGUL SYLLABLE GA (LV) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3]
 ÷ AC00 × 094D ÷	#  ÷ [0.2] HANGUL SYLLABLE GA (LV) × [9.0] DEVANAGARI SIGN VIRAMA (Extend_ConjunctLinkingScripts_ConjunctLinker_ExtCccZwj) ÷ [0.3]
 ÷ AC00 × 0308 × 094D ÷	#  ÷ [0.2] HANGUL SYLLABLE GA (LV) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN VIRAMA (Extend_ConjunctLinkingScripts_ConjunctLinker_ExtCccZwj) ÷ [0.3]
 ÷ AC00 × 200D ÷	#  ÷ [0.2] HANGUL SYLLABLE GA (LV) × [9.0] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) ÷ [0.3]
@@ -606,8 +582,8 @@
 ÷ AC01 × 0308 ÷ 000A ÷	#  ÷ [0.2] HANGUL SYLLABLE GAG (LVT) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] <LINE FEED (LF)> (LF) ÷ [0.3]
 ÷ AC01 ÷ 0001 ÷	#  ÷ [0.2] HANGUL SYLLABLE GAG (LVT) ÷ [5.0] <START OF HEADING> (Control) ÷ [0.3]
 ÷ AC01 × 0308 ÷ 0001 ÷	#  ÷ [0.2] HANGUL SYLLABLE GAG (LVT) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] <START OF HEADING> (Control) ÷ [0.3]
-÷ AC01 × 034F ÷	#  ÷ [0.2] HANGUL SYLLABLE GAG (LVT) × [9.0] COMBINING GRAPHEME JOINER (Extend) ÷ [0.3]
-÷ AC01 × 0308 × 034F ÷	#  ÷ [0.2] HANGUL SYLLABLE GAG (LVT) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] COMBINING GRAPHEME JOINER (Extend) ÷ [0.3]
+÷ AC01 × 200C ÷	#  ÷ [0.2] HANGUL SYLLABLE GAG (LVT) × [9.0] ZERO WIDTH NON-JOINER (Extend) ÷ [0.3]
+÷ AC01 × 0308 × 200C ÷	#  ÷ [0.2] HANGUL SYLLABLE GAG (LVT) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] ZERO WIDTH NON-JOINER (Extend) ÷ [0.3]
 ÷ AC01 ÷ 1F1E6 ÷	#  ÷ [0.2] HANGUL SYLLABLE GAG (LVT) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [0.3]
 ÷ AC01 × 0308 ÷ 1F1E6 ÷	#  ÷ [0.2] HANGUL SYLLABLE GAG (LVT) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [0.3]
 ÷ AC01 ÷ 0600 ÷	#  ÷ [0.2] HANGUL SYLLABLE GAG (LVT) ÷ [999.0] ARABIC NUMBER SIGN (Prepend) ÷ [0.3]
@@ -624,8 +600,6 @@
 ÷ AC01 × 0308 ÷ AC00 ÷	#  ÷ [0.2] HANGUL SYLLABLE GAG (LVT) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL SYLLABLE GA (LV) ÷ [0.3]
 ÷ AC01 ÷ AC01 ÷	#  ÷ [0.2] HANGUL SYLLABLE GAG (LVT) ÷ [999.0] HANGUL SYLLABLE GAG (LVT) ÷ [0.3]
 ÷ AC01 × 0308 ÷ AC01 ÷	#  ÷ [0.2] HANGUL SYLLABLE GAG (LVT) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL SYLLABLE GAG (LVT) ÷ [0.3]
-÷ AC01 × 0900 ÷	#  ÷ [0.2] HANGUL SYLLABLE GAG (LVT) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) ÷ [0.3]
-÷ AC01 × 0308 × 0900 ÷	#  ÷ [0.2] HANGUL SYLLABLE GAG (LVT) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) ÷ [0.3]
 ÷ AC01 × 0903 ÷	#  ÷ [0.2] HANGUL SYLLABLE GAG (LVT) × [9.1] DEVANAGARI SIGN VISARGA (SpacingMark_ConjunctLinkingScripts) ÷ [0.3]
 ÷ AC01 × 0308 × 0903 ÷	#  ÷ [0.2] HANGUL SYLLABLE GAG (LVT) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.1] DEVANAGARI SIGN VISARGA (SpacingMark_ConjunctLinkingScripts) ÷ [0.3]
 ÷ AC01 ÷ 0904 ÷	#  ÷ [0.2] HANGUL SYLLABLE GAG (LVT) ÷ [999.0] DEVANAGARI LETTER SHORT A (ConjunctLinkingScripts) ÷ [0.3]
@@ -638,62 +612,14 @@
 ÷ AC01 × 0308 ÷ 231A ÷	#  ÷ [0.2] HANGUL SYLLABLE GAG (LVT) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] WATCH (ExtPict) ÷ [0.3]
 ÷ AC01 × 0300 ÷	#  ÷ [0.2] HANGUL SYLLABLE GAG (LVT) × [9.0] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) ÷ [0.3]
 ÷ AC01 × 0308 × 0300 ÷	#  ÷ [0.2] HANGUL SYLLABLE GAG (LVT) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) ÷ [0.3]
-÷ AC01 × 093C ÷	#  ÷ [0.2] HANGUL SYLLABLE GAG (LVT) × [9.0] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3]
-÷ AC01 × 0308 × 093C ÷	#  ÷ [0.2] HANGUL SYLLABLE GAG (LVT) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3]
+÷ AC01 × 0900 ÷	#  ÷ [0.2] HANGUL SYLLABLE GAG (LVT) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3]
+÷ AC01 × 0308 × 0900 ÷	#  ÷ [0.2] HANGUL SYLLABLE GAG (LVT) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3]
 ÷ AC01 × 094D ÷	#  ÷ [0.2] HANGUL SYLLABLE GAG (LVT) × [9.0] DEVANAGARI SIGN VIRAMA (Extend_ConjunctLinkingScripts_ConjunctLinker_ExtCccZwj) ÷ [0.3]
 ÷ AC01 × 0308 × 094D ÷	#  ÷ [0.2] HANGUL SYLLABLE GAG (LVT) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN VIRAMA (Extend_ConjunctLinkingScripts_ConjunctLinker_ExtCccZwj) ÷ [0.3]
 ÷ AC01 × 200D ÷	#  ÷ [0.2] HANGUL SYLLABLE GAG (LVT) × [9.0] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) ÷ [0.3]
 ÷ AC01 × 0308 × 200D ÷	#  ÷ [0.2] HANGUL SYLLABLE GAG (LVT) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) ÷ [0.3]
 ÷ AC01 ÷ 0378 ÷	#  ÷ [0.2] HANGUL SYLLABLE GAG (LVT) ÷ [999.0] <reserved-0378> (Other) ÷ [0.3]
 ÷ AC01 × 0308 ÷ 0378 ÷	#  ÷ [0.2] HANGUL SYLLABLE GAG (LVT) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] <reserved-0378> (Other) ÷ [0.3]
-÷ 0900 ÷ 0020 ÷	#  ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) ÷ [999.0] SPACE (Other) ÷ [0.3]
-÷ 0900 × 0308 ÷ 0020 ÷	#  ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] SPACE (Other) ÷ [0.3]
-÷ 0900 ÷ 000D ÷	#  ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) ÷ [5.0] <CARRIAGE RETURN (CR)> (CR) ÷ [0.3]
-÷ 0900 × 0308 ÷ 000D ÷	#  ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] <CARRIAGE RETURN (CR)> (CR) ÷ [0.3]
-÷ 0900 ÷ 000A ÷	#  ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) ÷ [5.0] <LINE FEED (LF)> (LF) ÷ [0.3]
-÷ 0900 × 0308 ÷ 000A ÷	#  ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] <LINE FEED (LF)> (LF) ÷ [0.3]
-÷ 0900 ÷ 0001 ÷	#  ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) ÷ [5.0] <START OF HEADING> (Control) ÷ [0.3]
-÷ 0900 × 0308 ÷ 0001 ÷	#  ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] <START OF HEADING> (Control) ÷ [0.3]
-÷ 0900 × 034F ÷	#  ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) × [9.0] COMBINING GRAPHEME JOINER (Extend) ÷ [0.3]
-÷ 0900 × 0308 × 034F ÷	#  ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] COMBINING GRAPHEME JOINER (Extend) ÷ [0.3]
-÷ 0900 ÷ 1F1E6 ÷	#  ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [0.3]
-÷ 0900 × 0308 ÷ 1F1E6 ÷	#  ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [0.3]
-÷ 0900 ÷ 0600 ÷	#  ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) ÷ [999.0] ARABIC NUMBER SIGN (Prepend) ÷ [0.3]
-÷ 0900 × 0308 ÷ 0600 ÷	#  ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] ARABIC NUMBER SIGN (Prepend) ÷ [0.3]
-÷ 0900 × 0A03 ÷	#  ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) × [9.1] GURMUKHI SIGN VISARGA (SpacingMark) ÷ [0.3]
-÷ 0900 × 0308 × 0A03 ÷	#  ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.1] GURMUKHI SIGN VISARGA (SpacingMark) ÷ [0.3]
-÷ 0900 ÷ 1100 ÷	#  ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) ÷ [999.0] HANGUL CHOSEONG KIYEOK (L) ÷ [0.3]
-÷ 0900 × 0308 ÷ 1100 ÷	#  ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL CHOSEONG KIYEOK (L) ÷ [0.3]
-÷ 0900 ÷ 1160 ÷	#  ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) ÷ [999.0] HANGUL JUNGSEONG FILLER (V) ÷ [0.3]
-÷ 0900 × 0308 ÷ 1160 ÷	#  ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL JUNGSEONG FILLER (V) ÷ [0.3]
-÷ 0900 ÷ 11A8 ÷	#  ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) ÷ [999.0] HANGUL JONGSEONG KIYEOK (T) ÷ [0.3]
-÷ 0900 × 0308 ÷ 11A8 ÷	#  ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL JONGSEONG KIYEOK (T) ÷ [0.3]
-÷ 0900 ÷ AC00 ÷	#  ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) ÷ [999.0] HANGUL SYLLABLE GA (LV) ÷ [0.3]
-÷ 0900 × 0308 ÷ AC00 ÷	#  ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL SYLLABLE GA (LV) ÷ [0.3]
-÷ 0900 ÷ AC01 ÷	#  ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) ÷ [999.0] HANGUL SYLLABLE GAG (LVT) ÷ [0.3]
-÷ 0900 × 0308 ÷ AC01 ÷	#  ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL SYLLABLE GAG (LVT) ÷ [0.3]
-÷ 0900 × 0900 ÷	#  ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) ÷ [0.3]
-÷ 0900 × 0308 × 0900 ÷	#  ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) ÷ [0.3]
-÷ 0900 × 0903 ÷	#  ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) × [9.1] DEVANAGARI SIGN VISARGA (SpacingMark_ConjunctLinkingScripts) ÷ [0.3]
-÷ 0900 × 0308 × 0903 ÷	#  ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.1] DEVANAGARI SIGN VISARGA (SpacingMark_ConjunctLinkingScripts) ÷ [0.3]
-÷ 0900 ÷ 0904 ÷	#  ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) ÷ [999.0] DEVANAGARI LETTER SHORT A (ConjunctLinkingScripts) ÷ [0.3]
-÷ 0900 × 0308 ÷ 0904 ÷	#  ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] DEVANAGARI LETTER SHORT A (ConjunctLinkingScripts) ÷ [0.3]
-÷ 0900 ÷ 0D4E ÷	#  ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) ÷ [999.0] MALAYALAM LETTER DOT REPH (Prepend_ConjunctLinkingScripts) ÷ [0.3]
-÷ 0900 × 0308 ÷ 0D4E ÷	#  ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] MALAYALAM LETTER DOT REPH (Prepend_ConjunctLinkingScripts) ÷ [0.3]
-÷ 0900 ÷ 0915 ÷	#  ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) ÷ [999.0] DEVANAGARI LETTER KA (ConjunctLinkingScripts_LinkingConsonant) ÷ [0.3]
-÷ 0900 × 0308 ÷ 0915 ÷	#  ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] DEVANAGARI LETTER KA (ConjunctLinkingScripts_LinkingConsonant) ÷ [0.3]
-÷ 0900 ÷ 231A ÷	#  ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) ÷ [999.0] WATCH (ExtPict) ÷ [0.3]
-÷ 0900 × 0308 ÷ 231A ÷	#  ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] WATCH (ExtPict) ÷ [0.3]
-÷ 0900 × 0300 ÷	#  ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) × [9.0] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) ÷ [0.3]
-÷ 0900 × 0308 × 0300 ÷	#  ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) ÷ [0.3]
-÷ 0900 × 093C ÷	#  ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) × [9.0] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3]
-÷ 0900 × 0308 × 093C ÷	#  ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3]
-÷ 0900 × 094D ÷	#  ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) × [9.0] DEVANAGARI SIGN VIRAMA (Extend_ConjunctLinkingScripts_ConjunctLinker_ExtCccZwj) ÷ [0.3]
-÷ 0900 × 0308 × 094D ÷	#  ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN VIRAMA (Extend_ConjunctLinkingScripts_ConjunctLinker_ExtCccZwj) ÷ [0.3]
-÷ 0900 × 200D ÷	#  ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) × [9.0] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) ÷ [0.3]
-÷ 0900 × 0308 × 200D ÷	#  ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) ÷ [0.3]
-÷ 0900 ÷ 0378 ÷	#  ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) ÷ [999.0] <reserved-0378> (Other) ÷ [0.3]
-÷ 0900 × 0308 ÷ 0378 ÷	#  ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] <reserved-0378> (Other) ÷ [0.3]
 ÷ 0903 ÷ 0020 ÷	#  ÷ [0.2] DEVANAGARI SIGN VISARGA (SpacingMark_ConjunctLinkingScripts) ÷ [999.0] SPACE (Other) ÷ [0.3]
 ÷ 0903 × 0308 ÷ 0020 ÷	#  ÷ [0.2] DEVANAGARI SIGN VISARGA (SpacingMark_ConjunctLinkingScripts) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] SPACE (Other) ÷ [0.3]
 ÷ 0903 ÷ 000D ÷	#  ÷ [0.2] DEVANAGARI SIGN VISARGA (SpacingMark_ConjunctLinkingScripts) ÷ [5.0] <CARRIAGE RETURN (CR)> (CR) ÷ [0.3]
@@ -702,8 +628,8 @@
 ÷ 0903 × 0308 ÷ 000A ÷	#  ÷ [0.2] DEVANAGARI SIGN VISARGA (SpacingMark_ConjunctLinkingScripts) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] <LINE FEED (LF)> (LF) ÷ [0.3]
 ÷ 0903 ÷ 0001 ÷	#  ÷ [0.2] DEVANAGARI SIGN VISARGA (SpacingMark_ConjunctLinkingScripts) ÷ [5.0] <START OF HEADING> (Control) ÷ [0.3]
 ÷ 0903 × 0308 ÷ 0001 ÷	#  ÷ [0.2] DEVANAGARI SIGN VISARGA (SpacingMark_ConjunctLinkingScripts) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] <START OF HEADING> (Control) ÷ [0.3]
-÷ 0903 × 034F ÷	#  ÷ [0.2] DEVANAGARI SIGN VISARGA (SpacingMark_ConjunctLinkingScripts) × [9.0] COMBINING GRAPHEME JOINER (Extend) ÷ [0.3]
-÷ 0903 × 0308 × 034F ÷	#  ÷ [0.2] DEVANAGARI SIGN VISARGA (SpacingMark_ConjunctLinkingScripts) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] COMBINING GRAPHEME JOINER (Extend) ÷ [0.3]
+÷ 0903 × 200C ÷	#  ÷ [0.2] DEVANAGARI SIGN VISARGA (SpacingMark_ConjunctLinkingScripts) × [9.0] ZERO WIDTH NON-JOINER (Extend) ÷ [0.3]
+÷ 0903 × 0308 × 200C ÷	#  ÷ [0.2] DEVANAGARI SIGN VISARGA (SpacingMark_ConjunctLinkingScripts) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] ZERO WIDTH NON-JOINER (Extend) ÷ [0.3]
 ÷ 0903 ÷ 1F1E6 ÷	#  ÷ [0.2] DEVANAGARI SIGN VISARGA (SpacingMark_ConjunctLinkingScripts) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [0.3]
 ÷ 0903 × 0308 ÷ 1F1E6 ÷	#  ÷ [0.2] DEVANAGARI SIGN VISARGA (SpacingMark_ConjunctLinkingScripts) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [0.3]
 ÷ 0903 ÷ 0600 ÷	#  ÷ [0.2] DEVANAGARI SIGN VISARGA (SpacingMark_ConjunctLinkingScripts) ÷ [999.0] ARABIC NUMBER SIGN (Prepend) ÷ [0.3]
@@ -720,8 +646,6 @@
 ÷ 0903 × 0308 ÷ AC00 ÷	#  ÷ [0.2] DEVANAGARI SIGN VISARGA (SpacingMark_ConjunctLinkingScripts) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL SYLLABLE GA (LV) ÷ [0.3]
 ÷ 0903 ÷ AC01 ÷	#  ÷ [0.2] DEVANAGARI SIGN VISARGA (SpacingMark_ConjunctLinkingScripts) ÷ [999.0] HANGUL SYLLABLE GAG (LVT) ÷ [0.3]
 ÷ 0903 × 0308 ÷ AC01 ÷	#  ÷ [0.2] DEVANAGARI SIGN VISARGA (SpacingMark_ConjunctLinkingScripts) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL SYLLABLE GAG (LVT) ÷ [0.3]
-÷ 0903 × 0900 ÷	#  ÷ [0.2] DEVANAGARI SIGN VISARGA (SpacingMark_ConjunctLinkingScripts) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) ÷ [0.3]
-÷ 0903 × 0308 × 0900 ÷	#  ÷ [0.2] DEVANAGARI SIGN VISARGA (SpacingMark_ConjunctLinkingScripts) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) ÷ [0.3]
 ÷ 0903 × 0903 ÷	#  ÷ [0.2] DEVANAGARI SIGN VISARGA (SpacingMark_ConjunctLinkingScripts) × [9.1] DEVANAGARI SIGN VISARGA (SpacingMark_ConjunctLinkingScripts) ÷ [0.3]
 ÷ 0903 × 0308 × 0903 ÷	#  ÷ [0.2] DEVANAGARI SIGN VISARGA (SpacingMark_ConjunctLinkingScripts) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.1] DEVANAGARI SIGN VISARGA (SpacingMark_ConjunctLinkingScripts) ÷ [0.3]
 ÷ 0903 ÷ 0904 ÷	#  ÷ [0.2] DEVANAGARI SIGN VISARGA (SpacingMark_ConjunctLinkingScripts) ÷ [999.0] DEVANAGARI LETTER SHORT A (ConjunctLinkingScripts) ÷ [0.3]
@@ -734,8 +658,8 @@
 ÷ 0903 × 0308 ÷ 231A ÷	#  ÷ [0.2] DEVANAGARI SIGN VISARGA (SpacingMark_ConjunctLinkingScripts) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] WATCH (ExtPict) ÷ [0.3]
 ÷ 0903 × 0300 ÷	#  ÷ [0.2] DEVANAGARI SIGN VISARGA (SpacingMark_ConjunctLinkingScripts) × [9.0] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) ÷ [0.3]
 ÷ 0903 × 0308 × 0300 ÷	#  ÷ [0.2] DEVANAGARI SIGN VISARGA (SpacingMark_ConjunctLinkingScripts) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) ÷ [0.3]
-÷ 0903 × 093C ÷	#  ÷ [0.2] DEVANAGARI SIGN VISARGA (SpacingMark_ConjunctLinkingScripts) × [9.0] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3]
-÷ 0903 × 0308 × 093C ÷	#  ÷ [0.2] DEVANAGARI SIGN VISARGA (SpacingMark_ConjunctLinkingScripts) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3]
+÷ 0903 × 0900 ÷	#  ÷ [0.2] DEVANAGARI SIGN VISARGA (SpacingMark_ConjunctLinkingScripts) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3]
+÷ 0903 × 0308 × 0900 ÷	#  ÷ [0.2] DEVANAGARI SIGN VISARGA (SpacingMark_ConjunctLinkingScripts) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3]
 ÷ 0903 × 094D ÷	#  ÷ [0.2] DEVANAGARI SIGN VISARGA (SpacingMark_ConjunctLinkingScripts) × [9.0] DEVANAGARI SIGN VIRAMA (Extend_ConjunctLinkingScripts_ConjunctLinker_ExtCccZwj) ÷ [0.3]
 ÷ 0903 × 0308 × 094D ÷	#  ÷ [0.2] DEVANAGARI SIGN VISARGA (SpacingMark_ConjunctLinkingScripts) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN VIRAMA (Extend_ConjunctLinkingScripts_ConjunctLinker_ExtCccZwj) ÷ [0.3]
 ÷ 0903 × 200D ÷	#  ÷ [0.2] DEVANAGARI SIGN VISARGA (SpacingMark_ConjunctLinkingScripts) × [9.0] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) ÷ [0.3]
@@ -750,8 +674,8 @@
 ÷ 0904 × 0308 ÷ 000A ÷	#  ÷ [0.2] DEVANAGARI LETTER SHORT A (ConjunctLinkingScripts) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] <LINE FEED (LF)> (LF) ÷ [0.3]
 ÷ 0904 ÷ 0001 ÷	#  ÷ [0.2] DEVANAGARI LETTER SHORT A (ConjunctLinkingScripts) ÷ [5.0] <START OF HEADING> (Control) ÷ [0.3]
 ÷ 0904 × 0308 ÷ 0001 ÷	#  ÷ [0.2] DEVANAGARI LETTER SHORT A (ConjunctLinkingScripts) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] <START OF HEADING> (Control) ÷ [0.3]
-÷ 0904 × 034F ÷	#  ÷ [0.2] DEVANAGARI LETTER SHORT A (ConjunctLinkingScripts) × [9.0] COMBINING GRAPHEME JOINER (Extend) ÷ [0.3]
-÷ 0904 × 0308 × 034F ÷	#  ÷ [0.2] DEVANAGARI LETTER SHORT A (ConjunctLinkingScripts) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] COMBINING GRAPHEME JOINER (Extend) ÷ [0.3]
+÷ 0904 × 200C ÷	#  ÷ [0.2] DEVANAGARI LETTER SHORT A (ConjunctLinkingScripts) × [9.0] ZERO WIDTH NON-JOINER (Extend) ÷ [0.3]
+÷ 0904 × 0308 × 200C ÷	#  ÷ [0.2] DEVANAGARI LETTER SHORT A (ConjunctLinkingScripts) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] ZERO WIDTH NON-JOINER (Extend) ÷ [0.3]
 ÷ 0904 ÷ 1F1E6 ÷	#  ÷ [0.2] DEVANAGARI LETTER SHORT A (ConjunctLinkingScripts) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [0.3]
 ÷ 0904 × 0308 ÷ 1F1E6 ÷	#  ÷ [0.2] DEVANAGARI LETTER SHORT A (ConjunctLinkingScripts) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [0.3]
 ÷ 0904 ÷ 0600 ÷	#  ÷ [0.2] DEVANAGARI LETTER SHORT A (ConjunctLinkingScripts) ÷ [999.0] ARABIC NUMBER SIGN (Prepend) ÷ [0.3]
@@ -768,8 +692,6 @@
 ÷ 0904 × 0308 ÷ AC00 ÷	#  ÷ [0.2] DEVANAGARI LETTER SHORT A (ConjunctLinkingScripts) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL SYLLABLE GA (LV) ÷ [0.3]
 ÷ 0904 ÷ AC01 ÷	#  ÷ [0.2] DEVANAGARI LETTER SHORT A (ConjunctLinkingScripts) ÷ [999.0] HANGUL SYLLABLE GAG (LVT) ÷ [0.3]
 ÷ 0904 × 0308 ÷ AC01 ÷	#  ÷ [0.2] DEVANAGARI LETTER SHORT A (ConjunctLinkingScripts) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL SYLLABLE GAG (LVT) ÷ [0.3]
-÷ 0904 × 0900 ÷	#  ÷ [0.2] DEVANAGARI LETTER SHORT A (ConjunctLinkingScripts) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) ÷ [0.3]
-÷ 0904 × 0308 × 0900 ÷	#  ÷ [0.2] DEVANAGARI LETTER SHORT A (ConjunctLinkingScripts) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) ÷ [0.3]
 ÷ 0904 × 0903 ÷	#  ÷ [0.2] DEVANAGARI LETTER SHORT A (ConjunctLinkingScripts) × [9.1] DEVANAGARI SIGN VISARGA (SpacingMark_ConjunctLinkingScripts) ÷ [0.3]
 ÷ 0904 × 0308 × 0903 ÷	#  ÷ [0.2] DEVANAGARI LETTER SHORT A (ConjunctLinkingScripts) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.1] DEVANAGARI SIGN VISARGA (SpacingMark_ConjunctLinkingScripts) ÷ [0.3]
 ÷ 0904 ÷ 0904 ÷	#  ÷ [0.2] DEVANAGARI LETTER SHORT A (ConjunctLinkingScripts) ÷ [999.0] DEVANAGARI LETTER SHORT A (ConjunctLinkingScripts) ÷ [0.3]
@@ -782,8 +704,8 @@
 ÷ 0904 × 0308 ÷ 231A ÷	#  ÷ [0.2] DEVANAGARI LETTER SHORT A (ConjunctLinkingScripts) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] WATCH (ExtPict) ÷ [0.3]
 ÷ 0904 × 0300 ÷	#  ÷ [0.2] DEVANAGARI LETTER SHORT A (ConjunctLinkingScripts) × [9.0] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) ÷ [0.3]
 ÷ 0904 × 0308 × 0300 ÷	#  ÷ [0.2] DEVANAGARI LETTER SHORT A (ConjunctLinkingScripts) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) ÷ [0.3]
-÷ 0904 × 093C ÷	#  ÷ [0.2] DEVANAGARI LETTER SHORT A (ConjunctLinkingScripts) × [9.0] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3]
-÷ 0904 × 0308 × 093C ÷	#  ÷ [0.2] DEVANAGARI LETTER SHORT A (ConjunctLinkingScripts) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3]
+÷ 0904 × 0900 ÷	#  ÷ [0.2] DEVANAGARI LETTER SHORT A (ConjunctLinkingScripts) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3]
+÷ 0904 × 0308 × 0900 ÷	#  ÷ [0.2] DEVANAGARI LETTER SHORT A (ConjunctLinkingScripts) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3]
 ÷ 0904 × 094D ÷	#  ÷ [0.2] DEVANAGARI LETTER SHORT A (ConjunctLinkingScripts) × [9.0] DEVANAGARI SIGN VIRAMA (Extend_ConjunctLinkingScripts_ConjunctLinker_ExtCccZwj) ÷ [0.3]
 ÷ 0904 × 0308 × 094D ÷	#  ÷ [0.2] DEVANAGARI LETTER SHORT A (ConjunctLinkingScripts) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN VIRAMA (Extend_ConjunctLinkingScripts_ConjunctLinker_ExtCccZwj) ÷ [0.3]
 ÷ 0904 × 200D ÷	#  ÷ [0.2] DEVANAGARI LETTER SHORT A (ConjunctLinkingScripts) × [9.0] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) ÷ [0.3]
@@ -798,8 +720,8 @@
 ÷ 0D4E × 0308 ÷ 000A ÷	#  ÷ [0.2] MALAYALAM LETTER DOT REPH (Prepend_ConjunctLinkingScripts) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] <LINE FEED (LF)> (LF) ÷ [0.3]
 ÷ 0D4E ÷ 0001 ÷	#  ÷ [0.2] MALAYALAM LETTER DOT REPH (Prepend_ConjunctLinkingScripts) ÷ [5.0] <START OF HEADING> (Control) ÷ [0.3]
 ÷ 0D4E × 0308 ÷ 0001 ÷	#  ÷ [0.2] MALAYALAM LETTER DOT REPH (Prepend_ConjunctLinkingScripts) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] <START OF HEADING> (Control) ÷ [0.3]
-÷ 0D4E × 034F ÷	#  ÷ [0.2] MALAYALAM LETTER DOT REPH (Prepend_ConjunctLinkingScripts) × [9.0] COMBINING GRAPHEME JOINER (Extend) ÷ [0.3]
-÷ 0D4E × 0308 × 034F ÷	#  ÷ [0.2] MALAYALAM LETTER DOT REPH (Prepend_ConjunctLinkingScripts) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] COMBINING GRAPHEME JOINER (Extend) ÷ [0.3]
+÷ 0D4E × 200C ÷	#  ÷ [0.2] MALAYALAM LETTER DOT REPH (Prepend_ConjunctLinkingScripts) × [9.0] ZERO WIDTH NON-JOINER (Extend) ÷ [0.3]
+÷ 0D4E × 0308 × 200C ÷	#  ÷ [0.2] MALAYALAM LETTER DOT REPH (Prepend_ConjunctLinkingScripts) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] ZERO WIDTH NON-JOINER (Extend) ÷ [0.3]
 ÷ 0D4E × 1F1E6 ÷	#  ÷ [0.2] MALAYALAM LETTER DOT REPH (Prepend_ConjunctLinkingScripts) × [9.2] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [0.3]
 ÷ 0D4E × 0308 ÷ 1F1E6 ÷	#  ÷ [0.2] MALAYALAM LETTER DOT REPH (Prepend_ConjunctLinkingScripts) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [0.3]
 ÷ 0D4E × 0600 ÷	#  ÷ [0.2] MALAYALAM LETTER DOT REPH (Prepend_ConjunctLinkingScripts) × [9.2] ARABIC NUMBER SIGN (Prepend) ÷ [0.3]
@@ -816,8 +738,6 @@
 ÷ 0D4E × 0308 ÷ AC00 ÷	#  ÷ [0.2] MALAYALAM LETTER DOT REPH (Prepend_ConjunctLinkingScripts) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL SYLLABLE GA (LV) ÷ [0.3]
 ÷ 0D4E × AC01 ÷	#  ÷ [0.2] MALAYALAM LETTER DOT REPH (Prepend_ConjunctLinkingScripts) × [9.2] HANGUL SYLLABLE GAG (LVT) ÷ [0.3]
 ÷ 0D4E × 0308 ÷ AC01 ÷	#  ÷ [0.2] MALAYALAM LETTER DOT REPH (Prepend_ConjunctLinkingScripts) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL SYLLABLE GAG (LVT) ÷ [0.3]
-÷ 0D4E × 0900 ÷	#  ÷ [0.2] MALAYALAM LETTER DOT REPH (Prepend_ConjunctLinkingScripts) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) ÷ [0.3]
-÷ 0D4E × 0308 × 0900 ÷	#  ÷ [0.2] MALAYALAM LETTER DOT REPH (Prepend_ConjunctLinkingScripts) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) ÷ [0.3]
 ÷ 0D4E × 0903 ÷	#  ÷ [0.2] MALAYALAM LETTER DOT REPH (Prepend_ConjunctLinkingScripts) × [9.1] DEVANAGARI SIGN VISARGA (SpacingMark_ConjunctLinkingScripts) ÷ [0.3]
 ÷ 0D4E × 0308 × 0903 ÷	#  ÷ [0.2] MALAYALAM LETTER DOT REPH (Prepend_ConjunctLinkingScripts) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.1] DEVANAGARI SIGN VISARGA (SpacingMark_ConjunctLinkingScripts) ÷ [0.3]
 ÷ 0D4E × 0904 ÷	#  ÷ [0.2] MALAYALAM LETTER DOT REPH (Prepend_ConjunctLinkingScripts) × [9.2] DEVANAGARI LETTER SHORT A (ConjunctLinkingScripts) ÷ [0.3]
@@ -830,8 +750,8 @@
 ÷ 0D4E × 0308 ÷ 231A ÷	#  ÷ [0.2] MALAYALAM LETTER DOT REPH (Prepend_ConjunctLinkingScripts) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] WATCH (ExtPict) ÷ [0.3]
 ÷ 0D4E × 0300 ÷	#  ÷ [0.2] MALAYALAM LETTER DOT REPH (Prepend_ConjunctLinkingScripts) × [9.0] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) ÷ [0.3]
 ÷ 0D4E × 0308 × 0300 ÷	#  ÷ [0.2] MALAYALAM LETTER DOT REPH (Prepend_ConjunctLinkingScripts) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) ÷ [0.3]
-÷ 0D4E × 093C ÷	#  ÷ [0.2] MALAYALAM LETTER DOT REPH (Prepend_ConjunctLinkingScripts) × [9.0] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3]
-÷ 0D4E × 0308 × 093C ÷	#  ÷ [0.2] MALAYALAM LETTER DOT REPH (Prepend_ConjunctLinkingScripts) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3]
+÷ 0D4E × 0900 ÷	#  ÷ [0.2] MALAYALAM LETTER DOT REPH (Prepend_ConjunctLinkingScripts) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3]
+÷ 0D4E × 0308 × 0900 ÷	#  ÷ [0.2] MALAYALAM LETTER DOT REPH (Prepend_ConjunctLinkingScripts) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3]
 ÷ 0D4E × 094D ÷	#  ÷ [0.2] MALAYALAM LETTER DOT REPH (Prepend_ConjunctLinkingScripts) × [9.0] DEVANAGARI SIGN VIRAMA (Extend_ConjunctLinkingScripts_ConjunctLinker_ExtCccZwj) ÷ [0.3]
 ÷ 0D4E × 0308 × 094D ÷	#  ÷ [0.2] MALAYALAM LETTER DOT REPH (Prepend_ConjunctLinkingScripts) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN VIRAMA (Extend_ConjunctLinkingScripts_ConjunctLinker_ExtCccZwj) ÷ [0.3]
 ÷ 0D4E × 200D ÷	#  ÷ [0.2] MALAYALAM LETTER DOT REPH (Prepend_ConjunctLinkingScripts) × [9.0] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) ÷ [0.3]
@@ -846,8 +766,8 @@
 ÷ 0915 × 0308 ÷ 000A ÷	#  ÷ [0.2] DEVANAGARI LETTER KA (ConjunctLinkingScripts_LinkingConsonant) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] <LINE FEED (LF)> (LF) ÷ [0.3]
 ÷ 0915 ÷ 0001 ÷	#  ÷ [0.2] DEVANAGARI LETTER KA (ConjunctLinkingScripts_LinkingConsonant) ÷ [5.0] <START OF HEADING> (Control) ÷ [0.3]
 ÷ 0915 × 0308 ÷ 0001 ÷	#  ÷ [0.2] DEVANAGARI LETTER KA (ConjunctLinkingScripts_LinkingConsonant) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] <START OF HEADING> (Control) ÷ [0.3]
-÷ 0915 × 034F ÷	#  ÷ [0.2] DEVANAGARI LETTER KA (ConjunctLinkingScripts_LinkingConsonant) × [9.0] COMBINING GRAPHEME JOINER (Extend) ÷ [0.3]
-÷ 0915 × 0308 × 034F ÷	#  ÷ [0.2] DEVANAGARI LETTER KA (ConjunctLinkingScripts_LinkingConsonant) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] COMBINING GRAPHEME JOINER (Extend) ÷ [0.3]
+÷ 0915 × 200C ÷	#  ÷ [0.2] DEVANAGARI LETTER KA (ConjunctLinkingScripts_LinkingConsonant) × [9.0] ZERO WIDTH NON-JOINER (Extend) ÷ [0.3]
+÷ 0915 × 0308 × 200C ÷	#  ÷ [0.2] DEVANAGARI LETTER KA (ConjunctLinkingScripts_LinkingConsonant) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] ZERO WIDTH NON-JOINER (Extend) ÷ [0.3]
 ÷ 0915 ÷ 1F1E6 ÷	#  ÷ [0.2] DEVANAGARI LETTER KA (ConjunctLinkingScripts_LinkingConsonant) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [0.3]
 ÷ 0915 × 0308 ÷ 1F1E6 ÷	#  ÷ [0.2] DEVANAGARI LETTER KA (ConjunctLinkingScripts_LinkingConsonant) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [0.3]
 ÷ 0915 ÷ 0600 ÷	#  ÷ [0.2] DEVANAGARI LETTER KA (ConjunctLinkingScripts_LinkingConsonant) ÷ [999.0] ARABIC NUMBER SIGN (Prepend) ÷ [0.3]
@@ -864,8 +784,6 @@
 ÷ 0915 × 0308 ÷ AC00 ÷	#  ÷ [0.2] DEVANAGARI LETTER KA (ConjunctLinkingScripts_LinkingConsonant) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL SYLLABLE GA (LV) ÷ [0.3]
 ÷ 0915 ÷ AC01 ÷	#  ÷ [0.2] DEVANAGARI LETTER KA (ConjunctLinkingScripts_LinkingConsonant) ÷ [999.0] HANGUL SYLLABLE GAG (LVT) ÷ [0.3]
 ÷ 0915 × 0308 ÷ AC01 ÷	#  ÷ [0.2] DEVANAGARI LETTER KA (ConjunctLinkingScripts_LinkingConsonant) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL SYLLABLE GAG (LVT) ÷ [0.3]
-÷ 0915 × 0900 ÷	#  ÷ [0.2] DEVANAGARI LETTER KA (ConjunctLinkingScripts_LinkingConsonant) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) ÷ [0.3]
-÷ 0915 × 0308 × 0900 ÷	#  ÷ [0.2] DEVANAGARI LETTER KA (ConjunctLinkingScripts_LinkingConsonant) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) ÷ [0.3]
 ÷ 0915 × 0903 ÷	#  ÷ [0.2] DEVANAGARI LETTER KA (ConjunctLinkingScripts_LinkingConsonant) × [9.1] DEVANAGARI SIGN VISARGA (SpacingMark_ConjunctLinkingScripts) ÷ [0.3]
 ÷ 0915 × 0308 × 0903 ÷	#  ÷ [0.2] DEVANAGARI LETTER KA (ConjunctLinkingScripts_LinkingConsonant) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.1] DEVANAGARI SIGN VISARGA (SpacingMark_ConjunctLinkingScripts) ÷ [0.3]
 ÷ 0915 ÷ 0904 ÷	#  ÷ [0.2] DEVANAGARI LETTER KA (ConjunctLinkingScripts_LinkingConsonant) ÷ [999.0] DEVANAGARI LETTER SHORT A (ConjunctLinkingScripts) ÷ [0.3]
@@ -878,8 +796,8 @@
 ÷ 0915 × 0308 ÷ 231A ÷	#  ÷ [0.2] DEVANAGARI LETTER KA (ConjunctLinkingScripts_LinkingConsonant) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] WATCH (ExtPict) ÷ [0.3]
 ÷ 0915 × 0300 ÷	#  ÷ [0.2] DEVANAGARI LETTER KA (ConjunctLinkingScripts_LinkingConsonant) × [9.0] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) ÷ [0.3]
 ÷ 0915 × 0308 × 0300 ÷	#  ÷ [0.2] DEVANAGARI LETTER KA (ConjunctLinkingScripts_LinkingConsonant) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) ÷ [0.3]
-÷ 0915 × 093C ÷	#  ÷ [0.2] DEVANAGARI LETTER KA (ConjunctLinkingScripts_LinkingConsonant) × [9.0] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3]
-÷ 0915 × 0308 × 093C ÷	#  ÷ [0.2] DEVANAGARI LETTER KA (ConjunctLinkingScripts_LinkingConsonant) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3]
+÷ 0915 × 0900 ÷	#  ÷ [0.2] DEVANAGARI LETTER KA (ConjunctLinkingScripts_LinkingConsonant) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3]
+÷ 0915 × 0308 × 0900 ÷	#  ÷ [0.2] DEVANAGARI LETTER KA (ConjunctLinkingScripts_LinkingConsonant) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3]
 ÷ 0915 × 094D ÷	#  ÷ [0.2] DEVANAGARI LETTER KA (ConjunctLinkingScripts_LinkingConsonant) × [9.0] DEVANAGARI SIGN VIRAMA (Extend_ConjunctLinkingScripts_ConjunctLinker_ExtCccZwj) ÷ [0.3]
 ÷ 0915 × 0308 × 094D ÷	#  ÷ [0.2] DEVANAGARI LETTER KA (ConjunctLinkingScripts_LinkingConsonant) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN VIRAMA (Extend_ConjunctLinkingScripts_ConjunctLinker_ExtCccZwj) ÷ [0.3]
 ÷ 0915 × 200D ÷	#  ÷ [0.2] DEVANAGARI LETTER KA (ConjunctLinkingScripts_LinkingConsonant) × [9.0] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) ÷ [0.3]
@@ -894,8 +812,8 @@
 ÷ 231A × 0308 ÷ 000A ÷	#  ÷ [0.2] WATCH (ExtPict) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] <LINE FEED (LF)> (LF) ÷ [0.3]
 ÷ 231A ÷ 0001 ÷	#  ÷ [0.2] WATCH (ExtPict) ÷ [5.0] <START OF HEADING> (Control) ÷ [0.3]
 ÷ 231A × 0308 ÷ 0001 ÷	#  ÷ [0.2] WATCH (ExtPict) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] <START OF HEADING> (Control) ÷ [0.3]
-÷ 231A × 034F ÷	#  ÷ [0.2] WATCH (ExtPict) × [9.0] COMBINING GRAPHEME JOINER (Extend) ÷ [0.3]
-÷ 231A × 0308 × 034F ÷	#  ÷ [0.2] WATCH (ExtPict) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] COMBINING GRAPHEME JOINER (Extend) ÷ [0.3]
+÷ 231A × 200C ÷	#  ÷ [0.2] WATCH (ExtPict) × [9.0] ZERO WIDTH NON-JOINER (Extend) ÷ [0.3]
+÷ 231A × 0308 × 200C ÷	#  ÷ [0.2] WATCH (ExtPict) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] ZERO WIDTH NON-JOINER (Extend) ÷ [0.3]
 ÷ 231A ÷ 1F1E6 ÷	#  ÷ [0.2] WATCH (ExtPict) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [0.3]
 ÷ 231A × 0308 ÷ 1F1E6 ÷	#  ÷ [0.2] WATCH (ExtPict) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [0.3]
 ÷ 231A ÷ 0600 ÷	#  ÷ [0.2] WATCH (ExtPict) ÷ [999.0] ARABIC NUMBER SIGN (Prepend) ÷ [0.3]
@@ -912,8 +830,6 @@
 ÷ 231A × 0308 ÷ AC00 ÷	#  ÷ [0.2] WATCH (ExtPict) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL SYLLABLE GA (LV) ÷ [0.3]
 ÷ 231A ÷ AC01 ÷	#  ÷ [0.2] WATCH (ExtPict) ÷ [999.0] HANGUL SYLLABLE GAG (LVT) ÷ [0.3]
 ÷ 231A × 0308 ÷ AC01 ÷	#  ÷ [0.2] WATCH (ExtPict) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL SYLLABLE GAG (LVT) ÷ [0.3]
-÷ 231A × 0900 ÷	#  ÷ [0.2] WATCH (ExtPict) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) ÷ [0.3]
-÷ 231A × 0308 × 0900 ÷	#  ÷ [0.2] WATCH (ExtPict) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) ÷ [0.3]
 ÷ 231A × 0903 ÷	#  ÷ [0.2] WATCH (ExtPict) × [9.1] DEVANAGARI SIGN VISARGA (SpacingMark_ConjunctLinkingScripts) ÷ [0.3]
 ÷ 231A × 0308 × 0903 ÷	#  ÷ [0.2] WATCH (ExtPict) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.1] DEVANAGARI SIGN VISARGA (SpacingMark_ConjunctLinkingScripts) ÷ [0.3]
 ÷ 231A ÷ 0904 ÷	#  ÷ [0.2] WATCH (ExtPict) ÷ [999.0] DEVANAGARI LETTER SHORT A (ConjunctLinkingScripts) ÷ [0.3]
@@ -926,8 +842,8 @@
 ÷ 231A × 0308 ÷ 231A ÷	#  ÷ [0.2] WATCH (ExtPict) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] WATCH (ExtPict) ÷ [0.3]
 ÷ 231A × 0300 ÷	#  ÷ [0.2] WATCH (ExtPict) × [9.0] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) ÷ [0.3]
 ÷ 231A × 0308 × 0300 ÷	#  ÷ [0.2] WATCH (ExtPict) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) ÷ [0.3]
-÷ 231A × 093C ÷	#  ÷ [0.2] WATCH (ExtPict) × [9.0] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3]
-÷ 231A × 0308 × 093C ÷	#  ÷ [0.2] WATCH (ExtPict) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3]
+÷ 231A × 0900 ÷	#  ÷ [0.2] WATCH (ExtPict) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3]
+÷ 231A × 0308 × 0900 ÷	#  ÷ [0.2] WATCH (ExtPict) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3]
 ÷ 231A × 094D ÷	#  ÷ [0.2] WATCH (ExtPict) × [9.0] DEVANAGARI SIGN VIRAMA (Extend_ConjunctLinkingScripts_ConjunctLinker_ExtCccZwj) ÷ [0.3]
 ÷ 231A × 0308 × 094D ÷	#  ÷ [0.2] WATCH (ExtPict) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN VIRAMA (Extend_ConjunctLinkingScripts_ConjunctLinker_ExtCccZwj) ÷ [0.3]
 ÷ 231A × 200D ÷	#  ÷ [0.2] WATCH (ExtPict) × [9.0] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) ÷ [0.3]
@@ -942,8 +858,8 @@
 ÷ 0300 × 0308 ÷ 000A ÷	#  ÷ [0.2] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] <LINE FEED (LF)> (LF) ÷ [0.3]
 ÷ 0300 ÷ 0001 ÷	#  ÷ [0.2] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) ÷ [5.0] <START OF HEADING> (Control) ÷ [0.3]
 ÷ 0300 × 0308 ÷ 0001 ÷	#  ÷ [0.2] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] <START OF HEADING> (Control) ÷ [0.3]
-÷ 0300 × 034F ÷	#  ÷ [0.2] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) × [9.0] COMBINING GRAPHEME JOINER (Extend) ÷ [0.3]
-÷ 0300 × 0308 × 034F ÷	#  ÷ [0.2] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] COMBINING GRAPHEME JOINER (Extend) ÷ [0.3]
+÷ 0300 × 200C ÷	#  ÷ [0.2] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) × [9.0] ZERO WIDTH NON-JOINER (Extend) ÷ [0.3]
+÷ 0300 × 0308 × 200C ÷	#  ÷ [0.2] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] ZERO WIDTH NON-JOINER (Extend) ÷ [0.3]
 ÷ 0300 ÷ 1F1E6 ÷	#  ÷ [0.2] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [0.3]
 ÷ 0300 × 0308 ÷ 1F1E6 ÷	#  ÷ [0.2] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [0.3]
 ÷ 0300 ÷ 0600 ÷	#  ÷ [0.2] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) ÷ [999.0] ARABIC NUMBER SIGN (Prepend) ÷ [0.3]
@@ -960,8 +876,6 @@
 ÷ 0300 × 0308 ÷ AC00 ÷	#  ÷ [0.2] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL SYLLABLE GA (LV) ÷ [0.3]
 ÷ 0300 ÷ AC01 ÷	#  ÷ [0.2] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) ÷ [999.0] HANGUL SYLLABLE GAG (LVT) ÷ [0.3]
 ÷ 0300 × 0308 ÷ AC01 ÷	#  ÷ [0.2] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL SYLLABLE GAG (LVT) ÷ [0.3]
-÷ 0300 × 0900 ÷	#  ÷ [0.2] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) ÷ [0.3]
-÷ 0300 × 0308 × 0900 ÷	#  ÷ [0.2] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) ÷ [0.3]
 ÷ 0300 × 0903 ÷	#  ÷ [0.2] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) × [9.1] DEVANAGARI SIGN VISARGA (SpacingMark_ConjunctLinkingScripts) ÷ [0.3]
 ÷ 0300 × 0308 × 0903 ÷	#  ÷ [0.2] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.1] DEVANAGARI SIGN VISARGA (SpacingMark_ConjunctLinkingScripts) ÷ [0.3]
 ÷ 0300 ÷ 0904 ÷	#  ÷ [0.2] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) ÷ [999.0] DEVANAGARI LETTER SHORT A (ConjunctLinkingScripts) ÷ [0.3]
@@ -974,62 +888,60 @@
 ÷ 0300 × 0308 ÷ 231A ÷	#  ÷ [0.2] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] WATCH (ExtPict) ÷ [0.3]
 ÷ 0300 × 0300 ÷	#  ÷ [0.2] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) × [9.0] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) ÷ [0.3]
 ÷ 0300 × 0308 × 0300 ÷	#  ÷ [0.2] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) ÷ [0.3]
-÷ 0300 × 093C ÷	#  ÷ [0.2] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3]
-÷ 0300 × 0308 × 093C ÷	#  ÷ [0.2] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3]
+÷ 0300 × 0900 ÷	#  ÷ [0.2] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3]
+÷ 0300 × 0308 × 0900 ÷	#  ÷ [0.2] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3]
 ÷ 0300 × 094D ÷	#  ÷ [0.2] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN VIRAMA (Extend_ConjunctLinkingScripts_ConjunctLinker_ExtCccZwj) ÷ [0.3]
 ÷ 0300 × 0308 × 094D ÷	#  ÷ [0.2] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN VIRAMA (Extend_ConjunctLinkingScripts_ConjunctLinker_ExtCccZwj) ÷ [0.3]
 ÷ 0300 × 200D ÷	#  ÷ [0.2] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) × [9.0] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) ÷ [0.3]
 ÷ 0300 × 0308 × 200D ÷	#  ÷ [0.2] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) ÷ [0.3]
 ÷ 0300 ÷ 0378 ÷	#  ÷ [0.2] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) ÷ [999.0] <reserved-0378> (Other) ÷ [0.3]
 ÷ 0300 × 0308 ÷ 0378 ÷	#  ÷ [0.2] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] <reserved-0378> (Other) ÷ [0.3]
-÷ 093C ÷ 0020 ÷	#  ÷ [0.2] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [999.0] SPACE (Other) ÷ [0.3]
-÷ 093C × 0308 ÷ 0020 ÷	#  ÷ [0.2] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] SPACE (Other) ÷ [0.3]
-÷ 093C ÷ 000D ÷	#  ÷ [0.2] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [5.0] <CARRIAGE RETURN (CR)> (CR) ÷ [0.3]
-÷ 093C × 0308 ÷ 000D ÷	#  ÷ [0.2] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] <CARRIAGE RETURN (CR)> (CR) ÷ [0.3]
-÷ 093C ÷ 000A ÷	#  ÷ [0.2] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [5.0] <LINE FEED (LF)> (LF) ÷ [0.3]
-÷ 093C × 0308 ÷ 000A ÷	#  ÷ [0.2] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] <LINE FEED (LF)> (LF) ÷ [0.3]
-÷ 093C ÷ 0001 ÷	#  ÷ [0.2] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [5.0] <START OF HEADING> (Control) ÷ [0.3]
-÷ 093C × 0308 ÷ 0001 ÷	#  ÷ [0.2] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] <START OF HEADING> (Control) ÷ [0.3]
-÷ 093C × 034F ÷	#  ÷ [0.2] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) × [9.0] COMBINING GRAPHEME JOINER (Extend) ÷ [0.3]
-÷ 093C × 0308 × 034F ÷	#  ÷ [0.2] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] COMBINING GRAPHEME JOINER (Extend) ÷ [0.3]
-÷ 093C ÷ 1F1E6 ÷	#  ÷ [0.2] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [0.3]
-÷ 093C × 0308 ÷ 1F1E6 ÷	#  ÷ [0.2] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [0.3]
-÷ 093C ÷ 0600 ÷	#  ÷ [0.2] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [999.0] ARABIC NUMBER SIGN (Prepend) ÷ [0.3]
-÷ 093C × 0308 ÷ 0600 ÷	#  ÷ [0.2] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] ARABIC NUMBER SIGN (Prepend) ÷ [0.3]
-÷ 093C × 0A03 ÷	#  ÷ [0.2] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) × [9.1] GURMUKHI SIGN VISARGA (SpacingMark) ÷ [0.3]
-÷ 093C × 0308 × 0A03 ÷	#  ÷ [0.2] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.1] GURMUKHI SIGN VISARGA (SpacingMark) ÷ [0.3]
-÷ 093C ÷ 1100 ÷	#  ÷ [0.2] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [999.0] HANGUL CHOSEONG KIYEOK (L) ÷ [0.3]
-÷ 093C × 0308 ÷ 1100 ÷	#  ÷ [0.2] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL CHOSEONG KIYEOK (L) ÷ [0.3]
-÷ 093C ÷ 1160 ÷	#  ÷ [0.2] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [999.0] HANGUL JUNGSEONG FILLER (V) ÷ [0.3]
-÷ 093C × 0308 ÷ 1160 ÷	#  ÷ [0.2] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL JUNGSEONG FILLER (V) ÷ [0.3]
-÷ 093C ÷ 11A8 ÷	#  ÷ [0.2] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [999.0] HANGUL JONGSEONG KIYEOK (T) ÷ [0.3]
-÷ 093C × 0308 ÷ 11A8 ÷	#  ÷ [0.2] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL JONGSEONG KIYEOK (T) ÷ [0.3]
-÷ 093C ÷ AC00 ÷	#  ÷ [0.2] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [999.0] HANGUL SYLLABLE GA (LV) ÷ [0.3]
-÷ 093C × 0308 ÷ AC00 ÷	#  ÷ [0.2] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL SYLLABLE GA (LV) ÷ [0.3]
-÷ 093C ÷ AC01 ÷	#  ÷ [0.2] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [999.0] HANGUL SYLLABLE GAG (LVT) ÷ [0.3]
-÷ 093C × 0308 ÷ AC01 ÷	#  ÷ [0.2] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL SYLLABLE GAG (LVT) ÷ [0.3]
-÷ 093C × 0900 ÷	#  ÷ [0.2] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) ÷ [0.3]
-÷ 093C × 0308 × 0900 ÷	#  ÷ [0.2] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) ÷ [0.3]
-÷ 093C × 0903 ÷	#  ÷ [0.2] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) × [9.1] DEVANAGARI SIGN VISARGA (SpacingMark_ConjunctLinkingScripts) ÷ [0.3]
-÷ 093C × 0308 × 0903 ÷	#  ÷ [0.2] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.1] DEVANAGARI SIGN VISARGA (SpacingMark_ConjunctLinkingScripts) ÷ [0.3]
-÷ 093C ÷ 0904 ÷	#  ÷ [0.2] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [999.0] DEVANAGARI LETTER SHORT A (ConjunctLinkingScripts) ÷ [0.3]
-÷ 093C × 0308 ÷ 0904 ÷	#  ÷ [0.2] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] DEVANAGARI LETTER SHORT A (ConjunctLinkingScripts) ÷ [0.3]
-÷ 093C ÷ 0D4E ÷	#  ÷ [0.2] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [999.0] MALAYALAM LETTER DOT REPH (Prepend_ConjunctLinkingScripts) ÷ [0.3]
-÷ 093C × 0308 ÷ 0D4E ÷	#  ÷ [0.2] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] MALAYALAM LETTER DOT REPH (Prepend_ConjunctLinkingScripts) ÷ [0.3]
-÷ 093C ÷ 0915 ÷	#  ÷ [0.2] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [999.0] DEVANAGARI LETTER KA (ConjunctLinkingScripts_LinkingConsonant) ÷ [0.3]
-÷ 093C × 0308 ÷ 0915 ÷	#  ÷ [0.2] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] DEVANAGARI LETTER KA (ConjunctLinkingScripts_LinkingConsonant) ÷ [0.3]
-÷ 093C ÷ 231A ÷	#  ÷ [0.2] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [999.0] WATCH (ExtPict) ÷ [0.3]
-÷ 093C × 0308 ÷ 231A ÷	#  ÷ [0.2] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] WATCH (ExtPict) ÷ [0.3]
-÷ 093C × 0300 ÷	#  ÷ [0.2] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) × [9.0] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) ÷ [0.3]
-÷ 093C × 0308 × 0300 ÷	#  ÷ [0.2] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) ÷ [0.3]
-÷ 093C × 093C ÷	#  ÷ [0.2] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) × [9.0] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3]
-÷ 093C × 0308 × 093C ÷	#  ÷ [0.2] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3]
-÷ 093C × 094D ÷	#  ÷ [0.2] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) × [9.0] DEVANAGARI SIGN VIRAMA (Extend_ConjunctLinkingScripts_ConjunctLinker_ExtCccZwj) ÷ [0.3]
-÷ 093C × 0308 × 094D ÷	#  ÷ [0.2] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN VIRAMA (Extend_ConjunctLinkingScripts_ConjunctLinker_ExtCccZwj) ÷ [0.3]
-÷ 093C × 200D ÷	#  ÷ [0.2] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) × [9.0] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) ÷ [0.3]
-÷ 093C × 0308 × 200D ÷	#  ÷ [0.2] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) ÷ [0.3]
-÷ 093C ÷ 0378 ÷	#  ÷ [0.2] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [999.0] <reserved-0378> (Other) ÷ [0.3]
-÷ 093C × 0308 ÷ 0378 ÷	#  ÷ [0.2] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] <reserved-0378> (Other) ÷ [0.3]
+÷ 0900 ÷ 0020 ÷	#  ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [999.0] SPACE (Other) ÷ [0.3]
+÷ 0900 × 0308 ÷ 0020 ÷	#  ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] SPACE (Other) ÷ [0.3]
+÷ 0900 ÷ 000D ÷	#  ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [5.0] <CARRIAGE RETURN (CR)> (CR) ÷ [0.3]
+÷ 0900 × 0308 ÷ 000D ÷	#  ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] <CARRIAGE RETURN (CR)> (CR) ÷ [0.3]
+÷ 0900 ÷ 000A ÷	#  ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [5.0] <LINE FEED (LF)> (LF) ÷ [0.3]
+÷ 0900 × 0308 ÷ 000A ÷	#  ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] <LINE FEED (LF)> (LF) ÷ [0.3]
+÷ 0900 ÷ 0001 ÷	#  ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [5.0] <START OF HEADING> (Control) ÷ [0.3]
+÷ 0900 × 0308 ÷ 0001 ÷	#  ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] <START OF HEADING> (Control) ÷ [0.3]
+÷ 0900 × 200C ÷	#  ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) × [9.0] ZERO WIDTH NON-JOINER (Extend) ÷ [0.3]
+÷ 0900 × 0308 × 200C ÷	#  ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] ZERO WIDTH NON-JOINER (Extend) ÷ [0.3]
+÷ 0900 ÷ 1F1E6 ÷	#  ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [0.3]
+÷ 0900 × 0308 ÷ 1F1E6 ÷	#  ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [0.3]
+÷ 0900 ÷ 0600 ÷	#  ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [999.0] ARABIC NUMBER SIGN (Prepend) ÷ [0.3]
+÷ 0900 × 0308 ÷ 0600 ÷	#  ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] ARABIC NUMBER SIGN (Prepend) ÷ [0.3]
+÷ 0900 × 0A03 ÷	#  ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) × [9.1] GURMUKHI SIGN VISARGA (SpacingMark) ÷ [0.3]
+÷ 0900 × 0308 × 0A03 ÷	#  ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.1] GURMUKHI SIGN VISARGA (SpacingMark) ÷ [0.3]
+÷ 0900 ÷ 1100 ÷	#  ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [999.0] HANGUL CHOSEONG KIYEOK (L) ÷ [0.3]
+÷ 0900 × 0308 ÷ 1100 ÷	#  ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL CHOSEONG KIYEOK (L) ÷ [0.3]
+÷ 0900 ÷ 1160 ÷	#  ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [999.0] HANGUL JUNGSEONG FILLER (V) ÷ [0.3]
+÷ 0900 × 0308 ÷ 1160 ÷	#  ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL JUNGSEONG FILLER (V) ÷ [0.3]
+÷ 0900 ÷ 11A8 ÷	#  ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [999.0] HANGUL JONGSEONG KIYEOK (T) ÷ [0.3]
+÷ 0900 × 0308 ÷ 11A8 ÷	#  ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL JONGSEONG KIYEOK (T) ÷ [0.3]
+÷ 0900 ÷ AC00 ÷	#  ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [999.0] HANGUL SYLLABLE GA (LV) ÷ [0.3]
+÷ 0900 × 0308 ÷ AC00 ÷	#  ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL SYLLABLE GA (LV) ÷ [0.3]
+÷ 0900 ÷ AC01 ÷	#  ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [999.0] HANGUL SYLLABLE GAG (LVT) ÷ [0.3]
+÷ 0900 × 0308 ÷ AC01 ÷	#  ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL SYLLABLE GAG (LVT) ÷ [0.3]
+÷ 0900 × 0903 ÷	#  ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) × [9.1] DEVANAGARI SIGN VISARGA (SpacingMark_ConjunctLinkingScripts) ÷ [0.3]
+÷ 0900 × 0308 × 0903 ÷	#  ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.1] DEVANAGARI SIGN VISARGA (SpacingMark_ConjunctLinkingScripts) ÷ [0.3]
+÷ 0900 ÷ 0904 ÷	#  ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [999.0] DEVANAGARI LETTER SHORT A (ConjunctLinkingScripts) ÷ [0.3]
+÷ 0900 × 0308 ÷ 0904 ÷	#  ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] DEVANAGARI LETTER SHORT A (ConjunctLinkingScripts) ÷ [0.3]
+÷ 0900 ÷ 0D4E ÷	#  ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [999.0] MALAYALAM LETTER DOT REPH (Prepend_ConjunctLinkingScripts) ÷ [0.3]
+÷ 0900 × 0308 ÷ 0D4E ÷	#  ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] MALAYALAM LETTER DOT REPH (Prepend_ConjunctLinkingScripts) ÷ [0.3]
+÷ 0900 ÷ 0915 ÷	#  ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [999.0] DEVANAGARI LETTER KA (ConjunctLinkingScripts_LinkingConsonant) ÷ [0.3]
+÷ 0900 × 0308 ÷ 0915 ÷	#  ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] DEVANAGARI LETTER KA (ConjunctLinkingScripts_LinkingConsonant) ÷ [0.3]
+÷ 0900 ÷ 231A ÷	#  ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [999.0] WATCH (ExtPict) ÷ [0.3]
+÷ 0900 × 0308 ÷ 231A ÷	#  ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] WATCH (ExtPict) ÷ [0.3]
+÷ 0900 × 0300 ÷	#  ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) × [9.0] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) ÷ [0.3]
+÷ 0900 × 0308 × 0300 ÷	#  ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) ÷ [0.3]
+÷ 0900 × 0900 ÷	#  ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3]
+÷ 0900 × 0308 × 0900 ÷	#  ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3]
+÷ 0900 × 094D ÷	#  ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) × [9.0] DEVANAGARI SIGN VIRAMA (Extend_ConjunctLinkingScripts_ConjunctLinker_ExtCccZwj) ÷ [0.3]
+÷ 0900 × 0308 × 094D ÷	#  ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN VIRAMA (Extend_ConjunctLinkingScripts_ConjunctLinker_ExtCccZwj) ÷ [0.3]
+÷ 0900 × 200D ÷	#  ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) × [9.0] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) ÷ [0.3]
+÷ 0900 × 0308 × 200D ÷	#  ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) ÷ [0.3]
+÷ 0900 ÷ 0378 ÷	#  ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [999.0] <reserved-0378> (Other) ÷ [0.3]
+÷ 0900 × 0308 ÷ 0378 ÷	#  ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] <reserved-0378> (Other) ÷ [0.3]
 ÷ 094D ÷ 0020 ÷	#  ÷ [0.2] DEVANAGARI SIGN VIRAMA (Extend_ConjunctLinkingScripts_ConjunctLinker_ExtCccZwj) ÷ [999.0] SPACE (Other) ÷ [0.3]
 ÷ 094D × 0308 ÷ 0020 ÷	#  ÷ [0.2] DEVANAGARI SIGN VIRAMA (Extend_ConjunctLinkingScripts_ConjunctLinker_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] SPACE (Other) ÷ [0.3]
 ÷ 094D ÷ 000D ÷	#  ÷ [0.2] DEVANAGARI SIGN VIRAMA (Extend_ConjunctLinkingScripts_ConjunctLinker_ExtCccZwj) ÷ [5.0] <CARRIAGE RETURN (CR)> (CR) ÷ [0.3]
@@ -1038,8 +950,8 @@
 ÷ 094D × 0308 ÷ 000A ÷	#  ÷ [0.2] DEVANAGARI SIGN VIRAMA (Extend_ConjunctLinkingScripts_ConjunctLinker_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] <LINE FEED (LF)> (LF) ÷ [0.3]
 ÷ 094D ÷ 0001 ÷	#  ÷ [0.2] DEVANAGARI SIGN VIRAMA (Extend_ConjunctLinkingScripts_ConjunctLinker_ExtCccZwj) ÷ [5.0] <START OF HEADING> (Control) ÷ [0.3]
 ÷ 094D × 0308 ÷ 0001 ÷	#  ÷ [0.2] DEVANAGARI SIGN VIRAMA (Extend_ConjunctLinkingScripts_ConjunctLinker_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] <START OF HEADING> (Control) ÷ [0.3]
-÷ 094D × 034F ÷	#  ÷ [0.2] DEVANAGARI SIGN VIRAMA (Extend_ConjunctLinkingScripts_ConjunctLinker_ExtCccZwj) × [9.0] COMBINING GRAPHEME JOINER (Extend) ÷ [0.3]
-÷ 094D × 0308 × 034F ÷	#  ÷ [0.2] DEVANAGARI SIGN VIRAMA (Extend_ConjunctLinkingScripts_ConjunctLinker_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] COMBINING GRAPHEME JOINER (Extend) ÷ [0.3]
+÷ 094D × 200C ÷	#  ÷ [0.2] DEVANAGARI SIGN VIRAMA (Extend_ConjunctLinkingScripts_ConjunctLinker_ExtCccZwj) × [9.0] ZERO WIDTH NON-JOINER (Extend) ÷ [0.3]
+÷ 094D × 0308 × 200C ÷	#  ÷ [0.2] DEVANAGARI SIGN VIRAMA (Extend_ConjunctLinkingScripts_ConjunctLinker_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] ZERO WIDTH NON-JOINER (Extend) ÷ [0.3]
 ÷ 094D ÷ 1F1E6 ÷	#  ÷ [0.2] DEVANAGARI SIGN VIRAMA (Extend_ConjunctLinkingScripts_ConjunctLinker_ExtCccZwj) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [0.3]
 ÷ 094D × 0308 ÷ 1F1E6 ÷	#  ÷ [0.2] DEVANAGARI SIGN VIRAMA (Extend_ConjunctLinkingScripts_ConjunctLinker_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [0.3]
 ÷ 094D ÷ 0600 ÷	#  ÷ [0.2] DEVANAGARI SIGN VIRAMA (Extend_ConjunctLinkingScripts_ConjunctLinker_ExtCccZwj) ÷ [999.0] ARABIC NUMBER SIGN (Prepend) ÷ [0.3]
@@ -1056,8 +968,6 @@
 ÷ 094D × 0308 ÷ AC00 ÷	#  ÷ [0.2] DEVANAGARI SIGN VIRAMA (Extend_ConjunctLinkingScripts_ConjunctLinker_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL SYLLABLE GA (LV) ÷ [0.3]
 ÷ 094D ÷ AC01 ÷	#  ÷ [0.2] DEVANAGARI SIGN VIRAMA (Extend_ConjunctLinkingScripts_ConjunctLinker_ExtCccZwj) ÷ [999.0] HANGUL SYLLABLE GAG (LVT) ÷ [0.3]
 ÷ 094D × 0308 ÷ AC01 ÷	#  ÷ [0.2] DEVANAGARI SIGN VIRAMA (Extend_ConjunctLinkingScripts_ConjunctLinker_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL SYLLABLE GAG (LVT) ÷ [0.3]
-÷ 094D × 0900 ÷	#  ÷ [0.2] DEVANAGARI SIGN VIRAMA (Extend_ConjunctLinkingScripts_ConjunctLinker_ExtCccZwj) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) ÷ [0.3]
-÷ 094D × 0308 × 0900 ÷	#  ÷ [0.2] DEVANAGARI SIGN VIRAMA (Extend_ConjunctLinkingScripts_ConjunctLinker_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) ÷ [0.3]
 ÷ 094D × 0903 ÷	#  ÷ [0.2] DEVANAGARI SIGN VIRAMA (Extend_ConjunctLinkingScripts_ConjunctLinker_ExtCccZwj) × [9.1] DEVANAGARI SIGN VISARGA (SpacingMark_ConjunctLinkingScripts) ÷ [0.3]
 ÷ 094D × 0308 × 0903 ÷	#  ÷ [0.2] DEVANAGARI SIGN VIRAMA (Extend_ConjunctLinkingScripts_ConjunctLinker_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.1] DEVANAGARI SIGN VISARGA (SpacingMark_ConjunctLinkingScripts) ÷ [0.3]
 ÷ 094D ÷ 0904 ÷	#  ÷ [0.2] DEVANAGARI SIGN VIRAMA (Extend_ConjunctLinkingScripts_ConjunctLinker_ExtCccZwj) ÷ [999.0] DEVANAGARI LETTER SHORT A (ConjunctLinkingScripts) ÷ [0.3]
@@ -1070,8 +980,8 @@
 ÷ 094D × 0308 ÷ 231A ÷	#  ÷ [0.2] DEVANAGARI SIGN VIRAMA (Extend_ConjunctLinkingScripts_ConjunctLinker_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] WATCH (ExtPict) ÷ [0.3]
 ÷ 094D × 0300 ÷	#  ÷ [0.2] DEVANAGARI SIGN VIRAMA (Extend_ConjunctLinkingScripts_ConjunctLinker_ExtCccZwj) × [9.0] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) ÷ [0.3]
 ÷ 094D × 0308 × 0300 ÷	#  ÷ [0.2] DEVANAGARI SIGN VIRAMA (Extend_ConjunctLinkingScripts_ConjunctLinker_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) ÷ [0.3]
-÷ 094D × 093C ÷	#  ÷ [0.2] DEVANAGARI SIGN VIRAMA (Extend_ConjunctLinkingScripts_ConjunctLinker_ExtCccZwj) × [9.0] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3]
-÷ 094D × 0308 × 093C ÷	#  ÷ [0.2] DEVANAGARI SIGN VIRAMA (Extend_ConjunctLinkingScripts_ConjunctLinker_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3]
+÷ 094D × 0900 ÷	#  ÷ [0.2] DEVANAGARI SIGN VIRAMA (Extend_ConjunctLinkingScripts_ConjunctLinker_ExtCccZwj) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3]
+÷ 094D × 0308 × 0900 ÷	#  ÷ [0.2] DEVANAGARI SIGN VIRAMA (Extend_ConjunctLinkingScripts_ConjunctLinker_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3]
 ÷ 094D × 094D ÷	#  ÷ [0.2] DEVANAGARI SIGN VIRAMA (Extend_ConjunctLinkingScripts_ConjunctLinker_ExtCccZwj) × [9.0] DEVANAGARI SIGN VIRAMA (Extend_ConjunctLinkingScripts_ConjunctLinker_ExtCccZwj) ÷ [0.3]
 ÷ 094D × 0308 × 094D ÷	#  ÷ [0.2] DEVANAGARI SIGN VIRAMA (Extend_ConjunctLinkingScripts_ConjunctLinker_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN VIRAMA (Extend_ConjunctLinkingScripts_ConjunctLinker_ExtCccZwj) ÷ [0.3]
 ÷ 094D × 200D ÷	#  ÷ [0.2] DEVANAGARI SIGN VIRAMA (Extend_ConjunctLinkingScripts_ConjunctLinker_ExtCccZwj) × [9.0] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) ÷ [0.3]
@@ -1086,8 +996,8 @@
 ÷ 200D × 0308 ÷ 000A ÷	#  ÷ [0.2] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] <LINE FEED (LF)> (LF) ÷ [0.3]
 ÷ 200D ÷ 0001 ÷	#  ÷ [0.2] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) ÷ [5.0] <START OF HEADING> (Control) ÷ [0.3]
 ÷ 200D × 0308 ÷ 0001 ÷	#  ÷ [0.2] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] <START OF HEADING> (Control) ÷ [0.3]
-÷ 200D × 034F ÷	#  ÷ [0.2] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) × [9.0] COMBINING GRAPHEME JOINER (Extend) ÷ [0.3]
-÷ 200D × 0308 × 034F ÷	#  ÷ [0.2] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] COMBINING GRAPHEME JOINER (Extend) ÷ [0.3]
+÷ 200D × 200C ÷	#  ÷ [0.2] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) × [9.0] ZERO WIDTH NON-JOINER (Extend) ÷ [0.3]
+÷ 200D × 0308 × 200C ÷	#  ÷ [0.2] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] ZERO WIDTH NON-JOINER (Extend) ÷ [0.3]
 ÷ 200D ÷ 1F1E6 ÷	#  ÷ [0.2] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [0.3]
 ÷ 200D × 0308 ÷ 1F1E6 ÷	#  ÷ [0.2] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [0.3]
 ÷ 200D ÷ 0600 ÷	#  ÷ [0.2] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) ÷ [999.0] ARABIC NUMBER SIGN (Prepend) ÷ [0.3]
@@ -1104,8 +1014,6 @@
 ÷ 200D × 0308 ÷ AC00 ÷	#  ÷ [0.2] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL SYLLABLE GA (LV) ÷ [0.3]
 ÷ 200D ÷ AC01 ÷	#  ÷ [0.2] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) ÷ [999.0] HANGUL SYLLABLE GAG (LVT) ÷ [0.3]
 ÷ 200D × 0308 ÷ AC01 ÷	#  ÷ [0.2] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL SYLLABLE GAG (LVT) ÷ [0.3]
-÷ 200D × 0900 ÷	#  ÷ [0.2] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) ÷ [0.3]
-÷ 200D × 0308 × 0900 ÷	#  ÷ [0.2] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) ÷ [0.3]
 ÷ 200D × 0903 ÷	#  ÷ [0.2] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) × [9.1] DEVANAGARI SIGN VISARGA (SpacingMark_ConjunctLinkingScripts) ÷ [0.3]
 ÷ 200D × 0308 × 0903 ÷	#  ÷ [0.2] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.1] DEVANAGARI SIGN VISARGA (SpacingMark_ConjunctLinkingScripts) ÷ [0.3]
 ÷ 200D ÷ 0904 ÷	#  ÷ [0.2] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) ÷ [999.0] DEVANAGARI LETTER SHORT A (ConjunctLinkingScripts) ÷ [0.3]
@@ -1118,8 +1026,8 @@
 ÷ 200D × 0308 ÷ 231A ÷	#  ÷ [0.2] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] WATCH (ExtPict) ÷ [0.3]
 ÷ 200D × 0300 ÷	#  ÷ [0.2] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) × [9.0] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) ÷ [0.3]
 ÷ 200D × 0308 × 0300 ÷	#  ÷ [0.2] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) ÷ [0.3]
-÷ 200D × 093C ÷	#  ÷ [0.2] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) × [9.0] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3]
-÷ 200D × 0308 × 093C ÷	#  ÷ [0.2] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3]
+÷ 200D × 0900 ÷	#  ÷ [0.2] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3]
+÷ 200D × 0308 × 0900 ÷	#  ÷ [0.2] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3]
 ÷ 200D × 094D ÷	#  ÷ [0.2] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) × [9.0] DEVANAGARI SIGN VIRAMA (Extend_ConjunctLinkingScripts_ConjunctLinker_ExtCccZwj) ÷ [0.3]
 ÷ 200D × 0308 × 094D ÷	#  ÷ [0.2] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN VIRAMA (Extend_ConjunctLinkingScripts_ConjunctLinker_ExtCccZwj) ÷ [0.3]
 ÷ 200D × 200D ÷	#  ÷ [0.2] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) × [9.0] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) ÷ [0.3]
@@ -1134,8 +1042,8 @@
 ÷ 0378 × 0308 ÷ 000A ÷	#  ÷ [0.2] <reserved-0378> (Other) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] <LINE FEED (LF)> (LF) ÷ [0.3]
 ÷ 0378 ÷ 0001 ÷	#  ÷ [0.2] <reserved-0378> (Other) ÷ [5.0] <START OF HEADING> (Control) ÷ [0.3]
 ÷ 0378 × 0308 ÷ 0001 ÷	#  ÷ [0.2] <reserved-0378> (Other) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] <START OF HEADING> (Control) ÷ [0.3]
-÷ 0378 × 034F ÷	#  ÷ [0.2] <reserved-0378> (Other) × [9.0] COMBINING GRAPHEME JOINER (Extend) ÷ [0.3]
-÷ 0378 × 0308 × 034F ÷	#  ÷ [0.2] <reserved-0378> (Other) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] COMBINING GRAPHEME JOINER (Extend) ÷ [0.3]
+÷ 0378 × 200C ÷	#  ÷ [0.2] <reserved-0378> (Other) × [9.0] ZERO WIDTH NON-JOINER (Extend) ÷ [0.3]
+÷ 0378 × 0308 × 200C ÷	#  ÷ [0.2] <reserved-0378> (Other) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] ZERO WIDTH NON-JOINER (Extend) ÷ [0.3]
 ÷ 0378 ÷ 1F1E6 ÷	#  ÷ [0.2] <reserved-0378> (Other) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [0.3]
 ÷ 0378 × 0308 ÷ 1F1E6 ÷	#  ÷ [0.2] <reserved-0378> (Other) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [0.3]
 ÷ 0378 ÷ 0600 ÷	#  ÷ [0.2] <reserved-0378> (Other) ÷ [999.0] ARABIC NUMBER SIGN (Prepend) ÷ [0.3]
@@ -1152,8 +1060,6 @@
 ÷ 0378 × 0308 ÷ AC00 ÷	#  ÷ [0.2] <reserved-0378> (Other) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL SYLLABLE GA (LV) ÷ [0.3]
 ÷ 0378 ÷ AC01 ÷	#  ÷ [0.2] <reserved-0378> (Other) ÷ [999.0] HANGUL SYLLABLE GAG (LVT) ÷ [0.3]
 ÷ 0378 × 0308 ÷ AC01 ÷	#  ÷ [0.2] <reserved-0378> (Other) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL SYLLABLE GAG (LVT) ÷ [0.3]
-÷ 0378 × 0900 ÷	#  ÷ [0.2] <reserved-0378> (Other) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) ÷ [0.3]
-÷ 0378 × 0308 × 0900 ÷	#  ÷ [0.2] <reserved-0378> (Other) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) ÷ [0.3]
 ÷ 0378 × 0903 ÷	#  ÷ [0.2] <reserved-0378> (Other) × [9.1] DEVANAGARI SIGN VISARGA (SpacingMark_ConjunctLinkingScripts) ÷ [0.3]
 ÷ 0378 × 0308 × 0903 ÷	#  ÷ [0.2] <reserved-0378> (Other) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.1] DEVANAGARI SIGN VISARGA (SpacingMark_ConjunctLinkingScripts) ÷ [0.3]
 ÷ 0378 ÷ 0904 ÷	#  ÷ [0.2] <reserved-0378> (Other) ÷ [999.0] DEVANAGARI LETTER SHORT A (ConjunctLinkingScripts) ÷ [0.3]
@@ -1166,8 +1072,8 @@
 ÷ 0378 × 0308 ÷ 231A ÷	#  ÷ [0.2] <reserved-0378> (Other) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] WATCH (ExtPict) ÷ [0.3]
 ÷ 0378 × 0300 ÷	#  ÷ [0.2] <reserved-0378> (Other) × [9.0] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) ÷ [0.3]
 ÷ 0378 × 0308 × 0300 ÷	#  ÷ [0.2] <reserved-0378> (Other) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) ÷ [0.3]
-÷ 0378 × 093C ÷	#  ÷ [0.2] <reserved-0378> (Other) × [9.0] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3]
-÷ 0378 × 0308 × 093C ÷	#  ÷ [0.2] <reserved-0378> (Other) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3]
+÷ 0378 × 0900 ÷	#  ÷ [0.2] <reserved-0378> (Other) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3]
+÷ 0378 × 0308 × 0900 ÷	#  ÷ [0.2] <reserved-0378> (Other) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3]
 ÷ 0378 × 094D ÷	#  ÷ [0.2] <reserved-0378> (Other) × [9.0] DEVANAGARI SIGN VIRAMA (Extend_ConjunctLinkingScripts_ConjunctLinker_ExtCccZwj) ÷ [0.3]
 ÷ 0378 × 0308 × 094D ÷	#  ÷ [0.2] <reserved-0378> (Other) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN VIRAMA (Extend_ConjunctLinkingScripts_ConjunctLinker_ExtCccZwj) ÷ [0.3]
 ÷ 0378 × 200D ÷	#  ÷ [0.2] <reserved-0378> (Other) × [9.0] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) ÷ [0.3]
@@ -1190,10 +1096,10 @@
 ÷ 0061 × 0308 ÷ 0062 ÷	#  ÷ [0.2] LATIN SMALL LETTER A (Other) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] LATIN SMALL LETTER B (Other) ÷ [0.3]
 ÷ 0061 × 0903 ÷ 0062 ÷	#  ÷ [0.2] LATIN SMALL LETTER A (Other) × [9.1] DEVANAGARI SIGN VISARGA (SpacingMark_ConjunctLinkingScripts) ÷ [999.0] LATIN SMALL LETTER B (Other) ÷ [0.3]
 ÷ 0061 ÷ 0600 × 0062 ÷	#  ÷ [0.2] LATIN SMALL LETTER A (Other) ÷ [999.0] ARABIC NUMBER SIGN (Prepend) × [9.2] LATIN SMALL LETTER B (Other) ÷ [0.3]
-÷ 1F476 × 1F3FF ÷ 1F476 ÷	#  ÷ [0.2] BABY (ExtPict) × [9.0] EMOJI MODIFIER FITZPATRICK TYPE-6 (Extend) ÷ [999.0] BABY (ExtPict) ÷ [0.3]
-÷ 0061 × 1F3FF ÷ 1F476 ÷	#  ÷ [0.2] LATIN SMALL LETTER A (Other) × [9.0] EMOJI MODIFIER FITZPATRICK TYPE-6 (Extend) ÷ [999.0] BABY (ExtPict) ÷ [0.3]
-÷ 0061 × 1F3FF ÷ 1F476 × 200D × 1F6D1 ÷	#  ÷ [0.2] LATIN SMALL LETTER A (Other) × [9.0] EMOJI MODIFIER FITZPATRICK TYPE-6 (Extend) ÷ [999.0] BABY (ExtPict) × [9.0] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) × [11.0] OCTAGONAL SIGN (ExtPict) ÷ [0.3]
-÷ 1F476 × 1F3FF × 0308 × 200D × 1F476 × 1F3FF ÷	#  ÷ [0.2] BABY (ExtPict) × [9.0] EMOJI MODIFIER FITZPATRICK TYPE-6 (Extend) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) × [11.0] BABY (ExtPict) × [9.0] EMOJI MODIFIER FITZPATRICK TYPE-6 (Extend) ÷ [0.3]
+÷ 1F476 × 1F3FF ÷ 1F476 ÷	#  ÷ [0.2] BABY (ExtPict) × [9.0] EMOJI MODIFIER FITZPATRICK TYPE-6 (Extend_ExtCccZwj) ÷ [999.0] BABY (ExtPict) ÷ [0.3]
+÷ 0061 × 1F3FF ÷ 1F476 ÷	#  ÷ [0.2] LATIN SMALL LETTER A (Other) × [9.0] EMOJI MODIFIER FITZPATRICK TYPE-6 (Extend_ExtCccZwj) ÷ [999.0] BABY (ExtPict) ÷ [0.3]
+÷ 0061 × 1F3FF ÷ 1F476 × 200D × 1F6D1 ÷	#  ÷ [0.2] LATIN SMALL LETTER A (Other) × [9.0] EMOJI MODIFIER FITZPATRICK TYPE-6 (Extend_ExtCccZwj) ÷ [999.0] BABY (ExtPict) × [9.0] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) × [11.0] OCTAGONAL SIGN (ExtPict) ÷ [0.3]
+÷ 1F476 × 1F3FF × 0308 × 200D × 1F476 × 1F3FF ÷	#  ÷ [0.2] BABY (ExtPict) × [9.0] EMOJI MODIFIER FITZPATRICK TYPE-6 (Extend_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) × [11.0] BABY (ExtPict) × [9.0] EMOJI MODIFIER FITZPATRICK TYPE-6 (Extend_ExtCccZwj) ÷ [0.3]
 ÷ 1F6D1 × 200D × 1F6D1 ÷	#  ÷ [0.2] OCTAGONAL SIGN (ExtPict) × [9.0] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) × [11.0] OCTAGONAL SIGN (ExtPict) ÷ [0.3]
 ÷ 0061 × 200D ÷ 1F6D1 ÷	#  ÷ [0.2] LATIN SMALL LETTER A (Other) × [9.0] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) ÷ [999.0] OCTAGONAL SIGN (ExtPict) ÷ [0.3]
 ÷ 2701 × 200D × 2701 ÷	#  ÷ [0.2] UPPER BLADE SCISSORS (Other) × [9.0] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) × [11.0] UPPER BLADE SCISSORS (Other) ÷ [0.3]
@@ -1210,6 +1116,6 @@
 ÷ 003F × 094D ÷ 0924 ÷	#  ÷ [0.2] QUESTION MARK (Other) × [9.0] DEVANAGARI SIGN VIRAMA (Extend_ConjunctLinkingScripts_ConjunctLinker_ExtCccZwj) ÷ [999.0] DEVANAGARI LETTER TA (ConjunctLinkingScripts_LinkingConsonant) ÷ [0.3]
 ÷ 0915 × 094D × 094D × 0924 ÷	#  ÷ [0.2] DEVANAGARI LETTER KA (ConjunctLinkingScripts_LinkingConsonant) × [9.0] DEVANAGARI SIGN VIRAMA (Extend_ConjunctLinkingScripts_ConjunctLinker_ExtCccZwj) × [9.0] DEVANAGARI SIGN VIRAMA (Extend_ConjunctLinkingScripts_ConjunctLinker_ExtCccZwj) × [9.3] DEVANAGARI LETTER TA (ConjunctLinkingScripts_LinkingConsonant) ÷ [0.3]
 #
-# Lines: 1187
+# Lines: 1093
 #
 # EOF
diff --git a/libcxx/utils/data/unicode/emoji-data.txt b/libcxx/utils/data/unicode/emoji-data.txt
index 0ba10e9ce4c9a..ff99028248b5f 100644
--- a/libcxx/utils/data/unicode/emoji-data.txt
+++ b/libcxx/utils/data/unicode/emoji-data.txt
@@ -1,11 +1,11 @@
 # emoji-data.txt
-# Date: 2023-02-01, 02:22:54 GMT
-# © 2023 Unicode®, Inc.
+# Date: 2024-05-01, 21:25:24 GMT
+# © 2024 Unicode®, Inc.
 # Unicode and the Unicode Logo are registered trademarks of Unicode, Inc. in the U.S. and other countries.
-# For terms of use, see https://www.unicode.org/terms_of_use.html
+# For terms of use and license, see https://www.unicode.org/terms_of_use.html
 #
 # Emoji Data for UTS #51
-# Used with Emoji Version 15.1 and subsequent minor revisions (if any)
+# Used with Emoji Version 16.0 and subsequent minor revisions (if any)
 #
 # For documentation and usage, see https://www.unicode.org/reports/tr51
 #
@@ -407,6 +407,8 @@
 1FA80..1FA82  ; Emoji                # E12.0  [3] (🪀..🪂)    yo-yo..parachute
 1FA83..1FA86  ; Emoji                # E13.0  [4] (🪃..🪆)    boomerang..nesting dolls
 1FA87..1FA88  ; Emoji                # E15.0  [2] (🪇..🪈)    maracas..flute
+1FA89         ; Emoji                # E16.0  [1] (🪉)       harp
+1FA8F         ; Emoji                # E16.0  [1] (🪏)       shovel
 1FA90..1FA95  ; Emoji                # E12.0  [6] (🪐..🪕)    ringed planet..banjo
 1FA96..1FAA8  ; Emoji                # E13.0 [19] (🪖..🪨)    military helmet..rock
 1FAA9..1FAAC  ; Emoji                # E14.0  [4] (🪩..🪬)    mirror ball..hamsa
@@ -414,19 +416,24 @@
 1FAB0..1FAB6  ; Emoji                # E13.0  [7] (🪰..🪶)    fly..feather
 1FAB7..1FABA  ; Emoji                # E14.0  [4] (🪷..🪺)    lotus..nest with eggs
 1FABB..1FABD  ; Emoji                # E15.0  [3] (🪻..🪽)    hyacinth..wing
+1FABE         ; Emoji                # E16.0  [1] (🪾)       leafless tree
 1FABF         ; Emoji                # E15.0  [1] (🪿)       goose
 1FAC0..1FAC2  ; Emoji                # E13.0  [3] (🫀..🫂)    anatomical heart..people hugging
 1FAC3..1FAC5  ; Emoji                # E14.0  [3] (🫃..🫅)    pregnant man..person with crown
+1FAC6         ; Emoji                # E16.0  [1] (🫆)       fingerprint
 1FACE..1FACF  ; Emoji                # E15.0  [2] (🫎..🫏)    moose..donkey
 1FAD0..1FAD6  ; Emoji                # E13.0  [7] (🫐..🫖)    blueberries..teapot
 1FAD7..1FAD9  ; Emoji                # E14.0  [3] (🫗..🫙)    pouring liquid..jar
 1FADA..1FADB  ; Emoji                # E15.0  [2] (🫚..🫛)    ginger root..pea pod
+1FADC         ; Emoji                # E16.0  [1] (🫜)       root vegetable
+1FADF         ; Emoji                # E16.0  [1] (🫟)       splatter
 1FAE0..1FAE7  ; Emoji                # E14.0  [8] (🫠..🫧)    melting face..bubbles
 1FAE8         ; Emoji                # E15.0  [1] (🫨)       shaking face
+1FAE9         ; Emoji                # E16.0  [1] (🫩)       face with bags under eyes
 1FAF0..1FAF6  ; Emoji                # E14.0  [7] (🫰..🫶)    hand with index finger and thumb crossed..heart hands
 1FAF7..1FAF8  ; Emoji                # E15.0  [2] (🫷..🫸)    leftwards pushing hand..rightwards pushing hand
 
-# Total elements: 1424
+# Total elements: 1431
 
 # ================================================
 
@@ -696,6 +703,8 @@
 1FA80..1FA82  ; Emoji_Presentation   # E12.0  [3] (🪀..🪂)    yo-yo..parachute
 1FA83..1FA86  ; Emoji_Presentation   # E13.0  [4] (🪃..🪆)    boomerang..nesting dolls
 1FA87..1FA88  ; Emoji_Presentation   # E15.0  [2] (🪇..🪈)    maracas..flute
+1FA89         ; Emoji_Presentation   # E16.0  [1] (🪉)       harp
+1FA8F         ; Emoji_Presentation   # E16.0  [1] (🪏)       shovel
 1FA90..1FA95  ; Emoji_Presentation   # E12.0  [6] (🪐..🪕)    ringed planet..banjo
 1FA96..1FAA8  ; Emoji_Presentation   # E13.0 [19] (🪖..🪨)    military helmet..rock
 1FAA9..1FAAC  ; Emoji_Presentation   # E14.0  [4] (🪩..🪬)    mirror ball..hamsa
@@ -703,19 +712,24 @@
 1FAB0..1FAB6  ; Emoji_Presentation   # E13.0  [7] (🪰..🪶)    fly..feather
 1FAB7..1FABA  ; Emoji_Presentation   # E14.0  [4] (🪷..🪺)    lotus..nest with eggs
 1FABB..1FABD  ; Emoji_Presentation   # E15.0  [3] (🪻..🪽)    hyacinth..wing
+1FABE         ; Emoji_Presentation   # E16.0  [1] (🪾)       leafless tree
 1FABF         ; Emoji_Presentation   # E15.0  [1] (🪿)       goose
 1FAC0..1FAC2  ; Emoji_Presentation   # E13.0  [3] (🫀..🫂)    anatomical heart..people hugging
 1FAC3..1FAC5  ; Emoji_Presentation   # E14.0  [3] (🫃..🫅)    pregnant man..person with crown
+1FAC6         ; Emoji_Presentation   # E16.0  [1] (🫆)       fingerprint
 1FACE..1FACF  ; Emoji_Presentation   # E15.0  [2] (🫎..🫏)    moose..donkey
 1FAD0..1FAD6  ; Emoji_Presentation   # E13.0  [7] (🫐..🫖)    blueberries..teapot
 1FAD7..1FAD9  ; Emoji_Presentation   # E14.0  [3] (🫗..🫙)    pouring liquid..jar
 1FADA..1FADB  ; Emoji_Presentation   # E15.0  [2] (🫚..🫛)    ginger root..pea pod
+1FADC         ; Emoji_Presentation   # E16.0  [1] (🫜)       root vegetable
+1FADF         ; Emoji_Presentation   # E16.0  [1] (🫟)       splatter
 1FAE0..1FAE7  ; Emoji_Presentation   # E14.0  [8] (🫠..🫧)    melting face..bubbles
 1FAE8         ; Emoji_Presentation   # E15.0  [1] (🫨)       shaking face
+1FAE9         ; Emoji_Presentation   # E16.0  [1] (🫩)       face with bags under eyes
 1FAF0..1FAF6  ; Emoji_Presentation   # E14.0  [7] (🫰..🫶)    hand with index finger and thumb crossed..heart hands
 1FAF7..1FAF8  ; Emoji_Presentation   # E15.0  [2] (🫷..🫸)    leftwards pushing hand..rightwards pushing hand
 
-# Total elements: 1205
+# Total elements: 1212
 
 # ================================================
 
@@ -1289,7 +1303,9 @@ E0020..E007F  ; Emoji_Component      # E0.0  [96] (󠀠..󠁿)      tag space..c
 1FA80..1FA82  ; Extended_Pictographic# E12.0  [3] (🪀..🪂)    yo-yo..parachute
 1FA83..1FA86  ; Extended_Pictographic# E13.0  [4] (🪃..🪆)    boomerang..nesting dolls
 1FA87..1FA88  ; Extended_Pictographic# E15.0  [2] (🪇..🪈)    maracas..flute
-1FA89..1FA8F  ; Extended_Pictographic# E0.0   [7] (🪉..🪏)    <reserved-1FA89>..<reserved-1FA8F>
+1FA89         ; Extended_Pictographic# E16.0  [1] (🪉)       harp
+1FA8A..1FA8E  ; Extended_Pictographic# E0.0   [5] (🪊..🪎)    <reserved-1FA8A>..<reserved-1FA8E>
+1FA8F         ; Extended_Pictographic# E16.0  [1] (🪏)       shovel
 1FA90..1FA95  ; Extended_Pictographic# E12.0  [6] (🪐..🪕)    ringed planet..banjo
 1FA96..1FAA8  ; Extended_Pictographic# E13.0 [19] (🪖..🪨)    military helmet..rock
 1FAA9..1FAAC  ; Extended_Pictographic# E14.0  [4] (🪩..🪬)    mirror ball..hamsa
@@ -1297,19 +1313,23 @@ E0020..E007F  ; Emoji_Component      # E0.0  [96] (󠀠..󠁿)      tag space..c
 1FAB0..1FAB6  ; Extended_Pictographic# E13.0  [7] (🪰..🪶)    fly..feather
 1FAB7..1FABA  ; Extended_Pictographic# E14.0  [4] (🪷..🪺)    lotus..nest with eggs
 1FABB..1FABD  ; Extended_Pictographic# E15.0  [3] (🪻..🪽)    hyacinth..wing
-1FABE         ; Extended_Pictographic# E0.0   [1] (🪾)       <reserved-1FABE>
+1FABE         ; Extended_Pictographic# E16.0  [1] (🪾)       leafless tree
 1FABF         ; Extended_Pictographic# E15.0  [1] (🪿)       goose
 1FAC0..1FAC2  ; Extended_Pictographic# E13.0  [3] (🫀..🫂)    anatomical heart..people hugging
 1FAC3..1FAC5  ; Extended_Pictographic# E14.0  [3] (🫃..🫅)    pregnant man..person with crown
-1FAC6..1FACD  ; Extended_Pictographic# E0.0   [8] (🫆..🫍)    <reserved-1FAC6>..<reserved-1FACD>
+1FAC6         ; Extended_Pictographic# E16.0  [1] (🫆)       fingerprint
+1FAC7..1FACD  ; Extended_Pictographic# E0.0   [7] (🫇..🫍)    <reserved-1FAC7>..<reserved-1FACD>
 1FACE..1FACF  ; Extended_Pictographic# E15.0  [2] (🫎..🫏)    moose..donkey
 1FAD0..1FAD6  ; Extended_Pictographic# E13.0  [7] (🫐..🫖)    blueberries..teapot
 1FAD7..1FAD9  ; Extended_Pictographic# E14.0  [3] (🫗..🫙)    pouring liquid..jar
 1FADA..1FADB  ; Extended_Pictographic# E15.0  [2] (🫚..🫛)    ginger root..pea pod
-1FADC..1FADF  ; Extended_Pictographic# E0.0   [4] (🫜..🫟)    <reserved-1FADC>..<reserved-1FADF>
+1FADC         ; Extended_Pictographic# E16.0  [1] (🫜)       root vegetable
+1FADD..1FADE  ; Extended_Pictographic# E0.0   [2] (🫝..🫞)    <reserved-1FADD>..<reserved-1FADE>
+1FADF         ; Extended_Pictographic# E16.0  [1] (🫟)       splatter
 1FAE0..1FAE7  ; Extended_Pictographic# E14.0  [8] (🫠..🫧)    melting face..bubbles
 1FAE8         ; Extended_Pictographic# E15.0  [1] (🫨)       shaking face
-1FAE9..1FAEF  ; Extended_Pictographic# E0.0   [7] (🫩..🫯)    <reserved-1FAE9>..<reserved-1FAEF>
+1FAE9         ; Extended_Pictographic# E16.0  [1] (🫩)       face with bags under eyes
+1FAEA..1FAEF  ; Extended_Pictographic# E0.0   [6] (🫪..🫯)    <reserved-1FAEA>..<reserved-1FAEF>
 1FAF0..1FAF6  ; Extended_Pictographic# E14.0  [7] (🫰..🫶)    hand with index finger and thumb crossed..heart hands
 1FAF7..1FAF8  ; Extended_Pictographic# E15.0  [2] (🫷..🫸)    leftwards pushing hand..rightwards pushing hand
 1FAF9..1FAFF  ; Extended_Pictographic# E0.0   [7] (🫹..🫿)    <reserved-1FAF9>..<reserved-1FAFF>
diff --git a/libcxx/utils/generate_feature_test_macro_components.py b/libcxx/utils/generate_feature_test_macro_components.py
index c4065cdc1afef..d0f9d279cd0ea 100755
--- a/libcxx/utils/generate_feature_test_macro_components.py
+++ b/libcxx/utils/generate_feature_test_macro_components.py
@@ -2,8 +2,15 @@
 
 import os
 from builtins import range
+from dataclasses import dataclass
 from functools import reduce
-from typing import Any, Dict, List  # Needed for python 3.8 compatibility.
+from typing import (
+    Any,
+    Dict,
+    List,  # Needed for python 3.8 compatibility.
+    NewType,
+    Optional,
+)
 import functools
 import json
 
@@ -505,6 +512,17 @@ def add_version_header(tc):
             "test_suite_guard": "!defined(_LIBCPP_VERSION) || (_LIBCPP_HAS_FILESYSTEM && _LIBCPP_AVAILABILITY_HAS_FILESYSTEM_LIBRARY)",
             "libcxx_guard": "_LIBCPP_HAS_FILESYSTEM && _LIBCPP_AVAILABILITY_HAS_FILESYSTEM_LIBRARY",
         },
+        {
+            "name": "__cpp_lib_flat_map",
+            "values": {"c++23": 202207},
+            "headers": ["flat_map"],
+        },
+        {
+            "name": "__cpp_lib_flat_set",
+            "values": {"c++23": 202207},
+            "headers": ["flat_set"],
+            "unimplemented": True,
+        },
         {
             "name": "__cpp_lib_format",
             "values": {
@@ -1933,9 +1951,28 @@ def produce_docs():
         f.write(doc_str)
 
 
+Std = NewType("Std", str)  # Standard version number
+Ftm = NewType("Ftm", str)  # The name of a feature test macro
+Value = NewType("Value", str)  # The value of a feature test macro including the L suffix
+
+@dataclass
+class Metadata:
+    headers: List[str] = None
+    test_suite_guard: str = None
+    libcxx_guard: str = None
+
+
+@dataclass
+class VersionHeader:
+    value: Value = None
+    implemented: bool = None
+    need_undef: bool = None
+    condition: str = None
+
+
 def get_ftms(
-    data, std_dialects: List[str], use_implemented_status: bool
-) -> Dict[str, Dict[str, Any]]:
+    data, std_dialects: List[Std], use_implemented_status: bool
+) -> Dict[Ftm, Dict[Std, Optional[Value]]]:
     """Impementation for FeatureTestMacros.(standard|implemented)_ftms()."""
     result = dict()
     for feature in data:
@@ -1972,7 +2009,7 @@ def get_ftms(
     return result
 
 
-def generate_version_header_dialect_block(data: Dict[str, Any]) -> str:
+def generate_version_header_dialect_block(data: Dict[Ftm, VersionHeader]) -> str:
     """Generates the contents of the version header for a dialect.
 
     This generates the contents of a
@@ -1983,27 +2020,29 @@ def generate_version_header_dialect_block(data: Dict[str, Any]) -> str:
     result = ""
     for element in data:
         for ftm, entry in element.items():
-            if not entry["implemented"]:
+            if not entry.implemented:
                 # When a FTM is not implemented don't add the guards
                 # or undefine the (possibly) defined macro.
-                result += f'// define {ftm} {entry["value"]}\n'
+                result += f"// define {ftm} {entry.value}\n"
             else:
-                need_undef = entry["need_undef"]
-                if entry["condition"]:
-                    result += f'#  if {entry["condition"]}\n'
-                    if entry["need_undef"]:
+                need_undef = entry.need_undef
+                if entry.condition:
+                    result += f"#  if {entry.condition}\n"
+                    if entry.need_undef:
                         result += f"#    undef {ftm}\n"
-                    result += f'#    define {ftm} {entry["value"]}\n'
+                    result += f"#    define {ftm} {entry.value}\n"
                     result += f"#  endif\n"
                 else:
-                    if entry["need_undef"]:
+                    if entry.need_undef:
                         result += f"#  undef {ftm}\n"
-                    result += f'#  define {ftm} {entry["value"]}\n'
+                    result += f"#  define {ftm} {entry.value}\n"
 
     return result
 
 
-def generate_version_header_implementation(data: Dict[str, Dict[str, Any]]) -> str:
+def generate_version_header_implementation(
+    data: Dict[Std, Dict[Ftm, VersionHeader]]
+) -> str:
     """Generates the body of the version header."""
 
     template = """#if _LIBCPP_STD_VER >= {dialect}
@@ -2121,7 +2160,7 @@ def __init__(self, filename: str):
             self.__data = json.load(f)
 
     @functools.cached_property
-    def std_dialects(self) -> List[str]:
+    def std_dialects(self) -> List[Std]:
         """Returns the C++ dialects avaiable.
 
         The available dialects are based on the 'c++xy' keys found the 'values'
@@ -2140,63 +2179,44 @@ def std_dialects(self) -> List[str]:
         return sorted(list(dialects))
 
     @functools.cached_property
-    def standard_ftms(self) -> Dict[str, Dict[str, Any]]:
+    def standard_ftms(self) -> Dict[Ftm, Dict[Std, Optional[Value]]]:
         """Returns the FTM versions per dialect in the Standard.
 
         This function does not use the 'implemented' flag. The output contains
         the versions used in the Standard. When a FTM in libc++ is not
         implemented according to the Standard to output may opt to show the
         expected value.
-
-        The result is a dict with the following content
-        - key: Name of the feature test macro.
-        - value: A dict with the following content:
-          * key: The version of the C++ dialect.
-          * value: The value of the feature-test macro.
         """
         return get_ftms(self.__data, self.std_dialects, False)
 
     @functools.cached_property
-    def implemented_ftms(self) -> Dict[str, Dict[str, Any]]:
+    def implemented_ftms(self) -> Dict[Ftm, Dict[Std, Optional[Value]]]:
         """Returns the FTM versions per dialect implemented in libc++.
 
         Unlike `get_std_dialect_versions` this function uses the 'implemented'
         flag. This returns the actual implementation status in libc++.
-
-        The result is a dict with the following content
-        - key: Name of the feature test macro.
-        - value: A dict with the following content:
-          * key: The version of the C++ dialect.
-          * value: The value of the feature-test macro. When a feature-test
-            macro is not implemented its value is None.
         """
 
         return get_ftms(self.__data, self.std_dialects, True)
 
     @functools.cached_property
-    def ftm_metadata(self) -> Dict[str, Dict[str, Any]]:
+    def ftm_metadata(self) -> Dict[Ftm, Metadata]:
         """Returns the metadata of the FTMs defined in the Standard.
 
         The metadata does not depend on the C++ dialect used.
-        The result is a dict with the following contents:
-        - key: Name of the feature test macro.
-        - value: A dict with the following content:
-          * headers: The list of headers that should provide the FTM
-          * test_suite_guard: The condition for testing the FTM in the test suite.
-          * test_suite_guard: The condition for testing the FTM in the version header.
         """
         result = dict()
         for feature in self.__data:
-            entry = dict()
-            entry["headers"] = feature["headers"]
-            entry["test_suite_guard"] = feature.get("test_suite_guard", None)
-            entry["libcxx_guard"] = feature.get("libcxx_guard", None)
-            result[feature["name"]] = entry
+            result[feature["name"]] = Metadata(
+                feature["headers"],
+                feature.get("test_suite_guard", None),
+                feature.get("libcxx_guard", None),
+            )
 
         return result
 
     @property
-    def version_header_implementation(self) -> Dict[str, List[Dict[str, Any]]]:
+    def version_header_implementation(self) -> Dict[Std, Dict[Ftm, VersionHeader]]:
         """Generates the body of the version header."""
         result = dict()
         for std in self.std_dialects:
@@ -2212,11 +2232,13 @@ def version_header_implementation(self) -> Dict[str, List[Dict[str, Any]]]:
                     continue
                 last_value = value
 
-                entry = dict()
-                entry["value"] = value
-                entry["implemented"] = self.implemented_ftms[ftm][std] == self.standard_ftms[ftm][std]
-                entry["need_undef"] = last_entry is not None and last_entry["implemented"] and entry["implemented"]
-                entry["condition"] = self.ftm_metadata[ftm]["libcxx_guard"]
+                implemented = self.implemented_ftms[ftm][std] == self.standard_ftms[ftm][std]
+                entry = VersionHeader(
+                    value,
+                    implemented,
+                    last_entry is not None and last_entry.implemented and implemented,
+                    self.ftm_metadata[ftm].libcxx_guard,
+                )
 
                 last_entry = entry
                 result[get_std_number(std)].append(dict({ftm: entry}))
diff --git a/libcxx/utils/libcxx/test/params.py b/libcxx/utils/libcxx/test/params.py
index 947cfd2651364..7dba39bc5db04 100644
--- a/libcxx/utils/libcxx/test/params.py
+++ b/libcxx/utils/libcxx/test/params.py
@@ -31,6 +31,7 @@
     "-Wno-reserved-module-identifier",
     '-Wdeprecated-copy',
     '-Wdeprecated-copy-dtor',
+    "-Wshift-negative-value",
     # GCC warns about places where we might want to add sized allocation/deallocation
     # functions, but we know better what we're doing/testing in the test suite.
     "-Wno-sized-deallocation",
@@ -186,7 +187,8 @@ def getSuitableClangTidy(cfg):
             AddFeature(std),
             AddSubstitution("%{cxx_std}", re.sub(r"\+", "x", std)),
             AddCompileFlag(lambda cfg: getStdFlag(cfg, std)),
-        ],
+        ]
+        + [AddFeature(f"std-at-least-{s}") for s in _allStandards if s <= std],
     ),
     Parameter(
         name="optimization",
diff --git a/libcxxabi/src/cxa_exception_storage.cpp b/libcxxabi/src/cxa_exception_storage.cpp
index 733f0d470569a..05da6a48e313c 100644
--- a/libcxxabi/src/cxa_exception_storage.cpp
+++ b/libcxxabi/src/cxa_exception_storage.cpp
@@ -56,7 +56,7 @@ extern "C" {
 namespace __cxxabiv1 {
 namespace {
     std::__libcpp_tls_key key_;
-    std::__libcpp_exec_once_flag flag_ = _LIBCPP_EXEC_ONCE_INITIALIZER;
+    constinit std::__libcpp_exec_once_flag flag_ = _LIBCPP_EXEC_ONCE_INITIALIZER;
 
     void _LIBCPP_TLS_DESTRUCTOR_CC destruct_(void *p) {
         __free_with_fallback(p);
diff --git a/libcxxabi/src/stdlib_new_delete.cpp b/libcxxabi/src/stdlib_new_delete.cpp
index 73798e211c313..f386b28f0cfe6 100644
--- a/libcxxabi/src/stdlib_new_delete.cpp
+++ b/libcxxabi/src/stdlib_new_delete.cpp
@@ -63,7 +63,7 @@ static void* operator_new_impl(std::size_t size) {
   return p;
 }
 
-_LIBCPP_OVERRIDABLE_FUNCTION(_Znwm, void*, operator new, (std::size_t size)) _THROW_BAD_ALLOC {
+_LIBCPP_MAKE_OVERRIDABLE_FUNCTION_DETECTABLE _LIBCPP_WEAK void* operator new(std::size_t size) _THROW_BAD_ALLOC {
   void* p = operator_new_impl(size);
   if (p == nullptr)
     __throw_bad_alloc_shim();
@@ -74,7 +74,7 @@ _LIBCPP_WEAK void* operator new(size_t size, const std::nothrow_t&) noexcept {
 #if !_LIBCPP_HAS_EXCEPTIONS
 #  if _LIBCPP_CAN_DETECT_OVERRIDDEN_FUNCTION
   _LIBCPP_ASSERT_SHIM(
-      !std::__is_function_overridden<static_cast<void* (*)(std::size_t)>(&operator new)>(),
+      !std::__is_function_overridden(static_cast<void* (*)(std::size_t)>(&operator new)),
       "libc++ was configured with exceptions disabled and `operator new(size_t)` has been overridden, "
       "but `operator new(size_t, nothrow_t)` has not been overridden. This is problematic because "
       "`operator new(size_t, nothrow_t)` must call `operator new(size_t)`, which will terminate in case "
@@ -94,7 +94,7 @@ _LIBCPP_WEAK void* operator new(size_t size, const std::nothrow_t&) noexcept {
 #endif
 }
 
-_LIBCPP_OVERRIDABLE_FUNCTION(_Znam, void*, operator new[], (size_t size)) _THROW_BAD_ALLOC {
+_LIBCPP_MAKE_OVERRIDABLE_FUNCTION_DETECTABLE _LIBCPP_WEAK void* operator new[](size_t size) _THROW_BAD_ALLOC {
   return ::operator new(size);
 }
 
@@ -102,7 +102,7 @@ _LIBCPP_WEAK void* operator new[](size_t size, const std::nothrow_t&) noexcept {
 #if !_LIBCPP_HAS_EXCEPTIONS
 #  if _LIBCPP_CAN_DETECT_OVERRIDDEN_FUNCTION
   _LIBCPP_ASSERT_SHIM(
-      !std::__is_function_overridden<static_cast<void* (*)(std::size_t)>(&operator new[])>(),
+      !std::__is_function_overridden(static_cast<void* (*)(std::size_t)>(&operator new[])),
       "libc++ was configured with exceptions disabled and `operator new[](size_t)` has been overridden, "
       "but `operator new[](size_t, nothrow_t)` has not been overridden. This is problematic because "
       "`operator new[](size_t, nothrow_t)` must call `operator new[](size_t)`, which will terminate in case "
@@ -156,8 +156,8 @@ static void* operator_new_aligned_impl(std::size_t size, std::align_val_t alignm
   return p;
 }
 
-_LIBCPP_OVERRIDABLE_FUNCTION(_ZnwmSt11align_val_t, void*, operator new, (std::size_t size, std::align_val_t alignment))
-_THROW_BAD_ALLOC {
+_LIBCPP_MAKE_OVERRIDABLE_FUNCTION_DETECTABLE _LIBCPP_WEAK void*
+operator new(std::size_t size, std::align_val_t alignment) _THROW_BAD_ALLOC {
   void* p = operator_new_aligned_impl(size, alignment);
   if (p == nullptr)
     __throw_bad_alloc_shim();
@@ -168,7 +168,7 @@ _LIBCPP_WEAK void* operator new(size_t size, std::align_val_t alignment, const s
 #  if !_LIBCPP_HAS_EXCEPTIONS
 #    if _LIBCPP_CAN_DETECT_OVERRIDDEN_FUNCTION
   _LIBCPP_ASSERT_SHIM(
-      !std::__is_function_overridden<static_cast<void* (*)(std::size_t, std::align_val_t)>(&operator new)>(),
+      !std::__is_function_overridden(static_cast<void* (*)(std::size_t, std::align_val_t)>(&operator new)),
       "libc++ was configured with exceptions disabled and `operator new(size_t, align_val_t)` has been overridden, "
       "but `operator new(size_t, align_val_t, nothrow_t)` has not been overridden. This is problematic because "
       "`operator new(size_t, align_val_t, nothrow_t)` must call `operator new(size_t, align_val_t)`, which will "
@@ -188,14 +188,16 @@ _LIBCPP_WEAK void* operator new(size_t size, std::align_val_t alignment, const s
 #  endif
 }
 
-_LIBCPP_OVERRIDABLE_FUNCTION(_ZnamSt11align_val_t, void*, operator new[], (size_t size, std::align_val_t alignment))
-_THROW_BAD_ALLOC { return ::operator new(size, alignment); }
+_LIBCPP_MAKE_OVERRIDABLE_FUNCTION_DETECTABLE _LIBCPP_WEAK void*
+operator new[](size_t size, std::align_val_t alignment) _THROW_BAD_ALLOC {
+  return ::operator new(size, alignment);
+}
 
 _LIBCPP_WEAK void* operator new[](size_t size, std::align_val_t alignment, const std::nothrow_t&) noexcept {
 #  if !_LIBCPP_HAS_EXCEPTIONS
 #    if _LIBCPP_CAN_DETECT_OVERRIDDEN_FUNCTION
   _LIBCPP_ASSERT_SHIM(
-      !std::__is_function_overridden<static_cast<void* (*)(std::size_t, std::align_val_t)>(&operator new[])>(),
+      !std::__is_function_overridden(static_cast<void* (*)(std::size_t, std::align_val_t)>(&operator new[])),
       "libc++ was configured with exceptions disabled and `operator new[](size_t, align_val_t)` has been overridden, "
       "but `operator new[](size_t, align_val_t, nothrow_t)` has not been overridden. This is problematic because "
       "`operator new[](size_t, align_val_t, nothrow_t)` must call `operator new[](size_t, align_val_t)`, which will "
diff --git a/libunwind/src/UnwindCursor.hpp b/libunwind/src/UnwindCursor.hpp
index 3831d8e071ef3..0923052b1b588 100644
--- a/libunwind/src/UnwindCursor.hpp
+++ b/libunwind/src/UnwindCursor.hpp
@@ -31,8 +31,9 @@
 #endif
 
 #if defined(_LIBUNWIND_TARGET_LINUX) &&                                        \
-    (defined(_LIBUNWIND_TARGET_AARCH64) || defined(_LIBUNWIND_TARGET_RISCV) || \
-     defined(_LIBUNWIND_TARGET_S390X))
+    (defined(_LIBUNWIND_TARGET_AARCH64) ||                                     \
+     defined(_LIBUNWIND_TARGET_LOONGARCH) ||                                   \
+     defined(_LIBUNWIND_TARGET_RISCV) || defined(_LIBUNWIND_TARGET_S390X))
 #include <errno.h>
 #include <signal.h>
 #include <sys/syscall.h>
@@ -996,6 +997,10 @@ class UnwindCursor : public AbstractUnwindCursor{
   bool setInfoForSigReturn(Registers_arm64 &);
   int stepThroughSigReturn(Registers_arm64 &);
 #endif
+#if defined(_LIBUNWIND_TARGET_LOONGARCH)
+  bool setInfoForSigReturn(Registers_loongarch &);
+  int stepThroughSigReturn(Registers_loongarch &);
+#endif
 #if defined(_LIBUNWIND_TARGET_RISCV)
   bool setInfoForSigReturn(Registers_riscv &);
   int stepThroughSigReturn(Registers_riscv &);
@@ -2815,6 +2820,61 @@ int UnwindCursor<A, R>::stepThroughSigReturn() {
 #endif // defined(_LIBUNWIND_CHECK_LINUX_SIGRETURN) &&
        // defined(_LIBUNWIND_TARGET_AARCH64)
 
+#if defined(_LIBUNWIND_CHECK_LINUX_SIGRETURN) &&                               \
+    defined(_LIBUNWIND_TARGET_LOONGARCH)
+template <typename A, typename R>
+bool UnwindCursor<A, R>::setInfoForSigReturn(Registers_loongarch &) {
+  const pint_t pc = static_cast<pint_t>(getReg(UNW_REG_IP));
+  // The PC might contain an invalid address if the unwind info is bad, so
+  // directly accessing it could cause a SIGSEGV.
+  if (!isReadableAddr(pc))
+    return false;
+  const auto *instructions = reinterpret_cast<const uint32_t *>(pc);
+  // Look for the two instructions used in the sigreturn trampoline
+  // __vdso_rt_sigreturn:
+  //
+  // 0x03822c0b li a7,0x8b
+  // 0x002b0000 syscall 0
+  if (instructions[0] != 0x03822c0b || instructions[1] != 0x002b0000)
+    return false;
+
+  _info = {};
+  _info.start_ip = pc;
+  _info.end_ip = pc + 4;
+  _isSigReturn = true;
+  return true;
+}
+
+template <typename A, typename R>
+int UnwindCursor<A, R>::stepThroughSigReturn(Registers_loongarch &) {
+  // In the signal trampoline frame, sp points to an rt_sigframe[1], which is:
+  //  - 128-byte siginfo struct
+  //  - ucontext_t struct:
+  //     - 8-byte long (__uc_flags)
+  //     - 8-byte pointer (*uc_link)
+  //     - 24-byte uc_stack
+  //     - 8-byte uc_sigmask
+  //     - 120-byte of padding to allow sigset_t to be expanded in the future
+  //     - 8 bytes of padding because sigcontext has 16-byte alignment
+  //     - struct sigcontext uc_mcontext
+  // [1]
+  // https://github.com/torvalds/linux/blob/master/arch/loongarch/kernel/signal.c
+  const pint_t kOffsetSpToSigcontext = 128 + 8 + 8 + 24 + 8 + 128;
+
+  const pint_t sigctx = _registers.getSP() + kOffsetSpToSigcontext;
+  _registers.setIP(_addressSpace.get64(sigctx));
+  for (int i = UNW_LOONGARCH_R1; i <= UNW_LOONGARCH_R31; ++i) {
+    // skip R0
+    uint64_t value =
+        _addressSpace.get64(sigctx + static_cast<pint_t>((i + 1) * 8));
+    _registers.setRegister(i, value);
+  }
+  _isSignalFrame = true;
+  return UNW_STEP_SUCCESS;
+}
+#endif // defined(_LIBUNWIND_CHECK_LINUX_SIGRETURN) &&
+       // defined(_LIBUNWIND_TARGET_LOONGARCH)
+
 #if defined(_LIBUNWIND_CHECK_LINUX_SIGRETURN) &&                               \
     defined(_LIBUNWIND_TARGET_RISCV)
 template <typename A, typename R>
diff --git a/libunwind/test/signal_unwind.pass.cpp b/libunwind/test/signal_unwind.pass.cpp
index 1c1566415a4d4..4de271ecb886b 100644
--- a/libunwind/test/signal_unwind.pass.cpp
+++ b/libunwind/test/signal_unwind.pass.cpp
@@ -8,7 +8,7 @@
 //===----------------------------------------------------------------------===//
 
 // Ensure that the unwinder can cope with the signal handler.
-// REQUIRES: target={{(aarch64|riscv64|s390x|x86_64)-.+linux.*}}
+// REQUIRES: target={{(aarch64|loongarch64|riscv64|s390x|x86_64)-.+linux.*}}
 
 // TODO: Figure out why this fails with Memory Sanitizer.
 // XFAIL: msan
diff --git a/libunwind/test/unwind_leaffunction.pass.cpp b/libunwind/test/unwind_leaffunction.pass.cpp
index 98de7dc43260c..d336c159c131b 100644
--- a/libunwind/test/unwind_leaffunction.pass.cpp
+++ b/libunwind/test/unwind_leaffunction.pass.cpp
@@ -8,7 +8,7 @@
 //===----------------------------------------------------------------------===//
 
 // Ensure that leaf function can be unwund.
-// REQUIRES: target={{(aarch64|riscv64|s390x|x86_64)-.+linux.*}}
+// REQUIRES: target={{(aarch64|loongarch64|riscv64|s390x|x86_64)-.+linux.*}}
 
 // TODO: Figure out why this fails with Memory Sanitizer.
 // XFAIL: msan
diff --git a/lld/COFF/Chunks.cpp b/lld/COFF/Chunks.cpp
index 11e7cf4346b23..a01c69c709876 100644
--- a/lld/COFF/Chunks.cpp
+++ b/lld/COFF/Chunks.cpp
@@ -1172,11 +1172,12 @@ uint64_t Arm64XRelocVal::get() const {
 
 size_t Arm64XDynamicRelocEntry::getSize() const {
   switch (type) {
+  case IMAGE_DVRT_ARM64X_FIXUP_TYPE_ZEROFILL:
+    return sizeof(uint16_t); // Just a header.
   case IMAGE_DVRT_ARM64X_FIXUP_TYPE_VALUE:
     return sizeof(uint16_t) + size; // A header and a payload.
   case IMAGE_DVRT_ARM64X_FIXUP_TYPE_DELTA:
-  case IMAGE_DVRT_ARM64X_FIXUP_TYPE_ZEROFILL:
-    llvm_unreachable("unsupported type");
+    return 2 * sizeof(uint16_t); // A header and a delta.
   }
   llvm_unreachable("invalid type");
 }
@@ -1186,6 +1187,9 @@ void Arm64XDynamicRelocEntry::writeTo(uint8_t *buf) const {
   *out = (offset.get() & 0xfff) | (type << 12);
 
   switch (type) {
+  case IMAGE_DVRT_ARM64X_FIXUP_TYPE_ZEROFILL:
+    *out |= ((bit_width(size) - 1) << 14); // Encode the size.
+    break;
   case IMAGE_DVRT_ARM64X_FIXUP_TYPE_VALUE:
     *out |= ((bit_width(size) - 1) << 14); // Encode the size.
     switch (size) {
@@ -1203,8 +1207,23 @@ void Arm64XDynamicRelocEntry::writeTo(uint8_t *buf) const {
     }
     break;
   case IMAGE_DVRT_ARM64X_FIXUP_TYPE_DELTA:
-  case IMAGE_DVRT_ARM64X_FIXUP_TYPE_ZEROFILL:
-    llvm_unreachable("unsupported type");
+    int delta = value.get();
+    // Negative offsets use a sign bit in the header.
+    if (delta < 0) {
+      *out |= 1 << 14;
+      delta = -delta;
+    }
+    // Depending on the value, the delta is encoded with a shift of 2 or 3 bits.
+    if (delta & 7) {
+      assert(!(delta & 3));
+      delta >>= 2;
+    } else {
+      *out |= (1 << 15);
+      delta >>= 3;
+    }
+    out[1] = delta;
+    assert(!(delta & ~0xffff));
+    break;
   }
 }
 
diff --git a/lld/COFF/Config.h b/lld/COFF/Config.h
index cd280aa09964d..0c7c4e91402f1 100644
--- a/lld/COFF/Config.h
+++ b/lld/COFF/Config.h
@@ -164,7 +164,6 @@ struct Configuration {
   bool noimplib = false;
   std::set<std::string> delayLoads;
   std::map<std::string, int> dllOrder;
-  Symbol *delayLoadHelper = nullptr;
   Symbol *arm64ECIcallHelper = nullptr;
 
   llvm::DenseSet<llvm::StringRef> saveTempsArgs;
diff --git a/lld/COFF/DLL.cpp b/lld/COFF/DLL.cpp
index 6a3f8eb21e847..198b6e1cddd1e 100644
--- a/lld/COFF/DLL.cpp
+++ b/lld/COFF/DLL.cpp
@@ -132,7 +132,6 @@ class ImportDirectoryChunk : public NonSectionChunk {
 class NullChunk : public NonSectionChunk {
 public:
   explicit NullChunk(size_t n, uint32_t align) : size(n) {
-    hasData = false;
     setAlignment(align);
   }
   explicit NullChunk(COFFLinkerContext &ctx)
@@ -717,6 +716,63 @@ class ExportOrdinalChunk : public NonSectionChunk {
 void IdataContents::create(COFFLinkerContext &ctx) {
   std::vector<std::vector<DefinedImportData *>> v = binImports(ctx, imports);
 
+  // In hybrid images, EC and native code are usually very similar,
+  // resulting in a highly similar set of imported symbols. Consequently,
+  // their import tables can be shared, with ARM64X relocations handling any
+  // differences. Identify matching import files used by EC and native code, and
+  // merge them into a single hybrid import entry.
+  if (ctx.hybridSymtab) {
+    for (std::vector<DefinedImportData *> &syms : v) {
+      std::vector<DefinedImportData *> hybridSyms;
+      ImportFile *prev = nullptr;
+      for (DefinedImportData *sym : syms) {
+        ImportFile *file = sym->file;
+        // At this stage, symbols are sorted by base name, ensuring that
+        // compatible import files, if present, are adjacent. Check if the
+        // current symbol's file imports the same symbol as the previously added
+        // one (if any and if it was not already merged). Additionally, verify
+        // that one of them is native while the other is EC. In rare cases,
+        // separate matching import entries may exist within the same namespace,
+        // which cannot be merged.
+        if (!prev || file->isEC() == prev->isEC() ||
+            !file->isSameImport(prev)) {
+          // We can't merge the import file, just add it to hybridSyms
+          // and set prev to its file so that we can try to match the next
+          // symbol.
+          hybridSyms.push_back(sym);
+          prev = file;
+          continue;
+        }
+
+        // A matching symbol may appear in syms in any order. The native variant
+        // exposes a subset of EC symbols and chunks, so always use the EC
+        // variant as the hybrid import file. If the native file was already
+        // added, replace it with the EC symbol in hybridSyms. Otherwise, the EC
+        // variant is already pushed, so we can simply merge it.
+        if (file->isEC()) {
+          hybridSyms.pop_back();
+          hybridSyms.push_back(sym);
+        }
+
+        // Merge import files by storing their hybrid form in the corresponding
+        // file class.
+        prev->hybridFile = file;
+        file->hybridFile = prev;
+        prev = nullptr; // A hybrid import file cannot be merged again.
+      }
+
+      // Sort symbols by type: native-only files first, followed by merged
+      // hybrid files, and then EC-only files.
+      llvm::stable_sort(hybridSyms,
+                        [](DefinedImportData *a, DefinedImportData *b) {
+                          if (a->file->hybridFile)
+                            return !b->file->hybridFile && b->file->isEC();
+                          return !a->file->isEC() && b->file->isEC();
+                        });
+      syms = std::move(hybridSyms);
+    }
+  }
+
   // Create .idata contents for each DLL.
   for (std::vector<DefinedImportData *> &syms : v) {
     // Create lookup and address tables. If they have external names,
@@ -724,19 +780,56 @@ void IdataContents::create(COFFLinkerContext &ctx) {
     // If they don't (if they are import-by-ordinals), we store only
     // ordinal values to the table.
     size_t base = lookups.size();
+    Chunk *lookupsTerminator = nullptr, *addressesTerminator = nullptr;
     for (DefinedImportData *s : syms) {
       uint16_t ord = s->getOrdinal();
+      HintNameChunk *hintChunk = nullptr;
+      Chunk *lookupsChunk, *addressesChunk;
+
       if (s->getExternalName().empty()) {
-        lookups.push_back(make<OrdinalOnlyChunk>(ctx, ord));
-        addresses.push_back(make<OrdinalOnlyChunk>(ctx, ord));
+        lookupsChunk = make<OrdinalOnlyChunk>(ctx, ord);
+        addressesChunk = make<OrdinalOnlyChunk>(ctx, ord);
       } else {
-        auto *c = make<HintNameChunk>(s->getExternalName(), ord);
-        lookups.push_back(make<LookupChunk>(ctx, c));
-        addresses.push_back(make<LookupChunk>(ctx, c));
-        hints.push_back(c);
+        hintChunk = make<HintNameChunk>(s->getExternalName(), ord);
+        lookupsChunk = make<LookupChunk>(ctx, hintChunk);
+        addressesChunk = make<LookupChunk>(ctx, hintChunk);
+        hints.push_back(hintChunk);
+      }
+
+      // Detect the first EC-only import in the hybrid IAT. Emit null chunk
+      // as a terminator for the native view, and add an ARM64X relocation to
+      // replace it with the correct import for the EC view.
+      //
+      // Additionally, for MSVC compatibility, store the lookup and address
+      // chunks and append them at the end of EC-only imports, where a null
+      // terminator chunk would typically be placed. Since they appear after
+      // the native terminator, they will be ignored in the native view.
+      // In the EC view, they should act as terminators, so emit ZEROFILL
+      // relocations overriding them.
+      if (ctx.hybridSymtab && !lookupsTerminator && s->file->isEC() &&
+          !s->file->hybridFile) {
+        lookupsTerminator = lookupsChunk;
+        addressesTerminator = addressesChunk;
+        lookupsChunk = make<NullChunk>(ctx);
+        addressesChunk = make<NullChunk>(ctx);
+
+        Arm64XRelocVal relocVal = hintChunk;
+        if (!hintChunk)
+          relocVal = (1ULL << 63) | ord;
+        ctx.dynamicRelocs->add(IMAGE_DVRT_ARM64X_FIXUP_TYPE_VALUE,
+                               sizeof(uint64_t), lookupsChunk, relocVal);
+        ctx.dynamicRelocs->add(IMAGE_DVRT_ARM64X_FIXUP_TYPE_VALUE,
+                               sizeof(uint64_t), addressesChunk, relocVal);
+        ctx.dynamicRelocs->add(IMAGE_DVRT_ARM64X_FIXUP_TYPE_ZEROFILL,
+                               sizeof(uint64_t), lookupsTerminator);
+        ctx.dynamicRelocs->add(IMAGE_DVRT_ARM64X_FIXUP_TYPE_ZEROFILL,
+                               sizeof(uint64_t), addressesTerminator);
       }
 
-      if (s->file->impECSym) {
+      lookups.push_back(lookupsChunk);
+      addresses.push_back(addressesChunk);
+
+      if (s->file->isEC()) {
         auto chunk = make<AuxImportChunk>(s->file);
         auxIat.push_back(chunk);
         s->file->impECSym->setLocation(chunk);
@@ -744,18 +837,27 @@ void IdataContents::create(COFFLinkerContext &ctx) {
         chunk = make<AuxImportChunk>(s->file);
         auxIatCopy.push_back(chunk);
         s->file->auxImpCopySym->setLocation(chunk);
+      } else if (ctx.hybridSymtab) {
+        // Fill the auxiliary IAT with null chunks for native-only imports.
+        auxIat.push_back(make<NullChunk>(ctx));
+        auxIatCopy.push_back(make<NullChunk>(ctx));
       }
     }
     // Terminate with null values.
-    lookups.push_back(make<NullChunk>(ctx));
-    addresses.push_back(make<NullChunk>(ctx));
-    if (ctx.config.machine == ARM64EC) {
+    lookups.push_back(lookupsTerminator ? lookupsTerminator
+                                        : make<NullChunk>(ctx));
+    addresses.push_back(addressesTerminator ? addressesTerminator
+                                            : make<NullChunk>(ctx));
+    if (ctx.symtabEC) {
       auxIat.push_back(make<NullChunk>(ctx));
       auxIatCopy.push_back(make<NullChunk>(ctx));
     }
 
-    for (int i = 0, e = syms.size(); i < e; ++i)
+    for (int i = 0, e = syms.size(); i < e; ++i) {
       syms[i]->setLocation(addresses[base + i]);
+      if (syms[i]->file->hybridFile)
+        syms[i]->file->hybridFile->impSym->setLocation(addresses[base + i]);
+    }
 
     // Create the import table header.
     dllNames.push_back(make<StringChunk>(syms[0]->getDLLName()));
@@ -763,6 +865,27 @@ void IdataContents::create(COFFLinkerContext &ctx) {
     dir->lookupTab = lookups[base];
     dir->addressTab = addresses[base];
     dirs.push_back(dir);
+
+    if (ctx.hybridSymtab) {
+      // If native-only imports exist, they will appear as a prefix to all
+      // imports. Emit ARM64X relocations to skip them in the EC view.
+      uint32_t nativeOnly =
+          llvm::find_if(syms,
+                        [](DefinedImportData *s) { return s->file->isEC(); }) -
+          syms.begin();
+      if (nativeOnly) {
+        ctx.dynamicRelocs->add(
+            IMAGE_DVRT_ARM64X_FIXUP_TYPE_DELTA, 0,
+            Arm64XRelocVal(
+                dir, offsetof(ImportDirectoryTableEntry, ImportLookupTableRVA)),
+            nativeOnly * sizeof(uint64_t));
+        ctx.dynamicRelocs->add(
+            IMAGE_DVRT_ARM64X_FIXUP_TYPE_DELTA, 0,
+            Arm64XRelocVal(dir, offsetof(ImportDirectoryTableEntry,
+                                         ImportAddressTableRVA)),
+            nativeOnly * sizeof(uint64_t));
+      }
+    }
   }
   // Add null terminator.
   dirs.push_back(make<NullChunk>(sizeof(ImportDirectoryTableEntry), 4));
@@ -788,12 +911,9 @@ uint64_t DelayLoadContents::getDirSize() {
   return dirs.size() * sizeof(delay_import_directory_table_entry);
 }
 
-void DelayLoadContents::create(Defined *h) {
-  helper = h;
+void DelayLoadContents::create() {
   std::vector<std::vector<DefinedImportData *>> v = binImports(ctx, imports);
 
-  Chunk *unwind = newTailMergeUnwindInfoChunk();
-
   // Create .didat contents for each DLL.
   for (std::vector<DefinedImportData *> &syms : v) {
     // Create the delay import table header.
@@ -801,54 +921,89 @@ void DelayLoadContents::create(Defined *h) {
     auto *dir = make<DelayDirectoryChunk>(dllNames.back());
 
     size_t base = addresses.size();
-    Chunk *tm = newTailMergeChunk(dir);
-    Chunk *pdataChunk = unwind ? newTailMergePDataChunk(tm, unwind) : nullptr;
-    for (DefinedImportData *s : syms) {
-      Chunk *t = newThunkChunk(s, tm);
-      auto *a = make<DelayAddressChunk>(ctx, t);
-      addresses.push_back(a);
-      thunks.push_back(t);
-      StringRef extName = s->getExternalName();
-      if (extName.empty()) {
-        names.push_back(make<OrdinalOnlyChunk>(ctx, s->getOrdinal()));
-      } else {
-        auto *c = make<HintNameChunk>(extName, 0);
-        names.push_back(make<LookupChunk>(ctx, c));
-        hintNames.push_back(c);
-        // Add a synthetic symbol for this load thunk, using the "__imp___load"
-        // prefix, in case this thunk needs to be added to the list of valid
-        // call targets for Control Flow Guard.
-        StringRef symName = saver().save("__imp___load_" + extName);
-        s->loadThunkSym =
-            cast<DefinedSynthetic>(ctx.symtab.addSynthetic(symName, t));
+    ctx.forEachSymtab([&](SymbolTable &symtab) {
+      if (ctx.hybridSymtab && symtab.isEC()) {
+        // For hybrid images, emit null-terminated native import entries
+        // followed by null-terminated EC entries. If a view is missing imports
+        // for a given module, only terminators are emitted. Emit ARM64X
+        // relocations to skip native entries in the EC view.
+        ctx.dynamicRelocs->add(
+            IMAGE_DVRT_ARM64X_FIXUP_TYPE_DELTA, 0,
+            Arm64XRelocVal(dir, offsetof(delay_import_directory_table_entry,
+                                         DelayImportAddressTable)),
+            (addresses.size() - base) * sizeof(uint64_t));
+        ctx.dynamicRelocs->add(
+            IMAGE_DVRT_ARM64X_FIXUP_TYPE_DELTA, 0,
+            Arm64XRelocVal(dir, offsetof(delay_import_directory_table_entry,
+                                         DelayImportNameTable)),
+            (addresses.size() - base) * sizeof(uint64_t));
       }
 
-      if (s->file->impECSym) {
-        auto chunk = make<AuxImportChunk>(s->file);
-        auxIat.push_back(chunk);
-        s->file->impECSym->setLocation(chunk);
+      Chunk *tm = nullptr;
+
+      for (DefinedImportData *s : syms) {
+        // Process only the symbols belonging to the current symtab.
+        if (symtab.isEC() != s->file->isEC())
+          continue;
+
+        if (!tm) {
+          tm = newTailMergeChunk(symtab, dir);
+          Chunk *pdataChunk = newTailMergePDataChunk(symtab, tm);
+          if (pdataChunk)
+            pdata.push_back(pdataChunk);
+        }
+
+        Chunk *t = newThunkChunk(s, tm);
+        auto *a = make<DelayAddressChunk>(ctx, t);
+        addresses.push_back(a);
+        s->setLocation(a);
+        thunks.push_back(t);
+        StringRef extName = s->getExternalName();
+        if (extName.empty()) {
+          names.push_back(make<OrdinalOnlyChunk>(ctx, s->getOrdinal()));
+        } else {
+          auto *c = make<HintNameChunk>(extName, 0);
+          names.push_back(make<LookupChunk>(ctx, c));
+          hintNames.push_back(c);
+          // Add a synthetic symbol for this load thunk, using the
+          // "__imp___load" prefix, in case this thunk needs to be added to the
+          // list of valid call targets for Control Flow Guard.
+          StringRef symName = saver().save("__imp___load_" + extName);
+          s->loadThunkSym =
+              cast<DefinedSynthetic>(symtab.addSynthetic(symName, t));
+        }
+
+        if (symtab.isEC()) {
+          auto chunk = make<AuxImportChunk>(s->file);
+          auxIat.push_back(chunk);
+          s->file->impECSym->setLocation(chunk);
+
+          chunk = make<AuxImportChunk>(s->file);
+          auxIatCopy.push_back(chunk);
+          s->file->auxImpCopySym->setLocation(chunk);
+        } else if (ctx.hybridSymtab) {
+          // Fill the auxiliary IAT with null chunks for native imports.
+          auxIat.push_back(make<NullChunk>(ctx));
+          auxIatCopy.push_back(make<NullChunk>(ctx));
+        }
+      }
 
-        chunk = make<AuxImportChunk>(s->file);
-        auxIatCopy.push_back(chunk);
-        s->file->auxImpCopySym->setLocation(chunk);
+      if (tm) {
+        thunks.push_back(tm);
+        StringRef tmName =
+            saver().save("__tailMerge_" + syms[0]->getDLLName().lower());
+        symtab.addSynthetic(tmName, tm);
       }
-    }
-    thunks.push_back(tm);
-    if (pdataChunk)
-      pdata.push_back(pdataChunk);
-    StringRef tmName =
-        saver().save("__tailMerge_" + syms[0]->getDLLName().lower());
-    ctx.symtab.addSynthetic(tmName, tm);
-    // Terminate with null values.
-    addresses.push_back(make<NullChunk>(ctx, 8));
-    names.push_back(make<NullChunk>(ctx, 8));
-    if (ctx.config.machine == ARM64EC) {
-      auxIat.push_back(make<NullChunk>(ctx, 8));
-      auxIatCopy.push_back(make<NullChunk>(ctx, 8));
-    }
 
-    for (int i = 0, e = syms.size(); i < e; ++i)
-      syms[i]->setLocation(addresses[base + i]);
+      // Terminate with null values.
+      addresses.push_back(make<NullChunk>(ctx, 8));
+      names.push_back(make<NullChunk>(ctx, 8));
+      if (ctx.symtabEC) {
+        auxIat.push_back(make<NullChunk>(ctx, 8));
+        auxIatCopy.push_back(make<NullChunk>(ctx, 8));
+      }
+    });
+
     auto *mh = make<NullChunk>(8, 8);
     moduleHandles.push_back(mh);
 
@@ -859,15 +1014,18 @@ void DelayLoadContents::create(Defined *h) {
     dirs.push_back(dir);
   }
 
-  if (unwind)
-    unwindinfo.push_back(unwind);
+  ctx.forEachSymtab([&](SymbolTable &symtab) {
+    if (symtab.tailMergeUnwindInfoChunk)
+      unwindinfo.push_back(symtab.tailMergeUnwindInfoChunk);
+  });
   // Add null terminator.
   dirs.push_back(
       make<NullChunk>(sizeof(delay_import_directory_table_entry), 4));
 }
 
-Chunk *DelayLoadContents::newTailMergeChunk(Chunk *dir) {
-  switch (ctx.config.machine) {
+Chunk *DelayLoadContents::newTailMergeChunk(SymbolTable &symtab, Chunk *dir) {
+  auto helper = cast<Defined>(symtab.delayLoadHelper);
+  switch (symtab.machine) {
   case AMD64:
   case ARM64EC:
     return make<TailMergeChunkX64>(dir, helper);
@@ -882,21 +1040,14 @@ Chunk *DelayLoadContents::newTailMergeChunk(Chunk *dir) {
   }
 }
 
-Chunk *DelayLoadContents::newTailMergeUnwindInfoChunk() {
-  switch (ctx.config.machine) {
-  case AMD64:
-  case ARM64EC:
-    return make<TailMergeUnwindInfoX64>();
-    // FIXME: Add support for other architectures.
-  default:
-    return nullptr; // Just don't generate unwind info.
-  }
-}
-Chunk *DelayLoadContents::newTailMergePDataChunk(Chunk *tm, Chunk *unwind) {
-  switch (ctx.config.machine) {
+Chunk *DelayLoadContents::newTailMergePDataChunk(SymbolTable &symtab,
+                                                 Chunk *tm) {
+  switch (symtab.machine) {
   case AMD64:
   case ARM64EC:
-    return make<TailMergePDataChunkX64>(tm, unwind);
+    if (!symtab.tailMergeUnwindInfoChunk)
+      symtab.tailMergeUnwindInfoChunk = make<TailMergeUnwindInfoX64>();
+    return make<TailMergePDataChunkX64>(tm, symtab.tailMergeUnwindInfoChunk);
     // FIXME: Add support for other architectures.
   default:
     return nullptr; // Just don't generate unwind info.
@@ -905,7 +1056,7 @@ Chunk *DelayLoadContents::newTailMergePDataChunk(Chunk *tm, Chunk *unwind) {
 
 Chunk *DelayLoadContents::newThunkChunk(DefinedImportData *s,
                                         Chunk *tailMerge) {
-  switch (ctx.config.machine) {
+  switch (s->file->getMachineType()) {
   case AMD64:
   case ARM64EC:
     return make<ThunkChunkX64>(s, tailMerge);
diff --git a/lld/COFF/DLL.h b/lld/COFF/DLL.h
index 724a323d62d20..5105b79f15d31 100644
--- a/lld/COFF/DLL.h
+++ b/lld/COFF/DLL.h
@@ -42,7 +42,7 @@ class DelayLoadContents {
   DelayLoadContents(COFFLinkerContext &ctx) : ctx(ctx) {}
   void add(DefinedImportData *sym) { imports.push_back(sym); }
   bool empty() { return imports.empty(); }
-  void create(Defined *helper);
+  void create();
   std::vector<Chunk *> getChunks();
   std::vector<Chunk *> getDataChunks();
   ArrayRef<Chunk *> getCodeChunks() { return thunks; }
@@ -56,11 +56,9 @@ class DelayLoadContents {
 
 private:
   Chunk *newThunkChunk(DefinedImportData *s, Chunk *tailMerge);
-  Chunk *newTailMergeChunk(Chunk *dir);
-  Chunk *newTailMergePDataChunk(Chunk *tm, Chunk *unwind);
-  Chunk *newTailMergeUnwindInfoChunk();
+  Chunk *newTailMergeChunk(SymbolTable &symtab, Chunk *dir);
+  Chunk *newTailMergePDataChunk(SymbolTable &symtab, Chunk *tm);
 
-  Defined *helper;
   std::vector<DefinedImportData *> imports;
   std::vector<Chunk *> dirs;
   std::vector<Chunk *> moduleHandles;
diff --git a/lld/COFF/Driver.cpp b/lld/COFF/Driver.cpp
index 6eea11f5f451f..6433ce6643f9c 100644
--- a/lld/COFF/Driver.cpp
+++ b/lld/COFF/Driver.cpp
@@ -954,20 +954,31 @@ std::string LinkerDriver::getImportName(bool asLib) {
 
 void LinkerDriver::createImportLibrary(bool asLib) {
   llvm::TimeTraceScope timeScope("Create import library");
-  std::vector<COFFShortExport> exports;
-  for (Export &e1 : ctx.symtab.exports) {
-    COFFShortExport e2;
-    e2.Name = std::string(e1.name);
-    e2.SymbolName = std::string(e1.symbolName);
-    e2.ExtName = std::string(e1.extName);
-    e2.ExportAs = std::string(e1.exportAs);
-    e2.ImportName = std::string(e1.importName);
-    e2.Ordinal = e1.ordinal;
-    e2.Noname = e1.noname;
-    e2.Data = e1.data;
-    e2.Private = e1.isPrivate;
-    e2.Constant = e1.constant;
-    exports.push_back(e2);
+  std::vector<COFFShortExport> exports, nativeExports;
+
+  auto getExports = [](SymbolTable &symtab,
+                       std::vector<COFFShortExport> &exports) {
+    for (Export &e1 : symtab.exports) {
+      COFFShortExport e2;
+      e2.Name = std::string(e1.name);
+      e2.SymbolName = std::string(e1.symbolName);
+      e2.ExtName = std::string(e1.extName);
+      e2.ExportAs = std::string(e1.exportAs);
+      e2.ImportName = std::string(e1.importName);
+      e2.Ordinal = e1.ordinal;
+      e2.Noname = e1.noname;
+      e2.Data = e1.data;
+      e2.Private = e1.isPrivate;
+      e2.Constant = e1.constant;
+      exports.push_back(e2);
+    }
+  };
+
+  if (ctx.hybridSymtab) {
+    getExports(ctx.symtab, nativeExports);
+    getExports(*ctx.hybridSymtab, exports);
+  } else {
+    getExports(ctx.symtab, exports);
   }
 
   std::string libName = getImportName(asLib);
@@ -975,7 +986,7 @@ void LinkerDriver::createImportLibrary(bool asLib) {
 
   if (!ctx.config.incremental) {
     checkError(writeImportLibrary(libName, path, exports, ctx.config.machine,
-                                  ctx.config.mingw));
+                                  ctx.config.mingw, nativeExports));
     return;
   }
 
@@ -985,7 +996,7 @@ void LinkerDriver::createImportLibrary(bool asLib) {
       path, /*IsText=*/false, /*RequiresNullTerminator=*/false);
   if (!oldBuf) {
     checkError(writeImportLibrary(libName, path, exports, ctx.config.machine,
-                                  ctx.config.mingw));
+                                  ctx.config.mingw, nativeExports));
     return;
   }
 
@@ -995,8 +1006,9 @@ void LinkerDriver::createImportLibrary(bool asLib) {
     Fatal(ctx) << "cannot create temporary file for import library " << path
                << ": " << ec.message();
 
-  if (Error e = writeImportLibrary(libName, tmpName, exports,
-                                   ctx.config.machine, ctx.config.mingw)) {
+  if (Error e =
+          writeImportLibrary(libName, tmpName, exports, ctx.config.machine,
+                             ctx.config.mingw, nativeExports)) {
     checkError(std::move(e));
     return;
   }
@@ -1370,43 +1382,46 @@ void LinkerDriver::maybeExportMinGWSymbols(const opt::InputArgList &args) {
     if (!ctx.config.dll)
       return;
 
-    if (!ctx.symtab.exports.empty())
+    if (ctx.symtab.hadExplicitExports ||
+        (ctx.hybridSymtab && ctx.hybridSymtab->hadExplicitExports))
       return;
     if (args.hasArg(OPT_exclude_all_symbols))
       return;
   }
 
-  AutoExporter exporter(ctx, excludedSymbols);
+  ctx.forEachSymtab([&](SymbolTable &symtab) {
+    AutoExporter exporter(symtab, excludedSymbols);
 
-  for (auto *arg : args.filtered(OPT_wholearchive_file))
-    if (std::optional<StringRef> path = findFile(arg->getValue()))
-      exporter.addWholeArchive(*path);
+    for (auto *arg : args.filtered(OPT_wholearchive_file))
+      if (std::optional<StringRef> path = findFile(arg->getValue()))
+        exporter.addWholeArchive(*path);
 
-  for (auto *arg : args.filtered(OPT_exclude_symbols)) {
-    SmallVector<StringRef, 2> vec;
-    StringRef(arg->getValue()).split(vec, ',');
-    for (StringRef sym : vec)
-      exporter.addExcludedSymbol(ctx.symtab.mangle(sym));
-  }
+    for (auto *arg : args.filtered(OPT_exclude_symbols)) {
+      SmallVector<StringRef, 2> vec;
+      StringRef(arg->getValue()).split(vec, ',');
+      for (StringRef sym : vec)
+        exporter.addExcludedSymbol(symtab.mangle(sym));
+    }
 
-  ctx.symtab.forEachSymbol([&](Symbol *s) {
-    auto *def = dyn_cast<Defined>(s);
-    if (!exporter.shouldExport(def))
-      return;
+    symtab.forEachSymbol([&](Symbol *s) {
+      auto *def = dyn_cast<Defined>(s);
+      if (!exporter.shouldExport(def))
+        return;
 
-    if (!def->isGCRoot) {
-      def->isGCRoot = true;
-      ctx.config.gcroot.push_back(def);
-    }
+      if (!def->isGCRoot) {
+        def->isGCRoot = true;
+        ctx.config.gcroot.push_back(def);
+      }
 
-    Export e;
-    e.name = def->getName();
-    e.sym = def;
-    if (Chunk *c = def->getChunk())
-      if (!(c->getOutputCharacteristics() & IMAGE_SCN_MEM_EXECUTE))
-        e.data = true;
-    s->isUsedInRegularObj = true;
-    ctx.symtab.exports.push_back(e);
+      Export e;
+      e.name = def->getName();
+      e.sym = def;
+      if (Chunk *c = def->getChunk())
+        if (!(c->getOutputCharacteristics() & IMAGE_SCN_MEM_EXECUTE))
+          e.data = true;
+      s->isUsedInRegularObj = true;
+      symtab.exports.push_back(e);
+    });
   });
 }
 
@@ -2353,12 +2368,13 @@ void LinkerDriver::linkerMain(ArrayRef<const char *> argsArr) {
     llvm::TimeTraceScope timeScope("Delay load");
     for (auto *arg : args.filtered(OPT_delayload)) {
       config->delayLoads.insert(StringRef(arg->getValue()).lower());
-      if (config->machine == I386) {
-        config->delayLoadHelper = ctx.symtab.addGCRoot("___delayLoadHelper2@8");
-      } else {
-        config->delayLoadHelper =
-            ctx.symtab.addGCRoot("__delayLoadHelper2", true);
-      }
+      ctx.forEachSymtab([&](SymbolTable &symtab) {
+        if (symtab.machine == I386) {
+          symtab.delayLoadHelper = symtab.addGCRoot("___delayLoadHelper2@8");
+        } else {
+          symtab.delayLoadHelper = symtab.addGCRoot("__delayLoadHelper2", true);
+        }
+      });
     }
   }
 
@@ -2541,14 +2557,14 @@ void LinkerDriver::linkerMain(ArrayRef<const char *> argsArr) {
         // it.
         if (symtab.findUnderscore("_load_config_used"))
           symtab.addGCRoot(symtab.mangle("_load_config_used"));
-      });
 
-      if (args.hasArg(OPT_include_optional)) {
-        // Handle /includeoptional
-        for (auto *arg : args.filtered(OPT_include_optional))
-          if (isa_and_nonnull<LazyArchive>(ctx.symtab.find(arg->getValue())))
-            ctx.symtab.addGCRoot(arg->getValue());
-      }
+        if (args.hasArg(OPT_include_optional)) {
+          // Handle /includeoptional
+          for (auto *arg : args.filtered(OPT_include_optional))
+            if (isa_and_nonnull<LazyArchive>(symtab.find(arg->getValue())))
+              symtab.addGCRoot(arg->getValue());
+        }
+      });
     } while (run());
   }
 
@@ -2557,11 +2573,13 @@ void LinkerDriver::linkerMain(ArrayRef<const char *> argsArr) {
     ctx.symtab.addUndefinedGlob(pat);
 
   // Create wrapped symbols for -wrap option.
-  std::vector<WrappedSymbol> wrapped = addWrappedSymbols(ctx, args);
-  // Load more object files that might be needed for wrapped symbols.
-  if (!wrapped.empty())
-    while (run())
-      ;
+  ctx.forEachSymtab([&](SymbolTable &symtab) {
+    addWrappedSymbols(symtab, args);
+    // Load more object files that might be needed for wrapped symbols.
+    if (!symtab.wrapped.empty())
+      while (run())
+        ;
+  });
 
   if (config->autoImport || config->stdcallFixup) {
     // MinGW specific.
@@ -2631,8 +2649,10 @@ void LinkerDriver::linkerMain(ArrayRef<const char *> argsArr) {
   run();
 
   // Apply symbol renames for -wrap.
-  if (!wrapped.empty())
-    wrapSymbols(ctx, wrapped);
+  ctx.forEachSymtab([](SymbolTable &symtab) {
+    if (!symtab.wrapped.empty())
+      wrapSymbols(symtab);
+  });
 
   if (isArm64EC(config->machine))
     createECExportThunks();
@@ -2667,7 +2687,8 @@ void LinkerDriver::linkerMain(ArrayRef<const char *> argsArr) {
   // Windows specific -- when we are creating a .dll file, we also
   // need to create a .lib file. In MinGW mode, we only do that when the
   // -implib option is given explicitly, for compatibility with GNU ld.
-  if (!ctx.symtab.exports.empty() || config->dll) {
+  if (config->dll || !ctx.symtab.exports.empty() ||
+      (ctx.hybridSymtab && !ctx.hybridSymtab->exports.empty())) {
     llvm::TimeTraceScope timeScope("Create .lib exports");
     ctx.forEachSymtab([](SymbolTable &symtab) { symtab.fixupExports(); });
     if (!config->noimplib && (!config->mingw || !config->implib.empty()))
@@ -2678,7 +2699,7 @@ void LinkerDriver::linkerMain(ArrayRef<const char *> argsArr) {
 
   // Handle /output-def (MinGW specific).
   if (auto *arg = args.getLastArg(OPT_output_def))
-    writeDefFile(ctx, arg->getValue(), ctx.symtab.exports);
+    writeDefFile(ctx, arg->getValue(), mainSymtab.exports);
 
   // Set extra alignment for .comm symbols
   for (auto pair : config->alignComm) {
diff --git a/lld/COFF/InputFiles.cpp b/lld/COFF/InputFiles.cpp
index 47faf70e099e1..7b105fb4c17a2 100644
--- a/lld/COFF/InputFiles.cpp
+++ b/lld/COFF/InputFiles.cpp
@@ -1129,15 +1129,21 @@ void ObjFile::enqueuePdbFile(StringRef path, ObjFile *fromFile) {
 }
 
 ImportFile::ImportFile(COFFLinkerContext &ctx, MemoryBufferRef m)
-    : InputFile(ctx.symtab, ImportKind, m), live(!ctx.config.doGC) {}
+    : InputFile(ctx.getSymtab(getMachineType(m)), ImportKind, m),
+      live(!ctx.config.doGC) {}
 
-MachineTypes ImportFile::getMachineType() const {
+MachineTypes ImportFile::getMachineType(MemoryBufferRef m) {
   uint16_t machine =
-      reinterpret_cast<const coff_import_header *>(mb.getBufferStart())
-          ->Machine;
+      reinterpret_cast<const coff_import_header *>(m.getBufferStart())->Machine;
   return MachineTypes(machine);
 }
 
+bool ImportFile::isSameImport(const ImportFile *other) const {
+  if (!externalName.empty())
+    return other->externalName == externalName;
+  return hdr->OrdinalHint == other->hdr->OrdinalHint;
+}
+
 ImportThunkChunk *ImportFile::makeImportThunk() {
   switch (hdr->Machine) {
   case AMD64:
diff --git a/lld/COFF/InputFiles.h b/lld/COFF/InputFiles.h
index 823561cda247a..21b9aeef21d4f 100644
--- a/lld/COFF/InputFiles.h
+++ b/lld/COFF/InputFiles.h
@@ -351,11 +351,15 @@ class ImportFile : public InputFile {
   explicit ImportFile(COFFLinkerContext &ctx, MemoryBufferRef m);
 
   static bool classof(const InputFile *f) { return f->kind() == ImportKind; }
-  MachineTypes getMachineType() const override;
+  MachineTypes getMachineType() const override { return getMachineType(mb); }
+  static MachineTypes getMachineType(MemoryBufferRef m);
+  bool isSameImport(const ImportFile *other) const;
+  bool isEC() const { return impECSym != nullptr; }
 
   DefinedImportData *impSym = nullptr;
   Defined *thunkSym = nullptr;
   ImportThunkChunkARM64EC *impchkThunk = nullptr;
+  ImportFile *hybridFile = nullptr;
   std::string dllName;
 
 private:
diff --git a/lld/COFF/MinGW.cpp b/lld/COFF/MinGW.cpp
index 76f5a0a7500b9..ca5dfbcc7e47c 100644
--- a/lld/COFF/MinGW.cpp
+++ b/lld/COFF/MinGW.cpp
@@ -25,9 +25,8 @@ using namespace lld;
 using namespace lld::coff;
 
 AutoExporter::AutoExporter(
-    COFFLinkerContext &ctx,
-    const llvm::DenseSet<StringRef> &manualExcludeSymbols)
-    : manualExcludeSymbols(manualExcludeSymbols), ctx(ctx) {
+    SymbolTable &symtab, const llvm::DenseSet<StringRef> &manualExcludeSymbols)
+    : manualExcludeSymbols(manualExcludeSymbols), symtab(symtab) {
   excludeLibs = {
       "libgcc",
       "libgcc_s",
@@ -50,8 +49,7 @@ AutoExporter::AutoExporter(
       "libclang_rt.profile-x86_64",
       "libc++",
       "libc++abi",
-      "libFortranRuntime",
-      "libFortranDecimal",
+      "libflang_rt.runtime",
       "libunwind",
       "libmsvcrt",
       "libucrtbase",
@@ -84,7 +82,7 @@ AutoExporter::AutoExporter(
       "_NULL_THUNK_DATA",
   };
 
-  if (ctx.config.machine == I386) {
+  if (symtab.machine == I386) {
     excludeSymbols = {
         "__NULL_IMPORT_DESCRIPTOR",
         "__pei386_runtime_relocator",
@@ -94,6 +92,7 @@ AutoExporter::AutoExporter(
         "__fmode",
         "_environ",
         "___dso_handle",
+        "__load_config_used",
         // These are the MinGW names that differ from the standard
         // ones (lacking an extra underscore).
         "_DllMain@12",
@@ -111,6 +110,7 @@ AutoExporter::AutoExporter(
         "_fmode",
         "environ",
         "__dso_handle",
+        "_load_config_used",
         // These are the MinGW names that differ from the standard
         // ones (lacking an extra underscore).
         "DllMain",
@@ -119,6 +119,10 @@ AutoExporter::AutoExporter(
     };
     excludeSymbolPrefixes.insert("_head_");
   }
+  if (symtab.isEC()) {
+    excludeSymbols.insert("__chpe_metadata");
+    excludeSymbolPrefixes.insert("__os_arm64x_");
+  }
 }
 
 void AutoExporter::addWholeArchive(StringRef path) {
@@ -151,7 +155,7 @@ bool AutoExporter::shouldExport(Defined *sym) const {
       return false;
 
   // If a corresponding __imp_ symbol exists and is defined, don't export it.
-  if (ctx.symtab.find(("__imp_" + sym->getName()).str()))
+  if (symtab.find(("__imp_" + sym->getName()).str()))
     return false;
 
   // Check that file is non-null before dereferencing it, symbols not
@@ -203,8 +207,8 @@ static StringRef mangle(Twine sym, MachineTypes machine) {
 // This function instantiates wrapper symbols. At this point, they seem
 // like they are not being used at all, so we explicitly set some flags so
 // that LTO won't eliminate them.
-std::vector<WrappedSymbol>
-lld::coff::addWrappedSymbols(COFFLinkerContext &ctx, opt::InputArgList &args) {
+void lld::coff::addWrappedSymbols(SymbolTable &symtab,
+                                  opt::InputArgList &args) {
   std::vector<WrappedSymbol> v;
   DenseSet<StringRef> seen;
 
@@ -213,14 +217,14 @@ lld::coff::addWrappedSymbols(COFFLinkerContext &ctx, opt::InputArgList &args) {
     if (!seen.insert(name).second)
       continue;
 
-    Symbol *sym = ctx.symtab.findUnderscore(name);
+    Symbol *sym = symtab.findUnderscore(name);
     if (!sym)
       continue;
 
     Symbol *real =
-        ctx.symtab.addUndefined(mangle("__real_" + name, ctx.config.machine));
+        symtab.addUndefined(mangle("__real_" + name, symtab.machine));
     Symbol *wrap =
-        ctx.symtab.addUndefined(mangle("__wrap_" + name, ctx.config.machine));
+        symtab.addUndefined(mangle("__wrap_" + name, symtab.machine));
     v.push_back({sym, real, wrap});
 
     // These symbols may seem undefined initially, but don't bail out
@@ -239,7 +243,7 @@ lld::coff::addWrappedSymbols(COFFLinkerContext &ctx, opt::InputArgList &args) {
     if (!isa<Undefined>(wrap))
       wrap->isUsedInRegularObj = true;
   }
-  return v;
+  symtab.wrapped = std::move(v);
 }
 
 // Do renaming for -wrap by updating pointers to symbols.
@@ -247,29 +251,28 @@ lld::coff::addWrappedSymbols(COFFLinkerContext &ctx, opt::InputArgList &args) {
 // When this function is executed, only InputFiles and symbol table
 // contain pointers to symbol objects. We visit them to replace pointers,
 // so that wrapped symbols are swapped as instructed by the command line.
-void lld::coff::wrapSymbols(COFFLinkerContext &ctx,
-                            ArrayRef<WrappedSymbol> wrapped) {
+void lld::coff::wrapSymbols(SymbolTable &symtab) {
   DenseMap<Symbol *, Symbol *> map;
-  for (const WrappedSymbol &w : wrapped) {
+  for (const WrappedSymbol &w : symtab.wrapped) {
     map[w.sym] = w.wrap;
     map[w.real] = w.sym;
     if (Defined *d = dyn_cast<Defined>(w.wrap)) {
-      Symbol *imp = ctx.symtab.find(("__imp_" + w.sym->getName()).str());
+      Symbol *imp = symtab.find(("__imp_" + w.sym->getName()).str());
       // Create a new defined local import for the wrap symbol. If
       // no imp prefixed symbol existed, there's no need for it.
       // (We can't easily distinguish whether any object file actually
       // referenced it or not, though.)
       if (imp) {
         DefinedLocalImport *wrapimp = make<DefinedLocalImport>(
-            ctx, saver().save("__imp_" + w.wrap->getName()), d);
-        ctx.symtab.localImportChunks.push_back(wrapimp->getChunk());
+            symtab.ctx, saver().save("__imp_" + w.wrap->getName()), d);
+        symtab.localImportChunks.push_back(wrapimp->getChunk());
         map[imp] = wrapimp;
       }
     }
   }
 
   // Update pointers in input files.
-  parallelForEach(ctx.objFileInstances, [&](ObjFile *file) {
+  parallelForEach(symtab.ctx.objFileInstances, [&](ObjFile *file) {
     MutableArrayRef<Symbol *> syms = file->getMutableSymbols();
     for (auto &sym : syms)
       if (Symbol *s = map.lookup(sym))
diff --git a/lld/COFF/MinGW.h b/lld/COFF/MinGW.h
index ffa500b234777..48bce74c616bc 100644
--- a/lld/COFF/MinGW.h
+++ b/lld/COFF/MinGW.h
@@ -25,7 +25,7 @@ class COFFLinkerContext;
 // symbols for MinGW.
 class AutoExporter {
 public:
-  AutoExporter(COFFLinkerContext &ctx,
+  AutoExporter(SymbolTable &symtab,
                const llvm::DenseSet<StringRef> &manualExcludeSymbols);
 
   void addWholeArchive(StringRef path);
@@ -42,7 +42,7 @@ class AutoExporter {
   bool shouldExport(Defined *sym) const;
 
 private:
-  COFFLinkerContext &ctx;
+  SymbolTable &symtab;
 };
 
 void writeDefFile(COFFLinkerContext &, StringRef name,
@@ -55,17 +55,9 @@ void writeDefFile(COFFLinkerContext &, StringRef name,
 // symbol becomes accessible as `__real_foo`, so you can call that from your
 // wrapper.
 //
-// This data structure is instantiated for each -wrap option.
-struct WrappedSymbol {
-  Symbol *sym;
-  Symbol *real;
-  Symbol *wrap;
-};
-
-std::vector<WrappedSymbol> addWrappedSymbols(COFFLinkerContext &ctx,
-                                             llvm::opt::InputArgList &args);
+void addWrappedSymbols(SymbolTable &symtab, llvm::opt::InputArgList &args);
 
-void wrapSymbols(COFFLinkerContext &ctx, ArrayRef<WrappedSymbol> wrapped);
+void wrapSymbols(SymbolTable &symtab);
 
 } // namespace lld::coff
 
diff --git a/lld/COFF/SymbolTable.h b/lld/COFF/SymbolTable.h
index c8d7251838842..b124998809e3c 100644
--- a/lld/COFF/SymbolTable.h
+++ b/lld/COFF/SymbolTable.h
@@ -34,6 +34,13 @@ class LazyArchive;
 class SectionChunk;
 class Symbol;
 
+// This data structure is instantiated for each -wrap option.
+struct WrappedSymbol {
+  Symbol *sym;
+  Symbol *real;
+  Symbol *wrap;
+};
+
 // SymbolTable is a bucket of all known symbols, including defined,
 // undefined, or lazy symbols (the last one is symbols in archive
 // files whose archive members are not yet loaded).
@@ -158,6 +165,12 @@ class SymbolTable {
   Chunk *edataStart = nullptr;
   Chunk *edataEnd = nullptr;
 
+  Symbol *delayLoadHelper = nullptr;
+  Chunk *tailMergeUnwindInfoChunk = nullptr;
+
+  // A list of wrapped symbols.
+  std::vector<WrappedSymbol> wrapped;
+
   void fixupExports();
   void assignExportOrdinals();
   void parseModuleDefs(StringRef path);
diff --git a/lld/COFF/Writer.cpp b/lld/COFF/Writer.cpp
index bef2ced9f2957..484ac9cdcaf9e 100644
--- a/lld/COFF/Writer.cpp
+++ b/lld/COFF/Writer.cpp
@@ -1191,10 +1191,12 @@ void Writer::createMiscChunks() {
   }
 
   // Create thunks for locally-dllimported symbols.
-  if (!ctx.symtab.localImportChunks.empty()) {
-    for (Chunk *c : ctx.symtab.localImportChunks)
-      rdataSec->addChunk(c);
-  }
+  ctx.forEachSymtab([&](SymbolTable &symtab) {
+    if (!symtab.localImportChunks.empty()) {
+      for (Chunk *c : symtab.localImportChunks)
+        rdataSec->addChunk(c);
+    }
+  });
 
   // Create Debug Information Chunks
   debugInfoSec = config->mingw ? buildidSec : rdataSec;
@@ -1307,8 +1309,7 @@ void Writer::appendImportThunks() {
   }
 
   if (!delayIdata.empty()) {
-    Defined *helper = cast<Defined>(ctx.config.delayLoadHelper);
-    delayIdata.create(helper);
+    delayIdata.create();
     for (Chunk *c : delayIdata.getChunks())
       didatSec->addChunk(c);
     for (Chunk *c : delayIdata.getDataChunks())
diff --git a/lld/Common/BPSectionOrdererBase.cpp b/lld/Common/BPSectionOrdererBase.cpp
deleted file mode 100644
index 7d26a5fb84483..0000000000000
--- a/lld/Common/BPSectionOrdererBase.cpp
+++ /dev/null
@@ -1,371 +0,0 @@
-//===- BPSectionOrdererBase.cpp -------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "lld/Common/BPSectionOrdererBase.h"
-#include "lld/Common/ErrorHandler.h"
-#include "llvm/ADT/DenseSet.h"
-#include "llvm/ADT/SetVector.h"
-#include "llvm/ADT/SmallSet.h"
-#include "llvm/ADT/StringMap.h"
-#include "llvm/ProfileData/InstrProfReader.h"
-#include "llvm/Support/BalancedPartitioning.h"
-#include "llvm/Support/TimeProfiler.h"
-#include "llvm/Support/VirtualFileSystem.h"
-
-#define DEBUG_TYPE "bp-section-orderer"
-
-using namespace llvm;
-using namespace lld;
-
-using UtilityNodes = SmallVector<BPFunctionNode::UtilityNodeT>;
-
-static SmallVector<std::pair<unsigned, UtilityNodes>> getUnsForCompression(
-    ArrayRef<const BPSectionBase *> sections,
-    const DenseMap<const void *, uint64_t> &sectionToIdx,
-    ArrayRef<unsigned> sectionIdxs,
-    DenseMap<unsigned, SmallVector<unsigned>> *duplicateSectionIdxs,
-    BPFunctionNode::UtilityNodeT &maxUN) {
-  TimeTraceScope timeScope("Build nodes for compression");
-
-  SmallVector<std::pair<unsigned, SmallVector<uint64_t>>> sectionHashes;
-  sectionHashes.reserve(sectionIdxs.size());
-  SmallVector<uint64_t> hashes;
-
-  for (unsigned sectionIdx : sectionIdxs) {
-    const auto *isec = sections[sectionIdx];
-    isec->getSectionHashes(hashes, sectionToIdx);
-    sectionHashes.emplace_back(sectionIdx, std::move(hashes));
-    hashes.clear();
-  }
-
-  DenseMap<uint64_t, unsigned> hashFrequency;
-  for (auto &[sectionIdx, hashes] : sectionHashes)
-    for (auto hash : hashes)
-      ++hashFrequency[hash];
-
-  if (duplicateSectionIdxs) {
-    // Merge sections that are nearly identical
-    SmallVector<std::pair<unsigned, SmallVector<uint64_t>>> newSectionHashes;
-    DenseMap<uint64_t, unsigned> wholeHashToSectionIdx;
-    for (auto &[sectionIdx, hashes] : sectionHashes) {
-      uint64_t wholeHash = 0;
-      for (auto hash : hashes)
-        if (hashFrequency[hash] > 5)
-          wholeHash ^= hash;
-      auto [it, wasInserted] =
-          wholeHashToSectionIdx.insert(std::make_pair(wholeHash, sectionIdx));
-      if (wasInserted) {
-        newSectionHashes.emplace_back(sectionIdx, hashes);
-      } else {
-        (*duplicateSectionIdxs)[it->getSecond()].push_back(sectionIdx);
-      }
-    }
-    sectionHashes = newSectionHashes;
-
-    // Recompute hash frequencies
-    hashFrequency.clear();
-    for (auto &[sectionIdx, hashes] : sectionHashes)
-      for (auto hash : hashes)
-        ++hashFrequency[hash];
-  }
-
-  // Filter rare and common hashes and assign each a unique utility node that
-  // doesn't conflict with the trace utility nodes
-  DenseMap<uint64_t, BPFunctionNode::UtilityNodeT> hashToUN;
-  for (auto &[hash, frequency] : hashFrequency) {
-    if (frequency <= 1 || frequency * 2 > sectionHashes.size())
-      continue;
-    hashToUN[hash] = ++maxUN;
-  }
-
-  SmallVector<std::pair<unsigned, UtilityNodes>> sectionUns;
-  for (auto &[sectionIdx, hashes] : sectionHashes) {
-    UtilityNodes uns;
-    for (auto &hash : hashes) {
-      auto it = hashToUN.find(hash);
-      if (it != hashToUN.end())
-        uns.push_back(it->second);
-    }
-    sectionUns.emplace_back(sectionIdx, uns);
-  }
-  return sectionUns;
-}
-
-llvm::DenseMap<const BPSectionBase *, int>
-BPSectionBase::reorderSectionsByBalancedPartitioning(
-    llvm::StringRef profilePath, bool forFunctionCompression,
-    bool forDataCompression, bool compressionSortStartupFunctions, bool verbose,
-    SmallVector<std::unique_ptr<BPSectionBase>> &inputSections) {
-  TimeTraceScope timeScope("Setup Balanced Partitioning");
-  SmallVector<const BPSectionBase *> sections;
-  DenseMap<const void *, uint64_t> sectionToIdx;
-  StringMap<DenseSet<unsigned>> symbolToSectionIdxs;
-
-  // Process input sections
-  for (const auto &isec : inputSections) {
-    unsigned sectionIdx = sections.size();
-    sectionToIdx.try_emplace(isec->getSection(), sectionIdx);
-    sections.emplace_back(isec.get());
-    for (auto &sym : isec->getSymbols())
-      symbolToSectionIdxs[sym->getName()].insert(sectionIdx);
-  }
-  StringMap<DenseSet<unsigned>> rootSymbolToSectionIdxs;
-  for (auto &entry : symbolToSectionIdxs) {
-    StringRef name = entry.getKey();
-    auto &sectionIdxs = entry.getValue();
-    name = BPSectionBase::getRootSymbol(name);
-    rootSymbolToSectionIdxs[name].insert(sectionIdxs.begin(),
-                                         sectionIdxs.end());
-    if (auto resolvedLinkageName =
-            sections[*sectionIdxs.begin()]->getResolvedLinkageName(name))
-      rootSymbolToSectionIdxs[resolvedLinkageName.value()].insert(
-          sectionIdxs.begin(), sectionIdxs.end());
-  }
-
-  BPFunctionNode::UtilityNodeT maxUN = 0;
-  DenseMap<unsigned, UtilityNodes> startupSectionIdxUNs;
-  // Used to define the initial order for startup functions.
-  DenseMap<unsigned, size_t> sectionIdxToTimestamp;
-  std::unique_ptr<InstrProfReader> reader;
-  if (!profilePath.empty()) {
-    auto fs = vfs::getRealFileSystem();
-    auto readerOrErr = InstrProfReader::create(profilePath, *fs);
-    lld::checkError(readerOrErr.takeError());
-
-    reader = std::move(readerOrErr.get());
-    for (auto &entry : *reader) {
-      // Read all entries
-      (void)entry;
-    }
-    auto &traces = reader->getTemporalProfTraces();
-
-    DenseMap<unsigned, BPFunctionNode::UtilityNodeT> sectionIdxToFirstUN;
-    for (size_t traceIdx = 0; traceIdx < traces.size(); traceIdx++) {
-      uint64_t currentSize = 0, cutoffSize = 1;
-      size_t cutoffTimestamp = 1;
-      auto &trace = traces[traceIdx].FunctionNameRefs;
-      for (size_t timestamp = 0; timestamp < trace.size(); timestamp++) {
-        auto [Filename, ParsedFuncName] = getParsedIRPGOName(
-            reader->getSymtab().getFuncOrVarName(trace[timestamp]));
-        ParsedFuncName = BPSectionBase::getRootSymbol(ParsedFuncName);
-
-        auto sectionIdxsIt = rootSymbolToSectionIdxs.find(ParsedFuncName);
-        if (sectionIdxsIt == rootSymbolToSectionIdxs.end())
-          continue;
-        auto &sectionIdxs = sectionIdxsIt->getValue();
-        // If the same symbol is found in multiple sections, they might be
-        // identical, so we arbitrarily use the size from the first section.
-        currentSize += sections[*sectionIdxs.begin()]->getSize();
-
-        // Since BalancedPartitioning is sensitive to the initial order, we need
-        // to explicitly define it to be ordered by earliest timestamp.
-        for (unsigned sectionIdx : sectionIdxs) {
-          auto [it, wasInserted] =
-              sectionIdxToTimestamp.try_emplace(sectionIdx, timestamp);
-          if (!wasInserted)
-            it->getSecond() = std::min<size_t>(it->getSecond(), timestamp);
-        }
-
-        if (timestamp >= cutoffTimestamp || currentSize >= cutoffSize) {
-          ++maxUN;
-          cutoffSize = 2 * currentSize;
-          cutoffTimestamp = 2 * cutoffTimestamp;
-        }
-        for (unsigned sectionIdx : sectionIdxs)
-          sectionIdxToFirstUN.try_emplace(sectionIdx, maxUN);
-      }
-      for (auto &[sectionIdx, firstUN] : sectionIdxToFirstUN)
-        for (auto un = firstUN; un <= maxUN; ++un)
-          startupSectionIdxUNs[sectionIdx].push_back(un);
-      ++maxUN;
-      sectionIdxToFirstUN.clear();
-    }
-  }
-
-  SmallVector<unsigned> sectionIdxsForFunctionCompression,
-      sectionIdxsForDataCompression;
-  for (unsigned sectionIdx = 0; sectionIdx < sections.size(); sectionIdx++) {
-    if (startupSectionIdxUNs.count(sectionIdx))
-      continue;
-    const auto *isec = sections[sectionIdx];
-    if (isec->isCodeSection()) {
-      if (forFunctionCompression)
-        sectionIdxsForFunctionCompression.push_back(sectionIdx);
-    } else {
-      if (forDataCompression)
-        sectionIdxsForDataCompression.push_back(sectionIdx);
-    }
-  }
-
-  if (compressionSortStartupFunctions) {
-    SmallVector<unsigned> startupIdxs;
-    for (auto &[sectionIdx, uns] : startupSectionIdxUNs)
-      startupIdxs.push_back(sectionIdx);
-    auto unsForStartupFunctionCompression =
-        getUnsForCompression(sections, sectionToIdx, startupIdxs,
-                             /*duplicateSectionIdxs=*/nullptr, maxUN);
-    for (auto &[sectionIdx, compressionUns] :
-         unsForStartupFunctionCompression) {
-      auto &uns = startupSectionIdxUNs[sectionIdx];
-      uns.append(compressionUns);
-      llvm::sort(uns);
-      uns.erase(std::unique(uns.begin(), uns.end()), uns.end());
-    }
-  }
-
-  // Map a section index (order directly) to a list of duplicate section indices
-  // (not ordered directly).
-  DenseMap<unsigned, SmallVector<unsigned>> duplicateSectionIdxs;
-  auto unsForFunctionCompression = getUnsForCompression(
-      sections, sectionToIdx, sectionIdxsForFunctionCompression,
-      &duplicateSectionIdxs, maxUN);
-  auto unsForDataCompression = getUnsForCompression(
-      sections, sectionToIdx, sectionIdxsForDataCompression,
-      &duplicateSectionIdxs, maxUN);
-
-  std::vector<BPFunctionNode> nodesForStartup, nodesForFunctionCompression,
-      nodesForDataCompression;
-  for (auto &[sectionIdx, uns] : startupSectionIdxUNs)
-    nodesForStartup.emplace_back(sectionIdx, uns);
-  for (auto &[sectionIdx, uns] : unsForFunctionCompression)
-    nodesForFunctionCompression.emplace_back(sectionIdx, uns);
-  for (auto &[sectionIdx, uns] : unsForDataCompression)
-    nodesForDataCompression.emplace_back(sectionIdx, uns);
-
-  // Use the first timestamp to define the initial order for startup nodes.
-  llvm::sort(nodesForStartup, [&sectionIdxToTimestamp](auto &L, auto &R) {
-    return std::make_pair(sectionIdxToTimestamp[L.Id], L.Id) <
-           std::make_pair(sectionIdxToTimestamp[R.Id], R.Id);
-  });
-  // Sort compression nodes by their Id (which is the section index) because the
-  // input linker order tends to be not bad.
-  llvm::sort(nodesForFunctionCompression,
-             [](auto &L, auto &R) { return L.Id < R.Id; });
-  llvm::sort(nodesForDataCompression,
-             [](auto &L, auto &R) { return L.Id < R.Id; });
-
-  {
-    TimeTraceScope timeScope("Balanced Partitioning");
-    BalancedPartitioningConfig config;
-    BalancedPartitioning bp(config);
-    bp.run(nodesForStartup);
-    bp.run(nodesForFunctionCompression);
-    bp.run(nodesForDataCompression);
-  }
-
-  unsigned numStartupSections = 0;
-  unsigned numCodeCompressionSections = 0;
-  unsigned numDuplicateCodeSections = 0;
-  unsigned numDataCompressionSections = 0;
-  unsigned numDuplicateDataSections = 0;
-  SetVector<const BPSectionBase *> orderedSections;
-  // Order startup functions,
-  for (auto &node : nodesForStartup) {
-    const auto *isec = sections[node.Id];
-    if (orderedSections.insert(isec))
-      ++numStartupSections;
-  }
-  // then functions for compression,
-  for (auto &node : nodesForFunctionCompression) {
-    const auto *isec = sections[node.Id];
-    if (orderedSections.insert(isec))
-      ++numCodeCompressionSections;
-
-    auto It = duplicateSectionIdxs.find(node.Id);
-    if (It == duplicateSectionIdxs.end())
-      continue;
-    for (auto dupSecIdx : It->getSecond()) {
-      const auto *dupIsec = sections[dupSecIdx];
-      if (orderedSections.insert(dupIsec))
-        ++numDuplicateCodeSections;
-    }
-  }
-  // then data for compression.
-  for (auto &node : nodesForDataCompression) {
-    const auto *isec = sections[node.Id];
-    if (orderedSections.insert(isec))
-      ++numDataCompressionSections;
-    auto It = duplicateSectionIdxs.find(node.Id);
-    if (It == duplicateSectionIdxs.end())
-      continue;
-    for (auto dupSecIdx : It->getSecond()) {
-      const auto *dupIsec = sections[dupSecIdx];
-      if (orderedSections.insert(dupIsec))
-        ++numDuplicateDataSections;
-    }
-  }
-
-  if (verbose) {
-    unsigned numTotalOrderedSections =
-        numStartupSections + numCodeCompressionSections +
-        numDuplicateCodeSections + numDataCompressionSections +
-        numDuplicateDataSections;
-    dbgs()
-        << "Ordered " << numTotalOrderedSections
-        << " sections using balanced partitioning:\n  Functions for startup: "
-        << numStartupSections
-        << "\n  Functions for compression: " << numCodeCompressionSections
-        << "\n  Duplicate functions: " << numDuplicateCodeSections
-        << "\n  Data for compression: " << numDataCompressionSections
-        << "\n  Duplicate data: " << numDuplicateDataSections << "\n";
-
-    if (!profilePath.empty()) {
-      // Evaluate this function order for startup
-      StringMap<std::pair<uint64_t, uint64_t>> symbolToPageNumbers;
-      const uint64_t pageSize = (1 << 14);
-      uint64_t currentAddress = 0;
-      for (const auto *isec : orderedSections) {
-        for (auto &sym : isec->getSymbols()) {
-          uint64_t startAddress = currentAddress + sym->getValue().value_or(0);
-          uint64_t endAddress = startAddress + sym->getSize().value_or(0);
-          uint64_t firstPage = startAddress / pageSize;
-          // I think the kernel might pull in a few pages when one it touched,
-          // so it might be more accurate to force lastPage to be aligned by
-          // 4?
-          uint64_t lastPage = endAddress / pageSize;
-          StringRef rootSymbol = sym->getName();
-          rootSymbol = BPSectionBase::getRootSymbol(rootSymbol);
-          symbolToPageNumbers.try_emplace(rootSymbol, firstPage, lastPage);
-          if (auto resolvedLinkageName =
-                  isec->getResolvedLinkageName(rootSymbol))
-            symbolToPageNumbers.try_emplace(resolvedLinkageName.value(),
-                                            firstPage, lastPage);
-        }
-        currentAddress += isec->getSize();
-      }
-
-      // The area under the curve F where F(t) is the total number of page
-      // faults at step t.
-      unsigned area = 0;
-      for (auto &trace : reader->getTemporalProfTraces()) {
-        SmallSet<uint64_t, 0> touchedPages;
-        for (unsigned step = 0; step < trace.FunctionNameRefs.size(); step++) {
-          auto traceId = trace.FunctionNameRefs[step];
-          auto [Filename, ParsedFuncName] =
-              getParsedIRPGOName(reader->getSymtab().getFuncOrVarName(traceId));
-          ParsedFuncName = BPSectionBase::getRootSymbol(ParsedFuncName);
-          auto it = symbolToPageNumbers.find(ParsedFuncName);
-          if (it != symbolToPageNumbers.end()) {
-            auto &[firstPage, lastPage] = it->getValue();
-            for (uint64_t i = firstPage; i <= lastPage; i++)
-              touchedPages.insert(i);
-          }
-          area += touchedPages.size();
-        }
-      }
-      dbgs() << "Total area under the page fault curve: " << (float)area
-             << "\n";
-    }
-  }
-
-  DenseMap<const BPSectionBase *, int> sectionPriorities;
-  int prio = -orderedSections.size();
-  for (const auto *isec : orderedSections)
-    sectionPriorities[isec] = prio++;
-  return sectionPriorities;
-}
diff --git a/lld/Common/CMakeLists.txt b/lld/Common/CMakeLists.txt
index 43e91b85821db..4f503d04f7844 100644
--- a/lld/Common/CMakeLists.txt
+++ b/lld/Common/CMakeLists.txt
@@ -24,7 +24,6 @@ set_source_files_properties("${version_inc}"
 
 add_lld_library(lldCommon
   Args.cpp
-  BPSectionOrdererBase.cpp
   CommonLinkerContext.cpp
   DriverDispatcher.cpp
   DWARF.cpp
@@ -48,7 +47,6 @@ add_lld_library(lldCommon
   Demangle
   MC
   Option
-  ProfileData
   Support
   Target
   TargetParser
diff --git a/lld/Common/ErrorHandler.cpp b/lld/Common/ErrorHandler.cpp
index 716bce54258ce..a11960325a9cd 100644
--- a/lld/Common/ErrorHandler.cpp
+++ b/lld/Common/ErrorHandler.cpp
@@ -289,7 +289,7 @@ void ErrorHandler::error(const Twine &msg) {
 
 void ErrorHandler::error(const Twine &msg, ErrorTag tag,
                          ArrayRef<StringRef> args) {
-  if (errorHandlingScript.empty()) {
+  if (errorHandlingScript.empty() || disableOutput) {
     error(msg);
     return;
   }
diff --git a/lld/ELF/Arch/ARM.cpp b/lld/ELF/Arch/ARM.cpp
index de6e45c6cc65c..7d2953ddf64f0 100644
--- a/lld/ELF/Arch/ARM.cpp
+++ b/lld/ELF/Arch/ARM.cpp
@@ -1536,8 +1536,8 @@ template <typename ELFT> void elf::writeARMCmseImportLib(Ctx &ctx) {
   }
 
   if (auto e = buffer->commit())
-    Fatal(ctx) << "failed to write output '" << buffer->getPath()
-               << "': " << std::move(e);
+    Err(ctx) << "failed to write output '" << buffer->getPath()
+             << "': " << std::move(e);
 }
 
 void elf::setARMTargetInfo(Ctx &ctx) { ctx.target.reset(new ARM(ctx)); }
diff --git a/lld/ELF/Arch/RISCV.cpp b/lld/ELF/Arch/RISCV.cpp
index 36ae31be6ed2a..4d8989a21b501 100644
--- a/lld/ELF/Arch/RISCV.cpp
+++ b/lld/ELF/Arch/RISCV.cpp
@@ -885,7 +885,7 @@ static bool relax(Ctx &ctx, InputSection &sec) {
   }
   // Inform assignAddresses that the size has changed.
   if (!isUInt<32>(delta))
-    Fatal(ctx) << "section size decrease is too large: " << delta;
+    Err(ctx) << "section size decrease is too large: " << delta;
   sec.bytesDropped = delta;
   return changed;
 }
diff --git a/lld/ELF/BPSectionOrderer.cpp b/lld/ELF/BPSectionOrderer.cpp
new file mode 100644
index 0000000000000..01f77b33926f7
--- /dev/null
+++ b/lld/ELF/BPSectionOrderer.cpp
@@ -0,0 +1,95 @@
+//===- BPSectionOrderer.cpp -----------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "BPSectionOrderer.h"
+#include "InputFiles.h"
+#include "InputSection.h"
+#include "SymbolTable.h"
+#include "Symbols.h"
+#include "lld/Common/BPSectionOrdererBase.inc"
+#include "llvm/Support/Endian.h"
+
+using namespace llvm;
+using namespace lld::elf;
+
+namespace {
+struct BPOrdererELF;
+}
+template <> struct lld::BPOrdererTraits<struct BPOrdererELF> {
+  using Section = elf::InputSectionBase;
+  using Defined = elf::Defined;
+};
+namespace {
+struct BPOrdererELF : lld::BPOrderer<BPOrdererELF> {
+  DenseMap<const InputSectionBase *, Defined *> secToSym;
+
+  static uint64_t getSize(const Section &sec) { return sec.getSize(); }
+  static bool isCodeSection(const Section &sec) {
+    return sec.flags & ELF::SHF_EXECINSTR;
+  }
+  ArrayRef<Defined *> getSymbols(const Section &sec) {
+    auto it = secToSym.find(&sec);
+    if (it == secToSym.end())
+      return {};
+    return ArrayRef(it->second);
+  }
+
+  static void
+  getSectionHashes(const Section &sec, SmallVectorImpl<uint64_t> &hashes,
+                   const DenseMap<const void *, uint64_t> &sectionToIdx) {
+    constexpr unsigned windowSize = 4;
+
+    // Calculate content hashes: k-mers and the last k-1 bytes.
+    ArrayRef<uint8_t> data = sec.content();
+    if (data.size() >= windowSize)
+      for (size_t i = 0; i <= data.size() - windowSize; ++i)
+        hashes.push_back(support::endian::read32le(data.data() + i));
+    for (uint8_t byte : data.take_back(windowSize - 1))
+      hashes.push_back(byte);
+
+    llvm::sort(hashes);
+    hashes.erase(std::unique(hashes.begin(), hashes.end()), hashes.end());
+  }
+
+  static StringRef getSymName(const Defined &sym) { return sym.getName(); }
+  static uint64_t getSymValue(const Defined &sym) { return sym.value; }
+  static uint64_t getSymSize(const Defined &sym) { return sym.size; }
+};
+} // namespace
+
+DenseMap<const InputSectionBase *, int> elf::runBalancedPartitioning(
+    Ctx &ctx, StringRef profilePath, bool forFunctionCompression,
+    bool forDataCompression, bool compressionSortStartupFunctions,
+    bool verbose) {
+  // Collect candidate sections and associated symbols.
+  SmallVector<InputSectionBase *> sections;
+  DenseMap<CachedHashStringRef, std::set<unsigned>> rootSymbolToSectionIdxs;
+  BPOrdererELF orderer;
+
+  auto addSection = [&](Symbol &sym) {
+    auto *d = dyn_cast<Defined>(&sym);
+    if (!d)
+      return;
+    auto *sec = dyn_cast_or_null<InputSectionBase>(d->section);
+    if (!sec || sec->size == 0 || !orderer.secToSym.try_emplace(sec, d).second)
+      return;
+    rootSymbolToSectionIdxs[CachedHashStringRef(getRootSymbol(sym.getName()))]
+        .insert(sections.size());
+    sections.emplace_back(sec);
+  };
+
+  for (Symbol *sym : ctx.symtab->getSymbols())
+    addSection(*sym);
+  for (ELFFileBase *file : ctx.objectFiles)
+    for (Symbol *sym : file->getLocalSymbols())
+      addSection(*sym);
+  return orderer.computeOrder(profilePath, forFunctionCompression,
+                              forDataCompression,
+                              compressionSortStartupFunctions, verbose,
+                              sections, rootSymbolToSectionIdxs);
+}
diff --git a/lld/ELF/BPSectionOrderer.h b/lld/ELF/BPSectionOrderer.h
new file mode 100644
index 0000000000000..a0cb1360005a6
--- /dev/null
+++ b/lld/ELF/BPSectionOrderer.h
@@ -0,0 +1,37 @@
+//===- BPSectionOrderer.h -------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// This file uses Balanced Partitioning to order sections to improve startup
+/// time and compressed size.
+///
+//===----------------------------------------------------------------------===//
+
+#ifndef LLD_ELF_BPSECTION_ORDERER_H
+#define LLD_ELF_BPSECTION_ORDERER_H
+
+#include "llvm/ADT/DenseMap.h"
+#include "llvm/ADT/StringRef.h"
+
+namespace lld::elf {
+struct Ctx;
+class InputSectionBase;
+
+/// Run Balanced Partitioning to find the optimal function and data order to
+/// improve startup time and compressed size.
+///
+/// It is important that -ffunction-sections and -fdata-sections compiler flags
+/// are used to ensure functions and data are in their own sections and thus
+/// can be reordered.
+llvm::DenseMap<const InputSectionBase *, int>
+runBalancedPartitioning(Ctx &ctx, llvm::StringRef profilePath,
+                        bool forFunctionCompression, bool forDataCompression,
+                        bool compressionSortStartupFunctions, bool verbose);
+
+} // namespace lld::elf
+
+#endif
diff --git a/lld/ELF/CMakeLists.txt b/lld/ELF/CMakeLists.txt
index 83d816ddb0601..ec3f6382282b1 100644
--- a/lld/ELF/CMakeLists.txt
+++ b/lld/ELF/CMakeLists.txt
@@ -37,6 +37,7 @@ add_lld_library(lldELF
   Arch/X86.cpp
   Arch/X86_64.cpp
   ARMErrataFix.cpp
+  BPSectionOrderer.cpp
   CallGraphSort.cpp
   DWARF.cpp
   Driver.cpp
@@ -72,6 +73,7 @@ add_lld_library(lldELF
   Object
   Option
   Passes
+  ProfileData
   Support
   TargetParser
   TransformUtils
diff --git a/lld/ELF/Config.h b/lld/ELF/Config.h
index b2859486d58e9..f132b11b20c63 100644
--- a/lld/ELF/Config.h
+++ b/lld/ELF/Config.h
@@ -264,6 +264,12 @@ struct Config {
   bool armBe8 = false;
   BsymbolicKind bsymbolic = BsymbolicKind::None;
   CGProfileSortKind callGraphProfileSort;
+  llvm::StringRef irpgoProfilePath;
+  bool bpStartupFunctionSort = false;
+  bool bpCompressionSortStartupFunctions = false;
+  bool bpFunctionOrderForCompression = false;
+  bool bpDataOrderForCompression = false;
+  bool bpVerboseSectionOrderer = false;
   bool checkSections;
   bool checkDynamicRelocs;
   std::optional<llvm::DebugCompressionType> compressDebugSections;
@@ -292,7 +298,6 @@ struct Config {
   bool gdbIndex;
   bool gnuHash = false;
   bool gnuUnique;
-  bool hasDynSymTab;
   bool ignoreDataAddressEquality;
   bool ignoreFunctionAddressEquality;
   bool ltoCSProfileGenerate;
@@ -306,7 +311,6 @@ struct Config {
   bool mipsN32Abi = false;
   bool mmapOutputFile;
   bool nmagic;
-  bool noDynamicLinker = false;
   bool noinhibitExec;
   bool nostdlib;
   bool oFormatBinary;
@@ -407,7 +411,7 @@ struct Config {
   StringRef thinLTOJobs;
   unsigned timeTraceGranularity;
   int32_t splitStackAdjustSize;
-  StringRef packageMetadata;
+  SmallVector<uint8_t, 0> packageMetadata;
 
   // The following config options do not directly correspond to any
   // particular command line options.
@@ -619,6 +623,7 @@ struct Ctx : CommonLinkerContext {
   };
   ElfSym sym{};
   std::unique_ptr<SymbolTable> symtab;
+  SmallVector<Symbol *, 0> synthesizedSymbols;
 
   SmallVector<std::unique_ptr<MemoryBuffer>> memoryBuffers;
   SmallVector<ELFFileBase *, 0> objectFiles;
@@ -654,6 +659,8 @@ struct Ctx : CommonLinkerContext {
   std::unique_ptr<llvm::TarWriter> tar;
   // InputFile for linker created symbols with no source location.
   InputFile *internalFile = nullptr;
+  // True if symbols can be exported (isExported) or preemptible.
+  bool hasDynsym = false;
   // True if SHT_LLVM_SYMPART is used.
   std::atomic<bool> hasSympart{false};
   // True if there are TLS IE relocations. Set DF_STATIC_TLS if -shared.
diff --git a/lld/ELF/Driver.cpp b/lld/ELF/Driver.cpp
index 13e8f8ce6df20..7d14180a49926 100644
--- a/lld/ELF/Driver.cpp
+++ b/lld/ELF/Driver.cpp
@@ -105,6 +105,13 @@ llvm::raw_fd_ostream Ctx::openAuxiliaryFile(llvm::StringRef filename,
   using namespace llvm::sys::fs;
   OpenFlags flags =
       auxiliaryFiles.insert(filename).second ? OF_None : OF_Append;
+  if (e.disableOutput && filename == "-") {
+#ifdef _WIN32
+    filename = "NUL";
+#else
+    filename = "/dev/null";
+#endif
+  }
   return {filename, ec, flags};
 }
 
@@ -364,6 +371,9 @@ static void checkOptions(Ctx &ctx) {
       if (!ctx.arg.cmseOutputLib.empty())
         ErrAlways(ctx) << "--out-implib may not be used without --cmse-implib";
     }
+    if (ctx.arg.fixCortexA8 && !ctx.arg.isLE)
+      ErrAlways(ctx)
+          << "--fix-cortex-a8 is not supported on big endian targets";
   } else {
     if (ctx.arg.cmseImplib)
       ErrAlways(ctx) << "--cmse-implib is only supported on ARM targets";
@@ -371,30 +381,46 @@ static void checkOptions(Ctx &ctx) {
       ErrAlways(ctx) << "--in-implib is only supported on ARM targets";
     if (!ctx.arg.cmseOutputLib.empty())
       ErrAlways(ctx) << "--out-implib is only supported on ARM targets";
+    if (ctx.arg.fixCortexA8)
+      ErrAlways(ctx) << "--fix-cortex-a8 is only supported on ARM targets";
+    if (ctx.arg.armBe8)
+      ErrAlways(ctx) << "--be8 is only supported on ARM targets";
   }
 
-  if (ctx.arg.fixCortexA53Errata843419 && ctx.arg.emachine != EM_AARCH64)
-    ErrAlways(ctx)
-        << "--fix-cortex-a53-843419 is only supported on AArch64 targets";
-
-  if (ctx.arg.fixCortexA8 && ctx.arg.emachine != EM_ARM)
-    ErrAlways(ctx) << "--fix-cortex-a8 is only supported on ARM targets";
-
-  if (ctx.arg.armBe8 && ctx.arg.emachine != EM_ARM)
-    ErrAlways(ctx) << "--be8 is only supported on ARM targets";
-
-  if (ctx.arg.fixCortexA8 && !ctx.arg.isLE)
-    ErrAlways(ctx) << "--fix-cortex-a8 is not supported on big endian targets";
-
-  if (ctx.arg.tocOptimize && ctx.arg.emachine != EM_PPC64)
-    ErrAlways(ctx) << "--toc-optimize is only supported on PowerPC64 targets";
+  if (ctx.arg.emachine != EM_AARCH64) {
+    if (ctx.arg.executeOnly)
+      ErrAlways(ctx) << "--execute-only is only supported on AArch64 targets";
+    if (ctx.arg.fixCortexA53Errata843419)
+      ErrAlways(ctx) << "--fix-cortex-a53-843419 is only supported on AArch64";
+    if (ctx.arg.zPacPlt)
+      ErrAlways(ctx) << "-z pac-plt only supported on AArch64";
+    if (ctx.arg.zForceBti)
+      ErrAlways(ctx) << "-z force-bti only supported on AArch64";
+    if (ctx.arg.zBtiReport != "none")
+      ErrAlways(ctx) << "-z bti-report only supported on AArch64";
+    if (ctx.arg.zPauthReport != "none")
+      ErrAlways(ctx) << "-z pauth-report only supported on AArch64";
+    if (ctx.arg.zGcsReport != "none")
+      ErrAlways(ctx) << "-z gcs-report only supported on AArch64";
+    if (ctx.arg.zGcs != GcsPolicy::Implicit)
+      ErrAlways(ctx) << "-z gcs only supported on AArch64";
+  }
 
-  if (ctx.arg.pcRelOptimize && ctx.arg.emachine != EM_PPC64)
-    ErrAlways(ctx) << "--pcrel-optimize is only supported on PowerPC64 targets";
+  if (ctx.arg.emachine != EM_PPC64) {
+    if (ctx.arg.tocOptimize)
+      ErrAlways(ctx) << "--toc-optimize is only supported on PowerPC64 targets";
+    if (ctx.arg.pcRelOptimize)
+      ErrAlways(ctx)
+          << "--pcrel-optimize is only supported on PowerPC64 targets";
+  }
 
   if (ctx.arg.relaxGP && ctx.arg.emachine != EM_RISCV)
     ErrAlways(ctx) << "--relax-gp is only supported on RISC-V targets";
 
+  if (ctx.arg.emachine != EM_386 && ctx.arg.emachine != EM_X86_64 &&
+      ctx.arg.zCetReport != "none")
+    ErrAlways(ctx) << "-z cet-report only supported on X86 and X86_64";
+
   if (ctx.arg.pie && ctx.arg.shared)
     ErrAlways(ctx) << "-shared and -pie may not be used together";
 
@@ -428,9 +454,6 @@ static void checkOptions(Ctx &ctx) {
   }
 
   if (ctx.arg.executeOnly) {
-    if (ctx.arg.emachine != EM_AARCH64)
-      ErrAlways(ctx) << "--execute-only is only supported on AArch64 targets";
-
     if (ctx.arg.singleRoRx && !ctx.script->hasSectionsCommand)
       ErrAlways(ctx)
           << "--execute-only and --no-rosegment cannot be used together";
@@ -438,25 +461,6 @@ static void checkOptions(Ctx &ctx) {
 
   if (ctx.arg.zRetpolineplt && ctx.arg.zForceIbt)
     ErrAlways(ctx) << "-z force-ibt may not be used with -z retpolineplt";
-
-  if (ctx.arg.emachine != EM_AARCH64) {
-    if (ctx.arg.zPacPlt)
-      ErrAlways(ctx) << "-z pac-plt only supported on AArch64";
-    if (ctx.arg.zForceBti)
-      ErrAlways(ctx) << "-z force-bti only supported on AArch64";
-    if (ctx.arg.zBtiReport != "none")
-      ErrAlways(ctx) << "-z bti-report only supported on AArch64";
-    if (ctx.arg.zPauthReport != "none")
-      ErrAlways(ctx) << "-z pauth-report only supported on AArch64";
-    if (ctx.arg.zGcsReport != "none")
-      ErrAlways(ctx) << "-z gcs-report only supported on AArch64";
-    if (ctx.arg.zGcs != GcsPolicy::Implicit)
-      ErrAlways(ctx) << "-z gcs only supported on AArch64";
-  }
-
-  if (ctx.arg.emachine != EM_386 && ctx.arg.emachine != EM_X86_64 &&
-      ctx.arg.zCetReport != "none")
-    ErrAlways(ctx) << "-z cet-report only supported on X86 and X86_64";
 }
 
 static const char *getReproduceOption(opt::InputArgList &args) {
@@ -774,11 +778,8 @@ static StringRef getDynamicLinker(Ctx &ctx, opt::InputArgList &args) {
   auto *arg = args.getLastArg(OPT_dynamic_linker, OPT_no_dynamic_linker);
   if (!arg)
     return "";
-  if (arg->getOption().getID() == OPT_no_dynamic_linker) {
-    // --no-dynamic-linker suppresses undefined weak symbols in .dynsym
-    ctx.arg.noDynamicLinker = true;
+  if (arg->getOption().getID() == OPT_no_dynamic_linker)
     return "";
-  }
   return arg->getValue();
 }
 
@@ -815,6 +816,26 @@ static ICFLevel getICF(opt::InputArgList &args) {
   return ICFLevel::All;
 }
 
+static void parsePackageMetadata(Ctx &ctx, const opt::Arg &arg) {
+  unsigned c0, c1;
+  SmallVector<uint8_t, 0> decoded;
+  StringRef s = arg.getValue();
+  for (size_t i = 0, e = s.size(); i != e; ++i) {
+    if (s[i] != '%') {
+      decoded.push_back(s[i]);
+    } else if (i + 2 < e && (c1 = hexDigitValue(s[i + 1])) != -1u &&
+               (c0 = hexDigitValue(s[i + 2])) != -1u) {
+      decoded.push_back(uint8_t(c1 * 16 + c0));
+      i += 2;
+    } else {
+      ErrAlways(ctx) << arg.getSpelling() << ": invalid % escape at byte " << i
+                     << "; supports only %[0-9a-fA-F][0-9a-fA-F]";
+      return;
+    }
+  }
+  ctx.arg.packageMetadata = std::move(decoded);
+}
+
 static StripPolicy getStrip(Ctx &ctx, opt::InputArgList &args) {
   if (args.hasArg(OPT_relocatable))
     return StripPolicy::None;
@@ -1114,6 +1135,53 @@ static CGProfileSortKind getCGProfileSortKind(Ctx &ctx,
   return CGProfileSortKind::None;
 }
 
+static void parseBPOrdererOptions(Ctx &ctx, opt::InputArgList &args) {
+  if (auto *arg = args.getLastArg(OPT_bp_compression_sort)) {
+    StringRef s = arg->getValue();
+    if (s == "function") {
+      ctx.arg.bpFunctionOrderForCompression = true;
+    } else if (s == "data") {
+      ctx.arg.bpDataOrderForCompression = true;
+    } else if (s == "both") {
+      ctx.arg.bpFunctionOrderForCompression = true;
+      ctx.arg.bpDataOrderForCompression = true;
+    } else if (s != "none") {
+      ErrAlways(ctx) << arg->getSpelling()
+                     << ": expected [none|function|data|both]";
+    }
+    if (s != "none" && args.hasArg(OPT_call_graph_ordering_file))
+      ErrAlways(ctx) << "--bp-compression-sort is incompatible with "
+                        "--call-graph-ordering-file";
+  }
+  if (auto *arg = args.getLastArg(OPT_bp_startup_sort)) {
+    StringRef s = arg->getValue();
+    if (s == "function") {
+      ctx.arg.bpStartupFunctionSort = true;
+    } else if (s != "none") {
+      ErrAlways(ctx) << arg->getSpelling() << ": expected [none|function]";
+    }
+    if (s != "none" && args.hasArg(OPT_call_graph_ordering_file))
+      ErrAlways(ctx) << "--bp-startup-sort=function is incompatible with "
+                        "--call-graph-ordering-file";
+  }
+
+  ctx.arg.bpCompressionSortStartupFunctions =
+      args.hasFlag(OPT_bp_compression_sort_startup_functions,
+                   OPT_no_bp_compression_sort_startup_functions, false);
+  ctx.arg.bpVerboseSectionOrderer = args.hasArg(OPT_verbose_bp_section_orderer);
+
+  ctx.arg.irpgoProfilePath = args.getLastArgValue(OPT_irpgo_profile);
+  if (ctx.arg.irpgoProfilePath.empty()) {
+    if (ctx.arg.bpStartupFunctionSort)
+      ErrAlways(ctx) << "--bp-startup-sort=function must be used with "
+                        "--irpgo-profile";
+    if (ctx.arg.bpCompressionSortStartupFunctions)
+      ErrAlways(ctx)
+          << "--bp-compression-sort-startup-functions must be used with "
+             "--irpgo-profile";
+  }
+}
+
 static DebugCompressionType getCompressionType(Ctx &ctx, StringRef s,
                                                StringRef option) {
   DebugCompressionType type = StringSwitch<DebugCompressionType>(s)
@@ -1255,6 +1323,7 @@ static void readConfigs(Ctx &ctx, opt::InputArgList &args) {
       ctx.arg.bsymbolic = BsymbolicKind::All;
   }
   ctx.arg.callGraphProfileSort = getCGProfileSortKind(ctx, args);
+  parseBPOrdererOptions(ctx, args);
   ctx.arg.checkSections =
       args.hasFlag(OPT_check_sections, OPT_no_check_sections, true);
   ctx.arg.chroot = args.getLastArgValue(OPT_chroot);
@@ -1376,7 +1445,8 @@ static void readConfigs(Ctx &ctx, opt::InputArgList &args) {
   ctx.arg.optimize = args::getInteger(args, OPT_O, 1);
   ctx.arg.orphanHandling = getOrphanHandling(ctx, args);
   ctx.arg.outputFile = args.getLastArgValue(OPT_o);
-  ctx.arg.packageMetadata = args.getLastArgValue(OPT_package_metadata);
+  if (auto *arg = args.getLastArg(OPT_package_metadata))
+    parsePackageMetadata(ctx, *arg);
   ctx.arg.pie = args.hasFlag(OPT_pie, OPT_no_pie, false);
   ctx.arg.printIcfSections =
       args.hasFlag(OPT_print_icf_sections, OPT_no_print_icf_sections, false);
@@ -1456,7 +1526,8 @@ static void readConfigs(Ctx &ctx, opt::InputArgList &args) {
   }
   ctx.arg.thinLTOModulesToCompile =
       args::getStrings(args, OPT_thinlto_single_module_eq);
-  ctx.arg.timeTraceEnabled = args.hasArg(OPT_time_trace_eq);
+  ctx.arg.timeTraceEnabled =
+      args.hasArg(OPT_time_trace_eq) && !ctx.e.disableOutput;
   ctx.arg.timeTraceGranularity =
       args::getInteger(args, OPT_time_trace_granularity, 500);
   ctx.arg.trace = args.hasArg(OPT_trace);
@@ -2149,9 +2220,12 @@ static void excludeLibs(Ctx &ctx, opt::InputArgList &args) {
     ArrayRef<Symbol *> symbols = file->getSymbols();
     if (isa<ELFFileBase>(file))
       symbols = cast<ELFFileBase>(file)->getGlobalSymbols();
-    for (Symbol *sym : symbols)
-      if (!sym->isUndefined() && sym->file == file)
+    for (Symbol *sym : symbols) {
+      if (!sym->isUndefined() && sym->file == file) {
         sym->versionId = VER_NDX_LOCAL;
+        sym->isExported = false;
+      }
+    }
   };
 
   for (ELFFileBase *file : ctx.objectFiles)
@@ -2402,7 +2476,7 @@ static void findKeepUniqueSections(Ctx &ctx, opt::InputArgList &args) {
   // or DSOs, so we conservatively mark them as address-significant.
   bool icfSafe = ctx.arg.icf == ICFLevel::Safe;
   for (Symbol *sym : ctx.symtab->getSymbols())
-    if (sym->includeInDynsym(ctx))
+    if (sym->isExported)
       markAddrsig(icfSafe, sym);
 
   // Visit the address-significance table in each object file and mark each
@@ -2418,8 +2492,10 @@ static void findKeepUniqueSections(Ctx &ctx, opt::InputArgList &args) {
         unsigned size;
         const char *err = nullptr;
         uint64_t symIndex = decodeULEB128(cur, &size, contents.end(), &err);
-        if (err)
-          Fatal(ctx) << f << ": could not decode addrsig section: " << err;
+        if (err) {
+          Err(ctx) << f << ": could not decode addrsig section: " << err;
+          break;
+        }
         markAddrsig(icfSafe, syms[symIndex]);
         cur += size;
       }
@@ -2535,11 +2611,18 @@ void LinkerDriver::compileBitcodeFiles(bool skipLinkedOutput) {
     auto *obj = cast<ObjFile<ELFT>>(file.get());
     obj->parse(/*ignoreComdats=*/true);
 
-    // Parse '@' in symbol names for non-relocatable output.
+    // For defined symbols in non-relocatable output,
+    // compute isExported and parse '@'.
     if (!ctx.arg.relocatable)
-      for (Symbol *sym : obj->getGlobalSymbols())
+      for (Symbol *sym : obj->getGlobalSymbols()) {
+        if (!sym->isDefined())
+          continue;
+        if (ctx.hasDynsym && ctx.arg.exportDynamic &&
+            sym->computeBinding(ctx) != STB_LOCAL)
+          sym->isExported = true;
         if (sym->hasVersionSuffix)
           sym->parseSymbolVersion(ctx);
+      }
     ctx.objectFiles.push_back(obj);
   }
 }
@@ -2881,7 +2964,7 @@ template <class ELFT> void LinkerDriver::link(opt::InputArgList &args) {
   parseFiles(ctx, files);
 
   // Create dynamic sections for dynamic linking and static PIE.
-  ctx.arg.hasDynSymTab = !ctx.sharedFiles.empty() || ctx.arg.isPic;
+  ctx.hasDynsym = !ctx.sharedFiles.empty() || ctx.arg.isPic;
 
   // If an entry symbol is in a static archive, pull out that file now.
   if (Symbol *sym = ctx.symtab->find(ctx.arg.entry))
@@ -3051,7 +3134,7 @@ template <class ELFT> void LinkerDriver::link(opt::InputArgList &args) {
 
   // Handle --exclude-libs again because lto.tmp may reference additional
   // libcalls symbols defined in an excluded archive. This may override
-  // versionId set by scanVersionScript().
+  // versionId set by scanVersionScript() and isExported.
   if (args.hasArg(OPT_exclude_libs))
     excludeLibs(ctx, args);
 
diff --git a/lld/ELF/EhFrame.cpp b/lld/ELF/EhFrame.cpp
index 6e0120e14988b..5d5a7bc0ab966 100644
--- a/lld/ELF/EhFrame.cpp
+++ b/lld/ELF/EhFrame.cpp
@@ -41,11 +41,10 @@ class EhReader {
   bool hasLSDA();
 
 private:
-  template <class P> void failOn(const P *loc, const Twine &msg) {
+  template <class P> void errOn(const P *loc, const Twine &msg) {
     Ctx &ctx = isec->file->ctx;
-    Fatal(ctx) << "corrupted .eh_frame: " << msg << "\n>>> defined in "
-               << isec->getObjMsg((const uint8_t *)loc -
-                                  isec->content().data());
+    Err(ctx) << "corrupted .eh_frame: " << msg << "\n>>> defined in "
+             << isec->getObjMsg((const uint8_t *)loc - isec->content().data());
   }
 
   uint8_t readByte();
@@ -62,8 +61,10 @@ class EhReader {
 
 // Read a byte and advance D by one byte.
 uint8_t EhReader::readByte() {
-  if (d.empty())
-    failOn(d.data(), "unexpected end of CIE");
+  if (d.empty()) {
+    errOn(d.data(), "unexpected end of CIE");
+    return 0;
+  }
   uint8_t b = d.front();
   d = d.slice(1);
   return b;
@@ -71,15 +72,18 @@ uint8_t EhReader::readByte() {
 
 void EhReader::skipBytes(size_t count) {
   if (d.size() < count)
-    failOn(d.data(), "CIE is too small");
-  d = d.slice(count);
+    errOn(d.data(), "CIE is too small");
+  else
+    d = d.slice(count);
 }
 
 // Read a null-terminated string.
 StringRef EhReader::readString() {
   const uint8_t *end = llvm::find(d, '\0');
-  if (end == d.end())
-    failOn(d.data(), "corrupted CIE (failed to read string)");
+  if (end == d.end()) {
+    errOn(d.data(), "corrupted CIE (failed to read string)");
+    return {};
+  }
   StringRef s = toStringRef(d.slice(0, end - d.begin()));
   d = d.slice(s.size() + 1);
   return s;
@@ -97,7 +101,7 @@ void EhReader::skipLeb128() {
     if ((val & 0x80) == 0)
       return;
   }
-  failOn(errPos, "corrupted CIE (failed to read LEB128)");
+  errOn(errPos, "corrupted CIE (failed to read LEB128)");
 }
 
 static size_t getAugPSize(Ctx &ctx, unsigned enc) {
@@ -121,12 +125,12 @@ static size_t getAugPSize(Ctx &ctx, unsigned enc) {
 void EhReader::skipAugP() {
   uint8_t enc = readByte();
   if ((enc & 0xf0) == DW_EH_PE_aligned)
-    failOn(d.data() - 1, "DW_EH_PE_aligned encoding is not supported");
+    return errOn(d.data() - 1, "DW_EH_PE_aligned encoding is not supported");
   size_t size = getAugPSize(isec->getCtx(), enc);
   if (size == 0)
-    failOn(d.data() - 1, "unknown FDE encoding");
+    return errOn(d.data() - 1, "unknown FDE encoding");
   if (size >= d.size())
-    failOn(d.data() - 1, "corrupted CIE");
+    return errOn(d.data() - 1, "corrupted CIE");
   d = d.slice(size);
 }
 
@@ -141,9 +145,11 @@ bool elf::hasLSDA(const EhSectionPiece &p) {
 StringRef EhReader::getAugmentation() {
   skipBytes(8);
   int version = readByte();
-  if (version != 1 && version != 3)
-    failOn(d.data() - 1,
-           "FDE version 1 or 3 expected, but got " + Twine(version));
+  if (version != 1 && version != 3) {
+    errOn(d.data() - 1,
+          "FDE version 1 or 3 expected, but got " + Twine(version));
+    return {};
+  }
 
   StringRef aug = readString();
 
@@ -174,8 +180,10 @@ uint8_t EhReader::getFdeEncoding() {
       readByte();
     else if (c == 'P')
       skipAugP();
-    else if (c != 'B' && c != 'S' && c != 'G')
-      failOn(aug.data(), "unknown .eh_frame augmentation string: " + aug);
+    else if (c != 'B' && c != 'S' && c != 'G') {
+      errOn(aug.data(), "unknown .eh_frame augmentation string: " + aug);
+      break;
+    }
   }
   return DW_EH_PE_absptr;
 }
@@ -191,8 +199,10 @@ bool EhReader::hasLSDA() {
       skipAugP();
     else if (c == 'R')
       readByte();
-    else if (c != 'B' && c != 'S' && c != 'G')
-      failOn(aug.data(), "unknown .eh_frame augmentation string: " + aug);
+    else if (c != 'B' && c != 'S' && c != 'G') {
+      errOn(aug.data(), "unknown .eh_frame augmentation string: " + aug);
+      break;
+    }
   }
   return false;
 }
diff --git a/lld/ELF/InputFiles.cpp b/lld/ELF/InputFiles.cpp
index c3c6812c26202..d43de8ce6dfef 100644
--- a/lld/ELF/InputFiles.cpp
+++ b/lld/ELF/InputFiles.cpp
@@ -639,6 +639,13 @@ template <class ELFT> void ObjFile<ELFT>::parse(bool ignoreComdats) {
       }
       break;
     case EM_AARCH64:
+      // FIXME: BuildAttributes have been implemented in llvm, but not yet in
+      // lld. Remove the section so that it does not accumulate in the output
+      // file. When support is implemented we expect not to output a build
+      // attributes section in files of type ET_EXEC or ET_SHARED, but ld -r
+      // ouptut will need a single merged attributes section.
+      if (sec.sh_type == SHT_AARCH64_ATTRIBUTES)
+        sections[i] = &InputSection::discarded;
       // Producing a static binary with MTE globals is not currently supported,
       // remove all SHT_AARCH64_MEMTAG_GLOBALS_STATIC sections as they're unused
       // medatada, and we don't want them to end up in the output file for
@@ -700,13 +707,12 @@ bool ObjFile<ELFT>::shouldMerge(const Elf_Shdr &sec, StringRef name) {
   if (entSize == 0)
     return false;
   if (sec.sh_size % entSize)
-    Fatal(ctx) << this << ":(" << name << "): SHF_MERGE section size ("
-               << uint64_t(sec.sh_size)
-               << ") must be a multiple of sh_entsize (" << entSize << ")";
-
+    ErrAlways(ctx) << this << ":(" << name << "): SHF_MERGE section size ("
+                   << uint64_t(sec.sh_size)
+                   << ") must be a multiple of sh_entsize (" << entSize << ")";
   if (sec.sh_flags & SHF_WRITE)
-    Fatal(ctx) << this << ":(" << name
-               << "): writable SHF_MERGE section is not supported";
+    Err(ctx) << this << ":(" << name
+             << "): writable SHF_MERGE section is not supported";
 
   return true;
 }
@@ -891,9 +897,11 @@ void ObjFile<ELFT>::initializeSections(bool ignoreComdats,
     InputSectionBase *linkSec = nullptr;
     if (sec.sh_link < size)
       linkSec = this->sections[sec.sh_link];
-    if (!linkSec)
-      Fatal(ctx) << this
-                 << ": invalid sh_link index: " << uint32_t(sec.sh_link);
+    if (!linkSec) {
+      ErrAlways(ctx) << this
+                     << ": invalid sh_link index: " << uint32_t(sec.sh_link);
+      continue;
+    }
 
     // A SHF_LINK_ORDER section is discarded if its linked-to section is
     // discarded.
@@ -922,17 +930,18 @@ static void readGnuProperty(Ctx &ctx, const InputSection &sec,
   using Elf_Note = typename ELFT::Note;
 
   ArrayRef<uint8_t> data = sec.content();
-  auto reportFatal = [&](const uint8_t *place, const Twine &msg) {
-    Fatal(ctx) << sec.file << ":(" << sec.name << "+0x"
-               << Twine::utohexstr(place - sec.content().data())
-               << "): " << msg;
+  auto err = [&](const uint8_t *place) -> ELFSyncStream {
+    auto diag = Err(ctx);
+    diag << sec.file << ":(" << sec.name << "+0x"
+         << Twine::utohexstr(place - sec.content().data()) << "): ";
+    return diag;
   };
   while (!data.empty()) {
     // Read one NOTE record.
     auto *nhdr = reinterpret_cast<const Elf_Nhdr *>(data.data());
     if (data.size() < sizeof(Elf_Nhdr) ||
         data.size() < nhdr->getSize(sec.addralign))
-      reportFatal(data.data(), "data is too short");
+      return void(err(data.data()) << "data is too short");
 
     Elf_Note note(*nhdr);
     if (nhdr->n_type != NT_GNU_PROPERTY_TYPE_0 || note.getName() != "GNU") {
@@ -949,30 +958,32 @@ static void readGnuProperty(Ctx &ctx, const InputSection &sec,
     while (!desc.empty()) {
       const uint8_t *place = desc.data();
       if (desc.size() < 8)
-        reportFatal(place, "program property is too short");
+        return void(err(place) << "program property is too short");
       uint32_t type = read32<ELFT::Endianness>(desc.data());
       uint32_t size = read32<ELFT::Endianness>(desc.data() + 4);
       desc = desc.slice(8);
       if (desc.size() < size)
-        reportFatal(place, "program property is too short");
+        return void(err(place) << "program property is too short");
 
       if (type == featureAndType) {
         // We found a FEATURE_1_AND field. There may be more than one of these
         // in a .note.gnu.property section, for a relocatable object we
         // accumulate the bits set.
         if (size < 4)
-          reportFatal(place, "FEATURE_1_AND entry is too short");
+          return void(err(place) << "FEATURE_1_AND entry is too short");
         f.andFeatures |= read32<ELFT::Endianness>(desc.data());
       } else if (ctx.arg.emachine == EM_AARCH64 &&
                  type == GNU_PROPERTY_AARCH64_FEATURE_PAUTH) {
         if (!f.aarch64PauthAbiCoreInfo.empty()) {
-          reportFatal(data.data(),
-                      "multiple GNU_PROPERTY_AARCH64_FEATURE_PAUTH entries are "
-                      "not supported");
+          return void(
+              err(data.data())
+              << "multiple GNU_PROPERTY_AARCH64_FEATURE_PAUTH entries are "
+                 "not supported");
         } else if (size != 16) {
-          reportFatal(data.data(), "GNU_PROPERTY_AARCH64_FEATURE_PAUTH entry "
-                                   "is invalid: expected 16 bytes, but got " +
-                                       Twine(size));
+          return void(err(data.data())
+                      << "GNU_PROPERTY_AARCH64_FEATURE_PAUTH entry "
+                         "is invalid: expected 16 bytes, but got "
+                      << size);
         }
         f.aarch64PauthAbiCoreInfo = desc;
       }
@@ -1127,8 +1138,8 @@ void ObjFile<ELFT>::initializeSymbols(const object::ELFFile<ELFT> &obj) {
     sym->isUsedInRegularObj = true;
     if (LLVM_UNLIKELY(eSym.st_shndx == SHN_COMMON)) {
       if (value == 0 || value >= UINT32_MAX)
-        Fatal(ctx) << this << ": common symbol '" << sym->getName()
-                   << "' has invalid alignment: " << value;
+        Err(ctx) << this << ": common symbol '" << sym->getName()
+                 << "' has invalid alignment: " << value;
       hasCommonSyms = true;
       sym->resolve(ctx, CommonSymbol{ctx, this, StringRef(), binding, stOther,
                                      type, value, size});
@@ -1174,8 +1185,10 @@ void ObjFile<ELFT>::initSectionsAndLocalSyms(bool ignoreComdats) {
       secIdx = check(getExtendedSymbolTableIndex<ELFT>(eSym, i, shndxTable));
     else if (secIdx >= SHN_LORESERVE)
       secIdx = 0;
-    if (LLVM_UNLIKELY(secIdx >= sections.size()))
-      Fatal(ctx) << this << ": invalid section index: " << secIdx;
+    if (LLVM_UNLIKELY(secIdx >= sections.size())) {
+      Err(ctx) << this << ": invalid section index: " << secIdx;
+      secIdx = 0;
+    }
     if (LLVM_UNLIKELY(eSym.getBinding() != STB_LOCAL))
       ErrAlways(ctx) << this << ": non-local symbol (" << i
                      << ") found at index < .symtab's sh_info (" << end << ")";
@@ -1184,9 +1197,12 @@ void ObjFile<ELFT>::initSectionsAndLocalSyms(bool ignoreComdats) {
     uint8_t type = eSym.getType();
     if (type == STT_FILE)
       sourceFile = CHECK2(eSym.getName(stringTable), this);
-    if (LLVM_UNLIKELY(stringTable.size() <= eSym.st_name))
-      Fatal(ctx) << this << ": invalid symbol name offset";
-    StringRef name(stringTable.data() + eSym.st_name);
+    unsigned stName = eSym.st_name;
+    if (LLVM_UNLIKELY(stringTable.size() <= stName)) {
+      Err(ctx) << this << ": invalid symbol name offset";
+      stName = 0;
+    }
+    StringRef name(stringTable.data() + stName);
 
     symbols[i] = reinterpret_cast<Symbol *>(locals + i);
     if (eSym.st_shndx == SHN_UNDEF || sec == &InputSection::discarded)
@@ -1237,8 +1253,10 @@ template <class ELFT> void ObjFile<ELFT>::postParse() {
         secIdx = 0;
     }
 
-    if (LLVM_UNLIKELY(secIdx >= sections.size()))
-      Fatal(ctx) << this << ": invalid section index: " << secIdx;
+    if (LLVM_UNLIKELY(secIdx >= sections.size())) {
+      Err(ctx) << this << ": invalid section index: " << secIdx;
+      continue;
+    }
     InputSectionBase *sec = sections[secIdx];
     if (sec == &InputSection::discarded) {
       if (sym.traced) {
@@ -1373,16 +1391,22 @@ std::vector<uint32_t> SharedFile::parseVerneed(const ELFFile<ELFT> &obj,
   ArrayRef<uint8_t> data = CHECK2(obj.getSectionContents(*sec), this);
   const uint8_t *verneedBuf = data.begin();
   for (unsigned i = 0; i != sec->sh_info; ++i) {
-    if (verneedBuf + sizeof(typename ELFT::Verneed) > data.end())
-      Fatal(ctx) << this << " has an invalid Verneed";
+    if (verneedBuf + sizeof(typename ELFT::Verneed) > data.end()) {
+      Err(ctx) << this << " has an invalid Verneed";
+      break;
+    }
     auto *vn = reinterpret_cast<const typename ELFT::Verneed *>(verneedBuf);
     const uint8_t *vernauxBuf = verneedBuf + vn->vn_aux;
     for (unsigned j = 0; j != vn->vn_cnt; ++j) {
-      if (vernauxBuf + sizeof(typename ELFT::Vernaux) > data.end())
-        Fatal(ctx) << this << " has an invalid Vernaux";
+      if (vernauxBuf + sizeof(typename ELFT::Vernaux) > data.end()) {
+        Err(ctx) << this << " has an invalid Vernaux";
+        break;
+      }
       auto *aux = reinterpret_cast<const typename ELFT::Vernaux *>(vernauxBuf);
-      if (aux->vna_name >= this->stringTable.size())
-        Fatal(ctx) << this << " has a Vernaux with an invalid vna_name";
+      if (aux->vna_name >= this->stringTable.size()) {
+        Err(ctx) << this << " has a Vernaux with an invalid vna_name";
+        break;
+      }
       uint16_t version = aux->vna_other & VERSYM_VERSION;
       if (version >= verneeds.size())
         verneeds.resize(version + 1);
@@ -1470,13 +1494,17 @@ template <class ELFT> void SharedFile::parse() {
   for (const Elf_Dyn &dyn : dynamicTags) {
     if (dyn.d_tag == DT_NEEDED) {
       uint64_t val = dyn.getVal();
-      if (val >= this->stringTable.size())
-        Fatal(ctx) << this << ": invalid DT_NEEDED entry";
+      if (val >= this->stringTable.size()) {
+        Err(ctx) << this << ": invalid DT_NEEDED entry";
+        return;
+      }
       dtNeeded.push_back(this->stringTable.data() + val);
     } else if (dyn.d_tag == DT_SONAME) {
       uint64_t val = dyn.getVal();
-      if (val >= this->stringTable.size())
-        Fatal(ctx) << this << ": invalid DT_SONAME entry";
+      if (val >= this->stringTable.size()) {
+        Err(ctx) << this << ": invalid DT_SONAME entry";
+        return;
+      }
       soName = this->stringTable.data() + val;
     }
   }
@@ -1553,7 +1581,7 @@ template <class ELFT> void SharedFile::parse() {
       }
       Symbol *s = ctx.symtab->addSymbol(
           Undefined{this, name, sym.getBinding(), sym.st_other, sym.getType()});
-      s->exportDynamic = true;
+      s->isExported = true;
       if (sym.getBinding() != STB_WEAK &&
           ctx.arg.unresolvedSymbolsInShlib != UnresolvedPolicy::Ignore)
         requiredSymbols.push_back(s);
@@ -1750,7 +1778,7 @@ static void createBitcodeSymbol(Ctx &ctx, Symbol *&sym,
                    nullptr);
     // The definition can be omitted if all bitcode definitions satisfy
     // `canBeOmittedFromSymbolTable()` and isUsedInRegularObj is false.
-    // The latter condition is tested in Symbol::includeInDynsym.
+    // The latter condition is tested in parseVersionAndComputeIsPreemptible.
     sym->ltoCanOmit = objSym.canBeOmittedFromSymbolTable() &&
                       (!sym->isDefined() || sym->ltoCanOmit);
     sym->resolve(ctx, newSym);
diff --git a/lld/ELF/InputSection.cpp b/lld/ELF/InputSection.cpp
index 42ef530b79d89..52c472bb89caf 100644
--- a/lld/ELF/InputSection.cpp
+++ b/lld/ELF/InputSection.cpp
@@ -71,8 +71,10 @@ InputSectionBase::InputSectionBase(InputFile *file, StringRef name,
   // The ELF spec states that a value of 0 means the section has
   // no alignment constraints.
   uint32_t v = std::max<uint32_t>(addralign, 1);
-  if (!isPowerOf2_64(v))
-    Fatal(getCtx()) << this << ": sh_addralign is not a power of 2";
+  if (!isPowerOf2_64(v)) {
+    Err(getCtx()) << this << ": sh_addralign is not a power of 2";
+    v = 1;
+  }
   this->addralign = v;
 
   // If SHF_COMPRESSED is set, parse the header. The legacy .zdebug format is no
@@ -104,8 +106,10 @@ InputSectionBase::InputSectionBase(ObjFile<ELFT> &file,
   // We reject object files having insanely large alignments even though
   // they are allowed by the spec. I think 4GB is a reasonable limitation.
   // We might want to relax this in the future.
-  if (hdr.sh_addralign > UINT32_MAX)
-    Fatal(getCtx()) << &file << ": section sh_addralign is too large";
+  if (hdr.sh_addralign > UINT32_MAX) {
+    Err(getCtx()) << &file << ": section sh_addralign is too large";
+    addralign = 1;
+  }
 }
 
 size_t InputSectionBase::getSize() const {
@@ -123,7 +127,7 @@ static void decompressAux(Ctx &ctx, const InputSectionBase &sec, uint8_t *out,
   if (Error e = hdr->ch_type == ELFCOMPRESS_ZLIB
                     ? compression::zlib::decompress(compressed, out, size)
                     : compression::zstd::decompress(compressed, out, size))
-    Fatal(ctx) << &sec << ": decompress failed: " << std::move(e);
+    Err(ctx) << &sec << ": decompress failed: " << std::move(e);
 }
 
 void InputSectionBase::decompress() const {
@@ -649,9 +653,11 @@ static uint64_t getRISCVUndefinedRelativeWeakVA(uint64_t type, uint64_t p) {
 // of the RW segment.
 static uint64_t getARMStaticBase(const Symbol &sym) {
   OutputSection *os = sym.getOutputSection();
-  if (!os || !os->ptLoad || !os->ptLoad->firstSec)
-    Fatal(os->ctx) << "SBREL relocation to " << sym.getName()
-                   << " without static base";
+  if (!os || !os->ptLoad || !os->ptLoad->firstSec) {
+    Err(os->ctx) << "SBREL relocation to " << sym.getName()
+                 << " without static base";
+    return 0;
+  }
   return os->ptLoad->firstSec->addr;
 }
 
@@ -1304,7 +1310,7 @@ template <class ELFT> void InputSection::writeTo(Ctx &ctx, uint8_t *buf) {
     if (Error e = hdr->ch_type == ELFCOMPRESS_ZLIB
                       ? compression::zlib::decompress(compressed, buf, size)
                       : compression::zstd::decompress(compressed, buf, size))
-      Fatal(ctx) << this << ": decompress failed: " << std::move(e);
+      Err(ctx) << this << ": decompress failed: " << std::move(e);
     uint8_t *bufEnd = buf + size;
     relocate<ELFT>(ctx, buf, bufEnd);
     return;
@@ -1427,8 +1433,11 @@ static size_t findNull(StringRef s, size_t entSize) {
 void MergeInputSection::splitStrings(StringRef s, size_t entSize) {
   const bool live = !(flags & SHF_ALLOC) || !getCtx().arg.gcSections;
   const char *p = s.data(), *end = s.data() + s.size();
-  if (!std::all_of(end - entSize, end, [](char c) { return c == 0; }))
-    Fatal(getCtx()) << this << ": string is not null terminated";
+  if (!std::all_of(end - entSize, end, [](char c) { return c == 0; })) {
+    Err(getCtx()) << this << ": string is not null terminated";
+    pieces.emplace_back(entSize, 0, false);
+    return;
+  }
   if (entSize == 1) {
     // Optimize the common case.
     do {
@@ -1488,8 +1497,10 @@ void MergeInputSection::splitIntoPieces() {
 }
 
 SectionPiece &MergeInputSection::getSectionPiece(uint64_t offset) {
-  if (content().size() <= offset)
-    Fatal(getCtx()) << this << ": offset is outside the section";
+  if (content().size() <= offset) {
+    Err(getCtx()) << this << ": offset is outside the section";
+    return pieces[0];
+  }
   return partition_point(
       pieces, [=](SectionPiece p) { return p.inputOff <= offset; })[-1];
 }
diff --git a/lld/ELF/Options.td b/lld/ELF/Options.td
index c31875305952f..b3b12a0646875 100644
--- a/lld/ELF/Options.td
+++ b/lld/ELF/Options.td
@@ -141,6 +141,19 @@ def call_graph_profile_sort: JJ<"call-graph-profile-sort=">,
 def : FF<"no-call-graph-profile-sort">, Alias<call_graph_profile_sort>, AliasArgs<["none"]>,
   Flags<[HelpHidden]>;
 
+defm irpgo_profile: EEq<"irpgo-profile",
+  "Read a temporary profile file for use with --bp-startup-sort=">;
+def bp_compression_sort: JJ<"bp-compression-sort=">, MetaVarName<"[none,function,data,both]">,
+  HelpText<"Improve Lempel-Ziv compression by grouping similar sections together, resulting in a smaller compressed app size">;
+def bp_startup_sort: JJ<"bp-startup-sort=">, MetaVarName<"[none,function]">,
+  HelpText<"Utilize a temporal profile file to reduce page faults during program startup">;
+
+// Auxiliary options related to balanced partition
+defm bp_compression_sort_startup_functions: BB<"bp-compression-sort-startup-functions",
+  "When --irpgo-profile is pecified, prioritize function similarity for compression in addition to startup time", "">;
+def verbose_bp_section_orderer: FF<"verbose-bp-section-orderer">,
+  HelpText<"Print information on balanced partitioning">;
+
 // --chroot doesn't have a help text because it is an internal option.
 def chroot: Separate<["--"], "chroot">;
 
@@ -565,7 +578,7 @@ def z: JoinedOrSeparate<["-"], "z">, MetaVarName<"<option>">,
 def visual_studio_diagnostics_format : FF<"vs-diagnostics">,
 HelpText<"Format diagnostics for Visual Studio compatibility">;
 
-def package_metadata: JJ<"package-metadata=">, HelpText<"Emit package metadata note">;
+def package_metadata: JJ<"package-metadata=">, HelpText<"Emit a percent-encoded string to the .note.package section">;
 
 // Aliases
 def: Separate<["-"], "dT">, Alias<default_script>, HelpText<"Alias for --default-script">;
diff --git a/lld/ELF/Relocations.cpp b/lld/ELF/Relocations.cpp
index 76b151b93d517..629702b45965b 100644
--- a/lld/ELF/Relocations.cpp
+++ b/lld/ELF/Relocations.cpp
@@ -428,8 +428,10 @@ class OffsetGetter {
     if (j == fdes.begin() || j[-1].inputOff + j[-1].size <= off) {
       while (i != cies.end() && i->inputOff <= off)
         ++i;
-      if (i == cies.begin() || i[-1].inputOff + i[-1].size <= off)
-        Fatal(ctx) << ".eh_frame: relocation is not in any piece";
+      if (i == cies.begin() || i[-1].inputOff + i[-1].size <= off) {
+        Err(ctx) << ".eh_frame: relocation is not in any piece";
+        return 0;
+      }
       it = i;
     }
 
diff --git a/lld/ELF/ScriptLexer.cpp b/lld/ELF/ScriptLexer.cpp
index e0adf2e5b0c8c..8db44f55505f8 100644
--- a/lld/ELF/ScriptLexer.cpp
+++ b/lld/ELF/ScriptLexer.cpp
@@ -105,7 +105,7 @@ void ScriptLexer::lex() {
       curBuf = buffers.pop_back_val();
       continue;
     }
-    curTokState = inExpr;
+    curTokState = lexState;
 
     // Quoted token. Note that double-quote characters are parts of a token
     // because, in a glob match context, only unquoted tokens are interpreted
@@ -142,7 +142,13 @@ void ScriptLexer::lex() {
     // C-like languages, so that you can write "file-name.cpp" as one bare
     // token.
     size_t pos;
-    if (inExpr) {
+    switch (lexState) {
+    case State::Script:
+      pos = s.find_first_not_of(
+          "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"
+          "0123456789_.$/\\~=+[]*?-!^:");
+      break;
+    case State::Expr:
       pos = s.find_first_not_of(
           "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"
           "0123456789_.$");
@@ -150,10 +156,7 @@ void ScriptLexer::lex() {
           ((s[0] == s[1] && strchr("<>&|", s[0])) ||
            is_contained({"==", "!=", "<=", ">=", "<<", ">>"}, s.substr(0, 2))))
         pos = 2;
-    } else {
-      pos = s.find_first_not_of(
-          "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"
-          "0123456789_.$/\\~=+[]*?-!^:");
+      break;
     }
 
     if (pos == 0)
@@ -208,8 +211,8 @@ StringRef ScriptLexer::next() {
 }
 
 StringRef ScriptLexer::peek() {
-  // curTok is invalid if curTokState and inExpr mismatch.
-  if (curTok.size() && curTokState != inExpr) {
+  // curTok is invalid if curTokState and lexState mismatch.
+  if (curTok.size() && curTokState != lexState) {
     curBuf.s = StringRef(curTok.data(), curBuf.s.end() - curTok.data());
     curTok = {};
   }
diff --git a/lld/ELF/ScriptLexer.h b/lld/ELF/ScriptLexer.h
index d689a7e108f93..be691022f5385 100644
--- a/lld/ELF/ScriptLexer.h
+++ b/lld/ELF/ScriptLexer.h
@@ -41,6 +41,11 @@ class ScriptLexer {
   // Used to detect INCLUDE() cycles.
   llvm::DenseSet<StringRef> activeFilenames;
 
+  enum class State {
+    Script,
+    Expr,
+  };
+
   struct Token {
     StringRef str;
     explicit operator bool() const { return !str.empty(); }
@@ -54,8 +59,9 @@ class ScriptLexer {
   // expression state changes.
   StringRef curTok;
   size_t prevTokLine = 1;
-  // The inExpr state when curTok is cached.
-  bool curTokState = false;
+  // The lex state when curTok is cached.
+  State curTokState = State::Script;
+  State lexState = State::Script;
   bool eof = false;
 
 public:
@@ -75,7 +81,6 @@ class ScriptLexer {
   MemoryBufferRef getCurrentMB();
 
   std::vector<MemoryBufferRef> mbs;
-  bool inExpr = false;
 
 private:
   StringRef getLine();
diff --git a/lld/ELF/ScriptParser.cpp b/lld/ELF/ScriptParser.cpp
index 06a22613ee93a..a10af9565a1d6 100644
--- a/lld/ELF/ScriptParser.cpp
+++ b/lld/ELF/ScriptParser.cpp
@@ -289,7 +289,7 @@ void ScriptParser::readLinkerScript() {
 void ScriptParser::readDefsym() {
   if (errCount(ctx))
     return;
-  inExpr = true;
+  SaveAndRestore saved(lexState, State::Expr);
   StringRef name = readName();
   expect("=");
   Expr e = readExpr();
@@ -954,8 +954,8 @@ bool ScriptParser::readSectionDirective(OutputSection *cmd, StringRef tok) {
 // https://sourceware.org/binutils/docs/ld/Output-Section-Type.html
 void ScriptParser::readSectionAddressType(OutputSection *cmd) {
   if (consume("(")) {
-    // Temporarily set inExpr to support TYPE=<value> without spaces.
-    SaveAndRestore saved(inExpr, true);
+    // Temporarily set lexState to support TYPE=<value> without spaces.
+    SaveAndRestore saved(lexState, State::Expr);
     if (readSectionDirective(cmd, peek()))
       return;
     cmd->addrExpr = readExpr();
@@ -965,7 +965,7 @@ void ScriptParser::readSectionAddressType(OutputSection *cmd) {
   }
 
   if (consume("(")) {
-    SaveAndRestore saved(inExpr, true);
+    SaveAndRestore saved(lexState, State::Expr);
     StringRef tok = peek();
     if (!readSectionDirective(cmd, tok))
       setError("unknown section directive: " + tok);
@@ -1087,10 +1087,10 @@ OutputDesc *ScriptParser::readOutputSectionDescription(StringRef outSec) {
   osec->phdrs = readOutputSectionPhdrs();
 
   if (peek() == "=" || peek().starts_with("=")) {
-    inExpr = true;
+    lexState = State::Expr;
     consume("=");
     osec->filler = readFill();
-    inExpr = false;
+    lexState = State::Script;
   }
 
   // Consume optional comma following output section command.
@@ -1162,7 +1162,7 @@ SymbolAssignment *ScriptParser::readAssignment(StringRef tok) {
   bool savedSeenRelroEnd = ctx.script->seenRelroEnd;
   const StringRef op = peek();
   {
-    SaveAndRestore saved(inExpr, true);
+    SaveAndRestore saved(lexState, State::Expr);
     if (op.starts_with("=")) {
       // Support = followed by an expression without whitespace.
       cmd = readSymbolAssignment(unquote(tok));
@@ -1235,7 +1235,7 @@ SymbolAssignment *ScriptParser::readSymbolAssignment(StringRef name) {
 Expr ScriptParser::readExpr() {
   // Our lexer is context-aware. Set the in-expression bit so that
   // they apply different tokenization rules.
-  SaveAndRestore saved(inExpr, true);
+  SaveAndRestore saved(lexState, State::Expr);
   Expr e = readExpr1(readPrimary(), 0);
   return e;
 }
@@ -1452,12 +1452,11 @@ std::pair<uint64_t, uint64_t> ScriptParser::readInputSectionFlags() {
 
 StringRef ScriptParser::readParenName() {
   expect("(");
-  bool orig = inExpr;
-  inExpr = false;
-  StringRef tok = readName();
-  inExpr = orig;
+  auto saved = std::exchange(lexState, State::Script);
+  StringRef name = readName();
+  lexState = saved;
   expect(")");
-  return tok;
+  return name;
 }
 
 static void checkIfExists(LinkerScript &script, const OutputSection &osec,
diff --git a/lld/ELF/SymbolTable.cpp b/lld/ELF/SymbolTable.cpp
index 975700505facb..b8a70d4e898fc 100644
--- a/lld/ELF/SymbolTable.cpp
+++ b/lld/ELF/SymbolTable.cpp
@@ -203,7 +203,7 @@ void SymbolTable::handleDynamicList() {
       syms = findByVersion(ver);
 
     for (Symbol *sym : syms)
-      sym->exportDynamic = sym->inDynamicList = true;
+      sym->isExported = sym->inDynamicList = true;
   }
 }
 
@@ -350,10 +350,8 @@ void SymbolTable::scanVersionScript() {
         assignAsterisk(pat, &v, true);
   }
 
-  // isPreemptible is false at this point. To correctly compute the binding of a
-  // Defined (which is used by includeInDynsym(ctx)), we need to know if it is
-  // VER_NDX_LOCAL or not. Compute symbol versions before handling
-  // --dynamic-list.
+  // Handle --dynamic-list. If a specified symbol is also matched by local: in a
+  // version script, the version script takes precedence.
   handleDynamicList();
 }
 
diff --git a/lld/ELF/Symbols.cpp b/lld/ELF/Symbols.cpp
index 497db842bc9be..80b0691428007 100644
--- a/lld/ELF/Symbols.cpp
+++ b/lld/ELF/Symbols.cpp
@@ -268,20 +268,6 @@ uint8_t Symbol::computeBinding(Ctx &ctx) const {
   return binding;
 }
 
-bool Symbol::includeInDynsym(Ctx &ctx) const {
-  if (computeBinding(ctx) == STB_LOCAL)
-    return false;
-  if (!isDefined() && !isCommon())
-    // This should unconditionally return true, unfortunately glibc -static-pie
-    // expects undefined weak symbols not to exist in .dynsym, e.g.
-    // __pthread_mutex_lock reference in _dl_add_to_namespace_list,
-    // __pthread_initialize_minimal reference in csu/libc-start.c.
-    return !(isUndefWeak() && ctx.arg.noDynamicLinker);
-
-  return exportDynamic ||
-         (ctx.arg.exportDynamic && (isUsedInRegularObj || !ltoCanOmit));
-}
-
 // Print out a log message for --trace-symbol.
 void elf::printTraceSymbol(const Symbol &sym, StringRef name) {
   std::string s;
@@ -374,13 +360,25 @@ void elf::parseVersionAndComputeIsPreemptible(Ctx &ctx) {
   // Symbol themselves might know their versions because symbols
   // can contain versions in the form of <name>@<version>.
   // Let them parse and update their names to exclude version suffix.
-  bool hasDynSymTab = ctx.arg.hasDynSymTab;
+  // In addition, compute isExported and isPreemptible.
+  bool hasDynsym = ctx.hasDynsym;
+  bool maybePreemptible = ctx.sharedFiles.size() || ctx.arg.shared;
   for (Symbol *sym : ctx.symtab->getSymbols()) {
     if (sym->hasVersionSuffix)
       sym->parseSymbolVersion(ctx);
-    sym->isExported = sym->includeInDynsym(ctx);
-    if (hasDynSymTab)
-      sym->isPreemptible = sym->isExported && computeIsPreemptible(ctx, *sym);
+    if (!hasDynsym)
+      continue;
+    if (sym->computeBinding(ctx) == STB_LOCAL) {
+      sym->isExported = false;
+      continue;
+    }
+    if (!sym->isDefined() && !sym->isCommon()) {
+      sym->isPreemptible = maybePreemptible && computeIsPreemptible(ctx, *sym);
+    } else if (ctx.arg.exportDynamic &&
+               (sym->isUsedInRegularObj || !sym->ltoCanOmit)) {
+      sym->isExported = true;
+      sym->isPreemptible = computeIsPreemptible(ctx, *sym);
+    }
   }
 }
 
@@ -658,7 +656,7 @@ void Symbol::resolve(Ctx &ctx, const LazySymbol &other) {
 }
 
 void Symbol::resolve(Ctx &ctx, const SharedSymbol &other) {
-  exportDynamic = true;
+  isExported = true;
   if (isPlaceholder()) {
     other.overwrite(*this);
     return;
diff --git a/lld/ELF/Symbols.h b/lld/ELF/Symbols.h
index 48df6f60db864..64f2f6eaa8d09 100644
--- a/lld/ELF/Symbols.h
+++ b/lld/ELF/Symbols.h
@@ -105,6 +105,9 @@ class Symbol {
   uint8_t partition;
 
   // True if this symbol is preemptible at load time.
+  //
+  // Primarily set in two locations, (a) parseVersionAndComputeIsPreemptible and
+  // (b) demoteSymbolsAndComputeIsPreemptible.
   LLVM_PREFERRED_TYPE(bool)
   uint8_t isPreemptible : 1;
 
@@ -131,16 +134,9 @@ class Symbol {
   // - If -shared or --export-dynamic is specified, any symbol in an object
   //   file/bitcode sets this property, unless suppressed by LTO
   //   canBeOmittedFromSymbolTable().
-  //
-  // Primarily set in two locations, (a) after parseSymbolVersion and
-  // (b) during demoteSymbols.
   LLVM_PREFERRED_TYPE(bool)
   uint8_t isExported : 1;
 
-  // Used to compute isExported. Set when defined or referenced by a SharedFile.
-  LLVM_PREFERRED_TYPE(bool)
-  uint8_t exportDynamic : 1;
-
   LLVM_PREFERRED_TYPE(bool)
   uint8_t ltoCanOmit : 1;
 
@@ -159,7 +155,6 @@ class Symbol {
     stOther = (stOther & ~3) | visibility;
   }
 
-  bool includeInDynsym(Ctx &) const;
   uint8_t computeBinding(Ctx &) const;
   bool isGlobal() const { return binding == llvm::ELF::STB_GLOBAL; }
   bool isWeak() const { return binding == llvm::ELF::STB_WEAK; }
@@ -247,8 +242,8 @@ class Symbol {
   Symbol(Kind k, InputFile *file, StringRef name, uint8_t binding,
          uint8_t stOther, uint8_t type)
       : file(file), nameData(name.data()), nameSize(name.size()), type(type),
-        binding(binding), stOther(stOther), symbolKind(k), exportDynamic(false),
-        ltoCanOmit(false), archSpecificBit(false) {}
+        binding(binding), stOther(stOther), symbolKind(k), ltoCanOmit(false),
+        archSpecificBit(false) {}
 
   void overwrite(Symbol &sym, Kind k) const {
     if (sym.traced)
diff --git a/lld/ELF/SyntheticSections.cpp b/lld/ELF/SyntheticSections.cpp
index eb07d82fc9601..b03c4282ab1aa 100644
--- a/lld/ELF/SyntheticSections.cpp
+++ b/lld/ELF/SyntheticSections.cpp
@@ -4740,7 +4740,7 @@ template <class ELFT> void elf::createSyntheticSections(Ctx &ctx) {
 
   // Add MIPS-specific sections.
   if (ctx.arg.emachine == EM_MIPS) {
-    if (!ctx.arg.shared && ctx.arg.hasDynSymTab) {
+    if (!ctx.arg.shared && ctx.hasDynsym) {
       ctx.in.mipsRldMap = std::make_unique<MipsRldMapSection>(ctx);
       add(*ctx.in.mipsRldMap);
     }
@@ -4776,8 +4776,8 @@ template <class ELFT> void elf::createSyntheticSections(Ctx &ctx) {
       add(*part.buildId);
     }
 
-    // dynSymTab is always present to simplify sym->includeInDynsym(ctx) in
-    // finalizeSections.
+    // dynSymTab is always present to simplify several finalizeSections
+    // functions.
     part.dynStrTab = std::make_unique<StringTableSection>(ctx, ".dynstr", true);
     part.dynSymTab =
         std::make_unique<SymbolTableSection<ELFT>>(ctx, *part.dynStrTab);
@@ -4803,7 +4803,7 @@ template <class ELFT> void elf::createSyntheticSections(Ctx &ctx) {
       part.relaDyn = std::make_unique<RelocationSection<ELFT>>(
           ctx, relaDynName, ctx.arg.zCombreloc, threadCount);
 
-    if (ctx.arg.hasDynSymTab) {
+    if (ctx.hasDynsym) {
       add(*part.dynSymTab);
 
       part.verSym = std::make_unique<VersionTableSection>(ctx);
diff --git a/lld/ELF/Writer.cpp b/lld/ELF/Writer.cpp
index fe4a0a15ae835..858f92c001158 100644
--- a/lld/ELF/Writer.cpp
+++ b/lld/ELF/Writer.cpp
@@ -9,6 +9,7 @@
 #include "Writer.h"
 #include "AArch64ErrataFix.h"
 #include "ARMErrataFix.h"
+#include "BPSectionOrderer.h"
 #include "CallGraphSort.h"
 #include "Config.h"
 #include "InputFiles.h"
@@ -149,6 +150,7 @@ static Defined *addOptionalRegular(Ctx &ctx, StringRef name, SectionBase *sec,
   if (!s || s->isDefined() || s->isCommon())
     return nullptr;
 
+  ctx.synthesizedSymbols.push_back(s);
   s->resolve(ctx, Defined{ctx, ctx.internalFile, StringRef(), STB_GLOBAL,
                           stOther, STT_NOTYPE, val,
                           /*size=*/0, sec});
@@ -282,6 +284,8 @@ static void demoteDefined(Defined &sym, DenseMap<SectionBase *, size_t> &map) {
 static void demoteSymbolsAndComputeIsPreemptible(Ctx &ctx) {
   llvm::TimeTraceScope timeScope("Demote symbols");
   DenseMap<InputFile *, DenseMap<SectionBase *, size_t>> sectionIndexMap;
+  bool hasDynsym = ctx.hasDynsym;
+  bool maybePreemptible = ctx.sharedFiles.size() || ctx.arg.shared;
   for (Symbol *sym : ctx.symtab->getSymbols()) {
     if (auto *d = dyn_cast<Defined>(sym)) {
       if (d->section && !d->section->isLive())
@@ -297,9 +301,10 @@ static void demoteSymbolsAndComputeIsPreemptible(Ctx &ctx) {
       }
     }
 
-    sym->isExported = sym->includeInDynsym(ctx);
-    if (ctx.arg.hasDynSymTab)
-      sym->isPreemptible = sym->isExported && computeIsPreemptible(ctx, *sym);
+    if (hasDynsym)
+      sym->isPreemptible = maybePreemptible &&
+                           (sym->isUndefined() || sym->isExported) &&
+                           computeIsPreemptible(ctx, *sym);
   }
 }
 
@@ -381,9 +386,11 @@ template <class ELFT> void Writer<ELFT>::run() {
     if (errCount(ctx))
       return;
 
-    if (auto e = buffer->commit())
-      Err(ctx) << "failed to write output '" << buffer->getPath()
-               << "': " << std::move(e);
+    if (!ctx.e.disableOutput) {
+      if (auto e = buffer->commit())
+        Err(ctx) << "failed to write output '" << buffer->getPath()
+                 << "': " << std::move(e);
+    }
 
     if (!ctx.arg.cmseOutputLib.empty())
       writeARMCmseImportLib<ELFT>(ctx);
@@ -645,7 +652,6 @@ static bool isRelroSection(Ctx &ctx, const OutputSection *sec) {
 enum RankFlags {
   RF_NOT_ADDR_SET = 1 << 27,
   RF_NOT_ALLOC = 1 << 26,
-  RF_HIP_FATBIN = 1 << 19,
   RF_PARTITION = 1 << 18, // Partition number (8 bits)
   RF_LARGE_ALT = 1 << 15,
   RF_WRITE = 1 << 14,
@@ -743,15 +749,6 @@ unsigned elf::getSectionRank(Ctx &ctx, OutputSection &osec) {
   if (osec.type == SHT_NOBITS)
     rank |= RF_BSS;
 
-  // Put HIP fatbin related sections further away to avoid wasting relocation
-  // range to jump over them.  Make sure .hip_fatbin is the furthest.
-  if (osec.name == ".hipFatBinSegment")
-    rank |= RF_HIP_FATBIN;
-  if (osec.name == ".hip_gpubin_handle")
-    rank |= RF_HIP_FATBIN | 2;
-  if (osec.name == ".hip_fatbin")
-    rank |= RF_HIP_FATBIN | RF_WRITE | 3;
-
   // Some architectures have additional ordering restrictions for sections
   // within the same PT_LOAD.
   if (ctx.arg.emachine == EM_PPC64) {
@@ -1086,8 +1083,18 @@ static void maybeShuffle(Ctx &ctx,
 // that don't appear in the order file.
 static DenseMap<const InputSectionBase *, int> buildSectionOrder(Ctx &ctx) {
   DenseMap<const InputSectionBase *, int> sectionOrder;
-  if (!ctx.arg.callGraphProfile.empty())
+  if (ctx.arg.bpStartupFunctionSort || ctx.arg.bpFunctionOrderForCompression ||
+      ctx.arg.bpDataOrderForCompression) {
+    TimeTraceScope timeScope("Balanced Partitioning Section Orderer");
+    sectionOrder = runBalancedPartitioning(
+        ctx, ctx.arg.bpStartupFunctionSort ? ctx.arg.irpgoProfilePath : "",
+        ctx.arg.bpFunctionOrderForCompression,
+        ctx.arg.bpDataOrderForCompression,
+        ctx.arg.bpCompressionSortStartupFunctions,
+        ctx.arg.bpVerboseSectionOrderer);
+  } else if (!ctx.arg.callGraphProfile.empty()) {
     sectionOrder = computeCallGraphProfileOrder(ctx);
+  }
 
   if (ctx.arg.symbolOrderingFile.empty())
     return sectionOrder;
@@ -1844,6 +1851,13 @@ template <class ELFT> void Writer<ELFT>::finalizeSections() {
     }
   }
 
+  // If the previous code block defines any non-hidden symbols (e.g.
+  // __global_pointer$), they may be exported.
+  if (ctx.hasDynsym && ctx.arg.exportDynamic)
+    for (Symbol *sym : ctx.synthesizedSymbols)
+      if (sym->computeBinding(ctx) != STB_LOCAL)
+        sym->isExported = true;
+
   demoteSymbolsAndComputeIsPreemptible(ctx);
 
   if (ctx.arg.copyRelocs && ctx.arg.discard != DiscardPolicy::None)
@@ -1928,9 +1942,9 @@ template <class ELFT> void Writer<ELFT>::finalizeSections() {
       if (ctx.in.symTab)
         ctx.in.symTab->addSymbol(sym);
 
-      // computeBinding might localize a linker-synthesized hidden symbol
-      // (e.g. __global_pointer$) that was considered exported.
-      if (sym->isExported && !sym->isLocal()) {
+      // computeBinding might localize a symbol that was considered exported
+      // but then synthesized as hidden (e.g. _DYNAMIC).
+      if ((sym->isExported || sym->isPreemptible) && !sym->isLocal()) {
         ctx.partitions[sym->partition - 1].dynSymTab->addSymbol(sym);
         if (auto *file = dyn_cast<SharedFile>(sym->file))
           if (file->isNeeded && !sym->isUndefined())
diff --git a/lld/MachO/BPSectionOrderer.cpp b/lld/MachO/BPSectionOrderer.cpp
index 18c8aad58344f..689afd67712a4 100644
--- a/lld/MachO/BPSectionOrderer.cpp
+++ b/lld/MachO/BPSectionOrderer.cpp
@@ -8,41 +8,131 @@
 
 #include "BPSectionOrderer.h"
 #include "InputSection.h"
+#include "Relocations.h"
+#include "Symbols.h"
+#include "lld/Common/BPSectionOrdererBase.inc"
 #include "llvm/ADT/DenseMap.h"
+#include "llvm/ADT/StableHashing.h"
+#include "llvm/Support/Endian.h"
+#include "llvm/Support/xxhash.h"
 
 #define DEBUG_TYPE "bp-section-orderer"
 
 using namespace llvm;
 using namespace lld::macho;
 
+namespace {
+struct BPOrdererMachO;
+}
+template <> struct lld::BPOrdererTraits<struct BPOrdererMachO> {
+  using Section = macho::InputSection;
+  using Defined = macho::Defined;
+};
+namespace {
+struct BPOrdererMachO : lld::BPOrderer<BPOrdererMachO> {
+  static uint64_t getSize(const Section &sec) { return sec.getSize(); }
+  static bool isCodeSection(const Section &sec) {
+    return macho::isCodeSection(&sec);
+  }
+  static ArrayRef<Defined *> getSymbols(const Section &sec) {
+    return sec.symbols;
+  }
+
+  // Linkage names can be prefixed with "_" or "l_" on Mach-O. See
+  // Mangler::getNameWithPrefix() for details.
+  std::optional<StringRef> static getResolvedLinkageName(llvm::StringRef name) {
+    if (name.consume_front("_") || name.consume_front("l_"))
+      return name;
+    return {};
+  }
+
+  static void
+  getSectionHashes(const Section &sec, llvm::SmallVectorImpl<uint64_t> &hashes,
+                   const llvm::DenseMap<const void *, uint64_t> &sectionToIdx) {
+    constexpr unsigned windowSize = 4;
+
+    // Calculate content hashes: k-mers and the last k-1 bytes.
+    ArrayRef<uint8_t> data = sec.data;
+    if (data.size() >= windowSize)
+      for (size_t i = 0; i <= data.size() - windowSize; ++i)
+        hashes.push_back(llvm::support::endian::read32le(data.data() + i));
+    for (uint8_t byte : data.take_back(windowSize - 1))
+      hashes.push_back(byte);
+
+    // Calculate relocation hashes
+    for (const auto &r : sec.relocs) {
+      if (r.length == 0 || r.referent.isNull() || r.offset >= data.size())
+        continue;
+
+      uint64_t relocHash = getRelocHash(r, sectionToIdx);
+      uint32_t start = (r.offset < windowSize) ? 0 : r.offset - windowSize + 1;
+      for (uint32_t i = start; i < r.offset + r.length; i++) {
+        auto window = data.drop_front(i).take_front(windowSize);
+        hashes.push_back(xxh3_64bits(window) ^ relocHash);
+      }
+    }
+
+    llvm::sort(hashes);
+    hashes.erase(std::unique(hashes.begin(), hashes.end()), hashes.end());
+  }
+
+  static llvm::StringRef getSymName(const Defined &sym) {
+    return sym.getName();
+  }
+  static uint64_t getSymValue(const Defined &sym) { return sym.value; }
+  static uint64_t getSymSize(const Defined &sym) { return sym.size; }
+
+private:
+  static uint64_t
+  getRelocHash(const Reloc &reloc,
+               const llvm::DenseMap<const void *, uint64_t> &sectionToIdx) {
+    auto *isec = reloc.getReferentInputSection();
+    std::optional<uint64_t> sectionIdx;
+    if (auto it = sectionToIdx.find(isec); it != sectionToIdx.end())
+      sectionIdx = it->second;
+    uint64_t kind = -1, value = 0;
+    if (isec)
+      kind = uint64_t(isec->kind());
+
+    if (auto *sym = reloc.referent.dyn_cast<Symbol *>()) {
+      kind = (kind << 8) | uint8_t(sym->kind());
+      if (auto *d = llvm::dyn_cast<Defined>(sym))
+        value = d->value;
+    }
+    return llvm::stable_hash_combine(kind, sectionIdx.value_or(0), value,
+                                     reloc.addend);
+  }
+};
+} // namespace
+
 DenseMap<const InputSection *, int> lld::macho::runBalancedPartitioning(
     StringRef profilePath, bool forFunctionCompression, bool forDataCompression,
     bool compressionSortStartupFunctions, bool verbose) {
-
-  SmallVector<std::unique_ptr<BPSectionBase>> sections;
+  // Collect candidate sections and associated symbols.
+  SmallVector<InputSection *> sections;
+  DenseMap<CachedHashStringRef, std::set<unsigned>> rootSymbolToSectionIdxs;
   for (const auto *file : inputFiles) {
     for (auto *sec : file->sections) {
       for (auto &subsec : sec->subsections) {
         auto *isec = subsec.isec;
-        if (!isec || isec->data.empty() || !isec->data.data())
+        if (!isec || isec->data.empty())
           continue;
-        sections.emplace_back(
-            std::make_unique<BPSectionMacho>(isec, sections.size()));
+        size_t idx = sections.size();
+        sections.emplace_back(isec);
+        for (auto *sym : BPOrdererMachO::getSymbols(*isec)) {
+          auto rootName = getRootSymbol(sym->getName());
+          rootSymbolToSectionIdxs[CachedHashStringRef(rootName)].insert(idx);
+          if (auto linkageName =
+                  BPOrdererMachO::getResolvedLinkageName(rootName))
+            rootSymbolToSectionIdxs[CachedHashStringRef(*linkageName)].insert(
+                idx);
+        }
       }
     }
   }
 
-  auto reorderedSections = BPSectionBase::reorderSectionsByBalancedPartitioning(
-      profilePath, forFunctionCompression, forDataCompression,
-      compressionSortStartupFunctions, verbose, sections);
-
-  DenseMap<const InputSection *, int> result;
-  for (const auto &[sec, priority] : reorderedSections) {
-    if (auto *machoSection = dyn_cast<BPSectionMacho>(sec)) {
-      result.try_emplace(
-          static_cast<const InputSection *>(machoSection->getSection()),
-          priority);
-    }
-  }
-  return result;
+  return BPOrdererMachO().computeOrder(profilePath, forFunctionCompression,
+                                       forDataCompression,
+                                       compressionSortStartupFunctions, verbose,
+                                       sections, rootSymbolToSectionIdxs);
 }
diff --git a/lld/MachO/BPSectionOrderer.h b/lld/MachO/BPSectionOrderer.h
index 69c6b260f044c..a27f605cb1180 100644
--- a/lld/MachO/BPSectionOrderer.h
+++ b/lld/MachO/BPSectionOrderer.h
@@ -14,140 +14,18 @@
 #ifndef LLD_MACHO_BPSECTION_ORDERER_H
 #define LLD_MACHO_BPSECTION_ORDERER_H
 
-#include "InputSection.h"
-#include "Relocations.h"
-#include "Symbols.h"
-#include "lld/Common/BPSectionOrdererBase.h"
 #include "llvm/ADT/DenseMap.h"
-#include "llvm/ADT/StableHashing.h"
 #include "llvm/ADT/StringRef.h"
-#include "llvm/Support/Endian.h"
-#include "llvm/Support/xxhash.h"
 
 namespace lld::macho {
-
 class InputSection;
 
-class BPSymbolMacho : public BPSymbol {
-  const Symbol *sym;
-
-public:
-  explicit BPSymbolMacho(const Symbol *s) : sym(s) {}
-
-  llvm::StringRef getName() const override { return sym->getName(); }
-
-  const Defined *asDefined() const {
-    return llvm::dyn_cast_or_null<Defined>(sym);
-  }
-
-  std::optional<uint64_t> getValue() const override {
-    if (auto *d = asDefined())
-      return d->value;
-    return {};
-  }
-
-  std::optional<uint64_t> getSize() const override {
-    if (auto *d = asDefined())
-      return d->size;
-    return {};
-  }
-
-  const Symbol *getSymbol() const { return sym; }
-};
-
-class BPSectionMacho : public BPSectionBase {
-  const InputSection *isec;
-  uint64_t sectionIdx;
-
-public:
-  explicit BPSectionMacho(const InputSection *sec, uint64_t sectionIdx)
-      : isec(sec), sectionIdx(sectionIdx) {}
-
-  const void *getSection() const override { return isec; }
-
-  uint64_t getSize() const override { return isec->getSize(); }
-
-  uint64_t getSectionIdx() const { return sectionIdx; }
-
-  bool isCodeSection() const override { return macho::isCodeSection(isec); }
-
-  SmallVector<std::unique_ptr<BPSymbol>> getSymbols() const override {
-    SmallVector<std::unique_ptr<BPSymbol>> symbols;
-    for (auto *sym : isec->symbols)
-      if (auto *d = llvm::dyn_cast_or_null<Defined>(sym))
-        symbols.emplace_back(std::make_unique<BPSymbolMacho>(d));
-    return symbols;
-  }
-
-  // Linkage names can be prefixed with "_" or "l_" on Mach-O. See
-  // Mangler::getNameWithPrefix() for details.
-  std::optional<StringRef>
-  getResolvedLinkageName(llvm::StringRef name) const override {
-    if (name.consume_front("_") || name.consume_front("l_"))
-      return name;
-    return {};
-  }
-
-  void getSectionHashes(llvm::SmallVectorImpl<uint64_t> &hashes,
-                        const llvm::DenseMap<const void *, uint64_t>
-                            &sectionToIdx) const override {
-    constexpr unsigned windowSize = 4;
-
-    // Calculate content hashes: k-mers and the last k-1 bytes.
-    ArrayRef<uint8_t> data = isec->data;
-    if (data.size() >= windowSize)
-      for (size_t i = 0; i <= data.size() - windowSize; ++i)
-        hashes.push_back(llvm::support::endian::read32le(data.data() + i));
-    for (uint8_t byte : data.take_back(windowSize - 1))
-      hashes.push_back(byte);
-
-    // Calculate relocation hashes
-    for (const auto &r : isec->relocs) {
-      if (r.length == 0 || r.referent.isNull() || r.offset >= data.size())
-        continue;
-
-      uint64_t relocHash = getRelocHash(r, sectionToIdx);
-      uint32_t start = (r.offset < windowSize) ? 0 : r.offset - windowSize + 1;
-      for (uint32_t i = start; i < r.offset + r.length; i++) {
-        auto window = data.drop_front(i).take_front(windowSize);
-        hashes.push_back(xxh3_64bits(window) ^ relocHash);
-      }
-    }
-
-    llvm::sort(hashes);
-    hashes.erase(std::unique(hashes.begin(), hashes.end()), hashes.end());
-  }
-
-  static bool classof(const BPSectionBase *s) { return true; }
-
-private:
-  static uint64_t
-  getRelocHash(const Reloc &reloc,
-               const llvm::DenseMap<const void *, uint64_t> &sectionToIdx) {
-    auto *isec = reloc.getReferentInputSection();
-    std::optional<uint64_t> sectionIdx;
-    if (auto it = sectionToIdx.find(isec); it != sectionToIdx.end())
-      sectionIdx = it->second;
-    uint64_t kind = -1, value = 0;
-    if (isec)
-      kind = uint64_t(isec->kind());
-
-    if (auto *sym = reloc.referent.dyn_cast<Symbol *>()) {
-      kind = (kind << 8) | uint8_t(sym->kind());
-      if (auto *d = llvm::dyn_cast<Defined>(sym))
-        value = d->value;
-    }
-    return llvm::stable_hash_combine(kind, sectionIdx.value_or(0), value,
-                                     reloc.addend);
-  }
-};
-
 /// Run Balanced Partitioning to find the optimal function and data order to
 /// improve startup time and compressed size.
 ///
 /// It is important that .subsections_via_symbols is used to ensure functions
 /// and data are in their own sections and thus can be reordered.
-llvm::DenseMap<const lld::macho::InputSection *, int>
+llvm::DenseMap<const InputSection *, int>
 runBalancedPartitioning(llvm::StringRef profilePath,
                         bool forFunctionCompression, bool forDataCompression,
                         bool compressionSortStartupFunctions, bool verbose);
diff --git a/lld/MachO/CMakeLists.txt b/lld/MachO/CMakeLists.txt
index c778fcf7b6fff..ecf6ce609e59f 100644
--- a/lld/MachO/CMakeLists.txt
+++ b/lld/MachO/CMakeLists.txt
@@ -50,6 +50,7 @@ add_lld_library(lldMachO
   Object
   Option
   Passes
+  ProfileData
   Support
   TargetParser
   TextAPI
diff --git a/lld/MachO/ObjC.cpp b/lld/MachO/ObjC.cpp
index 272197b34e115..fe9cee9651cca 100644
--- a/lld/MachO/ObjC.cpp
+++ b/lld/MachO/ObjC.cpp
@@ -543,7 +543,7 @@ ObjcCategoryMerger::tryGetSymbolAtIsecOffset(const ConcatInputSection *isec,
   if (!reloc)
     return nullptr;
 
-  Symbol *sym = reloc->referent.dyn_cast<Symbol *>();
+  Symbol *sym = dyn_cast_if_present<Symbol *>(reloc->referent);
 
   if (reloc->addend && sym) {
     assert(isa<Defined>(sym) && "Expected defined for non-zero addend");
diff --git a/lld/MachO/Relocations.cpp b/lld/MachO/Relocations.cpp
index aac0e1bd3c9e0..78cc13388d6eb 100644
--- a/lld/MachO/Relocations.cpp
+++ b/lld/MachO/Relocations.cpp
@@ -32,7 +32,7 @@ InputSection *Reloc::getReferentInputSection() const {
 }
 
 StringRef Reloc::getReferentString() const {
-  if (auto *isec = referent.dyn_cast<InputSection *>()) {
+  if (auto *isec = dyn_cast<InputSection *>(referent)) {
     const auto *cisec = dyn_cast<CStringInputSection>(isec);
     assert(cisec && "referent must be a CStringInputSection");
     return cisec->getStringRefAtOffset(addend);
diff --git a/lld/MachO/Writer.cpp b/lld/MachO/Writer.cpp
index bec980e18e18b..d9856a46e8cb8 100644
--- a/lld/MachO/Writer.cpp
+++ b/lld/MachO/Writer.cpp
@@ -711,7 +711,7 @@ void Writer::scanRelocations() {
 
       // Canonicalize the referent so that later accesses in Writer won't
       // have to worry about it.
-      if (auto *referentIsec = r.referent.dyn_cast<InputSection *>())
+      if (auto *referentIsec = dyn_cast_if_present<InputSection *>(r.referent))
         r.referent = referentIsec->canonical();
 
       if (target->hasAttr(r.type, RelocAttrBits::SUBTRAHEND)) {
@@ -725,7 +725,7 @@ void Writer::scanRelocations() {
           it->referent = referentIsec->canonical();
         continue;
       }
-      if (auto *sym = r.referent.dyn_cast<Symbol *>()) {
+      if (auto *sym = dyn_cast_if_present<Symbol *>(r.referent)) {
         if (auto *undefined = dyn_cast<Undefined>(sym))
           treatUndefinedSymbol(*undefined, isec, r.offset);
         // treatUndefinedSymbol() can replace sym with a DylibSymbol; re-check.
diff --git a/lld/docs/ReleaseNotes.rst b/lld/docs/ReleaseNotes.rst
index 1995c80df652c..6f60efd87c975 100644
--- a/lld/docs/ReleaseNotes.rst
+++ b/lld/docs/ReleaseNotes.rst
@@ -26,40 +26,14 @@ Non-comprehensive list of changes in this release
 ELF Improvements
 ----------------
 
-* ``-z nosectionheader`` has been implemented to omit the section header table.
-  The operation is similar to ``llvm-objcopy --strip-sections``.
-  (`#101286 <https://github.com/llvm/llvm-project/pull/101286>`_)
-* Section ``CLASS`` linker script syntax binds input sections to named classes,
-  which are referenced later one or more times. This provides access to the
-  automatic spilling mechanism of `--enable-non-contiguous-regions` without
-  globally changing the semantics of section matching. It also independently
-  increases the expressive power of linker scripts.
-  (`#95323 <https://github.com/llvm/llvm-project/pull/95323>`_)
-* Supported relocation types for x86-64 target:
-  * ``R_X86_64_CODE_4_GOTPCRELX`` (`#109783 <https://github.com/llvm/llvm-project/pull/109783>`_) (`#116737 <https://github.com/llvm/llvm-project/pull/116737>`_)
-  * ``R_X86_64_CODE_4_GOTTPOFF`` (`#116634 <https://github.com/llvm/llvm-project/pull/116634>`_)
-  * ``R_X86_64_CODE_4_GOTPC32_TLSDESC`` (`#116909 <https://github.com/llvm/llvm-project/pull/116909>`_)
-  * ``R_X86_64_CODE_6_GOTTPOFF``  (`#117675 <https://github.com/llvm/llvm-project/pull/117675>`_)
-* Supported relocation types for LoongArch target: ``R_LARCH_TLS_{LD,GD,DESC}_PCREL20_S2``.
-  (`#100105 <https://github.com/llvm/llvm-project/pull/100105>`_)
-
 Breaking changes
 ----------------
 
-* Removed support for the (deprecated) `R_RISCV_RVC_LUI` relocation. This
-  was a binutils-internal relocation used during relaxation, and was not
-  emitted by compilers/assemblers.
-
 COFF Improvements
 -----------------
-* ``/includeglob`` has been implemented to match the behavior of ``--undefined-glob`` available for ELF.
-* ``/lldsavetemps`` allows saving select intermediate LTO compilation results (e.g. resolution, preopt, promote, internalize, import, opt, precodegen, prelink, combinedindex).
-* ``/machine:arm64ec`` support completed, enabling the linking of ARM64EC images.
-* COFF weak anti-dependency alias symbols are now supported.
 
 MinGW Improvements
 ------------------
-* ``--undefined-glob`` is now supported by translating into the ``/includeglob`` flag.
 
 MachO Improvements
 ------------------
diff --git a/lld/docs/ld.lld.1 b/lld/docs/ld.lld.1
index b28c6082f68b0..b5c1816ce6e5f 100644
--- a/lld/docs/ld.lld.1
+++ b/lld/docs/ld.lld.1
@@ -493,6 +493,10 @@ If
 .Fl -use-android-relr-tags
 is specified, use SHT_ANDROID_RELR instead of SHT_RELR.
 .Pp
+.It Fl -package-metadata
+Emit a percent-encoded string to the
+.Cm .note.package
+section. For example, %25 decodes to a single %.
 .It Fl -pic-veneer
 Always generate position independent thunks.
 .It Fl -pie , Fl -pic-executable
diff --git a/lld/include/lld/Common/BPSectionOrdererBase.h b/lld/include/lld/Common/BPSectionOrdererBase.h
deleted file mode 100644
index bbd05edc5e55e..0000000000000
--- a/lld/include/lld/Common/BPSectionOrdererBase.h
+++ /dev/null
@@ -1,70 +0,0 @@
-//===- BPSectionOrdererBase.h ---------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-//
-// This file defines the common interfaces which may be used by
-// BPSectionOrderer.
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LLD_COMMON_BP_SECTION_ORDERER_BASE_H
-#define LLD_COMMON_BP_SECTION_ORDERER_BASE_H
-
-#include "llvm/ADT/DenseMap.h"
-#include "llvm/ADT/SmallVector.h"
-#include "llvm/ADT/StringRef.h"
-#include "llvm/ADT/Twine.h"
-#include <memory>
-#include <optional>
-
-namespace lld {
-
-class BPSymbol {
-
-public:
-  virtual ~BPSymbol() = default;
-  virtual llvm::StringRef getName() const = 0;
-  virtual std::optional<uint64_t> getValue() const = 0;
-  virtual std::optional<uint64_t> getSize() const = 0;
-};
-
-class BPSectionBase {
-public:
-  virtual ~BPSectionBase() = default;
-  virtual uint64_t getSize() const = 0;
-  virtual bool isCodeSection() const = 0;
-  virtual llvm::SmallVector<std::unique_ptr<BPSymbol>> getSymbols() const = 0;
-  virtual const void *getSection() const = 0;
-  virtual void getSectionHashes(
-      llvm::SmallVectorImpl<uint64_t> &hashes,
-      const llvm::DenseMap<const void *, uint64_t> &sectionToIdx) const = 0;
-  virtual std::optional<llvm::StringRef>
-  getResolvedLinkageName(llvm::StringRef name) const = 0;
-
-  /// Symbols can be appended with "(.__uniq.xxxx)?.llvm.yyyy" where "xxxx" and
-  /// "yyyy" are numbers that could change between builds. We need to use the
-  /// root symbol name before this suffix so these symbols can be matched with
-  /// profiles which may have different suffixes.
-  static llvm::StringRef getRootSymbol(llvm::StringRef Name) {
-    auto [P0, S0] = Name.rsplit(".llvm.");
-    auto [P1, S1] = P0.rsplit(".__uniq.");
-    return P1;
-  }
-
-  /// Reorders sections using balanced partitioning algorithm based on profile
-  /// data.
-  static llvm::DenseMap<const BPSectionBase *, int>
-  reorderSectionsByBalancedPartitioning(
-      llvm::StringRef profilePath, bool forFunctionCompression,
-      bool forDataCompression, bool compressionSortStartupFunctions,
-      bool verbose,
-      llvm::SmallVector<std::unique_ptr<BPSectionBase>> &inputSections);
-};
-
-} // namespace lld
-
-#endif
diff --git a/lld/include/lld/Common/BPSectionOrdererBase.inc b/lld/include/lld/Common/BPSectionOrdererBase.inc
new file mode 100644
index 0000000000000..83b87f4c7817d
--- /dev/null
+++ b/lld/include/lld/Common/BPSectionOrdererBase.inc
@@ -0,0 +1,413 @@
+//===- BPSectionOrdererBase.inc ---------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file defines the common BPSectionOrderer interface using the Curiously
+// Recurring Template Pattern and dispatches to the BalancedPartitioning
+// algorithm implemented in LLVMSupport. The optimized section layout attempts
+// to group similar sections together (resulting in a smaller compressed app
+// size) and utilize a temporal profile file to reduce page faults during
+// program startup.
+//
+// Clients should derive from BPOrderer, providing concrete implementations for
+// section and symbol representations. Include this file in a .cpp file to
+// specialize the template for the derived class.
+//
+//===----------------------------------------------------------------------===//
+
+#include "lld/Common/ErrorHandler.h"
+#include "llvm/ADT/CachedHashString.h"
+#include "llvm/ADT/DenseMap.h"
+#include "llvm/ADT/MapVector.h"
+#include "llvm/ADT/SetVector.h"
+#include "llvm/ADT/SmallSet.h"
+#include "llvm/ADT/SmallVector.h"
+#include "llvm/ADT/StringMap.h"
+#include "llvm/ADT/StringRef.h"
+#include "llvm/ADT/Twine.h"
+#include "llvm/ProfileData/InstrProfReader.h"
+#include "llvm/Support/BalancedPartitioning.h"
+#include "llvm/Support/TimeProfiler.h"
+#include "llvm/Support/VirtualFileSystem.h"
+#include <memory>
+#include <optional>
+#include <set>
+
+#define DEBUG_TYPE "bp-section-orderer"
+
+using namespace llvm;
+using namespace lld;
+
+namespace lld {
+template <class D> struct BPOrdererTraits;
+
+template <class D> struct BPOrderer {
+  using Section = typename BPOrdererTraits<D>::Section;
+  using Defined = typename BPOrdererTraits<D>::Defined;
+
+  // Compute a section order using the Balanced Partitioning algorithm.
+  //
+  // * for*Compresion: Improve Lempel-Ziv compression by grouping
+  //   similar sections together.
+  // * profilePath: Utilize a temporal profile file to reduce page faults during
+  //   program startup.
+  // * compressionSortStartupFunctions: if profilePath is specified, allocate
+  //   extra utility vertices to prioritize nearby function similarity.
+  auto computeOrder(llvm::StringRef profilePath, bool forFunctionCompression,
+                    bool forDataCompression,
+                    bool compressionSortStartupFunctions, bool verbose,
+                    llvm::ArrayRef<Section *> sections,
+                    const DenseMap<CachedHashStringRef, std::set<unsigned>>
+                        &rootSymbolToSectionIdxs)
+      -> llvm::DenseMap<const Section *, int>;
+
+  std::optional<StringRef> static getResolvedLinkageName(StringRef name) {
+    return {};
+  }
+};
+} // namespace lld
+
+using UtilityNodes = SmallVector<BPFunctionNode::UtilityNodeT>;
+
+template <class D>
+static SmallVector<std::pair<unsigned, UtilityNodes>> getUnsForCompression(
+    ArrayRef<const typename D::Section *> sections,
+    const DenseMap<const void *, uint64_t> &sectionToIdx,
+    ArrayRef<unsigned> sectionIdxs,
+    DenseMap<unsigned, SmallVector<unsigned, 0>> *duplicateSectionIdxs,
+    BPFunctionNode::UtilityNodeT &maxUN) {
+  TimeTraceScope timeScope("Build nodes for compression");
+
+  SmallVector<std::pair<unsigned, SmallVector<uint64_t>>> sectionHashes;
+  sectionHashes.reserve(sectionIdxs.size());
+  SmallVector<uint64_t> hashes;
+
+  for (unsigned sectionIdx : sectionIdxs) {
+    const auto *isec = sections[sectionIdx];
+    D::getSectionHashes(*isec, hashes, sectionToIdx);
+    sectionHashes.emplace_back(sectionIdx, std::move(hashes));
+    hashes.clear();
+  }
+
+  MapVector<uint64_t, unsigned> hashFrequency;
+  for (auto &[sectionIdx, hashes] : sectionHashes)
+    for (auto hash : hashes)
+      ++hashFrequency[hash];
+
+  if (duplicateSectionIdxs) {
+    // Merge sections that are nearly identical
+    SmallVector<std::pair<unsigned, SmallVector<uint64_t>>> newSectionHashes;
+    DenseMap<uint64_t, unsigned> wholeHashToSectionIdx;
+    unsigned threshold = sectionHashes.size() > 10000 ? 5 : 0;
+    for (auto &[sectionIdx, hashes] : sectionHashes) {
+      uint64_t wholeHash = 0;
+      for (auto hash : hashes)
+        if (hashFrequency[hash] > threshold)
+          wholeHash ^= hash;
+      auto [it, wasInserted] =
+          wholeHashToSectionIdx.insert(std::make_pair(wholeHash, sectionIdx));
+      if (wasInserted) {
+        newSectionHashes.emplace_back(sectionIdx, hashes);
+      } else {
+        (*duplicateSectionIdxs)[it->getSecond()].push_back(sectionIdx);
+      }
+    }
+    sectionHashes = newSectionHashes;
+
+    // Recompute hash frequencies
+    hashFrequency.clear();
+    for (auto &[sectionIdx, hashes] : sectionHashes)
+      for (auto hash : hashes)
+        ++hashFrequency[hash];
+  }
+
+  // Filter rare and common hashes and assign each a unique utility node that
+  // doesn't conflict with the trace utility nodes
+  DenseMap<uint64_t, BPFunctionNode::UtilityNodeT> hashToUN;
+  for (auto &[hash, frequency] : hashFrequency) {
+    if (frequency <= 1 || frequency * 2 > sectionHashes.size())
+      continue;
+    hashToUN[hash] = ++maxUN;
+  }
+
+  SmallVector<std::pair<unsigned, UtilityNodes>> sectionUns;
+  for (auto &[sectionIdx, hashes] : sectionHashes) {
+    UtilityNodes uns;
+    for (auto &hash : hashes) {
+      auto it = hashToUN.find(hash);
+      if (it != hashToUN.end())
+        uns.push_back(it->second);
+    }
+    sectionUns.emplace_back(sectionIdx, uns);
+  }
+  return sectionUns;
+}
+
+/// Symbols can be appended with "(.__uniq.xxxx)?.llvm.yyyy" where "xxxx" and
+/// "yyyy" are numbers that could change between builds. We need to use the
+/// root symbol name before this suffix so these symbols can be matched with
+/// profiles which may have different suffixes.
+inline StringRef getRootSymbol(StringRef name) {
+  auto [P0, S0] = name.rsplit(".llvm.");
+  auto [P1, S1] = P0.rsplit(".__uniq.");
+  return P1;
+}
+
+template <class D>
+auto BPOrderer<D>::computeOrder(
+    StringRef profilePath, bool forFunctionCompression, bool forDataCompression,
+    bool compressionSortStartupFunctions, bool verbose,
+    ArrayRef<Section *> sections,
+    const DenseMap<CachedHashStringRef, std::set<unsigned>>
+        &rootSymbolToSectionIdxs) -> DenseMap<const Section *, int> {
+  TimeTraceScope timeScope("Setup Balanced Partitioning");
+  DenseMap<const void *, uint64_t> sectionToIdx;
+  for (auto [i, isec] : llvm::enumerate(sections))
+    sectionToIdx.try_emplace(isec, i);
+
+  BPFunctionNode::UtilityNodeT maxUN = 0;
+  DenseMap<unsigned, UtilityNodes> startupSectionIdxUNs;
+  // Used to define the initial order for startup functions.
+  DenseMap<unsigned, size_t> sectionIdxToTimestamp;
+  std::unique_ptr<InstrProfReader> reader;
+  if (!profilePath.empty()) {
+    auto fs = vfs::getRealFileSystem();
+    auto readerOrErr = InstrProfReader::create(profilePath, *fs);
+    lld::checkError(readerOrErr.takeError());
+
+    reader = std::move(readerOrErr.get());
+    for (auto &entry : *reader) {
+      // Read all entries
+      (void)entry;
+    }
+    auto &traces = reader->getTemporalProfTraces();
+
+    DenseMap<unsigned, BPFunctionNode::UtilityNodeT> sectionIdxToFirstUN;
+    for (size_t traceIdx = 0; traceIdx < traces.size(); traceIdx++) {
+      uint64_t currentSize = 0, cutoffSize = 1;
+      size_t cutoffTimestamp = 1;
+      auto &trace = traces[traceIdx].FunctionNameRefs;
+      for (size_t timestamp = 0; timestamp < trace.size(); timestamp++) {
+        auto [_, parsedFuncName] = getParsedIRPGOName(
+            reader->getSymtab().getFuncOrVarName(trace[timestamp]));
+        parsedFuncName = getRootSymbol(parsedFuncName);
+
+        auto sectionIdxsIt =
+            rootSymbolToSectionIdxs.find(CachedHashStringRef(parsedFuncName));
+        if (sectionIdxsIt == rootSymbolToSectionIdxs.end())
+          continue;
+        auto &sectionIdxs = sectionIdxsIt->second;
+        // If the same symbol is found in multiple sections, they might be
+        // identical, so we arbitrarily use the size from the first section.
+        currentSize += D::getSize(*sections[*sectionIdxs.begin()]);
+
+        // Since BalancedPartitioning is sensitive to the initial order, we need
+        // to explicitly define it to be ordered by earliest timestamp.
+        for (unsigned sectionIdx : sectionIdxs) {
+          auto [it, wasInserted] =
+              sectionIdxToTimestamp.try_emplace(sectionIdx, timestamp);
+          if (!wasInserted)
+            it->getSecond() = std::min<size_t>(it->getSecond(), timestamp);
+        }
+
+        if (timestamp >= cutoffTimestamp || currentSize >= cutoffSize) {
+          ++maxUN;
+          cutoffSize = 2 * currentSize;
+          cutoffTimestamp = 2 * cutoffTimestamp;
+        }
+        for (unsigned sectionIdx : sectionIdxs)
+          sectionIdxToFirstUN.try_emplace(sectionIdx, maxUN);
+      }
+      for (auto &[sectionIdx, firstUN] : sectionIdxToFirstUN)
+        for (auto un = firstUN; un <= maxUN; ++un)
+          startupSectionIdxUNs[sectionIdx].push_back(un);
+      ++maxUN;
+      sectionIdxToFirstUN.clear();
+    }
+  }
+
+  SmallVector<unsigned> sectionIdxsForFunctionCompression,
+      sectionIdxsForDataCompression;
+  for (unsigned sectionIdx = 0; sectionIdx < sections.size(); sectionIdx++) {
+    if (startupSectionIdxUNs.count(sectionIdx))
+      continue;
+    const auto *isec = sections[sectionIdx];
+    if (D::isCodeSection(*isec)) {
+      if (forFunctionCompression)
+        sectionIdxsForFunctionCompression.push_back(sectionIdx);
+    } else {
+      if (forDataCompression)
+        sectionIdxsForDataCompression.push_back(sectionIdx);
+    }
+  }
+
+  if (compressionSortStartupFunctions) {
+    SmallVector<unsigned> startupIdxs;
+    for (auto &[sectionIdx, uns] : startupSectionIdxUNs)
+      startupIdxs.push_back(sectionIdx);
+    auto unsForStartupFunctionCompression =
+        getUnsForCompression<D>(sections, sectionToIdx, startupIdxs,
+                                /*duplicateSectionIdxs=*/nullptr, maxUN);
+    for (auto &[sectionIdx, compressionUns] :
+         unsForStartupFunctionCompression) {
+      auto &uns = startupSectionIdxUNs[sectionIdx];
+      uns.append(compressionUns);
+      llvm::sort(uns);
+      uns.erase(std::unique(uns.begin(), uns.end()), uns.end());
+    }
+  }
+
+  // Map a section index (order directly) to a list of duplicate section indices
+  // (not ordered directly).
+  DenseMap<unsigned, SmallVector<unsigned, 0>> duplicateSectionIdxs;
+  auto unsForFunctionCompression = getUnsForCompression<D>(
+      sections, sectionToIdx, sectionIdxsForFunctionCompression,
+      &duplicateSectionIdxs, maxUN);
+  auto unsForDataCompression = getUnsForCompression<D>(
+      sections, sectionToIdx, sectionIdxsForDataCompression,
+      &duplicateSectionIdxs, maxUN);
+
+  std::vector<BPFunctionNode> nodesForStartup, nodesForFunctionCompression,
+      nodesForDataCompression;
+  for (auto &[sectionIdx, uns] : startupSectionIdxUNs)
+    nodesForStartup.emplace_back(sectionIdx, uns);
+  for (auto &[sectionIdx, uns] : unsForFunctionCompression)
+    nodesForFunctionCompression.emplace_back(sectionIdx, uns);
+  for (auto &[sectionIdx, uns] : unsForDataCompression)
+    nodesForDataCompression.emplace_back(sectionIdx, uns);
+
+  // Use the first timestamp to define the initial order for startup nodes.
+  llvm::sort(nodesForStartup, [&sectionIdxToTimestamp](auto &L, auto &R) {
+    return std::make_pair(sectionIdxToTimestamp[L.Id], L.Id) <
+           std::make_pair(sectionIdxToTimestamp[R.Id], R.Id);
+  });
+  // Sort compression nodes by their Id (which is the section index) because the
+  // input linker order tends to be not bad.
+  llvm::sort(nodesForFunctionCompression,
+             [](auto &L, auto &R) { return L.Id < R.Id; });
+  llvm::sort(nodesForDataCompression,
+             [](auto &L, auto &R) { return L.Id < R.Id; });
+
+  {
+    TimeTraceScope timeScope("Balanced Partitioning");
+    BalancedPartitioningConfig config;
+    BalancedPartitioning bp(config);
+    bp.run(nodesForStartup);
+    bp.run(nodesForFunctionCompression);
+    bp.run(nodesForDataCompression);
+  }
+
+  unsigned numStartupSections = 0;
+  unsigned numCodeCompressionSections = 0;
+  unsigned numDuplicateCodeSections = 0;
+  unsigned numDataCompressionSections = 0;
+  unsigned numDuplicateDataSections = 0;
+  SetVector<const Section *> orderedSections;
+  // Order startup functions,
+  for (auto &node : nodesForStartup) {
+    const auto *isec = sections[node.Id];
+    if (orderedSections.insert(isec))
+      ++numStartupSections;
+  }
+  // then functions for compression,
+  for (auto &node : nodesForFunctionCompression) {
+    const auto *isec = sections[node.Id];
+    if (orderedSections.insert(isec))
+      ++numCodeCompressionSections;
+
+    auto It = duplicateSectionIdxs.find(node.Id);
+    if (It == duplicateSectionIdxs.end())
+      continue;
+    for (auto dupSecIdx : It->getSecond()) {
+      const auto *dupIsec = sections[dupSecIdx];
+      if (orderedSections.insert(dupIsec))
+        ++numDuplicateCodeSections;
+    }
+  }
+  // then data for compression.
+  for (auto &node : nodesForDataCompression) {
+    const auto *isec = sections[node.Id];
+    if (orderedSections.insert(isec))
+      ++numDataCompressionSections;
+    auto It = duplicateSectionIdxs.find(node.Id);
+    if (It == duplicateSectionIdxs.end())
+      continue;
+    for (auto dupSecIdx : It->getSecond()) {
+      const auto *dupIsec = sections[dupSecIdx];
+      if (orderedSections.insert(dupIsec))
+        ++numDuplicateDataSections;
+    }
+  }
+
+  if (verbose) {
+    unsigned numTotalOrderedSections =
+        numStartupSections + numCodeCompressionSections +
+        numDuplicateCodeSections + numDataCompressionSections +
+        numDuplicateDataSections;
+    dbgs()
+        << "Ordered " << numTotalOrderedSections
+        << " sections using balanced partitioning:\n  Functions for startup: "
+        << numStartupSections
+        << "\n  Functions for compression: " << numCodeCompressionSections
+        << "\n  Duplicate functions: " << numDuplicateCodeSections
+        << "\n  Data for compression: " << numDataCompressionSections
+        << "\n  Duplicate data: " << numDuplicateDataSections << "\n";
+
+    if (!profilePath.empty()) {
+      // Evaluate this function order for startup
+      StringMap<std::pair<uint64_t, uint64_t>> symbolToPageNumbers;
+      const uint64_t pageSize = (1 << 14);
+      uint64_t currentAddress = 0;
+      for (const auto *isec : orderedSections) {
+        for (auto *sym : static_cast<D *>(this)->getSymbols(*isec)) {
+          uint64_t startAddress = currentAddress + D::getSymValue(*sym);
+          uint64_t endAddress = startAddress + D::getSymSize(*sym);
+          uint64_t firstPage = startAddress / pageSize;
+          // I think the kernel might pull in a few pages when one it touched,
+          // so it might be more accurate to force lastPage to be aligned by
+          // 4?
+          uint64_t lastPage = endAddress / pageSize;
+          StringRef rootSymbol = D::getSymName(*sym);
+          rootSymbol = getRootSymbol(rootSymbol);
+          symbolToPageNumbers.try_emplace(rootSymbol, firstPage, lastPage);
+          if (auto resolvedLinkageName = D::getResolvedLinkageName(rootSymbol))
+            symbolToPageNumbers.try_emplace(resolvedLinkageName.value(),
+                                            firstPage, lastPage);
+        }
+        currentAddress += D::getSize(*isec);
+      }
+
+      // The area under the curve F where F(t) is the total number of page
+      // faults at step t.
+      unsigned area = 0;
+      for (auto &trace : reader->getTemporalProfTraces()) {
+        SmallSet<uint64_t, 0> touchedPages;
+        for (unsigned step = 0; step < trace.FunctionNameRefs.size(); step++) {
+          auto traceId = trace.FunctionNameRefs[step];
+          auto [Filename, ParsedFuncName] =
+              getParsedIRPGOName(reader->getSymtab().getFuncOrVarName(traceId));
+          ParsedFuncName = getRootSymbol(ParsedFuncName);
+          auto it = symbolToPageNumbers.find(ParsedFuncName);
+          if (it != symbolToPageNumbers.end()) {
+            auto &[firstPage, lastPage] = it->getValue();
+            for (uint64_t i = firstPage; i <= lastPage; i++)
+              touchedPages.insert(i);
+          }
+          area += touchedPages.size();
+        }
+      }
+      dbgs() << "Total area under the page fault curve: " << (float)area
+             << "\n";
+    }
+  }
+
+  DenseMap<const Section *, int> sectionPriorities;
+  int prio = -orderedSections.size();
+  for (const auto *isec : orderedSections)
+    sectionPriorities[isec] = prio++;
+  return sectionPriorities;
+}
diff --git a/lld/test/COFF/arm64ec-import.test b/lld/test/COFF/arm64ec-import.test
index 033c27884be02..bb2b772081d59 100644
--- a/lld/test/COFF/arm64ec-import.test
+++ b/lld/test/COFF/arm64ec-import.test
@@ -160,6 +160,19 @@ BASERELOC-NEXT:     Type: DIR64
 BASERELOC-NEXT:     Address: 0x5020
 BASERELOC-NEXT:   }
 
+
+Build with -filealign:8 to enable precise size checking.
+
+RUN: lld-link -machine:arm64ec -dll -noentry -out:out-size.dll loadconfig-arm64ec.obj icall.obj hybmp.obj \
+RUN:          test.obj test-arm64ec.lib test2-arm64ec.lib -filealign:8
+
+RUN: llvm-readobj --headers out-size.dll | FileCheck --check-prefix=RDATA-HEADER %s
+
+RDATA-HEADER:      Name: .rdata (2E 72 64 61 74 61 00 00)
+RDATA-HEADER-NEXT: VirtualSize: 0x2030
+RDATA-HEADER-NEXT: VirtualAddress: 0x3000
+RDATA-HEADER-NEXT: RawDataSize: 8240
+
 #--- test.s
     .section .test, "r"
     .globl arm64ec_data_sym
diff --git a/lld/test/COFF/arm64ec-patchable-thunks.test b/lld/test/COFF/arm64ec-patchable-thunks.test
index 5cebe7cc27ad6..1e1ff1f7f2ee4 100644
--- a/lld/test/COFF/arm64ec-patchable-thunks.test
+++ b/lld/test/COFF/arm64ec-patchable-thunks.test
@@ -5,6 +5,7 @@ RUN: llvm-mc -filetype=obj -triple=arm64ec-windows arm64ec-patchable.s -o arm64e
 RUN: llvm-mc -filetype=obj -triple=arm64ec-windows arm64ec-alias.s -o arm64ec-alias.obj
 RUN: llvm-mc -filetype=obj -triple=arm64ec-windows test-sec.s -o test-sec.obj
 RUN: llvm-mc -filetype=obj -triple=arm64ec-windows %S/Inputs/loadconfig-arm64ec.s -o loadconfig-arm64ec.obj
+RUN: llvm-mc -filetype=obj -triple=aarch64-windows %S/Inputs/loadconfig-arm64.s -o loadconfig-arm64.obj
 
 RUN: lld-link -out:test.dll -machine:arm64ec arm64ec-patchable.obj test-sec.obj loadconfig-arm64ec.obj -dll -noentry
 
@@ -59,6 +60,18 @@ RUN: not lld-link -out:test4.dll -machine:arm64ec test-sec.obj loadconfig-arm64e
 ERR: error: undefined symbol: EXP+#patchable_func
 
 
+RUN: lld-link -out:testx.dll -machine:arm64x arm64ec-patchable.obj test-sec.obj \
+RUN:          loadconfig-arm64.obj loadconfig-arm64ec.obj -dll -noentry
+RUN: llvm-objdump -d testx.dll | FileCheck -check-prefix=PATCH-DISASM %s
+RUN: llvm-readobj --coff-load-config testx.dll | FileCheck -check-prefix=PATCH-CHPE %s
+
+
+RUN: lld-link -out:testx2.dll -machine:arm64x arm64ec-alias.obj loadconfig-arm64.obj \
+RUN:          loadconfig-arm64ec.obj -dll -noentry
+RUN: llvm-objdump -d testx2.dll | FileCheck -check-prefix=PATCH-DISASM %s
+RUN: llvm-readobj --coff-load-config testx2.dll | FileCheck -check-prefix=PATCH-CHPE %s
+
+
 #--- arm64ec-patchable.s
     .section ".text", "x", discard, "#patchable_func$hp_target"
     .globl "#patchable_func$hp_target"
diff --git a/lld/test/COFF/arm64x-delayimport.test b/lld/test/COFF/arm64x-delayimport.test
new file mode 100644
index 0000000000000..56923ef748d09
--- /dev/null
+++ b/lld/test/COFF/arm64x-delayimport.test
@@ -0,0 +1,363 @@
+REQUIRES: aarch64, x86
+RUN: split-file %s %t.dir && cd %t.dir
+
+RUN: llvm-mc -filetype=obj -triple=arm64ec-windows test-arm64ec.s -o test-arm64ec.obj
+RUN: llvm-mc -filetype=obj -triple=aarch64-windows test-arm64.s -o test-arm64.obj
+RUN: llvm-mc -filetype=obj -triple=arm64ec-windows arm64ec-helper.s -o arm64ec-helper.obj
+RUN: llvm-mc -filetype=obj -triple=aarch64-windows arm64-helper.s -o arm64-helper.obj
+RUN: llvm-mc -filetype=obj -triple=arm64ec-windows %S/Inputs/loadconfig-arm64ec.s -o loadconfig-arm64ec.obj
+RUN: llvm-mc -filetype=obj -triple=aarch64-windows %S/Inputs/loadconfig-arm64.s -o loadconfig-arm64.obj
+RUN: llvm-lib -machine:arm64ec -def:test.def -out:test-arm64ec.lib
+RUN: llvm-lib -machine:arm64 -def:test.def -out:test-arm64.lib
+
+# Test delayed-load import from both native and EC code.
+
+RUN: lld-link -machine:arm64x -dll -noentry -out:out.dll loadconfig-arm64.obj loadconfig-arm64ec.obj \
+RUN:          arm64-helper.obj arm64ec-helper.obj test-arm64.obj test-arm64ec.obj test-arm64.lib test-arm64ec.lib -delayload:test.dll
+
+RUN: llvm-readobj --coff-imports out.dll | FileCheck --check-prefix=IMPORTS %s
+IMPORTS:      DelayImport {
+IMPORTS-NEXT:   Name: test.dll
+IMPORTS-NEXT:   Attributes: 0x1
+IMPORTS-NEXT:   ModuleHandle: 0x6080
+IMPORTS-NEXT:   ImportAddressTable: 0x6088
+IMPORTS-NEXT:   ImportNameTable: 0x4390
+IMPORTS-NEXT:   BoundDelayImportTable: 0x0
+IMPORTS-NEXT:   UnloadDelayImportTable: 0x0
+IMPORTS-NEXT:   Import {
+IMPORTS-NEXT:     Symbol: func (0)
+IMPORTS-NEXT:     Address: 0x180001014
+IMPORTS-NEXT:   }
+IMPORTS-NEXT: }
+IMPORTS-NEXT: HybridObject {
+IMPORTS:        DelayImport {
+IMPORTS-NEXT:     Name: test.dll
+IMPORTS-NEXT:     Attributes: 0x1
+IMPORTS-NEXT:     ModuleHandle: 0x6080
+IMPORTS-NEXT:     ImportAddressTable: 0x6098
+IMPORTS-NEXT:     ImportNameTable: 0x43A0
+IMPORTS-NEXT:     BoundDelayImportTable: 0x0
+IMPORTS-NEXT:     UnloadDelayImportTable: 0x0
+IMPORTS-NEXT:     Import {
+IMPORTS-NEXT:       Symbol: func (0)
+IMPORTS-NEXT:       Address: 0x180003006
+IMPORTS-NEXT:     }
+IMPORTS-NEXT:   }
+IMPORTS-NEXT: }
+
+RUN: llvm-readobj --hex-dump=.test out.dll | FileCheck --check-prefix=TESTSEC %s
+TESTSEC: 0x180009000 10500000 98600000 00300000 10200000
+
+RUN: llvm-readobj --hex-dump=.testa out.dll | FileCheck --check-prefix=TESTSECA %s
+TESTSECA: 0x18000a000 88600000 08100000
+
+RUN: llvm-objdump -d out.dll | FileCheck --check-prefix=DISASM %s
+DISASM:      0000000180001000 <.text>:
+DISASM-NEXT: 180001000: 52800060     mov     w0, #0x3                // =3
+DISASM-NEXT: 180001004: d65f03c0     ret
+DISASM-NEXT: 180001008: b0000030     adrp    x16, 0x180006000
+DISASM-NEXT: 18000100c: f9404610     ldr     x16, [x16, #0x88]
+DISASM-NEXT: 180001010: d61f0200     br      x16
+DISASM-NEXT: 180001014: b0000031     adrp    x17, 0x180006000
+DISASM-NEXT: 180001018: 91022231     add     x17, x17, #0x88
+DISASM-NEXT: 18000101c: 14000001     b       0x180001020 <.text+0x20>
+DISASM-NEXT: 180001020: a9b37bfd     stp     x29, x30, [sp, #-0xd0]!
+DISASM-NEXT: 180001024: 910003fd     mov     x29, sp
+DISASM-NEXT: 180001028: a90107e0     stp     x0, x1, [sp, #0x10]
+DISASM-NEXT: 18000102c: a9020fe2     stp     x2, x3, [sp, #0x20]
+DISASM-NEXT: 180001030: a90317e4     stp     x4, x5, [sp, #0x30]
+DISASM-NEXT: 180001034: a9041fe6     stp     x6, x7, [sp, #0x40]
+DISASM-NEXT: 180001038: ad0287e0     stp     q0, q1, [sp, #0x50]
+DISASM-NEXT: 18000103c: ad038fe2     stp     q2, q3, [sp, #0x70]
+DISASM-NEXT: 180001040: ad0497e4     stp     q4, q5, [sp, #0x90]
+DISASM-NEXT: 180001044: ad059fe6     stp     q6, q7, [sp, #0xb0]
+DISASM-NEXT: 180001048: aa1103e1     mov     x1, x17
+DISASM-NEXT: 18000104c: f0000000     adrp    x0, 0x180004000
+DISASM-NEXT: 180001050: 910d4000     add     x0, x0, #0x350
+DISASM-NEXT: 180001054: 97ffffeb     bl      0x180001000 <.text>
+DISASM-NEXT: 180001058: aa0003f0     mov     x16, x0
+DISASM-NEXT: 18000105c: ad459fe6     ldp     q6, q7, [sp, #0xb0]
+DISASM-NEXT: 180001060: ad4497e4     ldp     q4, q5, [sp, #0x90]
+DISASM-NEXT: 180001064: ad438fe2     ldp     q2, q3, [sp, #0x70]
+DISASM-NEXT: 180001068: ad4287e0     ldp     q0, q1, [sp, #0x50]
+DISASM-NEXT: 18000106c: a9441fe6     ldp     x6, x7, [sp, #0x40]
+DISASM-NEXT: 180001070: a94317e4     ldp     x4, x5, [sp, #0x30]
+DISASM-NEXT: 180001074: a9420fe2     ldp     x2, x3, [sp, #0x20]
+DISASM-NEXT: 180001078: a94107e0     ldp     x0, x1, [sp, #0x10]
+DISASM-NEXT: 18000107c: a8cd7bfd     ldp     x29, x30, [sp], #0xd0
+DISASM-NEXT: 180001080: d61f0200     br      x16
+DISASM-NEXT:                 ...
+DISASM-NEXT: 180002000: 52800040     mov     w0, #0x2                // =2
+DISASM-NEXT: 180002004: d65f03c0     ret
+DISASM-NEXT: 180002008: 52800060     mov     w0, #0x3                // =3
+DISASM-NEXT: 18000200c: d65f03c0     ret
+DISASM-NEXT: 180002010: f0000010     adrp    x16, 0x180005000
+DISASM-NEXT: 180002014: f9400a10     ldr     x16, [x16, #0x10]
+DISASM-NEXT: 180002018: d61f0200     br      x16
+DISASM-NEXT: 18000201c: 9000002b     adrp    x11, 0x180006000
+DISASM-NEXT: 180002020: f9404d6b     ldr     x11, [x11, #0x98]
+DISASM-NEXT: 180002024: 9000000a     adrp    x10, 0x180002000 <.text+0x1000>
+DISASM-NEXT: 180002028: 9100c14a     add     x10, x10, #0x30
+DISASM-NEXT: 18000202c: 17fffff5     b       0x180002000 <.text+0x1000>
+DISASM-NEXT: 180002030: 52800080     mov     w0, #0x4                // =4
+DISASM-NEXT: 180002034: d65f03c0     ret
+DISASM-NEXT:                 ...
+DISASM-NEXT: 180003000: ff 25 92 30 00 00            jmpq    *0x3092(%rip)           # 0x180006098
+DISASM-NEXT: 180003006: 48 8d 05 8b 30 00 00         leaq    0x308b(%rip), %rax      # 0x180006098
+DISASM-NEXT: 18000300d: e9 00 00 00 00               jmp     0x180003012 <.text+0x2012>
+DISASM-NEXT: 180003012: 51                           pushq   %rcx
+DISASM-NEXT: 180003013: 52                           pushq   %rdx
+DISASM-NEXT: 180003014: 41 50                        pushq   %r8
+DISASM-NEXT: 180003016: 41 51                        pushq   %r9
+DISASM-NEXT: 180003018: 48 83 ec 48                  subq    $0x48, %rsp
+DISASM-NEXT: 18000301c: 66 0f 7f 04 24               movdqa  %xmm0, (%rsp)
+DISASM-NEXT: 180003021: 66 0f 7f 4c 24 10            movdqa  %xmm1, 0x10(%rsp)
+DISASM-NEXT: 180003027: 66 0f 7f 54 24 20            movdqa  %xmm2, 0x20(%rsp)
+DISASM-NEXT: 18000302d: 66 0f 7f 5c 24 30            movdqa  %xmm3, 0x30(%rsp)
+DISASM-NEXT: 180003033: 48 8b d0                     movq    %rax, %rdx
+DISASM-NEXT: 180003036: 48 8d 0d 13 13 00 00         leaq    0x1313(%rip), %rcx # 0x180004350
+DISASM-NEXT: 18000303d: e8 c6 ef ff ff               callq   0x180002008 <.text+0x1008>
+DISASM-NEXT: 180003042: 66 0f 6f 04 24               movdqa  (%rsp), %xmm0
+DISASM-NEXT: 180003047: 66 0f 6f 4c 24 10            movdqa  0x10(%rsp), %xmm1
+DISASM-NEXT: 18000304d: 66 0f 6f 54 24 20            movdqa  0x20(%rsp), %xmm2
+DISASM-NEXT: 180003053: 66 0f 6f 5c 24 30            movdqa  0x30(%rsp), %xmm3
+DISASM-NEXT: 180003059: 48 83 c4 48                  addq    $0x48, %rsp
+DISASM-NEXT: 18000305d: 41 59                        popq    %r9
+DISASM-NEXT: 18000305f: 41 58                        popq    %r8
+DISASM-NEXT: 180003061: 5a                           popq    %rdx
+DISASM-NEXT: 180003062: 59                           popq    %rcx
+DISASM-NEXT: 180003063: ff e0                        jmpq    *%rax
+
+RUN: llvm-readobj --coff-load-config out.dll | FileCheck --check-prefix=LOADCFG %s
+LOADCFG:      AuxiliaryDelayloadIAT: 0x5000
+LOADCFG-NEXT: AuxiliaryDelayloadIATCopy: 0x4140
+
+RUN: llvm-readobj --hex-dump=.rdata out.dll | FileCheck --check-prefix=AUXIAT %s
+AUXIAT:      0x180005000 00000000 00000000 00000000 00000000
+AUXIAT-NEXT: 0x180005010 1c200080 01000000 00000000 00000000
+
+
+# Test delayed-load import from native code only.
+
+RUN: lld-link -machine:arm64x -dll -noentry -out:out-native.dll loadconfig-arm64.obj loadconfig-arm64ec.obj \
+RUN:          arm64-helper.obj arm64ec-helper.obj test-arm64.obj test-arm64.lib test-arm64ec.lib -delayload:test.dll
+
+RUN: llvm-readobj --coff-imports out-native.dll | FileCheck --check-prefix=NATIVE-IMPORTS %s
+NATIVE-IMPORTS:      DelayImport {
+NATIVE-IMPORTS-NEXT:   Name: test.dll
+NATIVE-IMPORTS-NEXT:   Attributes: 0x1
+NATIVE-IMPORTS-NEXT:   ModuleHandle: 0x5080
+NATIVE-IMPORTS-NEXT:   ImportAddressTable: 0x5088
+NATIVE-IMPORTS-NEXT:   ImportNameTable: 0x3370
+NATIVE-IMPORTS-NEXT:   BoundDelayImportTable: 0x0
+NATIVE-IMPORTS-NEXT:   UnloadDelayImportTable: 0x0
+NATIVE-IMPORTS-NEXT:   Import {
+NATIVE-IMPORTS-NEXT:     Symbol: func (0)
+NATIVE-IMPORTS-NEXT:     Address: 0x180001014
+NATIVE-IMPORTS-NEXT:   }
+NATIVE-IMPORTS-NEXT: }
+NATIVE-IMPORTS-NEXT: HybridObject {
+NATIVE-IMPORTS-NEXT:   Format: COFF-ARM64EC
+NATIVE-IMPORTS-NEXT:   Arch: aarch64
+NATIVE-IMPORTS-NEXT:   AddressSize: 64bit
+NATIVE-IMPORTS-NEXT:   DelayImport {
+NATIVE-IMPORTS-NEXT:     Name: test.dll
+NATIVE-IMPORTS-NEXT:     Attributes: 0x1
+NATIVE-IMPORTS-NEXT:     ModuleHandle: 0x5080
+NATIVE-IMPORTS-NEXT:     ImportAddressTable: 0x5098
+NATIVE-IMPORTS-NEXT:     ImportNameTable: 0x3380
+NATIVE-IMPORTS-NEXT:     BoundDelayImportTable: 0x0
+NATIVE-IMPORTS-NEXT:     UnloadDelayImportTable: 0x0
+NATIVE-IMPORTS-NEXT:   }
+NATIVE-IMPORTS-NEXT: }
+
+RUN: llvm-readobj --hex-dump=.testa out-native.dll | FileCheck --check-prefix=NATIVE-TESTSECA %s
+NATIVE-TESTSECA: 0x180007000 88500000 08100000
+
+RUN: llvm-objdump -d out-native.dll | FileCheck --check-prefix=NATIVE-DISASM %s
+NATIVE-DISASM:      0000000180001000 <.text>:
+NATIVE-DISASM-NEXT: 180001000: 52800060     mov     w0, #0x3                // =3
+NATIVE-DISASM-NEXT: 180001004: d65f03c0     ret
+NATIVE-DISASM-NEXT: 180001008: 90000030     adrp    x16, 0x180005000
+NATIVE-DISASM-NEXT: 18000100c: f9404610     ldr     x16, [x16, #0x88]
+NATIVE-DISASM-NEXT: 180001010: d61f0200     br      x16
+NATIVE-DISASM-NEXT: 180001014: 90000031     adrp    x17, 0x180005000
+NATIVE-DISASM-NEXT: 180001018: 91022231     add     x17, x17, #0x88
+NATIVE-DISASM-NEXT: 18000101c: 14000001     b       0x180001020 <.text+0x20>
+NATIVE-DISASM-NEXT: 180001020: a9b37bfd     stp     x29, x30, [sp, #-0xd0]!
+NATIVE-DISASM-NEXT: 180001024: 910003fd     mov     x29, sp
+NATIVE-DISASM-NEXT: 180001028: a90107e0     stp     x0, x1, [sp, #0x10]
+NATIVE-DISASM-NEXT: 18000102c: a9020fe2     stp     x2, x3, [sp, #0x20]
+NATIVE-DISASM-NEXT: 180001030: a90317e4     stp     x4, x5, [sp, #0x30]
+NATIVE-DISASM-NEXT: 180001034: a9041fe6     stp     x6, x7, [sp, #0x40]
+NATIVE-DISASM-NEXT: 180001038: ad0287e0     stp     q0, q1, [sp, #0x50]
+NATIVE-DISASM-NEXT: 18000103c: ad038fe2     stp     q2, q3, [sp, #0x70]
+NATIVE-DISASM-NEXT: 180001040: ad0497e4     stp     q4, q5, [sp, #0x90]
+NATIVE-DISASM-NEXT: 180001044: ad059fe6     stp     q6, q7, [sp, #0xb0]
+NATIVE-DISASM-NEXT: 180001048: aa1103e1     mov     x1, x17
+NATIVE-DISASM-NEXT: 18000104c: d0000000     adrp    x0, 0x180003000
+NATIVE-DISASM-NEXT: 180001050: 910cc000     add     x0, x0, #0x330
+NATIVE-DISASM-NEXT: 180001054: 97ffffeb     bl      0x180001000 <.text>
+NATIVE-DISASM-NEXT: 180001058: aa0003f0     mov     x16, x0
+NATIVE-DISASM-NEXT: 18000105c: ad459fe6     ldp     q6, q7, [sp, #0xb0]
+NATIVE-DISASM-NEXT: 180001060: ad4497e4     ldp     q4, q5, [sp, #0x90]
+NATIVE-DISASM-NEXT: 180001064: ad438fe2     ldp     q2, q3, [sp, #0x70]
+NATIVE-DISASM-NEXT: 180001068: ad4287e0     ldp     q0, q1, [sp, #0x50]
+NATIVE-DISASM-NEXT: 18000106c: a9441fe6     ldp     x6, x7, [sp, #0x40]
+NATIVE-DISASM-NEXT: 180001070: a94317e4     ldp     x4, x5, [sp, #0x30]
+NATIVE-DISASM-NEXT: 180001074: a9420fe2     ldp     x2, x3, [sp, #0x20]
+NATIVE-DISASM-NEXT: 180001078: a94107e0     ldp     x0, x1, [sp, #0x10]
+NATIVE-DISASM-NEXT: 18000107c: a8cd7bfd     ldp     x29, x30, [sp], #0xd0
+NATIVE-DISASM-NEXT: 180001080: d61f0200     br      x16
+
+RUN: llvm-readobj --coff-load-config out-native.dll | FileCheck --check-prefix=NATIVE-LOADCFG %s
+NATIVE-LOADCFG:      AuxiliaryDelayloadIAT: 0x4000
+NATIVE-LOADCFG-NEXT: AuxiliaryDelayloadIATCopy: 0x3140
+
+RUN: llvm-readobj --hex-dump=.rdata out-native.dll | FileCheck --check-prefix=NATIVE-AUXIAT %s
+NATIVE-AUXIAT:      0x180004000 00000000 00000000 00000000 00000000
+NATIVE-AUXIAT-NEXT: 0x180004010 00000000 00000000
+
+
+# Test delayed-load import from EC code only.
+
+RUN: lld-link -machine:arm64x -dll -noentry -out:out-ec.dll loadconfig-arm64.obj loadconfig-arm64ec.obj \
+RUN:          arm64-helper.obj arm64ec-helper.obj test-arm64ec.obj test-arm64.lib test-arm64ec.lib -delayload:test.dll
+
+RUN: llvm-readobj --coff-imports out-ec.dll | FileCheck --check-prefix=EC-IMPORTS %s
+EC-IMPORTS:      DelayImport {
+EC-IMPORTS-NEXT:   Name: test.dll
+EC-IMPORTS-NEXT:   Attributes: 0x1
+EC-IMPORTS-NEXT:   ModuleHandle: 0x6080
+EC-IMPORTS-NEXT:   ImportAddressTable: 0x6088
+EC-IMPORTS-NEXT:   ImportNameTable: 0x4388
+EC-IMPORTS-NEXT:   BoundDelayImportTable: 0x0
+EC-IMPORTS-NEXT:   UnloadDelayImportTable: 0x0
+EC-IMPORTS-NEXT: }
+EC-IMPORTS-NEXT: HybridObject {
+EC-IMPORTS-NEXT:   Format: COFF-ARM64EC
+EC-IMPORTS-NEXT:   Arch: aarch64
+EC-IMPORTS-NEXT:   AddressSize: 64bit
+EC-IMPORTS-NEXT:   DelayImport {
+EC-IMPORTS-NEXT:     Name: test.dll
+EC-IMPORTS-NEXT:     Attributes: 0x1
+EC-IMPORTS-NEXT:     ModuleHandle: 0x6080
+EC-IMPORTS-NEXT:     ImportAddressTable: 0x6090
+EC-IMPORTS-NEXT:     ImportNameTable: 0x4390
+EC-IMPORTS-NEXT:     BoundDelayImportTable: 0x0
+EC-IMPORTS-NEXT:     UnloadDelayImportTable: 0x0
+EC-IMPORTS-NEXT:     Import {
+EC-IMPORTS-NEXT:       Symbol: func (0)
+EC-IMPORTS-NEXT:       Address: 0x180003006
+EC-IMPORTS-NEXT:     }
+EC-IMPORTS-NEXT:   }
+EC-IMPORTS-NEXT: }
+
+RUN: llvm-readobj --hex-dump=.test out-ec.dll | FileCheck --check-prefix=EC-TESTSEC %s
+EC-TESTSEC: 0x180009000 08500000 90600000 00300000 10200000
+
+RUN: llvm-objdump -d out-ec.dll | FileCheck --check-prefix=EC-DISASM %s
+EC-DISASM:      0000000180001000 <.text>:
+EC-DISASM-NEXT: 180001000: 52800060     mov     w0, #0x3                // =3
+EC-DISASM-NEXT: 180001004: d65f03c0     ret
+EC-DISASM-NEXT:                 ...
+EC-DISASM-NEXT: 180002000: 52800040     mov     w0, #0x2                // =2
+EC-DISASM-NEXT: 180002004: d65f03c0     ret
+EC-DISASM-NEXT: 180002008: 52800060     mov     w0, #0x3                // =3
+EC-DISASM-NEXT: 18000200c: d65f03c0     ret
+EC-DISASM-NEXT: 180002010: f0000010     adrp    x16, 0x180005000
+EC-DISASM-NEXT: 180002014: f9400610     ldr     x16, [x16, #0x8]
+EC-DISASM-NEXT: 180002018: d61f0200     br      x16
+EC-DISASM-NEXT: 18000201c: 9000002b     adrp    x11, 0x180006000
+EC-DISASM-NEXT: 180002020: f940496b     ldr     x11, [x11, #0x90]
+EC-DISASM-NEXT: 180002024: 9000000a     adrp    x10, 0x180002000 <.text+0x1000>
+EC-DISASM-NEXT: 180002028: 9100c14a     add     x10, x10, #0x30
+EC-DISASM-NEXT: 18000202c: 17fffff5     b       0x180002000 <.text+0x1000>
+EC-DISASM-NEXT: 180002030: 52800080     mov     w0, #0x4                // =4
+EC-DISASM-NEXT: 180002034: d65f03c0     ret
+EC-DISASM-NEXT:                 ...
+EC-DISASM-NEXT: 180003000: ff 25 8a 30 00 00            jmpq    *0x308a(%rip)           # 0x180006090
+EC-DISASM-NEXT: 180003006: 48 8d 05 83 30 00 00         leaq    0x3083(%rip), %rax      # 0x180006090
+EC-DISASM-NEXT: 18000300d: e9 00 00 00 00               jmp     0x180003012 <.text+0x2012>
+EC-DISASM-NEXT: 180003012: 51                           pushq   %rcx
+EC-DISASM-NEXT: 180003013: 52                           pushq   %rdx
+EC-DISASM-NEXT: 180003014: 41 50                        pushq   %r8
+EC-DISASM-NEXT: 180003016: 41 51                        pushq   %r9
+EC-DISASM-NEXT: 180003018: 48 83 ec 48                  subq    $0x48, %rsp
+EC-DISASM-NEXT: 18000301c: 66 0f 7f 04 24               movdqa  %xmm0, (%rsp)
+EC-DISASM-NEXT: 180003021: 66 0f 7f 4c 24 10            movdqa  %xmm1, 0x10(%rsp)
+EC-DISASM-NEXT: 180003027: 66 0f 7f 54 24 20            movdqa  %xmm2, 0x20(%rsp)
+EC-DISASM-NEXT: 18000302d: 66 0f 7f 5c 24 30            movdqa  %xmm3, 0x30(%rsp)
+EC-DISASM-NEXT: 180003033: 48 8b d0                     movq    %rax, %rdx
+EC-DISASM-NEXT: 180003036: 48 8d 0d 0b 13 00 00         leaq    0x130b(%rip), %rcx      # 0x180004348
+EC-DISASM-NEXT: 18000303d: e8 c6 ef ff ff               callq   0x180002008 <.text+0x1008>
+EC-DISASM-NEXT: 180003042: 66 0f 6f 04 24               movdqa  (%rsp), %xmm0
+EC-DISASM-NEXT: 180003047: 66 0f 6f 4c 24 10            movdqa  0x10(%rsp), %xmm1
+EC-DISASM-NEXT: 18000304d: 66 0f 6f 54 24 20            movdqa  0x20(%rsp), %xmm2
+EC-DISASM-NEXT: 180003053: 66 0f 6f 5c 24 30            movdqa  0x30(%rsp), %xmm3
+EC-DISASM-NEXT: 180003059: 48 83 c4 48                  addq    $0x48, %rsp
+EC-DISASM-NEXT: 18000305d: 41 59                        popq    %r9
+EC-DISASM-NEXT: 18000305f: 41 58                        popq    %r8
+EC-DISASM-NEXT: 180003061: 5a                           popq    %rdx
+EC-DISASM-NEXT: 180003062: 59                           popq    %rcx
+EC-DISASM-NEXT: 180003063: ff e0                        jmpq    *%rax
+
+RUN: llvm-readobj --coff-load-config out-ec.dll | FileCheck --check-prefix=EC-LOADCFG %s
+EC-LOADCFG:      AuxiliaryDelayloadIAT: 0x5000
+EC-LOADCFG-NEXT: AuxiliaryDelayloadIATCopy: 0x4140
+
+RUN: llvm-readobj --hex-dump=.rdata out-ec.dll | FileCheck --check-prefix=EC-AUXIAT %s
+EC-AUXIAT:      0x180005000 00000000 00000000 1c200080 01000000
+EC-AUXIAT-NEXT: 0x180005010 00000000 00000000
+
+
+#--- test-arm64ec.s
+    .section .test, "rd"
+    .rva __imp_func
+    .rva __imp_aux_func
+    .rva func
+    .rva "#func"
+
+#--- test-arm64.s
+    .section .testa, "rd"
+    .rva __imp_func
+    .rva func
+
+#--- arm64ec-helper.s
+    .section .text,"xr",discard,__icall_helper_arm64ec
+    .globl __icall_helper_arm64ec
+    .p2align 2, 0x0
+__icall_helper_arm64ec:
+    mov w0, #2
+    ret
+
+    .section .text,"xr",discard,"#__delayLoadHelper2"
+    .globl "#__delayLoadHelper2"
+    .p2align 2, 0x0
+"#__delayLoadHelper2":
+    mov w0, #3
+    ret
+
+    .section .hybmp$x, "yi"
+    .symidx __imp_func
+    .symidx func_exit_thunk
+    .word 4
+
+    .section .wowthk$aa,"xr",discard,func_exit_thunk
+    .globl func_exit_thunk
+func_exit_thunk:
+    mov w0, #4
+    ret
+
+#--- arm64-helper.s
+    .section .text,"xr",discard,__delayLoadHelper2
+    .globl __delayLoadHelper2
+    .p2align 2, 0x0
+__delayLoadHelper2:
+    mov w0, #3
+    ret
+
+#--- test.def
+NAME test.dll
+EXPORTS
+    func
diff --git a/lld/test/COFF/arm64x-export-all.s b/lld/test/COFF/arm64x-export-all.s
new file mode 100644
index 0000000000000..06ea9ec50259e
--- /dev/null
+++ b/lld/test/COFF/arm64x-export-all.s
@@ -0,0 +1,68 @@
+// REQUIRES: aarch64
+
+// RUN: llvm-mc -filetype=obj -triple=aarch64-windows %s -o %t.arm64.obj
+// RUN: llvm-mc -filetype=obj -triple=arm64ec-windows %s -o %t.arm64ec.obj
+// RUN: llvm-mc -filetype=obj -triple=arm64ec-windows %S/Inputs/loadconfig-arm64ec.s -o %t-loadconfig-arm64ec.obj
+// RUN: llvm-mc -filetype=obj -triple=aarch64-windows %S/Inputs/loadconfig-arm64.s -o %t-loadconfig-arm64.obj
+
+// Check that all symbols are exported in both EC and native views.
+
+// RUN: lld-link -machine:arm64x -lldmingw -dll -noentry -out:%t.dll %t.arm64.obj %t.arm64ec.obj %t-loadconfig-arm64.obj %t-loadconfig-arm64ec.obj
+
+// RUN: llvm-readobj --coff-exports %t.dll | FileCheck --check-prefix=EXP %s
+// EXP:      Format: COFF-ARM64X
+// EXP-NEXT: Arch: aarch64
+// EXP-NEXT: AddressSize: 64bit
+// EXP-NEXT: Export {
+// EXP-NEXT:   Ordinal: 1
+// EXP-NEXT:   Name: sym
+// EXP-NEXT:   RVA: 0x2000
+// EXP-NEXT: }
+// EXP-NEXT: Export {
+// EXP-NEXT:   Ordinal: 2
+// EXP-NEXT:   Name: sym2
+// EXP-NEXT:   RVA: 0x2004
+// EXP-NEXT: }
+// EXP-NEXT: HybridObject {
+// EXP-NEXT:   Format: COFF-ARM64EC
+// EXP-NEXT:   Arch: aarch64
+// EXP-NEXT:   AddressSize: 64bit
+// EXP-NEXT:   Export {
+// EXP-NEXT:     Ordinal: 1
+// EXP-NEXT:     Name: sym
+// EXP-NEXT:     RVA: 0x2008
+// EXP-NEXT:   }
+// EXP-NEXT:   Export {
+// EXP-NEXT:     Ordinal: 2
+// EXP-NEXT:     Name: sym2
+// EXP-NEXT:     RVA: 0x200C
+// EXP-NEXT:   }
+// EXP-NEXT: }
+
+// Check that an explicit export in the EC view is respected, preventing symbols from being auto-exported in both EC and native views.
+
+// RUN: lld-link -machine:arm64x -lldmingw -dll -noentry -out:%t2.dll %t.arm64.obj %t.arm64ec.obj -export:sym \
+// RUN:          %t-loadconfig-arm64.obj %t-loadconfig-arm64ec.obj
+
+// RUN: llvm-readobj --coff-exports %t2.dll | FileCheck --check-prefix=EXP2 %s
+// EXP2:      Format: COFF-ARM64X
+// EXP2-NEXT: Arch: aarch64
+// EXP2-NEXT: AddressSize: 64bit
+// EXP2-NEXT: HybridObject {
+// EXP2-NEXT:   Format: COFF-ARM64EC
+// EXP2-NEXT:   Arch: aarch64
+// EXP2-NEXT:   AddressSize: 64bit
+// EXP2-NEXT:   Export {
+// EXP2-NEXT:     Ordinal: 1
+// EXP2-NEXT:     Name: sym
+// EXP2-NEXT:     RVA: 0x2008
+// EXP2-NEXT:   }
+// EXP2-NEXT: }
+
+        .data
+        .globl sym
+sym:
+        .word 0
+        .globl sym2
+sym2:
+        .word 0
diff --git a/lld/test/COFF/arm64x-export.test b/lld/test/COFF/arm64x-export.test
index 3ae0725a67089..c83db2a436db5 100644
--- a/lld/test/COFF/arm64x-export.test
+++ b/lld/test/COFF/arm64x-export.test
@@ -14,7 +14,7 @@ RUN: llvm-mc -filetype=obj -triple=aarch64-windows %S/Inputs/loadconfig-arm64.s
 # A command-line export applies only to EC exports.
 
 RUN: lld-link -machine:arm64x -dll -out:out-cmd.dll arm64ec-func.obj arm64-func.obj \
-RUN:          loadconfig-arm64.obj loadconfig-arm64ec.obj -noentry -export:func
+RUN:          loadconfig-arm64.obj loadconfig-arm64ec.obj -noentry -export:func -output-def:out.def
 
 RUN: llvm-objdump -d out-cmd.dll | FileCheck --check-prefix=DISASM-EC %s
 DISASM-EC:      Disassembly of section .text:
@@ -48,12 +48,44 @@ EXPORTS-EC-NEXT:     RVA: 0x2000
 EXPORTS-EC-NEXT:   }
 EXPORTS-EC-NEXT: }
 
+RUN: llvm-readobj out-cmd.lib | FileCheck --check-prefixes=IMPLIB-HEADER,IMPLIB-EC %s
+IMPLIB-HEADER:      File: out-{{.*}}.lib(out{{.*}}.dll)
+IMPLIB-HEADER-NEXT: Format: COFF-ARM64
+IMPLIB-HEADER-NEXT: Arch: aarch64
+IMPLIB-HEADER-NEXT: AddressSize: 64bit
+IMPLIB-HEADER-EMPTY:
+IMPLIB-HEADER-NEXT: File: out-{{.*}}.lib(out{{.*}}.dll)
+IMPLIB-HEADER-NEXT: Format: COFF-ARM64
+IMPLIB-HEADER-NEXT: Arch: aarch64
+IMPLIB-HEADER-NEXT: AddressSize: 64bit
+IMPLIB-HEADER-EMPTY:
+IMPLIB-HEADER-NEXT: File: out-{{.*}}.lib(out{{.*}}.dll)
+IMPLIB-HEADER-NEXT: Format: COFF-ARM64
+IMPLIB-HEADER-NEXT: Arch: aarch64
+IMPLIB-HEADER-NEXT: AddressSize: 64bit
+IMPLIB-HEADER-EMPTY:
+IMPLIB-EC:      File: out{{.*}}.dll
+IMPLIB-EC-NEXT: Format: COFF-import-file-ARM64EC
+IMPLIB-EC-NEXT: Type: code
+IMPLIB-EC-NEXT: Name type: export as
+IMPLIB-EC-NEXT: Export name: func
+IMPLIB-EC-NEXT: Symbol: __imp_func
+IMPLIB-EC-NEXT: Symbol: func
+IMPLIB-EC-NEXT: Symbol: __imp_aux_func
+IMPLIB-EC-NEXT: Symbol: #func
+
+RUN: FileCheck --check-prefix=OUT-DEF %s < out.def
+OUT-DEF:      EXPORTS
+OUT-DEF-NEXT:     func @1
+
+
 # Export using the EC .drectve section.
 
 RUN: lld-link -machine:arm64x -dll -out:out-drectve-ec.dll arm64ec-func.obj arm64-func.obj \
 RUN:          loadconfig-arm64.obj loadconfig-arm64ec.obj arm64ec-drectve.obj -noentry
 RUN: llvm-objdump -d out-drectve-ec.dll | FileCheck --check-prefix=DISASM-EC %s
 RUN: llvm-readobj --headers --coff-exports out-drectve-ec.dll | FileCheck --check-prefix=EXPORTS-EC %s
+RUN: llvm-readobj out-drectve-ec.lib | FileCheck --check-prefixes=IMPLIB-HEADER,IMPLIB-EC %s
 
 # A command-line def file applies only to EC exports.
 
@@ -61,6 +93,7 @@ RUN: lld-link -machine:arm64x -dll -out:out-def-ec.dll arm64ec-func.obj arm64-fu
 RUN:          loadconfig-arm64.obj loadconfig-arm64ec.obj -def:func.def -noentry
 RUN: llvm-objdump -d out-def-ec.dll | FileCheck --check-prefix=DISASM-EC %s
 RUN: llvm-readobj --headers --coff-exports out-def-ec.dll | FileCheck --check-prefix=EXPORTS-EC %s
+RUN: llvm-readobj out-def-ec.lib | FileCheck --check-prefixes=IMPLIB-HEADER,IMPLIB-EC %s
 
 # Export using the EC .edata section.
 
@@ -111,6 +144,15 @@ EXPORTS-NATIVE:        ExportTableRVA: 0x0
 EXPORTS-NATIVE-NEXT:   ExportTableSize: 0x0
 EXPORTS-NATIVE-NOT:    Name: func
 
+RUN: llvm-readobj out-drectve-native.lib | FileCheck --check-prefixes=IMPLIB-HEADER,IMPLIB-NATIVE %s
+IMPLIB-NATIVE:      File: out{{.*}}.dll
+IMPLIB-NATIVE-NEXT: Format: COFF-import-file-ARM64
+IMPLIB-NATIVE-NEXT: Type: code
+IMPLIB-NATIVE-NEXT: Name type: name
+IMPLIB-NATIVE-NEXT: Export name: func
+IMPLIB-NATIVE-NEXT: Symbol: __imp_func
+IMPLIB-NATIVE-NEXT: Symbol: func
+
 # Export using the native .edata section.
 
 RUN: lld-link -machine:arm64x -dll -out:out-edata.dll arm64ec-func.obj arm64-func.obj \
@@ -162,12 +204,15 @@ EXPORTS-BOTH-NEXT:     RVA: 0x3000
 EXPORTS-BOTH-NEXT:   }
 EXPORTS-BOTH-NEXT: }
 
+RUN: llvm-readobj out-both.lib | FileCheck --check-prefixes=IMPLIB-HEADER,IMPLIB-EC,IMPLIB-NATIVE %s
+
 # Export using both the -def and -defarm64native arguments.
 
 RUN: lld-link -machine:arm64x -dll -out:out-def-both.dll arm64ec-func.obj arm64-func.obj \
 RUN:          loadconfig-arm64.obj loadconfig-arm64ec.obj -def:func.def -defarm64native:func.def -noentry
 RUN: llvm-objdump -d out-def-both.dll | FileCheck --check-prefix=DISASM-BOTH %s
 RUN: llvm-readobj --headers --coff-exports out-def-both.dll | FileCheck --check-prefix=EXPORTS-BOTH %s
+RUN: llvm-readobj out-def-both.lib | FileCheck --check-prefixes=IMPLIB-HEADER,IMPLIB-EC,IMPLIB-NATIVE %s
 
 # -defarm64native is ignored if -def is not specified.
 
diff --git a/lld/test/COFF/arm64x-import.test b/lld/test/COFF/arm64x-import.test
new file mode 100644
index 0000000000000..7441c71d87710
--- /dev/null
+++ b/lld/test/COFF/arm64x-import.test
@@ -0,0 +1,533 @@
+REQUIRES: aarch64, x86
+RUN: split-file %s %t.dir && cd %t.dir
+
+RUN: llvm-mc -filetype=obj -triple=arm64ec-windows func12-thunks-arm64ec.s -o func12-thunks-arm64ec.obj
+RUN: llvm-mc -filetype=obj -triple=aarch64-windows func12-thunks-arm64.s -o func12-thunks-arm64.obj
+RUN: llvm-mc -filetype=obj -triple=arm64ec-windows func12-arm64ec.s -o func12-arm64ec.obj
+RUN: llvm-mc -filetype=obj -triple=aarch64-windows func123-arm64.s -o func123-arm64.obj
+RUN: llvm-mc -filetype=obj -triple=arm64ec-windows func123-arm64ec.s -o func123-arm64ec.obj
+RUN: llvm-mc -filetype=obj -triple=aarch64-windows func12-arm64.s -o func12-arm64.obj
+RUN: llvm-mc -filetype=obj -triple=aarch64-windows func234-arm64.s -o func234-arm64.obj
+RUN: llvm-mc -filetype=obj -triple=arm64ec-windows func12o-arm64ec.s -o func12o-arm64ec.obj
+RUN: llvm-mc -filetype=obj -triple=aarch64-windows func34-arm64.s -o func34-arm64.obj
+RUN: llvm-mc -filetype=obj -triple=aarch64-windows func34o-arm64.s -o func34o-arm64.obj
+RUN: llvm-mc -filetype=obj -triple=aarch64-windows funco-arm64.s -o funco-arm64.obj
+RUN: llvm-mc -filetype=obj -triple=arm64ec-windows icall.s -o icall.obj
+RUN: llvm-mc -filetype=obj -triple=arm64ec-windows %S/Inputs/loadconfig-arm64ec.s -o loadconfig-arm64ec.obj
+RUN: llvm-mc -filetype=obj -triple=aarch64-windows %S/Inputs/loadconfig-arm64.s -o loadconfig-arm64.obj
+RUN: llvm-lib -machine:arm64ec -def:imp.def -out:imp-arm64ec.lib
+RUN: llvm-lib -machine:arm64 -def:imp.def -out:imp-arm64.lib
+RUN: llvm-lib -machine:arm64x -def:imp.def -defArm64Native:imp.def -out:imp-arm64x.lib
+RUN: llvm-lib -machine:arm64x -def:imp-ord10.def -defArm64Native:imp.def -out:imp-ecord.lib
+RUN: llvm-lib -machine:arm64x -def:imp-ord10.def -defArm64Native:imp-ord20.def -out:imp-ecord.lib
+RUN: llvm-lib -machine:arm64x -def:imp2.def -defArm64Native:imp2.def -out:imp2.lib
+RUN: llvm-lib -machine:arm64x -def:noname-ec.def -defArm64Native:noname-native.def -out:noname.lib
+RUN: llvm-lib -machine:arm64x -def:dup-ec.def -defArm64Native:dup-native.def -out:dup.lib
+
+
+# Link to the imported func1, func2, and func1's thunks from both native and EC code.
+
+RUN: lld-link -machine:arm64x -dll -noentry -out:test-12-thunks.dll loadconfig-arm64.obj loadconfig-arm64ec.obj \
+RUN:          icall.obj func12-thunks-arm64ec.obj func12-thunks-arm64.obj imp-arm64ec.lib imp-arm64.lib
+
+RUN: llvm-objdump -d test-12-thunks.dll | FileCheck --check-prefix=DISASM-12T %s
+DISASM-12T:      0000000180001000 <.text>:
+DISASM-12T-NEXT: 180001000: f0000010     adrp    x16, 0x180004000
+DISASM-12T-NEXT: 180001004: f9400610     ldr     x16, [x16, #0x8]
+DISASM-12T-NEXT: 180001008: d61f0200     br      x16
+DISASM-12T-NEXT:                 ...
+DISASM-12T-NEXT: 180002000: 52800040     mov     w0, #0x2                // =2
+DISASM-12T-NEXT: 180002004: d65f03c0     ret
+DISASM-12T-NEXT: 180002008: 90000030     adrp    x16, 0x180006000
+DISASM-12T-NEXT: 18000200c: f9400210     ldr     x16, [x16]
+DISASM-12T-NEXT: 180002010: d61f0200     br      x16
+DISASM-12T-NEXT: 180002014: d000000b     adrp    x11, 0x180004000
+DISASM-12T-NEXT: 180002018: f940016b     ldr     x11, [x11]
+DISASM-12T-NEXT: 18000201c: 9000000a     adrp    x10, 0x180002000 <.text+0x1000>
+DISASM-12T-NEXT: 180002020: 9100f14a     add     x10, x10, #0x3c
+DISASM-12T-NEXT: 180002024: 17fffff7     b       0x180002000 <.text+0x1000>
+DISASM-12T-NEXT: 180002028: d000000b     adrp    x11, 0x180004000
+DISASM-12T-NEXT: 18000202c: f940056b     ldr     x11, [x11, #0x8]
+DISASM-12T-NEXT: 180002030: d0ffffea     adrp    x10, 0x180000000
+DISASM-12T-NEXT: 180002034: 9100014a     add     x10, x10, #0x0
+DISASM-12T-NEXT: 180002038: 17fffff2     b       0x180002000 <.text+0x1000>
+DISASM-12T-NEXT: 18000203c: 52800060     mov     w0, #0x3                // =3
+DISASM-12T-NEXT: 180002040: d65f03c0     ret
+DISASM-12T-NEXT:                 ...
+DISASM-12T-NEXT: 180003000: ff 25 fa 0f 00 00            jmpq    *0xffa(%rip)            # 0x180004000
+
+RUN: llvm-readobj --coff-imports test-12-thunks.dll | FileCheck --check-prefix=IMPORTS-12 %s
+IMPORTS-12:      Import {
+IMPORTS-12-NEXT:   Name: test.dll
+IMPORTS-12-NEXT:   ImportLookupTableRVA: 0x5348
+IMPORTS-12-NEXT:   ImportAddressTableRVA: 0x4000
+IMPORTS-12-NEXT:   Symbol: func1 (0)
+IMPORTS-12-NEXT:   Symbol: func2 (0)
+IMPORTS-12-NEXT: }
+IMPORTS-12-NEXT: HybridObject {
+IMPORTS-12:        Import {
+IMPORTS-12-NEXT:     Name: test.dll
+IMPORTS-12-NEXT:     ImportLookupTableRVA: 0x5348
+IMPORTS-12-NEXT:     ImportAddressTableRVA: 0x4000
+IMPORTS-12-NEXT:     Symbol: func1 (0)
+IMPORTS-12-NEXT:     Symbol: func2 (0)
+IMPORTS-12-NEXT:   }
+IMPORTS-12-NEXT: }
+
+RUN: llvm-readobj --hex-dump=.test test-12-thunks.dll | FileCheck --check-prefix=FUNC-12-THUNKS %s
+FUNC-12-THUNKS:      0x180009000 00600000 00400000 00300000 08200000
+FUNC-12-THUNKS-NEXT: 0x180009010 08600000 08400000
+
+RUN: llvm-readobj --hex-dump=.testa test-12-thunks.dll | FileCheck --check-prefix=FUNC-12-THUNKSA %s
+FUNC-12-THUNKSA: 0x18000a000 00400000 08400000 00100000
+
+
+# If the ordinals of named imports don't match, use the EC value.
+
+RUN: lld-link -machine:arm64x -dll -noentry -out:test-12-thunks-ord.dll loadconfig-arm64.obj loadconfig-arm64ec.obj \
+RUN:          icall.obj func12-thunks-arm64ec.obj func12-thunks-arm64.obj imp-ecord.lib
+RUN: llvm-readobj --coff-imports test-12-thunks-ord.dll | FileCheck --check-prefix=IMPORTS-ORD %s
+
+IMPORTS-ORD:      Format: COFF-ARM64X
+IMPORTS-ORD-NEXT: Arch: aarch64
+IMPORTS-ORD-NEXT: AddressSize: 64bit
+IMPORTS-ORD-NEXT: Import {
+IMPORTS-ORD-NEXT:   Name: test.dll
+IMPORTS-ORD-NEXT:   ImportLookupTableRVA: 0x5348
+IMPORTS-ORD-NEXT:   ImportAddressTableRVA: 0x4000
+IMPORTS-ORD-NEXT:   Symbol: func1 (11)
+IMPORTS-ORD-NEXT:   Symbol: func2 (12)
+IMPORTS-ORD-NEXT: }
+IMPORTS-ORD-NEXT: HybridObject {
+IMPORTS-ORD-NEXT:   Format: COFF-ARM64EC
+IMPORTS-ORD-NEXT:   Arch: aarch64
+IMPORTS-ORD-NEXT:   AddressSize: 64bit
+IMPORTS-ORD-NEXT:   Import {
+IMPORTS-ORD-NEXT:     Name: test.dll
+IMPORTS-ORD-NEXT:     ImportLookupTableRVA: 0x5348
+IMPORTS-ORD-NEXT:     ImportAddressTableRVA: 0x4000
+IMPORTS-ORD-NEXT:     Symbol: func1 (11)
+IMPORTS-ORD-NEXT:     Symbol: func2 (12)
+IMPORTS-ORD-NEXT:   }
+IMPORTS-ORD-NEXT: }
+
+
+# Link to NONAME imports.
+
+RUN: lld-link -machine:arm64x -dll -noentry -out:test-noname.dll loadconfig-arm64.obj loadconfig-arm64ec.obj \
+RUN:          icall.obj func12-thunks-arm64ec.obj func12-thunks-arm64.obj noname.lib
+RUN: llvm-readobj --coff-imports test-noname.dll | FileCheck --check-prefix=IMPORTS-ORD2 %s
+
+IMPORTS-ORD2:      Format: COFF-ARM64X
+IMPORTS-ORD2-NEXT: Arch: aarch64
+IMPORTS-ORD2-NEXT: AddressSize: 64bit
+IMPORTS-ORD2-NEXT: Import {
+IMPORTS-ORD2-NEXT:   Name: test.dll
+IMPORTS-ORD2-NEXT:   ImportLookupTableRVA: 0x5348
+IMPORTS-ORD2-NEXT:   ImportAddressTableRVA: 0x4000
+IMPORTS-ORD2-NEXT:   Symbol:  (12)
+IMPORTS-ORD2-NEXT:   Symbol:  (11)
+IMPORTS-ORD2-NEXT: }
+IMPORTS-ORD2-NEXT: HybridObject {
+IMPORTS-ORD2-NEXT:   Format: COFF-ARM64EC
+IMPORTS-ORD2-NEXT:   Arch: aarch64
+IMPORTS-ORD2-NEXT:   AddressSize: 64bit
+IMPORTS-ORD2-NEXT:   Import {
+IMPORTS-ORD2-NEXT:     Name: test.dll
+IMPORTS-ORD2-NEXT:     ImportLookupTableRVA: 0x5350
+IMPORTS-ORD2-NEXT:     ImportAddressTableRVA: 0x4008
+IMPORTS-ORD2-NEXT:     Symbol:  (11)
+IMPORTS-ORD2-NEXT:     Symbol:  (10)
+IMPORTS-ORD2-NEXT:   }
+IMPORTS-ORD2-NEXT: }
+
+# Link to the imported func1 and func2 from both native and EC code, and func3 from native code.
+
+RUN: lld-link -machine:arm64x -dll -noentry -out:test2.dll loadconfig-arm64.obj loadconfig-arm64ec.obj \
+RUN:          icall.obj func12-arm64ec.obj func123-arm64.obj imp-arm64x.lib
+
+RUN: llvm-readobj --coff-imports test2.dll | FileCheck --check-prefix=IMPORTS-123-12 %s
+IMPORTS-123-12:      Import {
+IMPORTS-123-12-NEXT:   Name: test.dll
+IMPORTS-123-12-NEXT:   ImportLookupTableRVA: 0x3338
+IMPORTS-123-12-NEXT:   ImportAddressTableRVA: 0x2000
+IMPORTS-123-12-NEXT:   Symbol: func3 (0)
+IMPORTS-123-12-NEXT:   Symbol: func1 (0)
+IMPORTS-123-12-NEXT:   Symbol: func2 (0)
+IMPORTS-123-12-NEXT: }
+IMPORTS-123-12-NEXT: HybridObject {
+IMPORTS-123-12:        Import {
+IMPORTS-123-12-NEXT:     Name: test.dll
+IMPORTS-123-12-NEXT:     ImportLookupTableRVA: 0x3340
+IMPORTS-123-12-NEXT:     ImportAddressTableRVA: 0x2008
+IMPORTS-123-12-NEXT:     Symbol: func1 (0)
+IMPORTS-123-12-NEXT:     Symbol: func2 (0)
+IMPORTS-123-12-NEXT:   }
+IMPORTS-123-12-NEXT: }
+
+RUN: llvm-readobj --hex-dump=.test test2.dll | FileCheck --check-prefix=TEST-123-12 %s
+TEST-123-12: 0x180007000 08400000 08200000 10400000 10200000
+
+RUN: llvm-readobj --hex-dump=.testa test2.dll | FileCheck --check-prefix=TEST-123-12A %s
+TEST-123-12A: 0x180008000 08200000 10200000 00200000
+
+RUN: llvm-readobj --hex-dump=.rdata test2.dll | FileCheck --check-prefix=TEST-123-12AUX %s
+TEST-123-12AUX:      0x180004000 00000000 00000000 08100080 01000000
+TEST-123-12AUX-NEXT: 0x180004010 1c100080 01000000 00000000 00000000
+
+
+# Link to the imported func1 and func2 from both native and EC code, and func3 from EC code.
+
+RUN: lld-link -machine:arm64x -dll -noentry -out:func-12-123.dll loadconfig-arm64.obj loadconfig-arm64ec.obj \
+RUN:          icall.obj func123-arm64ec.obj func12-arm64.obj imp-arm64x.lib
+
+RUN: llvm-readobj --coff-imports func-12-123.dll | FileCheck --check-prefix=IMPORTS-12-123 %s
+IMPORTS-12-123:      Import {
+IMPORTS-12-123-NEXT:   Name: test.dll
+IMPORTS-12-123-NEXT:   ImportLookupTableRVA: 0x3338
+IMPORTS-12-123-NEXT:   ImportAddressTableRVA: 0x2000
+IMPORTS-12-123-NEXT:   Symbol: func1 (0)
+IMPORTS-12-123-NEXT:   Symbol: func2 (0)
+IMPORTS-12-123-NEXT: }
+IMPORTS-12-123-NEXT: HybridObject {
+IMPORTS-12-123:        Import {
+IMPORTS-12-123-NEXT:     Name: test.dll
+IMPORTS-12-123-NEXT:     ImportLookupTableRVA: 0x3338
+IMPORTS-12-123-NEXT:     ImportAddressTableRVA: 0x2000
+IMPORTS-12-123-NEXT:     Symbol: func1 (0)
+IMPORTS-12-123-NEXT:     Symbol: func2 (0)
+IMPORTS-12-123-NEXT:     Symbol: func3 (0)
+IMPORTS-12-123-NEXT:   }
+IMPORTS-12-123-NEXT: }
+
+RUN: llvm-readobj --hex-dump=.test func-12-123.dll | FileCheck --check-prefix=TEST-12-123 %s
+TEST-12-123:      0x180007000 00400000 00200000 08400000 08200000
+TEST-12-123-NEXT: 0x180007010 10400000 10200000
+
+RUN: llvm-readobj --hex-dump=.testa func-12-123.dll | FileCheck --check-prefix=TEST-12-123A %s
+TEST-12-123A: 0x180008000 00200000 08200000
+
+RUN: llvm-readobj --hex-dump=.rdata func-12-123.dll | FileCheck --check-prefix=TEST-12-123AUX %s
+TEST-12-123AUX:      0x180004000 08100080 01000000 1c100080 01000000
+TEST-12-123AUX-NEXT: 0x180004010 30100080 01000000 00000000 00000000
+
+
+# Link to the imported func2 and func3 from both native and EC code, func4 from native code,
+# and func1 from EC code.
+
+RUN: lld-link -machine:arm64x -dll -noentry -out:test-234-123.dll loadconfig-arm64.obj loadconfig-arm64ec.obj \
+RUN:          icall.obj func123-arm64ec.obj func234-arm64.obj imp-arm64x.lib
+
+RUN: llvm-readobj --coff-imports test-234-123.dll | FileCheck --check-prefix=IMPORTS-234-123 %s
+IMPORTS-234-123:      Import {
+IMPORTS-234-123-NEXT:   Name: test.dll
+IMPORTS-234-123-NEXT:   ImportLookupTableRVA: 0x3338
+IMPORTS-234-123-NEXT:   ImportAddressTableRVA: 0x2000
+IMPORTS-234-123-NEXT:   Symbol: func4 (0)
+IMPORTS-234-123-NEXT:   Symbol: func2 (0)
+IMPORTS-234-123-NEXT:   Symbol: func3 (0)
+IMPORTS-234-123-NEXT: }
+IMPORTS-234-123-NEXT: HybridObject {
+IMPORTS-234-123:        Import {
+IMPORTS-234-123-NEXT:     Name: test.dll
+IMPORTS-234-123-NEXT:     ImportLookupTableRVA: 0x3340
+IMPORTS-234-123-NEXT:     ImportAddressTableRVA: 0x2008
+IMPORTS-234-123-NEXT:     Symbol: func2 (0)
+IMPORTS-234-123-NEXT:     Symbol: func3 (0)
+IMPORTS-234-123-NEXT:     Symbol: func1 (0)
+IMPORTS-234-123-NEXT:   }
+IMPORTS-234-123-NEXT: }
+
+RUN: llvm-readobj --hex-dump=.test test-234-123.dll | FileCheck --check-prefix=TEST-234-123 %s
+TEST-234-123:      0x180007000 18400000 18200000 08400000 08200000
+TEST-234-123-NEXT: 0x180007010 10400000 1020000
+
+RUN: llvm-readobj --hex-dump=.testa test-234-123.dll | FileCheck --check-prefix=TEST-234-123A %s
+TEST-234-123A: 0x180008000 08200000 10200000 00200000
+
+
+# Link to the imported func3 and func4 from native code, and func1 and func2 from EC code.
+
+RUN: lld-link -machine:arm64x -dll -noentry -out:test-34-12.dll loadconfig-arm64.obj loadconfig-arm64ec.obj \
+RUN:          icall.obj func12o-arm64ec.obj func34o-arm64.obj imp-arm64x.lib imp2.lib
+
+RUN: llvm-readobj --coff-imports test-34-12.dll | FileCheck --check-prefix=IMPORTS-34-12 %s
+IMPORTS-34-12:      Import {
+IMPORTS-34-12-NEXT:   Name: test.dll
+IMPORTS-34-12-NEXT:   ImportLookupTableRVA: 0x3350
+IMPORTS-34-12-NEXT:   ImportAddressTableRVA: 0x2000
+IMPORTS-34-12-NEXT:   Symbol: func3 (0)
+IMPORTS-34-12-NEXT:   Symbol: func4 (0)
+IMPORTS-34-12-NEXT: }
+IMPORTS-34-12-NEXT: Import {
+IMPORTS-34-12-NEXT:   Name: test2.dll
+IMPORTS-34-12-NEXT:   ImportLookupTableRVA: 0x3378
+IMPORTS-34-12-NEXT:   ImportAddressTableRVA: 0x2028
+IMPORTS-34-12-NEXT:   Symbol: otherfunc (0)
+IMPORTS-34-12-NEXT: }
+IMPORTS-34-12-NEXT: HybridObject {
+IMPORTS-34-12:        Import {
+IMPORTS-34-12-NEXT:     Name: test.dll
+IMPORTS-34-12-NEXT:     ImportLookupTableRVA: 0x3360
+IMPORTS-34-12-NEXT:     ImportAddressTableRVA: 0x2010
+IMPORTS-34-12-NEXT:     Symbol: func1 (0)
+IMPORTS-34-12-NEXT:     Symbol: func2 (0)
+IMPORTS-34-12-NEXT:   }
+IMPORTS-34-12-NEXT:   Import {
+IMPORTS-34-12-NEXT:     Name: test2.dll
+IMPORTS-34-12-NEXT:     ImportLookupTableRVA: 0x3378
+IMPORTS-34-12-NEXT:     ImportAddressTableRVA: 0x2028
+IMPORTS-34-12-NEXT:     Symbol: otherfunc (0)
+IMPORTS-34-12-NEXT:   }
+IMPORTS-34-12-NEXT: }
+
+RUN: llvm-readobj --hex-dump=.test test-34-12.dll | FileCheck --check-prefix=TEST-23-12 %s
+TEST-23-12:      0x180007000 10400000 10200000 18400000 18200000
+TEST-23-12-NEXT: 0x180007010 28400000 28200000
+
+RUN: llvm-readobj --hex-dump=.testa test-34-12.dll | FileCheck --check-prefix=TEST-23-12A %s
+TEST-23-12A: 0x180008000 00200000 08200000 28200000
+
+
+# Link only to imported EC functions, with no native imports.
+
+RUN: lld-link -machine:arm64x -dll -noentry -out:test-ec12.dll loadconfig-arm64.obj loadconfig-arm64ec.obj \
+RUN:          icall.obj func12-arm64ec.obj funco-arm64.obj imp-arm64x.lib imp2.lib
+
+RUN: llvm-readobj --coff-imports test-ec12.dll | FileCheck --check-prefix=IMPORTS-EC12 %s
+
+IMPORTS-EC12:      File: test-ec12.dll
+IMPORTS-EC12-NEXT: Format: COFF-ARM64X
+IMPORTS-EC12-NEXT: Arch: aarch64
+IMPORTS-EC12-NEXT: AddressSize: 64bit
+IMPORTS-EC12-NEXT: Import {
+IMPORTS-EC12-NEXT:   Name: test.dll
+IMPORTS-EC12-NEXT:   ImportLookupTableRVA: 0x3350
+IMPORTS-EC12-NEXT:   ImportAddressTableRVA: 0x2000
+IMPORTS-EC12-NEXT: }
+IMPORTS-EC12-NEXT: Import {
+IMPORTS-EC12-NEXT:   Name: test2.dll
+IMPORTS-EC12-NEXT:   ImportLookupTableRVA: 0x3368
+IMPORTS-EC12-NEXT:   ImportAddressTableRVA: 0x2018
+IMPORTS-EC12-NEXT:   Symbol: otherfunc (0)
+IMPORTS-EC12-NEXT: }
+IMPORTS-EC12-NEXT: HybridObject {
+IMPORTS-EC12-NEXT:   Format: COFF-ARM64EC
+IMPORTS-EC12-NEXT:   Arch: aarch64
+IMPORTS-EC12-NEXT:   AddressSize: 64bit
+IMPORTS-EC12-NEXT:   Import {
+IMPORTS-EC12-NEXT:     Name: test.dll
+IMPORTS-EC12-NEXT:     ImportLookupTableRVA: 0x3350
+IMPORTS-EC12-NEXT:     ImportAddressTableRVA: 0x2000
+IMPORTS-EC12-NEXT:     Symbol: func1 (0)
+IMPORTS-EC12-NEXT:     Symbol: func2 (0)
+IMPORTS-EC12-NEXT:   }
+IMPORTS-EC12-NEXT:   Import {
+IMPORTS-EC12-NEXT:     Name: test2.dll
+IMPORTS-EC12-NEXT:     ImportLookupTableRVA: 0x3370
+IMPORTS-EC12-NEXT:     ImportAddressTableRVA: 0x2020
+IMPORTS-EC12-NEXT:   }
+IMPORTS-EC12-NEXT: }
+
+
+# Link only to imported native functions, with no EC imports.
+
+RUN: lld-link -machine:arm64x -dll -noentry -out:test-n12.dll loadconfig-arm64.obj loadconfig-arm64ec.obj \
+RUN:          func12-arm64.obj imp-arm64x.lib
+
+RUN: llvm-readobj --coff-imports test-n12.dll | FileCheck --check-prefix=IMPORTS-N12 %s
+
+IMPORTS-N12:      Arch: aarch64
+IMPORTS-N12-NEXT: AddressSize: 64bit
+IMPORTS-N12-NEXT: Import {
+IMPORTS-N12-NEXT:   Name: test.dll
+IMPORTS-N12-NEXT:   ImportLookupTableRVA: 0x2330
+IMPORTS-N12-NEXT:   ImportAddressTableRVA: 0x1000
+IMPORTS-N12-NEXT:   Symbol: func1 (0)
+IMPORTS-N12-NEXT:   Symbol: func2 (0)
+IMPORTS-N12-NEXT: }
+IMPORTS-N12-NEXT: HybridObject {
+IMPORTS-N12-NEXT:   Format: COFF-ARM64EC
+IMPORTS-N12-NEXT:   Arch: aarch64
+IMPORTS-N12-NEXT:   AddressSize: 64bit
+IMPORTS-N12-NEXT:   Import {
+IMPORTS-N12-NEXT:     Name: test.dll
+IMPORTS-N12-NEXT:     ImportLookupTableRVA: 0x2340
+IMPORTS-N12-NEXT:     ImportAddressTableRVA: 0x1010
+IMPORTS-N12-NEXT:   }
+IMPORTS-N12-NEXT: }
+
+
+RUN: lld-link -machine:arm64x -dll -noentry -out:test-dup.dll loadconfig-arm64.obj loadconfig-arm64ec.obj icall.obj \
+RUN:          func12-arm64ec.obj func34-arm64.obj dup.lib
+
+RUN: llvm-readobj --coff-imports test-dup.dll | FileCheck --check-prefix=IMPORTS-DUP %s
+IMPORTS-DUP:      Format: COFF-ARM64X
+IMPORTS-DUP-NEXT: Arch: aarch64
+IMPORTS-DUP-NEXT: AddressSize: 64bit
+IMPORTS-DUP-NEXT: Import {
+IMPORTS-DUP-NEXT:   Name: test.dll
+IMPORTS-DUP-NEXT:   ImportLookupTableRVA: 0x3338
+IMPORTS-DUP-NEXT:   ImportAddressTableRVA: 0x2000
+IMPORTS-DUP-NEXT:   Symbol: func4 (0)
+IMPORTS-DUP-NEXT:   Symbol: func4 (0)
+IMPORTS-DUP-NEXT: }
+IMPORTS-DUP-NEXT: HybridObject {
+IMPORTS-DUP-NEXT:   Format: COFF-ARM64EC
+IMPORTS-DUP-NEXT:   Arch: aarch64
+IMPORTS-DUP-NEXT:   AddressSize: 64bit
+IMPORTS-DUP-NEXT:   Import {
+IMPORTS-DUP-NEXT:     Name: test.dll
+IMPORTS-DUP-NEXT:     ImportLookupTableRVA: 0x3348
+IMPORTS-DUP-NEXT:     ImportAddressTableRVA: 0x2010
+IMPORTS-DUP-NEXT:     Symbol: func1 (0)
+IMPORTS-DUP-NEXT:     Symbol: func1 (0)
+IMPORTS-DUP-NEXT:   }
+IMPORTS-DUP-NEXT: }
+
+#--- func12-thunks-arm64ec.s
+    .section .test, "r"
+    .rva __imp_func1
+    .rva __imp_aux_func1
+    .rva func1
+    .rva "#func1"
+    .rva __imp_func2
+    .rva __imp_aux_func2
+
+#--- func12-thunks-arm64.s
+    .section .testa, "r"
+    .rva __imp_func1
+    .rva __imp_func2
+    .rva func2
+
+#--- func12-arm64ec.s
+    .section .test, "r"
+    .rva __imp_func1
+    .rva __imp_aux_func1
+    .rva __imp_func2
+    .rva __imp_aux_func2
+
+#--- func123-arm64.s
+    .section .testa, "r"
+    .rva __imp_func1
+    .rva __imp_func2
+    .rva __imp_func3
+
+#--- func123-arm64ec.s
+    .section .test, "r"
+    .rva __imp_func1
+    .rva __imp_aux_func1
+    .rva __imp_func2
+    .rva __imp_aux_func2
+    .rva __imp_func3
+    .rva __imp_aux_func3
+
+#--- func12-arm64.s
+    .section .testa, "r"
+    .rva __imp_func1
+    .rva __imp_func2
+
+#--- func234-arm64.s
+    .section .testa, "r"
+    .rva __imp_func2
+    .rva __imp_func3
+    .rva __imp_func4
+
+#--- func12o-arm64ec.s
+    .section .test, "r"
+    .rva __imp_func1
+    .rva __imp_aux_func1
+    .rva __imp_func2
+    .rva __imp_aux_func2
+    .rva __imp_otherfunc
+    .rva __imp_aux_otherfunc
+
+#--- func34-arm64.s
+    .section .testa, "r"
+    .rva __imp_func3
+    .rva __imp_func4
+
+#--- func34o-arm64.s
+    .section .testa, "r"
+    .rva __imp_func3
+    .rva __imp_func4
+    .rva __imp_otherfunc
+
+#--- funco-arm64.s
+    .section .testa, "r"
+    .rva __imp_otherfunc
+
+#--- icall.s
+    .text
+    .globl __icall_helper_arm64ec
+    .p2align 2, 0x0
+__icall_helper_arm64ec:
+    mov w0, #2
+    ret
+
+    .section .hybmp$x, "yi"
+    .symidx __imp_func1
+    .symidx func1_exit_thunk
+    .word 4
+
+    .section .wowthk$aa,"xr",discard,func1_exit_thunk
+    .globl func1_exit_thunk
+func1_exit_thunk:
+    mov w0, #3
+    ret
+
+#--- imp.def
+NAME test.dll
+EXPORTS
+    data_sym DATA
+    func1
+    func2
+    func3
+    func4
+
+#--- imp-ord10.def
+NAME test.dll
+EXPORTS
+    data_sym DATA @10
+    func1 @11
+    func2 @12
+    func3 @13
+    func4 @14
+
+#--- imp-ord20.def
+NAME test.dll
+EXPORTS
+    data_sym DATA @10
+    func1 @21
+    func2 @22
+    func3 @23
+    func4 @24
+
+#--- imp2.def
+NAME test2.dll
+EXPORTS
+    otherfunc
+
+#--- noname-ec.def
+NAME test.dll
+EXPORTS
+    func1 @10 NONAME
+    func2 @11 NONAME
+
+#--- noname-native.def
+NAME test.dll
+EXPORTS
+    func1 @12 NONAME
+    func2 @11 NONAME
+
+#--- dup-ec.def
+NAME test.dll
+EXPORTS
+    func1
+    func2 EXPORTAS func1
+
+#--- dup-native.def
+NAME test.dll
+EXPORTS
+    func3 EXPORTAS func4
+    func4
diff --git a/lld/test/COFF/arm64x-includeoptional.s b/lld/test/COFF/arm64x-includeoptional.s
new file mode 100644
index 0000000000000..bf02075d96779
--- /dev/null
+++ b/lld/test/COFF/arm64x-includeoptional.s
@@ -0,0 +1,17 @@
+// REQUIRES: aarch64
+
+// RUN: llvm-mc -filetype=obj -triple=aarch64-windows %s -o %t.arm64.obj
+// RUN: llvm-mc -filetype=obj -triple=arm64ec-windows %s -o %t.arm64ec.obj
+// RUN: llvm-mc -filetype=obj -triple=arm64ec-windows %S/Inputs/loadconfig-arm64ec.s -o %t-loadconfig-arm64ec.obj
+// RUN: llvm-mc -filetype=obj -triple=aarch64-windows %S/Inputs/loadconfig-arm64.s -o %t-loadconfig-arm64.obj
+
+// RUN: llvm-lib -machine:arm64x -out:%t-test.lib %t.arm64.obj %t.arm64ec.obj %t-loadconfig-arm64ec.obj %t-loadconfig-arm64.obj
+// RUN: lld-link -machine:arm64x -dll -noentry -out:%t.dll %t-test.lib -includeoptional:sym
+
+// RUN: llvm-readobj --hex-dump=.test %t.dll | FileCheck %s
+// CHECK: 0x180004000 01000000 01000000
+
+        .globl sym
+        .section .test,"dr"
+sym:
+        .word 1
diff --git a/lld/test/COFF/arm64x-wrap.s b/lld/test/COFF/arm64x-wrap.s
new file mode 100644
index 0000000000000..4f600e38f7a83
--- /dev/null
+++ b/lld/test/COFF/arm64x-wrap.s
@@ -0,0 +1,31 @@
+// REQUIRES: aarch64
+// RUN: split-file %s %t.dir && cd %t.dir
+
+// RUN: llvm-mc -filetype=obj -triple=arm64ec-windows test.s -o test-arm64ec.obj
+// RUN: llvm-mc -filetype=obj -triple=aarch64-windows test.s -o test-arm64.obj
+// RUN: llvm-mc -filetype=obj -triple=arm64ec-windows other.s -o other-arm64ec.obj
+// RUN: llvm-mc -filetype=obj -triple=aarch64-windows other.s -o other-arm64.obj
+// RUN: llvm-mc -filetype=obj -triple=arm64ec-windows %S/Inputs/loadconfig-arm64ec.s -o loadconfig-arm64ec.obj
+// RUN: llvm-mc -filetype=obj -triple=aarch64-windows %S/Inputs/loadconfig-arm64.s -o loadconfig-arm64.obj
+
+// RUN: lld-link -machine:arm64x -dll -noentry test-arm64.obj test-arm64ec.obj other-arm64.obj other-arm64ec.obj \
+// RUN:          loadconfig-arm64.obj loadconfig-arm64ec.obj -out:out.dll -wrap:sym -wrap:nosuchsym
+
+// RUN: llvm-readobj --hex-dump=.test out.dll | FileCheck %s
+// CHECK: 0x180004000 02000000 02000000 01000000 02000000
+// CHECK: 0x180004010 02000000 01000000
+
+#--- test.s
+        .section .test,"dr"
+        .word sym
+        .word __wrap_sym
+        .word __real_sym
+
+#--- other.s
+        .global sym
+        .global __wrap_sym
+        .global __real_sym
+
+sym = 1
+__wrap_sym = 2
+__real_sym = 3
diff --git a/lld/test/COFF/delayimporttables.yaml b/lld/test/COFF/delayimporttables.yaml
index f1e7c61f55a5e..cf54c0a7140a1 100644
--- a/lld/test/COFF/delayimporttables.yaml
+++ b/lld/test/COFF/delayimporttables.yaml
@@ -1,8 +1,9 @@
 # RUN: yaml2obj %p/Inputs/delayimporttables-dll1.yaml -o %t1.obj
 # RUN: yaml2obj %p/Inputs/delayimporttables-dll2.yaml -o %t2.obj
 # RUN: yaml2obj %s -o %t.obj
-# RUN: lld-link /nodefaultlib /entry:DllMain /out:%T/delayimporttables-dll1.dll /dll /implib:%t-dll1.lib %t1.obj
-# RUN: lld-link /nodefaultlib /entry:DllMain /out:%T/delayimporttables-dll2.dll /dll /implib:%t-dll2.lib %t2.obj
+# RUN: mkdir -p %t.dir
+# RUN: lld-link /nodefaultlib /entry:DllMain /out:%t.dir/delayimporttables-dll1.dll /dll /implib:%t-dll1.lib %t1.obj
+# RUN: lld-link /nodefaultlib /entry:DllMain /out:%t.dir/delayimporttables-dll2.dll /dll /implib:%t-dll2.lib %t2.obj
 # RUN: lld-link /nodefaultlib /entry:main /out:%t.exe /delayload:delayimporttables-dll1.dll /delayload:delayimporttables-dll2.dll %t.obj %t-dll1.lib %t-dll2.lib
 # RUN: llvm-readobj --coff-imports %t.exe |FileCheck %s
 
diff --git a/lld/test/COFF/export-all.s b/lld/test/COFF/export-all.s
index a932bf0bba427..cd0a6f5f7b35a 100644
--- a/lld/test/COFF/export-all.s
+++ b/lld/test/COFF/export-all.s
@@ -81,14 +81,14 @@ __imp__unexported:
 
 # RUN: echo -e ".global foobar\n.global DllMainCRTStartup\n.text\nDllMainCRTStartup:\nret\nfoobar:\ncall mingwfunc\ncall crtfunc\nret\n" > %t.main.s
 # RUN: llvm-mc -triple=x86_64-windows-gnu %t.main.s -filetype=obj -o %t.main.obj
-# RUN: mkdir -p %T/libs
-# RUN: echo -e ".global mingwfunc\n.text\nmingwfunc:\nret\n" > %T/libs/mingwfunc.s
-# RUN: llvm-mc -triple=x86_64-windows-gnu %T/libs/mingwfunc.s -filetype=obj -o %T/libs/mingwfunc.o
-# RUN: rm -f %T/libs/libmingwex.a
-# RUN: llvm-ar rcs %T/libs/libmingwex.a %T/libs/mingwfunc.o
-# RUN: echo -e ".global crtfunc\n.text\ncrtfunc:\nret\n" > %T/libs/crtfunc.s
-# RUN: llvm-mc -triple=x86_64-windows-gnu %T/libs/crtfunc.s -filetype=obj -o %T/libs/crt2.o
-# RUN: lld-link -safeseh:no -out:%t.dll -dll -entry:DllMainCRTStartup %t.main.obj -lldmingw %T/libs/crt2.o %T/libs/libmingwex.a -output-def:%t.def
+# RUN: mkdir -p %t.dir/libs
+# RUN: echo -e ".global mingwfunc\n.text\nmingwfunc:\nret\n" > %t.dir/libs/mingwfunc.s
+# RUN: llvm-mc -triple=x86_64-windows-gnu %t.dir/libs/mingwfunc.s -filetype=obj -o %t.dir/libs/mingwfunc.o
+# RUN: rm -f %t.dir/libs/libmingwex.a
+# RUN: llvm-ar rcs %t.dir/libs/libmingwex.a %t.dir/libs/mingwfunc.o
+# RUN: echo -e ".global crtfunc\n.text\ncrtfunc:\nret\n" > %t.dir/libs/crtfunc.s
+# RUN: llvm-mc -triple=x86_64-windows-gnu %t.dir/libs/crtfunc.s -filetype=obj -o %t.dir/libs/crt2.o
+# RUN: lld-link -safeseh:no -out:%t.dll -dll -entry:DllMainCRTStartup %t.main.obj -lldmingw %t.dir/libs/crt2.o %t.dir/libs/libmingwex.a -output-def:%t.def
 # RUN: echo "EOF" >> %t.def
 # RUN: cat %t.def | FileCheck -check-prefix=CHECK-EXCLUDE %s
 
@@ -99,7 +99,7 @@ __imp__unexported:
 # Test that libraries included with -wholearchive: are autoexported, even if
 # they are in a library that otherwise normally would be excluded.
 
-# RUN: lld-link -safeseh:no -out:%t.dll -dll -entry:DllMainCRTStartup %t.main.obj -lldmingw %T/libs/crt2.o -wholearchive:%T/libs/libmingwex.a -output-def:%t.def
+# RUN: lld-link -safeseh:no -out:%t.dll -dll -entry:DllMainCRTStartup %t.main.obj -lldmingw %t.dir/libs/crt2.o -wholearchive:%t.dir/libs/libmingwex.a -output-def:%t.def
 # RUN: echo "EOF" >> %t.def
 # RUN: cat %t.def | FileCheck -check-prefix=CHECK-WHOLEARCHIVE %s
 
diff --git a/lld/test/COFF/filename-casing.s b/lld/test/COFF/filename-casing.s
index 0834c034ee74f..46819fb282edf 100644
--- a/lld/test/COFF/filename-casing.s
+++ b/lld/test/COFF/filename-casing.s
@@ -1,10 +1,11 @@
 # REQUIRES: x86
 
-# RUN: llvm-mc -filetype=obj -triple=x86_64-windows-msvc -o %T/MixedCase.obj %s
-# RUN: not lld-link /entry:main %T/MixedCase.obj 2>&1 | FileCheck -check-prefix=OBJECT %s
+# RUN: mkdir -p %t.dir
+# RUN: llvm-mc -filetype=obj -triple=x86_64-windows-msvc -o %t.dir/MixedCase.obj %s
+# RUN: not lld-link /entry:main %t.dir/MixedCase.obj 2>&1 | FileCheck -check-prefix=OBJECT %s
 
-# RUN: llvm-lib /out:%T/MixedCase.lib %T/MixedCase.obj
-# RUN: not lld-link /machine:x64 /entry:main %T/MixedCase.lib 2>&1 | FileCheck -check-prefix=ARCHIVE %s
+# RUN: llvm-lib /out:%t.dir/MixedCase.lib %t.dir/MixedCase.obj
+# RUN: not lld-link /machine:x64 /entry:main %t.dir/MixedCase.lib 2>&1 | FileCheck -check-prefix=ARCHIVE %s
 
 # OBJECT: undefined symbol: f
 # OBJECT-NEXT: >>> referenced by {{.*}}MixedCase.obj:(main)
diff --git a/lld/test/COFF/guardcf-align.s b/lld/test/COFF/guardcf-align.s
index 0535dc8ac3d89..f70fd0a82c002 100644
--- a/lld/test/COFF/guardcf-align.s
+++ b/lld/test/COFF/guardcf-align.s
@@ -1,10 +1,11 @@
 # REQUIRES: x86
+# RUN: mkdir -p %t.dir
 # RUN: llvm-mc -triple x86_64-windows-msvc -filetype=obj -o %t.obj %s
 # RUN: yaml2obj %p/Inputs/guardcf-align-foobar.yaml \
-# RUN:     > %T/guardcf-align-foobar.obj
-# RUN: lld-link -out:%T/guardcf-align.exe -entry:main -guard:cf \
-# RUN:     %t.obj %T/guardcf-align-foobar.obj
-# RUN: llvm-readobj --coff-load-config %T/guardcf-align.exe | FileCheck %s
+# RUN:     > %t.dir/guardcf-align-foobar.obj
+# RUN: lld-link -out:%t.dir/guardcf-align.exe -entry:main -guard:cf \
+# RUN:     %t.obj %t.dir/guardcf-align-foobar.obj
+# RUN: llvm-readobj --coff-load-config %t.dir/guardcf-align.exe | FileCheck %s
 
 # Check that the gfids table contains at least one entry that ends in 0
 # and no entries that end in something other than 0.
diff --git a/lld/test/COFF/implib-name.test b/lld/test/COFF/implib-name.test
index 3ec8d823a91da..ffb74d7c53c5b 100644
--- a/lld/test/COFF/implib-name.test
+++ b/lld/test/COFF/implib-name.test
@@ -1,73 +1,73 @@
 # REQUIRES: x86
-# RUN: mkdir -p %T
-# RUN: llvm-mc -triple x86_64-unknown-windows-msvc -filetype obj -o %T/object.obj %S/Inputs/object.s
+# RUN: mkdir -p %t.dir
+# RUN: llvm-mc -triple x86_64-unknown-windows-msvc -filetype obj -o %t.dir/object.obj %S/Inputs/object.s
 
-# RUN: lld-link /dll /machine:x64 /def:%S/Inputs/named.def /out:%T/library.dll %T/object.obj /entry:f /subsystem:CONSOLE
-# RUN: llvm-ar t %T/library.lib | FileCheck %s -check-prefix CHECK-DEFAULT-DLL-EXT
+# RUN: lld-link /dll /machine:x64 /def:%S/Inputs/named.def /out:%t.dir/library.dll %t.dir/object.obj /entry:f /subsystem:CONSOLE
+# RUN: llvm-ar t %t.dir/library.lib | FileCheck %s -check-prefix CHECK-DEFAULT-DLL-EXT
 
-# RUN: lld-link /machine:x64 /def:%S/Inputs/named.def /out:%T/library.lib
-# RUN: llvm-ar t %T/library.lib | FileCheck %s -check-prefix CHECK-DEFAULT-DLL-EXT
+# RUN: lld-link /machine:x64 /def:%S/Inputs/named.def /out:%t.dir/library.lib
+# RUN: llvm-ar t %t.dir/library.lib | FileCheck %s -check-prefix CHECK-DEFAULT-DLL-EXT
 
 CHECK-DEFAULT-DLL-EXT: library.dll
 CHECK-DEFAULT-DLL-EXT: library.dll
 CHECK-DEFAULT-DLL-EXT: library.dll
 CHECK-DEFAULT-DLL-EXT: library.dll
 
-# RUN: lld-link /machine:x64 /def:%S/Inputs/named.def /out:%T/library.exe %T/object.obj /entry:f /subsystem:CONSOLE
-# RUN: llvm-ar t %T/library.lib | FileCheck %s -check-prefix CHECK-DEFAULT-EXE-EXT
+# RUN: lld-link /machine:x64 /def:%S/Inputs/named.def /out:%t.dir/library.exe %t.dir/object.obj /entry:f /subsystem:CONSOLE
+# RUN: llvm-ar t %t.dir/library.lib | FileCheck %s -check-prefix CHECK-DEFAULT-EXE-EXT
 
 CHECK-DEFAULT-EXE-EXT: library.exe
 CHECK-DEFAULT-EXE-EXT: library.exe
 CHECK-DEFAULT-EXE-EXT: library.exe
 CHECK-DEFAULT-EXE-EXT: library.exe
 
-# RUN: lld-link /dll /machine:x64 /def:%S/Inputs/extension.def /out:%T/extension.dll /entry:f /subsystem:CONSOLE
-# RUN: llvm-ar t %T/extension.lib | FileCheck %s -check-prefix CHECK-EXTENSION
+# RUN: lld-link /dll /machine:x64 /def:%S/Inputs/extension.def /out:%t.dir/extension.dll /entry:f /subsystem:CONSOLE
+# RUN: llvm-ar t %t.dir/extension.lib | FileCheck %s -check-prefix CHECK-EXTENSION
 
-# RUN: lld-link /machine:x64 /def:%S/Inputs/extension.def /out:%T/extension.exe /entry:f /subsystem:CONSOLE
-# RUN: llvm-ar t %T/extension.lib | FileCheck %s -check-prefix CHECK-EXTENSION
+# RUN: lld-link /machine:x64 /def:%S/Inputs/extension.def /out:%t.dir/extension.exe /entry:f /subsystem:CONSOLE
+# RUN: llvm-ar t %t.dir/extension.lib | FileCheck %s -check-prefix CHECK-EXTENSION
 
-# RUN: lld-link /machine:x64 /def:%S/Inputs/extension.def /out:%T/extension.lib
-# RUN: llvm-ar t %T/extension.lib | FileCheck %s -check-prefix CHECK-EXTENSION
+# RUN: lld-link /machine:x64 /def:%S/Inputs/extension.def /out:%t.dir/extension.lib
+# RUN: llvm-ar t %t.dir/extension.lib | FileCheck %s -check-prefix CHECK-EXTENSION
 
 CHECK-EXTENSION: library.ext
 CHECK-EXTENSION: library.ext
 CHECK-EXTENSION: library.ext
 CHECK-EXTENSION: library.ext
 
-# RUN: lld-link /dll /machine:x64 /def:%S/Inputs/default.def /out:%T/default.dll /entry:f /subsystem:CONSOLE
-# RUN: llvm-ar t %T/default.lib | FileCheck %s -check-prefix CHECK-OUTPUT-NAME-DLL
+# RUN: lld-link /dll /machine:x64 /def:%S/Inputs/default.def /out:%t.dir/default.dll /entry:f /subsystem:CONSOLE
+# RUN: llvm-ar t %t.dir/default.lib | FileCheck %s -check-prefix CHECK-OUTPUT-NAME-DLL
 
-# RUN: lld-link /machine:x64 /def:%S/Inputs/default.def /out:%T/default.lib
-# RUN: llvm-ar t %T/default.lib | FileCheck %s -check-prefix CHECK-OUTPUT-NAME-DLL
+# RUN: lld-link /machine:x64 /def:%S/Inputs/default.def /out:%t.dir/default.lib
+# RUN: llvm-ar t %t.dir/default.lib | FileCheck %s -check-prefix CHECK-OUTPUT-NAME-DLL
 
 CHECK-OUTPUT-NAME-DLL: default.dll
 CHECK-OUTPUT-NAME-DLL: default.dll
 CHECK-OUTPUT-NAME-DLL: default.dll
 CHECK-OUTPUT-NAME-DLL: default.dll
 
-# RUN: lld-link /machine:x64 /def:%S/Inputs/default.def /out:%T/default.exe %T/object.obj /entry:f /subsystem:CONSOLE
-# RUN: llvm-ar t %T/default.lib | FileCheck %s -check-prefix CHECK-OUTPUT-NAME-EXE
+# RUN: lld-link /machine:x64 /def:%S/Inputs/default.def /out:%t.dir/default.exe %t.dir/object.obj /entry:f /subsystem:CONSOLE
+# RUN: llvm-ar t %t.dir/default.lib | FileCheck %s -check-prefix CHECK-OUTPUT-NAME-EXE
 
 CHECK-OUTPUT-NAME-EXE: default.exe
 CHECK-OUTPUT-NAME-EXE: default.exe
 CHECK-OUTPUT-NAME-EXE: default.exe
 CHECK-OUTPUT-NAME-EXE: default.exe
 
-# RUN: lld-link /machine:x64 /out:%T/default.exe %T/object.obj /entry:f /subsystem:CONSOLE
-# RUN: llvm-ar t %T/default.lib | FileCheck %s -check-prefix CHECK-NODEF-EXE
+# RUN: lld-link /machine:x64 /out:%t.dir/default.exe %t.dir/object.obj /entry:f /subsystem:CONSOLE
+# RUN: llvm-ar t %t.dir/default.lib | FileCheck %s -check-prefix CHECK-NODEF-EXE
 
 CHECK-NODEF-EXE: default.exe
 CHECK-NODEF-EXE: default.exe
 CHECK-NODEF-EXE: default.exe
 CHECK-NODEF-EXE: default.exe
 
-# RUN: lld-link /machine:x64 /dll /out:%T/default.dll %T/object.obj /entry:f /subsystem:CONSOLE
-# RUN: llvm-ar t %T/default.lib | FileCheck %s -check-prefix CHECK-NODEF-DLL
+# RUN: lld-link /machine:x64 /dll /out:%t.dir/default.dll %t.dir/object.obj /entry:f /subsystem:CONSOLE
+# RUN: llvm-ar t %t.dir/default.lib | FileCheck %s -check-prefix CHECK-NODEF-DLL
 
 CHECK-NODEF-DLL: default.dll
 CHECK-NODEF-DLL: default.dll
 CHECK-NODEF-DLL: default.dll
 CHECK-NODEF-DLL: default.dll
 
-# RUN: lld-link /nologo /machine:x64 /out:%T/exe %T/object.obj /entry:f /subsystem:CONSOLE
+# RUN: lld-link /nologo /machine:x64 /out:%t.dir/exe %t.dir/object.obj /entry:f /subsystem:CONSOLE
diff --git a/lld/test/COFF/locally-imported-arm64x.s b/lld/test/COFF/locally-imported-arm64x.s
new file mode 100644
index 0000000000000..6091af2dfa167
--- /dev/null
+++ b/lld/test/COFF/locally-imported-arm64x.s
@@ -0,0 +1,32 @@
+// REQUIRES: aarch64
+
+// RUN: llvm-mc -filetype=obj -triple=aarch64-windows %s -o %t.arm64.obj
+// RUN: llvm-mc -filetype=obj -triple=arm64ec-windows %s -o %t.arm64ec.obj
+
+// RUN: lld-link -machine:arm64x -dll -noentry %t.arm64.obj %t.arm64ec.obj -out:%t.dll 2>&1 | FileCheck --check-prefix=WARN %s
+// WARN:      lld-link: warning: {{.*}}.arm64.obj: locally defined symbol imported: func
+// WARN-NEXT: lld-link: warning: {{.*}}.arm64ec.obj: locally defined symbol imported: func
+
+// RUN: llvm-readobj --hex-dump=.test %t.dll | FileCheck --check-prefix=TEST %s
+// TEST: 0x180005000 00300000 08300000
+
+// RUN: llvm-readobj --coff-basereloc %t.dll | FileCheck --check-prefix=RELOCS %s
+// RELOCS:      Entry {
+// RELOCS-NEXT:   Type: DIR64
+// RELOCS-NEXT:   Address: 0x3000
+// RELOCS-NEXT: }
+// RELOCS-NEXT: Entry {
+// RELOCS-NEXT:   Type: DIR64
+// RELOCS-NEXT:   Address: 0x3008
+// RELOCS-NEXT: }
+
+// RUN: llvm-readobj --hex-dump=.rdata %t.dll | FileCheck --check-prefix=RDATA %s
+// RDATA: 0x180003000 00100080 01000000 00200080 01000000
+
+    .text
+    .globl func
+func:
+    ret
+
+    .section .test, "r"
+    .rva __imp_func
diff --git a/lld/test/COFF/locally-imported-warn-multiple.s b/lld/test/COFF/locally-imported-warn-multiple.s
index 247ec58b0fdb6..f1d03415eb90d 100644
--- a/lld/test/COFF/locally-imported-warn-multiple.s
+++ b/lld/test/COFF/locally-imported-warn-multiple.s
@@ -1,10 +1,11 @@
 # REQUIRES: x86
 
-# RUN: llvm-mc -filetype=obj -triple=x86_64-windows-msvc -o %T/locally-imported-def.obj %S/Inputs/locally-imported-def.s
-# RUN: llvm-mc -filetype=obj -triple=x86_64-windows-msvc -o %T/locally-imported-imp1.obj %S/Inputs/locally-imported-imp.s
-# RUN: llvm-mc -filetype=obj -triple=x86_64-windows-msvc -o %T/locally-imported-imp2.obj %S/Inputs/locally-imported-imp.s
+# RUN: mkdir -p %t.dir
+# RUN: llvm-mc -filetype=obj -triple=x86_64-windows-msvc -o %t.dir/locally-imported-def.obj %S/Inputs/locally-imported-def.s
+# RUN: llvm-mc -filetype=obj -triple=x86_64-windows-msvc -o %t.dir/locally-imported-imp1.obj %S/Inputs/locally-imported-imp.s
+# RUN: llvm-mc -filetype=obj -triple=x86_64-windows-msvc -o %t.dir/locally-imported-imp2.obj %S/Inputs/locally-imported-imp.s
 # RUN: llvm-mc -filetype=obj -triple=x86_64-windows-msvc -o %t.obj %s
-# RUN: lld-link /entry:main %T/locally-imported-def.obj %T/locally-imported-imp1.obj %T/locally-imported-imp2.obj %t.obj 2>&1 | FileCheck %s
+# RUN: lld-link /entry:main %t.dir/locally-imported-def.obj %t.dir/locally-imported-imp1.obj %t.dir/locally-imported-imp2.obj %t.obj 2>&1 | FileCheck %s
 
 # CHECK: warning: [[TESTDIR:.+]]locally-imported-imp1.obj: locally defined symbol imported: f (defined in [[TESTDIR]]locally-imported-def.obj)
 # CHECK-NEXT: warning: [[TESTDIR:.+]]locally-imported-imp2.obj: locally defined symbol imported: f (defined in [[TESTDIR]]locally-imported-def.obj)
diff --git a/lld/test/COFF/lto-chkstk.ll b/lld/test/COFF/lto-chkstk.ll
index 331acea92c3ab..8703983cd519e 100644
--- a/lld/test/COFF/lto-chkstk.ll
+++ b/lld/test/COFF/lto-chkstk.ll
@@ -1,10 +1,11 @@
 ; REQUIRES: x86
+; RUN: mkdir -p %t.dir
 ; RUN: llvm-as -o %t.obj %s
-; RUN: llvm-mc -triple=x86_64-pc-windows-msvc -filetype=obj -o %T/lto-chkstk-foo.obj %S/Inputs/lto-chkstk-foo.s
-; RUN: llvm-mc -triple=x86_64-pc-windows-msvc -filetype=obj -o %T/lto-chkstk-chkstk.obj %S/Inputs/lto-chkstk-chkstk.s
+; RUN: llvm-mc -triple=x86_64-pc-windows-msvc -filetype=obj -o %t.dir/lto-chkstk-foo.obj %S/Inputs/lto-chkstk-foo.s
+; RUN: llvm-mc -triple=x86_64-pc-windows-msvc -filetype=obj -o %t.dir/lto-chkstk-chkstk.obj %S/Inputs/lto-chkstk-chkstk.s
 ; RUN: rm -f %t.lib
-; RUN: llvm-ar cru %t.lib %T/lto-chkstk-chkstk.obj
-; RUN: lld-link /out:%t.exe /entry:main /subsystem:console %t.obj %T/lto-chkstk-foo.obj %t.lib
+; RUN: llvm-ar cru %t.lib %t.dir/lto-chkstk-chkstk.obj
+; RUN: lld-link /out:%t.exe /entry:main /subsystem:console %t.obj %t.dir/lto-chkstk-foo.obj %t.lib
 
 target datalayout = "e-m:w-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-pc-windows-msvc"
diff --git a/lld/test/COFF/lto-comdat.ll b/lld/test/COFF/lto-comdat.ll
index 52e34b1635063..76ef8259695a5 100644
--- a/lld/test/COFF/lto-comdat.ll
+++ b/lld/test/COFF/lto-comdat.ll
@@ -1,44 +1,45 @@
 ; REQUIRES: x86
-; RUN: llvm-as -o %T/comdat-main.lto.obj %s
-; RUN: llvm-as -o %T/comdat1.lto.obj %S/Inputs/lto-comdat1.ll
-; RUN: llvm-as -o %T/comdat2.lto.obj %S/Inputs/lto-comdat2.ll
-; RUN: rm -f %T/comdat.lto.lib
-; RUN: llvm-ar cru %T/comdat.lto.lib %T/comdat1.lto.obj %T/comdat2.lto.obj
+; RUN: mkdir -p %t.dir
+; RUN: llvm-as -o %t.dir/comdat-main.lto.obj %s
+; RUN: llvm-as -o %t.dir/comdat1.lto.obj %S/Inputs/lto-comdat1.ll
+; RUN: llvm-as -o %t.dir/comdat2.lto.obj %S/Inputs/lto-comdat2.ll
+; RUN: rm -f %t.dir/comdat.lto.lib
+; RUN: llvm-ar cru %t.dir/comdat.lto.lib %t.dir/comdat1.lto.obj %t.dir/comdat2.lto.obj
 
-; RUN: llc -filetype=obj -o %T/comdat-main.obj %s
-; RUN: llc -filetype=obj -o %T/comdat1.obj %S/Inputs/lto-comdat1.ll
-; RUN: llc -filetype=obj -o %T/comdat2.obj %S/Inputs/lto-comdat2.ll
-; RUN: rm -f %T/comdat.lib
-; RUN: llvm-ar cru %T/comdat.lib %T/comdat1.obj %T/comdat2.obj
+; RUN: llc -filetype=obj -o %t.dir/comdat-main.obj %s
+; RUN: llc -filetype=obj -o %t.dir/comdat1.obj %S/Inputs/lto-comdat1.ll
+; RUN: llc -filetype=obj -o %t.dir/comdat2.obj %S/Inputs/lto-comdat2.ll
+; RUN: rm -f %t.dir/comdat.lib
+; RUN: llvm-ar cru %t.dir/comdat.lib %t.dir/comdat1.obj %t.dir/comdat2.obj
 
 ; Check that, when we use an LTO main with LTO objects, we optimize away all
 ; of f1, f2, and comdat.
-; RUN: lld-link /out:%T/comdat-main.exe /entry:main /subsystem:console %T/comdat-main.lto.obj %T/comdat1.lto.obj %T/comdat2.lto.obj
-; RUN: llvm-readobj --file-headers %T/comdat-main.exe | FileCheck -check-prefix=HEADERS-11 %s
-; RUN: llvm-objdump --no-print-imm-hex -d %T/comdat-main.exe | FileCheck --check-prefix=TEXT-11 %s
-; RUN: lld-link /out:%T/comdat-main.exe /entry:main /subsystem:console %T/comdat-main.lto.obj %T/comdat.lto.lib
-; RUN: llvm-readobj --file-headers %T/comdat-main.exe | FileCheck -check-prefix=HEADERS-11 %s
-; RUN: llvm-objdump --no-print-imm-hex -d %T/comdat-main.exe | FileCheck --check-prefix=TEXT-11 %s
+; RUN: lld-link /out:%t.dir/comdat-main.exe /entry:main /subsystem:console %t.dir/comdat-main.lto.obj %t.dir/comdat1.lto.obj %t.dir/comdat2.lto.obj
+; RUN: llvm-readobj --file-headers %t.dir/comdat-main.exe | FileCheck -check-prefix=HEADERS-11 %s
+; RUN: llvm-objdump --no-print-imm-hex -d %t.dir/comdat-main.exe | FileCheck --check-prefix=TEXT-11 %s
+; RUN: lld-link /out:%t.dir/comdat-main.exe /entry:main /subsystem:console %t.dir/comdat-main.lto.obj %t.dir/comdat.lto.lib
+; RUN: llvm-readobj --file-headers %t.dir/comdat-main.exe | FileCheck -check-prefix=HEADERS-11 %s
+; RUN: llvm-objdump --no-print-imm-hex -d %t.dir/comdat-main.exe | FileCheck --check-prefix=TEXT-11 %s
 
 ; Check that, when we use a non-LTO main with LTO objects, we pick the comdat
 ; implementation in LTO, elide calls to it from inside LTO, and retain the
 ; call to comdat from main.
-; RUN: lld-link /out:%T/comdat-main.exe /entry:main /subsystem:console %T/comdat-main.obj %T/comdat1.lto.obj %T/comdat2.lto.obj
-; RUN: llvm-readobj --file-headers %T/comdat-main.exe | FileCheck -check-prefix=HEADERS-01 %s
-; RUN: llvm-objdump --no-print-imm-hex -d %T/comdat-main.exe | FileCheck --check-prefix=TEXT-01 %s
-; RUN: lld-link /out:%T/comdat-main.exe /entry:main /subsystem:console %T/comdat-main.obj %T/comdat.lto.lib
-; RUN: llvm-readobj --file-headers %T/comdat-main.exe | FileCheck -check-prefix=HEADERS-01 %s
-; RUN: llvm-objdump --no-print-imm-hex -d %T/comdat-main.exe | FileCheck --check-prefix=TEXT-01 %s
+; RUN: lld-link /out:%t.dir/comdat-main.exe /entry:main /subsystem:console %t.dir/comdat-main.obj %t.dir/comdat1.lto.obj %t.dir/comdat2.lto.obj
+; RUN: llvm-readobj --file-headers %t.dir/comdat-main.exe | FileCheck -check-prefix=HEADERS-01 %s
+; RUN: llvm-objdump --no-print-imm-hex -d %t.dir/comdat-main.exe | FileCheck --check-prefix=TEXT-01 %s
+; RUN: lld-link /out:%t.dir/comdat-main.exe /entry:main /subsystem:console %t.dir/comdat-main.obj %t.dir/comdat.lto.lib
+; RUN: llvm-readobj --file-headers %t.dir/comdat-main.exe | FileCheck -check-prefix=HEADERS-01 %s
+; RUN: llvm-objdump --no-print-imm-hex -d %t.dir/comdat-main.exe | FileCheck --check-prefix=TEXT-01 %s
 
 ; Check that, when we use an LTO main with non-LTO objects, we pick the comdat
 ; implementation in LTO, elide the call to it from inside LTO, and keep the
 ; calls to comdat from the non-LTO objects.
-; RUN: lld-link /out:%T/comdat-main.exe /entry:main /subsystem:console %T/comdat-main.lto.obj %T/comdat1.obj %T/comdat2.obj
-; RUN: llvm-readobj --file-headers %T/comdat-main.exe | FileCheck -check-prefix=HEADERS-10 %s
-; RUN: llvm-objdump --no-print-imm-hex -d %T/comdat-main.exe | FileCheck --check-prefix=TEXT-10 %s
-; RUN: lld-link /out:%T/comdat-main.exe /entry:main /subsystem:console %T/comdat-main.lto.obj %T/comdat.lib
-; RUN: llvm-readobj --file-headers %T/comdat-main.exe | FileCheck -check-prefix=HEADERS-10 %s
-; RUN: llvm-objdump --no-print-imm-hex -d %T/comdat-main.exe | FileCheck --check-prefix=TEXT-10 %s
+; RUN: lld-link /out:%t.dir/comdat-main.exe /entry:main /subsystem:console %t.dir/comdat-main.lto.obj %t.dir/comdat1.obj %t.dir/comdat2.obj
+; RUN: llvm-readobj --file-headers %t.dir/comdat-main.exe | FileCheck -check-prefix=HEADERS-10 %s
+; RUN: llvm-objdump --no-print-imm-hex -d %t.dir/comdat-main.exe | FileCheck --check-prefix=TEXT-10 %s
+; RUN: lld-link /out:%t.dir/comdat-main.exe /entry:main /subsystem:console %t.dir/comdat-main.lto.obj %t.dir/comdat.lib
+; RUN: llvm-readobj --file-headers %t.dir/comdat-main.exe | FileCheck -check-prefix=HEADERS-10 %s
+; RUN: llvm-objdump --no-print-imm-hex -d %t.dir/comdat-main.exe | FileCheck --check-prefix=TEXT-10 %s
 
 ; HEADERS-11: AddressOfEntryPoint: 0x1000
 ; TEXT-11: Disassembly of section .text:
diff --git a/lld/test/COFF/lto-lazy-reference.ll b/lld/test/COFF/lto-lazy-reference.ll
index a6f4010eddd5d..49d61cad7a6f3 100644
--- a/lld/test/COFF/lto-lazy-reference.ll
+++ b/lld/test/COFF/lto-lazy-reference.ll
@@ -1,8 +1,9 @@
 ; REQUIRES: x86
-; RUN: llc -mtriple=i686-pc-windows-msvc -filetype=obj -o %T/lto-lazy-reference-quadruple.obj %S/Inputs/lto-lazy-reference-quadruple.ll
-; RUN: llvm-as -o %T/lto-lazy-reference-dummy.bc %S/Inputs/lto-lazy-reference-dummy.ll
+; RUN: mkdir -p %t.dir
+; RUN: llc -mtriple=i686-pc-windows-msvc -filetype=obj -o %t.dir/lto-lazy-reference-quadruple.obj %S/Inputs/lto-lazy-reference-quadruple.ll
+; RUN: llvm-as -o %t.dir/lto-lazy-reference-dummy.bc %S/Inputs/lto-lazy-reference-dummy.ll
 ; RUN: rm -f %t.lib
-; RUN: llvm-ar cru %t.lib %T/lto-lazy-reference-quadruple.obj %T/lto-lazy-reference-dummy.bc
+; RUN: llvm-ar cru %t.lib %t.dir/lto-lazy-reference-quadruple.obj %t.dir/lto-lazy-reference-dummy.bc
 ; RUN: llvm-as -o %t.obj %s
 ; RUN: lld-link /out:%t.exe /entry:main /subsystem:console %t.obj %t.lib
 
diff --git a/lld/test/COFF/lto-linker-opts.ll b/lld/test/COFF/lto-linker-opts.ll
index bbe67ae41da7d..1c615287e7519 100644
--- a/lld/test/COFF/lto-linker-opts.ll
+++ b/lld/test/COFF/lto-linker-opts.ll
@@ -1,6 +1,7 @@
 ; REQUIRES: x86
-; RUN: llvm-as -o %T/lto-linker-opts.obj %s
-; RUN: env LIB=%S/Inputs lld-link /out:%T/lto-linker-opts.exe /entry:main /subsystem:console %T/lto-linker-opts.obj
+; RUN: mkdir -p %t.dir
+; RUN: llvm-as -o %t.dir/lto-linker-opts.obj %s
+; RUN: env LIB=%S/Inputs lld-link /out:%t.dir/lto-linker-opts.exe /entry:main /subsystem:console %t.dir/lto-linker-opts.obj
 
 target datalayout = "e-m:w-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-pc-windows-msvc"
diff --git a/lld/test/COFF/lto.ll b/lld/test/COFF/lto.ll
index 7ea772f616374..e711dfc1dd79d 100644
--- a/lld/test/COFF/lto.ll
+++ b/lld/test/COFF/lto.ll
@@ -1,34 +1,35 @@
 ; REQUIRES: x86
-; RUN: llvm-as -o %T/main.lto.obj %s
-; RUN: llvm-as -o %T/foo.lto.obj %S/Inputs/lto-dep.ll
-; RUN: rm -f %T/foo.lto.lib
-; RUN: llvm-ar cru %T/foo.lto.lib %T/foo.lto.obj
+; RUN: mkdir -p %t.dir
+; RUN: llvm-as -o %t.dir/main.lto.obj %s
+; RUN: llvm-as -o %t.dir/foo.lto.obj %S/Inputs/lto-dep.ll
+; RUN: rm -f %t.dir/foo.lto.lib
+; RUN: llvm-ar cru %t.dir/foo.lto.lib %t.dir/foo.lto.obj
 
-; RUN: llc -filetype=obj -o %T/main.obj %s
-; RUN: llc -filetype=obj -o %T/foo.obj %S/Inputs/lto-dep.ll
-; RUN: rm -f %T/foo.lib
-; RUN: llvm-ar cru %T/foo.lib %T/foo.obj
+; RUN: llc -filetype=obj -o %t.dir/main.obj %s
+; RUN: llc -filetype=obj -o %t.dir/foo.obj %S/Inputs/lto-dep.ll
+; RUN: rm -f %t.dir/foo.lib
+; RUN: llvm-ar cru %t.dir/foo.lib %t.dir/foo.obj
 
-; RUN: lld-link /out:%T/main.exe /entry:main /include:f2 /subsystem:console %T/main.lto.obj %T/foo.lto.obj
-; RUN: llvm-readobj --file-headers %T/main.exe | FileCheck -check-prefix=HEADERS-11 %s
-; RUN: llvm-objdump --no-print-imm-hex -d %T/main.exe | FileCheck --check-prefix=TEXT-11 %s
-; RUN: lld-link /out:%T/main.exe /entry:main /include:f2 /subsystem:console %T/main.lto.obj %T/foo.lto.lib /verbose 2>&1 | FileCheck -check-prefix=VERBOSE %s
-; RUN: llvm-readobj --file-headers %T/main.exe | FileCheck -check-prefix=HEADERS-11 %s
-; RUN: llvm-objdump --no-print-imm-hex -d %T/main.exe | FileCheck --check-prefix=TEXT-11 %s
+; RUN: lld-link /out:%t.dir/main.exe /entry:main /include:f2 /subsystem:console %t.dir/main.lto.obj %t.dir/foo.lto.obj
+; RUN: llvm-readobj --file-headers %t.dir/main.exe | FileCheck -check-prefix=HEADERS-11 %s
+; RUN: llvm-objdump --no-print-imm-hex -d %t.dir/main.exe | FileCheck --check-prefix=TEXT-11 %s
+; RUN: lld-link /out:%t.dir/main.exe /entry:main /include:f2 /subsystem:console %t.dir/main.lto.obj %t.dir/foo.lto.lib /verbose 2>&1 | FileCheck -check-prefix=VERBOSE %s
+; RUN: llvm-readobj --file-headers %t.dir/main.exe | FileCheck -check-prefix=HEADERS-11 %s
+; RUN: llvm-objdump --no-print-imm-hex -d %t.dir/main.exe | FileCheck --check-prefix=TEXT-11 %s
 
-; RUN: lld-link /out:%T/main.exe /entry:main /subsystem:console %T/main.obj %T/foo.lto.obj
-; RUN: llvm-readobj --file-headers %T/main.exe | FileCheck -check-prefix=HEADERS-01 %s
-; RUN: llvm-objdump --no-print-imm-hex -d %T/main.exe | FileCheck --check-prefix=TEXT-01 %s
-; RUN: lld-link /out:%T/main.exe /entry:main /subsystem:console %T/main.obj %T/foo.lto.lib
-; RUN: llvm-readobj --file-headers %T/main.exe | FileCheck -check-prefix=HEADERS-01 %s
-; RUN: llvm-objdump --no-print-imm-hex -d %T/main.exe | FileCheck --check-prefix=TEXT-01 %s
+; RUN: lld-link /out:%t.dir/main.exe /entry:main /subsystem:console %t.dir/main.obj %t.dir/foo.lto.obj
+; RUN: llvm-readobj --file-headers %t.dir/main.exe | FileCheck -check-prefix=HEADERS-01 %s
+; RUN: llvm-objdump --no-print-imm-hex -d %t.dir/main.exe | FileCheck --check-prefix=TEXT-01 %s
+; RUN: lld-link /out:%t.dir/main.exe /entry:main /subsystem:console %t.dir/main.obj %t.dir/foo.lto.lib
+; RUN: llvm-readobj --file-headers %t.dir/main.exe | FileCheck -check-prefix=HEADERS-01 %s
+; RUN: llvm-objdump --no-print-imm-hex -d %t.dir/main.exe | FileCheck --check-prefix=TEXT-01 %s
 
-; RUN: lld-link /out:%T/main.exe /entry:main /subsystem:console %T/main.lto.obj %T/foo.obj
-; RUN: llvm-readobj --file-headers %T/main.exe | FileCheck -check-prefix=HEADERS-10 %s
-; RUN: llvm-objdump --no-print-imm-hex -d %T/main.exe | FileCheck --check-prefix=TEXT-10 %s
-; RUN: lld-link /out:%T/main.exe /entry:main /subsystem:console %T/main.lto.obj %T/foo.lib
-; RUN: llvm-readobj --file-headers %T/main.exe | FileCheck -check-prefix=HEADERS-10 %s
-; RUN: llvm-objdump --no-print-imm-hex -d %T/main.exe | FileCheck --check-prefix=TEXT-10 %s
+; RUN: lld-link /out:%t.dir/main.exe /entry:main /subsystem:console %t.dir/main.lto.obj %t.dir/foo.obj
+; RUN: llvm-readobj --file-headers %t.dir/main.exe | FileCheck -check-prefix=HEADERS-10 %s
+; RUN: llvm-objdump --no-print-imm-hex -d %t.dir/main.exe | FileCheck --check-prefix=TEXT-10 %s
+; RUN: lld-link /out:%t.dir/main.exe /entry:main /subsystem:console %t.dir/main.lto.obj %t.dir/foo.lib
+; RUN: llvm-readobj --file-headers %t.dir/main.exe | FileCheck -check-prefix=HEADERS-10 %s
+; RUN: llvm-objdump --no-print-imm-hex -d %t.dir/main.exe | FileCheck --check-prefix=TEXT-10 %s
 
 ; VERBOSE: foo.lto.lib({{.*}}foo.lto.obj)
 
diff --git a/lld/test/COFF/map.test b/lld/test/COFF/map.test
index 15c1ec104b07e..117ab39884793 100644
--- a/lld/test/COFF/map.test
+++ b/lld/test/COFF/map.test
@@ -1,13 +1,14 @@
+# RUN: mkdir -p %t.dir
 # RUN: yaml2obj %p/Inputs/export.yaml -o %t-dll.obj
 # RUN: lld-link /out:%t.dll /dll %t-dll.obj /implib:%t-dll.lib \
 # RUN:   /export:exportfn1 /export:exportfn2
 # RUN: yaml2obj %p/Inputs/map.yaml -o %t.obj
-# RUN: lld-link /out:%t.exe /entry:main %t.obj %t-dll.lib /map:%T/foo.map  /lldmap
-# RUN: FileCheck -check-prefix=MAP -strict-whitespace %s < %T/foo.map
+# RUN: lld-link /out:%t.exe /entry:main %t.obj %t-dll.lib /map:%t.dir/foo.map  /lldmap
+# RUN: FileCheck -check-prefix=MAP -strict-whitespace %s < %t.dir/foo.map
 # RUN: FileCheck -check-prefix=LLDMAP -strict-whitespace %s < %t.map
-# RUN: lld-link /out:%t.exe /entry:main %t.obj %t-dll.lib /map /lldmap:%T/foo-lld.map
+# RUN: lld-link /out:%t.exe /entry:main %t.obj %t-dll.lib /map /lldmap:%t.dir/foo-lld.map
 # RUN: FileCheck -check-prefix=MAP -strict-whitespace %s < %t.map
-# RUN: FileCheck -check-prefix=LLDMAP -strict-whitespace %s < %T/foo-lld.map
+# RUN: FileCheck -check-prefix=LLDMAP -strict-whitespace %s < %t.dir/foo-lld.map
 # RUN: lld-link /out:%t.dll /dll %t-dll.obj /export:exportfn1 \
 # RUN:   /export:foo=exportfn2 /map /mapinfo:exports
 # RUN: FileCheck -check-prefix=MAPINFO -strict-whitespace %s < %t.map
diff --git a/lld/test/COFF/out.test b/lld/test/COFF/out.test
index 478e7cffacd2d..533c1e427b000 100644
--- a/lld/test/COFF/out.test
+++ b/lld/test/COFF/out.test
@@ -1,16 +1,16 @@
 # RUN: yaml2obj %p/Inputs/ret42.yaml -o %t.obj
 
-# RUN: mkdir -p %T/out/tmp
-# RUN: cp %t.obj %T/out/out1.obj
-# RUN: cp %t.obj %T/out/tmp/out2
-# RUN: cp %t.obj %T/out/tmp/out3.xyz
-# RUN: lld-link /lib %t.obj /out:%T/out/out4.lib
+# RUN: mkdir -p %t.dir/out/tmp
+# RUN: cp %t.obj %t.dir/out/out1.obj
+# RUN: cp %t.obj %t.dir/out/tmp/out2
+# RUN: cp %t.obj %t.dir/out/tmp/out3.xyz
+# RUN: lld-link /lib %t.obj /out:%t.dir/out/out4.lib
 
 # RUN: rm -f out1.exe out2.exe out3.exe out3.dll out4.exe
-# RUN: lld-link /entry:main %T/out/out1.obj
-# RUN: lld-link /entry:main %T/out/tmp/out2
-# RUN: lld-link /dll /entry:main %T/out/tmp/out3.xyz
-# RUN: lld-link /entry:main -wholearchive:%T/out/out4.lib
+# RUN: lld-link /entry:main %t.dir/out/out1.obj
+# RUN: lld-link /entry:main %t.dir/out/tmp/out2
+# RUN: lld-link /dll /entry:main %t.dir/out/tmp/out3.xyz
+# RUN: lld-link /entry:main -wholearchive:%t.dir/out/out4.lib
 
 # RUN: llvm-readobj out1.exe | FileCheck %s
 # RUN: llvm-readobj out2.exe | FileCheck %s
diff --git a/lld/test/COFF/precomp-link.test b/lld/test/COFF/precomp-link.test
index 53d981c4e6228..ce90603d0bb87 100644
--- a/lld/test/COFF/precomp-link.test
+++ b/lld/test/COFF/precomp-link.test
@@ -40,10 +40,11 @@ WARNING-NOT: warning
 
 Check that a PCH with wrong signature, but with right LF_PRECOMP records count, works.
 
+RUN: mkdir -p %t.dir
 RUN: sed '16,19s/Signature: *545589255/Signature: 123456789/' < %t.precomp.yaml > precomp-wrong-sig.yaml
-RUN: yaml2obj precomp-wrong-sig.yaml -o %T/precomp.obj
-RUN: env LLD_IN_TEST=1 lld-link %T/precomp.obj %S/Inputs/precomp-a.obj %S/Inputs/precomp-b.obj /nodefaultlib /entry:main /debug:noghash /pdb:%t.pdb /out:%t.exe 2>&1 | FileCheck %s -check-prefix WARNING --allow-empty
-RUN: env LLD_IN_TEST=1 lld-link %T/precomp.obj %S/Inputs/precomp-a.obj %S/Inputs/precomp-b.obj /nodefaultlib /entry:main /debug:ghash /pdb:%t.pdb /out:%t.exe 2>&1 | FileCheck %s -check-prefix WARNING --allow-empty
+RUN: yaml2obj precomp-wrong-sig.yaml -o %t.dir/precomp.obj
+RUN: env LLD_IN_TEST=1 lld-link %t.dir/precomp.obj %S/Inputs/precomp-a.obj %S/Inputs/precomp-b.obj /nodefaultlib /entry:main /debug:noghash /pdb:%t.pdb /out:%t.exe 2>&1 | FileCheck %s -check-prefix WARNING --allow-empty
+RUN: env LLD_IN_TEST=1 lld-link %t.dir/precomp.obj %S/Inputs/precomp-a.obj %S/Inputs/precomp-b.obj /nodefaultlib /entry:main /debug:ghash /pdb:%t.pdb /out:%t.exe 2>&1 | FileCheck %s -check-prefix WARNING --allow-empty
 
 Check that two PCH objs with duplicate signatures are an error.
 
diff --git a/lld/test/COFF/savetemps-colon.ll b/lld/test/COFF/savetemps-colon.ll
index 983055043bc93..bc26ec3ac0420 100644
--- a/lld/test/COFF/savetemps-colon.ll
+++ b/lld/test/COFF/savetemps-colon.ll
@@ -1,55 +1,55 @@
 ; REQUIRES: x86
-; RUN: rm -fr %T/savetemps-colon
-; RUN: mkdir %T/savetemps-colon
-; RUN: opt -thinlto-bc -o %T/savetemps-colon/savetemps.obj %s
-; RUN: opt -thinlto-bc -o %T/savetemps-colon/thin1.obj %S/Inputs/thinlto.ll
+; RUN: rm -fr %t.dir/savetemps-colon
+; RUN: mkdir -p %t.dir/savetemps-colon
+; RUN: opt -thinlto-bc -o %t.dir/savetemps-colon/savetemps.obj %s
+; RUN: opt -thinlto-bc -o %t.dir/savetemps-colon/thin1.obj %S/Inputs/thinlto.ll
 
 ;; Check preopt
-; RUN: lld-link /lldsavetemps:preopt /out:%T/savetemps-colon/savetemps.exe /entry:main \
-; RUN:     /subsystem:console %T/savetemps-colon/savetemps.obj %T/savetemps-colon/thin1.obj
-; RUN: ls %T/savetemps-colon/*.obj.*.preopt.bc | count 2
+; RUN: lld-link /lldsavetemps:preopt /out:%t.dir/savetemps-colon/savetemps.exe /entry:main \
+; RUN:     /subsystem:console %t.dir/savetemps-colon/savetemps.obj %t.dir/savetemps-colon/thin1.obj
+; RUN: ls %t.dir/savetemps-colon/*.obj.*.preopt.bc | count 2
 
 ;; Check promote
-; RUN: lld-link /lldsavetemps:promote /out:%T/savetemps-colon/savetemps.exe /entry:main \
-; RUN:     /subsystem:console %T/savetemps-colon/savetemps.obj %T/savetemps-colon/thin1.obj
-; RUN: ls %T/savetemps-colon/*.obj.*.promote.bc | count 2
+; RUN: lld-link /lldsavetemps:promote /out:%t.dir/savetemps-colon/savetemps.exe /entry:main \
+; RUN:     /subsystem:console %t.dir/savetemps-colon/savetemps.obj %t.dir/savetemps-colon/thin1.obj
+; RUN: ls %t.dir/savetemps-colon/*.obj.*.promote.bc | count 2
 
 ;; Check internalize
-; RUN: lld-link /lldsavetemps:internalize /out:%T/savetemps-colon/savetemps.exe /entry:main \
-; RUN:     /subsystem:console %T/savetemps-colon/savetemps.obj %T/savetemps-colon/thin1.obj
-; RUN: ls %T/savetemps-colon/*.obj.*.internalize.bc | count 2
+; RUN: lld-link /lldsavetemps:internalize /out:%t.dir/savetemps-colon/savetemps.exe /entry:main \
+; RUN:     /subsystem:console %t.dir/savetemps-colon/savetemps.obj %t.dir/savetemps-colon/thin1.obj
+; RUN: ls %t.dir/savetemps-colon/*.obj.*.internalize.bc | count 2
 
 ;; Check import
-; RUN: lld-link /lldsavetemps:import /out:%T/savetemps-colon/savetemps.exe /entry:main \
-; RUN:     /subsystem:console %T/savetemps-colon/savetemps.obj %T/savetemps-colon/thin1.obj
-; RUN: ls %T/savetemps-colon/*.obj.*.import.bc | count 2
+; RUN: lld-link /lldsavetemps:import /out:%t.dir/savetemps-colon/savetemps.exe /entry:main \
+; RUN:     /subsystem:console %t.dir/savetemps-colon/savetemps.obj %t.dir/savetemps-colon/thin1.obj
+; RUN: ls %t.dir/savetemps-colon/*.obj.*.import.bc | count 2
 
 ;; Check opt
 ;; Not supported on Windows due to difficulty with escaping "opt" across platforms.
 
 ;; Check precodegen
-; RUN: lld-link /lldsavetemps:precodegen /out:%T/savetemps-colon/savetemps.exe /entry:main \
-; RUN:     /subsystem:console %T/savetemps-colon/savetemps.obj %T/savetemps-colon/thin1.obj
-; RUN: ls %T/savetemps-colon/*.obj.*.precodegen.bc | count 2
+; RUN: lld-link /lldsavetemps:precodegen /out:%t.dir/savetemps-colon/savetemps.exe /entry:main \
+; RUN:     /subsystem:console %t.dir/savetemps-colon/savetemps.obj %t.dir/savetemps-colon/thin1.obj
+; RUN: ls %t.dir/savetemps-colon/*.obj.*.precodegen.bc | count 2
 
 ;; Check combinedindex
-; RUN: lld-link /lldsavetemps:combinedindex /out:%T/savetemps-colon/savetemps.exe /entry:main \
-; RUN:     /subsystem:console %T/savetemps-colon/savetemps.obj %T/savetemps-colon/thin1.obj
-; RUN: ls %T/savetemps-colon/*.exe.index.bc | count 1
+; RUN: lld-link /lldsavetemps:combinedindex /out:%t.dir/savetemps-colon/savetemps.exe /entry:main \
+; RUN:     /subsystem:console %t.dir/savetemps-colon/savetemps.obj %t.dir/savetemps-colon/thin1.obj
+; RUN: ls %t.dir/savetemps-colon/*.exe.index.bc | count 1
 
 ;; Check prelink
-; RUN: lld-link /lldsavetemps:prelink /out:%T/savetemps-colon/savetemps.exe /entry:main \
-; RUN:     /subsystem:console %T/savetemps-colon/savetemps.obj %T/savetemps-colon/thin1.obj
-; RUN: ls %T/savetemps-colon/*.exe.lto.*.obj | count 2
+; RUN: lld-link /lldsavetemps:prelink /out:%t.dir/savetemps-colon/savetemps.exe /entry:main \
+; RUN:     /subsystem:console %t.dir/savetemps-colon/savetemps.obj %t.dir/savetemps-colon/thin1.obj
+; RUN: ls %t.dir/savetemps-colon/*.exe.lto.*.obj | count 2
 
 ;; Check resolution
-; RUN: lld-link /lldsavetemps:resolution /out:%T/savetemps-colon/savetemps.exe /entry:main \
-; RUN:     /subsystem:console %T/savetemps-colon/savetemps.obj %T/savetemps-colon/thin1.obj
-; RUN: ls %T/savetemps-colon/*.resolution.txt | count 1
+; RUN: lld-link /lldsavetemps:resolution /out:%t.dir/savetemps-colon/savetemps.exe /entry:main \
+; RUN:     /subsystem:console %t.dir/savetemps-colon/savetemps.obj %t.dir/savetemps-colon/thin1.obj
+; RUN: ls %t.dir/savetemps-colon/*.resolution.txt | count 1
 
 ;; Check error message
-; RUN: not lld-link /lldsavetemps:notastage /out:%T/savetemps-colon/savetemps.exe /entry:main \
-; RUN:     /subsystem:console %T/savetemps-colon/savetemps.obj %T/savetemps-colon/thin1.obj 2>&1 \
+; RUN: not lld-link /lldsavetemps:notastage /out:%t.dir/savetemps-colon/savetemps.exe /entry:main \
+; RUN:     /subsystem:console %t.dir/savetemps-colon/savetemps.obj %t.dir/savetemps-colon/thin1.obj 2>&1 \
 ; RUN: | FileCheck %s
 ; CHECK: unknown /lldsavetemps value: notastage
 
diff --git a/lld/test/COFF/savetemps.ll b/lld/test/COFF/savetemps.ll
index 64d0566108b82..a1ea5e5d7cfcc 100644
--- a/lld/test/COFF/savetemps.ll
+++ b/lld/test/COFF/savetemps.ll
@@ -1,21 +1,21 @@
 ; REQUIRES: x86
-; RUN: rm -fr %T/savetemps
-; RUN: mkdir %T/savetemps
-; RUN: llvm-as -o %T/savetemps/savetemps.obj %s
-; RUN: lld-link /out:%T/savetemps/savetemps.exe /entry:main \
-; RUN:     /subsystem:console %T/savetemps/savetemps.obj
-; RUN: not llvm-dis -o - %T/savetemps/savetemps.exe.0.0.preopt.bc
-; RUN: not llvm-dis -o - %T/savetemps/savetemps.exe.0.2.internalize.bc
-; RUN: not llvm-dis -o - %T/savetemps/savetemps.exe.0.4.opt.bc
-; RUN: not llvm-dis -o - %T/savetemps/savetemps.exe.0.5.precodegen.bc
-; RUN: not llvm-objdump -s %T/savetemps/savetemps.exe.lto.obj
-; RUN: lld-link /lldsavetemps /out:%T/savetemps/savetemps.exe /entry:main \
-; RUN:     /subsystem:console %T/savetemps/savetemps.obj
-; RUN: llvm-dis -o - %T/savetemps/savetemps.exe.0.0.preopt.bc | FileCheck %s
-; RUN: llvm-dis -o - %T/savetemps/savetemps.exe.0.2.internalize.bc | FileCheck %s
-; RUN: llvm-dis -o - %T/savetemps/savetemps.exe.0.4.opt.bc | FileCheck %s
-; RUN: llvm-dis -o - %T/savetemps/savetemps.exe.0.5.precodegen.bc | FileCheck %s
-; RUN: llvm-objdump -s %T/savetemps/savetemps.exe.lto.obj | \
+; RUN: rm -fr %t.dir/savetemps
+; RUN: mkdir -p %t.dir/savetemps
+; RUN: llvm-as -o %t.dir/savetemps/savetemps.obj %s
+; RUN: lld-link /out:%t.dir/savetemps/savetemps.exe /entry:main \
+; RUN:     /subsystem:console %t.dir/savetemps/savetemps.obj
+; RUN: not llvm-dis -o - %t.dir/savetemps/savetemps.exe.0.0.preopt.bc
+; RUN: not llvm-dis -o - %t.dir/savetemps/savetemps.exe.0.2.internalize.bc
+; RUN: not llvm-dis -o - %t.dir/savetemps/savetemps.exe.0.4.opt.bc
+; RUN: not llvm-dis -o - %t.dir/savetemps/savetemps.exe.0.5.precodegen.bc
+; RUN: not llvm-objdump -s %t.dir/savetemps/savetemps.exe.lto.obj
+; RUN: lld-link /lldsavetemps /out:%t.dir/savetemps/savetemps.exe /entry:main \
+; RUN:     /subsystem:console %t.dir/savetemps/savetemps.obj
+; RUN: llvm-dis -o - %t.dir/savetemps/savetemps.exe.0.0.preopt.bc | FileCheck %s
+; RUN: llvm-dis -o - %t.dir/savetemps/savetemps.exe.0.2.internalize.bc | FileCheck %s
+; RUN: llvm-dis -o - %t.dir/savetemps/savetemps.exe.0.4.opt.bc | FileCheck %s
+; RUN: llvm-dis -o - %t.dir/savetemps/savetemps.exe.0.5.precodegen.bc | FileCheck %s
+; RUN: llvm-objdump -s %t.dir/savetemps/savetemps.exe.lto.obj | \
 ; RUN:     FileCheck --check-prefix=CHECK-OBJDUMP %s
 
 ; CHECK: define {{(noundef )?}}i32 @main()
diff --git a/lld/test/COFF/thinlto-archives.ll b/lld/test/COFF/thinlto-archives.ll
index 9194db8f65289..aa26d0e848a76 100644
--- a/lld/test/COFF/thinlto-archives.ll
+++ b/lld/test/COFF/thinlto-archives.ll
@@ -1,15 +1,15 @@
 ; REQUIRES: x86
-; RUN: rm -fr %T/thinlto-archives
-; RUN: mkdir %T/thinlto-archives %T/thinlto-archives/a %T/thinlto-archives/b
-; RUN: opt -thinlto-bc -o %T/thinlto-archives/main.obj %s
-; RUN: opt -thinlto-bc -o %T/thinlto-archives/a/bar.obj %S/Inputs/lto-dep.ll
-; RUN: opt -thinlto-bc -o %T/thinlto-archives/b/bar.obj %S/Inputs/bar.ll
-; RUN: llvm-ar crs %T/thinlto-archives/a.lib %T/thinlto-archives/a/bar.obj
-; RUN: llvm-ar crs %T/thinlto-archives/b.lib %T/thinlto-archives/b/bar.obj
-; RUN: lld-link -out:%T/thinlto-archives/main.exe -entry:main \
-; RUN:     -lldsavetemps -subsystem:console %T/thinlto-archives/main.obj \
-; RUN:     %T/thinlto-archives/a.lib %T/thinlto-archives/b.lib
-; RUN: FileCheck %s < %T/thinlto-archives/main.exe.resolution.txt
+; RUN: rm -fr %t.dir/thinlto-archives
+; RUN: mkdir -p %t.dir/thinlto-archives %t.dir/thinlto-archives/a %t.dir/thinlto-archives/b
+; RUN: opt -thinlto-bc -o %t.dir/thinlto-archives/main.obj %s
+; RUN: opt -thinlto-bc -o %t.dir/thinlto-archives/a/bar.obj %S/Inputs/lto-dep.ll
+; RUN: opt -thinlto-bc -o %t.dir/thinlto-archives/b/bar.obj %S/Inputs/bar.ll
+; RUN: llvm-ar crs %t.dir/thinlto-archives/a.lib %t.dir/thinlto-archives/a/bar.obj
+; RUN: llvm-ar crs %t.dir/thinlto-archives/b.lib %t.dir/thinlto-archives/b/bar.obj
+; RUN: lld-link -out:%t.dir/thinlto-archives/main.exe -entry:main \
+; RUN:     -lldsavetemps -subsystem:console %t.dir/thinlto-archives/main.obj \
+; RUN:     %t.dir/thinlto-archives/a.lib %t.dir/thinlto-archives/b.lib
+; RUN: FileCheck %s < %t.dir/thinlto-archives/main.exe.resolution.txt
 
 ; CHECK: {{/thinlto-archives/main.obj$}}
 ; CHECK: {{^-r=.*/thinlto-archives/main.obj,main,px$}}
diff --git a/lld/test/COFF/thinlto-mangled.ll b/lld/test/COFF/thinlto-mangled.ll
index 2554328d05dbf..05be2fc0b7b9c 100644
--- a/lld/test/COFF/thinlto-mangled.ll
+++ b/lld/test/COFF/thinlto-mangled.ll
@@ -1,7 +1,8 @@
 ; REQUIRES: x86
+; RUN: mkdir -p %t.dir
 ; RUN: opt -thinlto-bc %s -o %t.obj
-; RUN: opt -thinlto-bc %S/Inputs/thinlto-mangled-qux.ll -o %T/thinlto-mangled-qux.obj
-; RUN: lld-link -out:%t.exe -entry:main %t.obj %T/thinlto-mangled-qux.obj
+; RUN: opt -thinlto-bc %S/Inputs/thinlto-mangled-qux.ll -o %t.dir/thinlto-mangled-qux.obj
+; RUN: lld-link -out:%t.exe -entry:main %t.obj %t.dir/thinlto-mangled-qux.obj
 
 target datalayout = "e-m:w-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-pc-windows-msvc19.0.24215"
diff --git a/lld/test/COFF/thinlto-whole-archives.ll b/lld/test/COFF/thinlto-whole-archives.ll
index 4ee95749d18d9..0a9cbf1d3b3df 100644
--- a/lld/test/COFF/thinlto-whole-archives.ll
+++ b/lld/test/COFF/thinlto-whole-archives.ll
@@ -1,14 +1,14 @@
 ; REQUIRES: x86
-; RUN: rm -fr %T/thinlto-whole-archives
-; RUN: mkdir %T/thinlto-whole-archives %T/thinlto-whole-archives/a %T/thinlto-whole-archives/b
-; RUN: opt -thinlto-bc -o %T/thinlto-whole-archives/main.obj %s
-; RUN: opt -thinlto-bc -o %T/thinlto-whole-archives/a/bar.obj %S/Inputs/lto-dep.ll
-; RUN: opt -thinlto-bc -o %T/thinlto-whole-archives/b/bar.obj %S/Inputs/bar.ll
-; RUN: llvm-ar crs %T/thinlto-whole-archives/a.lib %T/thinlto-whole-archives/a/bar.obj %T/thinlto-whole-archives/b/bar.obj
-; RUN: lld-link -out:%T/thinlto-whole-archives/main.exe -entry:main \
-; RUN:     -wholearchive -lldsavetemps -subsystem:console %T/thinlto-whole-archives/main.obj \
-; RUN:     %T/thinlto-whole-archives/a.lib
-; RUN: FileCheck %s < %T/thinlto-whole-archives/main.exe.resolution.txt
+; RUN: rm -fr %t.dir/thinlto-whole-archives
+; RUN: mkdir -p %t.dir/thinlto-whole-archives %t.dir/thinlto-whole-archives/a %t.dir/thinlto-whole-archives/b
+; RUN: opt -thinlto-bc -o %t.dir/thinlto-whole-archives/main.obj %s
+; RUN: opt -thinlto-bc -o %t.dir/thinlto-whole-archives/a/bar.obj %S/Inputs/lto-dep.ll
+; RUN: opt -thinlto-bc -o %t.dir/thinlto-whole-archives/b/bar.obj %S/Inputs/bar.ll
+; RUN: llvm-ar crs %t.dir/thinlto-whole-archives/a.lib %t.dir/thinlto-whole-archives/a/bar.obj %t.dir/thinlto-whole-archives/b/bar.obj
+; RUN: lld-link -out:%t.dir/thinlto-whole-archives/main.exe -entry:main \
+; RUN:     -wholearchive -lldsavetemps -subsystem:console %t.dir/thinlto-whole-archives/main.obj \
+; RUN:     %t.dir/thinlto-whole-archives/a.lib
+; RUN: FileCheck %s < %t.dir/thinlto-whole-archives/main.exe.resolution.txt
 
 ; CHECK: {{[/\\]thinlto-whole-archives[/\\]main.obj$}}
 ; CHECK: {{^-r=.*[/\\]thinlto-whole-archives[/\\]main.obj,main,px$}}
diff --git a/lld/test/COFF/thinlto.ll b/lld/test/COFF/thinlto.ll
index bfd819dd6757f..b711572d7fda7 100644
--- a/lld/test/COFF/thinlto.ll
+++ b/lld/test/COFF/thinlto.ll
@@ -1,25 +1,25 @@
 ; REQUIRES: x86
-; RUN: rm -fr %T/thinlto
-; RUN: mkdir %T/thinlto
-; RUN: opt -thinlto-bc -o %T/thinlto/main.obj %s
-; RUN: opt -thinlto-bc -o %T/thinlto/foo.obj %S/Inputs/lto-dep.ll
-; RUN: lld-link /lldsavetemps /out:%T/thinlto/main.exe /entry:main /subsystem:console %T/thinlto/main.obj %T/thinlto/foo.obj
-; RUN: llvm-nm %T/thinlto/main.exe.lto.foo.obj | FileCheck %s
+; RUN: rm -fr %t.dir/thinlto
+; RUN: mkdir -p %t.dir/thinlto
+; RUN: opt -thinlto-bc -o %t.dir/thinlto/main.obj %s
+; RUN: opt -thinlto-bc -o %t.dir/thinlto/foo.obj %S/Inputs/lto-dep.ll
+; RUN: lld-link /lldsavetemps /out:%t.dir/thinlto/main.exe /entry:main /subsystem:console %t.dir/thinlto/main.obj %t.dir/thinlto/foo.obj
+; RUN: llvm-nm %t.dir/thinlto/main.exe.lto.foo.obj | FileCheck %s
 
 ; Test various possible options for /opt:lldltojobs
-; RUN: lld-link /lldsavetemps /out:%T/thinlto/main.exe /entry:main /subsystem:console %T/thinlto/main.obj %T/thinlto/foo.obj /opt:lldltojobs=1
-; RUN: llvm-nm %T/thinlto/main.exe.lto.foo.obj | FileCheck %s
-; RUN: lld-link /lldsavetemps /out:%T/thinlto/main.exe /entry:main /subsystem:console %T/thinlto/main.obj %T/thinlto/foo.obj /opt:lldltojobs=all
-; RUN: llvm-nm %T/thinlto/main.exe.lto.foo.obj | FileCheck %s
-; RUN: lld-link /lldsavetemps /out:%T/thinlto/main.exe /entry:main /subsystem:console %T/thinlto/main.obj %T/thinlto/foo.obj /opt:lldltojobs=100
-; RUN: llvm-nm %T/thinlto/main.exe.lto.foo.obj | FileCheck %s
-; RUN: not lld-link /lldsavetemps /out:%T/thinlto/main.exe /entry:main /subsystem:console %T/thinlto/main.obj %T/thinlto/foo.obj /opt:lldltojobs=foo 2>&1 | FileCheck %s --check-prefix=BAD-JOBS
+; RUN: lld-link /lldsavetemps /out:%t.dir/thinlto/main.exe /entry:main /subsystem:console %t.dir/thinlto/main.obj %t.dir/thinlto/foo.obj /opt:lldltojobs=1
+; RUN: llvm-nm %t.dir/thinlto/main.exe.lto.foo.obj | FileCheck %s
+; RUN: lld-link /lldsavetemps /out:%t.dir/thinlto/main.exe /entry:main /subsystem:console %t.dir/thinlto/main.obj %t.dir/thinlto/foo.obj /opt:lldltojobs=all
+; RUN: llvm-nm %t.dir/thinlto/main.exe.lto.foo.obj | FileCheck %s
+; RUN: lld-link /lldsavetemps /out:%t.dir/thinlto/main.exe /entry:main /subsystem:console %t.dir/thinlto/main.obj %t.dir/thinlto/foo.obj /opt:lldltojobs=100
+; RUN: llvm-nm %t.dir/thinlto/main.exe.lto.foo.obj | FileCheck %s
+; RUN: not lld-link /lldsavetemps /out:%t.dir/thinlto/main.exe /entry:main /subsystem:console %t.dir/thinlto/main.obj %t.dir/thinlto/foo.obj /opt:lldltojobs=foo 2>&1 | FileCheck %s --check-prefix=BAD-JOBS
 ; BAD-JOBS: error: /opt:lldltojobs: invalid job count: foo
 
 ; This command will store full path to foo.obj in the archive %t.lib
 ; Check that /lldsavetemps is still usable in such case.
-; RUN: lld-link /lib %T/thinlto/foo.obj /out:%t.lib
-; RUN: lld-link /lldsavetemps /out:%t.exe /entry:main /subsystem:console %T/thinlto/main.obj %t.lib
+; RUN: lld-link /lib %t.dir/thinlto/foo.obj /out:%t.lib
+; RUN: lld-link /lldsavetemps /out:%t.exe /entry:main /subsystem:console %t.dir/thinlto/main.obj %t.lib
 
 ; CHECK-NOT: U foo
 
diff --git a/lld/test/ELF/aarch64-build-attributes.s b/lld/test/ELF/aarch64-build-attributes.s
new file mode 100644
index 0000000000000..24e15f94e3d4a
--- /dev/null
+++ b/lld/test/ELF/aarch64-build-attributes.s
@@ -0,0 +1,26 @@
+// REQUIRES: aarch64
+// RUN: llvm-mc -triple=aarch64 %s -filetype=obj -o %t.o
+// RUN: ld.lld %t.o --shared -o %t.so
+// RUN: llvm-readelf --sections %t.so | FileCheck %s
+// RUN: ld.lld %t.o -o %t
+// RUN: llvm-readelf --sections %t | FileCheck %s
+// RUN: ld.lld -r %t.o -o %t2.o
+// RUN: llvm-readelf --sections %t2.o | FileCheck %s
+
+/// File has a Build attributes section. This should not appear in
+/// ET_EXEC or ET_SHARED files as there is no requirement for it to
+/// do so. FIXME, the ld -r (relocatable link) should output a single
+/// merged build attributes section. When full support is added in
+/// ld.lld this test should be updated.
+
+// CHECK-NOT: .ARM.attributes
+
+.aeabi_subsection aeabi_feature_and_bits, optional, uleb128
+.aeabi_attribute Tag_Feature_BTI, 1
+.aeabi_attribute Tag_Feature_PAC, 1
+.aeabi_attribute Tag_Feature_GCS, 1
+
+.global _start
+.type _start, %function
+_start:
+ret
diff --git a/lld/test/ELF/archive-thin-missing-member.s b/lld/test/ELF/archive-thin-missing-member.s
index 1d770451fcbe8..45fcd0e3caa23 100644
--- a/lld/test/ELF/archive-thin-missing-member.s
+++ b/lld/test/ELF/archive-thin-missing-member.s
@@ -1,4 +1,6 @@
 # REQUIRES: x86
+# This test intentionally checks for fatal errors, and fatal errors aren't supported for testing when main is run twice.
+# XFAIL: main-run-twice
 
 # RUN: rm -f %t-no-syms.a
 # RUN: rm -f %t-syms.a
diff --git a/lld/test/ELF/arm-thumb-thunk-v6m-xo.s b/lld/test/ELF/arm-thumb-thunk-v6m-xo.s
index f1b6c0c194b38..caee025ef52b2 100644
--- a/lld/test/ELF/arm-thumb-thunk-v6m-xo.s
+++ b/lld/test/ELF/arm-thumb-thunk-v6m-xo.s
@@ -1,4 +1,6 @@
 // REQUIRES: arm
+// This test intentionally checks for fatal errors, and fatal errors aren't supported for testing when main is run twice.
+// XFAIL: main-run-twice
 // RUN: rm -rf %t && split-file %s %t
 // RUN: llvm-mc -arm-add-build-attributes -filetype=obj -triple=armv6m-none-eabi %t/a.s -o %t/a.o
 // RUN: ld.lld --no-rosegment --script %t/a.t %t/a.o -o %t/a
diff --git a/lld/test/ELF/arm-thunk-section-too-large.s b/lld/test/ELF/arm-thunk-section-too-large.s
index c6c058085968d..a2e8ff555eadc 100644
--- a/lld/test/ELF/arm-thunk-section-too-large.s
+++ b/lld/test/ELF/arm-thunk-section-too-large.s
@@ -1,4 +1,6 @@
 // REQUIRES: arm
+// This test intentionally checks for fatal errors, and fatal errors aren't supported for testing when main is run twice.
+// XFAIL: main-run-twice
 // RUN: llvm-mc %s -triple=armv7a-linux-gnueabihf -arm-add-build-attributes -filetype=obj -o %t.o
 // RUN: not ld.lld %t.o -o /dev/null 2>&1 | FileCheck %s
 
diff --git a/lld/test/ELF/arm-thunk-toolargesection.s b/lld/test/ELF/arm-thunk-toolargesection.s
index f91c4a4959315..88f8e6ba4cad3 100644
--- a/lld/test/ELF/arm-thunk-toolargesection.s
+++ b/lld/test/ELF/arm-thunk-toolargesection.s
@@ -1,4 +1,6 @@
 // REQUIRES: arm
+// This test intentionally checks for fatal errors, and fatal errors aren't supported for testing when main is run twice.
+// XFAIL: main-run-twice
 // RUN: llvm-mc -filetype=obj -triple=thumbv7a-none-linux-gnueabi %s -o %t
 // RUN: not ld.lld %t -o /dev/null 2>&1 | FileCheck %s
 
diff --git a/lld/test/ELF/arm-v5-reloc-error.s b/lld/test/ELF/arm-v5-reloc-error.s
index bd4b9ad68d10a..85e66f1d3850f 100644
--- a/lld/test/ELF/arm-v5-reloc-error.s
+++ b/lld/test/ELF/arm-v5-reloc-error.s
@@ -1,4 +1,6 @@
 // REQUIRES: arm
+// This test intentionally checks for fatal errors, and fatal errors aren't supported for testing when main is run twice.
+// XFAIL: main-run-twice
 // RUN: llvm-mc -filetype=obj -triple=armv7a-linux-gnueabi %s -o %t
 // RUN: echo "SECTIONS { \
 // RUN:       . = SIZEOF_HEADERS; \
diff --git a/lld/test/ELF/bad-archive.s b/lld/test/ELF/bad-archive.s
index ba6d674c2fa65..92aa1646f6c57 100644
--- a/lld/test/ELF/bad-archive.s
+++ b/lld/test/ELF/bad-archive.s
@@ -1,4 +1,6 @@
 // REQUIRES: x86
+// This test intentionally checks for fatal errors, and fatal errors aren't supported for testing when main is run twice.
+// XFAIL: main-run-twice
 
 // Check bad archive error reporting with --whole-archive
 // and without it.
diff --git a/lld/test/ELF/basic.s b/lld/test/ELF/basic.s
index 587fd1641500a..b01f51eb4a2c7 100644
--- a/lld/test/ELF/basic.s
+++ b/lld/test/ELF/basic.s
@@ -220,7 +220,7 @@ _start:
 
 ## Test erroring on a recursive response file, but only once.
 # RUN: echo @%t.responsefile > %t.responsefile
-# RUN: not ld.lld %t @%t.responsefile 2>&1 | FileCheck %s --check-prefix=RECRSP
+# RUN: env LLD_IN_TEST=1 not ld.lld %t @%t.responsefile 2>&1 | FileCheck %s --check-prefix=RECRSP
 # RECRSP: recursive expansion of: '{{.*}}.responsefile'
 # RECRSP-NOT: recursive expansion of
 
diff --git a/lld/test/ELF/bp-section-orderer-stress.s b/lld/test/ELF/bp-section-orderer-stress.s
new file mode 100644
index 0000000000000..da9670933949f
--- /dev/null
+++ b/lld/test/ELF/bp-section-orderer-stress.s
@@ -0,0 +1,104 @@
+# REQUIRES: aarch64
+
+## Generate a large test case and check that the output is deterministic.
+
+# RUN: %python %s %t.s %t.proftext
+
+# RUN: llvm-mc -filetype=obj -triple=aarch64 %t.s -o %t.o
+# RUN: llvm-profdata merge %t.proftext -o %t.profdata
+
+# RUN: ld.lld --icf=all -o %t1.o %t.o --irpgo-profile=%t.profdata --bp-startup-sort=function --bp-compression-sort-startup-functions --bp-compression-sort=both
+# RUN: ld.lld --icf=all -o %t2.o %t.o --irpgo-profile=%t.profdata --bp-startup-sort=function --bp-compression-sort-startup-functions --bp-compression-sort=both
+# RUN: cmp %t1.o %t2.o
+
+import random
+import sys
+
+assembly_filepath = sys.argv[1]
+proftext_filepath = sys.argv[2]
+
+random.seed(1234)
+num_functions = 1000
+num_data = 100
+num_traces = 10
+
+function_names = [f"f{n}" for n in range(num_functions)]
+data_names = [f"d{n}" for n in range(num_data)]
+profiled_functions = function_names[: int(num_functions / 2)]
+
+function_contents = [
+    f"""
+{name}:
+  add w0, w0, #{i % 4096}
+  add w1, w1, #{i % 10}
+  add w2, w0, #{i % 20}
+  adrp x3, {name}
+  ret
+"""
+    for i, name in enumerate(function_names)
+]
+
+data_contents = [
+      f"""
+{name}:
+  .ascii "s{i % 2}-{i % 3}-{i % 5}"
+  .xword {name}
+"""
+    for i, name in enumerate(data_names)
+]
+
+trace_contents = [
+    f"""
+# Weight
+1
+{", ".join(random.sample(profiled_functions, len(profiled_functions)))}
+"""
+    for i in range(num_traces)
+]
+
+profile_contents = [
+    f"""
+{name}
+# Func Hash:
+{i}
+# Num Counters:
+1
+# Counter Values:
+1
+"""
+    for i, name in enumerate(profiled_functions)
+]
+
+with open(assembly_filepath, "w") as f:
+    f.write(
+        f"""
+.text
+.globl _start
+
+_start:
+  ret
+
+{"".join(function_contents)}
+
+.data
+{"".join(data_contents)}
+
+"""
+    )
+
+with open(proftext_filepath, "w") as f:
+    f.write(
+        f"""
+:ir
+:temporal_prof_traces
+
+# Num Traces
+{num_traces}
+# Trace Stream Size:
+{num_traces}
+
+{"".join(trace_contents)}
+
+{"".join(profile_contents)}
+"""
+    )
diff --git a/lld/test/ELF/bp-section-orderer.s b/lld/test/ELF/bp-section-orderer.s
new file mode 100644
index 0000000000000..2e18107c02ca3
--- /dev/null
+++ b/lld/test/ELF/bp-section-orderer.s
@@ -0,0 +1,335 @@
+# REQUIRES: aarch64
+# RUN: rm -rf %t && split-file %s %t && cd %t
+
+## Check for incompatible cases
+# RUN: not ld.lld %t --irpgo-profile=/dev/null --bp-startup-sort=function --call-graph-ordering-file=/dev/null 2>&1 | FileCheck %s --check-prefix=BP-STARTUP-CALLGRAPH-ERR
+# RUN: not ld.lld --bp-compression-sort=function --call-graph-ordering-file /dev/null 2>&1 | FileCheck %s --check-prefix=BP-COMPRESSION-CALLGRAPH-ERR
+# RUN: not ld.lld --bp-startup-sort=function 2>&1 | FileCheck %s --check-prefix=BP-STARTUP-ERR
+# RUN: not ld.lld --bp-compression-sort-startup-functions 2>&1 | FileCheck %s --check-prefix=BP-STARTUP-COMPRESSION-ERR
+# RUN: not ld.lld --bp-startup-sort=invalid --bp-compression-sort=invalid 2>&1 | FileCheck %s --check-prefix=BP-INVALID
+
+# BP-STARTUP-CALLGRAPH-ERR: error: --bp-startup-sort=function is incompatible with --call-graph-ordering-file
+# BP-COMPRESSION-CALLGRAPH-ERR: error: --bp-compression-sort is incompatible with --call-graph-ordering-file
+# BP-STARTUP-ERR: error: --bp-startup-sort=function must be used with --irpgo-profile
+# BP-STARTUP-COMPRESSION-ERR: error: --bp-compression-sort-startup-functions must be used with --irpgo-profile
+
+# BP-INVALID: error: --bp-compression-sort=: expected [none|function|data|both]
+# BP-INVALID: error: --bp-startup-sort=: expected [none|function]
+
+# RUN: llvm-mc -filetype=obj -triple=aarch64 a.s -o a.o
+# RUN: llvm-profdata merge a.proftext -o a.profdata
+# RUN: ld.lld a.o --irpgo-profile=a.profdata --bp-startup-sort=function --verbose-bp-section-orderer --icf=all 2>&1 | FileCheck %s --check-prefix=STARTUP-FUNC-ORDER
+
+# STARTUP-FUNC-ORDER: Ordered 3 sections using balanced partitioning
+# STARTUP-FUNC-ORDER: Total area under the page fault curve: 3.
+
+# RUN: ld.lld -o out.s a.o --irpgo-profile=a.profdata --bp-startup-sort=function
+# RUN: llvm-nm -jn out.s | tr '\n' , | FileCheck %s --check-prefix=STARTUP
+# STARTUP: s5,s4,s3,s2,s1,A,B,C,F,E,D,_start,d4,d3,d2,d1,{{$}}
+
+# RUN: ld.lld -o out.os a.o --irpgo-profile=a.profdata --bp-startup-sort=function --symbol-ordering-file a.txt
+# RUN: llvm-nm -jn out.os | tr '\n' , | FileCheck %s --check-prefix=ORDER-STARTUP
+# ORDER-STARTUP: s2,s1,s5,s4,s3,A,F,E,D,B,C,_start,d3,d2,d4,d1,{{$}}
+
+# RUN: ld.lld -o out.cf a.o --verbose-bp-section-orderer --bp-compression-sort=function 2>&1 | FileCheck %s --check-prefix=BP-COMPRESSION-FUNC
+# RUN: llvm-nm -jn out.cf | tr '\n' , | FileCheck %s --check-prefix=CFUNC
+# CFUNC: s5,s4,s3,s2,s1,F,C,E,D,B,A,_start,d4,d3,d2,d1,{{$}}
+
+# RUN: ld.lld -o out.cd a.o --verbose-bp-section-orderer --bp-compression-sort=data 2>&1 | FileCheck %s --check-prefix=BP-COMPRESSION-DATA
+# RUN: llvm-nm -jn out.cd | tr '\n' , | FileCheck %s --check-prefix=CDATA
+# CDATA: s5,s3,s4,s2,s1,F,C,E,D,B,A,_start,d4,d1,d3,d2,{{$}}
+
+# RUN: ld.lld -o out.cb a.o --verbose-bp-section-orderer --bp-compression-sort=both 2>&1 | FileCheck %s --check-prefix=BP-COMPRESSION-BOTH
+# RUN: llvm-nm -jn out.cb | tr '\n' , | FileCheck %s --check-prefix=CDATA
+
+# RUN: ld.lld -o out.cbs a.o --verbose-bp-section-orderer --bp-compression-sort=both --irpgo-profile=a.profdata --bp-startup-sort=function 2>&1 | FileCheck %s --check-prefix=BP-COMPRESSION-BOTH
+# RUN: llvm-nm -jn out.cbs | tr '\n' , | FileCheck %s --check-prefix=CBOTH-STARTUP
+# CBOTH-STARTUP: s5,s3,s4,s2,s1,A,B,C,F,E,D,_start,d4,d1,d3,d2,{{$}}
+
+# BP-COMPRESSION-FUNC: Ordered 7 sections using balanced partitioning
+# BP-COMPRESSION-DATA: Ordered 9 sections using balanced partitioning
+# BP-COMPRESSION-BOTH: Ordered 16 sections using balanced partitioning
+
+#--- a.proftext
+:ir
+:temporal_prof_traces
+# Num Traces
+1
+# Trace Stream Size:
+1
+# Weight
+1
+A, B, C
+
+A
+# Func Hash:
+1111
+# Num Counters:
+1
+# Counter Values:
+1
+
+B
+# Func Hash:
+2222
+# Num Counters:
+1
+# Counter Values:
+1
+
+C
+# Func Hash:
+3333
+# Num Counters:
+1
+# Counter Values:
+1
+
+D
+# Func Hash:
+4444
+# Num Counters:
+1
+# Counter Values:
+1
+
+#--- a.txt
+A
+F
+E
+D
+s2
+s1
+d3
+d2
+
+#--- a.c
+const char s5[] = "engineering";
+const char s4[] = "computer program";
+const char s3[] = "hardware engineer";
+const char s2[] = "computer software";
+const char s1[] = "hello world program";
+int d4[] = {1,2,3,4,5,6};
+int d3[] = {5,6,7,8};
+int d2[] = {7,8,9,10};
+int d1[] = {3,4,5,6};
+
+int C(int a);
+int B(int a);
+void A();
+
+int F(int a) { return C(a + 3); }
+int E(int a) { return C(a + 2); }
+int D(int a) { return B(a + 2); }
+int C(int a) { A(); return a + 2; }
+int B(int a) { A(); return a + 1; }
+void A() {}
+
+int _start() { return 0; }
+
+#--- gen
+clang --target=aarch64-linux-gnu -O0 -ffunction-sections -fdata-sections -fno-asynchronous-unwind-tables -S a.c -o -
+;--- a.s
+	.file	"a.c"
+	.section	.text.F,"ax",@progbits
+	.globl	F                               // -- Begin function F
+	.p2align	2
+	.type	F,@function
+F:                                      // @F
+// %bb.0:                               // %entry
+	sub	sp, sp, #32
+	stp	x29, x30, [sp, #16]             // 16-byte Folded Spill
+	add	x29, sp, #16
+	stur	w0, [x29, #-4]
+	ldur	w8, [x29, #-4]
+	add	w0, w8, #3
+	bl	C
+	ldp	x29, x30, [sp, #16]             // 16-byte Folded Reload
+	add	sp, sp, #32
+	ret
+.Lfunc_end0:
+	.size	F, .Lfunc_end0-F
+                                        // -- End function
+	.section	.text.C,"ax",@progbits
+	.globl	C                               // -- Begin function C
+	.p2align	2
+	.type	C,@function
+C:                                      // @C
+// %bb.0:                               // %entry
+	sub	sp, sp, #32
+	stp	x29, x30, [sp, #16]             // 16-byte Folded Spill
+	add	x29, sp, #16
+	stur	w0, [x29, #-4]
+	bl	A
+	ldur	w8, [x29, #-4]
+	add	w0, w8, #2
+	ldp	x29, x30, [sp, #16]             // 16-byte Folded Reload
+	add	sp, sp, #32
+	ret
+.Lfunc_end1:
+	.size	C, .Lfunc_end1-C
+                                        // -- End function
+	.section	.text.E,"ax",@progbits
+	.globl	E                               // -- Begin function E
+	.p2align	2
+	.type	E,@function
+E:                                      // @E
+// %bb.0:                               // %entry
+	sub	sp, sp, #32
+	stp	x29, x30, [sp, #16]             // 16-byte Folded Spill
+	add	x29, sp, #16
+	stur	w0, [x29, #-4]
+	ldur	w8, [x29, #-4]
+	add	w0, w8, #2
+	bl	C
+	ldp	x29, x30, [sp, #16]             // 16-byte Folded Reload
+	add	sp, sp, #32
+	ret
+.Lfunc_end2:
+	.size	E, .Lfunc_end2-E
+                                        // -- End function
+	.section	.text.D,"ax",@progbits
+	.globl	D                               // -- Begin function D
+	.p2align	2
+	.type	D,@function
+D:                                      // @D
+// %bb.0:                               // %entry
+	sub	sp, sp, #32
+	stp	x29, x30, [sp, #16]             // 16-byte Folded Spill
+	add	x29, sp, #16
+	stur	w0, [x29, #-4]
+	ldur	w8, [x29, #-4]
+	add	w0, w8, #2
+	bl	B
+	ldp	x29, x30, [sp, #16]             // 16-byte Folded Reload
+	add	sp, sp, #32
+	ret
+.Lfunc_end3:
+	.size	D, .Lfunc_end3-D
+                                        // -- End function
+	.section	.text.B,"ax",@progbits
+	.globl	B                               // -- Begin function B
+	.p2align	2
+	.type	B,@function
+B:                                      // @B
+// %bb.0:                               // %entry
+	sub	sp, sp, #32
+	stp	x29, x30, [sp, #16]             // 16-byte Folded Spill
+	add	x29, sp, #16
+	stur	w0, [x29, #-4]
+	bl	A
+	ldur	w8, [x29, #-4]
+	add	w0, w8, #1
+	ldp	x29, x30, [sp, #16]             // 16-byte Folded Reload
+	add	sp, sp, #32
+	ret
+.Lfunc_end4:
+	.size	B, .Lfunc_end4-B
+                                        // -- End function
+	.section	.text.A,"ax",@progbits
+	.globl	A                               // -- Begin function A
+	.p2align	2
+	.type	A,@function
+A:                                      // @A
+// %bb.0:                               // %entry
+	ret
+.Lfunc_end5:
+	.size	A, .Lfunc_end5-A
+                                        // -- End function
+	.section	.text._start,"ax",@progbits
+	.globl	_start                          // -- Begin function _start
+	.p2align	2
+	.type	_start,@function
+_start:                                 // @_start
+// %bb.0:                               // %entry
+	mov	w0, wzr
+	ret
+.Lfunc_end6:
+	.size	_start, .Lfunc_end6-_start
+                                        // -- End function
+	.type	s5,@object                      // @s5
+	.section	.rodata.s5,"a",@progbits
+	.globl	s5
+s5:
+	.asciz	"engineering"
+	.size	s5, 12
+
+	.type	s4,@object                      // @s4
+	.section	.rodata.s4,"a",@progbits
+	.globl	s4
+s4:
+	.asciz	"computer program"
+	.size	s4, 17
+
+	.type	s3,@object                      // @s3
+	.section	.rodata.s3,"a",@progbits
+	.globl	s3
+s3:
+	.asciz	"hardware engineer"
+	.size	s3, 18
+
+	.type	s2,@object                      // @s2
+	.section	.rodata.s2,"a",@progbits
+	.globl	s2
+s2:
+	.asciz	"computer software"
+	.size	s2, 18
+
+	.type	s1,@object                      // @s1
+	.section	.rodata.s1,"a",@progbits
+	.globl	s1
+s1:
+	.asciz	"hello world program"
+	.size	s1, 20
+
+	.type	d4,@object                      // @d4
+	.section	.data.d4,"aw",@progbits
+	.globl	d4
+	.p2align	2, 0x0
+d4:
+	.word	1                               // 0x1
+	.word	2                               // 0x2
+	.word	3                               // 0x3
+	.word	4                               // 0x4
+	.word	5                               // 0x5
+	.word	6                               // 0x6
+	.size	d4, 24
+
+	.type	d3,@object                      // @d3
+	.section	.data.d3,"aw",@progbits
+	.globl	d3
+	.p2align	2, 0x0
+d3:
+	.word	5                               // 0x5
+	.word	6                               // 0x6
+	.word	7                               // 0x7
+	.word	8                               // 0x8
+	.size	d3, 16
+
+	.type	d2,@object                      // @d2
+	.section	.data.d2,"aw",@progbits
+	.globl	d2
+	.p2align	2, 0x0
+d2:
+	.word	7                               // 0x7
+	.word	8                               // 0x8
+	.word	9                               // 0x9
+	.word	10                              // 0xa
+	.size	d2, 16
+
+	.type	d1,@object                      // @d1
+	.section	.data.d1,"aw",@progbits
+	.globl	d1
+	.p2align	2, 0x0
+d1:
+	.word	3                               // 0x3
+	.word	4                               // 0x4
+	.word	5                               // 0x5
+	.word	6                               // 0x6
+	.size	d1, 16
+
+	.section	".note.GNU-stack","",@progbits
+	.addrsig
+	.addrsig_sym C
+	.addrsig_sym B
+	.addrsig_sym A
diff --git a/lld/test/ELF/compressed-input-err.s b/lld/test/ELF/compressed-input-err.s
index 83b1f62d7e495..7251585ed5d70 100644
--- a/lld/test/ELF/compressed-input-err.s
+++ b/lld/test/ELF/compressed-input-err.s
@@ -9,6 +9,7 @@
 
 # RUN: yaml2obj --docnum=3 %s -o %t3.o
 # RUN: not ld.lld %t3.o -o /dev/null -shared 2>&1 | FileCheck %s
+# RUN: ld.lld %t3.o -o /dev/null -shared --noinhibit-exec
 
 ## Check we are able to report zlib decompress errors.
 # CHECK: error: {{.*}}.o:(.debug_info): decompress failed: zlib error: Z_DATA_ERROR
diff --git a/lld/test/ELF/executable-undefined-ignoreall.s b/lld/test/ELF/executable-undefined-ignoreall.s
index 073b22bd84543..1f83b1b61830a 100644
--- a/lld/test/ELF/executable-undefined-ignoreall.s
+++ b/lld/test/ELF/executable-undefined-ignoreall.s
@@ -1,11 +1,13 @@
 # REQUIRES: x86
 
-## --unresolved-symbols=ignore-all behaves similar to -shared:
+## In dynamic linking, --unresolved-symbols=ignore-all behaves similar to -shared:
 ## for PLT relocations to undefined symbols, produce dynamic relocations if we
 ## emit .dynsym.
 
+# RUN: llvm-mc -filetype=obj -triple=x86_64 %S/Inputs/shared.s -o %ta.o
+# RUN: ld.lld -shared -soname=ta %ta.o -o %ta.so
 # RUN: llvm-mc -filetype=obj -triple=x86_64-unknown-linux %s -o %t.o
-# RUN: ld.lld %t.o -o %t --unresolved-symbols=ignore-all -pie
+# RUN: ld.lld %t.o %ta.so -o %t --unresolved-symbols=ignore-all -pie
 # RUN: llvm-readobj -r %t | FileCheck %s
 
 # CHECK:      Relocations [
diff --git a/lld/test/ELF/fatlto/fatlto.invalid.s b/lld/test/ELF/fatlto/fatlto.invalid.s
index a712b1dbb58b4..a0d5fd5597b76 100644
--- a/lld/test/ELF/fatlto/fatlto.invalid.s
+++ b/lld/test/ELF/fatlto/fatlto.invalid.s
@@ -1,4 +1,6 @@
 # REQUIRES: x86
+# This test intentionally checks for fatal errors, and fatal errors aren't supported for testing when main is run twice.
+# XFAIL: main-run-twice
 # RUN: llvm-mc -filetype=obj -triple=x86_64-unknown-linux %s -o %t
 # RUN: not ld.lld %t -o /dev/null --fat-lto-objects 2>&1 | FileCheck %s
 
diff --git a/lld/test/ELF/gc-sections.s b/lld/test/ELF/gc-sections.s
index 31e00d495146a..97cbf749ae8a5 100644
--- a/lld/test/ELF/gc-sections.s
+++ b/lld/test/ELF/gc-sections.s
@@ -5,11 +5,15 @@
 # RUN: llvm-readobj --sections --symbols %t2 | FileCheck -check-prefix=NOGC %s
 # RUN: ld.lld --gc-sections --print-gc-sections %t -o %t2 | FileCheck --check-prefix=GC1-DISCARD %s
 # RUN: llvm-readobj --sections --symbols %t2 | FileCheck -check-prefix=GC1 %s
-# RUN: ld.lld --export-dynamic --gc-sections %t -o %t2
+# RUN: ld.lld -pie --export-dynamic --gc-sections %t -o %t2
+# RUN: llvm-readobj --sections --symbols %t2 | FileCheck -check-prefix=GC2 %s
+
+## In non-pie static linking, --export-dynamic currently retains the global 'd' even if it is not exported.
+# RUN: ld.lld --export-dynamic --gc-sections --print-gc-sections %t -o %t1
 # RUN: llvm-readobj --sections --symbols %t2 | FileCheck -check-prefix=GC2 %s
 
 # RUN: llvm-mc -filetype=obj -triple=x86_64 --crel %s -o %t.o
-# RUN: ld.lld --gc-sections --print-gc-sections %t.o -o %t2 | FileCheck --check-prefix=GC1-DISCARD %s
+# RUN: ld.lld -pie --gc-sections --print-gc-sections %t.o -o %t2 | FileCheck --check-prefix=GC1-DISCARD %s
 # RUN: llvm-readobj --sections --symbols %t2 | FileCheck -check-prefix=GC1 %s
 
 # NOGC: Name: .eh_frame
diff --git a/lld/test/ELF/hexagon-plt.s b/lld/test/ELF/hexagon-plt.s
index 9eca0e83afe6f..679de82923a72 100644
--- a/lld/test/ELF/hexagon-plt.s
+++ b/lld/test/ELF/hexagon-plt.s
@@ -71,12 +71,14 @@
 # DIS-NEXT:   20054: { 	r14 = asr(r14,#2)
 # DIS-NEXT:   20058:   	jumpr r28 }
 # DIS-NEXT:   2005c: { 	trap0(#219) }
-## bar's plt slot
+# DIS-EMPTY:
+# DIS-NEXT: 00020060 <bar@plt>:
 # DIS-NEXT:   20060: { 	immext(#131072)
 # DIS-NEXT:   20064:   	r14 = add(pc,##131096) }
 # DIS-NEXT:   20068: { 	r28 = memw(r14+#0) }
 # DIS-NEXT:   2006c: { 	jumpr r28 }
-## weak's plt slot
+# DIS-EMPTY:
+# DIS-NEXT: 00020070 <weak@plt>:
 # DIS-NEXT:   20070: { 	immext(#131072)
 # DIS-NEXT:   20074:   	r14 = add(pc,##131084) }
 # DIS-NEXT:   20078: { 	r28 = memw(r14+#0) }
diff --git a/lld/test/ELF/hexagon-shared.s b/lld/test/ELF/hexagon-shared.s
index 01f7286584705..cc62662d278e2 100644
--- a/lld/test/ELF/hexagon-shared.s
+++ b/lld/test/ELF/hexagon-shared.s
@@ -80,6 +80,8 @@ pvar:
 # PLT-NEXT: { r14 = asr(r14,#2)
 # PLT-NEXT: jumpr r28 }
 # PLT-NEXT: { trap0(#219) }
+# PLT-EMPTY:
+# PLT-NEXT: 000102f0 <foo@plt>:
 # PLT-NEXT: immext(#131200)
 # PLT-NEXT: r14 = add(pc,##131252) }
 # PLT-NEXT: r28 = memw(r14+#0) }
diff --git a/lld/test/ELF/hip-section-layout.s b/lld/test/ELF/hip-section-layout.s
deleted file mode 100644
index b76141c6b41ae..0000000000000
--- a/lld/test/ELF/hip-section-layout.s
+++ /dev/null
@@ -1,41 +0,0 @@
-# REQUIRES: x86
-## Test HIP specific sections layout.
-
-# RUN: llvm-mc -filetype=obj -triple=x86_64-unknown-linux --defsym=HIP_SECTIONS=1 --defsym=NON_HIP_SECTIONS=1 %s -o %t.o
-# RUN: ld.lld %t.o -o %t.out
-# RUN: llvm-readobj --sections %t.out | FileCheck %s
-
-# RUN: llvm-mc -filetype=obj -triple=x86_64-unknown-linux --defsym=NON_HIP_SECTIONS=1 %s -o %t.1.o
-# RUN: llvm-mc -filetype=obj -triple=x86_64-unknown-linux --defsym=HIP_SECTIONS=1 %s -o %t.2.o
-# RUN: ld.lld %t.1.o %t.2.o -o %t.1.s.out
-# RUN: llvm-readobj --sections %t.1.s.out | FileCheck %s
-# RUN: ld.lld %t.2.o %t.1.o -o %t.2.s.out
-# RUN: llvm-readobj --sections %t.2.s.out | FileCheck %s
-
-.ifdef HIP_SECTIONS
-.section .hipFatBinSegment,"aw",@progbits; .space 1
-.section .hip_gpubin_handle,"aw",@progbits; .space 1
-.section .hip_fatbin,"a",@progbits; .space 1
-.endif
-
-.ifdef NON_HIP_SECTIONS
-.global _start
-.text
-_start:
-.section .bss,"aw",@nobits; .space 1
-.section .debug_info,"",@progbits
-.section .debug_line,"",@progbits
-.section .debug_str,"MS",@progbits,1
-.endif
-
-# Check that the HIP sections are placed towards the end but before non allocated sections
-
-// CHECK: Name: .text
-// CHECK: Name: .bss
-// CHECK: Name: .hipFatBinSegment
-// CHECK: Name: .hip_gpubin_handle
-// CHECK: Name: .hip_fatbin
-// CHECK: Name: .debug_info
-// CHECK: Name: .debug_line
-// CHECK: Name: .debug_str
-
diff --git a/lld/test/ELF/icf-safe.s b/lld/test/ELF/icf-safe.s
index 96776feccbc67..5381532609938 100644
--- a/lld/test/ELF/icf-safe.s
+++ b/lld/test/ELF/icf-safe.s
@@ -1,16 +1,19 @@
 # REQUIRES: x86
+# RUN: llvm-mc -filetype=obj -triple=x86_64 %S/Inputs/shared.s -o %ta.o
+# RUN: ld.lld -shared -soname=ta %ta.o -o %ta.so
 
 # RUN: llvm-mc -filetype=obj -triple=x86_64-unknown-linux %s -o %t1.o
 # RUN: llvm-objcopy %t1.o %t1copy.o
 # RUN: llvm-objcopy --localize-symbol=h1 %t1.o %t1changed.o
 # RUN: ld.lld -r %t1.o -o %t1reloc.o
 # RUN: llvm-mc -filetype=obj -triple=x86_64-unknown-linux %S/Inputs/icf-safe.s -o %t2.o
-# RUN: ld.lld %t1.o %t2.o -o %t2 --icf=safe --print-icf-sections | FileCheck %s
+# RUN: ld.lld %t1.o %t2.o -o %t2 --icf=safe --print-icf-sections --export-dynamic | FileCheck %s
 # RUN: ld.lld %t1copy.o %t2.o -o %t2 --icf=safe --print-icf-sections | FileCheck %s
 # RUN: ld.lld %t1.o %t2.o -o %t3 --icf=safe --print-icf-sections -shared | FileCheck --check-prefix=EXPORT %s
-# RUN: ld.lld %t1.o %t2.o -o %t3 --icf=safe --print-icf-sections --export-dynamic | FileCheck --check-prefix=EXPORT %s
+## Exported symbols are suppressed for ICF when dynamic linking is enabled.
+# RUN: ld.lld %t1.o %t2.o %ta.so -o %t3 --icf=safe --print-icf-sections --export-dynamic | FileCheck --check-prefix=EXPORT %s
 # RUN: ld.lld %t1.o %t2.o -o %t2 --icf=all --print-icf-sections | FileCheck --check-prefix=ALL %s
-# RUN: ld.lld %t1.o %t2.o -o %t2 --icf=all --print-icf-sections --export-dynamic | FileCheck --check-prefix=ALL-EXPORT %s
+# RUN: ld.lld %t1.o %t2.o %ta.so -o %t2 --icf=all --print-icf-sections --export-dynamic | FileCheck --check-prefix=ALL-EXPORT %s
 # RUN: ld.lld %t1changed.o -o %t4 --icf=safe 2>&1 | FileCheck --check-prefix=SH_LINK_0 %s
 # RUN: ld.lld %t1reloc.o -o %t4 --icf=safe 2>&1 | FileCheck --check-prefix=SH_LINK_0 %s
 
diff --git a/lld/test/ELF/invalid-cie-reference.s b/lld/test/ELF/invalid-cie-reference.s
index 158fc4e935f28..b2d3750ff1c7a 100644
--- a/lld/test/ELF/invalid-cie-reference.s
+++ b/lld/test/ELF/invalid-cie-reference.s
@@ -1,4 +1,6 @@
 // REQUIRES: x86
+// This test intentionally checks for fatal errors, and fatal errors aren't supported for testing when main is run twice.
+// XFAIL: main-run-twice
 
 // RUN: llvm-mc -filetype=obj -triple=x86_64-pc-linux %s -o %t
 // RUN: not ld.lld %t -o /dev/null 2>&1 | FileCheck %s
diff --git a/lld/test/ELF/invalid-eh-frame2.s b/lld/test/ELF/invalid-eh-frame2.s
index 87ce8ede72503..01f38738519b6 100644
--- a/lld/test/ELF/invalid-eh-frame2.s
+++ b/lld/test/ELF/invalid-eh-frame2.s
@@ -2,6 +2,7 @@
 
 # RUN: llvm-mc -filetype=obj -triple=x86_64-pc-linux %s -o %t
 # RUN: not ld.lld --eh-frame-hdr %t -o /dev/null 2>&1 | FileCheck %s
+# RUN: ld.lld --eh-frame-hdr %t -o /dev/null --noinhibit-exec
 
 # CHECK:      error: corrupted .eh_frame: corrupted CIE (failed to read string)
 # CHECK-NEXT: >>> defined in {{.*}}:(.eh_frame+0x9)
diff --git a/lld/test/ELF/invalid-eh-frame4.s b/lld/test/ELF/invalid-eh-frame4.s
index a567bd40d73ef..60bbc7f22717c 100644
--- a/lld/test/ELF/invalid-eh-frame4.s
+++ b/lld/test/ELF/invalid-eh-frame4.s
@@ -2,6 +2,7 @@
 
 # RUN: llvm-mc -filetype=obj -triple=x86_64-pc-linux %s -o %t
 # RUN: not ld.lld --eh-frame-hdr %t -o /dev/null 2>&1 | FileCheck %s
+# RUN: ld.lld --eh-frame-hdr %t -o /dev/null --noinhibit-exec 2>&1 | FileCheck %s
 
 # CHECK: corrupted .eh_frame: unknown .eh_frame augmentation string:
 
diff --git a/lld/test/ELF/invalid-eh-frame6.s b/lld/test/ELF/invalid-eh-frame6.s
index 77be15f54e6b1..6888419da3e3d 100644
--- a/lld/test/ELF/invalid-eh-frame6.s
+++ b/lld/test/ELF/invalid-eh-frame6.s
@@ -2,6 +2,7 @@
 
 # RUN: llvm-mc -filetype=obj -triple=x86_64-pc-linux %s -o %t
 # RUN: not ld.lld --eh-frame-hdr %t -o /dev/null 2>&1 | FileCheck %s
+# RUN: ld.lld --eh-frame-hdr %t -o /dev/null --noinhibit-exec
 
 # CHECK: error: corrupted .eh_frame: unknown FDE encoding
 # CHECK-NEXT: >>> defined in {{.*}}:(.eh_frame+0xe)
diff --git a/lld/test/ELF/invalid/bad-reloc-target.test b/lld/test/ELF/invalid/bad-reloc-target.test
index 88b4cdf96779f..6a1619e81b80b 100644
--- a/lld/test/ELF/invalid/bad-reloc-target.test
+++ b/lld/test/ELF/invalid/bad-reloc-target.test
@@ -51,7 +51,7 @@ Symbols:
 ## Relocation refers to a symbol with index larger than
 ## symbol table size. Check we report it.
 # RUN: yaml2obj --docnum=3 %s -o %t2.o
-# RUN: not ld.lld %t2.o -o /dev/null 2>&1 | FileCheck %s --check-prefix=ERR3
+# RUN: env LLD_IN_TEST=1 not ld.lld %t2.o -o /dev/null 2>&1 | FileCheck %s --check-prefix=ERR3
 # ERR3: error: {{.*}}.o: invalid symbol index
 
 --- !ELF
diff --git a/lld/test/ELF/invalid/comdat-broken.test b/lld/test/ELF/invalid/comdat-broken.test
index 02941070dc0d1..6064272272636 100644
--- a/lld/test/ELF/invalid/comdat-broken.test
+++ b/lld/test/ELF/invalid/comdat-broken.test
@@ -1,4 +1,6 @@
 # REQUIRES: x86
+# This test intentionally checks for fatal errors, and fatal errors aren't supported for testing when main is run twice.
+# XFAIL: main-run-twice
 
 # RUN: yaml2obj %s -o %t.o
 # RUN: not ld.lld %t.o -o /dev/null 2>&1 | FileCheck %s
diff --git a/lld/test/ELF/invalid/data-encoding.test b/lld/test/ELF/invalid/data-encoding.test
index 94862af79c3cf..361d53da1a862 100644
--- a/lld/test/ELF/invalid/data-encoding.test
+++ b/lld/test/ELF/invalid/data-encoding.test
@@ -1,4 +1,6 @@
 # REQUIRES: x86
+# This test intentionally checks for fatal errors, and fatal errors aren't supported for testing when main is run twice.
+# XFAIL: main-run-twice
 
 # The object in the archive has invalid data encoding.
 # Check we report this.
diff --git a/lld/test/ELF/invalid/dynamic-section-broken.test b/lld/test/ELF/invalid/dynamic-section-broken.test
index 62f311470a2e5..01fe9a05b8a5f 100644
--- a/lld/test/ELF/invalid/dynamic-section-broken.test
+++ b/lld/test/ELF/invalid/dynamic-section-broken.test
@@ -1,3 +1,5 @@
+# This test intentionally checks for fatal errors, and fatal errors aren't supported for testing when main is run twice.
+# XFAIL: main-run-twice
 ## .dynamic section has invalid sh_entsize, check we report it.
 # RUN: yaml2obj --docnum=1 %s -o %t.so
 # RUN: not ld.lld %t.so -o /dev/null 2>&1 | FileCheck %s --check-prefix=ERR1
diff --git a/lld/test/ELF/invalid/invalid-elf.test b/lld/test/ELF/invalid/invalid-elf.test
index 848a430eb053c..7282259b7305c 100644
--- a/lld/test/ELF/invalid/invalid-elf.test
+++ b/lld/test/ELF/invalid/invalid-elf.test
@@ -1,3 +1,5 @@
+# This test intentionally checks for fatal errors, and fatal errors aren't supported for testing when main is run twice.
+# XFAIL: main-run-twice
 # RUN: rm -rf %t && mkdir -p %t
 # RUN: echo > %t/empty.o
 # RUN: llvm-ar --format=gnu cr %t/not-elf.a %t/empty.o
diff --git a/lld/test/ELF/invalid/invalid-file-class.test b/lld/test/ELF/invalid/invalid-file-class.test
index 3f547861b3793..dda4648e255b6 100644
--- a/lld/test/ELF/invalid/invalid-file-class.test
+++ b/lld/test/ELF/invalid/invalid-file-class.test
@@ -1,4 +1,6 @@
 # REQUIRES: x86
+# This test intentionally checks for fatal errors, and fatal errors aren't supported for testing when main is run twice.
+# XFAIL: main-run-twice
 # RUN: rm -rf %t && mkdir -p %t
 
 ## In this test, we check that able to report objects with
diff --git a/lld/test/ELF/invalid/linkorder-invalid-sec.test b/lld/test/ELF/invalid/linkorder-invalid-sec.test
index a2f4ee8f5bc2b..e0132956f0ba1 100644
--- a/lld/test/ELF/invalid/linkorder-invalid-sec.test
+++ b/lld/test/ELF/invalid/linkorder-invalid-sec.test
@@ -1,6 +1,6 @@
 # REQUIRES: x86
 # RUN: yaml2obj %s -o %t.o
-# RUN: not ld.lld %t.o -o /dev/null 2>&1 | FileCheck %s
+# RUN: not ld.lld %t.o -o /dev/null --noinhibit-exec 2>&1 | FileCheck %s
 # CHECK: invalid sh_link index: 12345
 
 --- !ELF
diff --git a/lld/test/ELF/invalid/merge-invalid-size.s b/lld/test/ELF/invalid/merge-invalid-size.s
index 71c3f98e75529..82ad1f97b4a93 100644
--- a/lld/test/ELF/invalid/merge-invalid-size.s
+++ b/lld/test/ELF/invalid/merge-invalid-size.s
@@ -1,6 +1,6 @@
 // REQUIRES: x86
 // RUN: llvm-mc %s -o %t.o -filetype=obj -triple=x86_64-pc-linux
-// RUN: not ld.lld %t.o -o /dev/null 2>&1 | FileCheck %s
+// RUN: not ld.lld %t.o -o /dev/null --noinhibit-exec 2>&1 | FileCheck %s
 // CHECK: merge-invalid-size.s.tmp.o:(.foo): SHF_MERGE section size (2) must be a multiple of sh_entsize (4)
 
 .section .foo,"aM",@progbits,4
diff --git a/lld/test/ELF/invalid/merge-writable.s b/lld/test/ELF/invalid/merge-writable.s
index 0c5fe92481da0..24a274b193576 100644
--- a/lld/test/ELF/invalid/merge-writable.s
+++ b/lld/test/ELF/invalid/merge-writable.s
@@ -1,6 +1,7 @@
 // REQUIRES: x86
 // RUN: llvm-mc %s -o %t.o -filetype=obj -triple=x86_64-pc-linux
 // RUN: not ld.lld %t.o -o /dev/null 2>&1 | FileCheck %s
+// RUN: ld.lld %t.o -o /dev/null --noinhibit-exec 2>&1 | FileCheck %s
 // CHECK: merge-writable.s.tmp.o:(.foo): writable SHF_MERGE section is not supported
 
 .section .foo,"awM",@progbits,4
diff --git a/lld/test/ELF/invalid/section-alignment.test b/lld/test/ELF/invalid/section-alignment.test
index 8099ec01849b6..32e673f82992b 100644
--- a/lld/test/ELF/invalid/section-alignment.test
+++ b/lld/test/ELF/invalid/section-alignment.test
@@ -1,5 +1,6 @@
 # RUN: yaml2obj %s -o %t
 # RUN: not ld.lld %t -o /dev/null 2>&1 | FileCheck %s
+# RUN: ld.lld %t -o /dev/null --noinhibit-exec 2>&1 | FileCheck %s
 
 ## In current lld implementation, we do not accept sh_addralign
 ## larger than UINT32_MAX.
diff --git a/lld/test/ELF/invalid/section-alignment2.s b/lld/test/ELF/invalid/section-alignment2.s
index c130bbbaa071f..c180860ca4127 100644
--- a/lld/test/ELF/invalid/section-alignment2.s
+++ b/lld/test/ELF/invalid/section-alignment2.s
@@ -1,5 +1,6 @@
 # RUN: yaml2obj %s -o %t.o
 # RUN: not ld.lld %t.o -o /dev/null 2>&1 | FileCheck %s
+# RUN: ld.lld %t.o -o /dev/null --noinhibit-exec
 
 # CHECK: error: {{.*}}.o:(.text): sh_addralign is not a power of 2
 
diff --git a/lld/test/ELF/invalid/section-index.test b/lld/test/ELF/invalid/section-index.test
index cc8c6d067265a..370597b7b7a2d 100644
--- a/lld/test/ELF/invalid/section-index.test
+++ b/lld/test/ELF/invalid/section-index.test
@@ -3,6 +3,7 @@
 
 # RUN: yaml2obj %s -o %t1.o
 # RUN: not ld.lld %t1.o -o /dev/null 2>&1 | FileCheck %s
+# RUN: ld.lld %t1.o -o /dev/null --noinhibit-exec 2>&1 | FileCheck %s
 # CHECK: {{.*}}1.o: invalid section index: 256
 
 !ELF
diff --git a/lld/test/ELF/invalid/sht-group-wrong-section.test b/lld/test/ELF/invalid/sht-group-wrong-section.test
index 46209a213902c..0eea432c1ee45 100644
--- a/lld/test/ELF/invalid/sht-group-wrong-section.test
+++ b/lld/test/ELF/invalid/sht-group-wrong-section.test
@@ -1,4 +1,6 @@
 # REQUIRES: x86
+# This test intentionally checks for fatal errors, and fatal errors aren't supported for testing when main is run twice.
+# XFAIL: main-run-twice
 # RUN: yaml2obj %s -o %t.o
 # RUN: not ld.lld %t.o %t.o -o /dev/null 2>&1 | FileCheck %s
 # CHECK: error: {{.*}}.o: invalid section index in group: 12345
diff --git a/lld/test/ELF/invalid/sht-group.test b/lld/test/ELF/invalid/sht-group.test
index 9041bb61b3e95..d50fbe6a6ce75 100644
--- a/lld/test/ELF/invalid/sht-group.test
+++ b/lld/test/ELF/invalid/sht-group.test
@@ -1,4 +1,6 @@
 # REQUIRES: x86
+# This test intentionally checks for fatal errors, and fatal errors aren't supported for testing when main is run twice.
+# XFAIL: main-run-twice
 # RUN: yaml2obj %s -o %t.o
 # RUN: not ld.lld %t.o -o /dev/null 2>&1 | FileCheck %s
 # CHECK: invalid symbol index
diff --git a/lld/test/ELF/invalid/symbol-name.test b/lld/test/ELF/invalid/symbol-name.test
index 1ae76f0bd81e7..73284a1b9b842 100644
--- a/lld/test/ELF/invalid/symbol-name.test
+++ b/lld/test/ELF/invalid/symbol-name.test
@@ -1,5 +1,6 @@
 # RUN: yaml2obj %s -o %t.o
 # RUN: not ld.lld %t.o -o /dev/null 2>&1 | FileCheck %s
+# RUN: ld.lld %t.o -o /dev/null --noinhibit-exec
 # CHECK: error: {{.*}}.o: invalid symbol name offset
 
 ## YAML below contains symbol with name offset in st_name
diff --git a/lld/test/ELF/invalid/symtab-sh-info.s b/lld/test/ELF/invalid/symtab-sh-info.s
index 253bbf9e62d56..03570892c4628 100644
--- a/lld/test/ELF/invalid/symtab-sh-info.s
+++ b/lld/test/ELF/invalid/symtab-sh-info.s
@@ -1,3 +1,5 @@
+# This test intentionally checks for fatal errors, and fatal errors aren't supported for testing when main is run twice.
+# XFAIL: main-run-twice
 ## .symtab's sh_info contains zero value. First entry in a .symtab is a
 ## zero entry that must exist in a valid object, so sh_info can't be null.
 ## Check we report a proper error for that case.
diff --git a/lld/test/ELF/invalid/verneed-shared.test b/lld/test/ELF/invalid/verneed-shared.test
index 2e2ff494fb582..fd107e4f09781 100644
--- a/lld/test/ELF/invalid/verneed-shared.test
+++ b/lld/test/ELF/invalid/verneed-shared.test
@@ -1,4 +1,6 @@
 ## REQUIRES: x86
+# This test intentionally checks for fatal errors, and fatal errors aren't supported for testing when main is run twice.
+# XFAIL: main-run-twice
 ## Test that we can parse SHT_GNU_verneed in a shared object and report certain errors.
 
 # RUN: echo '.globl _start; _start:' | llvm-mc -filetype=obj -triple=x86_64 - -o %t.o
diff --git a/lld/test/ELF/lto/bitcode-nodatalayout.ll b/lld/test/ELF/lto/bitcode-nodatalayout.ll
index 7ddc122267218..39e8263aa7287 100644
--- a/lld/test/ELF/lto/bitcode-nodatalayout.ll
+++ b/lld/test/ELF/lto/bitcode-nodatalayout.ll
@@ -1,4 +1,6 @@
 ; REQUIRES: x86
+; This test intentionally checks for fatal errors, and fatal errors aren't supported for testing when main is run twice.
+; XFAIL: main-run-twice
 ; RUN: llvm-as %s -o %t.o
 ; RUN: not ld.lld %t.o -o /dev/null 2>&1 | FileCheck %s
 
diff --git a/lld/test/ELF/lto/bitcode-wrapper.ll b/lld/test/ELF/lto/bitcode-wrapper.ll
index 183d8c28383b3..c76be4854cca3 100644
--- a/lld/test/ELF/lto/bitcode-wrapper.ll
+++ b/lld/test/ELF/lto/bitcode-wrapper.ll
@@ -1,4 +1,6 @@
 ; REQUIRES: x86
+; This test intentionally checks for fatal errors, and fatal errors aren't supported for testing when main is run twice.
+; XFAIL: main-run-twice
 
 ;; The LLVM bitcode format allows for an optional wrapper header. This test
 ;; shows that LLD can handle bitcode wrapped in this way, and also that an
diff --git a/lld/test/ELF/lto/cache-warnings.ll b/lld/test/ELF/lto/cache-warnings.ll
index d8c5ea963ec13..d0224d5426ff3 100644
--- a/lld/test/ELF/lto/cache-warnings.ll
+++ b/lld/test/ELF/lto/cache-warnings.ll
@@ -1,4 +1,5 @@
 ; REQUIRES: x86, shell
+; UNSUPPORTED: main-run-twice
 
 ; RUN: opt -module-hash -module-summary %s -o %t.o
 ; RUN: opt -module-hash -module-summary %p/Inputs/cache.ll -o %t2.o
diff --git a/lld/test/ELF/lto/devirt_vcall_vis_export_dynamic.ll b/lld/test/ELF/lto/devirt_vcall_vis_export_dynamic.ll
index 2a52c5ad8ae42..189e3c0b821bd 100644
--- a/lld/test/ELF/lto/devirt_vcall_vis_export_dynamic.ll
+++ b/lld/test/ELF/lto/devirt_vcall_vis_export_dynamic.ll
@@ -28,6 +28,10 @@
 
 ;; Check that all WPD fails with --export-dynamic.
 
+; RUN: echo '.globl foo; foo:' > %ta.s
+; RUN: llvm-mc -filetype=obj -triple=x86_64 %ta.s -o %ta.o
+; RUN: ld.lld -shared -soname=ta %ta.o -o %ta.so
+
 ;; Index based WPD
 ; RUN: ld.lld %t2.o -o %t3 -save-temps --lto-whole-program-visibility \
 ; RUN:   -mllvm -pass-remarks=. \
@@ -49,19 +53,19 @@
 ;; Check that WPD fails for target _ZN1D1mEi with --export-dynamic-symbol=_ZTV1D.
 
 ;; Index based WPD
-; RUN: ld.lld %t2.o -o %t3 -save-temps --lto-whole-program-visibility \
+; RUN: ld.lld %t2.o %ta.so -o %t3 -save-temps --lto-whole-program-visibility \
 ; RUN:   -mllvm -pass-remarks=. \
 ; RUN:   --export-dynamic-symbol=_ZTV1D 2>&1 | FileCheck %s --check-prefix=REMARK-AONLY
 ; RUN: llvm-dis %t2.o.4.opt.bc -o - | FileCheck %s --check-prefix=CHECK-AONLY-IR
 
 ;; Hybrid WPD
-; RUN: ld.lld %t.o -o %t3 -save-temps --lto-whole-program-visibility \
+; RUN: ld.lld %t.o %ta.so -o %t3 -save-temps --lto-whole-program-visibility \
 ; RUN:   -mllvm -pass-remarks=. \
 ; RUN:   --export-dynamic-symbol=_ZTV1D 2>&1 | FileCheck %s --check-prefix=REMARK-AONLY
 ; RUN: llvm-dis %t.o.4.opt.bc -o - | FileCheck %s --check-prefix=CHECK-AONLY-IR
 
 ;; Regular LTO WPD
-; RUN: ld.lld %t4.o -o %t3 -save-temps --lto-whole-program-visibility \
+; RUN: ld.lld %t4.o %ta.so -o %t3 -save-temps --lto-whole-program-visibility \
 ; RUN:   -mllvm -pass-remarks=. \
 ; RUN:   --export-dynamic-symbol=_ZTV1D 2>&1 | FileCheck %s --check-prefix=REMARK-AONLY
 ; RUN: llvm-dis %t3.0.4.opt.bc -o - | FileCheck %s --check-prefix=CHECK-AONLY-IR
@@ -74,19 +78,19 @@
 ; RUN: echo "{ _ZTV1D; };" > %t.list
 
 ;; Index based WPD
-; RUN: ld.lld %t2.o -o %t3 -save-temps --lto-whole-program-visibility \
+; RUN: ld.lld %t2.o %ta.so -o %t3 -save-temps --lto-whole-program-visibility \
 ; RUN:   -mllvm -pass-remarks=. \
 ; RUN:   --dynamic-list=%t.list 2>&1 | FileCheck %s --check-prefix=REMARK-AONLY
 ; RUN: llvm-dis %t2.o.4.opt.bc -o - | FileCheck %s --check-prefix=CHECK-AONLY-IR
 
 ;; Hybrid WPD
-; RUN: ld.lld %t.o -o %t3 -save-temps --lto-whole-program-visibility \
+; RUN: ld.lld %t.o %ta.so -o %t3 -save-temps --lto-whole-program-visibility \
 ; RUN:   -mllvm -pass-remarks=. \
 ; RUN:   --dynamic-list=%t.list 2>&1 | FileCheck %s --check-prefix=REMARK-AONLY
 ; RUN: llvm-dis %t.o.4.opt.bc -o - | FileCheck %s --check-prefix=CHECK-AONLY-IR
 
 ;; Regular LTO WPD
-; RUN: ld.lld %t4.o -o %t3 -save-temps --lto-whole-program-visibility \
+; RUN: ld.lld %t4.o %ta.so -o %t3 -save-temps --lto-whole-program-visibility \
 ; RUN:   -mllvm -pass-remarks=. \
 ; RUN:   --dynamic-list=%t.list 2>&1 | FileCheck %s --check-prefix=REMARK-AONLY
 ; RUN: llvm-dis %t3.0.4.opt.bc -o - | FileCheck %s --check-prefix=CHECK-AONLY-IR
diff --git a/lld/test/ELF/lto/internalize-exportdyn.ll b/lld/test/ELF/lto/internalize-exportdyn.ll
index 25e319269bbdb..859f881cf3293 100644
--- a/lld/test/ELF/lto/internalize-exportdyn.ll
+++ b/lld/test/ELF/lto/internalize-exportdyn.ll
@@ -3,7 +3,7 @@
 ; RUN: llvm-as a.ll -o a.bc
 ; RUN: llvm-as %p/Inputs/internalize-exportdyn.ll -o b.bc
 ; RUN: llvm-mc -filetype=obj -triple=x86_64 lib.s -o lib.o
-; RUN: ld.lld a.bc b.bc lib.o -o out --export-dynamic -save-temps
+; RUN: ld.lld a.bc b.bc lib.o -o out --export-dynamic -save-temps -pie
 ; RUN: llvm-dis < out.0.2.internalize.bc | FileCheck %s
 ; RUN: ld.lld a.bc b.bc lib.o -o out2 -shared -save-temps
 ; RUN: llvm-dis < out2.0.2.internalize.bc | FileCheck %s --check-prefix=DSO
diff --git a/lld/test/ELF/lto/ltopasses-custom.ll b/lld/test/ELF/lto/ltopasses-custom.ll
index ecb024cecade4..e37083ca8b8c7 100644
--- a/lld/test/ELF/lto/ltopasses-custom.ll
+++ b/lld/test/ELF/lto/ltopasses-custom.ll
@@ -24,13 +24,13 @@ define void @barrier() {
 ; ATOMIC-NEXT: ret void
 
 ; Check that invalid passes are rejected gracefully.
-; RUN: not --crash ld.lld -m elf_x86_64 %t.o -o /dev/null \
+; RUN: env LLD_IN_TEST=1 not --crash ld.lld -m elf_x86_64 %t.o -o /dev/null \
 ; RUN:   --lto-newpm-passes=iamnotapass -shared 2>&1 | \
 ; RUN:   FileCheck %s --check-prefix=INVALID
 ; INVALID: unable to parse pass pipeline description 'iamnotapass': unknown pass name 'iamnotapass'
 
 ; Check that invalid AA pipelines are rejected gracefully.
-; RUN: not --crash ld.lld -m elf_x86_64 %t.o -o /dev/null \
+; RUN: env LLD_IN_TEST=1 not --crash ld.lld -m elf_x86_64 %t.o -o /dev/null \
 ; RUN:   --lto-newpm-passes=globaldce --lto-aa-pipeline=patatino \
 ; RUN:   -shared 2>&1 | \
 ; RUN:   FileCheck %s --check-prefix=INVALIDAA
diff --git a/lld/test/ELF/lto/relocation-model-pic.ll b/lld/test/ELF/lto/relocation-model-pic.ll
index 813ffa76b8d5b..92ac6e5c584ac 100644
--- a/lld/test/ELF/lto/relocation-model-pic.ll
+++ b/lld/test/ELF/lto/relocation-model-pic.ll
@@ -16,7 +16,7 @@ target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
 @foo = external global i32
-define i32 @main() {
+define i32 @_start() {
   %t = load i32, ptr @foo
   ret i32 %t
 }
diff --git a/lld/test/ELF/lto/relocation-model-static.ll b/lld/test/ELF/lto/relocation-model-static.ll
index 443c419624cb8..26503f2044f82 100644
--- a/lld/test/ELF/lto/relocation-model-static.ll
+++ b/lld/test/ELF/lto/relocation-model-static.ll
@@ -13,7 +13,7 @@ target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16
 target triple = "x86_64-unknown-linux-gnu"
 
 @foo = external dso_local global i32
-define i32 @main() {
+define i32 @_start() {
   %t = load i32, ptr @foo
   ret i32 %t
 }
diff --git a/lld/test/ELF/lto/verify-invalid.ll b/lld/test/ELF/lto/verify-invalid.ll
index d97d0e1b78b8c..cb8bb389a608b 100644
--- a/lld/test/ELF/lto/verify-invalid.ll
+++ b/lld/test/ELF/lto/verify-invalid.ll
@@ -1,10 +1,10 @@
 ; REQUIRES: x86
 ; RUN: llvm-as %s -o %t.o
-; RUN: ld.lld %t.o -o %t2 --lto-debug-pass-manager \
+; RUN: env LLD_IN_TEST=1 ld.lld %t.o -o %t2 --lto-debug-pass-manager \
 ; RUN:   2>&1 | FileCheck -check-prefix=DEFAULT-NPM %s
-; RUN: ld.lld %t.o -o %t2 --lto-debug-pass-manager \
+; RUN: env LLD_IN_TEST=1 ld.lld %t.o -o %t2 --lto-debug-pass-manager \
 ; RUN:   -disable-verify 2>&1 | FileCheck -check-prefix=DISABLE-NPM %s
-; RUN: ld.lld %t.o -o %t2 --lto-debug-pass-manager \
+; RUN: env LLD_IN_TEST=1 ld.lld %t.o -o %t2 --lto-debug-pass-manager \
 ; RUN:   --plugin-opt=disable-verify 2>&1 | FileCheck -check-prefix=DISABLE-NPM %s
 
 target triple = "x86_64-unknown-linux-gnu"
diff --git a/lld/test/ELF/lto/version-script.ll b/lld/test/ELF/lto/version-script.ll
index 52b9afc38eeed..54a5e01f2dee5 100644
--- a/lld/test/ELF/lto/version-script.ll
+++ b/lld/test/ELF/lto/version-script.ll
@@ -3,7 +3,7 @@
 ; RUN: echo "VERSION_1.0{ global: foo; local: *; }; VERSION_2.0{ global: bar; local: *; };" > %t.script
 ; RUN: ld.lld %t.o -o %t2 -shared --version-script %t.script -save-temps
 ; RUN: llvm-dis < %t2.0.0.preopt.bc | FileCheck %s
-; RUN: llvm-readobj --dyn-syms %t2 | FileCheck --check-prefix=DSO %s
+; RUN: llvm-readelf --dyn-syms %t2 | FileCheck --check-prefix=DSO %s
 
 target triple = "x86_64-unknown-linux-gnu"
 target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
@@ -19,32 +19,6 @@ define void @bar() {
 ; CHECK: define void @foo()
 ; CHECK: define void @bar()
 
-; DSO: DynamicSymbols [
-; DSO:   Symbol {
-; DSO:     Name:
-; DSO:     Value: 0x0
-; DSO:     Size: 0
-; DSO:     Binding: Local
-; DSO:     Type: None
-; DSO:     Other: 0
-; DSO:     Section: Undefined
-; DSO:   }
-; DSO:   Symbol {
-; DSO:     Name: foo@@VERSION_1.0
-; DSO:     Value:
-; DSO:     Size: 1
-; DSO:     Binding: Global
-; DSO:     Type: Function
-; DSO:     Other: 0
-; DSO:     Section: .text
-; DSO:   }
-; DSO:   Symbol {
-; DSO:     Name: bar@@VERSION_2.0
-; DSO:     Value:
-; DSO:     Size: 1
-; DSO:     Binding: Global
-; DSO:     Type: Function
-; DSO:     Other: 0
-; DSO:     Section: .text
-; DSO:   }
-; DSO: ]
+; DSO: Symbol table '.dynsym' contains 3 entries:
+; DSO:      1: {{.*}}               1 FUNC    GLOBAL DEFAULT [[#]] foo@@VERSION_1.0{{$}}
+; DSO:      2: {{.*}}               1 FUNC    GLOBAL DEFAULT [[#]] bar@@VERSION_2.0{{$}}
diff --git a/lld/test/ELF/lto/version-script2.ll b/lld/test/ELF/lto/version-script2.ll
index dab22750f77b8..5635731518fdb 100644
--- a/lld/test/ELF/lto/version-script2.ll
+++ b/lld/test/ELF/lto/version-script2.ll
@@ -17,16 +17,16 @@
 ; RUN: ld.lld %t.o %tbar.so -o %t.so -shared --version-script %t/ver
 ; RUN: llvm-readelf --dyn-syms %t.so | FileCheck %s
 
-; CHECK:      UND       bar@VER1
-; CHECK-NEXT: {{[1-9]}} foo@@VER1
+; CHECK:      UND       bar@VER1{{$}}
+; CHECK-NEXT: {{[1-9]}} foo@@VER1{{$}}
 
 ;; For relocatable output, @ should be retained in the symbol name.
 ;; Don't parse and drop `@VER1`. Also check that --version-script is ignored.
 ; RUN: ld.lld %t.o -o %t.ro -r --version-script %t/ver
 ; RUN: llvm-readelf -s %t.ro | FileCheck %s --check-prefix=RELOCATABLE
 
-; RELOCATABLE:      {{[1-9]}} foo@@VER1
-; RELOCATABLE-NEXT: UND       bar@VER1
+; RELOCATABLE:      {{[1-9]}} foo@@VER1{{$}}
+; RELOCATABLE-NEXT: UND       bar@VER1{{$}}
 
 ;--- ver
 VER1 {};
diff --git a/lld/test/ELF/merge-string-error.s b/lld/test/ELF/merge-string-error.s
deleted file mode 100644
index bd77a4c1dce87..0000000000000
--- a/lld/test/ELF/merge-string-error.s
+++ /dev/null
@@ -1,11 +0,0 @@
-// REQUIRES: x86
-// RUN: llvm-mc -filetype=obj -triple=x86_64-pc-linux %s -o %t.o
-// RUN: not ld.lld %t.o -o /dev/null -shared 2>&1 | FileCheck %s
-
-        .section	.rodata.str1.1,"aMS",@progbits,1
-	.asciz	"abc"
-
-        .data
-        .quad .rodata.str1.1 + 4
-
-// CHECK: merge-string-error.s.tmp.o:(.rodata.str1.1): offset is outside the section
diff --git a/lld/test/ELF/mergeable-errors.s b/lld/test/ELF/mergeable-errors.s
index d67cd91c97fbf..b155d581046a8 100644
--- a/lld/test/ELF/mergeable-errors.s
+++ b/lld/test/ELF/mergeable-errors.s
@@ -1,6 +1,7 @@
 # REQUIRES: x86
 # RUN: llvm-mc -filetype=obj -triple=x86_64-unknown-linux %s -o %t.o
 # RUN: not ld.lld %t.o -o /dev/null 2>&1 | FileCheck %s
+# RUN: ld.lld %t.o -o /dev/null --noinhibit-exec
 
 # CHECK: error: {{.*}}.o:(.mergeable): string is not null terminated
 
diff --git a/lld/test/ELF/package-metadata.s b/lld/test/ELF/package-metadata.s
index 29df499d7e98d..a70a8940d7c68 100644
--- a/lld/test/ELF/package-metadata.s
+++ b/lld/test/ELF/package-metadata.s
@@ -1,12 +1,15 @@
 # REQUIRES: x86
+# RUN: rm -rf %t && split-file %s %t && cd %t
+# RUN: llvm-mc -filetype=obj -triple=x86_64 a.s -o a.o
 
-# RUN: llvm-mc -filetype=obj -triple=x86_64 %s -o %t.o
+# RUN: ld.lld a.o --package-metadata='{}'
+# RUN: llvm-readelf -n a.out | FileCheck %s --check-prefixes=NOTE,FIRST
 
-# RUN: ld.lld %t.o -o %t --package-metadata='{}'
-# RUN: llvm-readelf -n %t | FileCheck %s --check-prefixes=NOTE,FIRST
+# RUN: ld.lld a.o --package-metadata='{"abc":123}'
+# RUN: llvm-readelf -n a.out | FileCheck %s --check-prefixes=NOTE,SECOND
 
-# RUN: ld.lld %t.o -o %t --package-metadata='{"abc":123}'
-# RUN: llvm-readelf -n %t | FileCheck %s --check-prefixes=NOTE,SECOND
+# RUN: ld.lld a.o --package-metadata='%7b%22abc%22:123%7D'
+# RUN: llvm-readelf -n a.out | FileCheck %s --check-prefixes=NOTE,SECOND
 
 # NOTE: .note.package
 # NOTE-NEXT: Owner
@@ -14,6 +17,13 @@
 # FIRST-NEXT: description data: 7b 7d 00
 # SECOND-NEXT: description data: 7b 22 61 62 63 22 3a 31 32 33 7d 00
 
+# RUN: not ld.lld a.o --package-metadata='%7b%' 2>&1 | FileCheck %s --check-prefix=ERR
+# RUN: not ld.lld a.o --package-metadata='%7b%7' 2>&1 | FileCheck %s --check-prefix=ERR
+# RUN: not ld.lld a.o --package-metadata='%7b%7g' 2>&1 | FileCheck %s --check-prefix=ERR
+
+# ERR: error: --package-metadata=: invalid % escape at byte 3; supports only %[0-9a-fA-F][0-9a-fA-F]
+
+#--- a.s
 .globl _start
 _start:
   ret
diff --git a/lld/test/ELF/partition-errors.s b/lld/test/ELF/partition-errors.s
index 6150fe20cdadb..476f377640871 100644
--- a/lld/test/ELF/partition-errors.s
+++ b/lld/test/ELF/partition-errors.s
@@ -1,16 +1,21 @@
 // REQUIRES: x86, mips
+/// Link against a DSO to ensure that sections are not discarded by --gc-sections.
+// RUN: llvm-mc %S/Inputs/shared.s -o %ts.o -filetype=obj --triple=x86_64
+// RUN: ld.lld -shared -soname=ts %ts.o -o %ts.so
 // RUN: llvm-mc -triple=x86_64-unknown-linux -filetype=obj -o %t.o %s
 // RUN: echo "SECTIONS {}" > %t.script
-// RUN: not ld.lld --export-dynamic %t.o %t.script 2>&1 | FileCheck %s
+// RUN: not ld.lld --export-dynamic %ts.so %t.o %t.script 2>&1 | FileCheck %s
 // RUN: echo "PHDRS { text PT_LOAD; }" > %t2.script
-// RUN: not ld.lld --export-dynamic %t.o %t2.script 2>&1 | FileCheck %s
-// RUN: not ld.lld --export-dynamic %t.o --section-start .text=0 2>&1 | FileCheck %s
-// RUN: not ld.lld --export-dynamic %t.o -Ttext=0 2>&1 | FileCheck %s
-// RUN: not ld.lld --export-dynamic %t.o -Tdata=0 2>&1 | FileCheck %s
-// RUN: not ld.lld --export-dynamic %t.o -Tbss=0 2>&1 | FileCheck %s
+// RUN: not ld.lld --export-dynamic %ts.so %t.o %t2.script 2>&1 | FileCheck %s
+// RUN: not ld.lld --export-dynamic %ts.so %t.o --section-start .text=0 2>&1 | FileCheck %s
+// RUN: not ld.lld --export-dynamic %ts.so %t.o -Ttext=0 2>&1 | FileCheck %s
+// RUN: not ld.lld --export-dynamic %ts.so %t.o -Tdata=0 2>&1 | FileCheck %s
+// RUN: not ld.lld --export-dynamic %ts.so %t.o -Tbss=0 2>&1 | FileCheck %s
 
+// RUN: llvm-mc %S/Inputs/shared.s -o %ts.o -filetype=obj --triple=mipsel-unknown-linux
+// RUN: ld.lld -shared -soname=ts %ts.o -o %ts.so
 // RUN: llvm-mc -triple=mipsel-unknown-linux -filetype=obj -o %t2.o %s
-// RUN: not ld.lld --export-dynamic %t2.o 2>&1 | FileCheck %s
+// RUN: not ld.lld --export-dynamic %t2.o %ts.so 2>&1 | FileCheck %s
 
 // CHECK: error: {{.*}}.o: partitions cannot be used
 
diff --git a/lld/test/ELF/partition-icf.s b/lld/test/ELF/partition-icf.s
index e8608c899fc19..cffb5d4c8fabf 100644
--- a/lld/test/ELF/partition-icf.s
+++ b/lld/test/ELF/partition-icf.s
@@ -1,6 +1,9 @@
 // REQUIRES: x86
+/// Link against a DSO to ensure that sections are not discarded by --gc-sections.
+// RUN: llvm-mc -filetype=obj -triple=x86_64 %S/Inputs/shared.s -o %ts.o
+// RUN: ld.lld -shared -soname=ts %ts.o -o %ts.so
 // RUN: llvm-mc %s -o %t.o -filetype=obj --triple=x86_64-unknown-linux
-// RUN: ld.lld %t.o -o %t --export-dynamic --gc-sections --icf=all
+// RUN: ld.lld %t.o %ts.so -o %t --export-dynamic --gc-sections --icf=all
 // RUN: llvm-readelf -S -s %t | FileCheck %s
 
 // CHECK: part1
diff --git a/lld/test/ELF/partition-move-to-main-startstop.s b/lld/test/ELF/partition-move-to-main-startstop.s
index 76c04957b3f72..9620412573471 100644
--- a/lld/test/ELF/partition-move-to-main-startstop.s
+++ b/lld/test/ELF/partition-move-to-main-startstop.s
@@ -1,6 +1,9 @@
 // REQUIRES: x86
+/// Link against a DSO to ensure that sections are not discarded by --gc-sections.
+// RUN: llvm-mc -filetype=obj -triple=x86_64 %S/Inputs/shared.s -o %ts.o
+// RUN: ld.lld -shared -soname=ts %ts.o -o %ts.so
 // RUN: llvm-mc %s -o %t.o -filetype=obj --triple=x86_64-unknown-linux
-// RUN: ld.lld %t.o -o %t --export-dynamic --gc-sections
+// RUN: ld.lld %t.o %ts.so -o %t --export-dynamic --gc-sections
 // RUN: llvm-readelf -S %t | FileCheck --implicit-check-not=has_startstop %s
 
 // We can't let the has_startstop section be split by partition because it is
diff --git a/lld/test/ELF/partition-thunk-reuse.s b/lld/test/ELF/partition-thunk-reuse.s
index 6425bdf345b03..c60adc4ebdd4d 100644
--- a/lld/test/ELF/partition-thunk-reuse.s
+++ b/lld/test/ELF/partition-thunk-reuse.s
@@ -1,6 +1,9 @@
 // REQUIRES: arm
+/// Link against a DSO to ensure that sections are not discarded by --gc-sections.
+// RUN: llvm-mc -filetype=obj -triple=armv7-unknown-linux %S/Inputs/shared.s -o %ts.o
+// RUN: ld.lld -shared -soname=ts %ts.o -o %ts.so
 // RUN: llvm-mc %s -o %t.o -filetype=obj --triple=armv7-unknown-linux -arm-add-build-attributes
-// RUN: ld.lld %t.o -o %t --export-dynamic --gc-sections
+// RUN: ld.lld %t.o %ts.so -o %t --export-dynamic --gc-sections
 // RUN: llvm-nm %t | FileCheck %s
 
 // CHECK: __Thumbv7ABSLongThunk__start
diff --git a/lld/test/ELF/partitions.s b/lld/test/ELF/partitions.s
index 103094e081a37..0fc25b4da8617 100644
--- a/lld/test/ELF/partitions.s
+++ b/lld/test/ELF/partitions.s
@@ -1,10 +1,15 @@
 // REQUIRES: aarch64, x86
+/// Link against a DSO to ensure that sections are not discarded by --gc-sections.
+// RUN: llvm-mc %S/Inputs/shared.s -o %ts.o -filetype=obj --triple=x86_64
+// RUN: ld.lld -shared -soname=ts %ts.o -o %ts.so
 // RUN: llvm-mc %s -o %t.o -filetype=obj --triple=x86_64-unknown-linux
-// RUN: ld.lld %t.o -o %t --export-dynamic --gc-sections -z max-page-size=65536
+// RUN: ld.lld %t.o %ts.so -o %t --export-dynamic --gc-sections -z max-page-size=65536
 // RUN: llvm-readelf -S -s %t | FileCheck %s
 
+// RUN: llvm-mc %S/Inputs/shared.s -o %ts.o -filetype=obj --triple=aarch64
+// RUN: ld.lld -shared -soname=ts %ts.o -o %ts.so
 // RUN: llvm-mc %s -o %t.o -filetype=obj --triple=aarch64 --crel
-// RUN: ld.lld %t.o -o %t --export-dynamic --gc-sections
+// RUN: ld.lld %t.o %ts.so -o %t --export-dynamic --gc-sections
 // RUN: llvm-readelf -S -s %t | FileCheck %s
 
 // This is basically lld/docs/partitions.dot in object file form.
diff --git a/lld/test/ELF/ppc32-weak-undef-call.s b/lld/test/ELF/ppc32-weak-undef-call.s
index dfb45e5fe18cf..1ad280a49c038 100644
--- a/lld/test/ELF/ppc32-weak-undef-call.s
+++ b/lld/test/ELF/ppc32-weak-undef-call.s
@@ -1,15 +1,15 @@
 # REQUIRES: ppc
 # RUN: llvm-mc -filetype=obj -triple=powerpc %s -o %t.o
 # RUN: ld.lld %t.o -o %t
-# RUN: llvm-objdump -d --no-show-raw-insn %t | FileCheck --check-prefix=PDE %s
+# RUN: llvm-objdump -d --no-show-raw-insn %t | FileCheck --check-prefix=STATIC %s
 # RUN: ld.lld -pie %t.o -o %t
-# RUN: llvm-objdump -d --no-show-raw-insn %t | FileCheck --check-prefix=PIC %s
+# RUN: llvm-objdump -d --no-show-raw-insn %t | FileCheck --check-prefix=STATIC %s
 # RUN: ld.lld -shared %t.o -o %t
 # RUN: llvm-objdump -d --no-show-raw-insn %t | FileCheck --check-prefix=PIC %s
 
 ## It does not really matter how we fixup it, but we cannot overflow and
 ## should not generate a call stub (this would waste space).
-# PDE: bl 0x100100b4
+# STATIC: bl {{.*}} <.text>
 
 ## With -pie or -shared, create a call stub. ld.bfd produces bl .+0
 # PIC:       bl 0x[[PLT:[0-9a-f]+]]
diff --git a/lld/test/ELF/ppc64-undefined-weak.s b/lld/test/ELF/ppc64-undefined-weak.s
index 7b1be5e36dd32..e9168020b216f 100644
--- a/lld/test/ELF/ppc64-undefined-weak.s
+++ b/lld/test/ELF/ppc64-undefined-weak.s
@@ -2,25 +2,25 @@
 
 # RUN: llvm-mc -filetype=obj -triple=powerpc64le %s -o %t.o
 # RUN: ld.lld %t.o -o %t
-# RUN: llvm-objdump -d --no-show-raw-insn %t | FileCheck %s --check-prefix=PDE
+# RUN: llvm-objdump -d --no-show-raw-insn %t | FileCheck %s --check-prefix=STATIC
 # RUN: ld.lld -pie %t.o -o %t
-# RUN: llvm-objdump -d --no-show-raw-insn %t | FileCheck %s --check-prefix=PIC
+# RUN: llvm-objdump -d --no-show-raw-insn %t | FileCheck %s --check-prefix=STATIC
 # RUN: ld.lld -shared %t.o -o %t.so
 # RUN: llvm-objdump -d --no-show-raw-insn %t.so | FileCheck %s --check-prefix=PIC
 
 # RUN: llvm-mc -filetype=obj -triple=powerpc64 %s -o %t.o
 # RUN: ld.lld %t.o -o %t
-# RUN: llvm-objdump -d --no-show-raw-insn %t | FileCheck %s --check-prefix=PDE
+# RUN: llvm-objdump -d --no-show-raw-insn %t | FileCheck %s --check-prefix=STATIC
 
 ## Branches to an undefined weak symbol need a thunk iff a dynamic relocation is
 ## produced. undefweak2 is hidden and does not need a dynamic relocation, so we
 ## suppress the thunk. undefweak1 needs a thunk iff -pie or -shared.
 
-# PDE-LABEL: <_start>:
-# PDE-NEXT:    bl {{.*}} <_start>
-# PDE-NEXT:    nop
-# PDE-NEXT:    bl {{.*}} <_start+0x8>
-# PDE-NEXT:    nop
+# STATIC-LABEL: <_start>:
+# STATIC-NEXT:    bl {{.*}} <_start>
+# STATIC-NEXT:    nop
+# STATIC-NEXT:    bl {{.*}} <_start+0x8>
+# STATIC-NEXT:    nop
 
 # PIC-LABEL: <_start>:
 # PIC-NEXT:    bl {{.*}} <__plt_undefweak1>
diff --git a/lld/test/ELF/relocation-past-merge-end.s b/lld/test/ELF/relocation-past-merge-end.s
index 15214a5a4fc05..1dced95c49ac2 100644
--- a/lld/test/ELF/relocation-past-merge-end.s
+++ b/lld/test/ELF/relocation-past-merge-end.s
@@ -1,9 +1,16 @@
 // REQUIRES: x86
 // RUN: llvm-mc %s -o %t.o -filetype=obj -triple=x86_64-pc-linux
 // RUN: not ld.lld %t.o -o /dev/null -shared 2>&1 | FileCheck %s
-// CHECK: relocation-past-merge-end.s.tmp.o:(.foo): offset is outside the section
+// RUN: ld.lld %t.o -o /dev/null -shared --noinhibit-exec 2>&1 | FileCheck %s
+// CHECK: .o:(.foo): offset is outside the section
+// CHECCK: .o:(.rodata.str1.1): offset is outside the section
 
 .data
 .quad .foo + 10
+.quad .rodata.str1.1 + 4
+
 .section	.foo,"aM",@progbits,4
 .quad 0
+
+.section	.rodata.str1.1,"aMS",@progbits,1
+.asciz	"abc"
diff --git a/lld/test/ELF/riscv-gp.s b/lld/test/ELF/riscv-gp.s
index e82e36ee9a7ae..2f715e1470f2d 100644
--- a/lld/test/ELF/riscv-gp.s
+++ b/lld/test/ELF/riscv-gp.s
@@ -14,10 +14,21 @@
 # SEC32: {{0*}}000039c0 0 NOTYPE GLOBAL DEFAULT [[#SDATA]] __global_pointer$
 
 # SEC64: [ [[#SDATA:]]] .sdata PROGBITS {{0*}}000032e0
+# SEC64:     '.dynsym'
+# SEC64-NOT: __global_pointer$
+# SEC64:     '.symtab'
 # SEC64: {{0*}}00003ae0 0 NOTYPE GLOBAL DEFAULT [[#SDATA]] __global_pointer$
 
 # ERR: error: relocation R_RISCV_PCREL_HI20 cannot be used against symbol '__global_pointer$'; recompile with -fPIC
 
+# RUN: ld.lld -pie --no-dynamic-linker --export-dynamic %t.64.o -o %t.64e
+# RUN: llvm-readelf -s %t.64e | FileCheck %s --check-prefix=STATICE
+
+# STATICE:     '.dynsym'
+# STATICE:     __global_pointer$
+# STATICE:     '.symtab'
+# STATICE:     __global_pointer$
+
 ## -r mode does not define __global_pointer$.
 # RUN: ld.lld -r %t.64.o -o %t.64.ro
 # RUN: llvm-readelf -s %t.64.ro | FileCheck --check-prefix=RELOCATABLE %s
diff --git a/lld/test/ELF/stdout.s b/lld/test/ELF/stdout.s
index 64cf64a72b4b6..e33ab3f792c40 100644
--- a/lld/test/ELF/stdout.s
+++ b/lld/test/ELF/stdout.s
@@ -1,7 +1,8 @@
 # REQUIRES: x86
 
 # RUN: llvm-mc -filetype=obj -triple=x86_64-unknown-linux %s -o %t.o
-# RUN: ld.lld %t.o -o - > %t1
+## Test that we only write to "-" once.
+# RUN: env LLD_IN_TEST=2 ld.lld %t.o -o - > %t1
 # RUN: llvm-objdump -d %t1 | FileCheck %s
 
 # CHECK: nop
diff --git a/lld/test/ELF/target-specific-options.s b/lld/test/ELF/target-specific-options.s
index 94dccd9e4a0c1..0f126f0186f8b 100644
--- a/lld/test/ELF/target-specific-options.s
+++ b/lld/test/ELF/target-specific-options.s
@@ -2,7 +2,7 @@
 # RUN: llvm-mc -filetype=obj -triple=x86_64 %s -o %t
 
 # RUN: not ld.lld %t --fix-cortex-a53-843419 -o /dev/null 2>&1 | FileCheck %s --check-prefix=ERR-843419
-# ERR-843419: error: --fix-cortex-a53-843419 is only supported on AArch64 targets
+# ERR-843419: error: --fix-cortex-a53-843419 is only supported on AArch64
 
 # RUN: not ld.lld %t --be8 -o /dev/null 2>&1 | FileCheck %s --check-prefix=ERR-BE8
 # ERR-BE8: error: --be8 is only supported on ARM targets
diff --git a/lld/test/ELF/unsupported-emachine.test b/lld/test/ELF/unsupported-emachine.test
index 43d907ea72d56..54a3402378bfb 100644
--- a/lld/test/ELF/unsupported-emachine.test
+++ b/lld/test/ELF/unsupported-emachine.test
@@ -1,3 +1,5 @@
+# This test intentionally checks for fatal errors, and fatal errors aren't supported for testing when main is run twice.
+# XFAIL: main-run-twice
 # RUN: yaml2obj %s -o %t.o
 # RUN: not ld.lld %t.o -o /dev/null 2>&1 | FileCheck %s
 
diff --git a/lld/test/ELF/weak-undef-lib.s b/lld/test/ELF/weak-undef-lib.s
index 19b59fddaa723..0ff1bc755f075 100644
--- a/lld/test/ELF/weak-undef-lib.s
+++ b/lld/test/ELF/weak-undef-lib.s
@@ -7,7 +7,7 @@
 # RUN: llvm-readobj --dyn-syms %t.so | FileCheck %s
 
 # RUN: ld.lld -pie -o %t %t1.o --start-lib %t2.o
-# RUN: llvm-readobj --dyn-syms %t | FileCheck %s
+# RUN: llvm-readelf --dyn-syms %t | FileCheck %s --check-prefix=STATICPIE
 
 # CHECK:      Name: foo
 # CHECK-NEXT: Value: 0x0
@@ -17,6 +17,11 @@
 # CHECK-NEXT: Other: 0
 # CHECK-NEXT: Section: Undefined
 
+# RUN: ld.lld -pie -o %t %t1.o --start-lib %t2.o --no-dynamic-linker
+# RUN: llvm-readelf --dyn-syms %t | FileCheck %s --check-prefix=STATICPIE
+
+# STATICPIE: Symbol table '.dynsym' contains 1
+
 ## -u specifies a STB_DEFAULT undefined symbol, so the definition from %t2.o is
 ## fetched.
 # RUN: ld.lld -u foo %t1.o --start-lib %t2.o -o %t1
diff --git a/lld/test/ELF/weak-undef-no-dynamic-linker.s b/lld/test/ELF/weak-undef-no-dynamic-linker.s
index fa6936e1ef393..11abd5351af9d 100644
--- a/lld/test/ELF/weak-undef-no-dynamic-linker.s
+++ b/lld/test/ELF/weak-undef-no-dynamic-linker.s
@@ -1,13 +1,12 @@
 # REQUIRES: x86
 # RUN: llvm-mc -filetype=obj -triple=x86_64 %s -o %t.o
 # RUN: ld.lld -pie %t.o -o %t
-# RUN: llvm-readobj --dyn-syms %t | FileCheck %s
+# RUN: llvm-readobj --dyn-syms %t | FileCheck --check-prefix=NO %s
 # RUN: ld.lld -pie --no-dynamic-linker %t.o -o %t
 # RUN: llvm-readobj --dyn-syms %t | FileCheck --check-prefix=NO %s
 
-## With --no-dynamic-linker, don't emit undefined weak symbols to .dynsym .
-## This will suppress a relocation.
-# CHECK: Name: foo
+## With static PIE (whether or not --no-dynamic-linker is specified), don't
+## emit undefined weak symbols to .dynsym . This suppresses relocations.
 # NO-NOT: Name: foo
 
 .weak foo
diff --git a/lld/test/ELF/weak-undef-rw.s b/lld/test/ELF/weak-undef-rw.s
index ec3e4ce734895..c5cf1bdcaba68 100644
--- a/lld/test/ELF/weak-undef-rw.s
+++ b/lld/test/ELF/weak-undef-rw.s
@@ -1,23 +1,23 @@
 # REQUIRES: x86
 # RUN: llvm-mc -filetype=obj -triple=x86_64-pc-linux %s -o %t.o
 # RUN: ld.lld %t.o -o %t --export-dynamic
-# RUN: llvm-readelf -r --hex-dump=.data %t | FileCheck %s --check-prefix=NOPIC
+# RUN: llvm-readelf -r --hex-dump=.data %t | FileCheck %s --check-prefix=STATIC
 # RUN: ld.lld %t.o -o %t.pie -pie
-# RUN: llvm-readobj -r %t.pie | FileCheck %s --check-prefix=PIC
+# RUN: llvm-readelf -r --hex-dump=.data %t.pie | FileCheck %s --check-prefix=STATIC
 # RUN: ld.lld %t.o -o %t.so -shared
 # RUN: llvm-readobj -r %t.so | FileCheck %s --check-prefix=PIC
 
 ## gABI leaves the behavior of weak undefined references implementation defined.
-## We choose to resolve them statically for -no-pie and produce dynamic relocations
-## for -pie and -shared.
+## We choose to resolve them statically for static linking and produce dynamic relocations
+## for dynamic linking (-shared or at least one input DSO).
 ##
 ## Note: Some ports of GNU ld support -z nodynamic-undefined-weak that we don't
 ## implement.
 
-# NOPIC: no relocations
-# NOPIC: Hex dump of section '.data':
-# NOPIC-NEXT: {{.*}} 00000000 00000000 
-# NOPIC-EMPTY:
+# STATIC:      no relocations
+# STATIC:      Hex dump of section '.data':
+# STATIC-NEXT: {{.*}} 00000000 00000000 .
+# STATIC-EMPTY:
 
 # PIC:      .rela.dyn {
 # PIC-NEXT:   R_X86_64_64 foobar 0x0
diff --git a/lld/test/MachO/objc-category-merging-erase-objc-name-test.s b/lld/test/MachO/objc-category-merging-erase-objc-name-test.s
index 9e4046a1447e7..bc0b27d198795 100644
--- a/lld/test/MachO/objc-category-merging-erase-objc-name-test.s
+++ b/lld/test/MachO/objc-category-merging-erase-objc-name-test.s
@@ -4,9 +4,11 @@
 ; then when merging the category into the base class (and deleting the category), we don't
 ; delete the 'MyTestProtocol' name
 
-; RUN: llvm-mc -filetype=obj -triple=arm64-apple-macos -o %T/erase-objc-name.o %s
-; RUN: %lld -no_objc_relative_method_lists -arch arm64 -dylib -o %T/erase-objc-name.dylib %T/erase-objc-name.o -objc_category_merging
-; RUN: llvm-objdump --objc-meta-data --macho %T/erase-objc-name.dylib | FileCheck %s --check-prefixes=MERGE_CATS
+; RUN: mkdir -p %t.dir
+
+; RUN: llvm-mc -filetype=obj -triple=arm64-apple-macos -o %t.dir/erase-objc-name.o %s
+; RUN: %lld -no_objc_relative_method_lists -arch arm64 -dylib -o %t.dir/erase-objc-name.dylib %t.dir/erase-objc-name.o -objc_category_merging
+; RUN: llvm-objdump --objc-meta-data --macho %t.dir/erase-objc-name.dylib | FileCheck %s --check-prefixes=MERGE_CATS
 
 ; === Check merge categories enabled ===
 ; Check that the original categories are not there
diff --git a/lld/test/lit.cfg.py b/lld/test/lit.cfg.py
index 859094e2b57db..9e6b0e839d9a8 100644
--- a/lld/test/lit.cfg.py
+++ b/lld/test/lit.cfg.py
@@ -104,10 +104,8 @@
     config.environment["LLD_IN_TEST"] = "1"
 else:
     config.environment["LLD_IN_TEST"] = "2"
-    # Many ELF tests fail in this mode.
-    config.excludes.append("ELF")
-    # Some old Mach-O backend tests fail, and it's due for removal anyway.
-    config.excludes.append("mach-o")
+    # Many wasm tests fail.
+    config.excludes.append("wasm")
     # Some new Mach-O backend tests fail; give them a way to mark themselves
     # unsupported in this mode.
     config.available_features.add("main-run-twice")
diff --git a/lld/test/wasm/shared-lazy.s b/lld/test/wasm/shared-lazy.s
new file mode 100644
index 0000000000000..f1044547203a2
--- /dev/null
+++ b/lld/test/wasm/shared-lazy.s
@@ -0,0 +1,79 @@
+##  Based on lld/test/ELF/shared-lazy.s
+
+# RUN: rm -rf %t && split-file %s %t && cd %t
+# RUN: llvm-mc -filetype=obj -triple=wasm32-unknown-emscripten a.s -o a.o
+# RUN: wasm-ld a.o --experimental-pic -shared -o a.so
+# RUN: llvm-mc -filetype=obj -triple=wasm32-unknown-emscripten b.s -o b.o
+# RUN: wasm-ld b.o --experimental-pic -shared -o b.so
+# RUN: llvm-ar rc a.a a.o
+# RUN: llvm-mc -filetype=obj -triple=wasm32-unknown-emscripten ref.s -o ref.o
+# RUN: wasm-ld a.a b.so ref.o --experimental-pic -shared -o 1.so
+# RUN: obj2yaml 1.so | FileCheck %s
+# RUN: wasm-ld a.so a.a ref.o --experimental-pic -shared -o 1.so
+# RUN: obj2yaml 1.so | FileCheck %s
+
+## The definitions from a.so are used and we don't extract a member from the
+## archive.
+
+# CHECK:   - Type:            IMPORT
+# CHECK:            - Module:          GOT.mem
+# CHECK-NEXT:         Field:           x1
+# CHECK-NEXT:         Kind:            GLOBAL
+# CHECK-NEXT:         GlobalType:      I32
+# CHECK-NEXT:         GlobalMutable:   true
+# CHECK-NEXT:       - Module:          GOT.mem
+# CHECK-NEXT:         Field:           x2
+# CHECK-NEXT:         Kind:            GLOBAL
+# CHECK-NEXT:         GlobalType:      I32
+# CHECK-NEXT:         GlobalMutable:   true
+
+## The extracted x1 is defined as STB_GLOBAL.
+# RUN: wasm-ld ref.o a.a b.so -o 2.so --experimental-pic -shared
+# RUN: obj2yaml 2.so | FileCheck %s --check-prefix=CHECK2
+# RUN: wasm-ld a.a ref.o b.so -o 2.so --experimental-pic -shared
+# RUN: obj2yaml 2.so | FileCheck %s --check-prefix=CHECK2
+
+# CHECK2:       - Type:            EXPORT
+# CHECK2-NEXT:    Exports:
+# CHECK2-NEXT:      - Name:            __wasm_call_ctors
+# CHECK2-NEXT:        Kind:            FUNCTION
+# CHECK2-NEXT:        Index:
+# CHECK2-NEXT:      - Name:            x1
+# CHECK2-NEXT:        Kind:            GLOBAL
+# CHECK2-NEXT:        Index:
+# CHECK2-NEXT:      - Name:            x2
+# CHECK2-NEXT:        Kind:            GLOBAL
+
+#--- a.s
+.section .data.x1,"",@
+.global x1
+x1:
+  .byte 0
+.size x1, 1
+
+.section .data.x2,"",@
+.weak x2
+x2:
+  .byte 0
+.size x2, 1
+#--- b.s
+.section .data.x1,"",@
+.globl x1
+x1:
+  .byte 0
+.size x1, 1
+
+.section .data.x2,"",@
+.globl x2
+x2:
+  .byte 0
+.size x2, 1
+#--- ref.s
+.globl x1
+.globl x2
+.globl d
+.section .data.d,"",@
+d:
+  .int x1
+  .int x2
+.size d, 8
diff --git a/lld/test/wasm/shared-weak-undefined.s b/lld/test/wasm/shared-weak-undefined.s
index cb3e29b5a9b7b..58c2e3cd46b5a 100644
--- a/lld/test/wasm/shared-weak-undefined.s
+++ b/lld/test/wasm/shared-weak-undefined.s
@@ -7,10 +7,11 @@
 # This verifies that LazySymbols (those found in library archives) are correctly
 # demoted to undefined symbols in the final link when they are only weakly
 # referenced.
+# RUN: mkdir -p %t.dir
 # RUN: llvm-mc -filetype=obj -triple=wasm32-unknown-unknown -o %t.ret32.o %p/Inputs/ret32.s
-# RUN: rm -f %T/libret32.a
-# RUN: llvm-ar cru %T/libret32.a %t.ret32.o
-# RUN: wasm-ld --experimental-pic -shared -o %t.ret32.wasm %t.o %T/libret32.a
+# RUN: rm -f %t.dir/libret32.a
+# RUN: llvm-ar cru %t.dir/libret32.a %t.ret32.o
+# RUN: wasm-ld --experimental-pic -shared -o %t.ret32.wasm %t.o %t.dir/libret32.a
 # RUN: obj2yaml %t.wasm | FileCheck %s
 # RUN: llvm-objdump -d %t.wasm | FileCheck %s -check-prefix=ASM
 
diff --git a/lld/wasm/SymbolTable.cpp b/lld/wasm/SymbolTable.cpp
index 6309cf35fe552..7e8b4aa632a32 100644
--- a/lld/wasm/SymbolTable.cpp
+++ b/lld/wasm/SymbolTable.cpp
@@ -363,7 +363,7 @@ Symbol *SymbolTable::addSharedFunction(StringRef name, uint32_t flags,
     replaceSymbol<SharedFunctionSymbol>(sym, name, flags, file, sig);
   };
 
-  if (wasInserted) {
+  if (wasInserted || s->isLazy()) {
     replaceSym(s);
     return s;
   }
@@ -408,10 +408,18 @@ Symbol *SymbolTable::addSharedData(StringRef name, uint32_t flags,
   bool wasInserted;
   std::tie(s, wasInserted) = insert(name, file);
 
-  if (wasInserted || s->isUndefined()) {
+  if (wasInserted || s->isLazy()) {
     replaceSymbol<SharedData>(s, name, flags, file);
+    return s;
+  }
+
+  // Shared symbols should never replace locally-defined ones
+  if (s->isDefined()) {
+    return s;
   }
 
+  checkDataType(s, file);
+  replaceSymbol<SharedData>(s, name, flags, file);
   return s;
 }
 
diff --git a/lldb/bindings/interface/SBFrameDocstrings.i b/lldb/bindings/interface/SBFrameDocstrings.i
index 05a876a685a91..f6fb774111a16 100644
--- a/lldb/bindings/interface/SBFrameDocstrings.i
+++ b/lldb/bindings/interface/SBFrameDocstrings.i
@@ -78,6 +78,21 @@ See also SBThread."
     See also GetFunctionName()."
 ) lldb::SBFrame::IsInlined;
 
+%feature("docstring", "
+    Returns an SBValueList which is an array of one or more register
+    sets that exist for this thread.  
+    Each SBValue in the SBValueList represents one register-set. 
+    The first register-set will be the general purpose registers -- 
+    the registers printed by the `register read` command-line in lldb, with 
+    no additional arguments.  
+    The register-set SBValue will have a name, e.g. 
+    SBFrame::GetRegisters().GetValueAtIndex(0).GetName() 
+    By convention, certain stubs choose to name their general-purpose 
+    register-set the 'General Purpose Registers', but that is not required.
+    A register-set SBValue will have children, one child per register 
+    in the register-set."
+) lldb::SBFrame::GetRegisters;
+
 %feature("docstring", "
     Return true if this frame is artificial (e.g a frame synthesized to
     capture a tail call). Local variables may not be available in an artificial
diff --git a/lldb/bindings/interface/SBFrameExtensions.i b/lldb/bindings/interface/SBFrameExtensions.i
index e0472280666ab..38d03abaee8f0 100644
--- a/lldb/bindings/interface/SBFrameExtensions.i
+++ b/lldb/bindings/interface/SBFrameExtensions.i
@@ -87,8 +87,8 @@ STRING_EXTENSION_OUTSIDE(SBFrame)
         args = property(get_arguments, None, doc='''A read only property that returns a list() that contains a collection of lldb.SBValue objects that represent the argument variables in this stack frame.''')
         arguments = property(get_arguments, None, doc='''A read only property that returns a list() that contains a collection of lldb.SBValue objects that represent the argument variables in this stack frame.''')
         statics = property(get_statics, None, doc='''A read only property that returns a list() that contains a collection of lldb.SBValue objects that represent the static variables in this stack frame.''')
-        registers = property(GetRegisters, None, doc='''A read only property that returns a list() that contains a collection of lldb.SBValue objects that represent the CPU registers for this stack frame.''')
-        regs = property(GetRegisters, None, doc='''A read only property that returns a list() that contains a collection of lldb.SBValue objects that represent the CPU registers for this stack frame.''')
+        registers = property(GetRegisters, None, doc='''Returns the register sets for this thread as a list().  See SBFrame::GetRegisters() for details.''')
+        regs = property(GetRegisters, None, doc='''Returns the register sets for this thread as a list().  See SBFrame::GetRegisters() for details.''')
         register = property(get_registers_access, None, doc='''A read only property that returns an helper object providing a flattened indexable view of the CPU registers for this stack frame.''')
         reg = property(get_registers_access, None, doc='''A read only property that returns an helper object providing a flattened indexable view of the CPU registers for this stack frame''')
         parent = property(get_parent_frame, None, doc='''A read only property that returns the parent (caller) frame of the current frame.''')
diff --git a/lldb/bindings/interface/SBProcessDocstrings.i b/lldb/bindings/interface/SBProcessDocstrings.i
index 1b98a79e4f6d3..b5b22052a6094 100644
--- a/lldb/bindings/interface/SBProcessDocstrings.i
+++ b/lldb/bindings/interface/SBProcessDocstrings.i
@@ -258,3 +258,14 @@ SBProcess supports thread iteration. For example (from test/lldbutil.py), ::
     Deallocates the block of memory (previously allocated using
     AllocateMemory) given in the argument."
 ) lldb::SBProcess::DeallocateMemory;
+
+%feature("docstring", "
+    Get a list of all the memory regions associated with this process.
+    ```
+        readable_regions = []
+        for region in process.GetMemoryRegions():
+            if region.IsReadable():
+                readable_regions.append(region)
+    ```
+"
+) lldb::SBProcess::GetMemoryRegions;
diff --git a/lldb/bindings/interface/SBSaveCoreOptionsDocstrings.i b/lldb/bindings/interface/SBSaveCoreOptionsDocstrings.i
index e69de29bb2d1d..08bbdf89d68de 100644
--- a/lldb/bindings/interface/SBSaveCoreOptionsDocstrings.i
+++ b/lldb/bindings/interface/SBSaveCoreOptionsDocstrings.i
@@ -0,0 +1,71 @@
+%feature("docstring",
+"A container to specify how to save a core file.
+
+SBSaveCoreOptions includes API's to specify the memory regions and threads to include
+when generating a core file. It extends the existing SaveCoreStyle option.
+
+* eSaveCoreFull will save off all thread and memory regions, ignoring the memory regions and threads in
+the options object.
+
+* eSaveCoreDirtyOnly pages will capture all threads and all rw- memory regions, in addition to the regions specified
+in the options object if they are not already captured.
+
+* eSaveCoreStackOnly will capture all threads, but no memory regions unless specified.
+
+* eSaveCoreCustomOnly Custom defers entirely to the SBSaveCoreOptions object and will only save what is specified. 
+  Picking custom and specifying nothing will result in an error being returned.
+
+Note that currently ELF Core files are not supported."
+) lldb::SBSaveCoreOptions;
+
+%feature("docstring", "
+    Set the plugin name to save a Core file with. Only plugins registered with Plugin manager will be accepted
+    Examples are Minidump and Mach-O."
+) lldb::SBSaveCoreOptions::SetPluginName;
+
+%feature("docstring", "
+    Get the specified plugin name, or None if the name is not set."
+) lldb::SBSaveCoreOptions::GetPluginName;
+
+%feature("docstring", "
+    Set the lldb.SaveCoreStyle."
+) lldb::SBSaveCoreOptions::SetStyle;
+
+%feature("docstring", "
+    Get the specified lldb.SaveCoreStyle, or eSaveCoreUnspecified if not set."
+) lldb::SBSaveCoreOptions::GetStyle;
+
+%feature("docstring", "
+    Set the file path to save the Core file at."
+) lldb::SBSaveCoreOptions::SetOutputFile;
+
+%feature("docstring", "
+    Get an SBFileSpec corresponding to the specified output path, or none if not set."
+) lldb::SBSaveCoreOptions::GetOutputFile;
+
+%feature("docstring", "
+    Set the process to save, or unset a process by providing a default SBProcess. 
+    Resetting will result in the reset of all process specific options, such as Threads to save."
+) lldb::SBSaveCoreOptions::SetProcess;
+
+%feature("docstring", "
+    Add an SBThread to be saved, an error will be returned if an SBThread from a different process is specified. 
+    The process is set either by the first SBThread added to the options container, or explicitly by the SetProcess call."
+) lldb::SBSaveCoreOptions::AddThread;
+
+%feature("docstring", "
+    Remove an SBthread if present in the container, returns true if a matching thread was found and removed."
+) lldb::SBSaveCoreOptions::RemoveThread;
+
+%feature("docstring", "
+    Add a memory region to save, an error will be returned in the region is invalid. 
+    Ranges that overlap will be unioned into a single region."
+) lldb::SBSaveCoreOptions::AddMemoryRegionToSave;
+
+%feature("docstring", "
+    Get an SBThreadCollection of all threads marked to be saved. This collection is not sorted according to insertion order."
+) lldb::SBSaveCoreOptions::GetThreadsToSave;
+
+%feature("docstring", "
+    Unset all options."
+) lldb::SBSaveCoreOptions::Clear;
diff --git a/lldb/bindings/python/python-swigsafecast.swig b/lldb/bindings/python/python-swigsafecast.swig
index 429baad158ca5..4721dfdc17e6a 100644
--- a/lldb/bindings/python/python-swigsafecast.swig
+++ b/lldb/bindings/python/python-swigsafecast.swig
@@ -9,6 +9,10 @@ PythonObject SWIGBridge::ToSWIGWrapper(std::unique_ptr<lldb::SBValue> value_sb)
   return ToSWIGHelper(value_sb.release(), SWIGTYPE_p_lldb__SBValue);
 }
 
+PythonObject SWIGBridge::ToSWIGWrapper(std::unique_ptr<lldb::SBCommandReturnObject> result_up) {
+  return ToSWIGHelper(result_up.release(), SWIGTYPE_p_lldb__SBCommandReturnObject);
+}
+
 PythonObject SWIGBridge::ToSWIGWrapper(lldb::ValueObjectSP value_sp) {
   return ToSWIGWrapper(std::unique_ptr<lldb::SBValue>(new lldb::SBValue(value_sp)));
 }
diff --git a/lldb/bindings/python/python-typemaps.swig b/lldb/bindings/python/python-typemaps.swig
index f8c33e15c03e6..88b6cd9ef6b6e 100644
--- a/lldb/bindings/python/python-typemaps.swig
+++ b/lldb/bindings/python/python-typemaps.swig
@@ -476,6 +476,25 @@ template <> bool SetNumberFromPyObject<double>(double &number, PyObject *obj) {
   $1 = $1 || PyCallable_Check(reinterpret_cast<PyObject *>($input));
 }
 
+// For lldb::SBCommandPrintCallback
+%typemap(in) (lldb::SBCommandPrintCallback callback, void *baton) {
+  if (!($input == Py_None ||
+        PyCallable_Check(reinterpret_cast<PyObject *>($input)))) {
+    PyErr_SetString(PyExc_TypeError, "Need a callable object or None!");
+    SWIG_fail;
+  }
+
+  // Don't lose the callback reference.
+  Py_INCREF($input);
+  $1 = LLDBSwigPythonCallPythonCommandPrintCallback;
+  $2 = $input;
+}
+
+%typemap(typecheck) (lldb::SBCommandPrintCallback callback, void *baton) {
+  $1 = $input == Py_None;
+  $1 = $1 || PyCallable_Check(reinterpret_cast<PyObject *>($input));
+}
+
 %typemap(in) (lldb::CommandOverrideCallback callback, void *baton) {
   if (!($input == Py_None ||
         PyCallable_Check(reinterpret_cast<PyObject *>($input)))) {
diff --git a/lldb/bindings/python/python-wrapper.swig b/lldb/bindings/python/python-wrapper.swig
index b72a462d04643..57c7ac387145e 100644
--- a/lldb/bindings/python/python-wrapper.swig
+++ b/lldb/bindings/python/python-wrapper.swig
@@ -727,7 +727,7 @@ lldb_private::python::SWIGBridge::LLDBSwigPythonHandleOptionArgumentCompletionFo
     dict_sp->AddBooleanItem("no-completion", true);
     return dict_sp;
   }
-    
+
 
   // Convert the return dictionary to a DictionarySP.
   StructuredData::ObjectSP result_obj_sp = result.CreateStructuredObject();
@@ -753,7 +753,7 @@ bool lldb_private::python::SWIGBridge::LLDBSwigPythonCallParsedCommandObject(
   auto pfunc = self.ResolveName<PythonCallable>("__call__");
 
   if (!pfunc.IsAllocated()) {
-    cmd_retobj.AppendError("Could not find '__call__' method in implementation class"); 
+    cmd_retobj.AppendError("Could not find '__call__' method in implementation class");
     return false;
   }
 
@@ -1012,6 +1012,26 @@ static void LLDBSwigPythonCallPythonLogOutputCallback(const char *str,
   }
 }
 
+// For CommandPrintCallback functions
+static CommandReturnObjectCallbackResult LLDBSwigPythonCallPythonCommandPrintCallback(SBCommandReturnObject& result, void *callback_baton) {
+  SWIG_Python_Thread_Block swig_thread_block;
+
+  PyErr_Cleaner py_err_cleaner(true);
+
+  PythonObject result_arg = SWIGBridge::ToSWIGWrapper(
+      std::make_unique<SBCommandReturnObject>(result));
+  PythonCallable callable =
+      Retain<PythonCallable>(reinterpret_cast<PyObject *>(callback_baton));
+
+  if (!callable.IsValid())
+    return eCommandReturnObjectPrintCallbackSkipped;
+
+  PythonObject callback_result = callable(result_arg);
+
+  long long ret_val = unwrapOrSetPythonException(As<long long>(callback_result));
+  return (CommandReturnObjectCallbackResult)ret_val;
+}
+
 // For DebuggerTerminateCallback functions
 static void LLDBSwigPythonCallPythonSBDebuggerTerminateCallback(lldb::user_id_t debugger_id,
                                                       void *baton) {
diff --git a/lldb/docs/resources/build.rst b/lldb/docs/resources/build.rst
index c0d3c580d631c..214c5f63f2c73 100644
--- a/lldb/docs/resources/build.rst
+++ b/lldb/docs/resources/build.rst
@@ -62,7 +62,7 @@ CMake configuration error.
 +-------------------+--------------------------------------------------------------+--------------------------+
 | Libxml2           | XML                                                          | ``LLDB_ENABLE_LIBXML2``  |
 +-------------------+--------------------------------------------------------------+--------------------------+
-| Python            | Python scripting. >= 3.0 is required, >= 3.8 is recommended. | ``LLDB_ENABLE_PYTHON``   |
+| Python            | Python scripting. >= 3.8 is required.                        | ``LLDB_ENABLE_PYTHON``   |
 +-------------------+--------------------------------------------------------------+--------------------------+
 | Lua               | Lua scripting. Lua 5.3 and 5.4 are supported.                | ``LLDB_ENABLE_LUA``      |
 +-------------------+--------------------------------------------------------------+--------------------------+
diff --git a/lldb/docs/resources/lldbgdbremote.md b/lldb/docs/resources/lldbgdbremote.md
index 5cac3736337a8..162815148a18b 100644
--- a/lldb/docs/resources/lldbgdbremote.md
+++ b/lldb/docs/resources/lldbgdbremote.md
@@ -2412,6 +2412,11 @@ value of errno if unlink failed.
 
 ## "x" - Binary memory read
 
+> **Warning:** The format of this packet was decided before GDB 16
+> introduced its own format for `x`. Future versions of LLDB may not
+> support the format described here, and new code should produce and
+> expect the format used by GDB.
+
 Like the `m` (read) and `M` (write) packets, this is a partner to the
 `X` (write binary data) packet, `x`.
 
diff --git a/lldb/docs/resources/test.rst b/lldb/docs/resources/test.rst
index 5f1bd0d573830..1272aeeeb969e 100644
--- a/lldb/docs/resources/test.rst
+++ b/lldb/docs/resources/test.rst
@@ -586,24 +586,60 @@ To run a specific test, pass a filter, for example:
 Running the Test Suite Remotely
 ```````````````````````````````
 
-Running the test-suite remotely is similar to the process of running a local
-test suite, but there are two things to have in mind:
-
-1. You must have the lldb-server running on the remote system, ready to accept
-   multiple connections. For more information on how to setup remote debugging
-   see the Remote debugging page.
-2. You must tell the test-suite how to connect to the remote system. This is
-   achieved using the ``LLDB_TEST_PLATFORM_URL``, ``LLDB_TEST_PLATFORM_WORKING_DIR``
-   flags to cmake, and ``--platform-name`` parameter to ``dotest.py``.
-   These parameters correspond to the platform select and platform connect
-   LLDB commands. You will usually also need to specify the compiler and
-   architecture for the remote system.
-3. Remote Shell tests execution is currently supported only for Linux target
-   platform. It's triggered when ``LLDB_TEST_SYSROOT`` is provided for building
-   test sources. It can be disabled by setting ``LLDB_TEST_SHELL_DISABLE_REMOTE=On``.
-   Shell tests are not guaranteed to pass against remote target if the compiler
-   being used is other than Clang.
+1. Run lldb-server on the remote system, so that it can accept multiple connections.
+   This is called "platform" mode:
 
+   ::
+
+      lldb-server platform --server --listen 0.0.0.0:<port A> --gdbserver-port <port B>
+
+   Assuming that ``port A`` and ``port B`` on the remote system can be reached
+   from your host system. If your remote system is a simulator on your host machine,
+   you may need to forward these ports to the host when you start the simulator.
+
+   For more information on how to setup remote debugging see :doc:`/use/remote`.
+
+2. Tell the test-suite how to connect to the remote system. This is done using the
+   ``LLDB_TEST_PLATFORM_URL`` and ``LLDB_TEST_PLATFORM_WORKING_DIR`` flags of CMake,
+   or the ``--platform-name``, ``--platform-url`` and ``--platform-working-dir``
+   parameters of ``dotest.py``. These parameters are passed on to the ``platform select``
+   and ``platform connect`` LLDB commands when the tests are run.
+
+   You will usually need to specify the compiler and architecture for the
+   remote system. This is done with CMake options ``LLDB_TEST_COMPILER`` and
+   ``LLDB_TEST_ARCH``, or the ``dotest.py`` options ``--compiler`` and ``--arch``.
+
+   .. note::
+      Even in cases where the two systems are the same architecture and run the
+      same operating system, there may be version differences between the two
+      which require you to use a different compiler version for remote testing.
+
+   For example, to run tests using ``dotest.py`` on a remote AArch64 Linux system
+   you might run:
+
+   ::
+
+      ./bin/lldb-dotest --platform-name remote-linux --arch aarch64 --compiler aarch64-none-linux-gnu-gcc --platform-url connect://<remote-ip>:<port A> --platform-working-dir /tmp/test_lldb -p <test-name>.py
+
+   This is the equivalent of:
+
+      * ``LLDB_TEST_ARCH`` = ``aarch64``
+      * ``LLDB_TEST_COMPILER`` = ``aarch64-none-linux-gnu-gcc``
+      * ``LLDB_TEST_PLATFORM_URL`` = ``connect://<remote-ip>:<port A>``
+      * ``LLDB_TEST_PLATFORM_WORKING_DIR`` = ``/tmp/test_lldb``
+
+   Setting these values using CMake allows you to run ``ninja check-lldb`` to run
+   tests on the remote system.
+
+   If you have a host build that you sometimes check on a remote system, but otherwise
+   test on the host, adding arguments to ``dotest.py`` manually is easier.
+
+.. note::
+   Remote Shell test execution is currently supported only for Linux targets.
+   It is enabled when ``LLDB_TEST_SYSROOT`` is set. Remote Shell testing can
+   be disabled by setting ``LLDB_TEST_SHELL_DISABLE_REMOTE=On``. Shell tests
+   are not guaranteed to pass against remote target if the test compiler is not
+   Clang.
 
 Running tests in QEMU System Emulation Environment
 ``````````````````````````````````````````````````
diff --git a/lldb/docs/use/aarch64-linux.md b/lldb/docs/use/aarch64-linux.md
index 393838dc0bb4f..164d25cef6721 100644
--- a/lldb/docs/use/aarch64-linux.md
+++ b/lldb/docs/use/aarch64-linux.md
@@ -229,3 +229,64 @@ bytes.
 
 `zt0`'s value and whether it is active or not will be saved prior to
 expression evaluation and restored afterwards.
+
+## Guarded Control Stack Extension (GCS)
+
+GCS support includes the following new registers:
+
+* `gcs_features_enabled`
+* `gcs_features_locked`
+* `gcspr_el0`
+
+These map to the registers ptrace provides. The first two have a `gcs_`
+prefix added as their names are too generic without it.
+
+When the GCS is enabled the kernel allocates a memory region for it. This region
+has a special attribute that LLDB will detect and presents like this:
+```
+  (lldb) memory region --all
+  <...>
+  [0x0000fffff7a00000-0x0000fffff7e00000) rw-
+  shadow stack: yes
+  [0x0000fffff7e00000-0x0000fffff7e10000) ---
+```
+
+`shadow stack` is a generic term used in the kernel for secure stack
+extensions like GCS.
+
+### Expression Evaluation
+
+To execute an expression when GCS is enabled, LLDB must push the return
+address of the expression wrapper (usually the entry point of the program)
+to the Guarded Control Stack. It does this by decrementing `gcspr_el0` and
+writing to the location now pointed to by `gcspr_el0` (instead of using the
+GCS push instructions).
+
+After an expression finishes, LLDB will restore the contents of all 3
+GCS registers, apart from the enable bit of `gcs_features_enabled`. This is
+because there are limits on how often and from where you can set this
+bit.
+
+GCS cannot be enabled from ptrace and it is expected that a process which
+has enabled and then disabled GCS, will not enable it again. The simplest
+choice was to not restore the enable bit at all. It is up to the user or
+program to manage that bit.
+
+The return address that LLDB pushed onto the Guarded Control Stack will be left
+in place. As will any values that were pushed to the stack by functions run
+during the expression.
+
+When the process resumes, `gcspr_el0` will be pointing to the original entry
+on the guarded control stack. So the other values will have no effect and
+likely be overwritten by future function calls.
+
+LLDB does not track and restore changes to general memory during expressions,
+so not restoring the GCS contents fits with the current behaviour.
+
+Note that if GCS is disabled and an expression enables it, LLDB will not
+be able to setup the return address and it is up to that expression to do that
+if it wants to return to LLDB correctly.
+
+If it does not do this, the expression will fail and although most process
+state will be restored, GCS will be left enabled. Which means that the program
+is very unlikely to be able to progress.
diff --git a/lldb/docs/use/variable.rst b/lldb/docs/use/variable.rst
index 22c2bee73fa59..3ad71cb93c51d 100644
--- a/lldb/docs/use/variable.rst
+++ b/lldb/docs/use/variable.rst
@@ -1226,13 +1226,13 @@ By default, several categories are created in LLDB:
 - CoreServices: CS classes
 - VectorTypes: compact display for several vector types
 
-If you want to use a custom category for your formatters, all the type ... add
-provide a --category (-w) option, that names the category to add the formatter
+If you want to use a custom category for your formatters, all the ``type ... add``
+provide a ``--category`` (``-w``) option, that names the category to add the formatter
 to. To delete the formatter, you then have to specify the correct category.
 
 Categories can be in one of two states: enabled and disabled. A category is
-initially disabled, and can be enabled using the type category enable command.
-To disable an enabled category, the command to use is type category disable.
+initially disabled, and can be enabled using the ``type category enable`` command.
+To disable an enabled category, the command to use is ``type category disable``.
 
 The order in which categories are enabled or disabled is significant, in that
 LLDB uses that order when looking for formatters. Therefore, when you enable a
diff --git a/lldb/include/lldb/API/LLDB.h b/lldb/include/lldb/API/LLDB.h
index 40368e036e0e4..126fcef31b416 100644
--- a/lldb/include/lldb/API/LLDB.h
+++ b/lldb/include/lldb/API/LLDB.h
@@ -54,6 +54,7 @@
 #include "lldb/API/SBProcess.h"
 #include "lldb/API/SBProcessInfo.h"
 #include "lldb/API/SBProcessInfoList.h"
+#include "lldb/API/SBProgress.h"
 #include "lldb/API/SBQueue.h"
 #include "lldb/API/SBQueueItem.h"
 #include "lldb/API/SBReproducer.h"
diff --git a/lldb/include/lldb/API/SBCommandInterpreter.h b/lldb/include/lldb/API/SBCommandInterpreter.h
index b7e39b7858616..dd475ddf6c170 100644
--- a/lldb/include/lldb/API/SBCommandInterpreter.h
+++ b/lldb/include/lldb/API/SBCommandInterpreter.h
@@ -247,13 +247,13 @@ class SBCommandInterpreter {
                                        lldb::SBStringList &matches,
                                        lldb::SBStringList &descriptions);
 
-  /// Returns whether an interrupt flag was raised either by the SBDebugger - 
+  /// Returns whether an interrupt flag was raised either by the SBDebugger -
   /// when the function is not running on the RunCommandInterpreter thread, or
   /// by SBCommandInterpreter::InterruptCommand if it is.  If your code is doing
-  /// interruptible work, check this API periodically, and interrupt if it 
+  /// interruptible work, check this API periodically, and interrupt if it
   /// returns true.
   bool WasInterrupted() const;
-  
+
   /// Interrupts the command currently executing in the RunCommandInterpreter
   /// thread.
   ///
@@ -331,6 +331,8 @@ class SBCommandInterpreter {
   /// this list. Otherwise this list is empty.
   SBStructuredData GetTranscript();
 
+  void SetPrintCallback(lldb::SBCommandPrintCallback callback, void *baton);
+
 protected:
   friend class lldb_private::CommandPluginInterfaceImplementation;
 
diff --git a/lldb/include/lldb/API/SBCommandReturnObject.h b/lldb/include/lldb/API/SBCommandReturnObject.h
index e8e20a3f3016b..9a63c1f96aa70 100644
--- a/lldb/include/lldb/API/SBCommandReturnObject.h
+++ b/lldb/include/lldb/API/SBCommandReturnObject.h
@@ -42,6 +42,10 @@ class LLDB_API SBCommandReturnObject {
 
   bool IsValid() const;
 
+  /// Get the command as the user typed it. Empty string if commands were run on
+  /// behalf of lldb.
+  const char *GetCommand();
+
   const char *GetOutput();
 
   const char *GetError();
diff --git a/lldb/include/lldb/API/SBDefines.h b/lldb/include/lldb/API/SBDefines.h
index 159a9ba799b18..ed5a80da117a5 100644
--- a/lldb/include/lldb/API/SBDefines.h
+++ b/lldb/include/lldb/API/SBDefines.h
@@ -95,6 +95,7 @@ class LLDB_API SBPlatformShellCommand;
 class LLDB_API SBProcess;
 class LLDB_API SBProcessInfo;
 class LLDB_API SBProcessInfoList;
+class LLDB_API SBProgress;
 class LLDB_API SBQueue;
 class LLDB_API SBQueueItem;
 class LLDB_API SBReplayOptions;
@@ -143,6 +144,9 @@ typedef bool (*SBBreakpointHitCallback)(void *baton, lldb::SBProcess &process,
 typedef void (*SBDebuggerDestroyCallback)(lldb::user_id_t debugger_id,
                                           void *baton);
 
+typedef lldb::CommandReturnObjectCallbackResult (*SBCommandPrintCallback)(
+    lldb::SBCommandReturnObject &result, void *baton);
+
 typedef lldb::SBError (*SBPlatformLocateModuleCallback)(
     void *baton, const lldb::SBModuleSpec &module_spec,
     lldb::SBFileSpec &module_file_spec, lldb::SBFileSpec &symbol_file_spec);
diff --git a/lldb/include/lldb/API/SBProgress.h b/lldb/include/lldb/API/SBProgress.h
index d2eaf0a743cb3..d574d1d2982b9 100644
--- a/lldb/include/lldb/API/SBProgress.h
+++ b/lldb/include/lldb/API/SBProgress.h
@@ -51,6 +51,10 @@ class LLDB_API SBProgress {
   SBProgress(const char *title, const char *details, uint64_t total_units,
              SBDebugger &debugger);
 
+#ifndef SWIG
+  SBProgress(SBProgress &&rhs);
+#endif
+
   ~SBProgress();
 
   void Increment(uint64_t amount, const char *description = nullptr);
@@ -59,6 +63,9 @@ class LLDB_API SBProgress {
   lldb_private::Progress &ref() const;
 
 private:
+  SBProgress(const SBProgress &rhs) = delete;
+  const SBProgress &operator=(const SBProgress &rhs) = delete;
+
   std::unique_ptr<lldb_private::Progress> m_opaque_up;
 }; // SBProgress
 } // namespace lldb
diff --git a/lldb/include/lldb/Core/Telemetry.h b/lldb/include/lldb/Core/Telemetry.h
new file mode 100644
index 0000000000000..60a7097de5eee
--- /dev/null
+++ b/lldb/include/lldb/Core/Telemetry.h
@@ -0,0 +1,74 @@
+//===-- Telemetry.h -------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLDB_CORE_TELEMETRY_H
+#define LLDB_CORE_TELEMETRY_H
+
+#include "lldb/Core/StructuredDataImpl.h"
+#include "lldb/Interpreter/CommandReturnObject.h"
+#include "lldb/Utility/StructuredData.h"
+#include "lldb/lldb-forward.h"
+#include "llvm/ADT/StringExtras.h"
+#include "llvm/ADT/StringRef.h"
+#include "llvm/Support/JSON.h"
+#include "llvm/Telemetry/Telemetry.h"
+#include <chrono>
+#include <ctime>
+#include <memory>
+#include <optional>
+#include <string>
+#include <unordered_map>
+
+namespace lldb_private {
+namespace telemetry {
+
+struct LLDBEntryKind : public ::llvm::telemetry::EntryKind {
+  static const llvm::telemetry::KindType BaseInfo = 0b11000;
+};
+
+/// Defines a convenient type for timestamp of various events.
+using SteadyTimePoint = std::chrono::time_point<std::chrono::steady_clock,
+                                                std::chrono::nanoseconds>;
+struct LLDBBaseTelemetryInfo : public llvm::telemetry::TelemetryInfo {
+  /// Start time of an event
+  SteadyTimePoint start_time;
+  /// End time of an event - may be empty if not meaningful.
+  std::optional<SteadyTimePoint> end_time;
+  // TBD: could add some memory stats here too?
+
+  Debugger *debugger;
+
+  // For dyn_cast, isa, etc operations.
+  llvm::telemetry::KindType getKind() const override {
+    return LLDBEntryKind::BaseInfo;
+  }
+
+  static bool classof(const llvm::telemetry::TelemetryInfo *t) {
+    // Subclasses of this is also acceptable.
+    return (t->getKind() & LLDBEntryKind::BaseInfo) == LLDBEntryKind::BaseInfo;
+  }
+
+  void serialize(llvm::telemetry::Serializer &serializer) const override;
+};
+
+/// The base Telemetry manager instance in LLDB
+/// This class declares additional instrumentation points
+/// applicable to LLDB.
+class TelemetryManager : public llvm::telemetry::Manager {
+public:
+  TelemetryManager(std::unique_ptr<llvm::telemetry::Config> config);
+
+  llvm::Error preDispatch(llvm::telemetry::TelemetryInfo *entry) override;
+
+private:
+  std::unique_ptr<llvm::telemetry::Config> m_config;
+};
+
+} // namespace telemetry
+} // namespace lldb_private
+#endif // LLDB_CORE_TELEMETRY_H
diff --git a/lldb/include/lldb/Core/UserSettingsController.h b/lldb/include/lldb/Core/UserSettingsController.h
index 32da7e05f7040..29e892fdba45b 100644
--- a/lldb/include/lldb/Core/UserSettingsController.h
+++ b/lldb/include/lldb/Core/UserSettingsController.h
@@ -38,9 +38,7 @@ class Properties {
 
   virtual ~Properties();
 
-  virtual lldb::OptionValuePropertiesSP GetValueProperties() const {
-    // This function is virtual in case subclasses want to lazily implement
-    // creating the properties.
+  lldb::OptionValuePropertiesSP GetValueProperties() const {
     return m_collection_sp;
   }
 
diff --git a/lldb/include/lldb/Core/Value.h b/lldb/include/lldb/Core/Value.h
index d0c338ffec0cd..3714621b469ec 100644
--- a/lldb/include/lldb/Core/Value.h
+++ b/lldb/include/lldb/Core/Value.h
@@ -109,8 +109,10 @@ class Value {
 
   Scalar &ResolveValue(ExecutionContext *exe_ctx, Module *module = nullptr);
 
+  /// See comment on m_scalar to understand what GetScalar returns.
   const Scalar &GetScalar() const { return m_value; }
 
+  /// See comment on m_scalar to understand what GetScalar returns.
   Scalar &GetScalar() { return m_value; }
 
   size_t ResizeData(size_t len);
@@ -148,6 +150,32 @@ class Value {
   static ValueType GetValueTypeFromAddressType(AddressType address_type);
 
 protected:
+  /// Represents a value, which can be a scalar, a load address, a file address,
+  /// or a host address.
+  ///
+  /// The interpretation of `m_value` depends on `m_value_type`:
+  /// - Scalar: `m_value` contains the scalar value.
+  /// - Load Address: `m_value` contains the load address.
+  /// - File Address: `m_value` contains the file address.
+  /// - Host Address: `m_value` contains a pointer to the start of the buffer in
+  ///    host memory.
+  ///   Currently, this can point to either:
+  ///     - The `m_data_buffer` of this Value instance (e.g., in DWARF
+  ///     computations).
+  ///     - The `m_data` of a Value Object containing this Value.
+  // TODO: the GetScalar() API relies on knowledge not codified by the type
+  // system, making it hard to understand and easy to misuse.
+  // - Separate the scalar from the variable that contains the address (be it a
+  //   load, file or host address).
+  // - Rename GetScalar() to something more indicative to what the scalar is,
+  //   like GetScalarOrAddress() for example.
+  // - Split GetScalar() into two functions, GetScalar() and GetAddress(), which
+  //   verify (or assert) what m_value_type is to make sure users of the class are
+  //   querying the right thing.
+  // TODO: It's confusing to point to multiple possible buffers when the
+  // ValueType is a host address. Value should probably always own its buffer.
+  // Perhaps as a shared pointer with a copy on write system if the same buffer
+  // can be shared by multiple classes.
   Scalar m_value;
   CompilerType m_compiler_type;
   void *m_context = nullptr;
diff --git a/lldb/include/lldb/Expression/ExpressionVariable.h b/lldb/include/lldb/Expression/ExpressionVariable.h
index fc36793b3a475..f5bd938921966 100644
--- a/lldb/include/lldb/Expression/ExpressionVariable.h
+++ b/lldb/include/lldb/Expression/ExpressionVariable.h
@@ -107,9 +107,18 @@ class ExpressionVariable
 
   FlagType m_flags; // takes elements of Flags
 
-  // these should be private
+  /// These members should be private.
+  /// @{
+  /// A value object whose value's data lives in host (lldb's) memory.
   lldb::ValueObjectSP m_frozen_sp;
+  /// The ValueObject counterpart to m_frozen_sp that tracks the value in
+  /// inferior memory. This object may not always exist; its presence depends on
+  /// whether it is logical for the value to exist in the inferior memory. For
+  /// example, when evaluating a C++ expression that generates an r-value, such
+  /// as a single function call, there is no memory address in the inferior to
+  /// track.
   lldb::ValueObjectSP m_live_sp;
+  /// @}
 };
 
 /// \class ExpressionVariableList ExpressionVariable.h
diff --git a/lldb/include/lldb/Host/Host.h b/lldb/include/lldb/Host/Host.h
index d8113a5fceead..4e19d15b06743 100644
--- a/lldb/include/lldb/Host/Host.h
+++ b/lldb/include/lldb/Host/Host.h
@@ -259,6 +259,8 @@ class Host {
                                               const FileSpec &file_spec,
                                               uint32_t line_no);
 
+  static llvm::Error OpenURL(llvm::StringRef url);
+
   /// Check if we're running in an interactive graphical session.
   ///
   /// \return
diff --git a/lldb/include/lldb/Interpreter/CommandInterpreter.h b/lldb/include/lldb/Interpreter/CommandInterpreter.h
index 2bafc30cc8e23..d5425788fe1c9 100644
--- a/lldb/include/lldb/Interpreter/CommandInterpreter.h
+++ b/lldb/include/lldb/Interpreter/CommandInterpreter.h
@@ -16,6 +16,7 @@
 #include "lldb/Interpreter/CommandObject.h"
 #include "lldb/Interpreter/ScriptInterpreter.h"
 #include "lldb/Utility/Args.h"
+#include "lldb/Utility/Baton.h"
 #include "lldb/Utility/Broadcaster.h"
 #include "lldb/Utility/CompletionRequest.h"
 #include "lldb/Utility/Event.h"
@@ -100,8 +101,7 @@ class CommandInterpreterRunOptions {
                                LazyBool stop_on_error, LazyBool stop_on_crash,
                                LazyBool echo_commands, LazyBool echo_comments,
                                LazyBool print_results, LazyBool print_errors,
-                               LazyBool add_to_history,
-                               LazyBool handle_repeats)
+                               LazyBool add_to_history, LazyBool handle_repeats)
       : m_stop_on_continue(stop_on_continue), m_stop_on_error(stop_on_error),
         m_stop_on_crash(stop_on_crash), m_echo_commands(echo_commands),
         m_echo_comment_commands(echo_comments), m_print_results(print_results),
@@ -248,13 +248,17 @@ class CommandInterpreter : public Broadcaster,
   enum CommandTypes {
     eCommandTypesBuiltin = 0x0001, //< native commands such as "frame"
     eCommandTypesUserDef = 0x0002, //< scripted commands
-    eCommandTypesUserMW  = 0x0004, //< multiword commands (command containers)
+    eCommandTypesUserMW = 0x0004,  //< multiword commands (command containers)
     eCommandTypesAliases = 0x0008, //< aliases such as "po"
-    eCommandTypesHidden  = 0x0010, //< commands prefixed with an underscore
+    eCommandTypesHidden = 0x0010,  //< commands prefixed with an underscore
     eCommandTypesAllThem = 0xFFFF  //< all commands
   };
 
-  // The CommandAlias and CommandInterpreter both have a hand in 
+  using CommandReturnObjectCallback =
+      std::function<lldb::CommandReturnObjectCallbackResult(
+          CommandReturnObject &)>;
+
+  // The CommandAlias and CommandInterpreter both have a hand in
   // substituting for alias commands.  They work by writing special tokens
   // in the template form of the Alias command, and then detecting them when the
   // command is executed.  These are the special tokens:
@@ -334,9 +338,8 @@ class CommandInterpreter : public Broadcaster,
   ///         dummy "contains everything MWC, so we return null here, but
   ///         in this case error.Success is true.
 
-  CommandObjectMultiword *VerifyUserMultiwordCmdPath(Args &path,
-                                                     bool leaf_is_command,
-                                                     Status &result);
+  CommandObjectMultiword *
+  VerifyUserMultiwordCmdPath(Args &path, bool leaf_is_command, Status &result);
 
   CommandAlias *AddAlias(llvm::StringRef alias_name,
                          lldb::CommandObjectSP &command_obj_sp,
@@ -596,7 +599,7 @@ class CommandInterpreter : public Broadcaster,
   void SetEchoCommentCommands(bool enable);
 
   bool GetRepeatPreviousCommand() const;
-  
+
   bool GetRequireCommandOverwrite() const;
 
   const CommandObject::CommandMap &GetUserCommands() const {
@@ -666,6 +669,8 @@ class CommandInterpreter : public Broadcaster,
     ++m_command_usages[cmd_obj.GetCommandName()];
   }
 
+  void SetPrintCallback(CommandReturnObjectCallback callback);
+
   llvm::json::Value GetStatistics();
   const StructuredData::Array &GetTranscript() const;
 
@@ -776,6 +781,9 @@ class CommandInterpreter : public Broadcaster,
   std::vector<uint32_t> m_command_source_flags;
   CommandInterpreterRunResult m_result;
 
+  /// An optional callback to handle printing the CommandReturnObject.
+  CommandReturnObjectCallback m_print_callback;
+
   // The exit code the user has requested when calling the 'quit' command.
   // No value means the user hasn't set a custom exit code so far.
   std::optional<int> m_quit_exit_code;
diff --git a/lldb/include/lldb/Interpreter/CommandReturnObject.h b/lldb/include/lldb/Interpreter/CommandReturnObject.h
index 9fef59337016d..803bcd76995ed 100644
--- a/lldb/include/lldb/Interpreter/CommandReturnObject.h
+++ b/lldb/include/lldb/Interpreter/CommandReturnObject.h
@@ -31,10 +31,16 @@ class CommandReturnObject {
 
   ~CommandReturnObject() = default;
 
+  /// Get the command as the user typed it. Empty string if commands were run on
+  /// behalf of lldb.
+  const std::string &GetCommand() const { return m_command; }
+
+  void SetCommand(std::string command) { m_command = std::move(command); }
+
   /// Format any inline diagnostics with an indentation of \c indent.
-  std::string GetInlineDiagnosticString(unsigned indent);
+  std::string GetInlineDiagnosticString(unsigned indent) const;
 
-  llvm::StringRef GetOutputString() {
+  llvm::StringRef GetOutputString() const {
     lldb::StreamSP stream_sp(m_out_stream.GetStreamAtIndex(eStreamStringIndex));
     if (stream_sp)
       return std::static_pointer_cast<StreamString>(stream_sp)->GetString();
@@ -46,7 +52,7 @@ class CommandReturnObject {
   /// If \c with_diagnostics is true, all diagnostics are also
   /// rendered into the string. Otherwise the expectation is that they
   /// are fetched with \ref GetInlineDiagnosticString().
-  std::string GetErrorString(bool with_diagnostics = true);
+  std::string GetErrorString(bool with_diagnostics = true) const;
   StructuredData::ObjectSP GetErrorData();
 
   Stream &GetOutputStream() {
@@ -95,11 +101,11 @@ class CommandReturnObject {
     m_err_stream.SetStreamAtIndex(eImmediateStreamIndex, stream_sp);
   }
 
-  lldb::StreamSP GetImmediateOutputStream() {
+  lldb::StreamSP GetImmediateOutputStream() const {
     return m_out_stream.GetStreamAtIndex(eImmediateStreamIndex);
   }
 
-  lldb::StreamSP GetImmediateErrorStream() {
+  lldb::StreamSP GetImmediateErrorStream() const {
     return m_err_stream.GetStreamAtIndex(eImmediateStreamIndex);
   }
 
@@ -182,6 +188,8 @@ class CommandReturnObject {
 private:
   enum { eStreamStringIndex = 0, eImmediateStreamIndex = 1 };
 
+  std::string m_command;
+
   StreamTee m_out_stream;
   StreamTee m_err_stream;
   std::vector<DiagnosticDetail> m_diagnostics;
diff --git a/lldb/include/lldb/Interpreter/OptionValueSInt64.h b/lldb/include/lldb/Interpreter/OptionValueSInt64.h
index 3cf41d38c0ef0..f7e72684e4102 100644
--- a/lldb/include/lldb/Interpreter/OptionValueSInt64.h
+++ b/lldb/include/lldb/Interpreter/OptionValueSInt64.h
@@ -68,11 +68,10 @@ class OptionValueSInt64 : public Cloneable<OptionValueSInt64, OptionValue> {
   }
 
   bool SetDefaultValue(int64_t value) {
-    if (value >= m_min_value && value <= m_max_value) {
-      m_default_value = value;
-      return true;
-    }
-    return false;
+    assert(value >= m_min_value && value <= m_max_value &&
+           "disallowed default value");
+    m_default_value = value;
+    return true;
   }
 
   void SetMinimumValue(int64_t v) { m_min_value = v; }
diff --git a/lldb/include/lldb/Interpreter/OptionValueUInt64.h b/lldb/include/lldb/Interpreter/OptionValueUInt64.h
index 07076075790c6..5cccff177cb1d 100644
--- a/lldb/include/lldb/Interpreter/OptionValueUInt64.h
+++ b/lldb/include/lldb/Interpreter/OptionValueUInt64.h
@@ -73,18 +73,17 @@ class OptionValueUInt64 : public Cloneable<OptionValueUInt64, OptionValue> {
   }
 
   bool SetDefaultValue(uint64_t value) {
-    if (value >= m_min_value && value <= m_max_value) {
-      m_default_value = value;
-      return true;
-    }
-    return false;
+    assert(value >= m_min_value && value <= m_max_value &&
+           "disallowed default value");
+    m_default_value = value;
+    return true;
   }
 
-  void SetMinimumValue(int64_t v) { m_min_value = v; }
+  void SetMinimumValue(uint64_t v) { m_min_value = v; }
 
   uint64_t GetMinimumValue() const { return m_min_value; }
 
-  void SetMaximumValue(int64_t v) { m_max_value = v; }
+  void SetMaximumValue(uint64_t v) { m_max_value = v; }
 
   uint64_t GetMaximumValue() const { return m_max_value; }
 
diff --git a/lldb/include/lldb/Interpreter/Options.h b/lldb/include/lldb/Interpreter/Options.h
index 9a6a17c2793fa..864bda6f24c8c 100644
--- a/lldb/include/lldb/Interpreter/Options.h
+++ b/lldb/include/lldb/Interpreter/Options.h
@@ -76,12 +76,12 @@ class Options {
   // This gets passed the short option as an integer...
   void OptionSeen(int short_option);
 
-  bool VerifyOptions(CommandReturnObject &result);
+  llvm::Error VerifyOptions();
 
   // Verify that the options given are in the options table and can be used
   // together, but there may be some required options that are missing (used to
   // verify options that get folded into command aliases).
-  bool VerifyPartialOptions(CommandReturnObject &result);
+  llvm::Error VerifyPartialOptions();
 
   void OutputFormattedUsageText(Stream &strm,
                                 const OptionDefinition &option_def,
diff --git a/lldb/include/lldb/Symbol/SymbolContext.h b/lldb/include/lldb/Symbol/SymbolContext.h
index 07769cd8dffae..69fbe544c73cd 100644
--- a/lldb/include/lldb/Symbol/SymbolContext.h
+++ b/lldb/include/lldb/Symbol/SymbolContext.h
@@ -165,8 +165,8 @@ class SymbolContext {
   ///     eSymbolContextSymbol is set in \a scope
   ///
   /// \param[in] scope
-  ///     A mask of symbol context bits telling this function which
-  ///     address ranges it can use when trying to extract one from
+  ///     A mask bits from the \b SymbolContextItem enum telling this function
+  ///     which address ranges it can use when trying to extract one from
   ///     the valid (non-nullptr) symbol context classes.
   ///
   /// \param[in] range_idx
@@ -192,6 +192,13 @@ class SymbolContext {
   bool GetAddressRange(uint32_t scope, uint32_t range_idx,
                        bool use_inline_block_range, AddressRange &range) const;
 
+  /// Get the address of the function or symbol represented by this symbol
+  /// context.
+  ///
+  /// If both fields are present, the address of the function is returned. If
+  /// both are empty, the result is an invalid address.
+  Address GetFunctionOrSymbolAddress() const;
+
   llvm::Error GetAddressRangeFromHereToEndLine(uint32_t end_line,
                                                AddressRange &range);
 
diff --git a/lldb/include/lldb/Symbol/UnwindTable.h b/lldb/include/lldb/Symbol/UnwindTable.h
index 29b7fa61b4849..3166fdec6ebaa 100644
--- a/lldb/include/lldb/Symbol/UnwindTable.h
+++ b/lldb/include/lldb/Symbol/UnwindTable.h
@@ -38,8 +38,9 @@ class UnwindTable {
   ArmUnwindInfo *GetArmUnwindInfo();
   SymbolFile *GetSymbolFile();
 
-  lldb::FuncUnwindersSP GetFuncUnwindersContainingAddress(const Address &addr,
-                                                          SymbolContext &sc);
+  lldb::FuncUnwindersSP
+  GetFuncUnwindersContainingAddress(const Address &addr,
+                                    const SymbolContext &sc);
 
   bool GetAllowAssemblyEmulationUnwindPlans();
 
diff --git a/lldb/include/lldb/Target/LanguageRuntime.h b/lldb/include/lldb/Target/LanguageRuntime.h
index 4a0214b04e235..f9ae2dc589632 100644
--- a/lldb/include/lldb/Target/LanguageRuntime.h
+++ b/lldb/include/lldb/Target/LanguageRuntime.h
@@ -105,12 +105,16 @@ class LanguageRuntime : public Runtime, public PluginInterface {
         "language doesn't support getting vtable information");
   }
 
-  // this call should return true if it could set the name and/or the type
-  virtual bool GetDynamicTypeAndAddress(ValueObject &in_value,
-                                        lldb::DynamicValueType use_dynamic,
-                                        TypeAndOrName &class_type_or_name,
-                                        Address &address,
-                                        Value::ValueType &value_type) = 0;
+  /// This call should return true if it could set the name and/or the type
+  /// Sets address to the address of the dynamic type if value_type is set to
+  /// a file or load address. Sets local_buffer to a buffer containing the data
+  /// of the dynamic type if value_type is set to a host address. Callers should
+  /// copy local_buffer over into their own buffer if they want to keep the data
+  /// alive.
+  virtual bool GetDynamicTypeAndAddress(
+      ValueObject &in_value, lldb::DynamicValueType use_dynamic,
+      TypeAndOrName &class_type_or_name, Address &address,
+      Value::ValueType &value_type, llvm::ArrayRef<uint8_t> &local_buffer) = 0;
 
   // This call should return a CompilerType given a generic type name and an
   // ExecutionContextScope in which one can actually fetch any specialization
diff --git a/lldb/include/lldb/Target/Process.h b/lldb/include/lldb/Target/Process.h
index a184e6dd891af..c3622a29bc772 100644
--- a/lldb/include/lldb/Target/Process.h
+++ b/lldb/include/lldb/Target/Process.h
@@ -1491,10 +1491,11 @@ class Process : public std::enable_shared_from_this<Process>,
   ///     otherwise.
   virtual bool IsAlive();
 
+  /// Check if a process is a live debug session, or a corefile/post-mortem.
   virtual bool IsLiveDebugSession() const { return true; };
 
   /// Provide a way to retrieve the core dump file that is loaded for debugging.
-  /// Only available if IsLiveDebugSession() returns true.
+  /// Only available if IsLiveDebugSession() returns false.
   ///
   /// \return
   ///     File path to the core file.
diff --git a/lldb/include/lldb/Target/Thread.h b/lldb/include/lldb/Target/Thread.h
index ef66fa11574db..9749fd8d575a1 100644
--- a/lldb/include/lldb/Target/Thread.h
+++ b/lldb/include/lldb/Target/Thread.h
@@ -470,6 +470,26 @@ class Thread : public std::enable_shared_from_this<Thread>,
 
   virtual void ClearStackFrames();
 
+  /// Sets the thread that is backed by this thread.
+  /// If backed_thread.GetBackedThread() is null, this method also calls
+  /// backed_thread.SetBackingThread(this).
+  /// If backed_thread.GetBackedThread() is non-null, asserts that it is equal
+  /// to `this`.
+  void SetBackedThread(Thread &backed_thread) {
+    m_backed_thread = backed_thread.shared_from_this();
+
+    // Ensure the bidrectional relationship is preserved.
+    Thread *backing_thread = backed_thread.GetBackingThread().get();
+    assert(backing_thread == nullptr || backing_thread == this);
+    if (backing_thread == nullptr)
+      backed_thread.SetBackingThread(shared_from_this());
+  }
+
+  void ClearBackedThread() { m_backed_thread.reset(); }
+
+  /// Returns the thread that is backed by this thread, if any.
+  lldb::ThreadSP GetBackedThread() const { return m_backed_thread.lock(); }
+
   virtual bool SetBackingThread(const lldb::ThreadSP &thread_sp) {
     return false;
   }
@@ -1349,6 +1369,9 @@ class Thread : public std::enable_shared_from_this<Thread>,
   LazyBool m_override_should_notify;
   mutable std::unique_ptr<ThreadPlanStack> m_null_plan_stack_up;
 
+  /// The Thread backed by this thread, if any.
+  lldb::ThreadWP m_backed_thread;
+
 private:
   bool m_extended_info_fetched; // Have we tried to retrieve the m_extended_info
                                 // for this thread?
diff --git a/lldb/include/lldb/Target/ThreadList.h b/lldb/include/lldb/Target/ThreadList.h
index f931bb83a8cea..bca01f5fe2083 100644
--- a/lldb/include/lldb/Target/ThreadList.h
+++ b/lldb/include/lldb/Target/ThreadList.h
@@ -101,8 +101,6 @@ class ThreadList : public ThreadCollection {
 
   lldb::ThreadSP GetThreadSPForThreadPtr(Thread *thread_ptr);
 
-  lldb::ThreadSP GetBackingThread(const lldb::ThreadSP &real_thread);
-
   bool ShouldStop(Event *event_ptr);
 
   Vote ShouldReportStop(Event *event_ptr);
diff --git a/lldb/include/lldb/Utility/StreamTee.h b/lldb/include/lldb/Utility/StreamTee.h
index 5695586171f35..571548e2e23fe 100644
--- a/lldb/include/lldb/Utility/StreamTee.h
+++ b/lldb/include/lldb/Utility/StreamTee.h
@@ -85,7 +85,7 @@ class StreamTee : public Stream {
     return result;
   }
 
-  lldb::StreamSP GetStreamAtIndex(uint32_t idx) {
+  lldb::StreamSP GetStreamAtIndex(uint32_t idx) const {
     lldb::StreamSP stream_sp;
     std::lock_guard<std::recursive_mutex> guard(m_streams_mutex);
     if (idx < m_streams.size())
diff --git a/lldb/include/lldb/ValueObject/DILLexer.h b/lldb/include/lldb/ValueObject/DILLexer.h
new file mode 100644
index 0000000000000..e1182da5b20ab
--- /dev/null
+++ b/lldb/include/lldb/ValueObject/DILLexer.h
@@ -0,0 +1,123 @@
+//===-- DILLexer.h ----------------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLDB_VALUEOBJECT_DILLEXER_H
+#define LLDB_VALUEOBJECT_DILLEXER_H
+
+#include "llvm/ADT/StringRef.h"
+#include "llvm/Support/Error.h"
+#include <cstdint>
+#include <memory>
+#include <string>
+#include <vector>
+
+namespace lldb_private::dil {
+
+/// Class defining the tokens generated by the DIL lexer and used by the
+/// DIL parser.
+class Token {
+public:
+  enum Kind {
+    coloncolon,
+    eof,
+    identifier,
+    l_paren,
+    r_paren,
+  };
+
+  Token(Kind kind, std::string spelling, uint32_t start)
+      : m_kind(kind), m_spelling(std::move(spelling)), m_start_pos(start) {}
+
+  Kind GetKind() const { return m_kind; }
+
+  std::string GetSpelling() const { return m_spelling; }
+
+  bool Is(Kind kind) const { return m_kind == kind; }
+
+  bool IsNot(Kind kind) const { return m_kind != kind; }
+
+  bool IsOneOf(Kind kind1, Kind kind2) const { return Is(kind1) || Is(kind2); }
+
+  template <typename... Ts> bool IsOneOf(Kind kind, Ts... Ks) const {
+    return Is(kind) || IsOneOf(Ks...);
+  }
+
+  uint32_t GetLocation() const { return m_start_pos; }
+
+  static llvm::StringRef GetTokenName(Kind kind);
+
+private:
+  Kind m_kind;
+  std::string m_spelling;
+  uint32_t m_start_pos; // within entire expression string
+};
+
+/// Class for doing the simple lexing required by DIL.
+class DILLexer {
+public:
+  /// Lexes all the tokens in expr and calls the private constructor
+  /// with the lexed tokens.
+  static llvm::Expected<DILLexer> Create(llvm::StringRef expr);
+
+  /// Return the current token to be handled by the DIL parser.
+  const Token &GetCurrentToken() { return m_lexed_tokens[m_tokens_idx]; }
+
+  /// Advance the current token position by N.
+  void Advance(uint32_t N = 1) {
+    if (m_tokens_idx + N >= m_lexed_tokens.size())
+      // N is too large; advance to the end of the lexed tokens.
+      m_tokens_idx = m_lexed_tokens.size() - 1;
+    else
+      m_tokens_idx += N;
+  }
+
+  /// Return the lexed token N positions ahead of the 'current' token
+  /// being handled by the DIL parser.
+  const Token &LookAhead(uint32_t N) {
+    if (m_tokens_idx + N < m_lexed_tokens.size())
+      return m_lexed_tokens[m_tokens_idx + N];
+
+    // Last token should be an 'eof' token.
+    return m_lexed_tokens.back();
+  }
+
+  /// Return the index for the 'current' token being handled by the DIL parser.
+  uint32_t GetCurrentTokenIdx() { return m_tokens_idx; }
+
+  /// Set the index for the 'current' token (to be handled by the parser)
+  /// to a particular position. Used for either committing 'look ahead' parsing
+  /// or rolling back tentative parsing.
+  void ResetTokenIdx(uint32_t new_value) {
+    assert(new_value < m_lexed_tokens.size());
+    m_tokens_idx = new_value;
+  }
+
+  uint32_t NumLexedTokens() { return m_lexed_tokens.size(); }
+
+private:
+  DILLexer(llvm::StringRef dil_expr, std::vector<Token> lexed_tokens)
+      : m_expr(dil_expr), m_lexed_tokens(std::move(lexed_tokens)),
+        m_tokens_idx(0) {}
+
+  static llvm::Expected<Token> Lex(llvm::StringRef expr,
+                                   llvm::StringRef &remainder);
+
+  // The input string we are lexing & parsing.
+  llvm::StringRef m_expr;
+
+  // Holds all of the tokens lexed so far.
+  std::vector<Token> m_lexed_tokens;
+
+  // Index into m_lexed_tokens; indicates which token the DIL parser is
+  // currently trying to parse/handle.
+  uint32_t m_tokens_idx;
+};
+
+} // namespace lldb_private::dil
+
+#endif // LLDB_VALUEOBJECT_DILLEXER_H
diff --git a/lldb/include/lldb/ValueObject/ValueObject.h b/lldb/include/lldb/ValueObject/ValueObject.h
index 4f77384bb8f13..a0f53d20327cd 100644
--- a/lldb/include/lldb/ValueObject/ValueObject.h
+++ b/lldb/include/lldb/ValueObject/ValueObject.h
@@ -865,6 +865,19 @@ class ValueObject {
 
   virtual void SetLanguageFlags(uint64_t flags) { m_language_flags = flags; }
 
+  /// Returns the local buffer that this ValueObject points to if it's
+  /// available.
+  /// \return
+  ///     The local buffer if this value object's value points to a
+  ///     host address, and if that buffer can be determined. Otherwise, returns
+  ///     an empty ArrayRef.
+  ///
+  /// TODO: Because a ValueObject's Value can point to any arbitrary memory
+  /// location, it is possible that we can't find what what buffer we're
+  /// pointing to, and thus also can't know its size. See the comment in
+  /// Value::m_value for a more thorough explanation of why that is.
+  llvm::ArrayRef<uint8_t> GetLocalBuffer() const;
+
 protected:
   typedef ClusterManager<ValueObject> ValueObjectManager;
 
diff --git a/lldb/include/lldb/ValueObject/ValueObjectConstResultImpl.h b/lldb/include/lldb/ValueObject/ValueObjectConstResultImpl.h
index dbd68160acb4d..5509886a8965d 100644
--- a/lldb/include/lldb/ValueObject/ValueObjectConstResultImpl.h
+++ b/lldb/include/lldb/ValueObject/ValueObjectConstResultImpl.h
@@ -66,6 +66,10 @@ class ValueObjectConstResultImpl {
 
 private:
   ValueObject *m_impl_backend;
+  /// The memory address in the inferior process that this ValueObject tracks.
+  /// This address is used to request additional memory when the actual data
+  /// size exceeds the initial local buffer size, such as when a dynamic type
+  /// resolution results in a type larger than its statically determined type.
   lldb::addr_t m_live_address;
   AddressType m_live_address_type;
   lldb::ValueObjectSP m_address_of_backend;
diff --git a/lldb/include/lldb/lldb-enumerations.h b/lldb/include/lldb/lldb-enumerations.h
index 50d2233509de6..fecf9cbb765f7 100644
--- a/lldb/include/lldb/lldb-enumerations.h
+++ b/lldb/include/lldb/lldb-enumerations.h
@@ -1368,6 +1368,15 @@ enum Severity {
   eSeverityInfo, // Equivalent to Remark used in clang.
 };
 
+/// Callback return value, indicating whether it handled printing the
+/// CommandReturnObject or deferred doing so to the CommandInterpreter.
+enum CommandReturnObjectCallbackResult {
+  /// The callback deferred printing the command return object.
+  eCommandReturnObjectPrintCallbackSkipped = 0,
+  /// The callback handled printing the command return object.
+  eCommandReturnObjectPrintCallbackHandled = 1,
+};
+
 } // namespace lldb
 
 #endif // LLDB_LLDB_ENUMERATIONS_H
diff --git a/lldb/packages/Python/lldbsuite/test/gdbclientutils.py b/lldb/packages/Python/lldbsuite/test/gdbclientutils.py
index 1784487323ad6..4b782b3b470fe 100644
--- a/lldb/packages/Python/lldbsuite/test/gdbclientutils.py
+++ b/lldb/packages/Python/lldbsuite/test/gdbclientutils.py
@@ -126,6 +126,9 @@ def respond(self, packet):
         if packet[0] == "m":
             addr, length = [int(x, 16) for x in packet[1:].split(",")]
             return self.readMemory(addr, length)
+        if packet[0] == "x":
+            addr, length = [int(x, 16) for x in packet[1:].split(",")]
+            return self.x(addr, length)
         if packet[0] == "M":
             location, encoded_data = packet[1:].split(":")
             addr, length = [int(x, 16) for x in location.split(",")]
@@ -267,6 +270,9 @@ def writeRegister(self, register, value_hex):
     def readMemory(self, addr, length):
         return "00" * length
 
+    def x(self, addr, length):
+        return ""
+
     def writeMemory(self, addr, data_hex):
         return "OK"
 
diff --git a/lldb/packages/Python/lldbsuite/test/lldbutil.py b/lldb/packages/Python/lldbsuite/test/lldbutil.py
index 07b5f8cc7d900..27e0040034370 100644
--- a/lldb/packages/Python/lldbsuite/test/lldbutil.py
+++ b/lldb/packages/Python/lldbsuite/test/lldbutil.py
@@ -1353,33 +1353,6 @@ def get_args_as_string(frame, showFuncName=True):
         return "(%s)" % (", ".join(args))
 
 
-def print_registers(frame, string_buffer=False):
-    """Prints all the register sets of the frame."""
-
-    output = io.StringIO() if string_buffer else sys.stdout
-
-    print("Register sets for " + str(frame), file=output)
-
-    registerSet = frame.GetRegisters()  # Return type of SBValueList.
-    print(
-        "Frame registers (size of register set = %d):" % registerSet.GetSize(),
-        file=output,
-    )
-    for value in registerSet:
-        # print(value, file=output)
-        print(
-            "%s (number of children = %d):" % (value.GetName(), value.GetNumChildren()),
-            file=output,
-        )
-        for child in value:
-            print(
-                "Name: %s, Value: %s" % (child.GetName(), child.GetValue()), file=output
-            )
-
-    if string_buffer:
-        return output.getvalue()
-
-
 def get_registers(frame, kind):
     """Returns the registers given the frame and the kind of registers desired.
 
@@ -1605,21 +1578,6 @@ def set_actions_for_signal(
     )
 
 
-class PrintableRegex(object):
-    def __init__(self, text):
-        self.regex = re.compile(text)
-        self.text = text
-
-    def match(self, str):
-        return self.regex.match(str)
-
-    def __str__(self):
-        return "%s" % (self.text)
-
-    def __repr__(self):
-        return "re.compile(%s) -> %s" % (self.text, self.regex)
-
-
 def skip_if_callable(test, mycallable, reason):
     if callable(mycallable):
         if mycallable(test):
diff --git a/lldb/packages/Python/lldbsuite/test/make/Makefile.rules b/lldb/packages/Python/lldbsuite/test/make/Makefile.rules
index 2da6ff226b615..06959f226066a 100644
--- a/lldb/packages/Python/lldbsuite/test/make/Makefile.rules
+++ b/lldb/packages/Python/lldbsuite/test/make/Makefile.rules
@@ -207,6 +207,10 @@ else
 		override ARCH :=
 		override ARCHFLAG :=
 	endif
+	ifeq "$(ARCH)" "riscv"
+		override ARCH :=
+		override ARCHFLAG :=
+	endif
 	ifeq "$(findstring mips,$(ARCH))" "mips"
 		override ARCHFLAG := -
 	endif
diff --git a/lldb/packages/Python/lldbsuite/test/tools/lldb-dap/dap_server.py b/lldb/packages/Python/lldbsuite/test/tools/lldb-dap/dap_server.py
index c29992ce9c784..043d82e2e2c7d 100644
--- a/lldb/packages/Python/lldbsuite/test/tools/lldb-dap/dap_server.py
+++ b/lldb/packages/Python/lldbsuite/test/tools/lldb-dap/dap_server.py
@@ -612,6 +612,28 @@ def request_attach(
         command_dict = {"command": "attach", "type": "request", "arguments": args_dict}
         return self.send_recv(command_dict)
 
+    def request_breakpointLocations(
+        self, file_path, line, end_line=None, column=None, end_column=None
+    ):
+        (dir, base) = os.path.split(file_path)
+        source_dict = {"name": base, "path": file_path}
+        args_dict = {}
+        args_dict["source"] = source_dict
+        if line is not None:
+            args_dict["line"] = line
+        if end_line is not None:
+            args_dict["endLine"] = end_line
+        if column is not None:
+            args_dict["column"] = column
+        if end_column is not None:
+            args_dict["endColumn"] = end_column
+        command_dict = {
+            "command": "breakpointLocations",
+            "type": "request",
+            "arguments": args_dict,
+        }
+        return self.send_recv(command_dict)
+
     def request_configurationDone(self):
         command_dict = {
             "command": "configurationDone",
@@ -851,6 +873,8 @@ def request_next(self, threadId, granularity="statement"):
     def request_stepIn(self, threadId, targetId, granularity="statement"):
         if self.exit_status is not None:
             raise ValueError("request_stepIn called after process exited")
+        if threadId is None:
+            threadId = self.get_thread_id()
         args_dict = {
             "threadId": threadId,
             "targetId": targetId,
@@ -911,18 +935,14 @@ def request_setBreakpoints(self, file_path, line_array, data=None):
                     breakpoint_data = data[i]
                 bp = {"line": line}
                 if breakpoint_data is not None:
-                    if "condition" in breakpoint_data and breakpoint_data["condition"]:
+                    if breakpoint_data.get("condition"):
                         bp["condition"] = breakpoint_data["condition"]
-                    if (
-                        "hitCondition" in breakpoint_data
-                        and breakpoint_data["hitCondition"]
-                    ):
+                    if breakpoint_data.get("hitCondition"):
                         bp["hitCondition"] = breakpoint_data["hitCondition"]
-                    if (
-                        "logMessage" in breakpoint_data
-                        and breakpoint_data["logMessage"]
-                    ):
+                    if breakpoint_data.get("logMessage"):
                         bp["logMessage"] = breakpoint_data["logMessage"]
+                    if breakpoint_data.get("column"):
+                        bp["column"] = breakpoint_data["column"]
                 breakpoints.append(bp)
             args_dict["breakpoints"] = breakpoints
 
diff --git a/lldb/packages/Python/lldbsuite/test/tools/lldb-dap/lldbdap_testcase.py b/lldb/packages/Python/lldbsuite/test/tools/lldb-dap/lldbdap_testcase.py
index a25466f07fa55..34e9b96dbcc3f 100644
--- a/lldb/packages/Python/lldbsuite/test/tools/lldb-dap/lldbdap_testcase.py
+++ b/lldb/packages/Python/lldbsuite/test/tools/lldb-dap/lldbdap_testcase.py
@@ -238,9 +238,10 @@ def set_global(self, name, value, id=None):
     def stepIn(
         self, threadId=None, targetId=None, waitForStop=True, granularity="statement"
     ):
-        self.dap_server.request_stepIn(
+        response = self.dap_server.request_stepIn(
             threadId=threadId, targetId=targetId, granularity=granularity
         )
+        self.assertTrue(response["success"])
         if waitForStop:
             return self.dap_server.wait_for_stopped()
         return None
diff --git a/lldb/source/API/SBCommandInterpreter.cpp b/lldb/source/API/SBCommandInterpreter.cpp
index 7a35473283684..d153e2acdf853 100644
--- a/lldb/source/API/SBCommandInterpreter.cpp
+++ b/lldb/source/API/SBCommandInterpreter.cpp
@@ -98,8 +98,8 @@ SBCommandInterpreter::SBCommandInterpreter(const SBCommandInterpreter &rhs)
 
 SBCommandInterpreter::~SBCommandInterpreter() = default;
 
-const SBCommandInterpreter &SBCommandInterpreter::
-operator=(const SBCommandInterpreter &rhs) {
+const SBCommandInterpreter &
+SBCommandInterpreter::operator=(const SBCommandInterpreter &rhs) {
   LLDB_INSTRUMENT_VA(this, rhs);
 
   m_opaque_ptr = rhs.m_opaque_ptr;
@@ -151,7 +151,7 @@ bool SBCommandInterpreter::WasInterrupted() const {
 
 bool SBCommandInterpreter::InterruptCommand() {
   LLDB_INSTRUMENT_VA(this);
-  
+
   return (IsValid() ? m_opaque_ptr->InterruptCommand() : false);
 }
 
@@ -222,8 +222,7 @@ void SBCommandInterpreter::HandleCommandsFromFile(
   if (override_context.get())
     m_opaque_ptr->HandleCommandsFromFile(tmp_spec,
                                          override_context.get()->Lock(true),
-                                         options.ref(),
-                                         result.ref());
+                                         options.ref(), result.ref());
 
   else
     m_opaque_ptr->HandleCommandsFromFile(tmp_spec, options.ref(), result.ref());
@@ -649,7 +648,8 @@ SBCommand::operator bool() const {
 const char *SBCommand::GetName() {
   LLDB_INSTRUMENT_VA(this);
 
-  return (IsValid() ? ConstString(m_opaque_sp->GetCommandName()).AsCString() : nullptr);
+  return (IsValid() ? ConstString(m_opaque_sp->GetCommandName()).AsCString()
+                    : nullptr);
 }
 
 const char *SBCommand::GetHelp() {
@@ -743,3 +743,15 @@ void SBCommand::SetFlags(uint32_t flags) {
   if (IsValid())
     m_opaque_sp->GetFlags().Set(flags);
 }
+
+void SBCommandInterpreter::SetPrintCallback(
+    lldb::SBCommandPrintCallback callback, void *baton) {
+  LLDB_INSTRUMENT_VA(this, callback, baton);
+
+  if (m_opaque_ptr)
+    m_opaque_ptr->SetPrintCallback(
+        [callback, baton](lldb_private::CommandReturnObject &result) {
+          SBCommandReturnObject sb_result(result);
+          return callback(sb_result, baton);
+        });
+}
diff --git a/lldb/source/API/SBCommandReturnObject.cpp b/lldb/source/API/SBCommandReturnObject.cpp
index 9df8aa48b9936..6f54581e64ef4 100644
--- a/lldb/source/API/SBCommandReturnObject.cpp
+++ b/lldb/source/API/SBCommandReturnObject.cpp
@@ -84,6 +84,13 @@ SBCommandReturnObject::operator bool() const {
   return true;
 }
 
+const char *SBCommandReturnObject::GetCommand() {
+  LLDB_INSTRUMENT_VA(this);
+
+  ConstString output(ref().GetCommand());
+  return output.AsCString(/*value_if_empty*/ "");
+}
+
 const char *SBCommandReturnObject::GetOutput() {
   LLDB_INSTRUMENT_VA(this);
 
diff --git a/lldb/source/API/SBDebugger.cpp b/lldb/source/API/SBDebugger.cpp
index 4e6b22492a0d1..bdb8e538b99f8 100644
--- a/lldb/source/API/SBDebugger.cpp
+++ b/lldb/source/API/SBDebugger.cpp
@@ -185,7 +185,7 @@ lldb::SBError SBDebugger::InitializeWithErrorHandling() {
     llvm::sys::DynamicLibrary dynlib =
         llvm::sys::DynamicLibrary::getPermanentLibrary(spec.GetPath().c_str());
     if (dynlib.isValid()) {
-      typedef bool (*LLDBCommandPluginInit)(lldb::SBDebugger & debugger);
+      typedef bool (*LLDBCommandPluginInit)(lldb::SBDebugger debugger);
 
       lldb::SBDebugger debugger_sb(debugger_sp);
       // This calls the bool lldb::PluginInitialize(lldb::SBDebugger debugger)
diff --git a/lldb/source/API/SBFunction.cpp b/lldb/source/API/SBFunction.cpp
index d07594c2e8c01..19861f6af3645 100644
--- a/lldb/source/API/SBFunction.cpp
+++ b/lldb/source/API/SBFunction.cpp
@@ -144,7 +144,7 @@ SBAddress SBFunction::GetStartAddress() {
 
   SBAddress addr;
   if (m_opaque_ptr)
-    addr.SetAddress(m_opaque_ptr->GetAddressRange().GetBaseAddress());
+    addr.SetAddress(m_opaque_ptr->GetAddress());
   return addr;
 }
 
diff --git a/lldb/source/API/SBProgress.cpp b/lldb/source/API/SBProgress.cpp
index d6ed5f0d15fc9..e67e289a60eff 100644
--- a/lldb/source/API/SBProgress.cpp
+++ b/lldb/source/API/SBProgress.cpp
@@ -32,6 +32,9 @@ SBProgress::SBProgress(const char *title, const char *details,
       lldb_private::Progress::Origin::eExternal);
 }
 
+SBProgress::SBProgress(SBProgress &&rhs)
+    : m_opaque_up(std::move(rhs.m_opaque_up)) {}
+
 SBProgress::~SBProgress() = default;
 
 void SBProgress::Increment(uint64_t amount, const char *description) {
diff --git a/lldb/source/API/SBTarget.cpp b/lldb/source/API/SBTarget.cpp
index 2a33161bc21ed..dd9caa724ea36 100644
--- a/lldb/source/API/SBTarget.cpp
+++ b/lldb/source/API/SBTarget.cpp
@@ -1342,7 +1342,8 @@ lldb::SBWatchpoint SBTarget::WatchAddress(lldb::addr_t addr, size_t size,
 
   SBWatchpointOptions options;
   options.SetWatchpointTypeRead(read);
-  options.SetWatchpointTypeWrite(eWatchpointWriteTypeOnModify);
+  if (modify)
+    options.SetWatchpointTypeWrite(eWatchpointWriteTypeOnModify);
   return WatchpointCreateByAddress(addr, size, options, error);
 }
 
diff --git a/lldb/source/Breakpoint/BreakpointSite.cpp b/lldb/source/Breakpoint/BreakpointSite.cpp
index 9700a57d3346e..8b964c5711468 100644
--- a/lldb/source/Breakpoint/BreakpointSite.cpp
+++ b/lldb/source/Breakpoint/BreakpointSite.cpp
@@ -12,6 +12,7 @@
 
 #include "lldb/Breakpoint/Breakpoint.h"
 #include "lldb/Breakpoint/BreakpointLocation.h"
+#include "lldb/Target/Thread.h"
 #include "lldb/Utility/Stream.h"
 
 using namespace lldb;
@@ -161,6 +162,8 @@ BreakpointLocationSP BreakpointSite::GetConstituentAtIndex(size_t index) {
 
 bool BreakpointSite::ValidForThisThread(Thread &thread) {
   std::lock_guard<std::recursive_mutex> guard(m_constituents_mutex);
+  if (ThreadSP backed_thread = thread.GetBackedThread())
+    return m_constituents.ValidForThisThread(*backed_thread);
   return m_constituents.ValidForThisThread(thread);
 }
 
diff --git a/lldb/source/Breakpoint/WatchpointList.cpp b/lldb/source/Breakpoint/WatchpointList.cpp
index f7564483e6f1f..57369b76c03af 100644
--- a/lldb/source/Breakpoint/WatchpointList.cpp
+++ b/lldb/source/Breakpoint/WatchpointList.cpp
@@ -236,7 +236,7 @@ void WatchpointList::RemoveAll(bool notify) {
       wp_collection::iterator pos, end = m_watchpoints.end();
       for (pos = m_watchpoints.begin(); pos != end; ++pos) {
         if ((*pos)->GetTarget().EventTypeHasListeners(
-                Target::eBroadcastBitBreakpointChanged)) {
+                Target::eBroadcastBitWatchpointChanged)) {
           auto data_sp = std::make_shared<Watchpoint::WatchpointEventData>(
               eWatchpointEventTypeRemoved, *pos);
           (*pos)->GetTarget().BroadcastEvent(
diff --git a/lldb/source/Commands/CommandObjectTarget.cpp b/lldb/source/Commands/CommandObjectTarget.cpp
index d8265e41a7384..da50fe04fa2b6 100644
--- a/lldb/source/Commands/CommandObjectTarget.cpp
+++ b/lldb/source/Commands/CommandObjectTarget.cpp
@@ -1621,12 +1621,15 @@ static void DumpSymbolContextList(
     if (!first_module)
       strm.EOL();
 
-    AddressRange range;
-
-    sc.GetAddressRange(eSymbolContextEverything, 0, true, range);
+    Address addr;
+    if (sc.line_entry.IsValid())
+      addr = sc.line_entry.range.GetBaseAddress();
+    else if (sc.block && sc.block->GetContainingInlinedBlock())
+      sc.block->GetContainingInlinedBlock()->GetStartAddress(addr);
+    else
+      addr = sc.GetFunctionOrSymbolAddress();
 
-    DumpAddress(exe_scope, range.GetBaseAddress(), verbose, all_ranges, strm,
-                settings);
+    DumpAddress(exe_scope, addr, verbose, all_ranges, strm, settings);
     first_module = false;
   }
   strm.IndentLess();
@@ -3471,6 +3474,17 @@ class CommandObjectTargetModulesShowUnwind : public CommandObjectParsed {
         m_type = eLookupTypeFunctionOrSymbol;
         break;
 
+      case 'c':
+        bool value, success;
+        value = OptionArgParser::ToBoolean(option_arg, false, &success);
+        if (success) {
+          m_cached = value;
+        } else {
+          return Status::FromErrorStringWithFormatv(
+              "invalid boolean value '%s' passed for -c option", option_arg);
+        }
+        break;
+
       default:
         llvm_unreachable("Unimplemented option");
       }
@@ -3482,6 +3496,7 @@ class CommandObjectTargetModulesShowUnwind : public CommandObjectParsed {
       m_type = eLookupTypeInvalid;
       m_str.clear();
       m_addr = LLDB_INVALID_ADDRESS;
+      m_cached = false;
     }
 
     llvm::ArrayRef<OptionDefinition> GetDefinitions() override {
@@ -3494,6 +3509,7 @@ class CommandObjectTargetModulesShowUnwind : public CommandObjectParsed {
                                      // parsing options
     std::string m_str; // Holds name lookup
     lldb::addr_t m_addr = LLDB_INVALID_ADDRESS; // Holds the address to lookup
+    bool m_cached = true;
   };
 
   CommandObjectTargetModulesShowUnwind(CommandInterpreter &interpreter)
@@ -3570,22 +3586,22 @@ class CommandObjectTargetModulesShowUnwind : public CommandObjectParsed {
         continue;
       if (!sc.module_sp || sc.module_sp->GetObjectFile() == nullptr)
         continue;
-      AddressRange range;
-      if (!sc.GetAddressRange(eSymbolContextFunction | eSymbolContextSymbol, 0,
-                              false, range))
-        continue;
-      if (!range.GetBaseAddress().IsValid())
+      Address addr = sc.GetFunctionOrSymbolAddress();
+      if (!addr.IsValid())
         continue;
       ConstString funcname(sc.GetFunctionName());
       if (funcname.IsEmpty())
         continue;
-      addr_t start_addr = range.GetBaseAddress().GetLoadAddress(target);
+      addr_t start_addr = addr.GetLoadAddress(target);
       if (abi)
         start_addr = abi->FixCodeAddress(start_addr);
 
-      FuncUnwindersSP func_unwinders_sp(
-          sc.module_sp->GetUnwindTable()
-              .GetUncachedFuncUnwindersContainingAddress(start_addr, sc));
+      UnwindTable &uw_table = sc.module_sp->GetUnwindTable();
+      FuncUnwindersSP func_unwinders_sp =
+          m_options.m_cached
+              ? uw_table.GetFuncUnwindersContainingAddress(start_addr, sc)
+              : uw_table.GetUncachedFuncUnwindersContainingAddress(start_addr,
+                                                                   sc);
       if (!func_unwinders_sp)
         continue;
 
diff --git a/lldb/source/Commands/Options.td b/lldb/source/Commands/Options.td
index 777f8c36c4916..8831fed38435b 100644
--- a/lldb/source/Commands/Options.td
+++ b/lldb/source/Commands/Options.td
@@ -965,6 +965,8 @@ let Command = "target modules show unwind" in {
   def target_modules_show_unwind_address : Option<"address", "a">, Group<2>,
     Arg<"AddressOrExpression">, Desc<"Show unwind instructions for a function "
     "or symbol containing an address">;
+  def target_modules_show_unwind_cached : Option<"cached", "c">,
+    Arg<"Boolean">, Desc<"Show cached unwind information">;
 }
 
 let Command = "target modules lookup" in {
diff --git a/lldb/source/Core/CMakeLists.txt b/lldb/source/Core/CMakeLists.txt
index 6d14f7a87764e..cf5f6ac9da489 100644
--- a/lldb/source/Core/CMakeLists.txt
+++ b/lldb/source/Core/CMakeLists.txt
@@ -16,6 +16,11 @@ if (LLDB_ENABLE_CURSES)
   endif()
 endif()
 
+if (LLVM_BUILD_TELEMETRY)
+   set(TELEMETRY_DEPS Telemetry)
+   add_definitions(-DLLDB_BUILD_TELEMETRY)
+endif()
+
 # TODO: Add property `NO_PLUGIN_DEPENDENCIES` to lldbCore
 add_lldb_library(lldbCore
   Address.cpp
@@ -52,10 +57,10 @@ add_lldb_library(lldbCore
   SourceLocationSpec.cpp
   SourceManager.cpp
   StreamAsynchronousIO.cpp
+  Telemetry.cpp
   ThreadedCommunication.cpp
   UserSettingsController.cpp
   Value.cpp
-
   DEPENDS
     clang-tablegen-targets
 
@@ -80,6 +85,7 @@ add_lldb_library(lldbCore
     Support
     Demangle
     TargetParser
+    ${TELEMETRY_DEPS}
   )
 
 add_dependencies(lldbCore
diff --git a/lldb/source/Core/FormatEntity.cpp b/lldb/source/Core/FormatEntity.cpp
index fb7043ac74b8d..7fe22994d7f7e 100644
--- a/lldb/source/Core/FormatEntity.cpp
+++ b/lldb/source/Core/FormatEntity.cpp
@@ -450,7 +450,7 @@ static bool DumpAddressOffsetFromFunction(Stream &s, const SymbolContext *sc,
 
     if (sc) {
       if (sc->function) {
-        func_addr = sc->function->GetAddressRange().GetBaseAddress();
+        func_addr = sc->function->GetAddress();
         if (sc->block && !concrete_only) {
           // Check to make sure we aren't in an inline function. If we are, use
           // the inline block range that contains "format_addr" since blocks
@@ -468,7 +468,7 @@ static bool DumpAddressOffsetFromFunction(Stream &s, const SymbolContext *sc,
     if (func_addr.IsValid()) {
       const char *addr_offset_padding = no_padding ? "" : " ";
 
-      if (func_addr.GetSection() == format_addr.GetSection()) {
+      if (func_addr.GetModule() == format_addr.GetModule()) {
         addr_t func_file_addr = func_addr.GetFileAddress();
         addr_t addr_file_addr = format_addr.GetFileAddress();
         if (addr_file_addr > func_file_addr ||
diff --git a/lldb/source/Core/Module.cpp b/lldb/source/Core/Module.cpp
index 9601c834d9b8f..33668c5d20dde 100644
--- a/lldb/source/Core/Module.cpp
+++ b/lldb/source/Core/Module.cpp
@@ -919,9 +919,8 @@ void Module::FindFunctions(const RegularExpression &regex,
               const SymbolContext &sc = sc_list[i];
               if (sc.block)
                 continue;
-              file_addr_to_index[sc.function->GetAddressRange()
-                                     .GetBaseAddress()
-                                     .GetFileAddress()] = i;
+              file_addr_to_index[sc.function->GetAddress().GetFileAddress()] =
+                  i;
             }
 
             FileAddrToIndexMap::const_iterator end = file_addr_to_index.end();
diff --git a/lldb/source/Core/SourceManager.cpp b/lldb/source/Core/SourceManager.cpp
index 27a9edeef4249..d63d42de14e80 100644
--- a/lldb/source/Core/SourceManager.cpp
+++ b/lldb/source/Core/SourceManager.cpp
@@ -435,9 +435,8 @@ SourceManager::GetDefaultFileAndLine() {
         for (const SymbolContext &sc : sc_list) {
           if (sc.function) {
             lldb_private::LineEntry line_entry;
-            if (sc.function->GetAddressRange()
-                    .GetBaseAddress()
-                    .CalculateSymbolContextLineEntry(line_entry)) {
+            if (sc.function->GetAddress().CalculateSymbolContextLineEntry(
+                    line_entry)) {
               SetDefaultFileAndLine(line_entry.file_sp, line_entry.line);
               return SupportFileAndLine(line_entry.file_sp, m_last_line);
             }
diff --git a/lldb/source/Core/Telemetry.cpp b/lldb/source/Core/Telemetry.cpp
new file mode 100644
index 0000000000000..99f5d43ccbaf0
--- /dev/null
+++ b/lldb/source/Core/Telemetry.cpp
@@ -0,0 +1,74 @@
+//===-- Telemetry.cpp -----------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifdef LLDB_BUILD_TELEMETRY
+
+#include "lldb/Core/Telemetry.h"
+#include "lldb/Core/Debugger.h"
+#include "lldb/Utility/LLDBLog.h"
+#include "lldb/Utility/UUID.h"
+#include "lldb/lldb-enumerations.h"
+#include "lldb/lldb-forward.h"
+#include "llvm/ADT/StringRef.h"
+#include "llvm/Support/Error.h"
+#include "llvm/Support/RandomNumberGenerator.h"
+#include "llvm/Telemetry/Telemetry.h"
+#include <chrono>
+#include <cstdlib>
+#include <memory>
+#include <string>
+#include <utility>
+
+namespace lldb_private {
+namespace telemetry {
+
+using ::llvm::Error;
+using ::llvm::telemetry::Destination;
+using ::llvm::telemetry::Serializer;
+using ::llvm::telemetry::TelemetryInfo;
+
+static uint64_t ToNanosec(const SteadyTimePoint Point) {
+  return std::chrono::nanoseconds(Point.time_since_epoch()).count();
+}
+
+void LLDBBaseTelemetryInfo::serialize(Serializer &serializer) const {
+  serializer.write("entry_kind", getKind());
+  serializer.write("session_id", SessionId);
+  serializer.write("start_time", ToNanosec(start_time));
+  if (end_time.has_value())
+    serializer.write("end_time", ToNanosec(end_time.value()));
+}
+
+[[maybe_unused]] static std::string MakeUUID(lldb_private::Debugger *debugger) {
+  uint8_t random_bytes[16];
+  if (auto ec = llvm::getRandomBytes(random_bytes, 16)) {
+    LLDB_LOG(GetLog(LLDBLog::Object),
+             "Failed to generate random bytes for UUID: {0}", ec.message());
+    // fallback to using timestamp + debugger ID.
+    return llvm::formatv(
+        "{0}_{1}", std::chrono::steady_clock::now().time_since_epoch().count(),
+        debugger->GetID());
+  }
+  return lldb_private::UUID(random_bytes).GetAsString();
+}
+
+TelemetryManager::TelemetryManager(
+    std::unique_ptr<llvm::telemetry::Config> config)
+    : m_config(std::move(config)) {}
+
+llvm::Error TelemetryManager::preDispatch(TelemetryInfo *entry) {
+  // Do nothing for now.
+  // In up-coming patch, this would be where the manager
+  // attach the session_uuid to the entry.
+  return Error::success();
+}
+
+} // namespace telemetry
+} // namespace lldb_private
+
+#endif // LLDB_BUILD_TELEMETRY
diff --git a/lldb/source/Core/UserSettingsController.cpp b/lldb/source/Core/UserSettingsController.cpp
index b57c1b0eef9b4..5408d64b40647 100644
--- a/lldb/source/Core/UserSettingsController.cpp
+++ b/lldb/source/Core/UserSettingsController.cpp
@@ -40,64 +40,45 @@ Properties::~Properties() = default;
 lldb::OptionValueSP
 Properties::GetPropertyValue(const ExecutionContext *exe_ctx,
                              llvm::StringRef path, Status &error) const {
-  OptionValuePropertiesSP properties_sp(GetValueProperties());
-  if (properties_sp)
-    return properties_sp->GetSubValue(exe_ctx, path, error);
-  return lldb::OptionValueSP();
+  return m_collection_sp->GetSubValue(exe_ctx, path, error);
 }
 
 Status Properties::SetPropertyValue(const ExecutionContext *exe_ctx,
                                     VarSetOperationType op,
                                     llvm::StringRef path,
                                     llvm::StringRef value) {
-  OptionValuePropertiesSP properties_sp(GetValueProperties());
-  if (properties_sp)
-    return properties_sp->SetSubValue(exe_ctx, op, path, value);
-  return Status::FromErrorString("no properties");
+  return m_collection_sp->SetSubValue(exe_ctx, op, path, value);
 }
 
 void Properties::DumpAllPropertyValues(const ExecutionContext *exe_ctx,
                                        Stream &strm, uint32_t dump_mask,
                                        bool is_json) {
-  OptionValuePropertiesSP properties_sp(GetValueProperties());
-  if (!properties_sp)
-    return;
-
   if (is_json) {
-    llvm::json::Value json = properties_sp->ToJSON(exe_ctx);
+    llvm::json::Value json = m_collection_sp->ToJSON(exe_ctx);
     strm.Printf("%s", llvm::formatv("{0:2}", json).str().c_str());
   } else
-    properties_sp->DumpValue(exe_ctx, strm, dump_mask);
+    m_collection_sp->DumpValue(exe_ctx, strm, dump_mask);
 }
 
 void Properties::DumpAllDescriptions(CommandInterpreter &interpreter,
                                      Stream &strm) const {
   strm.PutCString("Top level variables:\n\n");
 
-  OptionValuePropertiesSP properties_sp(GetValueProperties());
-  if (properties_sp)
-    return properties_sp->DumpAllDescriptions(interpreter, strm);
+  return m_collection_sp->DumpAllDescriptions(interpreter, strm);
 }
 
 Status Properties::DumpPropertyValue(const ExecutionContext *exe_ctx,
                                      Stream &strm,
                                      llvm::StringRef property_path,
                                      uint32_t dump_mask, bool is_json) {
-  OptionValuePropertiesSP properties_sp(GetValueProperties());
-  if (properties_sp) {
-    return properties_sp->DumpPropertyValue(exe_ctx, strm, property_path,
+  return m_collection_sp->DumpPropertyValue(exe_ctx, strm, property_path,
                                             dump_mask, is_json);
-  }
-  return Status::FromErrorString("empty property list");
 }
 
 size_t
 Properties::Apropos(llvm::StringRef keyword,
                     std::vector<const Property *> &matching_properties) const {
-  OptionValuePropertiesSP properties_sp(GetValueProperties());
-  if (properties_sp) {
-    properties_sp->Apropos(keyword, matching_properties);
-  }
+  m_collection_sp->Apropos(keyword, matching_properties);
   return matching_properties.size();
 }
 
diff --git a/lldb/source/Expression/Materializer.cpp b/lldb/source/Expression/Materializer.cpp
index 8cd050f9fdb7e..13a72a9921e1d 100644
--- a/lldb/source/Expression/Materializer.cpp
+++ b/lldb/source/Expression/Materializer.cpp
@@ -1187,6 +1187,9 @@ class EntityResultVariable : public Materializer::Entity {
 
 private:
   CompilerType m_type;
+  /// This is used both to control whether this result entity can (and should)
+  /// track the value in inferior memory, as well as to control whether LLDB
+  /// needs to allocate memory for the variable during materialization.
   bool m_is_program_reference;
   bool m_keep_in_memory;
 
diff --git a/lldb/source/Host/common/Host.cpp b/lldb/source/Host/common/Host.cpp
index fdb623667bc25..c52e11ad6ddf2 100644
--- a/lldb/source/Host/common/Host.cpp
+++ b/lldb/source/Host/common/Host.cpp
@@ -11,13 +11,17 @@
 #include <climits>
 #include <cstdlib>
 #include <sys/types.h>
+
 #ifndef _WIN32
 #include <dlfcn.h>
 #include <grp.h>
 #include <netdb.h>
 #include <pwd.h>
 #include <sys/stat.h>
+#include <sys/syscall.h>
+#include <sys/wait.h>
 #include <unistd.h>
+#include <spawn.h>
 #endif
 
 #if defined(__APPLE__)
@@ -26,16 +30,6 @@
 #include <mach/mach_port.h>
 #endif
 
-#if defined(__linux__) || defined(__FreeBSD__) ||                              \
-    defined(__FreeBSD_kernel__) || defined(__APPLE__) ||                       \
-    defined(__NetBSD__) || defined(__OpenBSD__) || defined(__EMSCRIPTEN__)
-#if !defined(__ANDROID__)
-#include <spawn.h>
-#endif
-#include <sys/syscall.h>
-#include <sys/wait.h>
-#endif
-
 #if defined(__FreeBSD__)
 #include <pthread_np.h>
 #endif
diff --git a/lldb/source/Host/common/Socket.cpp b/lldb/source/Host/common/Socket.cpp
index 296c2273ba419..f35e5ff43595b 100644
--- a/lldb/source/Host/common/Socket.cpp
+++ b/lldb/source/Host/common/Socket.cpp
@@ -40,16 +40,6 @@
 #include "lldb/Host/linux/AbstractSocket.h"
 #endif
 
-#ifdef __ANDROID__
-#include <arpa/inet.h>
-#include <asm-generic/errno-base.h>
-#include <cerrno>
-#include <fcntl.h>
-#include <linux/tcp.h>
-#include <sys/syscall.h>
-#include <unistd.h>
-#endif // __ANDROID__
-
 using namespace lldb;
 using namespace lldb_private;
 
diff --git a/lldb/source/Host/macosx/objcxx/Host.mm b/lldb/source/Host/macosx/objcxx/Host.mm
index a99ffbc3a3fac..5b3d04a6b587b 100644
--- a/lldb/source/Host/macosx/objcxx/Host.mm
+++ b/lldb/source/Host/macosx/objcxx/Host.mm
@@ -452,6 +452,38 @@ repeat with the_window in (get windows)\n\
 #endif // TARGET_OS_OSX
 }
 
+llvm::Error Host::OpenURL(llvm::StringRef url) {
+#if !TARGET_OS_OSX
+  return llvm::errorCodeToError(
+      std::error_code(ENOTSUP, std::system_category()));
+#else  // !TARGET_OS_OSX
+  if (url.empty())
+    return llvm::createStringError("Cannot open empty URL.");
+
+  LLDB_LOG(GetLog(LLDBLog::Host), "Opening URL: {0}", url);
+
+  CFCString url_cfstr(url.data(), kCFStringEncodingUTF8);
+  CFCReleaser<CFURLRef> cfurl = ::CFURLCreateWithString(
+      /*allocator=*/NULL,
+      /*URLString*/ url_cfstr.get(),
+      /*baseURL=*/NULL);
+
+  if (!cfurl.get())
+    return llvm::createStringError(
+        llvm::formatv("could not create CFURL from URL \"{0}\"", url));
+
+  OSStatus error = ::LSOpenCFURLRef(
+      /*inURL=*/cfurl.get(),
+      /*outLaunchedURL=*/NULL);
+
+  if (error != noErr)
+    return llvm::createStringError(
+        llvm::formatv("LSOpenCFURLRef failed: error {0:x}", error));
+
+  return llvm::Error::success();
+#endif // TARGET_OS_OSX
+}
+
 bool Host::IsInteractiveGraphicSession() {
 #if !TARGET_OS_OSX
   return false;
diff --git a/lldb/source/Host/posix/ProcessLauncherPosixFork.cpp b/lldb/source/Host/posix/ProcessLauncherPosixFork.cpp
index 7d856954684c4..3e956290c3055 100644
--- a/lldb/source/Host/posix/ProcessLauncherPosixFork.cpp
+++ b/lldb/source/Host/posix/ProcessLauncherPosixFork.cpp
@@ -230,16 +230,6 @@ MakeForkActions(const ProcessLaunchInfo &info) {
   return result;
 }
 
-static Environment::Envp FixupEnvironment(Environment env) {
-#ifdef __ANDROID__
-  // If there is no PATH variable specified inside the environment then set the
-  // path to /system/bin. It is required because the default path used by
-  // execve() is wrong on android.
-  env.try_emplace("PATH", "/system/bin");
-#endif
-  return env.getEnvp();
-}
-
 ForkLaunchInfo::ForkLaunchInfo(const ProcessLaunchInfo &info)
     : separate_process_group(
           info.GetFlags().Test(eLaunchFlagLaunchInSeparateProcessGroup)),
@@ -247,8 +237,7 @@ ForkLaunchInfo::ForkLaunchInfo(const ProcessLaunchInfo &info)
       disable_aslr(info.GetFlags().Test(eLaunchFlagDisableASLR)),
       wd(info.GetWorkingDirectory().GetPath()),
       argv(info.GetArguments().GetConstArgumentVector()),
-      envp(FixupEnvironment(info.GetEnvironment())),
-      actions(MakeForkActions(info)) {}
+      envp(info.GetEnvironment().getEnvp()), actions(MakeForkActions(info)) {}
 
 HostProcess
 ProcessLauncherPosixFork::LaunchProcess(const ProcessLaunchInfo &launch_info,
diff --git a/lldb/source/Interpreter/CommandAlias.cpp b/lldb/source/Interpreter/CommandAlias.cpp
index c5971b52f837f..b45fcca358a5e 100644
--- a/lldb/source/Interpreter/CommandAlias.cpp
+++ b/lldb/source/Interpreter/CommandAlias.cpp
@@ -10,6 +10,7 @@
 
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/Support/ErrorHandling.h"
+#include "llvm/Support/FormatAdapters.h"
 
 #include "lldb/Interpreter/CommandInterpreter.h"
 #include "lldb/Interpreter/CommandObject.h"
@@ -20,20 +21,17 @@
 using namespace lldb;
 using namespace lldb_private;
 
-static bool ProcessAliasOptionsArgs(lldb::CommandObjectSP &cmd_obj_sp,
-                                    llvm::StringRef options_args,
-                                    OptionArgVectorSP &option_arg_vector_sp) {
-  bool success = true;
+static llvm::Error
+ProcessAliasOptionsArgs(lldb::CommandObjectSP &cmd_obj_sp,
+                        llvm::StringRef options_args,
+                        OptionArgVectorSP &option_arg_vector_sp) {
   OptionArgVector *option_arg_vector = option_arg_vector_sp.get();
 
   if (options_args.size() < 1)
-    return true;
+    return llvm::Error::success();
 
   Args args(options_args);
   std::string options_string(options_args);
-  // TODO: Find a way to propagate errors in this CommandReturnObject up the
-  // stack.
-  CommandReturnObject result(false);
   // Check to see if the command being aliased can take any command options.
   Options *options = cmd_obj_sp->GetOptions();
   if (options) {
@@ -45,34 +43,30 @@ static bool ProcessAliasOptionsArgs(lldb::CommandObjectSP &cmd_obj_sp,
 
     llvm::Expected<Args> args_or =
         options->ParseAlias(args, option_arg_vector, options_string);
-    if (!args_or) {
-      result.AppendError(toString(args_or.takeError()));
-      result.AppendError("Unable to create requested alias.\n");
-      return false;
-    }
+    if (!args_or)
+      return llvm::createStringError(
+          llvm::formatv("unable to create alias: {0}",
+                        llvm::fmt_consume(args_or.takeError())));
     args = std::move(*args_or);
-    options->VerifyPartialOptions(result);
-    if (!result.Succeeded() &&
-        result.GetStatus() != lldb::eReturnStatusStarted) {
-      result.AppendError("Unable to create requested alias.\n");
-      return false;
-    }
+    if (llvm::Error error = options->VerifyPartialOptions())
+      return error;
   }
 
   if (!options_string.empty()) {
-    if (cmd_obj_sp->WantsRawCommandString())
-      option_arg_vector->emplace_back(CommandInterpreter::g_argument, 
-                                      -1, options_string);
-    else {
+    if (cmd_obj_sp->WantsRawCommandString()) {
+      option_arg_vector->emplace_back(CommandInterpreter::g_argument, -1,
+                                      options_string);
+    } else {
       for (auto &entry : args.entries()) {
         if (!entry.ref().empty())
-          option_arg_vector->emplace_back(std::string(CommandInterpreter::g_argument), -1,
-                                          std::string(entry.ref()));
+          option_arg_vector->emplace_back(
+              std::string(CommandInterpreter::g_argument), -1,
+              std::string(entry.ref()));
       }
     }
   }
 
-  return success;
+  return llvm::Error::success();
 }
 
 CommandAlias::CommandAlias(CommandInterpreter &interpreter,
@@ -85,10 +79,15 @@ CommandAlias::CommandAlias(CommandInterpreter &interpreter,
       m_option_args_sp(new OptionArgVector),
       m_is_dashdash_alias(eLazyBoolCalculate), m_did_set_help(false),
       m_did_set_help_long(false) {
-  if (ProcessAliasOptionsArgs(cmd_sp, options_args, m_option_args_sp)) {
+  if (llvm::Error error =
+          ProcessAliasOptionsArgs(cmd_sp, options_args, m_option_args_sp)) {
+    // FIXME: Find a way to percolate this error up.
+    LLDB_LOG_ERROR(GetLog(LLDBLog::Host), std::move(error),
+                   "ProcessAliasOptionsArgs failed: {0}");
+  } else {
     m_underlying_command_sp = cmd_sp;
     for (int i = 0;
-         auto cmd_entry = m_underlying_command_sp->GetArgumentEntryAtIndex(i);
+         auto *cmd_entry = m_underlying_command_sp->GetArgumentEntryAtIndex(i);
          i++) {
       m_arguments.push_back(*cmd_entry);
     }
diff --git a/lldb/source/Interpreter/CommandInterpreter.cpp b/lldb/source/Interpreter/CommandInterpreter.cpp
index 764dcfd1903b1..acdec84a1689b 100644
--- a/lldb/source/Interpreter/CommandInterpreter.cpp
+++ b/lldb/source/Interpreter/CommandInterpreter.cpp
@@ -113,7 +113,6 @@ const char *CommandInterpreter::g_no_argument = "<no-argument>";
 const char *CommandInterpreter::g_need_argument = "<need-argument>";
 const char *CommandInterpreter::g_argument = "<argument>";
 
-
 #define LLDB_PROPERTIES_interpreter
 #include "InterpreterProperties.inc"
 
@@ -285,8 +284,6 @@ bool CommandInterpreter::GetRequireCommandOverwrite() const {
 void CommandInterpreter::Initialize() {
   LLDB_SCOPED_TIMER();
 
-  CommandReturnObject result(m_debugger.GetUseColor());
-
   LoadCommandDictionary();
 
   // An alias arguments vector to reuse - reset it before use...
@@ -447,16 +444,17 @@ void CommandInterpreter::Initialize() {
     CommandAlias *parray_alias =
         AddAlias("parray", cmd_obj_sp, "--element-count %1 --");
     if (parray_alias) {
-        parray_alias->SetHelp
-          ("parray <COUNT> <EXPRESSION> -- lldb will evaluate EXPRESSION "
-           "to get a typed-pointer-to-an-array in memory, and will display "
-           "COUNT elements of that type from the array.");
-        parray_alias->SetHelpLong("");
-    }
-    CommandAlias *poarray_alias = AddAlias("poarray", cmd_obj_sp,
-             "--object-description --element-count %1 --");
+      parray_alias->SetHelp(
+          "parray <COUNT> <EXPRESSION> -- lldb will evaluate EXPRESSION "
+          "to get a typed-pointer-to-an-array in memory, and will display "
+          "COUNT elements of that type from the array.");
+      parray_alias->SetHelpLong("");
+    }
+    CommandAlias *poarray_alias = AddAlias(
+        "poarray", cmd_obj_sp, "--object-description --element-count %1 --");
     if (poarray_alias) {
-      poarray_alias->SetHelp("poarray <COUNT> <EXPRESSION> -- lldb will "
+      poarray_alias->SetHelp(
+          "poarray <COUNT> <EXPRESSION> -- lldb will "
           "evaluate EXPRESSION to get the address of an array of COUNT "
           "objects in memory, and will call po on them.");
       poarray_alias->SetHelpLong("");
@@ -536,9 +534,7 @@ void CommandInterpreter::Initialize() {
   }
 }
 
-void CommandInterpreter::Clear() {
-  m_command_io_handler_sp.reset();
-}
+void CommandInterpreter::Clear() { m_command_io_handler_sp.reset(); }
 
 const char *CommandInterpreter::ProcessEmbeddedScriptCommands(const char *arg) {
   // This function has not yet been implemented.
@@ -851,9 +847,12 @@ void CommandInterpreter::LoadCommandDictionary() {
     // now "bt 3" is the preferred form, in line with gdb.
     if (bt_regex_cmd_up->AddRegexCommand("^([[:digit:]]+)[[:space:]]*$",
                                          "thread backtrace -c %1") &&
-        bt_regex_cmd_up->AddRegexCommand("^(-[^[:space:]].*)$", "thread backtrace %1") &&
-        bt_regex_cmd_up->AddRegexCommand("^all[[:space:]]*$", "thread backtrace all") &&
-        bt_regex_cmd_up->AddRegexCommand("^[[:space:]]*$", "thread backtrace")) {
+        bt_regex_cmd_up->AddRegexCommand("^(-[^[:space:]].*)$",
+                                         "thread backtrace %1") &&
+        bt_regex_cmd_up->AddRegexCommand("^all[[:space:]]*$",
+                                         "thread backtrace all") &&
+        bt_regex_cmd_up->AddRegexCommand("^[[:space:]]*$",
+                                         "thread backtrace")) {
       CommandObjectSP command_sp(bt_regex_cmd_up.release());
       m_command_dict[std::string(command_sp->GetCommandName())] = command_sp;
     }
@@ -954,13 +953,14 @@ int CommandInterpreter::GetCommandNamesMatchingPartialString(
   return matches.GetSize();
 }
 
-CommandObjectMultiword *CommandInterpreter::VerifyUserMultiwordCmdPath(
-    Args &path, bool leaf_is_command, Status &result) {
+CommandObjectMultiword *
+CommandInterpreter::VerifyUserMultiwordCmdPath(Args &path, bool leaf_is_command,
+                                               Status &result) {
   result.Clear();
 
   auto get_multi_or_report_error =
       [&result](CommandObjectSP cmd_sp,
-                           const char *name) -> CommandObjectMultiword * {
+                const char *name) -> CommandObjectMultiword * {
     if (!cmd_sp) {
       result = Status::FromErrorStringWithFormat(
           "Path component: '%s' not found", name);
@@ -1265,8 +1265,8 @@ CommandInterpreter::GetCommandObject(llvm::StringRef cmd_str,
   // Try to find a match among commands and aliases. Allowing inexact matches,
   // but perferring exact matches.
   return GetCommandSP(cmd_str, /*include_aliases=*/true, /*exact=*/false,
-                             matches, descriptions)
-                    .get();
+                      matches, descriptions)
+      .get();
 }
 
 CommandObject *CommandInterpreter::GetUserCommandObject(
@@ -1299,8 +1299,8 @@ CommandObject *CommandInterpreter::GetUserCommandObject(
   StringList tmp_list;
   StringList *matches_ptr = matches ? matches : &tmp_list;
   AddNamesMatchingPartialString(GetUserCommands(), cmd_str, *matches_ptr);
-  AddNamesMatchingPartialString(GetUserMultiwordCommands(),
-                                cmd_str, *matches_ptr);
+  AddNamesMatchingPartialString(GetUserMultiwordCommands(), cmd_str,
+                                *matches_ptr);
 
   return {};
 }
@@ -1798,8 +1798,7 @@ Status CommandInterpreter::PreprocessCommand(std::string &command) {
   return error;
 }
 
-Status
-CommandInterpreter::PreprocessToken(std::string &expr_str) {
+Status CommandInterpreter::PreprocessToken(std::string &expr_str) {
   Status error;
   ExecutionContext exe_ctx(GetExecutionContext());
 
@@ -1819,9 +1818,8 @@ CommandInterpreter::PreprocessToken(std::string &expr_str) {
   options.SetTryAllThreads(true);
   options.SetTimeout(std::nullopt);
 
-  ExpressionResults expr_result =
-      target.EvaluateExpression(expr_str.c_str(), exe_ctx.GetFramePtr(),
-                                expr_result_valobj_sp, options);
+  ExpressionResults expr_result = target.EvaluateExpression(
+      expr_str.c_str(), exe_ctx.GetFramePtr(), expr_result_valobj_sp, options);
 
   if (expr_result == eExpressionCompleted) {
     Scalar scalar;
@@ -1889,12 +1887,13 @@ bool CommandInterpreter::HandleCommand(const char *command_line,
   std::string real_original_command_string(command_string);
 
   Log *log = GetLog(LLDBLog::Commands);
-  llvm::PrettyStackTraceFormat stack_trace("HandleCommand(command = \"%s\")",
-                                   command_line);
-
   LLDB_LOGF(log, "Processing command: %s", command_line);
   LLDB_SCOPED_TIMERF("Processing command: %s.", command_line);
 
+  // Set the command in the CommandReturnObject here so that it's there even if
+  // the command is interrupted.
+  result.SetCommand(command_line);
+
   if (INTERRUPT_REQUESTED(GetDebugger(), "Interrupted initiating command")) {
     result.AppendError("... Interrupted");
     return false;
@@ -2011,7 +2010,8 @@ bool CommandInterpreter::HandleCommand(const char *command_line,
   // has the command expanded to the full name.  For example, if the input was
   // "br s -n main", command_string is now "breakpoint set -n main".
   if (log) {
-    llvm::StringRef command_name = cmd_obj ? cmd_obj->GetCommandName() : "<not found>";
+    llvm::StringRef command_name =
+        cmd_obj ? cmd_obj->GetCommandName() : "<not found>";
     LLDB_LOGF(log, "HandleCommand, cmd_obj : '%s'", command_name.str().c_str());
     LLDB_LOGF(log, "HandleCommand, (revised) command_string: '%s'",
               command_string.c_str());
@@ -2216,11 +2216,15 @@ CommandInterpreter::GetAlias(llvm::StringRef alias_name) const {
   return nullptr;
 }
 
-bool CommandInterpreter::HasCommands() const { return (!m_command_dict.empty()); }
+bool CommandInterpreter::HasCommands() const {
+  return (!m_command_dict.empty());
+}
 
 bool CommandInterpreter::HasAliases() const { return (!m_alias_dict.empty()); }
 
-bool CommandInterpreter::HasUserCommands() const { return (!m_user_dict.empty()); }
+bool CommandInterpreter::HasUserCommands() const {
+  return (!m_user_dict.empty());
+}
 
 bool CommandInterpreter::HasUserMultiwordCommands() const {
   return (!m_user_mw_dict.empty());
@@ -2578,20 +2582,18 @@ bool CommandInterpreter::DidProcessStopAbnormally() const {
   return false;
 }
 
-void
-CommandInterpreter::HandleCommands(const StringList &commands,
-                                   const ExecutionContext &override_context,
-                                   const CommandInterpreterRunOptions &options,
-                                   CommandReturnObject &result) {
+void CommandInterpreter::HandleCommands(
+    const StringList &commands, const ExecutionContext &override_context,
+    const CommandInterpreterRunOptions &options, CommandReturnObject &result) {
 
   OverrideExecutionContext(override_context);
   HandleCommands(commands, options, result);
   RestoreExecutionContext();
 }
 
-void CommandInterpreter::HandleCommands(const StringList &commands,
-                                        const CommandInterpreterRunOptions &options,
-                                        CommandReturnObject &result) {
+void CommandInterpreter::HandleCommands(
+    const StringList &commands, const CommandInterpreterRunOptions &options,
+    CommandReturnObject &result) {
   size_t num_lines = commands.GetSize();
 
   // If we are going to continue past a "continue" then we need to run the
@@ -2643,7 +2645,8 @@ void CommandInterpreter::HandleCommands(const StringList &commands,
                                       (uint64_t)idx, cmd, error_msg);
         m_debugger.SetAsyncExecution(old_async_execution);
         return;
-      } else if (options.GetPrintResults()) {
+      }
+      if (options.GetPrintResults()) {
         result.AppendMessageWithFormatv("Command #{0} '{1}' failed with {2}",
                                         (uint64_t)idx + 1, cmd, error_msg);
       }
@@ -2728,8 +2731,9 @@ void CommandInterpreter::HandleCommandsFromFile(
   RestoreExecutionContext();
 }
 
-void CommandInterpreter::HandleCommandsFromFile(FileSpec &cmd_file,
-    const CommandInterpreterRunOptions &options, CommandReturnObject &result) {
+void CommandInterpreter::HandleCommandsFromFile(
+    FileSpec &cmd_file, const CommandInterpreterRunOptions &options,
+    CommandReturnObject &result) {
   if (!FileSystem::Instance().Exists(cmd_file)) {
     result.AppendErrorWithFormat(
         "Error reading commands from file %s - file not found.\n",
@@ -3134,9 +3138,9 @@ bool CommandInterpreter::EchoCommandNonInteractive(
 
 void CommandInterpreter::IOHandlerInputComplete(IOHandler &io_handler,
                                                 std::string &line) {
-    // If we were interrupted, bail out...
-    if (WasInterrupted())
-      return;
+  // If we were interrupted, bail out...
+  if (WasInterrupted())
+    return;
 
   const bool is_interactive = io_handler.GetIsInteractive();
   const bool allow_repeats =
@@ -3182,30 +3186,40 @@ void CommandInterpreter::IOHandlerInputComplete(IOHandler &io_handler,
   if ((result.Succeeded() &&
        io_handler.GetFlags().Test(eHandleCommandFlagPrintResult)) ||
       io_handler.GetFlags().Test(eHandleCommandFlagPrintErrors)) {
-    // Display any inline diagnostics first.
-    const bool inline_diagnostics = !result.GetImmediateErrorStream() &&
-                                    GetDebugger().GetShowInlineDiagnostics();
-    if (inline_diagnostics) {
-      unsigned prompt_len = m_debugger.GetPrompt().size();
-      if (auto indent = result.GetDiagnosticIndent()) {
-        std::string diags =
-            result.GetInlineDiagnosticString(prompt_len + *indent);
-        PrintCommandOutput(io_handler, diags, true);
+    auto DefaultPrintCallback = [&](const CommandReturnObject &result) {
+      // Display any inline diagnostics first.
+      const bool inline_diagnostics = !result.GetImmediateErrorStream() &&
+                                      GetDebugger().GetShowInlineDiagnostics();
+      if (inline_diagnostics) {
+        unsigned prompt_len = m_debugger.GetPrompt().size();
+        if (auto indent = result.GetDiagnosticIndent()) {
+          std::string diags =
+              result.GetInlineDiagnosticString(prompt_len + *indent);
+          PrintCommandOutput(io_handler, diags, true);
+        }
       }
-    }
 
-    // Display any STDOUT/STDERR _prior_ to emitting the command result text.
-    GetProcessOutput();
+      // Display any STDOUT/STDERR _prior_ to emitting the command result text.
+      GetProcessOutput();
 
-    if (!result.GetImmediateOutputStream()) {
-      llvm::StringRef output = result.GetOutputString();
-      PrintCommandOutput(io_handler, output, true);
-    }
+      if (!result.GetImmediateOutputStream()) {
+        llvm::StringRef output = result.GetOutputString();
+        PrintCommandOutput(io_handler, output, true);
+      }
 
-    // Now emit the command error text from the command we just executed.
-    if (!result.GetImmediateErrorStream()) {
-      std::string error = result.GetErrorString(!inline_diagnostics);
-      PrintCommandOutput(io_handler, error, false);
+      // Now emit the command error text from the command we just executed.
+      if (!result.GetImmediateErrorStream()) {
+        std::string error = result.GetErrorString(!inline_diagnostics);
+        PrintCommandOutput(io_handler, error, false);
+      }
+    };
+
+    if (m_print_callback) {
+      const auto callback_result = m_print_callback(result);
+      if (callback_result == eCommandReturnObjectPrintCallbackSkipped)
+        DefaultPrintCallback(result);
+    } else {
+      DefaultPrintCallback(result);
     }
   }
 
@@ -3656,3 +3670,8 @@ llvm::json::Value CommandInterpreter::GetStatistics() {
 const StructuredData::Array &CommandInterpreter::GetTranscript() const {
   return m_transcript;
 }
+
+void CommandInterpreter::SetPrintCallback(
+    CommandReturnObjectCallback callback) {
+  m_print_callback = callback;
+}
diff --git a/lldb/source/Interpreter/CommandObject.cpp b/lldb/source/Interpreter/CommandObject.cpp
index 6b044a28eb37e..7008253e32bca 100644
--- a/lldb/source/Interpreter/CommandObject.cpp
+++ b/lldb/source/Interpreter/CommandObject.cpp
@@ -120,17 +120,24 @@ bool CommandObject::ParseOptions(Args &args, CommandReturnObject &result) {
     if (args_or) {
       args = std::move(*args_or);
       error = options->NotifyOptionParsingFinished(&exe_ctx);
-    } else
+    } else {
       error = Status::FromError(args_or.takeError());
+    }
 
-    if (error.Success()) {
-      if (options->VerifyOptions(result))
-        return true;
-    } else {
+    if (error.Fail()) {
       result.SetError(error.takeError());
+      result.SetStatus(eReturnStatusFailed);
+      return false;
     }
-    result.SetStatus(eReturnStatusFailed);
-    return false;
+
+    if (llvm::Error error = options->VerifyOptions()) {
+      result.SetError(std::move(error));
+      result.SetStatus(eReturnStatusFailed);
+      return false;
+    }
+
+    result.SetStatus(eReturnStatusSuccessFinishNoResult);
+    return true;
   }
   return true;
 }
@@ -278,7 +285,6 @@ void CommandObject::HandleCompletion(CompletionRequest &request) {
   } else {
     // Can we do anything generic with the options?
     Options *cur_options = GetOptions();
-    CommandReturnObject result(m_interpreter.GetDebugger().GetUseColor());
     OptionElementVector opt_element_vector;
 
     if (cur_options != nullptr) {
diff --git a/lldb/source/Interpreter/CommandReturnObject.cpp b/lldb/source/Interpreter/CommandReturnObject.cpp
index b99b2bc7b36ce..0a2948e8e6ca4 100644
--- a/lldb/source/Interpreter/CommandReturnObject.cpp
+++ b/lldb/source/Interpreter/CommandReturnObject.cpp
@@ -147,7 +147,8 @@ void CommandReturnObject::SetError(llvm::Error error) {
   }
 }
 
-std::string CommandReturnObject::GetInlineDiagnosticString(unsigned indent) {
+std::string
+CommandReturnObject::GetInlineDiagnosticString(unsigned indent) const {
   StreamString diag_stream(m_colors);
   RenderDiagnosticDetails(diag_stream, indent, true, m_diagnostics);
   // Duplex the diagnostics to the secondary stream (but not inlined).
@@ -157,7 +158,7 @@ std::string CommandReturnObject::GetInlineDiagnosticString(unsigned indent) {
   return diag_stream.GetString().str();
 }
 
-std::string CommandReturnObject::GetErrorString(bool with_diagnostics) {
+std::string CommandReturnObject::GetErrorString(bool with_diagnostics) const {
   StreamString stream(m_colors);
   if (with_diagnostics)
     RenderDiagnosticDetails(stream, std::nullopt, false, m_diagnostics);
diff --git a/lldb/source/Interpreter/OptionArgParser.cpp b/lldb/source/Interpreter/OptionArgParser.cpp
index 800f22b6169dc..2d393a57452ee 100644
--- a/lldb/source/Interpreter/OptionArgParser.cpp
+++ b/lldb/source/Interpreter/OptionArgParser.cpp
@@ -262,8 +262,10 @@ OptionArgParser::DoToAddress(const ExecutionContext *exe_ctx, llvm::StringRef s,
   // 3: The symbol/reg name if there is an offset
   // 4: +/-
   // 5: The offset value.
+  // clang-format off
   static RegularExpression g_symbol_plus_offset_regex(
-      "^(\\$[^ +-]+)|(([^ +-]+)([-\\+])[[:space:]]*(0x[0-9A-Fa-f]+|[0-9]+)[[:space:]]*)$");
+      "^(\\$[^ +-]+)|(([^ +-]+)[[:space:]]*([-\\+])[[:space:]]*(0x[0-9A-Fa-f]+|[0-9]+)[[:space:]]*)$");
+  // clang-format on
 
   llvm::SmallVector<llvm::StringRef, 4> matches;
   if (g_symbol_plus_offset_regex.Execute(sref, &matches)) {
diff --git a/lldb/source/Interpreter/Options.cpp b/lldb/source/Interpreter/Options.cpp
index 893a3b71604ba..fdadba62987d3 100644
--- a/lldb/source/Interpreter/Options.cpp
+++ b/lldb/source/Interpreter/Options.cpp
@@ -138,46 +138,6 @@ void Options::OptionsSetUnion(const OptionSet &set_a, const OptionSet &set_b,
   }
 }
 
-bool Options::VerifyOptions(CommandReturnObject &result) {
-  bool options_are_valid = false;
-
-  int num_levels = GetRequiredOptions().size();
-  if (num_levels) {
-    for (int i = 0; i < num_levels && !options_are_valid; ++i) {
-      // This is the correct set of options if:  1). m_seen_options contains
-      // all of m_required_options[i] (i.e. all the required options at this
-      // level are a subset of m_seen_options); AND 2). { m_seen_options -
-      // m_required_options[i] is a subset of m_options_options[i] (i.e. all
-      // the rest of m_seen_options are in the set of optional options at this
-      // level.
-
-      // Check to see if all of m_required_options[i] are a subset of
-      // m_seen_options
-      if (IsASubset(GetRequiredOptions()[i], m_seen_options)) {
-        // Construct the set difference: remaining_options = {m_seen_options} -
-        // {m_required_options[i]}
-        OptionSet remaining_options;
-        OptionsSetDiff(m_seen_options, GetRequiredOptions()[i],
-                       remaining_options);
-        // Check to see if remaining_options is a subset of
-        // m_optional_options[i]
-        if (IsASubset(remaining_options, GetOptionalOptions()[i]))
-          options_are_valid = true;
-      }
-    }
-  } else {
-    options_are_valid = true;
-  }
-
-  if (options_are_valid) {
-    result.SetStatus(eReturnStatusSuccessFinishNoResult);
-  } else {
-    result.AppendError("invalid combination of options for the given command");
-  }
-
-  return options_are_valid;
-}
-
 // This is called in the Options constructor, though we could call it lazily if
 // that ends up being a performance problem.
 
@@ -590,13 +550,50 @@ void Options::GenerateOptionUsage(Stream &strm, CommandObject &cmd,
   strm.SetIndentLevel(save_indent_level);
 }
 
+llvm::Error Options::VerifyOptions() {
+  bool options_are_valid = false;
+
+  int num_levels = GetRequiredOptions().size();
+  if (num_levels) {
+    for (int i = 0; i < num_levels && !options_are_valid; ++i) {
+      // This is the correct set of options if:  1). m_seen_options contains
+      // all of m_required_options[i] (i.e. all the required options at this
+      // level are a subset of m_seen_options); AND 2). { m_seen_options -
+      // m_required_options[i] is a subset of m_options_options[i] (i.e. all
+      // the rest of m_seen_options are in the set of optional options at this
+      // level.
+
+      // Check to see if all of m_required_options[i] are a subset of
+      // m_seen_options
+      if (IsASubset(GetRequiredOptions()[i], m_seen_options)) {
+        // Construct the set difference: remaining_options = {m_seen_options} -
+        // {m_required_options[i]}
+        OptionSet remaining_options;
+        OptionsSetDiff(m_seen_options, GetRequiredOptions()[i],
+                       remaining_options);
+        // Check to see if remaining_options is a subset of
+        // m_optional_options[i]
+        if (IsASubset(remaining_options, GetOptionalOptions()[i]))
+          options_are_valid = true;
+      }
+    }
+  } else {
+    options_are_valid = true;
+  }
+
+  if (!options_are_valid)
+    return llvm::createStringError(
+        "invalid combination of options for the given command");
+
+  return llvm::Error::success();
+}
+
 // This function is called when we have been given a potentially incomplete set
 // of options, such as when an alias has been defined (more options might be
 // added at at the time the alias is invoked).  We need to verify that the
 // options in the set m_seen_options are all part of a set that may be used
 // together, but m_seen_options may be missing some of the "required" options.
-
-bool Options::VerifyPartialOptions(CommandReturnObject &result) {
+llvm::Error Options::VerifyPartialOptions() {
   bool options_are_valid = false;
 
   int num_levels = GetRequiredOptions().size();
@@ -613,7 +610,11 @@ bool Options::VerifyPartialOptions(CommandReturnObject &result) {
     }
   }
 
-  return options_are_valid;
+  if (!options_are_valid)
+    return llvm::createStringError(
+        "invalid combination of options for the given command");
+
+  return llvm::Error::success();
 }
 
 bool Options::HandleOptionCompletion(CompletionRequest &request,
diff --git a/lldb/source/Plugins/ABI/AArch64/ABISysV_arm64.cpp b/lldb/source/Plugins/ABI/AArch64/ABISysV_arm64.cpp
index 93b8141e97ef8..74047ea65788c 100644
--- a/lldb/source/Plugins/ABI/AArch64/ABISysV_arm64.cpp
+++ b/lldb/source/Plugins/ABI/AArch64/ABISysV_arm64.cpp
@@ -60,6 +60,69 @@ ABISysV_arm64::CreateInstance(lldb::ProcessSP process_sp, const ArchSpec &arch)
   return ABISP();
 }
 
+static Status PushToLinuxGuardedControlStack(addr_t return_addr,
+                                             RegisterContext *reg_ctx,
+                                             Thread &thread) {
+  Status err;
+
+  // If the Guarded Control Stack extension is present we may need to put the
+  // return address onto that stack.
+  const RegisterInfo *gcs_features_enabled_info =
+      reg_ctx->GetRegisterInfoByName("gcs_features_enabled");
+  if (!gcs_features_enabled_info)
+    return err;
+
+  uint64_t gcs_features_enabled = reg_ctx->ReadRegisterAsUnsigned(
+      gcs_features_enabled_info, LLDB_INVALID_ADDRESS);
+  if (gcs_features_enabled == LLDB_INVALID_ADDRESS)
+    return Status("Could not read GCS features enabled register.");
+
+  // Only attempt this if GCS is enabled. If it's not enabled then gcspr_el0
+  // may point to unmapped memory.
+  if ((gcs_features_enabled & 1) == 0)
+    return err;
+
+  const RegisterInfo *gcspr_el0_info =
+      reg_ctx->GetRegisterInfoByName("gcspr_el0");
+  if (!gcspr_el0_info)
+    return Status("Could not get register info for gcspr_el0.");
+
+  uint64_t gcspr_el0 =
+      reg_ctx->ReadRegisterAsUnsigned(gcspr_el0_info, LLDB_INVALID_ADDRESS);
+  if (gcspr_el0 == LLDB_INVALID_ADDRESS)
+    return Status("Could not read gcspr_el0.");
+
+  // A link register entry on the GCS is 8 bytes.
+  gcspr_el0 -= 8;
+  if (!reg_ctx->WriteRegisterFromUnsigned(gcspr_el0_info, gcspr_el0))
+    return Status(
+        "Attempted to decrement gcspr_el0, but could not write to it.");
+
+  Status error;
+  size_t wrote = thread.GetProcess()->WriteMemory(gcspr_el0, &return_addr,
+                                                  sizeof(return_addr), error);
+  if ((wrote != sizeof(return_addr) || error.Fail())) {
+    // When PrepareTrivialCall fails, the register context is not restored,
+    // unlike when an expression fails to execute. This is arguably a bug,
+    // see https://github.com/llvm/llvm-project/issues/124269.
+    // For now we are handling this here specifically. We can assume this
+    // write will work as the one to decrement the register did.
+    reg_ctx->WriteRegisterFromUnsigned(gcspr_el0_info, gcspr_el0 + 8);
+    return Status("Failed to write new Guarded Control Stack entry.");
+  }
+
+  Log *log = GetLog(LLDBLog::Expressions);
+  LLDB_LOGF(log,
+            "Pushed return address 0x%" PRIx64 " to Guarded Control Stack. "
+            "gcspr_el0 was 0%" PRIx64 ", is now 0x%" PRIx64 ".",
+            return_addr, gcspr_el0 - 8, gcspr_el0);
+
+  // gcspr_el0 will be restored to the original value by lldb-server after
+  // the call has finished, which serves as the "pop".
+
+  return err;
+}
+
 bool ABISysV_arm64::PrepareTrivialCall(Thread &thread, addr_t sp,
                                        addr_t func_addr, addr_t return_addr,
                                        llvm::ArrayRef<addr_t> args) const {
@@ -87,6 +150,18 @@ bool ABISysV_arm64::PrepareTrivialCall(Thread &thread, addr_t sp,
   if (args.size() > 8)
     return false;
 
+  // Do this first, as it's got the most chance of failing (though still very
+  // low).
+  if (GetProcessSP()->GetTarget().GetArchitecture().GetTriple().isOSLinux()) {
+    Status err = PushToLinuxGuardedControlStack(return_addr, reg_ctx, thread);
+    // If we could not manage the GCS, the expression will certainly fail,
+    // and if we just carried on, that failure would be a lot more cryptic.
+    if (err.Fail()) {
+      LLDB_LOGF(log, "Failed to setup Guarded Call Stack: %s", err.AsCString());
+      return false;
+    }
+  }
+
   for (size_t i = 0; i < args.size(); ++i) {
     const RegisterInfo *reg_info = reg_ctx->GetRegisterInfo(
         eRegisterKindGeneric, LLDB_REGNUM_GENERIC_ARG1 + i);
diff --git a/lldb/source/Plugins/ABI/X86/ABISysV_x86_64.cpp b/lldb/source/Plugins/ABI/X86/ABISysV_x86_64.cpp
index 54028b1b3261a..83b01b14aedc5 100644
--- a/lldb/source/Plugins/ABI/X86/ABISysV_x86_64.cpp
+++ b/lldb/source/Plugins/ABI/X86/ABISysV_x86_64.cpp
@@ -79,6 +79,7 @@ ABISysV_x86_64::CreateInstance(lldb::ProcessSP process_sp, const ArchSpec &arch)
     case llvm::Triple::OSType::IOS:
     case llvm::Triple::OSType::TvOS:
     case llvm::Triple::OSType::WatchOS:
+    case llvm::Triple::OSType::XROS:
       switch (os_env) {
       case llvm::Triple::EnvironmentType::MacABI:
       case llvm::Triple::EnvironmentType::Simulator:
diff --git a/lldb/source/Plugins/DynamicLoader/Darwin-Kernel/DynamicLoaderDarwinKernel.cpp b/lldb/source/Plugins/DynamicLoader/Darwin-Kernel/DynamicLoaderDarwinKernel.cpp
index ab013e79047ea..b8941dae01078 100644
--- a/lldb/source/Plugins/DynamicLoader/Darwin-Kernel/DynamicLoaderDarwinKernel.cpp
+++ b/lldb/source/Plugins/DynamicLoader/Darwin-Kernel/DynamicLoaderDarwinKernel.cpp
@@ -168,8 +168,9 @@ DynamicLoader *DynamicLoaderDarwinKernel::CreateInstance(Process *process,
     case llvm::Triple::IOS:
     case llvm::Triple::TvOS:
     case llvm::Triple::WatchOS:
-    case llvm::Triple::XROS:
     case llvm::Triple::BridgeOS:
+    case llvm::Triple::DriverKit:
+    case llvm::Triple::XROS:
       if (triple_ref.getVendor() != llvm::Triple::Apple) {
         return nullptr;
       }
@@ -243,6 +244,7 @@ DynamicLoaderDarwinKernel::SearchForKernelWithDebugHints(Process *process) {
   Status read_err;
   addr_t kernel_addresses_64[] = {
       0xfffffff000002010ULL,
+      0xfffffe0000004010ULL, // newest arm64 devices, large memory support
       0xfffffff000004010ULL, // newest arm64 devices
       0xffffff8000004010ULL, // 2014-2015-ish arm64 devices
       0xffffff8000002010ULL, // oldest arm64 devices
@@ -1092,7 +1094,7 @@ void DynamicLoaderDarwinKernel::LoadKernelModuleIfNeeded() {
       static ConstString arm64_T1Sz_value("gT1Sz");
       const Symbol *symbol =
           m_kernel.GetModule()->FindFirstSymbolWithNameAndType(
-              kext_summary_symbol, eSymbolTypeData);
+              kext_summary_symbol, eSymbolTypeAny);
       if (symbol) {
         m_kext_summary_header_ptr_addr = symbol->GetAddress();
         // Update all image infos
diff --git a/lldb/source/Plugins/DynamicLoader/Hexagon-DYLD/DynamicLoaderHexagonDYLD.cpp b/lldb/source/Plugins/DynamicLoader/Hexagon-DYLD/DynamicLoaderHexagonDYLD.cpp
index 96c94535c623c..480500e034bc1 100644
--- a/lldb/source/Plugins/DynamicLoader/Hexagon-DYLD/DynamicLoaderHexagonDYLD.cpp
+++ b/lldb/source/Plugins/DynamicLoader/Hexagon-DYLD/DynamicLoaderHexagonDYLD.cpp
@@ -426,9 +426,7 @@ DynamicLoaderHexagonDYLD::GetStepThroughTrampolinePlan(Thread &thread,
   typedef std::vector<lldb::addr_t> AddressVector;
   AddressVector addrs;
   for (const SymbolContext &context : target_symbols) {
-    AddressRange range;
-    context.GetAddressRange(eSymbolContextEverything, 0, false, range);
-    lldb::addr_t addr = range.GetBaseAddress().GetLoadAddress(&target);
+    addr_t addr = context.GetFunctionOrSymbolAddress().GetLoadAddress(&target);
     if (addr != LLDB_INVALID_ADDRESS)
       addrs.push_back(addr);
   }
diff --git a/lldb/source/Plugins/DynamicLoader/MacOSX-DYLD/DynamicLoaderDarwin.cpp b/lldb/source/Plugins/DynamicLoader/MacOSX-DYLD/DynamicLoaderDarwin.cpp
index 8949b14c57189..f9b49c50355d5 100644
--- a/lldb/source/Plugins/DynamicLoader/MacOSX-DYLD/DynamicLoaderDarwin.cpp
+++ b/lldb/source/Plugins/DynamicLoader/MacOSX-DYLD/DynamicLoaderDarwin.cpp
@@ -419,6 +419,10 @@ bool DynamicLoaderDarwin::JSONImageInformationIntoImageInfo(
         image_infos[i].os_type = llvm::Triple::WatchOS;
       else if (os_name == "bridgeos")
         image_infos[i].os_type = llvm::Triple::BridgeOS;
+      else if (os_name == "driverkit")
+        image_infos[i].os_type = llvm::Triple::DriverKit;
+      else if (os_name == "xros")
+        image_infos[i].os_type = llvm::Triple::XROS;
       else if (os_name == "maccatalyst") {
         image_infos[i].os_type = llvm::Triple::IOS;
         image_infos[i].os_env = llvm::Triple::MacABI;
@@ -431,6 +435,9 @@ bool DynamicLoaderDarwin::JSONImageInformationIntoImageInfo(
       } else if (os_name == "watchossimulator") {
         image_infos[i].os_type = llvm::Triple::WatchOS;
         image_infos[i].os_env = llvm::Triple::Simulator;
+      } else if (os_name == "xrsimulator") {
+        image_infos[i].os_type = llvm::Triple::XROS;
+        image_infos[i].os_env = llvm::Triple::Simulator;
       }
     }
     if (image->HasKey("min_version_os_sdk")) {
@@ -765,7 +772,8 @@ bool DynamicLoaderDarwin::AddModulesUsingPreloadedModules(
           (dyld_triple.getEnvironment() == llvm::Triple::Simulator &&
            (dyld_triple.getOS() == llvm::Triple::IOS ||
             dyld_triple.getOS() == llvm::Triple::TvOS ||
-            dyld_triple.getOS() == llvm::Triple::WatchOS)))
+            dyld_triple.getOS() == llvm::Triple::WatchOS ||
+            dyld_triple.getOS() == llvm::Triple::XROS)))
         image_module_sp->MergeArchitecture(dyld_spec);
     }
   }
@@ -835,7 +843,7 @@ lldb_private::ArchSpec DynamicLoaderDarwin::ImageInfo::GetArchitecture() const {
   }
   if (os_env == llvm::Triple::Simulator &&
       (os_type == llvm::Triple::IOS || os_type == llvm::Triple::TvOS ||
-       os_type == llvm::Triple::WatchOS)) {
+       os_type == llvm::Triple::WatchOS || os_type == llvm::Triple::XROS)) {
     llvm::Triple triple(llvm::Twine(arch_spec.GetArchitectureName()) +
                         "-apple-" + llvm::Triple::getOSTypeName(os_type) +
                         min_version_os_sdk + "-simulator");
@@ -939,13 +947,10 @@ DynamicLoaderDarwin::GetStepThroughTrampolinePlan(Thread &thread,
         images.FindSymbolsWithNameAndType(trampoline_name, eSymbolTypeCode,
                                           code_symbols);
         for (const SymbolContext &context : code_symbols) {
-          AddressRange addr_range;
-          context.GetAddressRange(eSymbolContextEverything, 0, false,
-                                  addr_range);
-          addresses.push_back(addr_range.GetBaseAddress());
+          Address addr = context.GetFunctionOrSymbolAddress();
+          addresses.push_back(addr);
           if (log) {
-            addr_t load_addr =
-                addr_range.GetBaseAddress().GetLoadAddress(target_sp.get());
+            addr_t load_addr = addr.GetLoadAddress(target_sp.get());
 
             LLDB_LOGF(log, "Found a trampoline target symbol at 0x%" PRIx64 ".",
                       load_addr);
@@ -980,13 +985,10 @@ DynamicLoaderDarwin::GetStepThroughTrampolinePlan(Thread &thread,
                                           indirect_symbols);
 
         for (const SymbolContext &context : indirect_symbols) {
-          AddressRange addr_range;
-          context.GetAddressRange(eSymbolContextEverything, 0, false,
-                                  addr_range);
-          addresses.push_back(addr_range.GetBaseAddress());
+          Address addr = context.GetFunctionOrSymbolAddress();
+          addresses.push_back(addr);
           if (log) {
-            addr_t load_addr =
-                addr_range.GetBaseAddress().GetLoadAddress(target_sp.get());
+            addr_t load_addr = addr.GetLoadAddress(target_sp.get());
 
             LLDB_LOGF(log, "Found an indirect target symbol at 0x%" PRIx64 ".",
                       load_addr);
@@ -1214,35 +1216,46 @@ DynamicLoaderDarwin::GetThreadLocalData(const lldb::ModuleSP module_sp,
 
 bool DynamicLoaderDarwin::UseDYLDSPI(Process *process) {
   Log *log = GetLog(LLDBLog::DynamicLoader);
-  bool use_new_spi_interface = false;
+  bool use_new_spi_interface = true;
 
   llvm::VersionTuple version = process->GetHostOSVersion();
   if (!version.empty()) {
-    const llvm::Triple::OSType os_type =
+    using namespace llvm;
+    const Triple::OSType os_type =
         process->GetTarget().GetArchitecture().GetTriple().getOS();
 
-    // macOS 10.12 and newer
-    if (os_type == llvm::Triple::MacOSX &&
-        version >= llvm::VersionTuple(10, 12))
-      use_new_spi_interface = true;
+    auto OlderThan = [os_type, version](llvm::Triple::OSType o,
+                                        llvm::VersionTuple v) -> bool {
+      return os_type == o && version < v;
+    };
+
+    if (OlderThan(Triple::MacOSX, VersionTuple(10, 12)))
+      use_new_spi_interface = false;
 
-    // iOS 10 and newer
-    if (os_type == llvm::Triple::IOS && version >= llvm::VersionTuple(10))
-      use_new_spi_interface = true;
+    if (OlderThan(Triple::IOS, VersionTuple(10)))
+      use_new_spi_interface = false;
 
-    // tvOS 10 and newer
-    if (os_type == llvm::Triple::TvOS && version >= llvm::VersionTuple(10))
-      use_new_spi_interface = true;
+    if (OlderThan(Triple::TvOS, VersionTuple(10)))
+      use_new_spi_interface = false;
 
-    // watchOS 3 and newer
-    if (os_type == llvm::Triple::WatchOS && version >= llvm::VersionTuple(3))
-      use_new_spi_interface = true;
+    if (OlderThan(Triple::WatchOS, VersionTuple(3)))
+      use_new_spi_interface = false;
 
-    // NEED_BRIDGEOS_TRIPLE // Any BridgeOS
-    // NEED_BRIDGEOS_TRIPLE if (os_type == llvm::Triple::BridgeOS)
-    // NEED_BRIDGEOS_TRIPLE   use_new_spi_interface = true;
+    // llvm::Triple::BridgeOS and llvm::Triple::XROS always use the new
+    // libdyld SPI interface.
+  } else {
+    // We could not get an OS version string, we are likely not
+    // connected to debugserver and the packets to call the libdyld SPI
+    // will not exist.
+    use_new_spi_interface = false;
   }
 
+  // Corefiles cannot use the libdyld SPI to get the inferior's
+  // binaries, we must find it through metadata or a scan
+  // of the corefile memory.
+  if (!process->IsLiveDebugSession())
+    use_new_spi_interface = false;
+
   if (log) {
     if (use_new_spi_interface)
       LLDB_LOGF(
diff --git a/lldb/source/Plugins/DynamicLoader/MacOSX-DYLD/DynamicLoaderMacOS.cpp b/lldb/source/Plugins/DynamicLoader/MacOSX-DYLD/DynamicLoaderMacOS.cpp
index 5b11059bcc50c..08bef4999eb9a 100644
--- a/lldb/source/Plugins/DynamicLoader/MacOSX-DYLD/DynamicLoaderMacOS.cpp
+++ b/lldb/source/Plugins/DynamicLoader/MacOSX-DYLD/DynamicLoaderMacOS.cpp
@@ -55,8 +55,9 @@ DynamicLoader *DynamicLoaderMacOS::CreateInstance(Process *process,
       case llvm::Triple::IOS:
       case llvm::Triple::TvOS:
       case llvm::Triple::WatchOS:
-      case llvm::Triple::XROS:
       case llvm::Triple::BridgeOS:
+      case llvm::Triple::DriverKit:
+      case llvm::Triple::XROS:
         create = triple_ref.getVendor() == llvm::Triple::Apple;
         break;
       default:
diff --git a/lldb/source/Plugins/DynamicLoader/MacOSX-DYLD/DynamicLoaderMacOSXDYLD.cpp b/lldb/source/Plugins/DynamicLoader/MacOSX-DYLD/DynamicLoaderMacOSXDYLD.cpp
index 8fc77cbe11701..b05ed1ce2c823 100644
--- a/lldb/source/Plugins/DynamicLoader/MacOSX-DYLD/DynamicLoaderMacOSXDYLD.cpp
+++ b/lldb/source/Plugins/DynamicLoader/MacOSX-DYLD/DynamicLoaderMacOSXDYLD.cpp
@@ -75,8 +75,9 @@ DynamicLoader *DynamicLoaderMacOSXDYLD::CreateInstance(Process *process,
       case llvm::Triple::IOS:
       case llvm::Triple::TvOS:
       case llvm::Triple::WatchOS:
-      case llvm::Triple::XROS:
       case llvm::Triple::BridgeOS:
+      case llvm::Triple::DriverKit:
+      case llvm::Triple::XROS:
         create = triple_ref.getVendor() == llvm::Triple::Apple;
         break;
       default:
diff --git a/lldb/source/Plugins/DynamicLoader/POSIX-DYLD/DynamicLoaderPOSIXDYLD.cpp b/lldb/source/Plugins/DynamicLoader/POSIX-DYLD/DynamicLoaderPOSIXDYLD.cpp
index 5614bc32468d5..53ba11ac21bd3 100644
--- a/lldb/source/Plugins/DynamicLoader/POSIX-DYLD/DynamicLoaderPOSIXDYLD.cpp
+++ b/lldb/source/Plugins/DynamicLoader/POSIX-DYLD/DynamicLoaderPOSIXDYLD.cpp
@@ -512,9 +512,7 @@ DynamicLoaderPOSIXDYLD::GetStepThroughTrampolinePlan(Thread &thread,
   typedef std::vector<lldb::addr_t> AddressVector;
   AddressVector addrs;
   for (const SymbolContext &context : target_symbols) {
-    AddressRange range;
-    context.GetAddressRange(eSymbolContextEverything, 0, false, range);
-    lldb::addr_t addr = range.GetBaseAddress().GetLoadAddress(&target);
+    addr_t addr = context.GetFunctionOrSymbolAddress().GetLoadAddress(&target);
     if (addr != LLDB_INVALID_ADDRESS)
       addrs.push_back(addr);
   }
diff --git a/lldb/source/Plugins/ExpressionParser/Clang/CxxModuleHandler.cpp b/lldb/source/Plugins/ExpressionParser/Clang/CxxModuleHandler.cpp
index c201153fd7ceb..3ae32d4afd27a 100644
--- a/lldb/source/Plugins/ExpressionParser/Clang/CxxModuleHandler.cpp
+++ b/lldb/source/Plugins/ExpressionParser/Clang/CxxModuleHandler.cpp
@@ -280,7 +280,8 @@ std::optional<Decl *> CxxModuleHandler::tryInstantiateStdTemplate(Decl *d) {
       new_class_template->getDeclContext(),
       new_class_template->getTemplatedDecl()->getLocation(),
       new_class_template->getLocation(), new_class_template, imported_args,
-      nullptr);
+      td->hasStrictPackMatch(),
+      /*PrevDecl=*/nullptr);
 
   new_class_template->AddSpecialization(result, InsertPos);
   if (new_class_template->isOutOfLine())
diff --git a/lldb/source/Plugins/ExpressionParser/Clang/IRForTarget.cpp b/lldb/source/Plugins/ExpressionParser/Clang/IRForTarget.cpp
index 6c728f3447489..a414ad652448e 100644
--- a/lldb/source/Plugins/ExpressionParser/Clang/IRForTarget.cpp
+++ b/lldb/source/Plugins/ExpressionParser/Clang/IRForTarget.cpp
@@ -66,7 +66,7 @@ static llvm::Value *FindEntryInstruction(llvm::Function *function) {
   if (function->empty())
     return nullptr;
 
-  return function->getEntryBlock().getFirstNonPHIOrDbg();
+  return &*function->getEntryBlock().getFirstNonPHIOrDbg();
 }
 
 IRForTarget::IRForTarget(lldb_private::ClangExpressionDeclMap *decl_map,
@@ -361,7 +361,7 @@ bool IRForTarget::CreateResultVariable(llvm::Function &llvm_function) {
     // there's nothing to put into its equivalent persistent variable.
 
     BasicBlock &entry_block(llvm_function.getEntryBlock());
-    Instruction *first_entry_instruction(entry_block.getFirstNonPHIOrDbg());
+    Instruction *first_entry_instruction(&*entry_block.getFirstNonPHIOrDbg());
 
     if (!first_entry_instruction)
       return false;
@@ -1505,7 +1505,7 @@ bool IRForTarget::ReplaceVariables(Function &llvm_function) {
   LLDB_LOG(log, "Arg: \"{0}\"", PrintValue(argument));
 
   BasicBlock &entry_block(llvm_function.getEntryBlock());
-  Instruction *FirstEntryInstruction(entry_block.getFirstNonPHIOrDbg());
+  Instruction *FirstEntryInstruction(&*entry_block.getFirstNonPHIOrDbg());
 
   if (!FirstEntryInstruction) {
     m_error_stream.Printf("Internal error [IRForTarget]: Couldn't find the "
diff --git a/lldb/source/Plugins/Language/CPlusPlus/CPlusPlusLanguage.cpp b/lldb/source/Plugins/Language/CPlusPlus/CPlusPlusLanguage.cpp
index 2bf574e97768e..4b045d12ad494 100644
--- a/lldb/source/Plugins/Language/CPlusPlus/CPlusPlusLanguage.cpp
+++ b/lldb/source/Plugins/Language/CPlusPlus/CPlusPlusLanguage.cpp
@@ -641,7 +641,7 @@ static void LoadLibCxxFormatters(lldb::TypeCategoryImplSP cpp_category_sp) {
       .SetSkipPointers(false)
       .SetSkipReferences(false)
       .SetDontShowChildren(true)
-      .SetDontShowValue(true)
+      .SetDontShowValue(false)
       .SetShowMembersOneLiner(false)
       .SetHideItemNames(false);
 
@@ -1204,7 +1204,7 @@ static void LoadLibStdcppFormatters(lldb::TypeCategoryImplSP cpp_category_sp) {
       .SetSkipPointers(false)
       .SetSkipReferences(false)
       .SetDontShowChildren(true)
-      .SetDontShowValue(true)
+      .SetDontShowValue(false)
       .SetShowMembersOneLiner(false)
       .SetHideItemNames(false);
 
diff --git a/lldb/source/Plugins/Language/CPlusPlus/LibCxx.cpp b/lldb/source/Plugins/Language/CPlusPlus/LibCxx.cpp
index 6d0ccdbbe4a71..2aa8fdba70634 100644
--- a/lldb/source/Plugins/Language/CPlusPlus/LibCxx.cpp
+++ b/lldb/source/Plugins/Language/CPlusPlus/LibCxx.cpp
@@ -430,12 +430,6 @@ size_t lldb_private::formatters::LibcxxUniquePtrSyntheticFrontEnd::
 
 bool lldb_private::formatters::LibcxxContainerSummaryProvider(
     ValueObject &valobj, Stream &stream, const TypeSummaryOptions &options) {
-  if (valobj.IsPointerType()) {
-    uint64_t value = valobj.GetValueAsUnsigned(0);
-    if (!value)
-      return false;
-    stream.Printf("0x%016" PRIx64 " ", value);
-  }
   return FormatEntity::FormatStringRef("size=${svar%#}", stream, nullptr,
                                        nullptr, nullptr, &valobj, false, false);
 }
diff --git a/lldb/source/Plugins/LanguageRuntime/CPlusPlus/CPPLanguageRuntime.cpp b/lldb/source/Plugins/LanguageRuntime/CPlusPlus/CPPLanguageRuntime.cpp
index fb706544ea560..42fa54634841c 100644
--- a/lldb/source/Plugins/LanguageRuntime/CPlusPlus/CPPLanguageRuntime.cpp
+++ b/lldb/source/Plugins/LanguageRuntime/CPlusPlus/CPPLanguageRuntime.cpp
@@ -142,10 +142,7 @@ line_entry_helper(Target &target, const SymbolContext &sc, Symbol *symbol,
 
   CPPLanguageRuntime::LibCppStdFunctionCallableInfo optional_info;
 
-  AddressRange range;
-  sc.GetAddressRange(eSymbolContextEverything, 0, false, range);
-
-  Address address = range.GetBaseAddress();
+  Address address = sc.GetFunctionOrSymbolAddress();
 
   Address addr;
   if (target.ResolveLoadAddress(address.GetCallableLoadAddress(&target),
diff --git a/lldb/source/Plugins/LanguageRuntime/CPlusPlus/ItaniumABI/ItaniumABILanguageRuntime.cpp b/lldb/source/Plugins/LanguageRuntime/CPlusPlus/ItaniumABI/ItaniumABILanguageRuntime.cpp
index 66cdab1307ce9..8faf7135217ac 100644
--- a/lldb/source/Plugins/LanguageRuntime/CPlusPlus/ItaniumABI/ItaniumABILanguageRuntime.cpp
+++ b/lldb/source/Plugins/LanguageRuntime/CPlusPlus/ItaniumABI/ItaniumABILanguageRuntime.cpp
@@ -289,7 +289,7 @@ llvm::Expected<LanguageRuntime::VTableInfo>
 bool ItaniumABILanguageRuntime::GetDynamicTypeAndAddress(
     ValueObject &in_value, lldb::DynamicValueType use_dynamic,
     TypeAndOrName &class_type_or_name, Address &dynamic_address,
-    Value::ValueType &value_type) {
+    Value::ValueType &value_type, llvm::ArrayRef<uint8_t> &local_buffer) {
   // For Itanium, if the type has a vtable pointer in the object, it will be at
   // offset 0 in the object.  That will point to the "address point" within the
   // vtable (not the beginning of the vtable.)  We can then look up the symbol
diff --git a/lldb/source/Plugins/LanguageRuntime/CPlusPlus/ItaniumABI/ItaniumABILanguageRuntime.h b/lldb/source/Plugins/LanguageRuntime/CPlusPlus/ItaniumABI/ItaniumABILanguageRuntime.h
index 0f7e73cfee075..7abf2f8547cd5 100644
--- a/lldb/source/Plugins/LanguageRuntime/CPlusPlus/ItaniumABI/ItaniumABILanguageRuntime.h
+++ b/lldb/source/Plugins/LanguageRuntime/CPlusPlus/ItaniumABI/ItaniumABILanguageRuntime.h
@@ -54,8 +54,8 @@ class ItaniumABILanguageRuntime : public lldb_private::CPPLanguageRuntime {
   bool GetDynamicTypeAndAddress(ValueObject &in_value,
                                 lldb::DynamicValueType use_dynamic,
                                 TypeAndOrName &class_type_or_name,
-                                Address &address,
-                                Value::ValueType &value_type) override;
+                                Address &address, Value::ValueType &value_type,
+                                llvm::ArrayRef<uint8_t> &local_buffer) override;
 
   TypeAndOrName FixUpDynamicType(const TypeAndOrName &type_and_or_name,
                                  ValueObject &static_value) override;
diff --git a/lldb/source/Plugins/LanguageRuntime/ObjC/AppleObjCRuntime/AppleObjCRuntime.cpp b/lldb/source/Plugins/LanguageRuntime/ObjC/AppleObjCRuntime/AppleObjCRuntime.cpp
index ceee19c136d25..ad60290382c02 100644
--- a/lldb/source/Plugins/LanguageRuntime/ObjC/AppleObjCRuntime/AppleObjCRuntime.cpp
+++ b/lldb/source/Plugins/LanguageRuntime/ObjC/AppleObjCRuntime/AppleObjCRuntime.cpp
@@ -276,7 +276,7 @@ bool AppleObjCRuntime::CouldHaveDynamicValue(ValueObject &in_value) {
 bool AppleObjCRuntime::GetDynamicTypeAndAddress(
     ValueObject &in_value, lldb::DynamicValueType use_dynamic,
     TypeAndOrName &class_type_or_name, Address &address,
-    Value::ValueType &value_type) {
+    Value::ValueType &value_type, llvm::ArrayRef<uint8_t> &local_buffer) {
   return false;
 }
 
diff --git a/lldb/source/Plugins/LanguageRuntime/ObjC/AppleObjCRuntime/AppleObjCRuntime.h b/lldb/source/Plugins/LanguageRuntime/ObjC/AppleObjCRuntime/AppleObjCRuntime.h
index da58d44db19a8..425a608d65c2c 100644
--- a/lldb/source/Plugins/LanguageRuntime/ObjC/AppleObjCRuntime/AppleObjCRuntime.h
+++ b/lldb/source/Plugins/LanguageRuntime/ObjC/AppleObjCRuntime/AppleObjCRuntime.h
@@ -54,8 +54,8 @@ class AppleObjCRuntime : public lldb_private::ObjCLanguageRuntime {
   bool GetDynamicTypeAndAddress(ValueObject &in_value,
                                 lldb::DynamicValueType use_dynamic,
                                 TypeAndOrName &class_type_or_name,
-                                Address &address,
-                                Value::ValueType &value_type) override;
+                                Address &address, Value::ValueType &value_type,
+                                llvm::ArrayRef<uint8_t> &local_buffer) override;
 
   TypeAndOrName FixUpDynamicType(const TypeAndOrName &type_and_or_name,
                                  ValueObject &static_value) override;
diff --git a/lldb/source/Plugins/LanguageRuntime/ObjC/AppleObjCRuntime/AppleObjCRuntimeV1.cpp b/lldb/source/Plugins/LanguageRuntime/ObjC/AppleObjCRuntime/AppleObjCRuntimeV1.cpp
index 93168c23f3547..db1317d70d060 100644
--- a/lldb/source/Plugins/LanguageRuntime/ObjC/AppleObjCRuntime/AppleObjCRuntimeV1.cpp
+++ b/lldb/source/Plugins/LanguageRuntime/ObjC/AppleObjCRuntime/AppleObjCRuntimeV1.cpp
@@ -48,7 +48,7 @@ AppleObjCRuntimeV1::AppleObjCRuntimeV1(Process *process)
 bool AppleObjCRuntimeV1::GetDynamicTypeAndAddress(
     ValueObject &in_value, lldb::DynamicValueType use_dynamic,
     TypeAndOrName &class_type_or_name, Address &address,
-    Value::ValueType &value_type) {
+    Value::ValueType &value_type, llvm::ArrayRef<uint8_t> &local_buffer) {
   class_type_or_name.Clear();
   value_type = Value::ValueType::Scalar;
   if (CouldHaveDynamicValue(in_value)) {
diff --git a/lldb/source/Plugins/LanguageRuntime/ObjC/AppleObjCRuntime/AppleObjCRuntimeV1.h b/lldb/source/Plugins/LanguageRuntime/ObjC/AppleObjCRuntime/AppleObjCRuntimeV1.h
index 46d8e89c906e3..c51ac24e690b8 100644
--- a/lldb/source/Plugins/LanguageRuntime/ObjC/AppleObjCRuntime/AppleObjCRuntimeV1.h
+++ b/lldb/source/Plugins/LanguageRuntime/ObjC/AppleObjCRuntime/AppleObjCRuntimeV1.h
@@ -100,8 +100,8 @@ class AppleObjCRuntimeV1 : public AppleObjCRuntime {
   bool GetDynamicTypeAndAddress(ValueObject &in_value,
                                 lldb::DynamicValueType use_dynamic,
                                 TypeAndOrName &class_type_or_name,
-                                Address &address,
-                                Value::ValueType &value_type) override;
+                                Address &address, Value::ValueType &value_type,
+                                llvm::ArrayRef<uint8_t> &local_buffer) override;
 
   llvm::Expected<std::unique_ptr<UtilityFunction>>
   CreateObjectChecker(std::string, ExecutionContext &exe_ctx) override;
diff --git a/lldb/source/Plugins/LanguageRuntime/ObjC/AppleObjCRuntime/AppleObjCRuntimeV2.cpp b/lldb/source/Plugins/LanguageRuntime/ObjC/AppleObjCRuntime/AppleObjCRuntimeV2.cpp
index c43871b08191d..a57099f3df454 100644
--- a/lldb/source/Plugins/LanguageRuntime/ObjC/AppleObjCRuntime/AppleObjCRuntimeV2.cpp
+++ b/lldb/source/Plugins/LanguageRuntime/ObjC/AppleObjCRuntime/AppleObjCRuntimeV2.cpp
@@ -770,7 +770,7 @@ AppleObjCRuntimeV2::GetPreferredLanguageRuntime(ValueObject &in_value) {
 bool AppleObjCRuntimeV2::GetDynamicTypeAndAddress(
     ValueObject &in_value, lldb::DynamicValueType use_dynamic,
     TypeAndOrName &class_type_or_name, Address &address,
-    Value::ValueType &value_type) {
+    Value::ValueType &value_type, llvm::ArrayRef<uint8_t> &local_buffer) {
   // We should never get here with a null process...
   assert(m_process != nullptr);
 
diff --git a/lldb/source/Plugins/LanguageRuntime/ObjC/AppleObjCRuntime/AppleObjCRuntimeV2.h b/lldb/source/Plugins/LanguageRuntime/ObjC/AppleObjCRuntime/AppleObjCRuntimeV2.h
index 2422539b13f13..79840f9be79b3 100644
--- a/lldb/source/Plugins/LanguageRuntime/ObjC/AppleObjCRuntime/AppleObjCRuntimeV2.h
+++ b/lldb/source/Plugins/LanguageRuntime/ObjC/AppleObjCRuntime/AppleObjCRuntimeV2.h
@@ -53,8 +53,8 @@ class AppleObjCRuntimeV2 : public AppleObjCRuntime {
   bool GetDynamicTypeAndAddress(ValueObject &in_value,
                                 lldb::DynamicValueType use_dynamic,
                                 TypeAndOrName &class_type_or_name,
-                                Address &address,
-                                Value::ValueType &value_type) override;
+                                Address &address, Value::ValueType &value_type,
+                                llvm::ArrayRef<uint8_t> &local_buffer) override;
 
   llvm::Expected<std::unique_ptr<UtilityFunction>>
   CreateObjectChecker(std::string name, ExecutionContext &exe_ctx) override;
diff --git a/lldb/source/Plugins/LanguageRuntime/ObjC/GNUstepObjCRuntime/GNUstepObjCRuntime.cpp b/lldb/source/Plugins/LanguageRuntime/ObjC/GNUstepObjCRuntime/GNUstepObjCRuntime.cpp
index d6ffb03ab55e2..a4b3e26474a55 100644
--- a/lldb/source/Plugins/LanguageRuntime/ObjC/GNUstepObjCRuntime/GNUstepObjCRuntime.cpp
+++ b/lldb/source/Plugins/LanguageRuntime/ObjC/GNUstepObjCRuntime/GNUstepObjCRuntime.cpp
@@ -127,7 +127,7 @@ bool GNUstepObjCRuntime::CouldHaveDynamicValue(ValueObject &in_value) {
 bool GNUstepObjCRuntime::GetDynamicTypeAndAddress(
     ValueObject &in_value, DynamicValueType use_dynamic,
     TypeAndOrName &class_type_or_name, Address &address,
-    Value::ValueType &value_type) {
+    Value::ValueType &value_type, llvm::ArrayRef<uint8_t> &local_buffer) {
   return false;
 }
 
diff --git a/lldb/source/Plugins/LanguageRuntime/ObjC/GNUstepObjCRuntime/GNUstepObjCRuntime.h b/lldb/source/Plugins/LanguageRuntime/ObjC/GNUstepObjCRuntime/GNUstepObjCRuntime.h
index de24466ebb003..94a5c9e1261a8 100644
--- a/lldb/source/Plugins/LanguageRuntime/ObjC/GNUstepObjCRuntime/GNUstepObjCRuntime.h
+++ b/lldb/source/Plugins/LanguageRuntime/ObjC/GNUstepObjCRuntime/GNUstepObjCRuntime.h
@@ -67,8 +67,8 @@ class GNUstepObjCRuntime : public lldb_private::ObjCLanguageRuntime {
   bool GetDynamicTypeAndAddress(ValueObject &in_value,
                                 lldb::DynamicValueType use_dynamic,
                                 TypeAndOrName &class_type_or_name,
-                                Address &address,
-                                Value::ValueType &value_type) override;
+                                Address &address, Value::ValueType &value_type,
+                                llvm::ArrayRef<uint8_t> &local_buffer) override;
 
   TypeAndOrName FixUpDynamicType(const TypeAndOrName &type_and_or_name,
                                  ValueObject &static_value) override;
diff --git a/lldb/source/Plugins/ObjectFile/Mach-O/ObjectFileMachO.cpp b/lldb/source/Plugins/ObjectFile/Mach-O/ObjectFileMachO.cpp
index bf2d293d2012c..4e356a7c8f5d9 100644
--- a/lldb/source/Plugins/ObjectFile/Mach-O/ObjectFileMachO.cpp
+++ b/lldb/source/Plugins/ObjectFile/Mach-O/ObjectFileMachO.cpp
@@ -2750,7 +2750,7 @@ void ObjectFileMachO::ParseSymtab(Symtab &symtab) {
         return;
 
         if (process_shared_cache_uuid.IsValid() &&
-          process_shared_cache_uuid != UUID::fromData(&cache_uuid, 16))
+          process_shared_cache_uuid != UUID(&cache_uuid, 16))
         return;
 
       dyld_shared_cache_for_each_image(shared_cache, ^(dyld_image_t image) {
@@ -2759,7 +2759,7 @@ void ObjectFileMachO::ParseSymtab(Symtab &symtab) {
           return;
 
         dyld_image_copy_uuid(image, &dsc_image_uuid);
-        if (image_uuid != UUID::fromData(dsc_image_uuid, 16))
+        if (image_uuid != UUID(dsc_image_uuid, 16))
           return;
 
         found_image = true;
@@ -2848,7 +2848,7 @@ void ObjectFileMachO::ParseSymtab(Symtab &symtab) {
                           "DSC unmapped local symbol[{0}] has invalid "
                           "string table offset {1:x} in {2}, ignoring symbol",
                           nlist_index, nlist.n_strx,
-                          module_sp->GetFileSpec().GetPath());
+                          module_sp->GetFileSpec().GetPath()));
                       continue;
                     }
                     if (symbol_name[0] == '\0')
@@ -6557,9 +6557,8 @@ bool ObjectFileMachO::SaveCore(const lldb::ProcessSP &process_sp,
        target_triple.getOS() == llvm::Triple::IOS ||
        target_triple.getOS() == llvm::Triple::WatchOS ||
        target_triple.getOS() == llvm::Triple::TvOS ||
+       target_triple.getOS() == llvm::Triple::BridgeOS ||
        target_triple.getOS() == llvm::Triple::XROS)) {
-    // NEED_BRIDGEOS_TRIPLE target_triple.getOS() == llvm::Triple::BridgeOS))
-    // {
     bool make_core = false;
     switch (target_arch.GetMachine()) {
     case llvm::Triple::aarch64:
diff --git a/lldb/source/Plugins/Platform/MacOSX/PlatformDarwinKernel.cpp b/lldb/source/Plugins/Platform/MacOSX/PlatformDarwinKernel.cpp
index 6f75e5ea70b6a..4fbead97e9c1a 100644
--- a/lldb/source/Plugins/Platform/MacOSX/PlatformDarwinKernel.cpp
+++ b/lldb/source/Plugins/Platform/MacOSX/PlatformDarwinKernel.cpp
@@ -126,8 +126,10 @@ PlatformSP PlatformDarwinKernel::CreateInstance(bool force,
       case llvm::Triple::MacOSX:
       case llvm::Triple::IOS:
       case llvm::Triple::WatchOS:
+      case llvm::Triple::XROS:
       case llvm::Triple::TvOS:
       case llvm::Triple::BridgeOS:
+      case llvm::Triple::DriverKit:
         break;
       // Only accept "vendor" for vendor if the host is Apple and it "unknown"
       // wasn't specified (it was just returned because it was NOT specified)
@@ -329,6 +331,8 @@ void PlatformDarwinKernel::CollectKextAndKernelDirectories() {
                                "/Platforms/AppleTVOS.platform/Developer/SDKs");
     AddSDKSubdirsToSearchPaths(developer_dir +
                                "/Platforms/WatchOS.platform/Developer/SDKs");
+    AddSDKSubdirsToSearchPaths(developer_dir +
+                               "/Platforms/XROS.platform/Developer/SDKs");
     AddSDKSubdirsToSearchPaths(developer_dir +
                                "/Platforms/BridgeOS.platform/Developer/SDKs");
   }
diff --git a/lldb/source/Plugins/Process/Linux/NativeRegisterContextLinux_arm64.cpp b/lldb/source/Plugins/Process/Linux/NativeRegisterContextLinux_arm64.cpp
index efd3385c46e92..884c7d4b9e359 100644
--- a/lldb/source/Plugins/Process/Linux/NativeRegisterContextLinux_arm64.cpp
+++ b/lldb/source/Plugins/Process/Linux/NativeRegisterContextLinux_arm64.cpp
@@ -1063,9 +1063,27 @@ Status NativeRegisterContextLinux_arm64::WriteAllRegisterValues(
           std::bind(&NativeRegisterContextLinux_arm64::WriteFPMR, this));
       break;
     case RegisterSetType::GCS:
+      // It is not permitted to enable GCS via ptrace. We can disable it, but
+      // to keep things simple we will not revert any change to the
+      // PR_SHADOW_STACK_ENABLE bit. Instead patch in the current enable bit
+      // into the registers we are about to restore.
+      m_gcs_is_valid = false;
+      error = ReadGCS();
+      if (error.Fail())
+        return error;
+
+      uint64_t enable_bit = m_gcs_regs.features_enabled & 1UL;
+      gcs_regs new_gcs_regs = *reinterpret_cast<const gcs_regs *>(src);
+      new_gcs_regs.features_enabled =
+          (new_gcs_regs.features_enabled & ~1UL) | enable_bit;
+
+      const uint8_t *new_gcs_src =
+          reinterpret_cast<const uint8_t *>(&new_gcs_regs);
       error = RestoreRegisters(
-          GetGCSBuffer(), &src, GetGCSBufferSize(), m_gcs_is_valid,
+          GetGCSBuffer(), &new_gcs_src, GetGCSBufferSize(), m_gcs_is_valid,
           std::bind(&NativeRegisterContextLinux_arm64::WriteGCS, this));
+      src += GetGCSBufferSize();
+
       break;
     }
 
diff --git a/lldb/source/Plugins/Process/Linux/NativeRegisterContextLinux_loongarch64.cpp b/lldb/source/Plugins/Process/Linux/NativeRegisterContextLinux_loongarch64.cpp
index b04018ee243fd..c4841950f1e07 100644
--- a/lldb/source/Plugins/Process/Linux/NativeRegisterContextLinux_loongarch64.cpp
+++ b/lldb/source/Plugins/Process/Linux/NativeRegisterContextLinux_loongarch64.cpp
@@ -27,18 +27,46 @@
 // struct iovec definition
 #include <sys/uio.h>
 
+// LoongArch SIMD eXtension registers
 #ifndef NT_LOONGARCH_LSX
-#define NT_LOONGARCH_LSX 0xa02 /* LoongArch SIMD eXtension registers */
+#define NT_LOONGARCH_LSX 0xa02
 #endif
 
+// LoongArch Advanced SIMD eXtension registers
 #ifndef NT_LOONGARCH_LASX
-#define NT_LOONGARCH_LASX                                                      \
-  0xa03 /* LoongArch Advanced SIMD eXtension registers */
+#define NT_LOONGARCH_LASX 0xa03
+#endif
+
+// LoongArch hardware breakpoint registers
+#ifndef NT_LOONGARCH_HW_BREAK
+#define NT_LOONGARCH_HW_BREAK 0xa05
+#endif
+
+// LoongArch hardware watchpoint registers
+#ifndef NT_LOONGARCH_HW_WATCH
+#define NT_LOONGARCH_HW_WATCH 0xa06
 #endif
 
 #define REG_CONTEXT_SIZE                                                       \
   (GetGPRSize() + GetFPRSize() + sizeof(m_lsx) + sizeof(m_lasx))
 
+// ptrace has a struct type user_watch_state, which was replaced by
+// user_watch_state_v2 when more watchpoints were added, so this file
+// may be built on systems with one or both in the system headers.
+// The type below has the same layout as user_watch_state_v2 but will
+// not clash with that name if it exists. We can use the v2 layout even
+// on old kernels as we will only see 8 watchpoints and the kernel will
+// truncate any extra data we send to it.
+struct loongarch_user_watch_state {
+  uint64_t dbg_info;
+  struct {
+    uint64_t addr;
+    uint64_t mask;
+    uint32_t ctrl;
+    uint32_t pad;
+  } dbg_regs[14];
+};
+
 using namespace lldb;
 using namespace lldb_private;
 using namespace lldb_private::process_linux;
@@ -528,7 +556,7 @@ llvm::Error NativeRegisterContextLinux_loongarch64::ReadHardwareDebugInfo() {
 
   int regset = NT_LOONGARCH_HW_WATCH;
   struct iovec ioVec;
-  struct user_watch_state dreg_state;
+  struct loongarch_user_watch_state dreg_state;
   Status error;
 
   ioVec.iov_base = &dreg_state;
@@ -556,7 +584,7 @@ llvm::Error NativeRegisterContextLinux_loongarch64::ReadHardwareDebugInfo() {
 llvm::Error NativeRegisterContextLinux_loongarch64::WriteHardwareDebugRegs(
     DREGType hwbType) {
   struct iovec ioVec;
-  struct user_watch_state dreg_state;
+  struct loongarch_user_watch_state dreg_state;
   int regset;
 
   memset(&dreg_state, 0, sizeof(dreg_state));
diff --git a/lldb/source/Plugins/Process/Utility/InferiorCallPOSIX.cpp b/lldb/source/Plugins/Process/Utility/InferiorCallPOSIX.cpp
index 99426c7d1f261..5226b9bbe7c78 100644
--- a/lldb/source/Plugins/Process/Utility/InferiorCallPOSIX.cpp
+++ b/lldb/source/Plugins/Process/Utility/InferiorCallPOSIX.cpp
@@ -52,9 +52,6 @@ bool lldb_private::InferiorCallMmap(Process *process, addr_t &allocated_addr,
   if (count > 0) {
     SymbolContext sc;
     if (sc_list.GetContextAtIndex(0, sc)) {
-      const uint32_t range_scope =
-          eSymbolContextFunction | eSymbolContextSymbol;
-      const bool use_inline_block_range = false;
       EvaluateExpressionOptions options;
       options.SetStopOthers(true);
       options.SetUnwindOnError(true);
@@ -77,9 +74,8 @@ bool lldb_private::InferiorCallMmap(Process *process, addr_t &allocated_addr,
           prot_arg |= PROT_WRITE;
       }
 
-      AddressRange mmap_range;
-      if (sc.GetAddressRange(range_scope, 0, use_inline_block_range,
-                             mmap_range)) {
+      Address mmap_addr = sc.GetFunctionOrSymbolAddress();
+      if (mmap_addr.IsValid()) {
         auto type_system_or_err =
             process->GetTarget().GetScratchTypeSystemForLanguage(
                 eLanguageTypeC);
@@ -96,9 +92,8 @@ bool lldb_private::InferiorCallMmap(Process *process, addr_t &allocated_addr,
         MmapArgList args =
             process->GetTarget().GetPlatform()->GetMmapArgumentList(
                 arch, addr, length, prot_arg, flags, fd, offset);
-        lldb::ThreadPlanSP call_plan_sp(
-            new ThreadPlanCallFunction(*thread, mmap_range.GetBaseAddress(),
-                                       void_ptr_type, args, options));
+        lldb::ThreadPlanSP call_plan_sp(new ThreadPlanCallFunction(
+            *thread, mmap_addr, void_ptr_type, args, options));
         if (call_plan_sp) {
           DiagnosticManager diagnostics;
 
@@ -149,9 +144,6 @@ bool lldb_private::InferiorCallMunmap(Process *process, addr_t addr,
   if (count > 0) {
     SymbolContext sc;
     if (sc_list.GetContextAtIndex(0, sc)) {
-      const uint32_t range_scope =
-          eSymbolContextFunction | eSymbolContextSymbol;
-      const bool use_inline_block_range = false;
       EvaluateExpressionOptions options;
       options.SetStopOthers(true);
       options.SetUnwindOnError(true);
@@ -161,13 +153,11 @@ bool lldb_private::InferiorCallMunmap(Process *process, addr_t addr,
       options.SetTimeout(process->GetUtilityExpressionTimeout());
       options.SetTrapExceptions(false);
 
-      AddressRange munmap_range;
-      if (sc.GetAddressRange(range_scope, 0, use_inline_block_range,
-                             munmap_range)) {
+      Address munmap_addr = sc.GetFunctionOrSymbolAddress();
+      if (munmap_addr.IsValid()) {
         lldb::addr_t args[] = {addr, length};
-        lldb::ThreadPlanSP call_plan_sp(
-            new ThreadPlanCallFunction(*thread, munmap_range.GetBaseAddress(),
-                                       CompilerType(), args, options));
+        lldb::ThreadPlanSP call_plan_sp(new ThreadPlanCallFunction(
+            *thread, munmap_addr, CompilerType(), args, options));
         if (call_plan_sp) {
           DiagnosticManager diagnostics;
 
diff --git a/lldb/source/Plugins/Process/Utility/RegisterFlagsDetector_arm64.cpp b/lldb/source/Plugins/Process/Utility/RegisterFlagsDetector_arm64.cpp
index 9f82c935c0e7e..042940b7dff6e 100644
--- a/lldb/source/Plugins/Process/Utility/RegisterFlagsDetector_arm64.cpp
+++ b/lldb/source/Plugins/Process/Utility/RegisterFlagsDetector_arm64.cpp
@@ -17,6 +17,7 @@
 #define HWCAP_ASIMDHP (1ULL << 10)
 #define HWCAP_DIT (1ULL << 24)
 #define HWCAP_SSBS (1ULL << 28)
+#define HWCAP_GCS (1ULL << 32)
 
 #define HWCAP2_BTI (1ULL << 17)
 #define HWCAP2_MTE (1ULL << 18)
@@ -50,6 +51,21 @@ Arm64RegisterFlagsDetector::DetectFPMRFields(uint64_t hwcap, uint64_t hwcap2) {
   };
 }
 
+Arm64RegisterFlagsDetector::Fields
+Arm64RegisterFlagsDetector::DetectGCSFeatureFields(uint64_t hwcap,
+                                                   uint64_t hwcap2) {
+  (void)hwcap2;
+
+  if (!(hwcap & HWCAP_GCS))
+    return {};
+
+  return {
+      {"PUSH", 2},
+      {"WRITE", 1},
+      {"ENABLE", 0},
+  };
+}
+
 Arm64RegisterFlagsDetector::Fields
 Arm64RegisterFlagsDetector::DetectSVCRFields(uint64_t hwcap, uint64_t hwcap2) {
   (void)hwcap;
diff --git a/lldb/source/Plugins/Process/Utility/RegisterFlagsDetector_arm64.h b/lldb/source/Plugins/Process/Utility/RegisterFlagsDetector_arm64.h
index 0f3d53d93892b..7daebcc71db04 100644
--- a/lldb/source/Plugins/Process/Utility/RegisterFlagsDetector_arm64.h
+++ b/lldb/source/Plugins/Process/Utility/RegisterFlagsDetector_arm64.h
@@ -61,6 +61,7 @@ class Arm64RegisterFlagsDetector {
   static Fields DetectMTECtrlFields(uint64_t hwcap, uint64_t hwcap2);
   static Fields DetectSVCRFields(uint64_t hwcap, uint64_t hwcap2);
   static Fields DetectFPMRFields(uint64_t hwcap, uint64_t hwcap2);
+  static Fields DetectGCSFeatureFields(uint64_t hwcap, uint64_t hwcap2);
 
   struct RegisterEntry {
     RegisterEntry(llvm::StringRef name, unsigned size, DetectorFn detector)
@@ -70,13 +71,15 @@ class Arm64RegisterFlagsDetector {
     llvm::StringRef m_name;
     RegisterFlags m_flags;
     DetectorFn m_detector;
-  } m_registers[6] = {
+  } m_registers[8] = {
       RegisterEntry("cpsr", 4, DetectCPSRFields),
       RegisterEntry("fpsr", 4, DetectFPSRFields),
       RegisterEntry("fpcr", 4, DetectFPCRFields),
       RegisterEntry("mte_ctrl", 8, DetectMTECtrlFields),
       RegisterEntry("svcr", 8, DetectSVCRFields),
       RegisterEntry("fpmr", 8, DetectFPMRFields),
+      RegisterEntry("gcs_features_enabled", 8, DetectGCSFeatureFields),
+      RegisterEntry("gcs_features_locked", 8, DetectGCSFeatureFields),
   };
 
   // Becomes true once field detection has been run for all registers.
diff --git a/lldb/source/Plugins/Process/Utility/ThreadMemory.h b/lldb/source/Plugins/Process/Utility/ThreadMemory.h
index d124f5780ea9b..cebb31538eaf2 100644
--- a/lldb/source/Plugins/Process/Utility/ThreadMemory.h
+++ b/lldb/source/Plugins/Process/Utility/ThreadMemory.h
@@ -33,7 +33,7 @@ class ThreadMemory : public lldb_private::Thread {
 
   const char *GetInfo() override {
     if (m_backing_thread_sp)
-      m_backing_thread_sp->GetInfo();
+      return m_backing_thread_sp->GetInfo();
     return nullptr;
   }
 
@@ -41,7 +41,7 @@ class ThreadMemory : public lldb_private::Thread {
     if (!m_name.empty())
       return m_name.c_str();
     if (m_backing_thread_sp)
-      m_backing_thread_sp->GetName();
+      return m_backing_thread_sp->GetName();
     return nullptr;
   }
 
@@ -49,7 +49,7 @@ class ThreadMemory : public lldb_private::Thread {
     if (!m_queue.empty())
       return m_queue.c_str();
     if (m_backing_thread_sp)
-      m_backing_thread_sp->GetQueueName();
+      return m_backing_thread_sp->GetQueueName();
     return nullptr;
   }
 
@@ -72,12 +72,17 @@ class ThreadMemory : public lldb_private::Thread {
 
   void ClearStackFrames() override;
 
-  void ClearBackingThread() override { m_backing_thread_sp.reset(); }
+  void ClearBackingThread() override {
+    if (m_backing_thread_sp)
+      m_backing_thread_sp->ClearBackedThread();
+    m_backing_thread_sp.reset();
+  }
 
   bool SetBackingThread(const lldb::ThreadSP &thread_sp) override {
     // printf ("Thread 0x%llx is being backed by thread 0x%llx\n", GetID(),
     // thread_sp->GetID());
     m_backing_thread_sp = thread_sp;
+    thread_sp->SetBackedThread(*this);
     return (bool)thread_sp;
   }
 
diff --git a/lldb/source/Plugins/Process/Windows/Common/NativeProcessWindows.cpp b/lldb/source/Plugins/Process/Windows/Common/NativeProcessWindows.cpp
index 9c330ff118670..79dd46ba319d6 100644
--- a/lldb/source/Plugins/Process/Windows/Common/NativeProcessWindows.cpp
+++ b/lldb/source/Plugins/Process/Windows/Common/NativeProcessWindows.cpp
@@ -292,7 +292,8 @@ NativeProcessWindows::GetAuxvData() const {
 
 llvm::Expected<llvm::ArrayRef<uint8_t>>
 NativeProcessWindows::GetSoftwareBreakpointTrapOpcode(size_t size_hint) {
-  static const uint8_t g_aarch64_opcode[] = {0x00, 0x00, 0x3e, 0xd4}; // brk #0xf000
+  static const uint8_t g_aarch64_opcode[] = {0x00, 0x00, 0x3e,
+                                             0xd4};     // brk #0xf000
   static const uint8_t g_thumb_opcode[] = {0xfe, 0xde}; // udf #0xfe
 
   switch (GetArchitecture().GetMachine()) {
@@ -309,9 +310,9 @@ NativeProcessWindows::GetSoftwareBreakpointTrapOpcode(size_t size_hint) {
 }
 
 size_t NativeProcessWindows::GetSoftwareBreakpointPCOffset() {
-    // Windows always reports an incremented PC after a breakpoint is hit,
-    // even on ARM.
-    return cantFail(GetSoftwareBreakpointTrapOpcode(0)).size();
+  // Windows always reports an incremented PC after a breakpoint is hit,
+  // even on ARM.
+  return cantFail(GetSoftwareBreakpointTrapOpcode(0)).size();
 }
 
 bool NativeProcessWindows::FindSoftwareBreakpoint(lldb::addr_t addr) {
@@ -463,6 +464,7 @@ NativeProcessWindows::OnDebugException(bool first_chance,
   switch (record.GetExceptionCode()) {
   case DWORD(STATUS_SINGLE_STEP):
   case STATUS_WX86_SINGLE_STEP: {
+#ifndef __aarch64__
     uint32_t wp_id = LLDB_INVALID_INDEX32;
     if (NativeThreadWindows *thread = GetThreadByID(record.GetThreadID())) {
       NativeRegisterContextWindows &reg_ctx = thread->GetRegisterContext();
@@ -483,6 +485,7 @@ NativeProcessWindows::OnDebugException(bool first_chance,
       }
     }
     if (wp_id == LLDB_INVALID_INDEX32)
+#endif
       StopThread(record.GetThreadID(), StopReason::eStopReasonTrace);
 
     SetState(eStateStopped, true);
@@ -492,23 +495,50 @@ NativeProcessWindows::OnDebugException(bool first_chance,
   }
   case DWORD(STATUS_BREAKPOINT):
   case STATUS_WX86_BREAKPOINT:
-    if (FindSoftwareBreakpoint(record.GetExceptionAddress())) {
-      LLDB_LOG(log, "Hit non-loader breakpoint at address {0:x}.",
-               record.GetExceptionAddress());
 
-      StopThread(record.GetThreadID(), StopReason::eStopReasonBreakpoint);
+    if (NativeThreadWindows *stop_thread =
+            GetThreadByID(record.GetThreadID())) {
+      auto &reg_ctx = stop_thread->GetRegisterContext();
+      const auto exception_addr = record.GetExceptionAddress();
+      const auto thread_id = record.GetThreadID();
 
-      if (NativeThreadWindows *stop_thread =
-              GetThreadByID(record.GetThreadID())) {
-        auto &register_context = stop_thread->GetRegisterContext();
-        uint32_t breakpoint_size = GetSoftwareBreakpointPCOffset();
+      if (FindSoftwareBreakpoint(exception_addr)) {
+        LLDB_LOG(log, "Hit non-loader breakpoint at address {0:x}.",
+                 exception_addr);
         // The current PC is AFTER the BP opcode, on all architectures.
-        uint64_t pc = register_context.GetPC() - breakpoint_size;
-        register_context.SetPC(pc);
+        reg_ctx.SetPC(reg_ctx.GetPC() - GetSoftwareBreakpointPCOffset());
+        StopThread(thread_id, StopReason::eStopReasonBreakpoint);
+        SetState(eStateStopped, true);
+        return ExceptionResult::MaskException;
+      } else {
+        // This block of code will only be entered in case of a hardware
+        // watchpoint or breakpoint hit on AArch64. However, we only handle
+        // hardware watchpoints below as breakpoints are not yet supported.
+        const std::vector<ULONG_PTR> &args = record.GetExceptionArguments();
+        // Check that the ExceptionInformation array of EXCEPTION_RECORD
+        // contains at least two elements: the first is a read-write flag
+        // indicating the type of data access operation (read or write) while
+        // the second contains the virtual address of the accessed data.
+        if (args.size() >= 2) {
+          uint32_t hw_id = LLDB_INVALID_INDEX32;
+          Status error = reg_ctx.GetWatchpointHitIndex(hw_id, args[1]);
+          if (error.Fail())
+            LLDB_LOG(log,
+                     "received error while checking for watchpoint hits, pid = "
+                     "{0}, error = {1}",
+                     thread_id, error);
+
+          if (hw_id != LLDB_INVALID_INDEX32) {
+            std::string desc =
+                formatv("{0} {1} {2}", reg_ctx.GetWatchpointAddress(hw_id),
+                        hw_id, exception_addr)
+                    .str();
+            StopThread(thread_id, StopReason::eStopReasonWatchpoint, desc);
+            SetState(eStateStopped, true);
+            return ExceptionResult::MaskException;
+          }
+        }
       }
-
-      SetState(eStateStopped, true);
-      return ExceptionResult::MaskException;
     }
 
     if (!initial_stop) {
diff --git a/lldb/source/Plugins/Process/Windows/Common/NativeRegisterContextWindows.cpp b/lldb/source/Plugins/Process/Windows/Common/NativeRegisterContextWindows.cpp
index 9128363eaa577..effe6df36b576 100644
--- a/lldb/source/Plugins/Process/Windows/Common/NativeRegisterContextWindows.cpp
+++ b/lldb/source/Plugins/Process/Windows/Common/NativeRegisterContextWindows.cpp
@@ -18,10 +18,6 @@
 using namespace lldb;
 using namespace lldb_private;
 
-NativeRegisterContextWindows::NativeRegisterContextWindows(
-    NativeThreadProtocol &thread, RegisterInfoInterface *reg_info_interface_p)
-    : NativeRegisterContextRegisterInfo(thread, reg_info_interface_p) {}
-
 lldb::thread_t NativeRegisterContextWindows::GetThreadHandle() const {
   auto wthread = static_cast<NativeThreadWindows *>(&m_thread);
   return wthread->GetHostThread().GetNativeThread().GetSystemHandle();
diff --git a/lldb/source/Plugins/Process/Windows/Common/NativeRegisterContextWindows.h b/lldb/source/Plugins/Process/Windows/Common/NativeRegisterContextWindows.h
index 841b8547f3e90..2b71f639d562e 100644
--- a/lldb/source/Plugins/Process/Windows/Common/NativeRegisterContextWindows.h
+++ b/lldb/source/Plugins/Process/Windows/Common/NativeRegisterContextWindows.h
@@ -17,16 +17,17 @@ namespace lldb_private {
 
 class NativeThreadWindows;
 
-class NativeRegisterContextWindows : public NativeRegisterContextRegisterInfo {
+class NativeRegisterContextWindows
+    : public virtual NativeRegisterContextRegisterInfo {
 public:
-  NativeRegisterContextWindows(
-      NativeThreadProtocol &native_thread,
-      RegisterInfoInterface *reg_info_interface_p);
-
   static std::unique_ptr<NativeRegisterContextWindows>
   CreateHostNativeRegisterContextWindows(const ArchSpec &target_arch,
                                          NativeThreadProtocol &native_thread);
 
+  // MSVC compiler deletes the default constructor due to virtual inheritance.
+  // Explicitly defining it ensures the class remains constructible.
+  NativeRegisterContextWindows() {}
+
 protected:
   lldb::thread_t GetThreadHandle() const;
 };
diff --git a/lldb/source/Plugins/Process/Windows/Common/NativeRegisterContextWindows_WoW64.cpp b/lldb/source/Plugins/Process/Windows/Common/NativeRegisterContextWindows_WoW64.cpp
index a9642d1c5e48d..069c327ee2f9a 100644
--- a/lldb/source/Plugins/Process/Windows/Common/NativeRegisterContextWindows_WoW64.cpp
+++ b/lldb/source/Plugins/Process/Windows/Common/NativeRegisterContextWindows_WoW64.cpp
@@ -88,8 +88,8 @@ static Status SetWoW64ThreadContextHelper(lldb::thread_t thread_handle,
 
 NativeRegisterContextWindows_WoW64::NativeRegisterContextWindows_WoW64(
     const ArchSpec &target_arch, NativeThreadProtocol &native_thread)
-    : NativeRegisterContextWindows(native_thread,
-                                   CreateRegisterInfoInterface(target_arch)) {}
+    : NativeRegisterContextRegisterInfo(
+          native_thread, CreateRegisterInfoInterface(target_arch)) {}
 
 bool NativeRegisterContextWindows_WoW64::IsGPR(uint32_t reg_index) const {
   return (reg_index >= k_first_gpr_i386 && reg_index < k_first_alias_i386);
diff --git a/lldb/source/Plugins/Process/Windows/Common/NativeRegisterContextWindows_arm.cpp b/lldb/source/Plugins/Process/Windows/Common/NativeRegisterContextWindows_arm.cpp
index 4209fdf3c7109..fd8a0c05c1b25 100644
--- a/lldb/source/Plugins/Process/Windows/Common/NativeRegisterContextWindows_arm.cpp
+++ b/lldb/source/Plugins/Process/Windows/Common/NativeRegisterContextWindows_arm.cpp
@@ -128,8 +128,8 @@ NativeRegisterContextWindows::CreateHostNativeRegisterContextWindows(
 
 NativeRegisterContextWindows_arm::NativeRegisterContextWindows_arm(
     const ArchSpec &target_arch, NativeThreadProtocol &native_thread)
-    : NativeRegisterContextWindows(native_thread,
-                                   CreateRegisterInfoInterface(target_arch)) {}
+    : NativeRegisterContextRegisterInfo(
+          native_thread, CreateRegisterInfoInterface(target_arch)) {}
 
 bool NativeRegisterContextWindows_arm::IsGPR(uint32_t reg_index) const {
   return (reg_index >= k_first_gpr_arm && reg_index <= k_last_gpr_arm);
diff --git a/lldb/source/Plugins/Process/Windows/Common/NativeRegisterContextWindows_arm64.cpp b/lldb/source/Plugins/Process/Windows/Common/NativeRegisterContextWindows_arm64.cpp
index 080a9140e36a6..d065ad6957f7d 100644
--- a/lldb/source/Plugins/Process/Windows/Common/NativeRegisterContextWindows_arm64.cpp
+++ b/lldb/source/Plugins/Process/Windows/Common/NativeRegisterContextWindows_arm64.cpp
@@ -10,7 +10,6 @@
 
 #include "NativeRegisterContextWindows_arm64.h"
 #include "NativeThreadWindows.h"
-#include "Plugins/Process/Utility/RegisterInfoPOSIX_arm64.h"
 #include "ProcessWindowsLog.h"
 #include "lldb/Host/HostInfo.h"
 #include "lldb/Host/HostThread.h"
@@ -143,8 +142,13 @@ NativeRegisterContextWindows::CreateHostNativeRegisterContextWindows(
 
 NativeRegisterContextWindows_arm64::NativeRegisterContextWindows_arm64(
     const ArchSpec &target_arch, NativeThreadProtocol &native_thread)
-    : NativeRegisterContextWindows(native_thread,
-                                   CreateRegisterInfoInterface(target_arch)) {}
+    : NativeRegisterContextRegisterInfo(
+          native_thread, CreateRegisterInfoInterface(target_arch)) {
+  // Currently, there is no API to query the maximum supported hardware
+  // breakpoints and watchpoints on Windows. The values set below are based
+  // on tests conducted on Windows 11 with Snapdragon Elite X hardware.
+  m_max_hwp_supported = 1;
+}
 
 bool NativeRegisterContextWindows_arm64::IsGPR(uint32_t reg_index) const {
   return (reg_index >= k_first_gpr_arm64 && reg_index <= k_last_gpr_arm64);
@@ -709,48 +713,37 @@ Status NativeRegisterContextWindows_arm64::WriteAllRegisterValues(
   return SetThreadContextHelper(GetThreadHandle(), &tls_context);
 }
 
-Status NativeRegisterContextWindows_arm64::IsWatchpointHit(uint32_t wp_index,
-                                                           bool &is_hit) {
-  return Status::FromErrorString("unimplemented");
-}
-
-Status NativeRegisterContextWindows_arm64::GetWatchpointHitIndex(
-    uint32_t &wp_index, lldb::addr_t trap_addr) {
-  return Status::FromErrorString("unimplemented");
-}
-
-Status NativeRegisterContextWindows_arm64::IsWatchpointVacant(uint32_t wp_index,
-                                                              bool &is_vacant) {
-  return Status::FromErrorString("unimplemented");
-}
-
-Status NativeRegisterContextWindows_arm64::SetHardwareWatchpointWithIndex(
-    lldb::addr_t addr, size_t size, uint32_t watch_flags, uint32_t wp_index) {
-  return Status::FromErrorString("unimplemented");
-}
+llvm::Error NativeRegisterContextWindows_arm64::ReadHardwareDebugInfo() {
+  ::CONTEXT tls_context;
+  Status error = GetThreadContextHelper(GetThreadHandle(), &tls_context,
+                                        CONTEXT_DEBUG_REGISTERS);
+  if (error.Fail())
+    return error.ToError();
 
-bool NativeRegisterContextWindows_arm64::ClearHardwareWatchpoint(
-    uint32_t wp_index) {
-  return false;
-}
+  for (uint32_t i = 0; i < m_max_hwp_supported; i++) {
+    m_hwp_regs[i].address = tls_context.Wvr[i];
+    m_hwp_regs[i].control = tls_context.Wcr[i];
+  }
 
-Status NativeRegisterContextWindows_arm64::ClearAllHardwareWatchpoints() {
-  return Status::FromErrorString("unimplemented");
+  return llvm::Error::success();
 }
 
-uint32_t NativeRegisterContextWindows_arm64::SetHardwareWatchpoint(
-    lldb::addr_t addr, size_t size, uint32_t watch_flags) {
-  return LLDB_INVALID_INDEX32;
-}
+llvm::Error
+NativeRegisterContextWindows_arm64::WriteHardwareDebugRegs(DREGType hwbType) {
+  ::CONTEXT tls_context;
+  Status error = GetThreadContextHelper(GetThreadHandle(), &tls_context,
+                                        CONTEXT_DEBUG_REGISTERS);
+  if (error.Fail())
+    return error.ToError();
 
-lldb::addr_t
-NativeRegisterContextWindows_arm64::GetWatchpointAddress(uint32_t wp_index) {
-  return LLDB_INVALID_ADDRESS;
-}
+  if (hwbType == eDREGTypeWATCH) {
+    for (uint32_t i = 0; i < m_max_hwp_supported; i++) {
+      tls_context.Wvr[i] = m_hwp_regs[i].address;
+      tls_context.Wcr[i] = m_hwp_regs[i].control;
+    }
+  }
 
-uint32_t NativeRegisterContextWindows_arm64::NumSupportedHardwareWatchpoints() {
-  // Not implemented
-  return 0;
+  return SetThreadContextHelper(GetThreadHandle(), &tls_context).ToError();
 }
 
 #endif // defined(__aarch64__) || defined(_M_ARM64)
diff --git a/lldb/source/Plugins/Process/Windows/Common/NativeRegisterContextWindows_arm64.h b/lldb/source/Plugins/Process/Windows/Common/NativeRegisterContextWindows_arm64.h
index 88afc1e7b18a2..e73a6af4cbf80 100644
--- a/lldb/source/Plugins/Process/Windows/Common/NativeRegisterContextWindows_arm64.h
+++ b/lldb/source/Plugins/Process/Windows/Common/NativeRegisterContextWindows_arm64.h
@@ -10,6 +10,8 @@
 #ifndef liblldb_NativeRegisterContextWindows_arm64_h_
 #define liblldb_NativeRegisterContextWindows_arm64_h_
 
+#include "Plugins/Process/Utility/NativeRegisterContextDBReg_arm64.h"
+#include "Plugins/Process/Utility/RegisterInfoPOSIX_arm64.h"
 #include "Plugins/Process/Utility/lldb-arm64-register-enums.h"
 
 #include "NativeRegisterContextWindows.h"
@@ -18,7 +20,9 @@ namespace lldb_private {
 
 class NativeThreadWindows;
 
-class NativeRegisterContextWindows_arm64 : public NativeRegisterContextWindows {
+class NativeRegisterContextWindows_arm64
+    : public NativeRegisterContextWindows,
+      public NativeRegisterContextDBReg_arm64 {
 public:
   NativeRegisterContextWindows_arm64(const ArchSpec &target_arch,
                                      NativeThreadProtocol &native_thread);
@@ -37,28 +41,6 @@ class NativeRegisterContextWindows_arm64 : public NativeRegisterContextWindows {
 
   Status WriteAllRegisterValues(const lldb::DataBufferSP &data_sp) override;
 
-  Status IsWatchpointHit(uint32_t wp_index, bool &is_hit) override;
-
-  Status GetWatchpointHitIndex(uint32_t &wp_index,
-                               lldb::addr_t trap_addr) override;
-
-  Status IsWatchpointVacant(uint32_t wp_index, bool &is_vacant) override;
-
-  bool ClearHardwareWatchpoint(uint32_t wp_index) override;
-
-  Status ClearAllHardwareWatchpoints() override;
-
-  Status SetHardwareWatchpointWithIndex(lldb::addr_t addr, size_t size,
-                                        uint32_t watch_flags,
-                                        uint32_t wp_index);
-
-  uint32_t SetHardwareWatchpoint(lldb::addr_t addr, size_t size,
-                                 uint32_t watch_flags) override;
-
-  lldb::addr_t GetWatchpointAddress(uint32_t wp_index) override;
-
-  uint32_t NumSupportedHardwareWatchpoints() override;
-
 protected:
   Status GPRRead(const uint32_t reg, RegisterValue &reg_value);
 
@@ -72,6 +54,10 @@ class NativeRegisterContextWindows_arm64 : public NativeRegisterContextWindows {
   bool IsGPR(uint32_t reg_index) const;
 
   bool IsFPR(uint32_t reg_index) const;
+
+  llvm::Error ReadHardwareDebugInfo() override;
+
+  llvm::Error WriteHardwareDebugRegs(DREGType hwbType) override;
 };
 
 } // namespace lldb_private
diff --git a/lldb/source/Plugins/Process/Windows/Common/NativeRegisterContextWindows_i386.cpp b/lldb/source/Plugins/Process/Windows/Common/NativeRegisterContextWindows_i386.cpp
index 53df986679394..7c13759df462d 100644
--- a/lldb/source/Plugins/Process/Windows/Common/NativeRegisterContextWindows_i386.cpp
+++ b/lldb/source/Plugins/Process/Windows/Common/NativeRegisterContextWindows_i386.cpp
@@ -92,8 +92,8 @@ NativeRegisterContextWindows::CreateHostNativeRegisterContextWindows(
 
 NativeRegisterContextWindows_i386::NativeRegisterContextWindows_i386(
     const ArchSpec &target_arch, NativeThreadProtocol &native_thread)
-    : NativeRegisterContextWindows(native_thread,
-                                   CreateRegisterInfoInterface(target_arch)) {}
+    : NativeRegisterContextRegisterInfo(
+          native_thread, CreateRegisterInfoInterface(target_arch)) {}
 
 bool NativeRegisterContextWindows_i386::IsGPR(uint32_t reg_index) const {
   return (reg_index < k_first_alias_i386);
diff --git a/lldb/source/Plugins/Process/Windows/Common/NativeRegisterContextWindows_x86_64.cpp b/lldb/source/Plugins/Process/Windows/Common/NativeRegisterContextWindows_x86_64.cpp
index 4c59273b845aa..c54c31e48e4e5 100644
--- a/lldb/source/Plugins/Process/Windows/Common/NativeRegisterContextWindows_x86_64.cpp
+++ b/lldb/source/Plugins/Process/Windows/Common/NativeRegisterContextWindows_x86_64.cpp
@@ -110,8 +110,8 @@ NativeRegisterContextWindows::CreateHostNativeRegisterContextWindows(
 
 NativeRegisterContextWindows_x86_64::NativeRegisterContextWindows_x86_64(
     const ArchSpec &target_arch, NativeThreadProtocol &native_thread)
-    : NativeRegisterContextWindows(native_thread,
-                                   CreateRegisterInfoInterface(target_arch)) {}
+    : NativeRegisterContextRegisterInfo(
+          native_thread, CreateRegisterInfoInterface(target_arch)) {}
 
 bool NativeRegisterContextWindows_x86_64::IsGPR(uint32_t reg_index) const {
   return (reg_index >= k_first_gpr_x86_64 && reg_index < k_first_alias_x86_64);
diff --git a/lldb/source/Plugins/Process/elf-core/RegisterContextPOSIXCore_arm64.cpp b/lldb/source/Plugins/Process/elf-core/RegisterContextPOSIXCore_arm64.cpp
index 2ddf8440aeb03..bd02bb0e69a4d 100644
--- a/lldb/source/Plugins/Process/elf-core/RegisterContextPOSIXCore_arm64.cpp
+++ b/lldb/source/Plugins/Process/elf-core/RegisterContextPOSIXCore_arm64.cpp
@@ -69,6 +69,15 @@ RegisterContextCorePOSIX_arm64::Create(Thread &thread, const ArchSpec &arch,
   if (fpmr_data.GetByteSize() >= sizeof(uint64_t))
     opt_regsets.Set(RegisterInfoPOSIX_arm64::eRegsetMaskFPMR);
 
+  DataExtractor gcs_data = getRegset(notes, arch.GetTriple(), AARCH64_GCS_Desc);
+  struct __attribute__((packed)) gcs_regs {
+    uint64_t features_enabled;
+    uint64_t features_locked;
+    uint64_t gcspr_e0;
+  };
+  if (gcs_data.GetByteSize() >= sizeof(gcs_regs))
+    opt_regsets.Set(RegisterInfoPOSIX_arm64::eRegsetMaskGCS);
+
   auto register_info_up =
       std::make_unique<RegisterInfoPOSIX_arm64>(arch, opt_regsets);
   return std::unique_ptr<RegisterContextCorePOSIX_arm64>(
@@ -136,6 +145,9 @@ RegisterContextCorePOSIX_arm64::RegisterContextCorePOSIX_arm64(
   if (m_register_info_up->IsFPMRPresent())
     m_fpmr_data = getRegset(notes, target_triple, AARCH64_FPMR_Desc);
 
+  if (m_register_info_up->IsGCSPresent())
+    m_gcs_data = getRegset(notes, target_triple, AARCH64_GCS_Desc);
+
   ConfigureRegisterContext();
 }
 
@@ -330,6 +342,11 @@ bool RegisterContextCorePOSIX_arm64::ReadRegister(const RegisterInfo *reg_info,
     assert(offset < m_mte_data.GetByteSize());
     value.SetFromMemoryData(*reg_info, m_mte_data.GetDataStart() + offset,
                             reg_info->byte_size, lldb::eByteOrderLittle, error);
+  } else if (IsGCS(reg)) {
+    offset = reg_info->byte_offset - m_register_info_up->GetGCSOffset();
+    assert(offset < m_gcs_data.GetByteSize());
+    value.SetFromMemoryData(*reg_info, m_gcs_data.GetDataStart() + offset,
+                            reg_info->byte_size, lldb::eByteOrderLittle, error);
   } else if (IsSME(reg)) {
     // If you had SME in the process, active or otherwise, there will at least
     // be a ZA header. No header, no SME at all.
diff --git a/lldb/source/Plugins/Process/elf-core/RegisterContextPOSIXCore_arm64.h b/lldb/source/Plugins/Process/elf-core/RegisterContextPOSIXCore_arm64.h
index 35588c40c2eb1..6140f805ffc78 100644
--- a/lldb/source/Plugins/Process/elf-core/RegisterContextPOSIXCore_arm64.h
+++ b/lldb/source/Plugins/Process/elf-core/RegisterContextPOSIXCore_arm64.h
@@ -63,6 +63,7 @@ class RegisterContextCorePOSIX_arm64 : public RegisterContextPOSIX_arm64 {
   lldb_private::DataExtractor m_mte_data;
   lldb_private::DataExtractor m_zt_data;
   lldb_private::DataExtractor m_fpmr_data;
+  lldb_private::DataExtractor m_gcs_data;
 
   SVEState m_sve_state = SVEState::Unknown;
   uint16_t m_sve_vector_length = 0;
diff --git a/lldb/source/Plugins/Process/elf-core/RegisterUtilities.h b/lldb/source/Plugins/Process/elf-core/RegisterUtilities.h
index b97279b0d735b..59382a12cde0a 100644
--- a/lldb/source/Plugins/Process/elf-core/RegisterUtilities.h
+++ b/lldb/source/Plugins/Process/elf-core/RegisterUtilities.h
@@ -148,6 +148,10 @@ constexpr RegsetDesc AARCH64_FPMR_Desc[] = {
     {llvm::Triple::Linux, llvm::Triple::aarch64, llvm::ELF::NT_ARM_FPMR},
 };
 
+constexpr RegsetDesc AARCH64_GCS_Desc[] = {
+    {llvm::Triple::Linux, llvm::Triple::aarch64, llvm::ELF::NT_ARM_GCS},
+};
+
 constexpr RegsetDesc PPC_VMX_Desc[] = {
     {llvm::Triple::FreeBSD, llvm::Triple::UnknownArch, llvm::ELF::NT_PPC_VMX},
     {llvm::Triple::Linux, llvm::Triple::UnknownArch, llvm::ELF::NT_PPC_VMX},
diff --git a/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunicationClient.cpp b/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunicationClient.cpp
index b3f1c6f052955..581dd8f8e0b6b 100644
--- a/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunicationClient.cpp
+++ b/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunicationClient.cpp
@@ -275,7 +275,6 @@ void GDBRemoteCommunicationClient::ResetDiscoverableSettings(bool did_exec) {
     m_supports_vCont_s = eLazyBoolCalculate;
     m_supports_vCont_S = eLazyBoolCalculate;
     m_supports_p = eLazyBoolCalculate;
-    m_supports_x = eLazyBoolCalculate;
     m_supports_QSaveRegisterState = eLazyBoolCalculate;
     m_qHostInfo_is_valid = eLazyBoolCalculate;
     m_curr_pid_is_valid = eLazyBoolCalculate;
@@ -295,6 +294,7 @@ void GDBRemoteCommunicationClient::ResetDiscoverableSettings(bool did_exec) {
     m_supports_qXfer_siginfo_read = eLazyBoolCalculate;
     m_supports_augmented_libraries_svr4_read = eLazyBoolCalculate;
     m_uses_native_signals = eLazyBoolCalculate;
+    m_x_packet_state.reset();
     m_supports_qProcessInfoPID = true;
     m_supports_qfProcessInfo = true;
     m_supports_qUserName = true;
@@ -348,6 +348,7 @@ void GDBRemoteCommunicationClient::GetRemoteQSupported() {
   m_supports_memory_tagging = eLazyBoolNo;
   m_supports_qSaveCore = eLazyBoolNo;
   m_uses_native_signals = eLazyBoolNo;
+  m_x_packet_state.reset();
 
   m_max_packet_size = UINT64_MAX; // It's supposed to always be there, but if
                                   // not, we assume no limit
@@ -401,6 +402,8 @@ void GDBRemoteCommunicationClient::GetRemoteQSupported() {
         m_supports_qSaveCore = eLazyBoolYes;
       else if (x == "native-signals+")
         m_uses_native_signals = eLazyBoolYes;
+      else if (x == "binary-upload+")
+        m_x_packet_state = xPacketState::Prefixed;
       // Look for a list of compressions in the features list e.g.
       // qXfer:features:read+;PacketSize=20000;qEcho+;SupportedCompressions=zlib-
       // deflate,lzma
@@ -715,19 +718,20 @@ Status GDBRemoteCommunicationClient::WriteMemoryTags(
   return status;
 }
 
-bool GDBRemoteCommunicationClient::GetxPacketSupported() {
-  if (m_supports_x == eLazyBoolCalculate) {
+GDBRemoteCommunicationClient::xPacketState
+GDBRemoteCommunicationClient::GetxPacketState() {
+  if (!m_x_packet_state)
+    GetRemoteQSupported();
+  if (!m_x_packet_state) {
     StringExtractorGDBRemote response;
-    m_supports_x = eLazyBoolNo;
-    char packet[256];
-    snprintf(packet, sizeof(packet), "x0,0");
-    if (SendPacketAndWaitForResponse(packet, response) ==
+    m_x_packet_state = xPacketState::Unimplemented;
+    if (SendPacketAndWaitForResponse("x0,0", response) ==
         PacketResult::Success) {
       if (response.IsOKResponse())
-        m_supports_x = eLazyBoolYes;
+        m_x_packet_state = xPacketState::Bare;
     }
   }
-  return m_supports_x;
+  return *m_x_packet_state;
 }
 
 lldb::pid_t GDBRemoteCommunicationClient::GetCurrentProcessID(bool allow_lazy) {
diff --git a/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunicationClient.h b/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunicationClient.h
index 898d176abc346..1118a76d7211b 100644
--- a/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunicationClient.h
+++ b/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunicationClient.h
@@ -218,7 +218,14 @@ class GDBRemoteCommunicationClient : public GDBRemoteClientBase {
 
   bool GetpPacketSupported(lldb::tid_t tid);
 
-  bool GetxPacketSupported();
+  enum class xPacketState {
+    Unimplemented,
+    Prefixed, // Successful responses start with a 'b' character. This is the
+              // style used by GDB.
+    Bare,     // No prefix, packets starts with the memory being read. This is
+              // LLDB's original style.
+  };
+  xPacketState GetxPacketState();
 
   bool GetVAttachOrWaitSupported();
 
@@ -541,7 +548,6 @@ class GDBRemoteCommunicationClient : public GDBRemoteClientBase {
   LazyBool m_attach_or_wait_reply = eLazyBoolCalculate;
   LazyBool m_prepare_for_reg_writing_reply = eLazyBoolCalculate;
   LazyBool m_supports_p = eLazyBoolCalculate;
-  LazyBool m_supports_x = eLazyBoolCalculate;
   LazyBool m_avoid_g_packets = eLazyBoolCalculate;
   LazyBool m_supports_QSaveRegisterState = eLazyBoolCalculate;
   LazyBool m_supports_qXfer_auxv_read = eLazyBoolCalculate;
@@ -561,6 +567,7 @@ class GDBRemoteCommunicationClient : public GDBRemoteClientBase {
   LazyBool m_supports_memory_tagging = eLazyBoolCalculate;
   LazyBool m_supports_qSaveCore = eLazyBoolCalculate;
   LazyBool m_uses_native_signals = eLazyBoolCalculate;
+  std::optional<xPacketState> m_x_packet_state;
 
   bool m_supports_qProcessInfoPID : 1, m_supports_qfProcessInfo : 1,
       m_supports_qUserName : 1, m_supports_qGroupName : 1,
diff --git a/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunicationServerCommon.cpp b/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunicationServerCommon.cpp
index c2fe05cad566e..67ba42f33d1dd 100644
--- a/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunicationServerCommon.cpp
+++ b/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunicationServerCommon.cpp
@@ -212,6 +212,8 @@ GDBRemoteCommunicationServerCommon::Handle_qHostInfo(
     response.PutCString("ostype:tvos;");
 #elif defined(TARGET_OS_WATCH) && TARGET_OS_WATCH == 1
     response.PutCString("ostype:watchos;");
+#elif defined(TARGET_OS_XR) && TARGET_OS_XR == 1
+    response.PutCString("ostype:xros;");
 #elif defined(TARGET_OS_BRIDGE) && TARGET_OS_BRIDGE == 1
     response.PutCString("ostype:bridgeos;");
 #else
diff --git a/lldb/source/Plugins/Process/gdb-remote/ProcessGDBRemote.cpp b/lldb/source/Plugins/Process/gdb-remote/ProcessGDBRemote.cpp
index 538c868014009..07b4470d06191 100644
--- a/lldb/source/Plugins/Process/gdb-remote/ProcessGDBRemote.cpp
+++ b/lldb/source/Plugins/Process/gdb-remote/ProcessGDBRemote.cpp
@@ -1726,10 +1726,6 @@ ThreadSP ProcessGDBRemote::SetThreadStopInfo(
 
   if (!thread_sp->StopInfoIsUpToDate()) {
     thread_sp->SetStopInfo(StopInfoSP());
-    // If there's a memory thread backed by this thread, we need to use it to
-    // calculate StopInfo.
-    if (ThreadSP memory_thread_sp = m_thread_list.GetBackingThread(thread_sp))
-      thread_sp = memory_thread_sp;
 
     if (exc_type != 0) {
       // For thread plan async interrupt, creating stop info on the
@@ -2609,11 +2605,15 @@ void ProcessGDBRemote::WillPublicStop() {
 // Process Memory
 size_t ProcessGDBRemote::DoReadMemory(addr_t addr, void *buf, size_t size,
                                       Status &error) {
+  using xPacketState = GDBRemoteCommunicationClient::xPacketState;
+
   GetMaxMemorySize();
-  bool binary_memory_read = m_gdb_comm.GetxPacketSupported();
+  xPacketState x_state = m_gdb_comm.GetxPacketState();
+
   // M and m packets take 2 bytes for 1 byte of memory
-  size_t max_memory_size =
-      binary_memory_read ? m_max_memory_size : m_max_memory_size / 2;
+  size_t max_memory_size = x_state != xPacketState::Unimplemented
+                               ? m_max_memory_size
+                               : m_max_memory_size / 2;
   if (size > max_memory_size) {
     // Keep memory read sizes down to a sane limit. This function will be
     // called multiple times in order to complete the task by
@@ -2624,8 +2624,8 @@ size_t ProcessGDBRemote::DoReadMemory(addr_t addr, void *buf, size_t size,
   char packet[64];
   int packet_len;
   packet_len = ::snprintf(packet, sizeof(packet), "%c%" PRIx64 ",%" PRIx64,
-                          binary_memory_read ? 'x' : 'm', (uint64_t)addr,
-                          (uint64_t)size);
+                          x_state != xPacketState::Unimplemented ? 'x' : 'm',
+                          (uint64_t)addr, (uint64_t)size);
   assert(packet_len + 1 < (int)sizeof(packet));
   UNUSED_IF_ASSERT_DISABLED(packet_len);
   StringExtractorGDBRemote response;
@@ -2634,19 +2634,25 @@ size_t ProcessGDBRemote::DoReadMemory(addr_t addr, void *buf, size_t size,
       GDBRemoteCommunication::PacketResult::Success) {
     if (response.IsNormalResponse()) {
       error.Clear();
-      if (binary_memory_read) {
+      if (x_state != xPacketState::Unimplemented) {
         // The lower level GDBRemoteCommunication packet receive layer has
         // already de-quoted any 0x7d character escaping that was present in
         // the packet
 
-        size_t data_received_size = response.GetBytesLeft();
-        if (data_received_size > size) {
-          // Don't write past the end of BUF if the remote debug server gave us
-          // too much data for some reason.
-          data_received_size = size;
+        llvm::StringRef data_received = response.GetStringRef();
+        if (x_state == xPacketState::Prefixed &&
+            !data_received.consume_front("b")) {
+          error = Status::FromErrorStringWithFormatv(
+              "unexpected response to GDB server memory read packet '{0}': "
+              "'{1}'",
+              packet, data_received);
+          return 0;
         }
-        memcpy(buf, response.GetStringRef().data(), data_received_size);
-        return data_received_size;
+        // Don't write past the end of BUF if the remote debug server gave us
+        // too much data for some reason.
+        size_t memcpy_size = std::min(size, data_received.size());
+        memcpy(buf, data_received.data(), memcpy_size);
+        return memcpy_size;
       } else {
         return response.GetHexBytes(
             llvm::MutableArrayRef<uint8_t>((uint8_t *)buf, size), '\xdd');
diff --git a/lldb/source/Plugins/ScriptInterpreter/Python/PythonDataObjects.cpp b/lldb/source/Plugins/ScriptInterpreter/Python/PythonDataObjects.cpp
index a0f8cf954f804..1340425aade43 100644
--- a/lldb/source/Plugins/ScriptInterpreter/Python/PythonDataObjects.cpp
+++ b/lldb/source/Plugins/ScriptInterpreter/Python/PythonDataObjects.cpp
@@ -73,10 +73,8 @@ Expected<std::string> python::As<std::string>(Expected<PythonObject> &&obj) {
 static bool python_is_finalizing() {
 #if PY_VERSION_HEX >= 0x030d0000
   return Py_IsFinalizing();
-#elif PY_VERSION_HEX >= 0x03070000
-  return _Py_IsFinalizing();
 #else
-  return _Py_Finalizing != nullptr;
+  return _Py_IsFinalizing();
 #endif
 }
 
@@ -810,7 +808,6 @@ bool PythonCallable::Check(PyObject *py_obj) {
   return PyCallable_Check(py_obj);
 }
 
-#if PY_VERSION_HEX >= 0x03030000
 static const char get_arg_info_script[] = R"(
 from inspect import signature, Parameter, ismethod
 from collections import namedtuple
@@ -832,15 +829,12 @@ def main(f):
             raise Exception(f'unknown parameter kind: {kind}')
     return ArgInfo(count, varargs)
 )";
-#endif
 
 Expected<PythonCallable::ArgInfo> PythonCallable::GetArgInfo() const {
   ArgInfo result = {};
   if (!IsValid())
     return nullDeref();
 
-#if PY_VERSION_HEX >= 0x03030000
-
   // no need to synchronize access to this global, we already have the GIL
   static PythonScript get_arg_info(get_arg_info_script);
   Expected<PythonObject> pyarginfo = get_arg_info(*this);
@@ -852,57 +846,6 @@ Expected<PythonCallable::ArgInfo> PythonCallable::GetArgInfo() const {
       cantFail(As<bool>(pyarginfo.get().GetAttribute("has_varargs")));
   result.max_positional_args = has_varargs ? ArgInfo::UNBOUNDED : count;
 
-#else
-  PyObject *py_func_obj;
-  bool is_bound_method = false;
-  bool is_class = false;
-
-  if (PyType_Check(m_py_obj) || PyClass_Check(m_py_obj)) {
-    auto init = GetAttribute("__init__");
-    if (!init)
-      return init.takeError();
-    py_func_obj = init.get().get();
-    is_class = true;
-  } else {
-    py_func_obj = m_py_obj;
-  }
-
-  if (PyMethod_Check(py_func_obj)) {
-    py_func_obj = PyMethod_GET_FUNCTION(py_func_obj);
-    PythonObject im_self = GetAttributeValue("im_self");
-    if (im_self.IsValid() && !im_self.IsNone())
-      is_bound_method = true;
-  } else {
-    // see if this is a callable object with an __call__ method
-    if (!PyFunction_Check(py_func_obj)) {
-      PythonObject __call__ = GetAttributeValue("__call__");
-      if (__call__.IsValid()) {
-        auto __callable__ = __call__.AsType<PythonCallable>();
-        if (__callable__.IsValid()) {
-          py_func_obj = PyMethod_GET_FUNCTION(__callable__.get());
-          PythonObject im_self = __callable__.GetAttributeValue("im_self");
-          if (im_self.IsValid() && !im_self.IsNone())
-            is_bound_method = true;
-        }
-      }
-    }
-  }
-
-  if (!py_func_obj)
-    return result;
-
-  PyCodeObject *code = (PyCodeObject *)PyFunction_GET_CODE(py_func_obj);
-  if (!code)
-    return result;
-
-  auto count = code->co_argcount;
-  bool has_varargs = !!(code->co_flags & CO_VARARGS);
-  result.max_positional_args =
-      has_varargs ? ArgInfo::UNBOUNDED
-                  : (count - (int)is_bound_method) - (int)is_class;
-
-#endif
-
   return result;
 }
 
diff --git a/lldb/source/Plugins/ScriptInterpreter/Python/SWIGPythonBridge.h b/lldb/source/Plugins/ScriptInterpreter/Python/SWIGPythonBridge.h
index 0f0e4a563e8b2..a2252d164ab83 100644
--- a/lldb/source/Plugins/ScriptInterpreter/Python/SWIGPythonBridge.h
+++ b/lldb/source/Plugins/ScriptInterpreter/Python/SWIGPythonBridge.h
@@ -81,6 +81,8 @@ template <typename T> class ScopedPythonObject : PythonObject {
 class SWIGBridge {
 public:
   static PythonObject ToSWIGWrapper(std::unique_ptr<lldb::SBValue> value_sb);
+  static PythonObject
+  ToSWIGWrapper(std::unique_ptr<lldb::SBCommandReturnObject> result_up);
   static PythonObject ToSWIGWrapper(lldb::ValueObjectSP value_sp);
   static PythonObject ToSWIGWrapper(lldb::TargetSP target_sp);
   static PythonObject ToSWIGWrapper(lldb::ProcessSP process_sp);
@@ -190,12 +192,11 @@ class SWIGBridge {
                                   lldb::DebuggerSP debugger, const char *args,
                                   lldb_private::CommandReturnObject &cmd_retobj,
                                   lldb::ExecutionContextRefSP exe_ctx_ref_sp);
-  static bool
-  LLDBSwigPythonCallParsedCommandObject(PyObject *implementor,
-                                  lldb::DebuggerSP debugger,  
-                                  StructuredDataImpl &args_impl,
-                                  lldb_private::CommandReturnObject &cmd_retobj,
-                                  lldb::ExecutionContextRefSP exe_ctx_ref_sp);
+  static bool LLDBSwigPythonCallParsedCommandObject(
+      PyObject *implementor, lldb::DebuggerSP debugger,
+      StructuredDataImpl &args_impl,
+      lldb_private::CommandReturnObject &cmd_retobj,
+      lldb::ExecutionContextRefSP exe_ctx_ref_sp);
 
   static std::optional<std::string>
   LLDBSwigPythonGetRepeatCommandForScriptedCommand(PyObject *implementor,
diff --git a/lldb/source/Plugins/ScriptInterpreter/Python/ScriptInterpreterPython.cpp b/lldb/source/Plugins/ScriptInterpreter/Python/ScriptInterpreterPython.cpp
index ef3c53ca5698d..f4efb00161f8b 100644
--- a/lldb/source/Plugins/ScriptInterpreter/Python/ScriptInterpreterPython.cpp
+++ b/lldb/source/Plugins/ScriptInterpreter/Python/ScriptInterpreterPython.cpp
@@ -70,8 +70,7 @@ extern "C" PyObject *PyInit__lldb(void);
 // Don't mess with the signal handlers on Windows.
 #define LLDB_USE_PYTHON_SET_INTERRUPT 0
 #else
-// PyErr_SetInterrupt was introduced in 3.2.
-#define LLDB_USE_PYTHON_SET_INTERRUPT PY_VERSION_HEX >= 0x03020000
+#define LLDB_USE_PYTHON_SET_INTERRUPT 1
 #endif
 
 static ScriptInterpreterPythonImpl *GetPythonInterpreter(Debugger &debugger) {
@@ -91,10 +90,8 @@ namespace {
 struct InitializePythonRAII {
 public:
   InitializePythonRAII() {
-#if PY_VERSION_HEX >= 0x03080000
     PyConfig config;
     PyConfig_InitPythonConfig(&config);
-#endif
 
 #if LLDB_EMBED_PYTHON_HOME
     static std::string g_python_home = []() -> std::string {
@@ -108,14 +105,7 @@ struct InitializePythonRAII {
       return spec.GetPath();
     }();
     if (!g_python_home.empty()) {
-#if PY_VERSION_HEX >= 0x03080000
       PyConfig_SetBytesString(&config, &config.home, g_python_home.c_str());
-#else
-      size_t size = 0;
-      wchar_t *python_home_w = Py_DecodeLocale(g_python_home.c_str(), &size);
-      Py_SetPythonHome(python_home_w);
-      PyMem_RawFree(python_home_w);
-#endif
     }
 #endif
 
@@ -142,23 +132,10 @@ struct InitializePythonRAII {
       PyImport_AppendInittab("_lldb", LLDBSwigPyInit);
     }
 
-#if PY_VERSION_HEX >= 0x03080000
     config.install_signal_handlers = 0;
     Py_InitializeFromConfig(&config);
     PyConfig_Clear(&config);
     InitializeThreadsPrivate();
-#else
-// Python < 3.2 and Python >= 3.2 reversed the ordering requirements for
-// calling `Py_Initialize` and `PyEval_InitThreads`.  < 3.2 requires that you
-// call `PyEval_InitThreads` first, and >= 3.2 requires that you call it last.
-#if PY_VERSION_HEX >= 0x03020000
-    Py_InitializeEx(0);
-    InitializeThreadsPrivate();
-#else
-    InitializeThreadsPrivate();
-    Py_InitializeEx(0);
-#endif
-#endif
   }
 
   ~InitializePythonRAII() {
@@ -175,17 +152,17 @@ struct InitializePythonRAII {
 
 private:
   void InitializeThreadsPrivate() {
-// Since Python 3.7 `Py_Initialize` calls `PyEval_InitThreads` inside itself,
-// so there is no way to determine whether the embedded interpreter
-// was already initialized by some external code. `PyEval_ThreadsInitialized`
-// would always return `true` and `PyGILState_Ensure/Release` flow would be
-// executed instead of unlocking GIL with `PyEval_SaveThread`. When
-// an another thread calls `PyGILState_Ensure` it would get stuck in deadlock.
-#if PY_VERSION_HEX >= 0x03070000
+    // Since Python 3.7 `Py_Initialize` calls `PyEval_InitThreads` inside
+    // itself, so there is no way to determine whether the embedded interpreter
+    // was already initialized by some external code.
+    // `PyEval_ThreadsInitialized` would always return `true` and
+    // `PyGILState_Ensure/Release` flow would be executed instead of unlocking
+    // GIL with `PyEval_SaveThread`. When an another thread calls
+    // `PyGILState_Ensure` it would get stuck in deadlock.
+
     // The only case we should go further and acquire the GIL: it is unlocked.
     if (PyGILState_Check())
       return;
-#endif
 
 // `PyEval_ThreadsInitialized` was deprecated in Python 3.9 and removed in
 // Python 3.13. It has been returning `true` always since Python 3.7.
diff --git a/lldb/source/Plugins/ScriptInterpreter/Python/lldb-python.h b/lldb/source/Plugins/ScriptInterpreter/Python/lldb-python.h
index b68598b9d59d9..4a6c11dba02e8 100644
--- a/lldb/source/Plugins/ScriptInterpreter/Python/lldb-python.h
+++ b/lldb/source/Plugins/ScriptInterpreter/Python/lldb-python.h
@@ -50,8 +50,8 @@ static llvm::Expected<bool> *g_fcxx_modules_workaround [[maybe_unused]];
 
 // Provide a meaningful diagnostic error if someone tries to compile this file
 // with a version of Python we don't support.
-static_assert(PY_VERSION_HEX >= 0x03000000,
-              "LLDB requires at least Python 3.0");
+static_assert(PY_VERSION_HEX >= 0x03080000,
+              "LLDB requires at least Python 3.8");
 #endif
 
 #endif // LLDB_PLUGINS_SCRIPTINTERPRETER_PYTHON_LLDB_PYTHON_H
diff --git a/lldb/source/Plugins/StructuredData/DarwinLog/StructuredDataDarwinLog.cpp b/lldb/source/Plugins/StructuredData/DarwinLog/StructuredDataDarwinLog.cpp
index 4ca8bd2f9085d..82f18c5fe37a2 100644
--- a/lldb/source/Plugins/StructuredData/DarwinLog/StructuredDataDarwinLog.cpp
+++ b/lldb/source/Plugins/StructuredData/DarwinLog/StructuredDataDarwinLog.cpp
@@ -975,8 +975,6 @@ EnableOptionsSP ParseAutoEnableOptions(Status &error, Debugger &debugger) {
   EnableOptionsSP options_sp(new EnableOptions());
   options_sp->NotifyOptionParsingStarting(&exe_ctx);
 
-  CommandReturnObject result(debugger.GetUseColor());
-
   // Parse the arguments.
   auto options_property_sp =
       debugger.GetPropertyValue(nullptr,
@@ -1013,8 +1011,13 @@ EnableOptionsSP ParseAutoEnableOptions(Status &error, Debugger &debugger) {
     return EnableOptionsSP();
   }
 
-  if (!options_sp->VerifyOptions(result))
+  if (llvm::Error error = options_sp->VerifyOptions()) {
+    LLDB_LOG_ERROR(
+        log, std::move(error),
+        "Parsing plugin.structured-data.darwin-log.auto-enable-options value "
+        "failed: {0}");
     return EnableOptionsSP();
+  }
 
   // We successfully parsed and validated the options.
   return options_sp;
diff --git a/lldb/source/Plugins/SymbolFile/DWARF/DWARFASTParserClang.cpp b/lldb/source/Plugins/SymbolFile/DWARF/DWARFASTParserClang.cpp
index e77188bfbd2e4..ec0004c70c6da 100644
--- a/lldb/source/Plugins/SymbolFile/DWARF/DWARFASTParserClang.cpp
+++ b/lldb/source/Plugins/SymbolFile/DWARF/DWARFASTParserClang.cpp
@@ -492,6 +492,10 @@ ParsedDWARFTypeAttributes::ParsedDWARFTypeAttributes(const DWARFDIE &die) {
     case DW_AT_reference:
       ref_qual = clang::RQ_LValue;
       break;
+    case DW_AT_APPLE_enum_kind:
+      enum_kind = static_cast<clang::EnumExtensibilityAttr::Kind>(
+          form_value.Unsigned());
+      break;
     }
   }
 }
@@ -1001,9 +1005,10 @@ TypeSP DWARFASTParserClang::ParseEnum(const SymbolContext &sc,
   }
 
   CompilerType clang_type = m_ast.CreateEnumerationType(
-      attrs.name.GetStringRef(), GetClangDeclContextContainingDIE(def_die, nullptr),
+      attrs.name.GetStringRef(),
+      GetClangDeclContextContainingDIE(def_die, nullptr),
       GetOwningClangModule(def_die), attrs.decl, enumerator_clang_type,
-      attrs.is_scoped_enum);
+      attrs.is_scoped_enum, attrs.enum_kind);
   TypeSP type_sp =
       dwarf->MakeType(def_die.GetID(), attrs.name, attrs.byte_size, nullptr,
                       attrs.type.Reference().GetID(), Type::eEncodingIsUID,
@@ -1019,16 +1024,7 @@ TypeSP DWARFASTParserClang::ParseEnum(const SymbolContext &sc,
     // Declaration DIE is inserted into the type map in ParseTypeFromDWARF
   }
 
-
-  if (TypeSystemClang::StartTagDeclarationDefinition(clang_type)) {
-    if (def_die.HasChildren()) {
-      bool is_signed = false;
-      enumerator_clang_type.IsIntegerType(is_signed);
-      ParseChildEnumerators(clang_type, is_signed,
-                            type_sp->GetByteSize(nullptr).value_or(0), def_die);
-    }
-    TypeSystemClang::CompleteTagDeclarationDefinition(clang_type);
-  } else {
+  if (!CompleteEnumType(def_die, type_sp.get(), clang_type)) {
     dwarf->GetObjectFile()->GetModule()->ReportError(
         "DWARF DIE at {0:x16} named \"{1}\" was not able to start its "
         "definition.\nPlease file a bug and attach the file at the "
@@ -1272,7 +1268,7 @@ DWARFASTParserClang::ParseSubroutine(const DWARFDIE &die,
     return_clang_type = m_ast.GetBasicType(eBasicTypeVoid);
 
   std::vector<CompilerType> function_param_types;
-  std::vector<clang::ParmVarDecl *> function_param_decls;
+  llvm::SmallVector<llvm::StringRef> function_param_names;
 
   // Parse the function children for the parameters
 
@@ -1284,7 +1280,7 @@ DWARFASTParserClang::ParseSubroutine(const DWARFDIE &die,
   if (die.HasChildren()) {
     ParseChildParameters(containing_decl_ctx, die, is_variadic,
                          has_template_params, function_param_types,
-                         function_param_decls);
+                         function_param_names);
   }
 
   bool is_cxx_method = DeclKindIsCXXClass(containing_decl_ctx->getDeclKind());
@@ -1414,12 +1410,14 @@ DWARFASTParserClang::ParseSubroutine(const DWARFDIE &die,
 
           LinkDeclContextToDIE(function_decl, die);
 
-          if (!function_param_decls.empty()) {
-            m_ast.SetFunctionParameters(function_decl, function_param_decls);
-            if (template_function_decl)
-              m_ast.SetFunctionParameters(template_function_decl,
-                                          function_param_decls);
-          }
+          const clang::FunctionProtoType *function_prototype(
+              llvm::cast<clang::FunctionProtoType>(
+                  ClangUtil::GetQualType(clang_type).getTypePtr()));
+          const auto params = m_ast.CreateParameterDeclarations(
+              function_decl, *function_prototype, function_param_names);
+          function_decl->setParams(params);
+          if (template_function_decl)
+            template_function_decl->setParams(params);
 
           ClangASTMetadata metadata;
           metadata.SetUserID(die.GetID());
@@ -2219,13 +2217,14 @@ bool DWARFASTParserClang::CompleteRecordType(const DWARFDIE &die,
 bool DWARFASTParserClang::CompleteEnumType(const DWARFDIE &die,
                                            lldb_private::Type *type,
                                            const CompilerType &clang_type) {
+  assert(clang_type.IsEnumerationType());
+
   if (TypeSystemClang::StartTagDeclarationDefinition(clang_type)) {
-    if (die.HasChildren()) {
-      bool is_signed = false;
-      clang_type.IsIntegerType(is_signed);
-      ParseChildEnumerators(clang_type, is_signed,
+    if (die.HasChildren())
+      ParseChildEnumerators(clang_type,
+                            clang_type.IsEnumerationIntegerTypeSigned(),
                             type->GetByteSize(nullptr).value_or(0), die);
-    }
+
     TypeSystemClang::CompleteTagDeclarationDefinition(clang_type);
   }
   return (bool)clang_type;
@@ -2327,8 +2326,7 @@ size_t DWARFASTParserClang::ParseChildEnumerators(
       continue;
 
     const char *name = nullptr;
-    bool got_value = false;
-    int64_t enum_value = 0;
+    std::optional<uint64_t> enum_value;
     Declaration decl;
 
     for (size_t i = 0; i < attributes.Size(); ++i) {
@@ -2337,7 +2335,6 @@ size_t DWARFASTParserClang::ParseChildEnumerators(
       if (attributes.ExtractFormValueAtIndex(i, form_value)) {
         switch (attr) {
         case DW_AT_const_value:
-          got_value = true;
           if (is_signed)
             enum_value = form_value.Signed();
           else
@@ -2366,9 +2363,9 @@ size_t DWARFASTParserClang::ParseChildEnumerators(
       }
     }
 
-    if (name && name[0] && got_value) {
+    if (name && name[0] && enum_value) {
       m_ast.AddEnumerationValueToEnumerationType(
-          clang_type, decl, name, enum_value, enumerator_byte_size * 8);
+          clang_type, decl, name, *enum_value, enumerator_byte_size * 8);
       ++enumerators_added;
     }
   }
@@ -2380,7 +2377,7 @@ DWARFASTParserClang::ConstructDemangledNameFromDWARF(const DWARFDIE &die) {
   bool is_variadic = false;
   bool has_template_params = false;
   std::vector<CompilerType> param_types;
-  std::vector<clang::ParmVarDecl *> param_decls;
+  llvm::SmallVector<llvm::StringRef> param_names;
   StreamString sstr;
 
   DWARFDeclContext decl_ctx = die.GetDWARFDeclContext();
@@ -2394,7 +2391,7 @@ DWARFASTParserClang::ConstructDemangledNameFromDWARF(const DWARFDIE &die) {
       die, GetCXXObjectParameter(die, *containing_decl_ctx));
 
   ParseChildParameters(containing_decl_ctx, die, is_variadic,
-                       has_template_params, param_types, param_decls);
+                       has_template_params, param_types, param_names);
   sstr << "(";
   for (size_t i = 0; i < param_types.size(); i++) {
     if (i > 0)
@@ -3157,7 +3154,7 @@ void DWARFASTParserClang::ParseChildParameters(
     clang::DeclContext *containing_decl_ctx, const DWARFDIE &parent_die,
     bool &is_variadic, bool &has_template_params,
     std::vector<CompilerType> &function_param_types,
-    std::vector<clang::ParmVarDecl *> &function_param_decls) {
+    llvm::SmallVectorImpl<llvm::StringRef> &function_param_names) {
   if (!parent_die)
     return;
 
@@ -3168,22 +3165,14 @@ void DWARFASTParserClang::ParseChildParameters(
       if (die.GetAttributeValueAsUnsigned(DW_AT_artificial, 0))
         continue;
 
-      const char *name = die.GetName();
       DWARFDIE param_type_die = die.GetAttributeValueAsReferenceDIE(DW_AT_type);
 
       Type *type = die.ResolveTypeUID(param_type_die);
       if (!type)
         break;
 
+      function_param_names.emplace_back(die.GetName());
       function_param_types.push_back(type->GetForwardCompilerType());
-
-      clang::ParmVarDecl *param_var_decl = m_ast.CreateParameterDeclaration(
-          containing_decl_ctx, GetOwningClangModule(die), name,
-          type->GetForwardCompilerType(), clang::StorageClass::SC_None);
-      assert(param_var_decl);
-      function_param_decls.push_back(param_var_decl);
-
-      m_ast.SetMetadataAsUserID(param_var_decl, die.GetID());
     } break;
 
     case DW_TAG_unspecified_parameters:
@@ -3205,6 +3194,8 @@ void DWARFASTParserClang::ParseChildParameters(
       break;
     }
   }
+
+  assert(function_param_names.size() == function_param_types.size());
 }
 
 clang::Decl *DWARFASTParserClang::GetClangDeclForDIE(const DWARFDIE &die) {
diff --git a/lldb/source/Plugins/SymbolFile/DWARF/DWARFASTParserClang.h b/lldb/source/Plugins/SymbolFile/DWARF/DWARFASTParserClang.h
index a5c3746ada4c3..135dd06186c4b 100644
--- a/lldb/source/Plugins/SymbolFile/DWARF/DWARFASTParserClang.h
+++ b/lldb/source/Plugins/SymbolFile/DWARF/DWARFASTParserClang.h
@@ -186,12 +186,12 @@ class DWARFASTParserClang : public lldb_private::plugin::dwarf::DWARFASTParser {
       const lldb::AccessType default_accessibility,
       lldb_private::ClangASTImporter::LayoutInfo &layout_info);
 
-  void
-  ParseChildParameters(clang::DeclContext *containing_decl_ctx,
-                       const lldb_private::plugin::dwarf::DWARFDIE &parent_die,
-                       bool &is_variadic, bool &has_template_params,
-                       std::vector<lldb_private::CompilerType> &function_args,
-                       std::vector<clang::ParmVarDecl *> &function_param_decls);
+  void ParseChildParameters(
+      clang::DeclContext *containing_decl_ctx,
+      const lldb_private::plugin::dwarf::DWARFDIE &parent_die,
+      bool &is_variadic, bool &has_template_params,
+      std::vector<lldb_private::CompilerType> &function_param_types,
+      llvm::SmallVectorImpl<llvm::StringRef> &function_param_names);
 
   size_t ParseChildEnumerators(
       const lldb_private::CompilerType &compiler_type, bool is_signed,
@@ -564,10 +564,14 @@ struct ParsedDWARFTypeAttributes {
   uint32_t bit_stride = 0;
   uint32_t byte_stride = 0;
   uint32_t encoding = 0;
-  clang::RefQualifierKind ref_qual =
-      clang::RQ_None; ///< Indicates ref-qualifier of
-                      ///< C++ member function if present.
-                      ///< Is RQ_None otherwise.
+
+  ///< Indicates ref-qualifier of C++ member function if present.
+  ///< Is RQ_None otherwise.
+  clang::RefQualifierKind ref_qual = clang::RQ_None;
+
+  ///< Has a value if this DIE represents an enum that was declared
+  ///< with enum_extensibility.
+  std::optional<clang::EnumExtensibilityAttr::Kind> enum_kind;
 };
 
 #endif // LLDB_SOURCE_PLUGINS_SYMBOLFILE_DWARF_DWARFASTPARSERCLANG_H
diff --git a/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARF.cpp b/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARF.cpp
index afc37f517559e..a96757afabddf 100644
--- a/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARF.cpp
+++ b/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARF.cpp
@@ -2455,18 +2455,12 @@ bool SymbolFileDWARF::ResolveFunction(const DWARFDIE &orig_die,
   }
   assert(die && die.Tag() == DW_TAG_subprogram);
   if (GetFunction(die, sc)) {
-    Address addr;
     // Parse all blocks if needed
     if (inlined_die) {
       Block &function_block = sc.function->GetBlock(true);
       sc.block = function_block.FindBlockByID(inlined_die.GetID());
       if (sc.block == nullptr)
         sc.block = function_block.FindBlockByID(inlined_die.GetOffset());
-      if (sc.block == nullptr || !sc.block->GetStartAddress(addr))
-        addr.Clear();
-    } else {
-      sc.block = nullptr;
-      addr = sc.function->GetAddressRange().GetBaseAddress();
     }
 
     sc_list.Append(sc);
diff --git a/lldb/source/Plugins/SymbolFile/NativePDB/PdbAstBuilder.cpp b/lldb/source/Plugins/SymbolFile/NativePDB/PdbAstBuilder.cpp
index 0c71df625ae34..5d4b22d08b111 100644
--- a/lldb/source/Plugins/SymbolFile/NativePDB/PdbAstBuilder.cpp
+++ b/lldb/source/Plugins/SymbolFile/NativePDB/PdbAstBuilder.cpp
@@ -1137,7 +1137,7 @@ void PdbAstBuilder::CreateFunctionParameters(PdbCompilandSymId func_id,
   }
 
   if (!params.empty() && params.size() == param_count)
-    m_clang.SetFunctionParameters(&function_decl, params);
+    function_decl.setParams(params);
 }
 
 clang::QualType PdbAstBuilder::CreateEnumType(PdbTypeSymId id,
diff --git a/lldb/source/Plugins/SymbolFile/PDB/PDBASTParser.cpp b/lldb/source/Plugins/SymbolFile/PDB/PDBASTParser.cpp
index fa3530a0c22ff..c6dd72e22fb4c 100644
--- a/lldb/source/Plugins/SymbolFile/PDB/PDBASTParser.cpp
+++ b/lldb/source/Plugins/SymbolFile/PDB/PDBASTParser.cpp
@@ -975,8 +975,8 @@ PDBASTParser::GetDeclForSymbol(const llvm::pdb::PDBSymbol &symbol) {
         }
       }
     }
-    if (params.size())
-      m_ast.SetFunctionParameters(decl, params);
+    if (params.size() && decl)
+      decl->setParams(params);
 
     m_uid_to_decl[sym_id] = decl;
 
@@ -1155,7 +1155,7 @@ bool PDBASTParser::AddEnumValue(CompilerType enum_type,
   Variant v = enum_value.getValue();
   std::string name =
       std::string(MSVCUndecoratedNameParser::DropScope(enum_value.getName()));
-  int64_t raw_value;
+  uint64_t raw_value;
   switch (v.Type) {
   case PDB_VariantType::Int8:
     raw_value = v.Value.Int8;
diff --git a/lldb/source/Plugins/SystemRuntime/MacOSX/SystemRuntimeMacOSX.cpp b/lldb/source/Plugins/SystemRuntime/MacOSX/SystemRuntimeMacOSX.cpp
index 0556b87777884..b23f64210cc80 100644
--- a/lldb/source/Plugins/SystemRuntime/MacOSX/SystemRuntimeMacOSX.cpp
+++ b/lldb/source/Plugins/SystemRuntime/MacOSX/SystemRuntimeMacOSX.cpp
@@ -62,8 +62,9 @@ SystemRuntime *SystemRuntimeMacOSX::CreateInstance(Process *process) {
       case llvm::Triple::IOS:
       case llvm::Triple::TvOS:
       case llvm::Triple::WatchOS:
-      case llvm::Triple::XROS:
       case llvm::Triple::BridgeOS:
+      case llvm::Triple::DriverKit:
+      case llvm::Triple::XROS:
         create = triple_ref.getVendor() == llvm::Triple::Apple;
         break;
       default:
@@ -632,10 +633,8 @@ bool SystemRuntimeMacOSX::BacktraceRecordingHeadersInitialized() {
   if (!sc_list.IsEmpty()) {
     SymbolContext sc;
     sc_list.GetContextAtIndex(0, sc);
-    AddressRange addr_range;
-    sc.GetAddressRange(eSymbolContextSymbol, 0, false, addr_range);
-    queue_info_version_address =
-        addr_range.GetBaseAddress().GetLoadAddress(&target);
+    Address addr = sc.GetFunctionOrSymbolAddress();
+    queue_info_version_address = addr.GetLoadAddress(&target);
   }
   sc_list.Clear();
 
@@ -646,10 +645,8 @@ bool SystemRuntimeMacOSX::BacktraceRecordingHeadersInitialized() {
   if (!sc_list.IsEmpty()) {
     SymbolContext sc;
     sc_list.GetContextAtIndex(0, sc);
-    AddressRange addr_range;
-    sc.GetAddressRange(eSymbolContextSymbol, 0, false, addr_range);
-    queue_info_data_offset_address =
-        addr_range.GetBaseAddress().GetLoadAddress(&target);
+    Address addr = sc.GetFunctionOrSymbolAddress();
+    queue_info_data_offset_address = addr.GetLoadAddress(&target);
   }
   sc_list.Clear();
 
@@ -660,10 +657,8 @@ bool SystemRuntimeMacOSX::BacktraceRecordingHeadersInitialized() {
   if (!sc_list.IsEmpty()) {
     SymbolContext sc;
     sc_list.GetContextAtIndex(0, sc);
-    AddressRange addr_range;
-    sc.GetAddressRange(eSymbolContextSymbol, 0, false, addr_range);
-    item_info_version_address =
-        addr_range.GetBaseAddress().GetLoadAddress(&target);
+    Address addr = sc.GetFunctionOrSymbolAddress();
+    item_info_version_address = addr.GetLoadAddress(&target);
   }
   sc_list.Clear();
 
@@ -674,10 +669,8 @@ bool SystemRuntimeMacOSX::BacktraceRecordingHeadersInitialized() {
   if (!sc_list.IsEmpty()) {
     SymbolContext sc;
     sc_list.GetContextAtIndex(0, sc);
-    AddressRange addr_range;
-    sc.GetAddressRange(eSymbolContextSymbol, 0, false, addr_range);
-    item_info_data_offset_address =
-        addr_range.GetBaseAddress().GetLoadAddress(&target);
+    Address addr = sc.GetFunctionOrSymbolAddress();
+    item_info_data_offset_address = addr.GetLoadAddress(&target);
   }
 
   if (queue_info_version_address != LLDB_INVALID_ADDRESS &&
diff --git a/lldb/source/Plugins/TypeSystem/Clang/TypeSystemClang.cpp b/lldb/source/Plugins/TypeSystem/Clang/TypeSystemClang.cpp
index 47051f2e68090..4901b6029d9ce 100644
--- a/lldb/source/Plugins/TypeSystem/Clang/TypeSystemClang.cpp
+++ b/lldb/source/Plugins/TypeSystem/Clang/TypeSystemClang.cpp
@@ -1666,6 +1666,12 @@ TypeSystemClang::CreateClassTemplateSpecializationDecl(
   ast.getTypeDeclType(class_template_specialization_decl, nullptr);
   class_template_specialization_decl->setDeclName(
       class_template_decl->getDeclName());
+
+  // FIXME: set to fixed value for now so it's not uninitialized.
+  // One way to determine StrictPackMatch would be
+  // Sema::CheckTemplateTemplateArgument.
+  class_template_specialization_decl->setStrictPackMatch(false);
+
   SetOwningModule(class_template_specialization_decl, owning_module);
   decl_ctx->addDecl(class_template_specialization_decl);
 
@@ -2217,12 +2223,6 @@ ParmVarDecl *TypeSystemClang::CreateParameterDeclaration(
   return decl;
 }
 
-void TypeSystemClang::SetFunctionParameters(
-    FunctionDecl *function_decl, llvm::ArrayRef<ParmVarDecl *> params) {
-  if (function_decl)
-    function_decl->setParams(params);
-}
-
 CompilerType
 TypeSystemClang::CreateBlockPointerType(const CompilerType &function_type) {
   QualType block_type = m_ast_up->getBlockPointerType(
@@ -2303,7 +2303,8 @@ CompilerType TypeSystemClang::GetOrCreateStructForIdentifier(
 CompilerType TypeSystemClang::CreateEnumerationType(
     llvm::StringRef name, clang::DeclContext *decl_ctx,
     OptionalClangModuleID owning_module, const Declaration &decl,
-    const CompilerType &integer_clang_type, bool is_scoped) {
+    const CompilerType &integer_clang_type, bool is_scoped,
+    std::optional<clang::EnumExtensibilityAttr::Kind> enum_kind) {
   // TODO: Do something intelligent with the Declaration object passed in
   // like maybe filling in the SourceLocation with it...
   ASTContext &ast = getASTContext();
@@ -2321,6 +2322,10 @@ CompilerType TypeSystemClang::CreateEnumerationType(
   if (decl_ctx)
     decl_ctx->addDecl(enum_decl);
 
+  if (enum_kind)
+    enum_decl->addAttr(
+        clang::EnumExtensibilityAttr::CreateImplicit(ast, *enum_kind));
+
   // TODO: check if we should be setting the promotion type too?
   enum_decl->setIntegerType(ClangUtil::GetQualType(integer_clang_type));
 
@@ -7708,6 +7713,32 @@ void TypeSystemClang::SetFloatingInitializerForVariable(
       ast, init_value, true, qt.getUnqualifiedType(), SourceLocation()));
 }
 
+llvm::SmallVector<clang::ParmVarDecl *>
+TypeSystemClang::CreateParameterDeclarations(
+    clang::FunctionDecl *func, const clang::FunctionProtoType &prototype,
+    const llvm::SmallVector<llvm::StringRef> &parameter_names) {
+  assert(func);
+  assert(parameter_names.empty() ||
+         parameter_names.size() == prototype.getNumParams());
+
+  llvm::SmallVector<clang::ParmVarDecl *> params;
+  for (unsigned param_index = 0; param_index < prototype.getNumParams();
+       ++param_index) {
+    llvm::StringRef name =
+        !parameter_names.empty() ? parameter_names[param_index] : "";
+
+    auto *param =
+        CreateParameterDeclaration(func, /*owning_module=*/{}, name.data(),
+                                   GetType(prototype.getParamType(param_index)),
+                                   clang::SC_None, /*add_decl=*/false);
+    assert(param);
+
+    params.push_back(param);
+  }
+
+  return params;
+}
+
 clang::CXXMethodDecl *TypeSystemClang::AddMethodToCXXRecordType(
     lldb::opaque_compiler_type_t type, llvm::StringRef name,
     const char *mangled_name, const CompilerType &method_clang_type,
@@ -7848,20 +7879,10 @@ clang::CXXMethodDecl *TypeSystemClang::AddMethodToCXXRecordType(
         getASTContext(), mangled_name, /*literal=*/false));
   }
 
-  // Populate the method decl with parameter decls
-
-  llvm::SmallVector<clang::ParmVarDecl *, 12> params;
-
-  for (unsigned param_index = 0; param_index < num_params; ++param_index) {
-    params.push_back(clang::ParmVarDecl::Create(
-        getASTContext(), cxx_method_decl, clang::SourceLocation(),
-        clang::SourceLocation(),
-        nullptr, // anonymous
-        method_function_prototype->getParamType(param_index), nullptr,
-        clang::SC_None, nullptr));
-  }
-
-  cxx_method_decl->setParams(llvm::ArrayRef<clang::ParmVarDecl *>(params));
+  // Parameters on member function declarations in DWARF generally don't
+  // have names, so we omit them when creating the ParmVarDecls.
+  cxx_method_decl->setParams(CreateParameterDeclarations(
+      cxx_method_decl, *method_function_prototype, /*parameter_names=*/{}));
 
   AddAccessSpecifierDecl(cxx_record_decl, getASTContext(),
                          GetCXXRecordDeclAccess(cxx_record_decl),
@@ -8531,12 +8552,10 @@ clang::EnumConstantDecl *TypeSystemClang::AddEnumerationValueToEnumerationType(
 
 clang::EnumConstantDecl *TypeSystemClang::AddEnumerationValueToEnumerationType(
     const CompilerType &enum_type, const Declaration &decl, const char *name,
-    int64_t enum_value, uint32_t enum_value_bit_size) {
-  CompilerType underlying_type = GetEnumerationIntegerType(enum_type);
-  bool is_signed = false;
-  underlying_type.IsIntegerType(is_signed);
-
-  llvm::APSInt value(enum_value_bit_size, !is_signed);
+    uint64_t enum_value, uint32_t enum_value_bit_size) {
+  assert(enum_type.IsEnumerationType());
+  llvm::APSInt value(enum_value_bit_size,
+                     !enum_type.IsEnumerationIntegerTypeSigned());
   value = enum_value;
 
   return AddEnumerationValueToEnumerationType(enum_type, decl, name, value);
diff --git a/lldb/source/Plugins/TypeSystem/Clang/TypeSystemClang.h b/lldb/source/Plugins/TypeSystem/Clang/TypeSystemClang.h
index 678eaed381fd4..99d9becffd128 100644
--- a/lldb/source/Plugins/TypeSystem/Clang/TypeSystemClang.h
+++ b/lldb/source/Plugins/TypeSystem/Clang/TypeSystemClang.h
@@ -22,6 +22,7 @@
 
 #include "clang/AST/ASTContext.h"
 #include "clang/AST/ASTFwd.h"
+#include "clang/AST/Attr.h"
 #include "clang/AST/Decl.h"
 #include "clang/AST/TemplateBase.h"
 #include "clang/AST/Type.h"
@@ -489,9 +490,6 @@ class TypeSystemClang : public TypeSystem {
                              const char *name, const CompilerType &param_type,
                              int storage, bool add_decl = false);
 
-  void SetFunctionParameters(clang::FunctionDecl *function_decl,
-                             llvm::ArrayRef<clang::ParmVarDecl *> params);
-
   CompilerType CreateBlockPointerType(const CompilerType &function_type);
 
   // Array Types
@@ -501,12 +499,12 @@ class TypeSystemClang : public TypeSystem {
                                bool is_vector);
 
   // Enumeration Types
-  CompilerType CreateEnumerationType(llvm::StringRef name,
-                                     clang::DeclContext *decl_ctx,
-                                     OptionalClangModuleID owning_module,
-                                     const Declaration &decl,
-                                     const CompilerType &integer_qual_type,
-                                     bool is_scoped);
+  CompilerType CreateEnumerationType(
+      llvm::StringRef name, clang::DeclContext *decl_ctx,
+      OptionalClangModuleID owning_module, const Declaration &decl,
+      const CompilerType &integer_qual_type, bool is_scoped,
+      std::optional<clang::EnumExtensibilityAttr::Kind> enum_kind =
+          std::nullopt);
 
   // Integer type functions
 
@@ -976,6 +974,24 @@ class TypeSystemClang : public TypeSystem {
   SetFloatingInitializerForVariable(clang::VarDecl *var,
                                     const llvm::APFloat &init_value);
 
+  /// For each parameter type of \c prototype, creates a \c clang::ParmVarDecl
+  /// whose \c clang::DeclContext is \c context.
+  ///
+  /// \param[in] context Non-null \c clang::FunctionDecl which will be the \c
+  /// clang::DeclContext of each parameter created/returned by this function.
+  /// \param[in] prototype The \c clang::FunctionProtoType of \c context.
+  /// \param[in] param_names The ith element of this vector contains the name
+  /// of the ith parameter. This parameter may be unnamed, in which case the
+  /// ith entry in \c param_names is an empty string. This vector is either
+  /// empty, or will have an entry for *each* parameter of the prototype
+  /// regardless of whether a parameter is unnamed or not.
+  ///
+  /// \returns A list of newly created of non-null \c clang::ParmVarDecl (one
+  /// for each parameter of \c prototype).
+  llvm::SmallVector<clang::ParmVarDecl *> CreateParameterDeclarations(
+      clang::FunctionDecl *context, const clang::FunctionProtoType &prototype,
+      const llvm::SmallVector<llvm::StringRef> &param_names);
+
   clang::CXXMethodDecl *AddMethodToCXXRecordType(
       lldb::opaque_compiler_type_t type, llvm::StringRef name,
       const char *mangled_name, const CompilerType &method_type,
@@ -1025,7 +1041,7 @@ class TypeSystemClang : public TypeSystem {
   // Modifying Enumeration types
   clang::EnumConstantDecl *AddEnumerationValueToEnumerationType(
       const CompilerType &enum_type, const Declaration &decl, const char *name,
-      int64_t enum_value, uint32_t enum_value_bit_size);
+      uint64_t enum_value, uint32_t enum_value_bit_size);
   clang::EnumConstantDecl *AddEnumerationValueToEnumerationType(
       const CompilerType &enum_type, const Declaration &decl, const char *name,
       const llvm::APSInt &value);
diff --git a/lldb/source/Symbol/SymbolContext.cpp b/lldb/source/Symbol/SymbolContext.cpp
index f4270ee839676..19f4f91e29d25 100644
--- a/lldb/source/Symbol/SymbolContext.cpp
+++ b/lldb/source/Symbol/SymbolContext.cpp
@@ -370,6 +370,16 @@ bool SymbolContext::GetAddressRange(uint32_t scope, uint32_t range_idx,
   return false;
 }
 
+Address SymbolContext::GetFunctionOrSymbolAddress() const {
+  if (function)
+    return function->GetAddress();
+
+  if (symbol)
+    return symbol->GetAddress();
+
+  return Address();
+}
+
 LanguageType SymbolContext::GetLanguage() const {
   LanguageType lang;
   if (function && (lang = function->GetLanguage()) != eLanguageTypeUnknown) {
diff --git a/lldb/source/Symbol/UnwindTable.cpp b/lldb/source/Symbol/UnwindTable.cpp
index da88b0c9c4ea1..61d51192bf3d1 100644
--- a/lldb/source/Symbol/UnwindTable.cpp
+++ b/lldb/source/Symbol/UnwindTable.cpp
@@ -86,6 +86,7 @@ void UnwindTable::Initialize() {
 void UnwindTable::ModuleWasUpdated() {
   std::lock_guard<std::mutex> guard(m_mutex);
   m_scanned_all_unwind_sources = false;
+  m_unwinds.clear();
 }
 
 UnwindTable::~UnwindTable() = default;
@@ -118,7 +119,7 @@ UnwindTable::GetAddressRange(const Address &addr, const SymbolContext &sc) {
 
 FuncUnwindersSP
 UnwindTable::GetFuncUnwindersContainingAddress(const Address &addr,
-                                               SymbolContext &sc) {
+                                               const SymbolContext &sc) {
   Initialize();
 
   std::lock_guard<std::mutex> guard(m_mutex);
diff --git a/lldb/source/Target/Process.cpp b/lldb/source/Target/Process.cpp
index 89731f798deda..428f8519b72fd 100644
--- a/lldb/source/Target/Process.cpp
+++ b/lldb/source/Target/Process.cpp
@@ -6677,11 +6677,8 @@ static void GetUserSpecifiedCoreFileSaveRanges(Process &process,
 
   for (const auto &range : regions) {
     auto entry = option_ranges.FindEntryThatContains(range.GetRange());
-    if (entry) {
-      ranges.Append(range.GetRange().GetRangeBase(),
-                    range.GetRange().GetByteSize(),
-                    CreateCoreFileMemoryRange(range));
-    }
+    if (entry)
+      AddRegion(range, true, ranges);
   }
 }
 
diff --git a/lldb/source/Target/ThreadList.cpp b/lldb/source/Target/ThreadList.cpp
index 6cbef330bf488..c0440d82fd1ff 100644
--- a/lldb/source/Target/ThreadList.cpp
+++ b/lldb/source/Target/ThreadList.cpp
@@ -191,20 +191,6 @@ ThreadSP ThreadList::GetThreadSPForThreadPtr(Thread *thread_ptr) {
   return thread_sp;
 }
 
-ThreadSP ThreadList::GetBackingThread(const ThreadSP &real_thread) {
-  std::lock_guard<std::recursive_mutex> guard(GetMutex());
-
-  ThreadSP thread_sp;
-  const uint32_t num_threads = m_threads.size();
-  for (uint32_t idx = 0; idx < num_threads; ++idx) {
-    if (m_threads[idx]->GetBackingThread() == real_thread) {
-      thread_sp = m_threads[idx];
-      break;
-    }
-  }
-  return thread_sp;
-}
-
 ThreadSP ThreadList::FindThreadByIndexID(uint32_t index_id, bool can_update) {
   std::lock_guard<std::recursive_mutex> guard(GetMutex());
 
diff --git a/lldb/source/Utility/ArchSpec.cpp b/lldb/source/Utility/ArchSpec.cpp
index b13e8ff1ec373..495215459336a 100644
--- a/lldb/source/Utility/ArchSpec.cpp
+++ b/lldb/source/Utility/ArchSpec.cpp
@@ -1040,6 +1040,12 @@ bool ArchSpec::IsMatch(const ArchSpec &rhs, MatchType match) const {
          rhs_triple_os == llvm::Triple::IOS &&
          rhs_triple_env == llvm::Triple::MacABI))
       return true;
+    // x86_64-apple-driverkit, x86_64-apple-macosx are compatible, no match.
+    if ((lhs_triple_os == llvm::Triple::DriverKit &&
+         rhs_triple_os == llvm::Triple::MacOSX) ||
+        (lhs_triple_os == llvm::Triple::MacOSX &&
+         rhs_triple_os == llvm::Triple::DriverKit))
+      return true;
   }
 
   // x86_64-apple-ios-macabi and x86_64-apple-ios are not compatible.
diff --git a/lldb/source/ValueObject/CMakeLists.txt b/lldb/source/ValueObject/CMakeLists.txt
index 70cb3d6d53f07..30c34472289e7 100644
--- a/lldb/source/ValueObject/CMakeLists.txt
+++ b/lldb/source/ValueObject/CMakeLists.txt
@@ -1,4 +1,5 @@
 add_lldb_library(lldbValueObject
+  DILLexer.cpp
   ValueObject.cpp
   ValueObjectCast.cpp
   ValueObjectChild.cpp
diff --git a/lldb/source/ValueObject/DILLexer.cpp b/lldb/source/ValueObject/DILLexer.cpp
new file mode 100644
index 0000000000000..c7acfec347af4
--- /dev/null
+++ b/lldb/source/ValueObject/DILLexer.cpp
@@ -0,0 +1,97 @@
+//===-- DILLexer.cpp ------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+// This implements the recursive descent parser for the Data Inspection
+// Language (DIL), and its helper functions, which will eventually underlie the
+// 'frame variable' command. The language that this parser recognizes is
+// described in lldb/docs/dil-expr-lang.ebnf
+//
+//===----------------------------------------------------------------------===//
+
+#include "lldb/ValueObject/DILLexer.h"
+#include "lldb/Utility/Status.h"
+#include "llvm/ADT/StringSwitch.h"
+
+namespace lldb_private::dil {
+
+llvm::StringRef Token::GetTokenName(Kind kind) {
+  switch (kind) {
+  case Kind::coloncolon:
+    return "coloncolon";
+  case Kind::eof:
+    return "eof";
+  case Kind::identifier:
+    return "identifier";
+  case Kind::l_paren:
+    return "l_paren";
+  case Kind::r_paren:
+    return "r_paren";
+  }
+}
+
+static bool IsLetter(char c) {
+  return ('a' <= c && c <= 'z') || ('A' <= c && c <= 'Z');
+}
+
+static bool IsDigit(char c) { return '0' <= c && c <= '9'; }
+
+// A word starts with a letter, underscore, or dollar sign, followed by
+// letters ('a'..'z','A'..'Z'), digits ('0'..'9'), and/or  underscores.
+static std::optional<llvm::StringRef> IsWord(llvm::StringRef expr,
+                                             llvm::StringRef &remainder) {
+  // Find the longest prefix consisting of letters, digits, underscors and
+  // '$'. If it doesn't start with a digit, then it's a word.
+  llvm::StringRef candidate = remainder.take_while(
+      [](char c) { return IsDigit(c) || IsLetter(c) || c == '_' || c == '$'; });
+  if (candidate.empty() || IsDigit(candidate[0]))
+    return std::nullopt;
+  remainder = remainder.drop_front(candidate.size());
+  return candidate;
+}
+
+llvm::Expected<DILLexer> DILLexer::Create(llvm::StringRef expr) {
+  std::vector<Token> tokens;
+  llvm::StringRef remainder = expr;
+  do {
+    if (llvm::Expected<Token> t = Lex(expr, remainder)) {
+      tokens.push_back(std::move(*t));
+    } else {
+      return t.takeError();
+    }
+  } while (tokens.back().GetKind() != Token::eof);
+  return DILLexer(expr, std::move(tokens));
+}
+
+llvm::Expected<Token> DILLexer::Lex(llvm::StringRef expr,
+                                    llvm::StringRef &remainder) {
+  // Skip over whitespace (spaces).
+  remainder = remainder.ltrim();
+  llvm::StringRef::iterator cur_pos = remainder.begin();
+
+  // Check to see if we've reached the end of our input string.
+  if (remainder.empty())
+    return Token(Token::eof, "", (uint32_t)expr.size());
+
+  uint32_t position = cur_pos - expr.begin();
+  std::optional<llvm::StringRef> maybe_word = IsWord(expr, remainder);
+  if (maybe_word)
+    return Token(Token::identifier, maybe_word->str(), position);
+
+  constexpr std::pair<Token::Kind, const char *> operators[] = {
+      {Token::l_paren, "("},
+      {Token::r_paren, ")"},
+      {Token::coloncolon, "::"},
+  };
+  for (auto [kind, str] : operators) {
+    if (remainder.consume_front(str))
+      return Token(kind, str, position);
+  }
+
+  // Unrecognized character(s) in string; unable to lex it.
+  return llvm::createStringError("Unable to lex input string");
+}
+
+} // namespace lldb_private::dil
diff --git a/lldb/source/ValueObject/ValueObject.cpp b/lldb/source/ValueObject/ValueObject.cpp
index 2864af107b925..9d98f62c0379b 100644
--- a/lldb/source/ValueObject/ValueObject.cpp
+++ b/lldb/source/ValueObject/ValueObject.cpp
@@ -849,6 +849,22 @@ bool ValueObject::SetData(DataExtractor &data, Status &error) {
   return true;
 }
 
+llvm::ArrayRef<uint8_t> ValueObject::GetLocalBuffer() const {
+  if (m_value.GetValueType() != Value::ValueType::HostAddress)
+    return {};
+  auto start = m_value.GetScalar().ULongLong(LLDB_INVALID_ADDRESS);
+  if (start == LLDB_INVALID_ADDRESS)
+    return {};
+  // Does our pointer point to this value object's m_data buffer?
+  if ((uint64_t)m_data.GetDataStart() == start)
+    return m_data.GetData();
+  // Does our pointer point to the value's buffer?
+  if ((uint64_t)m_value.GetBuffer().GetBytes() == start)
+    return m_value.GetBuffer().GetData();
+  // Our pointer points to something else. We can't know what the size is.
+  return {};
+}
+
 static bool CopyStringDataToBufferSP(const StreamString &source,
                                      lldb::WritableDataBufferSP &destination) {
   llvm::StringRef src = source.GetString();
diff --git a/lldb/source/ValueObject/ValueObjectDynamicValue.cpp b/lldb/source/ValueObject/ValueObjectDynamicValue.cpp
index 588c644bbfd07..ecd663af68c2d 100644
--- a/lldb/source/ValueObject/ValueObjectDynamicValue.cpp
+++ b/lldb/source/ValueObject/ValueObjectDynamicValue.cpp
@@ -145,6 +145,7 @@ bool ValueObjectDynamicValue::UpdateValue() {
   Address dynamic_address;
   bool found_dynamic_type = false;
   Value::ValueType value_type;
+  llvm::ArrayRef<uint8_t> local_buffer;
 
   LanguageRuntime *runtime = nullptr;
 
@@ -157,7 +158,7 @@ bool ValueObjectDynamicValue::UpdateValue() {
       // Try the preferred runtime first.
       found_dynamic_type = preferred_runtime->GetDynamicTypeAndAddress(
           *m_parent, m_use_dynamic, class_type_or_name, dynamic_address,
-          value_type);
+          value_type, local_buffer);
       if (found_dynamic_type)
         // Set the operative `runtime` for later use in this function.
         runtime = preferred_runtime;
@@ -166,20 +167,20 @@ bool ValueObjectDynamicValue::UpdateValue() {
       // Fallback to the runtime for `known_type`.
       found_dynamic_type = runtime->GetDynamicTypeAndAddress(
           *m_parent, m_use_dynamic, class_type_or_name, dynamic_address,
-          value_type);
+          value_type, local_buffer);
   } else {
     runtime = process->GetLanguageRuntime(lldb::eLanguageTypeC_plus_plus);
     if (runtime)
       found_dynamic_type = runtime->GetDynamicTypeAndAddress(
           *m_parent, m_use_dynamic, class_type_or_name, dynamic_address,
-          value_type);
+          value_type, local_buffer);
 
     if (!found_dynamic_type) {
       runtime = process->GetLanguageRuntime(lldb::eLanguageTypeObjC);
       if (runtime)
         found_dynamic_type = runtime->GetDynamicTypeAndAddress(
             *m_parent, m_use_dynamic, class_type_or_name, dynamic_address,
-            value_type);
+            value_type, local_buffer);
     }
   }
 
@@ -239,11 +240,29 @@ bool ValueObjectDynamicValue::UpdateValue() {
     if (m_address.IsValid())
       SetValueDidChange(true);
 
-    // We've moved, so we should be fine...
-    m_address = dynamic_address;
-    lldb::TargetSP target_sp(GetTargetSP());
-    lldb::addr_t load_address = m_address.GetLoadAddress(target_sp.get());
-    m_value.GetScalar() = load_address;
+    // If we found a host address, and the dynamic type fits in the local buffer
+    // that was found, point to that buffer. Later on this function will copy
+    // the buffer over.
+    if (value_type == Value::ValueType::HostAddress && !local_buffer.empty()) {
+      auto *exe_scope = exe_ctx.GetBestExecutionContextScope();
+      // If we found a host address but it doesn't fit in the buffer, there's
+      // nothing we can do.
+      if (local_buffer.size() <
+          m_dynamic_type_info.GetCompilerType().GetByteSize(exe_scope)) {
+        SetValueIsValid(false);
+        return false;
+      }
+
+      m_value.GetScalar() = (uint64_t)local_buffer.data();
+      m_address = LLDB_INVALID_ADDRESS;
+    } else {
+      // Otherwise we have a legitimate address on the target. Point to the load
+      // address.
+      m_address = dynamic_address;
+      lldb::TargetSP target_sp(GetTargetSP());
+      lldb::addr_t load_address = m_address.GetLoadAddress(target_sp.get());
+      m_value.GetScalar() = load_address;
+    }
   }
 
   if (runtime)
@@ -258,7 +277,11 @@ bool ValueObjectDynamicValue::UpdateValue() {
     LLDB_LOGF(log, "[%s %p] has a new dynamic type %s", GetName().GetCString(),
               static_cast<void *>(this), GetTypeName().GetCString());
 
-  if (m_address.IsValid() && m_dynamic_type_info) {
+  // m_address could be invalid but we could still have a local buffer
+  // containing the dynamic value.
+  if ((m_address.IsValid() ||
+       m_value.GetValueType() == Value::ValueType::HostAddress) &&
+      m_dynamic_type_info) {
     // The variable value is in the Scalar value inside the m_value. We can
     // point our m_data right to it.
     m_error = m_value.GetValueAsData(&exe_ctx, m_data, GetModule().get());
diff --git a/lldb/test/API/api/command-return-object/TestSBCommandReturnObject.py b/lldb/test/API/api/command-return-object/TestSBCommandReturnObject.py
index f2dbbbd7b4d42..2193b7270d0b4 100644
--- a/lldb/test/API/api/command-return-object/TestSBCommandReturnObject.py
+++ b/lldb/test/API/api/command-return-object/TestSBCommandReturnObject.py
@@ -24,3 +24,12 @@ def test_sb_command_return_object(self):
         # return exit code 0 to indicate success.  We can let this exception go
         # - the test harness will recognize it as a test failure.
         subprocess.check_call([self.driver_exe, self.driver_exe])
+
+    def test_get_command(self):
+        res = lldb.SBCommandReturnObject()
+        self.assertEqual(res.GetCommand(), "")
+
+        ci = self.dbg.GetCommandInterpreter()
+        ci.HandleCommand("help help", res)
+        self.assertTrue(res.Succeeded())
+        self.assertEqual(res.GetCommand(), "help help")
diff --git a/lldb/test/API/commands/register/register/register_command/TestRegisters.py b/lldb/test/API/commands/register/register/register_command/TestRegisters.py
index 99290e02cd2b0..5bf7aa5dee9c4 100644
--- a/lldb/test/API/commands/register/register/register_command/TestRegisters.py
+++ b/lldb/test/API/commands/register/register/register_command/TestRegisters.py
@@ -111,7 +111,6 @@ def test_fp_register_write(self):
     # "register read fstat" always return 0xffff
     @expectedFailureAndroid(archs=["i386"])
     @skipIf(archs=no_match(["amd64", "i386", "x86_64"]))
-    @skipIfOutOfTreeDebugserver
     @expectedFailureAll(oslist=["windows"], bugnumber="llvm.org/pr37995")
     def test_fp_special_purpose_register_read(self):
         """Test commands that read fpu special purpose registers."""
diff --git a/lldb/test/API/commands/watchpoints/unaligned-watchpoint/TestUnalignedWatchpoint.py b/lldb/test/API/commands/watchpoints/unaligned-watchpoint/TestUnalignedWatchpoint.py
index 68f239668cf08..5f9e52855da16 100644
--- a/lldb/test/API/commands/watchpoints/unaligned-watchpoint/TestUnalignedWatchpoint.py
+++ b/lldb/test/API/commands/watchpoints/unaligned-watchpoint/TestUnalignedWatchpoint.py
@@ -16,7 +16,6 @@
 class UnalignedWatchpointTestCase(TestBase):
     NO_DEBUG_INFO_TESTCASE = True
 
-    @skipIfOutOfTreeDebugserver
     def test_unaligned_watchpoint(self):
         """Test an unaligned watchpoint triggered by a larger aligned write."""
         self.build()
diff --git a/lldb/test/API/commands/watchpoints/watchpoint_events/TestWatchpointEvents.py b/lldb/test/API/commands/watchpoints/watchpoint_events/TestWatchpointEvents.py
index 726a9d93c29d4..6e05cf06204a7 100644
--- a/lldb/test/API/commands/watchpoints/watchpoint_events/TestWatchpointEvents.py
+++ b/lldb/test/API/commands/watchpoints/watchpoint_events/TestWatchpointEvents.py
@@ -82,27 +82,45 @@ def test_with_python_api(self):
             'make sure watchpoint condition is "' + condition + '"',
         )
 
-    def GetWatchpointEvent(self, event_type):
-        # We added a watchpoint so we should get a watchpoint added event.
-        event = lldb.SBEvent()
-        success = self.listener.WaitForEvent(1, event)
-        self.assertTrue(success, "Successfully got watchpoint event")
-        self.assertTrue(
-            lldb.SBWatchpoint.EventIsWatchpointEvent(event),
-            "Event is a watchpoint event.",
+        target.DeleteWatchpoint(local_watch.GetID())
+        self.GetWatchpointEvent(
+            lldb.eWatchpointEventTypeDisabled, lldb.eWatchpointEventTypeRemoved
         )
-        found_type = lldb.SBWatchpoint.GetWatchpointEventTypeFromEvent(event)
-        self.assertEqual(
-            found_type,
-            event_type,
-            "Event is not correct type, expected: %d, found: %d"
-            % (event_type, found_type),
+
+        # Re-create it so that we can check DeleteAllWatchpoints
+        local_watch = local_var.Watch(True, False, True, error)
+        if not error.Success():
+            self.fail(
+                "Failed to make watchpoint for local_var: %s" % (error.GetCString())
+            )
+        self.GetWatchpointEvent(lldb.eWatchpointEventTypeAdded)
+        target.DeleteAllWatchpoints()
+        self.GetWatchpointEvent(
+            lldb.eWatchpointEventTypeDisabled, lldb.eWatchpointEventTypeRemoved
         )
+
+    def GetWatchpointEvent(self, *event_types):
+        # We added a watchpoint so we should get a watchpoint added event.
+        event = lldb.SBEvent()
+        for event_type in event_types:
+            success = self.listener.WaitForEvent(1, event)
+            self.assertTrue(success, "Successfully got watchpoint event")
+            self.assertTrue(
+                lldb.SBWatchpoint.EventIsWatchpointEvent(event),
+                "Event is a watchpoint event.",
+            )
+            found_type = lldb.SBWatchpoint.GetWatchpointEventTypeFromEvent(event)
+            self.assertEqual(
+                found_type,
+                event_type,
+                "Event is not correct type, expected: %d, found: %d"
+                % (event_type, found_type),
+            )
         # There shouldn't be another event waiting around:
         found_event = self.listener.PeekAtNextEventForBroadcasterWithType(
-            self.target_bcast, lldb.SBTarget.eBroadcastBitBreakpointChanged, event
+            self.target_bcast, lldb.SBTarget.eBroadcastBitWatchpointChanged, event
         )
         if found_event:
-            print("Found an event I didn't expect: ", event)
+            print("Found an event I didn't expect: ", event.GetType())
 
-        self.assertTrue(not found_event, "Only one event per change.")
+        self.assertTrue(not found_event, f"Only expected {len(event_types)} events.")
diff --git a/lldb/test/API/driver/terminal/TestTerminalDimensions.py b/lldb/test/API/driver/terminal/TestTerminalDimensions.py
new file mode 100644
index 0000000000000..9ab3745613994
--- /dev/null
+++ b/lldb/test/API/driver/terminal/TestTerminalDimensions.py
@@ -0,0 +1,22 @@
+import lldb
+from lldbsuite.test.decorators import *
+from lldbsuite.test.lldbtest import *
+from lldbsuite.test import lldbutil
+from lldbsuite.test.lldbpexpect import PExpectTest
+
+
+class TerminalDimensionsTest(PExpectTest):
+    NO_DEBUG_INFO_TESTCASE = True
+
+    @skipIfAsan
+    def test(self):
+        """Test that the lldb driver correctly reports the (PExpect) terminal dimension."""
+        self.launch(dimensions=(40, 40))
+
+        # Tests clear all the settings so we lose the launch values. Resize the
+        # window to update the settings. These new values need to be different
+        # to trigger a SIGWINCH.
+        self.child.setwinsize(20, 60)
+
+        self.expect("settings show term-height", ["term-height (unsigned) = 20"])
+        self.expect("settings show term-width", ["term-width (unsigned) = 60"])
diff --git a/lldb/test/API/functionalities/abbreviation/TestAbbreviations.py b/lldb/test/API/functionalities/abbreviation/TestAbbreviations.py
index a8cbffbb7ba4a..cc767edaaa619 100644
--- a/lldb/test/API/functionalities/abbreviation/TestAbbreviations.py
+++ b/lldb/test/API/functionalities/abbreviation/TestAbbreviations.py
@@ -45,10 +45,12 @@ def test_command_abbreviations_and_aliases(self):
 
         # Make sure an unabbreviated command is not mangled.
         command_interpreter.ResolveCommand(
-            "breakpoint set --name main --line 123", result
+            "breakpoint set --name main --ignore-count 123", result
         )
         self.assertTrue(result.Succeeded())
-        self.assertEqual("breakpoint set --name main --line 123", result.GetOutput())
+        self.assertEqual(
+            "breakpoint set --name main --ignore-count 123", result.GetOutput()
+        )
 
         # Create some aliases.
         self.runCmd("com a alias com al")
@@ -72,10 +74,10 @@ def test_command_abbreviations_and_aliases(self):
             "process launch -s -o /dev/tty0 -e /dev/tty0", result.GetOutput()
         )
 
-        self.runCmd("alias xyzzy breakpoint set -n %1 -l %2")
+        self.runCmd("alias xyzzy breakpoint set -n %1 -i %2")
         command_interpreter.ResolveCommand("xyzzy main 123", result)
         self.assertTrue(result.Succeeded())
-        self.assertEqual("breakpoint set -n main -l 123", result.GetOutput().strip())
+        self.assertEqual("breakpoint set -n main -i 123", result.GetOutput().strip())
 
         # And again, without enough parameters.
         command_interpreter.ResolveCommand("xyzzy main", result)
@@ -92,6 +94,12 @@ def test_command_abbreviations_and_aliases(self):
         self.assertTrue(result.Succeeded())
         self.assertEqual("scripting run 1+1", result.GetOutput())
 
+        # Name and line are incompatible options.
+        command_interpreter.HandleCommand(
+            "alias zzyx breakpoint set -n %1 -l %2", result
+        )
+        self.assertFalse(result.Succeeded())
+
         # Prompt changing stuff should be tested, but this doesn't seem like the
         # right test to do it in.  It has nothing to do with aliases or abbreviations.
         # self.runCmd("com sou ./change_prompt.lldb")
diff --git a/lldb/test/API/functionalities/breakpoint/hardware_breakpoints/hardware_breakpoint_on_multiple_threads/TestHWBreakMultiThread.py b/lldb/test/API/functionalities/breakpoint/hardware_breakpoints/hardware_breakpoint_on_multiple_threads/TestHWBreakMultiThread.py
index 6fe74a8a3fb8c..1a0515aa04c07 100644
--- a/lldb/test/API/functionalities/breakpoint/hardware_breakpoints/hardware_breakpoint_on_multiple_threads/TestHWBreakMultiThread.py
+++ b/lldb/test/API/functionalities/breakpoint/hardware_breakpoints/hardware_breakpoint_on_multiple_threads/TestHWBreakMultiThread.py
@@ -15,14 +15,12 @@ class HardwareBreakpointMultiThreadTestCase(HardwareBreakpointTestBase):
     def does_not_support_hw_breakpoints(self):
         return not super().supports_hw_breakpoints()
 
-    @skipIfOutOfTreeDebugserver
     @skipTestIfFn(does_not_support_hw_breakpoints)
     def test_hw_break_set_delete_multi_thread_macos(self):
         self.build()
         self.setTearDownCleanup()
         self.break_multi_thread("delete")
 
-    @skipIfOutOfTreeDebugserver
     @skipTestIfFn(does_not_support_hw_breakpoints)
     def test_hw_break_set_disable_multi_thread_macos(self):
         self.build()
diff --git a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/atomic/TestLibCxxAtomic.py b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/atomic/TestLibCxxAtomic.py
index 241226d50df80..c6592ede03147 100644
--- a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/atomic/TestLibCxxAtomic.py
+++ b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/atomic/TestLibCxxAtomic.py
@@ -2,6 +2,7 @@
 Test lldb data formatter subsystem.
 """
 
+import re
 import lldb
 from lldbsuite.test.decorators import *
 from lldbsuite.test.lldbtest import *
@@ -30,9 +31,7 @@ def test(self):
 
         self.runCmd("run", RUN_SUCCEEDED)
 
-        lldbutil.skip_if_library_missing(
-            self, self.target(), lldbutil.PrintableRegex("libc\+\+")
-        )
+        lldbutil.skip_if_library_missing(self, self.target(), re.compile(r"libc\+\+"))
 
         # The stop reason of the thread should be breakpoint.
         self.expect(
diff --git a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/deque/TestDataFormatterLibcxxDeque.py b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/deque/TestDataFormatterLibcxxDeque.py
index 3596b546be306..ece1e4f5f049b 100644
--- a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/deque/TestDataFormatterLibcxxDeque.py
+++ b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/deque/TestDataFormatterLibcxxDeque.py
@@ -9,22 +9,27 @@
 
 
 class LibcxxDequeDataFormatterTestCase(TestBase):
-    def check_numbers(self, var_name):
+    def check_numbers(self, var_name, show_ptr=False):
+        patterns = []
+        substrs = [
+            "[0] = 1",
+            "[1] = 12",
+            "[2] = 123",
+            "[3] = 1234",
+            "[4] = 12345",
+            "[5] = 123456",
+            "[6] = 1234567",
+            "}",
+        ]
+        if show_ptr:
+            patterns = [var_name + " = 0x.* size=7"]
+        else:
+            substrs.insert(0, var_name + " = size=7")
         self.expect(
             "frame variable " + var_name,
-            substrs=[
-                var_name + " = size=7",
-                "[0] = 1",
-                "[1] = 12",
-                "[2] = 123",
-                "[3] = 1234",
-                "[4] = 12345",
-                "[5] = 123456",
-                "[6] = 1234567",
-                "}",
-            ],
+            patterns=patterns,
+            substrs=substrs,
         )
-
         self.expect_expr(
             var_name,
             result_summary="size=7",
@@ -75,7 +80,7 @@ def test_ref_and_ptr(self):
         )
 
         # The reference should display the same was as the value did
-        self.check_numbers("ref")
+        self.check_numbers("ref", True)
 
         # The pointer should just show the right number of elements:
         self.expect("frame variable ptr", substrs=["ptr =", " size=7"])
diff --git a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/initializerlist/TestInitializerList.py b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/initializerlist/TestInitializerList.py
index 93d5392830b50..b8a1dd3569d77 100644
--- a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/initializerlist/TestInitializerList.py
+++ b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/initializerlist/TestInitializerList.py
@@ -3,6 +3,7 @@
 """
 
 
+import re
 import lldb
 from lldbsuite.test.decorators import *
 from lldbsuite.test.lldbtest import *
@@ -24,9 +25,7 @@ def test(self):
 
         self.runCmd("run", RUN_SUCCEEDED)
 
-        lldbutil.skip_if_library_missing(
-            self, self.target(), lldbutil.PrintableRegex("libc\+\+")
-        )
+        lldbutil.skip_if_library_missing(self, self.target(), re.compile(r"libc\+\+"))
 
         # The stop reason of the thread should be breakpoint.
         self.expect(
diff --git a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/span/TestDataFormatterLibcxxSpan.py b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/span/TestDataFormatterLibcxxSpan.py
index d5de73ac14ca1..4df4fa1acc8e7 100644
--- a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/span/TestDataFormatterLibcxxSpan.py
+++ b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/span/TestDataFormatterLibcxxSpan.py
@@ -172,7 +172,4 @@ def test_ref_and_ptr(self):
 
         # The pointer should just show the right number of elements:
 
-        ptrAddr = self.findVariable("ptr").GetValue()
-        self.expect_expr(
-            "ptr", result_type="std::span<int, 5> *", result_summary=f"{ptrAddr} size=5"
-        )
+        self.expect("frame variable ptr", patterns=["ptr = 0x.*", " size=5"])
diff --git a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/variant/TestDataFormatterLibcxxVariant.py b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/variant/TestDataFormatterLibcxxVariant.py
index 4154ad3c297fa..47e07a5ce3f5b 100644
--- a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/variant/TestDataFormatterLibcxxVariant.py
+++ b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/variant/TestDataFormatterLibcxxVariant.py
@@ -47,7 +47,7 @@ def test_with_run_command(self):
 
         self.expect(
             "frame variable v1_ref",
-            substrs=["v1_ref =  Active Type = int : {", "Value = 12", "}"],
+            patterns=["v1_ref = 0x.* Active Type = int : {", "Value = 12", "}"],
         )
 
         self.expect(
diff --git a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/vector/TestDataFormatterLibcxxVector.py b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/vector/TestDataFormatterLibcxxVector.py
index a475c15d3da34..ca0f0e3693116 100644
--- a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/vector/TestDataFormatterLibcxxVector.py
+++ b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/vector/TestDataFormatterLibcxxVector.py
@@ -10,22 +10,28 @@
 
 
 class LibcxxVectorDataFormatterTestCase(TestBase):
-    def check_numbers(self, var_name):
+    def check_numbers(self, var_name, show_ptr=False):
+        patterns = []
+        substrs = [
+            "[0] = 1",
+            "[1] = 12",
+            "[2] = 123",
+            "[3] = 1234",
+            "[4] = 12345",
+            "[5] = 123456",
+            "[6] = 1234567",
+            "}",
+        ]
+        if show_ptr:
+            patterns = [var_name + " = 0x.* size=7"]
+        else:
+            substrs.insert(0, var_name + " = size=7")
+
         self.expect(
             "frame variable " + var_name,
-            substrs=[
-                var_name + " = size=7",
-                "[0] = 1",
-                "[1] = 12",
-                "[2] = 123",
-                "[3] = 1234",
-                "[4] = 12345",
-                "[5] = 123456",
-                "[6] = 1234567",
-                "}",
-            ],
+            patterns=patterns,
+            substrs=substrs,
         )
-
         self.expect_expr(
             var_name,
             result_summary="size=7",
@@ -174,7 +180,7 @@ def test_ref_and_ptr(self):
         )
 
         # The reference should display the same was as the value did
-        self.check_numbers("ref")
+        self.check_numbers("ref", True)
 
         # The pointer should just show the right number of elements:
 
diff --git a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libstdcpp/variant/TestDataFormatterLibStdcxxVariant.py b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libstdcpp/variant/TestDataFormatterLibStdcxxVariant.py
index ea4a53fcb4097..394e221809f7c 100644
--- a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libstdcpp/variant/TestDataFormatterLibStdcxxVariant.py
+++ b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libstdcpp/variant/TestDataFormatterLibStdcxxVariant.py
@@ -30,7 +30,7 @@ def test_with_run_command(self):
         for name in ["v1_ref", "v1_typedef_ref"]:
             self.expect(
                 "frame variable " + name,
-                substrs=[name + " =  Active Type = int : {", "Value = 12", "}"],
+                patterns=[name + " = 0x.*  Active Type = int : {", "Value = 12", "}"],
             )
 
         self.expect(
diff --git a/lldb/test/API/functionalities/gdb_remote_client/TestReadMemory.py b/lldb/test/API/functionalities/gdb_remote_client/TestReadMemory.py
new file mode 100644
index 0000000000000..81dcb54aef5d8
--- /dev/null
+++ b/lldb/test/API/functionalities/gdb_remote_client/TestReadMemory.py
@@ -0,0 +1,55 @@
+import lldb
+from lldbsuite.support import seven
+from lldbsuite.test.lldbtest import *
+from lldbsuite.test.decorators import *
+from lldbsuite.test.gdbclientutils import *
+from lldbsuite.test.lldbgdbclient import GDBRemoteTestBase
+
+
+class TestReadMemory(GDBRemoteTestBase):
+    def test_x_with_prefix(self):
+        class MyResponder(MockGDBServerResponder):
+            def qSupported(self, client_features):
+                # binary-upload+ indicates we use the gdb style of x packets
+                return super().qSupported(client_features) + ";binary-upload+"
+
+            def x(self, addr, length):
+                return "bfoobar" if addr == 0x1000 else "E01"
+
+        self.server.responder = MyResponder()
+        target = self.dbg.CreateTargetWithFileAndTargetTriple("", "x86_64-pc-linux")
+        process = self.connect(target)
+
+        error = lldb.SBError()
+        self.assertEqual(b"foobar", process.ReadMemory(0x1000, 10, error))
+
+    def test_x_bare(self):
+        class MyResponder(MockGDBServerResponder):
+            def x(self, addr, length):
+                # The OK response indicates we use the old lldb style.
+                if addr == 0 and length == 0:
+                    return "OK"
+                return "foobar" if addr == 0x1000 else "E01"
+
+        self.server.responder = MyResponder()
+        target = self.dbg.CreateTargetWithFileAndTargetTriple("", "x86_64-pc-linux")
+        process = self.connect(target)
+
+        error = lldb.SBError()
+        self.assertEqual(b"foobar", process.ReadMemory(0x1000, 10, error))
+
+    def test_m_fallback(self):
+        class MyResponder(MockGDBServerResponder):
+            def x(self, addr, length):
+                # If `x` is unsupported, we should fall back to `m`.
+                return ""
+
+            def readMemory(self, addr, length):
+                return seven.hexlify("foobar") if addr == 0x1000 else "E01"
+
+        self.server.responder = MyResponder()
+        target = self.dbg.CreateTargetWithFileAndTargetTriple("", "x86_64-pc-linux")
+        process = self.connect(target)
+
+        error = lldb.SBError()
+        self.assertEqual(b"foobar", process.ReadMemory(0x1000, 10, error))
diff --git a/lldb/test/API/functionalities/plugins/python_os_plugin/TestPythonOSPlugin.py b/lldb/test/API/functionalities/plugins/python_os_plugin/TestPythonOSPlugin.py
index 3ad7539018d5d..fe78edd98f4d4 100644
--- a/lldb/test/API/functionalities/plugins/python_os_plugin/TestPythonOSPlugin.py
+++ b/lldb/test/API/functionalities/plugins/python_os_plugin/TestPythonOSPlugin.py
@@ -160,6 +160,8 @@ def run_python_os_step(self):
         )
         self.assertTrue(process, PROCESS_IS_VALID)
 
+        core_thread_zero = process.GetThreadAtIndex(0)
+
         # Make sure there are no OS plug-in created thread when we first stop
         # at our breakpoint in main
         thread = process.GetThreadByID(0x111111111)
@@ -183,6 +185,10 @@ def run_python_os_step(self):
             thread.IsValid(),
             "Make sure there is a thread 0x111111111 after we load the python OS plug-in",
         )
+        # This OS plugin does not set thread names / queue names, so it should
+        # inherit the core thread's name.
+        self.assertEqual(core_thread_zero.GetName(), thread.GetName())
+        self.assertEqual(core_thread_zero.GetQueueName(), thread.GetQueueName())
 
         frame = thread.GetFrameAtIndex(0)
         self.assertTrue(
diff --git a/lldb/test/API/functionalities/process_save_core_minidump/TestProcessSaveCoreMinidump.py b/lldb/test/API/functionalities/process_save_core_minidump/TestProcessSaveCoreMinidump.py
index 808de687e6ea2..c2152640c425c 100644
--- a/lldb/test/API/functionalities/process_save_core_minidump/TestProcessSaveCoreMinidump.py
+++ b/lldb/test/API/functionalities/process_save_core_minidump/TestProcessSaveCoreMinidump.py
@@ -636,3 +636,42 @@ def minidump_saves_fs_base_region(self):
             self.assertTrue(self.dbg.DeleteTarget(target))
             if os.path.isfile(tls_file):
                 os.unlink(tls_file)
+
+    @skipUnlessPlatform(["linux"])
+    @skipUnlessArch("x86_64")
+    def test_invalid_custom_regions_not_included(self):
+        options = lldb.SBSaveCoreOptions()
+        self.build()
+        exe = self.getBuildArtifact("a.out")
+        output_file = self.getBuildArtifact("no_empty_regions.dmp")
+        try:
+            target = self.dbg.CreateTarget(exe)
+            process = target.LaunchSimple(
+                None, None, self.get_process_working_directory()
+            )
+            self.assertState(process.GetState(), lldb.eStateStopped)
+            options.SetPluginName("minidump")
+            options.SetOutputFile(lldb.SBFileSpec(output_file))
+            options.SetStyle(lldb.eSaveCoreCustomOnly)
+            region_one = lldb.SBMemoryRegionInfo()
+            process.GetMemoryRegions().GetMemoryRegionAtIndex(0, region_one)
+            options.AddMemoryRegionToSave(region_one)
+            empty_region = lldb.SBMemoryRegionInfo(
+                "empty region", 0x0, 0x0, 3, True, False
+            )
+            options.AddMemoryRegionToSave(empty_region)
+            region_with_no_permissions = lldb.SBMemoryRegionInfo(
+                "no permissions", 0x2AAA, 0x2BBB, 0, True, False
+            )
+            options.AddMemoryRegionToSave(region_with_no_permissions)
+            error = process.SaveCore(options)
+            self.assertTrue(error.Success(), error.GetCString())
+            core_target = self.dbg.CreateTarget(None)
+            core_process = core_target.LoadCore(output_file)
+            self.assertNotIn(
+                region_with_no_permissions, core_process.GetMemoryRegions()
+            )
+            self.assertNotIn(empty_region, core_process.GetMemoryRegions())
+        finally:
+            if os.path.isfile(output_file):
+                os.unlink(output_file)
diff --git a/lldb/test/API/functionalities/scripted_process/TestStackCoreScriptedProcess.py b/lldb/test/API/functionalities/scripted_process/TestStackCoreScriptedProcess.py
index bf9681ad678b6..a5c79378bab50 100644
--- a/lldb/test/API/functionalities/scripted_process/TestStackCoreScriptedProcess.py
+++ b/lldb/test/API/functionalities/scripted_process/TestStackCoreScriptedProcess.py
@@ -32,7 +32,6 @@ def get_module_with_name(self, target, name):
         return None
 
     @skipUnlessDarwin
-    @skipIfOutOfTreeDebugserver
     @skipIfRemote
     @skipIfAsan  # On ASAN builds, this test times-out (rdar://98678134)
     @skipIfDarwin
diff --git a/lldb/test/API/functionalities/thread/concurrent_events/TestConcurrentManyBreakpoints.py b/lldb/test/API/functionalities/thread/concurrent_events/TestConcurrentManyBreakpoints.py
index 79e704ac56962..bc41f5f3045a7 100644
--- a/lldb/test/API/functionalities/thread/concurrent_events/TestConcurrentManyBreakpoints.py
+++ b/lldb/test/API/functionalities/thread/concurrent_events/TestConcurrentManyBreakpoints.py
@@ -7,7 +7,6 @@
 class ConcurrentManyBreakpoints(ConcurrentEventsBase):
     # Atomic sequences are not supported yet for MIPS in LLDB.
     @skipIf(triple="^mips")
-    @skipIfOutOfTreeDebugserver
     @expectedFailureAll(
         archs=["aarch64"], oslist=["freebsd"], bugnumber="llvm.org/pr49433"
     )
diff --git a/lldb/test/API/functionalities/thread/concurrent_events/TestConcurrentManyCrash.py b/lldb/test/API/functionalities/thread/concurrent_events/TestConcurrentManyCrash.py
index af841b1dc41e6..c5d5e40986720 100644
--- a/lldb/test/API/functionalities/thread/concurrent_events/TestConcurrentManyCrash.py
+++ b/lldb/test/API/functionalities/thread/concurrent_events/TestConcurrentManyCrash.py
@@ -7,7 +7,6 @@
 class ConcurrentManyCrash(ConcurrentEventsBase):
     # Atomic sequences are not supported yet for MIPS in LLDB.
     @skipIf(triple="^mips")
-    @skipIfOutOfTreeDebugserver
     def test(self):
         """Test 100 threads that cause a segfault."""
         self.build()
diff --git a/lldb/test/API/functionalities/thread/concurrent_events/TestConcurrentManySignals.py b/lldb/test/API/functionalities/thread/concurrent_events/TestConcurrentManySignals.py
index 4d4e6516bf88a..d241aa76f61ea 100644
--- a/lldb/test/API/functionalities/thread/concurrent_events/TestConcurrentManySignals.py
+++ b/lldb/test/API/functionalities/thread/concurrent_events/TestConcurrentManySignals.py
@@ -10,7 +10,6 @@ class ConcurrentManySignals(ConcurrentEventsBase):
     # This test is flaky on Darwin.
     @skipIfDarwin
     @expectedFailureNetBSD
-    @skipIfOutOfTreeDebugserver
     def test(self):
         """Test 100 signals from 100 threads."""
         self.build()
diff --git a/lldb/test/API/functionalities/thread/concurrent_events/TestConcurrentManyWatchpoints.py b/lldb/test/API/functionalities/thread/concurrent_events/TestConcurrentManyWatchpoints.py
index 072ec06a5af94..30b7da4cddb1e 100644
--- a/lldb/test/API/functionalities/thread/concurrent_events/TestConcurrentManyWatchpoints.py
+++ b/lldb/test/API/functionalities/thread/concurrent_events/TestConcurrentManyWatchpoints.py
@@ -8,7 +8,6 @@ class ConcurrentManyWatchpoints(ConcurrentEventsBase):
     # Atomic sequences are not supported yet for MIPS in LLDB.
     @skipIf(triple="^mips")
     @add_test_categories(["watchpoint"])
-    @skipIfOutOfTreeDebugserver
     def test(self):
         """Test 100 watchpoints from 100 threads."""
         self.build()
diff --git a/lldb/test/API/functionalities/watchpoint/large-watchpoint/TestLargeWatchpoint.py b/lldb/test/API/functionalities/watchpoint/large-watchpoint/TestLargeWatchpoint.py
index c5e161497e628..8db242a17feb5 100644
--- a/lldb/test/API/functionalities/watchpoint/large-watchpoint/TestLargeWatchpoint.py
+++ b/lldb/test/API/functionalities/watchpoint/large-watchpoint/TestLargeWatchpoint.py
@@ -27,7 +27,6 @@ def continue_and_report_stop_reason(self, process, iter_str):
 
     # debugserver only gained the ability to watch larger regions
     # with this patch.
-    @skipIfOutOfTreeDebugserver
     def test_large_watchpoint(self):
         """Test watchpoint that covers a large region of memory."""
         self.build()
diff --git a/lldb/test/API/functionalities/watchpoint/unaligned-spanning-two-dwords/TestUnalignedSpanningDwords.py b/lldb/test/API/functionalities/watchpoint/unaligned-spanning-two-dwords/TestUnalignedSpanningDwords.py
index 9a0f451c64c30..bf0cb24e21888 100644
--- a/lldb/test/API/functionalities/watchpoint/unaligned-spanning-two-dwords/TestUnalignedSpanningDwords.py
+++ b/lldb/test/API/functionalities/watchpoint/unaligned-spanning-two-dwords/TestUnalignedSpanningDwords.py
@@ -32,7 +32,6 @@ def hit_watchpoint_and_continue(self, process, iter_str):
     # older debugservers will return the base address of the doubleword
     # which lldb doesn't understand, and will stop executing without a
     # proper stop reason.
-    @skipIfOutOfTreeDebugserver
     def test_unaligned_watchpoint(self):
         """Test a watchpoint that is handled by two hardware watchpoint registers."""
         self.build()
diff --git a/lldb/test/API/linux/aarch64/gcs/TestAArch64LinuxGCS.py b/lldb/test/API/linux/aarch64/gcs/TestAArch64LinuxGCS.py
index d3d4dbecf4a2a..f5a2ca356bbe8 100644
--- a/lldb/test/API/linux/aarch64/gcs/TestAArch64LinuxGCS.py
+++ b/lldb/test/API/linux/aarch64/gcs/TestAArch64LinuxGCS.py
@@ -3,7 +3,6 @@
 extension is enabled.
 """
 
-
 import lldb
 from lldbsuite.test.decorators import *
 from lldbsuite.test.lldbtest import *
@@ -84,6 +83,40 @@ def test_gcs_fault(self):
             ],
         )
 
+    # This helper reads all the GCS registers and optionally compares them
+    # against a previous state, then returns the current register values.
+    def check_gcs_registers(
+        self,
+        expected_gcs_features_enabled=None,
+        expected_gcs_features_locked=None,
+        expected_gcspr_el0=None,
+    ):
+        thread = self.dbg.GetSelectedTarget().process.GetThreadAtIndex(0)
+        registerSets = thread.GetFrameAtIndex(0).GetRegisters()
+        gcs_registers = registerSets.GetFirstValueByName(
+            r"Guarded Control Stack Registers"
+        )
+
+        gcs_features_enabled = gcs_registers.GetChildMemberWithName(
+            "gcs_features_enabled"
+        ).GetValueAsUnsigned()
+        if expected_gcs_features_enabled is not None:
+            self.assertEqual(expected_gcs_features_enabled, gcs_features_enabled)
+
+        gcs_features_locked = gcs_registers.GetChildMemberWithName(
+            "gcs_features_locked"
+        ).GetValueAsUnsigned()
+        if expected_gcs_features_locked is not None:
+            self.assertEqual(expected_gcs_features_locked, gcs_features_locked)
+
+        gcspr_el0 = gcs_registers.GetChildMemberWithName(
+            "gcspr_el0"
+        ).GetValueAsUnsigned()
+        if expected_gcspr_el0 is not None:
+            self.assertEqual(expected_gcspr_el0, gcspr_el0)
+
+        return gcs_features_enabled, gcs_features_locked, gcspr_el0
+
     @skipUnlessArch("aarch64")
     @skipUnlessPlatform(["linux"])
     def test_gcs_registers(self):
@@ -108,40 +141,7 @@ def test_gcs_registers(self):
 
         self.expect("register read --all", substrs=["Guarded Control Stack Registers:"])
 
-        # This helper reads all the GCS registers and optionally compares them
-        # against a previous state, then returns the current register values.
-        def check_gcs_registers(
-            expected_gcs_features_enabled=None,
-            expected_gcs_features_locked=None,
-            expected_gcspr_el0=None,
-        ):
-            thread = self.dbg.GetSelectedTarget().process.GetThreadAtIndex(0)
-            registerSets = thread.GetFrameAtIndex(0).GetRegisters()
-            gcs_registers = registerSets.GetFirstValueByName(
-                r"Guarded Control Stack Registers"
-            )
-
-            gcs_features_enabled = gcs_registers.GetChildMemberWithName(
-                "gcs_features_enabled"
-            ).GetValueAsUnsigned()
-            if expected_gcs_features_enabled is not None:
-                self.assertEqual(expected_gcs_features_enabled, gcs_features_enabled)
-
-            gcs_features_locked = gcs_registers.GetChildMemberWithName(
-                "gcs_features_locked"
-            ).GetValueAsUnsigned()
-            if expected_gcs_features_locked is not None:
-                self.assertEqual(expected_gcs_features_locked, gcs_features_locked)
-
-            gcspr_el0 = gcs_registers.GetChildMemberWithName(
-                "gcspr_el0"
-            ).GetValueAsUnsigned()
-            if expected_gcspr_el0 is not None:
-                self.assertEqual(expected_gcspr_el0, gcspr_el0)
-
-            return gcs_features_enabled, gcs_features_locked, gcspr_el0
-
-        enabled, locked, spr_el0 = check_gcs_registers()
+        enabled, locked, spr_el0 = self.check_gcs_registers()
 
         # Features enabled should have at least the enable bit set, it could have
         # others depending on what the C library did, but we can't rely on always
@@ -164,7 +164,7 @@ def check_gcs_registers(
             substrs=["stopped", "stop reason = breakpoint"],
         )
 
-        _, _, spr_el0 = check_gcs_registers(enabled, locked, spr_el0 - 8)
+        _, _, spr_el0 = self.check_gcs_registers(enabled, locked, spr_el0 - 8)
 
         # Any combination of GCS feature lock bits might have been set by the C
         # library, and could be set to 0 or 1. To check that we can modify them,
@@ -223,7 +223,10 @@ def check_gcs_registers(
         self.runCmd(f"register write gcs_features_enabled {enabled}")
         self.expect(
             "register read gcs_features_enabled",
-            substrs=[f"gcs_features_enabled = 0x{enabled:016x}"],
+            substrs=[
+                f"gcs_features_enabled = 0x{enabled:016x}",
+                f"= (PUSH = {(enabled >> 2) & 1}, WRITE = {(enabled >> 1) & 1}, ENABLE = {enabled & 1})",
+            ],
         )
 
         # With GCS disabled, the invalid guarded control stack pointer is not
@@ -235,3 +238,184 @@ def check_gcs_registers(
                 "exited with status = 0",
             ],
         )
+
+    @skipUnlessPlatform(["linux"])
+    def test_gcs_expression_simple(self):
+        if not self.isAArch64GCS():
+            self.skipTest("Target must support GCS.")
+
+        self.build()
+        self.runCmd("file " + self.getBuildArtifact("a.out"), CURRENT_EXECUTABLE_SET)
+
+        # Break before GCS has been enabled.
+        self.runCmd("b main")
+        # And after it has been enabled.
+        lldbutil.run_break_set_by_file_and_line(
+            self,
+            "main.c",
+            line_number("main.c", "// Set break point at this line."),
+            num_expected_locations=1,
+        )
+
+        self.runCmd("run", RUN_SUCCEEDED)
+
+        if self.process().GetState() == lldb.eStateExited:
+            self.fail("Test program failed to run.")
+
+        self.expect(
+            "thread list",
+            STOPPED_DUE_TO_BREAKPOINT,
+            substrs=["stopped", "stop reason = breakpoint"],
+        )
+
+        # GCS has not been enabled yet and the ABI plugin should know not to
+        # attempt pushing to the control stack.
+        before = self.check_gcs_registers()
+        expr_cmd = "p get_gcs_status()"
+        self.expect(expr_cmd, substrs=["(unsigned long) 0"])
+        self.check_gcs_registers(*before)
+
+        # Continue to when GCS has been enabled.
+        self.runCmd("continue")
+        self.expect(
+            "thread list",
+            STOPPED_DUE_TO_BREAKPOINT,
+            substrs=["stopped", "stop reason = breakpoint"],
+        )
+
+        # If we fail to setup the GCS entry, we should not leave any of the GCS registers
+        # changed. The last thing we do is write a new GCS entry to memory and
+        # to simulate the failure of that, temporarily point the GCS to the zero page.
+        #
+        # We use the value 8 here because LLDB will decrement it by 8 so it points to
+        # what we think will be an empty entry on the guarded control stack.
+        _, _, original_gcspr = self.check_gcs_registers()
+        self.runCmd("register write gcspr_el0 8")
+        before = self.check_gcs_registers()
+        self.expect(expr_cmd, error=True)
+        self.check_gcs_registers(*before)
+        # Point to the valid shadow stack region again.
+        self.runCmd(f"register write gcspr_el0 {original_gcspr}")
+
+        # This time we do need to push to the GCS and having done so, we can
+        # return from this expression without causing a fault.
+        before = self.check_gcs_registers()
+        self.expect(expr_cmd, substrs=["(unsigned long) 1"])
+        self.check_gcs_registers(*before)
+
+    @skipUnlessPlatform(["linux"])
+    def test_gcs_expression_disable_gcs(self):
+        if not self.isAArch64GCS():
+            self.skipTest("Target must support GCS.")
+
+        self.build()
+        self.runCmd("file " + self.getBuildArtifact("a.out"), CURRENT_EXECUTABLE_SET)
+
+        # Break after GCS is enabled.
+        lldbutil.run_break_set_by_file_and_line(
+            self,
+            "main.c",
+            line_number("main.c", "// Set break point at this line."),
+            num_expected_locations=1,
+        )
+
+        self.runCmd("run", RUN_SUCCEEDED)
+
+        if self.process().GetState() == lldb.eStateExited:
+            self.fail("Test program failed to run.")
+
+        self.expect(
+            "thread list",
+            STOPPED_DUE_TO_BREAKPOINT,
+            substrs=["stopped", "stop reason = breakpoint"],
+        )
+
+        # Unlock all features so the expression can enable them again.
+        self.runCmd("register write gcs_features_locked 0")
+        # Disable all features, but keep GCS itself enabled.
+        PR_SHADOW_STACK_ENABLE = 1
+        self.runCmd(f"register write gcs_features_enabled 0x{PR_SHADOW_STACK_ENABLE:x}")
+
+        enabled, locked, spr_el0 = self.check_gcs_registers()
+        # We restore everything apart GCS being enabled, as we are not allowed to
+        # go from disabled -> enabled via ptrace.
+        self.expect("p change_gcs_config(false)", substrs=["true"])
+        enabled &= ~1
+        self.check_gcs_registers(enabled, locked, spr_el0)
+
+    @skipUnlessPlatform(["linux"])
+    def test_gcs_expression_enable_gcs(self):
+        if not self.isAArch64GCS():
+            self.skipTest("Target must support GCS.")
+
+        self.build()
+        self.runCmd("file " + self.getBuildArtifact("a.out"), CURRENT_EXECUTABLE_SET)
+
+        # Break before GCS is enabled.
+        self.runCmd("b main")
+
+        self.runCmd("run", RUN_SUCCEEDED)
+
+        if self.process().GetState() == lldb.eStateExited:
+            self.fail("Test program failed to run.")
+
+        self.expect(
+            "thread list",
+            STOPPED_DUE_TO_BREAKPOINT,
+            substrs=["stopped", "stop reason = breakpoint"],
+        )
+
+        # Unlock all features so the expression can enable them again.
+        self.runCmd("register write gcs_features_locked 0")
+        # Disable all features. The program needs PR_SHADOW_STACK_PUSH, but it
+        # will enable that itself.
+        self.runCmd(f"register write gcs_features_enabled 0")
+
+        enabled, locked, spr_el0 = self.check_gcs_registers()
+        self.expect("p change_gcs_config(true)", substrs=["true"])
+        # Though we could disable GCS with ptrace, we choose not to to be
+        # consistent with the disabled -> enabled behaviour.
+        enabled |= 1
+        self.check_gcs_registers(enabled, locked, spr_el0)
+
+    @skipIfLLVMTargetMissing("AArch64")
+    def test_gcs_core_file(self):
+        # To re-generate the core file, build the test file and run it on a
+        # machine with GCS enabled. Note that because the kernel decides where
+        # the GCS is stored, the value of gcspr_el0 and which memory region it
+        # points to may change between runs.
+
+        self.runCmd("target create --core corefile")
+
+        self.expect(
+            "bt",
+            substrs=["stop reason = SIGSEGV: control protection fault"],
+        )
+
+        self.expect(
+            "register read --all",
+            substrs=[
+                "Guarded Control Stack Registers:",
+                "gcs_features_enabled = 0x0000000000000001",
+                "gcs_features_locked = 0x0000000000000000",
+                "gcspr_el0 = 0x0000ffffa83ffff0",
+            ],
+        )
+
+        # Should get register fields for both. They have the same fields.
+        self.expect(
+            "register read gcs_features_enabled",
+            substrs=["= (PUSH = 0, WRITE = 0, ENABLE = 1)"],
+        )
+        self.expect(
+            "register read gcs_features_locked",
+            substrs=["= (PUSH = 0, WRITE = 0, ENABLE = 0)"],
+        )
+
+        # Core files do not include /proc/pid/smaps, so we cannot see the
+        # shadow stack "ss" flag. gcspr_el0 should at least point to some mapped
+        # region.
+        self.expect(
+            "memory region $gcspr_el0",
+            substrs=["[0x0000ffffa8000000-0x0000ffffa8400000) rw-"],
+        )
diff --git a/lldb/test/API/linux/aarch64/gcs/corefile b/lldb/test/API/linux/aarch64/gcs/corefile
new file mode 100644
index 0000000000000..34faa98c4d783
Binary files /dev/null and b/lldb/test/API/linux/aarch64/gcs/corefile differ
diff --git a/lldb/test/API/linux/aarch64/gcs/main.c b/lldb/test/API/linux/aarch64/gcs/main.c
index 09354639af376..396aef7499ca9 100644
--- a/lldb/test/API/linux/aarch64/gcs/main.c
+++ b/lldb/test/API/linux/aarch64/gcs/main.c
@@ -1,4 +1,5 @@
 #include <asm/hwcap.h>
+#include <stdbool.h>
 #include <sys/auxv.h>
 #include <sys/prctl.h>
 
@@ -8,7 +9,12 @@
 
 #define PR_GET_SHADOW_STACK_STATUS 74
 #define PR_SET_SHADOW_STACK_STATUS 75
-#define PR_SHADOW_STACK_ENABLE (1UL)
+#define PR_LOCK_SHADOW_STACK_STATUS 76
+
+#define PR_SHADOW_STACK_ENABLE (1UL << 0)
+#define PR_SHADOW_STACK_WRITE (1UL << 1)
+#define PR_SHADOW_STACK_PUSH (1UL << 2)
+
 #define PRCTL_SYSCALL_NO 167
 
 // Once we enable GCS, we cannot return from the function that made the syscall
@@ -36,6 +42,36 @@ unsigned long get_gcs_status() {
   return mode;
 }
 
+extern void _start();
+bool change_gcs_config(bool enable) {
+  // The test unlocks and disables all features (excluding the main enable bit)
+  // before calling this expression. Enable them again.
+  unsigned long new_status =
+      enable | PR_SHADOW_STACK_PUSH | PR_SHADOW_STACK_WRITE;
+
+  if (enable) {
+    // We would not be able to return from prctl().
+    my_prctl(PR_SET_SHADOW_STACK_STATUS, new_status, 0, 0, 0);
+
+    // This is a stack, so we must push in reverse order to the pops we want to
+    // have later. So push the return of __lldb_expr (_start), then the return
+    // address of this function (__lldb_expr).
+    __asm__ __volatile__("sys	#3, C7, C7, #0, %0\n"  // gcspushm _start
+                         "sys	#3, C7, C7, #0, x30\n" // gcspushm x30
+                         :
+                         : "r"(_start));
+  } else {
+    if (prctl(PR_SET_SHADOW_STACK_STATUS, new_status, 0, 0, 0) != 0)
+      return false;
+  }
+
+  // Turn back on all locks.
+  if (prctl(PR_LOCK_SHADOW_STACK_STATUS, ~(0UL), 0, 0, 0) != 0)
+    return false;
+
+  return true;
+}
+
 void gcs_signal() {
   // If we enabled GCS manually, then we could just return from main to generate
   // a signal. However, if the C library enabled it, then we'd just exit
@@ -50,10 +86,11 @@ void gcs_signal() {
 }
 
 // These functions are used to observe gcspr_el0 changing as we enter them, and
-// the fault we cause by changing its value.
-void test_func2() { volatile int i = 99; }
+// the fault we cause by changing its value. Also used to check expression
+// eval can handle function calls.
+int test_func2() { return 99; }
 
-void test_func() { test_func2(); }
+int test_func() { return test_func2(); }
 
 int main() {
   if (!(getauxval(AT_HWCAP) & HWCAP_GCS))
@@ -71,7 +108,7 @@ int main() {
   // By now we should have one memory region where the GCS is stored.
 
   // For register read/write tests.
-  test_func();
+  volatile int i = test_func();
 
   // If this was a register test, we would have disabled GCS during the
   // test_func call. We cannot re-enable it from ptrace so skip this part in
diff --git a/lldb/test/API/macosx/corefile-exception-reason/TestCorefileExceptionReason.py b/lldb/test/API/macosx/corefile-exception-reason/TestCorefileExceptionReason.py
index 58f6c36ccfcc6..ada74a11ffd7f 100644
--- a/lldb/test/API/macosx/corefile-exception-reason/TestCorefileExceptionReason.py
+++ b/lldb/test/API/macosx/corefile-exception-reason/TestCorefileExceptionReason.py
@@ -11,7 +11,6 @@
 
 
 class TestCorefileExceptionReason(TestBase):
-    @skipIfOutOfTreeDebugserver  # newer debugserver required for these qMemoryRegionInfo types
     @no_debug_info_test
     @skipUnlessDarwin
     @skipIf(archs=no_match(["arm64", "arm64e"]))
diff --git a/lldb/test/API/macosx/debugserver-exit-code/TestDebugServerExitCode.py b/lldb/test/API/macosx/debugserver-exit-code/TestDebugServerExitCode.py
index 853d5635031f5..2d9c0f5f97773 100644
--- a/lldb/test/API/macosx/debugserver-exit-code/TestDebugServerExitCode.py
+++ b/lldb/test/API/macosx/debugserver-exit-code/TestDebugServerExitCode.py
@@ -11,7 +11,6 @@
 class TestCase(TestBase):
     @no_debug_info_test
     @skipUnlessDarwin
-    @skipIfOutOfTreeDebugserver
     def test_abort(self):
         self.build()
         target = self.dbg.CreateTarget(self.getBuildArtifact("a.out"))
diff --git a/lldb/test/API/macosx/early-process-launch/TestEarlyProcessLaunch.py b/lldb/test/API/macosx/early-process-launch/TestEarlyProcessLaunch.py
index ff704104f7892..4c1bbe7490018 100644
--- a/lldb/test/API/macosx/early-process-launch/TestEarlyProcessLaunch.py
+++ b/lldb/test/API/macosx/early-process-launch/TestEarlyProcessLaunch.py
@@ -12,7 +12,6 @@ class TestEarlyProcessLaunch(TestBase):
 
     @skipUnlessDarwin
     @skipIfAsan  # rdar://103359354
-    @skipIfOutOfTreeDebugserver  # 2022-12-13 FIXME: skipping system debugserver
     # until this feature is included in the system
     # debugserver.
     @add_test_categories(["pyapi"])
diff --git a/lldb/test/API/macosx/ignore_exceptions/TestIgnoredExceptions.py b/lldb/test/API/macosx/ignore_exceptions/TestIgnoredExceptions.py
index 429f1dbb1beba..8d2674f7bad30 100644
--- a/lldb/test/API/macosx/ignore_exceptions/TestIgnoredExceptions.py
+++ b/lldb/test/API/macosx/ignore_exceptions/TestIgnoredExceptions.py
@@ -12,7 +12,6 @@
 class TestDarwinSignalHandlers(TestBase):
     NO_DEBUG_INFO_TESTCASE = True
 
-    @skipIfOutOfTreeDebugserver
     @skipUnlessDarwin
     def test_ignored_thread(self):
         """It isn't possible to convert an EXC_BAD_ACCESS to a signal when
diff --git a/lldb/test/API/macosx/profile_vrs_detach/TestDetachVrsProfile.py b/lldb/test/API/macosx/profile_vrs_detach/TestDetachVrsProfile.py
index c2fa837ed01b8..b558b6215ceda 100644
--- a/lldb/test/API/macosx/profile_vrs_detach/TestDetachVrsProfile.py
+++ b/lldb/test/API/macosx/profile_vrs_detach/TestDetachVrsProfile.py
@@ -19,7 +19,6 @@ class TestDetachVrsProfile(TestBase):
     NO_DEBUG_INFO_TESTCASE = True
 
     @skipUnlessDarwin
-    @skipIfOutOfTreeDebugserver
     @skipIfRemote
     def test_profile_and_detach(self):
         """There can be many tests in a test case - describe this test here."""
diff --git a/lldb/test/API/macosx/simulator/TestSimulatorPlatform.py b/lldb/test/API/macosx/simulator/TestSimulatorPlatform.py
index 3f5645a486bcb..a69f7a055c79b 100644
--- a/lldb/test/API/macosx/simulator/TestSimulatorPlatform.py
+++ b/lldb/test/API/macosx/simulator/TestSimulatorPlatform.py
@@ -79,7 +79,6 @@ def run_with(self, arch, os, vers, env, expected_load_command):
     @skipUnlessDarwin
     @skipIfDarwinEmbedded
     @apple_simulator_test("iphone")
-    @skipIfOutOfTreeDebugserver
     def test_ios(self):
         """Test running an iOS simulator binary"""
         self.run_with(
@@ -94,7 +93,6 @@ def test_ios(self):
     @skipUnlessDarwin
     @skipIfDarwinEmbedded
     @apple_simulator_test("appletv")
-    @skipIfOutOfTreeDebugserver
     def test_tvos(self):
         """Test running an tvOS simulator binary"""
         self.run_with(
@@ -111,7 +109,6 @@ def test_tvos(self):
     @apple_simulator_test("watch")
     @skipIfDarwin  # rdar://problem/64552748
     @skipIf(archs=["arm64", "arm64e"])
-    @skipIfOutOfTreeDebugserver
     def test_watchos_i386(self):
         """Test running a 32-bit watchOS simulator binary"""
         self.run_with(
@@ -128,7 +125,6 @@ def test_watchos_i386(self):
     @apple_simulator_test("watch")
     @skipIfDarwin  # rdar://problem/64552748
     @skipIf(archs=["i386", "x86_64"])
-    @skipIfOutOfTreeDebugserver
     def test_watchos_armv7k(self):
         """Test running a 32-bit watchOS simulator binary"""
         self.run_with(
@@ -150,7 +146,6 @@ def test_watchos_armv7k(self):
 
     @skipUnlessDarwin
     @skipIfDarwinEmbedded
-    @skipIfOutOfTreeDebugserver
     def test_lc_version_min_macosx(self):
         """Test running a back-deploying non-simulator MacOS X binary"""
         self.run_with(
@@ -166,7 +161,6 @@ def test_lc_version_min_macosx(self):
     @skipIfDarwinEmbedded
     @apple_simulator_test("iphone")
     @skipIf(archs=["arm64", "arm64e"])
-    @skipIfOutOfTreeDebugserver
     def test_lc_version_min_iphoneos(self):
         """Test running a back-deploying iOS simulator binary
         with a legacy iOS load command"""
@@ -183,7 +177,6 @@ def test_lc_version_min_iphoneos(self):
     @skipIfDarwinEmbedded
     @apple_simulator_test("iphone")
     @skipIf(archs=["arm64", "arm64e"])
-    @skipIfOutOfTreeDebugserver
     def test_ios_backdeploy_x86(self):
         """Test running a back-deploying iOS simulator binary
         with a legacy iOS load command"""
@@ -200,7 +193,6 @@ def test_ios_backdeploy_x86(self):
     @skipIfDarwinEmbedded
     @apple_simulator_test("iphone")
     @skipIf(archs=["i386", "x86_64"])
-    @skipIfOutOfTreeDebugserver
     def test_ios_backdeploy_apple_silicon(self):
         """Test running a back-deploying iOS simulator binary"""
         self.run_with(
@@ -216,7 +208,6 @@ def test_ios_backdeploy_apple_silicon(self):
     @skipIfDarwinEmbedded
     @apple_simulator_test("appletv")
     @skipIf(archs=["arm64", "arm64e"])
-    @skipIfOutOfTreeDebugserver
     def test_lc_version_min_tvos(self):
         """Test running a back-deploying tvOS simulator binary
         with a legacy tvOS load command"""
@@ -233,7 +224,6 @@ def test_lc_version_min_tvos(self):
     @skipIfDarwinEmbedded
     @apple_simulator_test("appletv")
     @skipIf(archs=["i386", "x86_64"])
-    @skipIfOutOfTreeDebugserver
     def test_tvos_backdeploy_apple_silicon(self):
         """Test running a back-deploying tvOS simulator binary"""
         self.run_with(
@@ -250,7 +240,6 @@ def test_tvos_backdeploy_apple_silicon(self):
     @apple_simulator_test("watch")
     @skipIf(archs=["arm64", "arm64e"])
     @skipIfDarwin  # rdar://problem/64552748
-    @skipIfOutOfTreeDebugserver
     def test_lc_version_min_watchos(self):
         """Test running a back-deploying watchOS simulator binary
         with a legacy watchOS load command"""
@@ -268,7 +257,6 @@ def test_lc_version_min_watchos(self):
     @apple_simulator_test("watch")
     @skipIf(archs=["arm64", "arm64e"])
     @skipIfDarwin  # rdar://problem/64552748
-    @skipIfOutOfTreeDebugserver
     def test_watchos_backdeploy_apple_silicon(self):
         """Test running a back-deploying watchOS simulator binary"""
         self.run_with(
diff --git a/lldb/test/API/macosx/skinny-corefile/TestSkinnyCorefile.py b/lldb/test/API/macosx/skinny-corefile/TestSkinnyCorefile.py
index 138792d817dbf..02ab856aabc6b 100644
--- a/lldb/test/API/macosx/skinny-corefile/TestSkinnyCorefile.py
+++ b/lldb/test/API/macosx/skinny-corefile/TestSkinnyCorefile.py
@@ -12,7 +12,6 @@
 
 
 class TestSkinnyCorefile(TestBase):
-    @skipIfOutOfTreeDebugserver  # newer debugserver required for these qMemoryRegionInfo types
     @skipIf(
         debug_info=no_match(["dsym"]),
         bugnumber="This test is looking explicitly for a dSYM",
diff --git a/lldb/test/API/macosx/stack-corefile/TestStackCorefile.py b/lldb/test/API/macosx/stack-corefile/TestStackCorefile.py
index 60e9d44141480..b96bfe87ad48f 100644
--- a/lldb/test/API/macosx/stack-corefile/TestStackCorefile.py
+++ b/lldb/test/API/macosx/stack-corefile/TestStackCorefile.py
@@ -11,7 +11,6 @@
 
 
 class TestStackCorefile(TestBase):
-    @skipIfOutOfTreeDebugserver  # newer debugserver required for these qMemoryRegionInfo types
     @no_debug_info_test
     @skipUnlessDarwin
     @skipIfRemote
diff --git a/lldb/test/API/macosx/universal/TestUniversal.py b/lldb/test/API/macosx/universal/TestUniversal.py
index aecc8814b377e..3c043df641978 100644
--- a/lldb/test/API/macosx/universal/TestUniversal.py
+++ b/lldb/test/API/macosx/universal/TestUniversal.py
@@ -57,8 +57,6 @@ def test_sbdebugger_create_target_with_file_and_target_triple(self):
     @skipIf(compiler="clang", compiler_version=["<", "7.0"])
     def test_process_launch_for_universal(self):
         """Test process launch of a universal binary."""
-        from lldbsuite.test.lldbutil import print_registers
-
         if not haswellOrLater():
             return
 
diff --git a/lldb/test/API/macosx/unregistered-macho/TestUnregisteredMacho.py b/lldb/test/API/macosx/unregistered-macho/TestUnregisteredMacho.py
index 54c0f19164015..d887d55f11612 100644
--- a/lldb/test/API/macosx/unregistered-macho/TestUnregisteredMacho.py
+++ b/lldb/test/API/macosx/unregistered-macho/TestUnregisteredMacho.py
@@ -13,7 +13,6 @@
 class TestUnregisteredMacho(TestBase):
     # newer debugserver required for jGetLoadedDynamicLibrariesInfos
     # to support this
-    @skipIfOutOfTreeDebugserver
     @no_debug_info_test
     @skipUnlessDarwin
     def test(self):
diff --git a/lldb/test/API/python_api/interpreter_callback/Makefile b/lldb/test/API/python_api/interpreter_callback/Makefile
new file mode 100644
index 0000000000000..10495940055b6
--- /dev/null
+++ b/lldb/test/API/python_api/interpreter_callback/Makefile
@@ -0,0 +1,3 @@
+C_SOURCES := main.c
+
+include Makefile.rules
diff --git a/lldb/test/API/python_api/interpreter_callback/TestCommandInterepterPrintCallback.py b/lldb/test/API/python_api/interpreter_callback/TestCommandInterepterPrintCallback.py
new file mode 100644
index 0000000000000..c99c636bda3d6
--- /dev/null
+++ b/lldb/test/API/python_api/interpreter_callback/TestCommandInterepterPrintCallback.py
@@ -0,0 +1,68 @@
+import lldb
+from lldbsuite.test.decorators import *
+from lldbsuite.test.lldbtest import *
+from lldbsuite.test import lldbutil
+
+
+class CommandInterepterPrintCallbackTest(TestBase):
+    NO_DEBUG_INFO_TESTCASE = True
+
+    def run_command_interpreter_with_output_file(self, out_filename, input_str):
+        with open(out_filename, "w") as f:
+            self.dbg.SetOutputFileHandle(f, False)
+            self.dbg.SetInputString(input_str)
+            opts = lldb.SBCommandInterpreterRunOptions()
+            self.dbg.RunCommandInterpreter(True, False, opts, 0, False, False)
+
+    def test_command_interpreter_print_callback(self):
+        """Test the command interpreter print callback."""
+        self.build()
+        exe = self.getBuildArtifact("a.out")
+
+        target = self.dbg.CreateTarget(exe)
+        self.assertTrue(target, VALID_TARGET)
+
+        lldbutil.run_to_source_breakpoint(
+            self, "// Break here", lldb.SBFileSpec("main.c")
+        )
+
+        out_filename = self.getBuildArtifact("output")
+        ci = self.dbg.GetCommandInterpreter()
+        called = False
+
+        # The string we'll be looking for in the command output.
+        needle = "Show a list of all debugger commands"
+
+        # Test registering a callback that handles the printing. Make sure the
+        # result is passed to the callback and that we don't print the result.
+        def handling_callback(return_object):
+            nonlocal called
+            called = True
+            self.assertEqual("help help", return_object.GetCommand())
+            self.assertIn(needle, return_object.GetOutput())
+            return lldb.eCommandReturnObjectPrintCallbackHandled
+
+        ci.SetPrintCallback(handling_callback)
+        self.assertFalse(called)
+        self.run_command_interpreter_with_output_file(out_filename, "help help\n")
+        with open(out_filename, "r") as f:
+            self.assertNotIn(needle, f.read())
+
+        # Test registering a callback that defers the printing to lldb. Make
+        # sure the result is passed to the callback and that the result is
+        # printed by lldb.
+        def non_handling_callback(return_object):
+            nonlocal called
+            called = True
+            self.assertEqual("he help", return_object.GetCommand())
+            self.assertIn(needle, return_object.GetOutput())
+            return lldb.eCommandReturnObjectPrintCallbackSkipped
+
+        called = False
+        ci.SetPrintCallback(non_handling_callback)
+        self.assertFalse(called)
+        self.run_command_interpreter_with_output_file(out_filename, "he help\n")
+        self.assertTrue(called)
+
+        with open(out_filename, "r") as f:
+            self.assertIn(needle, f.read())
diff --git a/lldb/test/API/python_api/interpreter_callback/main.c b/lldb/test/API/python_api/interpreter_callback/main.c
new file mode 100644
index 0000000000000..78d5c0714714a
--- /dev/null
+++ b/lldb/test/API/python_api/interpreter_callback/main.c
@@ -0,0 +1,6 @@
+#include <stdio.h>
+
+int main() {
+  int i = 1;
+  return i; // Break here
+}
diff --git a/lldb/test/API/python_api/watchpoint/TestWatchpointRead.py b/lldb/test/API/python_api/watchpoint/TestWatchpointRead.py
new file mode 100644
index 0000000000000..f482ebe5b1ece
--- /dev/null
+++ b/lldb/test/API/python_api/watchpoint/TestWatchpointRead.py
@@ -0,0 +1,129 @@
+"""
+Use lldb Python SBTarget API to set read watchpoints
+"""
+
+import lldb
+from lldbsuite.test.decorators import *
+from lldbsuite.test.lldbtest import *
+from lldbsuite.test import lldbutil
+
+
+class SetReadOnlyWatchpointTestCase(TestBase):
+    NO_DEBUG_INFO_TESTCASE = True
+
+    def setUp(self):
+        # Call super's setUp().
+        TestBase.setUp(self)
+        # Our simple source filename.
+        self.source = "main.c"
+        # Find the line number to break inside main().
+        self.line = line_number(self.source, "// Set break point at this line.")
+        self.build()
+
+    # Intel hardware does not support read-only watchpoints
+    @expectedFailureAll(archs=["i386", "x86_64"])
+    def test_read_watchpoint_watch_address(self):
+        exe = self.getBuildArtifact("a.out")
+
+        target = self.dbg.CreateTarget(exe)
+        self.assertTrue(target, VALID_TARGET)
+
+        # Now create a breakpoint on main.c.
+        breakpoint = target.BreakpointCreateByLocation(self.source, self.line)
+        self.assertTrue(
+            breakpoint and breakpoint.GetNumLocations() == 1, VALID_BREAKPOINT
+        )
+
+        # Now launch the process, and do not stop at the entry point.
+        process = target.LaunchSimple(None, None, self.get_process_working_directory())
+
+        # We should be stopped due to the breakpoint.  Get frame #0.
+        process = target.GetProcess()
+        self.assertState(process.GetState(), lldb.eStateStopped, PROCESS_STOPPED)
+        thread = lldbutil.get_stopped_thread(process, lldb.eStopReasonBreakpoint)
+        frame0 = thread.GetFrameAtIndex(0)
+
+        value = frame0.FindValue("global", lldb.eValueTypeVariableGlobal)
+        local = frame0.FindValue("local", lldb.eValueTypeVariableLocal)
+        error = lldb.SBError()
+
+        watchpoint = target.WatchAddress(value.GetLoadAddress(), 1, True, False, error)
+        self.assertTrue(
+            value and local and watchpoint,
+            "Successfully found the values and set a watchpoint",
+        )
+        self.DebugSBValue(value)
+        self.DebugSBValue(local)
+
+        # Hide stdout if not running with '-t' option.
+        if not self.TraceOn():
+            self.HideStdout()
+
+        print(watchpoint)
+
+        # Continue.  Expect the program to stop due to the variable being
+        # read, but *not* written to.
+        process.Continue()
+
+        if self.TraceOn():
+            lldbutil.print_stacktraces(process)
+
+        self.assertTrue(
+            local.GetValueAsSigned() > 0, "The local variable has been incremented"
+        )
+
+    # Intel hardware does not support read-only watchpoints
+    @expectedFailureAll(archs=["i386", "x86_64"])
+    def test_read_watchpoint_watch_create_by_address(self):
+        exe = self.getBuildArtifact("a.out")
+
+        target = self.dbg.CreateTarget(exe)
+        self.assertTrue(target, VALID_TARGET)
+
+        # Now create a breakpoint on main.c.
+        breakpoint = target.BreakpointCreateByLocation(self.source, self.line)
+        self.assertTrue(
+            breakpoint and breakpoint.GetNumLocations() == 1, VALID_BREAKPOINT
+        )
+
+        # Now launch the process, and do not stop at the entry point.
+        process = target.LaunchSimple(None, None, self.get_process_working_directory())
+
+        # We should be stopped due to the breakpoint.  Get frame #0.
+        process = target.GetProcess()
+        self.assertState(process.GetState(), lldb.eStateStopped, PROCESS_STOPPED)
+        thread = lldbutil.get_stopped_thread(process, lldb.eStopReasonBreakpoint)
+        frame0 = thread.GetFrameAtIndex(0)
+
+        value = frame0.FindValue("global", lldb.eValueTypeVariableGlobal)
+        local = frame0.FindValue("local", lldb.eValueTypeVariableLocal)
+        error = lldb.SBError()
+
+        wp_opts = lldb.SBWatchpointOptions()
+        wp_opts.SetWatchpointTypeRead(True)
+        watchpoint = target.WatchpointCreateByAddress(
+            value.GetLoadAddress(), 1, wp_opts, error
+        )
+        self.assertTrue(
+            value and local and watchpoint,
+            "Successfully found the values and set a watchpoint",
+        )
+        self.DebugSBValue(value)
+        self.DebugSBValue(local)
+
+        # Hide stdout if not running with '-t' option.
+        if not self.TraceOn():
+            self.HideStdout()
+
+        print(watchpoint)
+
+        # Continue.  Expect the program to stop due to the variable being
+        # read, but *not* written to.
+        process.Continue()
+
+        if self.TraceOn():
+            lldbutil.print_stacktraces(process)
+
+        self.assertTrue(
+            local.GetValueAsSigned() > 0, "The local variable has been incremented"
+        )
diff --git a/lldb/test/API/python_api/watchpoint/watchlocation/TestTargetWatchAddress.py b/lldb/test/API/python_api/watchpoint/watchlocation/TestTargetWatchAddress.py
index cbab3c6382e43..7a0e42a4fc278 100644
--- a/lldb/test/API/python_api/watchpoint/watchlocation/TestTargetWatchAddress.py
+++ b/lldb/test/API/python_api/watchpoint/watchlocation/TestTargetWatchAddress.py
@@ -21,7 +21,7 @@ def setUp(self):
         # This is for verifying that watch location works.
         self.violating_func = "do_bad_thing_with_location"
 
-    def test_watch_address(self):
+    def test_watch_create_by_address(self):
         """Exercise SBTarget.WatchpointCreateByAddress() API to set a watchpoint."""
         self.build()
         exe = self.getBuildArtifact("a.out")
@@ -88,6 +88,75 @@ def test_watch_address(self):
 
         # This finishes our test.
 
+    def test_watch_address(self):
+        """Exercise SBTarget.WatchAddress() API to set a watchpoint.
+        Same as test_watch_create_by_address, but uses the simpler API.
+        """
+        self.build()
+        exe = self.getBuildArtifact("a.out")
+
+        # Create a target by the debugger.
+        target = self.dbg.CreateTarget(exe)
+        self.assertTrue(target, VALID_TARGET)
+
+        # Now create a breakpoint on main.c.
+        breakpoint = target.BreakpointCreateByLocation(self.source, self.line)
+        self.assertTrue(
+            breakpoint and breakpoint.GetNumLocations() == 1, VALID_BREAKPOINT
+        )
+
+        # Now launch the process, and do not stop at the entry point.
+        process = target.LaunchSimple(None, None, self.get_process_working_directory())
+
+        # We should be stopped due to the breakpoint.  Get frame #0.
+        process = target.GetProcess()
+        self.assertState(process.GetState(), lldb.eStateStopped, PROCESS_STOPPED)
+        thread = lldbutil.get_stopped_thread(process, lldb.eStopReasonBreakpoint)
+        frame0 = thread.GetFrameAtIndex(0)
+
+        value = frame0.FindValue("g_char_ptr", lldb.eValueTypeVariableGlobal)
+        pointee = value.CreateValueFromAddress(
+            "pointee", value.GetValueAsUnsigned(0), value.GetType().GetPointeeType()
+        )
+        # Watch for write to *g_char_ptr.
+        error = lldb.SBError()
+        watch_read = False
+        watch_write = True
+        watchpoint = target.WatchAddress(
+            value.GetValueAsUnsigned(), 1, watch_read, watch_write, error
+        )
+        self.assertTrue(
+            value and watchpoint, "Successfully found the pointer and set a watchpoint"
+        )
+        self.DebugSBValue(value)
+        self.DebugSBValue(pointee)
+
+        # Hide stdout if not running with '-t' option.
+        if not self.TraceOn():
+            self.HideStdout()
+
+        print(watchpoint)
+
+        # Continue.  Expect the program to stop due to the variable being
+        # written to.
+        process.Continue()
+
+        if self.TraceOn():
+            lldbutil.print_stacktraces(process)
+
+        thread = lldbutil.get_stopped_thread(process, lldb.eStopReasonWatchpoint)
+        self.assertTrue(thread, "The thread stopped due to watchpoint")
+        self.DebugSBValue(value)
+        self.DebugSBValue(pointee)
+
+        self.expect(
+            lldbutil.print_stacktrace(thread, string_buffer=True),
+            exe=False,
+            substrs=[self.violating_func],
+        )
+
+        # This finishes our test.
+
     # No size constraint on MIPS for watches
     @skipIf(archs=["mips", "mipsel", "mips64", "mips64el"])
     @skipIf(archs=["s390x"])  # Likewise on SystemZ
diff --git a/lldb/test/API/tools/lldb-dap/breakpoint/Makefile b/lldb/test/API/tools/lldb-dap/breakpoint/Makefile
index 7634f513e8523..06438b3e6e313 100644
--- a/lldb/test/API/tools/lldb-dap/breakpoint/Makefile
+++ b/lldb/test/API/tools/lldb-dap/breakpoint/Makefile
@@ -16,4 +16,4 @@ main-copy.cpp: main.cpp
 # The following shared library will be used to test breakpoints under dynamic loading
 libother:  other-copy.c
 	"$(MAKE)" -f $(MAKEFILE_RULES) \
-		DYLIB_ONLY=YES DYLIB_C_SOURCES=other-copy.c DYLIB_NAME=other 
+		DYLIB_ONLY=YES DYLIB_C_SOURCES=other-copy.c DYLIB_NAME=other
diff --git a/lldb/test/API/tools/lldb-dap/breakpoint/TestDAP_breakpointLocations.py b/lldb/test/API/tools/lldb-dap/breakpoint/TestDAP_breakpointLocations.py
new file mode 100644
index 0000000000000..1058157e2c668
--- /dev/null
+++ b/lldb/test/API/tools/lldb-dap/breakpoint/TestDAP_breakpointLocations.py
@@ -0,0 +1,88 @@
+"""
+Test lldb-dap breakpointLocations request
+"""
+
+
+import dap_server
+import shutil
+from lldbsuite.test.decorators import *
+from lldbsuite.test.lldbtest import *
+from lldbsuite.test import lldbutil
+import lldbdap_testcase
+import os
+
+
+class TestDAP_breakpointLocations(lldbdap_testcase.DAPTestCaseBase):
+    def setUp(self):
+        lldbdap_testcase.DAPTestCaseBase.setUp(self)
+
+        self.main_basename = "main-copy.cpp"
+        self.main_path = os.path.realpath(self.getBuildArtifact(self.main_basename))
+
+    @skipIfWindows
+    def test_column_breakpoints(self):
+        """Test retrieving the available breakpoint locations."""
+        program = self.getBuildArtifact("a.out")
+        self.build_and_launch(program, stopOnEntry=True)
+        loop_line = line_number(self.main_path, "// break loop")
+        self.dap_server.request_continue()
+
+        # Ask for the breakpoint locations based only on the line number
+        response = self.dap_server.request_breakpointLocations(
+            self.main_path, loop_line
+        )
+        self.assertTrue(response["success"])
+        self.assertEqual(
+            response["body"]["breakpoints"],
+            [
+                {"line": loop_line, "column": 9},
+                {"line": loop_line, "column": 13},
+                {"line": loop_line, "column": 20},
+                {"line": loop_line, "column": 23},
+                {"line": loop_line, "column": 25},
+                {"line": loop_line, "column": 34},
+                {"line": loop_line, "column": 37},
+                {"line": loop_line, "column": 39},
+                {"line": loop_line, "column": 51},
+            ],
+        )
+
+        # Ask for the breakpoint locations for a column range
+        response = self.dap_server.request_breakpointLocations(
+            self.main_path,
+            loop_line,
+            column=24,
+            end_column=46,
+        )
+        self.assertTrue(response["success"])
+        self.assertEqual(
+            response["body"]["breakpoints"],
+            [
+                {"line": loop_line, "column": 25},
+                {"line": loop_line, "column": 34},
+                {"line": loop_line, "column": 37},
+                {"line": loop_line, "column": 39},
+            ],
+        )
+
+        # Ask for the breakpoint locations for a range of line numbers
+        response = self.dap_server.request_breakpointLocations(
+            self.main_path,
+            line=loop_line,
+            end_line=loop_line + 2,
+            column=39,
+        )
+        self.maxDiff = None
+        self.assertTrue(response["success"])
+        # On some systems, there is an additional breakpoint available
+        # at line 41, column 3, i.e. at the end of the loop. To make this
+        # test more portable, only check that all expected breakpoints are
+        # presented, but also accept additional breakpoints.
+        expected_breakpoints = [
+            {"column": 39, "line": 40},
+            {"column": 51, "line": 40},
+            {"column": 3, "line": 42},
+            {"column": 18, "line": 42},
+        ]
+        for bp in expected_breakpoints:
+            self.assertIn(bp, response["body"]["breakpoints"])
diff --git a/lldb/test/API/tools/lldb-dap/breakpoint/TestDAP_setBreakpoints.py b/lldb/test/API/tools/lldb-dap/breakpoint/TestDAP_setBreakpoints.py
index 123fea79c5cda..c62feda64a125 100644
--- a/lldb/test/API/tools/lldb-dap/breakpoint/TestDAP_setBreakpoints.py
+++ b/lldb/test/API/tools/lldb-dap/breakpoint/TestDAP_setBreakpoints.py
@@ -125,20 +125,18 @@ def test_set_and_clear(self):
         # Set 3 breakpoints and verify that they got set correctly
         response = self.dap_server.request_setBreakpoints(self.main_path, lines)
         line_to_id = {}
-        if response:
-            breakpoints = response["body"]["breakpoints"]
-            self.assertEqual(
-                len(breakpoints),
-                len(lines),
-                "expect %u source breakpoints" % (len(lines)),
-            )
-            for breakpoint, index in zip(breakpoints, range(len(lines))):
-                line = breakpoint["line"]
-                self.assertTrue(line, lines[index])
-                # Store the "id" of the breakpoint that was set for later
-                line_to_id[line] = breakpoint["id"]
-                self.assertIn(line, lines, "line expected in lines array")
-                self.assertTrue(breakpoint["verified"], "expect breakpoint verified")
+        breakpoints = response["body"]["breakpoints"]
+        self.assertEqual(
+            len(breakpoints),
+            len(lines),
+            "expect %u source breakpoints" % (len(lines)),
+        )
+        for index, breakpoint in enumerate(breakpoints):
+            line = breakpoint["line"]
+            self.assertEqual(line, lines[index])
+            # Store the "id" of the breakpoint that was set for later
+            line_to_id[line] = breakpoint["id"]
+            self.assertTrue(breakpoint["verified"], "expect breakpoint verified")
 
         # There is no breakpoint delete packet, clients just send another
         # setBreakpoints packet with the same source file with fewer lines.
@@ -151,75 +149,66 @@ def test_set_and_clear(self):
         # Set 2 breakpoints and verify that the previous breakpoints that were
         # set above are still set.
         response = self.dap_server.request_setBreakpoints(self.main_path, lines)
-        if response:
-            breakpoints = response["body"]["breakpoints"]
+        breakpoints = response["body"]["breakpoints"]
+        self.assertEqual(
+            len(breakpoints),
+            len(lines),
+            "expect %u source breakpoints" % (len(lines)),
+        )
+        for index, breakpoint in enumerate(breakpoints):
+            line = breakpoint["line"]
+            self.assertEqual(line, lines[index])
+            # Verify the same breakpoints are still set within LLDB by
+            # making sure the breakpoint ID didn't change
             self.assertEqual(
-                len(breakpoints),
-                len(lines),
-                "expect %u source breakpoints" % (len(lines)),
+                line_to_id[line],
+                breakpoint["id"],
+                "verify previous breakpoints stayed the same",
             )
-            for breakpoint, index in zip(breakpoints, range(len(lines))):
-                line = breakpoint["line"]
-                self.assertTrue(line, lines[index])
-                # Verify the same breakpoints are still set within LLDB by
-                # making sure the breakpoint ID didn't change
-                self.assertEqual(
-                    line_to_id[line],
-                    breakpoint["id"],
-                    "verify previous breakpoints stayed the same",
-                )
-                self.assertIn(line, lines, "line expected in lines array")
-                self.assertTrue(
-                    breakpoint["verified"], "expect breakpoint still verified"
-                )
+            self.assertTrue(breakpoint["verified"], "expect breakpoint still verified")
 
         # Now get the full list of breakpoints set in the target and verify
         # we have only 2 breakpoints set. The response above could have told
         # us about 2 breakpoints, but we want to make sure we don't have the
         # third one still set in the target
         response = self.dap_server.request_testGetTargetBreakpoints()
-        if response:
-            breakpoints = response["body"]["breakpoints"]
+        breakpoints = response["body"]["breakpoints"]
+        self.assertEqual(
+            len(breakpoints),
+            len(lines),
+            "expect %u source breakpoints" % (len(lines)),
+        )
+        for breakpoint in breakpoints:
+            line = breakpoint["line"]
+            # Verify the same breakpoints are still set within LLDB by
+            # making sure the breakpoint ID didn't change
             self.assertEqual(
-                len(breakpoints),
-                len(lines),
-                "expect %u source breakpoints" % (len(lines)),
+                line_to_id[line],
+                breakpoint["id"],
+                "verify previous breakpoints stayed the same",
             )
-            for breakpoint in breakpoints:
-                line = breakpoint["line"]
-                # Verify the same breakpoints are still set within LLDB by
-                # making sure the breakpoint ID didn't change
-                self.assertEqual(
-                    line_to_id[line],
-                    breakpoint["id"],
-                    "verify previous breakpoints stayed the same",
-                )
-                self.assertIn(line, lines, "line expected in lines array")
-                self.assertTrue(
-                    breakpoint["verified"], "expect breakpoint still verified"
-                )
+            self.assertIn(line, lines, "line expected in lines array")
+            self.assertTrue(breakpoint["verified"], "expect breakpoint still verified")
 
         # Now clear all breakpoints for the source file by passing down an
         # empty lines array
         lines = []
         response = self.dap_server.request_setBreakpoints(self.main_path, lines)
-        if response:
-            breakpoints = response["body"]["breakpoints"]
-            self.assertEqual(
-                len(breakpoints),
-                len(lines),
-                "expect %u source breakpoints" % (len(lines)),
-            )
+        breakpoints = response["body"]["breakpoints"]
+        self.assertEqual(
+            len(breakpoints),
+            len(lines),
+            "expect %u source breakpoints" % (len(lines)),
+        )
 
         # Verify with the target that all breakpoints have been cleared
         response = self.dap_server.request_testGetTargetBreakpoints()
-        if response:
-            breakpoints = response["body"]["breakpoints"]
-            self.assertEqual(
-                len(breakpoints),
-                len(lines),
-                "expect %u source breakpoints" % (len(lines)),
-            )
+        breakpoints = response["body"]["breakpoints"]
+        self.assertEqual(
+            len(breakpoints),
+            len(lines),
+            "expect %u source breakpoints" % (len(lines)),
+        )
 
         # Now set a breakpoint again in the same source file and verify it
         # was added.
@@ -281,12 +270,11 @@ def test_clear_breakpoints_unset_breakpoints(self):
         self.assertEqual(
             len(breakpoints), len(lines), "expect %u source breakpoints" % (len(lines))
         )
-        for breakpoint, index in zip(breakpoints, range(len(lines))):
+        for index, breakpoint in enumerate(breakpoints):
             line = breakpoint["line"]
-            self.assertTrue(line, lines[index])
+            self.assertEqual(line, lines[index])
             # Store the "id" of the breakpoint that was set for later
             line_to_id[line] = breakpoint["id"]
-            self.assertIn(line, lines, "line expected in lines array")
             self.assertTrue(breakpoint["verified"], "expect breakpoint verified")
 
         # Now clear all breakpoints for the source file by not setting the
@@ -356,3 +344,49 @@ def test_functionality(self):
         self.continue_to_breakpoints(breakpoint_ids)
         i = int(self.dap_server.get_local_variable_value("i"))
         self.assertEqual(i, 7, "i != 7 showing post hitCondition hits every time")
+
+    @skipIfWindows
+    def test_column_breakpoints(self):
+        """Test setting multiple breakpoints in the same line at different columns."""
+        loop_line = line_number("main.cpp", "// break loop")
+
+        program = self.getBuildArtifact("a.out")
+        self.build_and_launch(program)
+
+        # Set two breakpoints on the loop line at different columns.
+        columns = [13, 39]
+        response = self.dap_server.request_setBreakpoints(
+            self.main_path, [loop_line, loop_line], list({"column": c} for c in columns)
+        )
+
+        # Verify the breakpoints were set correctly
+        breakpoints = response["body"]["breakpoints"]
+        breakpoint_ids = []
+        self.assertEqual(
+            len(breakpoints),
+            len(columns),
+            "expect %u source breakpoints" % (len(columns)),
+        )
+        for index, breakpoint in enumerate(breakpoints):
+            self.assertEqual(breakpoint["line"], loop_line)
+            self.assertEqual(breakpoint["column"], columns[index])
+            self.assertTrue(breakpoint["verified"], "expect breakpoint verified")
+            breakpoint_ids.append(breakpoint["id"])
+
+        # Continue to the first breakpoint,
+        self.continue_to_breakpoints([breakpoint_ids[0]])
+
+        # We should have stopped right before the call to `twelve`.
+        # Step into and check we are inside `twelve`.
+        self.stepIn()
+        func_name = self.get_stackFrames()[0]["name"]
+        self.assertEqual(func_name, "twelve(int)")
+
+        # Continue to the second breakpoint.
+        self.continue_to_breakpoints([breakpoint_ids[1]])
+
+        # We should have stopped right before the call to `fourteen`.
+        # Step into and check we are inside `fourteen`.
+        self.stepIn()
+        func_name = self.get_stackFrames()[0]["name"]
+        self.assertEqual(func_name, "a::fourteen(int)")
diff --git a/lldb/test/API/tools/lldb-server/register-reading/TestGdbRemoteGPacket.py b/lldb/test/API/tools/lldb-server/register-reading/TestGdbRemoteGPacket.py
index ad503e19de797..5bae98d326762 100644
--- a/lldb/test/API/tools/lldb-server/register-reading/TestGdbRemoteGPacket.py
+++ b/lldb/test/API/tools/lldb-server/register-reading/TestGdbRemoteGPacket.py
@@ -25,7 +25,6 @@ def _extract_register_value(reg_info, reg_bank, byte_order, bytes_per_entry=8):
 
 
 class TestGdbRemoteGPacket(gdbremote_testcase.GdbRemoteTestCaseBase):
-    @skipIfOutOfTreeDebugserver
     @skipUnlessDarwin  # G packet not supported
     def test_g_packet(self):
         self.build()
diff --git a/lldb/test/Shell/Commands/command-breakpoint-by-addr.test b/lldb/test/Shell/Commands/command-breakpoint-by-addr.test
new file mode 100644
index 0000000000000..0a9dfd916a9c7
--- /dev/null
+++ b/lldb/test/Shell/Commands/command-breakpoint-by-addr.test
@@ -0,0 +1,14 @@
+# RUN: %clang_host -g -O0 %S/Inputs/main.c -o %t.out
+# RUN: %lldb %t.out -b -s %s | FileCheck %s
+
+breakpoint set -a "main+26"
+breakpoint set -a "main+ 26"
+breakpoint set -a "main +26"
+breakpoint set -a "main + 26"
+breakpoint set -a "main  +    26"
+
+# CHECK: Breakpoint 1: address = 
+# CHECK: Breakpoint 2: address = 
+# CHECK: Breakpoint 3: address = 
+# CHECK: Breakpoint 4: address = 
+# CHECK: Breakpoint 5: address = 
diff --git a/lldb/test/Shell/Expr/TestEnumExtensibility.m b/lldb/test/Shell/Expr/TestEnumExtensibility.m
new file mode 100644
index 0000000000000..738b4fa2c7786
--- /dev/null
+++ b/lldb/test/Shell/Expr/TestEnumExtensibility.m
@@ -0,0 +1,33 @@
+// UNSUPPORTED: system-linux, system-windows
+
+// RUN: %clangxx_host %s -c -g -o %t
+// RUN: %lldb %t \
+// RUN:   -o "target var gClosed gOpen gNS gNSOpts" \
+// RUN:   -o "image dump ast" \
+// RUN:   2>&1 | FileCheck %s
+
+#import <Foundation/Foundation.h>
+
+enum __attribute__((enum_extensibility(closed))) Closed { C1 } gClosed;
+
+enum __attribute__((enum_extensibility(open))) Open { O1 } gOpen;
+
+typedef NS_ENUM(int, NS) { N1 } gNS;
+
+typedef NS_OPTIONS(int, NSO) { OPT1 } gNSOpts;
+
+// CHECK:      EnumDecl {{.*}} Closed
+// CHECK-NEXT: |-EnumExtensibilityAttr {{.*}} Closed
+// CHECK-NEXT: `-EnumConstantDecl {{.*}} C1 'Closed'
+
+// CHECK:      EnumDecl {{.*}} Open
+// CHECK-NEXT: |-EnumExtensibilityAttr {{.*}} Open
+// CHECK-NEXT: `-EnumConstantDecl {{.*}} O1 'Open'
+
+// CHECK:      EnumDecl {{.*}} NS
+// CHECK-NEXT: |-EnumExtensibilityAttr {{.*}} Open
+// CHECK-NEXT: `-EnumConstantDecl {{.*}} N1 'NS'
+
+// CHECK:      EnumDecl {{.*}} NSO
+// CHECK-NEXT: |-EnumExtensibilityAttr {{.*}} Open
+// CHECK-NEXT: `-EnumConstantDecl {{.*}} OPT1 'NSO'
diff --git a/lldb/test/Shell/SymbolFile/DWARF/x86/discontinuous-function.s b/lldb/test/Shell/SymbolFile/DWARF/x86/discontinuous-function.s
index 93ea9f33e762d..197ab9aa14910 100644
--- a/lldb/test/Shell/SymbolFile/DWARF/x86/discontinuous-function.s
+++ b/lldb/test/Shell/SymbolFile/DWARF/x86/discontinuous-function.s
@@ -6,17 +6,29 @@
 # The function bar has been placed "in the middle" of foo, and the function
 # entry point is deliberately not its lowest address.
 
-# RUN: llvm-mc -triple x86_64-pc-linux -filetype=obj %s -o %t
-# RUN: %lldb %t -o "image lookup -v -n foo" -o "expr -- &foo" -o exit | FileCheck %s
+# RUN: split-file %s %t
+# RUN: llvm-mc -triple x86_64-pc-linux -filetype=obj %t/input.s -o %t/input.o
+# RUN: %lldb %t/input.o -s %t/commands -o exit | FileCheck %s
 
-# CHECK-LABEL: image lookup
+#--- commands
+
+image lookup -v -n foo
+# CHECK-LABEL: image lookup -v -n foo
+# CHECK: 1 match found in {{.*}}
+# CHECK: Summary: input.o`foo
+# CHECK: Function: id = {{.*}}, name = "foo", ranges = [0x0000000000000000-0x000000000000000e)[0x0000000000000014-0x000000000000001c)
+
+image lookup -v --regex -n '^foo$'
+# CHECK-LABEL: image lookup -v --regex -n '^foo$'
 # CHECK: 1 match found in {{.*}}
-# CHECK: Summary: {{.*}}`foo
+# CHECK: Summary: input.o`foo
 # CHECK: Function: id = {{.*}}, name = "foo", ranges = [0x0000000000000000-0x000000000000000e)[0x0000000000000014-0x000000000000001c)
 
+expr -- &foo
 # CHECK-LABEL: expr -- &foo
 # CHECK: (void (*)()) $0 = 0x0000000000000007
 
+#--- input.s
         .text
 
 foo.__part.1:
diff --git a/lldb/test/Shell/SymbolFile/DWARF/x86/explicit-member-function-quals.cpp b/lldb/test/Shell/SymbolFile/DWARF/x86/explicit-member-function-quals.cpp
new file mode 100644
index 0000000000000..5d1222795dd83
--- /dev/null
+++ b/lldb/test/Shell/SymbolFile/DWARF/x86/explicit-member-function-quals.cpp
@@ -0,0 +1,22 @@
+// XFAIL: *
+
+// Tests that we correctly deduce the CV-quals and storage
+// class of explicit object member functions.
+//
+// RUN: %clangxx_host %s -target x86_64-pc-linux -g -std=c++23 -c -o %t
+// RUN: %lldb %t -b -o "type lookup Foo" 2>&1 | FileCheck %s
+//
+// CHECK:      (lldb) type lookup Foo
+// CHECK-NEXT: struct Foo {
+// CHECK-NEXT:      void Method(Foo);
+// CHECK-NEXT:      void cMethod(const Foo &) const;
+// CHECK-NEXT:      void vMethod(volatile Foo &) volatile;
+// CHECK-NEXT:      void cvMethod(const volatile Foo &) const volatile;
+// CHECK-NEXT: }
+
+struct Foo {
+  void Method(this Foo) {}
+  void cMethod(this Foo const &) {}
+  void vMethod(this Foo volatile &) {}
+  void cvMethod(this Foo const volatile &) {}
+} f;
diff --git a/lldb/test/Shell/SymbolFile/Inputs/target-symbols-add-unwind.c b/lldb/test/Shell/SymbolFile/Inputs/target-symbols-add-unwind.c
deleted file mode 100644
index 237c8ce181774..0000000000000
--- a/lldb/test/Shell/SymbolFile/Inputs/target-symbols-add-unwind.c
+++ /dev/null
@@ -1 +0,0 @@
-int main() {}
diff --git a/lldb/test/Shell/SymbolFile/target-symbols-add-unwind.test b/lldb/test/Shell/SymbolFile/target-symbols-add-unwind.test
index 5420213d405e8..249d2ef79a891 100644
--- a/lldb/test/Shell/SymbolFile/target-symbols-add-unwind.test
+++ b/lldb/test/Shell/SymbolFile/target-symbols-add-unwind.test
@@ -1,27 +1,107 @@
-# TODO: When it's possible to run "image show-unwind" without a running
-# process, we can remove the unsupported line below, and hard-code an ELF
-# triple in the test.
-# UNSUPPORTED: system-windows, system-darwin
-
-# RUN: cd %T
-# RUN: %clang_host %S/Inputs/target-symbols-add-unwind.c -g \
-# RUN:   -fno-unwind-tables -fno-asynchronous-unwind-tables \
-# RUN:   -o target-symbols-add-unwind.debug
-# RUN: llvm-objcopy --strip-debug target-symbols-add-unwind.debug \
-# RUN:   target-symbols-add-unwind.stripped
-# RUN: %lldb target-symbols-add-unwind.stripped -s %s -o quit | FileCheck %s
-
-process launch --stop-at-entry
-image show-unwind -n main
-# CHECK-LABEL: image show-unwind -n main
+# NB: The minidump core file exists only because "image show-unwind" currently
+# requires a process to exist. If that changes, it can be removed.
+
+# REQUIRES: x86, lld
+
+# RUN: split-file %s %t
+# RUN: yaml2obj %t/a.core.yaml -o %t/a.core
+# RUN: %clang -c --target=x86_64-pc-linux %t/a.s -o %t/a.o
+# RUN: ld.lld --shared %t/a.o -o %t/a.debug --build-id=0xdeadbeef \
+# RUN:   --image-base=0x10000
+# RUN: llvm-objcopy --strip-all %t/a.debug %t/a.stripped
+# RUN: cd %t
+# RUN: %lldb -c %t/a.core \
+# RUN:   -o "settings set interpreter.stop-command-source-on-error false" \
+# RUN:   -s %t/commands -o quit | FileCheck %s
+
+#--- commands
+
+image add a.stripped
+image load --file a.stripped --slide 0
+image list
+# CHECK-LABEL: image list
+# CHECK: [  0] DEADBEEF 0x0000000000010000 a.stripped
+
+## Due to missing symbol information this (incorrectly) prints the unwind
+## information for public_fn1
+image show-unwind -n public_fn1 --cached true
+# CHECK-LABEL: image show-unwind -n public_fn1
+# CHECK-NEXT: UNWIND PLANS for a.stripped`public_fn1 (start addr 0x12000)
 # CHECK-NOT: debug_frame UnwindPlan:
 
-target symbols add -s target-symbols-add-unwind.stripped target-symbols-add-unwind.debug
+target symbols add -s a.stripped a.debug
 # CHECK-LABEL: target symbols add
 # CHECK: symbol file {{.*}} has been added to {{.*}}
 
-image show-unwind -n main
-# CHECK-LABEL: image show-unwind -n main
+image show-unwind -n private_fn --cached true
+# CHECK-LABEL: image show-unwind -n private_fn
+# CHECK-NEXT: UNWIND PLANS for a.stripped`private_fn (start addr 0x12010)
 # CHECK: debug_frame UnwindPlan:
 # CHECK-NEXT: This UnwindPlan originally sourced from DWARF CFI
 # CHECK-NEXT: This UnwindPlan is sourced from the compiler: yes.
+# CHECK-NEXT: This UnwindPlan is valid at all instruction locations: no.
+# CHECK-NEXT: This UnwindPlan is for a trap handler function: no.
+# CHECK-NEXT: Address range of this UnwindPlan: [a.stripped.PT_LOAD[1]..text + 16-0x0000000000000013)
+
+
+#--- a.s
+
+        .text
+        .cfi_sections .debug_frame
+        .globl  public_fn1, public_fn2
+
+        .p2align 12
+public_fn1:
+        .cfi_startproc
+        pushq   %rbp
+        .cfi_def_cfa_offset 16
+        .cfi_offset %rbp, -16
+        popq    %rbp
+        .cfi_def_cfa %rsp, 8
+        retq
+        .cfi_endproc
+
+        .p2align 4
+private_fn:
+        .cfi_startproc
+        pushq   %rbp
+        .cfi_def_cfa_offset 16
+        .cfi_offset %rbp, -16
+        popq    %rbp
+        .cfi_def_cfa %rsp, 8
+        retq
+        .cfi_endproc
+
+        .p2align 4
+public_fn2:
+        .cfi_startproc
+        pushq   %rbp
+        .cfi_def_cfa_offset 16
+        .cfi_offset %rbp, -16
+        popq    %rbp
+        .cfi_def_cfa %rsp, 8
+        retq
+        .cfi_endproc
+
+#--- a.core.yaml
+--- !minidump
+Streams:
+  - Type:            SystemInfo
+    Processor Arch:  AMD64
+    Platform ID:     Linux
+    CPU:
+      Vendor ID:       GenuineIntel
+      Version Info:    0x00000000
+      Feature Info:    0x00000000
+  - Type:            ThreadList
+    Threads:
+      - Thread Id:       0x000074F3
+        Context:         0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000B001000000000006CAE000000006B7FC05A0000C81D415A0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000A2BF9E5A6B7F0000000000000000000000000000000000008850C14BFD7F00009850C14BFD7F00000100000000000000B04AC14BFD7F0000000000000000000060812D01000000000800000000000000B065E05A6B7F00008004400000000000E050C14BFD7F00000000000000000000000000000000000004400000000000007F03FFFF0000FFFFFFFFFFFF000000000000000000000000801F00006B7F00000400000000000000B84CC14BFD7F0000304D405A6B7F0000C84DC14BFD7F0000C0AA405A6B7F00004F033D0000000000B84DC14BFD7F0000E84DC14BFD7F0000000000000000000000000000000000000070E05A6B7F000078629E5A6B7F0000C81D415A6B7F0000804F9E5A6B7F00000000000001000000E603000001000000E093115A6B7F0000804EC14BFD7F0000584EC14BFD7F000099ADC05A6B7F00000100000000000000AAAAD77D0000000002000000000000000800000000000000B065E05A6B7F0000E6B7C05A6B7F0000010000006B7F0000884DC14BFD7F0000106F7C5A6B7F0000984EC14BFD7F0000488B7C5A6B7F0000C4A71CB90000000001000000000000000800000000000000B065E05A6B7F000048B6C05A6B7F0000702AE25A6B7F0000D84DC14BFD7F000030489E5A6B7F0000E84EC14BFD7F0000E05E9E5A6B7F00000991F0460000000001000000000000000800000000000000B065E05A6B7F000048B6C05A6B7F00000100000000000000284EC14BFD7F00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000
+        Stack:
+          Start of Memory Range: 0x00007FFD4BC15080
+          Content:         30044000000000000000000000000000
+  - Type:            MemoryList
+    Memory Ranges:
+      - Start of Memory Range: 0x00007FFD4BC15080
+        Content:         30044000000000000000000000000000
+...
diff --git a/lldb/tools/debugserver/source/RNBRemote.cpp b/lldb/tools/debugserver/source/RNBRemote.cpp
index efa015920c0d5..8a53094429aba 100644
--- a/lldb/tools/debugserver/source/RNBRemote.cpp
+++ b/lldb/tools/debugserver/source/RNBRemote.cpp
@@ -6369,6 +6369,8 @@ rnb_err_t RNBRemote::HandlePacket_qProcessInfo(const char *p) {
       rep << "ostype:bridgeos;";
 #elif defined(TARGET_OS_OSX) && TARGET_OS_OSX == 1
       rep << "ostype:macosx;";
+#elif defined(TARGET_OS_XR) && TARGET_OS_XR == 1
+      rep << "ostype:xros;";
 #else
       rep << "ostype:ios;";
 #endif
@@ -6422,6 +6424,8 @@ rnb_err_t RNBRemote::HandlePacket_qProcessInfo(const char *p) {
         rep << "ostype:watchos;";
 #elif defined(TARGET_OS_BRIDGE) && TARGET_OS_BRIDGE == 1
         rep << "ostype:bridgeos;";
+#elif defined(TARGET_OS_XR) && TARGET_OS_XR == 1
+        rep << "ostype:xros;";
 #else
         rep << "ostype:ios;";
 #endif
diff --git a/lldb/tools/lldb-dap/.vscodeignore b/lldb/tools/lldb-dap/.vscodeignore
new file mode 100644
index 0000000000000..0491ba879fc3f
--- /dev/null
+++ b/lldb/tools/lldb-dap/.vscodeignore
@@ -0,0 +1,9 @@
+// Ignore everything by default
+**/*
+
+// Only include specific files and directories
+!LICENSE.TXT
+!package.json
+!README.md
+!out/**
+!syntaxes/**
diff --git a/lldb/tools/lldb-dap/CMakeLists.txt b/lldb/tools/lldb-dap/CMakeLists.txt
index d68098bf7b326..43fc18873feb3 100644
--- a/lldb/tools/lldb-dap/CMakeLists.txt
+++ b/lldb/tools/lldb-dap/CMakeLists.txt
@@ -1,7 +1,3 @@
-if ( CMAKE_SYSTEM_NAME MATCHES "Windows" OR CMAKE_SYSTEM_NAME MATCHES "NetBSD" )
-  list(APPEND extra_libs lldbHost)
-endif ()
-
 if (HAVE_LIBPTHREAD)
   list(APPEND extra_libs pthread)
 endif ()
@@ -26,9 +22,11 @@ add_lldb_tool(lldb-dap
   lldb-dap.cpp
   Breakpoint.cpp
   BreakpointBase.cpp
+  DAP.cpp
   ExceptionBreakpoint.cpp
   FifoFiles.cpp
   FunctionBreakpoint.cpp
+  InstructionBreakpoint.cpp
   IOStream.cpp
   JSONUtils.cpp
   LLDBUtils.cpp
@@ -36,12 +34,11 @@ add_lldb_tool(lldb-dap
   ProgressEvent.cpp
   RunInTerminal.cpp
   SourceBreakpoint.cpp
-  DAP.cpp
   Watchpoint.cpp
-  InstructionBreakpoint.cpp
 
   LINK_LIBS
     liblldb
+    lldbHost
     ${extra_libs}
 
   LINK_COMPONENTS
diff --git a/lldb/tools/lldb-dap/DAP.cpp b/lldb/tools/lldb-dap/DAP.cpp
index 35250d9eef608..a67abe582abd4 100644
--- a/lldb/tools/lldb-dap/DAP.cpp
+++ b/lldb/tools/lldb-dap/DAP.cpp
@@ -6,34 +6,62 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include <chrono>
-#include <cstdarg>
-#include <fstream>
-#include <mutex>
-
 #include "DAP.h"
 #include "JSONUtils.h"
 #include "LLDBUtils.h"
+#include "OutputRedirector.h"
+#include "lldb/API/SBBreakpoint.h"
 #include "lldb/API/SBCommandInterpreter.h"
+#include "lldb/API/SBCommandReturnObject.h"
 #include "lldb/API/SBLanguageRuntime.h"
 #include "lldb/API/SBListener.h"
+#include "lldb/API/SBProcess.h"
 #include "lldb/API/SBStream.h"
+#include "lldb/Host/FileSystem.h"
+#include "lldb/Utility/Status.h"
+#include "lldb/lldb-defines.h"
+#include "lldb/lldb-enumerations.h"
+#include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/StringExtras.h"
+#include "llvm/ADT/Twine.h"
+#include "llvm/Support/Error.h"
+#include "llvm/Support/ErrorHandling.h"
 #include "llvm/Support/FormatVariadic.h"
+#include "llvm/Support/raw_ostream.h"
+#include <algorithm>
+#include <cassert>
+#include <chrono>
+#include <cstdarg>
+#include <cstdio>
+#include <fstream>
+#include <mutex>
+#include <utility>
 
 #if defined(_WIN32)
 #define NOMINMAX
 #include <fcntl.h>
 #include <io.h>
 #include <windows.h>
+#else
+#include <unistd.h>
 #endif
 
 using namespace lldb_dap;
 
+namespace {
+#ifdef _WIN32
+const char DEV_NULL[] = "nul";
+#else
+const char DEV_NULL[] = "/dev/null";
+#endif
+} // namespace
+
 namespace lldb_dap {
 
-DAP::DAP(llvm::StringRef path, ReplMode repl_mode)
-    : debug_adaptor_path(path), broadcaster("lldb-dap"),
+DAP::DAP(llvm::StringRef path, std::ofstream *log, ReplMode repl_mode,
+         StreamDescriptor input, StreamDescriptor output)
+    : debug_adaptor_path(path), log(log), input(std::move(input)),
+      output(std::move(output)), broadcaster("lldb-dap"),
       exception_breakpoints(), focus_tid(LLDB_INVALID_THREAD_ID),
       stop_at_entry(false), is_attach(false),
       enable_auto_variable_summaries(false),
@@ -43,21 +71,7 @@ DAP::DAP(llvm::StringRef path, ReplMode repl_mode)
       configuration_done_sent(false), waiting_for_run_in_terminal(false),
       progress_event_reporter(
           [&](const ProgressEvent &event) { SendJSON(event.ToJSON()); }),
-      reverse_request_seq(0), repl_mode(repl_mode) {
-  const char *log_file_path = getenv("LLDBDAP_LOG");
-#if defined(_WIN32)
-  // Windows opens stdout and stdin in text mode which converts \n to 13,10
-  // while the value is just 10 on Darwin/Linux. Setting the file mode to binary
-  // fixes this.
-  int result = _setmode(fileno(stdout), _O_BINARY);
-  assert(result);
-  result = _setmode(fileno(stdin), _O_BINARY);
-  UNUSED_IF_ASSERT_DISABLED(result);
-  assert(result);
-#endif
-  if (log_file_path)
-    log.reset(new std::ofstream(log_file_path));
-}
+      reverse_request_seq(0), repl_mode(repl_mode) {}
 
 DAP::~DAP() = default;
 
@@ -173,6 +187,45 @@ ExceptionBreakpoint *DAP::GetExceptionBreakpoint(const lldb::break_id_t bp_id) {
   return nullptr;
 }
 
+llvm::Error DAP::ConfigureIO(std::FILE *overrideOut, std::FILE *overrideErr) {
+  in = lldb::SBFile(std::fopen(DEV_NULL, "r"), /*transfer_ownership=*/true);
+
+  if (auto Error = out.RedirectTo([this](llvm::StringRef output) {
+        SendOutput(OutputType::Stdout, output);
+      }))
+    return Error;
+
+  if (overrideOut) {
+    auto fd = out.GetWriteFileDescriptor();
+    if (auto Error = fd.takeError())
+      return Error;
+
+    if (dup2(*fd, fileno(overrideOut)) == -1)
+      return llvm::errorCodeToError(llvm::errnoAsErrorCode());
+  }
+
+  if (auto Error = err.RedirectTo([this](llvm::StringRef output) {
+        SendOutput(OutputType::Stderr, output);
+      }))
+    return Error;
+
+  if (overrideErr) {
+    auto fd = err.GetWriteFileDescriptor();
+    if (auto Error = fd.takeError())
+      return Error;
+
+    if (dup2(*fd, fileno(overrideErr)) == -1)
+      return llvm::errorCodeToError(llvm::errnoAsErrorCode());
+  }
+
+  return llvm::Error::success();
+}
+
+void DAP::StopIO() {
+  out.Stop();
+  err.Stop();
+}
+
 // Send the JSON in "json_str" to the "out" stream. Correctly send the
 // "Content-Length:" field followed by the length, followed by the raw
 // JSON bytes.
@@ -208,19 +261,19 @@ std::string DAP::ReadJSON() {
   std::string json_str;
   int length;
 
-  if (!input.read_expected(log.get(), "Content-Length: "))
+  if (!input.read_expected(log, "Content-Length: "))
     return json_str;
 
-  if (!input.read_line(log.get(), length_str))
+  if (!input.read_line(log, length_str))
     return json_str;
 
   if (!llvm::to_integer(length_str, length))
     return json_str;
 
-  if (!input.read_expected(log.get(), "\r\n"))
+  if (!input.read_expected(log, "\r\n"))
     return json_str;
 
-  if (!input.read_full(log.get(), length, json_str))
+  if (!input.read_full(log, length, json_str))
     return json_str;
 
   if (log) {
diff --git a/lldb/tools/lldb-dap/DAP.h b/lldb/tools/lldb-dap/DAP.h
index ae496236f1336..b23be68ea002f 100644
--- a/lldb/tools/lldb-dap/DAP.h
+++ b/lldb/tools/lldb-dap/DAP.h
@@ -9,36 +9,38 @@
 #ifndef LLDB_TOOLS_LLDB_DAP_DAP_H
 #define LLDB_TOOLS_LLDB_DAP_DAP_H
 
-#include <cstdio>
-#include <iosfwd>
-#include <map>
-#include <optional>
-#include <thread>
-
-#include "llvm/ADT/DenseMap.h"
-#include "llvm/ADT/DenseSet.h"
-#include "llvm/ADT/StringMap.h"
-#include "llvm/ADT/StringRef.h"
-#include "llvm/Support/JSON.h"
-#include "llvm/Support/Threading.h"
-#include "llvm/Support/raw_ostream.h"
-
-#include "lldb/API/SBAttachInfo.h"
-#include "lldb/API/SBCommandInterpreter.h"
-#include "lldb/API/SBCommandReturnObject.h"
-#include "lldb/API/SBDebugger.h"
-#include "lldb/API/SBEvent.h"
-#include "lldb/API/SBFormat.h"
-#include "lldb/API/SBLaunchInfo.h"
-#include "lldb/API/SBTarget.h"
-#include "lldb/API/SBThread.h"
-
+#include "DAPForward.h"
 #include "ExceptionBreakpoint.h"
 #include "FunctionBreakpoint.h"
 #include "IOStream.h"
 #include "InstructionBreakpoint.h"
+#include "OutputRedirector.h"
 #include "ProgressEvent.h"
 #include "SourceBreakpoint.h"
+#include "lldb/API/SBBroadcaster.h"
+#include "lldb/API/SBCommandInterpreter.h"
+#include "lldb/API/SBDebugger.h"
+#include "lldb/API/SBError.h"
+#include "lldb/API/SBFile.h"
+#include "lldb/API/SBFormat.h"
+#include "lldb/API/SBFrame.h"
+#include "lldb/API/SBTarget.h"
+#include "lldb/API/SBThread.h"
+#include "lldb/API/SBValue.h"
+#include "lldb/API/SBValueList.h"
+#include "lldb/lldb-types.h"
+#include "llvm/ADT/DenseMap.h"
+#include "llvm/ADT/DenseSet.h"
+#include "llvm/ADT/StringMap.h"
+#include "llvm/ADT/StringRef.h"
+#include "llvm/Support/Error.h"
+#include "llvm/Support/JSON.h"
+#include "llvm/Support/Threading.h"
+#include <map>
+#include <mutex>
+#include <optional>
+#include <thread>
+#include <vector>
 
 #define VARREF_LOCALS (int64_t)1
 #define VARREF_GLOBALS (int64_t)2
@@ -48,7 +50,8 @@
 
 namespace lldb_dap {
 
-typedef llvm::DenseMap<uint32_t, SourceBreakpoint> SourceBreakpointMap;
+typedef llvm::DenseMap<std::pair<uint32_t, uint32_t>, SourceBreakpoint>
+    SourceBreakpointMap;
 typedef llvm::StringMap<FunctionBreakpoint> FunctionBreakpointMap;
 typedef llvm::DenseMap<lldb::addr_t, InstructionBreakpoint>
     InstructionBreakpointMap;
@@ -138,15 +141,18 @@ struct SendEventRequestHandler : public lldb::SBCommandPluginInterface {
 
 struct DAP {
   llvm::StringRef debug_adaptor_path;
+  std::ofstream *log;
   InputStream input;
   OutputStream output;
+  lldb::SBFile in;
+  OutputRedirector out;
+  OutputRedirector err;
   lldb::SBDebugger debugger;
   lldb::SBTarget target;
   Variables variables;
   lldb::SBBroadcaster broadcaster;
   std::thread event_thread;
   std::thread progress_event_thread;
-  std::unique_ptr<std::ofstream> log;
   llvm::StringMap<SourceBreakpointMap> source_breakpoints;
   FunctionBreakpointMap function_breakpoints;
   InstructionBreakpointMap instruction_breakpoints;
@@ -198,13 +204,23 @@ struct DAP {
   // will contain that expression.
   std::string last_nonempty_var_expression;
 
-  DAP(llvm::StringRef path, ReplMode repl_mode);
+  DAP(llvm::StringRef path, std::ofstream *log, ReplMode repl_mode,
+      StreamDescriptor input, StreamDescriptor output);
   ~DAP();
   DAP(const DAP &rhs) = delete;
   void operator=(const DAP &rhs) = delete;
   ExceptionBreakpoint *GetExceptionBreakpoint(const std::string &filter);
   ExceptionBreakpoint *GetExceptionBreakpoint(const lldb::break_id_t bp_id);
 
+  /// Redirect stdout and stderr fo the IDE's console output.
+  ///
+  /// Errors in this operation will be printed to the log file and the IDE's
+  /// console output as well.
+  llvm::Error ConfigureIO(std::FILE *overrideOut, std::FILE *overrideErr);
+
+  /// Stop the redirected IO threads and associated pipes.
+  void StopIO();
+
   // Serialize the JSON value into a string and send the JSON packet to
   // the "out" stream.
   void SendJSON(const llvm::json::Value &json);
diff --git a/lldb/tools/lldb-dap/IOStream.h b/lldb/tools/lldb-dap/IOStream.h
index 57d5fd458b716..c91b2f717893c 100644
--- a/lldb/tools/lldb-dap/IOStream.h
+++ b/lldb/tools/lldb-dap/IOStream.h
@@ -10,13 +10,8 @@
 #define LLDB_TOOLS_LLDB_DAP_IOSTREAM_H
 
 #if defined(_WIN32)
-// We need to #define NOMINMAX in order to skip `min()` and `max()` macro
-// definitions that conflict with other system headers.
-// We also need to #undef GetObject (which is defined to GetObjectW) because
-// the JSON code we use also has methods named `GetObject()` and we conflict
-// against these.
-#define NOMINMAX
-#include <windows.h>
+#include "lldb/Host/windows/windows.h"
+#include <winsock2.h>
 #else
 typedef int SOCKET;
 #endif
@@ -52,6 +47,9 @@ struct StreamDescriptor {
 struct InputStream {
   StreamDescriptor descriptor;
 
+  explicit InputStream(StreamDescriptor descriptor)
+      : descriptor(std::move(descriptor)) {}
+
   bool read_full(std::ofstream *log, size_t length, std::string &text);
 
   bool read_line(std::ofstream *log, std::string &line);
@@ -62,6 +60,9 @@ struct InputStream {
 struct OutputStream {
   StreamDescriptor descriptor;
 
+  explicit OutputStream(StreamDescriptor descriptor)
+      : descriptor(std::move(descriptor)) {}
+
   bool write_full(llvm::StringRef str);
 };
 } // namespace lldb_dap
diff --git a/lldb/tools/lldb-dap/OutputRedirector.cpp b/lldb/tools/lldb-dap/OutputRedirector.cpp
index 2c2f49569869b..a23572ab7ae04 100644
--- a/lldb/tools/lldb-dap/OutputRedirector.cpp
+++ b/lldb/tools/lldb-dap/OutputRedirector.cpp
@@ -6,6 +6,11 @@
 //
 //===----------------------------------------------------------------------===/
 
+#include "OutputRedirector.h"
+#include "DAP.h"
+#include "llvm/ADT/StringRef.h"
+#include "llvm/Support/Error.h"
+#include <system_error>
 #if defined(_WIN32)
 #include <fcntl.h>
 #include <io.h>
@@ -13,51 +18,78 @@
 #include <unistd.h>
 #endif
 
-#include "DAP.h"
-#include "OutputRedirector.h"
-#include "llvm/ADT/StringRef.h"
-
-using namespace llvm;
+using lldb_private::Pipe;
+using llvm::createStringError;
+using llvm::Error;
+using llvm::Expected;
+using llvm::inconvertibleErrorCode;
+using llvm::StringRef;
 
 namespace lldb_dap {
 
-Error RedirectFd(int fd, std::function<void(llvm::StringRef)> callback) {
+int OutputRedirector::kInvalidDescriptor = -1;
+
+OutputRedirector::OutputRedirector() : m_fd(kInvalidDescriptor) {}
+
+Expected<int> OutputRedirector::GetWriteFileDescriptor() {
+  if (m_fd == kInvalidDescriptor)
+    return createStringError(std::errc::bad_file_descriptor,
+                             "write handle is not open for writing");
+  return m_fd;
+}
+
+Error OutputRedirector::RedirectTo(std::function<void(StringRef)> callback) {
+  assert(m_fd == kInvalidDescriptor && "Output readirector already started.");
   int new_fd[2];
+
 #if defined(_WIN32)
-  if (_pipe(new_fd, 4096, O_TEXT) == -1) {
+  if (::_pipe(new_fd, OutputBufferSize, O_TEXT) == -1) {
 #else
-  if (pipe(new_fd) == -1) {
+  if (::pipe(new_fd) == -1) {
 #endif
     int error = errno;
     return createStringError(inconvertibleErrorCode(),
-                             "Couldn't create new pipe for fd %d. %s", fd,
-                             strerror(error));
-  }
-
-  if (dup2(new_fd[1], fd) == -1) {
-    int error = errno;
-    return createStringError(inconvertibleErrorCode(),
-                             "Couldn't override the fd %d. %s", fd,
-                             strerror(error));
+                             "Couldn't create new pipe %s", strerror(error));
   }
 
   int read_fd = new_fd[0];
-  std::thread t([read_fd, callback]() {
+  m_fd = new_fd[1];
+  m_forwarder = std::thread([this, callback, read_fd]() {
     char buffer[OutputBufferSize];
-    while (true) {
-      ssize_t bytes_count = read(read_fd, &buffer, sizeof(buffer));
+    while (!m_stopped) {
+      ssize_t bytes_count = ::read(read_fd, &buffer, sizeof(buffer));
+      // EOF detected.
       if (bytes_count == 0)
-        return;
+        break;
       if (bytes_count == -1) {
-        if (errno == EAGAIN || errno == EINTR)
+        // Skip non-fatal errors.
+        if (errno == EAGAIN || errno == EINTR || errno == EWOULDBLOCK)
           continue;
         break;
       }
+
       callback(StringRef(buffer, bytes_count));
     }
+    ::close(read_fd);
   });
-  t.detach();
+
   return Error::success();
 }
 
+void OutputRedirector::Stop() {
+  m_stopped = true;
+
+  if (m_fd != kInvalidDescriptor) {
+    int fd = m_fd;
+    m_fd = kInvalidDescriptor;
+    // Closing the pipe may not be sufficient to wake up the thread in case the
+    // write descriptor is duplicated (to stdout/err or to another process).
+    // Write a null byte to ensure the read call returns.
+    char buf[] = "\0";
+    (void)::write(fd, buf, sizeof(buf));
+    ::close(fd);
+    m_forwarder.join();
+  }
+}
+
 } // namespace lldb_dap
diff --git a/lldb/tools/lldb-dap/OutputRedirector.h b/lldb/tools/lldb-dap/OutputRedirector.h
index e26d1648b104f..d2bd39797f3d7 100644
--- a/lldb/tools/lldb-dap/OutputRedirector.h
+++ b/lldb/tools/lldb-dap/OutputRedirector.h
@@ -9,17 +9,41 @@
 #ifndef LLDB_TOOLS_LLDB_DAP_OUTPUT_REDIRECTOR_H
 #define LLDB_TOOLS_LLDB_DAP_OUTPUT_REDIRECTOR_H
 
+#include "lldb/Host/Pipe.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/Support/Error.h"
+#include <atomic>
+#include <functional>
+#include <thread>
 
 namespace lldb_dap {
 
-/// Redirects the output of a given file descriptor to a callback.
-///
-/// \return
-///     \a Error::success if the redirection was set up correctly, or an error
-///     otherwise.
-llvm::Error RedirectFd(int fd, std::function<void(llvm::StringRef)> callback);
+class OutputRedirector {
+public:
+  static int kInvalidDescriptor;
+
+  /// Creates writable file descriptor that will invoke the given callback on
+  /// each write in a background thread.
+  ///
+  /// \return
+  ///     \a Error::success if the redirection was set up correctly, or an error
+  ///     otherwise.
+  llvm::Error RedirectTo(std::function<void(llvm::StringRef)> callback);
+
+  llvm::Expected<int> GetWriteFileDescriptor();
+  void Stop();
+
+  ~OutputRedirector() { Stop(); }
+
+  OutputRedirector();
+  OutputRedirector(const OutputRedirector &) = delete;
+  OutputRedirector &operator=(const OutputRedirector &) = delete;
+
+private:
+  std::atomic<bool> m_stopped = false;
+  int m_fd;
+  std::thread m_forwarder;
+};
 
 } // namespace lldb_dap
 
diff --git a/lldb/tools/lldb-dap/lldb-dap.cpp b/lldb/tools/lldb-dap/lldb-dap.cpp
index 6b12569d90a83..e323990d8b6ed 100644
--- a/lldb/tools/lldb-dap/lldb-dap.cpp
+++ b/lldb/tools/lldb-dap/lldb-dap.cpp
@@ -10,10 +10,10 @@
 #include "FifoFiles.h"
 #include "JSONUtils.h"
 #include "LLDBUtils.h"
-#include "OutputRedirector.h"
 #include "RunInTerminal.h"
 #include "Watchpoint.h"
 #include "lldb/API/SBDeclaration.h"
+#include "lldb/API/SBEvent.h"
 #include "lldb/API/SBInstruction.h"
 #include "lldb/API/SBListener.h"
 #include "lldb/API/SBMemoryRegionInfo.h"
@@ -41,9 +41,11 @@
 #include <cassert>
 #include <climits>
 #include <cstdarg>
+#include <cstdint>
 #include <cstdio>
 #include <cstdlib>
 #include <cstring>
+#include <fcntl.h>
 #include <map>
 #include <memory>
 #include <optional>
@@ -140,15 +142,14 @@ lldb::SBValueList *GetTopLevelScope(DAP &dap, int64_t variablesReference) {
   }
 }
 
-SOCKET AcceptConnection(DAP &dap, int portno) {
+SOCKET AcceptConnection(std::ofstream *log, int portno) {
   // Accept a socket connection from any host on "portno".
   SOCKET newsockfd = -1;
   struct sockaddr_in serv_addr, cli_addr;
   SOCKET sockfd = socket(AF_INET, SOCK_STREAM, 0);
   if (sockfd < 0) {
-    if (dap.log)
-      *dap.log << "error: opening socket (" << strerror(errno) << ")"
-               << std::endl;
+    if (log)
+      *log << "error: opening socket (" << strerror(errno) << ")" << std::endl;
   } else {
     memset((char *)&serv_addr, 0, sizeof(serv_addr));
     serv_addr.sin_family = AF_INET;
@@ -156,9 +157,9 @@ SOCKET AcceptConnection(DAP &dap, int portno) {
     serv_addr.sin_addr.s_addr = htonl(INADDR_LOOPBACK);
     serv_addr.sin_port = htons(portno);
     if (bind(sockfd, (struct sockaddr *)&serv_addr, sizeof(serv_addr)) < 0) {
-      if (dap.log)
-        *dap.log << "error: binding socket (" << strerror(errno) << ")"
-                 << std::endl;
+      if (log)
+        *log << "error: binding socket (" << strerror(errno) << ")"
+             << std::endl;
     } else {
       listen(sockfd, 5);
       socklen_t clilen = sizeof(cli_addr);
@@ -166,8 +167,8 @@ SOCKET AcceptConnection(DAP &dap, int portno) {
           llvm::sys::RetryAfterSignal(static_cast<SOCKET>(-1), accept, sockfd,
                                       (struct sockaddr *)&cli_addr, &clilen);
       if (newsockfd < 0)
-        if (dap.log)
-          *dap.log << "error: accept (" << strerror(errno) << ")" << std::endl;
+        if (log)
+          *log << "error: accept (" << strerror(errno) << ")" << std::endl;
     }
 #if defined(_WIN32)
     closesocket(sockfd);
@@ -911,6 +912,196 @@ void request_attach(DAP &dap, const llvm::json::Object &request) {
   }
 }
 
+// "BreakpointLocationsRequest": {
+//   "allOf": [ { "$ref": "#/definitions/Request" }, {
+//     "type": "object",
+//     "description": "The `breakpointLocations` request returns all possible
+//     locations for source breakpoints in a given range.\nClients should only
+//     call this request if the corresponding capability
+//     `supportsBreakpointLocationsRequest` is true.",
+//     "properties": {
+//       "command": {
+//         "type": "string",
+//         "enum": [ "breakpointLocations" ]
+//       },
+//       "arguments": {
+//         "$ref": "#/definitions/BreakpointLocationsArguments"
+//       }
+//     },
+//     "required": [ "command" ]
+//   }]
+// },
+// "BreakpointLocationsArguments": {
+//   "type": "object",
+//   "description": "Arguments for `breakpointLocations` request.",
+//   "properties": {
+//     "source": {
+//       "$ref": "#/definitions/Source",
+//       "description": "The source location of the breakpoints; either
+//       `source.path` or `source.sourceReference` must be specified."
+//     },
+//     "line": {
+//       "type": "integer",
+//       "description": "Start line of range to search possible breakpoint
+//       locations in. If only the line is specified, the request returns all
+//       possible locations in that line."
+//     },
+//     "column": {
+//       "type": "integer",
+//       "description": "Start position within `line` to search possible
+//       breakpoint locations in. It is measured in UTF-16 code units and the
+//       client capability `columnsStartAt1` determines whether it is 0- or
+//       1-based. If no column is given, the first position in the start line is
+//       assumed."
+//     },
+//     "endLine": {
+//       "type": "integer",
+//       "description": "End line of range to search possible breakpoint
+//       locations in. If no end line is given, then the end line is assumed to
+//       be the start line."
+//     },
+//     "endColumn": {
+//       "type": "integer",
+//       "description": "End position within `endLine` to search possible
+//       breakpoint locations in. It is measured in UTF-16 code units and the
+//       client capability `columnsStartAt1` determines whether it is 0- or
+//       1-based. If no end column is given, the last position in the end line
+//       is assumed."
+//     }
+//   },
+//   "required": [ "source", "line" ]
+// },
+// "BreakpointLocationsResponse": {
+//   "allOf": [ { "$ref": "#/definitions/Response" }, {
+//     "type": "object",
+//     "description": "Response to `breakpointLocations` request.\nContains
+//     possible locations for source breakpoints.",
+//     "properties": {
+//       "body": {
+//         "type": "object",
+//         "properties": {
+//           "breakpoints": {
+//             "type": "array",
+//             "items": {
+//               "$ref": "#/definitions/BreakpointLocation"
+//             },
+//             "description": "Sorted set of possible breakpoint locations."
+//           }
+//         },
+//         "required": [ "breakpoints" ]
+//       }
+//     },
+//     "required": [ "body" ]
+//   }]
+// },
+// "BreakpointLocation": {
+//   "type": "object",
+//   "description": "Properties of a breakpoint location returned from the
+//   `breakpointLocations` request.",
+//   "properties": {
+//     "line": {
+//       "type": "integer",
+//       "description": "Start line of breakpoint location."
+//     },
+//     "column": {
+//       "type": "integer",
+//       "description": "The start position of a breakpoint location. Position
+//       is measured in UTF-16 code units and the client capability
+//       `columnsStartAt1` determines whether it is 0- or 1-based."
+//     },
+//     "endLine": {
+//       "type": "integer",
+//       "description": "The end line of breakpoint location if the location
+//       covers a range."
+//     },
+//     "endColumn": {
+//       "type": "integer",
+//       "description": "The end position of a breakpoint location (if the
+//       location covers a range). Position is measured in UTF-16 code units and
+//       the client capability `columnsStartAt1` determines whether it is 0- or
+//       1-based."
+//     }
+//   },
+//   "required": [ "line" ]
+// },
+void request_breakpointLocations(DAP &dap, const llvm::json::Object &request) {
+  llvm::json::Object response;
+  FillResponse(request, response);
+  auto *arguments = request.getObject("arguments");
+  auto *source = arguments->getObject("source");
+  std::string path = GetString(source, "path").str();
+  uint64_t start_line = GetUnsigned(arguments, "line", 0);
+  uint64_t start_column = GetUnsigned(arguments, "column", 0);
+  uint64_t end_line = GetUnsigned(arguments, "endLine", start_line);
+  uint64_t end_column =
+      GetUnsigned(arguments, "endColumn", std::numeric_limits<uint64_t>::max());
+
+  lldb::SBFileSpec file_spec(path.c_str(), true);
+  lldb::SBSymbolContextList compile_units =
+      dap.target.FindCompileUnits(file_spec);
+
+  // Find all relevant lines & columns
+  llvm::SmallVector<std::pair<uint32_t, uint32_t>, 8> locations;
+  for (uint32_t c_idx = 0, c_limit = compile_units.GetSize(); c_idx < c_limit;
+       ++c_idx) {
+    const lldb::SBCompileUnit &compile_unit =
+        compile_units.GetContextAtIndex(c_idx).GetCompileUnit();
+    if (!compile_unit.IsValid())
+      continue;
+    lldb::SBFileSpec primary_file_spec = compile_unit.GetFileSpec();
+
+    // Go through the line table and find all matching lines / columns
+    for (uint32_t l_idx = 0, l_limit = compile_unit.GetNumLineEntries();
+         l_idx < l_limit; ++l_idx) {
+      lldb::SBLineEntry line_entry = compile_unit.GetLineEntryAtIndex(l_idx);
+
+      // Filter by line / column
+      uint32_t line = line_entry.GetLine();
+      if (line < start_line || line > end_line)
+        continue;
+      uint32_t column = line_entry.GetColumn();
+      if (column == LLDB_INVALID_COLUMN_NUMBER)
+        continue;
+      if (line == start_line && column < start_column)
+        continue;
+      if (line == end_line && column > end_column)
+        continue;
+
+      // Make sure we are in the right file.
+      // We might have a match on line & column range and still
+      // be in the wrong file, e.g. for included files.
+      // Given that the involved pointers point into LLDB's string pool,
+      // we can directly compare the `const char*` pointers.
+      if (line_entry.GetFileSpec().GetFilename() !=
+              primary_file_spec.GetFilename() ||
+          line_entry.GetFileSpec().GetDirectory() !=
+              primary_file_spec.GetDirectory())
+        continue;
+
+      locations.emplace_back(line, column);
+    }
+  }
+
+  // The line entries are sorted by addresses, but we must return the list
+  // ordered by line / column position.
+  std::sort(locations.begin(), locations.end());
+  locations.erase(std::unique(locations.begin(), locations.end()),
+                  locations.end());
+
+  llvm::json::Array locations_json;
+  for (auto &l : locations) {
+    llvm::json::Object location;
+    location.try_emplace("line", l.first);
+    location.try_emplace("column", l.second);
+    locations_json.emplace_back(std::move(location));
+  }
+
+  llvm::json::Object body;
+  body.try_emplace("breakpoints", std::move(locations_json));
+  response.try_emplace("body", std::move(body));
+  dap.SendJSON(llvm::json::Value(std::move(response)));
+}
+
 // "ContinueRequest": {
 //   "allOf": [ { "$ref": "#/definitions/Request" }, {
 //     "type": "object",
@@ -1103,6 +1294,7 @@ void request_disconnect(DAP &dap, const llvm::json::Object &request) {
     dap.broadcaster.BroadcastEventByType(eBroadcastBitStopProgressThread);
     dap.progress_event_thread.join();
   }
+  dap.StopIO();
   dap.disconnecting = true;
 }
 
@@ -1872,7 +2064,36 @@ void request_initialize(DAP &dap, const llvm::json::Object &request) {
   // which may affect the outcome of tests.
   bool source_init_file = GetBoolean(arguments, "sourceInitFile", true);
 
-  dap.debugger = lldb::SBDebugger::Create(source_init_file);
+  // Do not source init files until in/out/err are configured.
+  dap.debugger = lldb::SBDebugger::Create(false);
+  dap.debugger.SetInputFile(dap.in);
+  auto out_fd = dap.out.GetWriteFileDescriptor();
+  if (llvm::Error err = out_fd.takeError()) {
+    response["success"] = false;
+    EmplaceSafeString(response, "message", llvm::toString(std::move(err)));
+    dap.SendJSON(llvm::json::Value(std::move(response)));
+    return;
+  }
+  dap.debugger.SetOutputFile(lldb::SBFile(*out_fd, "w", false));
+  auto err_fd = dap.err.GetWriteFileDescriptor();
+  if (llvm::Error err = err_fd.takeError()) {
+    response["success"] = false;
+    EmplaceSafeString(response, "message", llvm::toString(std::move(err)));
+    dap.SendJSON(llvm::json::Value(std::move(response)));
+    return;
+  }
+  dap.debugger.SetErrorFile(lldb::SBFile(*err_fd, "w", false));
+
+  auto interp = dap.debugger.GetCommandInterpreter();
+
+  if (source_init_file) {
+    dap.debugger.SkipLLDBInitFiles(false);
+    dap.debugger.SkipAppInitFiles(false);
+    lldb::SBCommandReturnObject init;
+    interp.SourceInitFileInGlobalDirectory(init);
+    interp.SourceInitFileInHomeDirectory(init);
+  }
+
   if (llvm::Error err = dap.RunPreInitCommands()) {
     response["success"] = false;
     EmplaceSafeString(response, "message", llvm::toString(std::move(err)));
@@ -1938,6 +2159,8 @@ void request_initialize(DAP &dap, const llvm::json::Object &request) {
   body.try_emplace("supportsCompletionsRequest", true);
   // The debug adapter supports the disassembly request.
   body.try_emplace("supportsDisassembleRequest", true);
+  // The debug adapter supports the `breakpointLocations` request.
+  body.try_emplace("supportsBreakpointLocationsRequest", true);
   // The debug adapter supports stepping granularities (argument `granularity`)
   // for the stepping requests.
   body.try_emplace("supportsSteppingGranularity", true);
@@ -2702,9 +2925,10 @@ void request_setBreakpoints(DAP &dap, const llvm::json::Object &request) {
       const auto *bp_obj = bp.getAsObject();
       if (bp_obj) {
         SourceBreakpoint src_bp(dap, *bp_obj);
-        request_bps.try_emplace(src_bp.line, src_bp);
+        std::pair<uint32_t, uint32_t> bp_pos(src_bp.line, src_bp.column);
+        request_bps.try_emplace(bp_pos, src_bp);
         const auto [iv, inserted] =
-            dap.source_breakpoints[path].try_emplace(src_bp.line, src_bp);
+            dap.source_breakpoints[path].try_emplace(bp_pos, src_bp);
         // We check if this breakpoint already exists to update it
         if (inserted)
           iv->getSecond().SetBreakpoint(path.data());
@@ -4775,6 +4999,8 @@ void request_setInstructionBreakpoints(DAP &dap,
 
 void RegisterRequestCallbacks(DAP &dap) {
   dap.RegisterRequestCallback("attach", request_attach);
+  dap.RegisterRequestCallback("breakpointLocations",
+                              request_breakpointLocations);
   dap.RegisterRequestCallback("completions", request_completions);
   dap.RegisterRequestCallback("continue", request_continue);
   dap.RegisterRequestCallback("configurationDone", request_configurationDone);
@@ -4911,36 +5137,14 @@ static void redirection_test() {
   fflush(stderr);
 }
 
-/// Redirect stdout and stderr fo the IDE's console output.
-///
-/// Errors in this operation will be printed to the log file and the IDE's
-/// console output as well.
-///
-/// \return
-///     A fd pointing to the original stdout.
-static int SetupStdoutStderrRedirection(DAP &dap) {
-  int stdoutfd = fileno(stdout);
-  int new_stdout_fd = dup(stdoutfd);
-  auto output_callback_stderr = [&dap](llvm::StringRef data) {
-    dap.SendOutput(OutputType::Stderr, data);
-  };
-  auto output_callback_stdout = [&dap](llvm::StringRef data) {
-    dap.SendOutput(OutputType::Stdout, data);
-  };
-  if (llvm::Error err = RedirectFd(stdoutfd, output_callback_stdout)) {
-    std::string error_message = llvm::toString(std::move(err));
-    if (dap.log)
-      *dap.log << error_message << std::endl;
-    output_callback_stderr(error_message);
-  }
-  if (llvm::Error err = RedirectFd(fileno(stderr), output_callback_stderr)) {
-    std::string error_message = llvm::toString(std::move(err));
-    if (dap.log)
-      *dap.log << error_message << std::endl;
-    output_callback_stderr(error_message);
-  }
-
-  return new_stdout_fd;
+/// Duplicates a file descriptor, setting FD_CLOEXEC if applicable.
+static int DuplicateFileDescriptor(int fd) {
+#if defined(F_DUPFD_CLOEXEC)
+  // Ensure FD_CLOEXEC is set.
+  return ::fcntl(fd, F_DUPFD_CLOEXEC, 0);
+#else
+  return ::dup(fd);
+#endif
 }
 
 int main(int argc, char *argv[]) {
@@ -5031,47 +5235,88 @@ int main(int argc, char *argv[]) {
   }
 #endif
 
+  std::unique_ptr<std::ofstream> log = nullptr;
+  const char *log_file_path = getenv("LLDBDAP_LOG");
+  if (log_file_path)
+    log = std::make_unique<std::ofstream>(log_file_path);
+
   // Initialize LLDB first before we do anything.
-  lldb::SBDebugger::Initialize();
+  lldb::SBError error = lldb::SBDebugger::InitializeWithErrorHandling();
+  if (error.Fail()) {
+    lldb::SBStream os;
+    error.GetDescription(os);
+    llvm::errs() << "lldb initialize failed: " << os.GetData() << "\n";
+    return EXIT_FAILURE;
+  }
 
   // Terminate the debugger before the C++ destructor chain kicks in.
   auto terminate_debugger =
       llvm::make_scope_exit([] { lldb::SBDebugger::Terminate(); });
 
-  DAP dap = DAP(program_path.str(), default_repl_mode);
-
-  RegisterRequestCallbacks(dap);
-
-  // stdout/stderr redirection to the IDE's console
-  int new_stdout_fd = SetupStdoutStderrRedirection(dap);
-
+  StreamDescriptor input;
+  StreamDescriptor output;
+  std::FILE *redirectOut = nullptr;
+  std::FILE *redirectErr = nullptr;
   if (portno != -1) {
     printf("Listening on port %i...\n", portno);
-    SOCKET socket_fd = AcceptConnection(dap, portno);
-    if (socket_fd >= 0) {
-      dap.input.descriptor = StreamDescriptor::from_socket(socket_fd, true);
-      dap.output.descriptor = StreamDescriptor::from_socket(socket_fd, false);
-    } else {
+    SOCKET socket_fd = AcceptConnection(log.get(), portno);
+    if (socket_fd < 0)
       return EXIT_FAILURE;
-    }
+
+    input = StreamDescriptor::from_socket(socket_fd, true);
+    output = StreamDescriptor::from_socket(socket_fd, false);
   } else {
-    dap.input.descriptor = StreamDescriptor::from_file(fileno(stdin), false);
-    dap.output.descriptor = StreamDescriptor::from_file(new_stdout_fd, false);
+#if defined(_WIN32)
+    // Windows opens stdout and stdin in text mode which converts \n to 13,10
+    // while the value is just 10 on Darwin/Linux. Setting the file mode to
+    // binary fixes this.
+    int result = _setmode(fileno(stdout), _O_BINARY);
+    assert(result);
+    result = _setmode(fileno(stdin), _O_BINARY);
+    UNUSED_IF_ASSERT_DISABLED(result);
+    assert(result);
+#endif
+
+    int stdout_fd = DuplicateFileDescriptor(fileno(stdout));
+    if (stdout_fd == -1) {
+      llvm::logAllUnhandledErrors(
+          llvm::errorCodeToError(llvm::errnoAsErrorCode()), llvm::errs(),
+          "Failed to configure stdout redirect: ");
+      return EXIT_FAILURE;
+    }
+
+    redirectOut = stdout;
+    redirectErr = stderr;
 
-    /// used only by TestVSCode_redirection_to_console.py
-    if (getenv("LLDB_DAP_TEST_STDOUT_STDERR_REDIRECTION") != nullptr)
-      redirection_test();
+    input = StreamDescriptor::from_file(fileno(stdin), false);
+    output = StreamDescriptor::from_file(stdout_fd, false);
   }
 
+  DAP dap = DAP(program_path.str(), log.get(), default_repl_mode,
+                std::move(input), std::move(output));
+
+  // stdout/stderr redirection to the IDE's console
+  if (auto Err = dap.ConfigureIO(redirectOut, redirectErr)) {
+    llvm::logAllUnhandledErrors(std::move(Err), llvm::errs(),
+                                "Failed to configure lldb-dap IO operations: ");
+    return EXIT_FAILURE;
+  }
+
+  RegisterRequestCallbacks(dap);
+
   for (const std::string &arg :
        input_args.getAllArgValues(OPT_pre_init_command)) {
     dap.pre_init_commands.push_back(arg);
   }
 
+  // used only by TestVSCode_redirection_to_console.py
+  if (getenv("LLDB_DAP_TEST_STDOUT_STDERR_REDIRECTION") != nullptr)
+    redirection_test();
+
   bool CleanExit = true;
   if (auto Err = dap.Loop()) {
-    if (dap.log)
-      *dap.log << "Transport Error: " << llvm::toString(std::move(Err)) << "\n";
+    if (log)
+      *log << "Transport Error: " << llvm::toString(std::move(Err)) << "\n";
     CleanExit = false;
   }
 
diff --git a/lldb/tools/lldb-dap/package-lock.json b/lldb/tools/lldb-dap/package-lock.json
index d1cb6d00ecf56..4c18474241421 100644
--- a/lldb/tools/lldb-dap/package-lock.json
+++ b/lldb/tools/lldb-dap/package-lock.json
@@ -1,136 +1,267 @@
 {
   "name": "lldb-dap",
-  "version": "0.2.8",
+  "version": "0.2.9",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "lldb-dap",
-      "version": "0.2.8",
+      "version": "0.2.9",
       "license": "Apache 2.0 License with LLVM exceptions",
       "devDependencies": {
         "@types/node": "^18.11.18",
-        "@types/vscode": "~1.74.0",
-        "@vscode/vsce": "^2.19.0",
-        "prettier": "^3.1.1",
-        "prettier-plugin-curly": "^0.1.3",
-        "typescript": "^4.6.4"
+        "@types/vscode": "1.75.0",
+        "@vscode/vsce": "^3.2.2",
+        "prettier": "^3.4.2",
+        "prettier-plugin-curly": "^0.3.1",
+        "typescript": "^5.7.3"
       },
       "engines": {
         "vscode": "^1.75.0"
       }
     },
-    "node_modules/@babel/code-frame": {
-      "version": "7.23.5",
-      "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.23.5.tgz",
-      "integrity": "sha512-CgH3s1a96LipHCmSUmYFPwY7MNx8C3avkq7i4Wl3cfa662ldtUe4VM1TPXX70pfmrlWTb6jLqTYrZyT2ZTJBgA==",
+    "node_modules/@azure/abort-controller": {
+      "version": "2.1.2",
+      "resolved": "https://registry.npmjs.org/@azure/abort-controller/-/abort-controller-2.1.2.tgz",
+      "integrity": "sha512-nBrLsEWm4J2u5LpAPjxADTlq3trDgVZZXHNKabeXZtpq3d3AbN/KGO82R87rdDz5/lYB024rtEf10/q0urNgsA==",
       "dev": true,
+      "license": "MIT",
       "dependencies": {
-        "@babel/highlight": "^7.23.4",
-        "chalk": "^2.4.2"
+        "tslib": "^2.6.2"
       },
       "engines": {
-        "node": ">=6.9.0"
+        "node": ">=18.0.0"
       }
     },
-    "node_modules/@babel/generator": {
-      "version": "7.23.6",
-      "resolved": "https://registry.npmjs.org/@babel/generator/-/generator-7.23.6.tgz",
-      "integrity": "sha512-qrSfCYxYQB5owCmGLbl8XRpX1ytXlpueOb0N0UmQwA073KZxejgQTzAmJezxvpwQD9uGtK2shHdi55QT+MbjIw==",
+    "node_modules/@azure/core-auth": {
+      "version": "1.9.0",
+      "resolved": "https://registry.npmjs.org/@azure/core-auth/-/core-auth-1.9.0.tgz",
+      "integrity": "sha512-FPwHpZywuyasDSLMqJ6fhbOK3TqUdviZNF8OqRGA4W5Ewib2lEEZ+pBsYcBa88B2NGO/SEnYPGhyBqNlE8ilSw==",
       "dev": true,
+      "license": "MIT",
       "dependencies": {
-        "@babel/types": "^7.23.6",
-        "@jridgewell/gen-mapping": "^0.3.2",
-        "@jridgewell/trace-mapping": "^0.3.17",
-        "jsesc": "^2.5.1"
+        "@azure/abort-controller": "^2.0.0",
+        "@azure/core-util": "^1.11.0",
+        "tslib": "^2.6.2"
       },
       "engines": {
-        "node": ">=6.9.0"
+        "node": ">=18.0.0"
       }
     },
-    "node_modules/@babel/helper-environment-visitor": {
-      "version": "7.22.20",
-      "resolved": "https://registry.npmjs.org/@babel/helper-environment-visitor/-/helper-environment-visitor-7.22.20.tgz",
-      "integrity": "sha512-zfedSIzFhat/gFhWfHtgWvlec0nqB9YEIVrpuwjruLlXfUSnA8cJB0miHKwqDnQ7d32aKo2xt88/xZptwxbfhA==",
+    "node_modules/@azure/core-client": {
+      "version": "1.9.2",
+      "resolved": "https://registry.npmjs.org/@azure/core-client/-/core-client-1.9.2.tgz",
+      "integrity": "sha512-kRdry/rav3fUKHl/aDLd/pDLcB+4pOFwPPTVEExuMyaI5r+JBbMWqRbCY1pn5BniDaU3lRxO9eaQ1AmSMehl/w==",
       "dev": true,
+      "license": "MIT",
+      "dependencies": {
+        "@azure/abort-controller": "^2.0.0",
+        "@azure/core-auth": "^1.4.0",
+        "@azure/core-rest-pipeline": "^1.9.1",
+        "@azure/core-tracing": "^1.0.0",
+        "@azure/core-util": "^1.6.1",
+        "@azure/logger": "^1.0.0",
+        "tslib": "^2.6.2"
+      },
       "engines": {
-        "node": ">=6.9.0"
+        "node": ">=18.0.0"
       }
     },
-    "node_modules/@babel/helper-function-name": {
-      "version": "7.23.0",
-      "resolved": "https://registry.npmjs.org/@babel/helper-function-name/-/helper-function-name-7.23.0.tgz",
-      "integrity": "sha512-OErEqsrxjZTJciZ4Oo+eoZqeW9UIiOcuYKRJA4ZAgV9myA+pOXhhmpfNCKjEH/auVfEYVFJ6y1Tc4r0eIApqiw==",
+    "node_modules/@azure/core-rest-pipeline": {
+      "version": "1.18.2",
+      "resolved": "https://registry.npmjs.org/@azure/core-rest-pipeline/-/core-rest-pipeline-1.18.2.tgz",
+      "integrity": "sha512-IkTf/DWKyCklEtN/WYW3lqEsIaUDshlzWRlZNNwSYtFcCBQz++OtOjxNpm8rr1VcbMS6RpjybQa3u6B6nG0zNw==",
       "dev": true,
+      "license": "MIT",
       "dependencies": {
-        "@babel/template": "^7.22.15",
-        "@babel/types": "^7.23.0"
+        "@azure/abort-controller": "^2.0.0",
+        "@azure/core-auth": "^1.8.0",
+        "@azure/core-tracing": "^1.0.1",
+        "@azure/core-util": "^1.11.0",
+        "@azure/logger": "^1.0.0",
+        "http-proxy-agent": "^7.0.0",
+        "https-proxy-agent": "^7.0.0",
+        "tslib": "^2.6.2"
       },
       "engines": {
-        "node": ">=6.9.0"
+        "node": ">=18.0.0"
       }
     },
-    "node_modules/@babel/helper-hoist-variables": {
-      "version": "7.22.5",
-      "resolved": "https://registry.npmjs.org/@babel/helper-hoist-variables/-/helper-hoist-variables-7.22.5.tgz",
-      "integrity": "sha512-wGjk9QZVzvknA6yKIUURb8zY3grXCcOZt+/7Wcy8O2uctxhplmUPkOdlgoNhmdVee2c92JXbf1xpMtVNbfoxRw==",
+    "node_modules/@azure/core-tracing": {
+      "version": "1.2.0",
+      "resolved": "https://registry.npmjs.org/@azure/core-tracing/-/core-tracing-1.2.0.tgz",
+      "integrity": "sha512-UKTiEJPkWcESPYJz3X5uKRYyOcJD+4nYph+KpfdPRnQJVrZfk0KJgdnaAWKfhsBBtAf/D58Az4AvCJEmWgIBAg==",
       "dev": true,
+      "license": "MIT",
       "dependencies": {
-        "@babel/types": "^7.22.5"
+        "tslib": "^2.6.2"
       },
       "engines": {
-        "node": ">=6.9.0"
+        "node": ">=18.0.0"
       }
     },
-    "node_modules/@babel/helper-split-export-declaration": {
-      "version": "7.22.6",
-      "resolved": "https://registry.npmjs.org/@babel/helper-split-export-declaration/-/helper-split-export-declaration-7.22.6.tgz",
-      "integrity": "sha512-AsUnxuLhRYsisFiaJwvp1QF+I3KjD5FOxut14q/GzovUe6orHLesW2C7d754kRm53h5gqrz6sFl6sxc4BVtE/g==",
+    "node_modules/@azure/core-util": {
+      "version": "1.11.0",
+      "resolved": "https://registry.npmjs.org/@azure/core-util/-/core-util-1.11.0.tgz",
+      "integrity": "sha512-DxOSLua+NdpWoSqULhjDyAZTXFdP/LKkqtYuxxz1SCN289zk3OG8UOpnCQAz/tygyACBtWp/BoO72ptK7msY8g==",
       "dev": true,
+      "license": "MIT",
       "dependencies": {
-        "@babel/types": "^7.22.5"
+        "@azure/abort-controller": "^2.0.0",
+        "tslib": "^2.6.2"
       },
       "engines": {
-        "node": ">=6.9.0"
+        "node": ">=18.0.0"
       }
     },
-    "node_modules/@babel/helper-string-parser": {
-      "version": "7.23.4",
-      "resolved": "https://registry.npmjs.org/@babel/helper-string-parser/-/helper-string-parser-7.23.4.tgz",
-      "integrity": "sha512-803gmbQdqwdf4olxrX4AJyFBV/RTr3rSmOj0rKwesmzlfhYNDEs+/iOcznzpNWlJlIlTJC2QfPFcHB6DlzdVLQ==",
+    "node_modules/@azure/identity": {
+      "version": "4.6.0",
+      "resolved": "https://registry.npmjs.org/@azure/identity/-/identity-4.6.0.tgz",
+      "integrity": "sha512-ANpO1iAvcZmpD4QY7/kaE/P2n66pRXsDp3nMUC6Ow3c9KfXOZF7qMU9VgqPw8m7adP7TVIbVyrCEmD9cth3KQQ==",
       "dev": true,
+      "license": "MIT",
+      "dependencies": {
+        "@azure/abort-controller": "^2.0.0",
+        "@azure/core-auth": "^1.9.0",
+        "@azure/core-client": "^1.9.2",
+        "@azure/core-rest-pipeline": "^1.17.0",
+        "@azure/core-tracing": "^1.0.0",
+        "@azure/core-util": "^1.11.0",
+        "@azure/logger": "^1.0.0",
+        "@azure/msal-browser": "^4.0.1",
+        "@azure/msal-node": "^2.15.0",
+        "events": "^3.0.0",
+        "jws": "^4.0.0",
+        "open": "^8.0.0",
+        "stoppable": "^1.1.0",
+        "tslib": "^2.2.0"
+      },
       "engines": {
-        "node": ">=6.9.0"
+        "node": ">=18.0.0"
       }
     },
-    "node_modules/@babel/helper-validator-identifier": {
-      "version": "7.22.20",
-      "resolved": "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-7.22.20.tgz",
-      "integrity": "sha512-Y4OZ+ytlatR8AI+8KZfKuL5urKp7qey08ha31L8b3BwewJAoJamTzyvxPR/5D+KkdJCGPq/+8TukHBlY10FX9A==",
+    "node_modules/@azure/logger": {
+      "version": "1.1.4",
+      "resolved": "https://registry.npmjs.org/@azure/logger/-/logger-1.1.4.tgz",
+      "integrity": "sha512-4IXXzcCdLdlXuCG+8UKEwLA1T1NHqUfanhXYHiQTn+6sfWCZXduqbtXDGceg3Ce5QxTGo7EqmbV6Bi+aqKuClQ==",
+      "dev": true,
+      "license": "MIT",
+      "dependencies": {
+        "tslib": "^2.6.2"
+      },
+      "engines": {
+        "node": ">=18.0.0"
+      }
+    },
+    "node_modules/@azure/msal-browser": {
+      "version": "4.2.0",
+      "resolved": "https://registry.npmjs.org/@azure/msal-browser/-/msal-browser-4.2.0.tgz",
+      "integrity": "sha512-MXQjgAgjg/2VRKV+UPWHESoZPcue2ZvWKfpBLCyTUyixP+mhCl0q5D1+xDiwBGV3lru2poKZVZDQAOE40wKmWg==",
+      "dev": true,
+      "license": "MIT",
+      "dependencies": {
+        "@azure/msal-common": "15.1.1"
+      },
+      "engines": {
+        "node": ">=0.8.0"
+      }
+    },
+    "node_modules/@azure/msal-common": {
+      "version": "15.1.1",
+      "resolved": "https://registry.npmjs.org/@azure/msal-common/-/msal-common-15.1.1.tgz",
+      "integrity": "sha512-bvLWYq9fleAcTJ6H+hfkG91On6vI/UhGyOB7Z6r0Bsa+KTL3zPtigmGCOJgdxrEklOYD88X9SehexLDH/5NRKQ==",
+      "dev": true,
+      "license": "MIT",
+      "engines": {
+        "node": ">=0.8.0"
+      }
+    },
+    "node_modules/@azure/msal-node": {
+      "version": "2.16.2",
+      "resolved": "https://registry.npmjs.org/@azure/msal-node/-/msal-node-2.16.2.tgz",
+      "integrity": "sha512-An7l1hEr0w1HMMh1LU+rtDtqL7/jw74ORlc9Wnh06v7TU/xpG39/Zdr1ZJu3QpjUfKJ+E0/OXMW8DRSWTlh7qQ==",
+      "dev": true,
+      "license": "MIT",
+      "dependencies": {
+        "@azure/msal-common": "14.16.0",
+        "jsonwebtoken": "^9.0.0",
+        "uuid": "^8.3.0"
+      },
+      "engines": {
+        "node": ">=16"
+      }
+    },
+    "node_modules/@azure/msal-node/node_modules/@azure/msal-common": {
+      "version": "14.16.0",
+      "resolved": "https://registry.npmjs.org/@azure/msal-common/-/msal-common-14.16.0.tgz",
+      "integrity": "sha512-1KOZj9IpcDSwpNiQNjt0jDYZpQvNZay7QAEi/5DLubay40iGYtLzya/jbjRPLyOTZhEKyL1MzPuw2HqBCjceYA==",
+      "dev": true,
+      "license": "MIT",
+      "engines": {
+        "node": ">=0.8.0"
+      }
+    },
+    "node_modules/@babel/code-frame": {
+      "version": "7.26.2",
+      "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.26.2.tgz",
+      "integrity": "sha512-RJlIHRueQgwWitWgF8OdFYGZX328Ax5BCemNGlqHfplnRT9ESi8JkFlvaVYbS+UubVY6dpv87Fs2u5M29iNFVQ==",
       "dev": true,
+      "license": "MIT",
+      "dependencies": {
+        "@babel/helper-validator-identifier": "^7.25.9",
+        "js-tokens": "^4.0.0",
+        "picocolors": "^1.0.0"
+      },
       "engines": {
         "node": ">=6.9.0"
       }
     },
-    "node_modules/@babel/highlight": {
-      "version": "7.23.4",
-      "resolved": "https://registry.npmjs.org/@babel/highlight/-/highlight-7.23.4.tgz",
-      "integrity": "sha512-acGdbYSfp2WheJoJm/EBBBLh/ID8KDc64ISZ9DYtBmC8/Q204PZJLHyzeB5qMzJ5trcOkybd78M4x2KWsUq++A==",
+    "node_modules/@babel/generator": {
+      "version": "7.26.5",
+      "resolved": "https://registry.npmjs.org/@babel/generator/-/generator-7.26.5.tgz",
+      "integrity": "sha512-2caSP6fN9I7HOe6nqhtft7V4g7/V/gfDsC3Ag4W7kEzzvRGKqiv0pu0HogPiZ3KaVSoNDhUws6IJjDjpfmYIXw==",
       "dev": true,
+      "license": "MIT",
       "dependencies": {
-        "@babel/helper-validator-identifier": "^7.22.20",
-        "chalk": "^2.4.2",
-        "js-tokens": "^4.0.0"
+        "@babel/parser": "^7.26.5",
+        "@babel/types": "^7.26.5",
+        "@jridgewell/gen-mapping": "^0.3.5",
+        "@jridgewell/trace-mapping": "^0.3.25",
+        "jsesc": "^3.0.2"
       },
       "engines": {
         "node": ">=6.9.0"
       }
     },
+    "node_modules/@babel/helper-string-parser": {
+      "version": "7.25.9",
+      "resolved": "https://registry.npmjs.org/@babel/helper-string-parser/-/helper-string-parser-7.25.9.tgz",
+      "integrity": "sha512-4A/SCr/2KLd5jrtOMFzaKjVtAei3+2r/NChoBNoZ3EyP/+GlhoaEGoWOZUmFmoITP7zOJyHIMm+DYRd8o3PvHA==",
+      "dev": true,
+      "license": "MIT",
+      "engines": {
+        "node": ">=6.9.0"
+      }
+    },
+    "node_modules/@babel/helper-validator-identifier": {
+      "version": "7.25.9",
+      "resolved": "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-7.25.9.tgz",
+      "integrity": "sha512-Ed61U6XJc3CVRfkERJWDz4dJwKe7iLmmJsbOGu9wSloNSFttHV0I8g6UAgb7qnK5ly5bGLPd4oXZlxCdANBOWQ==",
+      "dev": true,
+      "license": "MIT",
+      "engines": {
+        "node": ">=6.9.0"
+      }
+    },
     "node_modules/@babel/parser": {
-      "version": "7.23.6",
-      "resolved": "https://registry.npmjs.org/@babel/parser/-/parser-7.23.6.tgz",
-      "integrity": "sha512-Z2uID7YJ7oNvAI20O9X0bblw7Qqs8Q2hFy0R9tAfnfLkp5MW0UH9eUvnDSnFwKZ0AvgS1ucqR4KzvVHgnke1VQ==",
+      "version": "7.26.7",
+      "resolved": "https://registry.npmjs.org/@babel/parser/-/parser-7.26.7.tgz",
+      "integrity": "sha512-kEvgGGgEjRUutvdVvZhbn/BxVt+5VSpwXz1j3WYXQbXDo8KzFOPNG2GQbdAiNq8g6wn1yKk7C/qrke03a84V+w==",
       "dev": true,
+      "license": "MIT",
+      "dependencies": {
+        "@babel/types": "^7.26.7"
+      },
       "bin": {
         "parser": "bin/babel-parser.js"
       },
@@ -139,33 +270,32 @@
       }
     },
     "node_modules/@babel/template": {
-      "version": "7.22.15",
-      "resolved": "https://registry.npmjs.org/@babel/template/-/template-7.22.15.tgz",
-      "integrity": "sha512-QPErUVm4uyJa60rkI73qneDacvdvzxshT3kksGqlGWYdOTIUOwJ7RDUL8sGqslY1uXWSL6xMFKEXDS3ox2uF0w==",
+      "version": "7.25.9",
+      "resolved": "https://registry.npmjs.org/@babel/template/-/template-7.25.9.tgz",
+      "integrity": "sha512-9DGttpmPvIxBb/2uwpVo3dqJ+O6RooAFOS+lB+xDqoE2PVCE8nfoHMdZLpfCQRLwvohzXISPZcgxt80xLfsuwg==",
       "dev": true,
+      "license": "MIT",
       "dependencies": {
-        "@babel/code-frame": "^7.22.13",
-        "@babel/parser": "^7.22.15",
-        "@babel/types": "^7.22.15"
+        "@babel/code-frame": "^7.25.9",
+        "@babel/parser": "^7.25.9",
+        "@babel/types": "^7.25.9"
       },
       "engines": {
         "node": ">=6.9.0"
       }
     },
     "node_modules/@babel/traverse": {
-      "version": "7.23.7",
-      "resolved": "https://registry.npmjs.org/@babel/traverse/-/traverse-7.23.7.tgz",
-      "integrity": "sha512-tY3mM8rH9jM0YHFGyfC0/xf+SB5eKUu7HPj7/k3fpi9dAlsMc5YbQvDi0Sh2QTPXqMhyaAtzAr807TIyfQrmyg==",
-      "dev": true,
-      "dependencies": {
-        "@babel/code-frame": "^7.23.5",
-        "@babel/generator": "^7.23.6",
-        "@babel/helper-environment-visitor": "^7.22.20",
-        "@babel/helper-function-name": "^7.23.0",
-        "@babel/helper-hoist-variables": "^7.22.5",
-        "@babel/helper-split-export-declaration": "^7.22.6",
-        "@babel/parser": "^7.23.6",
-        "@babel/types": "^7.23.6",
+      "version": "7.26.7",
+      "resolved": "https://registry.npmjs.org/@babel/traverse/-/traverse-7.26.7.tgz",
+      "integrity": "sha512-1x1sgeyRLC3r5fQOM0/xtQKsYjyxmFjaOrLJNtZ81inNjyJHGIolTULPiSc/2qe1/qfpFLisLQYFnnZl7QoedA==",
+      "dev": true,
+      "license": "MIT",
+      "dependencies": {
+        "@babel/code-frame": "^7.26.2",
+        "@babel/generator": "^7.26.5",
+        "@babel/parser": "^7.26.7",
+        "@babel/template": "^7.25.9",
+        "@babel/types": "^7.26.7",
         "debug": "^4.3.1",
         "globals": "^11.1.0"
       },
@@ -174,62 +304,85 @@
       }
     },
     "node_modules/@babel/types": {
-      "version": "7.23.6",
-      "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.23.6.tgz",
-      "integrity": "sha512-+uarb83brBzPKN38NX1MkB6vb6+mwvR6amUulqAE7ccQw1pEl+bCia9TbdG1lsnFP7lZySvUn37CHyXQdfTwzg==",
+      "version": "7.26.7",
+      "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.26.7.tgz",
+      "integrity": "sha512-t8kDRGrKXyp6+tjUh7hw2RLyclsW4TRoRvRHtSyAX9Bb5ldlFh+90YAYY6awRXrlB4G5G2izNeGySpATlFzmOg==",
       "dev": true,
+      "license": "MIT",
       "dependencies": {
-        "@babel/helper-string-parser": "^7.23.4",
-        "@babel/helper-validator-identifier": "^7.22.20",
-        "to-fast-properties": "^2.0.0"
+        "@babel/helper-string-parser": "^7.25.9",
+        "@babel/helper-validator-identifier": "^7.25.9"
       },
       "engines": {
         "node": ">=6.9.0"
       }
     },
+    "node_modules/@isaacs/cliui": {
+      "version": "8.0.2",
+      "resolved": "https://registry.npmjs.org/@isaacs/cliui/-/cliui-8.0.2.tgz",
+      "integrity": "sha512-O8jcjabXaleOG9DQ0+ARXWZBTfnP4WNAqzuiJK7ll44AmxGKv/J2M4TPjxjY3znBCfvBXFzucm1twdyFybFqEA==",
+      "dev": true,
+      "license": "ISC",
+      "dependencies": {
+        "string-width": "^5.1.2",
+        "string-width-cjs": "npm:string-width@^4.2.0",
+        "strip-ansi": "^7.0.1",
+        "strip-ansi-cjs": "npm:strip-ansi@^6.0.1",
+        "wrap-ansi": "^8.1.0",
+        "wrap-ansi-cjs": "npm:wrap-ansi@^7.0.0"
+      },
+      "engines": {
+        "node": ">=12"
+      }
+    },
     "node_modules/@jridgewell/gen-mapping": {
-      "version": "0.3.3",
-      "resolved": "https://registry.npmjs.org/@jridgewell/gen-mapping/-/gen-mapping-0.3.3.tgz",
-      "integrity": "sha512-HLhSWOLRi875zjjMG/r+Nv0oCW8umGb0BgEhyX3dDX3egwZtB8PqLnjz3yedt8R5StBrzcg4aBpnh8UA9D1BoQ==",
+      "version": "0.3.8",
+      "resolved": "https://registry.npmjs.org/@jridgewell/gen-mapping/-/gen-mapping-0.3.8.tgz",
+      "integrity": "sha512-imAbBGkb+ebQyxKgzv5Hu2nmROxoDOXHh80evxdoXNOrvAnVx7zimzc1Oo5h9RlfV4vPXaE2iM5pOFbvOCClWA==",
       "dev": true,
+      "license": "MIT",
       "dependencies": {
-        "@jridgewell/set-array": "^1.0.1",
+        "@jridgewell/set-array": "^1.2.1",
         "@jridgewell/sourcemap-codec": "^1.4.10",
-        "@jridgewell/trace-mapping": "^0.3.9"
+        "@jridgewell/trace-mapping": "^0.3.24"
       },
       "engines": {
         "node": ">=6.0.0"
       }
     },
     "node_modules/@jridgewell/resolve-uri": {
-      "version": "3.1.1",
-      "resolved": "https://registry.npmjs.org/@jridgewell/resolve-uri/-/resolve-uri-3.1.1.tgz",
-      "integrity": "sha512-dSYZh7HhCDtCKm4QakX0xFpsRDqjjtZf/kjI/v3T3Nwt5r8/qz/M19F9ySyOqU94SXBmeG9ttTul+YnR4LOxFA==",
+      "version": "3.1.2",
+      "resolved": "https://registry.npmjs.org/@jridgewell/resolve-uri/-/resolve-uri-3.1.2.tgz",
+      "integrity": "sha512-bRISgCIjP20/tbWSPWMEi54QVPRZExkuD9lJL+UIxUKtwVJA8wW1Trb1jMs1RFXo1CBTNZ/5hpC9QvmKWdopKw==",
       "dev": true,
+      "license": "MIT",
       "engines": {
         "node": ">=6.0.0"
       }
     },
     "node_modules/@jridgewell/set-array": {
-      "version": "1.1.2",
-      "resolved": "https://registry.npmjs.org/@jridgewell/set-array/-/set-array-1.1.2.tgz",
-      "integrity": "sha512-xnkseuNADM0gt2bs+BvhO0p78Mk762YnZdsuzFV018NoG1Sj1SCQvpSqa7XUaTam5vAGasABV9qXASMKnFMwMw==",
+      "version": "1.2.1",
+      "resolved": "https://registry.npmjs.org/@jridgewell/set-array/-/set-array-1.2.1.tgz",
+      "integrity": "sha512-R8gLRTZeyp03ymzP/6Lil/28tGeGEzhx1q2k703KGWRAI1VdvPIXdG70VJc2pAMw3NA6JKL5hhFu1sJX0Mnn/A==",
       "dev": true,
+      "license": "MIT",
       "engines": {
         "node": ">=6.0.0"
       }
     },
     "node_modules/@jridgewell/sourcemap-codec": {
-      "version": "1.4.15",
-      "resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.4.15.tgz",
-      "integrity": "sha512-eF2rxCRulEKXHTRiDrDy6erMYWqNw4LPdQ8UQA4huuxaQsVeRPFl2oM8oDGxMFhJUWZf9McpLtJasDDZb/Bpeg==",
-      "dev": true
+      "version": "1.5.0",
+      "resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.5.0.tgz",
+      "integrity": "sha512-gv3ZRaISU3fjPAgNsriBRqGWQL6quFx04YMPW/zD8XMLsU32mhCCbfbO6KZFLjvYpCZ8zyDEgqsgf+PwPaM7GQ==",
+      "dev": true,
+      "license": "MIT"
     },
     "node_modules/@jridgewell/trace-mapping": {
-      "version": "0.3.20",
-      "resolved": "https://registry.npmjs.org/@jridgewell/trace-mapping/-/trace-mapping-0.3.20.tgz",
-      "integrity": "sha512-R8LcPeWZol2zR8mmH3JeKQ6QRCFb7XgUhV9ZlGhHLGyg4wpPiPZNQOOWhFZhxKw8u//yTbNGI42Bx/3paXEQ+Q==",
+      "version": "0.3.25",
+      "resolved": "https://registry.npmjs.org/@jridgewell/trace-mapping/-/trace-mapping-0.3.25.tgz",
+      "integrity": "sha512-vNk6aEwybGtawWmy/PzwnGDOjCkLWSD2wqvjGGAgOAwCGWySYXfYoxt00IJkTF+8Lb57DwOb3Aa0o9CApepiYQ==",
       "dev": true,
+      "license": "MIT",
       "dependencies": {
         "@jridgewell/resolve-uri": "^3.1.0",
         "@jridgewell/sourcemap-codec": "^1.4.14"
@@ -245,32 +398,38 @@
       }
     },
     "node_modules/@types/vscode": {
-      "version": "1.74.0",
-      "resolved": "https://registry.npmjs.org/@types/vscode/-/vscode-1.74.0.tgz",
-      "integrity": "sha512-LyeCIU3jb9d38w0MXFwta9r0Jx23ugujkAxdwLTNCyspdZTKUc43t7ppPbCiPoQ/Ivd/pnDFZrb4hWd45wrsgA==",
-      "dev": true
+      "version": "1.75.0",
+      "resolved": "https://registry.npmjs.org/@types/vscode/-/vscode-1.75.0.tgz",
+      "integrity": "sha512-SAr0PoOhJS6FUq5LjNr8C/StBKALZwDVm3+U4pjF/3iYkt3GioJOPV/oB1Sf1l7lROe4TgrMyL5N1yaEgTWycw==",
+      "dev": true,
+      "license": "MIT"
     },
     "node_modules/@vscode/vsce": {
-      "version": "2.22.0",
-      "resolved": "https://registry.npmjs.org/@vscode/vsce/-/vsce-2.22.0.tgz",
-      "integrity": "sha512-8df4uJiM3C6GZ2Sx/KilSKVxsetrTBBIUb3c0W4B1EWHcddioVs5mkyDKtMNP0khP/xBILVSzlXxhV+nm2rC9A==",
+      "version": "3.2.2",
+      "resolved": "https://registry.npmjs.org/@vscode/vsce/-/vsce-3.2.2.tgz",
+      "integrity": "sha512-4TqdUq/yKlQTHcQMk/DamR632bq/+IJDomSbexOMee/UAYWqYm0XHWA6scGslsCpzY+sCWEhhl0nqdOB0XW1kw==",
       "dev": true,
+      "license": "MIT",
       "dependencies": {
-        "azure-devops-node-api": "^11.0.1",
+        "@azure/identity": "^4.1.0",
+        "@vscode/vsce-sign": "^2.0.0",
+        "azure-devops-node-api": "^12.5.0",
         "chalk": "^2.4.2",
         "cheerio": "^1.0.0-rc.9",
-        "commander": "^6.2.1",
-        "glob": "^7.0.6",
+        "cockatiel": "^3.1.2",
+        "commander": "^12.1.0",
+        "form-data": "^4.0.0",
+        "glob": "^11.0.0",
         "hosted-git-info": "^4.0.2",
         "jsonc-parser": "^3.2.0",
         "leven": "^3.1.0",
-        "markdown-it": "^12.3.2",
+        "markdown-it": "^14.1.0",
         "mime": "^1.3.4",
         "minimatch": "^3.0.3",
         "parse-semver": "^1.1.1",
         "read": "^1.0.7",
         "semver": "^7.5.2",
-        "tmp": "^0.2.1",
+        "tmp": "^0.2.3",
         "typed-rest-client": "^1.8.4",
         "url-join": "^4.0.1",
         "xml2js": "^0.5.0",
@@ -281,12 +440,180 @@
         "vsce": "vsce"
       },
       "engines": {
-        "node": ">= 14"
+        "node": ">= 20"
       },
       "optionalDependencies": {
         "keytar": "^7.7.0"
       }
     },
+    "node_modules/@vscode/vsce-sign": {
+      "version": "2.0.5",
+      "resolved": "https://registry.npmjs.org/@vscode/vsce-sign/-/vsce-sign-2.0.5.tgz",
+      "integrity": "sha512-GfYWrsT/vypTMDMgWDm75iDmAOMe7F71sZECJ+Ws6/xyIfmB3ELVnVN+LwMFAvmXY+e6eWhR2EzNGF/zAhWY3Q==",
+      "dev": true,
+      "hasInstallScript": true,
+      "license": "SEE LICENSE IN LICENSE.txt",
+      "optionalDependencies": {
+        "@vscode/vsce-sign-alpine-arm64": "2.0.2",
+        "@vscode/vsce-sign-alpine-x64": "2.0.2",
+        "@vscode/vsce-sign-darwin-arm64": "2.0.2",
+        "@vscode/vsce-sign-darwin-x64": "2.0.2",
+        "@vscode/vsce-sign-linux-arm": "2.0.2",
+        "@vscode/vsce-sign-linux-arm64": "2.0.2",
+        "@vscode/vsce-sign-linux-x64": "2.0.2",
+        "@vscode/vsce-sign-win32-arm64": "2.0.2",
+        "@vscode/vsce-sign-win32-x64": "2.0.2"
+      }
+    },
+    "node_modules/@vscode/vsce-sign-alpine-arm64": {
+      "version": "2.0.2",
+      "resolved": "https://registry.npmjs.org/@vscode/vsce-sign-alpine-arm64/-/vsce-sign-alpine-arm64-2.0.2.tgz",
+      "integrity": "sha512-E80YvqhtZCLUv3YAf9+tIbbqoinWLCO/B3j03yQPbjT3ZIHCliKZlsy1peNc4XNZ5uIb87Jn0HWx/ZbPXviuAQ==",
+      "cpu": [
+        "arm64"
+      ],
+      "dev": true,
+      "license": "SEE LICENSE IN LICENSE.txt",
+      "optional": true,
+      "os": [
+        "alpine"
+      ]
+    },
+    "node_modules/@vscode/vsce-sign-alpine-x64": {
+      "version": "2.0.2",
+      "resolved": "https://registry.npmjs.org/@vscode/vsce-sign-alpine-x64/-/vsce-sign-alpine-x64-2.0.2.tgz",
+      "integrity": "sha512-n1WC15MSMvTaeJ5KjWCzo0nzjydwxLyoHiMJHu1Ov0VWTZiddasmOQHekA47tFRycnt4FsQrlkSCTdgHppn6bw==",
+      "cpu": [
+        "x64"
+      ],
+      "dev": true,
+      "license": "SEE LICENSE IN LICENSE.txt",
+      "optional": true,
+      "os": [
+        "alpine"
+      ]
+    },
+    "node_modules/@vscode/vsce-sign-darwin-arm64": {
+      "version": "2.0.2",
+      "resolved": "https://registry.npmjs.org/@vscode/vsce-sign-darwin-arm64/-/vsce-sign-darwin-arm64-2.0.2.tgz",
+      "integrity": "sha512-rz8F4pMcxPj8fjKAJIfkUT8ycG9CjIp888VY/6pq6cuI2qEzQ0+b5p3xb74CJnBbSC0p2eRVoe+WgNCAxCLtzQ==",
+      "cpu": [
+        "arm64"
+      ],
+      "dev": true,
+      "license": "SEE LICENSE IN LICENSE.txt",
+      "optional": true,
+      "os": [
+        "darwin"
+      ]
+    },
+    "node_modules/@vscode/vsce-sign-darwin-x64": {
+      "version": "2.0.2",
+      "resolved": "https://registry.npmjs.org/@vscode/vsce-sign-darwin-x64/-/vsce-sign-darwin-x64-2.0.2.tgz",
+      "integrity": "sha512-MCjPrQ5MY/QVoZ6n0D92jcRb7eYvxAujG/AH2yM6lI0BspvJQxp0o9s5oiAM9r32r9tkLpiy5s2icsbwefAQIw==",
+      "cpu": [
+        "x64"
+      ],
+      "dev": true,
+      "license": "SEE LICENSE IN LICENSE.txt",
+      "optional": true,
+      "os": [
+        "darwin"
+      ]
+    },
+    "node_modules/@vscode/vsce-sign-linux-arm": {
+      "version": "2.0.2",
+      "resolved": "https://registry.npmjs.org/@vscode/vsce-sign-linux-arm/-/vsce-sign-linux-arm-2.0.2.tgz",
+      "integrity": "sha512-Fkb5jpbfhZKVw3xwR6t7WYfwKZktVGNXdg1m08uEx1anO0oUPUkoQRsNm4QniL3hmfw0ijg00YA6TrxCRkPVOQ==",
+      "cpu": [
+        "arm"
+      ],
+      "dev": true,
+      "license": "SEE LICENSE IN LICENSE.txt",
+      "optional": true,
+      "os": [
+        "linux"
+      ]
+    },
+    "node_modules/@vscode/vsce-sign-linux-arm64": {
+      "version": "2.0.2",
+      "resolved": "https://registry.npmjs.org/@vscode/vsce-sign-linux-arm64/-/vsce-sign-linux-arm64-2.0.2.tgz",
+      "integrity": "sha512-Ybeu7cA6+/koxszsORXX0OJk9N0GgfHq70Wqi4vv2iJCZvBrOWwcIrxKjvFtwyDgdeQzgPheH5nhLVl5eQy7WA==",
+      "cpu": [
+        "arm64"
+      ],
+      "dev": true,
+      "license": "SEE LICENSE IN LICENSE.txt",
+      "optional": true,
+      "os": [
+        "linux"
+      ]
+    },
+    "node_modules/@vscode/vsce-sign-linux-x64": {
+      "version": "2.0.2",
+      "resolved": "https://registry.npmjs.org/@vscode/vsce-sign-linux-x64/-/vsce-sign-linux-x64-2.0.2.tgz",
+      "integrity": "sha512-NsPPFVtLaTlVJKOiTnO8Cl78LZNWy0Q8iAg+LlBiCDEgC12Gt4WXOSs2pmcIjDYzj2kY4NwdeN1mBTaujYZaPg==",
+      "cpu": [
+        "x64"
+      ],
+      "dev": true,
+      "license": "SEE LICENSE IN LICENSE.txt",
+      "optional": true,
+      "os": [
+        "linux"
+      ]
+    },
+    "node_modules/@vscode/vsce-sign-win32-arm64": {
+      "version": "2.0.2",
+      "resolved": "https://registry.npmjs.org/@vscode/vsce-sign-win32-arm64/-/vsce-sign-win32-arm64-2.0.2.tgz",
+      "integrity": "sha512-wPs848ymZ3Ny+Y1Qlyi7mcT6VSigG89FWQnp2qRYCyMhdJxOpA4lDwxzlpL8fG6xC8GjQjGDkwbkWUcCobvksQ==",
+      "cpu": [
+        "arm64"
+      ],
+      "dev": true,
+      "license": "SEE LICENSE IN LICENSE.txt",
+      "optional": true,
+      "os": [
+        "win32"
+      ]
+    },
+    "node_modules/@vscode/vsce-sign-win32-x64": {
+      "version": "2.0.2",
+      "resolved": "https://registry.npmjs.org/@vscode/vsce-sign-win32-x64/-/vsce-sign-win32-x64-2.0.2.tgz",
+      "integrity": "sha512-pAiRN6qSAhDM5SVOIxgx+2xnoVUePHbRNC7OD2aOR3WltTKxxF25OfpK8h8UQ7A0BuRkSgREbB59DBlFk4iAeg==",
+      "cpu": [
+        "x64"
+      ],
+      "dev": true,
+      "license": "SEE LICENSE IN LICENSE.txt",
+      "optional": true,
+      "os": [
+        "win32"
+      ]
+    },
+    "node_modules/agent-base": {
+      "version": "7.1.3",
+      "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-7.1.3.tgz",
+      "integrity": "sha512-jRR5wdylq8CkOe6hei19GGZnxM6rBGwFl3Bg0YItGDimvjGtAvdZk4Pu6Cl4u4Igsws4a1fd1Vq3ezrhn4KmFw==",
+      "dev": true,
+      "license": "MIT",
+      "engines": {
+        "node": ">= 14"
+      }
+    },
+    "node_modules/ansi-regex": {
+      "version": "6.1.0",
+      "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-6.1.0.tgz",
+      "integrity": "sha512-7HSX4QQb4CspciLpVFwyRe79O3xsIZDDLER21kERQ71oaPodF8jL725AgJMFAYbooIqolJoRLuM81SpeUkpkvA==",
+      "dev": true,
+      "license": "MIT",
+      "engines": {
+        "node": ">=12"
+      },
+      "funding": {
+        "url": "https://github.com/chalk/ansi-regex?sponsor=1"
+      }
+    },
     "node_modules/ansi-styles": {
       "version": "3.2.1",
       "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz",
@@ -303,13 +630,22 @@
       "version": "2.0.1",
       "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz",
       "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==",
-      "dev": true
+      "dev": true,
+      "license": "Python-2.0"
+    },
+    "node_modules/asynckit": {
+      "version": "0.4.0",
+      "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz",
+      "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==",
+      "dev": true,
+      "license": "MIT"
     },
     "node_modules/azure-devops-node-api": {
-      "version": "11.2.0",
-      "resolved": "https://registry.npmjs.org/azure-devops-node-api/-/azure-devops-node-api-11.2.0.tgz",
-      "integrity": "sha512-XdiGPhrpaT5J8wdERRKs5g8E0Zy1pvOYTli7z9E8nmOn3YGp4FhtjhrOyFmX/8veWCwdI69mCHKJw6l+4J/bHA==",
+      "version": "12.5.0",
+      "resolved": "https://registry.npmjs.org/azure-devops-node-api/-/azure-devops-node-api-12.5.0.tgz",
+      "integrity": "sha512-R5eFskGvOm3U/GzeAuxRkUsAl0hrAwGgWn6zAd2KrZmrEhWZVqLew4OOupbQlXUuojUzpGtq62SmdhJ06N88og==",
       "dev": true,
+      "license": "MIT",
       "dependencies": {
         "tunnel": "0.0.6",
         "typed-rest-client": "^1.8.4"
@@ -319,7 +655,8 @@
       "version": "1.0.2",
       "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz",
       "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==",
-      "dev": true
+      "dev": true,
+      "license": "MIT"
     },
     "node_modules/base64-js": {
       "version": "1.5.1",
@@ -365,6 +702,7 @@
       "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz",
       "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==",
       "dev": true,
+      "license": "MIT",
       "dependencies": {
         "balanced-match": "^1.0.0",
         "concat-map": "0.0.1"
@@ -404,15 +742,39 @@
         "node": "*"
       }
     },
-    "node_modules/call-bind": {
-      "version": "1.0.5",
-      "resolved": "https://registry.npmjs.org/call-bind/-/call-bind-1.0.5.tgz",
-      "integrity": "sha512-C3nQxfFZxFRVoJoGKKI8y3MOEo129NQ+FgQ08iye+Mk4zNZZGdjfs06bVTr+DBSlA66Q2VEcMki/cUCP4SercQ==",
+    "node_modules/buffer-equal-constant-time": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/buffer-equal-constant-time/-/buffer-equal-constant-time-1.0.1.tgz",
+      "integrity": "sha512-zRpUiDwd/xk6ADqPMATG8vc9VPrkck7T07OIx0gnjmJAnHnTVXNQG3vfvWNuiZIkwu9KrKdA1iJKfsfTVxE6NA==",
+      "dev": true,
+      "license": "BSD-3-Clause"
+    },
+    "node_modules/call-bind-apply-helpers": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/call-bind-apply-helpers/-/call-bind-apply-helpers-1.0.1.tgz",
+      "integrity": "sha512-BhYE+WDaywFg2TBWYNXAE+8B1ATnThNBqXHP5nQu0jWJdVvY2hvkpyB3qOmtmDePiS5/BDQ8wASEWGMWRG148g==",
       "dev": true,
+      "license": "MIT",
       "dependencies": {
-        "function-bind": "^1.1.2",
-        "get-intrinsic": "^1.2.1",
-        "set-function-length": "^1.1.1"
+        "es-errors": "^1.3.0",
+        "function-bind": "^1.1.2"
+      },
+      "engines": {
+        "node": ">= 0.4"
+      }
+    },
+    "node_modules/call-bound": {
+      "version": "1.0.3",
+      "resolved": "https://registry.npmjs.org/call-bound/-/call-bound-1.0.3.tgz",
+      "integrity": "sha512-YTd+6wGlNlPxSuri7Y6X8tY2dmm12UMH66RpKMhiX6rsk5wXXnYgbUcOt8kiS31/AjfoTOvCsE+w8nZQLQnzHA==",
+      "dev": true,
+      "license": "MIT",
+      "dependencies": {
+        "call-bind-apply-helpers": "^1.0.1",
+        "get-intrinsic": "^1.2.6"
+      },
+      "engines": {
+        "node": ">= 0.4"
       },
       "funding": {
         "url": "https://github.com/sponsors/ljharb"
@@ -477,6 +839,16 @@
       "dev": true,
       "optional": true
     },
+    "node_modules/cockatiel": {
+      "version": "3.2.1",
+      "resolved": "https://registry.npmjs.org/cockatiel/-/cockatiel-3.2.1.tgz",
+      "integrity": "sha512-gfrHV6ZPkquExvMh9IOkKsBzNDk6sDuZ6DdBGUBkvFnTCqCxzpuq48RySgP0AnaqQkw2zynOFj9yly6T1Q2G5Q==",
+      "dev": true,
+      "license": "MIT",
+      "engines": {
+        "node": ">=16"
+      }
+    },
     "node_modules/color-convert": {
       "version": "1.9.3",
       "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz",
@@ -492,20 +864,50 @@
       "integrity": "sha512-72fSenhMw2HZMTVHeCA9KCmpEIbzWiQsjN+BHcBbS9vr1mtt+vJjPdksIBNUmKAW8TFUDPJK5SUU3QhE9NEXDw==",
       "dev": true
     },
+    "node_modules/combined-stream": {
+      "version": "1.0.8",
+      "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz",
+      "integrity": "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==",
+      "dev": true,
+      "license": "MIT",
+      "dependencies": {
+        "delayed-stream": "~1.0.0"
+      },
+      "engines": {
+        "node": ">= 0.8"
+      }
+    },
     "node_modules/commander": {
-      "version": "6.2.1",
-      "resolved": "https://registry.npmjs.org/commander/-/commander-6.2.1.tgz",
-      "integrity": "sha512-U7VdrJFnJgo4xjrHpTzu0yrHPGImdsmD95ZlgYSEajAn2JKzDhDTPG9kBTefmObL2w/ngeZnilk+OV9CG3d7UA==",
+      "version": "12.1.0",
+      "resolved": "https://registry.npmjs.org/commander/-/commander-12.1.0.tgz",
+      "integrity": "sha512-Vw8qHK3bZM9y/P10u3Vib8o/DdkvA2OtPtZvD871QKjy74Wj1WSKFILMPRPSdUSx5RFK1arlJzEtA4PkFgnbuA==",
       "dev": true,
+      "license": "MIT",
       "engines": {
-        "node": ">= 6"
+        "node": ">=18"
       }
     },
     "node_modules/concat-map": {
       "version": "0.0.1",
       "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz",
       "integrity": "sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==",
-      "dev": true
+      "dev": true,
+      "license": "MIT"
+    },
+    "node_modules/cross-spawn": {
+      "version": "7.0.6",
+      "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz",
+      "integrity": "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==",
+      "dev": true,
+      "license": "MIT",
+      "dependencies": {
+        "path-key": "^3.1.0",
+        "shebang-command": "^2.0.0",
+        "which": "^2.0.1"
+      },
+      "engines": {
+        "node": ">= 8"
+      }
     },
     "node_modules/css-select": {
       "version": "5.1.0",
@@ -536,12 +938,13 @@
       }
     },
     "node_modules/debug": {
-      "version": "4.3.4",
-      "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz",
-      "integrity": "sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==",
+      "version": "4.4.0",
+      "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.0.tgz",
+      "integrity": "sha512-6WTZ/IxCY/T6BALoZHaE4ctp9xm+Z5kY/pzYaCHRFeyVhojxlrm+46y68HA6hr0TcwEssoxNiDEUJQjfPZ/RYA==",
       "dev": true,
+      "license": "MIT",
       "dependencies": {
-        "ms": "2.1.2"
+        "ms": "^2.1.3"
       },
       "engines": {
         "node": ">=6.0"
@@ -578,18 +981,24 @@
         "node": ">=4.0.0"
       }
     },
-    "node_modules/define-data-property": {
-      "version": "1.1.1",
-      "resolved": "https://registry.npmjs.org/define-data-property/-/define-data-property-1.1.1.tgz",
-      "integrity": "sha512-E7uGkTzkk1d0ByLeSc6ZsFS79Axg+m1P/VsgYsxHgiuc3tFSj+MjMIwe90FC4lOAZzNBdY7kkO2P2wKdsQ1vgQ==",
+    "node_modules/define-lazy-prop": {
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/define-lazy-prop/-/define-lazy-prop-2.0.0.tgz",
+      "integrity": "sha512-Ds09qNh8yw3khSjiJjiUInaGX9xlqZDY7JVryGxdxV7NPeuqQfplOpQ66yJFZut3jLa5zOwkXw1g9EI2uKh4Og==",
       "dev": true,
-      "dependencies": {
-        "get-intrinsic": "^1.2.1",
-        "gopd": "^1.0.1",
-        "has-property-descriptors": "^1.0.0"
-      },
+      "license": "MIT",
       "engines": {
-        "node": ">= 0.4"
+        "node": ">=8"
+      }
+    },
+    "node_modules/delayed-stream": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz",
+      "integrity": "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==",
+      "dev": true,
+      "license": "MIT",
+      "engines": {
+        "node": ">=0.4.0"
       }
     },
     "node_modules/detect-libc": {
@@ -657,6 +1066,45 @@
         "url": "https://github.com/fb55/domutils?sponsor=1"
       }
     },
+    "node_modules/dunder-proto": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/dunder-proto/-/dunder-proto-1.0.1.tgz",
+      "integrity": "sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A==",
+      "dev": true,
+      "license": "MIT",
+      "dependencies": {
+        "call-bind-apply-helpers": "^1.0.1",
+        "es-errors": "^1.3.0",
+        "gopd": "^1.2.0"
+      },
+      "engines": {
+        "node": ">= 0.4"
+      }
+    },
+    "node_modules/eastasianwidth": {
+      "version": "0.2.0",
+      "resolved": "https://registry.npmjs.org/eastasianwidth/-/eastasianwidth-0.2.0.tgz",
+      "integrity": "sha512-I88TYZWc9XiYHRQ4/3c5rjjfgkjhLyW2luGIheGERbNQ6OY7yTybanSpDXZa8y7VUP9YmDcYa+eyq4ca7iLqWA==",
+      "dev": true,
+      "license": "MIT"
+    },
+    "node_modules/ecdsa-sig-formatter": {
+      "version": "1.0.11",
+      "resolved": "https://registry.npmjs.org/ecdsa-sig-formatter/-/ecdsa-sig-formatter-1.0.11.tgz",
+      "integrity": "sha512-nagl3RYrbNv6kQkeJIpt6NJZy8twLB/2vtz6yN9Z4vRKHN4/QZJIEbqohALSgwKdnksuY3k5Addp5lg8sVoVcQ==",
+      "dev": true,
+      "license": "Apache-2.0",
+      "dependencies": {
+        "safe-buffer": "^5.0.1"
+      }
+    },
+    "node_modules/emoji-regex": {
+      "version": "9.2.2",
+      "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-9.2.2.tgz",
+      "integrity": "sha512-L18DaJsXSUk2+42pv8mLs5jJT2hqFkFE4j21wOmgbUqsZ2hL72NsUU785g9RXgo3s0ZNgVl42TiHp3ZtOv/Vyg==",
+      "dev": true,
+      "license": "MIT"
+    },
     "node_modules/end-of-stream": {
       "version": "1.4.4",
       "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.4.tgz",
@@ -679,6 +1127,39 @@
         "url": "https://github.com/fb55/entities?sponsor=1"
       }
     },
+    "node_modules/es-define-property": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/es-define-property/-/es-define-property-1.0.1.tgz",
+      "integrity": "sha512-e3nRfgfUZ4rNGL232gUgX06QNyyez04KdjFrF+LTRoOXmrOgFKDg4BCdsjW8EnT69eqdYGmRpJwiPVYNrCaW3g==",
+      "dev": true,
+      "license": "MIT",
+      "engines": {
+        "node": ">= 0.4"
+      }
+    },
+    "node_modules/es-errors": {
+      "version": "1.3.0",
+      "resolved": "https://registry.npmjs.org/es-errors/-/es-errors-1.3.0.tgz",
+      "integrity": "sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw==",
+      "dev": true,
+      "license": "MIT",
+      "engines": {
+        "node": ">= 0.4"
+      }
+    },
+    "node_modules/es-object-atoms": {
+      "version": "1.1.1",
+      "resolved": "https://registry.npmjs.org/es-object-atoms/-/es-object-atoms-1.1.1.tgz",
+      "integrity": "sha512-FGgH2h8zKNim9ljj7dankFPcICIK9Cp5bm+c2gQSYePhpaG5+esrLODihIorn+Pe6FGJzWhXQotPv73jTaldXA==",
+      "dev": true,
+      "license": "MIT",
+      "dependencies": {
+        "es-errors": "^1.3.0"
+      },
+      "engines": {
+        "node": ">= 0.4"
+      }
+    },
     "node_modules/escape-string-regexp": {
       "version": "1.0.5",
       "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz",
@@ -688,6 +1169,16 @@
         "node": ">=0.8.0"
       }
     },
+    "node_modules/events": {
+      "version": "3.3.0",
+      "resolved": "https://registry.npmjs.org/events/-/events-3.3.0.tgz",
+      "integrity": "sha512-mQw+2fkQbALzQ7V0MY0IqdnXNOeTtP4r0lN9z7AAawCXgqea7bDii20AYrIBrFd/Hx0M2Ocz6S111CaFkUcb0Q==",
+      "dev": true,
+      "license": "MIT",
+      "engines": {
+        "node": ">=0.8.x"
+      }
+    },
     "node_modules/expand-template": {
       "version": "2.0.3",
       "resolved": "https://registry.npmjs.org/expand-template/-/expand-template-2.0.3.tgz",
@@ -707,43 +1198,94 @@
         "pend": "~1.2.0"
       }
     },
-    "node_modules/fs-constants": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/fs-constants/-/fs-constants-1.0.0.tgz",
-      "integrity": "sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow==",
+    "node_modules/foreground-child": {
+      "version": "3.3.0",
+      "resolved": "https://registry.npmjs.org/foreground-child/-/foreground-child-3.3.0.tgz",
+      "integrity": "sha512-Ld2g8rrAyMYFXBhEqMz8ZAHBi4J4uS1i/CxGMDnjyFWddMXLVcDp051DZfu+t7+ab7Wv6SMqpWmyFIj5UbfFvg==",
       "dev": true,
-      "optional": true
+      "license": "ISC",
+      "dependencies": {
+        "cross-spawn": "^7.0.0",
+        "signal-exit": "^4.0.1"
+      },
+      "engines": {
+        "node": ">=14"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/isaacs"
+      }
+    },
+    "node_modules/form-data": {
+      "version": "4.0.1",
+      "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.1.tgz",
+      "integrity": "sha512-tzN8e4TX8+kkxGPK8D5u0FNmjPUjw3lwC9lSLxxoB/+GtsJG91CO8bSWy73APlgAZzZbXEYZJuxjkHH2w+Ezhw==",
+      "dev": true,
+      "license": "MIT",
+      "dependencies": {
+        "asynckit": "^0.4.0",
+        "combined-stream": "^1.0.8",
+        "mime-types": "^2.1.12"
+      },
+      "engines": {
+        "node": ">= 6"
+      }
     },
-    "node_modules/fs.realpath": {
+    "node_modules/fs-constants": {
       "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz",
-      "integrity": "sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw==",
-      "dev": true
+      "resolved": "https://registry.npmjs.org/fs-constants/-/fs-constants-1.0.0.tgz",
+      "integrity": "sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow==",
+      "dev": true,
+      "optional": true
     },
     "node_modules/function-bind": {
       "version": "1.1.2",
       "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.2.tgz",
       "integrity": "sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==",
       "dev": true,
+      "license": "MIT",
       "funding": {
         "url": "https://github.com/sponsors/ljharb"
       }
     },
     "node_modules/get-intrinsic": {
-      "version": "1.2.2",
-      "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.2.2.tgz",
-      "integrity": "sha512-0gSo4ml/0j98Y3lngkFEot/zhiCeWsbYIlZ+uZOVgzLyLaUw7wxUL+nCTP0XJvJg1AXulJRI3UJi8GsbDuxdGA==",
+      "version": "1.2.7",
+      "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.2.7.tgz",
+      "integrity": "sha512-VW6Pxhsrk0KAOqs3WEd0klDiF/+V7gQOpAvY1jVU/LHmaD/kQO4523aiJuikX/QAKYiW6x8Jh+RJej1almdtCA==",
       "dev": true,
+      "license": "MIT",
       "dependencies": {
+        "call-bind-apply-helpers": "^1.0.1",
+        "es-define-property": "^1.0.1",
+        "es-errors": "^1.3.0",
+        "es-object-atoms": "^1.0.0",
         "function-bind": "^1.1.2",
-        "has-proto": "^1.0.1",
-        "has-symbols": "^1.0.3",
-        "hasown": "^2.0.0"
+        "get-proto": "^1.0.0",
+        "gopd": "^1.2.0",
+        "has-symbols": "^1.1.0",
+        "hasown": "^2.0.2",
+        "math-intrinsics": "^1.1.0"
+      },
+      "engines": {
+        "node": ">= 0.4"
       },
       "funding": {
         "url": "https://github.com/sponsors/ljharb"
       }
     },
+    "node_modules/get-proto": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/get-proto/-/get-proto-1.0.1.tgz",
+      "integrity": "sha512-sTSfBjoXBp89JvIKIefqw7U2CCebsc74kiY6awiGogKtoSGbgjYE/G/+l9sF3MWFPNc9IcoOC4ODfKHfxFmp0g==",
+      "dev": true,
+      "license": "MIT",
+      "dependencies": {
+        "dunder-proto": "^1.0.1",
+        "es-object-atoms": "^1.0.0"
+      },
+      "engines": {
+        "node": ">= 0.4"
+      }
+    },
     "node_modules/github-from-package": {
       "version": "0.0.0",
       "resolved": "https://registry.npmjs.org/github-from-package/-/github-from-package-0.0.0.tgz",
@@ -752,20 +1294,50 @@
       "optional": true
     },
     "node_modules/glob": {
-      "version": "7.2.3",
-      "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz",
-      "integrity": "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==",
+      "version": "11.0.1",
+      "resolved": "https://registry.npmjs.org/glob/-/glob-11.0.1.tgz",
+      "integrity": "sha512-zrQDm8XPnYEKawJScsnM0QzobJxlT/kHOOlRTio8IH/GrmxRE5fjllkzdaHclIuNjUQTJYH2xHNIGfdpJkDJUw==",
       "dev": true,
+      "license": "ISC",
       "dependencies": {
-        "fs.realpath": "^1.0.0",
-        "inflight": "^1.0.4",
-        "inherits": "2",
-        "minimatch": "^3.1.1",
-        "once": "^1.3.0",
-        "path-is-absolute": "^1.0.0"
+        "foreground-child": "^3.1.0",
+        "jackspeak": "^4.0.1",
+        "minimatch": "^10.0.0",
+        "minipass": "^7.1.2",
+        "package-json-from-dist": "^1.0.0",
+        "path-scurry": "^2.0.0"
+      },
+      "bin": {
+        "glob": "dist/esm/bin.mjs"
       },
       "engines": {
-        "node": "*"
+        "node": "20 || >=22"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/isaacs"
+      }
+    },
+    "node_modules/glob/node_modules/brace-expansion": {
+      "version": "2.0.1",
+      "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.1.tgz",
+      "integrity": "sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==",
+      "dev": true,
+      "license": "MIT",
+      "dependencies": {
+        "balanced-match": "^1.0.0"
+      }
+    },
+    "node_modules/glob/node_modules/minimatch": {
+      "version": "10.0.1",
+      "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-10.0.1.tgz",
+      "integrity": "sha512-ethXTt3SGGR+95gudmqJ1eNhRO7eGEGIgYA9vnPatK4/etz2MEVDno5GMCibdMTuBMyElzIlgxMna3K94XDIDQ==",
+      "dev": true,
+      "license": "ISC",
+      "dependencies": {
+        "brace-expansion": "^2.0.1"
+      },
+      "engines": {
+        "node": "20 || >=22"
       },
       "funding": {
         "url": "https://github.com/sponsors/isaacs"
@@ -776,17 +1348,19 @@
       "resolved": "https://registry.npmjs.org/globals/-/globals-11.12.0.tgz",
       "integrity": "sha512-WOBp/EEGUiIsJSp7wcv/y6MO+lV9UoncWqxuFfm8eBwzWNgyfBd6Gz+IeKQ9jCmyhoH99g15M3T+QaVHFjizVA==",
       "dev": true,
+      "license": "MIT",
       "engines": {
         "node": ">=4"
       }
     },
     "node_modules/gopd": {
-      "version": "1.0.1",
-      "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.0.1.tgz",
-      "integrity": "sha512-d65bNlIadxvpb/A2abVdlqKqV563juRnZ1Wtk6s1sIR8uNsXR70xqIzVqxVf1eTqDunwT2MkczEeaezCKTZhwA==",
+      "version": "1.2.0",
+      "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.2.0.tgz",
+      "integrity": "sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg==",
       "dev": true,
-      "dependencies": {
-        "get-intrinsic": "^1.1.3"
+      "license": "MIT",
+      "engines": {
+        "node": ">= 0.4"
       },
       "funding": {
         "url": "https://github.com/sponsors/ljharb"
@@ -801,35 +1375,12 @@
         "node": ">=4"
       }
     },
-    "node_modules/has-property-descriptors": {
-      "version": "1.0.1",
-      "resolved": "https://registry.npmjs.org/has-property-descriptors/-/has-property-descriptors-1.0.1.tgz",
-      "integrity": "sha512-VsX8eaIewvas0xnvinAe9bw4WfIeODpGYikiWYLH+dma0Jw6KHYqWiWfhQlgOVK8D6PvjubK5Uc4P0iIhIcNVg==",
-      "dev": true,
-      "dependencies": {
-        "get-intrinsic": "^1.2.2"
-      },
-      "funding": {
-        "url": "https://github.com/sponsors/ljharb"
-      }
-    },
-    "node_modules/has-proto": {
-      "version": "1.0.1",
-      "resolved": "https://registry.npmjs.org/has-proto/-/has-proto-1.0.1.tgz",
-      "integrity": "sha512-7qE+iP+O+bgF9clE5+UoBFzE65mlBiVj3tKCrlNQ0Ogwm0BjpT/gK4SlLYDMybDh5I3TCTKnPPa0oMG7JDYrhg==",
-      "dev": true,
-      "engines": {
-        "node": ">= 0.4"
-      },
-      "funding": {
-        "url": "https://github.com/sponsors/ljharb"
-      }
-    },
     "node_modules/has-symbols": {
-      "version": "1.0.3",
-      "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.0.3.tgz",
-      "integrity": "sha512-l3LCuF6MgDNwTDKkdYGEihYjt5pRPbEg46rtlmnSPlUbgmB8LOIrKJbYYFBSbnPaJexMKtiPO8hmeRjRz2Td+A==",
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.1.0.tgz",
+      "integrity": "sha512-1cDNdwJ2Jaohmb3sg4OmKaMBwuC48sYni5HUw2DvsC8LjGTLK9h+eb1X6RyuOHe4hT0ULCW68iomhjUoKUqlPQ==",
       "dev": true,
+      "license": "MIT",
       "engines": {
         "node": ">= 0.4"
       },
@@ -838,10 +1389,11 @@
       }
     },
     "node_modules/hasown": {
-      "version": "2.0.0",
-      "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.0.tgz",
-      "integrity": "sha512-vUptKVTpIJhcczKBbgnS+RtcuYMB8+oNzPK2/Hp3hanz8JmpATdmmgLgSaadVREkDm+e2giHwY3ZRkyjSIDDFA==",
+      "version": "2.0.2",
+      "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.2.tgz",
+      "integrity": "sha512-0hJU9SCPvmMzIBdZFqNPXWa6dqh7WdH0cII9y+CyS8rG3nL48Bclra9HmKhVVUHyPWNH5Y7xDwAB7bfgSjkUMQ==",
       "dev": true,
+      "license": "MIT",
       "dependencies": {
         "function-bind": "^1.1.2"
       },
@@ -880,6 +1432,34 @@
         "entities": "^4.4.0"
       }
     },
+    "node_modules/http-proxy-agent": {
+      "version": "7.0.2",
+      "resolved": "https://registry.npmjs.org/http-proxy-agent/-/http-proxy-agent-7.0.2.tgz",
+      "integrity": "sha512-T1gkAiYYDWYx3V5Bmyu7HcfcvL7mUrTWiM6yOfa3PIphViJ/gFPbvidQ+veqSOHci/PxBcDabeUNCzpOODJZig==",
+      "dev": true,
+      "license": "MIT",
+      "dependencies": {
+        "agent-base": "^7.1.0",
+        "debug": "^4.3.4"
+      },
+      "engines": {
+        "node": ">= 14"
+      }
+    },
+    "node_modules/https-proxy-agent": {
+      "version": "7.0.6",
+      "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-7.0.6.tgz",
+      "integrity": "sha512-vK9P5/iUfdl95AI+JVyUuIcVtd4ofvtrOr3HNtM2yxC9bnMbEdp3x01OhQNnjb8IJYi38VlTE3mBXwcfvywuSw==",
+      "dev": true,
+      "license": "MIT",
+      "dependencies": {
+        "agent-base": "^7.1.2",
+        "debug": "4"
+      },
+      "engines": {
+        "node": ">= 14"
+      }
+    },
     "node_modules/ieee754": {
       "version": "1.2.1",
       "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.2.1.tgz",
@@ -901,21 +1481,12 @@
       ],
       "optional": true
     },
-    "node_modules/inflight": {
-      "version": "1.0.6",
-      "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz",
-      "integrity": "sha512-k92I/b08q4wvFscXCLvqfsHCrjrF7yiXsQuIVvVE7N82W3+aqpzuUdBbfhWcy/FZR3/4IgflMgKLOsvPDrGCJA==",
-      "dev": true,
-      "dependencies": {
-        "once": "^1.3.0",
-        "wrappy": "1"
-      }
-    },
     "node_modules/inherits": {
       "version": "2.0.4",
       "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz",
       "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==",
-      "dev": true
+      "dev": true,
+      "optional": true
     },
     "node_modules/ini": {
       "version": "1.3.8",
@@ -924,22 +1495,86 @@
       "dev": true,
       "optional": true
     },
+    "node_modules/is-docker": {
+      "version": "2.2.1",
+      "resolved": "https://registry.npmjs.org/is-docker/-/is-docker-2.2.1.tgz",
+      "integrity": "sha512-F+i2BKsFrH66iaUFc0woD8sLy8getkwTwtOBjvs56Cx4CgJDeKQeqfz8wAYiSb8JOprWhHH5p77PbmYCvvUuXQ==",
+      "dev": true,
+      "license": "MIT",
+      "bin": {
+        "is-docker": "cli.js"
+      },
+      "engines": {
+        "node": ">=8"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/sindresorhus"
+      }
+    },
+    "node_modules/is-fullwidth-code-point": {
+      "version": "3.0.0",
+      "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz",
+      "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==",
+      "dev": true,
+      "license": "MIT",
+      "engines": {
+        "node": ">=8"
+      }
+    },
+    "node_modules/is-wsl": {
+      "version": "2.2.0",
+      "resolved": "https://registry.npmjs.org/is-wsl/-/is-wsl-2.2.0.tgz",
+      "integrity": "sha512-fKzAra0rGJUUBwGBgNkHZuToZcn+TtXHpeCgmkMJMMYx1sQDYaCSyjJBSCa2nH1DGm7s3n1oBnohoVTBaN7Lww==",
+      "dev": true,
+      "license": "MIT",
+      "dependencies": {
+        "is-docker": "^2.0.0"
+      },
+      "engines": {
+        "node": ">=8"
+      }
+    },
+    "node_modules/isexe": {
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz",
+      "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==",
+      "dev": true,
+      "license": "ISC"
+    },
+    "node_modules/jackspeak": {
+      "version": "4.0.2",
+      "resolved": "https://registry.npmjs.org/jackspeak/-/jackspeak-4.0.2.tgz",
+      "integrity": "sha512-bZsjR/iRjl1Nk1UkjGpAzLNfQtzuijhn2g+pbZb98HQ1Gk8vM9hfbxeMBP+M2/UUdwj0RqGG3mlvk2MsAqwvEw==",
+      "dev": true,
+      "license": "BlueOak-1.0.0",
+      "dependencies": {
+        "@isaacs/cliui": "^8.0.2"
+      },
+      "engines": {
+        "node": "20 || >=22"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/isaacs"
+      }
+    },
     "node_modules/js-tokens": {
       "version": "4.0.0",
       "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz",
       "integrity": "sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ==",
-      "dev": true
+      "dev": true,
+      "license": "MIT"
     },
     "node_modules/jsesc": {
-      "version": "2.5.2",
-      "resolved": "https://registry.npmjs.org/jsesc/-/jsesc-2.5.2.tgz",
-      "integrity": "sha512-OYu7XEzjkCQ3C5Ps3QIZsQfNpqoJyZZA99wd9aWd05NCtC5pWOkShK2mkL6HXQR6/Cy2lbNdPlZBpuQHXE63gA==",
+      "version": "3.1.0",
+      "resolved": "https://registry.npmjs.org/jsesc/-/jsesc-3.1.0.tgz",
+      "integrity": "sha512-/sM3dO2FOzXjKQhJuo0Q173wf2KOo8t4I8vHy6lF9poUp7bKT0/NHE8fPX23PwfhnykfqnC2xRxOnVw5XuGIaA==",
       "dev": true,
+      "license": "MIT",
       "bin": {
         "jsesc": "bin/jsesc"
       },
       "engines": {
-        "node": ">=4"
+        "node": ">=6"
       }
     },
     "node_modules/jsonc-parser": {
@@ -948,6 +1583,75 @@
       "integrity": "sha512-gfFQZrcTc8CnKXp6Y4/CBT3fTc0OVuDofpre4aEeEpSBPV5X5v4+Vmx+8snU7RLPrNHPKSgLxGo9YuQzz20o+w==",
       "dev": true
     },
+    "node_modules/jsonwebtoken": {
+      "version": "9.0.2",
+      "resolved": "https://registry.npmjs.org/jsonwebtoken/-/jsonwebtoken-9.0.2.tgz",
+      "integrity": "sha512-PRp66vJ865SSqOlgqS8hujT5U4AOgMfhrwYIuIhfKaoSCZcirrmASQr8CX7cUg+RMih+hgznrjp99o+W4pJLHQ==",
+      "dev": true,
+      "license": "MIT",
+      "dependencies": {
+        "jws": "^3.2.2",
+        "lodash.includes": "^4.3.0",
+        "lodash.isboolean": "^3.0.3",
+        "lodash.isinteger": "^4.0.4",
+        "lodash.isnumber": "^3.0.3",
+        "lodash.isplainobject": "^4.0.6",
+        "lodash.isstring": "^4.0.1",
+        "lodash.once": "^4.0.0",
+        "ms": "^2.1.1",
+        "semver": "^7.5.4"
+      },
+      "engines": {
+        "node": ">=12",
+        "npm": ">=6"
+      }
+    },
+    "node_modules/jsonwebtoken/node_modules/jwa": {
+      "version": "1.4.1",
+      "resolved": "https://registry.npmjs.org/jwa/-/jwa-1.4.1.tgz",
+      "integrity": "sha512-qiLX/xhEEFKUAJ6FiBMbes3w9ATzyk5W7Hvzpa/SLYdxNtng+gcurvrI7TbACjIXlsJyr05/S1oUhZrc63evQA==",
+      "dev": true,
+      "license": "MIT",
+      "dependencies": {
+        "buffer-equal-constant-time": "1.0.1",
+        "ecdsa-sig-formatter": "1.0.11",
+        "safe-buffer": "^5.0.1"
+      }
+    },
+    "node_modules/jsonwebtoken/node_modules/jws": {
+      "version": "3.2.2",
+      "resolved": "https://registry.npmjs.org/jws/-/jws-3.2.2.tgz",
+      "integrity": "sha512-YHlZCB6lMTllWDtSPHz/ZXTsi8S00usEV6v1tjq8tOUZzw7DpSDWVXjXDre6ed1w/pd495ODpHZYSdkRTsa0HA==",
+      "dev": true,
+      "license": "MIT",
+      "dependencies": {
+        "jwa": "^1.4.1",
+        "safe-buffer": "^5.0.1"
+      }
+    },
+    "node_modules/jwa": {
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/jwa/-/jwa-2.0.0.tgz",
+      "integrity": "sha512-jrZ2Qx916EA+fq9cEAeCROWPTfCwi1IVHqT2tapuqLEVVDKFDENFw1oL+MwrTvH6msKxsd1YTDVw6uKEcsrLEA==",
+      "dev": true,
+      "license": "MIT",
+      "dependencies": {
+        "buffer-equal-constant-time": "1.0.1",
+        "ecdsa-sig-formatter": "1.0.11",
+        "safe-buffer": "^5.0.1"
+      }
+    },
+    "node_modules/jws": {
+      "version": "4.0.0",
+      "resolved": "https://registry.npmjs.org/jws/-/jws-4.0.0.tgz",
+      "integrity": "sha512-KDncfTmOZoOMTFG4mBlG0qUIOlc03fmzH+ru6RgYVZhPkyiy/92Owlt/8UEN+a4TXR1FQetfIpJE8ApdvdVxTg==",
+      "dev": true,
+      "license": "MIT",
+      "dependencies": {
+        "jwa": "^2.0.0",
+        "safe-buffer": "^5.0.1"
+      }
+    },
     "node_modules/keytar": {
       "version": "7.9.0",
       "resolved": "https://registry.npmjs.org/keytar/-/keytar-7.9.0.tgz",
@@ -970,14 +1674,64 @@
       }
     },
     "node_modules/linkify-it": {
-      "version": "3.0.3",
-      "resolved": "https://registry.npmjs.org/linkify-it/-/linkify-it-3.0.3.tgz",
-      "integrity": "sha512-ynTsyrFSdE5oZ/O9GEf00kPngmOfVwazR5GKDq6EYfhlpFug3J2zybX56a2PRRpc9P+FuSoGNAwjlbDs9jJBPQ==",
+      "version": "5.0.0",
+      "resolved": "https://registry.npmjs.org/linkify-it/-/linkify-it-5.0.0.tgz",
+      "integrity": "sha512-5aHCbzQRADcdP+ATqnDuhhJ/MRIqDkZX5pyjFHRRysS8vZ5AbqGEoFIb6pYHPZ+L/OC2Lc+xT8uHVVR5CAK/wQ==",
       "dev": true,
+      "license": "MIT",
       "dependencies": {
-        "uc.micro": "^1.0.1"
+        "uc.micro": "^2.0.0"
       }
     },
+    "node_modules/lodash.includes": {
+      "version": "4.3.0",
+      "resolved": "https://registry.npmjs.org/lodash.includes/-/lodash.includes-4.3.0.tgz",
+      "integrity": "sha512-W3Bx6mdkRTGtlJISOvVD/lbqjTlPPUDTMnlXZFnVwi9NKJ6tiAk6LVdlhZMm17VZisqhKcgzpO5Wz91PCt5b0w==",
+      "dev": true,
+      "license": "MIT"
+    },
+    "node_modules/lodash.isboolean": {
+      "version": "3.0.3",
+      "resolved": "https://registry.npmjs.org/lodash.isboolean/-/lodash.isboolean-3.0.3.tgz",
+      "integrity": "sha512-Bz5mupy2SVbPHURB98VAcw+aHh4vRV5IPNhILUCsOzRmsTmSQ17jIuqopAentWoehktxGd9e/hbIXq980/1QJg==",
+      "dev": true,
+      "license": "MIT"
+    },
+    "node_modules/lodash.isinteger": {
+      "version": "4.0.4",
+      "resolved": "https://registry.npmjs.org/lodash.isinteger/-/lodash.isinteger-4.0.4.tgz",
+      "integrity": "sha512-DBwtEWN2caHQ9/imiNeEA5ys1JoRtRfY3d7V9wkqtbycnAmTvRRmbHKDV4a0EYc678/dia0jrte4tjYwVBaZUA==",
+      "dev": true,
+      "license": "MIT"
+    },
+    "node_modules/lodash.isnumber": {
+      "version": "3.0.3",
+      "resolved": "https://registry.npmjs.org/lodash.isnumber/-/lodash.isnumber-3.0.3.tgz",
+      "integrity": "sha512-QYqzpfwO3/CWf3XP+Z+tkQsfaLL/EnUlXWVkIk5FUPc4sBdTehEqZONuyRt2P67PXAk+NXmTBcc97zw9t1FQrw==",
+      "dev": true,
+      "license": "MIT"
+    },
+    "node_modules/lodash.isplainobject": {
+      "version": "4.0.6",
+      "resolved": "https://registry.npmjs.org/lodash.isplainobject/-/lodash.isplainobject-4.0.6.tgz",
+      "integrity": "sha512-oSXzaWypCMHkPC3NvBEaPHf0KsA5mvPrOPgQWDsbg8n7orZ290M0BmC/jgRZ4vcJ6DTAhjrsSYgdsW/F+MFOBA==",
+      "dev": true,
+      "license": "MIT"
+    },
+    "node_modules/lodash.isstring": {
+      "version": "4.0.1",
+      "resolved": "https://registry.npmjs.org/lodash.isstring/-/lodash.isstring-4.0.1.tgz",
+      "integrity": "sha512-0wJxfxH1wgO3GrbuP+dTTk7op+6L41QCXbGINEmD+ny/G/eCqGzxyCsh7159S+mgDDcoarnBw6PC1PS5+wUGgw==",
+      "dev": true,
+      "license": "MIT"
+    },
+    "node_modules/lodash.once": {
+      "version": "4.1.1",
+      "resolved": "https://registry.npmjs.org/lodash.once/-/lodash.once-4.1.1.tgz",
+      "integrity": "sha512-Sb487aTOCr9drQVL8pIxOzVhafOjZN9UU54hiN8PU3uAiSV7lx1yYNpbNmex2PK6dSJoNTSJUUswT651yww3Mg==",
+      "dev": true,
+      "license": "MIT"
+    },
     "node_modules/lru-cache": {
       "version": "6.0.0",
       "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-6.0.0.tgz",
@@ -991,35 +1745,39 @@
       }
     },
     "node_modules/markdown-it": {
-      "version": "12.3.2",
-      "resolved": "https://registry.npmjs.org/markdown-it/-/markdown-it-12.3.2.tgz",
-      "integrity": "sha512-TchMembfxfNVpHkbtriWltGWc+m3xszaRD0CZup7GFFhzIgQqxIfn3eGj1yZpfuflzPvfkt611B2Q/Bsk1YnGg==",
+      "version": "14.1.0",
+      "resolved": "https://registry.npmjs.org/markdown-it/-/markdown-it-14.1.0.tgz",
+      "integrity": "sha512-a54IwgWPaeBCAAsv13YgmALOF1elABB08FxO9i+r4VFk5Vl4pKokRPeX8u5TCgSsPi6ec1otfLjdOpVcgbpshg==",
       "dev": true,
+      "license": "MIT",
       "dependencies": {
         "argparse": "^2.0.1",
-        "entities": "~2.1.0",
-        "linkify-it": "^3.0.1",
-        "mdurl": "^1.0.1",
-        "uc.micro": "^1.0.5"
+        "entities": "^4.4.0",
+        "linkify-it": "^5.0.0",
+        "mdurl": "^2.0.0",
+        "punycode.js": "^2.3.1",
+        "uc.micro": "^2.1.0"
       },
       "bin": {
-        "markdown-it": "bin/markdown-it.js"
+        "markdown-it": "bin/markdown-it.mjs"
       }
     },
-    "node_modules/markdown-it/node_modules/entities": {
-      "version": "2.1.0",
-      "resolved": "https://registry.npmjs.org/entities/-/entities-2.1.0.tgz",
-      "integrity": "sha512-hCx1oky9PFrJ611mf0ifBLBRW8lUUVRlFolb5gWRfIELabBlbp9xZvrqZLZAs+NxFnbfQoeGd8wDkygjg7U85w==",
+    "node_modules/math-intrinsics": {
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/math-intrinsics/-/math-intrinsics-1.1.0.tgz",
+      "integrity": "sha512-/IXtbwEk5HTPyEwyKX6hGkYXxM9nbj64B+ilVJnC/R6B0pH5G4V3b0pVbL7DBj4tkhBAppbQUlf6F6Xl9LHu1g==",
       "dev": true,
-      "funding": {
-        "url": "https://github.com/fb55/entities?sponsor=1"
+      "license": "MIT",
+      "engines": {
+        "node": ">= 0.4"
       }
     },
     "node_modules/mdurl": {
-      "version": "1.0.1",
-      "resolved": "https://registry.npmjs.org/mdurl/-/mdurl-1.0.1.tgz",
-      "integrity": "sha512-/sKlQJCBYVY9Ers9hqzKou4H6V5UWc/M59TH2dvkt+84itfnq7uFOMLpOiOS4ujvHP4etln18fmIxA5R5fll0g==",
-      "dev": true
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/mdurl/-/mdurl-2.0.0.tgz",
+      "integrity": "sha512-Lf+9+2r+Tdp5wXDXC4PcIBjTDtq4UKjCPMQhKIuzpJNW0b96kVqSwW0bT7FhRSfmAiFYgP+SCRvdrDozfh0U5w==",
+      "dev": true,
+      "license": "MIT"
     },
     "node_modules/mime": {
       "version": "1.6.0",
@@ -1033,6 +1791,29 @@
         "node": ">=4"
       }
     },
+    "node_modules/mime-db": {
+      "version": "1.52.0",
+      "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz",
+      "integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==",
+      "dev": true,
+      "license": "MIT",
+      "engines": {
+        "node": ">= 0.6"
+      }
+    },
+    "node_modules/mime-types": {
+      "version": "2.1.35",
+      "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz",
+      "integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==",
+      "dev": true,
+      "license": "MIT",
+      "dependencies": {
+        "mime-db": "1.52.0"
+      },
+      "engines": {
+        "node": ">= 0.6"
+      }
+    },
     "node_modules/mimic-response": {
       "version": "3.1.0",
       "resolved": "https://registry.npmjs.org/mimic-response/-/mimic-response-3.1.0.tgz",
@@ -1051,6 +1832,7 @@
       "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz",
       "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==",
       "dev": true,
+      "license": "ISC",
       "dependencies": {
         "brace-expansion": "^1.1.7"
       },
@@ -1068,6 +1850,16 @@
         "url": "https://github.com/sponsors/ljharb"
       }
     },
+    "node_modules/minipass": {
+      "version": "7.1.2",
+      "resolved": "https://registry.npmjs.org/minipass/-/minipass-7.1.2.tgz",
+      "integrity": "sha512-qOOzS1cBTWYF4BH8fVePDBOO9iptMnGUEZwNc/cMWnTV2nVLZ7VoNWEPHkYczZA0pdoA7dl6e7FL659nX9S2aw==",
+      "dev": true,
+      "license": "ISC",
+      "engines": {
+        "node": ">=16 || 14 >=14.17"
+      }
+    },
     "node_modules/mkdirp-classic": {
       "version": "0.5.3",
       "resolved": "https://registry.npmjs.org/mkdirp-classic/-/mkdirp-classic-0.5.3.tgz",
@@ -1076,10 +1868,11 @@
       "optional": true
     },
     "node_modules/ms": {
-      "version": "2.1.2",
-      "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz",
-      "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==",
-      "dev": true
+      "version": "2.1.3",
+      "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz",
+      "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==",
+      "dev": true,
+      "license": "MIT"
     },
     "node_modules/mute-stream": {
       "version": "0.0.8",
@@ -1127,10 +1920,14 @@
       }
     },
     "node_modules/object-inspect": {
-      "version": "1.13.1",
-      "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.13.1.tgz",
-      "integrity": "sha512-5qoj1RUiKOMsCCNLV1CBiPYE10sziTsnmNxkAI/rZhiD63CF7IqdFGC/XzjWjpSgLf0LxXX3bDFIh0E18f6UhQ==",
+      "version": "1.13.4",
+      "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.13.4.tgz",
+      "integrity": "sha512-W67iLl4J2EXEGTbfeHCffrjDfitvLANg0UlX3wFUUSTx92KXRFegMHUVgSqE+wvhAbi4WqjGg9czysTV2Epbew==",
       "dev": true,
+      "license": "MIT",
+      "engines": {
+        "node": ">= 0.4"
+      },
       "funding": {
         "url": "https://github.com/sponsors/ljharb"
       }
@@ -1140,10 +1937,36 @@
       "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz",
       "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==",
       "dev": true,
+      "optional": true,
       "dependencies": {
         "wrappy": "1"
       }
     },
+    "node_modules/open": {
+      "version": "8.4.2",
+      "resolved": "https://registry.npmjs.org/open/-/open-8.4.2.tgz",
+      "integrity": "sha512-7x81NCL719oNbsq/3mh+hVrAWmFuEYUqrq/Iw3kUzH8ReypT9QQ0BLoJS7/G9k6N81XjW4qHWtjWwe/9eLy1EQ==",
+      "dev": true,
+      "license": "MIT",
+      "dependencies": {
+        "define-lazy-prop": "^2.0.0",
+        "is-docker": "^2.1.1",
+        "is-wsl": "^2.2.0"
+      },
+      "engines": {
+        "node": ">=12"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/sindresorhus"
+      }
+    },
+    "node_modules/package-json-from-dist": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/package-json-from-dist/-/package-json-from-dist-1.0.1.tgz",
+      "integrity": "sha512-UEZIS3/by4OC8vL3P2dTXRETpebLI2NiI5vIrjaD/5UtrkFX/tNbwjTSRAGC/+7CAo2pIcBaRgWmcBBHcsaCIw==",
+      "dev": true,
+      "license": "BlueOak-1.0.0"
+    },
     "node_modules/parse-semver": {
       "version": "1.1.1",
       "resolved": "https://registry.npmjs.org/parse-semver/-/parse-semver-1.1.1.tgz",
@@ -1187,13 +2010,41 @@
         "url": "https://github.com/inikulin/parse5?sponsor=1"
       }
     },
-    "node_modules/path-is-absolute": {
-      "version": "1.0.1",
-      "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz",
-      "integrity": "sha512-AVbw3UJ2e9bq64vSaS9Am0fje1Pa8pbGqTTsmXfaIiMpnr5DlDhfJOuLj9Sf95ZPVDAUerDfEk88MPmPe7UCQg==",
+    "node_modules/path-key": {
+      "version": "3.1.1",
+      "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz",
+      "integrity": "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==",
       "dev": true,
+      "license": "MIT",
       "engines": {
-        "node": ">=0.10.0"
+        "node": ">=8"
+      }
+    },
+    "node_modules/path-scurry": {
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/path-scurry/-/path-scurry-2.0.0.tgz",
+      "integrity": "sha512-ypGJsmGtdXUOeM5u93TyeIEfEhM6s+ljAhrk5vAvSx8uyY/02OvrZnA0YNGUrPXfpJMgI1ODd3nwz8Npx4O4cg==",
+      "dev": true,
+      "license": "BlueOak-1.0.0",
+      "dependencies": {
+        "lru-cache": "^11.0.0",
+        "minipass": "^7.1.2"
+      },
+      "engines": {
+        "node": "20 || >=22"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/isaacs"
+      }
+    },
+    "node_modules/path-scurry/node_modules/lru-cache": {
+      "version": "11.0.2",
+      "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-11.0.2.tgz",
+      "integrity": "sha512-123qHRfJBmo2jXDbo/a5YOQrJoHF/GNQTLzQ5+IdK5pWpceK17yRc6ozlWd25FxvGKQbIUs91fDFkXmDHTKcyA==",
+      "dev": true,
+      "license": "ISC",
+      "engines": {
+        "node": "20 || >=22"
       }
     },
     "node_modules/pend": {
@@ -1202,6 +2053,13 @@
       "integrity": "sha512-F3asv42UuXchdzt+xXqfW1OGlVBe+mxa2mqI0pg5yAHZPvFmY3Y6drSf/GQ1A86WgWEN9Kzh/WrgKa6iGcHXLg==",
       "dev": true
     },
+    "node_modules/picocolors": {
+      "version": "1.1.1",
+      "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.1.1.tgz",
+      "integrity": "sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA==",
+      "dev": true,
+      "license": "ISC"
+    },
     "node_modules/prebuild-install": {
       "version": "7.1.1",
       "resolved": "https://registry.npmjs.org/prebuild-install/-/prebuild-install-7.1.1.tgz",
@@ -1230,10 +2088,11 @@
       }
     },
     "node_modules/prettier": {
-      "version": "3.1.1",
-      "resolved": "https://registry.npmjs.org/prettier/-/prettier-3.1.1.tgz",
-      "integrity": "sha512-22UbSzg8luF4UuZtzgiUOfcGM8s4tjBv6dJRT7j275NXsy2jb4aJa4NNveul5x4eqlF1wuhuR2RElK71RvmVaw==",
+      "version": "3.4.2",
+      "resolved": "https://registry.npmjs.org/prettier/-/prettier-3.4.2.tgz",
+      "integrity": "sha512-e9MewbtFo+Fevyuxn/4rrcDAaq0IYxPGLvObpQjiZBMAzB9IGmzlnG9RZy3FFas+eBMu2vA0CszMeduow5dIuQ==",
       "dev": true,
+      "license": "MIT",
       "bin": {
         "prettier": "bin/prettier.cjs"
       },
@@ -1245,13 +2104,15 @@
       }
     },
     "node_modules/prettier-plugin-curly": {
-      "version": "0.1.3",
-      "resolved": "https://registry.npmjs.org/prettier-plugin-curly/-/prettier-plugin-curly-0.1.3.tgz",
-      "integrity": "sha512-NYr2BPex/0fFwDbiZZr91kfgBko1tmaorLOrVAkT5rN91mIYYJRiWabRxWGFqzRSO7J0eoEcxakY9NWvJWAh4w==",
+      "version": "0.3.1",
+      "resolved": "https://registry.npmjs.org/prettier-plugin-curly/-/prettier-plugin-curly-0.3.1.tgz",
+      "integrity": "sha512-GU3JJ4DB+j2gNA+8SxiyuAOuqalFu+JrX9JMNChTEwvgLCs+XXu2WJ1NbP1uK57Cbz04E2OhrkDpQFLMH5MsXg==",
       "dev": true,
+      "license": "MIT",
       "dependencies": {
-        "@babel/parser": "^7.22.5",
-        "@babel/traverse": "^7.22.5"
+        "@babel/generator": "^7.25.7",
+        "@babel/parser": "^7.25.8",
+        "@babel/traverse": "^7.25.7"
       },
       "engines": {
         "node": ">=18"
@@ -1271,13 +2132,24 @@
         "once": "^1.3.1"
       }
     },
+    "node_modules/punycode.js": {
+      "version": "2.3.1",
+      "resolved": "https://registry.npmjs.org/punycode.js/-/punycode.js-2.3.1.tgz",
+      "integrity": "sha512-uxFIHU0YlHYhDQtV4R9J6a52SLx28BCjT+4ieh7IGbgwVJWO+km431c4yRlREUAsAmt/uMjQUyQHNEPf0M39CA==",
+      "dev": true,
+      "license": "MIT",
+      "engines": {
+        "node": ">=6"
+      }
+    },
     "node_modules/qs": {
-      "version": "6.11.2",
-      "resolved": "https://registry.npmjs.org/qs/-/qs-6.11.2.tgz",
-      "integrity": "sha512-tDNIz22aBzCDxLtVH++VnTfzxlfeK5CbqohpSqpJgj1Wg/cQbStNAz3NuqCs5vV+pjBsK4x4pN9HlVh7rcYRiA==",
+      "version": "6.14.0",
+      "resolved": "https://registry.npmjs.org/qs/-/qs-6.14.0.tgz",
+      "integrity": "sha512-YWWTjgABSKcvs/nWBi9PycY/JiPJqOD4JA6o9Sej2AtvSGarXxKC3OQSk4pAarbdQlKAh5D4FCQkJNkW+GAn3w==",
       "dev": true,
+      "license": "BSD-3-Clause",
       "dependencies": {
-        "side-channel": "^1.0.4"
+        "side-channel": "^1.1.0"
       },
       "engines": {
         "node": ">=0.6"
@@ -1329,21 +2201,6 @@
         "node": ">= 6"
       }
     },
-    "node_modules/rimraf": {
-      "version": "3.0.2",
-      "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-3.0.2.tgz",
-      "integrity": "sha512-JZkJMZkAGFFPP2YqXZXPbMlMBgsxzE8ILs4lMIX/2o0L9UBw9O/Y3o6wFw/i9YLapcUJWwqbi3kdxIPdC62TIA==",
-      "dev": true,
-      "dependencies": {
-        "glob": "^7.1.3"
-      },
-      "bin": {
-        "rimraf": "bin.js"
-      },
-      "funding": {
-        "url": "https://github.com/sponsors/isaacs"
-      }
-    },
     "node_modules/safe-buffer": {
       "version": "5.2.1",
       "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz",
@@ -1362,8 +2219,7 @@
           "type": "consulting",
           "url": "https://feross.org/support"
         }
-      ],
-      "optional": true
+      ]
     },
     "node_modules/sax": {
       "version": "1.3.0",
@@ -1386,35 +2242,118 @@
         "node": ">=10"
       }
     },
-    "node_modules/set-function-length": {
-      "version": "1.1.1",
-      "resolved": "https://registry.npmjs.org/set-function-length/-/set-function-length-1.1.1.tgz",
-      "integrity": "sha512-VoaqjbBJKiWtg4yRcKBQ7g7wnGnLV3M8oLvVWwOk2PdYY6PEFegR1vezXR0tw6fZGF9csVakIRjrJiy2veSBFQ==",
+    "node_modules/shebang-command": {
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz",
+      "integrity": "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==",
       "dev": true,
+      "license": "MIT",
       "dependencies": {
-        "define-data-property": "^1.1.1",
-        "get-intrinsic": "^1.2.1",
-        "gopd": "^1.0.1",
-        "has-property-descriptors": "^1.0.0"
+        "shebang-regex": "^3.0.0"
       },
       "engines": {
-        "node": ">= 0.4"
+        "node": ">=8"
+      }
+    },
+    "node_modules/shebang-regex": {
+      "version": "3.0.0",
+      "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-3.0.0.tgz",
+      "integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==",
+      "dev": true,
+      "license": "MIT",
+      "engines": {
+        "node": ">=8"
       }
     },
     "node_modules/side-channel": {
-      "version": "1.0.4",
-      "resolved": "https://registry.npmjs.org/side-channel/-/side-channel-1.0.4.tgz",
-      "integrity": "sha512-q5XPytqFEIKHkGdiMIrY10mvLRvnQh42/+GoBlFW3b2LXLE2xxJpZFdm94we0BaoV3RwJyGqg5wS7epxTv0Zvw==",
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/side-channel/-/side-channel-1.1.0.tgz",
+      "integrity": "sha512-ZX99e6tRweoUXqR+VBrslhda51Nh5MTQwou5tnUDgbtyM0dBgmhEDtWGP/xbKn6hqfPRHujUNwz5fy/wbbhnpw==",
+      "dev": true,
+      "license": "MIT",
+      "dependencies": {
+        "es-errors": "^1.3.0",
+        "object-inspect": "^1.13.3",
+        "side-channel-list": "^1.0.0",
+        "side-channel-map": "^1.0.1",
+        "side-channel-weakmap": "^1.0.2"
+      },
+      "engines": {
+        "node": ">= 0.4"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/ljharb"
+      }
+    },
+    "node_modules/side-channel-list": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/side-channel-list/-/side-channel-list-1.0.0.tgz",
+      "integrity": "sha512-FCLHtRD/gnpCiCHEiJLOwdmFP+wzCmDEkc9y7NsYxeF4u7Btsn1ZuwgwJGxImImHicJArLP4R0yX4c2KCrMrTA==",
       "dev": true,
+      "license": "MIT",
       "dependencies": {
-        "call-bind": "^1.0.0",
-        "get-intrinsic": "^1.0.2",
-        "object-inspect": "^1.9.0"
+        "es-errors": "^1.3.0",
+        "object-inspect": "^1.13.3"
+      },
+      "engines": {
+        "node": ">= 0.4"
       },
       "funding": {
         "url": "https://github.com/sponsors/ljharb"
       }
     },
+    "node_modules/side-channel-map": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/side-channel-map/-/side-channel-map-1.0.1.tgz",
+      "integrity": "sha512-VCjCNfgMsby3tTdo02nbjtM/ewra6jPHmpThenkTYh8pG9ucZ/1P8So4u4FGBek/BjpOVsDCMoLA/iuBKIFXRA==",
+      "dev": true,
+      "license": "MIT",
+      "dependencies": {
+        "call-bound": "^1.0.2",
+        "es-errors": "^1.3.0",
+        "get-intrinsic": "^1.2.5",
+        "object-inspect": "^1.13.3"
+      },
+      "engines": {
+        "node": ">= 0.4"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/ljharb"
+      }
+    },
+    "node_modules/side-channel-weakmap": {
+      "version": "1.0.2",
+      "resolved": "https://registry.npmjs.org/side-channel-weakmap/-/side-channel-weakmap-1.0.2.tgz",
+      "integrity": "sha512-WPS/HvHQTYnHisLo9McqBHOJk2FkHO/tlpvldyrnem4aeQp4hai3gythswg6p01oSoTl58rcpiFAjF2br2Ak2A==",
+      "dev": true,
+      "license": "MIT",
+      "dependencies": {
+        "call-bound": "^1.0.2",
+        "es-errors": "^1.3.0",
+        "get-intrinsic": "^1.2.5",
+        "object-inspect": "^1.13.3",
+        "side-channel-map": "^1.0.1"
+      },
+      "engines": {
+        "node": ">= 0.4"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/ljharb"
+      }
+    },
+    "node_modules/signal-exit": {
+      "version": "4.1.0",
+      "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-4.1.0.tgz",
+      "integrity": "sha512-bzyZ1e88w9O1iNJbKnOlvYTrWPDl46O1bG0D3XInv+9tkPrxrN8jUUTiFlDkkmKWgn1M6CfIA13SuGqOa9Korw==",
+      "dev": true,
+      "license": "ISC",
+      "engines": {
+        "node": ">=14"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/isaacs"
+      }
+    },
     "node_modules/simple-concat": {
       "version": "1.0.1",
       "resolved": "https://registry.npmjs.org/simple-concat/-/simple-concat-1.0.1.tgz",
@@ -1462,6 +2401,17 @@
         "simple-concat": "^1.0.0"
       }
     },
+    "node_modules/stoppable": {
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/stoppable/-/stoppable-1.1.0.tgz",
+      "integrity": "sha512-KXDYZ9dszj6bzvnEMRYvxgeTHU74QBFL54XKtP3nyMuJ81CFYtABZ3bAzL2EdFUaEwJOBOgENyFj3R7oTzDyyw==",
+      "dev": true,
+      "license": "MIT",
+      "engines": {
+        "node": ">=4",
+        "npm": ">=6"
+      }
+    },
     "node_modules/string_decoder": {
       "version": "1.3.0",
       "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz",
@@ -1472,6 +2422,110 @@
         "safe-buffer": "~5.2.0"
       }
     },
+    "node_modules/string-width": {
+      "version": "5.1.2",
+      "resolved": "https://registry.npmjs.org/string-width/-/string-width-5.1.2.tgz",
+      "integrity": "sha512-HnLOCR3vjcY8beoNLtcjZ5/nxn2afmME6lhrDrebokqMap+XbeW8n9TXpPDOqdGK5qcI3oT0GKTW6wC7EMiVqA==",
+      "dev": true,
+      "license": "MIT",
+      "dependencies": {
+        "eastasianwidth": "^0.2.0",
+        "emoji-regex": "^9.2.2",
+        "strip-ansi": "^7.0.1"
+      },
+      "engines": {
+        "node": ">=12"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/sindresorhus"
+      }
+    },
+    "node_modules/string-width-cjs": {
+      "name": "string-width",
+      "version": "4.2.3",
+      "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz",
+      "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==",
+      "dev": true,
+      "license": "MIT",
+      "dependencies": {
+        "emoji-regex": "^8.0.0",
+        "is-fullwidth-code-point": "^3.0.0",
+        "strip-ansi": "^6.0.1"
+      },
+      "engines": {
+        "node": ">=8"
+      }
+    },
+    "node_modules/string-width-cjs/node_modules/ansi-regex": {
+      "version": "5.0.1",
+      "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz",
+      "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==",
+      "dev": true,
+      "license": "MIT",
+      "engines": {
+        "node": ">=8"
+      }
+    },
+    "node_modules/string-width-cjs/node_modules/emoji-regex": {
+      "version": "8.0.0",
+      "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz",
+      "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==",
+      "dev": true,
+      "license": "MIT"
+    },
+    "node_modules/string-width-cjs/node_modules/strip-ansi": {
+      "version": "6.0.1",
+      "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz",
+      "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==",
+      "dev": true,
+      "license": "MIT",
+      "dependencies": {
+        "ansi-regex": "^5.0.1"
+      },
+      "engines": {
+        "node": ">=8"
+      }
+    },
+    "node_modules/strip-ansi": {
+      "version": "7.1.0",
+      "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-7.1.0.tgz",
+      "integrity": "sha512-iq6eVVI64nQQTRYq2KtEg2d2uU7LElhTJwsH4YzIHZshxlgZms/wIc4VoDQTlG/IvVIrBKG06CrZnp0qv7hkcQ==",
+      "dev": true,
+      "license": "MIT",
+      "dependencies": {
+        "ansi-regex": "^6.0.1"
+      },
+      "engines": {
+        "node": ">=12"
+      },
+      "funding": {
+        "url": "https://github.com/chalk/strip-ansi?sponsor=1"
+      }
+    },
+    "node_modules/strip-ansi-cjs": {
+      "name": "strip-ansi",
+      "version": "6.0.1",
+      "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz",
+      "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==",
+      "dev": true,
+      "license": "MIT",
+      "dependencies": {
+        "ansi-regex": "^5.0.1"
+      },
+      "engines": {
+        "node": ">=8"
+      }
+    },
+    "node_modules/strip-ansi-cjs/node_modules/ansi-regex": {
+      "version": "5.0.1",
+      "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz",
+      "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==",
+      "dev": true,
+      "license": "MIT",
+      "engines": {
+        "node": ">=8"
+      }
+    },
     "node_modules/strip-json-comments": {
       "version": "2.0.1",
       "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-2.0.1.tgz",
@@ -1525,31 +2579,28 @@
       }
     },
     "node_modules/tmp": {
-      "version": "0.2.1",
-      "resolved": "https://registry.npmjs.org/tmp/-/tmp-0.2.1.tgz",
-      "integrity": "sha512-76SUhtfqR2Ijn+xllcI5P1oyannHNHByD80W1q447gU3mp9G9PSpGdWmjUOHRDPiHYacIk66W7ubDTuPF3BEtQ==",
+      "version": "0.2.3",
+      "resolved": "https://registry.npmjs.org/tmp/-/tmp-0.2.3.tgz",
+      "integrity": "sha512-nZD7m9iCPC5g0pYmcaxogYKggSfLsdxl8of3Q/oIbqCqLLIO9IAF0GWjX1z9NZRHPiXv8Wex4yDCaZsgEw0Y8w==",
       "dev": true,
-      "dependencies": {
-        "rimraf": "^3.0.0"
-      },
+      "license": "MIT",
       "engines": {
-        "node": ">=8.17.0"
+        "node": ">=14.14"
       }
     },
-    "node_modules/to-fast-properties": {
-      "version": "2.0.0",
-      "resolved": "https://registry.npmjs.org/to-fast-properties/-/to-fast-properties-2.0.0.tgz",
-      "integrity": "sha512-/OaKK0xYrs3DmxRYqL/yDc+FxFUVYhDlXMhRmv3z915w2HF1tnN1omB354j8VUGO/hbRzyD6Y3sA7v7GS/ceog==",
+    "node_modules/tslib": {
+      "version": "2.8.1",
+      "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz",
+      "integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==",
       "dev": true,
-      "engines": {
-        "node": ">=4"
-      }
+      "license": "0BSD"
     },
     "node_modules/tunnel": {
       "version": "0.0.6",
       "resolved": "https://registry.npmjs.org/tunnel/-/tunnel-0.0.6.tgz",
       "integrity": "sha512-1h/Lnq9yajKY2PEbBadPXj3VxsDDu844OnaAo52UVmIzIvwwtBPIuNvkjuzBlTWpfJyUbG3ez0KSBibQkj4ojg==",
       "dev": true,
+      "license": "MIT",
       "engines": {
         "node": ">=0.6.11 <=0.7.0 || >=0.7.3"
       }
@@ -1572,6 +2623,7 @@
       "resolved": "https://registry.npmjs.org/typed-rest-client/-/typed-rest-client-1.8.11.tgz",
       "integrity": "sha512-5UvfMpd1oelmUPRbbaVnq+rHP7ng2cE4qoQkQeAqxRL6PklkxsM0g32/HL0yfvruK6ojQ5x8EE+HF4YV6DtuCA==",
       "dev": true,
+      "license": "MIT",
       "dependencies": {
         "qs": "^6.9.1",
         "tunnel": "0.0.6",
@@ -1579,29 +2631,32 @@
       }
     },
     "node_modules/typescript": {
-      "version": "4.9.5",
-      "resolved": "https://registry.npmjs.org/typescript/-/typescript-4.9.5.tgz",
-      "integrity": "sha512-1FXk9E2Hm+QzZQ7z+McJiHL4NW1F2EzMu9Nq9i3zAaGqibafqYwCVU6WyWAuyQRRzOlxou8xZSyXLEN8oKj24g==",
+      "version": "5.7.3",
+      "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.7.3.tgz",
+      "integrity": "sha512-84MVSjMEHP+FQRPy3pX9sTVV/INIex71s9TL2Gm5FG/WG1SqXeKyZ0k7/blY/4FdOzI12CBy1vGc4og/eus0fw==",
       "dev": true,
+      "license": "Apache-2.0",
       "bin": {
         "tsc": "bin/tsc",
         "tsserver": "bin/tsserver"
       },
       "engines": {
-        "node": ">=4.2.0"
+        "node": ">=14.17"
       }
     },
     "node_modules/uc.micro": {
-      "version": "1.0.6",
-      "resolved": "https://registry.npmjs.org/uc.micro/-/uc.micro-1.0.6.tgz",
-      "integrity": "sha512-8Y75pvTYkLJW2hWQHXxoqRgV7qb9B+9vFEtidML+7koHUFapnVJAZ6cKs+Qjz5Aw3aZWHMC6u0wJE3At+nSGwA==",
-      "dev": true
+      "version": "2.1.0",
+      "resolved": "https://registry.npmjs.org/uc.micro/-/uc.micro-2.1.0.tgz",
+      "integrity": "sha512-ARDJmphmdvUk6Glw7y9DQ2bFkKBHwQHLi2lsaH6PPmz/Ka9sFOBsBluozhDltWmnv9u/cF6Rt87znRTPV+yp/A==",
+      "dev": true,
+      "license": "MIT"
     },
     "node_modules/underscore": {
-      "version": "1.13.6",
-      "resolved": "https://registry.npmjs.org/underscore/-/underscore-1.13.6.tgz",
-      "integrity": "sha512-+A5Sja4HP1M08MaXya7p5LvjuM7K6q/2EaC0+iovj/wOcMsTzMvDFbasi/oSapiwOlt252IqsKqPjCl7huKS0A==",
-      "dev": true
+      "version": "1.13.7",
+      "resolved": "https://registry.npmjs.org/underscore/-/underscore-1.13.7.tgz",
+      "integrity": "sha512-GMXzWtsc57XAtguZgaQViUOzs0KTkk8ojr3/xAxXLITqf/3EMwxC0inyETfDFjH/Krbhuep0HNbbjI9i/q3F3g==",
+      "dev": true,
+      "license": "MIT"
     },
     "node_modules/undici-types": {
       "version": "5.26.5",
@@ -1622,11 +2677,169 @@
       "dev": true,
       "optional": true
     },
+    "node_modules/uuid": {
+      "version": "8.3.2",
+      "resolved": "https://registry.npmjs.org/uuid/-/uuid-8.3.2.tgz",
+      "integrity": "sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==",
+      "dev": true,
+      "license": "MIT",
+      "bin": {
+        "uuid": "dist/bin/uuid"
+      }
+    },
+    "node_modules/which": {
+      "version": "2.0.2",
+      "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz",
+      "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==",
+      "dev": true,
+      "license": "ISC",
+      "dependencies": {
+        "isexe": "^2.0.0"
+      },
+      "bin": {
+        "node-which": "bin/node-which"
+      },
+      "engines": {
+        "node": ">= 8"
+      }
+    },
+    "node_modules/wrap-ansi": {
+      "version": "8.1.0",
+      "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-8.1.0.tgz",
+      "integrity": "sha512-si7QWI6zUMq56bESFvagtmzMdGOtoxfR+Sez11Mobfc7tm+VkUckk9bW2UeffTGVUbOksxmSw0AA2gs8g71NCQ==",
+      "dev": true,
+      "license": "MIT",
+      "dependencies": {
+        "ansi-styles": "^6.1.0",
+        "string-width": "^5.0.1",
+        "strip-ansi": "^7.0.1"
+      },
+      "engines": {
+        "node": ">=12"
+      },
+      "funding": {
+        "url": "https://github.com/chalk/wrap-ansi?sponsor=1"
+      }
+    },
+    "node_modules/wrap-ansi-cjs": {
+      "name": "wrap-ansi",
+      "version": "7.0.0",
+      "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz",
+      "integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==",
+      "dev": true,
+      "license": "MIT",
+      "dependencies": {
+        "ansi-styles": "^4.0.0",
+        "string-width": "^4.1.0",
+        "strip-ansi": "^6.0.0"
+      },
+      "engines": {
+        "node": ">=10"
+      },
+      "funding": {
+        "url": "https://github.com/chalk/wrap-ansi?sponsor=1"
+      }
+    },
+    "node_modules/wrap-ansi-cjs/node_modules/ansi-regex": {
+      "version": "5.0.1",
+      "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz",
+      "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==",
+      "dev": true,
+      "license": "MIT",
+      "engines": {
+        "node": ">=8"
+      }
+    },
+    "node_modules/wrap-ansi-cjs/node_modules/ansi-styles": {
+      "version": "4.3.0",
+      "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz",
+      "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==",
+      "dev": true,
+      "license": "MIT",
+      "dependencies": {
+        "color-convert": "^2.0.1"
+      },
+      "engines": {
+        "node": ">=8"
+      },
+      "funding": {
+        "url": "https://github.com/chalk/ansi-styles?sponsor=1"
+      }
+    },
+    "node_modules/wrap-ansi-cjs/node_modules/color-convert": {
+      "version": "2.0.1",
+      "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz",
+      "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==",
+      "dev": true,
+      "license": "MIT",
+      "dependencies": {
+        "color-name": "~1.1.4"
+      },
+      "engines": {
+        "node": ">=7.0.0"
+      }
+    },
+    "node_modules/wrap-ansi-cjs/node_modules/color-name": {
+      "version": "1.1.4",
+      "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz",
+      "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==",
+      "dev": true,
+      "license": "MIT"
+    },
+    "node_modules/wrap-ansi-cjs/node_modules/emoji-regex": {
+      "version": "8.0.0",
+      "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz",
+      "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==",
+      "dev": true,
+      "license": "MIT"
+    },
+    "node_modules/wrap-ansi-cjs/node_modules/string-width": {
+      "version": "4.2.3",
+      "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz",
+      "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==",
+      "dev": true,
+      "license": "MIT",
+      "dependencies": {
+        "emoji-regex": "^8.0.0",
+        "is-fullwidth-code-point": "^3.0.0",
+        "strip-ansi": "^6.0.1"
+      },
+      "engines": {
+        "node": ">=8"
+      }
+    },
+    "node_modules/wrap-ansi-cjs/node_modules/strip-ansi": {
+      "version": "6.0.1",
+      "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz",
+      "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==",
+      "dev": true,
+      "license": "MIT",
+      "dependencies": {
+        "ansi-regex": "^5.0.1"
+      },
+      "engines": {
+        "node": ">=8"
+      }
+    },
+    "node_modules/wrap-ansi/node_modules/ansi-styles": {
+      "version": "6.2.1",
+      "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-6.2.1.tgz",
+      "integrity": "sha512-bN798gFfQX+viw3R7yrGWRqnrN2oRkEkUjjl4JNn4E8GxxbjtG3FbrEIIY3l8/hrwUwIeCZvi4QuOTP4MErVug==",
+      "dev": true,
+      "license": "MIT",
+      "engines": {
+        "node": ">=12"
+      },
+      "funding": {
+        "url": "https://github.com/chalk/ansi-styles?sponsor=1"
+      }
+    },
     "node_modules/wrappy": {
       "version": "1.0.2",
       "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz",
       "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==",
-      "dev": true
+      "dev": true,
+      "optional": true
     },
     "node_modules/xml2js": {
       "version": "0.5.0",
diff --git a/lldb/tools/lldb-dap/package.json b/lldb/tools/lldb-dap/package.json
index bbe65e1f73fd8..e866af0602d70 100644
--- a/lldb/tools/lldb-dap/package.json
+++ b/lldb/tools/lldb-dap/package.json
@@ -1,7 +1,7 @@
 {
   "name": "lldb-dap",
   "displayName": "LLDB DAP",
-  "version": "0.2.8",
+  "version": "0.2.9",
   "publisher": "llvm-vs-code-extensions",
   "homepage": "https://lldb.llvm.org",
   "description": "LLDB debugging from VSCode",
@@ -28,11 +28,11 @@
   ],
   "devDependencies": {
     "@types/node": "^18.11.18",
-    "@types/vscode": "~1.74.0",
-    "@vscode/vsce": "^2.19.0",
-    "prettier-plugin-curly": "^0.1.3",
-    "prettier": "^3.1.1",
-    "typescript": "^4.6.4"
+    "@types/vscode": "1.75.0",
+    "@vscode/vsce": "^3.2.2",
+    "prettier-plugin-curly": "^0.3.1",
+    "prettier": "^3.4.2",
+    "typescript": "^5.7.3"
   },
   "activationEvents": [
     "onDebug"
diff --git a/lldb/unittests/OperatingSystem/CMakeLists.txt b/lldb/unittests/OperatingSystem/CMakeLists.txt
new file mode 100644
index 0000000000000..ae810f37b0577
--- /dev/null
+++ b/lldb/unittests/OperatingSystem/CMakeLists.txt
@@ -0,0 +1,21 @@
+add_lldb_unittest(ProcessGdbRemoteTests
+  GDBRemoteClientBaseTest.cpp
+  GDBRemoteCommunicationClientTest.cpp
+  GDBRemoteCommunicationServerLLGSTest.cpp
+  GDBRemoteCommunicationServerTest.cpp
+  GDBRemoteCommunicationTest.cpp
+  GDBRemoteTestUtils.cpp
+
+  LINK_LIBS
+    LLVMTestingSupport
+    lldbCore
+    lldbHost
+    lldbInterpreter
+    lldbPluginProcessUtility
+    lldbSymbol
+    lldbTarget
+    lldbValueObject
+
+  LINK_COMPONENTS
+    Support
+  )
diff --git a/lldb/unittests/OperatingSystem/OperatingSystemPlugin.h b/lldb/unittests/OperatingSystem/OperatingSystemPlugin.h
new file mode 100644
index 0000000000000..c58362dec03aa
--- /dev/null
+++ b/lldb/unittests/OperatingSystem/OperatingSystemPlugin.h
@@ -0,0 +1,59 @@
+//===-- OperatingSystemPlugin.h ---------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "lldb/Core/PluginManager.h"
+#include "lldb/Target/OperatingSystem.h"
+#include "lldb/Target/Thread.h"
+#include "lldb/Target/ThreadList.h"
+
+/// An operating system plugin that does nothing: simply keeps the thread lists
+/// as they are.
+class OperatingSystemIdentityMap : public lldb_private::OperatingSystem {
+public:
+  OperatingSystemIdentityMap(lldb_private::Process *process)
+      : OperatingSystem(process) {}
+
+  static OperatingSystem *CreateInstance(lldb_private::Process *process,
+                                         bool force) {
+    return new OperatingSystemIdentityMap(process);
+  }
+  static llvm::StringRef GetPluginNameStatic() { return "identity map"; }
+  static llvm::StringRef GetPluginDescriptionStatic() { return ""; }
+
+  static void Initialize() {
+    lldb_private::PluginManager::RegisterPlugin(GetPluginNameStatic(),
+                                                GetPluginDescriptionStatic(),
+                                                CreateInstance, nullptr);
+  }
+  static void Terminate() {
+    lldb_private::PluginManager::UnregisterPlugin(CreateInstance);
+  }
+  llvm::StringRef GetPluginName() override { return GetPluginNameStatic(); }
+
+  // Simply adds the threads from real_thread_list into new_thread_list.
+  bool UpdateThreadList(lldb_private::ThreadList &old_thread_list,
+                        lldb_private::ThreadList &real_thread_list,
+                        lldb_private::ThreadList &new_thread_list) override {
+    for (const auto &real_thread : real_thread_list.Threads())
+      new_thread_list.AddThread(real_thread);
+    return true;
+  }
+
+  void ThreadWasSelected(lldb_private::Thread *thread) override {}
+
+  lldb::RegisterContextSP
+  CreateRegisterContextForThread(lldb_private::Thread *thread,
+                                 lldb::addr_t reg_data_addr) override {
+    return thread->GetRegisterContext();
+  }
+
+  lldb::StopInfoSP
+  CreateThreadStopReason(lldb_private::Thread *thread) override {
+    return thread->GetStopInfo();
+  }
+};
diff --git a/lldb/unittests/OperatingSystem/TestThreadSpecificBreakpoints.cpp b/lldb/unittests/OperatingSystem/TestThreadSpecificBreakpoints.cpp
new file mode 100644
index 0000000000000..369f5b3ae4670
--- /dev/null
+++ b/lldb/unittests/OperatingSystem/TestThreadSpecificBreakpoints.cpp
@@ -0,0 +1,10 @@
+//===-- OperatingSystemPlugin.h ---------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "OperatingSystemPlugin.h"
+LLDB_PLUGIN_DEFINE(OperatingSystemIdentityMap)
diff --git a/lldb/unittests/ScriptInterpreter/Python/PythonDataObjectsTests.cpp b/lldb/unittests/ScriptInterpreter/Python/PythonDataObjectsTests.cpp
index 365ebc8e52c24..2dd92fc00fea1 100644
--- a/lldb/unittests/ScriptInterpreter/Python/PythonDataObjectsTests.cpp
+++ b/lldb/unittests/ScriptInterpreter/Python/PythonDataObjectsTests.cpp
@@ -760,10 +760,6 @@ class NewStyle(object):
     EXPECT_EQ(arginfo.get().max_positional_args, 3u);
   }
 
-#if PY_VERSION_HEX >= 0x03030000
-
-  // the old implementation of GetArgInfo just doesn't work on builtins.
-
   {
     auto builtins = PythonModule::BuiltinsModule();
     auto hex = As<PythonCallable>(builtins.GetAttribute("hex"));
@@ -772,8 +768,6 @@ class NewStyle(object):
     ASSERT_THAT_EXPECTED(arginfo, llvm::Succeeded());
     EXPECT_EQ(arginfo.get().max_positional_args, 1u);
   }
-
-#endif
 }
 
 TEST_F(PythonDataObjectsTest, TestScript) {
diff --git a/lldb/unittests/Symbol/TestTypeSystemClang.cpp b/lldb/unittests/Symbol/TestTypeSystemClang.cpp
index a2d1f6db80277..23374062127e0 100644
--- a/lldb/unittests/Symbol/TestTypeSystemClang.cpp
+++ b/lldb/unittests/Symbol/TestTypeSystemClang.cpp
@@ -1040,3 +1040,45 @@ TEST_F(TestTypeSystemClang, GetDeclContextByNameWhenMissingSymbolFile) {
 
   EXPECT_TRUE(decls.empty());
 }
+
+TEST_F(TestTypeSystemClang, AddMethodToCXXRecordType_ParmVarDecls) {
+  // Tests that AddMethodToCXXRecordType creates ParmVarDecl's with
+  // a correct clang::DeclContext.
+
+  llvm::StringRef class_name = "S";
+  CompilerType t = clang_utils::createRecord(*m_ast, class_name);
+  m_ast->StartTagDeclarationDefinition(t);
+
+  CompilerType return_type = m_ast->GetBasicType(lldb::eBasicTypeVoid);
+  const bool is_virtual = false;
+  const bool is_static = false;
+  const bool is_inline = false;
+  const bool is_explicit = true;
+  const bool is_attr_used = false;
+  const bool is_artificial = false;
+
+  llvm::SmallVector<CompilerType> param_types{
+      m_ast->GetBasicType(lldb::eBasicTypeInt),
+      m_ast->GetBasicType(lldb::eBasicTypeShort)};
+  CompilerType function_type = m_ast->CreateFunctionType(
+      return_type, param_types.data(), /*num_params*/ param_types.size(),
+      /*variadic=*/false, /*quals*/ 0U);
+  m_ast->AddMethodToCXXRecordType(
+      t.GetOpaqueQualType(), "myFunc", nullptr, function_type,
+      lldb::AccessType::eAccessPublic, is_virtual, is_static, is_inline,
+      is_explicit, is_attr_used, is_artificial);
+
+  // Complete the definition and check the created record.
+  m_ast->CompleteTagDeclarationDefinition(t);
+
+  auto *record = llvm::cast<CXXRecordDecl>(ClangUtil::GetAsTagDecl(t));
+
+  auto method_it = record->method_begin();
+  ASSERT_NE(method_it, record->method_end());
+
+  EXPECT_EQ(method_it->getNumParams(), param_types.size());
+
+  // DeclContext of each parameter should be the CXXMethodDecl itself.
+  EXPECT_EQ(method_it->getParamDecl(0)->getDeclContext(), *method_it);
+  EXPECT_EQ(method_it->getParamDecl(1)->getDeclContext(), *method_it);
+}
diff --git a/lldb/unittests/SymbolFile/DWARF/DWARFASTParserClangTests.cpp b/lldb/unittests/SymbolFile/DWARF/DWARFASTParserClangTests.cpp
index 8adda6fba3a0b..6c77736113da3 100644
--- a/lldb/unittests/SymbolFile/DWARF/DWARFASTParserClangTests.cpp
+++ b/lldb/unittests/SymbolFile/DWARF/DWARFASTParserClangTests.cpp
@@ -1082,3 +1082,173 @@ TEST_F(DWARFASTParserClangTests, TestParseSubroutine_ExplicitObjectParameter) {
             clang::Qualifiers::fromCVRMask(clang::Qualifiers::Const |
                                            clang::Qualifiers::Volatile));
 }
+
+TEST_F(DWARFASTParserClangTests, TestParseSubroutine_ParameterCreation) {
+  // Tests parsing of a C++ free function will create clang::ParmVarDecls with
+  // the correct clang::DeclContext.
+  //
+  // Also ensures we attach names to the ParmVarDecls (even when DWARF contains
+  // a mix of named/unnamed parameters).
+
+  const char *yamldata = R"(
+--- !ELF
+FileHeader:
+  Class:   ELFCLASS64
+  Data:    ELFDATA2LSB
+  Type:    ET_EXEC
+  Machine: EM_AARCH64
+DWARF:
+  debug_str:
+    - func
+    - int
+    - short
+    - namedParam
+  debug_abbrev:
+    - ID:              0
+      Table:
+        - Code:            0x1
+          Tag:             DW_TAG_compile_unit
+          Children:        DW_CHILDREN_yes
+          Attributes:
+            - Attribute:       DW_AT_language
+              Form:            DW_FORM_data2
+        - Code:            0x2
+          Tag:             DW_TAG_structure_type
+          Children:        DW_CHILDREN_yes
+          Attributes:
+            - Attribute:       DW_AT_name
+              Form:            DW_FORM_strp
+        - Code:            0x3
+          Tag:             DW_TAG_subprogram
+          Children:        DW_CHILDREN_yes
+          Attributes:
+            - Attribute:       DW_AT_name
+              Form:            DW_FORM_strp
+            - Attribute:       DW_AT_declaration
+              Form:            DW_FORM_flag_present
+            - Attribute:       DW_AT_external
+              Form:            DW_FORM_flag_present
+        - Code:            0x4
+          Tag:             DW_TAG_formal_parameter
+          Children:        DW_CHILDREN_no
+          Attributes:
+            - Attribute:       DW_AT_type
+              Form:            DW_FORM_ref4
+        - Code:            0x5
+          Tag:             DW_TAG_formal_parameter
+          Children:        DW_CHILDREN_no
+          Attributes:
+            - Attribute:       DW_AT_type
+              Form:            DW_FORM_ref4
+            - Attribute:       DW_AT_name
+              Form:            DW_FORM_strp
+        - Code:            0x6
+          Tag:             DW_TAG_base_type
+          Children:        DW_CHILDREN_no
+          Attributes:
+            - Attribute:       DW_AT_name
+              Form:            DW_FORM_strp
+            - Attribute:       DW_AT_encoding
+              Form:            DW_FORM_data1
+            - Attribute:       DW_AT_byte_size
+              Form:            DW_FORM_data1
+  debug_info:
+     - Version:         5
+       UnitType:        DW_UT_compile
+       AddrSize:        8
+       Entries:
+
+# DW_TAG_compile_unit
+#   DW_AT_language [DW_FORM_data2]    (DW_LANG_C_plus_plus)
+
+        - AbbrCode:        0x1
+          Values:
+            - Value:           0x04
+
+#     DW_TAG_subprogram
+#       DW_AT_name [DW_FORM_strp] ("func")
+        - AbbrCode:        0x3
+          Values:
+            - Value:           0x0
+            - Value:           0x1
+            - Value:           0x1
+
+#       DW_TAG_formal_parameter
+#         DW_AT_type [DW_FORM_ref4] (int)
+        - AbbrCode:        0x4
+          Values:
+            - Value: 0x23
+
+#       DW_TAG_formal_parameter
+#         DW_AT_type [DW_FORM_ref4] (short)
+#         DW_AT_name [DW_FORM_strp] ("namedParam")
+        - AbbrCode:        0x5
+          Values:
+            - Value: 0x2a
+            - Value: 0xf
+
+        - AbbrCode: 0x0
+
+#   DW_TAG_base_type
+#     DW_AT_name      [DW_FORM_strp] ("int")
+#     DW_AT_encoding  [DW_FORM_data1]
+#     DW_AT_byte_size [DW_FORM_data1]
+
+        - AbbrCode:        0x6
+          Values:
+            - Value:           0x0000000000000005
+            - Value:           0x0000000000000005 # DW_ATE_signed
+            - Value:           0x0000000000000004
+
+#   DW_TAG_base_type
+#     DW_AT_name      [DW_FORM_strp] ("short")
+#     DW_AT_encoding  [DW_FORM_data1]
+#     DW_AT_byte_size [DW_FORM_data1]
+
+        - AbbrCode:        0x6
+          Values:
+            - Value:           0x0000000000000009
+            - Value:           0x0000000000000005 # DW_ATE_signed
+            - Value:           0x0000000000000004
+
+        - AbbrCode: 0x0
+...
+)";
+  YAMLModuleTester t(yamldata);
+
+  DWARFUnit *unit = t.GetDwarfUnit();
+  ASSERT_NE(unit, nullptr);
+  const DWARFDebugInfoEntry *cu_entry = unit->DIE().GetDIE();
+  ASSERT_EQ(cu_entry->Tag(), DW_TAG_compile_unit);
+  ASSERT_EQ(unit->GetDWARFLanguageType(), DW_LANG_C_plus_plus);
+  DWARFDIE cu_die(unit, cu_entry);
+
+  auto ts_or_err =
+      cu_die.GetDWARF()->GetTypeSystemForLanguage(eLanguageTypeC_plus_plus);
+  ASSERT_TRUE(static_cast<bool>(ts_or_err));
+  llvm::consumeError(ts_or_err.takeError());
+
+  auto *ts = static_cast<TypeSystemClang *>(ts_or_err->get());
+  auto *parser = static_cast<DWARFASTParserClang *>(ts->GetDWARFParser());
+
+  auto subprogram = cu_die.GetFirstChild();
+  ASSERT_TRUE(subprogram.IsValid());
+  ASSERT_EQ(subprogram.Tag(), DW_TAG_subprogram);
+
+  SymbolContext sc;
+  bool new_type;
+  auto type_sp = parser->ParseTypeFromDWARF(sc, subprogram, &new_type);
+  ASSERT_NE(type_sp, nullptr);
+
+  auto result = ts->GetTranslationUnitDecl()->lookup(
+      clang_utils::getDeclarationName(*ts, "func"));
+  ASSERT_TRUE(result.isSingleResult());
+
+  auto const *func = llvm::cast<clang::FunctionDecl>(result.front());
+
+  EXPECT_EQ(func->getNumParams(), 2U);
+  EXPECT_EQ(func->getParamDecl(0)->getDeclContext(), func);
+  EXPECT_TRUE(func->getParamDecl(0)->getName().empty());
+  EXPECT_EQ(func->getParamDecl(1)->getDeclContext(), func);
+  EXPECT_EQ(func->getParamDecl(1)->getName(), "namedParam");
+}
diff --git a/lldb/unittests/TestingSupport/Symbol/ClangTestUtils.h b/lldb/unittests/TestingSupport/Symbol/ClangTestUtils.h
index 21525266119b4..63b2ba8c8688a 100644
--- a/lldb/unittests/TestingSupport/Symbol/ClangTestUtils.h
+++ b/lldb/unittests/TestingSupport/Symbol/ClangTestUtils.h
@@ -21,20 +21,21 @@ inline clang::DeclarationName getDeclarationName(TypeSystemClang &ast,
   return ast.getASTContext().DeclarationNames.getIdentifier(&II);
 }
 
-inline CompilerType createRecord(TypeSystemClang &ast, llvm::StringRef name) {
+inline CompilerType
+createRecord(TypeSystemClang &ast, llvm::StringRef name,
+             lldb::LanguageType lang = lldb::LanguageType::eLanguageTypeC) {
   return ast.CreateRecordType(ast.getASTContext().getTranslationUnitDecl(),
                               OptionalClangModuleID(),
-                              lldb::AccessType::eAccessPublic, name, 0,
-                              lldb::LanguageType::eLanguageTypeC);
+                              lldb::AccessType::eAccessPublic, name, 0, lang);
 }
 
 /// Create a record with the given name and a field with the given type
 /// and name.
-inline CompilerType createRecordWithField(TypeSystemClang &ast,
-                                          llvm::StringRef record_name,
-                                          CompilerType field_type,
-                                          llvm::StringRef field_name) {
-  CompilerType t = createRecord(ast, record_name);
+inline CompilerType createRecordWithField(
+    TypeSystemClang &ast, llvm::StringRef record_name, CompilerType field_type,
+    llvm::StringRef field_name,
+    lldb::LanguageType lang = lldb::LanguageType::eLanguageTypeC) {
+  CompilerType t = createRecord(ast, record_name, lang);
 
   TypeSystemClang::StartTagDeclarationDefinition(t);
   ast.AddFieldToRecordType(t, field_name, field_type,
@@ -63,12 +64,13 @@ struct SourceASTWithRecord {
   CompilerType record_type;
   clang::RecordDecl *record_decl = nullptr;
   clang::FieldDecl *field_decl = nullptr;
-  SourceASTWithRecord() {
+  SourceASTWithRecord(
+      lldb::LanguageType lang = lldb::LanguageType::eLanguageTypeC) {
     holder = std::make_unique<TypeSystemClangHolder>("test ASTContext");
     ast = holder->GetAST();
     record_type = createRecordWithField(
         *ast, "Source", ast->GetBasicType(lldb::BasicType::eBasicTypeChar),
-        "a_field");
+        "a_field", lang);
     record_decl =
         llvm::cast<clang::RecordDecl>(ClangUtil::GetAsTagDecl(record_type));
     field_decl = *record_decl->fields().begin();
diff --git a/lldb/unittests/Utility/ArchSpecTest.cpp b/lldb/unittests/Utility/ArchSpecTest.cpp
index 74a4b48456b01..2c78629849c64 100644
--- a/lldb/unittests/Utility/ArchSpecTest.cpp
+++ b/lldb/unittests/Utility/ArchSpecTest.cpp
@@ -400,6 +400,12 @@ TEST(ArchSpecTest, Compatibility) {
     B.MergeFrom(A);
     ASSERT_TRUE(B.IsExactMatch(C));
   }
+  {
+    ArchSpec A("x86_64-apple-driverkit19.0");
+    ArchSpec B("x86_64-apple-macosx10.15.0");
+    ASSERT_FALSE(A.IsExactMatch(B));
+    ASSERT_TRUE(A.IsCompatibleMatch(B));
+  }
 }
 
 TEST(ArchSpecTest, OperatorBool) {
diff --git a/lldb/unittests/ValueObject/CMakeLists.txt b/lldb/unittests/ValueObject/CMakeLists.txt
index 8fcc8d62a7997..6ef0091647a59 100644
--- a/lldb/unittests/ValueObject/CMakeLists.txt
+++ b/lldb/unittests/ValueObject/CMakeLists.txt
@@ -1,10 +1,13 @@
 add_lldb_unittest(LLDBValueObjectTests
   DumpValueObjectOptionsTests.cpp
+  DILLexerTests.cpp
+  DynamicValueObjectLocalBuffer.cpp
 
   LINK_LIBS
     lldbValueObject
     lldbPluginPlatformLinux
     lldbPluginScriptInterpreterNone
+    LLVMTestingSupport
 
   LINK_COMPONENTS
     Support
diff --git a/lldb/unittests/ValueObject/DILLexerTests.cpp b/lldb/unittests/ValueObject/DILLexerTests.cpp
new file mode 100644
index 0000000000000..9e5b8efd7af80
--- /dev/null
+++ b/lldb/unittests/ValueObject/DILLexerTests.cpp
@@ -0,0 +1,156 @@
+//===-- DILLexerTests.cpp --------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "lldb/ValueObject/DILLexer.h"
+#include "llvm/ADT/StringRef.h"
+#include "llvm/Testing/Support/Error.h"
+#include "gtest/gtest.h"
+#include <string>
+
+using llvm::StringRef;
+
+using namespace lldb_private::dil;
+
+llvm::Expected<std::vector<std::pair<Token::Kind, std::string>>>
+ExtractTokenData(llvm::StringRef input_expr) {
+
+  llvm::Expected<DILLexer> maybe_lexer = DILLexer::Create(input_expr);
+  if (!maybe_lexer)
+    return maybe_lexer.takeError();
+  DILLexer lexer(*maybe_lexer);
+
+  std::vector<std::pair<Token::Kind, std::string>> data;
+  do {
+    Token tok = lexer.GetCurrentToken();
+    data.push_back(std::make_pair(tok.GetKind(), tok.GetSpelling()));
+    lexer.Advance();
+  } while (data.back().first != Token::eof);
+  // Don't return the eof token.
+  data.pop_back();
+  return data;
+}
+
+TEST(DILLexerTests, SimpleTest) {
+  StringRef input_expr("simple_var");
+  llvm::Expected<DILLexer> maybe_lexer = DILLexer::Create(input_expr);
+  ASSERT_THAT_EXPECTED(maybe_lexer, llvm::Succeeded());
+  DILLexer lexer(*maybe_lexer);
+  Token token = lexer.GetCurrentToken();
+
+  EXPECT_EQ(token.GetKind(), Token::identifier);
+  EXPECT_EQ(token.GetSpelling(), "simple_var");
+  lexer.Advance();
+  token = lexer.GetCurrentToken();
+  EXPECT_EQ(token.GetKind(), Token::eof);
+}
+
+TEST(DILLexerTests, TokenKindTest) {
+  Token token = Token(Token::identifier, "ident", 0);
+
+  EXPECT_TRUE(token.Is(Token::identifier));
+  EXPECT_FALSE(token.Is(Token::l_paren));
+  EXPECT_TRUE(token.IsOneOf(Token::eof, Token::identifier));
+  EXPECT_FALSE(token.IsOneOf(Token::l_paren, Token::r_paren, Token::coloncolon,
+                             Token::eof));
+}
+
+TEST(DILLexerTests, LookAheadTest) {
+  StringRef input_expr("(anonymous namespace)::some_var");
+  llvm::Expected<DILLexer> maybe_lexer = DILLexer::Create(input_expr);
+  ASSERT_THAT_EXPECTED(maybe_lexer, llvm::Succeeded());
+  DILLexer lexer(*maybe_lexer);
+  Token token = lexer.GetCurrentToken();
+
+  // Current token is '('; check the next 4 tokens, to make
+  // sure they are the identifier 'anonymous', the identifier 'namespace'
+  // ')' and '::', in that order.
+  EXPECT_EQ(token.GetKind(), Token::l_paren);
+  EXPECT_EQ(lexer.LookAhead(1).GetKind(), Token::identifier);
+  EXPECT_EQ(lexer.LookAhead(1).GetSpelling(), "anonymous");
+  EXPECT_EQ(lexer.LookAhead(2).GetKind(), Token::identifier);
+  EXPECT_EQ(lexer.LookAhead(2).GetSpelling(), "namespace");
+  EXPECT_EQ(lexer.LookAhead(3).GetKind(), Token::r_paren);
+  EXPECT_EQ(lexer.LookAhead(4).GetKind(), Token::coloncolon);
+
+  // Our current index should still be 0, as we only looked ahead; we are still
+  // officially on the '('.
+  EXPECT_EQ(lexer.GetCurrentTokenIdx(), 0u);
+
+  // Accept the 'lookahead', so our current token is '::', which has the index
+  // 4 in our vector of tokens (which starts at zero).
+  lexer.Advance(4);
+  token = lexer.GetCurrentToken();
+  EXPECT_EQ(token.GetKind(), Token::coloncolon);
+  EXPECT_EQ(lexer.GetCurrentTokenIdx(), 4u);
+
+  lexer.Advance();
+  token = lexer.GetCurrentToken();
+  EXPECT_EQ(token.GetKind(), Token::identifier);
+  EXPECT_EQ(token.GetSpelling(), "some_var");
+  EXPECT_EQ(lexer.GetCurrentTokenIdx(), 5u);
+  EXPECT_EQ(token.GetLocation(), strlen("(anonymous namespace)::"));
+
+  lexer.Advance();
+  token = lexer.GetCurrentToken();
+  EXPECT_EQ(token.GetKind(), Token::eof);
+}
+
+TEST(DILLexerTests, MultiTokenLexTest) {
+  EXPECT_THAT_EXPECTED(
+      ExtractTokenData("This string has (several ) ::identifiers"),
+      llvm::HasValue(testing::ElementsAre(
+          testing::Pair(Token::identifier, "This"),
+          testing::Pair(Token::identifier, "string"),
+          testing::Pair(Token::identifier, "has"),
+          testing::Pair(Token::l_paren, "("),
+          testing::Pair(Token::identifier, "several"),
+          testing::Pair(Token::r_paren, ")"),
+          testing::Pair(Token::coloncolon, "::"),
+          testing::Pair(Token::identifier, "identifiers"))));
+}
+
+TEST(DILLexerTests, IdentifiersTest) {
+  // These strings should lex into identifier tokens.
+  std::vector<std::string> valid_identifiers = {
+      "$My_name1", "$pc",  "abcd", "_", "_a",     "_a_",      "$",
+      "a_b",       "this", "self", "a", "MyName", "namespace"};
+
+  // The lexer can lex these strings, but they should not be identifiers.
+  std::vector<std::string> invalid_identifiers = {"", "::", "(", ")"};
+
+  // The lexer is expected to fail attempting to lex these strings (it cannot
+  // create valid tokens out of them).
+  std::vector<std::string> invalid_tok_strings = {"234", "2a", "2", "1MyName"};
+
+  // Verify that all of the valid identifiers come out as identifier tokens.
+  for (auto &str : valid_identifiers) {
+    SCOPED_TRACE(str);
+    EXPECT_THAT_EXPECTED(ExtractTokenData(str),
+                         llvm::HasValue(testing::ElementsAre(
+                             testing::Pair(Token::identifier, str))));
+  }
+
+  // Verify that the lexer fails on invalid token strings.
+  for (auto &str : invalid_tok_strings) {
+    SCOPED_TRACE(str);
+    auto maybe_lexer = DILLexer::Create(str);
+    EXPECT_THAT_EXPECTED(maybe_lexer, llvm::Failed());
+  }
+
+  // Verify that none of the invalid identifiers come out as identifier tokens.
+  for (auto &str : invalid_identifiers) {
+    SCOPED_TRACE(str);
+    llvm::Expected<DILLexer> maybe_lexer = DILLexer::Create(str);
+    EXPECT_THAT_EXPECTED(maybe_lexer, llvm::Succeeded());
+    DILLexer lexer(*maybe_lexer);
+    Token token = lexer.GetCurrentToken();
+    EXPECT_TRUE(token.IsNot(Token::identifier));
+    EXPECT_TRUE(token.IsOneOf(Token::eof, Token::coloncolon, Token::l_paren,
+                              Token::r_paren));
+  }
+}
diff --git a/lldb/unittests/ValueObject/DynamicValueObjectLocalBuffer.cpp b/lldb/unittests/ValueObject/DynamicValueObjectLocalBuffer.cpp
new file mode 100644
index 0000000000000..417708dd2dc22
--- /dev/null
+++ b/lldb/unittests/ValueObject/DynamicValueObjectLocalBuffer.cpp
@@ -0,0 +1,240 @@
+//===---DynamicValueObjectLocalBuffer.cpp-----------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "Plugins/Platform/Linux/PlatformLinux.h"
+#include "Plugins/ScriptInterpreter/None/ScriptInterpreterNone.h"
+#include "Plugins/TypeSystem/Clang/TypeSystemClang.h"
+#include "TestingSupport/SubsystemRAII.h"
+#include "TestingSupport/Symbol/ClangTestUtils.h"
+#include "lldb/Core/Debugger.h"
+#include "lldb/Core/PluginManager.h"
+#include "lldb/Target/Language.h"
+#include "lldb/Target/LanguageRuntime.h"
+#include "lldb/ValueObject/ValueObject.h"
+#include "lldb/ValueObject/ValueObjectConstResult.h"
+
+#include "gtest/gtest.h"
+
+using namespace lldb;
+using namespace lldb_private;
+using namespace lldb_private::clang_utils;
+
+// This entire class is boilerplate.
+struct MockLanguage : public Language {
+
+  llvm::StringRef GetPluginName() override { return "MockLanguage"; }
+  lldb::LanguageType GetLanguageType() const override {
+    return lldb::eLanguageTypeC_plus_plus;
+  };
+
+  static Language *CreateInstance(lldb::LanguageType language) {
+    return new MockLanguage();
+  }
+  static void Initialize() {
+    PluginManager::RegisterPlugin("MockLanguage", "Mock Language",
+                                  CreateInstance);
+  };
+
+  static void Terminate() { PluginManager::UnregisterPlugin(CreateInstance); }
+  bool IsSourceFile(llvm::StringRef file_path) const override { return true; }
+};
+LLDB_PLUGIN_DEFINE(MockLanguage)
+
+struct MockLanguageRuntime : public LanguageRuntime {
+  // This is the only method in this class that matters for this test.
+  // This will unconditionally succeed and return a type with size 4,
+  // a value_type of HostAddress, and a local buffer that points to the parent's
+  // local buffer.
+  // The tests will set that buffer to be either be larger or smaller than the
+  // type we're returning.
+  bool
+  GetDynamicTypeAndAddress(ValueObject &in_value,
+                           lldb::DynamicValueType use_dynamic,
+                           TypeAndOrName &class_type_or_name, Address &address,
+                           Value::ValueType &value_type,
+                           llvm::ArrayRef<uint8_t> &local_buffer) override {
+    auto ast = in_value.GetCompilerType()
+                   .GetTypeSystem()
+                   .dyn_cast_or_null<TypeSystemClang>();
+
+    auto int_type = createRecordWithField(
+        *ast, "TypeWitInt", ast->GetBasicType(lldb::BasicType::eBasicTypeInt),
+        "theIntField", LanguageType::eLanguageTypeC_plus_plus);
+    class_type_or_name.SetCompilerType(int_type);
+    local_buffer = in_value.GetLocalBuffer();
+    value_type = Value::ValueType::HostAddress;
+    return true;
+  }
+
+  // All of this is boilerplate.
+  MockLanguageRuntime(Process *process) : LanguageRuntime(process) {}
+  llvm::StringRef GetPluginName() override { return "MockLanguageRuntime"; }
+  lldb::LanguageType GetLanguageType() const override {
+    return lldb::eLanguageTypeC_plus_plus;
+  }
+
+  llvm::Error GetObjectDescription(Stream &str, ValueObject &object) override {
+    return llvm::Error::success();
+  }
+
+  llvm::Error GetObjectDescription(Stream &str, Value &value,
+                                   ExecutionContextScope *exe_scope) override {
+    return llvm::Error::success();
+  }
+
+  bool CouldHaveDynamicValue(ValueObject &in_value) override { return true; }
+
+  TypeAndOrName FixUpDynamicType(const TypeAndOrName &type_and_or_name,
+                                 ValueObject &static_value) override {
+    return type_and_or_name;
+  }
+
+  lldb::BreakpointResolverSP
+  CreateExceptionResolver(const lldb::BreakpointSP &bkpt, bool catch_bp,
+                          bool throw_bp) override {
+    return lldb::BreakpointResolverSP();
+  }
+
+  lldb::ThreadPlanSP GetStepThroughTrampolinePlan(Thread &thread,
+                                                  bool stop_others) override {
+    return {};
+  }
+
+  static LanguageRuntime *CreateInstance(Process *process,
+                                         LanguageType language) {
+    return new MockLanguageRuntime(process);
+  }
+
+  static void Initialize() {
+    PluginManager::RegisterPlugin(
+        "MockLanguageRuntime", "MockLanguageRuntime", CreateInstance,
+        [](CommandInterpreter &interpreter) -> lldb::CommandObjectSP {
+          return {};
+        },
+        [](lldb::LanguageType language,
+           bool throw_bp) -> BreakpointPreconditionSP { return {}; });
+  }
+
+  static void Terminate() { PluginManager::UnregisterPlugin(CreateInstance); }
+};
+LLDB_PLUGIN_DEFINE(MockLanguageRuntime)
+
+// This entire class is boilerplate.
+struct MockProcess : Process {
+  MockProcess(lldb::TargetSP target_sp, lldb::ListenerSP listener_sp)
+      : Process(target_sp, listener_sp) {}
+
+  llvm::StringRef GetPluginName() override { return "mock process"; }
+
+  bool CanDebug(lldb::TargetSP target, bool plugin_specified_by_name) override {
+    return false;
+  };
+
+  Status DoDestroy() override { return {}; }
+
+  void RefreshStateAfterStop() override {}
+
+  bool DoUpdateThreadList(ThreadList &old_thread_list,
+                          ThreadList &new_thread_list) override {
+    return false;
+  };
+
+  size_t DoReadMemory(lldb::addr_t vm_addr, void *buf, size_t size,
+                      Status &error) override {
+    // No need to read memory in these tests.
+    return size;
+  }
+};
+
+class DynamicValueObjectLocalBufferTest : public ::testing::Test {
+public:
+  void SetUp() override {
+    ArchSpec arch("i386-pc-linux");
+    Platform::SetHostPlatform(
+        platform_linux::PlatformLinux::CreateInstance(true, &arch));
+    // std::call_once(TestUtilities::g_debugger_initialize_flag,
+    //                []() { Debugger::Initialize(nullptr); });
+    m_debugger_sp = Debugger::CreateInstance();
+    ASSERT_TRUE(m_debugger_sp);
+    m_debugger_sp->GetTargetList().CreateTarget(*m_debugger_sp, "", arch,
+                                                eLoadDependentsNo,
+                                                m_platform_sp, m_target_sp);
+    ASSERT_TRUE(m_target_sp);
+    ASSERT_TRUE(m_target_sp->GetArchitecture().IsValid());
+    ASSERT_TRUE(m_platform_sp);
+    m_listener_sp = Listener::MakeListener("dummy");
+    m_process_sp = std::make_shared<MockProcess>(m_target_sp, m_listener_sp);
+    ASSERT_TRUE(m_process_sp);
+    m_exe_ctx = ExecutionContext(m_process_sp);
+
+    m_holder = std::make_unique<clang_utils::TypeSystemClangHolder>("test");
+    m_type_system = m_holder->GetAST();
+    LLDB_PLUGIN_INITIALIZE(MockLanguage);
+    LLDB_PLUGIN_INITIALIZE(MockLanguageRuntime);
+  }
+  void TearDown() override {
+    LLDB_PLUGIN_TERMINATE(MockLanguage);
+    LLDB_PLUGIN_TERMINATE(MockLanguageRuntime);
+  }
+
+  void TestValueObjectWithLocalBuffer(DataExtractor &data_extractor,
+                                      bool should_succeed) {
+    std::unique_ptr<TypeSystemClangHolder> holder =
+        std::make_unique<TypeSystemClangHolder>("test ASTContext");
+    TypeSystemClang *ast = holder->GetAST();
+    auto char_type = createRecordWithField(
+        *ast, "TypeWithChar",
+        ast->GetBasicType(lldb::BasicType::eBasicTypeChar), "theField");
+
+    ExecutionContextScope *exe_scope = m_exe_ctx.GetBestExecutionContextScope();
+    ConstString var_name("test_var");
+    auto valobj_sp = ValueObjectConstResult::Create(exe_scope, char_type,
+                                                    var_name, data_extractor);
+    auto dyn_valobj = valobj_sp->GetDynamicValue(lldb::eDynamicCanRunTarget);
+    ASSERT_TRUE(dyn_valobj->GetValueIsValid() == should_succeed);
+  }
+
+  SubsystemRAII<FileSystem, HostInfo, platform_linux::PlatformLinux,
+                ScriptInterpreterNone>
+      m_subsystems;
+  std::unique_ptr<clang_utils::TypeSystemClangHolder> m_holder;
+  lldb::DebuggerSP m_debugger_sp;
+  lldb::TargetSP m_target_sp;
+  lldb::PlatformSP m_platform_sp;
+  lldb::ListenerSP m_listener_sp;
+  lldb::ProcessSP m_process_sp;
+  ExecutionContext m_exe_ctx;
+  TypeSystemClang *m_type_system;
+};
+
+TEST_F(DynamicValueObjectLocalBufferTest, BufferTooSmall) {
+  /// Test that a value object with a buffer to small to fit the
+  /// "dynamic" type will return an invalid dynamic value object.
+  uint8_t value = 1;
+  ByteOrder endian = endian::InlHostByteOrder();
+  DataExtractor data_extractor{&value, sizeof(value), endian, 4};
+  TestValueObjectWithLocalBuffer(data_extractor, false);
+}
+
+TEST_F(DynamicValueObjectLocalBufferTest, BufferTooBig) {
+  /// Test that a value object with a buffer big enough fit the
+  /// "dynamic" type will return a valid dynamic value object.
+  uint64_t value = 1;
+  ByteOrder endian = endian::InlHostByteOrder();
+  DataExtractor data_extractor{&value, sizeof(value), endian, 4};
+  TestValueObjectWithLocalBuffer(data_extractor, true);
+}
+
+TEST_F(DynamicValueObjectLocalBufferTest, BufferExactlyRight) {
+  /// Test that a value object with a buffer exactly the size of the
+  /// "dynamic" type will return a valid dynamic value object.
+  uint32_t value = 1;
+  ByteOrder endian = endian::InlHostByteOrder();
+  DataExtractor data_extractor{&value, sizeof(value), endian, 4};
+  TestValueObjectWithLocalBuffer(data_extractor, true);
+}
diff --git a/lldb/utils/lldb-dotest/lldb-dotest.in b/lldb/utils/lldb-dotest/lldb-dotest.in
index f20859e87b1e0..9688b94d91842 100755
--- a/lldb/utils/lldb-dotest/lldb-dotest.in
+++ b/lldb/utils/lldb-dotest/lldb-dotest.in
@@ -4,7 +4,8 @@ import subprocess
 import sys
 
 dotest_path = '@LLDB_SOURCE_DIR_CONFIGURED@/test/API/dotest.py'
-dotest_args_str = '@LLDB_DOTEST_ARGS_CONFIGURED@'
+dotest_common_args_str = '@LLDB_TEST_COMMON_ARGS_CONFIGURED@'
+dotest_user_args_str = '@LLDB_TEST_USER_ARGS_CONFIGURED@'
 arch = '@LLDB_TEST_ARCH@'
 executable = '@LLDB_TEST_EXECUTABLE_CONFIGURED@'
 compiler = '@LLDB_TEST_COMPILER_CONFIGURED@'
@@ -26,8 +27,10 @@ if __name__ == '__main__':
     dotest_args = []
     # split on an empty string will produce [''] and if you
     # add that to the command, it will be treated as a directory...
-    if len(dotest_args_str) > 0:
-        dotest_args = dotest_args_str.split(';')
+    if dotest_common_args_str:
+        dotest_args.extend(dotest_common_args_str.split(';'))
+    if dotest_user_args_str:
+        dotest_args.extend(dotest_user_args_str.split(';'))
     # Build dotest.py command.
     cmd = [sys.executable, dotest_path]
     cmd.extend(['--arch', arch])
diff --git a/llvm/CMakeLists.txt b/llvm/CMakeLists.txt
index ad12100fdb5b8..f5293e8663243 100644
--- a/llvm/CMakeLists.txt
+++ b/llvm/CMakeLists.txt
@@ -157,6 +157,13 @@ if ("libc" IN_LIST LLVM_ENABLE_PROJECTS)
     "https://libc.llvm.org/ for building the runtimes.")
 endif()
 
+if ("compiler-rt" IN_LIST LLVM_ENABLE_PROJECTS)
+  message(WARNING "Using LLVM_ENABLE_PROJECTS=compiler-rt is deprecated now, and will "
+    "become a fatal error in the LLVM 21 release.  Please use "
+    "-DLLVM_ENABLE_RUNTIMES=compiler-rt or see the instructions at "
+    "https://compiler-rt.llvm.org/ for building the runtimes.")
+endif()
+
 # Select the runtimes to build
 #
 # As we migrate runtimes to using the bootstrapping build, the set of default runtimes
@@ -822,6 +829,7 @@ option (LLVM_ENABLE_DOXYGEN "Use doxygen to generate llvm API documentation." OF
 option (LLVM_ENABLE_SPHINX "Use Sphinx to generate llvm documentation." OFF)
 option (LLVM_ENABLE_OCAMLDOC "Build OCaml bindings documentation." ON)
 option (LLVM_ENABLE_BINDINGS "Build bindings." ON)
+option (LLVM_BUILD_TELEMETRY "Build the telemetry library. This does not enable telemetry." ON)
 
 set(LLVM_INSTALL_DOXYGEN_HTML_DIR "${CMAKE_INSTALL_DOCDIR}/llvm/doxygen-html"
     CACHE STRING "Doxygen-generated HTML documentation install directory")
@@ -1127,6 +1135,8 @@ if (NOT TENSORFLOW_AOT_PATH STREQUAL "")
     ${CMAKE_ARCHIVE_OUTPUT_DIRECTORY}/tf_runtime)
   install(TARGETS tf_xla_runtime EXPORT LLVMExports
     ARCHIVE DESTINATION lib${LLVM_LIBDIR_SUFFIX} COMPONENT tf_xla_runtime)
+  install(TARGETS tf_xla_runtime EXPORT LLVMDevelopmentExports
+    ARCHIVE DESTINATION lib${LLVM_LIBDIR_SUFFIX} COMPONENT tf_xla_runtime)
   set_property(GLOBAL APPEND PROPERTY LLVM_EXPORTS tf_xla_runtime)
   # Once we add more modules, we should handle this more automatically.
   if (DEFINED LLVM_OVERRIDE_MODEL_HEADER_INLINERSIZEMODEL)
diff --git a/llvm/Maintainers.md b/llvm/Maintainers.md
index 534d81e68d024..b9dd2041170a7 100644
--- a/llvm/Maintainers.md
+++ b/llvm/Maintainers.md
@@ -436,8 +436,8 @@ echristo@gmail.com (email), [echristo](https://github.com/echristo) (GitHub)
 
 #### Exception handling
 
-Anton Korobeynikov \
-anton@korobeynikov.info (email), [asl](https://github.com/asl) (GitHub)
+Reid Kleckner \
+rnk@google.com (email), [rnk](https://github.com/rnk) (GitHub)
 
 #### LLVM Buildbot
 
@@ -493,7 +493,7 @@ Hal Finkel (hfinkel@anl.gov, [hfinkel](https://github.com/hfinkel) -- AliasAnaly
 Renato Golin (rengolin@systemcall.eu, [rengolin](https://github.com/rengolin)) -- ARM backend \
 Venkatraman Govindaraju (venkatra@cs.wisc.edu, [vegovin](https://github.com/vegovin) -- Sparc backend \
 James Grosbach (grosbach@apple.com) -- MC layer \
-Anton Korobeynikov (anton@korobeynikov.info, [asl](https://github.com/asl)) -- ARM EABI, Windows codegen \
+Anton Korobeynikov (anton@korobeynikov.info, [asl](https://github.com/asl)) -- ARM EABI, Windows codegen, Exception handling \
 Benjamin Kramer (benny.kra@gmail.com, [d0k](https://github.com/d0k)) -- DWARF Parser \
 David Majnemer (david.majnemer@gmail.com, [majnemer](https://github.com/majnemer)) -- InstCombine, ConstantFold \
 Tim Northover (t.p.northover@gmail.com, [TNorthover](https://github.com/TNorthover)) -- AArch64 backend \
diff --git a/llvm/cmake/config-ix.cmake b/llvm/cmake/config-ix.cmake
index e906bdd9eac6e..767774812ade5 100644
--- a/llvm/cmake/config-ix.cmake
+++ b/llvm/cmake/config-ix.cmake
@@ -146,7 +146,7 @@ if( NOT PURE_WINDOWS )
     endif()
   endif()
   check_library_exists(dl dlopen "" HAVE_LIBDL)
-  check_library_exists(rt clock_gettime "" HAVE_LIBRT)
+  check_library_exists(rt shm_open "" HAVE_LIBRT)
 endif()
 
 # Check for libpfm.
diff --git a/llvm/cmake/modules/AddLLVM.cmake b/llvm/cmake/modules/AddLLVM.cmake
index d3e9377c8d2f7..39f0e28a249d3 100644
--- a/llvm/cmake/modules/AddLLVM.cmake
+++ b/llvm/cmake/modules/AddLLVM.cmake
@@ -1472,6 +1472,9 @@ if(NOT LLVM_TOOLCHAIN_TOOLS)
   if (LLVM_ENABLE_LIBXML2)
     list(APPEND LLVM_TOOLCHAIN_TOOLS llvm-mt)
   endif()
+  if (LLVM_TOOL_LLVM_DRIVER_BUILD)
+    list(APPEND LLVM_TOOLCHAIN_TOOLS llvm-driver)
+  endif()
 endif()
 
 macro(llvm_add_tool project name)
@@ -1495,6 +1498,11 @@ macro(llvm_add_tool project name)
                 ${export_to_llvmexports}
                 RUNTIME DESTINATION ${${project}_TOOLS_INSTALL_DIR}
                 COMPONENT ${name})
+        if (LLVM_ENABLE_PDB)
+          install(FILES $<TARGET_PDB_FILE:${name}> 
+                DESTINATION "${${project}_TOOLS_INSTALL_DIR}" COMPONENT ${name} 
+                OPTIONAL)
+        endif()
 
         if (NOT LLVM_ENABLE_IDE)
           add_llvm_install_targets(install-${name}
@@ -1525,6 +1533,11 @@ macro(add_llvm_example name)
   add_llvm_executable(${name} EXPORT_SYMBOLS ${ARGN})
   if( LLVM_BUILD_EXAMPLES )
     install(TARGETS ${name} RUNTIME DESTINATION "${LLVM_EXAMPLES_INSTALL_DIR}")
+    if (LLVM_ENABLE_PDB)
+      install(FILES $<TARGET_PDB_FILE:${name}> 
+              DESTINATION "${LLVM_EXAMPLES_INSTALL_DIR}" COMPONENT ${name} 
+              OPTIONAL)
+    endif()
   endif()
   get_subproject_title(subproject_title)
   set_target_properties(${name} PROPERTIES FOLDER "${subproject_title}/Examples")
@@ -1559,6 +1572,11 @@ macro(add_llvm_utility name)
               ${export_to_llvmexports}
               RUNTIME DESTINATION ${LLVM_UTILS_INSTALL_DIR}
               COMPONENT ${name})
+      if (LLVM_ENABLE_PDB)
+        install(FILES $<TARGET_PDB_FILE:${name}> 
+                DESTINATION "${LLVM_UTILS_INSTALL_DIR}" COMPONENT ${name} 
+                OPTIONAL)
+      endif()
 
       if (NOT LLVM_ENABLE_IDE)
         add_llvm_install_targets(install-${name}
diff --git a/llvm/cmake/modules/HandleLLVMOptions.cmake b/llvm/cmake/modules/HandleLLVMOptions.cmake
index f19125eb6bf27..5db06ccdadbeb 100644
--- a/llvm/cmake/modules/HandleLLVMOptions.cmake
+++ b/llvm/cmake/modules/HandleLLVMOptions.cmake
@@ -312,7 +312,7 @@ endif()
 
 # Pass -Wl,-z,defs. This makes sure all symbols are defined. Otherwise a DSO
 # build might work on ELF but fail on MachO/COFF.
-if(NOT (CMAKE_SYSTEM_NAME MATCHES "Darwin|FreeBSD|OpenBSD|DragonFly|AIX|OS390" OR
+if(NOT (CMAKE_SYSTEM_NAME MATCHES "Darwin|FreeBSD|OpenBSD|DragonFly|AIX|OS390|Emscripten" OR
         WIN32 OR CYGWIN) AND
    NOT LLVM_USE_SANITIZER)
   set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -Wl,-z,defs")
@@ -1212,7 +1212,7 @@ if (CLANG_CL AND (LLVM_BUILD_INSTRUMENTED OR LLVM_USE_SANITIZER))
   endif()
   file(TO_CMAKE_PATH "${clang_compiler_rt_file}" clang_compiler_rt_file)
   get_filename_component(clang_runtime_dir "${clang_compiler_rt_file}" DIRECTORY)
-  prepend("/libpath:\"${clang_runtime_dir}\""
+  prepend("${CMAKE_CXX_LINKER_WRAPPER_FLAG}/libpath:\"${clang_runtime_dir}\""
     CMAKE_EXE_LINKER_FLAGS
     CMAKE_MODULE_LINKER_FLAGS
     CMAKE_SHARED_LINKER_FLAGS)
@@ -1285,7 +1285,7 @@ elseif(LLVM_ENABLE_LTO)
   endif()
 endif()
 
-if(LLVM_ENABLE_FATLTO AND UNIX AND NOT APPLE)
+if(LLVM_ENABLE_FATLTO AND ((UNIX AND NOT APPLE) OR FUCHSIA))
   append("-ffat-lto-objects" CMAKE_C_FLAGS CMAKE_CXX_FLAGS)
   if(NOT LINKER_IS_LLD_LINK)
     append("-ffat-lto-objects" CMAKE_EXE_LINKER_FLAGS CMAKE_SHARED_LINKER_FLAGS CMAKE_MODULE_LINKER_FLAGS)
diff --git a/llvm/cmake/modules/LLVMConfig.cmake.in b/llvm/cmake/modules/LLVMConfig.cmake.in
index c49f10b9343ff..28655ee3ab87d 100644
--- a/llvm/cmake/modules/LLVMConfig.cmake.in
+++ b/llvm/cmake/modules/LLVMConfig.cmake.in
@@ -100,6 +100,8 @@ set(LLVM_ENABLE_PIC @LLVM_ENABLE_PIC@)
 
 set(LLVM_BUILD_32_BITS @LLVM_BUILD_32_BITS@)
 
+set(LLVM_BUILD_TELEMETRY @LLVM_BUILD_TELEMETRY@)
+
 if (NOT "@LLVM_PTHREAD_LIB@" STREQUAL "")
   set(LLVM_PTHREAD_LIB "@LLVM_PTHREAD_LIB@")
 endif()
diff --git a/llvm/docs/AMDGPUUsage.rst b/llvm/docs/AMDGPUUsage.rst
index 40b393224f15d..899b2cf3b4901 100644
--- a/llvm/docs/AMDGPUUsage.rst
+++ b/llvm/docs/AMDGPUUsage.rst
@@ -583,11 +583,11 @@ Generic processor code objects are versioned. See :ref:`amdgpu-generic-processor
                                                                                                   - ``v_dot2_f32_f16``
 
 
-     ``gfx9-4-generic``   ``amdgcn``     - ``gfx940``      - xnack            - Absolute flat   FP8 and BF8 instructions,
-                                         - ``gfx941``      - sramecc            scratch         FP8 and BF8 conversion instructions,
-                                         - ``gfx942``                                           as well as instructions with XF32 format support
-                                         - ``gfx950``                                           are not available.
-
+     ``gfx9-4-generic``   ``amdgcn``     - ``gfx940``      - sramecc          - Architected     FP8 and BF8 instructions,
+                                         - ``gfx941``      - tgsplit            flat scratch    FP8 and BF8 conversion
+                                         - ``gfx942``      - xnack            - Packed          instructions, as well as
+                                         - ``gfx950``      - kernarg preload    work-item       instructions with XF32 format
+                                                                                IDs             support are not available.
 
      ``gfx10-1-generic``  ``amdgcn``     - ``gfx1010``     - xnack            - Absolute flat   - The following instructions are
                                          - ``gfx1011``     - wavefrontsize64    scratch           not available on ``gfx1011``
@@ -1327,7 +1327,7 @@ The AMDGPU backend implements the following LLVM IR intrinsics.
                                                    with the fifth i32 operand. The i1 sixth operand is used to clamp
                                                    the output. The i1s preceding the vector operands decide the signedness.
 
-  llvm.amdgcn.sched_barrier                        Controls the types of instructions that may be allowed to cross the intrinsic
+  llvm.amdgcn.sched.barrier                        Controls the types of instructions that may be allowed to cross the intrinsic
                                                    during instruction scheduling. The parameter is a mask for the instruction types
                                                    that can cross the intrinsic.
 
@@ -1345,7 +1345,7 @@ The AMDGPU backend implements the following LLVM IR intrinsics.
                                                    - 0x0200: All DS write instructions may be scheduled across sched_barrier.
                                                    - 0x0400: All Transcendental (e.g. V_EXP) instructions may be scheduled across sched_barrier.
 
-  llvm.amdgcn.sched_group_barrier                  Creates schedule groups with specific properties to create custom scheduling
+  llvm.amdgcn.sched.group.barrier                  Creates schedule groups with specific properties to create custom scheduling
                                                    pipelines. The ordering between groups is enforced by the instruction scheduler.
                                                    The intrinsic applies to the code that preceeds the intrinsic. The intrinsic
                                                    takes three values that control the behavior of the schedule groups.
@@ -1369,7 +1369,7 @@ The AMDGPU backend implements the following LLVM IR intrinsics.
                                                    |  ``// 5 MFMA``
                                                    |  ``__builtin_amdgcn_sched_group_barrier(8, 5, 0)``
 
-  llvm.amdgcn.iglp_opt                             An **experimental** intrinsic for instruction group level parallelism. The intrinsic
+  llvm.amdgcn.iglp.opt                             An **experimental** intrinsic for instruction group level parallelism. The intrinsic
                                                    implements predefined intruction scheduling orderings. The intrinsic applies to the
                                                    surrounding scheduling region. The intrinsic takes a value that specifies the
                                                    strategy.  The compiler implements two strategies.
@@ -1422,6 +1422,19 @@ The AMDGPU backend implements the following LLVM IR intrinsics.
                                                    Returns a pair for the swapped registers. The first element of the return
                                                    corresponds to the swapped element of the first argument.
 
+  llvm.amdgcn.mov.dpp                              The llvm.amdgcn.mov.dpp.`<type>` intrinsic represents the mov.dpp operation in AMDGPU.
+                                                   This operation is being deprecated and can be replaced with llvm.amdgcn.update.dpp.
+
+  llvm.amdgcn.update.dpp                           The llvm.amdgcn.update.dpp.`<type>` intrinsic represents the update.dpp operation in AMDGPU.
+                                                   It takes an old value, a source operand, a DPP control operand, a row mask, a bank mask, and a bound control.
+                                                   Various data types are supported, including, bf16, f16, f32, f64, i16, i32, i64, p0, p3, p5, v2f16, v2f32, v2i16, v2i32, v2p0, v3i32, v4i32, v8f16.
+                                                   This operation is equivalent to a sequence of v_mov_b32 operations.
+                                                   It is preferred over llvm.amdgcn.mov.dpp.`<type>` for future use.
+                                                   `llvm.amdgcn.update.dpp.<type> <old> <src> <dpp_ctrl> <row_mask> <bank_mask> <bound_ctrl>`
+                                                   Should be equivalent to:
+                                                   - `v_mov_b32 <dest> <old>`
+                                                   - `v_mov_b32 <dest> <src> <dpp_ctrl> <row_mask> <bank_mask> <bound_ctrl>`
+
   ==============================================   ==========================================================
 
 .. TODO::
@@ -1536,167 +1549,180 @@ The AMDGPU backend supports the following LLVM IR attributes.
   .. table:: AMDGPU LLVM IR Attributes
      :name: amdgpu-llvm-ir-attributes-table
 
-     ======================================= ==========================================================
-     LLVM Attribute                          Description
-     ======================================= ==========================================================
-     "amdgpu-flat-work-group-size"="min,max" Specify the minimum and maximum flat work group sizes that
-                                             will be specified when the kernel is dispatched. Generated
-                                             by the ``amdgpu_flat_work_group_size`` CLANG attribute [CLANG-ATTR]_.
-                                             The IR implied default value is 1,1024. Clang may emit this attribute
-                                             with more restrictive bounds depending on language defaults.
-                                             If the actual block or workgroup size exceeds the limit at any point during
-                                             the execution, the behavior is undefined. For example, even if there is
-                                             only one active thread but the thread local id exceeds the limit, the
-                                             behavior is undefined.
-
-     "amdgpu-implicitarg-num-bytes"="n"      Number of kernel argument bytes to add to the kernel
-                                             argument block size for the implicit arguments. This
-                                             varies by OS and language (for OpenCL see
-                                             :ref:`opencl-kernel-implicit-arguments-appended-for-amdhsa-os-table`).
-     "amdgpu-num-sgpr"="n"                   Specifies the number of SGPRs to use. Generated by
-                                             the ``amdgpu_num_sgpr`` CLANG attribute [CLANG-ATTR]_.
-     "amdgpu-num-vgpr"="n"                   Specifies the number of VGPRs to use. Generated by the
-                                             ``amdgpu_num_vgpr`` CLANG attribute [CLANG-ATTR]_.
-     "amdgpu-waves-per-eu"="m,n"             Specify the minimum and maximum number of waves per
-                                             execution unit. Generated by the ``amdgpu_waves_per_eu``
-                                             CLANG attribute [CLANG-ATTR]_. This is an optimization hint,
-                                             and the backend may not be able to satisfy the request. If
-                                             the specified range is incompatible with the function's
-                                             "amdgpu-flat-work-group-size" value, the implied occupancy
-                                             bounds by the workgroup size takes precedence.
-
-     "amdgpu-ieee" true/false.               GFX6-GFX11 Only
-                                             Specify whether the function expects the IEEE field of the
-                                             mode register to be set on entry. Overrides the default for
-                                             the calling convention.
-     "amdgpu-dx10-clamp" true/false.         GFX6-GFX11 Only
-                                             Specify whether the function expects the DX10_CLAMP field of
-                                             the mode register to be set on entry. Overrides the default
-                                             for the calling convention.
-
-     "amdgpu-no-workitem-id-x"               Indicates the function does not depend on the value of the
-                                             llvm.amdgcn.workitem.id.x intrinsic. If a function is marked with this
-                                             attribute, or reached through a call site marked with this attribute, and
-                                             that intrinsic is called, the behavior of the program is undefined. (Whole-program
-                                             undefined behavior is used here because, for example, the absence of a required workitem
-                                             ID in the preloaded register set can mean that all other preloaded registers
-                                             are earlier than the compilation assumed they would be.) The backend can
-                                             generally infer this during code generation, so typically there is no
-                                             benefit to frontends marking functions with this.
-
-     "amdgpu-no-workitem-id-y"               The same as amdgpu-no-workitem-id-x, except for the
-                                             llvm.amdgcn.workitem.id.y intrinsic.
-
-     "amdgpu-no-workitem-id-z"               The same as amdgpu-no-workitem-id-x, except for the
-                                             llvm.amdgcn.workitem.id.z intrinsic.
-
-     "amdgpu-no-workgroup-id-x"              The same as amdgpu-no-workitem-id-x, except for the
-                                             llvm.amdgcn.workgroup.id.x intrinsic.
-
-     "amdgpu-no-workgroup-id-y"              The same as amdgpu-no-workitem-id-x, except for the
-                                             llvm.amdgcn.workgroup.id.y intrinsic.
-
-     "amdgpu-no-workgroup-id-z"              The same as amdgpu-no-workitem-id-x, except for the
-                                             llvm.amdgcn.workgroup.id.z intrinsic.
-
-     "amdgpu-no-dispatch-ptr"                The same as amdgpu-no-workitem-id-x, except for the
-                                             llvm.amdgcn.dispatch.ptr intrinsic.
-
-     "amdgpu-no-implicitarg-ptr"             The same as amdgpu-no-workitem-id-x, except for the
-                                             llvm.amdgcn.implicitarg.ptr intrinsic.
-
-     "amdgpu-no-dispatch-id"                 The same as amdgpu-no-workitem-id-x, except for the
-                                             llvm.amdgcn.dispatch.id intrinsic.
-
-     "amdgpu-no-queue-ptr"                   Similar to amdgpu-no-workitem-id-x, except for the
-                                             llvm.amdgcn.queue.ptr intrinsic. Note that unlike the other ABI hint
-                                             attributes, the queue pointer may be required in situations where the
-                                             intrinsic call does not directly appear in the program. Some subtargets
-                                             require the queue pointer for to handle some addrspacecasts, as well
-                                             as the llvm.amdgcn.is.shared, llvm.amdgcn.is.private, llvm.trap, and
-                                             llvm.debug intrinsics.
-
-     "amdgpu-no-hostcall-ptr"                Similar to amdgpu-no-implicitarg-ptr, except specific to the implicit
-                                             kernel argument that holds the pointer to the hostcall buffer. If this
-                                             attribute is absent, then the amdgpu-no-implicitarg-ptr is also removed.
-
-     "amdgpu-no-heap-ptr"                    Similar to amdgpu-no-implicitarg-ptr, except specific to the implicit
-                                             kernel argument that holds the pointer to an initialized memory buffer
-                                             that conforms to the requirements of the malloc/free device library V1
-                                             version implementation. If this attribute is absent, then the
-                                             amdgpu-no-implicitarg-ptr is also removed.
-
-     "amdgpu-no-multigrid-sync-arg"          Similar to amdgpu-no-implicitarg-ptr, except specific to the implicit
-                                             kernel argument that holds the multigrid synchronization pointer. If this
-                                             attribute is absent, then the amdgpu-no-implicitarg-ptr is also removed.
-
-     "amdgpu-no-default-queue"               Similar to amdgpu-no-implicitarg-ptr, except specific to the implicit
-                                             kernel argument that holds the default queue pointer. If this
-                                             attribute is absent, then the amdgpu-no-implicitarg-ptr is also removed.
-
-     "amdgpu-no-completion-action"           Similar to amdgpu-no-implicitarg-ptr, except specific to the implicit
-                                             kernel argument that holds the completion action pointer. If this
-                                             attribute is absent, then the amdgpu-no-implicitarg-ptr is also removed.
-
-     "amdgpu-lds-size"="min[,max]"           Min is the minimum number of bytes that will be allocated in the Local
-                                             Data Store at address zero. Variables are allocated within this frame
-                                             using absolute symbol metadata, primarily by the AMDGPULowerModuleLDS
-                                             pass. Optional max is the maximum number of bytes that will be allocated.
-                                             Note that min==max indicates that no further variables can be added to
-                                             the frame. This is an internal detail of how LDS variables are lowered,
-                                             language front ends should not set this attribute.
-
-     "amdgpu-gds-size"                       Bytes expected to be allocated at the start of GDS memory at entry.
-
-     "amdgpu-git-ptr-high"                   The hard-wired high half of the address of the global information table
-                                             for AMDPAL OS type. 0xffffffff represents no hard-wired high half, since
-                                             current hardware only allows a 16 bit value.
-
-     "amdgpu-32bit-address-high-bits"        Assumed high 32-bits for 32-bit address spaces which are really truncated
-                                             64-bit addresses (i.e., addrspace(6))
-
-     "amdgpu-color-export"                   Indicates shader exports color information if set to 1.
-                                             Defaults to 1 for :ref:`amdgpu_ps <amdgpu-cc>`, and 0 for other calling
-                                             conventions. Determines the necessity and type of null exports when a shader
-                                             terminates early by killing lanes.
-
-     "amdgpu-depth-export"                   Indicates shader exports depth information if set to 1. Determines the
-                                             necessity and type of null exports when a shader terminates early by killing
-                                             lanes. A depth-only shader will export to depth channel when no null export
-                                             target is available (GFX11+).
-
-     "InitialPSInputAddr"                    Set the initial value of the `spi_ps_input_addr` register for
-                                             :ref:`amdgpu_ps <amdgpu-cc>` shaders. Any bits enabled by this value will
-                                             be enabled in the final register value.
-
-     "amdgpu-wave-priority-threshold"        VALU instruction count threshold for adjusting wave priority. If exceeded,
-                                             temporarily raise the wave priority at the start of the shader function
-                                             until its last VMEM instructions to allow younger waves to issue their VMEM
-                                             instructions as well.
+     ============================================ ==========================================================
+     LLVM Attribute                               Description
+     ============================================ ==========================================================
+     "amdgpu-flat-work-group-size"="min,max"      Specify the minimum and maximum flat work group sizes that
+                                                  will be specified when the kernel is dispatched. Generated
+                                                  by the ``amdgpu_flat_work_group_size`` CLANG attribute [CLANG-ATTR]_.
+                                                  The IR implied default value is 1,1024. Clang may emit this attribute
+                                                  with more restrictive bounds depending on language defaults.
+                                                  If the actual block or workgroup size exceeds the limit at any point during
+                                                  the execution, the behavior is undefined. For example, even if there is
+                                                  only one active thread but the thread local id exceeds the limit, the
+                                                  behavior is undefined.
+
+     "amdgpu-implicitarg-num-bytes"="n"           Number of kernel argument bytes to add to the kernel
+                                                  argument block size for the implicit arguments. This
+                                                  varies by OS and language (for OpenCL see
+                                                  :ref:`opencl-kernel-implicit-arguments-appended-for-amdhsa-os-table`).
+     "amdgpu-num-sgpr"="n"                        Specifies the number of SGPRs to use. Generated by
+                                                  the ``amdgpu_num_sgpr`` CLANG attribute [CLANG-ATTR]_.
+     "amdgpu-num-vgpr"="n"                        Specifies the number of VGPRs to use. Generated by the
+                                                  ``amdgpu_num_vgpr`` CLANG attribute [CLANG-ATTR]_.
+     "amdgpu-waves-per-eu"="m,n"                  Specify the minimum and maximum number of waves per
+                                                  execution unit. Generated by the ``amdgpu_waves_per_eu``
+                                                  CLANG attribute [CLANG-ATTR]_. This is an optimization hint,
+                                                  and the backend may not be able to satisfy the request. If
+                                                  the specified range is incompatible with the function's
+                                                  "amdgpu-flat-work-group-size" value, the implied occupancy
+                                                  bounds by the workgroup size takes precedence.
+
+     "amdgpu-ieee" true/false.                    GFX6-GFX11 Only
+                                                  Specify whether the function expects the IEEE field of the
+                                                  mode register to be set on entry. Overrides the default for
+                                                  the calling convention.
+     "amdgpu-dx10-clamp" true/false.              GFX6-GFX11 Only
+                                                  Specify whether the function expects the DX10_CLAMP field of
+                                                  the mode register to be set on entry. Overrides the default
+                                                  for the calling convention.
+
+     "amdgpu-no-workitem-id-x"                    Indicates the function does not depend on the value of the
+                                                  llvm.amdgcn.workitem.id.x intrinsic. If a function is marked with this
+                                                  attribute, or reached through a call site marked with this attribute, and
+                                                  that intrinsic is called, the behavior of the program is undefined. (Whole-program
+                                                  undefined behavior is used here because, for example, the absence of a required workitem
+                                                  ID in the preloaded register set can mean that all other preloaded registers
+                                                  are earlier than the compilation assumed they would be.) The backend can
+                                                  generally infer this during code generation, so typically there is no
+                                                  benefit to frontends marking functions with this.
+
+     "amdgpu-no-workitem-id-y"                    The same as amdgpu-no-workitem-id-x, except for the
+                                                  llvm.amdgcn.workitem.id.y intrinsic.
+
+     "amdgpu-no-workitem-id-z"                    The same as amdgpu-no-workitem-id-x, except for the
+                                                  llvm.amdgcn.workitem.id.z intrinsic.
+
+     "amdgpu-no-workgroup-id-x"                   The same as amdgpu-no-workitem-id-x, except for the
+                                                  llvm.amdgcn.workgroup.id.x intrinsic.
+
+     "amdgpu-no-workgroup-id-y"                   The same as amdgpu-no-workitem-id-x, except for the
+                                                  llvm.amdgcn.workgroup.id.y intrinsic.
+
+     "amdgpu-no-workgroup-id-z"                   The same as amdgpu-no-workitem-id-x, except for the
+                                                  llvm.amdgcn.workgroup.id.z intrinsic.
+
+     "amdgpu-no-dispatch-ptr"                     The same as amdgpu-no-workitem-id-x, except for the
+                                                  llvm.amdgcn.dispatch.ptr intrinsic.
+
+     "amdgpu-no-implicitarg-ptr"                  The same as amdgpu-no-workitem-id-x, except for the
+                                                  llvm.amdgcn.implicitarg.ptr intrinsic.
+
+     "amdgpu-no-dispatch-id"                      The same as amdgpu-no-workitem-id-x, except for the
+                                                  llvm.amdgcn.dispatch.id intrinsic.
+
+     "amdgpu-no-queue-ptr"                        Similar to amdgpu-no-workitem-id-x, except for the
+                                                  llvm.amdgcn.queue.ptr intrinsic. Note that unlike the other ABI hint
+                                                  attributes, the queue pointer may be required in situations where the
+                                                  intrinsic call does not directly appear in the program. Some subtargets
+                                                  require the queue pointer for to handle some addrspacecasts, as well
+                                                  as the llvm.amdgcn.is.shared, llvm.amdgcn.is.private, llvm.trap, and
+                                                  llvm.debug intrinsics.
+
+     "amdgpu-no-hostcall-ptr"                     Similar to amdgpu-no-implicitarg-ptr, except specific to the implicit
+                                                  kernel argument that holds the pointer to the hostcall buffer. If this
+                                                  attribute is absent, then the amdgpu-no-implicitarg-ptr is also removed.
+
+     "amdgpu-no-heap-ptr"                         Similar to amdgpu-no-implicitarg-ptr, except specific to the implicit
+                                                  kernel argument that holds the pointer to an initialized memory buffer
+                                                  that conforms to the requirements of the malloc/free device library V1
+                                                  version implementation. If this attribute is absent, then the
+                                                  amdgpu-no-implicitarg-ptr is also removed.
+
+     "amdgpu-no-multigrid-sync-arg"               Similar to amdgpu-no-implicitarg-ptr, except specific to the implicit
+                                                  kernel argument that holds the multigrid synchronization pointer. If this
+                                                  attribute is absent, then the amdgpu-no-implicitarg-ptr is also removed.
+
+     "amdgpu-no-default-queue"                    Similar to amdgpu-no-implicitarg-ptr, except specific to the implicit
+                                                  kernel argument that holds the default queue pointer. If this
+                                                  attribute is absent, then the amdgpu-no-implicitarg-ptr is also removed.
+
+     "amdgpu-no-completion-action"                Similar to amdgpu-no-implicitarg-ptr, except specific to the implicit
+                                                  kernel argument that holds the completion action pointer. If this
+                                                  attribute is absent, then the amdgpu-no-implicitarg-ptr is also removed.
+
+     "amdgpu-lds-size"="min[,max]"                Min is the minimum number of bytes that will be allocated in the Local
+                                                  Data Store at address zero. Variables are allocated within this frame
+                                                  using absolute symbol metadata, primarily by the AMDGPULowerModuleLDS
+                                                  pass. Optional max is the maximum number of bytes that will be allocated.
+                                                  Note that min==max indicates that no further variables can be added to
+                                                  the frame. This is an internal detail of how LDS variables are lowered,
+                                                  language front ends should not set this attribute.
+
+     "amdgpu-gds-size"                            Bytes expected to be allocated at the start of GDS memory at entry.
+
+     "amdgpu-git-ptr-high"                        The hard-wired high half of the address of the global information table
+                                                  for AMDPAL OS type. 0xffffffff represents no hard-wired high half, since
+                                                  current hardware only allows a 16 bit value.
+
+     "amdgpu-32bit-address-high-bits"             Assumed high 32-bits for 32-bit address spaces which are really truncated
+                                                  64-bit addresses (i.e., addrspace(6))
+
+     "amdgpu-color-export"                        Indicates shader exports color information if set to 1.
+                                                  Defaults to 1 for :ref:`amdgpu_ps <amdgpu-cc>`, and 0 for other calling
+                                                  conventions. Determines the necessity and type of null exports when a shader
+                                                  terminates early by killing lanes.
+
+     "amdgpu-depth-export"                        Indicates shader exports depth information if set to 1. Determines the
+                                                  necessity and type of null exports when a shader terminates early by killing
+                                                  lanes. A depth-only shader will export to depth channel when no null export
+                                                  target is available (GFX11+).
+
+     "InitialPSInputAddr"                         Set the initial value of the `spi_ps_input_addr` register for
+                                                  :ref:`amdgpu_ps <amdgpu-cc>` shaders. Any bits enabled by this value will
+                                                  be enabled in the final register value.
+
+     "amdgpu-wave-priority-threshold"             VALU instruction count threshold for adjusting wave priority. If exceeded,
+                                                  temporarily raise the wave priority at the start of the shader function
+                                                  until its last VMEM instructions to allow younger waves to issue their VMEM
+                                                  instructions as well.
+
+     "amdgpu-memory-bound"                        Set internally by backend
+
+     "amdgpu-wave-limiter"                        Set internally by backend
+
+     "amdgpu-unroll-threshold"                    Set base cost threshold preference for loop unrolling within this function,
+                                                  default is 300. Actual threshold may be varied by per-loop metadata or
+                                                  reduced by heuristics.
+
+     "amdgpu-max-num-workgroups"="x,y,z"          Specify the maximum number of work groups for the kernel dispatch in the
+                                                  X, Y, and Z dimensions. Each number must be >= 1. Generated by the
+                                                  ``amdgpu_max_num_work_groups`` CLANG attribute [CLANG-ATTR]_. Clang only
+                                                  emits this attribute when all the three numbers are >= 1.
 
-     "amdgpu-memory-bound"                   Set internally by backend
+     "amdgpu-no-agpr"                             Indicates the function will not require allocating AGPRs. This is only
+                                                  relevant on subtargets with AGPRs. The behavior is undefined if a
+                                                  function which requires AGPRs is reached through any function marked
+                                                  with this attribute.
 
-     "amdgpu-wave-limiter"                   Set internally by backend
+     "amdgpu-hidden-argument"                     This attribute is used internally by the backend to mark function arguments
+                                                  as hidden. Hidden arguments are managed by the compiler and are not part of
+                                                  the explicit arguments supplied by the user.
 
-     "amdgpu-unroll-threshold"               Set base cost threshold preference for loop unrolling within this function,
-                                             default is 300. Actual threshold may be varied by per-loop metadata or
-                                             reduced by heuristics.
+     "amdgpu-sgpr-hazard-wait"                    Disabled SGPR hazard wait insertion if set to 0.
+                                                  Exists for testing performance impact of SGPR hazard waits only.
 
-     "amdgpu-max-num-workgroups"="x,y,z"     Specify the maximum number of work groups for the kernel dispatch in the
-                                             X, Y, and Z dimensions. Each number must be >= 1. Generated by the
-                                             ``amdgpu_max_num_work_groups`` CLANG attribute [CLANG-ATTR]_. Clang only
-                                             emits this attribute when all the three numbers are >= 1.
+     "amdgpu-sgpr-hazard-boundary-cull"           Enable insertion of SGPR hazard cull sequences at function call boundaries.
+                                                  Cull sequence reduces future hazard waits, but has a performance cost.
 
-     "amdgpu-no-agpr"                        Indicates the function will not require allocating AGPRs. This is only
-                                             relevant on subtargets with AGPRs. The behavior is undefined if a
-                                             function which requires AGPRs is reached through any function marked
-                                             with this attribute.
+     "amdgpu-sgpr-hazard-mem-wait-cull"           Enable insertion of SGPR hazard cull sequences before memory waits.
+                                                  Cull sequence reduces future hazard waits, but has a performance cost.
+                                                  Attempt to amortize cost by overlapping with memory accesses.
 
-     "amdgpu-hidden-argument"                This attribute is used internally by the backend to mark function arguments
-                                             as hidden. Hidden arguments are managed by the compiler and are not part of
-                                             the explicit arguments supplied by the user.
+     "amdgpu-sgpr-hazard-mem-wait-cull-threshold" Sets the number of active SGPR hazards that must be present before
+                                                  inserting a cull sequence at a memory wait.
 
-     ======================================= ==========================================================
+     ============================================ ==========================================================
 
 Calling Conventions
 ===================
@@ -18187,6 +18213,9 @@ terminated by an ``.end_amdhsa_kernel`` directive.
                                                                                                :ref:`amdgpu-amdhsa-compute_pgm_rsrc1-gfx6-gfx12-table`.
      ``.amdhsa_shared_vgpr_count``                            0                   GFX10-GFX11  Controls SHARED_VGPR_COUNT in
                                                                                                :ref:`amdgpu-amdhsa-compute_pgm_rsrc3-gfx10-gfx11-table`.
+     ``.amdhsa_inst_pref_size``                               0                   GFX11-GFX12  Controls INST_PREF_SIZE in
+                                                                                               :ref:`amdgpu-amdhsa-compute_pgm_rsrc3-gfx10-gfx11-table` or
+                                                                                               :ref:`amdgpu-amdhsa-compute_pgm_rsrc3-gfx12-table`
      ``.amdhsa_exception_fp_ieee_invalid_op``                 0                   GFX6-GFX12   Controls ENABLE_EXCEPTION_IEEE_754_FP_INVALID_OPERATION in
                                                                                                :ref:`amdgpu-amdhsa-compute_pgm_rsrc2-gfx6-gfx12-table`.
      ``.amdhsa_exception_fp_denorm_src``                      0                   GFX6-GFX12   Controls ENABLE_EXCEPTION_FP_DENORMAL_SOURCE in
diff --git a/llvm/docs/Contributing.rst b/llvm/docs/Contributing.rst
index 9311f39b6e697..5ee07fcec5bf9 100644
--- a/llvm/docs/Contributing.rst
+++ b/llvm/docs/Contributing.rst
@@ -73,15 +73,33 @@ recent commit:
 
   % git clang-format HEAD~1
 
-Note that this modifies the files, but doesn't commit them -- you'll likely want
-to run
+.. note::
+  For some patches, formatting them may add changes that obscure the intent of
+  the patch. For example, adding to an enum that was not previously formatted
+  may result in the entire enum being reformatted. This happens because not all
+  of the LLVM Project conforms to LLVM's clang-format style at this time.
+
+  If you think that this might be the case for your changes, or are unsure, we
+  recommend that you add the formatting changes as a **separate commit** within
+  the Pull Request.
+
+  Reviewers may request that this formatting commit be made into a separate Pull
+  Request that will be merged before your actual changes.
+
+  This means that if the formatting changes are the first commit, you will have
+  an easier time doing this. If they are not, that is ok too, but you will have
+  to do a bit more work to separate it out.
+
+Note that ``git clang-format`` modifies the files, but does not commit them --
+you will likely want to run one of the following to add the changes to a commit:
 
 .. code-block:: console
 
+  # To create a new commit.
+  % git commit -a
+  # To add to the most recent commit.
   % git commit --amend -a
 
-in order to update the last commit with all pending changes.
-
 .. note::
   If you don't already have ``clang-format`` or ``git clang-format`` installed
   on your system, the ``clang-format`` binary will be built alongside clang, and
diff --git a/llvm/docs/ConvergenceAndUniformity.rst b/llvm/docs/ConvergenceAndUniformity.rst
index 863cebd91a20b..bcc5b4b1cd41e 100644
--- a/llvm/docs/ConvergenceAndUniformity.rst
+++ b/llvm/docs/ConvergenceAndUniformity.rst
@@ -285,8 +285,8 @@ Dependence on Cycles Headers
 Contradictions in *convergence-before* are possible only between two
 nodes that are inside some cycle. The dynamic instances of such nodes
 may be interleaved in the same thread, and this interleaving may be
-different for different threads.
-
+different for different threads. Cycle headers serve as implicit
+*points of convergence* in the maximal converged-with relation.
 When a thread executes a node ``X`` once and then executes it again,
 it must have followed a closed path in the CFG that includes ``X``.
 Such a path must pass through the header of at least one cycle --- the
@@ -295,14 +295,6 @@ thread, two dynamic instances of ``X`` are either separated by the
 execution of at least one cycle header, or ``X`` itself is a cycle
 header.
 
-In reducible cycles (natural loops), each execution of the header is
-equivalent to the start of a new iteration of the cycle. But this
-analogy breaks down in the presence of explicit constraints on the
-converged-with relation, such as those described in :ref:`future
-work<convergence-note-convergence>`. Instead, cycle headers should be
-treated as implicit *points of convergence* in a maximal
-converged-with relation.
-
 Consider a sequence of nested cycles ``C1``, ``C2``, ..., ``Ck`` such
 that ``C1`` is the outermost cycle and ``Ck`` is the innermost cycle,
 with headers ``H1``, ``H2``, ..., ``Hk`` respectively. When a thread
diff --git a/llvm/docs/GitHub.rst b/llvm/docs/GitHub.rst
index 892b8abcc2d4e..b5b75db91e1c4 100644
--- a/llvm/docs/GitHub.rst
+++ b/llvm/docs/GitHub.rst
@@ -438,9 +438,13 @@ Releases
 Backporting Fixes to the Release Branches
 -----------------------------------------
 You can use special comments on issues or pull requests to make backport
-requests for the release branches.  This is done by making a comment containing
-the following command on any issue or pull request that has been added to one
-of the "X.Y.Z Release" milestones.
+requests for the release branches.  To do this, after your pull reuest has been
+merged:
+
+1. Edit "Milestone" at the right side of the isssue or pull request
+   to say "LLVM X.Y Release"
+
+2. Add a comment to it in the following format:
 
 ::
 
diff --git a/llvm/docs/HowToAddABuilder.rst b/llvm/docs/HowToAddABuilder.rst
index 3984ef692b6ca..fe79f6375568c 100644
--- a/llvm/docs/HowToAddABuilder.rst
+++ b/llvm/docs/HowToAddABuilder.rst
@@ -200,10 +200,9 @@ will:
 
 In order to use this "local testing" mode:
 
-* Within a checkout of `llvm-zorg <https://github.com/llvm/llvm-zorg>`_,
-  create and activate a Python `venv
+* Create and activate a Python `venv
   <https://docs.python.org/3/library/venv.html>`_ and install the necessary
-  dependencies.
+  dependencies. This step can be run from any directory.
 
     .. code-block:: bash
 
@@ -211,9 +210,22 @@ In order to use this "local testing" mode:
        source bbenv/bin/activate
        pip install buildbot{,-console-view,-grid-view,-waterfall-view,-worker,-www}==3.11.7 urllib3
 
-* Initialise the necessary buildmaster files, link to the configuration in
-  ``llvm-zorg`` and ask ``buildbot`` to check the configuration. This step can
-  be run from any directory.
+* If your system has Python 3.13 or newer you will need to additionally
+  install ``legacy-cgi`` and make a minor patch to the installed buildbot
+  package. This step does not need to be followed for earlier Python versions.
+
+    .. code-block:: bash
+
+       pip install legacy-cgi
+       sed -i \
+         -e 's/import pipes/import shlex/' \
+         -e 's/pipes\.quote/shlex.quote/' \
+         bbenv/lib/python3.13/site-packages/buildbot_worker/runprocess.py
+
+* Initialise the necessary buildmaster files, link to the configuration in a
+  local checkout out of `llvm-zorg <https://github.com/llvm/llvm-zorg>`_`, and
+  ask ``buildbot`` to check the configuration. This step can be run from any
+  directory.
 
     .. code-block:: bash
 
@@ -266,6 +278,15 @@ to the remote buildmaster by connecting to ``localhost:9900``:
 
        ssh -N -L 8011:localhost:8011 -L 9990:localhost:9990 username@buildmaster_server_address
 
+Be aware that some build configurations may checkout the current upstream
+``llvm-zorg`` repository in order to retrieve additional scripts used during
+the build process, meaning any local changes will not be reflected in this
+part of the build. If you wish to test changes to any of these scripts without
+committing them upstream, you will need to temporarily patch the builder logic
+in order to instead check out your own branch.
+Typically, ``addGetSourcecodeForProject`` from
+``zorg/buildbot/process/factory.py`` is used for this and you can edit the
+caller to specify your own ``repourl`` and/or ``branch`` keyword argument.
 
 Best Practices for Configuring a Fast Builder
 =============================================
diff --git a/llvm/docs/HowToReleaseLLVM.rst b/llvm/docs/HowToReleaseLLVM.rst
index 4274717dbdfa0..ca55ee573c002 100644
--- a/llvm/docs/HowToReleaseLLVM.rst
+++ b/llvm/docs/HowToReleaseLLVM.rst
@@ -349,8 +349,7 @@ Below are the rules regarding patching the release branch:
 
 #. *Bug fix releases* Patches should be limited to bug fixes or very safe
    and critical performance improvements.  Patches must maintain both API and
-   ABI compatibility with the previous major release.
-
+   ABI compatibility with the X.1.0 release.
 
 Release Final Tasks
 -------------------
diff --git a/llvm/docs/KernelInfo.rst b/llvm/docs/KernelInfo.rst
new file mode 100644
index 0000000000000..dac642f1ffc65
--- /dev/null
+++ b/llvm/docs/KernelInfo.rst
@@ -0,0 +1,63 @@
+==========
+KernelInfo
+==========
+
+.. contents::
+   :local:
+
+Introduction
+============
+
+This LLVM IR pass reports various statistics for codes compiled for GPUs.  The
+goal of these statistics is to help identify bad code patterns and ways to
+mitigate them.  The pass operates at the LLVM IR level so that it can, in
+theory, support any LLVM-based compiler for programming languages supporting
+GPUs.
+
+By default, the pass runs at the end of LTO, and options like
+``-Rpass=kernel-info`` enable its remarks.  Example ``opt`` and ``clang``
+command lines appear in the next section.
+
+Remarks include summary statistics (e.g., total size of static allocas) and
+individual occurrences (e.g., source location of each alloca).  Examples of the
+output appear in tests in `llvm/test/Analysis/KernelInfo`.
+
+Example Command Lines
+=====================
+
+To analyze a C program as it appears to an LLVM GPU backend at the end of LTO:
+
+.. code-block:: shell
+
+  $ clang -O2 -g -fopenmp --offload-arch=native test.c -foffload-lto \
+      -Rpass=kernel-info
+
+To analyze specified LLVM IR, perhaps previously generated by something like
+``clang -save-temps -g -fopenmp --offload-arch=native test.c``:
+
+.. code-block:: shell
+
+  $ opt -disable-output test-openmp-nvptx64-nvidia-cuda-sm_70.bc \
+      -pass-remarks=kernel-info -passes=kernel-info
+
+When specifying an LLVM pass pipeline on the command line, ``kernel-info`` still
+runs at the end of LTO by default.  ``-no-kernel-info-end-lto`` disables that
+behavior so you can position ``kernel-info`` explicitly:
+
+.. code-block:: shell
+
+  $ clang -O2 -g -fopenmp --offload-arch=native test.c -foffload-lto \
+      -Rpass=kernel-info \
+      -Xoffload-linker --lto-newpm-passes='lto<O2>'
+
+  $ clang -O2 -g -fopenmp --offload-arch=native test.c -foffload-lto \
+      -Rpass=kernel-info -mllvm -no-kernel-info-end-lto \
+      -Xoffload-linker --lto-newpm-passes='module(kernel-info),lto<O2>'
+
+  $ opt -disable-output test-openmp-nvptx64-nvidia-cuda-sm_70.bc \
+      -pass-remarks=kernel-info \
+      -passes='lto<O2>'
+
+  $ opt -disable-output test-openmp-nvptx64-nvidia-cuda-sm_70.bc \
+      -pass-remarks=kernel-info -no-kernel-info-end-lto \
+      -passes='module(kernel-info),lto<O2>'
diff --git a/llvm/docs/LangRef.rst b/llvm/docs/LangRef.rst
index b922636d6c914..2d72e548ec82a 100644
--- a/llvm/docs/LangRef.rst
+++ b/llvm/docs/LangRef.rst
@@ -191,7 +191,7 @@ symbol table entries. Here is an example of the "hello world" module:
     @.str = private unnamed_addr constant [13 x i8] c"hello world\0A\00"
 
     ; External declaration of the puts function
-    declare i32 @puts(ptr nocapture) nounwind
+    declare i32 @puts(ptr captures(none)) nounwind
 
     ; Definition of main function
     define i32 @main() {
@@ -1170,7 +1170,7 @@ spaces. For example:
 .. code-block:: llvm
 
     ; On function declarations/definitions:
-    declare i32 @printf(ptr noalias nocapture, ...)
+    declare i32 @printf(ptr noalias captures(none), ...)
     declare i32 @atoi(i8 zeroext)
     declare signext i8 @returns_signed_char()
     define void @baz(i32 "amdgpu-flat-work-group-size"="1,256" %x)
@@ -1380,11 +1380,10 @@ Currently, only the following parameter attributes are defined:
     memory locations that are *modified*, by any means, during the execution of
     the function. If there are other accesses not based on the argument or
     return value, the behavior is undefined. The attribute on a return value
-    also has additional semantics described below. The caller shares the
-    responsibility with the callee for described below. The caller shares the
-    responsibility with the callee for ensuring that these requirements are met.
-    For further details, please see the discussion of the NoAlias response in
-    :ref:`alias analysis <Must, May,  or No>`.
+    also has additional semantics, as described below. Both the caller and the
+    callee share the responsibility of ensuring that these requirements are
+    met. For further details, please see the discussion of the NoAlias response
+    in :ref:`alias analysis <Must, May,  or No>`.
 
     Note that this definition of ``noalias`` is intentionally similar
     to the definition of ``restrict`` in C99 for function arguments.
@@ -1433,24 +1432,6 @@ Currently, only the following parameter attributes are defined:
       is captured through the return value, and additionally whether the pointer
       is null is captured in some other way.
 
-.. _nocapture:
-
-``nocapture``
-    This indicates that the callee does not :ref:`capture <pointercapture>` the
-    pointer. This is not a valid attribute for return values.
-    This attribute applies only to the particular copy of the pointer passed in
-    this argument. A caller could pass two copies of the same pointer with one
-    being annotated nocapture and the other not, and the callee could validly
-    capture through the non annotated parameter.
-
-.. code-block:: llvm
-
-    define void @f(ptr nocapture %a, ptr %b) {
-      ; (capture %b)
-    }
-
-    call void @f(ptr @glb, ptr @glb) ; well-defined
-
 ``nofree``
     This indicates that callee does not free the pointer argument. This is not
     a valid attribute for return values.
@@ -1726,6 +1707,10 @@ Currently, only the following parameter attributes are defined:
     and negative values are allowed in case the argument points partway into
     an allocation. An empty list is not allowed.
 
+    On a ``byval`` argument, ``initializes`` refers to the given parts of the
+    callee copy being overwritten. A ``byval`` callee can never initialize the
+    original caller memory passed to the ``byval`` argument.
+
 ``dead_on_unwind``
     At a high level, this attribute indicates that the pointer argument is dead
     if the call unwinds, in the sense that the caller will not depend on the
@@ -13924,7 +13909,7 @@ Syntax:
 
       declare <pointer type>
         @llvm.experimental.gc.get.pointer.base(
-          <pointer type> readnone nocapture %derived_ptr)
+          <pointer type> readnone captures(none) %derived_ptr)
           nounwind willreturn memory(none)
 
 Overview:
@@ -13962,7 +13947,7 @@ Syntax:
 
       declare i64
         @llvm.experimental.gc.get.pointer.offset(
-          <pointer type> readnone nocapture %derived_ptr)
+          <pointer type> readnone captures(none) %derived_ptr)
           nounwind willreturn memory(none)
 
 Overview:
@@ -16141,6 +16126,111 @@ of the argument.
 When specified with the fast-math-flag 'afn', the result may be approximated
 using a less accurate calculation.
 
+'``llvm.sincospi.*``' Intrinsic
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Syntax:
+"""""""
+
+This is an overloaded intrinsic. You can use ``llvm.sincospi`` on any
+floating-point or vector of floating-point type. Not all targets support
+all types however.
+
+::
+
+      declare { float, float }          @llvm.sincospi.f32(float  %Val)
+      declare { double, double }        @llvm.sincospi.f64(double %Val)
+      declare { x86_fp80, x86_fp80 }    @llvm.sincospi.f80(x86_fp80  %Val)
+      declare { fp128, fp128 }          @llvm.sincospi.f128(fp128 %Val)
+      declare { ppc_fp128, ppc_fp128 }  @llvm.sincospi.ppcf128(ppc_fp128  %Val)
+      declare { <4 x float>, <4 x float> } @llvm.sincospi.v4f32(<4 x float>  %Val)
+
+Overview:
+"""""""""
+
+The '``llvm.sincospi.*``' intrinsics returns the sine and cosine of pi*operand.
+
+Arguments:
+""""""""""
+
+The argument is a :ref:`floating-point <t_floating>` value or
+:ref:`vector <t_vector>` of floating-point values. Returns two values matching
+the argument type in a struct.
+
+Semantics:
+""""""""""
+
+This is equivalent to the ``llvm.sincos.*`` intrinsic where the argument has been
+multiplied by pi, however, it computes the result more accurately especially
+for large input values.
+
+.. note::
+
+  Currently, the default lowering of this intrinsic relies on the ``sincospi[f|l]``
+  functions being available in the target's runtime (e.g. libc).
+
+When specified with the fast-math-flag 'afn', the result may be approximated
+using a less accurate calculation.
+
+'``llvm.modf.*``' Intrinsic
+^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Syntax:
+"""""""
+
+This is an overloaded intrinsic. You can use ``llvm.modf`` on any floating-point
+or vector of floating-point type. However, not all targets support all types.
+
+::
+
+ declare { float, float }             @llvm.modf.f32(float  %Val)
+ declare { double, double }           @llvm.modf.f64(double %Val)
+ declare { x86_fp80, x86_fp80 }       @llvm.modf.f80(x86_fp80  %Val)
+ declare { fp128, fp128 }             @llvm.modf.f128(fp128 %Val)
+ declare { ppc_fp128, ppc_fp128 }     @llvm.modf.ppcf128(ppc_fp128  %Val)
+ declare { <4 x float>, <4 x float> } @llvm.modf.v4f32(<4 x float>  %Val)
+
+Overview:
+"""""""""
+
+The '``llvm.modf.*``' intrinsics return the operand's integral and fractional
+parts.
+
+Arguments:
+""""""""""
+
+The argument is a :ref:`floating-point <t_floating>` value or
+:ref:`vector <t_vector>` of floating-point values. Returns two values matching
+the argument type in a struct.
+
+Semantics:
+""""""""""
+
+Return the same values as a corresponding libm '``modf``' function without
+trapping or setting ``errno``.
+
+The first result is the fractional part of the operand and the second result is
+the integral part of the operand. Both results have the same sign as the operand.
+
+Not including exceptional inputs (listed below), ``llvm.modf.*`` is semantically
+equivalent to:
+
+::
+
+  %fp = frem <fptype> %x, 1.0  ; Fractional part
+  %ip = fsub <fptype> %x, %fp  ; Integral part
+
+(assuming no floating-point precision errors)
+
+If the argument is a zero, returns a zero with the same sign for both the
+fractional and integral parts.
+
+If the argument is an infinity, returns a fractional part of zero with the same
+sign, and infinity with the same sign as the integral part.
+
+When specified with the fast-math-flag 'afn', the result may be approximated
+using a less accurate calculation.
+
 '``llvm.pow.*``' Intrinsic
 ^^^^^^^^^^^^^^^^^^^^^^^^^^
 
@@ -26223,7 +26313,7 @@ Syntax:
 
 ::
 
-      declare void @llvm.lifetime.start(i64 <size>, ptr nocapture <ptr>)
+      declare void @llvm.lifetime.start(i64 <size>, ptr captures(none) <ptr>)
 
 Overview:
 """""""""
@@ -26273,7 +26363,7 @@ Syntax:
 
 ::
 
-      declare void @llvm.lifetime.end(i64 <size>, ptr nocapture <ptr>)
+      declare void @llvm.lifetime.end(i64 <size>, ptr captures(none) <ptr>)
 
 Overview:
 """""""""
@@ -26313,7 +26403,7 @@ This is an overloaded intrinsic. The memory object can belong to any address spa
 
 ::
 
-      declare ptr @llvm.invariant.start.p0(i64 <size>, ptr nocapture <ptr>)
+      declare ptr @llvm.invariant.start.p0(i64 <size>, ptr captures(none) <ptr>)
 
 Overview:
 """""""""
@@ -26344,7 +26434,7 @@ This is an overloaded intrinsic. The memory object can belong to any address spa
 
 ::
 
-      declare void @llvm.invariant.end.p0(ptr <start>, i64 <size>, ptr nocapture <ptr>)
+      declare void @llvm.invariant.end.p0(ptr <start>, i64 <size>, ptr captures(none) <ptr>)
 
 Overview:
 """""""""
diff --git a/llvm/docs/LibFuzzer.rst b/llvm/docs/LibFuzzer.rst
index 9e34530e7e3e7..2137740896ddb 100644
--- a/llvm/docs/LibFuzzer.rst
+++ b/llvm/docs/LibFuzzer.rst
@@ -354,16 +354,18 @@ Output
 
 During operation the fuzzer prints information to ``stderr``, for example::
 
-  INFO: Seed: 1523017872
-  INFO: Loaded 1 modules (16 guards): [0x744e60, 0x744ea0),
-  INFO: -max_len is not provided, using 64
+  INFO: Running with entropic power schedule (0xFF, 100).
+  INFO: Seed: 1434179311
+  INFO: Loaded 1 modules   (8 inline 8-bit counters): 8 [0x5f03d189be90, 0x5f03d189be98),
+  INFO: Loaded 1 PC tables (8 PCs): 8 [0x5f03d189be98,0x5f03d189bf18),
+  INFO: -max_len is not provided; libFuzzer will not generate inputs larger than 4096 bytes
   INFO: A corpus is not provided, starting from an empty corpus
-  #0	READ units: 1
-  #1	INITED cov: 3 ft: 2 corp: 1/1b exec/s: 0 rss: 24Mb
-  #3811	NEW    cov: 4 ft: 3 corp: 2/2b exec/s: 0 rss: 25Mb L: 1 MS: 5 ChangeBit-ChangeByte-ChangeBit-ShuffleBytes-ChangeByte-
-  #3827	NEW    cov: 5 ft: 4 corp: 3/4b exec/s: 0 rss: 25Mb L: 2 MS: 1 CopyPart-
-  #3963	NEW    cov: 6 ft: 5 corp: 4/6b exec/s: 0 rss: 25Mb L: 2 MS: 2 ShuffleBytes-ChangeBit-
-  #4167	NEW    cov: 7 ft: 6 corp: 5/9b exec/s: 0 rss: 25Mb L: 3 MS: 1 InsertByte-
+  #2      INITED cov: 2 ft: 2 corp: 1/1b exec/s: 0 rss: 31Mb
+  #144    NEW    cov: 3 ft: 3 corp: 2/2b lim: 4 exec/s: 0 rss: 31Mb L: 1/1 MS: 2 ChangeByte-ChangeByte-
+  #157    NEW    cov: 4 ft: 4 corp: 3/4b lim: 4 exec/s: 0 rss: 31Mb L: 2/2 MS: 3 CrossOver-ChangeBit-CrossOver-
+  #1345   NEW    cov: 5 ft: 5 corp: 4/8b lim: 14 exec/s: 0 rss: 32Mb L: 4/4 MS: 3 InsertByte-ChangeBit-CrossOver-
+  #1696   NEW    cov: 6 ft: 6 corp: 5/10b lim: 17 exec/s: 0 rss: 32Mb L: 2/4 MS: 1 EraseBytes-
+  #1832   REDUCE cov: 6 ft: 6 corp: 5/9b lim: 17 exec/s: 0 rss: 32Mb L: 3/3 MS: 1 EraseBytes-
   ...
 
 The early parts of the output include information about the fuzzer options and
@@ -407,7 +409,7 @@ Each output line also reports the following statistics (when non-zero):
 ``corp:``
   Number of entries in the current in-memory test corpus and its size in bytes.
 ``lim:``
-  Current limit on the length of new entries in the corpus.  Increases over time
+  Current limit on the length of new entries in the corpus. Increases over time
   until the max length (``-max_len``) is reached.
 ``exec/s:``
   Number of fuzzer iterations per second.
@@ -418,7 +420,8 @@ For ``NEW`` and ``REDUCE`` events, the output line also includes information
 about the mutation operation that produced the new input:
 
 ``L:``
-  Size of the new input in bytes.
+  Size of the new/reduced input in bytes and the size of the largest input
+  in current in-memory test corpus.
 ``MS: <n> <operations>``
   Count and list of the mutation operations used to generate the input.
 
@@ -453,19 +456,26 @@ A simple function that does something interesting if it receives the input
 
 You should get an error pretty quickly::
 
-  INFO: Seed: 1523017872
-  INFO: Loaded 1 modules (16 guards): [0x744e60, 0x744ea0),
-  INFO: -max_len is not provided, using 64
+  INFO: Running with entropic power schedule (0xFF, 100).
+  INFO: Seed: 1434179311
+  INFO: Loaded 1 modules   (8 inline 8-bit counters): 8 [0x5f03d189be90, 0x5f03d189be98),
+  INFO: Loaded 1 PC tables (8 PCs): 8 [0x5f03d189be98,0x5f03d189bf18),
+  INFO: -max_len is not provided; libFuzzer will not generate inputs larger than 4096 bytes
   INFO: A corpus is not provided, starting from an empty corpus
-  #0	READ units: 1
-  #1	INITED cov: 3 ft: 2 corp: 1/1b exec/s: 0 rss: 24Mb
-  #3811	NEW    cov: 4 ft: 3 corp: 2/2b exec/s: 0 rss: 25Mb L: 1 MS: 5 ChangeBit-ChangeByte-ChangeBit-ShuffleBytes-ChangeByte-
-  #3827	NEW    cov: 5 ft: 4 corp: 3/4b exec/s: 0 rss: 25Mb L: 2 MS: 1 CopyPart-
-  #3963	NEW    cov: 6 ft: 5 corp: 4/6b exec/s: 0 rss: 25Mb L: 2 MS: 2 ShuffleBytes-ChangeBit-
-  #4167	NEW    cov: 7 ft: 6 corp: 5/9b exec/s: 0 rss: 25Mb L: 3 MS: 1 InsertByte-
-  ==31511== ERROR: libFuzzer: deadly signal
+  #2      INITED cov: 2 ft: 2 corp: 1/1b exec/s: 0 rss: 31Mb
+  #144    NEW    cov: 3 ft: 3 corp: 2/2b lim: 4 exec/s: 0 rss: 31Mb L: 1/1 MS: 2 ChangeByte-ChangeByte-
+  #157    NEW    cov: 4 ft: 4 corp: 3/4b lim: 4 exec/s: 0 rss: 31Mb L: 2/2 MS: 3 CrossOver-ChangeBit-CrossOver-
+  #1345   NEW    cov: 5 ft: 5 corp: 4/8b lim: 14 exec/s: 0 rss: 32Mb L: 4/4 MS: 3 InsertByte-ChangeBit-CrossOver-
+  #1696   NEW    cov: 6 ft: 6 corp: 5/10b lim: 17 exec/s: 0 rss: 32Mb L: 2/4 MS: 1 EraseBytes-
+  #1832   REDUCE cov: 6 ft: 6 corp: 5/9b lim: 17 exec/s: 0 rss: 32Mb L: 3/3 MS: 1 EraseBytes-
+  ==840148== ERROR: libFuzzer: deadly signal
   ...
-  artifact_prefix='./'; Test unit written to ./crash-b13e8756b13a00cf168300179061fb4b91fefbed
+  SUMMARY: libFuzzer: deadly signal
+  MS: 2 CopyPart-ChangeByte-; base unit: dbee5f8c7a5da845446e75b4a5708e74428b520a
+  0x48,0x49,0x21,
+  HI!
+  artifact_prefix='./'; Test unit written to ./crash-7a8dc3985d2a90fb6e62e94910fc11d31949c348
+  Base64: SEkh
 
 
 More examples
diff --git a/llvm/docs/NVPTXUsage.rst b/llvm/docs/NVPTXUsage.rst
index 64dd2b84a1763..1680b11433537 100644
--- a/llvm/docs/NVPTXUsage.rst
+++ b/llvm/docs/NVPTXUsage.rst
@@ -581,6 +581,46 @@ prefetched in terms of bytes and it must be a multiple of 16.
 For more information, refer PTX ISA
 `<https://docs.nvidia.com/cuda/parallel-thread-execution/#data-movement-and-conversion-instructions-cp-async-bulk-prefetch>`_.
 
+'``llvm.nvvm.prefetch.*``'
+^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Syntax:
+"""""""
+
+.. code-block:: llvm
+
+  declare void  @llvm.nvvm.prefetch.local.L1.evictnormal(ptr addrspace(5) %local_ptr)
+  declare void  @llvm.nvvm.prefetch.local.L2.evictnormal(ptr addrspace(5) %local_ptr)
+  
+  declare void  @llvm.nvvm.prefetch.global.L1.evictnormal(ptr addrspace(1) %global_ptr)
+  declare void  @llvm.nvvm.prefetch.global.L2.evictnormal(ptr addrspace(1) %global_ptr)
+  declare void  @llvm.nvvm.prefetch.global.L1.evictlast(ptr addrspace(1) %global_ptr)
+  declare void  @llvm.nvvm.prefetch.global.L2.evictlast(ptr addrspace(1) %global_ptr)
+  
+  declare void  @llvm.nvvm.prefetch.L1.evictnormal(ptr %ptr)
+  declare void  @llvm.nvvm.prefetch.L2.evictnormal(ptr %ptr)
+  
+  declare void  @llvm.nvvm.prefetchu.L1.evictnormal(ptr %ptr)
+
+Overview:
+"""""""""
+
+The '``@llvm.nvvm.prefetch.*``' and '``@llvm.nvvm.prefetchu.*``' intrinsic
+correspond to the '``prefetch.*``;' and '``prefetchu.*``' family of PTX instructions. 
+The '``prefetch.*``' instructions bring the cache line containing the
+specified address in global or local memory address space into the 
+specified cache level (L1 or L2). The '`prefetchu.*``' instruction brings the cache line 
+containing the specified generic address into the specified uniform cache level.
+If no address space is specified, it is assumed to be generic address. The intrinsic 
+uses and eviction priority which can be accessed by the '``.level::eviction_priority``' modifier.
+
+* A prefetch to a shared memory location performs no operation.
+* A prefetch into the uniform cache requires a generic address, 
+  and no operation occurs if the address maps to a const, local, or shared memory location.
+
+For more information, refer to the PTX ISA
+`<https://docs.nvidia.com/cuda/parallel-thread-execution/#data-movement-and-conversion-instructions-prefetch-prefetchu>`_.
+
 '``llvm.nvvm.cp.async.bulk.tensor.g2s.tile.[1-5]d``'
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
@@ -962,6 +1002,179 @@ The ``griddepcontrol`` intrinsics allows the dependent grids and prerequisite gr
 For more information, refer 
 `PTX ISA <https://docs.nvidia.com/cuda/parallel-thread-execution/#parallel-synchronization-and-communication-instructions-griddepcontrol>`__.
 
+TCGEN05 family of Intrinsics
+----------------------------
+
+The llvm.nvvm.tcgen05.* intrinsics model the TCGEN05 family of instructions
+exposed by PTX. These intrinsics use 'Tensor Memory' (henceforth ``tmem``).
+NVPTX represents this memory using ``addrspace(6)`` and is always 32-bits.
+
+For more information, refer to the PTX ISA
+`<https://docs.nvidia.com/cuda/parallel-thread-execution/#tensor-memory>`_.
+
+The tensor-memory pointers may only be used with the tcgen05 intrinsics.
+There are specialized load/store instructions provided (tcgen05.ld/st) to
+work with tensor-memory.
+
+See the PTX ISA for more information on tensor-memory load/store instructions
+`<https://docs.nvidia.com/cuda/parallel-thread-execution/#tensor-memory-and-register-load-store-instructions>`_.
+
+'``llvm.nvvm.tcgen05.alloc``'
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Syntax:
+"""""""
+
+.. code-block:: llvm
+
+  declare void @llvm.nvvm.tcgen05.alloc.cg1(ptr %dst, i32 %ncols)
+  declare void @llvm.nvvm.tcgen05.alloc.cg2(ptr %dst, i32 %ncols)
+  declare void @llvm.nvvm.tcgen05.alloc.shared.cg1(ptr addrspace(3) %dst, i32 %ncols)
+  declare void @llvm.nvvm.tcgen05.alloc.shared.cg2(ptr addrspace(3) %dst, i32 %ncols)
+
+Overview:
+"""""""""
+
+The '``@llvm.nvvm.tcgen05.alloc.*``' intrinsics correspond to the
+``tcgen05.alloc.cta_group*.sync.aligned.b32`` family of PTX instructions.
+The ``tcgen05.alloc`` is a potentially blocking instruction which dynamically
+allocates the specified number of columns in the Tensor Memory and writes
+the address of the allocated Tensor Memory into shared memory at the
+location specified by ``%dst``. The 32-bit operand ``%ncols`` specifies
+the number of columns to be allocated and it must be a power-of-two.
+The ``.shared`` variant explicitly uses shared memory address space for
+the ``%dst`` operand. The ``.cg1`` and ``.cg2`` variants generate
+``cta_group::1`` and ``cta_group::2`` variants of the instruction respectively.
+
+For more information, refer to the PTX ISA
+`<https://docs.nvidia.com/cuda/parallel-thread-execution/#tensor-memory-allocation-and-management-instructions>`_.
+
+'``llvm.nvvm.tcgen05.dealloc``'
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Syntax:
+"""""""
+
+.. code-block:: llvm
+
+  declare void @llvm.nvvm.tcgen05.dealloc.cg1(ptr addrspace(6) %tmem_addr, i32 %ncols)
+  declare void @llvm.nvvm.tcgen05.dealloc.cg2(ptr addrspace(6) %tmem_addr, i32 %ncols)
+
+Overview:
+"""""""""
+
+The '``@llvm.nvvm.tcgen05.dealloc.*``' intrinsics correspond to the
+``tcgen05.dealloc.*`` set of PTX instructions. The ``tcgen05.dealloc``
+instructions deallocates the Tensor Memory specified by the Tensor Memory
+address ``%tmem_addr``. The operand ``%tmem_addr`` must point to a previous
+Tensor Memory allocation. The 32-bit operand ``%ncols`` specifies the number
+of columns to be de-allocated. The ``.cg1`` and ``.cg2`` variants generate
+``cta_group::1`` and ``cta_group::2`` variants of the instruction respectively.
+
+For more information, refer to the PTX ISA
+`<https://docs.nvidia.com/cuda/parallel-thread-execution/#tensor-memory-allocation-and-management-instructions>`_.
+
+'``llvm.nvvm.tcgen05.relinq.alloc.permit``'
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Syntax:
+"""""""
+
+.. code-block:: llvm
+
+  declare void @llvm.nvvm.tcgen05.relinq.alloc.permit.cg1()
+  declare void @llvm.nvvm.tcgen05.relinq.alloc.permit.cg2()
+
+Overview:
+"""""""""
+
+The '``@llvm.nvvm.tcgen05.relinq.alloc.permit.*``' intrinsics correspond
+to the ``tcgen05.relinquish_alloc_permit.*`` set of PTX instructions.
+This instruction specifies that the CTA of the executing thread is
+relinquishing the right to allocate Tensor Memory. So, it is illegal
+for a CTA to perform ``tcgen05.alloc`` after any of its constituent
+threads execute ``tcgen05.relinquish_alloc_permit``. The ``.cg1``
+and ``.cg2`` variants generate ``cta_group::1`` and ``cta_group::2``
+flavors of the instruction respectively.
+
+For more information, refer to the PTX ISA
+`<https://docs.nvidia.com/cuda/parallel-thread-execution/#tensor-memory-allocation-and-management-instructions>`_.
+
+'``llvm.nvvm.tcgen05.commit``'
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Syntax:
+"""""""
+
+.. code-block:: llvm
+
+  declare void @llvm.nvvm.tcgen05.commit.{cg1,cg2}(ptr %mbar)
+  declare void @llvm.nvvm.tcgen05.commit.shared.{cg1,cg2}(ptr addrspace(3) %mbar)
+  declare void @llvm.nvvm.tcgen05.commit.mc.{cg1,cg2}(ptr %mbar, i16 %mc)
+  declare void @llvm.nvvm.tcgen05.commit.mc.shared.{cg1,cg2}(ptr addrspace(3) %mbar, i16 %mc)
+
+Overview:
+"""""""""
+
+The '``@llvm.nvvm.tcgen05.commit.*``' intrinsics correspond to the
+``tcgen05.commit.{cg1/cg2}.mbarrier::arrive::one.*`` set of PTX instructions.
+The ``tcgen05.commit`` is an asynchronous instruction which makes the mbarrier
+object (``%mbar``) track the completion of all prior asynchronous tcgen05 operations.
+The ``.mc`` variants allow signaling on the mbarrier objects of multiple CTAs
+(specified by ``%mc``) in the cluster. The ``.cg1`` and ``.cg2`` variants generate
+``cta_group::1`` and ``cta_group::2`` flavors of the instruction respectively.
+
+For more information, refer to the PTX ISA
+`<https://docs.nvidia.com/cuda/parallel-thread-execution/#tcgen-async-sync-operations-commit>`_.
+
+'``llvm.nvvm.tcgen05.wait``'
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Syntax:
+"""""""
+
+.. code-block:: llvm
+
+  declare void @llvm.nvvm.tcgen05.wait.ld()
+  declare void @llvm.nvvm.tcgen05.wait.st()
+
+Overview:
+"""""""""
+
+The '``@llvm.nvvm.tcgen05.wait.ld/st``' intrinsics correspond to
+the ``tcgen05.wait::{ld/st}.sync.aligned`` pair of PTX instructions.
+The ``tcgen05.wait::ld`` causes the executing thread to block until
+all prior ``tcgen05.ld`` operations issued by the executing thread
+have completed. The ``tcgen05.wait::st`` causes the executing thread
+to block until all prior ``tcgen05.st`` operations issued by the
+executing thread have completed.
+
+For more information, refer to the PTX ISA
+`<https://docs.nvidia.com/cuda/parallel-thread-execution/#tcgen05-instructions-tcgen05-wait>`_.
+
+'``llvm.nvvm.tcgen05.fence``'
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Syntax:
+"""""""
+
+.. code-block:: llvm
+
+  declare void @llvm.nvvm.tcgen05.fence.before.thread.sync()
+  declare void @llvm.nvvm.tcgen05.fence.after.thread.sync()
+
+Overview:
+"""""""""
+
+The '``@llvm.nvvm.tcgen05.fence.*``' intrinsics correspond to
+the ``tcgen05.fence::{before/after}_thread_sync`` pair of PTX instructions.
+These instructions act as code motion fences for asynchronous tcgen05
+operations.
+
+For more information, refer to the PTX ISA
+`<https://docs.nvidia.com/cuda/parallel-thread-execution/#tensorcore-5th-generation-instructions-tcgen05-fence>`_.
+
+
 Other Intrinsics
 ----------------
 
diff --git a/llvm/docs/Passes.rst b/llvm/docs/Passes.rst
index 5e436db62be3a..435e748199b87 100644
--- a/llvm/docs/Passes.rst
+++ b/llvm/docs/Passes.rst
@@ -5,6 +5,11 @@ LLVM's Analysis and Transform Passes
 .. contents::
     :local:
 
+.. toctree::
+   :hidden:
+
+   KernelInfo
+
 Introduction
 ============
 .. warning:: This document is not updated frequently, and the list of passes
@@ -148,6 +153,12 @@ This pass collects the count of all instructions and reports them.
 Bookkeeping for "interesting" users of expressions computed from induction
 variables.
 
+``kernel-info``: GPU Kernel Info
+--------------------------------
+
+Reports various statistics for codes compiled for GPUs.  This pass is
+:doc:`documented separately<KernelInfo>`.
+
 ``lazy-value-info``: Lazy Value Information Analysis
 ----------------------------------------------------
 
diff --git a/llvm/docs/RISCVUsage.rst b/llvm/docs/RISCVUsage.rst
index c83fd1db0ba9b..09fb59f94e84d 100644
--- a/llvm/docs/RISCVUsage.rst
+++ b/llvm/docs/RISCVUsage.rst
@@ -433,7 +433,7 @@ The current vendor extensions supported are:
   LLVM implements `version 0.2 of the Qualcomm uC Arithmetic extension specification <https://github.com/quic/riscv-unified-db/releases/latest>`__ by Qualcomm.  All instructions are prefixed with `qc.` as described in the specification. These instructions are only available for riscv32.
 
 ``experimental-Xqciac``
-  LLVM implements `version 0.2 of the Qualcomm uC Load-Store Address Calculation extension specification <https://github.com/quic/riscv-unified-db/releases/latest>`__ by Qualcomm.  All instructions are prefixed with `qc.` as described in the specification. These instructions are only available for riscv32.
+  LLVM implements `version 0.3 of the Qualcomm uC Load-Store Address Calculation extension specification <https://github.com/quic/riscv-unified-db/releases/latest>`__ by Qualcomm.  All instructions are prefixed with `qc.` as described in the specification. These instructions are only available for riscv32.
 
 ``experimental-Xqcicli``
   LLVM implements `version 0.2 of the Qualcomm uC Conditional Load Immediate extension specification <https://github.com/quic/riscv-unified-db/releases/latest>`__ by Qualcomm.  All instructions are prefixed with `qc.` as described in the specification. These instructions are only available for riscv32.
@@ -459,6 +459,12 @@ The current vendor extensions supported are:
 ``experimental-Xqcisls``
   LLVM implements `version 0.2 of the Qualcomm uC Scaled Load Store extension specification <https://github.com/quic/riscv-unified-db/releases/latest>`__ by Qualcomm.  All instructions are prefixed with `qc.` as described in the specification. These instructions are only available for riscv32.
 
+``Xmipscmove``
+  LLVM implements conditional move for the `p8700 processor <https://mips.com/products/hardware/p8700/>` by MIPS.
+
+``Xmipslsp``
+  LLVM implements load/store pair instructions for the `p8700 processor <https://mips.com/products/hardware/p8700/>` by MIPS.
+
 Experimental C Intrinsics
 =========================
 
diff --git a/llvm/docs/ReleaseNotes.md b/llvm/docs/ReleaseNotes.md
index 05d902641d093..204eeaeb31ee3 100644
--- a/llvm/docs/ReleaseNotes.md
+++ b/llvm/docs/ReleaseNotes.md
@@ -42,17 +42,6 @@ point (e.g. maybe you would like to give an example of the
 functionality, or simply have a lot to talk about), see the comment below
 for adding a new subsection. -->
 
-* Added a new IRNormalizer pass which aims to transform LLVM modules into
-  a normal form by reordering and renaming instructions while preserving the
-  same semantics. The normalizer makes it easier to spot semantic differences
-  when diffing two modules which have undergone different passes.
-
-* The SPIR-V backend is now an official LLVM target, providing OpenCL and SYCL
-  conformance and establishing a foundation for broader applicability to other
-  APIs, including Vulkan, GLSL, and HLSL. This backend aims to offer a unified
-  approach for diverse compute and graphics workloads, providing a robust
-  alternative to the Khronos SPIR-V LLVM Translator.
-
 * ...
 
 <!-- If you would like to document a larger change, then you can add a
@@ -67,60 +56,17 @@ Makes programs 10x faster by doing Special New Thing.
 Changes to the LLVM IR
 ----------------------
 
-* Types are no longer allowed to be recursive.
-
-* The `x86_mmx` IR type has been removed. It will be translated to
-  the standard vector type `<1 x i64>` in bitcode upgrade.
-* Renamed `llvm.experimental.stepvector` intrinsic to `llvm.stepvector`.
-
-* Added `usub_cond` and `usub_sat` operations to `atomicrmw`.
-
-* Introduced `noalias.addrspace` metadata.
-
-* Remove the following intrinsics which can be replaced with a `bitcast`:
-
-  * `llvm.nvvm.bitcast.f2i`
-  * `llvm.nvvm.bitcast.i2f`
-  * `llvm.nvvm.bitcast.d2ll`
-  * `llvm.nvvm.bitcast.ll2d`
-
-* Remove the following intrinsics which can be replaced with a funnel-shift:
-
-  * `llvm.nvvm.rotate.b32`
-  * `llvm.nvvm.rotate.right.b64`
-  * `llvm.nvvm.rotate.b64`
-
-* Remove the following intrinsics which can be replaced with an
-  `addrspacecast`:
-
-  * `llvm.nvvm.ptr.gen.to.global`
-  * `llvm.nvvm.ptr.gen.to.shared`
-  * `llvm.nvvm.ptr.gen.to.constant`
-  * `llvm.nvvm.ptr.gen.to.local`
-  * `llvm.nvvm.ptr.global.to.gen`
-  * `llvm.nvvm.ptr.shared.to.gen`
-  * `llvm.nvvm.ptr.constant.to.gen`
-  * `llvm.nvvm.ptr.local.to.gen`
-
-* Remove the following intrinsics which can be relaced with a load from
-  addrspace(1) with an !invariant.load metadata
-
-  * `llvm.nvvm.ldg.global.i`
-  * `llvm.nvvm.ldg.global.f`
-  * `llvm.nvvm.ldg.global.p`
-
-* Operand bundle values can now be metadata strings.
-
-* Fast math flags are now permitted on `fptrunc` and `fpext`.
+* The `nocapture` attribute has been replaced by `captures(none)`.
 
 Changes to LLVM infrastructure
 ------------------------------
 
+* Removed support for target intrinsics being defined in the target directories
+  themselves (i.e., the `TargetIntrinsicInfo` class).
+
 Changes to building LLVM
 ------------------------
 
-* Raised the minimum MSVC version to Visual Studio 2019 16.8.
-
 Changes to TableGen
 -------------------
 
@@ -130,41 +76,12 @@ Changes to Interprocedural Optimizations
 Changes to the AArch64 Backend
 ------------------------------
 
-* `.balign N, 0`, `.p2align N, 0`, `.align N, 0` in code sections will now fill
-  the required alignment space with a sequence of `0x0` bytes (the requested
-  fill value) rather than NOPs.
-
-* Assembler/disassembler support has been added for Armv9.6-A (2024)
-  architecture extensions.
-
-* Added support for the FUJITSU-MONAKA CPU.
-
 Changes to the AMDGPU Backend
 -----------------------------
 
-* Removed `llvm.amdgcn.flat.atomic.fadd` and
-  `llvm.amdgcn.global.atomic.fadd` intrinsics. Users should use the
-  {ref}`atomicrmw <i_atomicrmw>` instruction with `fadd` and
-  addrspace(0) or addrspace(1) instead.
-
 Changes to the ARM Backend
 --------------------------
 
-* `.balign N, 0`, `.p2align N, 0`, `.align N, 0` in code sections will now fill
-  the required alignment space with a sequence of `0x0` bytes (the requested
-  fill value) rather than NOPs.
-
-* The default behavior for frame pointers in leaf functions has been updated.
-  When the `-fno-omit-frame-pointer` option is specified, `FPKeepKindStr` is
-  set to `-mframe-pointer=all`, meaning the frame pointer (FP) is now retained
-  in leaf functions by default. To eliminate the frame pointer in leaf functions,
-  you must explicitly use the `-momit-leaf-frame-pointer` option.
-
-* When using the `MOVT` or `MOVW` instructions, the Assembler will now check to
-  ensure that any addend that is used is within a 16-bit signed value range. If the
-  addend falls outside of this range, the LLVM backend will emit an error like so
-  `Relocation Not In Range`.
-
 Changes to the AVR Backend
 --------------------------
 
@@ -177,189 +94,24 @@ Changes to the Hexagon Backend
 Changes to the LoongArch Backend
 --------------------------------
 
-* [Incorrect GOT usage](https://github.com/llvm/llvm-project/pull/117099) for `non-dso_local` function calls in large code model is fixed.
-
-* A [gprof support issue](https://github.com/llvm/llvm-project/issues/121103) is fixed.
-
-* A [SDAG hang issue](https://github.com/llvm/llvm-project/issues/107355) caused by `ISD::CONCAT_VECTORS` is fixed.
-
-* A [compiler crash issue](https://github.com/llvm/llvm-project/issues/118301) when converting `half` to `i32` is fixed.
-
-* Almost all of `la64v1.1` instructions can now be generated. The full list is
-  `frecipe.s`, `frecipe.d`, `frsqrte.s`, `frsqrte.d`, `vfrecipe.s`, `vfrecipe.d`,
-  `vfrsqrte.s`, `vfrsqrte.d`, `xvfrecipe.s`, `xvfrecipe.d`, `xvfrsqrte.s`,
-  `xvfrsqrte.d`, `sc.q`, `amcas.b`, `amcas.h`, `amcas.w`, `amcas.d`, `amcas_db.b`,
-  `amcas_db.h`, `amcas_db.w`, `amcas_db.d`, `amswap.b`, `amswap.h`, `amswap_db.b`,
-  `amswap_db.h`, `amadd.b`, `amadd.h`, `amadd_db.b`, `amadd_db.h`. Optionally
-  generate instructions `dbar 0x700`, `div.w`, `div.wu`, `mod.w` and `mod.wu`
-  when related target features are enabled. `llacq.w`, `screl.w`, `llacq.d` and
-  `screl.d` cannot be generated yet.
-
-* An llc option called `-loongarch-annotate-tablejump` is added to annotate
-  table jump instruction in the `.discard.tablejump_annotate` section. A typical
-  user of these annotations is the `objtool` in Linux kernel.
-
-* The default cpu in `MCSubtargetInfo` is changed from `la464` to `generic-la64`.
-  In addition, the `lsx` feature is added to `generic-la64`.
-
-* CFI instructions now allow register names and aliases, previously only numbers
-  were allowed.
-
-* `RuntimeDyld` now supports LoongArch, which means that programs relying on
-  `MCJIT` can now work.
-
-* `.balign N, 0`, `.p2align N, 0`, `.align N, 0` in code sections will now fill
-  the required alignment space with a sequence of `0x0` bytes (the requested
-  fill value) rather than NOPs.
-
-* `%ld_pcrel_20`, `%gd_pcrel_20` and `%desc_pcrel_20` operand modifiers are
-   supported by assembler.
-
-* A machine function pass called `LoongArch Merge Base Offset` is added to merge
-  the offset of address calculation into the offset field of instructions in a
-  global address lowering sequence.
-
-* The `LoopDataPrefetch` pass can now work on LoongArch, but it is disabled by
-  default due to the bad effect on Fortran benchmarks.
-
-* Enable alias analysis by default.
-
-* Avoid indirect branch jumps using the `$ra` register.
-
-* Other optimizations.
-
 Changes to the MIPS Backend
 ---------------------------
 
 Changes to the PowerPC Backend
 ------------------------------
 
-* The Linux `ppc64` LLC default cpu is updated from `ppc` to `ppc64`.
-* The AIX LLC default cpu is updated from `generic` to `pwr7`.
-
 Changes to the RISC-V Backend
 -----------------------------
 
-* `.balign N, 0`, `.p2align N, 0`, `.align N, 0` in code sections will now fill
-  the required alignment space with a sequence of `0x0` bytes (the requested
-  fill value) rather than NOPs.
-* Added Syntacore SCR4 and SCR5 CPUs: `-mcpu=syntacore-scr4/5-rv32/64`
-* `-mcpu=sifive-p470` was added.
-* Added Hazard3 CPU as taped out for RP2350: `-mcpu=rp2350-hazard3` (32-bit
-  only).
-* Fixed length vector support using RVV instructions now requires VLEN>=64. This
-  means Zve32x and Zve32f will also require Zvl64b. The prior support was
-  largely untested.
-* The `Zvbc32e` and `Zvkgs` extensions are now supported experimentally.
-* Added `Smctr`, `Ssctr` and `Svvptc` extensions.
-* `-mcpu=syntacore-scr7` was added.
-* `-mcpu=tt-ascalon-d8` was added.
-* `-mcpu=mips-p8700` was added.
-* `-mcpu=sifive-p550` was added.
-* The `Zacas` extension is no longer marked as experimental.
-* Added Smdbltrp, Ssdbltrp extensions to -march.
-* The `Smmpm`, `Smnpm`, `Ssnpm`, `Supm`, and `Sspm` pointer masking extensions
-  are no longer marked as experimental.
-* The `Sha` extension is now supported.
-* The RVA23U64, RVA23S64, RVB23U64, and RVB23S64 profiles are no longer marked
-  as experimental.
-* `.insn <length>, <raw encoding>` can be used to assemble 48- and 64-bit
-  instructions from raw integer values.
-* `.insn [<length>,] <raw encoding>` now accepts absolute expressions for both
-  expressions, so that they can be computed from constants and absolute symbols.
-* The following new inline assembly constraints and modifiers are accepted:
-  * `cr` constraint meaning an RVC-encoding compatible GPR (`x8`-`x15`)
-  * `cf` constraint meaning an RVC-encoding compatible FPR (`f8`-`f15`)
-  * `R` constraint meaning an even-odd GPR pair (prints as the even register,
-    but both registers in the pair are considered live).
-  * `N` modifer meaning print the register encoding (0-31) rather than the name.
-* `f` and `cf` inline assembly constraints, when using F-/D-/H-in-X extensions,
-  will use the relevant GPR rather than FPR. This makes inline assembly portable
-  between e.g. F and Zfinx code.
-* Adds experimental assembler support for the Qualcomm uC 'Xqcicsr` (CSR)
-  extension.
-* Adds experimental assembler support for the Qualcomm uC 'Xqcisls` (Scaled Load Store)
-  extension.
-* Adds experimental assembler support for the Qualcomm uC 'Xqcia` (Arithmetic)
-  extension.
-* Adds experimental assembler support for the Qualcomm uC 'Xqciac` (Load-Store Address Calculation)
-  extension.
-* Adds experimental assembler support for the Qualcomm uC 'Xqcics` (Conditonal Select)
-  extension.
-* Adds experimental assembler support for the Qualcomm uC 'Xqcilsm` (Load Store Multiple)
-  extension.
-* Adds experimental assembler support for the Qualcomm uC 'Xqcicli` (Conditional Load Immediate)
-  extension.
-* Adds experimental assembler support for the Qualcomm uC 'Xqcicm` (Conditonal Move)
-  extension.
-* Adds experimental assembler support for the Qualcomm uC 'Xqciint` (Interrupts)
-  extension.
-* Adds experimental assembler support for the Qualcomm uC 'Xqcilo` (Large Offset Load Store)
-  extension.
-* Added ``Sdext`` and ``Sdtrig`` extensions.
-
 Changes to the WebAssembly Backend
 ----------------------------------
 
-* The default target CPU, "generic", now enables the `-mnontrapping-fptoint`
-  and `-mbulk-memory` flags, which correspond to the [Bulk Memory Operations]
-  and [Non-trapping float-to-int Conversions] language features, which are
-  [widely implemented in engines].
-
-* A new Lime1 target CPU is added, `-mcpu=lime1`. This CPU follows the
-  definition of the Lime1 CPU [here], and enables `-mmultivalue`,
-  `-mmutable-globals`, `-mcall-indirect-overlong`, `-msign-ext`,
-  `-mbulk-memory-opt`, `-mnontrapping-fptoint`, and `-mextended-const`.
-
-* Support for the new standardized [Exception Handling] proposal is added.
-  The [legacy Exception Handling] proposal is still supported, and turned on by
-  the newly added `-wasm-use-legacy-eh` option. Given that major web browsers
-  still default to the legacy EH proposal, this option is turned on by default
-  for the moment.
-
-[Bulk Memory Operations]: https://github.com/WebAssembly/bulk-memory-operations/blob/master/proposals/bulk-memory-operations/Overview.md
-[Non-trapping float-to-int Conversions]: https://github.com/WebAssembly/spec/blob/master/proposals/nontrapping-float-to-int-conversion/Overview.md
-[Exception Handling]: https://github.com/WebAssembly/exception-handling/blob/main/proposals/exception-handling/Exceptions.md
-[legacy Exception Handling]: https://github.com/WebAssembly/exception-handling/blob/main/proposals/exception-handling/legacy/Exceptions.md
-[widely implemented in engines]: https://webassembly.org/features/
-[here]: https://github.com/WebAssembly/tool-conventions/blob/main/Lime.md#lime1
-
 Changes to the Windows Target
 -----------------------------
 
 Changes to the X86 Backend
 --------------------------
 
-* `.balign N, 0x90`, `.p2align N, 0x90`, and `.align N, 0x90` in code sections
-  now fill the required alignment space with repeating `0x90` bytes, rather than
-  using optimised NOP filling. Optimised NOP filling fills the space with NOP
-  instructions of various widths, not just those that use the `0x90` byte
-  encoding. To use optimised NOP filling in a code section, leave off the
-  "fillval" argument, i.e. `.balign N`, `.p2align N` or `.align N` respectively.
-
-* Due to the removal of the `x86_mmx` IR type, functions with
-  `x86_mmx` arguments or return values will use a different,
-  incompatible, calling convention ABI. Such functions are not
-  generally seen in the wild (Clang never generates them!), so this is
-  not expected to result in real-world compatibility problems.
-
-* Support ISA of `AVX10.2-256` and `AVX10.2-512`.
-
-* Supported instructions of `MOVRS AND AVX10.2`
-
-* Supported ISA of `SM4(EVEX)`.
-
-* Supported ISA of `MSR_IMM`.
-
-* Supported ``-mcpu=diamondrapids``
-
-* Supported emitting relocation types for x86-64 target:
-  * `R_X86_64_CODE_4_GOTPCRELX`
-  * `R_X86_64_CODE_4_GOTTPOFF`
-  * `R_X86_64_CODE_4_GOTPC32_TLSDESC`
-  * `R_X86_64_CODE_6_GOTTPOFF`
-
-
 Changes to the OCaml bindings
 -----------------------------
 
@@ -369,54 +121,6 @@ Changes to the Python bindings
 Changes to the C API
 --------------------
 
-* The following symbols are deleted due to the removal of the `x86_mmx` IR type:
-
-  * `LLVMX86_MMXTypeKind`
-  * `LLVMX86MMXTypeInContext`
-  * `LLVMX86MMXType`
-
- * The following functions are added to further support non-null-terminated strings:
-
-  * `LLVMGetNamedFunctionWithLength`
-  * `LLVMGetNamedGlobalWithLength`
-
-* The following functions are added to access the `LLVMContextRef` associated
-   with `LLVMValueRef` and `LLVMBuilderRef` objects:
-
-  * `LLVMGetValueContext`
-  * `LLVMGetBuilderContext`
-
-* The new pass manager can now be invoked with a custom alias analysis pipeline, using
-  the `LLVMPassBuilderOptionsSetAAPipeline` function.
-
-* It is now also possible to run the new pass manager on a single function, by calling
-  `LLVMRunPassesOnFunction` instead of `LLVMRunPasses`.
-
-* Support for creating instructions with custom synchronization scopes has been added:
-
-  * `LLVMGetSyncScopeID` to map a synchronization scope name to an ID.
-  * `LLVMBuildFenceSyncScope`, `LLVMBuildAtomicRMWSyncScope` and
-    `LLVMBuildAtomicCmpXchgSyncScope` versions of the existing builder functions
-    with an additional synchronization scope ID parameter.
-  * `LLVMGetAtomicSyncScopeID` and `LLVMSetAtomicSyncScopeID` to get and set the
-    synchronization scope of any atomic instruction.
-  * `LLVMIsAtomic` to check if an instruction is atomic, for use with the above functions.
-    Because of backwards compatibility, `LLVMIsAtomicSingleThread` and
-    `LLVMSetAtomicSingleThread` continue to work with any instruction type.
-
-* The `LLVMSetPersonalityFn` and `LLVMSetInitializer` APIs now support clearing the
-  personality function and initializer respectively by passing a null pointer.
-
-* The following functions are added to allow iterating over debug records attached to
-  instructions:
-
-  * `LLVMGetFirstDbgRecord`
-  * `LLVMGetLastDbgRecord`
-  * `LLVMGetNextDbgRecord`
-  * `LLVMGetPreviousDbgRecord`
-
-* Added `LLVMAtomicRMWBinOpUSubCond` and `LLVMAtomicRMWBinOpUSubSat` to `LLVMAtomicRMWBinOp` enum for AtomicRMW instructions.
-
 Changes to the CodeGen infrastructure
 -------------------------------------
 
@@ -429,149 +133,19 @@ Changes to the Debug Info
 Changes to the LLVM tools
 ---------------------------------
 
-* llvm-objcopy now supports the following options for Mach-O:
-  `--globalize-symbol`, `--globalize-symbols`,
-  `--keep-global-symbol`, `--keep-global-symbols`,
-  `--localize-symbol`, `--localize-symbols`,
-  `--skip-symbol`, `--skip-symbols`.
-
 Changes to LLDB
 ---------------------------------
 
-* It is now recommended that LLDB be built with Python >= 3.8, but no changes
-  have been made to the supported Python versions. The next release, LLDB 21,
-  will require Python >= 3.8.
-
-* LLDB now supports inline diagnostics for the expression evaluator and command line parser.
-
-  Old:
-  ```
-  (lldb) p a+b
-  error: <user expression 0>:1:1: use of undeclared identifier 'a'
-      1 | a+b
-        | ^
-  error: <user expression 0>:1:3: use of undeclared identifier 'b'
-      1 | a+b
-        |   ^
-  ```
+* When building LLDB with Python support, the minimum version of Python is now
+  3.8.
+* LLDB now supports hardware watchpoints for AArch64 Windows targets. Windows
+  does not provide API to query the number of supported hardware watchpoints.
+  Therefore current implementation allows only 1 watchpoint, as tested with
+  Windows 11 on the Microsoft SQ2 and Snapdragon Elite X platforms.
+
+### Changes to lldb-dap
 
-  New:
-
-  ```
-  (lldb) p a+b
-           ˄ ˄
-           │ ╰─ error: use of undeclared identifier 'b'
-           ╰─ error: use of undeclared identifier 'a'
-  ```
-
-* LLDB can now read the `fpmr` register from AArch64 Linux processes and core
-  files.
-
-* Program stdout/stderr redirection will now open the file with O_TRUNC flag, make sure to truncate the file if path already exists.
-  * eg. `settings set target.output-path/target.error-path <path/to/file>`
-
-* A new setting `target.launch-working-dir` can be used to set a persistent cwd that is used by default by `process launch` and `run`.
-
-* LLDB now parses shared libraries in parallel, resulting in an average 2x speedup when attaching (only available on Darwin platforms) and launching (available on all platforms).
-
-* It is now possible to implement lldb commands in Python that use lldb's native command-line parser.  In particular, that allows per-option/argument completion,
-  with all the basic completers automatically supported and auto-generated help.
-  The command template file in `lldb/examples/python/cmdtemplate.py` has been updated to show how to use this.
-
-* Breakpoints on "inlined call sites" are now supported.  Previous to this fix, breakpoints on source lines that only contained inlined call sites would be
-  moved to the next source line, causing you to miss the inlined executions.
-
-* On the command line, LLDB now limits tab completions to your terminal width to avoid wrapping.
-
-  Old:
-  ```
-  Available completions:
-          _regexp-attach    -- Attach to process by ID or name.
-          _regexp-break     -- Set a breakpoint using one of several shorthand
-  formats.
-          _regexp-bt        -- Show backtrace of the current thread's call sta
-  ck. Any numeric argument displays at most that many frames. The argument 'al
-  l' displays all threads. Use 'settings set frame-format' to customize the pr
-  inting of individual frames and 'settings set thread-format' to customize th
-  e thread header. Frame recognizers may filter thelist. Use 'thread backtrace
-  -u (--unfiltered)' to see them all.
-          _regexp-display   -- Evaluate an expression at every stop (see 'help
-  target stop-hook'.)
-
-  ```
-
-  New:
-  ```
-  Available completions:
-          _regexp-attach    -- Attach to process by ID or name.
-          _regexp-break     -- Set a breakpoint using one of several shorth...
-          _regexp-bt        -- Show backtrace of the current thread's call ...
-          _regexp-display   -- Evaluate an expression at every stop (see 'h...
-  ```
-* DWARF indexing speed (for binaries not using the debug_names index) increased
-  by [30-60%](https://github.com/llvm/llvm-project/pull/118657).
-
-* Minidumps generated by LLDB now support:
-  * 64 bit memory (due to 64b support, Minidumps are now paged to disk while being written).
-  * Capturing of TLS variables.
-  * Multiple signals or exceptions, including breakpoints.
-
-* [New Core File API](https://lldb.llvm.org/python_api/lldb.SBSaveCoreOptions.html). This gives greater control on the data captured into the core file, relative to the existing `process save-core` styles.
-
-* When opening ELF core files, LLDB will print additional information about the
-  signal that killed the process and the disassembly view will display actual
-  (relocated) targets of the jump instructions instead of raw offsets encoded in
-  the instruction. This matches existing behavior for live processes.
-
-  Old:
-  ```
-  * thread #1: tid = 329384, 0x0000000000401262, name = 'a.out', stop reason = signal SIGSEGV
-
-  0x7f1e3193e0a7 <+23>:  ja     0xfe100        ; <+112>
-  ```
-
-  New:
-  ```
-  * thread #1: tid = 329384, 0x0000000000401262, name = 'a.out', stop reason = SIGSEGV: address not mapped to object (fault address=0x0)
-
-  0x7f1e3193e0a7 <+23>:  ja     0x7f1e3193e100 ; <+112>
-  ```
-
-* `lldb-server` now listens to a single port for gdbserver connections and provides
-  that port to the connection handler processes. This means that only 2 ports need
-  to be opened in the firewall (one for the `lldb-server` platform, one for gdbserver connections).
-  In addition, due to this work, `lldb-server` now works on Windows in the server mode.
-  
-* LLDB now supports execution of user expressions for non-trivial cases for LoongArch and RISC-V targets, like function calls, when some code needs to be executed on the target.
-
-* LLDB now supports optionally enabled/disabled register sets (particularly floating point registers) for RISC-V 64. This happens for targets like `RV64IMAC` or `RV64IMACV`,
-  that have no floating point registers. The change is applied to native debugging and core-file usage.
-
-* LLDB now supports [core-file for LoongArch](https://github.com/llvm/llvm-project/pull/112296).
-
-* LLDB now supports [hardware breakpoint and watchpoint for LoongArch](https://github.com/llvm/llvm-project/pull/118770).
-
-* LLDB now supports [vector registers for LoongArch](https://github.com/llvm/llvm-project/pull/120664) when debugging a live process.
-
-* Incorrect floating-point register dwarf number for LoongArch is [fixed](https://github.com/llvm/llvm-project/pull/120391).
-
-* The `frame diagnose` now works on ELF-based systems. After a crash, LLDB will
-  try to determine the likely cause of the signal, matching Darwin behavior.
-  This feature requires using a new `lldb-server` version and (like Darwin) only
-  works on x86 binaries.
-
-  ```
-  * thread #1, name = 'a.out', stop reason = signal SIGSEGV: address not mapped to object (fault address=0x4)
-      frame #0: 0x00005555555551aa a.out`GetSum(f=0x0000555555558018) at main.c:21:37
-     18   }
-     19
-     20   int GetSum(struct Foo *f) {
-  -> 21     return SumTwoIntegers(f->a, f->b->d ? 0 : 1);
-     22   }
-     23
-     24   int main() {
-  Likely cause: f->b->d accessed 0x4
-  ```
+* Breakpoints can now be set for specific columns within a line.
 
 Changes to BOLT
 ---------------------------------
diff --git a/llvm/docs/RemoveDIsDebugInfo.md b/llvm/docs/RemoveDIsDebugInfo.md
index c9012548f7578..db22d1df758fa 100644
--- a/llvm/docs/RemoveDIsDebugInfo.md
+++ b/llvm/docs/RemoveDIsDebugInfo.md
@@ -24,7 +24,13 @@ The debug records are not instructions, do not appear in the instruction list, a
 
 # Great, what do I need to do!
 
-Very little -- we've already instrumented all of LLVM to handle these new records ("`DbgRecords`") and behave identically to past LLVM behaviour. This is currently being turned on by default, so that `DbgRecords` will be used by default in memory, IR, and bitcode.
+We've largely completed the migration. The remaining rough edge is that going forwards, instructions must be inserted into basic blocks using iterators rather than instruction pointers. In almost all circumstances you can just call `getIterator` on an instruction pointer -- however, if you call a function that returns the start of a basic block, such as:
+
+1. BasicBlock::begin
+2. BasicBlock::getFirstNonPHIIt
+3. BasicBlock::getFirstInsertionPt
+
+Then you must past that iterator into the insertion function without modification (the iterator carries a debug-info bit). That's all! Read on for a more detailed explanation.
 
 ## API Changes
 
diff --git a/llvm/docs/SPIRVUsage.rst b/llvm/docs/SPIRVUsage.rst
index 23c5fe37a9b87..b7601b26beb89 100644
--- a/llvm/docs/SPIRVUsage.rst
+++ b/llvm/docs/SPIRVUsage.rst
@@ -204,7 +204,7 @@ list of supported SPIR-V extensions, sorted alphabetically by their extension na
    * - ``SPV_KHR_non_semantic_info``
      - Adds the ability to declare extended instruction sets that have no semantic impact and can be safely removed from a module.
 
-To enable multiple extensions, list them separated by spaces. For example, to enable support for atomic operations on floating-point numbers and arbitrary precision integers, use:
+To enable multiple extensions, list them separated by comma. For example, to enable support for atomic operations on floating-point numbers and arbitrary precision integers, use:
 
 ``-spirv-ext=+SPV_EXT_shader_atomic_float_add,+SPV_INTEL_arbitrary_precision_integers``
 
diff --git a/llvm/docs/SandboxIR.md b/llvm/docs/SandboxIR.md
index 61bae4e36ef43..735190e19966e 100644
--- a/llvm/docs/SandboxIR.md
+++ b/llvm/docs/SandboxIR.md
@@ -6,7 +6,7 @@ Sandbox IR is an IR layer on top of LLVM IR that allows you to save/restore its
 
 Within your LLVM pass:
 
-```
+``` C++
 // 1. Include the necessary Sandbox IR header files.
 #include "llvm/SandboxIR/Context.h
 #include "llvm/SandboxIR/Function.h
diff --git a/llvm/docs/TableGen/ProgRef.rst b/llvm/docs/TableGen/ProgRef.rst
index cfe61382658ec..edb97109c9289 100644
--- a/llvm/docs/TableGen/ProgRef.rst
+++ b/llvm/docs/TableGen/ProgRef.rst
@@ -268,7 +268,7 @@ high-level types (e.g., ``dag``). This flexibility allows you to describe a
 wide range of records conveniently and compactly.
 
 .. productionlist::
-   Type: "bit" | "int" | "string" | "dag"
+   Type: "bit" | "int" | "string" | "dag" | "code"
        :| "bits" "<" `TokInteger` ">"
        :| "list" "<" `Type` ">"
        :| `ClassID`
@@ -285,6 +285,10 @@ wide range of records conveniently and compactly.
     The ``string`` type represents an ordered sequence of characters of arbitrary
     length.
 
+``code``
+    The keyword ``code`` is an alias for ``string`` which may be used to
+    indicate string values that are code.
+
 ``bits<``\ *n*\ ``>``
     The ``bits`` type is a fixed-sized integer of arbitrary length *n* that
     is treated as separate bits. These bits can be accessed individually.
@@ -363,7 +367,16 @@ Simple values
 The :token:`SimpleValue` has a number of forms.
 
 .. productionlist::
-   SimpleValue: `TokInteger` | `TokString`+ | `TokCode`
+   SimpleValue: `SimpleValue1`
+              :| `SimpleValue2`
+              :| `SimpleValue3`
+              :| `SimpleValue4`
+              :| `SimpleValue5`
+              :| `SimpleValue6`
+              :| `SimpleValue7`
+              :| `SimpleValue8`
+              :| `SimpleValue9`
+   SimpleValue1: `TokInteger` | `TokString`+ | `TokCode`
 
 A value can be an integer literal, a string literal, or a code literal.
 Multiple adjacent string literals are concatenated as in C/C++; the simple
@@ -498,6 +511,8 @@ arguments, producing a value for that bang operator. The ``!cond`` operator
 takes a list of pairs of arguments separated by colons. See `Appendix A:
 Bang Operators`_ for a description of each bang operator.
 
+The `Type` is only accepted for certain bang operators, and must not be
+``code``.
 
 Suffixed values
 ---------------
@@ -670,7 +685,7 @@ arguments.
 
 .. productionlist::
    Body: ";" | "{" `BodyItem`* "}"
-   BodyItem: (`Type` | "code") `TokIdentifier` ["=" `Value`] ";"
+   BodyItem: `Type` `TokIdentifier` ["=" `Value`] ";"
            :| "let" `TokIdentifier` ["{" `RangeList` "}"] "=" `Value` ";"
            :| "defvar" `TokIdentifier` "=" `Value` ";"
            :| `Assert`
@@ -678,8 +693,7 @@ arguments.
 A field definition in the body specifies a field to be included in the class
 or record. If no initial value is specified, then the field's value is
 uninitialized. The type must be specified; TableGen will not infer it from
-the value. The keyword ``code`` may be used to emphasize that the field
-has a string value that is code.
+the value.
 
 The ``let`` form is used to reset a field to a new value. This can be done
 for fields defined directly in the body or fields inherited from parent
@@ -1302,8 +1316,9 @@ output. It is intended for debugging purpose.
   instantiation point of the containing record.
 
 .. productionlist::
-   Dump: "dump"  `string` ";"
+   Dump: "dump" `Value` ";"
 
+The :token:`Value` is an arbitrary string expression.
 For example, it can be used in combination with `!repr` to investigate
 the values passed to a multiclass:
 
@@ -1350,11 +1365,12 @@ The ``assert`` statement checks a boolean condition to be sure that it is true
 and prints an error message if it is not.
 
 .. productionlist::
-   Assert: "assert" `condition` "," `message` ";"
+   Assert: "assert" `Value` "," `Value` ";"
 
-If the boolean condition is true, the statement does nothing. If the
-condition is false, it prints a nonfatal error message. The **message**, which
-can be an arbitrary string expression, is included in the error message as a
+The first :token:`Value` is a boolean condition. If it is true, the
+statement does nothing. If the condition is false, it prints a nonfatal
+error message. The second :token:`Value` is a message, which can be an
+arbitrary string expression. It is included in the error message as a
 note. The exact behavior of the ``assert`` statement depends on its
 placement.
 
diff --git a/llvm/docs/Vectorizers.rst b/llvm/docs/Vectorizers.rst
index f134a6df94a69..d19d03050436a 100644
--- a/llvm/docs/Vectorizers.rst
+++ b/llvm/docs/Vectorizers.rst
@@ -405,9 +405,11 @@ Early Exit Vectorization
 When vectorizing a loop with a single early exit, the loop blocks following the
 early exit are predicated and the vector loop will always exit via the latch.
 If the early exit has been taken, the vector loop's successor block
-(``middle.split`` below) branches to the early exit block. Otherwise
-``middle.block`` selects between the exit block from the latch or the scalar
-remainder loop.
+(``middle.split`` below) branches to the early exit block via an intermediate
+block (``vector.early.exit`` below). This intermediate block is responsible for
+calculating any exit values of loop-defined variables that are used in the
+early exit block. Otherwise, ``middle.block`` selects between the exit block
+from the latch or the scalar remainder loop.
 
 .. image:: vplan-early-exit.png
 
diff --git a/llvm/docs/tutorial/MyFirstLanguageFrontend/LangImpl09.rst b/llvm/docs/tutorial/MyFirstLanguageFrontend/LangImpl09.rst
index c75d8814918aa..a88b6ed48e8d4 100644
--- a/llvm/docs/tutorial/MyFirstLanguageFrontend/LangImpl09.rst
+++ b/llvm/docs/tutorial/MyFirstLanguageFrontend/LangImpl09.rst
@@ -299,7 +299,7 @@ or parser so we'll need to add it.
    }
 
 In this set of code we've added some functionality on how to keep track of the
-line and column of the "source file". As we lex every token we set our current
+line and column of the "source file". As we lex every token we set our
 current "lexical location" to the assorted line and column for the beginning
 of the token. We do this by overriding all of the previous calls to
 ``getchar()`` with our new ``advance()`` that keeps track of the information
diff --git a/llvm/docs/vplan-early-exit.dot b/llvm/docs/vplan-early-exit.dot
index 63490b0cdb2e4..980fc6939b6d4 100644
--- a/llvm/docs/vplan-early-exit.dot
+++ b/llvm/docs/vplan-early-exit.dot
@@ -19,23 +19,27 @@ compound=true
     "middle.split"
   ]
   N4 -> N5 [ label=""]
-  N4 -> N6 [ label=""]
+  N4 -> N7 [ label=""]
   N5 [label =
-    "early.exit"
+    "vector.early.exit"
   ]
+  N5 -> N6 [ label=""]
   N6 [label =
-    "middle.block"
+    "early.exit"
   ]
-  N6 -> N9 [ label=""]
-  N6 -> N7 [ label=""]
   N7 [label =
-    "scalar.ph"
+    "middle.block"
   ]
+  N7 -> N10 [ label=""]
   N7 -> N8 [ label=""]
   N8 [label =
-    "loop.header"
+    "scalar.ph"
   ]
+  N8 -> N9 [ label=""]
   N9 [label =
+    "loop.header"
+  ]
+  N10 [label =
     "latch.exit"
   ]
 }
diff --git a/llvm/docs/vplan-early-exit.png b/llvm/docs/vplan-early-exit.png
index 3cd293bcdbcc8..249b90c69e2b8 100644
Binary files a/llvm/docs/vplan-early-exit.png and b/llvm/docs/vplan-early-exit.png differ
diff --git a/llvm/examples/IRTransforms/SimplifyCFG.cpp b/llvm/examples/IRTransforms/SimplifyCFG.cpp
index a37060cedb4a7..5b8b9b0cef11e 100644
--- a/llvm/examples/IRTransforms/SimplifyCFG.cpp
+++ b/llvm/examples/IRTransforms/SimplifyCFG.cpp
@@ -286,7 +286,7 @@ static bool mergeIntoSinglePredecessor_v1(Function &F) {
     }
     // Move all instructions from BB to Pred.
     for (Instruction &I : make_early_inc_range(BB))
-      I.moveBefore(Pred->getTerminator());
+      I.moveBefore(Pred->getTerminator()->getIterator());
 
     // Remove the Pred's terminator (which jumped to BB). BB's terminator
     // will become Pred's terminator.
@@ -337,7 +337,7 @@ static bool mergeIntoSinglePredecessor_v2(Function &F, DominatorTree &DT) {
     }
     // Move all instructions from BB to Pred.
     for (Instruction &I : make_early_inc_range(BB))
-      I.moveBefore(Pred->getTerminator());
+      I.moveBefore(Pred->getTerminator()->getIterator());
 
     // Remove the Pred's terminator (which jumped to BB). BB's terminator
     // will become Pred's terminator.
diff --git a/llvm/include/llvm-c/Core.h b/llvm/include/llvm-c/Core.h
index dc8ecf4fb2ade..43bb775e5781a 100644
--- a/llvm/include/llvm-c/Core.h
+++ b/llvm/include/llvm-c/Core.h
@@ -321,11 +321,6 @@ typedef enum {
   LLVMRealPredicateTrue   /**< Always true (always folded) */
 } LLVMRealPredicate;
 
-typedef enum {
-  LLVMLandingPadCatch,    /**< A catch clause   */
-  LLVMLandingPadFilter    /**< A filter clause  */
-} LLVMLandingPadClauseTy;
-
 typedef enum {
   LLVMNotThreadLocal = 0,
   LLVMGeneralDynamicTLSModel,
diff --git a/llvm/include/llvm/ADT/GenericCycleImpl.h b/llvm/include/llvm/ADT/GenericCycleImpl.h
index 41ba8bf8fde14..4b2e01bcd9c53 100644
--- a/llvm/include/llvm/ADT/GenericCycleImpl.h
+++ b/llvm/include/llvm/ADT/GenericCycleImpl.h
@@ -454,7 +454,9 @@ void GenericCycleInfoCompute<ContextT>::dfs(BlockT *EntryBlock) {
     BlockT *Block = TraverseStack.back();
     LLVM_DEBUG(errs() << "DFS visiting block: " << Info.Context.print(Block)
                       << "\n");
-    if (!BlockDFSInfo.count(Block)) {
+    if (BlockDFSInfo.try_emplace(Block, Counter + 1).second) {
+      ++Counter;
+
       // We're visiting the block for the first time. Open its DFSInfo, add
       // successors to the traversal stack, and remember the traversal stack
       // depth at which the block was opened, so that we can correctly record
@@ -465,9 +467,6 @@ void GenericCycleInfoCompute<ContextT>::dfs(BlockT *EntryBlock) {
       DFSTreeStack.emplace_back(TraverseStack.size());
       llvm::append_range(TraverseStack, successors(Block));
 
-      bool Added = BlockDFSInfo.try_emplace(Block, ++Counter).second;
-      (void)Added;
-      assert(Added);
       BlockPreorder.push_back(Block);
       LLVM_DEBUG(errs() << "  preorder number: " << Counter << "\n");
     } else {
diff --git a/llvm/include/llvm/ADT/StableHashing.h b/llvm/include/llvm/ADT/StableHashing.h
index b220a0ed1f913..b6bd23c6945bf 100644
--- a/llvm/include/llvm/ADT/StableHashing.h
+++ b/llvm/include/llvm/ADT/StableHashing.h
@@ -54,14 +54,13 @@ inline stable_hash stable_hash_combine(stable_hash A, stable_hash B,
 // maintain closeness to the original name across different builds.
 inline StringRef get_stable_name(StringRef Name) {
   // Return the part after ".content." that represents contents.
-  auto [P0, S0] = Name.rsplit(".content.");
+  StringRef S0 = Name.rsplit(".content.").second;
   if (!S0.empty())
     return S0;
 
   // Ignore these suffixes.
-  auto [P1, S1] = Name.rsplit(".llvm.");
-  auto [P2, S2] = P1.rsplit(".__uniq.");
-  return P2;
+  StringRef P1 = Name.rsplit(".llvm.").first;
+  return P1.rsplit(".__uniq.").first;
 }
 
 // Generates a consistent hash value for a given input name across different
diff --git a/llvm/include/llvm/ADT/StringSwitch.h b/llvm/include/llvm/ADT/StringSwitch.h
index 7093da07663a0..86e591c71c92e 100644
--- a/llvm/include/llvm/ADT/StringSwitch.h
+++ b/llvm/include/llvm/ADT/StringSwitch.h
@@ -14,7 +14,6 @@
 #define LLVM_ADT_STRINGSWITCH_H
 
 #include "llvm/ADT/StringRef.h"
-#include "llvm/Support/Compiler.h"
 #include <cassert>
 #include <cstring>
 #include <optional>
@@ -67,9 +66,7 @@ class StringSwitch {
 
   // Case-sensitive case matchers
   StringSwitch &Case(StringLiteral S, T Value) {
-    if (!Result && Str == S) {
-      Result = std::move(Value);
-    }
+    CaseImpl(Value, S);
     return *this;
   }
 
@@ -88,61 +85,59 @@ class StringSwitch {
   }
 
   StringSwitch &Cases(StringLiteral S0, StringLiteral S1, T Value) {
-    return Case(S0, Value).Case(S1, Value);
+    return CasesImpl(Value, S0, S1);
   }
 
   StringSwitch &Cases(StringLiteral S0, StringLiteral S1, StringLiteral S2,
                       T Value) {
-    return Case(S0, Value).Cases(S1, S2, Value);
+    return CasesImpl(Value, S0, S1, S2);
   }
 
   StringSwitch &Cases(StringLiteral S0, StringLiteral S1, StringLiteral S2,
                       StringLiteral S3, T Value) {
-    return Case(S0, Value).Cases(S1, S2, S3, Value);
+    return CasesImpl(Value, S0, S1, S2, S3);
   }
 
   StringSwitch &Cases(StringLiteral S0, StringLiteral S1, StringLiteral S2,
                       StringLiteral S3, StringLiteral S4, T Value) {
-    return Case(S0, Value).Cases(S1, S2, S3, S4, Value);
+    return CasesImpl(Value, S0, S1, S2, S3, S4);
   }
 
   StringSwitch &Cases(StringLiteral S0, StringLiteral S1, StringLiteral S2,
                       StringLiteral S3, StringLiteral S4, StringLiteral S5,
                       T Value) {
-    return Case(S0, Value).Cases(S1, S2, S3, S4, S5, Value);
+    return CasesImpl(Value, S0, S1, S2, S3, S4, S5);
   }
 
   StringSwitch &Cases(StringLiteral S0, StringLiteral S1, StringLiteral S2,
                       StringLiteral S3, StringLiteral S4, StringLiteral S5,
                       StringLiteral S6, T Value) {
-    return Case(S0, Value).Cases(S1, S2, S3, S4, S5, S6, Value);
+    return CasesImpl(Value, S0, S1, S2, S3, S4, S5, S6);
   }
 
   StringSwitch &Cases(StringLiteral S0, StringLiteral S1, StringLiteral S2,
                       StringLiteral S3, StringLiteral S4, StringLiteral S5,
                       StringLiteral S6, StringLiteral S7, T Value) {
-    return Case(S0, Value).Cases(S1, S2, S3, S4, S5, S6, S7, Value);
+    return CasesImpl(Value, S0, S1, S2, S3, S4, S5, S6, S7);
   }
 
   StringSwitch &Cases(StringLiteral S0, StringLiteral S1, StringLiteral S2,
                       StringLiteral S3, StringLiteral S4, StringLiteral S5,
                       StringLiteral S6, StringLiteral S7, StringLiteral S8,
                       T Value) {
-    return Case(S0, Value).Cases(S1, S2, S3, S4, S5, S6, S7, S8, Value);
+    return CasesImpl(Value, S0, S1, S2, S3, S4, S5, S6, S7, S8);
   }
 
   StringSwitch &Cases(StringLiteral S0, StringLiteral S1, StringLiteral S2,
                       StringLiteral S3, StringLiteral S4, StringLiteral S5,
                       StringLiteral S6, StringLiteral S7, StringLiteral S8,
                       StringLiteral S9, T Value) {
-    return Case(S0, Value).Cases(S1, S2, S3, S4, S5, S6, S7, S8, S9, Value);
+    return CasesImpl(Value, S0, S1, S2, S3, S4, S5, S6, S7, S8, S9);
   }
 
   // Case-insensitive case matchers.
   StringSwitch &CaseLower(StringLiteral S, T Value) {
-    if (!Result && Str.equals_insensitive(S))
-      Result = std::move(Value);
-
+    CaseLowerImpl(Value, S);
     return *this;
   }
 
@@ -161,22 +156,22 @@ class StringSwitch {
   }
 
   StringSwitch &CasesLower(StringLiteral S0, StringLiteral S1, T Value) {
-    return CaseLower(S0, Value).CaseLower(S1, Value);
+    return CasesLowerImpl(Value, S0, S1);
   }
 
   StringSwitch &CasesLower(StringLiteral S0, StringLiteral S1, StringLiteral S2,
                            T Value) {
-    return CaseLower(S0, Value).CasesLower(S1, S2, Value);
+    return CasesLowerImpl(Value, S0, S1, S2);
   }
 
   StringSwitch &CasesLower(StringLiteral S0, StringLiteral S1, StringLiteral S2,
                            StringLiteral S3, T Value) {
-    return CaseLower(S0, Value).CasesLower(S1, S2, S3, Value);
+    return CasesLowerImpl(Value, S0, S1, S2, S3);
   }
 
   StringSwitch &CasesLower(StringLiteral S0, StringLiteral S1, StringLiteral S2,
                            StringLiteral S3, StringLiteral S4, T Value) {
-    return CaseLower(S0, Value).CasesLower(S1, S2, S3, S4, Value);
+    return CasesLowerImpl(Value, S0, S1, S2, S3, S4);
   }
 
   [[nodiscard]] R Default(T Value) {
@@ -189,6 +184,39 @@ class StringSwitch {
     assert(Result && "Fell off the end of a string-switch");
     return std::move(*Result);
   }
+
+private:
+  // Returns true when `Str` matches the `S` argument, and stores the result.
+  bool CaseImpl(T &Value, StringLiteral S) {
+    if (!Result && Str == S) {
+      Result = std::move(Value);
+      return true;
+    }
+    return false;
+  }
+
+  // Returns true when `Str` matches the `S` argument (case-insensitive), and
+  // stores the result.
+  bool CaseLowerImpl(T &Value, StringLiteral S) {
+    if (!Result && Str.equals_insensitive(S)) {
+      Result = std::move(Value);
+      return true;
+    }
+    return false;
+  }
+
+  template <typename... Args> StringSwitch &CasesImpl(T &Value, Args... Cases) {
+    // Stop matching after the string is found.
+    (... || CaseImpl(Value, Cases));
+    return *this;
+  }
+
+  template <typename... Args>
+  StringSwitch &CasesLowerImpl(T &Value, Args... Cases) {
+    // Stop matching after the string is found.
+    (... || CaseLowerImpl(Value, Cases));
+    return *this;
+  }
 };
 
 } // end namespace llvm
diff --git a/llvm/include/llvm/ADT/StringTable.h b/llvm/include/llvm/ADT/StringTable.h
index ce5efa1e06ea6..b3c4a414ed6b4 100644
--- a/llvm/include/llvm/ADT/StringTable.h
+++ b/llvm/include/llvm/ADT/StringTable.h
@@ -78,14 +78,11 @@ class StringTable {
     // support `constexpr`.
     assert(!Table.empty() && "Requires at least a valid empty string.");
     assert(Table.data()[0] == '\0' && "Offset zero must be the empty string.");
-    // Ensure that `strlen` from any offset cannot overflow the end of the table
-    // by insisting on a null byte at the end. We also insist on the last string
-    // within the table being *separately* null terminated. This structure is
-    // used to enable predictable iteration over all the strings when needed.
+    // Regardless of how many strings are in the table, the last one should also
+    // be null terminated. This also ensures that computing `strlen` on the
+    // strings can't accidentally run past the end of the table.
     assert(Table.data()[Table.size() - 1] == '\0' &&
            "Last byte must be a null byte.");
-    assert(Table.data()[Table.size() - 2] == '\0' &&
-           "Next-to-last byte must be a null byte.");
   }
 
   // Get a string from the table starting with the provided offset. The returned
diff --git a/llvm/include/llvm/Analysis/BlockFrequencyInfoImpl.h b/llvm/include/llvm/Analysis/BlockFrequencyInfoImpl.h
index 7a7a9594f4760..9509e5234840f 100644
--- a/llvm/include/llvm/Analysis/BlockFrequencyInfoImpl.h
+++ b/llvm/include/llvm/Analysis/BlockFrequencyInfoImpl.h
@@ -1143,14 +1143,15 @@ void BlockFrequencyInfoImpl<BT>::calculate(const FunctionT &F,
 template <class BT>
 void BlockFrequencyInfoImpl<BT>::setBlockFreq(const BlockT *BB,
                                               BlockFrequency Freq) {
-  if (Nodes.count(BB))
-    BlockFrequencyInfoImplBase::setBlockFreq(getNode(BB), Freq);
+  auto [It, Inserted] = Nodes.try_emplace(BB);
+  if (!Inserted)
+    BlockFrequencyInfoImplBase::setBlockFreq(It->second.first, Freq);
   else {
     // If BB is a newly added block after BFI is done, we need to create a new
     // BlockNode for it assigned with a new index. The index can be determined
     // by the size of Freqs.
     BlockNode NewNode(Freqs.size());
-    Nodes[BB] = {NewNode, BFICallbackVH(BB, this)};
+    It->second = {NewNode, BFICallbackVH(BB, this)};
     Freqs.emplace_back();
     BlockFrequencyInfoImplBase::setBlockFreq(NewNode, Freq);
   }
diff --git a/llvm/include/llvm/Analysis/CFGPrinter.h b/llvm/include/llvm/Analysis/CFGPrinter.h
index cd785331d1f14..b844e3f11c4a5 100644
--- a/llvm/include/llvm/Analysis/CFGPrinter.h
+++ b/llvm/include/llvm/Analysis/CFGPrinter.h
@@ -31,6 +31,8 @@
 #include "llvm/Support/FormatVariadic.h"
 
 namespace llvm {
+class ModuleSlotTracker;
+
 template <class GraphType> struct GraphTraits;
 class CFGViewerPass : public PassInfoMixin<CFGViewerPass> {
 public:
@@ -61,6 +63,7 @@ class DOTFuncInfo {
   const Function *F;
   const BlockFrequencyInfo *BFI;
   const BranchProbabilityInfo *BPI;
+  std::unique_ptr<ModuleSlotTracker> MSTStorage;
   uint64_t MaxFreq;
   bool ShowHeat;
   bool EdgeWeights;
@@ -68,14 +71,10 @@ class DOTFuncInfo {
 
 public:
   DOTFuncInfo(const Function *F) : DOTFuncInfo(F, nullptr, nullptr, 0) {}
+  ~DOTFuncInfo();
 
   DOTFuncInfo(const Function *F, const BlockFrequencyInfo *BFI,
-              const BranchProbabilityInfo *BPI, uint64_t MaxFreq)
-      : F(F), BFI(BFI), BPI(BPI), MaxFreq(MaxFreq) {
-    ShowHeat = false;
-    EdgeWeights = !!BPI; // Print EdgeWeights when BPI is available.
-    RawWeights = !!BFI;  // Print RawWeights when BFI is available.
-  }
+              const BranchProbabilityInfo *BPI, uint64_t MaxFreq);
 
   const BlockFrequencyInfo *getBFI() const { return BFI; }
 
@@ -83,6 +82,8 @@ class DOTFuncInfo {
 
   const Function *getFunction() const { return this->F; }
 
+  ModuleSlotTracker *getModuleSlotTracker();
+
   uint64_t getMaxFreq() const { return MaxFreq; }
 
   uint64_t getFreq(const BasicBlock *BB) const {
@@ -203,22 +204,12 @@ struct DOTGraphTraits<DOTFuncInfo *> : public DefaultDOTGraphTraits {
     return SimpleNodeLabelString(Node);
   }
 
-  static void printBasicBlock(raw_string_ostream &OS, const BasicBlock &Node) {
-    // Prepend label name
-    Node.printAsOperand(OS, false);
-    OS << ":\n";
-    for (const Instruction &Inst : Node)
-      OS << Inst << "\n";
-  }
-
   static std::string getCompleteNodeLabel(
       const BasicBlock *Node, DOTFuncInfo *,
       function_ref<void(raw_string_ostream &, const BasicBlock &)>
-          HandleBasicBlock = printBasicBlock,
-      function_ref<void(std::string &, unsigned &, unsigned)>
-          HandleComment = eraseComment) {
-    return CompleteNodeLabelString(Node, HandleBasicBlock, HandleComment);
-  }
+          HandleBasicBlock = {},
+      function_ref<void(std::string &, unsigned &, unsigned)> HandleComment =
+          eraseComment);
 
   std::string getNodeLabel(const BasicBlock *Node, DOTFuncInfo *CFGInfo) {
 
@@ -337,6 +328,6 @@ struct DOTGraphTraits<DOTFuncInfo *> : public DefaultDOTGraphTraits {
   bool isNodeHidden(const BasicBlock *Node, const DOTFuncInfo *CFGInfo);
   void computeDeoptOrUnreachablePaths(const Function *F);
 };
-} // End llvm namespace
+} // namespace llvm
 
 #endif
diff --git a/llvm/include/llvm/Analysis/DDG.h b/llvm/include/llvm/Analysis/DDG.h
index be5f746e31a57..dfd84a9addb97 100644
--- a/llvm/include/llvm/Analysis/DDG.h
+++ b/llvm/include/llvm/Analysis/DDG.h
@@ -453,7 +453,7 @@ bool DependenceGraphInfo<NodeType>::getDependencies(
   for (auto *SrcI : SrcIList)
     for (auto *DstI : DstIList)
       if (auto Dep =
-              const_cast<DependenceInfo *>(&DI)->depends(SrcI, DstI, true))
+              const_cast<DependenceInfo *>(&DI)->depends(SrcI, DstI))
         Deps.push_back(std::move(Dep));
 
   return !Deps.empty();
diff --git a/llvm/include/llvm/Analysis/DependenceAnalysis.h b/llvm/include/llvm/Analysis/DependenceAnalysis.h
index f0a09644e0f4b..426ac757b4b0d 100644
--- a/llvm/include/llvm/Analysis/DependenceAnalysis.h
+++ b/llvm/include/llvm/Analysis/DependenceAnalysis.h
@@ -303,12 +303,8 @@ namespace llvm {
     /// depends - Tests for a dependence between the Src and Dst instructions.
     /// Returns NULL if no dependence; otherwise, returns a Dependence (or a
     /// FullDependence) with as much information as can be gleaned.
-    /// The flag PossiblyLoopIndependent should be set by the caller
-    /// if it appears that control flow can reach from Src to Dst
-    /// without traversing a loop back edge.
     std::unique_ptr<Dependence> depends(Instruction *Src,
-                                        Instruction *Dst,
-                                        bool PossiblyLoopIndependent);
+                                        Instruction *Dst);
 
     /// getSplitIteration - Give a dependence that's splittable at some
     /// particular level, return the iteration that should be used to split
diff --git a/llvm/include/llvm/Analysis/KernelInfo.h b/llvm/include/llvm/Analysis/KernelInfo.h
new file mode 100644
index 0000000000000..75d92c202212b
--- /dev/null
+++ b/llvm/include/llvm/Analysis/KernelInfo.h
@@ -0,0 +1,35 @@
+//=- KernelInfo.h - Kernel Analysis -------------------------------*- C++ -*-=//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file defines the KernelInfoPrinter class used to emit remarks about
+// function properties from a GPU kernel.
+//
+// See llvm/docs/KernelInfo.rst.
+// ===---------------------------------------------------------------------===//
+
+#ifndef LLVM_ANALYSIS_KERNELINFO_H
+#define LLVM_ANALYSIS_KERNELINFO_H
+
+#include "llvm/IR/PassManager.h"
+
+namespace llvm {
+
+class TargetMachine;
+
+class KernelInfoPrinter : public PassInfoMixin<KernelInfoPrinter> {
+  TargetMachine *TM;
+
+public:
+  explicit KernelInfoPrinter(TargetMachine *TM) : TM(TM) {}
+
+  PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM);
+
+  static bool isRequired() { return true; }
+};
+} // namespace llvm
+#endif // LLVM_ANALYSIS_KERNELINFO_H
diff --git a/llvm/include/llvm/Analysis/LoopAccessAnalysis.h b/llvm/include/llvm/Analysis/LoopAccessAnalysis.h
index 31374a128856c..6fc6ca14d0889 100644
--- a/llvm/include/llvm/Analysis/LoopAccessAnalysis.h
+++ b/llvm/include/llvm/Analysis/LoopAccessAnalysis.h
@@ -853,6 +853,25 @@ bool sortPtrAccesses(ArrayRef<Value *> VL, Type *ElemTy, const DataLayout &DL,
 bool isConsecutiveAccess(Value *A, Value *B, const DataLayout &DL,
                          ScalarEvolution &SE, bool CheckType = true);
 
+/// Calculate Start and End points of memory access.
+/// Let's assume A is the first access and B is a memory access on N-th loop
+/// iteration. Then B is calculated as:
+///   B = A + Step*N .
+/// Step value may be positive or negative.
+/// N is a calculated back-edge taken count:
+///     N = (TripCount > 0) ? RoundDown(TripCount -1 , VF) : 0
+/// Start and End points are calculated in the following way:
+/// Start = UMIN(A, B) ; End = UMAX(A, B) + SizeOfElt,
+/// where SizeOfElt is the size of single memory access in bytes.
+///
+/// There is no conflict when the intervals are disjoint:
+/// NoConflict = (P2.Start >= P1.End) || (P1.Start >= P2.End)
+std::pair<const SCEV *, const SCEV *> getStartAndEndForAccess(
+    const Loop *Lp, const SCEV *PtrExpr, Type *AccessTy, const SCEV *MaxBECount,
+    ScalarEvolution *SE,
+    DenseMap<std::pair<const SCEV *, Type *>,
+             std::pair<const SCEV *, const SCEV *>> *PointerBounds);
+
 class LoopAccessInfoManager {
   /// The cache.
   DenseMap<Loop *, std::unique_ptr<LoopAccessInfo>> LoopAccessInfoMap;
diff --git a/llvm/include/llvm/Analysis/MemoryProfileInfo.h b/llvm/include/llvm/Analysis/MemoryProfileInfo.h
index 215139caef696..f75783a4fef50 100644
--- a/llvm/include/llvm/Analysis/MemoryProfileInfo.h
+++ b/llvm/include/llvm/Analysis/MemoryProfileInfo.h
@@ -56,6 +56,37 @@ class CallStackTrie {
     // Allocation types for call context sharing the context prefix at this
     // node.
     uint8_t AllocTypes;
+    // Updated as we add allocations to note if this is the deepest point in the
+    // trie that has an ambiguous allocation type (both Cold and NotCold). It is
+    // used to prune unneeded NotCold contexts, taking advantage of the fact
+    // that we later will only clone Cold contexts, as NotCold is the allocation
+    // default. We only need to keep as metadata the NotCold contexts that
+    // overlap the longest with Cold allocations, so that we know how deeply we
+    // need to clone. For example, assume we add the following contexts to the
+    // trie:
+    //    1 3 (notcold)
+    //    1 2 4 (cold)
+    //    1 2 5 (notcold)
+    //    1 2 6 (notcold)
+    // the trie looks like:
+    //         1
+    //        / \
+    //       2   3
+    //      /|\
+    //     4 5 6
+    //
+    // It is sufficient to prune all but one not cold contexts (either 1,2,5 or
+    // 1,2,6, we arbitrarily keep the first one we encounter which will be
+    // 1,2,5). We'll initially have DeepestAmbiguousAllocType set false for trie
+    // node 1 after the trie is built, and true for node 2. This indicates that
+    // the not cold context ending in 3 is not needed (its immediate callee has
+    // this value set false). The first notcold context we encounter when
+    // iterating the callers of node 2 will be the context ending in 5 (since
+    // std::map iteration is in sorted order of key), which will see that this
+    // field is true for its callee, so we will keep it. But at that point we
+    // set the callee's flag to false which prevents adding the not cold context
+    // ending in 6 unnecessarily.
+    bool DeepestAmbiguousAllocType = true;
     // If the user has requested reporting of hinted sizes, keep track of the
     // associated full stack id and profiled sizes. Can have more than one
     // after trimming (e.g. when building from metadata). This is only placed on
@@ -65,6 +96,15 @@ class CallStackTrie {
     std::map<uint64_t, CallStackTrieNode *> Callers;
     CallStackTrieNode(AllocationType Type)
         : AllocTypes(static_cast<uint8_t>(Type)) {}
+    void addAllocType(AllocationType AllocType) {
+      AllocTypes |= static_cast<uint8_t>(AllocType);
+    }
+    void removeAllocType(AllocationType AllocType) {
+      AllocTypes &= ~static_cast<uint8_t>(AllocType);
+    }
+    bool hasAllocType(AllocationType AllocType) const {
+      return AllocTypes & static_cast<uint8_t>(AllocType);
+    }
   };
 
   // The node for the allocation at the root.
@@ -85,11 +125,17 @@ class CallStackTrie {
   void collectContextSizeInfo(CallStackTrieNode *Node,
                               std::vector<ContextTotalSize> &ContextSizeInfo);
 
+  // Recursively convert hot allocation types to notcold, since we don't
+  // actually do any cloning for hot contexts, to facilitate more aggressive
+  // pruning of contexts.
+  void convertHotToNotCold(CallStackTrieNode *Node);
+
   // Recursive helper to trim contexts and create metadata nodes.
   bool buildMIBNodes(CallStackTrieNode *Node, LLVMContext &Ctx,
                      std::vector<uint64_t> &MIBCallStack,
                      std::vector<Metadata *> &MIBNodes,
-                     bool CalleeHasAmbiguousCallerContext);
+                     bool CalleeHasAmbiguousCallerContext,
+                     bool &CalleeDeepestAmbiguousAllocType);
 
 public:
   CallStackTrie() = default;
diff --git a/llvm/include/llvm/Analysis/RegionInfoImpl.h b/llvm/include/llvm/Analysis/RegionInfoImpl.h
index db3a9027549ce..eb99d8bc6fb23 100644
--- a/llvm/include/llvm/Analysis/RegionInfoImpl.h
+++ b/llvm/include/llvm/Analysis/RegionInfoImpl.h
@@ -720,16 +720,14 @@ void RegionInfoBase<Tr>::buildRegionsTree(DomTreeNodeT *N, RegionT *region) {
   while (BB == region->getExit())
     region = region->getParent();
 
-  typename BBtoRegionMap::iterator it = BBtoRegion.find(BB);
+  auto [It, Inserted] = BBtoRegion.try_emplace(BB, region);
 
   // This basic block is a start block of a region. It is already in the
   // BBtoRegion relation. Only the child basic blocks have to be updated.
-  if (it != BBtoRegion.end()) {
-    RegionT *newRegion = it->second;
+  if (!Inserted) {
+    RegionT *newRegion = It->second;
     region->addSubRegion(getTopMostParent(newRegion));
     region = newRegion;
-  } else {
-    BBtoRegion[BB] = region;
   }
 
   for (DomTreeNodeBase<BlockT> *C : *N) {
diff --git a/llvm/include/llvm/Analysis/ScalarEvolution.h b/llvm/include/llvm/Analysis/ScalarEvolution.h
index a011f16b4d95c..f729b07076d29 100644
--- a/llvm/include/llvm/Analysis/ScalarEvolution.h
+++ b/llvm/include/llvm/Analysis/ScalarEvolution.h
@@ -1188,21 +1188,19 @@ class ScalarEvolution {
                             ICmpInst::Predicate Pred);
 
   struct LoopInvariantPredicate {
-    ICmpInst::Predicate Pred;
+    CmpPredicate Pred;
     const SCEV *LHS;
     const SCEV *RHS;
 
-    LoopInvariantPredicate(ICmpInst::Predicate Pred, const SCEV *LHS,
-                           const SCEV *RHS)
+    LoopInvariantPredicate(CmpPredicate Pred, const SCEV *LHS, const SCEV *RHS)
         : Pred(Pred), LHS(LHS), RHS(RHS) {}
   };
   /// If the result of the predicate LHS `Pred` RHS is loop invariant with
   /// respect to L, return a LoopInvariantPredicate with LHS and RHS being
   /// invariants, available at L's entry. Otherwise, return std::nullopt.
   std::optional<LoopInvariantPredicate>
-  getLoopInvariantPredicate(ICmpInst::Predicate Pred, const SCEV *LHS,
-                            const SCEV *RHS, const Loop *L,
-                            const Instruction *CtxI = nullptr);
+  getLoopInvariantPredicate(CmpPredicate Pred, const SCEV *LHS, const SCEV *RHS,
+                            const Loop *L, const Instruction *CtxI = nullptr);
 
   /// If the result of the predicate LHS `Pred` RHS is loop invariant with
   /// respect to L at given Context during at least first MaxIter iterations,
diff --git a/llvm/include/llvm/Analysis/ScalarEvolutionExpressions.h b/llvm/include/llvm/Analysis/ScalarEvolutionExpressions.h
index 6eb1aca1cf76a..872746b7df5ca 100644
--- a/llvm/include/llvm/Analysis/ScalarEvolutionExpressions.h
+++ b/llvm/include/llvm/Analysis/ScalarEvolutionExpressions.h
@@ -944,10 +944,11 @@ class SCEVLoopAddRecRewriter
       Operands.push_back(visit(Op));
 
     const Loop *L = Expr->getLoop();
-    if (0 == Map.count(L))
+    auto It = Map.find(L);
+    if (It == Map.end())
       return SE.getAddRecExpr(Operands, L, Expr->getNoWrapFlags());
 
-    return SCEVAddRecExpr::evaluateAtIteration(Operands, Map[L], SE);
+    return SCEVAddRecExpr::evaluateAtIteration(Operands, It->second, SE);
   }
 
 private:
diff --git a/llvm/include/llvm/Analysis/TargetTransformInfo.h b/llvm/include/llvm/Analysis/TargetTransformInfo.h
index 71b204f9c3fec..08ab4ee2ec1cf 100644
--- a/llvm/include/llvm/Analysis/TargetTransformInfo.h
+++ b/llvm/include/llvm/Analysis/TargetTransformInfo.h
@@ -622,6 +622,11 @@ class TargetTransformInfo {
     /// Don't allow runtime unrolling if expanding the trip count takes more
     /// than SCEVExpansionBudget.
     unsigned SCEVExpansionBudget;
+    /// Allow runtime unrolling multi-exit loops. Should only be set if the
+    /// target determined that multi-exit unrolling is profitable for the loop.
+    /// Fall back to the generic logic to determine whether multi-exit unrolling
+    /// is profitable if set to false.
+    bool RuntimeUnrollMultiExit;
   };
 
   /// Get target-customized preferences for the generic loop unrolling
@@ -1512,6 +1517,19 @@ class TargetTransformInfo {
       Align Alignment, TTI::TargetCostKind CostKind = TTI::TCK_RecipThroughput,
       const Instruction *I = nullptr) const;
 
+  /// \return The cost of Expand Load or Compress Store operation
+  /// \p Opcode - is a type of memory access Load or Store
+  /// \p Src - a vector type of the data to be loaded or stored
+  /// \p VariableMask - true when the memory access is predicated with a mask
+  ///                   that is not a compile-time constant
+  /// \p Alignment - alignment of single element
+  /// \p I - the optional original context instruction, if one exists, e.g. the
+  ///        load/store to transform or the call to the gather/scatter intrinsic
+  InstructionCost getExpandCompressMemoryOpCost(
+      unsigned Opcode, Type *DataTy, bool VariableMask, Align Alignment,
+      TTI::TargetCostKind CostKind = TTI::TCK_RecipThroughput,
+      const Instruction *I = nullptr) const;
+
   /// \return The cost of strided memory operations.
   /// \p Opcode - is a type of memory access Load or Store
   /// \p DataTy - a vector type of the data to be loaded or stored
@@ -1886,6 +1904,11 @@ class TargetTransformInfo {
 
   /// @}
 
+  /// Collect kernel launch bounds for \p F into \p LB.
+  void collectKernelLaunchBounds(
+      const Function &F,
+      SmallVectorImpl<std::pair<StringRef, int64_t>> &LB) const;
+
 private:
   /// The abstract base class used to type erase specific TTI
   /// implementations.
@@ -2218,6 +2241,9 @@ class TargetTransformInfo::Concept {
                          bool VariableMask, Align Alignment,
                          TTI::TargetCostKind CostKind,
                          const Instruction *I = nullptr) = 0;
+  virtual InstructionCost getExpandCompressMemoryOpCost(
+      unsigned Opcode, Type *DataTy, bool VariableMask, Align Alignment,
+      TTI::TargetCostKind CostKind, const Instruction *I = nullptr) = 0;
   virtual InstructionCost
   getStridedMemoryOpCost(unsigned Opcode, Type *DataTy, const Value *Ptr,
                          bool VariableMask, Align Alignment,
@@ -2324,6 +2350,9 @@ class TargetTransformInfo::Concept {
   virtual unsigned getMaxNumArgs() const = 0;
   virtual unsigned getNumBytesToPadGlobalArray(unsigned Size,
                                                Type *ArrayType) const = 0;
+  virtual void collectKernelLaunchBounds(
+      const Function &F,
+      SmallVectorImpl<std::pair<StringRef, int64_t>> &LB) const = 0;
 };
 
 template <typename T>
@@ -2950,6 +2979,12 @@ class TargetTransformInfo::Model final : public TargetTransformInfo::Concept {
     return Impl.getGatherScatterOpCost(Opcode, DataTy, Ptr, VariableMask,
                                        Alignment, CostKind, I);
   }
+  InstructionCost getExpandCompressMemoryOpCost(
+      unsigned Opcode, Type *DataTy, bool VariableMask, Align Alignment,
+      TTI::TargetCostKind CostKind, const Instruction *I = nullptr) override {
+    return Impl.getExpandCompressMemoryOpCost(Opcode, DataTy, VariableMask,
+                                              Alignment, CostKind, I);
+  }
   InstructionCost
   getStridedMemoryOpCost(unsigned Opcode, Type *DataTy, const Value *Ptr,
                          bool VariableMask, Align Alignment,
@@ -3169,6 +3204,12 @@ class TargetTransformInfo::Model final : public TargetTransformInfo::Concept {
                                        Type *ArrayType) const override {
     return Impl.getNumBytesToPadGlobalArray(Size, ArrayType);
   }
+
+  void collectKernelLaunchBounds(
+      const Function &F,
+      SmallVectorImpl<std::pair<StringRef, int64_t>> &LB) const override {
+    Impl.collectKernelLaunchBounds(F, LB);
+  }
 };
 
 template <typename T>
diff --git a/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h b/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
index dcef4a1abcfa3..5128c6b86a5f0 100644
--- a/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
+++ b/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
@@ -774,6 +774,12 @@ class TargetTransformInfoImplBase {
     return 1;
   }
 
+  InstructionCost getExpandCompressMemoryOpCost(
+      unsigned Opcode, Type *DataTy, bool VariableMask, Align Alignment,
+      TTI::TargetCostKind CostKind, const Instruction *I = nullptr) const {
+    return 1;
+  }
+
   InstructionCost getStridedMemoryOpCost(unsigned Opcode, Type *DataTy,
                                          const Value *Ptr, bool VariableMask,
                                          Align Alignment,
@@ -1049,6 +1055,10 @@ class TargetTransformInfoImplBase {
     return 0;
   }
 
+  void collectKernelLaunchBounds(
+      const Function &F,
+      SmallVectorImpl<std::pair<StringRef, int64_t>> &LB) const {}
+
 protected:
   // Obtain the minimum required size to hold the value (without the sign)
   // In case of a vector it returns the min required size for one element.
diff --git a/llvm/include/llvm/Analysis/ValueTracking.h b/llvm/include/llvm/Analysis/ValueTracking.h
index b4918c2d1e8a1..dba54be4c92f8 100644
--- a/llvm/include/llvm/Analysis/ValueTracking.h
+++ b/llvm/include/llvm/Analysis/ValueTracking.h
@@ -164,11 +164,8 @@ bool isKnownNegative(const Value *V, const SimplifyQuery &SQ,
 
 /// Return true if the given values are known to be non-equal when defined.
 /// Supports scalar integer types only.
-bool isKnownNonEqual(const Value *V1, const Value *V2, const DataLayout &DL,
-                     AssumptionCache *AC = nullptr,
-                     const Instruction *CxtI = nullptr,
-                     const DominatorTree *DT = nullptr,
-                     bool UseInstrInfo = true);
+bool isKnownNonEqual(const Value *V1, const Value *V2, const SimplifyQuery &SQ,
+                     unsigned Depth = 0);
 
 /// Return true if 'V & Mask' is known to be zero. We use this predicate to
 /// simplify operations downstream. Mask is known to be zero for bits that V
diff --git a/llvm/include/llvm/Analysis/VectorUtils.h b/llvm/include/llvm/Analysis/VectorUtils.h
index 5d41d1cd14ef4..f21594c557e0e 100644
--- a/llvm/include/llvm/Analysis/VectorUtils.h
+++ b/llvm/include/llvm/Analysis/VectorUtils.h
@@ -748,12 +748,11 @@ class InterleavedAccessInfo {
   /// \returns the newly created interleave group.
   InterleaveGroup<Instruction> *
   createInterleaveGroup(Instruction *Instr, int Stride, Align Alignment) {
-    assert(!InterleaveGroupMap.count(Instr) &&
-           "Already in an interleaved access group");
-    InterleaveGroupMap[Instr] =
-        new InterleaveGroup<Instruction>(Instr, Stride, Alignment);
-    InterleaveGroups.insert(InterleaveGroupMap[Instr]);
-    return InterleaveGroupMap[Instr];
+    auto [It, Inserted] = InterleaveGroupMap.try_emplace(Instr);
+    assert(Inserted && "Already in an interleaved access group");
+    It->second = new InterleaveGroup<Instruction>(Instr, Stride, Alignment);
+    InterleaveGroups.insert(It->second);
+    return It->second;
   }
 
   /// Release the group and remove all the relationships.
diff --git a/llvm/include/llvm/AsmParser/LLToken.h b/llvm/include/llvm/AsmParser/LLToken.h
index 7b47bc88ddb25..4a431383e0a1c 100644
--- a/llvm/include/llvm/AsmParser/LLToken.h
+++ b/llvm/include/llvm/AsmParser/LLToken.h
@@ -202,10 +202,11 @@ enum Kind {
   kw_argmem,
   kw_inaccessiblemem,
 
-  // Legacy memory attributes:
+  // Legacy attributes:
   kw_argmemonly,
   kw_inaccessiblememonly,
   kw_inaccessiblemem_or_argmemonly,
+  kw_nocapture,
 
   // Captures attribute:
   kw_address,
@@ -497,6 +498,7 @@ enum Kind {
   DwarfMacinfo,     // DW_MACINFO_foo
   ChecksumKind,     // CSK_foo
   DbgRecordType,    // dbg_foo
+  DwarfEnumKind,    // DW_APPLE_ENUM_KIND_foo
 
   // Type valued tokens (TyVal).
   Type,
diff --git a/llvm/include/llvm/BinaryFormat/DXContainer.h b/llvm/include/llvm/BinaryFormat/DXContainer.h
index 21e28d546286e..fbab066bf4517 100644
--- a/llvm/include/llvm/BinaryFormat/DXContainer.h
+++ b/llvm/include/llvm/BinaryFormat/DXContainer.h
@@ -14,6 +14,8 @@
 #define LLVM_BINARYFORMAT_DXCONTAINER_H
 
 #include "llvm/ADT/StringRef.h"
+#include "llvm/Support/BinaryStreamError.h"
+#include "llvm/Support/Error.h"
 #include "llvm/Support/SwapByteOrder.h"
 #include "llvm/TargetParser/Triple.h"
 
@@ -152,6 +154,11 @@ enum class FeatureFlags : uint64_t {
 static_assert((uint64_t)FeatureFlags::NextUnusedBit <= 1ull << 63,
               "Shader flag bits exceed enum size.");
 
+#define ROOT_ELEMENT_FLAG(Num, Val) Val = 1ull << Num,
+enum class RootElementFlag : uint32_t {
+#include "DXContainerConstants.def"
+};
+
 PartType parsePartType(StringRef S);
 
 struct VertexPSVInfo {
@@ -541,6 +548,23 @@ struct ProgramSignatureElement {
 static_assert(sizeof(ProgramSignatureElement) == 32,
               "ProgramSignatureElement is misaligned");
 
+struct RootSignatureValidations {
+
+  static Expected<uint32_t> validateRootFlag(uint32_t Flags) {
+    if ((Flags & ~0x80000fff) != 0)
+      return llvm::make_error<BinaryStreamError>("Invalid Root Signature flag");
+    return Flags;
+  }
+
+  static Expected<uint32_t> validateVersion(uint32_t Version) {
+    if (Version == 1 || Version == 2)
+      return Version;
+
+    return llvm::make_error<BinaryStreamError>(
+        "Invalid Root Signature Version");
+  }
+};
+
 } // namespace dxbc
 } // namespace llvm
 
diff --git a/llvm/include/llvm/BinaryFormat/DXContainerConstants.def b/llvm/include/llvm/BinaryFormat/DXContainerConstants.def
index 96d4499c9cadc..6d44ea14df444 100644
--- a/llvm/include/llvm/BinaryFormat/DXContainerConstants.def
+++ b/llvm/include/llvm/BinaryFormat/DXContainerConstants.def
@@ -4,6 +4,7 @@ CONTAINER_PART(DXIL)
 CONTAINER_PART(SFI0)
 CONTAINER_PART(HASH)
 CONTAINER_PART(PSV0)
+CONTAINER_PART(RTS0)
 CONTAINER_PART(ISG1)
 CONTAINER_PART(OSG1)
 CONTAINER_PART(PSG1)
@@ -52,6 +53,26 @@ SHADER_FEATURE_FLAG(31, 36, NextUnusedBit, "Next reserved shader flag bit (not a
 #undef SHADER_FEATURE_FLAG
 #endif // SHADER_FEATURE_FLAG
 
+
+// ROOT_ELEMENT_FLAG(bit offset for the flag, name).
+#ifdef ROOT_ELEMENT_FLAG
+
+ROOT_ELEMENT_FLAG(0, AllowInputAssemblerInputLayout)
+ROOT_ELEMENT_FLAG(1, DenyVertexShaderRootAccess)
+ROOT_ELEMENT_FLAG(2, DenyHullShaderRootAccess)
+ROOT_ELEMENT_FLAG(3, DenyDomainShaderRootAccess)
+ROOT_ELEMENT_FLAG(4, DenyGeometryShaderRootAccess)
+ROOT_ELEMENT_FLAG(5, DenyPixelShaderRootAccess)
+ROOT_ELEMENT_FLAG(6, AllowStreamOutput)
+ROOT_ELEMENT_FLAG(7, LocalRootSignature)
+ROOT_ELEMENT_FLAG(8, DenyAmplificationShaderRootAccess)
+ROOT_ELEMENT_FLAG(9, DenyMeshShaderRootAccess)
+ROOT_ELEMENT_FLAG(10, CBVSRVUAVHeapDirectlyIndexed)
+ROOT_ELEMENT_FLAG(11, SamplerHeapDirectlyIndexed)
+#undef ROOT_ELEMENT_FLAG
+#endif // ROOT_ELEMENT_FLAG
+
+
 #ifdef DXIL_MODULE_FLAG
 
 // Only save DXIL module flags which not map to feature flags here.
diff --git a/llvm/include/llvm/BinaryFormat/Dwarf.def b/llvm/include/llvm/BinaryFormat/Dwarf.def
index 2bb84fbc864d8..724a14ccc7aea 100644
--- a/llvm/include/llvm/BinaryFormat/Dwarf.def
+++ b/llvm/include/llvm/BinaryFormat/Dwarf.def
@@ -24,7 +24,8 @@
       (defined HANDLE_DW_CFA && defined HANDLE_DW_CFA_PRED) ||                 \
       defined HANDLE_DW_APPLE_PROPERTY || defined HANDLE_DW_UT ||              \
       defined HANDLE_DWARF_SECTION || defined HANDLE_DW_IDX ||                 \
-      defined HANDLE_DW_END || defined HANDLE_DW_SECT)
+      defined HANDLE_DW_END || defined HANDLE_DW_SECT ||                       \
+      defined HANDLE_DW_APPLE_ENUM_KIND)
 #error "Missing macro definition of HANDLE_DW*"
 #endif
 
@@ -146,6 +147,10 @@
 #define HANDLE_DW_SECT(ID, NAME)
 #endif
 
+#ifndef HANDLE_DW_APPLE_ENUM_KIND
+#define HANDLE_DW_APPLE_ENUM_KIND(ID, NAME)
+#endif
+
 HANDLE_DW_TAG(0x0000, null, 2, DWARF, DW_KIND_NONE)
 HANDLE_DW_TAG(0x0001, array_type, 2, DWARF, DW_KIND_TYPE)
 HANDLE_DW_TAG(0x0002, class_type, 2, DWARF, DW_KIND_TYPE)
@@ -638,6 +643,7 @@ HANDLE_DW_AT(0x3fed, APPLE_property, 0, APPLE)
 HANDLE_DW_AT(0x3fee, APPLE_objc_direct, 0, APPLE)
 HANDLE_DW_AT(0x3fef, APPLE_sdk, 0, APPLE)
 HANDLE_DW_AT(0x3ff0, APPLE_origin, 0, APPLE)
+HANDLE_DW_AT(0x3ff1, APPLE_enum_kind, 0, APPLE)
 
 // Attribute form encodings.
 HANDLE_DW_FORM(0x01, addr, 2, DWARF)
@@ -1269,6 +1275,11 @@ HANDLE_DW_APPLE_PROPERTY(0x1000, nullability)
 HANDLE_DW_APPLE_PROPERTY(0x2000, null_resettable)
 HANDLE_DW_APPLE_PROPERTY(0x4000, class)
 
+// Enum kinds.
+// Keep in sync with EnumExtensibilityAttr::Kind.
+HANDLE_DW_APPLE_ENUM_KIND(0x00, Closed)
+HANDLE_DW_APPLE_ENUM_KIND(0x01, Open)
+
 // DWARF v5 Unit Types.
 HANDLE_DW_UT(0x01, compile)
 HANDLE_DW_UT(0x02, type)
@@ -1367,3 +1378,4 @@ HANDLE_DW_SECT(8, RNGLISTS)
 #undef HANDLE_DW_IDX
 #undef HANDLE_DW_END
 #undef HANDLE_DW_SECT
+#undef HANDLE_DW_APPLE_ENUM_KIND
diff --git a/llvm/include/llvm/BinaryFormat/Dwarf.h b/llvm/include/llvm/BinaryFormat/Dwarf.h
index 3be819c0a76ee..397b4b164386d 100644
--- a/llvm/include/llvm/BinaryFormat/Dwarf.h
+++ b/llvm/include/llvm/BinaryFormat/Dwarf.h
@@ -44,9 +44,10 @@ namespace dwarf {
 enum LLVMConstants : uint32_t {
   /// LLVM mock tags (see also llvm/BinaryFormat/Dwarf.def).
   /// \{
-  DW_TAG_invalid = ~0U,        ///< Tag for invalid results.
-  DW_VIRTUALITY_invalid = ~0U, ///< Virtuality for invalid results.
-  DW_MACINFO_invalid = ~0U,    ///< Macinfo type for invalid results.
+  DW_TAG_invalid = ~0U,             ///< Tag for invalid results.
+  DW_VIRTUALITY_invalid = ~0U,      ///< Virtuality for invalid results.
+  DW_MACINFO_invalid = ~0U,         ///< Macinfo type for invalid results.
+  DW_APPLE_ENUM_KIND_invalid = ~0U, ///< Enum kind for invalid results.
   /// \}
 
   /// Special values for an initial length field.
@@ -198,6 +199,12 @@ enum VirtualityAttribute {
   DW_VIRTUALITY_max = 0x02
 };
 
+enum EnumKindAttribute {
+#define HANDLE_DW_APPLE_ENUM_KIND(ID, NAME) DW_APPLE_ENUM_KIND_##NAME = ID,
+#include "llvm/BinaryFormat/Dwarf.def"
+  DW_APPLE_ENUM_KIND_max = 0x01
+};
+
 enum DefaultedMemberAttribute {
 #define HANDLE_DW_DEFAULTED(ID, NAME) DW_DEFAULTED_##NAME = ID,
 #include "llvm/BinaryFormat/Dwarf.def"
@@ -981,6 +988,7 @@ StringRef AccessibilityString(unsigned Access);
 StringRef DefaultedMemberString(unsigned DefaultedEncodings);
 StringRef VisibilityString(unsigned Visibility);
 StringRef VirtualityString(unsigned Virtuality);
+StringRef EnumKindString(unsigned EnumKind);
 StringRef LanguageString(unsigned Language);
 StringRef CaseString(unsigned Case);
 StringRef ConventionString(unsigned Convention);
@@ -1020,6 +1028,7 @@ unsigned getOperationEncoding(StringRef OperationEncodingString);
 unsigned getSubOperationEncoding(unsigned OpEncoding,
                                  StringRef SubOperationEncodingString);
 unsigned getVirtuality(StringRef VirtualityString);
+unsigned getEnumKind(StringRef EnumKindString);
 unsigned getLanguage(StringRef LanguageString);
 unsigned getCallingConvention(StringRef LanguageString);
 unsigned getAttributeEncoding(StringRef EncodingString);
diff --git a/llvm/include/llvm/BinaryFormat/ELF.h b/llvm/include/llvm/BinaryFormat/ELF.h
index 48ae0db80f43e..4b826bbf58f17 100644
--- a/llvm/include/llvm/BinaryFormat/ELF.h
+++ b/llvm/include/llvm/BinaryFormat/ELF.h
@@ -619,6 +619,7 @@ enum {
   EF_HEXAGON_MACH_V5 = 0x00000004,   // Hexagon V5
   EF_HEXAGON_MACH_V55 = 0x00000005,  // Hexagon V55
   EF_HEXAGON_MACH_V60 = 0x00000060,  // Hexagon V60
+  EF_HEXAGON_MACH_V61 = 0x00000061,  // Hexagon V61
   EF_HEXAGON_MACH_V62 = 0x00000062,  // Hexagon V62
   EF_HEXAGON_MACH_V65 = 0x00000065,  // Hexagon V65
   EF_HEXAGON_MACH_V66 = 0x00000066,  // Hexagon V66
@@ -630,7 +631,11 @@ enum {
   EF_HEXAGON_MACH_V71T = 0x00008071, // Hexagon V71T
   EF_HEXAGON_MACH_V73 = 0x00000073,  // Hexagon V73
   EF_HEXAGON_MACH_V75 = 0x00000075,  // Hexagon V75
+  EF_HEXAGON_MACH_V77 = 0x00000077,  // Hexagon V77
   EF_HEXAGON_MACH_V79 = 0x00000079,  // Hexagon V79
+  EF_HEXAGON_MACH_V81 = 0x00000081,  // Hexagon V81
+  EF_HEXAGON_MACH_V83 = 0x00000083,  // Hexagon V83
+  EF_HEXAGON_MACH_V85 = 0x00000085,  // Hexagon V85
   EF_HEXAGON_MACH = 0x000003ff,      // Hexagon V..
 
   // Highest ISA version flags
@@ -642,6 +647,7 @@ enum {
   EF_HEXAGON_ISA_V5 = 0x00000040,   // Hexagon V5 ISA
   EF_HEXAGON_ISA_V55 = 0x00000050,  // Hexagon V55 ISA
   EF_HEXAGON_ISA_V60 = 0x00000060,  // Hexagon V60 ISA
+  EF_HEXAGON_ISA_V61 = 0x00000061,  // Hexagon V61 ISA
   EF_HEXAGON_ISA_V62 = 0x00000062,  // Hexagon V62 ISA
   EF_HEXAGON_ISA_V65 = 0x00000065,  // Hexagon V65 ISA
   EF_HEXAGON_ISA_V66 = 0x00000066,  // Hexagon V66 ISA
@@ -651,7 +657,11 @@ enum {
   EF_HEXAGON_ISA_V71 = 0x00000071,  // Hexagon V71 ISA
   EF_HEXAGON_ISA_V73 = 0x00000073,  // Hexagon V73 ISA
   EF_HEXAGON_ISA_V75 = 0x00000075,  // Hexagon V75 ISA
+  EF_HEXAGON_ISA_V77 = 0x00000077,  // Hexagon V77 ISA
   EF_HEXAGON_ISA_V79 = 0x00000079,  // Hexagon V79 ISA
+  EF_HEXAGON_ISA_V81 = 0x00000081,  // Hexagon V81 ISA
+  EF_HEXAGON_ISA_V83 = 0x00000083,  // Hexagon V83 ISA
+  EF_HEXAGON_ISA_V85 = 0x00000085,  // Hexagon V85 ISA
   EF_HEXAGON_ISA = 0x000003ff,      // Hexagon V.. ISA
 };
 
@@ -1788,6 +1798,7 @@ enum : unsigned {
   GNU_PROPERTY_AARCH64_FEATURE_1_AND = 0xc0000000,
   GNU_PROPERTY_AARCH64_FEATURE_PAUTH = 0xc0000001,
   GNU_PROPERTY_X86_FEATURE_1_AND = 0xc0000002,
+  GNU_PROPERTY_RISCV_FEATURE_1_AND = 0xc0000000,
 
   GNU_PROPERTY_X86_UINT32_OR_LO = 0xc0008000,
   GNU_PROPERTY_X86_FEATURE_2_NEEDED = GNU_PROPERTY_X86_UINT32_OR_LO + 1,
@@ -1852,6 +1863,13 @@ enum : unsigned {
   GNU_PROPERTY_X86_ISA_1_V4 = 1 << 3,
 };
 
+// RISC-V processor feature bits.
+enum : unsigned {
+  GNU_PROPERTY_RISCV_FEATURE_1_CFI_LP_UNLABELED = 1 << 0,
+  GNU_PROPERTY_RISCV_FEATURE_1_CFI_SS = 1 << 1,
+  GNU_PROPERTY_RISCV_FEATURE_1_CFI_LP_FUNC_SIG = 1 << 2,
+};
+
 // FreeBSD note types.
 enum {
   NT_FREEBSD_ABI_TAG = 1,
diff --git a/llvm/include/llvm/CodeGen/AsmPrinter.h b/llvm/include/llvm/CodeGen/AsmPrinter.h
index 5291369b3b9f1..3da63af5ba571 100644
--- a/llvm/include/llvm/CodeGen/AsmPrinter.h
+++ b/llvm/include/llvm/CodeGen/AsmPrinter.h
@@ -893,6 +893,9 @@ class AsmPrinter : public MachineFunctionPass {
   // Internal Implementation Details
   //===------------------------------------------------------------------===//
 
+  void emitJumpTableImpl(const MachineJumpTableInfo &MJTI,
+                         ArrayRef<unsigned> JumpTableIndices,
+                         bool JTInDiffSection);
   void emitJumpTableEntry(const MachineJumpTableInfo &MJTI,
                           const MachineBasicBlock *MBB, unsigned uid) const;
 
diff --git a/llvm/include/llvm/CodeGen/BasicTTIImpl.h b/llvm/include/llvm/CodeGen/BasicTTIImpl.h
index 596db39239213..339b83637fa8f 100644
--- a/llvm/include/llvm/CodeGen/BasicTTIImpl.h
+++ b/llvm/include/llvm/CodeGen/BasicTTIImpl.h
@@ -1468,6 +1468,15 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
                                        true, CostKind);
   }
 
+  InstructionCost getExpandCompressMemoryOpCost(
+      unsigned Opcode, Type *DataTy, bool VariableMask, Align Alignment,
+      TTI::TargetCostKind CostKind, const Instruction *I = nullptr) {
+    // Treat expand load/compress store as gather/scatter operation.
+    // TODO: implement more precise cost estimation for these intrinsics.
+    return getCommonMaskedMemoryOpCost(Opcode, DataTy, Alignment, VariableMask,
+                                       /*IsGatherScatter*/ true, CostKind);
+  }
+
   InstructionCost getStridedMemoryOpCost(unsigned Opcode, Type *DataTy,
                                          const Value *Ptr, bool VariableMask,
                                          Align Alignment,
@@ -1776,6 +1785,21 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
       return thisT()->getGatherScatterOpCost(Instruction::Load, RetTy, Args[0],
                                              VarMask, Alignment, CostKind, I);
     }
+    case Intrinsic::masked_compressstore: {
+      const Value *Data = Args[0];
+      const Value *Mask = Args[2];
+      Align Alignment = I->getParamAlign(1).valueOrOne();
+      return thisT()->getExpandCompressMemoryOpCost(
+          Instruction::Store, Data->getType(), !isa<Constant>(Mask), Alignment,
+          CostKind, I);
+    }
+    case Intrinsic::masked_expandload: {
+      const Value *Mask = Args[1];
+      Align Alignment = I->getParamAlign(0).valueOrOne();
+      return thisT()->getExpandCompressMemoryOpCost(Instruction::Load, RetTy,
+                                                    !isa<Constant>(Mask),
+                                                    Alignment, CostKind, I);
+    }
     case Intrinsic::experimental_vp_strided_store: {
       const Value *Data = Args[0];
       const Value *Ptr = Args[1];
@@ -2077,6 +2101,12 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
     case Intrinsic::sincos:
       ISD = ISD::FSINCOS;
       break;
+    case Intrinsic::sincospi:
+      ISD = ISD::FSINCOSPI;
+      break;
+    case Intrinsic::modf:
+      ISD = ISD::FMODF;
+      break;
     case Intrinsic::tan:
       ISD = ISD::FTAN;
       break;
@@ -2204,6 +2234,20 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
       return thisT()->getMaskedMemoryOpCost(Instruction::Load, Ty, TyAlign, 0,
                                             CostKind);
     }
+    case Intrinsic::experimental_vp_strided_store: {
+      auto *Ty = cast<VectorType>(ICA.getArgTypes()[0]);
+      Align Alignment = thisT()->DL.getABITypeAlign(Ty->getElementType());
+      return thisT()->getStridedMemoryOpCost(
+          Instruction::Store, Ty, /*Ptr=*/nullptr, /*VariableMask=*/true,
+          Alignment, CostKind, ICA.getInst());
+    }
+    case Intrinsic::experimental_vp_strided_load: {
+      auto *Ty = cast<VectorType>(ICA.getReturnType());
+      Align Alignment = thisT()->DL.getABITypeAlign(Ty->getElementType());
+      return thisT()->getStridedMemoryOpCost(
+          Instruction::Load, Ty, /*Ptr=*/nullptr, /*VariableMask=*/true,
+          Alignment, CostKind, ICA.getInst());
+    }
     case Intrinsic::vector_reduce_add:
     case Intrinsic::vector_reduce_mul:
     case Intrinsic::vector_reduce_and:
diff --git a/llvm/include/llvm/CodeGen/CommandFlags.h b/llvm/include/llvm/CodeGen/CommandFlags.h
index d5448d781363d..000aed782a805 100644
--- a/llvm/include/llvm/CodeGen/CommandFlags.h
+++ b/llvm/include/llvm/CodeGen/CommandFlags.h
@@ -136,6 +136,8 @@ bool getEmitCallSiteInfo();
 
 bool getEnableMachineFunctionSplitter();
 
+bool getEnableStaticDataPartitioning();
+
 bool getEnableDebugEntryValues();
 
 bool getValueTrackingVariableLocations();
diff --git a/llvm/include/llvm/CodeGen/GlobalISel/LegalizerInfo.h b/llvm/include/llvm/CodeGen/GlobalISel/LegalizerInfo.h
index 5d7e03bbaeb7d..9472aa196f9b4 100644
--- a/llvm/include/llvm/CodeGen/GlobalISel/LegalizerInfo.h
+++ b/llvm/include/llvm/CodeGen/GlobalISel/LegalizerInfo.h
@@ -119,20 +119,17 @@ struct LegalityQuery {
     MemDesc(LLT MemoryTy, uint64_t AlignInBits, AtomicOrdering Ordering)
         : MemoryTy(MemoryTy), AlignInBits(AlignInBits), Ordering(Ordering) {}
     MemDesc(const MachineMemOperand &MMO)
-        : MemoryTy(MMO.getMemoryType()),
-          AlignInBits(MMO.getAlign().value() * 8),
-          Ordering(MMO.getSuccessOrdering()) {}
+        : MemDesc(MMO.getMemoryType(), MMO.getAlign().value() * 8,
+                  MMO.getSuccessOrdering()) {}
   };
 
   /// Operations which require memory can use this to place requirements on the
   /// memory type for each MMO.
   ArrayRef<MemDesc> MMODescrs;
 
-  constexpr LegalityQuery(unsigned Opcode, const ArrayRef<LLT> Types,
-                          const ArrayRef<MemDesc> MMODescrs)
+  constexpr LegalityQuery(unsigned Opcode, ArrayRef<LLT> Types,
+                          ArrayRef<MemDesc> MMODescrs = {})
       : Opcode(Opcode), Types(Types), MMODescrs(MMODescrs) {}
-  constexpr LegalityQuery(unsigned Opcode, const ArrayRef<LLT> Types)
-      : LegalityQuery(Opcode, Types, {}) {}
 
   raw_ostream &print(raw_ostream &OS) const;
 };
diff --git a/llvm/include/llvm/CodeGen/ISDOpcodes.h b/llvm/include/llvm/CodeGen/ISDOpcodes.h
index fd8784a4c1003..68ed812222dfd 100644
--- a/llvm/include/llvm/CodeGen/ISDOpcodes.h
+++ b/llvm/include/llvm/CodeGen/ISDOpcodes.h
@@ -1058,6 +1058,14 @@ enum NodeType {
   /// FSINCOS - Compute both fsin and fcos as a single operation.
   FSINCOS,
 
+  /// FSINCOSPI - Compute both the sine and cosine times pi more accurately
+  /// than FSINCOS(pi*x), especially for large x.
+  FSINCOSPI,
+
+  /// FMODF - Decomposes the operand into integral and fractional parts, each
+  /// having the same type and sign as the operand.
+  FMODF,
+
   /// Gets the current floating-point environment. The first operand is a token
   /// chain. The results are FP environment, represented by an integer value,
   /// and a token chain.
diff --git a/llvm/include/llvm/CodeGen/LiveIntervals.h b/llvm/include/llvm/CodeGen/LiveIntervals.h
index 540651ea11442..708917be497ef 100644
--- a/llvm/include/llvm/CodeGen/LiveIntervals.h
+++ b/llvm/include/llvm/CodeGen/LiveIntervals.h
@@ -149,8 +149,9 @@ class LiveIntervals {
   LiveInterval &createEmptyInterval(Register Reg) {
     assert(!hasInterval(Reg) && "Interval already exists!");
     VirtRegIntervals.grow(Reg.id());
-    VirtRegIntervals[Reg.id()] = createInterval(Reg);
-    return *VirtRegIntervals[Reg.id()];
+    auto &Interval = VirtRegIntervals[Reg.id()];
+    Interval = createInterval(Reg);
+    return *Interval;
   }
 
   LiveInterval &createAndComputeVirtRegInterval(Register Reg) {
@@ -168,8 +169,9 @@ class LiveIntervals {
 
   /// Interval removal.
   void removeInterval(Register Reg) {
-    delete VirtRegIntervals[Reg];
-    VirtRegIntervals[Reg] = nullptr;
+    auto &Interval = VirtRegIntervals[Reg];
+    delete Interval;
+    Interval = nullptr;
   }
 
   /// Given a register and an instruction, adds a live segment from that
diff --git a/llvm/include/llvm/CodeGen/MachineCopyPropagation.h b/llvm/include/llvm/CodeGen/MachineCopyPropagation.h
new file mode 100644
index 0000000000000..2fe26464b71b3
--- /dev/null
+++ b/llvm/include/llvm/CodeGen/MachineCopyPropagation.h
@@ -0,0 +1,35 @@
+//===- llvm/CodeGen/MachineCopyPropagation.h --------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CODEGEN_MACHINECOPYPROPAGATION_H
+#define LLVM_CODEGEN_MACHINECOPYPROPAGATION_H
+
+#include "llvm/CodeGen/MachinePassManager.h"
+
+namespace llvm {
+
+class MachineCopyPropagationPass
+    : public PassInfoMixin<MachineCopyPropagationPass> {
+  bool UseCopyInstr;
+
+public:
+  MachineCopyPropagationPass(bool UseCopyInstr = false)
+      : UseCopyInstr(UseCopyInstr) {}
+
+  PreservedAnalyses run(MachineFunction &MF,
+                        MachineFunctionAnalysisManager &MFAM);
+
+  MachineFunctionProperties getRequiredProperties() const {
+    return MachineFunctionProperties().set(
+        MachineFunctionProperties::Property::NoVRegs);
+  }
+};
+
+} // namespace llvm
+
+#endif // LLVM_CODEGEN_MACHINECOPYPROPAGATION_H
diff --git a/llvm/include/llvm/CodeGen/MachineFunction.h b/llvm/include/llvm/CodeGen/MachineFunction.h
index c3eb27b946287..f1e595cde54e3 100644
--- a/llvm/include/llvm/CodeGen/MachineFunction.h
+++ b/llvm/include/llvm/CodeGen/MachineFunction.h
@@ -918,6 +918,13 @@ class LLVM_ABI MachineFunction {
   bool verify(Pass *p = nullptr, const char *Banner = nullptr,
               raw_ostream *OS = nullptr, bool AbortOnError = true) const;
 
+  /// For New Pass Manager: Run the current MachineFunction through the machine
+  /// code verifier, useful for debugger use.
+  /// \returns true if no problems were found.
+  bool verify(MachineFunctionAnalysisManager &MFAM,
+              const char *Banner = nullptr, raw_ostream *OS = nullptr,
+              bool AbortOnError = true) const;
+
   /// Run the current MachineFunction through the machine code verifier, useful
   /// for debugger use.
   /// \returns true if no problems were found.
diff --git a/llvm/include/llvm/CodeGen/MachineOperand.h b/llvm/include/llvm/CodeGen/MachineOperand.h
index be1b4fb7d54fb..b4e704654495c 100644
--- a/llvm/include/llvm/CodeGen/MachineOperand.h
+++ b/llvm/include/llvm/CodeGen/MachineOperand.h
@@ -32,7 +32,6 @@ class MachineRegisterInfo;
 class MCCFIInstruction;
 class MDNode;
 class ModuleSlotTracker;
-class TargetIntrinsicInfo;
 class TargetRegisterInfo;
 class hash_code;
 class raw_ostream;
@@ -283,8 +282,7 @@ class MachineOperand {
   /// Providing a valid \p TRI and \p IntrinsicInfo results in a more
   /// target-specific printing. If \p TRI and \p IntrinsicInfo are null, the
   /// function will try to pick it up from the parent.
-  void print(raw_ostream &os, const TargetRegisterInfo *TRI = nullptr,
-             const TargetIntrinsicInfo *IntrinsicInfo = nullptr) const;
+  void print(raw_ostream &os, const TargetRegisterInfo *TRI = nullptr) const;
 
   /// More complex way of printing a MachineOperand.
   /// \param TypeToPrint specifies the generic type to be printed on uses and
@@ -310,14 +308,12 @@ class MachineOperand {
   void print(raw_ostream &os, ModuleSlotTracker &MST, LLT TypeToPrint,
              std::optional<unsigned> OpIdx, bool PrintDef, bool IsStandalone,
              bool ShouldPrintRegisterTies, unsigned TiedOperandIdx,
-             const TargetRegisterInfo *TRI,
-             const TargetIntrinsicInfo *IntrinsicInfo) const;
+             const TargetRegisterInfo *TRI) const;
 
   /// Same as print(os, TRI, IntrinsicInfo), but allows to specify the low-level
   /// type to be printed the same way the full version of print(...) does it.
   void print(raw_ostream &os, LLT TypeToPrint,
-             const TargetRegisterInfo *TRI = nullptr,
-             const TargetIntrinsicInfo *IntrinsicInfo = nullptr) const;
+             const TargetRegisterInfo *TRI = nullptr) const;
 
   void dump() const;
 
@@ -854,7 +850,7 @@ class MachineOperand {
     Op.IsEarlyClobber = isEarlyClobber;
     Op.TiedTo = 0;
     Op.IsDebug = isDebug;
-    Op.SmallContents.RegNo = Reg;
+    Op.SmallContents.RegNo = Reg.id();
     Op.Contents.Reg.Prev = nullptr;
     Op.Contents.Reg.Next = nullptr;
     Op.setSubReg(SubReg);
diff --git a/llvm/include/llvm/CodeGen/MachinePipeliner.h b/llvm/include/llvm/CodeGen/MachinePipeliner.h
index 8e47d0cead757..f95a02aad4559 100644
--- a/llvm/include/llvm/CodeGen/MachinePipeliner.h
+++ b/llvm/include/llvm/CodeGen/MachinePipeliner.h
@@ -390,6 +390,9 @@ class SwingSchedulerDAG : public ScheduleDAGInstrs {
 
   const SwingSchedulerDDG *getDDG() const { return DDG.get(); }
 
+  bool mayOverlapInLaterIter(const MachineInstr *BaseMI,
+                             const MachineInstr *OtherMI) const;
+
 private:
   void addLoopCarriedDependences(AAResults *AA);
   void updatePhiDependences();
@@ -409,7 +412,7 @@ class SwingSchedulerDAG : public ScheduleDAGInstrs {
   void computeNodeOrder(NodeSetType &NodeSets);
   void checkValidNodeOrder(const NodeSetType &Circuits) const;
   bool schedulePipeline(SMSchedule &Schedule);
-  bool computeDelta(MachineInstr &MI, unsigned &Delta) const;
+  bool computeDelta(const MachineInstr &MI, int &Delta) const;
   MachineInstr *findDefInLoop(Register Reg);
   bool canUseLastOffsetValue(MachineInstr *MI, unsigned &BasePos,
                              unsigned &OffsetPos, unsigned &NewBase,
diff --git a/llvm/include/llvm/CodeGen/MachineRegisterInfo.h b/llvm/include/llvm/CodeGen/MachineRegisterInfo.h
index 4fddc2033b81b..1c465741cb462 100644
--- a/llvm/include/llvm/CodeGen/MachineRegisterInfo.h
+++ b/llvm/include/llvm/CodeGen/MachineRegisterInfo.h
@@ -810,9 +810,10 @@ class MachineRegisterInfo {
   void setRegAllocationHint(Register VReg, unsigned Type, Register PrefReg) {
     assert(VReg.isVirtual());
     RegAllocHints.grow(Register::index2VirtReg(getNumVirtRegs()));
-    RegAllocHints[VReg].first  = Type;
-    RegAllocHints[VReg].second.clear();
-    RegAllocHints[VReg].second.push_back(PrefReg);
+    auto &Hint = RegAllocHints[VReg];
+    Hint.first = Type;
+    Hint.second.clear();
+    Hint.second.push_back(PrefReg);
   }
 
   /// addRegAllocationHint - Add a register allocation hint to the hints
@@ -843,9 +844,9 @@ class MachineRegisterInfo {
     assert(VReg.isVirtual());
     if (!RegAllocHints.inBounds(VReg))
       return {0, Register()};
-    Register BestHint = (RegAllocHints[VReg.id()].second.size() ?
-                         RegAllocHints[VReg.id()].second[0] : Register());
-    return {RegAllocHints[VReg.id()].first, BestHint};
+    auto &Hint = RegAllocHints[VReg.id()];
+    Register BestHint = (Hint.second.size() ? Hint.second[0] : Register());
+    return {Hint.first, BestHint};
   }
 
   /// getSimpleHint - same as getRegAllocationHint except it will only return
diff --git a/llvm/include/llvm/CodeGen/MachineScheduler.h b/llvm/include/llvm/CodeGen/MachineScheduler.h
index 660670ccdcd75..4762494e6ccb7 100644
--- a/llvm/include/llvm/CodeGen/MachineScheduler.h
+++ b/llvm/include/llvm/CodeGen/MachineScheduler.h
@@ -17,7 +17,7 @@
 // scheduled. Targets can override the DAG builder and scheduler without
 // replacing the pass as follows:
 //
-// ScheduleDAGInstrs *<Target>PassConfig::
+// ScheduleDAGInstrs *<Target>TargetMachine::
 // createMachineScheduler(MachineSchedContext *C) {
 //   return new CustomMachineScheduler(C);
 // }
@@ -29,7 +29,7 @@
 // plugin an alternate MachineSchedStrategy. The strategy is responsible for
 // selecting the highest priority node from the list:
 //
-// ScheduleDAGInstrs *<Target>PassConfig::
+// ScheduleDAGInstrs *<Target>TargetMachine::
 // createMachineScheduler(MachineSchedContext *C) {
 //   return new ScheduleDAGMILive(C, CustomStrategy(C));
 // }
@@ -39,7 +39,7 @@
 // can adjust dependencies based on target-specific knowledge or add weak edges
 // to aid heuristics:
 //
-// ScheduleDAGInstrs *<Target>PassConfig::
+// ScheduleDAGInstrs *<Target>TargetMachine::
 // createMachineScheduler(MachineSchedContext *C) {
 //   ScheduleDAGMI *DAG = createGenericSchedLive(C);
 //   DAG->addMutation(new CustomDAGMutation(...));
@@ -137,7 +137,7 @@ struct MachineSchedContext {
   MachineFunction *MF = nullptr;
   const MachineLoopInfo *MLI = nullptr;
   const MachineDominatorTree *MDT = nullptr;
-  const TargetPassConfig *PassConfig = nullptr;
+  const TargetMachine *TM = nullptr;
   AAResults *AA = nullptr;
   LiveIntervals *LIS = nullptr;
 
diff --git a/llvm/include/llvm/CodeGen/PostRASchedulerList.h b/llvm/include/llvm/CodeGen/PostRASchedulerList.h
new file mode 100644
index 0000000000000..06043bed5c088
--- /dev/null
+++ b/llvm/include/llvm/CodeGen/PostRASchedulerList.h
@@ -0,0 +1,32 @@
+//===- llvm/CodeGen/PostRASchedulerList.h ------------------------*- C++-*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CODEGEN_POSTRASCHEDULERLIST_H
+#define LLVM_CODEGEN_POSTRASCHEDULERLIST_H
+
+#include "llvm/CodeGen/MachinePassManager.h"
+
+namespace llvm {
+
+class PostRASchedulerPass : public PassInfoMixin<PostRASchedulerPass> {
+  const TargetMachine *TM;
+
+public:
+  PostRASchedulerPass(const TargetMachine *TM) : TM(TM) {}
+  PreservedAnalyses run(MachineFunction &MF,
+                        MachineFunctionAnalysisManager &MFAM);
+
+  MachineFunctionProperties getRequiredProperties() const {
+    return MachineFunctionProperties().set(
+        MachineFunctionProperties::Property::NoVRegs);
+  }
+};
+
+} // namespace llvm
+
+#endif // LLVM_CODEGEN_POSTRASCHEDULERLIST_H
diff --git a/llvm/include/llvm/CodeGen/RDFGraph.h b/llvm/include/llvm/CodeGen/RDFGraph.h
index cf7344e8c3e74..8a93afbcb5491 100644
--- a/llvm/include/llvm/CodeGen/RDFGraph.h
+++ b/llvm/include/llvm/CodeGen/RDFGraph.h
@@ -865,8 +865,9 @@ struct DataFlowGraph {
   using BlockRefsMap = RegisterAggrMap<NodeId>;
 
   void buildStmt(Block BA, MachineInstr &In);
-  void recordDefsForDF(BlockRefsMap &PhiM, Block BA);
-  void buildPhis(BlockRefsMap &PhiM, Block BA);
+  void recordDefsForDF(BlockRefsMap &PhiM, BlockRefsMap &PhiClobberM, Block BA);
+  void buildPhis(BlockRefsMap &PhiM, Block BA,
+                 const DefStackMap &DefM = DefStackMap());
   void removeUnusedPhis();
 
   void pushClobbers(Instr IA, DefStackMap &DM);
@@ -874,7 +875,7 @@ struct DataFlowGraph {
   template <typename T> void linkRefUp(Instr IA, NodeAddr<T> TA, DefStack &DS);
   template <typename Predicate>
   void linkStmtRefs(DefStackMap &DefM, Stmt SA, Predicate P);
-  void linkBlockRefs(DefStackMap &DefM, Block BA);
+  void linkBlockRefs(DefStackMap &DefM, BlockRefsMap &PhiClobberM, Block BA);
 
   void unlinkUseDF(Use UA);
   void unlinkDefDF(Def DA);
diff --git a/llvm/include/llvm/CodeGen/ReachingDefAnalysis.h b/llvm/include/llvm/CodeGen/ReachingDefAnalysis.h
index 2e976a88b4cea..978e84b09a736 100644
--- a/llvm/include/llvm/CodeGen/ReachingDefAnalysis.h
+++ b/llvm/include/llvm/CodeGen/ReachingDefAnalysis.h
@@ -114,8 +114,11 @@ class ReachingDefAnalysis : public MachineFunctionPass {
 private:
   MachineFunction *MF = nullptr;
   const TargetRegisterInfo *TRI = nullptr;
+  const TargetInstrInfo *TII = nullptr;
   LoopTraversal::TraversalOrder TraversedMBBOrder;
   unsigned NumRegUnits = 0;
+  unsigned NumStackObjects = 0;
+  int ObjectIndexBegin = 0;
   /// Instruction that defined each register, relative to the beginning of the
   /// current basic block.  When a LiveRegsDefInfo is used to represent a
   /// live-out register, this value is relative to the end of the basic block,
@@ -139,6 +142,13 @@ class ReachingDefAnalysis : public MachineFunctionPass {
 
   MBBReachingDefsInfo MBBReachingDefs;
 
+  /// MBBFrameObjsReachingDefs[{i, j}] is a list of instruction indices
+  /// (relative to begining of MBB i) that define frame index j in MBB i. This
+  /// is used in answering reaching definition queries.
+  using MBBFrameObjsReachingDefsInfo =
+      DenseMap<std::pair<unsigned, int>, SmallVector<int>>;
+  MBBFrameObjsReachingDefsInfo MBBFrameObjsReachingDefs;
+
   /// Default values are 'nothing happened a long time ago'.
   const int ReachingDefDefaultVal = -(1 << 21);
 
@@ -158,6 +168,7 @@ class ReachingDefAnalysis : public MachineFunctionPass {
     MachineFunctionPass::getAnalysisUsage(AU);
   }
 
+  void printAllReachingDefs(MachineFunction &MF);
   bool runOnMachineFunction(MachineFunction &MF) override;
 
   MachineFunctionProperties getRequiredProperties() const override {
@@ -177,6 +188,7 @@ class ReachingDefAnalysis : public MachineFunctionPass {
 
   /// Provides the instruction id of the closest reaching def instruction of
   /// Reg that reaches MI, relative to the begining of MI's basic block.
+  /// Note that Reg may represent a stack slot.
   int getReachingDef(MachineInstr *MI, Register Reg) const;
 
   /// Return whether A and B use the same def of Reg.
@@ -305,6 +317,7 @@ class ReachingDefAnalysis : public MachineFunctionPass {
 
   /// Provides the instruction of the closest reaching def instruction of
   /// Reg that reaches MI, relative to the begining of MI's basic block.
+  /// Note that Reg may represent a stack slot.
   MachineInstr *getReachingLocalMIDef(MachineInstr *MI, Register Reg) const;
 };
 
diff --git a/llvm/include/llvm/CodeGen/Register.h b/llvm/include/llvm/CodeGen/Register.h
index fac5f00110ef7..8a0bf3dc71ad2 100644
--- a/llvm/include/llvm/CodeGen/Register.h
+++ b/llvm/include/llvm/CodeGen/Register.h
@@ -98,6 +98,12 @@ class Register {
   /// register in a function will get the index 0.
   unsigned virtRegIndex() const { return virtReg2Index(Reg); }
 
+  /// Compute the frame index from a register value representing a stack slot.
+  int stackSlotIndex() const {
+    assert(isStack() && "Not a stack slot");
+    return static_cast<int>(Reg - MCRegister::FirstStackSlot);
+  }
+
   constexpr operator unsigned() const { return Reg; }
 
   constexpr unsigned id() const { return Reg; }
@@ -160,6 +166,37 @@ template <> struct DenseMapInfo<Register> {
   }
 };
 
+/// Wrapper class representing a virtual register or register unit.
+class VirtRegOrUnit {
+  unsigned VRegOrUnit;
+
+public:
+  constexpr explicit VirtRegOrUnit(MCRegUnit Unit) : VRegOrUnit(Unit) {
+    assert(!Register::isVirtualRegister(VRegOrUnit));
+  }
+  constexpr explicit VirtRegOrUnit(Register Reg) : VRegOrUnit(Reg.id()) {
+    assert(Reg.isVirtual());
+  }
+
+  constexpr bool isVirtualReg() const {
+    return Register::isVirtualRegister(VRegOrUnit);
+  }
+
+  constexpr MCRegUnit asMCRegUnit() const {
+    assert(!isVirtualReg() && "Not a register unit");
+    return VRegOrUnit;
+  }
+
+  constexpr Register asVirtualReg() const {
+    assert(isVirtualReg() && "Not a virtual register");
+    return Register(VRegOrUnit);
+  }
+
+  constexpr bool operator==(const VirtRegOrUnit &Other) const {
+    return VRegOrUnit == Other.VRegOrUnit;
+  }
+};
+
 } // namespace llvm
 
 #endif // LLVM_CODEGEN_REGISTER_H
diff --git a/llvm/include/llvm/CodeGen/RegisterCoalescerPass.h b/llvm/include/llvm/CodeGen/RegisterCoalescerPass.h
new file mode 100644
index 0000000000000..91f66dbf33b5f
--- /dev/null
+++ b/llvm/include/llvm/CodeGen/RegisterCoalescerPass.h
@@ -0,0 +1,28 @@
+//===- llvm/CodeGen/RegisterCoalescerPass.h ---------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CODEGEN_REGISTER_COALESCERPASS_H
+#define LLVM_CODEGEN_REGISTER_COALESCERPASS_H
+
+#include "llvm/CodeGen/MachinePassManager.h"
+
+namespace llvm {
+class RegisterCoalescerPass : public PassInfoMixin<RegisterCoalescerPass> {
+public:
+  PreservedAnalyses run(MachineFunction &MF,
+                        MachineFunctionAnalysisManager &MFAM);
+
+  MachineFunctionProperties getClearedProperties() const {
+    return MachineFunctionProperties().set(
+        MachineFunctionProperties::Property::IsSSA);
+  }
+};
+
+} // namespace llvm
+
+#endif // LLVM_CODEGEN_REGISTER_COALESCERPASS_H
diff --git a/llvm/include/llvm/CodeGen/RegisterPressure.h b/llvm/include/llvm/CodeGen/RegisterPressure.h
index 407afee343ce2..88f43116c0b46 100644
--- a/llvm/include/llvm/CodeGen/RegisterPressure.h
+++ b/llvm/include/llvm/CodeGen/RegisterPressure.h
@@ -278,7 +278,7 @@ class LiveRegSet {
 
   unsigned getSparseIndexFromReg(Register Reg) const {
     if (Reg.isVirtual())
-      return Register::virtReg2Index(Reg) + NumRegUnits;
+      return Reg.virtRegIndex() + NumRegUnits;
     assert(Reg < NumRegUnits);
     return Reg;
   }
diff --git a/llvm/include/llvm/CodeGen/RenameIndependentSubregs.h b/llvm/include/llvm/CodeGen/RenameIndependentSubregs.h
new file mode 100644
index 0000000000000..2f6afe6bea620
--- /dev/null
+++ b/llvm/include/llvm/CodeGen/RenameIndependentSubregs.h
@@ -0,0 +1,25 @@
+//===- llvm/CodeGen/RenameIndependentSubregs.h ------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CODEGEN_RENAME_INDEPENDENT_SUBREGS_H
+#define LLVM_CODEGEN_RENAME_INDEPENDENT_SUBREGS_H
+
+#include "llvm/CodeGen/MachinePassManager.h"
+
+namespace llvm {
+
+class RenameIndependentSubregsPass
+    : public PassInfoMixin<RenameIndependentSubregsPass> {
+public:
+  PreservedAnalyses run(MachineFunction &MF,
+                        MachineFunctionAnalysisManager &MFAM);
+};
+
+} // namespace llvm
+
+#endif // LLVM_CODEGEN_RENAME_INDEPENDENT_SUBREGS_H
diff --git a/llvm/include/llvm/CodeGen/RuntimeLibcallUtil.h b/llvm/include/llvm/CodeGen/RuntimeLibcallUtil.h
index 045ec7d365311..fb2d4ebd2f62f 100644
--- a/llvm/include/llvm/CodeGen/RuntimeLibcallUtil.h
+++ b/llvm/include/llvm/CodeGen/RuntimeLibcallUtil.h
@@ -62,9 +62,17 @@ Libcall getLDEXP(EVT RetVT);
 /// UNKNOWN_LIBCALL if there is none.
 Libcall getFREXP(EVT RetVT);
 
-/// getFSINCOS - Return the FSINCOS_* value for the given types, or
+/// getSINCOS - Return the SINCOS_* value for the given types, or
 /// UNKNOWN_LIBCALL if there is none.
-Libcall getFSINCOS(EVT RetVT);
+Libcall getSINCOS(EVT RetVT);
+
+/// getSINCOSPI - Return the SINCOSPI_* value for the given types, or
+/// UNKNOWN_LIBCALL if there is none.
+Libcall getSINCOSPI(EVT RetVT);
+
+/// getMODF - Return the MODF_* value for the given types, or
+/// UNKNOWN_LIBCALL if there is none.
+Libcall getMODF(EVT RetVT);
 
 /// Return the SYNC_FETCH_AND_* value for the given opcode and type, or
 /// UNKNOWN_LIBCALL if there is none.
diff --git a/llvm/include/llvm/CodeGen/SelectionDAGNodes.h b/llvm/include/llvm/CodeGen/SelectionDAGNodes.h
index 49467ce0a54cd..6eff6bfe8d5b1 100644
--- a/llvm/include/llvm/CodeGen/SelectionDAGNodes.h
+++ b/llvm/include/llvm/CodeGen/SelectionDAGNodes.h
@@ -1645,7 +1645,7 @@ class ShuffleVectorSDNode : public SDNode {
     return Mask[Idx];
   }
 
-  bool isSplat() const { return isSplatMask(Mask, getValueType(0)); }
+  bool isSplat() const { return isSplatMask(getMask()); }
 
   int getSplatIndex() const {
     assert(isSplat() && "Cannot get splat index for non-splat!");
@@ -1659,7 +1659,7 @@ class ShuffleVectorSDNode : public SDNode {
     return 0;
   }
 
-  static bool isSplatMask(const int *Mask, EVT VT);
+  static bool isSplatMask(ArrayRef<int> Mask);
 
   /// Change values in a shuffle permute mask assuming
   /// the two vector operands have swapped position.
@@ -3261,13 +3261,16 @@ namespace ISD {
   template <typename ConstNodeType>
   bool matchUnaryPredicateImpl(SDValue Op,
                                std::function<bool(ConstNodeType *)> Match,
-                               bool AllowUndefs = false);
+                               bool AllowUndefs = false,
+                               bool AllowTruncation = false);
 
   /// Hook for matching ConstantSDNode predicate
   inline bool matchUnaryPredicate(SDValue Op,
                                   std::function<bool(ConstantSDNode *)> Match,
-                                  bool AllowUndefs = false) {
-    return matchUnaryPredicateImpl<ConstantSDNode>(Op, Match, AllowUndefs);
+                                  bool AllowUndefs = false,
+                                  bool AllowTruncation = false) {
+    return matchUnaryPredicateImpl<ConstantSDNode>(Op, Match, AllowUndefs,
+                                                   AllowTruncation);
   }
 
   /// Hook for matching ConstantFPSDNode predicate
diff --git a/llvm/include/llvm/CodeGen/StackSlotColoring.h b/llvm/include/llvm/CodeGen/StackSlotColoring.h
new file mode 100644
index 0000000000000..8db59a339280d
--- /dev/null
+++ b/llvm/include/llvm/CodeGen/StackSlotColoring.h
@@ -0,0 +1,24 @@
+//===- llvm/CodeGen/StackSlotColoring.h -------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CODEGEN_STACKSLOTCOLORING_H
+#define LLVM_CODEGEN_STACKSLOTCOLORING_H
+
+#include "llvm/CodeGen/MachinePassManager.h"
+
+namespace llvm {
+
+class StackSlotColoringPass : public PassInfoMixin<StackSlotColoringPass> {
+public:
+  PreservedAnalyses run(MachineFunction &MF,
+                        MachineFunctionAnalysisManager &MFAM);
+};
+
+} // namespace llvm
+
+#endif // LLVM_CODEGEN_STACKSLOTCOLORING_H
diff --git a/llvm/include/llvm/CodeGen/TargetInstrInfo.h b/llvm/include/llvm/CodeGen/TargetInstrInfo.h
index a91cb0d4f603b..c8eba71c9bb0a 100644
--- a/llvm/include/llvm/CodeGen/TargetInstrInfo.h
+++ b/llvm/include/llvm/CodeGen/TargetInstrInfo.h
@@ -1566,7 +1566,7 @@ class TargetInstrInfo : public MCInstrInfo {
   ///   DAG->addMutation(createLoadClusterDAGMutation(DAG->TII, DAG->TRI));
   /// or
   ///   DAG->addMutation(createStoreClusterDAGMutation(DAG->TII, DAG->TRI));
-  /// to TargetPassConfig::createMachineScheduler() to have an effect.
+  /// to TargetMachine::createMachineScheduler() to have an effect.
   ///
   /// \p BaseOps1 and \p BaseOps2 are memory operands of two memory operations.
   /// \p Offset1 and \p Offset2 are the byte offsets for the memory
diff --git a/llvm/include/llvm/CodeGen/TargetLowering.h b/llvm/include/llvm/CodeGen/TargetLowering.h
index 861cffdc115a4..bbecc7a6ddaee 100644
--- a/llvm/include/llvm/CodeGen/TargetLowering.h
+++ b/llvm/include/llvm/CodeGen/TargetLowering.h
@@ -94,6 +94,7 @@ class TargetRegisterClass;
 class TargetRegisterInfo;
 class TargetTransformInfo;
 class Value;
+class VPIntrinsic;
 
 namespace Sched {
 
@@ -3156,6 +3157,30 @@ class TargetLoweringBase {
     return false;
   }
 
+  /// Lower an interleaved load to target specific intrinsics. Return
+  /// true on success.
+  ///
+  /// \p Load is a vp.load instruction.
+  /// \p Mask is a mask value
+  /// \p DeinterleaveRes is a list of deinterleaved results.
+  virtual bool
+  lowerDeinterleavedIntrinsicToVPLoad(VPIntrinsic *Load, Value *Mask,
+                                      ArrayRef<Value *> DeinterleaveRes) const {
+    return false;
+  }
+
+  /// Lower an interleaved store to target specific intrinsics. Return
+  /// true on success.
+  ///
+  /// \p Store is the vp.store instruction.
+  /// \p Mask is a mask value
+  /// \p InterleaveOps is a list of values being interleaved.
+  virtual bool
+  lowerInterleavedIntrinsicToVPStore(VPIntrinsic *Store, Value *Mask,
+                                     ArrayRef<Value *> InterleaveOps) const {
+    return false;
+  }
+
   /// Lower a deinterleave intrinsic to a target specific load intrinsic.
   /// Return true on success. Currently only supports
   /// llvm.vector.deinterleave2
@@ -5503,20 +5528,19 @@ class TargetLowering : public TargetLoweringBase {
   bool expandMULO(SDNode *Node, SDValue &Result, SDValue &Overflow,
                   SelectionDAG &DAG) const;
 
-  /// forceExpandWideMUL - Unconditionally expand a MUL into either a libcall or
-  /// brute force via a wide multiplication. The expansion works by
-  /// attempting to do a multiplication on a wider type twice the size of the
-  /// original operands. LL and LH represent the lower and upper halves of the
-  /// first operand. RL and RH represent the lower and upper halves of the
-  /// second operand. The upper and lower halves of the result are stored in Lo
-  /// and Hi.
-  void forceExpandWideMUL(SelectionDAG &DAG, const SDLoc &dl, bool Signed,
-                          EVT WideVT, const SDValue LL, const SDValue LH,
-                          const SDValue RL, const SDValue RH, SDValue &Lo,
-                          SDValue &Hi) const;
-
-  /// Same as above, but creates the upper halves of each operand by
-  /// sign/zero-extending the operands.
+  /// Calculate the product twice the width of LHS and RHS. If HiLHS/HiRHS are
+  /// non-null they will be included in the multiplication. The expansion works
+  /// by splitting the 2 inputs into 4 pieces that we can multiply and add
+  /// together without neding MULH or MUL_LOHI.
+  void forceExpandMultiply(SelectionDAG &DAG, const SDLoc &dl, bool Signed,
+                           SDValue &Lo, SDValue &Hi, SDValue LHS, SDValue RHS,
+                           SDValue HiLHS = SDValue(),
+                           SDValue HiRHS = SDValue()) const;
+
+  /// Calculate full product of LHS and RHS either via a libcall or through
+  /// brute force expansion of the multiplication. The expansion works by
+  /// splitting the 2 inputs into 4 pieces that we can multiply and add together
+  /// without needing MULH or MUL_LOHI.
   void forceExpandWideMUL(SelectionDAG &DAG, const SDLoc &dl, bool Signed,
                           const SDValue LHS, const SDValue RHS, SDValue &Lo,
                           SDValue &Hi) const;
@@ -5623,6 +5647,18 @@ class TargetLowering : public TargetLoweringBase {
   // joining their results. SDValue() is returned when expansion did not happen.
   SDValue expandVectorNaryOpBySplitting(SDNode *Node, SelectionDAG &DAG) const;
 
+  /// Replace an extraction of a load with a narrowed load.
+  ///
+  /// \param ResultVT type of the result extraction.
+  /// \param InVecVT type of the input vector to with bitcasts resolved.
+  /// \param EltNo index of the vector element to load.
+  /// \param OriginalLoad vector load that to be replaced.
+  /// \returns \p ResultVT Load on success SDValue() on failure.
+  SDValue scalarizeExtractedVectorLoad(EVT ResultVT, const SDLoc &DL,
+                                       EVT InVecVT, SDValue EltNo,
+                                       LoadSDNode *OriginalLoad,
+                                       SelectionDAG &DAG) const;
+
 private:
   SDValue foldSetCCWithAnd(EVT VT, SDValue N0, SDValue N1, ISD::CondCode Cond,
                            const SDLoc &DL, DAGCombinerInfo &DCI) const;
diff --git a/llvm/include/llvm/CodeGen/TargetLoweringObjectFileImpl.h b/llvm/include/llvm/CodeGen/TargetLoweringObjectFileImpl.h
index 3e2cd05387926..10f0594c267ae 100644
--- a/llvm/include/llvm/CodeGen/TargetLoweringObjectFileImpl.h
+++ b/llvm/include/llvm/CodeGen/TargetLoweringObjectFileImpl.h
@@ -76,6 +76,9 @@ class TargetLoweringObjectFileELF : public TargetLoweringObjectFile {
 
   MCSection *getSectionForJumpTable(const Function &F,
                                     const TargetMachine &TM) const override;
+  MCSection *
+  getSectionForJumpTable(const Function &F, const TargetMachine &TM,
+                         const MachineJumpTableEntry *JTE) const override;
   MCSection *getSectionForLSDA(const Function &F, const MCSymbol &FnSym,
                                const TargetMachine &TM) const override;
 
diff --git a/llvm/include/llvm/CodeGen/TargetPassConfig.h b/llvm/include/llvm/CodeGen/TargetPassConfig.h
index 66c79c74f2be5..1af7267fa9ecf 100644
--- a/llvm/include/llvm/CodeGen/TargetPassConfig.h
+++ b/llvm/include/llvm/CodeGen/TargetPassConfig.h
@@ -22,9 +22,7 @@
 namespace llvm {
 
 class TargetMachine;
-struct MachineSchedContext;
 class PassConfigImpl;
-class ScheduleDAGInstrs;
 class CSEConfigBase;
 class PassInstrumentationCallbacks;
 
@@ -300,27 +298,6 @@ class TargetPassConfig : public ImmutablePass {
   /// Fully developed targets will not generally override this.
   virtual void addMachinePasses();
 
-  /// Create an instance of ScheduleDAGInstrs to be run within the standard
-  /// MachineScheduler pass for this function and target at the current
-  /// optimization level.
-  ///
-  /// This can also be used to plug a new MachineSchedStrategy into an instance
-  /// of the standard ScheduleDAGMI:
-  ///   return new ScheduleDAGMI(C, std::make_unique<MyStrategy>(C), /*RemoveKillFlags=*/false)
-  ///
-  /// Return NULL to select the default (generic) machine scheduler.
-  virtual ScheduleDAGInstrs *
-  createMachineScheduler(MachineSchedContext *C) const {
-    return nullptr;
-  }
-
-  /// Similar to createMachineScheduler but used when postRA machine scheduling
-  /// is enabled.
-  virtual ScheduleDAGInstrs *
-  createPostMachineScheduler(MachineSchedContext *C) const {
-    return nullptr;
-  }
-
   /// printAndVerify - Add a pass to dump then verify the machine function, if
   /// those steps are enabled.
   void printAndVerify(const std::string &Banner);
diff --git a/llvm/include/llvm/CodeGen/TargetRegisterInfo.h b/llvm/include/llvm/CodeGen/TargetRegisterInfo.h
index 0bf72637de398..114149ff53d85 100644
--- a/llvm/include/llvm/CodeGen/TargetRegisterInfo.h
+++ b/llvm/include/llvm/CodeGen/TargetRegisterInfo.h
@@ -466,9 +466,9 @@ class TargetRegisterInfo : public MCRegisterInfo {
   }
 
   /// Returns true if Reg contains RegUnit.
-  bool hasRegUnit(MCRegister Reg, Register RegUnit) const {
+  bool hasRegUnit(MCRegister Reg, MCRegUnit RegUnit) const {
     for (MCRegUnit Unit : regunits(Reg))
-      if (Register(Unit) == RegUnit)
+      if (Unit == RegUnit)
         return true;
     return false;
   }
@@ -1383,9 +1383,7 @@ class BitMaskClassIterator {
 // This is useful when building IndexedMaps keyed on virtual registers
 struct VirtReg2IndexFunctor {
   using argument_type = Register;
-  unsigned operator()(Register Reg) const {
-    return Register::virtReg2Index(Reg);
-  }
+  unsigned operator()(Register Reg) const { return Reg.virtRegIndex(); }
 };
 
 /// Prints virtual and physical registers with or without a TRI instance.
diff --git a/llvm/include/llvm/DebugInfo/DWARF/DWARFDebugLine.h b/llvm/include/llvm/DebugInfo/DWARF/DWARFDebugLine.h
index ff7bf87d8e6b5..61f17a27b3d28 100644
--- a/llvm/include/llvm/DebugInfo/DWARF/DWARFDebugLine.h
+++ b/llvm/include/llvm/DebugInfo/DWARF/DWARFDebugLine.h
@@ -209,6 +209,9 @@ class DWARFDebugLine {
     unsigned LastRowIndex;
     bool Empty;
 
+    /// The offset into the line table where this sequence begins
+    uint64_t StmtSeqOffset = UINT64_MAX;
+
     void reset();
 
     static bool orderByHighPC(const Sequence &LHS, const Sequence &RHS) {
@@ -243,8 +246,20 @@ class DWARFDebugLine {
     uint32_t lookupAddress(object::SectionedAddress Address,
                            bool *IsApproximateLine = nullptr) const;
 
-    bool lookupAddressRange(object::SectionedAddress Address, uint64_t Size,
-                            std::vector<uint32_t> &Result) const;
+    /// Fills the Result argument with the indices of the rows that correspond
+    /// to the address range specified by \p Address and \p Size.
+    ///
+    /// \param Address - The starting address of the range.
+    /// \param Size - The size of the address range.
+    /// \param Result - The vector to fill with row indices.
+    /// \param StmtSequenceOffset - if provided, only rows from the sequence
+    /// starting at the matching offset will be added to the result.
+    ///
+    /// Returns true if any rows were found.
+    bool lookupAddressRange(
+        object::SectionedAddress Address, uint64_t Size,
+        std::vector<uint32_t> &Result,
+        std::optional<uint64_t> StmtSequenceOffset = std::nullopt) const;
 
     bool hasFileAtIndex(uint64_t FileIndex) const {
       return Prologue.hasFileAtIndex(FileIndex);
@@ -305,8 +320,20 @@ class DWARFDebugLine {
     uint32_t lookupAddressImpl(object::SectionedAddress Address,
                                bool *IsApproximateLine = nullptr) const;
 
-    bool lookupAddressRangeImpl(object::SectionedAddress Address, uint64_t Size,
-                                std::vector<uint32_t> &Result) const;
+    /// Fills the Result argument with the indices of the rows that correspond
+    /// to the address range specified by \p Address and \p Size.
+    ///
+    /// \param Address - The starting address of the range.
+    /// \param Size - The size of the address range.
+    /// \param Result - The vector to fill with row indices.
+    /// \param StmtSequenceOffset - if provided, only rows from the sequence
+    /// starting at the matching offset will be added to the result.
+    ///
+    /// Returns true if any rows were found.
+    bool
+    lookupAddressRangeImpl(object::SectionedAddress Address, uint64_t Size,
+                           std::vector<uint32_t> &Result,
+                           std::optional<uint64_t> StmtSequenceOffset) const;
   };
 
   const LineTable *getLineTable(uint64_t Offset) const;
@@ -376,7 +403,7 @@ class DWARFDebugLine {
     ParsingState(struct LineTable *LT, uint64_t TableOffset,
                  function_ref<void(Error)> ErrorHandler);
 
-    void resetRowAndSequence();
+    void resetRowAndSequence(uint64_t Offset);
     void appendRowToMatrix();
 
     struct AddrOpIndexDelta {
diff --git a/llvm/include/llvm/ExecutionEngine/JITLink/JITLink.h b/llvm/include/llvm/ExecutionEngine/JITLink/JITLink.h
index 297e603164b24..8f0dfea0c97ac 100644
--- a/llvm/include/llvm/ExecutionEngine/JITLink/JITLink.h
+++ b/llvm/include/llvm/ExecutionEngine/JITLink/JITLink.h
@@ -587,6 +587,9 @@ class Symbol {
     return static_cast<const Block &>(*Base);
   }
 
+  /// Return the Section for this Symbol (Symbol must be defined).
+  Section &getSection() const { return getBlock().getSection(); }
+
   /// Returns the offset for this symbol within the underlying addressable.
   orc::ExecutorAddrDiff getOffset() const { return Offset; }
 
@@ -1460,7 +1463,7 @@ class LinkGraph {
       A.setAddress(orc::ExecutorAddr());
     } else {
       assert(Sym.isDefined() && "Sym is not a defined symbol");
-      Section &Sec = Sym.getBlock().getSection();
+      Section &Sec = Sym.getSection();
       Sec.removeSymbol(Sym);
       Sym.makeExternal(createAddressable(orc::ExecutorAddr(), false));
     }
@@ -1488,7 +1491,7 @@ class LinkGraph {
       Sym.setScope(Scope::Local);
     } else {
       assert(Sym.isDefined() && "Sym is not a defined symbol");
-      Section &Sec = Sym.getBlock().getSection();
+      Section &Sec = Sym.getSection();
       Sec.removeSymbol(Sym);
       Sym.makeAbsolute(createAddressable(Address));
     }
@@ -1534,7 +1537,7 @@ class LinkGraph {
   transferDefinedSymbol(Symbol &Sym, Block &DestBlock,
                         orc::ExecutorAddrDiff NewOffset,
                         std::optional<orc::ExecutorAddrDiff> ExplicitNewSize) {
-    auto &OldSection = Sym.getBlock().getSection();
+    auto &OldSection = Sym.getSection();
     Sym.setBlock(DestBlock);
     Sym.setOffset(NewOffset);
     if (ExplicitNewSize)
@@ -1622,7 +1625,7 @@ class LinkGraph {
   /// Removes defined symbols. Does not remove the underlying block.
   void removeDefinedSymbol(Symbol &Sym) {
     assert(Sym.isDefined() && "Sym is not a defined symbol");
-    Sym.getBlock().getSection().removeSymbol(Sym);
+    Sym.getSection().removeSymbol(Sym);
     destroySymbol(Sym);
   }
 
diff --git a/llvm/include/llvm/ExecutionEngine/Orc/Core.h b/llvm/include/llvm/ExecutionEngine/Orc/Core.h
index db853362f6573..cecb4094c9a57 100644
--- a/llvm/include/llvm/ExecutionEngine/Orc/Core.h
+++ b/llvm/include/llvm/ExecutionEngine/Orc/Core.h
@@ -1204,8 +1204,13 @@ class JITDylib : public ThreadSafeRefCountedBase<JITDylib>,
 
   JITDylib(ExecutionSession &ES, std::string Name);
 
-  std::pair<AsynchronousSymbolQuerySet, std::shared_ptr<SymbolDependenceMap>>
-  IL_removeTracker(ResourceTracker &RT);
+  struct RemoveTrackerResult {
+    AsynchronousSymbolQuerySet QueriesToFail;
+    std::shared_ptr<SymbolDependenceMap> FailedSymbols;
+    std::vector<std::unique_ptr<MaterializationUnit>> DefunctMUs;
+  };
+
+  RemoveTrackerResult IL_removeTracker(ResourceTracker &RT);
 
   void transferTracker(ResourceTracker &DstRT, ResourceTracker &SrcRT);
 
@@ -1551,6 +1556,30 @@ class ExecutionSession {
     EPC->getDispatcher().dispatch(std::move(T));
   }
 
+  /// Returns the bootstrap map.
+  const StringMap<std::vector<char>> &getBootstrapMap() const {
+    return EPC->getBootstrapMap();
+  }
+
+  /// Look up and SPS-deserialize a bootstrap map value.
+  template <typename T, typename SPSTagT>
+  Error getBootstrapMapValue(StringRef Key, std::optional<T> &Val) const {
+    return EPC->getBootstrapMapValue<T, SPSTagT>(Key, Val);
+  }
+
+  /// Returns the bootstrap symbol map.
+  const StringMap<ExecutorAddr> &getBootstrapSymbolsMap() const {
+    return EPC->getBootstrapSymbolsMap();
+  }
+
+  /// For each (ExecutorAddr&, StringRef) pair, looks up the string in the
+  /// bootstrap symbols map and writes its address to the ExecutorAddr if
+  /// found. If any symbol is not found then the function returns an error.
+  Error getBootstrapSymbols(
+      ArrayRef<std::pair<ExecutorAddr &, StringRef>> Pairs) const {
+    return EPC->getBootstrapSymbols(Pairs);
+  }
+
   /// Run a wrapper function in the executor.
   ///
   /// The wrapper function should be callable as:
diff --git a/llvm/include/llvm/ExecutionEngine/Orc/ExecutorProcessControl.h b/llvm/include/llvm/ExecutionEngine/Orc/ExecutorProcessControl.h
index dcf5592f1717c..86e98e74b7055 100644
--- a/llvm/include/llvm/ExecutionEngine/Orc/ExecutorProcessControl.h
+++ b/llvm/include/llvm/ExecutionEngine/Orc/ExecutorProcessControl.h
@@ -20,6 +20,7 @@
 #include "llvm/ExecutionEngine/Orc/Shared/TargetProcessControlTypes.h"
 #include "llvm/ExecutionEngine/Orc/Shared/WrapperFunctionUtils.h"
 #include "llvm/ExecutionEngine/Orc/SymbolStringPool.h"
+#include "llvm/ExecutionEngine/Orc/TargetProcess/UnwindInfoManager.h"
 #include "llvm/ExecutionEngine/Orc/TaskDispatch.h"
 #include "llvm/Support/DynamicLibrary.h"
 #include "llvm/Support/MSVCErrorWorkarounds.h"
@@ -507,6 +508,9 @@ class SelfExecutorProcessControl : public ExecutorProcessControl,
                           SymbolLookupCompleteFn F) override;
 
   std::unique_ptr<jitlink::JITLinkMemoryManager> OwnedMemMgr;
+#ifdef __APPLE__
+  std::unique_ptr<UnwindInfoManager> UnwindInfoMgr;
+#endif // __APPLE__
   char GlobalManglingPrefix = 0;
 };
 
diff --git a/llvm/include/llvm/ExecutionEngine/Orc/MachOPlatform.h b/llvm/include/llvm/ExecutionEngine/Orc/MachOPlatform.h
index 6e99f6c03a7c6..91842714f6c4c 100644
--- a/llvm/include/llvm/ExecutionEngine/Orc/MachOPlatform.h
+++ b/llvm/include/llvm/ExecutionEngine/Orc/MachOPlatform.h
@@ -368,6 +368,7 @@ class MachOPlatform : public Platform {
   DenseMap<JITDylib *, SymbolLookupSet> RegisteredInitSymbols;
 
   std::mutex PlatformMutex;
+  bool ForceEHFrames = false;
   BootstrapInfo *Bootstrap = nullptr;
   DenseMap<JITDylib *, ExecutorAddr> JITDylibToHeaderAddr;
   DenseMap<ExecutorAddr, JITDylib *> HeaderAddrToJITDylib;
diff --git a/llvm/include/llvm/ExecutionEngine/Orc/Shared/MachOObjectFormat.h b/llvm/include/llvm/ExecutionEngine/Orc/Shared/MachOObjectFormat.h
index b927dfbce992a..8eb8c1b4cc89d 100644
--- a/llvm/include/llvm/ExecutionEngine/Orc/Shared/MachOObjectFormat.h
+++ b/llvm/include/llvm/ExecutionEngine/Orc/Shared/MachOObjectFormat.h
@@ -25,7 +25,7 @@ namespace orc {
 extern StringRef MachODataCommonSectionName;
 extern StringRef MachODataDataSectionName;
 extern StringRef MachOEHFrameSectionName;
-extern StringRef MachOCompactUnwindInfoSectionName;
+extern StringRef MachOCompactUnwindSectionName;
 extern StringRef MachOCStringSectionName;
 extern StringRef MachOModInitFuncSectionName;
 extern StringRef MachOObjCCatListSectionName;
@@ -53,6 +53,7 @@ extern StringRef MachOTextTextSectionName;
 extern StringRef MachOThreadBSSSectionName;
 extern StringRef MachOThreadDataSectionName;
 extern StringRef MachOThreadVarsSectionName;
+extern StringRef MachOUnwindInfoSectionName;
 
 extern StringRef MachOInitSectionNames[22];
 
diff --git a/llvm/include/llvm/ExecutionEngine/Orc/Shared/OrcRTBridge.h b/llvm/include/llvm/ExecutionEngine/Orc/Shared/OrcRTBridge.h
index aed43f6308cba..db5ff135a7164 100644
--- a/llvm/include/llvm/ExecutionEngine/Orc/Shared/OrcRTBridge.h
+++ b/llvm/include/llvm/ExecutionEngine/Orc/Shared/OrcRTBridge.h
@@ -88,6 +88,15 @@ using SPSRunAsMainSignature = int64_t(shared::SPSExecutorAddr,
 using SPSRunAsVoidFunctionSignature = int32_t(shared::SPSExecutorAddr);
 using SPSRunAsIntFunctionSignature = int32_t(shared::SPSExecutorAddr, int32_t);
 } // end namespace rt
+
+namespace rt_alt {
+extern const char *UnwindInfoManagerInstanceName;
+extern const char *UnwindInfoManagerFindSectionsHelperName;
+extern const char *UnwindInfoManagerEnableWrapperName;
+extern const char *UnwindInfoManagerDisableWrapperName;
+extern const char *UnwindInfoManagerRegisterActionName;
+extern const char *UnwindInfoManagerDeregisterActionName;
+} // end namespace rt_alt
 } // end namespace orc
 } // end namespace llvm
 
diff --git a/llvm/include/llvm/ExecutionEngine/Orc/TargetProcess/DefaultHostBootstrapValues.h b/llvm/include/llvm/ExecutionEngine/Orc/TargetProcess/DefaultHostBootstrapValues.h
new file mode 100644
index 0000000000000..d3277e61eeb7b
--- /dev/null
+++ b/llvm/include/llvm/ExecutionEngine/Orc/TargetProcess/DefaultHostBootstrapValues.h
@@ -0,0 +1,28 @@
+//===- DefaultHostBootstrapValues.h - Defaults for host process -*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Set sensible default bootstrap values for JIT execution in the host process.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_EXECUTIONENGINE_ORC_TARGETPROCESS_DEFAULTHOSTBOOTSTRAPVALUES_H
+#define LLVM_EXECUTIONENGINE_ORC_TARGETPROCESS_DEFAULTHOSTBOOTSTRAPVALUES_H
+
+#include "llvm/ADT/StringMap.h"
+#include "llvm/ExecutionEngine/Orc/Shared/ExecutorAddress.h"
+#include <vector>
+
+namespace llvm::orc {
+
+void addDefaultBootstrapValuesForHostProcess(
+    StringMap<std::vector<char>> &BootstrapMap,
+    StringMap<ExecutorAddr> &BootstrapSymbols);
+
+} // namespace llvm::orc
+
+#endif // LLVM_EXECUTIONENGINE_ORC_TARGETPROCESS_DEFAULTHOSTBOOTSTRAPVALUES_H
diff --git a/llvm/include/llvm/ExecutionEngine/Orc/TargetProcess/UnwindInfoManager.h b/llvm/include/llvm/ExecutionEngine/Orc/TargetProcess/UnwindInfoManager.h
new file mode 100644
index 0000000000000..fc7719f282122
--- /dev/null
+++ b/llvm/include/llvm/ExecutionEngine/Orc/TargetProcess/UnwindInfoManager.h
@@ -0,0 +1,78 @@
+//===--- UnwindInfoManager.h -- Register unwind info sections ---*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Utilities for managing eh-frame and compact-unwind registration and lookup
+// through libunwind's find_dynamic_unwind_sections mechanism.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_EXECUTIONENGINE_ORC_TARGETPROCESS_UNWINDINFOMANAGER_H
+#define LLVM_EXECUTIONENGINE_ORC_TARGETPROCESS_UNWINDINFOMANAGER_H
+
+#include "llvm/ExecutionEngine/Orc/Shared/ExecutorAddress.h"
+#include "llvm/ExecutionEngine/Orc/TargetProcess/ExecutorBootstrapService.h"
+#include "llvm/Support/Error.h"
+#include <map>
+#include <mutex>
+
+namespace llvm::orc {
+
+class UnwindInfoManager : public ExecutorBootstrapService {
+public:
+  // This struct's layout should match the unw_dynamic_unwind_sections struct
+  // from libunwind/src/libunwid_ext.h.
+  struct UnwindSections {
+    uintptr_t dso_base;
+    uintptr_t dwarf_section;
+    size_t dwarf_section_length;
+    uintptr_t compact_unwind_section;
+    size_t compact_unwind_section_length;
+  };
+
+  /// If the libunwind find-dynamic-unwind-info callback registration APIs are
+  /// available then this method will return an UnwindInfoManager instance,
+  /// otherwise it will return nullptr.
+  static std::unique_ptr<UnwindInfoManager> TryCreate();
+
+  Error shutdown() override;
+  void addBootstrapSymbols(StringMap<ExecutorAddr> &M) override;
+
+  Error enable(void *FindDynamicUnwindSections);
+  Error disable(void);
+
+  Error registerSections(ArrayRef<orc::ExecutorAddrRange> CodeRanges,
+                         orc::ExecutorAddr DSOBase,
+                         orc::ExecutorAddrRange DWARFEHFrame,
+                         orc::ExecutorAddrRange CompactUnwind);
+
+  Error deregisterSections(ArrayRef<orc::ExecutorAddrRange> CodeRanges);
+
+  int findSections(uintptr_t Addr, UnwindSections *Info);
+
+private:
+  UnwindInfoManager(int (*AddFindDynamicUnwindSections)(void *),
+                    int (*RemoveFindDynamicUnwindSections)(void *))
+      : AddFindDynamicUnwindSections(AddFindDynamicUnwindSections),
+        RemoveFindDynamicUnwindSections(RemoveFindDynamicUnwindSections) {}
+
+  static int findSectionsHelper(UnwindInfoManager *Instance, uintptr_t Addr,
+                                UnwindSections *Info);
+
+  std::mutex M;
+  std::map<uintptr_t, UnwindSections> UWSecs;
+
+  int (*AddFindDynamicUnwindSections)(void *) = nullptr;
+  int (*RemoveFindDynamicUnwindSections)(void *) = nullptr;
+  void *FindDynamicUnwindSections = nullptr;
+
+  static const char *AddFnName, *RemoveFnName;
+};
+
+} // namespace llvm::orc
+
+#endif // LLVM_EXECUTIONENGINE_ORC_TARGETPROCESS_UNWINDINFOMANAGER_H
diff --git a/llvm/include/llvm/ExecutionEngine/Orc/UnwindInfoRegistrationPlugin.h b/llvm/include/llvm/ExecutionEngine/Orc/UnwindInfoRegistrationPlugin.h
new file mode 100644
index 0000000000000..eb883a79a93d8
--- /dev/null
+++ b/llvm/include/llvm/ExecutionEngine/Orc/UnwindInfoRegistrationPlugin.h
@@ -0,0 +1,70 @@
+//===- UnwindInfoRegistrationPlugin.h -- libunwind registration -*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Register eh-frame and compact-unwind sections with libunwind
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_EXECUTIONENGINE_ORC_UNWINDINFOREGISTRATIONPLUGIN_H
+#define LLVM_EXECUTIONENGINE_ORC_UNWINDINFOREGISTRATIONPLUGIN_H
+
+#include "llvm/ExecutionEngine/Orc/LinkGraphLinkingLayer.h"
+
+namespace llvm::orc {
+
+class UnwindInfoRegistrationPlugin : public LinkGraphLinkingLayer::Plugin {
+public:
+  static Expected<std::shared_ptr<UnwindInfoRegistrationPlugin>>
+  Create(IRLayer &IRL, JITDylib &PlatformJD, ExecutorAddr Instance,
+         ExecutorAddr FindHelper, ExecutorAddr Enable, ExecutorAddr Disable,
+         ExecutorAddr Register, ExecutorAddr Deregister);
+
+  static Expected<std::shared_ptr<UnwindInfoRegistrationPlugin>>
+  Create(IRLayer &IRL, JITDylib &PlatformJD);
+
+  ~UnwindInfoRegistrationPlugin();
+
+  void modifyPassConfig(MaterializationResponsibility &MR,
+                        jitlink::LinkGraph &G,
+                        jitlink::PassConfiguration &PassConfig) override;
+
+  Error notifyEmitted(MaterializationResponsibility &MR) override {
+    return Error::success();
+  }
+
+  Error notifyFailed(MaterializationResponsibility &MR) override {
+    return Error::success();
+  }
+
+  Error notifyRemovingResources(JITDylib &JD, ResourceKey K) override {
+    return Error::success();
+  }
+
+  void notifyTransferringResources(JITDylib &JD, ResourceKey DstKey,
+                                   ResourceKey SrcKey) override {}
+
+private:
+  UnwindInfoRegistrationPlugin(ExecutionSession &ES, ExecutorAddr Instance,
+                               ExecutorAddr Disable, ExecutorAddr Register,
+                               ExecutorAddr Deregister)
+      : ES(ES), Instance(Instance), Disable(Disable), Register(Register),
+        Deregister(Deregister) {
+    DSOBaseName = ES.intern("__jitlink$libunwind_dso_base");
+  }
+
+  static Expected<ThreadSafeModule> makeBouncerModule(ExecutionSession &ES);
+  Error addUnwindInfoRegistrationActions(jitlink::LinkGraph &G);
+
+  ExecutionSession &ES;
+  SymbolStringPtr DSOBaseName;
+  ExecutorAddr Instance, Disable, Register, Deregister;
+};
+
+} // namespace llvm::orc
+
+#endif // LLVM_EXECUTIONENGINE_ORC_UNWINDINFOREGISTRATIONPLUGIN_H
diff --git a/llvm/include/llvm/Frontend/Offloading/Utility.h b/llvm/include/llvm/Frontend/Offloading/Utility.h
index ddcf0a946d635..7b717a4733b79 100644
--- a/llvm/include/llvm/Frontend/Offloading/Utility.h
+++ b/llvm/include/llvm/Frontend/Offloading/Utility.h
@@ -10,6 +10,7 @@
 #define LLVM_FRONTEND_OFFLOADING_UTILITY_H
 
 #include <cstdint>
+#include <memory>
 
 #include "llvm/ADT/StringMap.h"
 #include "llvm/ADT/StringRef.h"
@@ -24,11 +25,24 @@ namespace offloading {
 /// This is the record of an object that just be registered with the offloading
 /// runtime.
 struct EntryTy {
+  /// Reserved bytes used to detect an older version of the struct, always zero.
+  uint64_t Reserved = 0x0;
+  /// The current version of the struct for runtime forward compatibility.
+  uint16_t Version = 0x1;
+  /// The expected consumer of this entry, e.g. CUDA or OpenMP.
+  uint16_t Kind;
+  /// Flags associated with the global.
+  uint32_t Flags;
+  /// The address of the global to be registered by the runtime.
   void *Address;
+  /// The name of the symbol in the device image.
   char *SymbolName;
-  size_t Size;
-  int32_t Flags;
-  int32_t Data;
+  /// The number of bytes the symbol takes.
+  uint64_t Size;
+  /// Extra generic data used to register this entry.
+  uint64_t Data;
+  /// An extra pointer, usually null.
+  void *AuxAddr;
 };
 
 /// Offloading entry flags for CUDA / HIP. The first three bits indicate the
@@ -55,34 +69,35 @@ enum OffloadEntryKindFlag : uint32_t {
 /// globals that will be registered with the offloading runtime.
 StructType *getEntryTy(Module &M);
 
-/// Returns the struct type we store the two pointers for CUDA / HIP managed
-/// variables in. Necessary until we widen the offload entry struct.
-StructType *getManagedTy(Module &M);
-
 /// Create an offloading section struct used to register this global at
 /// runtime.
 ///
 /// \param M The module to be used
 /// \param Addr The pointer to the global being registered.
+/// \param Kind The offloading language expected to consume this.
 /// \param Name The symbol name associated with the global.
 /// \param Size The size in bytes of the global (0 for functions).
 /// \param Flags Flags associated with the entry.
 /// \param Data Extra data storage associated with the entry.
 /// \param SectionName The section this entry will be placed at.
-void emitOffloadingEntry(Module &M, Constant *Addr, StringRef Name,
-                         uint64_t Size, int32_t Flags, int32_t Data,
-                         StringRef SectionName);
+/// \param AuxAddr An extra pointer if needed.
+void emitOffloadingEntry(Module &M, object::OffloadKind Kind, Constant *Addr,
+                         StringRef Name, uint64_t Size, uint32_t Flags,
+                         uint64_t Data, Constant *AuxAddr = nullptr,
+                         StringRef SectionName = "llvm_offload_entries");
+
 /// Create a constant struct initializer used to register this global at
 /// runtime.
 /// \return the constant struct and the global variable holding the symbol name.
 std::pair<Constant *, GlobalVariable *>
-getOffloadingEntryInitializer(Module &M, Constant *Addr, StringRef Name,
-                              uint64_t Size, int32_t Flags, int32_t Data);
+getOffloadingEntryInitializer(Module &M, object::OffloadKind Kind,
+                              Constant *Addr, StringRef Name, uint64_t Size,
+                              uint32_t Flags, uint64_t Data, Constant *AuxAddr);
 
 /// Creates a pair of globals used to iterate the array of offloading entries by
 /// accessing the section variables provided by the linker.
 std::pair<GlobalVariable *, GlobalVariable *>
-getOffloadEntryArray(Module &M, StringRef SectionName);
+getOffloadEntryArray(Module &M, StringRef SectionName = "llvm_offload_entries");
 
 namespace amdgpu {
 /// Check if an image is compatible with current system's environment. The
@@ -138,6 +153,12 @@ Error getAMDGPUMetaDataFromImage(MemoryBufferRef MemBuffer,
                                  StringMap<AMDGPUKernelMetaData> &KernelInfoMap,
                                  uint16_t &ELFABIVersion);
 } // namespace amdgpu
+
+namespace intel {
+/// Containerizes an offloading binary into the ELF binary format expected by
+/// the Intel runtime offload plugin.
+Error containerizeOpenMPSPIRVImage(std::unique_ptr<MemoryBuffer> &Binary);
+} // namespace intel
 } // namespace offloading
 } // namespace llvm
 
diff --git a/llvm/include/llvm/Frontend/OpenMP/ClauseT.h b/llvm/include/llvm/Frontend/OpenMP/ClauseT.h
index 3e22b6ff71c8f..5dc1c4cb54c7c 100644
--- a/llvm/include/llvm/Frontend/OpenMP/ClauseT.h
+++ b/llvm/include/llvm/Frontend/OpenMP/ClauseT.h
@@ -846,6 +846,12 @@ struct NoOpenmpRoutinesT {
   using EmptyTrait = std::true_type;
 };
 
+// V6.0: [10.6.1] `assumption` clauses
+template <typename T, typename I, typename E> //
+struct NoOpenmpConstructsT {
+  using EmptyTrait = std::true_type;
+};
+
 // V5.2: [8.3.1] `assumption` clauses
 template <typename T, typename I, typename E> //
 struct NoParallelismT {
@@ -1239,7 +1245,7 @@ using EmptyClausesT = std::variant<
     ReverseOffloadT<T, I, E>, SeqCstT<T, I, E>, SimdT<T, I, E>,
     ThreadsT<T, I, E>, UnifiedAddressT<T, I, E>, UnifiedSharedMemoryT<T, I, E>,
     UnknownT<T, I, E>, UntiedT<T, I, E>, UseT<T, I, E>, WeakT<T, I, E>,
-    WriteT<T, I, E>>;
+    WriteT<T, I, E>, NoOpenmpConstructsT<T, I, E>>;
 
 template <typename T, typename I, typename E>
 using IncompleteClausesT =
diff --git a/llvm/include/llvm/Frontend/OpenMP/OMP.td b/llvm/include/llvm/Frontend/OpenMP/OMP.td
index a4c1964c3e88f..37b92f2339df9 100644
--- a/llvm/include/llvm/Frontend/OpenMP/OMP.td
+++ b/llvm/include/llvm/Frontend/OpenMP/OMP.td
@@ -265,6 +265,7 @@ def OMPC_Map : Clause<"map"> {
   let flangClass = "OmpMapClause";
 }
 def OMPC_Match : Clause<"match"> {
+  let flangClass = "OmpMatchClause";
 }
 def OMP_MEMORY_ORDER_SeqCst : ClauseVal<"seq_cst", 1, 1> {}
 def OMP_MEMORY_ORDER_AcqRel : ClauseVal<"acq_rel", 2, 1> {}
@@ -298,6 +299,9 @@ def OMPC_NoOpenMP : Clause<"no_openmp"> {
 def OMPC_NoOpenMPRoutines : Clause<"no_openmp_routines"> {
   let clangClass = "OMPNoOpenMPRoutinesClause";
 }
+def OMPC_NoOpenMPConstructs : Clause<"no_openmp_constructs"> {
+  let clangClass = "OMPNoOpenMPConstructsClause";
+}
 def OMPC_NoParallelism : Clause<"no_parallelism"> {
   let clangClass = "OMPNoParallelismClause";
 }
@@ -367,6 +371,10 @@ def OMPC_Ordered : Clause<"ordered"> {
   let flangClass = "ScalarIntConstantExpr";
   let isValueOptional = true;
 }
+def OMPC_Otherwise : Clause<"otherwise"> {
+  let flangClass = "OmpOtherwiseClause";
+  let isValueOptional = true;
+}
 def OMPC_Partial: Clause<"partial"> {
   let clangClass = "OMPPartialClause";
   let flangClass = "ScalarIntConstantExpr";
@@ -524,6 +532,7 @@ def OMPC_Weak : Clause<"weak"> {
   let clangClass = "OMPWeakClause";
 }
 def OMPC_When: Clause<"when"> {
+  let flangClass = "OmpWhenClause";
 }
 def OMPC_Write : Clause<"write"> {
   let clangClass = "OMPWriteClause";
@@ -568,6 +577,7 @@ def OMP_Assume : Directive<"assume"> {
     VersionedClause<OMPC_NoOpenMP, 51>,
     VersionedClause<OMPC_NoOpenMPRoutines, 51>,
     VersionedClause<OMPC_NoParallelism, 51>,
+    VersionedClause<OMPC_NoOpenMPConstructs, 60>,
   ];
 }
 def OMP_Atomic : Directive<"atomic"> {
@@ -845,7 +855,8 @@ def OMP_Metadirective : Directive<"metadirective"> {
     VersionedClause<OMPC_When>,
   ];
   let allowedOnceClauses = [
-    VersionedClause<OMPC_Default>,
+    VersionedClause<OMPC_Otherwise, 52>,
+    VersionedClause<OMPC_Default, 50, 51>,
   ];
   let association = AS_None;
   let category = CA_Meta;
diff --git a/llvm/include/llvm/Frontend/OpenMP/OMPContext.h b/llvm/include/llvm/Frontend/OpenMP/OMPContext.h
index b13b74ceab865..a501eaf2356ff 100644
--- a/llvm/include/llvm/Frontend/OpenMP/OMPContext.h
+++ b/llvm/include/llvm/Frontend/OpenMP/OMPContext.h
@@ -62,7 +62,7 @@ StringRef getOpenMPContextTraitSetName(TraitSet Kind);
 
 /// Parse \p Str and return the trait set it matches or
 /// TraitSelector::invalid.
-TraitSelector getOpenMPContextTraitSelectorKind(StringRef Str);
+TraitSelector getOpenMPContextTraitSelectorKind(StringRef Str, TraitSet Set);
 
 /// Return the trait selector for which \p Property is a property.
 TraitSelector getOpenMPContextTraitSelectorForProperty(TraitProperty Property);
@@ -139,6 +139,8 @@ struct VariantMatchInfo {
     // the raw string.
     if (Property == TraitProperty::device_isa___ANY)
       ISATraits.push_back(RawString);
+    if (Property == TraitProperty::target_device_isa___ANY)
+      ISATraits.push_back(RawString);
 
     RequiredTraits.set(unsigned(Property));
     if (Set == TraitSet::construct)
@@ -155,7 +157,8 @@ struct VariantMatchInfo {
 /// e.g., device={kind(host)}, and constructs traits which describe the nesting
 /// in OpenMP constructs at the location.
 struct OMPContext {
-  OMPContext(bool IsDeviceCompilation, Triple TargetTriple);
+  OMPContext(bool IsDeviceCompilation, Triple TargetTriple,
+             Triple TargetOffloadTriple, int DeviceNum);
   virtual ~OMPContext() = default;
 
   void addTrait(TraitProperty Property) {
diff --git a/llvm/include/llvm/Frontend/OpenMP/OMPGridValues.h b/llvm/include/llvm/Frontend/OpenMP/OMPGridValues.h
index bfac2d734b81d..788a3c8a56f38 100644
--- a/llvm/include/llvm/Frontend/OpenMP/OMPGridValues.h
+++ b/llvm/include/llvm/Frontend/OpenMP/OMPGridValues.h
@@ -120,6 +120,17 @@ static constexpr GV NVPTXGridValues = {
     128,       // GV_Default_WG_Size
 };
 
+/// For generic SPIR-V GPUs
+static constexpr GV SPIRVGridValues = {
+    256,       // GV_Slot_Size
+    64,        // GV_Warp_Size
+    (1 << 16), // GV_Max_Teams
+    440,       // GV_Default_Num_Teams
+    896,       // GV_SimpleBufferSize
+    1024,      // GV_Max_WG_Size,
+    256,       // GV_Default_WG_Size
+};
+
 } // namespace omp
 } // namespace llvm
 
diff --git a/llvm/include/llvm/Frontend/OpenMP/OMPIRBuilder.h b/llvm/include/llvm/Frontend/OpenMP/OMPIRBuilder.h
index 9802cbe8b7b94..d25077cae63e4 100644
--- a/llvm/include/llvm/Frontend/OpenMP/OMPIRBuilder.h
+++ b/llvm/include/llvm/Frontend/OpenMP/OMPIRBuilder.h
@@ -40,9 +40,10 @@ class OpenMPIRBuilder;
 /// not have any PHINodes. If \p CreateBranch is true, a branch instruction to
 /// \p New will be added such that there is no semantic change. Otherwise, the
 /// \p IP insert block remains degenerate and it is up to the caller to insert a
-/// terminator.
-void spliceBB(IRBuilderBase::InsertPoint IP, BasicBlock *New,
-              bool CreateBranch);
+/// terminator. \p DL is used as the debug location for the branch instruction
+/// if one is created.
+void spliceBB(IRBuilderBase::InsertPoint IP, BasicBlock *New, bool CreateBranch,
+              DebugLoc DL);
 
 /// Splice a BasicBlock at an IRBuilder's current insertion point. Its new
 /// insert location will stick to after the instruction before the insertion
@@ -58,9 +59,10 @@ void spliceBB(IRBuilder<> &Builder, BasicBlock *New, bool CreateBranch);
 /// is true, a branch to the new successor will new created such that
 /// semantically there is no change; otherwise the block of the insertion point
 /// remains degenerate and it is the caller's responsibility to insert a
-/// terminator. Returns the new successor block.
+/// terminator. \p DL is used as the debug location for the branch instruction
+/// if one is created. Returns the new successor block.
 BasicBlock *splitBB(IRBuilderBase::InsertPoint IP, bool CreateBranch,
-                    llvm::Twine Name = {});
+                    DebugLoc DL, llvm::Twine Name = {});
 
 /// Split a BasicBlock at \p Builder's insertion point, even if the block is
 /// degenerate (missing the terminator).  Its new insert location will stick to
diff --git a/llvm/include/llvm/Frontend/OpenMP/OMPKinds.def b/llvm/include/llvm/Frontend/OpenMP/OMPKinds.def
index 928a03148c416..44a9a37c70597 100644
--- a/llvm/include/llvm/Frontend/OpenMP/OMPKinds.def
+++ b/llvm/include/llvm/Frontend/OpenMP/OMPKinds.def
@@ -522,6 +522,7 @@ __OMP_RTL(__last, false, Void, )
 #define EnumAttrInt(Kind, N) Attribute::get(Ctx, Attribute::AttrKind::Kind, N)
 #define AllocSizeAttr(N, M) Attribute::getWithAllocSizeArgs(Ctx, N, M)
 #define MemoryAttr(ME) Attribute::getWithMemoryEffects(Ctx, ME)
+#define NoCaptureAttr Attribute::getWithCaptureInfo(Ctx, CaptureInfo::none())
 #define AttributeSet(...)                                                      \
   AttributeSet::get(Ctx, ArrayRef<Attribute>({__VA_ARGS__}))
 
@@ -610,7 +611,7 @@ __OMP_ATTRS_SET(ForkAttrs, OptimisticAttributes
 __OMP_ATTRS_SET(ReadOnlyPtrAttrs,
                 OptimisticAttributes
                     ? AttributeSet(EnumAttr(ReadOnly), EnumAttr(NoFree),
-                                   EnumAttr(NoCapture))
+                                   NoCaptureAttr)
                     : AttributeSet())
 
 __OMP_ATTRS_SET(DeviceAllocAttrs,
@@ -622,13 +623,13 @@ __OMP_ATTRS_SET(DeviceAllocAttrs,
 __OMP_ATTRS_SET(WriteOnlyPtrAttrs,
                 OptimisticAttributes
                     ? AttributeSet(EnumAttr(WriteOnly), EnumAttr(NoFree),
-                                   EnumAttr(NoCapture))
+                                   NoCaptureAttr)
                     : AttributeSet())
 #endif
 
 __OMP_ATTRS_SET(ArgPtrAttrs,
                 OptimisticAttributes
-                    ? AttributeSet(EnumAttr(NoCapture), EnumAttr(NoFree))
+                    ? AttributeSet(NoCaptureAttr, EnumAttr(NoFree))
                     : AttributeSet())
 
 __OMP_ATTRS_SET(ReturnPtrAttrs, AttributeSet(EnumAttr(NoAlias)))
@@ -726,8 +727,8 @@ __OMP_RTL_ATTRS(omp_get_cancellation, GetterAttrs, SExt, ParamAttrs())
 __OMP_RTL_ATTRS(omp_get_nested, GetterAttrs, SExt, ParamAttrs())
 __OMP_RTL_ATTRS(
     omp_get_schedule, GetterArgWriteAttrs, AttributeSet(),
-    ParamAttrs(AttributeSet(EnumAttr(NoCapture), EnumAttr(WriteOnly)),
-               AttributeSet(EnumAttr(NoCapture), EnumAttr(WriteOnly))))
+    ParamAttrs(AttributeSet(NoCaptureAttr, EnumAttr(WriteOnly)),
+               AttributeSet(NoCaptureAttr, EnumAttr(WriteOnly))))
 __OMP_RTL_ATTRS(omp_get_thread_limit, GetterAttrs, SExt, ParamAttrs())
 __OMP_RTL_ATTRS(omp_get_supported_active_levels, GetterAttrs, SExt, ParamAttrs())
 __OMP_RTL_ATTRS(omp_get_max_active_levels, GetterAttrs, SExt, ParamAttrs())
@@ -740,7 +741,7 @@ __OMP_RTL_ATTRS(omp_get_proc_bind, GetterAttrs, SExt, ParamAttrs())
 __OMP_RTL_ATTRS(omp_get_num_places, GetterAttrs, SExt, ParamAttrs())
 __OMP_RTL_ATTRS(omp_get_num_procs, GetterAttrs, SExt, ParamAttrs())
 __OMP_RTL_ATTRS(omp_get_place_proc_ids, GetterArgWriteAttrs, AttributeSet(),
-                ParamAttrs(SExt, AttributeSet(EnumAttr(NoCapture),
+                ParamAttrs(SExt, AttributeSet(NoCaptureAttr,
                                               EnumAttr(WriteOnly))))
 __OMP_RTL_ATTRS(omp_get_place_num, GetterAttrs, SExt, ParamAttrs())
 __OMP_RTL_ATTRS(omp_get_partition_num_places, GetterAttrs, SExt, ParamAttrs())
@@ -984,7 +985,7 @@ __OMP_RTL_ATTRS(__kmpc_alloc_shared,
                              AllocSizeAttr(0, std::nullopt)),
                 ReturnPtrAttrs, ParamAttrs(SizeTyExt))
 __OMP_RTL_ATTRS(__kmpc_free_shared, DeviceAllocAttrs, AttributeSet(),
-                ParamAttrs(AttributeSet(EnumAttr(NoCapture),
+                ParamAttrs(AttributeSet(NoCaptureAttr,
                                         EnumAttr(AllocatedPointer)),
                            SizeTyExt))
 __OMP_RTL_ATTRS(__kmpc_begin_sharing_variables, AttributeSet(), AttributeSet(),
@@ -1269,6 +1270,38 @@ __OMP_TRAIT_PROPERTY(device, arch, amdgcn)
 __OMP_TRAIT_PROPERTY(device, arch, nvptx)
 __OMP_TRAIT_PROPERTY(device, arch, nvptx64)
 
+__OMP_TRAIT_SET(target_device)
+
+__OMP_TRAIT_SELECTOR(target_device, kind, true)
+
+__OMP_TRAIT_PROPERTY(target_device, kind, host)
+__OMP_TRAIT_PROPERTY(target_device, kind, nohost)
+__OMP_TRAIT_PROPERTY(target_device, kind, cpu)
+__OMP_TRAIT_PROPERTY(target_device, kind, gpu)
+__OMP_TRAIT_PROPERTY(target_device, kind, fpga)
+__OMP_TRAIT_PROPERTY(target_device, kind, any)
+
+__OMP_TRAIT_SELECTOR(target_device, device_num, true)
+
+__OMP_TRAIT_PROPERTY(target_device, device_num, number)
+
+__OMP_TRAIT_SELECTOR(target_device, arch, true)
+
+__OMP_TRAIT_PROPERTY(target_device, arch, arm)
+__OMP_TRAIT_PROPERTY(target_device, arch, armeb)
+__OMP_TRAIT_PROPERTY(target_device, arch, aarch64)
+__OMP_TRAIT_PROPERTY(target_device, arch, aarch64_be)
+__OMP_TRAIT_PROPERTY(target_device, arch, aarch64_32)
+__OMP_TRAIT_PROPERTY(target_device, arch, ppc)
+__OMP_TRAIT_PROPERTY(target_device, arch, ppcle)
+__OMP_TRAIT_PROPERTY(target_device, arch, ppc64)
+__OMP_TRAIT_PROPERTY(target_device, arch, ppc64le)
+__OMP_TRAIT_PROPERTY(target_device, arch, x86)
+__OMP_TRAIT_PROPERTY(target_device, arch, x86_64)
+__OMP_TRAIT_PROPERTY(target_device, arch, amdgcn)
+__OMP_TRAIT_PROPERTY(target_device, arch, nvptx)
+__OMP_TRAIT_PROPERTY(target_device, arch, nvptx64)
+
 __OMP_TRAIT_SET(implementation)
 
 __OMP_TRAIT_SELECTOR(implementation, vendor, true)
@@ -1310,12 +1343,16 @@ __OMP_TRAIT_SELECTOR_AND_PROPERTY(construct, dispatch)
 // This allows us to issue warnings wrt. isa only if we match otherwise.
 __OMP_TRAIT_SELECTOR(device, isa, true)
 
+__OMP_TRAIT_SELECTOR(target_device, isa, true)
+
 // We use "__ANY" as a placeholder in the isa property to denote the
 // conceptual "any", not the literal `any` used in kind. The string we
 // we use is not important except that it will show up in diagnostics.
 OMP_TRAIT_PROPERTY(device_isa___ANY, device, device_isa,
                    "<any, entirely target dependent>")
 
+OMP_TRAIT_PROPERTY(target_device_isa___ANY, target_device, target_device_isa,
+                   "<any, entirely target dependent>")
 
 #undef OMP_TRAIT_SET
 #undef __OMP_TRAIT_SET
@@ -1373,6 +1410,7 @@ __OMP_ASSUME_CLAUSE(llvm::StringLiteral("contains"), false, true, false)
 __OMP_ASSUME_CLAUSE(llvm::StringLiteral("holds"), false, false, true)
 __OMP_ASSUME_CLAUSE(llvm::StringLiteral("no_openmp"), false, false, false)
 __OMP_ASSUME_CLAUSE(llvm::StringLiteral("no_openmp_routines"), false, false, false)
+__OMP_ASSUME_CLAUSE(llvm::StringLiteral("no_openmp_constructs"), false, false, false)
 __OMP_ASSUME_CLAUSE(llvm::StringLiteral("no_parallelism"), false, false, false)
 
 #undef __OMP_ASSUME_CLAUSE
diff --git a/llvm/include/llvm/IR/Attributes.h b/llvm/include/llvm/IR/Attributes.h
index 7612e553fe32e..d6533b9bcbea1 100644
--- a/llvm/include/llvm/IR/Attributes.h
+++ b/llvm/include/llvm/IR/Attributes.h
@@ -165,6 +165,7 @@ class Attribute {
   static Attribute getWithUWTableKind(LLVMContext &Context, UWTableKind Kind);
   static Attribute getWithMemoryEffects(LLVMContext &Context, MemoryEffects ME);
   static Attribute getWithNoFPClass(LLVMContext &Context, FPClassTest Mask);
+  static Attribute getWithCaptureInfo(LLVMContext &Context, CaptureInfo CI);
 
   /// For a typed attribute, return the equivalent attribute with the type
   /// changed to \p ReplacementTy.
diff --git a/llvm/include/llvm/IR/Attributes.td b/llvm/include/llvm/IR/Attributes.td
index 4396ec4d04c41..70b9a2c488d3e 100644
--- a/llvm/include/llvm/IR/Attributes.td
+++ b/llvm/include/llvm/IR/Attributes.td
@@ -180,9 +180,6 @@ def NoBuiltin : EnumAttr<"nobuiltin", IntersectPreserve, [FnAttr]>;
 /// Function cannot enter into caller's translation unit.
 def NoCallback : EnumAttr<"nocallback", IntersectAnd, [FnAttr]>;
 
-/// Function creates no aliases of pointer.
-def NoCapture : EnumAttr<"nocapture", IntersectAnd, [ParamAttr]>;
-
 /// Specify how the pointer may be captured.
 def Captures : IntAttr<"captures", IntersectCustom, [ParamAttr]>;
 
diff --git a/llvm/include/llvm/IR/AutoUpgrade.h b/llvm/include/llvm/IR/AutoUpgrade.h
index 97c3e4d7589d7..8c093568a1e03 100644
--- a/llvm/include/llvm/IR/AutoUpgrade.h
+++ b/llvm/include/llvm/IR/AutoUpgrade.h
@@ -61,6 +61,10 @@ namespace llvm {
   /// module is modified.
   bool UpgradeModuleFlags(Module &M);
 
+  /// Convert legacy nvvm.annotations metadata to appropriate function
+  /// attributes.
+  void UpgradeNVVMAnnotations(Module &M);
+
   /// Convert calls to ARC runtime functions to intrinsic calls and upgrade the
   /// old retain release marker to new module flag format.
   void UpgradeARCRuntime(Module &M);
diff --git a/llvm/include/llvm/IR/BasicBlock.h b/llvm/include/llvm/IR/BasicBlock.h
index e22fe1e7e7dc8..c9169cb601809 100644
--- a/llvm/include/llvm/IR/BasicBlock.h
+++ b/llvm/include/llvm/IR/BasicBlock.h
@@ -280,14 +280,24 @@ class BasicBlock final : public Value, // Basic blocks are data objects also
   /// When adding instructions to the beginning of the basic block, they should
   /// be added before the returned value, not before the first instruction,
   /// which might be PHI. Returns 0 is there's no non-PHI instruction.
-  const Instruction* getFirstNonPHI() const;
-  Instruction* getFirstNonPHI() {
-    return const_cast<Instruction *>(
-                       static_cast<const BasicBlock *>(this)->getFirstNonPHI());
-  }
-
-  /// Iterator returning form of getFirstNonPHI. Installed as a placeholder for
-  /// the RemoveDIs project that will eventually remove debug intrinsics.
+  ///
+  /// Deprecated in favour of getFirstNonPHIIt, which returns an iterator that
+  /// preserves some debugging information.
+  LLVM_DEPRECATED("Use iterators as instruction positions", "getFirstNonPHIIt")
+  const Instruction *getFirstNonPHI() const;
+  LLVM_DEPRECATED("Use iterators as instruction positions instead",
+                  "getFirstNonPHIIt")
+  Instruction *getFirstNonPHI();
+
+  /// Returns an iterator to the first instruction in this block that is not a
+  /// PHINode instruction.
+  ///
+  /// When adding instructions to the beginning of the basic block, they should
+  /// be added before the returned value, not before the first instruction,
+  /// which might be PHI. Returns end() if there's no non-PHI instruction.
+  ///
+  /// Avoid unwrapping the iterator to an Instruction* before inserting here,
+  /// as important debug-info is preserved in the iterator.
   InstListType::const_iterator getFirstNonPHIIt() const;
   InstListType::iterator getFirstNonPHIIt() {
     BasicBlock::iterator It =
@@ -299,22 +309,24 @@ class BasicBlock final : public Value, // Basic blocks are data objects also
   /// Returns a pointer to the first instruction in this block that is not a
   /// PHINode or a debug intrinsic, or any pseudo operation if \c SkipPseudoOp
   /// is true.
-  const Instruction *getFirstNonPHIOrDbg(bool SkipPseudoOp = true) const;
-  Instruction *getFirstNonPHIOrDbg(bool SkipPseudoOp = true) {
-    return const_cast<Instruction *>(
-        static_cast<const BasicBlock *>(this)->getFirstNonPHIOrDbg(
-            SkipPseudoOp));
+  InstListType::const_iterator
+  getFirstNonPHIOrDbg(bool SkipPseudoOp = true) const;
+  InstListType::iterator getFirstNonPHIOrDbg(bool SkipPseudoOp = true) {
+    return static_cast<const BasicBlock *>(this)
+        ->getFirstNonPHIOrDbg(SkipPseudoOp)
+        .getNonConst();
   }
 
   /// Returns a pointer to the first instruction in this block that is not a
   /// PHINode, a debug intrinsic, or a lifetime intrinsic, or any pseudo
   /// operation if \c SkipPseudoOp is true.
-  const Instruction *
+  InstListType::const_iterator
   getFirstNonPHIOrDbgOrLifetime(bool SkipPseudoOp = true) const;
-  Instruction *getFirstNonPHIOrDbgOrLifetime(bool SkipPseudoOp = true) {
-    return const_cast<Instruction *>(
-        static_cast<const BasicBlock *>(this)->getFirstNonPHIOrDbgOrLifetime(
-            SkipPseudoOp));
+  InstListType::iterator
+  getFirstNonPHIOrDbgOrLifetime(bool SkipPseudoOp = true) {
+    return static_cast<const BasicBlock *>(this)
+        ->getFirstNonPHIOrDbgOrLifetime(SkipPseudoOp)
+        .getNonConst();
   }
 
   /// Returns an iterator to the first instruction in this block that is
diff --git a/llvm/include/llvm/IR/DIBuilder.h b/llvm/include/llvm/IR/DIBuilder.h
index 6c479415b9ed2..8bee9f4703dd9 100644
--- a/llvm/include/llvm/IR/DIBuilder.h
+++ b/llvm/include/llvm/IR/DIBuilder.h
@@ -632,7 +632,8 @@ namespace llvm {
         DIScope *Scope, StringRef Name, DIFile *File, unsigned LineNumber,
         uint64_t SizeInBits, uint32_t AlignInBits, DINodeArray Elements,
         DIType *UnderlyingType, unsigned RunTimeLang = 0,
-        StringRef UniqueIdentifier = "", bool IsScoped = false);
+        StringRef UniqueIdentifier = "", bool IsScoped = false,
+        std::optional<uint32_t> EnumKind = std::nullopt);
     /// Create debugging information entry for a set.
     /// \param Scope          Scope in which this set is defined.
     /// \param Name           Set name.
@@ -667,19 +668,20 @@ namespace llvm {
     static DIType *createObjectPointerType(DIType *Ty, bool Implicit);
 
     /// Create a permanent forward-declared type.
-    DICompositeType *createForwardDecl(unsigned Tag, StringRef Name,
-                                       DIScope *Scope, DIFile *F, unsigned Line,
-                                       unsigned RuntimeLang = 0,
-                                       uint64_t SizeInBits = 0,
-                                       uint32_t AlignInBits = 0,
-                                       StringRef UniqueIdentifier = "");
+    DICompositeType *
+    createForwardDecl(unsigned Tag, StringRef Name, DIScope *Scope, DIFile *F,
+                      unsigned Line, unsigned RuntimeLang = 0,
+                      uint64_t SizeInBits = 0, uint32_t AlignInBits = 0,
+                      StringRef UniqueIdentifier = "",
+                      std::optional<uint32_t> EnumKind = std::nullopt);
 
     /// Create a temporary forward-declared type.
     DICompositeType *createReplaceableCompositeType(
         unsigned Tag, StringRef Name, DIScope *Scope, DIFile *F, unsigned Line,
         unsigned RuntimeLang = 0, uint64_t SizeInBits = 0,
         uint32_t AlignInBits = 0, DINode::DIFlags Flags = DINode::FlagFwdDecl,
-        StringRef UniqueIdentifier = "", DINodeArray Annotations = nullptr);
+        StringRef UniqueIdentifier = "", DINodeArray Annotations = nullptr,
+        std::optional<uint32_t> EnumKind = std::nullopt);
 
     /// Retain DIScope* in a module even if it is not referenced
     /// through debug info anchors.
diff --git a/llvm/include/llvm/IR/DebugInfoMetadata.h b/llvm/include/llvm/IR/DebugInfoMetadata.h
index 5ea8c0d7b448d..8515d8eda8568 100644
--- a/llvm/include/llvm/IR/DebugInfoMetadata.h
+++ b/llvm/include/llvm/IR/DebugInfoMetadata.h
@@ -1176,24 +1176,28 @@ class DICompositeType : public DIType {
   friend class MDNode;
 
   unsigned RuntimeLang;
+  std::optional<uint32_t> EnumKind;
 
   DICompositeType(LLVMContext &C, StorageType Storage, unsigned Tag,
                   unsigned Line, unsigned RuntimeLang, uint64_t SizeInBits,
                   uint32_t AlignInBits, uint64_t OffsetInBits,
-                  uint32_t NumExtraInhabitants, DIFlags Flags,
+                  uint32_t NumExtraInhabitants,
+                  std::optional<uint32_t> EnumKind, DIFlags Flags,
                   ArrayRef<Metadata *> Ops)
       : DIType(C, DICompositeTypeKind, Storage, Tag, Line, SizeInBits,
                AlignInBits, OffsetInBits, NumExtraInhabitants, Flags, Ops),
-        RuntimeLang(RuntimeLang) {}
+        RuntimeLang(RuntimeLang), EnumKind(EnumKind) {}
   ~DICompositeType() = default;
 
   /// Change fields in place.
   void mutate(unsigned Tag, unsigned Line, unsigned RuntimeLang,
               uint64_t SizeInBits, uint32_t AlignInBits, uint64_t OffsetInBits,
-              uint32_t NumExtraInhabitants, DIFlags Flags) {
+              uint32_t NumExtraInhabitants, std::optional<uint32_t> EnumKind,
+              DIFlags Flags) {
     assert(isDistinct() && "Only distinct nodes can mutate");
     assert(getRawIdentifier() && "Only ODR-uniqued nodes should mutate");
     this->RuntimeLang = RuntimeLang;
+    this->EnumKind = EnumKind;
     DIType::mutate(Tag, Line, SizeInBits, AlignInBits, OffsetInBits,
                    NumExtraInhabitants, Flags);
   }
@@ -1203,15 +1207,15 @@ class DICompositeType : public DIType {
           unsigned Line, DIScope *Scope, DIType *BaseType, uint64_t SizeInBits,
           uint32_t AlignInBits, uint64_t OffsetInBits, DIType *Specification,
           uint32_t NumExtraInhabitants, DIFlags Flags, DINodeArray Elements,
-          unsigned RuntimeLang, DIType *VTableHolder,
-          DITemplateParameterArray TemplateParams, StringRef Identifier,
-          DIDerivedType *Discriminator, Metadata *DataLocation,
-          Metadata *Associated, Metadata *Allocated, Metadata *Rank,
-          DINodeArray Annotations, StorageType Storage,
+          unsigned RuntimeLang, std::optional<uint32_t> EnumKind,
+          DIType *VTableHolder, DITemplateParameterArray TemplateParams,
+          StringRef Identifier, DIDerivedType *Discriminator,
+          Metadata *DataLocation, Metadata *Associated, Metadata *Allocated,
+          Metadata *Rank, DINodeArray Annotations, StorageType Storage,
           bool ShouldCreate = true) {
     return getImpl(Context, Tag, getCanonicalMDString(Context, Name), File,
                    Line, Scope, BaseType, SizeInBits, AlignInBits, OffsetInBits,
-                   Flags, Elements.get(), RuntimeLang, VTableHolder,
+                   Flags, Elements.get(), RuntimeLang, EnumKind, VTableHolder,
                    TemplateParams.get(),
                    getCanonicalMDString(Context, Identifier), Discriminator,
                    DataLocation, Associated, Allocated, Rank, Annotations.get(),
@@ -1222,21 +1226,21 @@ class DICompositeType : public DIType {
           unsigned Line, Metadata *Scope, Metadata *BaseType,
           uint64_t SizeInBits, uint32_t AlignInBits, uint64_t OffsetInBits,
           DIFlags Flags, Metadata *Elements, unsigned RuntimeLang,
-          Metadata *VTableHolder, Metadata *TemplateParams,
-          MDString *Identifier, Metadata *Discriminator, Metadata *DataLocation,
-          Metadata *Associated, Metadata *Allocated, Metadata *Rank,
-          Metadata *Annotations, Metadata *Specification,
-          uint32_t NumExtraInhabitants, StorageType Storage,
-          bool ShouldCreate = true);
+          std::optional<uint32_t> EnumKind, Metadata *VTableHolder,
+          Metadata *TemplateParams, MDString *Identifier,
+          Metadata *Discriminator, Metadata *DataLocation, Metadata *Associated,
+          Metadata *Allocated, Metadata *Rank, Metadata *Annotations,
+          Metadata *Specification, uint32_t NumExtraInhabitants,
+          StorageType Storage, bool ShouldCreate = true);
 
   TempDICompositeType cloneImpl() const {
     return getTemporary(
         getContext(), getTag(), getName(), getFile(), getLine(), getScope(),
         getBaseType(), getSizeInBits(), getAlignInBits(), getOffsetInBits(),
-        getFlags(), getElements(), getRuntimeLang(), getVTableHolder(),
-        getTemplateParams(), getIdentifier(), getDiscriminator(),
-        getRawDataLocation(), getRawAssociated(), getRawAllocated(),
-        getRawRank(), getAnnotations(), getSpecification(),
+        getFlags(), getElements(), getRuntimeLang(), getEnumKind(),
+        getVTableHolder(), getTemplateParams(), getIdentifier(),
+        getDiscriminator(), getRawDataLocation(), getRawAssociated(),
+        getRawAllocated(), getRawRank(), getAnnotations(), getSpecification(),
         getNumExtraInhabitants());
   }
 
@@ -1246,7 +1250,8 @@ class DICompositeType : public DIType {
       (unsigned Tag, StringRef Name, DIFile *File, unsigned Line,
        DIScope *Scope, DIType *BaseType, uint64_t SizeInBits,
        uint32_t AlignInBits, uint64_t OffsetInBits, DIFlags Flags,
-       DINodeArray Elements, unsigned RuntimeLang, DIType *VTableHolder,
+       DINodeArray Elements, unsigned RuntimeLang,
+       std::optional<uint32_t> EnumKind, DIType *VTableHolder,
        DITemplateParameterArray TemplateParams = nullptr,
        StringRef Identifier = "", DIDerivedType *Discriminator = nullptr,
        Metadata *DataLocation = nullptr, Metadata *Associated = nullptr,
@@ -1255,23 +1260,24 @@ class DICompositeType : public DIType {
        uint32_t NumExtraInhabitants = 0),
       (Tag, Name, File, Line, Scope, BaseType, SizeInBits, AlignInBits,
        OffsetInBits, Specification, NumExtraInhabitants, Flags, Elements,
-       RuntimeLang, VTableHolder, TemplateParams, Identifier, Discriminator,
-       DataLocation, Associated, Allocated, Rank, Annotations))
+       RuntimeLang, EnumKind, VTableHolder, TemplateParams, Identifier,
+       Discriminator, DataLocation, Associated, Allocated, Rank, Annotations))
   DEFINE_MDNODE_GET(
       DICompositeType,
       (unsigned Tag, MDString *Name, Metadata *File, unsigned Line,
        Metadata *Scope, Metadata *BaseType, uint64_t SizeInBits,
        uint32_t AlignInBits, uint64_t OffsetInBits, DIFlags Flags,
-       Metadata *Elements, unsigned RuntimeLang, Metadata *VTableHolder,
+       Metadata *Elements, unsigned RuntimeLang,
+       std::optional<uint32_t> EnumKind, Metadata *VTableHolder,
        Metadata *TemplateParams = nullptr, MDString *Identifier = nullptr,
        Metadata *Discriminator = nullptr, Metadata *DataLocation = nullptr,
        Metadata *Associated = nullptr, Metadata *Allocated = nullptr,
        Metadata *Rank = nullptr, Metadata *Annotations = nullptr,
        Metadata *Specification = nullptr, uint32_t NumExtraInhabitants = 0),
       (Tag, Name, File, Line, Scope, BaseType, SizeInBits, AlignInBits,
-       OffsetInBits, Flags, Elements, RuntimeLang, VTableHolder, TemplateParams,
-       Identifier, Discriminator, DataLocation, Associated, Allocated, Rank,
-       Annotations, Specification, NumExtraInhabitants))
+       OffsetInBits, Flags, Elements, RuntimeLang, EnumKind, VTableHolder,
+       TemplateParams, Identifier, Discriminator, DataLocation, Associated,
+       Allocated, Rank, Annotations, Specification, NumExtraInhabitants))
 
   TempDICompositeType clone() const { return cloneImpl(); }
 
@@ -1288,10 +1294,11 @@ class DICompositeType : public DIType {
              Metadata *BaseType, uint64_t SizeInBits, uint32_t AlignInBits,
              uint64_t OffsetInBits, Metadata *Specification,
              uint32_t NumExtraInhabitants, DIFlags Flags, Metadata *Elements,
-             unsigned RuntimeLang, Metadata *VTableHolder,
-             Metadata *TemplateParams, Metadata *Discriminator,
-             Metadata *DataLocation, Metadata *Associated, Metadata *Allocated,
-             Metadata *Rank, Metadata *Annotations);
+             unsigned RuntimeLang, std::optional<uint32_t> EnumKind,
+             Metadata *VTableHolder, Metadata *TemplateParams,
+             Metadata *Discriminator, Metadata *DataLocation,
+             Metadata *Associated, Metadata *Allocated, Metadata *Rank,
+             Metadata *Annotations);
   static DICompositeType *getODRTypeIfExists(LLVMContext &Context,
                                              MDString &Identifier);
 
@@ -1310,10 +1317,11 @@ class DICompositeType : public DIType {
                Metadata *BaseType, uint64_t SizeInBits, uint32_t AlignInBits,
                uint64_t OffsetInBits, Metadata *Specification,
                uint32_t NumExtraInhabitants, DIFlags Flags, Metadata *Elements,
-               unsigned RuntimeLang, Metadata *VTableHolder,
-               Metadata *TemplateParams, Metadata *Discriminator,
-               Metadata *DataLocation, Metadata *Associated,
-               Metadata *Allocated, Metadata *Rank, Metadata *Annotations);
+               unsigned RuntimeLang, std::optional<uint32_t> EnumKind,
+               Metadata *VTableHolder, Metadata *TemplateParams,
+               Metadata *Discriminator, Metadata *DataLocation,
+               Metadata *Associated, Metadata *Allocated, Metadata *Rank,
+               Metadata *Annotations);
 
   DIType *getBaseType() const { return cast_or_null<DIType>(getRawBaseType()); }
   DINodeArray getElements() const {
@@ -1327,6 +1335,7 @@ class DICompositeType : public DIType {
   }
   StringRef getIdentifier() const { return getStringOperand(7); }
   unsigned getRuntimeLang() const { return RuntimeLang; }
+  std::optional<uint32_t> getEnumKind() const { return EnumKind; }
 
   Metadata *getRawBaseType() const { return getOperand(3); }
   Metadata *getRawElements() const { return getOperand(4); }
diff --git a/llvm/include/llvm/IR/DerivedTypes.h b/llvm/include/llvm/IR/DerivedTypes.h
index b44f4f8c8687d..60606d34c32c3 100644
--- a/llvm/include/llvm/IR/DerivedTypes.h
+++ b/llvm/include/llvm/IR/DerivedTypes.h
@@ -536,6 +536,15 @@ class VectorType : public Type {
                            EltCnt.divideCoefficientBy(2));
   }
 
+  static VectorType *getOneNthElementsVectorType(VectorType *VTy,
+                                                 unsigned Denominator) {
+    auto EltCnt = VTy->getElementCount();
+    assert(EltCnt.isKnownMultipleOf(Denominator) &&
+           "Cannot take one-nth of a vector");
+    return VectorType::get(VTy->getScalarType(),
+                           EltCnt.divideCoefficientBy(Denominator));
+  }
+
   /// This static method returns a VectorType with twice as many elements as the
   /// input type and the same element type.
   static VectorType *getDoubleElementsVectorType(VectorType *VTy) {
diff --git a/llvm/include/llvm/IR/Function.h b/llvm/include/llvm/IR/Function.h
index e7afcbd31420c..29041688124bc 100644
--- a/llvm/include/llvm/IR/Function.h
+++ b/llvm/include/llvm/IR/Function.h
@@ -284,6 +284,18 @@ class LLVM_ABI Function : public GlobalObject, public ilist_node<Function> {
     setValueSubclassData((getSubclassDataFromValue() & 0xc00f) | (ID << 4));
   }
 
+  /// Does it have a kernel calling convention?
+  bool hasKernelCallingConv() const {
+    switch (getCallingConv()) {
+    default:
+      return false;
+    case CallingConv::PTX_Kernel:
+    case CallingConv::AMDGPU_KERNEL:
+    case CallingConv::SPIR_KERNEL:
+      return true;
+    }
+  }
+
   enum ProfileCountType { PCT_Real, PCT_Synthetic };
 
   /// Class to represent profile counts.
@@ -334,12 +346,6 @@ class LLVM_ABI Function : public GlobalObject, public ilist_node<Function> {
   /// sample PGO, to enable the same inlines as the profiled optimized binary.
   DenseSet<GlobalValue::GUID> getImportGUIDs() const;
 
-  /// Set the section prefix for this function.
-  void setSectionPrefix(StringRef Prefix);
-
-  /// Get the section prefix for this function.
-  std::optional<StringRef> getSectionPrefix() const;
-
   /// hasGC/getGC/setGC/clearGC - The name of the garbage collection algorithm
   ///                             to use during code generation.
   bool hasGC() const {
diff --git a/llvm/include/llvm/IR/GlobalObject.h b/llvm/include/llvm/IR/GlobalObject.h
index 08edc13d81f88..73c1b87d67717 100644
--- a/llvm/include/llvm/IR/GlobalObject.h
+++ b/llvm/include/llvm/IR/GlobalObject.h
@@ -124,6 +124,12 @@ class GlobalObject : public GlobalValue {
   /// appropriate default object file section.
   void setSection(StringRef S);
 
+  /// Set the section prefix for this global object.
+  void setSectionPrefix(StringRef Prefix);
+
+  /// Get the section prefix for this global object.
+  std::optional<StringRef> getSectionPrefix() const;
+
   bool hasComdat() const { return getComdat() != nullptr; }
   const Comdat *getComdat() const { return ObjComdat; }
   Comdat *getComdat() { return ObjComdat; }
diff --git a/llvm/include/llvm/IR/InstrTypes.h b/llvm/include/llvm/IR/InstrTypes.h
index 47ddc7555594c..90fe864d4ae71 100644
--- a/llvm/include/llvm/IR/InstrTypes.h
+++ b/llvm/include/llvm/IR/InstrTypes.h
@@ -1015,7 +1015,7 @@ struct OperandBundleUse {
   /// has the attribute A.
   bool operandHasAttr(unsigned Idx, Attribute::AttrKind A) const {
     if (isDeoptOperandBundle())
-      if (A == Attribute::ReadOnly || A == Attribute::NoCapture)
+      if (A == Attribute::ReadOnly)
         return Inputs[Idx]->getType()->isPointerTy();
 
     // Conservative answer:  no operands have any attributes.
@@ -1490,6 +1490,11 @@ class CallBase : public Instruction {
     Attrs = Attrs.addRetAttribute(getContext(), Attr);
   }
 
+  /// Adds attributes to the return value.
+  void addRetAttrs(const AttrBuilder &B) {
+    Attrs = Attrs.addRetAttributes(getContext(), B);
+  }
+
   /// Adds the attribute to the indicated argument
   void addParamAttr(unsigned ArgNo, Attribute::AttrKind Kind) {
     assert(ArgNo < arg_size() && "Out of bounds");
@@ -1502,6 +1507,12 @@ class CallBase : public Instruction {
     Attrs = Attrs.addParamAttribute(getContext(), ArgNo, Attr);
   }
 
+  /// Adds attributes to the indicated argument
+  void addParamAttrs(unsigned ArgNo, const AttrBuilder &B) {
+    assert(ArgNo < arg_size() && "Out of bounds");
+    Attrs = Attrs.addParamAttributes(getContext(), ArgNo, B);
+  }
+
   /// removes the attribute from the list of attributes.
   void removeAttributeAtIndex(unsigned i, Attribute::AttrKind Kind) {
     Attrs = Attrs.removeAttributeAtIndex(getContext(), i, Kind);
@@ -1591,6 +1602,14 @@ class CallBase : public Instruction {
   /// Determine whether the argument or parameter has the given attribute.
   bool paramHasAttr(unsigned ArgNo, Attribute::AttrKind Kind) const;
 
+  /// Return true if this argument has the nonnull attribute on either the
+  /// CallBase instruction or the called function. Also returns true if at least
+  /// one byte is known to be dereferenceable and the pointer is in
+  /// addrspace(0). If \p AllowUndefOrPoison is true, respect the semantics of
+  /// nonnull attribute and return true even if the argument can be undef or
+  /// poison.
+  bool paramHasNonNullAttr(unsigned ArgNo, bool AllowUndefOrPoison) const;
+
   /// Get the attribute of a given kind at a position.
   Attribute getAttributeAtIndex(unsigned i, Attribute::AttrKind Kind) const {
     return getAttributes().getAttributeAtIndex(i, Kind);
@@ -1663,16 +1682,14 @@ class CallBase : public Instruction {
     return bundleOperandHasAttr(i, Kind);
   }
 
+  /// Return which pointer components this operand may capture.
+  CaptureInfo getCaptureInfo(unsigned OpNo) const;
+
   /// Determine whether this data operand is not captured.
   // FIXME: Once this API is no longer duplicated in `CallSite`, rename this to
   // better indicate that this may return a conservative answer.
   bool doesNotCapture(unsigned OpNo) const {
-    // If the argument is passed byval, the callee does not have access to the
-    // original pointer and thus cannot capture it.
-    if (OpNo < arg_size() && isByValArgument(OpNo))
-      return true;
-
-    return dataOperandHasImpliedAttr(OpNo, Attribute::NoCapture);
+    return capturesNothing(getCaptureInfo(OpNo));
   }
 
   /// Determine whether this argument is passed by value.
diff --git a/llvm/include/llvm/IR/Instruction.h b/llvm/include/llvm/IR/Instruction.h
index 6cdd79ce16005..900384432d75d 100644
--- a/llvm/include/llvm/IR/Instruction.h
+++ b/llvm/include/llvm/IR/Instruction.h
@@ -206,6 +206,12 @@ class Instruction : public User,
 
   /// Insert an unlinked instruction into a basic block immediately before
   /// the specified instruction.
+  ///
+  /// Deprecated in favour of the iterator-accepting flavour. Iterators at the
+  /// start of a block such as BasicBlock::getFirstNonPHIIt must be passed into
+  /// insertBefore without unwrapping/rewrapping. For all other positions, call
+  /// getIterator to fetch the instruction iterator.
+  LLVM_DEPRECATED("Use iterators as instruction positions", "")
   void insertBefore(Instruction *InsertPos);
 
   /// Insert an unlinked instruction into a basic block immediately before
@@ -229,6 +235,12 @@ class Instruction : public User,
 
   /// Unlink this instruction from its current basic block and insert it into
   /// the basic block that MovePos lives in, right before MovePos.
+  ///
+  /// Deprecated in favour of the iterator-accepting flavour. Iterators at the
+  /// start of a block such as BasicBlock::getFirstNonPHIIt must be passed into
+  /// moveBefore without unwrapping/rewrapping. For all other positions, call
+  /// getIterator to fetch the instruction iterator.
+  LLVM_DEPRECATED("Use iterators as instruction positions", "")
   void moveBefore(Instruction *MovePos);
 
   /// Unlink this instruction from its current basic block and insert it into
@@ -238,8 +250,20 @@ class Instruction : public User,
   /// Perform a \ref moveBefore operation, while signalling that the caller
   /// intends to preserve the original ordering of instructions. This implicitly
   /// means that any adjacent debug-info should move with this instruction.
-  /// This method is currently a no-op placeholder, but it will become
-  /// meaningful when the "RemoveDIs" project is enabled.
+  void moveBeforePreserving(InstListType::iterator MovePos);
+
+  /// Perform a \ref moveBefore operation, while signalling that the caller
+  /// intends to preserve the original ordering of instructions. This implicitly
+  /// means that any adjacent debug-info should move with this instruction.
+  void moveBeforePreserving(BasicBlock &BB, InstListType::iterator I);
+
+  /// Perform a \ref moveBefore operation, while signalling that the caller
+  /// intends to preserve the original ordering of instructions. This implicitly
+  /// means that any adjacent debug-info should move with this instruction.
+  ///
+  /// Deprecated in favour of the iterator-accepting flavour of
+  /// moveBeforePreserving, as all insertions should be at iterator positions.
+  LLVM_DEPRECATED("Use iterators as instruction positions", "")
   void moveBeforePreserving(Instruction *MovePos);
 
 private:
@@ -253,11 +277,6 @@ class Instruction : public User,
   /// \pre I is a valid iterator into BB.
   void moveBefore(BasicBlock &BB, InstListType::iterator I);
 
-  void moveBeforePreserving(BasicBlock &BB, InstListType::iterator I);
-  /// Unlink this instruction from its current basic block and insert it into
-  /// the basic block that MovePos lives in, right before MovePos.
-  void moveBeforePreserving(InstListType::iterator I);
-
   /// Unlink this instruction from its current basic block and insert it into
   /// the basic block that MovePos lives in, right after MovePos.
   void moveAfter(Instruction *MovePos);
diff --git a/llvm/include/llvm/IR/Instructions.h b/llvm/include/llvm/IR/Instructions.h
index 9a41971b63373..a1f964352207f 100644
--- a/llvm/include/llvm/IR/Instructions.h
+++ b/llvm/include/llvm/IR/Instructions.h
@@ -1231,6 +1231,18 @@ class ICmpInst: public CmpInst {
     return getSwappedCmpPredicate(getCmpPredicate());
   }
 
+  /// @returns the non-strict predicate along with samesign information: static
+  /// variant.
+  static CmpPredicate getNonStrictCmpPredicate(CmpPredicate Pred) {
+    return {getNonStrictPredicate(Pred), Pred.hasSameSign()};
+  }
+
+  /// For example, SGT -> SGE, SLT -> SLE, ULT -> ULE, UGT -> UGE.
+  /// @returns the non-strict predicate along with samesign information.
+  Predicate getNonStrictCmpPredicate() const {
+    return getNonStrictCmpPredicate(getCmpPredicate());
+  }
+
   /// For example, EQ->EQ, SLE->SLE, UGT->SGT, etc.
   /// @returns the predicate that would be the result if the operand were
   /// regarded as signed.
diff --git a/llvm/include/llvm/IR/IntrinsicInst.h b/llvm/include/llvm/IR/IntrinsicInst.h
index 6ccbb6b185c7d..93750d6e3845e 100644
--- a/llvm/include/llvm/IR/IntrinsicInst.h
+++ b/llvm/include/llvm/IR/IntrinsicInst.h
@@ -1882,6 +1882,11 @@ class ConvergenceControlInst : public IntrinsicInst {
   bool isLoop() const {
     return getIntrinsicID() == Intrinsic::experimental_convergence_loop;
   }
+
+  static ConvergenceControlInst *CreateAnchor(BasicBlock &BB);
+  static ConvergenceControlInst *CreateEntry(BasicBlock &BB);
+  static ConvergenceControlInst *CreateLoop(BasicBlock &BB,
+                                            ConvergenceControlInst *Parent);
 };
 
 } // end namespace llvm
diff --git a/llvm/include/llvm/IR/Intrinsics.h b/llvm/include/llvm/IR/Intrinsics.h
index 82f72131b9d2f..65a7fc0ce2c1c 100644
--- a/llvm/include/llvm/IR/Intrinsics.h
+++ b/llvm/include/llvm/IR/Intrinsics.h
@@ -148,6 +148,9 @@ namespace Intrinsic {
       ExtendArgument,
       TruncArgument,
       HalfVecArgument,
+      OneThirdVecArgument,
+      OneFifthVecArgument,
+      OneSeventhVecArgument,
       SameVecWidthArgument,
       VecOfAnyPtrsToElt,
       VecElementArgument,
@@ -159,6 +162,9 @@ namespace Intrinsic {
       AArch64Svcount,
     } Kind;
 
+    // These three have to be contiguous.
+    static_assert(OneFifthVecArgument == OneThirdVecArgument + 1 &&
+                  OneSeventhVecArgument == OneFifthVecArgument + 1);
     union {
       unsigned Integer_Width;
       unsigned Float_Width;
@@ -178,15 +184,17 @@ namespace Intrinsic {
     unsigned getArgumentNumber() const {
       assert(Kind == Argument || Kind == ExtendArgument ||
              Kind == TruncArgument || Kind == HalfVecArgument ||
-             Kind == SameVecWidthArgument || Kind == VecElementArgument ||
-             Kind == Subdivide2Argument || Kind == Subdivide4Argument ||
-             Kind == VecOfBitcastsToInt);
+             Kind == OneThirdVecArgument || Kind == OneFifthVecArgument ||
+             Kind == OneSeventhVecArgument || Kind == SameVecWidthArgument ||
+             Kind == VecElementArgument || Kind == Subdivide2Argument ||
+             Kind == Subdivide4Argument || Kind == VecOfBitcastsToInt);
       return Argument_Info >> 3;
     }
     ArgKind getArgumentKind() const {
       assert(Kind == Argument || Kind == ExtendArgument ||
              Kind == TruncArgument || Kind == HalfVecArgument ||
-             Kind == SameVecWidthArgument ||
+             Kind == OneThirdVecArgument || Kind == OneFifthVecArgument ||
+             Kind == OneSeventhVecArgument || Kind == SameVecWidthArgument ||
              Kind == VecElementArgument || Kind == Subdivide2Argument ||
              Kind == Subdivide4Argument || Kind == VecOfBitcastsToInt);
       return (ArgKind)(Argument_Info & 7);
diff --git a/llvm/include/llvm/IR/Intrinsics.td b/llvm/include/llvm/IR/Intrinsics.td
index ee877349a3314..14ecae41ff08f 100644
--- a/llvm/include/llvm/IR/Intrinsics.td
+++ b/llvm/include/llvm/IR/Intrinsics.td
@@ -327,6 +327,9 @@ def IIT_I4 : IIT_Int<4, 58>;
 def IIT_AARCH64_SVCOUNT : IIT_VT<aarch64svcount, 59>;
 def IIT_V6 : IIT_Vec<6, 60>;
 def IIT_V10 : IIT_Vec<10, 61>;
+def IIT_ONE_THIRD_VEC_ARG : IIT_Base<62>;
+def IIT_ONE_FIFTH_VEC_ARG : IIT_Base<63>;
+def IIT_ONE_SEVENTH_VEC_ARG : IIT_Base<64>;
 }
 
 defvar IIT_all_FixedTypes = !filter(iit, IIT_all,
@@ -467,6 +470,15 @@ class LLVMVectorElementType<int num> : LLVMMatchType<num, IIT_VEC_ELEMENT>;
 class LLVMHalfElementsVectorType<int num>
   : LLVMMatchType<num, IIT_HALF_VEC_ARG>;
 
+class LLVMOneThirdElementsVectorType<int num>
+  : LLVMMatchType<num, IIT_ONE_THIRD_VEC_ARG>;
+
+class LLVMOneFifthElementsVectorType<int num>
+  : LLVMMatchType<num, IIT_ONE_FIFTH_VEC_ARG>;
+
+class LLVMOneSeventhElementsVectorType<int num>
+  : LLVMMatchType<num, IIT_ONE_SEVENTH_VEC_ARG>;
+
 // Match the type of another intrinsic parameter that is expected to be a
 // vector type (i.e. <N x iM>) but with each element subdivided to
 // form a vector with more elements that are smaller than the original.
@@ -1063,6 +1075,10 @@ let IntrProperties = [IntrNoMem, IntrSpeculatable, IntrWillReturn] in {
   def int_roundeven    : DefaultAttrsIntrinsic<[llvm_anyfloat_ty], [LLVMMatchType<0>]>;
   def int_sincos : DefaultAttrsIntrinsic<[LLVMMatchType<0>, LLVMMatchType<0>],
                              [llvm_anyfloat_ty]>;
+  def int_sincospi : DefaultAttrsIntrinsic<[LLVMMatchType<0>, LLVMMatchType<0>],
+                             [llvm_anyfloat_ty]>;
+  def int_modf : DefaultAttrsIntrinsic<[LLVMMatchType<0>, LLVMMatchType<0>],
+                             [llvm_anyfloat_ty]>;
 
   // Truncate a floating point number with a specific rounding mode
   def int_fptrunc_round : DefaultAttrsIntrinsic<[ llvm_anyfloat_ty ],
@@ -2728,6 +2744,54 @@ def int_vector_deinterleave2 : DefaultAttrsIntrinsic<[LLVMHalfElementsVectorType
                                                      [llvm_anyvector_ty],
                                                      [IntrNoMem]>;
 
+def int_vector_interleave3   : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
+                                                     [LLVMOneThirdElementsVectorType<0>,
+                                                      LLVMOneThirdElementsVectorType<0>,
+                                                      LLVMOneThirdElementsVectorType<0>],
+                                                     [IntrNoMem]>;
+
+def int_vector_deinterleave3 : DefaultAttrsIntrinsic<[LLVMOneThirdElementsVectorType<0>,
+                                                      LLVMOneThirdElementsVectorType<0>,
+                                                      LLVMOneThirdElementsVectorType<0>],
+                                                     [llvm_anyvector_ty],
+                                                     [IntrNoMem]>;
+
+def int_vector_interleave5   : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
+                                                     [LLVMOneFifthElementsVectorType<0>,
+                                                      LLVMOneFifthElementsVectorType<0>,
+                                                      LLVMOneFifthElementsVectorType<0>,
+                                                      LLVMOneFifthElementsVectorType<0>,
+                                                      LLVMOneFifthElementsVectorType<0>],
+                                                     [IntrNoMem]>;
+
+def int_vector_deinterleave5 : DefaultAttrsIntrinsic<[LLVMOneFifthElementsVectorType<0>,
+                                                      LLVMOneFifthElementsVectorType<0>,
+                                                      LLVMOneFifthElementsVectorType<0>,
+                                                      LLVMOneFifthElementsVectorType<0>,
+                                                      LLVMOneFifthElementsVectorType<0>],
+                                                     [llvm_anyvector_ty],
+                                                     [IntrNoMem]>;
+
+def int_vector_interleave7   : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
+                                                     [LLVMOneSeventhElementsVectorType<0>,
+                                                      LLVMOneSeventhElementsVectorType<0>,
+                                                      LLVMOneSeventhElementsVectorType<0>,
+                                                      LLVMOneSeventhElementsVectorType<0>,
+                                                      LLVMOneSeventhElementsVectorType<0>,
+                                                      LLVMOneSeventhElementsVectorType<0>,
+                                                      LLVMOneSeventhElementsVectorType<0>],
+                                                     [IntrNoMem]>;
+
+def int_vector_deinterleave7 : DefaultAttrsIntrinsic<[LLVMOneSeventhElementsVectorType<0>,
+                                                      LLVMOneSeventhElementsVectorType<0>,
+                                                      LLVMOneSeventhElementsVectorType<0>,
+                                                      LLVMOneSeventhElementsVectorType<0>,
+                                                      LLVMOneSeventhElementsVectorType<0>,
+                                                      LLVMOneSeventhElementsVectorType<0>,
+                                                      LLVMOneSeventhElementsVectorType<0>],
+                                                     [llvm_anyvector_ty],
+                                                     [IntrNoMem]>;
+
 //===-------------- Intrinsics to perform partial reduction ---------------===//
 
 def int_experimental_vector_partial_reduce_add : DefaultAttrsIntrinsic<[LLVMMatchType<0>],
diff --git a/llvm/include/llvm/IR/IntrinsicsAArch64.td b/llvm/include/llvm/IR/IntrinsicsAArch64.td
index b31a65d9bcc02..6dfc3c8f2a393 100644
--- a/llvm/include/llvm/IR/IntrinsicsAArch64.td
+++ b/llvm/include/llvm/IR/IntrinsicsAArch64.td
@@ -993,6 +993,82 @@ def int_aarch64_st64b: Intrinsic<[], !listconcat([llvm_ptr_ty], data512)>;
 def int_aarch64_st64bv: Intrinsic<[llvm_i64_ty], !listconcat([llvm_ptr_ty], data512)>;
 def int_aarch64_st64bv0: Intrinsic<[llvm_i64_ty], !listconcat([llvm_ptr_ty], data512)>;
 
+  //
+  // Neon FP8 intrinsics
+  //
+
+  // Conversions
+  class AdvSIMD_FP8_1VectorArg_Long_Intrinsic
+    : DefaultAttrsIntrinsic<[llvm_anyvector_ty], [llvm_anyvector_ty], [IntrReadMem, IntrInaccessibleMemOnly]>;
+
+  def int_aarch64_neon_fp8_cvtl1   : AdvSIMD_FP8_1VectorArg_Long_Intrinsic;
+  def int_aarch64_neon_fp8_cvtl2   : AdvSIMD_FP8_1VectorArg_Long_Intrinsic;
+
+  def int_aarch64_neon_fp8_fcvtn
+    : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
+                            [llvm_anyvector_ty,
+                             LLVMMatchType<1>],
+                            [IntrReadMem, IntrInaccessibleMemOnly]>;
+  def int_aarch64_neon_fp8_fcvtn2
+    : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
+                            [LLVMMatchType<0>,
+                             llvm_anyvector_ty,
+                             LLVMMatchType<1>],
+                            [IntrReadMem, IntrInaccessibleMemOnly]>;
+
+  // Dot-product
+  class AdvSIMD_FP8_DOT_Intrinsic
+    : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
+                            [LLVMMatchType<0>,
+                             llvm_anyvector_ty,
+                             LLVMMatchType<1>],
+                             [IntrReadMem, IntrInaccessibleMemOnly]>;
+  class AdvSIMD_FP8_DOT_LANE_Intrinsic
+    : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
+                            [LLVMMatchType<0>,
+                             llvm_anyvector_ty,
+                             llvm_v16i8_ty,
+                             llvm_i32_ty],
+                             [IntrReadMem, IntrInaccessibleMemOnly, ImmArg<ArgIndex<3>>]>;
+
+  def int_aarch64_neon_fp8_fdot2 : AdvSIMD_FP8_DOT_Intrinsic;
+  def int_aarch64_neon_fp8_fdot2_lane : AdvSIMD_FP8_DOT_LANE_Intrinsic;
+
+  def int_aarch64_neon_fp8_fdot4 : AdvSIMD_FP8_DOT_Intrinsic;
+  def int_aarch64_neon_fp8_fdot4_lane : AdvSIMD_FP8_DOT_LANE_Intrinsic;
+
+
+// Fused multiply-add
+  class AdvSIMD_FP8_FMLA_Intrinsic
+    : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
+                            [LLVMMatchType<0>,
+                             llvm_v16i8_ty,
+                             llvm_v16i8_ty],
+                             [IntrReadMem, IntrInaccessibleMemOnly]>;
+
+  class AdvSIMD_FP8_FMLA_LANE_Intrinsic
+    : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
+                            [LLVMMatchType<0>,
+                             llvm_v16i8_ty,
+                             llvm_v16i8_ty,
+                             llvm_i32_ty],
+                             [IntrReadMem, IntrInaccessibleMemOnly, ImmArg<ArgIndex<3>>]>;
+
+  def int_aarch64_neon_fp8_fmlalb : AdvSIMD_FP8_FMLA_Intrinsic;
+  def int_aarch64_neon_fp8_fmlalt : AdvSIMD_FP8_FMLA_Intrinsic;
+
+  def int_aarch64_neon_fp8_fmlallbb : AdvSIMD_FP8_FMLA_Intrinsic;
+  def int_aarch64_neon_fp8_fmlallbt : AdvSIMD_FP8_FMLA_Intrinsic;
+  def int_aarch64_neon_fp8_fmlalltb : AdvSIMD_FP8_FMLA_Intrinsic;
+  def int_aarch64_neon_fp8_fmlalltt : AdvSIMD_FP8_FMLA_Intrinsic;
+
+  def int_aarch64_neon_fp8_fmlalb_lane : AdvSIMD_FP8_FMLA_LANE_Intrinsic;
+  def int_aarch64_neon_fp8_fmlalt_lane : AdvSIMD_FP8_FMLA_LANE_Intrinsic;
+
+  def int_aarch64_neon_fp8_fmlallbb_lane : AdvSIMD_FP8_FMLA_LANE_Intrinsic;
+  def int_aarch64_neon_fp8_fmlallbt_lane : AdvSIMD_FP8_FMLA_LANE_Intrinsic;
+  def int_aarch64_neon_fp8_fmlalltb_lane : AdvSIMD_FP8_FMLA_LANE_Intrinsic;
+  def int_aarch64_neon_fp8_fmlalltt_lane : AdvSIMD_FP8_FMLA_LANE_Intrinsic;
 }
 
 def llvm_nxv1i1_ty  : LLVMType<nxv1i1>;
diff --git a/llvm/include/llvm/IR/IntrinsicsAMDGPU.td b/llvm/include/llvm/IR/IntrinsicsAMDGPU.td
index cc3584833202b..eb7bde6999491 100644
--- a/llvm/include/llvm/IR/IntrinsicsAMDGPU.td
+++ b/llvm/include/llvm/IR/IntrinsicsAMDGPU.td
@@ -237,7 +237,7 @@ def int_amdgcn_reloc_constant : DefaultAttrsIntrinsic<
 // the second one is copied to m0
 def int_amdgcn_s_sendmsg : ClangBuiltin<"__builtin_amdgcn_s_sendmsg">,
   Intrinsic <[], [llvm_i32_ty, llvm_i32_ty],
-  [ImmArg<ArgIndex<0>>, IntrNoMem, IntrHasSideEffects]>;
+  [ImmArg<ArgIndex<0>>, IntrNoMem, IntrHasSideEffects, IntrWillReturn]>;
 def int_amdgcn_s_sendmsghalt : ClangBuiltin<"__builtin_amdgcn_s_sendmsghalt">,
   Intrinsic <[], [llvm_i32_ty, llvm_i32_ty],
   [ImmArg<ArgIndex<0>>, IntrNoMem, IntrHasSideEffects]>;
@@ -246,7 +246,7 @@ def int_amdgcn_s_sendmsghalt : ClangBuiltin<"__builtin_amdgcn_s_sendmsghalt">,
 // gfx11 intrinsic
 // The first parameter is s_sendmsg immediate (i16). Return type is i32 or i64.
 def int_amdgcn_s_sendmsg_rtn : Intrinsic <[llvm_anyint_ty], [llvm_i32_ty],
-  [ImmArg<ArgIndex<0>>, IntrNoMem, IntrHasSideEffects]>;
+  [ImmArg<ArgIndex<0>>, IntrNoMem, IntrHasSideEffects, IntrWillReturn]>;
 
 // Vanilla workgroup sync-barrier
 def int_amdgcn_s_barrier : ClangBuiltin<"__builtin_amdgcn_s_barrier">,
@@ -2923,6 +2923,12 @@ def int_amdgcn_s_prefetch_data :
     "", [SDNPMemOperand]
   >;
 
+// llvm.amdgcn.ds.bpermute.fi.b32 <index> <src>
+def int_amdgcn_ds_bpermute_fi_b32 :
+  ClangBuiltin<"__builtin_amdgcn_ds_bpermute_fi_b32">,
+  Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty],
+    [IntrNoMem, IntrConvergent, IntrWillReturn, IntrNoCallback, IntrNoFree]>;
+
 //===----------------------------------------------------------------------===//
 // Deep learning intrinsics.
 //===----------------------------------------------------------------------===//
diff --git a/llvm/include/llvm/IR/IntrinsicsDirectX.td b/llvm/include/llvm/IR/IntrinsicsDirectX.td
index f21948697c8a6..beed84b144cec 100644
--- a/llvm/include/llvm/IR/IntrinsicsDirectX.td
+++ b/llvm/include/llvm/IR/IntrinsicsDirectX.td
@@ -105,6 +105,8 @@ def int_dx_wave_active_countbits : DefaultAttrsIntrinsic<[llvm_i32_ty], [llvm_i1
 def int_dx_wave_all : DefaultAttrsIntrinsic<[llvm_i1_ty], [llvm_i1_ty], [IntrConvergent, IntrNoMem]>;
 def int_dx_wave_any : DefaultAttrsIntrinsic<[llvm_i1_ty], [llvm_i1_ty], [IntrConvergent, IntrNoMem]>;
 def int_dx_wave_getlaneindex : DefaultAttrsIntrinsic<[llvm_i32_ty], [], [IntrConvergent, IntrNoMem]>;
+def int_dx_wave_reduce_max : DefaultAttrsIntrinsic<[llvm_any_ty], [LLVMMatchType<0>], [IntrConvergent, IntrNoMem]>;
+def int_dx_wave_reduce_umax : DefaultAttrsIntrinsic<[llvm_anyint_ty], [LLVMMatchType<0>], [IntrConvergent, IntrNoMem]>;
 def int_dx_wave_reduce_sum : DefaultAttrsIntrinsic<[llvm_any_ty], [LLVMMatchType<0>], [IntrConvergent, IntrNoMem]>;
 def int_dx_wave_reduce_usum : DefaultAttrsIntrinsic<[llvm_anyint_ty], [LLVMMatchType<0>], [IntrConvergent, IntrNoMem]>;
 def int_dx_wave_is_first_lane : DefaultAttrsIntrinsic<[llvm_i1_ty], [], [IntrConvergent]>;
diff --git a/llvm/include/llvm/IR/IntrinsicsNVVM.td b/llvm/include/llvm/IR/IntrinsicsNVVM.td
index 68c2373a1a454..6af1f2a166773 100644
--- a/llvm/include/llvm/IR/IntrinsicsNVVM.td
+++ b/llvm/include/llvm/IR/IntrinsicsNVVM.td
@@ -48,6 +48,8 @@
 
 def llvm_global_ptr_ty  : LLVMQualPointerType<1>;  // (global)ptr
 def llvm_shared_ptr_ty  : LLVMQualPointerType<3>;  // (shared)ptr
+def llvm_local_ptr_ty   : LLVMQualPointerType<5>;  // (local)ptr
+def llvm_tmem_ptr_ty    : LLVMQualPointerType<6>;  // (tensor memory)ptr
 
 //
 // MISC
@@ -1444,10 +1446,18 @@ let TargetPrefix = "nvvm" in {
       Intrinsic<[llvm_i32_ty], [llvm_float_ty], [IntrNoMem, IntrNoCallback]>;
   def int_nvvm_f2tf32_rn_relu : ClangBuiltin<"__nvvm_f2tf32_rn_relu">,
       Intrinsic<[llvm_i32_ty], [llvm_float_ty], [IntrNoMem, IntrNoCallback]>;
+  def int_nvvm_f2tf32_rn_satfinite : ClangBuiltin<"__nvvm_f2tf32_rn_satfinite">,
+      Intrinsic<[llvm_i32_ty], [llvm_float_ty], [IntrNoMem, IntrNoCallback]>;
+  def int_nvvm_f2tf32_rn_relu_satfinite : ClangBuiltin<"__nvvm_f2tf32_rn_relu_satfinite">,
+      Intrinsic<[llvm_i32_ty], [llvm_float_ty], [IntrNoMem, IntrNoCallback]>;
   def int_nvvm_f2tf32_rz : ClangBuiltin<"__nvvm_f2tf32_rz">,
       Intrinsic<[llvm_i32_ty], [llvm_float_ty], [IntrNoMem, IntrNoCallback]>;
   def int_nvvm_f2tf32_rz_relu : ClangBuiltin<"__nvvm_f2tf32_rz_relu">,
       Intrinsic<[llvm_i32_ty], [llvm_float_ty], [IntrNoMem, IntrNoCallback]>;
+  def int_nvvm_f2tf32_rz_satfinite : ClangBuiltin<"__nvvm_f2tf32_rz_satfinite">,
+      Intrinsic<[llvm_i32_ty], [llvm_float_ty], [IntrNoMem, IntrNoCallback]>;
+  def int_nvvm_f2tf32_rz_relu_satfinite : ClangBuiltin<"__nvvm_f2tf32_rz_relu_satfinite">,
+      Intrinsic<[llvm_i32_ty], [llvm_float_ty], [IntrNoMem, IntrNoCallback]>;
 
   def int_nvvm_ff_to_e4m3x2_rn : ClangBuiltin<"__nvvm_ff_to_e4m3x2_rn">,
       Intrinsic<[llvm_i16_ty], [llvm_float_ty, llvm_float_ty], [IntrNoMem, IntrNoCallback]>;
@@ -4990,6 +5000,26 @@ foreach dim = [1, 2, 3, 4, 5] in {
   }
 }
 
+// Intrinsics for Prefetch and Prefetchu
+foreach level = ["L1", "L2"] in {
+   foreach addr = ["global", "local", ""] in {
+     foreach evict = !if(!eq(addr, "global"), ["evictlast", "evictnormal"], ["evictnormal"]) in {
+      defvar suffix = "" # !if(!eq(addr, ""), "", addr # "_") # level # "_" # evict;             
+      def int_nvvm_prefetch_ # suffix : Intrinsic<[], 
+                                        !cond(
+                                        !eq(addr, "global") : [llvm_global_ptr_ty],
+                                        !eq(addr, "local") : [llvm_local_ptr_ty],
+                                        !eq(addr, "") : [llvm_ptr_ty]),
+                                        [IntrArgMemOnly, ReadOnly<ArgIndex<0>>,
+                                        NoCapture<ArgIndex<0>>]>;   
+    }
+  }
+}
+
+def int_nvvm_prefetchu_L1_evictnormal : Intrinsic<[], [llvm_ptr_ty],
+  [IntrArgMemOnly, ReadOnly<ArgIndex<0>>, NoCapture<ArgIndex<0>>]>;
+
+
 // Intrinsics for Bulk Copy using TMA (non-tensor)
 // From Global to Shared Cluster
 def int_nvvm_cp_async_bulk_global_to_shared_cluster
@@ -5047,4 +5077,65 @@ def int_nvvm_cp_async_bulk_prefetch_L2
 def int_nvvm_griddepcontrol_launch_dependents: Intrinsic<[], [], [IntrNoMem, IntrHasSideEffects]>;
 def int_nvvm_griddepcontrol_wait: Intrinsic<[], [], [IntrNoMem, IntrHasSideEffects]>;
 
+//
+// Tcgen05 family of Intrinsics
+//
+
+// Tcgen05 alloc/dealloc related intrinsics
+
+foreach cta_group = ["cg1", "cg2"] in {
+  def int_nvvm_tcgen05_alloc_ # cta_group : Intrinsic<[],
+    [llvm_ptr_ty,        // dst_ptr
+     llvm_i32_ty] ,      // num_columns
+    [IntrConvergent, IntrInaccessibleMemOrArgMemOnly,
+     WriteOnly<ArgIndex<0>>, NoCapture<ArgIndex<0>>]>;
+
+  def int_nvvm_tcgen05_alloc_shared_ # cta_group : Intrinsic<[],
+    [llvm_shared_ptr_ty, // dst_ptr
+     llvm_i32_ty],       // num_columns
+    [IntrConvergent, IntrInaccessibleMemOrArgMemOnly,
+     WriteOnly<ArgIndex<0>>, NoCapture<ArgIndex<0>>]>;
+
+  def int_nvvm_tcgen05_dealloc_ # cta_group : Intrinsic<[],
+    [llvm_tmem_ptr_ty,   // tmem_addr
+     llvm_i32_ty],       // num_columns
+    [IntrConvergent, IntrArgMemOnly,
+     NoCapture<ArgIndex<0>>]>;
+
+  def int_nvvm_tcgen05_relinq_alloc_permit_ # cta_group : Intrinsic<[], [],
+    [IntrConvergent, IntrInaccessibleMemOnly]>;
+
+  def int_nvvm_tcgen05_commit_ # cta_group : Intrinsic<[],
+    [llvm_ptr_ty],        // mbar_ptr
+    [IntrConvergent, IntrInaccessibleMemOrArgMemOnly,
+     NoCapture<ArgIndex<0>>]>;
+
+  def int_nvvm_tcgen05_commit_shared_ # cta_group : Intrinsic<[],
+    [llvm_shared_ptr_ty], // mbar_ptr
+    [IntrConvergent, IntrInaccessibleMemOrArgMemOnly,
+     NoCapture<ArgIndex<0>>]>;
+
+  def int_nvvm_tcgen05_commit_mc_ # cta_group : Intrinsic<[],
+    [llvm_ptr_ty, llvm_i16_ty], // mbar_ptr, cta_mask
+    [IntrConvergent, IntrInaccessibleMemOrArgMemOnly,
+     NoCapture<ArgIndex<0>>]>;
+
+  def int_nvvm_tcgen05_commit_mc_shared_ # cta_group : Intrinsic<[],
+    [llvm_shared_ptr_ty, llvm_i16_ty], // mbar_ptr, cta_mask
+    [IntrConvergent, IntrInaccessibleMemOrArgMemOnly,
+     NoCapture<ArgIndex<0>>]>;
+}
+
+// Tcgen05 wait_ld/st intrinsics
+def int_nvvm_tcgen05_wait_ld : Intrinsic<[], [],
+  [IntrConvergent, IntrInaccessibleMemOnly]>;
+def int_nvvm_tcgen05_wait_st : Intrinsic<[], [],
+  [IntrConvergent, IntrInaccessibleMemOnly]>;
+
+// Tcgen05 Fence intrinsics
+def int_nvvm_tcgen05_fence_before_thread_sync : Intrinsic<[], [],
+  [IntrNoMem, IntrHasSideEffects]>;
+def int_nvvm_tcgen05_fence_after_thread_sync : Intrinsic<[], [],
+  [IntrNoMem, IntrHasSideEffects]>;
+
 } // let TargetPrefix = "nvvm"
diff --git a/llvm/include/llvm/IR/IntrinsicsSPIRV.td b/llvm/include/llvm/IR/IntrinsicsSPIRV.td
index be337dbccaf8a..38910ee263ee3 100644
--- a/llvm/include/llvm/IR/IntrinsicsSPIRV.td
+++ b/llvm/include/llvm/IR/IntrinsicsSPIRV.td
@@ -91,6 +91,8 @@ let TargetPrefix = "spv" in {
   def int_spv_wave_active_countbits : DefaultAttrsIntrinsic<[llvm_i32_ty], [llvm_i1_ty], [IntrConvergent, IntrNoMem]>;
   def int_spv_wave_all : DefaultAttrsIntrinsic<[llvm_i1_ty], [llvm_i1_ty], [IntrConvergent, IntrNoMem]>;
   def int_spv_wave_any : DefaultAttrsIntrinsic<[llvm_i1_ty], [llvm_i1_ty], [IntrConvergent, IntrNoMem]>;
+  def int_spv_wave_reduce_umax : DefaultAttrsIntrinsic<[llvm_any_ty], [LLVMMatchType<0>], [IntrConvergent, IntrNoMem]>;
+  def int_spv_wave_reduce_max : DefaultAttrsIntrinsic<[llvm_any_ty], [LLVMMatchType<0>], [IntrConvergent, IntrNoMem]>;
   def int_spv_wave_reduce_sum : DefaultAttrsIntrinsic<[llvm_any_ty], [LLVMMatchType<0>], [IntrConvergent, IntrNoMem]>;
   def int_spv_wave_is_first_lane : DefaultAttrsIntrinsic<[llvm_i1_ty], [], [IntrConvergent]>;
   def int_spv_wave_readlane : DefaultAttrsIntrinsic<[llvm_any_ty], [LLVMMatchType<0>, llvm_i32_ty], [IntrConvergent, IntrNoMem]>;
diff --git a/llvm/include/llvm/IR/MDBuilder.h b/llvm/include/llvm/IR/MDBuilder.h
index e02ec8f5a3d8b..ce4e1da656049 100644
--- a/llvm/include/llvm/IR/MDBuilder.h
+++ b/llvm/include/llvm/IR/MDBuilder.h
@@ -89,8 +89,8 @@ class MDBuilder {
   MDNode *createFunctionEntryCount(uint64_t Count, bool Synthetic,
                                    const DenseSet<GlobalValue::GUID> *Imports);
 
-  /// Return metadata containing the section prefix for a function.
-  MDNode *createFunctionSectionPrefix(StringRef Prefix);
+  /// Return metadata containing the section prefix for a global object.
+  MDNode *createGlobalObjectSectionPrefix(StringRef Prefix);
 
   /// Return metadata containing the pseudo probe descriptor for a function.
   MDNode *createPseudoProbeDesc(uint64_t GUID, uint64_t Hash, StringRef FName);
diff --git a/llvm/include/llvm/IR/RuntimeLibcalls.def b/llvm/include/llvm/IR/RuntimeLibcalls.def
index 8153845b52c7a..a7963543c4350 100644
--- a/llvm/include/llvm/IR/RuntimeLibcalls.def
+++ b/llvm/include/llvm/IR/RuntimeLibcalls.def
@@ -354,6 +354,16 @@ HANDLE_LIBCALL(FREXP_F64, "frexp")
 HANDLE_LIBCALL(FREXP_F80, "frexpl")
 HANDLE_LIBCALL(FREXP_F128, "frexpl")
 HANDLE_LIBCALL(FREXP_PPCF128, "frexpl")
+HANDLE_LIBCALL(SINCOSPI_F32, "sincospif")
+HANDLE_LIBCALL(SINCOSPI_F64, "sincospi")
+HANDLE_LIBCALL(SINCOSPI_F80, "sincospil")
+HANDLE_LIBCALL(SINCOSPI_F128, "sincospil")
+HANDLE_LIBCALL(SINCOSPI_PPCF128, "sincospil")
+HANDLE_LIBCALL(MODF_F32, "modff")
+HANDLE_LIBCALL(MODF_F64, "modf")
+HANDLE_LIBCALL(MODF_F80, "modfl")
+HANDLE_LIBCALL(MODF_F128, "modfl")
+HANDLE_LIBCALL(MODF_PPCF128, "modfl")
 
 // Floating point environment
 HANDLE_LIBCALL(FEGETENV, "fegetenv")
diff --git a/llvm/include/llvm/IR/Use.h b/llvm/include/llvm/IR/Use.h
index 64b86f3a4396a..a86b9c46c1f69 100644
--- a/llvm/include/llvm/IR/Use.h
+++ b/llvm/include/llvm/IR/Use.h
@@ -11,14 +11,6 @@
 /// instruction or some other User instance which refers to a Value.  The Use
 /// class keeps the "use list" of the referenced value up to date.
 ///
-/// Pointer tagging is used to efficiently find the User corresponding to a Use
-/// without having to store a User pointer in every Use. A User is preceded in
-/// memory by all the Uses corresponding to its operands, and the low bits of
-/// one of the fields (Prev) of the Use class are used to encode offsets to be
-/// able to find that User given a pointer to any Use. For details, see:
-///
-///   http://www.llvm.org/docs/ProgrammersManual.html#UserLayout
-///
 //===----------------------------------------------------------------------===//
 
 #ifndef LLVM_IR_USE_H
diff --git a/llvm/include/llvm/IR/Value.h b/llvm/include/llvm/IR/Value.h
index 011aedece94ab..cfed12e2f5f8d 100644
--- a/llvm/include/llvm/IR/Value.h
+++ b/llvm/include/llvm/IR/Value.h
@@ -710,6 +710,10 @@ class Value {
   /// For example, for a value \p ExternalAnalysis might try to calculate a
   /// lower bound. If \p ExternalAnalysis is successful, it should return true.
   ///
+  /// If \p LookThroughIntToPtr is true then this method also looks through
+  /// IntToPtr and PtrToInt constant expressions. The returned pointer may not
+  /// have the same provenance as this value.
+  ///
   /// If this is called on a non-pointer value, it returns 'this' and the
   /// \p Offset is not modified.
   ///
@@ -722,17 +726,19 @@ class Value {
       const DataLayout &DL, APInt &Offset, bool AllowNonInbounds,
       bool AllowInvariantGroup = false,
       function_ref<bool(Value &Value, APInt &Offset)> ExternalAnalysis =
-          nullptr) const;
+          nullptr,
+      bool LookThroughIntToPtr = false) const;
 
   Value *stripAndAccumulateConstantOffsets(
       const DataLayout &DL, APInt &Offset, bool AllowNonInbounds,
       bool AllowInvariantGroup = false,
       function_ref<bool(Value &Value, APInt &Offset)> ExternalAnalysis =
-          nullptr) {
+          nullptr,
+      bool LookThroughIntToPtr = false) {
     return const_cast<Value *>(
         static_cast<const Value *>(this)->stripAndAccumulateConstantOffsets(
-            DL, Offset, AllowNonInbounds, AllowInvariantGroup,
-            ExternalAnalysis));
+            DL, Offset, AllowNonInbounds, AllowInvariantGroup, ExternalAnalysis,
+            LookThroughIntToPtr));
   }
 
   /// This is a wrapper around stripAndAccumulateConstantOffsets with the
diff --git a/llvm/include/llvm/InitializePasses.h b/llvm/include/llvm/InitializePasses.h
index 8111afcc1fb20..6d74d7f24bf9a 100644
--- a/llvm/include/llvm/InitializePasses.h
+++ b/llvm/include/llvm/InitializePasses.h
@@ -192,7 +192,7 @@ void initializeMachineBranchProbabilityInfoWrapperPassPass(PassRegistry &);
 void initializeMachineCFGPrinterPass(PassRegistry &);
 void initializeMachineCSELegacyPass(PassRegistry &);
 void initializeMachineCombinerPass(PassRegistry &);
-void initializeMachineCopyPropagationPass(PassRegistry &);
+void initializeMachineCopyPropagationLegacyPass(PassRegistry &);
 void initializeMachineCycleInfoPrinterPassPass(PassRegistry &);
 void initializeMachineCycleInfoWrapperPassPass(PassRegistry &);
 void initializeMachineDominanceFrontierPass(PassRegistry &);
@@ -241,7 +241,7 @@ void initializePostInlineEntryExitInstrumenterPass(PassRegistry &);
 void initializePostMachineSchedulerPass(PassRegistry &);
 void initializePostRAHazardRecognizerPass(PassRegistry &);
 void initializePostRAMachineSinkingPass(PassRegistry &);
-void initializePostRASchedulerPass(PassRegistry &);
+void initializePostRASchedulerLegacyPass(PassRegistry &);
 void initializePreISelIntrinsicLoweringLegacyPassPass(PassRegistry &);
 void initializePrintFunctionPassWrapperPass(PassRegistry &);
 void initializePrintModulePassWrapperPass(PassRegistry &);
@@ -266,10 +266,10 @@ void initializeRegionOnlyPrinterPass(PassRegistry &);
 void initializeRegionOnlyViewerPass(PassRegistry &);
 void initializeRegionPrinterPass(PassRegistry &);
 void initializeRegionViewerPass(PassRegistry &);
-void initializeRegisterCoalescerPass(PassRegistry &);
+void initializeRegisterCoalescerLegacyPass(PassRegistry &);
 void initializeRemoveLoadsIntoFakeUsesPass(PassRegistry &);
 void initializeRemoveRedundantDebugValuesPass(PassRegistry &);
-void initializeRenameIndependentSubregsPass(PassRegistry &);
+void initializeRenameIndependentSubregsLegacyPass(PassRegistry &);
 void initializeReplaceWithVeclibLegacyPass(PassRegistry &);
 void initializeResetMachineFunctionPass(PassRegistry &);
 void initializeSCEVAAWrapperPassPass(PassRegistry &);
@@ -298,7 +298,7 @@ void initializeStackMapLivenessPass(PassRegistry &);
 void initializeStackProtectorPass(PassRegistry &);
 void initializeStackSafetyGlobalInfoWrapperPassPass(PassRegistry &);
 void initializeStackSafetyInfoWrapperPassPass(PassRegistry &);
-void initializeStackSlotColoringPass(PassRegistry &);
+void initializeStackSlotColoringLegacyPass(PassRegistry &);
 void initializeStraightLineStrengthReduceLegacyPassPass(PassRegistry &);
 void initializeStripDebugMachineModulePass(PassRegistry &);
 void initializeStructurizeCFGLegacyPassPass(PassRegistry &);
diff --git a/llvm/include/llvm/MC/DXContainerRootSignature.h b/llvm/include/llvm/MC/DXContainerRootSignature.h
new file mode 100644
index 0000000000000..e1a9be5fc52d8
--- /dev/null
+++ b/llvm/include/llvm/MC/DXContainerRootSignature.h
@@ -0,0 +1,28 @@
+//===- llvm/MC/DXContainerRootSignature.h - RootSignature -*- C++ -*- ========//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include <cstdint>
+#include <limits>
+
+namespace llvm {
+
+class raw_ostream;
+
+namespace mcdxbc {
+struct RootSignatureHeader {
+  uint32_t Version = 2;
+  uint32_t NumParameters = 0;
+  uint32_t RootParametersOffset = 0;
+  uint32_t NumStaticSamplers = 0;
+  uint32_t StaticSamplersOffset = 0;
+  uint32_t Flags = 0;
+
+  void write(raw_ostream &OS);
+};
+} // namespace mcdxbc
+} // namespace llvm
diff --git a/llvm/include/llvm/MC/MCELFStreamer.h b/llvm/include/llvm/MC/MCELFStreamer.h
index 5a1cdd9e96cad..9c86b4ecc6599 100644
--- a/llvm/include/llvm/MC/MCELFStreamer.h
+++ b/llvm/include/llvm/MC/MCELFStreamer.h
@@ -54,7 +54,6 @@ class MCELFStreamer : public MCObjectStreamer {
   void emitLabelAtPos(MCSymbol *Symbol, SMLoc Loc, MCDataFragment &F,
                       uint64_t Offset) override;
   void emitAssemblerFlag(MCAssemblerFlag Flag) override;
-  void emitThumbFunc(MCSymbol *Func) override;
   void emitWeakReference(MCSymbol *Alias, const MCSymbol *Symbol) override;
   bool emitSymbolAttribute(MCSymbol *Symbol, MCSymbolAttr Attribute) override;
   void emitSymbolDesc(MCSymbol *Symbol, unsigned DescValue) override;
diff --git a/llvm/include/llvm/MC/MCObjectStreamer.h b/llvm/include/llvm/MC/MCObjectStreamer.h
index aaa13be6b2986..2e3ef417ce89a 100644
--- a/llvm/include/llvm/MC/MCObjectStreamer.h
+++ b/llvm/include/llvm/MC/MCObjectStreamer.h
@@ -63,7 +63,6 @@ class MCObjectStreamer : public MCStreamer {
   virtual void emitInstToData(const MCInst &Inst, const MCSubtargetInfo&) = 0;
   void emitCFIStartProcImpl(MCDwarfFrameInfo &Frame) override;
   void emitCFIEndProcImpl(MCDwarfFrameInfo &Frame) override;
-  MCSymbol *emitCFILabel() override;
   void emitInstructionImpl(const MCInst &Inst, const MCSubtargetInfo &STI);
   void resolvePendingFixups();
 
@@ -81,6 +80,7 @@ class MCObjectStreamer : public MCStreamer {
   bool isIntegratedAssemblerRequired() const override { return true; }
 
   void emitFrames(MCAsmBackend *MAB);
+  MCSymbol *emitCFILabel() override;
   void emitCFISections(bool EH, bool Debug) override;
 
   void insert(MCFragment *F) {
diff --git a/llvm/include/llvm/MC/MCStreamer.h b/llvm/include/llvm/MC/MCStreamer.h
index 558b14cebfd3d..4fb2158f26e1e 100644
--- a/llvm/include/llvm/MC/MCStreamer.h
+++ b/llvm/include/llvm/MC/MCStreamer.h
@@ -168,6 +168,8 @@ class ARMTargetStreamer : public MCTargetStreamer {
 
   virtual void annotateTLSDescriptorSequence(const MCSymbolRefExpr *SRE);
 
+  // Note in the output that the specified \p Symbol is a Thumb mode function.
+  virtual void emitThumbFunc(MCSymbol *Symbol);
   virtual void emitThumbSet(MCSymbol *Symbol, const MCExpr *Value);
 
   void emitConstantPools() override;
@@ -252,6 +254,12 @@ class MCStreamer {
   bool AllowAutoPadding = false;
 
 protected:
+  // True if we are processing SEH directives in an epilogue.
+  bool InEpilogCFI = false;
+
+  // Symbol of the current epilog for which we are processing SEH directives.
+  MCSymbol *CurrentEpilog = nullptr;
+
   MCFragment *CurFrag = nullptr;
 
   MCStreamer(MCContext &Ctx);
@@ -333,6 +341,10 @@ class MCStreamer {
     return WinFrameInfos;
   }
 
+  MCSymbol *getCurrentEpilog() const { return CurrentEpilog; }
+
+  bool isInEpilogCFI() const { return InEpilogCFI; }
+
   void generateCompactUnwindEncodings(MCAsmBackend *MAB);
 
   /// \name Assembly File Formatting.
@@ -491,10 +503,6 @@ class MCStreamer {
                             const Triple *DarwinTargetVariantTriple,
                             const VersionTuple &DarwinTargetVariantSDKVersion);
 
-  /// Note in the output that the specified \p Func is a Thumb mode
-  /// function (ARM target only).
-  virtual void emitThumbFunc(MCSymbol *Func);
-
   /// Emit an assignment of \p Value to \p Symbol.
   ///
   /// This corresponds to an assembler statement such as:
@@ -1056,6 +1064,8 @@ class MCStreamer {
                                  SMLoc Loc = SMLoc());
   virtual void emitWinCFIPushFrame(bool Code, SMLoc Loc = SMLoc());
   virtual void emitWinCFIEndProlog(SMLoc Loc = SMLoc());
+  virtual void emitWinCFIBeginEpilogue(SMLoc Loc = SMLoc());
+  virtual void emitWinCFIEndEpilogue(SMLoc Loc = SMLoc());
   virtual void emitWinEHHandler(const MCSymbol *Sym, bool Unwind, bool Except,
                                 SMLoc Loc = SMLoc());
   virtual void emitWinEHHandlerData(SMLoc Loc = SMLoc());
diff --git a/llvm/include/llvm/MC/MCWasmStreamer.h b/llvm/include/llvm/MC/MCWasmStreamer.h
index 77b5625efdfe8..4dffd83b12d0d 100644
--- a/llvm/include/llvm/MC/MCWasmStreamer.h
+++ b/llvm/include/llvm/MC/MCWasmStreamer.h
@@ -45,7 +45,6 @@ class MCWasmStreamer : public MCObjectStreamer {
   void emitLabelAtPos(MCSymbol *Symbol, SMLoc Loc, MCDataFragment &F,
                       uint64_t Offset) override;
   void emitAssemblerFlag(MCAssemblerFlag Flag) override;
-  void emitThumbFunc(MCSymbol *Func) override;
   void emitWeakReference(MCSymbol *Alias, const MCSymbol *Symbol) override;
   bool emitSymbolAttribute(MCSymbol *Symbol, MCSymbolAttr Attribute) override;
   void emitSymbolDesc(MCSymbol *Symbol, unsigned DescValue) override;
diff --git a/llvm/include/llvm/MC/MCWinCOFFStreamer.h b/llvm/include/llvm/MC/MCWinCOFFStreamer.h
index 2425abe51e6dd..4e2a495480980 100644
--- a/llvm/include/llvm/MC/MCWinCOFFStreamer.h
+++ b/llvm/include/llvm/MC/MCWinCOFFStreamer.h
@@ -46,7 +46,6 @@ class MCWinCOFFStreamer : public MCObjectStreamer {
   void changeSection(MCSection *Section, uint32_t Subsection = 0) override;
   void emitLabel(MCSymbol *Symbol, SMLoc Loc = SMLoc()) override;
   void emitAssemblerFlag(MCAssemblerFlag Flag) override;
-  void emitThumbFunc(MCSymbol *Func) override;
   bool emitSymbolAttribute(MCSymbol *Symbol, MCSymbolAttr Attribute) override;
   void emitSymbolDesc(MCSymbol *Symbol, unsigned DescValue) override;
   void beginCOFFSymbolDef(MCSymbol const *Symbol) override;
diff --git a/llvm/include/llvm/Object/DXContainer.h b/llvm/include/llvm/Object/DXContainer.h
index 19c83ba6c6e85..c3a2f756bd683 100644
--- a/llvm/include/llvm/Object/DXContainer.h
+++ b/llvm/include/llvm/Object/DXContainer.h
@@ -116,6 +116,28 @@ template <typename T> struct ViewArray {
 };
 
 namespace DirectX {
+
+class RootSignature {
+private:
+  uint32_t Version;
+  uint32_t NumParameters;
+  uint32_t RootParametersOffset;
+  uint32_t NumStaticSamplers;
+  uint32_t StaticSamplersOffset;
+  uint32_t Flags;
+
+public:
+  RootSignature() {}
+
+  Error parse(StringRef Data);
+  uint32_t getVersion() const { return Version; }
+  uint32_t getNumParameters() const { return NumParameters; }
+  uint32_t getRootParametersOffset() const { return RootParametersOffset; }
+  uint32_t getNumStaticSamplers() const { return NumStaticSamplers; }
+  uint32_t getStaticSamplersOffset() const { return StaticSamplersOffset; }
+  uint32_t getFlags() const { return Flags; }
+};
+
 class PSVRuntimeInfo {
 
   using ResourceArray = ViewArray<dxbc::PSV::v2::ResourceBindInfo>;
@@ -287,6 +309,7 @@ class DXContainer {
   std::optional<uint64_t> ShaderFeatureFlags;
   std::optional<dxbc::ShaderHash> Hash;
   std::optional<DirectX::PSVRuntimeInfo> PSVInfo;
+  std::optional<DirectX::RootSignature> RootSignature;
   DirectX::Signature InputSignature;
   DirectX::Signature OutputSignature;
   DirectX::Signature PatchConstantSignature;
@@ -296,6 +319,7 @@ class DXContainer {
   Error parseDXILHeader(StringRef Part);
   Error parseShaderFeatureFlags(StringRef Part);
   Error parseHash(StringRef Part);
+  Error parseRootSignature(StringRef Part);
   Error parsePSVInfo(StringRef Part);
   Error parseSignature(StringRef Part, DirectX::Signature &Array);
   friend class PartIterator;
@@ -382,6 +406,10 @@ class DXContainer {
 
   std::optional<dxbc::ShaderHash> getShaderHash() const { return Hash; }
 
+  std::optional<DirectX::RootSignature> getRootSignature() const {
+    return RootSignature;
+  }
+
   const std::optional<DirectX::PSVRuntimeInfo> &getPSVInfo() const {
     return PSVInfo;
   };
diff --git a/llvm/include/llvm/Object/SymbolicFile.h b/llvm/include/llvm/Object/SymbolicFile.h
index b13588c147d9b..2c857e72c3e5a 100644
--- a/llvm/include/llvm/Object/SymbolicFile.h
+++ b/llvm/include/llvm/Object/SymbolicFile.h
@@ -71,7 +71,7 @@ template <class content_type> class content_iterator {
 
 public:
   using iterator_category = std::forward_iterator_tag;
-  using value_type = content_type;
+  using value_type = const content_type;
   using difference_type = std::ptrdiff_t;
   using pointer = value_type *;
   using reference = value_type &;
diff --git a/llvm/include/llvm/ObjectYAML/DXContainerYAML.h b/llvm/include/llvm/ObjectYAML/DXContainerYAML.h
index 66ad057ab0e30..0200f5cb196ff 100644
--- a/llvm/include/llvm/ObjectYAML/DXContainerYAML.h
+++ b/llvm/include/llvm/ObjectYAML/DXContainerYAML.h
@@ -17,6 +17,7 @@
 
 #include "llvm/ADT/StringRef.h"
 #include "llvm/BinaryFormat/DXContainer.h"
+#include "llvm/Object/DXContainer.h"
 #include "llvm/ObjectYAML/YAML.h"
 #include "llvm/Support/YAMLTraits.h"
 #include <array>
@@ -72,6 +73,22 @@ struct ShaderHash {
   std::vector<llvm::yaml::Hex8> Digest;
 };
 
+#define ROOT_ELEMENT_FLAG(Num, Val) bool Val = false;
+struct RootSignatureDesc {
+  RootSignatureDesc() = default;
+  RootSignatureDesc(const object::DirectX::RootSignature &Data);
+
+  uint32_t Version;
+  uint32_t NumParameters;
+  uint32_t RootParametersOffset;
+  uint32_t NumStaticSamplers;
+  uint32_t StaticSamplersOffset;
+
+  uint32_t getEncodedFlags();
+
+#include "llvm/BinaryFormat/DXContainerConstants.def"
+};
+
 using ResourceFlags = dxbc::PSV::ResourceFlags;
 using ResourceBindInfo = dxbc::PSV::v2::ResourceBindInfo;
 
@@ -159,6 +176,7 @@ struct Part {
   std::optional<ShaderHash> Hash;
   std::optional<PSVInfo> Info;
   std::optional<DXContainerYAML::Signature> Signature;
+  std::optional<DXContainerYAML::RootSignatureDesc> RootSignature;
 };
 
 struct Object {
@@ -241,6 +259,11 @@ template <> struct MappingTraits<DXContainerYAML::Signature> {
   static void mapping(IO &IO, llvm::DXContainerYAML::Signature &El);
 };
 
+template <> struct MappingTraits<DXContainerYAML::RootSignatureDesc> {
+  static void mapping(IO &IO,
+                      DXContainerYAML::RootSignatureDesc &RootSignature);
+};
+
 } // namespace yaml
 
 } // namespace llvm
diff --git a/llvm/include/llvm/Passes/CodeGenPassBuilder.h b/llvm/include/llvm/Passes/CodeGenPassBuilder.h
index a84164bed46ce..7f91dd7ebf49d 100644
--- a/llvm/include/llvm/Passes/CodeGenPassBuilder.h
+++ b/llvm/include/llvm/Passes/CodeGenPassBuilder.h
@@ -45,6 +45,7 @@
 #include "llvm/CodeGen/LowerEmuTLS.h"
 #include "llvm/CodeGen/MIRPrinter.h"
 #include "llvm/CodeGen/MachineCSE.h"
+#include "llvm/CodeGen/MachineCopyPropagation.h"
 #include "llvm/CodeGen/MachineFunctionAnalysis.h"
 #include "llvm/CodeGen/MachineLICM.h"
 #include "llvm/CodeGen/MachineModuleInfo.h"
@@ -53,11 +54,14 @@
 #include "llvm/CodeGen/OptimizePHIs.h"
 #include "llvm/CodeGen/PHIElimination.h"
 #include "llvm/CodeGen/PeepholeOptimizer.h"
+#include "llvm/CodeGen/PostRASchedulerList.h"
 #include "llvm/CodeGen/PreISelIntrinsicLowering.h"
 #include "llvm/CodeGen/RegAllocFast.h"
 #include "llvm/CodeGen/RegUsageInfoCollector.h"
 #include "llvm/CodeGen/RegUsageInfoPropagate.h"
+#include "llvm/CodeGen/RegisterCoalescerPass.h"
 #include "llvm/CodeGen/RegisterUsageInfo.h"
+#include "llvm/CodeGen/RenameIndependentSubregs.h"
 #include "llvm/CodeGen/ReplaceWithVeclib.h"
 #include "llvm/CodeGen/SafeStack.h"
 #include "llvm/CodeGen/SelectOptimize.h"
@@ -65,6 +69,7 @@
 #include "llvm/CodeGen/SjLjEHPrepare.h"
 #include "llvm/CodeGen/StackColoring.h"
 #include "llvm/CodeGen/StackProtector.h"
+#include "llvm/CodeGen/StackSlotColoring.h"
 #include "llvm/CodeGen/TailDuplication.h"
 #include "llvm/CodeGen/TargetPassConfig.h"
 #include "llvm/CodeGen/TwoAddressInstructionPass.h"
@@ -957,7 +962,7 @@ Error CodeGenPassBuilder<Derived, TargetMachineT>::addMachinePasses(
     if (Opt.MISchedPostRA)
       addPass(PostMachineSchedulerPass());
     else
-      addPass(PostRASchedulerPass());
+      addPass(PostRASchedulerPass(&TM));
   }
 
   // GC
diff --git a/llvm/include/llvm/Passes/DroppedVariableStats.h b/llvm/include/llvm/Passes/DroppedVariableStats.h
index c4de849ca7554..30fbeae703b03 100644
--- a/llvm/include/llvm/Passes/DroppedVariableStats.h
+++ b/llvm/include/llvm/Passes/DroppedVariableStats.h
@@ -96,7 +96,10 @@ class DroppedVariableStats {
     DenseSet<VarID> &DebugVariablesBeforeSet =
         DbgVariables.DebugVariablesBefore;
     DenseSet<VarID> &DebugVariablesAfterSet = DbgVariables.DebugVariablesAfter;
-    DenseMap<VarID, DILocation *> &InlinedAtsMap = InlinedAts.back()[FuncName];
+    auto It = InlinedAts.back().find(FuncName);
+    if (It == InlinedAts.back().end())
+      return;
+    DenseMap<VarID, DILocation *> &InlinedAtsMap = It->second;
     // Find an Instruction that shares the same scope as the dropped #dbg_value
     // or has a scope that is the child of the scope of the #dbg_value, and has
     // an inlinedAt equal to the inlinedAt of the #dbg_value or it's inlinedAt
diff --git a/llvm/include/llvm/Passes/MachinePassRegistry.def b/llvm/include/llvm/Passes/MachinePassRegistry.def
index dfe3514360c3c..9f9922dfa5673 100644
--- a/llvm/include/llvm/Passes/MachinePassRegistry.def
+++ b/llvm/include/llvm/Passes/MachinePassRegistry.def
@@ -140,12 +140,14 @@ MACHINE_FUNCTION_PASS("early-machinelicm", EarlyMachineLICMPass())
 MACHINE_FUNCTION_PASS("early-tailduplication", EarlyTailDuplicatePass())
 MACHINE_FUNCTION_PASS("finalize-isel", FinalizeISelPass())
 MACHINE_FUNCTION_PASS("localstackalloc", LocalStackSlotAllocationPass())
+MACHINE_FUNCTION_PASS("machine-cp", MachineCopyPropagationPass())
 MACHINE_FUNCTION_PASS("machine-cse", MachineCSEPass())
 MACHINE_FUNCTION_PASS("machinelicm", MachineLICMPass())
 MACHINE_FUNCTION_PASS("no-op-machine-function", NoOpMachineFunctionPass())
 MACHINE_FUNCTION_PASS("opt-phis", OptimizePHIsPass())
 MACHINE_FUNCTION_PASS("peephole-opt", PeepholeOptimizerPass())
 MACHINE_FUNCTION_PASS("phi-node-elimination", PHIEliminationPass())
+MACHINE_FUNCTION_PASS("post-RA-sched", PostRASchedulerPass(TM))
 MACHINE_FUNCTION_PASS("print", PrintMIRPass())
 MACHINE_FUNCTION_PASS("print<livedebugvars>", LiveDebugVariablesPrinterPass(errs()))
 MACHINE_FUNCTION_PASS("print<live-intervals>", LiveIntervalsPrinterPass(errs()))
@@ -164,9 +166,12 @@ MACHINE_FUNCTION_PASS("print<slot-indexes>", SlotIndexesPrinterPass(errs()))
 MACHINE_FUNCTION_PASS("print<virtregmap>", VirtRegMapPrinterPass(errs()))
 MACHINE_FUNCTION_PASS("reg-usage-collector", RegUsageInfoCollectorPass())
 MACHINE_FUNCTION_PASS("reg-usage-propagation", RegUsageInfoPropagationPass())
+MACHINE_FUNCTION_PASS("register-coalescer", RegisterCoalescerPass())
+MACHINE_FUNCTION_PASS("rename-independent-subregs", RenameIndependentSubregsPass())
 MACHINE_FUNCTION_PASS("require-all-machine-function-properties",
                       RequireAllMachineFunctionPropertiesPass())
 MACHINE_FUNCTION_PASS("stack-coloring", StackColoringPass())
+MACHINE_FUNCTION_PASS("stack-slot-coloring", StackSlotColoringPass())
 MACHINE_FUNCTION_PASS("tailduplication", TailDuplicatePass())
 MACHINE_FUNCTION_PASS("trigger-verifier-error", TriggerVerifierErrorPass())
 MACHINE_FUNCTION_PASS("two-address-instruction", TwoAddressInstructionPass())
@@ -234,7 +239,6 @@ DUMMY_MACHINE_FUNCTION_PASS("legalizer", LegalizerPass)
 DUMMY_MACHINE_FUNCTION_PASS("livedebugvalues", LiveDebugValuesPass)
 DUMMY_MACHINE_FUNCTION_PASS("lrshrink", LiveRangeShrinkPass)
 DUMMY_MACHINE_FUNCTION_PASS("machine-combiner", MachineCombinerPass)
-DUMMY_MACHINE_FUNCTION_PASS("machine-cp", MachineCopyPropagationPass)
 DUMMY_MACHINE_FUNCTION_PASS("static-data-splitter", StaticDataSplitter)
 DUMMY_MACHINE_FUNCTION_PASS("machine-function-splitter", MachineFunctionSplitterPass)
 DUMMY_MACHINE_FUNCTION_PASS("machine-latecleanup", MachineLateInstrsCleanupPass)
@@ -245,7 +249,6 @@ DUMMY_MACHINE_FUNCTION_PASS("machine-uniformity", MachineUniformityInfoWrapperPa
 DUMMY_MACHINE_FUNCTION_PASS("machineinstr-printer", MachineFunctionPrinterPass)
 DUMMY_MACHINE_FUNCTION_PASS("mirfs-discriminators", MIRAddFSDiscriminatorsPass)
 DUMMY_MACHINE_FUNCTION_PASS("patchable-function", PatchableFunctionPass)
-DUMMY_MACHINE_FUNCTION_PASS("post-RA-sched", PostRASchedulerPass)
 DUMMY_MACHINE_FUNCTION_PASS("postmisched", PostMachineSchedulerPass)
 DUMMY_MACHINE_FUNCTION_PASS("postra-machine-sink", PostRAMachineSinkingPass)
 DUMMY_MACHINE_FUNCTION_PASS("postrapseudos", ExpandPostRAPseudosPass)
@@ -262,12 +265,9 @@ DUMMY_MACHINE_FUNCTION_PASS("regallocscoringpass", RegAllocScoringPass)
 DUMMY_MACHINE_FUNCTION_PASS("regbankselect", RegBankSelectPass)
 DUMMY_MACHINE_FUNCTION_PASS("remove-loads-into-fake-uses", RemoveLoadsIntoFakeUsesPass)
 DUMMY_MACHINE_FUNCTION_PASS("removeredundantdebugvalues", RemoveRedundantDebugValuesPass)
-DUMMY_MACHINE_FUNCTION_PASS("rename-independent-subregs", RenameIndependentSubregsPass)
 DUMMY_MACHINE_FUNCTION_PASS("reset-machine-function", ResetMachineFunctionPass)
 DUMMY_MACHINE_FUNCTION_PASS("shrink-wrap", ShrinkWrapPass)
-DUMMY_MACHINE_FUNCTION_PASS("simple-register-coalescing", RegisterCoalescerPass)
 DUMMY_MACHINE_FUNCTION_PASS("stack-frame-layout", StackFrameLayoutAnalysisPass)
-DUMMY_MACHINE_FUNCTION_PASS("stack-slot-coloring", StackSlotColoringPass)
 DUMMY_MACHINE_FUNCTION_PASS("stackmap-liveness", StackMapLivenessPass)
 DUMMY_MACHINE_FUNCTION_PASS("unpack-mi-bundles", UnpackMachineBundlesPass)
 DUMMY_MACHINE_FUNCTION_PASS("virtregrewriter", VirtRegRewriterPass)
diff --git a/llvm/include/llvm/Passes/PassBuilder.h b/llvm/include/llvm/Passes/PassBuilder.h
index e7bc3a58f414f..51ccaa53447d7 100644
--- a/llvm/include/llvm/Passes/PassBuilder.h
+++ b/llvm/include/llvm/Passes/PassBuilder.h
@@ -60,6 +60,10 @@ class PipelineTuningOptions {
   /// Tuning option to enable/disable loop unrolling. Its default value is true.
   bool LoopUnrolling;
 
+  /// Tuning option to enable/disable loop interchange. Its default value is
+  /// false.
+  bool LoopInterchange;
+
   /// Tuning option to forget all SCEV loops in LoopUnroll. Its default value
   /// is that of the flag: `-forget-scev-loop-unroll`.
   bool ForgetAllSCEVInLoopUnroll;
@@ -465,6 +469,17 @@ class PassBuilder {
     VectorizerStartEPCallbacks.push_back(C);
   }
 
+  /// Register a callback for a default optimizer pipeline extension
+  /// point
+  ///
+  /// This extension point allows adding optimization passes after the
+  /// vectorizer and other highly target specific optimization passes are
+  /// executed.
+  void registerVectorizerEndEPCallback(
+      const std::function<void(FunctionPassManager &, OptimizationLevel)> &C) {
+    VectorizerEndEPCallbacks.push_back(C);
+  }
+
   /// Register a callback for a default optimizer pipeline extension point.
   ///
   /// This extension point allows adding optimization once at the start of the
@@ -631,6 +646,8 @@ class PassBuilder {
                                            OptimizationLevel Level);
   void invokeVectorizerStartEPCallbacks(FunctionPassManager &FPM,
                                         OptimizationLevel Level);
+  void invokeVectorizerEndEPCallbacks(FunctionPassManager &FPM,
+                                      OptimizationLevel Level);
   void invokeOptimizerEarlyEPCallbacks(ModulePassManager &MPM,
                                        OptimizationLevel Level,
                                        ThinOrFullLTOPhase Phase);
@@ -759,6 +776,8 @@ class PassBuilder {
       CGSCCOptimizerLateEPCallbacks;
   SmallVector<std::function<void(FunctionPassManager &, OptimizationLevel)>, 2>
       VectorizerStartEPCallbacks;
+  SmallVector<std::function<void(FunctionPassManager &, OptimizationLevel)>, 2>
+      VectorizerEndEPCallbacks;
   // Module callbacks
   SmallVector<std::function<void(ModulePassManager &, OptimizationLevel,
                                  ThinOrFullLTOPhase)>,
diff --git a/llvm/include/llvm/ProfileData/Coverage/CoverageMapping.h b/llvm/include/llvm/ProfileData/Coverage/CoverageMapping.h
index 5a20a9ef63287..4fc0133443192 100644
--- a/llvm/include/llvm/ProfileData/Coverage/CoverageMapping.h
+++ b/llvm/include/llvm/ProfileData/Coverage/CoverageMapping.h
@@ -541,9 +541,9 @@ struct MCDCRecord {
 
   std::string getConditionHeaderString(unsigned Condition) {
     std::ostringstream OS;
-    OS << "Condition C" << Condition + 1 << " --> (";
-    OS << CondLoc[Condition].first << ":" << CondLoc[Condition].second;
-    OS << ")\n";
+    const auto &[Line, Col] = CondLoc[Condition];
+    OS << "Condition C" << Condition + 1 << " --> (" << Line << ":" << Col
+       << ")\n";
     return OS.str();
   }
 
diff --git a/llvm/include/llvm/SandboxIR/Context.h b/llvm/include/llvm/SandboxIR/Context.h
index 7fe97d984b958..a88b0003f55bd 100644
--- a/llvm/include/llvm/SandboxIR/Context.h
+++ b/llvm/include/llvm/SandboxIR/Context.h
@@ -218,6 +218,8 @@ class Context {
 public:
   Context(LLVMContext &LLVMCtx);
   ~Context();
+  /// Clears function-level state.
+  void clear();
 
   Tracker &getTracker() { return IRTracker; }
   /// Convenience function for `getTracker().save()`
diff --git a/llvm/include/llvm/SandboxIR/Region.h b/llvm/include/llvm/SandboxIR/Region.h
index 8133e01734ea7..c1195141cb54c 100644
--- a/llvm/include/llvm/SandboxIR/Region.h
+++ b/llvm/include/llvm/SandboxIR/Region.h
@@ -6,18 +6,55 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef LLVM_TRANSFORMS_VECTORIZE_SANDBOXVECTORIZER_REGION_H
-#define LLVM_TRANSFORMS_VECTORIZE_SANDBOXVECTORIZER_REGION_H
+#ifndef LLVM_SANDBOXIR_REGION_H
+#define LLVM_SANDBOXIR_REGION_H
 
 #include <memory>
 
 #include "llvm/ADT/SetVector.h"
 #include "llvm/ADT/iterator_range.h"
+#include "llvm/Analysis/TargetTransformInfo.h"
 #include "llvm/SandboxIR/Instruction.h"
 #include "llvm/Support/raw_ostream.h"
 
 namespace llvm::sandboxir {
 
+class Region;
+
+class ScoreBoard {
+  const Region &Rgn;
+  TargetTransformInfo &TTI;
+  constexpr static TTI::TargetCostKind CostKind = TTI::TCK_RecipThroughput;
+  /// The cost of all instructions added to the region.
+  InstructionCost AfterCost = 0;
+  /// The cost of all instructions that got removed and replaced by new ones.
+  InstructionCost BeforeCost = 0;
+  /// Helper for both add() and remove(). \Returns the TTI cost of \p I.
+  InstructionCost getCost(Instruction *I) const;
+  /// No need to allow copies.
+  ScoreBoard(const ScoreBoard &) = delete;
+  const ScoreBoard &operator=(const ScoreBoard &) = delete;
+
+public:
+  ScoreBoard(Region &Rgn, TargetTransformInfo &TTI) : Rgn(Rgn), TTI(TTI) {}
+  /// Mark \p I as a newly added instruction to the region.
+  void add(Instruction *I) { AfterCost += getCost(I); }
+  /// Mark \p I as a deleted instruction from the region.
+  void remove(Instruction *I);
+  /// \Returns the cost of the newly added instructions.
+  InstructionCost getAfterCost() const { return AfterCost; }
+  /// \Returns the cost of the Removed instructions.
+  InstructionCost getBeforeCost() const { return BeforeCost; }
+
+#ifndef NDEBUG
+  void dump(raw_ostream &OS) const {
+    OS << "BeforeCost: " << BeforeCost << "\n";
+    OS << "AfterCost:  " << AfterCost << "\n";
+  }
+  LLVM_DUMP_METHOD void dump() const;
+#endif // NDEBUG
+};
+
 /// The main job of the Region is to point to new instructions generated by
 /// vectorization passes. It is the unit that RegionPasses operate on with their
 /// runOnRegion() function.
@@ -62,6 +99,8 @@ class Region {
   static constexpr const char *RegionStr = "sandboxregion";
 
   Context &Ctx;
+  /// Keeps track of cost of instructions added and removed.
+  ScoreBoard Scoreboard;
 
   /// ID (for later deregistration) of the "create instruction" callback.
   Context::CallbackID CreateInstCB;
@@ -72,7 +111,7 @@ class Region {
   // TODO: Add a way to encode/decode region info to/from metadata.
 
 public:
-  Region(Context &Ctx);
+  Region(Context &Ctx, TargetTransformInfo &TTI);
   ~Region();
 
   Context &getContext() const { return Ctx; }
@@ -91,7 +130,10 @@ class Region {
   iterator end() { return Insts.end(); }
   iterator_range<iterator> insts() { return make_range(begin(), end()); }
 
-  static SmallVector<std::unique_ptr<Region>> createRegionsFromMD(Function &F);
+  static SmallVector<std::unique_ptr<Region>>
+  createRegionsFromMD(Function &F, TargetTransformInfo &TTI);
+  /// \Returns the ScoreBoard data structure that keeps track of instr costs.
+  const ScoreBoard &getScoreboard() const { return Scoreboard; }
 
 #ifndef NDEBUG
   /// This is an expensive check, meant for testing.
@@ -109,4 +151,4 @@ class Region {
 
 } // namespace llvm::sandboxir
 
-#endif // LLVM_TRANSFORMS_VECTORIZE_SANDBOXVECTORIZER_REGION_H
+#endif // LLVM_SANDBOXIR_REGION_H
diff --git a/llvm/include/llvm/SandboxIR/Tracker.h b/llvm/include/llvm/SandboxIR/Tracker.h
index 9a031f3270837..f7b469965eae8 100644
--- a/llvm/include/llvm/SandboxIR/Tracker.h
+++ b/llvm/include/llvm/SandboxIR/Tracker.h
@@ -440,8 +440,9 @@ class ShuffleVectorSetMask final : public IRChangeBase {
 class Tracker {
 public:
   enum class TrackerState {
-    Disabled, ///> Tracking is disabled
-    Record,   ///> Tracking changes
+    Disabled,  ///> Tracking is disabled
+    Record,    ///> Tracking changes
+    Reverting, ///> Reverting changes
   };
 
 private:
@@ -473,6 +474,8 @@ class Tracker {
 
   ~Tracker();
   Context &getContext() const { return Ctx; }
+  /// \Returns true if there are no changes tracked.
+  bool empty() const { return Changes.empty(); }
   /// Record \p Change and take ownership. This is the main function used to
   /// track Sandbox IR changes.
   void track(std::unique_ptr<IRChangeBase> &&Change) {
diff --git a/llvm/include/llvm/SandboxIR/Utils.h b/llvm/include/llvm/SandboxIR/Utils.h
index d58fe52214395..5c6f0d9edd618 100644
--- a/llvm/include/llvm/SandboxIR/Utils.h
+++ b/llvm/include/llvm/SandboxIR/Utils.h
@@ -17,6 +17,8 @@
 #include "llvm/Analysis/MemoryLocation.h"
 #include "llvm/Analysis/ScalarEvolution.h"
 #include "llvm/Analysis/ValueTracking.h"
+#include "llvm/IR/Verifier.h"
+#include "llvm/SandboxIR/Function.h"
 #include "llvm/SandboxIR/Instruction.h"
 #include <optional>
 
@@ -122,6 +124,13 @@ class Utils {
                              const std::optional<MemoryLocation> &OptLoc) {
     return BatchAA.getModRefInfo(cast<llvm::Instruction>(I->Val), OptLoc);
   }
+
+  /// Equivalent to llvm::verifyFunction().
+  /// \Returns true if the IR is broken.
+  static bool verifyFunction(const Function *F, raw_ostream &OS) {
+    const auto &LLVMF = *cast<llvm::Function>(F->Val);
+    return llvm::verifyFunction(LLVMF, &OS);
+  }
 };
 
 } // namespace llvm::sandboxir
diff --git a/llvm/include/llvm/SandboxIR/Value.h b/llvm/include/llvm/SandboxIR/Value.h
index 243195f4c1c4b..28e33ca0f2312 100644
--- a/llvm/include/llvm/SandboxIR/Value.h
+++ b/llvm/include/llvm/SandboxIR/Value.h
@@ -167,6 +167,7 @@ class Value {
   // Region needs to manipulate metadata in the underlying LLVM Value, we don't
   // expose metadata in sandboxir.
   friend class Region;
+  friend class ScoreBoard; // Needs access to `Val` for the instruction cost.
 
   /// All values point to the context.
   Context &Ctx;
diff --git a/llvm/include/llvm/Support/AArch64BuildAttributes.h b/llvm/include/llvm/Support/AArch64BuildAttributes.h
index ea293b72f9bb1..2479992cf8e79 100644
--- a/llvm/include/llvm/Support/AArch64BuildAttributes.h
+++ b/llvm/include/llvm/Support/AArch64BuildAttributes.h
@@ -22,7 +22,7 @@
 
 namespace llvm {
 
-namespace AArch64BuildAttributes {
+namespace AArch64BuildAttrs {
 
 /// AArch64 build attributes vendors IDs (a.k.a subsection name)
 enum VendorID : unsigned {
@@ -69,7 +69,7 @@ enum FeatureAndBitsFlag : unsigned {
   Feature_PAC_Flag = 1 << 1,
   Feature_GCS_Flag = 1 << 2
 };
-} // namespace AArch64BuildAttributes
+} // namespace AArch64BuildAttrs
 } // namespace llvm
 
 #endif // LLVM_SUPPORT_AARCH64BUILDATTRIBUTES_H
\ No newline at end of file
diff --git a/llvm/include/llvm/Support/Error.h b/llvm/include/llvm/Support/Error.h
index 90120156ec2ea..c1b809a09bb80 100644
--- a/llvm/include/llvm/Support/Error.h
+++ b/llvm/include/llvm/Support/Error.h
@@ -1404,6 +1404,23 @@ inline Error createFileError(const Twine &F, size_t Line, std::error_code EC) {
   return createFileError(F, Line, errorCodeToError(EC));
 }
 
+/// Create a StringError with the specified error code and prepend the file path
+/// to it.
+inline Error createFileError(const Twine &F, std::error_code EC,
+                             const Twine &S) {
+  Error E = createStringError(EC, S);
+  return createFileError(F, std::move(E));
+}
+
+/// Create a StringError with the specified error code and prepend the file path
+/// to it.
+template <typename... Ts>
+inline Error createFileError(const Twine &F, std::error_code EC,
+                             char const *Fmt, const Ts &...Vals) {
+  Error E = createStringError(EC, Fmt, Vals...);
+  return createFileError(F, std::move(E));
+}
+
 Error createFileError(const Twine &F, ErrorSuccess) = delete;
 
 /// Helper for check-and-exit error handling.
diff --git a/llvm/include/llvm/Support/GlobPattern.h b/llvm/include/llvm/Support/GlobPattern.h
index eff8957d1b682..5e5e5c869a71f 100644
--- a/llvm/include/llvm/Support/GlobPattern.h
+++ b/llvm/include/llvm/Support/GlobPattern.h
@@ -22,31 +22,32 @@
 namespace llvm {
 
 /// This class implements a glob pattern matcher similar to the one found in
-/// bash, but with some key differences. Namely, that \p "*" matches all
+/// bash, but with some key differences. Namely, that `*` matches all
 /// characters and does not exclude path separators.
 ///
-/// * \p "?" matches a single character.
-/// * \p "*" matches zero or more characters.
-/// * \p "[<chars>]" matches one character in the bracket. Character ranges,
-///   e.g., \p "[a-z]", and negative sets via \p "[^ab]" or \p "[!ab]" are also
+/// * `?` matches a single character.
+/// * `*` matches zero or more characters.
+/// * `[<chars>]` matches one character in the bracket. Character ranges,
+///   e.g., `[a-z]`, and negative sets via `[^ab]` or `[!ab]` are also
 ///   supported.
-/// * \p "{<glob>,...}" matches one of the globs in the list. Nested brace
+/// * `{<glob>,...}` matches one of the globs in the list. Nested brace
 ///   expansions are not supported. If \p MaxSubPatterns is empty then
-///   brace expansions are not supported and characters \p "{,}" are treated as
+///   brace expansions are not supported and characters `{,}` are treated as
 ///   literals.
-/// * \p "\\" (a single backslash) escapes the next character so it is treated
-///   as a literal.
+/// * `\` escapes the next character so it is treated as a literal.
 ///
 /// Some known edge cases are:
-/// * \p "]" is allowed as the first character in a character class, i.e.,
-///   \p "[]]" is valid and matches the literal \p "]".
-/// * The empty character class, i.e., \p "[]", is invalid.
-/// * Empty or singleton brace expansions, e.g., \p "{}", \p "{a}", are invalid.
-/// * \p "}" and \p "," that are not inside a brace expansion are taken as
-///   literals, e.g., \p ",}" is valid but \p "{" is not.
+/// * The literal `]` is allowed as the first character in a character class,
+///    i.e., `[]]` is valid and matches the literal `]`.
+/// * The empty character class, i.e., `[]`, is invalid.
+/// * Empty or singleton brace expansions, e.g., `{}`, `{a}`, are invalid.
+/// * The literals `}` and `,` that are not inside a brace expansion are taken
+///   as literals, e.g., `,}` is valid but `{` is not.
 ///
-/// For example, \p "*[/\\\\]foo.{c,cpp}" (with two backslashes) will match
-/// (unix or windows) paths to all files named \p "foo.c" or \p "foo.cpp".
+/// Examples:
+/// * `*[/\\]foo.{c,cpp}` will match (unix or windows) paths to files named
+///   `foo.c` or `foo.cpp`.
+/// * `_Z{N,NK,}S[tabsoid]*` will match mangled C++ standard library functions.
 class GlobPattern {
 public:
   /// \param Pat the pattern to match against
diff --git a/llvm/include/llvm/Support/MathExtras.h b/llvm/include/llvm/Support/MathExtras.h
index 574e9a6116603..5a6f51adc07f3 100644
--- a/llvm/include/llvm/Support/MathExtras.h
+++ b/llvm/include/llvm/Support/MathExtras.h
@@ -43,38 +43,37 @@ using common_sint =
 /// Mathematical constants.
 namespace numbers {
 // TODO: Track C++20 std::numbers.
-// TODO: Favor using the hexadecimal FP constants (requires C++17).
 // clang-format off
-constexpr double e          = 2.7182818284590452354, // (0x1.5bf0a8b145769P+1) https://oeis.org/A001113
-                 egamma     = .57721566490153286061, // (0x1.2788cfc6fb619P-1) https://oeis.org/A001620
-                 ln2        = .69314718055994530942, // (0x1.62e42fefa39efP-1) https://oeis.org/A002162
-                 ln10       = 2.3025850929940456840, // (0x1.26bb1bbb55516P+1) https://oeis.org/A002392
-                 log2e      = 1.4426950408889634074, // (0x1.71547652b82feP+0)
-                 log10e     = .43429448190325182765, // (0x1.bcb7b1526e50eP-2)
-                 pi         = 3.1415926535897932385, // (0x1.921fb54442d18P+1) https://oeis.org/A000796
-                 inv_pi     = .31830988618379067154, // (0x1.45f306dc9c883P-2) https://oeis.org/A049541
-                 sqrtpi     = 1.7724538509055160273, // (0x1.c5bf891b4ef6bP+0) https://oeis.org/A002161
-                 inv_sqrtpi = .56418958354775628695, // (0x1.20dd750429b6dP-1) https://oeis.org/A087197
-                 sqrt2      = 1.4142135623730950488, // (0x1.6a09e667f3bcdP+0) https://oeis.org/A00219
-                 inv_sqrt2  = .70710678118654752440, // (0x1.6a09e667f3bcdP-1)
-                 sqrt3      = 1.7320508075688772935, // (0x1.bb67ae8584caaP+0) https://oeis.org/A002194
-                 inv_sqrt3  = .57735026918962576451, // (0x1.279a74590331cP-1)
-                 phi        = 1.6180339887498948482; // (0x1.9e3779b97f4a8P+0) https://oeis.org/A001622
-constexpr float ef          = 2.71828183F, // (0x1.5bf0a8P+1) https://oeis.org/A001113
-                egammaf     = .577215665F, // (0x1.2788d0P-1) https://oeis.org/A001620
-                ln2f        = .693147181F, // (0x1.62e430P-1) https://oeis.org/A002162
-                ln10f       = 2.30258509F, // (0x1.26bb1cP+1) https://oeis.org/A002392
-                log2ef      = 1.44269504F, // (0x1.715476P+0)
-                log10ef     = .434294482F, // (0x1.bcb7b2P-2)
-                pif         = 3.14159265F, // (0x1.921fb6P+1) https://oeis.org/A000796
-                inv_pif     = .318309886F, // (0x1.45f306P-2) https://oeis.org/A049541
-                sqrtpif     = 1.77245385F, // (0x1.c5bf8aP+0) https://oeis.org/A002161
-                inv_sqrtpif = .564189584F, // (0x1.20dd76P-1) https://oeis.org/A087197
-                sqrt2f      = 1.41421356F, // (0x1.6a09e6P+0) https://oeis.org/A002193
-                inv_sqrt2f  = .707106781F, // (0x1.6a09e6P-1)
-                sqrt3f      = 1.73205081F, // (0x1.bb67aeP+0) https://oeis.org/A002194
-                inv_sqrt3f  = .577350269F, // (0x1.279a74P-1)
-                phif        = 1.61803399F; // (0x1.9e377aP+0) https://oeis.org/A001622
+constexpr double e          = 0x1.5bf0a8b145769P+1, // (2.7182818284590452354) https://oeis.org/A001113
+                 egamma     = 0x1.2788cfc6fb619P-1, // (.57721566490153286061) https://oeis.org/A001620
+                 ln2        = 0x1.62e42fefa39efP-1, // (.69314718055994530942) https://oeis.org/A002162
+                 ln10       = 0x1.26bb1bbb55516P+1, // (2.3025850929940456840) https://oeis.org/A002392
+                 log2e      = 0x1.71547652b82feP+0, // (1.4426950408889634074)
+                 log10e     = 0x1.bcb7b1526e50eP-2, // (.43429448190325182765)
+                 pi         = 0x1.921fb54442d18P+1, // (3.1415926535897932385) https://oeis.org/A000796
+                 inv_pi     = 0x1.45f306dc9c883P-2, // (.31830988618379067154) https://oeis.org/A049541
+                 sqrtpi     = 0x1.c5bf891b4ef6bP+0, // (1.7724538509055160273) https://oeis.org/A002161
+                 inv_sqrtpi = 0x1.20dd750429b6dP-1, // (.56418958354775628695) https://oeis.org/A087197
+                 sqrt2      = 0x1.6a09e667f3bcdP+0, // (1.4142135623730950488) https://oeis.org/A00219
+                 inv_sqrt2  = 0x1.6a09e667f3bcdP-1, // (.70710678118654752440)
+                 sqrt3      = 0x1.bb67ae8584caaP+0, // (1.7320508075688772935) https://oeis.org/A002194
+                 inv_sqrt3  = 0x1.279a74590331cP-1, // (.57735026918962576451)
+                 phi        = 0x1.9e3779b97f4a8P+0; // (1.6180339887498948482) https://oeis.org/A001622
+constexpr float ef          = 0x1.5bf0a8P+1F, // (2.71828183) https://oeis.org/A001113
+                egammaf     = 0x1.2788d0P-1F, // (.577215665) https://oeis.org/A001620
+                ln2f        = 0x1.62e430P-1F, // (.693147181) https://oeis.org/A002162
+                ln10f       = 0x1.26bb1cP+1F, // (2.30258509) https://oeis.org/A002392
+                log2ef      = 0x1.715476P+0F, // (1.44269504)
+                log10ef     = 0x1.bcb7b2P-2F, // (.434294482)
+                pif         = 0x1.921fb6P+1F, // (3.14159265) https://oeis.org/A000796
+                inv_pif     = 0x1.45f306P-2F, // (.318309886) https://oeis.org/A049541
+                sqrtpif     = 0x1.c5bf8aP+0F, // (1.77245385) https://oeis.org/A002161
+                inv_sqrtpif = 0x1.20dd76P-1F, // (.564189584) https://oeis.org/A087197
+                sqrt2f      = 0x1.6a09e6P+0F, // (1.41421356) https://oeis.org/A002193
+                inv_sqrt2f  = 0x1.6a09e6P-1F, // (.707106781)
+                sqrt3f      = 0x1.bb67aeP+0F, // (1.73205081) https://oeis.org/A002194
+                inv_sqrt3f  = 0x1.279a74P-1F, // (.577350269)
+                phif        = 0x1.9e377aP+0F; // (1.61803399) https://oeis.org/A001622
 // clang-format on
 } // namespace numbers
 
diff --git a/llvm/include/llvm/Support/ModRef.h b/llvm/include/llvm/Support/ModRef.h
index 9ecdab71ec8ca..a8ce9a8e6e69c 100644
--- a/llvm/include/llvm/Support/ModRef.h
+++ b/llvm/include/llvm/Support/ModRef.h
@@ -327,6 +327,9 @@ class CaptureInfo {
   CaptureInfo(CaptureComponents Components)
       : OtherComponents(Components), RetComponents(Components) {}
 
+  /// Create CaptureInfo that does not capture any components of the pointer
+  static CaptureInfo none() { return CaptureInfo(CaptureComponents::None); }
+
   /// Create CaptureInfo that may capture all components of the pointer.
   static CaptureInfo all() { return CaptureInfo(CaptureComponents::All); }
 
@@ -360,6 +363,20 @@ class CaptureInfo {
                        RetComponents & Other.RetComponents);
   }
 
+  /// Compute union of CaptureInfos in-place.
+  CaptureInfo &operator|=(CaptureInfo Other) {
+    OtherComponents |= Other.OtherComponents;
+    RetComponents |= Other.RetComponents;
+    return *this;
+  }
+
+  /// Compute intersection of CaptureInfos in-place.
+  CaptureInfo &operator&=(CaptureInfo Other) {
+    OtherComponents &= Other.OtherComponents;
+    RetComponents &= Other.RetComponents;
+    return *this;
+  }
+
   static CaptureInfo createFromIntValue(uint32_t Data) {
     return CaptureInfo(CaptureComponents(Data >> 4),
                        CaptureComponents(Data & 0xf));
diff --git a/llvm/include/llvm/Support/NVPTXAddrSpace.h b/llvm/include/llvm/Support/NVPTXAddrSpace.h
index 93eae39e3d230..486a396621da1 100644
--- a/llvm/include/llvm/Support/NVPTXAddrSpace.h
+++ b/llvm/include/llvm/Support/NVPTXAddrSpace.h
@@ -17,17 +17,41 @@
 
 namespace llvm {
 namespace NVPTXAS {
+
 enum AddressSpace : unsigned {
   ADDRESS_SPACE_GENERIC = 0,
   ADDRESS_SPACE_GLOBAL = 1,
   ADDRESS_SPACE_SHARED = 3,
   ADDRESS_SPACE_CONST = 4,
   ADDRESS_SPACE_LOCAL = 5,
+  ADDRESS_SPACE_TENSOR = 6,
 
   ADDRESS_SPACE_PARAM = 101,
 };
-} // end namespace NVPTXAS
 
+// According to official PTX Writer's Guide, DWARF debug information should
+// contain DW_AT_address_class attribute for all variables and parameters.
+// It's required for cuda-gdb to be able to properly reflect the memory space
+// of variable address. Acceptable address class codes are listed in this enum.
+//
+// More detailed information:
+// https://docs.nvidia.com/cuda/ptx-writers-guide-to-interoperability/index.html#cuda-specific-dwarf-definitions
+enum DWARF_AddressSpace : unsigned {
+  DWARF_ADDR_code_space = 1,
+  DWARF_ADDR_reg_space = 2,
+  DWARF_ADDR_sreg_space = 3,
+  DWARF_ADDR_const_space = 4,
+  DWARF_ADDR_global_space = 5,
+  DWARF_ADDR_local_space = 6,
+  DWARF_ADDR_param_space = 7,
+  DWARF_ADDR_shared_space = 8,
+  DWARF_ADDR_surf_space = 9,
+  DWARF_ADDR_tex_space = 10,
+  DWARF_ADDR_tex_sampler_space = 11,
+  DWARF_ADDR_generic_space = 12
+};
+
+} // end namespace NVPTXAS
 } // end namespace llvm
 
 #endif // LLVM_SUPPORT_NVPTXADDRSPACE_H
diff --git a/llvm/include/llvm/TableGen/Record.h b/llvm/include/llvm/TableGen/Record.h
index d9930a48e8084..e04ed34823148 100644
--- a/llvm/include/llvm/TableGen/Record.h
+++ b/llvm/include/llvm/TableGen/Record.h
@@ -1523,7 +1523,7 @@ class RecordVal {
   bool IsUsed = false;
 
   /// Reference locations to this record value.
-  SmallVector<SMRange> ReferenceLocs;
+  SmallVector<SMRange, 0> ReferenceLocs;
 
 public:
   RecordVal(const Init *N, const RecTy *T, FieldKind K);
diff --git a/llvm/include/llvm/TableGen/StringToOffsetTable.h b/llvm/include/llvm/TableGen/StringToOffsetTable.h
index d4bb685acce32..e716411514bd6 100644
--- a/llvm/include/llvm/TableGen/StringToOffsetTable.h
+++ b/llvm/include/llvm/TableGen/StringToOffsetTable.h
@@ -27,6 +27,12 @@ class StringToOffsetTable {
   std::string AggregateString;
 
 public:
+  StringToOffsetTable() {
+    // Ensure we always put the empty string at offset zero. That lets empty
+    // initialization also be zero initialization for offsets into the table.
+    GetOrAddStringOffset("");
+  }
+
   bool empty() const { return StringOffset.empty(); }
   size_t size() const { return AggregateString.size(); }
 
@@ -51,28 +57,71 @@ class StringToOffsetTable {
     return II->second;
   }
 
-  // Emit the string using string literal concatenation, for better readability
-  // and searchability.
-  void EmitStringLiteralDef(raw_ostream &OS, const Twine &Decl,
-                            const Twine &Indent = "  ") const {
+  // Emit a string table definition with the provided name and indent.
+  //
+  // When possible, this uses string-literal concatenation to emit the string
+  // contents in a readable and searchable way. However, for (very) large string
+  // tables MSVC cannot reliably use string literals and so there we use a large
+  // character array. We still use a line oriented emission and add comments to
+  // provide searchability even in this case.
+  //
+  // The string table, and its input string contents, are always emitted as both
+  // `static` and `constexpr`. Both `Name` and (`Name` + "Storage") must be
+  // valid identifiers to declare.
+  void EmitStringTableDef(raw_ostream &OS, const Twine &Name,
+                          const Twine &Indent = "") const {
     OS << formatv(R"(
 #ifdef __GNUC__
 #pragma GCC diagnostic push
 #pragma GCC diagnostic ignored "-Woverlength-strings"
 #endif
-{0}{1} = )",
-                  Indent, Decl);
+{0}static constexpr char {1}Storage[] = )",
+                  Indent, Name);
+
+    // MSVC silently miscompiles string literals longer than 64k in some
+    // circumstances. When the string table is longer, emit it as an array of
+    // character literals.
+    bool UseChars = AggregateString.size() > (64 * 1024);
+    OS << (UseChars ? "{\n" : "\n");
+
+    llvm::ListSeparator LineSep(UseChars ? ",\n" : "\n");
+    llvm::SmallVector<StringRef> Strings(split(AggregateString, '\0'));
+    // We should always have an empty string at the start, and because these are
+    // null terminators rather than separators, we'll have one at the end as
+    // well. Skip the end one.
+    assert(Strings.front().empty() && "Expected empty initial string!");
+    assert(Strings.back().empty() &&
+           "Expected empty string at the end due to terminators!");
+    Strings.pop_back();
+    for (StringRef Str : Strings) {
+      OS << LineSep << Indent << "  ";
+      // If we can, just emit this as a string literal to be concatenated.
+      if (!UseChars) {
+        OS << "\"";
+        OS.write_escaped(Str);
+        OS << "\\0\"";
+        continue;
+      }
 
-    for (StringRef Str : split(AggregateString, '\0')) {
-      OS << "\n" << Indent << "  \"";
-      OS.write_escaped(Str);
-      OS << "\\0\"";
+      llvm::ListSeparator CharSep(", ");
+      for (char C : Str) {
+        OS << CharSep << "'";
+        OS.write_escaped(StringRef(&C, 1));
+        OS << "'";
+      }
+      OS << CharSep << "'\\0'";
     }
-    OS << R"(;
+    OS << LineSep << Indent << (UseChars ? "};" : "  ;");
+
+    OS << formatv(R"(
 #ifdef __GNUC__
 #pragma GCC diagnostic pop
 #endif
-)";
+
+{0}static constexpr llvm::StringTable {1} =
+{0}    {1}Storage;
+)",
+                  Indent, Name);
   }
 
   // Emit the string as one single string.
@@ -110,26 +159,6 @@ class StringToOffsetTable {
     }
     O << "\"";
   }
-
-  /// Emit the string using character literals. MSVC has a limitation that
-  /// string literals cannot be longer than 64K.
-  void EmitCharArray(raw_ostream &O) {
-    assert(AggregateString.find(')') == std::string::npos &&
-           "can't emit raw string with closing parens");
-    int Count = 0;
-    O << ' ';
-    for (char C : AggregateString) {
-      O << " \'";
-      O.write_escaped(StringRef(&C, 1));
-      O << "\',";
-      Count++;
-      if (Count > 14) {
-        O << "\n ";
-        Count = 0;
-      }
-    }
-    O << '\n';
-  }
 };
 
 } // end namespace llvm
diff --git a/llvm/include/llvm/Target/Target.td b/llvm/include/llvm/Target/Target.td
index f568a64971f09..e8b460aaf803b 100644
--- a/llvm/include/llvm/Target/Target.td
+++ b/llvm/include/llvm/Target/Target.td
@@ -43,8 +43,11 @@ def DefaultMode : HwMode<"", []>;
 // "Objects", which is a list of the same length as the list of modes.
 // The n-th element on the Objects list will be associated with the n-th
 // element on the Modes list.
-class HwModeSelect<list<HwMode> Ms> {
+class HwModeSelect<list<HwMode> Ms, int ObjectsLength> {
   list<HwMode> Modes = Ms;
+
+  assert !eq(ObjectsLength, !size(Modes)),
+     "The Objects and Modes lists must be the same length";
 }
 
 // A common class that implements a counterpart of ValueType, which is
@@ -53,7 +56,7 @@ class HwModeSelect<list<HwMode> Ms> {
 // objects could be used. This is specifically applicable to selection
 // patterns.
 class ValueTypeByHwMode<list<HwMode> Ms, list<ValueType> Ts>
-    : HwModeSelect<Ms>, ValueType<0, 0> {
+    : HwModeSelect<Ms, !size(Ts)>, ValueType<0, 0> {
   // The length of this list must be the same as the length of Ms.
   list<ValueType> Objects = Ts;
 }
@@ -64,7 +67,9 @@ class ValueTypeByHwMode<list<HwMode> Ms, list<ValueType> Ts>
 // or ValueType could be used. This is specifically applicable to selection
 // patterns.
 class PtrValueTypeByHwMode<ValueTypeByHwMode scalar, int addrspace>
-    : HwModeSelect<scalar.Modes>, PtrValueType<ValueType<0, 0>, addrspace> {
+    : HwModeSelect<scalar.Modes, !size(scalar.Objects)>,
+      PtrValueType<ValueType<0, 0>, addrspace> {
+  // The length of this list must be the same as the length of Ms.
   list<ValueType> Objects = scalar.Objects;
 }
 
@@ -78,7 +83,7 @@ class RegInfo<int RS, int SS, int SA> {
 
 // The register size/alignment information, parameterized by a HW mode.
 class RegInfoByHwMode<list<HwMode> Ms = [], list<RegInfo> Ts = []>
-    : HwModeSelect<Ms> {
+    : HwModeSelect<Ms, !size(Ts)> {
   // The length of this list must be the same as the length of Ms.
   list<RegInfo> Objects = Ts;
 }
@@ -89,7 +94,7 @@ class SubRegRange<int size, int offset = 0> {
 }
 
 class SubRegRangeByHwMode<list<HwMode> Ms = [], list<SubRegRange> Ts = []>
-    : HwModeSelect<Ms> {
+    : HwModeSelect<Ms, !size(Ts)> {
   // The length of this list must be the same as the length of Ms.
   list<SubRegRange> Objects = Ts;
 }
@@ -574,7 +579,7 @@ class InstructionEncoding {
 // an EncodingByHwMode, its Inst and Size members are ignored and Ts are used
 // to encode and decode based on HwMode.
 class EncodingByHwMode<list<HwMode> Ms = [], list<InstructionEncoding> Ts = []>
-    : HwModeSelect<Ms> {
+    : HwModeSelect<Ms, !size(Ts)> {
   // The length of this list must be the same as the length of Ms.
   list<InstructionEncoding> Objects = Ts;
 }
diff --git a/llvm/include/llvm/Target/TargetIntrinsicInfo.h b/llvm/include/llvm/Target/TargetIntrinsicInfo.h
deleted file mode 100644
index dc59f11c8d9a1..0000000000000
--- a/llvm/include/llvm/Target/TargetIntrinsicInfo.h
+++ /dev/null
@@ -1,68 +0,0 @@
-//===-- llvm/Target/TargetIntrinsicInfo.h - Instruction Info ----*- C++ -*-===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-//
-// This file describes the target intrinsic instructions to the code generator.
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LLVM_TARGET_TARGETINTRINSICINFO_H
-#define LLVM_TARGET_TARGETINTRINSICINFO_H
-
-#include "llvm/ADT/StringRef.h"
-#include <string>
-
-namespace llvm {
-
-class Function;
-class Module;
-class Type;
-
-//---------------------------------------------------------------------------
-///
-/// TargetIntrinsicInfo - Interface to description of machine instruction set
-///
-class TargetIntrinsicInfo {
-  TargetIntrinsicInfo(const TargetIntrinsicInfo &) = delete;
-  void operator=(const TargetIntrinsicInfo &) = delete;
-public:
-  TargetIntrinsicInfo();
-  virtual ~TargetIntrinsicInfo();
-
-  /// Return the name of a target intrinsic, e.g. "llvm.bfin.ssync".
-  /// The Tys and numTys parameters are for intrinsics with overloaded types
-  /// (e.g., those using iAny or fAny). For a declaration for an overloaded
-  /// intrinsic, Tys should point to an array of numTys pointers to Type,
-  /// and must provide exactly one type for each overloaded type in the
-  /// intrinsic.
-  virtual std::string getName(unsigned IID, Type **Tys = nullptr,
-                              unsigned numTys = 0) const = 0;
-
-  /// Look up target intrinsic by name. Return intrinsic ID or 0 for unknown
-  /// names.
-  virtual unsigned lookupName(const char *Name, unsigned Len) const =0;
-
-  unsigned lookupName(StringRef Name) const {
-    return lookupName(Name.data(), Name.size());
-  }
-
-  /// Return the target intrinsic ID of a function, or 0.
-  virtual unsigned getIntrinsicID(const Function *F) const;
-
-  /// Returns true if the intrinsic can be overloaded.
-  virtual bool isOverloaded(unsigned IID) const = 0;
-
-  /// Create or insert an LLVM Function declaration for an intrinsic,
-  /// and return it. The Tys and numTys are for intrinsics with overloaded
-  /// types. See above for more information.
-  virtual Function *getDeclaration(Module *M, unsigned ID, Type **Tys = nullptr,
-                                   unsigned numTys = 0) const = 0;
-};
-
-} // End llvm namespace
-
-#endif
diff --git a/llvm/include/llvm/Target/TargetLoweringObjectFile.h b/llvm/include/llvm/Target/TargetLoweringObjectFile.h
index b523d6a47b41a..a5ed1b29dc1bc 100644
--- a/llvm/include/llvm/Target/TargetLoweringObjectFile.h
+++ b/llvm/include/llvm/Target/TargetLoweringObjectFile.h
@@ -21,6 +21,7 @@
 namespace llvm {
 
 struct Align;
+struct MachineJumpTableEntry;
 class Constant;
 class DataLayout;
 class Function;
@@ -135,6 +136,10 @@ class TargetLoweringObjectFile : public MCObjectFileInfo {
 
   virtual MCSection *getSectionForJumpTable(const Function &F,
                                             const TargetMachine &TM) const;
+  virtual MCSection *
+  getSectionForJumpTable(const Function &F, const TargetMachine &TM,
+                         const MachineJumpTableEntry *JTE) const;
+
   virtual MCSection *getSectionForLSDA(const Function &, const MCSymbol &,
                                        const TargetMachine &) const {
     return LSDASection;
diff --git a/llvm/include/llvm/Target/TargetMachine.h b/llvm/include/llvm/Target/TargetMachine.h
index 9bdb110bd3683..27eeb415ed644 100644
--- a/llvm/include/llvm/Target/TargetMachine.h
+++ b/llvm/include/llvm/Target/TargetMachine.h
@@ -19,6 +19,7 @@
 #include "llvm/MC/MCStreamer.h"
 #include "llvm/Support/Allocator.h"
 #include "llvm/Support/CodeGen.h"
+#include "llvm/Support/CommandLine.h"
 #include "llvm/Support/Error.h"
 #include "llvm/Support/PGOOptions.h"
 #include "llvm/Target/CGPassBuilderOption.h"
@@ -28,6 +29,8 @@
 #include <string>
 #include <utility>
 
+extern llvm::cl::opt<bool> NoKernelInfoEndLTO;
+
 namespace llvm {
 
 class AAManager;
@@ -36,6 +39,7 @@ using ModulePassManager = PassManager<Module>;
 class Function;
 class GlobalValue;
 class MachineModuleInfoWrapperPass;
+struct MachineSchedContext;
 class Mangler;
 class MCAsmInfo;
 class MCContext;
@@ -47,10 +51,10 @@ class raw_pwrite_stream;
 class PassBuilder;
 class PassInstrumentationCallbacks;
 struct PerFunctionMIParsingState;
+class ScheduleDAGInstrs;
 class SMDiagnostic;
 class SMRange;
 class Target;
-class TargetIntrinsicInfo;
 class TargetIRAnalysis;
 class TargetTransformInfo;
 class TargetLoweringObjectFile;
@@ -144,6 +148,28 @@ class TargetMachine {
     return nullptr;
   }
 
+  /// Create an instance of ScheduleDAGInstrs to be run within the standard
+  /// MachineScheduler pass for this function and target at the current
+  /// optimization level.
+  ///
+  /// This can also be used to plug a new MachineSchedStrategy into an instance
+  /// of the standard ScheduleDAGMI:
+  ///   return new ScheduleDAGMI(C, std::make_unique<MyStrategy>(C),
+  ///   /*RemoveKillFlags=*/false)
+  ///
+  /// Return NULL to select the default (generic) machine scheduler.
+  virtual ScheduleDAGInstrs *
+  createMachineScheduler(MachineSchedContext *C) const {
+    return nullptr;
+  }
+
+  /// Similar to createMachineScheduler but used when postRA machine scheduling
+  /// is enabled.
+  virtual ScheduleDAGInstrs *
+  createPostMachineScheduler(MachineSchedContext *C) const {
+    return nullptr;
+  }
+
   /// Allocate and return a default initialized instance of the YAML
   /// representation for the MachineFunctionInfo.
   virtual yaml::MachineFunctionInfo *createDefaultFuncInfoYAML() const {
@@ -215,11 +241,6 @@ class TargetMachine {
   const MCInstrInfo *getMCInstrInfo() const { return MII.get(); }
   const MCSubtargetInfo *getMCSubtargetInfo() const { return STI.get(); }
 
-  /// If intrinsic information is available, return it.  If not, return null.
-  virtual const TargetIntrinsicInfo *getIntrinsicInfo() const {
-    return nullptr;
-  }
-
   bool requiresStructuredCFG() const { return RequireStructuredCFG; }
   void setRequiresStructuredCFG(bool Value) { RequireStructuredCFG = Value; }
 
@@ -305,6 +326,10 @@ class TargetMachine {
     return Options.FunctionSections;
   }
 
+  bool getEnableStaticDataPartitioning() const {
+    return Options.EnableStaticDataPartitioning;
+  }
+
   /// Return true if visibility attribute should not be emitted in XCOFF,
   /// corresponding to -mignore-xcoff-visibility.
   bool getIgnoreXCOFFVisibility() const {
diff --git a/llvm/include/llvm/Target/TargetOptions.h b/llvm/include/llvm/Target/TargetOptions.h
index 88f253805ca99..fd8dad4f6f791 100644
--- a/llvm/include/llvm/Target/TargetOptions.h
+++ b/llvm/include/llvm/Target/TargetOptions.h
@@ -145,7 +145,8 @@ namespace llvm {
           TrapUnreachable(false), NoTrapAfterNoreturn(false), TLSSize(0),
           EmulatedTLS(false), EnableTLSDESC(false), EnableIPRA(false),
           EmitStackSizeSection(false), EnableMachineOutliner(false),
-          EnableMachineFunctionSplitter(false), SupportsDefaultOutlining(false),
+          EnableMachineFunctionSplitter(false),
+          EnableStaticDataPartitioning(false), SupportsDefaultOutlining(false),
           EmitAddrsig(false), BBAddrMap(false), EmitCallSiteInfo(false),
           SupportsDebugEntryValues(false), EnableDebugEntryValues(false),
           ValueTrackingVariableLocations(false), ForceDwarfFrameSection(false),
@@ -312,6 +313,9 @@ namespace llvm {
     /// Enables the MachineFunctionSplitter pass.
     unsigned EnableMachineFunctionSplitter : 1;
 
+    /// Enables the StaticDataSplitter pass.
+    unsigned EnableStaticDataPartitioning : 1;
+
     /// Set if the target supports default outlining behaviour.
     unsigned SupportsDefaultOutlining : 1;
 
diff --git a/llvm/include/llvm/Target/TargetSelectionDAG.td b/llvm/include/llvm/Target/TargetSelectionDAG.td
index 7f3c2be90d820..42a5fbec95174 100644
--- a/llvm/include/llvm/Target/TargetSelectionDAG.td
+++ b/llvm/include/llvm/Target/TargetSelectionDAG.td
@@ -840,6 +840,10 @@ def vector_insert_subvec : SDNode<"ISD::INSERT_SUBVECTOR",
 def extract_subvector : SDNode<"ISD::EXTRACT_SUBVECTOR", SDTSubVecExtract, []>;
 def insert_subvector : SDNode<"ISD::INSERT_SUBVECTOR", SDTSubVecInsert, []>;
 
+def find_last_active
+    : SDNode<"ISD::VECTOR_FIND_LAST_ACTIVE",
+             SDTypeProfile<1, 1, [SDTCisInt<0>, SDTCisVec<1>]>, []>;
+
 // Nodes for intrinsics, you should use the intrinsic itself and let tblgen use
 // these internally.  Don't reference these directly.
 def intrinsic_void : SDNode<"ISD::INTRINSIC_VOID",
diff --git a/llvm/include/llvm/TargetParser/Triple.h b/llvm/include/llvm/TargetParser/Triple.h
index 8097300c6e630..09c0d223d9b4d 100644
--- a/llvm/include/llvm/TargetParser/Triple.h
+++ b/llvm/include/llvm/TargetParser/Triple.h
@@ -498,6 +498,9 @@ class Triple {
     return getArchPointerBitWidth(getArch());
   }
 
+  /// Returns the trampoline size in bytes for this configuration.
+  unsigned getTrampolineSize() const;
+
   /// Test whether the architecture is 64-bit
   ///
   /// Note that this tests for 64-bit pointer width, and nothing else. Note
@@ -653,6 +656,9 @@ class Triple {
     return getOS() == Triple::Win32;
   }
 
+  /// Tests whether the OS is Windows or UEFI.
+  bool isOSWindowsOrUEFI() const { return isOSWindows() || isUEFI(); }
+
   /// Checks if the environment is MSVC.
   bool isKnownWindowsMSVCEnvironment() const {
     return isOSWindows() && getEnvironment() == Triple::MSVC;
@@ -1117,9 +1123,10 @@ class Triple {
            isWindowsCygwinEnvironment() || isOHOSFamily();
   }
 
-  /// True if the target supports both general-dynamic and TLSDESC, and TLSDESC
-  /// is enabled by default.
-  bool hasDefaultTLSDESC() const { return isAndroid() && isRISCV64(); }
+  /// True if the target uses TLSDESC by default.
+  bool hasDefaultTLSDESC() const {
+    return isAArch64() || (isAndroid() && isRISCV64()) || isOSFuchsia();
+  }
 
   /// Tests whether the target uses -data-sections as default.
   bool hasDefaultDataSections() const {
diff --git a/llvm/include/llvm/Transforms/Coroutines/CoroInstr.h b/llvm/include/llvm/Transforms/Coroutines/CoroInstr.h
index 3aa30bec85c3a..fbc76219ead86 100644
--- a/llvm/include/llvm/Transforms/Coroutines/CoroInstr.h
+++ b/llvm/include/llvm/Transforms/Coroutines/CoroInstr.h
@@ -170,7 +170,7 @@ class CoroIdInst : public AnyCoroIdInst {
       Inst->eraseFromParent();
       return;
     }
-    Inst->moveBefore(getCoroBegin()->getNextNode());
+    Inst->moveBefore(std::next(getCoroBegin()->getIterator()));
   }
 
   // Info argument of coro.id is
diff --git a/llvm/include/llvm/Transforms/IPO/Attributor.h b/llvm/include/llvm/Transforms/IPO/Attributor.h
index 8589314699749..995ee54a73ce4 100644
--- a/llvm/include/llvm/Transforms/IPO/Attributor.h
+++ b/llvm/include/llvm/Transforms/IPO/Attributor.h
@@ -1954,8 +1954,12 @@ struct Attributor {
 private:
   /// Helper to check \p Attrs for \p AK, if not found, check if \p
   /// AAType::isImpliedByIR is true, and if not, create AAType for \p IRP.
+  /// If \p SkipHasAttrCheck is true, don't check whether the attribute is set
+  /// first. This should be used if only some values of a complex IR attribute
+  /// imply the AAType.
   template <Attribute::AttrKind AK, typename AAType>
-  void checkAndQueryIRAttr(const IRPosition &IRP, AttributeSet Attrs);
+  void checkAndQueryIRAttr(const IRPosition &IRP, AttributeSet Attrs,
+                           bool SkipHasAttrCheck = false);
 
   /// Helper to apply \p CB on all attributes of type \p AttrDescs of \p IRP.
   template <typename DescTy>
@@ -4355,7 +4359,7 @@ struct AAInstanceInfo : public StateWrapper<BooleanState, AbstractAttribute> {
 /// An abstract interface for all nocapture attributes.
 struct AANoCapture
     : public IRAttribute<
-          Attribute::NoCapture,
+          Attribute::Captures,
           StateWrapper<BitIntegerState<uint16_t, 7, 0>, AbstractAttribute>,
           AANoCapture> {
   AANoCapture(const IRPosition &IRP, Attributor &A) : IRAttribute(IRP) {}
@@ -6513,7 +6517,7 @@ bool hasAssumedIRAttr(Attributor &A, const AbstractAttribute *QueryingAA,
     CASE(NoUnwind, AANoUnwind, );
     CASE(WillReturn, AAWillReturn, );
     CASE(NoFree, AANoFree, );
-    CASE(NoCapture, AANoCapture, );
+    CASE(Captures, AANoCapture, );
     CASE(NoRecurse, AANoRecurse, );
     CASE(NoReturn, AANoReturn, );
     CASE(NoSync, AANoSync, );
diff --git a/llvm/include/llvm/Transforms/Instrumentation/LowerAllowCheckPass.h b/llvm/include/llvm/Transforms/Instrumentation/LowerAllowCheckPass.h
index 3ee907606e12b..2c6e60138f2aa 100644
--- a/llvm/include/llvm/Transforms/Instrumentation/LowerAllowCheckPass.h
+++ b/llvm/include/llvm/Transforms/Instrumentation/LowerAllowCheckPass.h
@@ -25,7 +25,7 @@ namespace llvm {
 class LowerAllowCheckPass : public PassInfoMixin<LowerAllowCheckPass> {
 public:
   struct Options {
-    std::vector<unsigned int> placeholder; // TODO: cutoffs
+    std::vector<unsigned int> cutoffs;
   };
 
   explicit LowerAllowCheckPass(LowerAllowCheckPass::Options Opts)
@@ -33,6 +33,8 @@ class LowerAllowCheckPass : public PassInfoMixin<LowerAllowCheckPass> {
   PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM);
 
   static bool IsRequested();
+  void printPipeline(raw_ostream &OS,
+                     function_ref<StringRef(StringRef)> MapClassName2PassName);
 
 private:
   LowerAllowCheckPass::Options Opts;
diff --git a/llvm/include/llvm/Transforms/Utils/BasicBlockUtils.h b/llvm/include/llvm/Transforms/Utils/BasicBlockUtils.h
index b447942ffbd67..6faff3d1fd8e3 100644
--- a/llvm/include/llvm/Transforms/Utils/BasicBlockUtils.h
+++ b/llvm/include/llvm/Transforms/Utils/BasicBlockUtils.h
@@ -540,7 +540,7 @@ inline void SplitBlockAndInsertIfThenElse(Value *Cond, Instruction *SplitBefore,
 /// SplitBefore.  Returns the first insert point in the loop body, and the
 /// PHINode for the induction variable (i.e. "i" above).
 std::pair<Instruction*, Value*>
-SplitBlockAndInsertSimpleForLoop(Value *End, Instruction *SplitBefore);
+SplitBlockAndInsertSimpleForLoop(Value *End, BasicBlock::iterator SplitBefore);
 
 /// Utility function for performing a given action on each lane of a vector
 /// with \p EC elements.  To simplify porting legacy code, this defaults to
@@ -550,9 +550,9 @@ SplitBlockAndInsertSimpleForLoop(Value *End, Instruction *SplitBefore);
 /// IRBuilder whose insert point is correctly set for instantiating the
 /// given index, and a value which is (at runtime) the index to access.
 /// This index *may* be a constant.
-void SplitBlockAndInsertForEachLane(ElementCount EC, Type *IndexTy,
-    Instruction *InsertBefore,
-    std::function<void(IRBuilderBase&, Value*)> Func);
+void SplitBlockAndInsertForEachLane(
+    ElementCount EC, Type *IndexTy, BasicBlock::iterator InsertBefore,
+    std::function<void(IRBuilderBase &, Value *)> Func);
 
 /// Utility function for performing a given action on each lane of a vector
 /// with \p EVL effective length. EVL is assumed > 0. To simplify porting legacy
@@ -563,7 +563,7 @@ void SplitBlockAndInsertForEachLane(ElementCount EC, Type *IndexTy,
 /// the given index, and a value which is (at runtime) the index to access. This
 /// index *may* be a constant.
 void SplitBlockAndInsertForEachLane(
-    Value *End, Instruction *InsertBefore,
+    Value *End, BasicBlock::iterator InsertBefore,
     std::function<void(IRBuilderBase &, Value *)> Func);
 
 /// Check whether BB is the merge point of a if-region.
diff --git a/llvm/include/llvm/Transforms/Utils/MemoryTaggingSupport.h b/llvm/include/llvm/Transforms/Utils/MemoryTaggingSupport.h
index 60255d5736409..8b7daf616b110 100644
--- a/llvm/include/llvm/Transforms/Utils/MemoryTaggingSupport.h
+++ b/llvm/include/llvm/Transforms/Utils/MemoryTaggingSupport.h
@@ -91,7 +91,6 @@ class StackInfoBuilder {
 
 uint64_t getAllocaSizeInBytes(const AllocaInst &AI);
 void alignAndPadAlloca(memtag::AllocaInfo &Info, llvm::Align Align);
-bool isLifetimeIntrinsic(Value *V);
 
 Value *readRegister(IRBuilder<> &IRB, StringRef Name);
 Value *getFP(IRBuilder<> &IRB);
diff --git a/llvm/include/llvm/Transforms/Utils/UnrollLoop.h b/llvm/include/llvm/Transforms/Utils/UnrollLoop.h
index 8cf17ced458c8..ed560f6f6e2fa 100644
--- a/llvm/include/llvm/Transforms/Utils/UnrollLoop.h
+++ b/llvm/include/llvm/Transforms/Utils/UnrollLoop.h
@@ -76,6 +76,7 @@ struct UnrollLoopOptions {
   bool ForgetAllSCEV;
   const Instruction *Heart = nullptr;
   unsigned SCEVExpansionBudget;
+  bool RuntimeUnrollMultiExit = false;
 };
 
 LoopUnrollResult UnrollLoop(Loop *L, UnrollLoopOptions ULO, LoopInfo *LI,
@@ -91,7 +92,8 @@ bool UnrollRuntimeLoopRemainder(
     bool UseEpilogRemainder, bool UnrollRemainder, bool ForgetAllSCEV,
     LoopInfo *LI, ScalarEvolution *SE, DominatorTree *DT, AssumptionCache *AC,
     const TargetTransformInfo *TTI, bool PreserveLCSSA,
-    unsigned SCEVExpansionBudget, Loop **ResultLoop = nullptr);
+    unsigned SCEVExpansionBudget, bool RuntimeUnrollMultiExit,
+    Loop **ResultLoop = nullptr);
 
 LoopUnrollResult UnrollAndJamLoop(Loop *L, unsigned Count, unsigned TripCount,
                                   unsigned TripMultiple, bool UnrollRemainder,
diff --git a/llvm/include/llvm/Transforms/Utils/VNCoercion.h b/llvm/include/llvm/Transforms/Utils/VNCoercion.h
index f1ea94bf60fcc..ed4dbad50ee85 100644
--- a/llvm/include/llvm/Transforms/Utils/VNCoercion.h
+++ b/llvm/include/llvm/Transforms/Utils/VNCoercion.h
@@ -23,6 +23,7 @@
 
 namespace llvm {
 class Constant;
+class Function;
 class StoreInst;
 class LoadInst;
 class MemIntrinsic;
@@ -35,7 +36,7 @@ namespace VNCoercion {
 /// Return true if CoerceAvailableValueToLoadType would succeed if it was
 /// called.
 bool canCoerceMustAliasedValueToLoad(Value *StoredVal, Type *LoadTy,
-                                     const DataLayout &DL);
+                                     Function *F);
 
 /// If we saw a store of a value to memory, and then a load from a must-aliased
 /// pointer of a different type, try to coerce the stored value to the loaded
@@ -44,7 +45,7 @@ bool canCoerceMustAliasedValueToLoad(Value *StoredVal, Type *LoadTy,
 ///
 /// If we can't do it, return null.
 Value *coerceAvailableValueToLoadType(Value *StoredVal, Type *LoadedTy,
-                                      IRBuilderBase &IRB, const DataLayout &DL);
+                                      IRBuilderBase &IRB, Function *F);
 
 /// This function determines whether a value for the pointer LoadPtr can be
 /// extracted from the store at DepSI.
@@ -75,7 +76,7 @@ int analyzeLoadFromClobberingMemInst(Type *LoadTy, Value *LoadPtr,
 /// It inserts instructions to do so at InsertPt, and returns the extracted
 /// value.
 Value *getValueForLoad(Value *SrcVal, unsigned Offset, Type *LoadTy,
-                            Instruction *InsertPt, const DataLayout &DL);
+                       Instruction *InsertPt, Function *F);
 // This is the same as getValueForLoad, except it performs no insertion.
 // It only allows constant inputs.
 Constant *getConstantValueForLoad(Constant *SrcVal, unsigned Offset,
diff --git a/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/DependencyGraph.h b/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/DependencyGraph.h
index b2d7c9b8aa8bb..54cb8fa6ea848 100644
--- a/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/DependencyGraph.h
+++ b/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/DependencyGraph.h
@@ -45,15 +45,6 @@ class DGNode;
 class MemDGNode;
 class DependencyGraph;
 
-/// While OpIt points to a Value that is not an Instruction keep incrementing
-/// it. \Returns the first iterator that points to an Instruction, or end.
-[[nodiscard]] static User::op_iterator skipNonInstr(User::op_iterator OpIt,
-                                                    User::op_iterator OpItE) {
-  while (OpIt != OpItE && !isa<Instruction>((*OpIt).get()))
-    ++OpIt;
-  return OpIt;
-}
-
 /// Iterate over both def-use and mem dependencies.
 class PredIterator {
   User::op_iterator OpIt;
@@ -72,6 +63,12 @@ class PredIterator {
   friend class DGNode;    // For constructor
   friend class MemDGNode; // For constructor
 
+  /// Skip iterators that don't point instructions or are outside \p DAG,
+  /// starting from \p OpIt and ending before \p OpItE.n
+  static User::op_iterator skipBadIt(User::op_iterator OpIt,
+                                     User::op_iterator OpItE,
+                                     const DependencyGraph &DAG);
+
 public:
   using difference_type = std::ptrdiff_t;
   using value_type = DGNode *;
@@ -124,6 +121,7 @@ class DGNode {
     assert(UnscheduledSuccs > 0 && "Counting error!");
     --UnscheduledSuccs;
   }
+  void incrUnscheduledSuccs() { ++UnscheduledSuccs; }
   /// \Returns true if all dependent successors have been scheduled.
   bool ready() const { return UnscheduledSuccs == 0; }
   /// \Returns true if this node has been scheduled.
@@ -135,8 +133,9 @@ class DGNode {
   bool comesBefore(const DGNode *Other) { return I->comesBefore(Other->I); }
   using iterator = PredIterator;
   virtual iterator preds_begin(DependencyGraph &DAG) {
-    return PredIterator(skipNonInstr(I->op_begin(), I->op_end()), I->op_end(),
-                        this, DAG);
+    return PredIterator(
+        PredIterator::skipBadIt(I->op_begin(), I->op_end(), DAG), I->op_end(),
+        this, DAG);
   }
   virtual iterator preds_end(DependencyGraph &DAG) {
     return PredIterator(I->op_end(), I->op_end(), this, DAG);
@@ -218,12 +217,14 @@ class MemDGNode final : public DGNode {
   friend class PredIterator; // For MemPreds.
   /// Creates both edges: this<->N.
   void setNextNode(MemDGNode *N) {
+    assert(N != this && "About to point to self!");
     NextMemN = N;
     if (NextMemN != nullptr)
       NextMemN->PrevMemN = this;
   }
   /// Creates both edges: N<->this.
   void setPrevNode(MemDGNode *N) {
+    assert(N != this && "About to point to self!");
     PrevMemN = N;
     if (PrevMemN != nullptr)
       PrevMemN->NextMemN = this;
@@ -247,8 +248,8 @@ class MemDGNode final : public DGNode {
   }
   iterator preds_begin(DependencyGraph &DAG) override {
     auto OpEndIt = I->op_end();
-    return PredIterator(skipNonInstr(I->op_begin(), OpEndIt), OpEndIt,
-                        MemPreds.begin(), this, DAG);
+    return PredIterator(PredIterator::skipBadIt(I->op_begin(), OpEndIt, DAG),
+                        OpEndIt, MemPreds.begin(), this, DAG);
   }
   iterator preds_end(DependencyGraph &DAG) override {
     return PredIterator(I->op_end(), I->op_end(), MemPreds.end(), this, DAG);
@@ -263,6 +264,7 @@ class MemDGNode final : public DGNode {
   void addMemPred(MemDGNode *PredN) {
     [[maybe_unused]] auto Inserted = MemPreds.insert(PredN).second;
     assert(Inserted && "PredN already exists!");
+    assert(PredN != this && "Trying to add a dependency to self!");
     if (!Scheduled) {
       ++PredN->UnscheduledSuccs;
     }
@@ -348,13 +350,15 @@ class DependencyGraph {
   void createNewNodes(const Interval<Instruction> &NewInterval);
 
   /// Helper for `notify*Instr()`. \Returns the first MemDGNode that comes
-  /// before \p N, including or excluding \p N based on \p IncludingN, or
-  /// nullptr if not found.
-  MemDGNode *getMemDGNodeBefore(DGNode *N, bool IncludingN) const;
+  /// before \p N, skipping \p SkipN, including or excluding \p N based on
+  /// \p IncludingN, or nullptr if not found.
+  MemDGNode *getMemDGNodeBefore(DGNode *N, bool IncludingN,
+                                MemDGNode *SkipN = nullptr) const;
   /// Helper for `notifyMoveInstr()`. \Returns the first MemDGNode that comes
-  /// after \p N, including or excluding \p N based on \p IncludingN, or nullptr
-  /// if not found.
-  MemDGNode *getMemDGNodeAfter(DGNode *N, bool IncludingN) const;
+  /// after \p N, skipping \p SkipN, including or excluding \p N based on \p
+  /// IncludingN, or nullptr if not found.
+  MemDGNode *getMemDGNodeAfter(DGNode *N, bool IncludingN,
+                               MemDGNode *SkipN = nullptr) const;
 
   /// Called by the callbacks when a new instruction \p I has been created.
   void notifyCreateInstr(Instruction *I);
diff --git a/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/Interval.h b/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/Interval.h
index 18cd29e9e14ee..f6c5a20467337 100644
--- a/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/Interval.h
+++ b/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/Interval.h
@@ -108,6 +108,10 @@ template <typename T> class Interval {
     return (Top == I || Top->comesBefore(I)) &&
            (I == Bottom || I->comesBefore(Bottom));
   }
+  /// \Returns true if \p Elm is right before the top or right after the bottom.
+  bool touches(T *Elm) const {
+    return Top == Elm->getNextNode() || Bottom == Elm->getPrevNode();
+  }
   T *top() const { return Top; }
   T *bottom() const { return Bottom; }
 
diff --git a/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/Legality.h b/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/Legality.h
index f10c535aa820e..132b12a7b4e6c 100644
--- a/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/Legality.h
+++ b/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/Legality.h
@@ -91,6 +91,8 @@ enum class ResultReason {
   DiffTypes,
   DiffMathFlags,
   DiffWrapFlags,
+  DiffBBs,
+  RepeatedInstrs,
   NotConsecutive,
   CantSchedule,
   Unimplemented,
@@ -127,6 +129,10 @@ struct ToStr {
       return "DiffMathFlags";
     case ResultReason::DiffWrapFlags:
       return "DiffWrapFlags";
+    case ResultReason::DiffBBs:
+      return "DiffBBs";
+    case ResultReason::RepeatedInstrs:
+      return "RepeatedInstrs";
     case ResultReason::NotConsecutive:
       return "NotConsecutive";
     case ResultReason::CantSchedule:
diff --git a/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/Passes/TransactionAcceptOrRevert.h b/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/Passes/TransactionAcceptOrRevert.h
new file mode 100644
index 0000000000000..fce9cc0c1bde7
--- /dev/null
+++ b/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/Passes/TransactionAcceptOrRevert.h
@@ -0,0 +1,30 @@
+//===- TransactionAcceptOrRevert.h ------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This is a region pass that checks the region cost before/after vectorization
+// and accepts the state of Sandbox IR if the cost is better, or otherwise
+// reverts it.
+//
+
+#ifndef LLVM_TRANSFORMS_VECTORIZE_SANDBOXVECTORIZER_PASSES_TRANSACTIONACCEPTORREVERT_H
+#define LLVM_TRANSFORMS_VECTORIZE_SANDBOXVECTORIZER_PASSES_TRANSACTIONACCEPTORREVERT_H
+
+#include "llvm/SandboxIR/Pass.h"
+#include "llvm/SandboxIR/Region.h"
+
+namespace llvm::sandboxir {
+
+class TransactionAcceptOrRevert : public RegionPass {
+public:
+  TransactionAcceptOrRevert() : RegionPass("tr-accept-or-revert") {}
+  bool runOnRegion(Region &Rgn, const Analyses &A) final;
+};
+
+} // namespace llvm::sandboxir
+
+#endif // LLVM_TRANSFORMS_VECTORIZE_SANDBOXVECTORIZER_PASSES_TRANSACTIONACCEPTORREVERT_H
diff --git a/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/Passes/TransactionAlwaysAccept.h b/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/Passes/TransactionAlwaysAccept.h
new file mode 100644
index 0000000000000..ed6cf1bf7cf51
--- /dev/null
+++ b/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/Passes/TransactionAlwaysAccept.h
@@ -0,0 +1,34 @@
+//===- TransactionAlwaysAccept.h --------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This is a region pass that always accepts the transaction without checking
+// its cost. This is mainly used as a final pass in lit tests.
+//
+
+#ifndef LLVM_TRANSFORMS_VECTORIZE_SANDBOXVECTORIZER_PASSES_TRANSACTIONALWAYSACCEPT_H
+#define LLVM_TRANSFORMS_VECTORIZE_SANDBOXVECTORIZER_PASSES_TRANSACTIONALWAYSACCEPT_H
+
+#include "llvm/SandboxIR/Pass.h"
+#include "llvm/SandboxIR/Region.h"
+
+namespace llvm::sandboxir {
+
+class TransactionAlwaysAccept : public RegionPass {
+public:
+  TransactionAlwaysAccept() : RegionPass("tr-accept") {}
+  bool runOnRegion(Region &Rgn, const Analyses &A) final {
+    auto &Tracker = Rgn.getContext().getTracker();
+    bool HasChanges = !Tracker.empty();
+    Tracker.accept();
+    return HasChanges;
+  }
+};
+
+} // namespace llvm::sandboxir
+
+#endif // LLVM_TRANSFORMS_VECTORIZE_SANDBOXVECTORIZER_PASSES_TRANSACTIONALWAYSACCEPT_H
diff --git a/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/SandboxVectorizer.h b/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/SandboxVectorizer.h
index 09369dbb496fc..7ea9386f08bee 100644
--- a/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/SandboxVectorizer.h
+++ b/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/SandboxVectorizer.h
@@ -24,10 +24,18 @@ class SandboxVectorizerPass : public PassInfoMixin<SandboxVectorizerPass> {
   TargetTransformInfo *TTI = nullptr;
   AAResults *AA = nullptr;
   ScalarEvolution *SE = nullptr;
-
+  // NOTE: We define the Context as a pass-scope object instead of local object
+  // in runOnFunction() because the passes defined in the pass-manager need
+  // access to it for registering/deregistering callbacks during construction
+  // and destruction.
   std::unique_ptr<sandboxir::Context> Ctx;
 
   // A pipeline of SandboxIR function passes run by the vectorizer.
+  // NOTE: We define this as a pass-scope object to avoid recreating the
+  // pass-pipeline every time in runOnFunction(). The downside is that the
+  // Context also needs to be defined as a pass-scope object because the passes
+  // within FPM may register/unregister callbacks, so they need access to
+  // Context.
   sandboxir::FunctionPassManager FPM;
 
   bool runImpl(Function &F);
diff --git a/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/Scheduler.h b/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/Scheduler.h
index 25432e1396c73..6c2315af0e797 100644
--- a/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/Scheduler.h
+++ b/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/Scheduler.h
@@ -30,8 +30,23 @@ namespace llvm::sandboxir {
 class PriorityCmp {
 public:
   bool operator()(const DGNode *N1, const DGNode *N2) {
-    // TODO: This should be a hierarchical comparator.
-    return N1->getInstruction()->comesBefore(N2->getInstruction());
+    // Given that the DAG does not model dependencies such that PHIs are always
+    // at the top, or terminators always at the bottom, we need to force the
+    // priority here in the comparator of the ready list container.
+    auto *I1 = N1->getInstruction();
+    auto *I2 = N2->getInstruction();
+    bool IsTerm1 = I1->isTerminator();
+    bool IsTerm2 = I2->isTerminator();
+    if (IsTerm1 != IsTerm2)
+      // Terminators have the lowest priority.
+      return IsTerm1 > IsTerm2;
+    bool IsPHI1 = isa<PHINode>(I1);
+    bool IsPHI2 = isa<PHINode>(I2);
+    if (IsPHI1 != IsPHI2)
+      // PHIs have the highest priority.
+      return IsPHI1 < IsPHI2;
+    // Otherwise rely on the instruction order.
+    return I2->comesBefore(I1);
   }
 };
 
@@ -54,6 +69,22 @@ class ReadyListContainer {
   }
   bool empty() const { return List.empty(); }
   void clear() { List = {}; }
+  /// \Removes \p N if found in the ready list.
+  void remove(DGNode *N) {
+    // TODO: Use a more efficient data-structure for the ready list because the
+    // priority queue does not support fast removals.
+    SmallVector<DGNode *, 8> Keep;
+    Keep.reserve(List.size());
+    while (!List.empty()) {
+      auto *Top = List.top();
+      List.pop();
+      if (Top == N)
+        break;
+      Keep.push_back(Top);
+    }
+    for (auto *KeepN : Keep)
+      List.push(KeepN);
+  }
 #ifndef NDEBUG
   void dump(raw_ostream &OS) const;
   LLVM_DUMP_METHOD void dump() const;
@@ -116,12 +147,22 @@ class Scheduler {
   /// The dependency graph is used by the scheduler to determine the legal
   /// ordering of instructions.
   DependencyGraph DAG;
+  friend class SchedulerInternalsAttorney; // For DAG.
+  Context &Ctx;
   /// This is the top of the schedule, i.e. the location where the scheduler
   /// is about to place the scheduled instructions. It gets updated as we
   /// schedule.
   std::optional<BasicBlock::iterator> ScheduleTopItOpt;
   // TODO: This is wasting memory in exchange for fast removal using a raw ptr.
   DenseMap<SchedBundle *, std::unique_ptr<SchedBundle>> Bndls;
+  /// The BB that we are currently scheduling.
+  BasicBlock *ScheduledBB = nullptr;
+  /// The ID of the callback we register with Sandbox IR.
+  std::optional<Context::CallbackID> CreateInstrCB;
+  /// Called by Sandbox IR's callback system, after \p I has been created.
+  /// NOTE: This should run after DAG's callback has run.
+  // TODO: Perhaps call DAG's notify function from within this one?
+  void notifyCreateInstr(Instruction *I);
 
   /// \Returns a scheduling bundle containing \p Instrs.
   SchedBundle *createBundle(ArrayRef<Instruction *> Instrs);
@@ -151,8 +192,16 @@ class Scheduler {
   Scheduler &operator=(const Scheduler &) = delete;
 
 public:
-  Scheduler(AAResults &AA, Context &Ctx) : DAG(AA, Ctx) {}
-  ~Scheduler() {}
+  Scheduler(AAResults &AA, Context &Ctx) : DAG(AA, Ctx), Ctx(Ctx) {
+    // NOTE: The scheduler's callback depends on the DAG's callback running
+    // before it and updating the DAG accordingly.
+    CreateInstrCB = Ctx.registerCreateInstrCallback(
+        [this](Instruction *I) { notifyCreateInstr(I); });
+  }
+  ~Scheduler() {
+    if (CreateInstrCB)
+      Ctx.unregisterCreateInstrCallback(*CreateInstrCB);
+  }
   /// Tries to build a schedule that includes all of \p Instrs scheduled at the
   /// same scheduling cycle. This essentially checks that there are no
   /// dependencies among \p Instrs. This function may involve scheduling
@@ -166,8 +215,10 @@ class Scheduler {
     DAG.clear();
     ReadyList.clear();
     ScheduleTopItOpt = std::nullopt;
+    ScheduledBB = nullptr;
     assert(Bndls.empty() && DAG.empty() && ReadyList.empty() &&
-           !ScheduleTopItOpt && "Expected empty state!");
+           !ScheduleTopItOpt && ScheduledBB == nullptr &&
+           "Expected empty state!");
   }
 
 #ifndef NDEBUG
@@ -176,6 +227,13 @@ class Scheduler {
 #endif
 };
 
+/// A client-attorney class for accessing the Scheduler's internals (used for
+/// unit tests).
+class SchedulerInternalsAttorney {
+public:
+  static DependencyGraph &getDAG(Scheduler &Sched) { return Sched.DAG; }
+};
+
 } // namespace llvm::sandboxir
 
 #endif // LLVM_TRANSFORMS_VECTORIZE_SANDBOXVECTORIZER_SCHEDULER_H
diff --git a/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/VecUtils.h b/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/VecUtils.h
index 64090febc5a09..bec1cecf241f6 100644
--- a/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/VecUtils.h
+++ b/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/VecUtils.h
@@ -111,10 +111,12 @@ class VecUtils {
     return LowestI;
   }
   /// \Returns the lowest instruction in \p Vals, or nullptr if no instructions
-  /// are found or if not in the same BB.
-  static Instruction *getLowest(ArrayRef<Value *> Vals) {
-    // Find the first Instruction in Vals.
-    auto It = find_if(Vals, [](Value *V) { return isa<Instruction>(V); });
+  /// are found. Skips instructions not in \p BB.
+  static Instruction *getLowest(ArrayRef<Value *> Vals, BasicBlock *BB) {
+    // Find the first Instruction in Vals that is also in `BB`.
+    auto It = find_if(Vals, [BB](Value *V) {
+      return isa<Instruction>(V) && cast<Instruction>(V)->getParent() == BB;
+    });
     // If we couldn't find an instruction return nullptr.
     if (It == Vals.end())
       return nullptr;
@@ -122,15 +124,14 @@ class VecUtils {
     // Now look for the lowest instruction in Vals starting from one position
     // after FirstI.
     Instruction *LowestI = FirstI;
-    auto *LowestBB = LowestI->getParent();
     for (auto *V : make_range(std::next(It), Vals.end())) {
       auto *I = dyn_cast<Instruction>(V);
       // Skip non-instructions.
       if (I == nullptr)
         continue;
-      // If the instructions are in different BBs return nullptr.
-      if (I->getParent() != LowestBB)
-        return nullptr;
+      // Skips instructions not in \p BB.
+      if (I->getParent() != BB)
+        continue;
       // If `LowestI` comes before `I` then `I` is the new lowest.
       if (LowestI->comesBefore(I))
         LowestI = I;
diff --git a/llvm/lib/Analysis/BasicAliasAnalysis.cpp b/llvm/lib/Analysis/BasicAliasAnalysis.cpp
index b2a3f3390e000..85b8253408ebf 100644
--- a/llvm/lib/Analysis/BasicAliasAnalysis.cpp
+++ b/llvm/lib/Analysis/BasicAliasAnalysis.cpp
@@ -1349,8 +1349,10 @@ AliasResult BasicAAResult::aliasGEP(
     const VariableGEPIndex &Var1 = DecompGEP1.VarIndices[1];
     if (Var0.hasNegatedScaleOf(Var1) && Var0.Val.TruncBits == 0 &&
         Var0.Val.hasSameCastsAs(Var1.Val) && !AAQI.MayBeCrossIteration &&
-        isKnownNonEqual(Var0.Val.V, Var1.Val.V, DL, &AC, /* CxtI */ nullptr,
-                        DT))
+        isKnownNonEqual(Var0.Val.V, Var1.Val.V,
+                        SimplifyQuery(DL, DT, &AC, /*CxtI=*/Var0.CxtI
+                                                       ? Var0.CxtI
+                                                       : Var1.CxtI)))
       MinAbsVarIndex = Var0.Scale.abs();
   }
 
diff --git a/llvm/lib/Analysis/CFGPrinter.cpp b/llvm/lib/Analysis/CFGPrinter.cpp
index af18fb6626e3b..38aad849755be 100644
--- a/llvm/lib/Analysis/CFGPrinter.cpp
+++ b/llvm/lib/Analysis/CFGPrinter.cpp
@@ -19,6 +19,7 @@
 
 #include "llvm/Analysis/CFGPrinter.h"
 #include "llvm/ADT/PostOrderIterator.h"
+#include "llvm/IR/ModuleSlotTracker.h"
 #include "llvm/Support/CommandLine.h"
 #include "llvm/Support/FileSystem.h"
 #include "llvm/Support/GraphWriter.h"
@@ -90,6 +91,22 @@ static void viewCFG(Function &F, const BlockFrequencyInfo *BFI,
   ViewGraph(&CFGInfo, "cfg." + F.getName(), CFGOnly);
 }
 
+DOTFuncInfo::DOTFuncInfo(const Function *F, const BlockFrequencyInfo *BFI,
+                         const BranchProbabilityInfo *BPI, uint64_t MaxFreq)
+    : F(F), BFI(BFI), BPI(BPI), MaxFreq(MaxFreq) {
+  ShowHeat = false;
+  EdgeWeights = !!BPI; // Print EdgeWeights when BPI is available.
+  RawWeights = !!BFI;  // Print RawWeights when BFI is available.
+}
+
+DOTFuncInfo::~DOTFuncInfo() = default;
+
+ModuleSlotTracker *DOTFuncInfo::getModuleSlotTracker() {
+  if (!MSTStorage)
+    MSTStorage = std::make_unique<ModuleSlotTracker>(F->getParent());
+  return &*MSTStorage;
+}
+
 PreservedAnalyses CFGViewerPass::run(Function &F, FunctionAnalysisManager &AM) {
   if (!CFGFuncName.empty() && !F.getName().contains(CFGFuncName))
     return PreservedAnalyses::all();
@@ -208,3 +225,38 @@ bool DOTGraphTraits<DOTFuncInfo *>::isNodeHidden(const BasicBlock *Node,
   }
   return false;
 }
+
+std::string DOTGraphTraits<DOTFuncInfo *>::getCompleteNodeLabel(
+    const BasicBlock *Node, DOTFuncInfo *CFGInfo,
+    function_ref<void(raw_string_ostream &, const BasicBlock &)>
+        HandleBasicBlock,
+    function_ref<void(std::string &, unsigned &, unsigned)> HandleComment) {
+  if (HandleBasicBlock)
+    return CompleteNodeLabelString(Node, HandleBasicBlock, HandleComment);
+
+  // Default basic block printing
+  std::optional<ModuleSlotTracker> MSTStorage;
+  ModuleSlotTracker *MST = nullptr;
+
+  if (CFGInfo) {
+    MST = CFGInfo->getModuleSlotTracker();
+  } else {
+    MSTStorage.emplace(Node->getModule());
+    MST = &*MSTStorage;
+  }
+
+  return CompleteNodeLabelString(
+      Node,
+      function_ref<void(raw_string_ostream &, const BasicBlock &)>(
+          [MST](raw_string_ostream &OS, const BasicBlock &Node) -> void {
+            // Prepend label name
+            Node.printAsOperand(OS, false, *MST);
+            OS << ":\n";
+
+            for (const Instruction &Inst : Node) {
+              Inst.print(OS, *MST, /* IsForDebug */ false);
+              OS << '\n';
+            }
+          }),
+      HandleComment);
+}
diff --git a/llvm/lib/Analysis/CMakeLists.txt b/llvm/lib/Analysis/CMakeLists.txt
index 0db5b80f336cb..a44f6c6a135ef 100644
--- a/llvm/lib/Analysis/CMakeLists.txt
+++ b/llvm/lib/Analysis/CMakeLists.txt
@@ -79,6 +79,7 @@ add_llvm_component_library(LLVMAnalysis
   InstructionPrecedenceTracking.cpp
   InstructionSimplify.cpp
   InteractiveModelRunner.cpp
+  KernelInfo.cpp
   LastRunTrackingAnalysis.cpp
   LazyBranchProbabilityInfo.cpp
   LazyBlockFrequencyInfo.cpp
diff --git a/llvm/lib/Analysis/CaptureTracking.cpp b/llvm/lib/Analysis/CaptureTracking.cpp
index a353842eb809c..49baf2eb84bb3 100644
--- a/llvm/lib/Analysis/CaptureTracking.cpp
+++ b/llvm/lib/Analysis/CaptureTracking.cpp
@@ -318,8 +318,8 @@ UseCaptureKind llvm::DetermineUseCaptureKind(
       return UseCaptureKind::NO_CAPTURE;
 
     // Not captured if only passed via 'nocapture' arguments.
-    if (Call->isDataOperand(&U) &&
-        !Call->doesNotCapture(Call->getDataOperandNo(&U))) {
+    assert(Call->isDataOperand(&U) && "Non-callee must be data operand");
+    if (!Call->doesNotCapture(Call->getDataOperandNo(&U))) {
       // The parameter is not marked 'nocapture' - captured.
       return UseCaptureKind::MAY_CAPTURE;
     }
diff --git a/llvm/lib/Analysis/CmpInstAnalysis.cpp b/llvm/lib/Analysis/CmpInstAnalysis.cpp
index 3599428c5ff41..5c0d1dd1c74b0 100644
--- a/llvm/lib/Analysis/CmpInstAnalysis.cpp
+++ b/llvm/lib/Analysis/CmpInstAnalysis.cpp
@@ -168,6 +168,7 @@ llvm::decomposeBitTestICmp(Value *LHS, Value *RHS, CmpInst::Predicate Pred,
 
 std::optional<DecomposedBitTest>
 llvm::decomposeBitTest(Value *Cond, bool LookThruTrunc, bool AllowNonZeroC) {
+  using namespace PatternMatch;
   if (auto *ICmp = dyn_cast<ICmpInst>(Cond)) {
     // Don't allow pointers. Splat vectors are fine.
     if (!ICmp->getOperand(0)->getType()->isIntOrIntVectorTy())
@@ -176,6 +177,19 @@ llvm::decomposeBitTest(Value *Cond, bool LookThruTrunc, bool AllowNonZeroC) {
                                 ICmp->getPredicate(), LookThruTrunc,
                                 AllowNonZeroC);
   }
+  Value *X;
+  if (Cond->getType()->isIntOrIntVectorTy(1) &&
+      (match(Cond, m_Trunc(m_Value(X))) ||
+       match(Cond, m_Not(m_Trunc(m_Value(X)))))) {
+    DecomposedBitTest Result;
+    Result.X = X;
+    unsigned BitWidth = X->getType()->getScalarSizeInBits();
+    Result.Mask = APInt(BitWidth, 1);
+    Result.C = APInt::getZero(BitWidth);
+    Result.Pred = isa<TruncInst>(Cond) ? ICmpInst::ICMP_NE : ICmpInst::ICMP_EQ;
+
+    return Result;
+  }
 
   return std::nullopt;
 }
diff --git a/llvm/lib/Analysis/ConstantFolding.cpp b/llvm/lib/Analysis/ConstantFolding.cpp
index 80c1277e63165..d645bf8f7b621 100644
--- a/llvm/lib/Analysis/ConstantFolding.cpp
+++ b/llvm/lib/Analysis/ConstantFolding.cpp
@@ -1258,11 +1258,16 @@ Constant *llvm::ConstantFoldCompareInstOperands(
     if (Ops0->getType()->isPointerTy() && !ICmpInst::isSigned(Predicate)) {
       unsigned IndexWidth = DL.getIndexTypeSizeInBits(Ops0->getType());
       APInt Offset0(IndexWidth, 0);
-      Value *Stripped0 =
-          Ops0->stripAndAccumulateInBoundsConstantOffsets(DL, Offset0);
+      bool IsEqPred = ICmpInst::isEquality(Predicate);
+      Value *Stripped0 = Ops0->stripAndAccumulateConstantOffsets(
+          DL, Offset0, /*AllowNonInbounds=*/IsEqPred,
+          /*AllowInvariantGroup=*/false, /*ExternalAnalysis=*/nullptr,
+          /*LookThroughIntToPtr=*/IsEqPred);
       APInt Offset1(IndexWidth, 0);
-      Value *Stripped1 =
-          Ops1->stripAndAccumulateInBoundsConstantOffsets(DL, Offset1);
+      Value *Stripped1 = Ops1->stripAndAccumulateConstantOffsets(
+          DL, Offset1, /*AllowNonInbounds=*/IsEqPred,
+          /*AllowInvariantGroup=*/false, /*ExternalAnalysis=*/nullptr,
+          /*LookThroughIntToPtr=*/IsEqPred);
       if (Stripped0 == Stripped1)
         return ConstantInt::getBool(
             Ops0->getContext(),
diff --git a/llvm/lib/Analysis/DDG.cpp b/llvm/lib/Analysis/DDG.cpp
index a0774096c5129..0907a7fb021fc 100644
--- a/llvm/lib/Analysis/DDG.cpp
+++ b/llvm/lib/Analysis/DDG.cpp
@@ -241,9 +241,10 @@ bool DataDependenceGraph::addNode(DDGNode &N) {
 }
 
 const PiBlockDDGNode *DataDependenceGraph::getPiBlock(const NodeType &N) const {
-  if (!PiBlockMap.contains(&N))
+  auto It = PiBlockMap.find(&N);
+  if (It == PiBlockMap.end())
     return nullptr;
-  auto *Pi = PiBlockMap.find(&N)->second;
+  auto *Pi = It->second;
   assert(!PiBlockMap.contains(Pi) && "Nested pi-blocks detected.");
   return Pi;
 }
diff --git a/llvm/lib/Analysis/DependenceAnalysis.cpp b/llvm/lib/Analysis/DependenceAnalysis.cpp
index a4a98ea0bae14..cd252c62ba9cd 100644
--- a/llvm/lib/Analysis/DependenceAnalysis.cpp
+++ b/llvm/lib/Analysis/DependenceAnalysis.cpp
@@ -187,7 +187,7 @@ static void dumpExampleDependence(raw_ostream &OS, DependenceInfo *DA,
         if (DstI->mayReadOrWriteMemory()) {
           OS << "Src:" << *SrcI << " --> Dst:" << *DstI << "\n";
           OS << "  da analyze - ";
-          if (auto D = DA->depends(&*SrcI, &*DstI, true)) {
+          if (auto D = DA->depends(&*SrcI, &*DstI)) {
             // Normalize negative direction vectors if required by clients.
             if (NormalizeResults && D->normalize(&SE))
                 OS << "normalized - ";
@@ -216,7 +216,8 @@ void DependenceAnalysisWrapperPass::print(raw_ostream &OS,
 
 PreservedAnalyses
 DependenceAnalysisPrinterPass::run(Function &F, FunctionAnalysisManager &FAM) {
-  OS << "'Dependence Analysis' for function '" << F.getName() << "':\n";
+  OS << "Printing analysis 'Dependence Analysis' for function '" << F.getName()
+     << "':\n";
   dumpExampleDependence(OS, &FAM.getResult<DependenceAnalysis>(F),
                         FAM.getResult<ScalarEvolutionAnalysis>(F),
                         NormalizeResults);
@@ -722,7 +723,10 @@ static AliasResult underlyingObjectsAlias(AAResults *AA,
       MemoryLocation::getBeforeOrAfter(LocA.Ptr, LocA.AATags);
   MemoryLocation LocBS =
       MemoryLocation::getBeforeOrAfter(LocB.Ptr, LocB.AATags);
-  if (AA->isNoAlias(LocAS, LocBS))
+  BatchAAResults BAA(*AA);
+  BAA.enableCrossIterationMode();
+
+  if (BAA.isNoAlias(LocAS, LocBS))
     return AliasResult::NoAlias;
 
   // Check the underlying objects are the same
@@ -743,7 +747,6 @@ static AliasResult underlyingObjectsAlias(AAResults *AA,
   return AliasResult::NoAlias;
 }
 
-
 // Returns true if the load or store can be analyzed. Atomic and volatile
 // operations have properties which this analysis does not understand.
 static
@@ -3586,8 +3589,8 @@ bool DependenceInfo::invalidate(Function &F, const PreservedAnalyses &PA,
 // Care is required to keep the routine below, getSplitIteration(),
 // up to date with respect to this routine.
 std::unique_ptr<Dependence>
-DependenceInfo::depends(Instruction *Src, Instruction *Dst,
-                        bool PossiblyLoopIndependent) {
+DependenceInfo::depends(Instruction *Src, Instruction *Dst) {
+  bool PossiblyLoopIndependent = true;
   if (Src == Dst)
     PossiblyLoopIndependent = false;
 
diff --git a/llvm/lib/Analysis/DependenceGraphBuilder.cpp b/llvm/lib/Analysis/DependenceGraphBuilder.cpp
index 7ee2adf49ebb4..5664e2f27a61a 100644
--- a/llvm/lib/Analysis/DependenceGraphBuilder.cpp
+++ b/llvm/lib/Analysis/DependenceGraphBuilder.cpp
@@ -240,9 +240,7 @@ template <class G> void AbstractDependenceGraphBuilder<G>::createDefUseEdges() {
         Instruction *UI = dyn_cast<Instruction>(U);
         if (!UI)
           continue;
-        NodeType *DstNode = nullptr;
-        if (IMap.find(UI) != IMap.end())
-          DstNode = IMap.find(UI)->second;
+        NodeType *DstNode = IMap.lookup(UI);
 
         // In the case of loops, the scope of the subgraph is all the
         // basic blocks (and instructions within them) belonging to the loop. We
@@ -297,7 +295,7 @@ void AbstractDependenceGraphBuilder<G>::createMemoryDependencyEdges() {
       bool BackwardEdgeCreated = false;
       for (Instruction *ISrc : SrcIList) {
         for (Instruction *IDst : DstIList) {
-          auto D = DI.depends(ISrc, IDst, true);
+          auto D = DI.depends(ISrc, IDst);
           if (!D)
             continue;
 
diff --git a/llvm/lib/Analysis/InstructionSimplify.cpp b/llvm/lib/Analysis/InstructionSimplify.cpp
index d69747e30f884..59002cd934ab1 100644
--- a/llvm/lib/Analysis/InstructionSimplify.cpp
+++ b/llvm/lib/Analysis/InstructionSimplify.cpp
@@ -27,6 +27,7 @@
 #include "llvm/Analysis/CmpInstAnalysis.h"
 #include "llvm/Analysis/ConstantFolding.h"
 #include "llvm/Analysis/InstSimplifyFolder.h"
+#include "llvm/Analysis/Loads.h"
 #include "llvm/Analysis/LoopAnalysisManager.h"
 #include "llvm/Analysis/MemoryBuiltins.h"
 #include "llvm/Analysis/OverflowInstAnalysis.h"
@@ -2685,27 +2686,14 @@ static Constant *computePointerICmp(CmpPredicate Pred, Value *LHS, Value *RHS,
   const DataLayout &DL = Q.DL;
   const TargetLibraryInfo *TLI = Q.TLI;
 
-  // We can only fold certain predicates on pointer comparisons.
-  switch (Pred) {
-  default:
+  // We fold equality and unsigned predicates on pointer comparisons, but forbid
+  // signed predicates since a GEP with inbounds could cross the sign boundary.
+  if (CmpInst::isSigned(Pred))
     return nullptr;
 
-    // Equality comparisons are easy to fold.
-  case CmpInst::ICMP_EQ:
-  case CmpInst::ICMP_NE:
-    break;
-
-    // We can only handle unsigned relational comparisons because 'inbounds' on
-    // a GEP only protects against unsigned wrapping.
-  case CmpInst::ICMP_UGT:
-  case CmpInst::ICMP_UGE:
-  case CmpInst::ICMP_ULT:
-  case CmpInst::ICMP_ULE:
-    // However, we have to switch them to their signed variants to handle
-    // negative indices from the base pointer.
-    Pred = ICmpInst::getSignedPredicate(Pred);
-    break;
-  }
+  // We have to switch to a signed predicate to handle negative indices from
+  // the base pointer.
+  Pred = ICmpInst::getSignedPredicate(Pred);
 
   // Strip off any constant offsets so that we can reason about them.
   // It's tempting to use getUnderlyingObject or even just stripInBoundsOffsets
@@ -2729,7 +2717,7 @@ static Constant *computePointerICmp(CmpPredicate Pred, Value *LHS, Value *RHS,
                             ICmpInst::compare(LHSOffset, RHSOffset, Pred));
 
   // Various optimizations for (in)equality comparisons.
-  if (Pred == CmpInst::ICMP_EQ || Pred == CmpInst::ICMP_NE) {
+  if (ICmpInst::isEquality(Pred)) {
     // Different non-empty allocations that exist at the same time have
     // different addresses (if the program can tell). If the offsets are
     // within the bounds of their allocations (and not one-past-the-end!
@@ -3994,7 +3982,7 @@ static Value *simplifyICmpInst(CmpPredicate Pred, Value *LHS, Value *RHS,
   // This is potentially expensive, and we have already computedKnownBits for
   // compares with 0 above here, so only try this for a non-zero compare.
   if (ICmpInst::isEquality(Pred) && !match(RHS, m_Zero()) &&
-      isKnownNonEqual(LHS, RHS, Q.DL, Q.AC, Q.CxtI, Q.DT, Q.IIQ.UseInstrInfo)) {
+      isKnownNonEqual(LHS, RHS, Q)) {
     return Pred == ICmpInst::ICMP_NE ? getTrue(ITy) : getFalse(ITy);
   }
 
@@ -4612,12 +4600,11 @@ static Value *simplifyCmpSelOfMaxMin(Value *CmpLHS, Value *CmpRHS,
   return nullptr;
 }
 
-/// An alternative way to test if a bit is set or not uses sgt/slt instead of
-/// eq/ne.
-static Value *simplifySelectWithFakeICmpEq(Value *CmpLHS, Value *CmpRHS,
-                                           CmpPredicate Pred, Value *TrueVal,
-                                           Value *FalseVal) {
-  if (auto Res = decomposeBitTestICmp(CmpLHS, CmpRHS, Pred))
+/// An alternative way to test if a bit is set or not.
+/// uses e.g. sgt/slt or trunc instead of eq/ne.
+static Value *simplifySelectWithBitTest(Value *CondVal, Value *TrueVal,
+                                        Value *FalseVal) {
+  if (auto Res = decomposeBitTest(CondVal))
     return simplifySelectBitTest(TrueVal, FalseVal, Res->X, &Res->Mask,
                                  Res->Pred == ICmpInst::ICMP_EQ);
 
@@ -4728,21 +4715,20 @@ static Value *simplifySelectWithICmpCond(Value *CondVal, Value *TrueVal,
       return FalseVal;
   }
 
-  // Check for other compares that behave like bit test.
-  if (Value *V =
-          simplifySelectWithFakeICmpEq(CmpLHS, CmpRHS, Pred, TrueVal, FalseVal))
-    return V;
-
   // If we have a scalar equality comparison, then we know the value in one of
   // the arms of the select. See if substituting this value into the arm and
   // simplifying the result yields the same value as the other arm.
   if (Pred == ICmpInst::ICMP_EQ) {
-    if (Value *V = simplifySelectWithEquivalence({{CmpLHS, CmpRHS}}, TrueVal,
-                                                 FalseVal, Q, MaxRecurse))
-      return V;
-    if (Value *V = simplifySelectWithEquivalence({{CmpRHS, CmpLHS}}, TrueVal,
-                                                 FalseVal, Q, MaxRecurse))
-      return V;
+    if (CmpLHS->getType()->isIntOrIntVectorTy() ||
+        canReplacePointersIfEqual(CmpLHS, CmpRHS, Q.DL))
+      if (Value *V = simplifySelectWithEquivalence({{CmpLHS, CmpRHS}}, TrueVal,
+                                                   FalseVal, Q, MaxRecurse))
+        return V;
+    if (CmpLHS->getType()->isIntOrIntVectorTy() ||
+        canReplacePointersIfEqual(CmpRHS, CmpLHS, Q.DL))
+      if (Value *V = simplifySelectWithEquivalence({{CmpRHS, CmpLHS}}, TrueVal,
+                                                   FalseVal, Q, MaxRecurse))
+        return V;
 
     Value *X;
     Value *Y;
@@ -4984,6 +4970,9 @@ static Value *simplifySelectInst(Value *Cond, Value *TrueVal, Value *FalseVal,
           simplifySelectWithICmpCond(Cond, TrueVal, FalseVal, Q, MaxRecurse))
     return V;
 
+  if (Value *V = simplifySelectWithBitTest(Cond, TrueVal, FalseVal))
+    return V;
+
   if (Value *V = simplifySelectWithFCmp(Cond, TrueVal, FalseVal, Q, MaxRecurse))
     return V;
 
diff --git a/llvm/lib/Analysis/KernelInfo.cpp b/llvm/lib/Analysis/KernelInfo.cpp
new file mode 100644
index 0000000000000..9c8f8699401ce
--- /dev/null
+++ b/llvm/lib/Analysis/KernelInfo.cpp
@@ -0,0 +1,326 @@
+//===- KernelInfo.cpp - Kernel Analysis -----------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file defines the KernelInfoPrinter class used to emit remarks about
+// function properties from a GPU kernel.
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/Analysis/KernelInfo.h"
+#include "llvm/ADT/SmallString.h"
+#include "llvm/ADT/StringExtras.h"
+#include "llvm/Analysis/OptimizationRemarkEmitter.h"
+#include "llvm/Analysis/TargetTransformInfo.h"
+#include "llvm/IR/DebugInfo.h"
+#include "llvm/IR/Dominators.h"
+#include "llvm/IR/Instructions.h"
+#include "llvm/IR/Metadata.h"
+#include "llvm/IR/Module.h"
+#include "llvm/IR/PassManager.h"
+
+using namespace llvm;
+
+#define DEBUG_TYPE "kernel-info"
+
+namespace {
+
+/// Data structure holding function info for kernels.
+class KernelInfo {
+  void updateForBB(const BasicBlock &BB, OptimizationRemarkEmitter &ORE);
+
+public:
+  static void emitKernelInfo(Function &F, FunctionAnalysisManager &FAM,
+                             TargetMachine *TM);
+
+  /// Whether the function has external linkage and is not a kernel function.
+  bool ExternalNotKernel = false;
+
+  /// Launch bounds.
+  SmallVector<std::pair<StringRef, int64_t>> LaunchBounds;
+
+  /// The number of alloca instructions inside the function, the number of those
+  /// with allocation sizes that cannot be determined at compile time, and the
+  /// sum of the sizes that can be.
+  ///
+  /// With the current implementation for at least some GPU archs,
+  /// AllocasDyn > 0 might not be possible, but we report AllocasDyn anyway in
+  /// case the implementation changes.
+  int64_t Allocas = 0;
+  int64_t AllocasDyn = 0;
+  int64_t AllocasStaticSizeSum = 0;
+
+  /// Number of direct/indirect calls (anything derived from CallBase).
+  int64_t DirectCalls = 0;
+  int64_t IndirectCalls = 0;
+
+  /// Number of direct calls made from this function to other functions
+  /// defined in this module.
+  int64_t DirectCallsToDefinedFunctions = 0;
+
+  /// Number of direct calls to inline assembly.
+  int64_t InlineAssemblyCalls = 0;
+
+  /// Number of calls of type InvokeInst.
+  int64_t Invokes = 0;
+
+  /// Target-specific flat address space.
+  unsigned FlatAddrspace;
+
+  /// Number of flat address space memory accesses (via load, store, etc.).
+  int64_t FlatAddrspaceAccesses = 0;
+};
+
+} // end anonymous namespace
+
+static void identifyCallee(OptimizationRemark &R, const Module *M,
+                           const Value *V, StringRef Kind = "") {
+  SmallString<100> Name; // might be function name or asm expression
+  if (const Function *F = dyn_cast<Function>(V)) {
+    if (auto *SubProgram = F->getSubprogram()) {
+      if (SubProgram->isArtificial())
+        R << "artificial ";
+      Name = SubProgram->getName();
+    }
+  }
+  if (Name.empty()) {
+    raw_svector_ostream OS(Name);
+    V->printAsOperand(OS, /*PrintType=*/false, M);
+  }
+  if (!Kind.empty())
+    R << Kind << " ";
+  R << "'" << Name << "'";
+}
+
+static void identifyFunction(OptimizationRemark &R, const Function &F) {
+  identifyCallee(R, F.getParent(), &F, "function");
+}
+
+static void remarkAlloca(OptimizationRemarkEmitter &ORE, const Function &Caller,
+                         const AllocaInst &Alloca,
+                         TypeSize::ScalarTy StaticSize) {
+  ORE.emit([&] {
+    StringRef DbgName;
+    DebugLoc Loc;
+    bool Artificial = false;
+    auto DVRs = findDVRDeclares(&const_cast<AllocaInst &>(Alloca));
+    if (!DVRs.empty()) {
+      const DbgVariableRecord &DVR = **DVRs.begin();
+      DbgName = DVR.getVariable()->getName();
+      Loc = DVR.getDebugLoc();
+      Artificial = DVR.Variable->isArtificial();
+    }
+    OptimizationRemark R(DEBUG_TYPE, "Alloca", DiagnosticLocation(Loc),
+                         Alloca.getParent());
+    R << "in ";
+    identifyFunction(R, Caller);
+    R << ", ";
+    if (Artificial)
+      R << "artificial ";
+    SmallString<20> ValName;
+    raw_svector_ostream OS(ValName);
+    Alloca.printAsOperand(OS, /*PrintType=*/false, Caller.getParent());
+    R << "alloca ('" << ValName << "') ";
+    if (!DbgName.empty())
+      R << "for '" << DbgName << "' ";
+    else
+      R << "without debug info ";
+    R << "with ";
+    if (StaticSize)
+      R << "static size of " << itostr(StaticSize) << " bytes";
+    else
+      R << "dynamic size";
+    return R;
+  });
+}
+
+static void remarkCall(OptimizationRemarkEmitter &ORE, const Function &Caller,
+                       const CallBase &Call, StringRef CallKind,
+                       StringRef RemarkKind) {
+  ORE.emit([&] {
+    OptimizationRemark R(DEBUG_TYPE, RemarkKind, &Call);
+    R << "in ";
+    identifyFunction(R, Caller);
+    R << ", " << CallKind << ", callee is ";
+    identifyCallee(R, Caller.getParent(), Call.getCalledOperand());
+    return R;
+  });
+}
+
+static void remarkFlatAddrspaceAccess(OptimizationRemarkEmitter &ORE,
+                                      const Function &Caller,
+                                      const Instruction &Inst) {
+  ORE.emit([&] {
+    OptimizationRemark R(DEBUG_TYPE, "FlatAddrspaceAccess", &Inst);
+    R << "in ";
+    identifyFunction(R, Caller);
+    if (const IntrinsicInst *II = dyn_cast<IntrinsicInst>(&Inst)) {
+      R << ", '" << II->getCalledFunction()->getName() << "' call";
+    } else {
+      R << ", '" << Inst.getOpcodeName() << "' instruction";
+    }
+    if (!Inst.getType()->isVoidTy()) {
+      SmallString<20> Name;
+      raw_svector_ostream OS(Name);
+      Inst.printAsOperand(OS, /*PrintType=*/false, Caller.getParent());
+      R << " ('" << Name << "')";
+    }
+    R << " accesses memory in flat address space";
+    return R;
+  });
+}
+
+void KernelInfo::updateForBB(const BasicBlock &BB,
+                             OptimizationRemarkEmitter &ORE) {
+  const Function &F = *BB.getParent();
+  const Module &M = *F.getParent();
+  const DataLayout &DL = M.getDataLayout();
+  for (const Instruction &I : BB.instructionsWithoutDebug()) {
+    if (const AllocaInst *Alloca = dyn_cast<AllocaInst>(&I)) {
+      ++Allocas;
+      TypeSize::ScalarTy StaticSize = 0;
+      if (std::optional<TypeSize> Size = Alloca->getAllocationSize(DL)) {
+        StaticSize = Size->getFixedValue();
+        assert(StaticSize <=
+               (TypeSize::ScalarTy)std::numeric_limits<int64_t>::max());
+        AllocasStaticSizeSum += StaticSize;
+      } else {
+        ++AllocasDyn;
+      }
+      remarkAlloca(ORE, F, *Alloca, StaticSize);
+    } else if (const CallBase *Call = dyn_cast<CallBase>(&I)) {
+      SmallString<40> CallKind;
+      SmallString<40> RemarkKind;
+      if (Call->isIndirectCall()) {
+        ++IndirectCalls;
+        CallKind += "indirect";
+        RemarkKind += "Indirect";
+      } else {
+        ++DirectCalls;
+        CallKind += "direct";
+        RemarkKind += "Direct";
+      }
+      if (isa<InvokeInst>(Call)) {
+        ++Invokes;
+        CallKind += " invoke";
+        RemarkKind += "Invoke";
+      } else {
+        CallKind += " call";
+        RemarkKind += "Call";
+      }
+      if (!Call->isIndirectCall()) {
+        if (const Function *Callee = Call->getCalledFunction()) {
+          if (!Callee->isIntrinsic() && !Callee->isDeclaration()) {
+            ++DirectCallsToDefinedFunctions;
+            CallKind += " to defined function";
+            RemarkKind += "ToDefinedFunction";
+          }
+        } else if (Call->isInlineAsm()) {
+          ++InlineAssemblyCalls;
+          CallKind += " to inline assembly";
+          RemarkKind += "ToInlineAssembly";
+        }
+      }
+      remarkCall(ORE, F, *Call, CallKind, RemarkKind);
+      if (const AnyMemIntrinsic *MI = dyn_cast<AnyMemIntrinsic>(Call)) {
+        if (MI->getDestAddressSpace() == FlatAddrspace) {
+          ++FlatAddrspaceAccesses;
+          remarkFlatAddrspaceAccess(ORE, F, I);
+        } else if (const AnyMemTransferInst *MT =
+                       dyn_cast<AnyMemTransferInst>(MI)) {
+          if (MT->getSourceAddressSpace() == FlatAddrspace) {
+            ++FlatAddrspaceAccesses;
+            remarkFlatAddrspaceAccess(ORE, F, I);
+          }
+        }
+      }
+    } else if (const LoadInst *Load = dyn_cast<LoadInst>(&I)) {
+      if (Load->getPointerAddressSpace() == FlatAddrspace) {
+        ++FlatAddrspaceAccesses;
+        remarkFlatAddrspaceAccess(ORE, F, I);
+      }
+    } else if (const StoreInst *Store = dyn_cast<StoreInst>(&I)) {
+      if (Store->getPointerAddressSpace() == FlatAddrspace) {
+        ++FlatAddrspaceAccesses;
+        remarkFlatAddrspaceAccess(ORE, F, I);
+      }
+    } else if (const AtomicRMWInst *At = dyn_cast<AtomicRMWInst>(&I)) {
+      if (At->getPointerAddressSpace() == FlatAddrspace) {
+        ++FlatAddrspaceAccesses;
+        remarkFlatAddrspaceAccess(ORE, F, I);
+      }
+    } else if (const AtomicCmpXchgInst *At = dyn_cast<AtomicCmpXchgInst>(&I)) {
+      if (At->getPointerAddressSpace() == FlatAddrspace) {
+        ++FlatAddrspaceAccesses;
+        remarkFlatAddrspaceAccess(ORE, F, I);
+      }
+    }
+  }
+}
+
+static void remarkProperty(OptimizationRemarkEmitter &ORE, const Function &F,
+                           StringRef Name, int64_t Value) {
+  ORE.emit([&] {
+    OptimizationRemark R(DEBUG_TYPE, Name, &F);
+    R << "in ";
+    identifyFunction(R, F);
+    R << ", " << Name << " = " << itostr(Value);
+    return R;
+  });
+}
+
+static std::optional<int64_t> parseFnAttrAsInteger(Function &F,
+                                                   StringRef Name) {
+  if (!F.hasFnAttribute(Name))
+    return std::nullopt;
+  return F.getFnAttributeAsParsedInteger(Name);
+}
+
+void KernelInfo::emitKernelInfo(Function &F, FunctionAnalysisManager &FAM,
+                                TargetMachine *TM) {
+  KernelInfo KI;
+  TargetTransformInfo &TheTTI = FAM.getResult<TargetIRAnalysis>(F);
+  KI.FlatAddrspace = TheTTI.getFlatAddressSpace();
+
+  // Record function properties.
+  KI.ExternalNotKernel = F.hasExternalLinkage() && !F.hasKernelCallingConv();
+  for (StringRef Name : {"omp_target_num_teams", "omp_target_thread_limit"}) {
+    if (auto Val = parseFnAttrAsInteger(F, Name))
+      KI.LaunchBounds.push_back({Name, *Val});
+  }
+  TheTTI.collectKernelLaunchBounds(F, KI.LaunchBounds);
+
+  auto &ORE = FAM.getResult<OptimizationRemarkEmitterAnalysis>(F);
+  for (const auto &BB : F)
+    KI.updateForBB(BB, ORE);
+
+#define REMARK_PROPERTY(PROP_NAME)                                             \
+  remarkProperty(ORE, F, #PROP_NAME, KI.PROP_NAME)
+  REMARK_PROPERTY(ExternalNotKernel);
+  for (auto LB : KI.LaunchBounds)
+    remarkProperty(ORE, F, LB.first, LB.second);
+  REMARK_PROPERTY(Allocas);
+  REMARK_PROPERTY(AllocasStaticSizeSum);
+  REMARK_PROPERTY(AllocasDyn);
+  REMARK_PROPERTY(DirectCalls);
+  REMARK_PROPERTY(IndirectCalls);
+  REMARK_PROPERTY(DirectCallsToDefinedFunctions);
+  REMARK_PROPERTY(InlineAssemblyCalls);
+  REMARK_PROPERTY(Invokes);
+  REMARK_PROPERTY(FlatAddrspaceAccesses);
+#undef REMARK_PROPERTY
+
+  return;
+}
+
+PreservedAnalyses KernelInfoPrinter::run(Function &F,
+                                         FunctionAnalysisManager &AM) {
+  // Skip it if remarks are not enabled as it will do nothing useful.
+  if (F.getContext().getDiagHandlerPtr()->isPassedOptRemarkEnabled(DEBUG_TYPE))
+    KernelInfo::emitKernelInfo(F, AM, TM);
+  return PreservedAnalyses::all();
+}
diff --git a/llvm/lib/Analysis/LazyValueInfo.cpp b/llvm/lib/Analysis/LazyValueInfo.cpp
index 20f69a0955f51..5cd179df436de 100644
--- a/llvm/lib/Analysis/LazyValueInfo.cpp
+++ b/llvm/lib/Analysis/LazyValueInfo.cpp
@@ -398,6 +398,8 @@ class LazyValueInfoImpl {
   std::optional<ValueLatticeElement>
   getValueFromICmpCondition(Value *Val, ICmpInst *ICI, bool isTrueDest,
                             bool UseBlockValue);
+  ValueLatticeElement getValueFromTrunc(Value *Val, TruncInst *Trunc,
+                                        bool IsTrueDest);
 
   std::optional<ValueLatticeElement>
   getValueFromCondition(Value *Val, Value *Cond, bool IsTrueDest,
@@ -622,10 +624,12 @@ LazyValueInfoImpl::solveBlockValueImpl(Value *Val, BasicBlock *BB) {
   return getFromRangeMetadata(BBI);
 }
 
-static void AddNonNullPointer(Value *Ptr, NonNullPointerSet &PtrSet) {
+static void AddNonNullPointer(Value *Ptr, NonNullPointerSet &PtrSet,
+                              bool IsDereferenced = true) {
   // TODO: Use NullPointerIsDefined instead.
   if (Ptr->getType()->getPointerAddressSpace() == 0)
-    PtrSet.insert(getUnderlyingObject(Ptr));
+    PtrSet.insert(IsDereferenced ? getUnderlyingObject(Ptr)
+                                 : Ptr->stripInBoundsOffsets());
 }
 
 static void AddNonNullPointersByInstruction(
@@ -644,6 +648,13 @@ static void AddNonNullPointersByInstruction(
     AddNonNullPointer(MI->getRawDest(), PtrSet);
     if (MemTransferInst *MTI = dyn_cast<MemTransferInst>(MI))
       AddNonNullPointer(MTI->getRawSource(), PtrSet);
+  } else if (auto *CB = dyn_cast<CallBase>(I)) {
+    for (auto &U : CB->args()) {
+      if (U->getType()->isPointerTy() &&
+          CB->paramHasNonNullAttr(CB->getArgOperandNo(&U),
+                                  /*AllowUndefOrPoison=*/false))
+        AddNonNullPointer(U.get(), PtrSet, /*IsDereferenced=*/false);
+    }
   }
 }
 
@@ -1283,6 +1294,27 @@ std::optional<ValueLatticeElement> LazyValueInfoImpl::getValueFromICmpCondition(
   return ValueLatticeElement::getOverdefined();
 }
 
+ValueLatticeElement LazyValueInfoImpl::getValueFromTrunc(Value *Val,
+                                                         TruncInst *Trunc,
+                                                         bool IsTrueDest) {
+  assert(Trunc->getType()->isIntOrIntVectorTy(1));
+
+  if (Trunc->getOperand(0) != Val)
+    return ValueLatticeElement::getOverdefined();
+
+  Type *Ty = Val->getType();
+
+  if (Trunc->hasNoUnsignedWrap()) {
+    if (IsTrueDest)
+      return ValueLatticeElement::get(ConstantInt::get(Ty, 1));
+    return ValueLatticeElement::get(Constant::getNullValue(Ty));
+  }
+
+  if (IsTrueDest)
+    return ValueLatticeElement::getNot(Constant::getNullValue(Ty));
+  return ValueLatticeElement::getNot(Constant::getAllOnesValue(Ty));
+}
+
 // Handle conditions of the form
 // extractvalue(op.with.overflow(%x, C), 1).
 static ValueLatticeElement getValueFromOverflowCondition(
@@ -1312,6 +1344,9 @@ LazyValueInfoImpl::getValueFromCondition(Value *Val, Value *Cond,
   if (ICmpInst *ICI = dyn_cast<ICmpInst>(Cond))
     return getValueFromICmpCondition(Val, ICI, IsTrueDest, UseBlockValue);
 
+  if (auto *Trunc = dyn_cast<TruncInst>(Cond))
+    return getValueFromTrunc(Val, Trunc, IsTrueDest);
+
   if (auto *EVI = dyn_cast<ExtractValueInst>(Cond))
     if (auto *WO = dyn_cast<WithOverflowInst>(EVI->getAggregateOperand()))
       if (EVI->getNumIndices() == 1 && *EVI->idx_begin() == 1)
diff --git a/llvm/lib/Analysis/Loads.cpp b/llvm/lib/Analysis/Loads.cpp
index 9279f19b72a3f..733a7988e5a73 100644
--- a/llvm/lib/Analysis/Loads.cpp
+++ b/llvm/lib/Analysis/Loads.cpp
@@ -13,6 +13,7 @@
 #include "llvm/Analysis/Loads.h"
 #include "llvm/Analysis/AliasAnalysis.h"
 #include "llvm/Analysis/AssumeBundleQueries.h"
+#include "llvm/Analysis/LoopAccessAnalysis.h"
 #include "llvm/Analysis/LoopInfo.h"
 #include "llvm/Analysis/MemoryBuiltins.h"
 #include "llvm/Analysis/MemoryLocation.h"
@@ -277,84 +278,90 @@ static bool AreEquivalentAddressValues(const Value *A, const Value *B) {
 bool llvm::isDereferenceableAndAlignedInLoop(
     LoadInst *LI, Loop *L, ScalarEvolution &SE, DominatorTree &DT,
     AssumptionCache *AC, SmallVectorImpl<const SCEVPredicate *> *Predicates) {
+  const Align Alignment = LI->getAlign();
   auto &DL = LI->getDataLayout();
   Value *Ptr = LI->getPointerOperand();
-
   APInt EltSize(DL.getIndexTypeSizeInBits(Ptr->getType()),
                 DL.getTypeStoreSize(LI->getType()).getFixedValue());
-  const Align Alignment = LI->getAlign();
-
-  Instruction *HeaderFirstNonPHI = &*L->getHeader()->getFirstNonPHIIt();
 
   // If given a uniform (i.e. non-varying) address, see if we can prove the
   // access is safe within the loop w/o needing predication.
   if (L->isLoopInvariant(Ptr))
-    return isDereferenceableAndAlignedPointer(Ptr, Alignment, EltSize, DL,
-                                              HeaderFirstNonPHI, AC, &DT);
+    return isDereferenceableAndAlignedPointer(
+        Ptr, Alignment, EltSize, DL, &*L->getHeader()->getFirstNonPHIIt(), AC,
+        &DT);
+
+  const SCEV *PtrScev = SE.getSCEV(Ptr);
+  auto *AddRec = dyn_cast<SCEVAddRecExpr>(PtrScev);
 
-  // Otherwise, check to see if we have a repeating access pattern where we can
-  // prove that all accesses are well aligned and dereferenceable.
-  auto *AddRec = dyn_cast<SCEVAddRecExpr>(SE.getSCEV(Ptr));
+  // Check to see if we have a repeating access pattern and it's possible
+  // to prove all accesses are well aligned.
   if (!AddRec || AddRec->getLoop() != L || !AddRec->isAffine())
     return false;
-  auto* Step = dyn_cast<SCEVConstant>(AddRec->getStepRecurrence(SE));
+
+  auto *Step = dyn_cast<SCEVConstant>(AddRec->getStepRecurrence(SE));
   if (!Step)
     return false;
 
-  auto TC = SE.getSmallConstantMaxTripCount(L, Predicates);
-  if (!TC)
+  // For the moment, restrict ourselves to the case where the access size is a
+  // multiple of the requested alignment and the base is aligned.
+  // TODO: generalize if a case found which warrants
+  if (EltSize.urem(Alignment.value()) != 0)
     return false;
 
   // TODO: Handle overlapping accesses.
-  // We should be computing AccessSize as (TC - 1) * Step + EltSize.
-  if (EltSize.sgt(Step->getAPInt()))
+  if (EltSize.ugt(Step->getAPInt().abs()))
+    return false;
+
+  const SCEV *MaxBECount =
+      Predicates ? SE.getPredicatedConstantMaxBackedgeTakenCount(L, *Predicates)
+                 : SE.getConstantMaxBackedgeTakenCount(L);
+  if (isa<SCEVCouldNotCompute>(MaxBECount))
+    return false;
+
+  const auto &[AccessStart, AccessEnd] = getStartAndEndForAccess(
+      L, PtrScev, LI->getType(), MaxBECount, &SE, nullptr);
+  if (isa<SCEVCouldNotCompute>(AccessStart) ||
+      isa<SCEVCouldNotCompute>(AccessEnd))
     return false;
 
-  // Compute the total access size for access patterns with unit stride and
-  // patterns with gaps. For patterns with unit stride, Step and EltSize are the
-  // same.
-  // For patterns with gaps (i.e. non unit stride), we are
-  // accessing EltSize bytes at every Step.
-  APInt AccessSize = TC * Step->getAPInt();
+  // Try to get the access size.
+  const SCEV *PtrDiff = SE.getMinusSCEV(AccessEnd, AccessStart);
+  APInt MaxPtrDiff = SE.getUnsignedRangeMax(PtrDiff);
 
-  assert(SE.isLoopInvariant(AddRec->getStart(), L) &&
-         "implied by addrec definition");
   Value *Base = nullptr;
-  if (auto *StartS = dyn_cast<SCEVUnknown>(AddRec->getStart())) {
-    Base = StartS->getValue();
-  } else if (auto *StartS = dyn_cast<SCEVAddExpr>(AddRec->getStart())) {
-    // Handle (NewBase + offset) as start value.
-    const auto *Offset = dyn_cast<SCEVConstant>(StartS->getOperand(0));
-    const auto *NewBase = dyn_cast<SCEVUnknown>(StartS->getOperand(1));
-    if (StartS->getNumOperands() == 2 && Offset && NewBase) {
-      // The following code below assumes the offset is unsigned, but GEP
-      // offsets are treated as signed so we can end up with a signed value
-      // here too. For example, suppose the initial PHI value is (i8 255),
-      // the offset will be treated as (i8 -1) and sign-extended to (i64 -1).
-      if (Offset->getAPInt().isNegative())
-        return false;
+  APInt AccessSize;
+  if (const SCEVUnknown *NewBase = dyn_cast<SCEVUnknown>(AccessStart)) {
+    Base = NewBase->getValue();
+    AccessSize = MaxPtrDiff;
+  } else if (auto *MinAdd = dyn_cast<SCEVAddExpr>(AccessStart)) {
+    if (MinAdd->getNumOperands() != 2)
+      return false;
 
-      // For the moment, restrict ourselves to the case where the offset is a
-      // multiple of the requested alignment and the base is aligned.
-      // TODO: generalize if a case found which warrants
-      if (Offset->getAPInt().urem(Alignment.value()) != 0)
-        return false;
-      Base = NewBase->getValue();
-      bool Overflow = false;
-      AccessSize = AccessSize.uadd_ov(Offset->getAPInt(), Overflow);
-      if (Overflow)
-        return false;
-    }
-  }
+    const auto *Offset = dyn_cast<SCEVConstant>(MinAdd->getOperand(0));
+    const auto *NewBase = dyn_cast<SCEVUnknown>(MinAdd->getOperand(1));
+    if (!Offset || !NewBase)
+      return false;
 
-  if (!Base)
-    return false;
+    // The following code below assumes the offset is unsigned, but GEP
+    // offsets are treated as signed so we can end up with a signed value
+    // here too. For example, suppose the initial PHI value is (i8 255),
+    // the offset will be treated as (i8 -1) and sign-extended to (i64 -1).
+    if (Offset->getAPInt().isNegative())
+      return false;
 
-  // For the moment, restrict ourselves to the case where the access size is a
-  // multiple of the requested alignment and the base is aligned.
-  // TODO: generalize if a case found which warrants
-  if (EltSize.urem(Alignment.value()) != 0)
+    // For the moment, restrict ourselves to the case where the offset is a
+    // multiple of the requested alignment and the base is aligned.
+    // TODO: generalize if a case found which warrants
+    if (Offset->getAPInt().urem(Alignment.value()) != 0)
+      return false;
+
+    AccessSize = MaxPtrDiff + Offset->getAPInt();
+    Base = NewBase->getValue();
+  } else
     return false;
+
+  Instruction *HeaderFirstNonPHI = &*L->getHeader()->getFirstNonPHIIt();
   return isDereferenceableAndAlignedPointer(Base, Alignment, AccessSize, DL,
                                             HeaderFirstNonPHI, AC, &DT);
 }
diff --git a/llvm/lib/Analysis/LoopAccessAnalysis.cpp b/llvm/lib/Analysis/LoopAccessAnalysis.cpp
index 2a68979add666..3202ba81be78e 100644
--- a/llvm/lib/Analysis/LoopAccessAnalysis.cpp
+++ b/llvm/lib/Analysis/LoopAccessAnalysis.cpp
@@ -190,31 +190,20 @@ RuntimeCheckingPtrGroup::RuntimeCheckingPtrGroup(
   Members.push_back(Index);
 }
 
-/// Calculate Start and End points of memory access.
-/// Let's assume A is the first access and B is a memory access on N-th loop
-/// iteration. Then B is calculated as:
-///   B = A + Step*N .
-/// Step value may be positive or negative.
-/// N is a calculated back-edge taken count:
-///     N = (TripCount > 0) ? RoundDown(TripCount -1 , VF) : 0
-/// Start and End points are calculated in the following way:
-/// Start = UMIN(A, B) ; End = UMAX(A, B) + SizeOfElt,
-/// where SizeOfElt is the size of single memory access in bytes.
-///
-/// There is no conflict when the intervals are disjoint:
-/// NoConflict = (P2.Start >= P1.End) || (P1.Start >= P2.End)
-static std::pair<const SCEV *, const SCEV *> getStartAndEndForAccess(
-    const Loop *Lp, const SCEV *PtrExpr, Type *AccessTy,
-    PredicatedScalarEvolution &PSE,
+std::pair<const SCEV *, const SCEV *> llvm::getStartAndEndForAccess(
+    const Loop *Lp, const SCEV *PtrExpr, Type *AccessTy, const SCEV *MaxBECount,
+    ScalarEvolution *SE,
     DenseMap<std::pair<const SCEV *, Type *>,
-             std::pair<const SCEV *, const SCEV *>> &PointerBounds) {
-  ScalarEvolution *SE = PSE.getSE();
-
-  auto [Iter, Ins] = PointerBounds.insert(
-      {{PtrExpr, AccessTy},
-       {SE->getCouldNotCompute(), SE->getCouldNotCompute()}});
-  if (!Ins)
-    return Iter->second;
+             std::pair<const SCEV *, const SCEV *>> *PointerBounds) {
+  std::pair<const SCEV *, const SCEV *> *PtrBoundsPair;
+  if (PointerBounds) {
+    auto [Iter, Ins] = PointerBounds->insert(
+        {{PtrExpr, AccessTy},
+         {SE->getCouldNotCompute(), SE->getCouldNotCompute()}});
+    if (!Ins)
+      return Iter->second;
+    PtrBoundsPair = &Iter->second;
+  }
 
   const SCEV *ScStart;
   const SCEV *ScEnd;
@@ -222,10 +211,8 @@ static std::pair<const SCEV *, const SCEV *> getStartAndEndForAccess(
   if (SE->isLoopInvariant(PtrExpr, Lp)) {
     ScStart = ScEnd = PtrExpr;
   } else if (auto *AR = dyn_cast<SCEVAddRecExpr>(PtrExpr)) {
-    const SCEV *Ex = PSE.getSymbolicMaxBackedgeTakenCount();
-
     ScStart = AR->getStart();
-    ScEnd = AR->evaluateAtIteration(Ex, *SE);
+    ScEnd = AR->evaluateAtIteration(MaxBECount, *SE);
     const SCEV *Step = AR->getStepRecurrence(*SE);
 
     // For expressions with negative step, the upper bound is ScStart and the
@@ -244,7 +231,7 @@ static std::pair<const SCEV *, const SCEV *> getStartAndEndForAccess(
     return {SE->getCouldNotCompute(), SE->getCouldNotCompute()};
 
   assert(SE->isLoopInvariant(ScStart, Lp) && "ScStart needs to be invariant");
-  assert(SE->isLoopInvariant(ScEnd, Lp)&& "ScEnd needs to be invariant");
+  assert(SE->isLoopInvariant(ScEnd, Lp) && "ScEnd needs to be invariant");
 
   // Add the size of the pointed element to ScEnd.
   auto &DL = Lp->getHeader()->getDataLayout();
@@ -252,8 +239,10 @@ static std::pair<const SCEV *, const SCEV *> getStartAndEndForAccess(
   const SCEV *EltSizeSCEV = SE->getStoreSizeOfExpr(IdxTy, AccessTy);
   ScEnd = SE->getAddExpr(ScEnd, EltSizeSCEV);
 
-  Iter->second = {ScStart, ScEnd};
-  return Iter->second;
+  std::pair<const SCEV *, const SCEV *> Res = {ScStart, ScEnd};
+  if (PointerBounds)
+    *PtrBoundsPair = Res;
+  return Res;
 }
 
 /// Calculate Start and End points of memory access using
@@ -263,8 +252,9 @@ void RuntimePointerChecking::insert(Loop *Lp, Value *Ptr, const SCEV *PtrExpr,
                                     unsigned DepSetId, unsigned ASId,
                                     PredicatedScalarEvolution &PSE,
                                     bool NeedsFreeze) {
+  const SCEV *MaxBECount = PSE.getSymbolicMaxBackedgeTakenCount();
   const auto &[ScStart, ScEnd] = getStartAndEndForAccess(
-      Lp, PtrExpr, AccessTy, PSE, DC.getPointerBounds());
+      Lp, PtrExpr, AccessTy, MaxBECount, PSE.getSE(), &DC.getPointerBounds());
   assert(!isa<SCEVCouldNotCompute>(ScStart) &&
          !isa<SCEVCouldNotCompute>(ScEnd) &&
          "must be able to compute both start and end expressions");
@@ -1448,7 +1438,7 @@ llvm::getPtrStride(PredicatedScalarEvolution &PSE, Type *AccessTy, Value *Ptr,
                    bool Assume, bool ShouldCheckWrap) {
   const SCEV *PtrScev = replaceSymbolicStrideSCEV(PSE, StridesMap, Ptr);
   if (PSE.getSE()->isLoopInvariant(PtrScev, Lp))
-    return {0};
+    return 0;
 
   Type *Ty = Ptr->getType();
   assert(Ty->isPointerTy() && "Unexpected non-ptr");
@@ -1938,10 +1928,11 @@ MemoryDepChecker::getDependenceDistanceStrideAndSize(
   // required for correctness.
   if (SE.isLoopInvariant(Src, InnermostLoop) ||
       SE.isLoopInvariant(Sink, InnermostLoop)) {
-    const auto &[SrcStart_, SrcEnd_] =
-        getStartAndEndForAccess(InnermostLoop, Src, ATy, PSE, PointerBounds);
-    const auto &[SinkStart_, SinkEnd_] =
-        getStartAndEndForAccess(InnermostLoop, Sink, BTy, PSE, PointerBounds);
+    const SCEV *MaxBECount = PSE.getSymbolicMaxBackedgeTakenCount();
+    const auto &[SrcStart_, SrcEnd_] = getStartAndEndForAccess(
+        InnermostLoop, Src, ATy, MaxBECount, PSE.getSE(), &PointerBounds);
+    const auto &[SinkStart_, SinkEnd_] = getStartAndEndForAccess(
+        InnermostLoop, Sink, BTy, MaxBECount, PSE.getSE(), &PointerBounds);
     if (!isa<SCEVCouldNotCompute>(SrcStart_) &&
         !isa<SCEVCouldNotCompute>(SrcEnd_) &&
         !isa<SCEVCouldNotCompute>(SinkStart_) &&
@@ -2602,7 +2593,7 @@ bool LoopAccessInfo::analyzeLoop(AAResults *AA, const LoopInfo *LI,
     bool IsReadOnlyPtr = false;
     Type *AccessTy = getLoadStoreType(LD);
     if (Seen.insert({Ptr, AccessTy}).second ||
-        !getPtrStride(*PSE, LD->getType(), Ptr, TheLoop, SymbolicStrides).value_or(0)) {
+        !getPtrStride(*PSE, AccessTy, Ptr, TheLoop, SymbolicStrides)) {
       ++NumReads;
       IsReadOnlyPtr = true;
     }
@@ -2868,7 +2859,7 @@ static Value *stripGetElementPtr(Value *Ptr, ScalarEvolution *SE, Loop *Lp) {
 /// strides "a[i*stride]". Returns the symbolic stride, or null otherwise.
 static const SCEV *getStrideFromPointer(Value *Ptr, ScalarEvolution *SE, Loop *Lp) {
   auto *PtrTy = dyn_cast<PointerType>(Ptr->getType());
-  if (!PtrTy || PtrTy->isAggregateType())
+  if (!PtrTy)
     return nullptr;
 
   // Try to remove a gep instruction to make the pointer (actually index at this
@@ -2876,18 +2867,15 @@ static const SCEV *getStrideFromPointer(Value *Ptr, ScalarEvolution *SE, Loop *L
   // pointer, otherwise, we are analyzing the index.
   Value *OrigPtr = Ptr;
 
-  // The size of the pointer access.
-  int64_t PtrAccessSize = 1;
-
   Ptr = stripGetElementPtr(Ptr, SE, Lp);
   const SCEV *V = SE->getSCEV(Ptr);
 
   if (Ptr != OrigPtr)
     // Strip off casts.
-    while (const SCEVIntegralCastExpr *C = dyn_cast<SCEVIntegralCastExpr>(V))
+    while (auto *C = dyn_cast<SCEVIntegralCastExpr>(V))
       V = C->getOperand();
 
-  const SCEVAddRecExpr *S = dyn_cast<SCEVAddRecExpr>(V);
+  auto *S = dyn_cast<SCEVAddRecExpr>(V);
   if (!S)
     return nullptr;
 
@@ -2897,25 +2885,20 @@ static const SCEV *getStrideFromPointer(Value *Ptr, ScalarEvolution *SE, Loop *L
     return nullptr;
 
   V = S->getStepRecurrence(*SE);
-  if (!V)
-    return nullptr;
 
   // Strip off the size of access multiplication if we are still analyzing the
   // pointer.
   if (OrigPtr == Ptr) {
-    if (const SCEVMulExpr *M = dyn_cast<SCEVMulExpr>(V)) {
-      if (M->getOperand(0)->getSCEVType() != scConstant)
+    if (auto *M = dyn_cast<SCEVMulExpr>(V)) {
+      auto *StepConst = dyn_cast<SCEVConstant>(M->getOperand(0));
+      if (!StepConst)
         return nullptr;
 
-      const APInt &APStepVal = cast<SCEVConstant>(M->getOperand(0))->getAPInt();
-
-      // Huge step value - give up.
-      if (APStepVal.getBitWidth() > 64)
+      auto StepVal = StepConst->getAPInt().trySExtValue();
+      // Bail out on a non-unit pointer access size.
+      if (!StepVal || StepVal != 1)
         return nullptr;
 
-      int64_t StepVal = APStepVal.getSExtValue();
-      if (PtrAccessSize != StepVal)
-        return nullptr;
       V = M->getOperand(1);
     }
   }
@@ -2929,7 +2912,7 @@ static const SCEV *getStrideFromPointer(Value *Ptr, ScalarEvolution *SE, Loop *L
   if (isa<SCEVUnknown>(V))
     return V;
 
-  if (const auto *C = dyn_cast<SCEVIntegralCastExpr>(V))
+  if (auto *C = dyn_cast<SCEVIntegralCastExpr>(V))
     if (isa<SCEVUnknown>(C->getOperand()))
       return V;
 
@@ -3095,7 +3078,6 @@ const LoopAccessInfo &LoopAccessInfoManager::getInfo(Loop &L) {
   return *It->second;
 }
 void LoopAccessInfoManager::clear() {
-  SmallVector<Loop *> ToRemove;
   // Collect LoopAccessInfo entries that may keep references to IR outside the
   // analyzed loop or SCEVs that may have been modified or invalidated. At the
   // moment, that is loops requiring memory or SCEV runtime checks, as those cache
@@ -3104,11 +3086,8 @@ void LoopAccessInfoManager::clear() {
     if (LAI->getRuntimePointerChecking()->getChecks().empty() &&
         LAI->getPSE().getPredicate().isAlwaysTrue())
       continue;
-    ToRemove.push_back(L);
-  }
-
-  for (Loop *L : ToRemove)
     LoopAccessInfoMap.erase(L);
+  }
 }
 
 bool LoopAccessInfoManager::invalidate(
diff --git a/llvm/lib/Analysis/LoopCacheAnalysis.cpp b/llvm/lib/Analysis/LoopCacheAnalysis.cpp
index 2897b922f61e4..050c32707596a 100644
--- a/llvm/lib/Analysis/LoopCacheAnalysis.cpp
+++ b/llvm/lib/Analysis/LoopCacheAnalysis.cpp
@@ -224,7 +224,7 @@ IndexedReference::hasTemporalReuse(const IndexedReference &Other,
   }
 
   std::unique_ptr<Dependence> D =
-      DI.depends(&StoreOrLoadInst, &Other.StoreOrLoadInst, true);
+      DI.depends(&StoreOrLoadInst, &Other.StoreOrLoadInst);
 
   if (D == nullptr) {
     LLVM_DEBUG(dbgs().indent(2) << "No temporal reuse: no dependence\n");
diff --git a/llvm/lib/Analysis/MemoryProfileInfo.cpp b/llvm/lib/Analysis/MemoryProfileInfo.cpp
index 2f3c87a89f9f9..a22344e19d045 100644
--- a/llvm/lib/Analysis/MemoryProfileInfo.cpp
+++ b/llvm/lib/Analysis/MemoryProfileInfo.cpp
@@ -42,10 +42,22 @@ cl::opt<unsigned> MemProfMinAveLifetimeAccessDensityHotThreshold(
     cl::desc("The minimum TotalLifetimeAccessDensity / AllocCount for an "
              "allocation to be considered hot"));
 
+cl::opt<bool>
+    MemProfUseHotHints("memprof-use-hot-hints", cl::init(false), cl::Hidden,
+                       cl::desc("Enable use of hot hints (only supported for "
+                                "unambigously hot allocations)"));
+
 cl::opt<bool> MemProfReportHintedSizes(
     "memprof-report-hinted-sizes", cl::init(false), cl::Hidden,
     cl::desc("Report total allocation sizes of hinted allocations"));
 
+// This is useful if we have enabled reporting of hinted sizes, and want to get
+// information from the indexing step for all contexts (especially for testing),
+// or have specified a value less than 100% for -memprof-cloning-cold-threshold.
+cl::opt<bool> MemProfKeepAllNotColdContexts(
+    "memprof-keep-all-not-cold-contexts", cl::init(false), cl::Hidden,
+    cl::desc("Keep all non-cold contexts (increases cloning overheads)"));
+
 AllocationType llvm::memprof::getAllocType(uint64_t TotalLifetimeAccessDensity,
                                            uint64_t AllocCount,
                                            uint64_t TotalLifetime) {
@@ -60,8 +72,9 @@ AllocationType llvm::memprof::getAllocType(uint64_t TotalLifetimeAccessDensity,
 
   // The access densities are multiplied by 100 to hold 2 decimal places of
   // precision, so need to divide by 100.
-  if (((float)TotalLifetimeAccessDensity) / AllocCount / 100 >
-      MemProfMinAveLifetimeAccessDensityHotThreshold)
+  if (MemProfUseHotHints &&
+      ((float)TotalLifetimeAccessDensity) / AllocCount / 100 >
+          MemProfMinAveLifetimeAccessDensityHotThreshold)
     return AllocationType::Hot;
 
   return AllocationType::NotCold;
@@ -141,7 +154,7 @@ void CallStackTrie::addCallStack(
       First = false;
       if (Alloc) {
         assert(AllocStackId == StackId);
-        Alloc->AllocTypes |= static_cast<uint8_t>(AllocType);
+        Alloc->addAllocType(AllocType);
       } else {
         AllocStackId = StackId;
         Alloc = new CallStackTrieNode(AllocType);
@@ -150,10 +163,16 @@ void CallStackTrie::addCallStack(
       continue;
     }
     // Update existing caller node if it exists.
+    CallStackTrieNode *Prev = nullptr;
     auto Next = Curr->Callers.find(StackId);
     if (Next != Curr->Callers.end()) {
+      Prev = Curr;
       Curr = Next->second;
-      Curr->AllocTypes |= static_cast<uint8_t>(AllocType);
+      Curr->addAllocType(AllocType);
+      // If this node has an ambiguous alloc type, its callee is not the deepest
+      // point where we have an ambigous allocation type.
+      if (!hasSingleAllocType(Curr->AllocTypes))
+        Prev->DeepestAmbiguousAllocType = false;
       continue;
     }
     // Otherwise add a new caller node.
@@ -222,20 +241,50 @@ void CallStackTrie::collectContextSizeInfo(
     collectContextSizeInfo(Caller.second, ContextSizeInfo);
 }
 
+void CallStackTrie::convertHotToNotCold(CallStackTrieNode *Node) {
+  if (Node->hasAllocType(AllocationType::Hot)) {
+    Node->removeAllocType(AllocationType::Hot);
+    Node->addAllocType(AllocationType::NotCold);
+  }
+  for (auto &Caller : Node->Callers)
+    convertHotToNotCold(Caller.second);
+}
+
 // Recursive helper to trim contexts and create metadata nodes.
 // Caller should have pushed Node's loc to MIBCallStack. Doing this in the
 // caller makes it simpler to handle the many early returns in this method.
 bool CallStackTrie::buildMIBNodes(CallStackTrieNode *Node, LLVMContext &Ctx,
                                   std::vector<uint64_t> &MIBCallStack,
                                   std::vector<Metadata *> &MIBNodes,
-                                  bool CalleeHasAmbiguousCallerContext) {
+                                  bool CalleeHasAmbiguousCallerContext,
+                                  bool &CalleeDeepestAmbiguousAllocType) {
   // Trim context below the first node in a prefix with a single alloc type.
   // Add an MIB record for the current call stack prefix.
   if (hasSingleAllocType(Node->AllocTypes)) {
-    std::vector<ContextTotalSize> ContextSizeInfo;
-    collectContextSizeInfo(Node, ContextSizeInfo);
-    MIBNodes.push_back(createMIBNode(
-        Ctx, MIBCallStack, (AllocationType)Node->AllocTypes, ContextSizeInfo));
+    // Because we only clone cold contexts (we don't clone for exposing NotCold
+    // contexts as that is the default allocation behavior), we create MIB
+    // metadata for this context if any of the following are true:
+    // 1) It is cold.
+    // 2) The immediate callee is the deepest point where we have an ambiguous
+    //    allocation type (i.e. the other callers that are cold need to know
+    //    that we have a not cold context overlapping to this point so that we
+    //    know how deep to clone).
+    // 3) MemProfKeepAllNotColdContexts is enabled, which is useful if we are
+    //    reporting hinted sizes, and want to get information from the indexing
+    //    step for all contexts, or have specified a value less than 100% for
+    //    -memprof-cloning-cold-threshold.
+    if (Node->hasAllocType(AllocationType::Cold) ||
+        CalleeDeepestAmbiguousAllocType || MemProfKeepAllNotColdContexts) {
+      std::vector<ContextTotalSize> ContextSizeInfo;
+      collectContextSizeInfo(Node, ContextSizeInfo);
+      MIBNodes.push_back(createMIBNode(Ctx, MIBCallStack,
+                                       (AllocationType)Node->AllocTypes,
+                                       ContextSizeInfo));
+      // If we just emitted an MIB for a not cold caller, don't need to emit
+      // another one for the callee to correctly disambiguate its cold callers.
+      if (!Node->hasAllocType(AllocationType::Cold))
+        CalleeDeepestAmbiguousAllocType = false;
+    }
     return true;
   }
 
@@ -246,9 +295,9 @@ bool CallStackTrie::buildMIBNodes(CallStackTrieNode *Node, LLVMContext &Ctx,
     bool AddedMIBNodesForAllCallerContexts = true;
     for (auto &Caller : Node->Callers) {
       MIBCallStack.push_back(Caller.first);
-      AddedMIBNodesForAllCallerContexts &=
-          buildMIBNodes(Caller.second, Ctx, MIBCallStack, MIBNodes,
-                        NodeHasAmbiguousCallerContext);
+      AddedMIBNodesForAllCallerContexts &= buildMIBNodes(
+          Caller.second, Ctx, MIBCallStack, MIBNodes,
+          NodeHasAmbiguousCallerContext, Node->DeepestAmbiguousAllocType);
       // Remove Caller.
       MIBCallStack.pop_back();
     }
@@ -301,15 +350,37 @@ bool CallStackTrie::buildAndAttachMIBMetadata(CallBase *CI) {
                                 "single");
     return false;
   }
+  // If there were any hot allocation contexts, the Alloc trie node would have
+  // the Hot type set. If so, because we don't currently support cloning for hot
+  // contexts, they should be converted to NotCold. This happens in the cloning
+  // support anyway, however, doing this now enables more aggressive context
+  // trimming when building the MIB metadata (and possibly may make the
+  // allocation have a single NotCold allocation type), greatly reducing
+  // overheads in bitcode, cloning memory and cloning time.
+  if (Alloc->hasAllocType(AllocationType::Hot)) {
+    convertHotToNotCold(Alloc);
+    // Check whether we now have a single alloc type.
+    if (hasSingleAllocType(Alloc->AllocTypes)) {
+      addSingleAllocTypeAttribute(CI, (AllocationType)Alloc->AllocTypes,
+                                  "single");
+      return false;
+    }
+  }
   auto &Ctx = CI->getContext();
   std::vector<uint64_t> MIBCallStack;
   MIBCallStack.push_back(AllocStackId);
   std::vector<Metadata *> MIBNodes;
   assert(!Alloc->Callers.empty() && "addCallStack has not been called yet");
-  // The last parameter is meant to say whether the callee of the given node
-  // has more than one caller. Here the node being passed in is the alloc
-  // and it has no callees. So it's false.
-  if (buildMIBNodes(Alloc, Ctx, MIBCallStack, MIBNodes, false)) {
+  // The CalleeHasAmbiguousCallerContext flag is meant to say whether the
+  // callee of the given node has more than one caller. Here the node being
+  // passed in is the alloc and it has no callees. So it's false.
+  // Similarly, the last parameter is meant to say whether the callee of the
+  // given node is the deepest point where we have ambiguous alloc types, which
+  // is also false as the alloc has no callees.
+  bool DeepestAmbiguousAllocType = true;
+  if (buildMIBNodes(Alloc, Ctx, MIBCallStack, MIBNodes,
+                    /*CalleeHasAmbiguousCallerContext=*/false,
+                    DeepestAmbiguousAllocType)) {
     assert(MIBCallStack.size() == 1 &&
            "Should only be left with Alloc's location in stack");
     CI->setMetadata(LLVMContext::MD_memprof, MDNode::get(Ctx, MIBNodes));
diff --git a/llvm/lib/Analysis/ScalarEvolution.cpp b/llvm/lib/Analysis/ScalarEvolution.cpp
index 7d7d37b3d228d..8f74c1c398ced 100644
--- a/llvm/lib/Analysis/ScalarEvolution.cpp
+++ b/llvm/lib/Analysis/ScalarEvolution.cpp
@@ -5917,20 +5917,18 @@ const SCEV *ScalarEvolution::createAddRecFromPHI(PHINode *PN) {
     //   PHI(f(0), f({1,+,1})) --> f({0,+,1})
 
     // Do not allow refinement in rewriting of BEValue.
-    if (isGuaranteedNotToCauseUB(BEValue)) {
-      const SCEV *Shifted = SCEVShiftRewriter::rewrite(BEValue, L, *this);
-      const SCEV *Start = SCEVInitRewriter::rewrite(Shifted, L, *this, false);
-      if (Shifted != getCouldNotCompute() && Start != getCouldNotCompute() &&
-          ::impliesPoison(BEValue, Start)) {
-        const SCEV *StartVal = getSCEV(StartValueV);
-        if (Start == StartVal) {
-          // Okay, for the entire analysis of this edge we assumed the PHI
-          // to be symbolic.  We now need to go back and purge all of the
-          // entries for the scalars that use the symbolic expression.
-          forgetMemoizedResults(SymbolicName);
-          insertValueToMap(PN, Shifted);
-          return Shifted;
-        }
+    const SCEV *Shifted = SCEVShiftRewriter::rewrite(BEValue, L, *this);
+    const SCEV *Start = SCEVInitRewriter::rewrite(Shifted, L, *this, false);
+    if (Shifted != getCouldNotCompute() && Start != getCouldNotCompute() &&
+        isGuaranteedNotToCauseUB(Shifted) && ::impliesPoison(Shifted, Start)) {
+      const SCEV *StartVal = getSCEV(StartValueV);
+      if (Start == StartVal) {
+        // Okay, for the entire analysis of this edge we assumed the PHI
+        // to be symbolic.  We now need to go back and purge all of the
+        // entries for the scalars that use the symbolic expression.
+        forgetMemoizedResults(SymbolicName);
+        insertValueToMap(PN, Shifted);
+        return Shifted;
       }
     }
   }
@@ -11167,9 +11165,8 @@ ScalarEvolution::getMonotonicPredicateTypeImpl(const SCEVAddRecExpr *LHS,
 }
 
 std::optional<ScalarEvolution::LoopInvariantPredicate>
-ScalarEvolution::getLoopInvariantPredicate(ICmpInst::Predicate Pred,
-                                           const SCEV *LHS, const SCEV *RHS,
-                                           const Loop *L,
+ScalarEvolution::getLoopInvariantPredicate(CmpPredicate Pred, const SCEV *LHS,
+                                           const SCEV *RHS, const Loop *L,
                                            const Instruction *CtxI) {
   // If there is a loop-invariant, force it into the RHS, otherwise bail out.
   if (!isLoopInvariant(RHS, L)) {
@@ -11177,7 +11174,7 @@ ScalarEvolution::getLoopInvariantPredicate(ICmpInst::Predicate Pred,
       return std::nullopt;
 
     std::swap(LHS, RHS);
-    Pred = ICmpInst::getSwappedPredicate(Pred);
+    Pred = ICmpInst::getSwappedCmpPredicate(Pred);
   }
 
   const SCEVAddRecExpr *ArLHS = dyn_cast<SCEVAddRecExpr>(LHS);
@@ -11205,7 +11202,7 @@ ScalarEvolution::getLoopInvariantPredicate(ICmpInst::Predicate Pred,
   // A similar reasoning applies for a monotonically decreasing predicate, by
   // replacing true with false and false with true in the above two bullets.
   bool Increasing = *MonotonicType == ScalarEvolution::MonotonicallyIncreasing;
-  auto P = Increasing ? Pred : ICmpInst::getInversePredicate(Pred);
+  auto P = Increasing ? Pred : ICmpInst::getInverseCmpPredicate(Pred);
 
   if (isLoopBackedgeGuardedByCond(L, P, LHS, RHS))
     return ScalarEvolution::LoopInvariantPredicate(Pred, ArLHS->getStart(),
@@ -11635,8 +11632,9 @@ bool ScalarEvolution::isBasicBlockEntryGuardedByCond(const BasicBlock *BB,
   // non-strict comparison is known from ranges and non-equality is known from
   // dominating predicates. If we are proving strict comparison, we always try
   // to prove non-equality and non-strict comparison separately.
-  auto NonStrictPredicate = ICmpInst::getNonStrictPredicate(Pred);
-  const bool ProvingStrictComparison = (Pred != NonStrictPredicate);
+  CmpPredicate NonStrictPredicate = ICmpInst::getNonStrictCmpPredicate(Pred);
+  const bool ProvingStrictComparison =
+      (Pred != static_cast<CmpInst::Predicate>(NonStrictPredicate));
   bool ProvedNonStrictComparison = false;
   bool ProvedNonEquality = false;
 
@@ -12404,6 +12402,12 @@ bool ScalarEvolution::isImpliedViaMerge(CmpPredicate Pred, const SCEV *LHS,
       // iteration of a loop.
       if (!properlyDominates(L, LBB))
         return false;
+      // Addrecs are considered to properly dominate their loop, so are missed
+      // by the previous check. Discard any values that have computable
+      // evolution in this loop.
+      if (auto *Loop = LI.getLoopFor(LBB))
+        if (hasComputableLoopEvolution(L, Loop))
+          return false;
       if (!ProvedEasily(L, RHS))
         return false;
     }
diff --git a/llvm/lib/Analysis/TargetTransformInfo.cpp b/llvm/lib/Analysis/TargetTransformInfo.cpp
index 8b9722d047edc..dc066099bdc1d 100644
--- a/llvm/lib/Analysis/TargetTransformInfo.cpp
+++ b/llvm/lib/Analysis/TargetTransformInfo.cpp
@@ -1153,6 +1153,15 @@ InstructionCost TargetTransformInfo::getGatherScatterOpCost(
   return Cost;
 }
 
+InstructionCost TargetTransformInfo::getExpandCompressMemoryOpCost(
+    unsigned Opcode, Type *DataTy, bool VariableMask, Align Alignment,
+    TTI::TargetCostKind CostKind, const Instruction *I) const {
+  InstructionCost Cost = TTIImpl->getExpandCompressMemoryOpCost(
+      Opcode, DataTy, VariableMask, Alignment, CostKind, I);
+  assert(Cost >= 0 && "TTI should not produce negative costs!");
+  return Cost;
+}
+
 InstructionCost TargetTransformInfo::getStridedMemoryOpCost(
     unsigned Opcode, Type *DataTy, const Value *Ptr, bool VariableMask,
     Align Alignment, TTI::TargetCostKind CostKind, const Instruction *I) const {
@@ -1441,6 +1450,12 @@ TargetTransformInfo::getNumBytesToPadGlobalArray(unsigned Size,
   return TTIImpl->getNumBytesToPadGlobalArray(Size, ArrayType);
 }
 
+void TargetTransformInfo::collectKernelLaunchBounds(
+    const Function &F,
+    SmallVectorImpl<std::pair<StringRef, int64_t>> &LB) const {
+  return TTIImpl->collectKernelLaunchBounds(F, LB);
+}
+
 TargetTransformInfo::Concept::~Concept() = default;
 
 TargetIRAnalysis::TargetIRAnalysis() : TTICallback(&getDefaultTTI) {}
diff --git a/llvm/lib/Analysis/ValueTracking.cpp b/llvm/lib/Analysis/ValueTracking.cpp
index eba728c7c8c36..28d7e1ce401e4 100644
--- a/llvm/lib/Analysis/ValueTracking.cpp
+++ b/llvm/lib/Analysis/ValueTracking.cpp
@@ -113,24 +113,6 @@ static const Instruction *safeCxtI(const Value *V, const Instruction *CxtI) {
   return nullptr;
 }
 
-static const Instruction *safeCxtI(const Value *V1, const Value *V2, const Instruction *CxtI) {
-  // If we've been provided with a context instruction, then use that (provided
-  // it has been inserted).
-  if (CxtI && CxtI->getParent())
-    return CxtI;
-
-  // If the value is really an already-inserted instruction, then use that.
-  CxtI = dyn_cast<Instruction>(V1);
-  if (CxtI && CxtI->getParent())
-    return CxtI;
-
-  CxtI = dyn_cast<Instruction>(V2);
-  if (CxtI && CxtI->getParent())
-    return CxtI;
-
-  return nullptr;
-}
-
 static bool getShuffleDemandedElts(const ShuffleVectorInst *Shuf,
                                    const APInt &DemandedElts,
                                    APInt &DemandedLHS, APInt &DemandedRHS) {
@@ -316,18 +298,14 @@ static bool isKnownNonEqual(const Value *V1, const Value *V2,
                             const SimplifyQuery &Q);
 
 bool llvm::isKnownNonEqual(const Value *V1, const Value *V2,
-                           const DataLayout &DL, AssumptionCache *AC,
-                           const Instruction *CxtI, const DominatorTree *DT,
-                           bool UseInstrInfo) {
+                           const SimplifyQuery &Q, unsigned Depth) {
   // We don't support looking through casts.
   if (V1 == V2 || V1->getType() != V2->getType())
     return false;
   auto *FVTy = dyn_cast<FixedVectorType>(V1->getType());
   APInt DemandedElts =
       FVTy ? APInt::getAllOnes(FVTy->getNumElements()) : APInt(1, 1);
-  return ::isKnownNonEqual(
-      V1, V2, DemandedElts, 0,
-      SimplifyQuery(DL, DT, AC, safeCxtI(V2, V1, CxtI), UseInstrInfo));
+  return ::isKnownNonEqual(V1, V2, DemandedElts, Depth, Q);
 }
 
 bool llvm::MaskedValueIsZero(const Value *V, const APInt &Mask,
@@ -593,11 +571,14 @@ static bool cmpExcludesZero(CmpInst::Predicate Pred, const Value *RHS) {
 }
 
 static void breakSelfRecursivePHI(const Use *U, const PHINode *PHI,
-                                  Value *&ValOut, Instruction *&CtxIOut) {
+                                  Value *&ValOut, Instruction *&CtxIOut,
+                                  const PHINode **PhiOut = nullptr) {
   ValOut = U->get();
   if (ValOut == PHI)
     return;
   CtxIOut = PHI->getIncomingBlock(*U)->getTerminator();
+  if (PhiOut)
+    *PhiOut = PHI;
   Value *V;
   // If the Use is a select of this phi, compute analysis on other arm to break
   // recursion.
@@ -610,11 +591,13 @@ static void breakSelfRecursivePHI(const Use *U, const PHINode *PHI,
   // incoming value to break recursion.
   // TODO: We could handle any number of incoming edges as long as we only have
   // two unique values.
-  else if (auto *IncPhi = dyn_cast<PHINode>(ValOut);
-           IncPhi && IncPhi->getNumIncomingValues() == 2) {
+  if (auto *IncPhi = dyn_cast<PHINode>(ValOut);
+      IncPhi && IncPhi->getNumIncomingValues() == 2) {
     for (int Idx = 0; Idx < 2; ++Idx) {
       if (IncPhi->getIncomingValue(Idx) == PHI) {
         ValOut = IncPhi->getIncomingValue(1 - Idx);
+        if (PhiOut)
+          *PhiOut = IncPhi;
         CtxIOut = IncPhi->getIncomingBlock(1 - Idx)->getTerminator();
         break;
       }
@@ -700,33 +683,30 @@ static void computeKnownBitsFromCmp(const Value *V, CmpInst::Predicate Pred,
 
   Value *Y;
   const APInt *Mask, *C;
+  if (!match(RHS, m_APInt(C)))
+    return;
+
   uint64_t ShAmt;
   switch (Pred) {
   case ICmpInst::ICMP_EQ:
     // assume(V = C)
-    if (match(LHS, m_V) && match(RHS, m_APInt(C))) {
+    if (match(LHS, m_V)) {
       Known = Known.unionWith(KnownBits::makeConstant(*C));
       // assume(V & Mask = C)
-    } else if (match(LHS, m_c_And(m_V, m_Value(Y))) &&
-               match(RHS, m_APInt(C))) {
+    } else if (match(LHS, m_c_And(m_V, m_Value(Y)))) {
       // For one bits in Mask, we can propagate bits from C to V.
       Known.One |= *C;
       if (match(Y, m_APInt(Mask)))
         Known.Zero |= ~*C & *Mask;
       // assume(V | Mask = C)
-    } else if (match(LHS, m_c_Or(m_V, m_Value(Y))) && match(RHS, m_APInt(C))) {
+    } else if (match(LHS, m_c_Or(m_V, m_Value(Y)))) {
       // For zero bits in Mask, we can propagate bits from C to V.
       Known.Zero |= ~*C;
       if (match(Y, m_APInt(Mask)))
         Known.One |= *C & ~*Mask;
-      // assume(V ^ Mask = C)
-    } else if (match(LHS, m_Xor(m_V, m_APInt(Mask))) &&
-               match(RHS, m_APInt(C))) {
-      // Equivalent to assume(V == Mask ^ C)
-      Known = Known.unionWith(KnownBits::makeConstant(*C ^ *Mask));
       // assume(V << ShAmt = C)
     } else if (match(LHS, m_Shl(m_V, m_ConstantInt(ShAmt))) &&
-               match(RHS, m_APInt(C)) && ShAmt < BitWidth) {
+               ShAmt < BitWidth) {
       // For those bits in C that are known, we can propagate them to known
       // bits in V shifted to the right by ShAmt.
       KnownBits RHSKnown = KnownBits::makeConstant(*C);
@@ -735,7 +715,7 @@ static void computeKnownBitsFromCmp(const Value *V, CmpInst::Predicate Pred,
       Known = Known.unionWith(RHSKnown);
       // assume(V >> ShAmt = C)
     } else if (match(LHS, m_Shr(m_V, m_ConstantInt(ShAmt))) &&
-               match(RHS, m_APInt(C)) && ShAmt < BitWidth) {
+               ShAmt < BitWidth) {
       KnownBits RHSKnown = KnownBits::makeConstant(*C);
       // For those bits in RHS that are known, we can propagate them to known
       // bits in V shifted to the right by C.
@@ -746,38 +726,36 @@ static void computeKnownBitsFromCmp(const Value *V, CmpInst::Predicate Pred,
   case ICmpInst::ICMP_NE: {
     // assume (V & B != 0) where B is a power of 2
     const APInt *BPow2;
-    if (match(LHS, m_And(m_V, m_Power2(BPow2))) && match(RHS, m_Zero()))
+    if (C->isZero() && match(LHS, m_And(m_V, m_Power2(BPow2))))
       Known.One |= *BPow2;
     break;
   }
-  default:
-    if (match(RHS, m_APInt(C))) {
-      const APInt *Offset = nullptr;
-      if (match(LHS, m_CombineOr(m_V, m_AddLike(m_V, m_APInt(Offset))))) {
-        ConstantRange LHSRange = ConstantRange::makeAllowedICmpRegion(Pred, *C);
-        if (Offset)
-          LHSRange = LHSRange.sub(*Offset);
-        Known = Known.unionWith(LHSRange.toKnownBits());
-      }
-      if (Pred == ICmpInst::ICMP_UGT || Pred == ICmpInst::ICMP_UGE) {
-        // X & Y u> C     -> X u> C && Y u> C
-        // X nuw- Y u> C  -> X u> C
-        if (match(LHS, m_c_And(m_V, m_Value())) ||
-            match(LHS, m_NUWSub(m_V, m_Value())))
-          Known.One.setHighBits(
-              (*C + (Pred == ICmpInst::ICMP_UGT)).countLeadingOnes());
-      }
-      if (Pred == ICmpInst::ICMP_ULT || Pred == ICmpInst::ICMP_ULE) {
-        // X | Y u< C    -> X u< C && Y u< C
-        // X nuw+ Y u< C -> X u< C && Y u< C
-        if (match(LHS, m_c_Or(m_V, m_Value())) ||
-            match(LHS, m_c_NUWAdd(m_V, m_Value()))) {
-          Known.Zero.setHighBits(
-              (*C - (Pred == ICmpInst::ICMP_ULT)).countLeadingZeros());
-        }
+  default: {
+    const APInt *Offset = nullptr;
+    if (match(LHS, m_CombineOr(m_V, m_AddLike(m_V, m_APInt(Offset))))) {
+      ConstantRange LHSRange = ConstantRange::makeAllowedICmpRegion(Pred, *C);
+      if (Offset)
+        LHSRange = LHSRange.sub(*Offset);
+      Known = Known.unionWith(LHSRange.toKnownBits());
+    }
+    if (Pred == ICmpInst::ICMP_UGT || Pred == ICmpInst::ICMP_UGE) {
+      // X & Y u> C     -> X u> C && Y u> C
+      // X nuw- Y u> C  -> X u> C
+      if (match(LHS, m_c_And(m_V, m_Value())) ||
+          match(LHS, m_NUWSub(m_V, m_Value())))
+        Known.One.setHighBits(
+            (*C + (Pred == ICmpInst::ICMP_UGT)).countLeadingOnes());
+    }
+    if (Pred == ICmpInst::ICMP_ULT || Pred == ICmpInst::ICMP_ULE) {
+      // X | Y u< C    -> X u< C && Y u< C
+      // X nuw+ Y u< C -> X u< C && Y u< C
+      if (match(LHS, m_c_Or(m_V, m_Value())) ||
+          match(LHS, m_c_NUWAdd(m_V, m_Value()))) {
+        Known.Zero.setHighBits(
+            (*C - (Pred == ICmpInst::ICMP_ULT)).countLeadingZeros());
       }
     }
-    break;
+  } break;
   }
 }
 
@@ -793,7 +771,10 @@ static void computeKnownBitsFromICmpCond(const Value *V, ICmpInst *Cmp,
   if (match(LHS, m_Trunc(m_Specific(V)))) {
     KnownBits DstKnown(LHS->getType()->getScalarSizeInBits());
     computeKnownBitsFromCmp(LHS, Pred, LHS, RHS, DstKnown, SQ);
-    Known = Known.unionWith(DstKnown.anyext(Known.getBitWidth()));
+    if (cast<TruncInst>(LHS)->hasNoUnsignedWrap())
+      Known = Known.unionWith(DstKnown.zext(Known.getBitWidth()));
+    else
+      Known = Known.unionWith(DstKnown.anyext(Known.getBitWidth()));
     return;
   }
 
@@ -816,10 +797,31 @@ static void computeKnownBitsFromCond(const Value *V, Value *Cond,
     else
       Known2 = Known2.intersectWith(Known3);
     Known = Known.unionWith(Known2);
+    return;
   }
 
-  if (auto *Cmp = dyn_cast<ICmpInst>(Cond))
+  if (auto *Cmp = dyn_cast<ICmpInst>(Cond)) {
     computeKnownBitsFromICmpCond(V, Cmp, Known, SQ, Invert);
+    return;
+  }
+
+  if (match(Cond, m_Trunc(m_Specific(V)))) {
+    KnownBits DstKnown(1);
+    if (Invert) {
+      DstKnown.setAllZero();
+    } else {
+      DstKnown.setAllOnes();
+    }
+    if (cast<TruncInst>(Cond)->hasNoUnsignedWrap()) {
+      Known = Known.unionWith(DstKnown.zext(Known.getBitWidth()));
+      return;
+    }
+    Known = Known.unionWith(DstKnown.anyext(Known.getBitWidth()));
+    return;
+  }
+
+  if (Depth < MaxAnalysisRecursionDepth && match(Cond, m_Not(m_Value(A))))
+    computeKnownBitsFromCond(V, A, Known, Depth + 1, SQ, !Invert);
 }
 
 void llvm::computeKnownBitsFromContext(const Value *V, KnownBits &Known,
@@ -1440,7 +1442,22 @@ static void computeKnownBitsFromOperator(const Operator *I,
     computeKnownBits(I->getOperand(0), Known, Depth + 1, Q);
     // Accumulate the constant indices in a separate variable
     // to minimize the number of calls to computeForAddSub.
-    APInt AccConstIndices(BitWidth, 0, /*IsSigned*/ true);
+    unsigned IndexWidth = Q.DL.getIndexTypeSizeInBits(I->getType());
+    APInt AccConstIndices(IndexWidth, 0);
+
+    auto AddIndexToKnown = [&](KnownBits IndexBits) {
+      if (IndexWidth == BitWidth) {
+        // Note that inbounds does *not* guarantee nsw for the addition, as only
+        // the offset is signed, while the base address is unsigned.
+        Known = KnownBits::add(Known, IndexBits);
+      } else {
+        // If the index width is smaller than the pointer width, only add the
+        // value to the low bits.
+        assert(IndexWidth < BitWidth &&
+               "Index width can't be larger than pointer width");
+        Known.insertBits(KnownBits::add(Known.trunc(IndexWidth), IndexBits), 0);
+      }
+    };
 
     gep_type_iterator GTI = gep_type_begin(I);
     for (unsigned i = 1, e = I->getNumOperands(); i != e; ++i, ++GTI) {
@@ -1478,43 +1495,34 @@ static void computeKnownBitsFromOperator(const Operator *I,
         break;
       }
 
-      unsigned IndexBitWidth = Index->getType()->getScalarSizeInBits();
-      KnownBits IndexBits(IndexBitWidth);
-      computeKnownBits(Index, IndexBits, Depth + 1, Q);
-      TypeSize IndexTypeSize = GTI.getSequentialElementStride(Q.DL);
-      uint64_t TypeSizeInBytes = IndexTypeSize.getKnownMinValue();
-      KnownBits ScalingFactor(IndexBitWidth);
+      TypeSize Stride = GTI.getSequentialElementStride(Q.DL);
+      uint64_t StrideInBytes = Stride.getKnownMinValue();
+      if (!Stride.isScalable()) {
+        // Fast path for constant offset.
+        if (auto *CI = dyn_cast<ConstantInt>(Index)) {
+          AccConstIndices +=
+              CI->getValue().sextOrTrunc(IndexWidth) * StrideInBytes;
+          continue;
+        }
+      }
+
+      KnownBits IndexBits =
+          computeKnownBits(Index, Depth + 1, Q).sextOrTrunc(IndexWidth);
+      KnownBits ScalingFactor(IndexWidth);
       // Multiply by current sizeof type.
       // &A[i] == A + i * sizeof(*A[i]).
-      if (IndexTypeSize.isScalable()) {
+      if (Stride.isScalable()) {
         // For scalable types the only thing we know about sizeof is
         // that this is a multiple of the minimum size.
-        ScalingFactor.Zero.setLowBits(llvm::countr_zero(TypeSizeInBytes));
-      } else if (IndexBits.isConstant()) {
-        APInt IndexConst = IndexBits.getConstant();
-        APInt ScalingFactor(IndexBitWidth, TypeSizeInBytes);
-        IndexConst *= ScalingFactor;
-        AccConstIndices += IndexConst.sextOrTrunc(BitWidth);
-        continue;
+        ScalingFactor.Zero.setLowBits(llvm::countr_zero(StrideInBytes));
       } else {
         ScalingFactor =
-            KnownBits::makeConstant(APInt(IndexBitWidth, TypeSizeInBytes));
+            KnownBits::makeConstant(APInt(IndexWidth, StrideInBytes));
       }
-      IndexBits = KnownBits::mul(IndexBits, ScalingFactor);
-
-      // If the offsets have a different width from the pointer, according
-      // to the language reference we need to sign-extend or truncate them
-      // to the width of the pointer.
-      IndexBits = IndexBits.sextOrTrunc(BitWidth);
-
-      // Note that inbounds does *not* guarantee nsw for the addition, as only
-      // the offset is signed, while the base address is unsigned.
-      Known = KnownBits::add(Known, IndexBits);
-    }
-    if (!Known.isUnknown() && !AccConstIndices.isZero()) {
-      KnownBits Index = KnownBits::makeConstant(AccConstIndices);
-      Known = KnownBits::add(Known, Index);
+      AddIndexToKnown(KnownBits::mul(IndexBits, ScalingFactor));
     }
+    if (!Known.isUnknown() && !AccConstIndices.isZero())
+      AddIndexToKnown(KnownBits::makeConstant(AccConstIndices));
     break;
   }
   case Instruction::PHI: {
@@ -1673,8 +1681,9 @@ static void computeKnownBitsFromOperator(const Operator *I,
       Known.One.setAllBits();
       for (const Use &U : P->operands()) {
         Value *IncValue;
+        const PHINode *CxtPhi;
         Instruction *CxtI;
-        breakSelfRecursivePHI(&U, P, IncValue, CxtI);
+        breakSelfRecursivePHI(&U, P, IncValue, CxtI, &CxtPhi);
         // Skip direct self references.
         if (IncValue == P)
           continue;
@@ -1705,9 +1714,10 @@ static void computeKnownBitsFromOperator(const Operator *I,
                     m_Br(m_c_ICmp(Pred, m_Specific(IncValue), m_APInt(RHSC)),
                          m_BasicBlock(TrueSucc), m_BasicBlock(FalseSucc)))) {
             // Check for cases of duplicate successors.
-            if ((TrueSucc == P->getParent()) != (FalseSucc == P->getParent())) {
+            if ((TrueSucc == CxtPhi->getParent()) !=
+                (FalseSucc == CxtPhi->getParent())) {
               // If we're using the false successor, invert the predicate.
-              if (FalseSucc == P->getParent())
+              if (FalseSucc == CxtPhi->getParent())
                 Pred = CmpInst::getInversePredicate(Pred);
               // Get the knownbits implied by the incoming phi condition.
               auto CR = ConstantRange::makeExactICmpRegion(Pred, *RHSC);
@@ -2645,40 +2655,42 @@ static bool isKnownNonNullFromDominatingCondition(const Value *V,
     return false;
 
   unsigned NumUsesExplored = 0;
-  for (const auto *U : V->users()) {
+  for (auto &U : V->uses()) {
     // Avoid massive lists
     if (NumUsesExplored >= DomConditionsMaxUses)
       break;
     NumUsesExplored++;
 
+    const Instruction *UI = cast<Instruction>(U.getUser());
     // If the value is used as an argument to a call or invoke, then argument
     // attributes may provide an answer about null-ness.
-    if (const auto *CB = dyn_cast<CallBase>(U))
-      if (auto *CalledFunc = CB->getCalledFunction())
-        for (const Argument &Arg : CalledFunc->args())
-          if (CB->getArgOperand(Arg.getArgNo()) == V &&
-              Arg.hasNonNullAttr(/* AllowUndefOrPoison */ false) &&
-              DT->dominates(CB, CtxI))
-            return true;
+    if (V->getType()->isPointerTy()) {
+      if (const auto *CB = dyn_cast<CallBase>(UI)) {
+        if (CB->isArgOperand(&U) &&
+            CB->paramHasNonNullAttr(CB->getArgOperandNo(&U),
+                                    /*AllowUndefOrPoison=*/false) &&
+            DT->dominates(CB, CtxI))
+          return true;
+      }
+    }
 
     // If the value is used as a load/store, then the pointer must be non null.
-    if (V == getLoadStorePointerOperand(U)) {
-      const Instruction *I = cast<Instruction>(U);
-      if (!NullPointerIsDefined(I->getFunction(),
+    if (V == getLoadStorePointerOperand(UI)) {
+      if (!NullPointerIsDefined(UI->getFunction(),
                                 V->getType()->getPointerAddressSpace()) &&
-          DT->dominates(I, CtxI))
+          DT->dominates(UI, CtxI))
         return true;
     }
 
-    if ((match(U, m_IDiv(m_Value(), m_Specific(V))) ||
-         match(U, m_IRem(m_Value(), m_Specific(V)))) &&
-        isValidAssumeForContext(cast<Instruction>(U), CtxI, DT))
+    if ((match(UI, m_IDiv(m_Value(), m_Specific(V))) ||
+         match(UI, m_IRem(m_Value(), m_Specific(V)))) &&
+        isValidAssumeForContext(UI, CtxI, DT))
       return true;
 
     // Consider only compare instructions uniquely controlling a branch
     Value *RHS;
     CmpPredicate Pred;
-    if (!match(U, m_c_ICmp(Pred, m_Specific(V), m_Value(RHS))))
+    if (!match(UI, m_c_ICmp(Pred, m_Specific(V), m_Value(RHS))))
       continue;
 
     bool NonNullIfTrue;
@@ -2691,7 +2703,7 @@ static bool isKnownNonNullFromDominatingCondition(const Value *V,
 
     SmallVector<const User *, 4> WorkList;
     SmallPtrSet<const User *, 4> Visited;
-    for (const auto *CmpU : U->users()) {
+    for (const auto *CmpU : UI->users()) {
       assert(WorkList.empty() && "Should be!");
       if (Visited.insert(CmpU).second)
         WorkList.push_back(CmpU);
@@ -4943,6 +4955,11 @@ static void computeKnownFPClassFromCond(const Value *V, Value *Cond,
                                 KnownFromContext);
     return;
   }
+  if (Depth < MaxAnalysisRecursionDepth && match(Cond, m_Not(m_Value(A)))) {
+    computeKnownFPClassFromCond(V, A, Depth + 1, !CondIsTrue, CxtI,
+                                KnownFromContext);
+    return;
+  }
   CmpPredicate Pred;
   Value *LHS;
   uint64_t ClassVal = 0;
@@ -8608,7 +8625,7 @@ bool llvm::isKnownInversion(const Value *X, const Value *Y) {
     return false;
 
   // They must both have samesign flag or not.
-  if (cast<ICmpInst>(X)->hasSameSign() != cast<ICmpInst>(Y)->hasSameSign())
+  if (Pred1.hasSameSign() != Pred2.hasSameSign())
     return false;
 
   if (B == C)
@@ -8620,8 +8637,7 @@ bool llvm::isKnownInversion(const Value *X, const Value *Y) {
     return false;
 
   // Sign bits of two RHSCs should match.
-  if (cast<ICmpInst>(X)->hasSameSign() &&
-      RHSC1->isNonNegative() != RHSC2->isNonNegative())
+  if (Pred1.hasSameSign() && RHSC1->isNonNegative() != RHSC2->isNonNegative())
     return false;
 
   const auto CR1 = ConstantRange::makeExactICmpRegion(Pred1, *RHSC1);
@@ -10221,10 +10237,9 @@ void llvm::findValuesAffectedByCondition(
       if (ICmpInst::isEquality(Pred)) {
         if (HasRHSC) {
           Value *Y;
-          // (X & C) or (X | C) or (X ^ C).
+          // (X & C) or (X | C).
           // (X << C) or (X >>_s C) or (X >>_u C).
-          if (match(A, m_BitwiseLogic(m_Value(X), m_ConstantInt())) ||
-              match(A, m_Shift(m_Value(X), m_ConstantInt())))
+          if (match(A, m_Shift(m_Value(X), m_ConstantInt())))
             AddAffected(X);
           else if (match(A, m_And(m_Value(X), m_Value(Y))) ||
                    match(A, m_Or(m_Value(X), m_Value(Y)))) {
@@ -10283,6 +10298,13 @@ void llvm::findValuesAffectedByCondition(
                                                            m_Value()))) {
       // Handle patterns that computeKnownFPClass() support.
       AddAffected(A);
+    } else if (!IsAssume && match(V, m_Trunc(m_Value(X)))) {
+      // Assume is checked here as X is already added above for assumes in
+      // addValueAffectedByCondition
+      AddAffected(X);
+    } else if (!IsAssume && match(V, m_Not(m_Value(X)))) {
+      // Assume is checked here to avoid issues with ephemeral values
+      Worklist.push_back(X);
     }
   }
 }
diff --git a/llvm/lib/AsmParser/LLLexer.cpp b/llvm/lib/AsmParser/LLLexer.cpp
index 5ea507c009bdc..438824f84e2d0 100644
--- a/llvm/lib/AsmParser/LLLexer.cpp
+++ b/llvm/lib/AsmParser/LLLexer.cpp
@@ -704,6 +704,7 @@ lltok::Kind LLLexer::LexIdentifier() {
   KEYWORD(argmemonly);
   KEYWORD(inaccessiblememonly);
   KEYWORD(inaccessiblemem_or_argmemonly);
+  KEYWORD(nocapture);
   KEYWORD(address_is_null);
   KEYWORD(address);
   KEYWORD(provenance);
@@ -975,6 +976,7 @@ lltok::Kind LLLexer::LexIdentifier() {
   DWKEYWORD(CC, DwarfCC);
   DWKEYWORD(OP, DwarfOp);
   DWKEYWORD(MACINFO, DwarfMacinfo);
+  DWKEYWORD(APPLE_ENUM_KIND, DwarfEnumKind);
 
 #undef DWKEYWORD
 
diff --git a/llvm/lib/AsmParser/LLParser.cpp b/llvm/lib/AsmParser/LLParser.cpp
index fa0079bac435c..ad52a9f493eae 100644
--- a/llvm/lib/AsmParser/LLParser.cpp
+++ b/llvm/lib/AsmParser/LLParser.cpp
@@ -448,6 +448,7 @@ bool LLParser::validateEndOfModule(bool UpgradeDebugInfo) {
     llvm::UpgradeDebugInfo(*M);
 
   UpgradeModuleFlags(*M);
+  UpgradeNVVMAnnotations(*M);
   UpgradeSectionAttributes(*M);
 
   if (PreserveInputDbgFormat != cl::boolOrDefault::BOU_TRUE)
@@ -2004,6 +2005,12 @@ bool LLParser::parseOptionalParamOrReturnAttrs(AttrBuilder &B, bool IsParam) {
       continue;
     }
 
+    if (Token == lltok::kw_nocapture) {
+      Lex.Lex();
+      B.addCapturesAttr(CaptureInfo::none());
+      continue;
+    }
+
     SMLoc Loc = Lex.getLoc();
     Attribute::AttrKind Attr = tokenToAttribute(Token);
     if (Attr == Attribute::None)
@@ -4697,6 +4704,12 @@ struct DwarfCCField : public MDUnsignedField {
   DwarfCCField() : MDUnsignedField(0, dwarf::DW_CC_hi_user) {}
 };
 
+struct DwarfEnumKindField : public MDUnsignedField {
+  DwarfEnumKindField()
+      : MDUnsignedField(dwarf::DW_APPLE_ENUM_KIND_invalid,
+                        dwarf::DW_APPLE_ENUM_KIND_max) {}
+};
+
 struct EmissionKindField : public MDUnsignedField {
   EmissionKindField() : MDUnsignedField(0, DICompileUnit::LastEmissionKind) {}
 };
@@ -4870,6 +4883,25 @@ bool LLParser::parseMDField(LocTy Loc, StringRef Name,
   return false;
 }
 
+template <>
+bool LLParser::parseMDField(LocTy Loc, StringRef Name,
+                            DwarfEnumKindField &Result) {
+  if (Lex.getKind() == lltok::APSInt)
+    return parseMDField(Loc, Name, static_cast<MDUnsignedField &>(Result));
+
+  if (Lex.getKind() != lltok::DwarfEnumKind)
+    return tokError("expected DWARF enum kind code");
+
+  unsigned EnumKind = dwarf::getEnumKind(Lex.getStrVal());
+  if (EnumKind == dwarf::DW_APPLE_ENUM_KIND_invalid)
+    return tokError("invalid DWARF enum kind code" + Twine(" '") +
+                    Lex.getStrVal() + "'");
+  assert(EnumKind <= Result.Max && "Expected valid DWARF enum kind code");
+  Result.assign(EnumKind);
+  Lex.Lex();
+  return false;
+}
+
 template <>
 bool LLParser::parseMDField(LocTy Loc, StringRef Name, DwarfLangField &Result) {
   if (Lex.getKind() == lltok::APSInt)
@@ -5489,6 +5521,7 @@ bool LLParser::parseDICompositeType(MDNode *&Result, bool IsDistinct) {
   OPTIONAL(flags, DIFlagField, );                                              \
   OPTIONAL(elements, MDField, );                                               \
   OPTIONAL(runtimeLang, DwarfLangField, );                                     \
+  OPTIONAL(enumKind, DwarfEnumKindField, );                                    \
   OPTIONAL(vtableHolder, MDField, );                                           \
   OPTIONAL(templateParams, MDField, );                                         \
   OPTIONAL(identifier, MDStringField, );                                       \
@@ -5510,15 +5543,19 @@ bool LLParser::parseDICompositeType(MDNode *&Result, bool IsDistinct) {
   else if (rank.isMDField())
     Rank = rank.getMDFieldValue();
 
+  std::optional<unsigned> EnumKind;
+  if (enumKind.Val != dwarf::DW_APPLE_ENUM_KIND_invalid)
+    EnumKind = enumKind.Val;
+
   // If this has an identifier try to build an ODR type.
   if (identifier.Val)
     if (auto *CT = DICompositeType::buildODRType(
             Context, *identifier.Val, tag.Val, name.Val, file.Val, line.Val,
             scope.Val, baseType.Val, size.Val, align.Val, offset.Val,
             specification.Val, num_extra_inhabitants.Val, flags.Val,
-            elements.Val, runtimeLang.Val, vtableHolder.Val, templateParams.Val,
-            discriminator.Val, dataLocation.Val, associated.Val, allocated.Val,
-            Rank, annotations.Val)) {
+            elements.Val, runtimeLang.Val, EnumKind, vtableHolder.Val,
+            templateParams.Val, discriminator.Val, dataLocation.Val,
+            associated.Val, allocated.Val, Rank, annotations.Val)) {
       Result = CT;
       return false;
     }
@@ -5529,9 +5566,10 @@ bool LLParser::parseDICompositeType(MDNode *&Result, bool IsDistinct) {
       DICompositeType,
       (Context, tag.Val, name.Val, file.Val, line.Val, scope.Val, baseType.Val,
        size.Val, align.Val, offset.Val, flags.Val, elements.Val,
-       runtimeLang.Val, vtableHolder.Val, templateParams.Val, identifier.Val,
-       discriminator.Val, dataLocation.Val, associated.Val, allocated.Val, Rank,
-       annotations.Val, specification.Val, num_extra_inhabitants.Val));
+       runtimeLang.Val, EnumKind, vtableHolder.Val, templateParams.Val,
+       identifier.Val, discriminator.Val, dataLocation.Val, associated.Val,
+       allocated.Val, Rank, annotations.Val, specification.Val,
+       num_extra_inhabitants.Val));
   return false;
 }
 
@@ -10703,12 +10741,15 @@ bool LLParser::parseOptionalCallsites(std::vector<CallsiteInfo> &Callsites) {
       return true;
 
     SmallVector<unsigned> StackIdIndices;
-    do {
-      uint64_t StackId = 0;
-      if (parseUInt64(StackId))
-        return true;
-      StackIdIndices.push_back(Index->addOrGetStackIdIndex(StackId));
-    } while (EatIfPresent(lltok::comma));
+    // Synthesized callsite records will not have a stack id list.
+    if (Lex.getKind() != lltok::rparen) {
+      do {
+        uint64_t StackId = 0;
+        if (parseUInt64(StackId))
+          return true;
+        StackIdIndices.push_back(Index->addOrGetStackIdIndex(StackId));
+      } while (EatIfPresent(lltok::comma));
+    }
 
     if (parseToken(lltok::rparen, "expected ')' in stackIds"))
       return true;
diff --git a/llvm/lib/BinaryFormat/Dwarf.cpp b/llvm/lib/BinaryFormat/Dwarf.cpp
index 0cd5dfbd023e4..b9b10a541b263 100644
--- a/llvm/lib/BinaryFormat/Dwarf.cpp
+++ b/llvm/lib/BinaryFormat/Dwarf.cpp
@@ -390,6 +390,25 @@ unsigned llvm::dwarf::getVirtuality(StringRef VirtualityString) {
       .Default(DW_VIRTUALITY_invalid);
 }
 
+StringRef llvm::dwarf::EnumKindString(unsigned EnumKind) {
+  switch (EnumKind) {
+  default:
+    return StringRef();
+#define HANDLE_DW_APPLE_ENUM_KIND(ID, NAME)                                    \
+  case DW_APPLE_ENUM_KIND_##NAME:                                              \
+    return "DW_APPLE_ENUM_KIND_" #NAME;
+#include "llvm/BinaryFormat/Dwarf.def"
+  }
+}
+
+unsigned llvm::dwarf::getEnumKind(StringRef EnumKindString) {
+  return StringSwitch<unsigned>(EnumKindString)
+#define HANDLE_DW_APPLE_ENUM_KIND(ID, NAME)                                    \
+  .Case("DW_APPLE_ENUM_KIND_" #NAME, DW_APPLE_ENUM_KIND_##NAME)
+#include "llvm/BinaryFormat/Dwarf.def"
+      .Default(DW_APPLE_ENUM_KIND_invalid);
+}
+
 StringRef llvm::dwarf::LanguageString(unsigned Language) {
   switch (Language) {
   default:
@@ -741,6 +760,8 @@ StringRef llvm::dwarf::AttributeValueString(uint16_t Attr, unsigned Val) {
     return LanguageString(Val);
   case DW_AT_defaulted:
     return DefaultedMemberString(Val);
+  case DW_AT_APPLE_enum_kind:
+    return EnumKindString(Val);
   }
 
   return StringRef();
diff --git a/llvm/lib/Bitcode/Reader/BitcodeReader.cpp b/llvm/lib/Bitcode/Reader/BitcodeReader.cpp
index 551dfd4af88bb..1a09e80c4fbb2 100644
--- a/llvm/lib/Bitcode/Reader/BitcodeReader.cpp
+++ b/llvm/lib/Bitcode/Reader/BitcodeReader.cpp
@@ -1865,7 +1865,7 @@ static uint64_t getRawAttributeMask(Attribute::AttrKind Val) {
   case Attribute::StackProtect:    return 1 << 14;
   case Attribute::StackProtectReq: return 1 << 15;
   case Attribute::Alignment:       return 31 << 16;
-  case Attribute::NoCapture:       return 1 << 21;
+  // 1ULL << 21 is NoCapture, which is upgraded separately.
   case Attribute::NoRedZone:       return 1 << 22;
   case Attribute::NoImplicitFloat: return 1 << 23;
   case Attribute::Naked:           return 1 << 24;
@@ -1986,6 +1986,12 @@ static void decodeLLVMAttributesForBitcode(AttrBuilder &B,
       B.addMemoryAttr(ME);
   }
 
+  // Upgrade nocapture to captures(none).
+  if (Attrs & (1ULL << 21)) {
+    Attrs &= ~(1ULL << 21);
+    B.addCapturesAttr(CaptureInfo::none());
+  }
+
   addRawAttributeValue(B, Attrs);
 }
 
@@ -2098,8 +2104,6 @@ static Attribute::AttrKind getAttrFromCode(uint64_t Code) {
     return Attribute::NoBuiltin;
   case bitc::ATTR_KIND_NO_CALLBACK:
     return Attribute::NoCallback;
-  case bitc::ATTR_KIND_NO_CAPTURE:
-    return Attribute::NoCapture;
   case bitc::ATTR_KIND_NO_DIVERGENCE_SOURCE:
     return Attribute::NoDivergenceSource;
   case bitc::ATTR_KIND_NO_DUPLICATE:
@@ -2349,6 +2353,11 @@ Error BitcodeReader::parseAttributeGroupBlock() {
               upgradeOldMemoryAttribute(ME, EncodedKind))
             continue;
 
+          if (EncodedKind == bitc::ATTR_KIND_NO_CAPTURE) {
+            B.addCapturesAttr(CaptureInfo::none());
+            continue;
+          }
+
           if (Error Err = parseAttrKind(EncodedKind, &Kind))
             return Err;
 
@@ -7148,6 +7157,8 @@ Error BitcodeReader::materializeModule() {
 
   UpgradeModuleFlags(*TheModule);
 
+  UpgradeNVVMAnnotations(*TheModule);
+
   UpgradeARCRuntime(*TheModule);
 
   return Error::success();
diff --git a/llvm/lib/Bitcode/Reader/MetadataLoader.cpp b/llvm/lib/Bitcode/Reader/MetadataLoader.cpp
index 1caa8480efc51..413d9f68e6cc3 100644
--- a/llvm/lib/Bitcode/Reader/MetadataLoader.cpp
+++ b/llvm/lib/Bitcode/Reader/MetadataLoader.cpp
@@ -1600,7 +1600,7 @@ Error MetadataLoader::MetadataLoaderImpl::parseOneMetadata(
     break;
   }
   case bitc::METADATA_COMPOSITE_TYPE: {
-    if (Record.size() < 16 || Record.size() > 24)
+    if (Record.size() < 16 || Record.size() > 25)
       return error("Invalid record");
 
     // If we have a UUID and this is not a forward declaration, lookup the
@@ -1622,6 +1622,8 @@ Error MetadataLoader::MetadataLoaderImpl::parseOneMetadata(
     DINode::DIFlags Flags = static_cast<DINode::DIFlags>(Record[10]);
     Metadata *Elements = nullptr;
     unsigned RuntimeLang = Record[12];
+    std::optional<uint32_t> EnumKind;
+
     Metadata *VTableHolder = nullptr;
     Metadata *TemplateParams = nullptr;
     Metadata *Discriminator = nullptr;
@@ -1683,24 +1685,28 @@ Error MetadataLoader::MetadataLoaderImpl::parseOneMetadata(
         Specification = getMDOrNull(Record[23]);
       }
     }
+
+    if (Record.size() > 25 && Record[25] != dwarf::DW_APPLE_ENUM_KIND_invalid)
+      EnumKind = Record[25];
+
     DICompositeType *CT = nullptr;
     if (Identifier)
       CT = DICompositeType::buildODRType(
           Context, *Identifier, Tag, Name, File, Line, Scope, BaseType,
           SizeInBits, AlignInBits, OffsetInBits, Specification,
-          NumExtraInhabitants, Flags, Elements, RuntimeLang, VTableHolder,
-          TemplateParams, Discriminator, DataLocation, Associated, Allocated,
-          Rank, Annotations);
+          NumExtraInhabitants, Flags, Elements, RuntimeLang, EnumKind,
+          VTableHolder, TemplateParams, Discriminator, DataLocation, Associated,
+          Allocated, Rank, Annotations);
 
     // Create a node if we didn't get a lazy ODR type.
     if (!CT)
       CT = GET_OR_DISTINCT(DICompositeType,
                            (Context, Tag, Name, File, Line, Scope, BaseType,
                             SizeInBits, AlignInBits, OffsetInBits, Flags,
-                            Elements, RuntimeLang, VTableHolder, TemplateParams,
-                            Identifier, Discriminator, DataLocation, Associated,
-                            Allocated, Rank, Annotations, Specification,
-                            NumExtraInhabitants));
+                            Elements, RuntimeLang, EnumKind, VTableHolder,
+                            TemplateParams, Identifier, Discriminator,
+                            DataLocation, Associated, Allocated, Rank,
+                            Annotations, Specification, NumExtraInhabitants));
     if (!IsNotUsedInTypeRef && Identifier)
       MetadataList.addTypeRef(*Identifier, *cast<DICompositeType>(CT));
 
diff --git a/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp b/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp
index 31c96400dd0fe..7ca63c2c7251d 100644
--- a/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp
+++ b/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp
@@ -23,6 +23,7 @@
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/StringMap.h"
 #include "llvm/ADT/StringRef.h"
+#include "llvm/BinaryFormat/Dwarf.h"
 #include "llvm/Bitcode/BitcodeCommon.h"
 #include "llvm/Bitcode/BitcodeReader.h"
 #include "llvm/Bitcode/LLVMBitCodes.h"
@@ -768,8 +769,6 @@ static uint64_t getAttrKindEncoding(Attribute::AttrKind Kind) {
     return bitc::ATTR_KIND_NO_BUILTIN;
   case Attribute::NoCallback:
     return bitc::ATTR_KIND_NO_CALLBACK;
-  case Attribute::NoCapture:
-    return bitc::ATTR_KIND_NO_CAPTURE;
   case Attribute::NoDivergenceSource:
     return bitc::ATTR_KIND_NO_DIVERGENCE_SOURCE;
   case Attribute::NoDuplicate:
@@ -1959,6 +1958,8 @@ void ModuleBitcodeWriter::writeDICompositeType(
   Record.push_back(VE.getMetadataOrNullID(N->getAnnotations().get()));
   Record.push_back(N->getNumExtraInhabitants());
   Record.push_back(VE.getMetadataOrNullID(N->getRawSpecification()));
+  Record.push_back(
+      N->getEnumKind().value_or(dwarf::DW_APPLE_ENUM_KIND_invalid));
 
   Stream.EmitRecord(bitc::METADATA_COMPOSITE_TYPE, Record, Abbrev);
   Record.clear();
diff --git a/llvm/lib/CMakeLists.txt b/llvm/lib/CMakeLists.txt
index f6465612d30c0..d0a2bc9294381 100644
--- a/llvm/lib/CMakeLists.txt
+++ b/llvm/lib/CMakeLists.txt
@@ -41,7 +41,9 @@ add_subdirectory(ProfileData)
 add_subdirectory(Passes)
 add_subdirectory(TargetParser)
 add_subdirectory(TextAPI)
-add_subdirectory(Telemetry)
+if (LLVM_BUILD_TELEMETRY)
+  add_subdirectory(Telemetry)
+endif()
 add_subdirectory(ToolDrivers)
 add_subdirectory(XRay)
 if (LLVM_INCLUDE_TESTS)
diff --git a/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp b/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp
index b2a4721f37b26..44b10c3ef9972 100644
--- a/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp
+++ b/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp
@@ -2853,7 +2853,6 @@ void AsmPrinter::emitConstantPool() {
 // Print assembly representations of the jump tables used by the current
 // function.
 void AsmPrinter::emitJumpTableInfo() {
-  const DataLayout &DL = MF->getDataLayout();
   const MachineJumpTableInfo *MJTI = MF->getJumpTableInfo();
   if (!MJTI) return;
   if (MJTI->getEntryKind() == MachineJumpTableInfo::EK_Inline) return;
@@ -2868,42 +2867,83 @@ void AsmPrinter::emitJumpTableInfo() {
       MJTI->getEntryKind() == MachineJumpTableInfo::EK_LabelDifference32 ||
           MJTI->getEntryKind() == MachineJumpTableInfo::EK_LabelDifference64,
       F);
+
+  if (!TM.Options.EnableStaticDataPartitioning) {
+    emitJumpTableImpl(*MJTI, llvm::to_vector(llvm::seq<unsigned>(JT.size())),
+                      JTInDiffSection);
+    return;
+  }
+
+  SmallVector<unsigned> HotJumpTableIndices, ColdJumpTableIndices;
+  // When static data partitioning is enabled, collect jump table entries that
+  // go into the same section together to reduce the amount of section switch
+  // statements.
+  for (unsigned JTI = 0, JTSize = JT.size(); JTI < JTSize; ++JTI) {
+    if (JT[JTI].Hotness == MachineFunctionDataHotness::Cold) {
+      ColdJumpTableIndices.push_back(JTI);
+    } else {
+      HotJumpTableIndices.push_back(JTI);
+    }
+  }
+
+  emitJumpTableImpl(*MJTI, HotJumpTableIndices, JTInDiffSection);
+  emitJumpTableImpl(*MJTI, ColdJumpTableIndices, JTInDiffSection);
+}
+
+void AsmPrinter::emitJumpTableImpl(const MachineJumpTableInfo &MJTI,
+                                   ArrayRef<unsigned> JumpTableIndices,
+                                   bool JTInDiffSection) {
+  if (JumpTableIndices.empty())
+    return;
+
+  const TargetLoweringObjectFile &TLOF = getObjFileLowering();
+  const Function &F = MF->getFunction();
+  const std::vector<MachineJumpTableEntry> &JT = MJTI.getJumpTables();
+  MCSection *JumpTableSection = nullptr;
+  if (TM.Options.EnableStaticDataPartitioning) {
+    JumpTableSection =
+        TLOF.getSectionForJumpTable(F, TM, &JT[JumpTableIndices.front()]);
+  } else {
+    JumpTableSection = TLOF.getSectionForJumpTable(F, TM);
+  }
+
+  const DataLayout &DL = MF->getDataLayout();
   if (JTInDiffSection) {
-    // Drop it in the readonly section.
-    MCSection *ReadOnlySection = TLOF.getSectionForJumpTable(F, TM);
-    OutStreamer->switchSection(ReadOnlySection);
+    OutStreamer->switchSection(JumpTableSection);
   }
 
-  emitAlignment(Align(MJTI->getEntryAlignment(DL)));
+  emitAlignment(Align(MJTI.getEntryAlignment(MF->getDataLayout())));
 
   // Jump tables in code sections are marked with a data_region directive
   // where that's supported.
   if (!JTInDiffSection)
     OutStreamer->emitDataRegion(MCDR_DataRegionJT32);
 
-  for (unsigned JTI = 0, e = JT.size(); JTI != e; ++JTI) {
-    const std::vector<MachineBasicBlock*> &JTBBs = JT[JTI].MBBs;
+  for (const unsigned JumpTableIndex : JumpTableIndices) {
+    ArrayRef<MachineBasicBlock *> JTBBs = JT[JumpTableIndex].MBBs;
 
     // If this jump table was deleted, ignore it.
-    if (JTBBs.empty()) continue;
+    if (JTBBs.empty())
+      continue;
 
     // For the EK_LabelDifference32 entry, if using .set avoids a relocation,
     /// emit a .set directive for each unique entry.
-    if (MJTI->getEntryKind() == MachineJumpTableInfo::EK_LabelDifference32 &&
+    if (MJTI.getEntryKind() == MachineJumpTableInfo::EK_LabelDifference32 &&
         MAI->doesSetDirectiveSuppressReloc()) {
-      SmallPtrSet<const MachineBasicBlock*, 16> EmittedSets;
+      SmallPtrSet<const MachineBasicBlock *, 16> EmittedSets;
       const TargetLowering *TLI = MF->getSubtarget().getTargetLowering();
-      const MCExpr *Base = TLI->getPICJumpTableRelocBaseExpr(MF,JTI,OutContext);
+      const MCExpr *Base =
+          TLI->getPICJumpTableRelocBaseExpr(MF, JumpTableIndex, OutContext);
       for (const MachineBasicBlock *MBB : JTBBs) {
         if (!EmittedSets.insert(MBB).second)
           continue;
 
         // .set LJTSet, LBB32-base
         const MCExpr *LHS =
-          MCSymbolRefExpr::create(MBB->getSymbol(), OutContext);
-        OutStreamer->emitAssignment(GetJTSetSymbol(JTI, MBB->getNumber()),
-                                    MCBinaryExpr::createSub(LHS, Base,
-                                                            OutContext));
+            MCSymbolRefExpr::create(MBB->getSymbol(), OutContext);
+        OutStreamer->emitAssignment(
+            GetJTSetSymbol(JumpTableIndex, MBB->getNumber()),
+            MCBinaryExpr::createSub(LHS, Base, OutContext));
       }
     }
 
@@ -2915,19 +2955,19 @@ void AsmPrinter::emitJumpTableInfo() {
       // FIXME: This doesn't have to have any specific name, just any randomly
       // named and numbered local label started with 'l' would work.  Simplify
       // GetJTISymbol.
-      OutStreamer->emitLabel(GetJTISymbol(JTI, true));
+      OutStreamer->emitLabel(GetJTISymbol(JumpTableIndex, true));
 
-    MCSymbol* JTISymbol = GetJTISymbol(JTI);
+    MCSymbol *JTISymbol = GetJTISymbol(JumpTableIndex);
     OutStreamer->emitLabel(JTISymbol);
 
     // Defer MCAssembler based constant folding due to a performance issue. The
     // label differences will be evaluated at write time.
     for (const MachineBasicBlock *MBB : JTBBs)
-      emitJumpTableEntry(*MJTI, MBB, JTI);
+      emitJumpTableEntry(MJTI, MBB, JumpTableIndex);
   }
 
   if (EmitJumpTableSizesSection)
-    emitJumpTableSizesSection(*MJTI, F);
+    emitJumpTableSizesSection(MJTI, MF->getFunction());
 
   if (!JTInDiffSection)
     OutStreamer->emitDataRegion(MCDR_DataRegionEnd);
diff --git a/llvm/lib/CodeGen/AsmPrinter/DbgEntityHistoryCalculator.cpp b/llvm/lib/CodeGen/AsmPrinter/DbgEntityHistoryCalculator.cpp
index 55a0afcf7a33f..c9efec37b0bc6 100644
--- a/llvm/lib/CodeGen/AsmPrinter/DbgEntityHistoryCalculator.cpp
+++ b/llvm/lib/CodeGen/AsmPrinter/DbgEntityHistoryCalculator.cpp
@@ -414,9 +414,10 @@ static void handleNewDebugValue(InlinedEntity Var, const MachineInstr &DV,
         dropRegDescribedVar(RegVars, I.first, Var);
 
     // Drop all entries that have ended, and mark the new entry as live.
+    auto &Entries = LiveEntries[Var];
     for (auto Index : IndicesToErase)
-      LiveEntries[Var].erase(Index);
-    LiveEntries[Var].insert(NewIndex);
+      Entries.erase(Index);
+    Entries.insert(NewIndex);
   }
 }
 
diff --git a/llvm/lib/CodeGen/AsmPrinter/DwarfCompileUnit.cpp b/llvm/lib/CodeGen/AsmPrinter/DwarfCompileUnit.cpp
index 2f96366b78e97..ddf0275ddfe6a 100644
--- a/llvm/lib/CodeGen/AsmPrinter/DwarfCompileUnit.cpp
+++ b/llvm/lib/CodeGen/AsmPrinter/DwarfCompileUnit.cpp
@@ -33,6 +33,7 @@
 #include "llvm/MC/MCSymbolWasm.h"
 #include "llvm/MC/MachineLocation.h"
 #include "llvm/Support/CommandLine.h"
+#include "llvm/Support/NVPTXAddrSpace.h"
 #include "llvm/Target/TargetLoweringObjectFile.h"
 #include "llvm/Target/TargetMachine.h"
 #include "llvm/Target/TargetOptions.h"
@@ -75,6 +76,26 @@ static dwarf::Tag GetCompileUnitType(UnitKind Kind, DwarfDebug *DW) {
   return dwarf::DW_TAG_compile_unit;
 }
 
+/// Translate NVVM IR address space code to DWARF correspondent value
+static unsigned translateToNVVMDWARFAddrSpace(unsigned AddrSpace) {
+  switch (AddrSpace) {
+  case NVPTXAS::ADDRESS_SPACE_GENERIC:
+    return NVPTXAS::DWARF_ADDR_generic_space;
+  case NVPTXAS::ADDRESS_SPACE_GLOBAL:
+    return NVPTXAS::DWARF_ADDR_global_space;
+  case NVPTXAS::ADDRESS_SPACE_SHARED:
+    return NVPTXAS::DWARF_ADDR_shared_space;
+  case NVPTXAS::ADDRESS_SPACE_CONST:
+    return NVPTXAS::DWARF_ADDR_const_space;
+  case NVPTXAS::ADDRESS_SPACE_LOCAL:
+    return NVPTXAS::DWARF_ADDR_local_space;
+  default:
+    llvm_unreachable(
+        "Cannot translate unknown address space to DWARF address space");
+    return AddrSpace;
+  }
+}
+
 DwarfCompileUnit::DwarfCompileUnit(unsigned UID, const DICompileUnit *Node,
                                    AsmPrinter *A, DwarfDebug *DW,
                                    DwarfFile *DWU, UnitKind Kind)
@@ -264,14 +285,11 @@ void DwarfCompileUnit::addLocationAttribute(
     }
 
     if (Expr) {
-      // According to
-      // https://docs.nvidia.com/cuda/archive/10.0/ptx-writers-guide-to-interoperability/index.html#cuda-specific-dwarf
-      // cuda-gdb requires DW_AT_address_class for all variables to be able to
-      // correctly interpret address space of the variable address.
+      // cuda-gdb special requirement. See NVPTXAS::DWARF_AddressSpace
       // Decode DW_OP_constu <DWARF Address Space> DW_OP_swap DW_OP_xderef
-      // sequence for the NVPTX + gdb target.
-      unsigned LocalNVPTXAddressSpace;
+      // sequence to specify corresponding address space.
       if (Asm->TM.getTargetTriple().isNVPTX() && DD->tuneForGDB()) {
+        unsigned LocalNVPTXAddressSpace;
         const DIExpression *NewExpr =
             DIExpression::extractAddressClass(Expr, LocalNVPTXAddressSpace);
         if (NewExpr != Expr) {
@@ -363,6 +381,10 @@ void DwarfCompileUnit::addLocationAttribute(
         DD->addArangeLabel(SymbolCU(this, Sym));
         addOpAddress(*Loc, Sym);
       }
+      if (Asm->TM.getTargetTriple().isNVPTX() && DD->tuneForGDB() &&
+          !NVPTXAddressSpace)
+        NVPTXAddressSpace =
+            translateToNVVMDWARFAddrSpace(Global->getType()->getAddressSpace());
     }
     // Global variables attached to symbols are memory locations.
     // It would be better if this were unconditional, but malformed input that
@@ -373,13 +395,9 @@ void DwarfCompileUnit::addLocationAttribute(
     DwarfExpr->addExpression(Expr);
   }
   if (Asm->TM.getTargetTriple().isNVPTX() && DD->tuneForGDB()) {
-    // According to
-    // https://docs.nvidia.com/cuda/archive/10.0/ptx-writers-guide-to-interoperability/index.html#cuda-specific-dwarf
-    // cuda-gdb requires DW_AT_address_class for all variables to be able to
-    // correctly interpret address space of the variable address.
-    const unsigned NVPTX_ADDR_global_space = 5;
+    // cuda-gdb special requirement. See NVPTXAS::DWARF_AddressSpace
     addUInt(*VariableDIE, dwarf::DW_AT_address_class, dwarf::DW_FORM_data1,
-            NVPTXAddressSpace.value_or(NVPTX_ADDR_global_space));
+            NVPTXAddressSpace.value_or(NVPTXAS::DWARF_ADDR_global_space));
   }
   if (Loc)
     addBlock(*VariableDIE, dwarf::DW_AT_location, DwarfExpr->finalize());
@@ -793,10 +811,10 @@ void DwarfCompileUnit::applyConcreteDbgVariableAttributes(
   const DbgValueLoc *DVal = &Single.getValueLoc();
   if (Asm->TM.getTargetTriple().isNVPTX() && DD->tuneForGDB() &&
       !Single.getExpr()) {
-    // Lack of expression means it is a register.  Registers for PTX need to
-    // be marked with DW_AT_address_class = 2.  See
-    // https://docs.nvidia.com/cuda/archive/10.0/ptx-writers-guide-to-interoperability/index.html#cuda-specific-dwarf
-    addUInt(VariableDie, dwarf::DW_AT_address_class, dwarf::DW_FORM_data1, 2);
+    // cuda-gdb special requirement. See NVPTXAS::DWARF_AddressSpace
+    // Lack of expression means it is a register.
+    addUInt(VariableDie, dwarf::DW_AT_address_class, dwarf::DW_FORM_data1,
+            NVPTXAS::DWARF_ADDR_reg_space);
   }
   if (!DVal->isVariadic()) {
     const DbgValueLocEntry *Entry = DVal->getLocEntries().begin();
@@ -922,14 +940,11 @@ void DwarfCompileUnit::applyConcreteDbgVariableAttributes(const Loc::MMI &MMI,
     SmallVector<uint64_t, 8> Ops;
     TRI->getOffsetOpcodes(Offset, Ops);
 
-    // According to
-    // https://docs.nvidia.com/cuda/archive/10.0/ptx-writers-guide-to-interoperability/index.html#cuda-specific-dwarf
-    // cuda-gdb requires DW_AT_address_class for all variables to be
-    // able to correctly interpret address space of the variable
-    // address. Decode DW_OP_constu <DWARF Address Space> DW_OP_swap
-    // DW_OP_xderef sequence for the NVPTX + gdb target.
-    unsigned LocalNVPTXAddressSpace;
+    // cuda-gdb special requirement. See NVPTXAS::DWARF_AddressSpace.
+    // Decode DW_OP_constu <DWARF Address Space> DW_OP_swap
+    // DW_OP_xderef sequence to specify address space.
     if (Asm->TM.getTargetTriple().isNVPTX() && DD->tuneForGDB()) {
+      unsigned LocalNVPTXAddressSpace;
       const DIExpression *NewExpr =
           DIExpression::extractAddressClass(Expr, LocalNVPTXAddressSpace);
       if (NewExpr != Expr) {
@@ -949,14 +964,9 @@ void DwarfCompileUnit::applyConcreteDbgVariableAttributes(const Loc::MMI &MMI,
     DwarfExpr.addExpression(std::move(Cursor));
   }
   if (Asm->TM.getTargetTriple().isNVPTX() && DD->tuneForGDB()) {
-    // According to
-    // https://docs.nvidia.com/cuda/archive/10.0/ptx-writers-guide-to-interoperability/index.html#cuda-specific-dwarf
-    // cuda-gdb requires DW_AT_address_class for all variables to be
-    // able to correctly interpret address space of the variable
-    // address.
-    const unsigned NVPTX_ADDR_local_space = 6;
+    // cuda-gdb special requirement. See NVPTXAS::DWARF_AddressSpace.
     addUInt(VariableDie, dwarf::DW_AT_address_class, dwarf::DW_FORM_data1,
-            NVPTXAddressSpace.value_or(NVPTX_ADDR_local_space));
+            NVPTXAddressSpace.value_or(NVPTXAS::DWARF_ADDR_local_space));
   }
   addBlock(VariableDie, dwarf::DW_AT_location, DwarfExpr.finalize());
   if (DwarfExpr.TagOffset)
@@ -1747,8 +1757,11 @@ void DwarfCompileUnit::createBaseTypeDIEs() {
 DIE *DwarfCompileUnit::getLexicalBlockDIE(const DILexicalBlock *LB) {
   // Assume if there is an abstract tree all the DIEs are already emitted.
   bool isAbstract = getAbstractScopeDIEs().count(LB->getSubprogram());
-  if (isAbstract && getAbstractScopeDIEs().count(LB))
-    return getAbstractScopeDIEs()[LB];
+  if (isAbstract) {
+    auto &DIEs = getAbstractScopeDIEs();
+    if (auto It = DIEs.find(LB); It != DIEs.end())
+      return It->second;
+  }
   assert(!isAbstract && "Missed lexical block DIE in abstract tree!");
 
   // Return a concrete DIE if it exists or nullptr otherwise.
diff --git a/llvm/lib/CodeGen/AsmPrinter/DwarfUnit.cpp b/llvm/lib/CodeGen/AsmPrinter/DwarfUnit.cpp
index d3450b8b0556f..c0f2878c84bc8 100644
--- a/llvm/lib/CodeGen/AsmPrinter/DwarfUnit.cpp
+++ b/llvm/lib/CodeGen/AsmPrinter/DwarfUnit.cpp
@@ -232,6 +232,42 @@ void DwarfUnit::addUInt(DIEValueList &Block, dwarf::Form Form,
   addUInt(Block, (dwarf::Attribute)0, Form, Integer);
 }
 
+void DwarfUnit::addIntAsBlock(DIE &Die, dwarf::Attribute Attribute, const APInt &Val) {
+  DIEBlock *Block = new (DIEValueAllocator) DIEBlock;
+
+  // Get the raw data form of the large APInt.
+  const uint64_t *Ptr64 = Val.getRawData();
+
+  int NumBytes = Val.getBitWidth() / 8; // 8 bits per byte.
+  bool LittleEndian = Asm->getDataLayout().isLittleEndian();
+
+  // Output the constant to DWARF one byte at a time.
+  for (int i = 0; i < NumBytes; i++) {
+    uint8_t c;
+    if (LittleEndian)
+      c = Ptr64[i / 8] >> (8 * (i & 7));
+    else
+      c = Ptr64[(NumBytes - 1 - i) / 8] >> (8 * ((NumBytes - 1 - i) & 7));
+    addUInt(*Block, dwarf::DW_FORM_data1, c);
+  }
+
+  addBlock(Die, Attribute, Block);
+}
+
+void DwarfUnit::addInt(DIE &Die, dwarf::Attribute Attribute,
+		       const APInt &Val, bool Unsigned) {
+  unsigned CIBitWidth = Val.getBitWidth();
+  if (CIBitWidth <= 64) {
+    if (Unsigned)
+      addUInt(Die, Attribute, std::nullopt, Val.getZExtValue());
+    else
+      addSInt(Die, Attribute, std::nullopt, Val.getSExtValue());
+    return;
+  }
+
+  addIntAsBlock(Die, Attribute, Val);
+}
+
 void DwarfUnit::addSInt(DIEValueList &Die, dwarf::Attribute Attribute,
                         std::optional<dwarf::Form> Form, int64_t Integer) {
   if (!Form)
@@ -484,25 +520,7 @@ void DwarfUnit::addConstantValue(DIE &Die, const APInt &Val, bool Unsigned) {
     return;
   }
 
-  DIEBlock *Block = new (DIEValueAllocator) DIEBlock;
-
-  // Get the raw data form of the large APInt.
-  const uint64_t *Ptr64 = Val.getRawData();
-
-  int NumBytes = Val.getBitWidth() / 8; // 8 bits per byte.
-  bool LittleEndian = Asm->getDataLayout().isLittleEndian();
-
-  // Output the constant to DWARF one byte at a time.
-  for (int i = 0; i < NumBytes; i++) {
-    uint8_t c;
-    if (LittleEndian)
-      c = Ptr64[i / 8] >> (8 * (i & 7));
-    else
-      c = Ptr64[(NumBytes - 1 - i) / 8] >> (8 * ((NumBytes - 1 - i) & 7));
-    addUInt(*Block, dwarf::DW_FORM_data1, c);
-  }
-
-  addBlock(Die, dwarf::DW_AT_const_value, Block);
+  addIntAsBlock(Die, dwarf::DW_AT_const_value, Val);
 }
 
 void DwarfUnit::addLinkageName(DIE &Die, StringRef LinkageName) {
@@ -849,9 +867,7 @@ void DwarfUnit::constructTypeDIE(DIE &Buffer, const DIDerivedType *DTy) {
   }
 }
 
-DIE *DwarfUnit::constructSubprogramArguments(DIE &Buffer, DITypeRefArray Args) {
-  // Args[0] is the return type.
-  DIE *ObjectPointer = nullptr;
+void DwarfUnit::constructSubprogramArguments(DIE &Buffer, DITypeRefArray Args) {
   for (unsigned i = 1, N = Args.size(); i < N; ++i) {
     const DIType *Ty = Args[i];
     if (!Ty) {
@@ -862,14 +878,8 @@ DIE *DwarfUnit::constructSubprogramArguments(DIE &Buffer, DITypeRefArray Args) {
       addType(Arg, Ty);
       if (Ty->isArtificial())
         addFlag(Arg, dwarf::DW_AT_artificial);
-      if (Ty->isObjectPointer()) {
-        assert(!ObjectPointer && "Can't have more than one object pointer");
-        ObjectPointer = &Arg;
-      }
     }
   }
-
-  return ObjectPointer;
 }
 
 void DwarfUnit::constructTypeDIE(DIE &Buffer, const DISubroutineType *CTy) {
@@ -980,12 +990,8 @@ void DwarfUnit::constructTypeDIE(DIE &Buffer, const DICompositeType *CTy) {
           DIE &Variant = createAndAddDIE(dwarf::DW_TAG_variant, Buffer);
           if (const ConstantInt *CI =
               dyn_cast_or_null<ConstantInt>(DDTy->getDiscriminantValue())) {
-            if (DD->isUnsignedDIType(Discriminator->getBaseType()))
-              addUInt(Variant, dwarf::DW_AT_discr_value, std::nullopt,
-                      CI->getZExtValue());
-            else
-              addSInt(Variant, dwarf::DW_AT_discr_value, std::nullopt,
-                      CI->getSExtValue());
+	    addInt(Variant, dwarf::DW_AT_discr_value, CI->getValue(),
+		   DD->isUnsignedDIType(Discriminator->getBaseType()));
           }
           constructMemberDIE(Variant, DDTy);
         } else {
@@ -1366,8 +1372,7 @@ void DwarfUnit::applySubprogramAttributes(const DISubprogram *SP, DIE &SPDie,
 
     // Add arguments. Do not add arguments for subprogram definition. They will
     // be handled while processing variables.
-    if (auto *ObjectPointer = constructSubprogramArguments(SPDie, Args))
-      addDIEEntry(SPDie, dwarf::DW_AT_object_pointer, *ObjectPointer);
+    constructSubprogramArguments(SPDie, Args);
   }
 
   addThrownTypes(SPDie, SP->getThrownTypes());
@@ -1624,6 +1629,9 @@ void DwarfUnit::constructEnumTypeDIE(DIE &Buffer, const DICompositeType *CTy) {
       addFlag(Buffer, dwarf::DW_AT_enum_class);
   }
 
+  if (auto Kind = CTy->getEnumKind())
+    addUInt(Buffer, dwarf::DW_AT_APPLE_enum_kind, dwarf::DW_FORM_data1, *Kind);
+
   auto *Context = CTy->getScope();
   bool IndexEnumerators = !Context || isa<DICompileUnit>(Context) || isa<DIFile>(Context) ||
       isa<DINamespace>(Context) || isa<DICommonBlock>(Context);
diff --git a/llvm/lib/CodeGen/AsmPrinter/DwarfUnit.h b/llvm/lib/CodeGen/AsmPrinter/DwarfUnit.h
index 7a5295d826a48..9ddd6f8c14175 100644
--- a/llvm/lib/CodeGen/AsmPrinter/DwarfUnit.h
+++ b/llvm/lib/CodeGen/AsmPrinter/DwarfUnit.h
@@ -167,6 +167,10 @@ class DwarfUnit : public DIEUnit {
 
   void addSInt(DIELoc &Die, std::optional<dwarf::Form> Form, int64_t Integer);
 
+  /// Add an integer attribute data and value; value may be any width.
+  void addInt(DIE &Die, dwarf::Attribute Attribute, const APInt &Integer,
+	      bool Unsigned);
+
   /// Add a string attribute data and value.
   ///
   /// We always emit a reference to the string pool instead of immediate
@@ -268,9 +272,7 @@ class DwarfUnit : public DIEUnit {
   void constructContainingTypeDIEs();
 
   /// Construct function argument DIEs.
-  ///
-  /// \returns DIE of the object pointer if one exists. Nullptr otherwise.
-  DIE *constructSubprogramArguments(DIE &Buffer, DITypeRefArray Args);
+  void constructSubprogramArguments(DIE &Buffer, DITypeRefArray Args);
 
   /// Create a DIE with the given Tag, add the DIE to its parent, and
   /// call insertDIE if MD is not null.
@@ -336,6 +338,10 @@ class DwarfUnit : public DIEUnit {
   void emitCommonHeader(bool UseOffsets, dwarf::UnitType UT);
 
 private:
+  /// A helper to add a wide integer constant to a DIE using a block
+  /// form.
+  void addIntAsBlock(DIE &Die, dwarf::Attribute Attribute, const APInt &Val);
+
   void constructTypeDIE(DIE &Buffer, const DIBasicType *BTy);
   void constructTypeDIE(DIE &Buffer, const DIStringType *BTy);
   void constructTypeDIE(DIE &Buffer, const DIDerivedType *DTy);
diff --git a/llvm/lib/CodeGen/CFIFixup.cpp b/llvm/lib/CodeGen/CFIFixup.cpp
index 02152a136a225..7986f7d213454 100644
--- a/llvm/lib/CodeGen/CFIFixup.cpp
+++ b/llvm/lib/CodeGen/CFIFixup.cpp
@@ -70,6 +70,7 @@
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/PostOrderIterator.h"
 #include "llvm/ADT/STLExtras.h"
+#include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/iterator_range.h"
 #include "llvm/CodeGen/MachineBasicBlock.h"
 #include "llvm/CodeGen/MachineFunction.h"
@@ -122,6 +123,62 @@ findPrologueEnd(MachineFunction &MF, MachineBasicBlock::iterator &PrologueEnd) {
   return nullptr;
 }
 
+// Represents a basic block's relationship to the call frame. This metadata
+// reflects what the state *should* be, which may differ from the actual state
+// after final machine basic block layout.
+struct BlockFlags {
+  bool Reachable : 1;
+  bool StrongNoFrameOnEntry : 1;
+  bool HasFrameOnEntry : 1;
+  bool HasFrameOnExit : 1;
+  BlockFlags()
+      : Reachable(false), StrongNoFrameOnEntry(false), HasFrameOnEntry(false),
+        HasFrameOnExit(false) {}
+};
+
+// Most functions will have <= 32 basic blocks.
+using BlockFlagsVector = SmallVector<BlockFlags, 32>;
+
+// Computes the frame information for each block in the function. Frame info
+// for a block is inferred from its predecessors.
+static BlockFlagsVector
+computeBlockInfo(const MachineFunction &MF,
+                 const MachineBasicBlock *PrologueBlock) {
+  BlockFlagsVector BlockInfo(MF.getNumBlockIDs());
+  BlockInfo[0].Reachable = true;
+  BlockInfo[0].StrongNoFrameOnEntry = true;
+
+  // Compute the presence/absence of frame at each basic block.
+  ReversePostOrderTraversal<const MachineBasicBlock *> RPOT(&*MF.begin());
+  for (const MachineBasicBlock *MBB : RPOT) {
+    BlockFlags &Info = BlockInfo[MBB->getNumber()];
+
+    // Set to true if the current block contains the prologue or the epilogue,
+    // respectively.
+    bool HasPrologue = MBB == PrologueBlock;
+    bool HasEpilogue = false;
+
+    if (Info.HasFrameOnEntry || HasPrologue)
+      HasEpilogue = containsEpilogue(*MBB);
+
+    // If the function has a call frame at the entry of the current block or the
+    // current block contains the prologue, then the function has a call frame
+    // at the exit of the block, unless the block contains the epilogue.
+    Info.HasFrameOnExit = (Info.HasFrameOnEntry || HasPrologue) && !HasEpilogue;
+
+    // Set the successors' state on entry.
+    for (MachineBasicBlock *Succ : MBB->successors()) {
+      BlockFlags &SuccInfo = BlockInfo[Succ->getNumber()];
+      SuccInfo.Reachable = true;
+      SuccInfo.StrongNoFrameOnEntry |=
+          Info.StrongNoFrameOnEntry && !HasPrologue;
+      SuccInfo.HasFrameOnEntry = Info.HasFrameOnExit;
+    }
+  }
+
+  return BlockInfo;
+}
+
 // Represents the point within a basic block where we can insert an instruction.
 // Note that we need the MachineBasicBlock* as well as the iterator since the
 // iterator can point to the end of the block. Instructions are inserted
@@ -181,13 +238,69 @@ static InsertionPoint cloneCfiPrologue(const InsertionPoint &PrologueEnd,
   return DstInsertPt;
 }
 
-bool CFIFixup::runOnMachineFunction(MachineFunction &MF) {
+// Fixes up the CFI instructions in a basic block to be consistent with the
+// intended frame state, adding or removing CFI instructions as necessary.
+// Returns true if a change was made and false otherwise.
+static bool
+fixupBlock(MachineBasicBlock &CurrBB, const BlockFlagsVector &BlockInfo,
+           SmallDenseMap<MBBSectionID, InsertionPoint> &InsertionPts,
+           const InsertionPoint &Prologue) {
+  const MachineFunction &MF = *CurrBB.getParent();
   const TargetFrameLowering &TFL = *MF.getSubtarget().getFrameLowering();
-  if (!TFL.enableCFIFixup(MF))
+  const BlockFlags &Info = BlockInfo[CurrBB.getNumber()];
+
+  if (!Info.Reachable)
     return false;
 
-  const unsigned NumBlocks = MF.getNumBlockIDs();
-  if (NumBlocks < 2)
+  // If the previous block and the current block are in the same section,
+  // the frame info will propagate from the previous block to the current one.
+  const BlockFlags &PrevInfo =
+      BlockInfo[std::prev(CurrBB.getIterator())->getNumber()];
+  bool HasFrame = PrevInfo.HasFrameOnExit && !CurrBB.isBeginSection();
+  bool NeedsFrame = Info.HasFrameOnEntry && !Info.StrongNoFrameOnEntry;
+
+#ifndef NDEBUG
+  if (!Info.StrongNoFrameOnEntry) {
+    for (auto *Pred : CurrBB.predecessors()) {
+      const BlockFlags &PredInfo = BlockInfo[Pred->getNumber()];
+      assert((!PredInfo.Reachable ||
+              Info.HasFrameOnEntry == PredInfo.HasFrameOnExit) &&
+             "Inconsistent call frame state");
+    }
+  }
+#endif
+
+  if (HasFrame == NeedsFrame)
+    return false;
+
+  if (!NeedsFrame) {
+    // Reset to the state upon function entry.
+    TFL.resetCFIToInitialState(CurrBB);
+    return true;
+  }
+
+  // Reset to the "after prologue" state.
+  InsertionPoint &InsertPt = InsertionPts[CurrBB.getSectionID()];
+  if (InsertPt.MBB == nullptr) {
+    // CurBB is the first block in its section, so there is no "after
+    // prologue" state. Clone the CFI instructions from the prologue block
+    // to create it.
+    InsertPt = cloneCfiPrologue(Prologue, {&CurrBB, CurrBB.begin()});
+  } else {
+    // There's an earlier block known to have a stack frame. Insert a
+    // `.cfi_remember_state` instruction into that block and a
+    // `.cfi_restore_state` instruction at the beginning of the current
+    // block.
+    InsertPt = insertRememberRestorePair(InsertPt, {&CurrBB, CurrBB.begin()});
+  }
+  return true;
+}
+
+bool CFIFixup::runOnMachineFunction(MachineFunction &MF) {
+  if (!MF.getSubtarget().getFrameLowering()->enableCFIFixup(MF))
+    return false;
+
+  if (MF.getNumBlockIDs() < 2)
     return false;
 
   // Find the prologue and the point where we can issue the first
@@ -197,44 +310,7 @@ bool CFIFixup::runOnMachineFunction(MachineFunction &MF) {
   if (PrologueBlock == nullptr)
     return false;
 
-  struct BlockFlags {
-    bool Reachable : 1;
-    bool StrongNoFrameOnEntry : 1;
-    bool HasFrameOnEntry : 1;
-    bool HasFrameOnExit : 1;
-  };
-  SmallVector<BlockFlags, 32> BlockInfo(NumBlocks,
-                                        {false, false, false, false});
-  BlockInfo[0].Reachable = true;
-  BlockInfo[0].StrongNoFrameOnEntry = true;
-
-  // Compute the presence/absence of frame at each basic block.
-  ReversePostOrderTraversal<MachineBasicBlock *> RPOT(&*MF.begin());
-  for (MachineBasicBlock *MBB : RPOT) {
-    BlockFlags &Info = BlockInfo[MBB->getNumber()];
-
-    // Set to true if the current block contains the prologue or the epilogue,
-    // respectively.
-    bool HasPrologue = MBB == PrologueBlock;
-    bool HasEpilogue = false;
-
-    if (Info.HasFrameOnEntry || HasPrologue)
-      HasEpilogue = containsEpilogue(*MBB);
-
-    // If the function has a call frame at the entry of the current block or the
-    // current block contains the prologue, then the function has a call frame
-    // at the exit of the block, unless the block contains the epilogue.
-    Info.HasFrameOnExit = (Info.HasFrameOnEntry || HasPrologue) && !HasEpilogue;
-
-    // Set the successors' state on entry.
-    for (MachineBasicBlock *Succ : MBB->successors()) {
-      BlockFlags &SuccInfo = BlockInfo[Succ->getNumber()];
-      SuccInfo.Reachable = true;
-      SuccInfo.StrongNoFrameOnEntry |=
-          Info.StrongNoFrameOnEntry && !HasPrologue;
-      SuccInfo.HasFrameOnEntry = Info.HasFrameOnExit;
-    }
-  }
+  BlockFlagsVector BlockInfo = computeBlockInfo(MF, PrologueBlock);
 
   // Walk the blocks of the function in "physical" order.
   // Every block inherits the frame state (as recorded in the unwind tables)
@@ -253,57 +329,10 @@ bool CFIFixup::runOnMachineFunction(MachineFunction &MF) {
   // No point starting before the prologue block.
   // TODO: the unwind tables will still be incorrect if an epilogue physically
   // preceeds the prologue.
-  MachineFunction::iterator CurrBB = std::next(PrologueBlock->getIterator());
-  bool HasFrame = BlockInfo[PrologueBlock->getNumber()].HasFrameOnExit;
-  while (CurrBB != MF.end()) {
-    const BlockFlags &Info = BlockInfo[CurrBB->getNumber()];
-    if (!Info.Reachable) {
-      ++CurrBB;
-      continue;
-    }
-
-#ifndef NDEBUG
-    if (!Info.StrongNoFrameOnEntry) {
-      for (auto *Pred : CurrBB->predecessors()) {
-        BlockFlags &PredInfo = BlockInfo[Pred->getNumber()];
-        assert((!PredInfo.Reachable ||
-                Info.HasFrameOnEntry == PredInfo.HasFrameOnExit) &&
-               "Inconsistent call frame state");
-      }
-    }
-#endif
-
-    // If the block is the first block in its section, then it doesn't have a
-    // frame on entry.
-    HasFrame &= !CurrBB->isBeginSection();
-    if (!Info.StrongNoFrameOnEntry && Info.HasFrameOnEntry && !HasFrame) {
-      // Reset to the "after prologue" state.
-
-      InsertionPoint &InsertPt = InsertionPts[CurrBB->getSectionID()];
-      if (InsertPt.MBB == nullptr) {
-        // CurBB is the first block in its section, so there is no "after
-        // prologue" state. Clone the CFI instructions from the prologue block
-        // to create it.
-        InsertPt = cloneCfiPrologue({PrologueBlock, PrologueEnd},
-                                    {&*CurrBB, CurrBB->begin()});
-      } else {
-        // There's an earlier block known to have a stack frame. Insert a
-        // `.cfi_remember_state` instruction into that block and a
-        // `.cfi_restore_state` instruction at the beginning of the current
-        // block.
-        InsertPt =
-            insertRememberRestorePair(InsertPt, {&*CurrBB, CurrBB->begin()});
-      }
-      Change = true;
-    } else if ((Info.StrongNoFrameOnEntry || !Info.HasFrameOnEntry) &&
-               HasFrame) {
-      // Reset to the state upon function entry.
-      TFL.resetCFIToInitialState(*CurrBB);
-      Change = true;
-    }
-
-    HasFrame = Info.HasFrameOnExit;
-    ++CurrBB;
+  for (MachineBasicBlock &MBB :
+       make_range(std::next(PrologueBlock->getIterator()), MF.end())) {
+    Change |=
+        fixupBlock(MBB, BlockInfo, InsertionPts, {PrologueBlock, PrologueEnd});
   }
 
   return Change;
diff --git a/llvm/lib/CodeGen/CodeGen.cpp b/llvm/lib/CodeGen/CodeGen.cpp
index ed871519e33bc..d69a24f00871e 100644
--- a/llvm/lib/CodeGen/CodeGen.cpp
+++ b/llvm/lib/CodeGen/CodeGen.cpp
@@ -77,7 +77,7 @@ void llvm::initializeCodeGen(PassRegistry &Registry) {
   initializeMachineCFGPrinterPass(Registry);
   initializeMachineCSELegacyPass(Registry);
   initializeMachineCombinerPass(Registry);
-  initializeMachineCopyPropagationPass(Registry);
+  initializeMachineCopyPropagationLegacyPass(Registry);
   initializeMachineCycleInfoPrinterPassPass(Registry);
   initializeMachineCycleInfoWrapperPassPass(Registry);
   initializeMachineDominatorTreeWrapperPassPass(Registry);
@@ -108,7 +108,7 @@ void llvm::initializeCodeGen(PassRegistry &Registry) {
   initializePostMachineSchedulerPass(Registry);
   initializePostRAHazardRecognizerPass(Registry);
   initializePostRAMachineSinkingPass(Registry);
-  initializePostRASchedulerPass(Registry);
+  initializePostRASchedulerLegacyPass(Registry);
   initializePreISelIntrinsicLoweringLegacyPassPass(Registry);
   initializeProcessImplicitDefsPass(Registry);
   initializeRABasicPass(Registry);
@@ -116,10 +116,10 @@ void llvm::initializeCodeGen(PassRegistry &Registry) {
   initializeRegAllocFastPass(Registry);
   initializeRegUsageInfoCollectorLegacyPass(Registry);
   initializeRegUsageInfoPropagationLegacyPass(Registry);
-  initializeRegisterCoalescerPass(Registry);
+  initializeRegisterCoalescerLegacyPass(Registry);
   initializeRemoveLoadsIntoFakeUsesPass(Registry);
   initializeRemoveRedundantDebugValuesPass(Registry);
-  initializeRenameIndependentSubregsPass(Registry);
+  initializeRenameIndependentSubregsLegacyPass(Registry);
   initializeSafeStackLegacyPassPass(Registry);
   initializeSelectOptimizePass(Registry);
   initializeShadowStackGCLoweringPass(Registry);
@@ -130,7 +130,7 @@ void llvm::initializeCodeGen(PassRegistry &Registry) {
   initializeStackFrameLayoutAnalysisPassPass(Registry);
   initializeStackMapLivenessPass(Registry);
   initializeStackProtectorPass(Registry);
-  initializeStackSlotColoringPass(Registry);
+  initializeStackSlotColoringLegacyPass(Registry);
   initializeStaticDataSplitterPass(Registry);
   initializeStripDebugMachineModulePass(Registry);
   initializeTailDuplicateLegacyPass(Registry);
diff --git a/llvm/lib/CodeGen/CodeGenPrepare.cpp b/llvm/lib/CodeGen/CodeGenPrepare.cpp
index 7e9d705a7bef6..088062afab177 100644
--- a/llvm/lib/CodeGen/CodeGenPrepare.cpp
+++ b/llvm/lib/CodeGen/CodeGenPrepare.cpp
@@ -990,7 +990,7 @@ bool CodeGenPrepare::isMergingEmptyBlockProfitable(BasicBlock *BB,
                  isa<IndirectBrInst>(Pred->getTerminator())))
     return true;
 
-  if (BB->getTerminator() != BB->getFirstNonPHIOrDbg())
+  if (BB->getTerminator() != &*BB->getFirstNonPHIOrDbg())
     return true;
 
   // We use a simple cost heuristic which determine skipping merging is
@@ -2935,13 +2935,13 @@ bool CodeGenPrepare::dupRetToEnableTailCallOpts(BasicBlock *BB,
 
   // Make sure there are no instructions between the first instruction
   // and return.
-  const Instruction *BI = BB->getFirstNonPHI();
+  BasicBlock::const_iterator BI = BB->getFirstNonPHIIt();
   // Skip over debug and the bitcast.
-  while (isa<DbgInfoIntrinsic>(BI) || BI == BCI || BI == EVI ||
-         isa<PseudoProbeInst>(BI) || isLifetimeEndOrBitCastFor(BI) ||
-         isFakeUse(BI))
-    BI = BI->getNextNode();
-  if (BI != RetI)
+  while (isa<DbgInfoIntrinsic>(BI) || &*BI == BCI || &*BI == EVI ||
+         isa<PseudoProbeInst>(BI) || isLifetimeEndOrBitCastFor(&*BI) ||
+         isFakeUse(&*BI))
+    BI = std::next(BI);
+  if (&*BI != RetI)
     return false;
 
   /// Only dup the ReturnInst if the CallInst is likely to be emitted as a tail
@@ -3074,6 +3074,14 @@ struct ExtAddrMode : public TargetLowering::AddrMode {
   void print(raw_ostream &OS) const;
   void dump() const;
 
+  // Replace From in ExtAddrMode with To.
+  // E.g., SExt insts may be promoted and deleted. We should replace them with
+  // the promoted values.
+  void replaceWith(Value *From, Value *To) {
+    if (ScaledReg == From)
+      ScaledReg = To;
+  }
+
   FieldName compare(const ExtAddrMode &other) {
     // First check that the types are the same on each field, as differing types
     // is something we can't cope with later on.
@@ -3265,8 +3273,8 @@ class TypePromotionTransaction {
     /// Either an instruction:
     /// - Is the first in a basic block: BB is used.
     /// - Has a previous instruction: PrevInst is used.
-    union {
-      Instruction *PrevInst;
+    struct {
+      BasicBlock::iterator PrevInst;
       BasicBlock *BB;
     } Point;
     std::optional<DbgRecord::self_iterator> BeforeDbgRecord = std::nullopt;
@@ -3286,7 +3294,7 @@ class TypePromotionTransaction {
         BeforeDbgRecord = Inst->getDbgReinsertionPosition();
 
       if (HasPrevInstruction) {
-        Point.PrevInst = &*std::prev(Inst->getIterator());
+        Point.PrevInst = std::prev(Inst->getIterator());
       } else {
         Point.BB = BB;
       }
@@ -3297,7 +3305,7 @@ class TypePromotionTransaction {
       if (HasPrevInstruction) {
         if (Inst->getParent())
           Inst->removeFromParent();
-        Inst->insertAfter(&*Point.PrevInst);
+        Inst->insertAfter(Point.PrevInst);
       } else {
         BasicBlock::iterator Position = Point.BB->getFirstInsertionPt();
         if (Inst->getParent())
@@ -3317,7 +3325,7 @@ class TypePromotionTransaction {
 
   public:
     /// Move \p Inst before \p Before.
-    InstructionMoveBefore(Instruction *Inst, Instruction *Before)
+    InstructionMoveBefore(Instruction *Inst, BasicBlock::iterator Before)
         : TypePromotionAction(Inst), Position(Inst) {
       LLVM_DEBUG(dbgs() << "Do: move: " << *Inst << "\nbefore: " << *Before
                         << "\n");
@@ -5365,6 +5373,9 @@ bool AddressingModeMatcher::matchOperationAddr(User *AddrInst, unsigned Opcode,
       TPT.rollback(LastKnownGood);
       return false;
     }
+
+    // SExt has been deleted. Make sure it is not referenced by the AddrMode.
+    AddrMode.replaceWith(Ext, PromotedOperand);
     return true;
   }
   case Instruction::Call:
diff --git a/llvm/lib/CodeGen/CommandFlags.cpp b/llvm/lib/CodeGen/CommandFlags.cpp
index d180cfcea658c..023656cde0089 100644
--- a/llvm/lib/CodeGen/CommandFlags.cpp
+++ b/llvm/lib/CodeGen/CommandFlags.cpp
@@ -103,6 +103,7 @@ CGOPT(bool, EnableStackSizeSection)
 CGOPT(bool, EnableAddrsig)
 CGOPT(bool, EmitCallSiteInfo)
 CGOPT(bool, EnableMachineFunctionSplitter)
+CGOPT(bool, EnableStaticDataPartitioning)
 CGOPT(bool, EnableDebugEntryValues)
 CGOPT(bool, ForceDwarfFrameSection)
 CGOPT(bool, XRayFunctionIndex)
@@ -480,6 +481,12 @@ codegen::RegisterCodeGenFlags::RegisterCodeGenFlags() {
       cl::init(false));
   CGBINDOPT(EnableMachineFunctionSplitter);
 
+  static cl::opt<bool> EnableStaticDataPartitioning(
+      "partition-static-data-sections",
+      cl::desc("Partition data sections using profile information."),
+      cl::init(false));
+  CGBINDOPT(EnableStaticDataPartitioning);
+
   static cl::opt<bool> ForceDwarfFrameSection(
       "force-dwarf-frame-section",
       cl::desc("Always emit a debug frame section."), cl::init(false));
@@ -586,6 +593,7 @@ codegen::InitTargetOptionsFromCodeGenFlags(const Triple &TheTriple) {
   Options.ExceptionModel = getExceptionModel();
   Options.EmitStackSizeSection = getEnableStackSizeSection();
   Options.EnableMachineFunctionSplitter = getEnableMachineFunctionSplitter();
+  Options.EnableStaticDataPartitioning = getEnableStaticDataPartitioning();
   Options.EmitAddrsig = getEnableAddrsig();
   Options.EmitCallSiteInfo = getEmitCallSiteInfo();
   Options.EnableDebugEntryValues = getEnableDebugEntryValues();
diff --git a/llvm/lib/CodeGen/DetectDeadLanes.cpp b/llvm/lib/CodeGen/DetectDeadLanes.cpp
index 86e9f3abe010d..a6d2640ed044f 100644
--- a/llvm/lib/CodeGen/DetectDeadLanes.cpp
+++ b/llvm/lib/CodeGen/DetectDeadLanes.cpp
@@ -118,7 +118,7 @@ void DeadLaneDetector::addUsedLanesOnOperand(const MachineOperand &MO,
     UsedLanes = TRI->composeSubRegIndexLaneMask(MOSubReg, UsedLanes);
   UsedLanes &= MRI->getMaxLaneMaskForVReg(MOReg);
 
-  unsigned MORegIdx = Register::virtReg2Index(MOReg);
+  unsigned MORegIdx = MOReg.virtRegIndex();
   DeadLaneDetector::VRegInfo &MORegInfo = VRegInfos[MORegIdx];
   LaneBitmask PrevUsedLanes = MORegInfo.UsedLanes;
   // Any change at all?
@@ -147,7 +147,7 @@ DeadLaneDetector::transferUsedLanes(const MachineInstr &MI,
                                     const MachineOperand &MO) const {
   unsigned OpNum = MO.getOperandNo();
   assert(lowersToCopies(MI) &&
-         DefinedByCopy[Register::virtReg2Index(MI.getOperand(0).getReg())]);
+         DefinedByCopy[MI.getOperand(0).getReg().virtRegIndex()]);
 
   switch (MI.getOpcode()) {
   case TargetOpcode::COPY:
@@ -204,7 +204,7 @@ void DeadLaneDetector::transferDefinedLanesStep(const MachineOperand &Use,
   Register DefReg = Def.getReg();
   if (!DefReg.isVirtual())
     return;
-  unsigned DefRegIdx = Register::virtReg2Index(DefReg);
+  unsigned DefRegIdx = DefReg.virtRegIndex();
   if (!DefinedByCopy.test(DefRegIdx))
     return;
 
@@ -433,7 +433,7 @@ bool DetectDeadLanes::isUndefInput(const DeadLaneDetector &DLD,
   Register DefReg = Def.getReg();
   if (!DefReg.isVirtual())
     return false;
-  unsigned DefRegIdx = Register::virtReg2Index(DefReg);
+  unsigned DefRegIdx = DefReg.virtRegIndex();
   if (!DLD.isDefinedByCopy(DefRegIdx))
     return false;
 
@@ -506,7 +506,7 @@ DetectDeadLanes::modifySubRegisterOperandStatus(const DeadLaneDetector &DLD,
         Register Reg = MO.getReg();
         if (!Reg.isVirtual())
           continue;
-        unsigned RegIdx = Register::virtReg2Index(Reg);
+        unsigned RegIdx = Reg.virtRegIndex();
         const DeadLaneDetector::VRegInfo &RegInfo = DLD.getVRegInfo(RegIdx);
         if (MO.isDef() && !MO.isDead() && RegInfo.UsedLanes.none()) {
           LLVM_DEBUG(dbgs()
diff --git a/llvm/lib/CodeGen/GlobalISel/CSEInfo.cpp b/llvm/lib/CodeGen/GlobalISel/CSEInfo.cpp
index 3a9069848ca1d..0222069cfc576 100644
--- a/llvm/lib/CodeGen/GlobalISel/CSEInfo.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/CSEInfo.cpp
@@ -181,10 +181,7 @@ MachineInstr *GISelCSEInfo::getMachineInstrIfExists(FoldingSetNodeID &ID,
 
 void GISelCSEInfo::countOpcodeHit(unsigned Opc) {
 #ifndef NDEBUG
-  if (OpcodeHitTable.count(Opc))
-    OpcodeHitTable[Opc] += 1;
-  else
-    OpcodeHitTable[Opc] = 1;
+  ++OpcodeHitTable[Opc];
 #endif
   // Else do nothing.
 }
diff --git a/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp b/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp
index b193d8bb0aa18..0dfbb91f2ac54 100644
--- a/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp
@@ -5641,6 +5641,7 @@ bool CombinerHelper::matchUMulHToLShr(MachineInstr &MI) const {
   Register RHS = MI.getOperand(2).getReg();
   Register Dst = MI.getOperand(0).getReg();
   LLT Ty = MRI.getType(Dst);
+  LLT RHSTy = MRI.getType(RHS);
   LLT ShiftAmtTy = getTargetLowering().getPreferredShiftAmountTy(Ty);
   auto MatchPow2ExceptOne = [&](const Constant *C) {
     if (auto *CI = dyn_cast<ConstantInt>(C))
@@ -5649,7 +5650,10 @@ bool CombinerHelper::matchUMulHToLShr(MachineInstr &MI) const {
   };
   if (!matchUnaryPredicate(MRI, RHS, MatchPow2ExceptOne, false))
     return false;
-  return isLegalOrBeforeLegalizer({TargetOpcode::G_LSHR, {Ty, ShiftAmtTy}});
+  // We need to check both G_LSHR and G_CTLZ because the combine uses G_CTLZ to
+  // get log base 2, and it is not always legal for on a target.
+  return isLegalOrBeforeLegalizer({TargetOpcode::G_LSHR, {Ty, ShiftAmtTy}}) &&
+         isLegalOrBeforeLegalizer({TargetOpcode::G_CTLZ, {RHSTy, RHSTy}});
 }
 
 void CombinerHelper::applyUMulHToLShr(MachineInstr &MI) const {
@@ -6595,8 +6599,9 @@ bool CombinerHelper::matchRedundantBinOpInEquality(MachineInstr &MI,
 static std::optional<unsigned>
 getMinUselessShift(KnownBits ValueKB, unsigned Opcode,
                    std::optional<int64_t> &Result) {
-  assert(Opcode == TargetOpcode::G_SHL || Opcode == TargetOpcode::G_LSHR ||
-         Opcode == TargetOpcode::G_ASHR && "Expect G_SHL, G_LSHR or G_ASHR.");
+  assert((Opcode == TargetOpcode::G_SHL || Opcode == TargetOpcode::G_LSHR ||
+          Opcode == TargetOpcode::G_ASHR) &&
+         "Expect G_SHL, G_LSHR or G_ASHR.");
   auto SignificantBits = 0;
   switch (Opcode) {
   case TargetOpcode::G_SHL:
diff --git a/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp b/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
index 21622ea43724c..d01de29826cad 100644
--- a/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
@@ -79,7 +79,6 @@
 #include "llvm/Support/ErrorHandling.h"
 #include "llvm/Support/MathExtras.h"
 #include "llvm/Support/raw_ostream.h"
-#include "llvm/Target/TargetIntrinsicInfo.h"
 #include "llvm/Target/TargetMachine.h"
 #include "llvm/Transforms/Utils/Local.h"
 #include "llvm/Transforms/Utils/MemoryOpRemark.h"
@@ -296,8 +295,21 @@ void IRTranslator::addMachineCFGPred(CFGEdge Edge, MachineBasicBlock *NewPred) {
   MachinePreds[Edge].push_back(NewPred);
 }
 
+static bool containsBF16Type(const User &U) {
+  // BF16 cannot currently be represented by LLT, to avoid miscompiles we
+  // prevent any instructions using them. FIXME: This can be removed once LLT
+  // supports bfloat.
+  return U.getType()->getScalarType()->isBFloatTy() ||
+         any_of(U.operands(), [](Value *V) {
+           return V->getType()->getScalarType()->isBFloatTy();
+         });
+}
+
 bool IRTranslator::translateBinaryOp(unsigned Opcode, const User &U,
                                      MachineIRBuilder &MIRBuilder) {
+  if (containsBF16Type(U))
+    return false;
+
   // Get or create a virtual register for each value.
   // Unless the value is a Constant => loadimm cst?
   // or inline constant each time?
@@ -317,6 +329,9 @@ bool IRTranslator::translateBinaryOp(unsigned Opcode, const User &U,
 
 bool IRTranslator::translateUnaryOp(unsigned Opcode, const User &U,
                                     MachineIRBuilder &MIRBuilder) {
+  if (containsBF16Type(U))
+    return false;
+
   Register Op0 = getOrCreateVReg(*U.getOperand(0));
   Register Res = getOrCreateVReg(U);
   uint32_t Flags = 0;
@@ -334,6 +349,9 @@ bool IRTranslator::translateFNeg(const User &U, MachineIRBuilder &MIRBuilder) {
 
 bool IRTranslator::translateCompare(const User &U,
                                     MachineIRBuilder &MIRBuilder) {
+  if (containsBF16Type(U))
+    return false;
+
   auto *CI = cast<CmpInst>(&U);
   Register Op0 = getOrCreateVReg(*U.getOperand(0));
   Register Op1 = getOrCreateVReg(*U.getOperand(1));
@@ -1553,8 +1571,7 @@ bool IRTranslator::translateBitCast(const User &U,
 
 bool IRTranslator::translateCast(unsigned Opcode, const User &U,
                                  MachineIRBuilder &MIRBuilder) {
-  if (U.getType()->getScalarType()->isBFloatTy() ||
-      U.getOperand(0)->getType()->getScalarType()->isBFloatTy())
+  if (containsBF16Type(U))
     return false;
 
   uint32_t Flags = 0;
@@ -2423,9 +2440,7 @@ bool IRTranslator::translateKnownIntrinsic(const CallInst &CI, Intrinsic::ID ID,
     return true;
   }
   case Intrinsic::invariant_start: {
-    LLT PtrTy = getLLTForType(*CI.getArgOperand(0)->getType(), *DL);
-    Register Undef = MRI->createGenericVirtualRegister(PtrTy);
-    MIRBuilder.buildUndef(Undef);
+    MIRBuilder.buildUndef(getOrCreateVReg(CI));
     return true;
   }
   case Intrinsic::invariant_end:
@@ -2647,6 +2662,8 @@ bool IRTranslator::translateKnownIntrinsic(const CallInst &CI, Intrinsic::ID ID,
 
 bool IRTranslator::translateInlineAsm(const CallBase &CB,
                                       MachineIRBuilder &MIRBuilder) {
+  if (containsBF16Type(CB))
+    return false;
 
   const InlineAsmLowering *ALI = MF->getSubtarget().getInlineAsmLowering();
 
@@ -2736,8 +2753,10 @@ bool IRTranslator::translateCallBase(const CallBase &CB,
 }
 
 bool IRTranslator::translateCall(const User &U, MachineIRBuilder &MIRBuilder) {
+  if (containsBF16Type(U))
+    return false;
+
   const CallInst &CI = cast<CallInst>(U);
-  auto TII = MF->getTarget().getIntrinsicInfo();
   const Function *F = CI.getCalledFunction();
 
   // FIXME: support Windows dllimport function calls and calls through
@@ -2761,11 +2780,8 @@ bool IRTranslator::translateCall(const User &U, MachineIRBuilder &MIRBuilder) {
   diagnoseDontCall(CI);
 
   Intrinsic::ID ID = Intrinsic::not_intrinsic;
-  if (F && F->isIntrinsic()) {
+  if (F && F->isIntrinsic())
     ID = F->getIntrinsicID();
-    if (TII && ID == Intrinsic::not_intrinsic)
-      ID = static_cast<Intrinsic::ID>(TII->getIntrinsicID(F));
-  }
 
   if (!F || !F->isIntrinsic() || ID == Intrinsic::not_intrinsic)
     return translateCallBase(CI, MIRBuilder);
@@ -3371,6 +3387,9 @@ bool IRTranslator::translateAtomicCmpXchg(const User &U,
 
 bool IRTranslator::translateAtomicRMW(const User &U,
                                       MachineIRBuilder &MIRBuilder) {
+  if (containsBF16Type(U))
+    return false;
+
   const AtomicRMWInst &I = cast<AtomicRMWInst>(U);
   auto Flags = TLI->getAtomicMemOperandFlags(I, *DL);
 
diff --git a/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp b/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp
index d0a62340a5f32..d4cb224c35d74 100644
--- a/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp
@@ -411,6 +411,10 @@ static RTLIB::Libcall getRTLibDesc(unsigned Opcode, unsigned Size) {
   } while (0)
 
   switch (Opcode) {
+  case TargetOpcode::G_LROUND:
+    RTLIBCASE(LROUND_F);
+  case TargetOpcode::G_LLROUND:
+    RTLIBCASE(LLROUND_F);
   case TargetOpcode::G_MUL:
     RTLIBCASE_INT(MUL_I);
   case TargetOpcode::G_SDIV:
@@ -1267,6 +1271,8 @@ LegalizerHelper::libcall(MachineInstr &MI, LostDebugLocObserver &LocObserver) {
       return Status;
     break;
   }
+  case TargetOpcode::G_LROUND:
+  case TargetOpcode::G_LLROUND:
   case TargetOpcode::G_INTRINSIC_LRINT:
   case TargetOpcode::G_INTRINSIC_LLRINT: {
     LLT LLTy = MRI.getType(MI.getOperand(1).getReg());
@@ -3342,6 +3348,15 @@ LegalizerHelper::widenScalar(MachineInstr &MI, unsigned TypeIdx, LLT WideTy) {
     Observer.changedInstr(MI);
     return Legalized;
   }
+  case TargetOpcode::G_VECREDUCE_ADD: {
+    if (TypeIdx != 1)
+      return UnableToLegalize;
+    Observer.changingInstr(MI);
+    widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_ANYEXT);
+    widenScalarDst(MI, WideTy.getScalarType(), 0, TargetOpcode::G_TRUNC);
+    Observer.changedInstr(MI);
+    return Legalized;
+  }
   case TargetOpcode::G_VECREDUCE_FADD:
   case TargetOpcode::G_VECREDUCE_FMUL:
   case TargetOpcode::G_VECREDUCE_FMIN:
@@ -7255,7 +7270,7 @@ LegalizerHelper::lowerFunnelShiftAsShifts(MachineInstr &MI) {
     }
   }
 
-  MIRBuilder.buildOr(Dst, ShX, ShY);
+  MIRBuilder.buildOr(Dst, ShX, ShY, MachineInstr::Disjoint);
   MI.eraseFromParent();
   return Legalized;
 }
diff --git a/llvm/lib/CodeGen/GlobalMerge.cpp b/llvm/lib/CodeGen/GlobalMerge.cpp
index 41e01a1d3ccd5..5993fc939a08a 100644
--- a/llvm/lib/CodeGen/GlobalMerge.cpp
+++ b/llvm/lib/CodeGen/GlobalMerge.cpp
@@ -271,8 +271,7 @@ bool GlobalMergeImpl::doMerge(SmallVectorImpl<GlobalVariable *> &Globals,
 
   // If we want to just blindly group all globals together, do so.
   if (!GlobalMergeGroupByUse || (Opt.MergeConstAggressive && isConst)) {
-    BitVector AllGlobals(Globals.size());
-    AllGlobals.set();
+    BitVector AllGlobals(Globals.size(), true);
     return doMerge(Globals, AllGlobals, M, isConst, AddrSpace);
   }
 
@@ -437,13 +436,15 @@ bool GlobalMergeImpl::doMerge(SmallVectorImpl<GlobalVariable *> &Globals,
     return doMerge(Globals, AllGlobals, M, isConst, AddrSpace);
   }
 
-  // Now we found a bunch of sets of globals used together. We accumulated
+  // Now we found a bunch of sets of globals used together.  We accumulated
   // the number of times we encountered the sets (i.e., the number of functions
-  // that use that exact set of globals). Multiply that by the size of the set
-  // to give us a crude profitability metric.
+  // that use that exact set of globals).
+  //
+  // Multiply that by the size of the set to give us a crude profitability
+  // metric.
   llvm::stable_sort(UsedGlobalSets,
                     [](const UsedGlobalSet &UGS1, const UsedGlobalSet &UGS2) {
-                      return UGS1.Globals.count() * UGS1.UsageCount >=
+                      return UGS1.Globals.count() * UGS1.UsageCount <
                              UGS2.Globals.count() * UGS2.UsageCount;
                     });
 
@@ -456,7 +457,7 @@ bool GlobalMergeImpl::doMerge(SmallVectorImpl<GlobalVariable *> &Globals,
   BitVector PickedGlobals(Globals.size());
   bool Changed = false;
 
-  for (const UsedGlobalSet &UGS : UsedGlobalSets) {
+  for (const UsedGlobalSet &UGS : llvm::reverse(UsedGlobalSets)) {
     if (UGS.UsageCount == 0)
       continue;
     if (PickedGlobals.anyCommon(UGS.Globals))
@@ -727,7 +728,8 @@ bool GlobalMergeImpl::run(Module &M) {
 
     Type *Ty = GV.getValueType();
     TypeSize AllocSize = DL.getTypeAllocSize(Ty);
-    if (AllocSize < Opt.MaxOffset && AllocSize >= Opt.MinSize) {
+    bool CanMerge = AllocSize < Opt.MaxOffset && AllocSize >= Opt.MinSize;
+    if (CanMerge) {
       if (TM &&
           TargetLoweringObjectFile::getKindForGlobal(&GV, *TM).isBSS())
         BSSGlobals[{AddressSpace, Section}].push_back(&GV);
@@ -736,11 +738,8 @@ bool GlobalMergeImpl::run(Module &M) {
       else
         Globals[{AddressSpace, Section}].push_back(&GV);
     }
-    LLVM_DEBUG(dbgs() << "GV "
-                      << ((DL.getTypeAllocSize(Ty) < Opt.MaxOffset)
-                              ? "to merge: "
-                              : "not to merge: ")
-                      << GV << "\n");
+    LLVM_DEBUG(dbgs() << "GV " << (CanMerge ? "" : "not ") << "to merge: " << GV
+                      << "\n");
   }
 
   for (auto &P : Globals)
diff --git a/llvm/lib/CodeGen/InitUndef.cpp b/llvm/lib/CodeGen/InitUndef.cpp
index 5fe52035d063f..6c0e9f9e930b9 100644
--- a/llvm/lib/CodeGen/InitUndef.cpp
+++ b/llvm/lib/CodeGen/InitUndef.cpp
@@ -142,8 +142,7 @@ bool InitUndef::handleSubReg(MachineFunction &MF, MachineInstr &MI,
     Register Reg = UseMO.getReg();
     if (NewRegs.count(Reg))
       continue;
-    DeadLaneDetector::VRegInfo Info =
-        DLD.getVRegInfo(Register::virtReg2Index(Reg));
+    DeadLaneDetector::VRegInfo Info = DLD.getVRegInfo(Reg.virtRegIndex());
 
     if (Info.UsedLanes == Info.DefinedLanes)
       continue;
diff --git a/llvm/lib/CodeGen/InlineSpiller.cpp b/llvm/lib/CodeGen/InlineSpiller.cpp
index d98254650a001..302dd37ff3d67 100644
--- a/llvm/lib/CodeGen/InlineSpiller.cpp
+++ b/llvm/lib/CodeGen/InlineSpiller.cpp
@@ -1320,13 +1320,16 @@ void HoistSpillHelper::addToMergeableSpills(MachineInstr &Spill, int StackSlot,
   LiveInterval &OrigLI = LIS.getInterval(Original);
   // save a copy of LiveInterval in StackSlotToOrigLI because the original
   // LiveInterval may be cleared after all its references are spilled.
-  if (!StackSlotToOrigLI.contains(StackSlot)) {
+
+  auto [Place, Inserted] = StackSlotToOrigLI.try_emplace(StackSlot);
+  if (Inserted) {
     auto LI = std::make_unique<LiveInterval>(OrigLI.reg(), OrigLI.weight());
     LI->assign(OrigLI, Allocator);
-    StackSlotToOrigLI[StackSlot] = std::move(LI);
+    Place->second = std::move(LI);
   }
+
   SlotIndex Idx = LIS.getInstructionIndex(Spill);
-  VNInfo *OrigVNI = StackSlotToOrigLI[StackSlot]->getVNInfoAt(Idx.getRegSlot());
+  VNInfo *OrigVNI = Place->second->getVNInfoAt(Idx.getRegSlot());
   std::pair<int, VNInfo *> MIdx = std::make_pair(StackSlot, OrigVNI);
   MergeableSpills[MIdx].insert(&Spill);
 }
@@ -1529,10 +1532,12 @@ void HoistSpillHelper::runHoistSpills(
     MachineBasicBlock *Block = (*RIt)->getBlock();
 
     // If Block contains an original spill, simply continue.
-    if (SpillsToKeep.contains(*RIt) && !SpillsToKeep[*RIt]) {
-      SpillsInSubTreeMap[*RIt].first.insert(*RIt);
-      // SpillsInSubTreeMap[*RIt].second contains the cost of spill.
-      SpillsInSubTreeMap[*RIt].second = MBFI.getBlockFreq(Block);
+    if (auto It = SpillsToKeep.find(*RIt);
+        It != SpillsToKeep.end() && !It->second) {
+      auto &SIt = SpillsInSubTreeMap[*RIt];
+      SIt.first.insert(*RIt);
+      // Sit.second contains the cost of spill.
+      SIt.second = MBFI.getBlockFreq(Block);
       continue;
     }
 
@@ -1578,7 +1583,8 @@ void HoistSpillHelper::runHoistSpills(
       for (auto *const SpillBB : SpillsInSubTree) {
         // When SpillBB is a BB contains original spill, insert the spill
         // to SpillsToRm.
-        if (SpillsToKeep.contains(SpillBB) && !SpillsToKeep[SpillBB]) {
+        if (auto It = SpillsToKeep.find(SpillBB);
+            It != SpillsToKeep.end() && !It->second) {
           MachineInstr *SpillToRm = SpillBBToSpill[SpillBB];
           SpillsToRm.push_back(SpillToRm);
         }
diff --git a/llvm/lib/CodeGen/InterleavedAccessPass.cpp b/llvm/lib/CodeGen/InterleavedAccessPass.cpp
index 3f6a69ecb7d72..3261f2858b236 100644
--- a/llvm/lib/CodeGen/InterleavedAccessPass.cpp
+++ b/llvm/lib/CodeGen/InterleavedAccessPass.cpp
@@ -630,11 +630,37 @@ getVectorDeinterleaveFactor(IntrinsicInst *II,
   return true;
 }
 
+// Return the corresponded deinterleaved mask, or nullptr if there is no valid
+// mask.
+static Value *getMask(Value *WideMask, unsigned Factor,
+                      VectorType *LeafValueTy) {
+  using namespace llvm::PatternMatch;
+  if (auto *IMI = dyn_cast<IntrinsicInst>(WideMask)) {
+    SmallVector<Value *, 8> Operands;
+    SmallVector<Instruction *, 8> DeadInsts;
+    if (getVectorInterleaveFactor(IMI, Operands, DeadInsts)) {
+      assert(!Operands.empty());
+      if (Operands.size() == Factor && llvm::all_equal(Operands))
+        return Operands[0];
+    }
+  }
+
+  if (match(WideMask, m_AllOnes())) {
+    // Scale the vector length of all-ones mask.
+    ElementCount OrigEC =
+        cast<VectorType>(WideMask->getType())->getElementCount();
+    assert(OrigEC.getKnownMinValue() % Factor == 0);
+    return ConstantVector::getSplat(OrigEC.divideCoefficientBy(Factor),
+                                    cast<Constant>(WideMask)->getSplatValue());
+  }
+
+  return nullptr;
+}
+
 bool InterleavedAccessImpl::lowerDeinterleaveIntrinsic(
     IntrinsicInst *DI, SmallSetVector<Instruction *, 32> &DeadInsts) {
-  LoadInst *LI = dyn_cast<LoadInst>(DI->getOperand(0));
-
-  if (!LI || !LI->hasOneUse() || !LI->isSimple())
+  Value *LoadedVal = DI->getOperand(0);
+  if (!LoadedVal->hasOneUse() || !isa<LoadInst, VPIntrinsic>(LoadedVal))
     return false;
 
   SmallVector<Value *, 8> DeinterleaveValues;
@@ -643,16 +669,43 @@ bool InterleavedAccessImpl::lowerDeinterleaveIntrinsic(
                                    DeinterleaveDeadInsts))
     return false;
 
-  LLVM_DEBUG(dbgs() << "IA: Found a deinterleave intrinsic: " << *DI
-                    << " with factor = " << DeinterleaveValues.size() << "\n");
+  const unsigned Factor = DeinterleaveValues.size();
 
-  // Try and match this with target specific intrinsics.
-  if (!TLI->lowerDeinterleaveIntrinsicToLoad(LI, DeinterleaveValues))
-    return false;
+  if (auto *VPLoad = dyn_cast<VPIntrinsic>(LoadedVal)) {
+    if (VPLoad->getIntrinsicID() != Intrinsic::vp_load)
+      return false;
+    // Check mask operand. Handle both all-true and interleaved mask.
+    Value *WideMask = VPLoad->getOperand(1);
+    Value *Mask = getMask(WideMask, Factor,
+                          cast<VectorType>(DeinterleaveValues[0]->getType()));
+    if (!Mask)
+      return false;
+
+    LLVM_DEBUG(dbgs() << "IA: Found a vp.load with deinterleave intrinsic "
+                      << *DI << " and factor = " << Factor << "\n");
+
+    // Since lowerInterleaveLoad expects Shuffles and LoadInst, use special
+    // TLI function to emit target-specific interleaved instruction.
+    if (!TLI->lowerDeinterleavedIntrinsicToVPLoad(VPLoad, Mask,
+                                                  DeinterleaveValues))
+      return false;
+
+  } else {
+    auto *LI = cast<LoadInst>(LoadedVal);
+    if (!LI->isSimple())
+      return false;
+
+    LLVM_DEBUG(dbgs() << "IA: Found a load with deinterleave intrinsic " << *DI
+                      << " and factor = " << Factor << "\n");
+
+    // Try and match this with target specific intrinsics.
+    if (!TLI->lowerDeinterleaveIntrinsicToLoad(LI, DeinterleaveValues))
+      return false;
+  }
 
   DeadInsts.insert(DeinterleaveDeadInsts.begin(), DeinterleaveDeadInsts.end());
   // We now have a target-specific load, so delete the old one.
-  DeadInsts.insert(LI);
+  DeadInsts.insert(cast<Instruction>(LoadedVal));
   return true;
 }
 
@@ -660,10 +713,8 @@ bool InterleavedAccessImpl::lowerInterleaveIntrinsic(
     IntrinsicInst *II, SmallSetVector<Instruction *, 32> &DeadInsts) {
   if (!II->hasOneUse())
     return false;
-
-  StoreInst *SI = dyn_cast<StoreInst>(*(II->users().begin()));
-
-  if (!SI || !SI->isSimple())
+  Value *StoredBy = II->user_back();
+  if (!isa<StoreInst, VPIntrinsic>(StoredBy))
     return false;
 
   SmallVector<Value *, 8> InterleaveValues;
@@ -671,15 +722,41 @@ bool InterleavedAccessImpl::lowerInterleaveIntrinsic(
   if (!getVectorInterleaveFactor(II, InterleaveValues, InterleaveDeadInsts))
     return false;
 
-  LLVM_DEBUG(dbgs() << "IA: Found an interleave intrinsic: " << *II
-                    << " with factor = " << InterleaveValues.size() << "\n");
+  const unsigned Factor = InterleaveValues.size();
 
-  // Try and match this with target specific intrinsics.
-  if (!TLI->lowerInterleaveIntrinsicToStore(SI, InterleaveValues))
-    return false;
+  if (auto *VPStore = dyn_cast<VPIntrinsic>(StoredBy)) {
+    if (VPStore->getIntrinsicID() != Intrinsic::vp_store)
+      return false;
+
+    Value *WideMask = VPStore->getOperand(2);
+    Value *Mask = getMask(WideMask, Factor,
+                          cast<VectorType>(InterleaveValues[0]->getType()));
+    if (!Mask)
+      return false;
+
+    LLVM_DEBUG(dbgs() << "IA: Found a vp.store with interleave intrinsic "
+                      << *II << " and factor = " << Factor << "\n");
+
+    // Since lowerInterleavedStore expects Shuffle and StoreInst, use special
+    // TLI function to emit target-specific interleaved instruction.
+    if (!TLI->lowerInterleavedIntrinsicToVPStore(VPStore, Mask,
+                                                 InterleaveValues))
+      return false;
+  } else {
+    auto *SI = cast<StoreInst>(StoredBy);
+    if (!SI->isSimple())
+      return false;
+
+    LLVM_DEBUG(dbgs() << "IA: Found a store with interleave intrinsic " << *II
+                      << " and factor = " << Factor << "\n");
+
+    // Try and match this with target specific intrinsics.
+    if (!TLI->lowerInterleaveIntrinsicToStore(SI, InterleaveValues))
+      return false;
+  }
 
   // We now have a target-specific store, so delete the old one.
-  DeadInsts.insert(SI);
+  DeadInsts.insert(cast<Instruction>(StoredBy));
   DeadInsts.insert(InterleaveDeadInsts.begin(), InterleaveDeadInsts.end());
   return true;
 }
diff --git a/llvm/lib/CodeGen/LiveIntervals.cpp b/llvm/lib/CodeGen/LiveIntervals.cpp
index 4fdfcf547542d..3485a27335f13 100644
--- a/llvm/lib/CodeGen/LiveIntervals.cpp
+++ b/llvm/lib/CodeGen/LiveIntervals.cpp
@@ -1080,10 +1080,10 @@ class LiveIntervals::HMEditor {
           for (LiveInterval::SubRange &S : LI.subranges()) {
             if ((S.LaneMask & LaneMask).none())
               continue;
-            updateRange(S, Reg, S.LaneMask);
+            updateRange(S, VirtRegOrUnit(Reg), S.LaneMask);
           }
         }
-        updateRange(LI, Reg, LaneBitmask::getNone());
+        updateRange(LI, VirtRegOrUnit(Reg), LaneBitmask::getNone());
         // If main range has a hole and we are moving a subrange use across
         // the hole updateRange() cannot properly handle it since it only
         // gets the LiveRange and not the whole LiveInterval. As a result
@@ -1110,7 +1110,7 @@ class LiveIntervals::HMEditor {
       // precomputed live range.
       for (MCRegUnit Unit : TRI.regunits(Reg.asMCReg()))
         if (LiveRange *LR = getRegUnitLI(Unit))
-          updateRange(*LR, Unit, LaneBitmask::getNone());
+          updateRange(*LR, VirtRegOrUnit(Unit), LaneBitmask::getNone());
     }
     if (hasRegMask)
       updateRegMaskSlots();
@@ -1119,24 +1119,25 @@ class LiveIntervals::HMEditor {
 private:
   /// Update a single live range, assuming an instruction has been moved from
   /// OldIdx to NewIdx.
-  void updateRange(LiveRange &LR, Register Reg, LaneBitmask LaneMask) {
+  void updateRange(LiveRange &LR, VirtRegOrUnit VRegOrUnit,
+                   LaneBitmask LaneMask) {
     if (!Updated.insert(&LR).second)
       return;
     LLVM_DEBUG({
       dbgs() << "     ";
-      if (Reg.isVirtual()) {
-        dbgs() << printReg(Reg);
+      if (VRegOrUnit.isVirtualReg()) {
+        dbgs() << printReg(VRegOrUnit.asVirtualReg());
         if (LaneMask.any())
           dbgs() << " L" << PrintLaneMask(LaneMask);
       } else {
-        dbgs() << printRegUnit(Reg, &TRI);
+        dbgs() << printRegUnit(VRegOrUnit.asMCRegUnit(), &TRI);
       }
       dbgs() << ":\t" << LR << '\n';
     });
     if (SlotIndex::isEarlierInstr(OldIdx, NewIdx))
       handleMoveDown(LR);
     else
-      handleMoveUp(LR, Reg, LaneMask);
+      handleMoveUp(LR, VRegOrUnit, LaneMask);
     LLVM_DEBUG(dbgs() << "        -->\t" << LR << '\n');
     assert(LR.verify());
   }
@@ -1316,7 +1317,8 @@ class LiveIntervals::HMEditor {
 
   /// Update LR to reflect an instruction has been moved upwards from OldIdx
   /// to NewIdx (NewIdx < OldIdx).
-  void handleMoveUp(LiveRange &LR, Register Reg, LaneBitmask LaneMask) {
+  void handleMoveUp(LiveRange &LR, VirtRegOrUnit VRegOrUnit,
+                    LaneBitmask LaneMask) {
     LiveRange::iterator E = LR.end();
     // Segment going into OldIdx.
     LiveRange::iterator OldIdxIn = LR.find(OldIdx.getBaseIndex());
@@ -1340,7 +1342,7 @@ class LiveIntervals::HMEditor {
       SlotIndex DefBeforeOldIdx
         = std::max(OldIdxIn->start.getDeadSlot(),
                    NewIdx.getRegSlot(OldIdxIn->end.isEarlyClobber()));
-      OldIdxIn->end = findLastUseBefore(DefBeforeOldIdx, Reg, LaneMask);
+      OldIdxIn->end = findLastUseBefore(DefBeforeOldIdx, VRegOrUnit, LaneMask);
 
       // Did we have a Def at OldIdx? If not we are done now.
       OldIdxOut = std::next(OldIdxIn);
@@ -1498,11 +1500,12 @@ class LiveIntervals::HMEditor {
   }
 
   // Return the last use of reg between NewIdx and OldIdx.
-  SlotIndex findLastUseBefore(SlotIndex Before, Register Reg,
+  SlotIndex findLastUseBefore(SlotIndex Before, VirtRegOrUnit VRegOrUnit,
                               LaneBitmask LaneMask) {
-    if (Reg.isVirtual()) {
+    if (VRegOrUnit.isVirtualReg()) {
       SlotIndex LastUse = Before;
-      for (MachineOperand &MO : MRI.use_nodbg_operands(Reg)) {
+      for (MachineOperand &MO :
+           MRI.use_nodbg_operands(VRegOrUnit.asVirtualReg())) {
         if (MO.isUndef())
           continue;
         unsigned SubReg = MO.getSubReg();
@@ -1545,7 +1548,7 @@ class LiveIntervals::HMEditor {
       // Check if MII uses Reg.
       for (MIBundleOperands MO(*MII); MO.isValid(); ++MO)
         if (MO->isReg() && !MO->isUndef() && MO->getReg().isPhysical() &&
-            TRI.hasRegUnit(MO->getReg(), Reg))
+            TRI.hasRegUnit(MO->getReg(), VRegOrUnit.asMCRegUnit()))
           return Idx.getRegSlot();
     }
     // Didn't reach Before. It must be the first instruction in the block.
diff --git a/llvm/lib/CodeGen/MIRParser/MIParser.cpp b/llvm/lib/CodeGen/MIRParser/MIParser.cpp
index 19c73374c3703..a4e513d05f9cd 100644
--- a/llvm/lib/CodeGen/MIRParser/MIParser.cpp
+++ b/llvm/lib/CodeGen/MIRParser/MIParser.cpp
@@ -66,7 +66,6 @@
 #include "llvm/Support/MemoryBuffer.h"
 #include "llvm/Support/SMLoc.h"
 #include "llvm/Support/SourceMgr.h"
-#include "llvm/Target/TargetIntrinsicInfo.h"
 #include "llvm/Target/TargetMachine.h"
 #include <cassert>
 #include <cctype>
@@ -485,7 +484,7 @@ class MIParser {
   bool parseDILocation(MDNode *&Expr);
   bool parseMetadataOperand(MachineOperand &Dest);
   bool parseCFIOffset(int &Offset);
-  bool parseCFIRegister(Register &Reg);
+  bool parseCFIRegister(unsigned &Reg);
   bool parseCFIAddressSpace(unsigned &AddressSpace);
   bool parseCFIEscapeValues(std::string& Values);
   bool parseCFIOperand(MachineOperand &Dest);
@@ -2446,7 +2445,7 @@ bool MIParser::parseCFIOffset(int &Offset) {
   return false;
 }
 
-bool MIParser::parseCFIRegister(Register &Reg) {
+bool MIParser::parseCFIRegister(unsigned &Reg) {
   if (Token.isNot(MIToken::NamedRegister))
     return error("expected a cfi register");
   Register LLVMReg;
@@ -2491,7 +2490,7 @@ bool MIParser::parseCFIOperand(MachineOperand &Dest) {
   auto Kind = Token.kind();
   lex();
   int Offset;
-  Register Reg;
+  unsigned Reg;
   unsigned AddressSpace;
   unsigned CFIIndex;
   switch (Kind) {
@@ -2564,7 +2563,7 @@ bool MIParser::parseCFIOperand(MachineOperand &Dest) {
     CFIIndex = MF.addFrameInst(MCCFIInstruction::createUndefined(nullptr, Reg));
     break;
   case MIToken::kw_cfi_register: {
-    Register Reg2;
+    unsigned Reg2;
     if (parseCFIRegister(Reg) || expectAndConsume(MIToken::comma) ||
         parseCFIRegister(Reg2))
       return true;
@@ -2669,13 +2668,8 @@ bool MIParser::parseIntrinsicOperand(MachineOperand &Dest) {
   if (expectAndConsume(MIToken::rparen))
     return error("expected ')' to terminate intrinsic name");
 
-  // Find out what intrinsic we're dealing with, first try the global namespace
-  // and then the target's private intrinsics if that fails.
-  const TargetIntrinsicInfo *TII = MF.getTarget().getIntrinsicInfo();
+  // Find out what intrinsic we're dealing with.
   Intrinsic::ID ID = Intrinsic::lookupIntrinsicID(Name);
-  if (ID == Intrinsic::not_intrinsic && TII)
-    ID = static_cast<Intrinsic::ID>(TII->lookupName(Name));
-
   if (ID == Intrinsic::not_intrinsic)
     return error("unknown intrinsic name");
   Dest = MachineOperand::CreateIntrinsicID(ID);
diff --git a/llvm/lib/CodeGen/MIRPrinter.cpp b/llvm/lib/CodeGen/MIRPrinter.cpp
index 0b41c90442a5d..789c628a64a23 100644
--- a/llvm/lib/CodeGen/MIRPrinter.cpp
+++ b/llvm/lib/CodeGen/MIRPrinter.cpp
@@ -452,7 +452,7 @@ void MIRPrinter::convertStackObjects(yaml::MachineFunction &YMF,
     YamlObject.IsAliased = MFI.isAliasedObjectIndex(I);
     // Save the ID' position in FixedStackObjects storage vector.
     FixedStackObjectsIdx[ID] = YMF.FixedStackObjects.size();
-    YMF.FixedStackObjects.push_back(YamlObject);
+    YMF.FixedStackObjects.push_back(std::move(YamlObject));
     StackObjectOperandMapping.insert(
         std::make_pair(I, FrameIndexOperand::createFixed(ID)));
   }
@@ -506,11 +506,11 @@ void MIRPrinter::convertStackObjects(yaml::MachineFunction &YMF,
         auto &Object =
             YMF.FixedStackObjects
                 [FixedStackObjectsIdx[FrameIdx + MFI.getNumFixedObjects()]];
-        Object.CalleeSavedRegister = Reg;
+        Object.CalleeSavedRegister = std::move(Reg);
         Object.CalleeSavedRestored = CSInfo.isRestored();
       } else {
         auto &Object = YMF.StackObjects[StackObjectsIdx[FrameIdx]];
-        Object.CalleeSavedRegister = Reg;
+        Object.CalleeSavedRegister = std::move(Reg);
         Object.CalleeSavedRestored = CSInfo.isRestored();
       }
     }
@@ -576,7 +576,7 @@ void MIRPrinter::convertCallSiteObjects(yaml::MachineFunction &YMF,
       printRegMIR(ArgReg.Reg, YmlArgReg.Reg, TRI);
       YmlCS.ArgForwardingRegs.emplace_back(YmlArgReg);
     }
-    YMF.CallSitesInfo.push_back(YmlCS);
+    YMF.CallSitesInfo.push_back(std::move(YmlCS));
   }
 
   // Sort call info by position of call instructions.
@@ -597,7 +597,7 @@ void MIRPrinter::convertMachineMetadataNodes(yaml::MachineFunction &YMF,
     std::string NS;
     raw_string_ostream StrOS(NS);
     MD.second->print(StrOS, MST, MF.getFunction().getParent());
-    YMF.MachineMetadataNodes.push_back(NS);
+    YMF.MachineMetadataNodes.push_back(std::move(NS));
   }
 }
 
@@ -612,7 +612,7 @@ void MIRPrinter::convertCalledGlobals(yaml::MachineFunction &YMF,
 
     yaml::CalledGlobal YamlCG{CallSite, CG.Callee->getName().str(),
                               CG.TargetFlags};
-    YMF.CalledGlobals.push_back(YamlCG);
+    YMF.CalledGlobals.push_back(std::move(YamlCG));
   }
 
   // Sort by position of call instructions.
@@ -638,11 +638,11 @@ void MIRPrinter::convert(yaml::MachineFunction &MF,
 
     yaml::MachineConstantPoolValue YamlConstant;
     YamlConstant.ID = ID++;
-    YamlConstant.Value = Str;
+    YamlConstant.Value = std::move(Str);
     YamlConstant.Alignment = Constant.getAlign();
     YamlConstant.IsTargetSpecific = Constant.isMachineConstantPoolEntry();
 
-    MF.Constants.push_back(YamlConstant);
+    MF.Constants.push_back(std::move(YamlConstant));
   }
 }
 
@@ -661,7 +661,7 @@ void MIRPrinter::convert(ModuleSlotTracker &MST,
       Entry.Blocks.push_back(Str);
       Str.clear();
     }
-    YamlJTI.Entries.push_back(Entry);
+    YamlJTI.Entries.push_back(std::move(Entry));
   }
 }
 
@@ -1007,10 +1007,9 @@ void MIPrinter::print(const MachineInstr &MI, unsigned OpIdx,
     unsigned TiedOperandIdx = 0;
     if (ShouldPrintRegisterTies && Op.isReg() && Op.isTied() && !Op.isDef())
       TiedOperandIdx = Op.getParent()->findTiedOperandIdx(OpIdx);
-    const TargetIntrinsicInfo *TII = MI.getMF()->getTarget().getIntrinsicInfo();
     Op.print(OS, MST, TypeToPrint, OpIdx, PrintDef, /*IsStandalone=*/false,
-             ShouldPrintRegisterTies, TiedOperandIdx, TRI, TII);
-      OS << formatOperandComment(MOComment);
+             ShouldPrintRegisterTies, TiedOperandIdx, TRI);
+    OS << formatOperandComment(MOComment);
     break;
   }
   case MachineOperand::MO_FrameIndex:
diff --git a/llvm/lib/CodeGen/MLRegAllocEvictAdvisor.cpp b/llvm/lib/CodeGen/MLRegAllocEvictAdvisor.cpp
index 9c6487b40d606..9656774c6eaae 100644
--- a/llvm/lib/CodeGen/MLRegAllocEvictAdvisor.cpp
+++ b/llvm/lib/CodeGen/MLRegAllocEvictAdvisor.cpp
@@ -42,6 +42,7 @@
 #include <array>
 #include <bitset>
 #include <memory>
+#include <unordered_map>
 
 using namespace llvm;
 
@@ -63,11 +64,11 @@ static cl::opt<std::string> InteractiveChannelBaseName(
         "outgoing name should be "
         "<regalloc-evict-interactive-channel-base>.out"));
 
-static cl::opt<unsigned>
-    MaxCascade("mlregalloc-max-cascade", cl::Hidden,
-               cl::desc("The maximum number of times a live range can be "
-                        "evicted before preventing it from being evicted"),
-               cl::init(20));
+static cl::opt<unsigned> MaxEvictionCount(
+    "mlregalloc-max-eviction-count", cl::Hidden,
+    cl::desc("The maximum number of times a live range can be "
+             "evicted before preventing it from being evicted"),
+    cl::init(100));
 
 // Options that only make sense in development mode
 #ifdef LLVM_HAVE_TFLITE
@@ -364,6 +365,22 @@ class MLEvictAdvisor : public RegAllocEvictionAdvisor {
 
   using RegID = unsigned;
   mutable DenseMap<RegID, LIFeatureComponents> CachedFeatures;
+
+  mutable std::unordered_map<unsigned, unsigned> VirtRegEvictionCounts;
+
+  void onEviction(Register RegBeingEvicted) const {
+    // If we cannot find the virtual register in the map, we just assume it has
+    // not been evicted before and thus has a value of zero (which is what the
+    // subscript operator returns by default).
+    ++VirtRegEvictionCounts[RegBeingEvicted.id()];
+  }
+
+  unsigned getEvictionCount(Register Reg) const {
+    auto EvictionCountIt = VirtRegEvictionCounts.find(Reg.id());
+    if (EvictionCountIt != VirtRegEvictionCounts.end())
+      return EvictionCountIt->second;
+    return 0;
+  }
 };
 
 #define _DECL_FEATURES(type, name, shape, _)                                   \
@@ -657,7 +674,7 @@ bool MLEvictAdvisor::loadInterferenceFeatures(
       // threshold, prevent the range from being evicted. We still let the
       // range through if it is urgent as we are required to produce an
       // eviction if the candidate is not spillable.
-      if (IntfCascade >= MaxCascade && !Urgent)
+      if (getEvictionCount(Intf->reg()) > MaxEvictionCount && !Urgent)
         return false;
 
       // Only evict older cascades or live ranges without a cascade.
@@ -803,6 +820,22 @@ MCRegister MLEvictAdvisor::tryFindEvictionCandidate(
   }
   assert(CandidatePos < ValidPosLimit);
   (void)ValidPosLimit;
+
+  // Update information about how many times the virtual registers being
+  // evicted have been evicted so that we can prevent the model from evicting
+  // the same ranges continually and eating compile time.
+  if (CandidatePos == CandidateVirtRegPos) {
+    onEviction(VirtReg.reg());
+  } else {
+    for (MCRegUnit Unit : TRI->regunits(Regs[CandidatePos].first)) {
+      LiveIntervalUnion::Query &Q = Matrix->query(VirtReg, Unit);
+      const auto &IFIntervals = Q.interferingVRegs(EvictInterferenceCutoff);
+      for (const LiveInterval *Intf : reverse(IFIntervals)) {
+        onEviction(Intf->reg());
+      }
+    }
+  }
+
   return Regs[CandidatePos].first;
 }
 
diff --git a/llvm/lib/CodeGen/MachineCopyPropagation.cpp b/llvm/lib/CodeGen/MachineCopyPropagation.cpp
index d44b064dcb4b2..1105b8c15515f 100644
--- a/llvm/lib/CodeGen/MachineCopyPropagation.cpp
+++ b/llvm/lib/CodeGen/MachineCopyPropagation.cpp
@@ -48,6 +48,7 @@
 //
 //===----------------------------------------------------------------------===//
 
+#include "llvm/CodeGen/MachineCopyPropagation.h"
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/SetVector.h"
@@ -449,7 +450,7 @@ class CopyTracker {
   }
 };
 
-class MachineCopyPropagation : public MachineFunctionPass {
+class MachineCopyPropagation {
   const TargetRegisterInfo *TRI = nullptr;
   const TargetInstrInfo *TII = nullptr;
   const MachineRegisterInfo *MRI = nullptr;
@@ -458,24 +459,10 @@ class MachineCopyPropagation : public MachineFunctionPass {
   bool UseCopyInstr;
 
 public:
-  static char ID; // Pass identification, replacement for typeid
-
   MachineCopyPropagation(bool CopyInstr = false)
-      : MachineFunctionPass(ID), UseCopyInstr(CopyInstr || MCPUseCopyInstr) {
-    initializeMachineCopyPropagationPass(*PassRegistry::getPassRegistry());
-  }
+      : UseCopyInstr(CopyInstr || MCPUseCopyInstr) {}
 
-  void getAnalysisUsage(AnalysisUsage &AU) const override {
-    AU.setPreservesCFG();
-    MachineFunctionPass::getAnalysisUsage(AU);
-  }
-
-  bool runOnMachineFunction(MachineFunction &MF) override;
-
-  MachineFunctionProperties getRequiredProperties() const override {
-    return MachineFunctionProperties().set(
-        MachineFunctionProperties::Property::NoVRegs);
-  }
+  bool run(MachineFunction &MF);
 
 private:
   typedef enum { DebugUse = false, RegularUse = true } DebugType;
@@ -510,13 +497,35 @@ class MachineCopyPropagation : public MachineFunctionPass {
   bool Changed = false;
 };
 
+class MachineCopyPropagationLegacy : public MachineFunctionPass {
+  bool UseCopyInstr;
+
+public:
+  static char ID; // pass identification
+
+  MachineCopyPropagationLegacy(bool UseCopyInstr = false)
+      : MachineFunctionPass(ID), UseCopyInstr(UseCopyInstr) {}
+
+  void getAnalysisUsage(AnalysisUsage &AU) const override {
+    AU.setPreservesCFG();
+    MachineFunctionPass::getAnalysisUsage(AU);
+  }
+
+  bool runOnMachineFunction(MachineFunction &MF) override;
+
+  MachineFunctionProperties getRequiredProperties() const override {
+    return MachineFunctionProperties().set(
+        MachineFunctionProperties::Property::NoVRegs);
+  }
+};
+
 } // end anonymous namespace
 
-char MachineCopyPropagation::ID = 0;
+char MachineCopyPropagationLegacy::ID = 0;
 
-char &llvm::MachineCopyPropagationID = MachineCopyPropagation::ID;
+char &llvm::MachineCopyPropagationID = MachineCopyPropagationLegacy::ID;
 
-INITIALIZE_PASS(MachineCopyPropagation, DEBUG_TYPE,
+INITIALIZE_PASS(MachineCopyPropagationLegacy, DEBUG_TYPE,
                 "Machine Copy Propagation Pass", false, false)
 
 void MachineCopyPropagation::ReadRegister(MCRegister Reg, MachineInstr &Reader,
@@ -1009,18 +1018,30 @@ void MachineCopyPropagation::ForwardCopyPropagateBlock(MachineBasicBlock &MBB) {
           continue;
         }
 
-        LLVM_DEBUG(dbgs() << "MCP: Removing copy due to regmask clobbering: ";
-                   MaybeDead->dump());
-
         // Invalidate all entries in the copy map which are not preserved by
         // this register mask.
-        for (unsigned RegUnit : TRI->regunits(Reg))
+        bool MIRefedinCopyInfo = false;
+        for (unsigned RegUnit : TRI->regunits(Reg)) {
           if (!PreservedRegUnits.test(RegUnit))
             Tracker.clobberRegUnit(RegUnit, *TRI, *TII, UseCopyInstr);
+          else {
+            if (MaybeDead == Tracker.findCopyForUnit(RegUnit, *TRI)) {
+              MIRefedinCopyInfo = true;
+            }
+          }
+        }
 
         // erase() will return the next valid iterator pointing to the next
         // element after the erased one.
         DI = MaybeDeadCopies.erase(DI);
+
+        // Preserved by RegMask, DO NOT remove copy
+        if (MIRefedinCopyInfo)
+          continue;
+
+        LLVM_DEBUG(dbgs() << "MCP: Removing copy due to regmask clobbering: "
+                          << *MaybeDead);
+
         MaybeDead->eraseFromParent();
         Changed = true;
         ++NumDeletes;
@@ -1056,8 +1077,9 @@ void MachineCopyPropagation::ForwardCopyPropagateBlock(MachineBasicBlock &MBB) {
       assert(!MRI->isReserved(DestReg));
 
       // Update matching debug values, if any.
-      SmallVector<MachineInstr *> MaybeDeadDbgUsers(
-          CopyDbgUsers[MaybeDead].begin(), CopyDbgUsers[MaybeDead].end());
+      const auto &DbgUsers = CopyDbgUsers[MaybeDead];
+      SmallVector<MachineInstr *> MaybeDeadDbgUsers(DbgUsers.begin(),
+                                                    DbgUsers.end());
       MRI->updateDbgUsersToReg(DestReg.asMCReg(), SrcReg.asMCReg(),
                                MaybeDeadDbgUsers);
 
@@ -1229,8 +1251,9 @@ void MachineCopyPropagation::BackwardCopyPropagateBlock(
         isCopyInstr(*Copy, *TII, UseCopyInstr);
     Register Src = CopyOperands->Source->getReg();
     Register Def = CopyOperands->Destination->getReg();
-    SmallVector<MachineInstr *> MaybeDeadDbgUsers(CopyDbgUsers[Copy].begin(),
-                                                  CopyDbgUsers[Copy].end());
+    const auto &DbgUsers = CopyDbgUsers[Copy];
+    SmallVector<MachineInstr *> MaybeDeadDbgUsers(DbgUsers.begin(),
+                                                  DbgUsers.end());
 
     MRI->updateDbgUsersToReg(Src.asMCReg(), Def.asMCReg(), MaybeDeadDbgUsers);
     Copy->eraseFromParent();
@@ -1563,10 +1586,25 @@ void MachineCopyPropagation::EliminateSpillageCopies(MachineBasicBlock &MBB) {
   Tracker.clear();
 }
 
-bool MachineCopyPropagation::runOnMachineFunction(MachineFunction &MF) {
+bool MachineCopyPropagationLegacy::runOnMachineFunction(MachineFunction &MF) {
   if (skipFunction(MF.getFunction()))
     return false;
 
+  return MachineCopyPropagation(UseCopyInstr).run(MF);
+}
+
+PreservedAnalyses
+MachineCopyPropagationPass::run(MachineFunction &MF,
+                                MachineFunctionAnalysisManager &) {
+  MFPropsModifier _(*this, MF);
+  if (!MachineCopyPropagation(UseCopyInstr).run(MF))
+    return PreservedAnalyses::all();
+  auto PA = getMachineFunctionPassPreservedAnalyses();
+  PA.preserveSet<CFGAnalyses>();
+  return PA;
+}
+
+bool MachineCopyPropagation::run(MachineFunction &MF) {
   bool isSpillageCopyElimEnabled = false;
   switch (EnableSpillageCopyElimination) {
   case cl::BOU_UNSET:
@@ -1599,5 +1637,5 @@ bool MachineCopyPropagation::runOnMachineFunction(MachineFunction &MF) {
 
 MachineFunctionPass *
 llvm::createMachineCopyPropagationPass(bool UseCopyInstr = false) {
-  return new MachineCopyPropagation(UseCopyInstr);
+  return new MachineCopyPropagationLegacy(UseCopyInstr);
 }
diff --git a/llvm/lib/CodeGen/MachineFunction.cpp b/llvm/lib/CodeGen/MachineFunction.cpp
index ab3609b6141b8..6e0342a763d15 100644
--- a/llvm/lib/CodeGen/MachineFunction.cpp
+++ b/llvm/lib/CodeGen/MachineFunction.cpp
@@ -967,13 +967,13 @@ void MachineFunction::copyAdditionalCallInfo(const MachineInstr *Old,
   CallSiteInfoMap::iterator CSIt = getCallSiteInfo(OldCallMI);
   if (CSIt != CallSitesInfo.end()) {
     CallSiteInfo CSInfo = CSIt->second;
-    CallSitesInfo[New] = CSInfo;
+    CallSitesInfo[New] = std::move(CSInfo);
   }
 
   CalledGlobalsMap::iterator CGIt = CalledGlobalsInfo.find(OldCallMI);
   if (CGIt != CalledGlobalsInfo.end()) {
     CalledGlobalInfo CGInfo = CGIt->second;
-    CalledGlobalsInfo[New] = CGInfo;
+    CalledGlobalsInfo[New] = std::move(CGInfo);
   }
 }
 
@@ -991,14 +991,14 @@ void MachineFunction::moveAdditionalCallInfo(const MachineInstr *Old,
   if (CSIt != CallSitesInfo.end()) {
     CallSiteInfo CSInfo = std::move(CSIt->second);
     CallSitesInfo.erase(CSIt);
-    CallSitesInfo[New] = CSInfo;
+    CallSitesInfo[New] = std::move(CSInfo);
   }
 
   CalledGlobalsMap::iterator CGIt = CalledGlobalsInfo.find(OldCallMI);
   if (CGIt != CalledGlobalsInfo.end()) {
     CalledGlobalInfo CGInfo = std::move(CGIt->second);
     CalledGlobalsInfo.erase(CGIt);
-    CalledGlobalsInfo[New] = CGInfo;
+    CalledGlobalsInfo[New] = std::move(CGInfo);
   }
 }
 
@@ -1054,7 +1054,7 @@ auto MachineFunction::salvageCopySSA(
   // Check whether this copy-like instruction has already been salvaged into
   // an operand pair.
   Register Dest;
-  if (auto CopyDstSrc = TII.isCopyInstr(MI)) {
+  if (auto CopyDstSrc = TII.isCopyLikeInstr(MI)) {
     Dest = CopyDstSrc->Destination->getReg();
   } else {
     assert(MI.isSubregToReg());
@@ -1138,7 +1138,7 @@ auto MachineFunction::salvageCopySSAImpl(MachineInstr &MI)
     CurInst = Inst.getIterator();
 
     // Any non-copy instruction is the defining instruction we're seeking.
-    if (!Inst.isCopyLike() && !TII.isCopyInstr(Inst))
+    if (!Inst.isCopyLike() && !TII.isCopyLikeInstr(Inst))
       break;
     State = GetRegAndSubreg(Inst);
   };
diff --git a/llvm/lib/CodeGen/MachineInstr.cpp b/llvm/lib/CodeGen/MachineInstr.cpp
index a9f756b684360..52c977a3651a3 100644
--- a/llvm/lib/CodeGen/MachineInstr.cpp
+++ b/llvm/lib/CodeGen/MachineInstr.cpp
@@ -70,18 +70,15 @@ static const MachineFunction *getMFIfAvailable(const MachineInstr &MI) {
   return nullptr;
 }
 
-// Try to crawl up to the machine function and get TRI and IntrinsicInfo from
-// it.
+// Try to crawl up to the machine function and get TRI/MRI/TII from it.
 static void tryToGetTargetInfo(const MachineInstr &MI,
                                const TargetRegisterInfo *&TRI,
                                const MachineRegisterInfo *&MRI,
-                               const TargetIntrinsicInfo *&IntrinsicInfo,
                                const TargetInstrInfo *&TII) {
 
   if (const MachineFunction *MF = getMFIfAvailable(MI)) {
     TRI = MF->getSubtarget().getRegisterInfo();
     MRI = &MF->getRegInfo();
-    IntrinsicInfo = MF->getTarget().getIntrinsicInfo();
     TII = MF->getSubtarget().getInstrInfo();
   }
 }
@@ -1753,8 +1750,7 @@ void MachineInstr::print(raw_ostream &OS, ModuleSlotTracker &MST,
   // We can be a bit tidier if we know the MachineFunction.
   const TargetRegisterInfo *TRI = nullptr;
   const MachineRegisterInfo *MRI = nullptr;
-  const TargetIntrinsicInfo *IntrinsicInfo = nullptr;
-  tryToGetTargetInfo(*this, TRI, MRI, IntrinsicInfo, TII);
+  tryToGetTargetInfo(*this, TRI, MRI, TII);
 
   if (isCFIInstruction())
     assert(getNumOperands() == 1 && "Expected 1 operand in CFI instruction");
@@ -1784,7 +1780,7 @@ void MachineInstr::print(raw_ostream &OS, ModuleSlotTracker &MST,
     LLT TypeToPrint = MRI ? getTypeToPrint(StartOp, PrintedTypes, *MRI) : LLT{};
     unsigned TiedOperandIdx = getTiedOperandIdx(StartOp);
     MO.print(OS, MST, TypeToPrint, StartOp, /*PrintDef=*/false, IsStandalone,
-             ShouldPrintRegisterTies, TiedOperandIdx, TRI, IntrinsicInfo);
+             ShouldPrintRegisterTies, TiedOperandIdx, TRI);
     ++StartOp;
   }
 
@@ -1846,9 +1842,9 @@ void MachineInstr::print(raw_ostream &OS, ModuleSlotTracker &MST,
     const unsigned OpIdx = InlineAsm::MIOp_AsmString;
     LLT TypeToPrint = MRI ? getTypeToPrint(OpIdx, PrintedTypes, *MRI) : LLT{};
     unsigned TiedOperandIdx = getTiedOperandIdx(OpIdx);
-    getOperand(OpIdx).print(OS, MST, TypeToPrint, OpIdx, /*PrintDef=*/true, IsStandalone,
-                            ShouldPrintRegisterTies, TiedOperandIdx, TRI,
-                            IntrinsicInfo);
+    getOperand(OpIdx).print(OS, MST, TypeToPrint, OpIdx, /*PrintDef=*/true,
+                            IsStandalone, ShouldPrintRegisterTies,
+                            TiedOperandIdx, TRI);
 
     // Print HasSideEffects, MayLoad, MayStore, IsAlignStack
     unsigned ExtraInfo = getOperand(InlineAsm::MIOp_ExtraInfo).getImm();
@@ -1886,7 +1882,7 @@ void MachineInstr::print(raw_ostream &OS, ModuleSlotTracker &MST,
         LLT TypeToPrint = MRI ? getTypeToPrint(i, PrintedTypes, *MRI) : LLT{};
         unsigned TiedOperandIdx = getTiedOperandIdx(i);
         MO.print(OS, MST, TypeToPrint, i, /*PrintDef=*/true, IsStandalone,
-                 ShouldPrintRegisterTies, TiedOperandIdx, TRI, IntrinsicInfo);
+                 ShouldPrintRegisterTies, TiedOperandIdx, TRI);
       }
     } else if (isDebugLabel() && MO.isMetadata()) {
       // Pretty print DBG_LABEL instructions.
@@ -1897,7 +1893,7 @@ void MachineInstr::print(raw_ostream &OS, ModuleSlotTracker &MST,
         LLT TypeToPrint = MRI ? getTypeToPrint(i, PrintedTypes, *MRI) : LLT{};
         unsigned TiedOperandIdx = getTiedOperandIdx(i);
         MO.print(OS, MST, TypeToPrint, i, /*PrintDef=*/true, IsStandalone,
-                 ShouldPrintRegisterTies, TiedOperandIdx, TRI, IntrinsicInfo);
+                 ShouldPrintRegisterTies, TiedOperandIdx, TRI);
       }
     } else if (i == AsmDescOp && MO.isImm()) {
       // Pretty print the inline asm operand descriptor.
@@ -1941,7 +1937,7 @@ void MachineInstr::print(raw_ostream &OS, ModuleSlotTracker &MST,
         MachineOperand::printSubRegIdx(OS, MO.getImm(), TRI);
       else
         MO.print(OS, MST, TypeToPrint, i, /*PrintDef=*/true, IsStandalone,
-                 ShouldPrintRegisterTies, TiedOperandIdx, TRI, IntrinsicInfo);
+                 ShouldPrintRegisterTies, TiedOperandIdx, TRI);
     }
   }
 
diff --git a/llvm/lib/CodeGen/MachineOperand.cpp b/llvm/lib/CodeGen/MachineOperand.cpp
index d11ac614ace35..231d66607b700 100644
--- a/llvm/lib/CodeGen/MachineOperand.cpp
+++ b/llvm/lib/CodeGen/MachineOperand.cpp
@@ -27,7 +27,6 @@
 #include "llvm/IR/Instructions.h"
 #include "llvm/IR/ModuleSlotTracker.h"
 #include "llvm/MC/MCDwarf.h"
-#include "llvm/Target/TargetIntrinsicInfo.h"
 #include "llvm/Target/TargetMachine.h"
 #include <optional>
 
@@ -71,13 +70,13 @@ void MachineOperand::setReg(Register Reg) {
   if (MachineFunction *MF = getMFIfAvailable(*this)) {
     MachineRegisterInfo &MRI = MF->getRegInfo();
     MRI.removeRegOperandFromUseList(this);
-    SmallContents.RegNo = Reg;
+    SmallContents.RegNo = Reg.id();
     MRI.addRegOperandToUseList(this);
     return;
   }
 
   // Otherwise, just change the register, no problem.  :)
-  SmallContents.RegNo = Reg;
+  SmallContents.RegNo = Reg.id();
 }
 
 void MachineOperand::substVirtReg(Register Reg, unsigned SubIdx,
@@ -291,7 +290,7 @@ void MachineOperand::ChangeToRegister(Register Reg, bool isDef, bool isImp,
   assert(!(isDead && !isDef) && "Dead flag on non-def");
   assert(!(isKill && isDef) && "Kill flag on def");
   OpKind = MO_Register;
-  SmallContents.RegNo = Reg;
+  SmallContents.RegNo = Reg.id();
   SubReg_TargetFlags = 0;
   IsDef = isDef;
   IsImp = isImp;
@@ -390,7 +389,8 @@ hash_code llvm::hash_value(const MachineOperand &MO) {
   switch (MO.getType()) {
   case MachineOperand::MO_Register:
     // Register operands don't have target flags.
-    return hash_combine(MO.getType(), (unsigned)MO.getReg(), MO.getSubReg(), MO.isDef());
+    return hash_combine(MO.getType(), MO.getReg().id(), MO.getSubReg(),
+                        MO.isDef());
   case MachineOperand::MO_Immediate:
     return hash_combine(MO.getType(), MO.getTargetFlags(), MO.getImm());
   case MachineOperand::MO_CImmediate:
@@ -449,14 +449,11 @@ hash_code llvm::hash_value(const MachineOperand &MO) {
   llvm_unreachable("Invalid machine operand type");
 }
 
-// Try to crawl up to the machine function and get TRI and IntrinsicInfo from
-// it.
+// Try to crawl up to the machine function and get TRI from it.
 static void tryToGetTargetInfo(const MachineOperand &MO,
-                               const TargetRegisterInfo *&TRI,
-                               const TargetIntrinsicInfo *&IntrinsicInfo) {
+                               const TargetRegisterInfo *&TRI) {
   if (const MachineFunction *MF = getMFIfAvailable(MO)) {
     TRI = MF->getSubtarget().getRegisterInfo();
-    IntrinsicInfo = MF->getTarget().getIntrinsicInfo();
   }
 }
 
@@ -780,20 +777,19 @@ static void printCFI(raw_ostream &OS, const MCCFIInstruction &CFI,
   }
 }
 
-void MachineOperand::print(raw_ostream &OS, const TargetRegisterInfo *TRI,
-                           const TargetIntrinsicInfo *IntrinsicInfo) const {
-  print(OS, LLT{}, TRI, IntrinsicInfo);
+void MachineOperand::print(raw_ostream &OS,
+                           const TargetRegisterInfo *TRI) const {
+  print(OS, LLT{}, TRI);
 }
 
 void MachineOperand::print(raw_ostream &OS, LLT TypeToPrint,
-                           const TargetRegisterInfo *TRI,
-                           const TargetIntrinsicInfo *IntrinsicInfo) const {
-  tryToGetTargetInfo(*this, TRI, IntrinsicInfo);
+                           const TargetRegisterInfo *TRI) const {
+  tryToGetTargetInfo(*this, TRI);
   ModuleSlotTracker DummyMST(nullptr);
   print(OS, DummyMST, TypeToPrint, std::nullopt, /*PrintDef=*/false,
         /*IsStandalone=*/true,
         /*ShouldPrintRegisterTies=*/true,
-        /*TiedOperandIdx=*/0, TRI, IntrinsicInfo);
+        /*TiedOperandIdx=*/0, TRI);
 }
 
 void MachineOperand::print(raw_ostream &OS, ModuleSlotTracker &MST,
@@ -801,8 +797,7 @@ void MachineOperand::print(raw_ostream &OS, ModuleSlotTracker &MST,
                            bool PrintDef, bool IsStandalone,
                            bool ShouldPrintRegisterTies,
                            unsigned TiedOperandIdx,
-                           const TargetRegisterInfo *TRI,
-                           const TargetIntrinsicInfo *IntrinsicInfo) const {
+                           const TargetRegisterInfo *TRI) const {
   printTargetFlags(OS, *this);
   switch (getType()) {
   case MachineOperand::MO_Register: {
@@ -1003,8 +998,6 @@ void MachineOperand::print(raw_ostream &OS, ModuleSlotTracker &MST,
     Intrinsic::ID ID = getIntrinsicID();
     if (ID < Intrinsic::num_intrinsics)
       OS << "intrinsic(@" << Intrinsic::getBaseName(ID) << ')';
-    else if (IntrinsicInfo)
-      OS << "intrinsic(@" << IntrinsicInfo->getName(ID) << ')';
     else
       OS << "intrinsic(" << ID << ')';
     break;
diff --git a/llvm/lib/CodeGen/MachinePassManager.cpp b/llvm/lib/CodeGen/MachinePassManager.cpp
index 6d540808d4cc9..3eed874903f7c 100644
--- a/llvm/lib/CodeGen/MachinePassManager.cpp
+++ b/llvm/lib/CodeGen/MachinePassManager.cpp
@@ -152,6 +152,7 @@ PassManager<MachineFunction>::run(MachineFunction &MF,
     PI.runAfterPass(*Pass, MF, PassPA);
     PA.intersect(std::move(PassPA));
   }
+  PA.preserveSet<AllAnalysesOn<MachineFunction>>();
   return PA;
 }
 
diff --git a/llvm/lib/CodeGen/MachinePipeliner.cpp b/llvm/lib/CodeGen/MachinePipeliner.cpp
index 0d5dc96159003..064b0a0f11747 100644
--- a/llvm/lib/CodeGen/MachinePipeliner.cpp
+++ b/llvm/lib/CodeGen/MachinePipeliner.cpp
@@ -511,7 +511,7 @@ bool MachinePipeliner::runWindowScheduler(MachineLoop &L) {
   Context.MF = MF;
   Context.MLI = MLI;
   Context.MDT = MDT;
-  Context.PassConfig = &getAnalysis<TargetPassConfig>();
+  Context.TM = &getAnalysis<TargetPassConfig>().getTM<TargetMachine>();
   Context.AA = &getAnalysis<AAResultsWrapperPass>().getAAResults();
   Context.LIS = &getAnalysis<LiveIntervalsWrapperPass>().getLIS();
   Context.RegClassInfo->runOnMachineFunction(*MF);
@@ -2521,9 +2521,104 @@ bool SwingSchedulerDAG::schedulePipeline(SMSchedule &Schedule) {
   return scheduleFound && Schedule.getMaxStageCount() > 0;
 }
 
+static Register findUniqueOperandDefinedInLoop(const MachineInstr &MI) {
+  const MachineRegisterInfo &MRI = MI.getParent()->getParent()->getRegInfo();
+  Register Result;
+  for (const MachineOperand &Use : MI.all_uses()) {
+    Register Reg = Use.getReg();
+    if (!Reg.isVirtual())
+      return Register();
+    if (MRI.getVRegDef(Reg)->getParent() != MI.getParent())
+      continue;
+    if (Result)
+      return Register();
+    Result = Reg;
+  }
+  return Result;
+}
+
+/// When Op is a value that is incremented recursively in a loop and there is a
+/// unique instruction that increments it, returns true and sets Value.
+static bool findLoopIncrementValue(const MachineOperand &Op, int &Value) {
+  if (!Op.isReg() || !Op.getReg().isVirtual())
+    return false;
+
+  Register OrgReg = Op.getReg();
+  Register CurReg = OrgReg;
+  const MachineBasicBlock *LoopBB = Op.getParent()->getParent();
+  const MachineRegisterInfo &MRI = LoopBB->getParent()->getRegInfo();
+
+  const TargetInstrInfo *TII =
+      LoopBB->getParent()->getSubtarget().getInstrInfo();
+  const TargetRegisterInfo *TRI =
+      LoopBB->getParent()->getSubtarget().getRegisterInfo();
+
+  MachineInstr *Phi = nullptr;
+  MachineInstr *Increment = nullptr;
+
+  // Traverse definitions until it reaches Op or an instruction that does not
+  // satisfy the condition.
+  // Acceptable example:
+  //   bb.0:
+  //     %0 = PHI %3, %bb.0, ...
+  //     %2 = ADD %0, Value
+  //     ... = LOAD %2(Op)
+  //     %3 = COPY %2
+  while (true) {
+    if (!CurReg.isValid() || !CurReg.isVirtual())
+      return false;
+    MachineInstr *Def = MRI.getVRegDef(CurReg);
+    if (Def->getParent() != LoopBB)
+      return false;
+
+    if (Def->isCopy()) {
+      // Ignore copy instructions unless they contain subregisters
+      if (Def->getOperand(0).getSubReg() || Def->getOperand(1).getSubReg())
+        return false;
+      CurReg = Def->getOperand(1).getReg();
+    } else if (Def->isPHI()) {
+      // There must be just one Phi
+      if (Phi)
+        return false;
+      Phi = Def;
+      CurReg = getLoopPhiReg(*Def, LoopBB);
+    } else if (TII->getIncrementValue(*Def, Value)) {
+      // Potentially a unique increment
+      if (Increment)
+        // Multiple increments exist
+        return false;
+
+      const MachineOperand *BaseOp;
+      int64_t Offset;
+      bool OffsetIsScalable;
+      if (TII->getMemOperandWithOffset(*Def, BaseOp, Offset, OffsetIsScalable,
+                                       TRI)) {
+        // Pre/post increment instruction
+        CurReg = BaseOp->getReg();
+      } else {
+        // If only one of the operands is defined within the loop, it is assumed
+        // to be an incremented value.
+        CurReg = findUniqueOperandDefinedInLoop(*Def);
+        if (!CurReg.isValid())
+          return false;
+      }
+      Increment = Def;
+    } else {
+      return false;
+    }
+    if (CurReg == OrgReg)
+      break;
+  }
+
+  if (!Phi || !Increment)
+    return false;
+
+  return true;
+}
+
 /// Return true if we can compute the amount the instruction changes
 /// during each iteration. Set Delta to the amount of the change.
-bool SwingSchedulerDAG::computeDelta(MachineInstr &MI, unsigned &Delta) const {
+bool SwingSchedulerDAG::computeDelta(const MachineInstr &MI, int &Delta) const {
   const TargetRegisterInfo *TRI = MF.getSubtarget().getRegisterInfo();
   const MachineOperand *BaseOp;
   int64_t Offset;
@@ -2538,24 +2633,7 @@ bool SwingSchedulerDAG::computeDelta(MachineInstr &MI, unsigned &Delta) const {
   if (!BaseOp->isReg())
     return false;
 
-  Register BaseReg = BaseOp->getReg();
-
-  MachineRegisterInfo &MRI = MF.getRegInfo();
-  // Check if there is a Phi. If so, get the definition in the loop.
-  MachineInstr *BaseDef = MRI.getVRegDef(BaseReg);
-  if (BaseDef && BaseDef->isPHI()) {
-    BaseReg = getLoopPhiReg(*BaseDef, MI.getParent());
-    BaseDef = MRI.getVRegDef(BaseReg);
-  }
-  if (!BaseDef)
-    return false;
-
-  int D = 0;
-  if (!TII->getIncrementValue(*BaseDef, D) && D >= 0)
-    return false;
-
-  Delta = D;
-  return true;
+  return findLoopIncrementValue(*BaseOp, Delta);
 }
 
 /// Check if we can change the instruction to use an offset value from the
@@ -2673,6 +2751,100 @@ MachineInstr *SwingSchedulerDAG::findDefInLoop(Register Reg) {
   return Def;
 }
 
+/// Return false if there is no overlap between the region accessed by BaseMI in
+/// an iteration and the region accessed by OtherMI in subsequent iterations.
+bool SwingSchedulerDAG::mayOverlapInLaterIter(
+    const MachineInstr *BaseMI, const MachineInstr *OtherMI) const {
+  int DeltaB, DeltaO, Delta;
+  if (!computeDelta(*BaseMI, DeltaB) || !computeDelta(*OtherMI, DeltaO) ||
+      DeltaB != DeltaO)
+    return true;
+  Delta = DeltaB;
+
+  const MachineOperand *BaseOpB, *BaseOpO;
+  int64_t OffsetB, OffsetO;
+  bool OffsetBIsScalable, OffsetOIsScalable;
+  const TargetRegisterInfo *TRI = MF.getSubtarget().getRegisterInfo();
+  if (!TII->getMemOperandWithOffset(*BaseMI, BaseOpB, OffsetB,
+                                    OffsetBIsScalable, TRI) ||
+      !TII->getMemOperandWithOffset(*OtherMI, BaseOpO, OffsetO,
+                                    OffsetOIsScalable, TRI))
+    return true;
+
+  if (OffsetBIsScalable || OffsetOIsScalable)
+    return true;
+
+  if (!BaseOpB->isIdenticalTo(*BaseOpO)) {
+    // Pass cases with different base operands but same initial values.
+    // Typically for when pre/post increment is used.
+
+    if (!BaseOpB->isReg() || !BaseOpO->isReg())
+      return true;
+    Register RegB = BaseOpB->getReg(), RegO = BaseOpO->getReg();
+    if (!RegB.isVirtual() || !RegO.isVirtual())
+      return true;
+
+    MachineInstr *DefB = MRI.getVRegDef(BaseOpB->getReg());
+    MachineInstr *DefO = MRI.getVRegDef(BaseOpO->getReg());
+    if (!DefB || !DefO || !DefB->isPHI() || !DefO->isPHI())
+      return true;
+
+    unsigned InitValB = 0;
+    unsigned LoopValB = 0;
+    unsigned InitValO = 0;
+    unsigned LoopValO = 0;
+    getPhiRegs(*DefB, BB, InitValB, LoopValB);
+    getPhiRegs(*DefO, BB, InitValO, LoopValO);
+    MachineInstr *InitDefB = MRI.getVRegDef(InitValB);
+    MachineInstr *InitDefO = MRI.getVRegDef(InitValO);
+
+    if (!InitDefB->isIdenticalTo(*InitDefO))
+      return true;
+  }
+
+  LocationSize AccessSizeB = (*BaseMI->memoperands_begin())->getSize();
+  LocationSize AccessSizeO = (*OtherMI->memoperands_begin())->getSize();
+
+  // This is the main test, which checks the offset values and the loop
+  // increment value to determine if the accesses may be loop carried.
+  if (!AccessSizeB.hasValue() || !AccessSizeO.hasValue())
+    return true;
+
+  LLVM_DEBUG({
+    dbgs() << "Overlap check:\n";
+    dbgs() << "  BaseMI: ";
+    BaseMI->dump();
+    dbgs() << "    Base + " << OffsetB << " + I * " << Delta
+           << ", Len: " << AccessSizeB.getValue() << "\n";
+    dbgs() << "  OtherMI: ";
+    OtherMI->dump();
+    dbgs() << "    Base + " << OffsetO << " + I * " << Delta
+           << ", Len: " << AccessSizeO.getValue() << "\n";
+  });
+
+  // Excessive overlap may be detected in strided patterns.
+  // For example, the memory addresses of the store and the load in
+  //   for (i=0; i<n; i+=2) a[i+1] = a[i];
+  // are assumed to overlap.
+  if (Delta < 0) {
+    int64_t BaseMinAddr = OffsetB;
+    int64_t OhterNextIterMaxAddr = OffsetO + Delta + AccessSizeO.getValue() - 1;
+    if (BaseMinAddr > OhterNextIterMaxAddr) {
+      LLVM_DEBUG(dbgs() << "  Result: No overlap\n");
+      return false;
+    }
+  } else {
+    int64_t BaseMaxAddr = OffsetB + AccessSizeB.getValue() - 1;
+    int64_t OtherNextIterMinAddr = OffsetO + Delta;
+    if (BaseMaxAddr < OtherNextIterMinAddr) {
+      LLVM_DEBUG(dbgs() << "  Result: No overlap\n");
+      return false;
+    }
+  }
+  LLVM_DEBUG(dbgs() << "  Result: Overlap\n");
+  return true;
+}
+
 /// Return true for an order or output dependence that is loop carried
 /// potentially. A dependence is loop carried if the destination defines a value
 /// that may be used or defined by the source in a subsequent iteration.
@@ -2704,61 +2876,7 @@ bool SwingSchedulerDAG::isLoopCarriedDep(
   // The conservative assumption is that a dependence between memory operations
   // may be loop carried. The following code checks when it can be proved that
   // there is no loop carried dependence.
-  unsigned DeltaS, DeltaD;
-  if (!computeDelta(*SI, DeltaS) || !computeDelta(*DI, DeltaD))
-    return true;
-
-  const MachineOperand *BaseOpS, *BaseOpD;
-  int64_t OffsetS, OffsetD;
-  bool OffsetSIsScalable, OffsetDIsScalable;
-  const TargetRegisterInfo *TRI = MF.getSubtarget().getRegisterInfo();
-  if (!TII->getMemOperandWithOffset(*SI, BaseOpS, OffsetS, OffsetSIsScalable,
-                                    TRI) ||
-      !TII->getMemOperandWithOffset(*DI, BaseOpD, OffsetD, OffsetDIsScalable,
-                                    TRI))
-    return true;
-
-  assert(!OffsetSIsScalable && !OffsetDIsScalable &&
-         "Expected offsets to be byte offsets");
-
-  MachineInstr *DefS = MRI.getVRegDef(BaseOpS->getReg());
-  MachineInstr *DefD = MRI.getVRegDef(BaseOpD->getReg());
-  if (!DefS || !DefD || !DefS->isPHI() || !DefD->isPHI())
-    return true;
-
-  unsigned InitValS = 0;
-  unsigned LoopValS = 0;
-  unsigned InitValD = 0;
-  unsigned LoopValD = 0;
-  getPhiRegs(*DefS, BB, InitValS, LoopValS);
-  getPhiRegs(*DefD, BB, InitValD, LoopValD);
-  MachineInstr *InitDefS = MRI.getVRegDef(InitValS);
-  MachineInstr *InitDefD = MRI.getVRegDef(InitValD);
-
-  if (!InitDefS->isIdenticalTo(*InitDefD))
-    return true;
-
-  // Check that the base register is incremented by a constant value for each
-  // iteration.
-  MachineInstr *LoopDefS = MRI.getVRegDef(LoopValS);
-  int D = 0;
-  if (!LoopDefS || !TII->getIncrementValue(*LoopDefS, D))
-    return true;
-
-  LocationSize AccessSizeS = (*SI->memoperands_begin())->getSize();
-  LocationSize AccessSizeD = (*DI->memoperands_begin())->getSize();
-
-  // This is the main test, which checks the offset values and the loop
-  // increment value to determine if the accesses may be loop carried.
-  if (!AccessSizeS.hasValue() || !AccessSizeD.hasValue())
-    return true;
-
-  if (DeltaS != DeltaD || DeltaS < AccessSizeS.getValue() ||
-      DeltaD < AccessSizeD.getValue())
-    return true;
-
-  return (OffsetS + (int64_t)AccessSizeS.getValue() <
-          OffsetD + (int64_t)AccessSizeD.getValue());
+  return mayOverlapInLaterIter(DI, SI);
 }
 
 void SwingSchedulerDAG::postProcessDAG() {
diff --git a/llvm/lib/CodeGen/MachineScheduler.cpp b/llvm/lib/CodeGen/MachineScheduler.cpp
index 393530f56cc27..3f72e8486c06e 100644
--- a/llvm/lib/CodeGen/MachineScheduler.cpp
+++ b/llvm/lib/CodeGen/MachineScheduler.cpp
@@ -58,6 +58,7 @@
 #include "llvm/Support/ErrorHandling.h"
 #include "llvm/Support/GraphWriter.h"
 #include "llvm/Support/raw_ostream.h"
+#include "llvm/Target/TargetMachine.h"
 #include <algorithm>
 #include <cassert>
 #include <cstdint>
@@ -219,9 +220,7 @@ namespace {
 class MachineSchedulerBase : public MachineSchedContext,
                              public MachineFunctionPass {
 public:
-  MachineSchedulerBase(char &ID): MachineFunctionPass(ID) {}
-
-  void print(raw_ostream &O, const Module* = nullptr) const override;
+  MachineSchedulerBase(char &ID) : MachineFunctionPass(ID) {}
 
 protected:
   void scheduleRegions(ScheduleDAGInstrs &Scheduler, bool FixKillFlags);
@@ -392,8 +391,11 @@ ScheduleDAGInstrs *MachineScheduler::createMachineScheduler() {
   if (Ctor != useDefaultMachineSched)
     return Ctor(this);
 
+  const TargetMachine &TM =
+      getAnalysis<TargetPassConfig>().getTM<TargetMachine>();
+
   // Get the default scheduler set by the target for this function.
-  ScheduleDAGInstrs *Scheduler = PassConfig->createMachineScheduler(this);
+  ScheduleDAGInstrs *Scheduler = TM.createMachineScheduler(this);
   if (Scheduler)
     return Scheduler;
 
@@ -405,8 +407,10 @@ ScheduleDAGInstrs *MachineScheduler::createMachineScheduler() {
 /// the caller. We don't have a command line option to override the postRA
 /// scheduler. The Target must configure it.
 ScheduleDAGInstrs *PostMachineScheduler::createPostMachineScheduler() {
+  const TargetMachine &TM =
+      getAnalysis<TargetPassConfig>().getTM<TargetMachine>();
   // Get the postRA scheduler set by the target for this function.
-  ScheduleDAGInstrs *Scheduler = PassConfig->createPostMachineScheduler(this);
+  ScheduleDAGInstrs *Scheduler = TM.createPostMachineScheduler(this);
   if (Scheduler)
     return Scheduler;
 
@@ -446,7 +450,6 @@ bool MachineScheduler::runOnMachineFunction(MachineFunction &mf) {
   MF = &mf;
   MLI = &getAnalysis<MachineLoopInfoWrapperPass>().getLI();
   MDT = &getAnalysis<MachineDominatorTreeWrapperPass>().getDomTree();
-  PassConfig = &getAnalysis<TargetPassConfig>();
   AA = &getAnalysis<AAResultsWrapperPass>().getAAResults();
 
   LIS = &getAnalysis<LiveIntervalsWrapperPass>().getLIS();
@@ -484,7 +487,6 @@ bool PostMachineScheduler::runOnMachineFunction(MachineFunction &mf) {
   // Initialize the context of the pass.
   MF = &mf;
   MLI = &getAnalysis<MachineLoopInfoWrapperPass>().getLI();
-  PassConfig = &getAnalysis<TargetPassConfig>();
   AA = &getAnalysis<AAResultsWrapperPass>().getAAResults();
 
   if (VerifyScheduling)
@@ -662,10 +664,6 @@ void MachineSchedulerBase::scheduleRegions(ScheduleDAGInstrs &Scheduler,
   Scheduler.finalizeSchedule();
 }
 
-void MachineSchedulerBase::print(raw_ostream &O, const Module* m) const {
-  // unimplemented
-}
-
 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
 LLVM_DUMP_METHOD void ReadyQueue::dump() const {
   dbgs() << "Queue " << Name << ": ";
@@ -1293,9 +1291,9 @@ void ScheduleDAGMILive::initRegPressure() {
     updatePressureDiffs(LiveUses);
   }
 
-  LLVM_DEBUG(dbgs() << "Top Pressure:\n";
+  LLVM_DEBUG(dbgs() << "Top Pressure: ";
              dumpRegSetPressure(TopRPTracker.getRegSetPressureAtPos(), TRI);
-             dbgs() << "Bottom Pressure:\n";
+             dbgs() << "Bottom Pressure: ";
              dumpRegSetPressure(BotRPTracker.getRegSetPressureAtPos(), TRI););
 
   assert((BotRPTracker.getPos() == RegionEnd ||
@@ -1316,11 +1314,14 @@ void ScheduleDAGMILive::initRegPressure() {
       RegionCriticalPSets.push_back(PressureChange(i));
     }
   }
-  LLVM_DEBUG(dbgs() << "Excess PSets: ";
-             for (const PressureChange &RCPS
-                  : RegionCriticalPSets) dbgs()
-             << TRI->getRegPressureSetName(RCPS.getPSet()) << " ";
-             dbgs() << "\n");
+  LLVM_DEBUG({
+    if (RegionCriticalPSets.size() > 0) {
+      dbgs() << "Excess PSets: ";
+      for (const PressureChange &RCPS : RegionCriticalPSets)
+        dbgs() << TRI->getRegPressureSetName(RCPS.getPSet()) << " ";
+      dbgs() << "\n";
+    }
+  });
 }
 
 void ScheduleDAGMILive::
@@ -1374,10 +1375,14 @@ void ScheduleDAGMILive::updatePressureDiffs(ArrayRef<VRegMaskOrUnit> LiveUses) {
 
         PressureDiff &PDiff = getPressureDiff(&SU);
         PDiff.addPressureChange(Reg, Decrement, &MRI);
-        LLVM_DEBUG(dbgs() << "  UpdateRegP: SU(" << SU.NodeNum << ") "
-                          << printReg(Reg, TRI) << ':'
-                          << PrintLaneMask(P.LaneMask) << ' ' << *SU.getInstr();
-                   dbgs() << "              to "; PDiff.dump(*TRI););
+        if (llvm::any_of(PDiff, [](const PressureChange &Change) {
+              return Change.isValid();
+            }))
+          LLVM_DEBUG(dbgs()
+                         << "  UpdateRegPressure: SU(" << SU.NodeNum << ") "
+                         << printReg(Reg, TRI) << ':'
+                         << PrintLaneMask(P.LaneMask) << ' ' << *SU.getInstr();
+                     dbgs() << "                     to "; PDiff.dump(*TRI););
       }
     } else {
       assert(P.LaneMask.any());
@@ -1409,9 +1414,13 @@ void ScheduleDAGMILive::updatePressureDiffs(ArrayRef<VRegMaskOrUnit> LiveUses) {
           if (LRQ.valueIn() == VNI) {
             PressureDiff &PDiff = getPressureDiff(SU);
             PDiff.addPressureChange(Reg, true, &MRI);
-            LLVM_DEBUG(dbgs() << "  UpdateRegP: SU(" << SU->NodeNum << ") "
-                              << *SU->getInstr();
-                       dbgs() << "              to "; PDiff.dump(*TRI););
+            if (llvm::any_of(PDiff, [](const PressureChange &Change) {
+                  return Change.isValid();
+                }))
+              LLVM_DEBUG(dbgs() << "  UpdateRegPressure: SU(" << SU->NodeNum
+                                << ") " << *SU->getInstr();
+                         dbgs() << "                     to ";
+                         PDiff.dump(*TRI););
           }
         }
       }
@@ -1671,7 +1680,7 @@ void ScheduleDAGMILive::scheduleMI(SUnit *SU, bool IsTopNode) {
 
       TopRPTracker.advance(RegOpers);
       assert(TopRPTracker.getPos() == CurrentTop && "out of sync");
-      LLVM_DEBUG(dbgs() << "Top Pressure:\n"; dumpRegSetPressure(
+      LLVM_DEBUG(dbgs() << "Top Pressure: "; dumpRegSetPressure(
                      TopRPTracker.getRegSetPressureAtPos(), TRI););
 
       updateScheduledPressure(SU, TopRPTracker.getPressure().MaxSetPressure);
@@ -1709,7 +1718,7 @@ void ScheduleDAGMILive::scheduleMI(SUnit *SU, bool IsTopNode) {
       SmallVector<VRegMaskOrUnit, 8> LiveUses;
       BotRPTracker.recede(RegOpers, &LiveUses);
       assert(BotRPTracker.getPos() == CurrentBottom && "out of sync");
-      LLVM_DEBUG(dbgs() << "Bottom Pressure:\n"; dumpRegSetPressure(
+      LLVM_DEBUG(dbgs() << "Bottom Pressure: "; dumpRegSetPressure(
                      BotRPTracker.getRegSetPressureAtPos(), TRI););
 
       updateScheduledPressure(SU, BotRPTracker.getPressure().MaxSetPressure);
diff --git a/llvm/lib/CodeGen/MachineUniformityAnalysis.cpp b/llvm/lib/CodeGen/MachineUniformityAnalysis.cpp
index a4b78c1c75ceb..b5dc487e7cb0e 100644
--- a/llvm/lib/CodeGen/MachineUniformityAnalysis.cpp
+++ b/llvm/lib/CodeGen/MachineUniformityAnalysis.cpp
@@ -185,11 +185,11 @@ MachineUniformityAnalysisPass::MachineUniformityAnalysisPass()
 }
 
 INITIALIZE_PASS_BEGIN(MachineUniformityAnalysisPass, "machine-uniformity",
-                      "Machine Uniformity Info Analysis", true, true)
+                      "Machine Uniformity Info Analysis", false, true)
 INITIALIZE_PASS_DEPENDENCY(MachineCycleInfoWrapperPass)
 INITIALIZE_PASS_DEPENDENCY(MachineDominatorTreeWrapperPass)
 INITIALIZE_PASS_END(MachineUniformityAnalysisPass, "machine-uniformity",
-                    "Machine Uniformity Info Analysis", true, true)
+                    "Machine Uniformity Info Analysis", false, true)
 
 void MachineUniformityAnalysisPass::getAnalysisUsage(AnalysisUsage &AU) const {
   AU.setPreservesAll();
diff --git a/llvm/lib/CodeGen/MachineVerifier.cpp b/llvm/lib/CodeGen/MachineVerifier.cpp
index d41b11307e7bc..05afcbee701a8 100644
--- a/llvm/lib/CodeGen/MachineVerifier.cpp
+++ b/llvm/lib/CodeGen/MachineVerifier.cpp
@@ -313,7 +313,7 @@ struct MachineVerifier {
   void report(const Twine &Msg, const MachineInstr *MI);
 
   void report_context(const LiveInterval &LI) const;
-  void report_context(const LiveRange &LR, Register VRegUnit,
+  void report_context(const LiveRange &LR, VirtRegOrUnit VRegOrUnit,
                       LaneBitmask LaneMask) const;
   void report_context(const LiveRange::Segment &S) const;
   void report_context(const VNInfo &VNI) const;
@@ -322,18 +322,18 @@ struct MachineVerifier {
   void report_context_liverange(const LiveRange &LR) const;
   void report_context_lanemask(LaneBitmask LaneMask) const;
   void report_context_vreg(Register VReg) const;
-  void report_context_vreg_regunit(Register VRegOrUnit) const;
+  void report_context_vreg_regunit(VirtRegOrUnit VRegOrUnit) const;
 
   void verifyInlineAsm(const MachineInstr *MI);
 
   void checkLiveness(const MachineOperand *MO, unsigned MONum);
   void checkLivenessAtUse(const MachineOperand *MO, unsigned MONum,
                           SlotIndex UseIdx, const LiveRange &LR,
-                          Register VRegOrUnit,
+                          VirtRegOrUnit VRegOrUnit,
                           LaneBitmask LaneMask = LaneBitmask::getNone());
   void checkLivenessAtDef(const MachineOperand *MO, unsigned MONum,
                           SlotIndex DefIdx, const LiveRange &LR,
-                          Register VRegOrUnit, bool SubRangeCheck = false,
+                          VirtRegOrUnit VRegOrUnit, bool SubRangeCheck = false,
                           LaneBitmask LaneMask = LaneBitmask::getNone());
 
   void markReachable(const MachineBasicBlock *MBB);
@@ -344,12 +344,12 @@ struct MachineVerifier {
   void verifyLiveVariables();
   void verifyLiveIntervals();
   void verifyLiveInterval(const LiveInterval &);
-  void verifyLiveRangeValue(const LiveRange &, const VNInfo *, Register,
+  void verifyLiveRangeValue(const LiveRange &, const VNInfo *, VirtRegOrUnit,
                             LaneBitmask);
   void verifyLiveRangeSegment(const LiveRange &,
-                              const LiveRange::const_iterator I, Register,
+                              const LiveRange::const_iterator I, VirtRegOrUnit,
                               LaneBitmask);
-  void verifyLiveRange(const LiveRange &, Register,
+  void verifyLiveRange(const LiveRange &, VirtRegOrUnit,
                        LaneBitmask LaneMask = LaneBitmask::getNone());
 
   void verifyStackFrame();
@@ -431,6 +431,12 @@ bool MachineFunction::verify(Pass *p, const char *Banner, raw_ostream *OS,
   return MachineVerifier(p, Banner, OS, AbortOnError).verify(*this);
 }
 
+bool MachineFunction::verify(MachineFunctionAnalysisManager &MFAM,
+                             const char *Banner, raw_ostream *OS,
+                             bool AbortOnError) const {
+  return MachineVerifier(MFAM, Banner, OS, AbortOnError).verify(*this);
+}
+
 bool MachineFunction::verify(LiveIntervals *LiveInts, SlotIndexes *Indexes,
                              const char *Banner, raw_ostream *OS,
                              bool AbortOnError) const {
@@ -636,10 +642,11 @@ void MachineVerifier::report_context(const LiveInterval &LI) const {
   OS << "- interval:    " << LI << '\n';
 }
 
-void MachineVerifier::report_context(const LiveRange &LR, Register VRegUnit,
+void MachineVerifier::report_context(const LiveRange &LR,
+                                     VirtRegOrUnit VRegOrUnit,
                                      LaneBitmask LaneMask) const {
   report_context_liverange(LR);
-  report_context_vreg_regunit(VRegUnit);
+  report_context_vreg_regunit(VRegOrUnit);
   if (LaneMask.any())
     report_context_lanemask(LaneMask);
 }
@@ -664,11 +671,13 @@ void MachineVerifier::report_context_vreg(Register VReg) const {
   OS << "- v. register: " << printReg(VReg, TRI) << '\n';
 }
 
-void MachineVerifier::report_context_vreg_regunit(Register VRegOrUnit) const {
-  if (VRegOrUnit.isVirtual()) {
-    report_context_vreg(VRegOrUnit);
+void MachineVerifier::report_context_vreg_regunit(
+    VirtRegOrUnit VRegOrUnit) const {
+  if (VRegOrUnit.isVirtualReg()) {
+    report_context_vreg(VRegOrUnit.asVirtualReg());
   } else {
-    OS << "- regunit:     " << printRegUnit(VRegOrUnit, TRI) << '\n';
+    OS << "- regunit:     " << printRegUnit(VRegOrUnit.asMCRegUnit(), TRI)
+       << '\n';
   }
 }
 
@@ -1278,6 +1287,19 @@ void MachineVerifier::verifyPreISelGenericInstruction(const MachineInstr *MI) {
         if (TypeSize::isKnownGT(MMO.getSize().getValue(),
                                 ValTy.getSizeInBytes()))
           report("load memory size cannot exceed result size", MI);
+
+        if (MMO.getRanges()) {
+          ConstantInt *i =
+              mdconst::extract<ConstantInt>(MMO.getRanges()->getOperand(0));
+          const LLT RangeTy = LLT::scalar(i->getIntegerType()->getBitWidth());
+          const LLT MemTy = MMO.getMemoryType();
+          if (MemTy.getScalarType() != RangeTy ||
+              ValTy.isScalar() != MemTy.isScalar() ||
+              (ValTy.isVector() &&
+               ValTy.getNumElements() != MemTy.getNumElements())) {
+            report("range is incompatible with the result type", MI);
+          }
+        }
       } else if (MI->getOpcode() == TargetOpcode::G_STORE) {
         if (TypeSize::isKnownLT(ValTy.getSizeInBytes(),
                                 MMO.getSize().getValue()))
@@ -2828,7 +2850,7 @@ MachineVerifier::visitMachineOperand(const MachineOperand *MO, unsigned MONum) {
 void MachineVerifier::checkLivenessAtUse(const MachineOperand *MO,
                                          unsigned MONum, SlotIndex UseIdx,
                                          const LiveRange &LR,
-                                         Register VRegOrUnit,
+                                         VirtRegOrUnit VRegOrUnit,
                                          LaneBitmask LaneMask) {
   const MachineInstr *MI = MO->getParent();
 
@@ -2863,7 +2885,7 @@ void MachineVerifier::checkLivenessAtUse(const MachineOperand *MO,
 void MachineVerifier::checkLivenessAtDef(const MachineOperand *MO,
                                          unsigned MONum, SlotIndex DefIdx,
                                          const LiveRange &LR,
-                                         Register VRegOrUnit,
+                                         VirtRegOrUnit VRegOrUnit,
                                          bool SubRangeCheck,
                                          LaneBitmask LaneMask) {
   if (!LR.verify()) {
@@ -2908,7 +2930,7 @@ void MachineVerifier::checkLivenessAtDef(const MachineOperand *MO,
   if (MO->isDead()) {
     LiveQueryResult LRQ = LR.Query(DefIdx);
     if (!LRQ.isDeadDef()) {
-      assert(VRegOrUnit.isVirtual() && "Expecting a virtual register.");
+      assert(VRegOrUnit.isVirtualReg() && "Expecting a virtual register.");
       // A dead subreg def only tells us that the specific subreg is dead. There
       // could be other non-dead defs of other subregs, or we could have other
       // parts of the register being live through the instruction. So unless we
@@ -2973,13 +2995,13 @@ void MachineVerifier::checkLiveness(const MachineOperand *MO, unsigned MONum) {
           if (MRI->isReservedRegUnit(Unit))
             continue;
           if (const LiveRange *LR = LiveInts->getCachedRegUnit(Unit))
-            checkLivenessAtUse(MO, MONum, UseIdx, *LR, Unit);
+            checkLivenessAtUse(MO, MONum, UseIdx, *LR, VirtRegOrUnit(Unit));
         }
       }
 
       if (Reg.isVirtual()) {
         // This is a virtual register interval.
-        checkLivenessAtUse(MO, MONum, UseIdx, *LI, Reg);
+        checkLivenessAtUse(MO, MONum, UseIdx, *LI, VirtRegOrUnit(Reg));
 
         if (LI->hasSubRanges() && !MO->isDef()) {
           LaneBitmask MOMask = SubRegIdx != 0
@@ -2989,7 +3011,8 @@ void MachineVerifier::checkLiveness(const MachineOperand *MO, unsigned MONum) {
           for (const LiveInterval::SubRange &SR : LI->subranges()) {
             if ((MOMask & SR.LaneMask).none())
               continue;
-            checkLivenessAtUse(MO, MONum, UseIdx, SR, Reg, SR.LaneMask);
+            checkLivenessAtUse(MO, MONum, UseIdx, SR, VirtRegOrUnit(Reg),
+                               SR.LaneMask);
             LiveQueryResult LRQ = SR.Query(UseIdx);
             if (LRQ.valueIn() || (MI->isPHI() && LRQ.valueOut()))
               LiveInMask |= SR.LaneMask;
@@ -3081,7 +3104,7 @@ void MachineVerifier::checkLiveness(const MachineOperand *MO, unsigned MONum) {
       DefIdx = DefIdx.getRegSlot(MO->isEarlyClobber());
 
       if (Reg.isVirtual()) {
-        checkLivenessAtDef(MO, MONum, DefIdx, *LI, Reg);
+        checkLivenessAtDef(MO, MONum, DefIdx, *LI, VirtRegOrUnit(Reg));
 
         if (LI->hasSubRanges()) {
           LaneBitmask MOMask = SubRegIdx != 0
@@ -3090,7 +3113,8 @@ void MachineVerifier::checkLiveness(const MachineOperand *MO, unsigned MONum) {
           for (const LiveInterval::SubRange &SR : LI->subranges()) {
             if ((SR.LaneMask & MOMask).none())
               continue;
-            checkLivenessAtDef(MO, MONum, DefIdx, SR, Reg, true, SR.LaneMask);
+            checkLivenessAtDef(MO, MONum, DefIdx, SR, VirtRegOrUnit(Reg), true,
+                               SR.LaneMask);
           }
         }
       }
@@ -3160,7 +3184,7 @@ struct VRegFilter {
     for (Register Reg : FromRegSet) {
       if (!Reg.isVirtual())
         continue;
-      unsigned Index = Register::virtReg2Index(Reg);
+      unsigned Index = Reg.virtRegIndex();
       if (Index < SparseUniverseMax) {
         if (Index < SparseUniverse && Sparse.test(Index))
           continue;
@@ -3183,7 +3207,7 @@ struct VRegFilter {
     Dense.reserve(NewDenseSize);
     for (unsigned I = Begin; I < End; ++I) {
       Register Reg = ToVRegs[I];
-      unsigned Index = Register::virtReg2Index(Reg);
+      unsigned Index = Reg.virtRegIndex();
       if (Index < SparseUniverseMax)
         Sparse.set(Index);
       else
@@ -3532,11 +3556,12 @@ void MachineVerifier::verifyLiveIntervals() {
   // Verify all the cached regunit intervals.
   for (unsigned i = 0, e = TRI->getNumRegUnits(); i != e; ++i)
     if (const LiveRange *LR = LiveInts->getCachedRegUnit(i))
-      verifyLiveRange(*LR, i);
+      verifyLiveRange(*LR, VirtRegOrUnit(i));
 }
 
 void MachineVerifier::verifyLiveRangeValue(const LiveRange &LR,
-                                           const VNInfo *VNI, Register Reg,
+                                           const VNInfo *VNI,
+                                           VirtRegOrUnit VRegOrUnit,
                                            LaneBitmask LaneMask) {
   if (VNI->isUnused())
     return;
@@ -3545,14 +3570,14 @@ void MachineVerifier::verifyLiveRangeValue(const LiveRange &LR,
 
   if (!DefVNI) {
     report("Value not live at VNInfo def and not marked unused", MF);
-    report_context(LR, Reg, LaneMask);
+    report_context(LR, VRegOrUnit, LaneMask);
     report_context(*VNI);
     return;
   }
 
   if (DefVNI != VNI) {
     report("Live segment at def has different VNInfo", MF);
-    report_context(LR, Reg, LaneMask);
+    report_context(LR, VRegOrUnit, LaneMask);
     report_context(*VNI);
     return;
   }
@@ -3560,7 +3585,7 @@ void MachineVerifier::verifyLiveRangeValue(const LiveRange &LR,
   const MachineBasicBlock *MBB = LiveInts->getMBBFromIndex(VNI->def);
   if (!MBB) {
     report("Invalid VNInfo definition index", MF);
-    report_context(LR, Reg, LaneMask);
+    report_context(LR, VRegOrUnit, LaneMask);
     report_context(*VNI);
     return;
   }
@@ -3568,7 +3593,7 @@ void MachineVerifier::verifyLiveRangeValue(const LiveRange &LR,
   if (VNI->isPHIDef()) {
     if (VNI->def != LiveInts->getMBBStartIdx(MBB)) {
       report("PHIDef VNInfo is not defined at MBB start", MBB);
-      report_context(LR, Reg, LaneMask);
+      report_context(LR, VRegOrUnit, LaneMask);
       report_context(*VNI);
     }
     return;
@@ -3578,57 +3603,56 @@ void MachineVerifier::verifyLiveRangeValue(const LiveRange &LR,
   const MachineInstr *MI = LiveInts->getInstructionFromIndex(VNI->def);
   if (!MI) {
     report("No instruction at VNInfo def index", MBB);
-    report_context(LR, Reg, LaneMask);
+    report_context(LR, VRegOrUnit, LaneMask);
     report_context(*VNI);
     return;
   }
 
-  if (Reg != 0) {
-    bool hasDef = false;
-    bool isEarlyClobber = false;
-    for (ConstMIBundleOperands MOI(*MI); MOI.isValid(); ++MOI) {
-      if (!MOI->isReg() || !MOI->isDef())
+  bool hasDef = false;
+  bool isEarlyClobber = false;
+  for (ConstMIBundleOperands MOI(*MI); MOI.isValid(); ++MOI) {
+    if (!MOI->isReg() || !MOI->isDef())
+      continue;
+    if (VRegOrUnit.isVirtualReg()) {
+      if (MOI->getReg() != VRegOrUnit.asVirtualReg())
         continue;
-      if (Reg.isVirtual()) {
-        if (MOI->getReg() != Reg)
-          continue;
-      } else {
-        if (!MOI->getReg().isPhysical() || !TRI->hasRegUnit(MOI->getReg(), Reg))
-          continue;
-      }
-      if (LaneMask.any() &&
-          (TRI->getSubRegIndexLaneMask(MOI->getSubReg()) & LaneMask).none())
+    } else {
+      if (!MOI->getReg().isPhysical() ||
+          !TRI->hasRegUnit(MOI->getReg(), VRegOrUnit.asMCRegUnit()))
         continue;
-      hasDef = true;
-      if (MOI->isEarlyClobber())
-        isEarlyClobber = true;
     }
+    if (LaneMask.any() &&
+        (TRI->getSubRegIndexLaneMask(MOI->getSubReg()) & LaneMask).none())
+      continue;
+    hasDef = true;
+    if (MOI->isEarlyClobber())
+      isEarlyClobber = true;
+  }
 
-    if (!hasDef) {
-      report("Defining instruction does not modify register", MI);
-      report_context(LR, Reg, LaneMask);
-      report_context(*VNI);
-    }
+  if (!hasDef) {
+    report("Defining instruction does not modify register", MI);
+    report_context(LR, VRegOrUnit, LaneMask);
+    report_context(*VNI);
+  }
 
-    // Early clobber defs begin at USE slots, but other defs must begin at
-    // DEF slots.
-    if (isEarlyClobber) {
-      if (!VNI->def.isEarlyClobber()) {
-        report("Early clobber def must be at an early-clobber slot", MBB);
-        report_context(LR, Reg, LaneMask);
-        report_context(*VNI);
-      }
-    } else if (!VNI->def.isRegister()) {
-      report("Non-PHI, non-early clobber def must be at a register slot", MBB);
-      report_context(LR, Reg, LaneMask);
+  // Early clobber defs begin at USE slots, but other defs must begin at
+  // DEF slots.
+  if (isEarlyClobber) {
+    if (!VNI->def.isEarlyClobber()) {
+      report("Early clobber def must be at an early-clobber slot", MBB);
+      report_context(LR, VRegOrUnit, LaneMask);
       report_context(*VNI);
     }
+  } else if (!VNI->def.isRegister()) {
+    report("Non-PHI, non-early clobber def must be at a register slot", MBB);
+    report_context(LR, VRegOrUnit, LaneMask);
+    report_context(*VNI);
   }
 }
 
 void MachineVerifier::verifyLiveRangeSegment(const LiveRange &LR,
                                              const LiveRange::const_iterator I,
-                                             Register Reg,
+                                             VirtRegOrUnit VRegOrUnit,
                                              LaneBitmask LaneMask) {
   const LiveRange::Segment &S = *I;
   const VNInfo *VNI = S.valno;
@@ -3636,28 +3660,28 @@ void MachineVerifier::verifyLiveRangeSegment(const LiveRange &LR,
 
   if (VNI->id >= LR.getNumValNums() || VNI != LR.getValNumInfo(VNI->id)) {
     report("Foreign valno in live segment", MF);
-    report_context(LR, Reg, LaneMask);
+    report_context(LR, VRegOrUnit, LaneMask);
     report_context(S);
     report_context(*VNI);
   }
 
   if (VNI->isUnused()) {
     report("Live segment valno is marked unused", MF);
-    report_context(LR, Reg, LaneMask);
+    report_context(LR, VRegOrUnit, LaneMask);
     report_context(S);
   }
 
   const MachineBasicBlock *MBB = LiveInts->getMBBFromIndex(S.start);
   if (!MBB) {
     report("Bad start of live segment, no basic block", MF);
-    report_context(LR, Reg, LaneMask);
+    report_context(LR, VRegOrUnit, LaneMask);
     report_context(S);
     return;
   }
   SlotIndex MBBStartIdx = LiveInts->getMBBStartIdx(MBB);
   if (S.start != MBBStartIdx && S.start != VNI->def) {
     report("Live segment must begin at MBB entry or valno def", MBB);
-    report_context(LR, Reg, LaneMask);
+    report_context(LR, VRegOrUnit, LaneMask);
     report_context(S);
   }
 
@@ -3665,7 +3689,7 @@ void MachineVerifier::verifyLiveRangeSegment(const LiveRange &LR,
     LiveInts->getMBBFromIndex(S.end.getPrevSlot());
   if (!EndMBB) {
     report("Bad end of live segment, no basic block", MF);
-    report_context(LR, Reg, LaneMask);
+    report_context(LR, VRegOrUnit, LaneMask);
     report_context(S);
     return;
   }
@@ -3673,7 +3697,7 @@ void MachineVerifier::verifyLiveRangeSegment(const LiveRange &LR,
   // Checks for non-live-out segments.
   if (S.end != LiveInts->getMBBEndIdx(EndMBB)) {
     // RegUnit intervals are allowed dead phis.
-    if (!Reg.isVirtual() && VNI->isPHIDef() && S.start == VNI->def &&
+    if (!VRegOrUnit.isVirtualReg() && VNI->isPHIDef() && S.start == VNI->def &&
         S.end == VNI->def.getDeadSlot())
       return;
 
@@ -3682,7 +3706,7 @@ void MachineVerifier::verifyLiveRangeSegment(const LiveRange &LR,
         LiveInts->getInstructionFromIndex(S.end.getPrevSlot());
     if (!MI) {
       report("Live segment doesn't end at a valid instruction", EndMBB);
-      report_context(LR, Reg, LaneMask);
+      report_context(LR, VRegOrUnit, LaneMask);
       report_context(S);
       return;
     }
@@ -3690,7 +3714,7 @@ void MachineVerifier::verifyLiveRangeSegment(const LiveRange &LR,
     // The block slot must refer to a basic block boundary.
     if (S.end.isBlock()) {
       report("Live segment ends at B slot of an instruction", EndMBB);
-      report_context(LR, Reg, LaneMask);
+      report_context(LR, VRegOrUnit, LaneMask);
       report_context(S);
     }
 
@@ -3699,7 +3723,7 @@ void MachineVerifier::verifyLiveRangeSegment(const LiveRange &LR,
       // That means there must be a dead def.
       if (!SlotIndex::isSameInstr(S.start, S.end)) {
         report("Live segment ending at dead slot spans instructions", EndMBB);
-        report_context(LR, Reg, LaneMask);
+        report_context(LR, VRegOrUnit, LaneMask);
         report_context(S);
       }
     }
@@ -3715,21 +3739,21 @@ void MachineVerifier::verifyLiveRangeSegment(const LiveRange &LR,
         report("Live segment ending at early clobber slot must be "
                "redefined by an EC def in the same instruction",
                EndMBB);
-        report_context(LR, Reg, LaneMask);
+        report_context(LR, VRegOrUnit, LaneMask);
         report_context(S);
       }
     }
 
     // The following checks only apply to virtual registers. Physreg liveness
     // is too weird to check.
-    if (Reg.isVirtual()) {
+    if (VRegOrUnit.isVirtualReg()) {
       // A live segment can end with either a redefinition, a kill flag on a
       // use, or a dead flag on a def.
       bool hasRead = false;
       bool hasSubRegDef = false;
       bool hasDeadDef = false;
       for (ConstMIBundleOperands MOI(*MI); MOI.isValid(); ++MOI) {
-        if (!MOI->isReg() || MOI->getReg() != Reg)
+        if (!MOI->isReg() || MOI->getReg() != VRegOrUnit.asVirtualReg())
           continue;
         unsigned Sub = MOI->getSubReg();
         LaneBitmask SLM =
@@ -3758,18 +3782,18 @@ void MachineVerifier::verifyLiveRangeSegment(const LiveRange &LR,
           report(
               "Instruction ending live segment on dead slot has no dead flag",
               MI);
-          report_context(LR, Reg, LaneMask);
+          report_context(LR, VRegOrUnit, LaneMask);
           report_context(S);
         }
       } else {
         if (!hasRead) {
           // When tracking subregister liveness, the main range must start new
           // values on partial register writes, even if there is no read.
-          if (!MRI->shouldTrackSubRegLiveness(Reg) || LaneMask.any() ||
-              !hasSubRegDef) {
+          if (!MRI->shouldTrackSubRegLiveness(VRegOrUnit.asVirtualReg()) ||
+              LaneMask.any() || !hasSubRegDef) {
             report("Instruction ending live segment doesn't read the register",
                    MI);
-            report_context(LR, Reg, LaneMask);
+            report_context(LR, VRegOrUnit, LaneMask);
             report_context(S);
           }
         }
@@ -3790,14 +3814,14 @@ void MachineVerifier::verifyLiveRangeSegment(const LiveRange &LR,
 
   SmallVector<SlotIndex, 4> Undefs;
   if (LaneMask.any()) {
-    LiveInterval &OwnerLI = LiveInts->getInterval(Reg);
+    LiveInterval &OwnerLI = LiveInts->getInterval(VRegOrUnit.asVirtualReg());
     OwnerLI.computeSubRangeUndefs(Undefs, LaneMask, *MRI, *Indexes);
   }
 
   while (true) {
     assert(LiveInts->isLiveInToMBB(LR, &*MFI));
     // We don't know how to track physregs into a landing pad.
-    if (!Reg.isVirtual() && MFI->isEHPad()) {
+    if (!VRegOrUnit.isVirtualReg() && MFI->isEHPad()) {
       if (&*MFI == EndMBB)
         break;
       ++MFI;
@@ -3830,7 +3854,7 @@ void MachineVerifier::verifyLiveRangeSegment(const LiveRange &LR,
         if (LiveRangeCalc::isJointlyDominated(Pred, Undefs, *Indexes))
           continue;
         report("Register not marked live out of predecessor", Pred);
-        report_context(LR, Reg, LaneMask);
+        report_context(LR, VRegOrUnit, LaneMask);
         report_context(*VNI);
         OS << " live into " << printMBBReference(*MFI) << '@'
            << LiveInts->getMBBStartIdx(&*MFI) << ", not live before " << PEnd
@@ -3841,7 +3865,7 @@ void MachineVerifier::verifyLiveRangeSegment(const LiveRange &LR,
       // Only PHI-defs can take different predecessor values.
       if (!IsPHI && PVNI != VNI) {
         report("Different value live out of predecessor", Pred);
-        report_context(LR, Reg, LaneMask);
+        report_context(LR, VRegOrUnit, LaneMask);
         OS << "Valno #" << PVNI->id << " live out of "
            << printMBBReference(*Pred) << '@' << PEnd << "\nValno #" << VNI->id
            << " live into " << printMBBReference(*MFI) << '@'
@@ -3854,19 +3878,20 @@ void MachineVerifier::verifyLiveRangeSegment(const LiveRange &LR,
   }
 }
 
-void MachineVerifier::verifyLiveRange(const LiveRange &LR, Register Reg,
+void MachineVerifier::verifyLiveRange(const LiveRange &LR,
+                                      VirtRegOrUnit VRegOrUnit,
                                       LaneBitmask LaneMask) {
   for (const VNInfo *VNI : LR.valnos)
-    verifyLiveRangeValue(LR, VNI, Reg, LaneMask);
+    verifyLiveRangeValue(LR, VNI, VRegOrUnit, LaneMask);
 
   for (LiveRange::const_iterator I = LR.begin(), E = LR.end(); I != E; ++I)
-    verifyLiveRangeSegment(LR, I, Reg, LaneMask);
+    verifyLiveRangeSegment(LR, I, VRegOrUnit, LaneMask);
 }
 
 void MachineVerifier::verifyLiveInterval(const LiveInterval &LI) {
   Register Reg = LI.reg();
   assert(Reg.isVirtual());
-  verifyLiveRange(LI, Reg);
+  verifyLiveRange(LI, VirtRegOrUnit(Reg));
 
   if (LI.hasSubRanges()) {
     LaneBitmask Mask;
@@ -3882,10 +3907,10 @@ void MachineVerifier::verifyLiveInterval(const LiveInterval &LI) {
       }
       if (SR.empty()) {
         report("Subrange must not be empty", MF);
-        report_context(SR, LI.reg(), SR.LaneMask);
+        report_context(SR, VirtRegOrUnit(LI.reg()), SR.LaneMask);
       }
       Mask |= SR.LaneMask;
-      verifyLiveRange(SR, LI.reg(), SR.LaneMask);
+      verifyLiveRange(SR, VirtRegOrUnit(LI.reg()), SR.LaneMask);
       if (!LI.covers(SR)) {
         report("A Subrange is not covered by the main range", MF);
         report_context(LI);
diff --git a/llvm/lib/CodeGen/ModuloSchedule.cpp b/llvm/lib/CodeGen/ModuloSchedule.cpp
index d99b6ace01000..f9fe812f7e65c 100644
--- a/llvm/lib/CodeGen/ModuloSchedule.cpp
+++ b/llvm/lib/CodeGen/ModuloSchedule.cpp
@@ -397,8 +397,9 @@ void ModuloScheduleExpander::generateExistingPhis(
     // The Phi value from the loop body typically is defined in the loop, but
     // not always. So, we need to check if the value is defined in the loop.
     unsigned PhiOp2 = LoopVal;
-    if (VRMap[LastStageNum].count(LoopVal))
-      PhiOp2 = VRMap[LastStageNum][LoopVal];
+    if (auto It = VRMap[LastStageNum].find(LoopVal);
+        It != VRMap[LastStageNum].end())
+      PhiOp2 = It->second;
 
     int StageScheduled = Schedule.getStage(&*BBI);
     int LoopValStage = Schedule.getStage(MRI.getVRegDef(LoopVal));
@@ -1055,8 +1056,8 @@ void ModuloScheduleExpander::updateInstruction(MachineInstr *NewMI,
         // Make an adjustment to get the last definition.
         StageNum -= StageDiff;
       }
-      if (VRMap[StageNum].count(reg))
-        MO.setReg(VRMap[StageNum][reg]);
+      if (auto It = VRMap[StageNum].find(reg); It != VRMap[StageNum].end())
+        MO.setReg(It->second);
     }
   }
 }
@@ -1710,8 +1711,8 @@ void PeelingModuloScheduleExpander::moveStageBetweenBlocks(
     for (MachineOperand &MO : I->uses()) {
       if (!MO.isReg())
         continue;
-      if (Remaps.count(MO.getReg()))
-        MO.setReg(Remaps[MO.getReg()]);
+      if (auto It = Remaps.find(MO.getReg()); It != Remaps.end())
+        MO.setReg(It->second);
       else {
         // If we are using a phi from the source block we need to add a new phi
         // pointing to the old one.
diff --git a/llvm/lib/CodeGen/PeepholeOptimizer.cpp b/llvm/lib/CodeGen/PeepholeOptimizer.cpp
index 48c25d5039bfd..745c0d4b36a62 100644
--- a/llvm/lib/CodeGen/PeepholeOptimizer.cpp
+++ b/llvm/lib/CodeGen/PeepholeOptimizer.cpp
@@ -153,7 +153,7 @@ class RecurrenceInstr;
 class Rewriter {
 protected:
   MachineInstr &CopyLike;
-  unsigned CurrentSrcIdx = 0; ///< The index of the source being rewritten.
+  int CurrentSrcIdx = 0; ///< The index of the source being rewritten.
 public:
   Rewriter(MachineInstr &CopyLike) : CopyLike(CopyLike) {}
   virtual ~Rewriter() = default;
@@ -201,14 +201,11 @@ class CopyRewriter : public Rewriter {
 
   bool getNextRewritableSource(RegSubRegPair &Src,
                                RegSubRegPair &Dst) override {
-    // CurrentSrcIdx > 0 means this function has already been called.
-    if (CurrentSrcIdx > 0)
+    if (++CurrentSrcIdx > 1)
       return false;
-    // This is the first call to getNextRewritableSource.
-    // Move the CurrentSrcIdx to remember that we made that call.
-    CurrentSrcIdx = 1;
+
     // The rewritable source is the argument.
-    const MachineOperand &MOSrc = CopyLike.getOperand(1);
+    const MachineOperand &MOSrc = CopyLike.getOperand(CurrentSrcIdx);
     Src = RegSubRegPair(MOSrc.getReg(), MOSrc.getSubReg());
     // What we track are the alternative sources of the definition.
     const MachineOperand &MODef = CopyLike.getOperand(0);
@@ -217,8 +214,6 @@ class CopyRewriter : public Rewriter {
   }
 
   bool RewriteCurrentSource(Register NewReg, unsigned NewSubReg) override {
-    if (CurrentSrcIdx != 1)
-      return false;
     MachineOperand &MOSrc = CopyLike.getOperand(CurrentSrcIdx);
     MOSrc.setReg(NewReg);
     MOSrc.setSubReg(NewSubReg);
@@ -229,7 +224,7 @@ class CopyRewriter : public Rewriter {
 /// Helper class to rewrite uncoalescable copy like instructions
 /// into new COPY (coalescable friendly) instructions.
 class UncoalescableRewriter : public Rewriter {
-  unsigned NumDefs; ///< Number of defs in the bitcast.
+  int NumDefs; ///< Number of defs in the bitcast.
 
 public:
   UncoalescableRewriter(MachineInstr &MI) : Rewriter(MI) {
@@ -383,6 +378,7 @@ class RegSequenceRewriter : public Rewriter {
 public:
   RegSequenceRewriter(MachineInstr &MI) : Rewriter(MI) {
     assert(MI.isRegSequence() && "Invalid instruction");
+    CurrentSrcIdx = -1;
   }
 
   /// \see Rewriter::getNextRewritableSource()
@@ -404,16 +400,10 @@ class RegSequenceRewriter : public Rewriter {
   bool getNextRewritableSource(RegSubRegPair &Src,
                                RegSubRegPair &Dst) override {
     // We are looking at v0 = REG_SEQUENCE v1, sub1, v2, sub2, etc.
+    CurrentSrcIdx += 2;
+    if (static_cast<unsigned>(CurrentSrcIdx) >= CopyLike.getNumOperands())
+      return false;
 
-    // If this is the first call, move to the first argument.
-    if (CurrentSrcIdx == 0) {
-      CurrentSrcIdx = 1;
-    } else {
-      // Otherwise, move to the next argument and check that it is valid.
-      CurrentSrcIdx += 2;
-      if (CurrentSrcIdx >= CopyLike.getNumOperands())
-        return false;
-    }
     const MachineOperand &MOInsertedReg = CopyLike.getOperand(CurrentSrcIdx);
     Src.Reg = MOInsertedReg.getReg();
     // If we have to compose sub-register indices, bail out.
@@ -431,9 +421,10 @@ class RegSequenceRewriter : public Rewriter {
   }
 
   bool RewriteCurrentSource(Register NewReg, unsigned NewSubReg) override {
-    // We cannot rewrite out of bound operands.
-    // Moreover, rewritable sources are at odd positions.
-    if ((CurrentSrcIdx & 1) != 1 || CurrentSrcIdx > CopyLike.getNumOperands())
+    // Do not introduce new subregister uses in a reg_sequence. Until composing
+    // subregister indices is supported while folding, we're just blocking
+    // folding of subregister copies later in the function.
+    if (NewSubReg)
       return false;
 
     MachineOperand &MO = CopyLike.getOperand(CurrentSrcIdx);
@@ -474,7 +465,8 @@ class PeepholeOptimizer : private MachineFunction::Delegate {
   bool optimizeUncoalescableCopy(MachineInstr &MI,
                                  SmallPtrSetImpl<MachineInstr *> &LocalMIs);
   bool optimizeRecurrence(MachineInstr &PHI);
-  bool findNextSource(RegSubRegPair RegSubReg, RewriteMapTy &RewriteMap);
+  bool findNextSource(const TargetRegisterClass *DefRC, unsigned DefSubReg,
+                      RegSubRegPair RegSubReg, RewriteMapTy &RewriteMap);
   bool isMoveImmediate(MachineInstr &MI, SmallSet<Register, 4> &ImmDefRegs,
                        DenseMap<Register, MachineInstr *> &ImmDefMIs);
   bool foldImmediate(MachineInstr &MI, SmallSet<Register, 4> &ImmDefRegs,
@@ -513,7 +505,7 @@ class PeepholeOptimizer : private MachineFunction::Delegate {
 
   /// Check whether \p MI is understood by the register coalescer
   /// but may require some rewriting.
-  bool isCoalescableCopy(const MachineInstr &MI) {
+  static bool isCoalescableCopy(const MachineInstr &MI) {
     // SubregToRegs are not interesting, because they are already register
     // coalescer friendly.
     return MI.isCopy() ||
@@ -523,7 +515,7 @@ class PeepholeOptimizer : private MachineFunction::Delegate {
 
   /// Check whether \p MI is a copy like instruction that is
   /// not recognized by the register coalescer.
-  bool isUncoalescableCopy(const MachineInstr &MI) {
+  static bool isUncoalescableCopy(const MachineInstr &MI) {
     return MI.isBitcast() || (!DisableAdvCopyOpt && (MI.isRegSequenceLike() ||
                                                      MI.isInsertSubregLike() ||
                                                      MI.isExtractSubregLike()));
@@ -1000,8 +992,10 @@ bool PeepholeOptimizer::optimizeCondBranch(MachineInstr &MI) {
   return TII->optimizeCondBranch(MI);
 }
 
-/// Try to find the next source that share the same register file
-/// for the value defined by \p Reg and \p SubReg.
+/// Try to find a better source value that shares the same register file to
+/// replace \p RegSubReg in an instruction like
+/// `DefRC.DefSubReg = COPY RegSubReg`
+///
 /// When true is returned, the \p RewriteMap can be used by the client to
 /// retrieve all Def -> Use along the way up to the next source. Any found
 /// Use that is not itself a key for another entry, is the next source to
@@ -1011,17 +1005,15 @@ bool PeepholeOptimizer::optimizeCondBranch(MachineInstr &MI) {
 /// share the same register file as \p Reg and \p SubReg. The client should
 /// then be capable to rewrite all intermediate PHIs to get the next source.
 /// \return False if no alternative sources are available. True otherwise.
-bool PeepholeOptimizer::findNextSource(RegSubRegPair RegSubReg,
+bool PeepholeOptimizer::findNextSource(const TargetRegisterClass *DefRC,
+                                       unsigned DefSubReg,
+                                       RegSubRegPair RegSubReg,
                                        RewriteMapTy &RewriteMap) {
   // Do not try to find a new source for a physical register.
   // So far we do not have any motivating example for doing that.
   // Thus, instead of maintaining untested code, we will revisit that if
   // that changes at some point.
   Register Reg = RegSubReg.Reg;
-  if (Reg.isPhysical())
-    return false;
-  const TargetRegisterClass *DefRC = MRI->getRegClass(Reg);
-
   SmallVector<RegSubRegPair, 4> SrcToLook;
   RegSubRegPair CurSrcPair = RegSubReg;
   SrcToLook.push_back(CurSrcPair);
@@ -1044,8 +1036,11 @@ bool PeepholeOptimizer::findNextSource(RegSubRegPair RegSubReg,
         return false;
 
       // Insert the Def -> Use entry for the recently found source.
-      ValueTrackerResult CurSrcRes = RewriteMap.lookup(CurSrcPair);
-      if (CurSrcRes.isValid()) {
+      auto [InsertPt, WasInserted] = RewriteMap.try_emplace(CurSrcPair, Res);
+
+      if (!WasInserted) {
+        const ValueTrackerResult &CurSrcRes = InsertPt->second;
+
         assert(CurSrcRes == Res && "ValueTrackerResult found must match");
         // An existent entry with multiple sources is a PHI cycle we must avoid.
         // Otherwise it's an entry with a valid next source we already found.
@@ -1056,7 +1051,6 @@ bool PeepholeOptimizer::findNextSource(RegSubRegPair RegSubReg,
         }
         break;
       }
-      RewriteMap.insert(std::make_pair(CurSrcPair, Res));
 
       // ValueTrackerResult usually have one source unless it's the result from
       // a PHI instruction. Add the found PHI edges to be looked up further.
@@ -1083,7 +1077,7 @@ bool PeepholeOptimizer::findNextSource(RegSubRegPair RegSubReg,
 
       // Keep following the chain if the value isn't any better yet.
       const TargetRegisterClass *SrcRC = MRI->getRegClass(CurSrcPair.Reg);
-      if (!TRI->shouldRewriteCopySrc(DefRC, RegSubReg.SubReg, SrcRC,
+      if (!TRI->shouldRewriteCopySrc(DefRC, DefSubReg, SrcRC,
                                      CurSrcPair.SubReg))
         continue;
 
@@ -1191,21 +1185,33 @@ bool PeepholeOptimizer::optimizeCoalescableCopyImpl(Rewriter &&CpyRewriter) {
   bool Changed = false;
   // Get the right rewriter for the current copy.
   // Rewrite each rewritable source.
-  RegSubRegPair Src;
+  RegSubRegPair Dst;
   RegSubRegPair TrackPair;
-  while (CpyRewriter.getNextRewritableSource(Src, TrackPair)) {
+  while (CpyRewriter.getNextRewritableSource(TrackPair, Dst)) {
+    if (Dst.Reg.isPhysical()) {
+      // Do not try to find a new source for a physical register.
+      // So far we do not have any motivating example for doing that.
+      // Thus, instead of maintaining untested code, we will revisit that if
+      // that changes at some point.
+      continue;
+    }
+
+    const TargetRegisterClass *DefRC = MRI->getRegClass(Dst.Reg);
+
     // Keep track of PHI nodes and its incoming edges when looking for sources.
     RewriteMapTy RewriteMap;
     // Try to find a more suitable source. If we failed to do so, or get the
     // actual source, move to the next source.
-    if (!findNextSource(TrackPair, RewriteMap))
+    if (!findNextSource(DefRC, Dst.SubReg, TrackPair, RewriteMap))
       continue;
 
     // Get the new source to rewrite. TODO: Only enable handling of multiple
     // sources (PHIs) once we have a motivating example and testcases for it.
     RegSubRegPair NewSrc = getNewSource(MRI, TII, TrackPair, RewriteMap,
                                         /*HandleMultipleSources=*/false);
-    if (Src.Reg == NewSrc.Reg || NewSrc.Reg == 0)
+    assert(TrackPair.Reg != NewSrc.Reg &&
+           "should not rewrite source to original value");
+    if (!NewSrc.Reg)
       continue;
 
     // Rewrite source.
@@ -1332,9 +1338,14 @@ bool PeepholeOptimizer::optimizeUncoalescableCopy(
     if (Def.Reg.isPhysical())
       return false;
 
+    // FIXME: Uncoalescable copies are treated differently by
+    // UncoalescableRewriter, and this probably should not share
+    // API. getNextRewritableSource really finds rewritable defs.
+    const TargetRegisterClass *DefRC = MRI->getRegClass(Def.Reg);
+
     // If we do not know how to rewrite this definition, there is no point
     // in trying to kill this instruction.
-    if (!findNextSource(Def, RewriteMap))
+    if (!findNextSource(DefRC, Def.SubReg, Def, RewriteMap))
       return false;
 
     RewritePairs.push_back(Def);
@@ -1965,22 +1976,7 @@ ValueTrackerResult ValueTracker::getNextSourceFromRegSequence() {
   assert((Def->isRegSequence() || Def->isRegSequenceLike()) &&
          "Invalid definition");
 
-  if (Def->getOperand(DefIdx).getSubReg())
-    // If we are composing subregs, bail out.
-    // The case we are checking is Def.<subreg> = REG_SEQUENCE.
-    // This should almost never happen as the SSA property is tracked at
-    // the register level (as opposed to the subreg level).
-    // I.e.,
-    // Def.sub0 =
-    // Def.sub1 =
-    // is a valid SSA representation for Def.sub0 and Def.sub1, but not for
-    // Def. Thus, it must not be generated.
-    // However, some code could theoretically generates a single
-    // Def.sub0 (i.e, not defining the other subregs) and we would
-    // have this case.
-    // If we can ascertain (or force) that this never happens, we could
-    // turn that into an assertion.
-    return ValueTrackerResult();
+  assert(!Def->getOperand(DefIdx).getSubReg() && "illegal subregister def");
 
   SmallVector<RegSubRegPairAndIdx, 8> RegSeqInputRegs;
   if (!TII->getRegSequenceInputs(*Def, DefIdx, RegSeqInputRegs))
diff --git a/llvm/lib/CodeGen/PostRASchedulerList.cpp b/llvm/lib/CodeGen/PostRASchedulerList.cpp
index badfd9a68d6a0..039a473382732 100644
--- a/llvm/lib/CodeGen/PostRASchedulerList.cpp
+++ b/llvm/lib/CodeGen/PostRASchedulerList.cpp
@@ -17,6 +17,7 @@
 //
 //===----------------------------------------------------------------------===//
 
+#include "llvm/CodeGen/PostRASchedulerList.h"
 #include "llvm/ADT/Statistic.h"
 #include "llvm/Analysis/AliasAnalysis.h"
 #include "llvm/CodeGen/AntiDepBreaker.h"
@@ -39,6 +40,7 @@
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/ErrorHandling.h"
 #include "llvm/Support/raw_ostream.h"
+#include "llvm/Target/TargetMachine.h"
 using namespace llvm;
 
 #define DEBUG_TYPE "post-RA-sched"
@@ -73,124 +75,134 @@ DebugMod("postra-sched-debugmod",
 AntiDepBreaker::~AntiDepBreaker() = default;
 
 namespace {
-  class PostRAScheduler : public MachineFunctionPass {
-    const TargetInstrInfo *TII = nullptr;
-    RegisterClassInfo RegClassInfo;
-
-  public:
-    static char ID;
-    PostRAScheduler() : MachineFunctionPass(ID) {}
-
-    void getAnalysisUsage(AnalysisUsage &AU) const override {
-      AU.setPreservesCFG();
-      AU.addRequired<AAResultsWrapperPass>();
-      AU.addRequired<TargetPassConfig>();
-      AU.addRequired<MachineDominatorTreeWrapperPass>();
-      AU.addPreserved<MachineDominatorTreeWrapperPass>();
-      AU.addRequired<MachineLoopInfoWrapperPass>();
-      AU.addPreserved<MachineLoopInfoWrapperPass>();
-      MachineFunctionPass::getAnalysisUsage(AU);
-    }
+class PostRAScheduler {
+  const TargetInstrInfo *TII = nullptr;
+  MachineLoopInfo *MLI = nullptr;
+  AliasAnalysis *AA = nullptr;
+  const TargetMachine *TM = nullptr;
+  RegisterClassInfo RegClassInfo;
+
+public:
+  PostRAScheduler(MachineFunction &MF, MachineLoopInfo *MLI, AliasAnalysis *AA,
+                  const TargetMachine *TM)
+      : TII(MF.getSubtarget().getInstrInfo()), MLI(MLI), AA(AA), TM(TM) {}
+  bool run(MachineFunction &MF);
+};
+
+class PostRASchedulerLegacy : public MachineFunctionPass {
+public:
+  static char ID;
+  PostRASchedulerLegacy() : MachineFunctionPass(ID) {}
+
+  void getAnalysisUsage(AnalysisUsage &AU) const override {
+    AU.setPreservesCFG();
+    AU.addRequired<AAResultsWrapperPass>();
+    AU.addRequired<TargetPassConfig>();
+    AU.addRequired<MachineDominatorTreeWrapperPass>();
+    AU.addPreserved<MachineDominatorTreeWrapperPass>();
+    AU.addRequired<MachineLoopInfoWrapperPass>();
+    AU.addPreserved<MachineLoopInfoWrapperPass>();
+    MachineFunctionPass::getAnalysisUsage(AU);
+  }
 
-    MachineFunctionProperties getRequiredProperties() const override {
-      return MachineFunctionProperties().set(
-          MachineFunctionProperties::Property::NoVRegs);
-    }
+  MachineFunctionProperties getRequiredProperties() const override {
+    return MachineFunctionProperties().set(
+        MachineFunctionProperties::Property::NoVRegs);
+  }
 
-    bool runOnMachineFunction(MachineFunction &Fn) override;
-  };
-  char PostRAScheduler::ID = 0;
+  bool runOnMachineFunction(MachineFunction &Fn) override;
+};
+char PostRASchedulerLegacy::ID = 0;
 
-  class SchedulePostRATDList : public ScheduleDAGInstrs {
-    /// AvailableQueue - The priority queue to use for the available SUnits.
-    ///
-    LatencyPriorityQueue AvailableQueue;
+class SchedulePostRATDList : public ScheduleDAGInstrs {
+  /// AvailableQueue - The priority queue to use for the available SUnits.
+  ///
+  LatencyPriorityQueue AvailableQueue;
 
-    /// PendingQueue - This contains all of the instructions whose operands have
-    /// been issued, but their results are not ready yet (due to the latency of
-    /// the operation).  Once the operands becomes available, the instruction is
-    /// added to the AvailableQueue.
-    std::vector<SUnit*> PendingQueue;
+  /// PendingQueue - This contains all of the instructions whose operands have
+  /// been issued, but their results are not ready yet (due to the latency of
+  /// the operation).  Once the operands becomes available, the instruction is
+  /// added to the AvailableQueue.
+  std::vector<SUnit *> PendingQueue;
 
-    /// HazardRec - The hazard recognizer to use.
-    ScheduleHazardRecognizer *HazardRec;
+  /// HazardRec - The hazard recognizer to use.
+  ScheduleHazardRecognizer *HazardRec;
 
-    /// AntiDepBreak - Anti-dependence breaking object, or NULL if none
-    AntiDepBreaker *AntiDepBreak;
+  /// AntiDepBreak - Anti-dependence breaking object, or NULL if none
+  AntiDepBreaker *AntiDepBreak;
 
-    /// AA - AliasAnalysis for making memory reference queries.
-    AliasAnalysis *AA;
+  /// AA - AliasAnalysis for making memory reference queries.
+  AliasAnalysis *AA;
 
-    /// The schedule. Null SUnit*'s represent noop instructions.
-    std::vector<SUnit*> Sequence;
+  /// The schedule. Null SUnit*'s represent noop instructions.
+  std::vector<SUnit *> Sequence;
 
-    /// Ordered list of DAG postprocessing steps.
-    std::vector<std::unique_ptr<ScheduleDAGMutation>> Mutations;
+  /// Ordered list of DAG postprocessing steps.
+  std::vector<std::unique_ptr<ScheduleDAGMutation>> Mutations;
 
-    /// The index in BB of RegionEnd.
-    ///
-    /// This is the instruction number from the top of the current block, not
-    /// the SlotIndex. It is only used by the AntiDepBreaker.
-    unsigned EndIndex = 0;
+  /// The index in BB of RegionEnd.
+  ///
+  /// This is the instruction number from the top of the current block, not
+  /// the SlotIndex. It is only used by the AntiDepBreaker.
+  unsigned EndIndex = 0;
 
-  public:
-    SchedulePostRATDList(
-        MachineFunction &MF, MachineLoopInfo &MLI, AliasAnalysis *AA,
-        const RegisterClassInfo &,
-        TargetSubtargetInfo::AntiDepBreakMode AntiDepMode,
-        SmallVectorImpl<const TargetRegisterClass *> &CriticalPathRCs);
+public:
+  SchedulePostRATDList(
+      MachineFunction &MF, MachineLoopInfo &MLI, AliasAnalysis *AA,
+      const RegisterClassInfo &,
+      TargetSubtargetInfo::AntiDepBreakMode AntiDepMode,
+      SmallVectorImpl<const TargetRegisterClass *> &CriticalPathRCs);
 
-    ~SchedulePostRATDList() override;
+  ~SchedulePostRATDList() override;
 
-    /// startBlock - Initialize register live-range state for scheduling in
-    /// this block.
-    ///
-    void startBlock(MachineBasicBlock *BB) override;
+  /// startBlock - Initialize register live-range state for scheduling in
+  /// this block.
+  ///
+  void startBlock(MachineBasicBlock *BB) override;
 
-    // Set the index of RegionEnd within the current BB.
-    void setEndIndex(unsigned EndIdx) { EndIndex = EndIdx; }
+  // Set the index of RegionEnd within the current BB.
+  void setEndIndex(unsigned EndIdx) { EndIndex = EndIdx; }
 
-    /// Initialize the scheduler state for the next scheduling region.
-    void enterRegion(MachineBasicBlock *bb,
-                     MachineBasicBlock::iterator begin,
-                     MachineBasicBlock::iterator end,
-                     unsigned regioninstrs) override;
+  /// Initialize the scheduler state for the next scheduling region.
+  void enterRegion(MachineBasicBlock *bb, MachineBasicBlock::iterator begin,
+                   MachineBasicBlock::iterator end,
+                   unsigned regioninstrs) override;
 
-    /// Notify that the scheduler has finished scheduling the current region.
-    void exitRegion() override;
+  /// Notify that the scheduler has finished scheduling the current region.
+  void exitRegion() override;
 
-    /// Schedule - Schedule the instruction range using list scheduling.
-    ///
-    void schedule() override;
+  /// Schedule - Schedule the instruction range using list scheduling.
+  ///
+  void schedule() override;
 
-    void EmitSchedule();
+  void EmitSchedule();
 
-    /// Observe - Update liveness information to account for the current
-    /// instruction, which will not be scheduled.
-    ///
-    void Observe(MachineInstr &MI, unsigned Count);
+  /// Observe - Update liveness information to account for the current
+  /// instruction, which will not be scheduled.
+  ///
+  void Observe(MachineInstr &MI, unsigned Count);
 
-    /// finishBlock - Clean up register live-range state.
-    ///
-    void finishBlock() override;
+  /// finishBlock - Clean up register live-range state.
+  ///
+  void finishBlock() override;
 
-  private:
-    /// Apply each ScheduleDAGMutation step in order.
-    void postProcessDAG();
+private:
+  /// Apply each ScheduleDAGMutation step in order.
+  void postProcessDAG();
 
-    void ReleaseSucc(SUnit *SU, SDep *SuccEdge);
-    void ReleaseSuccessors(SUnit *SU);
-    void ScheduleNodeTopDown(SUnit *SU, unsigned CurCycle);
-    void ListScheduleTopDown();
+  void ReleaseSucc(SUnit *SU, SDep *SuccEdge);
+  void ReleaseSuccessors(SUnit *SU);
+  void ScheduleNodeTopDown(SUnit *SU, unsigned CurCycle);
+  void ListScheduleTopDown();
 
-    void dumpSchedule() const;
-    void emitNoop(unsigned CurCycle);
-  };
-}
+  void dumpSchedule() const;
+  void emitNoop(unsigned CurCycle);
+};
+} // namespace
 
-char &llvm::PostRASchedulerID = PostRAScheduler::ID;
+char &llvm::PostRASchedulerID = PostRASchedulerLegacy::ID;
 
-INITIALIZE_PASS(PostRAScheduler, DEBUG_TYPE,
+INITIALIZE_PASS(PostRASchedulerLegacy, DEBUG_TYPE,
                 "Post RA top-down list latency scheduler", false, false)
 
 SchedulePostRATDList::SchedulePostRATDList(
@@ -263,19 +275,12 @@ static bool enablePostRAScheduler(const TargetSubtargetInfo &ST,
          OptLevel >= ST.getOptLevelToEnablePostRAScheduler();
 }
 
-bool PostRAScheduler::runOnMachineFunction(MachineFunction &Fn) {
-  if (skipFunction(Fn.getFunction()))
-    return false;
-
-  const auto &Subtarget = Fn.getSubtarget();
-  TargetPassConfig *PassConfig = &getAnalysis<TargetPassConfig>();
+bool PostRAScheduler::run(MachineFunction &MF) {
+  const auto &Subtarget = MF.getSubtarget();
   // Check that post-RA scheduling is enabled for this target.
-  if (!enablePostRAScheduler(Subtarget, PassConfig->getOptLevel()))
+  if (!enablePostRAScheduler(Subtarget, TM->getOptLevel()))
     return false;
 
-  TII = Subtarget.getInstrInfo();
-  MachineLoopInfo &MLI = getAnalysis<MachineLoopInfoWrapperPass>().getLI();
-  AliasAnalysis *AA = &getAnalysis<AAResultsWrapperPass>().getAAResults();
   TargetSubtargetInfo::AntiDepBreakMode AntiDepMode =
       Subtarget.getAntiDepBreakMode();
   if (EnableAntiDepBreaking.getPosition() > 0) {
@@ -287,22 +292,22 @@ bool PostRAScheduler::runOnMachineFunction(MachineFunction &Fn) {
   }
   SmallVector<const TargetRegisterClass *, 4> CriticalPathRCs;
   Subtarget.getCriticalPathRCs(CriticalPathRCs);
-  RegClassInfo.runOnMachineFunction(Fn);
+  RegClassInfo.runOnMachineFunction(MF);
 
   LLVM_DEBUG(dbgs() << "PostRAScheduler\n");
 
-  SchedulePostRATDList Scheduler(Fn, MLI, AA, RegClassInfo, AntiDepMode,
+  SchedulePostRATDList Scheduler(MF, *MLI, AA, RegClassInfo, AntiDepMode,
                                  CriticalPathRCs);
 
   // Loop over all of the basic blocks
-  for (auto &MBB : Fn) {
+  for (auto &MBB : MF) {
 #ifndef NDEBUG
     // If DebugDiv > 0 then only schedule MBB with (ID % DebugDiv) == DebugMod
     if (DebugDiv > 0) {
       static int bbcnt = 0;
       if (bbcnt++ % DebugDiv != DebugMod)
         continue;
-      dbgs() << "*** DEBUG scheduling " << Fn.getName() << ":"
+      dbgs() << "*** DEBUG scheduling " << MF.getName() << ":"
              << printMBBReference(MBB) << " ***\n";
     }
 #endif
@@ -320,7 +325,7 @@ bool PostRAScheduler::runOnMachineFunction(MachineFunction &Fn) {
       // Calls are not scheduling boundaries before register allocation, but
       // post-ra we don't gain anything by scheduling across calls since we
       // don't need to worry about register pressure.
-      if (MI.isCall() || TII->isSchedulingBoundary(MI, &MBB, Fn)) {
+      if (MI.isCall() || TII->isSchedulingBoundary(MI, &MBB, MF)) {
         Scheduler.enterRegion(&MBB, I, Current, CurrentCount - Count);
         Scheduler.setEndIndex(CurrentCount);
         Scheduler.schedule();
@@ -353,6 +358,39 @@ bool PostRAScheduler::runOnMachineFunction(MachineFunction &Fn) {
   return true;
 }
 
+bool PostRASchedulerLegacy::runOnMachineFunction(MachineFunction &MF) {
+  if (skipFunction(MF.getFunction()))
+    return false;
+
+  MachineLoopInfo *MLI = &getAnalysis<MachineLoopInfoWrapperPass>().getLI();
+  AliasAnalysis *AA = &getAnalysis<AAResultsWrapperPass>().getAAResults();
+  const TargetMachine *TM =
+      &getAnalysis<TargetPassConfig>().getTM<TargetMachine>();
+  PostRAScheduler Impl(MF, MLI, AA, TM);
+  return Impl.run(MF);
+}
+
+PreservedAnalyses
+PostRASchedulerPass::run(MachineFunction &MF,
+                         MachineFunctionAnalysisManager &MFAM) {
+  MFPropsModifier _(*this, MF);
+
+  MachineLoopInfo *MLI = &MFAM.getResult<MachineLoopAnalysis>(MF);
+  auto &FAM = MFAM.getResult<FunctionAnalysisManagerMachineFunctionProxy>(MF)
+                  .getManager();
+  AliasAnalysis *AA = &FAM.getResult<AAManager>(MF.getFunction());
+  PostRAScheduler Impl(MF, MLI, AA, TM);
+  bool Changed = Impl.run(MF);
+  if (!Changed)
+    return PreservedAnalyses::all();
+
+  PreservedAnalyses PA = getMachineFunctionPassPreservedAnalyses();
+  PA.preserveSet<CFGAnalyses>();
+  PA.preserve<MachineDominatorTreeAnalysis>();
+  PA.preserve<MachineLoopAnalysis>();
+  return PA;
+}
+
 /// StartBlock - Initialize register live-range state for scheduling in
 /// this block.
 ///
diff --git a/llvm/lib/CodeGen/PreISelIntrinsicLowering.cpp b/llvm/lib/CodeGen/PreISelIntrinsicLowering.cpp
index 048a6a49e4cb9..27fa0b43d74f6 100644
--- a/llvm/lib/CodeGen/PreISelIntrinsicLowering.cpp
+++ b/llvm/lib/CodeGen/PreISelIntrinsicLowering.cpp
@@ -32,6 +32,7 @@
 #include "llvm/Support/Casting.h"
 #include "llvm/Target/TargetMachine.h"
 #include "llvm/Transforms/Scalar/LowerConstantIntrinsics.h"
+#include "llvm/Transforms/Utils/BuildLibCalls.h"
 #include "llvm/Transforms/Utils/LowerMemIntrinsics.h"
 #include "llvm/Transforms/Utils/LowerVectorIntrinsics.h"
 
@@ -233,6 +234,60 @@ static bool canEmitLibcall(const TargetMachine *TM, Function *F,
   return TLI->getLibcallName(LC) != nullptr;
 }
 
+// Return a value appropriate for use with the memset_pattern16 libcall, if
+// possible and if we know how. (Adapted from equivalent helper in
+// LoopIdiomRecognize).
+static Constant *getMemSetPattern16Value(MemSetPatternInst *Inst,
+                                         const TargetLibraryInfo &TLI) {
+  // TODO: This could check for UndefValue because it can be merged into any
+  // other valid pattern.
+
+  // Don't emit libcalls if a non-default address space is being used.
+  if (Inst->getRawDest()->getType()->getPointerAddressSpace() != 0)
+    return nullptr;
+
+  Value *V = Inst->getValue();
+  Type *VTy = V->getType();
+  const DataLayout &DL = Inst->getDataLayout();
+  Module *M = Inst->getModule();
+
+  if (!isLibFuncEmittable(M, &TLI, LibFunc_memset_pattern16))
+    return nullptr;
+
+  // If the value isn't a constant, we can't promote it to being in a constant
+  // array.  We could theoretically do a store to an alloca or something, but
+  // that doesn't seem worthwhile.
+  Constant *C = dyn_cast<Constant>(V);
+  if (!C || isa<ConstantExpr>(C))
+    return nullptr;
+
+  // Only handle simple values that are a power of two bytes in size.
+  uint64_t Size = DL.getTypeSizeInBits(VTy);
+  if (!DL.typeSizeEqualsStoreSize(VTy) || !isPowerOf2_64(Size))
+    return nullptr;
+
+  // Don't care enough about darwin/ppc to implement this.
+  if (DL.isBigEndian())
+    return nullptr;
+
+  // Convert to size in bytes.
+  Size /= 8;
+
+  // TODO: If CI is larger than 16-bytes, we can try slicing it in half to see
+  // if the top and bottom are the same (e.g. for vectors and large integers).
+  if (Size > 16)
+    return nullptr;
+
+  // If the constant is exactly 16 bytes, just use it.
+  if (Size == 16)
+    return C;
+
+  // Otherwise, we'll use an array of the constants.
+  uint64_t ArraySize = 16 / Size;
+  ArrayType *AT = ArrayType::get(V->getType(), ArraySize);
+  return ConstantArray::get(AT, std::vector<Constant *>(ArraySize, C));
+}
+
 // TODO: Handle atomic memcpy and memcpy.inline
 // TODO: Pass ScalarEvolution
 bool PreISelIntrinsicLowering::expandMemIntrinsicUses(Function &F) const {
@@ -323,7 +378,56 @@ bool PreISelIntrinsicLowering::expandMemIntrinsicUses(Function &F) const {
     }
     case Intrinsic::experimental_memset_pattern: {
       auto *Memset = cast<MemSetPatternInst>(Inst);
-      expandMemSetPatternAsLoop(Memset);
+      const TargetLibraryInfo &TLI = LookupTLI(*Memset->getFunction());
+      Constant *PatternValue = getMemSetPattern16Value(Memset, TLI);
+      if (!PatternValue) {
+        // If it isn't possible to emit a memset_pattern16 libcall, expand to
+        // a loop instead.
+        expandMemSetPatternAsLoop(Memset);
+        Changed = true;
+        Memset->eraseFromParent();
+        break;
+      }
+      // FIXME: There is currently no profitability calculation for emitting
+      // the libcall vs expanding the memset.pattern directly.
+      IRBuilder<> Builder(Inst);
+      Module *M = Memset->getModule();
+      const DataLayout &DL = Memset->getDataLayout();
+
+      StringRef FuncName = "memset_pattern16";
+      FunctionCallee MSP = getOrInsertLibFunc(
+          M, TLI, LibFunc_memset_pattern16, Builder.getVoidTy(),
+          Memset->getRawDest()->getType(), Builder.getPtrTy(),
+          Memset->getLength()->getType());
+      inferNonMandatoryLibFuncAttrs(M, FuncName, TLI);
+
+      // Otherwise we should form a memset_pattern16.  PatternValue is known
+      // to be an constant array of 16-bytes. Put the value into a mergable
+      // global.
+      assert(Memset->getRawDest()->getType()->getPointerAddressSpace() == 0 &&
+             "Should have skipped if non-zero AS");
+      GlobalVariable *GV = new GlobalVariable(
+          *M, PatternValue->getType(), /*isConstant=*/true,
+          GlobalValue::PrivateLinkage, PatternValue, ".memset_pattern");
+      GV->setUnnamedAddr(
+          GlobalValue::UnnamedAddr::Global); // Ok to merge these.
+      // TODO: Consider relaxing alignment requirement.
+      GV->setAlignment(Align(16));
+      Value *PatternPtr = GV;
+      Value *NumBytes = Builder.CreateMul(
+          Builder.getInt64(DL.getTypeSizeInBits(Memset->getValue()->getType()) /
+                           8),
+          Memset->getLength());
+      CallInst *MemsetPattern16Call =
+          Builder.CreateCall(MSP, {Memset->getRawDest(), PatternPtr, NumBytes});
+      MemsetPattern16Call->setAAMetadata(Memset->getAAMetadata());
+      // Preserve any call site attributes on the destination pointer
+      // argument (e.g. alignment).
+      AttrBuilder ArgAttrs(Memset->getContext(),
+                           Memset->getAttributes().getParamAttrs(0));
+      MemsetPattern16Call->setAttributes(
+          MemsetPattern16Call->getAttributes().addParamAttributes(
+              Memset->getContext(), 0, ArgAttrs));
       Changed = true;
       Memset->eraseFromParent();
       break;
diff --git a/llvm/lib/CodeGen/PrologEpilogInserter.cpp b/llvm/lib/CodeGen/PrologEpilogInserter.cpp
index 51e9a067707e2..eb8929cae069e 100644
--- a/llvm/lib/CodeGen/PrologEpilogInserter.cpp
+++ b/llvm/lib/CodeGen/PrologEpilogInserter.cpp
@@ -208,8 +208,8 @@ static void stashEntryDbgValues(MachineBasicBlock &MBB,
   }
 
   // Remove stashed debug values from the block.
-  if (EntryDbgValues.count(&MBB))
-    for (auto *MI : EntryDbgValues[&MBB])
+  if (auto It = EntryDbgValues.find(&MBB); It != EntryDbgValues.end())
+    for (auto *MI : It->second)
       MI->removeFromParent();
 }
 
diff --git a/llvm/lib/CodeGen/RDFGraph.cpp b/llvm/lib/CodeGen/RDFGraph.cpp
index 483e61db788f4..805b0ee7be0bc 100644
--- a/llvm/lib/CodeGen/RDFGraph.cpp
+++ b/llvm/lib/CodeGen/RDFGraph.cpp
@@ -966,15 +966,18 @@ void DataFlowGraph::build(const Config &config) {
 
   // Build a map "PhiM" which will contain, for each block, the set
   // of references that will require phi definitions in that block.
+  // "PhiClobberM" map contains references that require phis for clobbering defs
   BlockRefsMap PhiM(getPRI());
+  BlockRefsMap PhiClobberM(getPRI());
   for (Block BA : Blocks)
-    recordDefsForDF(PhiM, BA);
+    recordDefsForDF(PhiM, PhiClobberM, BA);
   for (Block BA : Blocks)
     buildPhis(PhiM, BA);
 
   // Link all the refs. This will recursively traverse the dominator tree.
+  // Phis for clobbering defs are added here.
   DefStackMap DM;
-  linkBlockRefs(DM, EA);
+  linkBlockRefs(DM, PhiClobberM, EA);
 
   // Finally, remove all unused phi nodes.
   if (!(BuildCfg.Options & BuildOptions::KeepDeadPhis))
@@ -1378,7 +1381,9 @@ void DataFlowGraph::buildStmt(Block BA, MachineInstr &In) {
 
 // Scan all defs in the block node BA and record in PhiM the locations of
 // phi nodes corresponding to these defs.
-void DataFlowGraph::recordDefsForDF(BlockRefsMap &PhiM, Block BA) {
+// Clobbering defs in BA are recorded in PhiClobberM
+void DataFlowGraph::recordDefsForDF(BlockRefsMap &PhiM,
+                                    BlockRefsMap &PhiClobberM, Block BA) {
   // Check all defs from block BA and record them in each block in BA's
   // iterated dominance frontier. This information will later be used to
   // create phi nodes.
@@ -1394,11 +1399,17 @@ void DataFlowGraph::recordDefsForDF(BlockRefsMap &PhiM, Block BA) {
   // This is done to make sure that each defined reference gets only one
   // phi node, even if it is defined multiple times.
   RegisterAggr Defs(getPRI());
+  RegisterAggr ClobberDefs(getPRI());
   for (Instr IA : BA.Addr->members(*this)) {
     for (Ref RA : IA.Addr->members_if(IsDef, *this)) {
       RegisterRef RR = RA.Addr->getRegRef(*this);
-      if (RR.isReg() && isTracked(RR))
+      if (!isTracked(RR))
+        continue;
+      if (RR.isReg())
         Defs.insert(RR);
+      // Clobbering def
+      else if (RR.isMask())
+        ClobberDefs.insert(RR);
     }
   }
 
@@ -1416,12 +1427,14 @@ void DataFlowGraph::recordDefsForDF(BlockRefsMap &PhiM, Block BA) {
   for (auto *DB : IDF) {
     Block DBA = findBlock(DB);
     PhiM[DBA.Id].insert(Defs);
+    PhiClobberM[DBA.Id].insert(ClobberDefs);
   }
 }
 
 // Given the locations of phi nodes in the map PhiM, create the phi nodes
 // that are located in the block node BA.
-void DataFlowGraph::buildPhis(BlockRefsMap &PhiM, Block BA) {
+void DataFlowGraph::buildPhis(BlockRefsMap &PhiM, Block BA,
+                              const DefStackMap &DefM) {
   // Check if this blocks has any DF defs, i.e. if there are any defs
   // that this block is in the iterated dominance frontier of.
   auto HasDF = PhiM.find(BA.Id);
@@ -1434,10 +1447,37 @@ void DataFlowGraph::buildPhis(BlockRefsMap &PhiM, Block BA) {
   for (MachineBasicBlock *PB : MBB->predecessors())
     Preds.push_back(findBlock(PB));
 
+  RegisterAggr PhiDefs(getPRI());
+  // DefM will be non empty when we are building phis
+  // for clobbering defs
+  if (!DefM.empty()) {
+    for (Instr IA : BA.Addr->members_if(IsPhi, *this)) {
+      for (Def DA : IA.Addr->members_if(IsDef, *this)) {
+        auto DR = DA.Addr->getRegRef(*this);
+        PhiDefs.insert(DR);
+      }
+    }
+  }
+
+  MachineRegisterInfo &MRI = MF.getRegInfo();
   const RegisterAggr &Defs = PhiM[BA.Id];
   uint16_t PhiFlags = NodeAttrs::PhiRef | NodeAttrs::Preserving;
 
   for (RegisterRef RR : Defs.refs()) {
+    if (!DefM.empty()) {
+      auto F = DefM.find(RR.Reg);
+      // Do not create a phi for unallocatable registers, or for registers
+      // that are never livein to BA.
+      // If a phi exists for RR, do not create another.
+      if (!MRI.isAllocatable(RR.Reg) || PhiDefs.hasCoverOf(RR) ||
+          F == DefM.end() || F->second.empty())
+        continue;
+      // Do not create a phi, if all reaching defs are clobbering
+      auto RDef = F->second.top();
+      if (RDef->Addr->getFlags() & NodeAttrs::Clobbering)
+        continue;
+      PhiDefs.insert(RR);
+    }
     Phi PA = newPhi(BA);
     PA.Addr->addMember(newDef(PA, RR, PhiFlags), *this);
 
@@ -1576,7 +1616,15 @@ void DataFlowGraph::linkStmtRefs(DefStackMap &DefM, Stmt SA, Predicate P) {
 
 // Create data-flow links for all instructions in the block node BA. This
 // will include updating any phi nodes in BA.
-void DataFlowGraph::linkBlockRefs(DefStackMap &DefM, Block BA) {
+void DataFlowGraph::linkBlockRefs(DefStackMap &DefM, BlockRefsMap &PhiClobberM,
+                                  Block BA) {
+  // Create phi nodes for clobbering defs.
+  // Since a huge number of registers can get clobbered, it would result in many
+  // phi nodes being created in the graph. Only create phi nodes that have a non
+  // clobbering reaching def. Use DefM to get not clobbering defs reaching a
+  // block.
+  buildPhis(PhiClobberM, BA, DefM);
+
   // Push block delimiters.
   markBlock(BA.Id, DefM);
 
@@ -1613,7 +1661,7 @@ void DataFlowGraph::linkBlockRefs(DefStackMap &DefM, Block BA) {
   for (auto *I : *N) {
     MachineBasicBlock *SB = I->getBlock();
     Block SBA = findBlock(SB);
-    linkBlockRefs(DefM, SBA);
+    linkBlockRefs(DefM, PhiClobberM, SBA);
   }
 
   // Link the phi uses from the successor blocks.
diff --git a/llvm/lib/CodeGen/ReachingDefAnalysis.cpp b/llvm/lib/CodeGen/ReachingDefAnalysis.cpp
index 9459904d56e4c..59ad9ffae5bf9 100644
--- a/llvm/lib/CodeGen/ReachingDefAnalysis.cpp
+++ b/llvm/lib/CodeGen/ReachingDefAnalysis.cpp
@@ -10,6 +10,8 @@
 #include "llvm/ADT/SetOperations.h"
 #include "llvm/ADT/SmallSet.h"
 #include "llvm/CodeGen/LiveRegUnits.h"
+#include "llvm/CodeGen/MachineFrameInfo.h"
+#include "llvm/CodeGen/TargetInstrInfo.h"
 #include "llvm/CodeGen/TargetRegisterInfo.h"
 #include "llvm/CodeGen/TargetSubtargetInfo.h"
 #include "llvm/Support/Debug.h"
@@ -18,6 +20,10 @@ using namespace llvm;
 
 #define DEBUG_TYPE "reaching-defs-analysis"
 
+static cl::opt<bool> PrintAllReachingDefs("print-all-reaching-defs", cl::Hidden,
+                                          cl::desc("Used for test purpuses"),
+                                          cl::Hidden);
+
 char ReachingDefAnalysis::ID = 0;
 INITIALIZE_PASS(ReachingDefAnalysis, DEBUG_TYPE, "ReachingDefAnalysis", false,
                 true)
@@ -48,6 +54,16 @@ static bool isValidRegDefOf(const MachineOperand &MO, Register Reg,
   return TRI->regsOverlap(MO.getReg(), Reg);
 }
 
+static bool isFIDef(const MachineInstr &MI, int FrameIndex,
+                    const TargetInstrInfo *TII) {
+  int DefFrameIndex = 0;
+  int SrcFrameIndex = 0;
+  if (TII->isStoreToStackSlot(MI, DefFrameIndex) ||
+      TII->isStackSlotCopy(MI, DefFrameIndex, SrcFrameIndex))
+    return DefFrameIndex == FrameIndex;
+  return false;
+}
+
 void ReachingDefAnalysis::enterBasicBlock(MachineBasicBlock *MBB) {
   unsigned MBBNumber = MBB->getNumber();
   assert(MBBNumber < MBBReachingDefs.numBlockIDs() &&
@@ -126,6 +142,13 @@ void ReachingDefAnalysis::processDefs(MachineInstr *MI) {
          "Unexpected basic block number.");
 
   for (auto &MO : MI->operands()) {
+    if (MO.isFI()) {
+      int FrameIndex = MO.getIndex();
+      assert(FrameIndex >= 0 && "Can't handle negative frame indicies yet!");
+      if (!isFIDef(*MI, FrameIndex, TII))
+        continue;
+      MBBFrameObjsReachingDefs[{MBBNumber, FrameIndex}].push_back(CurInstr);
+    }
     if (!isValidRegDef(MO))
       continue;
     for (MCRegUnit Unit : TRI->regunits(MO.getReg().asMCReg())) {
@@ -209,12 +232,58 @@ void ReachingDefAnalysis::processBasicBlock(
   leaveBasicBlock(MBB);
 }
 
+void ReachingDefAnalysis::printAllReachingDefs(MachineFunction &MF) {
+  dbgs() << "RDA results for " << MF.getName() << "\n";
+  int Num = 0;
+  DenseMap<MachineInstr *, int> InstToNumMap;
+  SmallPtrSet<MachineInstr *, 2> Defs;
+  for (MachineBasicBlock &MBB : MF) {
+    for (MachineInstr &MI : MBB) {
+      for (MachineOperand &MO : MI.operands()) {
+        Register Reg;
+        if (MO.isFI()) {
+          int FrameIndex = MO.getIndex();
+          assert(FrameIndex >= 0 &&
+                 "Can't handle negative frame indicies yet!");
+          Reg = Register::index2StackSlot(FrameIndex);
+        } else if (MO.isReg()) {
+          if (MO.isDef())
+            continue;
+          Reg = MO.getReg();
+          if (!Reg.isValid())
+            continue;
+        } else
+          continue;
+        Defs.clear();
+        getGlobalReachingDefs(&MI, Reg, Defs);
+        MO.print(dbgs(), TRI);
+        SmallVector<int, 0> Nums;
+        for (MachineInstr *Def : Defs)
+          Nums.push_back(InstToNumMap[Def]);
+        llvm::sort(Nums);
+        dbgs() << ":{ ";
+        for (int Num : Nums)
+          dbgs() << Num << " ";
+        dbgs() << "}\n";
+      }
+      dbgs() << Num << ": " << MI << "\n";
+      InstToNumMap[&MI] = Num;
+      ++Num;
+    }
+  }
+}
+
 bool ReachingDefAnalysis::runOnMachineFunction(MachineFunction &mf) {
   MF = &mf;
   TRI = MF->getSubtarget().getRegisterInfo();
+  const TargetSubtargetInfo &STI = MF->getSubtarget();
+  TRI = STI.getRegisterInfo();
+  TII = STI.getInstrInfo();
   LLVM_DEBUG(dbgs() << "********** REACHING DEFINITION ANALYSIS **********\n");
   init();
   traverse();
+  if (PrintAllReachingDefs)
+    printAllReachingDefs(*MF);
   return false;
 }
 
@@ -222,6 +291,7 @@ void ReachingDefAnalysis::releaseMemory() {
   // Clear the internal vectors.
   MBBOutRegsInfos.clear();
   MBBReachingDefs.clear();
+  MBBFrameObjsReachingDefs.clear();
   InstIds.clear();
   LiveRegs.clear();
 }
@@ -234,6 +304,8 @@ void ReachingDefAnalysis::reset() {
 
 void ReachingDefAnalysis::init() {
   NumRegUnits = TRI->getNumRegUnits();
+  NumStackObjects = MF->getFrameInfo().getNumObjects();
+  ObjectIndexBegin = MF->getFrameInfo().getObjectIndexBegin();
   MBBReachingDefs.init(MF->getNumBlockIDs());
   // Initialize the MBBOutRegsInfos
   MBBOutRegsInfos.resize(MF->getNumBlockIDs());
@@ -268,6 +340,23 @@ int ReachingDefAnalysis::getReachingDef(MachineInstr *MI, Register Reg) const {
   assert(MBBNumber < MBBReachingDefs.numBlockIDs() &&
          "Unexpected basic block number.");
   int LatestDef = ReachingDefDefaultVal;
+
+  if (Reg.isStack()) {
+    // Check that there was a reaching def.
+    int FrameIndex = Reg.stackSlotIndex();
+    auto Lookup = MBBFrameObjsReachingDefs.find({MBBNumber, FrameIndex});
+    if (Lookup == MBBFrameObjsReachingDefs.end())
+      return LatestDef;
+    auto &Defs = Lookup->second;
+    for (int Def : Defs) {
+      if (Def >= InstId)
+        break;
+      DefRes = Def;
+    }
+    LatestDef = std::max(LatestDef, DefRes);
+    return LatestDef;
+  }
+
   for (MCRegUnit Unit : TRI->regunits(Reg)) {
     for (int Def : MBBReachingDefs.defs(MBBNumber, Unit)) {
       if (Def >= InstId)
@@ -419,7 +508,7 @@ void ReachingDefAnalysis::getLiveOuts(MachineBasicBlock *MBB, Register Reg,
   VisitedBBs.insert(MBB);
   LiveRegUnits LiveRegs(*TRI);
   LiveRegs.addLiveOuts(*MBB);
-  if (LiveRegs.available(Reg))
+  if (Reg.isPhysical() && LiveRegs.available(Reg))
     return;
 
   if (auto *Def = getLocalLiveOutMIDef(MBB, Reg))
@@ -500,7 +589,7 @@ bool ReachingDefAnalysis::isReachingDefLiveOut(MachineInstr *MI,
   MachineBasicBlock *MBB = MI->getParent();
   LiveRegUnits LiveRegs(*TRI);
   LiveRegs.addLiveOuts(*MBB);
-  if (LiveRegs.available(Reg))
+  if (Reg.isPhysical() && LiveRegs.available(Reg))
     return false;
 
   auto Last = MBB->getLastNonDebugInstr();
@@ -520,14 +609,21 @@ MachineInstr *ReachingDefAnalysis::getLocalLiveOutMIDef(MachineBasicBlock *MBB,
                                                         Register Reg) const {
   LiveRegUnits LiveRegs(*TRI);
   LiveRegs.addLiveOuts(*MBB);
-  if (LiveRegs.available(Reg))
+  if (Reg.isPhysical() && LiveRegs.available(Reg))
     return nullptr;
 
   auto Last = MBB->getLastNonDebugInstr();
   if (Last == MBB->end())
     return nullptr;
 
+  if (Reg.isStack()) {
+    int FrameIndex = Reg.stackSlotIndex();
+    if (isFIDef(*Last, FrameIndex, TII))
+      return &*Last;
+  }
+
   int Def = getReachingDef(&*Last, Reg);
+
   for (auto &MO : Last->operands())
     if (isValidRegDefOf(MO, Reg, TRI))
       return &*Last;
diff --git a/llvm/lib/CodeGen/RegAllocBasic.cpp b/llvm/lib/CodeGen/RegAllocBasic.cpp
index f3f34f890be11..e1f05406297d2 100644
--- a/llvm/lib/CodeGen/RegAllocBasic.cpp
+++ b/llvm/lib/CodeGen/RegAllocBasic.cpp
@@ -134,7 +134,7 @@ INITIALIZE_PASS_BEGIN(RABasic, "regallocbasic", "Basic Register Allocator",
 INITIALIZE_PASS_DEPENDENCY(LiveDebugVariablesWrapperLegacy)
 INITIALIZE_PASS_DEPENDENCY(SlotIndexesWrapperPass)
 INITIALIZE_PASS_DEPENDENCY(LiveIntervalsWrapperPass)
-INITIALIZE_PASS_DEPENDENCY(RegisterCoalescer)
+INITIALIZE_PASS_DEPENDENCY(RegisterCoalescerLegacy)
 INITIALIZE_PASS_DEPENDENCY(MachineScheduler)
 INITIALIZE_PASS_DEPENDENCY(LiveStacksWrapperLegacy)
 INITIALIZE_PASS_DEPENDENCY(AAResultsWrapperPass)
diff --git a/llvm/lib/CodeGen/RegAllocFast.cpp b/llvm/lib/CodeGen/RegAllocFast.cpp
index e2309b65cf9a2..14128dafbe4ee 100644
--- a/llvm/lib/CodeGen/RegAllocFast.cpp
+++ b/llvm/lib/CodeGen/RegAllocFast.cpp
@@ -207,9 +207,7 @@ class RegAllocFastImpl {
 
     explicit LiveReg(Register VirtReg) : VirtReg(VirtReg) {}
 
-    unsigned getSparseSetIndex() const {
-      return Register::virtReg2Index(VirtReg);
-    }
+    unsigned getSparseSetIndex() const { return VirtReg.virtRegIndex(); }
   };
 
   using LiveRegMap = SparseSet<LiveReg, identity<unsigned>, uint16_t>;
@@ -349,11 +347,11 @@ class RegAllocFastImpl {
   unsigned calcSpillCost(MCPhysReg PhysReg) const;
 
   LiveRegMap::iterator findLiveVirtReg(Register VirtReg) {
-    return LiveVirtRegs.find(Register::virtReg2Index(VirtReg));
+    return LiveVirtRegs.find(VirtReg.virtRegIndex());
   }
 
   LiveRegMap::const_iterator findLiveVirtReg(Register VirtReg) const {
-    return LiveVirtRegs.find(Register::virtReg2Index(VirtReg));
+    return LiveVirtRegs.find(VirtReg.virtRegIndex());
   }
 
   void assignVirtToPhysReg(MachineInstr &MI, LiveReg &, MCPhysReg PhysReg);
@@ -493,7 +491,7 @@ static bool dominates(InstrPosIndexes &PosIndexes, const MachineInstr &A,
 
 /// Returns false if \p VirtReg is known to not live out of the current block.
 bool RegAllocFastImpl::mayLiveOut(Register VirtReg) {
-  if (MayLiveAcrossBlocks.test(Register::virtReg2Index(VirtReg))) {
+  if (MayLiveAcrossBlocks.test(VirtReg.virtRegIndex())) {
     // Cannot be live-out if there are no successors.
     return !MBB->succ_empty();
   }
@@ -506,7 +504,7 @@ bool RegAllocFastImpl::mayLiveOut(Register VirtReg) {
     // Find the first def in the self loop MBB.
     for (const MachineInstr &DefInst : MRI->def_instructions(VirtReg)) {
       if (DefInst.getParent() != MBB) {
-        MayLiveAcrossBlocks.set(Register::virtReg2Index(VirtReg));
+        MayLiveAcrossBlocks.set(VirtReg.virtRegIndex());
         return true;
       } else {
         if (!SelfLoopDef || dominates(PosIndexes, DefInst, *SelfLoopDef))
@@ -514,7 +512,7 @@ bool RegAllocFastImpl::mayLiveOut(Register VirtReg) {
       }
     }
     if (!SelfLoopDef) {
-      MayLiveAcrossBlocks.set(Register::virtReg2Index(VirtReg));
+      MayLiveAcrossBlocks.set(VirtReg.virtRegIndex());
       return true;
     }
   }
@@ -525,7 +523,7 @@ bool RegAllocFastImpl::mayLiveOut(Register VirtReg) {
   unsigned C = 0;
   for (const MachineInstr &UseInst : MRI->use_nodbg_instructions(VirtReg)) {
     if (UseInst.getParent() != MBB || ++C >= Limit) {
-      MayLiveAcrossBlocks.set(Register::virtReg2Index(VirtReg));
+      MayLiveAcrossBlocks.set(VirtReg.virtRegIndex());
       // Cannot be live-out if there are no successors.
       return !MBB->succ_empty();
     }
@@ -535,7 +533,7 @@ bool RegAllocFastImpl::mayLiveOut(Register VirtReg) {
       // value inside a self looping block.
       if (SelfLoopDef == &UseInst ||
           !dominates(PosIndexes, *SelfLoopDef, UseInst)) {
-        MayLiveAcrossBlocks.set(Register::virtReg2Index(VirtReg));
+        MayLiveAcrossBlocks.set(VirtReg.virtRegIndex());
         return true;
       }
     }
@@ -546,7 +544,7 @@ bool RegAllocFastImpl::mayLiveOut(Register VirtReg) {
 
 /// Returns false if \p VirtReg is known to not be live into the current block.
 bool RegAllocFastImpl::mayLiveIn(Register VirtReg) {
-  if (MayLiveAcrossBlocks.test(Register::virtReg2Index(VirtReg)))
+  if (MayLiveAcrossBlocks.test(VirtReg.virtRegIndex()))
     return !MBB->pred_empty();
 
   // See if the first \p Limit def of the register are all in the current block.
@@ -554,7 +552,7 @@ bool RegAllocFastImpl::mayLiveIn(Register VirtReg) {
   unsigned C = 0;
   for (const MachineInstr &DefInst : MRI->def_instructions(VirtReg)) {
     if (DefInst.getParent() != MBB || ++C >= Limit) {
-      MayLiveAcrossBlocks.set(Register::virtReg2Index(VirtReg));
+      MayLiveAcrossBlocks.set(VirtReg.virtRegIndex());
       return !MBB->pred_empty();
     }
   }
diff --git a/llvm/lib/CodeGen/RegAllocGreedy.cpp b/llvm/lib/CodeGen/RegAllocGreedy.cpp
index 6077cfd514de2..465c4e8feffbb 100644
--- a/llvm/lib/CodeGen/RegAllocGreedy.cpp
+++ b/llvm/lib/CodeGen/RegAllocGreedy.cpp
@@ -154,7 +154,7 @@ INITIALIZE_PASS_BEGIN(RAGreedy, "greedy",
 INITIALIZE_PASS_DEPENDENCY(LiveDebugVariablesWrapperLegacy)
 INITIALIZE_PASS_DEPENDENCY(SlotIndexesWrapperPass)
 INITIALIZE_PASS_DEPENDENCY(LiveIntervalsWrapperPass)
-INITIALIZE_PASS_DEPENDENCY(RegisterCoalescer)
+INITIALIZE_PASS_DEPENDENCY(RegisterCoalescerLegacy)
 INITIALIZE_PASS_DEPENDENCY(MachineScheduler)
 INITIALIZE_PASS_DEPENDENCY(LiveStacksWrapperLegacy)
 INITIALIZE_PASS_DEPENDENCY(MachineDominatorTreeWrapperPass)
diff --git a/llvm/lib/CodeGen/RegisterCoalescer.cpp b/llvm/lib/CodeGen/RegisterCoalescer.cpp
index 8313927dd2aa1..f0b597e21f6fd 100644
--- a/llvm/lib/CodeGen/RegisterCoalescer.cpp
+++ b/llvm/lib/CodeGen/RegisterCoalescer.cpp
@@ -20,20 +20,22 @@
 #include "llvm/ADT/SmallPtrSet.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/Statistic.h"
-#include "llvm/Analysis/AliasAnalysis.h"
 #include "llvm/CodeGen/LiveInterval.h"
 #include "llvm/CodeGen/LiveIntervals.h"
 #include "llvm/CodeGen/LiveRangeEdit.h"
 #include "llvm/CodeGen/MachineBasicBlock.h"
+#include "llvm/CodeGen/MachineDominators.h"
 #include "llvm/CodeGen/MachineFunction.h"
 #include "llvm/CodeGen/MachineFunctionPass.h"
 #include "llvm/CodeGen/MachineInstr.h"
 #include "llvm/CodeGen/MachineInstrBuilder.h"
 #include "llvm/CodeGen/MachineLoopInfo.h"
 #include "llvm/CodeGen/MachineOperand.h"
+#include "llvm/CodeGen/MachinePassManager.h"
 #include "llvm/CodeGen/MachineRegisterInfo.h"
 #include "llvm/CodeGen/Passes.h"
 #include "llvm/CodeGen/RegisterClassInfo.h"
+#include "llvm/CodeGen/RegisterCoalescerPass.h"
 #include "llvm/CodeGen/SlotIndexes.h"
 #include "llvm/CodeGen/TargetInstrInfo.h"
 #include "llvm/CodeGen/TargetOpcodes.h"
@@ -62,15 +64,15 @@ using namespace llvm;
 
 #define DEBUG_TYPE "regalloc"
 
-STATISTIC(numJoins    , "Number of interval joins performed");
-STATISTIC(numCrossRCs , "Number of cross class joins performed");
-STATISTIC(numCommutes , "Number of instruction commuting performed");
-STATISTIC(numExtends  , "Number of copies extended");
-STATISTIC(NumReMats   , "Number of instructions re-materialized");
-STATISTIC(NumInflated , "Number of register classes inflated");
+STATISTIC(numJoins, "Number of interval joins performed");
+STATISTIC(numCrossRCs, "Number of cross class joins performed");
+STATISTIC(numCommutes, "Number of instruction commuting performed");
+STATISTIC(numExtends, "Number of copies extended");
+STATISTIC(NumReMats, "Number of instructions re-materialized");
+STATISTIC(NumInflated, "Number of register classes inflated");
 STATISTIC(NumLaneConflicts, "Number of dead lane conflicts tested");
-STATISTIC(NumLaneResolves,  "Number of dead lane conflicts resolved");
-STATISTIC(NumShrinkToUses,  "Number of shrinkToUses called");
+STATISTIC(NumLaneResolves, "Number of dead lane conflicts resolved");
+STATISTIC(NumShrinkToUses, "Number of shrinkToUses called");
 
 static cl::opt<bool> EnableJoining("join-liveintervals",
                                    cl::desc("Coalesce copies (default=true)"),
@@ -81,20 +83,20 @@ static cl::opt<bool> UseTerminalRule("terminal-rule",
                                      cl::init(false), cl::Hidden);
 
 /// Temporary flag to test critical edge unsplitting.
-static cl::opt<bool>
-EnableJoinSplits("join-splitedges",
-  cl::desc("Coalesce copies on split edges (default=subtarget)"), cl::Hidden);
+static cl::opt<bool> EnableJoinSplits(
+    "join-splitedges",
+    cl::desc("Coalesce copies on split edges (default=subtarget)"), cl::Hidden);
 
 /// Temporary flag to test global copy optimization.
-static cl::opt<cl::boolOrDefault>
-EnableGlobalCopies("join-globalcopies",
-  cl::desc("Coalesce copies that span blocks (default=subtarget)"),
-  cl::init(cl::BOU_UNSET), cl::Hidden);
+static cl::opt<cl::boolOrDefault> EnableGlobalCopies(
+    "join-globalcopies",
+    cl::desc("Coalesce copies that span blocks (default=subtarget)"),
+    cl::init(cl::BOU_UNSET), cl::Hidden);
 
-static cl::opt<bool>
-VerifyCoalescing("verify-coalescing",
-         cl::desc("Verify machine instrs before and after register coalescing"),
-         cl::Hidden);
+static cl::opt<bool> VerifyCoalescing(
+    "verify-coalescing",
+    cl::desc("Verify machine instrs before and after register coalescing"),
+    cl::Hidden);
 
 static cl::opt<unsigned> LateRematUpdateThreshold(
     "late-remat-update-threshold", cl::Hidden,
@@ -120,317 +122,322 @@ static cl::opt<unsigned> LargeIntervalFreqThreshold(
 
 namespace {
 
-  class JoinVals;
-
-  class RegisterCoalescer : public MachineFunctionPass,
-                            private LiveRangeEdit::Delegate {
-    MachineFunction* MF = nullptr;
-    MachineRegisterInfo* MRI = nullptr;
-    const TargetRegisterInfo* TRI = nullptr;
-    const TargetInstrInfo* TII = nullptr;
-    LiveIntervals *LIS = nullptr;
-    const MachineLoopInfo* Loops = nullptr;
-    AliasAnalysis *AA = nullptr;
-    RegisterClassInfo RegClassInfo;
-
-    /// Position and VReg of a PHI instruction during coalescing.
-    struct PHIValPos {
-      SlotIndex SI;    ///< Slot where this PHI occurs.
-      Register Reg;    ///< VReg the PHI occurs in.
-      unsigned SubReg; ///< Qualifying subregister for Reg.
-    };
-
-    /// Map from debug instruction number to PHI position during coalescing.
-    DenseMap<unsigned, PHIValPos> PHIValToPos;
-    /// Index of, for each VReg, which debug instruction numbers and
-    /// corresponding PHIs are sensitive to coalescing. Each VReg may have
-    /// multiple PHI defs, at different positions.
-    DenseMap<Register, SmallVector<unsigned, 2>> RegToPHIIdx;
-
-    /// Debug variable location tracking -- for each VReg, maintain an
-    /// ordered-by-slot-index set of DBG_VALUEs, to help quick
-    /// identification of whether coalescing may change location validity.
-    using DbgValueLoc = std::pair<SlotIndex, MachineInstr*>;
-    DenseMap<Register, std::vector<DbgValueLoc>> DbgVRegToValues;
-
-    /// A LaneMask to remember on which subregister live ranges we need to call
-    /// shrinkToUses() later.
-    LaneBitmask ShrinkMask;
-
-    /// True if the main range of the currently coalesced intervals should be
-    /// checked for smaller live intervals.
-    bool ShrinkMainRange = false;
-
-    /// True if the coalescer should aggressively coalesce global copies
-    /// in favor of keeping local copies.
-    bool JoinGlobalCopies = false;
-
-    /// True if the coalescer should aggressively coalesce fall-thru
-    /// blocks exclusively containing copies.
-    bool JoinSplitEdges = false;
-
-    /// Copy instructions yet to be coalesced.
-    SmallVector<MachineInstr*, 8> WorkList;
-    SmallVector<MachineInstr*, 8> LocalWorkList;
-
-    /// Set of instruction pointers that have been erased, and
-    /// that may be present in WorkList.
-    SmallPtrSet<MachineInstr*, 8> ErasedInstrs;
-
-    /// Dead instructions that are about to be deleted.
-    SmallVector<MachineInstr*, 8> DeadDefs;
-
-    /// Virtual registers to be considered for register class inflation.
-    SmallVector<Register, 8> InflateRegs;
-
-    /// The collection of live intervals which should have been updated
-    /// immediately after rematerialiation but delayed until
-    /// lateLiveIntervalUpdate is called.
-    DenseSet<Register> ToBeUpdated;
-
-    /// Record how many times the large live interval with many valnos
-    /// has been tried to join with other live interval.
-    DenseMap<Register, unsigned long> LargeLIVisitCounter;
-
-    /// Recursively eliminate dead defs in DeadDefs.
-    void eliminateDeadDefs(LiveRangeEdit *Edit = nullptr);
-
-    /// LiveRangeEdit callback for eliminateDeadDefs().
-    void LRE_WillEraseInstruction(MachineInstr *MI) override;
-
-    /// Coalesce the LocalWorkList.
-    void coalesceLocals();
-
-    /// Join compatible live intervals
-    void joinAllIntervals();
-
-    /// Coalesce copies in the specified MBB, putting
-    /// copies that cannot yet be coalesced into WorkList.
-    void copyCoalesceInMBB(MachineBasicBlock *MBB);
-
-    /// Tries to coalesce all copies in CurrList. Returns true if any progress
-    /// was made.
-    bool copyCoalesceWorkList(MutableArrayRef<MachineInstr*> CurrList);
-
-    /// If one def has many copy like uses, and those copy uses are all
-    /// rematerialized, the live interval update needed for those
-    /// rematerializations will be delayed and done all at once instead
-    /// of being done multiple times. This is to save compile cost because
-    /// live interval update is costly.
-    void lateLiveIntervalUpdate();
-
-    /// Check if the incoming value defined by a COPY at \p SLRQ in the subrange
-    /// has no value defined in the predecessors. If the incoming value is the
-    /// same as defined by the copy itself, the value is considered undefined.
-    bool copyValueUndefInPredecessors(LiveRange &S,
-                                      const MachineBasicBlock *MBB,
-                                      LiveQueryResult SLRQ);
-
-    /// Set necessary undef flags on subregister uses after pruning out undef
-    /// lane segments from the subrange.
-    void setUndefOnPrunedSubRegUses(LiveInterval &LI, Register Reg,
-                                    LaneBitmask PrunedLanes);
-
-    /// Attempt to join intervals corresponding to SrcReg/DstReg, which are the
-    /// src/dst of the copy instruction CopyMI.  This returns true if the copy
-    /// was successfully coalesced away. If it is not currently possible to
-    /// coalesce this interval, but it may be possible if other things get
-    /// coalesced, then it returns true by reference in 'Again'.
-    bool joinCopy(MachineInstr *CopyMI, bool &Again,
-                  SmallPtrSetImpl<MachineInstr *> &CurrentErasedInstrs);
-
-    /// Attempt to join these two intervals.  On failure, this
-    /// returns false.  The output "SrcInt" will not have been modified, so we
-    /// can use this information below to update aliases.
-    bool joinIntervals(CoalescerPair &CP);
-
-    /// Attempt joining two virtual registers. Return true on success.
-    bool joinVirtRegs(CoalescerPair &CP);
-
-    /// If a live interval has many valnos and is coalesced with other
-    /// live intervals many times, we regard such live interval as having
-    /// high compile time cost.
-    bool isHighCostLiveInterval(LiveInterval &LI);
-
-    /// Attempt joining with a reserved physreg.
-    bool joinReservedPhysReg(CoalescerPair &CP);
-
-    /// Add the LiveRange @p ToMerge as a subregister liverange of @p LI.
-    /// Subranges in @p LI which only partially interfere with the desired
-    /// LaneMask are split as necessary. @p LaneMask are the lanes that
-    /// @p ToMerge will occupy in the coalescer register. @p LI has its subrange
-    /// lanemasks already adjusted to the coalesced register.
-    void mergeSubRangeInto(LiveInterval &LI, const LiveRange &ToMerge,
-                           LaneBitmask LaneMask, CoalescerPair &CP,
-                           unsigned DstIdx);
-
-    /// Join the liveranges of two subregisters. Joins @p RRange into
-    /// @p LRange, @p RRange may be invalid afterwards.
-    void joinSubRegRanges(LiveRange &LRange, LiveRange &RRange,
-                          LaneBitmask LaneMask, const CoalescerPair &CP);
-
-    /// We found a non-trivially-coalescable copy. If the source value number is
-    /// defined by a copy from the destination reg see if we can merge these two
-    /// destination reg valno# into a single value number, eliminating a copy.
-    /// This returns true if an interval was modified.
-    bool adjustCopiesBackFrom(const CoalescerPair &CP, MachineInstr *CopyMI);
-
-    /// Return true if there are definitions of IntB
-    /// other than BValNo val# that can reach uses of AValno val# of IntA.
-    bool hasOtherReachingDefs(LiveInterval &IntA, LiveInterval &IntB,
-                              VNInfo *AValNo, VNInfo *BValNo);
-
-    /// We found a non-trivially-coalescable copy.
-    /// If the source value number is defined by a commutable instruction and
-    /// its other operand is coalesced to the copy dest register, see if we
-    /// can transform the copy into a noop by commuting the definition.
-    /// This returns a pair of two flags:
-    /// - the first element is true if an interval was modified,
-    /// - the second element is true if the destination interval needs
-    ///   to be shrunk after deleting the copy.
-    std::pair<bool,bool> removeCopyByCommutingDef(const CoalescerPair &CP,
-                                                  MachineInstr *CopyMI);
-
-    /// We found a copy which can be moved to its less frequent predecessor.
-    bool removePartialRedundancy(const CoalescerPair &CP, MachineInstr &CopyMI);
-
-    /// If the source of a copy is defined by a
-    /// trivial computation, replace the copy by rematerialize the definition.
-    bool reMaterializeTrivialDef(const CoalescerPair &CP, MachineInstr *CopyMI,
-                                 bool &IsDefCopy);
-
-    /// Return true if a copy involving a physreg should be joined.
-    bool canJoinPhys(const CoalescerPair &CP);
-
-    /// Replace all defs and uses of SrcReg to DstReg and update the subregister
-    /// number if it is not zero. If DstReg is a physical register and the
-    /// existing subregister number of the def / use being updated is not zero,
-    /// make sure to set it to the correct physical subregister.
-    void updateRegDefsUses(Register SrcReg, Register DstReg, unsigned SubIdx);
-
-    /// If the given machine operand reads only undefined lanes add an undef
-    /// flag.
-    /// This can happen when undef uses were previously concealed by a copy
-    /// which we coalesced. Example:
-    ///    %0:sub0<def,read-undef> = ...
-    ///    %1 = COPY %0           <-- Coalescing COPY reveals undef
-    ///       = use %1:sub1       <-- hidden undef use
-    void addUndefFlag(const LiveInterval &Int, SlotIndex UseIdx,
-                      MachineOperand &MO, unsigned SubRegIdx);
-
-    /// Handle copies of undef values. If the undef value is an incoming
-    /// PHI value, it will convert @p CopyMI to an IMPLICIT_DEF.
-    /// Returns nullptr if @p CopyMI was not in any way eliminable. Otherwise,
-    /// it returns @p CopyMI (which could be an IMPLICIT_DEF at this point).
-    MachineInstr *eliminateUndefCopy(MachineInstr *CopyMI);
-
-    /// Check whether or not we should apply the terminal rule on the
-    /// destination (Dst) of \p Copy.
-    /// When the terminal rule applies, Copy is not profitable to
-    /// coalesce.
-    /// Dst is terminal if it has exactly one affinity (Dst, Src) and
-    /// at least one interference (Dst, Dst2). If Dst is terminal, the
-    /// terminal rule consists in checking that at least one of
-    /// interfering node, say Dst2, has an affinity of equal or greater
-    /// weight with Src.
-    /// In that case, Dst2 and Dst will not be able to be both coalesced
-    /// with Src. Since Dst2 exposes more coalescing opportunities than
-    /// Dst, we can drop \p Copy.
-    bool applyTerminalRule(const MachineInstr &Copy) const;
-
-    /// Wrapper method for \see LiveIntervals::shrinkToUses.
-    /// This method does the proper fixing of the live-ranges when the afore
-    /// mentioned method returns true.
-    void shrinkToUses(LiveInterval *LI,
-                      SmallVectorImpl<MachineInstr * > *Dead = nullptr) {
-      NumShrinkToUses++;
-      if (LIS->shrinkToUses(LI, Dead)) {
-        /// Check whether or not \p LI is composed by multiple connected
-        /// components and if that is the case, fix that.
-        SmallVector<LiveInterval*, 8> SplitLIs;
-        LIS->splitSeparateComponents(*LI, SplitLIs);
-      }
-    }
+class JoinVals;
+
+class RegisterCoalescer : private LiveRangeEdit::Delegate {
+  MachineFunction *MF = nullptr;
+  MachineRegisterInfo *MRI = nullptr;
+  const TargetRegisterInfo *TRI = nullptr;
+  const TargetInstrInfo *TII = nullptr;
+  LiveIntervals *LIS = nullptr;
+  SlotIndexes *SI = nullptr;
+  const MachineLoopInfo *Loops = nullptr;
+  RegisterClassInfo RegClassInfo;
+
+  /// Position and VReg of a PHI instruction during coalescing.
+  struct PHIValPos {
+    SlotIndex SI;    ///< Slot where this PHI occurs.
+    Register Reg;    ///< VReg the PHI occurs in.
+    unsigned SubReg; ///< Qualifying subregister for Reg.
+  };
 
-    /// Wrapper Method to do all the necessary work when an Instruction is
-    /// deleted.
-    /// Optimizations should use this to make sure that deleted instructions
-    /// are always accounted for.
-    void deleteInstr(MachineInstr* MI) {
-      ErasedInstrs.insert(MI);
-      LIS->RemoveMachineInstrFromMaps(*MI);
-      MI->eraseFromParent();
+  /// Map from debug instruction number to PHI position during coalescing.
+  DenseMap<unsigned, PHIValPos> PHIValToPos;
+  /// Index of, for each VReg, which debug instruction numbers and
+  /// corresponding PHIs are sensitive to coalescing. Each VReg may have
+  /// multiple PHI defs, at different positions.
+  DenseMap<Register, SmallVector<unsigned, 2>> RegToPHIIdx;
+
+  /// Debug variable location tracking -- for each VReg, maintain an
+  /// ordered-by-slot-index set of DBG_VALUEs, to help quick
+  /// identification of whether coalescing may change location validity.
+  using DbgValueLoc = std::pair<SlotIndex, MachineInstr *>;
+  DenseMap<Register, std::vector<DbgValueLoc>> DbgVRegToValues;
+
+  /// A LaneMask to remember on which subregister live ranges we need to call
+  /// shrinkToUses() later.
+  LaneBitmask ShrinkMask;
+
+  /// True if the main range of the currently coalesced intervals should be
+  /// checked for smaller live intervals.
+  bool ShrinkMainRange = false;
+
+  /// True if the coalescer should aggressively coalesce global copies
+  /// in favor of keeping local copies.
+  bool JoinGlobalCopies = false;
+
+  /// True if the coalescer should aggressively coalesce fall-thru
+  /// blocks exclusively containing copies.
+  bool JoinSplitEdges = false;
+
+  /// Copy instructions yet to be coalesced.
+  SmallVector<MachineInstr *, 8> WorkList;
+  SmallVector<MachineInstr *, 8> LocalWorkList;
+
+  /// Set of instruction pointers that have been erased, and
+  /// that may be present in WorkList.
+  SmallPtrSet<MachineInstr *, 8> ErasedInstrs;
+
+  /// Dead instructions that are about to be deleted.
+  SmallVector<MachineInstr *, 8> DeadDefs;
+
+  /// Virtual registers to be considered for register class inflation.
+  SmallVector<Register, 8> InflateRegs;
+
+  /// The collection of live intervals which should have been updated
+  /// immediately after rematerialiation but delayed until
+  /// lateLiveIntervalUpdate is called.
+  DenseSet<Register> ToBeUpdated;
+
+  /// Record how many times the large live interval with many valnos
+  /// has been tried to join with other live interval.
+  DenseMap<Register, unsigned long> LargeLIVisitCounter;
+
+  /// Recursively eliminate dead defs in DeadDefs.
+  void eliminateDeadDefs(LiveRangeEdit *Edit = nullptr);
+
+  /// LiveRangeEdit callback for eliminateDeadDefs().
+  void LRE_WillEraseInstruction(MachineInstr *MI) override;
+
+  /// Coalesce the LocalWorkList.
+  void coalesceLocals();
+
+  /// Join compatible live intervals
+  void joinAllIntervals();
+
+  /// Coalesce copies in the specified MBB, putting
+  /// copies that cannot yet be coalesced into WorkList.
+  void copyCoalesceInMBB(MachineBasicBlock *MBB);
+
+  /// Tries to coalesce all copies in CurrList. Returns true if any progress
+  /// was made.
+  bool copyCoalesceWorkList(MutableArrayRef<MachineInstr *> CurrList);
+
+  /// If one def has many copy like uses, and those copy uses are all
+  /// rematerialized, the live interval update needed for those
+  /// rematerializations will be delayed and done all at once instead
+  /// of being done multiple times. This is to save compile cost because
+  /// live interval update is costly.
+  void lateLiveIntervalUpdate();
+
+  /// Check if the incoming value defined by a COPY at \p SLRQ in the subrange
+  /// has no value defined in the predecessors. If the incoming value is the
+  /// same as defined by the copy itself, the value is considered undefined.
+  bool copyValueUndefInPredecessors(LiveRange &S, const MachineBasicBlock *MBB,
+                                    LiveQueryResult SLRQ);
+
+  /// Set necessary undef flags on subregister uses after pruning out undef
+  /// lane segments from the subrange.
+  void setUndefOnPrunedSubRegUses(LiveInterval &LI, Register Reg,
+                                  LaneBitmask PrunedLanes);
+
+  /// Attempt to join intervals corresponding to SrcReg/DstReg, which are the
+  /// src/dst of the copy instruction CopyMI.  This returns true if the copy
+  /// was successfully coalesced away. If it is not currently possible to
+  /// coalesce this interval, but it may be possible if other things get
+  /// coalesced, then it returns true by reference in 'Again'.
+  bool joinCopy(MachineInstr *CopyMI, bool &Again,
+                SmallPtrSetImpl<MachineInstr *> &CurrentErasedInstrs);
+
+  /// Attempt to join these two intervals.  On failure, this
+  /// returns false.  The output "SrcInt" will not have been modified, so we
+  /// can use this information below to update aliases.
+  bool joinIntervals(CoalescerPair &CP);
+
+  /// Attempt joining two virtual registers. Return true on success.
+  bool joinVirtRegs(CoalescerPair &CP);
+
+  /// If a live interval has many valnos and is coalesced with other
+  /// live intervals many times, we regard such live interval as having
+  /// high compile time cost.
+  bool isHighCostLiveInterval(LiveInterval &LI);
+
+  /// Attempt joining with a reserved physreg.
+  bool joinReservedPhysReg(CoalescerPair &CP);
+
+  /// Add the LiveRange @p ToMerge as a subregister liverange of @p LI.
+  /// Subranges in @p LI which only partially interfere with the desired
+  /// LaneMask are split as necessary. @p LaneMask are the lanes that
+  /// @p ToMerge will occupy in the coalescer register. @p LI has its subrange
+  /// lanemasks already adjusted to the coalesced register.
+  void mergeSubRangeInto(LiveInterval &LI, const LiveRange &ToMerge,
+                         LaneBitmask LaneMask, CoalescerPair &CP,
+                         unsigned DstIdx);
+
+  /// Join the liveranges of two subregisters. Joins @p RRange into
+  /// @p LRange, @p RRange may be invalid afterwards.
+  void joinSubRegRanges(LiveRange &LRange, LiveRange &RRange,
+                        LaneBitmask LaneMask, const CoalescerPair &CP);
+
+  /// We found a non-trivially-coalescable copy. If the source value number is
+  /// defined by a copy from the destination reg see if we can merge these two
+  /// destination reg valno# into a single value number, eliminating a copy.
+  /// This returns true if an interval was modified.
+  bool adjustCopiesBackFrom(const CoalescerPair &CP, MachineInstr *CopyMI);
+
+  /// Return true if there are definitions of IntB
+  /// other than BValNo val# that can reach uses of AValno val# of IntA.
+  bool hasOtherReachingDefs(LiveInterval &IntA, LiveInterval &IntB,
+                            VNInfo *AValNo, VNInfo *BValNo);
+
+  /// We found a non-trivially-coalescable copy.
+  /// If the source value number is defined by a commutable instruction and
+  /// its other operand is coalesced to the copy dest register, see if we
+  /// can transform the copy into a noop by commuting the definition.
+  /// This returns a pair of two flags:
+  /// - the first element is true if an interval was modified,
+  /// - the second element is true if the destination interval needs
+  ///   to be shrunk after deleting the copy.
+  std::pair<bool, bool> removeCopyByCommutingDef(const CoalescerPair &CP,
+                                                 MachineInstr *CopyMI);
+
+  /// We found a copy which can be moved to its less frequent predecessor.
+  bool removePartialRedundancy(const CoalescerPair &CP, MachineInstr &CopyMI);
+
+  /// If the source of a copy is defined by a
+  /// trivial computation, replace the copy by rematerialize the definition.
+  bool reMaterializeTrivialDef(const CoalescerPair &CP, MachineInstr *CopyMI,
+                               bool &IsDefCopy);
+
+  /// Return true if a copy involving a physreg should be joined.
+  bool canJoinPhys(const CoalescerPair &CP);
+
+  /// Replace all defs and uses of SrcReg to DstReg and update the subregister
+  /// number if it is not zero. If DstReg is a physical register and the
+  /// existing subregister number of the def / use being updated is not zero,
+  /// make sure to set it to the correct physical subregister.
+  void updateRegDefsUses(Register SrcReg, Register DstReg, unsigned SubIdx);
+
+  /// If the given machine operand reads only undefined lanes add an undef
+  /// flag.
+  /// This can happen when undef uses were previously concealed by a copy
+  /// which we coalesced. Example:
+  ///    %0:sub0<def,read-undef> = ...
+  ///    %1 = COPY %0           <-- Coalescing COPY reveals undef
+  ///       = use %1:sub1       <-- hidden undef use
+  void addUndefFlag(const LiveInterval &Int, SlotIndex UseIdx,
+                    MachineOperand &MO, unsigned SubRegIdx);
+
+  /// Handle copies of undef values. If the undef value is an incoming
+  /// PHI value, it will convert @p CopyMI to an IMPLICIT_DEF.
+  /// Returns nullptr if @p CopyMI was not in any way eliminable. Otherwise,
+  /// it returns @p CopyMI (which could be an IMPLICIT_DEF at this point).
+  MachineInstr *eliminateUndefCopy(MachineInstr *CopyMI);
+
+  /// Check whether or not we should apply the terminal rule on the
+  /// destination (Dst) of \p Copy.
+  /// When the terminal rule applies, Copy is not profitable to
+  /// coalesce.
+  /// Dst is terminal if it has exactly one affinity (Dst, Src) and
+  /// at least one interference (Dst, Dst2). If Dst is terminal, the
+  /// terminal rule consists in checking that at least one of
+  /// interfering node, say Dst2, has an affinity of equal or greater
+  /// weight with Src.
+  /// In that case, Dst2 and Dst will not be able to be both coalesced
+  /// with Src. Since Dst2 exposes more coalescing opportunities than
+  /// Dst, we can drop \p Copy.
+  bool applyTerminalRule(const MachineInstr &Copy) const;
+
+  /// Wrapper method for \see LiveIntervals::shrinkToUses.
+  /// This method does the proper fixing of the live-ranges when the afore
+  /// mentioned method returns true.
+  void shrinkToUses(LiveInterval *LI,
+                    SmallVectorImpl<MachineInstr *> *Dead = nullptr) {
+    NumShrinkToUses++;
+    if (LIS->shrinkToUses(LI, Dead)) {
+      /// Check whether or not \p LI is composed by multiple connected
+      /// components and if that is the case, fix that.
+      SmallVector<LiveInterval *, 8> SplitLIs;
+      LIS->splitSeparateComponents(*LI, SplitLIs);
     }
+  }
 
-    /// Walk over function and initialize the DbgVRegToValues map.
-    void buildVRegToDbgValueMap(MachineFunction &MF);
+  /// Wrapper Method to do all the necessary work when an Instruction is
+  /// deleted.
+  /// Optimizations should use this to make sure that deleted instructions
+  /// are always accounted for.
+  void deleteInstr(MachineInstr *MI) {
+    ErasedInstrs.insert(MI);
+    LIS->RemoveMachineInstrFromMaps(*MI);
+    MI->eraseFromParent();
+  }
 
-    /// Test whether, after merging, any DBG_VALUEs would refer to a
-    /// different value number than before merging, and whether this can
-    /// be resolved. If not, mark the DBG_VALUE as being undef.
-    void checkMergingChangesDbgValues(CoalescerPair &CP, LiveRange &LHS,
-                                      JoinVals &LHSVals, LiveRange &RHS,
-                                      JoinVals &RHSVals);
+  /// Walk over function and initialize the DbgVRegToValues map.
+  void buildVRegToDbgValueMap(MachineFunction &MF);
 
-    void checkMergingChangesDbgValuesImpl(Register Reg, LiveRange &OtherRange,
-                                          LiveRange &RegRange, JoinVals &Vals2);
+  /// Test whether, after merging, any DBG_VALUEs would refer to a
+  /// different value number than before merging, and whether this can
+  /// be resolved. If not, mark the DBG_VALUE as being undef.
+  void checkMergingChangesDbgValues(CoalescerPair &CP, LiveRange &LHS,
+                                    JoinVals &LHSVals, LiveRange &RHS,
+                                    JoinVals &RHSVals);
 
-  public:
-    static char ID; ///< Class identification, replacement for typeinfo
+  void checkMergingChangesDbgValuesImpl(Register Reg, LiveRange &OtherRange,
+                                        LiveRange &RegRange, JoinVals &Vals2);
 
-    RegisterCoalescer() : MachineFunctionPass(ID) {
-      initializeRegisterCoalescerPass(*PassRegistry::getPassRegistry());
-    }
+public:
+  // For legacy pass only.
+  RegisterCoalescer() {}
+  RegisterCoalescer &operator=(RegisterCoalescer &&Other) = default;
 
-    void getAnalysisUsage(AnalysisUsage &AU) const override;
+  RegisterCoalescer(LiveIntervals *LIS, SlotIndexes *SI,
+                    const MachineLoopInfo *Loops)
+      : LIS(LIS), SI(SI), Loops(Loops) {}
 
-    MachineFunctionProperties getClearedProperties() const override {
-      return MachineFunctionProperties().set(
-          MachineFunctionProperties::Property::IsSSA);
-    }
+  bool run(MachineFunction &MF);
+};
+
+class RegisterCoalescerLegacy : public MachineFunctionPass {
+public:
+  static char ID; ///< Class identification, replacement for typeinfo
 
-    void releaseMemory() override;
+  RegisterCoalescerLegacy() : MachineFunctionPass(ID) {
+    initializeRegisterCoalescerLegacyPass(*PassRegistry::getPassRegistry());
+  }
 
-    /// This is the pass entry point.
-    bool runOnMachineFunction(MachineFunction&) override;
+  void getAnalysisUsage(AnalysisUsage &AU) const override;
 
-    /// Implement the dump method.
-    void print(raw_ostream &O, const Module* = nullptr) const override;
-  };
+  MachineFunctionProperties getClearedProperties() const override {
+    return MachineFunctionProperties().set(
+        MachineFunctionProperties::Property::IsSSA);
+  }
+
+  /// This is the pass entry point.
+  bool runOnMachineFunction(MachineFunction &) override;
+};
 
 } // end anonymous namespace
 
-char RegisterCoalescer::ID = 0;
+char RegisterCoalescerLegacy::ID = 0;
 
-char &llvm::RegisterCoalescerID = RegisterCoalescer::ID;
+char &llvm::RegisterCoalescerID = RegisterCoalescerLegacy::ID;
 
-INITIALIZE_PASS_BEGIN(RegisterCoalescer, "register-coalescer",
+INITIALIZE_PASS_BEGIN(RegisterCoalescerLegacy, "register-coalescer",
                       "Register Coalescer", false, false)
 INITIALIZE_PASS_DEPENDENCY(LiveIntervalsWrapperPass)
 INITIALIZE_PASS_DEPENDENCY(SlotIndexesWrapperPass)
 INITIALIZE_PASS_DEPENDENCY(MachineLoopInfoWrapperPass)
-INITIALIZE_PASS_DEPENDENCY(AAResultsWrapperPass)
-INITIALIZE_PASS_END(RegisterCoalescer, "register-coalescer",
+INITIALIZE_PASS_END(RegisterCoalescerLegacy, "register-coalescer",
                     "Register Coalescer", false, false)
 
 [[nodiscard]] static bool isMoveInstr(const TargetRegisterInfo &tri,
                                       const MachineInstr *MI, Register &Src,
                                       Register &Dst, unsigned &SrcSub,
                                       unsigned &DstSub) {
-    if (MI->isCopy()) {
-      Dst = MI->getOperand(0).getReg();
-      DstSub = MI->getOperand(0).getSubReg();
-      Src = MI->getOperand(1).getReg();
-      SrcSub = MI->getOperand(1).getSubReg();
-    } else if (MI->isSubregToReg()) {
-      Dst = MI->getOperand(0).getReg();
-      DstSub = tri.composeSubRegIndices(MI->getOperand(0).getSubReg(),
-                                        MI->getOperand(3).getImm());
-      Src = MI->getOperand(2).getReg();
-      SrcSub = MI->getOperand(2).getSubReg();
-    } else
-      return false;
-    return true;
+  if (MI->isCopy()) {
+    Dst = MI->getOperand(0).getReg();
+    DstSub = MI->getOperand(0).getSubReg();
+    Src = MI->getOperand(1).getReg();
+    SrcSub = MI->getOperand(1).getSubReg();
+  } else if (MI->isSubregToReg()) {
+    Dst = MI->getOperand(0).getReg();
+    DstSub = tri.composeSubRegIndices(MI->getOperand(0).getSubReg(),
+                                      MI->getOperand(3).getImm());
+    Src = MI->getOperand(2).getReg();
+    SrcSub = MI->getOperand(2).getSubReg();
+  } else
+    return false;
+  return true;
 }
 
 /// Return true if this block should be vacated by the coalescer to eliminate
@@ -476,14 +483,16 @@ bool CoalescerPair::setRegisters(const MachineInstr *MI) {
     // Eliminate DstSub on a physreg.
     if (DstSub) {
       Dst = TRI.getSubReg(Dst, DstSub);
-      if (!Dst) return false;
+      if (!Dst)
+        return false;
       DstSub = 0;
     }
 
     // Eliminate SrcSub by picking a corresponding Dst superregister.
     if (SrcSub) {
       Dst = TRI.getMatchingSuperReg(Dst, SrcSub, MRI.getRegClass(Src));
-      if (!Dst) return false;
+      if (!Dst)
+        return false;
     } else if (!MRI.getRegClass(Src)->contains(Dst)) {
       return false;
     }
@@ -498,8 +507,8 @@ bool CoalescerPair::setRegisters(const MachineInstr *MI) {
       if (Src == Dst && SrcSub != DstSub)
         return false;
 
-      NewRC = TRI.getCommonSuperRegClass(SrcRC, SrcSub, DstRC, DstSub,
-                                         SrcIdx, DstIdx);
+      NewRC = TRI.getCommonSuperRegClass(SrcRC, SrcSub, DstRC, DstSub, SrcIdx,
+                                         DstIdx);
       if (!NewRC)
         return false;
     } else if (DstSub) {
@@ -585,9 +594,9 @@ bool CoalescerPair::isCoalescable(const MachineInstr *MI) const {
   }
 }
 
-void RegisterCoalescer::getAnalysisUsage(AnalysisUsage &AU) const {
+void RegisterCoalescerLegacy::getAnalysisUsage(AnalysisUsage &AU) const {
   AU.setPreservesCFG();
-  AU.addRequired<AAResultsWrapperPass>();
+  AU.addUsedIfAvailable<SlotIndexesWrapperPass>();
   AU.addRequired<LiveIntervalsWrapperPass>();
   AU.addPreserved<LiveIntervalsWrapperPass>();
   AU.addPreserved<SlotIndexesWrapperPass>();
@@ -603,8 +612,8 @@ void RegisterCoalescer::eliminateDeadDefs(LiveRangeEdit *Edit) {
     return;
   }
   SmallVector<Register, 8> NewRegs;
-  LiveRangeEdit(nullptr, NewRegs, *MF, *LIS,
-                nullptr, this).eliminateDeadDefs(DeadDefs);
+  LiveRangeEdit(nullptr, NewRegs, *MF, *LIS, nullptr, this)
+      .eliminateDeadDefs(DeadDefs);
 }
 
 void RegisterCoalescer::LRE_WillEraseInstruction(MachineInstr *MI) {
@@ -618,9 +627,9 @@ bool RegisterCoalescer::adjustCopiesBackFrom(const CoalescerPair &CP,
   assert(!CP.isPhys() && "This doesn't work for physreg copies.");
 
   LiveInterval &IntA =
-    LIS->getInterval(CP.isFlipped() ? CP.getDstReg() : CP.getSrcReg());
+      LIS->getInterval(CP.isFlipped() ? CP.getDstReg() : CP.getSrcReg());
   LiveInterval &IntB =
-    LIS->getInterval(CP.isFlipped() ? CP.getSrcReg() : CP.getDstReg());
+      LIS->getInterval(CP.isFlipped() ? CP.getSrcReg() : CP.getDstReg());
   SlotIndex CopyIdx = LIS->getInstructionIndex(*CopyMI).getRegSlot();
 
   // We have a non-trivially-coalescable copy with IntA being the source and
@@ -640,19 +649,22 @@ bool RegisterCoalescer::adjustCopiesBackFrom(const CoalescerPair &CP,
   // BValNo is a value number in B that is defined by a copy from A.  'B1' in
   // the example above.
   LiveInterval::iterator BS = IntB.FindSegmentContaining(CopyIdx);
-  if (BS == IntB.end()) return false;
+  if (BS == IntB.end())
+    return false;
   VNInfo *BValNo = BS->valno;
 
   // Get the location that B is defined at.  Two options: either this value has
   // an unknown definition point or it is defined at CopyIdx.  If unknown, we
   // can't process it.
-  if (BValNo->def != CopyIdx) return false;
+  if (BValNo->def != CopyIdx)
+    return false;
 
   // AValNo is the value number in A that defines the copy, A3 in the example.
   SlotIndex CopyUseIdx = CopyIdx.getRegSlot(true);
   LiveInterval::iterator AS = IntA.FindSegmentContaining(CopyUseIdx);
   // The live segment might not exist after fun with physreg coalescing.
-  if (AS == IntA.end()) return false;
+  if (AS == IntA.end())
+    return false;
   VNInfo *AValNo = AS->valno;
 
   // If AValNo is defined as a copy from IntB, we can potentially process this.
@@ -664,21 +676,22 @@ bool RegisterCoalescer::adjustCopiesBackFrom(const CoalescerPair &CP,
 
   // Get the Segment in IntB that this value number starts with.
   LiveInterval::iterator ValS =
-    IntB.FindSegmentContaining(AValNo->def.getPrevSlot());
+      IntB.FindSegmentContaining(AValNo->def.getPrevSlot());
   if (ValS == IntB.end())
     return false;
 
   // Make sure that the end of the live segment is inside the same block as
   // CopyMI.
   MachineInstr *ValSEndInst =
-    LIS->getInstructionFromIndex(ValS->end.getPrevSlot());
+      LIS->getInstructionFromIndex(ValS->end.getPrevSlot());
   if (!ValSEndInst || ValSEndInst->getParent() != CopyMI->getParent())
     return false;
 
   // Okay, we now know that ValS ends in the same block that the CopyMI
   // live-range starts.  If there are no intervening live segments between them
   // in IntB, we can merge them.
-  if (ValS+1 != BS) return false;
+  if (ValS + 1 != BS)
+    return false;
 
   LLVM_DEBUG(dbgs() << "Extending: " << printReg(IntB.reg(), TRI));
 
@@ -751,8 +764,7 @@ bool RegisterCoalescer::adjustCopiesBackFrom(const CoalescerPair &CP,
 }
 
 bool RegisterCoalescer::hasOtherReachingDefs(LiveInterval &IntA,
-                                             LiveInterval &IntB,
-                                             VNInfo *AValNo,
+                                             LiveInterval &IntB, VNInfo *AValNo,
                                              VNInfo *BValNo) {
   // If AValNo has PHI kills, conservatively assume that IntB defs can reach
   // the PHI values.
@@ -760,7 +772,8 @@ bool RegisterCoalescer::hasOtherReachingDefs(LiveInterval &IntA,
     return true;
 
   for (LiveRange::Segment &ASeg : IntA.segments) {
-    if (ASeg.valno != AValNo) continue;
+    if (ASeg.valno != AValNo)
+      continue;
     LiveInterval::iterator BI = llvm::upper_bound(IntB, ASeg.start);
     if (BI != IntB.begin())
       --BI;
@@ -778,9 +791,10 @@ bool RegisterCoalescer::hasOtherReachingDefs(LiveInterval &IntA,
 
 /// Copy segments with value number @p SrcValNo from liverange @p Src to live
 /// range @Dst and use value number @p DstValNo there.
-static std::pair<bool,bool>
-addSegmentsWithValNo(LiveRange &Dst, VNInfo *DstValNo, const LiveRange &Src,
-                     const VNInfo *SrcValNo) {
+static std::pair<bool, bool> addSegmentsWithValNo(LiveRange &Dst,
+                                                  VNInfo *DstValNo,
+                                                  const LiveRange &Src,
+                                                  const VNInfo *SrcValNo) {
   bool Changed = false;
   bool MergedWithDead = false;
   for (const LiveRange::Segment &S : Src.segments) {
@@ -801,7 +815,7 @@ addSegmentsWithValNo(LiveRange &Dst, VNInfo *DstValNo, const LiveRange &Src,
   return std::make_pair(Changed, MergedWithDead);
 }
 
-std::pair<bool,bool>
+std::pair<bool, bool>
 RegisterCoalescer::removeCopyByCommutingDef(const CoalescerPair &CP,
                                             MachineInstr *CopyMI) {
   assert(!CP.isPhys());
@@ -841,19 +855,19 @@ RegisterCoalescer::removeCopyByCommutingDef(const CoalescerPair &CP,
   VNInfo *AValNo = IntA.getVNInfoAt(CopyIdx.getRegSlot(true));
   assert(AValNo && !AValNo->isUnused() && "COPY source not live");
   if (AValNo->isPHIDef())
-    return { false, false };
+    return {false, false};
   MachineInstr *DefMI = LIS->getInstructionFromIndex(AValNo->def);
   if (!DefMI)
-    return { false, false };
+    return {false, false};
   if (!DefMI->isCommutable())
-    return { false, false };
+    return {false, false};
   // If DefMI is a two-address instruction then commuting it will change the
   // destination register.
   int DefIdx = DefMI->findRegisterDefOperandIdx(IntA.reg(), /*TRI=*/nullptr);
   assert(DefIdx != -1);
   unsigned UseOpIdx;
   if (!DefMI->isRegTiedToUseOperand(DefIdx, &UseOpIdx))
-    return { false, false };
+    return {false, false};
 
   // FIXME: The code below tries to commute 'UseOpIdx' operand with some other
   // commutable operand which is expressed by 'CommuteAnyOperandIndex'value
@@ -866,17 +880,17 @@ RegisterCoalescer::removeCopyByCommutingDef(const CoalescerPair &CP,
   // op#2<->op#3) of commute transformation should be considered/tried here.
   unsigned NewDstIdx = TargetInstrInfo::CommuteAnyOperandIndex;
   if (!TII->findCommutedOpIndices(*DefMI, UseOpIdx, NewDstIdx))
-    return { false, false };
+    return {false, false};
 
   MachineOperand &NewDstMO = DefMI->getOperand(NewDstIdx);
   Register NewReg = NewDstMO.getReg();
   if (NewReg != IntB.reg() || !IntB.Query(AValNo->def).isKill())
-    return { false, false };
+    return {false, false};
 
   // Make sure there are no other definitions of IntB that would reach the
   // uses which the new definition can reach.
   if (hasOtherReachingDefs(IntA, IntB, AValNo, BValNo))
-    return { false, false };
+    return {false, false};
 
   // If some of the uses of IntA.reg is already coalesced away, return false.
   // It's not possible to determine whether it's safe to perform the coalescing.
@@ -889,7 +903,7 @@ RegisterCoalescer::removeCopyByCommutingDef(const CoalescerPair &CP,
       continue;
     // If this use is tied to a def, we can't rewrite the register.
     if (UseMI->isRegTiedToDefOperand(OpNo))
-      return { false, false };
+      return {false, false};
   }
 
   LLVM_DEBUG(dbgs() << "\tremoveCopyByCommutingDef: " << AValNo->def << '\t'
@@ -901,10 +915,10 @@ RegisterCoalescer::removeCopyByCommutingDef(const CoalescerPair &CP,
   MachineInstr *NewMI =
       TII->commuteInstruction(*DefMI, false, UseOpIdx, NewDstIdx);
   if (!NewMI)
-    return { false, false };
+    return {false, false};
   if (IntA.reg().isVirtual() && IntB.reg().isVirtual() &&
       !MRI->constrainRegClass(IntB.reg(), MRI->getRegClass(IntA.reg())))
-    return { false, false };
+    return {false, false};
   if (NewMI != DefMI) {
     LIS->ReplaceMachineInstrInMaps(*DefMI, *NewMI);
     MachineBasicBlock::iterator Pos = DefMI;
@@ -1035,7 +1049,7 @@ RegisterCoalescer::removeCopyByCommutingDef(const CoalescerPair &CP,
 
   LLVM_DEBUG(dbgs() << "\t\ttrimmed:  " << IntA << '\n');
   ++numCommutes;
-  return { true, ShrinkB };
+  return {true, ShrinkB};
 }
 
 /// For copy B = A in BB2, if A is defined by A = B in BB0 which is a
@@ -1194,9 +1208,9 @@ bool RegisterCoalescer::removePartialRedundancy(const CoalescerPair &CP,
     for (LiveInterval::SubRange &SR : IntB.subranges())
       SR.createDeadDef(NewCopyIdx, LIS->getVNInfoAllocator());
 
-    // If the newly created Instruction has an address of an instruction that was
-    // deleted before (object recycled by the allocator) it needs to be removed from
-    // the deleted list.
+    // If the newly created Instruction has an address of an instruction that
+    // was deleted before (object recycled by the allocator) it needs to be
+    // removed from the deleted list.
     ErasedInstrs.erase(NewCopyMI);
   } else {
     LLVM_DEBUG(dbgs() << "\tremovePartialRedundancy: Remove the copy from "
@@ -1247,7 +1261,7 @@ bool RegisterCoalescer::removePartialRedundancy(const CoalescerPair &CP,
     // to because the copy has been removed.  We can go ahead and remove that
     // endpoint; there is no other situation here that there could be a use at
     // the same place as we know that the copy is a full copy.
-    for (unsigned I = 0; I != EndPoints.size(); ) {
+    for (unsigned I = 0; I != EndPoints.size();) {
       if (SlotIndex::isSameInstr(EndPoints[I], CopyIdx)) {
         EndPoints[I] = EndPoints.back();
         EndPoints.pop_back();
@@ -1390,7 +1404,7 @@ bool RegisterCoalescer::reMaterializeTrivialDef(const CoalescerPair &CP,
   DebugLoc DL = CopyMI->getDebugLoc();
   MachineBasicBlock *MBB = CopyMI->getParent();
   MachineBasicBlock::iterator MII =
-    std::next(MachineBasicBlock::iterator(CopyMI));
+      std::next(MachineBasicBlock::iterator(CopyMI));
   Edit.rematerializeAt(*MBB, MII, DstReg, RM, *TRI, false, SrcIdx, CopyMI);
   MachineInstr &NewMI = *std::prev(MII);
   NewMI.setDebugLoc(DL);
@@ -1404,11 +1418,11 @@ bool RegisterCoalescer::reMaterializeTrivialDef(const CoalescerPair &CP,
   if (DstIdx != 0) {
     MachineOperand &DefMO = NewMI.getOperand(0);
     if (DefMO.getSubReg() == DstIdx) {
-      assert(SrcIdx == 0 && CP.isFlipped()
-             && "Shouldn't have SrcIdx+DstIdx at this point");
+      assert(SrcIdx == 0 && CP.isFlipped() &&
+             "Shouldn't have SrcIdx+DstIdx at this point");
       const TargetRegisterClass *DstRC = MRI->getRegClass(DstReg);
       const TargetRegisterClass *CommonRC =
-        TRI->getCommonSubClass(DefRC, DstRC);
+          TRI->getCommonSubClass(DefRC, DstRC);
       if (CommonRC != nullptr) {
         NewRC = CommonRC;
 
@@ -1438,7 +1452,8 @@ bool RegisterCoalescer::reMaterializeTrivialDef(const CoalescerPair &CP,
        I != E; ++I) {
     MachineOperand &MO = CopyMI->getOperand(I);
     if (MO.isReg()) {
-      assert(MO.isImplicit() && "No explicit operands after implicit operands.");
+      assert(MO.isImplicit() &&
+             "No explicit operands after implicit operands.");
       assert((MO.getReg().isPhysical() ||
               (MO.getSubReg() == 0 && MO.getReg() == DstOperand.getReg())) &&
              "unexpected implicit virtual register def");
@@ -1563,7 +1578,7 @@ bool RegisterCoalescer::reMaterializeTrivialDef(const CoalescerPair &CP,
       SlotIndex DefIndex =
           CurrIdx.getRegSlot(NewMI.getOperand(0).isEarlyClobber());
       LaneBitmask MaxMask = MRI->getMaxLaneMaskForVReg(DstReg);
-      VNInfo::Allocator& Alloc = LIS->getVNInfoAllocator();
+      VNInfo::Allocator &Alloc = LIS->getVNInfoAllocator();
       for (LiveInterval::SubRange &SR : DstInt.subranges()) {
         if (!SR.liveAt(DefIndex))
           SR.createDeadDef(DefIndex, Alloc);
@@ -1723,7 +1738,7 @@ MachineInstr *RegisterCoalescer::eliminateUndefCopy(MachineInstr *CopyMI) {
   // at this point.
   Register SrcReg, DstReg;
   unsigned SrcSubIdx = 0, DstSubIdx = 0;
-  if(!isMoveInstr(*TRI, CopyMI, SrcReg, DstReg, SrcSubIdx, DstSubIdx))
+  if (!isMoveInstr(*TRI, CopyMI, SrcReg, DstReg, SrcSubIdx, DstSubIdx))
     return nullptr;
 
   SlotIndex Idx = LIS->getInstructionIndex(*CopyMI);
@@ -1752,20 +1767,20 @@ MachineInstr *RegisterCoalescer::eliminateUndefCopy(MachineInstr *CopyMI) {
   // copy introduced a live value.
   if (((V && V->isPHIDef()) || (!V && !DstLI.liveAt(Idx)))) {
     for (unsigned i = CopyMI->getNumOperands(); i != 0; --i) {
-      MachineOperand &MO = CopyMI->getOperand(i-1);
+      MachineOperand &MO = CopyMI->getOperand(i - 1);
       if (MO.isReg()) {
         if (MO.isUse())
           CopyMI->removeOperand(i - 1);
       } else {
         assert(MO.isImm() &&
                CopyMI->getOpcode() == TargetOpcode::SUBREG_TO_REG);
-        CopyMI->removeOperand(i-1);
+        CopyMI->removeOperand(i - 1);
       }
     }
 
     CopyMI->setDesc(TII->get(TargetOpcode::IMPLICIT_DEF));
     LLVM_DEBUG(dbgs() << "\tReplaced copy of <undef> value with an "
-               "implicit def\n");
+                         "implicit def\n");
     return CopyMI;
   }
 
@@ -1877,10 +1892,10 @@ void RegisterCoalescer::updateRegDefsUses(Register SrcReg, Register DstReg,
     }
   }
 
-  SmallPtrSet<MachineInstr*, 8> Visited;
-  for (MachineRegisterInfo::reg_instr_iterator
-       I = MRI->reg_instr_begin(SrcReg), E = MRI->reg_instr_end();
-       I != E; ) {
+  SmallPtrSet<MachineInstr *, 8> Visited;
+  for (MachineRegisterInfo::reg_instr_iterator I = MRI->reg_instr_begin(SrcReg),
+                                               E = MRI->reg_instr_end();
+       I != E;) {
     MachineInstr *UseMI = &*(I++);
 
     // Each instruction can only be rewritten once because sub-register
@@ -1891,7 +1906,7 @@ void RegisterCoalescer::updateRegDefsUses(Register SrcReg, Register DstReg,
     if (SrcReg == DstReg && !Visited.insert(UseMI).second)
       continue;
 
-    SmallVector<unsigned,8> Ops;
+    SmallVector<unsigned, 8> Ops;
     bool Reads, Writes;
     std::tie(Reads, Writes) = UseMI->readsWritesVirtualRegister(SrcReg, &Ops);
 
@@ -1928,8 +1943,8 @@ void RegisterCoalescer::updateRegDefsUses(Register SrcReg, Register DstReg,
             DstInt->createSubRange(Allocator, UnusedLanes);
           }
           SlotIndex MIIdx = UseMI->isDebugInstr()
-            ? LIS->getSlotIndexes()->getIndexBefore(*UseMI)
-            : LIS->getInstructionIndex(*UseMI);
+                                ? LIS->getSlotIndexes()->getIndexBefore(*UseMI)
+                                : LIS->getInstructionIndex(*UseMI);
           SlotIndex UseIdx = MIIdx.getRegSlot(true);
           addUndefFlag(*DstInt, UseIdx, MO, SubUseIdx);
         }
@@ -2056,7 +2071,7 @@ bool RegisterCoalescer::joinCopy(
       if (UndefMI->isImplicitDef())
         return false;
       deleteInstr(CopyMI);
-      return false;  // Not coalescable.
+      return false; // Not coalescable.
     }
   }
 
@@ -2096,8 +2111,8 @@ bool RegisterCoalescer::joinCopy(
 
       LI.MergeValueNumberInto(DefVNI, ReadVNI);
       if (PrunedLanes.any()) {
-        LLVM_DEBUG(dbgs() << "Pruning undef incoming lanes: "
-                          << PrunedLanes << '\n');
+        LLVM_DEBUG(dbgs() << "Pruning undef incoming lanes: " << PrunedLanes
+                          << '\n');
         setUndefOnPrunedSubRegUses(LI, CP.getSrcReg(), PrunedLanes);
       }
 
@@ -2119,13 +2134,13 @@ bool RegisterCoalescer::joinCopy(
       if (reMaterializeTrivialDef(CP, CopyMI, IsDefCopy))
         return true;
       if (IsDefCopy)
-        Again = true;  // May be possible to coalesce later.
+        Again = true; // May be possible to coalesce later.
       return false;
     }
   } else {
     // When possible, let DstReg be the larger interval.
     if (!CP.isPartial() && LIS->getInterval(CP.getSrcReg()).size() >
-                           LIS->getInterval(CP.getDstReg()).size())
+                               LIS->getInterval(CP.getDstReg()).size())
       CP.flip();
 
     LLVM_DEBUG({
@@ -2186,7 +2201,7 @@ bool RegisterCoalescer::joinCopy(
 
     // Otherwise, we are unable to join the intervals.
     LLVM_DEBUG(dbgs() << "\tInterference!\n");
-    Again = true;  // May be possible to coalesce later.
+    Again = true; // May be possible to coalesce later.
     return false;
   }
 
@@ -2472,7 +2487,7 @@ class JoinVals {
   const bool TrackSubRegLiveness;
 
   /// Values that will be present in the final live range.
-  SmallVectorImpl<VNInfo*> &NewVNInfo;
+  SmallVectorImpl<VNInfo *> &NewVNInfo;
 
   const CoalescerPair &CP;
   LiveIntervals *LIS;
@@ -2483,7 +2498,7 @@ class JoinVals {
   /// NewVNInfo. This is suitable for passing to LiveInterval::join().
   SmallVector<int, 8> Assignments;
 
-  public:
+public:
   /// Conflict resolution for overlapping values.
   enum ConflictResolution {
     /// No overlap, simply keep this value.
@@ -2512,7 +2527,7 @@ class JoinVals {
     CR_Impossible
   };
 
-  private:
+private:
   /// Per-value info for LI. The lane bit masks are all relative to the final
   /// joined register, so they can be compared directly between SrcReg and
   /// DstReg.
@@ -2584,7 +2599,8 @@ class JoinVals {
   /// Find the ultimate value that VNI was copied from.
   std::pair<const VNInfo *, Register> followCopyChain(const VNInfo *VNI) const;
 
-  bool valuesIdentical(VNInfo *Value0, VNInfo *Value1, const JoinVals &Other) const;
+  bool valuesIdentical(VNInfo *Value0, VNInfo *Value1,
+                       const JoinVals &Other) const;
 
   /// Analyze ValNo in this live range, and set all fields of Vals[ValNo].
   /// Return a conflict resolution when possible, but leave the hard cases as
@@ -2674,7 +2690,7 @@ class JoinVals {
   /// Add erased instructions to ErasedInstrs.
   /// Add foreign virtual registers to ShrinkRegs if their live range ended at
   /// the erased instrs.
-  void eraseInstrs(SmallPtrSetImpl<MachineInstr*> &ErasedInstrs,
+  void eraseInstrs(SmallPtrSetImpl<MachineInstr *> &ErasedInstrs,
                    SmallVectorImpl<Register> &ShrinkRegs,
                    LiveInterval *LI = nullptr);
 
@@ -2692,14 +2708,14 @@ class JoinVals {
 
 } // end anonymous namespace
 
-LaneBitmask JoinVals::computeWriteLanes(const MachineInstr *DefMI, bool &Redef)
-  const {
+LaneBitmask JoinVals::computeWriteLanes(const MachineInstr *DefMI,
+                                        bool &Redef) const {
   LaneBitmask L;
   for (const MachineOperand &MO : DefMI->all_defs()) {
     if (MO.getReg() != Reg)
       continue;
     L |= TRI->getSubRegIndexLaneMask(
-           TRI->composeSubRegIndices(SubIdx, MO.getSubReg()));
+        TRI->composeSubRegIndices(SubIdx, MO.getSubReg()));
     if (MO.readsReg())
       Redef = true;
   }
@@ -2783,8 +2799,8 @@ bool JoinVals::valuesIdentical(VNInfo *Value0, VNInfo *Value1,
   return Orig0->def == Orig1->def && Reg0 == Reg1;
 }
 
-JoinVals::ConflictResolution
-JoinVals::analyzeValue(unsigned ValNo, JoinVals &Other) {
+JoinVals::ConflictResolution JoinVals::analyzeValue(unsigned ValNo,
+                                                    JoinVals &Other) {
   Val &V = Vals[ValNo];
   assert(!V.isAnalyzed() && "Value has already been analyzed!");
   VNInfo *VNI = LR.getValNumInfo(ValNo);
@@ -2922,9 +2938,9 @@ JoinVals::analyzeValue(unsigned ValNo, JoinVals &Other) {
     if (DefMI &&
         (DefMI->getParent() != OtherMBB || LIS->isLiveInToMBB(LR, OtherMBB))) {
       LLVM_DEBUG(dbgs() << "IMPLICIT_DEF defined at " << V.OtherVNI->def
-                 << " extends into "
-                 << printMBBReference(*DefMI->getParent())
-                 << ", keeping it.\n");
+                        << " extends into "
+                        << printMBBReference(*DefMI->getParent())
+                        << ", keeping it.\n");
       OtherV.mustKeepImplicitDef(*TRI, *OtherImpDef);
     } else if (OtherMBB->hasEHPadSuccessor()) {
       // If OtherV is defined in a basic block that has EH pad successors then
@@ -3113,9 +3129,9 @@ bool JoinVals::mapValues(JoinVals &Other) {
   return true;
 }
 
-bool JoinVals::
-taintExtent(unsigned ValNo, LaneBitmask TaintedLanes, JoinVals &Other,
-            SmallVectorImpl<std::pair<SlotIndex, LaneBitmask>> &TaintExtent) {
+bool JoinVals::taintExtent(
+    unsigned ValNo, LaneBitmask TaintedLanes, JoinVals &Other,
+    SmallVectorImpl<std::pair<SlotIndex, LaneBitmask>> &TaintExtent) {
   VNInfo *VNI = LR.getValNumInfo(ValNo);
   MachineBasicBlock *MBB = Indexes->getMBBFromIndex(VNI->def);
   SlotIndex MBBEnd = Indexes->getMBBEndIdx(MBB);
@@ -3176,8 +3192,8 @@ bool JoinVals::resolveConflicts(JoinVals &Other) {
     if (V.Resolution != CR_Unresolved)
       continue;
     LLVM_DEBUG(dbgs() << "\t\tconflict at " << printReg(Reg) << ':' << i << '@'
-                      << LR.getValNumInfo(i)->def
-                      << ' ' << PrintLaneMask(LaneMask) << '\n');
+                      << LR.getValNumInfo(i)->def << ' '
+                      << PrintLaneMask(LaneMask) << '\n');
     if (SubRangeJoin)
       return false;
 
@@ -3210,7 +3226,7 @@ bool JoinVals::resolveConflicts(JoinVals &Other) {
     assert(!SlotIndex::isSameInstr(VNI->def, TaintExtent.front().first) &&
            "Interference ends on VNI->def. Should have been handled earlier");
     MachineInstr *LastMI =
-      Indexes->getInstructionFromIndex(TaintExtent.front().first);
+        Indexes->getInstructionFromIndex(TaintExtent.front().first);
     assert(LastMI && "Range must end at a proper instruction");
     unsigned TaintNum = 0;
     while (true) {
@@ -3268,8 +3284,8 @@ void JoinVals::pruneValues(JoinVals &Other,
       // predecessors, so the instruction should simply go away once its value
       // has been replaced.
       Val &OtherV = Other.Vals[Vals[i].OtherVNI->id];
-      bool EraseImpDef = OtherV.ErasableImplicitDef &&
-                         OtherV.Resolution == CR_Keep;
+      bool EraseImpDef =
+          OtherV.ErasableImplicitDef && OtherV.Resolution == CR_Keep;
       if (!Def.isBlock()) {
         if (changeInstrs) {
           // Remove <def,read-undef> flags. This def is now a partial redef.
@@ -3384,12 +3400,12 @@ void JoinVals::pruneSubRegValues(LiveInterval &LI, LaneBitmask &ShrinkMask) {
       // If a subrange starts at the copy then an undefined value has been
       // copied and we must remove that subrange value as well.
       VNInfo *ValueOut = Q.valueOutOrDead();
-      if (ValueOut != nullptr && (Q.valueIn() == nullptr ||
-                                  (V.Identical && V.Resolution == CR_Erase &&
-                                   ValueOut->def == Def))) {
+      if (ValueOut != nullptr &&
+          (Q.valueIn() == nullptr ||
+           (V.Identical && V.Resolution == CR_Erase && ValueOut->def == Def))) {
         LLVM_DEBUG(dbgs() << "\t\tPrune sublane " << PrintLaneMask(S.LaneMask)
                           << " at " << Def << "\n");
-        SmallVector<SlotIndex,8> EndPoints;
+        SmallVector<SlotIndex, 8> EndPoints;
         LIS->pruneValue(S, Def, &EndPoints);
         DidPrune = true;
         // Mark value number as unused.
@@ -3438,7 +3454,7 @@ static bool isDefInSubRange(LiveInterval &LI, SlotIndex Def) {
 }
 
 void JoinVals::pruneMainSegments(LiveInterval &LI, bool &ShrinkMainRange) {
-  assert(&static_cast<LiveRange&>(LI) == &LR);
+  assert(&static_cast<LiveRange &>(LI) == &LR);
 
   for (unsigned i = 0, e = LR.getNumValNums(); i != e; ++i) {
     if (Vals[i].Resolution != CR_Keep)
@@ -3463,7 +3479,7 @@ void JoinVals::removeImplicitDefs() {
   }
 }
 
-void JoinVals::eraseInstrs(SmallPtrSetImpl<MachineInstr*> &ErasedInstrs,
+void JoinVals::eraseInstrs(SmallPtrSetImpl<MachineInstr *> &ErasedInstrs,
                            SmallVectorImpl<Register> &ShrinkRegs,
                            LiveInterval *LI) {
   for (unsigned i = 0, e = LR.getNumValNums(); i != e; ++i) {
@@ -3502,7 +3518,7 @@ void JoinVals::eraseInstrs(SmallPtrSetImpl<MachineInstr*> &ErasedInstrs,
       VNI->markUnused();
 
       if (LI != nullptr && LI->hasSubRanges()) {
-        assert(static_cast<LiveRange*>(LI) == &LR);
+        assert(static_cast<LiveRange *>(LI) == &LR);
         // Determine the end point based on the subrange information:
         // minimum of (earliest def of next segment,
         //             latest end point of containing segment)
@@ -3560,11 +3576,11 @@ void JoinVals::eraseInstrs(SmallPtrSetImpl<MachineInstr*> &ErasedInstrs,
 void RegisterCoalescer::joinSubRegRanges(LiveRange &LRange, LiveRange &RRange,
                                          LaneBitmask LaneMask,
                                          const CoalescerPair &CP) {
-  SmallVector<VNInfo*, 16> NewVNInfo;
-  JoinVals RHSVals(RRange, CP.getSrcReg(), CP.getSrcIdx(), LaneMask,
-                   NewVNInfo, CP, LIS, TRI, true, true);
-  JoinVals LHSVals(LRange, CP.getDstReg(), CP.getDstIdx(), LaneMask,
-                   NewVNInfo, CP, LIS, TRI, true, true);
+  SmallVector<VNInfo *, 16> NewVNInfo;
+  JoinVals RHSVals(RRange, CP.getSrcReg(), CP.getSrcIdx(), LaneMask, NewVNInfo,
+                   CP, LIS, TRI, true, true);
+  JoinVals LHSVals(LRange, CP.getDstReg(), CP.getDstIdx(), LaneMask, NewVNInfo,
+                   CP, LIS, TRI, true, true);
 
   // Compute NewVNInfo and resolve conflicts (see also joinVirtRegs())
   // We should be able to resolve all conflicts here as we could successfully do
@@ -3600,8 +3616,8 @@ void RegisterCoalescer::joinSubRegRanges(LiveRange &LRange, LiveRange &RRange,
   LRange.join(RRange, LHSVals.getAssignments(), RHSVals.getAssignments(),
               NewVNInfo);
 
-  LLVM_DEBUG(dbgs() << "\t\tjoined lanes: " << PrintLaneMask(LaneMask)
-                    << ' ' << LRange << "\n");
+  LLVM_DEBUG(dbgs() << "\t\tjoined lanes: " << PrintLaneMask(LaneMask) << ' '
+                    << LRange << "\n");
   if (EndPoints.empty())
     return;
 
@@ -3611,7 +3627,7 @@ void RegisterCoalescer::joinSubRegRanges(LiveRange &LRange, LiveRange &RRange,
     dbgs() << "\t\trestoring liveness to " << EndPoints.size() << " points: ";
     for (unsigned i = 0, n = EndPoints.size(); i != n; ++i) {
       dbgs() << EndPoints[i];
-      if (i != n-1)
+      if (i != n - 1)
         dbgs() << ',';
     }
     dbgs() << ":  " << LRange << '\n';
@@ -3651,7 +3667,7 @@ bool RegisterCoalescer::isHighCostLiveInterval(LiveInterval &LI) {
 }
 
 bool RegisterCoalescer::joinVirtRegs(CoalescerPair &CP) {
-  SmallVector<VNInfo*, 16> NewVNInfo;
+  SmallVector<VNInfo *, 16> NewVNInfo;
   LiveInterval &RHS = LIS->getInterval(CP.getSrcReg());
   LiveInterval &LHS = LIS->getInterval(CP.getDstReg());
   bool TrackSubRegLiveness = MRI->shouldTrackSubRegLiveness(*CP.getNewRC());
@@ -3820,12 +3836,12 @@ bool RegisterCoalescer::joinVirtRegs(CoalescerPair &CP) {
       dbgs() << "\t\trestoring liveness to " << EndPoints.size() << " points: ";
       for (unsigned i = 0, n = EndPoints.size(); i != n; ++i) {
         dbgs() << EndPoints[i];
-        if (i != n-1)
+        if (i != n - 1)
           dbgs() << ',';
       }
       dbgs() << ":  " << LHS << '\n';
     });
-    LIS->extendToIndices((LiveRange&)LHS, EndPoints);
+    LIS->extendToIndices((LiveRange &)LHS, EndPoints);
   }
 
   return true;
@@ -3835,8 +3851,7 @@ bool RegisterCoalescer::joinIntervals(CoalescerPair &CP) {
   return CP.isPhys() ? joinReservedPhysReg(CP) : joinVirtRegs(CP);
 }
 
-void RegisterCoalescer::buildVRegToDbgValueMap(MachineFunction &MF)
-{
+void RegisterCoalescer::buildVRegToDbgValueMap(MachineFunction &MF) {
   const SlotIndexes &Slots = *LIS->getSlotIndexes();
   SmallVector<MachineInstr *, 8> ToInsert;
 
@@ -3939,8 +3954,8 @@ void RegisterCoalescer::checkMergingChangesDbgValuesImpl(Register Reg,
     // was coalesced and Reg deleted. It's safe to refer to the other register
     // (which will be the source of the copy).
     auto Resolution = RegVals.getResolution(OtherIt->valno->id);
-    LastUndefResult = Resolution != JoinVals::CR_Keep &&
-                      Resolution != JoinVals::CR_Erase;
+    LastUndefResult =
+        Resolution != JoinVals::CR_Keep && Resolution != JoinVals::CR_Erase;
     LastUndefIdx = Idx;
     return LastUndefResult;
   };
@@ -3977,7 +3992,7 @@ struct MBBPriorityInfo {
   bool IsSplit;
 
   MBBPriorityInfo(MachineBasicBlock *mbb, unsigned depth, bool issplit)
-    : MBB(mbb), Depth(depth), IsSplit(issplit) {}
+      : MBB(mbb), Depth(depth), IsSplit(issplit) {}
 };
 
 } // end anonymous namespace
@@ -4020,8 +4035,8 @@ static bool isLocalCopy(MachineInstr *Copy, const LiveIntervals *LIS) {
   if (SrcReg.isPhysical() || DstReg.isPhysical())
     return false;
 
-  return LIS->intervalIsInOneMBB(LIS->getInterval(SrcReg))
-    || LIS->intervalIsInOneMBB(LIS->getInterval(DstReg));
+  return LIS->intervalIsInOneMBB(LIS->getInterval(SrcReg)) ||
+         LIS->intervalIsInOneMBB(LIS->getInterval(DstReg));
 }
 
 void RegisterCoalescer::lateLiveIntervalUpdate() {
@@ -4036,8 +4051,8 @@ void RegisterCoalescer::lateLiveIntervalUpdate() {
   ToBeUpdated.clear();
 }
 
-bool RegisterCoalescer::
-copyCoalesceWorkList(MutableArrayRef<MachineInstr*> CurrList) {
+bool RegisterCoalescer::copyCoalesceWorkList(
+    MutableArrayRef<MachineInstr *> CurrList) {
   bool Progress = false;
   SmallPtrSet<MachineInstr *, 4> CurrentErasedInstrs;
   for (MachineInstr *&MI : CurrList) {
@@ -4113,7 +4128,7 @@ bool RegisterCoalescer::applyTerminalRule(const MachineInstr &Copy) const {
     Register OtherSrcReg, OtherReg;
     unsigned OtherSrcSubReg = 0, OtherSubReg = 0;
     if (!isMoveInstr(*TRI, &Copy, OtherSrcReg, OtherReg, OtherSrcSubReg,
-                OtherSubReg))
+                     OtherSubReg))
       return false;
     if (OtherReg == SrcReg)
       OtherReg = OtherSrcReg;
@@ -4130,16 +4145,15 @@ bool RegisterCoalescer::applyTerminalRule(const MachineInstr &Copy) const {
   return false;
 }
 
-void
-RegisterCoalescer::copyCoalesceInMBB(MachineBasicBlock *MBB) {
+void RegisterCoalescer::copyCoalesceInMBB(MachineBasicBlock *MBB) {
   LLVM_DEBUG(dbgs() << MBB->getName() << ":\n");
 
   // Collect all copy-like instructions in MBB. Don't start coalescing anything
   // yet, it might invalidate the iterator.
   const unsigned PrevSize = WorkList.size();
   if (JoinGlobalCopies) {
-    SmallVector<MachineInstr*, 2> LocalTerminals;
-    SmallVector<MachineInstr*, 2> GlobalTerminals;
+    SmallVector<MachineInstr *, 2> LocalTerminals;
+    SmallVector<MachineInstr *, 2> GlobalTerminals;
     // Coalesce copies bottom-up to coalesce local defs before local uses. They
     // are not inherently easier to resolve, but slightly preferable until we
     // have local live range splitting. In particular this is required by
@@ -4163,9 +4177,8 @@ RegisterCoalescer::copyCoalesceInMBB(MachineBasicBlock *MBB) {
     // Append the copies evicted by the terminal rule at the end of the list.
     LocalWorkList.append(LocalTerminals.begin(), LocalTerminals.end());
     WorkList.append(GlobalTerminals.begin(), GlobalTerminals.end());
-  }
-  else {
-    SmallVector<MachineInstr*, 2> Terminals;
+  } else {
+    SmallVector<MachineInstr *, 2> Terminals;
     for (MachineInstr &MII : *MBB)
       if (MII.isCopyLike()) {
         if (applyTerminalRule(MII))
@@ -4179,11 +4192,12 @@ RegisterCoalescer::copyCoalesceInMBB(MachineBasicBlock *MBB) {
   // Try coalescing the collected copies immediately, and remove the nulls.
   // This prevents the WorkList from getting too large since most copies are
   // joinable on the first attempt.
-  MutableArrayRef<MachineInstr*>
-    CurrList(WorkList.begin() + PrevSize, WorkList.end());
+  MutableArrayRef<MachineInstr *> CurrList(WorkList.begin() + PrevSize,
+                                           WorkList.end());
   if (copyCoalesceWorkList(CurrList))
-    WorkList.erase(std::remove(WorkList.begin() + PrevSize, WorkList.end(),
-                               nullptr), WorkList.end());
+    WorkList.erase(
+        std::remove(WorkList.begin() + PrevSize, WorkList.end(), nullptr),
+        WorkList.end());
 }
 
 void RegisterCoalescer::coalesceLocals() {
@@ -4223,19 +4237,39 @@ void RegisterCoalescer::joinAllIntervals() {
   // Joining intervals can allow other intervals to be joined.  Iteratively join
   // until we make no progress.
   while (copyCoalesceWorkList(WorkList))
-    /* empty */ ;
+    /* empty */;
   lateLiveIntervalUpdate();
 }
 
-void RegisterCoalescer::releaseMemory() {
-  ErasedInstrs.clear();
-  WorkList.clear();
-  DeadDefs.clear();
-  InflateRegs.clear();
-  LargeLIVisitCounter.clear();
+PreservedAnalyses
+RegisterCoalescerPass::run(MachineFunction &MF,
+                           MachineFunctionAnalysisManager &MFAM) {
+  MFPropsModifier _(*this, MF);
+  auto &LIS = MFAM.getResult<LiveIntervalsAnalysis>(MF);
+  auto &Loops = MFAM.getResult<MachineLoopAnalysis>(MF);
+  auto *SI = MFAM.getCachedResult<SlotIndexesAnalysis>(MF);
+  RegisterCoalescer Impl(&LIS, SI, &Loops);
+  if (!Impl.run(MF))
+    return PreservedAnalyses::all();
+  auto PA = getMachineFunctionPassPreservedAnalyses();
+  PA.preserveSet<CFGAnalyses>();
+  PA.preserve<LiveIntervalsAnalysis>();
+  PA.preserve<SlotIndexesAnalysis>();
+  PA.preserve<MachineLoopAnalysis>();
+  PA.preserve<MachineDominatorTreeAnalysis>();
+  return PA;
 }
 
-bool RegisterCoalescer::runOnMachineFunction(MachineFunction &fn) {
+bool RegisterCoalescerLegacy::runOnMachineFunction(MachineFunction &MF) {
+  auto *LIS = &getAnalysis<LiveIntervalsWrapperPass>().getLIS();
+  auto *Loops = &getAnalysis<MachineLoopInfoWrapperPass>().getLI();
+  auto *SIWrapper = getAnalysisIfAvailable<SlotIndexesWrapperPass>();
+  SlotIndexes *SI = SIWrapper ? &SIWrapper->getSI() : nullptr;
+  RegisterCoalescer Impl(LIS, SI, Loops);
+  return Impl.run(MF);
+}
+
+bool RegisterCoalescer::run(MachineFunction &fn) {
   LLVM_DEBUG(dbgs() << "********** REGISTER COALESCER **********\n"
                     << "********** Function: " << fn.getName() << '\n');
 
@@ -4258,9 +4292,6 @@ bool RegisterCoalescer::runOnMachineFunction(MachineFunction &fn) {
   const TargetSubtargetInfo &STI = fn.getSubtarget();
   TRI = STI.getRegisterInfo();
   TII = STI.getInstrInfo();
-  LIS = &getAnalysis<LiveIntervalsWrapperPass>().getLIS();
-  AA = &getAnalysis<AAResultsWrapperPass>().getAAResults();
-  Loops = &getAnalysis<MachineLoopInfoWrapperPass>().getLI();
   if (EnableGlobalCopies == cl::BOU_UNSET)
     JoinGlobalCopies = STI.enableJoinGlobalCopies();
   else
@@ -4285,7 +4316,7 @@ bool RegisterCoalescer::runOnMachineFunction(MachineFunction &fn) {
   JoinSplitEdges = EnableJoinSplits;
 
   if (VerifyCoalescing)
-    MF->verify(this, "Before register coalescing", &errs());
+    MF->verify(LIS, SI, "Before register coalescing", &errs());
 
   DbgVRegToValues.clear();
   buildVRegToDbgValueMap(fn);
@@ -4343,12 +4374,9 @@ bool RegisterCoalescer::runOnMachineFunction(MachineFunction &fn) {
   PHIValToPos.clear();
   RegToPHIIdx.clear();
 
-  LLVM_DEBUG(dump());
+  LLVM_DEBUG(LIS->dump());
+
   if (VerifyCoalescing)
-    MF->verify(this, "After register coalescing", &errs());
+    MF->verify(LIS, SI, "After register coalescing", &errs());
   return true;
 }
-
-void RegisterCoalescer::print(raw_ostream &O, const Module* m) const {
-  LIS->print(O);
-}
diff --git a/llvm/lib/CodeGen/RegisterCoalescer.h b/llvm/lib/CodeGen/RegisterCoalescer.h
index f265d93fb0d63..ec1940805ea2e 100644
--- a/llvm/lib/CodeGen/RegisterCoalescer.h
+++ b/llvm/lib/CodeGen/RegisterCoalescer.h
@@ -14,6 +14,7 @@
 #ifndef LLVM_LIB_CODEGEN_REGISTERCOALESCER_H
 #define LLVM_LIB_CODEGEN_REGISTERCOALESCER_H
 
+#include "llvm/CodeGen/MachinePassManager.h"
 #include "llvm/CodeGen/Register.h"
 
 namespace llvm {
@@ -22,92 +23,92 @@ class MachineInstr;
 class TargetRegisterClass;
 class TargetRegisterInfo;
 
-  /// A helper class for register coalescers. When deciding if
-  /// two registers can be coalesced, CoalescerPair can determine if a copy
-  /// instruction would become an identity copy after coalescing.
-  class CoalescerPair {
-    const TargetRegisterInfo &TRI;
+/// A helper class for register coalescers. When deciding if
+/// two registers can be coalesced, CoalescerPair can determine if a copy
+/// instruction would become an identity copy after coalescing.
+class CoalescerPair {
+  const TargetRegisterInfo &TRI;
 
-    /// The register that will be left after coalescing. It can be a
-    /// virtual or physical register.
-    Register DstReg;
+  /// The register that will be left after coalescing. It can be a
+  /// virtual or physical register.
+  Register DstReg;
 
-    /// The virtual register that will be coalesced into dstReg.
-    Register SrcReg;
+  /// The virtual register that will be coalesced into dstReg.
+  Register SrcReg;
 
-    /// The sub-register index of the old DstReg in the new coalesced register.
-    unsigned DstIdx = 0;
+  /// The sub-register index of the old DstReg in the new coalesced register.
+  unsigned DstIdx = 0;
 
-    /// The sub-register index of the old SrcReg in the new coalesced register.
-    unsigned SrcIdx = 0;
+  /// The sub-register index of the old SrcReg in the new coalesced register.
+  unsigned SrcIdx = 0;
 
-    /// True when the original copy was a partial subregister copy.
-    bool Partial = false;
+  /// True when the original copy was a partial subregister copy.
+  bool Partial = false;
 
-    /// True when both regs are virtual and newRC is constrained.
-    bool CrossClass = false;
+  /// True when both regs are virtual and newRC is constrained.
+  bool CrossClass = false;
 
-    /// True when DstReg and SrcReg are reversed from the original
-    /// copy instruction.
-    bool Flipped = false;
+  /// True when DstReg and SrcReg are reversed from the original
+  /// copy instruction.
+  bool Flipped = false;
 
-    /// The register class of the coalesced register, or NULL if DstReg
-    /// is a physreg. This register class may be a super-register of both
-    /// SrcReg and DstReg.
-    const TargetRegisterClass *NewRC = nullptr;
+  /// The register class of the coalesced register, or NULL if DstReg
+  /// is a physreg. This register class may be a super-register of both
+  /// SrcReg and DstReg.
+  const TargetRegisterClass *NewRC = nullptr;
 
-  public:
-    CoalescerPair(const TargetRegisterInfo &tri) : TRI(tri) {}
+public:
+  CoalescerPair(const TargetRegisterInfo &tri) : TRI(tri) {}
 
-    /// Create a CoalescerPair representing a virtreg-to-physreg copy.
-    /// No need to call setRegisters().
-    CoalescerPair(Register VirtReg, MCRegister PhysReg,
-                  const TargetRegisterInfo &tri)
-        : TRI(tri), DstReg(PhysReg), SrcReg(VirtReg) {}
+  /// Create a CoalescerPair representing a virtreg-to-physreg copy.
+  /// No need to call setRegisters().
+  CoalescerPair(Register VirtReg, MCRegister PhysReg,
+                const TargetRegisterInfo &tri)
+      : TRI(tri), DstReg(PhysReg), SrcReg(VirtReg) {}
 
-    /// Set registers to match the copy instruction MI. Return
-    /// false if MI is not a coalescable copy instruction.
-    bool setRegisters(const MachineInstr*);
+  /// Set registers to match the copy instruction MI. Return
+  /// false if MI is not a coalescable copy instruction.
+  bool setRegisters(const MachineInstr *);
 
-    /// Swap SrcReg and DstReg. Return false if swapping is impossible
-    /// because DstReg is a physical register, or SubIdx is set.
-    bool flip();
+  /// Swap SrcReg and DstReg. Return false if swapping is impossible
+  /// because DstReg is a physical register, or SubIdx is set.
+  bool flip();
 
-    /// Return true if MI is a copy instruction that will become
-    /// an identity copy after coalescing.
-    bool isCoalescable(const MachineInstr*) const;
+  /// Return true if MI is a copy instruction that will become
+  /// an identity copy after coalescing.
+  bool isCoalescable(const MachineInstr *) const;
 
-    /// Return true if DstReg is a physical register.
-    bool isPhys() const { return !NewRC; }
+  /// Return true if DstReg is a physical register.
+  bool isPhys() const { return !NewRC; }
 
-    /// Return true if the original copy instruction did not copy
-    /// the full register, but was a subreg operation.
-    bool isPartial() const { return Partial; }
+  /// Return true if the original copy instruction did not copy
+  /// the full register, but was a subreg operation.
+  bool isPartial() const { return Partial; }
 
-    /// Return true if DstReg is virtual and NewRC is a smaller
-    /// register class than DstReg's.
-    bool isCrossClass() const { return CrossClass; }
+  /// Return true if DstReg is virtual and NewRC is a smaller
+  /// register class than DstReg's.
+  bool isCrossClass() const { return CrossClass; }
 
-    /// Return true when getSrcReg is the register being defined by
-    /// the original copy instruction.
-    bool isFlipped() const { return Flipped; }
+  /// Return true when getSrcReg is the register being defined by
+  /// the original copy instruction.
+  bool isFlipped() const { return Flipped; }
 
-    /// Return the register (virtual or physical) that will remain
-    /// after coalescing.
-    Register getDstReg() const { return DstReg; }
+  /// Return the register (virtual or physical) that will remain
+  /// after coalescing.
+  Register getDstReg() const { return DstReg; }
 
-    /// Return the virtual register that will be coalesced away.
-    Register getSrcReg() const { return SrcReg; }
+  /// Return the virtual register that will be coalesced away.
+  Register getSrcReg() const { return SrcReg; }
 
-    /// Return the subregister index that DstReg will be coalesced into, or 0.
-    unsigned getDstIdx() const { return DstIdx; }
+  /// Return the subregister index that DstReg will be coalesced into, or 0.
+  unsigned getDstIdx() const { return DstIdx; }
 
-    /// Return the subregister index that SrcReg will be coalesced into, or 0.
-    unsigned getSrcIdx() const { return SrcIdx; }
+  /// Return the subregister index that SrcReg will be coalesced into, or 0.
+  unsigned getSrcIdx() const { return SrcIdx; }
 
-    /// Return the register class of the coalesced register.
-    const TargetRegisterClass *getNewRC() const { return NewRC; }
-  };
+  /// Return the register class of the coalesced register.
+  const TargetRegisterClass *getNewRC() const { return NewRC; }
+};
 
 } // end namespace llvm
 
diff --git a/llvm/lib/CodeGen/RegisterPressure.cpp b/llvm/lib/CodeGen/RegisterPressure.cpp
index e8e6db1e3b3bd..ca51b670b46cc 100644
--- a/llvm/lib/CodeGen/RegisterPressure.cpp
+++ b/llvm/lib/CodeGen/RegisterPressure.cpp
@@ -79,15 +79,12 @@ static void decreaseSetPressure(std::vector<unsigned> &CurrSetPressure,
 LLVM_DUMP_METHOD
 void llvm::dumpRegSetPressure(ArrayRef<unsigned> SetPressure,
                               const TargetRegisterInfo *TRI) {
-  bool Empty = true;
   for (unsigned i = 0, e = SetPressure.size(); i < e; ++i) {
     if (SetPressure[i] != 0) {
-      dbgs() << TRI->getRegPressureSetName(i) << "=" << SetPressure[i] << '\n';
-      Empty = false;
+      dbgs() << TRI->getRegPressureSetName(i) << "=" << SetPressure[i] << ' ';
     }
   }
-  if (Empty)
-    dbgs() << "\n";
+  dbgs() << "\n";
 }
 
 LLVM_DUMP_METHOD
diff --git a/llvm/lib/CodeGen/RegisterScavenging.cpp b/llvm/lib/CodeGen/RegisterScavenging.cpp
index 126f8e6d039d5..350ae2048bfa4 100644
--- a/llvm/lib/CodeGen/RegisterScavenging.cpp
+++ b/llvm/lib/CodeGen/RegisterScavenging.cpp
@@ -412,8 +412,7 @@ static bool scavengeFrameVirtualRegsInBlock(MachineRegisterInfo &MRI,
         // We only care about virtual registers and ignore virtual registers
         // created by the target callbacks in the process (those will be handled
         // in a scavenging round).
-        if (!Reg.isVirtual() ||
-            Register::virtReg2Index(Reg) >= InitialNumVirtRegs)
+        if (!Reg.isVirtual() || Reg.virtRegIndex() >= InitialNumVirtRegs)
           continue;
         if (!MO.readsReg())
           continue;
@@ -432,8 +431,7 @@ static bool scavengeFrameVirtualRegsInBlock(MachineRegisterInfo &MRI,
         continue;
       Register Reg = MO.getReg();
       // Only vregs, no newly created vregs (see above).
-      if (!Reg.isVirtual() ||
-          Register::virtReg2Index(Reg) >= InitialNumVirtRegs)
+      if (!Reg.isVirtual() || Reg.virtRegIndex() >= InitialNumVirtRegs)
         continue;
       // We have to look at all operands anyway so we can precalculate here
       // whether there is a reading operand. This allows use to skip the use
diff --git a/llvm/lib/CodeGen/RemoveLoadsIntoFakeUses.cpp b/llvm/lib/CodeGen/RemoveLoadsIntoFakeUses.cpp
index ef7a58670c3ac..384a049acfe34 100644
--- a/llvm/lib/CodeGen/RemoveLoadsIntoFakeUses.cpp
+++ b/llvm/lib/CodeGen/RemoveLoadsIntoFakeUses.cpp
@@ -32,6 +32,7 @@
 #include "llvm/IR/Function.h"
 #include "llvm/InitializePasses.h"
 #include "llvm/Support/Debug.h"
+#include "llvm/Target/TargetMachine.h"
 
 using namespace llvm;
 
@@ -74,6 +75,10 @@ INITIALIZE_PASS_END(RemoveLoadsIntoFakeUses, DEBUG_TYPE,
                     "Remove Loads Into Fake Uses", false, false)
 
 bool RemoveLoadsIntoFakeUses::runOnMachineFunction(MachineFunction &MF) {
+  // Skip this pass if we would use VarLoc-based LDV, as there may be DBG_VALUE
+  // instructions of the restored values that would become invalid.
+  if (!MF.useDebugInstrRef())
+    return false;
   // Only run this for functions that have fake uses.
   if (!MF.hasFakeUses() || skipFunction(MF.getFunction()))
     return false;
@@ -86,20 +91,20 @@ bool RemoveLoadsIntoFakeUses::runOnMachineFunction(MachineFunction &MF) {
   const TargetInstrInfo *TII = ST.getInstrInfo();
   const TargetRegisterInfo *TRI = ST.getRegisterInfo();
 
-  SmallDenseMap<Register, SmallVector<MachineInstr *>> RegFakeUses;
+  SmallVector<MachineInstr *> RegFakeUses;
   LivePhysRegs.init(*TRI);
   SmallVector<MachineInstr *, 16> Statepoints;
   for (MachineBasicBlock *MBB : post_order(&MF)) {
+    RegFakeUses.clear();
     LivePhysRegs.addLiveOuts(*MBB);
 
     for (MachineInstr &MI : make_early_inc_range(reverse(*MBB))) {
       if (MI.isFakeUse()) {
-        for (const MachineOperand &MO : MI.operands()) {
-          // Track the Fake Uses that use this register so that we can delete
-          // them if we delete the corresponding load.
-          if (MO.isReg())
-            RegFakeUses[MO.getReg()].push_back(&MI);
-        }
+        if (MI.getNumOperands() == 0 || !MI.getOperand(0).isReg())
+          continue;
+        // Track the Fake Uses that use these register units so that we can
+        // delete them if we delete the corresponding load.
+        RegFakeUses.push_back(&MI);
         // Do not record FAKE_USE uses in LivePhysRegs so that we can recognize
         // otherwise-unused loads.
         continue;
@@ -109,31 +114,38 @@ bool RemoveLoadsIntoFakeUses::runOnMachineFunction(MachineFunction &MF) {
       // reload of a spilled register.
       if (MI.getRestoreSize(TII)) {
         Register Reg = MI.getOperand(0).getReg();
-        assert(Reg.isPhysical() && "VReg seen in function with NoVRegs set?");
         // Don't delete live physreg defs, or any reserved register defs.
         if (!LivePhysRegs.available(Reg) || MRI->isReserved(Reg))
           continue;
-        // There should be an exact match between the loaded register and the
-        // FAKE_USE use. If not, this is a load that is unused by anything? It
-        // should probably be deleted, but that's outside of this pass' scope.
-        if (RegFakeUses.contains(Reg)) {
+        // There should typically be an exact match between the loaded register
+        // and the FAKE_USE, but sometimes regalloc will choose to load a larger
+        // value than is needed. Therefore, as long as the load isn't used by
+        // anything except at least one FAKE_USE, we will delete it. If it isn't
+        // used by any fake uses, it should still be safe to delete but we
+        // choose to ignore it so that this pass has no side effects unrelated
+        // to fake uses.
+        SmallDenseSet<MachineInstr *> FakeUsesToDelete;
+        SmallVector<MachineInstr *> RemainingFakeUses;
+        for (MachineInstr *&FakeUse : reverse(RegFakeUses)) {
+          if (FakeUse->readsRegister(Reg, TRI)) {
+            FakeUsesToDelete.insert(FakeUse);
+            RegFakeUses.erase(&FakeUse);
+          }
+        }
+        if (!FakeUsesToDelete.empty()) {
           LLVM_DEBUG(dbgs() << "RemoveLoadsIntoFakeUses: DELETING: " << MI);
-          // It is possible that some DBG_VALUE instructions refer to this
-          // instruction. They will be deleted in the live debug variable
-          // analysis.
+          // Since this load only exists to restore a spilled register and we
+          // haven't, run LiveDebugValues yet, there shouldn't be any DBG_VALUEs
+          // for this load; otherwise, deleting this would be incorrect.
           MI.eraseFromParent();
           AnyChanges = true;
           ++NumLoadsDeleted;
-          // Each FAKE_USE now appears to be a fake use of the previous value
-          // of the loaded register; delete them to avoid incorrectly
-          // interpreting them as such.
-          for (MachineInstr *FakeUse : RegFakeUses[Reg]) {
+          for (MachineInstr *FakeUse : FakeUsesToDelete) {
             LLVM_DEBUG(dbgs()
                        << "RemoveLoadsIntoFakeUses: DELETING: " << *FakeUse);
             FakeUse->eraseFromParent();
           }
-          NumFakeUsesDeleted += RegFakeUses[Reg].size();
-          RegFakeUses[Reg].clear();
+          NumFakeUsesDeleted += FakeUsesToDelete.size();
         }
         continue;
       }
@@ -143,13 +155,15 @@ bool RemoveLoadsIntoFakeUses::runOnMachineFunction(MachineFunction &MF) {
       // that register.
       if (!RegFakeUses.empty()) {
         for (const MachineOperand &MO : MI.operands()) {
-          if (MO.isReg() && MO.isDef()) {
-            Register Reg = MO.getReg();
-            assert(Reg.isPhysical() &&
-                   "VReg seen in function with NoVRegs set?");
-            for (MCRegUnit Unit : TRI->regunits(Reg))
-              RegFakeUses.erase(Unit);
-          }
+          if (!MO.isReg())
+            continue;
+          Register Reg = MO.getReg();
+          // We clear RegFakeUses for this register and all subregisters,
+          // because any such FAKE_USE encountered prior is no longer relevant
+          // for later encountered loads.
+          for (MachineInstr *&FakeUse : reverse(RegFakeUses))
+            if (FakeUse->readsRegister(Reg, TRI))
+              RegFakeUses.erase(&FakeUse);
         }
       }
       LivePhysRegs.stepBackward(MI);
diff --git a/llvm/lib/CodeGen/RenameIndependentSubregs.cpp b/llvm/lib/CodeGen/RenameIndependentSubregs.cpp
index 0128f87748a77..58f212e1ba521 100644
--- a/llvm/lib/CodeGen/RenameIndependentSubregs.cpp
+++ b/llvm/lib/CodeGen/RenameIndependentSubregs.cpp
@@ -26,6 +26,7 @@
 //
 //===----------------------------------------------------------------------===//
 
+#include "llvm/CodeGen/RenameIndependentSubregs.h"
 #include "LiveRangeUtils.h"
 #include "PHIEliminationUtils.h"
 #include "llvm/CodeGen/LiveInterval.h"
@@ -43,25 +44,11 @@ using namespace llvm;
 
 namespace {
 
-class RenameIndependentSubregs : public MachineFunctionPass {
+class RenameIndependentSubregs {
 public:
-  static char ID;
-  RenameIndependentSubregs() : MachineFunctionPass(ID) {}
-
-  StringRef getPassName() const override {
-    return "Rename Disconnected Subregister Components";
-  }
-
-  void getAnalysisUsage(AnalysisUsage &AU) const override {
-    AU.setPreservesCFG();
-    AU.addRequired<LiveIntervalsWrapperPass>();
-    AU.addPreserved<LiveIntervalsWrapperPass>();
-    AU.addRequired<SlotIndexesWrapperPass>();
-    AU.addPreserved<SlotIndexesWrapperPass>();
-    MachineFunctionPass::getAnalysisUsage(AU);
-  }
+  RenameIndependentSubregs(LiveIntervals *LIS) : LIS(LIS) {}
 
-  bool runOnMachineFunction(MachineFunction &MF) override;
+  bool run(MachineFunction &MF);
 
 private:
   struct SubRangeInfo {
@@ -106,17 +93,36 @@ class RenameIndependentSubregs : public MachineFunctionPass {
   const TargetInstrInfo *TII = nullptr;
 };
 
+class RenameIndependentSubregsLegacy : public MachineFunctionPass {
+public:
+  static char ID;
+  RenameIndependentSubregsLegacy() : MachineFunctionPass(ID) {}
+  bool runOnMachineFunction(MachineFunction &MF) override;
+  StringRef getPassName() const override {
+    return "Rename Disconnected Subregister Components";
+  }
+
+  void getAnalysisUsage(AnalysisUsage &AU) const override {
+    AU.setPreservesCFG();
+    AU.addRequired<LiveIntervalsWrapperPass>();
+    AU.addPreserved<LiveIntervalsWrapperPass>();
+    AU.addRequired<SlotIndexesWrapperPass>();
+    AU.addPreserved<SlotIndexesWrapperPass>();
+    MachineFunctionPass::getAnalysisUsage(AU);
+  }
+};
+
 } // end anonymous namespace
 
-char RenameIndependentSubregs::ID;
+char RenameIndependentSubregsLegacy::ID;
 
-char &llvm::RenameIndependentSubregsID = RenameIndependentSubregs::ID;
+char &llvm::RenameIndependentSubregsID = RenameIndependentSubregsLegacy::ID;
 
-INITIALIZE_PASS_BEGIN(RenameIndependentSubregs, DEBUG_TYPE,
+INITIALIZE_PASS_BEGIN(RenameIndependentSubregsLegacy, DEBUG_TYPE,
                       "Rename Independent Subregisters", false, false)
 INITIALIZE_PASS_DEPENDENCY(SlotIndexesWrapperPass)
 INITIALIZE_PASS_DEPENDENCY(LiveIntervalsWrapperPass)
-INITIALIZE_PASS_END(RenameIndependentSubregs, DEBUG_TYPE,
+INITIALIZE_PASS_END(RenameIndependentSubregsLegacy, DEBUG_TYPE,
                     "Rename Independent Subregisters", false, false)
 
 bool RenameIndependentSubregs::renameComponents(LiveInterval &LI) const {
@@ -381,7 +387,25 @@ void RenameIndependentSubregs::computeMainRangesFixFlags(
   }
 }
 
-bool RenameIndependentSubregs::runOnMachineFunction(MachineFunction &MF) {
+PreservedAnalyses
+RenameIndependentSubregsPass::run(MachineFunction &MF,
+                                  MachineFunctionAnalysisManager &MFAM) {
+  auto &LIS = MFAM.getResult<LiveIntervalsAnalysis>(MF);
+  if (!RenameIndependentSubregs(&LIS).run(MF))
+    return PreservedAnalyses::all();
+  auto PA = getMachineFunctionPassPreservedAnalyses();
+  PA.preserveSet<CFGAnalyses>();
+  PA.preserve<LiveIntervalsAnalysis>();
+  PA.preserve<SlotIndexesAnalysis>();
+  return PA;
+}
+
+bool RenameIndependentSubregsLegacy::runOnMachineFunction(MachineFunction &MF) {
+  auto &LIS = getAnalysis<LiveIntervalsWrapperPass>().getLIS();
+  return RenameIndependentSubregs(&LIS).run(MF);
+}
+
+bool RenameIndependentSubregs::run(MachineFunction &MF) {
   // Skip renaming if liveness of subregister is not tracked.
   MRI = &MF.getRegInfo();
   if (!MRI->subRegLivenessEnabled())
@@ -390,7 +414,6 @@ bool RenameIndependentSubregs::runOnMachineFunction(MachineFunction &MF) {
   LLVM_DEBUG(dbgs() << "Renaming independent subregister live ranges in "
                     << MF.getName() << '\n');
 
-  LIS = &getAnalysis<LiveIntervalsWrapperPass>().getLIS();
   TII = MF.getSubtarget().getInstrInfo();
 
   // Iterate over all vregs. Note that we query getNumVirtRegs() the newly
diff --git a/llvm/lib/CodeGen/ScheduleDAGInstrs.cpp b/llvm/lib/CodeGen/ScheduleDAGInstrs.cpp
index cc98c52e90ea6..a26804707dd1f 100644
--- a/llvm/lib/CodeGen/ScheduleDAGInstrs.cpp
+++ b/llvm/lib/CodeGen/ScheduleDAGInstrs.cpp
@@ -209,13 +209,25 @@ void ScheduleDAGInstrs::addSchedBarrierDeps() {
   ExitSU.setInstr(ExitMI);
   // Add dependencies on the defs and uses of the instruction.
   if (ExitMI) {
+    const MCInstrDesc &MIDesc = ExitMI->getDesc();
     for (const MachineOperand &MO : ExitMI->all_uses()) {
+      unsigned OpIdx = MO.getOperandNo();
       Register Reg = MO.getReg();
       if (Reg.isPhysical()) {
+        // addPhysRegDataDeps uses the provided operand index to retrieve
+        // the operand use cycle from the scheduling model. If the operand
+        // is "fake" (e.g., an operand of a call instruction used to pass
+        // an argument to the called function.), the scheduling model may not
+        // have an entry for it. If this is the case, pass -1 as operand index,
+        // which will cause addPhysRegDataDeps to add an artificial dependency.
+        // FIXME: Using hasImplicitUseOfPhysReg here is inaccurate as it misses
+        //  aliases. When fixing, make sure to update addPhysRegDataDeps, too.
+        bool IsRealUse = OpIdx < MIDesc.getNumOperands() ||
+                         MIDesc.hasImplicitUseOfPhysReg(Reg);
         for (MCRegUnit Unit : TRI->regunits(Reg))
-          Uses.insert(PhysRegSUOper(&ExitSU, -1, Unit));
+          Uses.insert(PhysRegSUOper(&ExitSU, IsRealUse ? OpIdx : -1, Unit));
       } else if (Reg.isVirtual() && MO.readsReg()) {
-        addVRegUseDeps(&ExitSU, MO.getOperandNo());
+        addVRegUseDeps(&ExitSU, OpIdx);
       }
     }
   }
diff --git a/llvm/lib/CodeGen/SelectOptimize.cpp b/llvm/lib/CodeGen/SelectOptimize.cpp
index b7600a3b7fba7..9c8dcacc912f5 100644
--- a/llvm/lib/CodeGen/SelectOptimize.cpp
+++ b/llvm/lib/CodeGen/SelectOptimize.cpp
@@ -483,9 +483,9 @@ static Value *getTrueOrFalseValue(
     BasicBlock *B) {
   Value *V = isTrue ? SI.getTrueValue() : SI.getFalseValue();
   if (V) {
-    auto *IV = dyn_cast<Instruction>(V);
-    if (IV && OptSelects.count(IV))
-      return isTrue ? OptSelects[IV].first : OptSelects[IV].second;
+    if (auto *IV = dyn_cast<Instruction>(V))
+      if (auto It = OptSelects.find(IV); It != OptSelects.end())
+        return isTrue ? It->second.first : It->second.second;
     return V;
   }
 
@@ -508,9 +508,8 @@ static Value *getTrueOrFalseValue(
 
   unsigned OtherIdx = 1 - CondIdx;
   if (auto *IV = dyn_cast<Instruction>(CBO->getOperand(OtherIdx))) {
-    if (OptSelects.count(IV))
-      CBO->setOperand(OtherIdx,
-                      isTrue ? OptSelects[IV].first : OptSelects[IV].second);
+    if (auto It = OptSelects.find(IV); It != OptSelects.end())
+      CBO->setOperand(OtherIdx, isTrue ? It->second.first : It->second.second);
   }
   CBO->insertBefore(B->getTerminator()->getIterator());
   return CBO;
@@ -1305,9 +1304,9 @@ bool SelectOptimizeImpl::computeLoopCosts(
           auto UI = dyn_cast<Instruction>(U.get());
           if (!UI)
             continue;
-          if (InstCostMap.count(UI)) {
-            IPredCost = std::max(IPredCost, InstCostMap[UI].PredCost);
-            INonPredCost = std::max(INonPredCost, InstCostMap[UI].NonPredCost);
+          if (auto It = InstCostMap.find(UI); It != InstCostMap.end()) {
+            IPredCost = std::max(IPredCost, It->second.PredCost);
+            INonPredCost = std::max(INonPredCost, It->second.NonPredCost);
           }
         }
         auto ILatency = computeInstCost(&I);
@@ -1338,8 +1337,8 @@ bool SelectOptimizeImpl::computeLoopCosts(
 
           Scaled64 CondCost = Scaled64::getZero();
           if (auto *CI = dyn_cast<Instruction>(SG->Condition))
-            if (InstCostMap.count(CI))
-              CondCost = InstCostMap[CI].NonPredCost;
+            if (auto It = InstCostMap.find(CI); It != InstCostMap.end())
+              CondCost = It->second.NonPredCost;
           Scaled64 MispredictCost = getMispredictionCost(SI, CondCost);
 
           INonPredCost = PredictedPathCost + MispredictCost;
diff --git a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
index a0c703d2df8a2..ef58da873c59c 100644
--- a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
@@ -385,17 +385,6 @@ namespace {
     bool getTruncatedStoreValue(StoreSDNode *ST, SDValue &Val);
     bool extendLoadedValueToExtension(LoadSDNode *LD, SDValue &Val);
 
-    /// Replace an ISD::EXTRACT_VECTOR_ELT of a load with a narrowed
-    ///   load.
-    ///
-    /// \param EVE ISD::EXTRACT_VECTOR_ELT to be replaced.
-    /// \param InVecVT type of the input vector to EVE with bitcasts resolved.
-    /// \param EltNo index of the vector element to load.
-    /// \param OriginalLoad load that EVE came from to be replaced.
-    /// \returns EVE on success SDValue() on failure.
-    SDValue scalarizeExtractedVectorLoad(SDNode *EVE, EVT InVecVT,
-                                         SDValue EltNo,
-                                         LoadSDNode *OriginalLoad);
     void ReplaceLoadWithPromotedLoad(SDNode *Load, SDNode *ExtLoad);
     SDValue PromoteOperand(SDValue Op, EVT PVT, bool &Replace);
     SDValue SExtPromoteOperand(SDValue Op, EVT PVT);
@@ -11005,8 +10994,8 @@ SDValue DAGCombiner::visitFunnelShift(SDNode *N) {
       auto *RHS = dyn_cast<LoadSDNode>(N1);
       if (LHS && RHS && LHS->isSimple() && RHS->isSimple() &&
           LHS->getAddressSpace() == RHS->getAddressSpace() &&
-          (LHS->hasOneUse() || RHS->hasOneUse()) && ISD::isNON_EXTLoad(RHS) &&
-          ISD::isNON_EXTLoad(LHS)) {
+          (LHS->hasNUsesOfValue(1, 0) || RHS->hasNUsesOfValue(1, 0)) &&
+          ISD::isNON_EXTLoad(RHS) && ISD::isNON_EXTLoad(LHS)) {
         if (DAG.areNonVolatileConsecutiveLoads(LHS, RHS, BitWidth / 8, 1)) {
           SDLoc DL(RHS);
           uint64_t PtrOff =
@@ -11024,9 +11013,8 @@ SDValue DAGCombiner::visitFunnelShift(SDNode *N) {
                 VT, DL, RHS->getChain(), NewPtr,
                 RHS->getPointerInfo().getWithOffset(PtrOff), NewAlign,
                 RHS->getMemOperand()->getFlags(), RHS->getAAInfo());
-            // Replace the old load's chain with the new load's chain.
-            WorklistRemover DeadNodes(*this);
-            DAG.ReplaceAllUsesOfValueWith(N1.getValue(1), Load.getValue(1));
+            DAG.makeEquivalentMemoryOrdering(LHS, Load.getValue(1));
+            DAG.makeEquivalentMemoryOrdering(RHS, Load.getValue(1));
             return Load;
           }
         }
@@ -22720,81 +22708,6 @@ SDValue DAGCombiner::visitINSERT_VECTOR_ELT(SDNode *N) {
   return SDValue();
 }
 
-SDValue DAGCombiner::scalarizeExtractedVectorLoad(SDNode *EVE, EVT InVecVT,
-                                                  SDValue EltNo,
-                                                  LoadSDNode *OriginalLoad) {
-  assert(OriginalLoad->isSimple());
-
-  EVT ResultVT = EVE->getValueType(0);
-  EVT VecEltVT = InVecVT.getVectorElementType();
-
-  // If the vector element type is not a multiple of a byte then we are unable
-  // to correctly compute an address to load only the extracted element as a
-  // scalar.
-  if (!VecEltVT.isByteSized())
-    return SDValue();
-
-  ISD::LoadExtType ExtTy =
-      ResultVT.bitsGT(VecEltVT) ? ISD::EXTLOAD : ISD::NON_EXTLOAD;
-  if (!TLI.isOperationLegalOrCustom(ISD::LOAD, VecEltVT) ||
-      !TLI.shouldReduceLoadWidth(OriginalLoad, ExtTy, VecEltVT))
-    return SDValue();
-
-  Align Alignment = OriginalLoad->getAlign();
-  MachinePointerInfo MPI;
-  SDLoc DL(EVE);
-  if (auto *ConstEltNo = dyn_cast<ConstantSDNode>(EltNo)) {
-    int Elt = ConstEltNo->getZExtValue();
-    unsigned PtrOff = VecEltVT.getSizeInBits() * Elt / 8;
-    MPI = OriginalLoad->getPointerInfo().getWithOffset(PtrOff);
-    Alignment = commonAlignment(Alignment, PtrOff);
-  } else {
-    // Discard the pointer info except the address space because the memory
-    // operand can't represent this new access since the offset is variable.
-    MPI = MachinePointerInfo(OriginalLoad->getPointerInfo().getAddrSpace());
-    Alignment = commonAlignment(Alignment, VecEltVT.getSizeInBits() / 8);
-  }
-
-  unsigned IsFast = 0;
-  if (!TLI.allowsMemoryAccess(*DAG.getContext(), DAG.getDataLayout(), VecEltVT,
-                              OriginalLoad->getAddressSpace(), Alignment,
-                              OriginalLoad->getMemOperand()->getFlags(),
-                              &IsFast) ||
-      !IsFast)
-    return SDValue();
-
-  SDValue NewPtr = TLI.getVectorElementPointer(DAG, OriginalLoad->getBasePtr(),
-                                               InVecVT, EltNo);
-
-  // We are replacing a vector load with a scalar load. The new load must have
-  // identical memory op ordering to the original.
-  SDValue Load;
-  if (ResultVT.bitsGT(VecEltVT)) {
-    // If the result type of vextract is wider than the load, then issue an
-    // extending load instead.
-    ISD::LoadExtType ExtType =
-        TLI.isLoadExtLegal(ISD::ZEXTLOAD, ResultVT, VecEltVT) ? ISD::ZEXTLOAD
-                                                              : ISD::EXTLOAD;
-    Load = DAG.getExtLoad(ExtType, DL, ResultVT, OriginalLoad->getChain(),
-                          NewPtr, MPI, VecEltVT, Alignment,
-                          OriginalLoad->getMemOperand()->getFlags(),
-                          OriginalLoad->getAAInfo());
-    DAG.makeEquivalentMemoryOrdering(OriginalLoad, Load);
-  } else {
-    // The result type is narrower or the same width as the vector element
-    Load = DAG.getLoad(VecEltVT, DL, OriginalLoad->getChain(), NewPtr, MPI,
-                       Alignment, OriginalLoad->getMemOperand()->getFlags(),
-                       OriginalLoad->getAAInfo());
-    DAG.makeEquivalentMemoryOrdering(OriginalLoad, Load);
-    if (ResultVT.bitsLT(VecEltVT))
-      Load = DAG.getNode(ISD::TRUNCATE, DL, ResultVT, Load);
-    else
-      Load = DAG.getBitcast(ResultVT, Load);
-  }
-  ++OpsNarrowed;
-  return Load;
-}
-
 /// Transform a vector binary operation into a scalar binary operation by moving
 /// the math/logic after an extract element of a vector.
 static SDValue scalarizeExtractedBinOp(SDNode *ExtElt, SelectionDAG &DAG,
@@ -23186,6 +23099,11 @@ SDValue DAGCombiner::visitEXTRACT_VECTOR_ELT(SDNode *N) {
     }
 
     if (SVInVec.getOpcode() == ISD::BUILD_VECTOR) {
+      // TODO: Check if shuffle mask is legal?
+      if (LegalOperations && TLI.isOperationLegal(ISD::VECTOR_SHUFFLE, VecVT) &&
+          !VecOp.hasOneUse())
+        return SDValue();
+
       SDValue InOp = SVInVec.getOperand(OrigElt);
       if (InOp.getValueType() != ScalarVT) {
         assert(InOp.getValueType().isInteger() && ScalarVT.isInteger());
@@ -23268,8 +23186,13 @@ SDValue DAGCombiner::visitEXTRACT_VECTOR_ELT(SDNode *N) {
       ISD::isNormalLoad(VecOp.getNode()) &&
       !Index->hasPredecessor(VecOp.getNode())) {
     auto *VecLoad = dyn_cast<LoadSDNode>(VecOp);
-    if (VecLoad && VecLoad->isSimple())
-      return scalarizeExtractedVectorLoad(N, VecVT, Index, VecLoad);
+    if (VecLoad && VecLoad->isSimple()) {
+      if (SDValue Scalarized = TLI.scalarizeExtractedVectorLoad(
+              ScalarVT, SDLoc(N), VecVT, Index, VecLoad, DAG)) {
+        ++OpsNarrowed;
+        return Scalarized;
+      }
+    }
   }
 
   // Perform only after legalization to ensure build_vector / vector_shuffle
@@ -23357,7 +23280,13 @@ SDValue DAGCombiner::visitEXTRACT_VECTOR_ELT(SDNode *N) {
   if (Elt == -1)
     return DAG.getUNDEF(LVT);
 
-  return scalarizeExtractedVectorLoad(N, VecVT, Index, LN0);
+  if (SDValue Scalarized =
+          TLI.scalarizeExtractedVectorLoad(LVT, DL, VecVT, Index, LN0, DAG)) {
+    ++OpsNarrowed;
+    return Scalarized;
+  }
+
+  return SDValue();
 }
 
 // Simplify (build_vec (ext )) to (bitcast (build_vec ))
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
index c6475f0219903..f56097fdbb51a 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
@@ -2284,7 +2284,7 @@ SelectionDAGLegalize::ExpandDivRemLibCall(SDNode *Node,
 
 /// Return true if sincos libcall is available.
 static bool isSinCosLibcallAvailable(SDNode *Node, const TargetLowering &TLI) {
-  RTLIB::Libcall LC = RTLIB::getFSINCOS(Node->getSimpleValueType(0).SimpleTy);
+  RTLIB::Libcall LC = RTLIB::getSINCOS(Node->getSimpleValueType(0).SimpleTy);
   return TLI.getLibcallName(LC) != nullptr;
 }
 
@@ -3970,8 +3970,7 @@ bool SelectionDAGLegalize::ExpandNode(SDNode *Node) {
     else
       Index = DAG.getNode(ISD::MUL, dl, Index.getValueType(), Index,
                           DAG.getConstant(EntrySize, dl, Index.getValueType()));
-    SDValue Addr = DAG.getNode(ISD::ADD, dl, Index.getValueType(),
-                               Index, Table);
+    SDValue Addr = DAG.getMemBasePlusOffset(Table, Index, dl);
 
     EVT MemVT = EVT::getIntegerVT(*DAG.getContext(), EntrySize * 8);
     SDValue LD = DAG.getExtLoad(
@@ -3980,10 +3979,10 @@ bool SelectionDAGLegalize::ExpandNode(SDNode *Node) {
     Addr = LD;
     if (TLI.isJumpTableRelative()) {
       // For PIC, the sequence is:
-      // BRIND(load(Jumptable + index) + RelocBase)
+      // BRIND(RelocBase + load(Jumptable + index))
       // RelocBase can be JumpTable, GOT or some sort of global base.
-      Addr = DAG.getNode(ISD::ADD, dl, PTy, Addr,
-                          TLI.getPICJumpTableRelocBase(Table, DAG));
+      Addr = DAG.getMemBasePlusOffset(TLI.getPICJumpTableRelocBase(Table, DAG),
+                                      Addr, dl);
     }
 
     Tmp1 = TLI.expandIndirectJTBranch(dl, LD.getValue(1), Addr, JTI, DAG);
@@ -4521,11 +4520,15 @@ void SelectionDAGLegalize::ConvertNodeToLibcall(SDNode *Node) {
     ExpandFPLibCall(Node, RTLIB::TANH_F32, RTLIB::TANH_F64, RTLIB::TANH_F80,
                     RTLIB::TANH_F128, RTLIB::TANH_PPCF128, Results);
     break;
-  case ISD::FSINCOS: {
-    RTLIB::Libcall LC = RTLIB::getFSINCOS(Node->getValueType(0));
+  case ISD::FSINCOS:
+  case ISD::FSINCOSPI: {
+    EVT VT = Node->getValueType(0);
+    RTLIB::Libcall LC = Node->getOpcode() == ISD::FSINCOS
+                            ? RTLIB::getSINCOS(VT)
+                            : RTLIB::getSINCOSPI(VT);
     bool Expanded = DAG.expandMultipleResultFPLibCall(LC, Node, Results);
     if (!Expanded)
-      llvm_unreachable("Expected scalar FSINCOS to expand to libcall!");
+      llvm_unreachable("Expected scalar FSINCOS[PI] to expand to libcall!");
     break;
   }
   case ISD::FLOG:
@@ -4610,12 +4613,15 @@ void SelectionDAGLegalize::ConvertNodeToLibcall(SDNode *Node) {
     ExpandFPLibCall(Node, RTLIB::LDEXP_F32, RTLIB::LDEXP_F64, RTLIB::LDEXP_F80,
                     RTLIB::LDEXP_F128, RTLIB::LDEXP_PPCF128, Results);
     break;
+  case ISD::FMODF:
   case ISD::FFREXP: {
-    RTLIB::Libcall LC = RTLIB::getFREXP(Node->getValueType(0));
+    EVT VT = Node->getValueType(0);
+    RTLIB::Libcall LC = Node->getOpcode() == ISD::FMODF ? RTLIB::getMODF(VT)
+                                                        : RTLIB::getFREXP(VT);
     bool Expanded = DAG.expandMultipleResultFPLibCall(LC, Node, Results,
                                                       /*CallRetResNo=*/0);
     if (!Expanded)
-      llvm_unreachable("Expected scalar FFREXP to expand to libcall!");
+      llvm_unreachable("Expected scalar FFREXP/FMODF to expand to libcall!");
     break;
   }
   case ISD::FPOWI:
@@ -5504,9 +5510,11 @@ void SelectionDAGLegalize::PromoteNode(SDNode *Node) {
     Results.push_back(Tmp2.getValue(1));
     break;
   }
-  case ISD::FSINCOS: {
+  case ISD::FMODF:
+  case ISD::FSINCOS:
+  case ISD::FSINCOSPI: {
     Tmp1 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(0));
-    Tmp2 = DAG.getNode(ISD::FSINCOS, dl, DAG.getVTList(NVT, NVT), Tmp1,
+    Tmp2 = DAG.getNode(Node->getOpcode(), dl, DAG.getVTList(NVT, NVT), Tmp1,
                        Node->getFlags());
     Tmp3 = DAG.getIntPtrConstant(0, dl, /*isTarget=*/true);
     for (unsigned ResNum = 0; ResNum < Node->getNumValues(); ResNum++)
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeFloatTypes.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeFloatTypes.cpp
index 71f100bfa0343..4b79bd28e2750 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeFloatTypes.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeFloatTypes.cpp
@@ -833,7 +833,7 @@ DAGTypeLegalizer::SoftenFloatRes_UnaryWithTwoFPResults(SDNode *N,
 
 SDValue DAGTypeLegalizer::SoftenFloatRes_FSINCOS(SDNode *N) {
   return SoftenFloatRes_UnaryWithTwoFPResults(
-      N, RTLIB::getFSINCOS(N->getValueType(0)));
+      N, RTLIB::getSINCOS(N->getValueType(0)));
 }
 
 SDValue DAGTypeLegalizer::SoftenFloatRes_FREM(SDNode *N) {
@@ -2766,10 +2766,11 @@ void DAGTypeLegalizer::PromoteFloatResult(SDNode *N, unsigned ResNo) {
     case ISD::FLDEXP:     R = PromoteFloatRes_ExpOp(N); break;
     case ISD::FFREXP:     R = PromoteFloatRes_FFREXP(N); break;
 
+    case ISD::FMODF:
     case ISD::FSINCOS:
+    case ISD::FSINCOSPI:
       R = PromoteFloatRes_UnaryWithTwoFPResults(N);
       break;
-
     case ISD::FP_ROUND:   R = PromoteFloatRes_FP_ROUND(N); break;
     case ISD::STRICT_FP_ROUND:
       R = PromoteFloatRes_STRICT_FP_ROUND(N);
@@ -3228,7 +3229,9 @@ void DAGTypeLegalizer::SoftPromoteHalfResult(SDNode *N, unsigned ResNo) {
 
   case ISD::FFREXP:      R = SoftPromoteHalfRes_FFREXP(N); break;
 
+  case ISD::FMODF:
   case ISD::FSINCOS:
+  case ISD::FSINCOSPI:
     R = SoftPromoteHalfRes_UnaryWithTwoFPResults(N);
     break;
 
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp
index b0a624680231e..a0f29496df777 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp
@@ -566,6 +566,29 @@ SDValue DAGTypeLegalizer::PromoteIntRes_BITCAST(SDNode *N) {
     }
   }
 
+  // TODO: Handle big endian
+  if (!NOutVT.isVector() && InOp.getValueType().isVector() &&
+      DAG.getDataLayout().isLittleEndian()) {
+    // Pad the vector operand with undef and cast to a wider integer.
+    EVT EltVT = InOp.getValueType().getVectorElementType();
+    TypeSize EltSize = EltVT.getSizeInBits();
+    TypeSize OutSize = NOutVT.getSizeInBits();
+
+    if (OutSize.hasKnownScalarFactor(EltSize)) {
+      unsigned NumEltsWithPadding = OutSize.getKnownScalarFactor(EltSize);
+      EVT WideVecVT =
+          EVT::getVectorVT(*DAG.getContext(), EltVT, NumEltsWithPadding);
+
+      if (isTypeLegal(WideVecVT)) {
+        SDValue Inserted = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, WideVecVT,
+                                       DAG.getUNDEF(WideVecVT), InOp,
+                                       DAG.getVectorIdxConstant(0, dl));
+
+        return DAG.getNode(ISD::BITCAST, dl, NOutVT, Inserted);
+      }
+    }
+  }
+
   return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT,
                      CreateStackStoreLoad(InOp, OutVT));
 }
@@ -2181,9 +2204,43 @@ SDValue DAGTypeLegalizer::PromoteIntOp_ATOMIC_STORE(AtomicSDNode *N) {
 }
 
 SDValue DAGTypeLegalizer::PromoteIntOp_BITCAST(SDNode *N) {
+  EVT OutVT = N->getValueType(0);
+  SDValue InOp = N->getOperand(0);
+  EVT InVT = InOp.getValueType();
+  EVT NInVT = TLI.getTypeToTransformTo(*DAG.getContext(), InVT);
+  SDLoc dl(N);
+
+  switch (getTypeAction(InVT)) {
+  case TargetLowering::TypePromoteInteger: {
+    // TODO: Handle big endian
+    if (OutVT.isVector() && DAG.getDataLayout().isLittleEndian()) {
+      EVT EltVT = OutVT.getVectorElementType();
+      TypeSize EltSize = EltVT.getSizeInBits();
+      TypeSize NInSize = NInVT.getSizeInBits();
+
+      if (NInSize.hasKnownScalarFactor(EltSize)) {
+        unsigned NumEltsWithPadding = NInSize.getKnownScalarFactor(EltSize);
+        EVT WideVecVT =
+            EVT::getVectorVT(*DAG.getContext(), EltVT, NumEltsWithPadding);
+
+        if (isTypeLegal(WideVecVT)) {
+          SDValue Promoted = GetPromotedInteger(InOp);
+          SDValue Cast = DAG.getNode(ISD::BITCAST, dl, WideVecVT, Promoted);
+          return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, OutVT, Cast,
+                             DAG.getVectorIdxConstant(0, dl));
+        }
+      }
+    }
+
+    break;
+  }
+  default:
+    break;
+  }
+
   // This should only occur in unusual situations like bitcasting to an
   // x86_fp80, so just turn it into a store+load
-  return CreateStackStoreLoad(N->getOperand(0), N->getValueType(0));
+  return CreateStackStoreLoad(InOp, OutVT);
 }
 
 SDValue DAGTypeLegalizer::PromoteIntOp_BR_CC(SDNode *N, unsigned OpNo) {
@@ -4296,8 +4353,7 @@ void DAGTypeLegalizer::ExpandIntRes_MUL(SDNode *N,
   if (LC == RTLIB::UNKNOWN_LIBCALL || !TLI.getLibcallName(LC)) {
     // Perform a wide multiplication where the wide type is the original VT and
     // the 4 parts are the split arguments.
-    TLI.forceExpandWideMUL(DAG, dl, /*Signed=*/true, VT, LL, LH, RL, RH, Lo,
-                           Hi);
+    TLI.forceExpandMultiply(DAG, dl, /*Signed=*/false, Lo, Hi, LL, RL, LH, RH);
     return;
   }
 
@@ -5825,15 +5881,19 @@ SDValue DAGTypeLegalizer::PromoteIntRes_VECTOR_SPLICE(SDNode *N) {
 }
 
 SDValue DAGTypeLegalizer::PromoteIntRes_VECTOR_INTERLEAVE_DEINTERLEAVE(SDNode *N) {
-  SDLoc dl(N);
+  SDLoc DL(N);
+  unsigned Factor = N->getNumOperands();
+
+  SmallVector<SDValue, 8> Ops(Factor);
+  for (unsigned i = 0; i != Factor; i++)
+    Ops[i] = GetPromotedInteger(N->getOperand(i));
+
+  SmallVector<EVT, 8> ResVTs(Factor, Ops[0].getValueType());
+  SDValue Res = DAG.getNode(N->getOpcode(), DL, DAG.getVTList(ResVTs), Ops);
+
+  for (unsigned i = 0; i != Factor; i++)
+    SetPromotedInteger(SDValue(N, i), Res.getValue(i));
 
-  SDValue V0 = GetPromotedInteger(N->getOperand(0));
-  SDValue V1 = GetPromotedInteger(N->getOperand(1));
-  EVT ResVT = V0.getValueType();
-  SDValue Res = DAG.getNode(N->getOpcode(), dl,
-                            DAG.getVTList(ResVT, ResVT), V0, V1);
-  SetPromotedInteger(SDValue(N, 0), Res.getValue(0));
-  SetPromotedInteger(SDValue(N, 1), Res.getValue(1));
   return SDValue();
 }
 
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp
index 6ad08bce44b0a..7e8bae4b0f785 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp
@@ -454,7 +454,9 @@ SDValue VectorLegalizer::LegalizeOp(SDValue Op) {
   case ISD::UMULO:
   case ISD::FCANONICALIZE:
   case ISD::FFREXP:
+  case ISD::FMODF:
   case ISD::FSINCOS:
+  case ISD::FSINCOSPI:
   case ISD::SADDSAT:
   case ISD::UADDSAT:
   case ISD::SSUBSAT:
@@ -1216,13 +1218,24 @@ void VectorLegalizer::Expand(SDNode *Node, SmallVectorImpl<SDValue> &Results) {
       return;
 
     break;
-  case ISD::FSINCOS: {
-    RTLIB::Libcall LC =
-        RTLIB::getFSINCOS(Node->getValueType(0).getVectorElementType());
+  case ISD::FSINCOS:
+  case ISD::FSINCOSPI: {
+    EVT VT = Node->getValueType(0).getVectorElementType();
+    RTLIB::Libcall LC = Node->getOpcode() == ISD::FSINCOS
+                            ? RTLIB::getSINCOS(VT)
+                            : RTLIB::getSINCOSPI(VT);
     if (DAG.expandMultipleResultFPLibCall(LC, Node, Results))
       return;
     break;
   }
+  case ISD::FMODF: {
+    RTLIB::Libcall LC =
+        RTLIB::getMODF(Node->getValueType(0).getVectorElementType());
+    if (DAG.expandMultipleResultFPLibCall(LC, Node, Results,
+                                          /*CallRetResNo=*/0))
+      return;
+    break;
+  }
   case ISD::VECTOR_COMPRESS:
     Results.push_back(TLI.expandVECTOR_COMPRESS(Node, DAG));
     return;
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
index f39d9ca15496a..915ee2d110332 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
@@ -133,8 +133,10 @@ void DAGTypeLegalizer::ScalarizeVectorResult(SDNode *N, unsigned ResNo) {
   case ISD::ADDRSPACECAST:
     R = ScalarizeVecRes_ADDRSPACECAST(N);
     break;
+  case ISD::FMODF:
   case ISD::FFREXP:
   case ISD::FSINCOS:
+  case ISD::FSINCOSPI:
     R = ScalarizeVecRes_UnaryOpWithTwoResults(N, ResNo);
     break;
   case ISD::ADD:
@@ -1261,8 +1263,10 @@ void DAGTypeLegalizer::SplitVectorResult(SDNode *N, unsigned ResNo) {
   case ISD::ADDRSPACECAST:
     SplitVecRes_ADDRSPACECAST(N, Lo, Hi);
     break;
+  case ISD::FMODF:
   case ISD::FFREXP:
   case ISD::FSINCOS:
+  case ISD::FSINCOSPI:
     SplitVecRes_UnaryOpWithTwoResults(N, ResNo, Lo, Hi);
     break;
 
@@ -1668,6 +1672,15 @@ void DAGTypeLegalizer::SplitVecRes_INSERT_SUBVECTOR(SDNode *N, SDValue &Lo,
     return;
   }
 
+  if (getTypeAction(SubVecVT) == TargetLowering::TypeWidenVector &&
+      Vec.isUndef() && SubVecVT.getVectorElementType() == MVT::i1) {
+    SDValue WideSubVec = GetWidenedVector(SubVec);
+    if (WideSubVec.getValueType() == VecVT) {
+      std::tie(Lo, Hi) = DAG.SplitVector(WideSubVec, SDLoc(WideSubVec));
+      return;
+    }
+  }
+
   // Spill the vector to the stack.
   // In cases where the vector is illegal it will be broken down into parts
   // and stored in parts - we should use the alignment for the smallest part.
@@ -3183,34 +3196,53 @@ void DAGTypeLegalizer::SplitVecRes_VP_REVERSE(SDNode *N, SDValue &Lo,
 }
 
 void DAGTypeLegalizer::SplitVecRes_VECTOR_DEINTERLEAVE(SDNode *N) {
+  unsigned Factor = N->getNumOperands();
+
+  SmallVector<SDValue, 8> Ops(Factor * 2);
+  for (unsigned i = 0; i != Factor; ++i) {
+    SDValue OpLo, OpHi;
+    GetSplitVector(N->getOperand(i), OpLo, OpHi);
+    Ops[i * 2] = OpLo;
+    Ops[i * 2 + 1] = OpHi;
+  }
+
+  SmallVector<EVT, 8> VTs(Factor, Ops[0].getValueType());
 
-  SDValue Op0Lo, Op0Hi, Op1Lo, Op1Hi;
-  GetSplitVector(N->getOperand(0), Op0Lo, Op0Hi);
-  GetSplitVector(N->getOperand(1), Op1Lo, Op1Hi);
-  EVT VT = Op0Lo.getValueType();
   SDLoc DL(N);
-  SDValue ResLo = DAG.getNode(ISD::VECTOR_DEINTERLEAVE, DL,
-                              DAG.getVTList(VT, VT), Op0Lo, Op0Hi);
-  SDValue ResHi = DAG.getNode(ISD::VECTOR_DEINTERLEAVE, DL,
-                              DAG.getVTList(VT, VT), Op1Lo, Op1Hi);
+  SDValue ResLo = DAG.getNode(ISD::VECTOR_DEINTERLEAVE, DL, VTs,
+                              ArrayRef(Ops).slice(0, Factor));
+  SDValue ResHi = DAG.getNode(ISD::VECTOR_DEINTERLEAVE, DL, VTs,
+                              ArrayRef(Ops).slice(Factor, Factor));
 
-  SetSplitVector(SDValue(N, 0), ResLo.getValue(0), ResHi.getValue(0));
-  SetSplitVector(SDValue(N, 1), ResLo.getValue(1), ResHi.getValue(1));
+  for (unsigned i = 0; i != Factor; ++i)
+    SetSplitVector(SDValue(N, i), ResLo.getValue(i), ResHi.getValue(i));
 }
 
 void DAGTypeLegalizer::SplitVecRes_VECTOR_INTERLEAVE(SDNode *N) {
-  SDValue Op0Lo, Op0Hi, Op1Lo, Op1Hi;
-  GetSplitVector(N->getOperand(0), Op0Lo, Op0Hi);
-  GetSplitVector(N->getOperand(1), Op1Lo, Op1Hi);
-  EVT VT = Op0Lo.getValueType();
+  unsigned Factor = N->getNumOperands();
+
+  SmallVector<SDValue, 8> Ops(Factor * 2);
+  for (unsigned i = 0; i != Factor; ++i) {
+    SDValue OpLo, OpHi;
+    GetSplitVector(N->getOperand(i), OpLo, OpHi);
+    Ops[i] = OpLo;
+    Ops[i + Factor] = OpHi;
+  }
+
+  SmallVector<EVT, 8> VTs(Factor, Ops[0].getValueType());
+
   SDLoc DL(N);
-  SDValue Res[] = {DAG.getNode(ISD::VECTOR_INTERLEAVE, DL,
-                               DAG.getVTList(VT, VT), Op0Lo, Op1Lo),
-                   DAG.getNode(ISD::VECTOR_INTERLEAVE, DL,
-                               DAG.getVTList(VT, VT), Op0Hi, Op1Hi)};
+  SDValue Res[] = {DAG.getNode(ISD::VECTOR_INTERLEAVE, DL, VTs,
+                               ArrayRef(Ops).slice(0, Factor)),
+                   DAG.getNode(ISD::VECTOR_INTERLEAVE, DL, VTs,
+                               ArrayRef(Ops).slice(Factor, Factor))};
 
-  SetSplitVector(SDValue(N, 0), Res[0].getValue(0), Res[0].getValue(1));
-  SetSplitVector(SDValue(N, 1), Res[1].getValue(0), Res[1].getValue(1));
+  for (unsigned i = 0; i != Factor; ++i) {
+    unsigned IdxLo = 2 * i;
+    unsigned IdxHi = 2 * i + 1;
+    SetSplitVector(SDValue(N, i), Res[IdxLo / Factor].getValue(IdxLo % Factor),
+                   Res[IdxHi / Factor].getValue(IdxHi % Factor));
+  }
 }
 
 //===----------------------------------------------------------------------===//
@@ -4783,8 +4815,10 @@ void DAGTypeLegalizer::WidenVectorResult(SDNode *N, unsigned ResNo) {
   case ISD::VP_FSHR:
     Res = WidenVecRes_Ternary(N);
     break;
+  case ISD::FMODF:
   case ISD::FFREXP:
-  case ISD::FSINCOS: {
+  case ISD::FSINCOS:
+  case ISD::FSINCOSPI: {
     if (!unrollExpandedOp())
       Res = WidenVecRes_UnaryOpWithTwoResults(N, ResNo);
     break;
@@ -7040,6 +7074,7 @@ SDValue DAGTypeLegalizer::WidenVecOp_INSERT_SUBVECTOR(SDNode *N) {
   SDValue SubVec = N->getOperand(1);
   SDValue InVec = N->getOperand(0);
 
+  EVT OrigVT = SubVec.getValueType();
   if (getTypeAction(SubVec.getValueType()) == TargetLowering::TypeWidenVector)
     SubVec = GetWidenedVector(SubVec);
 
@@ -7064,14 +7099,34 @@ SDValue DAGTypeLegalizer::WidenVecOp_INSERT_SUBVECTOR(SDNode *N) {
     }
   }
 
+  SDLoc DL(N);
+
   // We need to make sure that the indices are still valid, otherwise we might
   // widen what was previously well-defined to something undefined.
   if (IndicesValid && InVec.isUndef() && N->getConstantOperandVal(2) == 0)
-    return DAG.getNode(ISD::INSERT_SUBVECTOR, SDLoc(N), VT, InVec, SubVec,
+    return DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT, InVec, SubVec,
                        N->getOperand(2));
 
-  report_fatal_error("Don't know how to widen the operands for "
-                     "INSERT_SUBVECTOR");
+  if (!IndicesValid || OrigVT.isScalableVector())
+    report_fatal_error(
+        "Don't know how to widen the operands for INSERT_SUBVECTOR");
+
+  // If the operands can't be widened legally, just replace the INSERT_SUBVECTOR
+  // with a series of INSERT_VECTOR_ELT
+  unsigned Idx = N->getConstantOperandVal(2);
+
+  SDValue InsertElt = InVec;
+  EVT VectorIdxTy = TLI.getVectorIdxTy(DAG.getDataLayout());
+  for (unsigned I = 0, E = OrigVT.getVectorNumElements(); I != E; ++I) {
+    SDValue ExtractElt =
+        DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT.getVectorElementType(),
+                    SubVec, DAG.getConstant(I, DL, VectorIdxTy));
+    InsertElt =
+        DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, VT, InsertElt, ExtractElt,
+                    DAG.getConstant(I + Idx, DL, VectorIdxTy));
+  }
+
+  return InsertElt;
 }
 
 SDValue DAGTypeLegalizer::WidenVecOp_EXTRACT_SUBVECTOR(SDNode *N) {
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
index 0f9790a10a139..164493d6dd860 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
@@ -363,7 +363,7 @@ bool ISD::isFreezeUndef(const SDNode *N) {
 template <typename ConstNodeType>
 bool ISD::matchUnaryPredicateImpl(SDValue Op,
                                   std::function<bool(ConstNodeType *)> Match,
-                                  bool AllowUndefs) {
+                                  bool AllowUndefs, bool AllowTruncation) {
   // FIXME: Add support for scalar UNDEF cases?
   if (auto *C = dyn_cast<ConstNodeType>(Op))
     return Match(C);
@@ -382,16 +382,17 @@ bool ISD::matchUnaryPredicateImpl(SDValue Op,
     }
 
     auto *Cst = dyn_cast<ConstNodeType>(Op.getOperand(i));
-    if (!Cst || Cst->getValueType(0) != SVT || !Match(Cst))
+    if (!Cst || (!AllowTruncation && Cst->getValueType(0) != SVT) ||
+        !Match(Cst))
       return false;
   }
   return true;
 }
 // Build used template types.
 template bool ISD::matchUnaryPredicateImpl<ConstantSDNode>(
-    SDValue, std::function<bool(ConstantSDNode *)>, bool);
+    SDValue, std::function<bool(ConstantSDNode *)>, bool, bool);
 template bool ISD::matchUnaryPredicateImpl<ConstantFPSDNode>(
-    SDValue, std::function<bool(ConstantFPSDNode *)>, bool);
+    SDValue, std::function<bool(ConstantFPSDNode *)>, bool, bool);
 
 bool ISD::matchBinaryPredicate(
     SDValue LHS, SDValue RHS,
@@ -7296,15 +7297,15 @@ SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
     // it's worth handling here.
     if (N2CV && N2CV->isZero())
       return N1;
-    if ((Opcode == ISD::ADD || Opcode == ISD::SUB) && VT.isVector() &&
-        VT.getVectorElementType() == MVT::i1)
+    if ((Opcode == ISD::ADD || Opcode == ISD::SUB) &&
+        VT.getScalarType() == MVT::i1)
       return getNode(ISD::XOR, DL, VT, N1, N2);
     break;
   case ISD::MUL:
     assert(VT.isInteger() && "This operator does not apply to FP types!");
     assert(N1.getValueType() == N2.getValueType() &&
            N1.getValueType() == VT && "Binary operator types must match!");
-    if (VT.isVector() && VT.getVectorElementType() == MVT::i1)
+    if (VT.getScalarType() == MVT::i1)
       return getNode(ISD::AND, DL, VT, N1, N2);
     if (N2C && (N1.getOpcode() == ISD::VSCALE) && Flags.hasNoSignedWrap()) {
       const APInt &MulImm = N1->getConstantOperandAPInt(0);
@@ -7325,7 +7326,7 @@ SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
     assert(VT.isInteger() && "This operator does not apply to FP types!");
     assert(N1.getValueType() == N2.getValueType() &&
            N1.getValueType() == VT && "Binary operator types must match!");
-    if (VT.isVector() && VT.getVectorElementType() == MVT::i1) {
+    if (VT.getScalarType() == MVT::i1) {
       // fold (add_sat x, y) -> (or x, y) for bool types.
       if (Opcode == ISD::SADDSAT || Opcode == ISD::UADDSAT)
         return getNode(ISD::OR, DL, VT, N1, N2);
@@ -7358,7 +7359,7 @@ SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
     assert(VT.isInteger() && "This operator does not apply to FP types!");
     assert(N1.getValueType() == N2.getValueType() &&
            N1.getValueType() == VT && "Binary operator types must match!");
-    if (VT.isVector() && VT.getVectorElementType() == MVT::i1)
+    if (VT.getScalarType() == MVT::i1)
       return getNode(ISD::XOR, DL, VT, N1, N2);
     break;
   case ISD::SMIN:
@@ -7366,7 +7367,7 @@ SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
     assert(VT.isInteger() && "This operator does not apply to FP types!");
     assert(N1.getValueType() == N2.getValueType() &&
            N1.getValueType() == VT && "Binary operator types must match!");
-    if (VT.isVector() && VT.getVectorElementType() == MVT::i1)
+    if (VT.getScalarType() == MVT::i1)
       return getNode(ISD::OR, DL, VT, N1, N2);
     break;
   case ISD::SMAX:
@@ -7374,7 +7375,7 @@ SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
     assert(VT.isInteger() && "This operator does not apply to FP types!");
     assert(N1.getValueType() == N2.getValueType() &&
            N1.getValueType() == VT && "Binary operator types must match!");
-    if (VT.isVector() && VT.getVectorElementType() == MVT::i1)
+    if (VT.getScalarType() == MVT::i1)
       return getNode(ISD::AND, DL, VT, N1, N2);
     break;
   case ISD::FADD:
@@ -10398,12 +10399,12 @@ SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
   case ISD::VP_ADD:
   case ISD::VP_SUB:
     // If it is VP_ADD/VP_SUB mask operation then turn it to VP_XOR
-    if (VT.isVector() && VT.getVectorElementType() == MVT::i1)
+    if (VT.getScalarType() == MVT::i1)
       Opcode = ISD::VP_XOR;
     break;
   case ISD::VP_MUL:
     // If it is VP_MUL mask operation then turn it to VP_AND
-    if (VT.isVector() && VT.getVectorElementType() == MVT::i1)
+    if (VT.getScalarType() == MVT::i1)
       Opcode = ISD::VP_AND;
     break;
   case ISD::VP_REDUCE_MUL:
@@ -10508,9 +10509,8 @@ SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList,
       return getNode(ISD::MERGE_VALUES, DL, VTList, {N1, ZeroOverFlow}, Flags);
     }
 
-    if (VTList.VTs[0].isVector() &&
-        VTList.VTs[0].getVectorElementType() == MVT::i1 &&
-        VTList.VTs[1].getVectorElementType() == MVT::i1) {
+    if (VTList.VTs[0].getScalarType() == MVT::i1 &&
+        VTList.VTs[1].getScalarType() == MVT::i1) {
       SDValue F1 = getFreeze(N1);
       SDValue F2 = getFreeze(N2);
       // {vXi1,vXi1} (u/s)addo(vXi1 x, vXi1y) -> {xor(x,y),and(x,y)}
@@ -13415,10 +13415,10 @@ BuildVectorSDNode::isConstantSequence() const {
   return std::make_pair(Start, Stride);
 }
 
-bool ShuffleVectorSDNode::isSplatMask(const int *Mask, EVT VT) {
+bool ShuffleVectorSDNode::isSplatMask(ArrayRef<int> Mask) {
   // Find the first non-undef value in the shuffle mask.
   unsigned i, e;
-  for (i = 0, e = VT.getVectorNumElements(); i != e && Mask[i] < 0; ++i)
+  for (i = 0, e = Mask.size(); i != e && Mask[i] < 0; ++i)
     /* search */;
 
   // If all elements are undefined, this shuffle can be considered a splat
@@ -13635,7 +13635,7 @@ void SelectionDAG::copyExtraInfo(SDNode *From, SDNode *To) {
   // Use of operator[] on the DenseMap may cause an insertion, which invalidates
   // the iterator, hence the need to make a copy to prevent a use-after-free.
   NodeExtraInfo NEI = I->second;
-  if (LLVM_LIKELY(!NEI.PCSections) && LLVM_LIKELY(!NEI.MMRA)) {
+  if (LLVM_LIKELY(!NEI.PCSections)) {
     // No deep copy required for the types of extra info set.
     //
     // FIXME: Investigate if other types of extra info also need deep copy. This
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
index 428e7a316d247..5a5596a542f72 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
@@ -97,7 +97,6 @@
 #include "llvm/Support/InstructionCost.h"
 #include "llvm/Support/MathExtras.h"
 #include "llvm/Support/raw_ostream.h"
-#include "llvm/Target/TargetIntrinsicInfo.h"
 #include "llvm/Target/TargetMachine.h"
 #include "llvm/Target/TargetOptions.h"
 #include "llvm/TargetParser/Triple.h"
@@ -6977,7 +6976,9 @@ void SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I,
                              getValue(I.getArgOperand(0)),
                              getValue(I.getArgOperand(1)), Flags));
     return;
+  case Intrinsic::modf:
   case Intrinsic::sincos:
+  case Intrinsic::sincospi:
   case Intrinsic::frexp: {
     unsigned Opcode;
     switch (Intrinsic) {
@@ -6986,6 +6987,12 @@ void SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I,
     case Intrinsic::sincos:
       Opcode = ISD::FSINCOS;
       break;
+    case Intrinsic::sincospi:
+      Opcode = ISD::FSINCOSPI;
+      break;
+    case Intrinsic::modf:
+      Opcode = ISD::FMODF;
+      break;
     case Intrinsic::frexp:
       Opcode = ISD::FFREXP;
       break;
@@ -8251,10 +8258,28 @@ void SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I,
     visitCallBrLandingPad(I);
     return;
   case Intrinsic::vector_interleave2:
-    visitVectorInterleave(I);
+    visitVectorInterleave(I, 2);
+    return;
+  case Intrinsic::vector_interleave3:
+    visitVectorInterleave(I, 3);
+    return;
+  case Intrinsic::vector_interleave5:
+    visitVectorInterleave(I, 5);
+    return;
+  case Intrinsic::vector_interleave7:
+    visitVectorInterleave(I, 7);
     return;
   case Intrinsic::vector_deinterleave2:
-    visitVectorDeinterleave(I);
+    visitVectorDeinterleave(I, 2);
+    return;
+  case Intrinsic::vector_deinterleave3:
+    visitVectorDeinterleave(I, 3);
+    return;
+  case Intrinsic::vector_deinterleave5:
+    visitVectorDeinterleave(I, 5);
+    return;
+  case Intrinsic::vector_deinterleave7:
+    visitVectorDeinterleave(I, 7);
     return;
   case Intrinsic::experimental_vector_compress:
     setValue(&I, DAG.getNode(ISD::VECTOR_COMPRESS, sdl,
@@ -9342,13 +9367,8 @@ void SelectionDAGBuilder::visitCall(const CallInst &I) {
 
   if (Function *F = I.getCalledFunction()) {
     if (F->isDeclaration()) {
-      // Is this an LLVM intrinsic or a target-specific intrinsic?
-      unsigned IID = F->getIntrinsicID();
-      if (!IID)
-        if (const TargetIntrinsicInfo *II = TM.getIntrinsicInfo())
-          IID = II->getIntrinsicID(F);
-
-      if (IID) {
+      // Is this an LLVM intrinsic?
+      if (unsigned IID = F->getIntrinsicID()) {
         visitIntrinsicCall(I, IID);
         return;
       }
@@ -12565,26 +12585,31 @@ void SelectionDAGBuilder::visitVectorReverse(const CallInst &I) {
   setValue(&I, DAG.getVectorShuffle(VT, DL, V, DAG.getUNDEF(VT), Mask));
 }
 
-void SelectionDAGBuilder::visitVectorDeinterleave(const CallInst &I) {
+void SelectionDAGBuilder::visitVectorDeinterleave(const CallInst &I,
+                                                  unsigned Factor) {
   auto DL = getCurSDLoc();
   SDValue InVec = getValue(I.getOperand(0));
-  EVT OutVT =
-      InVec.getValueType().getHalfNumVectorElementsVT(*DAG.getContext());
 
+  SmallVector<EVT, 4> ValueVTs;
+  ComputeValueVTs(DAG.getTargetLoweringInfo(), DAG.getDataLayout(), I.getType(),
+                  ValueVTs);
+
+  EVT OutVT = ValueVTs[0];
   unsigned OutNumElts = OutVT.getVectorMinNumElements();
 
-  // ISD Node needs the input vectors split into two equal parts
-  SDValue Lo = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, OutVT, InVec,
-                           DAG.getVectorIdxConstant(0, DL));
-  SDValue Hi = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, OutVT, InVec,
-                           DAG.getVectorIdxConstant(OutNumElts, DL));
+  SmallVector<SDValue, 4> SubVecs(Factor);
+  for (unsigned i = 0; i != Factor; ++i) {
+    assert(ValueVTs[i] == OutVT && "Expected VTs to be the same");
+    SubVecs[i] = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, OutVT, InVec,
+                             DAG.getVectorIdxConstant(OutNumElts * i, DL));
+  }
 
-  // Use VECTOR_SHUFFLE for fixed-length vectors to benefit from existing
-  // legalisation and combines.
-  if (OutVT.isFixedLengthVector()) {
-    SDValue Even = DAG.getVectorShuffle(OutVT, DL, Lo, Hi,
+  // Use VECTOR_SHUFFLE for fixed-length vectors with factor of 2 to benefit
+  // from existing legalisation and combines.
+  if (OutVT.isFixedLengthVector() && Factor == 2) {
+    SDValue Even = DAG.getVectorShuffle(OutVT, DL, SubVecs[0], SubVecs[1],
                                         createStrideMask(0, 2, OutNumElts));
-    SDValue Odd = DAG.getVectorShuffle(OutVT, DL, Lo, Hi,
+    SDValue Odd = DAG.getVectorShuffle(OutVT, DL, SubVecs[0], SubVecs[1],
                                        createStrideMask(1, 2, OutNumElts));
     SDValue Res = DAG.getMergeValues({Even, Odd}, getCurSDLoc());
     setValue(&I, Res);
@@ -12592,32 +12617,43 @@ void SelectionDAGBuilder::visitVectorDeinterleave(const CallInst &I) {
   }
 
   SDValue Res = DAG.getNode(ISD::VECTOR_DEINTERLEAVE, DL,
-                            DAG.getVTList(OutVT, OutVT), Lo, Hi);
+                            DAG.getVTList(ValueVTs), SubVecs);
   setValue(&I, Res);
 }
 
-void SelectionDAGBuilder::visitVectorInterleave(const CallInst &I) {
+void SelectionDAGBuilder::visitVectorInterleave(const CallInst &I,
+                                                unsigned Factor) {
   auto DL = getCurSDLoc();
-  EVT InVT = getValue(I.getOperand(0)).getValueType();
-  SDValue InVec0 = getValue(I.getOperand(0));
-  SDValue InVec1 = getValue(I.getOperand(1));
   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
+  EVT InVT = getValue(I.getOperand(0)).getValueType();
   EVT OutVT = TLI.getValueType(DAG.getDataLayout(), I.getType());
 
-  // Use VECTOR_SHUFFLE for fixed-length vectors to benefit from existing
-  // legalisation and combines.
-  if (OutVT.isFixedLengthVector()) {
+  SmallVector<SDValue, 8> InVecs(Factor);
+  for (unsigned i = 0; i < Factor; ++i) {
+    InVecs[i] = getValue(I.getOperand(i));
+    assert(InVecs[i].getValueType() == InVecs[0].getValueType() &&
+           "Expected VTs to be the same");
+  }
+
+  // Use VECTOR_SHUFFLE for fixed-length vectors with factor of 2 to benefit
+  // from existing legalisation and combines.
+  if (OutVT.isFixedLengthVector() && Factor == 2) {
     unsigned NumElts = InVT.getVectorMinNumElements();
-    SDValue V = DAG.getNode(ISD::CONCAT_VECTORS, DL, OutVT, InVec0, InVec1);
+    SDValue V = DAG.getNode(ISD::CONCAT_VECTORS, DL, OutVT, InVecs);
     setValue(&I, DAG.getVectorShuffle(OutVT, DL, V, DAG.getUNDEF(OutVT),
                                       createInterleaveMask(NumElts, 2)));
     return;
   }
 
-  SDValue Res = DAG.getNode(ISD::VECTOR_INTERLEAVE, DL,
-                            DAG.getVTList(InVT, InVT), InVec0, InVec1);
-  Res = DAG.getNode(ISD::CONCAT_VECTORS, DL, OutVT, Res.getValue(0),
-                    Res.getValue(1));
+  SmallVector<EVT, 8> ValueVTs(Factor, InVT);
+  SDValue Res =
+      DAG.getNode(ISD::VECTOR_INTERLEAVE, DL, DAG.getVTList(ValueVTs), InVecs);
+
+  SmallVector<SDValue, 8> Results(Factor);
+  for (unsigned i = 0; i < Factor; ++i)
+    Results[i] = Res.getValue(i);
+
+  Res = DAG.getNode(ISD::CONCAT_VECTORS, DL, OutVT, Results);
   setValue(&I, Res);
 }
 
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h
index ed85deef64fa7..ece48c9bedf72 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h
@@ -659,8 +659,8 @@ class SelectionDAGBuilder {
   void visitVectorReduce(const CallInst &I, unsigned Intrinsic);
   void visitVectorReverse(const CallInst &I);
   void visitVectorSplice(const CallInst &I);
-  void visitVectorInterleave(const CallInst &I);
-  void visitVectorDeinterleave(const CallInst &I);
+  void visitVectorInterleave(const CallInst &I, unsigned Factor);
+  void visitVectorDeinterleave(const CallInst &I, unsigned Factor);
   void visitStepVector(const CallInst &I);
 
   void visitUserOp1(const Instruction &I) {
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp
index f63c8dd3df1c8..8de537173e52c 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp
@@ -43,7 +43,6 @@
 #include "llvm/Support/ErrorHandling.h"
 #include "llvm/Support/Printable.h"
 #include "llvm/Support/raw_ostream.h"
-#include "llvm/Target/TargetIntrinsicInfo.h"
 #include "llvm/Target/TargetMachine.h"
 #include <cstdint>
 #include <iterator>
@@ -164,8 +163,6 @@ std::string SDNode::getOperationName(const SelectionDAG *G) const {
       return Intrinsic::getBaseName((Intrinsic::ID)IID).str();
     if (!G)
       return "Unknown intrinsic";
-    if (const TargetIntrinsicInfo *TII = G->getTarget().getIntrinsicInfo())
-      return TII->getName(IID);
     llvm_unreachable("Invalid intrinsic ID");
   }
 
@@ -219,6 +216,8 @@ std::string SDNode::getOperationName(const SelectionDAG *G) const {
   case ISD::FCOS:                       return "fcos";
   case ISD::STRICT_FCOS:                return "strict_fcos";
   case ISD::FSINCOS:                    return "fsincos";
+  case ISD::FSINCOSPI:                  return "fsincospi";
+  case ISD::FMODF:                      return "fmodf";
   case ISD::FTAN:                       return "ftan";
   case ISD::STRICT_FTAN:                return "strict_ftan";
   case ISD::FASIN:                      return "fasin";
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp
index 899f83bbc6064..61e5aa270bc11 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp
@@ -98,7 +98,6 @@
 #include "llvm/Support/KnownBits.h"
 #include "llvm/Support/Timer.h"
 #include "llvm/Support/raw_ostream.h"
-#include "llvm/Target/TargetIntrinsicInfo.h"
 #include "llvm/Target/TargetMachine.h"
 #include "llvm/Target/TargetOptions.h"
 #include "llvm/Transforms/Utils/BasicBlockUtils.h"
@@ -698,7 +697,7 @@ bool SelectionDAGISel::runOnMachineFunction(MachineFunction &mf) {
         Def->getParent()->insert(std::next(InsertPos), MI);
       } else
         LLVM_DEBUG(dbgs() << "Dropping debug info for dead vreg"
-                          << Register::virtReg2Index(Reg) << "\n");
+                          << printReg(Reg) << '\n');
     }
 
     // Don't try and extend through copies in instruction referencing mode.
@@ -4424,8 +4423,6 @@ void SelectionDAGISel::CannotYetSelect(SDNode *N) {
     unsigned iid = N->getConstantOperandVal(HasInputChain);
     if (iid < Intrinsic::num_intrinsics)
       Msg << "intrinsic %" << Intrinsic::getBaseName((Intrinsic::ID)iid);
-    else if (const TargetIntrinsicInfo *TII = TM.getIntrinsicInfo())
-      Msg << "target intrinsic %" << TII->getName(iid);
     else
       Msg << "unknown intrinsic #" << iid;
   }
diff --git a/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
index 0d039860b9f0f..adfb96041c5c0 100644
--- a/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
@@ -7971,7 +7971,7 @@ static bool isNonZeroModBitWidthOrUndef(SDValue Z, unsigned BW) {
   return ISD::matchUnaryPredicate(
       Z,
       [=](ConstantSDNode *C) { return !C || C->getAPIntValue().urem(BW) != 0; },
-      true);
+      /*AllowUndef=*/true, /*AllowTruncation=*/true);
 }
 
 static SDValue expandVPFunnelShift(SDNode *Node, SelectionDAG &DAG) {
@@ -10857,135 +10857,22 @@ SDValue TargetLowering::expandShlSat(SDNode *Node, SelectionDAG &DAG) const {
   return DAG.getSelect(dl, VT, Cond, SatVal, Result);
 }
 
-void TargetLowering::forceExpandWideMUL(SelectionDAG &DAG, const SDLoc &dl,
-                                        bool Signed, EVT WideVT,
-                                        const SDValue LL, const SDValue LH,
-                                        const SDValue RL, const SDValue RH,
-                                        SDValue &Lo, SDValue &Hi) const {
-  // We can fall back to a libcall with an illegal type for the MUL if we
-  // have a libcall big enough.
-  // Also, we can fall back to a division in some cases, but that's a big
-  // performance hit in the general case.
-  RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
-  if (WideVT == MVT::i16)
-    LC = RTLIB::MUL_I16;
-  else if (WideVT == MVT::i32)
-    LC = RTLIB::MUL_I32;
-  else if (WideVT == MVT::i64)
-    LC = RTLIB::MUL_I64;
-  else if (WideVT == MVT::i128)
-    LC = RTLIB::MUL_I128;
-
-  if (LC == RTLIB::UNKNOWN_LIBCALL || !getLibcallName(LC)) {
-    // We'll expand the multiplication by brute force because we have no other
-    // options. This is a trivially-generalized version of the code from
-    // Hacker's Delight (itself derived from Knuth's Algorithm M from section
-    // 4.3.1).
-    EVT VT = LL.getValueType();
-    unsigned Bits = VT.getSizeInBits();
-    unsigned HalfBits = Bits >> 1;
-    SDValue Mask =
-        DAG.getConstant(APInt::getLowBitsSet(Bits, HalfBits), dl, VT);
-    SDValue LLL = DAG.getNode(ISD::AND, dl, VT, LL, Mask);
-    SDValue RLL = DAG.getNode(ISD::AND, dl, VT, RL, Mask);
-
-    SDValue T = DAG.getNode(ISD::MUL, dl, VT, LLL, RLL);
-    SDValue TL = DAG.getNode(ISD::AND, dl, VT, T, Mask);
-
-    SDValue Shift = DAG.getShiftAmountConstant(HalfBits, VT, dl);
-    SDValue TH = DAG.getNode(ISD::SRL, dl, VT, T, Shift);
-    SDValue LLH = DAG.getNode(ISD::SRL, dl, VT, LL, Shift);
-    SDValue RLH = DAG.getNode(ISD::SRL, dl, VT, RL, Shift);
-
-    SDValue U = DAG.getNode(ISD::ADD, dl, VT,
-                            DAG.getNode(ISD::MUL, dl, VT, LLH, RLL), TH);
-    SDValue UL = DAG.getNode(ISD::AND, dl, VT, U, Mask);
-    SDValue UH = DAG.getNode(ISD::SRL, dl, VT, U, Shift);
-
-    SDValue V = DAG.getNode(ISD::ADD, dl, VT,
-                            DAG.getNode(ISD::MUL, dl, VT, LLL, RLH), UL);
-    SDValue VH = DAG.getNode(ISD::SRL, dl, VT, V, Shift);
-
-    SDValue W =
-        DAG.getNode(ISD::ADD, dl, VT, DAG.getNode(ISD::MUL, dl, VT, LLH, RLH),
-                    DAG.getNode(ISD::ADD, dl, VT, UH, VH));
-    Lo = DAG.getNode(ISD::ADD, dl, VT, TL,
-                     DAG.getNode(ISD::SHL, dl, VT, V, Shift));
-
-    Hi = DAG.getNode(ISD::ADD, dl, VT, W,
-                     DAG.getNode(ISD::ADD, dl, VT,
-                                 DAG.getNode(ISD::MUL, dl, VT, RH, LL),
-                                 DAG.getNode(ISD::MUL, dl, VT, RL, LH)));
-  } else {
-    // Attempt a libcall.
-    SDValue Ret;
-    TargetLowering::MakeLibCallOptions CallOptions;
-    CallOptions.setIsSigned(Signed);
-    CallOptions.setIsPostTypeLegalization(true);
-    if (shouldSplitFunctionArgumentsAsLittleEndian(DAG.getDataLayout())) {
-      // Halves of WideVT are packed into registers in different order
-      // depending on platform endianness. This is usually handled by
-      // the C calling convention, but we can't defer to it in
-      // the legalizer.
-      SDValue Args[] = {LL, LH, RL, RH};
-      Ret = makeLibCall(DAG, LC, WideVT, Args, CallOptions, dl).first;
-    } else {
-      SDValue Args[] = {LH, LL, RH, RL};
-      Ret = makeLibCall(DAG, LC, WideVT, Args, CallOptions, dl).first;
-    }
-    assert(Ret.getOpcode() == ISD::MERGE_VALUES &&
-           "Ret value is a collection of constituent nodes holding result.");
-    if (DAG.getDataLayout().isLittleEndian()) {
-      // Same as above.
-      Lo = Ret.getOperand(0);
-      Hi = Ret.getOperand(1);
-    } else {
-      Lo = Ret.getOperand(1);
-      Hi = Ret.getOperand(0);
-    }
-  }
-}
-
-void TargetLowering::forceExpandWideMUL(SelectionDAG &DAG, const SDLoc &dl,
-                                        bool Signed, const SDValue LHS,
-                                        const SDValue RHS, SDValue &Lo,
-                                        SDValue &Hi) const {
+void TargetLowering::forceExpandMultiply(SelectionDAG &DAG, const SDLoc &dl,
+                                         bool Signed, SDValue &Lo, SDValue &Hi,
+                                         SDValue LHS, SDValue RHS,
+                                         SDValue HiLHS, SDValue HiRHS) const {
   EVT VT = LHS.getValueType();
   assert(RHS.getValueType() == VT && "Mismatching operand types");
-  EVT WideVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits() * 2);
-  // We can fall back to a libcall with an illegal type for the MUL if we
-  // have a libcall big enough.
-  RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
-  if (WideVT == MVT::i16)
-    LC = RTLIB::MUL_I16;
-  else if (WideVT == MVT::i32)
-    LC = RTLIB::MUL_I32;
-  else if (WideVT == MVT::i64)
-    LC = RTLIB::MUL_I64;
-  else if (WideVT == MVT::i128)
-    LC = RTLIB::MUL_I128;
 
-  if (LC != RTLIB::UNKNOWN_LIBCALL && getLibcallName(LC)) {
-    SDValue HiLHS, HiRHS;
-    if (Signed) {
-      // The high part is obtained by SRA'ing all but one of the bits of low
-      // part.
-      unsigned LoSize = VT.getFixedSizeInBits();
-      SDValue Shift = DAG.getShiftAmountConstant(LoSize - 1, VT, dl);
-      HiLHS = DAG.getNode(ISD::SRA, dl, VT, LHS, Shift);
-      HiRHS = DAG.getNode(ISD::SRA, dl, VT, RHS, Shift);
-    } else {
-      HiLHS = DAG.getConstant(0, dl, VT);
-      HiRHS = DAG.getConstant(0, dl, VT);
-    }
-    forceExpandWideMUL(DAG, dl, Signed, WideVT, LHS, HiLHS, RHS, HiRHS, Lo, Hi);
-    return;
-  }
+  assert((HiLHS && HiRHS) || (!HiLHS && !HiRHS));
+  assert((!Signed || !HiLHS) &&
+         "Signed flag should only be set when HiLHS and RiRHS are null");
 
-  // Expand the multiplication by brute force. This is a generalized-version of
-  // the code from Hacker's Delight (itself derived from Knuth's Algorithm M
-  // from section 4.3.1) combined with the Hacker's delight code
-  // for calculating mulhs.
+  // We'll expand the multiplication by brute force because we have no other
+  // options. This is a trivially-generalized version of the code from
+  // Hacker's Delight (itself derived from Knuth's Algorithm M from section
+  // 4.3.1). If Signed is set, we can use arithmetic right shifts to propagate
+  // sign bits while calculating the Hi half.
   unsigned Bits = VT.getSizeInBits();
   unsigned HalfBits = Bits / 2;
   SDValue Mask = DAG.getConstant(APInt::getLowBitsSet(Bits, HalfBits), dl, VT);
@@ -11017,6 +10904,80 @@ void TargetLowering::forceExpandWideMUL(SelectionDAG &DAG, const SDLoc &dl,
 
   Hi = DAG.getNode(ISD::ADD, dl, VT, DAG.getNode(ISD::MUL, dl, VT, LH, RH),
                    DAG.getNode(ISD::ADD, dl, VT, UH, VH));
+
+  // If HiLHS and HiRHS are set, multiply them by the opposite low part and add
+  // the products to Hi.
+  if (HiLHS) {
+    Hi = DAG.getNode(ISD::ADD, dl, VT, Hi,
+                     DAG.getNode(ISD::ADD, dl, VT,
+                                 DAG.getNode(ISD::MUL, dl, VT, HiRHS, LHS),
+                                 DAG.getNode(ISD::MUL, dl, VT, RHS, HiLHS)));
+  }
+}
+
+void TargetLowering::forceExpandWideMUL(SelectionDAG &DAG, const SDLoc &dl,
+                                        bool Signed, const SDValue LHS,
+                                        const SDValue RHS, SDValue &Lo,
+                                        SDValue &Hi) const {
+  EVT VT = LHS.getValueType();
+  assert(RHS.getValueType() == VT && "Mismatching operand types");
+  EVT WideVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits() * 2);
+  // We can fall back to a libcall with an illegal type for the MUL if we
+  // have a libcall big enough.
+  RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
+  if (WideVT == MVT::i16)
+    LC = RTLIB::MUL_I16;
+  else if (WideVT == MVT::i32)
+    LC = RTLIB::MUL_I32;
+  else if (WideVT == MVT::i64)
+    LC = RTLIB::MUL_I64;
+  else if (WideVT == MVT::i128)
+    LC = RTLIB::MUL_I128;
+
+  if (LC == RTLIB::UNKNOWN_LIBCALL || !getLibcallName(LC)) {
+    forceExpandMultiply(DAG, dl, Signed, Lo, Hi, LHS, RHS);
+    return;
+  }
+
+  SDValue HiLHS, HiRHS;
+  if (Signed) {
+    // The high part is obtained by SRA'ing all but one of the bits of low
+    // part.
+    unsigned LoSize = VT.getFixedSizeInBits();
+    SDValue Shift = DAG.getShiftAmountConstant(LoSize - 1, VT, dl);
+    HiLHS = DAG.getNode(ISD::SRA, dl, VT, LHS, Shift);
+    HiRHS = DAG.getNode(ISD::SRA, dl, VT, RHS, Shift);
+  } else {
+    HiLHS = DAG.getConstant(0, dl, VT);
+    HiRHS = DAG.getConstant(0, dl, VT);
+  }
+
+  // Attempt a libcall.
+  SDValue Ret;
+  TargetLowering::MakeLibCallOptions CallOptions;
+  CallOptions.setIsSigned(Signed);
+  CallOptions.setIsPostTypeLegalization(true);
+  if (shouldSplitFunctionArgumentsAsLittleEndian(DAG.getDataLayout())) {
+    // Halves of WideVT are packed into registers in different order
+    // depending on platform endianness. This is usually handled by
+    // the C calling convention, but we can't defer to it in
+    // the legalizer.
+    SDValue Args[] = {LHS, HiLHS, RHS, HiRHS};
+    Ret = makeLibCall(DAG, LC, WideVT, Args, CallOptions, dl).first;
+  } else {
+    SDValue Args[] = {HiLHS, LHS, HiRHS, RHS};
+    Ret = makeLibCall(DAG, LC, WideVT, Args, CallOptions, dl).first;
+  }
+  assert(Ret.getOpcode() == ISD::MERGE_VALUES &&
+         "Ret value is a collection of constituent nodes holding result.");
+  if (DAG.getDataLayout().isLittleEndian()) {
+    // Same as above.
+    Lo = Ret.getOperand(0);
+    Hi = Ret.getOperand(1);
+  } else {
+    Lo = Ret.getOperand(1);
+    Hi = Ret.getOperand(0);
+  }
 }
 
 SDValue
@@ -12153,3 +12114,77 @@ SDValue TargetLowering::expandVectorNaryOpBySplitting(SDNode *Node,
   SDValue SplitOpHi = DAG.getNode(Opcode, DL, HiVT, HiOps);
   return DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, SplitOpLo, SplitOpHi);
 }
+
+SDValue TargetLowering::scalarizeExtractedVectorLoad(EVT ResultVT,
+                                                     const SDLoc &DL,
+                                                     EVT InVecVT, SDValue EltNo,
+                                                     LoadSDNode *OriginalLoad,
+                                                     SelectionDAG &DAG) const {
+  assert(OriginalLoad->isSimple());
+
+  EVT VecEltVT = InVecVT.getVectorElementType();
+
+  // If the vector element type is not a multiple of a byte then we are unable
+  // to correctly compute an address to load only the extracted element as a
+  // scalar.
+  if (!VecEltVT.isByteSized())
+    return SDValue();
+
+  ISD::LoadExtType ExtTy =
+      ResultVT.bitsGT(VecEltVT) ? ISD::EXTLOAD : ISD::NON_EXTLOAD;
+  if (!isOperationLegalOrCustom(ISD::LOAD, VecEltVT) ||
+      !shouldReduceLoadWidth(OriginalLoad, ExtTy, VecEltVT))
+    return SDValue();
+
+  Align Alignment = OriginalLoad->getAlign();
+  MachinePointerInfo MPI;
+  if (auto *ConstEltNo = dyn_cast<ConstantSDNode>(EltNo)) {
+    int Elt = ConstEltNo->getZExtValue();
+    unsigned PtrOff = VecEltVT.getSizeInBits() * Elt / 8;
+    MPI = OriginalLoad->getPointerInfo().getWithOffset(PtrOff);
+    Alignment = commonAlignment(Alignment, PtrOff);
+  } else {
+    // Discard the pointer info except the address space because the memory
+    // operand can't represent this new access since the offset is variable.
+    MPI = MachinePointerInfo(OriginalLoad->getPointerInfo().getAddrSpace());
+    Alignment = commonAlignment(Alignment, VecEltVT.getSizeInBits() / 8);
+  }
+
+  unsigned IsFast = 0;
+  if (!allowsMemoryAccess(*DAG.getContext(), DAG.getDataLayout(), VecEltVT,
+                          OriginalLoad->getAddressSpace(), Alignment,
+                          OriginalLoad->getMemOperand()->getFlags(), &IsFast) ||
+      !IsFast)
+    return SDValue();
+
+  SDValue NewPtr =
+      getVectorElementPointer(DAG, OriginalLoad->getBasePtr(), InVecVT, EltNo);
+
+  // We are replacing a vector load with a scalar load. The new load must have
+  // identical memory op ordering to the original.
+  SDValue Load;
+  if (ResultVT.bitsGT(VecEltVT)) {
+    // If the result type of vextract is wider than the load, then issue an
+    // extending load instead.
+    ISD::LoadExtType ExtType = isLoadExtLegal(ISD::ZEXTLOAD, ResultVT, VecEltVT)
+                                   ? ISD::ZEXTLOAD
+                                   : ISD::EXTLOAD;
+    Load = DAG.getExtLoad(ExtType, DL, ResultVT, OriginalLoad->getChain(),
+                          NewPtr, MPI, VecEltVT, Alignment,
+                          OriginalLoad->getMemOperand()->getFlags(),
+                          OriginalLoad->getAAInfo());
+    DAG.makeEquivalentMemoryOrdering(OriginalLoad, Load);
+  } else {
+    // The result type is narrower or the same width as the vector element
+    Load = DAG.getLoad(VecEltVT, DL, OriginalLoad->getChain(), NewPtr, MPI,
+                       Alignment, OriginalLoad->getMemOperand()->getFlags(),
+                       OriginalLoad->getAAInfo());
+    DAG.makeEquivalentMemoryOrdering(OriginalLoad, Load);
+    if (ResultVT.bitsLT(VecEltVT))
+      Load = DAG.getNode(ISD::TRUNCATE, DL, ResultVT, Load);
+    else
+      Load = DAG.getBitcast(ResultVT, Load);
+  }
+
+  return Load;
+}
diff --git a/llvm/lib/CodeGen/StackColoring.cpp b/llvm/lib/CodeGen/StackColoring.cpp
index b77b8dbdd6e59..552dfdfe16ce4 100644
--- a/llvm/lib/CodeGen/StackColoring.cpp
+++ b/llvm/lib/CodeGen/StackColoring.cpp
@@ -914,10 +914,10 @@ void StackColoring::remapInstructions(DenseMap<int, int> &SlotRemap) {
     if (!VI.Var || !VI.inStackSlot())
       continue;
     int Slot = VI.getStackSlot();
-    if (SlotRemap.count(Slot)) {
+    if (auto It = SlotRemap.find(Slot); It != SlotRemap.end()) {
       LLVM_DEBUG(dbgs() << "Remapping debug info for ["
                         << cast<DILocalVariable>(VI.Var)->getName() << "].\n");
-      VI.updateStackSlot(SlotRemap[Slot]);
+      VI.updateStackSlot(It->second);
       FixedDbg++;
     }
   }
@@ -1004,10 +1004,11 @@ void StackColoring::remapInstructions(DenseMap<int, int> &SlotRemap) {
         if (!AI)
           continue;
 
-        if (!Allocas.count(AI))
+        auto It = Allocas.find(AI);
+        if (It == Allocas.end())
           continue;
 
-        MMO->setValue(Allocas[AI]);
+        MMO->setValue(It->second);
         FixedMemOp++;
       }
 
@@ -1114,9 +1115,10 @@ void StackColoring::remapInstructions(DenseMap<int, int> &SlotRemap) {
   if (WinEHFuncInfo *EHInfo = MF->getWinEHFuncInfo())
     for (WinEHTryBlockMapEntry &TBME : EHInfo->TryBlockMap)
       for (WinEHHandlerType &H : TBME.HandlerArray)
-        if (H.CatchObj.FrameIndex != std::numeric_limits<int>::max() &&
-            SlotRemap.count(H.CatchObj.FrameIndex))
-          H.CatchObj.FrameIndex = SlotRemap[H.CatchObj.FrameIndex];
+        if (H.CatchObj.FrameIndex != std::numeric_limits<int>::max())
+          if (auto It = SlotRemap.find(H.CatchObj.FrameIndex);
+              It != SlotRemap.end())
+            H.CatchObj.FrameIndex = It->second;
 
   LLVM_DEBUG(dbgs() << "Fixed " << FixedMemOp << " machine memory operands.\n");
   LLVM_DEBUG(dbgs() << "Fixed " << FixedDbg << " debug locations.\n");
@@ -1173,8 +1175,8 @@ void StackColoring::expungeSlotMap(DenseMap<int, int> &SlotRemap,
   // Expunge slot remap map.
   for (unsigned i=0; i < NumSlots; ++i) {
     // If we are remapping i
-    if (SlotRemap.count(i)) {
-      int Target = SlotRemap[i];
+    if (auto It = SlotRemap.find(i); It != SlotRemap.end()) {
+      int Target = It->second;
       // As long as our target is mapped to something else, follow it.
       while (SlotRemap.count(Target)) {
         Target = SlotRemap[Target];
diff --git a/llvm/lib/CodeGen/StackSlotColoring.cpp b/llvm/lib/CodeGen/StackSlotColoring.cpp
index 4dc5dc87ba3fc..22c5c2e080b05 100644
--- a/llvm/lib/CodeGen/StackSlotColoring.cpp
+++ b/llvm/lib/CodeGen/StackSlotColoring.cpp
@@ -10,6 +10,7 @@
 //
 //===----------------------------------------------------------------------===//
 
+#include "llvm/CodeGen/StackSlotColoring.h"
 #include "llvm/ADT/BitVector.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/Statistic.h"
@@ -20,6 +21,7 @@
 #include "llvm/CodeGen/LiveStacks.h"
 #include "llvm/CodeGen/MachineBasicBlock.h"
 #include "llvm/CodeGen/MachineBlockFrequencyInfo.h"
+#include "llvm/CodeGen/MachineDominators.h"
 #include "llvm/CodeGen/MachineFrameInfo.h"
 #include "llvm/CodeGen/MachineFunction.h"
 #include "llvm/CodeGen/MachineFunctionPass.h"
@@ -59,136 +61,145 @@ STATISTIC(NumDead,       "Number of trivially dead stack accesses eliminated");
 
 namespace {
 
-  class StackSlotColoring : public MachineFunctionPass {
-    LiveStacks *LS = nullptr;
-    MachineFrameInfo *MFI = nullptr;
-    const TargetInstrInfo *TII = nullptr;
-    const MachineBlockFrequencyInfo *MBFI = nullptr;
-    SlotIndexes *Indexes = nullptr;
-
-    // SSIntervals - Spill slot intervals.
-    std::vector<LiveInterval*> SSIntervals;
-
-    // SSRefs - Keep a list of MachineMemOperands for each spill slot.
-    // MachineMemOperands can be shared between instructions, so we need
-    // to be careful that renames like [FI0, FI1] -> [FI1, FI2] do not
-    // become FI0 -> FI1 -> FI2.
-    SmallVector<SmallVector<MachineMemOperand *, 8>, 16> SSRefs;
-
-    // OrigAlignments - Alignments of stack objects before coloring.
-    SmallVector<Align, 16> OrigAlignments;
-
-    // OrigSizes - Sizes of stack objects before coloring.
-    SmallVector<unsigned, 16> OrigSizes;
-
-    // AllColors - If index is set, it's a spill slot, i.e. color.
-    // FIXME: This assumes PEI locate spill slot with smaller indices
-    // closest to stack pointer / frame pointer. Therefore, smaller
-    // index == better color. This is per stack ID.
-    SmallVector<BitVector, 2> AllColors;
-
-    // NextColor - Next "color" that's not yet used. This is per stack ID.
-    SmallVector<int, 2> NextColors = { -1 };
-
-    // UsedColors - "Colors" that have been assigned. This is per stack ID
-    SmallVector<BitVector, 2> UsedColors;
-
-    // Join all intervals sharing one color into a single LiveIntervalUnion to
-    // speedup range overlap test.
-    class ColorAssignmentInfo {
-      // Single liverange (used to avoid creation of LiveIntervalUnion).
-      LiveInterval *SingleLI = nullptr;
-      // LiveIntervalUnion to perform overlap test.
-      LiveIntervalUnion *LIU = nullptr;
-      // LiveIntervalUnion has a parameter in its constructor so doing this
-      // dirty magic.
-      uint8_t LIUPad[sizeof(LiveIntervalUnion)];
-
-    public:
-      ~ColorAssignmentInfo() {
-        if (LIU)
-          LIU->~LiveIntervalUnion(); // Dirty magic again.
-      }
-
-      // Return true if LiveInterval overlaps with any
-      // intervals that have already been assigned to this color.
-      bool overlaps(LiveInterval *LI) const {
-        if (LIU)
-          return LiveIntervalUnion::Query(*LI, *LIU).checkInterference();
-        return SingleLI ? SingleLI->overlaps(*LI) : false;
-      }
-
-      // Add new LiveInterval to this color.
-      void add(LiveInterval *LI, LiveIntervalUnion::Allocator &Alloc) {
-        assert(!overlaps(LI));
-        if (LIU) {
-          LIU->unify(*LI, *LI);
-        } else if (SingleLI) {
-          LIU = new (LIUPad) LiveIntervalUnion(Alloc);
-          LIU->unify(*SingleLI, *SingleLI);
-          LIU->unify(*LI, *LI);
-          SingleLI = nullptr;
-        } else
-          SingleLI = LI;
-      }
-    };
-
-    LiveIntervalUnion::Allocator LIUAlloc;
-
-    // Assignments - Color to intervals mapping.
-    SmallVector<ColorAssignmentInfo, 16> Assignments;
+class StackSlotColoring {
+  MachineFrameInfo *MFI = nullptr;
+  const TargetInstrInfo *TII = nullptr;
+  LiveStacks *LS = nullptr;
+  const MachineBlockFrequencyInfo *MBFI = nullptr;
+  SlotIndexes *Indexes = nullptr;
+
+  // SSIntervals - Spill slot intervals.
+  std::vector<LiveInterval *> SSIntervals;
+
+  // SSRefs - Keep a list of MachineMemOperands for each spill slot.
+  // MachineMemOperands can be shared between instructions, so we need
+  // to be careful that renames like [FI0, FI1] -> [FI1, FI2] do not
+  // become FI0 -> FI1 -> FI2.
+  SmallVector<SmallVector<MachineMemOperand *, 8>, 16> SSRefs;
+
+  // OrigAlignments - Alignments of stack objects before coloring.
+  SmallVector<Align, 16> OrigAlignments;
+
+  // OrigSizes - Sizes of stack objects before coloring.
+  SmallVector<unsigned, 16> OrigSizes;
+
+  // AllColors - If index is set, it's a spill slot, i.e. color.
+  // FIXME: This assumes PEI locate spill slot with smaller indices
+  // closest to stack pointer / frame pointer. Therefore, smaller
+  // index == better color. This is per stack ID.
+  SmallVector<BitVector, 2> AllColors;
+
+  // NextColor - Next "color" that's not yet used. This is per stack ID.
+  SmallVector<int, 2> NextColors = {-1};
+
+  // UsedColors - "Colors" that have been assigned. This is per stack ID
+  SmallVector<BitVector, 2> UsedColors;
+
+  // Join all intervals sharing one color into a single LiveIntervalUnion to
+  // speedup range overlap test.
+  class ColorAssignmentInfo {
+    // Single liverange (used to avoid creation of LiveIntervalUnion).
+    LiveInterval *SingleLI = nullptr;
+    // LiveIntervalUnion to perform overlap test.
+    LiveIntervalUnion *LIU = nullptr;
+    // LiveIntervalUnion has a parameter in its constructor so doing this
+    // dirty magic.
+    uint8_t LIUPad[sizeof(LiveIntervalUnion)];
 
   public:
-    static char ID; // Pass identification
+    ~ColorAssignmentInfo() {
+      if (LIU)
+        LIU->~LiveIntervalUnion(); // Dirty magic again.
+    }
 
-    StackSlotColoring() : MachineFunctionPass(ID) {
-      initializeStackSlotColoringPass(*PassRegistry::getPassRegistry());
+    // Return true if LiveInterval overlaps with any
+    // intervals that have already been assigned to this color.
+    bool overlaps(LiveInterval *LI) const {
+      if (LIU)
+        return LiveIntervalUnion::Query(*LI, *LIU).checkInterference();
+      return SingleLI ? SingleLI->overlaps(*LI) : false;
     }
 
-    void getAnalysisUsage(AnalysisUsage &AU) const override {
-      AU.setPreservesCFG();
-      AU.addRequired<SlotIndexesWrapperPass>();
-      AU.addPreserved<SlotIndexesWrapperPass>();
-      AU.addRequired<LiveStacksWrapperLegacy>();
-      AU.addRequired<MachineBlockFrequencyInfoWrapperPass>();
-      AU.addPreserved<MachineBlockFrequencyInfoWrapperPass>();
-      AU.addPreservedID(MachineDominatorsID);
-
-      // In some Target's pipeline, register allocation (RA) might be
-      // split into multiple phases based on register class. So, this pass
-      // may be invoked multiple times requiring it to save these analyses to be
-      // used by RA later.
-      AU.addPreserved<LiveIntervalsWrapperPass>();
-      AU.addPreserved<LiveDebugVariablesWrapperLegacy>();
-
-      MachineFunctionPass::getAnalysisUsage(AU);
+    // Add new LiveInterval to this color.
+    void add(LiveInterval *LI, LiveIntervalUnion::Allocator &Alloc) {
+      assert(!overlaps(LI));
+      if (LIU) {
+        LIU->unify(*LI, *LI);
+      } else if (SingleLI) {
+        LIU = new (LIUPad) LiveIntervalUnion(Alloc);
+        LIU->unify(*SingleLI, *SingleLI);
+        LIU->unify(*LI, *LI);
+        SingleLI = nullptr;
+      } else
+        SingleLI = LI;
     }
+  };
+
+  LiveIntervalUnion::Allocator LIUAlloc;
+
+  // Assignments - Color to intervals mapping.
+  SmallVector<ColorAssignmentInfo, 16> Assignments;
+
+public:
+  StackSlotColoring(MachineFunction &MF, LiveStacks *LS,
+                    MachineBlockFrequencyInfo *MBFI, SlotIndexes *Indexes)
+      : MFI(&MF.getFrameInfo()), TII(MF.getSubtarget().getInstrInfo()), LS(LS),
+        MBFI(MBFI), Indexes(Indexes) {}
+  bool run(MachineFunction &MF);
+
+private:
+  void InitializeSlots();
+  void ScanForSpillSlotRefs(MachineFunction &MF);
+  int ColorSlot(LiveInterval *li);
+  bool ColorSlots(MachineFunction &MF);
+  void RewriteInstruction(MachineInstr &MI, SmallVectorImpl<int> &SlotMapping,
+                          MachineFunction &MF);
+  bool RemoveDeadStores(MachineBasicBlock *MBB);
+};
 
-    bool runOnMachineFunction(MachineFunction &MF) override;
+class StackSlotColoringLegacy : public MachineFunctionPass {
+public:
+  static char ID; // Pass identification
 
-  private:
-    void InitializeSlots();
-    void ScanForSpillSlotRefs(MachineFunction &MF);
-    int ColorSlot(LiveInterval *li);
-    bool ColorSlots(MachineFunction &MF);
-    void RewriteInstruction(MachineInstr &MI, SmallVectorImpl<int> &SlotMapping,
-                            MachineFunction &MF);
-    bool RemoveDeadStores(MachineBasicBlock* MBB);
-  };
+  StackSlotColoringLegacy() : MachineFunctionPass(ID) {
+    initializeStackSlotColoringLegacyPass(*PassRegistry::getPassRegistry());
+  }
+
+  void getAnalysisUsage(AnalysisUsage &AU) const override {
+    AU.setPreservesCFG();
+    AU.addRequired<SlotIndexesWrapperPass>();
+    AU.addPreserved<SlotIndexesWrapperPass>();
+    AU.addRequired<LiveStacksWrapperLegacy>();
+    AU.addRequired<MachineBlockFrequencyInfoWrapperPass>();
+    AU.addPreserved<MachineBlockFrequencyInfoWrapperPass>();
+    AU.addPreservedID(MachineDominatorsID);
+
+    // In some Target's pipeline, register allocation (RA) might be
+    // split into multiple phases based on register class. So, this pass
+    // may be invoked multiple times requiring it to save these analyses to be
+    // used by RA later.
+    AU.addPreserved<LiveIntervalsWrapperPass>();
+    AU.addPreserved<LiveDebugVariablesWrapperLegacy>();
+
+    MachineFunctionPass::getAnalysisUsage(AU);
+  }
+
+  bool runOnMachineFunction(MachineFunction &MF) override;
+};
 
 } // end anonymous namespace
 
-char StackSlotColoring::ID = 0;
+char StackSlotColoringLegacy::ID = 0;
 
-char &llvm::StackSlotColoringID = StackSlotColoring::ID;
+char &llvm::StackSlotColoringID = StackSlotColoringLegacy::ID;
 
-INITIALIZE_PASS_BEGIN(StackSlotColoring, DEBUG_TYPE,
-                "Stack Slot Coloring", false, false)
+INITIALIZE_PASS_BEGIN(StackSlotColoringLegacy, DEBUG_TYPE,
+                      "Stack Slot Coloring", false, false)
 INITIALIZE_PASS_DEPENDENCY(SlotIndexesWrapperPass)
 INITIALIZE_PASS_DEPENDENCY(LiveStacksWrapperLegacy)
 INITIALIZE_PASS_DEPENDENCY(MachineLoopInfoWrapperPass)
-INITIALIZE_PASS_END(StackSlotColoring, DEBUG_TYPE,
-                "Stack Slot Coloring", false, false)
+INITIALIZE_PASS_END(StackSlotColoringLegacy, DEBUG_TYPE, "Stack Slot Coloring",
+                    false, false)
 
 namespace {
 
@@ -266,7 +277,7 @@ void StackSlotColoring::InitializeSlots() {
   for (auto *I : Intervals) {
     LiveInterval &li = I->second;
     LLVM_DEBUG(li.dump());
-    int FI = Register::stackSlot2Index(li.reg());
+    int FI = li.reg().stackSlotIndex();
     if (MFI->isDeadObjectIndex(FI))
       continue;
 
@@ -300,7 +311,7 @@ void StackSlotColoring::InitializeSlots() {
 int StackSlotColoring::ColorSlot(LiveInterval *li) {
   int Color = -1;
   bool Share = false;
-  int FI = Register::stackSlot2Index(li->reg());
+  int FI = li->reg().stackSlotIndex();
   uint8_t StackID = MFI->getStackID(FI);
 
   if (!DisableSharing) {
@@ -361,7 +372,7 @@ bool StackSlotColoring::ColorSlots(MachineFunction &MF) {
   LLVM_DEBUG(dbgs() << "Color spill slot intervals:\n");
   bool Changed = false;
   for (LiveInterval *li : SSIntervals) {
-    int SS = Register::stackSlot2Index(li->reg());
+    int SS = li->reg().stackSlotIndex();
     int NewSS = ColorSlot(li);
     assert(NewSS >= 0 && "Stack coloring failed?");
     SlotMapping[SS] = NewSS;
@@ -373,7 +384,7 @@ bool StackSlotColoring::ColorSlots(MachineFunction &MF) {
 
   LLVM_DEBUG(dbgs() << "\nSpill slots after coloring:\n");
   for (LiveInterval *li : SSIntervals) {
-    int SS = Register::stackSlot2Index(li->reg());
+    int SS = li->reg().stackSlotIndex();
     li->setWeight(SlotWeights[SS]);
   }
   // Sort them by new weight.
@@ -511,21 +522,12 @@ bool StackSlotColoring::RemoveDeadStores(MachineBasicBlock* MBB) {
   return changed;
 }
 
-bool StackSlotColoring::runOnMachineFunction(MachineFunction &MF) {
+bool StackSlotColoring::run(MachineFunction &MF) {
   LLVM_DEBUG({
     dbgs() << "********** Stack Slot Coloring **********\n"
            << "********** Function: " << MF.getName() << '\n';
   });
 
-  if (skipFunction(MF.getFunction()))
-    return false;
-
-  MFI = &MF.getFrameInfo();
-  TII = MF.getSubtarget().getInstrInfo();
-  LS = &getAnalysis<LiveStacksWrapperLegacy>().getLS();
-  MBFI = &getAnalysis<MachineBlockFrequencyInfoWrapperPass>().getMBFI();
-  Indexes = &getAnalysis<SlotIndexesWrapperPass>().getSI();
-
   bool Changed = false;
 
   unsigned NumSlots = LS->getNumIntervals();
@@ -559,3 +561,37 @@ bool StackSlotColoring::runOnMachineFunction(MachineFunction &MF) {
 
   return Changed;
 }
+
+bool StackSlotColoringLegacy::runOnMachineFunction(MachineFunction &MF) {
+  if (skipFunction(MF.getFunction()))
+    return false;
+
+  LiveStacks *LS = &getAnalysis<LiveStacksWrapperLegacy>().getLS();
+  MachineBlockFrequencyInfo *MBFI =
+      &getAnalysis<MachineBlockFrequencyInfoWrapperPass>().getMBFI();
+  SlotIndexes *Indexes = &getAnalysis<SlotIndexesWrapperPass>().getSI();
+  StackSlotColoring Impl(MF, LS, MBFI, Indexes);
+  return Impl.run(MF);
+}
+
+PreservedAnalyses
+StackSlotColoringPass::run(MachineFunction &MF,
+                           MachineFunctionAnalysisManager &MFAM) {
+  LiveStacks *LS = &MFAM.getResult<LiveStacksAnalysis>(MF);
+  MachineBlockFrequencyInfo *MBFI =
+      &MFAM.getResult<MachineBlockFrequencyAnalysis>(MF);
+  SlotIndexes *Indexes = &MFAM.getResult<SlotIndexesAnalysis>(MF);
+  StackSlotColoring Impl(MF, LS, MBFI, Indexes);
+  bool Changed = Impl.run(MF);
+  if (!Changed)
+    return PreservedAnalyses::all();
+
+  auto PA = getMachineFunctionPassPreservedAnalyses();
+  PA.preserveSet<CFGAnalyses>();
+  PA.preserve<SlotIndexesAnalysis>();
+  PA.preserve<MachineBlockFrequencyAnalysis>();
+  PA.preserve<MachineDominatorTreeAnalysis>();
+  PA.preserve<LiveIntervalsAnalysis>();
+  PA.preserve<LiveDebugVariablesAnalysis>();
+  return PA;
+}
diff --git a/llvm/lib/CodeGen/StaticDataSplitter.cpp b/llvm/lib/CodeGen/StaticDataSplitter.cpp
index 25f02fde8a4b8..0965fe85acfc7 100644
--- a/llvm/lib/CodeGen/StaticDataSplitter.cpp
+++ b/llvm/lib/CodeGen/StaticDataSplitter.cpp
@@ -35,31 +35,24 @@ using namespace llvm;
 
 #define DEBUG_TYPE "static-data-splitter"
 
-STATISTIC(NumHotJumpTables, "Number of hot jump tables seen");
-STATISTIC(NumColdJumpTables, "Number of cold jump tables seen");
+STATISTIC(NumHotJumpTables, "Number of hot jump tables seen.");
+STATISTIC(NumColdJumpTables, "Number of cold jump tables seen.");
 STATISTIC(NumUnknownJumpTables,
-          "Number of jump tables with unknown hotness. Option "
-          "-static-data-default-hotness specifies the hotness.");
-
-static cl::opt<MachineFunctionDataHotness> StaticDataDefaultHotness(
-    "static-data-default-hotness", cl::Hidden,
-    cl::desc("This option specifies the hotness of static data when profile "
-             "information is unavailable"),
-    cl::init(MachineFunctionDataHotness::Hot),
-    cl::values(clEnumValN(MachineFunctionDataHotness::Hot, "hot", "Hot"),
-               clEnumValN(MachineFunctionDataHotness::Cold, "cold", "Cold")));
+          "Number of jump tables with unknown hotness. They are from functions "
+          "without profile information.");
 
 class StaticDataSplitter : public MachineFunctionPass {
   const MachineBranchProbabilityInfo *MBPI = nullptr;
   const MachineBlockFrequencyInfo *MBFI = nullptr;
   const ProfileSummaryInfo *PSI = nullptr;
 
-  // Returns true iff any jump table is hot-cold categorized.
-  bool splitJumpTables(MachineFunction &MF);
+  // Update LLVM statistics for a machine function without profiles.
+  void updateStatsWithoutProfiles(const MachineFunction &MF);
+  // Update LLVM statistics for a machine function with profiles.
+  void updateStatsWithProfiles(const MachineFunction &MF);
 
-  // Same as above but works on functions with profile information.
-  bool splitJumpTablesWithProfiles(const MachineFunction &MF,
-                                   MachineJumpTableInfo &MJTI);
+  // Use profiles to partition static data.
+  bool partitionStaticDataWithProfiles(MachineFunction &MF);
 
 public:
   static char ID;
@@ -85,13 +78,25 @@ bool StaticDataSplitter::runOnMachineFunction(MachineFunction &MF) {
   MBFI = &getAnalysis<MachineBlockFrequencyInfoWrapperPass>().getMBFI();
   PSI = &getAnalysis<ProfileSummaryInfoWrapperPass>().getPSI();
 
-  return splitJumpTables(MF);
+  const bool ProfileAvailable = PSI && PSI->hasProfileSummary() && MBFI &&
+                                MF.getFunction().hasProfileData();
+
+  if (!ProfileAvailable) {
+    updateStatsWithoutProfiles(MF);
+    return false;
+  }
+
+  bool Changed = partitionStaticDataWithProfiles(MF);
+
+  updateStatsWithProfiles(MF);
+  return Changed;
 }
 
-bool StaticDataSplitter::splitJumpTablesWithProfiles(
-    const MachineFunction &MF, MachineJumpTableInfo &MJTI) {
+bool StaticDataSplitter::partitionStaticDataWithProfiles(MachineFunction &MF) {
   int NumChangedJumpTables = 0;
 
+  MachineJumpTableInfo *MJTI = MF.getJumpTableInfo();
+
   // Jump table could be used by either terminating instructions or
   // non-terminating ones, so we walk all instructions and use
   // `MachineOperand::isJTI()` to identify jump table operands.
@@ -100,70 +105,55 @@ bool StaticDataSplitter::splitJumpTablesWithProfiles(
   for (const auto &MBB : MF) {
     for (const MachineInstr &I : MBB) {
       for (const MachineOperand &Op : I.operands()) {
-        if (!Op.isJTI())
-          continue;
-        const int JTI = Op.getIndex();
-        // This is not a source block of jump table.
-        if (JTI == -1)
-          continue;
-
-        auto Hotness = MachineFunctionDataHotness::Hot;
-
-        // Hotness is based on source basic block hotness.
-        // TODO: PSI APIs are about instruction hotness. Introduce API for data
-        // access hotness.
-        if (PSI->isColdBlock(&MBB, MBFI))
-          Hotness = MachineFunctionDataHotness::Cold;
-
-        if (MJTI.updateJumpTableEntryHotness(JTI, Hotness))
-          ++NumChangedJumpTables;
+        if (Op.isJTI()) {
+          assert(MJTI != nullptr && "Jump table info is not available.");
+          const int JTI = Op.getIndex();
+          // This is not a source block of jump table.
+          if (JTI == -1)
+            continue;
+
+          auto Hotness = MachineFunctionDataHotness::Hot;
+
+          // Hotness is based on source basic block hotness.
+          // TODO: PSI APIs are about instruction hotness. Introduce API for
+          // data access hotness.
+          if (PSI->isColdBlock(&MBB, MBFI))
+            Hotness = MachineFunctionDataHotness::Cold;
+
+          if (MJTI->updateJumpTableEntryHotness(JTI, Hotness))
+            ++NumChangedJumpTables;
+        }
       }
     }
   }
   return NumChangedJumpTables > 0;
 }
 
-bool StaticDataSplitter::splitJumpTables(MachineFunction &MF) {
-  MachineJumpTableInfo *MJTI = MF.getJumpTableInfo();
-  if (!MJTI || MJTI->getJumpTables().empty())
-    return false;
-
-  const bool ProfileAvailable = PSI && PSI->hasProfileSummary() && MBFI &&
-                                MF.getFunction().hasProfileData();
-  auto statOnExit = llvm::make_scope_exit([&] {
-    if (!AreStatisticsEnabled())
-      return;
-
-    if (!ProfileAvailable) {
-      NumUnknownJumpTables += MJTI->getJumpTables().size();
-      return;
-    }
+void StaticDataSplitter::updateStatsWithProfiles(const MachineFunction &MF) {
+  if (!AreStatisticsEnabled())
+    return;
 
-    for (size_t JTI = 0; JTI < MJTI->getJumpTables().size(); JTI++) {
-      auto Hotness = MJTI->getJumpTables()[JTI].Hotness;
-      if (Hotness == MachineFunctionDataHotness::Hot) {
+  if (const MachineJumpTableInfo *MJTI = MF.getJumpTableInfo()) {
+    for (const auto &JumpTable : MJTI->getJumpTables()) {
+      if (JumpTable.Hotness == MachineFunctionDataHotness::Hot) {
         ++NumHotJumpTables;
       } else {
-        assert(Hotness == MachineFunctionDataHotness::Cold &&
+        assert(JumpTable.Hotness == MachineFunctionDataHotness::Cold &&
                "A jump table is either hot or cold when profile information is "
                "available.");
         ++NumColdJumpTables;
       }
     }
-  });
-
-  // Place jump tables according to block hotness if function has profile data.
-  if (ProfileAvailable)
-    return splitJumpTablesWithProfiles(MF, *MJTI);
+  }
+}
 
-  // If function profile is unavailable (e.g., module not instrumented, or new
-  // code paths lacking samples), -static-data-default-hotness specifies the
-  // hotness.
-  for (size_t JTI = 0; JTI < MJTI->getJumpTables().size(); JTI++)
-    MF.getJumpTableInfo()->updateJumpTableEntryHotness(
-        JTI, StaticDataDefaultHotness);
+void StaticDataSplitter::updateStatsWithoutProfiles(const MachineFunction &MF) {
+  if (!AreStatisticsEnabled())
+    return;
 
-  return true;
+  if (const MachineJumpTableInfo *MJTI = MF.getJumpTableInfo()) {
+    NumUnknownJumpTables += MJTI->getJumpTables().size();
+  }
 }
 
 char StaticDataSplitter::ID = 0;
diff --git a/llvm/lib/CodeGen/TargetLoweringBase.cpp b/llvm/lib/CodeGen/TargetLoweringBase.cpp
index 9c56912aa6ba0..51cde7ce139e2 100644
--- a/llvm/lib/CodeGen/TargetLoweringBase.cpp
+++ b/llvm/lib/CodeGen/TargetLoweringBase.cpp
@@ -402,11 +402,21 @@ RTLIB::Libcall RTLIB::getFREXP(EVT RetVT) {
                       FREXP_PPCF128);
 }
 
-RTLIB::Libcall RTLIB::getFSINCOS(EVT RetVT) {
+RTLIB::Libcall RTLIB::getSINCOS(EVT RetVT) {
   return getFPLibCall(RetVT, SINCOS_F32, SINCOS_F64, SINCOS_F80, SINCOS_F128,
                       SINCOS_PPCF128);
 }
 
+RTLIB::Libcall RTLIB::getSINCOSPI(EVT RetVT) {
+  return getFPLibCall(RetVT, SINCOSPI_F32, SINCOSPI_F64, SINCOSPI_F80,
+                      SINCOSPI_F128, SINCOSPI_PPCF128);
+}
+
+RTLIB::Libcall RTLIB::getMODF(EVT RetVT) {
+  return getFPLibCall(RetVT, MODF_F32, MODF_F64, MODF_F80, MODF_F128,
+                      MODF_PPCF128);
+}
+
 RTLIB::Libcall RTLIB::getOutlineAtomicHelper(const Libcall (&LC)[5][4],
                                              AtomicOrdering Order,
                                              uint64_t MemSize) {
@@ -775,9 +785,9 @@ void TargetLoweringBase::initActions() {
     setOperationAction({ISD::BITREVERSE, ISD::PARITY}, VT, Expand);
 
     // These library functions default to expand.
-    setOperationAction(
-        {ISD::FROUND, ISD::FPOWI, ISD::FLDEXP, ISD::FFREXP, ISD::FSINCOS}, VT,
-        Expand);
+    setOperationAction({ISD::FROUND, ISD::FPOWI, ISD::FLDEXP, ISD::FFREXP,
+                        ISD::FSINCOS, ISD::FSINCOSPI, ISD::FMODF},
+                       VT, Expand);
 
     // These operations default to expand for vector types.
     if (VT.isVector())
diff --git a/llvm/lib/CodeGen/TargetLoweringObjectFileImpl.cpp b/llvm/lib/CodeGen/TargetLoweringObjectFileImpl.cpp
index fbbd92a2e0ca4..6cbc4b9776a1b 100644
--- a/llvm/lib/CodeGen/TargetLoweringObjectFileImpl.cpp
+++ b/llvm/lib/CodeGen/TargetLoweringObjectFileImpl.cpp
@@ -24,6 +24,7 @@
 #include "llvm/CodeGen/BasicBlockSectionUtils.h"
 #include "llvm/CodeGen/MachineBasicBlock.h"
 #include "llvm/CodeGen/MachineFunction.h"
+#include "llvm/CodeGen/MachineJumpTableInfo.h"
 #include "llvm/CodeGen/MachineModuleInfo.h"
 #include "llvm/CodeGen/MachineModuleInfoImpls.h"
 #include "llvm/IR/Comdat.h"
@@ -648,7 +649,8 @@ static StringRef getSectionPrefixForGlobal(SectionKind Kind, bool IsLarge) {
 static SmallString<128>
 getELFSectionNameForGlobal(const GlobalObject *GO, SectionKind Kind,
                            Mangler &Mang, const TargetMachine &TM,
-                           unsigned EntrySize, bool UniqueSectionName) {
+                           unsigned EntrySize, bool UniqueSectionName,
+                           const MachineJumpTableEntry *JTE) {
   SmallString<128> Name =
       getSectionPrefixForGlobal(Kind, TM.isLargeGlobalValue(GO));
   if (Kind.isMergeableCString()) {
@@ -669,7 +671,24 @@ getELFSectionNameForGlobal(const GlobalObject *GO, SectionKind Kind,
 
   bool HasPrefix = false;
   if (const auto *F = dyn_cast<Function>(GO)) {
-    if (std::optional<StringRef> Prefix = F->getSectionPrefix()) {
+    // Jump table hotness takes precedence over its enclosing function's hotness
+    // if it's known. The function's section prefix is used if jump table entry
+    // hotness is unknown.
+    if (JTE && JTE->Hotness != MachineFunctionDataHotness::Unknown) {
+      if (JTE->Hotness == MachineFunctionDataHotness::Hot) {
+        raw_svector_ostream(Name) << ".hot";
+      } else {
+        assert(JTE->Hotness == MachineFunctionDataHotness::Cold &&
+               "Hotness must be cold");
+        raw_svector_ostream(Name) << ".unlikely";
+      }
+      HasPrefix = true;
+    } else if (std::optional<StringRef> Prefix = F->getSectionPrefix()) {
+      raw_svector_ostream(Name) << '.' << *Prefix;
+      HasPrefix = true;
+    }
+  } else if (const auto *GV = dyn_cast<GlobalVariable>(GO)) {
+    if (std::optional<StringRef> Prefix = GV->getSectionPrefix()) {
       raw_svector_ostream(Name) << '.' << *Prefix;
       HasPrefix = true;
     }
@@ -732,7 +751,7 @@ calcUniqueIDUpdateFlagsAndSize(const GlobalObject *GO, StringRef SectionName,
   // that section can be assigned an incorrect entry size. To avoid this we
   // usually put symbols of the same size into distinct mergeable sections with
   // the same name. Doing so relies on the ",unique ," assembly feature. This
-  // feature is not avalible until bintuils version 2.35
+  // feature is not available until binutils version 2.35
   // (https://sourceware.org/bugzilla/show_bug.cgi?id=25380).
   const bool SupportsUnique = Ctx.getAsmInfo()->useIntegratedAssembler() ||
                               Ctx.getAsmInfo()->binutilsIsAtLeast(2, 35);
@@ -745,7 +764,7 @@ calcUniqueIDUpdateFlagsAndSize(const GlobalObject *GO, StringRef SectionName,
   const bool SymbolMergeable = Flags & ELF::SHF_MERGE;
   const bool SeenSectionNameBefore =
       Ctx.isELFGenericMergeableSection(SectionName);
-  // If this is the first ocurrence of this section name, treat it as the
+  // If this is the first occurrence of this section name, treat it as the
   // generic section
   if (!SymbolMergeable && !SeenSectionNameBefore) {
     if (TM.getSeparateNamedSections())
@@ -767,8 +786,8 @@ calcUniqueIDUpdateFlagsAndSize(const GlobalObject *GO, StringRef SectionName,
   // implicitly for this symbol e.g. .rodata.str1.1, then we don't need
   // to unique the section as the entry size for this symbol will be
   // compatible with implicitly created sections.
-  SmallString<128> ImplicitSectionNameStem =
-      getELFSectionNameForGlobal(GO, Kind, Mang, TM, EntrySize, false);
+  SmallString<128> ImplicitSectionNameStem = getELFSectionNameForGlobal(
+      GO, Kind, Mang, TM, EntrySize, false, /*MJTE=*/nullptr);
   if (SymbolMergeable &&
       Ctx.isELFImplicitMergeableSectionNamePrefix(SectionName) &&
       SectionName.starts_with(ImplicitSectionNameStem))
@@ -874,7 +893,8 @@ MCSection *TargetLoweringObjectFileELF::getExplicitSectionGlobal(
 static MCSectionELF *selectELFSectionForGlobal(
     MCContext &Ctx, const GlobalObject *GO, SectionKind Kind, Mangler &Mang,
     const TargetMachine &TM, bool EmitUniqueSection, unsigned Flags,
-    unsigned *NextUniqueID, const MCSymbolELF *AssociatedSymbol) {
+    unsigned *NextUniqueID, const MCSymbolELF *AssociatedSymbol,
+    const MachineJumpTableEntry *MJTE = nullptr) {
 
   auto [Group, IsComdat, ExtraFlags] = getGlobalObjectInfo(GO, TM);
   Flags |= ExtraFlags;
@@ -893,7 +913,7 @@ static MCSectionELF *selectELFSectionForGlobal(
     }
   }
   SmallString<128> Name = getELFSectionNameForGlobal(
-      GO, Kind, Mang, TM, EntrySize, UniqueSectionName);
+      GO, Kind, Mang, TM, EntrySize, UniqueSectionName, MJTE);
 
   // Use 0 as the unique ID for execute-only text.
   if (Kind.isExecuteOnly())
@@ -967,17 +987,23 @@ MCSection *TargetLoweringObjectFileELF::getUniqueSectionForFunction(
 
 MCSection *TargetLoweringObjectFileELF::getSectionForJumpTable(
     const Function &F, const TargetMachine &TM) const {
+  return getSectionForJumpTable(F, TM, /*JTE=*/nullptr);
+}
+
+MCSection *TargetLoweringObjectFileELF::getSectionForJumpTable(
+    const Function &F, const TargetMachine &TM,
+    const MachineJumpTableEntry *JTE) const {
   // If the function can be removed, produce a unique section so that
   // the table doesn't prevent the removal.
   const Comdat *C = F.getComdat();
   bool EmitUniqueSection = TM.getFunctionSections() || C;
-  if (!EmitUniqueSection)
+  if (!EmitUniqueSection && !TM.getEnableStaticDataPartitioning())
     return ReadOnlySection;
 
   return selectELFSectionForGlobal(getContext(), &F, SectionKind::getReadOnly(),
                                    getMangler(), TM, EmitUniqueSection,
                                    ELF::SHF_ALLOC, &NextUniqueID,
-                                   /* AssociatedSymbol */ nullptr);
+                                   /* AssociatedSymbol */ nullptr, JTE);
 }
 
 MCSection *TargetLoweringObjectFileELF::getSectionForLSDA(
diff --git a/llvm/lib/CodeGen/TargetPassConfig.cpp b/llvm/lib/CodeGen/TargetPassConfig.cpp
index d8d9f38da3eae..5d9da9df9092a 100644
--- a/llvm/lib/CodeGen/TargetPassConfig.cpp
+++ b/llvm/lib/CodeGen/TargetPassConfig.cpp
@@ -1017,7 +1017,7 @@ bool TargetPassConfig::addCoreISelPasses() {
   if (Selector != SelectorType::GlobalISel || !isGlobalISelAbortEnabled())
     DebugifyIsSafe = false;
 
-  // Add instruction selector passes.
+  // Add instruction selector passes for global isel if enabled.
   if (Selector == SelectorType::GlobalISel) {
     SaveAndRestore SavedAddingMachinePasses(AddingMachinePasses, true);
     if (addIRTranslator())
@@ -1039,19 +1039,20 @@ bool TargetPassConfig::addCoreISelPasses() {
 
     if (addGlobalInstructionSelect())
       return true;
+  }
 
-    // Pass to reset the MachineFunction if the ISel failed.
+  // Pass to reset the MachineFunction if the ISel failed. Outside of the above
+  // if so that the verifier is not added to it.
+  if (Selector == SelectorType::GlobalISel)
     addPass(createResetMachineFunctionPass(
         reportDiagnosticWhenGlobalISelFallback(), isGlobalISelAbortEnabled()));
 
-    // Provide a fallback path when we do not want to abort on
-    // not-yet-supported input.
-    if (!isGlobalISelAbortEnabled() && addInstSelector())
+  // Run the SDAG InstSelector, providing a fallback path when we do not want to
+  // abort on not-yet-supported input.
+  if (Selector != SelectorType::GlobalISel || !isGlobalISelAbortEnabled())
+    if (addInstSelector())
       return true;
 
-  } else if (addInstSelector())
-    return true;
-
   // Expand pseudo-instructions emitted by ISel. Don't run the verifier before
   // FinalizeISel.
   addPass(&FinalizeISelID);
@@ -1256,7 +1257,7 @@ void TargetPassConfig::addMachinePasses() {
       }
     }
     addPass(createMachineFunctionSplitterPass());
-    if (SplitStaticData)
+    if (SplitStaticData || TM->Options.EnableStaticDataPartitioning)
       addPass(createStaticDataSplitterPass());
   }
   // We run the BasicBlockSections pass if either we need BB sections or BB
diff --git a/llvm/lib/CodeGen/TargetRegisterInfo.cpp b/llvm/lib/CodeGen/TargetRegisterInfo.cpp
index ba528f66980fa..e735c904e1b60 100644
--- a/llvm/lib/CodeGen/TargetRegisterInfo.cpp
+++ b/llvm/lib/CodeGen/TargetRegisterInfo.cpp
@@ -109,14 +109,14 @@ Printable printReg(Register Reg, const TargetRegisterInfo *TRI,
   return Printable([Reg, TRI, SubIdx, MRI](raw_ostream &OS) {
     if (!Reg)
       OS << "$noreg";
-    else if (Register::isStackSlot(Reg))
-      OS << "SS#" << Register::stackSlot2Index(Reg);
+    else if (Reg.isStack())
+      OS << "SS#" << Reg.stackSlotIndex();
     else if (Reg.isVirtual()) {
       StringRef Name = MRI ? MRI->getVRegName(Reg) : "";
       if (Name != "") {
         OS << '%' << Name;
       } else {
-        OS << '%' << Register::virtReg2Index(Reg);
+        OS << '%' << Reg.virtRegIndex();
       }
     } else if (!TRI)
       OS << '$' << "physreg" << Reg.id();
@@ -420,7 +420,10 @@ static bool shareSameRegisterFile(const TargetRegisterInfo &TRI,
                                   const TargetRegisterClass *SrcRC,
                                   unsigned SrcSubReg) {
   // Same register class.
-  if (DefRC == SrcRC)
+  //
+  // When processing uncoalescable copies / bitcasts, it is possible we reach
+  // here with the same register class, but mismatched subregister indices.
+  if (DefRC == SrcRC && DefSubReg == SrcSubReg)
     return true;
 
   // Both operands are sub registers. Check if they share a register class.
diff --git a/llvm/lib/CodeGen/TwoAddressInstructionPass.cpp b/llvm/lib/CodeGen/TwoAddressInstructionPass.cpp
index fb6274b09919b..6236268f77ab1 100644
--- a/llvm/lib/CodeGen/TwoAddressInstructionPass.cpp
+++ b/llvm/lib/CodeGen/TwoAddressInstructionPass.cpp
@@ -489,6 +489,9 @@ MachineInstr *TwoAddressInstructionImpl::findOnlyInterestingUse(
     bool &IsDstPhys) const {
   MachineOperand *UseOp = nullptr;
   for (MachineOperand &MO : MRI->use_nodbg_operands(Reg)) {
+    if (MO.isUndef())
+      continue;
+
     MachineInstr *MI = MO.getParent();
     if (MI->getParent() != MBB)
       return nullptr;
diff --git a/llvm/lib/CodeGen/WinEHPrepare.cpp b/llvm/lib/CodeGen/WinEHPrepare.cpp
index 6d85f07829033..b98523cac1f2f 100644
--- a/llvm/lib/CodeGen/WinEHPrepare.cpp
+++ b/llvm/lib/CodeGen/WinEHPrepare.cpp
@@ -251,7 +251,8 @@ void llvm::calculateCXXStateForAsynchEH(const BasicBlock *BB, int State,
     const BasicBlock *BB = WI->Block;
     int State = WI->State;
     delete WI;
-    if (EHInfo.BlockToStateMap.count(BB) && EHInfo.BlockToStateMap[BB] <= State)
+    if (auto It = EHInfo.BlockToStateMap.find(BB);
+        It != EHInfo.BlockToStateMap.end() && It->second <= State)
       continue; // skip blocks already visited by lower State
 
     BasicBlock::const_iterator It = BB->getFirstNonPHIIt();
@@ -312,7 +313,8 @@ void llvm::calculateSEHStateForAsynchEH(const BasicBlock *BB, int State,
     const BasicBlock *BB = WI->Block;
     int State = WI->State;
     delete WI;
-    if (EHInfo.BlockToStateMap.count(BB) && EHInfo.BlockToStateMap[BB] <= State)
+    if (auto It = EHInfo.BlockToStateMap.find(BB);
+        It != EHInfo.BlockToStateMap.end() && It->second <= State)
       continue; // skip blocks already visited by lower State
 
     BasicBlock::const_iterator It = BB->getFirstNonPHIIt();
diff --git a/llvm/lib/CodeGen/WindowScheduler.cpp b/llvm/lib/CodeGen/WindowScheduler.cpp
index f1658e36ae1e9..379740cae78d5 100644
--- a/llvm/lib/CodeGen/WindowScheduler.cpp
+++ b/llvm/lib/CodeGen/WindowScheduler.cpp
@@ -45,6 +45,7 @@
 #include "llvm/Support/CommandLine.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/TimeProfiler.h"
+#include "llvm/Target/TargetMachine.h"
 
 using namespace llvm;
 
@@ -167,7 +168,7 @@ WindowScheduler::createMachineScheduler(bool OnlyBuildGraph) {
              ? new ScheduleDAGMI(
                    Context, std::make_unique<PostGenericScheduler>(Context),
                    true)
-             : Context->PassConfig->createMachineScheduler(Context);
+             : Context->TM->createMachineScheduler(Context);
 }
 
 bool WindowScheduler::initialize() {
@@ -356,8 +357,8 @@ void WindowScheduler::generateTripleMBB() {
           // ==================================
           //          < Terminators >
           // ==================================
-          if (DefPairs.count(NewUse))
-            NewUse = DefPairs[NewUse];
+          if (auto It = DefPairs.find(NewUse); It != DefPairs.end())
+            NewUse = It->second;
           NewMI->substituteRegister(DefRegPair.first, NewUse, 0, *TRI);
         }
       // DefPairs is updated at last.
@@ -581,9 +582,10 @@ DenseMap<MachineInstr *, int> WindowScheduler::getIssueOrder(unsigned Offset,
   DenseMap<MachineInstr *, int> IssueOrder;
   int Id = 0;
   for (int Cycle = 0; Cycle < (int)II; ++Cycle) {
-    if (!CycleToMIs.count(Cycle))
+    auto It = CycleToMIs.find(Cycle);
+    if (It == CycleToMIs.end())
       continue;
-    for (auto *MI : CycleToMIs[Cycle])
+    for (auto *MI : It->second)
       IssueOrder[MI] = Id++;
   }
   return IssueOrder;
diff --git a/llvm/lib/DebugInfo/DWARF/DWARFContext.cpp b/llvm/lib/DebugInfo/DWARF/DWARFContext.cpp
index 99e1642ff23ad..15eac10688f18 100644
--- a/llvm/lib/DebugInfo/DWARF/DWARFContext.cpp
+++ b/llvm/lib/DebugInfo/DWARF/DWARFContext.cpp
@@ -1869,7 +1869,7 @@ DWARFContext::getInliningInfoForAddress(object::SectionedAddress Address,
           LineTable->getFileLineInfoForAddress(
               {Address.Address, Address.SectionIndex}, Spec.ApproximateLine,
               CU->getCompilationDir(), Spec.FLIKind, Frame);
-      } else {
+      } else if (CallLine != 0) {
         // Otherwise, use call file, call line and call column from
         // previous DIE in inlined chain.
         if (LineTable)
diff --git a/llvm/lib/DebugInfo/DWARF/DWARFDebugLine.cpp b/llvm/lib/DebugInfo/DWARF/DWARFDebugLine.cpp
index 939a5163d55ab..62bf3d4ecaaf0 100644
--- a/llvm/lib/DebugInfo/DWARF/DWARFDebugLine.cpp
+++ b/llvm/lib/DebugInfo/DWARF/DWARFDebugLine.cpp
@@ -531,6 +531,7 @@ void DWARFDebugLine::Sequence::reset() {
   FirstRowIndex = 0;
   LastRowIndex = 0;
   Empty = true;
+  StmtSeqOffset = UINT64_MAX;
 }
 
 DWARFDebugLine::LineTable::LineTable() { clear(); }
@@ -561,13 +562,12 @@ void DWARFDebugLine::LineTable::clear() {
 DWARFDebugLine::ParsingState::ParsingState(
     struct LineTable *LT, uint64_t TableOffset,
     function_ref<void(Error)> ErrorHandler)
-    : LineTable(LT), LineTableOffset(TableOffset), ErrorHandler(ErrorHandler) {
-  resetRowAndSequence();
-}
+    : LineTable(LT), LineTableOffset(TableOffset), ErrorHandler(ErrorHandler) {}
 
-void DWARFDebugLine::ParsingState::resetRowAndSequence() {
+void DWARFDebugLine::ParsingState::resetRowAndSequence(uint64_t Offset) {
   Row.reset(LineTable->Prologue.DefaultIsStmt);
   Sequence.reset();
+  Sequence.StmtSeqOffset = Offset;
 }
 
 void DWARFDebugLine::ParsingState::appendRowToMatrix() {
@@ -848,6 +848,10 @@ Error DWARFDebugLine::LineTable::parse(
     *OS << '\n';
     Row::dumpTableHeader(*OS, /*Indent=*/Verbose ? 12 : 0);
   }
+  // *OffsetPtr points to the end of the prologue - i.e. the start of the first
+  // sequence. So initialize the first sequence offset accordingly.
+  State.resetRowAndSequence(*OffsetPtr);
+
   bool TombstonedAddress = false;
   auto EmitRow = [&] {
     if (!TombstonedAddress) {
@@ -912,7 +916,9 @@ Error DWARFDebugLine::LineTable::parse(
         // into this code path - if it were invalid, the default case would be
         // followed.
         EmitRow();
-        State.resetRowAndSequence();
+        // Cursor now points to right after the end_sequence opcode - so points
+        // to the start of the next sequence - if one exists.
+        State.resetRowAndSequence(Cursor.tell());
         break;
 
       case DW_LNE_set_address:
@@ -1268,13 +1274,14 @@ Error DWARFDebugLine::LineTable::parse(
 
   // Sort all sequences so that address lookup will work faster.
   if (!Sequences.empty()) {
-    llvm::sort(Sequences, Sequence::orderByHighPC);
+    llvm::stable_sort(Sequences, Sequence::orderByHighPC);
     // Note: actually, instruction address ranges of sequences should not
     // overlap (in shared objects and executables). If they do, the address
     // lookup would still work, though, but result would be ambiguous.
     // We don't report warning in this case. For example,
     // sometimes .so compiled from multiple object files contains a few
     // rudimentary sequences for address ranges [0x0, 0xsomething).
+    // Address ranges may also overlap when using ICF.
   }
 
   // Terminate the table with a final blank line to clearly delineate it from
@@ -1364,10 +1371,11 @@ DWARFDebugLine::LineTable::lookupAddressImpl(object::SectionedAddress Address,
 
 bool DWARFDebugLine::LineTable::lookupAddressRange(
     object::SectionedAddress Address, uint64_t Size,
-    std::vector<uint32_t> &Result) const {
+    std::vector<uint32_t> &Result,
+    std::optional<uint64_t> StmtSequenceOffset) const {
 
   // Search for relocatable addresses
-  if (lookupAddressRangeImpl(Address, Size, Result))
+  if (lookupAddressRangeImpl(Address, Size, Result, StmtSequenceOffset))
     return true;
 
   if (Address.SectionIndex == object::SectionedAddress::UndefSection)
@@ -1375,12 +1383,13 @@ bool DWARFDebugLine::LineTable::lookupAddressRange(
 
   // Search for absolute addresses
   Address.SectionIndex = object::SectionedAddress::UndefSection;
-  return lookupAddressRangeImpl(Address, Size, Result);
+  return lookupAddressRangeImpl(Address, Size, Result, StmtSequenceOffset);
 }
 
 bool DWARFDebugLine::LineTable::lookupAddressRangeImpl(
     object::SectionedAddress Address, uint64_t Size,
-    std::vector<uint32_t> &Result) const {
+    std::vector<uint32_t> &Result,
+    std::optional<uint64_t> StmtSequenceOffset) const {
   if (Sequences.empty())
     return false;
   uint64_t EndAddr = Address.Address + Size;
@@ -1389,16 +1398,38 @@ bool DWARFDebugLine::LineTable::lookupAddressRangeImpl(
   Sequence.SectionIndex = Address.SectionIndex;
   Sequence.HighPC = Address.Address;
   SequenceIter LastSeq = Sequences.end();
-  SequenceIter SeqPos = llvm::upper_bound(
-      Sequences, Sequence, DWARFDebugLine::Sequence::orderByHighPC);
-  if (SeqPos == LastSeq || !SeqPos->containsPC(Address))
+  SequenceIter SeqPos;
+
+  if (StmtSequenceOffset) {
+    // If we have a statement sequence offset, find the specific sequence.
+    // Linear search for sequence with matching StmtSeqOffset
+    SeqPos = std::find_if(Sequences.begin(), LastSeq,
+                          [&](const DWARFDebugLine::Sequence &S) {
+                            return S.StmtSeqOffset == *StmtSequenceOffset;
+                          });
+
+    // If sequence not found, return false
+    if (SeqPos == LastSeq)
+      return false;
+
+    // Set LastSeq to the next sequence since we only want the one matching
+    // sequence (sequences are guaranteed to have unique StmtSeqOffset)
+    LastSeq = SeqPos + 1;
+  } else {
+    // No specific sequence requested, find first sequence containing address
+    SeqPos = std::upper_bound(Sequences.begin(), LastSeq, Sequence,
+                              DWARFDebugLine::Sequence::orderByHighPC);
+    if (SeqPos == LastSeq)
+      return false;
+  }
+
+  // If the start sequence doesn't contain the address, nothing to do
+  if (!SeqPos->containsPC(Address))
     return false;
 
   SequenceIter StartPos = SeqPos;
 
-  // Add the rows from the first sequence to the vector, starting with the
-  // index we just calculated
-
+  // Process sequences that overlap with the desired range
   while (SeqPos != LastSeq && SeqPos->LowPC < EndAddr) {
     const DWARFDebugLine::Sequence &CurSeq = *SeqPos;
     // For the first sequence, we need to find which row in the sequence is the
@@ -1508,7 +1539,8 @@ bool DWARFDebugLine::LineTable::getFileLineInfoForAddress(
     return false;
   // Take file number and line/column from the row.
   const auto &Row = Rows[RowIndex];
-  if (!getFileNameByIndex(Row.File, CompDir, Kind, Result.FileName))
+  if (Row.Line == 0 ||
+      !getFileNameByIndex(Row.File, CompDir, Kind, Result.FileName))
     return false;
   Result.Line = Row.Line;
   Result.Column = Row.Column;
diff --git a/llvm/lib/DebugInfo/DWARF/DWARFVerifier.cpp b/llvm/lib/DebugInfo/DWARF/DWARFVerifier.cpp
index 8bf513538de7c..107e79cc5a05a 100644
--- a/llvm/lib/DebugInfo/DWARF/DWARFVerifier.cpp
+++ b/llvm/lib/DebugInfo/DWARF/DWARFVerifier.cpp
@@ -967,21 +967,20 @@ void DWARFVerifier::verifyDebugLineStmtOffsets() {
       // here because we validate this in the .debug_info verifier.
       continue;
     }
-    auto Iter = StmtListToDie.find(LineTableOffset);
-    if (Iter != StmtListToDie.end()) {
+    auto [Iter, Inserted] = StmtListToDie.try_emplace(LineTableOffset, Die);
+    if (!Inserted) {
       ++NumDebugLineErrors;
+      const auto &OldDie = Iter->second;
       ErrorCategory.Report("Identical DW_AT_stmt_list section offset", [&]() {
         error() << "two compile unit DIEs, "
-                << format("0x%08" PRIx64, Iter->second.getOffset()) << " and "
+                << format("0x%08" PRIx64, OldDie.getOffset()) << " and "
                 << format("0x%08" PRIx64, Die.getOffset())
                 << ", have the same DW_AT_stmt_list section offset:\n";
-        dump(Iter->second);
+        dump(OldDie);
         dump(Die) << '\n';
       });
       // Already verified this line table before, no need to do it again.
-      continue;
     }
-    StmtListToDie[LineTableOffset] = Die;
   }
 }
 
diff --git a/llvm/lib/ExecutionEngine/IntelJITEvents/CMakeLists.txt b/llvm/lib/ExecutionEngine/IntelJITEvents/CMakeLists.txt
index b4fd04d65e263..56c529c08937c 100644
--- a/llvm/lib/ExecutionEngine/IntelJITEvents/CMakeLists.txt
+++ b/llvm/lib/ExecutionEngine/IntelJITEvents/CMakeLists.txt
@@ -1,5 +1,8 @@
 include_directories( ${CMAKE_CURRENT_SOURCE_DIR}/.. )
-include_directories( ${PROJECT_BINARY_DIR}/ittapi/include/ )
+if(NOT DEFINED ITTAPI_SOURCE_DIR)
+    set(ITTAPI_SOURCE_DIR ${PROJECT_BINARY_DIR})
+endif()
+include_directories( ${ITTAPI_SOURCE_DIR}/ittapi/include/ )
 add_llvm_component_library(LLVMIntelJITEvents
   IntelJITEventListener.cpp
 
diff --git a/llvm/lib/ExecutionEngine/JITLink/CMakeLists.txt b/llvm/lib/ExecutionEngine/JITLink/CMakeLists.txt
index e5f5a99c39bc0..65dd0c7468ae1 100644
--- a/llvm/lib/ExecutionEngine/JITLink/CMakeLists.txt
+++ b/llvm/lib/ExecutionEngine/JITLink/CMakeLists.txt
@@ -3,6 +3,7 @@ tablegen(LLVM COFFOptions.inc -gen-opt-parser-defs)
 add_public_tablegen_target(JITLinkTableGen)
 
 add_llvm_component_library(LLVMJITLink
+  CompactUnwindSupport.cpp
   DWARFRecordSectionSplitter.cpp
   EHFrameSupport.cpp
   JITLink.cpp
diff --git a/llvm/lib/ExecutionEngine/JITLink/COFF_x86_64.cpp b/llvm/lib/ExecutionEngine/JITLink/COFF_x86_64.cpp
index 151f1b337087d..8ceb08051e423 100644
--- a/llvm/lib/ExecutionEngine/JITLink/COFF_x86_64.cpp
+++ b/llvm/lib/ExecutionEngine/JITLink/COFF_x86_64.cpp
@@ -221,8 +221,7 @@ class COFFLinkGraphLowering_x86_64 {
         }
         case EdgeKind_coff_x86_64::SecRel32: {
           E.setAddend(E.getAddend() -
-                      getSectionStart(E.getTarget().getBlock().getSection())
-                          .getValue());
+                      getSectionStart(E.getTarget().getSection()).getValue());
           E.setKind(x86_64::Pointer32);
           break;
         }
diff --git a/llvm/lib/ExecutionEngine/JITLink/CompactUnwindSupport.cpp b/llvm/lib/ExecutionEngine/JITLink/CompactUnwindSupport.cpp
new file mode 100644
index 0000000000000..51e3d26479ffd
--- /dev/null
+++ b/llvm/lib/ExecutionEngine/JITLink/CompactUnwindSupport.cpp
@@ -0,0 +1,103 @@
+//=------- CompactUnwindSupport.cpp - Compact Unwind format support -------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Compact Unwind support.
+//
+//===----------------------------------------------------------------------===//
+
+#include "CompactUnwindSupport.h"
+
+#include "llvm/ADT/Sequence.h"
+
+#define DEBUG_TYPE "jitlink"
+
+namespace llvm {
+namespace jitlink {
+
+Error splitCompactUnwindBlocks(LinkGraph &G, Section &CompactUnwindSection,
+                               size_t RecordSize) {
+
+  std::vector<Block *> OriginalBlocks(CompactUnwindSection.blocks().begin(),
+                                      CompactUnwindSection.blocks().end());
+  LLVM_DEBUG({
+    dbgs() << "In " << G.getName() << " splitting compact unwind section "
+           << CompactUnwindSection.getName() << " containing "
+           << OriginalBlocks.size() << " initial blocks...\n";
+  });
+
+  while (!OriginalBlocks.empty()) {
+    auto *B = OriginalBlocks.back();
+    OriginalBlocks.pop_back();
+
+    if (B->getSize() == 0) {
+      LLVM_DEBUG({
+        dbgs() << "  Skipping empty block at "
+               << formatv("{0:x16}", B->getAddress()) << "\n";
+      });
+      continue;
+    }
+
+    unsigned NumBlocks = B->getSize() / RecordSize;
+
+    LLVM_DEBUG({
+      dbgs() << "  Splitting block at " << formatv("{0:x16}", B->getAddress())
+             << " into " << NumBlocks << " compact unwind record(s)\n";
+    });
+
+    if (B->getSize() % RecordSize)
+      return make_error<JITLinkError>(
+          "Error splitting compact unwind record in " + G.getName() +
+          ": block at " + formatv("{0:x}", B->getAddress()) + " has size " +
+          formatv("{0:x}", B->getSize()) +
+          " (not a multiple of CU record size of " +
+          formatv("{0:x}", RecordSize) + ")");
+
+    auto Blocks =
+        G.splitBlock(*B, map_range(seq(1U, NumBlocks), [=](Edge::OffsetT Idx) {
+          return Idx * RecordSize;
+        }));
+
+    for (auto *CURec : Blocks) {
+      bool AddedKeepAlive = false;
+
+      for (auto &E : CURec->edges()) {
+        if (E.getOffset() == 0) {
+          LLVM_DEBUG({
+            dbgs() << "    Updating compact unwind record at "
+                   << CURec->getAddress() << " to point to "
+                   << (E.getTarget().hasName() ? *E.getTarget().getName()
+                                               : StringRef())
+                   << " (at " << E.getTarget().getAddress() << ")\n";
+          });
+
+          if (E.getTarget().isExternal())
+            return make_error<JITLinkError>(
+                "Error adding keep-alive edge for compact unwind record at " +
+                formatv("{0:x}", CURec->getAddress()) + ": target " +
+                *E.getTarget().getName() + " is an external symbol");
+          auto &TgtBlock = E.getTarget().getBlock();
+          auto &CURecSym =
+              G.addAnonymousSymbol(*CURec, 0, RecordSize, false, false);
+          TgtBlock.addEdge(Edge::KeepAlive, 0, CURecSym, 0);
+          AddedKeepAlive = true;
+        }
+      }
+
+      if (!AddedKeepAlive)
+        return make_error<JITLinkError>(
+            "Error adding keep-alive edge for compact unwind record at " +
+            formatv("{0:x}", CURec->getAddress()) +
+            ": no outgoing target edge at offset 0");
+    }
+  }
+
+  return Error::success();
+}
+
+} // end namespace jitlink
+} // end namespace llvm
diff --git a/llvm/lib/ExecutionEngine/JITLink/CompactUnwindSupport.h b/llvm/lib/ExecutionEngine/JITLink/CompactUnwindSupport.h
new file mode 100644
index 0000000000000..c3f8834949335
--- /dev/null
+++ b/llvm/lib/ExecutionEngine/JITLink/CompactUnwindSupport.h
@@ -0,0 +1,682 @@
+//===- CompactUnwindSupportImpl.h - Compact Unwind format impl --*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Compact Unwind format support implementation details.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LIB_EXECUTIONENGINE_JITLINK_COMPACTUNWINDSUPPORTIMPL_H
+#define LIB_EXECUTIONENGINE_JITLINK_COMPACTUNWINDSUPPORTIMPL_H
+
+#include "llvm/ADT/STLExtras.h"
+#include "llvm/ExecutionEngine/JITLink/MachO.h"
+#include "llvm/Support/Debug.h"
+#include "llvm/Support/Endian.h"
+
+#define DEBUG_TYPE "jitlink_cu"
+
+namespace llvm {
+namespace jitlink {
+
+/// Split blocks in an __LD,__compact_unwind section on record boundaries.
+/// When this function returns edges within each record are guaranteed to be
+/// sorted by offset.
+Error splitCompactUnwindBlocks(LinkGraph &G, Section &CompactUnwindSection,
+                               size_t RecordSize);
+
+/// CRTP base for compact unwind traits classes. Automatically provides derived
+/// constants.
+///
+/// FIXME: Passing PtrSize as a template parameter is a hack to work around a
+///        bug in older MSVC compilers (until at least MSVC 15) where constexpr
+///        fields in the CRTP impl class were not visible to the base class.
+///        Once we no longer need to support these compilers the PtrSize
+///        template argument should be removed and PointerSize should be
+///        defined as a member in the CRTP Impl classes.
+template <typename CRTPImpl, size_t PtrSize> struct CompactUnwindTraits {
+  static constexpr size_t PointerSize = PtrSize;
+  static constexpr size_t Size = 3 * PointerSize + 2 * 4;
+  static constexpr size_t FnFieldOffset = 0;
+  static constexpr size_t SizeFieldOffset = FnFieldOffset + PointerSize;
+  static constexpr size_t EncodingFieldOffset = SizeFieldOffset + 4;
+  static constexpr size_t PersonalityFieldOffset = EncodingFieldOffset + 4;
+  static constexpr size_t LSDAFieldOffset =
+      PersonalityFieldOffset + PointerSize;
+
+  static uint32_t readPCRangeSize(ArrayRef<char> RecordContent) {
+    assert(SizeFieldOffset + 4 <= RecordContent.size() &&
+           "Truncated CU record?");
+    return support::endian::read32<CRTPImpl::Endianness>(RecordContent.data() +
+                                                         SizeFieldOffset);
+  }
+
+  static uint32_t readEncoding(ArrayRef<char> RecordContent) {
+    assert(EncodingFieldOffset + 4 <= RecordContent.size() &&
+           "Truncated CU record?");
+    return support::endian::read32<CRTPImpl::Endianness>(RecordContent.data() +
+                                                         EncodingFieldOffset);
+  }
+
+  static std::optional<uint32_t> encodeDWARFOffset(size_t Delta) {
+    uint32_t Encoded =
+        static_cast<uint32_t>(Delta) & CRTPImpl::DWARFSectionOffsetMask;
+    if (Encoded != Delta)
+      return std::nullopt;
+    return Encoded;
+  }
+};
+
+/// Architecture specific implementation of CompactUnwindManager.
+template <typename CURecTraits> class CompactUnwindManager {
+public:
+  CompactUnwindManager(StringRef CompactUnwindSectionName,
+                       StringRef UnwindInfoSectionName,
+                       StringRef EHFrameSectionName)
+      : CompactUnwindSectionName(CompactUnwindSectionName),
+        UnwindInfoSectionName(UnwindInfoSectionName),
+        EHFrameSectionName(EHFrameSectionName) {}
+
+  // Split compact unwind records, add keep-alive edges from functions to
+  // compact unwind records, and from compact unwind records to FDEs where
+  // needed.
+  //
+  // This method must be called *after* __eh_frame has been processed: it
+  // assumes that eh-frame records have been split up and keep-alive edges have
+  // been inserted.
+  Error prepareForPrune(LinkGraph &G) {
+    Section *CUSec = G.findSectionByName(CompactUnwindSectionName);
+    if (!CUSec || CUSec->empty()) {
+      LLVM_DEBUG({
+        dbgs() << "Compact unwind: No compact unwind info for " << G.getName()
+               << "\n";
+      });
+      return Error::success();
+    }
+
+    LLVM_DEBUG({
+      dbgs() << "Compact unwind: preparing " << G.getName() << " for prune\n";
+    });
+
+    Section *EHFrameSec = G.findSectionByName(EHFrameSectionName);
+
+    if (auto Err = splitCompactUnwindBlocks(G, *CUSec, CURecTraits::Size))
+      return Err;
+
+    LLVM_DEBUG({
+      dbgs() << "  Preparing " << CUSec->blocks_size() << " blocks in "
+             << CompactUnwindSectionName << "\n";
+    });
+
+    for (auto *B : CUSec->blocks()) {
+
+      // Find target function edge.
+      Edge *PCBeginEdge = nullptr;
+      for (auto &E : B->edges_at(CURecTraits::FnFieldOffset)) {
+        PCBeginEdge = &E;
+        break;
+      }
+
+      if (!PCBeginEdge)
+        return make_error<JITLinkError>(
+            "In " + G.getName() + ", compact unwind record at " +
+            formatv("{0:x}", B->getAddress()) + " has no pc-begin edge");
+
+      if (!PCBeginEdge->getTarget().isDefined())
+        return make_error<JITLinkError>(
+            "In " + G.getName() + ", compact unwind record at " +
+            formatv("{0:x}", B->getAddress()) + " points at external symbol " +
+            *PCBeginEdge->getTarget().getName());
+
+      auto &Fn = PCBeginEdge->getTarget();
+
+      if (!Fn.isDefined()) {
+        LLVM_DEBUG({
+          dbgs() << "In " << CompactUnwindSectionName << " for " << G.getName()
+                 << " encountered unexpected pc-edge to undefined symbol "
+                 << Fn.getName() << "\n";
+        });
+        continue;
+      }
+
+      uint32_t Encoding = CURecTraits::readEncoding(B->getContent());
+      bool NeedsDWARF = CURecTraits::encodingSpecifiesDWARF(Encoding);
+
+      LLVM_DEBUG({
+        dbgs() << "    Found record for function ";
+        if (Fn.hasName())
+          dbgs() << Fn.getName();
+        else
+          dbgs() << "<anon @ " << Fn.getAddress() << '>';
+        dbgs() << ": encoding = " << formatv("{0:x}", Encoding);
+        if (NeedsDWARF)
+          dbgs() << " (needs DWARF)";
+        dbgs() << "\n";
+      });
+
+      auto &CURecSym =
+          G.addAnonymousSymbol(*B, 0, CURecTraits::Size, false, false);
+
+      bool KeepAliveAlreadyPresent = false;
+      if (EHFrameSec) {
+        Edge *KeepAliveEdge = nullptr;
+        for (auto &E : Fn.getBlock().edges_at(0)) {
+          if (E.getKind() == Edge::KeepAlive && E.getTarget().isDefined() &&
+              &E.getTarget().getSection() == EHFrameSec) {
+            KeepAliveEdge = &E;
+            break;
+          }
+        }
+
+        if (KeepAliveEdge) {
+          // Found a keep-alive edge to an FDE in the eh-frame. Switch the keep
+          // alive edge to point to the CU and if the CU needs DWARF then add
+          // an extra keep-alive edge from the CU to the FDE.
+          auto &FDE = KeepAliveEdge->getTarget();
+          KeepAliveEdge->setTarget(CURecSym);
+          KeepAliveAlreadyPresent = true;
+          if (NeedsDWARF) {
+            LLVM_DEBUG({
+              dbgs() << "      Adding keep-alive edge to FDE at "
+                     << FDE.getAddress() << "\n";
+            });
+            B->addEdge(Edge::KeepAlive, 0, FDE, 0);
+          }
+        } else {
+          if (NeedsDWARF)
+            return make_error<JITLinkError>(
+                "In " + G.getName() + ", compact unwind recard ot " +
+                formatv("{0:x}", B->getAddress()) +
+                " needs DWARF, but no FDE was found");
+        }
+      } else {
+        if (NeedsDWARF)
+          return make_error<JITLinkError>(
+              "In " + G.getName() + ", compact unwind recard ot " +
+              formatv("{0:x}", B->getAddress()) + " needs DWARF, but no " +
+              EHFrameSectionName + " section exists");
+      }
+
+      if (!KeepAliveAlreadyPresent) {
+        // No FDE edge. We'll need to add a new edge from the function back
+        // to the CU record.
+        Fn.getBlock().addEdge(Edge::KeepAlive, 0, CURecSym, 0);
+      }
+    }
+
+    return Error::success();
+  }
+
+  /// Process all __compact_unwind records and reserve space for __unwind_info.
+  Error processAndReserveUnwindInfo(LinkGraph &G) {
+    // Bail out early if no unwind info.
+    Section *CUSec = G.findSectionByName(CompactUnwindSectionName);
+    if (!CUSec)
+      return Error::success();
+
+    // The __LD/__compact_unwind section is only used as input for the linker.
+    // We'll create a new __TEXT,__unwind_info section for unwind info output.
+    CUSec->setMemLifetime(orc::MemLifetime::NoAlloc);
+
+    // Find / make a mach-header to act as the base for unwind-info offsets
+    // (and to report the arch / subarch to libunwind).
+    if (auto Err = getOrCreateCompactUnwindBase(G))
+      return Err;
+
+    // Error out if there's already unwind-info in the graph: We have no idea
+    // how to merge unwind-info sections.
+    if (G.findSectionByName(UnwindInfoSectionName))
+      return make_error<JITLinkError>("In " + G.getName() + ", " +
+                                      UnwindInfoSectionName +
+                                      " already exists");
+
+    // Process the __compact_unwind section to build the Records vector that
+    // we'll use for writing the __unwind_info section.
+    if (auto Err = processCompactUnwind(G, *CUSec))
+      return Err;
+
+    // Calculate the size of __unwind_info.
+    size_t UnwindInfoSectionSize =
+        UnwindInfoSectionHeaderSize +
+        Personalities.size() * PersonalityEntrySize +
+        (NumSecondLevelPages + 1) * IndexEntrySize + NumLSDAs * LSDAEntrySize +
+        NumSecondLevelPages * SecondLevelPageHeaderSize +
+        Records.size() * SecondLevelPageEntrySize;
+
+    LLVM_DEBUG({
+      dbgs() << "In " << G.getName() << ", reserving "
+             << formatv("{0:x}", UnwindInfoSectionSize) << " bytes for "
+             << UnwindInfoSectionName << "\n";
+    });
+
+    // Create the __unwind_info section and reserve space for it.
+    Section &UnwindInfoSec =
+        G.createSection(UnwindInfoSectionName, orc::MemProt::Read);
+
+    auto UnwindInfoSectionContent = G.allocateBuffer(UnwindInfoSectionSize);
+    memset(UnwindInfoSectionContent.data(), 0, UnwindInfoSectionContent.size());
+    auto &B = G.createMutableContentBlock(
+        UnwindInfoSec, UnwindInfoSectionContent, orc::ExecutorAddr(), 8, 0);
+
+    // Add Keep-alive edges from the __unwind_info block to all of the target
+    // functions.
+    for (auto &R : Records)
+      B.addEdge(Edge::KeepAlive, 0, *R.Fn, 0);
+
+    return Error::success();
+  }
+
+  Error writeUnwindInfo(LinkGraph &G) {
+    Section *CUSec = G.findSectionByName(CompactUnwindSectionName);
+    if (!CUSec || CUSec->empty())
+      return Error::success();
+
+    Section *UnwindInfoSec = G.findSectionByName(UnwindInfoSectionName);
+    if (!UnwindInfoSec)
+      return make_error<JITLinkError>("In " + G.getName() + ", " +
+                                      UnwindInfoSectionName +
+                                      " missing after allocation");
+
+    if (UnwindInfoSec->blocks_size() != 1)
+      return make_error<JITLinkError>(
+          "In " + G.getName() + ", " + UnwindInfoSectionName +
+          " contains more than one block post-allocation");
+
+    LLVM_DEBUG(
+        { dbgs() << "Writing unwind info for " << G.getName() << "...\n"; });
+
+    mergeRecords();
+
+    auto &UnwindInfoBlock = **UnwindInfoSec->blocks().begin();
+    auto Content = UnwindInfoBlock.getMutableContent(G);
+    BinaryStreamWriter Writer(
+        {reinterpret_cast<uint8_t *>(Content.data()), Content.size()},
+        CURecTraits::Endianness);
+
+    // __unwind_info format, from mach-o/compact_unwind_encoding.h on Darwin:
+    //
+    // #define UNWIND_SECTION_VERSION 1
+    // struct unwind_info_section_header
+    // {
+    //     uint32_t    version;            // UNWIND_SECTION_VERSION
+    //     uint32_t    commonEncodingsArraySectionOffset;
+    //     uint32_t    commonEncodingsArrayCount;
+    //     uint32_t    personalityArraySectionOffset;
+    //     uint32_t    personalityArrayCount;
+    //     uint32_t    indexSectionOffset;
+    //     uint32_t    indexCount;
+    //     // compact_unwind_encoding_t[]
+    //     // uint32_t personalities[]
+    //     // unwind_info_section_header_index_entry[]
+    //     // unwind_info_section_header_lsda_index_entry[]
+    // };
+
+    if (auto Err = writeHeader(G, Writer))
+      return Err;
+
+    // Skip common encodings: JITLink doesn't use them.
+
+    if (auto Err = writePersonalities(G, Writer))
+      return Err;
+
+    // Calculate the offset to the LSDAs.
+    size_t SectionOffsetToLSDAs =
+        Writer.getOffset() + (NumSecondLevelPages + 1) * IndexEntrySize;
+
+    // Calculate offset to the 1st second-level page.
+    size_t SectionOffsetToSecondLevelPages =
+        SectionOffsetToLSDAs + NumLSDAs * LSDAEntrySize;
+
+    if (auto Err = writeIndexes(G, Writer, SectionOffsetToLSDAs,
+                                SectionOffsetToSecondLevelPages))
+      return Err;
+
+    if (auto Err = writeLSDAs(G, Writer))
+      return Err;
+
+    if (auto Err = writeSecondLevelPages(G, Writer))
+      return Err;
+
+    LLVM_DEBUG({
+      dbgs() << "    Wrote " << formatv("{0:x}", Writer.getOffset())
+             << " bytes of unwind info.\n";
+    });
+
+    return Error::success();
+  }
+
+private:
+  // Calculate the size of unwind-info.
+  static constexpr size_t MaxPersonalities = 4;
+  static constexpr size_t PersonalityShift = 28;
+
+  static constexpr size_t UnwindInfoSectionHeaderSize = 4 * 7;
+  static constexpr size_t PersonalityEntrySize = 4;
+  static constexpr size_t IndexEntrySize = 3 * 4;
+  static constexpr size_t LSDAEntrySize = 2 * 4;
+  static constexpr size_t SecondLevelPageSize = 4096;
+  static constexpr size_t SecondLevelPageHeaderSize = 8;
+  static constexpr size_t SecondLevelPageEntrySize = 8;
+  static constexpr size_t NumRecordsPerSecondLevelPage =
+      (SecondLevelPageSize - SecondLevelPageHeaderSize) /
+      SecondLevelPageEntrySize;
+
+  struct CompactUnwindRecord {
+    Symbol *Fn = nullptr;
+    uint32_t Size = 0;
+    uint32_t Encoding = 0;
+    Symbol *LSDA = nullptr;
+    Symbol *FDE = nullptr;
+  };
+
+  Error processCompactUnwind(LinkGraph &G, Section &CUSec) {
+    // TODO: Reset NumLSDAs, Personalities and CompactUnwindRecords if
+    // processing more than once.
+    assert(NumLSDAs == 0 && "NumLSDAs should be zero");
+    assert(Records.empty() && "CompactUnwindRecords vector should be empty.");
+    assert(Personalities.empty() && "Personalities vector should be empty.");
+
+    SmallVector<CompactUnwindRecord> NonUniquedRecords;
+    NonUniquedRecords.reserve(CUSec.blocks_size());
+
+    // Process __compact_unwind blocks.
+    for (auto *B : CUSec.blocks()) {
+      CompactUnwindRecord R;
+      R.Encoding = CURecTraits::readEncoding(B->getContent());
+      for (auto &E : B->edges()) {
+        switch (E.getOffset()) {
+        case CURecTraits::FnFieldOffset:
+          // This could be the function-pointer, or the FDE keep-alive. Check
+          // the type to decide.
+          if (E.getKind() == Edge::KeepAlive)
+            R.FDE = &E.getTarget();
+          else
+            R.Fn = &E.getTarget();
+          break;
+        case CURecTraits::PersonalityFieldOffset: {
+          // Add the Personality to the Personalities map and update the
+          // encoding.
+          size_t PersonalityIdx = 0;
+          for (; PersonalityIdx != Personalities.size(); ++PersonalityIdx)
+            if (Personalities[PersonalityIdx] == &E.getTarget())
+              break;
+          if (PersonalityIdx == MaxPersonalities)
+            return make_error<JITLinkError>(
+                "In " + G.getName() +
+                ", __compact_unwind contains too many personalities (max " +
+                formatv("{}", MaxPersonalities) + ")");
+          if (PersonalityIdx == Personalities.size())
+            Personalities.push_back(&E.getTarget());
+
+          R.Encoding |= (PersonalityIdx + 1) << PersonalityShift;
+          break;
+        }
+        case CURecTraits::LSDAFieldOffset:
+          ++NumLSDAs;
+          R.LSDA = &E.getTarget();
+          break;
+        default:
+          return make_error<JITLinkError>("In " + G.getName() +
+                                          ", compact unwind record at " +
+                                          formatv("{0:x}", B->getAddress()) +
+                                          " has unrecognized edge at offset " +
+                                          formatv("{0:x}", E.getOffset()));
+        }
+      }
+      Records.push_back(R);
+    }
+
+    // Sort the records into ascending order.
+    llvm::sort(Records, [](const CompactUnwindRecord &LHS,
+                           const CompactUnwindRecord &RHS) {
+      return LHS.Fn->getAddress() < RHS.Fn->getAddress();
+    });
+
+    // Calculate the number of second-level pages required.
+    NumSecondLevelPages = (Records.size() + NumRecordsPerSecondLevelPage - 1) /
+                          NumRecordsPerSecondLevelPage;
+
+    // Convert personality symbols to GOT entry pointers.
+    typename CURecTraits::GOTManager GOT(G);
+    for (auto &Personality : Personalities)
+      Personality = &GOT.getEntryForTarget(G, *Personality);
+
+    LLVM_DEBUG({
+      dbgs() << "  In " << G.getName() << ", " << CompactUnwindSectionName
+             << ": raw records = " << Records.size()
+             << ", personalities = " << Personalities.size()
+             << ", lsdas = " << NumLSDAs << "\n";
+    });
+
+    return Error::success();
+  }
+
+  void mergeRecords() {
+    SmallVector<CompactUnwindRecord> NonUniqued = std::move(Records);
+    Records.reserve(NonUniqued.size());
+
+    Records.push_back(NonUniqued.front());
+    for (size_t I = 1; I != NonUniqued.size(); ++I) {
+      auto &Next = NonUniqued[I];
+      auto &Last = Records.back();
+
+      bool NextNeedsDWARF = CURecTraits::encodingSpecifiesDWARF(Next.Encoding);
+      bool CannotBeMerged = CURecTraits::encodingCannotBeMerged(Next.Encoding);
+      if (NextNeedsDWARF || (Next.Encoding != Last.Encoding) ||
+          CannotBeMerged || Next.LSDA || Last.LSDA)
+        Records.push_back(Next);
+    }
+
+    // Recalculate derived values that may have changed.
+    NumSecondLevelPages = (Records.size() + NumRecordsPerSecondLevelPage - 1) /
+                          NumRecordsPerSecondLevelPage;
+  }
+
+  Error writeHeader(LinkGraph &G, BinaryStreamWriter &W) {
+    if (!isUInt<32>(NumSecondLevelPages + 1))
+      return make_error<JITLinkError>("In " + G.getName() + ", too many " +
+                                      UnwindInfoSectionName +
+                                      "second-level pages required");
+
+    // Write __unwind_info header.
+    size_t IndexArrayOffset = UnwindInfoSectionHeaderSize +
+                              Personalities.size() * PersonalityEntrySize;
+
+    cantFail(W.writeInteger<uint32_t>(1));
+    cantFail(W.writeInteger<uint32_t>(UnwindInfoSectionHeaderSize));
+    cantFail(W.writeInteger<uint32_t>(0));
+    cantFail(W.writeInteger<uint32_t>(UnwindInfoSectionHeaderSize));
+    cantFail(W.writeInteger<uint32_t>(Personalities.size()));
+    cantFail(W.writeInteger<uint32_t>(IndexArrayOffset));
+    cantFail(W.writeInteger<uint32_t>(NumSecondLevelPages + 1));
+
+    return Error::success();
+  }
+
+  Error writePersonalities(LinkGraph &G, BinaryStreamWriter &W) {
+    // Write personalities.
+    for (auto *PSym : Personalities) {
+      auto Delta = PSym->getAddress() - CompactUnwindBase->getAddress();
+      if (!isUInt<32>(Delta))
+        return makePersonalityRangeError(G, *PSym);
+      cantFail(W.writeInteger<uint32_t>(Delta));
+    }
+    return Error::success();
+  }
+
+  Error writeIndexes(LinkGraph &G, BinaryStreamWriter &W,
+                     size_t SectionOffsetToLSDAs,
+                     size_t SectionOffsetToSecondLevelPages) {
+    // Assume that function deltas are ok in this method -- we'll error
+    // check all of them when we write the second level pages.
+
+    // Write the header index entries.
+    size_t RecordIdx = 0;
+    size_t NumPreviousLSDAs = 0;
+    for (auto &R : Records) {
+      // If this record marks the start of a new second level page.
+      if (RecordIdx % NumRecordsPerSecondLevelPage == 0) {
+        auto FnDelta = R.Fn->getAddress() - CompactUnwindBase->getAddress();
+        auto SecondLevelPageOffset =
+            SectionOffsetToSecondLevelPages +
+            SecondLevelPageSize * (RecordIdx / NumRecordsPerSecondLevelPage);
+        auto LSDAOffset =
+            SectionOffsetToLSDAs + NumPreviousLSDAs * LSDAEntrySize;
+
+        cantFail(W.writeInteger<uint32_t>(FnDelta));
+        cantFail(W.writeInteger<uint32_t>(SecondLevelPageOffset));
+        cantFail(W.writeInteger<uint32_t>(LSDAOffset));
+      }
+      if (R.LSDA)
+        ++NumPreviousLSDAs;
+      ++RecordIdx;
+    }
+
+    // Write the index array terminator.
+    {
+      auto FnEndDelta =
+          Records.back().Fn->getRange().End - CompactUnwindBase->getAddress();
+
+      if (LLVM_UNLIKELY(!isUInt<32>(FnEndDelta)))
+        return make_error<JITLinkError>(
+            "In " + G.getName() + " " + UnwindInfoSectionName +
+            ", delta to end of functions  " +
+            formatv("{0:x}", Records.back().Fn->getRange().End) +
+            " exceeds 32 bits");
+
+      cantFail(W.writeInteger<uint32_t>(FnEndDelta));
+      cantFail(W.writeInteger<uint32_t>(0));
+      cantFail(W.writeInteger<uint32_t>(SectionOffsetToSecondLevelPages));
+    }
+
+    return Error::success();
+  }
+
+  Error writeLSDAs(LinkGraph &G, BinaryStreamWriter &W) {
+    // As with writeIndexes, assume that function deltas are ok for now.
+    for (auto &R : Records) {
+      if (R.LSDA) {
+        auto FnDelta = R.Fn->getAddress() - CompactUnwindBase->getAddress();
+        auto LSDADelta = R.LSDA->getAddress() - CompactUnwindBase->getAddress();
+
+        if (LLVM_UNLIKELY(!isUInt<32>(LSDADelta)))
+          return make_error<JITLinkError>(
+              "In " + G.getName() + " " + UnwindInfoSectionName +
+              ", delta to lsda at " + formatv("{0:x}", R.LSDA->getAddress()) +
+              " exceeds 32 bits");
+
+        cantFail(W.writeInteger<uint32_t>(FnDelta));
+        cantFail(W.writeInteger<uint32_t>(LSDADelta));
+      }
+    }
+
+    return Error::success();
+  }
+
+  Error writeSecondLevelPages(LinkGraph &G, BinaryStreamWriter &W) {
+    size_t RecordIdx = 0;
+
+    for (auto &R : Records) {
+      // When starting a new second-level page, write the page header:
+      //
+      //   2     : uint32_t    -- UNWIND_SECOND_LEVEL_REGULAR
+      //   8     : uint16_t    -- size of second level page table header
+      //   count : uint16_t    -- num entries in this second-level page
+      if (RecordIdx % NumRecordsPerSecondLevelPage == 0) {
+        constexpr uint32_t SecondLevelPageHeaderKind = 2;
+        constexpr uint16_t SecondLevelPageHeaderSize = 8;
+        uint16_t SecondLevelPageNumEntries =
+            std::min(Records.size() - RecordIdx, NumRecordsPerSecondLevelPage);
+
+        cantFail(W.writeInteger<uint32_t>(SecondLevelPageHeaderKind));
+        cantFail(W.writeInteger<uint16_t>(SecondLevelPageHeaderSize));
+        cantFail(W.writeInteger<uint16_t>(SecondLevelPageNumEntries));
+      }
+
+      // Write entry.
+      auto FnDelta = R.Fn->getAddress() - CompactUnwindBase->getAddress();
+
+      if (LLVM_UNLIKELY(!isUInt<32>(FnDelta)))
+        return make_error<JITLinkError>(
+            "In " + G.getName() + " " + UnwindInfoSectionName +
+            ", delta to function at " + formatv("{0:x}", R.Fn->getAddress()) +
+            " exceeds 32 bits");
+
+      auto Encoding = R.Encoding;
+
+      if (LLVM_UNLIKELY(CURecTraits::encodingSpecifiesDWARF(R.Encoding))) {
+        if (!EHFrameBase)
+          EHFrameBase = SectionRange(R.FDE->getSection()).getStart();
+        auto FDEDelta = R.FDE->getAddress() - EHFrameBase;
+
+        if (auto EncodedFDEDelta = CURecTraits::encodeDWARFOffset(FDEDelta))
+          Encoding |= *EncodedFDEDelta;
+        else
+          return make_error<JITLinkError>(
+              "In " + G.getName() + " " + UnwindInfoSectionName +
+              ", cannot encode delta " + formatv("{0:x}", FDEDelta) +
+              " to FDE at " + formatv("{0:x}", R.FDE->getAddress()));
+      }
+
+      cantFail(W.writeInteger<uint32_t>(FnDelta));
+      cantFail(W.writeInteger<uint32_t>(Encoding));
+
+      ++RecordIdx;
+    }
+
+    return Error::success();
+  }
+
+  Error getOrCreateCompactUnwindBase(LinkGraph &G) {
+    auto Name = G.intern("__jitlink$libunwind_dso_base");
+    CompactUnwindBase = G.findAbsoluteSymbolByName(Name);
+    if (!CompactUnwindBase) {
+      if (auto LocalCUBase = getOrCreateLocalMachOHeader(G)) {
+        CompactUnwindBase = &*LocalCUBase;
+        auto &B = LocalCUBase->getBlock();
+        G.addDefinedSymbol(B, 0, *Name, B.getSize(), Linkage::Strong,
+                           Scope::Local, false, true);
+      } else
+        return LocalCUBase.takeError();
+    }
+    CompactUnwindBase->setLive(true);
+    return Error::success();
+  }
+
+  Error makePersonalityRangeError(LinkGraph &G, Symbol &PSym) {
+    std::string ErrMsg;
+    {
+      raw_string_ostream ErrStream(ErrMsg);
+      ErrStream << "In " << G.getName() << " " << UnwindInfoSectionName
+                << ", personality ";
+      if (PSym.hasName())
+        ErrStream << PSym.getName() << " ";
+      ErrStream << "at " << PSym.getAddress()
+                << " is out of 32-bit delta range of compact-unwind base at "
+                << CompactUnwindBase->getAddress();
+    }
+    return make_error<JITLinkError>(std::move(ErrMsg));
+  }
+
+  StringRef CompactUnwindSectionName;
+  StringRef UnwindInfoSectionName;
+  StringRef EHFrameSectionName;
+  Symbol *CompactUnwindBase = nullptr;
+  orc::ExecutorAddr EHFrameBase;
+
+  size_t NumLSDAs = 0;
+  size_t NumSecondLevelPages = 0;
+  SmallVector<Symbol *, MaxPersonalities> Personalities;
+  SmallVector<CompactUnwindRecord> Records;
+};
+
+} // end namespace jitlink
+} // end namespace llvm
+
+#undef DEBUG_TYPE
+
+#endif // LIB_EXECUTIONENGINE_JITLINK_COMPACTUNWINDSUPPORTIMPL_H
diff --git a/llvm/lib/ExecutionEngine/JITLink/JITLink.cpp b/llvm/lib/ExecutionEngine/JITLink/JITLink.cpp
index 6b77330bb764b..e8ce9b2b9527d 100644
--- a/llvm/lib/ExecutionEngine/JITLink/JITLink.cpp
+++ b/llvm/lib/ExecutionEngine/JITLink/JITLink.cpp
@@ -424,7 +424,7 @@ Error makeTargetOutOfRangeError(const LinkGraph &G, const Block &B,
     if (E.getTarget().hasName()) {
       ErrStream << "\"" << E.getTarget().getName() << "\"";
     } else
-      ErrStream << E.getTarget().getBlock().getSection().getName() << " + "
+      ErrStream << E.getTarget().getSection().getName() << " + "
                 << formatv("{0:x}", E.getOffset());
     ErrStream << " at address " << formatv("{0:x}", E.getTarget().getAddress())
               << " is out of range of " << G.getEdgeKindName(E.getKind())
diff --git a/llvm/lib/ExecutionEngine/JITLink/JITLinkGeneric.h b/llvm/lib/ExecutionEngine/JITLink/JITLinkGeneric.h
index e5d05e6b1b7bf..0bf714d6fcdf3 100644
--- a/llvm/lib/ExecutionEngine/JITLink/JITLinkGeneric.h
+++ b/llvm/lib/ExecutionEngine/JITLink/JITLinkGeneric.h
@@ -162,7 +162,7 @@ template <typename LinkerImpl> class JITLinker : public JITLinkerBase {
           // If B is a block in a Standard or Finalize section then make sure
           // that no edges point to symbols in NoAlloc sections.
           assert((NoAllocSection || !E.getTarget().isDefined() ||
-                  E.getTarget().getBlock().getSection().getMemLifetime() !=
+                  E.getTarget().getSection().getMemLifetime() !=
                       orc::MemLifetime::NoAlloc) &&
                  "Block in allocated section has edge pointing to no-alloc "
                  "section");
diff --git a/llvm/lib/ExecutionEngine/JITLink/MachOLinkGraphBuilder.cpp b/llvm/lib/ExecutionEngine/JITLink/MachOLinkGraphBuilder.cpp
index 3e757f780b550..179e458c3cd1f 100644
--- a/llvm/lib/ExecutionEngine/JITLink/MachOLinkGraphBuilder.cpp
+++ b/llvm/lib/ExecutionEngine/JITLink/MachOLinkGraphBuilder.cpp
@@ -733,121 +733,5 @@ Error MachOLinkGraphBuilder::graphifyCStringSection(
   return Error::success();
 }
 
-Error CompactUnwindSplitter::operator()(LinkGraph &G) {
-  auto *CUSec = G.findSectionByName(CompactUnwindSectionName);
-  if (!CUSec)
-    return Error::success();
-
-  if (!G.getTargetTriple().isOSBinFormatMachO())
-    return make_error<JITLinkError>(
-        "Error linking " + G.getName() +
-        ": compact unwind splitting not supported on non-macho target " +
-        G.getTargetTriple().str());
-
-  unsigned CURecordSize = 0;
-  unsigned PersonalityEdgeOffset = 0;
-  unsigned LSDAEdgeOffset = 0;
-  switch (G.getTargetTriple().getArch()) {
-  case Triple::aarch64:
-  case Triple::x86_64:
-    // 64-bit compact-unwind record format:
-    // Range start: 8 bytes.
-    // Range size:  4 bytes.
-    // CU encoding: 4 bytes.
-    // Personality: 8 bytes.
-    // LSDA:        8 bytes.
-    CURecordSize = 32;
-    PersonalityEdgeOffset = 16;
-    LSDAEdgeOffset = 24;
-    break;
-  default:
-    return make_error<JITLinkError>(
-        "Error linking " + G.getName() +
-        ": compact unwind splitting not supported on " +
-        G.getTargetTriple().getArchName());
-  }
-
-  std::vector<Block *> OriginalBlocks(CUSec->blocks().begin(),
-                                      CUSec->blocks().end());
-  LLVM_DEBUG({
-    dbgs() << "In " << G.getName() << " splitting compact unwind section "
-           << CompactUnwindSectionName << " containing "
-           << OriginalBlocks.size() << " initial blocks...\n";
-  });
-
-  while (!OriginalBlocks.empty()) {
-    auto *B = OriginalBlocks.back();
-    OriginalBlocks.pop_back();
-
-    if (B->getSize() == 0) {
-      LLVM_DEBUG({
-        dbgs() << "  Skipping empty block at "
-               << formatv("{0:x16}", B->getAddress()) << "\n";
-      });
-      continue;
-    }
-
-    unsigned NumBlocks = B->getSize() / CURecordSize;
-
-    LLVM_DEBUG({
-      dbgs() << "  Splitting block at " << formatv("{0:x16}", B->getAddress())
-             << " into " << NumBlocks << " compact unwind record(s)\n";
-    });
-
-    if (B->getSize() % CURecordSize)
-      return make_error<JITLinkError>(
-          "Error splitting compact unwind record in " + G.getName() +
-          ": block at " + formatv("{0:x}", B->getAddress()) + " has size " +
-          formatv("{0:x}", B->getSize()) +
-          " (not a multiple of CU record size of " +
-          formatv("{0:x}", CURecordSize) + ")");
-
-    auto Blocks =
-        G.splitBlock(*B, map_range(seq(1U, NumBlocks), [=](Edge::OffsetT Idx) {
-          return Idx * CURecordSize;
-        }));
-
-    for (auto *CURec : Blocks) {
-      bool AddedKeepAlive = false;
-
-      for (auto &E : CURec->edges()) {
-        if (E.getOffset() == 0) {
-          LLVM_DEBUG({
-            dbgs() << "    Updating compact unwind record at "
-                   << CURec->getAddress() << " to point to "
-                   << (E.getTarget().hasName() ? *E.getTarget().getName()
-                                               : StringRef())
-                   << " (at " << E.getTarget().getAddress() << ")\n";
-          });
-
-          if (E.getTarget().isExternal())
-            return make_error<JITLinkError>(
-                "Error adding keep-alive edge for compact unwind record at " +
-                formatv("{0:x}", CURec->getAddress()) + ": target " +
-                *E.getTarget().getName() + " is an external symbol");
-          auto &TgtBlock = E.getTarget().getBlock();
-          auto &CURecSym =
-              G.addAnonymousSymbol(*CURec, 0, CURecordSize, false, false);
-          TgtBlock.addEdge(Edge::KeepAlive, 0, CURecSym, 0);
-          AddedKeepAlive = true;
-        } else if (E.getOffset() != PersonalityEdgeOffset &&
-                   E.getOffset() != LSDAEdgeOffset)
-          return make_error<JITLinkError>(
-              "Unexpected edge at offset " + formatv("{0:x}", E.getOffset()) +
-              " in compact unwind record at " +
-              formatv("{0:x}", CURec->getAddress()));
-      }
-
-      if (!AddedKeepAlive)
-        return make_error<JITLinkError>(
-            "Error adding keep-alive edge for compact unwind record at " +
-            formatv("{0:x}", CURec->getAddress()) +
-            ": no outgoing target edge at offset 0");
-    }
-  }
-
-  return Error::success();
-}
-
 } // end namespace jitlink
 } // end namespace llvm
diff --git a/llvm/lib/ExecutionEngine/JITLink/MachOLinkGraphBuilder.h b/llvm/lib/ExecutionEngine/JITLink/MachOLinkGraphBuilder.h
index 6afa01250f62d..343218ec9ad18 100644
--- a/llvm/lib/ExecutionEngine/JITLink/MachOLinkGraphBuilder.h
+++ b/llvm/lib/ExecutionEngine/JITLink/MachOLinkGraphBuilder.h
@@ -236,17 +236,6 @@ class MachOLinkGraphBuilder {
   StringMap<SectionParserFunction> CustomSectionParserFunctions;
 };
 
-/// A pass to split up __LD,__compact_unwind sections.
-class CompactUnwindSplitter {
-public:
-  CompactUnwindSplitter(StringRef CompactUnwindSectionName)
-      : CompactUnwindSectionName(CompactUnwindSectionName) {}
-  Error operator()(LinkGraph &G);
-
-private:
-  StringRef CompactUnwindSectionName;
-};
-
 } // end namespace jitlink
 } // end namespace llvm
 
diff --git a/llvm/lib/ExecutionEngine/JITLink/MachO_arm64.cpp b/llvm/lib/ExecutionEngine/JITLink/MachO_arm64.cpp
index 29061fff9c2ae..3af0c0cdeb7c3 100644
--- a/llvm/lib/ExecutionEngine/JITLink/MachO_arm64.cpp
+++ b/llvm/lib/ExecutionEngine/JITLink/MachO_arm64.cpp
@@ -13,7 +13,9 @@
 #include "llvm/ExecutionEngine/JITLink/MachO_arm64.h"
 #include "llvm/ExecutionEngine/JITLink/DWARFRecordSectionSplitter.h"
 #include "llvm/ExecutionEngine/JITLink/aarch64.h"
+#include "llvm/ExecutionEngine/Orc/Shared/MachOObjectFormat.h"
 
+#include "CompactUnwindSupport.h"
 #include "DefineExternalSectionStartAndEndSymbols.h"
 #include "MachOLinkGraphBuilder.h"
 
@@ -625,6 +627,28 @@ static Error applyPACSigningToModInitPointers(LinkGraph &G) {
   return Error::success();
 }
 
+struct CompactUnwindTraits_MachO_arm64
+    : public CompactUnwindTraits<CompactUnwindTraits_MachO_arm64,
+                                 /* PointerSize = */ 8> {
+  // FIXME: Reinstate once we no longer need the MSVC workaround. See
+  //        FIXME for CompactUnwindTraits in CompactUnwindSupport.h.
+  // constexpr static size_t PointerSize = 8;
+
+  constexpr static endianness Endianness = endianness::little;
+
+  constexpr static uint32_t EncodingModeMask = 0x0f000000;
+  constexpr static uint32_t DWARFSectionOffsetMask = 0x00ffffff;
+
+  using GOTManager = aarch64::GOTTableManager;
+
+  static bool encodingSpecifiesDWARF(uint32_t Encoding) {
+    constexpr uint32_t DWARFMode = 0x03000000;
+    return (Encoding & EncodingModeMask) == DWARFMode;
+  }
+
+  static bool encodingCannotBeMerged(uint32_t Encoding) { return false; }
+};
+
 void link_MachO_arm64(std::unique_ptr<LinkGraph> G,
                       std::unique_ptr<JITLinkContext> Ctx) {
 
@@ -637,16 +661,21 @@ void link_MachO_arm64(std::unique_ptr<LinkGraph> G,
     else
       Config.PrePrunePasses.push_back(markAllSymbolsLive);
 
-    // Add compact unwind splitter pass.
-    Config.PrePrunePasses.push_back(
-        CompactUnwindSplitter("__LD,__compact_unwind"));
-
     // Add eh-frame passes.
-    // FIXME: Prune eh-frames for which compact-unwind is available once
-    // we support compact-unwind registration with libunwind.
     Config.PrePrunePasses.push_back(createEHFrameSplitterPass_MachO_arm64());
     Config.PrePrunePasses.push_back(createEHFrameEdgeFixerPass_MachO_arm64());
 
+    // Create a compact-unwind manager for use in passes below.
+    auto CompactUnwindMgr =
+        std::make_shared<CompactUnwindManager<CompactUnwindTraits_MachO_arm64>>(
+            orc::MachOCompactUnwindSectionName, orc::MachOUnwindInfoSectionName,
+            orc::MachOEHFrameSectionName);
+
+    // Add compact unwind prepare pass.
+    Config.PrePrunePasses.push_back([CompactUnwindMgr](LinkGraph &G) {
+      return CompactUnwindMgr->prepareForPrune(G);
+    });
+
     // Resolve any external section start / end symbols.
     Config.PostAllocationPasses.push_back(
         createDefineExternalSectionStartAndEndSymbolsPass(
@@ -663,6 +692,16 @@ void link_MachO_arm64(std::unique_ptr<LinkGraph> G,
       Config.PreFixupPasses.push_back(
           aarch64::lowerPointer64AuthEdgesToSigningFunction);
     }
+
+    // Reserve unwind-info space.
+    Config.PostPrunePasses.push_back([CompactUnwindMgr](LinkGraph &G) {
+      return CompactUnwindMgr->processAndReserveUnwindInfo(G);
+    });
+
+    // Translate compact-unwind to unwind-info.
+    Config.PreFixupPasses.push_back([CompactUnwindMgr](LinkGraph &G) {
+      return CompactUnwindMgr->writeUnwindInfo(G);
+    });
   }
 
   if (auto Err = Ctx->modifyPassConfig(*G, Config))
@@ -673,11 +712,11 @@ void link_MachO_arm64(std::unique_ptr<LinkGraph> G,
 }
 
 LinkGraphPassFunction createEHFrameSplitterPass_MachO_arm64() {
-  return DWARFRecordSectionSplitter("__TEXT,__eh_frame");
+  return DWARFRecordSectionSplitter(orc::MachOEHFrameSectionName);
 }
 
 LinkGraphPassFunction createEHFrameEdgeFixerPass_MachO_arm64() {
-  return EHFrameEdgeFixer("__TEXT,__eh_frame", aarch64::PointerSize,
+  return EHFrameEdgeFixer(orc::MachOEHFrameSectionName, aarch64::PointerSize,
                           aarch64::Pointer32, aarch64::Pointer64,
                           aarch64::Delta32, aarch64::Delta64,
                           aarch64::NegDelta32);
diff --git a/llvm/lib/ExecutionEngine/JITLink/MachO_x86_64.cpp b/llvm/lib/ExecutionEngine/JITLink/MachO_x86_64.cpp
index 9547266dc9789..bb5f3ab7ed43c 100644
--- a/llvm/lib/ExecutionEngine/JITLink/MachO_x86_64.cpp
+++ b/llvm/lib/ExecutionEngine/JITLink/MachO_x86_64.cpp
@@ -13,7 +13,9 @@
 #include "llvm/ExecutionEngine/JITLink/MachO_x86_64.h"
 #include "llvm/ExecutionEngine/JITLink/DWARFRecordSectionSplitter.h"
 #include "llvm/ExecutionEngine/JITLink/x86_64.h"
+#include "llvm/ExecutionEngine/Orc/Shared/MachOObjectFormat.h"
 
+#include "CompactUnwindSupport.h"
 #include "DefineExternalSectionStartAndEndSymbols.h"
 #include "MachOLinkGraphBuilder.h"
 
@@ -500,26 +502,58 @@ Expected<std::unique_ptr<LinkGraph>> createLinkGraphFromMachOObject_x86_64(
       .buildGraph();
 }
 
+struct CompactUnwindTraits_MachO_x86_64
+    : public CompactUnwindTraits<CompactUnwindTraits_MachO_x86_64,
+                                 /* PointerSize = */ 8> {
+  // FIXME: Reinstate once we no longer need the MSVC workaround. See
+  //        FIXME for CompactUnwindTraits in CompactUnwindSupport.h.
+  // constexpr static size_t PointerSize = 8;
+
+  constexpr static endianness Endianness = endianness::little;
+
+  constexpr static uint32_t EncodingModeMask = 0x0f000000;
+  constexpr static uint32_t DWARFSectionOffsetMask = 0x00ffffff;
+
+  using GOTManager = x86_64::GOTTableManager;
+
+  static bool encodingSpecifiesDWARF(uint32_t Encoding) {
+    constexpr uint32_t DWARFMode = 0x04000000;
+    return (Encoding & EncodingModeMask) == DWARFMode;
+  }
+
+  static bool encodingCannotBeMerged(uint32_t Encoding) {
+    constexpr uint32_t StackIndirectMode = 0x03000000;
+    return (Encoding & EncodingModeMask) == StackIndirectMode;
+  }
+};
+
 void link_MachO_x86_64(std::unique_ptr<LinkGraph> G,
                        std::unique_ptr<JITLinkContext> Ctx) {
 
   PassConfiguration Config;
 
   if (Ctx->shouldAddDefaultTargetPasses(G->getTargetTriple())) {
-    // Add eh-frame passes.
-    Config.PrePrunePasses.push_back(createEHFrameSplitterPass_MachO_x86_64());
-    Config.PrePrunePasses.push_back(createEHFrameEdgeFixerPass_MachO_x86_64());
-
-    // Add compact unwind splitter pass.
-    Config.PrePrunePasses.push_back(
-        CompactUnwindSplitter("__LD,__compact_unwind"));
-
     // Add a mark-live pass.
     if (auto MarkLive = Ctx->getMarkLivePass(G->getTargetTriple()))
       Config.PrePrunePasses.push_back(std::move(MarkLive));
     else
       Config.PrePrunePasses.push_back(markAllSymbolsLive);
 
+    // Add eh-frame passes.
+    Config.PrePrunePasses.push_back(createEHFrameSplitterPass_MachO_x86_64());
+    Config.PrePrunePasses.push_back(createEHFrameEdgeFixerPass_MachO_x86_64());
+
+    // Create a compact-unwind manager for use in passes below.
+    auto CompactUnwindMgr = std::make_shared<
+        CompactUnwindManager<CompactUnwindTraits_MachO_x86_64>>(
+        orc::MachOCompactUnwindSectionName, orc::MachOUnwindInfoSectionName,
+        orc::MachOEHFrameSectionName);
+
+    // Add compact unwind prepare pass.
+    Config.PrePrunePasses.push_back([CompactUnwindMgr](LinkGraph &G) {
+      return CompactUnwindMgr->prepareForPrune(G);
+    });
+
     // Resolve any external section start / end symbols.
     Config.PostAllocationPasses.push_back(
         createDefineExternalSectionStartAndEndSymbolsPass(
@@ -528,6 +562,16 @@ void link_MachO_x86_64(std::unique_ptr<LinkGraph> G,
     // Add an in-place GOT/Stubs pass.
     Config.PostPrunePasses.push_back(buildGOTAndStubs_MachO_x86_64);
 
+    // Reserve space for unwind-info.
+    Config.PostPrunePasses.push_back([CompactUnwindMgr](LinkGraph &G) {
+      return CompactUnwindMgr->processAndReserveUnwindInfo(G);
+    });
+
+    // Translate compact-unwind to unwind-info.
+    Config.PreFixupPasses.push_back([CompactUnwindMgr](LinkGraph &G) {
+      return CompactUnwindMgr->writeUnwindInfo(G);
+    });
+
     // Add GOT/Stubs optimizer pass.
     Config.PreFixupPasses.push_back(x86_64::optimizeGOTAndStubAccesses);
   }
@@ -540,11 +584,11 @@ void link_MachO_x86_64(std::unique_ptr<LinkGraph> G,
 }
 
 LinkGraphPassFunction createEHFrameSplitterPass_MachO_x86_64() {
-  return DWARFRecordSectionSplitter("__TEXT,__eh_frame");
+  return DWARFRecordSectionSplitter(orc::MachOEHFrameSectionName);
 }
 
 LinkGraphPassFunction createEHFrameEdgeFixerPass_MachO_x86_64() {
-  return EHFrameEdgeFixer("__TEXT,__eh_frame", x86_64::PointerSize,
+  return EHFrameEdgeFixer(orc::MachOEHFrameSectionName, x86_64::PointerSize,
                           x86_64::Pointer32, x86_64::Pointer64, x86_64::Delta32,
                           x86_64::Delta64, x86_64::NegDelta32);
 }
diff --git a/llvm/lib/ExecutionEngine/JITLink/aarch32.cpp b/llvm/lib/ExecutionEngine/JITLink/aarch32.cpp
index 7b255717f2383..01bb6e0403578 100644
--- a/llvm/lib/ExecutionEngine/JITLink/aarch32.cpp
+++ b/llvm/lib/ExecutionEngine/JITLink/aarch32.cpp
@@ -874,7 +874,7 @@ bool StubsManager_prev7::visitEdge(LinkGraph &G, Block *B, Edge &E) {
   LLVM_DEBUG({
     dbgs() << "    Using " << (UseThumb ? "Thumb" : "Arm") << " entrypoint "
            << *StubEntrypoint << " in "
-           << StubEntrypoint->getBlock().getSection().getName() << "\n";
+           << StubEntrypoint->getSection().getName() << "\n";
   });
 
   E.setTarget(*StubEntrypoint);
@@ -919,8 +919,8 @@ bool StubsManager_v7::visitEdge(LinkGraph &G, Block *B, Edge &E) {
          "Instruction set states of stub and relocation site should be equal");
   LLVM_DEBUG({
     dbgs() << "    Using " << (MakeThumb ? "Thumb" : "Arm") << " entry "
-           << *StubSymbol << " in "
-           << StubSymbol->getBlock().getSection().getName() << "\n";
+           << *StubSymbol << " in " << StubSymbol->getSection().getName()
+           << "\n";
   });
 
   E.setTarget(*StubSymbol);
diff --git a/llvm/lib/ExecutionEngine/JITLink/aarch64.cpp b/llvm/lib/ExecutionEngine/JITLink/aarch64.cpp
index e2364ad786a42..8067e2a173a00 100644
--- a/llvm/lib/ExecutionEngine/JITLink/aarch64.cpp
+++ b/llvm/lib/ExecutionEngine/JITLink/aarch64.cpp
@@ -38,6 +38,8 @@ const char *getEdgeKindName(Edge::Kind R) {
   switch (R) {
   case Pointer64:
     return "Pointer64";
+  case Pointer64Authenticated:
+    return "Pointer64Authenticated";
   case Pointer32:
     return "Pointer32";
   case Delta64:
@@ -315,53 +317,50 @@ Error lowerPointer64AuthEdgesToSigningFunction(LinkGraph &G) {
   };
 
   for (auto *B : G.blocks()) {
-    for (auto EI = B->edges().begin(); EI != B->edges().end();) {
-      auto &E = *EI;
-      if (E.getKind() == aarch64::Pointer64Authenticated) {
-        uint64_t EncodedInfo = E.getAddend();
-        int32_t RealAddend = (uint32_t)(EncodedInfo & 0xffffffff);
-        uint32_t InitialDiscriminator = (EncodedInfo >> 32) & 0xffff;
-        bool AddressDiversify = (EncodedInfo >> 48) & 0x1;
-        uint32_t Key = (EncodedInfo >> 49) & 0x3;
-        uint32_t HighBits = EncodedInfo >> 51;
-        auto ValueToSign = E.getTarget().getAddress() + RealAddend;
-
-        if (HighBits != 0x1000)
-          return make_error<JITLinkError>(
-              "Pointer64Auth edge at " +
-              formatv("{0:x}", B->getFixupAddress(E).getValue()) +
-              " has invalid encoded addend  " + formatv("{0:x}", EncodedInfo));
-
-#ifndef NDEBUG
+    for (auto &E : B->edges()) {
+      // We're only concerned with Pointer64Authenticated edges here.
+      if (E.getKind() != aarch64::Pointer64Authenticated)
+        continue;
+
+      uint64_t EncodedInfo = E.getAddend();
+      int32_t RealAddend = (uint32_t)(EncodedInfo & 0xffffffff);
+      uint32_t InitialDiscriminator = (EncodedInfo >> 32) & 0xffff;
+      bool AddressDiversify = (EncodedInfo >> 48) & 0x1;
+      uint32_t Key = (EncodedInfo >> 49) & 0x3;
+      uint32_t HighBits = EncodedInfo >> 51;
+      auto ValueToSign = E.getTarget().getAddress() + RealAddend;
+
+      if (HighBits != 0x1000)
+        return make_error<JITLinkError>(
+            "Pointer64Auth edge at " +
+            formatv("{0:x}", B->getFixupAddress(E).getValue()) +
+            " has invalid encoded addend  " + formatv("{0:x}", EncodedInfo));
+
+      LLVM_DEBUG({
         const char *const KeyNames[] = {"IA", "IB", "DA", "DB"};
-#endif // NDEBUG
-        LLVM_DEBUG({
-          dbgs() << "  " << B->getFixupAddress(E) << " <- " << ValueToSign
-                 << " : key = " << KeyNames[Key] << ", discriminator = "
-                 << formatv("{0:x4}", InitialDiscriminator)
-                 << ", address diversified = "
-                 << (AddressDiversify ? "yes" : "no") << "\n";
-        });
-
-        // Materialize pointer value.
-        cantFail(
-            writeMovRegImm64Seq(AppendInstr, Reg1, ValueToSign.getValue()));
-
-        // Materialize fixup pointer.
-        cantFail(writeMovRegImm64Seq(AppendInstr, Reg2,
-                                     B->getFixupAddress(E).getValue()));
-
-        // Write signing instruction(s).
-        cantFail(writePACSignSeq(AppendInstr, Reg1, ValueToSign, Reg2, Reg3,
-                                 Key, InitialDiscriminator, AddressDiversify));
-
-        // Store signed pointer.
-        cantFail(writeStoreRegSeq(AppendInstr, Reg2, Reg1));
-
-        // Remove this edge.
-        EI = B->removeEdge(EI);
-      } else
-        ++EI;
+        dbgs() << "  " << B->getFixupAddress(E) << " <- " << ValueToSign
+               << " : key = " << KeyNames[Key] << ", discriminator = "
+               << formatv("{0:x4}", InitialDiscriminator)
+               << ", address diversified = "
+               << (AddressDiversify ? "yes" : "no") << "\n";
+      });
+
+      // Materialize pointer value.
+      cantFail(writeMovRegImm64Seq(AppendInstr, Reg1, ValueToSign.getValue()));
+
+      // Materialize fixup pointer.
+      cantFail(writeMovRegImm64Seq(AppendInstr, Reg2,
+                                   B->getFixupAddress(E).getValue()));
+
+      // Write signing instruction(s).
+      cantFail(writePACSignSeq(AppendInstr, Reg1, ValueToSign, Reg2, Reg3, Key,
+                               InitialDiscriminator, AddressDiversify));
+
+      // Store signed pointer.
+      cantFail(writeStoreRegSeq(AppendInstr, Reg2, Reg1));
+
+      // Replace edge with a keep-alive to preserve dependence info.
+      E.setKind(Edge::KeepAlive);
     }
   }
 
diff --git a/llvm/lib/ExecutionEngine/Orc/CMakeLists.txt b/llvm/lib/ExecutionEngine/Orc/CMakeLists.txt
index 2ab5d6dd39b63..8a866294eee25 100644
--- a/llvm/lib/ExecutionEngine/Orc/CMakeLists.txt
+++ b/llvm/lib/ExecutionEngine/Orc/CMakeLists.txt
@@ -57,6 +57,7 @@ add_llvm_component_library(LLVMOrcJIT
   ExecutorProcessControl.cpp
   TaskDispatch.cpp
   ThreadSafeModule.cpp
+  UnwindInfoRegistrationPlugin.cpp
   RedirectionManager.cpp
   JITLinkRedirectableSymbolManager.cpp
   ReOptimizeLayer.cpp
diff --git a/llvm/lib/ExecutionEngine/Orc/CompileUtils.cpp b/llvm/lib/ExecutionEngine/Orc/CompileUtils.cpp
index 5d2f3cd4a8be8..c4d65af1b57f8 100644
--- a/llvm/lib/ExecutionEngine/Orc/CompileUtils.cpp
+++ b/llvm/lib/ExecutionEngine/Orc/CompileUtils.cpp
@@ -33,6 +33,9 @@ irManglingOptionsFromTargetOptions(const TargetOptions &Opts) {
 
 /// Compile a Module to an ObjectFile.
 Expected<SimpleCompiler::CompileResult> SimpleCompiler::operator()(Module &M) {
+  if (M.getDataLayout().isDefault())
+    M.setDataLayout(TM.createDataLayout());
+
   CompileResult CachedObject = tryToLoadFromObjectCache(M);
   if (CachedObject)
     return std::move(CachedObject);
diff --git a/llvm/lib/ExecutionEngine/Orc/Core.cpp b/llvm/lib/ExecutionEngine/Orc/Core.cpp
index d47eb4416d3c2..9f466e725668a 100644
--- a/llvm/lib/ExecutionEngine/Orc/Core.cpp
+++ b/llvm/lib/ExecutionEngine/Orc/Core.cpp
@@ -1251,9 +1251,7 @@ JITDylib::JITDylib(ExecutionSession &ES, std::string Name)
   LinkOrder.push_back({this, JITDylibLookupFlags::MatchAllSymbols});
 }
 
-std::pair<JITDylib::AsynchronousSymbolQuerySet,
-          std::shared_ptr<SymbolDependenceMap>>
-JITDylib::IL_removeTracker(ResourceTracker &RT) {
+JITDylib::RemoveTrackerResult JITDylib::IL_removeTracker(ResourceTracker &RT) {
   // Note: Should be called under the session lock.
   assert(State != Closed && "JD is defunct");
 
@@ -1292,7 +1290,10 @@ JITDylib::IL_removeTracker(ResourceTracker &RT) {
       SymbolsToFail.push_back(Sym);
   }
 
-  auto Result = ES.IL_failSymbols(*this, std::move(SymbolsToFail));
+  auto [QueriesToFail, FailedSymbols] =
+      ES.IL_failSymbols(*this, std::move(SymbolsToFail));
+
+  std::vector<std::unique_ptr<MaterializationUnit>> DefunctMUs;
 
   // Removed symbols should be taken out of the table altogether.
   for (auto &Sym : SymbolsToRemove) {
@@ -1302,7 +1303,12 @@ JITDylib::IL_removeTracker(ResourceTracker &RT) {
     // Remove Materializer if present.
     if (I->second.hasMaterializerAttached()) {
       // FIXME: Should this discard the symbols?
-      UnmaterializedInfos.erase(Sym);
+      auto J = UnmaterializedInfos.find(Sym);
+      assert(J != UnmaterializedInfos.end() &&
+             "Symbol table indicates MU present, but no UMI record");
+      if (J->second->MU)
+        DefunctMUs.push_back(std::move(J->second->MU));
+      UnmaterializedInfos.erase(J);
     } else {
       assert(!UnmaterializedInfos.count(Sym) &&
              "Symbol has materializer attached");
@@ -1313,7 +1319,8 @@ JITDylib::IL_removeTracker(ResourceTracker &RT) {
 
   shrinkMaterializationInfoMemory();
 
-  return Result;
+  return {std::move(QueriesToFail), std::move(FailedSymbols),
+          std::move(DefunctMUs)};
 }
 
 void JITDylib::transferTracker(ResourceTracker &DstRT, ResourceTracker &SrcRT) {
@@ -2180,16 +2187,17 @@ Error ExecutionSession::removeResourceTracker(ResourceTracker &RT) {
   });
   std::vector<ResourceManager *> CurrentResourceManagers;
 
-  JITDylib::AsynchronousSymbolQuerySet QueriesToFail;
-  std::shared_ptr<SymbolDependenceMap> FailedSymbols;
+  JITDylib::RemoveTrackerResult R;
 
   runSessionLocked([&] {
     CurrentResourceManagers = ResourceManagers;
     RT.makeDefunct();
-    std::tie(QueriesToFail, FailedSymbols) =
-        RT.getJITDylib().IL_removeTracker(RT);
+    R = RT.getJITDylib().IL_removeTracker(RT);
   });
 
+  // Release any defunct MaterializationUnits.
+  R.DefunctMUs.clear();
+
   Error Err = Error::success();
 
   auto &JD = RT.getJITDylib();
@@ -2197,9 +2205,9 @@ Error ExecutionSession::removeResourceTracker(ResourceTracker &RT) {
     Err = joinErrors(std::move(Err),
                      L->handleRemoveResources(JD, RT.getKeyUnsafe()));
 
-  for (auto &Q : QueriesToFail)
-    Q->handleFailed(
-        make_error<FailedToMaterialize>(getSymbolStringPool(), FailedSymbols));
+  for (auto &Q : R.QueriesToFail)
+    Q->handleFailed(make_error<FailedToMaterialize>(getSymbolStringPool(),
+                                                    R.FailedSymbols));
 
   return Err;
 }
diff --git a/llvm/lib/ExecutionEngine/Orc/Debugging/PerfSupportPlugin.cpp b/llvm/lib/ExecutionEngine/Orc/Debugging/PerfSupportPlugin.cpp
index 7a970f3cccf46..bc5b3bb538e39 100644
--- a/llvm/lib/ExecutionEngine/Orc/Debugging/PerfSupportPlugin.cpp
+++ b/llvm/lib/ExecutionEngine/Orc/Debugging/PerfSupportPlugin.cpp
@@ -113,7 +113,7 @@ getCodeLoadRecord(const Symbol &Sym, std::atomic<uint64_t> &CodeIndex) {
 
 static std::optional<PerfJITDebugInfoRecord>
 getDebugInfoRecord(const Symbol &Sym, DWARFContext &DC) {
-  auto &Section = Sym.getBlock().getSection();
+  auto &Section = Sym.getSection();
   auto Addr = Sym.getAddress();
   auto Size = Sym.getSize();
   auto SAddr = object::SectionedAddress{Addr.getValue(), Section.getOrdinal()};
diff --git a/llvm/lib/ExecutionEngine/Orc/Debugging/VTuneSupportPlugin.cpp b/llvm/lib/ExecutionEngine/Orc/Debugging/VTuneSupportPlugin.cpp
index 9715a503629bf..1f4557217cf24 100644
--- a/llvm/lib/ExecutionEngine/Orc/Debugging/VTuneSupportPlugin.cpp
+++ b/llvm/lib/ExecutionEngine/Orc/Debugging/VTuneSupportPlugin.cpp
@@ -63,7 +63,7 @@ static VTuneMethodBatch getMethodBatch(LinkGraph &G, bool EmitDebugInfo) {
     if (!EmitDebugInfo)
       continue;
 
-    auto &Section = Sym->getBlock().getSection();
+    auto &Section = Sym->getSection();
     auto Addr = Sym->getAddress();
     auto SAddr =
         object::SectionedAddress{Addr.getValue(), Section.getOrdinal()};
diff --git a/llvm/lib/ExecutionEngine/Orc/EHFrameRegistrationPlugin.cpp b/llvm/lib/ExecutionEngine/Orc/EHFrameRegistrationPlugin.cpp
index 217c693dae9c9..161bd68ff0854 100644
--- a/llvm/lib/ExecutionEngine/Orc/EHFrameRegistrationPlugin.cpp
+++ b/llvm/lib/ExecutionEngine/Orc/EHFrameRegistrationPlugin.cpp
@@ -9,6 +9,7 @@
 #include "llvm/ExecutionEngine/Orc/EHFrameRegistrationPlugin.h"
 
 #include "llvm/ExecutionEngine/JITLink/EHFrameSupport.h"
+#include "llvm/ExecutionEngine/Orc/Shared/MachOObjectFormat.h"
 
 #define DEBUG_TYPE "orc"
 
@@ -21,11 +22,19 @@ EHFrameRegistrationPlugin::EHFrameRegistrationPlugin(
     : ES(ES), Registrar(std::move(Registrar)) {}
 
 void EHFrameRegistrationPlugin::modifyPassConfig(
-    MaterializationResponsibility &MR, LinkGraph &G,
+    MaterializationResponsibility &MR, LinkGraph &LG,
     PassConfiguration &PassConfig) {
 
+  if (LG.getTargetTriple().isOSBinFormatMachO())
+    PassConfig.PrePrunePasses.insert(
+        PassConfig.PrePrunePasses.begin(), [](LinkGraph &G) {
+          if (auto *CUSec = G.findSectionByName(MachOCompactUnwindSectionName))
+            G.removeSection(*CUSec);
+          return Error::success();
+        });
+
   PassConfig.PostFixupPasses.push_back(createEHFrameRecorderPass(
-      G.getTargetTriple(), [this, &MR](ExecutorAddr Addr, size_t Size) {
+      LG.getTargetTriple(), [this, &MR](ExecutorAddr Addr, size_t Size) {
         if (Addr) {
           std::lock_guard<std::mutex> Lock(EHFramePluginMutex);
           assert(!InProcessLinks.count(&MR) &&
diff --git a/llvm/lib/ExecutionEngine/Orc/ELFNixPlatform.cpp b/llvm/lib/ExecutionEngine/Orc/ELFNixPlatform.cpp
index aae7369fc29c4..7f0a45941cf9b 100644
--- a/llvm/lib/ExecutionEngine/Orc/ELFNixPlatform.cpp
+++ b/llvm/lib/ExecutionEngine/Orc/ELFNixPlatform.cpp
@@ -1,5 +1,4 @@
-//===------ ELFNixPlatform.cpp - Utilities for executing ELFNix in Orc
-//-----===//
+//===----- ELFNixPlatform.cpp - Utilities for executing ELFNix in Orc -----===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
diff --git a/llvm/lib/ExecutionEngine/Orc/ExecutorProcessControl.cpp b/llvm/lib/ExecutionEngine/Orc/ExecutorProcessControl.cpp
index aa799687e6d5d..7b38150ab4b65 100644
--- a/llvm/lib/ExecutionEngine/Orc/ExecutorProcessControl.cpp
+++ b/llvm/lib/ExecutionEngine/Orc/ExecutorProcessControl.cpp
@@ -9,8 +9,7 @@
 #include "llvm/ExecutionEngine/Orc/ExecutorProcessControl.h"
 
 #include "llvm/ExecutionEngine/Orc/Core.h"
-#include "llvm/ExecutionEngine/Orc/Shared/OrcRTBridge.h"
-#include "llvm/ExecutionEngine/Orc/TargetProcess/RegisterEHFrames.h"
+#include "llvm/ExecutionEngine/Orc/TargetProcess/DefaultHostBootstrapValues.h"
 #include "llvm/ExecutionEngine/Orc/TargetProcess/TargetExecutionUtils.h"
 #include "llvm/Support/Process.h"
 #include "llvm/TargetParser/Host.h"
@@ -45,13 +44,20 @@ SelfExecutorProcessControl::SelfExecutorProcessControl(
   this->DylibMgr = this;
   this->JDI = {ExecutorAddr::fromPtr(jitDispatchViaWrapperFunctionManager),
                ExecutorAddr::fromPtr(this)};
+
   if (this->TargetTriple.isOSBinFormatMachO())
     GlobalManglingPrefix = '_';
 
-  this->BootstrapSymbols[rt::RegisterEHFrameSectionWrapperName] =
-      ExecutorAddr::fromPtr(&llvm_orc_registerEHFrameSectionWrapper);
-  this->BootstrapSymbols[rt::DeregisterEHFrameSectionWrapperName] =
-      ExecutorAddr::fromPtr(&llvm_orc_deregisterEHFrameSectionWrapper);
+  addDefaultBootstrapValuesForHostProcess(BootstrapMap, BootstrapSymbols);
+
+#ifdef __APPLE__
+  // FIXME: Don't add an UnwindInfoManager by default -- it's redundant when
+  //        the ORC runtime is loaded. We'll need a way to document this and
+  //        allow clients to choose.
+  this->UnwindInfoMgr = UnwindInfoManager::TryCreate();
+  if (this->UnwindInfoMgr)
+    this->UnwindInfoMgr->addBootstrapSymbols(this->BootstrapSymbols);
+#endif // __APPLE__
 }
 
 Expected<std::unique_ptr<SelfExecutorProcessControl>>
diff --git a/llvm/lib/ExecutionEngine/Orc/LLJIT.cpp b/llvm/lib/ExecutionEngine/Orc/LLJIT.cpp
index b1dadbae93fec..dd844ae3a42bc 100644
--- a/llvm/lib/ExecutionEngine/Orc/LLJIT.cpp
+++ b/llvm/lib/ExecutionEngine/Orc/LLJIT.cpp
@@ -21,6 +21,7 @@
 #include "llvm/ExecutionEngine/Orc/ObjectTransformLayer.h"
 #include "llvm/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.h"
 #include "llvm/ExecutionEngine/Orc/TargetProcess/RegisterEHFrames.h"
+#include "llvm/ExecutionEngine/Orc/UnwindInfoRegistrationPlugin.h"
 #include "llvm/ExecutionEngine/SectionMemoryManager.h"
 #include "llvm/IR/GlobalVariable.h"
 #include "llvm/IR/IRBuilder.h"
@@ -833,14 +834,7 @@ Error LLJITBuilderState::prepareForConstruction() {
       CreateObjectLinkingLayer =
           [](ExecutionSession &ES,
              const Triple &) -> Expected<std::unique_ptr<ObjectLayer>> {
-        auto ObjLinkingLayer = std::make_unique<ObjectLinkingLayer>(ES);
-        if (auto EHFrameRegistrar = EPCEHFrameRegistrar::Create(ES))
-          ObjLinkingLayer->addPlugin(
-              std::make_unique<EHFrameRegistrationPlugin>(
-                  ES, std::move(*EHFrameRegistrar)));
-        else
-          return EHFrameRegistrar.takeError();
-        return std::move(ObjLinkingLayer);
+        return std::make_unique<ObjectLinkingLayer>(ES);
       };
     }
   }
@@ -1225,6 +1219,49 @@ Expected<JITDylibSP> setUpGenericLLVMIRPlatform(LLJIT &J) {
   auto &PlatformJD = J.getExecutionSession().createBareJITDylib("<Platform>");
   PlatformJD.addToLinkOrder(*ProcessSymbolsJD);
 
+  if (auto *OLL = dyn_cast<ObjectLinkingLayer>(&J.getObjLinkingLayer())) {
+
+    bool UseEHFrames = true;
+
+    // Enable compact-unwind support if possible.
+    if (J.getTargetTriple().isOSDarwin() ||
+        J.getTargetTriple().isOSBinFormatMachO()) {
+
+      // Check if the bootstrap map says that we should force eh-frames:
+      // Older libunwinds require this as they don't have a dynamic
+      // registration API for compact-unwind.
+      std::optional<bool> ForceEHFrames;
+      if (auto Err = J.getExecutionSession().getBootstrapMapValue<bool, bool>(
+              "darwin-use-ehframes-only", ForceEHFrames))
+        return Err;
+      if (ForceEHFrames.has_value())
+        UseEHFrames = *ForceEHFrames;
+      else
+        UseEHFrames = false;
+
+      // If UseEHFrames hasn't been set then we're good to use compact-unwind.
+      if (!UseEHFrames) {
+        if (auto UIRP = UnwindInfoRegistrationPlugin::Create(
+                J.getIRCompileLayer(), PlatformJD)) {
+          OLL->addPlugin(std::move(*UIRP));
+          LLVM_DEBUG(dbgs() << "Enabled compact-unwind support.\n");
+        } else
+          return UIRP.takeError();
+      }
+    }
+
+    // Otherwise fall back to standard unwind registration.
+    if (UseEHFrames) {
+      auto &ES = J.getExecutionSession();
+      if (auto EHFrameRegistrar = EPCEHFrameRegistrar::Create(ES)) {
+        OLL->addPlugin(std::make_unique<EHFrameRegistrationPlugin>(
+            ES, std::move(*EHFrameRegistrar)));
+        LLVM_DEBUG(dbgs() << "Enabled eh-frame support.\n");
+      } else
+        return EHFrameRegistrar.takeError();
+    }
+  }
+
   J.setPlatformSupport(
       std::make_unique<GenericLLVMIRPlatformSupport>(J, PlatformJD));
 
diff --git a/llvm/lib/ExecutionEngine/Orc/LinkGraphLinkingLayer.cpp b/llvm/lib/ExecutionEngine/Orc/LinkGraphLinkingLayer.cpp
index b4f78c617ae78..1bb444077ad15 100644
--- a/llvm/lib/ExecutionEngine/Orc/LinkGraphLinkingLayer.cpp
+++ b/llvm/lib/ExecutionEngine/Orc/LinkGraphLinkingLayer.cpp
@@ -499,7 +499,10 @@ LinkGraphLinkingLayer::LinkGraphLinkingLayer(
 }
 
 LinkGraphLinkingLayer::~LinkGraphLinkingLayer() {
-  assert(Allocs.empty() && "Layer destroyed with resources still attached");
+  assert(Allocs.empty() &&
+         "Layer destroyed with resources still attached "
+         "(ExecutionSession::endSession() must be called prior to "
+         "destruction)");
   getExecutionSession().deregisterResourceManager(*this);
 }
 
diff --git a/llvm/lib/ExecutionEngine/Orc/MachOPlatform.cpp b/llvm/lib/ExecutionEngine/Orc/MachOPlatform.cpp
index 48d54190fafb6..d4e341a96f5b1 100644
--- a/llvm/lib/ExecutionEngine/Orc/MachOPlatform.cpp
+++ b/llvm/lib/ExecutionEngine/Orc/MachOPlatform.cpp
@@ -9,6 +9,7 @@
 #include "llvm/ExecutionEngine/Orc/MachOPlatform.h"
 
 #include "llvm/BinaryFormat/MachO.h"
+#include "llvm/ExecutionEngine/JITLink/EHFrameSupport.h"
 #include "llvm/ExecutionEngine/JITLink/MachO.h"
 #include "llvm/ExecutionEngine/JITLink/aarch64.h"
 #include "llvm/ExecutionEngine/JITLink/x86_64.h"
@@ -480,6 +481,15 @@ MachOPlatform::MachOPlatform(
   ObjLinkingLayer.addPlugin(std::make_unique<MachOPlatformPlugin>(*this));
   PlatformJD.addGenerator(std::move(OrcRuntimeGenerator));
 
+  {
+    // Check for force-eh-frame
+    std::optional<bool> ForceEHFrames;
+    if ((Err = ES.getBootstrapMapValue<bool, bool>("darwin-use-ehframes-only",
+                                                   ForceEHFrames)))
+      return;
+    this->ForceEHFrames = ForceEHFrames.has_value() ? *ForceEHFrames : false;
+  }
+
   BootstrapInfo BI;
   Bootstrap = &BI;
 
@@ -810,6 +820,12 @@ void MachOPlatform::MachOPlatformPlugin::modifyPassConfig(
       HeaderAddr = I->second;
   }
 
+  // If we're forcing eh-frame use then discard the compact-unwind section
+  // immediately to prevent FDEs from being stripped.
+  if (MP.ForceEHFrames)
+    if (auto *CUSec = LG.findSectionByName(MachOCompactUnwindSectionName))
+      LG.removeSection(*CUSec);
+
   // Point the libunwind dso-base absolute symbol at the header for the
   // JITDylib. This will prevent us from synthesizing a new header for
   // every object.
@@ -1070,7 +1086,7 @@ Error MachOPlatform::MachOPlatformPlugin::processObjCImageInfo(
       for (auto *B : Sec.blocks())
         for (auto &E : B->edges())
           if (E.getTarget().isDefined() &&
-              &E.getTarget().getBlock().getSection() == ObjCImageInfo)
+              &E.getTarget().getSection() == ObjCImageInfo)
             return make_error<StringError>(MachOObjCImageInfoSectionName +
                                                " is referenced within file " +
                                                G.getName(),
@@ -1256,7 +1272,8 @@ MachOPlatform::MachOPlatformPlugin::findUnwindSectionInfo(
   // ScanSection records a section range and adds any executable blocks that
   // that section points to to the CodeBlocks vector.
   SmallVector<Block *> CodeBlocks;
-  auto ScanUnwindInfoSection = [&](Section &Sec, ExecutorAddrRange &SecRange) {
+  auto ScanUnwindInfoSection = [&](Section &Sec, ExecutorAddrRange &SecRange,
+                                   auto AddCodeBlocks) {
     if (Sec.blocks().empty())
       return;
     SecRange = (*Sec.blocks().begin())->getRange();
@@ -1264,23 +1281,31 @@ MachOPlatform::MachOPlatformPlugin::findUnwindSectionInfo(
       auto R = B->getRange();
       SecRange.Start = std::min(SecRange.Start, R.Start);
       SecRange.End = std::max(SecRange.End, R.End);
-      for (auto &E : B->edges()) {
-        if (E.getKind() != Edge::KeepAlive || !E.getTarget().isDefined())
-          continue;
-        auto &TargetBlock = E.getTarget().getBlock();
-        auto &TargetSection = TargetBlock.getSection();
-        if ((TargetSection.getMemProt() & MemProt::Exec) == MemProt::Exec)
-          CodeBlocks.push_back(&TargetBlock);
-      }
+      AddCodeBlocks(*B);
     }
   };
 
-  if (Section *EHFrameSec = G.findSectionByName(MachOEHFrameSectionName))
-    ScanUnwindInfoSection(*EHFrameSec, US.DwarfSection);
+  if (Section *EHFrameSec = G.findSectionByName(MachOEHFrameSectionName)) {
+    ScanUnwindInfoSection(*EHFrameSec, US.DwarfSection, [&](Block &B) {
+      if (auto *Fn = jitlink::EHFrameCFIBlockInspector::FromEdgeScan(B)
+                         .getPCBeginEdge())
+        if (Fn->getTarget().isDefined())
+          CodeBlocks.push_back(&Fn->getTarget().getBlock());
+    });
+  }
 
-  if (Section *CUInfoSec =
-          G.findSectionByName(MachOCompactUnwindInfoSectionName))
-    ScanUnwindInfoSection(*CUInfoSec, US.CompactUnwindSection);
+  if (Section *CUInfoSec = G.findSectionByName(MachOUnwindInfoSectionName)) {
+    ScanUnwindInfoSection(
+        *CUInfoSec, US.CompactUnwindSection, [&](Block &B) {
+          for (auto &E : B.edges()) {
+            assert(E.getTarget().isDefined() &&
+                   "unwind-info record edge has external target");
+            assert(E.getKind() == Edge::KeepAlive &&
+                   "unwind-info record has unexpected edge kind");
+            CodeBlocks.push_back(&E.getTarget().getBlock());
+          }
+        });
+  }
 
   // If we didn't find any pointed-to code-blocks then there's no need to
   // register any info.
diff --git a/llvm/lib/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.cpp b/llvm/lib/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.cpp
index a73b2310d193a..88cceacb71184 100644
--- a/llvm/lib/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.cpp
+++ b/llvm/lib/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.cpp
@@ -88,7 +88,10 @@ RTDyldObjectLinkingLayer::RTDyldObjectLinkingLayer(
 }
 
 RTDyldObjectLinkingLayer::~RTDyldObjectLinkingLayer() {
-  assert(MemMgrs.empty() && "Layer destroyed with resources still attached");
+  assert(MemMgrs.empty() &&
+         "Layer destroyed with resources still attached"
+         "(ExecutionSession::endSession() must be called prior to "
+         "destruction)");
 }
 
 void RTDyldObjectLinkingLayer::emit(
diff --git a/llvm/lib/ExecutionEngine/Orc/Shared/MachOObjectFormat.cpp b/llvm/lib/ExecutionEngine/Orc/Shared/MachOObjectFormat.cpp
index 11e8eb7bc3a19..53488b75a2686 100644
--- a/llvm/lib/ExecutionEngine/Orc/Shared/MachOObjectFormat.cpp
+++ b/llvm/lib/ExecutionEngine/Orc/Shared/MachOObjectFormat.cpp
@@ -18,7 +18,7 @@ namespace orc {
 StringRef MachODataCommonSectionName = "__DATA,__common";
 StringRef MachODataDataSectionName = "__DATA,__data";
 StringRef MachOEHFrameSectionName = "__TEXT,__eh_frame";
-StringRef MachOCompactUnwindInfoSectionName = "__TEXT,__unwind_info";
+StringRef MachOCompactUnwindSectionName = "__LD,__compact_unwind";
 StringRef MachOCStringSectionName = "__TEXT,__cstring";
 StringRef MachOModInitFuncSectionName = "__DATA,__mod_init_func";
 StringRef MachOObjCCatListSectionName = "__DATA,__objc_catlist";
@@ -46,6 +46,7 @@ StringRef MachOTextTextSectionName = "__TEXT,__text";
 StringRef MachOThreadBSSSectionName = "__DATA,__thread_bss";
 StringRef MachOThreadDataSectionName = "__DATA,__thread_data";
 StringRef MachOThreadVarsSectionName = "__DATA,__thread_vars";
+StringRef MachOUnwindInfoSectionName = "__TEXT,__unwind_info";
 
 StringRef MachOInitSectionNames[22] = {
     MachOModInitFuncSectionName,         MachOObjCCatListSectionName,
diff --git a/llvm/lib/ExecutionEngine/Orc/Shared/OrcRTBridge.cpp b/llvm/lib/ExecutionEngine/Orc/Shared/OrcRTBridge.cpp
index 54a25c007c589..fef3ff989a52a 100644
--- a/llvm/lib/ExecutionEngine/Orc/Shared/OrcRTBridge.cpp
+++ b/llvm/lib/ExecutionEngine/Orc/Shared/OrcRTBridge.cpp
@@ -64,5 +64,19 @@ const char *RunAsIntFunctionWrapperName =
     "__llvm_orc_bootstrap_run_as_int_function_wrapper";
 
 } // end namespace rt
+namespace rt_alt {
+const char *UnwindInfoManagerInstanceName =
+    "orc_rt_alt_UnwindInfoManager_Instance";
+const char *UnwindInfoManagerFindSectionsHelperName =
+    "orc_rt_alt_UnwindInfoManager_findSectionsHelper";
+const char *UnwindInfoManagerEnableWrapperName =
+    "orc_rt_alt_UnwindInfoManager_enable";
+const char *UnwindInfoManagerDisableWrapperName =
+    "orc_rt_alt_UnwindInfoManager_disable";
+const char *UnwindInfoManagerRegisterActionName =
+    "orc_rt_alt_UnwindInfoManager_register";
+const char *UnwindInfoManagerDeregisterActionName =
+    "orc_rt_alt_UnwindInfoManager_deregister";
+} // end namespace rt_alt
 } // end namespace orc
 } // end namespace llvm
diff --git a/llvm/lib/ExecutionEngine/Orc/TargetProcess/CMakeLists.txt b/llvm/lib/ExecutionEngine/Orc/TargetProcess/CMakeLists.txt
index 3d1dfe758c79d..9f3abac156adb 100644
--- a/llvm/lib/ExecutionEngine/Orc/TargetProcess/CMakeLists.txt
+++ b/llvm/lib/ExecutionEngine/Orc/TargetProcess/CMakeLists.txt
@@ -6,11 +6,15 @@ set(intel_jit_profiling )
 if( LLVM_USE_INTEL_JITEVENTS )
   set(intel_jit_profiling IntelJITProfiling)
   include_directories(${CMAKE_CURRENT_SOURCE_DIR}/../../IntelJITProfiling)
-  include_directories(${PROJECT_BINARY_DIR}/ittapi/include/ )
+  if(NOT DEFINED ITTAPI_SOURCE_DIR)
+      set(ITTAPI_SOURCE_DIR ${PROJECT_BINARY_DIR})
+  endif()
+  include_directories(${ITTAPI_SOURCE_DIR}/ittapi/include/ )
 endif()
 
 add_llvm_component_library(LLVMOrcTargetProcess
   ExecutorSharedMemoryMapperService.cpp
+  DefaultHostBootstrapValues.cpp
   JITLoaderGDB.cpp
   JITLoaderPerf.cpp
   JITLoaderVTune.cpp
@@ -20,6 +24,7 @@ add_llvm_component_library(LLVMOrcTargetProcess
   SimpleExecutorMemoryManager.cpp
   SimpleRemoteEPCServer.cpp
   TargetExecutionUtils.cpp
+  UnwindInfoManager.cpp
 
   ADDITIONAL_HEADER_DIRS
   ${LLVM_MAIN_INCLUDE_DIR}/llvm/ExecutionEngine/Orc
diff --git a/llvm/lib/ExecutionEngine/Orc/TargetProcess/DefaultHostBootstrapValues.cpp b/llvm/lib/ExecutionEngine/Orc/TargetProcess/DefaultHostBootstrapValues.cpp
new file mode 100644
index 0000000000000..c95b7ac5159fe
--- /dev/null
+++ b/llvm/lib/ExecutionEngine/Orc/TargetProcess/DefaultHostBootstrapValues.cpp
@@ -0,0 +1,36 @@
+//===----- DefaultHostBootstrapValues.cpp - Defaults for host process -----===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/ExecutionEngine/Orc/TargetProcess/DefaultHostBootstrapValues.h"
+
+#include "llvm/ExecutionEngine/Orc/Shared/OrcRTBridge.h"
+#include "llvm/ExecutionEngine/Orc/TargetProcess/RegisterEHFrames.h"
+
+#ifdef __APPLE__
+#include <dlfcn.h>
+#endif // __APPLE__
+
+namespace llvm::orc {
+
+void addDefaultBootstrapValuesForHostProcess(
+    StringMap<std::vector<char>> &BootstrapMap,
+    StringMap<ExecutorAddr> &BootstrapSymbols) {
+
+  // FIXME: We probably shouldn't set these on Windows?
+  BootstrapSymbols[rt::RegisterEHFrameSectionWrapperName] =
+      ExecutorAddr::fromPtr(&llvm_orc_registerEHFrameSectionWrapper);
+  BootstrapSymbols[rt::DeregisterEHFrameSectionWrapperName] =
+      ExecutorAddr::fromPtr(&llvm_orc_deregisterEHFrameSectionWrapper);
+
+#ifdef __APPLE__
+  if (!dlsym(RTLD_DEFAULT, "__unw_add_find_dynamic_unwind_sections"))
+    BootstrapMap["darwin-use-ehframes-only"].push_back(1);
+#endif // __APPLE__
+}
+
+} // namespace llvm::orc
diff --git a/llvm/lib/ExecutionEngine/Orc/TargetProcess/OrcRTBootstrap.cpp b/llvm/lib/ExecutionEngine/Orc/TargetProcess/OrcRTBootstrap.cpp
index d88fbbfc86385..c4f201b353d27 100644
--- a/llvm/lib/ExecutionEngine/Orc/TargetProcess/OrcRTBootstrap.cpp
+++ b/llvm/lib/ExecutionEngine/Orc/TargetProcess/OrcRTBootstrap.cpp
@@ -106,10 +106,6 @@ void addTo(StringMap<ExecutorAddr> &M) {
       ExecutorAddr::fromPtr(&writeBuffersWrapper);
   M[rt::MemoryWritePointersWrapperName] =
       ExecutorAddr::fromPtr(&writePointersWrapper);
-  M[rt::RegisterEHFrameSectionWrapperName] =
-      ExecutorAddr::fromPtr(&llvm_orc_registerEHFrameSectionWrapper);
-  M[rt::DeregisterEHFrameSectionWrapperName] =
-      ExecutorAddr::fromPtr(&llvm_orc_deregisterEHFrameSectionWrapper);
   M[rt::RunAsMainWrapperName] = ExecutorAddr::fromPtr(&runAsMainWrapper);
   M[rt::RunAsVoidFunctionWrapperName] =
       ExecutorAddr::fromPtr(&runAsVoidFunctionWrapper);
diff --git a/llvm/lib/ExecutionEngine/Orc/TargetProcess/UnwindInfoManager.cpp b/llvm/lib/ExecutionEngine/Orc/TargetProcess/UnwindInfoManager.cpp
new file mode 100644
index 0000000000000..9f748154c03e5
--- /dev/null
+++ b/llvm/lib/ExecutionEngine/Orc/TargetProcess/UnwindInfoManager.cpp
@@ -0,0 +1,188 @@
+//===------- UnwindInfoManager.cpp - Register unwind info sections --------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/ExecutionEngine/Orc/TargetProcess/UnwindInfoManager.h"
+#include "llvm/ExecutionEngine/Orc/Shared/OrcRTBridge.h"
+#include "llvm/ExecutionEngine/Orc/Shared/WrapperFunctionUtils.h"
+#include "llvm/Support/DynamicLibrary.h"
+
+#define DEBUG_TYPE "orc"
+
+using namespace llvm;
+using namespace llvm::orc;
+using namespace llvm::orc::shared;
+
+static orc::shared::CWrapperFunctionResult
+llvm_orc_rt_alt_UnwindInfoManager_enable(const char *Data, uint64_t Size) {
+  return WrapperFunction<SPSError(SPSExecutorAddr, SPSExecutorAddr)>::handle(
+             Data, Size,
+             [](ExecutorAddr Instance, ExecutorAddr FindFn) {
+               return Instance.toPtr<UnwindInfoManager *>()->enable(
+                   FindFn.toPtr<void *>());
+             })
+      .release();
+}
+
+static orc::shared::CWrapperFunctionResult
+llvm_orc_rt_alt_UnwindInfoManager_disable(const char *Data, uint64_t Size) {
+  return WrapperFunction<SPSError(SPSExecutorAddr)>::handle(
+             Data, Size,
+             [](ExecutorAddr Instance) {
+               return Instance.toPtr<UnwindInfoManager *>()->disable();
+             })
+      .release();
+}
+
+static orc::shared::CWrapperFunctionResult
+llvm_orc_rt_alt_UnwindInfoManager_register(const char *Data, uint64_t Size) {
+  using SPSSig =
+      SPSError(SPSExecutorAddr, SPSSequence<SPSExecutorAddrRange>,
+               SPSExecutorAddr, SPSExecutorAddrRange, SPSExecutorAddrRange);
+
+  return WrapperFunction<SPSSig>::handle(
+             Data, Size,
+             [](ExecutorAddr Instance,
+                std::vector<ExecutorAddrRange> CodeRanges, ExecutorAddr DSOBase,
+                ExecutorAddrRange DWARFRange,
+                ExecutorAddrRange CompactUnwindRange) {
+               return Instance.toPtr<UnwindInfoManager *>()->registerSections(
+                   CodeRanges, DSOBase, DWARFRange, CompactUnwindRange);
+             })
+      .release();
+}
+
+static orc::shared::CWrapperFunctionResult
+llvm_orc_rt_alt_UnwindInfoManager_deregister(const char *Data, uint64_t Size) {
+  using SPSSig = SPSError(SPSExecutorAddr, SPSSequence<SPSExecutorAddrRange>);
+
+  return WrapperFunction<SPSSig>::handle(
+             Data, Size,
+             [](ExecutorAddr Instance,
+                std::vector<ExecutorAddrRange> CodeRanges) {
+               return Instance.toPtr<UnwindInfoManager *>()->deregisterSections(
+                   CodeRanges);
+             })
+      .release();
+}
+
+namespace llvm::orc {
+
+const char *UnwindInfoManager::AddFnName =
+    "__unw_add_find_dynamic_unwind_sections";
+const char *UnwindInfoManager::RemoveFnName =
+    "__unw_remove_find_dynamic_unwind_sections";
+
+std::unique_ptr<UnwindInfoManager> UnwindInfoManager::TryCreate() {
+  std::string ErrMsg;
+  auto DL = sys::DynamicLibrary::getPermanentLibrary(nullptr, &ErrMsg);
+  if (!DL.isValid())
+    return nullptr;
+
+  auto AddFindDynamicUnwindSections =
+      (int (*)(void *))DL.getAddressOfSymbol(AddFnName);
+  if (!AddFindDynamicUnwindSections)
+    return nullptr;
+
+  auto RemoveFindDynamicUnwindSections =
+      (int (*)(void *))DL.getAddressOfSymbol(RemoveFnName);
+  if (!RemoveFindDynamicUnwindSections)
+    return nullptr;
+
+  return std::unique_ptr<UnwindInfoManager>(new UnwindInfoManager(
+      AddFindDynamicUnwindSections, RemoveFindDynamicUnwindSections));
+}
+
+Error UnwindInfoManager::shutdown() { return Error::success(); }
+
+void UnwindInfoManager::addBootstrapSymbols(StringMap<ExecutorAddr> &M) {
+  M[rt_alt::UnwindInfoManagerInstanceName] = ExecutorAddr::fromPtr(this);
+  M[rt_alt::UnwindInfoManagerFindSectionsHelperName] =
+      ExecutorAddr::fromPtr(&findSectionsHelper);
+  M[rt_alt::UnwindInfoManagerEnableWrapperName] =
+      ExecutorAddr::fromPtr(llvm_orc_rt_alt_UnwindInfoManager_enable);
+  M[rt_alt::UnwindInfoManagerDisableWrapperName] =
+      ExecutorAddr::fromPtr(llvm_orc_rt_alt_UnwindInfoManager_disable);
+  M[rt_alt::UnwindInfoManagerRegisterActionName] =
+      ExecutorAddr::fromPtr(llvm_orc_rt_alt_UnwindInfoManager_register);
+  M[rt_alt::UnwindInfoManagerDeregisterActionName] =
+      ExecutorAddr::fromPtr(llvm_orc_rt_alt_UnwindInfoManager_deregister);
+}
+
+Error UnwindInfoManager::enable(void *FindDynamicUnwindSections) {
+  LLVM_DEBUG(dbgs() << "Enabling UnwindInfoManager.\n");
+
+  if (auto Err = AddFindDynamicUnwindSections(FindDynamicUnwindSections))
+    return make_error<StringError>(Twine("Could not register function via ") +
+                                       AddFnName +
+                                       ", error code = " + Twine(Err),
+                                   inconvertibleErrorCode());
+
+  this->FindDynamicUnwindSections = FindDynamicUnwindSections;
+  return Error::success();
+}
+
+Error UnwindInfoManager::disable(void) {
+  LLVM_DEBUG(dbgs() << "Disabling UnwindInfoManager.\n");
+
+  if (FindDynamicUnwindSections)
+    if (auto Err = RemoveFindDynamicUnwindSections(FindDynamicUnwindSections))
+      return make_error<StringError>(
+          Twine("Could not deregister function via ") + RemoveFnName +
+              "error code = " + Twine(Err),
+          inconvertibleErrorCode());
+
+  FindDynamicUnwindSections = nullptr;
+  return Error::success();
+}
+
+Error UnwindInfoManager::registerSections(
+    ArrayRef<ExecutorAddrRange> CodeRanges, ExecutorAddr DSOBase,
+    ExecutorAddrRange DWARFEHFrame, ExecutorAddrRange CompactUnwind) {
+  std::lock_guard<std::mutex> Lock(M);
+  for (auto &R : CodeRanges)
+    UWSecs[R.Start.getValue()] =
+        UnwindSections{static_cast<uintptr_t>(DSOBase.getValue()),
+                       static_cast<uintptr_t>(DWARFEHFrame.Start.getValue()),
+                       static_cast<size_t>(DWARFEHFrame.size()),
+                       static_cast<uintptr_t>(CompactUnwind.Start.getValue()),
+                       static_cast<size_t>(CompactUnwind.size())};
+  return Error::success();
+}
+
+Error UnwindInfoManager::deregisterSections(
+    ArrayRef<ExecutorAddrRange> CodeRanges) {
+  std::lock_guard<std::mutex> Lock(M);
+  for (auto &R : CodeRanges) {
+    auto I = UWSecs.find(R.Start.getValue());
+    if (I == UWSecs.end())
+      return make_error<StringError>(
+          "No unwind-info sections registered for range " +
+              formatv("{0:x} - {1:x}", R.Start, R.End),
+          inconvertibleErrorCode());
+    UWSecs.erase(I);
+  }
+  return Error::success();
+}
+
+int UnwindInfoManager::findSections(uintptr_t Addr, UnwindSections *Info) {
+  std::lock_guard<std::mutex> Lock(M);
+  auto I = UWSecs.upper_bound(Addr);
+  if (I == UWSecs.begin())
+    return 0;
+  --I;
+  *Info = I->second;
+  return 1;
+}
+
+int UnwindInfoManager::findSectionsHelper(UnwindInfoManager *Instance,
+                                          uintptr_t Addr,
+                                          UnwindSections *Info) {
+  return Instance->findSections(Addr, Info);
+}
+
+} // namespace llvm::orc
diff --git a/llvm/lib/ExecutionEngine/Orc/UnwindInfoRegistrationPlugin.cpp b/llvm/lib/ExecutionEngine/Orc/UnwindInfoRegistrationPlugin.cpp
new file mode 100644
index 0000000000000..ae1f3f98269db
--- /dev/null
+++ b/llvm/lib/ExecutionEngine/Orc/UnwindInfoRegistrationPlugin.cpp
@@ -0,0 +1,238 @@
+//===----- UnwindInfoRegistrationPlugin.cpp - libunwind registration ------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/ExecutionEngine/Orc/UnwindInfoRegistrationPlugin.h"
+
+#include "llvm/ADT/ScopeExit.h"
+#include "llvm/ExecutionEngine/Orc/AbsoluteSymbols.h"
+#include "llvm/ExecutionEngine/Orc/Shared/MachOObjectFormat.h"
+#include "llvm/ExecutionEngine/Orc/Shared/OrcRTBridge.h"
+#include "llvm/IR/IRBuilder.h"
+#include "llvm/IR/Module.h"
+
+#define DEBUG_TYPE "orc"
+
+using namespace llvm::jitlink;
+
+static const char *FindDynamicUnwindSectionsFunctionName =
+    "_orc_rt_alt_find_dynamic_unwind_sections";
+
+namespace llvm::orc {
+
+Expected<std::shared_ptr<UnwindInfoRegistrationPlugin>>
+UnwindInfoRegistrationPlugin::Create(IRLayer &IRL, JITDylib &PlatformJD,
+                                     ExecutorAddr Instance,
+                                     ExecutorAddr FindHelper,
+                                     ExecutorAddr Enable, ExecutorAddr Disable,
+                                     ExecutorAddr Register,
+                                     ExecutorAddr Deregister) {
+
+  auto &ES = IRL.getExecutionSession();
+
+  // Build bouncer module.
+  auto M = makeBouncerModule(ES);
+  if (!M)
+    return M.takeError();
+
+  auto BouncerRT = PlatformJD.createResourceTracker();
+  auto RemoveBouncerModule = make_scope_exit([&]() {
+    if (auto Err = BouncerRT->remove())
+      ES.reportError(std::move(Err));
+  });
+
+  if (auto Err = PlatformJD.define(absoluteSymbols(
+          {{ES.intern(rt_alt::UnwindInfoManagerInstanceName),
+            ExecutorSymbolDef(Instance, JITSymbolFlags())},
+           {ES.intern(rt_alt::UnwindInfoManagerFindSectionsHelperName),
+            ExecutorSymbolDef(FindHelper, JITSymbolFlags::Callable)}})))
+    return std::move(Err);
+
+  if (auto Err = IRL.add(BouncerRT, std::move(*M)))
+    return Err;
+
+  auto FindUnwindSections =
+      ES.lookup({&PlatformJD}, FindDynamicUnwindSectionsFunctionName);
+  if (!FindUnwindSections)
+    return FindUnwindSections.takeError();
+
+  using namespace shared;
+  using SPSEnableSig = SPSError(SPSExecutorAddr, SPSExecutorAddr);
+  Error CallErr = Error::success();
+  if (auto Err = ES.callSPSWrapper<SPSEnableSig>(
+          Enable, CallErr, Instance, FindUnwindSections->getAddress())) {
+    consumeError(std::move(CallErr));
+    return std::move(Err);
+  }
+
+  if (CallErr)
+    return std::move(CallErr);
+
+  RemoveBouncerModule.release();
+
+  return std::shared_ptr<UnwindInfoRegistrationPlugin>(
+      new UnwindInfoRegistrationPlugin(ES, Instance, Disable, Register,
+                                       Deregister));
+}
+
+Expected<std::shared_ptr<UnwindInfoRegistrationPlugin>>
+UnwindInfoRegistrationPlugin::Create(IRLayer &IRL, JITDylib &PlatformJD) {
+
+  ExecutorAddr Instance, FindHelper, Enable, Disable, Register, Deregister;
+
+  auto &EPC = IRL.getExecutionSession().getExecutorProcessControl();
+  if (auto Err = EPC.getBootstrapSymbols(
+          {{Instance, rt_alt::UnwindInfoManagerInstanceName},
+           {FindHelper, rt_alt::UnwindInfoManagerFindSectionsHelperName},
+           {Enable, rt_alt::UnwindInfoManagerEnableWrapperName},
+           {Disable, rt_alt::UnwindInfoManagerDisableWrapperName},
+           {Register, rt_alt::UnwindInfoManagerRegisterActionName},
+           {Deregister, rt_alt::UnwindInfoManagerDeregisterActionName}}))
+    return std::move(Err);
+
+  return Create(IRL, PlatformJD, Instance, FindHelper, Enable, Disable,
+                Register, Deregister);
+}
+
+UnwindInfoRegistrationPlugin::~UnwindInfoRegistrationPlugin() {
+  using namespace shared;
+  using SPSDisableSig = SPSError(SPSExecutorAddr);
+  Error CallErr = Error::success();
+  if (auto Err = ES.callSPSWrapper<SPSDisableSig>(Disable, CallErr, Instance)) {
+    consumeError(std::move(CallErr));
+    ES.reportError(std::move(Err));
+  }
+  if (CallErr)
+    ES.reportError(std::move(CallErr));
+}
+
+void UnwindInfoRegistrationPlugin::modifyPassConfig(
+    MaterializationResponsibility &MR, LinkGraph &G,
+    PassConfiguration &PassConfig) {
+
+  PassConfig.PostFixupPasses.push_back(
+      [this](LinkGraph &G) { return addUnwindInfoRegistrationActions(G); });
+}
+
+Expected<ThreadSafeModule>
+UnwindInfoRegistrationPlugin::makeBouncerModule(ExecutionSession &ES) {
+  auto Ctx = std::make_unique<LLVMContext>();
+  auto M = std::make_unique<Module>("__libunwind_find_unwind_bouncer", *Ctx);
+  M->setTargetTriple(ES.getTargetTriple().str());
+
+  auto EscapeName = [](const char *N) { return std::string("\01") + N; };
+
+  auto *PtrTy = PointerType::getUnqual(*Ctx);
+  auto *OpaqueStructTy = StructType::create(*Ctx, "UnwindInfoMgr");
+  auto *UnwindMgrInstance = new GlobalVariable(
+      *M, OpaqueStructTy, true, GlobalValue::ExternalLinkage, nullptr,
+      EscapeName(rt_alt::UnwindInfoManagerInstanceName));
+
+  auto *Int64Ty = Type::getInt64Ty(*Ctx);
+  auto *FindHelperTy = FunctionType::get(Int64Ty, {PtrTy, PtrTy, PtrTy}, false);
+  auto *FindHelperFn = Function::Create(
+      FindHelperTy, GlobalValue::ExternalLinkage,
+      EscapeName(rt_alt::UnwindInfoManagerFindSectionsHelperName), *M);
+
+  auto *FindFnTy = FunctionType::get(Int64Ty, {PtrTy, PtrTy}, false);
+  auto *FindFn =
+      Function::Create(FindFnTy, GlobalValue::ExternalLinkage,
+                       EscapeName(FindDynamicUnwindSectionsFunctionName), *M);
+  auto *EntryBlock = BasicBlock::Create(M->getContext(), StringRef(), FindFn);
+  IRBuilder<> IB(EntryBlock);
+
+  std::vector<Value *> FindHelperArgs;
+  FindHelperArgs.push_back(UnwindMgrInstance);
+  for (auto &Arg : FindFn->args())
+    FindHelperArgs.push_back(&Arg);
+
+  IB.CreateRet(IB.CreateCall(FindHelperFn, FindHelperArgs));
+
+  return ThreadSafeModule(std::move(M), std::move(Ctx));
+}
+
+Error UnwindInfoRegistrationPlugin::addUnwindInfoRegistrationActions(
+    LinkGraph &G) {
+  ExecutorAddrRange EHFrameRange, UnwindInfoRange;
+
+  std::vector<Block *> CodeBlocks;
+
+  auto ScanUnwindInfoSection = [&](Section &Sec, ExecutorAddrRange &SecRange) {
+    if (Sec.empty())
+      return;
+
+    SecRange.Start = (*Sec.blocks().begin())->getAddress();
+    for (auto *B : Sec.blocks()) {
+      auto R = B->getRange();
+      SecRange.Start = std::min(SecRange.Start, R.Start);
+      SecRange.End = std::max(SecRange.End, R.End);
+      for (auto &E : B->edges()) {
+        if (E.getKind() != Edge::KeepAlive || !E.getTarget().isDefined())
+          continue;
+        auto &TargetBlock = E.getTarget().getBlock();
+        auto &TargetSection = TargetBlock.getSection();
+        if ((TargetSection.getMemProt() & MemProt::Exec) == MemProt::Exec)
+          CodeBlocks.push_back(&TargetBlock);
+      }
+    }
+  };
+
+  if (auto *EHFrame = G.findSectionByName(MachOEHFrameSectionName))
+    ScanUnwindInfoSection(*EHFrame, EHFrameRange);
+
+  if (auto *UnwindInfo = G.findSectionByName(MachOUnwindInfoSectionName))
+    ScanUnwindInfoSection(*UnwindInfo, UnwindInfoRange);
+
+  if (CodeBlocks.empty())
+    return Error::success();
+
+  if ((EHFrameRange == ExecutorAddrRange() &&
+       UnwindInfoRange == ExecutorAddrRange()))
+    return Error::success();
+
+  llvm::sort(CodeBlocks, [](const Block *LHS, const Block *RHS) {
+    return LHS->getAddress() < RHS->getAddress();
+  });
+
+  SmallVector<ExecutorAddrRange> CodeRanges;
+  for (auto *B : CodeBlocks) {
+    if (CodeRanges.empty() || CodeRanges.back().End != B->getAddress())
+      CodeRanges.push_back(B->getRange());
+    else
+      CodeRanges.back().End = B->getRange().End;
+  }
+
+  ExecutorAddr DSOBase;
+  if (auto *DSOBaseSym = G.findAbsoluteSymbolByName(DSOBaseName))
+    DSOBase = DSOBaseSym->getAddress();
+  else if (auto *DSOBaseSym = G.findExternalSymbolByName(DSOBaseName))
+    DSOBase = DSOBaseSym->getAddress();
+  else if (auto *DSOBaseSym = G.findDefinedSymbolByName(DSOBaseName))
+    DSOBase = DSOBaseSym->getAddress();
+  else
+    return make_error<StringError>("In " + G.getName() +
+                                       " could not find dso base symbol",
+                                   inconvertibleErrorCode());
+
+  using namespace shared;
+  using SPSRegisterArgs =
+      SPSArgList<SPSExecutorAddr, SPSSequence<SPSExecutorAddrRange>,
+                 SPSExecutorAddr, SPSExecutorAddrRange, SPSExecutorAddrRange>;
+  using SPSDeregisterArgs =
+      SPSArgList<SPSExecutorAddr, SPSSequence<SPSExecutorAddrRange>>;
+
+  G.allocActions().push_back(
+      {cantFail(WrapperFunctionCall::Create<SPSRegisterArgs>(
+           Register, Instance, CodeRanges, DSOBase, EHFrameRange,
+           UnwindInfoRange)),
+       cantFail(WrapperFunctionCall::Create<SPSDeregisterArgs>(
+           Deregister, Instance, CodeRanges))});
+
+  return Error::success();
+}
+
+} // namespace llvm::orc
diff --git a/llvm/lib/Frontend/Offloading/CMakeLists.txt b/llvm/lib/Frontend/Offloading/CMakeLists.txt
index ce445ad9cc4cb..8e1ede9c72b39 100644
--- a/llvm/lib/Frontend/Offloading/CMakeLists.txt
+++ b/llvm/lib/Frontend/Offloading/CMakeLists.txt
@@ -12,6 +12,7 @@ add_llvm_component_library(LLVMFrontendOffloading
   Core
   BinaryFormat
   Object
+  ObjectYAML
   Support
   TransformUtils
   TargetParser
diff --git a/llvm/lib/Frontend/Offloading/OffloadWrapper.cpp b/llvm/lib/Frontend/Offloading/OffloadWrapper.cpp
index 478cc8ab05d1a..b494c3748ac67 100644
--- a/llvm/lib/Frontend/Offloading/OffloadWrapper.cpp
+++ b/llvm/lib/Frontend/Offloading/OffloadWrapper.cpp
@@ -181,7 +181,7 @@ GlobalVariable *createBinDesc(Module &M, ArrayRef<ArrayRef<char>> Bufs,
       ConstantInt::get(Type::getInt32Ty(C), ImagesInits.size()), ImagesB,
       EntriesB, EntriesE);
 
-  return new GlobalVariable(M, DescInit->getType(), /*isConstant*/ true,
+  return new GlobalVariable(M, DescInit->getType(), /*isConstant=*/true,
                             GlobalValue::InternalLinkage, DescInit,
                             ".omp_offloading.descriptor" + Suffix);
 }
@@ -317,7 +317,10 @@ GlobalVariable *createFatbinDesc(Module &M, ArrayRef<char> Image, bool IsHIP,
 /// void __cudaRegisterTest(void **fatbinHandle) {
 ///   for (struct __tgt_offload_entry *entry = &__start_cuda_offloading_entries;
 ///        entry != &__stop_cuda_offloading_entries; ++entry) {
-///     if (!entry->size)
+///     if (entry->Kind != OFK_CUDA)
+///       continue
+///
+///     if (!entry->Size)
 ///       __cudaRegisterFunction(fatbinHandle, entry->addr, entry->name,
 ///                              entry->name, -1, 0, 0, 0, 0, 0);
 ///     else
@@ -391,6 +394,7 @@ Function *createRegisterGlobalsFunction(Module &M, bool IsHIP,
   // Create the loop to register all the entries.
   IRBuilder<> Builder(BasicBlock::Create(C, "entry", RegGlobalsFn));
   auto *EntryBB = BasicBlock::Create(C, "while.entry", RegGlobalsFn);
+  auto *IfKindBB = BasicBlock::Create(C, "if.kind", RegGlobalsFn);
   auto *IfThenBB = BasicBlock::Create(C, "if.then", RegGlobalsFn);
   auto *IfElseBB = BasicBlock::Create(C, "if.else", RegGlobalsFn);
   auto *SwGlobalBB = BasicBlock::Create(C, "sw.global", RegGlobalsFn);
@@ -406,30 +410,42 @@ Function *createRegisterGlobalsFunction(Module &M, bool IsHIP,
   auto *Entry = Builder.CreatePHI(PointerType::getUnqual(C), 2, "entry");
   auto *AddrPtr =
       Builder.CreateInBoundsGEP(offloading::getEntryTy(M), Entry,
-                                {ConstantInt::get(getSizeTTy(M), 0),
-                                 ConstantInt::get(Type::getInt32Ty(C), 0)});
+                                {ConstantInt::get(Type::getInt32Ty(C), 0),
+                                 ConstantInt::get(Type::getInt32Ty(C), 4)});
   auto *Addr = Builder.CreateLoad(Int8PtrTy, AddrPtr, "addr");
+  auto *AuxAddrPtr =
+      Builder.CreateInBoundsGEP(offloading::getEntryTy(M), Entry,
+                                {ConstantInt::get(Type::getInt32Ty(C), 0),
+                                 ConstantInt::get(Type::getInt32Ty(C), 8)});
+  auto *AuxAddr = Builder.CreateLoad(Int8PtrTy, AuxAddrPtr, "aux_addr");
+  auto *KindPtr =
+      Builder.CreateInBoundsGEP(offloading::getEntryTy(M), Entry,
+                                {ConstantInt::get(Type::getInt32Ty(C), 0),
+                                 ConstantInt::get(Type::getInt32Ty(C), 2)});
+  auto *Kind = Builder.CreateLoad(Type::getInt16Ty(C), KindPtr, "kind");
   auto *NamePtr =
       Builder.CreateInBoundsGEP(offloading::getEntryTy(M), Entry,
-                                {ConstantInt::get(getSizeTTy(M), 0),
-                                 ConstantInt::get(Type::getInt32Ty(C), 1)});
+                                {ConstantInt::get(Type::getInt32Ty(C), 0),
+                                 ConstantInt::get(Type::getInt32Ty(C), 5)});
   auto *Name = Builder.CreateLoad(Int8PtrTy, NamePtr, "name");
   auto *SizePtr =
       Builder.CreateInBoundsGEP(offloading::getEntryTy(M), Entry,
-                                {ConstantInt::get(getSizeTTy(M), 0),
-                                 ConstantInt::get(Type::getInt32Ty(C), 2)});
-  auto *Size = Builder.CreateLoad(getSizeTTy(M), SizePtr, "size");
+                                {ConstantInt::get(Type::getInt32Ty(C), 0),
+                                 ConstantInt::get(Type::getInt32Ty(C), 6)});
+  auto *Size = Builder.CreateLoad(Type::getInt64Ty(C), SizePtr, "size");
   auto *FlagsPtr =
       Builder.CreateInBoundsGEP(offloading::getEntryTy(M), Entry,
-                                {ConstantInt::get(getSizeTTy(M), 0),
+                                {ConstantInt::get(Type::getInt32Ty(C), 0),
                                  ConstantInt::get(Type::getInt32Ty(C), 3)});
   auto *Flags = Builder.CreateLoad(Type::getInt32Ty(C), FlagsPtr, "flags");
   auto *DataPtr =
       Builder.CreateInBoundsGEP(offloading::getEntryTy(M), Entry,
-                                {ConstantInt::get(getSizeTTy(M), 0),
-                                 ConstantInt::get(Type::getInt32Ty(C), 4)});
-  auto *Data = Builder.CreateLoad(Type::getInt32Ty(C), DataPtr, "textype");
-  auto *Kind = Builder.CreateAnd(
+                                {ConstantInt::get(Type::getInt32Ty(C), 0),
+                                 ConstantInt::get(Type::getInt32Ty(C), 7)});
+  auto *Data = Builder.CreateTrunc(
+      Builder.CreateLoad(Type::getInt64Ty(C), DataPtr, "data"),
+      Type::getInt32Ty(C));
+  auto *Type = Builder.CreateAnd(
       Flags, ConstantInt::get(Type::getInt32Ty(C), 0x7), "type");
 
   // Extract the flags stored in the bit-field and convert them to C booleans.
@@ -448,8 +464,14 @@ Function *createRegisterGlobalsFunction(Module &M, bool IsHIP,
                               llvm::offloading::OffloadGlobalNormalized));
   auto *Normalized = Builder.CreateLShr(
       NormalizedBit, ConstantInt::get(Type::getInt32Ty(C), 5), "normalized");
-  auto *FnCond =
-      Builder.CreateICmpEQ(Size, ConstantInt::getNullValue(getSizeTTy(M)));
+  auto *KindCond = Builder.CreateICmpEQ(
+      Kind, ConstantInt::get(Type::getInt16Ty(C),
+                             IsHIP ? object::OffloadKind::OFK_HIP
+                                   : object::OffloadKind::OFK_Cuda));
+  Builder.CreateCondBr(KindCond, IfKindBB, IfEndBB);
+  Builder.SetInsertPoint(IfKindBB);
+  auto *FnCond = Builder.CreateICmpEQ(
+      Size, ConstantInt::getNullValue(Type::getInt64Ty(C)));
   Builder.CreateCondBr(FnCond, IfThenBB, IfElseBB);
 
   // Create kernel registration code.
@@ -464,7 +486,7 @@ Function *createRegisterGlobalsFunction(Module &M, bool IsHIP,
   Builder.CreateBr(IfEndBB);
   Builder.SetInsertPoint(IfElseBB);
 
-  auto *Switch = Builder.CreateSwitch(Kind, IfEndBB);
+  auto *Switch = Builder.CreateSwitch(Type, IfEndBB);
   // Create global variable registration code.
   Builder.SetInsertPoint(SwGlobalBB);
   Builder.CreateCall(RegVar,
@@ -476,12 +498,8 @@ Function *createRegisterGlobalsFunction(Module &M, bool IsHIP,
 
   // Create managed variable registration code.
   Builder.SetInsertPoint(SwManagedBB);
-  auto *ManagedVar = Builder.CreateLoad(Int8PtrTy, Addr, "managed.addr");
-  auto *ManagedAddr = Builder.CreateInBoundsGEP(
-      Int8PtrTy, Addr, {ConstantInt::get(Builder.getInt64Ty(), 1)});
-  auto *Managed = Builder.CreateLoad(Int8PtrTy, ManagedAddr, "managed.addr");
-  Builder.CreateCall(RegManagedVar, {RegGlobalsFn->arg_begin(), ManagedVar,
-                                     Managed, Name, Size, Data});
+  Builder.CreateCall(RegManagedVar, {RegGlobalsFn->arg_begin(), AuxAddr, Addr,
+                                     Name, Size, Data});
   Builder.CreateBr(IfEndBB);
   Switch->addCase(Builder.getInt32(llvm::offloading::OffloadGlobalManagedEntry),
                   SwManagedBB);
diff --git a/llvm/lib/Frontend/Offloading/Utility.cpp b/llvm/lib/Frontend/Offloading/Utility.cpp
index 26dab0b22fa12..5d34bbffe4a75 100644
--- a/llvm/lib/Frontend/Offloading/Utility.cpp
+++ b/llvm/lib/Frontend/Offloading/Utility.cpp
@@ -15,6 +15,8 @@
 #include "llvm/IR/GlobalVariable.h"
 #include "llvm/IR/Value.h"
 #include "llvm/Object/ELFObjectFile.h"
+#include "llvm/ObjectYAML/ELFYAML.h"
+#include "llvm/ObjectYAML/yaml2obj.h"
 #include "llvm/Support/MemoryBufferRef.h"
 #include "llvm/Transforms/Utils/ModuleUtils.h"
 
@@ -27,31 +29,23 @@ StructType *offloading::getEntryTy(Module &M) {
       StructType::getTypeByName(C, "struct.__tgt_offload_entry");
   if (!EntryTy)
     EntryTy = StructType::create(
-        "struct.__tgt_offload_entry", PointerType::getUnqual(C),
-        PointerType::getUnqual(C), M.getDataLayout().getIntPtrType(C),
-        Type::getInt32Ty(C), Type::getInt32Ty(C));
+        "struct.__tgt_offload_entry", Type::getInt64Ty(C), Type::getInt16Ty(C),
+        Type::getInt16Ty(C), Type::getInt32Ty(C), PointerType::getUnqual(C),
+        PointerType::getUnqual(C), Type::getInt64Ty(C), Type::getInt64Ty(C),
+        PointerType::getUnqual(C));
   return EntryTy;
 }
 
-StructType *offloading::getManagedTy(Module &M) {
-  LLVMContext &C = M.getContext();
-  StructType *StructTy = StructType::getTypeByName(C, "struct.__managed_var");
-  if (!StructTy)
-    StructTy = llvm::StructType::create("struct.__managed_var",
-                                        PointerType::getUnqual(M.getContext()),
-                                        PointerType::getUnqual(M.getContext()));
-  return StructTy;
-}
-
-// TODO: Rework this interface to be more generic.
 std::pair<Constant *, GlobalVariable *>
-offloading::getOffloadingEntryInitializer(Module &M, Constant *Addr,
-                                          StringRef Name, uint64_t Size,
-                                          int32_t Flags, int32_t Data) {
+offloading::getOffloadingEntryInitializer(Module &M, object::OffloadKind Kind,
+                                          Constant *Addr, StringRef Name,
+                                          uint64_t Size, uint32_t Flags,
+                                          uint64_t Data, Constant *AuxAddr) {
   llvm::Triple Triple(M.getTargetTriple());
-  Type *Int8PtrTy = PointerType::getUnqual(M.getContext());
+  Type *PtrTy = PointerType::getUnqual(M.getContext());
+  Type *Int64Ty = Type::getInt64Ty(M.getContext());
   Type *Int32Ty = Type::getInt32Ty(M.getContext());
-  Type *SizeTy = M.getDataLayout().getIntPtrType(M.getContext());
+  Type *Int16Ty = Type::getInt16Ty(M.getContext());
 
   Constant *AddrName = ConstantDataArray::getString(M.getContext(), Name);
 
@@ -74,23 +68,29 @@ offloading::getOffloadingEntryInitializer(Module &M, Constant *Addr,
 
   // Construct the offloading entry.
   Constant *EntryData[] = {
-      ConstantExpr::getPointerBitCastOrAddrSpaceCast(Addr, Int8PtrTy),
-      ConstantExpr::getPointerBitCastOrAddrSpaceCast(Str, Int8PtrTy),
-      ConstantInt::get(SizeTy, Size),
+      ConstantExpr::getNullValue(Int64Ty),
+      ConstantInt::get(Int16Ty, 1),
+      ConstantInt::get(Int16Ty, Kind),
       ConstantInt::get(Int32Ty, Flags),
-      ConstantInt::get(Int32Ty, Data),
-  };
+      ConstantExpr::getPointerBitCastOrAddrSpaceCast(Addr, PtrTy),
+      ConstantExpr::getPointerBitCastOrAddrSpaceCast(Str, PtrTy),
+      ConstantInt::get(Int64Ty, Size),
+      ConstantInt::get(Int64Ty, Data),
+      AuxAddr ? ConstantExpr::getPointerBitCastOrAddrSpaceCast(AuxAddr, PtrTy)
+              : ConstantExpr::getNullValue(PtrTy)};
   Constant *EntryInitializer = ConstantStruct::get(getEntryTy(M), EntryData);
   return {EntryInitializer, Str};
 }
 
-void offloading::emitOffloadingEntry(Module &M, Constant *Addr, StringRef Name,
-                                     uint64_t Size, int32_t Flags, int32_t Data,
+void offloading::emitOffloadingEntry(Module &M, object::OffloadKind Kind,
+                                     Constant *Addr, StringRef Name,
+                                     uint64_t Size, uint32_t Flags,
+                                     uint64_t Data, Constant *AuxAddr,
                                      StringRef SectionName) {
   llvm::Triple Triple(M.getTargetTriple());
 
-  auto [EntryInitializer, NameGV] =
-      getOffloadingEntryInitializer(M, Addr, Name, Size, Flags, Data);
+  auto [EntryInitializer, NameGV] = getOffloadingEntryInitializer(
+      M, Kind, Addr, Name, Size, Flags, Data, AuxAddr);
 
   StringRef Prefix =
       Triple.isNVPTX() ? "$offloading$entry$" : ".offloading.entry.";
@@ -105,7 +105,7 @@ void offloading::emitOffloadingEntry(Module &M, Constant *Addr, StringRef Name,
     Entry->setSection((SectionName + "$OE").str());
   else
     Entry->setSection(SectionName);
-  Entry->setAlignment(Align(1));
+  Entry->setAlignment(Align(object::OffloadBinary::getAlignment()));
 }
 
 std::pair<GlobalVariable *, GlobalVariable *>
@@ -137,6 +137,7 @@ offloading::getOffloadEntryArray(Module &M, StringRef SectionName) {
         M, ZeroInitilaizer->getType(), true, GlobalVariable::InternalLinkage,
         ZeroInitilaizer, "__dummy." + SectionName);
     DummyEntry->setSection(SectionName);
+    DummyEntry->setAlignment(Align(object::OffloadBinary::getAlignment()));
     appendToCompilerUsed(M, DummyEntry);
   } else {
     // The COFF linker will merge sections containing a '$' together into a
@@ -375,3 +376,86 @@ Error llvm::offloading::amdgpu::getAMDGPUMetaDataFromImage(
   }
   return Error::success();
 }
+Error offloading::intel::containerizeOpenMPSPIRVImage(
+    std::unique_ptr<MemoryBuffer> &Img) {
+  constexpr char INTEL_ONEOMP_OFFLOAD_VERSION[] = "1.0";
+  constexpr int NT_INTEL_ONEOMP_OFFLOAD_VERSION = 1;
+  constexpr int NT_INTEL_ONEOMP_OFFLOAD_IMAGE_COUNT = 2;
+  constexpr int NT_INTEL_ONEOMP_OFFLOAD_IMAGE_AUX = 3;
+
+  // Start creating notes for the ELF container.
+  std::vector<ELFYAML::NoteEntry> Notes;
+  std::string Version = toHex(INTEL_ONEOMP_OFFLOAD_VERSION);
+  Notes.emplace_back(ELFYAML::NoteEntry{"INTELONEOMPOFFLOAD",
+                                        yaml::BinaryRef(Version),
+                                        NT_INTEL_ONEOMP_OFFLOAD_VERSION});
+
+  // The AuxInfo string will hold auxiliary information for the image.
+  // ELFYAML::NoteEntry structures will hold references to the
+  // string, so we have to make sure the string is valid.
+  std::string AuxInfo;
+
+  // TODO: Pass compile/link opts
+  StringRef CompileOpts = "";
+  StringRef LinkOpts = "";
+
+  unsigned ImageFmt = 1; // SPIR-V format
+
+  AuxInfo = toHex((Twine(0) + Twine('\0') + Twine(ImageFmt) + Twine('\0') +
+                   CompileOpts + Twine('\0') + LinkOpts)
+                      .str());
+  Notes.emplace_back(ELFYAML::NoteEntry{"INTELONEOMPOFFLOAD",
+                                        yaml::BinaryRef(AuxInfo),
+                                        NT_INTEL_ONEOMP_OFFLOAD_IMAGE_AUX});
+
+  std::string ImgCount = toHex(Twine(1).str()); // always one image per ELF
+  Notes.emplace_back(ELFYAML::NoteEntry{"INTELONEOMPOFFLOAD",
+                                        yaml::BinaryRef(ImgCount),
+                                        NT_INTEL_ONEOMP_OFFLOAD_IMAGE_COUNT});
+
+  std::string YamlFile;
+  llvm::raw_string_ostream YamlFileStream(YamlFile);
+
+  // Write the YAML template file.
+
+  // We use 64-bit little-endian ELF currently.
+  ELFYAML::FileHeader Header{};
+  Header.Class = ELF::ELFCLASS64;
+  Header.Data = ELF::ELFDATA2LSB;
+  Header.Type = ELF::ET_DYN;
+  // Use an existing Intel machine type as there is not one specifically for
+  // Intel GPUs.
+  Header.Machine = ELF::EM_IA_64;
+
+  // Create a section with notes.
+  ELFYAML::NoteSection Section{};
+  Section.Type = ELF::SHT_NOTE;
+  Section.AddressAlign = 0;
+  Section.Name = ".note.inteloneompoffload";
+  Section.Notes.emplace(std::move(Notes));
+
+  ELFYAML::Object Object{};
+  Object.Header = Header;
+  Object.Chunks.push_back(
+      std::make_unique<ELFYAML::NoteSection>(std::move(Section)));
+
+  // Create the section that will hold the image
+  ELFYAML::RawContentSection ImageSection{};
+  ImageSection.Type = ELF::SHT_PROGBITS;
+  ImageSection.AddressAlign = 0;
+  std::string Name = "__openmp_offload_spirv_0";
+  ImageSection.Name = Name;
+  ImageSection.Content =
+      llvm::yaml::BinaryRef(arrayRefFromStringRef(Img->getBuffer()));
+  Object.Chunks.push_back(
+      std::make_unique<ELFYAML::RawContentSection>(std::move(ImageSection)));
+  Error Err = Error::success();
+  llvm::yaml::yaml2elf(
+      Object, YamlFileStream,
+      [&Err](const Twine &Msg) { Err = createStringError(Msg); }, UINT64_MAX);
+  if (Err)
+    return Err;
+
+  Img = MemoryBuffer::getMemBufferCopy(YamlFile);
+  return Error::success();
+}
diff --git a/llvm/lib/Frontend/OpenMP/OMPContext.cpp b/llvm/lib/Frontend/OpenMP/OMPContext.cpp
index c26f5b49b6008..5e13da172d677 100644
--- a/llvm/lib/Frontend/OpenMP/OMPContext.cpp
+++ b/llvm/lib/Frontend/OpenMP/OMPContext.cpp
@@ -24,44 +24,91 @@
 using namespace llvm;
 using namespace omp;
 
-OMPContext::OMPContext(bool IsDeviceCompilation, Triple TargetTriple) {
-  // Add the appropriate device kind trait based on the triple and the
-  // IsDeviceCompilation flag.
-  ActiveTraits.set(unsigned(IsDeviceCompilation
-                                ? TraitProperty::device_kind_nohost
-                                : TraitProperty::device_kind_host));
-  switch (TargetTriple.getArch()) {
-  case Triple::arm:
-  case Triple::armeb:
-  case Triple::aarch64:
-  case Triple::aarch64_be:
-  case Triple::aarch64_32:
-  case Triple::loongarch64:
-  case Triple::mips:
-  case Triple::mipsel:
-  case Triple::mips64:
-  case Triple::mips64el:
-  case Triple::ppc:
-  case Triple::ppcle:
-  case Triple::ppc64:
-  case Triple::ppc64le:
-  case Triple::systemz:
-  case Triple::x86:
-  case Triple::x86_64:
-    ActiveTraits.set(unsigned(TraitProperty::device_kind_cpu));
-    break;
-  case Triple::amdgcn:
-  case Triple::nvptx:
-  case Triple::nvptx64:
-    ActiveTraits.set(unsigned(TraitProperty::device_kind_gpu));
-    break;
-  default:
-    break;
+OMPContext::OMPContext(bool IsDeviceCompilation, Triple TargetTriple,
+                       Triple TargetOffloadTriple, int DeviceNum) {
+  // Add the appropriate target device kind trait based on the target triple
+  if (!TargetOffloadTriple.getTriple().empty() && DeviceNum > -1) {
+    // If target triple is present, then target device is not a host
+    ActiveTraits.set(unsigned(TraitProperty::target_device_kind_nohost));
+    switch (TargetOffloadTriple.getArch()) {
+    case Triple::arm:
+    case Triple::armeb:
+    case Triple::aarch64:
+    case Triple::aarch64_be:
+    case Triple::aarch64_32:
+    case Triple::mips:
+    case Triple::mipsel:
+    case Triple::mips64:
+    case Triple::mips64el:
+    case Triple::ppc:
+    case Triple::ppcle:
+    case Triple::ppc64:
+    case Triple::ppc64le:
+    case Triple::systemz:
+    case Triple::x86:
+    case Triple::x86_64:
+      ActiveTraits.set(unsigned(TraitProperty::target_device_kind_cpu));
+      break;
+    case Triple::amdgcn:
+    case Triple::nvptx:
+    case Triple::nvptx64:
+      ActiveTraits.set(unsigned(TraitProperty::target_device_kind_gpu));
+      break;
+    default:
+      break;
+    }
+    // Add the appropriate device architecture trait based on the triple.
+#define OMP_TRAIT_PROPERTY(Enum, TraitSetEnum, TraitSelectorEnum, Str)         \
+  if (TraitSelector::TraitSelectorEnum == TraitSelector::target_device_arch) { \
+    if (TargetOffloadTriple.getArch() ==                                       \
+        TargetOffloadTriple.getArchTypeForLLVMName(Str))                       \
+      ActiveTraits.set(unsigned(TraitProperty::Enum));                         \
+    if (StringRef(Str) == "x86_64" &&                                          \
+        TargetOffloadTriple.getArch() == Triple::x86_64)                       \
+      ActiveTraits.set(unsigned(TraitProperty::Enum));                         \
   }
+#include "llvm/Frontend/OpenMP/OMPKinds.def"
+  } else {
+    // Add the appropriate device kind trait based on the triple and the
+    // IsDeviceCompilation flag.
+    ActiveTraits.set(unsigned(IsDeviceCompilation
+                                  ? TraitProperty::device_kind_nohost
+                                  : TraitProperty::device_kind_host));
+    ActiveTraits.set(unsigned(TraitProperty::target_device_kind_host));
+    switch (TargetTriple.getArch()) {
+    case Triple::arm:
+    case Triple::armeb:
+    case Triple::aarch64:
+    case Triple::aarch64_be:
+    case Triple::aarch64_32:
+    case Triple::mips:
+    case Triple::mipsel:
+    case Triple::mips64:
+    case Triple::mips64el:
+    case Triple::ppc:
+    case Triple::ppcle:
+    case Triple::ppc64:
+    case Triple::ppc64le:
+    case Triple::systemz:
+    case Triple::x86:
+    case Triple::x86_64:
+      ActiveTraits.set(unsigned(TraitProperty::device_kind_cpu));
+      ActiveTraits.set(unsigned(TraitProperty::target_device_kind_cpu));
+      break;
+    case Triple::amdgcn:
+    case Triple::nvptx:
+    case Triple::nvptx64:
+      ActiveTraits.set(unsigned(TraitProperty::device_kind_gpu));
+      ActiveTraits.set(unsigned(TraitProperty::target_device_kind_gpu));
+      break;
+    default:
+      break;
+    }
 
-  // Add the appropriate device architecture trait based on the triple.
+    // Add the appropriate device architecture trait based on the triple.
 #define OMP_TRAIT_PROPERTY(Enum, TraitSetEnum, TraitSelectorEnum, Str)         \
-  if (TraitSelector::TraitSelectorEnum == TraitSelector::device_arch) {        \
+  if (TraitSelector::TraitSelectorEnum == TraitSelector::device_arch ||        \
+      TraitSelector::TraitSelectorEnum == TraitSelector::target_device_arch) { \
     if (TargetTriple.getArch() == TargetTriple.getArchTypeForLLVMName(Str))    \
       ActiveTraits.set(unsigned(TraitProperty::Enum));                         \
     if (StringRef(Str) == "x86_64" &&                                          \
@@ -70,29 +117,30 @@ OMPContext::OMPContext(bool IsDeviceCompilation, Triple TargetTriple) {
   }
 #include "llvm/Frontend/OpenMP/OMPKinds.def"
 
-  // TODO: What exactly do we want to see as device ISA trait?
-  //       The discussion on the list did not seem to have come to an agreed
-  //       upon solution.
+    // TODO: What exactly do we want to see as device ISA trait?
+    //       The discussion on the list did not seem to have come to an agreed
+    //       upon solution.
 
-  // LLVM is the "OpenMP vendor" but we could also interpret vendor as the
-  // target vendor.
-  ActiveTraits.set(unsigned(TraitProperty::implementation_vendor_llvm));
+    // LLVM is the "OpenMP vendor" but we could also interpret vendor as the
+    // target vendor.
+    ActiveTraits.set(unsigned(TraitProperty::implementation_vendor_llvm));
 
-  // The user condition true is accepted but not false.
-  ActiveTraits.set(unsigned(TraitProperty::user_condition_true));
+    // The user condition true is accepted but not false.
+    ActiveTraits.set(unsigned(TraitProperty::user_condition_true));
 
-  // This is for sure some device.
-  ActiveTraits.set(unsigned(TraitProperty::device_kind_any));
+    // This is for sure some device.
+    ActiveTraits.set(unsigned(TraitProperty::device_kind_any));
 
-  LLVM_DEBUG({
-    dbgs() << "[" << DEBUG_TYPE
-           << "] New OpenMP context with the following properties:\n";
-    for (unsigned Bit : ActiveTraits.set_bits()) {
-      TraitProperty Property = TraitProperty(Bit);
-      dbgs() << "\t " << getOpenMPContextTraitPropertyFullName(Property)
-             << "\n";
-    }
-  });
+    LLVM_DEBUG({
+      dbgs() << "[" << DEBUG_TYPE
+             << "] New OpenMP context with the following properties:\n";
+      for (unsigned Bit : ActiveTraits.set_bits()) {
+        TraitProperty Property = TraitProperty(Bit);
+        dbgs() << "\t " << getOpenMPContextTraitPropertyFullName(Property)
+               << "\n";
+      }
+    });
+  }
 }
 
 /// Return true if \p C0 is a subset of \p C1. Note that both arrays are
@@ -213,6 +261,10 @@ static int isVariantApplicableInContextHelper(
       IsActiveTrait = llvm::all_of(VMI.ISATraits, [&](StringRef RawString) {
         return Ctx.matchesISATrait(RawString);
       });
+    if (Property == TraitProperty::target_device_isa___ANY)
+      IsActiveTrait = llvm::all_of(VMI.ISATraits, [&](StringRef RawString) {
+        return Ctx.matchesISATrait(RawString);
+      });
 
     if (std::optional<bool> Result = HandleTrait(Property, IsActiveTrait))
       return *Result;
@@ -298,6 +350,9 @@ static APInt getVariantMatchScore(const VariantMatchInfo &VMI,
     case TraitSet::device:
       // Handled separately below.
       break;
+    case TraitSet::target_device:
+      // TODO: Handling separately.
+      break;
     case TraitSet::invalid:
       llvm_unreachable("Unknown trait set is not to be used!");
     }
@@ -305,6 +360,8 @@ static APInt getVariantMatchScore(const VariantMatchInfo &VMI,
     // device={kind(any)} is "as if" no kind selector was specified.
     if (Property == TraitProperty::device_kind_any)
       continue;
+    if (Property == TraitProperty::target_device_kind_any)
+      continue;
 
     switch (getOpenMPContextTraitSelectorForProperty(Property)) {
     case TraitSelector::device_kind:
@@ -316,6 +373,15 @@ static APInt getVariantMatchScore(const VariantMatchInfo &VMI,
     case TraitSelector::device_isa:
       Score += (1ULL << (NoConstructTraits + 2));
       continue;
+    case TraitSelector::target_device_kind:
+      Score += (1ULL << (NoConstructTraits + 0));
+      continue;
+    case TraitSelector::target_device_arch:
+      Score += (1ULL << (NoConstructTraits + 1));
+      continue;
+    case TraitSelector::target_device_isa:
+      Score += (1ULL << (NoConstructTraits + 2));
+      continue;
     default:
       continue;
     }
@@ -412,7 +478,14 @@ StringRef llvm::omp::getOpenMPContextTraitSetName(TraitSet Kind) {
   llvm_unreachable("Unknown trait set!");
 }
 
-TraitSelector llvm::omp::getOpenMPContextTraitSelectorKind(StringRef S) {
+TraitSelector llvm::omp::getOpenMPContextTraitSelectorKind(StringRef S,
+                                                           TraitSet Set) {
+  if (Set == TraitSet::target_device && S == "kind")
+    return TraitSelector::target_device_kind;
+  if (Set == TraitSet::target_device && S == "arch")
+    return TraitSelector::target_device_arch;
+  if (Set == TraitSet::target_device && S == "isa")
+    return TraitSelector::target_device_isa;
   return StringSwitch<TraitSelector>(S)
 #define OMP_TRAIT_SELECTOR(Enum, TraitSetEnum, Str, ReqProp)                   \
   .Case(Str, TraitSelector::Enum)
@@ -445,6 +518,9 @@ TraitProperty llvm::omp::getOpenMPContextTraitPropertyKind(
   // up to the target to decide if the feature is available.
   if (Set == TraitSet::device && Selector == TraitSelector::device_isa)
     return TraitProperty::device_isa___ANY;
+  if (Set == TraitSet::target_device &&
+      Selector == TraitSelector::target_device_isa)
+    return TraitProperty::target_device_isa___ANY;
 #define OMP_TRAIT_PROPERTY(Enum, TraitSetEnum, TraitSelectorEnum, Str)         \
   if (Set == TraitSet::TraitSetEnum && Str == S)                               \
     return TraitProperty::Enum;
@@ -466,6 +542,8 @@ StringRef llvm::omp::getOpenMPContextTraitPropertyName(TraitProperty Kind,
                                                        StringRef RawString) {
   if (Kind == TraitProperty::device_isa___ANY)
     return RawString;
+  if (Kind == TraitProperty::target_device_isa___ANY)
+    return RawString;
   switch (Kind) {
 #define OMP_TRAIT_PROPERTY(Enum, TraitSetEnum, TraitSelectorEnum, Str)         \
   case TraitProperty::Enum:                                                    \
@@ -488,7 +566,8 @@ bool llvm::omp::isValidTraitSelectorForTraitSet(TraitSelector Selector,
                                                 TraitSet Set,
                                                 bool &AllowsTraitScore,
                                                 bool &RequiresProperty) {
-  AllowsTraitScore = Set != TraitSet::construct && Set != TraitSet::device;
+  AllowsTraitScore = Set != TraitSet::construct && Set != TraitSet::device &&
+                     Set != TraitSet::target_device;
   switch (Selector) {
 #define OMP_TRAIT_SELECTOR(Enum, TraitSetEnum, Str, ReqProp)                   \
   case TraitSelector::Enum:                                                    \
diff --git a/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp b/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
index 777391327f77c..ea8fab94a2256 100644
--- a/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
+++ b/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
@@ -38,6 +38,8 @@
 #include "llvm/IR/Function.h"
 #include "llvm/IR/GlobalVariable.h"
 #include "llvm/IR/IRBuilder.h"
+#include "llvm/IR/InstIterator.h"
+#include "llvm/IR/IntrinsicInst.h"
 #include "llvm/IR/LLVMContext.h"
 #include "llvm/IR/MDBuilder.h"
 #include "llvm/IR/Metadata.h"
@@ -159,6 +161,8 @@ static const omp::GV &getGridValue(const Triple &T, Function *Kernel) {
   }
   if (T.isNVPTX())
     return omp::NVPTXGridValues;
+  if (T.isSPIRV())
+    return omp::SPIRVGridValues;
   llvm_unreachable("No grid value available for this architecture!");
 }
 
@@ -313,7 +317,7 @@ static void redirectTo(BasicBlock *Source, BasicBlock *Target, DebugLoc DL) {
 }
 
 void llvm::spliceBB(IRBuilderBase::InsertPoint IP, BasicBlock *New,
-                    bool CreateBranch) {
+                    bool CreateBranch, DebugLoc DL) {
   assert(New->getFirstInsertionPt() == New->begin() &&
          "Target BB must not have PHI nodes");
 
@@ -321,15 +325,17 @@ void llvm::spliceBB(IRBuilderBase::InsertPoint IP, BasicBlock *New,
   BasicBlock *Old = IP.getBlock();
   New->splice(New->begin(), Old, IP.getPoint(), Old->end());
 
-  if (CreateBranch)
-    BranchInst::Create(New, Old);
+  if (CreateBranch) {
+    auto *NewBr = BranchInst::Create(New, Old);
+    NewBr->setDebugLoc(DL);
+  }
 }
 
 void llvm::spliceBB(IRBuilder<> &Builder, BasicBlock *New, bool CreateBranch) {
   DebugLoc DebugLoc = Builder.getCurrentDebugLocation();
   BasicBlock *Old = Builder.GetInsertBlock();
 
-  spliceBB(Builder.saveIP(), New, CreateBranch);
+  spliceBB(Builder.saveIP(), New, CreateBranch, DebugLoc);
   if (CreateBranch)
     Builder.SetInsertPoint(Old->getTerminator());
   else
@@ -341,12 +347,12 @@ void llvm::spliceBB(IRBuilder<> &Builder, BasicBlock *New, bool CreateBranch) {
 }
 
 BasicBlock *llvm::splitBB(IRBuilderBase::InsertPoint IP, bool CreateBranch,
-                          llvm::Twine Name) {
+                          DebugLoc DL, llvm::Twine Name) {
   BasicBlock *Old = IP.getBlock();
   BasicBlock *New = BasicBlock::Create(
       Old->getContext(), Name.isTriviallyEmpty() ? Old->getName() : Name,
       Old->getParent(), Old->getNextNode());
-  spliceBB(IP, New, CreateBranch);
+  spliceBB(IP, New, CreateBranch, DL);
   New->replaceSuccessorsPhiUsesWith(Old, New);
   return New;
 }
@@ -354,7 +360,7 @@ BasicBlock *llvm::splitBB(IRBuilderBase::InsertPoint IP, bool CreateBranch,
 BasicBlock *llvm::splitBB(IRBuilderBase &Builder, bool CreateBranch,
                           llvm::Twine Name) {
   DebugLoc DebugLoc = Builder.getCurrentDebugLocation();
-  BasicBlock *New = splitBB(Builder.saveIP(), CreateBranch, Name);
+  BasicBlock *New = splitBB(Builder.saveIP(), CreateBranch, DebugLoc, Name);
   if (CreateBranch)
     Builder.SetInsertPoint(Builder.GetInsertBlock()->getTerminator());
   else
@@ -368,7 +374,7 @@ BasicBlock *llvm::splitBB(IRBuilderBase &Builder, bool CreateBranch,
 BasicBlock *llvm::splitBB(IRBuilder<> &Builder, bool CreateBranch,
                           llvm::Twine Name) {
   DebugLoc DebugLoc = Builder.getCurrentDebugLocation();
-  BasicBlock *New = splitBB(Builder.saveIP(), CreateBranch, Name);
+  BasicBlock *New = splitBB(Builder.saveIP(), CreateBranch, DebugLoc, Name);
   if (CreateBranch)
     Builder.SetInsertPoint(Builder.GetInsertBlock()->getTerminator());
   else
@@ -678,7 +684,7 @@ void OpenMPIRBuilder::initialize() { initializeTypes(M); }
 static void raiseUserConstantDataAllocasToEntryBlock(IRBuilderBase &Builder,
                                                      Function *Function) {
   BasicBlock &EntryBlock = Function->getEntryBlock();
-  Instruction *MoveLocInst = EntryBlock.getFirstNonPHI();
+  BasicBlock::iterator MoveLocInst = EntryBlock.getFirstNonPHIIt();
 
   // Loop over blocks looking for constant allocas, skipping the entry block
   // as any allocas there are already in the desired location.
@@ -5303,7 +5309,7 @@ void OpenMPIRBuilder::createIfVersion(CanonicalLoopInfo *CanonicalLoop,
       Builder.CreateCondBr(IfCond, ThenBlock, /*ifFalse*/ ElseBlock);
   InsertPointTy IP{BrInstr->getParent(), ++BrInstr->getIterator()};
   // Then block contains branch to omp loop which needs to be vectorized
-  spliceBB(IP, ThenBlock, false);
+  spliceBB(IP, ThenBlock, false, Builder.getCurrentDebugLocation());
   ThenBlock->replaceSuccessorsPhiUsesWith(Head, ThenBlock);
 
   Builder.SetInsertPoint(ElseBlock);
@@ -6468,6 +6474,10 @@ void OpenMPIRBuilder::setOutlinedTargetRegionFunctionAttributes(
     OutlinedFn->setVisibility(GlobalValue::ProtectedVisibility);
     if (T.isAMDGCN())
       OutlinedFn->setCallingConv(CallingConv::AMDGPU_KERNEL);
+    else if (T.isNVPTX())
+      OutlinedFn->setCallingConv(CallingConv::PTX_Kernel);
+    else if (T.isSPIRV())
+      OutlinedFn->setCallingConv(CallingConv::SPIR_KERNEL);
   }
 }
 
@@ -6802,6 +6812,72 @@ FunctionCallee OpenMPIRBuilder::createDispatchDeinitFunction() {
   return getOrCreateRuntimeFunction(M, omp::OMPRTL___kmpc_dispatch_deinit);
 }
 
+static void FixupDebugInfoForOutlinedFunction(
+    OpenMPIRBuilder &OMPBuilder, IRBuilderBase &Builder, Function *Func,
+    DenseMap<Value *, std::tuple<Value *, unsigned>> &ValueReplacementMap) {
+
+  DISubprogram *NewSP = Func->getSubprogram();
+  if (!NewSP)
+    return;
+
+  DenseMap<const MDNode *, MDNode *> Cache;
+  SmallDenseMap<DILocalVariable *, DILocalVariable *> RemappedVariables;
+
+  auto GetUpdatedDIVariable = [&](DILocalVariable *OldVar, unsigned arg) {
+    auto NewSP = Func->getSubprogram();
+    DILocalVariable *&NewVar = RemappedVariables[OldVar];
+    // Only use cached variable if the arg number matches. This is important
+    // so that DIVariable created for privatized variables are not discarded.
+    if (NewVar && (arg == NewVar->getArg()))
+      return NewVar;
+
+    DILocalScope *NewScope = DILocalScope::cloneScopeForSubprogram(
+        *OldVar->getScope(), *NewSP, Builder.getContext(), Cache);
+    NewVar = llvm::DILocalVariable::get(
+        Builder.getContext(), NewScope, OldVar->getName(), OldVar->getFile(),
+        OldVar->getLine(), OldVar->getType(), arg, OldVar->getFlags(),
+        OldVar->getAlignInBits(), OldVar->getAnnotations());
+    return NewVar;
+  };
+
+  auto UpdateDebugRecord = [&](auto *DR) {
+    DILocalVariable *OldVar = DR->getVariable();
+    unsigned ArgNo = 0;
+    for (auto Loc : DR->location_ops()) {
+      auto Iter = ValueReplacementMap.find(Loc);
+      if (Iter != ValueReplacementMap.end()) {
+        DR->replaceVariableLocationOp(Loc, std::get<0>(Iter->second));
+        ArgNo = std::get<1>(Iter->second) + 1;
+      }
+    }
+    DR->setVariable(GetUpdatedDIVariable(OldVar, ArgNo));
+  };
+
+  // The location and scope of variable intrinsics and records still point to
+  // the parent function of the target region. Update them.
+  for (Instruction &I : instructions(Func)) {
+    if (auto *DDI = dyn_cast<llvm::DbgVariableIntrinsic>(&I))
+      UpdateDebugRecord(DDI);
+
+    for (DbgVariableRecord &DVR : filterDbgVars(I.getDbgRecordRange()))
+      UpdateDebugRecord(&DVR);
+  }
+  // An extra argument is passed to the device. Create the debug data for it.
+  if (OMPBuilder.Config.isTargetDevice()) {
+    DICompileUnit *CU = NewSP->getUnit();
+    Module *M = Func->getParent();
+    DIBuilder DB(*M, true, CU);
+    DIType *VoidPtrTy =
+        DB.createQualifiedType(dwarf::DW_TAG_pointer_type, nullptr);
+    DILocalVariable *Var = DB.createParameterVariable(
+        NewSP, "dyn_ptr", /*ArgNo*/ 1, NewSP->getFile(), /*LineNo=*/0,
+        VoidPtrTy, /*AlwaysPreserve=*/false, DINode::DIFlags::FlagArtificial);
+    auto Loc = DILocation::get(Func->getContext(), 0, 0, NewSP, 0);
+    DB.insertDeclare(&(*Func->arg_begin()), Var, DB.createExpression(), Loc,
+                     &(*Func->begin()));
+  }
+}
+
 static Expected<Function *> createOutlinedFunction(
     OpenMPIRBuilder &OMPBuilder, IRBuilderBase &Builder,
     const OpenMPIRBuilder::TargetKernelDefaultAttrs &DefaultAttrs,
@@ -6916,7 +6992,7 @@ static Expected<Function *> createOutlinedFunction(
   Builder.CreateRetVoid();
 
   // New Alloca IP at entry point of created device function.
-  Builder.SetInsertPoint(EntryBB->getFirstNonPHI());
+  Builder.SetInsertPoint(EntryBB->getFirstNonPHIIt());
   auto AllocaIP = Builder.saveIP();
 
   Builder.SetInsertPoint(UserCodeEntryBB->getFirstNonPHIOrDbg());
@@ -6927,6 +7003,8 @@ static Expected<Function *> createOutlinedFunction(
           ? make_range(Func->arg_begin() + 1, Func->arg_end())
           : Func->args();
 
+  DenseMap<Value *, std::tuple<Value *, unsigned>> ValueReplacementMap;
+
   auto ReplaceValue = [](Value *Input, Value *InputCopy, Function *Func) {
     // Things like GEP's can come in the form of Constants. Constants and
     // ConstantExpr's do not have access to the knowledge of what they're
@@ -6968,6 +7046,7 @@ static Expected<Function *> createOutlinedFunction(
     if (!AfterIP)
       return AfterIP.takeError();
     Builder.restoreIP(*AfterIP);
+    ValueReplacementMap[Input] = std::make_tuple(InputCopy, Arg.getArgNo());
 
     // In certain cases a Global may be set up for replacement, however, this
     // Global may be used in multiple arguments to the kernel, just segmented
@@ -6999,6 +7078,8 @@ static Expected<Function *> createOutlinedFunction(
   for (auto Deferred : DeferredReplacement)
     ReplaceValue(std::get<0>(Deferred), std::get<1>(Deferred), Func);
 
+  FixupDebugInfoForOutlinedFunction(OMPBuilder, Builder, Func,
+                                    ValueReplacementMap);
   return Func;
 }
 
@@ -7256,10 +7337,12 @@ OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::emitTargetTask(
     // If `HasNoWait == true`, we call  @__kmpc_omp_target_task_alloc to provide
     // the DeviceID to the deferred task and also since
     // @__kmpc_omp_target_task_alloc creates an untied/async task.
+    bool NeedsTargetTask = HasNoWait && DeviceID;
     Function *TaskAllocFn =
-        !HasNoWait ? getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_omp_task_alloc)
-                   : getOrCreateRuntimeFunctionPtr(
-                         OMPRTL___kmpc_omp_target_task_alloc);
+        !NeedsTargetTask
+            ? getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_omp_task_alloc)
+            : getOrCreateRuntimeFunctionPtr(
+                  OMPRTL___kmpc_omp_target_task_alloc);
 
     // Arguments - `loc_ref` (Ident) and `gtid` (ThreadID)
     // call.
@@ -7308,8 +7391,10 @@ OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::emitTargetTask(
         /*sizeof_task=*/TaskSize, /*sizeof_shared=*/SharedsSize,
         /*task_func=*/ProxyFn};
 
-    if (HasNoWait)
+    if (NeedsTargetTask) {
+      assert(DeviceID && "Expected non-empty device ID.");
       TaskAllocArgs.push_back(DeviceID);
+    }
 
     TaskData = Builder.CreateCall(TaskAllocFn, TaskAllocArgs);
 
@@ -7331,7 +7416,7 @@ OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::emitTargetTask(
     // ---------------------------------------------------------------
     // The above means that the lack of a nowait on the target construct
     // translates to '#pragma omp task if(0)'
-    if (!HasNoWait) {
+    if (!NeedsTargetTask) {
       if (DepArray) {
         Function *TaskWaitFn =
             getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_omp_wait_deps);
@@ -7432,10 +7517,15 @@ emitTargetCall(OpenMPIRBuilder &OMPBuilder, IRBuilderBase &Builder,
       // '@.omp_target_task_proxy_func' in the pseudo code above)
       // "@.omp_target_task_proxy_func' is generated by
       // emitTargetTaskProxyFunction.
-      if (OutlinedFnID)
+      if (OutlinedFnID && DeviceID)
         return OMPBuilder.emitKernelLaunch(Builder, OutlinedFnID,
                                            EmitTargetCallFallbackCB, KArgs,
                                            DeviceID, RTLoc, TargetTaskAllocaIP);
+
+      // We only need to do the outlining if `DeviceID` is set to avoid calling
+      // `emitKernelLaunch` if we want to code-gen for the host; e.g. if we are
+      // generating the `else` branch of an `if` clause.
+      //
       // When OutlinedFnID is set to nullptr, then it's not an offloading call.
       // In this case, we execute the host implementation directly.
       return EmitTargetCallFallbackCB(OMPBuilder.Builder.saveIP());
@@ -9213,8 +9303,8 @@ void OpenMPIRBuilder::createOffloadEntry(Constant *ID, Constant *Addr,
                                          StringRef Name) {
   if (!Config.isGPU()) {
     llvm::offloading::emitOffloadingEntry(
-        M, ID, Name.empty() ? Addr->getName() : Name, Size, Flags, /*Data=*/0,
-        "omp_offloading_entries");
+        M, object::OffloadKind::OFK_OpenMP, ID,
+        Name.empty() ? Addr->getName() : Name, Size, Flags, /*Data=*/0);
     return;
   }
   // TODO: Add support for global variables on the device after declare target
@@ -9223,20 +9313,8 @@ void OpenMPIRBuilder::createOffloadEntry(Constant *ID, Constant *Addr,
   if (!Fn)
     return;
 
-  Module &M = *(Fn->getParent());
-  LLVMContext &Ctx = M.getContext();
-
-  // Get "nvvm.annotations" metadata node.
-  NamedMDNode *MD = M.getOrInsertNamedMetadata("nvvm.annotations");
-
-  Metadata *MDVals[] = {
-      ConstantAsMetadata::get(Fn), MDString::get(Ctx, "kernel"),
-      ConstantAsMetadata::get(ConstantInt::get(Type::getInt32Ty(Ctx), 1))};
-  // Append metadata to nvvm.annotations.
-  MD->addOperand(MDNode::get(Ctx, MDVals));
-
   // Add a function attribute for the kernel.
-  Fn->addFnAttr(Attribute::get(Ctx, "kernel"));
+  Fn->addFnAttr("kernel");
   if (T.isAMDGCN())
     Fn->addFnAttr("uniform-work-group-size", "true");
   Fn->addFnAttr(Attribute::MustProgress);
@@ -9400,10 +9478,11 @@ void OpenMPIRBuilder::createOffloadEntriesAndInfoMetadata(
   //       entries should be redesigned to better suit this use-case.
   if (Config.hasRequiresFlags() && !Config.isTargetDevice())
     offloading::emitOffloadingEntry(
-        M, Constant::getNullValue(PointerType::getUnqual(M.getContext())),
-        /*Name=*/"",
-        /*Size=*/0, OffloadEntriesInfoManager::OMPTargetGlobalRegisterRequires,
-        Config.getRequiresFlags(), "omp_offloading_entries");
+        M, object::OffloadKind::OFK_OpenMP,
+        Constant::getNullValue(PointerType::getUnqual(M.getContext())),
+        ".requires", /*Size=*/0,
+        OffloadEntriesInfoManager::OMPTargetGlobalRegisterRequires,
+        Config.getRequiresFlags());
 }
 
 void TargetRegionEntryInfo::getTargetRegionEntryFnName(
diff --git a/llvm/lib/IR/AsmWriter.cpp b/llvm/lib/IR/AsmWriter.cpp
index a37a8901489cf..57e9cccdc0fb6 100644
--- a/llvm/lib/IR/AsmWriter.cpp
+++ b/llvm/lib/IR/AsmWriter.cpp
@@ -2004,7 +2004,7 @@ void MDFieldPrinter::printNameTableKind(StringRef Name,
 template <class IntTy, class Stringifier>
 void MDFieldPrinter::printDwarfEnum(StringRef Name, IntTy Value,
                                     Stringifier toString, bool ShouldSkipZero) {
-  if (!Value)
+  if (ShouldSkipZero && !Value)
     return;
 
   Out << FS << Name << ": ";
@@ -2257,6 +2257,11 @@ static void writeDICompositeType(raw_ostream &Out, const DICompositeType *N,
   Printer.printMetadata("annotations", N->getRawAnnotations());
   if (auto *Specification = N->getRawSpecification())
     Printer.printMetadata("specification", Specification);
+
+  if (auto EnumKind = N->getEnumKind())
+    Printer.printDwarfEnum("enumKind", *EnumKind, dwarf::EnumKindString,
+                           /*ShouldSkipZero=*/false);
+
   Out << ")";
 }
 
diff --git a/llvm/lib/IR/Assumptions.cpp b/llvm/lib/IR/Assumptions.cpp
index d1f8bcde53b26..ad874485fb3a0 100644
--- a/llvm/lib/IR/Assumptions.cpp
+++ b/llvm/lib/IR/Assumptions.cpp
@@ -103,10 +103,11 @@ bool llvm::addAssumptions(CallBase &CB,
 }
 
 StringSet<> llvm::KnownAssumptionStrings({
-    "omp_no_openmp",          // OpenMP 5.1
-    "omp_no_openmp_routines", // OpenMP 5.1
-    "omp_no_parallelism",     // OpenMP 5.1
-    "ompx_spmd_amenable",     // OpenMPOpt extension
-    "ompx_no_call_asm",       // OpenMPOpt extension
-    "ompx_aligned_barrier",   // OpenMPOpt extension
+    "omp_no_openmp",            // OpenMP 5.1
+    "omp_no_openmp_routines",   // OpenMP 5.1
+    "omp_no_parallelism",       // OpenMP 5.1
+    "omp_no_openmp_constructs", // OpenMP 6.0
+    "ompx_spmd_amenable",       // OpenMPOpt extension
+    "ompx_no_call_asm",         // OpenMPOpt extension
+    "ompx_aligned_barrier",     // OpenMPOpt extension
 });
diff --git a/llvm/lib/IR/Attributes.cpp b/llvm/lib/IR/Attributes.cpp
index ceb31856283c9..ef0591ef31744 100644
--- a/llvm/lib/IR/Attributes.cpp
+++ b/llvm/lib/IR/Attributes.cpp
@@ -288,6 +288,10 @@ Attribute Attribute::getWithNoFPClass(LLVMContext &Context,
   return get(Context, NoFPClass, ClassMask);
 }
 
+Attribute Attribute::getWithCaptureInfo(LLVMContext &Context, CaptureInfo CI) {
+  return get(Context, Captures, CI.toIntValue());
+}
+
 Attribute
 Attribute::getWithAllocSizeArgs(LLVMContext &Context, unsigned ElemSizeArg,
                                 const std::optional<unsigned> &NumElemsArg) {
@@ -2373,7 +2377,6 @@ AttributeMask AttributeFuncs::typeIncompatible(Type *Ty, AttributeSet AS,
     // Attributes that only apply to pointers.
     if (ASK & ASK_SAFE_TO_DROP)
       Incompatible.addAttribute(Attribute::NoAlias)
-          .addAttribute(Attribute::NoCapture)
           .addAttribute(Attribute::NonNull)
           .addAttribute(Attribute::ReadNone)
           .addAttribute(Attribute::ReadOnly)
diff --git a/llvm/lib/IR/AutoUpgrade.cpp b/llvm/lib/IR/AutoUpgrade.cpp
index 3725f412b8930..e886a6012b219 100644
--- a/llvm/lib/IR/AutoUpgrade.cpp
+++ b/llvm/lib/IR/AutoUpgrade.cpp
@@ -17,6 +17,7 @@
 #include "llvm/ADT/StringSwitch.h"
 #include "llvm/BinaryFormat/Dwarf.h"
 #include "llvm/IR/AttributeMask.h"
+#include "llvm/IR/CallingConv.h"
 #include "llvm/IR/Constants.h"
 #include "llvm/IR/DebugInfo.h"
 #include "llvm/IR/DebugInfoMetadata.h"
@@ -5019,6 +5020,72 @@ bool llvm::UpgradeDebugInfo(Module &M) {
   return Modified;
 }
 
+bool static upgradeSingleNVVMAnnotation(GlobalValue *GV, StringRef K,
+                                        const Metadata *V) {
+  if (K == "kernel") {
+    if (!mdconst::extract<ConstantInt>(V)->isZero())
+      cast<Function>(GV)->setCallingConv(CallingConv::PTX_Kernel);
+    return true;
+  }
+  if (K == "align") {
+    // V is a bitfeild specifying two 16-bit values. The alignment value is
+    // specfied in low 16-bits, The index is specified in the high bits. For the
+    // index, 0 indicates the return value while higher values correspond to
+    // each parameter (idx = param + 1).
+    const uint64_t AlignIdxValuePair =
+        mdconst::extract<ConstantInt>(V)->getZExtValue();
+    const unsigned Idx = (AlignIdxValuePair >> 16);
+    const Align StackAlign = Align(AlignIdxValuePair & 0xFFFF);
+    // TODO: Skip adding the stackalign attribute for returns, for now.
+    if (!Idx)
+      return false;
+    cast<Function>(GV)->addAttributeAtIndex(
+        Idx, Attribute::getWithStackAlignment(GV->getContext(), StackAlign));
+    return true;
+  }
+
+  return false;
+}
+
+void llvm::UpgradeNVVMAnnotations(Module &M) {
+  NamedMDNode *NamedMD = M.getNamedMetadata("nvvm.annotations");
+  if (!NamedMD)
+    return;
+
+  SmallVector<MDNode *, 8> NewNodes;
+  SmallSet<const MDNode *, 8> SeenNodes;
+  for (MDNode *MD : NamedMD->operands()) {
+    if (!SeenNodes.insert(MD).second)
+      continue;
+
+    auto *GV = mdconst::dyn_extract_or_null<GlobalValue>(MD->getOperand(0));
+    if (!GV)
+      continue;
+
+    assert((MD->getNumOperands() % 2) == 1 && "Invalid number of operands");
+
+    SmallVector<Metadata *, 8> NewOperands{MD->getOperand(0)};
+    // Each nvvm.annotations metadata entry will be of the following form:
+    //   !{ ptr @gv, !"key1", value1, !"key2", value2, ... }
+    // start index = 1, to skip the global variable key
+    // increment = 2, to skip the value for each property-value pairs
+    for (unsigned j = 1, je = MD->getNumOperands(); j < je; j += 2) {
+      MDString *K = cast<MDString>(MD->getOperand(j));
+      const MDOperand &V = MD->getOperand(j + 1);
+      bool Upgraded = upgradeSingleNVVMAnnotation(GV, K->getString(), V);
+      if (!Upgraded)
+        NewOperands.append({K, V});
+    }
+
+    if (NewOperands.size() > 1)
+      NewNodes.push_back(MDNode::get(M.getContext(), NewOperands));
+  }
+
+  NamedMD->clearOperands();
+  for (MDNode *N : NewNodes)
+    NamedMD->addOperand(N);
+}
+
 /// This checks for objc retain release marker which should be upgraded. It
 /// returns true if module is modified.
 static bool upgradeRetainReleaseMarker(Module &M) {
diff --git a/llvm/lib/IR/BasicBlock.cpp b/llvm/lib/IR/BasicBlock.cpp
index 0efc04cb2c867..dca42a57fa9e3 100644
--- a/llvm/lib/IR/BasicBlock.cpp
+++ b/llvm/lib/IR/BasicBlock.cpp
@@ -371,19 +371,31 @@ const Instruction* BasicBlock::getFirstNonPHI() const {
   return nullptr;
 }
 
+Instruction *BasicBlock::getFirstNonPHI() {
+  for (Instruction &I : *this)
+    if (!isa<PHINode>(I))
+      return &I;
+  return nullptr;
+}
+
 BasicBlock::const_iterator BasicBlock::getFirstNonPHIIt() const {
-  const Instruction *I = getFirstNonPHI();
-  if (!I)
-    return end();
-  BasicBlock::const_iterator It = I->getIterator();
-  // Set the head-inclusive bit to indicate that this iterator includes
-  // any debug-info at the start of the block. This is a no-op unless the
-  // appropriate CMake flag is set.
-  It.setHeadBit(true);
-  return It;
+  for (const Instruction &I : *this) {
+    if (isa<PHINode>(I))
+      continue;
+
+    BasicBlock::const_iterator It = I.getIterator();
+    // Set the head-inclusive bit to indicate that this iterator includes
+    // any debug-info at the start of the block. This is a no-op unless the
+    // appropriate CMake flag is set.
+    It.setHeadBit(true);
+    return It;
+  }
+
+  return end();
 }
 
-const Instruction *BasicBlock::getFirstNonPHIOrDbg(bool SkipPseudoOp) const {
+BasicBlock::const_iterator
+BasicBlock::getFirstNonPHIOrDbg(bool SkipPseudoOp) const {
   for (const Instruction &I : *this) {
     if (isa<PHINode>(I) || isa<DbgInfoIntrinsic>(I))
       continue;
@@ -391,12 +403,16 @@ const Instruction *BasicBlock::getFirstNonPHIOrDbg(bool SkipPseudoOp) const {
     if (SkipPseudoOp && isa<PseudoProbeInst>(I))
       continue;
 
-    return &I;
+    BasicBlock::const_iterator It = I.getIterator();
+    // This position comes after any debug records, the head bit should remain
+    // unset.
+    assert(!It.getHeadBit());
+    return It;
   }
-  return nullptr;
+  return end();
 }
 
-const Instruction *
+BasicBlock::const_iterator
 BasicBlock::getFirstNonPHIOrDbgOrLifetime(bool SkipPseudoOp) const {
   for (const Instruction &I : *this) {
     if (isa<PHINode>(I) || isa<DbgInfoIntrinsic>(I))
@@ -408,17 +424,21 @@ BasicBlock::getFirstNonPHIOrDbgOrLifetime(bool SkipPseudoOp) const {
     if (SkipPseudoOp && isa<PseudoProbeInst>(I))
       continue;
 
-    return &I;
+    BasicBlock::const_iterator It = I.getIterator();
+    // This position comes after any debug records, the head bit should remain
+    // unset.
+    assert(!It.getHeadBit());
+
+    return It;
   }
-  return nullptr;
+  return end();
 }
 
 BasicBlock::const_iterator BasicBlock::getFirstInsertionPt() const {
-  const Instruction *FirstNonPHI = getFirstNonPHI();
-  if (!FirstNonPHI)
+  const_iterator InsertPt = getFirstNonPHIIt();
+  if (InsertPt == end())
     return end();
 
-  const_iterator InsertPt = FirstNonPHI->getIterator();
   if (InsertPt->isEHPad()) ++InsertPt;
   // Set the head-inclusive bit to indicate that this iterator includes
   // any debug-info at the start of the block. This is a no-op unless the
@@ -428,11 +448,10 @@ BasicBlock::const_iterator BasicBlock::getFirstInsertionPt() const {
 }
 
 BasicBlock::const_iterator BasicBlock::getFirstNonPHIOrDbgOrAlloca() const {
-  const Instruction *FirstNonPHI = getFirstNonPHI();
-  if (!FirstNonPHI)
+  const_iterator InsertPt = getFirstNonPHIIt();
+  if (InsertPt == end())
     return end();
 
-  const_iterator InsertPt = FirstNonPHI->getIterator();
   if (InsertPt->isEHPad())
     ++InsertPt;
 
@@ -448,6 +467,9 @@ BasicBlock::const_iterator BasicBlock::getFirstNonPHIOrDbgOrAlloca() const {
       ++InsertPt;
     }
   }
+
+  // Signal that this comes after any debug records.
+  InsertPt.setHeadBit(false);
   return InsertPt;
 }
 
@@ -543,7 +565,7 @@ void BasicBlock::removePredecessor(BasicBlock *Pred,
 }
 
 bool BasicBlock::canSplitPredecessors() const {
-  const Instruction *FirstNonPHI = getFirstNonPHI();
+  const_iterator FirstNonPHI = getFirstNonPHIIt();
   if (isa<LandingPadInst>(FirstNonPHI))
     return true;
   // This is perhaps a little conservative because constructs like
@@ -675,11 +697,11 @@ void BasicBlock::replaceSuccessorsPhiUsesWith(BasicBlock *New) {
 }
 
 bool BasicBlock::isLandingPad() const {
-  return isa<LandingPadInst>(getFirstNonPHI());
+  return isa<LandingPadInst>(getFirstNonPHIIt());
 }
 
 const LandingPadInst *BasicBlock::getLandingPadInst() const {
-  return dyn_cast<LandingPadInst>(getFirstNonPHI());
+  return dyn_cast<LandingPadInst>(getFirstNonPHIIt());
 }
 
 std::optional<uint64_t> BasicBlock::getIrrLoopHeaderWeight() const {
diff --git a/llvm/lib/IR/DIBuilder.cpp b/llvm/lib/IR/DIBuilder.cpp
index 8f9462ab46d88..b49b4e4f3fd2d 100644
--- a/llvm/lib/IR/DIBuilder.cpp
+++ b/llvm/lib/IR/DIBuilder.cpp
@@ -511,8 +511,8 @@ DICompositeType *DIBuilder::createClassType(
   auto *R = DICompositeType::get(
       VMContext, dwarf::DW_TAG_class_type, Name, File, LineNumber,
       getNonCompileUnitScope(Context), DerivedFrom, SizeInBits, AlignInBits,
-      OffsetInBits, Flags, Elements, RunTimeLang, VTableHolder,
-      cast_or_null<MDTuple>(TemplateParams), UniqueIdentifier);
+      OffsetInBits, Flags, Elements, RunTimeLang, /*EnumKind=*/std::nullopt,
+      VTableHolder, cast_or_null<MDTuple>(TemplateParams), UniqueIdentifier);
   trackIfUnresolved(R);
   return R;
 }
@@ -526,9 +526,9 @@ DICompositeType *DIBuilder::createStructType(
   auto *R = DICompositeType::get(
       VMContext, dwarf::DW_TAG_structure_type, Name, File, LineNumber,
       getNonCompileUnitScope(Context), DerivedFrom, SizeInBits, AlignInBits, 0,
-      Flags, Elements, RunTimeLang, VTableHolder, nullptr, UniqueIdentifier,
-      nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, Specification,
-      NumExtraInhabitants);
+      Flags, Elements, RunTimeLang, /*EnumKind=*/std::nullopt, VTableHolder,
+      nullptr, UniqueIdentifier, nullptr, nullptr, nullptr, nullptr, nullptr,
+      nullptr, Specification, NumExtraInhabitants);
   trackIfUnresolved(R);
   return R;
 }
@@ -540,7 +540,8 @@ DICompositeType *DIBuilder::createUnionType(
   auto *R = DICompositeType::get(
       VMContext, dwarf::DW_TAG_union_type, Name, File, LineNumber,
       getNonCompileUnitScope(Scope), nullptr, SizeInBits, AlignInBits, 0, Flags,
-      Elements, RunTimeLang, nullptr, nullptr, UniqueIdentifier);
+      Elements, RunTimeLang, /*EnumKind=*/std::nullopt, nullptr, nullptr,
+      UniqueIdentifier);
   trackIfUnresolved(R);
   return R;
 }
@@ -554,7 +555,8 @@ DIBuilder::createVariantPart(DIScope *Scope, StringRef Name, DIFile *File,
   auto *R = DICompositeType::get(
       VMContext, dwarf::DW_TAG_variant_part, Name, File, LineNumber,
       getNonCompileUnitScope(Scope), nullptr, SizeInBits, AlignInBits, 0, Flags,
-      Elements, 0, nullptr, nullptr, UniqueIdentifier, Discriminator);
+      Elements, 0, /*EnumKind=*/std::nullopt, nullptr, nullptr,
+      UniqueIdentifier, Discriminator);
   trackIfUnresolved(R);
   return R;
 }
@@ -565,17 +567,16 @@ DISubroutineType *DIBuilder::createSubroutineType(DITypeRefArray ParameterTypes,
   return DISubroutineType::get(VMContext, Flags, CC, ParameterTypes);
 }
 
-DICompositeType *
-DIBuilder::createEnumerationType(DIScope *Scope, StringRef Name, DIFile *File,
-                                 unsigned LineNumber, uint64_t SizeInBits,
-                                 uint32_t AlignInBits, DINodeArray Elements,
-                                 DIType *UnderlyingType, unsigned RunTimeLang,
-                                 StringRef UniqueIdentifier, bool IsScoped) {
+DICompositeType *DIBuilder::createEnumerationType(
+    DIScope *Scope, StringRef Name, DIFile *File, unsigned LineNumber,
+    uint64_t SizeInBits, uint32_t AlignInBits, DINodeArray Elements,
+    DIType *UnderlyingType, unsigned RunTimeLang, StringRef UniqueIdentifier,
+    bool IsScoped, std::optional<uint32_t> EnumKind) {
   auto *CTy = DICompositeType::get(
       VMContext, dwarf::DW_TAG_enumeration_type, Name, File, LineNumber,
       getNonCompileUnitScope(Scope), UnderlyingType, SizeInBits, AlignInBits, 0,
       IsScoped ? DINode::FlagEnumClass : DINode::FlagZero, Elements,
-      RunTimeLang, nullptr, nullptr, UniqueIdentifier);
+      RunTimeLang, EnumKind, nullptr, nullptr, UniqueIdentifier);
   AllEnumTypes.emplace_back(CTy);
   trackIfUnresolved(CTy);
   return CTy;
@@ -602,8 +603,8 @@ DIBuilder::createArrayType(uint64_t Size, uint32_t AlignInBits, DIType *Ty,
                            PointerUnion<DIExpression *, DIVariable *> RK) {
   auto *R = DICompositeType::get(
       VMContext, dwarf::DW_TAG_array_type, "", nullptr, 0, nullptr, Ty, Size,
-      AlignInBits, 0, DINode::FlagZero, Subscripts, 0, nullptr, nullptr, "",
-      nullptr,
+      AlignInBits, 0, DINode::FlagZero, Subscripts, 0,
+      /*EnumKind=*/std::nullopt, nullptr, nullptr, "", nullptr,
       isa<DIExpression *>(DL) ? (Metadata *)cast<DIExpression *>(DL)
                               : (Metadata *)cast<DIVariable *>(DL),
       isa<DIExpression *>(AS) ? (Metadata *)cast<DIExpression *>(AS)
@@ -621,7 +622,8 @@ DICompositeType *DIBuilder::createVectorType(uint64_t Size,
                                              DINodeArray Subscripts) {
   auto *R = DICompositeType::get(VMContext, dwarf::DW_TAG_array_type, "",
                                  nullptr, 0, nullptr, Ty, Size, AlignInBits, 0,
-                                 DINode::FlagVector, Subscripts, 0, nullptr);
+                                 DINode::FlagVector, Subscripts, 0,
+                                 /*EnumKind=*/std::nullopt, nullptr);
   trackIfUnresolved(R);
   return R;
 }
@@ -666,17 +668,16 @@ void DIBuilder::retainType(DIScope *T) {
 
 DIBasicType *DIBuilder::createUnspecifiedParameter() { return nullptr; }
 
-DICompositeType *
-DIBuilder::createForwardDecl(unsigned Tag, StringRef Name, DIScope *Scope,
-                             DIFile *F, unsigned Line, unsigned RuntimeLang,
-                             uint64_t SizeInBits, uint32_t AlignInBits,
-                             StringRef UniqueIdentifier) {
+DICompositeType *DIBuilder::createForwardDecl(
+    unsigned Tag, StringRef Name, DIScope *Scope, DIFile *F, unsigned Line,
+    unsigned RuntimeLang, uint64_t SizeInBits, uint32_t AlignInBits,
+    StringRef UniqueIdentifier, std::optional<uint32_t> EnumKind) {
   // FIXME: Define in terms of createReplaceableForwardDecl() by calling
   // replaceWithUniqued().
   auto *RetTy = DICompositeType::get(
       VMContext, Tag, Name, F, Line, getNonCompileUnitScope(Scope), nullptr,
       SizeInBits, AlignInBits, 0, DINode::FlagFwdDecl, nullptr, RuntimeLang,
-      nullptr, nullptr, UniqueIdentifier);
+      /*EnumKind=*/EnumKind, nullptr, nullptr, UniqueIdentifier);
   trackIfUnresolved(RetTy);
   return RetTy;
 }
@@ -684,14 +685,14 @@ DIBuilder::createForwardDecl(unsigned Tag, StringRef Name, DIScope *Scope,
 DICompositeType *DIBuilder::createReplaceableCompositeType(
     unsigned Tag, StringRef Name, DIScope *Scope, DIFile *F, unsigned Line,
     unsigned RuntimeLang, uint64_t SizeInBits, uint32_t AlignInBits,
-    DINode::DIFlags Flags, StringRef UniqueIdentifier,
-    DINodeArray Annotations) {
+    DINode::DIFlags Flags, StringRef UniqueIdentifier, DINodeArray Annotations,
+    std::optional<uint32_t> EnumKind) {
   auto *RetTy =
       DICompositeType::getTemporary(
           VMContext, Tag, Name, F, Line, getNonCompileUnitScope(Scope), nullptr,
-          SizeInBits, AlignInBits, 0, Flags, nullptr, RuntimeLang, nullptr,
-          nullptr, UniqueIdentifier, nullptr, nullptr, nullptr, nullptr,
-          nullptr, Annotations)
+          SizeInBits, AlignInBits, 0, Flags, nullptr, RuntimeLang, EnumKind,
+          nullptr, nullptr, UniqueIdentifier, nullptr, nullptr, nullptr,
+          nullptr, nullptr, Annotations)
           .release();
   trackIfUnresolved(RetTy);
   return RetTy;
diff --git a/llvm/lib/IR/DataLayout.cpp b/llvm/lib/IR/DataLayout.cpp
index 95a5e5989ad00..0cf0bfc9702d3 100644
--- a/llvm/lib/IR/DataLayout.cpp
+++ b/llvm/lib/IR/DataLayout.cpp
@@ -178,7 +178,7 @@ const char *DataLayout::getManglingComponent(const Triple &T) {
     return "-m:l";
   if (T.isOSBinFormatMachO())
     return "-m:o";
-  if ((T.isOSWindows() || T.isUEFI()) && T.isOSBinFormatCOFF())
+  if (T.isOSWindowsOrUEFI() && T.isOSBinFormatCOFF())
     return T.getArch() == Triple::x86 ? "-m:x" : "-m:w";
   if (T.isOSBinFormatXCOFF())
     return "-m:a";
diff --git a/llvm/lib/IR/DebugInfo.cpp b/llvm/lib/IR/DebugInfo.cpp
index 4ce518009bd3e..ea1d79d436041 100644
--- a/llvm/lib/IR/DebugInfo.cpp
+++ b/llvm/lib/IR/DebugInfo.cpp
@@ -798,7 +798,11 @@ class DebugTypeInfoRemoval {
 
       return getReplacementMDNode(N);
     };
-    Replacements[N] = doRemap(N);
+    // Seperate recursive doRemap and operator [] into 2 lines to avoid
+    // out-of-order evaluations since both of them can access the same memory
+    // location in map Replacements.
+    auto Value = doRemap(N);
+    Replacements[N] = Value;
   }
 
   /// Do the remapping traversal.
diff --git a/llvm/lib/IR/DebugInfoMetadata.cpp b/llvm/lib/IR/DebugInfoMetadata.cpp
index 915cdd301f2c7..32e659d43edcf 100644
--- a/llvm/lib/IR/DebugInfoMetadata.cpp
+++ b/llvm/lib/IR/DebugInfoMetadata.cpp
@@ -767,11 +767,12 @@ DICompositeType *DICompositeType::getImpl(
     LLVMContext &Context, unsigned Tag, MDString *Name, Metadata *File,
     unsigned Line, Metadata *Scope, Metadata *BaseType, uint64_t SizeInBits,
     uint32_t AlignInBits, uint64_t OffsetInBits, DIFlags Flags,
-    Metadata *Elements, unsigned RuntimeLang, Metadata *VTableHolder,
-    Metadata *TemplateParams, MDString *Identifier, Metadata *Discriminator,
-    Metadata *DataLocation, Metadata *Associated, Metadata *Allocated,
-    Metadata *Rank, Metadata *Annotations, Metadata *Specification,
-    uint32_t NumExtraInhabitants, StorageType Storage, bool ShouldCreate) {
+    Metadata *Elements, unsigned RuntimeLang, std::optional<uint32_t> EnumKind,
+    Metadata *VTableHolder, Metadata *TemplateParams, MDString *Identifier,
+    Metadata *Discriminator, Metadata *DataLocation, Metadata *Associated,
+    Metadata *Allocated, Metadata *Rank, Metadata *Annotations,
+    Metadata *Specification, uint32_t NumExtraInhabitants, StorageType Storage,
+    bool ShouldCreate) {
   assert(isCanonical(Name) && "Expected canonical MDString");
 
   // Keep this in sync with buildODRType.
@@ -787,7 +788,7 @@ DICompositeType *DICompositeType::getImpl(
                      Rank,          Annotations,  Specification};
   DEFINE_GETIMPL_STORE(DICompositeType,
                        (Tag, Line, RuntimeLang, SizeInBits, AlignInBits,
-                        OffsetInBits, NumExtraInhabitants, Flags),
+                        OffsetInBits, NumExtraInhabitants, EnumKind, Flags),
                        Ops);
 }
 
@@ -796,10 +797,10 @@ DICompositeType *DICompositeType::buildODRType(
     Metadata *File, unsigned Line, Metadata *Scope, Metadata *BaseType,
     uint64_t SizeInBits, uint32_t AlignInBits, uint64_t OffsetInBits,
     Metadata *Specification, uint32_t NumExtraInhabitants, DIFlags Flags,
-    Metadata *Elements, unsigned RuntimeLang, Metadata *VTableHolder,
-    Metadata *TemplateParams, Metadata *Discriminator, Metadata *DataLocation,
-    Metadata *Associated, Metadata *Allocated, Metadata *Rank,
-    Metadata *Annotations) {
+    Metadata *Elements, unsigned RuntimeLang, std::optional<uint32_t> EnumKind,
+    Metadata *VTableHolder, Metadata *TemplateParams, Metadata *Discriminator,
+    Metadata *DataLocation, Metadata *Associated, Metadata *Allocated,
+    Metadata *Rank, Metadata *Annotations) {
   assert(!Identifier.getString().empty() && "Expected valid identifier");
   if (!Context.isODRUniquingDebugTypes())
     return nullptr;
@@ -808,9 +809,9 @@ DICompositeType *DICompositeType::buildODRType(
     return CT = DICompositeType::getDistinct(
                Context, Tag, Name, File, Line, Scope, BaseType, SizeInBits,
                AlignInBits, OffsetInBits, Flags, Elements, RuntimeLang,
-               VTableHolder, TemplateParams, &Identifier, Discriminator,
-               DataLocation, Associated, Allocated, Rank, Annotations,
-               Specification, NumExtraInhabitants);
+               EnumKind, VTableHolder, TemplateParams, &Identifier,
+               Discriminator, DataLocation, Associated, Allocated, Rank,
+               Annotations, Specification, NumExtraInhabitants);
   if (CT->getTag() != Tag)
     return nullptr;
 
@@ -821,7 +822,7 @@ DICompositeType *DICompositeType::buildODRType(
 
   // Mutate CT in place.  Keep this in sync with getImpl.
   CT->mutate(Tag, Line, RuntimeLang, SizeInBits, AlignInBits, OffsetInBits,
-             NumExtraInhabitants, Flags);
+             NumExtraInhabitants, EnumKind, Flags);
   Metadata *Ops[] = {File,          Scope,        Name,           BaseType,
                      Elements,      VTableHolder, TemplateParams, &Identifier,
                      Discriminator, DataLocation, Associated,     Allocated,
@@ -839,10 +840,10 @@ DICompositeType *DICompositeType::getODRType(
     Metadata *File, unsigned Line, Metadata *Scope, Metadata *BaseType,
     uint64_t SizeInBits, uint32_t AlignInBits, uint64_t OffsetInBits,
     Metadata *Specification, uint32_t NumExtraInhabitants, DIFlags Flags,
-    Metadata *Elements, unsigned RuntimeLang, Metadata *VTableHolder,
-    Metadata *TemplateParams, Metadata *Discriminator, Metadata *DataLocation,
-    Metadata *Associated, Metadata *Allocated, Metadata *Rank,
-    Metadata *Annotations) {
+    Metadata *Elements, unsigned RuntimeLang, std::optional<uint32_t> EnumKind,
+    Metadata *VTableHolder, Metadata *TemplateParams, Metadata *Discriminator,
+    Metadata *DataLocation, Metadata *Associated, Metadata *Allocated,
+    Metadata *Rank, Metadata *Annotations) {
   assert(!Identifier.getString().empty() && "Expected valid identifier");
   if (!Context.isODRUniquingDebugTypes())
     return nullptr;
@@ -850,9 +851,10 @@ DICompositeType *DICompositeType::getODRType(
   if (!CT) {
     CT = DICompositeType::getDistinct(
         Context, Tag, Name, File, Line, Scope, BaseType, SizeInBits,
-        AlignInBits, OffsetInBits, Flags, Elements, RuntimeLang, VTableHolder,
-        TemplateParams, &Identifier, Discriminator, DataLocation, Associated,
-        Allocated, Rank, Annotations, Specification, NumExtraInhabitants);
+        AlignInBits, OffsetInBits, Flags, Elements, RuntimeLang, EnumKind,
+        VTableHolder, TemplateParams, &Identifier, Discriminator, DataLocation,
+        Associated, Allocated, Rank, Annotations, Specification,
+        NumExtraInhabitants);
   } else {
     if (CT->getTag() != Tag)
       return nullptr;
diff --git a/llvm/lib/IR/Function.cpp b/llvm/lib/IR/Function.cpp
index 9c5dd5aeb92e9..5666f0a53866f 100644
--- a/llvm/lib/IR/Function.cpp
+++ b/llvm/lib/IR/Function.cpp
@@ -287,7 +287,7 @@ bool Argument::hasNoAliasAttr() const {
 
 bool Argument::hasNoCaptureAttr() const {
   if (!getType()->isPointerTy()) return false;
-  return hasAttribute(Attribute::NoCapture);
+  return capturesNothing(getAttributes().getCaptureInfo());
 }
 
 bool Argument::hasNoFreeAttr() const {
@@ -1164,22 +1164,6 @@ DenseSet<GlobalValue::GUID> Function::getImportGUIDs() const {
   return R;
 }
 
-void Function::setSectionPrefix(StringRef Prefix) {
-  MDBuilder MDB(getContext());
-  setMetadata(LLVMContext::MD_section_prefix,
-              MDB.createFunctionSectionPrefix(Prefix));
-}
-
-std::optional<StringRef> Function::getSectionPrefix() const {
-  if (MDNode *MD = getMetadata(LLVMContext::MD_section_prefix)) {
-    assert(cast<MDString>(MD->getOperand(0))->getString() ==
-               "function_section_prefix" &&
-           "Metadata not match");
-    return cast<MDString>(MD->getOperand(1))->getString();
-  }
-  return std::nullopt;
-}
-
 bool Function::nullPointerIsDefined() const {
   return hasFnAttribute(Attribute::NullPointerIsValid);
 }
diff --git a/llvm/lib/IR/Globals.cpp b/llvm/lib/IR/Globals.cpp
index db5e1cb57b1ba..148f8e26535d2 100644
--- a/llvm/lib/IR/Globals.cpp
+++ b/llvm/lib/IR/Globals.cpp
@@ -18,6 +18,7 @@
 #include "llvm/IR/GlobalAlias.h"
 #include "llvm/IR/GlobalValue.h"
 #include "llvm/IR/GlobalVariable.h"
+#include "llvm/IR/MDBuilder.h"
 #include "llvm/IR/Module.h"
 #include "llvm/Support/Error.h"
 #include "llvm/Support/ErrorHandling.h"
@@ -286,6 +287,24 @@ void GlobalObject::setSection(StringRef S) {
   setGlobalObjectFlag(HasSectionHashEntryBit, !S.empty());
 }
 
+void GlobalObject::setSectionPrefix(StringRef Prefix) {
+  MDBuilder MDB(getContext());
+  setMetadata(LLVMContext::MD_section_prefix,
+              MDB.createGlobalObjectSectionPrefix(Prefix));
+}
+
+std::optional<StringRef> GlobalObject::getSectionPrefix() const {
+  if (MDNode *MD = getMetadata(LLVMContext::MD_section_prefix)) {
+    [[maybe_unused]] StringRef MDName =
+        cast<MDString>(MD->getOperand(0))->getString();
+    assert((MDName == "section_prefix" ||
+            (isa<Function>(this) && MDName == "function_section_prefix")) &&
+           "Metadata not match");
+    return cast<MDString>(MD->getOperand(1))->getString();
+  }
+  return std::nullopt;
+}
+
 bool GlobalValue::isNobuiltinFnDef() const {
   const Function *F = dyn_cast<Function>(this);
   if (!F || F->empty())
diff --git a/llvm/lib/IR/Instruction.cpp b/llvm/lib/IR/Instruction.cpp
index 4ab47edf3ed7d..84d9306ca6700 100644
--- a/llvm/lib/IR/Instruction.cpp
+++ b/llvm/lib/IR/Instruction.cpp
@@ -1169,7 +1169,7 @@ bool Instruction::mayThrow(bool IncludePhaseOneUnwind) const {
     // Landingpads themselves don't unwind -- however, an invoke of a skipped
     // landingpad may continue unwinding.
     BasicBlock *UnwindDest = cast<InvokeInst>(this)->getUnwindDest();
-    Instruction *Pad = UnwindDest->getFirstNonPHI();
+    BasicBlock::iterator Pad = UnwindDest->getFirstNonPHIIt();
     if (auto *LP = dyn_cast<LandingPadInst>(Pad))
       return canUnwindPastLandingPad(LP, IncludePhaseOneUnwind);
     return false;
diff --git a/llvm/lib/IR/Instructions.cpp b/llvm/lib/IR/Instructions.cpp
index c9f5807765e40..e2d607368e94b 100644
--- a/llvm/lib/IR/Instructions.cpp
+++ b/llvm/lib/IR/Instructions.cpp
@@ -432,6 +432,23 @@ bool CallBase::paramHasAttr(unsigned ArgNo, Attribute::AttrKind Kind) const {
   }
 }
 
+bool CallBase::paramHasNonNullAttr(unsigned ArgNo,
+                                   bool AllowUndefOrPoison) const {
+  assert(getArgOperand(ArgNo)->getType()->isPointerTy() &&
+         "Argument must be a pointer");
+  if (paramHasAttr(ArgNo, Attribute::NonNull) &&
+      (AllowUndefOrPoison || paramHasAttr(ArgNo, Attribute::NoUndef)))
+    return true;
+
+  if (getParamDereferenceableBytes(ArgNo) > 0 &&
+      !NullPointerIsDefined(
+          getCaller(),
+          getArgOperand(ArgNo)->getType()->getPointerAddressSpace()))
+    return true;
+
+  return false;
+}
+
 bool CallBase::hasFnAttrOnCalledFunction(Attribute::AttrKind Kind) const {
   if (auto *F = dyn_cast<Function>(getCalledOperand()))
     return F->getAttributes().hasFnAttr(Kind);
@@ -675,6 +692,25 @@ void CallBase::setOnlyAccessesInaccessibleMemOrArgMem() {
                    MemoryEffects::inaccessibleOrArgMemOnly());
 }
 
+CaptureInfo CallBase::getCaptureInfo(unsigned OpNo) const {
+  if (OpNo < arg_size()) {
+    // If the argument is passed byval, the callee does not have access to the
+    // original pointer and thus cannot capture it.
+    if (isByValArgument(OpNo))
+      return CaptureInfo::none();
+
+    CaptureInfo CI = getParamAttributes(OpNo).getCaptureInfo();
+    if (auto *Fn = dyn_cast<Function>(getCalledOperand()))
+      CI &= Fn->getAttributes().getParamAttrs(OpNo).getCaptureInfo();
+    return CI;
+  }
+
+  // deopt operand bundles are captures(none)
+  auto &BOI = getBundleOpInfoForOperand(OpNo);
+  auto OBU = operandBundleFromBundleOpInfo(BOI);
+  return OBU.isDeoptOperandBundle() ? CaptureInfo::none() : CaptureInfo::all();
+}
+
 //===----------------------------------------------------------------------===//
 //                        CallInst Implementation
 //===----------------------------------------------------------------------===//
@@ -2666,8 +2702,7 @@ BinaryOperator *BinaryOperator::CreateNot(Value *Op, const Twine &Name,
 
 // Exchange the two operands to this instruction. This instruction is safe to
 // use on any binary instruction and does not modify the semantics of the
-// instruction. If the instruction is order-dependent (SetLT f.e.), the opcode
-// is changed.
+// instruction.
 bool BinaryOperator::swapOperands() {
   if (!isCommutative())
     return true; // Can't commute operands
diff --git a/llvm/lib/IR/IntrinsicInst.cpp b/llvm/lib/IR/IntrinsicInst.cpp
index ad174b1487a64..256bce1abe71f 100644
--- a/llvm/lib/IR/IntrinsicInst.cpp
+++ b/llvm/lib/IR/IntrinsicInst.cpp
@@ -885,3 +885,31 @@ Value *GCRelocateInst::getDerivedPtr() const {
     return *(Opt->Inputs.begin() + getDerivedPtrIndex());
   return *(GCInst->arg_begin() + getDerivedPtrIndex());
 }
+
+ConvergenceControlInst *ConvergenceControlInst::CreateAnchor(BasicBlock &BB) {
+  Module *M = BB.getModule();
+  Function *Fn = Intrinsic::getOrInsertDeclaration(
+      M, llvm::Intrinsic::experimental_convergence_anchor);
+  auto *Call = CallInst::Create(Fn, "", BB.getFirstInsertionPt());
+  return cast<ConvergenceControlInst>(Call);
+}
+
+ConvergenceControlInst *ConvergenceControlInst::CreateEntry(BasicBlock &BB) {
+  Module *M = BB.getModule();
+  Function *Fn = Intrinsic::getOrInsertDeclaration(
+      M, llvm::Intrinsic::experimental_convergence_entry);
+  auto *Call = CallInst::Create(Fn, "", BB.getFirstInsertionPt());
+  return cast<ConvergenceControlInst>(Call);
+}
+
+ConvergenceControlInst *
+ConvergenceControlInst::CreateLoop(BasicBlock &BB,
+                                   ConvergenceControlInst *ParentToken) {
+  Module *M = BB.getModule();
+  Function *Fn = Intrinsic::getOrInsertDeclaration(
+      M, llvm::Intrinsic::experimental_convergence_loop);
+  llvm::Value *BundleArgs[] = {ParentToken};
+  llvm::OperandBundleDef OB("convergencectrl", BundleArgs);
+  auto *Call = CallInst::Create(Fn, {}, {OB}, "", BB.getFirstInsertionPt());
+  return cast<ConvergenceControlInst>(Call);
+}
diff --git a/llvm/lib/IR/Intrinsics.cpp b/llvm/lib/IR/Intrinsics.cpp
index ec1184e8d835d..6e3c49b6d50b9 100644
--- a/llvm/lib/IR/Intrinsics.cpp
+++ b/llvm/lib/IR/Intrinsics.cpp
@@ -12,6 +12,7 @@
 
 #include "llvm/IR/Intrinsics.h"
 #include "llvm/ADT/StringExtras.h"
+#include "llvm/ADT/StringTable.h"
 #include "llvm/IR/Function.h"
 #include "llvm/IR/IntrinsicsAArch64.h"
 #include "llvm/IR/IntrinsicsAMDGPU.h"
@@ -40,7 +41,7 @@ using namespace llvm;
 
 StringRef Intrinsic::getBaseName(ID id) {
   assert(id < num_intrinsics && "Invalid intrinsic ID!");
-  return IntrinsicNameTable + IntrinsicNameOffsetTable[id];
+  return IntrinsicNameTable[IntrinsicNameOffsetTable[id]];
 }
 
 StringRef Intrinsic::getName(ID id) {
@@ -362,6 +363,24 @@ DecodeIITType(unsigned &NextElt, ArrayRef<unsigned char> Infos,
         IITDescriptor::get(IITDescriptor::HalfVecArgument, ArgInfo));
     return;
   }
+  case IIT_ONE_THIRD_VEC_ARG: {
+    unsigned ArgInfo = (NextElt == Infos.size() ? 0 : Infos[NextElt++]);
+    OutputTable.push_back(
+        IITDescriptor::get(IITDescriptor::OneThirdVecArgument, ArgInfo));
+    return;
+  }
+  case IIT_ONE_FIFTH_VEC_ARG: {
+    unsigned ArgInfo = (NextElt == Infos.size() ? 0 : Infos[NextElt++]);
+    OutputTable.push_back(
+        IITDescriptor::get(IITDescriptor::OneFifthVecArgument, ArgInfo));
+    return;
+  }
+  case IIT_ONE_SEVENTH_VEC_ARG: {
+    unsigned ArgInfo = (NextElt == Infos.size() ? 0 : Infos[NextElt++]);
+    OutputTable.push_back(
+        IITDescriptor::get(IITDescriptor::OneSeventhVecArgument, ArgInfo));
+    return;
+  }
   case IIT_SAME_VEC_WIDTH_ARG: {
     unsigned ArgInfo = (NextElt == Infos.size() ? 0 : Infos[NextElt++]);
     OutputTable.push_back(
@@ -555,6 +574,12 @@ static Type *DecodeFixedType(ArrayRef<Intrinsic::IITDescriptor> &Infos,
   case IITDescriptor::HalfVecArgument:
     return VectorType::getHalfElementsVectorType(
         cast<VectorType>(Tys[D.getArgumentNumber()]));
+  case IITDescriptor::OneThirdVecArgument:
+  case IITDescriptor::OneFifthVecArgument:
+  case IITDescriptor::OneSeventhVecArgument:
+    return VectorType::getOneNthElementsVectorType(
+        cast<VectorType>(Tys[D.getArgumentNumber()]),
+        3 + (D.Kind - IITDescriptor::OneThirdVecArgument) * 2);
   case IITDescriptor::SameVecWidthArgument: {
     Type *EltTy = DecodeFixedType(Infos, Tys, Context);
     Type *Ty = Tys[D.getArgumentNumber()];
@@ -649,20 +674,20 @@ static int lookupLLVMIntrinsicByName(ArrayRef<unsigned> NameOffsetTable,
       // `equal_range` requires the comparison to work with either side being an
       // offset or the value. Detect which kind each side is to set up the
       // compared strings.
-      const char *LHSStr;
+      StringRef LHSStr;
       if constexpr (std::is_integral_v<decltype(LHS)>) {
-        LHSStr = &IntrinsicNameTable[LHS];
+        LHSStr = IntrinsicNameTable[LHS];
       } else {
         LHSStr = LHS;
       }
-      const char *RHSStr;
+      StringRef RHSStr;
       if constexpr (std::is_integral_v<decltype(RHS)>) {
-        RHSStr = &IntrinsicNameTable[RHS];
+        RHSStr = IntrinsicNameTable[RHS];
       } else {
         RHSStr = RHS;
       }
-      return strncmp(LHSStr + CmpStart, RHSStr + CmpStart, CmpEnd - CmpStart) <
-             0;
+      return strncmp(LHSStr.data() + CmpStart, RHSStr.data() + CmpStart,
+                     CmpEnd - CmpStart) < 0;
     };
     LastLow = Low;
     std::tie(Low, High) = std::equal_range(Low, High, Name.data(), Cmp);
@@ -672,7 +697,7 @@ static int lookupLLVMIntrinsicByName(ArrayRef<unsigned> NameOffsetTable,
 
   if (LastLow == NameOffsetTable.end())
     return -1;
-  StringRef NameFound = &IntrinsicNameTable[*LastLow];
+  StringRef NameFound = IntrinsicNameTable[*LastLow];
   if (Name == NameFound ||
       (Name.starts_with(NameFound) && Name[NameFound.size()] == '.'))
     return LastLow - NameOffsetTable.begin();
@@ -716,7 +741,7 @@ Intrinsic::ID Intrinsic::lookupIntrinsicID(StringRef Name) {
 
   // If the intrinsic is not overloaded, require an exact match. If it is
   // overloaded, require either exact or prefix match.
-  const auto MatchSize = strlen(&IntrinsicNameTable[NameOffsetTable[Idx]]);
+  const auto MatchSize = IntrinsicNameTable[NameOffsetTable[Idx]].size();
   assert(Name.size() >= MatchSize && "Expected either exact or prefix match");
   bool IsExactMatch = Name.size() == MatchSize;
   return IsExactMatch || Intrinsic::isOverloaded(ID) ? ID
@@ -931,6 +956,16 @@ matchIntrinsicType(Type *Ty, ArrayRef<Intrinsic::IITDescriptor> &Infos,
     return !isa<VectorType>(ArgTys[D.getArgumentNumber()]) ||
            VectorType::getHalfElementsVectorType(
                cast<VectorType>(ArgTys[D.getArgumentNumber()])) != Ty;
+  case IITDescriptor::OneThirdVecArgument:
+  case IITDescriptor::OneFifthVecArgument:
+  case IITDescriptor::OneSeventhVecArgument:
+    // If this is a forward reference, defer the check for later.
+    if (D.getArgumentNumber() >= ArgTys.size())
+      return IsDeferredCheck || DeferCheck(Ty);
+    return !isa<VectorType>(ArgTys[D.getArgumentNumber()]) ||
+           VectorType::getOneNthElementsVectorType(
+               cast<VectorType>(ArgTys[D.getArgumentNumber()]),
+               3 + (D.Kind - IITDescriptor::OneThirdVecArgument) * 2) != Ty;
   case IITDescriptor::SameVecWidthArgument: {
     if (D.getArgumentNumber() >= ArgTys.size()) {
       // Defer check and subsequent check for the vector element type.
diff --git a/llvm/lib/IR/MDBuilder.cpp b/llvm/lib/IR/MDBuilder.cpp
index 26c8ab9fc36c8..b6aa8844a7eaf 100644
--- a/llvm/lib/IR/MDBuilder.cpp
+++ b/llvm/lib/IR/MDBuilder.cpp
@@ -87,9 +87,9 @@ MDNode *MDBuilder::createFunctionEntryCount(
   return MDNode::get(Context, Ops);
 }
 
-MDNode *MDBuilder::createFunctionSectionPrefix(StringRef Prefix) {
-  return MDNode::get(
-      Context, {createString("function_section_prefix"), createString(Prefix)});
+MDNode *MDBuilder::createGlobalObjectSectionPrefix(StringRef Prefix) {
+  return MDNode::get(Context,
+                     {createString("section_prefix"), createString(Prefix)});
 }
 
 MDNode *MDBuilder::createRange(const APInt &Lo, const APInt &Hi) {
diff --git a/llvm/lib/IR/Value.cpp b/llvm/lib/IR/Value.cpp
index eddb67282fca4..b5a69b9ecdde4 100644
--- a/llvm/lib/IR/Value.cpp
+++ b/llvm/lib/IR/Value.cpp
@@ -714,7 +714,8 @@ const Value *Value::stripPointerCastsForAliasAnalysis() const {
 const Value *Value::stripAndAccumulateConstantOffsets(
     const DataLayout &DL, APInt &Offset, bool AllowNonInbounds,
     bool AllowInvariantGroup,
-    function_ref<bool(Value &, APInt &)> ExternalAnalysis) const {
+    function_ref<bool(Value &, APInt &)> ExternalAnalysis,
+    bool LookThroughIntToPtr) const {
   if (!getType()->isPtrOrPtrVectorTy())
     return this;
 
@@ -775,6 +776,24 @@ const Value *Value::stripAndAccumulateConstantOffsets(
           V = RV;
         if (AllowInvariantGroup && Call->isLaunderOrStripInvariantGroup())
           V = Call->getArgOperand(0);
+    } else if (auto *Int2Ptr = dyn_cast<Operator>(V)) {
+      // Try to accumulate across (inttoptr (add (ptrtoint p), off)).
+      if (!AllowNonInbounds || !LookThroughIntToPtr || !Int2Ptr ||
+          Int2Ptr->getOpcode() != Instruction::IntToPtr ||
+          Int2Ptr->getOperand(0)->getType()->getScalarSizeInBits() != BitWidth)
+        return V;
+
+      auto *Add = dyn_cast<AddOperator>(Int2Ptr->getOperand(0));
+      if (!Add)
+        return V;
+
+      auto *Ptr2Int = dyn_cast<PtrToIntOperator>(Add->getOperand(0));
+      auto *CI = dyn_cast<ConstantInt>(Add->getOperand(1));
+      if (!Ptr2Int || !CI)
+        return V;
+
+      Offset += CI->getValue();
+      V = Ptr2Int->getOperand(0);
     }
     assert(V->getType()->isPtrOrPtrVectorTy() && "Unexpected operand type!");
   } while (Visited.insert(V).second);
diff --git a/llvm/lib/IR/Verifier.cpp b/llvm/lib/IR/Verifier.cpp
index 54de812517438..8432779c107de 100644
--- a/llvm/lib/IR/Verifier.cpp
+++ b/llvm/lib/IR/Verifier.cpp
@@ -6521,8 +6521,10 @@ void Verifier::visitIntrinsicCall(Intrinsic::ID ID, CallBase &Call) {
       const ColorVector &CV = BlockEHFuncletColors.find(CallBB)->second;
       assert(CV.size() > 0 && "Uncolored block");
       for (BasicBlock *ColorFirstBB : CV)
-        if (dyn_cast_or_null<FuncletPadInst>(ColorFirstBB->getFirstNonPHI()))
-          InEHFunclet = true;
+        if (auto It = ColorFirstBB->getFirstNonPHIIt();
+            It != ColorFirstBB->end())
+          if (dyn_cast_or_null<FuncletPadInst>(&*It))
+            InEHFunclet = true;
 
       // Check for funclet operand bundle
       bool HasToken = false;
diff --git a/llvm/lib/Linker/IRMover.cpp b/llvm/lib/Linker/IRMover.cpp
index 43fcfe75ba46b..62e2af4da57bb 100644
--- a/llvm/lib/Linker/IRMover.cpp
+++ b/llvm/lib/Linker/IRMover.cpp
@@ -1244,6 +1244,7 @@ Error IRLinker::linkModuleFlagsMetadata() {
 
   // Check for module flag for updates before do anything.
   UpgradeModuleFlags(*SrcM);
+  UpgradeNVVMAnnotations(*SrcM);
 
   // If the destination module doesn't have module flags yet, then just copy
   // over the source module's flags.
diff --git a/llvm/lib/MC/CMakeLists.txt b/llvm/lib/MC/CMakeLists.txt
index e1d19196c8766..f49f14c848b90 100644
--- a/llvm/lib/MC/CMakeLists.txt
+++ b/llvm/lib/MC/CMakeLists.txt
@@ -1,6 +1,7 @@
 add_llvm_component_library(LLVMMC
   ConstantPools.cpp
   DXContainerPSVInfo.cpp
+  DXContainerRootSignature.cpp
   ELFObjectWriter.cpp
   GOFFObjectWriter.cpp
   MCAsmBackend.cpp
diff --git a/llvm/lib/MC/DXContainerRootSignature.cpp b/llvm/lib/MC/DXContainerRootSignature.cpp
new file mode 100644
index 0000000000000..000d23f24d241
--- /dev/null
+++ b/llvm/lib/MC/DXContainerRootSignature.cpp
@@ -0,0 +1,23 @@
+//===- llvm/MC/DXContainerRootSignature.cpp - RootSignature -*- C++ -*-=======//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/MC/DXContainerRootSignature.h"
+#include "llvm/Support/EndianStream.h"
+
+using namespace llvm;
+using namespace llvm::mcdxbc;
+
+void RootSignatureHeader::write(raw_ostream &OS) {
+
+  support::endian::write(OS, Version, llvm::endianness::little);
+  support::endian::write(OS, NumParameters, llvm::endianness::little);
+  support::endian::write(OS, RootParametersOffset, llvm::endianness::little);
+  support::endian::write(OS, NumStaticSamplers, llvm::endianness::little);
+  support::endian::write(OS, StaticSamplersOffset, llvm::endianness::little);
+  support::endian::write(OS, Flags, llvm::endianness::little);
+}
diff --git a/llvm/lib/MC/ELFObjectWriter.cpp b/llvm/lib/MC/ELFObjectWriter.cpp
index 5f586fe19a5bb..68e7f1785fa23 100644
--- a/llvm/lib/MC/ELFObjectWriter.cpp
+++ b/llvm/lib/MC/ELFObjectWriter.cpp
@@ -71,9 +71,12 @@ STATISTIC(StrtabBytes, "Total size of SHT_STRTAB sections");
 STATISTIC(SymtabBytes, "Total size of SHT_SYMTAB sections");
 STATISTIC(RelocationBytes, "Total size of relocation sections");
 STATISTIC(DynsymBytes, "Total size of SHT_DYNSYM sections");
-STATISTIC(DebugBytes, "Total size of debug info sections");
+STATISTIC(
+    DebugBytes,
+    "Total size of debug info sections (not including those written to .dwo)");
 STATISTIC(UnwindBytes, "Total size of unwind sections");
 STATISTIC(OtherBytes, "Total size of uncategorized sections");
+STATISTIC(DwoBytes, "Total size of sections written to .dwo file");
 
 } // namespace stats
 
@@ -969,7 +972,9 @@ void ELFWriter::writeSectionHeaders(const MCAssembler &Asm) {
       return Section->getFlags() & Flag;
     };
 
-    if (Section->getName().starts_with(".debug")) {
+    if (Mode == DwoOnly) {
+      stats::DwoBytes += Size;
+    } else if (Section->getName().starts_with(".debug")) {
       stats::DebugBytes += Size;
     } else if (Section->getName().starts_with(".eh_frame")) {
       stats::UnwindBytes += Size;
diff --git a/llvm/lib/MC/MCAsmStreamer.cpp b/llvm/lib/MC/MCAsmStreamer.cpp
index dd8058c6d5cd8..4db405051f7f3 100644
--- a/llvm/lib/MC/MCAsmStreamer.cpp
+++ b/llvm/lib/MC/MCAsmStreamer.cpp
@@ -191,7 +191,6 @@ class MCAsmStreamer final : public MCStreamer {
   void emitDarwinTargetVariantBuildVersion(unsigned Platform, unsigned Major,
                                            unsigned Minor, unsigned Update,
                                            VersionTuple SDKVersion) override;
-  void emitThumbFunc(MCSymbol *Func) override;
 
   void emitAssignment(MCSymbol *Symbol, const MCExpr *Value) override;
   void emitConditionalAssignment(MCSymbol *Symbol,
@@ -398,6 +397,8 @@ class MCAsmStreamer final : public MCStreamer {
                          SMLoc Loc) override;
   void emitWinCFIPushFrame(bool Code, SMLoc Loc) override;
   void emitWinCFIEndProlog(SMLoc Loc) override;
+  void emitWinCFIBeginEpilogue(SMLoc Loc) override;
+  void emitWinCFIEndEpilogue(SMLoc Loc) override;
 
   void emitWinEHHandler(const MCSymbol *Sym, bool Unwind, bool Except,
                         SMLoc Loc) override;
@@ -695,18 +696,6 @@ void MCAsmStreamer::emitDarwinTargetVariantBuildVersion(
   emitBuildVersion(Platform, Major, Minor, Update, SDKVersion);
 }
 
-void MCAsmStreamer::emitThumbFunc(MCSymbol *Func) {
-  // This needs to emit to a temporary string to get properly quoted
-  // MCSymbols when they have spaces in them.
-  OS << "\t.thumb_func";
-  // Only Mach-O hasSubsectionsViaSymbols()
-  if (MAI->hasSubsectionsViaSymbols()) {
-    OS << '\t';
-    Func->print(OS, MAI);
-  }
-  EmitEOL();
-}
-
 void MCAsmStreamer::emitAssignment(MCSymbol *Symbol, const MCExpr *Value) {
   // Do not emit a .set on inlined target assignments.
   bool EmitSet = true;
@@ -2348,6 +2337,20 @@ void MCAsmStreamer::emitWinCFIEndProlog(SMLoc Loc) {
   EmitEOL();
 }
 
+void MCAsmStreamer::emitWinCFIBeginEpilogue(SMLoc Loc) {
+  MCStreamer::emitWinCFIBeginEpilogue(Loc);
+
+  OS << "\t.seh_startepilogue";
+  EmitEOL();
+}
+
+void MCAsmStreamer::emitWinCFIEndEpilogue(SMLoc Loc) {
+  MCStreamer::emitWinCFIEndEpilogue(Loc);
+
+  OS << "\t.seh_endepilogue";
+  EmitEOL();
+}
+
 void MCAsmStreamer::emitCGProfileEntry(const MCSymbolRefExpr *From,
                                        const MCSymbolRefExpr *To,
                                        uint64_t Count) {
diff --git a/llvm/lib/MC/MCContext.cpp b/llvm/lib/MC/MCContext.cpp
index 46222fcaa5b15..335febde3687c 100644
--- a/llvm/lib/MC/MCContext.cpp
+++ b/llvm/lib/MC/MCContext.cpp
@@ -85,7 +85,7 @@ MCContext::MCContext(const Triple &TheTriple, const MCAsmInfo *mai,
     Env = IsMachO;
     break;
   case Triple::COFF:
-    if (!TheTriple.isOSWindows() && !TheTriple.isUEFI())
+    if (!TheTriple.isOSWindowsOrUEFI())
       report_fatal_error(
           "Cannot initialize MC for non-Windows COFF object files.");
 
diff --git a/llvm/lib/MC/MCDisassembler/MCDisassembler.cpp b/llvm/lib/MC/MCDisassembler/MCDisassembler.cpp
index 6aa4b0e4fcb99..0dfa17eb1346f 100644
--- a/llvm/lib/MC/MCDisassembler/MCDisassembler.cpp
+++ b/llvm/lib/MC/MCDisassembler/MCDisassembler.cpp
@@ -29,17 +29,21 @@ bool MCDisassembler::tryAddingSymbolicOperand(MCInst &Inst, int64_t Value,
                                               uint64_t Address, bool IsBranch,
                                               uint64_t Offset, uint64_t OpSize,
                                               uint64_t InstSize) const {
-  if (Symbolizer)
+  if (Symbolizer) {
+    assert(CommentStream && "CommentStream is not set.");
     return Symbolizer->tryAddingSymbolicOperand(Inst, *CommentStream, Value,
                                                 Address, IsBranch, Offset,
                                                 OpSize, InstSize);
+  }
   return false;
 }
 
 void MCDisassembler::tryAddingPcLoadReferenceComment(int64_t Value,
                                                      uint64_t Address) const {
-  if (Symbolizer)
+  if (Symbolizer) {
+    assert(CommentStream && "CommentStream is not set.");
     Symbolizer->tryAddingPcLoadReferenceComment(*CommentStream, Value, Address);
+  }
 }
 
 void MCDisassembler::setSymbolizer(std::unique_ptr<MCSymbolizer> Symzer) {
diff --git a/llvm/lib/MC/MCELFStreamer.cpp b/llvm/lib/MC/MCELFStreamer.cpp
index 282c82198507d..65bb88eb6f32a 100644
--- a/llvm/lib/MC/MCELFStreamer.cpp
+++ b/llvm/lib/MC/MCELFStreamer.cpp
@@ -616,10 +616,6 @@ void MCELFStreamer::finishImpl() {
   this->MCObjectStreamer::finishImpl();
 }
 
-void MCELFStreamer::emitThumbFunc(MCSymbol *Func) {
-  llvm_unreachable("Generic ELF doesn't support this directive");
-}
-
 void MCELFStreamer::emitSymbolDesc(MCSymbol *Symbol, unsigned DescValue) {
   llvm_unreachable("ELF doesn't support this directive");
 }
diff --git a/llvm/lib/MC/MCMachOStreamer.cpp b/llvm/lib/MC/MCMachOStreamer.cpp
index aa79d1bc5b869..25352a0e1d5c9 100644
--- a/llvm/lib/MC/MCMachOStreamer.cpp
+++ b/llvm/lib/MC/MCMachOStreamer.cpp
@@ -99,7 +99,6 @@ class MCMachOStreamer : public MCObjectStreamer {
   void emitDarwinTargetVariantBuildVersion(unsigned Platform, unsigned Major,
                                            unsigned Minor, unsigned Update,
                                            VersionTuple SDKVersion) override;
-  void emitThumbFunc(MCSymbol *Func) override;
   bool emitSymbolAttribute(MCSymbol *Symbol, MCSymbolAttr Attribute) override;
   void emitSymbolDesc(MCSymbol *Symbol, unsigned DescValue) override;
   void emitCommonSymbol(MCSymbol *Symbol, uint64_t Size,
@@ -269,13 +268,6 @@ void MCMachOStreamer::emitDarwinTargetVariantBuildVersion(
                                            Minor, Update, SDKVersion);
 }
 
-void MCMachOStreamer::emitThumbFunc(MCSymbol *Symbol) {
-  // Remember that the function is a thumb function. Fixup and relocation
-  // values will need adjusted.
-  getAssembler().setIsThumbFunc(Symbol);
-  cast<MCSymbolMachO>(Symbol)->setThumbFunc();
-}
-
 bool MCMachOStreamer::emitSymbolAttribute(MCSymbol *Sym,
                                           MCSymbolAttr Attribute) {
   MCSymbolMachO *Symbol = cast<MCSymbolMachO>(Sym);
diff --git a/llvm/lib/MC/MCParser/COFFAsmParser.cpp b/llvm/lib/MC/MCParser/COFFAsmParser.cpp
index dd5ce9964a194..4618e5675e47b 100644
--- a/llvm/lib/MC/MCParser/COFFAsmParser.cpp
+++ b/llvm/lib/MC/MCParser/COFFAsmParser.cpp
@@ -92,6 +92,10 @@ class COFFAsmParser : public MCAsmParserExtension {
         ".seh_stackalloc");
     addDirectiveHandler<&COFFAsmParser::parseSEHDirectiveEndProlog>(
         ".seh_endprologue");
+    addDirectiveHandler<&COFFAsmParser::ParseSEHDirectiveBeginEpilog>(
+        ".seh_startepilogue");
+    addDirectiveHandler<&COFFAsmParser::ParseSEHDirectiveEndEpilog>(
+        ".seh_endepilogue");
   }
 
   bool parseSectionDirectiveText(StringRef, SMLoc) {
@@ -141,6 +145,8 @@ class COFFAsmParser : public MCAsmParserExtension {
   bool parseSEHDirectiveHandlerData(StringRef, SMLoc);
   bool parseSEHDirectiveAllocStack(StringRef, SMLoc);
   bool parseSEHDirectiveEndProlog(StringRef, SMLoc);
+  bool ParseSEHDirectiveBeginEpilog(StringRef, SMLoc);
+  bool ParseSEHDirectiveEndEpilog(StringRef, SMLoc);
 
   bool parseAtUnwindOrAtExcept(bool &unwind, bool &except);
   bool parseDirectiveSymbolAttribute(StringRef Directive, SMLoc);
@@ -749,6 +755,18 @@ bool COFFAsmParser::parseSEHDirectiveEndProlog(StringRef, SMLoc Loc) {
   return false;
 }
 
+bool COFFAsmParser::ParseSEHDirectiveBeginEpilog(StringRef, SMLoc Loc) {
+  Lex();
+  getStreamer().emitWinCFIBeginEpilogue(Loc);
+  return false;
+}
+
+bool COFFAsmParser::ParseSEHDirectiveEndEpilog(StringRef, SMLoc Loc) {
+  Lex();
+  getStreamer().emitWinCFIEndEpilogue(Loc);
+  return false;
+}
+
 bool COFFAsmParser::parseAtUnwindOrAtExcept(bool &unwind, bool &except) {
   StringRef identifier;
   if (getLexer().isNot(AsmToken::At) && getLexer().isNot(AsmToken::Percent))
diff --git a/llvm/lib/MC/MCStreamer.cpp b/llvm/lib/MC/MCStreamer.cpp
index e690723c0e502..dd04e1e30720a 100644
--- a/llvm/lib/MC/MCStreamer.cpp
+++ b/llvm/lib/MC/MCStreamer.cpp
@@ -1013,6 +1013,36 @@ void MCStreamer::emitWinCFIEndProlog(SMLoc Loc) {
   CurFrame->PrologEnd = Label;
 }
 
+void MCStreamer::emitWinCFIBeginEpilogue(SMLoc Loc) {
+  WinEH::FrameInfo *CurFrame = EnsureValidWinFrameInfo(Loc);
+  if (!CurFrame)
+    return;
+
+  if (!CurFrame->PrologEnd)
+    return getContext().reportError(
+        Loc, "starting epilogue (.seh_startepilogue) before prologue has ended "
+             "(.seh_endprologue) in " +
+                 CurFrame->Function->getName());
+
+  InEpilogCFI = true;
+  CurrentEpilog = emitCFILabel();
+}
+
+void MCStreamer::emitWinCFIEndEpilogue(SMLoc Loc) {
+  WinEH::FrameInfo *CurFrame = EnsureValidWinFrameInfo(Loc);
+  if (!CurFrame)
+    return;
+
+  if (!InEpilogCFI)
+    return getContext().reportError(Loc, "Stray .seh_endepilogue in " +
+                                             CurFrame->Function->getName());
+
+  InEpilogCFI = false;
+  MCSymbol *Label = emitCFILabel();
+  CurFrame->EpilogMap[CurrentEpilog].End = Label;
+  CurrentEpilog = nullptr;
+}
+
 void MCStreamer::emitCOFFSafeSEH(MCSymbol const *Symbol) {}
 
 void MCStreamer::emitCOFFSymbolIndex(MCSymbol const *Symbol) {}
@@ -1199,7 +1229,6 @@ void MCStreamer::emitAbsoluteSymbolDiffAsULEB128(const MCSymbol *Hi,
 }
 
 void MCStreamer::emitAssemblerFlag(MCAssemblerFlag Flag) {}
-void MCStreamer::emitThumbFunc(MCSymbol *Func) {}
 void MCStreamer::emitSymbolDesc(MCSymbol *Symbol, unsigned DescValue) {}
 void MCStreamer::beginCOFFSymbolDef(const MCSymbol *Symbol) {
   llvm_unreachable("this directive only supported on COFF targets");
diff --git a/llvm/lib/MC/MCWasmStreamer.cpp b/llvm/lib/MC/MCWasmStreamer.cpp
index 8560f0ebeb1d8..d4806b3a5ecb6 100644
--- a/llvm/lib/MC/MCWasmStreamer.cpp
+++ b/llvm/lib/MC/MCWasmStreamer.cpp
@@ -238,10 +238,6 @@ void MCWasmStreamer::fixSymbolsInTLSFixups(const MCExpr *expr) {
   }
 }
 
-void MCWasmStreamer::emitThumbFunc(MCSymbol *Func) {
-  llvm_unreachable("Generic Wasm doesn't support this directive");
-}
-
 void MCWasmStreamer::emitSymbolDesc(MCSymbol *Symbol, unsigned DescValue) {
   llvm_unreachable("Wasm doesn't support this directive");
 }
diff --git a/llvm/lib/MC/MCWinCOFFStreamer.cpp b/llvm/lib/MC/MCWinCOFFStreamer.cpp
index 8fd46bc8b0255..3d7eb525392e1 100644
--- a/llvm/lib/MC/MCWinCOFFStreamer.cpp
+++ b/llvm/lib/MC/MCWinCOFFStreamer.cpp
@@ -208,10 +208,6 @@ void MCWinCOFFStreamer::emitAssemblerFlag(MCAssemblerFlag Flag) {
   }
 }
 
-void MCWinCOFFStreamer::emitThumbFunc(MCSymbol *Func) {
-  llvm_unreachable("not implemented");
-}
-
 bool MCWinCOFFStreamer::emitSymbolAttribute(MCSymbol *S,
                                             MCSymbolAttr Attribute) {
   auto *Symbol = cast<MCSymbolCOFF>(S);
diff --git a/llvm/lib/MC/TargetRegistry.cpp b/llvm/lib/MC/TargetRegistry.cpp
index e1879f97aa567..a9e33c8349bdc 100644
--- a/llvm/lib/MC/TargetRegistry.cpp
+++ b/llvm/lib/MC/TargetRegistry.cpp
@@ -31,8 +31,7 @@ MCStreamer *Target::createMCObjectStreamer(
   case Triple::UnknownObjectFormat:
     llvm_unreachable("Unknown object format");
   case Triple::COFF:
-    assert((T.isOSWindows() || T.isUEFI()) &&
-           "only Windows and UEFI COFF are supported");
+    assert(T.isOSWindowsOrUEFI() && "only Windows and UEFI COFF are supported");
     S = COFFStreamerCtorFn(Ctx, std::move(TAB), std::move(OW),
                            std::move(Emitter));
     break;
diff --git a/llvm/lib/MCA/InstrBuilder.cpp b/llvm/lib/MCA/InstrBuilder.cpp
index 9304e6c57a385..2cb1908695308 100644
--- a/llvm/lib/MCA/InstrBuilder.cpp
+++ b/llvm/lib/MCA/InstrBuilder.cpp
@@ -735,7 +735,7 @@ InstrBuilder::createInstruction(const MCInst &MCI,
       // Skip non-register operands.
       if (!Op.isReg())
         continue;
-      RegID = Op.getReg();
+      RegID = Op.getReg().id();
     } else {
       // Implicit read.
       RegID = RD.RegisterID;
@@ -800,8 +800,8 @@ InstrBuilder::createInstruction(const MCInst &MCI,
   unsigned WriteIndex = 0;
   Idx = 0U;
   for (const WriteDescriptor &WD : D.Writes) {
-    RegID = WD.isImplicitWrite() ? MCRegister(WD.RegisterID)
-                                 : MCI.getOperand(WD.OpIndex).getReg();
+    RegID = WD.isImplicitWrite() ? WD.RegisterID
+                                 : MCI.getOperand(WD.OpIndex).getReg().id();
     // Check if this is a optional definition that references NoReg or a write
     // to a constant register.
     if ((WD.IsOptionalDef && !RegID) || MRI.isConstant(RegID)) {
diff --git a/llvm/lib/ObjCopy/ELF/ELFObjcopy.cpp b/llvm/lib/ObjCopy/ELF/ELFObjcopy.cpp
index 5aa0079f3fbc7..9c78f7433ad33 100644
--- a/llvm/lib/ObjCopy/ELF/ELFObjcopy.cpp
+++ b/llvm/lib/ObjCopy/ELF/ELFObjcopy.cpp
@@ -186,27 +186,28 @@ static std::unique_ptr<Writer> createWriter(const CommonConfig &Config,
 }
 
 static Error dumpSectionToFile(StringRef SecName, StringRef Filename,
-                               Object &Obj) {
+                               StringRef InputFilename, Object &Obj) {
   for (auto &Sec : Obj.sections()) {
     if (Sec.Name == SecName) {
       if (Sec.Type == SHT_NOBITS)
-        return createStringError(object_error::parse_failed,
-                                 "cannot dump section '%s': it has no contents",
-                                 SecName.str().c_str());
+        return createFileError(InputFilename, object_error::parse_failed,
+                               "cannot dump section '%s': it has no contents",
+                               SecName.str().c_str());
       Expected<std::unique_ptr<FileOutputBuffer>> BufferOrErr =
           FileOutputBuffer::create(Filename, Sec.OriginalData.size());
       if (!BufferOrErr)
-        return BufferOrErr.takeError();
+        return createFileError(Filename, BufferOrErr.takeError());
       std::unique_ptr<FileOutputBuffer> Buf = std::move(*BufferOrErr);
       std::copy(Sec.OriginalData.begin(), Sec.OriginalData.end(),
                 Buf->getBufferStart());
       if (Error E = Buf->commit())
-        return E;
+        return createFileError(Filename, std::move(E));
       return Error::success();
     }
   }
-  return createStringError(object_error::parse_failed, "section '%s' not found",
-                           SecName.str().c_str());
+
+  return createFileError(InputFilename, object_error::parse_failed,
+                         "section '%s' not found", SecName.str().c_str());
 }
 
 Error Object::compressOrDecompressSections(const CommonConfig &Config) {
@@ -798,7 +799,8 @@ static Error handleArgs(const CommonConfig &Config, const ELFConfig &ELFConfig,
     StringRef SectionName;
     StringRef FileName;
     std::tie(SectionName, FileName) = Flag.split('=');
-    if (Error E = dumpSectionToFile(SectionName, FileName, Obj))
+    if (Error E =
+            dumpSectionToFile(SectionName, FileName, Config.InputFilename, Obj))
       return E;
   }
 
@@ -807,10 +809,10 @@ static Error handleArgs(const CommonConfig &Config, const ELFConfig &ELFConfig,
   // us to avoid reporting the inappropriate errors about removing symbols
   // named in relocations.
   if (Error E = replaceAndRemoveSections(Config, ELFConfig, Obj))
-    return E;
+    return createFileError(Config.InputFilename, std::move(E));
 
   if (Error E = updateAndRemoveSymbols(Config, ELFConfig, Obj))
-    return E;
+    return createFileError(Config.InputFilename, std::move(E));
 
   if (!Config.SetSectionAlignment.empty()) {
     for (SectionBase &Sec : Obj.sections()) {
@@ -826,8 +828,8 @@ static Error handleArgs(const CommonConfig &Config, const ELFConfig &ELFConfig,
         if (Config.ChangeSectionLMAValAll > 0 &&
             Seg.PAddr > std::numeric_limits<uint64_t>::max() -
                             Config.ChangeSectionLMAValAll) {
-          return createStringError(
-              errc::invalid_argument,
+          return createFileError(
+              Config.InputFilename, errc::invalid_argument,
               "address 0x" + Twine::utohexstr(Seg.PAddr) +
                   " cannot be increased by 0x" +
                   Twine::utohexstr(Config.ChangeSectionLMAValAll) +
@@ -835,8 +837,8 @@ static Error handleArgs(const CommonConfig &Config, const ELFConfig &ELFConfig,
         } else if (Config.ChangeSectionLMAValAll < 0 &&
                    Seg.PAddr < std::numeric_limits<uint64_t>::min() -
                                    Config.ChangeSectionLMAValAll) {
-          return createStringError(
-              errc::invalid_argument,
+          return createFileError(
+              Config.InputFilename, errc::invalid_argument,
               "address 0x" + Twine::utohexstr(Seg.PAddr) +
                   " cannot be decreased by 0x" +
                   Twine::utohexstr(std::abs(Config.ChangeSectionLMAValAll)) +
@@ -849,10 +851,9 @@ static Error handleArgs(const CommonConfig &Config, const ELFConfig &ELFConfig,
 
   if (!Config.ChangeSectionAddress.empty()) {
     if (Obj.Type != ELF::ET_REL)
-      return createStringError(
-          object_error::invalid_file_type,
+      return createFileError(
+          Config.InputFilename, object_error::invalid_file_type,
           "cannot change section address in a non-relocatable file");
-
     StringMap<AddressUpdate> SectionsToUpdateAddress;
     for (const SectionPatternAddressUpdate &PatternUpdate :
          make_range(Config.ChangeSectionAddress.rbegin(),
@@ -863,8 +864,8 @@ static Error handleArgs(const CommonConfig &Config, const ELFConfig &ELFConfig,
                 .second) {
           if (PatternUpdate.Update.Kind == AdjustKind::Subtract &&
               Sec.Addr < PatternUpdate.Update.Value) {
-            return createStringError(
-                errc::invalid_argument,
+            return createFileError(
+                Config.InputFilename, errc::invalid_argument,
                 "address 0x" + Twine::utohexstr(Sec.Addr) +
                     " cannot be decreased by 0x" +
                     Twine::utohexstr(PatternUpdate.Update.Value) +
@@ -873,8 +874,8 @@ static Error handleArgs(const CommonConfig &Config, const ELFConfig &ELFConfig,
           if (PatternUpdate.Update.Kind == AdjustKind::Add &&
               Sec.Addr > std::numeric_limits<uint64_t>::max() -
                              PatternUpdate.Update.Value) {
-            return createStringError(
-                errc::invalid_argument,
+            return createFileError(
+                Config.InputFilename, errc::invalid_argument,
                 "address 0x" + Twine::utohexstr(Sec.Addr) +
                     " cannot be increased by 0x" +
                     Twine::utohexstr(PatternUpdate.Update.Value) +
@@ -909,7 +910,7 @@ static Error handleArgs(const CommonConfig &Config, const ELFConfig &ELFConfig,
   if (!ELFConfig.NotesToRemove.empty()) {
     if (Error Err =
             removeNotes(Obj, E, ELFConfig.NotesToRemove, Config.ErrorCallback))
-      return Err;
+      return createFileError(Config.InputFilename, std::move(Err));
   }
 
   for (const NewSectionInfo &AddedSection : Config.AddSection) {
@@ -924,7 +925,7 @@ static Error handleArgs(const CommonConfig &Config, const ELFConfig &ELFConfig,
       return Error::success();
     };
     if (Error E = handleUserSection(AddedSection, AddSection))
-      return E;
+      return createFileError(Config.InputFilename, std::move(E));
   }
 
   for (const NewSectionInfo &NewSection : Config.UpdateSection) {
@@ -932,7 +933,7 @@ static Error handleArgs(const CommonConfig &Config, const ELFConfig &ELFConfig,
       return Obj.updateSection(Name, Data);
     };
     if (Error E = handleUserSection(NewSection, UpdateSection))
-      return E;
+      return createFileError(Config.InputFilename, std::move(E));
   }
 
   if (!Config.AddGnuDebugLink.empty())
@@ -943,7 +944,7 @@ static Error handleArgs(const CommonConfig &Config, const ELFConfig &ELFConfig,
   // before adding new symbols.
   if (!Obj.SymbolTable && !Config.SymbolsToAdd.empty())
     if (Error E = Obj.addNewSymbolTable())
-      return E;
+      return createFileError(Config.InputFilename, std::move(E));
 
   for (const NewSymbolInfo &SI : Config.SymbolsToAdd)
     addSymbol(Obj, SI, ELFConfig.NewSymbolVisibility);
@@ -955,7 +956,7 @@ static Error handleArgs(const CommonConfig &Config, const ELFConfig &ELFConfig,
       if (Iter != Config.SetSectionFlags.end()) {
         const SectionFlagsUpdate &SFU = Iter->second;
         if (Error E = setSectionFlagsAndType(Sec, SFU.NewFlags, Obj.Machine))
-          return E;
+          return createFileError(Config.InputFilename, std::move(E));
       }
       auto It2 = Config.SetSectionType.find(Sec.Name);
       if (It2 != Config.SetSectionType.end())
@@ -974,7 +975,7 @@ static Error handleArgs(const CommonConfig &Config, const ELFConfig &ELFConfig,
         Sec.Name = std::string(SR.NewName);
         if (SR.NewFlags) {
           if (Error E = setSectionFlagsAndType(Sec, *SR.NewFlags, Obj.Machine))
-            return E;
+            return createFileError(Config.InputFilename, std::move(E));
         }
         RenamedSections.insert(&Sec);
       } else if (RelocSec && !(Sec.Flags & SHF_ALLOC))
@@ -1091,7 +1092,7 @@ Error objcopy::elf::executeObjcopyOnBinary(const CommonConfig &Config,
                                     : getOutputElfType(In);
 
   if (Error E = handleArgs(Config, ELFConfig, OutputElfType, **Obj))
-    return createFileError(Config.InputFilename, std::move(E));
+    return E;
 
   if (Error E = writeOutput(Config, **Obj, Out, OutputElfType))
     return createFileError(Config.InputFilename, std::move(E));
diff --git a/llvm/lib/ObjCopy/MachO/MachOObjcopy.cpp b/llvm/lib/ObjCopy/MachO/MachOObjcopy.cpp
index a188425b283fa..682edffc84f34 100644
--- a/llvm/lib/ObjCopy/MachO/MachOObjcopy.cpp
+++ b/llvm/lib/ObjCopy/MachO/MachOObjcopy.cpp
@@ -306,25 +306,25 @@ static Error processLoadCommands(const MachOConfig &MachOConfig, Object &Obj) {
 }
 
 static Error dumpSectionToFile(StringRef SecName, StringRef Filename,
-                               Object &Obj) {
+                               StringRef InputFilename, Object &Obj) {
   for (LoadCommand &LC : Obj.LoadCommands)
     for (const std::unique_ptr<Section> &Sec : LC.Sections) {
       if (Sec->CanonicalName == SecName) {
         Expected<std::unique_ptr<FileOutputBuffer>> BufferOrErr =
             FileOutputBuffer::create(Filename, Sec->Content.size());
         if (!BufferOrErr)
-          return BufferOrErr.takeError();
+          return createFileError(Filename, BufferOrErr.takeError());
         std::unique_ptr<FileOutputBuffer> Buf = std::move(*BufferOrErr);
         llvm::copy(Sec->Content, Buf->getBufferStart());
 
         if (Error E = Buf->commit())
-          return E;
+          return createFileError(Filename, std::move(E));
         return Error::success();
       }
     }
 
-  return createStringError(object_error::parse_failed, "section '%s' not found",
-                           SecName.str().c_str());
+  return createFileError(InputFilename, object_error::parse_failed,
+                         "section '%s' not found", SecName.str().c_str());
 }
 
 static Error addSection(const NewSectionInfo &NewSection, Object &Obj) {
@@ -426,12 +426,13 @@ static Error handleArgs(const CommonConfig &Config,
     StringRef SectionName;
     StringRef FileName;
     std::tie(SectionName, FileName) = Flag.split('=');
-    if (Error E = dumpSectionToFile(SectionName, FileName, Obj))
+    if (Error E =
+            dumpSectionToFile(SectionName, FileName, Config.InputFilename, Obj))
       return E;
   }
 
   if (Error E = removeSections(Config, Obj))
-    return E;
+    return createFileError(Config.InputFilename, std::move(E));
 
   // Mark symbols to determine which symbols are still needed.
   if (Config.StripAll)
@@ -446,20 +447,20 @@ static Error handleArgs(const CommonConfig &Config,
 
   for (const NewSectionInfo &NewSection : Config.AddSection) {
     if (Error E = isValidMachOCannonicalName(NewSection.SectionName))
-      return E;
+      return createFileError(Config.InputFilename, std::move(E));
     if (Error E = addSection(NewSection, Obj))
-      return E;
+      return createFileError(Config.InputFilename, std::move(E));
   }
 
   for (const NewSectionInfo &NewSection : Config.UpdateSection) {
     if (Error E = isValidMachOCannonicalName(NewSection.SectionName))
-      return E;
+      return createFileError(Config.InputFilename, std::move(E));
     if (Error E = updateSection(NewSection, Obj))
-      return E;
+      return createFileError(Config.InputFilename, std::move(E));
   }
 
   if (Error E = processLoadCommands(MachOConfig, Obj))
-    return E;
+    return createFileError(Config.InputFilename, std::move(E));
 
   return Error::success();
 }
@@ -479,7 +480,7 @@ Error objcopy::macho::executeObjcopyOnBinary(const CommonConfig &Config,
                              Config.InputFilename.str().c_str());
 
   if (Error E = handleArgs(Config, MachOConfig, **O))
-    return createFileError(Config.InputFilename, std::move(E));
+    return E;
 
   // Page size used for alignment of segment sizes in Mach-O executables and
   // dynamic libraries.
diff --git a/llvm/lib/ObjCopy/wasm/WasmObjcopy.cpp b/llvm/lib/ObjCopy/wasm/WasmObjcopy.cpp
index cf3d884bee3bd..57fd0f5ad233c 100644
--- a/llvm/lib/ObjCopy/wasm/WasmObjcopy.cpp
+++ b/llvm/lib/ObjCopy/wasm/WasmObjcopy.cpp
@@ -38,23 +38,23 @@ static bool isCommentSection(const Section &Sec) {
 }
 
 static Error dumpSectionToFile(StringRef SecName, StringRef Filename,
-                               Object &Obj) {
+                               StringRef InputFilename, Object &Obj) {
   for (const Section &Sec : Obj.Sections) {
     if (Sec.Name == SecName) {
       ArrayRef<uint8_t> Contents = Sec.Contents;
       Expected<std::unique_ptr<FileOutputBuffer>> BufferOrErr =
           FileOutputBuffer::create(Filename, Contents.size());
       if (!BufferOrErr)
-        return BufferOrErr.takeError();
+        return createFileError(Filename, BufferOrErr.takeError());
       std::unique_ptr<FileOutputBuffer> Buf = std::move(*BufferOrErr);
       std::copy(Contents.begin(), Contents.end(), Buf->getBufferStart());
       if (Error E = Buf->commit())
-        return E;
+        return createFileError(Filename, std::move(E));
       return Error::success();
     }
   }
-  return createStringError(errc::invalid_argument, "section '%s' not found",
-                           SecName.str().c_str());
+  return createFileError(Filename, errc::invalid_argument,
+                         "section '%s' not found", SecName.str().c_str());
 }
 
 static void removeSections(const CommonConfig &Config, Object &Obj) {
@@ -115,8 +115,9 @@ static Error handleArgs(const CommonConfig &Config, Object &Obj) {
     StringRef SecName;
     StringRef FileName;
     std::tie(SecName, FileName) = Flag.split("=");
-    if (Error E = dumpSectionToFile(SecName, FileName, Obj))
-      return createFileError(FileName, std::move(E));
+    if (Error E =
+            dumpSectionToFile(SecName, FileName, Config.InputFilename, Obj))
+      return E;
   }
 
   removeSections(Config, Obj);
diff --git a/llvm/lib/Object/DXContainer.cpp b/llvm/lib/Object/DXContainer.cpp
index 3b1a6203a1f8f..f28b096008b2f 100644
--- a/llvm/lib/Object/DXContainer.cpp
+++ b/llvm/lib/Object/DXContainer.cpp
@@ -10,6 +10,7 @@
 #include "llvm/BinaryFormat/DXContainer.h"
 #include "llvm/Object/Error.h"
 #include "llvm/Support/Alignment.h"
+#include "llvm/Support/Endian.h"
 #include "llvm/Support/FormatVariadic.h"
 
 using namespace llvm;
@@ -92,6 +93,15 @@ Error DXContainer::parseHash(StringRef Part) {
   return Error::success();
 }
 
+Error DXContainer::parseRootSignature(StringRef Part) {
+  if (RootSignature)
+    return parseFailed("More than one RTS0 part is present in the file");
+  RootSignature = DirectX::RootSignature();
+  if (Error Err = RootSignature->parse(Part))
+    return Err;
+  return Error::success();
+}
+
 Error DXContainer::parsePSVInfo(StringRef Part) {
   if (PSVInfo)
     return parseFailed("More than one PSV0 part is present in the file");
@@ -193,6 +203,10 @@ Error DXContainer::parsePartOffsets() {
       break;
     case dxbc::PartType::Unknown:
       break;
+    case dxbc::PartType::RTS0:
+      if (Error Err = parseRootSignature(PartData))
+        return Err;
+      break;
     }
   }
 
@@ -228,6 +242,53 @@ void DXContainer::PartIterator::updateIteratorImpl(const uint32_t Offset) {
   IteratorState.Offset = Offset;
 }
 
+Error DirectX::RootSignature::parse(StringRef Data) {
+  const char *Current = Data.begin();
+
+  // Root Signature headers expects 6 integers to be present.
+  if (Data.size() < 6 * sizeof(uint32_t))
+    return parseFailed(
+        "Invalid root signature, insufficient space for header.");
+
+  uint32_t VValue =
+      support::endian::read<uint32_t, llvm::endianness::little>(Current);
+  Current += sizeof(uint32_t);
+
+  Expected<uint32_t> MaybeVersion =
+      dxbc::RootSignatureValidations::validateVersion(VValue);
+  if (Error E = MaybeVersion.takeError())
+    return E;
+  Version = MaybeVersion.get();
+
+  NumParameters =
+      support::endian::read<uint32_t, llvm::endianness::little>(Current);
+  Current += sizeof(uint32_t);
+
+  RootParametersOffset =
+      support::endian::read<uint32_t, llvm::endianness::little>(Current);
+  Current += sizeof(uint32_t);
+
+  NumStaticSamplers =
+      support::endian::read<uint32_t, llvm::endianness::little>(Current);
+  Current += sizeof(uint32_t);
+
+  StaticSamplersOffset =
+      support::endian::read<uint32_t, llvm::endianness::little>(Current);
+  Current += sizeof(uint32_t);
+
+  uint32_t FValue =
+      support::endian::read<uint32_t, llvm::endianness::little>(Current);
+  Current += sizeof(uint32_t);
+
+  Expected<uint32_t> MaybeFlag =
+      dxbc::RootSignatureValidations::validateRootFlag(FValue);
+  if (Error E = MaybeFlag.takeError())
+    return E;
+  Flags = MaybeFlag.get();
+
+  return Error::success();
+}
+
 Error DirectX::PSVRuntimeInfo::parse(uint16_t ShaderKind) {
   Triple::EnvironmentType ShaderStage = dxbc::getShaderStage(ShaderKind);
 
diff --git a/llvm/lib/Object/ELF.cpp b/llvm/lib/Object/ELF.cpp
index b6d0699ee4fe0..bf42c92a242a1 100644
--- a/llvm/lib/Object/ELF.cpp
+++ b/llvm/lib/Object/ELF.cpp
@@ -747,13 +747,25 @@ decodeBBAddrMapImpl(const ELFFile<ELFT> &EF,
     assert(RelaSec &&
            "Can't read a SHT_LLVM_BB_ADDR_MAP section in a relocatable "
            "object file without providing a relocation section.");
-    Expected<typename ELFFile<ELFT>::Elf_Rela_Range> Relas = EF.relas(*RelaSec);
-    if (!Relas)
-      return createError("unable to read relocations for section " +
-                         describe(EF, Sec) + ": " +
-                         toString(Relas.takeError()));
-    for (typename ELFFile<ELFT>::Elf_Rela Rela : *Relas)
-      FunctionOffsetTranslations[Rela.r_offset] = Rela.r_addend;
+    if (RelaSec->sh_type == ELF::SHT_CREL) {
+      Expected<typename ELFFile<ELFT>::RelsOrRelas> Relas = EF.crels(*RelaSec);
+      if (!Relas)
+        return createError("unable to read CREL relocations for section " +
+                           describe(EF, Sec) + ": " +
+                           toString(Relas.takeError()));
+      for (typename ELFFile<ELFT>::Elf_Rela Rela : std::get<1>(*Relas)) {
+        FunctionOffsetTranslations[Rela.r_offset] = Rela.r_addend;
+      }
+    } else {
+      Expected<typename ELFFile<ELFT>::Elf_Rela_Range> Relas =
+          EF.relas(*RelaSec);
+      if (!Relas)
+        return createError("unable to read relocations for section " +
+                           describe(EF, Sec) + ": " +
+                           toString(Relas.takeError()));
+      for (typename ELFFile<ELFT>::Elf_Rela Rela : *Relas)
+        FunctionOffsetTranslations[Rela.r_offset] = Rela.r_addend;
+    }
   }
   auto GetAddressForRelocation =
       [&](unsigned RelocationOffsetInSection) -> Expected<unsigned> {
@@ -958,7 +970,8 @@ ELFFile<ELFT>::getSectionAndRelocations(
         continue;
     }
 
-    if (Sec.sh_type != ELF::SHT_RELA && Sec.sh_type != ELF::SHT_REL)
+    if (Sec.sh_type != ELF::SHT_RELA && Sec.sh_type != ELF::SHT_REL &&
+        Sec.sh_type != ELF::SHT_CREL)
       continue;
 
     Expected<const Elf_Shdr *> RelSecOrErr = this->getSection(Sec.sh_info);
diff --git a/llvm/lib/Object/ELFObjectFile.cpp b/llvm/lib/Object/ELFObjectFile.cpp
index 1ddfadaf1e271..2d3d70db50c39 100644
--- a/llvm/lib/Object/ELFObjectFile.cpp
+++ b/llvm/lib/Object/ELFObjectFile.cpp
@@ -802,6 +802,10 @@ std::vector<ELFPltEntry> ELFObjectFileBase::getPltEntries() const {
     case Triple::aarch64_be:
       JumpSlotReloc = ELF::R_AARCH64_JUMP_SLOT;
       break;
+    case Triple::hexagon:
+      JumpSlotReloc = ELF::R_HEX_JMP_SLOT;
+      GlobDatReloc = ELF::R_HEX_GLOB_DAT;
+      break;
     default:
       return {};
   }
diff --git a/llvm/lib/ObjectYAML/DWARFEmitter.cpp b/llvm/lib/ObjectYAML/DWARFEmitter.cpp
index 421dfe7dfa30e..ec5e08082b0ca 100644
--- a/llvm/lib/ObjectYAML/DWARFEmitter.cpp
+++ b/llvm/lib/ObjectYAML/DWARFEmitter.cpp
@@ -96,12 +96,11 @@ Error DWARFYAML::emitDebugStr(raw_ostream &OS, const DWARFYAML::Data &DI) {
 StringRef DWARFYAML::Data::getAbbrevTableContentByIndex(uint64_t Index) const {
   assert(Index < DebugAbbrev.size() &&
          "Index should be less than the size of DebugAbbrev array");
-  auto It = AbbrevTableContents.find(Index);
-  if (It != AbbrevTableContents.cend())
+  auto [It, Inserted] = AbbrevTableContents.try_emplace(Index);
+  if (!Inserted)
     return It->second;
 
-  std::string AbbrevTableBuffer;
-  raw_string_ostream OS(AbbrevTableBuffer);
+  raw_string_ostream OS(It->second);
 
   uint64_t AbbrevCode = 0;
   for (const DWARFYAML::Abbrev &AbbrevDecl : DebugAbbrev[Index].Table) {
@@ -123,9 +122,7 @@ StringRef DWARFYAML::Data::getAbbrevTableContentByIndex(uint64_t Index) const {
   // consisting of a 0 byte for the abbreviation code.
   OS.write_zeros(1);
 
-  AbbrevTableContents.insert({Index, AbbrevTableBuffer});
-
-  return AbbrevTableContents[Index];
+  return It->second;
 }
 
 Error DWARFYAML::emitDebugAbbrev(raw_ostream &OS, const DWARFYAML::Data &DI) {
diff --git a/llvm/lib/ObjectYAML/DXContainerEmitter.cpp b/llvm/lib/ObjectYAML/DXContainerEmitter.cpp
index 175f1a12f9314..b7d1c6558fa1f 100644
--- a/llvm/lib/ObjectYAML/DXContainerEmitter.cpp
+++ b/llvm/lib/ObjectYAML/DXContainerEmitter.cpp
@@ -13,6 +13,7 @@
 
 #include "llvm/BinaryFormat/DXContainer.h"
 #include "llvm/MC/DXContainerPSVInfo.h"
+#include "llvm/MC/DXContainerRootSignature.h"
 #include "llvm/ObjectYAML/ObjectYAML.h"
 #include "llvm/ObjectYAML/yaml2obj.h"
 #include "llvm/Support/Errc.h"
@@ -261,6 +262,20 @@ void DXContainerWriter::writeParts(raw_ostream &OS) {
     }
     case dxbc::PartType::Unknown:
       break; // Skip any handling for unrecognized parts.
+    case dxbc::PartType::RTS0:
+      if (!P.RootSignature.has_value())
+        continue;
+
+      mcdxbc::RootSignatureHeader Header;
+      Header.Flags = P.RootSignature->getEncodedFlags();
+      Header.Version = P.RootSignature->Version;
+      Header.NumParameters = P.RootSignature->NumParameters;
+      Header.RootParametersOffset = P.RootSignature->RootParametersOffset;
+      Header.NumStaticSamplers = P.RootSignature->NumStaticSamplers;
+      Header.StaticSamplersOffset = P.RootSignature->StaticSamplersOffset;
+
+      Header.write(OS);
+      break;
     }
     uint64_t BytesWritten = OS.tell() - DataStart;
     RollingOffset += BytesWritten;
diff --git a/llvm/lib/ObjectYAML/DXContainerYAML.cpp b/llvm/lib/ObjectYAML/DXContainerYAML.cpp
index 5dee1221b27c0..0869fd4fa9785 100644
--- a/llvm/lib/ObjectYAML/DXContainerYAML.cpp
+++ b/llvm/lib/ObjectYAML/DXContainerYAML.cpp
@@ -29,6 +29,27 @@ DXContainerYAML::ShaderFeatureFlags::ShaderFeatureFlags(uint64_t FlagData) {
 #include "llvm/BinaryFormat/DXContainerConstants.def"
 }
 
+DXContainerYAML::RootSignatureDesc::RootSignatureDesc(
+    const object::DirectX::RootSignature &Data)
+    : Version(Data.getVersion()), NumParameters(Data.getNumParameters()),
+      RootParametersOffset(Data.getRootParametersOffset()),
+      NumStaticSamplers(Data.getNumStaticSamplers()),
+      StaticSamplersOffset(Data.getStaticSamplersOffset()) {
+  uint32_t Flags = Data.getFlags();
+#define ROOT_ELEMENT_FLAG(Num, Val)                                            \
+  Val = (Flags & (uint32_t)dxbc::RootElementFlag::Val) > 0;
+#include "llvm/BinaryFormat/DXContainerConstants.def"
+}
+
+uint32_t DXContainerYAML::RootSignatureDesc::getEncodedFlags() {
+  uint64_t Flag = 0;
+#define ROOT_ELEMENT_FLAG(Num, Val)                                            \
+  if (Val)                                                                     \
+    Flag |= (uint32_t)dxbc::RootElementFlag::Val;
+#include "llvm/BinaryFormat/DXContainerConstants.def"
+  return Flag;
+}
+
 uint64_t DXContainerYAML::ShaderFeatureFlags::getEncodedFlags() {
   uint64_t Flag = 0;
 #define SHADER_FEATURE_FLAG(Num, DxilModuleNum, Val, Str)                      \
@@ -188,6 +209,17 @@ void MappingTraits<DXContainerYAML::Signature>::mapping(
   IO.mapRequired("Parameters", S.Parameters);
 }
 
+void MappingTraits<DXContainerYAML::RootSignatureDesc>::mapping(
+    IO &IO, DXContainerYAML::RootSignatureDesc &S) {
+  IO.mapRequired("Version", S.Version);
+  IO.mapRequired("NumParameters", S.NumParameters);
+  IO.mapRequired("RootParametersOffset", S.RootParametersOffset);
+  IO.mapRequired("NumStaticSamplers", S.NumStaticSamplers);
+  IO.mapRequired("StaticSamplersOffset", S.StaticSamplersOffset);
+#define ROOT_ELEMENT_FLAG(Num, Val) IO.mapOptional(#Val, S.Val, false);
+#include "llvm/BinaryFormat/DXContainerConstants.def"
+}
+
 void MappingTraits<DXContainerYAML::Part>::mapping(IO &IO,
                                                    DXContainerYAML::Part &P) {
   IO.mapRequired("Name", P.Name);
@@ -197,6 +229,7 @@ void MappingTraits<DXContainerYAML::Part>::mapping(IO &IO,
   IO.mapOptional("Hash", P.Hash);
   IO.mapOptional("PSVInfo", P.Info);
   IO.mapOptional("Signature", P.Signature);
+  IO.mapOptional("RootSignature", P.RootSignature);
 }
 
 void MappingTraits<DXContainerYAML::Object>::mapping(
diff --git a/llvm/lib/Passes/PassBuilder.cpp b/llvm/lib/Passes/PassBuilder.cpp
index 1e97cef22045d..e7ba7213a76fe 100644
--- a/llvm/lib/Passes/PassBuilder.cpp
+++ b/llvm/lib/Passes/PassBuilder.cpp
@@ -46,6 +46,7 @@
 #include "llvm/Analysis/InlineAdvisor.h"
 #include "llvm/Analysis/InlineSizeEstimatorAnalysis.h"
 #include "llvm/Analysis/InstCount.h"
+#include "llvm/Analysis/KernelInfo.h"
 #include "llvm/Analysis/LastRunTrackingAnalysis.h"
 #include "llvm/Analysis/LazyCallGraph.h"
 #include "llvm/Analysis/LazyValueInfo.h"
@@ -109,6 +110,7 @@
 #include "llvm/CodeGen/MachineBlockFrequencyInfo.h"
 #include "llvm/CodeGen/MachineBranchProbabilityInfo.h"
 #include "llvm/CodeGen/MachineCSE.h"
+#include "llvm/CodeGen/MachineCopyPropagation.h"
 #include "llvm/CodeGen/MachineDominators.h"
 #include "llvm/CodeGen/MachineFunctionAnalysis.h"
 #include "llvm/CodeGen/MachineLICM.h"
@@ -122,11 +124,14 @@
 #include "llvm/CodeGen/OptimizePHIs.h"
 #include "llvm/CodeGen/PHIElimination.h"
 #include "llvm/CodeGen/PeepholeOptimizer.h"
+#include "llvm/CodeGen/PostRASchedulerList.h"
 #include "llvm/CodeGen/PreISelIntrinsicLowering.h"
 #include "llvm/CodeGen/RegAllocFast.h"
 #include "llvm/CodeGen/RegUsageInfoCollector.h"
 #include "llvm/CodeGen/RegUsageInfoPropagate.h"
+#include "llvm/CodeGen/RegisterCoalescerPass.h"
 #include "llvm/CodeGen/RegisterUsageInfo.h"
+#include "llvm/CodeGen/RenameIndependentSubregs.h"
 #include "llvm/CodeGen/SafeStack.h"
 #include "llvm/CodeGen/SelectOptimize.h"
 #include "llvm/CodeGen/ShadowStackGCLowering.h"
@@ -135,6 +140,7 @@
 #include "llvm/CodeGen/SpillPlacement.h"
 #include "llvm/CodeGen/StackColoring.h"
 #include "llvm/CodeGen/StackProtector.h"
+#include "llvm/CodeGen/StackSlotColoring.h"
 #include "llvm/CodeGen/TailDuplication.h"
 #include "llvm/CodeGen/TargetPassConfig.h"
 #include "llvm/CodeGen/TwoAddressInstructionPass.h"
@@ -828,11 +834,65 @@ parseLowerAllowCheckPassOptions(StringRef Params) {
     StringRef ParamName;
     std::tie(ParamName, Params) = Params.split(';');
 
-    return make_error<StringError>(
-        formatv("invalid LowerAllowCheck pass parameter '{0}' ", ParamName)
-            .str(),
-        inconvertibleErrorCode());
+    // Format is <cutoffs[1,2,3]=70000;cutoffs[5,6,8]=90000>
+    //
+    // Parsing allows duplicate indices (last one takes precedence).
+    // It would technically be in spec to specify
+    //   cutoffs[0]=70000,cutoffs[1]=90000,cutoffs[0]=80000,...
+    if (ParamName.starts_with("cutoffs[")) {
+      StringRef IndicesStr;
+      StringRef CutoffStr;
+
+      std::tie(IndicesStr, CutoffStr) = ParamName.split("]=");
+      //       cutoffs[1,2,3
+      //                   70000
+
+      int cutoff;
+      if (CutoffStr.getAsInteger(0, cutoff))
+        return make_error<StringError>(
+            formatv("invalid LowerAllowCheck pass cutoffs parameter '{0}' "
+                    "({1})",
+                    CutoffStr, Params)
+                .str(),
+            inconvertibleErrorCode());
+
+      if (!IndicesStr.consume_front("cutoffs[") || IndicesStr == "")
+        return make_error<StringError>(
+            formatv("invalid LowerAllowCheck pass index parameter '{0}' "
+                    "({1})",
+                    IndicesStr, CutoffStr)
+                .str(),
+            inconvertibleErrorCode());
+
+      while (IndicesStr != "") {
+        StringRef firstIndexStr;
+        std::tie(firstIndexStr, IndicesStr) = IndicesStr.split('|');
+
+        unsigned int index;
+        if (firstIndexStr.getAsInteger(0, index))
+          return make_error<StringError>(
+              formatv("invalid LowerAllowCheck pass index parameter '{0}' "
+                      "({1}) {2}",
+                      firstIndexStr, IndicesStr)
+                  .str(),
+              inconvertibleErrorCode());
+
+        // In the common case (sequentially increasing indices), we will issue
+        // O(n) resize requests. We assume the underlying data structure has
+        // O(1) runtime for each added element.
+        if (index >= Result.cutoffs.size())
+          Result.cutoffs.resize(index + 1, 0);
+
+        Result.cutoffs[index] = cutoff;
+      }
+    } else {
+      return make_error<StringError>(
+          formatv("invalid LowerAllowCheck pass parameter '{0}' ", ParamName)
+              .str(),
+          inconvertibleErrorCode());
+    }
   }
+
   return Result;
 }
 
diff --git a/llvm/lib/Passes/PassBuilderPipelines.cpp b/llvm/lib/Passes/PassBuilderPipelines.cpp
index 4ec0fb8fc81ea..63e70d7e182bd 100644
--- a/llvm/lib/Passes/PassBuilderPipelines.cpp
+++ b/llvm/lib/Passes/PassBuilderPipelines.cpp
@@ -200,9 +200,9 @@ static cl::opt<bool> ExtraVectorizerPasses(
 static cl::opt<bool> RunNewGVN("enable-newgvn", cl::init(false), cl::Hidden,
                                cl::desc("Run the NewGVN pass"));
 
-static cl::opt<bool> EnableLoopInterchange(
-    "enable-loopinterchange", cl::init(false), cl::Hidden,
-    cl::desc("Enable the experimental LoopInterchange Pass"));
+static cl::opt<bool>
+    EnableLoopInterchange("enable-loopinterchange", cl::init(false), cl::Hidden,
+                          cl::desc("Enable the LoopInterchange Pass"));
 
 static cl::opt<bool> EnableUnrollAndJam("enable-unroll-and-jam",
                                         cl::init(false), cl::Hidden,
@@ -316,6 +316,7 @@ PipelineTuningOptions::PipelineTuningOptions() {
   LoopVectorization = true;
   SLPVectorization = false;
   LoopUnrolling = true;
+  LoopInterchange = EnableLoopInterchange;
   ForgetAllSCEVInLoopUnroll = ForgetSCEVInLoopUnroll;
   LicmMssaOptCap = SetLicmMssaOptCap;
   LicmMssaNoAccForPromotionCap = SetLicmMssaNoAccForPromotionCap;
@@ -360,6 +361,11 @@ void PassBuilder::invokeVectorizerStartEPCallbacks(FunctionPassManager &FPM,
   for (auto &C : VectorizerStartEPCallbacks)
     C(FPM, Level);
 }
+void PassBuilder::invokeVectorizerEndEPCallbacks(FunctionPassManager &FPM,
+                                                 OptimizationLevel Level) {
+  for (auto &C : VectorizerEndEPCallbacks)
+    C(FPM, Level);
+}
 void PassBuilder::invokeOptimizerEarlyEPCallbacks(ModulePassManager &MPM,
                                                   OptimizationLevel Level,
                                                   ThinOrFullLTOPhase Phase) {
@@ -480,7 +486,7 @@ PassBuilder::buildO1FunctionSimplificationPipeline(OptimizationLevel Level,
 
   LPM2.addPass(LoopDeletionPass());
 
-  if (EnableLoopInterchange)
+  if (PTO.LoopInterchange)
     LPM2.addPass(LoopInterchangePass());
 
   // Do not enable unrolling in PreLinkThinLTO phase during sample PGO
@@ -671,7 +677,7 @@ PassBuilder::buildFunctionSimplificationPipeline(OptimizationLevel Level,
 
   LPM2.addPass(LoopDeletionPass());
 
-  if (EnableLoopInterchange)
+  if (PTO.LoopInterchange)
     LPM2.addPass(LoopInterchangePass());
 
   // Do not enable unrolling in PreLinkThinLTO phase during sample PGO
@@ -964,7 +970,7 @@ PassBuilder::buildInlinerPipeline(OptimizationLevel Level,
   // Try to perform OpenMP specific optimizations. This is a (quick!) no-op if
   // there are no OpenMP runtime calls present in the module.
   if (Level == OptimizationLevel::O2 || Level == OptimizationLevel::O3)
-    MainCGPipeline.addPass(OpenMPOptCGSCCPass());
+    MainCGPipeline.addPass(OpenMPOptCGSCCPass(Phase));
 
   invokeCGSCCOptimizerLateEPCallbacks(MainCGPipeline, Level);
 
@@ -1132,7 +1138,7 @@ PassBuilder::buildModuleSimplificationPipeline(OptimizationLevel Level,
 
   // Try to perform OpenMP specific optimizations on the module. This is a
   // (quick!) no-op if there are no OpenMP runtime calls present in the module.
-  MPM.addPass(OpenMPOptPass());
+  MPM.addPass(OpenMPOptPass(Phase));
 
   if (AttributorRun & AttributorRunOption::MODULE)
     MPM.addPass(AttributorPass());
@@ -1534,6 +1540,8 @@ PassBuilder::buildModuleOptimizationPipeline(OptimizationLevel Level,
 
   addVectorPasses(Level, OptimizePM, /* IsFullLTO */ false);
 
+  invokeVectorizerEndEPCallbacks(OptimizePM, Level);
+
   // LoopSink pass sinks instructions hoisted by LICM, which serves as a
   // canonicalization pass that enables other optimizations. As a result,
   // LoopSink pass needs to be a very late IR pass to avoid undoing LICM
@@ -1594,9 +1602,7 @@ PassBuilder::buildModuleOptimizationPipeline(OptimizationLevel Level,
     MPM.addPass(CGProfilePass(LTOPhase == ThinOrFullLTOPhase::FullLTOPostLink ||
                               LTOPhase == ThinOrFullLTOPhase::ThinLTOPostLink));
 
-  // TODO: Relative look table converter pass caused an issue when full lto is
-  // enabled. See https://reviews.llvm.org/D94355 for more details.
-  // Until the issue fixed, disable this pass during pre-linking phase.
+  // RelLookupTableConverterPass runs later in LTO post-link pipeline.
   if (!LTOPreLink)
     MPM.addPass(RelLookupTableConverterPass());
 
@@ -2048,6 +2054,8 @@ PassBuilder::buildLTODefaultPipeline(OptimizationLevel Level,
   MainFPM.addPass(MoveAutoInitPass());
   MainFPM.addPass(MergedLoadStoreMotionPass());
 
+  invokeVectorizerStartEPCallbacks(MainFPM, Level);
+
   LoopPassManager LPM;
   if (EnableLoopFlatten && Level.getSpeedupLevel() > 1)
     LPM.addPass(LoopFlattenPass());
@@ -2068,6 +2076,8 @@ PassBuilder::buildLTODefaultPipeline(OptimizationLevel Level,
 
   addVectorPasses(Level, MainFPM, /* IsFullLTO */ true);
 
+  invokeVectorizerEndEPCallbacks(MainFPM, Level);
+
   // Run the OpenMPOpt CGSCC pass again late.
   MPM.addPass(createModuleToPostOrderCGSCCPassAdaptor(
       OpenMPOptCGSCCPass(ThinOrFullLTOPhase::FullLTOPostLink)));
@@ -2121,6 +2131,8 @@ PassBuilder::buildLTODefaultPipeline(OptimizationLevel Level,
   if (PTO.MergeFunctions)
     MPM.addPass(MergeFunctionsPass());
 
+  MPM.addPass(RelLookupTableConverterPass());
+
   if (PTO.CallGraphProfile)
     MPM.addPass(CGProfilePass(/*InLTOPostLink=*/true));
 
@@ -2231,6 +2243,13 @@ PassBuilder::buildO0DefaultPipeline(OptimizationLevel Level,
       MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM)));
   }
 
+  if (!VectorizerEndEPCallbacks.empty()) {
+    FunctionPassManager FPM;
+    invokeVectorizerEndEPCallbacks(FPM, Level);
+    if (!FPM.isEmpty())
+      MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM)));
+  }
+
   ModulePassManager CoroPM;
   CoroPM.addPass(CoroEarlyPass());
   CGSCCPassManager CGPM;
diff --git a/llvm/lib/Passes/PassRegistry.def b/llvm/lib/Passes/PassRegistry.def
index 0eb050c8adb04..9300a3dfca1dc 100644
--- a/llvm/lib/Passes/PassRegistry.def
+++ b/llvm/lib/Passes/PassRegistry.def
@@ -391,6 +391,7 @@ FUNCTION_PASS("irce", IRCEPass())
 FUNCTION_PASS("jump-threading", JumpThreadingPass())
 FUNCTION_PASS("jump-table-to-switch", JumpTableToSwitchPass());
 FUNCTION_PASS("kcfi", KCFIPass())
+FUNCTION_PASS("kernel-info", KernelInfoPrinter(TM))
 FUNCTION_PASS("lcssa", LCSSAPass())
 FUNCTION_PASS("libcalls-shrinkwrap", LibCallsShrinkWrapPass())
 FUNCTION_PASS("lint", LintPass())
diff --git a/llvm/lib/SandboxIR/Context.cpp b/llvm/lib/SandboxIR/Context.cpp
index 42ca456881fd0..830f2832853fe 100644
--- a/llvm/lib/SandboxIR/Context.cpp
+++ b/llvm/lib/SandboxIR/Context.cpp
@@ -611,6 +611,12 @@ Context::Context(LLVMContext &LLVMCtx)
 
 Context::~Context() {}
 
+void Context::clear() {
+  // TODO: Ideally we should clear only function-scope objects, and keep global
+  // objects, like Constants to avoid recreating them.
+  LLVMValueToValueMap.clear();
+}
+
 Module *Context::getModule(llvm::Module *LLVMM) const {
   auto It = LLVMModuleToModuleMap.find(LLVMM);
   if (It != LLVMModuleToModuleMap.end())
@@ -628,12 +634,15 @@ Module *Context::getOrCreateModule(llvm::Module *LLVMM) {
 }
 
 Function *Context::createFunction(llvm::Function *F) {
-  assert(getValue(F) == nullptr && "Already exists!");
   // Create the module if needed before we create the new sandboxir::Function.
   // Note: this won't fully populate the module. The only globals that will be
   // available will be the ones being used within the function.
   getOrCreateModule(F->getParent());
 
+  // There may be a function declaration already defined. Regardless destroy it.
+  if (Function *ExistingF = cast_or_null<Function>(getValue(F)))
+    detach(ExistingF);
+
   auto NewFPtr = std::unique_ptr<Function>(new Function(F, *this));
   auto *SBF = cast<Function>(registerValue(std::move(NewFPtr)));
   // Create arguments.
diff --git a/llvm/lib/SandboxIR/Instruction.cpp b/llvm/lib/SandboxIR/Instruction.cpp
index cc961418600e3..956047cf87b6b 100644
--- a/llvm/lib/SandboxIR/Instruction.cpp
+++ b/llvm/lib/SandboxIR/Instruction.cpp
@@ -129,7 +129,7 @@ void Instruction::insertBefore(Instruction *BeforeI) {
 
   // Insert the LLVM IR Instructions in program order.
   for (llvm::Instruction *I : getLLVMInstrs())
-    I->insertBefore(BeforeTopI);
+    I->insertBefore(BeforeTopI->getIterator());
 }
 
 void Instruction::insertAfter(Instruction *AfterI) {
diff --git a/llvm/lib/SandboxIR/Region.cpp b/llvm/lib/SandboxIR/Region.cpp
index 1455012440f90..dbb000e5dd925 100644
--- a/llvm/lib/SandboxIR/Region.cpp
+++ b/llvm/lib/SandboxIR/Region.cpp
@@ -11,7 +11,29 @@
 
 namespace llvm::sandboxir {
 
-Region::Region(Context &Ctx) : Ctx(Ctx) {
+InstructionCost ScoreBoard::getCost(Instruction *I) const {
+  auto *LLVMI = cast<llvm::Instruction>(I->Val);
+  SmallVector<const llvm::Value *> Operands(LLVMI->operands());
+  return TTI.getInstructionCost(LLVMI, Operands, CostKind);
+}
+
+void ScoreBoard::remove(Instruction *I) {
+  auto Cost = getCost(I);
+  if (Rgn.contains(I))
+    // If `I` is one the newly added ones, then we should adjust `AfterCost`
+    AfterCost -= Cost;
+  else
+    // If `I` is one of the original instructions (outside the region) then it
+    // is part of the original code, so adjust `BeforeCost`.
+    BeforeCost += Cost;
+}
+
+#ifndef NDEBUG
+void ScoreBoard::dump() const { dump(dbgs()); }
+#endif
+
+Region::Region(Context &Ctx, TargetTransformInfo &TTI)
+    : Ctx(Ctx), Scoreboard(*this, TTI) {
   LLVMContext &LLVMCtx = Ctx.LLVMCtx;
   auto *RegionStrMD = MDString::get(LLVMCtx, RegionStr);
   RegionMDN = MDNode::getDistinct(LLVMCtx, {RegionStrMD});
@@ -31,9 +53,15 @@ void Region::add(Instruction *I) {
   Insts.insert(I);
   // TODO: Consider tagging instructions lazily.
   cast<llvm::Instruction>(I->Val)->setMetadata(MDKind, RegionMDN);
+  // Keep track of the instruction cost.
+  Scoreboard.add(I);
 }
 
 void Region::remove(Instruction *I) {
+  // Keep track of the instruction cost. This need to be done *before* we remove
+  // `I` from the region.
+  Scoreboard.remove(I);
+
   Insts.remove(I);
   cast<llvm::Instruction>(I->Val)->setMetadata(MDKind, nullptr);
 }
@@ -58,23 +86,20 @@ void Region::dump() const {
 }
 #endif // NDEBUG
 
-SmallVector<std::unique_ptr<Region>> Region::createRegionsFromMD(Function &F) {
+SmallVector<std::unique_ptr<Region>>
+Region::createRegionsFromMD(Function &F, TargetTransformInfo &TTI) {
   SmallVector<std::unique_ptr<Region>> Regions;
   DenseMap<MDNode *, Region *> MDNToRegion;
   auto &Ctx = F.getContext();
   for (BasicBlock &BB : F) {
     for (Instruction &Inst : BB) {
       if (auto *MDN = cast<llvm::Instruction>(Inst.Val)->getMetadata(MDKind)) {
-        Region *R = nullptr;
-        auto It = MDNToRegion.find(MDN);
-        if (It == MDNToRegion.end()) {
-          Regions.push_back(std::make_unique<Region>(Ctx));
-          R = Regions.back().get();
-          MDNToRegion[MDN] = R;
-        } else {
-          R = It->second;
+        auto [It, Inserted] = MDNToRegion.try_emplace(MDN);
+        if (Inserted) {
+          Regions.push_back(std::make_unique<Region>(Ctx, TTI));
+          It->second = Regions.back().get();
         }
-        R->add(&Inst);
+        It->second->add(&Inst);
       }
     }
   }
diff --git a/llvm/lib/SandboxIR/Tracker.cpp b/llvm/lib/SandboxIR/Tracker.cpp
index 27ed37aa9bdd3..4fa9e11ae0d4e 100644
--- a/llvm/lib/SandboxIR/Tracker.cpp
+++ b/llvm/lib/SandboxIR/Tracker.cpp
@@ -175,7 +175,7 @@ void EraseFromParent::revert(Tracker &Tracker) {
   // Place the bottom-most instruction first.
   auto [Operands, BotLLVMI] = InstrData[0];
   if (auto *NextLLVMI = dyn_cast<llvm::Instruction *>(NextLLVMIOrBB)) {
-    BotLLVMI->insertBefore(NextLLVMI);
+    BotLLVMI->insertBefore(NextLLVMI->getIterator());
   } else {
     auto *LLVMBB = cast<llvm::BasicBlock *>(NextLLVMIOrBB);
     BotLLVMI->insertInto(LLVMBB, LLVMBB->end());
@@ -185,7 +185,7 @@ void EraseFromParent::revert(Tracker &Tracker) {
 
   // Go over the rest of the instructions and stack them on top.
   for (auto [Operands, LLVMI] : drop_begin(InstrData)) {
-    LLVMI->insertBefore(BotLLVMI);
+    LLVMI->insertBefore(BotLLVMI->getIterator());
     for (auto [OpNum, Op] : enumerate(Operands))
       LLVMI->setOperand(OpNum, Op);
     BotLLVMI = LLVMI;
@@ -347,13 +347,14 @@ void Tracker::save() {
 
 void Tracker::revert() {
   assert(State == TrackerState::Record && "Forgot to save()!");
-  State = TrackerState::Disabled;
+  State = TrackerState::Reverting;
   for (auto &Change : reverse(Changes))
     Change->revert(*this);
   Changes.clear();
 #if !defined(NDEBUG) && defined(EXPENSIVE_CHECKS)
   SnapshotChecker.expectNoDiff();
 #endif
+  State = TrackerState::Disabled;
 }
 
 void Tracker::accept() {
diff --git a/llvm/lib/Support/AArch64BuildAttributes.cpp b/llvm/lib/Support/AArch64BuildAttributes.cpp
index 4a6b2fd538803..e36667ca711e0 100644
--- a/llvm/lib/Support/AArch64BuildAttributes.cpp
+++ b/llvm/lib/Support/AArch64BuildAttributes.cpp
@@ -10,9 +10,9 @@
 #include "llvm/ADT/StringSwitch.h"
 
 using namespace llvm;
-using namespace llvm::AArch64BuildAttributes;
+using namespace llvm::AArch64BuildAttrs;
 
-StringRef AArch64BuildAttributes::getVendorName(unsigned Vendor) {
+StringRef AArch64BuildAttrs::getVendorName(unsigned Vendor) {
   switch (Vendor) {
   case AEABI_FEATURE_AND_BITS:
     return "aeabi_feature_and_bits";
@@ -25,14 +25,14 @@ StringRef AArch64BuildAttributes::getVendorName(unsigned Vendor) {
     return "";
   }
 }
-VendorID AArch64BuildAttributes::getVendorID(StringRef Vendor) {
+VendorID AArch64BuildAttrs::getVendorID(StringRef Vendor) {
   return StringSwitch<VendorID>(Vendor)
       .Case("aeabi_feature_and_bits", AEABI_FEATURE_AND_BITS)
       .Case("aeabi_pauthabi", AEABI_PAUTHABI)
       .Default(VENDOR_UNKNOWN);
 }
 
-StringRef AArch64BuildAttributes::getOptionalStr(unsigned Optional) {
+StringRef AArch64BuildAttrs::getOptionalStr(unsigned Optional) {
   switch (Optional) {
   case REQUIRED:
     return "required";
@@ -43,18 +43,18 @@ StringRef AArch64BuildAttributes::getOptionalStr(unsigned Optional) {
     return "";
   }
 }
-SubsectionOptional AArch64BuildAttributes::getOptionalID(StringRef Optional) {
+SubsectionOptional AArch64BuildAttrs::getOptionalID(StringRef Optional) {
   return StringSwitch<SubsectionOptional>(Optional)
       .Case("required", REQUIRED)
       .Case("optional", OPTIONAL)
       .Default(OPTIONAL_NOT_FOUND);
 }
-StringRef AArch64BuildAttributes::getSubsectionOptionalUnknownError() {
+StringRef AArch64BuildAttrs::getSubsectionOptionalUnknownError() {
   return "unknown AArch64 build attributes optionality, expected "
          "required|optional";
 }
 
-StringRef AArch64BuildAttributes::getTypeStr(unsigned Type) {
+StringRef AArch64BuildAttrs::getTypeStr(unsigned Type) {
   switch (Type) {
   case ULEB128:
     return "uleb128";
@@ -65,17 +65,17 @@ StringRef AArch64BuildAttributes::getTypeStr(unsigned Type) {
     return "";
   }
 }
-SubsectionType AArch64BuildAttributes::getTypeID(StringRef Type) {
+SubsectionType AArch64BuildAttrs::getTypeID(StringRef Type) {
   return StringSwitch<SubsectionType>(Type)
       .Cases("uleb128", "ULEB128", ULEB128)
       .Cases("ntbs", "NTBS", NTBS)
       .Default(TYPE_NOT_FOUND);
 }
-StringRef AArch64BuildAttributes::getSubsectionTypeUnknownError() {
+StringRef AArch64BuildAttrs::getSubsectionTypeUnknownError() {
   return "unknown AArch64 build attributes type, expected uleb128|ntbs";
 }
 
-StringRef AArch64BuildAttributes::getPauthABITagsStr(unsigned PauthABITag) {
+StringRef AArch64BuildAttrs::getPauthABITagsStr(unsigned PauthABITag) {
   switch (PauthABITag) {
   case TAG_PAUTH_PLATFORM:
     return "Tag_PAuth_Platform";
@@ -87,7 +87,7 @@ StringRef AArch64BuildAttributes::getPauthABITagsStr(unsigned PauthABITag) {
   }
 }
 
-PauthABITags AArch64BuildAttributes::getPauthABITagsID(StringRef PauthABITag) {
+PauthABITags AArch64BuildAttrs::getPauthABITagsID(StringRef PauthABITag) {
   return StringSwitch<PauthABITags>(PauthABITag)
       .Case("Tag_PAuth_Platform", TAG_PAUTH_PLATFORM)
       .Case("Tag_PAuth_Schema", TAG_PAUTH_SCHEMA)
@@ -95,7 +95,7 @@ PauthABITags AArch64BuildAttributes::getPauthABITagsID(StringRef PauthABITag) {
 }
 
 StringRef
-AArch64BuildAttributes::getFeatureAndBitsTagsStr(unsigned FeatureAndBitsTag) {
+AArch64BuildAttrs::getFeatureAndBitsTagsStr(unsigned FeatureAndBitsTag) {
   switch (FeatureAndBitsTag) {
   case TAG_FEATURE_BTI:
     return "Tag_Feature_BTI";
@@ -110,7 +110,7 @@ AArch64BuildAttributes::getFeatureAndBitsTagsStr(unsigned FeatureAndBitsTag) {
 }
 
 FeatureAndBitsTags
-AArch64BuildAttributes::getFeatureAndBitsTagsID(StringRef FeatureAndBitsTag) {
+AArch64BuildAttrs::getFeatureAndBitsTagsID(StringRef FeatureAndBitsTag) {
   return StringSwitch<FeatureAndBitsTags>(FeatureAndBitsTag)
       .Case("Tag_Feature_BTI", TAG_FEATURE_BTI)
       .Case("Tag_Feature_PAC", TAG_FEATURE_PAC)
diff --git a/llvm/lib/Support/ARMBuildAttrs.cpp b/llvm/lib/Support/ARMBuildAttributes.cpp
similarity index 100%
rename from llvm/lib/Support/ARMBuildAttrs.cpp
rename to llvm/lib/Support/ARMBuildAttributes.cpp
diff --git a/llvm/lib/Support/BalancedPartitioning.cpp b/llvm/lib/Support/BalancedPartitioning.cpp
index 19977c57c08db..7b807b167c0ef 100644
--- a/llvm/lib/Support/BalancedPartitioning.cpp
+++ b/llvm/lib/Support/BalancedPartitioning.cpp
@@ -305,7 +305,7 @@ void BalancedPartitioning::split(const FunctionNodeRange Nodes,
   unsigned NumNodes = std::distance(Nodes.begin(), Nodes.end());
   auto NodesMid = Nodes.begin() + (NumNodes + 1) / 2;
 
-  std::nth_element(Nodes.begin(), NodesMid, Nodes.end(), [](auto &L, auto &R) {
+  llvm::sort(Nodes.begin(), Nodes.end(), [](auto &L, auto &R) {
     return L.InputOrderIndex < R.InputOrderIndex;
   });
 
diff --git a/llvm/lib/Support/CMakeLists.txt b/llvm/lib/Support/CMakeLists.txt
index 122240c27b1fc..a6d8a25818866 100644
--- a/llvm/lib/Support/CMakeLists.txt
+++ b/llvm/lib/Support/CMakeLists.txt
@@ -143,7 +143,7 @@ add_llvm_component_library(LLVMSupport
   APFloat.cpp
   APInt.cpp
   APSInt.cpp
-  ARMBuildAttrs.cpp
+  ARMBuildAttributes.cpp
   AArch64BuildAttributes.cpp
   ARMAttributeParser.cpp
   ARMWinEH.cpp
diff --git a/llvm/lib/Support/MemAlloc.cpp b/llvm/lib/Support/MemAlloc.cpp
index 07a26cf26480b..6adc9abd75c5b 100644
--- a/llvm/lib/Support/MemAlloc.cpp
+++ b/llvm/lib/Support/MemAlloc.cpp
@@ -13,12 +13,15 @@
 
 LLVM_ATTRIBUTE_RETURNS_NONNULL LLVM_ATTRIBUTE_RETURNS_NOALIAS void *
 llvm::allocate_buffer(size_t Size, size_t Alignment) {
-  return ::operator new(Size
+  void *Result = ::operator new(Size,
 #ifdef __cpp_aligned_new
-                        ,
-                        std::align_val_t(Alignment)
+                                std::align_val_t(Alignment),
 #endif
-  );
+                                std::nothrow);
+  if (Result == nullptr) {
+    report_bad_alloc_error("Buffer allocation failed");
+  }
+  return Result;
 }
 
 void llvm::deallocate_buffer(void *Ptr, size_t Size, size_t Alignment) {
diff --git a/llvm/lib/Support/Unix/Path.inc b/llvm/lib/Support/Unix/Path.inc
index 1ad3a549a5a54..6004e007c0c7a 100644
--- a/llvm/lib/Support/Unix/Path.inc
+++ b/llvm/lib/Support/Unix/Path.inc
@@ -29,7 +29,6 @@
 
 #include <dirent.h>
 #include <pwd.h>
-#include <sys/file.h>
 
 #ifdef __APPLE__
 #include <copyfile.h>
diff --git a/llvm/lib/Support/Unix/Signals.inc b/llvm/lib/Support/Unix/Signals.inc
index 9a12663228a36..30e5f40193974 100644
--- a/llvm/lib/Support/Unix/Signals.inc
+++ b/llvm/lib/Support/Unix/Signals.inc
@@ -80,7 +80,7 @@
 
 using namespace llvm;
 
-static void SignalHandler(int Sig);     // defined below.
+static void SignalHandler(int Sig, siginfo_t *Info, void *);
 static void InfoSignalHandler(int Sig); // defined below.
 
 using SignalHandlerFunctionType = void (*)();
@@ -313,8 +313,8 @@ static void RegisterHandlers() { // Not signal-safe.
 
     switch (Kind) {
     case SignalKind::IsKill:
-      NewHandler.sa_handler = SignalHandler;
-      NewHandler.sa_flags = SA_NODEFER | SA_RESETHAND | SA_ONSTACK;
+      NewHandler.sa_sigaction = SignalHandler;
+      NewHandler.sa_flags = SA_NODEFER | SA_RESETHAND | SA_ONSTACK | SA_SIGINFO;
       break;
     case SignalKind::IsInfo:
       NewHandler.sa_handler = InfoSignalHandler;
@@ -370,7 +370,7 @@ void sys::CleanupOnSignal(uintptr_t Context) {
 }
 
 // The signal handler that runs.
-static void SignalHandler(int Sig) {
+static void SignalHandler(int Sig, siginfo_t *Info, void *) {
   // Restore the signal behavior to default, so that the program actually
   // crashes when we return and the signal reissues.  This also ensures that if
   // we crash in our signal handler that the program will terminate immediately
@@ -412,6 +412,11 @@ static void SignalHandler(int Sig) {
   if (Sig == SIGILL || Sig == SIGFPE || Sig == SIGTRAP)
     raise(Sig);
 #endif
+
+  // Signal sent from another process, do not assume that continuing the
+  // execution would re-raise it.
+  if (Info->si_pid != getpid())
+    raise(Sig);
 }
 
 static void InfoSignalHandler(int Sig) {
@@ -468,7 +473,8 @@ void llvm::sys::AddSignalHandler(sys::SignalHandlerCallback FnPtr,
 
 #if ENABLE_BACKTRACES && defined(HAVE_BACKTRACE) &&                            \
     (defined(__linux__) || defined(__FreeBSD__) ||                             \
-     defined(__FreeBSD_kernel__) || defined(__NetBSD__))
+     defined(__FreeBSD_kernel__) || defined(__NetBSD__) ||                     \
+     defined(__OpenBSD__) || defined(__DragonFly__))
 struct DlIteratePhdrData {
   void **StackTrace;
   int depth;
diff --git a/llvm/lib/Support/Unix/Threading.inc b/llvm/lib/Support/Unix/Threading.inc
index aedcd9aa34b99..15a5b008604c3 100644
--- a/llvm/lib/Support/Unix/Threading.inc
+++ b/llvm/lib/Support/Unix/Threading.inc
@@ -387,7 +387,7 @@ static int computeHostNumPhysicalCores() {
 static int computeHostNumPhysicalCores() {
   return sysconf(_SC_NPROCESSORS_ONLN);
 }
-#elif defined(__linux__) && !defined(__ANDROID__)
+#elif defined(__linux__)
 static int computeHostNumPhysicalCores() {
   cpu_set_t Affinity;
   if (sched_getaffinity(0, sizeof(Affinity), &Affinity) == 0)
diff --git a/llvm/lib/TableGen/Main.cpp b/llvm/lib/TableGen/Main.cpp
index 55a99cbfc58ac..35600bf2f1f86 100644
--- a/llvm/lib/TableGen/Main.cpp
+++ b/llvm/lib/TableGen/Main.cpp
@@ -128,6 +128,11 @@ int llvm::TableGenMain(const char *argv0,
     return 1;
   Timer.stopTimer();
 
+  // Return early if any other errors were generated during parsing
+  // (e.g., assert failures).
+  if (ErrorsPrinted > 0)
+    return reportError(argv0, Twine(ErrorsPrinted) + " errors.\n");
+
   // Write output to memory.
   Timer.startBackendTimer("Backend overall");
   std::string OutString;
diff --git a/llvm/lib/TableGen/TGParser.cpp b/llvm/lib/TableGen/TGParser.cpp
index d2115ab7627da..9a8301cffb930 100644
--- a/llvm/lib/TableGen/TGParser.cpp
+++ b/llvm/lib/TableGen/TGParser.cpp
@@ -286,11 +286,13 @@ bool TGParser::SetValue(Record *CurRec, SMLoc Loc, const Init *ValName,
       InitType = (Twine("' of type bit initializer with length ") +
                   Twine(BI->getNumBits())).str();
     else if (const auto *TI = dyn_cast<TypedInit>(V))
-      InitType = (Twine("' of type '") + TI->getType()->getAsString()).str();
+      InitType =
+          (Twine("' of type '") + TI->getType()->getAsString() + "'").str();
+
     return Error(Loc, "Field '" + ValName->getAsUnquotedString() +
                           "' of type '" + RV->getType()->getAsString() +
-                          "' is incompatible with value '" +
-                          V->getAsString() + InitType + "'");
+                          "' is incompatible with value '" + V->getAsString() +
+                          InitType);
   }
   return false;
 }
diff --git a/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp b/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp
index 8d8520c68232b..f1f25b65fc53f 100644
--- a/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp
+++ b/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp
@@ -78,11 +78,6 @@ static cl::opt<PtrauthCheckMode> PtrauthAuthChecks(
     cl::desc("Check pointer authentication auth/resign failures"),
     cl::init(Default));
 
-static cl::opt<bool> EnableImportCallOptimization(
-    "aarch64-win-import-call-optimization", cl::Hidden,
-    cl::desc("Enable import call optimization for AArch64 Windows"),
-    cl::init(false));
-
 #define DEBUG_TYPE "asm-printer"
 
 namespace {
@@ -95,6 +90,7 @@ class AArch64AsmPrinter : public AsmPrinter {
 #ifndef NDEBUG
   unsigned InstsEmitted;
 #endif
+  bool EnableImportCallOptimization = false;
   DenseMap<MCSection *, std::vector<std::pair<MCSymbol *, MCSymbol *>>>
       SectionToImportedFunctionCalls;
 
@@ -344,6 +340,9 @@ void AArch64AsmPrinter::emitStartOfAsmFile(Module &M) {
     OutStreamer->emitSymbolAttribute(S, MCSA_Global);
     OutStreamer->emitAssignment(
         S, MCConstantExpr::create(Feat00Value, MMI->getContext()));
+
+    if (M.getModuleFlag("import-call-optimization"))
+      EnableImportCallOptimization = true;
   }
 
   if (!TT.isOSBinFormatELF())
@@ -359,7 +358,7 @@ void AArch64AsmPrinter::emitStartOfAsmFile(Module &M) {
   if (const auto *BTE = mdconst::extract_or_null<ConstantInt>(
           M.getModuleFlag("branch-target-enforcement"))) {
     if (!BTE->isZero()) {
-      BAFlags |= AArch64BuildAttributes::FeatureAndBitsFlag::Feature_BTI_Flag;
+      BAFlags |= AArch64BuildAttrs::FeatureAndBitsFlag::Feature_BTI_Flag;
       GNUFlags |= ELF::GNU_PROPERTY_AARCH64_FEATURE_1_BTI;
     }
   }
@@ -367,7 +366,7 @@ void AArch64AsmPrinter::emitStartOfAsmFile(Module &M) {
   if (const auto *GCS = mdconst::extract_or_null<ConstantInt>(
           M.getModuleFlag("guarded-control-stack"))) {
     if (!GCS->isZero()) {
-      BAFlags |= AArch64BuildAttributes::FeatureAndBitsFlag::Feature_GCS_Flag;
+      BAFlags |= AArch64BuildAttrs::FeatureAndBitsFlag::Feature_GCS_Flag;
       GNUFlags |= ELF::GNU_PROPERTY_AARCH64_FEATURE_1_GCS;
     }
   }
@@ -375,7 +374,7 @@ void AArch64AsmPrinter::emitStartOfAsmFile(Module &M) {
   if (const auto *Sign = mdconst::extract_or_null<ConstantInt>(
           M.getModuleFlag("sign-return-address"))) {
     if (!Sign->isZero()) {
-      BAFlags |= AArch64BuildAttributes::FeatureAndBitsFlag::Feature_PAC_Flag;
+      BAFlags |= AArch64BuildAttrs::FeatureAndBitsFlag::Feature_PAC_Flag;
       GNUFlags |= ELF::GNU_PROPERTY_AARCH64_FEATURE_1_PAC;
     }
   }
@@ -478,45 +477,35 @@ void AArch64AsmPrinter::emitAttributes(unsigned Flags,
 
   if (PAuthABIPlatform || PAuthABIVersion) {
     TS->emitAtributesSubsection(
-        AArch64BuildAttributes::getVendorName(
-            AArch64BuildAttributes::AEABI_PAUTHABI),
-        AArch64BuildAttributes::SubsectionOptional::REQUIRED,
-        AArch64BuildAttributes::SubsectionType::ULEB128);
-    TS->emitAttribute(AArch64BuildAttributes::getVendorName(
-                          AArch64BuildAttributes::AEABI_PAUTHABI),
-                      AArch64BuildAttributes::TAG_PAUTH_PLATFORM,
-                      PAuthABIPlatform, "", false);
-    TS->emitAttribute(AArch64BuildAttributes::getVendorName(
-                          AArch64BuildAttributes::AEABI_PAUTHABI),
-                      AArch64BuildAttributes::TAG_PAUTH_SCHEMA, PAuthABIVersion,
-                      "", false);
-  }
-
-  unsigned BTIValue =
-      (Flags & AArch64BuildAttributes::Feature_BTI_Flag) ? 1 : 0;
-  unsigned PACValue =
-      (Flags & AArch64BuildAttributes::Feature_PAC_Flag) ? 1 : 0;
-  unsigned GCSValue =
-      (Flags & AArch64BuildAttributes::Feature_GCS_Flag) ? 1 : 0;
+        AArch64BuildAttrs::getVendorName(AArch64BuildAttrs::AEABI_PAUTHABI),
+        AArch64BuildAttrs::SubsectionOptional::REQUIRED,
+        AArch64BuildAttrs::SubsectionType::ULEB128);
+    TS->emitAttribute(
+        AArch64BuildAttrs::getVendorName(AArch64BuildAttrs::AEABI_PAUTHABI),
+        AArch64BuildAttrs::TAG_PAUTH_PLATFORM, PAuthABIPlatform, "", false);
+    TS->emitAttribute(
+        AArch64BuildAttrs::getVendorName(AArch64BuildAttrs::AEABI_PAUTHABI),
+        AArch64BuildAttrs::TAG_PAUTH_SCHEMA, PAuthABIVersion, "", false);
+  }
+
+  unsigned BTIValue = (Flags & AArch64BuildAttrs::Feature_BTI_Flag) ? 1 : 0;
+  unsigned PACValue = (Flags & AArch64BuildAttrs::Feature_PAC_Flag) ? 1 : 0;
+  unsigned GCSValue = (Flags & AArch64BuildAttrs::Feature_GCS_Flag) ? 1 : 0;
 
   if (BTIValue || PACValue || GCSValue) {
-    TS->emitAtributesSubsection(
-        AArch64BuildAttributes::getVendorName(
-            AArch64BuildAttributes::AEABI_FEATURE_AND_BITS),
-        AArch64BuildAttributes::SubsectionOptional::OPTIONAL,
-        AArch64BuildAttributes::SubsectionType::ULEB128);
-    TS->emitAttribute(AArch64BuildAttributes::getVendorName(
-                          AArch64BuildAttributes::AEABI_FEATURE_AND_BITS),
-                      AArch64BuildAttributes::TAG_FEATURE_BTI, BTIValue, "",
-                      false);
-    TS->emitAttribute(AArch64BuildAttributes::getVendorName(
-                          AArch64BuildAttributes::AEABI_FEATURE_AND_BITS),
-                      AArch64BuildAttributes::TAG_FEATURE_PAC, PACValue, "",
-                      false);
-    TS->emitAttribute(AArch64BuildAttributes::getVendorName(
-                          AArch64BuildAttributes::AEABI_FEATURE_AND_BITS),
-                      AArch64BuildAttributes::TAG_FEATURE_GCS, GCSValue, "",
-                      false);
+    TS->emitAtributesSubsection(AArch64BuildAttrs::getVendorName(
+                                    AArch64BuildAttrs::AEABI_FEATURE_AND_BITS),
+                                AArch64BuildAttrs::SubsectionOptional::OPTIONAL,
+                                AArch64BuildAttrs::SubsectionType::ULEB128);
+    TS->emitAttribute(AArch64BuildAttrs::getVendorName(
+                          AArch64BuildAttrs::AEABI_FEATURE_AND_BITS),
+                      AArch64BuildAttrs::TAG_FEATURE_BTI, BTIValue, "", false);
+    TS->emitAttribute(AArch64BuildAttrs::getVendorName(
+                          AArch64BuildAttrs::AEABI_FEATURE_AND_BITS),
+                      AArch64BuildAttrs::TAG_FEATURE_PAC, PACValue, "", false);
+    TS->emitAttribute(AArch64BuildAttrs::getVendorName(
+                          AArch64BuildAttrs::AEABI_FEATURE_AND_BITS),
+                      AArch64BuildAttrs::TAG_FEATURE_GCS, GCSValue, "", false);
   }
 }
 
@@ -3182,8 +3171,7 @@ void AArch64AsmPrinter::emitInstruction(const MachineInstr *MI) {
 
 void AArch64AsmPrinter::recordIfImportCall(
     const llvm::MachineInstr *BranchInst) {
-  if (!EnableImportCallOptimization ||
-      !TM.getTargetTriple().isOSBinFormatCOFF())
+  if (!EnableImportCallOptimization)
     return;
 
   auto [GV, OpFlags] = BranchInst->getMF()->tryGetCalledGlobal(BranchInst);
diff --git a/llvm/lib/Target/AArch64/AArch64Features.td b/llvm/lib/Target/AArch64/AArch64Features.td
index 0a91edb4c1661..d47dcfe92ff19 100644
--- a/llvm/lib/Target/AArch64/AArch64Features.td
+++ b/llvm/lib/Target/AArch64/AArch64Features.td
@@ -473,13 +473,13 @@ def FeatureD128 : ExtensionWithMArch<"d128", "D128",
 //===----------------------------------------------------------------------===//
 
 def FeatureFAMINMAX: ExtensionWithMArch<"faminmax", "FAMINMAX", "FEAT_FAMINMAX",
- "Enable FAMIN and FAMAX instructions">;
+ "Enable FAMIN and FAMAX instructions", [FeatureNEON]>;
 
 def FeatureLUT: ExtensionWithMArch<"lut", "LUT", "FEAT_LUT",
- "Enable Lookup Table instructions">;
+ "Enable Lookup Table instructions", [FeatureNEON]>;
 
 def FeatureFP8 : ExtensionWithMArch<"fp8", "FP8", "FEAT_FP8",
-  "Enable FP8 instructions", [FeatureFAMINMAX, FeatureLUT, FeatureBF16]>;
+  "Enable FP8 instructions", [FeatureNEON]>;
 
 def FeatureFP8FMA : ExtensionWithMArch<"fp8fma", "FP8FMA", "FEAT_FP8FMA",
   "Enable Armv9.5-A FP8 multiply-add instructions", [FeatureFP8]>;
@@ -488,10 +488,10 @@ def FeatureSSVE_FP8FMA : ExtensionWithMArch<"ssve-fp8fma", "SSVE_FP8FMA", "FEAT_
   "Enable SVE2 FP8 multiply-add instructions", [FeatureSME2, FeatureFP8]>;
 
 def FeatureFP8DOT4: ExtensionWithMArch<"fp8dot4", "FP8DOT4", "FEAT_FP8DOT4",
-  "Enable FP8 4-way dot instructions", [FeatureNEON, FeatureFP8]>;
+  "Enable FP8 4-way dot instructions", [FeatureFP8]>;
 
 def FeatureFP8DOT2: ExtensionWithMArch<"fp8dot2", "FP8DOT2", "FEAT_FP8DOT2",
-  "Enable FP8 2-way dot instructions", [FeatureNEON, FeatureFP8]>;
+  "Enable FP8 2-way dot instructions", [FeatureFP8]>;
 
 def FeatureSSVE_FP8DOT4 : ExtensionWithMArch<"ssve-fp8dot4", "SSVE_FP8DOT4", "FEAT_SSVE_FP8DOT4",
   "Enable SVE2 FP8 4-way dot product instructions", [FeatureSME2, FeatureFP8]>;
@@ -805,10 +805,14 @@ def FeatureLdpAlignedOnly : SubtargetFeature<"ldp-aligned-only", "HasLdpAlignedO
 def FeatureStpAlignedOnly : SubtargetFeature<"stp-aligned-only", "HasStpAlignedOnly",
     "true", "In order to emit stp, first check if the store will be aligned to 2 * element_size">;
 
-def FeatureUseFixedOverScalableIfEqualCost: SubtargetFeature<"use-fixed-over-scalable-if-equal-cost",
+def FeatureUseFixedOverScalableIfEqualCost : SubtargetFeature<"use-fixed-over-scalable-if-equal-cost",
   "UseFixedOverScalableIfEqualCost", "true",
   "Prefer fixed width loop vectorization over scalable if the cost-model assigns equal costs">;
 
+// For performance reasons we prefer to use ldapr to ldapur on certain cores.
+def FeatureAvoidLDAPUR : SubtargetFeature<"avoid-ldapur", "AvoidLDAPUR", "true",
+  "Prefer add+ldapr to offset ldapur">;
+
 //===----------------------------------------------------------------------===//
 // Architectures.
 //
diff --git a/llvm/lib/Target/AArch64/AArch64FrameLowering.cpp b/llvm/lib/Target/AArch64/AArch64FrameLowering.cpp
index eabe64361938b..d3abd79b85a75 100644
--- a/llvm/lib/Target/AArch64/AArch64FrameLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64FrameLowering.cpp
@@ -1491,13 +1491,6 @@ static MachineBasicBlock::iterator convertCalleeSaveRestoreToSPPrePostIncDec(
     NewOpc = AArch64::LDRQpost;
     break;
   }
-  // Get rid of the SEH code associated with the old instruction.
-  if (NeedsWinCFI) {
-    auto SEH = std::next(MBBI);
-    if (AArch64InstrInfo::isSEHInstruction(*SEH))
-      SEH->eraseFromParent();
-  }
-
   TypeSize Scale = TypeSize::getFixed(1), Width = TypeSize::getFixed(0);
   int64_t MinOffset, MaxOffset;
   bool Success = static_cast<const AArch64InstrInfo *>(TII)->getMemOpInfo(
@@ -1512,16 +1505,27 @@ static MachineBasicBlock::iterator convertCalleeSaveRestoreToSPPrePostIncDec(
       CSStackSizeInc > MaxOffset * (int64_t)Scale.getFixedValue()) {
     // If we are destroying the frame, make sure we add the increment after the
     // last frame operation.
-    if (FrameFlag == MachineInstr::FrameDestroy)
+    if (FrameFlag == MachineInstr::FrameDestroy) {
       ++MBBI;
+      // Also skip the SEH instruction, if needed
+      if (NeedsWinCFI && AArch64InstrInfo::isSEHInstruction(*MBBI))
+        ++MBBI;
+    }
     emitFrameOffset(MBB, MBBI, DL, AArch64::SP, AArch64::SP,
                     StackOffset::getFixed(CSStackSizeInc), TII, FrameFlag,
-                    false, false, nullptr, EmitCFI,
+                    false, NeedsWinCFI, HasWinCFI, EmitCFI,
                     StackOffset::getFixed(CFAOffset));
 
     return std::prev(MBBI);
   }
 
+  // Get rid of the SEH code associated with the old instruction.
+  if (NeedsWinCFI) {
+    auto SEH = std::next(MBBI);
+    if (AArch64InstrInfo::isSEHInstruction(*SEH))
+      SEH->eraseFromParent();
+  }
+
   MachineInstrBuilder MIB = BuildMI(MBB, MBBI, DL, TII->get(NewOpc));
   MIB.addReg(AArch64::SP, RegState::Define);
 
@@ -1630,6 +1634,9 @@ static bool IsSVECalleeSave(MachineBasicBlock::iterator I) {
   case AArch64::STR_PXI:
   case AArch64::LDR_ZXI:
   case AArch64::LDR_PXI:
+  case AArch64::PTRUE_B:
+  case AArch64::CPY_ZPzI_B:
+  case AArch64::CMPNE_PPzZI_B:
     return I->getFlag(MachineInstr::FrameSetup) ||
            I->getFlag(MachineInstr::FrameDestroy);
   }
@@ -3261,7 +3268,8 @@ bool AArch64FrameLowering::spillCalleeSavedRegisters(
       StrOpc = RPI.isPaired() ? AArch64::ST1B_2Z_IMM : AArch64::STR_ZXI;
       break;
     case RegPairInfo::PPR:
-      StrOpc = AArch64::STR_PXI;
+      StrOpc =
+          Size == 16 ? AArch64::SPILL_PPR_TO_ZPR_SLOT_PSEUDO : AArch64::STR_PXI;
       break;
     case RegPairInfo::VG:
       StrOpc = AArch64::STRXui;
@@ -3490,7 +3498,8 @@ bool AArch64FrameLowering::restoreCalleeSavedRegisters(
       LdrOpc = RPI.isPaired() ? AArch64::LD1B_2Z_IMM : AArch64::LDR_ZXI;
       break;
     case RegPairInfo::PPR:
-      LdrOpc = AArch64::LDR_PXI;
+      LdrOpc = Size == 16 ? AArch64::FILL_PPR_FROM_ZPR_SLOT_PSEUDO
+                          : AArch64::LDR_PXI;
       break;
     case RegPairInfo::VG:
       continue;
@@ -3716,6 +3725,14 @@ void AArch64FrameLowering::determineCalleeSaves(MachineFunction &MF,
       continue;
     }
 
+    // Always save P4 when PPR spills are ZPR-sized and a predicate above p8 is
+    // spilled. If all of p0-p3 are used as return values p4 is must be free
+    // to reload p8-p15.
+    if (RegInfo->getSpillSize(AArch64::PPRRegClass) == 16 &&
+        AArch64::PPR_p8to15RegClass.contains(Reg)) {
+      SavedRegs.set(AArch64::P4);
+    }
+
     // MachO's compact unwind format relies on all registers being stored in
     // pairs.
     // FIXME: the usual format is actually better if unwinding isn't needed.
@@ -4155,8 +4172,312 @@ int64_t AArch64FrameLowering::assignSVEStackObjectOffsets(
                                         true);
 }
 
+/// Attempts to scavenge a register from \p ScavengeableRegs given the used
+/// registers in \p UsedRegs.
+static Register tryScavengeRegister(LiveRegUnits const &UsedRegs,
+                                    BitVector const &ScavengeableRegs,
+                                    Register PreferredReg) {
+  if (PreferredReg != AArch64::NoRegister && UsedRegs.available(PreferredReg))
+    return PreferredReg;
+  for (auto Reg : ScavengeableRegs.set_bits()) {
+    if (UsedRegs.available(Reg))
+      return Reg;
+  }
+  return AArch64::NoRegister;
+}
+
+/// Propagates frame-setup/destroy flags from \p SourceMI to all instructions in
+/// \p MachineInstrs.
+static void propagateFrameFlags(MachineInstr &SourceMI,
+                                ArrayRef<MachineInstr *> MachineInstrs) {
+  for (MachineInstr *MI : MachineInstrs) {
+    if (SourceMI.getFlag(MachineInstr::FrameSetup))
+      MI->setFlag(MachineInstr::FrameSetup);
+    if (SourceMI.getFlag(MachineInstr::FrameDestroy))
+      MI->setFlag(MachineInstr::FrameDestroy);
+  }
+}
+
+/// RAII helper class for scavenging or spilling a register. On construction
+/// attempts to find a free register of class \p RC (given \p UsedRegs and \p
+/// AllocatableRegs), if no register can be found spills \p SpillCandidate to \p
+/// MaybeSpillFI to free a register. The free'd register is returned via the \p
+/// FreeReg output parameter. On destruction, if there is a spill, its previous
+/// value is reloaded. The spilling and scavenging is only valid at the
+/// insertion point \p MBBI, this class should _not_ be used in places that
+/// create or manipulate basic blocks, moving the expected insertion point.
+struct ScopedScavengeOrSpill {
+  ScopedScavengeOrSpill(const ScopedScavengeOrSpill &) = delete;
+  ScopedScavengeOrSpill(ScopedScavengeOrSpill &&) = delete;
+
+  ScopedScavengeOrSpill(MachineFunction &MF, MachineBasicBlock &MBB,
+                        MachineBasicBlock::iterator MBBI,
+                        Register SpillCandidate, const TargetRegisterClass &RC,
+                        LiveRegUnits const &UsedRegs,
+                        BitVector const &AllocatableRegs,
+                        std::optional<int> *MaybeSpillFI,
+                        Register PreferredReg = AArch64::NoRegister)
+      : MBB(MBB), MBBI(MBBI), RC(RC), TII(static_cast<const AArch64InstrInfo &>(
+                                          *MF.getSubtarget().getInstrInfo())),
+        TRI(*MF.getSubtarget().getRegisterInfo()) {
+    FreeReg = tryScavengeRegister(UsedRegs, AllocatableRegs, PreferredReg);
+    if (FreeReg != AArch64::NoRegister)
+      return;
+    assert(MaybeSpillFI && "Expected emergency spill slot FI information "
+                           "(attempted to spill in prologue/epilogue?)");
+    if (!MaybeSpillFI->has_value()) {
+      MachineFrameInfo &MFI = MF.getFrameInfo();
+      *MaybeSpillFI = MFI.CreateSpillStackObject(TRI.getSpillSize(RC),
+                                                 TRI.getSpillAlign(RC));
+    }
+    FreeReg = SpillCandidate;
+    SpillFI = MaybeSpillFI->value();
+    TII.storeRegToStackSlot(MBB, MBBI, FreeReg, false, *SpillFI, &RC, &TRI,
+                            Register());
+  }
+
+  bool hasSpilled() const { return SpillFI.has_value(); }
+
+  /// Returns the free register (found from scavenging or spilling a register).
+  Register freeRegister() const { return FreeReg; }
+
+  Register operator*() const { return freeRegister(); }
+
+  ~ScopedScavengeOrSpill() {
+    if (hasSpilled())
+      TII.loadRegFromStackSlot(MBB, MBBI, FreeReg, *SpillFI, &RC, &TRI,
+                               Register());
+  }
+
+private:
+  MachineBasicBlock &MBB;
+  MachineBasicBlock::iterator MBBI;
+  const TargetRegisterClass &RC;
+  const AArch64InstrInfo &TII;
+  const TargetRegisterInfo &TRI;
+  Register FreeReg = AArch64::NoRegister;
+  std::optional<int> SpillFI;
+};
+
+/// Emergency stack slots for expanding SPILL_PPR_TO_ZPR_SLOT_PSEUDO and
+/// FILL_PPR_FROM_ZPR_SLOT_PSEUDO.
+struct EmergencyStackSlots {
+  std::optional<int> ZPRSpillFI;
+  std::optional<int> PPRSpillFI;
+  std::optional<int> GPRSpillFI;
+};
+
+/// Registers available for scavenging (ZPR, PPR3b, GPR).
+struct ScavengeableRegs {
+  BitVector ZPRRegs;
+  BitVector PPR3bRegs;
+  BitVector GPRRegs;
+};
+
+static bool isInPrologueOrEpilogue(const MachineInstr &MI) {
+  return MI.getFlag(MachineInstr::FrameSetup) ||
+         MI.getFlag(MachineInstr::FrameDestroy);
+}
+
+/// Expands:
+/// ```
+/// SPILL_PPR_TO_ZPR_SLOT_PSEUDO $p0, %stack.0, 0
+/// ```
+/// To:
+/// ```
+/// $z0 = CPY_ZPzI_B $p0, 1, 0
+/// STR_ZXI $z0, $stack.0, 0
+/// ```
+/// While ensuring a ZPR ($z0 in this example) is free for the predicate (
+/// spilling if necessary).
+static void expandSpillPPRToZPRSlotPseudo(MachineBasicBlock &MBB,
+                                          MachineInstr &MI,
+                                          const TargetRegisterInfo &TRI,
+                                          LiveRegUnits const &UsedRegs,
+                                          ScavengeableRegs const &SR,
+                                          EmergencyStackSlots &SpillSlots) {
+  MachineFunction &MF = *MBB.getParent();
+  auto *TII =
+      static_cast<const AArch64InstrInfo *>(MF.getSubtarget().getInstrInfo());
+
+  ScopedScavengeOrSpill ZPredReg(
+      MF, MBB, MI, AArch64::Z0, AArch64::ZPRRegClass, UsedRegs, SR.ZPRRegs,
+      isInPrologueOrEpilogue(MI) ? nullptr : &SpillSlots.ZPRSpillFI);
+
+  SmallVector<MachineInstr *, 2> MachineInstrs;
+  const DebugLoc &DL = MI.getDebugLoc();
+  MachineInstrs.push_back(BuildMI(MBB, MI, DL, TII->get(AArch64::CPY_ZPzI_B))
+                              .addReg(*ZPredReg, RegState::Define)
+                              .add(MI.getOperand(0))
+                              .addImm(1)
+                              .addImm(0)
+                              .getInstr());
+  MachineInstrs.push_back(BuildMI(MBB, MI, DL, TII->get(AArch64::STR_ZXI))
+                              .addReg(*ZPredReg)
+                              .add(MI.getOperand(1))
+                              .addImm(MI.getOperand(2).getImm())
+                              .setMemRefs(MI.memoperands())
+                              .getInstr());
+  propagateFrameFlags(MI, MachineInstrs);
+}
+
+/// Expands:
+/// ```
+/// $p0 = FILL_PPR_FROM_ZPR_SLOT_PSEUDO %stack.0, 0
+/// ```
+/// To:
+/// ```
+/// $z0 = LDR_ZXI %stack.0, 0
+/// $p0 = PTRUE_B 31, implicit $vg
+/// $p0 = CMPNE_PPzZI_B $p0, $z0, 0, implicit-def $nzcv, implicit-def $nzcv
+/// ```
+/// While ensuring a ZPR ($z0 in this example) is free for the predicate (
+/// spilling if necessary). If the status flags are in use at the point of
+/// expansion they are preserved (by moving them to/from a GPR). This may cause
+/// an additional spill if no GPR is free at the expansion point.
+static bool expandFillPPRFromZPRSlotPseudo(
+    MachineBasicBlock &MBB, MachineInstr &MI, const TargetRegisterInfo &TRI,
+    LiveRegUnits const &UsedRegs, ScavengeableRegs const &SR,
+    MachineInstr *&LastPTrue, EmergencyStackSlots &SpillSlots) {
+  MachineFunction &MF = *MBB.getParent();
+  auto *TII =
+      static_cast<const AArch64InstrInfo *>(MF.getSubtarget().getInstrInfo());
+
+  ScopedScavengeOrSpill ZPredReg(
+      MF, MBB, MI, AArch64::Z0, AArch64::ZPRRegClass, UsedRegs, SR.ZPRRegs,
+      isInPrologueOrEpilogue(MI) ? nullptr : &SpillSlots.ZPRSpillFI);
+
+  ScopedScavengeOrSpill PredReg(
+      MF, MBB, MI, AArch64::P0, AArch64::PPR_3bRegClass, UsedRegs, SR.PPR3bRegs,
+      isInPrologueOrEpilogue(MI) ? nullptr : &SpillSlots.PPRSpillFI,
+      /*PreferredReg=*/
+      LastPTrue ? LastPTrue->getOperand(0).getReg() : AArch64::NoRegister);
+
+  // Elide NZCV spills if we know it is not used.
+  bool IsNZCVUsed = !UsedRegs.available(AArch64::NZCV);
+  std::optional<ScopedScavengeOrSpill> NZCVSaveReg;
+  if (IsNZCVUsed)
+    NZCVSaveReg.emplace(
+        MF, MBB, MI, AArch64::X0, AArch64::GPR64RegClass, UsedRegs, SR.GPRRegs,
+        isInPrologueOrEpilogue(MI) ? nullptr : &SpillSlots.GPRSpillFI);
+  SmallVector<MachineInstr *, 4> MachineInstrs;
+  const DebugLoc &DL = MI.getDebugLoc();
+  MachineInstrs.push_back(BuildMI(MBB, MI, DL, TII->get(AArch64::LDR_ZXI))
+                              .addReg(*ZPredReg, RegState::Define)
+                              .add(MI.getOperand(1))
+                              .addImm(MI.getOperand(2).getImm())
+                              .setMemRefs(MI.memoperands())
+                              .getInstr());
+  if (IsNZCVUsed)
+    MachineInstrs.push_back(
+        BuildMI(MBB, MI, DL, TII->get(AArch64::MRS))
+            .addReg(NZCVSaveReg->freeRegister(), RegState::Define)
+            .addImm(AArch64SysReg::NZCV)
+            .addReg(AArch64::NZCV, RegState::Implicit)
+            .getInstr());
+
+  // Reuse previous ptrue if we know it has not been clobbered.
+  if (LastPTrue) {
+    assert(*PredReg == LastPTrue->getOperand(0).getReg());
+    LastPTrue->moveBefore(&MI);
+  } else {
+    LastPTrue = BuildMI(MBB, MI, DL, TII->get(AArch64::PTRUE_B))
+                    .addReg(*PredReg, RegState::Define)
+                    .addImm(31);
+  }
+  MachineInstrs.push_back(LastPTrue);
+  MachineInstrs.push_back(
+      BuildMI(MBB, MI, DL, TII->get(AArch64::CMPNE_PPzZI_B))
+          .addReg(MI.getOperand(0).getReg(), RegState::Define)
+          .addReg(*PredReg)
+          .addReg(*ZPredReg)
+          .addImm(0)
+          .addReg(AArch64::NZCV, RegState::ImplicitDefine)
+          .getInstr());
+  if (IsNZCVUsed)
+    MachineInstrs.push_back(BuildMI(MBB, MI, DL, TII->get(AArch64::MSR))
+                                .addImm(AArch64SysReg::NZCV)
+                                .addReg(NZCVSaveReg->freeRegister())
+                                .addReg(AArch64::NZCV, RegState::ImplicitDefine)
+                                .getInstr());
+
+  propagateFrameFlags(MI, MachineInstrs);
+  return PredReg.hasSpilled();
+}
+
+/// Expands all FILL_PPR_FROM_ZPR_SLOT_PSEUDO and SPILL_PPR_TO_ZPR_SLOT_PSEUDO
+/// operations within the MachineBasicBlock \p MBB.
+static bool expandSMEPPRToZPRSpillPseudos(MachineBasicBlock &MBB,
+                                          const TargetRegisterInfo &TRI,
+                                          ScavengeableRegs const &SR,
+                                          EmergencyStackSlots &SpillSlots) {
+  LiveRegUnits UsedRegs(TRI);
+  UsedRegs.addLiveOuts(MBB);
+  bool HasPPRSpills = false;
+  MachineInstr *LastPTrue = nullptr;
+  for (MachineInstr &MI : make_early_inc_range(reverse(MBB))) {
+    UsedRegs.stepBackward(MI);
+    switch (MI.getOpcode()) {
+    case AArch64::FILL_PPR_FROM_ZPR_SLOT_PSEUDO:
+      if (LastPTrue &&
+          MI.definesRegister(LastPTrue->getOperand(0).getReg(), &TRI))
+        LastPTrue = nullptr;
+      HasPPRSpills |= expandFillPPRFromZPRSlotPseudo(MBB, MI, TRI, UsedRegs, SR,
+                                                     LastPTrue, SpillSlots);
+      MI.eraseFromParent();
+      break;
+    case AArch64::SPILL_PPR_TO_ZPR_SLOT_PSEUDO:
+      expandSpillPPRToZPRSlotPseudo(MBB, MI, TRI, UsedRegs, SR, SpillSlots);
+      MI.eraseFromParent();
+      [[fallthrough]];
+    default:
+      LastPTrue = nullptr;
+      break;
+    }
+  }
+
+  return HasPPRSpills;
+}
+
 void AArch64FrameLowering::processFunctionBeforeFrameFinalized(
     MachineFunction &MF, RegScavenger *RS) const {
+
+  AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>();
+  const TargetSubtargetInfo &TSI = MF.getSubtarget();
+  const TargetRegisterInfo &TRI = *TSI.getRegisterInfo();
+
+  // If predicates spills are 16-bytes we may need to expand
+  // SPILL_PPR_TO_ZPR_SLOT_PSEUDO/FILL_PPR_FROM_ZPR_SLOT_PSEUDO.
+  if (AFI->hasStackFrame() && TRI.getSpillSize(AArch64::PPRRegClass) == 16) {
+    auto ComputeScavengeableRegisters = [&](unsigned RegClassID) {
+      BitVector Regs = TRI.getAllocatableSet(MF, TRI.getRegClass(RegClassID));
+      assert(Regs.count() > 0 && "Expected scavengeable registers");
+      return Regs;
+    };
+
+    ScavengeableRegs SR{};
+    SR.ZPRRegs = ComputeScavengeableRegisters(AArch64::ZPRRegClassID);
+    // Only p0-7 are possible as the second operand of cmpne (needed for fills).
+    SR.PPR3bRegs = ComputeScavengeableRegisters(AArch64::PPR_3bRegClassID);
+    SR.GPRRegs = ComputeScavengeableRegisters(AArch64::GPR64RegClassID);
+
+    EmergencyStackSlots SpillSlots;
+    for (MachineBasicBlock &MBB : MF) {
+      // In the case we had to spill a predicate (in the range p0-p7) to reload
+      // a predicate (>= p8), additional spill/fill pseudos will be created.
+      // These need an additional expansion pass. Note: There will only be at
+      // most two expansion passes, as spilling/filling a predicate in the range
+      // p0-p7 never requires spilling another predicate.
+      for (int Pass = 0; Pass < 2; Pass++) {
+        bool HasPPRSpills =
+            expandSMEPPRToZPRSpillPseudos(MBB, TRI, SR, SpillSlots);
+        assert((Pass == 0 || !HasPPRSpills) && "Did not expect PPR spills");
+        if (!HasPPRSpills)
+          break;
+      }
+    }
+  }
+
   MachineFrameInfo &MFI = MF.getFrameInfo();
 
   assert(getStackGrowthDirection() == TargetFrameLowering::StackGrowsDown &&
@@ -4166,7 +4487,6 @@ void AArch64FrameLowering::processFunctionBeforeFrameFinalized(
   int64_t SVEStackSize =
       assignSVEStackObjectOffsets(MFI, MinCSFrameIndex, MaxCSFrameIndex);
 
-  AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>();
   AFI->setStackSizeSVE(alignTo(SVEStackSize, 16U));
   AFI->setMinMaxSVECSFrameIndex(MinCSFrameIndex, MaxCSFrameIndex);
 
@@ -5200,9 +5520,13 @@ void AArch64FrameLowering::emitRemarks(
 
           unsigned RegTy = StackAccess::AccessType::GPR;
           if (MFI.getStackID(FrameIdx) == TargetStackID::ScalableVector) {
-            if (AArch64::PPRRegClass.contains(MI.getOperand(0).getReg()))
+            // SPILL_PPR_TO_ZPR_SLOT_PSEUDO and FILL_PPR_FROM_ZPR_SLOT_PSEUDO
+            // spill/fill the predicate as a data vector (so are an FPR acess).
+            if (MI.getOpcode() != AArch64::SPILL_PPR_TO_ZPR_SLOT_PSEUDO &&
+                MI.getOpcode() != AArch64::FILL_PPR_FROM_ZPR_SLOT_PSEUDO &&
+                AArch64::PPRRegClass.contains(MI.getOperand(0).getReg())) {
               RegTy = StackAccess::PPR;
-            else
+            } else
               RegTy = StackAccess::FPR;
           } else if (AArch64InstrInfo::isFpOrNEON(MI)) {
             RegTy = StackAccess::FPR;
diff --git a/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp b/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp
index 6aa8cd4f0232a..1387a224fa660 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp
@@ -665,6 +665,10 @@ static AArch64_AM::ShiftExtendType getShiftTypeForNode(SDValue N) {
   }
 }
 
+static bool isMemOpOrPrefetch(SDNode *N) {
+  return isa<MemSDNode>(*N) || N->getOpcode() == AArch64ISD::PREFETCH;
+}
+
 /// Determine whether it is worth it to fold SHL into the addressing
 /// mode.
 static bool isWorthFoldingSHL(SDValue V) {
@@ -682,9 +686,9 @@ static bool isWorthFoldingSHL(SDValue V) {
   // computation, since the computation will be kept.
   const SDNode *Node = V.getNode();
   for (SDNode *UI : Node->users())
-    if (!isa<MemSDNode>(*UI))
+    if (!isMemOpOrPrefetch(UI))
       for (SDNode *UII : UI->users())
-        if (!isa<MemSDNode>(*UII))
+        if (!isMemOpOrPrefetch(UII))
           return false;
   return true;
 }
@@ -1248,7 +1252,7 @@ bool AArch64DAGToDAGISel::SelectAddrModeWRO(SDValue N, unsigned Size,
   // computation, since the computation will be kept.
   const SDNode *Node = N.getNode();
   for (SDNode *UI : Node->users()) {
-    if (!isa<MemSDNode>(*UI))
+    if (!isMemOpOrPrefetch(UI))
       return false;
   }
 
@@ -1332,7 +1336,7 @@ bool AArch64DAGToDAGISel::SelectAddrModeXRO(SDValue N, unsigned Size,
   // computation, since the computation will be kept.
   const SDNode *Node = N.getNode();
   for (SDNode *UI : Node->users()) {
-    if (!isa<MemSDNode>(*UI))
+    if (!isMemOpOrPrefetch(UI))
       return false;
   }
 
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index 4ede1fb93fe5f..4263be1098899 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -530,6 +530,9 @@ AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM,
   setOperationAction(ISD::XOR, MVT::i32, Custom);
   setOperationAction(ISD::XOR, MVT::i64, Custom);
 
+  setOperationAction(ISD::ADDRSPACECAST, MVT::i32, Custom);
+  setOperationAction(ISD::ADDRSPACECAST, MVT::i64, Custom);
+
   // Virtually no operation on f128 is legal, but LLVM can't expand them when
   // there's a valid register class, so we need custom operations in most cases.
   setOperationAction(ISD::FABS, MVT::f128, Expand);
@@ -732,19 +735,20 @@ AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM,
     setOperationAction(ISD::FCOPYSIGN, MVT::bf16, Promote);
   }
 
-  for (auto Op : {ISD::FREM,          ISD::FPOW,          ISD::FPOWI,
-                  ISD::FCOS,          ISD::FSIN,          ISD::FSINCOS,
-                  ISD::FACOS,         ISD::FASIN,         ISD::FATAN,
-                  ISD::FATAN2,        ISD::FCOSH,         ISD::FSINH,
-                  ISD::FTANH,         ISD::FTAN,          ISD::FEXP,
-                  ISD::FEXP2,         ISD::FEXP10,        ISD::FLOG,
-                  ISD::FLOG2,         ISD::FLOG10,        ISD::STRICT_FREM,
-                  ISD::STRICT_FPOW,   ISD::STRICT_FPOWI,  ISD::STRICT_FCOS,
-                  ISD::STRICT_FSIN,   ISD::STRICT_FACOS,  ISD::STRICT_FASIN,
-                  ISD::STRICT_FATAN,  ISD::STRICT_FATAN2, ISD::STRICT_FCOSH,
-                  ISD::STRICT_FSINH,  ISD::STRICT_FTANH,  ISD::STRICT_FEXP,
-                  ISD::STRICT_FEXP2,  ISD::STRICT_FLOG,   ISD::STRICT_FLOG2,
-                  ISD::STRICT_FLOG10, ISD::STRICT_FTAN}) {
+  for (auto Op : {ISD::FREM,          ISD::FPOW,         ISD::FPOWI,
+                  ISD::FCOS,          ISD::FSIN,         ISD::FSINCOS,
+                  ISD::FSINCOSPI,     ISD::FMODF,        ISD::FACOS,
+                  ISD::FASIN,         ISD::FATAN,        ISD::FATAN2,
+                  ISD::FCOSH,         ISD::FSINH,        ISD::FTANH,
+                  ISD::FTAN,          ISD::FEXP,         ISD::FEXP2,
+                  ISD::FEXP10,        ISD::FLOG,         ISD::FLOG2,
+                  ISD::FLOG10,        ISD::STRICT_FREM,  ISD::STRICT_FPOW,
+                  ISD::STRICT_FPOWI,  ISD::STRICT_FCOS,  ISD::STRICT_FSIN,
+                  ISD::STRICT_FACOS,  ISD::STRICT_FASIN, ISD::STRICT_FATAN,
+                  ISD::STRICT_FATAN2, ISD::STRICT_FCOSH, ISD::STRICT_FSINH,
+                  ISD::STRICT_FTANH,  ISD::STRICT_FEXP,  ISD::STRICT_FEXP2,
+                  ISD::STRICT_FLOG,   ISD::STRICT_FLOG2, ISD::STRICT_FLOG10,
+                  ISD::STRICT_FTAN}) {
     setOperationAction(Op, MVT::f16, Promote);
     setOperationAction(Op, MVT::v4f16, Expand);
     setOperationAction(Op, MVT::v8f16, Expand);
@@ -1140,6 +1144,8 @@ AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM,
 
   setTargetDAGCombine(ISD::SCALAR_TO_VECTOR);
 
+  setTargetDAGCombine(ISD::SHL);
+
   // In case of strict alignment, avoid an excessive number of byte wide stores.
   MaxStoresPerMemsetOptSize = 8;
   MaxStoresPerMemset =
@@ -1203,7 +1209,7 @@ AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM,
           ISD::FSIN,              ISD::FCOS,           ISD::FTAN,
           ISD::FASIN,             ISD::FACOS,          ISD::FATAN,
           ISD::FSINH,             ISD::FCOSH,          ISD::FTANH,
-          ISD::FPOW,              ISD::FLOG,           ISD::FLOG2,          
+          ISD::FPOW,              ISD::FLOG,           ISD::FLOG2,
           ISD::FLOG10,            ISD::FEXP,           ISD::FEXP2,
           ISD::FEXP10,            ISD::FRINT,          ISD::FROUND,
           ISD::FROUNDEVEN,        ISD::FTRUNC,         ISD::FMINNUM,
@@ -1212,7 +1218,7 @@ AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM,
           ISD::STRICT_FADD,       ISD::STRICT_FSUB,    ISD::STRICT_FMUL,
           ISD::STRICT_FDIV,       ISD::STRICT_FMA,     ISD::STRICT_FCEIL,
           ISD::STRICT_FFLOOR,     ISD::STRICT_FSQRT,   ISD::STRICT_FRINT,
-          ISD::STRICT_FNEARBYINT, ISD::STRICT_FROUND,  ISD::STRICT_FTRUNC,  
+          ISD::STRICT_FNEARBYINT, ISD::STRICT_FROUND,  ISD::STRICT_FTRUNC,
           ISD::STRICT_FROUNDEVEN, ISD::STRICT_FMINNUM, ISD::STRICT_FMAXNUM,
           ISD::STRICT_FMINIMUM,   ISD::STRICT_FMAXIMUM})
       setOperationAction(Op, MVT::v1f64, Expand);
@@ -1447,6 +1453,8 @@ AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM,
       setOperationAction(ISD::VECTOR_DEINTERLEAVE, VT, Custom);
       setOperationAction(ISD::VECTOR_INTERLEAVE, VT, Custom);
     }
+    for (auto VT : {MVT::nxv16i1, MVT::nxv8i1, MVT::nxv4i1, MVT::nxv2i1})
+      setOperationAction(ISD::VECTOR_FIND_LAST_ACTIVE, VT, Legal);
   }
 
   if (Subtarget->isSVEorStreamingSVEAvailable()) {
@@ -6878,6 +6886,38 @@ static SDValue LowerTruncateVectorStore(SDLoc DL, StoreSDNode *ST,
                       ST->getBasePtr(), ST->getMemOperand());
 }
 
+static SDValue LowerADDRSPACECAST(SDValue Op, SelectionDAG &DAG) {
+  SDLoc dl(Op);
+  SDValue Src = Op.getOperand(0);
+  MVT DestVT = Op.getSimpleValueType();
+  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
+  AddrSpaceCastSDNode *N = cast<AddrSpaceCastSDNode>(Op.getNode());
+
+  unsigned SrcAS = N->getSrcAddressSpace();
+  unsigned DestAS = N->getDestAddressSpace();
+  assert(SrcAS != DestAS &&
+         "addrspacecast must be between different address spaces");
+  assert(TLI.getTargetMachine().getPointerSize(SrcAS) !=
+             TLI.getTargetMachine().getPointerSize(DestAS) &&
+         "addrspacecast must be between different ptr sizes");
+  (void)TLI;
+
+  if (SrcAS == ARM64AS::PTR32_SPTR) {
+    return DAG.getNode(ISD::SIGN_EXTEND, dl, DestVT, Src,
+                       DAG.getTargetConstant(0, dl, DestVT));
+  } else if (SrcAS == ARM64AS::PTR32_UPTR) {
+    return DAG.getNode(ISD::ZERO_EXTEND, dl, DestVT, Src,
+                       DAG.getTargetConstant(0, dl, DestVT));
+  } else if ((DestAS == ARM64AS::PTR32_SPTR) ||
+             (DestAS == ARM64AS::PTR32_UPTR)) {
+    SDValue Ext = DAG.getAnyExtOrTrunc(Src, dl, DestVT);
+    SDValue Trunc = DAG.getZeroExtendInReg(Ext, dl, DestVT);
+    return Trunc;
+  } else {
+    return Src;
+  }
+}
+
 // Custom lowering for any store, vector or scalar and/or default or with
 // a truncate operations.  Currently only custom lower truncate operation
 // from vector v4i16 to v4i8 or volatile stores of i128.
@@ -7539,6 +7579,8 @@ SDValue AArch64TargetLowering::LowerOperation(SDValue Op,
   case ISD::SIGN_EXTEND:
   case ISD::ZERO_EXTEND:
     return LowerFixedLengthVectorIntExtendToSVE(Op, DAG);
+  case ISD::ADDRSPACECAST:
+    return LowerADDRSPACECAST(Op, DAG);
   case ISD::SIGN_EXTEND_INREG: {
     // Only custom lower when ExtraVT has a legal byte based element type.
     EVT ExtraVT = cast<VTSDNode>(Op.getOperand(1))->getVT();
@@ -8748,56 +8790,6 @@ static bool checkZExtBool(SDValue Arg, const SelectionDAG &DAG) {
   return ZExtBool;
 }
 
-// The FORM_TRANSPOSED_REG_TUPLE pseudo should only be used if the
-// input operands are copy nodes where the source register is in a
-// StridedOrContiguous class. For example:
-//
-//   %3:zpr2stridedorcontiguous = LD1B_2Z_IMM_PSEUDO ..
-//   %4:zpr = COPY %3.zsub1:zpr2stridedorcontiguous
-//   %5:zpr = COPY %3.zsub0:zpr2stridedorcontiguous
-//   %6:zpr2stridedorcontiguous = LD1B_2Z_PSEUDO ..
-//   %7:zpr = COPY %6.zsub1:zpr2stridedorcontiguous
-//   %8:zpr = COPY %6.zsub0:zpr2stridedorcontiguous
-//   %9:zpr2mul2 = FORM_TRANSPOSED_REG_TUPLE_X2_PSEUDO %5:zpr, %8:zpr
-//
-bool shouldUseFormStridedPseudo(MachineInstr &MI) {
-  MachineRegisterInfo &MRI = MI.getMF()->getRegInfo();
-
-  const TargetRegisterClass *RegClass = nullptr;
-  switch (MI.getOpcode()) {
-  case AArch64::FORM_TRANSPOSED_REG_TUPLE_X2_PSEUDO:
-    RegClass = &AArch64::ZPR2StridedOrContiguousRegClass;
-    break;
-  case AArch64::FORM_TRANSPOSED_REG_TUPLE_X4_PSEUDO:
-    RegClass = &AArch64::ZPR4StridedOrContiguousRegClass;
-    break;
-  default:
-    llvm_unreachable("Unexpected opcode.");
-  }
-
-  MCRegister SubReg = MCRegister::NoRegister;
-  for (unsigned I = 1; I < MI.getNumOperands(); ++I) {
-    MachineOperand &MO = MI.getOperand(I);
-    assert(MO.isReg() && "Unexpected operand to FORM_TRANSPOSED_REG_TUPLE");
-
-    MachineOperand *Def = MRI.getOneDef(MO.getReg());
-    if (!Def || !Def->getParent()->isCopy())
-      return false;
-
-    const MachineOperand &CopySrc = Def->getParent()->getOperand(1);
-    unsigned OpSubReg = CopySrc.getSubReg();
-    if (SubReg == MCRegister::NoRegister)
-      SubReg = OpSubReg;
-
-    MachineOperand *CopySrcOp = MRI.getOneDef(CopySrc.getReg());
-    if (!CopySrcOp || !CopySrcOp->isReg() || OpSubReg != SubReg ||
-        MRI.getRegClass(CopySrcOp->getReg()) != RegClass)
-      return false;
-  }
-
-  return true;
-}
-
 void AArch64TargetLowering::AdjustInstrPostInstrSelection(MachineInstr &MI,
                                                           SDNode *Node) const {
   // Live-in physreg copies that are glued to SMSTART are applied as
@@ -8823,27 +8815,6 @@ void AArch64TargetLowering::AdjustInstrPostInstrSelection(MachineInstr &MI,
     }
   }
 
-  if (MI.getOpcode() == AArch64::FORM_TRANSPOSED_REG_TUPLE_X2_PSEUDO ||
-      MI.getOpcode() == AArch64::FORM_TRANSPOSED_REG_TUPLE_X4_PSEUDO) {
-    // If input values to the FORM_TRANSPOSED_REG_TUPLE pseudo aren't copies
-    // from a StridedOrContiguous class, fall back on REG_SEQUENCE node.
-    if (shouldUseFormStridedPseudo(MI))
-      return;
-
-    const TargetInstrInfo *TII = Subtarget->getInstrInfo();
-    MachineInstrBuilder MIB = BuildMI(*MI.getParent(), MI, MI.getDebugLoc(),
-                                      TII->get(TargetOpcode::REG_SEQUENCE),
-                                      MI.getOperand(0).getReg());
-
-    for (unsigned I = 1; I < MI.getNumOperands(); ++I) {
-      MIB.add(MI.getOperand(I));
-      MIB.addImm(AArch64::zsub0 + (I - 1));
-    }
-
-    MI.eraseFromParent();
-    return;
-  }
-
   // Add an implicit use of 'VG' for ADDXri/SUBXri, which are instructions that
   // have nothing to do with VG, were it not that they are used to materialise a
   // frame-address. If they contain a frame-index to a scalable vector, this
@@ -9576,7 +9547,8 @@ AArch64TargetLowering::LowerCall(CallLoweringInfo &CLI,
 
     DAG.addNoMergeSiteInfo(Ret.getNode(), CLI.NoMerge);
     DAG.addCallSiteInfo(Ret.getNode(), std::move(CSInfo));
-    if (CalledGlobal)
+    if (CalledGlobal &&
+        MF.getFunction().getParent()->getModuleFlag("import-call-optimization"))
       DAG.addCalledGlobal(Ret.getNode(), CalledGlobal, OpFlags);
     return Ret;
   }
@@ -9589,7 +9561,8 @@ AArch64TargetLowering::LowerCall(CallLoweringInfo &CLI,
   DAG.addNoMergeSiteInfo(Chain.getNode(), CLI.NoMerge);
   InGlue = Chain.getValue(1);
   DAG.addCallSiteInfo(Chain.getNode(), std::move(CSInfo));
-  if (CalledGlobal)
+  if (CalledGlobal &&
+      MF.getFunction().getParent()->getModuleFlag("import-call-optimization"))
     DAG.addCalledGlobal(Chain.getNode(), CalledGlobal, OpFlags);
 
   uint64_t CalleePopBytes =
@@ -15535,7 +15508,7 @@ bool AArch64TargetLowering::isShuffleMaskLegal(ArrayRef<int> M, EVT VT) const {
 
   unsigned EltSize = VT.getScalarSizeInBits();
   unsigned NumElts = VT.getVectorNumElements();
-  return (ShuffleVectorSDNode::isSplatMask(&M[0], VT) ||
+  return (ShuffleVectorSDNode::isSplatMask(M) ||
           isREVMask(M, EltSize, NumElts, 64) ||
           isREVMask(M, EltSize, NumElts, 32) ||
           isREVMask(M, EltSize, NumElts, 16) ||
@@ -19760,6 +19733,33 @@ performLastTrueTestVectorCombine(SDNode *N,
   return getPTest(DAG, N->getValueType(0), Pg, N0, AArch64CC::LAST_ACTIVE);
 }
 
+static SDValue
+performExtractLastActiveCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI,
+                                const AArch64Subtarget *Subtarget) {
+  assert(N->getOpcode() == ISD::EXTRACT_VECTOR_ELT);
+  SelectionDAG &DAG = DCI.DAG;
+  SDValue Vec = N->getOperand(0);
+  SDValue Idx = N->getOperand(1);
+
+  if (DCI.isBeforeLegalize() || Idx.getOpcode() != ISD::VECTOR_FIND_LAST_ACTIVE)
+    return SDValue();
+
+  // Only legal for 8, 16, 32, and 64 bit element types.
+  EVT EltVT = Vec.getValueType().getVectorElementType();
+  if (!is_contained(ArrayRef({MVT::i8, MVT::i16, MVT::i32, MVT::i64, MVT::f16,
+                              MVT::bf16, MVT::f32, MVT::f64}),
+                    EltVT.getSimpleVT().SimpleTy))
+    return SDValue();
+
+  SDValue Mask = Idx.getOperand(0);
+  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
+  if (!TLI.isOperationLegal(ISD::VECTOR_FIND_LAST_ACTIVE, Mask.getValueType()))
+    return SDValue();
+
+  return DAG.getNode(AArch64ISD::LASTB, SDLoc(N), N->getValueType(0), Mask,
+                     Vec);
+}
+
 static SDValue
 performExtractVectorEltCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI,
                                const AArch64Subtarget *Subtarget) {
@@ -19768,6 +19768,8 @@ performExtractVectorEltCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI,
     return Res;
   if (SDValue Res = performLastTrueTestVectorCombine(N, DCI, Subtarget))
     return Res;
+  if (SDValue Res = performExtractLastActiveCombine(N, DCI, Subtarget))
+    return Res;
 
   SelectionDAG &DAG = DCI.DAG;
   SDValue N0 = N->getOperand(0), N1 = N->getOperand(1);
@@ -22362,6 +22364,9 @@ static SDValue performZExtDeinterleaveShuffleCombine(SDNode *N,
   if (!IsDeInterleave)
     IsUndefDeInterleave =
         Shuffle->getOperand(1).isUndef() &&
+        all_of(
+            Shuffle->getMask().slice(ExtOffset, VT.getVectorNumElements() / 2),
+            [](int M) { return M < 0; }) &&
         ShuffleVectorInst::isDeInterleaveMaskOfFactor(
             Shuffle->getMask().slice(ExtOffset + VT.getVectorNumElements() / 2,
                                      VT.getVectorNumElements() / 2),
@@ -23556,6 +23561,26 @@ static SDValue performLOADCombine(SDNode *N,
     performTBISimplification(N->getOperand(1), DCI, DAG);
 
   LoadSDNode *LD = cast<LoadSDNode>(N);
+  EVT RegVT = LD->getValueType(0);
+  EVT MemVT = LD->getMemoryVT();
+  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
+  SDLoc DL(LD);
+
+  // Cast ptr32 and ptr64 pointers to the default address space before a load.
+  unsigned AddrSpace = LD->getAddressSpace();
+  if (AddrSpace == ARM64AS::PTR64 || AddrSpace == ARM64AS::PTR32_SPTR ||
+      AddrSpace == ARM64AS::PTR32_UPTR) {
+    MVT PtrVT = TLI.getPointerTy(DAG.getDataLayout());
+    if (PtrVT != LD->getBasePtr().getSimpleValueType()) {
+      SDValue Cast =
+          DAG.getAddrSpaceCast(DL, PtrVT, LD->getBasePtr(), AddrSpace, 0);
+      return DAG.getExtLoad(LD->getExtensionType(), DL, RegVT, LD->getChain(),
+                            Cast, LD->getPointerInfo(), MemVT,
+                            LD->getOriginalAlign(),
+                            LD->getMemOperand()->getFlags());
+    }
+  }
+
   if (LD->isVolatile() || !Subtarget->isLittleEndian())
     return SDValue(N, 0);
 
@@ -23565,13 +23590,11 @@ static SDValue performLOADCombine(SDNode *N,
   if (!LD->isNonTemporal())
     return SDValue(N, 0);
 
-  EVT MemVT = LD->getMemoryVT();
   if (MemVT.isScalableVector() || MemVT.getSizeInBits() <= 256 ||
       MemVT.getSizeInBits() % 256 == 0 ||
       256 % MemVT.getScalarSizeInBits() != 0)
     return SDValue(N, 0);
 
-  SDLoc DL(LD);
   SDValue Chain = LD->getChain();
   SDValue BasePtr = LD->getBasePtr();
   SDNodeFlags Flags = LD->getFlags();
@@ -23831,12 +23854,28 @@ static SDValue performSTORECombine(SDNode *N,
   SDValue Value = ST->getValue();
   SDValue Ptr = ST->getBasePtr();
   EVT ValueVT = Value.getValueType();
+  EVT MemVT = ST->getMemoryVT();
+  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
+  SDLoc DL(ST);
 
   auto hasValidElementTypeForFPTruncStore = [](EVT VT) {
     EVT EltVT = VT.getVectorElementType();
     return EltVT == MVT::f32 || EltVT == MVT::f64;
   };
 
+  // Cast ptr32 and ptr64 pointers to the default address space before a store.
+  unsigned AddrSpace = ST->getAddressSpace();
+  if (AddrSpace == ARM64AS::PTR64 || AddrSpace == ARM64AS::PTR32_SPTR ||
+      AddrSpace == ARM64AS::PTR32_UPTR) {
+    MVT PtrVT = TLI.getPointerTy(DAG.getDataLayout());
+    if (PtrVT != Ptr.getSimpleValueType()) {
+      SDValue Cast = DAG.getAddrSpaceCast(DL, PtrVT, Ptr, AddrSpace, 0);
+      return DAG.getStore(Chain, DL, Value, Cast, ST->getPointerInfo(),
+                          ST->getOriginalAlign(),
+                          ST->getMemOperand()->getFlags(), ST->getAAInfo());
+    }
+  }
+
   if (SDValue Res = combineI8TruncStore(ST, DAG, Subtarget))
     return Res;
 
@@ -23850,8 +23889,8 @@ static SDValue performSTORECombine(SDNode *N,
       ValueVT.isFixedLengthVector() &&
       ValueVT.getFixedSizeInBits() >= Subtarget->getMinSVEVectorSizeInBits() &&
       hasValidElementTypeForFPTruncStore(Value.getOperand(0).getValueType()))
-    return DAG.getTruncStore(Chain, SDLoc(N), Value.getOperand(0), Ptr,
-                             ST->getMemoryVT(), ST->getMemOperand());
+    return DAG.getTruncStore(Chain, DL, Value.getOperand(0), Ptr, MemVT,
+                             ST->getMemOperand());
 
   if (SDValue Split = splitStores(N, DCI, DAG, Subtarget))
     return Split;
@@ -24848,6 +24887,39 @@ static SDValue reassociateCSELOperandsForCSE(SDNode *N, SelectionDAG &DAG) {
   }
 }
 
+static SDValue foldCSELofLASTB(SDNode *Op, SelectionDAG &DAG) {
+  AArch64CC::CondCode OpCC =
+      static_cast<AArch64CC::CondCode>(Op->getConstantOperandVal(2));
+
+  if (OpCC != AArch64CC::NE)
+    return SDValue();
+
+  SDValue PTest = Op->getOperand(3);
+  if (PTest.getOpcode() != AArch64ISD::PTEST_ANY)
+    return SDValue();
+
+  SDValue TruePred = PTest.getOperand(0);
+  SDValue AnyPred = PTest.getOperand(1);
+
+  if (TruePred.getOpcode() == AArch64ISD::REINTERPRET_CAST)
+    TruePred = TruePred.getOperand(0);
+
+  if (AnyPred.getOpcode() == AArch64ISD::REINTERPRET_CAST)
+    AnyPred = AnyPred.getOperand(0);
+
+  if (TruePred != AnyPred && TruePred.getOpcode() != AArch64ISD::PTRUE)
+    return SDValue();
+
+  SDValue LastB = Op->getOperand(0);
+  SDValue Default = Op->getOperand(1);
+
+  if (LastB.getOpcode() != AArch64ISD::LASTB || LastB.getOperand(0) != AnyPred)
+    return SDValue();
+
+  return DAG.getNode(AArch64ISD::CLASTB_N, SDLoc(Op), Op->getValueType(0),
+                     AnyPred, Default, LastB.getOperand(1));
+}
+
 // Optimize CSEL instructions
 static SDValue performCSELCombine(SDNode *N,
                                   TargetLowering::DAGCombinerInfo &DCI,
@@ -24893,6 +24965,10 @@ static SDValue performCSELCombine(SDNode *N,
     }
   }
 
+  // CSEL (LASTB P, Z), X, NE(ANY P) -> CLASTB P, X, Z
+  if (SDValue CondLast = foldCSELofLASTB(N, DAG))
+    return CondLast;
+
   return performCONDCombine(N, DCI, DAG, 2, 3);
 }
 
@@ -26339,6 +26415,50 @@ performScalarToVectorCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI,
   return NVCAST;
 }
 
+/// If the operand is a bitwise AND with a constant RHS, and the shift has a
+/// constant RHS and is the only use, we can pull it out of the shift, i.e.
+///
+///   (shl (and X, C1), C2) -> (and (shl X, C2), (shl C1, C2))
+///
+/// We prefer this canonical form to match existing isel patterns.
+static SDValue performSHLCombine(SDNode *N,
+                                 TargetLowering::DAGCombinerInfo &DCI,
+                                 SelectionDAG &DAG) {
+  if (DCI.isBeforeLegalizeOps())
+    return SDValue();
+
+  SDValue Op0 = N->getOperand(0);
+  if (Op0.getOpcode() != ISD::AND || !Op0.hasOneUse())
+    return SDValue();
+
+  SDValue C1 = Op0->getOperand(1);
+  SDValue C2 = N->getOperand(1);
+  if (!isa<ConstantSDNode>(C1) || !isa<ConstantSDNode>(C2))
+    return SDValue();
+
+  // Might be folded into shifted op, do not lower.
+  if (N->hasOneUse()) {
+    unsigned UseOpc = N->user_begin()->getOpcode();
+    if (UseOpc == ISD::ADD || UseOpc == ISD::SUB || UseOpc == ISD::SETCC ||
+        UseOpc == AArch64ISD::ADDS || UseOpc == AArch64ISD::SUBS)
+      return SDValue();
+  }
+
+  SDLoc DL(N);
+  EVT VT = N->getValueType(0);
+
+  // Don't combine unless (shl C1, C2) can be constant folded. Otherwise,
+  // DAGCombiner will simplify (and (op x...), (op y...)) -> (op (and x, y))
+  // causing infinite loop. Result may also be worse.
+  SDValue NewRHS = DAG.getNode(ISD::SHL, DL, VT, C1, C2);
+  if (!isa<ConstantSDNode>(NewRHS))
+    return SDValue();
+
+  SDValue X = Op0->getOperand(0);
+  SDValue NewShift = DAG.getNode(ISD::SHL, DL, VT, X, C2);
+  return DAG.getNode(ISD::AND, DL, VT, NewShift, NewRHS);
+}
+
 SDValue AArch64TargetLowering::PerformDAGCombine(SDNode *N,
                                                  DAGCombinerInfo &DCI) const {
   SelectionDAG &DAG = DCI.DAG;
@@ -26684,6 +26804,8 @@ SDValue AArch64TargetLowering::PerformDAGCombine(SDNode *N,
     return performCTLZCombine(N, DAG, Subtarget);
   case ISD::SCALAR_TO_VECTOR:
     return performScalarToVectorCombine(N, DCI, DAG);
+  case ISD::SHL:
+    return performSHLCombine(N, DCI, DAG);
   }
   return SDValue();
 }
@@ -27346,6 +27468,11 @@ void AArch64TargetLowering::ReplaceNodeResults(
     ReplaceATOMIC_LOAD_128Results(N, Results, DAG, Subtarget);
     return;
   }
+  case ISD::ADDRSPACECAST: {
+    SDValue V = LowerADDRSPACECAST(SDValue(N, 0), DAG);
+    Results.push_back(V);
+    return;
+  }
   case ISD::ATOMIC_LOAD:
   case ISD::LOAD: {
     MemSDNode *LoadNode = cast<MemSDNode>(N);
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.h b/llvm/lib/Target/AArch64/AArch64ISelLowering.h
index 470ed2a06b706..b26f28dc79f88 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.h
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.h
@@ -563,6 +563,10 @@ const unsigned StackProbeMaxLoopUnroll = 4;
 
 } // namespace AArch64
 
+namespace ARM64AS {
+enum : unsigned { PTR32_SPTR = 270, PTR32_UPTR = 271, PTR64 = 272 };
+}
+
 class AArch64Subtarget;
 
 class AArch64TargetLowering : public TargetLowering {
@@ -594,11 +598,19 @@ class AArch64TargetLowering : public TargetLowering {
                                            unsigned Depth) const override;
 
   MVT getPointerTy(const DataLayout &DL, uint32_t AS = 0) const override {
-    // Returning i64 unconditionally here (i.e. even for ILP32) means that the
-    // *DAG* representation of pointers will always be 64-bits. They will be
-    // truncated and extended when transferred to memory, but the 64-bit DAG
-    // allows us to use AArch64's addressing modes much more easily.
-    return MVT::getIntegerVT(64);
+    if ((AS == ARM64AS::PTR32_SPTR) || (AS == ARM64AS::PTR32_UPTR)) {
+      // These are 32-bit pointers created using the `__ptr32` extension or
+      // similar. They are handled by marking them as being in a different
+      // address space, and will be extended to 64-bits when used as the target
+      // of a load or store operation, or cast to a 64-bit pointer type.
+      return MVT::i32;
+    } else {
+      // Returning i64 unconditionally here (i.e. even for ILP32) means that the
+      // *DAG* representation of pointers will always be 64-bits. They will be
+      // truncated and extended when transferred to memory, but the 64-bit DAG
+      // allows us to use AArch64's addressing modes much more easily.
+      return MVT::i64;
+    }
   }
 
   bool targetShrinkDemandedConstant(SDValue Op, const APInt &DemandedBits,
diff --git a/llvm/lib/Target/AArch64/AArch64InstrAtomics.td b/llvm/lib/Target/AArch64/AArch64InstrAtomics.td
index de94cf64c9801..5e6db9d007a55 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrAtomics.td
+++ b/llvm/lib/Target/AArch64/AArch64InstrAtomics.td
@@ -575,7 +575,7 @@ let Predicates = [HasRCPC3, HasNEON] in {
 }
 
 // v8.4a FEAT_LRCPC2 patterns
-let Predicates = [HasRCPC_IMMO] in {
+let Predicates = [HasRCPC_IMMO, UseLDAPUR] in {
   // Load-Acquire RCpc Register unscaled loads
   def : Pat<(acquiring_load<atomic_load_az_8>
                (am_unscaled8 GPR64sp:$Rn, simm9:$offset)),
@@ -589,7 +589,9 @@ let Predicates = [HasRCPC_IMMO] in {
   def : Pat<(acquiring_load<atomic_load_64>
                (am_unscaled64 GPR64sp:$Rn, simm9:$offset)),
           (LDAPURXi GPR64sp:$Rn, simm9:$offset)>;
+}
 
+let Predicates = [HasRCPC_IMMO] in {
   // Store-Release Register unscaled stores
   def : Pat<(releasing_store<atomic_store_8>
                (am_unscaled8 GPR64sp:$Rn, simm9:$offset), GPR32:$val),
diff --git a/llvm/lib/Target/AArch64/AArch64InstrFormats.td b/llvm/lib/Target/AArch64/AArch64InstrFormats.td
index 6a3a9492e031c..d13bb908df78e 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrFormats.td
+++ b/llvm/lib/Target/AArch64/AArch64InstrFormats.td
@@ -5511,6 +5511,15 @@ multiclass IntegerToFPSIMDScalar<bits<2> rmode, bits<3> opcode, string asm, SDPa
     let Inst{31} = 1; // 64-bit FPR flag
     let Inst{23-22} = 0b00; // 32-bit FPR flag
   }
+
+  def : Pat<(f16 (node (i32 (extractelt (v4i32 V128:$Rn), (i64 0))))),
+          (!cast<Instruction>(NAME # HSr) (EXTRACT_SUBREG $Rn, ssub))>;
+  def : Pat<(f64 (node (i32 (extractelt (v4i32 V128:$Rn), (i64 0))))),
+          (!cast<Instruction>(NAME # DSr) (EXTRACT_SUBREG $Rn, ssub))>;
+  def : Pat<(f16 (node (i64 (extractelt (v2i64 V128:$Rn), (i64 0))))),
+          (!cast<Instruction>(NAME # HDr) (EXTRACT_SUBREG $Rn, dsub))>;
+  def : Pat<(f32 (node (i64 (extractelt (v2i64 V128:$Rn), (i64 0))))),
+          (!cast<Instruction>(NAME # SDr) (EXTRACT_SUBREG $Rn, dsub))>;
 }
 
 //---
@@ -6519,14 +6528,15 @@ multiclass SIMDThreeSameVectorFML<bit U, bit b13, bits<3> size, string asm,
                                          v4f32, v8f16, OpNode>;
 }
 
-multiclass SIMDThreeSameVectorMLA<bit Q, string asm>{
+multiclass SIMDThreeSameVectorMLA<bit Q, string asm, SDPatternOperator op> {
+
   def v8f16 : BaseSIMDThreeSameVectorDot<Q, 0b0, 0b11, 0b1111, asm, ".8h", ".16b",
-                                         V128, v8f16, v16i8, null_frag>;
+                                         V128, v8f16, v16i8, op>;
 }
 
-multiclass SIMDThreeSameVectorMLAL<bit Q, bits<2> sz, string asm>{
+multiclass SIMDThreeSameVectorMLAL<bit Q, bits<2> sz, string asm, SDPatternOperator op> {
   def v4f32 : BaseSIMDThreeSameVectorDot<Q, 0b0, sz, 0b1000, asm, ".4s", ".16b",
-                                         V128, v4f32, v16i8, null_frag>;
+                                         V128, v4f32, v16i8, op>;
 }
 
 // FP8 assembly/disassembly classes
@@ -6559,32 +6569,48 @@ class BaseSIMDThreeVectors<bit Q, bit U, bits<2> size, bits<4> op,
 
 
 // FCVTN (FP16 to FP8)
-multiclass SIMDThreeSameSizeVectorCvt<string asm> {
-   def v8f8 : BaseSIMDThreeVectors<0b0, 0b0, 0b01, 0b1110, V64, V64, asm, ".8b",".4h">;
-   def v16f8 : BaseSIMDThreeVectors<0b1, 0b0, 0b01, 0b1110,  V128, V128, asm, ".16b", ".8h">;
+multiclass SIMD_FP8_CVTN_F16<string asm, SDPatternOperator Op> {
+  let Uses = [FPMR, FPCR], mayLoad = 1 in {
+    def v8f8 : BaseSIMDThreeVectors<0b0, 0b0, 0b01, 0b1110, V64, V64, asm, ".8b",".4h">;
+    def v16f8 : BaseSIMDThreeVectors<0b1, 0b0, 0b01, 0b1110,  V128, V128, asm, ".16b", ".8h">;
+  }
+  def : Pat<(v8i8 (Op (v4f16 V64:$Rn), (v4f16 V64:$Rm))),
+            (!cast<Instruction>(NAME # v8f8) V64:$Rn, V64:$Rm)>;
+  def : Pat<(v16i8 (Op (v8f16 V128:$Rn), (v8f16 V128:$Rm))),
+            (!cast<Instruction>(NAME # v16f8) V128:$Rn, V128:$Rm)>;
 }
 
-// TODO : Create v16f8 value type
 // FCVTN, FCVTN2 (FP32 to FP8)
-multiclass SIMDThreeVectorCvt<string asm> {
-   def v8f8 : BaseSIMDThreeVectors<0b0, 0b0, 0b00, 0b1110, V64, V128, asm, ".8b", ".4s">;
-   def 2v16f8 : BaseSIMDThreeSameVectorDot<0b1, 0b0, 0b00, 0b1110, asm#2, ".16b", ".4s",
-                                           V128, v16i8, v4f32, null_frag>;
+multiclass SIMD_FP8_CVTN_F32<string asm, SDPatternOperator Op> {
+  let Uses = [FPMR, FPCR], mayLoad = 1 in {
+    def v8f8 : BaseSIMDThreeVectors<0b0, 0b0, 0b00, 0b1110, V64, V128, asm, ".8b", ".4s">;
+    def 2v16f8 : BaseSIMDThreeSameVectorDot<0b1, 0b0, 0b00, 0b1110, asm#2, ".16b", ".4s",
+                                            V128, v16i8, v4f32, null_frag>;
+  }
+
+  def : Pat<(v8i8 (Op (v4f32 V128:$Rn), (v4f32 V128:$Rm))),
+            (!cast<Instruction>(NAME # v8f8) V128:$Rn, V128:$Rm)>;
+
+  def : Pat<(v16i8 (!cast<SDPatternOperator>(Op # 2) (v16i8 V128:$_Rd), (v4f32 V128:$Rn), (v4f32 V128:$Rm))),
+            (!cast<Instruction>(NAME # 2v16f8) V128:$_Rd, V128:$Rn, V128:$Rm)>;
 }
 
-// TODO: Create a new Value Type v8f8 and v16f8
-multiclass SIMDThreeSameVectorDOT2<string asm> {
-   def v4f16 : BaseSIMDThreeSameVectorDot<0b0, 0b0, 0b01, 0b1111, asm, ".4h", ".8b",
-                                          V64, v4f16, v8i8, null_frag>;
-   def v8f16 : BaseSIMDThreeSameVectorDot<0b1, 0b0, 0b01, 0b1111, asm, ".8h", ".16b",
-                                          V128, v8f16, v16i8, null_frag>;
+multiclass SIMD_FP8_Dot2<string asm, SDPatternOperator op> {
+  let Uses = [FPMR, FPCR], mayLoad = 1 in {
+    def v4f16 : BaseSIMDThreeSameVectorDot<0b0, 0b0, 0b01, 0b1111, asm, ".4h", ".8b",
+                                           V64, v4f16, v8i8, op>;
+    def v8f16 : BaseSIMDThreeSameVectorDot<0b1, 0b0, 0b01, 0b1111, asm, ".8h", ".16b",
+                                           V128, v8f16, v16i8, op>;
+  }
 }
 
-multiclass SIMDThreeSameVectorDOT4<string asm> {
-   def v2f32 : BaseSIMDThreeSameVectorDot<0b0, 0b0, 0b00, 0b1111, asm, ".2s", ".8b",
-                                          V64, v2f32, v8i8, null_frag>;
-   def v4f32 : BaseSIMDThreeSameVectorDot<0b1, 0b0, 0b00, 0b1111, asm, ".4s", ".16b",
-                                          V128, v4f32, v16i8, null_frag>;
+multiclass SIMD_FP8_Dot4<string asm, SDPatternOperator op> {
+  let Uses = [FPMR, FPCR], mayLoad = 1 in {
+    def v2f32 : BaseSIMDThreeSameVectorDot<0b0, 0b0, 0b00, 0b1111, asm, ".2s", ".8b",
+                                           V64, v2f32, v8i8, op>;
+    def v4f32 : BaseSIMDThreeSameVectorDot<0b1, 0b0, 0b00, 0b1111, asm, ".4s", ".16b",
+                                           V128, v4f32, v16i8, op>;
+  }
 }
 
 let mayRaiseFPException = 1, Uses = [FPCR] in
@@ -7033,11 +7059,18 @@ multiclass SIMDMixedTwoVector<bit U, bits<5> opc, string asm,
 //----------------------------------------------------------------------------
 // FP8 Advanced SIMD two-register miscellaneous
 //----------------------------------------------------------------------------
-multiclass SIMDMixedTwoVectorFP8<bits<2>sz, string asm> {
-  def v8f16 : BaseSIMDMixedTwoVector<0b0, 0b1, sz, 0b10111, V64, V128,
-                                     asm, ".8h", ".8b", []>;
-  def 2v8f16 : BaseSIMDMixedTwoVector<0b1, 0b1, sz, 0b10111, V128, V128,
-                                     asm#2, ".8h", ".16b", []>;
+multiclass SIMD_FP8_CVTL<bits<2>sz, string asm, ValueType dty, SDPatternOperator Op> {
+  let Uses=[FPMR, FPCR], mayLoad = 1 in {
+    def NAME : BaseSIMDMixedTwoVector<0b0, 0b1, sz, 0b10111, V64, V128,
+                                      asm, ".8h", ".8b", []>;
+    def NAME#2 : BaseSIMDMixedTwoVector<0b1, 0b1, sz, 0b10111, V128, V128,
+                                        asm#2, ".8h", ".16b", []>;
+  }
+  def : Pat<(dty (Op (v8i8 V64:$Rn))),
+            (!cast<Instruction>(NAME) V64:$Rn)>;
+
+  def : Pat<(dty (Op (v16i8 V128:$Rn))),
+            (!cast<Instruction>(NAME#2) V128:$Rn)>;
 }
 
 class BaseSIMDCmpTwoVector<bit Q, bit U, bits<2> size, bits<2> size2,
@@ -9081,7 +9114,7 @@ class BaseSIMDThreeSameVectorIndexB<bit Q, bit U, bits<2> sz, bits<4> opc,
                                     RegisterOperand RegType,
                                     RegisterOperand RegType_lo>
   : BaseSIMDIndexedTied<Q, U, 0b0, sz, opc,
-                        RegType, RegType, RegType_lo, VectorIndexB,
+                        RegType, RegType, RegType_lo, VectorIndexB32b_timm,
                         asm, "", dst_kind, ".16b", ".b", []> {
 
   // idx = H:L:M
@@ -9090,14 +9123,24 @@ class BaseSIMDThreeSameVectorIndexB<bit Q, bit U, bits<2> sz, bits<4> opc,
   let Inst{21-19} = idx{2-0};
 }
 
-multiclass SIMDThreeSameVectorMLAIndex<bit Q, string asm> {
-  def v8f16 : BaseSIMDThreeSameVectorIndexB<Q, 0b0, 0b11, 0b0000, asm, ".8h",
-                                            V128, V128_0to7>;
+multiclass SIMDThreeSameVectorMLAIndex<bit Q, string asm, SDPatternOperator op> {
+  let Uses = [FPMR, FPCR], mayLoad = 1 in {
+    def v8f16 : BaseSIMDThreeSameVectorIndexB<Q, 0b0, 0b11, 0b0000, asm, ".8h",
+                                              V128, V128_0to7>;
+  }
+
+  def : Pat<(v8f16 (op (v8f16 V128:$Rd), (v16i8 V128:$Rn), (v16i8 V128_0to7:$Rm), VectorIndexB32b_timm:$Idx)),
+            (!cast<Instruction>(NAME # v8f16) $Rd, $Rn, $Rm, $Idx)>;
 }
 
-multiclass SIMDThreeSameVectorMLALIndex<bit Q, bits<2> sz, string asm> {
-  def v4f32 : BaseSIMDThreeSameVectorIndexB<Q, 0b1, sz, 0b1000, asm, ".4s",
-                                            V128, V128_0to7>;
+multiclass SIMDThreeSameVectorMLALIndex<bit Q, bits<2> sz, string asm, SDPatternOperator op> {
+  let Uses = [FPMR, FPCR], mayLoad = 1 in {
+    def v4f32 : BaseSIMDThreeSameVectorIndexB<Q, 0b1, sz, 0b1000, asm, ".4s",
+                                              V128, V128_0to7>;
+  }
+
+  def : Pat<(v4f32 (op (v4f32 V128:$Rd), (v16i8 V128:$Rn), (v16i8 V128_0to7:$Rm), VectorIndexB32b_timm:$Idx)),
+            (!cast<Instruction>(NAME # v4f32) $Rd, $Rn, $Rm, $Idx)>;
 }
 
 //----------------------------------------------------------------------------
@@ -9118,15 +9161,16 @@ class BaseSIMDThreeSameVectorIndexS<bit Q, bit U, bits<2> size, bits<4> opc, str
                                     string dst_kind, string lhs_kind, string rhs_kind,
                                     RegisterOperand RegType,
                                     ValueType AccumType, ValueType InputType,
+                                    AsmVectorIndexOpnd VIdx,
                                     SDPatternOperator OpNode> :
         BaseSIMDIndexedTied<Q, U, 0b0, size, opc, RegType, RegType, V128,
-                            VectorIndexS, asm, "", dst_kind, lhs_kind, rhs_kind,
+                            VIdx, asm, "", dst_kind, lhs_kind, rhs_kind,
         [(set (AccumType RegType:$dst),
               (AccumType (OpNode (AccumType RegType:$Rd),
                                  (InputType RegType:$Rn),
                                  (InputType (bitconvert (AccumType
                                     (AArch64duplane32 (v4i32 V128:$Rm),
-                                        VectorIndexS:$idx)))))))]> {
+                                        VIdx:$idx)))))))]> {
   bits<2> idx;
   let Inst{21}    = idx{0};  // L
   let Inst{11}    = idx{1};  // H
@@ -9135,17 +9179,24 @@ class BaseSIMDThreeSameVectorIndexS<bit Q, bit U, bits<2> size, bits<4> opc, str
 multiclass SIMDThreeSameVectorDotIndex<bit U, bit Mixed, bits<2> size, string asm,
                                        SDPatternOperator OpNode> {
   def v8i8  : BaseSIMDThreeSameVectorIndexS<0, U, size, {0b111, Mixed}, asm, ".2s", ".8b", ".4b",
-                                              V64, v2i32, v8i8, OpNode>;
+                                              V64, v2i32, v8i8, VectorIndexS, OpNode>;
   def v16i8 : BaseSIMDThreeSameVectorIndexS<1, U, size, {0b111, Mixed}, asm, ".4s", ".16b", ".4b",
-                                              V128, v4i32, v16i8, OpNode>;
+                                              V128, v4i32, v16i8, VectorIndexS, OpNode>;
 }
 
-// TODO: The vectors v8i8 and v16i8 should be v8f8 and v16f8
-multiclass SIMDThreeSameVectorFP8DOT4Index<string asm> {
-  def v8f8 : BaseSIMDThreeSameVectorIndexS<0b0, 0b0, 0b00, 0b0000, asm, ".2s", ".8b", ".4b",
-                                           V64, v2f32, v8i8, null_frag>;
-  def v16f8 : BaseSIMDThreeSameVectorIndexS<0b1, 0b0, 0b00, 0b0000, asm, ".4s", ".16b",".4b",
-                                            V128, v4f32, v16i8, null_frag>;
+multiclass SIMD_FP8_Dot4_Index<string asm, SDPatternOperator op> {
+  let Uses = [FPMR, FPCR], mayLoad = 1 in {
+    def v2f32 : BaseSIMDThreeSameVectorIndexS<0b0, 0b0, 0b00, 0b0000, asm, ".2s", ".8b", ".4b",
+                                              V64, v2f32, v8i8, VectorIndexS32b_timm, null_frag>;
+    def v4f32 : BaseSIMDThreeSameVectorIndexS<0b1, 0b0, 0b00, 0b0000, asm, ".4s", ".16b",".4b",
+                                              V128, v4f32, v16i8, VectorIndexS32b_timm, null_frag>;
+  }
+
+  def : Pat<(v2f32 (op (v2f32 V64:$Rd), (v8i8 V64:$Rn), (v16i8 V128:$Rm), VectorIndexS32b_timm:$Idx)),
+            (!cast<Instruction>(NAME # v2f32) $Rd, $Rn, $Rm, $Idx)>;
+
+  def : Pat<(v4f32 (op (v4f32 V128:$Rd), (v16i8 V128:$Rn), (v16i8 V128:$Rm), VectorIndexS32b_timm:$Idx)),
+            (!cast<Instruction>(NAME # v4f32) $Rd, $Rn, $Rm, $Idx)>;
 }
 
 // ARMv8.2-A Fused Multiply Add-Long Instructions (Indexed)
@@ -9154,14 +9205,15 @@ class BaseSIMDThreeSameVectorIndexH<bit Q, bit U, bits<2> sz, bits<4> opc, strin
                                       string dst_kind, string lhs_kind,
                                       string rhs_kind, RegisterOperand RegType,
                                       RegisterOperand RegType_lo, ValueType AccumType,
-                                      ValueType InputType, SDPatternOperator OpNode> :
+                                      ValueType InputType, AsmVectorIndexOpnd VIdx,
+                                      SDPatternOperator OpNode> :
         BaseSIMDIndexedTied<Q, U, 0, sz, opc, RegType, RegType, RegType_lo,
-                            VectorIndexH, asm, "", dst_kind, lhs_kind, rhs_kind,
+                            VIdx, asm, "", dst_kind, lhs_kind, rhs_kind,
           [(set (AccumType RegType:$dst),
                 (AccumType (OpNode (AccumType RegType:$Rd),
                                    (InputType RegType:$Rn),
                                    (InputType (AArch64duplane16 (v8f16 V128_lo:$Rm),
-                                                VectorIndexH:$idx)))))]> {
+                                                VIdx:$idx)))))]> {
   // idx = H:L:M
   bits<3> idx;
   let Inst{11} = idx{2}; // H
@@ -9172,19 +9224,25 @@ class BaseSIMDThreeSameVectorIndexH<bit Q, bit U, bits<2> sz, bits<4> opc, strin
 multiclass SIMDThreeSameVectorFMLIndex<bit U, bits<4> opc, string asm,
                                        SDPatternOperator OpNode> {
   def v4f16 : BaseSIMDThreeSameVectorIndexH<0, U, 0b10, opc, asm, ".2s", ".2h", ".h",
-                                              V64, V128_lo, v2f32, v4f16, OpNode>;
+                                              V64, V128_lo, v2f32, v4f16, VectorIndexH, OpNode>;
   def v8f16 : BaseSIMDThreeSameVectorIndexH<1, U, 0b10, opc, asm, ".4s", ".4h", ".h",
-                                              V128, V128_lo, v4f32, v8f16, OpNode>;
+                                              V128, V128_lo, v4f32, v8f16, VectorIndexH, OpNode>;
 }
 
 //----------------------------------------------------------------------------
 // FP8 Advanced SIMD vector x indexed element
-// TODO: Replace value types v8i8 and v16i8 by v8f8 and v16f8
-multiclass SIMDThreeSameVectorFP8DOT2Index<string asm> {
-  def v4f16 : BaseSIMDThreeSameVectorIndexH<0b0, 0b0, 0b01, 0b0000, asm, ".4h", ".8b", ".2b",
-                                            V64, V128_lo, v4f16, v8i8, null_frag>;
-  def v8f16 : BaseSIMDThreeSameVectorIndexH<0b1, 0b0, 0b01, 0b0000, asm, ".8h", ".16b", ".2b",
-                                            V128, V128_lo, v8f16, v8i16, null_frag>;
+multiclass SIMD_FP8_Dot2_Index<string asm, SDPatternOperator op> {
+  let Uses = [FPMR, FPCR], mayLoad = 1 in { 
+    def v4f16 : BaseSIMDThreeSameVectorIndexH<0b0, 0b0, 0b01, 0b0000, asm, ".4h", ".8b", ".2b",
+                                              V64, V128_lo, v4f16, v8i8, VectorIndexH32b_timm, null_frag>;
+    def v8f16 : BaseSIMDThreeSameVectorIndexH<0b1, 0b0, 0b01, 0b0000, asm, ".8h", ".16b", ".2b",
+                                              V128, V128_lo, v8f16, v16i8, VectorIndexH32b_timm, null_frag>;
+  }
+  def : Pat<(v4f16 (op (v4f16 V64:$Rd), (v8i8 V64:$Rn), (v16i8 V128_lo:$Rm), VectorIndexH32b_timm:$Idx)),
+            (!cast<Instruction>(NAME # v4f16) $Rd, $Rn, $Rm, $Idx)>;
+
+  def : Pat<(v8f16 (op (v8f16 V128:$Rd), (v16i8 V128:$Rn), (v16i8 V128_lo:$Rm), VectorIndexH32b_timm:$Idx)),
+            (!cast<Instruction>(NAME # v8f16) $Rd, $Rn, $Rm, $Idx)>;
 }
 
 multiclass SIMDFPIndexed<bit U, bits<4> opc, string asm,
@@ -13270,4 +13328,3 @@ multiclass SIMDThreeSameVectorFP8MatrixMul<string asm>{
       let Predicates = [HasNEON, HasF8F32MM];
     }
 }
-
diff --git a/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp b/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp
index 17dd8a073eff0..0f2b969fba35c 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp
+++ b/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp
@@ -81,7 +81,7 @@ static cl::opt<unsigned>
 AArch64InstrInfo::AArch64InstrInfo(const AArch64Subtarget &STI)
     : AArch64GenInstrInfo(AArch64::ADJCALLSTACKDOWN, AArch64::ADJCALLSTACKUP,
                           AArch64::CATCHRET),
-      RI(STI.getTargetTriple()), Subtarget(STI) {}
+      RI(STI.getTargetTriple(), STI.getHwMode()), Subtarget(STI) {}
 
 /// GetInstSize - Return the number of bytes of code the specified
 /// instruction may be.  This returns the maximum number of bytes.
@@ -2438,6 +2438,8 @@ unsigned AArch64InstrInfo::getLoadStoreImmIdx(unsigned Opc) {
   case AArch64::STZ2Gi:
   case AArch64::STZGi:
   case AArch64::TAGPstack:
+  case AArch64::SPILL_PPR_TO_ZPR_SLOT_PSEUDO:
+  case AArch64::FILL_PPR_FROM_ZPR_SLOT_PSEUDO:
     return 2;
   case AArch64::LD1B_D_IMM:
   case AArch64::LD1B_H_IMM:
@@ -4223,6 +4225,8 @@ bool AArch64InstrInfo::getMemOpInfo(unsigned Opcode, TypeSize &Scale,
     MinOffset = -256;
     MaxOffset = 254;
     break;
+  case AArch64::SPILL_PPR_TO_ZPR_SLOT_PSEUDO:
+  case AArch64::FILL_PPR_FROM_ZPR_SLOT_PSEUDO:
   case AArch64::LDR_ZXI:
   case AArch64::STR_ZXI:
     Scale = TypeSize::getScalable(16);
@@ -5355,6 +5359,11 @@ void AArch64InstrInfo::storeRegToStackSlot(MachineBasicBlock &MBB,
              "Unexpected register store without SVE store instructions");
       Opc = AArch64::STR_ZXI;
       StackID = TargetStackID::ScalableVector;
+    } else if (AArch64::PPRRegClass.hasSubClassEq(RC)) {
+      assert(Subtarget.isSVEorStreamingSVEAvailable() &&
+             "Unexpected predicate store without SVE store instructions");
+      Opc = AArch64::SPILL_PPR_TO_ZPR_SLOT_PSEUDO;
+      StackID = TargetStackID::ScalableVector;
     }
     break;
   case 24:
@@ -5527,6 +5536,11 @@ void AArch64InstrInfo::loadRegFromStackSlot(
              "Unexpected register load without SVE load instructions");
       Opc = AArch64::LDR_ZXI;
       StackID = TargetStackID::ScalableVector;
+    } else if (AArch64::PPRRegClass.hasSubClassEq(RC)) {
+      assert(Subtarget.isSVEorStreamingSVEAvailable() &&
+             "Unexpected predicate load without SVE load instructions");
+      Opc = AArch64::FILL_PPR_FROM_ZPR_SLOT_PSEUDO;
+      StackID = TargetStackID::ScalableVector;
     }
     break;
   case 24:
diff --git a/llvm/lib/Target/AArch64/AArch64InstrInfo.td b/llvm/lib/Target/AArch64/AArch64InstrInfo.td
index fa6385409f30c..658ac7490eb33 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrInfo.td
+++ b/llvm/lib/Target/AArch64/AArch64InstrInfo.td
@@ -389,6 +389,8 @@ def NoUseScalarIncVL : Predicate<"!Subtarget->useScalarIncVL()">;
 
 def UseSVEFPLD1R : Predicate<"!Subtarget->noSVEFPLD1R()">;
 
+def UseLDAPUR : Predicate<"!Subtarget->avoidLDAPUR()">;
+
 def AArch64LocalRecover : SDNode<"ISD::LOCAL_RECOVER",
                                   SDTypeProfile<1, 1, [SDTCisSameAs<0, 1>,
                                                        SDTCisInt<1>]>>;
@@ -426,7 +428,6 @@ def SDT_AArch64cbz : SDTypeProfile<0, 2, [SDTCisInt<0>, SDTCisVT<1, OtherVT>]>;
 def SDT_AArch64tbz : SDTypeProfile<0, 3, [SDTCisInt<0>, SDTCisInt<1>,
                                         SDTCisVT<2, OtherVT>]>;
 
-
 def SDT_AArch64CSel  : SDTypeProfile<1, 4,
                                    [SDTCisSameAs<0, 1>,
                                     SDTCisSameAs<0, 2>,
@@ -449,6 +450,7 @@ def SDT_AArch64FCCMP : SDTypeProfile<1, 5,
 def SDT_AArch64FCmp  : SDTypeProfile<1, 2, [SDTCisVT<0, i32>,
                                             SDTCisFP<1>,
                                             SDTCisSameAs<2, 1>]>;
+def SDT_AArch64Rev : SDTypeProfile<1, 1, [SDTCisSameAs<0, 1>]>;
 def SDT_AArch64Dup   : SDTypeProfile<1, 1, [SDTCisVec<0>]>;
 def SDT_AArch64DupLane   : SDTypeProfile<1, 2, [SDTCisVec<0>, SDTCisInt<2>]>;
 def SDT_AArch64Insr  : SDTypeProfile<1, 2, [SDTCisVec<0>]>;
@@ -815,11 +817,9 @@ def AArch64mvni_msl : SDNode<"AArch64ISD::MVNImsl", SDT_AArch64MOVIshift>;
 def AArch64movi : SDNode<"AArch64ISD::MOVI", SDT_AArch64MOVIedit>;
 def AArch64fmov : SDNode<"AArch64ISD::FMOV", SDT_AArch64MOVIedit>;
 
-def AArch64rev16_scalar : SDNode<"AArch64ISD::REV16", SDTIntUnaryOp>;
-
-def AArch64rev16 : SDNode<"AArch64ISD::REV16", SDT_AArch64UnaryVec>;
-def AArch64rev32 : SDNode<"AArch64ISD::REV32", SDT_AArch64UnaryVec>;
-def AArch64rev64 : SDNode<"AArch64ISD::REV64", SDT_AArch64UnaryVec>;
+def AArch64rev16 : SDNode<"AArch64ISD::REV16", SDT_AArch64Rev>;
+def AArch64rev32 : SDNode<"AArch64ISD::REV32", SDT_AArch64Rev>;
+def AArch64rev64 : SDNode<"AArch64ISD::REV64", SDT_AArch64Rev>;
 def AArch64ext : SDNode<"AArch64ISD::EXT", SDT_AArch64ExtVec>;
 
 def AArch64vashr : SDNode<"AArch64ISD::VASHR", SDT_AArch64vshift>;
@@ -1025,9 +1025,9 @@ def add_and_or_is_add : PatFrags<(ops node:$lhs, node:$rhs),
    return CurDAG->isADDLike(SDValue(N,0));
 }]> {
   let GISelPredicateCode = [{
-     // Only handle G_ADD for now. FIXME. build capability to compute whether
-     // operands of G_OR have common bits set or not.
-     return MI.getOpcode() == TargetOpcode::G_ADD;
+     return MI.getOpcode() == TargetOpcode::G_ADD ||
+            (MI.getOpcode() == TargetOpcode::G_OR &&
+             MI.getFlag(MachineInstr::MIFlag::Disjoint));
   }];
 }
 
@@ -1495,7 +1495,7 @@ class BaseSIMDSUDOTIndex<bit Q, string dst_kind, string lhs_kind,
                          ValueType AccumType, ValueType InputType>
       : BaseSIMDThreeSameVectorIndexS<Q, 0, 0b00, 0b1111, "sudot", dst_kind,
                                         lhs_kind, rhs_kind, RegType, AccumType,
-                                        InputType, null_frag> {
+                                        InputType, VectorIndexS, null_frag> {
   let Pattern = [(set (AccumType RegType:$dst),
                       (AccumType (AArch64usdot (AccumType RegType:$Rd),
                                  (InputType (bitconvert (AccumType
@@ -1771,7 +1771,12 @@ def : InstAlias<"xpaclri", (XPACLRI), 0>;
 
 let Uses = [LR, SP], Defs = [LR] in {
 // Insertion point of LR signing code.
-def PAUTH_PROLOGUE : Pseudo<(outs), (ins), []>, Sched<[]>;
+def PAUTH_PROLOGUE : Pseudo<(outs), (ins), []>, Sched<[]> {
+  // When using PAuthLR, the address of one of the instructions this expands
+  // into is used as an input to the signature calculation, so this must not be
+  // duplicated.
+  let isNotDuplicable = 1;
+}
 // Insertion point of LR authentication code.
 // The RET terminator of the containing machine basic block may be replaced
 // with a combined RETA(A|B) instruction when rewriting this Pseudo.
@@ -2993,8 +2998,8 @@ def : Pat<(bswap (rotr GPR64:$Rn, (i64 32))), (REV32Xr GPR64:$Rn)>;
 def : Pat<(srl (bswap top16Zero:$Rn), (i64 16)), (REV16Wr GPR32:$Rn)>;
 def : Pat<(srl (bswap top32Zero:$Rn), (i64 32)), (REV32Xr GPR64:$Rn)>;
 
-def : Pat<(AArch64rev16_scalar GPR32:$Rn), (REV16Wr GPR32:$Rn)>;
-def : Pat<(AArch64rev16_scalar GPR64:$Rn), (REV16Xr GPR64:$Rn)>;
+def : Pat<(AArch64rev16 GPR32:$Rn), (REV16Wr GPR32:$Rn)>;
+def : Pat<(AArch64rev16 GPR64:$Rn), (REV16Xr GPR64:$Rn)>;
 
 def : Pat<(or (and (srl GPR64:$Rn, (i64 8)), (i64 0x00ff00ff00ff00ff)),
               (and (shl GPR64:$Rn, (i64 8)), (i64 0xff00ff00ff00ff00))),
@@ -5068,8 +5073,8 @@ defm SCVTF : IntegerToFP<0b00, 0b010, "scvtf", any_sint_to_fp>;
 defm UCVTF : IntegerToFP<0b00, 0b011, "ucvtf", any_uint_to_fp>;
 
 let Predicates = [HasNEON, HasFPRCVT] in {
-  defm SCVTF : IntegerToFPSIMDScalar<0b11, 0b100, "scvtf">;
-  defm UCVTF : IntegerToFPSIMDScalar<0b11, 0b101, "ucvtf">;
+  defm SCVTF : IntegerToFPSIMDScalar<0b11, 0b100, "scvtf", any_sint_to_fp>;
+  defm UCVTF : IntegerToFPSIMDScalar<0b11, 0b101, "ucvtf", any_uint_to_fp>;
 }
 
 def : Pat<(f16 (fdiv (f16 (any_sint_to_fp (i32 GPR32:$Rn))), fixedpoint_f16_i32:$scale)),
@@ -6625,60 +6630,43 @@ defm : Neon_addl_extract_patterns<sub, sext, "SSUB">;
 // CodeGen patterns for addhn and subhn instructions, which can actually be
 // written in LLVM IR without too much difficulty.
 
-// Prioritize ADDHN and SUBHN over UZP2.
-let AddedComplexity = 10 in {
-
-// ADDHN
-def : Pat<(v8i8 (trunc (v8i16 (AArch64vlshr (add V128:$Rn, V128:$Rm), (i32 8))))),
-          (ADDHNv8i16_v8i8 V128:$Rn, V128:$Rm)>;
-def : Pat<(v4i16 (trunc (v4i32 (AArch64vlshr (add V128:$Rn, V128:$Rm),
-                                           (i32 16))))),
-          (ADDHNv4i32_v4i16 V128:$Rn, V128:$Rm)>;
-def : Pat<(v2i32 (trunc (v2i64 (AArch64vlshr (add V128:$Rn, V128:$Rm),
-                                           (i32 32))))),
-          (ADDHNv2i64_v2i32 V128:$Rn, V128:$Rm)>;
-def : Pat<(concat_vectors (v8i8 V64:$Rd),
-                          (trunc (v8i16 (AArch64vlshr (add V128:$Rn, V128:$Rm),
-                                                    (i32 8))))),
-          (ADDHNv8i16_v16i8 (SUBREG_TO_REG (i32 0), V64:$Rd, dsub),
-                            V128:$Rn, V128:$Rm)>;
-def : Pat<(concat_vectors (v4i16 V64:$Rd),
-                          (trunc (v4i32 (AArch64vlshr (add V128:$Rn, V128:$Rm),
-                                                    (i32 16))))),
-          (ADDHNv4i32_v8i16 (SUBREG_TO_REG (i32 0), V64:$Rd, dsub),
-                            V128:$Rn, V128:$Rm)>;
-def : Pat<(concat_vectors (v2i32 V64:$Rd),
-                          (trunc (v2i64 (AArch64vlshr (add V128:$Rn, V128:$Rm),
-                                                    (i32 32))))),
-          (ADDHNv2i64_v4i32 (SUBREG_TO_REG (i32 0), V64:$Rd, dsub),
-                            V128:$Rn, V128:$Rm)>;
-
-// SUBHN
-def : Pat<(v8i8 (trunc (v8i16 (AArch64vlshr (sub V128:$Rn, V128:$Rm), (i32 8))))),
-          (SUBHNv8i16_v8i8 V128:$Rn, V128:$Rm)>;
-def : Pat<(v4i16 (trunc (v4i32 (AArch64vlshr (sub V128:$Rn, V128:$Rm),
-                                           (i32 16))))),
-          (SUBHNv4i32_v4i16 V128:$Rn, V128:$Rm)>;
-def : Pat<(v2i32 (trunc (v2i64 (AArch64vlshr (sub V128:$Rn, V128:$Rm),
-                                           (i32 32))))),
-          (SUBHNv2i64_v2i32 V128:$Rn, V128:$Rm)>;
-def : Pat<(concat_vectors (v8i8 V64:$Rd),
-                          (trunc (v8i16 (AArch64vlshr (sub V128:$Rn, V128:$Rm),
-                                                    (i32 8))))),
-          (SUBHNv8i16_v16i8 (SUBREG_TO_REG (i32 0), V64:$Rd, dsub),
-                            V128:$Rn, V128:$Rm)>;
-def : Pat<(concat_vectors (v4i16 V64:$Rd),
-                          (trunc (v4i32 (AArch64vlshr (sub V128:$Rn, V128:$Rm),
-                                                    (i32 16))))),
-          (SUBHNv4i32_v8i16 (SUBREG_TO_REG (i32 0), V64:$Rd, dsub),
-                            V128:$Rn, V128:$Rm)>;
-def : Pat<(concat_vectors (v2i32 V64:$Rd),
-                          (trunc (v2i64 (AArch64vlshr (sub V128:$Rn, V128:$Rm),
-                                                    (i32 32))))),
-          (SUBHNv2i64_v4i32 (SUBREG_TO_REG (i32 0), V64:$Rd, dsub),
-                            V128:$Rn, V128:$Rm)>;
-
-} // AddedComplexity = 10
+multiclass AddSubHNPatterns<Instruction ADDHN, Instruction ADDHN2, Instruction SUBHN,
+                            Instruction SUBHN2, ValueType VT64, ValueType VT128, int Shift> {
+  def : Pat<(VT64 (trunc (VT128 (AArch64vlshr (add V128:$Rn, V128:$Rm), (i32 Shift))))),
+            (ADDHN V128:$Rn, V128:$Rm)>;
+  let AddedComplexity = 10 in
+  def : Pat<(concat_vectors (VT64 V64:$Rd),
+                            (trunc (VT128 (AArch64vlshr (add V128:$Rn, V128:$Rm),
+                                                        (i32 Shift))))),
+            (ADDHN2 (SUBREG_TO_REG (i32 0), V64:$Rd, dsub), V128:$Rn, V128:$Rm)>;
+
+  def : Pat<(VT64 (trunc (VT128 (AArch64vlshr (sub V128:$Rn, V128:$Rm), (i32 Shift))))),
+            (SUBHN V128:$Rn, V128:$Rm)>;
+  let AddedComplexity = 10 in
+  def : Pat<(concat_vectors (VT64 V64:$Rd),
+                            (trunc (VT128 (AArch64vlshr (sub V128:$Rn, V128:$Rm),
+                                                        (i32 Shift))))),
+            (SUBHN2 (SUBREG_TO_REG (i32 0), V64:$Rd, dsub), V128:$Rn, V128:$Rm)>;
+
+  // xor by -1 can also be treated as sub
+  def : Pat<(VT64 (trunc (VT128 (AArch64vlshr (xor V128:$Rn, immAllOnesV:$Rm), (i32 Shift))))),
+            (SUBHN V128:$Rm, V128:$Rn)>;
+  let AddedComplexity = 10 in
+  def : Pat<(concat_vectors (VT64 V64:$Rd),
+                            (trunc (VT128 (AArch64vlshr (xor V128:$Rn, immAllOnesV:$Rm),
+                                                        (i32 Shift))))),
+            (SUBHN2 (SUBREG_TO_REG (i32 0), V64:$Rd, dsub), V128:$Rm, V128:$Rn)>;
+}
+
+defm : AddSubHNPatterns<ADDHNv8i16_v8i8, ADDHNv8i16_v16i8,
+                        SUBHNv8i16_v8i8, SUBHNv8i16_v16i8,
+                        v8i8, v8i16, 8>;
+defm : AddSubHNPatterns<ADDHNv4i32_v4i16, ADDHNv4i32_v8i16,
+                        SUBHNv4i32_v4i16, SUBHNv4i32_v8i16,
+                        v4i16, v4i32, 16>;
+defm : AddSubHNPatterns<ADDHNv2i64_v2i32, ADDHNv2i64_v4i32,
+                        SUBHNv2i64_v2i32, SUBHNv2i64_v4i32,
+                        v2i32, v2i64, 32>;
 
 //----------------------------------------------------------------------------
 // AdvSIMD bitwise extract from vector instruction.
@@ -10322,13 +10310,13 @@ let Predicates = [HasD128] in {
 // 2023 Architecture Extensions:
 //===----------------------------===//
 
-let Uses = [FPMR, FPCR], Predicates = [HasFP8] in {
-  defm F1CVTL  : SIMDMixedTwoVectorFP8<0b00, "f1cvtl">;
-  defm F2CVTL  : SIMDMixedTwoVectorFP8<0b01, "f2cvtl">;
-  defm BF1CVTL : SIMDMixedTwoVectorFP8<0b10, "bf1cvtl">;
-  defm BF2CVTL : SIMDMixedTwoVectorFP8<0b11, "bf2cvtl">;
-  defm FCVTN_F16_F8 : SIMDThreeSameSizeVectorCvt<"fcvtn">;
-  defm FCVTN_F32_F8 : SIMDThreeVectorCvt<"fcvtn">;
+let Predicates = [HasFP8] in {
+  defm F1CVTL  : SIMD_FP8_CVTL<0b00, "f1cvtl", v8f16, int_aarch64_neon_fp8_cvtl1>;
+  defm F2CVTL  : SIMD_FP8_CVTL<0b01, "f2cvtl", v8f16, int_aarch64_neon_fp8_cvtl2>;
+  defm BF1CVTL : SIMD_FP8_CVTL<0b10, "bf1cvtl", v8bf16, int_aarch64_neon_fp8_cvtl1>;
+  defm BF2CVTL : SIMD_FP8_CVTL<0b11, "bf2cvtl", v8bf16, int_aarch64_neon_fp8_cvtl2>;
+  defm FCVTN_F16 : SIMD_FP8_CVTN_F16<"fcvtn", int_aarch64_neon_fp8_fcvtn>;
+  defm FCVTN_F32 : SIMD_FP8_CVTN_F32<"fcvtn", int_aarch64_neon_fp8_fcvtn>;
   defm FSCALE : SIMDThreeVectorFscale<0b1, 0b1, 0b111, "fscale", int_aarch64_neon_fp8_fscale>;
 } // End let Predicates = [HasFP8]
 
@@ -10351,30 +10339,32 @@ let Predicates = [HasNEON, HasFAMINMAX] in {
  defm FAMIN : SIMDThreeSameVectorFP<0b1, 0b1, 0b011, "famin", AArch64famin>;
 } // End let Predicates = [HasNEON, HasFAMINMAX]
 
-let Uses = [FPMR, FPCR], Predicates = [HasFP8FMA] in {
- defm FMLALBlane : SIMDThreeSameVectorMLAIndex<0b0, "fmlalb">;
- defm FMLALTlane : SIMDThreeSameVectorMLAIndex<0b1, "fmlalt">;
- defm FMLALLBBlane : SIMDThreeSameVectorMLALIndex<0b0, 0b00, "fmlallbb">;
- defm FMLALLBTlane : SIMDThreeSameVectorMLALIndex<0b0, 0b01, "fmlallbt">;
- defm FMLALLTBlane : SIMDThreeSameVectorMLALIndex<0b1, 0b00, "fmlalltb">;
- defm FMLALLTTlane : SIMDThreeSameVectorMLALIndex<0b1, 0b01, "fmlalltt">;
-
- defm FMLALB : SIMDThreeSameVectorMLA<0b0, "fmlalb">;
- defm FMLALT : SIMDThreeSameVectorMLA<0b1, "fmlalt">;
- defm FMLALLBB : SIMDThreeSameVectorMLAL<0b0, 0b00, "fmlallbb">;
- defm FMLALLBT : SIMDThreeSameVectorMLAL<0b0, 0b01, "fmlallbt">;
- defm FMLALLTB : SIMDThreeSameVectorMLAL<0b1, 0b00, "fmlalltb">;
- defm FMLALLTT : SIMDThreeSameVectorMLAL<0b1, 0b01, "fmlalltt">;
+let Predicates = [HasFP8FMA] in {
+ defm FMLALBlane : SIMDThreeSameVectorMLAIndex<0b0, "fmlalb", int_aarch64_neon_fp8_fmlalb_lane>;
+ defm FMLALTlane : SIMDThreeSameVectorMLAIndex<0b1, "fmlalt", int_aarch64_neon_fp8_fmlalt_lane>;
+ defm FMLALLBBlane : SIMDThreeSameVectorMLALIndex<0b0, 0b00, "fmlallbb", int_aarch64_neon_fp8_fmlallbb_lane>;
+ defm FMLALLBTlane : SIMDThreeSameVectorMLALIndex<0b0, 0b01, "fmlallbt", int_aarch64_neon_fp8_fmlallbt_lane>;
+ defm FMLALLTBlane : SIMDThreeSameVectorMLALIndex<0b1, 0b00, "fmlalltb", int_aarch64_neon_fp8_fmlalltb_lane>;
+ defm FMLALLTTlane : SIMDThreeSameVectorMLALIndex<0b1, 0b01, "fmlalltt", int_aarch64_neon_fp8_fmlalltt_lane>;
+}
+
+let Predicates = [HasFP8FMA], Uses = [FPMR, FPCR], mayLoad = 1 in {
+ defm FMLALB : SIMDThreeSameVectorMLA<0b0, "fmlalb", int_aarch64_neon_fp8_fmlalb>;
+ defm FMLALT : SIMDThreeSameVectorMLA<0b1, "fmlalt", int_aarch64_neon_fp8_fmlalt>;
+ defm FMLALLBB : SIMDThreeSameVectorMLAL<0b0, 0b00, "fmlallbb", int_aarch64_neon_fp8_fmlallbb>;
+ defm FMLALLBT : SIMDThreeSameVectorMLAL<0b0, 0b01, "fmlallbt", int_aarch64_neon_fp8_fmlallbt>;
+ defm FMLALLTB : SIMDThreeSameVectorMLAL<0b1, 0b00, "fmlalltb", int_aarch64_neon_fp8_fmlalltb>;
+ defm FMLALLTT : SIMDThreeSameVectorMLAL<0b1, 0b01, "fmlalltt", int_aarch64_neon_fp8_fmlalltt>;
 } // End let Predicates = [HasFP8FMA]
 
-let Uses = [FPMR, FPCR], Predicates = [HasFP8DOT2] in {
- defm FDOTlane : SIMDThreeSameVectorFP8DOT2Index<"fdot">;
- defm FDOT : SIMDThreeSameVectorDOT2<"fdot">;
+let Predicates = [HasFP8DOT2] in {
+ defm FDOTlane : SIMD_FP8_Dot2_Index<"fdot", int_aarch64_neon_fp8_fdot2_lane>;
+ defm FDOT : SIMD_FP8_Dot2<"fdot", int_aarch64_neon_fp8_fdot2>;
 } // End let Predicates = [HasFP8DOT2]
 
-let Uses = [FPMR, FPCR], Predicates = [HasFP8DOT4] in {
- defm FDOTlane : SIMDThreeSameVectorFP8DOT4Index<"fdot">;
- defm FDOT : SIMDThreeSameVectorDOT4<"fdot">;
+let Predicates = [HasFP8DOT4] in {
+ defm FDOTlane : SIMD_FP8_Dot4_Index<"fdot", int_aarch64_neon_fp8_fdot4_lane>;
+ defm FDOT : SIMD_FP8_Dot4<"fdot", int_aarch64_neon_fp8_fdot4>;
 } // End let Predicates = [HasFP8DOT4]
 
 //===----------------------------------------------------------------------===//
diff --git a/llvm/lib/Target/AArch64/AArch64MacroFusion.h b/llvm/lib/Target/AArch64/AArch64MacroFusion.h
index 2999e7a8aa909..62da0545875b6 100644
--- a/llvm/lib/Target/AArch64/AArch64MacroFusion.h
+++ b/llvm/lib/Target/AArch64/AArch64MacroFusion.h
@@ -20,7 +20,7 @@ namespace llvm {
 
 /// Note that you have to add:
 ///   DAG.addMutation(createAArch64MacroFusionDAGMutation());
-/// to AArch64PassConfig::createMachineScheduler() to have an effect.
+/// to AArch64TargetMachine::createMachineScheduler() to have an effect.
 std::unique_ptr<ScheduleDAGMutation> createAArch64MacroFusionDAGMutation();
 
 } // llvm
diff --git a/llvm/lib/Target/AArch64/AArch64Processors.td b/llvm/lib/Target/AArch64/AArch64Processors.td
index 0e3c4e8397f52..d1d4986d12550 100644
--- a/llvm/lib/Target/AArch64/AArch64Processors.td
+++ b/llvm/lib/Target/AArch64/AArch64Processors.td
@@ -240,6 +240,7 @@ def TuneX3 : SubtargetFeature<"cortex-x3", "ARMProcFamily", "CortexX3",
                                FeaturePostRAScheduler,
                                FeatureEnableSelectOptimize,
                                FeatureUseFixedOverScalableIfEqualCost,
+                               FeatureAvoidLDAPUR,
                                FeaturePredictableSelectIsExpensive]>;
 
 def TuneX4 : SubtargetFeature<"cortex-x4", "ARMProcFamily", "CortexX4",
@@ -250,6 +251,7 @@ def TuneX4 : SubtargetFeature<"cortex-x4", "ARMProcFamily", "CortexX4",
                                FeaturePostRAScheduler,
                                FeatureEnableSelectOptimize,
                                FeatureUseFixedOverScalableIfEqualCost,
+                               FeatureAvoidLDAPUR,
                                FeaturePredictableSelectIsExpensive]>;
 
 def TuneX925 : SubtargetFeature<"cortex-x925", "ARMProcFamily",
@@ -260,6 +262,7 @@ def TuneX925 : SubtargetFeature<"cortex-x925", "ARMProcFamily",
                                 FeaturePostRAScheduler,
                                 FeatureEnableSelectOptimize,
                                 FeatureUseFixedOverScalableIfEqualCost,
+                                FeatureAvoidLDAPUR,
                                 FeaturePredictableSelectIsExpensive]>;
 
 def TuneA64FX : SubtargetFeature<"a64fx", "ARMProcFamily", "A64FX",
@@ -540,6 +543,7 @@ def TuneNeoverseV2 : SubtargetFeature<"neoversev2", "ARMProcFamily", "NeoverseV2
                                       FeaturePostRAScheduler,
                                       FeatureEnableSelectOptimize,
                                       FeatureUseFixedOverScalableIfEqualCost,
+                                      FeatureAvoidLDAPUR,
                                       FeaturePredictableSelectIsExpensive]>;
 
 def TuneNeoverseV3 : SubtargetFeature<"neoversev3", "ARMProcFamily", "NeoverseV3",
@@ -549,6 +553,7 @@ def TuneNeoverseV3 : SubtargetFeature<"neoversev3", "ARMProcFamily", "NeoverseV3
                                       FeatureFuseAdrpAdd,
                                       FeaturePostRAScheduler,
                                       FeatureEnableSelectOptimize,
+                                      FeatureAvoidLDAPUR,
                                       FeaturePredictableSelectIsExpensive]>;
 
 def TuneNeoverseV3AE : SubtargetFeature<"neoversev3AE", "ARMProcFamily", "NeoverseV3",
@@ -558,6 +563,7 @@ def TuneNeoverseV3AE : SubtargetFeature<"neoversev3AE", "ARMProcFamily", "Neover
                                       FeatureFuseAdrpAdd,
                                       FeaturePostRAScheduler,
                                       FeatureEnableSelectOptimize,
+                                      FeatureAvoidLDAPUR,
                                       FeaturePredictableSelectIsExpensive]>;
 
 def TuneSaphira  : SubtargetFeature<"saphira", "ARMProcFamily", "Saphira",
@@ -862,7 +868,7 @@ def ProcessorFeatures {
                                    FeatureSSBS, FeatureLS64, FeatureCLRBHB,
                                    FeatureSPECRES2, FeatureSVEAES, FeatureSVE2SM4,
                                    FeatureSVE2SHA3, FeatureSVE2, FeatureSVEBitPerm, FeatureETE,
-                                   FeatureMEC, FeatureFP8DOT2];
+                                   FeatureMEC, FeatureFAMINMAX, FeatureFP8DOT2, FeatureLUT];
   list<SubtargetFeature> Carmel   = [HasV8_2aOps, FeatureNEON, FeatureSHA2, FeatureAES,
                                      FeatureFullFP16, FeatureCRC, FeatureLSE, FeatureRAS, FeatureRDM,
                                      FeatureFPARMv8];
diff --git a/llvm/lib/Target/AArch64/AArch64RegisterInfo.cpp b/llvm/lib/Target/AArch64/AArch64RegisterInfo.cpp
index 5973b63b5a802..8fd34325bb00d 100644
--- a/llvm/lib/Target/AArch64/AArch64RegisterInfo.cpp
+++ b/llvm/lib/Target/AArch64/AArch64RegisterInfo.cpp
@@ -20,6 +20,7 @@
 #include "MCTargetDesc/AArch64InstPrinter.h"
 #include "llvm/ADT/BitVector.h"
 #include "llvm/BinaryFormat/Dwarf.h"
+#include "llvm/CodeGen/LiveRegMatrix.h"
 #include "llvm/CodeGen/MachineFrameInfo.h"
 #include "llvm/CodeGen/MachineInstrBuilder.h"
 #include "llvm/CodeGen/MachineRegisterInfo.h"
@@ -38,8 +39,8 @@ using namespace llvm;
 #define GET_REGINFO_TARGET_DESC
 #include "AArch64GenRegisterInfo.inc"
 
-AArch64RegisterInfo::AArch64RegisterInfo(const Triple &TT)
-    : AArch64GenRegisterInfo(AArch64::LR), TT(TT) {
+AArch64RegisterInfo::AArch64RegisterInfo(const Triple &TT, unsigned HwMode)
+    : AArch64GenRegisterInfo(AArch64::LR, 0, 0, 0, HwMode), TT(TT) {
   AArch64_MC::initLLVMToCVRegMapping(this);
 }
 
@@ -1097,7 +1098,11 @@ bool AArch64RegisterInfo::getRegAllocationHints(
     Register VirtReg, ArrayRef<MCPhysReg> Order,
     SmallVectorImpl<MCPhysReg> &Hints, const MachineFunction &MF,
     const VirtRegMap *VRM, const LiveRegMatrix *Matrix) const {
-  const MachineRegisterInfo &MRI = MF.getRegInfo();
+
+  auto &ST = MF.getSubtarget<AArch64Subtarget>();
+  if (!ST.hasSME() || !ST.isStreaming())
+    return TargetRegisterInfo::getRegAllocationHints(VirtReg, Order, Hints, MF,
+                                                     VRM);
 
   // The SVE calling convention preserves registers Z8-Z23. As a result, there
   // are no ZPR2Strided or ZPR4Strided registers that do not overlap with the
@@ -1107,26 +1112,127 @@ bool AArch64RegisterInfo::getRegAllocationHints(
   // FORM_TRANSPOSED_REG_TUPLE pseudo, we want to favour reducing copy
   // instructions over reducing the number of clobbered callee-save registers,
   // so we add the strided registers as a hint.
+  const MachineRegisterInfo &MRI = MF.getRegInfo();
   unsigned RegID = MRI.getRegClass(VirtReg)->getID();
-  // Look through uses of the register for FORM_TRANSPOSED_REG_TUPLE.
-  if ((RegID == AArch64::ZPR2StridedOrContiguousRegClassID ||
-       RegID == AArch64::ZPR4StridedOrContiguousRegClassID) &&
-      any_of(MRI.use_nodbg_instructions(VirtReg), [](const MachineInstr &Use) {
-        return Use.getOpcode() ==
-                   AArch64::FORM_TRANSPOSED_REG_TUPLE_X2_PSEUDO ||
-               Use.getOpcode() == AArch64::FORM_TRANSPOSED_REG_TUPLE_X4_PSEUDO;
-      })) {
-    const TargetRegisterClass *StridedRC =
-        RegID == AArch64::ZPR2StridedOrContiguousRegClassID
-            ? &AArch64::ZPR2StridedRegClass
-            : &AArch64::ZPR4StridedRegClass;
-
-    for (MCPhysReg Reg : Order)
-      if (StridedRC->contains(Reg))
-        Hints.push_back(Reg);
+  if (RegID == AArch64::ZPR2StridedOrContiguousRegClassID ||
+      RegID == AArch64::ZPR4StridedOrContiguousRegClassID) {
+
+    // Look through uses of the register for FORM_TRANSPOSED_REG_TUPLE.
+    for (const MachineInstr &Use : MRI.use_nodbg_instructions(VirtReg)) {
+      if (Use.getOpcode() != AArch64::FORM_TRANSPOSED_REG_TUPLE_X2_PSEUDO &&
+          Use.getOpcode() != AArch64::FORM_TRANSPOSED_REG_TUPLE_X4_PSEUDO)
+        continue;
+
+      unsigned UseOps = Use.getNumOperands() - 1;
+      const TargetRegisterClass *StridedRC;
+      switch (RegID) {
+      case AArch64::ZPR2StridedOrContiguousRegClassID:
+        StridedRC = &AArch64::ZPR2StridedRegClass;
+        break;
+      case AArch64::ZPR4StridedOrContiguousRegClassID:
+        StridedRC = &AArch64::ZPR4StridedRegClass;
+        break;
+      default:
+        llvm_unreachable("Unexpected RegID");
+      }
 
-    return TargetRegisterInfo::getRegAllocationHints(VirtReg, Order, Hints, MF,
-                                                     VRM);
+      SmallVector<MCPhysReg, 4> StridedOrder;
+      for (MCPhysReg Reg : Order)
+        if (StridedRC->contains(Reg))
+          StridedOrder.push_back(Reg);
+
+      int OpIdx = Use.findRegisterUseOperandIdx(VirtReg, this);
+      assert(OpIdx != -1 && "Expected operand index from register use.");
+
+      unsigned TupleID = MRI.getRegClass(Use.getOperand(0).getReg())->getID();
+      bool IsMulZPR = TupleID == AArch64::ZPR2Mul2RegClassID ||
+                      TupleID == AArch64::ZPR4Mul4RegClassID;
+
+      const MachineOperand *AssignedRegOp = llvm::find_if(
+          make_range(Use.operands_begin() + 1, Use.operands_end()),
+          [&VRM](const MachineOperand &Op) {
+            return VRM->hasPhys(Op.getReg());
+          });
+
+      // Example:
+      //
+      // When trying to find a suitable register allocation for VirtReg %v2 in:
+      //
+      //  %v0:zpr2stridedorcontiguous = ld1 p0/z, [...]
+      //  %v1:zpr2stridedorcontiguous = ld1 p0/z, [...]
+      //  %v2:zpr2stridedorcontiguous = ld1 p0/z, [...]
+      //  %v3:zpr2stridedorcontiguous = ld1 p0/z, [...]
+      //  %v4:zpr4mul4 = FORM_TRANSPOSED_X4 %v0:0, %v1:0, %v2:0, %v3:0
+      //
+      // One such suitable allocation would be:
+      //
+      //  { z0, z8 }  = ld1 p0/z, [...]
+      //  { z1, z9 }  = ld1 p0/z, [...]
+      //  { z2, z10 } = ld1 p0/z, [...]
+      //  { z3, z11 } = ld1 p0/z, [...]
+      //  { z0, z1, z2, z3 } =
+      //     FORM_TRANSPOSED_X4 {z0, z8}:0, {z1, z9}:0, {z2, z10}:0, {z3, z11}:0
+      //
+      // Below we distinguish two cases when trying to find a register:
+      // * None of the registers used by FORM_TRANSPOSED_X4 have been assigned
+      //   yet. In this case the code muse ensure that there are at least UseOps
+      //   free consecutive registers. If IsMulZPR is true, then the first of
+      //   registers must also be a multiple of UseOps, e.g. { z0, z1, z2, z3 }
+      //   is valid but { z1, z2, z3, z5 } is not.
+      // * One or more of the registers used by FORM_TRANSPOSED_X4 is already
+      //   assigned a physical register, which means only checking that a
+      //   consectutive range of free tuple registers exists which includes
+      //   the assigned register.
+      //   e.g. in the example above, if { z0, z8 } is already allocated for
+      //   %v0, we just need to ensure that { z1, z9 }, { z2, z10 } and
+      //   { z3, z11 } are also free. If so, we add { z2, z10 }.
+
+      if (AssignedRegOp == Use.operands_end()) {
+        // There are no registers already assigned to any of the pseudo
+        // operands. Look for a valid starting register for the group.
+        for (unsigned I = 0; I < StridedOrder.size(); ++I) {
+          MCPhysReg Reg = StridedOrder[I];
+          SmallVector<MCPhysReg> Regs;
+
+          // If the FORM_TRANSPOSE nodes use the ZPRMul classes, the starting
+          // register of the first load should be a multiple of 2 or 4.
+          unsigned SubRegIdx = Use.getOperand(OpIdx).getSubReg();
+          if (IsMulZPR && (getSubReg(Reg, SubRegIdx) - AArch64::Z0) % UseOps !=
+                              ((unsigned)OpIdx - 1))
+            continue;
+
+          // In the example above, if VirtReg is the third operand of the
+          // tuple (%v2) and Reg == Z2_Z10, then we need to make sure that
+          // Z0_Z8, Z1_Z9 and Z3_Z11 are also available.
+          auto IsFreeConsecutiveReg = [&](unsigned UseOp) {
+            unsigned R = Reg - (OpIdx - 1) + UseOp;
+            return StridedRC->contains(R) &&
+                   (UseOp == 0 ||
+                    ((getSubReg(R, AArch64::zsub0) - AArch64::Z0) ==
+                     (getSubReg(R - 1, AArch64::zsub0) - AArch64::Z0) + 1)) &&
+                   !Matrix->isPhysRegUsed(R);
+          };
+          if (all_of(iota_range<unsigned>(0U, UseOps, /*Inclusive=*/false),
+                     IsFreeConsecutiveReg))
+            Hints.push_back(Reg);
+        }
+      } else {
+        // At least one operand already has a physical register assigned.
+        // Find the starting sub-register of this and use it to work out the
+        // correct strided register to suggest based on the current op index.
+        MCPhysReg TargetStartReg =
+            getSubReg(VRM->getPhys(AssignedRegOp->getReg()), AArch64::zsub0) +
+            (OpIdx - AssignedRegOp->getOperandNo());
+
+        for (unsigned I = 0; I < StridedOrder.size(); ++I)
+          if (getSubReg(StridedOrder[I], AArch64::zsub0) == TargetStartReg)
+            Hints.push_back(StridedOrder[I]);
+      }
+
+      if (!Hints.empty())
+        return TargetRegisterInfo::getRegAllocationHints(VirtReg, Order, Hints,
+                                                         MF, VRM);
+    }
   }
 
   for (MachineInstr &MI : MRI.def_instructions(VirtReg)) {
diff --git a/llvm/lib/Target/AArch64/AArch64RegisterInfo.h b/llvm/lib/Target/AArch64/AArch64RegisterInfo.h
index 11da624af4881..898a509f75908 100644
--- a/llvm/lib/Target/AArch64/AArch64RegisterInfo.h
+++ b/llvm/lib/Target/AArch64/AArch64RegisterInfo.h
@@ -27,7 +27,7 @@ class AArch64RegisterInfo final : public AArch64GenRegisterInfo {
   const Triple &TT;
 
 public:
-  AArch64RegisterInfo(const Triple &TT);
+  AArch64RegisterInfo(const Triple &TT, unsigned HwMode);
 
   // FIXME: This should be tablegen'd like getDwarfRegNum is
   int getSEHRegNum(unsigned i) const {
diff --git a/llvm/lib/Target/AArch64/AArch64RegisterInfo.td b/llvm/lib/Target/AArch64/AArch64RegisterInfo.td
index dd4f2549929f8..fed9b7b173e9c 100644
--- a/llvm/lib/Target/AArch64/AArch64RegisterInfo.td
+++ b/llvm/lib/Target/AArch64/AArch64RegisterInfo.td
@@ -979,10 +979,19 @@ class ZPRRegOp <string Suffix, AsmOperandClass C, ElementSizeEnum Size,
 //******************************************************************************
 
 // SVE predicate register classes.
+
+// Note: This hardware mode is enabled in AArch64Subtarget::getHwModeSet()
+// (without the use of the table-gen'd predicates).
+def SMEWithZPRPredicateSpills : HwMode<"", [Predicate<"false">]>;
+
+def PPRSpillFillRI : RegInfoByHwMode<
+      [DefaultMode,              SMEWithZPRPredicateSpills],
+      [RegInfo<16,16,16>,        RegInfo<16,128,128>]>;
+
 class PPRClass<int firstreg, int lastreg, int step = 1> : RegisterClass<"AArch64",
                                   [ nxv16i1, nxv8i1, nxv4i1, nxv2i1, nxv1i1 ], 16,
                                   (sequence "P%u", firstreg, lastreg, step)> {
-  let Size = 16;
+  let RegInfos = PPRSpillFillRI;
 }
 
 def PPR    : PPRClass<0, 15> {
diff --git a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
index 6d5e2697160ab..28aecd14e33fa 100644
--- a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
+++ b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
@@ -3379,6 +3379,20 @@ let Predicates = [HasSVE_or_SME] in {
   def : Pat<(i64 (vector_extract nxv2i64:$vec, VectorIndexD:$index)),
             (UMOVvi64 (v2i64 (EXTRACT_SUBREG ZPR:$vec, zsub)), VectorIndexD:$index)>;
 
+  // Find index of last active lane. This is a fallback in case we miss the
+  // opportunity to fold into a lastb or clastb directly.
+  def : Pat<(i64(find_last_active nxv16i1:$P1)),
+            (INSERT_SUBREG(IMPLICIT_DEF), (LASTB_RPZ_B $P1, (INDEX_II_B 0, 1)),
+                sub_32)>;
+  def : Pat<(i64(find_last_active nxv8i1:$P1)),
+            (INSERT_SUBREG(IMPLICIT_DEF), (LASTB_RPZ_H $P1, (INDEX_II_H 0, 1)),
+                sub_32)>;
+  def : Pat<(i64(find_last_active nxv4i1:$P1)),
+            (INSERT_SUBREG(IMPLICIT_DEF), (LASTB_RPZ_S $P1, (INDEX_II_S 0, 1)),
+                sub_32)>;
+  def : Pat<(i64(find_last_active nxv2i1:$P1)), (LASTB_RPZ_D $P1, (INDEX_II_D 0,
+                                                                      1))>;
+
   // Move element from the bottom 128-bits of a scalable vector to a single-element vector.
   // Alternative case where insertelement is just scalar_to_vector rather than vector_insert.
   def : Pat<(v1f64 (scalar_to_vector
@@ -4280,13 +4294,13 @@ let Predicates = [HasSVE2p2_or_SME2p2] in {
   defm SCVTF_ZPzZ  : sve_fp_z2op_p_zd_c<0b0, "scvtf", "int_aarch64_sve_scvtf", AArch64scvtf_mt>;
   defm UCVTF_ZPzZ  : sve_fp_z2op_p_zd_c<0b1, "ucvtf", "int_aarch64_sve_ucvtf", AArch64ucvtf_mt>;
   // Signed integer base 2 logarithm of fp value, zeroing predicate
-  defm FLOGB_ZPzZ : sve_fp_z2op_p_zd_d_flogb<"flogb">;
+  defm FLOGB_ZPzZ : sve_fp_z2op_p_zd_d_flogb<"flogb", int_aarch64_sve_flogb>;
 
   // SVE2 integer unary operations, zeroing predicate
-  def URECPE_ZPzZ  : sve2_int_un_pred_arit_z<0b10, 0b00, "urecpe", ZPR32>;
-  def URSQRTE_ZPzZ : sve2_int_un_pred_arit_z<0b10, 0b01, "ursqrte", ZPR32>;
-  defm SQABS_ZPzZ  : sve2_int_un_pred_arit_z<0b10, "sqabs">;
-  defm SQNEG_ZPzZ  : sve2_int_un_pred_arit_z<0b11, "sqneg">;
+  defm URECPE_ZPzZ  : sve2_int_un_pred_arit_z_S<0b00, "urecpe",  int_aarch64_sve_urecpe>;
+  defm URSQRTE_ZPzZ : sve2_int_un_pred_arit_z_S<0b01, "ursqrte", int_aarch64_sve_ursqrte>;
+  defm SQABS_ZPzZ   : sve2_int_un_pred_arit_z<  0b10, "sqabs",   int_aarch64_sve_sqabs>;
+  defm SQNEG_ZPzZ   : sve2_int_un_pred_arit_z<  0b11, "sqneg",   int_aarch64_sve_sqneg>;
 
   // Floating point round to integral fp value in integer size range
   // Merging
@@ -4301,17 +4315,17 @@ let Predicates = [HasSVE2p2_or_SME2p2] in {
   defm FRINT64X_ZPzZ : sve_fp_z2op_p_zd_frint<0b11, "frint64x">;
 
   // Floating-point round to integral fp value, zeroing predicate
-  defm FRINTN_ZPzZ : sve_fp_z2op_p_zd_hsd<0b00000, "frintn">;
-  defm FRINTP_ZPzZ : sve_fp_z2op_p_zd_hsd<0b00001, "frintp">;
-  defm FRINTM_ZPzZ : sve_fp_z2op_p_zd_hsd<0b00010, "frintm">;
-  defm FRINTZ_ZPzZ : sve_fp_z2op_p_zd_hsd<0b00011, "frintz">;
-  defm FRINTA_ZPzZ : sve_fp_z2op_p_zd_hsd<0b00100, "frinta">;
-  defm FRINTX_ZPzZ : sve_fp_z2op_p_zd_hsd<0b00110, "frintx">;
-  defm FRINTI_ZPzZ : sve_fp_z2op_p_zd_hsd<0b00111, "frinti">;
+  defm FRINTN_ZPzZ : sve_fp_z2op_p_zd_hsd<0b00000, "frintn", AArch64frintn_mt>;
+  defm FRINTP_ZPzZ : sve_fp_z2op_p_zd_hsd<0b00001, "frintp", AArch64frintp_mt>;
+  defm FRINTM_ZPzZ : sve_fp_z2op_p_zd_hsd<0b00010, "frintm", AArch64frintm_mt>;
+  defm FRINTZ_ZPzZ : sve_fp_z2op_p_zd_hsd<0b00011, "frintz", AArch64frintz_mt>;
+  defm FRINTA_ZPzZ : sve_fp_z2op_p_zd_hsd<0b00100, "frinta", AArch64frinta_mt>;
+  defm FRINTX_ZPzZ : sve_fp_z2op_p_zd_hsd<0b00110, "frintx", AArch64frintx_mt>;
+  defm FRINTI_ZPzZ : sve_fp_z2op_p_zd_hsd<0b00111, "frinti", AArch64frinti_mt>;
   // Floating-point invert exponent, zeroing predicate
-  defm FRECPX_ZPzZ : sve_fp_z2op_p_zd_hsd<0b01100, "frecpx">;
+  defm FRECPX_ZPzZ : sve_fp_z2op_p_zd_hsd<0b01100, "frecpx", AArch64frecpx_mt>;
   // Floating-point square root, zeroing predicate
-  defm FSQRT_ZPZz  : sve_fp_z2op_p_zd_hsd<0b01101, "fsqrt">;
+  defm FSQRT_ZPZz  : sve_fp_z2op_p_zd_hsd<0b01101, "fsqrt", AArch64fsqrt_mt>;
 
   // SVE2p2 integer unary arithmetic (bitwise), zeroing predicate
   defm CLS_ZPzZ  : sve_int_un_pred_arit_bitwise_z<0b000,  "cls", AArch64cls_mt>;
@@ -4325,25 +4339,25 @@ let Predicates = [HasSVE2p2_or_SME2p2] in {
   defm FNEG_ZPzZ : sve_int_un_pred_arit_bitwise_fp_z<0b101, "fneg", AArch64fneg_mt>;
 
   // SVE2p2 integer unary arithmetic, zeroing predicate
-  defm SXTB_ZPzZ  : sve_int_un_pred_arit_h_z<0b000, "sxtb">;
-  defm UXTB_ZPzZ  : sve_int_un_pred_arit_h_z<0b001, "uxtb">;
-  defm SXTH_ZPzZ  : sve_int_un_pred_arit_w_z<0b010, "sxth">;
-  defm UXTH_ZPzZ  : sve_int_un_pred_arit_w_z<0b011, "uxth">;
+  defm SXTB_ZPzZ  : sve_int_un_pred_arit_h_z<0b000, "sxtb", AArch64sxt_mt>;
+  defm UXTB_ZPzZ  : sve_int_un_pred_arit_h_z<0b001, "uxtb", AArch64uxt_mt>;
+  defm SXTH_ZPzZ  : sve_int_un_pred_arit_w_z<0b010, "sxth", AArch64sxt_mt>;
+  defm UXTH_ZPzZ  : sve_int_un_pred_arit_w_z<0b011, "uxth", AArch64uxt_mt>;
   defm ABS_ZPzZ   : sve_int_un_pred_arit_z<  0b110, "abs", AArch64abs_mt>;
   defm NEG_ZPzZ   : sve_int_un_pred_arit_z<  0b111, "neg", AArch64neg_mt>;
-  def SXTW_ZPzZ_D : sve_int_un_pred_arit_z<0b11, 0b1000, "sxtw", ZPR64>;
-  def UXTW_ZPzZ_D : sve_int_un_pred_arit_z<0b11, 0b1010, "uxtw", ZPR64>;
+  defm SXTW_ZPzZ  : sve_int_un_pred_arit_d_z<0b100, "sxtw", AArch64sxt_mt>;
+  defm UXTW_ZPzZ  : sve_int_un_pred_arit_d_z<0b101, "uxtw", AArch64uxt_mt>;
 
   // SVE predicate count
   defm FIRSTP_XPP : sve_int_pcount_pred_tmp<0b001, "firstp">;
   defm LASTP_XPP  : sve_int_pcount_pred_tmp<0b010, "lastp">;
 
   // SVE reverse within elements, zeroing predicate
-  defm RBIT_ZPzZ : sve_int_perm_rev_rbit_z<"rbit">;
-  defm REVB_ZPzZ : sve_int_perm_rev_revb_z<"revb">;
-  defm REVH_ZPzZ : sve_int_perm_rev_revh_z<"revh">;
-  def  REVW_ZPzZ : sve_int_perm_rev_z<0b11, 0b0110, "revw", ZPR64>;
-  def  REVD_ZPzZ : sve_int_perm_rev_z<0b00, 0b1110, "revd", ZPR128>;
+  defm RBIT_ZPzZ : sve_int_perm_rev_rbit_z<"rbit", AArch64rbit_mt>;
+  defm REVB_ZPzZ : sve_int_perm_rev_revb_z<"revb", AArch64revb_mt>;
+  defm REVH_ZPzZ : sve_int_perm_rev_revh_z<"revh", AArch64revh_mt>;
+  defm REVW_ZPzZ : sve_int_perm_rev_revw_z<"revw", AArch64revw_mt>;
+  defm REVD_ZPzZ : sve_int_perm_rev_revd_z<"revd", AArch64revd_mt>;
 } // End HasSME2p2orSVE2p2
 
 //===----------------------------------------------------------------------===//
diff --git a/llvm/lib/Target/AArch64/AArch64StackTagging.cpp b/llvm/lib/Target/AArch64/AArch64StackTagging.cpp
index 694ee17f8e5f1..fad83bbebd5d9 100644
--- a/llvm/lib/Target/AArch64/AArch64StackTagging.cpp
+++ b/llvm/lib/Target/AArch64/AArch64StackTagging.cpp
@@ -575,7 +575,7 @@ bool AArch64StackTagging::runOnFunction(Function &Fn) {
       TagPCall->setName(Info.AI->getName() + ".tag");
     // Does not replace metadata, so we don't have to handle DbgVariableRecords.
     Info.AI->replaceUsesWithIf(TagPCall, [&](const Use &U) {
-      return !memtag::isLifetimeIntrinsic(U.getUser());
+      return !isa<LifetimeIntrinsic>(U.getUser());
     });
     TagPCall->setOperand(0, Info.AI);
 
diff --git a/llvm/lib/Target/AArch64/AArch64StackTaggingPreRA.cpp b/llvm/lib/Target/AArch64/AArch64StackTaggingPreRA.cpp
index f819540391801..51bf4b8c30044 100644
--- a/llvm/lib/Target/AArch64/AArch64StackTaggingPreRA.cpp
+++ b/llvm/lib/Target/AArch64/AArch64StackTaggingPreRA.cpp
@@ -284,9 +284,8 @@ std::optional<int> AArch64StackTaggingPreRA::findFirstSlotCandidate() {
             WorkList.push_back(DstReg);
           continue;
         }
-        LLVM_DEBUG(dbgs() << "[" << ST.FI << ":" << ST.Tag << "] use of %"
-                          << Register::virtReg2Index(UseReg) << " in " << UseI
-                          << "\n");
+        LLVM_DEBUG(dbgs() << "[" << ST.FI << ":" << ST.Tag << "] use of "
+                          << printReg(UseReg) << " in " << UseI << "\n");
         Score++;
       }
     }
diff --git a/llvm/lib/Target/AArch64/AArch64Subtarget.cpp b/llvm/lib/Target/AArch64/AArch64Subtarget.cpp
index bc921f07e1dbf..bb36af8fce5cc 100644
--- a/llvm/lib/Target/AArch64/AArch64Subtarget.cpp
+++ b/llvm/lib/Target/AArch64/AArch64Subtarget.cpp
@@ -86,6 +86,11 @@ static cl::alias AArch64StreamingStackHazardSize(
     cl::desc("alias for -aarch64-streaming-hazard-size"),
     cl::aliasopt(AArch64StreamingHazardSize));
 
+static cl::opt<bool> EnableZPRPredicateSpills(
+    "aarch64-enable-zpr-predicate-spills", cl::init(false), cl::Hidden,
+    cl::desc(
+        "Enables spilling/reloading SVE predicates as data vectors (ZPRs)"));
+
 // Subreg liveness tracking is disabled by default for now until all issues
 // are ironed out. This option allows the feature to be used in tests.
 static cl::opt<bool>
@@ -125,7 +130,12 @@ void AArch64Subtarget::initializeProperties(bool HasMinSize) {
   // this in the future so we can specify it together with the subtarget
   // features.
   switch (ARMProcFamily) {
-  case Others:
+  case Generic:
+    // Using TuneCPU=generic we avoid ldapur instructions to line up with the
+    // cpus that use the AvoidLDAPUR feature. We don't want this to be on
+    // forever, so it is enabled between armv8.4 and armv8.7/armv9.2.
+    if (hasV8_4aOps() && !hasV8_8aOps())
+      AvoidLDAPUR = true;
     break;
   case Carmel:
     CacheLineSize = 64;
@@ -400,6 +410,20 @@ AArch64Subtarget::AArch64Subtarget(const Triple &TT, StringRef CPU,
   EnableSubregLiveness = EnableSubregLivenessTracking.getValue();
 }
 
+unsigned AArch64Subtarget::getHwModeSet() const {
+  AArch64HwModeBits Modes = AArch64HwModeBits::DefaultMode;
+
+  // Use a special hardware mode in streaming[-compatible] functions with
+  // aarch64-enable-zpr-predicate-spills. This changes the spill size (and
+  // alignment) for the predicate register class.
+  if (EnableZPRPredicateSpills.getValue() &&
+      (isStreaming() || isStreamingCompatible())) {
+    Modes |= AArch64HwModeBits::SMEWithZPRPredicateSpills;
+  }
+
+  return to_underlying(Modes);
+}
+
 const CallLowering *AArch64Subtarget::getCallLowering() const {
   return CallLoweringInfo.get();
 }
diff --git a/llvm/lib/Target/AArch64/AArch64Subtarget.h b/llvm/lib/Target/AArch64/AArch64Subtarget.h
index d22991224d496..c6eb77e3bc3ba 100644
--- a/llvm/lib/Target/AArch64/AArch64Subtarget.h
+++ b/llvm/lib/Target/AArch64/AArch64Subtarget.h
@@ -38,7 +38,7 @@ class Triple;
 class AArch64Subtarget final : public AArch64GenSubtargetInfo {
 public:
   enum ARMProcFamilyEnum : uint8_t {
-    Others,
+    Generic,
 #define ARM_PROCESSOR_FAMILY(ENUM) ENUM,
 #include "llvm/TargetParser/AArch64TargetParserDef.inc"
 #undef ARM_PROCESSOR_FAMILY
@@ -46,7 +46,7 @@ class AArch64Subtarget final : public AArch64GenSubtargetInfo {
 
 protected:
   /// ARMProcFamily - ARM processor family: Cortex-A53, Cortex-A57, and others.
-  ARMProcFamilyEnum ARMProcFamily = Others;
+  ARMProcFamilyEnum ARMProcFamily = Generic;
 
   // Enable 64-bit vectorization in SLP.
   unsigned MinVectorRegisterBitWidth = 64;
@@ -130,6 +130,8 @@ class AArch64Subtarget final : public AArch64GenSubtargetInfo {
                    bool IsStreaming = false, bool IsStreamingCompatible = false,
                    bool HasMinSize = false);
 
+  virtual unsigned getHwModeSet() const override;
+
 // Getters for SubtargetFeatures defined in tablegen
 #define GET_SUBTARGETINFO_MACRO(ATTRIBUTE, DEFAULT, GETTER)                    \
   bool GETTER() const { return ATTRIBUTE; }
diff --git a/llvm/lib/Target/AArch64/AArch64TargetMachine.cpp b/llvm/lib/Target/AArch64/AArch64TargetMachine.cpp
index 07f072446081a..d10a0c0a08f89 100644
--- a/llvm/lib/Target/AArch64/AArch64TargetMachine.cpp
+++ b/llvm/lib/Target/AArch64/AArch64TargetMachine.cpp
@@ -480,6 +480,33 @@ AArch64TargetMachine::getSubtargetImpl(const Function &F) const {
   return I.get();
 }
 
+ScheduleDAGInstrs *
+AArch64TargetMachine::createMachineScheduler(MachineSchedContext *C) const {
+  const AArch64Subtarget &ST = C->MF->getSubtarget<AArch64Subtarget>();
+  ScheduleDAGMILive *DAG = createGenericSchedLive(C);
+  DAG->addMutation(createLoadClusterDAGMutation(DAG->TII, DAG->TRI));
+  DAG->addMutation(createStoreClusterDAGMutation(DAG->TII, DAG->TRI));
+  if (ST.hasFusion())
+    DAG->addMutation(createAArch64MacroFusionDAGMutation());
+  return DAG;
+}
+
+ScheduleDAGInstrs *
+AArch64TargetMachine::createPostMachineScheduler(MachineSchedContext *C) const {
+  const AArch64Subtarget &ST = C->MF->getSubtarget<AArch64Subtarget>();
+  ScheduleDAGMI *DAG =
+      new ScheduleDAGMI(C, std::make_unique<AArch64PostRASchedStrategy>(C),
+                        /* RemoveKillFlags=*/true);
+  if (ST.hasFusion()) {
+    // Run the Macro Fusion after RA again since literals are expanded from
+    // pseudos then (v. addPreSched2()).
+    DAG->addMutation(createAArch64MacroFusionDAGMutation());
+    return DAG;
+  }
+
+  return DAG;
+}
+
 void AArch64leTargetMachine::anchor() { }
 
 AArch64leTargetMachine::AArch64leTargetMachine(
@@ -512,33 +539,6 @@ class AArch64PassConfig : public TargetPassConfig {
     return getTM<AArch64TargetMachine>();
   }
 
-  ScheduleDAGInstrs *
-  createMachineScheduler(MachineSchedContext *C) const override {
-    const AArch64Subtarget &ST = C->MF->getSubtarget<AArch64Subtarget>();
-    ScheduleDAGMILive *DAG = createGenericSchedLive(C);
-    DAG->addMutation(createLoadClusterDAGMutation(DAG->TII, DAG->TRI));
-    DAG->addMutation(createStoreClusterDAGMutation(DAG->TII, DAG->TRI));
-    if (ST.hasFusion())
-      DAG->addMutation(createAArch64MacroFusionDAGMutation());
-    return DAG;
-  }
-
-  ScheduleDAGInstrs *
-  createPostMachineScheduler(MachineSchedContext *C) const override {
-    const AArch64Subtarget &ST = C->MF->getSubtarget<AArch64Subtarget>();
-    ScheduleDAGMI *DAG =
-        new ScheduleDAGMI(C, std::make_unique<AArch64PostRASchedStrategy>(C),
-                          /* RemoveKillFlags=*/true);
-    if (ST.hasFusion()) {
-      // Run the Macro Fusion after RA again since literals are expanded from
-      // pseudos then (v. addPreSched2()).
-      DAG->addMutation(createAArch64MacroFusionDAGMutation());
-      return DAG;
-    }
-
-    return DAG;
-  }
-
   void addIRPasses()  override;
   bool addPreISel() override;
   void addCodeGenPrepare() override;
diff --git a/llvm/lib/Target/AArch64/AArch64TargetMachine.h b/llvm/lib/Target/AArch64/AArch64TargetMachine.h
index 76b1c9d917ecd..f8ba41f215430 100644
--- a/llvm/lib/Target/AArch64/AArch64TargetMachine.h
+++ b/llvm/lib/Target/AArch64/AArch64TargetMachine.h
@@ -68,9 +68,13 @@ class AArch64TargetMachine : public CodeGenTargetMachineImpl {
 
   /// Returns true if a cast between SrcAS and DestAS is a noop.
   bool isNoopAddrSpaceCast(unsigned SrcAS, unsigned DestAS) const override {
-    // Addrspacecasts are always noops.
-    return true;
+    return getPointerSize(SrcAS) == getPointerSize(DestAS);
   }
+  ScheduleDAGInstrs *
+  createMachineScheduler(MachineSchedContext *C) const override;
+
+  ScheduleDAGInstrs *
+  createPostMachineScheduler(MachineSchedContext *C) const override;
 
 private:
   bool isLittle;
diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
index e2389145cf33f..bd0d55f571234 100644
--- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
+++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
@@ -940,6 +940,16 @@ AArch64TTIImpl::getIntrinsicInstrCost(const IntrinsicCostAttributes &ICA,
     }
     break;
   }
+  case Intrinsic::experimental_cttz_elts: {
+    EVT ArgVT = getTLI()->getValueType(DL, ICA.getArgTypes()[0]);
+    if (!getTLI()->shouldExpandCttzElements(ArgVT)) {
+      // This will consist of a SVE brkb and a cntp instruction. These
+      // typically have the same latency and half the throughput as a vector
+      // add instruction.
+      return 4;
+    }
+    break;
+  }
   default:
     break;
   }
@@ -2188,7 +2198,7 @@ static std::optional<Instruction *> instCombineDMB(InstCombiner &IC,
     NI = NI->getNextNonDebugInstruction();
     if (!NI) {
       if (auto *SuccBB = NIBB->getUniqueSuccessor())
-        NI = SuccBB->getFirstNonPHIOrDbgOrLifetime();
+        NI = &*SuccBB->getFirstNonPHIOrDbgOrLifetime();
       else
         break;
     }
@@ -4272,7 +4282,7 @@ void AArch64TTIImpl::getUnrollingPreferences(Loop *L, ScalarEvolution &SE,
   // If mcpu is omitted, getProcFamily() returns AArch64Subtarget::Others, so by
   // checking for that case, we can ensure that the default behaviour is
   // unchanged
-  if (ST->getProcFamily() != AArch64Subtarget::Others &&
+  if (ST->getProcFamily() != AArch64Subtarget::Generic &&
       !ST->getSchedModel().isOutOfOrder()) {
     UP.Runtime = true;
     UP.Partial = true;
@@ -4682,13 +4692,25 @@ InstructionCost AArch64TTIImpl::getPartialReductionCost(
   EVT InputEVT = EVT::getEVT(InputTypeA);
   EVT AccumEVT = EVT::getEVT(AccumType);
 
-  if (VF.isScalable() && !ST->isSVEorStreamingSVEAvailable())
-    return Invalid;
-  if (VF.isFixed() && (!ST->isNeonAvailable() || !ST->hasDotProd()))
+  unsigned VFMinValue = VF.getKnownMinValue();
+
+  if (VF.isScalable()) {
+    if (!ST->isSVEorStreamingSVEAvailable())
+      return Invalid;
+
+    // Don't accept a partial reduction if the scaled accumulator is vscale x 1,
+    // since we can't lower that type.
+    unsigned Scale =
+        AccumEVT.getScalarSizeInBits() / InputEVT.getScalarSizeInBits();
+    if (VFMinValue == Scale)
+      return Invalid;
+  }
+  if (VF.isFixed() &&
+      (!ST->isNeonAvailable() || !ST->hasDotProd() || AccumEVT == MVT::i64))
     return Invalid;
 
   if (InputEVT == MVT::i8) {
-    switch (VF.getKnownMinValue()) {
+    switch (VFMinValue) {
     default:
       return Invalid;
     case 8:
@@ -4707,7 +4729,7 @@ InstructionCost AArch64TTIImpl::getPartialReductionCost(
   } else if (InputEVT == MVT::i16) {
     // FIXME: Allow i32 accumulator but increase cost, as we would extend
     //        it to i64.
-    if (VF.getKnownMinValue() != 8 || AccumEVT != MVT::i64)
+    if (VFMinValue != 8 || AccumEVT != MVT::i64)
       return Invalid;
   } else
     return Invalid;
diff --git a/llvm/lib/Target/AArch64/AsmParser/AArch64AsmParser.cpp b/llvm/lib/Target/AArch64/AsmParser/AArch64AsmParser.cpp
index d3eda48f3276e..335b46b76688f 100644
--- a/llvm/lib/Target/AArch64/AsmParser/AArch64AsmParser.cpp
+++ b/llvm/lib/Target/AArch64/AsmParser/AArch64AsmParser.cpp
@@ -4621,7 +4621,7 @@ ParseStatus AArch64AsmParser::tryParseVectorList(OperandVector &Operands,
     return ParseStatus::NoMatch;
   };
 
-  int NumRegs = getNumRegsForRegKind(VectorKind);
+  unsigned NumRegs = getNumRegsForRegKind(VectorKind);
   SMLoc S = getLoc();
   auto LCurly = getTok();
   Lex(); // Eat left bracket token.
@@ -4638,10 +4638,10 @@ ParseStatus AArch64AsmParser::tryParseVectorList(OperandVector &Operands,
   if (!ParseRes.isSuccess())
     return ParseRes;
 
-  int64_t PrevReg = FirstReg;
+  MCRegister PrevReg = FirstReg;
   unsigned Count = 1;
 
-  int Stride = 1;
+  unsigned Stride = 1;
   if (parseOptionalToken(AsmToken::Minus)) {
     SMLoc Loc = getLoc();
     StringRef NextKind;
@@ -4656,7 +4656,7 @@ ParseStatus AArch64AsmParser::tryParseVectorList(OperandVector &Operands,
       return Error(Loc, "mismatched register size suffix");
 
     unsigned Space =
-        (PrevReg < Reg) ? (Reg - PrevReg) : (Reg + NumRegs - PrevReg);
+        (PrevReg < Reg) ? (Reg - PrevReg) : (NumRegs - (PrevReg - Reg));
 
     if (Space == 0 || Space > 3)
       return Error(Loc, "invalid number of vectors");
@@ -4682,7 +4682,7 @@ ParseStatus AArch64AsmParser::tryParseVectorList(OperandVector &Operands,
           getContext().getRegisterInfo()->getEncodingValue(PrevReg);
       if (!HasCalculatedStride) {
         Stride = (PrevRegVal < RegVal) ? (RegVal - PrevRegVal)
-                                       : (RegVal + NumRegs - PrevRegVal);
+                                       : (NumRegs - (PrevRegVal - RegVal));
         HasCalculatedStride = true;
       }
 
@@ -5017,7 +5017,9 @@ bool AArch64AsmParser::parseOperand(OperandVector &Operands, bool isCondCode,
       return true;
     E = SMLoc::getFromPointer(getLoc().getPointer() - 1);
     Operands.push_back(AArch64Operand::CreateImm(IdVal, S, E, getContext()));
-    return false;
+
+    // Parse an optional shift/extend modifier.
+    return parseOptionalShiftExtend(getTok());
   }
   case AsmToken::Integer:
   case AsmToken::Real:
@@ -7841,10 +7843,10 @@ bool AArch64AsmParser::parseDirectiveAeabiSubSectionHeader(SMLoc L) {
 
   // Consume the name (subsection name)
   StringRef SubsectionName;
-  AArch64BuildAttributes::VendorID SubsectionNameID;
+  AArch64BuildAttrs::VendorID SubsectionNameID;
   if (Parser.getTok().is(AsmToken::Identifier)) {
     SubsectionName = Parser.getTok().getIdentifier();
-    SubsectionNameID = AArch64BuildAttributes::getVendorID(SubsectionName);
+    SubsectionNameID = AArch64BuildAttrs::getVendorID(SubsectionName);
   } else {
     Error(Parser.getTok().getLoc(), "subsection name not found");
     return true;
@@ -7861,14 +7863,14 @@ bool AArch64AsmParser::parseDirectiveAeabiSubSectionHeader(SMLoc L) {
       getTargetStreamer().getAtributesSubsectionByName(SubsectionName);
 
   // Consume the first parameter (optionality parameter)
-  AArch64BuildAttributes::SubsectionOptional IsOptional;
+  AArch64BuildAttrs::SubsectionOptional IsOptional;
   // options: optional/required
   if (Parser.getTok().is(AsmToken::Identifier)) {
     StringRef Optionality = Parser.getTok().getIdentifier();
-    IsOptional = AArch64BuildAttributes::getOptionalID(Optionality);
-    if (AArch64BuildAttributes::OPTIONAL_NOT_FOUND == IsOptional) {
+    IsOptional = AArch64BuildAttrs::getOptionalID(Optionality);
+    if (AArch64BuildAttrs::OPTIONAL_NOT_FOUND == IsOptional) {
       Error(Parser.getTok().getLoc(),
-            AArch64BuildAttributes::getSubsectionOptionalUnknownError() + ": " +
+            AArch64BuildAttrs::getSubsectionOptionalUnknownError() + ": " +
                 Optionality);
       return true;
     }
@@ -7877,10 +7879,10 @@ bool AArch64AsmParser::parseDirectiveAeabiSubSectionHeader(SMLoc L) {
         Error(Parser.getTok().getLoc(),
               "optionality mismatch! subsection '" + SubsectionName +
                   "' already exists with optionality defined as '" +
-                  AArch64BuildAttributes::getOptionalStr(
+                  AArch64BuildAttrs::getOptionalStr(
                       SubsectionExists->IsOptional) +
                   "' and not '" +
-                  AArch64BuildAttributes::getOptionalStr(IsOptional) + "'");
+                  AArch64BuildAttrs::getOptionalStr(IsOptional) + "'");
         return true;
       }
     }
@@ -7890,15 +7892,15 @@ bool AArch64AsmParser::parseDirectiveAeabiSubSectionHeader(SMLoc L) {
     return true;
   }
   // Check for possible IsOptional unaccepted values for known subsections
-  if (AArch64BuildAttributes::AEABI_FEATURE_AND_BITS == SubsectionNameID) {
-    if (AArch64BuildAttributes::REQUIRED == IsOptional) {
+  if (AArch64BuildAttrs::AEABI_FEATURE_AND_BITS == SubsectionNameID) {
+    if (AArch64BuildAttrs::REQUIRED == IsOptional) {
       Error(Parser.getTok().getLoc(),
             "aeabi_feature_and_bits must be marked as optional");
       return true;
     }
   }
-  if (AArch64BuildAttributes::AEABI_PAUTHABI == SubsectionNameID) {
-    if (AArch64BuildAttributes::OPTIONAL == IsOptional) {
+  if (AArch64BuildAttrs::AEABI_PAUTHABI == SubsectionNameID) {
+    if (AArch64BuildAttrs::OPTIONAL == IsOptional) {
       Error(Parser.getTok().getLoc(),
             "aeabi_pauthabi must be marked as required");
       return true;
@@ -7911,25 +7913,23 @@ bool AArch64AsmParser::parseDirectiveAeabiSubSectionHeader(SMLoc L) {
   }
 
   // Consume the second parameter (type parameter)
-  AArch64BuildAttributes::SubsectionType Type;
+  AArch64BuildAttrs::SubsectionType Type;
   if (Parser.getTok().is(AsmToken::Identifier)) {
     StringRef Name = Parser.getTok().getIdentifier();
-    Type = AArch64BuildAttributes::getTypeID(Name);
-    if (AArch64BuildAttributes::TYPE_NOT_FOUND == Type) {
+    Type = AArch64BuildAttrs::getTypeID(Name);
+    if (AArch64BuildAttrs::TYPE_NOT_FOUND == Type) {
       Error(Parser.getTok().getLoc(),
-            AArch64BuildAttributes::getSubsectionTypeUnknownError() + ": " +
-                Name);
+            AArch64BuildAttrs::getSubsectionTypeUnknownError() + ": " + Name);
       return true;
     }
     if (SubsectionExists) {
       if (Type != SubsectionExists->ParameterType) {
-        Error(Parser.getTok().getLoc(),
-              "type mismatch! subsection '" + SubsectionName +
-                  "' already exists with type defined as '" +
-                  AArch64BuildAttributes::getTypeStr(
-                      SubsectionExists->ParameterType) +
-                  "' and not '" + AArch64BuildAttributes::getTypeStr(Type) +
-                  "'");
+        Error(
+            Parser.getTok().getLoc(),
+            "type mismatch! subsection '" + SubsectionName +
+                "' already exists with type defined as '" +
+                AArch64BuildAttrs::getTypeStr(SubsectionExists->ParameterType) +
+                "' and not '" + AArch64BuildAttrs::getTypeStr(Type) + "'");
         return true;
       }
     }
@@ -7939,9 +7939,9 @@ bool AArch64AsmParser::parseDirectiveAeabiSubSectionHeader(SMLoc L) {
     return true;
   }
   // Check for possible unaccepted 'type' values for known subsections
-  if (AArch64BuildAttributes::AEABI_FEATURE_AND_BITS == SubsectionNameID ||
-      AArch64BuildAttributes::AEABI_PAUTHABI == SubsectionNameID) {
-    if (AArch64BuildAttributes::NTBS == Type) {
+  if (AArch64BuildAttrs::AEABI_FEATURE_AND_BITS == SubsectionNameID ||
+      AArch64BuildAttrs::AEABI_PAUTHABI == SubsectionNameID) {
+    if (AArch64BuildAttrs::NTBS == Type) {
       Error(Parser.getTok().getLoc(),
             SubsectionName + " must be marked as ULEB128");
       return true;
@@ -7976,14 +7976,13 @@ bool AArch64AsmParser::parseDirectiveAeabiAArch64Attr(SMLoc L) {
   StringRef ActiveSubsectionName = ActiveSubsection->VendorName;
   unsigned ActiveSubsectionType = ActiveSubsection->ParameterType;
 
-  unsigned ActiveSubsectionID = AArch64BuildAttributes::VENDOR_UNKNOWN;
-  if (AArch64BuildAttributes::getVendorName(
-          AArch64BuildAttributes::AEABI_PAUTHABI) == ActiveSubsectionName)
-    ActiveSubsectionID = AArch64BuildAttributes::AEABI_PAUTHABI;
-  if (AArch64BuildAttributes::getVendorName(
-          AArch64BuildAttributes::AEABI_FEATURE_AND_BITS) ==
+  unsigned ActiveSubsectionID = AArch64BuildAttrs::VENDOR_UNKNOWN;
+  if (AArch64BuildAttrs::getVendorName(AArch64BuildAttrs::AEABI_PAUTHABI) ==
       ActiveSubsectionName)
-    ActiveSubsectionID = AArch64BuildAttributes::AEABI_FEATURE_AND_BITS;
+    ActiveSubsectionID = AArch64BuildAttrs::AEABI_PAUTHABI;
+  if (AArch64BuildAttrs::getVendorName(
+          AArch64BuildAttrs::AEABI_FEATURE_AND_BITS) == ActiveSubsectionName)
+    ActiveSubsectionID = AArch64BuildAttrs::AEABI_FEATURE_AND_BITS;
 
   StringRef TagStr = "";
   unsigned Tag;
@@ -7993,21 +7992,21 @@ bool AArch64AsmParser::parseDirectiveAeabiAArch64Attr(SMLoc L) {
     default:
       assert(0 && "Subsection name error");
       break;
-    case AArch64BuildAttributes::VENDOR_UNKNOWN:
+    case AArch64BuildAttrs::VENDOR_UNKNOWN:
       // Private subsection, accept any tag.
       break;
-    case AArch64BuildAttributes::AEABI_PAUTHABI:
-      Tag = AArch64BuildAttributes::getPauthABITagsID(TagStr);
-      if (AArch64BuildAttributes::PAUTHABI_TAG_NOT_FOUND == Tag) {
+    case AArch64BuildAttrs::AEABI_PAUTHABI:
+      Tag = AArch64BuildAttrs::getPauthABITagsID(TagStr);
+      if (AArch64BuildAttrs::PAUTHABI_TAG_NOT_FOUND == Tag) {
         Error(Parser.getTok().getLoc(), "unknown AArch64 build attribute '" +
                                             TagStr + "' for subsection '" +
                                             ActiveSubsectionName + "'");
         return true;
       }
       break;
-    case AArch64BuildAttributes::AEABI_FEATURE_AND_BITS:
-      Tag = AArch64BuildAttributes::getFeatureAndBitsTagsID(TagStr);
-      if (AArch64BuildAttributes::FEATURE_AND_BITS_TAG_NOT_FOUND == Tag) {
+    case AArch64BuildAttrs::AEABI_FEATURE_AND_BITS:
+      Tag = AArch64BuildAttrs::getFeatureAndBitsTagsID(TagStr);
+      if (AArch64BuildAttrs::FEATURE_AND_BITS_TAG_NOT_FOUND == Tag) {
         Error(Parser.getTok().getLoc(), "unknown AArch64 build attribute '" +
                                             TagStr + "' for subsection '" +
                                             ActiveSubsectionName + "'");
@@ -8033,7 +8032,7 @@ bool AArch64AsmParser::parseDirectiveAeabiAArch64Attr(SMLoc L) {
   unsigned ValueInt = unsigned(-1);
   std::string ValueStr = "";
   if (Parser.getTok().is(AsmToken::Integer)) {
-    if (AArch64BuildAttributes::NTBS == ActiveSubsectionType) {
+    if (AArch64BuildAttrs::NTBS == ActiveSubsectionType) {
       Error(
           Parser.getTok().getLoc(),
           "active subsection type is NTBS (string), found ULEB128 (unsigned)");
@@ -8041,7 +8040,7 @@ bool AArch64AsmParser::parseDirectiveAeabiAArch64Attr(SMLoc L) {
     }
     ValueInt = getTok().getIntVal();
   } else if (Parser.getTok().is(AsmToken::Identifier)) {
-    if (AArch64BuildAttributes::ULEB128 == ActiveSubsectionType) {
+    if (AArch64BuildAttrs::ULEB128 == ActiveSubsectionType) {
       Error(
           Parser.getTok().getLoc(),
           "active subsection type is ULEB128 (unsigned), found NTBS (string)");
@@ -8049,7 +8048,7 @@ bool AArch64AsmParser::parseDirectiveAeabiAArch64Attr(SMLoc L) {
     }
     ValueStr = Parser.getTok().getIdentifier();
   } else if (Parser.getTok().is(AsmToken::String)) {
-    if (AArch64BuildAttributes::ULEB128 == ActiveSubsectionType) {
+    if (AArch64BuildAttrs::ULEB128 == ActiveSubsectionType) {
       Error(
           Parser.getTok().getLoc(),
           "active subsection type is ULEB128 (unsigned), found NTBS (string)");
@@ -8062,7 +8061,7 @@ bool AArch64AsmParser::parseDirectiveAeabiAArch64Attr(SMLoc L) {
   }
   // Check for possible unaccepted values for known tags (AEABI_PAUTHABI,
   // AEABI_FEATURE_AND_BITS)
-  if (!(ActiveSubsectionID == AArch64BuildAttributes::VENDOR_UNKNOWN) &&
+  if (!(ActiveSubsectionID == AArch64BuildAttrs::VENDOR_UNKNOWN) &&
       TagStr != "") { // TagStr was a recognized string
     if (0 != ValueInt && 1 != ValueInt) {
       Error(Parser.getTok().getLoc(),
diff --git a/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp b/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp
index 07f03644336cd..467094e9befef 100644
--- a/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp
+++ b/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp
@@ -2999,9 +2999,8 @@ bool AArch64InstructionSelector::select(MachineInstr &I) {
     bool IsZExtLoad = I.getOpcode() == TargetOpcode::G_ZEXTLOAD;
     LLT PtrTy = MRI.getType(LdSt.getPointerReg());
 
+    // Can only handle AddressSpace 0, 64-bit pointers.
     if (PtrTy != LLT::pointer(0, 64)) {
-      LLVM_DEBUG(dbgs() << "Load/Store pointer has type: " << PtrTy
-                        << ", expected: " << LLT::pointer(0, 64) << '\n');
       return false;
     }
 
diff --git a/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp b/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp
index 93461e39f9559..fdedf44e0ba1b 100644
--- a/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp
+++ b/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp
@@ -1215,11 +1215,13 @@ AArch64LegalizerInfo::AArch64LegalizerInfo(const AArch64Subtarget &ST)
                  {s32, v4s32},
                  {s32, v2s32},
                  {s64, v2s64}})
+      .moreElementsToNextPow2(1)
       .clampMaxNumElements(1, s64, 2)
       .clampMaxNumElements(1, s32, 4)
       .clampMaxNumElements(1, s16, 8)
       .clampMaxNumElements(1, s8, 16)
-      .lower();
+      .widenVectorEltsToVectorMinSize(1, 64)
+      .scalarize(1);
 
   getActionDefinitionsBuilder({G_VECREDUCE_FMIN, G_VECREDUCE_FMAX,
                                G_VECREDUCE_FMINIMUM, G_VECREDUCE_FMAXIMUM})
diff --git a/llvm/lib/Target/AArch64/GISel/AArch64PreLegalizerCombiner.cpp b/llvm/lib/Target/AArch64/GISel/AArch64PreLegalizerCombiner.cpp
index bbf1883925a69..76520fe5ce20f 100644
--- a/llvm/lib/Target/AArch64/GISel/AArch64PreLegalizerCombiner.cpp
+++ b/llvm/lib/Target/AArch64/GISel/AArch64PreLegalizerCombiner.cpp
@@ -267,10 +267,12 @@ bool matchExtAddvToUdotAddv(MachineInstr &MI, MachineRegisterInfo &MRI,
     SrcTy = MRI.getType(ExtMI1->getOperand(1).getReg());
     std::get<0>(MatchInfo) = ExtMI1->getOperand(1).getReg();
     std::get<1>(MatchInfo) = ExtMI2->getOperand(1).getReg();
-  } else {
+  } else if (I1Opc == TargetOpcode::G_ZEXT || I1Opc == TargetOpcode::G_SEXT) {
     SrcTy = MRI.getType(I1->getOperand(1).getReg());
     std::get<0>(MatchInfo) = I1->getOperand(1).getReg();
     std::get<1>(MatchInfo) = 0;
+  } else {
+    return false;
   }
 
   if (I1Opc == TargetOpcode::G_ZEXT)
diff --git a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64ELFStreamer.cpp b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64ELFStreamer.cpp
index 9f7a60074daeb..6b5c5f36cbd4b 100644
--- a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64ELFStreamer.cpp
+++ b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64ELFStreamer.cpp
@@ -160,13 +160,13 @@ class AArch64TargetAsmStreamer : public AArch64TargetStreamer {
       return;
     }
 
-    unsigned VendorID = AArch64BuildAttributes::getVendorID(VendorName);
+    unsigned VendorID = AArch64BuildAttrs::getVendorID(VendorName);
 
     switch (VendorID) {
     default:
       assert(0 && "Subsection name error");
       break;
-    case AArch64BuildAttributes::VENDOR_UNKNOWN:
+    case AArch64BuildAttrs::VENDOR_UNKNOWN:
       if (unsigned(-1) != Value) {
         OS << "\t.aeabi_attribute" << "\t" << Tag << ", " << Value;
         AArch64TargetStreamer::emitAttribute(VendorName, Tag, Value, "",
@@ -179,7 +179,7 @@ class AArch64TargetAsmStreamer : public AArch64TargetStreamer {
       }
       break;
     // Note: AEABI_FEATURE_AND_BITS takes only unsigned values
-    case AArch64BuildAttributes::AEABI_FEATURE_AND_BITS:
+    case AArch64BuildAttrs::AEABI_FEATURE_AND_BITS:
       switch (Tag) {
       default: // allow emitting any attribute by number
         OS << "\t.aeabi_attribute" << "\t" << Tag << ", " << Value;
@@ -188,19 +188,18 @@ class AArch64TargetAsmStreamer : public AArch64TargetStreamer {
         AArch64TargetStreamer::emitAttribute(VendorName, Tag, Value, "",
                                              Override);
         break;
-      case AArch64BuildAttributes::TAG_FEATURE_BTI:
-      case AArch64BuildAttributes::TAG_FEATURE_GCS:
-      case AArch64BuildAttributes::TAG_FEATURE_PAC:
+      case AArch64BuildAttrs::TAG_FEATURE_BTI:
+      case AArch64BuildAttrs::TAG_FEATURE_GCS:
+      case AArch64BuildAttrs::TAG_FEATURE_PAC:
         OS << "\t.aeabi_attribute" << "\t"
-           << AArch64BuildAttributes::getFeatureAndBitsTagsStr(Tag) << ", "
-           << Value;
+           << AArch64BuildAttrs::getFeatureAndBitsTagsStr(Tag) << ", " << Value;
         AArch64TargetStreamer::emitAttribute(VendorName, Tag, Value, "",
                                              Override);
         break;
       }
       break;
     // Note: AEABI_PAUTHABI takes only unsigned values
-    case AArch64BuildAttributes::AEABI_PAUTHABI:
+    case AArch64BuildAttrs::AEABI_PAUTHABI:
       switch (Tag) {
       default: // allow emitting any attribute by number
         OS << "\t.aeabi_attribute" << "\t" << Tag << ", " << Value;
@@ -209,10 +208,10 @@ class AArch64TargetAsmStreamer : public AArch64TargetStreamer {
         AArch64TargetStreamer::emitAttribute(VendorName, Tag, Value, "",
                                              Override);
         break;
-      case AArch64BuildAttributes::TAG_PAUTH_PLATFORM:
-      case AArch64BuildAttributes::TAG_PAUTH_SCHEMA:
+      case AArch64BuildAttrs::TAG_PAUTH_PLATFORM:
+      case AArch64BuildAttrs::TAG_PAUTH_SCHEMA:
         OS << "\t.aeabi_attribute" << "\t"
-           << AArch64BuildAttributes::getPauthABITagsStr(Tag) << ", " << Value;
+           << AArch64BuildAttrs::getPauthABITagsStr(Tag) << ", " << Value;
         AArch64TargetStreamer::emitAttribute(VendorName, Tag, Value, "",
                                              Override);
         break;
@@ -223,19 +222,18 @@ class AArch64TargetAsmStreamer : public AArch64TargetStreamer {
   }
 
   void emitAtributesSubsection(
-      StringRef SubsectionName,
-      AArch64BuildAttributes::SubsectionOptional Optional,
-      AArch64BuildAttributes::SubsectionType ParameterType) override {
+      StringRef SubsectionName, AArch64BuildAttrs::SubsectionOptional Optional,
+      AArch64BuildAttrs::SubsectionType ParameterType) override {
     // The AArch64 build attributes assembly subsection header format:
     // ".aeabi_subsection name, optional, parameter type"
     // optional: required (0) optional (1)
     // parameter type: uleb128 or ULEB128 (0) ntbs or NTBS (1)
-    unsigned SubsectionID = AArch64BuildAttributes::getVendorID(SubsectionName);
+    unsigned SubsectionID = AArch64BuildAttrs::getVendorID(SubsectionName);
 
     assert((0 == Optional || 1 == Optional) &&
-           AArch64BuildAttributes::getSubsectionOptionalUnknownError().data());
+           AArch64BuildAttrs::getSubsectionOptionalUnknownError().data());
     assert((0 == ParameterType || 1 == ParameterType) &&
-           AArch64BuildAttributes::getSubsectionTypeUnknownError().data());
+           AArch64BuildAttrs::getSubsectionTypeUnknownError().data());
 
     std::string SubsectionTag = ".aeabi_subsection";
     StringRef OptionalStr = getOptionalStr(Optional);
@@ -246,20 +244,20 @@ class AArch64TargetAsmStreamer : public AArch64TargetStreamer {
       // Treated as a private subsection
       break;
     }
-    case AArch64BuildAttributes::AEABI_PAUTHABI: {
-      assert(AArch64BuildAttributes::REQUIRED == Optional &&
+    case AArch64BuildAttrs::AEABI_PAUTHABI: {
+      assert(AArch64BuildAttrs::REQUIRED == Optional &&
              "subsection .aeabi-pauthabi should be marked as "
              "required and not as optional");
-      assert(AArch64BuildAttributes::ULEB128 == ParameterType &&
+      assert(AArch64BuildAttrs::ULEB128 == ParameterType &&
              "subsection .aeabi-pauthabi should be "
              "marked as uleb128 and not as ntbs");
       break;
     }
-    case AArch64BuildAttributes::AEABI_FEATURE_AND_BITS: {
-      assert(AArch64BuildAttributes::OPTIONAL == Optional &&
+    case AArch64BuildAttrs::AEABI_FEATURE_AND_BITS: {
+      assert(AArch64BuildAttrs::OPTIONAL == Optional &&
              "subsection .aeabi_feature_and_bits should be "
              "marked as optional and not as required");
-      assert(AArch64BuildAttributes::ULEB128 == ParameterType &&
+      assert(AArch64BuildAttrs::ULEB128 == ParameterType &&
              "subsection .aeabi_feature_and_bits should "
              "be marked as uleb128 and not as ntbs");
       break;
@@ -421,8 +419,8 @@ AArch64ELFStreamer &AArch64TargetELFStreamer::getStreamer() {
 }
 
 void AArch64TargetELFStreamer::emitAtributesSubsection(
-    StringRef VendorName, AArch64BuildAttributes::SubsectionOptional IsOptional,
-    AArch64BuildAttributes::SubsectionType ParameterType) {
+    StringRef VendorName, AArch64BuildAttrs::SubsectionOptional IsOptional,
+    AArch64BuildAttrs::SubsectionType ParameterType) {
   AArch64TargetStreamer::emitAtributesSubsection(VendorName, IsOptional,
                                                  ParameterType);
 }
diff --git a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64TargetStreamer.cpp b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64TargetStreamer.cpp
index 74ffe5f97f1b6..1ed4a81a97673 100644
--- a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64TargetStreamer.cpp
+++ b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64TargetStreamer.cpp
@@ -153,8 +153,8 @@ MCTargetStreamer *llvm::createAArch64NullTargetStreamer(MCStreamer &S) {
 }
 
 void AArch64TargetStreamer::emitAtributesSubsection(
-    StringRef VendorName, AArch64BuildAttributes::SubsectionOptional IsOptional,
-    AArch64BuildAttributes::SubsectionType ParameterType) {
+    StringRef VendorName, AArch64BuildAttrs::SubsectionOptional IsOptional,
+    AArch64BuildAttrs::SubsectionType ParameterType) {
 
   // If exists, return.
   for (MCELFStreamer::AttributeSubSection &SubSection : AttributeSubSections) {
diff --git a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64TargetStreamer.h b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64TargetStreamer.h
index b2b9afe867073..9183fb4cc5f56 100644
--- a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64TargetStreamer.h
+++ b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64TargetStreamer.h
@@ -97,8 +97,8 @@ class AArch64TargetStreamer : public MCTargetStreamer {
   /// Build attributes implementation
   virtual void
   emitAtributesSubsection(StringRef VendorName,
-                          AArch64BuildAttributes::SubsectionOptional IsOptional,
-                          AArch64BuildAttributes::SubsectionType ParameterType);
+                          AArch64BuildAttrs::SubsectionOptional IsOptional,
+                          AArch64BuildAttrs::SubsectionType ParameterType);
   virtual void emitAttribute(StringRef VendorName, unsigned Tag, unsigned Value,
                              std::string String, bool Override);
   void activateAtributesSubsection(StringRef VendorName);
@@ -124,9 +124,8 @@ class AArch64TargetELFStreamer : public AArch64TargetStreamer {
 
   /// Build attributes implementation
   void emitAtributesSubsection(
-      StringRef VendorName,
-      AArch64BuildAttributes::SubsectionOptional IsOptional,
-      AArch64BuildAttributes::SubsectionType ParameterType) override;
+      StringRef VendorName, AArch64BuildAttrs::SubsectionOptional IsOptional,
+      AArch64BuildAttrs::SubsectionType ParameterType) override;
   void emitAttribute(StringRef VendorName, unsigned Tag, unsigned Value,
                      std::string String, bool Override = false) override;
   void emitInst(uint32_t Inst) override;
@@ -138,12 +137,6 @@ class AArch64TargetELFStreamer : public AArch64TargetStreamer {
 };
 
 class AArch64TargetWinCOFFStreamer : public llvm::AArch64TargetStreamer {
-private:
-  // True if we are processing SEH directives in an epilogue.
-  bool InEpilogCFI = false;
-
-  // Symbol of the current epilog for which we are processing SEH directives.
-  MCSymbol *CurrentEpilog = nullptr;
 public:
   AArch64TargetWinCOFFStreamer(llvm::MCStreamer &S)
     : AArch64TargetStreamer(S) {}
diff --git a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64WinCOFFStreamer.cpp b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64WinCOFFStreamer.cpp
index 208d43502cb88..8075f53ae3086 100644
--- a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64WinCOFFStreamer.cpp
+++ b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64WinCOFFStreamer.cpp
@@ -73,8 +73,8 @@ void AArch64TargetWinCOFFStreamer::emitARM64WinUnwindCode(unsigned UnwindCode,
   if (!CurFrame)
     return;
   auto Inst = WinEH::Instruction(UnwindCode, /*Label=*/nullptr, Reg, Offset);
-  if (InEpilogCFI)
-    CurFrame->EpilogMap[CurrentEpilog].Instructions.push_back(Inst);
+  if (S.isInEpilogCFI())
+    CurFrame->EpilogMap[S.getCurrentEpilog()].Instructions.push_back(Inst);
   else
     CurFrame->Instructions.push_back(Inst);
 }
@@ -183,13 +183,7 @@ void AArch64TargetWinCOFFStreamer::emitARM64WinCFIPrologEnd() {
 }
 
 void AArch64TargetWinCOFFStreamer::emitARM64WinCFIEpilogStart() {
-  auto &S = getStreamer();
-  WinEH::FrameInfo *CurFrame = S.EnsureValidWinFrameInfo(SMLoc());
-  if (!CurFrame)
-    return;
-
-  InEpilogCFI = true;
-  CurrentEpilog = S.emitCFILabel();
+  getStreamer().emitWinCFIBeginEpilogue();
 }
 
 void AArch64TargetWinCOFFStreamer::emitARM64WinCFIEpilogEnd() {
@@ -198,13 +192,12 @@ void AArch64TargetWinCOFFStreamer::emitARM64WinCFIEpilogEnd() {
   if (!CurFrame)
     return;
 
-  InEpilogCFI = false;
-  WinEH::Instruction Inst =
-      WinEH::Instruction(Win64EH::UOP_End, /*Label=*/nullptr, -1, 0);
-  CurFrame->EpilogMap[CurrentEpilog].Instructions.push_back(Inst);
-  MCSymbol *Label = S.emitCFILabel();
-  CurFrame->EpilogMap[CurrentEpilog].End = Label;
-  CurrentEpilog = nullptr;
+  if (S.isInEpilogCFI()) {
+    WinEH::Instruction Inst =
+        WinEH::Instruction(Win64EH::UOP_End, /*Label=*/nullptr, -1, 0);
+    CurFrame->EpilogMap[S.getCurrentEpilog()].Instructions.push_back(Inst);
+  }
+  S.emitWinCFIEndEpilogue();
 }
 
 void AArch64TargetWinCOFFStreamer::emitARM64WinCFITrapFrame() {
diff --git a/llvm/lib/Target/AArch64/SMEInstrFormats.td b/llvm/lib/Target/AArch64/SMEInstrFormats.td
index a01d59d0e5c43..4f6a413ba5e5c 100644
--- a/llvm/lib/Target/AArch64/SMEInstrFormats.td
+++ b/llvm/lib/Target/AArch64/SMEInstrFormats.td
@@ -36,27 +36,40 @@ let WantsRoot = true in
 def am_sme_indexed_b4 : ComplexPattern<iPTR, 2, "SelectAddrModeIndexedSVE<0, 15>">;
 
 // The FORM_TRANSPOSED_REG_TUPLE pseudos defined below are intended to
-// improve register allocation for intrinsics which use strided and contiguous
-// multi-vector registers, avoiding unnecessary copies.
-// If the operands of the pseudo are copies where the source register is in
-// the StridedOrContiguous class, the pseudo is used to provide a hint to the
-// register allocator suggesting a contigious multi-vector register which
-// matches the subregister sequence used by the operands.
-// If the operands do not match this pattern, the pseudos are expanded
-// to a REG_SEQUENCE using the post-isel hook.
+// improve register allocation for intrinsics which use strided and
+// contiguous multi-vector registers, avoiding unnecessary copies.
+// The SMEPeepholeOpt pass will replace a REG_SEQUENCE instruction with the
+// FORM_TRANSPOSED_REG_TUPLE pseudo if the operands are copies where the
+// source register is in the StridedOrContiguous class. The operands in the
+// sequence must all have the same subreg index.
+// The pseudo is then used to provide a hint to the register allocator
+// suggesting a contigious multi-vector register which matches the
+// subregister sequence used by the operands.
 
 def FORM_TRANSPOSED_REG_TUPLE_X2_PSEUDO :
   Pseudo<(outs ZPR2:$tup),
          (ins ZPR:$zn0, ZPR:$zn1), []>, Sched<[]>{
   let hasSideEffects = 0;
-  let hasPostISelHook = 1;
 }
 
 def FORM_TRANSPOSED_REG_TUPLE_X4_PSEUDO :
   Pseudo<(outs ZPR4:$tup),
          (ins ZPR:$zn0, ZPR:$zn1, ZPR:$zn2, ZPR:$zn3), []>, Sched<[]>{
   let hasSideEffects = 0;
-  let hasPostISelHook = 1;
+}
+
+def SPILL_PPR_TO_ZPR_SLOT_PSEUDO :
+  Pseudo<(outs), (ins PPRorPNRAny:$Pt, GPR64sp:$Rn, simm9:$imm9), []>, Sched<[]>
+{
+  let mayStore = 1;
+  let hasSideEffects = 0;
+}
+
+def FILL_PPR_FROM_ZPR_SLOT_PSEUDO :
+  Pseudo<(outs PPRorPNRAny:$Pt), (ins GPR64sp:$Rn, simm9:$imm9), []>, Sched<[]>
+{
+  let mayLoad = 1;
+  let hasSideEffects = 0;
 }
 
 def SDTZALoadStore : SDTypeProfile<0, 3, [SDTCisInt<0>, SDTCisPtrTy<1>, SDTCisInt<2>]>;
@@ -164,14 +177,14 @@ class SME2_ZA_TwoOp_Multi_Single_Pat<string name, SDPatternOperator intrinsic, O
 class SME2_ZA_TwoOp_VG2_Multi_Single_Pat<string name, SDPatternOperator intrinsic, Operand index_ty, ZPRRegOp zpr_ty,
                                          ValueType vt, ComplexPattern tileslice>
     : Pat<(intrinsic (i32 (tileslice MatrixIndexGPR32Op8_11:$base, index_ty:$offset)), vt:$Zn1, vt:$Zn2, vt:$Zm),
-          (!cast<Instruction>(name # _PSEUDO) $base, $offset, (FORM_TRANSPOSED_REG_TUPLE_X2_PSEUDO vt:$Zn1, vt:$Zn2),
+          (!cast<Instruction>(name # _PSEUDO) $base, $offset, (REG_SEQUENCE ZPR2, vt:$Zn1, zsub0, vt:$Zn2, zsub1),
                                               zpr_ty:$Zm)>;
 class SME2_ZA_TwoOp_VG4_Multi_Single_Pat<string name, SDPatternOperator intrinsic, Operand index_ty, ZPRRegOp zpr_ty,
                                          ValueType vt, ComplexPattern tileslice>
     : Pat<(intrinsic (i32 (tileslice MatrixIndexGPR32Op8_11:$base, index_ty:$offset)),
                      vt:$Zn1, vt:$Zn2, vt:$Zn3, vt:$Zn4, vt:$Zm),
           (!cast<Instruction>(name # _PSEUDO) $base, $offset,
-                                              (FORM_TRANSPOSED_REG_TUPLE_X4_PSEUDO vt:$Zn1, vt:$Zn2, vt:$Zn3, vt:$Zn4),
+                                              (REG_SEQUENCE ZPR4, vt:$Zn1, zsub0, vt:$Zn2, zsub1, vt:$Zn3, zsub2, vt:$Zn4, zsub3),
                                               zpr_ty:$Zm)>;
 
 class SME2_ZA_TwoOp_VG2_Multi_Multi_Pat<string name, SDPatternOperator intrinsic, Operand index_ty, ValueType vt, ComplexPattern tileslice>
@@ -197,14 +210,14 @@ class SME2_ZA_TwoOp_VG2_Multi_Index_Pat<string name, SDPatternOperator intrinsic
                                         Operand imm_ty, ComplexPattern tileslice>
     : Pat<(intrinsic (i32 (tileslice MatrixIndexGPR32Op8_11:$base, index_ty:$offset)), vt:$Zn1, vt:$Zn2, vt:$Zm, (i32 imm_ty:$i)),
           (!cast<Instruction>(name # _PSEUDO) $base, $offset,
-                                              (FORM_TRANSPOSED_REG_TUPLE_X2_PSEUDO vt:$Zn1,vt:$Zn2), zpr_ty:$Zm, imm_ty:$i)>;
+                                              (REG_SEQUENCE ZPR2Mul2, vt:$Zn1, zsub0, vt:$Zn2, zsub1), zpr_ty:$Zm, imm_ty:$i)>;
 
 class SME2_ZA_TwoOp_VG4_Multi_Index_Pat<string name, SDPatternOperator intrinsic, Operand index_ty, ZPRRegOp zpr_ty, ValueType vt,
                                         Operand imm_ty, ComplexPattern tileslice>
     : Pat<(intrinsic (i32 (tileslice MatrixIndexGPR32Op8_11:$base, index_ty:$offset)),
                      vt:$Zn1, vt:$Zn2, vt:$Zn3, vt:$Zn4, vt:$Zm, (i32 imm_ty:$i)),
           (!cast<Instruction>(name # _PSEUDO) $base, $offset,
-                                              (FORM_TRANSPOSED_REG_TUPLE_X4_PSEUDO vt:$Zn1, vt:$Zn2, vt:$Zn3, vt:$Zn4),
+                                              (REG_SEQUENCE ZPR4Mul4, vt:$Zn1, zsub0, vt:$Zn2, zsub1, vt:$Zn3, zsub2, vt:$Zn4, zsub3),
                                               zpr_ty:$Zm, imm_ty:$i)>;
 
 class SME2_Sat_Shift_VG2_Pat<string name, SDPatternOperator intrinsic, ValueType out_vt, ValueType in_vt, Operand imm_ty>
diff --git a/llvm/lib/Target/AArch64/SMEPeepholeOpt.cpp b/llvm/lib/Target/AArch64/SMEPeepholeOpt.cpp
index 4a0312d5b276f..2ffd4d7b3a382 100644
--- a/llvm/lib/Target/AArch64/SMEPeepholeOpt.cpp
+++ b/llvm/lib/Target/AArch64/SMEPeepholeOpt.cpp
@@ -45,6 +45,7 @@ struct SMEPeepholeOpt : public MachineFunctionPass {
 
   bool optimizeStartStopPairs(MachineBasicBlock &MBB,
                               bool &HasRemovedAllSMChanges) const;
+  bool visitRegSequence(MachineInstr &MI);
 };
 
 char SMEPeepholeOpt::ID = 0;
@@ -225,6 +226,81 @@ bool SMEPeepholeOpt::optimizeStartStopPairs(
   return Changed;
 }
 
+// Using the FORM_TRANSPOSED_REG_TUPLE pseudo can improve register allocation
+// of multi-vector intrinsics. However, the psuedo should only be emitted if
+// the input registers of the REG_SEQUENCE are copy nodes where the source
+// register is in a StridedOrContiguous class. For example:
+//
+//   %3:zpr2stridedorcontiguous = LD1B_2Z_IMM_PSEUDO ..
+//   %4:zpr = COPY %3.zsub1:zpr2stridedorcontiguous
+//   %5:zpr = COPY %3.zsub0:zpr2stridedorcontiguous
+//   %6:zpr2stridedorcontiguous = LD1B_2Z_PSEUDO ..
+//   %7:zpr = COPY %6.zsub1:zpr2stridedorcontiguous
+//   %8:zpr = COPY %6.zsub0:zpr2stridedorcontiguous
+//   %9:zpr2mul2 = REG_SEQUENCE %5:zpr, %subreg.zsub0, %8:zpr, %subreg.zsub1
+//
+//   ->  %9:zpr2mul2 = FORM_TRANSPOSED_REG_TUPLE_X2_PSEUDO %5:zpr, %8:zpr
+//
+bool SMEPeepholeOpt::visitRegSequence(MachineInstr &MI) {
+  assert(MI.getMF()->getRegInfo().isSSA() && "Expected to be run on SSA form!");
+
+  MachineRegisterInfo &MRI = MI.getMF()->getRegInfo();
+  switch (MRI.getRegClass(MI.getOperand(0).getReg())->getID()) {
+  case AArch64::ZPR2RegClassID:
+  case AArch64::ZPR4RegClassID:
+  case AArch64::ZPR2Mul2RegClassID:
+  case AArch64::ZPR4Mul4RegClassID:
+    break;
+  default:
+    return false;
+  }
+
+  // The first operand is the register class created by the REG_SEQUENCE.
+  // Each operand pair after this consists of a vreg + subreg index, so
+  // for example a sequence of 2 registers will have a total of 5 operands.
+  if (MI.getNumOperands() != 5 && MI.getNumOperands() != 9)
+    return false;
+
+  MCRegister SubReg = MCRegister::NoRegister;
+  for (unsigned I = 1; I < MI.getNumOperands(); I += 2) {
+    MachineOperand &MO = MI.getOperand(I);
+
+    MachineOperand *Def = MRI.getOneDef(MO.getReg());
+    if (!Def || !Def->getParent()->isCopy())
+      return false;
+
+    const MachineOperand &CopySrc = Def->getParent()->getOperand(1);
+    unsigned OpSubReg = CopySrc.getSubReg();
+    if (SubReg == MCRegister::NoRegister)
+      SubReg = OpSubReg;
+
+    MachineOperand *CopySrcOp = MRI.getOneDef(CopySrc.getReg());
+    if (!CopySrcOp || !CopySrcOp->isReg() || OpSubReg != SubReg ||
+        CopySrcOp->getReg().isPhysical())
+      return false;
+
+    const TargetRegisterClass *CopySrcClass =
+        MRI.getRegClass(CopySrcOp->getReg());
+    if (CopySrcClass != &AArch64::ZPR2StridedOrContiguousRegClass &&
+        CopySrcClass != &AArch64::ZPR4StridedOrContiguousRegClass)
+      return false;
+  }
+
+  unsigned Opc = MI.getNumOperands() == 5
+                     ? AArch64::FORM_TRANSPOSED_REG_TUPLE_X2_PSEUDO
+                     : AArch64::FORM_TRANSPOSED_REG_TUPLE_X4_PSEUDO;
+
+  const TargetInstrInfo *TII =
+      MI.getMF()->getSubtarget<AArch64Subtarget>().getInstrInfo();
+  MachineInstrBuilder MIB = BuildMI(*MI.getParent(), MI, MI.getDebugLoc(),
+                                    TII->get(Opc), MI.getOperand(0).getReg());
+  for (unsigned I = 1; I < MI.getNumOperands(); I += 2)
+    MIB.addReg(MI.getOperand(I).getReg());
+
+  MI.eraseFromParent();
+  return true;
+}
+
 INITIALIZE_PASS(SMEPeepholeOpt, "aarch64-sme-peephole-opt",
                 "SME Peephole Optimization", false, false)
 
@@ -247,6 +323,12 @@ bool SMEPeepholeOpt::runOnMachineFunction(MachineFunction &MF) {
     bool BlockHasAllSMChangesRemoved;
     Changed |= optimizeStartStopPairs(MBB, BlockHasAllSMChangesRemoved);
     FunctionHasAllSMChangesRemoved |= BlockHasAllSMChangesRemoved;
+
+    if (MF.getSubtarget<AArch64Subtarget>().isStreaming()) {
+      for (MachineInstr &MI : make_early_inc_range(MBB))
+        if (MI.getOpcode() == AArch64::REG_SEQUENCE)
+          Changed |= visitRegSequence(MI);
+    }
   }
 
   AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>();
diff --git a/llvm/lib/Target/AArch64/SVEInstrFormats.td b/llvm/lib/Target/AArch64/SVEInstrFormats.td
index 2ee9910da5079..e443c5ab150bd 100644
--- a/llvm/lib/Target/AArch64/SVEInstrFormats.td
+++ b/llvm/lib/Target/AArch64/SVEInstrFormats.td
@@ -662,6 +662,17 @@ multiclass SVE_InReg_Extend_PassthruUndef<ValueType vt, SDPatternOperator op, Va
             (inst $PassThru, $Pg, $Src)>;
 }
 
+multiclass SVE_InReg_Extend_PassthruUndefZero<ValueType vt, SDPatternOperator op, ValueType pt,
+                                         ValueType inreg_vt, Instruction inst> {
+  let AddedComplexity = 1 in {
+    def : Pat<(vt (op pt:$Pg, vt:$Src, inreg_vt, (vt (SVEDup0Undef)))),
+               (inst $Pg, $Src)>;
+
+    def : Pat<(vt (op (pt (SVEAllActive:$Pg)), vt:$Src, inreg_vt, (vt (SVEAny)))),
+              (inst $Pg, $Src)>;
+  }
+}
+
 class SVE_Shift_DupImm_Pred_Pat<ValueType vt, SDPatternOperator op,
                                 ValueType pt, ValueType it,
                                 ComplexPattern cast, Instruction inst>
@@ -3270,10 +3281,17 @@ multiclass sve_fp_z2op_p_zd<string asm, SDPatternOperator op> {
   defm : SVE_3_Op_UndefZero_Pat<nxv4f32, op, nxv4f32, nxv2i1, nxv2f64, !cast<Instruction>(NAME # _DtoS)>;
 }
 
-multiclass sve_fp_z2op_p_zd_hsd<bits<5> opc, string asm> {
+multiclass sve_fp_z2op_p_zd_hsd<bits<5> opc, string asm, SDPatternOperator op> {
   def _H : sve_fp_z2op_p_zd<{ 0b01, opc }, asm, ZPR16, ZPR16>;
   def _S : sve_fp_z2op_p_zd<{ 0b10, opc }, asm, ZPR32, ZPR32>;
   def _D : sve_fp_z2op_p_zd<{ 0b11, opc }, asm, ZPR64, ZPR64>;
+
+  defm : SVE_1_Op_PassthruUndefZero_Pat<nxv8f16, op, nxv8i1, nxv8f16, !cast<Instruction>(NAME # _H)>;
+  defm : SVE_1_Op_PassthruUndefZero_Pat<nxv4f16, op, nxv4i1, nxv4f16, !cast<Instruction>(NAME # _H)>;
+  defm : SVE_1_Op_PassthruUndefZero_Pat<nxv2f16, op, nxv2i1, nxv2f16, !cast<Instruction>(NAME # _H)>;
+  defm : SVE_1_Op_PassthruUndefZero_Pat<nxv4f32, op, nxv4i1, nxv4f32, !cast<Instruction>(NAME # _S)>;
+  defm : SVE_1_Op_PassthruUndefZero_Pat<nxv2f32, op, nxv2i1, nxv2f32, !cast<Instruction>(NAME # _S)>;
+  defm : SVE_1_Op_PassthruUndefZero_Pat<nxv2f64, op, nxv2i1, nxv2f64, !cast<Instruction>(NAME # _D)>;
 }
 
 multiclass sve_fp_z2op_p_zd_frint<bits<2> opc, string asm> {
@@ -3325,10 +3343,14 @@ multiclass sve_fp_z2op_p_zd_c<bit U, string asm, string int_op, SDPatternOperato
   defm : SVE_1_Op_PassthruUndefZero_Pat<nxv2f64, ir_op, nxv2i1,nxv2i64, !cast<Instruction>(NAME # _DtoD)>;
 }
 
-multiclass sve_fp_z2op_p_zd_d_flogb<string asm> {
+multiclass sve_fp_z2op_p_zd_d_flogb<string asm, SDPatternOperator op> {
   def _H : sve_fp_z2op_p_zd<0b0011001, asm, ZPR16, ZPR16>;
   def _S : sve_fp_z2op_p_zd<0b0011010, asm, ZPR32, ZPR32>;
   def _D : sve_fp_z2op_p_zd<0b0011011, asm, ZPR64, ZPR64>;
+
+  defm : SVE_3_Op_UndefZero_Pat<nxv8i16, op, nxv8i16, nxv8i1, nxv8f16, !cast<Instruction>(NAME # _H)>;
+  defm : SVE_3_Op_UndefZero_Pat<nxv4i32, op, nxv4i32, nxv4i1, nxv4f32, !cast<Instruction>(NAME # _S)>;
+  defm : SVE_3_Op_UndefZero_Pat<nxv2i64, op, nxv2i64, nxv2i1, nxv2f64, !cast<Instruction>(NAME # _D)>;
 }
 
 multiclass sve_fp_z2op_p_zd_b_0<string asm, string op> {
@@ -4189,11 +4211,22 @@ multiclass sve2_int_un_pred_arit<bits<2> opc, string asm, SDPatternOperator op>
   defm : SVE_3_Op_Undef_Pat<nxv2i64, op, nxv2i64, nxv2i1,  nxv2i64, !cast<Pseudo>(NAME # _D_UNDEF)>;
 }
 
-multiclass sve2_int_un_pred_arit_z<bits<2> opc, string asm> {
+multiclass sve2_int_un_pred_arit_z_S<bits<2> opc, string asm, SDPatternOperator op> {
+  def _S : sve2_int_un_pred_arit_z<0b10, opc, asm, ZPR32>;
+
+  defm : SVE_3_Op_UndefZero_Pat<nxv4i32, op, nxv4i32, nxv4i1, nxv4i32, !cast<Instruction>(NAME # _S)>;
+}
+
+multiclass sve2_int_un_pred_arit_z<bits<2> opc, string asm, SDPatternOperator op> {
   def _B : sve2_int_un_pred_arit_z<0b00, opc, asm, ZPR8>;
   def _H : sve2_int_un_pred_arit_z<0b01, opc, asm, ZPR16>;
   def _S : sve2_int_un_pred_arit_z<0b10, opc, asm, ZPR32>;
   def _D : sve2_int_un_pred_arit_z<0b11, opc, asm, ZPR64>;
+
+  defm : SVE_3_Op_UndefZero_Pat<nxv16i8, op, nxv16i8, nxv16i1, nxv16i8, !cast<Instruction>(NAME # _B)>;
+  defm : SVE_3_Op_UndefZero_Pat<nxv8i16, op, nxv8i16, nxv8i1,  nxv8i16, !cast<Instruction>(NAME # _H)>;
+  defm : SVE_3_Op_UndefZero_Pat<nxv4i32, op, nxv4i32, nxv4i1, nxv4i32, !cast<Instruction>(NAME # _S)>;
+  defm : SVE_3_Op_UndefZero_Pat<nxv2i64, op, nxv2i64, nxv2i1,  nxv2i64, !cast<Instruction>(NAME # _D)>;
 }
 
 //===----------------------------------------------------------------------===//
@@ -4908,10 +4941,14 @@ multiclass sve_int_un_pred_arit_h<bits<3> opc, string asm,
   defm : SVE_InReg_Extend_PassthruUndef<nxv2i64, op, nxv2i1, nxv2i8, !cast<Pseudo>(NAME # _D_UNDEF)>;
 }
 
-multiclass sve_int_un_pred_arit_h_z<bits<3> opc, string asm> {
+multiclass sve_int_un_pred_arit_h_z<bits<3> opc, string asm, SDPatternOperator op> {
   def _H : sve_int_un_pred_arit_z<0b01, { opc, 0b0 }, asm, ZPR16>;
   def _S : sve_int_un_pred_arit_z<0b10, { opc, 0b0 }, asm, ZPR32>;
   def _D : sve_int_un_pred_arit_z<0b11, { opc, 0b0 }, asm, ZPR64>;
+
+  defm : SVE_InReg_Extend_PassthruUndefZero<nxv8i16, op, nxv8i1, nxv8i8, !cast<Instruction>(NAME # _H)>;
+  defm : SVE_InReg_Extend_PassthruUndefZero<nxv4i32, op, nxv4i1, nxv4i8, !cast<Instruction>(NAME # _S)>;
+  defm : SVE_InReg_Extend_PassthruUndefZero<nxv2i64, op, nxv2i1, nxv2i8, !cast<Instruction>(NAME # _D)>;
 }
 
 multiclass sve_int_un_pred_arit_w<bits<3> opc, string asm,
@@ -4931,9 +4968,12 @@ multiclass sve_int_un_pred_arit_w<bits<3> opc, string asm,
   defm : SVE_InReg_Extend_PassthruUndef<nxv2i64, op, nxv2i1, nxv2i16, !cast<Pseudo>(NAME # _D_UNDEF)>;
 }
 
-multiclass sve_int_un_pred_arit_w_z<bits<3> opc, string asm> {
+multiclass sve_int_un_pred_arit_w_z<bits<3> opc, string asm, SDPatternOperator op> {
   def _S : sve_int_un_pred_arit_z<0b10, { opc, 0b0 }, asm, ZPR32>;
   def _D : sve_int_un_pred_arit_z<0b11, { opc, 0b0 }, asm, ZPR64>;
+
+  defm : SVE_InReg_Extend_PassthruUndefZero<nxv4i32, op, nxv4i1, nxv4i16, !cast<Instruction>(NAME # _S)>;
+  defm : SVE_InReg_Extend_PassthruUndefZero<nxv2i64, op, nxv2i1, nxv2i16, !cast<Instruction>(NAME # _D)>;
 }
 
 multiclass sve_int_un_pred_arit_d<bits<3> opc, string asm,
@@ -4948,6 +4988,12 @@ multiclass sve_int_un_pred_arit_d<bits<3> opc, string asm,
   defm : SVE_InReg_Extend_PassthruUndef<nxv2i64, op, nxv2i1, nxv2i32, !cast<Pseudo>(NAME # _D_UNDEF)>;
 }
 
+multiclass sve_int_un_pred_arit_d_z<bits<3> opc, string asm, SDPatternOperator op> {
+  def _D : sve_int_un_pred_arit_z<0b11, {opc, 0b0}, asm, ZPR64>;
+
+  defm : SVE_InReg_Extend_PassthruUndefZero<nxv2i64, op, nxv2i1, nxv2i32, !cast<Instruction>(NAME # _D)>;
+}
+
 multiclass sve_int_un_pred_arit_bitwise<bits<3> opc, string asm,
                                         SDPatternOperator op> {
   def _B : sve_int_un_pred_arit<0b00, { opc, 0b1 }, asm, ZPR8>,
@@ -7635,22 +7681,54 @@ class sve_int_perm_rev_z<bits<2> sz, bits<4> opc, string asm,
   let hasSideEffects = 0;
 }
 
-multiclass sve_int_perm_rev_rbit_z<string asm> {
+multiclass sve_int_perm_rev_rbit_z<string asm, SDPatternOperator op> {
   def _B : sve_int_perm_rev_z<0b00, 0b0111, asm, ZPR8>;
   def _H : sve_int_perm_rev_z<0b01, 0b0111, asm, ZPR16>;
   def _S : sve_int_perm_rev_z<0b10, 0b0111, asm, ZPR32>;
   def _D : sve_int_perm_rev_z<0b11, 0b0111, asm, ZPR64>;
+
+  defm : SVE_1_Op_PassthruUndefZero_Pat<nxv16i8, op, nxv16i1, nxv16i8, !cast<Instruction>(NAME # _B)>;
+  defm : SVE_1_Op_PassthruUndefZero_Pat<nxv8i16, op, nxv8i1,  nxv8i16, !cast<Instruction>(NAME # _H)>;
+  defm : SVE_1_Op_PassthruUndefZero_Pat<nxv4i32, op, nxv4i1,  nxv4i32, !cast<Instruction>(NAME # _S)>;
+  defm : SVE_1_Op_PassthruUndefZero_Pat<nxv2i64, op, nxv2i1,  nxv2i64, !cast<Instruction>(NAME # _D)>;
 }
 
-multiclass sve_int_perm_rev_revb_z<string asm> {
+multiclass sve_int_perm_rev_revb_z<string asm, SDPatternOperator op> {
   def _H : sve_int_perm_rev_z<0b01, 0b0100, asm, ZPR16>;
   def _S : sve_int_perm_rev_z<0b10, 0b0100, asm, ZPR32>;
   def _D : sve_int_perm_rev_z<0b11, 0b0100, asm, ZPR64>;
+
+  defm : SVE_1_Op_PassthruUndefZero_Pat<nxv8i16, op, nxv8i1, nxv8i16, !cast<Instruction>(NAME # _H)>;
+  defm : SVE_1_Op_PassthruUndefZero_Pat<nxv4i32, op, nxv4i1, nxv4i32, !cast<Instruction>(NAME # _S)>;
+  defm : SVE_1_Op_PassthruUndefZero_Pat<nxv2i64, op, nxv2i1, nxv2i64, !cast<Instruction>(NAME # _D)>;
 }
 
-multiclass sve_int_perm_rev_revh_z<string asm> {
+multiclass sve_int_perm_rev_revh_z<string asm, SDPatternOperator op> {
   def _S : sve_int_perm_rev_z<0b10, 0b0101, asm, ZPR32>;
   def _D : sve_int_perm_rev_z<0b11, 0b0101, asm, ZPR64>;
+
+  defm : SVE_1_Op_PassthruUndefZero_Pat<nxv4i32, op, nxv4i1, nxv4i32, !cast<Instruction>(NAME # _S)>;
+  defm : SVE_1_Op_PassthruUndefZero_Pat<nxv2i64, op, nxv2i1, nxv2i64, !cast<Instruction>(NAME # _D)>;
+}
+
+multiclass sve_int_perm_rev_revw_z<string asm, SDPatternOperator op> {
+  def _D : sve_int_perm_rev_z<0b11, 0b0110, asm, ZPR64>;
+
+  defm : SVE_1_Op_PassthruUndefZero_Pat<nxv2i64, op, nxv2i1, nxv2i64, !cast<Instruction>(NAME # _D)>;
+}
+
+multiclass sve_int_perm_rev_revd_z<string asm, SDPatternOperator op> {
+  def NAME : sve_int_perm_rev_z<0b00, 0b1110, asm, ZPR128>;
+
+  defm : SVE_1_Op_PassthruUndefZero_Pat<nxv16i8, op, nxv16i1, nxv16i8, !cast<Instruction>(NAME)>;
+  defm : SVE_1_Op_PassthruUndefZero_Pat<nxv8i16, op, nxv8i1,  nxv8i16, !cast<Instruction>(NAME)>;
+  defm : SVE_1_Op_PassthruUndefZero_Pat<nxv4i32, op, nxv4i1,  nxv4i32, !cast<Instruction>(NAME)>;
+  defm : SVE_1_Op_PassthruUndefZero_Pat<nxv2i64, op, nxv2i1,  nxv2i64, !cast<Instruction>(NAME)>;
+
+  defm : SVE_1_Op_PassthruUndefZero_Pat<nxv8bf16, op, nxv8i1, nxv8bf16, !cast<Instruction>(NAME)>;
+  defm : SVE_1_Op_PassthruUndefZero_Pat<nxv8f16,  op, nxv8i1, nxv8f16,  !cast<Instruction>(NAME)>;
+  defm : SVE_1_Op_PassthruUndefZero_Pat<nxv4f32,  op, nxv4i1, nxv4f32,  !cast<Instruction>(NAME)>;
+  defm : SVE_1_Op_PassthruUndefZero_Pat<nxv2f64,  op, nxv2i1, nxv2f64,  !cast<Instruction>(NAME)>;
 }
 
 class sve_int_perm_cpy_r<bits<2> sz8_64, string asm, ZPRRegOp zprty,
diff --git a/llvm/lib/Target/AMDGPU/AMDGPU.h b/llvm/lib/Target/AMDGPU/AMDGPU.h
index 3d5a44a3623a0..67bad5884c260 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPU.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPU.h
@@ -41,7 +41,7 @@ FunctionPass *createSIPeepholeSDWALegacyPass();
 FunctionPass *createSILowerI1CopiesLegacyPass();
 FunctionPass *createSIShrinkInstructionsLegacyPass();
 FunctionPass *createSILoadStoreOptimizerLegacyPass();
-FunctionPass *createSIWholeQuadModePass();
+FunctionPass *createSIWholeQuadModeLegacyPass();
 FunctionPass *createSIFixControlFlowLiveIntervalsPass();
 FunctionPass *createSIOptimizeExecMaskingPreRAPass();
 FunctionPass *createSIOptimizeVGPRLiveRangeLegacyPass();
@@ -63,7 +63,7 @@ ModulePass *
 createAMDGPULowerModuleLDSLegacyPass(const AMDGPUTargetMachine *TM = nullptr);
 ModulePass *createAMDGPULowerBufferFatPointersPass();
 FunctionPass *createSIModeRegisterPass();
-FunctionPass *createGCNPreRAOptimizationsPass();
+FunctionPass *createGCNPreRAOptimizationsLegacyPass();
 FunctionPass *createAMDGPUPreloadKernArgPrologLegacyPass();
 
 struct AMDGPUSimplifyLibCallsPass : PassInfoMixin<AMDGPUSimplifyLibCallsPass> {
@@ -204,7 +204,7 @@ extern char &SILowerSGPRSpillsLegacyID;
 void initializeSILoadStoreOptimizerLegacyPass(PassRegistry &);
 extern char &SILoadStoreOptimizerLegacyID;
 
-void initializeSIWholeQuadModePass(PassRegistry &);
+void initializeSIWholeQuadModeLegacyPass(PassRegistry &);
 extern char &SIWholeQuadModeID;
 
 void initializeSILowerControlFlowLegacyPass(PassRegistry &);
@@ -451,10 +451,10 @@ extern char &AMDGPUOpenCLEnqueuedBlockLoweringLegacyID;
 void initializeGCNNSAReassignPass(PassRegistry &);
 extern char &GCNNSAReassignID;
 
-void initializeGCNPreRALongBranchRegPass(PassRegistry &);
+void initializeGCNPreRALongBranchRegLegacyPass(PassRegistry &);
 extern char &GCNPreRALongBranchRegID;
 
-void initializeGCNPreRAOptimizationsPass(PassRegistry &);
+void initializeGCNPreRAOptimizationsLegacyPass(PassRegistry &);
 extern char &GCNPreRAOptimizationsID;
 
 FunctionPass *createAMDGPUSetWavePriorityPass();
@@ -463,6 +463,9 @@ void initializeAMDGPUSetWavePriorityPass(PassRegistry &);
 void initializeGCNRewritePartialRegUsesPass(llvm::PassRegistry &);
 extern char &GCNRewritePartialRegUsesID;
 
+void initializeAMDGPUWaitSGPRHazardsLegacyPass(PassRegistry &);
+extern char &AMDGPUWaitSGPRHazardsLegacyID;
+
 namespace AMDGPU {
 enum TargetIndex {
   TI_CONSTDATA_START,
diff --git a/llvm/lib/Target/AMDGPU/AMDGPU.td b/llvm/lib/Target/AMDGPU/AMDGPU.td
index 7ad6720b8001a..6439149d801f6 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPU.td
+++ b/llvm/lib/Target/AMDGPU/AMDGPU.td
@@ -2498,6 +2498,9 @@ def HasNotMADIntraFwdBug : Predicate<"!Subtarget->hasMADIntraFwdBug()">;
 def HasSALUFloatInsts : Predicate<"Subtarget->hasSALUFloatInsts()">,
   AssemblerPredicate<(all_of FeatureSALUFloatInsts)>;
 
+def NotHasSALUFloatInsts : Predicate<"!Subtarget->hasSALUFloatInsts()">,
+  AssemblerPredicate<(all_of (not FeatureSALUFloatInsts))>;
+
 def HasPseudoScalarTrans : Predicate<"Subtarget->hasPseudoScalarTrans()">,
   AssemblerPredicate<(all_of FeaturePseudoScalarTrans)>;
 
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUCallLowering.cpp b/llvm/lib/Target/AMDGPU/AMDGPUCallLowering.cpp
index bb00442342d84..478a4c161fce7 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUCallLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUCallLowering.cpp
@@ -816,7 +816,7 @@ bool AMDGPUCallLowering::passSpecialInputs(MachineIRBuilder &MIRBuilder,
     Register InputReg = MRI.createGenericVirtualRegister(ArgTy);
 
     if (IncomingArg) {
-      LI->loadInputValue(InputReg, MIRBuilder, IncomingArg, ArgRC, ArgTy);
+      LI->buildLoadInputValue(InputReg, MIRBuilder, IncomingArg, ArgRC, ArgTy);
     } else if (InputID == AMDGPUFunctionArgInfo::IMPLICIT_ARG_PTR) {
       LI->getImplicitArgPtr(InputReg, MRI, MIRBuilder);
     } else if (InputID == AMDGPUFunctionArgInfo::LDS_KERNEL_ID) {
@@ -883,8 +883,9 @@ bool AMDGPUCallLowering::passSpecialInputs(MachineIRBuilder &MIRBuilder,
       NeedWorkItemIDX) {
     if (ST.getMaxWorkitemID(MF.getFunction(), 0) != 0) {
       InputReg = MRI.createGenericVirtualRegister(S32);
-      LI->loadInputValue(InputReg, MIRBuilder, IncomingArgX,
-                         std::get<1>(WorkitemIDX), std::get<2>(WorkitemIDX));
+      LI->buildLoadInputValue(InputReg, MIRBuilder, IncomingArgX,
+                              std::get<1>(WorkitemIDX),
+                              std::get<2>(WorkitemIDX));
     } else {
       InputReg = MIRBuilder.buildConstant(S32, 0).getReg(0);
     }
@@ -893,8 +894,8 @@ bool AMDGPUCallLowering::passSpecialInputs(MachineIRBuilder &MIRBuilder,
   if (IncomingArgY && !IncomingArgY->isMasked() && CalleeArgInfo->WorkItemIDY &&
       NeedWorkItemIDY && ST.getMaxWorkitemID(MF.getFunction(), 1) != 0) {
     Register Y = MRI.createGenericVirtualRegister(S32);
-    LI->loadInputValue(Y, MIRBuilder, IncomingArgY, std::get<1>(WorkitemIDY),
-                       std::get<2>(WorkitemIDY));
+    LI->buildLoadInputValue(Y, MIRBuilder, IncomingArgY,
+                            std::get<1>(WorkitemIDY), std::get<2>(WorkitemIDY));
 
     Y = MIRBuilder.buildShl(S32, Y, MIRBuilder.buildConstant(S32, 10)).getReg(0);
     InputReg = InputReg ? MIRBuilder.buildOr(S32, InputReg, Y).getReg(0) : Y;
@@ -903,8 +904,8 @@ bool AMDGPUCallLowering::passSpecialInputs(MachineIRBuilder &MIRBuilder,
   if (IncomingArgZ && !IncomingArgZ->isMasked() && CalleeArgInfo->WorkItemIDZ &&
       NeedWorkItemIDZ && ST.getMaxWorkitemID(MF.getFunction(), 2) != 0) {
     Register Z = MRI.createGenericVirtualRegister(S32);
-    LI->loadInputValue(Z, MIRBuilder, IncomingArgZ, std::get<1>(WorkitemIDZ),
-                       std::get<2>(WorkitemIDZ));
+    LI->buildLoadInputValue(Z, MIRBuilder, IncomingArgZ,
+                            std::get<1>(WorkitemIDZ), std::get<2>(WorkitemIDZ));
 
     Z = MIRBuilder.buildShl(S32, Z, MIRBuilder.buildConstant(S32, 20)).getReg(0);
     InputReg = InputReg ? MIRBuilder.buildOr(S32, InputReg, Z).getReg(0) : Z;
@@ -925,8 +926,8 @@ bool AMDGPUCallLowering::passSpecialInputs(MachineIRBuilder &MIRBuilder,
       ArgDescriptor IncomingArg = ArgDescriptor::createArg(
         IncomingArgX ? *IncomingArgX :
         IncomingArgY ? *IncomingArgY : *IncomingArgZ, ~0u);
-      LI->loadInputValue(InputReg, MIRBuilder, &IncomingArg,
-                         &AMDGPU::VGPR_32RegClass, S32);
+      LI->buildLoadInputValue(InputReg, MIRBuilder, &IncomingArg,
+                              &AMDGPU::VGPR_32RegClass, S32);
     }
   }
 
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUGlobalISelUtils.cpp b/llvm/lib/Target/AMDGPU/AMDGPUGlobalISelUtils.cpp
index d64337c4cb909..0b18c6b0e923a 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUGlobalISelUtils.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUGlobalISelUtils.cpp
@@ -56,7 +56,7 @@ AMDGPU::getBaseWithConstantOffset(MachineRegisterInfo &MRI, Register Reg,
 
   Register Base;
   if (KnownBits && mi_match(Reg, MRI, m_GOr(m_Reg(Base), m_ICst(Offset))) &&
-      KnownBits->maskedValueIsZero(Base, APInt(32, Offset)))
+      KnownBits->maskedValueIsZero(Base, APInt(32, Offset, /*isSigned=*/true)))
     return std::pair(Base, Offset);
 
   // Handle G_PTRTOINT (G_PTR_ADD base, const) case
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp b/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp
index cca9fa72d0ca5..792e17eeedab1 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp
@@ -4217,18 +4217,21 @@ SDValue AMDGPUTargetLowering::performTruncateCombine(
   // trunc (srl (bitcast (build_vector x, y))), 16 -> trunc (bitcast y)
   if (Src.getOpcode() == ISD::SRL && !VT.isVector()) {
     if (auto *K = isConstOrConstSplat(Src.getOperand(1))) {
-      if (2 * K->getZExtValue() == Src.getValueType().getScalarSizeInBits()) {
-        SDValue BV = stripBitcast(Src.getOperand(0));
-        if (BV.getOpcode() == ISD::BUILD_VECTOR &&
-            BV.getValueType().getVectorNumElements() == 2) {
-          SDValue SrcElt = BV.getOperand(1);
-          EVT SrcEltVT = SrcElt.getValueType();
-          if (SrcEltVT.isFloatingPoint()) {
-            SrcElt = DAG.getNode(ISD::BITCAST, SL,
-                                 SrcEltVT.changeTypeToInteger(), SrcElt);
+      SDValue BV = stripBitcast(Src.getOperand(0));
+      if (BV.getOpcode() == ISD::BUILD_VECTOR) {
+        EVT SrcEltVT = BV.getOperand(0).getValueType();
+        unsigned SrcEltSize = SrcEltVT.getSizeInBits();
+        unsigned BitIndex = K->getZExtValue();
+        unsigned PartIndex = BitIndex / SrcEltSize;
+
+        if (PartIndex * SrcEltSize == BitIndex &&
+            PartIndex < BV.getNumOperands()) {
+          if (SrcEltVT.getSizeInBits() == VT.getSizeInBits()) {
+            SDValue SrcElt =
+                DAG.getNode(ISD::BITCAST, SL, SrcEltVT.changeTypeToInteger(),
+                            BV.getOperand(PartIndex));
+            return DAG.getNode(ISD::TRUNCATE, SL, VT, SrcElt);
           }
-
-          return DAG.getNode(ISD::TRUNCATE, SL, VT, SrcElt);
         }
       }
     }
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp b/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp
index 40eaba2c09209..3bbbbcf71d8ae 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp
@@ -1207,9 +1207,8 @@ static int getV_CMPOpcode(CmpInst::Predicate P, unsigned Size,
                           unsigned FakeS16Opc, unsigned S32Opc,
                           unsigned S64Opc) {
     if (Size == 16)
-      // FIXME-TRUE16 use TrueS16Opc when realtrue16 is supported for CMP code
       return ST.hasTrue16BitInsts()
-                 ? ST.useRealTrue16Insts() ? FakeS16Opc : FakeS16Opc
+                 ? ST.useRealTrue16Insts() ? TrueS16Opc : FakeS16Opc
                  : S16Opc;
     if (Size == 32)
       return S32Opc;
diff --git a/llvm/lib/Target/AMDGPU/AMDGPULateCodeGenPrepare.cpp b/llvm/lib/Target/AMDGPU/AMDGPULateCodeGenPrepare.cpp
index f4e651ec477d3..b8109db821bcc 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPULateCodeGenPrepare.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPULateCodeGenPrepare.cpp
@@ -367,11 +367,11 @@ bool LiveRegOptimizer::optimizeLiveType(
   for (Instruction *U : Uses) {
     // Replace all converted operands for a use.
     for (auto [OpIdx, Op] : enumerate(U->operands())) {
-      if (ValMap.contains(Op) && ValMap[Op]) {
+      if (Value *Val = ValMap.lookup(Op)) {
         Value *NewVal = nullptr;
         if (BBUseValMap.contains(U->getParent()) &&
-            BBUseValMap[U->getParent()].contains(ValMap[Op]))
-          NewVal = BBUseValMap[U->getParent()][ValMap[Op]];
+            BBUseValMap[U->getParent()].contains(Val))
+          NewVal = BBUseValMap[U->getParent()][Val];
         else {
           BasicBlock::iterator InsertPt = U->getParent()->getFirstNonPHIIt();
           // We may pick up ops that were previously converted for users in
diff --git a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
index e9e47eaadd557..908d323c7fec9 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
@@ -4275,10 +4275,11 @@ verifyCFIntrinsic(MachineInstr &MI, MachineRegisterInfo &MRI, MachineInstr *&Br,
   return UseMI;
 }
 
-bool AMDGPULegalizerInfo::loadInputValue(Register DstReg, MachineIRBuilder &B,
-                                         const ArgDescriptor *Arg,
-                                         const TargetRegisterClass *ArgRC,
-                                         LLT ArgTy) const {
+void AMDGPULegalizerInfo::buildLoadInputValue(Register DstReg,
+                                              MachineIRBuilder &B,
+                                              const ArgDescriptor *Arg,
+                                              const TargetRegisterClass *ArgRC,
+                                              LLT ArgTy) const {
   MCRegister SrcReg = Arg->getRegister();
   assert(SrcReg.isPhysical() && "Physical register expected");
   assert(DstReg.isVirtual() && "Virtual register expected");
@@ -4304,8 +4305,6 @@ bool AMDGPULegalizerInfo::loadInputValue(Register DstReg, MachineIRBuilder &B,
   } else {
     B.buildCopy(DstReg, LiveIn);
   }
-
-  return true;
 }
 
 bool AMDGPULegalizerInfo::loadInputValue(
@@ -4369,7 +4368,8 @@ bool AMDGPULegalizerInfo::loadInputValue(
 
   if (!Arg->isRegister() || !Arg->getRegister().isValid())
     return false; // TODO: Handle these
-  return loadInputValue(DstReg, B, Arg, ArgRC, ArgTy);
+  buildLoadInputValue(DstReg, B, Arg, ArgRC, ArgTy);
+  return true;
 }
 
 bool AMDGPULegalizerInfo::legalizePreloadedArgIntrin(
diff --git a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.h b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.h
index 86c15197805d2..03b7c36fc450f 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.h
@@ -111,9 +111,9 @@ class AMDGPULegalizerInfo final : public LegalizerInfo {
   bool legalizeCTLZ_ZERO_UNDEF(MachineInstr &MI, MachineRegisterInfo &MRI,
                                MachineIRBuilder &B) const;
 
-  bool loadInputValue(Register DstReg, MachineIRBuilder &B,
-                      const ArgDescriptor *Arg,
-                      const TargetRegisterClass *ArgRC, LLT ArgTy) const;
+  void buildLoadInputValue(Register DstReg, MachineIRBuilder &B,
+                           const ArgDescriptor *Arg,
+                           const TargetRegisterClass *ArgRC, LLT ArgTy) const;
   bool loadInputValue(Register DstReg, MachineIRBuilder &B,
                       AMDGPUFunctionArgInfo::PreloadedValue ArgType) const;
 
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUMacroFusion.h b/llvm/lib/Target/AMDGPU/AMDGPUMacroFusion.h
index ad198a301dbe4..8aee23d824e97 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUMacroFusion.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPUMacroFusion.h
@@ -16,7 +16,7 @@ namespace llvm {
 
 /// Note that you have to add:
 ///   DAG.addMutation(createAMDGPUMacroFusionDAGMutation());
-/// to AMDGPUPassConfig::createMachineScheduler() to have an effect.
+/// to AMDGPUTargetMachine::createMachineScheduler() to have an effect.
 std::unique_ptr<ScheduleDAGMutation> createAMDGPUMacroFusionDAGMutation();
 
 } // llvm
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUMarkLastScratchLoad.cpp b/llvm/lib/Target/AMDGPU/AMDGPUMarkLastScratchLoad.cpp
index 8eef0c5892109..ba35a1d417173 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUMarkLastScratchLoad.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUMarkLastScratchLoad.cpp
@@ -88,7 +88,7 @@ bool AMDGPUMarkLastScratchLoad::runOnMachineFunction(MachineFunction &MF) {
       if (Segment.end.isBlock())
         continue;
 
-      const int FrameIndex = Register::stackSlot2Index(LI.reg());
+      const int FrameIndex = LI.reg().stackSlotIndex();
       MachineInstr *LastLoad = nullptr;
 
       MachineInstr *MISegmentEnd = SI->getInstructionFromIndex(Segment.end);
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def b/llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def
index 142b41f030635..d9d97928062f5 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def
+++ b/llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def
@@ -20,7 +20,6 @@ MODULE_PASS("amdgpu-always-inline", AMDGPUAlwaysInlinePass())
 MODULE_PASS("amdgpu-lower-buffer-fat-pointers",
             AMDGPULowerBufferFatPointersPass(*this))
 MODULE_PASS("amdgpu-lower-ctor-dtor", AMDGPUCtorDtorLoweringPass())
-MODULE_PASS("amdgpu-sw-lower-lds", AMDGPUSwLowerLDSPass(*this))
 MODULE_PASS("amdgpu-lower-enqueued-block", AMDGPUOpenCLEnqueuedBlockLoweringPass())
 MODULE_PASS("amdgpu-lower-module-lds", AMDGPULowerModuleLDSPass(*this))
 MODULE_PASS("amdgpu-perf-hint",
@@ -28,6 +27,7 @@ MODULE_PASS("amdgpu-perf-hint",
               *static_cast<const GCNTargetMachine *>(this)))
 MODULE_PASS("amdgpu-printf-runtime-binding", AMDGPUPrintfRuntimeBindingPass())
 MODULE_PASS("amdgpu-remove-incompatible-functions", AMDGPURemoveIncompatibleFunctionsPass(*this))
+MODULE_PASS("amdgpu-sw-lower-lds", AMDGPUSwLowerLDSPass(*this))
 MODULE_PASS("amdgpu-unify-metadata", AMDGPUUnifyMetadataPass())
 #undef MODULE_PASS
 
@@ -56,13 +56,13 @@ FUNCTION_PASS("amdgpu-lower-kernel-arguments",
               AMDGPULowerKernelArgumentsPass(*this))
 FUNCTION_PASS("amdgpu-lower-kernel-attributes",
               AMDGPULowerKernelAttributesPass())
-FUNCTION_PASS("amdgpu-simplifylib", AMDGPUSimplifyLibCallsPass())
 FUNCTION_PASS("amdgpu-promote-alloca", AMDGPUPromoteAllocaPass(*this))
 FUNCTION_PASS("amdgpu-promote-alloca-to-vector",
               AMDGPUPromoteAllocaToVectorPass(*this))
 FUNCTION_PASS("amdgpu-promote-kernel-arguments",
               AMDGPUPromoteKernelArgumentsPass())
 FUNCTION_PASS("amdgpu-rewrite-undef-for-phi", AMDGPURewriteUndefForPHIPass())
+FUNCTION_PASS("amdgpu-simplifylib", AMDGPUSimplifyLibCallsPass())
 FUNCTION_PASS("amdgpu-unify-divergent-exit-nodes",
               AMDGPUUnifyDivergentExitNodesPass())
 FUNCTION_PASS("amdgpu-usenative", AMDGPUUseNativeCallsPass())
@@ -97,11 +97,13 @@ FUNCTION_PASS_WITH_PARAMS(
 #define MACHINE_FUNCTION_PASS(NAME, CREATE_PASS)
 #endif
 MACHINE_FUNCTION_PASS("amdgpu-isel", AMDGPUISelDAGToDAGPass(*this))
+MACHINE_FUNCTION_PASS("amdgpu-pre-ra-long-branch-reg", GCNPreRALongBranchRegPass())
+MACHINE_FUNCTION_PASS("amdgpu-pre-ra-optimizations", GCNPreRAOptimizationsPass())
+MACHINE_FUNCTION_PASS("gcn-dpp-combine", GCNDPPCombinePass())
 MACHINE_FUNCTION_PASS("si-fix-sgpr-copies", SIFixSGPRCopiesPass())
-MACHINE_FUNCTION_PASS("si-i1-copies", SILowerI1CopiesPass())
 MACHINE_FUNCTION_PASS("si-fix-vgpr-copies", SIFixVGPRCopiesPass())
 MACHINE_FUNCTION_PASS("si-fold-operands", SIFoldOperandsPass());
-MACHINE_FUNCTION_PASS("gcn-dpp-combine", GCNDPPCombinePass())
+MACHINE_FUNCTION_PASS("si-i1-copies", SILowerI1CopiesPass())
 MACHINE_FUNCTION_PASS("si-load-store-opt", SILoadStoreOptimizerPass())
 MACHINE_FUNCTION_PASS("si-lower-control-flow", SILowerControlFlowPass())
 MACHINE_FUNCTION_PASS("si-lower-sgpr-spills", SILowerSGPRSpillsPass())
@@ -111,4 +113,39 @@ MACHINE_FUNCTION_PASS("si-optimize-exec-masking", SIOptimizeExecMaskingPass())
 MACHINE_FUNCTION_PASS("si-peephole-sdwa", SIPeepholeSDWAPass())
 MACHINE_FUNCTION_PASS("si-pre-allocate-wwm-regs", SIPreAllocateWWMRegsPass())
 MACHINE_FUNCTION_PASS("si-shrink-instructions", SIShrinkInstructionsPass())
+MACHINE_FUNCTION_PASS("si-wqm", SIWholeQuadModePass())
 #undef MACHINE_FUNCTION_PASS
+
+#define DUMMY_MACHINE_FUNCTION_PASS(NAME, CREATE_PASS)
+DUMMY_MACHINE_FUNCTION_PASS("amdgpu-insert-delay-alu", AMDGPUInsertDelayAluPass())
+DUMMY_MACHINE_FUNCTION_PASS("amdgpu-nsa-reassign", GCNNSAReassignPass())
+DUMMY_MACHINE_FUNCTION_PASS("amdgpu-rewrite-partial-reg-uses", GCNRewritePartialRegUsesPass())
+DUMMY_MACHINE_FUNCTION_PASS("amdgpu-set-wave-priority", AMDGPUSetWavePriorityPass())
+
+DUMMY_MACHINE_FUNCTION_PASS("si-form-memory-clauses", SIFormMemoryClausesPass())
+DUMMY_MACHINE_FUNCTION_PASS("si-insert-hard-clauses", SIInsertHardClausesPass())
+DUMMY_MACHINE_FUNCTION_PASS("si-insert-waitcnts", SIInsertWaitcntsPass())
+DUMMY_MACHINE_FUNCTION_PASS("si-late-branch-lowering", SILateBranchLoweringPass())
+DUMMY_MACHINE_FUNCTION_PASS("si-memory-legalizer", SIMemoryLegalizerPass())
+DUMMY_MACHINE_FUNCTION_PASS("si-mode-register", SIModeRegisterPass())
+DUMMY_MACHINE_FUNCTION_PASS("si-optimize-exec-masking-pre-ra", SIOptimizeExecMaskingPreRAPass())
+DUMMY_MACHINE_FUNCTION_PASS("si-pre-emit-peephole", SIPreEmitPeepholePass())
+// TODO: Move amdgpu-preload-kern-arg-prolog to MACHINE_FUNCTION_PASS since it
+// already exists.
+DUMMY_MACHINE_FUNCTION_PASS("amdgpu-preload-kern-arg-prolog", AMDGPUPreloadKernArgPrologPass())
+
+// Global ISel passes
+DUMMY_MACHINE_FUNCTION_PASS("amdgpu-prelegalizer-combiner", AMDGPUPreLegalizerCombinerPass())
+DUMMY_MACHINE_FUNCTION_PASS("amdgpu-postlegalizer-combiner", AMDGPUPostLegalizerCombinerPass())
+DUMMY_MACHINE_FUNCTION_PASS("amdgpu-global-isel-divergence-lowering", AMDGPUGlobalISelDivergenceLoweringPass())
+DUMMY_MACHINE_FUNCTION_PASS("amdgpu-regbankselect", AMDGPURegBankSelectPass())
+DUMMY_MACHINE_FUNCTION_PASS("amdgpu-regbanklegalize", AMDGPURegBankLegalizePass())
+DUMMY_MACHINE_FUNCTION_PASS("amdgpu-regbank-combiner", AMDGPURegBankCombinerPass())
+
+#undef DUMMY_MACHINE_FUNCTION_PASS
+
+
+#define DUMMY_CGSCC_PASS(NAME, CREATE_PASS)
+DUMMY_CGSCC_PASS("amdgpu-annotate-kernel-features", AMDGPUAnnotateKernelFeaturesPass())
+
+#undef DUMMY_CGSCC_PASS
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUPromoteAlloca.cpp b/llvm/lib/Target/AMDGPU/AMDGPUPromoteAlloca.cpp
index 907f82ed7fc52..69ddb384e1a40 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUPromoteAlloca.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUPromoteAlloca.cpp
@@ -839,10 +839,9 @@ bool AMDGPUPromoteAllocaImpl::tryPromoteAllocaToVector(AllocaInst &Alloca) {
         return RejectUser(Inst, "mem transfer inst length is non-constant or "
                                 "not a multiple of the vector element size");
 
-      if (!TransferInfo.count(TransferInst)) {
+      if (TransferInfo.try_emplace(TransferInst).second) {
         DeferredInsts.push_back(Inst);
         WorkList.push_back(Inst);
-        TransferInfo[TransferInst] = MemTransferInfo();
       }
 
       auto getPointerIndexOfAlloca = [&](Value *Ptr) -> ConstantInt * {
@@ -1556,12 +1555,24 @@ bool AMDGPUPromoteAllocaImpl::tryPromoteAllocaToLDS(AllocaInst &I,
     case Intrinsic::invariant_start:
     case Intrinsic::invariant_end:
     case Intrinsic::launder_invariant_group:
-    case Intrinsic::strip_invariant_group:
+    case Intrinsic::strip_invariant_group: {
+      SmallVector<Value *> Args;
+      if (Intr->getIntrinsicID() == Intrinsic::invariant_start) {
+        Args.emplace_back(Intr->getArgOperand(0));
+      } else if (Intr->getIntrinsicID() == Intrinsic::invariant_end) {
+        Args.emplace_back(Intr->getArgOperand(0));
+        Args.emplace_back(Intr->getArgOperand(1));
+      }
+      Args.emplace_back(Offset);
+      Function *F = Intrinsic::getOrInsertDeclaration(
+          Intr->getModule(), Intr->getIntrinsicID(), Offset->getType());
+      CallInst *NewIntr =
+          CallInst::Create(F, Args, Intr->getName(), Intr->getIterator());
+      Intr->mutateType(NewIntr->getType());
+      Intr->replaceAllUsesWith(NewIntr);
       Intr->eraseFromParent();
-      // FIXME: I think the invariant marker should still theoretically apply,
-      // but the intrinsics need to be changed to accept pointers with any
-      // address space.
       continue;
+    }
     case Intrinsic::objectsize: {
       Value *Src = Intr->getOperand(0);
 
diff --git a/llvm/lib/Target/AMDGPU/AMDGPURegisterBankInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPURegisterBankInfo.cpp
index 224c368cff4a1..2e5f42c3bdc40 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPURegisterBankInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPURegisterBankInfo.cpp
@@ -4675,6 +4675,7 @@ AMDGPURegisterBankInfo::getInstrMapping(const MachineInstr &MI) const {
     case Intrinsic::amdgcn_set_inactive:
     case Intrinsic::amdgcn_set_inactive_chain_arg:
     case Intrinsic::amdgcn_permlane64:
+    case Intrinsic::amdgcn_ds_bpermute_fi_b32:
       return getDefaultMappingAllVGPR(MI);
     case Intrinsic::amdgcn_cvt_pkrtz:
       if (Subtarget.hasSALUFloatInsts() && isSALUMapping(MI))
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
index 1f29589146c80..8b387a191efba 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
@@ -29,8 +29,11 @@
 #include "AMDGPUTargetObjectFile.h"
 #include "AMDGPUTargetTransformInfo.h"
 #include "AMDGPUUnifyDivergentExitNodes.h"
+#include "AMDGPUWaitSGPRHazards.h"
 #include "GCNDPPCombine.h"
 #include "GCNIterativeScheduler.h"
+#include "GCNPreRALongBranchReg.h"
+#include "GCNPreRAOptimizations.h"
 #include "GCNSchedStrategy.h"
 #include "GCNVOPDUtils.h"
 #include "R600.h"
@@ -49,10 +52,12 @@
 #include "SIPeepholeSDWA.h"
 #include "SIPreAllocateWWMRegs.h"
 #include "SIShrinkInstructions.h"
+#include "SIWholeQuadMode.h"
 #include "TargetInfo/AMDGPUTargetInfo.h"
 #include "Utils/AMDGPUBaseInfo.h"
 #include "llvm/Analysis/CGSCCPassManager.h"
 #include "llvm/Analysis/CallGraphSCCPass.h"
+#include "llvm/Analysis/KernelInfo.h"
 #include "llvm/Analysis/UniformityAnalysis.h"
 #include "llvm/CodeGen/AtomicExpand.h"
 #include "llvm/CodeGen/DeadMachineInstructionElim.h"
@@ -526,7 +531,7 @@ extern "C" LLVM_EXTERNAL_VISIBILITY void LLVMInitializeAMDGPUTarget() {
   initializeSIInsertHardClausesPass(*PR);
   initializeSIInsertWaitcntsPass(*PR);
   initializeSIModeRegisterPass(*PR);
-  initializeSIWholeQuadModePass(*PR);
+  initializeSIWholeQuadModeLegacyPass(*PR);
   initializeSILowerControlFlowLegacyPass(*PR);
   initializeSIPreEmitPeepholePass(*PR);
   initializeSILateBranchLoweringPass(*PR);
@@ -543,11 +548,12 @@ extern "C" LLVM_EXTERNAL_VISIBILITY void LLVMInitializeAMDGPUTarget() {
   initializeAMDGPUPrintfRuntimeBindingPass(*PR);
   initializeAMDGPUResourceUsageAnalysisPass(*PR);
   initializeGCNNSAReassignPass(*PR);
-  initializeGCNPreRAOptimizationsPass(*PR);
-  initializeGCNPreRALongBranchRegPass(*PR);
+  initializeGCNPreRAOptimizationsLegacyPass(*PR);
+  initializeGCNPreRALongBranchRegLegacyPass(*PR);
   initializeGCNRewritePartialRegUsesPass(*PR);
   initializeGCNRegPressurePrinterPass(*PR);
   initializeAMDGPUPreloadKernArgPrologLegacyPass(*PR);
+  initializeAMDGPUWaitSGPRHazardsLegacyPass(*PR);
 }
 
 static std::unique_ptr<TargetLoweringObjectFile> createTLOF(const Triple &TT) {
@@ -726,6 +732,16 @@ StringRef AMDGPUTargetMachine::getFeatureString(const Function &F) const {
                           : getTargetFeatureString();
 }
 
+llvm::ScheduleDAGInstrs *
+AMDGPUTargetMachine::createMachineScheduler(MachineSchedContext *C) const {
+  const GCNSubtarget &ST = C->MF->getSubtarget<GCNSubtarget>();
+  ScheduleDAGMILive *DAG = createGenericSchedLive(C);
+  DAG->addMutation(createLoadClusterDAGMutation(DAG->TII, DAG->TRI));
+  if (ST.shouldClusterStores())
+    DAG->addMutation(createStoreClusterDAGMutation(DAG->TII, DAG->TRI));
+  return DAG;
+}
+
 /// Predicate for Internalize pass.
 static bool mustPreserveGV(const GlobalValue &GV) {
   if (const Function *F = dyn_cast<Function>(&GV))
@@ -788,7 +804,8 @@ void AMDGPUTargetMachine::registerPassBuilderCallbacks(PassBuilder &PB) {
   PB.registerPipelineEarlySimplificationEPCallback(
       [](ModulePassManager &PM, OptimizationLevel Level,
          ThinOrFullLTOPhase Phase) {
-        PM.addPass(AMDGPUPrintfRuntimeBindingPass());
+        if (!isLTOPreLink(Phase))
+          PM.addPass(AMDGPUPrintfRuntimeBindingPass());
 
         if (Level == OptimizationLevel::O0)
           return;
@@ -879,6 +896,11 @@ void AMDGPUTargetMachine::registerPassBuilderCallbacks(PassBuilder &PB) {
             PM.addPass(AMDGPUAttributorPass(*this, Opt));
           }
         }
+        if (!NoKernelInfoEndLTO) {
+          FunctionPassManager FPM;
+          FPM.addPass(KernelInfoPrinter(this));
+          PM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM)));
+        }
       });
 
   PB.registerRegClassFilterParsingCallback(
@@ -1038,6 +1060,43 @@ Error GCNTargetMachine::buildCodeGenPipeline(
   return CGPB.buildPipeline(MPM, Out, DwoOut, FileType);
 }
 
+ScheduleDAGInstrs *
+GCNTargetMachine::createMachineScheduler(MachineSchedContext *C) const {
+  const GCNSubtarget &ST = C->MF->getSubtarget<GCNSubtarget>();
+  if (ST.enableSIScheduler())
+    return createSIMachineScheduler(C);
+
+  Attribute SchedStrategyAttr =
+      C->MF->getFunction().getFnAttribute("amdgpu-sched-strategy");
+  StringRef SchedStrategy = SchedStrategyAttr.isValid()
+                                ? SchedStrategyAttr.getValueAsString()
+                                : AMDGPUSchedStrategy;
+
+  if (SchedStrategy == "max-ilp")
+    return createGCNMaxILPMachineScheduler(C);
+
+  if (SchedStrategy == "max-memory-clause")
+    return createGCNMaxMemoryClauseMachineScheduler(C);
+
+  return createGCNMaxOccupancyMachineScheduler(C);
+}
+
+ScheduleDAGInstrs *
+GCNTargetMachine::createPostMachineScheduler(MachineSchedContext *C) const {
+  ScheduleDAGMI *DAG =
+      new GCNPostScheduleDAGMILive(C, std::make_unique<PostGenericScheduler>(C),
+                                   /*RemoveKillFlags=*/true);
+  const GCNSubtarget &ST = C->MF->getSubtarget<GCNSubtarget>();
+  DAG->addMutation(createLoadClusterDAGMutation(DAG->TII, DAG->TRI));
+  if (ST.shouldClusterStores())
+    DAG->addMutation(createStoreClusterDAGMutation(DAG->TII, DAG->TRI));
+  DAG->addMutation(createIGroupLPDAGMutation(AMDGPU::SchedulingPhase::PostRA));
+  if ((EnableVOPD.getNumOccurrences() ||
+       getOptLevel() >= CodeGenOptLevel::Less) &&
+      EnableVOPD)
+    DAG->addMutation(createVOPDPairingMutation());
+  return DAG;
+}
 //===----------------------------------------------------------------------===//
 // AMDGPU Legacy Pass Setup
 //===----------------------------------------------------------------------===//
@@ -1063,25 +1122,6 @@ class GCNPassConfig final : public AMDGPUPassConfig {
     return getTM<GCNTargetMachine>();
   }
 
-  ScheduleDAGInstrs *
-  createMachineScheduler(MachineSchedContext *C) const override;
-
-  ScheduleDAGInstrs *
-  createPostMachineScheduler(MachineSchedContext *C) const override {
-    ScheduleDAGMI *DAG = new GCNPostScheduleDAGMILive(
-        C, std::make_unique<PostGenericScheduler>(C),
-        /*RemoveKillFlags=*/true);
-    const GCNSubtarget &ST = C->MF->getSubtarget<GCNSubtarget>();
-    DAG->addMutation(createLoadClusterDAGMutation(DAG->TII, DAG->TRI));
-    if (ST.shouldClusterStores())
-      DAG->addMutation(createStoreClusterDAGMutation(DAG->TII, DAG->TRI));
-    DAG->addMutation(
-        createIGroupLPDAGMutation(AMDGPU::SchedulingPhase::PostRA));
-    if (isPassEnabled(EnableVOPD, CodeGenOptLevel::Less))
-      DAG->addMutation(createVOPDPairingMutation());
-    return DAG;
-  }
-
   bool addPreISel() override;
   void addMachineSSAOptimization() override;
   bool addILPOpts() override;
@@ -1308,41 +1348,10 @@ bool AMDGPUPassConfig::addGCPasses() {
   return false;
 }
 
-llvm::ScheduleDAGInstrs *
-AMDGPUPassConfig::createMachineScheduler(MachineSchedContext *C) const {
-  const GCNSubtarget &ST = C->MF->getSubtarget<GCNSubtarget>();
-  ScheduleDAGMILive *DAG = createGenericSchedLive(C);
-  DAG->addMutation(createLoadClusterDAGMutation(DAG->TII, DAG->TRI));
-  if (ST.shouldClusterStores())
-    DAG->addMutation(createStoreClusterDAGMutation(DAG->TII, DAG->TRI));
-  return DAG;
-}
-
 //===----------------------------------------------------------------------===//
 // GCN Legacy Pass Setup
 //===----------------------------------------------------------------------===//
 
-ScheduleDAGInstrs *GCNPassConfig::createMachineScheduler(
-  MachineSchedContext *C) const {
-  const GCNSubtarget &ST = C->MF->getSubtarget<GCNSubtarget>();
-  if (ST.enableSIScheduler())
-    return createSIMachineScheduler(C);
-
-  Attribute SchedStrategyAttr =
-      C->MF->getFunction().getFnAttribute("amdgpu-sched-strategy");
-  StringRef SchedStrategy = SchedStrategyAttr.isValid()
-                                ? SchedStrategyAttr.getValueAsString()
-                                : AMDGPUSchedStrategy;
-
-  if (SchedStrategy == "max-ilp")
-    return createGCNMaxILPMachineScheduler(C);
-
-  if (SchedStrategy == "max-memory-clause")
-    return createGCNMaxMemoryClauseMachineScheduler(C);
-
-  return createGCNMaxOccupancyMachineScheduler(C);
-}
-
 bool GCNPassConfig::addPreISel() {
   AMDGPUPassConfig::addPreISel();
 
@@ -1672,6 +1681,8 @@ void GCNPassConfig::addPreEmitPass() {
   // cases.
   addPass(&PostRAHazardRecognizerID);
 
+  addPass(&AMDGPUWaitSGPRHazardsLegacyID);
+
   if (isPassEnabled(EnableInsertDelayAlu, CodeGenOptLevel::Less))
     addPass(&AMDGPUInsertDelayAluID);
 
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.h b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.h
index 24b4da3a68f67..1455494d0ef7d 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.h
@@ -71,6 +71,8 @@ class AMDGPUTargetMachine : public CodeGenTargetMachineImpl {
   bool splitModule(Module &M, unsigned NumParts,
                    function_ref<void(std::unique_ptr<Module> MPart)>
                        ModuleCallback) override;
+  ScheduleDAGInstrs *
+  createMachineScheduler(MachineSchedContext *C) const override;
 };
 
 //===----------------------------------------------------------------------===//
@@ -115,6 +117,10 @@ class GCNTargetMachine final : public AMDGPUTargetMachine {
                                 PerFunctionMIParsingState &PFS,
                                 SMDiagnostic &Error,
                                 SMRange &SourceRange) const override;
+  ScheduleDAGInstrs *
+  createMachineScheduler(MachineSchedContext *C) const override;
+  ScheduleDAGInstrs *
+  createPostMachineScheduler(MachineSchedContext *C) const override;
 };
 
 //===----------------------------------------------------------------------===//
@@ -128,10 +134,6 @@ class AMDGPUPassConfig : public TargetPassConfig {
   AMDGPUTargetMachine &getAMDGPUTargetMachine() const {
     return getTM<AMDGPUTargetMachine>();
   }
-
-  ScheduleDAGInstrs *
-  createMachineScheduler(MachineSchedContext *C) const override;
-
   void addEarlyCSEOrGVNPass();
   void addStraightLineScalarOptimizationPasses();
   void addIRPasses() override;
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp
index 5160851f8c442..09f7877b13b3a 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp
@@ -416,8 +416,6 @@ int64_t GCNTTIImpl::getMaxMemIntrinsicInlineSizeThreshold() const {
   return 1024;
 }
 
-// FIXME: Should we use narrower types for local/region, or account for when
-// unaligned access is legal?
 Type *GCNTTIImpl::getMemcpyLoopLoweringType(
     LLVMContext &Context, Value *Length, unsigned SrcAddrSpace,
     unsigned DestAddrSpace, Align SrcAlign, Align DestAlign,
@@ -426,29 +424,12 @@ Type *GCNTTIImpl::getMemcpyLoopLoweringType(
   if (AtomicElementSize)
     return Type::getIntNTy(Context, *AtomicElementSize * 8);
 
-  Align MinAlign = std::min(SrcAlign, DestAlign);
-
-  // A (multi-)dword access at an address == 2 (mod 4) will be decomposed by the
-  // hardware into byte accesses. If you assume all alignments are equally
-  // probable, it's more efficient on average to use short accesses for this
-  // case.
-  if (MinAlign == Align(2))
-    return Type::getInt16Ty(Context);
-
-  // Not all subtargets have 128-bit DS instructions, and we currently don't
-  // form them by default.
-  if (SrcAddrSpace == AMDGPUAS::LOCAL_ADDRESS ||
-      SrcAddrSpace == AMDGPUAS::REGION_ADDRESS ||
-      DestAddrSpace == AMDGPUAS::LOCAL_ADDRESS ||
-      DestAddrSpace == AMDGPUAS::REGION_ADDRESS) {
-    return FixedVectorType::get(Type::getInt32Ty(Context), 2);
-  }
-
-  // Global memory works best with 16-byte accesses.
+  // 16-byte accesses achieve the highest copy throughput.
   // If the operation has a fixed known length that is large enough, it is
   // worthwhile to return an even wider type and let legalization lower it into
-  // multiple accesses, effectively unrolling the memcpy loop. Private memory
-  // also hits this, although accesses may be decomposed.
+  // multiple accesses, effectively unrolling the memcpy loop.
+  // We also rely on legalization to decompose into smaller accesses for
+  // subtargets and address spaces where it is necessary.
   //
   // Don't unroll if Length is not a constant, since unrolling leads to worse
   // performance for length values that are smaller or slightly larger than the
@@ -473,26 +454,22 @@ void GCNTTIImpl::getMemcpyLoopResidualLoweringType(
         OpsOut, Context, RemainingBytes, SrcAddrSpace, DestAddrSpace, SrcAlign,
         DestAlign, AtomicCpySize);
 
-  Align MinAlign = std::min(SrcAlign, DestAlign);
-
-  if (MinAlign != Align(2)) {
-    Type *I32x4Ty = FixedVectorType::get(Type::getInt32Ty(Context), 4);
-    while (RemainingBytes >= 16) {
-      OpsOut.push_back(I32x4Ty);
-      RemainingBytes -= 16;
-    }
+  Type *I32x4Ty = FixedVectorType::get(Type::getInt32Ty(Context), 4);
+  while (RemainingBytes >= 16) {
+    OpsOut.push_back(I32x4Ty);
+    RemainingBytes -= 16;
+  }
 
-    Type *I64Ty = Type::getInt64Ty(Context);
-    while (RemainingBytes >= 8) {
-      OpsOut.push_back(I64Ty);
-      RemainingBytes -= 8;
-    }
+  Type *I64Ty = Type::getInt64Ty(Context);
+  while (RemainingBytes >= 8) {
+    OpsOut.push_back(I64Ty);
+    RemainingBytes -= 8;
+  }
 
-    Type *I32Ty = Type::getInt32Ty(Context);
-    while (RemainingBytes >= 4) {
-      OpsOut.push_back(I32Ty);
-      RemainingBytes -= 4;
-    }
+  Type *I32Ty = Type::getInt32Ty(Context);
+  while (RemainingBytes >= 4) {
+    OpsOut.push_back(I32Ty);
+    RemainingBytes -= 4;
   }
 
   Type *I16Ty = Type::getInt16Ty(Context);
@@ -1430,3 +1407,19 @@ unsigned GCNTTIImpl::getPrefetchDistance() const {
 bool GCNTTIImpl::shouldPrefetchAddressSpace(unsigned AS) const {
   return AMDGPU::isFlatGlobalAddrSpace(AS);
 }
+
+void GCNTTIImpl::collectKernelLaunchBounds(
+    const Function &F,
+    SmallVectorImpl<std::pair<StringRef, int64_t>> &LB) const {
+  SmallVector<unsigned> MaxNumWorkgroups = ST->getMaxNumWorkGroups(F);
+  LB.push_back({"amdgpu-max-num-workgroups[0]", MaxNumWorkgroups[0]});
+  LB.push_back({"amdgpu-max-num-workgroups[1]", MaxNumWorkgroups[1]});
+  LB.push_back({"amdgpu-max-num-workgroups[2]", MaxNumWorkgroups[2]});
+  std::pair<unsigned, unsigned> FlatWorkGroupSize =
+      ST->getFlatWorkGroupSizes(F);
+  LB.push_back({"amdgpu-flat-work-group-size[0]", FlatWorkGroupSize.first});
+  LB.push_back({"amdgpu-flat-work-group-size[1]", FlatWorkGroupSize.second});
+  std::pair<unsigned, unsigned> WavesPerEU = ST->getWavesPerEU(F);
+  LB.push_back({"amdgpu-waves-per-eu[0]", WavesPerEU.first});
+  LB.push_back({"amdgpu-waves-per-eu[1]", WavesPerEU.second});
+}
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.h b/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.h
index 585f38fc02c29..a0d62008d9ddc 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.h
@@ -273,6 +273,9 @@ class GCNTTIImpl final : public BasicTTIImplBase<GCNTTIImpl> {
 
   /// \return if target want to issue a prefetch in address space \p AS.
   bool shouldPrefetchAddressSpace(unsigned AS) const override;
+  void collectKernelLaunchBounds(
+      const Function &F,
+      SmallVectorImpl<std::pair<StringRef, int64_t>> &LB) const;
 };
 
 } // end namespace llvm
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUUnifyDivergentExitNodes.cpp b/llvm/lib/Target/AMDGPU/AMDGPUUnifyDivergentExitNodes.cpp
index fda2a38c2464e..d087fbc86545c 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUUnifyDivergentExitNodes.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUUnifyDivergentExitNodes.cpp
@@ -215,7 +215,10 @@ bool AMDGPUUnifyDivergentExitNodesImpl::run(Function &F, DominatorTree *DT,
       PDT.roots(), [&](auto BB) { return !isUniformlyReached(UA, *BB); });
 
   for (BasicBlock *BB : PDT.roots()) {
-    if (isa<ReturnInst>(BB->getTerminator())) {
+    if (auto *RI = dyn_cast<ReturnInst>(BB->getTerminator())) {
+      auto *CI = dyn_cast_or_null<CallInst>(RI->getPrevNode());
+      if (CI && CI->isMustTailCall())
+        continue;
       if (HasDivergentExitBlock)
         ReturningBlocks.push_back(BB);
     } else if (isa<UnreachableInst>(BB->getTerminator())) {
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUWaitSGPRHazards.cpp b/llvm/lib/Target/AMDGPU/AMDGPUWaitSGPRHazards.cpp
new file mode 100644
index 0000000000000..e70d6aab306fe
--- /dev/null
+++ b/llvm/lib/Target/AMDGPU/AMDGPUWaitSGPRHazards.cpp
@@ -0,0 +1,517 @@
+//===- AMDGPUWaitSGPRHazards.cpp - Insert waits for SGPR read hazards -----===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+/// \file
+/// Insert s_wait_alu instructions to mitigate SGPR read hazards on GFX12.
+//
+//===----------------------------------------------------------------------===//
+
+#include "AMDGPUWaitSGPRHazards.h"
+#include "AMDGPU.h"
+#include "GCNSubtarget.h"
+#include "MCTargetDesc/AMDGPUMCTargetDesc.h"
+#include "SIInstrInfo.h"
+#include "llvm/ADT/SetVector.h"
+
+using namespace llvm;
+
+#define DEBUG_TYPE "amdgpu-wait-sgpr-hazards"
+
+static cl::opt<bool> GlobalEnableSGPRHazardWaits(
+    "amdgpu-sgpr-hazard-wait", cl::init(true), cl::Hidden,
+    cl::desc("Enable required s_wait_alu on SGPR hazards"));
+
+static cl::opt<bool> GlobalCullSGPRHazardsOnFunctionBoundary(
+    "amdgpu-sgpr-hazard-boundary-cull", cl::init(false), cl::Hidden,
+    cl::desc("Cull hazards on function boundaries"));
+
+static cl::opt<bool>
+    GlobalCullSGPRHazardsAtMemWait("amdgpu-sgpr-hazard-mem-wait-cull",
+                                   cl::init(false), cl::Hidden,
+                                   cl::desc("Cull hazards on memory waits"));
+
+static cl::opt<unsigned> GlobalCullSGPRHazardsMemWaitThreshold(
+    "amdgpu-sgpr-hazard-mem-wait-cull-threshold", cl::init(8), cl::Hidden,
+    cl::desc("Number of tracked SGPRs before initiating hazard cull on memory "
+             "wait"));
+
+namespace {
+
+class AMDGPUWaitSGPRHazards {
+public:
+  const SIInstrInfo *TII;
+  const SIRegisterInfo *TRI;
+  const MachineRegisterInfo *MRI;
+  unsigned DsNopCount;
+
+  bool EnableSGPRHazardWaits;
+  bool CullSGPRHazardsOnFunctionBoundary;
+  bool CullSGPRHazardsAtMemWait;
+  unsigned CullSGPRHazardsMemWaitThreshold;
+
+  AMDGPUWaitSGPRHazards() {}
+
+  // Return the numeric ID 0-127 for a given SGPR.
+  static std::optional<unsigned> sgprNumber(Register Reg,
+                                            const SIRegisterInfo &TRI) {
+    switch (Reg) {
+    case AMDGPU::M0:
+    case AMDGPU::EXEC:
+    case AMDGPU::EXEC_LO:
+    case AMDGPU::EXEC_HI:
+    case AMDGPU::SGPR_NULL:
+    case AMDGPU::SGPR_NULL64:
+      return {};
+    default:
+      break;
+    }
+    unsigned RegN = TRI.getHWRegIndex(Reg);
+    if (RegN > 127)
+      return {};
+    return RegN;
+  }
+
+  static inline bool isVCC(Register Reg) {
+    return Reg == AMDGPU::VCC || Reg == AMDGPU::VCC_LO || Reg == AMDGPU::VCC_HI;
+  }
+
+  // Adjust global offsets for instructions bundled with S_GETPC_B64 after
+  // insertion of a new instruction.
+  static void updateGetPCBundle(MachineInstr *NewMI) {
+    if (!NewMI->isBundled())
+      return;
+
+    // Find start of bundle.
+    auto I = NewMI->getIterator();
+    while (I->isBundledWithPred())
+      I--;
+    if (I->isBundle())
+      I++;
+
+    // Bail if this is not an S_GETPC bundle.
+    if (I->getOpcode() != AMDGPU::S_GETPC_B64)
+      return;
+
+    // Update offsets of any references in the bundle.
+    const unsigned NewBytes = 4;
+    assert(NewMI->getOpcode() == AMDGPU::S_WAITCNT_DEPCTR &&
+           "Unexpected instruction insertion in bundle");
+    auto NextMI = std::next(NewMI->getIterator());
+    auto End = NewMI->getParent()->end();
+    while (NextMI != End && NextMI->isBundledWithPred()) {
+      for (auto &Operand : NextMI->operands()) {
+        if (Operand.isGlobal())
+          Operand.setOffset(Operand.getOffset() + NewBytes);
+      }
+      NextMI++;
+    }
+  }
+
+  struct HazardState {
+    static constexpr unsigned None = 0;
+    static constexpr unsigned SALU = (1 << 0);
+    static constexpr unsigned VALU = (1 << 1);
+
+    std::bitset<64> Tracked;      // SGPR banks ever read by VALU
+    std::bitset<128> SALUHazards; // SGPRs with uncommitted values from SALU
+    std::bitset<128> VALUHazards; // SGPRs with uncommitted values from VALU
+    unsigned VCCHazard = None;    // Source of current VCC writes
+    bool ActiveFlat = false;      // Has unwaited flat instructions
+
+    bool merge(const HazardState &RHS) {
+      HazardState Orig(*this);
+      *this |= RHS;
+      return (*this != Orig);
+    }
+
+    bool operator==(const HazardState &RHS) const {
+      return Tracked == RHS.Tracked && SALUHazards == RHS.SALUHazards &&
+             VALUHazards == RHS.VALUHazards && VCCHazard == RHS.VCCHazard &&
+             ActiveFlat == RHS.ActiveFlat;
+    }
+
+    bool operator!=(const HazardState &RHS) const { return !(*this == RHS); }
+
+    void operator|=(const HazardState &RHS) {
+      Tracked |= RHS.Tracked;
+      SALUHazards |= RHS.SALUHazards;
+      VALUHazards |= RHS.VALUHazards;
+      VCCHazard |= RHS.VCCHazard;
+      ActiveFlat |= RHS.ActiveFlat;
+    }
+  };
+
+  struct BlockHazardState {
+    HazardState In;
+    HazardState Out;
+  };
+
+  DenseMap<const MachineBasicBlock *, BlockHazardState> BlockState;
+
+  static constexpr unsigned WAVE32_NOPS = 4;
+  static constexpr unsigned WAVE64_NOPS = 8;
+
+  void insertHazardCull(MachineBasicBlock &MBB,
+                        MachineBasicBlock::instr_iterator &MI) {
+    assert(!MI->isBundled());
+    unsigned Count = DsNopCount;
+    while (Count--)
+      BuildMI(MBB, MI, MI->getDebugLoc(), TII->get(AMDGPU::DS_NOP));
+  }
+
+  bool runOnMachineBasicBlock(MachineBasicBlock &MBB, bool Emit) {
+    enum { WA_VALU = 0x1, WA_SALU = 0x2, WA_VCC = 0x4 };
+
+    HazardState State = BlockState[&MBB].In;
+    SmallSet<Register, 8> SeenRegs;
+    bool Emitted = false;
+    unsigned DsNops = 0;
+
+    for (MachineBasicBlock::instr_iterator MI = MBB.instr_begin(),
+                                           E = MBB.instr_end();
+         MI != E; ++MI) {
+      if (MI->isMetaInstruction())
+        continue;
+
+      // Clear tracked SGPRs if sufficient DS_NOPs occur
+      if (MI->getOpcode() == AMDGPU::DS_NOP) {
+        if (++DsNops >= DsNopCount)
+          State.Tracked.reset();
+        continue;
+      }
+      DsNops = 0;
+
+      // Snoop FLAT instructions to avoid adding culls before scratch/lds loads.
+      // Culls could be disproportionate in cost to load time.
+      if (SIInstrInfo::isFLAT(*MI) && !SIInstrInfo::isFLATGlobal(*MI))
+        State.ActiveFlat = true;
+
+      // SMEM or VMEM clears hazards
+      if (SIInstrInfo::isVMEM(*MI) || SIInstrInfo::isSMRD(*MI)) {
+        State.VCCHazard = HazardState::None;
+        State.SALUHazards.reset();
+        State.VALUHazards.reset();
+        continue;
+      }
+
+      // Existing S_WAITALU can clear hazards
+      if (MI->getOpcode() == AMDGPU::S_WAITCNT_DEPCTR) {
+        unsigned int Mask = MI->getOperand(0).getImm();
+        if (AMDGPU::DepCtr::decodeFieldVaVcc(Mask) == 0)
+          State.VCCHazard &= ~HazardState::VALU;
+        if (AMDGPU::DepCtr::decodeFieldSaSdst(Mask) == 0) {
+          State.SALUHazards.reset();
+          State.VCCHazard &= ~HazardState::SALU;
+        }
+        if (AMDGPU::DepCtr::decodeFieldVaSdst(Mask) == 0)
+          State.VALUHazards.reset();
+        continue;
+      }
+
+      // Snoop counter waits to insert culls
+      if (CullSGPRHazardsAtMemWait &&
+          (MI->getOpcode() == AMDGPU::S_WAIT_LOADCNT ||
+           MI->getOpcode() == AMDGPU::S_WAIT_SAMPLECNT ||
+           MI->getOpcode() == AMDGPU::S_WAIT_BVHCNT) &&
+          (MI->getOperand(0).isImm() && MI->getOperand(0).getImm() == 0) &&
+          (State.Tracked.count() >= CullSGPRHazardsMemWaitThreshold)) {
+        if (MI->getOpcode() == AMDGPU::S_WAIT_LOADCNT && State.ActiveFlat) {
+          State.ActiveFlat = false;
+        } else {
+          State.Tracked.reset();
+          if (Emit)
+            insertHazardCull(MBB, MI);
+          continue;
+        }
+      }
+
+      // Process only VALUs and SALUs
+      bool IsVALU = SIInstrInfo::isVALU(*MI);
+      bool IsSALU = SIInstrInfo::isSALU(*MI);
+      if (!IsVALU && !IsSALU)
+        continue;
+
+      unsigned Wait = 0;
+
+      auto processOperand = [&](const MachineOperand &Op, bool IsUse) {
+        if (!Op.isReg())
+          return;
+        Register Reg = Op.getReg();
+        assert(!Op.getSubReg());
+        if (!TRI->isSGPRReg(*MRI, Reg))
+          return;
+
+        // Only visit each register once
+        if (!SeenRegs.insert(Reg).second)
+          return;
+
+        auto RegNumber = sgprNumber(Reg, *TRI);
+        if (!RegNumber)
+          return;
+
+        // Track SGPRs by pair -- numeric ID of an 64b SGPR pair.
+        // i.e. SGPR0 = SGPR0_SGPR1 = 0, SGPR3 = SGPR2_SGPR3 = 1, etc
+        unsigned RegN = *RegNumber;
+        unsigned PairN = (RegN >> 1) & 0x3f;
+
+        // Read/write of untracked register is safe; but must record any new
+        // reads.
+        if (!State.Tracked[PairN]) {
+          if (IsVALU && IsUse)
+            State.Tracked.set(PairN);
+          return;
+        }
+
+        uint8_t SGPRCount =
+            AMDGPU::getRegBitWidth(*TRI->getRegClassForReg(*MRI, Reg)) / 32;
+
+        if (IsUse) {
+          // SALU reading SGPR clears VALU hazards
+          if (IsSALU) {
+            if (isVCC(Reg)) {
+              if (State.VCCHazard & HazardState::VALU)
+                State.VCCHazard = HazardState::None;
+            } else {
+              State.VALUHazards.reset();
+            }
+          }
+          // Compute required waits
+          for (uint8_t RegIdx = 0; RegIdx < SGPRCount; ++RegIdx) {
+            Wait |= State.SALUHazards[RegN + RegIdx] ? WA_SALU : 0;
+            Wait |= IsVALU && State.VALUHazards[RegN + RegIdx] ? WA_VALU : 0;
+          }
+          if (isVCC(Reg) && State.VCCHazard) {
+            // Note: it's possible for both SALU and VALU to exist if VCC
+            // was updated differently by merged predecessors.
+            if (State.VCCHazard & HazardState::SALU)
+              Wait |= WA_SALU;
+            if (State.VCCHazard & HazardState::VALU)
+              Wait |= WA_VCC;
+          }
+        } else {
+          // Update hazards
+          if (isVCC(Reg)) {
+            State.VCCHazard = IsSALU ? HazardState::SALU : HazardState::VALU;
+          } else {
+            for (uint8_t RegIdx = 0; RegIdx < SGPRCount; ++RegIdx) {
+              if (IsSALU)
+                State.SALUHazards.set(RegN + RegIdx);
+              else
+                State.VALUHazards.set(RegN + RegIdx);
+            }
+          }
+        }
+      };
+
+      const bool IsSetPC =
+          (MI->isCall() || MI->isReturn() || MI->isIndirectBranch()) &&
+          MI->getOpcode() != AMDGPU::S_ENDPGM &&
+          MI->getOpcode() != AMDGPU::S_ENDPGM_SAVED;
+
+      // Only consider implicit VCC specified by instruction descriptor.
+      const bool HasImplicitVCC =
+          llvm::any_of(MI->getDesc().implicit_uses(),
+                       [](MCPhysReg Reg) { return isVCC(Reg); }) ||
+          llvm::any_of(MI->getDesc().implicit_defs(),
+                       [](MCPhysReg Reg) { return isVCC(Reg); });
+
+      if (IsSetPC) {
+        // All SGPR writes before a call/return must be flushed as the
+        // callee/caller will not will not see the hazard chain.
+        if (State.VCCHazard & HazardState::VALU)
+          Wait |= WA_VCC;
+        if (State.SALUHazards.any() || (State.VCCHazard & HazardState::SALU))
+          Wait |= WA_SALU;
+        if (State.VALUHazards.any())
+          Wait |= WA_VALU;
+        if (CullSGPRHazardsOnFunctionBoundary && State.Tracked.any()) {
+          State.Tracked.reset();
+          if (Emit)
+            insertHazardCull(MBB, MI);
+        }
+      } else {
+        // Process uses to determine required wait.
+        SeenRegs.clear();
+        for (const MachineOperand &Op : MI->all_uses()) {
+          if (Op.isImplicit() &&
+              (!HasImplicitVCC || !Op.isReg() || !isVCC(Op.getReg())))
+            continue;
+          processOperand(Op, true);
+        }
+      }
+
+      // Apply wait
+      if (Wait) {
+        unsigned Mask = 0xffff;
+        if (Wait & WA_VCC) {
+          State.VCCHazard &= ~HazardState::VALU;
+          Mask = AMDGPU::DepCtr::encodeFieldVaVcc(Mask, 0);
+        }
+        if (Wait & WA_SALU) {
+          State.SALUHazards.reset();
+          State.VCCHazard &= ~HazardState::SALU;
+          Mask = AMDGPU::DepCtr::encodeFieldSaSdst(Mask, 0);
+        }
+        if (Wait & WA_VALU) {
+          State.VALUHazards.reset();
+          Mask = AMDGPU::DepCtr::encodeFieldVaSdst(Mask, 0);
+        }
+        if (Emit) {
+          auto NewMI = BuildMI(MBB, MI, MI->getDebugLoc(),
+                               TII->get(AMDGPU::S_WAITCNT_DEPCTR))
+                           .addImm(Mask);
+          updateGetPCBundle(NewMI);
+          Emitted = true;
+        }
+      }
+
+      // On return from a call SGPR state is unknown, so all potential hazards.
+      if (MI->isCall() && !CullSGPRHazardsOnFunctionBoundary)
+        State.Tracked.set();
+
+      // Update hazards based on defs.
+      SeenRegs.clear();
+      for (const MachineOperand &Op : MI->all_defs()) {
+        if (Op.isImplicit() &&
+            (!HasImplicitVCC || !Op.isReg() || !isVCC(Op.getReg())))
+          continue;
+        processOperand(Op, false);
+      }
+    }
+
+    bool Changed = State != BlockState[&MBB].Out;
+    if (Emit) {
+      assert(!Changed && "Hazard state should not change on emit pass");
+      return Emitted;
+    }
+    if (Changed)
+      BlockState[&MBB].Out = State;
+    return Changed;
+  }
+
+  bool run(MachineFunction &MF) {
+    const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
+    if (!ST.hasVALUReadSGPRHazard())
+      return false;
+
+    // Parse settings
+    EnableSGPRHazardWaits = GlobalEnableSGPRHazardWaits;
+    CullSGPRHazardsOnFunctionBoundary = GlobalCullSGPRHazardsOnFunctionBoundary;
+    CullSGPRHazardsAtMemWait = GlobalCullSGPRHazardsAtMemWait;
+    CullSGPRHazardsMemWaitThreshold = GlobalCullSGPRHazardsMemWaitThreshold;
+
+    if (!GlobalEnableSGPRHazardWaits.getNumOccurrences())
+      EnableSGPRHazardWaits = MF.getFunction().getFnAttributeAsParsedInteger(
+          "amdgpu-sgpr-hazard-wait", EnableSGPRHazardWaits);
+    if (!GlobalCullSGPRHazardsOnFunctionBoundary.getNumOccurrences())
+      CullSGPRHazardsOnFunctionBoundary =
+          MF.getFunction().hasFnAttribute("amdgpu-sgpr-hazard-boundary-cull");
+    if (!GlobalCullSGPRHazardsAtMemWait.getNumOccurrences())
+      CullSGPRHazardsAtMemWait =
+          MF.getFunction().hasFnAttribute("amdgpu-sgpr-hazard-mem-wait-cull");
+    if (!GlobalCullSGPRHazardsMemWaitThreshold.getNumOccurrences())
+      CullSGPRHazardsMemWaitThreshold =
+          MF.getFunction().getFnAttributeAsParsedInteger(
+              "amdgpu-sgpr-hazard-mem-wait-cull-threshold",
+              CullSGPRHazardsMemWaitThreshold);
+
+    // Bail if disabled
+    if (!EnableSGPRHazardWaits)
+      return false;
+
+    TII = ST.getInstrInfo();
+    TRI = ST.getRegisterInfo();
+    MRI = &MF.getRegInfo();
+    DsNopCount = ST.isWave64() ? WAVE64_NOPS : WAVE32_NOPS;
+
+    auto CallingConv = MF.getFunction().getCallingConv();
+    if (!AMDGPU::isEntryFunctionCC(CallingConv) &&
+        !CullSGPRHazardsOnFunctionBoundary) {
+      // Callee must consider all SGPRs as tracked.
+      LLVM_DEBUG(dbgs() << "Is called function, track all SGPRs.\n");
+      MachineBasicBlock &EntryBlock = MF.front();
+      BlockState[&EntryBlock].In.Tracked.set();
+    }
+
+    // Calculate the hazard state for each basic block.
+    // Iterate until a fixed point is reached.
+    // Fixed point is guaranteed as merge function only ever increases
+    // the hazard set, and all backedges will cause a merge.
+    //
+    // Note: we have to take care of the entry block as this technically
+    // has an edge from outside the function. Failure to treat this as
+    // a merge could prevent fixed point being reached.
+    SetVector<MachineBasicBlock *> Worklist;
+    for (auto &MBB : reverse(MF))
+      Worklist.insert(&MBB);
+    while (!Worklist.empty()) {
+      auto &MBB = *Worklist.pop_back_val();
+      bool Changed = runOnMachineBasicBlock(MBB, false);
+      if (Changed) {
+        // Note: take a copy of state here in case it is reallocated by map
+        HazardState NewState = BlockState[&MBB].Out;
+        // Propagate to all successor blocks
+        for (auto Succ : MBB.successors()) {
+          // We only need to merge hazards at CFG merge points.
+          auto &SuccState = BlockState[Succ];
+          if (Succ->getSinglePredecessor() && !Succ->isEntryBlock()) {
+            if (SuccState.In != NewState) {
+              SuccState.In = NewState;
+              Worklist.insert(Succ);
+            }
+          } else if (SuccState.In.merge(NewState)) {
+            Worklist.insert(Succ);
+          }
+        }
+      }
+    }
+
+    LLVM_DEBUG(dbgs() << "Emit s_wait_alu instructions\n");
+
+    // Final to emit wait instructions.
+    bool Changed = false;
+    for (auto &MBB : MF)
+      Changed |= runOnMachineBasicBlock(MBB, true);
+
+    BlockState.clear();
+    return Changed;
+  }
+};
+
+class AMDGPUWaitSGPRHazardsLegacy : public MachineFunctionPass {
+public:
+  static char ID;
+
+  AMDGPUWaitSGPRHazardsLegacy() : MachineFunctionPass(ID) {}
+
+  bool runOnMachineFunction(MachineFunction &MF) override {
+    return AMDGPUWaitSGPRHazards().run(MF);
+  }
+
+  void getAnalysisUsage(AnalysisUsage &AU) const override {
+    AU.setPreservesCFG();
+    MachineFunctionPass::getAnalysisUsage(AU);
+  }
+};
+
+} // namespace
+
+char AMDGPUWaitSGPRHazardsLegacy::ID = 0;
+
+char &llvm::AMDGPUWaitSGPRHazardsLegacyID = AMDGPUWaitSGPRHazardsLegacy::ID;
+
+INITIALIZE_PASS(AMDGPUWaitSGPRHazardsLegacy, DEBUG_TYPE,
+                "AMDGPU Insert waits for SGPR read hazards", false, false)
+
+PreservedAnalyses
+AMDGPUWaitSGPRHazardsPass::run(MachineFunction &MF,
+                               MachineFunctionAnalysisManager &MFAM) {
+  if (AMDGPUWaitSGPRHazards().run(MF))
+    return PreservedAnalyses::none();
+  return PreservedAnalyses::all();
+}
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUWaitSGPRHazards.h b/llvm/lib/Target/AMDGPU/AMDGPUWaitSGPRHazards.h
new file mode 100644
index 0000000000000..58e9bca4c3ede
--- /dev/null
+++ b/llvm/lib/Target/AMDGPU/AMDGPUWaitSGPRHazards.h
@@ -0,0 +1,25 @@
+//===--- AMDGPUWaitSGPRHazards.h --------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIB_TARGET_AMDGPU_AMDGPUWAITSGPRHAZARDS_H
+#define LLVM_LIB_TARGET_AMDGPU_AMDGPUWAITSGPRHAZARDS_H
+
+#include "llvm/CodeGen/MachinePassManager.h"
+
+namespace llvm {
+
+class AMDGPUWaitSGPRHazardsPass
+    : public PassInfoMixin<AMDGPUWaitSGPRHazardsPass> {
+public:
+  PreservedAnalyses run(MachineFunction &MF,
+                        MachineFunctionAnalysisManager &MFAM);
+};
+
+} // namespace llvm
+
+#endif // LLVM_LIB_TARGET_AMDGPU_AMDGPUWAITSGPRHAZARDS_H
diff --git a/llvm/lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp b/llvm/lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp
index d8f441d1ccfe4..4ff9cff09f31d 100644
--- a/llvm/lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp
+++ b/llvm/lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp
@@ -33,6 +33,7 @@
 #include "llvm/MC/MCParser/MCAsmParser.h"
 #include "llvm/MC/MCParser/MCParsedAsmOperand.h"
 #include "llvm/MC/MCParser/MCTargetAsmParser.h"
+#include "llvm/MC/MCRegisterInfo.h"
 #include "llvm/MC/MCSymbol.h"
 #include "llvm/MC/TargetRegistry.h"
 #include "llvm/Support/AMDGPUMetadata.h"
@@ -1536,6 +1537,10 @@ class AMDGPUAsmParser : public MCTargetAsmParser {
     return getFeatureBits()[AMDGPU::FeatureFlatInstOffsets];
   }
 
+  bool hasTrue16Insts() const {
+    return getFeatureBits()[AMDGPU::FeatureTrue16BitInsts];
+  }
+
   bool hasArchitectedFlatScratch() const {
     return getFeatureBits()[AMDGPU::FeatureArchitectedFlatScratch];
   }
@@ -1777,6 +1782,7 @@ class AMDGPUAsmParser : public MCTargetAsmParser {
   bool validateMIMGDim(const MCInst &Inst, const OperandVector &Operands);
   bool validateMIMGMSAA(const MCInst &Inst);
   bool validateOpSel(const MCInst &Inst);
+  bool validateTrue16OpSel(const MCInst &Inst);
   bool validateNeg(const MCInst &Inst, int OpName);
   bool validateDPP(const MCInst &Inst, const OperandVector &Operands);
   bool validateVccOperand(MCRegister Reg) const;
@@ -4651,6 +4657,39 @@ bool AMDGPUAsmParser::validateOpSel(const MCInst &Inst) {
   return true;
 }
 
+bool AMDGPUAsmParser::validateTrue16OpSel(const MCInst &Inst) {
+  if (!hasTrue16Insts())
+    return true;
+  const MCRegisterInfo *MRI = getMRI();
+  const unsigned Opc = Inst.getOpcode();
+  int OpSelIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::op_sel);
+  if (OpSelIdx == -1)
+    return true;
+  unsigned OpSelOpValue = Inst.getOperand(OpSelIdx).getImm();
+  // If the value is 0 we could have a default OpSel Operand, so conservatively
+  // allow it.
+  if (OpSelOpValue == 0)
+    return true;
+  unsigned OpCount = 0;
+  for (int OpName : {AMDGPU::OpName::src0, AMDGPU::OpName::src1,
+                     AMDGPU::OpName::src2, AMDGPU::OpName::vdst}) {
+    int OpIdx = AMDGPU::getNamedOperandIdx(Inst.getOpcode(), OpName);
+    if (OpIdx == -1)
+      continue;
+    const MCOperand &Op = Inst.getOperand(OpIdx);
+    if (Op.isReg() &&
+        MRI->getRegClass(AMDGPU::VGPR_16RegClassID).contains(Op.getReg())) {
+      bool VGPRSuffixIsHi = AMDGPU::isHi16Reg(Op.getReg(), *MRI);
+      bool OpSelOpIsHi = ((OpSelOpValue & (1 << OpCount)) != 0);
+      if (OpSelOpIsHi != VGPRSuffixIsHi)
+        return false;
+    }
+    ++OpCount;
+  }
+
+  return true;
+}
+
 bool AMDGPUAsmParser::validateNeg(const MCInst &Inst, int OpName) {
   assert(OpName == AMDGPU::OpName::neg_lo || OpName == AMDGPU::OpName::neg_hi);
 
@@ -4853,7 +4892,10 @@ bool AMDGPUAsmParser::validateAGPRLdSt(const MCInst &Inst) const {
 
 bool AMDGPUAsmParser::validateVGPRAlign(const MCInst &Inst) const {
   auto FB = getFeatureBits();
-  if (!FB[AMDGPU::FeatureGFX90AInsts])
+  unsigned Opc = Inst.getOpcode();
+  // DS_READ_B96_TR_B6 is the only DS instruction in GFX950, that allows
+  // unaligned VGPR. All others only allow even aligned VGPRs.
+  if (!(FB[AMDGPU::FeatureGFX90AInsts]) || Opc == AMDGPU::DS_READ_B96_TR_B6_vi)
     return true;
 
   const MCRegisterInfo *MRI = getMRI();
@@ -5132,6 +5174,11 @@ bool AMDGPUAsmParser::validateInstruction(const MCInst &Inst,
     Error(getRegLoc(LDS_DIRECT, Operands), *ErrMsg);
     return false;
   }
+  if (!validateTrue16OpSel(Inst)) {
+    Error(getImmLoc(AMDGPUOperand::ImmTyOpSel, Operands),
+          "op_sel operand conflicts with 16-bit operand suffix");
+    return false;
+  }
   if (!validateSOPLiteral(Inst)) {
     Error(getLitLoc(Operands),
       "only one unique literal operand is allowed");
@@ -5829,6 +5876,18 @@ bool AMDGPUAsmParser::ParseDirectiveAMDHSAKernel() {
       PARSE_BITS_ENTRY(KD.compute_pgm_rsrc3,
                        COMPUTE_PGM_RSRC3_GFX10_GFX11_SHARED_VGPR_COUNT, ExprVal,
                        ValRange);
+    } else if (ID == ".amdhsa_inst_pref_size") {
+      if (IVersion.Major < 11)
+        return Error(IDRange.Start, "directive requires gfx11+", IDRange);
+      if (IVersion.Major == 11) {
+        PARSE_BITS_ENTRY(KD.compute_pgm_rsrc3,
+                         COMPUTE_PGM_RSRC3_GFX11_INST_PREF_SIZE, ExprVal,
+                         ValRange);
+      } else {
+        PARSE_BITS_ENTRY(KD.compute_pgm_rsrc3,
+                         COMPUTE_PGM_RSRC3_GFX12_PLUS_INST_PREF_SIZE, ExprVal,
+                         ValRange);
+      }
     } else if (ID == ".amdhsa_exception_fp_ieee_invalid_op") {
       PARSE_BITS_ENTRY(
           KD.compute_pgm_rsrc2,
diff --git a/llvm/lib/Target/AMDGPU/CMakeLists.txt b/llvm/lib/Target/AMDGPU/CMakeLists.txt
index 91cae76256306..408da0536237e 100644
--- a/llvm/lib/Target/AMDGPU/CMakeLists.txt
+++ b/llvm/lib/Target/AMDGPU/CMakeLists.txt
@@ -110,6 +110,7 @@ add_llvm_target(AMDGPUCodeGen
   AMDGPUTargetMachine.cpp
   AMDGPUTargetObjectFile.cpp
   AMDGPUTargetTransformInfo.cpp
+  AMDGPUWaitSGPRHazards.cpp
   AMDGPUUnifyDivergentExitNodes.cpp
   AMDGPUUnifyMetadata.cpp
   R600MachineCFGStructurizer.cpp
diff --git a/llvm/lib/Target/AMDGPU/DSInstructions.td b/llvm/lib/Target/AMDGPU/DSInstructions.td
index bc217e10e0fbd..9ca853befba73 100644
--- a/llvm/lib/Target/AMDGPU/DSInstructions.td
+++ b/llvm/lib/Target/AMDGPU/DSInstructions.td
@@ -699,7 +699,6 @@ def DS_PERMUTE_B32  : DS_1A1D_PERMUTE <"ds_permute_b32",
                                        int_amdgcn_ds_permute>;
 def DS_BPERMUTE_B32 : DS_1A1D_PERMUTE <"ds_bpermute_b32",
                                        int_amdgcn_ds_bpermute>;
-def DS_BPERMUTE_FI_B32 : DS_1A1D_PERMUTE <"ds_bpermute_fi_b32">;
 }
 
 } // let SubtargetPredicate = isGFX8Plus
@@ -737,6 +736,8 @@ defm DS_COND_SUB_U32      : DS_1A1D_NORET_mc<"ds_cond_sub_u32">;
 defm DS_COND_SUB_RTN_U32  : DS_1A1D_RET_mc<"ds_cond_sub_rtn_u32", VGPR_32>;
 defm DS_SUB_CLAMP_U32     : DS_1A1D_NORET_mc<"ds_sub_clamp_u32">;
 defm DS_SUB_CLAMP_RTN_U32 : DS_1A1D_RET_mc<"ds_sub_clamp_rtn_u32", VGPR_32>;
+def DS_BPERMUTE_FI_B32    : DS_1A1D_PERMUTE <"ds_bpermute_fi_b32",
+                                             int_amdgcn_ds_bpermute_fi_b32>;
 
 multiclass DSAtomicRetNoRetPatIntrinsic_mc<DS_Pseudo inst, DS_Pseudo noRetInst,
                                   ValueType vt, string frag> {
diff --git a/llvm/lib/Target/AMDGPU/Disassembler/AMDGPUDisassembler.cpp b/llvm/lib/Target/AMDGPU/Disassembler/AMDGPUDisassembler.cpp
index 58cdbe6cf373e..02ad08740049d 100644
--- a/llvm/lib/Target/AMDGPU/Disassembler/AMDGPUDisassembler.cpp
+++ b/llvm/lib/Target/AMDGPU/Disassembler/AMDGPUDisassembler.cpp
@@ -2233,15 +2233,15 @@ Expected<bool> AMDGPUDisassembler::decodeCOMPUTE_PGM_RSRC3(
 
     // Bits [4-11].
     if (isGFX11()) {
-      PRINT_PSEUDO_DIRECTIVE_COMMENT("INST_PREF_SIZE",
-                                     COMPUTE_PGM_RSRC3_GFX11_INST_PREF_SIZE);
+      PRINT_DIRECTIVE(".amdhsa_inst_pref_size",
+                      COMPUTE_PGM_RSRC3_GFX11_INST_PREF_SIZE);
       PRINT_PSEUDO_DIRECTIVE_COMMENT("TRAP_ON_START",
                                      COMPUTE_PGM_RSRC3_GFX11_TRAP_ON_START);
       PRINT_PSEUDO_DIRECTIVE_COMMENT("TRAP_ON_END",
                                      COMPUTE_PGM_RSRC3_GFX11_TRAP_ON_END);
     } else if (isGFX12Plus()) {
-      PRINT_PSEUDO_DIRECTIVE_COMMENT(
-          "INST_PREF_SIZE", COMPUTE_PGM_RSRC3_GFX12_PLUS_INST_PREF_SIZE);
+      PRINT_DIRECTIVE(".amdhsa_inst_pref_size",
+                      COMPUTE_PGM_RSRC3_GFX12_PLUS_INST_PREF_SIZE);
     } else {
       CHECK_RESERVED_BITS_DESC_MSG(COMPUTE_PGM_RSRC3_GFX10_RESERVED1,
                                    "COMPUTE_PGM_RSRC3",
diff --git a/llvm/lib/Target/AMDGPU/GCNHazardRecognizer.cpp b/llvm/lib/Target/AMDGPU/GCNHazardRecognizer.cpp
index 873d18e30a430..b0f087737afa7 100644
--- a/llvm/lib/Target/AMDGPU/GCNHazardRecognizer.cpp
+++ b/llvm/lib/Target/AMDGPU/GCNHazardRecognizer.cpp
@@ -14,7 +14,6 @@
 #include "GCNSubtarget.h"
 #include "MCTargetDesc/AMDGPUMCTargetDesc.h"
 #include "SIMachineFunctionInfo.h"
-#include "llvm/ADT/PostOrderIterator.h"
 #include "llvm/CodeGen/MachineFrameInfo.h"
 #include "llvm/CodeGen/MachineFunction.h"
 #include "llvm/CodeGen/ScheduleDAG.h"
@@ -45,10 +44,6 @@ static cl::opt<unsigned, false, MFMAPaddingRatioParser>
                      cl::desc("Fill a percentage of the latency between "
                               "neighboring MFMA with s_nops."));
 
-static cl::opt<unsigned> MaxExhaustiveHazardSearch(
-    "amdgpu-max-exhaustive-hazard-search", cl::init(128), cl::Hidden,
-    cl::desc("Maximum function size for exhausive hazard search"));
-
 //===----------------------------------------------------------------------===//
 // Hazard Recognizer Implementation
 //===----------------------------------------------------------------------===//
@@ -60,7 +55,6 @@ GCNHazardRecognizer::GCNHazardRecognizer(const MachineFunction &MF)
     : IsHazardRecognizerMode(false), CurrCycleInstr(nullptr), MF(MF),
       ST(MF.getSubtarget<GCNSubtarget>()), TII(*ST.getInstrInfo()),
       TRI(TII.getRegisterInfo()), TSchedModel(TII.getSchedModel()),
-      UseVALUReadHazardExhaustiveSearch(false),
       ClauseUses(TRI.getNumRegUnits()), ClauseDefs(TRI.getNumRegUnits()) {
   MaxLookAhead = MF.getRegInfo().isPhysRegUsed(AMDGPU::AGPR0) ? 19 : 5;
   RunLdsBranchVmemWARHazardFixup = shouldRunLdsBranchVmemWARHazardFixup(MF, ST);
@@ -1217,7 +1211,6 @@ void GCNHazardRecognizer::fixHazards(MachineInstr *MI) {
   fixWMMAHazards(MI);
   fixShift64HighRegBug(MI);
   fixVALUMaskWriteHazard(MI);
-  fixVALUReadSGPRHazard(MI);
   fixRequiredExportPriority(MI);
 }
 
@@ -2297,12 +2290,14 @@ GFX940_SMFMA_N_PassWritesVGPROverlappedSrcABWaitStates(int NumPasses) {
   return NumPasses + 2;
 }
 
-static int GFX940_XDL_N_PassWritesVGPROverlappedSrcABWaitStates(int NumPasses) {
-  // 2 pass -> 5
-  // 4 pass -> 7
-  // 8 pass -> 11
-  // 16 pass -> 19
-  return NumPasses + 3;
+static int GFX940_XDL_N_PassWritesVGPROverlappedSrcABWaitStates(int NumPasses,
+                                                                bool IsGFX950) {
+  // xdl def cycles | gfx940 | gfx950
+  // 2 pass         |  5        5
+  // 4 pass         |  7        8
+  // 8 pass         |  11       12
+  // 16 pass        |  19       20
+  return NumPasses + 3 + (NumPasses != 2 && IsGFX950);
 }
 
 int GCNHazardRecognizer::checkMAIHazards90A(MachineInstr *MI) {
@@ -2471,7 +2466,7 @@ int GCNHazardRecognizer::checkMAIHazards90A(MachineInstr *MI) {
           NeedWaitStates =
               isXDL(ST, *MI1)
                   ? GFX940_XDL_N_PassWritesVGPROverlappedSrcABWaitStates(
-                        NumPasses)
+                        NumPasses, ST.hasGFX950Insts())
                   : GFX940_SMFMA_N_PassWritesVGPROverlappedSrcABWaitStates(
                         NumPasses);
           break;
@@ -2603,20 +2598,24 @@ static int GFX940_SMFMA_N_PassWriteVgprVALUWawWaitStates(int NumPasses) {
   return NumPasses + 2;
 }
 
-static int GFX940_XDL_N_PassWriteVgprVALUWawWaitStates(int NumPasses) {
-  // 2 pass -> 5
-  // 4 pass -> 7
-  // 8 pass -> 11
-  // 16 pass -> 19
-  return NumPasses + 3;
+static int GFX940_XDL_N_PassWriteVgprVALUWawWaitStates(int NumPasses,
+                                                       bool IsGFX950) {
+  // xdl def cycles | gfx940 | gfx950
+  // 2 pass         |  5        5
+  // 4 pass         |  7        8
+  // 8 pass         |  11       12
+  // 16 pass        |  19       20
+  return NumPasses + 3 + (NumPasses != 2 && IsGFX950);
 }
 
-static int GFX940_XDL_N_PassWriteVgprVALUMemExpReadWaitStates(int NumPasses) {
-  // 2 pass -> 5
-  // 4 pass -> 7
-  // 8 pass -> 11
-  // 16 pass -> 19
-  return NumPasses + 3;
+static int GFX940_XDL_N_PassWriteVgprVALUMemExpReadWaitStates(int NumPasses,
+                                                              bool IsGFX950) {
+  // xdl def cycles | gfx940 | gfx950
+  // 2 pass         |  5        5
+  // 4 pass         |  7        8
+  // 8 pass         |  11       12
+  // 16 pass        |  19       20
+  return NumPasses + 3 + (NumPasses != 2 && IsGFX950);
 }
 
 static int GFX940_SMFMA_N_PassWriteVgprVALUMemExpReadWaitStates(int NumPasses) {
@@ -2767,7 +2766,8 @@ int GCNHazardRecognizer::checkMAIVALUHazards(MachineInstr *MI) {
       } else if (ST.hasGFX940Insts()) {
         NeedWaitStates =
             isXDL(ST, *MFMA)
-                ? GFX940_XDL_N_PassWriteVgprVALUMemExpReadWaitStates(NumPasses)
+                ? GFX940_XDL_N_PassWriteVgprVALUMemExpReadWaitStates(
+                      NumPasses, ST.hasGFX950Insts())
                 : GFX940_SMFMA_N_PassWriteVgprVALUMemExpReadWaitStates(
                       NumPasses);
       } else {
@@ -2853,7 +2853,8 @@ int GCNHazardRecognizer::checkMAIVALUHazards(MachineInstr *MI) {
       } else if (ST.hasGFX940Insts()) {
         NeedWaitStates =
             isXDL(ST, *MFMA)
-                ? GFX940_XDL_N_PassWriteVgprVALUWawWaitStates(NumPasses)
+                ? GFX940_XDL_N_PassWriteVgprVALUWawWaitStates(
+                      NumPasses, ST.hasGFX950Insts())
                 : GFX940_SMFMA_N_PassWriteVgprVALUWawWaitStates(NumPasses);
       } else {
         switch (NumPasses) {
@@ -3106,274 +3107,6 @@ bool GCNHazardRecognizer::fixVALUMaskWriteHazard(MachineInstr *MI) {
   return true;
 }
 
-// Return the numeric ID 0-63 of an 64b SGPR pair for a given SGPR.
-// i.e. SGPR0 = SGPR0_SGPR1 = 0, SGPR3 = SGPR2_SGPR3 = 1, etc
-static std::optional<unsigned> sgprPairNumber(Register Reg,
-                                              const SIRegisterInfo &TRI) {
-  switch (Reg) {
-  case AMDGPU::M0:
-  case AMDGPU::EXEC:
-  case AMDGPU::EXEC_LO:
-  case AMDGPU::EXEC_HI:
-  case AMDGPU::SGPR_NULL:
-  case AMDGPU::SGPR_NULL64:
-    return {};
-  default:
-    break;
-  }
-  unsigned RegN = TRI.getEncodingValue(Reg);
-  if (RegN > 127)
-    return {};
-  return (RegN >> 1) & 0x3f;
-}
-
-// For VALUReadSGPRHazard: pre-compute a bit vector of all SGPRs used by VALUs.
-void GCNHazardRecognizer::computeVALUHazardSGPRs(MachineFunction *MMF) {
-  assert(MMF == &MF);
-
-  // Assume non-empty vector means it has already been computed.
-  if (!VALUReadHazardSGPRs.empty())
-    return;
-
-  auto CallingConv = MF.getFunction().getCallingConv();
-  bool IsCallFree =
-      AMDGPU::isEntryFunctionCC(CallingConv) && !MF.getFrameInfo().hasCalls();
-
-  // Exhaustive search is only viable in non-caller/callee functions where
-  // VALUs will be exposed to the hazard recognizer.
-  UseVALUReadHazardExhaustiveSearch =
-      IsCallFree && MF.getTarget().getOptLevel() > CodeGenOptLevel::None &&
-      MF.getInstructionCount() <= MaxExhaustiveHazardSearch;
-
-  // Consider all SGPRs hazards if the shader uses function calls or is callee.
-  bool UseVALUUseCache =
-      IsCallFree && MF.getTarget().getOptLevel() > CodeGenOptLevel::None;
-  VALUReadHazardSGPRs.resize(64, !UseVALUUseCache);
-  if (!UseVALUUseCache)
-    return;
-
-  // Perform a post ordered reverse scan to find VALUs which read an SGPR
-  // before a SALU write to the same SGPR.  This provides a reduction in
-  // hazard insertion when all VALU access to an SGPR occurs after its last
-  // SALU write, when compared to a linear scan.
-  const MachineRegisterInfo &MRI = MF.getRegInfo();
-  BitVector SALUWriteSGPRs(64), ReadSGPRs(64);
-  MachineCycleInfo CI;
-  CI.compute(*MMF);
-
-  for (auto *MBB : post_order(&MF)) {
-    bool InCycle = CI.getCycle(MBB) != nullptr;
-    for (auto &MI : reverse(MBB->instrs())) {
-      bool IsVALU = SIInstrInfo::isVALU(MI);
-      bool IsSALU = SIInstrInfo::isSALU(MI);
-      if (!IsVALU && !IsSALU)
-        continue;
-
-      for (const MachineOperand &Op : MI.operands()) {
-        if (!Op.isReg())
-          continue;
-        Register Reg = Op.getReg();
-        assert(!Op.getSubReg());
-        // Only consider implicit operands of VCC.
-        if (Op.isImplicit() && !(Reg == AMDGPU::VCC_LO ||
-                                 Reg == AMDGPU::VCC_HI || Reg == AMDGPU::VCC))
-          continue;
-        if (!TRI.isSGPRReg(MRI, Reg))
-          continue;
-        auto RegN = sgprPairNumber(Reg, TRI);
-        if (!RegN)
-          continue;
-        if (IsVALU && Op.isUse()) {
-          // Note: any access within a cycle must be considered a hazard.
-          if (InCycle || (ReadSGPRs[*RegN] && SALUWriteSGPRs[*RegN]))
-            VALUReadHazardSGPRs.set(*RegN);
-          ReadSGPRs.set(*RegN);
-        } else if (IsSALU) {
-          if (Op.isDef())
-            SALUWriteSGPRs.set(*RegN);
-          else
-            ReadSGPRs.set(*RegN);
-        }
-      }
-    }
-  }
-}
-
-bool GCNHazardRecognizer::fixVALUReadSGPRHazard(MachineInstr *MI) {
-  if (!ST.hasVALUReadSGPRHazard())
-    return false;
-
-  // The hazard sequence is fundamentally three instructions:
-  //   1. VALU reads SGPR
-  //   2. SALU writes SGPR
-  //   3. VALU/SALU reads SGPR
-  // Try to avoid searching for (1) because the expiry point of the hazard is
-  // indeterminate; however, the hazard between (2) and (3) can expire if the
-  // gap contains sufficient SALU instructions with no usage of SGPR from (1).
-  // Note: SGPRs must be considered as 64-bit pairs as hazard exists
-  // even if individual SGPRs are accessed.
-
-  bool MIIsSALU = SIInstrInfo::isSALU(*MI);
-  bool MIIsVALU = SIInstrInfo::isVALU(*MI);
-  if (!(MIIsSALU || MIIsVALU))
-    return false;
-
-  // Avoid expensive search when compile time is priority by
-  // mitigating every SALU which writes an SGPR.
-  if (MF.getTarget().getOptLevel() == CodeGenOptLevel::None) {
-    if (!SIInstrInfo::isSALU(*MI) || SIInstrInfo::isSOPP(*MI))
-      return false;
-
-    const MachineOperand *SDSTOp =
-        TII.getNamedOperand(*MI, AMDGPU::OpName::sdst);
-    if (!SDSTOp || !SDSTOp->isReg())
-      return false;
-
-    const Register HazardReg = SDSTOp->getReg();
-    if (HazardReg == AMDGPU::EXEC || HazardReg == AMDGPU::EXEC_LO ||
-        HazardReg == AMDGPU::EXEC_HI || HazardReg == AMDGPU::M0)
-      return false;
-
-    // Add s_wait_alu sa_sdst(0) after SALU write.
-    auto NextMI = std::next(MI->getIterator());
-    auto NewMI = BuildMI(*MI->getParent(), NextMI, MI->getDebugLoc(),
-                         TII.get(AMDGPU::S_WAITCNT_DEPCTR))
-                     .addImm(AMDGPU::DepCtr::encodeFieldSaSdst(0));
-
-    // SALU write may be s_getpc in a bundle.
-    updateGetPCBundle(NewMI);
-
-    return true;
-  }
-
-  // Pre-compute set of SGPR pairs read by VALUs.
-  // Note: pass mutable pointer to MachineFunction for CycleInfo.
-  computeVALUHazardSGPRs(MI->getMF());
-
-  // If no VALUs hazard SGPRs exist then nothing to do.
-  if (VALUReadHazardSGPRs.none())
-    return false;
-
-  // All SGPR writes before a call/return must be flushed as the callee/caller
-  // will not will not see the hazard chain, i.e. (2) to (3) described above.
-  const bool IsSetPC = (MI->isCall() || MI->isReturn()) &&
-                       !(MI->getOpcode() == AMDGPU::S_ENDPGM ||
-                         MI->getOpcode() == AMDGPU::S_ENDPGM_SAVED);
-
-  // Collect all SGPR sources for MI which are read by a VALU.
-  const MachineRegisterInfo &MRI = MF.getRegInfo();
-  SmallSet<Register, 4> SGPRsUsed;
-
-  if (!IsSetPC) {
-    for (const MachineOperand &Op : MI->all_uses()) {
-      Register OpReg = Op.getReg();
-
-      // Only consider VCC implicit uses on VALUs.
-      // The only expected SALU implicit access is SCC which is no hazard.
-      if (MIIsSALU && Op.isImplicit())
-        continue;
-
-      if (!TRI.isSGPRReg(MRI, OpReg))
-        continue;
-
-      auto RegN = sgprPairNumber(OpReg, TRI);
-      if (!RegN)
-        continue;
-
-      if (!VALUReadHazardSGPRs[*RegN])
-        continue;
-
-      SGPRsUsed.insert(OpReg);
-    }
-
-    // No SGPRs -> nothing to do.
-    if (SGPRsUsed.empty())
-      return false;
-  }
-
-  // A hazard is any SALU which writes one of the SGPRs read by MI.
-  auto IsHazardFn = [this, IsSetPC, &SGPRsUsed](const MachineInstr &I) {
-    if (!SIInstrInfo::isSALU(I))
-      return false;
-    // Ensure SGPR flush before call/return by conservatively assuming every
-    // SALU writes an SGPR.
-    if (IsSetPC && I.getNumDefs() > 0)
-      return true;
-    // Check for any register writes.
-    return any_of(SGPRsUsed, [this, &I](Register Reg) {
-      return I.modifiesRegister(Reg, &TRI);
-    });
-  };
-
-  const int SALUExpiryCount = SIInstrInfo::isSALU(*MI) ? 10 : 11;
-  auto IsExpiredFn = [&](const MachineInstr &I, int Count) {
-    if (Count >= SALUExpiryCount)
-      return true;
-    // s_wait_alu sa_sdst(0) on path mitigates hazard.
-    if (I.getOpcode() == AMDGPU::S_WAITCNT_DEPCTR &&
-        AMDGPU::DepCtr::decodeFieldSaSdst(I.getOperand(0).getImm()) == 0)
-      return true;
-    return false;
-  };
-
-  auto WaitStatesFn = [this, &SGPRsUsed](const MachineInstr &I) {
-    // Only count true SALUs as wait states.
-    if (!SIInstrInfo::isSALU(I) || SIInstrInfo::isSOPP(I))
-      return 0;
-    // SALU must be unrelated to any hazard registers.
-    if (any_of(SGPRsUsed,
-               [this, &I](Register Reg) { return I.readsRegister(Reg, &TRI); }))
-      return 0;
-    return 1;
-  };
-
-  // Check for the hazard.
-  DenseSet<const MachineBasicBlock *> Visited;
-  int WaitStates = ::getWaitStatesSince(IsHazardFn, MI->getParent(),
-                                        std::next(MI->getReverseIterator()), 0,
-                                        IsExpiredFn, Visited, WaitStatesFn);
-
-  if (WaitStates >= SALUExpiryCount)
-    return false;
-
-  // Validate hazard through an exhaustive search.
-  if (UseVALUReadHazardExhaustiveSearch) {
-    // A hazard is any VALU which reads one of the paired SGPRs read by MI.
-    // This is searching for (1) in the hazard description.
-    auto hazardPair = [this](Register Reg) {
-      if (Reg == AMDGPU::VCC || Reg == AMDGPU::VCC_LO || Reg == AMDGPU::VCC_HI)
-        return Register(AMDGPU::VCC);
-      auto RegN = sgprPairNumber(Reg, TRI);
-      return Register(AMDGPU::SGPR0_SGPR1 + *RegN);
-    };
-    auto SearchHazardFn = [this, hazardPair,
-                           &SGPRsUsed](const MachineInstr &I) {
-      if (!SIInstrInfo::isVALU(I))
-        return false;
-      // Check for any register reads.
-      return any_of(SGPRsUsed, [this, hazardPair, &I](Register Reg) {
-        return I.readsRegister(hazardPair(Reg), &TRI);
-      });
-    };
-    auto SearchExpiredFn = [&](const MachineInstr &I, int Count) {
-      return false;
-    };
-    if (::getWaitStatesSince(SearchHazardFn, MI, SearchExpiredFn) ==
-        std::numeric_limits<int>::max())
-      return false;
-  }
-
-  // Add s_wait_alu sa_sdst(0) before SALU read.
-  auto NewMI = BuildMI(*MI->getParent(), MI, MI->getDebugLoc(),
-                       TII.get(AMDGPU::S_WAITCNT_DEPCTR))
-                   .addImm(AMDGPU::DepCtr::encodeFieldSaSdst(0));
-
-  // SALU read may be after s_getpc in a bundle.
-  updateGetPCBundle(NewMI);
-
-  return true;
-}
-
 static bool ensureEntrySetPrio(MachineFunction *MF, int Priority,
                                const SIInstrInfo &TII) {
   MachineBasicBlock &EntryMBB = MF->front();
diff --git a/llvm/lib/Target/AMDGPU/GCNHazardRecognizer.h b/llvm/lib/Target/AMDGPU/GCNHazardRecognizer.h
index 83ce100c58f0a..bbc55851bf967 100644
--- a/llvm/lib/Target/AMDGPU/GCNHazardRecognizer.h
+++ b/llvm/lib/Target/AMDGPU/GCNHazardRecognizer.h
@@ -48,8 +48,6 @@ class GCNHazardRecognizer final : public ScheduleHazardRecognizer {
   const SIRegisterInfo &TRI;
   const TargetSchedModel &TSchedModel;
   bool RunLdsBranchVmemWARHazardFixup;
-  BitVector VALUReadHazardSGPRs;
-  bool UseVALUReadHazardExhaustiveSearch;
 
   /// RegUnits of uses in the current soft memory clause.
   BitVector ClauseUses;
@@ -109,8 +107,6 @@ class GCNHazardRecognizer final : public ScheduleHazardRecognizer {
   bool fixWMMAHazards(MachineInstr *MI);
   bool fixShift64HighRegBug(MachineInstr *MI);
   bool fixVALUMaskWriteHazard(MachineInstr *MI);
-  void computeVALUHazardSGPRs(MachineFunction *MMF);
-  bool fixVALUReadSGPRHazard(MachineInstr *MI);
   bool fixRequiredExportPriority(MachineInstr *MI);
 
   int checkMAIHazards(MachineInstr *MI);
diff --git a/llvm/lib/Target/AMDGPU/GCNPreRALongBranchReg.cpp b/llvm/lib/Target/AMDGPU/GCNPreRALongBranchReg.cpp
index 76a41487af7e9..355bbeb2ffeba 100644
--- a/llvm/lib/Target/AMDGPU/GCNPreRALongBranchReg.cpp
+++ b/llvm/lib/Target/AMDGPU/GCNPreRALongBranchReg.cpp
@@ -14,6 +14,7 @@
 // distrance threshold tuning of what is considered "long" is handled through
 // amdgpu-long-branch-factor cl argument which sets LongBranchFactor.
 //===----------------------------------------------------------------------===//
+#include "GCNPreRALongBranchReg.h"
 #include "AMDGPU.h"
 #include "GCNSubtarget.h"
 #include "SIMachineFunctionInfo.h"
@@ -36,7 +37,7 @@ static cl::opt<double> LongBranchFactor(
              "reserved. We lean towards always reserving a register for  "
              "long jumps"));
 
-class GCNPreRALongBranchReg : public MachineFunctionPass {
+class GCNPreRALongBranchReg {
 
   struct BasicBlockInfo {
     // Offset - Distance from the beginning of the function to the beginning
@@ -48,27 +49,39 @@ class GCNPreRALongBranchReg : public MachineFunctionPass {
   void generateBlockInfo(MachineFunction &MF,
                          SmallVectorImpl<BasicBlockInfo> &BlockInfo);
 
+public:
+  GCNPreRALongBranchReg() = default;
+  bool run(MachineFunction &MF);
+};
+
+class GCNPreRALongBranchRegLegacy : public MachineFunctionPass {
 public:
   static char ID;
-  GCNPreRALongBranchReg() : MachineFunctionPass(ID) {
-    initializeGCNPreRALongBranchRegPass(*PassRegistry::getPassRegistry());
+  GCNPreRALongBranchRegLegacy() : MachineFunctionPass(ID) {
+    initializeGCNPreRALongBranchRegLegacyPass(*PassRegistry::getPassRegistry());
   }
-  bool runOnMachineFunction(MachineFunction &MF) override;
+
+  bool runOnMachineFunction(MachineFunction &MF) override {
+    return GCNPreRALongBranchReg().run(MF);
+  }
+
   StringRef getPassName() const override {
     return "AMDGPU Pre-RA Long Branch Reg";
   }
+
   void getAnalysisUsage(AnalysisUsage &AU) const override {
     AU.setPreservesAll();
     MachineFunctionPass::getAnalysisUsage(AU);
   }
 };
 } // End anonymous namespace.
-char GCNPreRALongBranchReg::ID = 0;
 
-INITIALIZE_PASS(GCNPreRALongBranchReg, DEBUG_TYPE,
+char GCNPreRALongBranchRegLegacy::ID = 0;
+
+INITIALIZE_PASS(GCNPreRALongBranchRegLegacy, DEBUG_TYPE,
                 "AMDGPU Pre-RA Long Branch Reg", false, false)
 
-char &llvm::GCNPreRALongBranchRegID = GCNPreRALongBranchReg::ID;
+char &llvm::GCNPreRALongBranchRegID = GCNPreRALongBranchRegLegacy::ID;
 void GCNPreRALongBranchReg::generateBlockInfo(
     MachineFunction &MF, SmallVectorImpl<BasicBlockInfo> &BlockInfo) {
 
@@ -99,7 +112,8 @@ void GCNPreRALongBranchReg::generateBlockInfo(
     PrevNum = Num;
   }
 }
-bool GCNPreRALongBranchReg::runOnMachineFunction(MachineFunction &MF) {
+
+bool GCNPreRALongBranchReg::run(MachineFunction &MF) {
   const GCNSubtarget &STM = MF.getSubtarget<GCNSubtarget>();
   const SIInstrInfo *TII = STM.getInstrInfo();
   const SIRegisterInfo *TRI = STM.getRegisterInfo();
@@ -136,3 +150,10 @@ bool GCNPreRALongBranchReg::runOnMachineFunction(MachineFunction &MF) {
   }
   return false;
 }
+
+PreservedAnalyses
+GCNPreRALongBranchRegPass::run(MachineFunction &MF,
+                               MachineFunctionAnalysisManager &MFAM) {
+  GCNPreRALongBranchReg().run(MF);
+  return PreservedAnalyses::all();
+}
diff --git a/llvm/lib/Target/AMDGPU/GCNPreRALongBranchReg.h b/llvm/lib/Target/AMDGPU/GCNPreRALongBranchReg.h
new file mode 100644
index 0000000000000..4cd7dea83a061
--- /dev/null
+++ b/llvm/lib/Target/AMDGPU/GCNPreRALongBranchReg.h
@@ -0,0 +1,23 @@
+//===- GCNPreRALongBranchReg.h ----------------------------------*- C++- *-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIB_TARGET_AMDGPU_GCNPRERALONGBRANCHREG_H
+#define LLVM_LIB_TARGET_AMDGPU_GCNPRERALONGBRANCHREG_H
+
+#include "llvm/CodeGen/MachinePassManager.h"
+
+namespace llvm {
+class GCNPreRALongBranchRegPass
+    : public PassInfoMixin<GCNPreRALongBranchRegPass> {
+public:
+  PreservedAnalyses run(MachineFunction &MF,
+                        MachineFunctionAnalysisManager &MFAM);
+};
+} // namespace llvm
+
+#endif // LLVM_LIB_TARGET_AMDGPU_GCNPRERALONGBRANCHREG_H
diff --git a/llvm/lib/Target/AMDGPU/GCNPreRAOptimizations.cpp b/llvm/lib/Target/AMDGPU/GCNPreRAOptimizations.cpp
index a6112b39325ee..0f008f70a6c3d 100644
--- a/llvm/lib/Target/AMDGPU/GCNPreRAOptimizations.cpp
+++ b/llvm/lib/Target/AMDGPU/GCNPreRAOptimizations.cpp
@@ -24,6 +24,7 @@
 ///
 //===----------------------------------------------------------------------===//
 
+#include "GCNPreRAOptimizations.h"
 #include "AMDGPU.h"
 #include "GCNSubtarget.h"
 #include "MCTargetDesc/AMDGPUMCTargetDesc.h"
@@ -37,7 +38,7 @@ using namespace llvm;
 
 namespace {
 
-class GCNPreRAOptimizations : public MachineFunctionPass {
+class GCNPreRAOptimizationsImpl {
 private:
   const SIInstrInfo *TII;
   const SIRegisterInfo *TRI;
@@ -46,11 +47,17 @@ class GCNPreRAOptimizations : public MachineFunctionPass {
 
   bool processReg(Register Reg);
 
+public:
+  GCNPreRAOptimizationsImpl(LiveIntervals *LS) : LIS(LS) {}
+  bool run(MachineFunction &MF);
+};
+
+class GCNPreRAOptimizationsLegacy : public MachineFunctionPass {
 public:
   static char ID;
 
-  GCNPreRAOptimizations() : MachineFunctionPass(ID) {
-    initializeGCNPreRAOptimizationsPass(*PassRegistry::getPassRegistry());
+  GCNPreRAOptimizationsLegacy() : MachineFunctionPass(ID) {
+    initializeGCNPreRAOptimizationsLegacyPass(*PassRegistry::getPassRegistry());
   }
 
   bool runOnMachineFunction(MachineFunction &MF) override;
@@ -65,24 +72,23 @@ class GCNPreRAOptimizations : public MachineFunctionPass {
     MachineFunctionPass::getAnalysisUsage(AU);
   }
 };
-
 } // End anonymous namespace.
 
-INITIALIZE_PASS_BEGIN(GCNPreRAOptimizations, DEBUG_TYPE,
+INITIALIZE_PASS_BEGIN(GCNPreRAOptimizationsLegacy, DEBUG_TYPE,
                       "AMDGPU Pre-RA optimizations", false, false)
 INITIALIZE_PASS_DEPENDENCY(LiveIntervalsWrapperPass)
-INITIALIZE_PASS_END(GCNPreRAOptimizations, DEBUG_TYPE, "Pre-RA optimizations",
-                    false, false)
+INITIALIZE_PASS_END(GCNPreRAOptimizationsLegacy, DEBUG_TYPE,
+                    "Pre-RA optimizations", false, false)
 
-char GCNPreRAOptimizations::ID = 0;
+char GCNPreRAOptimizationsLegacy::ID = 0;
 
-char &llvm::GCNPreRAOptimizationsID = GCNPreRAOptimizations::ID;
+char &llvm::GCNPreRAOptimizationsID = GCNPreRAOptimizationsLegacy::ID;
 
-FunctionPass *llvm::createGCNPreRAOptimizationsPass() {
-  return new GCNPreRAOptimizations();
+FunctionPass *llvm::createGCNPreRAOptimizationsLegacyPass() {
+  return new GCNPreRAOptimizationsLegacy();
 }
 
-bool GCNPreRAOptimizations::processReg(Register Reg) {
+bool GCNPreRAOptimizationsImpl::processReg(Register Reg) {
   MachineInstr *Def0 = nullptr;
   MachineInstr *Def1 = nullptr;
   uint64_t Init = 0;
@@ -212,14 +218,25 @@ bool GCNPreRAOptimizations::processReg(Register Reg) {
   return true;
 }
 
-bool GCNPreRAOptimizations::runOnMachineFunction(MachineFunction &MF) {
+bool GCNPreRAOptimizationsLegacy::runOnMachineFunction(MachineFunction &MF) {
   if (skipFunction(MF.getFunction()))
     return false;
+  LiveIntervals *LIS = &getAnalysis<LiveIntervalsWrapperPass>().getLIS();
+  return GCNPreRAOptimizationsImpl(LIS).run(MF);
+}
+
+PreservedAnalyses
+GCNPreRAOptimizationsPass::run(MachineFunction &MF,
+                               MachineFunctionAnalysisManager &MFAM) {
+  LiveIntervals *LIS = &MFAM.getResult<LiveIntervalsAnalysis>(MF);
+  GCNPreRAOptimizationsImpl(LIS).run(MF);
+  return PreservedAnalyses::all();
+}
 
+bool GCNPreRAOptimizationsImpl::run(MachineFunction &MF) {
   const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
   TII = ST.getInstrInfo();
   MRI = &MF.getRegInfo();
-  LIS = &getAnalysis<LiveIntervalsWrapperPass>().getLIS();
   TRI = ST.getRegisterInfo();
 
   bool Changed = false;
diff --git a/llvm/lib/Target/AMDGPU/GCNPreRAOptimizations.h b/llvm/lib/Target/AMDGPU/GCNPreRAOptimizations.h
new file mode 100644
index 0000000000000..295152b64b4c6
--- /dev/null
+++ b/llvm/lib/Target/AMDGPU/GCNPreRAOptimizations.h
@@ -0,0 +1,23 @@
+//===- GCNPreRAOptimizations.h ----------------------------------*- C++- *-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIB_TARGET_AMDGPU_GCNPRERAOPTIMIZATIONS_H
+#define LLVM_LIB_TARGET_AMDGPU_GCNPRERAOPTIMIZATIONS_H
+
+#include "llvm/CodeGen/MachinePassManager.h"
+
+namespace llvm {
+class GCNPreRAOptimizationsPass
+    : public PassInfoMixin<GCNPreRAOptimizationsPass> {
+public:
+  PreservedAnalyses run(MachineFunction &MF,
+                        MachineFunctionAnalysisManager &MFAM);
+};
+} // namespace llvm
+
+#endif // LLVM_LIB_TARGET_AMDGPU_GCNPRERAOPTIMIZATIONS_H
diff --git a/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp b/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp
index b00105ae9bd52..176586e3fbbb6 100644
--- a/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp
+++ b/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp
@@ -1615,6 +1615,64 @@ void GCNSchedStage::revertScheduling() {
   DAG.Regions[RegionIdx] = std::pair(DAG.RegionBegin, DAG.RegionEnd);
 }
 
+bool PreRARematStage::allUsesAvailableAt(const MachineInstr *InstToRemat,
+                                         SlotIndex OriginalIdx,
+                                         SlotIndex RematIdx) const {
+
+  LiveIntervals *LIS = DAG.LIS;
+  MachineRegisterInfo &MRI = DAG.MRI;
+  OriginalIdx = OriginalIdx.getRegSlot(true);
+  RematIdx = std::max(RematIdx, RematIdx.getRegSlot(true));
+  for (const MachineOperand &MO : InstToRemat->operands()) {
+    if (!MO.isReg() || !MO.getReg() || !MO.readsReg())
+      continue;
+
+    if (!MO.getReg().isVirtual()) {
+      // Do not attempt to reason about PhysRegs
+      // TODO: better analysis of PhysReg livness
+      if (!DAG.MRI.isConstantPhysReg(MO.getReg()) &&
+          !DAG.TII->isIgnorableUse(MO))
+        return false;
+
+      // Constant PhysRegs and IgnorableUses are okay
+      continue;
+    }
+
+    LiveInterval &LI = LIS->getInterval(MO.getReg());
+    const VNInfo *OVNI = LI.getVNInfoAt(OriginalIdx);
+    assert(OVNI);
+
+    // Don't allow rematerialization immediately after the original def.
+    // It would be incorrect if InstToRemat redefines the register.
+    // See PR14098.
+    if (SlotIndex::isSameInstr(OriginalIdx, RematIdx))
+      return false;
+
+    if (OVNI != LI.getVNInfoAt(RematIdx))
+      return false;
+
+    // Check that subrange is live at RematIdx.
+    if (LI.hasSubRanges()) {
+      const TargetRegisterInfo *TRI = MRI.getTargetRegisterInfo();
+      unsigned SubReg = MO.getSubReg();
+      LaneBitmask LM = SubReg ? TRI->getSubRegIndexLaneMask(SubReg)
+                              : MRI.getMaxLaneMaskForVReg(MO.getReg());
+      for (LiveInterval::SubRange &SR : LI.subranges()) {
+        if ((SR.LaneMask & LM).none())
+          continue;
+        if (!SR.liveAt(RematIdx))
+          return false;
+
+        // Early exit if all used lanes are checked. No need to continue.
+        LM &= ~SR.LaneMask;
+        if (LM.none())
+          break;
+      }
+    }
+  }
+  return true;
+}
+
 void PreRARematStage::collectRematerializableInstructions() {
   const SIRegisterInfo *SRI = static_cast<const SIRegisterInfo *>(DAG.TRI);
   for (unsigned I = 0, E = DAG.MRI.getNumVirtRegs(); I != E; ++I) {
@@ -1636,6 +1694,47 @@ void PreRARematStage::collectRematerializableInstructions() {
     if (Def->getParent() == UseI->getParent())
       continue;
 
+    bool HasRematDependency = false;
+    // Check if this instruction uses any registers that are planned to be
+    // rematerialized
+    for (auto &RematEntry : RematerializableInsts) {
+      if (find_if(RematEntry.second,
+                  [&Def](std::pair<MachineInstr *, MachineInstr *> &Remat) {
+                    for (MachineOperand &MO : Def->operands()) {
+                      if (!MO.isReg())
+                        continue;
+                      if (MO.getReg() == Remat.first->getOperand(0).getReg())
+                        return true;
+                    }
+                    return false;
+                  }) != RematEntry.second.end()) {
+        HasRematDependency = true;
+        break;
+      }
+    }
+    // Do not rematerialize an instruction if it uses an instruction that we
+    // have designated for rematerialization.
+    // FIXME: Allow for rematerialization chains: this requires 1. updating
+    // remat points to account for uses that are rematerialized, and 2. either
+    // rematerializing the candidates in careful ordering, or deferring the MBB
+    // RP walk until the entire chain has been rematerialized.
+    if (HasRematDependency)
+      continue;
+
+    // Similarly, check if the UseI is planned to be remat.
+    for (auto &RematEntry : RematerializableInsts) {
+      if (find_if(RematEntry.second,
+                  [&UseI](std::pair<MachineInstr *, MachineInstr *> &Remat) {
+                    return Remat.first == UseI;
+                  }) != RematEntry.second.end()) {
+        HasRematDependency = true;
+        break;
+      }
+    }
+
+    if (HasRematDependency)
+      break;
+
     // We are only collecting defs that are defined in another block and are
     // live-through or used inside regions at MinOccupancy. This means that the
     // register must be in the live-in set for the region.
@@ -1644,8 +1743,13 @@ void PreRARematStage::collectRematerializableInstructions() {
       auto It = DAG.LiveIns[I].find(Reg);
       if (It != DAG.LiveIns[I].end() && !It->second.none()) {
         if (DAG.RegionsWithMinOcc[I]) {
-          RematerializableInsts[I][Def] = UseI;
-          AddedToRematList = true;
+          SlotIndex DefIdx = DAG.LIS->getInstructionIndex(*Def);
+          SlotIndex UseIdx =
+              DAG.LIS->getInstructionIndex(*UseI).getRegSlot(true);
+          if (allUsesAvailableAt(Def, DefIdx, UseIdx)) {
+            RematerializableInsts[I][Def] = UseI;
+            AddedToRematList = true;
+          }
         }
 
         // Collect regions with rematerializable reg as live-in to avoid
@@ -1719,6 +1823,35 @@ bool PreRARematStage::sinkTriviallyRematInsts(const GCNSubtarget &ST,
       Register DefReg = Def->getOperand(0).getReg();
       TotalSinkableRegs +=
           SIRegisterInfo::getNumCoveredRegs(NewLiveIns[I][DefReg]);
+#ifdef EXPENSIVE_CHECKS
+      // All uses are known to be available / live at the remat point. Thus, the
+      // uses should already be live in to the region.
+      for (MachineOperand &MO : Def->operands()) {
+        if (!MO.isReg() || !MO.getReg() || !MO.readsReg())
+          continue;
+
+        Register UseReg = MO.getReg();
+        if (!UseReg.isVirtual())
+          continue;
+
+        LiveInterval &LI = LIS->getInterval(UseReg);
+        LaneBitmask LM = DAG.MRI.getMaxLaneMaskForVReg(MO.getReg());
+        if (LI.hasSubRanges() && MO.getSubReg())
+          LM = DAG.TRI->getSubRegIndexLaneMask(MO.getSubReg());
+
+        assert(NewLiveIns[I].contains(UseReg));
+        LaneBitmask LiveInMask = NewLiveIns[I][UseReg];
+        LaneBitmask UncoveredLanes = LM & ~(LiveInMask & LM);
+        // If this register has lanes not covered by the LiveIns, be sure they
+        // do not map to any subrange. ref:
+        // machine-scheduler-sink-trivial-remats.mir::omitted_subrange
+        if (UncoveredLanes.any()) {
+          assert(LI.hasSubRanges());
+          for (LiveInterval::SubRange &SR : LI.subranges())
+            assert((SR.LaneMask & UncoveredLanes).none());
+        }
+      }
+#endif
     }
     int VGPRsAfterSink = VGPRUsage - TotalSinkableRegs;
     unsigned OptimisticOccupancy = ST.getOccupancyWithNumVGPRs(VGPRsAfterSink);
@@ -1734,10 +1867,7 @@ bool PreRARematStage::sinkTriviallyRematInsts(const GCNSubtarget &ST,
       MachineBasicBlock::iterator InsertPos =
           MachineBasicBlock::iterator(It.second);
       Register Reg = Def->getOperand(0).getReg();
-      // Rematerialize MI to its use block. Since we are only rematerializing
-      // instructions that do not have any virtual reg uses, we do not need to
-      // call LiveRangeEdit::allUsesAvailableAt() and
-      // LiveRangeEdit::canRematerializeAt().
+      // Rematerialize MI to its use block.
       TII->reMaterialize(*InsertPos->getParent(), InsertPos, Reg,
                          Def->getOperand(0).getSubReg(), *Def, *DAG.TRI);
       MachineInstr *NewMI = &*std::prev(InsertPos);
@@ -1842,14 +1972,19 @@ bool PreRARematStage::sinkTriviallyRematInsts(const GCNSubtarget &ST,
   return true;
 }
 
-// Copied from MachineLICM
 bool PreRARematStage::isTriviallyReMaterializable(const MachineInstr &MI) {
   if (!DAG.TII->isTriviallyReMaterializable(MI))
     return false;
 
-  for (const MachineOperand &MO : MI.all_uses())
-    if (MO.getReg().isVirtual())
+  for (const MachineOperand &MO : MI.all_uses()) {
+    // We can't remat physreg uses, unless it is a constant or an ignorable
+    // use (e.g. implicit exec use on VALU instructions)
+    if (MO.getReg().isPhysical()) {
+      if (DAG.MRI.isConstantPhysReg(MO.getReg()) || DAG.TII->isIgnorableUse(MO))
+        continue;
       return false;
+    }
+  }
 
   return true;
 }
diff --git a/llvm/lib/Target/AMDGPU/GCNSchedStrategy.h b/llvm/lib/Target/AMDGPU/GCNSchedStrategy.h
index 44db834a41f82..e3da8d3005629 100644
--- a/llvm/lib/Target/AMDGPU/GCNSchedStrategy.h
+++ b/llvm/lib/Target/AMDGPU/GCNSchedStrategy.h
@@ -442,7 +442,7 @@ class PreRARematStage : public GCNSchedStage {
 
   // Map a trivially rematerializable def to a list of regions at MinOccupancy
   // that has the defined reg as a live-in.
-  DenseMap<MachineInstr *, SmallVector<unsigned, 4>> RematDefToLiveInRegions;
+  MapVector<MachineInstr *, SmallVector<unsigned, 4>> RematDefToLiveInRegions;
 
   // Collect all trivially rematerializable VGPR instructions with a single def
   // and single use outside the defining block into RematerializableInsts.
@@ -456,6 +456,12 @@ class PreRARematStage : public GCNSchedStage {
   bool sinkTriviallyRematInsts(const GCNSubtarget &ST,
                                const TargetInstrInfo *TII);
 
+  /// \p Returns true if all the uses in \p InstToRemat defined at \p
+  /// OriginalIdx are live at \p RematIdx. This only checks liveness of virtual
+  /// reg uses.
+  bool allUsesAvailableAt(const MachineInstr *InstToRemat,
+                          SlotIndex OriginalIdx, SlotIndex RematIdx) const;
+
 public:
   bool initGCNSchedStage() override;
 
diff --git a/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUTargetStreamer.cpp b/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUTargetStreamer.cpp
index eccd77d6c00f0..059bab5838526 100644
--- a/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUTargetStreamer.cpp
+++ b/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUTargetStreamer.cpp
@@ -579,7 +579,17 @@ void AMDGPUTargetAsmStreamer::EmitAmdhsaKernelDescriptor(
                amdhsa::COMPUTE_PGM_RSRC3_GFX10_GFX11_SHARED_VGPR_COUNT,
                ".amdhsa_shared_vgpr_count");
   }
+  if (IVersion.Major == 11) {
+    PrintField(KD.compute_pgm_rsrc3,
+               amdhsa::COMPUTE_PGM_RSRC3_GFX11_INST_PREF_SIZE_SHIFT,
+               amdhsa::COMPUTE_PGM_RSRC3_GFX11_INST_PREF_SIZE,
+               ".amdhsa_inst_pref_size");
+  }
   if (IVersion.Major >= 12) {
+    PrintField(KD.compute_pgm_rsrc3,
+               amdhsa::COMPUTE_PGM_RSRC3_GFX12_PLUS_INST_PREF_SIZE_SHIFT,
+               amdhsa::COMPUTE_PGM_RSRC3_GFX12_PLUS_INST_PREF_SIZE,
+               ".amdhsa_inst_pref_size");
     PrintField(KD.compute_pgm_rsrc1,
                amdhsa::COMPUTE_PGM_RSRC1_GFX12_PLUS_ENABLE_WG_RR_EN_SHIFT,
                amdhsa::COMPUTE_PGM_RSRC1_GFX12_PLUS_ENABLE_WG_RR_EN,
diff --git a/llvm/lib/Target/AMDGPU/R600OptimizeVectorRegisters.cpp b/llvm/lib/Target/AMDGPU/R600OptimizeVectorRegisters.cpp
index a20319ea4f9d3..ac11526da0919 100644
--- a/llvm/lib/Target/AMDGPU/R600OptimizeVectorRegisters.cpp
+++ b/llvm/lib/Target/AMDGPU/R600OptimizeVectorRegisters.cpp
@@ -287,10 +287,10 @@ bool R600VectorRegMerger::tryMergeUsingFreeSlot(RegSeqInfo &RSI,
     RegSeqInfo &CompatibleRSI,
     std::vector<std::pair<unsigned, unsigned>> &RemapChan) {
   unsigned NeededUndefs = 4 - RSI.UndefReg.size();
-  if (PreviousRegSeqByUndefCount[NeededUndefs].empty())
-    return false;
   std::vector<MachineInstr *> &MIs =
       PreviousRegSeqByUndefCount[NeededUndefs];
+  if (MIs.empty())
+    return false;
   CompatibleRSI = PreviousRegSeq[MIs.back()];
   tryMergeVector(&CompatibleRSI, &RSI, RemapChan);
   return true;
diff --git a/llvm/lib/Target/AMDGPU/R600TargetMachine.cpp b/llvm/lib/Target/AMDGPU/R600TargetMachine.cpp
index a9e5327d2c6c8..10552a1f0b1bc 100644
--- a/llvm/lib/Target/AMDGPU/R600TargetMachine.cpp
+++ b/llvm/lib/Target/AMDGPU/R600TargetMachine.cpp
@@ -90,17 +90,17 @@ R600TargetMachine::getTargetTransformInfo(const Function &F) const {
   return TargetTransformInfo(R600TTIImpl(this, F));
 }
 
+ScheduleDAGInstrs *
+R600TargetMachine::createMachineScheduler(MachineSchedContext *C) const {
+  return createR600MachineScheduler(C);
+}
+
 namespace {
 class R600PassConfig final : public AMDGPUPassConfig {
 public:
   R600PassConfig(TargetMachine &TM, PassManagerBase &PM)
       : AMDGPUPassConfig(TM, PM) {}
 
-  ScheduleDAGInstrs *
-  createMachineScheduler(MachineSchedContext *C) const override {
-    return createR600MachineScheduler(C);
-  }
-
   bool addPreISel() override;
   bool addInstSelector() override;
   void addPreRegAlloc() override;
diff --git a/llvm/lib/Target/AMDGPU/R600TargetMachine.h b/llvm/lib/Target/AMDGPU/R600TargetMachine.h
index b7f123a07a9c1..eb4cb91cb704d 100644
--- a/llvm/lib/Target/AMDGPU/R600TargetMachine.h
+++ b/llvm/lib/Target/AMDGPU/R600TargetMachine.h
@@ -53,6 +53,8 @@ class R600TargetMachine final : public AMDGPUTargetMachine {
   MachineFunctionInfo *
   createMachineFunctionInfo(BumpPtrAllocator &Allocator, const Function &F,
                             const TargetSubtargetInfo *STI) const override;
+  ScheduleDAGInstrs *
+  createMachineScheduler(MachineSchedContext *C) const override;
 };
 
 //===----------------------------------------------------------------------===//
diff --git a/llvm/lib/Target/AMDGPU/SIAnnotateControlFlow.cpp b/llvm/lib/Target/AMDGPU/SIAnnotateControlFlow.cpp
index 4ff6fc32b642d..df0c2080e0795 100644
--- a/llvm/lib/Target/AMDGPU/SIAnnotateControlFlow.cpp
+++ b/llvm/lib/Target/AMDGPU/SIAnnotateControlFlow.cpp
@@ -232,7 +232,7 @@ Value *SIAnnotateControlFlow::handleLoopCondition(
     } else if (L->contains(Inst)) {
       Insert = Term;
     } else {
-      Insert = L->getHeader()->getFirstNonPHIOrDbgOrLifetime();
+      Insert = &*L->getHeader()->getFirstNonPHIOrDbgOrLifetime();
     }
 
     return CreateBreak(Insert);
@@ -247,7 +247,7 @@ Value *SIAnnotateControlFlow::handleLoopCondition(
   }
 
   if (isa<Argument>(Cond)) {
-    Instruction *Insert = L->getHeader()->getFirstNonPHIOrDbgOrLifetime();
+    Instruction *Insert = &*L->getHeader()->getFirstNonPHIOrDbgOrLifetime();
     return CreateBreak(Insert);
   }
 
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
index 1aeca7f370aa1..b632c50dae0e3 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
@@ -419,8 +419,9 @@ SITargetLowering::SITargetLowering(const TargetMachine &TM,
   }
 
   setOperationAction(ISD::VECTOR_SHUFFLE,
-                     {MVT::v8i32, MVT::v8f32, MVT::v16i32, MVT::v16f32},
-                     Expand);
+                     {MVT::v4i32, MVT::v4f32, MVT::v8i32, MVT::v8f32,
+                      MVT::v16i32, MVT::v16f32, MVT::v32i32, MVT::v32f32},
+                     Custom);
 
   if (Subtarget->hasPkMovB32()) {
     // TODO: 16-bit element vectors should be legal with even aligned elements.
@@ -7589,15 +7590,38 @@ static bool elementPairIsContiguous(ArrayRef<int> Mask, int Elt) {
   return Mask[Elt + 1] == Mask[Elt] + 1 && (Mask[Elt] % 2 == 0);
 }
 
+static bool elementPairIsOddToEven(ArrayRef<int> Mask, int Elt) {
+  assert(Elt % 2 == 0);
+  return Mask[Elt] >= 0 && Mask[Elt + 1] >= 0 && (Mask[Elt] & 1) &&
+         !(Mask[Elt + 1] & 1);
+}
+
 SDValue SITargetLowering::lowerVECTOR_SHUFFLE(SDValue Op,
                                               SelectionDAG &DAG) const {
   SDLoc SL(Op);
   EVT ResultVT = Op.getValueType();
   ShuffleVectorSDNode *SVN = cast<ShuffleVectorSDNode>(Op);
   MVT EltVT = ResultVT.getVectorElementType().getSimpleVT();
-  MVT PackVT = MVT::getVectorVT(EltVT, 2);
+  const int NewSrcNumElts = 2;
+  MVT PackVT = MVT::getVectorVT(EltVT, NewSrcNumElts);
   int SrcNumElts = Op.getOperand(0).getValueType().getVectorNumElements();
 
+  // Break up the shuffle into registers sized pieces.
+  //
+  // We're trying to form sub-shuffles that the register allocation pipeline
+  // won't be able to figure out, like how to use v_pk_mov_b32 to do a register
+  // blend or 16-bit op_sel. It should be able to figure out how to reassemble a
+  // pair of copies into a consecutive register copy, so use the ordinary
+  // extract_vector_elt lowering unless we can use the shuffle.
+  //
+  // TODO: This is a bit of hack, and we should probably always use
+  // extract_subvector for the largest possible subvector we can (or at least
+  // use it for PackVT aligned pieces). However we have worse support for
+  // combines on them don't directly treat extract_subvector / insert_subvector
+  // as legal. The DAG scheduler also ends up doing a worse job with the
+  // extract_subvectors.
+  const bool ShouldUseConsecutiveExtract = EltVT.getSizeInBits() == 16;
+
   // vector_shuffle <0,1,6,7> lhs, rhs
   // -> concat_vectors (extract_subvector lhs, 0), (extract_subvector rhs, 2)
   //
@@ -7608,9 +7632,18 @@ SDValue SITargetLowering::lowerVECTOR_SHUFFLE(SDValue Op,
   // -> concat_vectors (extract_subvector rhs, 2), (extract_subvector lhs, 0)
 
   // Avoid scalarizing when both halves are reading from consecutive elements.
-  SmallVector<SDValue, 4> Pieces;
+
+  // If we're treating 2 element shuffles as legal, also create odd-to-even
+  // shuffles of neighboring pairs.
+  //
+  // vector_shuffle <3,2,7,6> lhs, rhs
+  //  -> concat_vectors vector_shuffle <1, 0> (extract_subvector lhs, 0)
+  //                    vector_shuffle <1, 0> (extract_subvector rhs, 2)
+
+  SmallVector<SDValue, 16> Pieces;
   for (int I = 0, N = ResultVT.getVectorNumElements(); I != N; I += 2) {
-    if (elementPairIsContiguous(SVN->getMask(), I)) {
+    if (ShouldUseConsecutiveExtract &&
+        elementPairIsContiguous(SVN->getMask(), I)) {
       const int Idx = SVN->getMaskElt(I);
       int VecIdx = Idx < SrcNumElts ? 0 : 1;
       int EltIdx = Idx < SrcNumElts ? Idx : Idx - SrcNumElts;
@@ -7618,6 +7651,48 @@ SDValue SITargetLowering::lowerVECTOR_SHUFFLE(SDValue Op,
                                    SVN->getOperand(VecIdx),
                                    DAG.getConstant(EltIdx, SL, MVT::i32));
       Pieces.push_back(SubVec);
+    } else if (elementPairIsOddToEven(SVN->getMask(), I) &&
+               isOperationLegal(ISD::VECTOR_SHUFFLE, PackVT)) {
+      int Idx0 = SVN->getMaskElt(I);
+      int Idx1 = SVN->getMaskElt(I + 1);
+
+      SDValue SrcOp0 = SVN->getOperand(0);
+      SDValue SrcOp1 = SrcOp0;
+      if (Idx0 >= SrcNumElts) {
+        SrcOp0 = SVN->getOperand(1);
+        Idx0 -= SrcNumElts;
+      }
+
+      if (Idx1 >= SrcNumElts) {
+        SrcOp1 = SVN->getOperand(1);
+        Idx1 -= SrcNumElts;
+      }
+
+      int AlignedIdx0 = Idx0 & ~(NewSrcNumElts - 1);
+      int AlignedIdx1 = Idx1 & ~(NewSrcNumElts - 1);
+
+      // Extract nearest even aligned piece.
+      SDValue SubVec0 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, SL, PackVT, SrcOp0,
+                                    DAG.getConstant(AlignedIdx0, SL, MVT::i32));
+      SDValue SubVec1 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, SL, PackVT, SrcOp1,
+                                    DAG.getConstant(AlignedIdx1, SL, MVT::i32));
+
+      int NewMaskIdx0 = Idx0 - AlignedIdx0;
+      int NewMaskIdx1 = Idx1 - AlignedIdx1;
+
+      SDValue Result0 = SubVec0;
+      SDValue Result1 = SubVec0;
+
+      if (SubVec0 != SubVec1) {
+        NewMaskIdx1 += NewSrcNumElts;
+        Result1 = SubVec1;
+      } else {
+        Result1 = DAG.getUNDEF(PackVT);
+      }
+
+      SDValue Shuf = DAG.getVectorShuffle(PackVT, SL, Result0, Result1,
+                                          {NewMaskIdx0, NewMaskIdx1});
+      Pieces.push_back(Shuf);
     } else {
       const int Idx0 = SVN->getMaskElt(I);
       const int Idx1 = SVN->getMaskElt(I + 1);
diff --git a/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp b/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp
index de2095fa60ffd..3d6419778f4b1 100644
--- a/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp
+++ b/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp
@@ -53,6 +53,11 @@ static cl::opt<bool>
                                "s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)"),
                       cl::init(false), cl::Hidden);
 
+static cl::opt<bool> ForceEmitZeroLoadFlag(
+    "amdgpu-waitcnt-load-forcezero",
+    cl::desc("Force all waitcnt load counters to wait until 0"),
+    cl::init(false), cl::Hidden);
+
 namespace {
 // Class of object that encapsulates latest instruction counter score
 // associated with the operand.  Used for determining whether
@@ -1850,6 +1855,9 @@ bool SIInsertWaitcnts::generateWaitcntInstBefore(MachineInstr &MI,
       Wait.BvhCnt = 0;
   }
 
+  if (ForceEmitZeroLoadFlag && Wait.LoadCnt != ~0u)
+    Wait.LoadCnt = 0;
+
   return generateWaitcnt(Wait, MI.getIterator(), *MI.getParent(), ScoreBrackets,
                          OldWaitcntInstr);
 }
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
index 1d98d68a2ea5d..35667801c809d 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
@@ -2366,11 +2366,11 @@ bool SIInstrInfo::expandPostRAPseudo(MachineInstr &MI) const {
     assert(ST.useVGPRIndexMode());
     Register VecReg = MI.getOperand(0).getReg();
     bool IsUndef = MI.getOperand(1).isUndef();
-    Register Idx = MI.getOperand(3).getReg();
+    MachineOperand &Idx = MI.getOperand(3);
     Register SubReg = MI.getOperand(4).getImm();
 
     MachineInstr *SetOn = BuildMI(MBB, MI, DL, get(AMDGPU::S_SET_GPR_IDX_ON))
-                              .addReg(Idx)
+                              .add(Idx)
                               .addImm(AMDGPU::VGPRIndexMode::DST_ENABLE);
     SetOn->getOperand(3).setIsUndef();
 
@@ -5573,8 +5573,12 @@ unsigned SIInstrInfo::getVALUOp(const MachineInstr &MI) const {
   case AMDGPU::S_SUB_F16: return AMDGPU::V_SUB_F16_fake16_e64;
   case AMDGPU::S_MIN_F16: return AMDGPU::V_MIN_F16_fake16_e64;
   case AMDGPU::S_MAX_F16: return AMDGPU::V_MAX_F16_fake16_e64;
-  case AMDGPU::S_MINIMUM_F16: return AMDGPU::V_MINIMUM_F16_e64;
-  case AMDGPU::S_MAXIMUM_F16: return AMDGPU::V_MAXIMUM_F16_e64;
+  case AMDGPU::S_MINIMUM_F16:
+    return ST.useRealTrue16Insts() ? AMDGPU::V_MINIMUM_F16_t16_e64
+                                   : AMDGPU::V_MINIMUM_F16_fake16_e64;
+  case AMDGPU::S_MAXIMUM_F16:
+    return ST.useRealTrue16Insts() ? AMDGPU::V_MAXIMUM_F16_t16_e64
+                                   : AMDGPU::V_MAXIMUM_F16_fake16_e64;
   case AMDGPU::S_MUL_F16: return AMDGPU::V_MUL_F16_fake16_e64;
   case AMDGPU::S_CVT_PK_RTZ_F16_F32: return AMDGPU::V_CVT_PKRTZ_F16_F32_e64;
   case AMDGPU::S_FMAC_F32: return AMDGPU::V_FMAC_F32_e64;
@@ -7547,9 +7551,7 @@ void SIInstrInfo::moveToVALUImpl(SIInstrWorklist &Worklist,
     return;
   }
   case AMDGPU::S_MINIMUM_F32:
-  case AMDGPU::S_MAXIMUM_F32:
-  case AMDGPU::S_MINIMUM_F16:
-  case AMDGPU::S_MAXIMUM_F16: {
+  case AMDGPU::S_MAXIMUM_F32: {
     const DebugLoc &DL = Inst.getDebugLoc();
     Register NewDst = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
     MachineInstr *NewInstr = BuildMI(*MBB, Inst, DL, get(NewOpcode), NewDst)
@@ -7566,6 +7568,26 @@ void SIInstrInfo::moveToVALUImpl(SIInstrWorklist &Worklist,
     Inst.eraseFromParent();
     return;
   }
+  case AMDGPU::S_MINIMUM_F16:
+  case AMDGPU::S_MAXIMUM_F16: {
+    const DebugLoc &DL = Inst.getDebugLoc();
+    Register NewDst = MRI.createVirtualRegister(ST.useRealTrue16Insts()
+                                                    ? &AMDGPU::VGPR_16RegClass
+                                                    : &AMDGPU::VGPR_32RegClass);
+    MachineInstr *NewInstr = BuildMI(*MBB, Inst, DL, get(NewOpcode), NewDst)
+                                 .addImm(0) // src0_modifiers
+                                 .add(Inst.getOperand(1))
+                                 .addImm(0) // src1_modifiers
+                                 .add(Inst.getOperand(2))
+                                 .addImm(0)  // clamp
+                                 .addImm(0)  // omod
+                                 .addImm(0); // opsel0
+    MRI.replaceRegWith(Inst.getOperand(0).getReg(), NewDst);
+    legalizeOperands(*NewInstr, MDT);
+    addUsersToMoveToVALUWorklist(NewDst, MRI, Worklist);
+    Inst.eraseFromParent();
+    return;
+  }
   }
 
   if (NewOpcode == AMDGPU::INSTRUCTION_LIST_END) {
diff --git a/llvm/lib/Target/AMDGPU/SIInstructions.td b/llvm/lib/Target/AMDGPU/SIInstructions.td
index 15c77c2a723e4..6e08aff24ec23 100644
--- a/llvm/lib/Target/AMDGPU/SIInstructions.td
+++ b/llvm/lib/Target/AMDGPU/SIInstructions.td
@@ -228,16 +228,39 @@ def S_INVERSE_BALLOT_U64 : SPseudoInstSI<
 // Pseudo instructions used for @llvm.fptrunc.round. The final codegen is done
 // in the ModeRegister pass.
 let Uses = [MODE, EXEC] in {
+let True16Predicate = NotHasTrue16BitInsts in
 def FPTRUNC_ROUND_F16_F32_PSEUDO : VPseudoInstSI <(outs VGPR_32:$vdst),
   (ins VGPR_32:$src0, i32imm:$round)>;
 
+let True16Predicate = UseFakeTrue16Insts in
+def FPTRUNC_ROUND_F16_F32_PSEUDO_fake16_e32 : VPseudoInstSI <(outs VGPR_32:$vdst),
+  (ins VGPR_32:$src0, i32imm:$round)>;
+
+let True16Predicate = UseRealTrue16Insts in
+// The operands of these pseudos should match V_CVT_F16_F32_t16_e64
+def FPTRUNC_ROUND_F16_F32_PSEUDO_t16_e64 : VPseudoInstSI <(outs VOPDstOperand_t16:$vdst),
+  (ins FP32InputMods:$src0_modifiers, VSrc_f32:$src0, Clamp0:$clamp, omod0:$omod, op_sel0:$op_sel, i32imm:$round)> {
+   let FPClamp = 1;
+   let ClampLo = 1;
+   let UseNamedOperandTable = 1;
+}
+
 def FPTRUNC_ROUND_F32_F64_PSEUDO : VPseudoInstSI <(outs VGPR_32:$vdst),
   (ins VReg_64:$src0, i32imm:$round)>;
 } // End Uses = [MODE, EXEC]
 
+let True16Predicate = NotHasTrue16BitInsts in
 def : GCNPat <(f16 (fptrunc_round f32:$src0, (i32 SupportedRoundMode:$round))),
      (FPTRUNC_ROUND_F16_F32_PSEUDO $src0, (as_hw_round_mode $round))>;
 
+let True16Predicate = UseFakeTrue16Insts in
+def : GCNPat <(f16 (fptrunc_round f32:$src0, (i32 SupportedRoundMode:$round))),
+     (FPTRUNC_ROUND_F16_F32_PSEUDO_fake16_e32 $src0, (as_hw_round_mode $round))>;
+
+let True16Predicate = UseRealTrue16Insts in
+def : GCNPat <(f16 (fptrunc_round (f32 (VOP3OpSelMods f32:$src0, i32:$src0_modifiers)), (i32 SupportedRoundMode:$round))),
+     (FPTRUNC_ROUND_F16_F32_PSEUDO_t16_e64 $src0_modifiers, $src0, (as_hw_round_mode $round))>;
+
 def : GCNPat <(f32 (fptrunc_round f64:$src0, (i32 SupportedRoundMode:$round))),
      (FPTRUNC_ROUND_F32_F64_PSEUDO $src0, (as_hw_round_mode $round))>;
 
@@ -1053,39 +1076,40 @@ def : GCNPat<
   (SI_ELSE $src, $target)
 >;
 
-def : Pat <
+def : GCNPat <
   (int_amdgcn_kill i1:$src),
   (SI_KILL_I1_PSEUDO SCSrc_i1:$src, 0)
 >;
 
-def : Pat <
+def : GCNPat <
   (int_amdgcn_kill (i1 (not i1:$src))),
   (SI_KILL_I1_PSEUDO SCSrc_i1:$src, -1)
 >;
 
-def : Pat <
+let SubtargetPredicate = NotHasSALUFloatInsts in
+def : GCNPat <
   (int_amdgcn_kill (i1 (setcc f32:$src, InlineImmFP32:$imm, cond:$cond))),
   (SI_KILL_F32_COND_IMM_PSEUDO VSrc_b32:$src, (bitcast_fpimm_to_i32 $imm), (cond_as_i32imm $cond))
 >;
 
-def : Pat <
+def : GCNPat <
   (int_amdgcn_wqm_demote i1:$src),
   (SI_DEMOTE_I1 SCSrc_i1:$src, 0)
 >;
 
-def : Pat <
+def : GCNPat <
   (int_amdgcn_wqm_demote (i1 (not i1:$src))),
   (SI_DEMOTE_I1 SCSrc_i1:$src, -1)
 >;
 
   // TODO: we could add more variants for other types of conditionals
 
-def : Pat <
+def : GCNPat <
   (i64 (int_amdgcn_icmp i1:$src, (i1 0), (i32 33))),
   (COPY $src) // Return the SGPRs representing i1 src
 >;
 
-def : Pat <
+def : GCNPat <
   (i32 (int_amdgcn_icmp i1:$src, (i1 0), (i32 33))),
   (COPY $src) // Return the SGPRs representing i1 src
 >;
@@ -2679,15 +2703,20 @@ class FPToI1Pat<Instruction Inst, int KOne, ValueType kone_type, ValueType vt, S
   (i1 (Inst 0, (kone_type KOne), $src0_modifiers, $src0, DSTCLAMP.NONE))
 >;
 
-let OtherPredicates = [NotHasTrue16BitInsts] in {
+let True16Predicate = NotHasTrue16BitInsts in {
   def : FPToI1Pat<V_CMP_EQ_F16_e64, CONST.FP16_ONE, i16, f16, fp_to_uint>;
   def : FPToI1Pat<V_CMP_EQ_F16_e64, CONST.FP16_NEG_ONE, i16, f16, fp_to_sint>;
-} // end OtherPredicates = [NotHasTrue16BitInsts]
+} // end True16Predicate = NotHasTrue16BitInsts
+
+let True16Predicate = UseRealTrue16Insts in {
+  def : FPToI1Pat<V_CMP_EQ_F16_t16_e64, CONST.FP16_ONE, i16, f16, fp_to_uint>;
+  def : FPToI1Pat<V_CMP_EQ_F16_t16_e64, CONST.FP16_NEG_ONE, i16, f16, fp_to_sint>;
+} // end True16Predicate = UseRealTrue16BitInsts
 
-let OtherPredicates = [HasTrue16BitInsts] in {
+let True16Predicate = UseFakeTrue16Insts in {
   def : FPToI1Pat<V_CMP_EQ_F16_fake16_e64, CONST.FP16_ONE, i16, f16, fp_to_uint>;
   def : FPToI1Pat<V_CMP_EQ_F16_fake16_e64, CONST.FP16_NEG_ONE, i16, f16, fp_to_sint>;
-} // end OtherPredicates = [HasTrue16BitInsts]
+} // end True16Predicate = UseFakeTrue16BitInsts
 
 def : FPToI1Pat<V_CMP_EQ_F32_e64, CONST.FP32_ONE, i32, f32, fp_to_uint>;
 def : FPToI1Pat<V_CMP_EQ_F32_e64, CONST.FP32_NEG_ONE, i32, f32, fp_to_sint>;
@@ -3766,6 +3795,13 @@ def : FPMinCanonMaxPat<V_MINMAX_F32_e64, f32, fmaxnum_like, fminnum_like_oneuse>
 def : FPMinCanonMaxPat<V_MAXMIN_F32_e64, f32, fminnum_like, fmaxnum_like_oneuse>;
 }
 
+let True16Predicate = UseRealTrue16Insts in {
+def : FPMinMaxPat<V_MINMAX_F16_t16_e64, f16, fmaxnum_like, fminnum_like_oneuse>;
+def : FPMinMaxPat<V_MAXMIN_F16_t16_e64, f16, fminnum_like, fmaxnum_like_oneuse>;
+def : FPMinCanonMaxPat<V_MINMAX_F16_t16_e64, f16, fmaxnum_like, fminnum_like_oneuse>;
+def : FPMinCanonMaxPat<V_MAXMIN_F16_t16_e64, f16, fminnum_like, fmaxnum_like_oneuse>;
+}
+
 let True16Predicate = UseFakeTrue16Insts in {
 def : FPMinMaxPat<V_MINMAX_F16_fake16_e64, f16, fmaxnum_like, fminnum_like_oneuse>;
 def : FPMinMaxPat<V_MAXMIN_F16_fake16_e64, f16, fminnum_like, fmaxnum_like_oneuse>;
@@ -3788,15 +3824,25 @@ let True16Predicate = UseFakeTrue16Insts in {
 }
 } // End SubtargetPredicate = [isGFX9Plus]
 
-let OtherPredicates = [isGFX12Plus] in {
+let SubtargetPredicate = isGFX12Plus in {
 def : FPMinMaxPat<V_MINIMUMMAXIMUM_F32_e64, f32, DivergentBinFrag<fmaximum>, fminimum_oneuse>;
 def : FPMinMaxPat<V_MAXIMUMMINIMUM_F32_e64, f32, DivergentBinFrag<fminimum>, fmaximum_oneuse>;
-def : FPMinMaxPat<V_MINIMUMMAXIMUM_F16_e64, f16, DivergentBinFrag<fmaximum>, fminimum_oneuse>;
-def : FPMinMaxPat<V_MAXIMUMMINIMUM_F16_e64, f16, DivergentBinFrag<fminimum>, fmaximum_oneuse>;
 def : FPMinCanonMaxPat<V_MINIMUMMAXIMUM_F32_e64, f32, DivergentBinFrag<fmaximum>, fminimum_oneuse>;
 def : FPMinCanonMaxPat<V_MAXIMUMMINIMUM_F32_e64, f32, DivergentBinFrag<fminimum>, fmaximum_oneuse>;
-def : FPMinCanonMaxPat<V_MINIMUMMAXIMUM_F16_e64, f16, DivergentBinFrag<fmaximum>, fminimum_oneuse>;
-def : FPMinCanonMaxPat<V_MAXIMUMMINIMUM_F16_e64, f16, DivergentBinFrag<fminimum>, fmaximum_oneuse>;
+}
+
+let True16Predicate = UseRealTrue16Insts, SubtargetPredicate = isGFX12Plus in {
+def : FPMinMaxPat<V_MINIMUMMAXIMUM_F16_t16_e64, f16, DivergentBinFrag<fmaximum>, fminimum_oneuse>;
+def : FPMinMaxPat<V_MAXIMUMMINIMUM_F16_t16_e64, f16, DivergentBinFrag<fminimum>, fmaximum_oneuse>;
+def : FPMinCanonMaxPat<V_MINIMUMMAXIMUM_F16_t16_e64, f16, DivergentBinFrag<fmaximum>, fminimum_oneuse>;
+def : FPMinCanonMaxPat<V_MAXIMUMMINIMUM_F16_t16_e64, f16, DivergentBinFrag<fminimum>, fmaximum_oneuse>;
+}
+
+let True16Predicate = UseFakeTrue16Insts, SubtargetPredicate = isGFX12Plus in {
+def : FPMinMaxPat<V_MINIMUMMAXIMUM_F16_fake16_e64, f16, DivergentBinFrag<fmaximum>, fminimum_oneuse>;
+def : FPMinMaxPat<V_MAXIMUMMINIMUM_F16_fake16_e64, f16, DivergentBinFrag<fminimum>, fmaximum_oneuse>;
+def : FPMinCanonMaxPat<V_MINIMUMMAXIMUM_F16_fake16_e64, f16, DivergentBinFrag<fmaximum>, fminimum_oneuse>;
+def : FPMinCanonMaxPat<V_MAXIMUMMINIMUM_F16_fake16_e64, f16, DivergentBinFrag<fminimum>, fmaximum_oneuse>;
 }
 
 // Convert a floating-point power of 2 to the integer exponent.
diff --git a/llvm/lib/Target/AMDGPU/SILoadStoreOptimizer.cpp b/llvm/lib/Target/AMDGPU/SILoadStoreOptimizer.cpp
index 904321a344db1..708acc9f88445 100644
--- a/llvm/lib/Target/AMDGPU/SILoadStoreOptimizer.cpp
+++ b/llvm/lib/Target/AMDGPU/SILoadStoreOptimizer.cpp
@@ -2174,12 +2174,13 @@ bool SILoadStoreOptimizer::promoteConstantOffsetToImm(
 
   // Step1: Find the base-registers and a 64bit constant offset.
   MachineOperand &Base = *TII->getNamedOperand(MI, AMDGPU::OpName::vaddr);
+  auto [It, Inserted] = Visited.try_emplace(&MI);
   MemAddress MAddr;
-  if (!Visited.contains(&MI)) {
+  if (Inserted) {
     processBaseWithConstOffset(Base, MAddr);
-    Visited[&MI] = MAddr;
+    It->second = MAddr;
   } else
-    MAddr = Visited[&MI];
+    MAddr = It->second;
 
   if (MAddr.Offset == 0) {
     LLVM_DEBUG(dbgs() << "  Failed to extract constant-offset or there are no"
@@ -2239,11 +2240,12 @@ bool SILoadStoreOptimizer::promoteConstantOffsetToImm(
     const MachineOperand &BaseNext =
       *TII->getNamedOperand(MINext, AMDGPU::OpName::vaddr);
     MemAddress MAddrNext;
-    if (!Visited.contains(&MINext)) {
+    auto [It, Inserted] = Visited.try_emplace(&MINext);
+    if (Inserted) {
       processBaseWithConstOffset(BaseNext, MAddrNext);
-      Visited[&MINext] = MAddrNext;
+      It->second = MAddrNext;
     } else
-      MAddrNext = Visited[&MINext];
+      MAddrNext = It->second;
 
     if (MAddrNext.Base.LoReg != MAddr.Base.LoReg ||
         MAddrNext.Base.HiReg != MAddr.Base.HiReg ||
diff --git a/llvm/lib/Target/AMDGPU/SIMachineScheduler.cpp b/llvm/lib/Target/AMDGPU/SIMachineScheduler.cpp
index 77b4f25021c75..b3fa65512e4c4 100644
--- a/llvm/lib/Target/AMDGPU/SIMachineScheduler.cpp
+++ b/llvm/lib/Target/AMDGPU/SIMachineScheduler.cpp
@@ -284,10 +284,9 @@ void SIScheduleBlock::fastSchedule() {
 }
 
 // Returns if the register was set between first and last.
-static bool isDefBetween(unsigned Reg,
-                           SlotIndex First, SlotIndex Last,
-                           const MachineRegisterInfo *MRI,
-                           const LiveIntervals *LIS) {
+static bool isDefBetween(Register Reg, SlotIndex First, SlotIndex Last,
+                         const MachineRegisterInfo *MRI,
+                         const LiveIntervals *LIS) {
   for (MachineRegisterInfo::def_instr_iterator
        UI = MRI->def_instr_begin(Reg),
        UE = MRI->def_instr_end(); UI != UE; ++UI) {
@@ -581,11 +580,11 @@ void SIScheduleBlock::printDebug(bool full) {
            << LiveOutPressure[AMDGPU::RegisterPressureSets::SReg_32] << ' '
            << LiveOutPressure[AMDGPU::RegisterPressureSets::VGPR_32] << "\n\n";
     dbgs() << "LiveIns:\n";
-    for (unsigned Reg : LiveInRegs)
+    for (Register Reg : LiveInRegs)
       dbgs() << printVRegOrUnit(Reg, DAG->getTRI()) << ' ';
 
     dbgs() << "\nLiveOuts:\n";
-    for (unsigned Reg : LiveOutRegs)
+    for (Register Reg : LiveOutRegs)
       dbgs() << printVRegOrUnit(Reg, DAG->getTRI()) << ' ';
   }
 
@@ -1413,12 +1412,12 @@ SIScheduleBlockScheduler::SIScheduleBlockScheduler(SIScheduleDAGMI *DAG,
   // highest topological index.
   LiveOutRegsNumUsages.resize(Blocks.size());
   for (SIScheduleBlock *Block : Blocks) {
-    for (unsigned Reg : Block->getInRegs()) {
+    for (Register Reg : Block->getInRegs()) {
       bool Found = false;
       int topoInd = -1;
       for (SIScheduleBlock* Pred: Block->getPreds()) {
-        std::set<unsigned> PredOutRegs = Pred->getOutRegs();
-        std::set<unsigned>::iterator RegPos = PredOutRegs.find(Reg);
+        std::set<Register> PredOutRegs = Pred->getOutRegs();
+        std::set<Register>::iterator RegPos = PredOutRegs.find(Reg);
 
         if (RegPos != PredOutRegs.end()) {
           Found = true;
@@ -1453,18 +1452,18 @@ SIScheduleBlockScheduler::SIScheduleBlockScheduler(SIScheduleDAGMI *DAG,
   }
 #endif
 
-  std::set<unsigned> InRegs = DAG->getInRegs();
+  std::set<Register> InRegs = DAG->getInRegs();
   addLiveRegs(InRegs);
 
   // Increase LiveOutRegsNumUsages for blocks
   // producing registers consumed in another
   // scheduling region.
-  for (unsigned Reg : DAG->getOutRegs()) {
+  for (Register Reg : DAG->getOutRegs()) {
     for (unsigned i = 0, e = Blocks.size(); i != e; ++i) {
       // Do reverse traversal
       int ID = BlocksStruct.TopDownIndex2Block[Blocks.size()-1-i];
       SIScheduleBlock *Block = Blocks[ID];
-      const std::set<unsigned> &OutRegs = Block->getOutRegs();
+      const std::set<Register> &OutRegs = Block->getOutRegs();
 
       if (OutRegs.find(Reg) == OutRegs.end())
         continue;
@@ -1477,11 +1476,11 @@ SIScheduleBlockScheduler::SIScheduleBlockScheduler(SIScheduleDAGMI *DAG,
   // Fill LiveRegsConsumers for regs that were already
   // defined before scheduling.
   for (SIScheduleBlock *Block : Blocks) {
-    for (unsigned Reg : Block->getInRegs()) {
+    for (Register Reg : Block->getInRegs()) {
       bool Found = false;
       for (SIScheduleBlock* Pred: Block->getPreds()) {
-        std::set<unsigned> PredOutRegs = Pred->getOutRegs();
-        std::set<unsigned>::iterator RegPos = PredOutRegs.find(Reg);
+        std::set<Register> PredOutRegs = Pred->getOutRegs();
+        std::set<Register>::iterator RegPos = PredOutRegs.find(Reg);
 
         if (RegPos != PredOutRegs.end()) {
           Found = true;
@@ -1573,13 +1572,11 @@ SIScheduleBlock *SIScheduleBlockScheduler::pickBlock() {
   if (SregCurrentUsage > maxSregUsage)
     maxSregUsage = SregCurrentUsage;
   LLVM_DEBUG(dbgs() << "Picking New Blocks\n"; dbgs() << "Available: ";
-             for (SIScheduleBlock *Block
-                  : ReadyBlocks) dbgs()
-             << Block->getID() << ' ';
+             for (SIScheduleBlock *Block : ReadyBlocks)
+               dbgs() << Block->getID() << ' ';
              dbgs() << "\nCurrent Live:\n";
-             for (unsigned Reg
-                  : LiveRegs) dbgs()
-             << printVRegOrUnit(Reg, DAG->getTRI()) << ' ';
+             for (Register Reg : LiveRegs)
+               dbgs() << printVRegOrUnit(Reg, DAG->getTRI()) << ' ';
              dbgs() << '\n';
              dbgs() << "Current VGPRs: " << VregCurrentUsage << '\n';
              dbgs() << "Current SGPRs: " << SregCurrentUsage << '\n';);
@@ -1634,7 +1631,7 @@ SIScheduleBlock *SIScheduleBlockScheduler::pickBlock() {
 
 // Tracking of currently alive registers to determine VGPR Usage.
 
-void SIScheduleBlockScheduler::addLiveRegs(std::set<unsigned> &Regs) {
+void SIScheduleBlockScheduler::addLiveRegs(std::set<Register> &Regs) {
   for (Register Reg : Regs) {
     // For now only track virtual registers.
     if (!Reg.isVirtual())
@@ -1645,10 +1642,10 @@ void SIScheduleBlockScheduler::addLiveRegs(std::set<unsigned> &Regs) {
 }
 
 void SIScheduleBlockScheduler::decreaseLiveRegs(SIScheduleBlock *Block,
-                                       std::set<unsigned> &Regs) {
-  for (unsigned Reg : Regs) {
+                                                std::set<Register> &Regs) {
+  for (Register Reg : Regs) {
     // For now only track virtual registers.
-    std::set<unsigned>::iterator Pos = LiveRegs.find(Reg);
+    std::set<Register>::iterator Pos = LiveRegs.find(Reg);
     assert (Pos != LiveRegs.end() && // Reg must be live.
                LiveRegsConsumers.find(Reg) != LiveRegsConsumers.end() &&
                LiveRegsConsumers[Reg] >= 1);
@@ -1687,8 +1684,8 @@ void SIScheduleBlockScheduler::blockScheduled(SIScheduleBlock *Block) {
 }
 
 std::vector<int>
-SIScheduleBlockScheduler::checkRegUsageImpact(std::set<unsigned> &InRegs,
-                                     std::set<unsigned> &OutRegs) {
+SIScheduleBlockScheduler::checkRegUsageImpact(std::set<Register> &InRegs,
+                                              std::set<Register> &OutRegs) {
   std::vector<int> DiffSetPressure;
   DiffSetPressure.assign(DAG->getTRI()->getNumRegPressureSets(), 0);
 
diff --git a/llvm/lib/Target/AMDGPU/SIMachineScheduler.h b/llvm/lib/Target/AMDGPU/SIMachineScheduler.h
index f8f4b5aae338e..b219cbd5672f0 100644
--- a/llvm/lib/Target/AMDGPU/SIMachineScheduler.h
+++ b/llvm/lib/Target/AMDGPU/SIMachineScheduler.h
@@ -81,8 +81,8 @@ class SIScheduleBlock {
   // Note that some registers are not 32 bits,
   // and thus the pressure is not equal
   // to the number of live registers.
-  std::set<unsigned> LiveInRegs;
-  std::set<unsigned> LiveOutRegs;
+  std::set<Register> LiveInRegs;
+  std::set<Register> LiveOutRegs;
 
   bool Scheduled = false;
   bool HighLatencyBlock = false;
@@ -157,8 +157,8 @@ class SIScheduleBlock {
     return InternalAdditionalPressure;
   }
 
-  std::set<unsigned> &getInRegs() { return LiveInRegs; }
-  std::set<unsigned> &getOutRegs() { return LiveOutRegs; }
+  std::set<Register> &getInRegs() { return LiveInRegs; }
+  std::set<Register> &getOutRegs() { return LiveOutRegs; }
 
   void printDebug(bool Full);
 
@@ -320,10 +320,10 @@ class SIScheduleBlockScheduler {
   SISchedulerBlockSchedulerVariant Variant;
   std::vector<SIScheduleBlock*> Blocks;
 
-  std::vector<std::map<unsigned, unsigned>> LiveOutRegsNumUsages;
-  std::set<unsigned> LiveRegs;
+  std::vector<std::map<Register, unsigned>> LiveOutRegsNumUsages;
+  std::set<Register> LiveRegs;
   // Num of schedulable unscheduled blocks reading the register.
-  std::map<unsigned, unsigned> LiveRegsConsumers;
+  std::map<Register, unsigned> LiveRegsConsumers;
 
   std::vector<unsigned> LastPosHighLatencyParentScheduled;
   int LastPosWaitedHighLatency;
@@ -389,15 +389,15 @@ class SIScheduleBlockScheduler {
                             SIBlockSchedCandidate &TryCand);
   SIScheduleBlock *pickBlock();
 
-  void addLiveRegs(std::set<unsigned> &Regs);
-  void decreaseLiveRegs(SIScheduleBlock *Block, std::set<unsigned> &Regs);
+  void addLiveRegs(std::set<Register> &Regs);
+  void decreaseLiveRegs(SIScheduleBlock *Block, std::set<Register> &Regs);
   void releaseBlockSuccs(SIScheduleBlock *Parent);
   void blockScheduled(SIScheduleBlock *Block);
 
   // Check register pressure change
   // by scheduling a block with these LiveIn and LiveOut.
-  std::vector<int> checkRegUsageImpact(std::set<unsigned> &InRegs,
-                                       std::set<unsigned> &OutRegs);
+  std::vector<int> checkRegUsageImpact(std::set<Register> &InRegs,
+                                       std::set<Register> &OutRegs);
 
   void schedule();
 };
@@ -462,8 +462,8 @@ class SIScheduleDAGMI final : public ScheduleDAGMILive {
                                                      unsigned &VgprUsage,
                                                      unsigned &SgprUsage);
 
-  std::set<unsigned> getInRegs() {
-    std::set<unsigned> InRegs;
+  std::set<Register> getInRegs() {
+    std::set<Register> InRegs;
     for (const auto &RegMaskPair : RPTracker.getPressure().LiveInRegs) {
       InRegs.insert(RegMaskPair.RegUnit);
     }
diff --git a/llvm/lib/Target/AMDGPU/SIModeRegister.cpp b/llvm/lib/Target/AMDGPU/SIModeRegister.cpp
index 412e2f2fe45d1..99aea52c184d6 100644
--- a/llvm/lib/Target/AMDGPU/SIModeRegister.cpp
+++ b/llvm/lib/Target/AMDGPU/SIModeRegister.cpp
@@ -166,6 +166,8 @@ Status SIModeRegister::getInstructionMode(MachineInstr &MI,
   unsigned Opcode = MI.getOpcode();
   if (TII->usesFPDPRounding(MI) ||
       Opcode == AMDGPU::FPTRUNC_ROUND_F16_F32_PSEUDO ||
+      Opcode == AMDGPU::FPTRUNC_ROUND_F16_F32_PSEUDO_fake16_e32 ||
+      Opcode == AMDGPU::FPTRUNC_ROUND_F16_F32_PSEUDO_t16_e64 ||
       Opcode == AMDGPU::FPTRUNC_ROUND_F32_F64_PSEUDO) {
     switch (Opcode) {
     case AMDGPU::V_INTERP_P1LL_F16:
@@ -177,19 +179,19 @@ Status SIModeRegister::getInstructionMode(MachineInstr &MI,
     case AMDGPU::FPTRUNC_ROUND_F16_F32_PSEUDO: {
       unsigned Mode = MI.getOperand(2).getImm();
       MI.removeOperand(2);
-      // Replacing the pseudo by a real instruction in place
-      if (TII->getSubtarget().hasTrue16BitInsts()) {
-        MachineBasicBlock &MBB = *MI.getParent();
-        MachineInstrBuilder B(*MBB.getParent(), MI);
-        MI.setDesc(TII->get(AMDGPU::V_CVT_F16_F32_fake16_e64));
-        MachineOperand Src0 = MI.getOperand(1);
-        MI.removeOperand(1);
-        B.addImm(0); // src0_modifiers
-        B.add(Src0); // re-add src0 operand
-        B.addImm(0); // clamp
-        B.addImm(0); // omod
-      } else
-        MI.setDesc(TII->get(AMDGPU::V_CVT_F16_F32_e32));
+      MI.setDesc(TII->get(AMDGPU::V_CVT_F16_F32_e32));
+      return Status(FP_ROUND_MODE_DP(3), FP_ROUND_MODE_DP(Mode));
+    }
+    case AMDGPU::FPTRUNC_ROUND_F16_F32_PSEUDO_fake16_e32: {
+      unsigned Mode = MI.getOperand(2).getImm();
+      MI.removeOperand(2);
+      MI.setDesc(TII->get(AMDGPU::V_CVT_F16_F32_fake16_e32));
+      return Status(FP_ROUND_MODE_DP(3), FP_ROUND_MODE_DP(Mode));
+    }
+    case AMDGPU::FPTRUNC_ROUND_F16_F32_PSEUDO_t16_e64: {
+      unsigned Mode = MI.getOperand(6).getImm();
+      MI.removeOperand(6);
+      MI.setDesc(TII->get(AMDGPU::V_CVT_F16_F32_t16_e64));
       return Status(FP_ROUND_MODE_DP(3), FP_ROUND_MODE_DP(Mode));
     }
     case AMDGPU::FPTRUNC_ROUND_F32_F64_PSEUDO: {
diff --git a/llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp b/llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp
index 6fc57dec6a826..71c720ed09b5f 100644
--- a/llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp
@@ -3516,30 +3516,6 @@ bool SIRegisterInfo::opCanUseInlineConstant(unsigned OpType) const {
          OpType <= AMDGPU::OPERAND_SRC_LAST;
 }
 
-bool SIRegisterInfo::shouldRewriteCopySrc(
-  const TargetRegisterClass *DefRC,
-  unsigned DefSubReg,
-  const TargetRegisterClass *SrcRC,
-  unsigned SrcSubReg) const {
-  // We want to prefer the smallest register class possible, so we don't want to
-  // stop and rewrite on anything that looks like a subregister
-  // extract. Operations mostly don't care about the super register class, so we
-  // only want to stop on the most basic of copies between the same register
-  // class.
-  //
-  // e.g. if we have something like
-  // %0 = ...
-  // %1 = ...
-  // %2 = REG_SEQUENCE %0, sub0, %1, sub1, %2, sub2
-  // %3 = COPY %2, sub0
-  //
-  // We want to look through the COPY to find:
-  //  => %3 = COPY %0
-
-  // Plain copy.
-  return getCommonSubClass(DefRC, SrcRC) != nullptr;
-}
-
 bool SIRegisterInfo::opCanUseLiteralConstant(unsigned OpType) const {
   // TODO: 64-bit operands have extending behavior from 32-bit literal.
   return OpType >= AMDGPU::OPERAND_REG_IMM_FIRST &&
diff --git a/llvm/lib/Target/AMDGPU/SIRegisterInfo.h b/llvm/lib/Target/AMDGPU/SIRegisterInfo.h
index 8e481e3ac2304..a434efb70d052 100644
--- a/llvm/lib/Target/AMDGPU/SIRegisterInfo.h
+++ b/llvm/lib/Target/AMDGPU/SIRegisterInfo.h
@@ -275,11 +275,6 @@ class SIRegisterInfo final : public AMDGPUGenRegisterInfo {
                            const TargetRegisterClass *SubRC,
                            unsigned SubIdx) const;
 
-  bool shouldRewriteCopySrc(const TargetRegisterClass *DefRC,
-                            unsigned DefSubReg,
-                            const TargetRegisterClass *SrcRC,
-                            unsigned SrcSubReg) const override;
-
   /// \returns True if operands defined with this operand type can accept
   /// a literal constant (i.e. any 32-bit immediate).
   bool opCanUseLiteralConstant(unsigned OpType) const;
diff --git a/llvm/lib/Target/AMDGPU/SIWholeQuadMode.cpp b/llvm/lib/Target/AMDGPU/SIWholeQuadMode.cpp
index 38f86ca3e9afc..3293602db0901 100644
--- a/llvm/lib/Target/AMDGPU/SIWholeQuadMode.cpp
+++ b/llvm/lib/Target/AMDGPU/SIWholeQuadMode.cpp
@@ -67,6 +67,7 @@
 ///
 //===----------------------------------------------------------------------===//
 
+#include "SIWholeQuadMode.h"
 #include "AMDGPU.h"
 #include "GCNSubtarget.h"
 #include "MCTargetDesc/AMDGPUMCTargetDesc.h"
@@ -148,11 +149,19 @@ struct WorkItem {
   WorkItem(MachineInstr *MI) : MI(MI) {}
 };
 
-class SIWholeQuadMode : public MachineFunctionPass {
+class SIWholeQuadMode {
+public:
+  SIWholeQuadMode(MachineFunction &MF, LiveIntervals *LIS,
+                  MachineDominatorTree *MDT, MachinePostDominatorTree *PDT)
+      : ST(&MF.getSubtarget<GCNSubtarget>()), TII(ST->getInstrInfo()),
+        TRI(&TII->getRegisterInfo()), MRI(&MF.getRegInfo()), LIS(LIS), MDT(MDT),
+        PDT(PDT) {}
+  bool run(MachineFunction &MF);
+
 private:
+  const GCNSubtarget *ST;
   const SIInstrInfo *TII;
   const SIRegisterInfo *TRI;
-  const GCNSubtarget *ST;
   MachineRegisterInfo *MRI;
   LiveIntervals *LIS;
   MachineDominatorTree *MDT;
@@ -212,14 +221,12 @@ class SIWholeQuadMode : public MachineFunctionPass {
                       MachineBasicBlock::iterator Before, Register SavedOrig,
                       char NonStrictState, char CurrentStrictState);
 
-  MachineBasicBlock *splitBlock(MachineBasicBlock *BB, MachineInstr *TermMI);
+  void splitBlock(MachineInstr *TermMI);
+  MachineInstr *lowerKillI1(MachineInstr &MI, bool IsWQM);
+  MachineInstr *lowerKillF32(MachineInstr &MI);
 
-  MachineInstr *lowerKillI1(MachineBasicBlock &MBB, MachineInstr &MI,
-                            bool IsWQM);
-  MachineInstr *lowerKillF32(MachineBasicBlock &MBB, MachineInstr &MI);
-
-  void lowerBlock(MachineBasicBlock &MBB);
-  void processBlock(MachineBasicBlock &MBB, bool IsEntry);
+  void lowerBlock(MachineBasicBlock &MBB, BlockInfo &BI);
+  void processBlock(MachineBasicBlock &MBB, BlockInfo &BI, bool IsEntry);
 
   bool lowerLiveMaskQueries();
   bool lowerCopyInstrs();
@@ -227,12 +234,13 @@ class SIWholeQuadMode : public MachineFunctionPass {
   void lowerInitExec(MachineInstr &MI);
   MachineBasicBlock::iterator lowerInitExecInstrs(MachineBasicBlock &Entry,
                                                   bool &Changed);
+};
 
+class SIWholeQuadModeLegacy : public MachineFunctionPass {
 public:
   static char ID;
 
-  SIWholeQuadMode() :
-    MachineFunctionPass(ID) { }
+  SIWholeQuadModeLegacy() : MachineFunctionPass(ID) {}
 
   bool runOnMachineFunction(MachineFunction &MF) override;
 
@@ -252,23 +260,22 @@ class SIWholeQuadMode : public MachineFunctionPass {
         MachineFunctionProperties::Property::IsSSA);
   }
 };
-
 } // end anonymous namespace
 
-char SIWholeQuadMode::ID = 0;
+char SIWholeQuadModeLegacy::ID = 0;
 
-INITIALIZE_PASS_BEGIN(SIWholeQuadMode, DEBUG_TYPE, "SI Whole Quad Mode", false,
-                      false)
+INITIALIZE_PASS_BEGIN(SIWholeQuadModeLegacy, DEBUG_TYPE, "SI Whole Quad Mode",
+                      false, false)
 INITIALIZE_PASS_DEPENDENCY(LiveIntervalsWrapperPass)
 INITIALIZE_PASS_DEPENDENCY(MachineDominatorTreeWrapperPass)
 INITIALIZE_PASS_DEPENDENCY(MachinePostDominatorTreeWrapperPass)
-INITIALIZE_PASS_END(SIWholeQuadMode, DEBUG_TYPE, "SI Whole Quad Mode", false,
-                    false)
+INITIALIZE_PASS_END(SIWholeQuadModeLegacy, DEBUG_TYPE, "SI Whole Quad Mode",
+                    false, false)
 
-char &llvm::SIWholeQuadModeID = SIWholeQuadMode::ID;
+char &llvm::SIWholeQuadModeID = SIWholeQuadModeLegacy::ID;
 
-FunctionPass *llvm::createSIWholeQuadModePass() {
-  return new SIWholeQuadMode;
+FunctionPass *llvm::createSIWholeQuadModeLegacyPass() {
+  return new SIWholeQuadModeLegacy;
 }
 
 #ifndef NDEBUG
@@ -746,8 +753,8 @@ SIWholeQuadMode::saveSCC(MachineBasicBlock &MBB,
   return Restore;
 }
 
-MachineBasicBlock *SIWholeQuadMode::splitBlock(MachineBasicBlock *BB,
-                                               MachineInstr *TermMI) {
+void SIWholeQuadMode::splitBlock(MachineInstr *TermMI) {
+  MachineBasicBlock *BB = TermMI->getParent();
   LLVM_DEBUG(dbgs() << "Split block " << printMBBReference(*BB) << " @ "
                     << *TermMI << "\n");
 
@@ -796,12 +803,9 @@ MachineBasicBlock *SIWholeQuadMode::splitBlock(MachineBasicBlock *BB,
             .addMBB(SplitBB);
     LIS->InsertMachineInstrInMaps(*MI);
   }
-
-  return SplitBB;
 }
 
-MachineInstr *SIWholeQuadMode::lowerKillF32(MachineBasicBlock &MBB,
-                                            MachineInstr &MI) {
+MachineInstr *SIWholeQuadMode::lowerKillF32(MachineInstr &MI) {
   assert(LiveMaskReg.isVirtual());
 
   const DebugLoc &DL = MI.getDebugLoc();
@@ -869,6 +873,8 @@ MachineInstr *SIWholeQuadMode::lowerKillF32(MachineBasicBlock &MBB,
     llvm_unreachable("invalid ISD:SET cond code");
   }
 
+  MachineBasicBlock &MBB = *MI.getParent();
+
   // Pick opcode based on comparison type.
   MachineInstr *VcmpMI;
   const MachineOperand &Op0 = MI.getOperand(0);
@@ -919,10 +925,11 @@ MachineInstr *SIWholeQuadMode::lowerKillF32(MachineBasicBlock &MBB,
   return NewTerm;
 }
 
-MachineInstr *SIWholeQuadMode::lowerKillI1(MachineBasicBlock &MBB,
-                                           MachineInstr &MI, bool IsWQM) {
+MachineInstr *SIWholeQuadMode::lowerKillI1(MachineInstr &MI, bool IsWQM) {
   assert(LiveMaskReg.isVirtual());
 
+  MachineBasicBlock &MBB = *MI.getParent();
+
   const DebugLoc &DL = MI.getDebugLoc();
   MachineInstr *MaskUpdateMI = nullptr;
 
@@ -1035,12 +1042,7 @@ MachineInstr *SIWholeQuadMode::lowerKillI1(MachineBasicBlock &MBB,
 // Replace (or supplement) instructions accessing live mask.
 // This can only happen once all the live mask registers have been created
 // and the execute state (WQM/StrictWWM/Exact) of instructions is known.
-void SIWholeQuadMode::lowerBlock(MachineBasicBlock &MBB) {
-  auto *BII = Blocks.find(&MBB);
-  if (BII == Blocks.end())
-    return;
-
-  const BlockInfo &BI = BII->second;
+void SIWholeQuadMode::lowerBlock(MachineBasicBlock &MBB, BlockInfo &BI) {
   if (!BI.NeedsLowering)
     return;
 
@@ -1052,17 +1054,18 @@ void SIWholeQuadMode::lowerBlock(MachineBasicBlock &MBB) {
 
   for (MachineInstr &MI : llvm::make_early_inc_range(
            llvm::make_range(MBB.getFirstNonPHI(), MBB.end()))) {
-    if (StateTransition.count(&MI))
-      State = StateTransition[&MI];
+    auto MIState = StateTransition.find(&MI);
+    if (MIState != StateTransition.end())
+      State = MIState->second;
 
     MachineInstr *SplitPoint = nullptr;
     switch (MI.getOpcode()) {
     case AMDGPU::SI_DEMOTE_I1:
     case AMDGPU::SI_KILL_I1_TERMINATOR:
-      SplitPoint = lowerKillI1(MBB, MI, State == StateWQM);
+      SplitPoint = lowerKillI1(MI, State == StateWQM);
       break;
     case AMDGPU::SI_KILL_F32_COND_IMM_TERMINATOR:
-      SplitPoint = lowerKillF32(MBB, MI);
+      SplitPoint = lowerKillF32(MI);
       break;
     case AMDGPU::ENTER_STRICT_WWM:
       ActiveLanesReg = MI.getOperand(0).getReg();
@@ -1088,12 +1091,8 @@ void SIWholeQuadMode::lowerBlock(MachineBasicBlock &MBB) {
   }
 
   // Perform splitting after instruction scan to simplify iteration.
-  if (!SplitPoints.empty()) {
-    MachineBasicBlock *BB = &MBB;
-    for (MachineInstr *MI : SplitPoints) {
-      BB = splitBlock(BB, MI);
-    }
-  }
+  for (MachineInstr *MI : SplitPoints)
+    splitBlock(MI);
 }
 
 // Return an iterator in the (inclusive) range [First, Last] at which
@@ -1260,13 +1259,8 @@ void SIWholeQuadMode::fromStrictMode(MachineBasicBlock &MBB,
   StateTransition[MI] = NonStrictState;
 }
 
-void SIWholeQuadMode::processBlock(MachineBasicBlock &MBB, bool IsEntry) {
-  auto *BII = Blocks.find(&MBB);
-  if (BII == Blocks.end())
-    return;
-
-  BlockInfo &BI = BII->second;
-
+void SIWholeQuadMode::processBlock(MachineBasicBlock &MBB, BlockInfo &BI,
+                                   bool IsEntry) {
   // This is a non-entry block that is WQM throughout, so no need to do
   // anything.
   if (!IsEntry && BI.Needs == StateWQM && BI.OutNeeds != StateExact) {
@@ -1555,19 +1549,18 @@ bool SIWholeQuadMode::lowerCopyInstrs() {
 
 bool SIWholeQuadMode::lowerKillInstrs(bool IsWQM) {
   for (MachineInstr *MI : KillInstrs) {
-    MachineBasicBlock *MBB = MI->getParent();
     MachineInstr *SplitPoint = nullptr;
     switch (MI->getOpcode()) {
     case AMDGPU::SI_DEMOTE_I1:
     case AMDGPU::SI_KILL_I1_TERMINATOR:
-      SplitPoint = lowerKillI1(*MBB, *MI, IsWQM);
+      SplitPoint = lowerKillI1(*MI, IsWQM);
       break;
     case AMDGPU::SI_KILL_F32_COND_IMM_TERMINATOR:
-      SplitPoint = lowerKillF32(*MBB, *MI);
+      SplitPoint = lowerKillF32(*MI);
       break;
     }
     if (SplitPoint)
-      splitBlock(MBB, SplitPoint);
+      splitBlock(SplitPoint);
   }
   return !KillInstrs.empty();
 }
@@ -1705,7 +1698,7 @@ SIWholeQuadMode::lowerInitExecInstrs(MachineBasicBlock &Entry, bool &Changed) {
   return InsertPt;
 }
 
-bool SIWholeQuadMode::runOnMachineFunction(MachineFunction &MF) {
+bool SIWholeQuadMode::run(MachineFunction &MF) {
   LLVM_DEBUG(dbgs() << "SI Whole Quad Mode on " << MF.getName()
                     << " ------------- \n");
   LLVM_DEBUG(MF.dump(););
@@ -1720,18 +1713,6 @@ bool SIWholeQuadMode::runOnMachineFunction(MachineFunction &MF) {
   SetInactiveInstrs.clear();
   StateTransition.clear();
 
-  ST = &MF.getSubtarget<GCNSubtarget>();
-
-  TII = ST->getInstrInfo();
-  TRI = &TII->getRegisterInfo();
-  MRI = &MF.getRegInfo();
-  LIS = &getAnalysis<LiveIntervalsWrapperPass>().getLIS();
-  auto *MDTWrapper = getAnalysisIfAvailable<MachineDominatorTreeWrapperPass>();
-  MDT = MDTWrapper ? &MDTWrapper->getDomTree() : nullptr;
-  auto *PDTWrapper =
-      getAnalysisIfAvailable<MachinePostDominatorTreeWrapperPass>();
-  PDT = PDTWrapper ? &PDTWrapper->getPostDomTree() : nullptr;
-
   if (ST->isWave32()) {
     AndOpc = AMDGPU::S_AND_B32;
     AndTermOpc = AMDGPU::S_AND_B32_term;
@@ -1809,11 +1790,11 @@ bool SIWholeQuadMode::runOnMachineFunction(MachineFunction &MF) {
     if (GlobalFlags & StateWQM)
       Blocks[&Entry].InNeeds |= StateWQM;
     // Wave mode switching requires full lowering pass.
-    for (auto BII : Blocks)
-      processBlock(*BII.first, BII.first == &Entry);
+    for (auto &BII : Blocks)
+      processBlock(*BII.first, BII.second, BII.first == &Entry);
     // Lowering blocks causes block splitting so perform as a second pass.
-    for (auto BII : Blocks)
-      lowerBlock(*BII.first);
+    for (auto &BII : Blocks)
+      lowerBlock(*BII.first, BII.second);
     Changed = true;
   }
 
@@ -1832,3 +1813,38 @@ bool SIWholeQuadMode::runOnMachineFunction(MachineFunction &MF) {
 
   return Changed;
 }
+
+bool SIWholeQuadModeLegacy::runOnMachineFunction(MachineFunction &MF) {
+  LiveIntervals *LIS = &getAnalysis<LiveIntervalsWrapperPass>().getLIS();
+  auto *MDTWrapper = getAnalysisIfAvailable<MachineDominatorTreeWrapperPass>();
+  MachineDominatorTree *MDT = MDTWrapper ? &MDTWrapper->getDomTree() : nullptr;
+  auto *PDTWrapper =
+      getAnalysisIfAvailable<MachinePostDominatorTreeWrapperPass>();
+  MachinePostDominatorTree *PDT =
+      PDTWrapper ? &PDTWrapper->getPostDomTree() : nullptr;
+  SIWholeQuadMode Impl(MF, LIS, MDT, PDT);
+  return Impl.run(MF);
+}
+
+PreservedAnalyses
+SIWholeQuadModePass::run(MachineFunction &MF,
+                         MachineFunctionAnalysisManager &MFAM) {
+  MFPropsModifier _(*this, MF);
+
+  LiveIntervals *LIS = &MFAM.getResult<LiveIntervalsAnalysis>(MF);
+  MachineDominatorTree *MDT =
+      MFAM.getCachedResult<MachineDominatorTreeAnalysis>(MF);
+  MachinePostDominatorTree *PDT =
+      MFAM.getCachedResult<MachinePostDominatorTreeAnalysis>(MF);
+  SIWholeQuadMode Impl(MF, LIS, MDT, PDT);
+  bool Changed = Impl.run(MF);
+  if (!Changed)
+    return PreservedAnalyses::all();
+
+  PreservedAnalyses PA = getMachineFunctionPassPreservedAnalyses();
+  PA.preserve<SlotIndexesAnalysis>();
+  PA.preserve<LiveIntervalsAnalysis>();
+  PA.preserve<MachineDominatorTreeAnalysis>();
+  PA.preserve<MachinePostDominatorTreeAnalysis>();
+  return PA;
+}
diff --git a/llvm/lib/Target/AMDGPU/SIWholeQuadMode.h b/llvm/lib/Target/AMDGPU/SIWholeQuadMode.h
new file mode 100644
index 0000000000000..e30b46721841b
--- /dev/null
+++ b/llvm/lib/Target/AMDGPU/SIWholeQuadMode.h
@@ -0,0 +1,27 @@
+//===- SIWholeQuadMode.h ----------------------------------------*- C++- *-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIB_TARGET_AMDGPU_SIWHOLEQUADMODE_H
+#define LLVM_LIB_TARGET_AMDGPU_SIWHOLEQUADMODE_H
+
+#include "llvm/CodeGen/MachinePassManager.h"
+
+namespace llvm {
+class SIWholeQuadModePass : public PassInfoMixin<SIWholeQuadModePass> {
+public:
+  PreservedAnalyses run(MachineFunction &MF,
+                        MachineFunctionAnalysisManager &MFAM);
+
+  MachineFunctionProperties getClearedProperties() const {
+    return MachineFunctionProperties().set(
+        MachineFunctionProperties::Property::IsSSA);
+  }
+};
+} // namespace llvm
+
+#endif // LLVM_LIB_TARGET_AMDGPU_SIWHOLEQUADMODE_H
diff --git a/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp b/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp
index d9c0aa300855f..59afcbed35294 100644
--- a/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp
@@ -134,6 +134,12 @@ unsigned getLoadcntStorecntBitShift(unsigned VersionMajor) {
   return VersionMajor >= 12 ? 8 : 0;
 }
 
+/// \returns VaSdst bit width
+inline unsigned getVaSdstBitWidth() { return 3; }
+
+/// \returns VaSdst bit shift
+inline unsigned getVaSdstBitShift() { return 9; }
+
 /// \returns VmVsrc bit width
 inline unsigned getVmVsrcBitWidth() { return 3; }
 
@@ -146,6 +152,12 @@ inline unsigned getVaVdstBitWidth() { return 4; }
 /// \returns VaVdst bit shift
 inline unsigned getVaVdstBitShift() { return 12; }
 
+/// \returns VaVcc bit width
+inline unsigned getVaVccBitWidth() { return 1; }
+
+/// \returns VaVcc bit shift
+inline unsigned getVaVccBitShift() { return 1; }
+
 /// \returns SaSdst bit width
 inline unsigned getSaSdstBitWidth() { return 1; }
 
@@ -1719,6 +1731,14 @@ unsigned decodeFieldSaSdst(unsigned Encoded) {
   return unpackBits(Encoded, getSaSdstBitShift(), getSaSdstBitWidth());
 }
 
+unsigned decodeFieldVaSdst(unsigned Encoded) {
+  return unpackBits(Encoded, getVaSdstBitShift(), getVaSdstBitWidth());
+}
+
+unsigned decodeFieldVaVcc(unsigned Encoded) {
+  return unpackBits(Encoded, getVaVccBitShift(), getVaVccBitWidth());
+}
+
 unsigned encodeFieldVmVsrc(unsigned Encoded, unsigned VmVsrc) {
   return packBits(VmVsrc, Encoded, getVmVsrcBitShift(), getVmVsrcBitWidth());
 }
@@ -1743,6 +1763,22 @@ unsigned encodeFieldSaSdst(unsigned SaSdst) {
   return encodeFieldSaSdst(0xffff, SaSdst);
 }
 
+unsigned encodeFieldVaSdst(unsigned Encoded, unsigned VaSdst) {
+  return packBits(VaSdst, Encoded, getVaSdstBitShift(), getVaSdstBitWidth());
+}
+
+unsigned encodeFieldVaSdst(unsigned VaSdst) {
+  return encodeFieldVaSdst(0xffff, VaSdst);
+}
+
+unsigned encodeFieldVaVcc(unsigned Encoded, unsigned VaVcc) {
+  return packBits(VaVcc, Encoded, getVaVccBitShift(), getVaVccBitWidth());
+}
+
+unsigned encodeFieldVaVcc(unsigned VaVcc) {
+  return encodeFieldVaVcc(0xffff, VaVcc);
+}
+
 } // namespace DepCtr
 
 //===----------------------------------------------------------------------===//
diff --git a/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h b/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h
index 3c9246d5e107d..67bebfb3418d5 100644
--- a/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h
+++ b/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h
@@ -976,8 +976,7 @@ struct Waitcnt {
   Waitcnt() = default;
   // Pre-gfx12 constructor.
   Waitcnt(unsigned VmCnt, unsigned ExpCnt, unsigned LgkmCnt, unsigned VsCnt)
-      : LoadCnt(VmCnt), ExpCnt(ExpCnt), DsCnt(LgkmCnt), StoreCnt(VsCnt),
-        SampleCnt(~0u), BvhCnt(~0u), KmCnt(~0u) {}
+      : LoadCnt(VmCnt), ExpCnt(ExpCnt), DsCnt(LgkmCnt), StoreCnt(VsCnt) {}
 
   // gfx12+ constructor.
   Waitcnt(unsigned LoadCnt, unsigned ExpCnt, unsigned DsCnt, unsigned StoreCnt,
@@ -1168,6 +1167,12 @@ unsigned decodeFieldVmVsrc(unsigned Encoded);
 /// \returns Decoded SaSdst from given immediate \p Encoded.
 unsigned decodeFieldSaSdst(unsigned Encoded);
 
+/// \returns Decoded VaSdst from given immediate \p Encoded.
+unsigned decodeFieldVaSdst(unsigned Encoded);
+
+/// \returns Decoded VaVcc from given immediate \p Encoded.
+unsigned decodeFieldVaVcc(unsigned Encoded);
+
 /// \returns \p VmVsrc as an encoded Depctr immediate.
 unsigned encodeFieldVmVsrc(unsigned VmVsrc);
 
@@ -1186,6 +1191,18 @@ unsigned encodeFieldSaSdst(unsigned SaSdst);
 /// \returns \p Encoded combined with encoded \p SaSdst.
 unsigned encodeFieldSaSdst(unsigned Encoded, unsigned SaSdst);
 
+/// \returns \p VaSdst as an encoded Depctr immediate.
+unsigned encodeFieldVaSdst(unsigned VaSdst);
+
+/// \returns \p Encoded combined with encoded \p VaSdst.
+unsigned encodeFieldVaSdst(unsigned Encoded, unsigned VaSdst);
+
+/// \returns \p VaVcc as an encoded Depctr immediate.
+unsigned encodeFieldVaVcc(unsigned VaVcc);
+
+/// \returns \p Encoded combined with encoded \p VaVcc.
+unsigned encodeFieldVaVcc(unsigned Encoded, unsigned VaVcc);
+
 } // namespace DepCtr
 
 namespace Exp {
diff --git a/llvm/lib/Target/AMDGPU/Utils/AMDGPUPALMetadata.cpp b/llvm/lib/Target/AMDGPU/Utils/AMDGPUPALMetadata.cpp
index 1e76bf7056cc8..296031e4a068f 100644
--- a/llvm/lib/Target/AMDGPU/Utils/AMDGPUPALMetadata.cpp
+++ b/llvm/lib/Target/AMDGPU/Utils/AMDGPUPALMetadata.cpp
@@ -27,6 +27,28 @@
 using namespace llvm;
 using namespace llvm::AMDGPU;
 
+// Return the PAL metadata hardware shader stage name.
+static const char *getStageName(CallingConv::ID CC) {
+  switch (CC) {
+  case CallingConv::AMDGPU_PS:
+    return ".ps";
+  case CallingConv::AMDGPU_VS:
+    return ".vs";
+  case CallingConv::AMDGPU_GS:
+    return ".gs";
+  case CallingConv::AMDGPU_ES:
+    return ".es";
+  case CallingConv::AMDGPU_HS:
+    return ".hs";
+  case CallingConv::AMDGPU_LS:
+    return ".ls";
+  case CallingConv::AMDGPU_Gfx:
+    llvm_unreachable("Callable shader has no hardware stage");
+  default:
+    return ".cs";
+  }
+}
+
 // Read the PAL metadata from IR metadata, where it was put by the frontend.
 void AMDGPUPALMetadata::readFromIR(Module &M) {
   auto *NamedMD = M.getNamedMetadata("amdgpu.pal.metadata.msgpack");
@@ -232,8 +254,18 @@ void AMDGPUPALMetadata::setEntryPoint(unsigned CC, StringRef Name) {
   if (isLegacy())
     return;
   // Msgpack format.
+  // Entry point is updated to .entry_point_symbol and is set to the function
+  // name
   getHwStage(CC)[".entry_point_symbol"] =
       MsgPackDoc.getNode(Name, /*Copy=*/true);
+
+  // Set .entry_point which is defined
+  // to be _amdgpu_<stage> and _amdgpu_cs for non-shader functions
+  SmallString<16> EPName("_amdgpu_");
+  raw_svector_ostream EPNameOS(EPName);
+  EPNameOS << getStageName(CC) + 1;
+  getHwStage(CC)[".entry_point"] =
+      MsgPackDoc.getNode(EPNameOS.str(), /*Copy=*/true);
 }
 
 // Set the number of used vgprs in the metadata. This is an optional
@@ -943,28 +975,6 @@ msgpack::MapDocNode AMDGPUPALMetadata::getGraphicsRegisters() {
   return GraphicsRegisters.getMap();
 }
 
-// Return the PAL metadata hardware shader stage name.
-static const char *getStageName(CallingConv::ID CC) {
-  switch (CC) {
-  case CallingConv::AMDGPU_PS:
-    return ".ps";
-  case CallingConv::AMDGPU_VS:
-    return ".vs";
-  case CallingConv::AMDGPU_GS:
-    return ".gs";
-  case CallingConv::AMDGPU_ES:
-    return ".es";
-  case CallingConv::AMDGPU_HS:
-    return ".hs";
-  case CallingConv::AMDGPU_LS:
-    return ".ls";
-  case CallingConv::AMDGPU_Gfx:
-    llvm_unreachable("Callable shader has no hardware stage");
-  default:
-    return ".cs";
-  }
-}
-
 msgpack::DocNode &AMDGPUPALMetadata::refHwStage() {
   auto &N =
       MsgPackDoc.getRoot()
diff --git a/llvm/lib/Target/AMDGPU/VOP1Instructions.td b/llvm/lib/Target/AMDGPU/VOP1Instructions.td
index b9c73e6ce8ef2..a407ae797a48b 100644
--- a/llvm/lib/Target/AMDGPU/VOP1Instructions.td
+++ b/llvm/lib/Target/AMDGPU/VOP1Instructions.td
@@ -973,12 +973,12 @@ multiclass VOP1_Real_FULL_with_name_gfx11_gfx12<bits<9> op, string opName,
   VOP1_Real_FULL_with_name<GFX11Gen, op, opName, asmName>,
   VOP1_Real_FULL_with_name<GFX12Gen, op, opName, asmName>;
 
-multiclass VOP1_Real_FULL_t16_and_fake16_gfx11_gfx12<bits<9> op, string asmName,
-                                          string opName = NAME> {
+multiclass VOP1_Real_FULL_t16_and_fake16_gfx11_gfx12<
+    bits<9> op, string asmName = !tolower(NAME), string opName = NAME> {
   defm opName#"_t16" :
-	VOP1_Real_FULL_with_name_gfx11_gfx12<op, opName#"_t16", asmName>;
+       VOP1_Real_FULL_with_name_gfx11_gfx12<op, opName#"_t16", asmName>;
   defm opName#"_fake16":
-	VOP1_Real_FULL_with_name_gfx11_gfx12<op, opName#"_fake16", asmName>;
+       VOP1_Real_FULL_with_name_gfx11_gfx12<op, opName#"_fake16", asmName>;
 }
 
 multiclass VOP1Only_Real_gfx11_gfx12<bits<9> op> :
@@ -1018,14 +1018,14 @@ defm V_CLS_I32             : VOP1_Real_FULL_with_name_gfx11_gfx12<0x03b,
 defm V_SWAP_B16              : VOP1Only_Real_gfx11_gfx12<0x066>;
 defm V_PERMLANE64_B32        : VOP1Only_Real_gfx11_gfx12<0x067>;
 defm V_MOV_B16_t16           : VOP1_Real_FULL_t16_gfx11_gfx12<0x01c, "v_mov_b16">;
-defm V_NOT_B16               : VOP1_Real_FULL_t16_and_fake16_gfx11_gfx12<0x069, "v_not_b16">;
-defm V_CVT_I32_I16           : VOP1_Real_FULL_t16_and_fake16_gfx11_gfx12<0x06a, "v_cvt_i32_i16">;
-defm V_CVT_U32_U16           : VOP1_Real_FULL_t16_and_fake16_gfx11_gfx12<0x06b, "v_cvt_u32_u16">;
-
-defm V_CVT_F16_U16           : VOP1_Real_FULL_t16_and_fake16_gfx11_gfx12<0x050, "v_cvt_f16_u16">;
-defm V_CVT_F16_I16           : VOP1_Real_FULL_t16_and_fake16_gfx11_gfx12<0x051, "v_cvt_f16_i16">;
-defm V_CVT_U16_F16           : VOP1_Real_FULL_t16_and_fake16_gfx11_gfx12<0x052, "v_cvt_u16_f16">;
-defm V_CVT_I16_F16           : VOP1_Real_FULL_t16_and_fake16_gfx11_gfx12<0x053, "v_cvt_i16_f16">;
+defm V_NOT_B16               : VOP1_Real_FULL_t16_and_fake16_gfx11_gfx12<0x069>;
+defm V_CVT_I32_I16           : VOP1_Real_FULL_t16_and_fake16_gfx11_gfx12<0x06a>;
+defm V_CVT_U32_U16           : VOP1_Real_FULL_t16_and_fake16_gfx11_gfx12<0x06b>;
+
+defm V_CVT_F16_U16           : VOP1_Real_FULL_t16_and_fake16_gfx11_gfx12<0x050>;
+defm V_CVT_F16_I16           : VOP1_Real_FULL_t16_and_fake16_gfx11_gfx12<0x051>;
+defm V_CVT_U16_F16           : VOP1_Real_FULL_t16_and_fake16_gfx11_gfx12<0x052>;
+defm V_CVT_I16_F16           : VOP1_Real_FULL_t16_and_fake16_gfx11_gfx12<0x053>;
 defm V_RCP_F16_t16           : VOP1_Real_FULL_t16_gfx11_gfx12<0x054, "v_rcp_f16">;
 defm V_RCP_F16_fake16        : VOP1_Real_FULL_t16_gfx11_gfx12<0x054, "v_rcp_f16">;
 defm V_SQRT_F16_t16          : VOP1_Real_FULL_t16_gfx11_gfx12<0x055, "v_sqrt_f16">;
@@ -1036,23 +1036,23 @@ defm V_LOG_F16_t16           : VOP1_Real_FULL_t16_gfx11_gfx12<0x057, "v_log_f16"
 defm V_LOG_F16_fake16        : VOP1_Real_FULL_t16_gfx11_gfx12<0x057, "v_log_f16">;
 defm V_EXP_F16_t16           : VOP1_Real_FULL_t16_gfx11_gfx12<0x058, "v_exp_f16">;
 defm V_EXP_F16_fake16        : VOP1_Real_FULL_t16_gfx11_gfx12<0x058, "v_exp_f16">;
-defm V_FREXP_MANT_F16        : VOP1_Real_FULL_t16_and_fake16_gfx11_gfx12<0x059, "v_frexp_mant_f16">;
-defm V_FREXP_EXP_I16_F16     : VOP1_Real_FULL_t16_and_fake16_gfx11_gfx12<0x05a, "v_frexp_exp_i16_f16">;
+defm V_FREXP_MANT_F16        : VOP1_Real_FULL_t16_and_fake16_gfx11_gfx12<0x059>;
+defm V_FREXP_EXP_I16_F16     : VOP1_Real_FULL_t16_and_fake16_gfx11_gfx12<0x05a>;
 defm V_FLOOR_F16_t16         : VOP1_Real_FULL_t16_gfx11_gfx12<0x05b, "v_floor_f16">;
 defm V_FLOOR_F16_fake16      : VOP1_Real_FULL_t16_gfx11_gfx12<0x05b, "v_floor_f16">;
 defm V_CEIL_F16_t16          : VOP1_Real_FULL_t16_gfx11_gfx12<0x05c, "v_ceil_f16">;
 defm V_CEIL_F16_fake16       : VOP1_Real_FULL_t16_gfx11_gfx12<0x05c, "v_ceil_f16">;
-defm V_TRUNC_F16             : VOP1_Real_FULL_t16_and_fake16_gfx11_gfx12<0x05d, "v_trunc_f16">;
-defm V_RNDNE_F16             : VOP1_Real_FULL_t16_and_fake16_gfx11_gfx12<0x05e, "v_rndne_f16">;
-defm V_FRACT_F16             : VOP1_Real_FULL_t16_and_fake16_gfx11_gfx12<0x05f, "v_fract_f16">;
-defm V_SIN_F16               : VOP1_Real_FULL_t16_and_fake16_gfx11_gfx12<0x060, "v_sin_f16">;
-defm V_COS_F16               : VOP1_Real_FULL_t16_and_fake16_gfx11_gfx12<0x061, "v_cos_f16">;
-defm V_SAT_PK_U8_I16         : VOP1_Real_FULL_t16_and_fake16_gfx11_gfx12<0x062, "v_sat_pk_u8_i16">;
-defm V_CVT_NORM_I16_F16      : VOP1_Real_FULL_t16_and_fake16_gfx11_gfx12<0x063, "v_cvt_norm_i16_f16">;
-defm V_CVT_NORM_U16_F16      : VOP1_Real_FULL_t16_and_fake16_gfx11_gfx12<0x064, "v_cvt_norm_u16_f16">;
-
-defm V_CVT_F16_F32           : VOP1_Real_FULL_t16_and_fake16_gfx11_gfx12<0x00a, "v_cvt_f16_f32">;
-defm V_CVT_F32_F16           : VOP1_Real_FULL_t16_and_fake16_gfx11_gfx12<0x00b, "v_cvt_f32_f16">;
+defm V_TRUNC_F16             : VOP1_Real_FULL_t16_and_fake16_gfx11_gfx12<0x05d>;
+defm V_RNDNE_F16             : VOP1_Real_FULL_t16_and_fake16_gfx11_gfx12<0x05e>;
+defm V_FRACT_F16             : VOP1_Real_FULL_t16_and_fake16_gfx11_gfx12<0x05f>;
+defm V_SIN_F16               : VOP1_Real_FULL_t16_and_fake16_gfx11_gfx12<0x060>;
+defm V_COS_F16               : VOP1_Real_FULL_t16_and_fake16_gfx11_gfx12<0x061>;
+defm V_SAT_PK_U8_I16         : VOP1_Real_FULL_t16_and_fake16_gfx11_gfx12<0x062>;
+defm V_CVT_NORM_I16_F16      : VOP1_Real_FULL_t16_and_fake16_gfx11_gfx12<0x063>;
+defm V_CVT_NORM_U16_F16      : VOP1_Real_FULL_t16_and_fake16_gfx11_gfx12<0x064>;
+
+defm V_CVT_F16_F32           : VOP1_Real_FULL_t16_and_fake16_gfx11_gfx12<0x00a>;
+defm V_CVT_F32_F16           : VOP1_Real_FULL_t16_and_fake16_gfx11_gfx12<0x00b>;
 
 //===----------------------------------------------------------------------===//
 // GFX10.
diff --git a/llvm/lib/Target/AMDGPU/VOP3Instructions.td b/llvm/lib/Target/AMDGPU/VOP3Instructions.td
index 947ac5c27620f..afafc2ecccfaf 100644
--- a/llvm/lib/Target/AMDGPU/VOP3Instructions.td
+++ b/llvm/lib/Target/AMDGPU/VOP3Instructions.td
@@ -170,8 +170,8 @@ defm V_MUL_HI_I32 : VOP3Inst <"v_mul_hi_i32", V_MUL_PROF<VOP_I32_I32_I32>, mulhs
 let SubtargetPredicate = isGFX12Plus, ReadsModeReg = 0, AddedComplexity = 1 in {
 defm V_MINIMUM_F32 : VOP3Inst <"v_minimum_f32", VOP3_Profile<VOP_F32_F32_F32>, DivergentBinFrag<fminimum>>;
 defm V_MAXIMUM_F32 : VOP3Inst <"v_maximum_f32", VOP3_Profile<VOP_F32_F32_F32>, DivergentBinFrag<fmaximum>>;
-defm V_MINIMUM_F16 : VOP3Inst <"v_minimum_f16", VOP3_Profile<VOP_F16_F16_F16>, DivergentBinFrag<fminimum>>;
-defm V_MAXIMUM_F16 : VOP3Inst <"v_maximum_f16", VOP3_Profile<VOP_F16_F16_F16>, DivergentBinFrag<fmaximum>>;
+defm V_MINIMUM_F16 : VOP3Inst_t16 <"v_minimum_f16", VOP_F16_F16_F16, DivergentBinFrag<fminimum>>;
+defm V_MAXIMUM_F16 : VOP3Inst_t16 <"v_maximum_f16", VOP_F16_F16_F16, DivergentBinFrag<fmaximum>>;
 
 let SchedRW = [WriteDoubleAdd] in {
 defm V_MINIMUM_F64 : VOP3Inst <"v_minimum_f64", VOP3_Profile<VOP_F64_F64_F64>, fminimum>;
@@ -212,7 +212,13 @@ defm V_BFE_U32 : VOP3Inst <"v_bfe_u32", VOP3_Profile<VOP_I32_I32_I32_I32>, AMDGP
 defm V_BFE_I32 : VOP3Inst <"v_bfe_i32", VOP3_Profile<VOP_I32_I32_I32_I32>, AMDGPUbfe_i32>;
 defm V_BFI_B32 : VOP3Inst <"v_bfi_b32", VOP3_Profile<VOP_I32_I32_I32_I32>, AMDGPUbfi>;
 defm V_ALIGNBIT_B32 : VOP3Inst <"v_alignbit_b32", VOP3_Profile<VOP_I32_I32_I32_I32>, fshr>;
+
+let True16Predicate = NotHasTrue16BitInsts in
 defm V_ALIGNBYTE_B32 : VOP3Inst <"v_alignbyte_b32", VOP3_Profile<VOP_I32_I32_I32_I32>, int_amdgcn_alignbyte>;
+let True16Predicate = UseRealTrue16Insts in
+defm V_ALIGNBYTE_B32_t16 : VOP3Inst <"v_alignbyte_b32_t16", VOP3_Profile_True16<VOP_I32_I32_I32_I16, VOP3_OPSEL>>;
+let True16Predicate = UseFakeTrue16Insts in 
+defm V_ALIGNBYTE_B32_fake16 : VOP3Inst <"v_alignbyte_b32_fake16", VOP3_Profile_Fake16<VOP_I32_I32_I32_I16, VOP3_OPSEL>>;
 
 // XXX - No FPException seems suspect but manual doesn't say it does
 let mayRaiseFPException = 0 in {
@@ -250,6 +256,25 @@ let SchedRW = [WriteDoubleAdd], FPDPRounding = 1 in {
 } // End SchedRW = [WriteDoubleAdd], FPDPRounding = 1
 } // End isReMaterializable = 1
 
+let True16Predicate = UseFakeTrue16Insts in
+def : GCNPat <
+(i32 (int_amdgcn_alignbyte (i32 (VOP3OpSelMods i32:$src0, i32:$src0_modifiers)),
+                           (i32 (VOP3OpSelMods i32:$src1, i32:$src1_modifiers)),
+                           (i32 (VOP3OpSelMods i32:$src2, i32:$src2_modifiers)))),
+(V_ALIGNBYTE_B32_fake16_e64 i32:$src0_modifiers, VSrc_b32:$src0,
+                            i32:$src1_modifiers, VSrc_b32:$src1,
+                            i32:$src2_modifiers, VGPR_32:$src2)
+>;
+
+let True16Predicate = UseRealTrue16Insts in
+def : GCNPat <
+(i32 (int_amdgcn_alignbyte (i32 (VOP3OpSelMods i32:$src0, i32:$src0_modifiers)),
+                           (i32 (VOP3OpSelMods i32:$src1, i32:$src1_modifiers)),
+                           (i32 (VOP3OpSelMods i32:$src2, i32:$src2_modifiers)))),
+(V_ALIGNBYTE_B32_t16_e64 i32:$src0_modifiers, VSrc_b32:$src0,
+                         i32:$src1_modifiers, VSrc_b32:$src1,
+                         i32:$src2_modifiers, (i16 (EXTRACT_SUBREG VGPR_32:$src2, lo16)))
+>;
 
 let mayRaiseFPException = 0 in { // Seems suspicious but manual doesn't say it does.
   let SchedRW = [WriteFloatFMA, WriteSALU] in
@@ -446,6 +471,9 @@ multiclass Ternary_i16_Pats_gfx9<SDPatternOperator op1, SDPatternOperator op2,
   >;
 }
 
+let True16Predicate = UseRealTrue16Insts in {
+  defm: Ternary_i16_Pats_gfx9<mul, add, V_MAD_U16_gfx9_t16_e64>;
+} // End True16Predicates = UseRealTrue16Insts
 let True16Predicate = UseFakeTrue16Insts in {
   defm: Ternary_i16_Pats_gfx9<mul, add, V_MAD_U16_gfx9_fake16_e64>;
 } // End True16Predicates = UseFakeTrue16Insts
@@ -634,16 +662,16 @@ defm V_MAX3_I16 : VOP3Inst_t16 <"v_max3_i16", VOP_I16_I16_I16_I16, AMDGPUsmax3>;
 defm V_MAX3_U16 : VOP3Inst_t16 <"v_max3_u16", VOP_I16_I16_I16_I16, AMDGPUumax3>;
 
 let SubtargetPredicate = HasMinimum3Maximum3F16, ReadsModeReg = 0 in {
-  defm V_MINIMUM3_F16 : VOP3Inst <"v_minimum3_f16", VOP3_Profile<VOP_F16_F16_F16_F16, VOP3_OPSEL>, AMDGPUfminimum3>;
-  defm V_MAXIMUM3_F16 : VOP3Inst <"v_maximum3_f16", VOP3_Profile<VOP_F16_F16_F16_F16, VOP3_OPSEL>, AMDGPUfmaximum3>;
+  defm V_MINIMUM3_F16 : VOP3Inst_t16 <"v_minimum3_f16", VOP_F16_F16_F16_F16, AMDGPUfminimum3>;
+  defm V_MAXIMUM3_F16 : VOP3Inst_t16 <"v_maximum3_f16", VOP_F16_F16_F16_F16, AMDGPUfmaximum3>;
 } // End SubtargetPredicate = isGFX12Plus, ReadsModeReg = 0
 
 defm V_ADD_I16 : VOP3Inst_t16 <"v_add_i16", VOP_I16_I16_I16>;
 defm V_SUB_I16 : VOP3Inst_t16 <"v_sub_i16", VOP_I16_I16_I16>;
 
 let isCommutable = 1 in {
-  defm V_MAD_U32_U16 : VOP3Inst <"v_mad_u32_u16", VOP3_Profile<VOP_I32_I16_I16_I32, VOP3_OPSEL>>;
-  defm V_MAD_I32_I16 : VOP3Inst <"v_mad_i32_i16", VOP3_Profile<VOP_I32_I16_I16_I32, VOP3_OPSEL>>;
+  defm V_MAD_U32_U16 : VOP3Inst_t16 <"v_mad_u32_u16", VOP_I32_I16_I16_I32>;
+  defm V_MAD_I32_I16 : VOP3Inst_t16 <"v_mad_i32_i16", VOP_I32_I16_I16_I32>;
 } // End isCommutable = 1
 
 defm V_CVT_PKNORM_I16_F16 : VOP3Inst_t16 <"v_cvt_pknorm_i16_f16", VOP_B32_F16_F16>;
@@ -1440,8 +1468,8 @@ let SubtargetPredicate = HasF32ToF16BF16ConversionSRInsts in {
 let SubtargetPredicate = isGFX12Plus, ReadsModeReg = 0 in {
   defm V_MAXIMUMMINIMUM_F32 : VOP3Inst<"v_maximumminimum_f32", VOP3_Profile<VOP_F32_F32_F32_F32>>;
   defm V_MINIMUMMAXIMUM_F32 : VOP3Inst<"v_minimummaximum_f32", VOP3_Profile<VOP_F32_F32_F32_F32>>;
-  defm V_MAXIMUMMINIMUM_F16 : VOP3Inst<"v_maximumminimum_f16", VOP3_Profile<VOP_F16_F16_F16_F16, VOP3_OPSEL>>;
-  defm V_MINIMUMMAXIMUM_F16 : VOP3Inst<"v_minimummaximum_f16", VOP3_Profile<VOP_F16_F16_F16_F16, VOP3_OPSEL>>;
+  defm V_MAXIMUMMINIMUM_F16 : VOP3Inst_t16<"v_maximumminimum_f16", VOP_F16_F16_F16_F16>;
+  defm V_MINIMUMMAXIMUM_F16 : VOP3Inst_t16<"v_minimummaximum_f16", VOP_F16_F16_F16_F16>;
 } // End SubtargetPredicate = isGFX12Plus, ReadsModeReg = 0
 
 let SubtargetPredicate = HasDot9Insts, IsDOT=1 in {
@@ -1591,8 +1619,8 @@ defm V_MIN3_NUM_F16       : VOP3_Realtriple_t16_and_fake16_gfx12<0x22b, "v_min3_
 defm V_MAX3_NUM_F16       : VOP3_Realtriple_t16_and_fake16_gfx12<0x22c, "v_max3_num_f16", "V_MAX3_F16", "v_max3_f16">;
 defm V_MINIMUM3_F32       : VOP3Only_Realtriple_gfx12<0x22d>;
 defm V_MAXIMUM3_F32       : VOP3Only_Realtriple_gfx12<0x22e>;
-defm V_MINIMUM3_F16       : VOP3Only_Realtriple_t16_gfx12<0x22f>;
-defm V_MAXIMUM3_F16       : VOP3Only_Realtriple_t16_gfx12<0x230>;
+defm V_MINIMUM3_F16       : VOP3Only_Realtriple_t16_and_fake16_gfx12<0x22f, "v_minimum3_f16">;
+defm V_MAXIMUM3_F16       : VOP3Only_Realtriple_t16_and_fake16_gfx12<0x230, "v_maximum3_f16">;
 defm V_MED3_NUM_F32       : VOP3_Realtriple_with_name_gfx12<0x231, "V_MED3_F32", "v_med3_num_f32">;
 defm V_MED3_NUM_F16       : VOP3_Realtriple_t16_and_fake16_gfx12<0x232, "v_med3_num_f16", "V_MED3_F16", "v_med3_f16">;
 defm V_MINMAX_NUM_F32     : VOP3_Realtriple_with_name_gfx12<0x268, "V_MINMAX_F32", "v_minmax_num_f32">;
@@ -1601,8 +1629,8 @@ defm V_MINMAX_NUM_F16     : VOP3_Realtriple_t16_and_fake16_gfx12<0x26a, "v_minma
 defm V_MAXMIN_NUM_F16     : VOP3_Realtriple_t16_and_fake16_gfx12<0x26b, "v_maxmin_num_f16", "V_MAXMIN_F16", "v_maxmin_f16">;
 defm V_MINIMUMMAXIMUM_F32 : VOP3Only_Realtriple_gfx12<0x26c>;
 defm V_MAXIMUMMINIMUM_F32 : VOP3Only_Realtriple_gfx12<0x26d>;
-defm V_MINIMUMMAXIMUM_F16 : VOP3Only_Realtriple_t16_gfx12<0x26e>;
-defm V_MAXIMUMMINIMUM_F16 : VOP3Only_Realtriple_t16_gfx12<0x26f>;
+defm V_MINIMUMMAXIMUM_F16 : VOP3Only_Realtriple_t16_and_fake16_gfx12<0x26e, "v_minimummaximum_f16">;
+defm V_MAXIMUMMINIMUM_F16 : VOP3Only_Realtriple_t16_and_fake16_gfx12<0x26f, "v_maximumminimum_f16">;
 defm V_S_EXP_F32          : VOP3Only_Real_Base_gfx12<0x280>;
 defm V_S_EXP_F16          : VOP3Only_Real_Base_gfx12<0x281>;
 defm V_S_LOG_F32          : VOP3Only_Real_Base_gfx12<0x282>;
@@ -1619,8 +1647,8 @@ defm V_MINIMUM_F64        : VOP3Only_Real_Base_gfx12<0x341>;
 defm V_MAXIMUM_F64        : VOP3Only_Real_Base_gfx12<0x342>;
 defm V_MINIMUM_F32        : VOP3Only_Realtriple_gfx12<0x365>;
 defm V_MAXIMUM_F32        : VOP3Only_Realtriple_gfx12<0x366>;
-defm V_MINIMUM_F16        : VOP3Only_Realtriple_t16_gfx12<0x367>;
-defm V_MAXIMUM_F16        : VOP3Only_Realtriple_t16_gfx12<0x368>;
+defm V_MINIMUM_F16        : VOP3Only_Realtriple_t16_and_fake16_gfx12<0x367, "v_minimum_f16">;
+defm V_MAXIMUM_F16        : VOP3Only_Realtriple_t16_and_fake16_gfx12<0x368, "v_maximum_f16">;
 
 defm V_PERMLANE16_VAR_B32  : VOP3Only_Real_Base_gfx12<0x30f>;
 defm V_PERMLANEX16_VAR_B32 : VOP3Only_Real_Base_gfx12<0x310>;
@@ -1687,7 +1715,7 @@ defm V_FMA_F32             : VOP3_Realtriple_gfx11_gfx12<0x213>;
 defm V_FMA_F64             : VOP3_Real_Base_gfx11_gfx12<0x214>;
 defm V_LERP_U8             : VOP3_Realtriple_gfx11_gfx12<0x215>;
 defm V_ALIGNBIT_B32        : VOP3_Realtriple_gfx11_gfx12<0x216>;
-defm V_ALIGNBYTE_B32       : VOP3_Realtriple_gfx11_gfx12<0x217>;
+defm V_ALIGNBYTE_B32       : VOP3_Realtriple_t16_and_fake16_gfx11_gfx12<0x217, "v_alignbyte_b32">;
 defm V_MULLIT_F32          : VOP3_Realtriple_gfx11_gfx12<0x218>;
 defm V_MIN3_F32            : VOP3_Realtriple_gfx11<0x219>;
 defm V_MIN3_I32            : VOP3_Realtriple_gfx11_gfx12<0x21a>;
@@ -1733,8 +1761,8 @@ defm V_ADD3_U32            : VOP3_Realtriple_gfx11_gfx12<0x255>;
 defm V_LSHL_OR_B32         : VOP3_Realtriple_gfx11_gfx12<0x256>;
 defm V_AND_OR_B32          : VOP3_Realtriple_gfx11_gfx12<0x257>;
 defm V_OR3_B32             : VOP3_Realtriple_gfx11_gfx12<0x258>;
-defm V_MAD_U32_U16         : VOP3_Realtriple_gfx11_gfx12<0x259>;
-defm V_MAD_I32_I16         : VOP3_Realtriple_gfx11_gfx12<0x25a>;
+defm V_MAD_U32_U16         : VOP3_Realtriple_t16_and_fake16_gfx11_gfx12<0x259, "v_mad_u32_u16">;
+defm V_MAD_I32_I16         : VOP3_Realtriple_t16_and_fake16_gfx11_gfx12<0x25a, "v_mad_i32_i16">;
 defm V_PERMLANE16_B32      : VOP3_Real_Base_gfx11_gfx12<0x25b>;
 defm V_PERMLANEX16_B32     : VOP3_Real_Base_gfx11_gfx12<0x25c>;
 defm V_MAXMIN_F32          : VOP3_Realtriple_gfx11<0x25e>;
diff --git a/llvm/lib/Target/AMDGPU/VOP3PInstructions.td b/llvm/lib/Target/AMDGPU/VOP3PInstructions.td
index 1afd68767cd3b..5e825e7259a95 100644
--- a/llvm/lib/Target/AMDGPU/VOP3PInstructions.td
+++ b/llvm/lib/Target/AMDGPU/VOP3PInstructions.td
@@ -154,10 +154,12 @@ defm V_PK_MAXIMUM3_F16 : VOP3PInst<"v_pk_maximum3_f16", VOP3P_Profile<VOP_V2F16_
 multiclass MadFmaMixPats<SDPatternOperator fma_like,
                          Instruction mix_inst,
                          Instruction mixlo_inst,
-                         Instruction mixhi_inst> {
+                         Instruction mixhi_inst,
+                         bit HasFP32Denormals> {
   // At least one of the operands needs to be an fpextend of an f16
   // for this to be worthwhile, so we need three patterns here.
   // TODO: Could we use a predicate to inspect src1/2/3 instead?
+  let OtherPredicates = !if(HasFP32Denormals, [TruePredicate], [NoFP32Denormals]) in {
   def : GCNPat <
     (f32 (fma_like (f32 (VOP3PMadMixModsExt f16:$src0, i32:$src0_mods)),
                    (f32 (VOP3PMadMixMods f16:$src1, i32:$src1_mods)),
@@ -177,6 +179,45 @@ multiclass MadFmaMixPats<SDPatternOperator fma_like,
     (mix_inst $src0_mods, $src0, $src1_mods, $src1, $src2_mods, $src2,
               DSTCLAMP.NONE)>;
 
+  def : GCNPat <
+    (AMDGPUclamp (build_vector
+      (f16 (fpround (fma_like (f32 (VOP3PMadMixMods f16:$lo_src0, i32:$lo_src0_modifiers)),
+                         (f32 (VOP3PMadMixMods f16:$lo_src1, i32:$lo_src1_modifiers)),
+                         (f32 (VOP3PMadMixMods f16:$lo_src2, i32:$lo_src2_modifiers))))),
+      (f16 (fpround (fma_like (f32 (VOP3PMadMixMods f16:$hi_src0, i32:$hi_src0_modifiers)),
+                         (f32 (VOP3PMadMixMods f16:$hi_src1, i32:$hi_src1_modifiers)),
+                         (f32 (VOP3PMadMixMods f16:$hi_src2, i32:$hi_src2_modifiers))))))),
+    (v2f16 (mixhi_inst $hi_src0_modifiers, $hi_src0,
+                       $hi_src1_modifiers, $hi_src1,
+                       $hi_src2_modifiers, $hi_src2,
+                       DSTCLAMP.ENABLE,
+                       (mixlo_inst $lo_src0_modifiers, $lo_src0,
+                                   $lo_src1_modifiers, $lo_src1,
+                                   $lo_src2_modifiers, $lo_src2,
+                                   DSTCLAMP.ENABLE,
+                                   (i32 (IMPLICIT_DEF)))))
+  >;
+
+  def : GCNPat <
+    (f16 (fpround (fmul (f32 (VOP3PMadMixMods f32:$src0, i32:$src0_modifiers)),
+                        (f32 (VOP3PMadMixMods f32:$src1, i32:$src1_modifiers))))),
+    (mixlo_inst $src0_modifiers, $src0,
+                $src1_modifiers, $src1,
+                (i32 0), (i32 0),
+                DSTCLAMP.NONE,
+                (i32 (IMPLICIT_DEF)))
+  >;
+
+  def : GCNPat <
+    (build_vector f16:$elt0, (f16 (fpround (fmul (f32 (VOP3PMadMixMods f32:$src0, i32:$src0_modifiers)),
+                                            (f32 (VOP3PMadMixMods f32:$src1, i32:$src1_modifiers)))))),
+    (v2f16 (mixhi_inst $src0_modifiers, $src0,
+                       $src1_modifiers, $src1,
+                       (i32 0), (i32 0),
+                       DSTCLAMP.NONE,
+                       VGPR_32:$elt0))
+  >;
+
   def : GCNPat <
     (f16 (fpround (fma_like (f32 (VOP3PMadMixMods f16:$src0, i32:$src0_modifiers)),
                             (f32 (VOP3PMadMixMods f16:$src1, i32:$src1_modifiers)),
@@ -187,10 +228,14 @@ multiclass MadFmaMixPats<SDPatternOperator fma_like,
                 DSTCLAMP.NONE,
                 (i32 (IMPLICIT_DEF)))
   >;
+  } // End OtherPredicates
 
   // FIXME: Special case handling for maxhi (especially for clamp)
   // because dealing with the write to high half of the register is
   // difficult.
+  foreach p = [NotHasTrue16BitInsts, UseFakeTrue16Insts] in
+  let OtherPredicates = !if(HasFP32Denormals, [TruePredicate], [NoFP32Denormals]), True16Predicate = p in {
+
   def : GCNPat <
     (build_vector f16:$elt0, (f16 (fpround (fma_like (f32 (VOP3PMadMixMods f16:$src0, i32:$src0_modifiers)),
                                                      (f32 (VOP3PMadMixMods f16:$src1, i32:$src1_modifiers)),
@@ -215,44 +260,44 @@ multiclass MadFmaMixPats<SDPatternOperator fma_like,
                        VGPR_32:$elt0))
   >;
 
-  def : GCNPat <
-    (AMDGPUclamp (build_vector
-      (f16 (fpround (fma_like (f32 (VOP3PMadMixMods f16:$lo_src0, i32:$lo_src0_modifiers)),
-                         (f32 (VOP3PMadMixMods f16:$lo_src1, i32:$lo_src1_modifiers)),
-                         (f32 (VOP3PMadMixMods f16:$lo_src2, i32:$lo_src2_modifiers))))),
-      (f16 (fpround (fma_like (f32 (VOP3PMadMixMods f16:$hi_src0, i32:$hi_src0_modifiers)),
-                         (f32 (VOP3PMadMixMods f16:$hi_src1, i32:$hi_src1_modifiers)),
-                         (f32 (VOP3PMadMixMods f16:$hi_src2, i32:$hi_src2_modifiers))))))),
-    (v2f16 (mixhi_inst $hi_src0_modifiers, $hi_src0,
-                       $hi_src1_modifiers, $hi_src1,
-                       $hi_src2_modifiers, $hi_src2,
-                       DSTCLAMP.ENABLE,
-                       (mixlo_inst $lo_src0_modifiers, $lo_src0,
-                                   $lo_src1_modifiers, $lo_src1,
-                                   $lo_src2_modifiers, $lo_src2,
-                                   DSTCLAMP.ENABLE,
-                                   (i32 (IMPLICIT_DEF)))))
-  >;
+  } // end OtherPredicates
 
+  let OtherPredicates = !if(HasFP32Denormals, [TruePredicate], [NoFP32Denormals]), True16Predicate = UseRealTrue16Insts in {
   def : GCNPat <
-    (f16 (fpround (fmul (f32 (VOP3PMadMixMods f32:$src0, i32:$src0_modifiers)),
-                        (f32 (VOP3PMadMixMods f32:$src1, i32:$src1_modifiers))))),
-    (mixlo_inst $src0_modifiers, $src0,
+    (build_vector (f16 (fpround (fma_like (f32 (VOP3PMadMixMods f16:$src0, i32:$src0_modifiers)),
+                            (f32 (VOP3PMadMixMods f16:$src1, i32:$src1_modifiers)),
+                            (f32 (VOP3PMadMixMods f16:$src2, i32:$src2_modifiers))))), f16:$elt1),
+    (v2f16 (mixlo_inst $src0_modifiers, $src0,
                 $src1_modifiers, $src1,
-                (i32 0), (i32 0),
+                $src2_modifiers, $src2,
                 DSTCLAMP.NONE,
-                (i32 (IMPLICIT_DEF)))
+                (REG_SEQUENCE VGPR_32, (f16 (IMPLICIT_DEF)), lo16, $elt1, hi16)))
   >;
 
   def : GCNPat <
-    (build_vector f16:$elt0, (f16 (fpround (fmul (f32 (VOP3PMadMixMods f32:$src0, i32:$src0_modifiers)),
-                                            (f32 (VOP3PMadMixMods f32:$src1, i32:$src1_modifiers)))))),
+    (build_vector f16:$elt0, (f16 (fpround (fma_like (f32 (VOP3PMadMixMods f16:$src0, i32:$src0_modifiers)),
+                                                (f32 (VOP3PMadMixMods f16:$src1, i32:$src1_modifiers)),
+                                                (f32 (VOP3PMadMixMods f16:$src2, i32:$src2_modifiers)))))),
     (v2f16 (mixhi_inst $src0_modifiers, $src0,
                        $src1_modifiers, $src1,
-                       (i32 0), (i32 0),
+                       $src2_modifiers, $src2,
                        DSTCLAMP.NONE,
-                       VGPR_32:$elt0))
+                       (REG_SEQUENCE VGPR_32, $elt0, lo16, (f16 (IMPLICIT_DEF)), hi16)))
+  >;
+
+  def : GCNPat <
+    (build_vector
+      f16:$elt0,
+      (AMDGPUclamp (f16 (fpround (fma_like (f32 (VOP3PMadMixMods f16:$src0, i32:$src0_modifiers)),
+                                      (f32 (VOP3PMadMixMods f16:$src1, i32:$src1_modifiers)),
+                                      (f32 (VOP3PMadMixMods f16:$src2, i32:$src2_modifiers))))))),
+    (v2f16 (mixhi_inst $src0_modifiers, $src0,
+                       $src1_modifiers, $src1,
+                       $src2_modifiers, $src2,
+                       DSTCLAMP.ENABLE,
+                       (REG_SEQUENCE VGPR_32, $elt0, lo16, (f16 (IMPLICIT_DEF)), hi16)))
   >;
+  } // end OtherPredicates
 }
 
 class MinimumMaximumByMinimum3Maximum3VOP3P<SDPatternOperator node,
@@ -266,7 +311,8 @@ def : MinimumMaximumByMinimum3Maximum3VOP3P<fminimum, V_PK_MINIMUM3_F16>;
 def : MinimumMaximumByMinimum3Maximum3VOP3P<fmaximum, V_PK_MAXIMUM3_F16>;
 }
 
-let SubtargetPredicate = HasMadMixInsts, OtherPredicates = [NoFP32Denormals] in {
+let SubtargetPredicate = HasMadMixInsts in {
+let OtherPredicates = [NoFP32Denormals] in {
 
 // These are VOP3a-like opcodes which accept no omod.
 // Size of src arguments (16/32) is controlled by op_sel.
@@ -284,9 +330,10 @@ defm V_MAD_MIXHI_F16 : VOP3_VOP3PInst<"v_mad_mixhi_f16", VOP3P_Mix_Profile<VOP_F
 }
 } // End FPDPRounding = 1
 }
+} // OtherPredicates = [NoFP32Denormals]
 
-defm : MadFmaMixPats<fmad, V_MAD_MIX_F32, V_MAD_MIXLO_F16, V_MAD_MIXHI_F16>;
-} // End SubtargetPredicate = HasMadMixInsts, OtherPredicates = [NoFP32Denormals]
+defm : MadFmaMixPats<fmad, V_MAD_MIX_F32, V_MAD_MIXLO_F16, V_MAD_MIXHI_F16, 0 /*HasFP32Denormals*/>;
+} // End SubtargetPredicate = HasMadMixInsts
 
 
 // Essentially the same as the mad_mix versions
@@ -306,7 +353,7 @@ defm V_FMA_MIXHI_F16 : VOP3_VOP3PInst<"v_fma_mixhi_f16", VOP3P_Mix_Profile<VOP_F
 } // End FPDPRounding = 1
 }
 
-defm : MadFmaMixPats<fma, V_FMA_MIX_F32, V_FMA_MIXLO_F16, V_FMA_MIXHI_F16>;
+defm : MadFmaMixPats<fma, V_FMA_MIX_F32, V_FMA_MIXLO_F16, V_FMA_MIXHI_F16, 1 /*HasPF32Denormals*/>;
 }
 
 // Defines patterns that extract signed 4bit from each Idx[0].
diff --git a/llvm/lib/Target/AMDGPU/VOPCInstructions.td b/llvm/lib/Target/AMDGPU/VOPCInstructions.td
index aa930249c5003..ca5ed5cd24603 100644
--- a/llvm/lib/Target/AMDGPU/VOPCInstructions.td
+++ b/llvm/lib/Target/AMDGPU/VOPCInstructions.td
@@ -389,10 +389,10 @@ multiclass VOPC_Pseudos <string opName,
     let SchedRW = P.Schedule;
     let isCompare = 1;
     let isCommutable = 1;
-    let AsmMatchConverter =
-        !if (P.HasOpSel, "cvtVOP3OpSel",
-             !if (!or(P.HasModifiers, P.HasOMod, P.HasIntClamp), "cvtVOP3",
-                  ""));
+    let AsmMatchConverter = !cond(
+        P.HasOpSel : "cvtVOP3OpSel",
+        !or(P.HasModifiers, P.HasOMod, P.HasIntClamp) : "cvtVOP3",
+        1 : "");
   }
 
   if P.HasExtSDWA then
@@ -454,6 +454,10 @@ multiclass VOPCX_Pseudos <string opName,
     let isCommutable = 1;
     let SubtargetPredicate = HasNoSdstCMPX;
     let IsVCMPX = 1;
+    let AsmMatchConverter = !cond(
+        P_NoSDst.HasOpSel : "cvtVOP3OpSel",
+        !or(P_NoSDst.HasModifiers, P_NoSDst.HasOMod, P_NoSDst.HasIntClamp) : "cvtVOP3",
+        1 : "");
   }
 
   if P_NoSDst.HasExtSDWA then
@@ -1035,6 +1039,20 @@ multiclass VOPCClassPat64<string inst_name> {
   >;
 }
 
+multiclass VOPCClassPat64_t16<string inst_name> {
+  defvar inst = !cast<VOP_Pseudo>(inst_name#"_t16_e64");
+  defvar P = inst.Pfl;
+  def : GCNPat <
+    (i1:$sdst
+      (AMDGPUfp_class
+        (P.Src0VT (VOP3ModsNonCanonicalizing P.Src0VT:$src0, i32:$src0_modifiers)),
+        i32:$src1)),
+    (inst i32:$src0_modifiers, VSrcT_f16:$src0,
+          0 /* src1_modifiers */, (f16 (EXTRACT_SUBREG VGPR_32:$src1, lo16)),
+          0) /* op_sel */
+  >;
+}
+
 multiclass VOPCClassPat64_fake16<string inst_name> {
   defvar inst = !cast<VOP_Pseudo>(inst_name#"_fake16_e64");
   defvar P = inst.Pfl;
@@ -1065,6 +1083,10 @@ multiclass VOPC_Class_Pseudos <string opName, VOPC_Profile p, bit DefExec,
              VCMPXNoSDstTable<1, opName#"_e64"> {
     let Defs = !if(DefExec, [EXEC], []);
     let SchedRW = p.Schedule;
+    let AsmMatchConverter = !cond(
+        p.HasOpSel : "cvtVOP3OpSel",
+        !or(p.HasModifiers, p.HasOMod, p.HasIntClamp) : "cvtVOP3",
+        1 : "");
   }
 
   if p.HasExtSDWA then
@@ -1113,6 +1135,10 @@ multiclass VOPCX_Class_Pseudos <string opName,
     let Defs = [EXEC];
     let SchedRW = P_NoSDst.Schedule;
     let SubtargetPredicate = HasNoSdstCMPX;
+    let AsmMatchConverter = !cond(
+       P_NoSDst.HasOpSel : "cvtVOP3OpSel",
+       !or(P_NoSDst.HasModifiers, P_NoSDst.HasOMod, P_NoSDst.HasIntClamp) : "cvtVOP3",
+       1 : "");
   }
 
   if P_NoSDst.HasExtSDWA then
@@ -1158,6 +1184,7 @@ multiclass VOPC_CLASS_F16 <string opName> {
   }
   let True16Predicate = UseRealTrue16Insts in {
     defm _t16 : VOPC_Class_Pseudos <opName#"_t16", VOPC_I1_F16_I16_t16, 0>;
+    defm : VOPCClassPat64_t16<NAME>;
   }
   let True16Predicate = UseFakeTrue16Insts in {
     defm _fake16 : VOPC_Class_Pseudos <opName#"_fake16", VOPC_I1_F16_I16_fake16, 0>;
@@ -1207,27 +1234,30 @@ defm V_CMPX_CLASS_F16 : VOPCX_CLASS_F16 <"v_cmpx_class_f16">;
 
 // We need to use COPY_TO_REGCLASS to w/a the problem when ReplaceAllUsesWith()
 // complaints it cannot replace i1 <-> i64/i32 if node was not morphed in place.
-multiclass ICMP_Pattern <PatFrags cond, Instruction inst, ValueType vt> {
+multiclass ICMP_Pattern <PatFrags cond, Instruction inst, ValueType vt, dag dstInst = (inst $src0, $src1)> {
   let WaveSizePredicate = isWave64 in
   def : GCNPat <
     (i64 (AMDGPUsetcc vt:$src0, vt:$src1, cond)),
-    (i64 (COPY_TO_REGCLASS (inst $src0, $src1), SReg_64))
+    (i64 (COPY_TO_REGCLASS dstInst, SReg_64))
   >;
 
   let WaveSizePredicate = isWave32 in {
     def : GCNPat <
       (i32 (AMDGPUsetcc vt:$src0, vt:$src1, cond)),
-      (i32 (COPY_TO_REGCLASS (inst $src0, $src1), SReg_32))
+      (i32 (COPY_TO_REGCLASS dstInst, SReg_32))
     >;
 
     // Support codegen of i64 setcc in wave32 mode.
     def : GCNPat <
       (i64 (AMDGPUsetcc vt:$src0, vt:$src1, cond)),
-      (i64 (REG_SEQUENCE SReg_64, (inst $src0, $src1), sub0, (S_MOV_B32 (i32 0)), sub1))
+      (i64 (REG_SEQUENCE SReg_64, dstInst, sub0, (S_MOV_B32 (i32 0)), sub1))
     >;
   }
 }
 
+multiclass ICMP_Pattern_t16<PatFrags cond, Instruction inst, ValueType vt>
+    : ICMP_Pattern<cond, inst, vt, (inst 0, $src0, 0, $src1)>;
+
 defm : ICMP_Pattern <COND_EQ, V_CMP_EQ_U32_e64, i32>;
 defm : ICMP_Pattern <COND_NE, V_CMP_NE_U32_e64, i32>;
 defm : ICMP_Pattern <COND_UGT, V_CMP_GT_U32_e64, i32>;
@@ -1250,6 +1280,19 @@ defm : ICMP_Pattern <COND_SGE, V_CMP_GE_I64_e64, i64>;
 defm : ICMP_Pattern <COND_SLT, V_CMP_LT_I64_e64, i64>;
 defm : ICMP_Pattern <COND_SLE, V_CMP_LE_I64_e64, i64>;
 
+let True16Predicate = UseRealTrue16Insts in {
+defm : ICMP_Pattern_t16 <COND_EQ, V_CMP_EQ_U16_t16_e64, i16>;
+defm : ICMP_Pattern_t16 <COND_NE, V_CMP_NE_U16_t16_e64, i16>;
+defm : ICMP_Pattern_t16 <COND_UGT, V_CMP_GT_U16_t16_e64, i16>;
+defm : ICMP_Pattern_t16 <COND_UGE, V_CMP_GE_U16_t16_e64, i16>;
+defm : ICMP_Pattern_t16 <COND_ULT, V_CMP_LT_U16_t16_e64, i16>;
+defm : ICMP_Pattern_t16 <COND_ULE, V_CMP_LE_U16_t16_e64, i16>;
+defm : ICMP_Pattern_t16 <COND_SGT, V_CMP_GT_I16_t16_e64, i16>;
+defm : ICMP_Pattern_t16 <COND_SGE, V_CMP_GE_I16_t16_e64, i16>;
+defm : ICMP_Pattern_t16 <COND_SLT, V_CMP_LT_I16_t16_e64, i16>;
+defm : ICMP_Pattern_t16 <COND_SLE, V_CMP_LE_I16_t16_e64, i16>;
+} // End True16Predicate = UseRealTrue16Insts
+
 let True16Predicate = UseFakeTrue16Insts in {
 defm : ICMP_Pattern <COND_EQ, V_CMP_EQ_U16_fake16_e64, i16>;
 defm : ICMP_Pattern <COND_NE, V_CMP_NE_U16_fake16_e64, i16>;
@@ -1335,6 +1378,24 @@ defm : FCMP_Pattern <COND_UGE, V_CMP_NLT_F64_e64, f64>;
 defm : FCMP_Pattern <COND_ULT, V_CMP_NGE_F64_e64, f64>;
 defm : FCMP_Pattern <COND_ULE, V_CMP_NGT_F64_e64, f64>;
 
+let True16Predicate = UseRealTrue16Insts in {
+defm : FCMP_Pattern <COND_O, V_CMP_O_F16_t16_e64, f16>;
+defm : FCMP_Pattern <COND_UO, V_CMP_U_F16_t16_e64, f16>;
+defm : FCMP_Pattern <COND_OEQ, V_CMP_EQ_F16_t16_e64, f16>;
+defm : FCMP_Pattern <COND_ONE, V_CMP_NEQ_F16_t16_e64, f16>;
+defm : FCMP_Pattern <COND_OGT, V_CMP_GT_F16_t16_e64, f16>;
+defm : FCMP_Pattern <COND_OGE, V_CMP_GE_F16_t16_e64, f16>;
+defm : FCMP_Pattern <COND_OLT, V_CMP_LT_F16_t16_e64, f16>;
+defm : FCMP_Pattern <COND_OLE, V_CMP_LE_F16_t16_e64, f16>;
+
+defm : FCMP_Pattern <COND_UEQ, V_CMP_NLG_F16_t16_e64, f16>;
+defm : FCMP_Pattern <COND_UNE, V_CMP_NEQ_F16_t16_e64, f16>;
+defm : FCMP_Pattern <COND_UGT, V_CMP_NLE_F16_t16_e64, f16>;
+defm : FCMP_Pattern <COND_UGE, V_CMP_NLT_F16_t16_e64, f16>;
+defm : FCMP_Pattern <COND_ULT, V_CMP_NGE_F16_t16_e64, f16>;
+defm : FCMP_Pattern <COND_ULE, V_CMP_NGT_F16_t16_e64, f16>;
+} // End True16Predicate = UseRealTrue16Insts
+
 let True16Predicate = UseFakeTrue16Insts in {
 defm : FCMP_Pattern <COND_O, V_CMP_O_F16_fake16_e64, f16>;
 defm : FCMP_Pattern <COND_UO, V_CMP_U_F16_fake16_e64, f16>;
@@ -1975,22 +2036,22 @@ defm V_CMP_CLASS_F16     : VOPC_Real_t16_and_fake16_gfx11_gfx12<0x07d, "v_cmp_cl
 defm V_CMP_CLASS_F32     : VOPC_Real_gfx11_gfx12<0x07e>;
 defm V_CMP_CLASS_F64     : VOPC_Real_gfx11_gfx12<0x07f>;
 
-defm V_CMPX_F_F16_fake16     : VOPCX_Real_t16_gfx11<0x080, "v_cmpx_f_f16">;
+defm V_CMPX_F_F16            : VOPCX_Real_t16_and_fake16_gfx11<0x080, "v_cmpx_f_f16">;
 defm V_CMPX_LT_F16           : VOPCX_Real_t16_and_fake16_gfx11_gfx12<0x081, "v_cmpx_lt_f16">;
-defm V_CMPX_EQ_F16_fake16    : VOPCX_Real_t16_gfx11_gfx12<0x082, "v_cmpx_eq_f16">;
-defm V_CMPX_LE_F16_fake16    : VOPCX_Real_t16_gfx11_gfx12<0x083, "v_cmpx_le_f16">;
-defm V_CMPX_GT_F16_fake16    : VOPCX_Real_t16_gfx11_gfx12<0x084, "v_cmpx_gt_f16">;
-defm V_CMPX_LG_F16_fake16    : VOPCX_Real_t16_gfx11_gfx12<0x085, "v_cmpx_lg_f16">;
-defm V_CMPX_GE_F16_fake16    : VOPCX_Real_t16_gfx11_gfx12<0x086, "v_cmpx_ge_f16">;
-defm V_CMPX_O_F16_fake16     : VOPCX_Real_t16_gfx11_gfx12<0x087, "v_cmpx_o_f16">;
-defm V_CMPX_U_F16_fake16     : VOPCX_Real_t16_gfx11_gfx12<0x088, "v_cmpx_u_f16">;
-defm V_CMPX_NGE_F16_fake16   : VOPCX_Real_t16_gfx11_gfx12<0x089, "v_cmpx_nge_f16">;
-defm V_CMPX_NLG_F16_fake16   : VOPCX_Real_t16_gfx11_gfx12<0x08a, "v_cmpx_nlg_f16">;
-defm V_CMPX_NGT_F16_fake16   : VOPCX_Real_t16_gfx11_gfx12<0x08b, "v_cmpx_ngt_f16">;
-defm V_CMPX_NLE_F16_fake16   : VOPCX_Real_t16_gfx11_gfx12<0x08c, "v_cmpx_nle_f16">;
-defm V_CMPX_NEQ_F16_fake16   : VOPCX_Real_t16_gfx11_gfx12<0x08d, "v_cmpx_neq_f16">;
-defm V_CMPX_NLT_F16_fake16   : VOPCX_Real_t16_gfx11_gfx12<0x08e, "v_cmpx_nlt_f16">;
-defm V_CMPX_T_F16_fake16     : VOPCX_Real_with_name_gfx11<0x08f, "V_CMPX_TRU_F16_fake16", "v_cmpx_t_f16", "v_cmpx_tru_f16">;
+defm V_CMPX_EQ_F16           : VOPCX_Real_t16_and_fake16_gfx11_gfx12<0x082, "v_cmpx_eq_f16">;
+defm V_CMPX_LE_F16           : VOPCX_Real_t16_and_fake16_gfx11_gfx12<0x083, "v_cmpx_le_f16">;
+defm V_CMPX_GT_F16           : VOPCX_Real_t16_and_fake16_gfx11_gfx12<0x084, "v_cmpx_gt_f16">;
+defm V_CMPX_LG_F16           : VOPCX_Real_t16_and_fake16_gfx11_gfx12<0x085, "v_cmpx_lg_f16">;
+defm V_CMPX_GE_F16           : VOPCX_Real_t16_and_fake16_gfx11_gfx12<0x086, "v_cmpx_ge_f16">;
+defm V_CMPX_O_F16            : VOPCX_Real_t16_and_fake16_gfx11_gfx12<0x087, "v_cmpx_o_f16">;
+defm V_CMPX_U_F16            : VOPCX_Real_t16_and_fake16_gfx11_gfx12<0x088, "v_cmpx_u_f16">;
+defm V_CMPX_NGE_F16          : VOPCX_Real_t16_and_fake16_gfx11_gfx12<0x089, "v_cmpx_nge_f16">;
+defm V_CMPX_NLG_F16          : VOPCX_Real_t16_and_fake16_gfx11_gfx12<0x08a, "v_cmpx_nlg_f16">;
+defm V_CMPX_NGT_F16          : VOPCX_Real_t16_and_fake16_gfx11_gfx12<0x08b, "v_cmpx_ngt_f16">;
+defm V_CMPX_NLE_F16          : VOPCX_Real_t16_and_fake16_gfx11_gfx12<0x08c, "v_cmpx_nle_f16">;
+defm V_CMPX_NEQ_F16          : VOPCX_Real_t16_and_fake16_gfx11_gfx12<0x08d, "v_cmpx_neq_f16">;
+defm V_CMPX_NLT_F16          : VOPCX_Real_t16_and_fake16_gfx11_gfx12<0x08e, "v_cmpx_nlt_f16">;
+defm V_CMPX_T_F16            : VOPCX_Real_t16_and_fake16_gfx11<0x08f, "v_cmpx_t_f16", "V_CMPX_TRU_F16", "v_cmpx_tru_f16">;
 
 defm V_CMPX_F_F32     : VOPCX_Real_gfx11<0x090>;
 defm V_CMPX_LT_F32    : VOPCX_Real_gfx11_gfx12<0x091>;
diff --git a/llvm/lib/Target/AMDGPU/VOPInstructions.td b/llvm/lib/Target/AMDGPU/VOPInstructions.td
index 3b5358b737aa4..d6ad01c8f9b35 100644
--- a/llvm/lib/Target/AMDGPU/VOPInstructions.td
+++ b/llvm/lib/Target/AMDGPU/VOPInstructions.td
@@ -1947,9 +1947,6 @@ multiclass VOP3Only_Realtriple_gfx12<bits<10> op, bit isSingle = 0> :
 multiclass VOP3Only_Real_Base_gfx12<bits<10> op> :
   VOP3_Real_Base<GFX12Gen, op, NAME, 1/*IsSingle*/>;
 
-multiclass VOP3Only_Realtriple_t16_gfx12<bits<10> op> :
-  VOP3Only_Realtriple<GFX12Gen, op>;
-
 multiclass VOP3_Realtriple_t16_gfx12<bits<10> op, string asmName, string opName = NAME,
                                      string pseudo_mnemonic = "", bit isSingle = 0> :
   VOP3_Realtriple_with_name<GFX12Gen, op, opName, asmName, pseudo_mnemonic, isSingle>;
@@ -1960,6 +1957,16 @@ multiclass VOP3_Realtriple_t16_and_fake16_gfx12<bits<10> op, string asmName, str
   defm _fake16:VOP3_Realtriple_t16_gfx12<op, asmName, opName#"_fake16", pseudo_mnemonic, isSingle>;
 }
 
+multiclass VOP3Only_Realtriple_t16_gfx12<bits<10> op, string asmName,
+                                     string opName = NAME, string pseudo_mnemonic = "">
+  : VOP3_Realtriple_t16_gfx12<op, asmName, opName, pseudo_mnemonic, 1>;
+
+multiclass VOP3Only_Realtriple_t16_and_fake16_gfx12<bits<10> op, string asmName,
+                                     string opName = NAME, string pseudo_mnemonic = ""> {
+  defm _t16 : VOP3Only_Realtriple_t16_gfx12<op, asmName, opName#"_t16", pseudo_mnemonic>;
+  defm _fake16 : VOP3Only_Realtriple_t16_gfx12<op, asmName, opName#"_fake16", pseudo_mnemonic>;
+}
+
 multiclass VOP3be_Real_with_name_gfx12<bits<10> op, string opName,
                                        string asmName, bit isSingle = 0> {
   defvar ps = !cast<VOP3_Pseudo>(opName#"_e64");
@@ -2012,6 +2019,10 @@ def : ClassPat<V_CMP_CLASS_F16_e64, f16> {
   let True16Predicate = NotHasTrue16BitInsts;
 }
 
+def : ClassPat_t16<V_CMP_CLASS_F16_t16_e64, f16> {
+  let True16Predicate = UseRealTrue16Insts;
+}
+
 def : ClassPat_t16<V_CMP_CLASS_F16_fake16_e64, f16> {
   let True16Predicate = UseFakeTrue16Insts;
 }
diff --git a/llvm/lib/Target/ARM/ARMAsmPrinter.cpp b/llvm/lib/Target/ARM/ARMAsmPrinter.cpp
index af47d0e176f36..b8772e1665f8a 100644
--- a/llvm/lib/Target/ARM/ARMAsmPrinter.cpp
+++ b/llvm/lib/Target/ARM/ARMAsmPrinter.cpp
@@ -63,9 +63,11 @@ void ARMAsmPrinter::emitFunctionBodyEnd() {
 }
 
 void ARMAsmPrinter::emitFunctionEntryLabel() {
+  auto &TS =
+      static_cast<ARMTargetStreamer &>(*OutStreamer->getTargetStreamer());
   if (AFI->isThumbFunction()) {
     OutStreamer->emitAssemblerFlag(MCAF_Code16);
-    OutStreamer->emitThumbFunc(CurrentFnSym);
+    TS.emitThumbFunc(CurrentFnSym);
   } else {
     OutStreamer->emitAssemblerFlag(MCAF_Code32);
   }
diff --git a/llvm/lib/Target/ARM/ARMISelLowering.cpp b/llvm/lib/Target/ARM/ARMISelLowering.cpp
index bd8d6079e1ba8..5c4fe9d922f4c 100644
--- a/llvm/lib/Target/ARM/ARMISelLowering.cpp
+++ b/llvm/lib/Target/ARM/ARMISelLowering.cpp
@@ -8479,7 +8479,7 @@ bool ARMTargetLowering::isShuffleMaskLegal(ArrayRef<int> M, EVT VT) const {
 
   unsigned EltSize = VT.getScalarSizeInBits();
   if (EltSize >= 32 ||
-      ShuffleVectorSDNode::isSplatMask(&M[0], VT) ||
+      ShuffleVectorSDNode::isSplatMask(M) ||
       ShuffleVectorInst::isIdentityMask(M, M.size()) ||
       isVREVMask(M, VT, 64) ||
       isVREVMask(M, VT, 32) ||
diff --git a/llvm/lib/Target/ARM/ARMLatencyMutations.h b/llvm/lib/Target/ARM/ARMLatencyMutations.h
index a4b8de0be51f7..b22025aedf854 100644
--- a/llvm/lib/Target/ARM/ARMLatencyMutations.h
+++ b/llvm/lib/Target/ARM/ARMLatencyMutations.h
@@ -47,7 +47,7 @@ class ARMOverrideBypasses : public ScheduleDAGMutation {
 
 /// Note that you have to add:
 ///   DAG.addMutation(createARMLatencyMutation(ST, AA));
-/// to ARMPassConfig::createMachineScheduler() to have an effect.
+/// to ARMTargetMachine::createMachineScheduler() to have an effect.
 std::unique_ptr<ScheduleDAGMutation>
 createARMLatencyMutations(const class ARMSubtarget &, AAResults *AA);
 
diff --git a/llvm/lib/Target/ARM/ARMMacroFusion.h b/llvm/lib/Target/ARM/ARMMacroFusion.h
index 4896a4a2544db..b8642dc536c89 100644
--- a/llvm/lib/Target/ARM/ARMMacroFusion.h
+++ b/llvm/lib/Target/ARM/ARMMacroFusion.h
@@ -20,7 +20,7 @@ namespace llvm {
 
 /// Note that you have to add:
 ///   DAG.addMutation(createARMMacroFusionDAGMutation());
-/// to ARMPassConfig::createMachineScheduler() to have an effect.
+/// to ARMTargetMachine::createMachineScheduler() to have an effect.
 std::unique_ptr<ScheduleDAGMutation> createARMMacroFusionDAGMutation();
 
 } // llvm
diff --git a/llvm/lib/Target/ARM/ARMProcessors.td b/llvm/lib/Target/ARM/ARMProcessors.td
index 213b104bcfdb2..7453727a7cff0 100644
--- a/llvm/lib/Target/ARM/ARMProcessors.td
+++ b/llvm/lib/Target/ARM/ARMProcessors.td
@@ -377,6 +377,7 @@ def : ProcessorModel<"star-mc1", CortexM4Model,         [ARMv8mMainline,
                                                          FeatureHasSlowFPVFMx,
                                                          FeatureUseMISched,
                                                          FeatureHasNoBranchPredictor,
+                                                         FeatureAvoidMULS,
                                                          FeatureFixCMSE_CVE_2021_35465]>;
 
 def : ProcessorModel<"cortex-m35p", CortexM4Model,      [ARMv8mMainline,
diff --git a/llvm/lib/Target/ARM/ARMTargetMachine.cpp b/llvm/lib/Target/ARM/ARMTargetMachine.cpp
index 8e7b756b67905..98bdf310dea91 100644
--- a/llvm/lib/Target/ARM/ARMTargetMachine.cpp
+++ b/llvm/lib/Target/ARM/ARMTargetMachine.cpp
@@ -325,6 +325,28 @@ ARMBaseTargetMachine::getTargetTransformInfo(const Function &F) const {
   return TargetTransformInfo(ARMTTIImpl(this, F));
 }
 
+ScheduleDAGInstrs *
+ARMBaseTargetMachine::createMachineScheduler(MachineSchedContext *C) const {
+  ScheduleDAGMILive *DAG = createGenericSchedLive(C);
+  // add DAG Mutations here.
+  const ARMSubtarget &ST = C->MF->getSubtarget<ARMSubtarget>();
+  if (ST.hasFusion())
+    DAG->addMutation(createARMMacroFusionDAGMutation());
+  return DAG;
+}
+
+ScheduleDAGInstrs *
+ARMBaseTargetMachine::createPostMachineScheduler(MachineSchedContext *C) const {
+  ScheduleDAGMI *DAG = createGenericSchedPostRA(C);
+  // add DAG Mutations here.
+  const ARMSubtarget &ST = C->MF->getSubtarget<ARMSubtarget>();
+  if (ST.hasFusion())
+    DAG->addMutation(createARMMacroFusionDAGMutation());
+  if (auto Mutation = createARMLatencyMutations(ST, C->AA))
+    DAG->addMutation(std::move(Mutation));
+  return DAG;
+}
+
 ARMLETargetMachine::ARMLETargetMachine(const Target &T, const Triple &TT,
                                        StringRef CPU, StringRef FS,
                                        const TargetOptions &Options,
@@ -353,28 +375,6 @@ class ARMPassConfig : public TargetPassConfig {
     return getTM<ARMBaseTargetMachine>();
   }
 
-  ScheduleDAGInstrs *
-  createMachineScheduler(MachineSchedContext *C) const override {
-    ScheduleDAGMILive *DAG = createGenericSchedLive(C);
-    // add DAG Mutations here.
-    const ARMSubtarget &ST = C->MF->getSubtarget<ARMSubtarget>();
-    if (ST.hasFusion())
-      DAG->addMutation(createARMMacroFusionDAGMutation());
-    return DAG;
-  }
-
-  ScheduleDAGInstrs *
-  createPostMachineScheduler(MachineSchedContext *C) const override {
-    ScheduleDAGMI *DAG = createGenericSchedPostRA(C);
-    // add DAG Mutations here.
-    const ARMSubtarget &ST = C->MF->getSubtarget<ARMSubtarget>();
-    if (ST.hasFusion())
-      DAG->addMutation(createARMMacroFusionDAGMutation());
-    if (auto Mutation = createARMLatencyMutations(ST, C->AA))
-      DAG->addMutation(std::move(Mutation));
-    return DAG;
-  }
-
   void addIRPasses() override;
   void addCodeGenPrepare() override;
   bool addPreISel() override;
diff --git a/llvm/lib/Target/ARM/ARMTargetMachine.h b/llvm/lib/Target/ARM/ARMTargetMachine.h
index 94d48fcae15eb..99fd817c81f89 100644
--- a/llvm/lib/Target/ARM/ARMTargetMachine.h
+++ b/llvm/lib/Target/ARM/ARMTargetMachine.h
@@ -96,6 +96,10 @@ class ARMBaseTargetMachine : public CodeGenTargetMachineImpl {
                                 PerFunctionMIParsingState &PFS,
                                 SMDiagnostic &Error,
                                 SMRange &SourceRange) const override;
+  ScheduleDAGInstrs *
+  createMachineScheduler(MachineSchedContext *C) const override;
+  ScheduleDAGInstrs *
+  createPostMachineScheduler(MachineSchedContext *C) const override;
 };
 
 /// ARM/Thumb little endian target machine.
diff --git a/llvm/lib/Target/ARM/AsmParser/ARMAsmParser.cpp b/llvm/lib/Target/ARM/AsmParser/ARMAsmParser.cpp
index dad91c6a969e8..6cc95b7e1cd2a 100644
--- a/llvm/lib/Target/ARM/AsmParser/ARMAsmParser.cpp
+++ b/llvm/lib/Target/ARM/AsmParser/ARMAsmParser.cpp
@@ -8652,6 +8652,37 @@ bool ARMAsmParser::validateInstruction(MCInst &Inst,
                    "coprocessor must be configured as GCP");
     break;
   }
+
+  case ARM::VTOSHH:
+  case ARM::VTOUHH:
+  case ARM::VTOSLH:
+  case ARM::VTOULH:
+  case ARM::VTOSHS:
+  case ARM::VTOUHS:
+  case ARM::VTOSLS:
+  case ARM::VTOULS:
+  case ARM::VTOSHD:
+  case ARM::VTOUHD:
+  case ARM::VTOSLD:
+  case ARM::VTOULD:
+  case ARM::VSHTOH:
+  case ARM::VUHTOH:
+  case ARM::VSLTOH:
+  case ARM::VULTOH:
+  case ARM::VSHTOS:
+  case ARM::VUHTOS:
+  case ARM::VSLTOS:
+  case ARM::VULTOS:
+  case ARM::VSHTOD:
+  case ARM::VUHTOD:
+  case ARM::VSLTOD:
+  case ARM::VULTOD: {
+    if (Operands[MnemonicOpsEndInd]->getReg() !=
+        Operands[MnemonicOpsEndInd + 1]->getReg())
+      return Error(Operands[MnemonicOpsEndInd]->getStartLoc(),
+                   "source and destination registers must be the same");
+    break;
+  }
   }
 
   return false;
@@ -11681,7 +11712,7 @@ void ARMAsmParser::doBeforeLabelEmit(MCSymbol *Symbol, SMLoc IDLoc) {
 
 void ARMAsmParser::onLabelParsed(MCSymbol *Symbol) {
   if (NextSymbolIsThumb) {
-    getParser().getStreamer().emitThumbFunc(Symbol);
+    getTargetStreamer().emitThumbFunc(Symbol);
     NextSymbolIsThumb = false;
   }
 }
@@ -11701,7 +11732,7 @@ bool ARMAsmParser::parseDirectiveThumbFunc(SMLoc L) {
         Parser.getTok().is(AsmToken::String)) {
       MCSymbol *Func = getParser().getContext().getOrCreateSymbol(
           Parser.getTok().getIdentifier());
-      getParser().getStreamer().emitThumbFunc(Func);
+      getTargetStreamer().emitThumbFunc(Func);
       Parser.Lex();
       if (parseEOL())
         return true;
diff --git a/llvm/lib/Target/ARM/MCTargetDesc/ARMAsmBackend.cpp b/llvm/lib/Target/ARM/MCTargetDesc/ARMAsmBackend.cpp
index 6e2886a192923..dfaad24e1b877 100644
--- a/llvm/lib/Target/ARM/MCTargetDesc/ARMAsmBackend.cpp
+++ b/llvm/lib/Target/ARM/MCTargetDesc/ARMAsmBackend.cpp
@@ -1395,7 +1395,7 @@ static MCAsmBackend *createARMAsmBackend(const Target &T,
   case Triple::ELF:
     assert(TheTriple.isOSBinFormatELF() && "using ELF for non-ELF target");
     uint8_t OSABI = Options.FDPIC
-                        ? ELF::ELFOSABI_ARM_FDPIC
+                        ? static_cast<uint8_t>(ELF::ELFOSABI_ARM_FDPIC)
                         : MCELFObjectTargetWriter::getOSABI(TheTriple.getOS());
     return new ARMAsmBackendELF(T, STI.getTargetTriple().isThumb(), OSABI,
                                 Endian);
diff --git a/llvm/lib/Target/ARM/MCTargetDesc/ARMELFStreamer.cpp b/llvm/lib/Target/ARM/MCTargetDesc/ARMELFStreamer.cpp
index c528526382a2b..ac30a0b7c2a01 100644
--- a/llvm/lib/Target/ARM/MCTargetDesc/ARMELFStreamer.cpp
+++ b/llvm/lib/Target/ARM/MCTargetDesc/ARMELFStreamer.cpp
@@ -100,6 +100,7 @@ class ARMTargetAsmStreamer : public ARMTargetStreamer {
   void finishAttributeSection() override;
 
   void annotateTLSDescriptorSequence(const MCSymbolRefExpr *SRE) override;
+  void emitThumbFunc(MCSymbol *Symbol) override;
   void emitThumbSet(MCSymbol *Symbol, const MCExpr *Value) override;
 
   void emitARMWinCFIAllocStack(unsigned Size, bool Wide) override;
@@ -260,6 +261,17 @@ void ARMTargetAsmStreamer::annotateTLSDescriptorSequence(
   OS << "\t.tlsdescseq\t" << S->getSymbol().getName() << "\n";
 }
 
+void ARMTargetAsmStreamer::emitThumbFunc(MCSymbol *Symbol) {
+  const MCAsmInfo *MAI = Streamer.getContext().getAsmInfo();
+  OS << "\t.thumb_func";
+  // Only Mach-O hasSubsectionsViaSymbols()
+  if (MAI->hasSubsectionsViaSymbols()) {
+    OS << '\t';
+    Symbol->print(OS, MAI);
+  }
+  OS << '\n';
+}
+
 void ARMTargetAsmStreamer::emitThumbSet(MCSymbol *Symbol, const MCExpr *Value) {
   const MCAsmInfo *MAI = Streamer.getContext().getAsmInfo();
 
@@ -422,6 +434,7 @@ class ARMTargetELFStreamer : public ARMTargetStreamer {
   void emitLabel(MCSymbol *Symbol) override;
 
   void annotateTLSDescriptorSequence(const MCSymbolRefExpr *SRE) override;
+  void emitThumbFunc(MCSymbol *Symbol) override;
   void emitThumbSet(MCSymbol *Symbol, const MCExpr *Value) override;
 
   // Reset state between object emissions
@@ -686,11 +699,6 @@ class ARMELFStreamer : public MCELFStreamer {
     Symbol->setBinding(ELF::STB_LOCAL);
   }
 
-  void emitThumbFunc(MCSymbol *Func) override {
-    getAssembler().setIsThumbFunc(Func);
-    emitSymbolAttribute(Func, MCSA_ELF_TypeFunction);
-  }
-
   // Helper functions for ARM exception handling directives
   void EHReset();
 
@@ -1089,7 +1097,7 @@ void ARMTargetELFStreamer::emitLabel(MCSymbol *Symbol) {
   Streamer.getAssembler().registerSymbol(*Symbol);
   unsigned Type = cast<MCSymbolELF>(Symbol)->getType();
   if (Type == ELF::STT_FUNC || Type == ELF::STT_GNU_IFUNC)
-    Streamer.emitThumbFunc(Symbol);
+    emitThumbFunc(Symbol);
 }
 
 void ARMTargetELFStreamer::annotateTLSDescriptorSequence(
@@ -1097,6 +1105,11 @@ void ARMTargetELFStreamer::annotateTLSDescriptorSequence(
   getStreamer().EmitFixup(S, FK_Data_4);
 }
 
+void ARMTargetELFStreamer::emitThumbFunc(MCSymbol *Symbol) {
+  getStreamer().getAssembler().setIsThumbFunc(Symbol);
+  getStreamer().emitSymbolAttribute(Symbol, MCSA_ELF_TypeFunction);
+}
+
 void ARMTargetELFStreamer::emitThumbSet(MCSymbol *Symbol, const MCExpr *Value) {
   if (const MCSymbolRefExpr *SRE = dyn_cast<MCSymbolRefExpr>(Value)) {
     const MCSymbol &Sym = SRE->getSymbol();
@@ -1106,7 +1119,7 @@ void ARMTargetELFStreamer::emitThumbSet(MCSymbol *Symbol, const MCExpr *Value) {
     }
   }
 
-  getStreamer().emitThumbFunc(Symbol);
+  emitThumbFunc(Symbol);
   getStreamer().emitAssignment(Symbol, Value);
 }
 
diff --git a/llvm/lib/Target/ARM/MCTargetDesc/ARMMCTargetDesc.h b/llvm/lib/Target/ARM/MCTargetDesc/ARMMCTargetDesc.h
index 39b189780f1ff..5a67417351f81 100644
--- a/llvm/lib/Target/ARM/MCTargetDesc/ARMMCTargetDesc.h
+++ b/llvm/lib/Target/ARM/MCTargetDesc/ARMMCTargetDesc.h
@@ -71,6 +71,7 @@ MCTargetStreamer *createARMTargetAsmStreamer(MCStreamer &S,
 MCTargetStreamer *createARMObjectTargetStreamer(MCStreamer &S,
                                                 const MCSubtargetInfo &STI);
 MCTargetStreamer *createARMObjectTargetELFStreamer(MCStreamer &S);
+MCTargetStreamer *createARMObjectTargetMachOStreamer(MCStreamer &S);
 MCTargetStreamer *createARMObjectTargetWinCOFFStreamer(MCStreamer &S);
 
 MCCodeEmitter *createARMLEMCCodeEmitter(const MCInstrInfo &MCII,
diff --git a/llvm/lib/Target/ARM/MCTargetDesc/ARMMachObjectWriter.cpp b/llvm/lib/Target/ARM/MCTargetDesc/ARMMachObjectWriter.cpp
index 357654615e002..c37ed1198c3bb 100644
--- a/llvm/lib/Target/ARM/MCTargetDesc/ARMMachObjectWriter.cpp
+++ b/llvm/lib/Target/ARM/MCTargetDesc/ARMMachObjectWriter.cpp
@@ -16,8 +16,11 @@
 #include "llvm/MC/MCExpr.h"
 #include "llvm/MC/MCFixup.h"
 #include "llvm/MC/MCMachObjectWriter.h"
+#include "llvm/MC/MCObjectStreamer.h"
 #include "llvm/MC/MCSection.h"
+#include "llvm/MC/MCSymbolMachO.h"
 #include "llvm/MC/MCValue.h"
+#include "llvm/Support/Casting.h"
 #include "llvm/Support/ErrorHandling.h"
 
 using namespace llvm;
@@ -489,3 +492,23 @@ llvm::createARMMachObjectWriter(bool Is64Bit, uint32_t CPUType,
                                 uint32_t CPUSubtype) {
   return std::make_unique<ARMMachObjectWriter>(Is64Bit, CPUType, CPUSubtype);
 }
+
+namespace {
+class ARMTargetMachOStreamer : public ARMTargetStreamer {
+public:
+  ARMTargetMachOStreamer(MCStreamer &S) : ARMTargetStreamer(S) {}
+  MCObjectStreamer &getStreamer() {
+    return static_cast<MCObjectStreamer &>(Streamer);
+  }
+  void emitThumbFunc(MCSymbol *Symbol) override {
+    // Remember that the function is a thumb function. Fixup and relocation
+    // values will need adjusted.
+    getStreamer().getAssembler().setIsThumbFunc(Symbol);
+    cast<MCSymbolMachO>(Symbol)->setThumbFunc();
+  }
+};
+} // namespace
+
+MCTargetStreamer *llvm::createARMObjectTargetMachOStreamer(MCStreamer &S) {
+  return new ARMTargetMachOStreamer(S);
+}
diff --git a/llvm/lib/Target/ARM/MCTargetDesc/ARMTargetStreamer.cpp b/llvm/lib/Target/ARM/MCTargetDesc/ARMTargetStreamer.cpp
index 84a0fb44c9acd..1dc79438d2fc5 100644
--- a/llvm/lib/Target/ARM/MCTargetDesc/ARMTargetStreamer.cpp
+++ b/llvm/lib/Target/ARM/MCTargetDesc/ARMTargetStreamer.cpp
@@ -115,6 +115,7 @@ void ARMTargetStreamer::emitFPU(ARM::FPUKind FPU) {}
 void ARMTargetStreamer::finishAttributeSection() {}
 void ARMTargetStreamer::annotateTLSDescriptorSequence(
     const MCSymbolRefExpr *SRE) {}
+void ARMTargetStreamer::emitThumbFunc(MCSymbol *Symbol) {}
 void ARMTargetStreamer::emitThumbSet(MCSymbol *Symbol, const MCExpr *Value) {}
 
 void ARMTargetStreamer::emitARMWinCFIAllocStack(unsigned Size, bool Wide) {}
@@ -329,5 +330,7 @@ llvm::createARMObjectTargetStreamer(MCStreamer &S, const MCSubtargetInfo &STI) {
     return createARMObjectTargetELFStreamer(S);
   if (TT.isOSBinFormatCOFF())
     return createARMObjectTargetWinCOFFStreamer(S);
+  if (TT.isOSBinFormatMachO())
+    return createARMObjectTargetMachOStreamer(S);
   return new ARMTargetStreamer(S);
 }
diff --git a/llvm/lib/Target/ARM/MCTargetDesc/ARMWinCOFFStreamer.cpp b/llvm/lib/Target/ARM/MCTargetDesc/ARMWinCOFFStreamer.cpp
index e66059c2a0e09..6566aa4c243be 100644
--- a/llvm/lib/Target/ARM/MCTargetDesc/ARMWinCOFFStreamer.cpp
+++ b/llvm/lib/Target/ARM/MCTargetDesc/ARMWinCOFFStreamer.cpp
@@ -31,7 +31,6 @@ class ARMWinCOFFStreamer : public MCWinCOFFStreamer {
   void emitWindowsUnwindTables() override;
   void emitWindowsUnwindTables(WinEH::FrameInfo *Frame) override;
 
-  void emitThumbFunc(MCSymbol *Symbol) override;
   void finishImpl() override;
 };
 
@@ -54,10 +53,6 @@ void ARMWinCOFFStreamer::emitWindowsUnwindTables() {
   EHStreamer.Emit(*this);
 }
 
-void ARMWinCOFFStreamer::emitThumbFunc(MCSymbol *Symbol) {
-  getAssembler().setIsThumbFunc(Symbol);
-}
-
 void ARMWinCOFFStreamer::finishImpl() {
   emitFrames(nullptr);
   emitWindowsUnwindTables();
@@ -77,16 +72,14 @@ llvm::createARMWinCOFFStreamer(MCContext &Context,
 
 namespace {
 class ARMTargetWinCOFFStreamer : public llvm::ARMTargetStreamer {
-private:
-  // True if we are processing SEH directives in an epilogue.
-  bool InEpilogCFI = false;
-
-  // Symbol of the current epilog for which we are processing SEH directives.
-  MCSymbol *CurrentEpilog = nullptr;
-
 public:
   ARMTargetWinCOFFStreamer(llvm::MCStreamer &S) : ARMTargetStreamer(S) {}
 
+  ARMWinCOFFStreamer &getStreamer() {
+    return static_cast<ARMWinCOFFStreamer &>(Streamer);
+  }
+  void emitThumbFunc(MCSymbol *Symbol) override;
+
   // The unwind codes on ARM Windows are documented at
   // https://docs.microsoft.com/en-us/cpp/build/arm-exception-handling
   void emitARMWinCFIAllocStack(unsigned Size, bool Wide) override;
@@ -104,6 +97,10 @@ class ARMTargetWinCOFFStreamer : public llvm::ARMTargetStreamer {
   void emitARMWinUnwindCode(unsigned UnwindCode, int Reg, int Offset);
 };
 
+void ARMTargetWinCOFFStreamer::emitThumbFunc(MCSymbol *Symbol) {
+  getStreamer().getAssembler().setIsThumbFunc(Symbol);
+}
+
 // Helper function to common out unwind code setup for those codes that can
 // belong to both prolog and epilog.
 void ARMTargetWinCOFFStreamer::emitARMWinUnwindCode(unsigned UnwindCode,
@@ -114,8 +111,8 @@ void ARMTargetWinCOFFStreamer::emitARMWinUnwindCode(unsigned UnwindCode,
     return;
   MCSymbol *Label = S.emitCFILabel();
   auto Inst = WinEH::Instruction(UnwindCode, Label, Reg, Offset);
-  if (InEpilogCFI)
-    CurFrame->EpilogMap[CurrentEpilog].Instructions.push_back(Inst);
+  if (S.isInEpilogCFI())
+    CurFrame->EpilogMap[S.getCurrentEpilog()].Instructions.push_back(Inst);
   else
     CurFrame->Instructions.push_back(Inst);
 }
@@ -224,9 +221,10 @@ void ARMTargetWinCOFFStreamer::emitARMWinCFIEpilogStart(unsigned Condition) {
   if (!CurFrame)
     return;
 
-  InEpilogCFI = true;
-  CurrentEpilog = S.emitCFILabel();
-  CurFrame->EpilogMap[CurrentEpilog].Condition = Condition;
+  S.emitWinCFIBeginEpilogue();
+  if (S.isInEpilogCFI()) {
+    CurFrame->EpilogMap[S.getCurrentEpilog()].Condition = Condition;
+  }
 }
 
 void ARMTargetWinCOFFStreamer::emitARMWinCFIEpilogEnd() {
@@ -235,33 +233,26 @@ void ARMTargetWinCOFFStreamer::emitARMWinCFIEpilogEnd() {
   if (!CurFrame)
     return;
 
-  if (!CurrentEpilog) {
-    S.getContext().reportError(SMLoc(), "Stray .seh_endepilogue in " +
-                                            CurFrame->Function->getName());
-    return;
-  }
-
-  std::vector<WinEH::Instruction> &Epilog =
-      CurFrame->EpilogMap[CurrentEpilog].Instructions;
-
-  unsigned UnwindCode = Win64EH::UOP_End;
-  if (!Epilog.empty()) {
-    WinEH::Instruction EndInstr = Epilog.back();
-    if (EndInstr.Operation == Win64EH::UOP_Nop) {
-      UnwindCode = Win64EH::UOP_EndNop;
-      Epilog.pop_back();
-    } else if (EndInstr.Operation == Win64EH::UOP_WideNop) {
-      UnwindCode = Win64EH::UOP_WideEndNop;
-      Epilog.pop_back();
+  if (S.isInEpilogCFI()) {
+    std::vector<WinEH::Instruction> &Epilog =
+        CurFrame->EpilogMap[S.getCurrentEpilog()].Instructions;
+
+    unsigned UnwindCode = Win64EH::UOP_End;
+    if (!Epilog.empty()) {
+      WinEH::Instruction EndInstr = Epilog.back();
+      if (EndInstr.Operation == Win64EH::UOP_Nop) {
+        UnwindCode = Win64EH::UOP_EndNop;
+        Epilog.pop_back();
+      } else if (EndInstr.Operation == Win64EH::UOP_WideNop) {
+        UnwindCode = Win64EH::UOP_WideEndNop;
+        Epilog.pop_back();
+      }
     }
-  }
 
-  InEpilogCFI = false;
-  WinEH::Instruction Inst = WinEH::Instruction(UnwindCode, nullptr, -1, 0);
-  CurFrame->EpilogMap[CurrentEpilog].Instructions.push_back(Inst);
-  MCSymbol *Label = S.emitCFILabel();
-  CurFrame->EpilogMap[CurrentEpilog].End = Label;
-  CurrentEpilog = nullptr;
+    WinEH::Instruction Inst = WinEH::Instruction(UnwindCode, nullptr, -1, 0);
+    CurFrame->EpilogMap[S.getCurrentEpilog()].Instructions.push_back(Inst);
+  }
+  S.emitWinCFIEndEpilogue();
 }
 
 void ARMTargetWinCOFFStreamer::emitARMWinCFICustom(unsigned Opcode) {
diff --git a/llvm/lib/Target/BPF/BPFMIPeephole.cpp b/llvm/lib/Target/BPF/BPFMIPeephole.cpp
index 2b17f2aaefe2b..106572cdeb840 100644
--- a/llvm/lib/Target/BPF/BPFMIPeephole.cpp
+++ b/llvm/lib/Target/BPF/BPFMIPeephole.cpp
@@ -24,6 +24,7 @@
 #include "BPFInstrInfo.h"
 #include "BPFTargetMachine.h"
 #include "llvm/ADT/Statistic.h"
+#include "llvm/ADT/StringExtras.h"
 #include "llvm/CodeGen/LivePhysRegs.h"
 #include "llvm/CodeGen/MachineFrameInfo.h"
 #include "llvm/CodeGen/MachineFunctionPass.h"
@@ -322,6 +323,7 @@ struct BPFMIPreEmitPeephole : public MachineFunctionPass {
   bool eliminateRedundantMov();
   bool adjustBranch();
   bool insertMissingCallerSavedSpills();
+  bool removeMayGotoZero();
 
 public:
 
@@ -337,6 +339,7 @@ struct BPFMIPreEmitPeephole : public MachineFunctionPass {
     if (SupportGotol)
       Changed = adjustBranch() || Changed;
     Changed |= insertMissingCallerSavedSpills();
+    Changed |= removeMayGotoZero();
     return Changed;
   }
 };
@@ -682,6 +685,59 @@ bool BPFMIPreEmitPeephole::insertMissingCallerSavedSpills() {
   return Changed;
 }
 
+bool BPFMIPreEmitPeephole::removeMayGotoZero() {
+  bool Changed = false;
+  MachineBasicBlock *Prev_MBB, *Curr_MBB = nullptr;
+
+  for (MachineBasicBlock &MBB : make_early_inc_range(reverse(*MF))) {
+    Prev_MBB = Curr_MBB;
+    Curr_MBB = &MBB;
+    if (Prev_MBB == nullptr || Curr_MBB->empty())
+      continue;
+
+    MachineInstr &MI = Curr_MBB->back();
+    if (MI.getOpcode() != TargetOpcode::INLINEASM_BR)
+      continue;
+
+    const char *AsmStr = MI.getOperand(0).getSymbolName();
+    SmallVector<StringRef, 4> AsmPieces;
+    SplitString(AsmStr, AsmPieces, ";\n");
+
+    // Do not support multiple insns in one inline asm.
+    if (AsmPieces.size() != 1)
+      continue;
+
+    // The asm insn must be a may_goto insn.
+    SmallVector<StringRef, 4> AsmOpPieces;
+    SplitString(AsmPieces[0], AsmOpPieces, " ");
+    if (AsmOpPieces.size() != 2 || AsmOpPieces[0] != "may_goto")
+      continue;
+    // Enforce the format of 'may_goto <label>'.
+    if (AsmOpPieces[1] != "${0:l}" && AsmOpPieces[1] != "$0")
+      continue;
+
+    // Get the may_goto branch target.
+    MachineOperand &MO = MI.getOperand(InlineAsm::MIOp_FirstOperand + 1);
+    if (!MO.isMBB() || MO.getMBB() != Prev_MBB)
+      continue;
+
+    Changed = true;
+    if (Curr_MBB->begin() == MI) {
+      // Single 'may_goto' insn in the same basic block.
+      Curr_MBB->removeSuccessor(Prev_MBB);
+      for (MachineBasicBlock *Pred : Curr_MBB->predecessors())
+        Pred->replaceSuccessor(Curr_MBB, Prev_MBB);
+      Curr_MBB->eraseFromParent();
+      Curr_MBB = Prev_MBB;
+    } else {
+      // Remove 'may_goto' insn.
+      MI.eraseFromParent();
+    }
+  }
+
+  return Changed;
+}
+
 } // end default namespace
 
 INITIALIZE_PASS(BPFMIPreEmitPeephole, "bpf-mi-pemit-peephole",
diff --git a/llvm/lib/Target/CMakeLists.txt b/llvm/lib/Target/CMakeLists.txt
index 2739233f9ccb3..9472288229cac 100644
--- a/llvm/lib/Target/CMakeLists.txt
+++ b/llvm/lib/Target/CMakeLists.txt
@@ -4,7 +4,6 @@ list(APPEND LLVM_TABLEGEN_FLAGS -I ${LLVM_MAIN_SRC_DIR}/lib/Target)
 
 add_llvm_component_library(LLVMTarget
   Target.cpp
-  TargetIntrinsicInfo.cpp
   TargetLoweringObjectFile.cpp
   TargetMachine.cpp
   TargetMachineC.cpp
diff --git a/llvm/lib/Target/DirectX/DXIL.td b/llvm/lib/Target/DirectX/DXIL.td
index d099bb395449d..2f6b4d676edfd 100644
--- a/llvm/lib/Target/DirectX/DXIL.td
+++ b/llvm/lib/Target/DirectX/DXIL.td
@@ -50,6 +50,7 @@ def HandleTy : DXILOpParamType;
 def ResBindTy : DXILOpParamType;
 def ResPropsTy : DXILOpParamType;
 def SplitDoubleTy : DXILOpParamType;
+def BinaryWithCarryTy : DXILOpParamType;
 
 class DXILOpClass;
 
@@ -738,6 +739,18 @@ def UMin : DXILOp<40, binary> {
   let attributes = [Attributes<DXIL1_0, [ReadNone]>];
 }
 
+def UAddc : DXILOp<44, binaryWithCarryOrBorrow > {
+  let Doc = "Unsigned 32-bit integer arithmetic add with carry. uaddc(a,b) = (a+b, a+b overflowed ? 1 : 0)";
+  // TODO: This `let intrinsics = ...` line may be uncommented when 
+  // https://github.com/llvm/llvm-project/issues/113192 is fixed
+  // let intrinsics = [IntrinSelect<int_uadd_with_overflow>];
+  let arguments = [OverloadTy, OverloadTy];
+  let result = BinaryWithCarryTy;
+  let overloads = [Overloads<DXIL1_0, [Int32Ty]>];
+  let stages = [Stages<DXIL1_0, [all_stages]>];
+  let attributes = [Attributes<DXIL1_0, [ReadNone]>];
+}
+
 def FMad : DXILOp<46, tertiary> {
   let Doc = "Floating point arithmetic multiply/add operation. fmad(m,a,b) = m "
             "* a + b.";
@@ -1000,6 +1013,16 @@ def WaveActiveOp : DXILOp<119, waveActiveOp> {
                    IntrinArgIndex<0>, IntrinArgI8<WaveOpKind_Sum>,
                    IntrinArgI8<SignedOpKind_Unsigned>
                  ]>,
+    IntrinSelect<int_dx_wave_reduce_max,
+                 [
+                   IntrinArgIndex<0>, IntrinArgI8<WaveOpKind_Max>,
+                   IntrinArgI8<SignedOpKind_Signed>
+                 ]>,
+    IntrinSelect<int_dx_wave_reduce_umax,
+                 [
+                   IntrinArgIndex<0>, IntrinArgI8<WaveOpKind_Max>,
+                   IntrinArgI8<SignedOpKind_Unsigned>
+                 ]>,
   ];
 
   let arguments = [OverloadTy, Int8Ty, Int8Ty];
@@ -1008,7 +1031,7 @@ def WaveActiveOp : DXILOp<119, waveActiveOp> {
     Overloads<DXIL1_0, [HalfTy, FloatTy, DoubleTy, Int16Ty, Int32Ty, Int64Ty]>
   ];
   let stages = [Stages<DXIL1_0, [all_stages]>];
-  let attributes = [Attributes<DXIL1_0, [ReadNone]>];
+  let attributes = [Attributes<DXIL1_0, []>];
 }
 
 def WaveAllBitCount : DXILOp<135, waveAllOp> {
diff --git a/llvm/lib/Target/DirectX/DXILDataScalarization.cpp b/llvm/lib/Target/DirectX/DXILDataScalarization.cpp
index 2ab2daaff5b51..a0dd17904f6fa 100644
--- a/llvm/lib/Target/DirectX/DXILDataScalarization.cpp
+++ b/llvm/lib/Target/DirectX/DXILDataScalarization.cpp
@@ -91,7 +91,7 @@ bool DataScalarizerVisitor::visitLoadInst(LoadInst &LI) {
     if (CE && CE->getOpcode() == Instruction::GetElementPtr) {
       GetElementPtrInst *OldGEP =
           cast<GetElementPtrInst>(CE->getAsInstruction());
-      OldGEP->insertBefore(&LI);
+      OldGEP->insertBefore(LI.getIterator());
       IRBuilder<> Builder(&LI);
       LoadInst *NewLoad =
           Builder.CreateLoad(LI.getType(), OldGEP, LI.getName());
@@ -115,7 +115,7 @@ bool DataScalarizerVisitor::visitStoreInst(StoreInst &SI) {
     if (CE && CE->getOpcode() == Instruction::GetElementPtr) {
       GetElementPtrInst *OldGEP =
           cast<GetElementPtrInst>(CE->getAsInstruction());
-      OldGEP->insertBefore(&SI);
+      OldGEP->insertBefore(SI.getIterator());
       IRBuilder<> Builder(&SI);
       StoreInst *NewStore = Builder.CreateStore(SI.getValueOperand(), OldGEP);
       NewStore->setAlignment(SI.getAlign());
diff --git a/llvm/lib/Target/DirectX/DXILFlattenArrays.cpp b/llvm/lib/Target/DirectX/DXILFlattenArrays.cpp
index 53fc1c713a8c2..a3163a8969642 100644
--- a/llvm/lib/Target/DirectX/DXILFlattenArrays.cpp
+++ b/llvm/lib/Target/DirectX/DXILFlattenArrays.cpp
@@ -164,7 +164,7 @@ bool DXILFlattenArraysVisitor::visitLoadInst(LoadInst &LI) {
     if (CE && CE->getOpcode() == Instruction::GetElementPtr) {
       GetElementPtrInst *OldGEP =
           cast<GetElementPtrInst>(CE->getAsInstruction());
-      OldGEP->insertBefore(&LI);
+      OldGEP->insertBefore(LI.getIterator());
 
       IRBuilder<> Builder(&LI);
       LoadInst *NewLoad =
@@ -187,7 +187,7 @@ bool DXILFlattenArraysVisitor::visitStoreInst(StoreInst &SI) {
     if (CE && CE->getOpcode() == Instruction::GetElementPtr) {
       GetElementPtrInst *OldGEP =
           cast<GetElementPtrInst>(CE->getAsInstruction());
-      OldGEP->insertBefore(&SI);
+      OldGEP->insertBefore(SI.getIterator());
 
       IRBuilder<> Builder(&SI);
       StoreInst *NewStore = Builder.CreateStore(SI.getValueOperand(), OldGEP);
diff --git a/llvm/lib/Target/DirectX/DXILOpBuilder.cpp b/llvm/lib/Target/DirectX/DXILOpBuilder.cpp
index badd5aabd6432..f0f1bbabb6b23 100644
--- a/llvm/lib/Target/DirectX/DXILOpBuilder.cpp
+++ b/llvm/lib/Target/DirectX/DXILOpBuilder.cpp
@@ -230,6 +230,14 @@ static StructType *getSplitDoubleType(LLVMContext &Context) {
   return StructType::create({Int32Ty, Int32Ty}, "dx.types.splitdouble");
 }
 
+static StructType *getBinaryWithCarryType(LLVMContext &Context) {
+  if (auto *ST = StructType::getTypeByName(Context, "dx.types.i32c"))
+    return ST;
+  Type *Int32Ty = Type::getInt32Ty(Context);
+  Type *Int1Ty = Type::getInt1Ty(Context);
+  return StructType::create({Int32Ty, Int1Ty}, "dx.types.i32c");
+}
+
 static Type *getTypeFromOpParamType(OpParamType Kind, LLVMContext &Ctx,
                                     Type *OverloadTy) {
   switch (Kind) {
@@ -273,6 +281,8 @@ static Type *getTypeFromOpParamType(OpParamType Kind, LLVMContext &Ctx,
     return getResPropsType(Ctx);
   case OpParamType::SplitDoubleTy:
     return getSplitDoubleType(Ctx);
+  case OpParamType::BinaryWithCarryTy:
+    return getBinaryWithCarryType(Ctx);
   }
   llvm_unreachable("Invalid parameter kind");
   return nullptr;
@@ -539,6 +549,10 @@ StructType *DXILOpBuilder::getSplitDoubleType(LLVMContext &Context) {
   return ::getSplitDoubleType(Context);
 }
 
+StructType *DXILOpBuilder::getBinaryWithCarryType(LLVMContext &Context) {
+  return ::getBinaryWithCarryType(Context);
+}
+
 StructType *DXILOpBuilder::getHandleType() {
   return ::getHandleType(IRB.getContext());
 }
diff --git a/llvm/lib/Target/DirectX/DXILOpBuilder.h b/llvm/lib/Target/DirectX/DXILOpBuilder.h
index df5a0240870f4..8e13b87a2be10 100644
--- a/llvm/lib/Target/DirectX/DXILOpBuilder.h
+++ b/llvm/lib/Target/DirectX/DXILOpBuilder.h
@@ -53,6 +53,9 @@ class DXILOpBuilder {
   /// Get the `%dx.types.splitdouble` type.
   StructType *getSplitDoubleType(LLVMContext &Context);
 
+  /// Get the `%dx.types.i32c` type.
+  StructType *getBinaryWithCarryType(LLVMContext &Context);
+
   /// Get the `%dx.types.Handle` type.
   StructType *getHandleType();
 
diff --git a/llvm/lib/Target/DirectX/DXILOpLowering.cpp b/llvm/lib/Target/DirectX/DXILOpLowering.cpp
index 0c245c1a43d31..c9e3d7e284963 100644
--- a/llvm/lib/Target/DirectX/DXILOpLowering.cpp
+++ b/llvm/lib/Target/DirectX/DXILOpLowering.cpp
@@ -359,17 +359,16 @@ class OpLowerer {
     return lowerToBindAndAnnotateHandle(F);
   }
 
-  Error replaceSplitDoubleCallUsages(CallInst *Intrin, CallInst *Op) {
+  Error replaceAggregateTypeOfCallUsages(CallInst *Intrin, CallInst *Op) {
     for (Use &U : make_early_inc_range(Intrin->uses())) {
       if (auto *EVI = dyn_cast<ExtractValueInst>(U.getUser())) {
-
-        if (EVI->getNumIndices() != 1)
-          return createStringError(std::errc::invalid_argument,
-                                   "Splitdouble has only 2 elements");
         EVI->setOperand(0, Op);
+      } else if (auto *IVI = dyn_cast<InsertValueInst>(U.getUser())) {
+        IVI->setOperand(0, Op);
       } else {
         return make_error<StringError>(
-            "Splitdouble use is not ExtractValueInst",
+            (Intrin->getCalledFunction()->getName() +
+             " use is not a ExtractValueInst or InsertValueInst"),
             inconvertibleErrorCode());
       }
     }
@@ -821,7 +820,16 @@ class OpLowerer {
             F, OpCode::SplitDouble,
             OpBuilder.getSplitDoubleType(M.getContext()),
             [&](CallInst *CI, CallInst *Op) {
-              return replaceSplitDoubleCallUsages(CI, Op);
+              return replaceAggregateTypeOfCallUsages(CI, Op);
+            });
+        break;
+      // TODO: this can be removed when
+      // https://github.com/llvm/llvm-project/issues/113192 is fixed
+      case Intrinsic::uadd_with_overflow:
+        HasErrors |= replaceFunctionWithNamedStructOp(
+            F, OpCode::UAddc, OpBuilder.getBinaryWithCarryType(M.getContext()),
+            [&](CallInst *CI, CallInst *Op) {
+              return replaceAggregateTypeOfCallUsages(CI, Op);
             });
         break;
       case Intrinsic::ctpop:
diff --git a/llvm/lib/Target/DirectX/DXILPrepare.cpp b/llvm/lib/Target/DirectX/DXILPrepare.cpp
index 375e6ce712924..355c79ae0edc9 100644
--- a/llvm/lib/Target/DirectX/DXILPrepare.cpp
+++ b/llvm/lib/Target/DirectX/DXILPrepare.cpp
@@ -53,7 +53,6 @@ constexpr bool isValidForDXIL(Attribute::AttrKind Attr) {
                        Attribute::Nest,
                        Attribute::NoAlias,
                        Attribute::NoBuiltin,
-                       Attribute::NoCapture,
                        Attribute::NoDuplicate,
                        Attribute::NoImplicitFloat,
                        Attribute::NoInline,
diff --git a/llvm/lib/Target/DirectX/DXILWriter/DXILBitcodeWriter.cpp b/llvm/lib/Target/DirectX/DXILWriter/DXILBitcodeWriter.cpp
index 0e064d7227cea..543fbd10db230 100644
--- a/llvm/lib/Target/DirectX/DXILWriter/DXILBitcodeWriter.cpp
+++ b/llvm/lib/Target/DirectX/DXILWriter/DXILBitcodeWriter.cpp
@@ -644,8 +644,6 @@ uint64_t DXILBitcodeWriter::getAttrKindEncoding(Attribute::AttrKind Kind) {
     return bitc::ATTR_KIND_NO_ALIAS;
   case Attribute::NoBuiltin:
     return bitc::ATTR_KIND_NO_BUILTIN;
-  case Attribute::NoCapture:
-    return bitc::ATTR_KIND_NO_CAPTURE;
   case Attribute::NoDuplicate:
     return bitc::ATTR_KIND_NO_DUPLICATE;
   case Attribute::NoImplicitFloat:
diff --git a/llvm/lib/Target/DirectX/DirectXTargetTransformInfo.cpp b/llvm/lib/Target/DirectX/DirectXTargetTransformInfo.cpp
index 4e6e01bc5edbc..ba656dc737140 100644
--- a/llvm/lib/Target/DirectX/DirectXTargetTransformInfo.cpp
+++ b/llvm/lib/Target/DirectX/DirectXTargetTransformInfo.cpp
@@ -40,6 +40,8 @@ bool DirectXTTIImpl::isTargetIntrinsicTriviallyScalarizable(
   switch (ID) {
   case Intrinsic::dx_frac:
   case Intrinsic::dx_rsqrt:
+  case Intrinsic::dx_wave_reduce_max:
+  case Intrinsic::dx_wave_reduce_umax:
   case Intrinsic::dx_wave_reduce_sum:
   case Intrinsic::dx_wave_reduce_usum:
   case Intrinsic::dx_wave_readlane:
diff --git a/llvm/lib/Target/Hexagon/AsmParser/HexagonAsmParser.cpp b/llvm/lib/Target/Hexagon/AsmParser/HexagonAsmParser.cpp
index e99858372a3c0..20881de1d94f4 100644
--- a/llvm/lib/Target/Hexagon/AsmParser/HexagonAsmParser.cpp
+++ b/llvm/lib/Target/Hexagon/AsmParser/HexagonAsmParser.cpp
@@ -74,7 +74,7 @@ static cl::opt<bool> WarnSignedMismatch(
     cl::init(false));
 static cl::opt<bool> WarnNoncontigiousRegister(
     "mwarn-noncontigious-register",
-    cl::desc("Warn for register names that arent contigious"), cl::init(true));
+    cl::desc("Warn for register names that aren't contigious"), cl::init(true));
 static cl::opt<bool> ErrorNoncontigiousRegister(
     "merror-noncontigious-register",
     cl::desc("Error for register names that aren't contigious"),
@@ -1330,7 +1330,7 @@ unsigned HexagonAsmParser::validateTargetOperandClass(MCParsedAsmOperand &AsmOp,
   return Match_InvalidOperand;
 }
 
-// FIXME: Calls to OutOfRange shoudl propagate failure up to parseStatement.
+// FIXME: Calls to OutOfRange should propagate failure up to parseStatement.
 bool HexagonAsmParser::OutOfRange(SMLoc IDLoc, long long Val, long long Max) {
   std::string errStr;
   raw_string_ostream ES(errStr);
diff --git a/llvm/lib/Target/Hexagon/BitTracker.h b/llvm/lib/Target/Hexagon/BitTracker.h
index 08c0359a4b7f0..6d52070c1a111 100644
--- a/llvm/lib/Target/Hexagon/BitTracker.h
+++ b/llvm/lib/Target/Hexagon/BitTracker.h
@@ -100,9 +100,9 @@ struct BitTracker {
       bool operator()(const MachineInstr *MI, const MachineInstr *MJ) const;
       DenseMap<const MachineInstr*,unsigned> &Dist;
     };
-    std::priority_queue<MachineInstr*, std::vector<MachineInstr*>, Cmp> Uses;
     DenseSet<const MachineInstr*> Set; // Set to avoid adding duplicate entries.
     DenseMap<const MachineInstr*,unsigned> Dist;
+    std::priority_queue<MachineInstr *, std::vector<MachineInstr *>, Cmp> Uses;
   };
 
   void reset();
diff --git a/llvm/lib/Target/Hexagon/Disassembler/HexagonDisassembler.cpp b/llvm/lib/Target/Hexagon/Disassembler/HexagonDisassembler.cpp
index 4a80ca81c672b..98b711f6b014b 100644
--- a/llvm/lib/Target/Hexagon/Disassembler/HexagonDisassembler.cpp
+++ b/llvm/lib/Target/Hexagon/Disassembler/HexagonDisassembler.cpp
@@ -172,7 +172,9 @@ extern "C" LLVM_EXTERNAL_VISIBILITY void LLVMInitializeHexagonDisassembler() {
 DecodeStatus HexagonDisassembler::getInstruction(MCInst &MI, uint64_t &Size,
                                                  ArrayRef<uint8_t> Bytes,
                                                  uint64_t Address,
-                                                 raw_ostream &cs) const {
+                                                 raw_ostream &CS) const {
+  CommentStream = &CS;
+
   DecodeStatus Result = DecodeStatus::Success;
   bool Complete = false;
   Size = 0;
@@ -184,7 +186,7 @@ DecodeStatus HexagonDisassembler::getInstruction(MCInst &MI, uint64_t &Size,
     if (Bytes.size() < HEXAGON_INSTR_SIZE)
       return MCDisassembler::Fail;
     MCInst *Inst = getContext().createMCInst();
-    Result = getSingleInstruction(*Inst, MI, Bytes, Address, cs, Complete);
+    Result = getSingleInstruction(*Inst, MI, Bytes, Address, CS, Complete);
     MI.addOperand(MCOperand::createInst(Inst));
     Size += HEXAGON_INSTR_SIZE;
     Bytes = Bytes.slice(HEXAGON_INSTR_SIZE);
diff --git a/llvm/lib/Target/Hexagon/HexagonBitSimplify.cpp b/llvm/lib/Target/Hexagon/HexagonBitSimplify.cpp
index 1922675917752..67d822a67e53c 100644
--- a/llvm/lib/Target/Hexagon/HexagonBitSimplify.cpp
+++ b/llvm/lib/Target/Hexagon/HexagonBitSimplify.cpp
@@ -2854,7 +2854,7 @@ bool HexagonBitSimplify::runOnMachineFunction(MachineFunction &MF) {
 // Recognize loops where the code at the end of the loop matches the code
 // before the entry of the loop, and the matching code is such that is can
 // be simplified. This pass relies on the bit simplification above and only
-// prepares code in a way that can be handled by the bit simplifcation.
+// prepares code in a way that can be handled by the bit simplification.
 //
 // This is the motivating testcase (and explanation):
 //
diff --git a/llvm/lib/Target/Hexagon/HexagonConstExtenders.cpp b/llvm/lib/Target/Hexagon/HexagonConstExtenders.cpp
index 86ce6b4e05ed2..55e8572f33198 100644
--- a/llvm/lib/Target/Hexagon/HexagonConstExtenders.cpp
+++ b/llvm/lib/Target/Hexagon/HexagonConstExtenders.cpp
@@ -253,7 +253,7 @@ namespace {
                           /*Kill*/false, /*Dead*/false, /*Undef*/false,
                           /*EarlyClobber*/false, Sub);
         if (Reg.isStack()) {
-          int FI = llvm::Register::stackSlot2Index(Reg);
+          int FI = Reg.stackSlotIndex();
           return MachineOperand::CreateFI(FI);
         }
         llvm_unreachable("Cannot create MachineOperand");
@@ -1040,7 +1040,7 @@ unsigned HCE::getDirectRegReplacement(unsigned ExtOpc) const {
 // extender. It may be possible for MI to be tweaked to work for a register
 // defined with a slightly different value. For example
 //   ... = L2_loadrub_io Rb, 1
-// can be modifed to be
+// can be modified to be
 //   ... = L2_loadrub_io Rb', 0
 // if Rb' = Rb+1.
 // The range for Rb would be [Min+1, Max+1], where [Min, Max] is a range
@@ -1148,8 +1148,8 @@ void HCE::recordExtender(MachineInstr &MI, unsigned OpNum) {
   bool IsStore = MI.mayStore();
 
   // Fixed stack slots have negative indexes, and they cannot be used
-  // with TRI::stackSlot2Index and TRI::index2StackSlot. This is somewhat
-  // unfortunate, but should not be a frequent thing.
+  // with Register::stackSlotIndex and Register::index2StackSlot. This is
+  // somewhat unfortunate, but should not be a frequent thing.
   for (MachineOperand &Op : MI.operands())
     if (Op.isFI() && Op.getIndex() < 0)
       return;
diff --git a/llvm/lib/Target/Hexagon/HexagonConstPropagation.cpp b/llvm/lib/Target/Hexagon/HexagonConstPropagation.cpp
index f68444c0b8d46..92374199b2c91 100644
--- a/llvm/lib/Target/Hexagon/HexagonConstPropagation.cpp
+++ b/llvm/lib/Target/Hexagon/HexagonConstPropagation.cpp
@@ -101,7 +101,7 @@ namespace {
 
   // Lattice cell, based on that was described in the W-Z paper on constant
   // propagation.
-  // Latice cell will be allowed to hold multiple constant values. While
+  // Lattice cell will be allowed to hold multiple constant values. While
   // multiple values would normally indicate "bottom", we can still derive
   // some useful information from them. For example, comparison X > 0
   // could be folded if all the values in the cell associated with X are
@@ -795,7 +795,7 @@ void MachineConstPropagator::visitUsesOf(unsigned Reg) {
   LLVM_DEBUG(dbgs() << "Visiting uses of " << printReg(Reg, &MCE.TRI)
                     << Cells.get(Reg) << '\n');
   for (MachineInstr &MI : MRI->use_nodbg_instructions(Reg)) {
-    // Do not process non-executable instructions. They can become exceutable
+    // Do not process non-executable instructions. They can become executable
     // later (via a flow-edge in the work queue). In such case, the instruc-
     // tion will be visited at that time.
     if (!InstrExec.count(&MI))
diff --git a/llvm/lib/Target/Hexagon/HexagonCopyToCombine.cpp b/llvm/lib/Target/Hexagon/HexagonCopyToCombine.cpp
index 99ac0c346d0c0..7b681fa44f4d6 100644
--- a/llvm/lib/Target/Hexagon/HexagonCopyToCombine.cpp
+++ b/llvm/lib/Target/Hexagon/HexagonCopyToCombine.cpp
@@ -385,7 +385,7 @@ bool HexagonCopyToCombine::isSafeToMoveTogether(MachineInstr &I1,
   return true;
 }
 
-/// findPotentialNewifiableTFRs - Finds tranfers that feed stores that could be
+/// findPotentialNewifiableTFRs - Finds transfers that feed stores that could be
 /// newified. (A use of a 64 bit register define can not be newified)
 void
 HexagonCopyToCombine::findPotentialNewifiableTFRs(MachineBasicBlock &BB) {
diff --git a/llvm/lib/Target/Hexagon/HexagonExpandCondsets.cpp b/llvm/lib/Target/Hexagon/HexagonExpandCondsets.cpp
index 06f6abe5bf847..02e2d670ebee2 100644
--- a/llvm/lib/Target/Hexagon/HexagonExpandCondsets.cpp
+++ b/llvm/lib/Target/Hexagon/HexagonExpandCondsets.cpp
@@ -720,7 +720,7 @@ bool HexagonExpandCondsets::split(MachineInstr &MI,
     }
   }
 
-  // First, create the two invididual conditional transfers, and add each
+  // First, create the two individual conditional transfers, and add each
   // of them to the live intervals information. Do that first and then remove
   // the old instruction from live intervals.
   MachineInstr *TfrT =
diff --git a/llvm/lib/Target/Hexagon/HexagonHardwareLoops.cpp b/llvm/lib/Target/Hexagon/HexagonHardwareLoops.cpp
index 9334746349240..743991447ad1f 100644
--- a/llvm/lib/Target/Hexagon/HexagonHardwareLoops.cpp
+++ b/llvm/lib/Target/Hexagon/HexagonHardwareLoops.cpp
@@ -825,7 +825,7 @@ CountValue *HexagonHardwareLoops::computeCount(MachineLoop *Loop,
   // a computation of it into the preheader.
 
   // If the induction variable bump is not a power of 2, quit.
-  // Othwerise we'd need a general integer division.
+  // Otherwise we'd need a general integer division.
   if (!isPowerOf2_64(std::abs(IVBump)))
     return nullptr;
 
@@ -1398,10 +1398,10 @@ bool HexagonHardwareLoops::phiMayWrapOrUnderflow(
 /// counter if it is <= 1. We only need to perform this analysis if the
 /// initial value is a register.
 ///
-/// This function assumes the initial value may underfow unless proven
+/// This function assumes the initial value may underflow unless proven
 /// otherwise. If the type is signed, then we don't care because signed
 /// underflow is undefined. We attempt to prove the initial value is not
-/// zero by perfoming a crude analysis of the loop counter. This function
+/// zero by performing a crude analysis of the loop counter. This function
 /// checks if the initial value is used in any comparison prior to the loop
 /// and, if so, assumes the comparison is a range check. This is inexact,
 /// but will catch the simple cases.
diff --git a/llvm/lib/Target/Hexagon/HexagonISelDAGToDAGHVX.cpp b/llvm/lib/Target/Hexagon/HexagonISelDAGToDAGHVX.cpp
index db9aa7e18f5e7..7635eb634380c 100644
--- a/llvm/lib/Target/Hexagon/HexagonISelDAGToDAGHVX.cpp
+++ b/llvm/lib/Target/Hexagon/HexagonISelDAGToDAGHVX.cpp
@@ -262,8 +262,7 @@ bool Coloring::color() {
 
   // Explicitly assign "None" to all uncolored nodes.
   for (unsigned I = 0; I != Order.size(); ++I)
-    if (Colors.count(I) == 0)
-      Colors[I] = ColorKind::None;
+    Colors.try_emplace(I, ColorKind::None);
 
   return true;
 }
@@ -1985,7 +1984,7 @@ SmallVector<uint32_t, 8> HvxSelector::getPerfectCompletions(ShuffleMask SM,
   // same as in P. This implies that P == Q.
 
   // There can be a situation where there are more entries with the same
-  // bits set than there are set bits (e.g. value 9 occuring more than 2
+  // bits set than there are set bits (e.g. value 9 occurring more than 2
   // times). In such cases it will be impossible to complete this to a
   // perfect shuffle.
   SmallVector<uint32_t, 8> Sorted(Worklist);
@@ -1996,7 +1995,7 @@ SmallVector<uint32_t, 8> HvxSelector::getPerfectCompletions(ShuffleMask SM,
     while (++I != E && P == Sorted[I])
       ++Count;
     if ((unsigned)llvm::popcount(P) < Count) {
-      // Reset all occurences of P, if there are more occurrences of P
+      // Reset all occurrences of P, if there are more occurrences of P
       // than there are bits in P.
       llvm::replace(Worklist, P, 0U);
     }
@@ -2224,7 +2223,7 @@ OpRef HvxSelector::perfect(ShuffleMask SM, OpRef Va, ResultStack &Results) {
   // V6_vdeal{b,h}
   // V6_vshuff{b,h}
 
-  // V6_vshufoe{b,h}  those are quivalent to vshuffvdd(..,{1,2})
+  // V6_vshufoe{b,h}  those are equivalent to vshuffvdd(..,{1,2})
   // V6_vshuffvdd (V6_vshuff)
   // V6_dealvdd (V6_vdeal)
 
@@ -2269,7 +2268,7 @@ OpRef HvxSelector::perfect(ShuffleMask SM, OpRef Va, ResultStack &Results) {
   // For example, with the inputs as above, the result will be:
   //   0 8  2 A  4 C  6 E
   //   1 9  3 B  5 D  7 F
-  // Now, this result can be tranposed again, but with the group size of 2:
+  // Now, this result can be transposed again, but with the group size of 2:
   //   08 19  4C 5D
   //   2A 3B  6E 7F
   // If we then transpose that result, but with the group size of 4, we get:
diff --git a/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp b/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp
index 12ca0c505bd06..1a7667fe42fbc 100644
--- a/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp
+++ b/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp
@@ -814,7 +814,7 @@ SDValue HexagonTargetLowering::LowerFormalArguments(
   // stack where the return value will be stored. For Hexagon, the location on
   // caller's stack is passed only when the struct size is smaller than (and
   // equal to) 8 bytes. If not, no address will be passed into callee and
-  // callee return the result direclty through R0/R1.
+  // callee return the result directly through R0/R1.
   auto NextSingleReg = [] (const TargetRegisterClass &RC, unsigned Reg) {
     switch (RC.getID()) {
     case Hexagon::IntRegsRegClassID:
@@ -979,7 +979,7 @@ HexagonTargetLowering::LowerVASTART(SDValue Op, SelectionDAG &DAG) const {
   // If first Vararg register is odd, add 4 bytes to start of
   // saved register area to point to the first register location.
   // This is because the saved register area has to be 8 byte aligned.
-  // Incase of an odd start register, there will be 4 bytes of padding in
+  // In case of an odd start register, there will be 4 bytes of padding in
   // the beginning of saved register area. If all registers area used up,
   // the following condition will handle it correctly.
   SDValue SavedRegAreaStartFrameIndex =
@@ -1321,7 +1321,7 @@ HexagonTargetLowering::GetDynamicTLSAddr(SelectionDAG &DAG, SDValue Chain,
 }
 
 //
-// Lower using the intial executable model for TLS addresses
+// Lower using the initial executable model for TLS addresses
 //
 SDValue
 HexagonTargetLowering::LowerToTLSInitialExecModel(GlobalAddressSDNode *GA,
@@ -3320,7 +3320,7 @@ HexagonTargetLowering::LowerEH_RETURN(SDValue Op, SelectionDAG &DAG) const {
   Chain = DAG.getStore(Chain, dl, Handler, StoreAddr, MachinePointerInfo());
   Chain = DAG.getCopyToReg(Chain, dl, OffsetReg, Offset);
 
-  // Not needed we already use it as explict input to EH_RETURN.
+  // Not needed we already use it as explicit input to EH_RETURN.
   // MF.getRegInfo().addLiveOut(OffsetReg);
 
   return DAG.getNode(HexagonISD::EH_RETURN, dl, MVT::Other, Chain);
diff --git a/llvm/lib/Target/Hexagon/HexagonISelLoweringHVX.cpp b/llvm/lib/Target/Hexagon/HexagonISelLoweringHVX.cpp
index 816e063f8dbbe..deffe6369df17 100644
--- a/llvm/lib/Target/Hexagon/HexagonISelLoweringHVX.cpp
+++ b/llvm/lib/Target/Hexagon/HexagonISelLoweringHVX.cpp
@@ -2400,7 +2400,7 @@ HexagonTargetLowering::VectorPair
 HexagonTargetLowering::emitHvxAddWithOverflow(SDValue A, SDValue B,
       const SDLoc &dl, bool Signed, SelectionDAG &DAG) const {
   // Compute A+B, return {A+B, O}, where O = vector predicate indicating
-  // whether an overflow has occured.
+  // whether an overflow has occurred.
   MVT ResTy = ty(A);
   assert(ResTy == ty(B));
   MVT PredTy = MVT::getVectorVT(MVT::i1, ResTy.getVectorNumElements());
@@ -2911,7 +2911,7 @@ HexagonTargetLowering::CreateTLWrapper(SDValue Op, SelectionDAG &DAG) const {
 #ifndef NDEBUG
     Op.dump(&DAG);
 #endif
-    llvm_unreachable("Unepected operator");
+    llvm_unreachable("Unexpected operator");
   }
 
   const SDLoc &dl(Op);
diff --git a/llvm/lib/Target/Hexagon/HexagonInstrInfo.cpp b/llvm/lib/Target/Hexagon/HexagonInstrInfo.cpp
index c54114513ac03..bd6ece19f9ad7 100644
--- a/llvm/lib/Target/Hexagon/HexagonInstrInfo.cpp
+++ b/llvm/lib/Target/Hexagon/HexagonInstrInfo.cpp
@@ -2240,7 +2240,7 @@ bool HexagonInstrInfo::isDependent(const MachineInstr &ProdMI,
   return false;
 }
 
-// Returns true if the instruction is alread a .cur.
+// Returns true if the instruction is already a .cur.
 bool HexagonInstrInfo::isDotCurInst(const MachineInstr &MI) const {
   switch (MI.getOpcode()) {
   case Hexagon::V6_vL32b_cur_pi:
@@ -3860,7 +3860,7 @@ int HexagonInstrInfo::getDotNewPredJumpOp(const MachineInstr &MI,
 int HexagonInstrInfo::getDotNewPredOp(const MachineInstr &MI,
       const MachineBranchProbabilityInfo *MBPI) const {
   switch (MI.getOpcode()) {
-  // Condtional Jumps
+  // Conditional Jumps
   case Hexagon::J2_jumpt:
   case Hexagon::J2_jumpf:
     return getDotNewPredJumpOp(MI, MBPI);
@@ -4325,7 +4325,7 @@ unsigned HexagonInstrInfo::getInstrTimingClassLatency(
 /// operand latency. But it may not be possible for instructions with variable
 /// number of defs / uses.
 ///
-/// This is a raw interface to the itinerary that may be directly overriden by
+/// This is a raw interface to the itinerary that may be directly overridden by
 /// a target. Use computeOperandLatency to get the best estimate of latency.
 std::optional<unsigned> HexagonInstrInfo::getOperandLatency(
     const InstrItineraryData *ItinData, const MachineInstr &DefMI,
diff --git a/llvm/lib/Target/Hexagon/HexagonInstrInfo.h b/llvm/lib/Target/Hexagon/HexagonInstrInfo.h
index 6bfb6d42095ba..2debdde78bfb2 100644
--- a/llvm/lib/Target/Hexagon/HexagonInstrInfo.h
+++ b/llvm/lib/Target/Hexagon/HexagonInstrInfo.h
@@ -145,7 +145,7 @@ class HexagonInstrInfo : public HexagonGenInstrInfo {
 
   /// Second variant of isProfitableToIfCvt. This one
   /// checks for the case where two basic blocks from true and false path
-  /// of a if-then-else (diamond) are predicated on mutally exclusive
+  /// of a if-then-else (diamond) are predicated on mutually exclusive
   /// predicates, where the probability of the true path being taken is given
   /// by Probability, and Confidence is a measure of our confidence that it
   /// will be properly predicted.
@@ -307,8 +307,9 @@ class HexagonInstrInfo : public HexagonGenInstrInfo {
   /// operand latency. But it may not be possible for instructions with variable
   /// number of defs / uses.
   ///
-  /// This is a raw interface to the itinerary that may be directly overriden by
-  /// a target. Use computeOperandLatency to get the best estimate of latency.
+  /// This is a raw interface to the itinerary that may be directly overridden
+  /// by a target. Use computeOperandLatency to get the best estimate of
+  /// latency.
   std::optional<unsigned> getOperandLatency(const InstrItineraryData *ItinData,
                                             const MachineInstr &DefMI,
                                             unsigned DefIdx,
diff --git a/llvm/lib/Target/Hexagon/HexagonLoopAlign.cpp b/llvm/lib/Target/Hexagon/HexagonLoopAlign.cpp
index 7d4c9eacbabfb..8ebead8a41c90 100644
--- a/llvm/lib/Target/Hexagon/HexagonLoopAlign.cpp
+++ b/llvm/lib/Target/Hexagon/HexagonLoopAlign.cpp
@@ -49,7 +49,7 @@ static cl::opt<uint32_t> TinyLoopBndlAlignLimit(
 
 static cl::opt<uint32_t>
     LoopEdgeThreshold("hexagon-loop-edge-threshold", cl::Hidden, cl::init(7500),
-                      cl::desc("Set hexagon loop align edge theshold"));
+                      cl::desc("Set hexagon loop align edge threshold"));
 
 namespace llvm {
 FunctionPass *createHexagonLoopAlign();
diff --git a/llvm/lib/Target/Hexagon/HexagonLoopIdiomRecognition.cpp b/llvm/lib/Target/Hexagon/HexagonLoopIdiomRecognition.cpp
index d2ae2ef7bd7ff..41803e66a0f8e 100644
--- a/llvm/lib/Target/Hexagon/HexagonLoopIdiomRecognition.cpp
+++ b/llvm/lib/Target/Hexagon/HexagonLoopIdiomRecognition.cpp
@@ -1388,14 +1388,12 @@ bool PolynomialMultiplyRecognize::convertShiftsToLeft(BasicBlock *LoopB,
 
   CastMapType CastMap;
 
-  auto upcast = [] (CastMapType &CM, IRBuilder<> &IRB, Value *V,
-        IntegerType *Ty) -> Value* {
-    auto H = CM.find(std::make_pair(V, Ty));
-    if (H != CM.end())
-      return H->second;
-    Value *CV = IRB.CreateIntCast(V, Ty, false);
-    CM.insert(std::make_pair(std::make_pair(V, Ty), CV));
-    return CV;
+  auto upcast = [](CastMapType &CM, IRBuilder<> &IRB, Value *V,
+                   IntegerType *Ty) -> Value * {
+    auto [H, Inserted] = CM.try_emplace(std::make_pair(V, Ty));
+    if (Inserted)
+      H->second = IRB.CreateIntCast(V, Ty, false);
+    return H->second;
   };
 
   for (auto I = LoopB->begin(), E = LoopB->end(); I != E; ++I) {
diff --git a/llvm/lib/Target/Hexagon/HexagonMachineFunctionInfo.h b/llvm/lib/Target/Hexagon/HexagonMachineFunctionInfo.h
index cb4a7a8c393fc..c5df02fa3b89c 100644
--- a/llvm/lib/Target/Hexagon/HexagonMachineFunctionInfo.h
+++ b/llvm/lib/Target/Hexagon/HexagonMachineFunctionInfo.h
@@ -70,12 +70,12 @@ class HexagonMachineFunctionInfo : public MachineFunctionInfo {
     PacketInfo[MI] |= Hexagon::EndPacket;
   }
   bool isStartPacket(const MachineInstr* MI) const {
-    return (PacketInfo.count(MI) &&
-            (PacketInfo.find(MI)->second & Hexagon::StartPacket));
+    auto It = PacketInfo.find(MI);
+    return It != PacketInfo.end() && (It->second & Hexagon::StartPacket);
   }
   bool isEndPacket(const MachineInstr* MI) const {
-    return (PacketInfo.count(MI) &&
-            (PacketInfo.find(MI)->second & Hexagon::EndPacket));
+    auto It = PacketInfo.find(MI);
+    return It != PacketInfo.end() && (It->second & Hexagon::EndPacket);
   }
   void setHasClobberLR(bool v) { HasClobberLR = v;  }
   bool hasClobberLR() const { return HasClobberLR; }
diff --git a/llvm/lib/Target/Hexagon/HexagonNewValueJump.cpp b/llvm/lib/Target/Hexagon/HexagonNewValueJump.cpp
index 36c52dfe81d5a..ee01ebc4daa26 100644
--- a/llvm/lib/Target/Hexagon/HexagonNewValueJump.cpp
+++ b/llvm/lib/Target/Hexagon/HexagonNewValueJump.cpp
@@ -13,10 +13,10 @@
 // Having said that, we should re-attempt to pull this earlier at some point
 // in future.
 
-// The basic approach looks for sequence of predicated jump, compare instruciton
-// that genereates the predicate and, the feeder to the predicate. Once it finds
+// The basic approach looks for sequence of predicated jump, compare instruction
+// that generates the predicate and, the feeder to the predicate. Once it finds
 // all, it collapses compare and jump instruction into a new value jump
-// intstructions.
+// instructions.
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/llvm/lib/Target/Hexagon/HexagonOptAddrMode.cpp b/llvm/lib/Target/Hexagon/HexagonOptAddrMode.cpp
index 693649c2e8e58..66832f31e94ed 100644
--- a/llvm/lib/Target/Hexagon/HexagonOptAddrMode.cpp
+++ b/llvm/lib/Target/Hexagon/HexagonOptAddrMode.cpp
@@ -152,10 +152,10 @@ bool HexagonOptAddrMode::hasRepForm(MachineInstr &MI, unsigned TfrDefR) {
   }
 
   if (HII->getAddrMode(MI) == HexagonII::BaseRegOffset)
-    // Tranform to Absolute plus register offset.
+    // Transform to Absolute plus register offset.
     return (HII->changeAddrMode_rr_ur(MI) >= 0);
   else if (HII->getAddrMode(MI) == HexagonII::BaseImmOffset)
-    // Tranform to absolute addressing mode.
+    // Transform to absolute addressing mode.
     return (HII->changeAddrMode_io_abs(MI) >= 0);
 
   return false;
@@ -163,7 +163,7 @@ bool HexagonOptAddrMode::hasRepForm(MachineInstr &MI, unsigned TfrDefR) {
 
 // Check if addasl instruction can be removed. This is possible only
 // if it's feeding to only load/store instructions with base + register
-// offset as these instruction can be tranformed to use 'absolute plus
+// offset as these instruction can be transformed to use 'absolute plus
 // shifted register offset'.
 // ex:
 // Rs = ##foo
@@ -551,7 +551,7 @@ bool HexagonOptAddrMode::processAddBases(NodeAddr<StmtNode *> AddSN,
     return Changed;
   ProcessedAddiInsts.insert(AddMI);
 
-  // Get the base register that would be shared by other Addi Intructions
+  // Get the base register that would be shared by other Addi Instructions
   Register BaseReg = AddMI->getOperand(1).getReg();
 
   // Store a list of all Addi instructions that share the above common base
@@ -638,7 +638,7 @@ bool HexagonOptAddrMode::processAddBases(NodeAddr<StmtNode *> AddSN,
 
     NewOffset = CurrentMIImmOp.getImm() - FirstReachedMIImmOp.getImm();
 
-    // This is the first occuring Addi, so skip modifying this
+    // This is the first occurring Addi, so skip modifying this
     if (CurrentMI == FirstReachedMI) {
       continue;
     }
@@ -1084,7 +1084,7 @@ bool HexagonOptAddrMode::processBlock(NodeAddr<BlockNode *> BA) {
 
     // Analyze all uses of 'add'. If the output of 'add' is used as an address
     // in the base+immediate addressing mode load/store instructions, see if
-    // they can be updated to use the immediate value as an offet. Thus,
+    // they can be updated to use the immediate value as an offset. Thus,
     // providing us the opportunity to eliminate 'add'.
     // Ex: Rx= add(Rt,#12)
     //     memw(Rx+#0) = Rs
diff --git a/llvm/lib/Target/Hexagon/HexagonOptimizeSZextends.cpp b/llvm/lib/Target/Hexagon/HexagonOptimizeSZextends.cpp
index bfd02802b7829..c29cf034ce089 100644
--- a/llvm/lib/Target/Hexagon/HexagonOptimizeSZextends.cpp
+++ b/llvm/lib/Target/Hexagon/HexagonOptimizeSZextends.cpp
@@ -81,7 +81,7 @@ bool HexagonOptimizeSZextends::runOnFunction(Function &F) {
             assert (EVT::getEVT(SI->getType()) ==
                     (EVT::getEVT(Use->getType())));
             Use->replaceAllUsesWith(SI);
-            Instruction* First = &F.getEntryBlock().front();
+            BasicBlock::iterator First = F.getEntryBlock().begin();
             SI->insertBefore(First);
             Use->eraseFromParent();
           }
diff --git a/llvm/lib/Target/Hexagon/HexagonPeephole.cpp b/llvm/lib/Target/Hexagon/HexagonPeephole.cpp
index 532a8401f9dac..349f2f1d91dc9 100644
--- a/llvm/lib/Target/Hexagon/HexagonPeephole.cpp
+++ b/llvm/lib/Target/Hexagon/HexagonPeephole.cpp
@@ -1,4 +1,4 @@
-//===-- HexagonPeephole.cpp - Hexagon Peephole Optimiztions ---------------===//
+//===-- HexagonPeephole.cpp - Hexagon Peephole Optimizations --------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
diff --git a/llvm/lib/Target/Hexagon/HexagonPseudo.td b/llvm/lib/Target/Hexagon/HexagonPseudo.td
index 94b6af024d280..c2f38a9ccf5b3 100644
--- a/llvm/lib/Target/Hexagon/HexagonPseudo.td
+++ b/llvm/lib/Target/Hexagon/HexagonPseudo.td
@@ -185,7 +185,7 @@ def PS_call_stk : T_Call<"">;
 // This pseudo instruction is used to replace int_hexagon_instrprof_custom intrinsic
 // with a call to custom handler passed as the first argument to the intrinsic.
 
-// Pleae Note:
+// Please Note:
 // 1) The call to the custom handler is being treated as a special one as the
 //    callee is responsible for saving and restoring all the registers it needs
 //    to modify. This includes caller saved registers as well as r0-r5 argument
diff --git a/llvm/lib/Target/Hexagon/HexagonSplitDouble.cpp b/llvm/lib/Target/Hexagon/HexagonSplitDouble.cpp
index b29cd86cc3412..2231fcd0c51c5 100644
--- a/llvm/lib/Target/Hexagon/HexagonSplitDouble.cpp
+++ b/llvm/lib/Target/Hexagon/HexagonSplitDouble.cpp
@@ -264,7 +264,7 @@ void HexagonSplitDoubleRegs::partitionRegisters(UUSetMap &P2Rs) {
         }
         if (MRI->getRegClass(T) != DoubleRC)
           continue;
-        unsigned u = Register::virtReg2Index(T);
+        unsigned u = T.virtRegIndex();
         if (FixedRegs[u])
           continue;
         LLVM_DEBUG(dbgs() << ' ' << printReg(T, TRI));
diff --git a/llvm/lib/Target/Hexagon/HexagonSubtarget.cpp b/llvm/lib/Target/Hexagon/HexagonSubtarget.cpp
index b340bfee3d7e2..ffe9ce750bf50 100644
--- a/llvm/lib/Target/Hexagon/HexagonSubtarget.cpp
+++ b/llvm/lib/Target/Hexagon/HexagonSubtarget.cpp
@@ -686,7 +686,7 @@ bool HexagonSubtarget::isBestZeroLatency(SUnit *Src, SUnit *Dst,
       restoreLatency(Src, DstBest);
   }
 
-  // Attempt to find another opprotunity for zero latency in a different
+  // Attempt to find another opportunity for zero latency in a different
   // dependence.
   if (SrcBest && DstBest)
     // If there is an edge from SrcBest to DstBst, then try to change that
diff --git a/llvm/lib/Target/Hexagon/HexagonTargetMachine.cpp b/llvm/lib/Target/Hexagon/HexagonTargetMachine.cpp
index a97bc1985c614..ff4cb3cc5a881 100644
--- a/llvm/lib/Target/Hexagon/HexagonTargetMachine.cpp
+++ b/llvm/lib/Target/Hexagon/HexagonTargetMachine.cpp
@@ -351,6 +351,11 @@ MachineFunctionInfo *HexagonTargetMachine::createMachineFunctionInfo(
 
 HexagonTargetMachine::~HexagonTargetMachine() = default;
 
+ScheduleDAGInstrs *
+HexagonTargetMachine::createMachineScheduler(MachineSchedContext *C) const {
+  return createVLIWMachineSched(C);
+}
+
 namespace {
 /// Hexagon Code Generator Pass Configuration Options.
 class HexagonPassConfig : public TargetPassConfig {
@@ -362,11 +367,6 @@ class HexagonPassConfig : public TargetPassConfig {
     return getTM<HexagonTargetMachine>();
   }
 
-  ScheduleDAGInstrs *
-  createMachineScheduler(MachineSchedContext *C) const override {
-    return createVLIWMachineSched(C);
-  }
-
   void addIRPasses() override;
   bool addInstSelector() override;
   void addPreRegAlloc() override;
diff --git a/llvm/lib/Target/Hexagon/HexagonTargetMachine.h b/llvm/lib/Target/Hexagon/HexagonTargetMachine.h
index 65f5b6e7e8eb7..48e0c08c0cab2 100644
--- a/llvm/lib/Target/Hexagon/HexagonTargetMachine.h
+++ b/llvm/lib/Target/Hexagon/HexagonTargetMachine.h
@@ -50,6 +50,8 @@ class HexagonTargetMachine : public CodeGenTargetMachineImpl {
   bool isNoopAddrSpaceCast(unsigned SrcAS, unsigned DestAS) const override {
     return true;
   }
+  ScheduleDAGInstrs *
+  createMachineScheduler(MachineSchedContext *C) const override;
 };
 
 } // end namespace llvm
diff --git a/llvm/lib/Target/Hexagon/HexagonTfrCleanup.cpp b/llvm/lib/Target/Hexagon/HexagonTfrCleanup.cpp
index 8a995f671895e..ba23461cd6da9 100644
--- a/llvm/lib/Target/Hexagon/HexagonTfrCleanup.cpp
+++ b/llvm/lib/Target/Hexagon/HexagonTfrCleanup.cpp
@@ -5,8 +5,8 @@
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
-// This pass is to address a situation that appears after register allocaion
-// evey now and then, namely a register copy from a source that was defined
+// This pass is to address a situation that appears after register allocation
+// every now and then, namely a register copy from a source that was defined
 // as an immediate value in the same block (usually just before the copy).
 //
 // Here is an example of actual code emitted that shows this problem:
diff --git a/llvm/lib/Target/Hexagon/HexagonVLIWPacketizer.cpp b/llvm/lib/Target/Hexagon/HexagonVLIWPacketizer.cpp
index a92e2b0197d09..7f9b9ffc34e7d 100644
--- a/llvm/lib/Target/Hexagon/HexagonVLIWPacketizer.cpp
+++ b/llvm/lib/Target/Hexagon/HexagonVLIWPacketizer.cpp
@@ -440,7 +440,7 @@ bool HexagonPacketizerList::canPromoteToDotCur(const MachineInstr &MI,
     return false;
 
   // Check for existing uses of a vector register within the packet which
-  // would be affected by converting a vector load into .cur formt.
+  // would be affected by converting a vector load into .cur format.
   for (auto *BI : CurrentPacketMIs) {
     LLVM_DEBUG(dbgs() << "packet has "; BI->dump(););
     if (BI->readsRegister(DepReg, MF.getSubtarget().getRegisterInfo()))
@@ -750,7 +750,7 @@ bool HexagonPacketizerList::canPromoteToNewValueStore(const MachineInstr &MI,
   // modified by they should not be modified between the producer and the store
   // instruction as it will make them both conditional on different values.
   // We already know this to be true for all the instructions before and
-  // including PacketMI. Howerver, we need to perform the check for the
+  // including PacketMI. However, we need to perform the check for the
   // remaining instructions in the packet.
 
   unsigned StartCheck = 0;
@@ -868,7 +868,7 @@ bool HexagonPacketizerList::canPromoteToDotNew(const MachineInstr &MI,
   if (PI.isImplicitDef())
     return false;
 
-  // If dependency is trough an implicitly defined register, we should not
+  // If dependency is through an implicitly defined register, we should not
   // newify the use.
   if (isImplicitDependency(PI, true, DepReg) ||
       isImplicitDependency(MI, false, DepReg))
@@ -989,7 +989,7 @@ bool HexagonPacketizerList::arePredicatesComplements(MachineInstr &MI1,
   // We attempt to detect it by analyzing existing dependencies in the packet.
 
   // Analyze relationships between all existing members of the packet.
-  // Look for Anti dependecy on the same predicate reg as used in the
+  // Look for Anti dependency on the same predicate reg as used in the
   // candidate.
   for (auto *I : CurrentPacketMIs) {
     // Scheduling Unit for current insn in the packet.
@@ -1276,7 +1276,7 @@ bool HexagonPacketizerList::hasRegMaskDependence(const MachineInstr &I,
   // occur on calls, and the problematic case is when we add an instruction
   // defining a register R to a packet that has a call that clobbers R via
   // a regmask. Those cannot be packetized together, because the call will
-  // be executed last. That's also a reson why it is ok to add a call
+  // be executed last. That's also a reason why it is ok to add a call
   // clobbering R to a packet that defines R.
 
   // Look for regmasks in J.
@@ -1450,7 +1450,7 @@ bool HexagonPacketizerList::isLegalToPacketizeTogether(SUnit *SUI, SUnit *SUJ) {
           continue;
     }
 
-    // Data dpendence ok if we have load.cur.
+    // Data dependence ok if we have load.cur.
     if (DepType == SDep::Data && HII->isDotCurInst(J)) {
       if (HII->isHVXVec(I))
         continue;
@@ -1843,7 +1843,7 @@ bool HexagonPacketizerList::shouldAddToPacket(const MachineInstr &MI) {
   // with any other instruction in the existing packet.
   auto &HST = MI.getParent()->getParent()->getSubtarget<HexagonSubtarget>();
   // Constraint 1: Only one duplex allowed per packet.
-  // Constraint 2: Consider duplex checks only if there is atleast one
+  // Constraint 2: Consider duplex checks only if there is at least one
   // instruction in a packet.
   // Constraint 3: If one of the existing instructions in the packet has a
   // SLOT0 only instruction that can not be duplexed, do not attempt to form
diff --git a/llvm/lib/Target/Hexagon/HexagonVLIWPacketizer.h b/llvm/lib/Target/Hexagon/HexagonVLIWPacketizer.h
index 6a709e566f868..8eced652b2e32 100644
--- a/llvm/lib/Target/Hexagon/HexagonVLIWPacketizer.h
+++ b/llvm/lib/Target/Hexagon/HexagonVLIWPacketizer.h
@@ -97,7 +97,7 @@ class HexagonPacketizerList : public VLIWPacketizerList {
   // together.
   bool isLegalToPacketizeTogether(SUnit *SUI, SUnit *SUJ) override;
 
-  // isLegalToPruneDependencies - Is it legal to prune dependece between SUI
+  // isLegalToPruneDependencies - Is it legal to prune dependence between SUI
   // and SUJ.
   bool isLegalToPruneDependencies(SUnit *SUI, SUnit *SUJ) override;
 
diff --git a/llvm/lib/Target/Hexagon/HexagonVectorCombine.cpp b/llvm/lib/Target/Hexagon/HexagonVectorCombine.cpp
index ce933108b83b1..4e4467c76aff5 100644
--- a/llvm/lib/Target/Hexagon/HexagonVectorCombine.cpp
+++ b/llvm/lib/Target/Hexagon/HexagonVectorCombine.cpp
@@ -342,7 +342,8 @@ class AlignVectors {
   MoveList createLoadGroups(const AddrList &Group) const;
   MoveList createStoreGroups(const AddrList &Group) const;
   bool moveTogether(MoveGroup &Move) const;
-  template <typename T> InstMap cloneBefore(Instruction *To, T &&Insts) const;
+  template <typename T>
+  InstMap cloneBefore(BasicBlock::iterator To, T &&Insts) const;
 
   void realignLoadGroup(IRBuilderBase &Builder, const ByteSpan &VSpan,
                         int ScLen, Value *AlignVal, Value *AlignAddr) const;
@@ -985,7 +986,7 @@ auto AlignVectors::createStoreGroups(const AddrList &Group) const -> MoveList {
     assert(!Move.Main.empty() && "Move group should have non-empty Main");
     if (Move.Main.size() >= SizeLimit)
       return false;
-    // For stores with return values we'd have to collect downward depenencies.
+    // For stores with return values we'd have to collect downward dependencies.
     // There are no such stores that we handle at the moment, so omit that.
     assert(Info.Inst->getType()->isVoidTy() &&
            "Not handling stores with return values");
@@ -1046,7 +1047,7 @@ auto AlignVectors::moveTogether(MoveGroup &Move) const -> bool {
   if (Move.IsLoad) {
     // Move all the loads (and dependencies) to where the first load is.
     // Clone all deps to before Where, keeping order.
-    Move.Clones = cloneBefore(Where, Move.Deps);
+    Move.Clones = cloneBefore(Where->getIterator(), Move.Deps);
     // Move all main instructions to after Where, keeping order.
     ArrayRef<Instruction *> Main(Move.Main);
     for (Instruction *M : Main) {
@@ -1067,7 +1068,7 @@ auto AlignVectors::moveTogether(MoveGroup &Move) const -> bool {
     // Move all main instructions to before Where, inverting order.
     ArrayRef<Instruction *> Main(Move.Main);
     for (Instruction *M : Main.drop_front(1)) {
-      M->moveBefore(Where);
+      M->moveBefore(Where->getIterator());
       Where = M;
     }
   }
@@ -1076,7 +1077,8 @@ auto AlignVectors::moveTogether(MoveGroup &Move) const -> bool {
 }
 
 template <typename T>
-auto AlignVectors::cloneBefore(Instruction *To, T &&Insts) const -> InstMap {
+auto AlignVectors::cloneBefore(BasicBlock::iterator To, T &&Insts) const
+    -> InstMap {
   InstMap Map;
 
   for (Instruction *I : Insts) {
@@ -1200,10 +1202,10 @@ auto AlignVectors::realignLoadGroup(IRBuilderBase &Builder,
                             VSpan.section(Start, Width).values());
   };
 
-  auto moveBefore = [this](Instruction *In, Instruction *To) {
+  auto moveBefore = [this](BasicBlock::iterator In, BasicBlock::iterator To) {
     // Move In and its upward dependencies to before To.
     assert(In->getParent() == To->getParent());
-    DepList Deps = getUpwardDeps(In, To);
+    DepList Deps = getUpwardDeps(&*In, &*To);
     In->moveBefore(To);
     // DepList is sorted with respect to positions in the basic block.
     InstMap Map = cloneBefore(In, Deps);
@@ -1236,7 +1238,7 @@ auto AlignVectors::realignLoadGroup(IRBuilderBase &Builder,
       // in order to check legality.
       if (auto *Load = dyn_cast<Instruction>(Loads[Index])) {
         if (!HVC.isSafeToMoveBeforeInBB(*Load, BasePos))
-          moveBefore(Load, &*BasePos);
+          moveBefore(Load->getIterator(), BasePos);
       }
       LLVM_DEBUG(dbgs() << "Loads[" << Index << "]:" << *Loads[Index] << '\n');
     }
diff --git a/llvm/lib/Target/Hexagon/HexagonVectorLoopCarriedReuse.cpp b/llvm/lib/Target/Hexagon/HexagonVectorLoopCarriedReuse.cpp
index 145cdbce27273..7ada5400b5916 100644
--- a/llvm/lib/Target/Hexagon/HexagonVectorLoopCarriedReuse.cpp
+++ b/llvm/lib/Target/Hexagon/HexagonVectorLoopCarriedReuse.cpp
@@ -323,7 +323,7 @@ bool HexagonVectorLoopCarriedReuse::isEquivalentOperation(Instruction *I1,
     return false;
   // This check is in place specifically for intrinsics. isSameOperationAs will
   // return two for any two hexagon intrinsics because they are essentially the
-  // same instruciton (CallInst). We need to scratch the surface to see if they
+  // same instruction (CallInst). We need to scratch the surface to see if they
   // are calls to the same function.
   if (CallInst *C1 = dyn_cast<CallInst>(I1)) {
     if (CallInst *C2 = dyn_cast<CallInst>(I2)) {
diff --git a/llvm/lib/Target/Hexagon/HexagonVectorLoopCarriedReuse.h b/llvm/lib/Target/Hexagon/HexagonVectorLoopCarriedReuse.h
index f826b2eb568fc..7daa0f2a91b7e 100644
--- a/llvm/lib/Target/Hexagon/HexagonVectorLoopCarriedReuse.h
+++ b/llvm/lib/Target/Hexagon/HexagonVectorLoopCarriedReuse.h
@@ -55,7 +55,7 @@
 //  cond_branch <Loop>
 //
 // In this pass, we look for PHIs such as X2 whose incoming values come only
-// from the Loop Preheader and over the backedge and additionaly, both these
+// from the Loop Preheader and over the backedge and additionally, both these
 // values are the results of the same operation in terms of opcode. We call such
 // a PHI node a dependence chain or DepChain. In this case, the dependence of X2
 // over X1 is carried over only one iteration and so the DepChain is only one
diff --git a/llvm/lib/Target/Hexagon/MCTargetDesc/HexagonMCCompound.cpp b/llvm/lib/Target/Hexagon/MCTargetDesc/HexagonMCCompound.cpp
index c1a9a01aaf7ab..9aa90a0f8da22 100644
--- a/llvm/lib/Target/Hexagon/MCTargetDesc/HexagonMCCompound.cpp
+++ b/llvm/lib/Target/Hexagon/MCTargetDesc/HexagonMCCompound.cpp
@@ -395,7 +395,7 @@ static bool lookForCompound(MCInstrInfo const &MCII, MCContext &Context,
 }
 
 /// tryCompound - Given a bundle check for compound insns when one
-/// is found update the contents fo the bundle with the compound insn.
+/// is found update the contents of the bundle with the compound insn.
 /// If a compound instruction is found then the bundle will have one
 /// additional slot.
 void HexagonMCInstrInfo::tryCompound(MCInstrInfo const &MCII, MCSubtargetInfo const &STI,
diff --git a/llvm/lib/Target/Hexagon/MCTargetDesc/HexagonMCInstrInfo.cpp b/llvm/lib/Target/Hexagon/MCTargetDesc/HexagonMCInstrInfo.cpp
index a2ac8b70b9db4..b27b8bbfa71a7 100644
--- a/llvm/lib/Target/Hexagon/MCTargetDesc/HexagonMCInstrInfo.cpp
+++ b/llvm/lib/Target/Hexagon/MCTargetDesc/HexagonMCInstrInfo.cpp
@@ -435,7 +435,7 @@ unsigned HexagonMCInstrInfo::getCVIResources(MCInstrInfo const &MCII,
   int SchedClass = HexagonMCInstrInfo::getDesc(MCII, MCI).getSchedClass();
   int Size = II[SchedClass].LastStage - II[SchedClass].FirstStage;
 
-  // HVX resources used are currenty located at the second to last stage.
+  // HVX resources used are currently located at the second to last stage.
   // This could also be done with a linear search of the stages looking for:
   // CVI_ALL, CVI_MPY01, CVI_XLSHF, CVI_MPY0, CVI_MPY1, CVI_SHIFT, CVI_XLANE,
   // CVI_ZW
diff --git a/llvm/lib/Target/Hexagon/MCTargetDesc/HexagonMCTargetDesc.cpp b/llvm/lib/Target/Hexagon/MCTargetDesc/HexagonMCTargetDesc.cpp
index a98f6048b051c..3b331b06e6a3a 100644
--- a/llvm/lib/Target/Hexagon/MCTargetDesc/HexagonMCTargetDesc.cpp
+++ b/llvm/lib/Target/Hexagon/MCTargetDesc/HexagonMCTargetDesc.cpp
@@ -614,7 +614,7 @@ MCSubtargetInfo *Hexagon_MC::createHexagonMCSubtargetInfo(const Triple &TT,
   }
 
   // Add qfloat subtarget feature by default to v68 and above
-  // unless explicitely disabled
+  // unless explicitly disabled
   if (checkFeature(X, Hexagon::ExtensionHVXV68) &&
       !ArchFS.contains("-hvx-qfloat")) {
     llvm::FeatureBitset Features = X->getFeatureBits();
@@ -734,8 +734,42 @@ class HexagonMCInstrAnalysis : public MCInstrAnalysis {
     Target = Value;
     return true;
   }
+
+  uint32_t getValueFromMask(uint32_t Instruction, uint32_t Mask) const {
+    uint32_t Result = 0;
+    uint32_t Offset = 0;
+    while (Mask) {
+      if (Instruction & (Mask & -Mask))
+        Result |= (1 << Offset);
+      Mask &= (Mask - 1);
+      ++Offset;
+    }
+    return Result;
+  }
+
+  std::vector<std::pair<uint64_t, uint64_t>>
+  findPltEntries(uint64_t PltSectionVA, ArrayRef<uint8_t> PltContents,
+                 const Triple &TargetTriple) const override {
+    // Do a lightweight parsing of PLT entries.
+    std::vector<std::pair<uint64_t, uint64_t>> Result;
+    for (uint64_t Byte = 0x0, End = PltContents.size(); Byte < End; Byte += 4) {
+      // Recognize immext(##gotpltn)
+      uint32_t ImmExt = support::endian::read32le(PltContents.data() + Byte);
+      if ((ImmExt & 0x00004000) != 0x00004000)
+        continue;
+      uint32_t LoadGotPlt =
+          support::endian::read32le(PltContents.data() + Byte + 4);
+      if ((LoadGotPlt & 0x6a49c00c) != 0x6a49c00c)
+        continue;
+      uint32_t Address = (getValueFromMask(ImmExt, 0xfff3fff) << 6) +
+                         getValueFromMask(LoadGotPlt, 0x1f80) + PltSectionVA +
+                         Byte;
+      Result.emplace_back(PltSectionVA + Byte, Address);
+    }
+    return Result;
+  }
 };
-}
+} // namespace
 
 static MCInstrAnalysis *createHexagonMCInstrAnalysis(const MCInstrInfo *Info) {
   return new HexagonMCInstrAnalysis(Info);
diff --git a/llvm/lib/Target/LoongArch/LoongArchMergeBaseOffset.cpp b/llvm/lib/Target/LoongArch/LoongArchMergeBaseOffset.cpp
index 7f98f7718a538..2aae498e1f2de 100644
--- a/llvm/lib/Target/LoongArch/LoongArchMergeBaseOffset.cpp
+++ b/llvm/lib/Target/LoongArch/LoongArchMergeBaseOffset.cpp
@@ -37,6 +37,8 @@ class LoongArchMergeBaseOffsetOpt : public MachineFunctionPass {
   bool detectFoldable(MachineInstr &Hi20, MachineInstr *&Lo12,
                       MachineInstr *&Lo20, MachineInstr *&Hi12,
                       MachineInstr *&Last);
+  bool detectFoldable(MachineInstr &Hi20, MachineInstr *&Add,
+                      MachineInstr *&Lo12);
 
   bool detectAndFoldOffset(MachineInstr &Hi20, MachineInstr &Lo12,
                            MachineInstr *&Lo20, MachineInstr *&Hi12,
@@ -176,7 +178,80 @@ bool LoongArchMergeBaseOffsetOpt::detectFoldable(MachineInstr &Hi20,
   return true;
 }
 
-// Update the offset in Hi20, Lo12, Lo20 and Hi12 instructions.
+// Detect the pattern:
+//
+// (small/medium):
+//   lu12i.w  vreg1, %le_hi20_r(s)
+//   add.w/d  vreg2, vreg1, r2, %le_add_r(s)
+//   addi.w/d vreg3, vreg2, %le_lo12_r(s)
+
+// The pattern is only accepted if:
+//    1) The first instruction has only one use, which is the PseudoAddTPRel.
+//       The second instruction has only one use, which is the ADDI. The
+//       second instruction's last operand is the tp register.
+//    2) The address operands have the appropriate type, reflecting the
+//       lowering of a thread_local global address using the pattern.
+//    3) The offset value in the ThreadLocal Global Address is 0.
+bool LoongArchMergeBaseOffsetOpt::detectFoldable(MachineInstr &Hi20,
+                                                 MachineInstr *&Add,
+                                                 MachineInstr *&Lo12) {
+  if (Hi20.getOpcode() != LoongArch::LU12I_W)
+    return false;
+
+  auto isGlobalOrCPI = [](const MachineOperand &Op) {
+    return Op.isGlobal() || Op.isCPI();
+  };
+
+  const MachineOperand &Hi20Op1 = Hi20.getOperand(1);
+  if (LoongArchII::getDirectFlags(Hi20Op1) != LoongArchII::MO_LE_HI_R ||
+      !isGlobalOrCPI(Hi20Op1) || Hi20Op1.getOffset() != 0)
+    return false;
+
+  Register HiDestReg = Hi20.getOperand(0).getReg();
+  if (!MRI->hasOneUse(HiDestReg))
+    return false;
+
+  Add = &*MRI->use_instr_begin(HiDestReg);
+  if ((ST->is64Bit() && Add->getOpcode() != LoongArch::PseudoAddTPRel_D) ||
+      (!ST->is64Bit() && Add->getOpcode() != LoongArch::PseudoAddTPRel_W))
+    return false;
+
+  if (Add->getOperand(2).getReg() != LoongArch::R2)
+    return false;
+
+  const MachineOperand &AddOp3 = Add->getOperand(3);
+  if (LoongArchII::getDirectFlags(AddOp3) != LoongArchII::MO_LE_ADD_R ||
+      !(isGlobalOrCPI(AddOp3) || AddOp3.isMCSymbol()) ||
+      AddOp3.getOffset() != 0)
+    return false;
+
+  Register AddDestReg = Add->getOperand(0).getReg();
+  if (!MRI->hasOneUse(AddDestReg))
+    return false;
+
+  Lo12 = &*MRI->use_instr_begin(AddDestReg);
+  if ((ST->is64Bit() && Lo12->getOpcode() != LoongArch::ADDI_D) ||
+      (!ST->is64Bit() && Lo12->getOpcode() != LoongArch::ADDI_W))
+    return false;
+
+  const MachineOperand &Lo12Op2 = Lo12->getOperand(2);
+  if (LoongArchII::getDirectFlags(Lo12Op2) != LoongArchII::MO_LE_LO_R ||
+      !(isGlobalOrCPI(Lo12Op2) || Lo12Op2.isMCSymbol()) ||
+      Lo12Op2.getOffset() != 0)
+    return false;
+
+  if (Hi20Op1.isGlobal()) {
+    LLVM_DEBUG(dbgs() << "  Found lowered global address: "
+                      << *Hi20Op1.getGlobal() << "\n");
+  } else if (Hi20Op1.isCPI()) {
+    LLVM_DEBUG(dbgs() << "  Found lowered constant pool: " << Hi20Op1.getIndex()
+                      << "\n");
+  }
+
+  return true;
+}
+
+// Update the offset in Hi20, (Add), Lo12, (Lo20 and Hi12) instructions.
 // Delete the tail instruction and update all the uses to use the
 // output from Last.
 void LoongArchMergeBaseOffsetOpt::foldOffset(
@@ -190,31 +265,49 @@ void LoongArchMergeBaseOffsetOpt::foldOffset(
     Lo20->getOperand(2).setOffset(Offset);
     Hi12->getOperand(2).setOffset(Offset);
   }
+
+  // For tls-le, offset of the second PseudoAddTPRel instr should also be
+  // updated.
+  MachineInstr *Add = &*MRI->use_instr_begin(Hi20.getOperand(0).getReg());
+  if (Hi20.getOpcode() == LoongArch::LU12I_W)
+    Add->getOperand(3).setOffset(Offset);
+
   // Delete the tail instruction.
   MachineInstr *Def = Last ? Last : &Lo12;
   MRI->constrainRegClass(Def->getOperand(0).getReg(),
                          MRI->getRegClass(Tail.getOperand(0).getReg()));
   MRI->replaceRegWith(Tail.getOperand(0).getReg(), Def->getOperand(0).getReg());
   Tail.eraseFromParent();
+
   LLVM_DEBUG(dbgs() << "  Merged offset " << Offset << " into base.\n"
-                    << "     " << Hi20 << "     " << Lo12;);
+                    << "     " << Hi20;);
+  if (Hi20.getOpcode() == LoongArch::LU12I_W) {
+    LLVM_DEBUG(dbgs() << "     " << *Add;);
+  }
+  LLVM_DEBUG(dbgs() << "     " << Lo12;);
   if (Lo20 && Hi12) {
     LLVM_DEBUG(dbgs() << "     " << *Lo20 << "     " << *Hi12;);
   }
 }
 
 // Detect patterns for large offsets that are passed into an ADD instruction.
-// If the pattern is found, updates the offset in Hi20, Lo12, Lo20 and Hi12
-// instructions and deletes TailAdd and the instructions that produced the
-// offset.
+// If the pattern is found, updates the offset in Hi20, (Add), Lo12,
+// (Lo20 and Hi12) instructions and deletes TailAdd and the instructions that
+// produced the offset.
 //
 //   (The instructions marked with "!" are not necessarily present)
 //
 //        Base address lowering is of the form:
-//           Hi20:  pcalau12i vreg1, %pc_hi20(s)
-//        +- Lo12:  addi.d vreg2, vreg1, %pc_lo12(s)
-//        |  Lo20:  lu32i.d vreg2, %pc64_lo20(s) !
-//        +- Hi12:  lu52i.d vreg2, vreg2, %pc64_hi12(s) !
+//           1) pcala:
+//             Hi20:  pcalau12i vreg1, %pc_hi20(s)
+//        +--- Lo12:  addi.d vreg2, vreg1, %pc_lo12(s)
+//        |    Lo20:  lu32i.d vreg2, %pc64_lo20(s) !
+//        +--- Hi12:  lu52i.d vreg2, vreg2, %pc64_hi12(s) !
+//        |
+//        |  2) tls-le:
+//        |    Hi20:  lu12i.w vreg1, %le_hi20_r(s)
+//        |    Add:   add.w/d vreg1, vreg1, r2, %le_add_r(s)
+//        +--- Lo12:  addi.w/d vreg2, vreg1, %le_lo12_r(s)
 //        |
 //        | The large offset can be one of the forms:
 //        |
@@ -334,7 +427,8 @@ bool LoongArchMergeBaseOffsetOpt::detectAndFoldOffset(MachineInstr &Hi20,
 
   // Look for arithmetic instructions we can get an offset from.
   // We might be able to remove the arithmetic instructions by folding the
-  // offset into the PCALAU12I+(ADDI/ADDI+LU32I+LU52I).
+  // offset into the PCALAU12I+(ADDI/ADDI+LU32I+LU52I) or
+  // LU12I_W+PseudoAddTPRel+ADDI.
   if (!MRI->hasOneUse(DestReg))
     return false;
 
@@ -454,6 +548,7 @@ bool LoongArchMergeBaseOffsetOpt::foldIntoMemoryOps(MachineInstr &Hi20,
   // If all the uses are memory ops with the same offset, we can transform:
   //
   // 1. (small/medium):
+  //  1.1. pcala
   //   pcalau12i vreg1, %pc_hi20(s)
   //   addi.d    vreg2, vreg1, %pc_lo12(s)
   //   ld.w      vreg3, 8(vreg2)
@@ -463,6 +558,18 @@ bool LoongArchMergeBaseOffsetOpt::foldIntoMemoryOps(MachineInstr &Hi20,
   //   pcalau12i vreg1, %pc_hi20(s+8)
   //   ld.w      vreg3, vreg1, %pc_lo12(s+8)(vreg1)
   //
+  //  1.2. tls-le
+  //   lu12i.w  vreg1, %le_hi20_r(s)
+  //   add.w/d  vreg2, vreg1, r2, %le_add_r(s)
+  //   addi.w/d vreg3, vreg2, %le_lo12_r(s)
+  //   ld.w     vreg4, 8(vreg3)
+  //
+  //   =>
+  //
+  //   lu12i.w vreg1, %le_hi20_r(s+8)
+  //   add.w/d vreg2, vreg1, r2, %le_add_r(s+8)
+  //   ld.w    vreg4, vreg2, %le_lo12_r(s+8)(vreg2)
+  //
   // 2. (large):
   //   pcalau12i vreg1, %pc_hi20(s)
   //   addi.d    vreg2, $zero, %pc_lo12(s)
@@ -598,7 +705,8 @@ bool LoongArchMergeBaseOffsetOpt::foldIntoMemoryOps(MachineInstr &Hi20,
     return false;
 
   // If optimized by this pass successfully, MO_RELAX bitmask target-flag should
-  // be removed from the code sequence.
+  // be removed from the pcala code sequence. Code sequence of tls-le can still
+  // be relaxed after being optimized.
   //
   // For example:
   //   pcalau12i $a0, %pc_hi20(symbol)
@@ -614,15 +722,20 @@ bool LoongArchMergeBaseOffsetOpt::foldIntoMemoryOps(MachineInstr &Hi20,
   // optimized, it cannot be relaxed any more. So MO_RELAX flag should not be
   // carried by them.
   Hi20.getOperand(1).setOffset(NewOffset);
-  Hi20.getOperand(1).setTargetFlags(
-      LoongArchII::getDirectFlags(Hi20.getOperand(1)));
   MachineOperand &ImmOp = Lo12.getOperand(2);
   ImmOp.setOffset(NewOffset);
-  ImmOp.setTargetFlags(LoongArchII::getDirectFlags(ImmOp));
   if (Lo20 && Hi12) {
     Lo20->getOperand(2).setOffset(NewOffset);
     Hi12->getOperand(2).setOffset(NewOffset);
   }
+  if (Hi20.getOpcode() == LoongArch::PCALAU12I) {
+    Hi20.getOperand(1).setTargetFlags(
+        LoongArchII::getDirectFlags(Hi20.getOperand(1)));
+    ImmOp.setTargetFlags(LoongArchII::getDirectFlags(ImmOp));
+  } else if (Hi20.getOpcode() == LoongArch::LU12I_W) {
+    MachineInstr *Add = &*MRI->use_instr_begin(Hi20.getOperand(0).getReg());
+    Add->getOperand(3).setOffset(NewOffset);
+  }
 
   // Update the immediate in the load/store instructions to add the offset.
   const LoongArchInstrInfo &TII = *ST->getInstrInfo();
@@ -673,7 +786,14 @@ bool LoongArchMergeBaseOffsetOpt::foldIntoMemoryOps(MachineInstr &Hi20,
     return true;
   }
 
-  MRI->replaceRegWith(Lo12.getOperand(0).getReg(), Hi20.getOperand(0).getReg());
+  if (Hi20.getOpcode() == LoongArch::PCALAU12I) {
+    MRI->replaceRegWith(Lo12.getOperand(0).getReg(),
+                        Hi20.getOperand(0).getReg());
+  } else if (Hi20.getOpcode() == LoongArch::LU12I_W) {
+    MachineInstr *Add = &*MRI->use_instr_begin(Hi20.getOperand(0).getReg());
+    MRI->replaceRegWith(Lo12.getOperand(0).getReg(),
+                        Add->getOperand(0).getReg());
+  }
   Lo12.eraseFromParent();
   return true;
 }
@@ -693,8 +813,21 @@ bool LoongArchMergeBaseOffsetOpt::runOnMachineFunction(MachineFunction &Fn) {
       MachineInstr *Lo20 = nullptr;
       MachineInstr *Hi12 = nullptr;
       MachineInstr *Last = nullptr;
-      if (!detectFoldable(Hi20, Lo12, Lo20, Hi12, Last))
+      if (Hi20.getOpcode() == LoongArch::PCALAU12I) {
+        // Detect foldable pcala code sequence in small/medium/large code model.
+        if (!detectFoldable(Hi20, Lo12, Lo20, Hi12, Last))
+          continue;
+      } else if (Hi20.getOpcode() == LoongArch::LU12I_W) {
+        MachineInstr *Add = nullptr;
+        // Detect foldable tls-le code sequence in small/medium code model.
+        if (!detectFoldable(Hi20, Add, Lo12))
+          continue;
+      } else {
         continue;
+      }
+      // For tls-le, we do not pass the second PseudoAddTPRel instr in order to
+      // reuse the existing hooks and the last three paramaters should always be
+      // nullptr.
       MadeChange |= detectAndFoldOffset(Hi20, *Lo12, Lo20, Hi12, Last);
       MadeChange |= foldIntoMemoryOps(Hi20, *Lo12, Lo20, Hi12, Last);
     }
diff --git a/llvm/lib/Target/M68k/M68kExpandPseudo.cpp b/llvm/lib/Target/M68k/M68kExpandPseudo.cpp
index c7fdd7d7c3502..1ba265a60c3d6 100644
--- a/llvm/lib/Target/M68k/M68kExpandPseudo.cpp
+++ b/llvm/lib/Target/M68k/M68kExpandPseudo.cpp
@@ -193,31 +193,23 @@ bool M68kExpandPseudo::ExpandMI(MachineBasicBlock &MBB,
   case M68k::MOV8dc:
     return TII->ExpandCCR(MIB, /*IsToCCR=*/false);
 
-  case M68k::MOVM8jm_P:
-    return TII->ExpandMOVEM(MIB, TII->get(M68k::MOVM32jm), /*IsRM=*/false);
   case M68k::MOVM16jm_P:
-    return TII->ExpandMOVEM(MIB, TII->get(M68k::MOVM32jm), /*IsRM=*/false);
+    return TII->ExpandMOVEM(MIB, TII->get(M68k::MOVM16jm), /*IsRM=*/false);
   case M68k::MOVM32jm_P:
     return TII->ExpandMOVEM(MIB, TII->get(M68k::MOVM32jm), /*IsRM=*/false);
 
-  case M68k::MOVM8pm_P:
-    return TII->ExpandMOVEM(MIB, TII->get(M68k::MOVM32pm), /*IsRM=*/false);
   case M68k::MOVM16pm_P:
-    return TII->ExpandMOVEM(MIB, TII->get(M68k::MOVM32pm), /*IsRM=*/false);
+    return TII->ExpandMOVEM(MIB, TII->get(M68k::MOVM16pm), /*IsRM=*/false);
   case M68k::MOVM32pm_P:
     return TII->ExpandMOVEM(MIB, TII->get(M68k::MOVM32pm), /*IsRM=*/false);
 
-  case M68k::MOVM8mj_P:
-    return TII->ExpandMOVEM(MIB, TII->get(M68k::MOVM32mj), /*IsRM=*/true);
   case M68k::MOVM16mj_P:
-    return TII->ExpandMOVEM(MIB, TII->get(M68k::MOVM32mj), /*IsRM=*/true);
+    return TII->ExpandMOVEM(MIB, TII->get(M68k::MOVM16mj), /*IsRM=*/true);
   case M68k::MOVM32mj_P:
     return TII->ExpandMOVEM(MIB, TII->get(M68k::MOVM32mj), /*IsRM=*/true);
 
-  case M68k::MOVM8mp_P:
-    return TII->ExpandMOVEM(MIB, TII->get(M68k::MOVM32mp), /*IsRM=*/true);
   case M68k::MOVM16mp_P:
-    return TII->ExpandMOVEM(MIB, TII->get(M68k::MOVM32mp), /*IsRM=*/true);
+    return TII->ExpandMOVEM(MIB, TII->get(M68k::MOVM16mp), /*IsRM=*/true);
   case M68k::MOVM32mp_P:
     return TII->ExpandMOVEM(MIB, TII->get(M68k::MOVM32mp), /*IsRM=*/true);
 
diff --git a/llvm/lib/Target/M68k/M68kInstrData.td b/llvm/lib/Target/M68k/M68kInstrData.td
index a7d7f1826f7f9..398c55fa6da4a 100644
--- a/llvm/lib/Target/M68k/M68kInstrData.td
+++ b/llvm/lib/Target/M68k/M68kInstrData.td
@@ -337,20 +337,16 @@ class MxMOVEM_RM_Pseudo<MxType TYPE, MxOperand MEMOp>
     : MxPseudo<(outs TYPE.ROp:$dst), (ins MEMOp:$src)>;
 
 // Mem <- Reg
-def MOVM8jm_P  : MxMOVEM_MR_Pseudo<MxType8d,  MxType8.JOp>;
 def MOVM16jm_P : MxMOVEM_MR_Pseudo<MxType16r, MxType16.JOp>;
 def MOVM32jm_P : MxMOVEM_MR_Pseudo<MxType32r, MxType32.JOp>;
 
-def MOVM8pm_P  : MxMOVEM_MR_Pseudo<MxType8d,  MxType8.POp>;
 def MOVM16pm_P : MxMOVEM_MR_Pseudo<MxType16r, MxType16.POp>;
 def MOVM32pm_P : MxMOVEM_MR_Pseudo<MxType32r, MxType32.POp>;
 
 // Reg <- Mem
-def MOVM8mj_P  : MxMOVEM_RM_Pseudo<MxType8d,  MxType8.JOp>;
 def MOVM16mj_P : MxMOVEM_RM_Pseudo<MxType16r, MxType16.JOp>;
 def MOVM32mj_P : MxMOVEM_RM_Pseudo<MxType32r, MxType32.JOp>;
 
-def MOVM8mp_P  : MxMOVEM_RM_Pseudo<MxType8d,  MxType8.POp>;
 def MOVM16mp_P : MxMOVEM_RM_Pseudo<MxType16r, MxType16.POp>;
 def MOVM32mp_P : MxMOVEM_RM_Pseudo<MxType32r, MxType32.POp>;
 
diff --git a/llvm/lib/Target/M68k/M68kInstrInfo.cpp b/llvm/lib/Target/M68k/M68kInstrInfo.cpp
index 182582642c50e..febd020f3f2a5 100644
--- a/llvm/lib/Target/M68k/M68kInstrInfo.cpp
+++ b/llvm/lib/Target/M68k/M68kInstrInfo.cpp
@@ -593,7 +593,6 @@ bool M68kInstrInfo::ExpandCCR(MachineInstrBuilder &MIB, bool IsToCCR) const {
 bool M68kInstrInfo::ExpandMOVEM(MachineInstrBuilder &MIB,
                                 const MCInstrDesc &Desc, bool IsRM) const {
   int Reg = 0, Offset = 0, Base = 0;
-  auto XR32 = RI.getRegClass(M68k::XR32RegClassID);
   auto DL = MIB->getDebugLoc();
   auto MI = MIB.getInstr();
   auto &MBB = *MIB->getParent();
@@ -608,13 +607,6 @@ bool M68kInstrInfo::ExpandMOVEM(MachineInstrBuilder &MIB,
     Reg = MIB->getOperand(2).getReg();
   }
 
-  // If the register is not in XR32 then it is smaller than 32 bit, we
-  // implicitly promote it to 32
-  if (!XR32->contains(Reg)) {
-    Reg = RI.getMatchingMegaReg(Reg, XR32);
-    assert(Reg && "Has not meaningful MEGA register");
-  }
-
   unsigned Mask = 1 << RI.getSpillRegisterOrder(Reg);
   if (IsRM) {
     BuildMI(MBB, MI, DL, Desc)
@@ -799,22 +791,25 @@ namespace {
 unsigned getLoadStoreRegOpcode(unsigned Reg, const TargetRegisterClass *RC,
                                const TargetRegisterInfo *TRI,
                                const M68kSubtarget &STI, bool load) {
-  switch (TRI->getRegSizeInBits(*RC)) {
+  switch (TRI->getSpillSize(*RC)) {
   default:
+    LLVM_DEBUG(
+        dbgs() << "Cannot determine appropriate opcode for load/store to/from "
+               << TRI->getName(Reg) << " of class " << TRI->getRegClassName(RC)
+               << " with spill size " << TRI->getSpillSize(*RC) << '\n');
     llvm_unreachable("Unknown spill size");
-  case 8:
+  case 2:
+    if (M68k::XR16RegClass.hasSubClassEq(RC))
+      return load ? M68k::MOVM16mp_P : M68k::MOVM16pm_P;
     if (M68k::DR8RegClass.hasSubClassEq(RC))
-      return load ? M68k::MOV8dp : M68k::MOV8pd;
+      return load ? M68k::MOVM16mp_P : M68k::MOVM16pm_P;
     if (M68k::CCRCRegClass.hasSubClassEq(RC))
-      return load ? M68k::MOV16cp : M68k::MOV16pc;
-
-    llvm_unreachable("Unknown 1-byte regclass");
-  case 16:
-    assert(M68k::XR16RegClass.hasSubClassEq(RC) && "Unknown 2-byte regclass");
-    return load ? M68k::MOVM16mp_P : M68k::MOVM16pm_P;
-  case 32:
-    assert(M68k::XR32RegClass.hasSubClassEq(RC) && "Unknown 4-byte regclass");
-    return load ? M68k::MOVM32mp_P : M68k::MOVM32pm_P;
+      return load ? M68k::MOVM16mp_P : M68k::MOVM16pm_P;
+    llvm_unreachable("Unknown 2-byte regclass");
+  case 4:
+    if (M68k::XR32RegClass.hasSubClassEq(RC))
+      return load ? M68k::MOVM32mp_P : M68k::MOVM32pm_P;
+    llvm_unreachable("Unknown 4-byte regclass");
   }
 }
 
diff --git a/llvm/lib/Target/M68k/M68kRegisterInfo.td b/llvm/lib/Target/M68k/M68kRegisterInfo.td
index 45b492eba4ec0..4942636ffd529 100644
--- a/llvm/lib/Target/M68k/M68kRegisterInfo.td
+++ b/llvm/lib/Target/M68k/M68kRegisterInfo.td
@@ -99,52 +99,77 @@ class MxRegClass<list<ValueType> regTypes, int alignment, dag regList>
     : RegisterClass<"M68k", regTypes, alignment, regList>;
 
 // Data Registers
+let RegInfos = RegInfoByHwMode<[DefaultMode], [RegInfo<8,16,16>]> in
 def DR8  : MxRegClass<[i8],  16, (sequence "BD%u", 0, 7)>;
+let RegInfos = RegInfoByHwMode<[DefaultMode], [RegInfo<16,16,16>]> in
 def DR16 : MxRegClass<[i16], 16, (sequence "WD%u", 0, 7)>;
+let RegInfos = RegInfoByHwMode<[DefaultMode], [RegInfo<32,32,32>]> in
 def DR32 : MxRegClass<[i32], 32, (sequence "D%u",  0, 7)>;
 
 // Address Registers
+let RegInfos = RegInfoByHwMode<[DefaultMode], [RegInfo<16,16,16>]> in
 def AR16 : MxRegClass<[i16], 16, (add (sequence "WA%u", 0, 6), WSP)>;
+let RegInfos = RegInfoByHwMode<[DefaultMode], [RegInfo<32,32,32>]> in
 def AR32 : MxRegClass<[i32], 32, (add (sequence "A%u", 0, 6), SP)>;
 
+let RegInfos = RegInfoByHwMode<[DefaultMode], [RegInfo<32,32,32>]> in
 def AR32_NOSP : MxRegClass<[i32], 32, (sequence "A%u", 0, 6)>;
 
 // Index Register Classes
 // FIXME try alternative ordering like `D0, D1, A0, A1, ...`
+let RegInfos = RegInfoByHwMode<[DefaultMode], [RegInfo<16,16,16>]> in
 def XR16 : MxRegClass<[i16], 16, (add DR16, AR16)>;
+let RegInfos = RegInfoByHwMode<[DefaultMode], [RegInfo<32,32,32>]> in
 def XR32 : MxRegClass<[i32], 32, (add DR32, AR32)>;
 
+let RegInfos = RegInfoByHwMode<[DefaultMode], [RegInfo<32,32,32>]> in
 def SPC  : MxRegClass<[i32], 32, (add SP)>;
 
 // Floating Point Data Registers
+let RegInfos = RegInfoByHwMode<[DefaultMode], [RegInfo<32,32,32>]> in
 def FPDR32 : MxRegClass<[f32], 32, (sequence "FP%u", 0, 7)>;
+let RegInfos = RegInfoByHwMode<[DefaultMode], [RegInfo<64,64,32>]> in
 def FPDR64 : MxRegClass<[f64], 32, (add FPDR32)>;
+let RegInfos = RegInfoByHwMode<[DefaultMode], [RegInfo<80,128,32>]> in
 def FPDR80 : MxRegClass<[f80], 32, (add FPDR32)>;
 
 let CopyCost = -1 in {
+  let RegInfos = RegInfoByHwMode<[DefaultMode], [RegInfo<8,16,16>]> in
   def CCRC : MxRegClass<[i8],  16, (add CCR)>;
+  let RegInfos = RegInfoByHwMode<[DefaultMode], [RegInfo<16,16,16>]> in
   def SRC  : MxRegClass<[i16], 16, (add SR)>;
 
   // Float Point System Control Registers
-  def FPIC   : MxRegClass<[i32], 32, (add FPIAR)>;
-  def FPCSC  : MxRegClass<[i32], 32, (add FPC, FPS)>;
-  def FPSYSC : MxRegClass<[i32], 32, (add FPCSC, FPIC)>;
+  let RegInfos = RegInfoByHwMode<[DefaultMode], [RegInfo<32,32,32>]> in {
+    def FPIC   : MxRegClass<[i32], 32, (add FPIAR)>;
+    def FPCSC  : MxRegClass<[i32], 32, (add FPC, FPS)>;
+    def FPSYSC : MxRegClass<[i32], 32, (add FPCSC, FPIC)>;
+  }
 }
 
 let isAllocatable = 0 in {
+  let RegInfos = RegInfoByHwMode<[DefaultMode], [RegInfo<32,32,32>]> in
   def PCC  : MxRegClass<[i32], 32, (add PC)>;
 }
 
 // Register used with tail call
+let RegInfos = RegInfoByHwMode<[DefaultMode], [RegInfo<16,16,16>]> in
 def DR16_TC : MxRegClass<[i16], 16, (add D0, D1)>;
+let RegInfos = RegInfoByHwMode<[DefaultMode], [RegInfo<32,32,32>]> in
 def DR32_TC : MxRegClass<[i32], 32, (add D0, D1)>;
 
+let RegInfos = RegInfoByHwMode<[DefaultMode], [RegInfo<16,16,16>]> in
 def AR16_TC : MxRegClass<[i16], 16, (add A0, A1)>;
+let RegInfos = RegInfoByHwMode<[DefaultMode], [RegInfo<32,32,32>]> in
 def AR32_TC : MxRegClass<[i32], 32, (add A0, A1)>;
 
+let RegInfos = RegInfoByHwMode<[DefaultMode], [RegInfo<16,16,16>]> in
 def XR16_TC : MxRegClass<[i16], 16, (add DR16_TC, AR16_TC)>;
+let RegInfos = RegInfoByHwMode<[DefaultMode], [RegInfo<32,32,32>]> in
 def XR32_TC : MxRegClass<[i32], 32, (add DR32_TC, AR32_TC)>;
 
 // These classes provide spill/restore order if used with MOVEM instruction
-def SPILL   : MxRegClass<[i32], 32, (add XR32)>;
-def SPILL_R : MxRegClass<[i32], 32, (add SP, (sequence "A%u", 6, 0), (sequence "D%u", 7, 0))>;
+let RegInfos = RegInfoByHwMode<[DefaultMode], [RegInfo<32,32,32>]> in {
+  def SPILL   : MxRegClass<[i32], 32, (add XR32)>;
+  def SPILL_R : MxRegClass<[i32], 32, (add SP, (sequence "A%u", 6, 0), (sequence "D%u", 7, 0))>;
+}
diff --git a/llvm/lib/Target/Mips/MipsISelLowering.cpp b/llvm/lib/Target/Mips/MipsISelLowering.cpp
index 7c4257c222c0b..afec52e289e22 100644
--- a/llvm/lib/Target/Mips/MipsISelLowering.cpp
+++ b/llvm/lib/Target/Mips/MipsISelLowering.cpp
@@ -1004,7 +1004,7 @@ static SDValue performORCombine(SDNode *N, SelectionDAG &DAG,
       return DAG.getNode(
           MipsISD::Ins, DL, N->getValueType(0),
           isConstCase
-              ? DAG.getConstant(CN1->getSExtValue() >> SMPos0, DL, ValTy)
+              ? DAG.getSignedConstant(CN1->getSExtValue() >> SMPos0, DL, ValTy)
               : SrlX,
           DAG.getConstant(SMPos0, DL, MVT::i32),
           DAG.getConstant(ValTy.getSizeInBits() / 8 < 8 ? SMSize0 & 31
@@ -1138,13 +1138,21 @@ static SDValue performADDCombine(SDNode *N, SelectionDAG &DAG,
     return SDValue();
   }
 
+  // When loading from a jump table, push the Lo node to the position that
+  // allows folding it into a load immediate.
   // (add v0, (add v1, abs_lo(tjt))) => (add (add v0, v1), abs_lo(tjt))
-  SDValue Add = N->getOperand(1);
-
-  if (Add.getOpcode() != ISD::ADD)
+  // (add (add abs_lo(tjt), v1), v0) => (add (add v0, v1), abs_lo(tjt))
+  SDValue InnerAdd = N->getOperand(1);
+  SDValue Index = N->getOperand(0);
+  if (InnerAdd.getOpcode() != ISD::ADD)
+    std::swap(InnerAdd, Index);
+  if (InnerAdd.getOpcode() != ISD::ADD)
     return SDValue();
 
-  SDValue Lo = Add.getOperand(1);
+  SDValue Lo = InnerAdd.getOperand(0);
+  SDValue Other = InnerAdd.getOperand(1);
+  if (Lo.getOpcode() != MipsISD::Lo)
+    std::swap(Lo, Other);
 
   if ((Lo.getOpcode() != MipsISD::Lo) ||
       (Lo.getOperand(0).getOpcode() != ISD::TargetJumpTable))
@@ -1153,8 +1161,7 @@ static SDValue performADDCombine(SDNode *N, SelectionDAG &DAG,
   EVT ValTy = N->getValueType(0);
   SDLoc DL(N);
 
-  SDValue Add1 = DAG.getNode(ISD::ADD, DL, ValTy, N->getOperand(0),
-                             Add.getOperand(0));
+  SDValue Add1 = DAG.getNode(ISD::ADD, DL, ValTy, Index, Other);
   return DAG.getNode(ISD::ADD, DL, ValTy, Add1, Lo);
 }
 
diff --git a/llvm/lib/Target/NVPTX/NVPTXAsmPrinter.cpp b/llvm/lib/Target/NVPTX/NVPTXAsmPrinter.cpp
index cb756246b8d11..75d930d9f7b6f 100644
--- a/llvm/lib/Target/NVPTX/NVPTXAsmPrinter.cpp
+++ b/llvm/lib/Target/NVPTX/NVPTXAsmPrinter.cpp
@@ -91,11 +91,6 @@
 
 using namespace llvm;
 
-static cl::opt<bool>
-    LowerCtorDtor("nvptx-lower-global-ctor-dtor",
-                  cl::desc("Lower GPU ctor / dtors to globals on the device."),
-                  cl::init(false), cl::Hidden);
-
 #define DEPOTNAME "__local_depot"
 
 /// DiscoverDependentGlobals - Return a set of GlobalVariables on which \p V
@@ -343,61 +338,32 @@ void NVPTXAsmPrinter::printReturnValStr(const Function *F, raw_ostream &O) {
   const auto *TLI = cast<NVPTXTargetLowering>(STI.getTargetLowering());
 
   Type *Ty = F->getReturnType();
-
-  bool isABI = (STI.getSmVersion() >= 20);
-
   if (Ty->getTypeID() == Type::VoidTyID)
     return;
   O << " (";
 
-  if (isABI) {
-    if ((Ty->isFloatingPointTy() || Ty->isIntegerTy()) &&
-        !ShouldPassAsArray(Ty)) {
-      unsigned size = 0;
-      if (auto *ITy = dyn_cast<IntegerType>(Ty)) {
-        size = ITy->getBitWidth();
-      } else {
-        assert(Ty->isFloatingPointTy() && "Floating point type expected here");
-        size = Ty->getPrimitiveSizeInBits();
-      }
-      size = promoteScalarArgumentSize(size);
-      O << ".param .b" << size << " func_retval0";
-    } else if (isa<PointerType>(Ty)) {
-      O << ".param .b" << TLI->getPointerTy(DL).getSizeInBits()
-        << " func_retval0";
-    } else if (ShouldPassAsArray(Ty)) {
-      unsigned totalsz = DL.getTypeAllocSize(Ty);
-      Align RetAlignment = TLI->getFunctionArgumentAlignment(
-          F, Ty, AttributeList::ReturnIndex, DL);
-      O << ".param .align " << RetAlignment.value() << " .b8 func_retval0["
-        << totalsz << "]";
-    } else
-      llvm_unreachable("Unknown return type");
-  } else {
-    SmallVector<EVT, 16> vtparts;
-    ComputeValueVTs(*TLI, DL, Ty, vtparts);
-    unsigned idx = 0;
-    for (unsigned i = 0, e = vtparts.size(); i != e; ++i) {
-      unsigned elems = 1;
-      EVT elemtype = vtparts[i];
-      if (vtparts[i].isVector()) {
-        elems = vtparts[i].getVectorNumElements();
-        elemtype = vtparts[i].getVectorElementType();
-      }
-
-      for (unsigned j = 0, je = elems; j != je; ++j) {
-        unsigned sz = elemtype.getSizeInBits();
-        if (elemtype.isInteger())
-          sz = promoteScalarArgumentSize(sz);
-        O << ".reg .b" << sz << " func_retval" << idx;
-        if (j < je - 1)
-          O << ", ";
-        ++idx;
-      }
-      if (i < e - 1)
-        O << ", ";
+  if ((Ty->isFloatingPointTy() || Ty->isIntegerTy()) &&
+      !ShouldPassAsArray(Ty)) {
+    unsigned size = 0;
+    if (auto *ITy = dyn_cast<IntegerType>(Ty)) {
+      size = ITy->getBitWidth();
+    } else {
+      assert(Ty->isFloatingPointTy() && "Floating point type expected here");
+      size = Ty->getPrimitiveSizeInBits();
     }
-  }
+    size = promoteScalarArgumentSize(size);
+    O << ".param .b" << size << " func_retval0";
+  } else if (isa<PointerType>(Ty)) {
+    O << ".param .b" << TLI->getPointerTy(DL).getSizeInBits()
+      << " func_retval0";
+  } else if (ShouldPassAsArray(Ty)) {
+    unsigned totalsz = DL.getTypeAllocSize(Ty);
+    Align RetAlignment = TLI->getFunctionArgumentAlignment(
+        F, Ty, AttributeList::ReturnIndex, DL);
+    O << ".param .align " << RetAlignment.value() << " .b8 func_retval0["
+      << totalsz << "]";
+  } else
+    llvm_unreachable("Unknown return type");
   O << ") ";
 }
 
@@ -795,13 +761,6 @@ void NVPTXAsmPrinter::emitDeclarations(const Module &M, raw_ostream &O) {
     emitAliasDeclaration(&GA, O);
 }
 
-static bool isEmptyXXStructor(GlobalVariable *GV) {
-  if (!GV) return true;
-  const ConstantArray *InitList = dyn_cast<ConstantArray>(GV->getInitializer());
-  if (!InitList) return true;  // Not an array; we don't know how to parse.
-  return InitList->getNumOperands() == 0;
-}
-
 void NVPTXAsmPrinter::emitStartOfAsmFile(Module &M) {
   // Construct a default subtarget off of the TargetMachine defaults. The
   // rest of NVPTX isn't friendly to change subtargets per function and
@@ -823,22 +782,6 @@ bool NVPTXAsmPrinter::doInitialization(Module &M) {
   if (M.alias_size() && (STI.getPTXVersion() < 63 || STI.getSmVersion() < 30))
     report_fatal_error(".alias requires PTX version >= 6.3 and sm_30");
 
-  // OpenMP supports NVPTX global constructors and destructors.
-  bool IsOpenMP = M.getModuleFlag("openmp") != nullptr;
-
-  if (!isEmptyXXStructor(M.getNamedGlobal("llvm.global_ctors")) &&
-      !LowerCtorDtor && !IsOpenMP) {
-    report_fatal_error(
-        "Module has a nontrivial global ctor, which NVPTX does not support.");
-    return true;  // error
-  }
-  if (!isEmptyXXStructor(M.getNamedGlobal("llvm.global_dtors")) &&
-      !LowerCtorDtor && !IsOpenMP) {
-    report_fatal_error(
-        "Module has a nontrivial global dtor, which NVPTX does not support.");
-    return true;  // error
-  }
-
   // We need to call the parent's one explicitly.
   bool Result = AsmPrinter::doInitialization(M);
 
@@ -1513,7 +1456,6 @@ void NVPTXAsmPrinter::emitFunctionParamList(const Function *F, raw_ostream &O) {
   unsigned paramIndex = 0;
   bool first = true;
   bool isKernelFunc = isKernelFunction(*F);
-  bool isABI = (STI.getSmVersion() >= 20);
 
   if (F->arg_empty() && !F->isVarArg()) {
     O << "()";
@@ -1646,10 +1588,7 @@ void NVPTXAsmPrinter::emitFunctionParamList(const Function *F, raw_ostream &O) {
         sz = PTySizeInBits;
       } else
         sz = Ty->getPrimitiveSizeInBits();
-      if (isABI)
-        O << "\t.param .b" << sz << " ";
-      else
-        O << "\t.reg .b" << sz << " ";
+      O << "\t.param .b" << sz << " ";
       O << TLI->getParamName(F, paramIndex);
       continue;
     }
@@ -1658,53 +1597,20 @@ void NVPTXAsmPrinter::emitFunctionParamList(const Function *F, raw_ostream &O) {
     Type *ETy = PAL.getParamByValType(paramIndex);
     assert(ETy && "Param should have byval type");
 
-    if (isABI || isKernelFunc) {
-      // Just print .param .align <a> .b8 .param[size];
-      // <a>  = optimal alignment for the element type; always multiple of
-      //        PAL.getParamAlignment
-      // size = typeallocsize of element type
-      Align OptimalAlign =
-          isKernelFunc
-              ? getOptimalAlignForParam(ETy)
-              : TLI->getFunctionByValParamAlign(
-                    F, ETy, PAL.getParamAlignment(paramIndex).valueOrOne(), DL);
-
-      unsigned sz = DL.getTypeAllocSize(ETy);
-      O << "\t.param .align " << OptimalAlign.value() << " .b8 ";
-      O << TLI->getParamName(F, paramIndex);
-      O << "[" << sz << "]";
-      continue;
-    } else {
-      // Split the ETy into constituent parts and
-      // print .param .b<size> <name> for each part.
-      // Further, if a part is vector, print the above for
-      // each vector element.
-      SmallVector<EVT, 16> vtparts;
-      ComputeValueVTs(*TLI, DL, ETy, vtparts);
-      for (unsigned i = 0, e = vtparts.size(); i != e; ++i) {
-        unsigned elems = 1;
-        EVT elemtype = vtparts[i];
-        if (vtparts[i].isVector()) {
-          elems = vtparts[i].getVectorNumElements();
-          elemtype = vtparts[i].getVectorElementType();
-        }
-
-        for (unsigned j = 0, je = elems; j != je; ++j) {
-          unsigned sz = elemtype.getSizeInBits();
-          if (elemtype.isInteger())
-            sz = promoteScalarArgumentSize(sz);
-          O << "\t.reg .b" << sz << " ";
-          O << TLI->getParamName(F, paramIndex);
-          if (j < je - 1)
-            O << ",\n";
-          ++paramIndex;
-        }
-        if (i < e - 1)
-          O << ",\n";
-      }
-      --paramIndex;
-      continue;
-    }
+    // Print .param .align <a> .b8 .param[size];
+    // <a>  = optimal alignment for the element type; always multiple of
+    //        PAL.getParamAlignment
+    // size = typeallocsize of element type
+    Align OptimalAlign =
+        isKernelFunc
+            ? getOptimalAlignForParam(ETy)
+            : TLI->getFunctionByValParamAlign(
+                  F, ETy, PAL.getParamAlignment(paramIndex).valueOrOne(), DL);
+
+    unsigned sz = DL.getTypeAllocSize(ETy);
+    O << "\t.param .align " << OptimalAlign.value() << " .b8 ";
+    O << TLI->getParamName(F, paramIndex);
+    O << "[" << sz << "]";
   }
 
   if (F->isVarArg()) {
diff --git a/llvm/lib/Target/NVPTX/NVPTXGenericToNVVM.cpp b/llvm/lib/Target/NVPTX/NVPTXGenericToNVVM.cpp
index 75fcf6829c504..c734d3d430073 100644
--- a/llvm/lib/Target/NVPTX/NVPTXGenericToNVVM.cpp
+++ b/llvm/lib/Target/NVPTX/NVPTXGenericToNVVM.cpp
@@ -86,7 +86,7 @@ bool GenericToNVVM::runOnModule(Module &M) {
     if (F.isDeclaration()) {
       continue;
     }
-    IRBuilder<> Builder(F.getEntryBlock().getFirstNonPHIOrDbg());
+    IRBuilder<> Builder(&*F.getEntryBlock().getFirstNonPHIOrDbg());
     for (BasicBlock &BB : F) {
       for (Instruction &II : BB) {
         for (unsigned i = 0, e = II.getNumOperands(); i < e; ++i) {
diff --git a/llvm/lib/Target/NVPTX/NVPTXISelDAGToDAG.cpp b/llvm/lib/Target/NVPTX/NVPTXISelDAGToDAG.cpp
index ac8ce05724750..e96c1758676a1 100644
--- a/llvm/lib/Target/NVPTX/NVPTXISelDAGToDAG.cpp
+++ b/llvm/lib/Target/NVPTX/NVPTXISelDAGToDAG.cpp
@@ -24,7 +24,6 @@
 #include "llvm/Support/CommandLine.h"
 #include "llvm/Support/ErrorHandling.h"
 #include "llvm/Support/FormatVariadic.h"
-#include "llvm/Target/TargetIntrinsicInfo.h"
 
 using namespace llvm;
 
@@ -648,9 +647,50 @@ static unsigned int getFenceOp(NVPTX::Ordering O, NVPTX::Scope S,
   if (S == NVPTX::Scope::Cluster)
     T->failIfClustersUnsupported(".cluster scope fence");
 
+  // Fall back to .acq_rel if .acquire, .release is not supported.
+  if (!T->hasSplitAcquireAndReleaseFences() &&
+      (O == NVPTX::Ordering::Acquire || O == NVPTX::Ordering::Release))
+    O = NVPTX::Ordering::AcquireRelease;
+
   switch (O) {
   case NVPTX::Ordering::Acquire:
+    switch (S) {
+    case NVPTX::Scope::System:
+      return T->hasMemoryOrdering() ? NVPTX::atomic_thread_fence_acquire_sys
+                                    : NVPTX::INT_MEMBAR_SYS;
+    case NVPTX::Scope::Block:
+      return T->hasMemoryOrdering() ? NVPTX::atomic_thread_fence_acquire_cta
+                                    : NVPTX::INT_MEMBAR_CTA;
+    case NVPTX::Scope::Cluster:
+      return NVPTX::atomic_thread_fence_acquire_cluster;
+    case NVPTX::Scope::Device:
+      return T->hasMemoryOrdering() ? NVPTX::atomic_thread_fence_acquire_gpu
+                                    : NVPTX::INT_MEMBAR_GL;
+    case NVPTX::Scope::Thread:
+      report_fatal_error(
+          formatv("Unsupported scope \"{}\" for acquire/release/acq_rel fence.",
+                  ScopeToString(S)));
+    }
+    break;
   case NVPTX::Ordering::Release:
+    switch (S) {
+    case NVPTX::Scope::System:
+      return T->hasMemoryOrdering() ? NVPTX::atomic_thread_fence_release_sys
+                                    : NVPTX::INT_MEMBAR_SYS;
+    case NVPTX::Scope::Block:
+      return T->hasMemoryOrdering() ? NVPTX::atomic_thread_fence_release_cta
+                                    : NVPTX::INT_MEMBAR_CTA;
+    case NVPTX::Scope::Cluster:
+      return NVPTX::atomic_thread_fence_release_cluster;
+    case NVPTX::Scope::Device:
+      return T->hasMemoryOrdering() ? NVPTX::atomic_thread_fence_release_gpu
+                                    : NVPTX::INT_MEMBAR_GL;
+    case NVPTX::Scope::Thread:
+      report_fatal_error(
+          formatv("Unsupported scope \"{}\" for acquire/release/acq_rel fence.",
+                  ScopeToString(S)));
+    }
+    break;
   case NVPTX::Ordering::AcquireRelease: {
     switch (S) {
     case NVPTX::Scope::System:
diff --git a/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp b/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp
index 773c97f7b4dc0..58ad92a8934a6 100644
--- a/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp
+++ b/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp
@@ -286,7 +286,7 @@ static void ComputePTXValueVTs(const TargetLowering &TLI, const DataLayout &DL,
     if (VT.isVector()) {
       unsigned NumElts = VT.getVectorNumElements();
       EVT EltVT = VT.getVectorElementType();
-      // We require power-of-2 sized vectors becuase
+      // We require power-of-2 sized vectors because
       // TargetLoweringBase::getVectorTypeBreakdown() which is invoked in
       // ComputePTXValueVTs() cannot currently break down non-power-of-2 sized
       // vectors.
@@ -1156,11 +1156,6 @@ std::string NVPTXTargetLowering::getPrototype(
     const CallBase &CB, unsigned UniqueCallSite) const {
   auto PtrVT = getPointerTy(DL);
 
-  bool isABI = (STI.getSmVersion() >= 20);
-  assert(isABI && "Non-ABI compilation is not supported");
-  if (!isABI)
-    return "";
-
   std::string Prototype;
   raw_string_ostream O(Prototype);
   O << "prototype_" << UniqueCallSite << " : .callprototype ";
@@ -1429,11 +1424,6 @@ SDValue NVPTXTargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
   const CallBase *CB = CLI.CB;
   const DataLayout &DL = DAG.getDataLayout();
 
-  bool isABI = (STI.getSmVersion() >= 20);
-  assert(isABI && "Non-ABI compilation is not supported");
-  if (!isABI)
-    return Chain;
-
   // Variadic arguments.
   //
   // Normally, for each argument, we declare a param scalar or a param
@@ -3091,11 +3081,6 @@ SDValue NVPTXTargetLowering::LowerFormalArguments(
   SDValue Root = DAG.getRoot();
   std::vector<SDValue> OutChains;
 
-  bool isABI = (STI.getSmVersion() >= 20);
-  assert(isABI && "Non-ABI compilation is not supported");
-  if (!isABI)
-    return Chain;
-
   std::vector<Type *> argTypes;
   std::vector<const Argument *> theArgs;
   for (const Argument &I : F->args()) {
@@ -3310,11 +3295,6 @@ NVPTXTargetLowering::LowerReturn(SDValue Chain, CallingConv::ID CallConv,
   const Function &F = MF.getFunction();
   Type *RetTy = MF.getFunction().getReturnType();
 
-  bool isABI = (STI.getSmVersion() >= 20);
-  assert(isABI && "Non-ABI compilation is not supported");
-  if (!isABI)
-    return Chain;
-
   const DataLayout &DL = DAG.getDataLayout();
   SmallVector<SDValue, 16> PromotedOutVals;
   SmallVector<EVT, 16> VTs;
diff --git a/llvm/lib/Target/NVPTX/NVPTXInstrInfo.td b/llvm/lib/Target/NVPTX/NVPTXInstrInfo.td
index f17799c130015..7d9697e40e6ab 100644
--- a/llvm/lib/Target/NVPTX/NVPTXInstrInfo.td
+++ b/llvm/lib/Target/NVPTX/NVPTXInstrInfo.td
@@ -142,6 +142,7 @@ def hasLDU : Predicate<"Subtarget->hasLDU()">;
 def hasPTXASUnreachableBug : Predicate<"Subtarget->hasPTXASUnreachableBug()">;
 def noPTXASUnreachableBug : Predicate<"!Subtarget->hasPTXASUnreachableBug()">;
 def hasOptEnabled : Predicate<"TM.getOptLevel() != CodeGenOptLevel::None">;
+def hasAcceleratedFeatures : Predicate<"Subtarget->hasAAFeatures()">;
 
 def doF32FTZ : Predicate<"useF32FTZ()">;
 def doNoF32FTZ : Predicate<"!useF32FTZ()">;
@@ -163,6 +164,7 @@ def do_SQRTF32_RN : Predicate<"usePrecSqrtF32()">;
 def hasHWROT32 : Predicate<"Subtarget->hasHWROT32()">;
 def noHWROT32 : Predicate<"!Subtarget->hasHWROT32()">;
 def hasDotInstructions : Predicate<"Subtarget->hasDotInstructions()">;
+def hasTcgen05Instructions : Predicate<"Subtarget->hasTcgen05Instructions()">;
 
 def True : Predicate<"true">;
 def False : Predicate<"false">;
@@ -764,6 +766,11 @@ let hasSideEffects = false in {
   defm CVT_to_tf32_rz_relu  : CVT_TO_TF32<"rz.relu">;
   defm CVT_to_tf32_rna      : CVT_TO_TF32<"rna", [hasPTX<70>, hasSM<80>]>;
   defm CVT_to_tf32_rna_satf : CVT_TO_TF32<"rna.satfinite", [hasPTX<81>, hasSM<89>]>;
+
+  defm CVT_to_tf32_rn_satf : CVT_TO_TF32<"rn.satfinite", [hasPTX<86>, hasSM<100>]>;
+  defm CVT_to_tf32_rz_satf : CVT_TO_TF32<"rz.satfinite", [hasPTX<86>, hasSM<100>]>;
+  defm CVT_to_tf32_rn_relu_satf  : CVT_TO_TF32<"rn.relu.satfinite", [hasPTX<86>, hasSM<100>]>;
+  defm CVT_to_tf32_rz_relu_satf  : CVT_TO_TF32<"rz.relu.satfinite", [hasPTX<86>, hasSM<100>]>;
 }
 
 def fpround_oneuse : PatFrag<(ops node:$a), (fpround node:$a), [{
@@ -3861,33 +3868,16 @@ def : Pat <
 // PTX Fence instructions
 ////////////////////////////////////////////////////////////////////////////////
 
-def atomic_thread_fence_seq_cst_sys :
-  NVPTXInst<(outs), (ins), "fence.sc.sys;", []>,
-  Requires<[hasPTX<60>, hasSM<70>]>;
-def atomic_thread_fence_acq_rel_sys :
-  NVPTXInst<(outs), (ins), "fence.acq_rel.sys;", []>,
-  Requires<[hasPTX<60>, hasSM<70>]>;
-
-def atomic_thread_fence_seq_cst_gpu :
-  NVPTXInst<(outs), (ins), "fence.sc.gpu;", []>,
-  Requires<[hasPTX<60>, hasSM<70>]>;
-def atomic_thread_fence_acq_rel_gpu :
-  NVPTXInst<(outs), (ins), "fence.acq_rel.gpu;", []>,
-  Requires<[hasPTX<60>, hasSM<70>]>;
-
-def atomic_thread_fence_seq_cst_cluster :
-  NVPTXInst<(outs), (ins), "fence.sc.cluster;", []>,
-  Requires<[hasPTX<78>, hasSM<90>]>;
-def atomic_thread_fence_acq_rel_cluster :
-  NVPTXInst<(outs), (ins), "fence.acq_rel.cluster;", []>,
-  Requires<[hasPTX<78>, hasSM<90>]>;
-
-def atomic_thread_fence_seq_cst_cta :
-  NVPTXInst<(outs), (ins), "fence.sc.cta;", []>,
-  Requires<[hasPTX<60>, hasSM<70>]>;
-def atomic_thread_fence_acq_rel_cta :
-  NVPTXInst<(outs), (ins), "fence.acq_rel.cta;", []>,
-  Requires<[hasPTX<60>, hasSM<70>]>;
+class NVPTXFenceInst<string scope, string sem, Predicate ptx>:
+    NVPTXInst<(outs), (ins), "fence."#sem#"."#scope#";", []>,
+    Requires<[ptx, hasSM<70>]>;
+
+foreach scope = ["sys", "gpu", "cluster", "cta"] in {
+  def atomic_thread_fence_seq_cst_#scope: NVPTXFenceInst<scope, "sc", hasPTX<60>>;
+  def atomic_thread_fence_acq_rel_#scope: NVPTXFenceInst<scope, "acq_rel", hasPTX<60>>;
+  def atomic_thread_fence_acquire_#scope: NVPTXFenceInst<scope, "acquire", hasPTX<87>>;
+  def atomic_thread_fence_release_#scope: NVPTXFenceInst<scope, "release", hasPTX<87>>;
+}
 
 def fpimm_any_zero : FPImmLeaf<fAny, [{
   return Imm.isZero();
diff --git a/llvm/lib/Target/NVPTX/NVPTXIntrinsics.td b/llvm/lib/Target/NVPTX/NVPTXIntrinsics.td
index 56d8b734bf01d..39dac65d67eb9 100644
--- a/llvm/lib/Target/NVPTX/NVPTXIntrinsics.td
+++ b/llvm/lib/Target/NVPTX/NVPTXIntrinsics.td
@@ -740,6 +740,33 @@ foreach dim = [1, 2, 3, 4, 5] in {
   }
 }
 
+//Prefetch and Prefetchu 
+class Join<string sep, list<string> lst> {
+  string ret = !foldl("", lst, a, b, !if(!eq(a, ""), b, !if(!eq(b,""), a, !strconcat(a, sep, b))));
+}
+
+class PREFETCH_INTRS<string InstName> :
+          NVPTXInst<(outs), (ins Int64Regs:$addr),
+          InstName # " [$addr];",
+          [(!cast<Intrinsic>(!strconcat("int_nvvm_",
+          !subst(".", "_", InstName))) i64:$addr)]>,
+          Requires<[hasPTX<80>, hasSM<90>]>;
+   
+
+// Only global supports evictlast and evictnormal.
+// Other variants (local and default) only support evictnormal
+foreach level = ["L1", "L2"] in {
+  foreach addr = ["global", "local", ""] in {
+    foreach evict = !if(!eq(addr, "global"), ["evictlast", "evictnormal"], ["evictnormal"]) in {
+      defvar suffix = Join<"_", [addr, level, evict]>.ret;
+      defvar inst_name = "prefetch." # !subst("_", ".", suffix);
+      def PREFETCH_# suffix : PREFETCH_INTRS<inst_name>;
+    }
+  }
+}
+
+def PREFETCHU_L1_EVICTNORMAL : PREFETCH_INTRS<"prefetchu.L1.evictnormal">;
+
 //-----------------------------------
 // MBarrier Functions
 //-----------------------------------
@@ -7547,7 +7574,7 @@ multiclass SET_MAXNREG<string Action, Intrinsic Intr> {
   def : NVPTXInst<(outs), (ins i32imm:$reg_count),
           "setmaxnreg." # Action # ".sync.aligned.u32 $reg_count;",
           [(Intr timm:$reg_count)]>,
-    Requires<[hasSM90a, hasPTX<80>]>;
+    Requires<[hasAcceleratedFeatures, hasSM<90>, hasPTX<80>]>;
 }
 
 defm INT_SET_MAXNREG_INC : SET_MAXNREG<"inc", int_nvvm_setmaxnreg_inc_sync_aligned_u32>;
@@ -7582,3 +7609,91 @@ def GRIDDEPCONTROL_WAIT :
                 Requires<[hasSM<90>, hasPTX<78>]>;
 
 def INT_EXIT : NVPTXInst<(outs), (ins), "exit;", [(int_nvvm_exit)]>;
+
+// Tcgen05 intrinsics
+let isConvergent = true in {
+
+multiclass TCGEN05_ALLOC_INTR<NVPTXRegClass rc, string AS, string num, Intrinsic Intr> {
+  def NAME : NVPTXInst<(outs),
+             (ins rc:$dst, Int32Regs:$ncols),
+             !strconcat("tcgen05.alloc.cta_group::", num, ".sync.aligned", AS, ".b32 [$dst], $ncols;"),
+             [(Intr rc:$dst, Int32Regs:$ncols)]>,
+             Requires<[hasTcgen05Instructions]>;
+}
+
+defm TCGEN05_ALLOC_CG1 : TCGEN05_ALLOC_INTR<Int64Regs, "", "1", int_nvvm_tcgen05_alloc_cg1>;
+defm TCGEN05_ALLOC_CG2 : TCGEN05_ALLOC_INTR<Int64Regs, "", "2", int_nvvm_tcgen05_alloc_cg2>;
+
+defm TCGEN05_ALLOC_S64_CG1 : TCGEN05_ALLOC_INTR<Int64Regs, ".shared::cta", "1", int_nvvm_tcgen05_alloc_shared_cg1>;
+defm TCGEN05_ALLOC_S64_CG2 : TCGEN05_ALLOC_INTR<Int64Regs, ".shared::cta", "2", int_nvvm_tcgen05_alloc_shared_cg2>;
+
+defm TCGEN05_ALLOC_S32_CG1 : TCGEN05_ALLOC_INTR<Int32Regs, ".shared::cta", "1", int_nvvm_tcgen05_alloc_shared_cg1>;
+defm TCGEN05_ALLOC_S32_CG2 : TCGEN05_ALLOC_INTR<Int32Regs, ".shared::cta", "2", int_nvvm_tcgen05_alloc_shared_cg2>;
+
+multiclass TCGEN05_DEALLOC_INTR<string num, Intrinsic Intr> {
+  def NAME : NVPTXInst<(outs),
+             (ins Int32Regs:$tmem_addr, Int32Regs:$ncols),
+             !strconcat("tcgen05.dealloc.cta_group::", num, ".sync.aligned.b32 $tmem_addr, $ncols;"),
+             [(Intr Int32Regs:$tmem_addr, Int32Regs:$ncols)]>,
+             Requires<[hasTcgen05Instructions]>;
+}
+defm TCGEN05_DEALLOC_CG1: TCGEN05_DEALLOC_INTR<"1", int_nvvm_tcgen05_dealloc_cg1>;
+defm TCGEN05_DEALLOC_CG2: TCGEN05_DEALLOC_INTR<"2", int_nvvm_tcgen05_dealloc_cg2>;
+
+multiclass TCGEN05_RELINQ_PERMIT_INTR<string num, Intrinsic Intr> {
+  def NAME : NVPTXInst<(outs), (ins),
+             !strconcat("tcgen05.relinquish_alloc_permit.cta_group::", num, ".sync.aligned;"),
+             [(Intr)]>,
+             Requires<[hasTcgen05Instructions]>;
+}
+defm TCGEN05_RELINQ_CG1: TCGEN05_RELINQ_PERMIT_INTR<"1", int_nvvm_tcgen05_relinq_alloc_permit_cg1>;
+defm TCGEN05_RELINQ_CG2: TCGEN05_RELINQ_PERMIT_INTR<"2", int_nvvm_tcgen05_relinq_alloc_permit_cg2>;
+
+def tcgen05_wait_ld: NVPTXInst<(outs), (ins), "tcgen05.wait::ld.sync.aligned;",
+  [(int_nvvm_tcgen05_wait_ld)]>,
+  Requires<[hasTcgen05Instructions]>;
+
+def tcgen05_wait_st: NVPTXInst<(outs), (ins), "tcgen05.wait::st.sync.aligned;",
+  [(int_nvvm_tcgen05_wait_st)]>,
+  Requires<[hasTcgen05Instructions]>;
+
+multiclass TCGEN05_COMMIT_INTR<NVPTXRegClass rc, string AS, string num> {
+  defvar prefix = "tcgen05.commit.cta_group::" # num;
+  defvar suffix = ".mbarrier::arrive::one.shared::cluster";
+
+  defvar intr_suffix = !if(!eq(AS, "shared"), "_shared", "") # "_cg" # num;
+  defvar Intr = !cast<Intrinsic>("int_nvvm_tcgen05_commit" # intr_suffix);
+  defvar IntrMC = !cast<Intrinsic>("int_nvvm_tcgen05_commit_mc" # intr_suffix);
+
+  def NAME : NVPTXInst<(outs), (ins rc:$mbar),
+             !strconcat(prefix, suffix, ".b64 [$mbar];"),
+             [(Intr rc:$mbar)]>,
+             Requires<[hasTcgen05Instructions]>;
+  def NAME # _MC : NVPTXInst<(outs), (ins rc:$mbar, Int16Regs:$mc),
+                   !strconcat(prefix, suffix, ".multicast::cluster.b64 [$mbar], $mc;"),
+                   [(IntrMC rc:$mbar, Int16Regs:$mc)]>,
+                   Requires<[hasTcgen05Instructions]>;
+}
+
+defm TCGEN05_COMMIT_CG1 : TCGEN05_COMMIT_INTR<Int64Regs, "", "1">;
+defm TCGEN05_COMMIT_CG2 : TCGEN05_COMMIT_INTR<Int64Regs, "", "2">;
+defm TCGEN05_COMMIT_S64_CG1 : TCGEN05_COMMIT_INTR<Int64Regs, "shared", "1">;
+defm TCGEN05_COMMIT_S64_CG2 : TCGEN05_COMMIT_INTR<Int64Regs, "shared", "2">;
+defm TCGEN05_COMMIT_S32_CG1 : TCGEN05_COMMIT_INTR<Int32Regs, "shared", "1">;
+defm TCGEN05_COMMIT_S32_CG2 : TCGEN05_COMMIT_INTR<Int32Regs, "shared", "2">;
+
+} // isConvergent
+
+let hasSideEffects = 1 in {
+
+def tcgen05_fence_before_thread_sync: NVPTXInst<(outs), (ins),
+  "tcgen05.fence::before_thread_sync;",
+  [(int_nvvm_tcgen05_fence_before_thread_sync)]>,
+  Requires<[hasTcgen05Instructions]>;
+
+def tcgen05_fence_after_thread_sync: NVPTXInst<(outs), (ins),
+  "tcgen05.fence::after_thread_sync;",
+  [(int_nvvm_tcgen05_fence_after_thread_sync)]>,
+  Requires<[hasTcgen05Instructions]>;
+
+} // hasSideEffects
diff --git a/llvm/lib/Target/NVPTX/NVPTXSubtarget.h b/llvm/lib/Target/NVPTX/NVPTXSubtarget.h
index 919f487c70141..851c9152e4cb8 100644
--- a/llvm/lib/Target/NVPTX/NVPTXSubtarget.h
+++ b/llvm/lib/Target/NVPTX/NVPTXSubtarget.h
@@ -88,11 +88,30 @@ class NVPTXSubtarget : public NVPTXGenSubtargetInfo {
   // Does SM & PTX support memory orderings (weak and atomic: relaxed, acquire,
   // release, acq_rel, sc) ?
   bool hasMemoryOrdering() const { return SmVersion >= 70 && PTXVersion >= 60; }
+  // Does SM & PTX support .acquire and .release qualifiers for fence?
+  bool hasSplitAcquireAndReleaseFences() const {
+    return SmVersion >= 90 && PTXVersion >= 86;
+  }
   // Does SM & PTX support atomic relaxed MMIO operations ?
   bool hasRelaxedMMIO() const { return SmVersion >= 70 && PTXVersion >= 82; }
   bool hasDotInstructions() const {
     return SmVersion >= 61 && PTXVersion >= 50;
   }
+  // Tcgen05 instructions in Blackwell family
+  bool hasTcgen05Instructions() const {
+    bool HasTcgen05 = false;
+    switch (FullSmVersion) {
+    default:
+      break;
+    case 1001: // sm_100a
+    case 1011: // sm_101a
+      HasTcgen05 = true;
+      break;
+    }
+
+    return HasTcgen05 && PTXVersion >= 86;
+  }
+
   // Prior to CUDA 12.3 ptxas did not recognize that the trap instruction
   // terminates a basic block. Instead, it would assume that control flow
   // continued to the next instruction. The next instruction could be in the
diff --git a/llvm/lib/Target/NVPTX/NVPTXTargetMachine.cpp b/llvm/lib/Target/NVPTX/NVPTXTargetMachine.cpp
index 6d4b82aa54a2b..f2afa6fc20bfa 100644
--- a/llvm/lib/Target/NVPTX/NVPTXTargetMachine.cpp
+++ b/llvm/lib/Target/NVPTX/NVPTXTargetMachine.cpp
@@ -21,6 +21,7 @@
 #include "NVPTXTargetObjectFile.h"
 #include "NVPTXTargetTransformInfo.h"
 #include "TargetInfo/NVPTXTargetInfo.h"
+#include "llvm/Analysis/KernelInfo.h"
 #include "llvm/Analysis/TargetTransformInfo.h"
 #include "llvm/CodeGen/Passes.h"
 #include "llvm/CodeGen/TargetPassConfig.h"
@@ -139,6 +140,9 @@ static std::string computeDataLayout(bool is64Bit, bool UseShortPointers) {
   else if (UseShortPointers)
     Ret += "-p3:32:32-p4:32:32-p5:32:32";
 
+  // Tensor Memory (addrspace:6) is always 32-bits.
+  Ret += "-p6:32:32";
+
   Ret += "-i64:64-i128:128-v16:16-v32:32-n16:32:64";
 
   return Ret;
@@ -266,6 +270,15 @@ void NVPTXTargetMachine::registerPassBuilderCallbacks(PassBuilder &PB) {
           FPM.addPass(NVPTXCopyByValArgsPass());
         PM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM)));
       });
+
+  if (!NoKernelInfoEndLTO) {
+    PB.registerFullLinkTimeOptimizationLastEPCallback(
+        [this](ModulePassManager &PM, OptimizationLevel Level) {
+          FunctionPassManager FPM;
+          FPM.addPass(KernelInfoPrinter(this));
+          PM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM)));
+        });
+  }
 }
 
 TargetTransformInfo
diff --git a/llvm/lib/Target/NVPTX/NVPTXTargetTransformInfo.cpp b/llvm/lib/Target/NVPTX/NVPTXTargetTransformInfo.cpp
index 4ec2ec100ab08..85e99d7fe97a2 100644
--- a/llvm/lib/Target/NVPTX/NVPTXTargetTransformInfo.cpp
+++ b/llvm/lib/Target/NVPTX/NVPTXTargetTransformInfo.cpp
@@ -562,4 +562,18 @@ Value *NVPTXTTIImpl::rewriteIntrinsicWithAddressSpace(IntrinsicInst *II,
   }
   }
   return nullptr;
-}
\ No newline at end of file
+}
+
+void NVPTXTTIImpl::collectKernelLaunchBounds(
+    const Function &F,
+    SmallVectorImpl<std::pair<StringRef, int64_t>> &LB) const {
+  std::optional<unsigned> Val;
+  if ((Val = getMaxClusterRank(F)))
+    LB.push_back({"maxclusterrank", *Val});
+  if ((Val = getMaxNTIDx(F)))
+    LB.push_back({"maxntidx", *Val});
+  if ((Val = getMaxNTIDy(F)))
+    LB.push_back({"maxntidy", *Val});
+  if ((Val = getMaxNTIDz(F)))
+    LB.push_back({"maxntidz", *Val});
+}
diff --git a/llvm/lib/Target/NVPTX/NVPTXTargetTransformInfo.h b/llvm/lib/Target/NVPTX/NVPTXTargetTransformInfo.h
index 0f4fb280b2d99..b0a846a9c7f96 100644
--- a/llvm/lib/Target/NVPTX/NVPTXTargetTransformInfo.h
+++ b/llvm/lib/Target/NVPTX/NVPTXTargetTransformInfo.h
@@ -129,6 +129,10 @@ class NVPTXTTIImpl : public BasicTTIImplBase<NVPTXTTIImpl> {
 
   Value *rewriteIntrinsicWithAddressSpace(IntrinsicInst *II, Value *OldV,
                                           Value *NewV) const;
+
+  void collectKernelLaunchBounds(
+      const Function &F,
+      SmallVectorImpl<std::pair<StringRef, int64_t>> &LB) const;
 };
 
 } // end namespace llvm
diff --git a/llvm/lib/Target/NVPTX/NVPTXUtilities.cpp b/llvm/lib/Target/NVPTX/NVPTXUtilities.cpp
index 0f2bec711b249..a41943880807c 100644
--- a/llvm/lib/Target/NVPTX/NVPTXUtilities.cpp
+++ b/llvm/lib/Target/NVPTX/NVPTXUtilities.cpp
@@ -310,30 +310,21 @@ std::optional<unsigned> getMaxNReg(const Function &F) {
   return findOneNVVMAnnotation(&F, "maxnreg");
 }
 
-bool isKernelFunction(const Function &F) {
-  if (F.getCallingConv() == CallingConv::PTX_Kernel)
-    return true;
-
-  if (const auto X = findOneNVVMAnnotation(&F, "kernel"))
-    return (*X == 1);
-
-  return false;
-}
-
 MaybeAlign getAlign(const Function &F, unsigned Index) {
   // First check the alignstack metadata
   if (MaybeAlign StackAlign =
           F.getAttributes().getAttributes(Index).getStackAlignment())
     return StackAlign;
 
-  // If that is missing, check the legacy nvvm metadata
-  std::vector<unsigned> Vs;
-  bool retval = findAllNVVMAnnotation(&F, "align", Vs);
-  if (!retval)
-    return std::nullopt;
-  for (unsigned V : Vs)
-    if ((V >> 16) == Index)
-      return Align(V & 0xFFFF);
+  // check the legacy nvvm metadata only for the return value since llvm does
+  // not support stackalign attribute for this.
+  if (Index == 0) {
+    std::vector<unsigned> Vs;
+    if (findAllNVVMAnnotation(&F, "align", Vs))
+      for (unsigned V : Vs)
+        if ((V >> 16) == Index)
+          return Align(V & 0xFFFF);
+  }
 
   return std::nullopt;
 }
diff --git a/llvm/lib/Target/NVPTX/NVPTXUtilities.h b/llvm/lib/Target/NVPTX/NVPTXUtilities.h
index 7ce00b9b5688d..cf35eaf4cbae5 100644
--- a/llvm/lib/Target/NVPTX/NVPTXUtilities.h
+++ b/llvm/lib/Target/NVPTX/NVPTXUtilities.h
@@ -16,6 +16,7 @@
 #include "NVPTX.h"
 #include "llvm/ADT/StringExtras.h"
 #include "llvm/CodeGen/ValueTypes.h"
+#include "llvm/IR/CallingConv.h"
 #include "llvm/IR/Function.h"
 #include "llvm/IR/GlobalVariable.h"
 #include "llvm/IR/IntrinsicInst.h"
@@ -63,7 +64,11 @@ std::optional<unsigned> getClusterDimz(const Function &);
 std::optional<unsigned> getMaxClusterRank(const Function &);
 std::optional<unsigned> getMinCTASm(const Function &);
 std::optional<unsigned> getMaxNReg(const Function &);
-bool isKernelFunction(const Function &);
+
+inline bool isKernelFunction(const Function &F) {
+  return F.getCallingConv() == CallingConv::PTX_Kernel;
+}
+
 bool isParamGridConstant(const Value &);
 
 MaybeAlign getAlign(const Function &, unsigned);
diff --git a/llvm/lib/Target/PowerPC/PPCISelLowering.cpp b/llvm/lib/Target/PowerPC/PPCISelLowering.cpp
index 4ca328bd9a9ba..bdc1ac7c7da58 100644
--- a/llvm/lib/Target/PowerPC/PPCISelLowering.cpp
+++ b/llvm/lib/Target/PowerPC/PPCISelLowering.cpp
@@ -8593,7 +8593,7 @@ SDValue PPCTargetLowering::LowerFP_TO_INT(SDValue Op, SelectionDAG &DAG,
 // to insert our load, L, into the chain as a peer of O. To do this, we give L
 // the same chain operand as O, we create a token factor from the chain results
 // of O and L, and we replace all uses of O's chain result with that token
-// factor (see spliceIntoChain below for this last part).
+// factor (this last part is handled by makeEquivalentMemoryOrdering).
 bool PPCTargetLowering::canReuseLoadAddress(SDValue Op, EVT MemVT,
                                             ReuseLoadInfo &RLI,
                                             SelectionDAG &DAG,
@@ -8648,27 +8648,6 @@ bool PPCTargetLowering::canReuseLoadAddress(SDValue Op, EVT MemVT,
   return true;
 }
 
-// Given the head of the old chain, ResChain, insert a token factor containing
-// it and NewResChain, and make users of ResChain now be users of that token
-// factor.
-// TODO: Remove and use DAG::makeEquivalentMemoryOrdering() instead.
-void PPCTargetLowering::spliceIntoChain(SDValue ResChain,
-                                        SDValue NewResChain,
-                                        SelectionDAG &DAG) const {
-  if (!ResChain)
-    return;
-
-  SDLoc dl(NewResChain);
-
-  SDValue TF = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
-                           NewResChain, DAG.getUNDEF(MVT::Other));
-  assert(TF.getNode() != NewResChain.getNode() &&
-         "A new TF really is required here");
-
-  DAG.ReplaceAllUsesOfValueWith(ResChain, TF);
-  DAG.UpdateNodeOperands(TF.getNode(), ResChain, NewResChain);
-}
-
 /// Analyze profitability of direct move
 /// prefer float load to int load plus direct move
 /// when there is no integer use of int load
@@ -8930,7 +8909,8 @@ SDValue PPCTargetLowering::LowerINT_TO_FP(SDValue Op,
     if (canReuseLoadAddress(SINT, MVT::i64, RLI, DAG)) {
       Bits = DAG.getLoad(MVT::f64, dl, RLI.Chain, RLI.Ptr, RLI.MPI,
                          RLI.Alignment, RLI.MMOFlags(), RLI.AAInfo, RLI.Ranges);
-      spliceIntoChain(RLI.ResChain, Bits.getValue(1), DAG);
+      if (RLI.ResChain)
+        DAG.makeEquivalentMemoryOrdering(RLI.ResChain, Bits.getValue(1));
     } else if (Subtarget.hasLFIWAX() &&
                canReuseLoadAddress(SINT, MVT::i32, RLI, DAG, ISD::SEXTLOAD)) {
       MachineMemOperand *MMO =
@@ -8940,7 +8920,8 @@ SDValue PPCTargetLowering::LowerINT_TO_FP(SDValue Op,
       Bits = DAG.getMemIntrinsicNode(PPCISD::LFIWAX, dl,
                                      DAG.getVTList(MVT::f64, MVT::Other),
                                      Ops, MVT::i32, MMO);
-      spliceIntoChain(RLI.ResChain, Bits.getValue(1), DAG);
+      if (RLI.ResChain)
+        DAG.makeEquivalentMemoryOrdering(RLI.ResChain, Bits.getValue(1));
     } else if (Subtarget.hasFPCVT() &&
                canReuseLoadAddress(SINT, MVT::i32, RLI, DAG, ISD::ZEXTLOAD)) {
       MachineMemOperand *MMO =
@@ -8950,7 +8931,8 @@ SDValue PPCTargetLowering::LowerINT_TO_FP(SDValue Op,
       Bits = DAG.getMemIntrinsicNode(PPCISD::LFIWZX, dl,
                                      DAG.getVTList(MVT::f64, MVT::Other),
                                      Ops, MVT::i32, MMO);
-      spliceIntoChain(RLI.ResChain, Bits.getValue(1), DAG);
+      if (RLI.ResChain)
+        DAG.makeEquivalentMemoryOrdering(RLI.ResChain, Bits.getValue(1));
     } else if (((Subtarget.hasLFIWAX() &&
                  SINT.getOpcode() == ISD::SIGN_EXTEND) ||
                 (Subtarget.hasFPCVT() &&
@@ -9046,8 +9028,9 @@ SDValue PPCTargetLowering::LowerINT_TO_FP(SDValue Op,
                                  DAG.getVTList(MVT::f64, MVT::Other), Ops,
                                  MVT::i32, MMO);
     Chain = Ld.getValue(1);
-    if (ReusingLoad)
-      spliceIntoChain(RLI.ResChain, Ld.getValue(1), DAG);
+    if (ReusingLoad && RLI.ResChain) {
+      DAG.makeEquivalentMemoryOrdering(RLI.ResChain, Ld.getValue(1));
+    }
   } else {
     assert(Subtarget.isPPC64() &&
            "i32->FP without LFIWAX supported only on PPC64");
@@ -11669,7 +11652,8 @@ SDValue PPCTargetLowering::LowerSCALAR_TO_VECTOR(SDValue Op,
     SDValue Bits = DAG.getMemIntrinsicNode(
         PPCISD::LD_SPLAT, dl, DAG.getVTList(MVT::v4i32, MVT::Other), Ops,
         MVT::i32, MMO);
-    spliceIntoChain(RLI.ResChain, Bits.getValue(1), DAG);
+    if (RLI.ResChain)
+      DAG.makeEquivalentMemoryOrdering(RLI.ResChain, Bits.getValue(1));
     return Bits.getValue(0);
   }
 
diff --git a/llvm/lib/Target/PowerPC/PPCISelLowering.h b/llvm/lib/Target/PowerPC/PPCISelLowering.h
index cc01cab7a2089..514329bbe92d7 100644
--- a/llvm/lib/Target/PowerPC/PPCISelLowering.h
+++ b/llvm/lib/Target/PowerPC/PPCISelLowering.h
@@ -1226,8 +1226,6 @@ namespace llvm {
     bool canReuseLoadAddress(SDValue Op, EVT MemVT, ReuseLoadInfo &RLI,
                              SelectionDAG &DAG,
                              ISD::LoadExtType ET = ISD::NON_EXTLOAD) const;
-    void spliceIntoChain(SDValue ResChain, SDValue NewResChain,
-                         SelectionDAG &DAG) const;
 
     void LowerFP_TO_INTForReuse(SDValue Op, ReuseLoadInfo &RLI,
                                 SelectionDAG &DAG, const SDLoc &dl) const;
diff --git a/llvm/lib/Target/PowerPC/PPCMIPeephole.cpp b/llvm/lib/Target/PowerPC/PPCMIPeephole.cpp
index 43d81f64613b7..1b6da5781ac6b 100644
--- a/llvm/lib/Target/PowerPC/PPCMIPeephole.cpp
+++ b/llvm/lib/Target/PowerPC/PPCMIPeephole.cpp
@@ -187,12 +187,11 @@ struct PPCMIPeephole : public MachineFunctionPass {
 
 #define addRegToUpdate(R) addRegToUpdateWithLine(R, __LINE__)
 void PPCMIPeephole::addRegToUpdateWithLine(Register Reg, int Line) {
-  if (!Register::isVirtualRegister(Reg))
+  if (!Reg.isVirtual())
     return;
   if (RegsToUpdate.insert(Reg).second)
-    LLVM_DEBUG(dbgs() << "Adding register: " << Register::virtReg2Index(Reg)
-                      << " on line " << Line
-                      << " for re-computation of kill flags\n");
+    LLVM_DEBUG(dbgs() << "Adding register: " << printReg(Reg) << " on line "
+                      << Line << " for re-computation of kill flags\n");
 }
 
 // Initialize class variables.
diff --git a/llvm/lib/Target/PowerPC/PPCMacroFusion.h b/llvm/lib/Target/PowerPC/PPCMacroFusion.h
index cbf49ee779ceb..22d15cac2ef55 100644
--- a/llvm/lib/Target/PowerPC/PPCMacroFusion.h
+++ b/llvm/lib/Target/PowerPC/PPCMacroFusion.h
@@ -20,7 +20,7 @@ namespace llvm {
 
 /// Note that you have to add:
 ///   DAG.addMutation(createPowerPCMacroFusionDAGMutation());
-/// to PPCPassConfig::createMachineScheduler() to have an effect.
+/// to PPCTargetMachine::createMachineScheduler() to have an effect.
 std::unique_ptr<ScheduleDAGMutation> createPowerPCMacroFusionDAGMutation();
 } // llvm
 
diff --git a/llvm/lib/Target/PowerPC/PPCRegisterInfo.h b/llvm/lib/Target/PowerPC/PPCRegisterInfo.h
index 36b8a24ba502d..274c7cb68ae0a 100644
--- a/llvm/lib/Target/PowerPC/PPCRegisterInfo.h
+++ b/llvm/lib/Target/PowerPC/PPCRegisterInfo.h
@@ -65,9 +65,10 @@ class PPCRegisterInfo : public PPCGenRegisterInfo {
   /// for a given imm form load/store opcode \p ImmFormOpcode.
   /// FIXME: move this to PPCInstrInfo class.
   unsigned getMappedIdxOpcForImmOpc(unsigned ImmOpcode) const {
-    if (!ImmToIdxMap.count(ImmOpcode))
+    auto It = ImmToIdxMap.find(ImmOpcode);
+    if (It == ImmToIdxMap.end())
       return PPC::INSTRUCTION_LIST_END;
-    return ImmToIdxMap.find(ImmOpcode)->second;
+    return It->second;
   }
 
   /// getPointerRegClass - Return the register class to use to hold pointers.
diff --git a/llvm/lib/Target/PowerPC/PPCTargetMachine.cpp b/llvm/lib/Target/PowerPC/PPCTargetMachine.cpp
index b1ad041bde790..5ee13a92cf993 100644
--- a/llvm/lib/Target/PowerPC/PPCTargetMachine.cpp
+++ b/llvm/lib/Target/PowerPC/PPCTargetMachine.cpp
@@ -403,6 +403,16 @@ PPCTargetMachine::getSubtargetImpl(const Function &F) const {
   return I.get();
 }
 
+ScheduleDAGInstrs *
+PPCTargetMachine::createMachineScheduler(MachineSchedContext *C) const {
+  return createPPCMachineScheduler(C);
+}
+
+ScheduleDAGInstrs *
+PPCTargetMachine::createPostMachineScheduler(MachineSchedContext *C) const {
+  return createPPCPostMachineScheduler(C);
+}
+
 //===----------------------------------------------------------------------===//
 // Pass Pipeline Configuration
 //===----------------------------------------------------------------------===//
@@ -438,15 +448,6 @@ class PPCPassConfig : public TargetPassConfig {
   bool addLegalizeMachineIR() override;
   bool addRegBankSelect() override;
   bool addGlobalInstructionSelect() override;
-
-  ScheduleDAGInstrs *
-  createMachineScheduler(MachineSchedContext *C) const override {
-    return createPPCMachineScheduler(C);
-  }
-  ScheduleDAGInstrs *
-  createPostMachineScheduler(MachineSchedContext *C) const override {
-    return createPPCPostMachineScheduler(C);
-  }
 };
 
 } // end anonymous namespace
diff --git a/llvm/lib/Target/PowerPC/PPCTargetMachine.h b/llvm/lib/Target/PowerPC/PPCTargetMachine.h
index 026bf2f26a816..cb02b446fadb3 100644
--- a/llvm/lib/Target/PowerPC/PPCTargetMachine.h
+++ b/llvm/lib/Target/PowerPC/PPCTargetMachine.h
@@ -63,6 +63,10 @@ class PPCTargetMachine final : public CodeGenTargetMachineImpl {
   MachineFunctionInfo *
   createMachineFunctionInfo(BumpPtrAllocator &Allocator, const Function &F,
                             const TargetSubtargetInfo *STI) const override;
+  ScheduleDAGInstrs *
+  createMachineScheduler(MachineSchedContext *C) const override;
+  ScheduleDAGInstrs *
+  createPostMachineScheduler(MachineSchedContext *C) const override;
 
   bool isELFv2ABI() const { return TargetABI == PPC_ABI_ELFv2; }
   bool hasGlibcHWCAPAccess() const { return HasGlibcHWCAPAccess; }
diff --git a/llvm/lib/Target/RISCV/AsmParser/RISCVAsmParser.cpp b/llvm/lib/Target/RISCV/AsmParser/RISCVAsmParser.cpp
index 227a6361730da..ac87d72b7595c 100644
--- a/llvm/lib/Target/RISCV/AsmParser/RISCVAsmParser.cpp
+++ b/llvm/lib/Target/RISCV/AsmParser/RISCVAsmParser.cpp
@@ -130,7 +130,7 @@ class RISCVAsmParser : public MCTargetAsmParser {
   void emitToStreamer(MCStreamer &S, const MCInst &Inst);
 
   // Helper to emit a combination of LUI, ADDI(W), and SLLI instructions that
-  // synthesize the desired immedate value into the destination register.
+  // synthesize the desired immediate value into the destination register.
   void emitLoadImm(MCRegister DestReg, int64_t Value, MCStreamer &Out);
 
   // Helper to emit a combination of AUIPC and SecondOpcode. Used to implement
@@ -280,7 +280,7 @@ class RISCVAsmParser : public MCTargetAsmParser {
   std::unique_ptr<RISCVOperand> defaultFRMArgLegacyOp() const;
 
 public:
-  enum RISCVMatchResultTy {
+  enum RISCVMatchResultTy : unsigned {
     Match_Dummy = FIRST_TARGET_MATCH_RESULT_TY,
 #define GET_OPERAND_DIAGNOSTIC_TYPES
 #include "RISCVGenAsmMatcher.inc"
@@ -877,6 +877,16 @@ struct RISCVOperand final : public MCParsedAsmOperand {
            VK == RISCVMCExpr::VK_RISCV_None;
   }
 
+  bool isUImm7Lsb000() const {
+    if (!isImm())
+      return false;
+    int64_t Imm;
+    RISCVMCExpr::VariantKind VK = RISCVMCExpr::VK_RISCV_None;
+    bool IsConstantImm = evaluateConstantImm(getImm(), Imm, VK);
+    return IsConstantImm && isShiftedUInt<4, 3>(Imm) &&
+           VK == RISCVMCExpr::VK_RISCV_None;
+  }
+
   bool isUImm8Lsb00() const {
     if (!isImm())
       return false;
@@ -1106,18 +1116,21 @@ struct RISCVOperand final : public MCParsedAsmOperand {
 
     switch (Kind) {
     case KindTy::Immediate:
-      OS << *getImm();
+      OS << "<imm: " << *Imm.Val << " " << (Imm.IsRV64 ? "rv64" : "rv32")
+         << ">";
       break;
     case KindTy::FPImmediate:
+      OS << "<fpimm: " << FPImm.Val << ">";
       break;
     case KindTy::Register:
-      OS << "<register " << RegName(getReg()) << ">";
+      OS << "<reg: " << RegName(Reg.RegNum) << " (" << Reg.RegNum
+         << (Reg.IsGPRAsFPR ? ") GPRasFPR>" : ")>");
       break;
     case KindTy::Token:
       OS << "'" << getToken() << "'";
       break;
     case KindTy::SystemRegister:
-      OS << "<sysreg: " << getSysReg() << '>';
+      OS << "<sysreg: " << getSysReg() << " (" << SysReg.Encoding << ")>";
       break;
     case KindTy::VType:
       OS << "<vtype: ";
@@ -1514,10 +1527,6 @@ bool RISCVAsmParser::matchAndEmitInstruction(SMLoc IDLoc, unsigned &Opcode,
         std::numeric_limits<uint32_t>::max(),
         "operand either must be a bare symbol name or an immediate integer in "
         "the range");
-  case Match_InvalidImmZero: {
-    SMLoc ErrorLoc = ((RISCVOperand &)*Operands[ErrorInfo]).getStartLoc();
-    return Error(ErrorLoc, "immediate must be zero");
-  }
   case Match_InvalidUImmLog2XLen:
     if (isRV64())
       return generateImmOutOfRangeError(Operands, ErrorInfo, 0, (1 << 6) - 1);
@@ -1644,43 +1653,10 @@ bool RISCVAsmParser::matchAndEmitInstruction(SMLoc IDLoc, unsigned &Opcode,
                                       "operand must be a valid system register "
                                       "name or an integer in the range");
   }
-  case Match_InvalidLoadFPImm: {
-    SMLoc ErrorLoc = ((RISCVOperand &)*Operands[ErrorInfo]).getStartLoc();
-    return Error(ErrorLoc, "operand must be a valid floating-point constant");
-  }
-  case Match_InvalidBareSymbol: {
-    SMLoc ErrorLoc = ((RISCVOperand &)*Operands[ErrorInfo]).getStartLoc();
-    return Error(ErrorLoc, "operand must be a bare symbol name");
-  }
-  case Match_InvalidPseudoJumpSymbol: {
-    SMLoc ErrorLoc = ((RISCVOperand &)*Operands[ErrorInfo]).getStartLoc();
-    return Error(ErrorLoc, "operand must be a valid jump target");
-  }
-  case Match_InvalidCallSymbol: {
-    SMLoc ErrorLoc = ((RISCVOperand &)*Operands[ErrorInfo]).getStartLoc();
-    return Error(ErrorLoc, "operand must be a bare symbol name");
-  }
-  case Match_InvalidTPRelAddSymbol: {
-    SMLoc ErrorLoc = ((RISCVOperand &)*Operands[ErrorInfo]).getStartLoc();
-    return Error(ErrorLoc, "operand must be a symbol with %tprel_add modifier");
-  }
-  case Match_InvalidTLSDESCCallSymbol: {
-    SMLoc ErrorLoc = ((RISCVOperand &)*Operands[ErrorInfo]).getStartLoc();
-    return Error(ErrorLoc,
-                 "operand must be a symbol with %tlsdesc_call modifier");
-  }
-  case Match_InvalidRTZArg: {
-    SMLoc ErrorLoc = ((RISCVOperand &)*Operands[ErrorInfo]).getStartLoc();
-    return Error(ErrorLoc, "operand must be 'rtz' floating-point rounding mode");
-  }
   case Match_InvalidVTypeI: {
     SMLoc ErrorLoc = ((RISCVOperand &)*Operands[ErrorInfo]).getStartLoc();
     return generateVTypeError(ErrorLoc);
   }
-  case Match_InvalidVMaskRegister: {
-    SMLoc ErrorLoc = ((RISCVOperand &)*Operands[ErrorInfo]).getStartLoc();
-    return Error(ErrorLoc, "operand must be v0.t");
-  }
   case Match_InvalidSImm5Plus1: {
     return generateImmOutOfRangeError(Operands, ErrorInfo, -(1 << 4) + 1,
                                       (1 << 4),
@@ -1689,26 +1665,14 @@ bool RISCVAsmParser::matchAndEmitInstruction(SMLoc IDLoc, unsigned &Opcode,
   case Match_InvalidSImm26:
     return generateImmOutOfRangeError(Operands, ErrorInfo, -(1 << 25),
                                       (1 << 25) - 1);
-  case Match_InvalidRlist: {
-    SMLoc ErrorLoc = ((RISCVOperand &)*Operands[ErrorInfo]).getStartLoc();
-    return Error(
-        ErrorLoc,
-        "operand must be {ra [, s0[-sN]]} or {x1 [, x8[-x9][, x18[-xN]]]}");
-  }
-  case Match_InvalidStackAdj: {
-    SMLoc ErrorLoc = ((RISCVOperand &)*Operands[ErrorInfo]).getStartLoc();
-    return Error(
-        ErrorLoc,
-        "stack adjustment is invalid for this instruction and register list; "
-        "refer to Zc spec for a detailed range of stack adjustment");
-  }
   case Match_InvalidRnumArg: {
     return generateImmOutOfRangeError(Operands, ErrorInfo, 0, 10);
   }
-  case Match_InvalidRegReg: {
-    SMLoc ErrorLoc = ((RISCVOperand &)*Operands[ErrorInfo]).getStartLoc();
-    return Error(ErrorLoc, "operands must be register and register");
   }
+
+  if (const char *MatchDiag = getMatchKindDiag((RISCVMatchResultTy)Result)) {
+    SMLoc ErrorLoc = ((RISCVOperand &)*Operands[ErrorInfo]).getStartLoc();
+    return Error(ErrorLoc, MatchDiag);
   }
 
   llvm_unreachable("Unknown match type detected!");
@@ -2612,7 +2576,7 @@ ParseStatus RISCVAsmParser::parseZeroOffsetMemOp(OperandVector &Operands) {
   std::unique_ptr<RISCVOperand> OptionalImmOp;
 
   if (getLexer().isNot(AsmToken::LParen)) {
-    // Parse an Integer token. We do not accept arbritrary constant expressions
+    // Parse an Integer token. We do not accept arbitrary constant expressions
     // in the offset field (because they may include parens, which complicates
     // parsing a lot).
     int64_t ImmVal;
diff --git a/llvm/lib/Target/RISCV/Disassembler/RISCVDisassembler.cpp b/llvm/lib/Target/RISCV/Disassembler/RISCVDisassembler.cpp
index a0b87f7c7ff25..01648ec0cbe9e 100644
--- a/llvm/lib/Target/RISCV/Disassembler/RISCVDisassembler.cpp
+++ b/llvm/lib/Target/RISCV/Disassembler/RISCVDisassembler.cpp
@@ -297,6 +297,16 @@ static DecodeStatus DecodeVRM8RegisterClass(MCInst &Inst, uint32_t RegNo,
   return MCDisassembler::Success;
 }
 
+static DecodeStatus DecodeVMV0RegisterClass(MCInst &Inst, uint32_t RegNo,
+                                            uint64_t Address,
+                                            const MCDisassembler *Decoder) {
+  if (RegNo)
+    return MCDisassembler::Fail;
+
+  Inst.addOperand(MCOperand::createReg(RISCV::V0));
+  return MCDisassembler::Success;
+}
+
 static DecodeStatus decodeVMaskReg(MCInst &Inst, uint32_t RegNo,
                                    uint64_t Address,
                                    const MCDisassembler *Decoder) {
@@ -671,6 +681,11 @@ DecodeStatus RISCVDisassembler::getInstruction32(MCInst &MI, uint64_t &Size,
                         "SiFive sf.cflush.d.l1 custom opcode table");
   TRY_TO_DECODE_FEATURE(RISCV::FeatureVendorXSfcease, DecoderTableXSfcease32,
                         "SiFive sf.cease custom opcode table");
+  TRY_TO_DECODE_FEATURE(RISCV::FeatureVendorXMIPSLSP, DecoderTableXmipslsp32,
+                        "MIPS mips.lsp custom opcode table");
+  TRY_TO_DECODE_FEATURE(RISCV::FeatureVendorXMIPSCMove,
+                        DecoderTableXmipscmove32,
+                        "MIPS mips.ccmov custom opcode table");
   TRY_TO_DECODE_FEATURE(RISCV::FeatureVendorXCVbitmanip,
                         DecoderTableXCVbitmanip32,
                         "CORE-V Bit Manipulation custom opcode table");
@@ -774,6 +789,7 @@ DecodeStatus RISCVDisassembler::getInstruction(MCInst &MI, uint64_t &Size,
                                                ArrayRef<uint8_t> Bytes,
                                                uint64_t Address,
                                                raw_ostream &CS) const {
+  CommentStream = &CS;
   // It's a 16 bit instruction if bit 0 and 1 are not 0b11.
   if ((Bytes[0] & 0b11) != 0b11)
     return getInstruction16(MI, Size, Bytes, Address, CS);
diff --git a/llvm/lib/Target/RISCV/GISel/RISCVInstructionSelector.cpp b/llvm/lib/Target/RISCV/GISel/RISCVInstructionSelector.cpp
index 3f1539da4a9c8..62fbe55dffba1 100644
--- a/llvm/lib/Target/RISCV/GISel/RISCVInstructionSelector.cpp
+++ b/llvm/lib/Target/RISCV/GISel/RISCVInstructionSelector.cpp
@@ -57,6 +57,17 @@ class RISCVInstructionSelector : public InstructionSelector {
   const TargetRegisterClass *
   getRegClassForTypeOnBank(LLT Ty, const RegisterBank &RB) const;
 
+  static constexpr unsigned MaxRecursionDepth = 6;
+
+  bool hasAllNBitUsers(const MachineInstr &MI, unsigned Bits,
+                       const unsigned Depth = 0) const;
+  bool hasAllHUsers(const MachineInstr &MI) const {
+    return hasAllNBitUsers(MI, 16);
+  }
+  bool hasAllWUsers(const MachineInstr &MI) const {
+    return hasAllNBitUsers(MI, 32);
+  }
+
   bool isRegInGprb(Register Reg) const;
   bool isRegInFprb(Register Reg) const;
 
@@ -128,8 +139,6 @@ class RISCVInstructionSelector : public InstructionSelector {
                           int OpIdx) const;
   void renderImmPlus1(MachineInstrBuilder &MIB, const MachineInstr &MI,
                       int OpIdx) const;
-  void renderImm(MachineInstrBuilder &MIB, const MachineInstr &MI,
-                 int OpIdx) const;
   void renderFrameIndex(MachineInstrBuilder &MIB, const MachineInstr &MI,
                         int OpIdx) const;
 
@@ -186,6 +195,79 @@ RISCVInstructionSelector::RISCVInstructionSelector(
 {
 }
 
+// Mimics optimizations in ISel and RISCVOptWInst Pass
+bool RISCVInstructionSelector::hasAllNBitUsers(const MachineInstr &MI,
+                                               unsigned Bits,
+                                               const unsigned Depth) const {
+
+  assert((MI.getOpcode() == TargetOpcode::G_ADD ||
+          MI.getOpcode() == TargetOpcode::G_SUB ||
+          MI.getOpcode() == TargetOpcode::G_MUL ||
+          MI.getOpcode() == TargetOpcode::G_SHL ||
+          MI.getOpcode() == TargetOpcode::G_LSHR ||
+          MI.getOpcode() == TargetOpcode::G_AND ||
+          MI.getOpcode() == TargetOpcode::G_OR ||
+          MI.getOpcode() == TargetOpcode::G_XOR ||
+          MI.getOpcode() == TargetOpcode::G_SEXT_INREG || Depth != 0) &&
+         "Unexpected opcode");
+
+  if (Depth >= RISCVInstructionSelector::MaxRecursionDepth)
+    return false;
+
+  auto DestReg = MI.getOperand(0).getReg();
+  for (auto &UserOp : MRI->use_nodbg_operands(DestReg)) {
+    assert(UserOp.getParent() && "UserOp must have a parent");
+    const MachineInstr &UserMI = *UserOp.getParent();
+    unsigned OpIdx = UserOp.getOperandNo();
+
+    switch (UserMI.getOpcode()) {
+    default:
+      return false;
+    case RISCV::ADDW:
+    case RISCV::ADDIW:
+    case RISCV::SUBW:
+      if (Bits >= 32)
+        break;
+      return false;
+    case RISCV::SLL:
+    case RISCV::SRA:
+    case RISCV::SRL:
+      // Shift amount operands only use log2(Xlen) bits.
+      if (OpIdx == 2 && Bits >= Log2_32(Subtarget->getXLen()))
+        break;
+      return false;
+    case RISCV::SLLI:
+      // SLLI only uses the lower (XLen - ShAmt) bits.
+      if (Bits >= Subtarget->getXLen() - UserMI.getOperand(2).getImm())
+        break;
+      return false;
+    case RISCV::ANDI:
+      if (Bits >= (unsigned)llvm::bit_width<uint64_t>(
+                      (uint64_t)UserMI.getOperand(2).getImm()))
+        break;
+      goto RecCheck;
+    case RISCV::AND:
+    case RISCV::OR:
+    case RISCV::XOR:
+    RecCheck:
+      if (hasAllNBitUsers(UserMI, Bits, Depth + 1))
+        break;
+      return false;
+    case RISCV::SRLI: {
+      unsigned ShAmt = UserMI.getOperand(2).getImm();
+      // If we are shifting right by less than Bits, and users don't demand any
+      // bits that were shifted into [Bits-1:0], then we can consider this as an
+      // N-Bit user.
+      if (Bits > ShAmt && hasAllNBitUsers(UserMI, Bits - ShAmt, Depth + 1))
+        break;
+      return false;
+    }
+    }
+  }
+
+  return true;
+}
+
 InstructionSelector::ComplexRendererFns
 RISCVInstructionSelector::selectShiftMask(MachineOperand &Root,
                                           unsigned ShiftWidth) const {
@@ -539,7 +621,7 @@ static void getOperandsForBranch(Register CondReg, RISCVCC::CondCode &CC,
     return;
   }
 
-  // We found an ICmp, do some canonicalizations.
+  // We found an ICmp, do some canonicalization.
 
   // Adjust comparisons to use comparison with 0 if possible.
   if (auto Constant = getIConstantVRegSExtVal(RHS, MRI)) {
@@ -653,7 +735,7 @@ bool RISCVInstructionSelector::select(MachineInstr &MI) {
     return true;
   }
   case TargetOpcode::G_FCONSTANT: {
-    // TODO: Use constant pool for complext constants.
+    // TODO: Use constant pool for complex constants.
     // TODO: Optimize +0.0 to use fcvt.d.w for s64 on rv32.
     Register DstReg = MI.getOperand(0).getReg();
     const APFloat &FPimm = MI.getOperand(1).getFPImm()->getValueAPF();
@@ -837,15 +919,6 @@ void RISCVInstructionSelector::renderImmPlus1(MachineInstrBuilder &MIB,
   MIB.addImm(CstVal + 1);
 }
 
-void RISCVInstructionSelector::renderImm(MachineInstrBuilder &MIB,
-                                         const MachineInstr &MI,
-                                         int OpIdx) const {
-  assert(MI.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 &&
-         "Expected G_CONSTANT");
-  int64_t CstVal = MI.getOperand(1).getCImm()->getSExtValue();
-  MIB.addImm(CstVal);
-}
-
 void RISCVInstructionSelector::renderFrameIndex(MachineInstrBuilder &MIB,
                                                 const MachineInstr &MI,
                                                 int OpIdx) const {
diff --git a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVBaseInfo.h b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVBaseInfo.h
index e9abc90d69a13..2f4b569041a6f 100644
--- a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVBaseInfo.h
+++ b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVBaseInfo.h
@@ -308,6 +308,7 @@ enum OperandType : unsigned {
   OPERAND_UIMM6_LSB0,
   OPERAND_UIMM7,
   OPERAND_UIMM7_LSB00,
+  OPERAND_UIMM7_LSB000,
   OPERAND_UIMM8_LSB00,
   OPERAND_UIMM8,
   OPERAND_UIMM8_LSB000,
diff --git a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVMatInt.cpp b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVMatInt.cpp
index 26725cf7decbe..06ae8e1296e51 100644
--- a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVMatInt.cpp
+++ b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVMatInt.cpp
@@ -175,7 +175,7 @@ static unsigned extractRotateInfo(int64_t Val) {
 
 static void generateInstSeqLeadingZeros(int64_t Val, const MCSubtargetInfo &STI,
                                         RISCVMatInt::InstSeq &Res) {
-  assert(Val > 0 && "Expected postive val");
+  assert(Val > 0 && "Expected positive val");
 
   unsigned LeadingZeros = llvm::countl_zero((uint64_t)Val);
   uint64_t ShiftedVal = (uint64_t)Val << LeadingZeros;
diff --git a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVTargetStreamer.cpp b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVTargetStreamer.cpp
index 99f57f47835ab..72b3e56c8a72f 100644
--- a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVTargetStreamer.cpp
+++ b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVTargetStreamer.cpp
@@ -21,7 +21,7 @@
 
 using namespace llvm;
 
-// This option controls wether or not we emit ELF attributes for ABI features,
+// This option controls whether or not we emit ELF attributes for ABI features,
 // like RISC-V atomics or X3 usage.
 static cl::opt<bool> RiscvAbiAttr(
     "riscv-abi-attributes",
diff --git a/llvm/lib/Target/RISCV/RISCV.h b/llvm/lib/Target/RISCV/RISCV.h
index d7bab601d545c..b1aee98739e85 100644
--- a/llvm/lib/Target/RISCV/RISCV.h
+++ b/llvm/lib/Target/RISCV/RISCV.h
@@ -84,6 +84,8 @@ void initializeRISCVMoveMergePass(PassRegistry &);
 
 FunctionPass *createRISCVPushPopOptimizationPass();
 void initializeRISCVPushPopOptPass(PassRegistry &);
+FunctionPass *createRISCVLoadStoreOptPass();
+void initializeRISCVLoadStoreOptPass(PassRegistry &);
 
 FunctionPass *createRISCVZacasABIFixPass();
 void initializeRISCVZacasABIFixPass(PassRegistry &);
diff --git a/llvm/lib/Target/RISCV/RISCV.td b/llvm/lib/Target/RISCV/RISCV.td
index 4e0c64a5ca2c6..87c07c3cd505f 100644
--- a/llvm/lib/Target/RISCV/RISCV.td
+++ b/llvm/lib/Target/RISCV/RISCV.td
@@ -50,6 +50,7 @@ include "RISCVSchedMIPSP8700.td"
 include "RISCVSchedRocket.td"
 include "RISCVSchedSiFive7.td"
 include "RISCVSchedSiFiveP400.td"
+include "RISCVSchedSiFiveP500.td"
 include "RISCVSchedSiFiveP600.td"
 include "RISCVSchedSyntacoreSCR1.td"
 include "RISCVSchedSyntacoreSCR345.td"
diff --git a/llvm/lib/Target/RISCV/RISCVAsmPrinter.cpp b/llvm/lib/Target/RISCV/RISCVAsmPrinter.cpp
index b1990409754b0..7dcf2ba2ac405 100644
--- a/llvm/lib/Target/RISCV/RISCVAsmPrinter.cpp
+++ b/llvm/lib/Target/RISCV/RISCVAsmPrinter.cpp
@@ -1089,7 +1089,7 @@ static bool lowerRISCVVMachineInstrToMCInst(const MachineInstr *MI,
   bool hasVLOutput = RISCV::isFaultFirstLoad(*MI);
   for (unsigned OpNo = 0; OpNo != NumOps; ++OpNo) {
     const MachineOperand &MO = MI->getOperand(OpNo);
-    // Skip vl ouput. It should be the second output.
+    // Skip vl output. It should be the second output.
     if (hasVLOutput && OpNo == 1)
       continue;
 
diff --git a/llvm/lib/Target/RISCV/RISCVCallingConv.td b/llvm/lib/Target/RISCV/RISCVCallingConv.td
index ad06f47743770..98e05b7f8eca7 100644
--- a/llvm/lib/Target/RISCV/RISCVCallingConv.td
+++ b/llvm/lib/Target/RISCV/RISCVCallingConv.td
@@ -42,6 +42,8 @@ def CSR_ILP32D_LP64D_V
 // Needed for implementation of RISCVRegisterInfo::getNoPreservedMask()
 def CSR_NoRegs : CalleeSavedRegs<(add)>;
 
+def CSR_IPRA : CalleeSavedRegs<(add X1)>;
+
 // Interrupt handler needs to save/restore all registers that are used,
 // both Caller and Callee saved registers.
 def CSR_Interrupt : CalleeSavedRegs<(add X1, (sequence "X%u", 5, 31))>;
diff --git a/llvm/lib/Target/RISCV/RISCVExpandPseudoInsts.cpp b/llvm/lib/Target/RISCV/RISCVExpandPseudoInsts.cpp
index eb3e1a1fe9fd5..0656bfbef6b35 100644
--- a/llvm/lib/Target/RISCV/RISCVExpandPseudoInsts.cpp
+++ b/llvm/lib/Target/RISCV/RISCVExpandPseudoInsts.cpp
@@ -341,15 +341,15 @@ bool RISCVExpandPseudo::expandRV32ZdinxStore(MachineBasicBlock &MBB,
                 .addImm(MBBI->getOperand(2).getImm() + 4);
   }
 
-  if (!MBBI->memoperands_empty()) {
-    assert(MBBI->hasOneMemOperand() && "Expected mem operand");
-    MachineMemOperand *OldMMO = MBBI->memoperands().front();
-    MachineFunction *MF = MBB.getParent();
-    MachineMemOperand *MMOLo = MF->getMachineMemOperand(OldMMO, 0, 4);
-    MachineMemOperand *MMOHi = MF->getMachineMemOperand(OldMMO, 4, 4);
-    MIBLo.setMemRefs(MMOLo);
-    MIBHi.setMemRefs(MMOHi);
+  MachineFunction *MF = MBB.getParent();
+  SmallVector<MachineMemOperand *> NewLoMMOs;
+  SmallVector<MachineMemOperand *> NewHiMMOs;
+  for (const MachineMemOperand *MMO : MBBI->memoperands()) {
+    NewLoMMOs.push_back(MF->getMachineMemOperand(MMO, 0, 4));
+    NewHiMMOs.push_back(MF->getMachineMemOperand(MMO, 4, 4));
   }
+  MIBLo.setMemRefs(NewLoMMOs);
+  MIBHi.setMemRefs(NewHiMMOs);
 
   MBBI->eraseFromParent();
   return true;
@@ -401,15 +401,15 @@ bool RISCVExpandPseudo::expandRV32ZdinxLoad(MachineBasicBlock &MBB,
                 .add(MBBI->getOperand(2));
   }
 
-  if (!MBBI->memoperands_empty()) {
-    assert(MBBI->hasOneMemOperand() && "Expected mem operand");
-    MachineMemOperand *OldMMO = MBBI->memoperands().front();
-    MachineFunction *MF = MBB.getParent();
-    MachineMemOperand *MMOLo = MF->getMachineMemOperand(OldMMO, 0, 4);
-    MachineMemOperand *MMOHi = MF->getMachineMemOperand(OldMMO, 4, 4);
-    MIBLo.setMemRefs(MMOLo);
-    MIBHi.setMemRefs(MMOHi);
+  MachineFunction *MF = MBB.getParent();
+  SmallVector<MachineMemOperand *> NewLoMMOs;
+  SmallVector<MachineMemOperand *> NewHiMMOs;
+  for (const MachineMemOperand *MMO : MBBI->memoperands()) {
+    NewLoMMOs.push_back(MF->getMachineMemOperand(MMO, 0, 4));
+    NewHiMMOs.push_back(MF->getMachineMemOperand(MMO, 4, 4));
   }
+  MIBLo.setMemRefs(NewLoMMOs);
+  MIBHi.setMemRefs(NewHiMMOs);
 
   MBBI->eraseFromParent();
   return true;
diff --git a/llvm/lib/Target/RISCV/RISCVFeatures.td b/llvm/lib/Target/RISCV/RISCVFeatures.td
index 4119dd77804f1..51aa8d7d307e4 100644
--- a/llvm/lib/Target/RISCV/RISCVFeatures.td
+++ b/llvm/lib/Target/RISCV/RISCVFeatures.td
@@ -1020,7 +1020,7 @@ def HasStdExtSmctrOrSsctr : Predicate<"Subtarget->hasStdExtSmctrOrSsctr()">,
 // Vendor extensions
 //===----------------------------------------------------------------------===//
 
-// Ventana Extenions
+// Ventana Extensions
 
 def FeatureVendorXVentanaCondOps
     : RISCVExtension<1, 0, "Ventana Conditional Ops">;
@@ -1228,6 +1228,21 @@ def HasVendorXCVbi
     : Predicate<"Subtarget->hasVendorXCVbi()">,
       AssemblerPredicate<(all_of FeatureVendorXCVbi),
                          "'XCVbi' (CORE-V Immediate Branching)">;
+// MIPS Extensions
+
+def FeatureVendorXMIPSCMove
+    : RISCVExtension<1, 0, "MIPS conditional move instruction(s) (ccmov)">;
+def HasVendorXMIPSCMove
+    : Predicate<"Subtarget->hasVendorXMIPSCMove()">,
+      AssemblerPredicate<(all_of FeatureVendorXMIPSCMove),
+                         "'Xmipscmove' ('mips.ccmov' instruction)">;
+def UseCCMovInsn : Predicate<"Subtarget->useCCMovInsn()">;
+def FeatureVendorXMIPSLSP
+    : RISCVExtension<1, 0, "MIPS optimization for hardware load-store bonding">;
+def HasVendorXMIPSLSP
+    : Predicate<"Subtarget->hasVendorXMIPSLSP()">,
+      AssemblerPredicate<(all_of FeatureVendorXMIPSLSP),
+                         "'Xmipslsp' (load and store pair instructions)">;
 
 // WCH / Nanjing Qinheng Microelectronics Extension(s)
 
@@ -1279,7 +1294,7 @@ def HasVendorXqcilsm
                          "'Xqcilsm' (Qualcomm uC Load Store Multiple Extension)">;
 
 def FeatureVendorXqciac
-    : RISCVExperimentalExtension<0, 2, "Qualcomm uC Load-Store Address Calculation Extension",
+    : RISCVExperimentalExtension<0, 3, "Qualcomm uC Load-Store Address Calculation Extension",
                                  [FeatureStdExtZca]>;
 def HasVendorXqciac
     : Predicate<"Subtarget->hasVendorXqciac()">,
@@ -1322,7 +1337,7 @@ def HasVendorXqcilo
 // LLVM specific features and extensions
 //===----------------------------------------------------------------------===//
 
-// Feature32Bit exists to mark CPUs that support RV32 to distinquish them from
+// Feature32Bit exists to mark CPUs that support RV32 to distinguish them from
 // tuning CPU names.
 def Feature32Bit
     : SubtargetFeature<"32bit", "IsRV32", "true", "Implements RV32">;
diff --git a/llvm/lib/Target/RISCV/RISCVFrameLowering.cpp b/llvm/lib/Target/RISCV/RISCVFrameLowering.cpp
index 333c8060f37f4..6abf45591d78e 100644
--- a/llvm/lib/Target/RISCV/RISCVFrameLowering.cpp
+++ b/llvm/lib/Target/RISCV/RISCVFrameLowering.cpp
@@ -102,13 +102,13 @@ RISCVFrameLowering::RISCVFrameLowering(const RISCVSubtarget &STI)
       STI(STI) {}
 
 // The register used to hold the frame pointer.
-static constexpr Register FPReg = RISCV::X8;
+static constexpr MCPhysReg FPReg = RISCV::X8;
 
 // The register used to hold the stack pointer.
-static constexpr Register SPReg = RISCV::X2;
+static constexpr MCPhysReg SPReg = RISCV::X2;
 
 // The register used to hold the return address.
-static constexpr Register RAReg = RISCV::X1;
+static constexpr MCPhysReg RAReg = RISCV::X1;
 
 // Offsets which need to be scale by XLen representing locations of CSRs which
 // are given a fixed location by save/restore libcalls or Zcmp Push/Pop.
@@ -250,17 +250,17 @@ static int getLibCallID(const MachineFunction &MF,
   if (CSI.empty() || !RVFI->useSaveRestoreLibCalls(MF))
     return -1;
 
-  Register MaxReg = RISCV::NoRegister;
+  Register MaxReg;
   for (auto &CS : CSI)
     // assignCalleeSavedSpillSlots assigns negative frame indexes to
     // registers which can be saved by libcall.
     if (CS.getFrameIdx() < 0)
       MaxReg = std::max(MaxReg.id(), CS.getReg().id());
 
-  if (MaxReg == RISCV::NoRegister)
+  if (!MaxReg)
     return -1;
 
-  switch (MaxReg) {
+  switch (MaxReg.id()) {
   default:
     llvm_unreachable("Something has gone wrong!");
     // clang-format off
@@ -339,7 +339,7 @@ getRestoreLibCallName(const MachineFunction &MF,
 // representing registers to store/load.
 static std::pair<unsigned, unsigned>
 getPushPopEncodingAndNum(const Register MaxReg) {
-  switch (MaxReg) {
+  switch (MaxReg.id()) {
   default:
     llvm_unreachable("Unexpected Reg for Push/Pop Inst");
   case RISCV::X27: /*s11*/
@@ -810,8 +810,7 @@ void RISCVFrameLowering::emitPrologue(MachineFunction &MF,
 
   auto FirstFrameSetup = MBBI;
 
-  // Since spillCalleeSavedRegisters may have inserted a libcall, skip past
-  // any instructions marked as FrameSetup
+  // Skip past all callee-saved register spill instructions.
   while (MBBI != MBB.end() && MBBI->getFlag(MachineInstr::FrameSetup))
     ++MBBI;
 
@@ -820,6 +819,12 @@ void RISCVFrameLowering::emitPrologue(MachineFunction &MF,
 
   const auto &CSI = MFI.getCalleeSavedInfo();
 
+  // Skip to before the spills of scalar callee-saved registers
+  // FIXME: assumes exactly one instruction is used to restore each
+  // callee-saved register.
+  MBBI = std::prev(MBBI, getRVVCalleeSavedInfo(MF, CSI).size() +
+                             getUnmanagedCSI(MF, CSI).size());
+
   // If libcalls are used to spill and restore callee-saved registers, the frame
   // has two sections; the opaque section managed by the libcalls, and the
   // section managed by MachineFrameInfo which can also hold callee saved
@@ -1076,8 +1081,7 @@ void RISCVFrameLowering::emitEpilogue(MachineFunction &MF,
 
     MBBI = MBB.getFirstTerminator();
 
-    // If callee-saved registers are saved via libcall, place stack adjustment
-    // before this call.
+    // Skip to before the restores of all callee-saved registers.
     while (MBBI != MBB.begin() &&
            std::prev(MBBI)->getFlag(MachineInstr::FrameDestroy))
       --MBBI;
@@ -1088,7 +1092,8 @@ void RISCVFrameLowering::emitEpilogue(MachineFunction &MF,
   // Skip to before the restores of scalar callee-saved registers
   // FIXME: assumes exactly one instruction is used to restore each
   // callee-saved register.
-  auto LastFrameDestroy = std::prev(MBBI, getUnmanagedCSI(MF, CSI).size());
+  auto FirstScalarCSRRestoreInsn =
+      std::next(MBBI, getRVVCalleeSavedInfo(MF, CSI).size());
 
   uint64_t FirstSPAdjustAmount = getFirstSPAdjustAmount(MF);
   uint64_t RealStackSize = FirstSPAdjustAmount ? FirstSPAdjustAmount
@@ -1105,20 +1110,20 @@ void RISCVFrameLowering::emitEpilogue(MachineFunction &MF,
     // If RestoreSPFromFP the stack pointer will be restored using the frame
     // pointer value.
     if (!RestoreSPFromFP)
-      RI->adjustReg(MBB, LastFrameDestroy, DL, SPReg, SPReg,
+      RI->adjustReg(MBB, FirstScalarCSRRestoreInsn, DL, SPReg, SPReg,
                     StackOffset::getScalable(RVVStackSize),
                     MachineInstr::FrameDestroy, getStackAlign());
 
     if (!hasFP(MF)) {
       unsigned CFIIndex = MF.addFrameInst(MCCFIInstruction::cfiDefCfa(
           nullptr, RI->getDwarfRegNum(SPReg, true), RealStackSize));
-      BuildMI(MBB, LastFrameDestroy, DL,
+      BuildMI(MBB, FirstScalarCSRRestoreInsn, DL,
               TII->get(TargetOpcode::CFI_INSTRUCTION))
           .addCFIIndex(CFIIndex)
           .setMIFlag(MachineInstr::FrameDestroy);
     }
 
-    emitCalleeSavedRVVEpilogCFI(MBB, LastFrameDestroy);
+    emitCalleeSavedRVVEpilogCFI(MBB, FirstScalarCSRRestoreInsn);
   }
 
   if (FirstSPAdjustAmount) {
@@ -1130,14 +1135,14 @@ void RISCVFrameLowering::emitEpilogue(MachineFunction &MF,
     // If RestoreSPFromFP the stack pointer will be restored using the frame
     // pointer value.
     if (!RestoreSPFromFP)
-      RI->adjustReg(MBB, LastFrameDestroy, DL, SPReg, SPReg,
+      RI->adjustReg(MBB, FirstScalarCSRRestoreInsn, DL, SPReg, SPReg,
                     StackOffset::getFixed(SecondSPAdjustAmount),
                     MachineInstr::FrameDestroy, getStackAlign());
 
     if (!hasFP(MF)) {
       unsigned CFIIndex = MF.addFrameInst(
           MCCFIInstruction::cfiDefCfaOffset(nullptr, FirstSPAdjustAmount));
-      BuildMI(MBB, LastFrameDestroy, DL,
+      BuildMI(MBB, FirstScalarCSRRestoreInsn, DL,
               TII->get(TargetOpcode::CFI_INSTRUCTION))
           .addCFIIndex(CFIIndex)
           .setMIFlag(MachineInstr::FrameDestroy);
@@ -1156,7 +1161,7 @@ void RISCVFrameLowering::emitEpilogue(MachineFunction &MF,
   // have vector objects in stack.
   if (RestoreSPFromFP) {
     assert(hasFP(MF) && "frame pointer should not have been eliminated");
-    RI->adjustReg(MBB, LastFrameDestroy, DL, SPReg, FPReg,
+    RI->adjustReg(MBB, FirstScalarCSRRestoreInsn, DL, SPReg, FPReg,
                   StackOffset::getFixed(-FPOffset), MachineInstr::FrameDestroy,
                   getStackAlign());
   }
@@ -1164,14 +1169,20 @@ void RISCVFrameLowering::emitEpilogue(MachineFunction &MF,
   if (hasFP(MF)) {
     unsigned CFIIndex = MF.addFrameInst(MCCFIInstruction::cfiDefCfa(
         nullptr, RI->getDwarfRegNum(SPReg, true), RealStackSize));
-    BuildMI(MBB, LastFrameDestroy, DL, TII->get(TargetOpcode::CFI_INSTRUCTION))
+    BuildMI(MBB, FirstScalarCSRRestoreInsn, DL,
+            TII->get(TargetOpcode::CFI_INSTRUCTION))
         .addCFIIndex(CFIIndex)
         .setMIFlag(MachineInstr::FrameDestroy);
   }
 
+  // Skip to after the restores of scalar callee-saved registers
+  // FIXME: assumes exactly one instruction is used to restore each
+  // callee-saved register.
+  MBBI = std::next(FirstScalarCSRRestoreInsn, getUnmanagedCSI(MF, CSI).size());
+
   if (getLibCallID(MF, CSI) != -1) {
     // tail __riscv_restore_[0-12] instruction is considered as a terminator,
-    // therefor it is unnecessary to place any CFI instructions after it. Just
+    // therefore it is unnecessary to place any CFI instructions after it. Just
     // deallocate stack if needed and return.
     if (StackSize != 0)
       deallocateStack(MF, MBB, MBBI, DL, StackSize,
@@ -1798,7 +1809,7 @@ bool RISCVFrameLowering::assignCalleeSavedSpillSlots(
   const TargetRegisterInfo *RegInfo = MF.getSubtarget().getRegisterInfo();
 
   for (auto &CS : CSI) {
-    unsigned Reg = CS.getReg();
+    Register Reg = CS.getReg();
     const TargetRegisterClass *RC = RegInfo->getMinimalPhysRegClass(Reg);
     unsigned Size = RegInfo->getSpillSize(*RC);
 
@@ -1836,11 +1847,16 @@ bool RISCVFrameLowering::assignCalleeSavedSpillSlots(
       MFI.setStackID(FrameIdx, TargetStackID::ScalableVector);
   }
 
-  // Allocate a fixed object that covers the full push or libcall size.
   if (RVFI->isPushable(MF)) {
-    if (int64_t PushSize = RVFI->getRVPushStackSize())
-      MFI.CreateFixedSpillStackObject(PushSize, -PushSize);
+    // Allocate a fixed object that covers all the registers that are pushed.
+    if (unsigned PushedRegs = RVFI->getRVPushRegs()) {
+      int64_t PushedRegsBytes =
+          static_cast<int64_t>(PushedRegs) * (STI.getXLen() / 8);
+      MFI.CreateFixedSpillStackObject(PushedRegsBytes, -PushedRegsBytes);
+    }
   } else if (int LibCallRegs = getLibCallID(MF, CSI) + 1) {
+    // Allocate a fixed object that covers all of the stack allocated by the
+    // libcall.
     int64_t LibCallFrameSize =
         alignTo((STI.getXLen() / 8) * LibCallRegs, getStackAlign());
     MFI.CreateFixedSpillStackObject(LibCallFrameSize, -LibCallFrameSize);
@@ -1898,7 +1914,8 @@ bool RISCVFrameLowering::spillCalleeSavedRegisters(
       Register Reg = CS.getReg();
       const TargetRegisterClass *RC = TRI->getMinimalPhysRegClass(Reg);
       TII.storeRegToStackSlot(MBB, MI, Reg, !MBB.isLiveIn(Reg),
-                              CS.getFrameIdx(), RC, TRI, Register());
+                              CS.getFrameIdx(), RC, TRI, Register(),
+                              MachineInstr::FrameSetup);
     }
   };
   storeRegsToStackSlots(UnmanagedCSI);
@@ -2009,7 +2026,7 @@ bool RISCVFrameLowering::restoreCalleeSavedRegisters(
       Register Reg = CS.getReg();
       const TargetRegisterClass *RC = TRI->getMinimalPhysRegClass(Reg);
       TII.loadRegFromStackSlot(MBB, MI, Reg, CS.getFrameIdx(), RC, TRI,
-                               Register());
+                               Register(), MachineInstr::FrameDestroy);
       assert(MI != MBB.begin() &&
              "loadRegFromStackSlot didn't insert any code!");
     }
diff --git a/llvm/lib/Target/RISCV/RISCVGatherScatterLowering.cpp b/llvm/lib/Target/RISCV/RISCVGatherScatterLowering.cpp
index 39c0af7985971..82c0d8d4738a4 100644
--- a/llvm/lib/Target/RISCV/RISCVGatherScatterLowering.cpp
+++ b/llvm/lib/Target/RISCV/RISCVGatherScatterLowering.cpp
@@ -131,7 +131,7 @@ static std::pair<Value *, Value *> matchStridedStart(Value *Start,
   }
 
   // Not a constant, maybe it's a strided constant with a splat added or
-  // multipled.
+  // multiplied.
   auto *BO = dyn_cast<BinaryOperator>(Start);
   if (!BO || (BO->getOpcode() != Instruction::Add &&
               BO->getOpcode() != Instruction::Or &&
diff --git a/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.cpp b/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.cpp
index 9855028ead9e2..ec2e8f1d50264 100644
--- a/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.cpp
+++ b/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.cpp
@@ -119,6 +119,19 @@ void RISCVDAGToDAGISel::PreprocessISelDAG() {
                                            MachineMemOperand::MOLoad);
       break;
     }
+    case ISD::FP_EXTEND: {
+      // We only have vector patterns for riscv_fpextend_vl in isel.
+      SDLoc DL(N);
+      MVT VT = N->getSimpleValueType(0);
+      if (!VT.isVector())
+        break;
+      SDValue VLMAX = CurDAG->getRegister(RISCV::X0, Subtarget->getXLenVT());
+      SDValue TrueMask = CurDAG->getNode(
+          RISCVISD::VMSET_VL, DL, VT.changeVectorElementType(MVT::i1), VLMAX);
+      Result = CurDAG->getNode(RISCVISD::FP_EXTEND_VL, DL, VT, N->getOperand(0),
+                               TrueMask, VLMAX);
+      break;
+    }
     }
 
     if (Result) {
@@ -3486,7 +3499,7 @@ bool RISCVDAGToDAGISel::selectSimm5Shl2(SDValue N, SDValue &Simm5,
 }
 
 // Select VL as a 5 bit immediate or a value that will become a register. This
-// allows us to choose betwen VSETIVLI or VSETVLI later.
+// allows us to choose between VSETIVLI or VSETVLI later.
 bool RISCVDAGToDAGISel::selectVLOp(SDValue N, SDValue &VL) {
   auto *C = dyn_cast<ConstantSDNode>(N);
   if (C && isUInt<5>(C->getZExtValue())) {
@@ -3538,7 +3551,8 @@ bool RISCVDAGToDAGISel::selectVSplat(SDValue N, SDValue &SplatVal) {
 static bool selectVSplatImmHelper(SDValue N, SDValue &SplatVal,
                                   SelectionDAG &DAG,
                                   const RISCVSubtarget &Subtarget,
-                                  std::function<bool(int64_t)> ValidateImm) {
+                                  std::function<bool(int64_t)> ValidateImm,
+                                  bool Decrement = false) {
   SDValue Splat = findVSplat(N);
   if (!Splat || !isa<ConstantSDNode>(Splat.getOperand(1)))
     return false;
@@ -3561,6 +3575,9 @@ static bool selectVSplatImmHelper(SDValue N, SDValue &SplatVal,
   if (!ValidateImm(SplatImm))
     return false;
 
+  if (Decrement)
+    SplatImm -= 1;
+
   SplatVal =
       DAG.getSignedTargetConstant(SplatImm, SDLoc(N), Subtarget.getXLenVT());
   return true;
@@ -3574,15 +3591,18 @@ bool RISCVDAGToDAGISel::selectVSplatSimm5(SDValue N, SDValue &SplatVal) {
 bool RISCVDAGToDAGISel::selectVSplatSimm5Plus1(SDValue N, SDValue &SplatVal) {
   return selectVSplatImmHelper(
       N, SplatVal, *CurDAG, *Subtarget,
-      [](int64_t Imm) { return (isInt<5>(Imm) && Imm != -16) || Imm == 16; });
+      [](int64_t Imm) { return (isInt<5>(Imm) && Imm != -16) || Imm == 16; },
+      /*Decrement=*/true);
 }
 
 bool RISCVDAGToDAGISel::selectVSplatSimm5Plus1NonZero(SDValue N,
                                                       SDValue &SplatVal) {
   return selectVSplatImmHelper(
-      N, SplatVal, *CurDAG, *Subtarget, [](int64_t Imm) {
+      N, SplatVal, *CurDAG, *Subtarget,
+      [](int64_t Imm) {
         return Imm != 0 && ((isInt<5>(Imm) && Imm != -16) || Imm == 16);
-      });
+      },
+      /*Decrement=*/true);
 }
 
 bool RISCVDAGToDAGISel::selectVSplatUimm(SDValue N, unsigned Bits,
diff --git a/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.h b/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.h
index 592f517358506..bb786e4b2bb40 100644
--- a/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.h
+++ b/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.h
@@ -278,6 +278,7 @@ struct VLX_VSXPseudo {
 #define GET_RISCVVSETable_DECL
 #define GET_RISCVVLXTable_DECL
 #define GET_RISCVVSXTable_DECL
+#include "RISCVGenSearchableTables.inc"
 } // namespace RISCV
 
 } // namespace llvm
diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
index 5e5bc0819a10c..ff44ff5249973 100644
--- a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
+++ b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
@@ -22,6 +22,7 @@
 #include "llvm/ADT/SmallSet.h"
 #include "llvm/ADT/Statistic.h"
 #include "llvm/Analysis/MemoryLocation.h"
+#include "llvm/Analysis/ValueTracking.h"
 #include "llvm/Analysis/VectorUtils.h"
 #include "llvm/CodeGen/MachineFrameInfo.h"
 #include "llvm/CodeGen/MachineFunction.h"
@@ -409,7 +410,9 @@ RISCVTargetLowering::RISCVTargetLowering(const TargetMachine &TM,
     setOperationAction(ISD::ABS, MVT::i32, Custom);
   }
 
-  if (!Subtarget.hasVendorXTHeadCondMov())
+  if (Subtarget.useCCMovInsn())
+    setOperationAction(ISD::SELECT, XLenVT, Legal);
+  else if (!Subtarget.hasVendorXTHeadCondMov())
     setOperationAction(ISD::SELECT, XLenVT, Custom);
 
   static const unsigned FPLegalNodeTypes[] = {
@@ -1186,6 +1189,9 @@ RISCVTargetLowering::RISCVTargetLowering(const TargetMachine &TM,
             {ISD::BUILD_VECTOR, ISD::CONCAT_VECTORS, ISD::VECTOR_REVERSE}, VT,
             Custom);
 
+        setOperationAction({ISD::VECTOR_INTERLEAVE, ISD::VECTOR_DEINTERLEAVE},
+                           VT, Custom);
+
         setOperationAction({ISD::INSERT_VECTOR_ELT, ISD::EXTRACT_VECTOR_ELT},
                            VT, Custom);
 
@@ -1341,6 +1347,9 @@ RISCVTargetLowering::RISCVTargetLowering(const TargetMachine &TM,
                             ISD::VECTOR_SHUFFLE, ISD::VECTOR_COMPRESS},
                            VT, Custom);
 
+        setOperationAction({ISD::VECTOR_INTERLEAVE, ISD::VECTOR_DEINTERLEAVE},
+                           VT, Custom);
+
         setOperationAction({ISD::LOAD, ISD::STORE, ISD::MLOAD, ISD::MSTORE,
                             ISD::MGATHER, ISD::MSCATTER},
                            VT, Custom);
@@ -1524,18 +1533,16 @@ RISCVTargetLowering::RISCVTargetLowering(const TargetMachine &TM,
     setTargetDAGCombine({ISD::ZERO_EXTEND, ISD::FP_TO_SINT, ISD::FP_TO_UINT,
                          ISD::FP_TO_SINT_SAT, ISD::FP_TO_UINT_SAT});
   if (Subtarget.hasVInstructions())
-    setTargetDAGCombine({ISD::FCOPYSIGN,     ISD::MGATHER,
-                         ISD::MSCATTER,      ISD::VP_GATHER,
-                         ISD::VP_SCATTER,    ISD::SRA,
-                         ISD::SRL,           ISD::SHL,
-                         ISD::STORE,         ISD::SPLAT_VECTOR,
-                         ISD::BUILD_VECTOR,  ISD::CONCAT_VECTORS,
-                         ISD::VP_STORE,      ISD::EXPERIMENTAL_VP_REVERSE,
-                         ISD::MUL,           ISD::SDIV,
-                         ISD::UDIV,          ISD::SREM,
-                         ISD::UREM,          ISD::INSERT_VECTOR_ELT,
-                         ISD::ABS,           ISD::CTPOP,
-                         ISD::VECTOR_SHUFFLE, ISD::VSELECT});
+    setTargetDAGCombine(
+        {ISD::FCOPYSIGN,    ISD::MGATHER,      ISD::MSCATTER,
+         ISD::VP_GATHER,    ISD::VP_SCATTER,   ISD::SRA,
+         ISD::SRL,          ISD::SHL,          ISD::STORE,
+         ISD::SPLAT_VECTOR, ISD::BUILD_VECTOR, ISD::CONCAT_VECTORS,
+         ISD::VP_STORE,     ISD::VP_TRUNCATE,  ISD::EXPERIMENTAL_VP_REVERSE,
+         ISD::MUL,          ISD::SDIV,         ISD::UDIV,
+         ISD::SREM,         ISD::UREM,         ISD::INSERT_VECTOR_ELT,
+         ISD::ABS,          ISD::CTPOP,        ISD::VECTOR_SHUFFLE,
+         ISD::VSELECT});
 
   if (Subtarget.hasVendorXTHeadMemPair())
     setTargetDAGCombine({ISD::LOAD, ISD::STORE});
@@ -2070,7 +2077,7 @@ bool RISCVTargetLowering::shouldConvertConstantLoadToIntImm(const APInt &Imm,
   if (isInt<32>(Val))
     return true;
 
-  // A constant pool entry may be more aligned thant he load we're trying to
+  // A constant pool entry may be more aligned than the load we're trying to
   // replace. If we don't support unaligned scalar mem, prefer the constant
   // pool.
   // TODO: Can the caller pass down the alignment?
@@ -2914,7 +2921,7 @@ static SDValue lowerFP_TO_INT_SAT(SDValue Op, SelectionDAG &DAG,
   bool IsSigned = Op.getOpcode() == ISD::FP_TO_SINT_SAT;
 
   if (!DstVT.isVector()) {
-    // For bf16 or for f16 in absense of Zfh, promote to f32, then saturate
+    // For bf16 or for f16 in absence of Zfh, promote to f32, then saturate
     // the result.
     if ((Src.getValueType() == MVT::f16 && !Subtarget.hasStdExtZfhOrZhinx()) ||
         Src.getValueType() == MVT::bf16) {
@@ -3179,7 +3186,7 @@ lowerVectorFTRUNC_FCEIL_FFLOOR_FROUND(SDValue Op, SelectionDAG &DAG,
 
 // Expand vector STRICT_FTRUNC, STRICT_FCEIL, STRICT_FFLOOR, STRICT_FROUND
 // STRICT_FROUNDEVEN and STRICT_FNEARBYINT by converting sNan of the source to
-// qNan and coverting the new source to integer and back to FP.
+// qNan and converting the new source to integer and back to FP.
 static SDValue
 lowerVectorStrictFTRUNC_FCEIL_FFLOOR_FROUND(SDValue Op, SelectionDAG &DAG,
                                             const RISCVSubtarget &Subtarget) {
@@ -3199,7 +3206,7 @@ lowerVectorStrictFTRUNC_FCEIL_FFLOOR_FROUND(SDValue Op, SelectionDAG &DAG,
   // Freeze the source since we are increasing the number of uses.
   Src = DAG.getFreeze(Src);
 
-  // Covert sNan to qNan by executing x + x for all unordered elemenet x in Src.
+  // Convert sNan to qNan by executing x + x for all unordered element x in Src.
   MVT MaskVT = Mask.getSimpleValueType();
   SDValue Unorder = DAG.getNode(RISCVISD::STRICT_FSETCC_VL, DL,
                                 DAG.getVTList(MaskVT, MVT::Other),
@@ -3717,7 +3724,7 @@ static SDValue lowerBuildVectorOfConstants(SDValue Op, SelectionDAG &DAG,
     unsigned NumViaIntegerBits = std::clamp(NumElts, 8u, Subtarget.getXLen());
     NumViaIntegerBits = std::min(NumViaIntegerBits, Subtarget.getELen());
     // If we have to use more than one INSERT_VECTOR_ELT then this
-    // optimization is likely to increase code size; avoid peforming it in
+    // optimization is likely to increase code size; avoid performing it in
     // such a case. We can use a load from a constant pool in this case.
     if (DAG.shouldOptForSize() && NumElts > NumViaIntegerBits)
       return SDValue();
@@ -4510,7 +4517,8 @@ static SDValue getSingleShuffleSrc(MVT VT, MVT ContainerVT, SDValue V1,
 
   // Src needs to have twice the number of elements.
   unsigned NumElts = VT.getVectorNumElements();
-  if (Src.getValueType().getVectorNumElements() != (NumElts * 2))
+  if (!Src.getValueType().isFixedLengthVector() ||
+      Src.getValueType().getVectorNumElements() != (NumElts * 2))
     return SDValue();
 
   // The extracts must extract the two halves of the source.
@@ -4610,7 +4618,7 @@ static int isElementRotate(int &LoSrc, int &HiSrc, ArrayRef<int> Mask) {
     int MaskSrc = M < Size ? 0 : 1;
 
     // Compute which of the two target values this index should be assigned to.
-    // This reflects whether the high elements are remaining or the low elemnts
+    // This reflects whether the high elements are remaining or the low elements
     // are remaining.
     int &TargetSrc = StartIdx < 0 ? HiSrc : LoSrc;
 
@@ -5212,17 +5220,21 @@ static SDValue lowerShuffleViaVRegSplitting(ShuffleVectorSDNode *SVN,
     SmallDenseMap<unsigned, SDValue, 4> Values;
     for (unsigned I : seq<unsigned>(Data.size())) {
       const auto &[Idx1, Idx2, _] = Data[I];
-      if (Values.contains(Idx1)) {
-        assert(Idx2 != UINT_MAX && Values.contains(Idx2) &&
-               "Expected both indices to be extracted already.");
-        break;
+      // If the shuffle contains permutation of odd number of elements,
+      // Idx1 might be used already in the first iteration.
+      //
+      // Idx1 = shuffle Idx1, Idx2
+      // Idx1 = shuffle Idx1, Idx3
+      SDValue &V = Values.try_emplace(Idx1).first->getSecond();
+      if (!V)
+        V = ExtractValue(Idx1 >= NumOfSrcRegs ? V2 : V1,
+                         (Idx1 % NumOfSrcRegs) * NumOpElts);
+      if (Idx2 != UINT_MAX) {
+        SDValue &V = Values.try_emplace(Idx2).first->getSecond();
+        if (!V)
+          V = ExtractValue(Idx2 >= NumOfSrcRegs ? V2 : V1,
+                           (Idx2 % NumOfSrcRegs) * NumOpElts);
       }
-      SDValue V = ExtractValue(Idx1 >= NumOfSrcRegs ? V2 : V1,
-                               (Idx1 % NumOfSrcRegs) * NumOpElts);
-      Values[Idx1] = V;
-      if (Idx2 != UINT_MAX)
-        Values[Idx2] = ExtractValue(Idx2 >= NumOfSrcRegs ? V2 : V1,
-                                    (Idx2 % NumOfSrcRegs) * NumOpElts);
     }
     SDValue V;
     for (const auto &[Idx1, Idx2, Mask] : Data) {
@@ -5323,6 +5335,21 @@ static SDValue lowerDisjointIndicesShuffle(ShuffleVectorSDNode *SVN,
   return DAG.getVectorShuffle(VT, DL, Select, DAG.getUNDEF(VT), NewMask);
 }
 
+/// Is this mask local (i.e. elements only move within their local span), and
+/// repeating (that is, the same rearrangement is being done within each span)?
+static bool isLocalRepeatingShuffle(ArrayRef<int> Mask, int Span) {
+  // TODO: Could improve the case where undef elements exist in the first span.
+  for (auto [I, M] : enumerate(Mask)) {
+    if (M == -1)
+      continue;
+    int ChunkLo = I - (I % Span);
+    int ChunkHi = ChunkLo + Span;
+    if (M < ChunkLo || M >= ChunkHi || M - ChunkLo != Mask[I % Span])
+      return false;
+  }
+  return true;
+}
+
 /// Try to widen element type to get a new mask value for a better permutation
 /// sequence.  This doesn't try to inspect the widened mask for profitability;
 /// we speculate the widened form is equal or better.  This has the effect of
@@ -5601,6 +5628,47 @@ static SDValue lowerVECTOR_SHUFFLE(SDValue Op, SelectionDAG &DAG,
     if (SDValue V = lowerVECTOR_SHUFFLEAsRotate(SVN, DAG, Subtarget))
       return V;
 
+    // Match a spread(4,8) which can be done via extend and shift.  Spread(2)
+    // is fully covered in interleave(2) above, so it is ignored here.
+    if (VT.getScalarSizeInBits() < Subtarget.getELen()) {
+      unsigned MaxFactor = Subtarget.getELen() / VT.getScalarSizeInBits();
+      assert(MaxFactor == 2 || MaxFactor == 4 || MaxFactor == 8);
+      for (unsigned Factor = 4; Factor <= MaxFactor; Factor <<= 1) {
+        unsigned Index;
+        if (isSpreadMask(Mask, Factor, Index)) {
+          MVT NarrowVT =
+              MVT::getVectorVT(VT.getVectorElementType(), NumElts / Factor);
+          SDValue Src = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, NarrowVT, V1,
+                                    DAG.getVectorIdxConstant(0, DL));
+          return getWideningSpread(Src, Factor, Index, DL, DAG);
+        }
+      }
+    }
+
+    // If only a prefix of the source elements influence a prefix of the
+    // destination elements, try to see if we can reduce the required LMUL
+    unsigned MinVLen = Subtarget.getRealMinVLen();
+    unsigned MinVLMAX = MinVLen / VT.getScalarSizeInBits();
+    if (NumElts > MinVLMAX) {
+      unsigned MaxIdx = 0;
+      for (auto [I, M] : enumerate(Mask)) {
+        if (M == -1)
+          continue;
+        MaxIdx = std::max(std::max((unsigned)I, (unsigned)M), MaxIdx);
+      }
+      unsigned NewNumElts =
+          std::max((uint64_t)MinVLMAX, PowerOf2Ceil(MaxIdx + 1));
+      if (NewNumElts != NumElts) {
+        MVT NewVT = MVT::getVectorVT(VT.getVectorElementType(), NewNumElts);
+        SDValue ZeroIdx = DAG.getVectorIdxConstant(0, DL);
+        V1 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, NewVT, V1, ZeroIdx);
+        SDValue Res = DAG.getVectorShuffle(NewVT, DL, V1, DAG.getUNDEF(NewVT),
+                                           Mask.take_front(NewNumElts));
+        return DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT, DAG.getUNDEF(VT), Res,
+                           ZeroIdx);
+      }
+    }
+
     // Before hitting generic lowering fallbacks, try to widen the mask
     // to a wider SEW.
     if (SDValue V = tryWidenMaskForShuffle(Op, DAG))
@@ -5625,23 +5693,6 @@ static SDValue lowerVECTOR_SHUFFLE(SDValue Op, SelectionDAG &DAG,
                          DAG.getUNDEF(VT));
     }
 
-    // Match a spread(4,8) which can be done via extend and shift.  Spread(2)
-    // is fully covered in interleave(2) above, so it is ignored here.
-    if (VT.getScalarSizeInBits() < Subtarget.getELen()) {
-      unsigned MaxFactor = Subtarget.getELen() / VT.getScalarSizeInBits();
-      assert(MaxFactor == 2 || MaxFactor == 4 || MaxFactor == 8);
-      for (unsigned Factor = 4; Factor <= MaxFactor; Factor <<= 1) {
-        unsigned Index;
-        if (isSpreadMask(Mask, Factor, Index)) {
-          MVT NarrowVT =
-              MVT::getVectorVT(VT.getVectorElementType(), NumElts / Factor);
-          SDValue Src = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, NarrowVT, V1,
-                                    DAG.getVectorIdxConstant(0, DL));
-          return getWideningSpread(Src, Factor, Index, DL, DAG);
-        }
-      }
-    }
-
     if (VT.getScalarSizeInBits() == 8 &&
         any_of(Mask, [&](const auto &Idx) { return Idx > 255; })) {
       // On such a vector we're unable to use i8 as the index type.
@@ -5684,10 +5735,41 @@ static SDValue lowerVECTOR_SHUFFLE(SDValue Op, SelectionDAG &DAG,
                                  : DAG.getUNDEF(XLenVT));
     }
     SDValue LHSIndices = DAG.getBuildVector(IndexVT, DL, GatherIndicesLHS);
-    LHSIndices = convertToScalableVector(IndexContainerVT, LHSIndices, DAG,
-                                         Subtarget);
-    SDValue Gather = DAG.getNode(GatherVVOpc, DL, ContainerVT, V1, LHSIndices,
-                                 DAG.getUNDEF(ContainerVT), TrueMask, VL);
+    LHSIndices =
+        convertToScalableVector(IndexContainerVT, LHSIndices, DAG, Subtarget);
+
+    SDValue Gather;
+    // If we have a locally repeating mask, then we can reuse the first register
+    // in the index register group for all registers within the source register
+    // group.  TODO: This generalizes to m2, and m4.
+    const MVT M1VT = getLMUL1VT(ContainerVT);
+    auto VLMAX = RISCVTargetLowering::computeVLMAXBounds(M1VT, Subtarget).first;
+    if (ContainerVT.bitsGT(M1VT) && isLocalRepeatingShuffle(Mask, VLMAX)) {
+      EVT SubIndexVT = M1VT.changeVectorElementType(IndexVT.getScalarType());
+      SDValue SubIndex =
+          DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, SubIndexVT, LHSIndices,
+                      DAG.getVectorIdxConstant(0, DL));
+      auto [InnerTrueMask, InnerVL] =
+          getDefaultScalableVLOps(M1VT, DL, DAG, Subtarget);
+      int N = ContainerVT.getVectorMinNumElements() /
+              M1VT.getVectorMinNumElements();
+      assert(isPowerOf2_32(N) && N <= 8);
+      Gather = DAG.getUNDEF(ContainerVT);
+      for (int i = 0; i < N; i++) {
+        SDValue SubIdx =
+            DAG.getVectorIdxConstant(M1VT.getVectorMinNumElements() * i, DL);
+        SDValue SubV1 =
+            DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, M1VT, V1, SubIdx);
+        SDValue SubVec =
+            DAG.getNode(GatherVVOpc, DL, M1VT, SubV1, SubIndex,
+                        DAG.getUNDEF(M1VT), InnerTrueMask, InnerVL);
+        Gather = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, ContainerVT, Gather,
+                             SubVec, SubIdx);
+      }
+    } else {
+      Gather = DAG.getNode(GatherVVOpc, DL, ContainerVT, V1, LHSIndices,
+                           DAG.getUNDEF(ContainerVT), TrueMask, VL);
+    }
     return convertFromScalableVector(VT, Gather, DAG, Subtarget);
   }
 
@@ -5708,8 +5790,8 @@ static SDValue lowerVECTOR_SHUFFLE(SDValue Op, SelectionDAG &DAG,
   // as a vselect + a single source vrgather.vv. Don't do this if we think the
   // operands may end up being lowered to something cheaper than a vrgather.vv.
   if (!DAG.isSplatValue(V2) && !DAG.isSplatValue(V1) &&
-      !ShuffleVectorSDNode::isSplatMask(ShuffleMaskLHS.data(), VT) &&
-      !ShuffleVectorSDNode::isSplatMask(ShuffleMaskRHS.data(), VT) &&
+      !ShuffleVectorSDNode::isSplatMask(ShuffleMaskLHS) &&
+      !ShuffleVectorSDNode::isSplatMask(ShuffleMaskRHS) &&
       !ShuffleVectorInst::isIdentityMask(ShuffleMaskLHS, NumElts) &&
       !ShuffleVectorInst::isIdentityMask(ShuffleMaskRHS, NumElts))
     if (SDValue V = lowerDisjointIndicesShuffle(SVN, DAG, Subtarget))
@@ -5752,9 +5834,10 @@ bool RISCVTargetLowering::isShuffleMaskLegal(ArrayRef<int> M, EVT VT) const {
     return false;
 
   // Support splats for any type. These should type legalize well.
-  if (ShuffleVectorSDNode::isSplatMask(M.data(), VT))
+  if (ShuffleVectorSDNode::isSplatMask(M))
     return true;
 
+  const unsigned NumElts = M.size();
   MVT SVT = VT.getSimpleVT();
 
   // Not for i1 vectors.
@@ -5762,7 +5845,8 @@ bool RISCVTargetLowering::isShuffleMaskLegal(ArrayRef<int> M, EVT VT) const {
     return false;
 
   int Dummy1, Dummy2;
-  return (isElementRotate(Dummy1, Dummy2, M) > 0) ||
+  return ShuffleVectorInst::isReverseMask(M, NumElts) ||
+         (isElementRotate(Dummy1, Dummy2, M) > 0) ||
          isInterleaveShuffle(M, SVT, Dummy1, Dummy2, Subtarget);
 }
 
@@ -8376,6 +8460,33 @@ SDValue RISCVTargetLowering::lowerSELECT(SDValue Op, SelectionDAG &DAG) const {
     if (isa<ConstantSDNode>(TrueV) && isa<ConstantSDNode>(FalseV)) {
       const APInt &TrueVal = TrueV->getAsAPIntVal();
       const APInt &FalseVal = FalseV->getAsAPIntVal();
+
+      // Prefer these over Zicond to avoid materializing an immediate:
+      //   (select (x < 0), y, z)  -> x >> (XLEN - 1) & (y - z) + z
+      //   (select (x > -1), z, y) -> x >> (XLEN - 1) & (y - z) + z
+      if (CondV.getOpcode() == ISD::SETCC &&
+          CondV.getOperand(0).getValueType() == VT && CondV.hasOneUse()) {
+        ISD::CondCode CCVal = cast<CondCodeSDNode>(CondV.getOperand(2))->get();
+        if ((CCVal == ISD::SETLT && isNullConstant(CondV.getOperand(1))) ||
+            (CCVal == ISD::SETGT && isAllOnesConstant(CondV.getOperand(1)))) {
+          int64_t TrueImm = TrueVal.getSExtValue();
+          int64_t FalseImm = FalseVal.getSExtValue();
+          if (CCVal == ISD::SETGT)
+            std::swap(TrueImm, FalseImm);
+          if (isInt<12>(TrueImm) && isInt<12>(FalseImm) &&
+              isInt<12>(TrueImm - FalseImm)) {
+            SDValue SRA =
+                DAG.getNode(ISD::SRA, DL, VT, CondV.getOperand(0),
+                            DAG.getConstant(Subtarget.getXLen() - 1, DL, VT));
+            SDValue AND =
+                DAG.getNode(ISD::AND, DL, VT, SRA,
+                            DAG.getSignedConstant(TrueImm - FalseImm, DL, VT));
+            return DAG.getNode(ISD::ADD, DL, VT, AND,
+                               DAG.getSignedConstant(FalseImm, DL, VT));
+          }
+        }
+      }
+
       const int TrueValCost = RISCVMatInt::getIntMatCost(
           TrueVal, Subtarget.getXLen(), Subtarget, /*CompressionCost=*/true);
       const int FalseValCost = RISCVMatInt::getIntMatCost(
@@ -8456,7 +8567,7 @@ SDValue RISCVTargetLowering::lowerSELECT(SDValue Op, SelectionDAG &DAG) const {
   SDValue RHS = CondV.getOperand(1);
   ISD::CondCode CCVal = cast<CondCodeSDNode>(CondV.getOperand(2))->get();
 
-  // Special case for a select of 2 constants that have a diffence of 1.
+  // Special case for a select of 2 constants that have a difference of 1.
   // Normally this is done by DAGCombine, but if the select is introduced by
   // type legalization or op legalization, we miss it. Restricting to SETLT
   // case for now because that is what signed saturating add/sub need.
@@ -9010,6 +9121,10 @@ RISCVTargetLowering::lowerVectorFPExtendOrRoundLike(SDValue Op,
 
   bool IsDirectConv = IsDirectExtend || IsDirectTrunc;
 
+  // We have regular SD node patterns for direct non-VL extends.
+  if (VT.isScalableVector() && IsDirectConv && !IsVP)
+    return Op;
+
   // Prepare any fixed-length vector operands.
   MVT ContainerVT = VT;
   SDValue Mask, VL;
@@ -9602,7 +9717,7 @@ static SDValue lowerVectorIntrinsicScalars(SDValue Op, SelectionDAG &DAG,
 // We need to convert from a scalable VF to a vsetvli with VLMax equal to
 // (vscale * VF). The vscale and VF are independent of element width. We use
 // SEW=8 for the vsetvli because it is the only element width that supports all
-// fractional LMULs. The LMUL is choosen so that with SEW=8 the VLMax is
+// fractional LMULs. The LMUL is chosen so that with SEW=8 the VLMax is
 // (vscale * VF). Where vscale is defined as VLEN/RVVBitsPerBlock. The
 // InsertVSETVLI pass can fix up the vtype of the vsetvli if a different
 // SEW and LMUL are better for the surrounding vector instructions.
@@ -10972,78 +11087,161 @@ SDValue RISCVTargetLowering::lowerVECTOR_DEINTERLEAVE(SDValue Op,
   SDLoc DL(Op);
   MVT VecVT = Op.getSimpleValueType();
 
-  assert(VecVT.isScalableVector() &&
-         "vector_interleave on non-scalable vector!");
+  const unsigned Factor = Op->getNumValues();
+  assert(Factor <= 8);
 
   // 1 bit element vectors need to be widened to e8
   if (VecVT.getVectorElementType() == MVT::i1)
     return widenVectorOpsToi8(Op, DL, DAG);
 
-  // If the VT is LMUL=8, we need to split and reassemble.
-  if (VecVT.getSizeInBits().getKnownMinValue() ==
+  // Convert to scalable vectors first.
+  if (VecVT.isFixedLengthVector()) {
+    MVT ContainerVT = getContainerForFixedLengthVector(VecVT);
+    SmallVector<SDValue, 8> Ops(Factor);
+    for (unsigned i = 0U; i < Factor; ++i)
+      Ops[i] = convertToScalableVector(ContainerVT, Op.getOperand(i), DAG,
+                                       Subtarget);
+
+    SmallVector<EVT, 8> VTs(Factor, ContainerVT);
+    SDValue NewDeinterleave =
+        DAG.getNode(ISD::VECTOR_DEINTERLEAVE, DL, VTs, Ops);
+
+    SmallVector<SDValue, 8> Res(Factor);
+    for (unsigned i = 0U; i < Factor; ++i)
+      Res[i] = convertFromScalableVector(VecVT, NewDeinterleave.getValue(i),
+                                         DAG, Subtarget);
+    return DAG.getMergeValues(Res, DL);
+  }
+
+  // If concatenating would exceed LMUL=8, we need to split.
+  if ((VecVT.getSizeInBits().getKnownMinValue() * Factor) >
       (8 * RISCV::RVVBitsPerBlock)) {
-    auto [Op0Lo, Op0Hi] = DAG.SplitVectorOperand(Op.getNode(), 0);
-    auto [Op1Lo, Op1Hi] = DAG.SplitVectorOperand(Op.getNode(), 1);
-    EVT SplitVT = Op0Lo.getValueType();
-
-    SDValue ResLo = DAG.getNode(ISD::VECTOR_DEINTERLEAVE, DL,
-                                DAG.getVTList(SplitVT, SplitVT), Op0Lo, Op0Hi);
-    SDValue ResHi = DAG.getNode(ISD::VECTOR_DEINTERLEAVE, DL,
-                                DAG.getVTList(SplitVT, SplitVT), Op1Lo, Op1Hi);
-
-    SDValue Even = DAG.getNode(ISD::CONCAT_VECTORS, DL, VecVT,
-                               ResLo.getValue(0), ResHi.getValue(0));
-    SDValue Odd = DAG.getNode(ISD::CONCAT_VECTORS, DL, VecVT, ResLo.getValue(1),
-                              ResHi.getValue(1));
-    return DAG.getMergeValues({Even, Odd}, DL);
+    SmallVector<SDValue, 8> Ops(Factor * 2);
+    for (unsigned i = 0; i != Factor; ++i) {
+      auto [OpLo, OpHi] = DAG.SplitVectorOperand(Op.getNode(), i);
+      Ops[i * 2] = OpLo;
+      Ops[i * 2 + 1] = OpHi;
+    }
+
+    SmallVector<EVT, 8> VTs(Factor, Ops[0].getValueType());
+
+    SDValue Lo = DAG.getNode(ISD::VECTOR_DEINTERLEAVE, DL, VTs,
+                             ArrayRef(Ops).slice(0, Factor));
+    SDValue Hi = DAG.getNode(ISD::VECTOR_DEINTERLEAVE, DL, VTs,
+                             ArrayRef(Ops).slice(Factor, Factor));
+
+    SmallVector<SDValue, 8> Res(Factor);
+    for (unsigned i = 0; i != Factor; ++i)
+      Res[i] = DAG.getNode(ISD::CONCAT_VECTORS, DL, VecVT, Lo.getValue(i),
+                           Hi.getValue(i));
+
+    return DAG.getMergeValues(Res, DL);
   }
 
-  // Concatenate the two vectors as one vector to deinterleave
+  SmallVector<SDValue, 8> Ops(Op->op_values());
+
+  // Concatenate the vectors as one vector to deinterleave
   MVT ConcatVT =
       MVT::getVectorVT(VecVT.getVectorElementType(),
-                       VecVT.getVectorElementCount().multiplyCoefficientBy(2));
-  SDValue Concat = DAG.getNode(ISD::CONCAT_VECTORS, DL, ConcatVT,
-                               Op.getOperand(0), Op.getOperand(1));
+                       VecVT.getVectorElementCount().multiplyCoefficientBy(
+                           PowerOf2Ceil(Factor)));
+  if (Ops.size() < PowerOf2Ceil(Factor))
+    Ops.append(PowerOf2Ceil(Factor) - Factor, DAG.getUNDEF(VecVT));
+  SDValue Concat = DAG.getNode(ISD::CONCAT_VECTORS, DL, ConcatVT, Ops);
+
+  if (Factor == 2) {
+    // We can deinterleave through vnsrl.wi if the element type is smaller than
+    // ELEN
+    if (VecVT.getScalarSizeInBits() < Subtarget.getELen()) {
+      SDValue Even = getDeinterleaveShiftAndTrunc(DL, VecVT, Concat, 2, 0, DAG);
+      SDValue Odd = getDeinterleaveShiftAndTrunc(DL, VecVT, Concat, 2, 1, DAG);
+      return DAG.getMergeValues({Even, Odd}, DL);
+    }
+
+    // For the indices, use the vmv.v.x of an i8 constant to fill the largest
+    // possibly mask vector, then extract the required subvector.  Doing this
+    // (instead of a vid, vmsne sequence) reduces LMUL, and allows the mask
+    // creation to be rematerialized during register allocation to reduce
+    // register pressure if needed.
+
+    MVT MaskVT = ConcatVT.changeVectorElementType(MVT::i1);
+
+    SDValue EvenSplat = DAG.getConstant(0b01010101, DL, MVT::nxv8i8);
+    EvenSplat = DAG.getBitcast(MVT::nxv64i1, EvenSplat);
+    SDValue EvenMask = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, MaskVT,
+                                   EvenSplat, DAG.getVectorIdxConstant(0, DL));
+
+    SDValue OddSplat = DAG.getConstant(0b10101010, DL, MVT::nxv8i8);
+    OddSplat = DAG.getBitcast(MVT::nxv64i1, OddSplat);
+    SDValue OddMask = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, MaskVT, OddSplat,
+                                  DAG.getVectorIdxConstant(0, DL));
+
+    // vcompress the even and odd elements into two separate vectors
+    SDValue EvenWide = DAG.getNode(ISD::VECTOR_COMPRESS, DL, ConcatVT, Concat,
+                                   EvenMask, DAG.getUNDEF(ConcatVT));
+    SDValue OddWide = DAG.getNode(ISD::VECTOR_COMPRESS, DL, ConcatVT, Concat,
+                                  OddMask, DAG.getUNDEF(ConcatVT));
+
+    // Extract the result half of the gather for even and odd
+    SDValue Even = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VecVT, EvenWide,
+                               DAG.getVectorIdxConstant(0, DL));
+    SDValue Odd = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VecVT, OddWide,
+                              DAG.getVectorIdxConstant(0, DL));
 
-  // We can deinterleave through vnsrl.wi if the element type is smaller than
-  // ELEN
-  if (VecVT.getScalarSizeInBits() < Subtarget.getELen()) {
-    SDValue Even = getDeinterleaveShiftAndTrunc(DL, VecVT, Concat, 2, 0, DAG);
-    SDValue Odd = getDeinterleaveShiftAndTrunc(DL, VecVT, Concat, 2, 1, DAG);
     return DAG.getMergeValues({Even, Odd}, DL);
   }
 
-  // For the indices, use the vmv.v.x of an i8 constant to fill the largest
-  // possibly mask vector, then extract the required subvector.  Doing this
-  // (instead of a vid, vmsne sequence) reduces LMUL, and allows the mask
-  // creation to be rematerialized during register allocation to reduce
-  // register pressure if needed.
+  // Store with unit-stride store and load it back with segmented load.
+  MVT XLenVT = Subtarget.getXLenVT();
+  SDValue VL = getDefaultScalableVLOps(ConcatVT, DL, DAG, Subtarget).second;
+  SDValue Passthru = DAG.getUNDEF(ConcatVT);
 
-  MVT MaskVT = ConcatVT.changeVectorElementType(MVT::i1);
+  // Allocate a stack slot.
+  Align Alignment = DAG.getReducedAlign(VecVT, /*UseABI=*/false);
+  SDValue StackPtr =
+      DAG.CreateStackTemporary(ConcatVT.getStoreSize(), Alignment);
+  auto &MF = DAG.getMachineFunction();
+  auto FrameIndex = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
+  auto PtrInfo = MachinePointerInfo::getFixedStack(MF, FrameIndex);
 
-  SDValue EvenSplat = DAG.getConstant(0b01010101, DL, MVT::nxv8i8);
-  EvenSplat = DAG.getBitcast(MVT::nxv64i1, EvenSplat);
-  SDValue EvenMask = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, MaskVT, EvenSplat,
-                                 DAG.getVectorIdxConstant(0, DL));
+  SDValue StoreOps[] = {DAG.getEntryNode(),
+                        DAG.getTargetConstant(Intrinsic::riscv_vse, DL, XLenVT),
+                        Concat, StackPtr, VL};
 
-  SDValue OddSplat = DAG.getConstant(0b10101010, DL, MVT::nxv8i8);
-  OddSplat = DAG.getBitcast(MVT::nxv64i1, OddSplat);
-  SDValue OddMask = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, MaskVT, OddSplat,
-                                DAG.getVectorIdxConstant(0, DL));
+  SDValue Chain = DAG.getMemIntrinsicNode(
+      ISD::INTRINSIC_VOID, DL, DAG.getVTList(MVT::Other), StoreOps,
+      ConcatVT.getVectorElementType(), PtrInfo, Alignment,
+      MachineMemOperand::MOStore, MemoryLocation::UnknownSize);
 
-  // vcompress the even and odd elements into two separate vectors
-  SDValue EvenWide = DAG.getNode(ISD::VECTOR_COMPRESS, DL, ConcatVT, Concat,
-                                 EvenMask, DAG.getUNDEF(ConcatVT));
-  SDValue OddWide = DAG.getNode(ISD::VECTOR_COMPRESS, DL, ConcatVT, Concat,
-                                OddMask, DAG.getUNDEF(ConcatVT));
+  static const Intrinsic::ID VlsegIntrinsicsIds[] = {
+      Intrinsic::riscv_vlseg2, Intrinsic::riscv_vlseg3, Intrinsic::riscv_vlseg4,
+      Intrinsic::riscv_vlseg5, Intrinsic::riscv_vlseg6, Intrinsic::riscv_vlseg7,
+      Intrinsic::riscv_vlseg8};
 
-  // Extract the result half of the gather for even and odd
-  SDValue Even = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VecVT, EvenWide,
-                             DAG.getVectorIdxConstant(0, DL));
-  SDValue Odd = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VecVT, OddWide,
-                            DAG.getVectorIdxConstant(0, DL));
+  SDValue LoadOps[] = {
+      Chain,
+      DAG.getTargetConstant(VlsegIntrinsicsIds[Factor - 2], DL, XLenVT),
+      Passthru,
+      StackPtr,
+      VL,
+      DAG.getTargetConstant(Log2_64(VecVT.getScalarSizeInBits()), DL, XLenVT)};
+
+  unsigned Sz =
+      Factor * VecVT.getVectorMinNumElements() * VecVT.getScalarSizeInBits();
+  EVT VecTupTy = MVT::getRISCVVectorTupleVT(Sz, Factor);
+
+  SDValue Load = DAG.getMemIntrinsicNode(
+      ISD::INTRINSIC_W_CHAIN, DL, DAG.getVTList({VecTupTy, MVT::Other}),
+      LoadOps, ConcatVT.getVectorElementType(), PtrInfo, Alignment,
+      MachineMemOperand::MOLoad, MemoryLocation::UnknownSize);
+
+  SmallVector<SDValue, 8> Res(Factor);
 
-  return DAG.getMergeValues({Even, Odd}, DL);
+  for (unsigned i = 0U; i < Factor; ++i)
+    Res[i] = DAG.getNode(RISCVISD::TUPLE_EXTRACT, DL, VecVT, Load,
+                         DAG.getVectorIdxConstant(i, DL));
+
+  return DAG.getMergeValues(Res, DL);
 }
 
 SDValue RISCVTargetLowering::lowerVECTOR_INTERLEAVE(SDValue Op,
@@ -11051,36 +11249,125 @@ SDValue RISCVTargetLowering::lowerVECTOR_INTERLEAVE(SDValue Op,
   SDLoc DL(Op);
   MVT VecVT = Op.getSimpleValueType();
 
-  assert(VecVT.isScalableVector() &&
-         "vector_interleave on non-scalable vector!");
+  const unsigned Factor = Op.getNumOperands();
+  assert(Factor <= 8);
 
   // i1 vectors need to be widened to i8
   if (VecVT.getVectorElementType() == MVT::i1)
     return widenVectorOpsToi8(Op, DL, DAG);
 
+  // Convert to scalable vectors first.
+  if (VecVT.isFixedLengthVector()) {
+    MVT ContainerVT = getContainerForFixedLengthVector(VecVT);
+    SmallVector<SDValue, 8> Ops(Factor);
+    for (unsigned i = 0U; i < Factor; ++i)
+      Ops[i] = convertToScalableVector(ContainerVT, Op.getOperand(i), DAG,
+                                       Subtarget);
+
+    SmallVector<EVT, 8> VTs(Factor, ContainerVT);
+    SDValue NewInterleave = DAG.getNode(ISD::VECTOR_INTERLEAVE, DL, VTs, Ops);
+
+    SmallVector<SDValue, 8> Res(Factor);
+    for (unsigned i = 0U; i < Factor; ++i)
+      Res[i] = convertFromScalableVector(VecVT, NewInterleave.getValue(i), DAG,
+                                         Subtarget);
+    return DAG.getMergeValues(Res, DL);
+  }
+
   MVT XLenVT = Subtarget.getXLenVT();
   SDValue VL = DAG.getRegister(RISCV::X0, XLenVT);
 
-  // If the VT is LMUL=8, we need to split and reassemble.
-  if (VecVT.getSizeInBits().getKnownMinValue() == (8 * RISCV::RVVBitsPerBlock)) {
-    auto [Op0Lo, Op0Hi] = DAG.SplitVectorOperand(Op.getNode(), 0);
-    auto [Op1Lo, Op1Hi] = DAG.SplitVectorOperand(Op.getNode(), 1);
-    EVT SplitVT = Op0Lo.getValueType();
+  // If the VT is larger than LMUL=8, we need to split and reassemble.
+  if ((VecVT.getSizeInBits().getKnownMinValue() * Factor) >
+      (8 * RISCV::RVVBitsPerBlock)) {
+    SmallVector<SDValue, 8> Ops(Factor * 2);
+    for (unsigned i = 0; i != Factor; ++i) {
+      auto [OpLo, OpHi] = DAG.SplitVectorOperand(Op.getNode(), i);
+      Ops[i] = OpLo;
+      Ops[i + Factor] = OpHi;
+    }
+
+    SmallVector<EVT, 8> VTs(Factor, Ops[0].getValueType());
+
+    SDValue Res[] = {DAG.getNode(ISD::VECTOR_INTERLEAVE, DL, VTs,
+                                 ArrayRef(Ops).take_front(Factor)),
+                     DAG.getNode(ISD::VECTOR_INTERLEAVE, DL, VTs,
+                                 ArrayRef(Ops).drop_front(Factor))};
 
-    SDValue ResLo = DAG.getNode(ISD::VECTOR_INTERLEAVE, DL,
-                                DAG.getVTList(SplitVT, SplitVT), Op0Lo, Op1Lo);
-    SDValue ResHi = DAG.getNode(ISD::VECTOR_INTERLEAVE, DL,
-                                DAG.getVTList(SplitVT, SplitVT), Op0Hi, Op1Hi);
+    SmallVector<SDValue, 8> Concats(Factor);
+    for (unsigned i = 0; i != Factor; ++i) {
+      unsigned IdxLo = 2 * i;
+      unsigned IdxHi = 2 * i + 1;
+      Concats[i] = DAG.getNode(ISD::CONCAT_VECTORS, DL, VecVT,
+                               Res[IdxLo / Factor].getValue(IdxLo % Factor),
+                               Res[IdxHi / Factor].getValue(IdxHi % Factor));
+    }
 
-    SDValue Lo = DAG.getNode(ISD::CONCAT_VECTORS, DL, VecVT,
-                             ResLo.getValue(0), ResLo.getValue(1));
-    SDValue Hi = DAG.getNode(ISD::CONCAT_VECTORS, DL, VecVT,
-                             ResHi.getValue(0), ResHi.getValue(1));
-    return DAG.getMergeValues({Lo, Hi}, DL);
+    return DAG.getMergeValues(Concats, DL);
   }
 
   SDValue Interleaved;
 
+  // Spill to the stack using a segment store for simplicity.
+  if (Factor != 2) {
+    EVT MemVT =
+        EVT::getVectorVT(*DAG.getContext(), VecVT.getVectorElementType(),
+                         VecVT.getVectorElementCount() * Factor);
+
+    // Allocate a stack slot.
+    Align Alignment = DAG.getReducedAlign(VecVT, /*UseABI=*/false);
+    SDValue StackPtr =
+        DAG.CreateStackTemporary(MemVT.getStoreSize(), Alignment);
+    EVT PtrVT = StackPtr.getValueType();
+    auto &MF = DAG.getMachineFunction();
+    auto FrameIndex = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
+    auto PtrInfo = MachinePointerInfo::getFixedStack(MF, FrameIndex);
+
+    static const Intrinsic::ID IntrIds[] = {
+        Intrinsic::riscv_vsseg2, Intrinsic::riscv_vsseg3,
+        Intrinsic::riscv_vsseg4, Intrinsic::riscv_vsseg5,
+        Intrinsic::riscv_vsseg6, Intrinsic::riscv_vsseg7,
+        Intrinsic::riscv_vsseg8,
+    };
+
+    unsigned Sz =
+        Factor * VecVT.getVectorMinNumElements() * VecVT.getScalarSizeInBits();
+    EVT VecTupTy = MVT::getRISCVVectorTupleVT(Sz, Factor);
+
+    SDValue StoredVal = DAG.getUNDEF(VecTupTy);
+    for (unsigned i = 0; i < Factor; i++)
+      StoredVal = DAG.getNode(RISCVISD::TUPLE_INSERT, DL, VecTupTy, StoredVal,
+                              Op.getOperand(i), DAG.getConstant(i, DL, XLenVT));
+
+    SDValue Ops[] = {DAG.getEntryNode(),
+                     DAG.getTargetConstant(IntrIds[Factor - 2], DL, XLenVT),
+                     StoredVal,
+                     StackPtr,
+                     VL,
+                     DAG.getTargetConstant(Log2_64(VecVT.getScalarSizeInBits()),
+                                           DL, XLenVT)};
+
+    SDValue Chain = DAG.getMemIntrinsicNode(
+        ISD::INTRINSIC_VOID, DL, DAG.getVTList(MVT::Other), Ops,
+        VecVT.getVectorElementType(), PtrInfo, Alignment,
+        MachineMemOperand::MOStore, MemoryLocation::UnknownSize);
+
+    SmallVector<SDValue, 8> Loads(Factor);
+
+    SDValue Increment =
+        DAG.getVScale(DL, PtrVT,
+                      APInt(PtrVT.getFixedSizeInBits(),
+                            VecVT.getStoreSize().getKnownMinValue()));
+    for (unsigned i = 0; i != Factor; ++i) {
+      if (i != 0)
+        StackPtr = DAG.getNode(ISD::ADD, DL, PtrVT, StackPtr, Increment);
+
+      Loads[i] = DAG.getLoad(VecVT, DL, Chain, StackPtr, PtrInfo);
+    }
+
+    return DAG.getMergeValues(Loads, DL);
+  }
+
   // If the element type is smaller than ELEN, then we can interleave with
   // vwaddu.vv and vwmaccu.vx
   if (VecVT.getScalarSizeInBits() < Subtarget.getELen()) {
@@ -12916,7 +13203,7 @@ void RISCVTargetLowering::ReplaceNodeResults(SDNode *N,
         return;
       if (IsStrict) {
         SDValue Chain = N->getOperand(0);
-        // In absense of Zfh, promote f16 to f32, then convert.
+        // In absence of Zfh, promote f16 to f32, then convert.
         if (Op0.getValueType() == MVT::f16 &&
             !Subtarget.hasStdExtZfhOrZhinx()) {
           Op0 = DAG.getNode(ISD::STRICT_FP_EXTEND, DL, {MVT::f32, MVT::Other},
@@ -12933,7 +13220,7 @@ void RISCVTargetLowering::ReplaceNodeResults(SDNode *N,
         Results.push_back(Res.getValue(1));
         return;
       }
-      // For bf16, or f16 in absense of Zfh, promote [b]f16 to f32 and then
+      // For bf16, or f16 in absence of Zfh, promote [b]f16 to f32 and then
       // convert.
       if ((Op0.getValueType() == MVT::f16 &&
            !Subtarget.hasStdExtZfhOrZhinx()) ||
@@ -12976,7 +13263,7 @@ void RISCVTargetLowering::ReplaceNodeResults(SDNode *N,
       if (!isTypeLegal(Op0VT))
         return;
 
-      // In absense of Zfh, promote f16 to f32, then convert.
+      // In absence of Zfh, promote f16 to f32, then convert.
       if (Op0.getValueType() == MVT::f16 && !Subtarget.hasStdExtZfhOrZhinx())
         Op0 = DAG.getNode(ISD::FP_EXTEND, DL, MVT::f32, Op0);
 
@@ -13603,7 +13890,7 @@ void RISCVTargetLowering::ReplaceNodeResults(SDNode *N,
 static unsigned getVecReduceOpcode(unsigned Opc) {
   switch (Opc) {
   default:
-    llvm_unreachable("Unhandled binary to transfrom reduction");
+    llvm_unreachable("Unhandled binary to transform reduction");
   case ISD::ADD:
     return ISD::VECREDUCE_ADD;
   case ISD::UMAX:
@@ -13733,7 +14020,7 @@ static SDValue combineBinOpToReduce(SDNode *N, SelectionDAG &DAG,
   auto BinOpToRVVReduce = [](unsigned Opc) {
     switch (Opc) {
     default:
-      llvm_unreachable("Unhandled binary to transfrom reduction");
+      llvm_unreachable("Unhandled binary to transform reduction");
     case ISD::ADD:
       return RISCVISD::VECREDUCE_ADD_VL;
     case ISD::UMAX:
@@ -15290,7 +15577,7 @@ struct NodeExtensionHelper {
 
   bool isSupportedFPExtend(SDNode *Root, MVT NarrowEltVT,
                            const RISCVSubtarget &Subtarget) {
-    // Any f16 extension will neeed zvfh
+    // Any f16 extension will need zvfh
     if (NarrowEltVT == MVT::f16 && !Subtarget.hasVInstructionsF16())
       return false;
     // The only bf16 extension we can do is vfmadd_vl -> vfwmadd_vl with
@@ -16039,7 +16326,7 @@ static SDValue performMemPairCombine(SDNode *N,
       if (Base1 != Base2)
         continue;
 
-      // Check if the offsets match the XTHeadMemPair encoding contraints.
+      // Check if the offsets match the XTHeadMemPair encoding constraints.
       bool Valid = false;
       if (MemVT == MVT::i32) {
         // Check for adjacent i32 values and a 2-bit index.
@@ -16295,7 +16582,7 @@ static SDValue performVP_REVERSECombine(SDNode *N, SelectionDAG &DAG,
   SDValue Temp2 = DAG.getNode(ISD::MUL, DL, XLenVT, Temp1,
                               DAG.getConstant(ElemWidthByte, DL, XLenVT));
   SDValue Base = DAG.getNode(ISD::ADD, DL, XLenVT, VPLoad->getBasePtr(), Temp2);
-  SDValue Stride = DAG.getConstant(-ElemWidthByte, DL, XLenVT);
+  SDValue Stride = DAG.getSignedConstant(-ElemWidthByte, DL, XLenVT);
 
   MachineFunction &MF = DAG.getMachineFunction();
   MachinePointerInfo PtrInfo(VPLoad->getAddressSpace());
@@ -16356,7 +16643,7 @@ static SDValue performVP_STORECombine(SDNode *N, SelectionDAG &DAG,
                               DAG.getConstant(ElemWidthByte, DL, XLenVT));
   SDValue Base =
       DAG.getNode(ISD::ADD, DL, XLenVT, VPStore->getBasePtr(), Temp2);
-  SDValue Stride = DAG.getConstant(-ElemWidthByte, DL, XLenVT);
+  SDValue Stride = DAG.getSignedConstant(-ElemWidthByte, DL, XLenVT);
 
   MachineFunction &MF = DAG.getMachineFunction();
   MachinePointerInfo PtrInfo(VPStore->getAddressSpace());
@@ -16371,6 +16658,93 @@ static SDValue performVP_STORECombine(SDNode *N, SelectionDAG &DAG,
       VPStore->isTruncatingStore(), VPStore->isCompressingStore());
 }
 
+// Peephole avgceil pattern.
+//   %1 = zext <N x i8> %a to <N x i32>
+//   %2 = zext <N x i8> %b to <N x i32>
+//   %3 = add nuw nsw <N x i32> %1, splat (i32 1)
+//   %4 = add nuw nsw <N x i32> %3, %2
+//   %5 = lshr <N x i32> %4, splat (i32 1)
+//   %6 = trunc <N x i32> %5 to <N x i8>
+static SDValue performVP_TRUNCATECombine(SDNode *N, SelectionDAG &DAG,
+                                         const RISCVSubtarget &Subtarget) {
+  EVT VT = N->getValueType(0);
+
+  // Ignore fixed vectors.
+  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
+  if (!VT.isScalableVector() || !TLI.isTypeLegal(VT))
+    return SDValue();
+
+  SDValue In = N->getOperand(0);
+  SDValue Mask = N->getOperand(1);
+  SDValue VL = N->getOperand(2);
+
+  // Input should be a vp_srl with same mask and VL.
+  if (In.getOpcode() != ISD::VP_SRL || In.getOperand(2) != Mask ||
+      In.getOperand(3) != VL)
+    return SDValue();
+
+  // Shift amount should be 1.
+  if (!isOneOrOneSplat(In.getOperand(1)))
+    return SDValue();
+
+  // Shifted value should be a vp_add with same mask and VL.
+  SDValue LHS = In.getOperand(0);
+  if (LHS.getOpcode() != ISD::VP_ADD || LHS.getOperand(2) != Mask ||
+      LHS.getOperand(3) != VL)
+    return SDValue();
+
+  SDValue Operands[3];
+
+  // Matches another VP_ADD with same VL and Mask.
+  auto FindAdd = [&](SDValue V, SDValue Other) {
+    if (V.getOpcode() != ISD::VP_ADD || V.getOperand(2) != Mask ||
+        V.getOperand(3) != VL)
+      return false;
+
+    Operands[0] = Other;
+    Operands[1] = V.getOperand(1);
+    Operands[2] = V.getOperand(0);
+    return true;
+  };
+
+  // We need to find another VP_ADD in one of the operands.
+  SDValue LHS0 = LHS.getOperand(0);
+  SDValue LHS1 = LHS.getOperand(1);
+  if (!FindAdd(LHS0, LHS1) && !FindAdd(LHS1, LHS0))
+    return SDValue();
+
+  // Now we have three operands of two additions. Check that one of them is a
+  // constant vector with ones.
+  auto I = llvm::find_if(Operands,
+                         [](const SDValue &Op) { return isOneOrOneSplat(Op); });
+  if (I == std::end(Operands))
+    return SDValue();
+  // We found a vector with ones, move if it to the end of the Operands array.
+  std::swap(*I, Operands[2]);
+
+  // Make sure the other 2 operands can be promoted from the result type.
+  for (SDValue Op : drop_end(Operands)) {
+    if (Op.getOpcode() != ISD::VP_ZERO_EXTEND || Op.getOperand(1) != Mask ||
+        Op.getOperand(2) != VL)
+      return SDValue();
+    // Input must be the same size or smaller than our result.
+    if (Op.getOperand(0).getScalarValueSizeInBits() > VT.getScalarSizeInBits())
+      return SDValue();
+  }
+
+  // Pattern is detected.
+  // Rebuild the zero extends in case the inputs are smaller than our result.
+  SDValue NewOp0 = DAG.getNode(ISD::VP_ZERO_EXTEND, SDLoc(Operands[0]), VT,
+                               Operands[0].getOperand(0), Mask, VL);
+  SDValue NewOp1 = DAG.getNode(ISD::VP_ZERO_EXTEND, SDLoc(Operands[1]), VT,
+                               Operands[1].getOperand(0), Mask, VL);
+  // Build a AVGCEILU_VL which will be selected as a VAADDU with RNU rounding
+  // mode.
+  SDLoc DL(N);
+  return DAG.getNode(RISCVISD::AVGCEILU_VL, DL, VT,
+                     {NewOp0, NewOp1, DAG.getUNDEF(VT), Mask, VL});
+}
+
 // Convert from one FMA opcode to another based on whether we are negating the
 // multiply result and/or the accumulator.
 // NOTE: Only supports RVV operations with VL.
@@ -16580,7 +16954,7 @@ static SDValue performSRACombine(SDNode *N, SelectionDAG &DAG,
 }
 
 // Invert (and/or (set cc X, Y), (xor Z, 1)) to (or/and (set !cc X, Y)), Z) if
-// the result is used as the conditon of a br_cc or select_cc we can invert,
+// the result is used as the condition of a br_cc or select_cc we can invert,
 // inverting the setcc is free, and Z is 0/1. Caller will invert the
 // br_cc/select_cc.
 static SDValue tryDemorganOfBooleanCondition(SDValue Cond, SelectionDAG &DAG) {
@@ -16641,7 +17015,7 @@ static SDValue tryDemorganOfBooleanCondition(SDValue Cond, SelectionDAG &DAG) {
   return DAG.getNode(Opc, SDLoc(Cond), VT, Setcc, Xor.getOperand(0));
 }
 
-// Perform common combines for BR_CC and SELECT_CC condtions.
+// Perform common combines for BR_CC and SELECT_CC conditions.
 static bool combine_CC(SDValue &LHS, SDValue &RHS, SDValue &CC, const SDLoc &DL,
                        SelectionDAG &DAG, const RISCVSubtarget &Subtarget) {
   ISD::CondCode CCVal = cast<CondCodeSDNode>(CC)->get();
@@ -17671,6 +18045,83 @@ static SDValue combineScalarCTPOPToVCPOP(SDNode *N, SelectionDAG &DAG,
   return DAG.getZExtOrTrunc(Pop, DL, VT);
 }
 
+static SDValue performSHLCombine(SDNode *N,
+                                 TargetLowering::DAGCombinerInfo &DCI,
+                                 const RISCVSubtarget &Subtarget) {
+  // (shl (zext x), y) -> (vwsll   x, y)
+  if (SDValue V = combineOp_VLToVWOp_VL(N, DCI, Subtarget))
+    return V;
+
+  // (shl (sext x), C) -> (vwmulsu x, 1u << C)
+  // (shl (zext x), C) -> (vwmulu  x, 1u << C)
+
+  if (!DCI.isAfterLegalizeDAG())
+    return SDValue();
+
+  SDValue LHS = N->getOperand(0);
+  if (!LHS.hasOneUse())
+    return SDValue();
+  unsigned Opcode;
+  switch (LHS.getOpcode()) {
+  case ISD::SIGN_EXTEND:
+  case RISCVISD::VSEXT_VL:
+    Opcode = RISCVISD::VWMULSU_VL;
+    break;
+  case ISD::ZERO_EXTEND:
+  case RISCVISD::VZEXT_VL:
+    Opcode = RISCVISD::VWMULU_VL;
+    break;
+  default:
+    return SDValue();
+  }
+
+  SDValue RHS = N->getOperand(1);
+  APInt ShAmt;
+  uint64_t ShAmtInt;
+  if (ISD::isConstantSplatVector(RHS.getNode(), ShAmt))
+    ShAmtInt = ShAmt.getZExtValue();
+  else if (RHS.getOpcode() == RISCVISD::VMV_V_X_VL &&
+           RHS.getOperand(1).getOpcode() == ISD::Constant)
+    ShAmtInt = RHS.getConstantOperandVal(1);
+  else
+    return SDValue();
+
+  // Better foldings:
+  // (shl (sext x), 1) -> (vwadd  x, x)
+  // (shl (zext x), 1) -> (vwaddu x, x)
+  if (ShAmtInt <= 1)
+    return SDValue();
+
+  SDValue NarrowOp = LHS.getOperand(0);
+  MVT NarrowVT = NarrowOp.getSimpleValueType();
+  uint64_t NarrowBits = NarrowVT.getScalarSizeInBits();
+  if (ShAmtInt >= NarrowBits)
+    return SDValue();
+  MVT VT = N->getSimpleValueType(0);
+  if (NarrowBits * 2 != VT.getScalarSizeInBits())
+    return SDValue();
+
+  SelectionDAG &DAG = DCI.DAG;
+  SDLoc DL(N);
+  SDValue Passthru, Mask, VL;
+  switch (N->getOpcode()) {
+  case ISD::SHL:
+    Passthru = DAG.getUNDEF(VT);
+    std::tie(Mask, VL) = getDefaultScalableVLOps(VT, DL, DAG, Subtarget);
+    break;
+  case RISCVISD::SHL_VL:
+    Passthru = N->getOperand(2);
+    Mask = N->getOperand(3);
+    VL = N->getOperand(4);
+    break;
+  default:
+    llvm_unreachable("Expected SHL");
+  }
+  return DAG.getNode(Opcode, DL, VT, NarrowOp,
+                     DAG.getConstant(1ULL << ShAmtInt, SDLoc(RHS), NarrowVT),
+                     Passthru, Mask, VL);
+}
+
 SDValue RISCVTargetLowering::PerformDAGCombine(SDNode *N,
                                                DAGCombinerInfo &DCI) const {
   SelectionDAG &DAG = DCI.DAG;
@@ -17928,6 +18379,8 @@ SDValue RISCVTargetLowering::PerformDAGCombine(SDNode *N,
     if (SDValue V = combineTruncOfSraSext(N, DAG))
       return V;
     return combineTruncToVnclip(N, DAG, Subtarget);
+  case ISD::VP_TRUNCATE:
+    return performVP_TRUNCATECombine(N, DAG, Subtarget);
   case ISD::TRUNCATE:
     return performTRUNCATECombine(N, DAG, Subtarget);
   case ISD::SELECT:
@@ -18150,7 +18603,7 @@ SDValue RISCVTargetLowering::PerformDAGCombine(SDNode *N,
         const int64_t Addend = SimpleVID->Addend;
 
         // Note: We don't need to check alignment here since (by assumption
-        // from the existance of the gather), our offsets must be sufficiently
+        // from the existence of the gather), our offsets must be sufficiently
         // aligned.
 
         const EVT PtrVT = getPointerTy(DAG.getDataLayout());
@@ -18302,7 +18755,7 @@ SDValue RISCVTargetLowering::PerformDAGCombine(SDNode *N,
     break;
   }
   case RISCVISD::SHL_VL:
-    if (SDValue V = combineOp_VLToVWOp_VL(N, DCI, Subtarget))
+    if (SDValue V = performSHLCombine(N, DCI, Subtarget))
       return V;
     [[fallthrough]];
   case RISCVISD::SRA_VL:
@@ -18327,7 +18780,7 @@ SDValue RISCVTargetLowering::PerformDAGCombine(SDNode *N,
   case ISD::SRL:
   case ISD::SHL: {
     if (N->getOpcode() == ISD::SHL) {
-      if (SDValue V = combineOp_VLToVWOp_VL(N, DCI, Subtarget))
+      if (SDValue V = performSHLCombine(N, DCI, Subtarget))
         return V;
     }
     SDValue ShAmt = N->getOperand(1);
@@ -20186,7 +20639,7 @@ SDValue RISCVTargetLowering::LowerFormalArguments(
   EVT PtrVT = getPointerTy(DAG.getDataLayout());
   MVT XLenVT = Subtarget.getXLenVT();
   unsigned XLenInBytes = Subtarget.getXLen() / 8;
-  // Used with vargs to acumulate store chains.
+  // Used with vargs to accumulate store chains.
   std::vector<SDValue> OutChains;
 
   // Assign locations to all of the incoming arguments.
@@ -22606,6 +23059,231 @@ bool RISCVTargetLowering::lowerInterleaveIntrinsicToStore(
   return true;
 }
 
+static bool isMultipleOfN(const Value *V, const DataLayout &DL, unsigned N) {
+  assert(N);
+  if (N == 1)
+    return true;
+
+  if (isPowerOf2_32(N)) {
+    KnownBits KB = llvm::computeKnownBits(V, DL);
+    return KB.countMinTrailingZeros() >= Log2_32(N);
+  }
+
+  using namespace PatternMatch;
+  // Right now we're only recognizing the simplest pattern.
+  uint64_t C;
+  return match(V, m_c_Mul(m_Value(), m_ConstantInt(C))) && C && C % N == 0;
+}
+
+/// Lower an interleaved vp.load into a vlsegN intrinsic.
+///
+/// E.g. Lower an interleaved vp.load (Factor = 2):
+///   %l = call <vscale x 64 x i8> @llvm.vp.load.nxv64i8.p0(ptr %ptr,
+///                                                         %mask,
+///                                                         i32 %wide.rvl)
+///   %dl = tail call { <vscale x 32 x i8>, <vscale x 32 x i8> }
+///             @llvm.vector.deinterleave2.nxv64i8(
+///               <vscale x 64 x i8> %l)
+///   %r0 = extractvalue { <vscale x 32 x i8>, <vscale x 32 x i8> } %dl, 0
+///   %r1 = extractvalue { <vscale x 32 x i8>, <vscale x 32 x i8> } %dl, 1
+///
+/// Into:
+///   %rvl = udiv %wide.rvl, 2
+///   %sl = call { <vscale x 32 x i8>, <vscale x 32 x i8> }
+///             @llvm.riscv.vlseg2.mask.nxv32i8.i64(<vscale x 32 x i8> undef,
+///                                                 <vscale x 32 x i8> undef,
+///                                                 ptr %ptr,
+///                                                 %mask,
+///                                                 i64 %rvl,
+///                                                 i64 1)
+///   %r0 = extractvalue { <vscale x 32 x i8>, <vscale x 32 x i8> } %sl, 0
+///   %r1 = extractvalue { <vscale x 32 x i8>, <vscale x 32 x i8> } %sl, 1
+///
+/// NOTE: the deinterleave2 intrinsic won't be touched and is expected to be
+/// removed by the caller
+/// TODO: We probably can loosen the dependency on matching extractvalue when
+/// dealing with factor of 2 (extractvalue is still required for most of other
+/// factors though).
+bool RISCVTargetLowering::lowerDeinterleavedIntrinsicToVPLoad(
+    VPIntrinsic *Load, Value *Mask,
+    ArrayRef<Value *> DeinterleaveResults) const {
+  assert(Mask && "Expect a valid mask");
+  assert(Load->getIntrinsicID() == Intrinsic::vp_load &&
+         "Unexpected intrinsic");
+
+  const unsigned Factor = DeinterleaveResults.size();
+
+  auto *WideVTy = dyn_cast<ScalableVectorType>(Load->getType());
+  // TODO: Support fixed vectors.
+  if (!WideVTy)
+    return false;
+
+  unsigned WideNumElements = WideVTy->getElementCount().getKnownMinValue();
+  assert(WideNumElements % Factor == 0 &&
+         "ElementCount of a wide load must be divisible by interleave factor");
+  auto *VTy =
+      VectorType::get(WideVTy->getScalarType(), WideNumElements / Factor,
+                      WideVTy->isScalableTy());
+  auto &DL = Load->getModule()->getDataLayout();
+  Align Alignment = Load->getParamAlign(0).value_or(
+      DL.getABITypeAlign(WideVTy->getElementType()));
+  if (!isLegalInterleavedAccessType(
+          VTy, Factor, Alignment,
+          Load->getArgOperand(0)->getType()->getPointerAddressSpace(), DL))
+    return false;
+
+  IRBuilder<> Builder(Load);
+  Value *WideEVL = Load->getArgOperand(2);
+  // Conservatively check if EVL is a multiple of factor, otherwise some
+  // (trailing) elements might be lost after the transformation.
+  if (!isMultipleOfN(WideEVL, Load->getDataLayout(), Factor))
+    return false;
+
+  auto *XLenTy = Type::getIntNTy(Load->getContext(), Subtarget.getXLen());
+  Value *EVL = Builder.CreateZExt(
+      Builder.CreateUDiv(WideEVL, ConstantInt::get(WideEVL->getType(), Factor)),
+      XLenTy);
+
+  static const Intrinsic::ID IntrMaskIds[] = {
+      Intrinsic::riscv_vlseg2_mask, Intrinsic::riscv_vlseg3_mask,
+      Intrinsic::riscv_vlseg4_mask, Intrinsic::riscv_vlseg5_mask,
+      Intrinsic::riscv_vlseg6_mask, Intrinsic::riscv_vlseg7_mask,
+      Intrinsic::riscv_vlseg8_mask,
+  };
+
+  unsigned SEW = DL.getTypeSizeInBits(VTy->getElementType());
+  unsigned NumElts = VTy->getElementCount().getKnownMinValue();
+  Type *VecTupTy = TargetExtType::get(
+      Load->getContext(), "riscv.vector.tuple",
+      ScalableVectorType::get(Type::getInt8Ty(Load->getContext()),
+                              NumElts * SEW / 8),
+      Factor);
+
+  Value *PoisonVal = PoisonValue::get(VecTupTy);
+
+  Function *VlsegNFunc = Intrinsic::getOrInsertDeclaration(
+      Load->getModule(), IntrMaskIds[Factor - 2],
+      {VecTupTy, Mask->getType(), EVL->getType()});
+
+  Value *Operands[] = {
+      PoisonVal,
+      Load->getArgOperand(0),
+      Mask,
+      EVL,
+      ConstantInt::get(XLenTy, RISCVII::TAIL_AGNOSTIC | RISCVII::MASK_AGNOSTIC),
+      ConstantInt::get(XLenTy, Log2_64(SEW))};
+
+  CallInst *VlsegN = Builder.CreateCall(VlsegNFunc, Operands);
+
+  SmallVector<Type *, 8> AggrTypes{Factor, VTy};
+  Value *Return =
+      PoisonValue::get(StructType::get(Load->getContext(), AggrTypes));
+  Function *VecExtractFunc = Intrinsic::getOrInsertDeclaration(
+      Load->getModule(), Intrinsic::riscv_tuple_extract, {VTy, VecTupTy});
+  for (unsigned i = 0; i < Factor; ++i) {
+    Value *VecExtract =
+        Builder.CreateCall(VecExtractFunc, {VlsegN, Builder.getInt32(i)});
+    Return = Builder.CreateInsertValue(Return, VecExtract, i);
+  }
+
+  for (auto [Idx, DIO] : enumerate(DeinterleaveResults)) {
+    // We have to create a brand new ExtractValue to replace each
+    // of these old ExtractValue instructions.
+    Value *NewEV =
+        Builder.CreateExtractValue(Return, {static_cast<unsigned>(Idx)});
+    DIO->replaceAllUsesWith(NewEV);
+  }
+
+  return true;
+}
+
+/// Lower an interleaved vp.store into a vssegN intrinsic.
+///
+/// E.g. Lower an interleaved vp.store (Factor = 2):
+///
+///   %is = tail call <vscale x 64 x i8>
+///             @llvm.vector.interleave2.nxv64i8(
+///                               <vscale x 32 x i8> %load0,
+///                               <vscale x 32 x i8> %load1
+///   %wide.rvl = shl nuw nsw i32 %rvl, 1
+///   tail call void @llvm.vp.store.nxv64i8.p0(
+///                               <vscale x 64 x i8> %is, ptr %ptr,
+///                               %mask,
+///                               i32 %wide.rvl)
+///
+/// Into:
+///   call void @llvm.riscv.vsseg2.mask.nxv32i8.i64(
+///                               <vscale x 32 x i8> %load1,
+///                               <vscale x 32 x i8> %load2, ptr %ptr,
+///                               %mask,
+///                               i64 %rvl)
+bool RISCVTargetLowering::lowerInterleavedIntrinsicToVPStore(
+    VPIntrinsic *Store, Value *Mask,
+    ArrayRef<Value *> InterleaveOperands) const {
+  assert(Mask && "Expect a valid mask");
+  assert(Store->getIntrinsicID() == Intrinsic::vp_store &&
+         "Unexpected intrinsic");
+
+  const unsigned Factor = InterleaveOperands.size();
+
+  auto *VTy = dyn_cast<ScalableVectorType>(InterleaveOperands[0]->getType());
+  // TODO: Support fixed vectors.
+  if (!VTy)
+    return false;
+
+  const DataLayout &DL = Store->getDataLayout();
+  Align Alignment = Store->getParamAlign(1).value_or(
+      DL.getABITypeAlign(VTy->getElementType()));
+  if (!isLegalInterleavedAccessType(
+          VTy, Factor, Alignment,
+          Store->getArgOperand(1)->getType()->getPointerAddressSpace(), DL))
+    return false;
+
+  IRBuilder<> Builder(Store);
+  Value *WideEVL = Store->getArgOperand(3);
+  // Conservatively check if EVL is a multiple of factor, otherwise some
+  // (trailing) elements might be lost after the transformation.
+  if (!isMultipleOfN(WideEVL, Store->getDataLayout(), Factor))
+    return false;
+
+  auto *XLenTy = Type::getIntNTy(Store->getContext(), Subtarget.getXLen());
+  Value *EVL = Builder.CreateZExt(
+      Builder.CreateUDiv(WideEVL, ConstantInt::get(WideEVL->getType(), Factor)),
+      XLenTy);
+
+  static const Intrinsic::ID IntrMaskIds[] = {
+      Intrinsic::riscv_vsseg2_mask, Intrinsic::riscv_vsseg3_mask,
+      Intrinsic::riscv_vsseg4_mask, Intrinsic::riscv_vsseg5_mask,
+      Intrinsic::riscv_vsseg6_mask, Intrinsic::riscv_vsseg7_mask,
+      Intrinsic::riscv_vsseg8_mask,
+  };
+
+  unsigned SEW = DL.getTypeSizeInBits(VTy->getElementType());
+  unsigned NumElts = VTy->getElementCount().getKnownMinValue();
+  Type *VecTupTy = TargetExtType::get(
+      Store->getContext(), "riscv.vector.tuple",
+      ScalableVectorType::get(Type::getInt8Ty(Store->getContext()),
+                              NumElts * SEW / 8),
+      Factor);
+
+  Function *VecInsertFunc = Intrinsic::getOrInsertDeclaration(
+      Store->getModule(), Intrinsic::riscv_tuple_insert, {VecTupTy, VTy});
+  Value *StoredVal = PoisonValue::get(VecTupTy);
+  for (unsigned i = 0; i < Factor; ++i)
+    StoredVal = Builder.CreateCall(
+        VecInsertFunc, {StoredVal, InterleaveOperands[i], Builder.getInt32(i)});
+
+  Function *VssegNFunc = Intrinsic::getOrInsertDeclaration(
+      Store->getModule(), IntrMaskIds[Factor - 2],
+      {VecTupTy, Mask->getType(), EVL->getType()});
+
+  Value *Operands[] = {StoredVal, Store->getArgOperand(1), Mask, EVL,
+                       ConstantInt::get(XLenTy, Log2_64(SEW))};
+
+  Builder.CreateCall(VssegNFunc, Operands);
+  return true;
+}
+
 MachineInstr *
 RISCVTargetLowering::EmitKCFICheck(MachineBasicBlock &MBB,
                                    MachineBasicBlock::instr_iterator &MBBI,
diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.h b/llvm/lib/Target/RISCV/RISCVISelLowering.h
index 77605a3076a80..e9dd8ff96fa37 100644
--- a/llvm/lib/Target/RISCV/RISCVISelLowering.h
+++ b/llvm/lib/Target/RISCV/RISCVISelLowering.h
@@ -910,6 +910,14 @@ class RISCVTargetLowering : public TargetLowering {
   bool lowerInterleaveIntrinsicToStore(
       StoreInst *SI, ArrayRef<Value *> InterleaveValues) const override;
 
+  bool lowerDeinterleavedIntrinsicToVPLoad(
+      VPIntrinsic *Load, Value *Mask,
+      ArrayRef<Value *> DeinterleaveRes) const override;
+
+  bool lowerInterleavedIntrinsicToVPStore(
+      VPIntrinsic *Store, Value *Mask,
+      ArrayRef<Value *> InterleaveOps) const override;
+
   bool supportKCFIBundles() const override { return true; }
 
   SDValue expandIndirectJTBranch(const SDLoc &dl, SDValue Value, SDValue Addr,
diff --git a/llvm/lib/Target/RISCV/RISCVIndirectBranchTracking.cpp b/llvm/lib/Target/RISCV/RISCVIndirectBranchTracking.cpp
index c006fba4af4bc..4660a975b20ae 100644
--- a/llvm/lib/Target/RISCV/RISCVIndirectBranchTracking.cpp
+++ b/llvm/lib/Target/RISCV/RISCVIndirectBranchTracking.cpp
@@ -7,7 +7,7 @@
 //===----------------------------------------------------------------------===//
 //
 // The pass adds LPAD (AUIPC with rs1 = X0) machine instructions at the
-// beginning of each basic block or function that is referenced by an indrect
+// beginning of each basic block or function that is referenced by an indirect
 // jump/call instruction.
 //
 //===----------------------------------------------------------------------===//
diff --git a/llvm/lib/Target/RISCV/RISCVInsertVSETVLI.cpp b/llvm/lib/Target/RISCV/RISCVInsertVSETVLI.cpp
index 1fd130d7e040e..4a74906ed3cc3 100644
--- a/llvm/lib/Target/RISCV/RISCVInsertVSETVLI.cpp
+++ b/llvm/lib/Target/RISCV/RISCVInsertVSETVLI.cpp
@@ -1069,7 +1069,7 @@ RISCVInsertVSETVLI::computeInfoForInstr(const MachineInstr &MI) const {
     const MachineOperand &VLOp = MI.getOperand(getVLOpNum(MI));
     if (VLOp.isImm()) {
       int64_t Imm = VLOp.getImm();
-      // Conver the VLMax sentintel to X0 register.
+      // Convert the VLMax sentintel to X0 register.
       if (Imm == RISCV::VLMaxSentinel) {
         // If we know the exact VLEN, see if we can use the constant encoding
         // for the VLMAX instead.  This reduces register pressure slightly.
@@ -1719,8 +1719,7 @@ void RISCVInsertVSETVLI::coalesceVSETVLIs(MachineBasicBlock &MBB) const {
       ToDelete.push_back(VLOpDef);
   };
 
-  for (MachineInstr &MI :
-       make_early_inc_range(make_range(MBB.rbegin(), MBB.rend()))) {
+  for (MachineInstr &MI : make_early_inc_range(reverse(MBB))) {
 
     if (!isVectorConfigInstr(MI)) {
       Used.doUnion(getDemanded(MI, ST));
diff --git a/llvm/lib/Target/RISCV/RISCVInsertWriteVXRM.cpp b/llvm/lib/Target/RISCV/RISCVInsertWriteVXRM.cpp
index fe593a3cabad7..7df04fc225b0b 100644
--- a/llvm/lib/Target/RISCV/RISCVInsertWriteVXRM.cpp
+++ b/llvm/lib/Target/RISCV/RISCVInsertWriteVXRM.cpp
@@ -170,10 +170,10 @@ struct BlockData {
   // Indicates if the block uses VXRM. Uninitialized means no use.
   VXRMInfo VXRMUse;
 
-  // Indicates the VXRM output from the block. Unitialized means transparent.
+  // Indicates the VXRM output from the block. Uninitialized means transparent.
   VXRMInfo VXRMOut;
 
-  // Keeps track of the available VXRM value at the start of the basic bloc.
+  // Keeps track of the available VXRM value at the start of the basic block.
   VXRMInfo AvailableIn;
 
   // Keeps track of the available VXRM value at the end of the basic block.
@@ -384,8 +384,8 @@ void RISCVInsertWriteVXRM::emitWriteVXRM(MachineBasicBlock &MBB) {
             PInfo.AvailableOut.getVXRMImm() ==
                 BBInfo.AnticipatedIn.getVXRMImm())
           continue;
-        // If the predecessor anticipates this value for all its succesors,
-        // then a write to VXRM would have already occured before this block is
+        // If the predecessor anticipates this value for all its successors,
+        // then a write to VXRM would have already occurred before this block is
         // executed.
         if (PInfo.AnticipatedOut.isStatic() &&
             PInfo.AnticipatedOut.getVXRMImm() ==
@@ -429,7 +429,7 @@ void RISCVInsertWriteVXRM::emitWriteVXRM(MachineBasicBlock &MBB) {
   // If all our successors anticipate a value, do the insert.
   // NOTE: It's possible that not all predecessors of our successor provide the
   // correct value. This can occur on critical edges. If we don't split the
-  // critical edge we'll also have a write vxrm in the succesor that is
+  // critical edge we'll also have a write vxrm in the successor that is
   // redundant with this one.
   if (PendingInsert ||
       (BBInfo.AnticipatedOut.isStatic() &&
diff --git a/llvm/lib/Target/RISCV/RISCVInstrFormats.td b/llvm/lib/Target/RISCV/RISCVInstrFormats.td
index 013c26c72bfd5..cea28bdce284c 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrFormats.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrFormats.td
@@ -232,7 +232,7 @@ class RVInstCommon<dag outs, dag ins, string opcodestr, string argstr,
   bit UsesVXRM = 0;
   let TSFlags{20} =  UsesVXRM;
 
-  // Indicates whther these instructions can partially overlap between source
+  // Indicates whether these instructions can partially overlap between source
   // registers and destination registers according to the vector spec.
   // 0 -> not a vector pseudo
   // 1 -> default value for vector pseudos. not widening or narrowing.
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfo.cpp b/llvm/lib/Target/RISCV/RISCVInstrInfo.cpp
index bd02880b0d712..1ec299e3c8cc0 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfo.cpp
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfo.cpp
@@ -437,6 +437,11 @@ void RISCVInstrInfo::copyPhysRegVector(
       MIB.addReg(RISCV::VL, RegState::Implicit);
       MIB.addReg(RISCV::VTYPE, RegState::Implicit);
     }
+    // Add an implicit read of the original source to silence the verifier
+    // in the cases where some of the smaller VRs we're copying from might be
+    // undef, caused by the fact that the original, larger source VR might not
+    // be fully initialized at the time this COPY happens.
+    MIB.addReg(SrcReg, RegState::Implicit);
 
     // If we are copying reversely, we should decrease the encoding.
     SrcEncoding += (ReversedCopy ? -NumCopied : NumCopied);
@@ -650,7 +655,8 @@ void RISCVInstrInfo::storeRegToStackSlot(MachineBasicBlock &MBB,
     BuildMI(MBB, I, DebugLoc(), get(Opcode))
         .addReg(SrcReg, getKillRegState(IsKill))
         .addFrameIndex(FI)
-        .addMemOperand(MMO);
+        .addMemOperand(MMO)
+        .setMIFlag(Flags);
   } else {
     MachineMemOperand *MMO = MF->getMachineMemOperand(
         MachinePointerInfo::getFixedStack(*MF, FI), MachineMemOperand::MOStore,
@@ -660,7 +666,8 @@ void RISCVInstrInfo::storeRegToStackSlot(MachineBasicBlock &MBB,
         .addReg(SrcReg, getKillRegState(IsKill))
         .addFrameIndex(FI)
         .addImm(0)
-        .addMemOperand(MMO);
+        .addMemOperand(MMO)
+        .setMIFlag(Flags);
   }
 }
 
@@ -670,6 +677,8 @@ void RISCVInstrInfo::loadRegFromStackSlot(
     Register VReg, MachineInstr::MIFlag Flags) const {
   MachineFunction *MF = MBB.getParent();
   MachineFrameInfo &MFI = MF->getFrameInfo();
+  DebugLoc DL =
+      Flags & MachineInstr::FrameDestroy ? MBB.findDebugLoc(I) : DebugLoc();
 
   unsigned Opcode;
   bool IsScalableVector = true;
@@ -734,18 +743,20 @@ void RISCVInstrInfo::loadRegFromStackSlot(
         LocationSize::beforeOrAfterPointer(), MFI.getObjectAlign(FI));
 
     MFI.setStackID(FI, TargetStackID::ScalableVector);
-    BuildMI(MBB, I, DebugLoc(), get(Opcode), DstReg)
+    BuildMI(MBB, I, DL, get(Opcode), DstReg)
         .addFrameIndex(FI)
-        .addMemOperand(MMO);
+        .addMemOperand(MMO)
+        .setMIFlag(Flags);
   } else {
     MachineMemOperand *MMO = MF->getMachineMemOperand(
         MachinePointerInfo::getFixedStack(*MF, FI), MachineMemOperand::MOLoad,
         MFI.getObjectSize(FI), MFI.getObjectAlign(FI));
 
-    BuildMI(MBB, I, DebugLoc(), get(Opcode), DstReg)
+    BuildMI(MBB, I, DL, get(Opcode), DstReg)
         .addFrameIndex(FI)
         .addImm(0)
-        .addMemOperand(MMO);
+        .addMemOperand(MMO)
+        .setMIFlag(Flags);
   }
 }
 
@@ -1510,7 +1521,7 @@ RISCVInstrInfo::optimizeSelect(MachineInstr &MI,
   SeenMIs.erase(DefMI);
 
   // If MI is inside a loop, and DefMI is outside the loop, then kill flags on
-  // DefMI would be invalid when tranferred inside the loop.  Checking for a
+  // DefMI would be invalid when transferred inside the loop.  Checking for a
   // loop is expensive, but at least remove kill flags if they are in different
   // BBs.
   if (DefMI->getParent() != MI.getParent())
@@ -2488,6 +2499,9 @@ bool RISCVInstrInfo::verifyInstruction(const MachineInstr &MI,
         case RISCVOp::OPERAND_UIMM7_LSB00:
           Ok = isShiftedUInt<5, 2>(Imm);
           break;
+        case RISCVOp::OPERAND_UIMM7_LSB000:
+          Ok = isShiftedUInt<4, 3>(Imm);
+          break;
         case RISCVOp::OPERAND_UIMM8_LSB00:
           Ok = isShiftedUInt<6, 2>(Imm);
           break;
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfo.td b/llvm/lib/Target/RISCV/RISCVInstrInfo.td
index bb5bb6352c32a..fde7dc89dd693 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfo.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfo.td
@@ -125,6 +125,7 @@ def ImmZeroAsmOperand : AsmOperandClass {
   let Name = "ImmZero";
   let RenderMethod = "addImmOperands";
   let DiagnosticType = !strconcat("Invalid", Name);
+  let DiagnosticString = "immediate must be zero";
 }
 
 // A parse method for (${gpr}) or 0(${gpr}), where the 0 is be silently ignored.
@@ -323,6 +324,7 @@ def BareSymbol : AsmOperandClass {
   let Name = "BareSymbol";
   let RenderMethod = "addImmOperands";
   let DiagnosticType = "InvalidBareSymbol";
+  let DiagnosticString = "operand must be a bare symbol name";
   let ParserMethod = "parseBareSymbol";
 }
 
@@ -335,6 +337,7 @@ def CallSymbol : AsmOperandClass {
   let Name = "CallSymbol";
   let RenderMethod = "addImmOperands";
   let DiagnosticType = "InvalidCallSymbol";
+  let DiagnosticString = "operand must be a bare symbol name";
   let ParserMethod = "parseCallSymbol";
 }
 
@@ -347,6 +350,7 @@ def PseudoJumpSymbol : AsmOperandClass {
   let Name = "PseudoJumpSymbol";
   let RenderMethod = "addImmOperands";
   let DiagnosticType = "InvalidPseudoJumpSymbol";
+  let DiagnosticString = "operand must be a valid jump target";
   let ParserMethod = "parsePseudoJumpSymbol";
 }
 
@@ -359,6 +363,7 @@ def TPRelAddSymbol : AsmOperandClass {
   let Name = "TPRelAddSymbol";
   let RenderMethod = "addImmOperands";
   let DiagnosticType = "InvalidTPRelAddSymbol";
+  let DiagnosticString = "operand must be a symbol with %tprel_add modifier";
   let ParserMethod = "parseOperandWithModifier";
 }
 
@@ -399,6 +404,10 @@ def ixlenimm_li_restricted : Operand<XLenVT> {
 
 // Standalone (codegen-only) immleaf patterns.
 
+// A 12-bit signed immediate plus one where the imm range will be -2047~2048.
+def simm12_plus1 : ImmLeaf<XLenVT,
+  [{return (isInt<12>(Imm) && Imm != -2048) || Imm == 2048;}]>;
+
 // A 6-bit constant greater than 32.
 def uimm6gt32 : ImmLeaf<XLenVT, [{
   return isUInt<6>(Imm) && Imm > 32;
@@ -1775,6 +1784,7 @@ def TLSDESCCallSymbol : AsmOperandClass {
   let Name = "TLSDESCCallSymbol";
   let RenderMethod = "addImmOperands";
   let DiagnosticType = "InvalidTLSDESCCallSymbol";
+  let DiagnosticString = "operand must be a symbol with %tlsdesc_call modifier";
   let ParserMethod = "parseOperandWithModifier";
 }
 
@@ -1945,15 +1955,19 @@ class binop_allhusers<SDPatternOperator operator>
     : PatFrag<(ops node:$lhs, node:$rhs),
               (XLenVT (operator node:$lhs, node:$rhs)), [{
   return hasAllHUsers(Node);
-}]>;
+}]> {
+  let GISelPredicateCode = [{ return hasAllHUsers(MI); }];
+}
 
 // PatFrag to allow ADDW/SUBW/MULW/SLLW to be selected from i64 add/sub/mul/shl
 // if only the lower 32 bits of their result is used.
 class binop_allwusers<SDPatternOperator operator>
-    : PatFrag<(ops node:$lhs, node:$rhs),
-              (i64 (operator node:$lhs, node:$rhs)), [{
+    : PatFrag<(ops node:$lhs, node:$rhs), (i64 (operator node:$lhs, node:$rhs)),
+              [{
   return hasAllWUsers(Node);
-}]>;
+}]> {
+  let GISelPredicateCode = [{ return hasAllWUsers(MI); }];
+}
 
 def sexti32_allwusers : PatFrag<(ops node:$src),
                                 (sext_inreg node:$src, i32), [{
@@ -2133,6 +2147,7 @@ include "RISCVInstrInfoSFB.td"
 include "RISCVInstrInfoXCV.td"
 include "RISCVInstrInfoXwch.td"
 include "RISCVInstrInfoXqci.td"
+include "RISCVInstrInfoXMips.td"
 
 //===----------------------------------------------------------------------===//
 // Global ISel
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoV.td b/llvm/lib/Target/RISCV/RISCVInstrInfoV.td
index 24a881dc6810f..fdb2334b131da 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfoV.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfoV.td
@@ -48,6 +48,15 @@ def VMaskAsmOperand : AsmOperandClass {
   let IsOptional = 1;
   let DefaultMethod = "defaultMaskRegOp";
   let DiagnosticType = "InvalidVMaskRegister";
+  let DiagnosticString = "operand must be v0.t";
+}
+
+def VMaskCarryInAsmOperand : AsmOperandClass {
+  let Name = "RVVMaskCarryInRegOpOperand";
+  let RenderMethod = "addRegOperands";
+  let PredicateMethod = "isV0Reg";
+  let DiagnosticType = "InvalidVMaskCarryInRegister";
+  let DiagnosticString = "operand must be v0";
 }
 
 def VMaskOp : RegisterOperand<VMV0> {
@@ -57,6 +66,11 @@ def VMaskOp : RegisterOperand<VMV0> {
   let DecoderMethod = "decodeVMaskReg";
 }
 
+def VMaskCarryInOp : RegisterOperand<VMV0> {
+  let ParserMatchClass = VMaskCarryInAsmOperand;
+  let EncoderMethod = "getVMaskReg";
+}
+
 def simm5 : RISCVSImmLeafOp<5> {
   let MCOperandPredicate = [{
     int64_t Imm;
@@ -442,10 +456,8 @@ class VALUVV<bits<6> funct6, RISCVVFormat opv, string opcodestr>
 // op vd, vs2, vs1, v0 (without mask, use v0 as carry input)
 class VALUmVV<bits<6> funct6, RISCVVFormat opv, string opcodestr>
     : RVInstVV<funct6, opv, (outs VR:$vd),
-                (ins VR:$vs2, VR:$vs1, VMV0:$v0),
-                opcodestr, "$vd, $vs2, $vs1, v0"> {
-  let vm = 0;
-}
+                (ins VR:$vs2, VR:$vs1, VMaskCarryInOp:$vm),
+                opcodestr, "$vd, $vs2, $vs1, $vm">;
 
 // op vd, vs1, vs2, vm (reverse the order of vs1 and vs2)
 class VALUrVV<bits<6> funct6, RISCVVFormat opv, string opcodestr,
@@ -474,10 +486,8 @@ class VALUVX<bits<6> funct6, RISCVVFormat opv, string opcodestr>
 // op vd, vs2, rs1, v0 (without mask, use v0 as carry input)
 class VALUmVX<bits<6> funct6, RISCVVFormat opv, string opcodestr>
     : RVInstVX<funct6, opv, (outs VR:$vd),
-                (ins VR:$vs2, GPR:$rs1, VMV0:$v0),
-                opcodestr, "$vd, $vs2, $rs1, v0"> {
-  let vm = 0;
-}
+                (ins VR:$vs2, GPR:$rs1, VMaskCarryInOp:$vm),
+                opcodestr, "$vd, $vs2, $rs1, $vm">;
 
 // op vd, rs1, vs2, vm (reverse the order of rs1 and vs2)
 class VALUrVX<bits<6> funct6, RISCVVFormat opv, string opcodestr,
@@ -506,10 +516,8 @@ class VALUVI<bits<6> funct6, string opcodestr, Operand optype = simm5>
 // op vd, vs2, imm, v0 (without mask, use v0 as carry input)
 class VALUmVI<bits<6> funct6, string opcodestr, Operand optype = simm5>
     : RVInstIVI<funct6, (outs VR:$vd),
-                (ins VR:$vs2, optype:$imm, VMV0:$v0),
-                opcodestr, "$vd, $vs2, $imm, v0"> {
-  let vm = 0;
-}
+                (ins VR:$vs2, optype:$imm, VMaskCarryInOp:$vm),
+                opcodestr, "$vd, $vs2, $imm, $vm">;
 
 // op vd, vs2, imm, vm
 class VALUVINoVm<bits<6> funct6, string opcodestr, Operand optype = simm5>
@@ -1458,10 +1466,9 @@ defm VFCLASS_V : VCLS_FV_VS2<"vfclass.v", 0b010011, 0b10000>;
 let hasSideEffects = 0, mayLoad = 0, mayStore = 0 in {
 
 // Vector Floating-Point Merge Instruction
-let vm = 0 in
 def VFMERGE_VFM : RVInstVX<0b010111, OPFVF, (outs VR:$vd),
-                           (ins VR:$vs2, FPR32:$rs1, VMV0:$v0),
-                           "vfmerge.vfm", "$vd, $vs2, $rs1, v0">,
+                           (ins VR:$vs2, FPR32:$rs1, VMaskCarryInOp:$vm),
+                           "vfmerge.vfm", "$vd, $vs2, $rs1, $vm">,
                   SchedBinaryMC<"WriteVFMergeV", "ReadVFMergeV", "ReadVFMergeF">;
 
 // Vector Floating-Point Move Instruction
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td b/llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td
index 268bfe70673a2..fe85d4b074c87 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td
@@ -2136,23 +2136,6 @@ multiclass VPseudoBinaryRoundingMode<VReg RetClass,
 }
 
 
-multiclass VPseudoBinaryM<VReg RetClass,
-                          VReg Op1Class,
-                          DAGOperand Op2Class,
-                          LMULInfo MInfo,
-                          string Constraint = "",
-                          bits<2> TargetConstraintType = 1,
-                          bit Commutable = 0> {
-  let VLMul = MInfo.value, isCommutable = Commutable in {
-    def "_" # MInfo.MX : VPseudoBinaryNoMask<RetClass, Op1Class, Op2Class,
-                                             Constraint, TargetConstraintType>;
-    let ForceTailAgnostic = true in
-    def "_" # MInfo.MX # "_MASK" : VPseudoBinaryMOutMask<RetClass, Op1Class,
-                                                         Op2Class, Constraint, TargetConstraintType>,
-                                   RISCVMaskedPseudo<MaskIdx=3>;
-  }
-}
-
 multiclass VPseudoBinaryEmul<VReg RetClass,
                              VReg Op1Class,
                              DAGOperand Op2Class,
@@ -2633,28 +2616,35 @@ multiclass PseudoVEXT_VF8 {
 //  lowest-numbered part of the source register group".
 // With LMUL<=1 the source and dest occupy a single register so any overlap
 // is in the lowest-numbered part.
-multiclass VPseudoBinaryM_VV<LMULInfo m, bits<2> TargetConstraintType = 1,
-                             bit Commutable = 0> {
-  defm _VV : VPseudoBinaryM<VR, m.vrclass, m.vrclass, m,
+multiclass VPseudoBinaryM<DAGOperand Op2Class, LMULInfo m, bit Commutable = 0> {
+  let VLMul = m.value, isCommutable = Commutable in {
+    def "_" # m.MX :
+      VPseudoBinaryNoMask<VR, m.vrclass, Op2Class,
+                          !if(!ge(m.octuple, 16), "@earlyclobber $rd", ""),
+                          TargetConstraintType = 2>;
+    let ForceTailAgnostic = true in
+    def "_" # m.MX # "_MASK" :
+      VPseudoBinaryMOutMask<VR, m.vrclass, Op2Class,
                             !if(!ge(m.octuple, 16), "@earlyclobber $rd", ""),
-                            TargetConstraintType, Commutable=Commutable>;
+                            TargetConstraintType = 2>,
+      RISCVMaskedPseudo<MaskIdx=3>;
+  }
 }
 
-multiclass VPseudoBinaryM_VX<LMULInfo m, bits<2> TargetConstraintType = 1> {
-  defm "_VX" :
-    VPseudoBinaryM<VR, m.vrclass, GPR, m,
-                   !if(!ge(m.octuple, 16), "@earlyclobber $rd", ""), TargetConstraintType>;
+multiclass VPseudoBinaryM_VV<LMULInfo m, bit Commutable = 0> {
+  defm _VV : VPseudoBinaryM<m.vrclass, m, Commutable=Commutable>;
 }
 
-multiclass VPseudoBinaryM_VF<LMULInfo m, FPR_Info f, bits<2> TargetConstraintType = 1> {
-  defm "_V" # f.FX :
-    VPseudoBinaryM<VR, m.vrclass, f.fprclass, m,
-                   !if(!ge(m.octuple, 16), "@earlyclobber $rd", ""), TargetConstraintType>;
+multiclass VPseudoBinaryM_VX<LMULInfo m> {
+  defm _VX : VPseudoBinaryM<GPR, m>;
 }
 
-multiclass VPseudoBinaryM_VI<LMULInfo m, bits<2> TargetConstraintType = 1> {
-  defm _VI : VPseudoBinaryM<VR, m.vrclass, simm5, m,
-                            !if(!ge(m.octuple, 16), "@earlyclobber $rd", ""), TargetConstraintType>;
+multiclass VPseudoBinaryM_VF<LMULInfo m, FPR_Info f> {
+  defm "_V" # f.FX : VPseudoBinaryM<f.fprclass, m>;
+}
+
+multiclass VPseudoBinaryM_VI<LMULInfo m> {
+  defm _VI : VPseudoBinaryM<simm5, m>;
 }
 
 multiclass VPseudoVGTR_VV_VX_VI {
@@ -3397,11 +3387,11 @@ multiclass VPseudoVWMAC_VV_VF_BF_RM {
 multiclass VPseudoVCMPM_VV_VX_VI<bit Commutable = 0> {
   foreach m = MxList in {
     defvar mx = m.MX;
-    defm "" : VPseudoBinaryM_VV<m, TargetConstraintType=2, Commutable=Commutable>,
+    defm "" : VPseudoBinaryM_VV<m, Commutable=Commutable>,
               SchedBinary<"WriteVICmpV", "ReadVICmpV", "ReadVICmpV", mx>;
-    defm "" : VPseudoBinaryM_VX<m, TargetConstraintType=2>,
+    defm "" : VPseudoBinaryM_VX<m>,
               SchedBinary<"WriteVICmpX", "ReadVICmpV", "ReadVICmpX", mx>;
-    defm "" : VPseudoBinaryM_VI<m, TargetConstraintType=2>,
+    defm "" : VPseudoBinaryM_VI<m>,
               SchedUnary<"WriteVICmpI", "ReadVICmpV", mx>;
   }
 }
@@ -3409,22 +3399,32 @@ multiclass VPseudoVCMPM_VV_VX_VI<bit Commutable = 0> {
 multiclass VPseudoVCMPM_VV_VX {
   foreach m = MxList in {
     defvar mx = m.MX;
-    defm "" : VPseudoBinaryM_VV<m, TargetConstraintType=2>,
+    defm "" : VPseudoBinaryM_VV<m>,
               SchedBinary<"WriteVICmpV", "ReadVICmpV", "ReadVICmpV", mx>;
-    defm "" : VPseudoBinaryM_VX<m, TargetConstraintType=2>,
+    defm "" : VPseudoBinaryM_VX<m>,
+              SchedBinary<"WriteVICmpX", "ReadVICmpV", "ReadVICmpX", mx>;
+  }
+}
+
+multiclass VPseudoVCMPM_VX_VI {
+  foreach m = MxList in {
+    defvar mx = m.MX;
+    defm "" : VPseudoBinaryM_VX<m>,
               SchedBinary<"WriteVICmpX", "ReadVICmpV", "ReadVICmpX", mx>;
+    defm "" : VPseudoBinaryM_VI<m>,
+              SchedUnary<"WriteVICmpI", "ReadVICmpV", mx>;
   }
 }
 
 multiclass VPseudoVCMPM_VV_VF {
   foreach m = MxListF in {
-    defm "" : VPseudoBinaryM_VV<m, TargetConstraintType=2>,
+    defm "" : VPseudoBinaryM_VV<m>,
               SchedBinary<"WriteVFCmpV", "ReadVFCmpV", "ReadVFCmpV", m.MX>;
   }
 
   foreach f = FPList in {
     foreach m = f.MxList in {
-      defm "" : VPseudoBinaryM_VF<m, f, TargetConstraintType=2>,
+      defm "" : VPseudoBinaryM_VF<m, f>,
                 SchedBinary<"WriteVFCmpF", "ReadVFCmpV", "ReadVFCmpF", m.MX>;
     }
   }
@@ -3433,22 +3433,12 @@ multiclass VPseudoVCMPM_VV_VF {
 multiclass VPseudoVCMPM_VF {
   foreach f = FPList in {
     foreach m = f.MxList in {
-      defm "" : VPseudoBinaryM_VF<m, f, TargetConstraintType=2>,
+      defm "" : VPseudoBinaryM_VF<m, f>,
                 SchedBinary<"WriteVFCmpF", "ReadVFCmpV", "ReadVFCmpF", m.MX>;
     }
   }
 }
 
-multiclass VPseudoVCMPM_VX_VI {
-  foreach m = MxList in {
-    defvar mx = m.MX;
-    defm "" : VPseudoBinaryM_VX<m, TargetConstraintType=2>,
-              SchedBinary<"WriteVICmpX", "ReadVICmpV", "ReadVICmpX", mx>;
-    defm "" : VPseudoBinaryM_VI<m, TargetConstraintType=2>,
-              SchedUnary<"WriteVICmpI", "ReadVICmpV", mx>;
-  }
-}
-
 multiclass VPseudoVRED_VS {
   foreach m = MxList in {
     defvar mx = m.MX;
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoVSDPatterns.td b/llvm/lib/Target/RISCV/RISCVInstrInfoVSDPatterns.td
index 880ea0ae0a976..c588e047c2ac8 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfoVSDPatterns.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfoVSDPatterns.td
@@ -313,29 +313,10 @@ multiclass VPatIntegerSetCCSDNode_VX_Swappable<string instruction_name,
                                           SplatPat, GPR>;
 
 multiclass VPatIntegerSetCCSDNode_VI_Swappable<string instruction_name,
-                                               CondCode cc, CondCode invcc>
+                                               CondCode cc, CondCode invcc,
+                                               ComplexPattern splatpat_kind = SplatPat_simm5>
     : VPatIntegerSetCCSDNode_XI_Swappable<instruction_name, cc, invcc, "VI",
-                                          SplatPat_simm5, simm5>;
-
-multiclass VPatIntegerSetCCSDNode_VIPlus1_Swappable<string instruction_name,
-                                                    CondCode cc, CondCode invcc,
-                                                    ComplexPattern splatpat_kind> {
-  foreach vti = AllIntegerVectors in {
-    defvar instruction = !cast<Instruction>(instruction_name#"_VI_"#vti.LMul.MX);
-    let Predicates = GetVTypePredicates<vti>.Predicates in {
-      def : Pat<(vti.Mask (setcc (vti.Vector vti.RegClass:$rs1),
-                                 (vti.Vector (splatpat_kind simm5:$rs2)),
-                                 cc)),
-                (instruction vti.RegClass:$rs1, (DecImm simm5:$rs2),
-                             vti.AVL, vti.Log2SEW)>;
-      def : Pat<(vti.Mask (setcc (vti.Vector (splatpat_kind simm5:$rs2)),
-                                 (vti.Vector vti.RegClass:$rs1),
-                                 invcc)),
-                (instruction vti.RegClass:$rs1, (DecImm simm5:$rs2),
-                             vti.AVL, vti.Log2SEW)>;
-    }
-  }
-}
+                                          splatpat_kind, simm5>;
 
 multiclass VPatFPSetCCSDNode_VV_VF_FV<CondCode cc,
                                       string inst_name,
@@ -555,19 +536,19 @@ multiclass VPatWidenBinaryFPSDNode_VV_VF<SDNode op, string instruction_name> {
     defvar wti = vtiToWti.Wti;
     let Predicates = !listconcat(GetVTypePredicates<vti>.Predicates,
                                  GetVTypePredicates<wti>.Predicates) in {
-      def : Pat<(op (wti.Vector (riscv_fpextend_vl_oneuse
+      def : Pat<(op (wti.Vector (riscv_fpextend_vl_sameuser
                                      (vti.Vector vti.RegClass:$rs2),
                                      (vti.Mask true_mask), (XLenVT srcvalue))),
-                    (wti.Vector (riscv_fpextend_vl_oneuse
+                    (wti.Vector (riscv_fpextend_vl_sameuser
                                      (vti.Vector vti.RegClass:$rs1),
                                      (vti.Mask true_mask), (XLenVT srcvalue)))),
                 (!cast<Instruction>(instruction_name#"_VV_"#vti.LMul.MX)
                   (wti.Vector (IMPLICIT_DEF)), vti.RegClass:$rs2,
                   vti.RegClass:$rs1, vti.AVL, vti.Log2SEW, TA_MA)>;
-      def : Pat<(op (wti.Vector (riscv_fpextend_vl_oneuse
+      def : Pat<(op (wti.Vector (riscv_fpextend_vl_sameuser
                                      (vti.Vector vti.RegClass:$rs2),
                                      (vti.Mask true_mask), (XLenVT srcvalue))),
-                    (wti.Vector (riscv_fpextend_vl_oneuse
+                    (wti.Vector (riscv_fpextend_vl_sameuser
                                      (vti.Vector (SplatFPOp vti.ScalarRegClass:$rs1)),
                                      (vti.Mask true_mask), (XLenVT srcvalue)))),
                 (!cast<Instruction>(instruction_name#"_V"#vti.ScalarSuffix#"_"#vti.LMul.MX)
@@ -590,10 +571,10 @@ multiclass VPatWidenBinaryFPSDNode_VV_VF_RM<SDNode op, string instruction_name>
     defvar wti = vtiToWti.Wti;
     let Predicates = !listconcat(GetVTypePredicates<vti>.Predicates,
                                  GetVTypePredicates<wti>.Predicates) in {
-      def : Pat<(op (wti.Vector (riscv_fpextend_vl_oneuse
+      def : Pat<(op (wti.Vector (riscv_fpextend_vl_sameuser
                                      (vti.Vector vti.RegClass:$rs2),
                                      (vti.Mask true_mask), (XLenVT srcvalue))),
-                    (wti.Vector (riscv_fpextend_vl_oneuse
+                    (wti.Vector (riscv_fpextend_vl_sameuser
                                      (vti.Vector vti.RegClass:$rs1),
                                      (vti.Mask true_mask), (XLenVT srcvalue)))),
                 (!cast<Instruction>(instruction_name#"_VV_"#vti.LMul.MX#"_E"#vti.SEW)
@@ -603,10 +584,10 @@ multiclass VPatWidenBinaryFPSDNode_VV_VF_RM<SDNode op, string instruction_name>
                    // RISCVInsertReadWriteCSR
                    FRM_DYN,
                   vti.AVL, vti.Log2SEW, TA_MA)>;
-      def : Pat<(op (wti.Vector (riscv_fpextend_vl_oneuse
+      def : Pat<(op (wti.Vector (riscv_fpextend_vl_sameuser
                                      (vti.Vector vti.RegClass:$rs2),
                                      (vti.Mask true_mask), (XLenVT srcvalue))),
-                    (wti.Vector (riscv_fpextend_vl_oneuse
+                    (wti.Vector (riscv_fpextend_vl_sameuser
                                      (vti.Vector (SplatFPOp (vti.Scalar vti.ScalarRegClass:$rs1))),
                                      (vti.Mask true_mask), (XLenVT srcvalue)))),
                 (!cast<Instruction>(instruction_name#"_V"#vti.ScalarSuffix#"_"#vti.LMul.MX#"_E"#vti.SEW)
@@ -688,10 +669,10 @@ multiclass VPatWidenFPMulAccSDNode_VV_VF_RM<string instruction_name,
                                  !if(!eq(vti.Scalar, bf16),
                                      [HasStdExtZvfbfwma],
                                      [])) in {
-      def : Pat<(fma (wti.Vector (riscv_fpextend_vl_oneuse
+      def : Pat<(fma (wti.Vector (riscv_fpextend_vl_sameuser
                                       (vti.Vector vti.RegClass:$rs1),
                                       (vti.Mask true_mask), (XLenVT srcvalue))),
-                     (wti.Vector (riscv_fpextend_vl_oneuse
+                     (wti.Vector (riscv_fpextend_vl_sameuser
                                       (vti.Vector vti.RegClass:$rs2),
                                       (vti.Mask true_mask), (XLenVT srcvalue))),
                      (wti.Vector wti.RegClass:$rd)),
@@ -768,11 +749,11 @@ multiclass VPatWidenFPMulSacSDNode_VV_VF_RM<string instruction_name> {
     defvar suffix = vti.LMul.MX # "_E" # vti.SEW;
     let Predicates = !listconcat(GetVTypePredicates<vti>.Predicates,
                                  GetVTypePredicates<wti>.Predicates) in {
-      def : Pat<(fma (wti.Vector (riscv_fpextend_vl_oneuse
+      def : Pat<(fma (wti.Vector (riscv_fpextend_vl_sameuser
                                       (vti.Vector vti.RegClass:$rs1),
                                       (vti.Mask true_mask), (XLenVT srcvalue))),
-                     (riscv_fpextend_vl_oneuse (vti.Vector vti.RegClass:$rs2),
-                                               (vti.Mask true_mask), (XLenVT srcvalue)),
+                     (riscv_fpextend_vl_sameuser (vti.Vector vti.RegClass:$rs2),
+                                                 (vti.Mask true_mask), (XLenVT srcvalue)),
                      (fneg wti.RegClass:$rd)),
                 (!cast<Instruction>(instruction_name#"_VV_"#suffix)
                    wti.RegClass:$rd, vti.RegClass:$rs1, vti.RegClass:$rs2,
@@ -1021,14 +1002,14 @@ defm : VPatIntegerSetCCSDNode_VI_Swappable<"PseudoVMSLEU", SETULE, SETUGE>;
 defm : VPatIntegerSetCCSDNode_VI_Swappable<"PseudoVMSGT",  SETGT, SETLT>;
 defm : VPatIntegerSetCCSDNode_VI_Swappable<"PseudoVMSGTU", SETUGT, SETULT>;
 
-defm : VPatIntegerSetCCSDNode_VIPlus1_Swappable<"PseudoVMSLE", SETLT, SETGT,
-                                                SplatPat_simm5_plus1>;
-defm : VPatIntegerSetCCSDNode_VIPlus1_Swappable<"PseudoVMSLEU", SETULT, SETUGT,
-                                                SplatPat_simm5_plus1_nonzero>;
-defm : VPatIntegerSetCCSDNode_VIPlus1_Swappable<"PseudoVMSGT", SETGE, SETLE,
-                                                SplatPat_simm5_plus1>;
-defm : VPatIntegerSetCCSDNode_VIPlus1_Swappable<"PseudoVMSGTU", SETUGE, SETULE,
-                                                SplatPat_simm5_plus1_nonzero>;
+defm : VPatIntegerSetCCSDNode_VI_Swappable<"PseudoVMSLE", SETLT, SETGT,
+                                           SplatPat_simm5_plus1>;
+defm : VPatIntegerSetCCSDNode_VI_Swappable<"PseudoVMSLEU", SETULT, SETUGT,
+                                           SplatPat_simm5_plus1_nonzero>;
+defm : VPatIntegerSetCCSDNode_VI_Swappable<"PseudoVMSGT", SETGE, SETLE,
+                                           SplatPat_simm5_plus1>;
+defm : VPatIntegerSetCCSDNode_VI_Swappable<"PseudoVMSGTU", SETUGE, SETULE,
+                                           SplatPat_simm5_plus1_nonzero>;
 
 // 11.9. Vector Integer Min/Max Instructions
 defm : VPatBinarySDNode_VV_VX<umin, "PseudoVMINU">;
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoVVLPatterns.td b/llvm/lib/Target/RISCV/RISCVInstrInfoVVLPatterns.td
index 2026ba79e623d..f63c1560f6253 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfoVVLPatterns.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfoVVLPatterns.td
@@ -554,6 +554,11 @@ def riscv_fpextend_vl_oneuse : PatFrag<(ops node:$A, node:$B, node:$C),
   return N->hasOneUse();
 }]>;
 
+def riscv_fpextend_vl_sameuser : PatFrag<(ops node:$A, node:$B, node:$C),
+                                 (riscv_fpextend_vl node:$A, node:$B, node:$C), [{
+  return !N->use_empty() && all_equal(N->users());
+}]>;
+
 def riscv_vfmadd_vl_oneuse : PatFrag<(ops node:$A, node:$B, node:$C, node:$D,
                                           node:$E),
                                      (riscv_vfmadd_vl node:$A, node:$B,
@@ -1052,32 +1057,8 @@ multiclass VPatIntegerSetCCVL_VX_Swappable<VTypeInfo vti, string instruction_nam
 }
 
 multiclass VPatIntegerSetCCVL_VI_Swappable<VTypeInfo vti, string instruction_name,
-                                           CondCode cc, CondCode invcc> {
-  defvar instruction_masked = !cast<Instruction>(instruction_name#"_VI_"#vti.LMul.MX#"_MASK");
-  def : Pat<(vti.Mask (riscv_setcc_vl (vti.Vector vti.RegClass:$rs1),
-                                      (SplatPat_simm5 simm5:$rs2), cc,
-                                      VR:$passthru,
-                                      (vti.Mask V0),
-                                      VLOpFrag)),
-            (instruction_masked VR:$passthru, vti.RegClass:$rs1,
-                                XLenVT:$rs2, (vti.Mask V0), GPR:$vl,
-                                vti.Log2SEW)>;
-
-  // FIXME: Can do some canonicalization to remove these patterns.
-  def : Pat<(vti.Mask (riscv_setcc_vl (SplatPat_simm5 simm5:$rs2),
-                                      (vti.Vector vti.RegClass:$rs1), invcc,
-                                      VR:$passthru,
-                                      (vti.Mask V0),
-                                      VLOpFrag)),
-            (instruction_masked VR:$passthru, vti.RegClass:$rs1,
-                                simm5:$rs2, (vti.Mask V0), GPR:$vl,
-                                vti.Log2SEW)>;
-}
-
-multiclass VPatIntegerSetCCVL_VIPlus1_Swappable<VTypeInfo vti,
-                                                string instruction_name,
-                                                CondCode cc, CondCode invcc,
-                                                ComplexPattern splatpat_kind> {
+                                           CondCode cc, CondCode invcc,
+                                           ComplexPattern splatpat_kind = SplatPat_simm5> {
   defvar instruction_masked = !cast<Instruction>(instruction_name#"_VI_"#vti.LMul.MX#"_MASK");
   def : Pat<(vti.Mask (riscv_setcc_vl (vti.Vector vti.RegClass:$rs1),
                                       (splatpat_kind simm5:$rs2), cc,
@@ -1085,7 +1066,7 @@ multiclass VPatIntegerSetCCVL_VIPlus1_Swappable<VTypeInfo vti,
                                       (vti.Mask V0),
                                       VLOpFrag)),
             (instruction_masked VR:$passthru, vti.RegClass:$rs1,
-                                (DecImm simm5:$rs2), (vti.Mask V0), GPR:$vl,
+                                XLenVT:$rs2, (vti.Mask V0), GPR:$vl,
                                 vti.Log2SEW)>;
 
   // FIXME: Can do some canonicalization to remove these patterns.
@@ -1095,7 +1076,7 @@ multiclass VPatIntegerSetCCVL_VIPlus1_Swappable<VTypeInfo vti,
                                       (vti.Mask V0),
                                       VLOpFrag)),
             (instruction_masked VR:$passthru, vti.RegClass:$rs1,
-                                (DecImm simm5:$rs2), (vti.Mask V0), GPR:$vl,
+                                simm5:$rs2, (vti.Mask V0), GPR:$vl,
                                 vti.Log2SEW)>;
 }
 
@@ -1436,29 +1417,6 @@ multiclass VPatWidenReductionVL<SDNode vop, PatFrags extop, string instruction_n
   }
 }
 
-multiclass VPatWidenReductionVL_RM<SDNode vop, PatFrags extop, string instruction_name, bit is_float> {
-  foreach vtiToWti = !if(is_float, AllWidenableFloatVectors, AllWidenableIntVectors) in {
-    defvar vti = vtiToWti.Vti;
-    defvar wti = vtiToWti.Wti;
-    defvar wti_m1 = !cast<VTypeInfo>(!if(is_float, "VF", "VI") # wti.SEW # "M1");
-    let Predicates = !listconcat(GetVTypePredicates<vti>.Predicates,
-                                 GetVTypePredicates<wti>.Predicates) in {
-      def: Pat<(wti_m1.Vector (vop (wti_m1.Vector VR:$passthru),
-                                   (wti.Vector (extop (vti.Vector vti.RegClass:$rs1))),
-                                   VR:$rs2, (vti.Mask V0), VLOpFrag,
-                                   (XLenVT timm:$policy))),
-               (!cast<Instruction>(instruction_name#"_VS_"#vti.LMul.MX#"_E"#vti.SEW#"_MASK")
-                  (wti_m1.Vector VR:$passthru), (vti.Vector vti.RegClass:$rs1),
-                  (wti_m1.Vector VR:$rs2), (vti.Mask V0),
-                  // Value to indicate no rounding mode change in
-                  // RISCVInsertReadWriteCSR
-                  FRM_DYN,
-                  GPR:$vl, vti.Log2SEW,
-                  (XLenVT timm:$policy))>;
-    }
-  }
-}
-
 multiclass VPatWidenReductionVL_Ext_VL<SDNode vop, PatFrags extop, string instruction_name, bit is_float> {
   foreach vtiToWti = !if(is_float, AllWidenableFloatVectors, AllWidenableIntVectors) in {
     defvar vti = vtiToWti.Vti;
@@ -1467,7 +1425,7 @@ multiclass VPatWidenReductionVL_Ext_VL<SDNode vop, PatFrags extop, string instru
     let Predicates = !listconcat(GetVTypePredicates<vti>.Predicates,
                                  GetVTypePredicates<wti>.Predicates) in {
       def: Pat<(wti_m1.Vector (vop (wti_m1.Vector VR:$passthru),
-                                   (wti.Vector (extop (vti.Vector vti.RegClass:$rs1), (vti.Mask true_mask), VLOpFrag)),
+                                   (wti.Vector (extop (vti.Vector vti.RegClass:$rs1), (vti.Mask true_mask), (XLenVT srcvalue))),
                                    VR:$rs2, (vti.Mask V0), VLOpFrag,
                                    (XLenVT timm:$policy))),
                (!cast<Instruction>(instruction_name#"_VS_"#vti.LMul.MX#"_E"#vti.SEW#"_MASK")
@@ -1486,7 +1444,7 @@ multiclass VPatWidenReductionVL_Ext_VL_RM<SDNode vop, PatFrags extop, string ins
     let Predicates = !listconcat(GetVTypePredicates<vti>.Predicates,
                                  GetVTypePredicates<wti>.Predicates) in {
       def: Pat<(wti_m1.Vector (vop (wti_m1.Vector VR:$passthru),
-                                   (wti.Vector (extop (vti.Vector vti.RegClass:$rs1), (vti.Mask true_mask), VLOpFrag)),
+                                   (wti.Vector (extop (vti.Vector vti.RegClass:$rs1), (vti.Mask true_mask), (XLenVT srcvalue))),
                                    VR:$rs2, (vti.Mask V0), VLOpFrag,
                                    (XLenVT timm:$policy))),
                (!cast<Instruction>(instruction_name#"_VS_"#vti.LMul.MX#"_E"#vti.SEW#"_MASK")
@@ -2173,14 +2131,14 @@ foreach vti = AllIntegerVectors in {
     defm : VPatIntegerSetCCVL_VI_Swappable<vti, "PseudoVMSGT",  SETGT,  SETLT>;
     defm : VPatIntegerSetCCVL_VI_Swappable<vti, "PseudoVMSGTU", SETUGT, SETULT>;
 
-    defm : VPatIntegerSetCCVL_VIPlus1_Swappable<vti, "PseudoVMSLE",  SETLT, SETGT,
-                                                SplatPat_simm5_plus1>;
-    defm : VPatIntegerSetCCVL_VIPlus1_Swappable<vti, "PseudoVMSLEU", SETULT, SETUGT,
-                                                SplatPat_simm5_plus1_nonzero>;
-    defm : VPatIntegerSetCCVL_VIPlus1_Swappable<vti, "PseudoVMSGT",  SETGE, SETLE,
-                                                SplatPat_simm5_plus1>;
-    defm : VPatIntegerSetCCVL_VIPlus1_Swappable<vti, "PseudoVMSGTU", SETUGE, SETULE,
-                                                SplatPat_simm5_plus1_nonzero>;
+    defm : VPatIntegerSetCCVL_VI_Swappable<vti, "PseudoVMSLE",  SETLT, SETGT,
+                                           SplatPat_simm5_plus1>;
+    defm : VPatIntegerSetCCVL_VI_Swappable<vti, "PseudoVMSLEU", SETULT, SETUGT,
+                                           SplatPat_simm5_plus1_nonzero>;
+    defm : VPatIntegerSetCCVL_VI_Swappable<vti, "PseudoVMSGT",  SETGE, SETLE,
+                                           SplatPat_simm5_plus1>;
+    defm : VPatIntegerSetCCVL_VI_Swappable<vti, "PseudoVMSGTU", SETUGE, SETULE,
+                                           SplatPat_simm5_plus1_nonzero>;
   }
 } // foreach vti = AllIntegerVectors
 
@@ -2677,13 +2635,9 @@ defm : VPatReductionVL<rvv_vecreduce_FMIN_vl,     "PseudoVFREDMIN", is_float=1>;
 defm : VPatReductionVL<rvv_vecreduce_FMAX_vl,     "PseudoVFREDMAX", is_float=1>;
 
 // 14.4. Vector Widening Floating-Point Reduction Instructions
-defm : VPatWidenReductionVL_RM<rvv_vecreduce_SEQ_FADD_vl, fpext_oneuse,
-                               "PseudoVFWREDOSUM", is_float=1>;
 defm : VPatWidenReductionVL_Ext_VL_RM<rvv_vecreduce_SEQ_FADD_vl,
                                       riscv_fpextend_vl_oneuse,
                                       "PseudoVFWREDOSUM", is_float=1>;
-defm : VPatWidenReductionVL_RM<rvv_vecreduce_FADD_vl, fpext_oneuse,
-                               "PseudoVFWREDUSUM", is_float=1>;
 defm : VPatWidenReductionVL_Ext_VL_RM<rvv_vecreduce_FADD_vl,
                                       riscv_fpextend_vl_oneuse,
                                       "PseudoVFWREDUSUM", is_float=1>;
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoXCV.td b/llvm/lib/Target/RISCV/RISCVInstrInfoXCV.td
index b98934d8c6396..bade4863ad348 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfoXCV.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfoXCV.td
@@ -514,6 +514,7 @@ def CVrrAsmOperand : AsmOperandClass {
   let Name = "RegReg";
   let ParserMethod = "parseRegReg";
   let DiagnosticType = "InvalidRegReg";
+  let DiagnosticString = "operands must be register and register";
 }
 
 def CVrr : Operand<i32>,
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoXMips.td b/llvm/lib/Target/RISCV/RISCVInstrInfoXMips.td
new file mode 100644
index 0000000000000..9be424310d660
--- /dev/null
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfoXMips.td
@@ -0,0 +1,149 @@
+//===-- RISCVInstrInfoXMips.td -----------------------------*- tablegen -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file describes the vendor extensions defined by MIPS.
+//
+//===----------------------------------------------------------------------===//
+
+//===----------------------------------------------------------------------===//
+// Operand definitions.
+//===----------------------------------------------------------------------===//
+
+// A 7-bit unsigned immediate where the least significant three bits are zero.
+def uimm7_lsb000 : RISCVOp,
+                   ImmLeaf<XLenVT, [{return isShiftedUInt<4, 3>(Imm);}]> {
+  let ParserMatchClass = UImmAsmOperand<7, "Lsb000">;
+  let EncoderMethod = "getImmOpValue";
+  let DecoderMethod = "decodeUImmOperand<7>";
+  let OperandType = "OPERAND_UIMM7_LSB000";
+  let MCOperandPredicate = [{
+    int64_t Imm;
+    if (!MCOp.evaluateAsConstantImm(Imm))
+      return false;
+    return isShiftedUInt<4, 3>(Imm);
+  }];
+}
+
+//===----------------------------------------------------------------------===//
+// MIPS custom instruction formats
+//===----------------------------------------------------------------------===//
+
+// Load double pair format.
+class LDPFormat<dag outs, dag ins, string opcodestr, string argstr>
+    : RVInst<outs, ins, opcodestr, argstr, [], InstFormatI> {
+  bits<7> imm7;
+  bits<5> rs1;
+  bits<5> rd1;
+  bits<5> rd2;
+
+  let Inst{31-27} = rd2;
+  let Inst{26-23} = imm7{6-3};
+  let Inst{22-20} = 0b000;
+  let Inst{19-15} = rs1;
+  let Inst{14-12} = 0b100;
+  let Inst{11-7} = rd1;
+  let Inst{6-0} = OPC_CUSTOM_0.Value;
+}
+
+// Load word pair format.
+class LWPFormat<dag outs, dag ins, string opcodestr, string argstr>
+    : RVInst<outs, ins, opcodestr, argstr, [], InstFormatI> {
+  bits<7> imm7;
+  bits<5> rs1;
+  bits<5> rd1;
+  bits<5> rd2;
+
+  let Inst{31-27} = rd2;
+  let Inst{26-22} = imm7{6-2};
+  let Inst{21-20} = 0b01;
+  let Inst{19-15} = rs1;
+  let Inst{14-12} = 0b100;
+  let Inst{11-7} = rd1;
+  let Inst{6-0} = OPC_CUSTOM_0.Value;
+}
+
+// Store double pair format.
+class SDPFormat<dag outs, dag ins, string opcodestr, string argstr>
+    : RVInst<outs, ins, opcodestr, argstr, [], InstFormatI> {
+  bits<7> imm7;
+  bits<5> rs3;
+  bits<5> rs2;
+  bits<5> rs1;
+
+  let Inst{31-27} = rs3;
+  let Inst{26-25} = imm7{6-5};
+  let Inst{24-20} = rs2;
+  let Inst{19-15} = rs1;
+  let Inst{14-12} = 0b101;
+  let Inst{11-10} = imm7{4-3};
+  let Inst{9-7} = 0b000;
+  let Inst{6-0} = OPC_CUSTOM_0.Value;
+}
+
+// Store word pair format.
+class SWPFormat<dag outs, dag ins, string opcodestr, string argstr>
+    : RVInst<outs, ins, opcodestr, argstr, [], InstFormatI> {
+  bits<7> imm7;
+  bits<5> rs3;
+  bits<5> rs2;
+  bits<5> rs1;
+
+  let Inst{31-27} = rs3;
+  let Inst{26-25} = imm7{6-5};
+  let Inst{24-20} = rs2;
+  let Inst{19-15} = rs1;
+  let Inst{14-12} = 0b101;
+  let Inst{11-9} = imm7{4-2};
+  let Inst{8-7} = 0b01;
+  let Inst{6-0} = OPC_CUSTOM_0.Value;
+}
+
+//===----------------------------------------------------------------------===//
+// MIPS extensions
+//===----------------------------------------------------------------------===//
+
+let Predicates = [HasVendorXMIPSCMove], hasSideEffects = 0, mayLoad = 0, mayStore = 0,
+                 DecoderNamespace = "Xmipscmove" in {
+def MIPS_CCMOV : RVInstR4<0b11, 0b011, OPC_CUSTOM_0, (outs GPR:$rd),
+                          (ins GPR:$rs1, GPR:$rs2, GPR:$rs3),
+                          "mips.ccmov", "$rd, $rs2, $rs1, $rs3">,
+                 Sched<[]>;
+}
+
+let Predicates = [UseCCMovInsn] in {
+def : Pat<(select (riscv_setne (XLenVT GPR:$rs2)),
+                  (XLenVT GPR:$rs1), (XLenVT GPR:$rs3)),
+          (MIPS_CCMOV GPR:$rs1, GPR:$rs2, GPR:$rs3)>;
+def : Pat<(select (riscv_seteq (XLenVT GPR:$rs2)),
+                  (XLenVT GPR:$rs3), (XLenVT GPR:$rs1)),
+          (MIPS_CCMOV GPR:$rs1, GPR:$rs2, GPR:$rs3)>;
+
+def : Pat<(select (XLenVT GPR:$rs2), (XLenVT GPR:$rs1), (XLenVT GPR:$rs3)),
+          (MIPS_CCMOV GPR:$rs1, GPR:$rs2, GPR:$rs3)>;
+}
+
+let Predicates = [HasVendorXMIPSLSP], hasSideEffects = 0,
+                 DecoderNamespace = "Xmipslsp" in {
+let mayLoad = 1, mayStore = 0 in {
+def MIPS_LWP : LWPFormat<(outs GPR:$rd1, GPR:$rd2), (ins GPR:$rs1, uimm7_lsb00:$imm7),
+                         "mips.lwp", "$rd1, $rd2, ${imm7}(${rs1})">,
+               Sched<[WriteLDW, WriteLDW, ReadMemBase]>;
+def MIPS_LDP : LDPFormat<(outs GPR:$rd1, GPR:$rd2), (ins GPR:$rs1, uimm7_lsb000:$imm7),
+                         "mips.ldp", "$rd1, $rd2, ${imm7}(${rs1})">,
+               Sched<[WriteLDD, WriteLDD, ReadMemBase]>;
+} // mayLoad = 1, mayStore = 0
+
+let mayLoad = 0, mayStore = 1 in {
+def MIPS_SWP : SWPFormat<(outs), (ins GPR:$rs2, GPR:$rs3, GPR:$rs1, uimm7_lsb00:$imm7),
+                         "mips.swp", "$rs2, $rs3, ${imm7}(${rs1})">,
+               Sched<[WriteSTW, ReadStoreData, ReadStoreData, ReadMemBase]>;
+def MIPS_SDP : SDPFormat<(outs), (ins GPR:$rs2, GPR:$rs3, GPR:$rs1, uimm7_lsb000:$imm7),
+                         "mips.sdp", "$rs2, $rs3, ${imm7}(${rs1})">,
+               Sched<[WriteSTD, ReadStoreData, ReadStoreData, ReadMemBase]>;
+} // mayLoad = 0, mayStore = 1
+} // Predicates = [HasVendorXMIPSLSP], hasSideEffects = 0, DecoderNamespace = "Xmipslsp"
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoXqci.td b/llvm/lib/Target/RISCV/RISCVInstrInfoXqci.td
index f746cce8c9a0f..1f042b0f47e96 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfoXqci.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfoXqci.td
@@ -299,9 +299,9 @@ let hasSideEffects = 0, mayLoad = 0, mayStore = 0 in {
 
 let Predicates = [HasVendorXqciac, IsRV32], DecoderNamespace = "Xqciac" in {
 let hasSideEffects = 0, mayLoad = 0, mayStore = 0 in {
-  def QC_C_MULADDI : RVInst16CL<0b001, 0b10, (outs GPRC:$rd_wb),
+  def QC_C_MULIADD : RVInst16CL<0b001, 0b10, (outs GPRC:$rd_wb),
                                (ins GPRC:$rd, GPRC:$rs1, uimm5:$uimm),
-                               "qc.c.muladdi", "$rd, $rs1, $uimm"> {
+                               "qc.c.muliadd", "$rd, $rs1, $uimm"> {
     let Constraints = "$rd = $rd_wb";
     bits<5> uimm;
 
@@ -310,9 +310,9 @@ let hasSideEffects = 0, mayLoad = 0, mayStore = 0 in {
     let Inst{5} = uimm{4};
   }
 
-  def QC_MULADDI : RVInstI<0b110, OPC_CUSTOM_0, (outs GPRNoX0:$rd_wb),
+  def QC_MULIADD : RVInstI<0b110, OPC_CUSTOM_0, (outs GPRNoX0:$rd_wb),
                            (ins GPRNoX0:$rd, GPRNoX0:$rs1, simm12:$imm12),
-                           "qc.muladdi", "$rd, $rs1, $imm12"> {
+                           "qc.muliadd", "$rd, $rs1, $imm12"> {
     let Constraints = "$rd = $rd_wb";
   }
 
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoZc.td b/llvm/lib/Target/RISCV/RISCVInstrInfoZc.td
index 5cc16765d4ae2..9dfbcf678d6eb 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfoZc.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfoZc.td
@@ -39,12 +39,15 @@ def RlistAsmOperand : AsmOperandClass {
   let Name = "Rlist";
   let ParserMethod = "parseReglist";
   let DiagnosticType = "InvalidRlist";
+  let DiagnosticString = "operand must be {ra [, s0[-sN]]} or {x1 [, x8[-x9][, x18[-xN]]]}";
 }
 
 def StackAdjAsmOperand : AsmOperandClass {
   let Name = "StackAdj";
   let ParserMethod = "parseZcmpStackAdj";
   let DiagnosticType = "InvalidStackAdj";
+  let DiagnosticString = "stack adjustment is invalid for this instruction and register list; "
+        "refer to Zc spec for a detailed range of stack adjustment";
   let PredicateMethod = "isSpimm";
   let RenderMethod = "addSpimmOperands";
 }
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoZfa.td b/llvm/lib/Target/RISCV/RISCVInstrInfoZfa.td
index ab54b45f4de93..a539ca82b7462 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfoZfa.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfoZfa.td
@@ -30,6 +30,7 @@ def LoadFPImmOperand : AsmOperandClass {
   let ParserMethod = "parseFPImm";
   let RenderMethod = "addFPImmOperands";
   let DiagnosticType = "InvalidLoadFPImm";
+  let DiagnosticString = "operand must be a valid floating-point constant";
 }
 
 def loadfpimm : Operand<XLenVT> {
@@ -43,6 +44,7 @@ def RTZArg : AsmOperandClass {
   let Name = "RTZArg";
   let RenderMethod = "addFRMArgOperands";
   let DiagnosticType = "InvalidRTZArg";
+  let DiagnosticString = "operand must be 'rtz' floating-point rounding mode";
   let ParserMethod = "parseFRMArg";
 }
 
diff --git a/llvm/lib/Target/RISCV/RISCVMakeCompressible.cpp b/llvm/lib/Target/RISCV/RISCVMakeCompressible.cpp
index df5501e37f831..5453753fa4579 100644
--- a/llvm/lib/Target/RISCV/RISCVMakeCompressible.cpp
+++ b/llvm/lib/Target/RISCV/RISCVMakeCompressible.cpp
@@ -17,7 +17,7 @@
 // For case 1, if a compressed register is available, then the uncompressed
 // register is copied to the compressed register and its uses are replaced.
 //
-// For example, storing zero uses the uncompressible zero register:
+// For example, storing zero uses the incompressible zero register:
 //   sw zero, 0(a0)   # if zero
 //   sw zero, 8(a0)   # if zero
 //   sw zero, 4(a0)   # if zero
@@ -275,7 +275,7 @@ static RegImmPair getRegImmPairPreventingCompression(const MachineInstr &MI) {
       // rather than used.
       //
       // For stores, we can change SrcDest (and Base if SrcDest == Base) but
-      // cannot resolve an uncompressible offset in this case.
+      // cannot resolve an incompressible offset in this case.
       if (isCompressibleStore(MI)) {
         if (!SrcDestCompressed && (BaseCompressed || SrcDest == Base) &&
             !NewBaseAdjust)
@@ -313,7 +313,7 @@ static Register analyzeCompressibleUses(MachineInstr &FirstMI,
     // If RegImm.Reg is modified by this instruction, then we cannot optimize
     // past this instruction. If the register is already compressed, then it may
     // possible to optimize a large offset in the current instruction - this
-    // will have been detected by the preceeding call to
+    // will have been detected by the preceding call to
     // getRegImmPairPreventingCompression.
     if (MI.modifiesRegister(RegImm.Reg, TRI))
       break;
@@ -409,7 +409,7 @@ bool RISCVMakeCompressibleOpt::runOnMachineFunction(MachineFunction &Fn) {
     LLVM_DEBUG(dbgs() << "MBB: " << MBB.getName() << "\n");
     for (MachineInstr &MI : MBB) {
       // Determine if this instruction would otherwise be compressed if not for
-      // an uncompressible register or offset.
+      // an incompressible register or offset.
       RegImmPair RegImm = getRegImmPairPreventingCompression(MI);
       if (!RegImm.Reg && RegImm.Imm == 0)
         continue;
diff --git a/llvm/lib/Target/RISCV/RISCVMergeBaseOffset.cpp b/llvm/lib/Target/RISCV/RISCVMergeBaseOffset.cpp
index a324deb4e48f5..bbbb1e1595982 100644
--- a/llvm/lib/Target/RISCV/RISCVMergeBaseOffset.cpp
+++ b/llvm/lib/Target/RISCV/RISCVMergeBaseOffset.cpp
@@ -434,8 +434,8 @@ bool RISCVMergeBaseOffsetOpt::foldIntoMemoryOps(MachineInstr &Hi,
 
         // Memory constraints have two operands.
         if (NumOps != 2 || !Flags.isMemKind()) {
-          // If the register is used by something other than a memory contraint,
-          // we should not fold.
+          // If the register is used by something other than a memory
+          // constraint, we should not fold.
           for (unsigned J = 0; J < NumOps; ++J) {
             const MachineOperand &MO = UseMI.getOperand(I + 1 + J);
             if (MO.isReg() && MO.getReg() == DestReg)
diff --git a/llvm/lib/Target/RISCV/RISCVOptWInstrs.cpp b/llvm/lib/Target/RISCV/RISCVOptWInstrs.cpp
index effec2cc776d8..28bee83837654 100644
--- a/llvm/lib/Target/RISCV/RISCVOptWInstrs.cpp
+++ b/llvm/lib/Target/RISCV/RISCVOptWInstrs.cpp
@@ -326,6 +326,7 @@ static bool hasAllNBitUsers(const MachineInstr &OrigMI,
         break;
 
       case RISCV::PseudoCCMOVGPR:
+      case RISCV::PseudoCCMOVGPRNoX0:
         // Either operand 4 or operand 5 is returned by this instruction. If
         // only the lower word of the result is used, then only the lower word
         // of operand 4 and 5 is used.
@@ -538,6 +539,7 @@ static bool isSignExtendedW(Register SrcReg, const RISCVSubtarget &ST,
     case RISCV::MIN:
     case RISCV::MINU:
     case RISCV::PseudoCCMOVGPR:
+    case RISCV::PseudoCCMOVGPRNoX0:
     case RISCV::PseudoCCAND:
     case RISCV::PseudoCCOR:
     case RISCV::PseudoCCXOR:
@@ -546,7 +548,7 @@ static bool isSignExtendedW(Register SrcReg, const RISCVSubtarget &ST,
       // MIN, MAX, or PHI is also sign-extended.
 
       // The input registers for PHI are operand 1, 3, ...
-      // The input registers for PseudoCCMOVGPR are 4 and 5.
+      // The input registers for PseudoCCMOVGPR(NoX0) are 4 and 5.
       // The input registers for PseudoCCAND/OR/XOR are 4, 5, and 6.
       // The input registers for others are operand 1 and 2.
       unsigned B = 1, E = 3, D = 1;
@@ -556,6 +558,7 @@ static bool isSignExtendedW(Register SrcReg, const RISCVSubtarget &ST,
         D = 2;
         break;
       case RISCV::PseudoCCMOVGPR:
+      case RISCV::PseudoCCMOVGPRNoX0:
         B = 4;
         E = 6;
         break;
diff --git a/llvm/lib/Target/RISCV/RISCVProcessors.td b/llvm/lib/Target/RISCV/RISCVProcessors.td
index 6dfed7ddeb9f6..b5eea138732a5 100644
--- a/llvm/lib/Target/RISCV/RISCVProcessors.td
+++ b/llvm/lib/Target/RISCV/RISCVProcessors.td
@@ -116,7 +116,9 @@ def MIPS_P8700 : RISCVProcessorModel<"mips-p8700",
                                       FeatureStdExtZba,
                                       FeatureStdExtZbb,
                                       FeatureStdExtZifencei,
-                                      FeatureStdExtZicsr],
+                                      FeatureStdExtZicsr,
+                                      FeatureVendorXMIPSCMove,
+                                      FeatureVendorXMIPSLSP],
                                      [TuneMIPSP8700]>;
 
 def ROCKET_RV32 : RISCVProcessorModel<"rocket-rv32",
@@ -327,7 +329,7 @@ defvar SiFiveP500TuneFeatures = [TuneNoDefaultUnroll,
                                  TuneAUIPCADDIFusion,
                                  TunePostRAScheduler];
 
-def SIFIVE_P550 : RISCVProcessorModel<"sifive-p550", NoSchedModel,
+def SIFIVE_P550 : RISCVProcessorModel<"sifive-p550", SiFiveP500Model,
                                       [Feature64Bit,
                                        FeatureStdExtI,
                                        FeatureStdExtZifencei,
diff --git a/llvm/lib/Target/RISCV/RISCVRegisterInfo.cpp b/llvm/lib/Target/RISCV/RISCVRegisterInfo.cpp
index b9c70fe60fb50..12d54313a96ab 100644
--- a/llvm/lib/Target/RISCV/RISCVRegisterInfo.cpp
+++ b/llvm/lib/Target/RISCV/RISCVRegisterInfo.cpp
@@ -55,6 +55,11 @@ RISCVRegisterInfo::RISCVRegisterInfo(unsigned HwMode)
     : RISCVGenRegisterInfo(RISCV::X1, /*DwarfFlavour*/0, /*EHFlavor*/0,
                            /*PC*/0, HwMode) {}
 
+const MCPhysReg *
+RISCVRegisterInfo::getIPRACSRegs(const MachineFunction *MF) const {
+  return CSR_IPRA_SaveList;
+}
+
 const MCPhysReg *
 RISCVRegisterInfo::getCalleeSavedRegs(const MachineFunction *MF) const {
   auto &Subtarget = MF->getSubtarget<RISCVSubtarget>();
@@ -281,7 +286,7 @@ void RISCVRegisterInfo::adjustReg(MachineBasicBlock &MBB,
   // instruction.  This saves 1 instruction over the full lui/addi+add fallback
   // path.  We avoid anything which can be done with a single lui as it might
   // be compressible.  Note that the sh1add case is fully covered by the 2x addi
-  // case just above and is thus ommitted.
+  // case just above and is thus omitted.
   if (ST.hasStdExtZba() && (Val & 0xFFF) != 0) {
     unsigned Opc = 0;
     if (isShiftedInt<12, 3>(Val)) {
@@ -926,6 +931,26 @@ bool RISCVRegisterInfo::getRegAllocationHints(
         tryAddHint(MO, MI.getOperand(0), NeedGPRC);
       }
     }
+
+    // Add a hint if it would allow auipc/lui+addi(w) fusion.
+    if ((MI.getOpcode() == RISCV::ADDIW || MI.getOpcode() == RISCV::ADDI) &&
+        MI.getOperand(1).isReg()) {
+      const MachineBasicBlock &MBB = *MI.getParent();
+      MachineBasicBlock::const_iterator I = MI.getIterator();
+      // Is the previous instruction a LUI or AUIPC that can be fused?
+      if (I != MBB.begin()) {
+        I = skipDebugInstructionsBackward(std::prev(I), MBB.begin());
+        if (((I->getOpcode() == RISCV::LUI && Subtarget.hasLUIADDIFusion()) ||
+             (I->getOpcode() == RISCV::AUIPC &&
+              Subtarget.hasAUIPCADDIFusion())) &&
+            I->getOperand(0).getReg() == MI.getOperand(1).getReg()) {
+          if (OpIdx == 0)
+            tryAddHint(MO, MI.getOperand(1), /*NeedGPRC=*/false);
+          else
+            tryAddHint(MO, MI.getOperand(0), /*NeedGPRC=*/false);
+        }
+      }
+    }
   }
 
   for (MCPhysReg OrderReg : Order)
diff --git a/llvm/lib/Target/RISCV/RISCVRegisterInfo.h b/llvm/lib/Target/RISCV/RISCVRegisterInfo.h
index 3ab79694e175c..6c4e9c7b1bdc7 100644
--- a/llvm/lib/Target/RISCV/RISCVRegisterInfo.h
+++ b/llvm/lib/Target/RISCV/RISCVRegisterInfo.h
@@ -62,6 +62,8 @@ struct RISCVRegisterInfo : public RISCVGenRegisterInfo {
 
   const MCPhysReg *getCalleeSavedRegs(const MachineFunction *MF) const override;
 
+  const MCPhysReg *getIPRACSRegs(const MachineFunction *MF) const override;
+
   BitVector getReservedRegs(const MachineFunction &MF) const override;
   bool isAsmClobberable(const MachineFunction &MF,
                         MCRegister PhysReg) const override;
diff --git a/llvm/lib/Target/RISCV/RISCVRegisterInfo.td b/llvm/lib/Target/RISCV/RISCVRegisterInfo.td
index 7eb93973459c0..a5dfb5ba1a2fc 100644
--- a/llvm/lib/Target/RISCV/RISCVRegisterInfo.td
+++ b/llvm/lib/Target/RISCV/RISCVRegisterInfo.td
@@ -247,12 +247,26 @@ def GPR : GPRRegisterClass<(add (sequence "X%u", 10, 17),
                                 (sequence "X%u", 0, 4))>;
 
 def GPRX0 : GPRRegisterClass<(add X0)>;
-def GPRX1 : GPRRegisterClass<(add X1)>;
-def GPRX5 : GPRRegisterClass<(add X5)>;
 
-def GPRNoX0 : GPRRegisterClass<(sub GPR, X0)>;
+def GPRX1 : GPRRegisterClass<(add X1)> {
+  let DiagnosticType = "InvalidRegClassGPRX1";
+  let DiagnosticString = "register must be ra (x1)";
+}
+
+def GPRX5 : GPRRegisterClass<(add X5)> {
+  let DiagnosticType = "InvalidRegClassGPRX5";
+  let DiagnosticString = "register must be t0 (x5)";
+}
+
+def GPRNoX0 : GPRRegisterClass<(sub GPR, X0)> {
+  let DiagnosticType = "InvalidRegClassGPRX0";
+  let DiagnosticString = "register must be a GPR excluding zero (x0)";
+}
 
-def GPRNoX0X2 : GPRRegisterClass<(sub GPR, X0, X2)>;
+def GPRNoX0X2 : GPRRegisterClass<(sub GPR, X0, X2)> {
+  let DiagnosticType = "InvalidRegClassGPRNoX0X2";
+  let DiagnosticString = "register must be a GPR excluding zero (x0) and sp (x2)";
+}
 
 def GPRX7 : GPRRegisterClass<(add X7)>;
 
@@ -276,13 +290,19 @@ def GPRTC : GPRRegisterClass<(add (sequence "X%u", 6, 7),
                                   (sequence "X%u", 28, 31))>;
 def GPRTCNonX7 : GPRRegisterClass<(sub GPRTC, X7)>;
 
-def SP : GPRRegisterClass<(add X2)>;
+def SP : GPRRegisterClass<(add X2)> {
+  let DiagnosticType = "InvalidRegClassSP";
+  let DiagnosticString = "register must be sp (x2)";
+}
 
 // Saved Registers from s0 to s7, for C.MVA01S07 instruction in Zcmp extension
 def SR07 : GPRRegisterClass<(add (sequence "X%u", 8, 9),
                                  (sequence "X%u", 18, 23))>;
 
-def GPRX1X5 :  GPRRegisterClass<(add X1, X5)>;
+def GPRX1X5 :  GPRRegisterClass<(add X1, X5)> {
+  let DiagnosticType = "InvalidRegClassGPRX1X5";
+  let DiagnosticString = "register must be ra or t0 (x1 or x5)";
+}
 
 //===----------------------------------------------------------------------===//
 // Even-Odd GPR Pairs
diff --git a/llvm/lib/Target/RISCV/RISCVSchedSiFive7.td b/llvm/lib/Target/RISCV/RISCVSchedSiFive7.td
index 32fddeead3449..f4d2073d3b52d 100644
--- a/llvm/lib/Target/RISCV/RISCVSchedSiFive7.td
+++ b/llvm/lib/Target/RISCV/RISCVSchedSiFive7.td
@@ -159,7 +159,7 @@ class SiFive7GetReductionCycles<string mx, int sew> {
   );
 }
 
-/// Cycles for ordered reductions take approximatley 6*VL cycles
+/// Cycles for ordered reductions take approximately 6*VL cycles
 class SiFive7GetOrderedReductionCycles<string mx, int sew> {
   defvar VLEN = 512;
   // (VLEN * LMUL) / SEW
@@ -224,7 +224,7 @@ def SiFive7VS          : ProcResource<1>; // Store sequencer
 // vector unit is ready to dequeue them. The unit dequeues up to one instruction
 // per cycle, in order, as soon as the sequencer for that type of instruction is
 // available. This resource is meant to be used for 1 cycle by all vector
-// instructions, to model that only one vector instruction may be dequed at a
+// instructions, to model that only one vector instruction may be dequeued at a
 // time. The actual dequeueing into the sequencer is modeled by the VA, VL, and
 // VS sequencer resources below. Each of them will only accept a single
 // instruction at a time and remain busy for the number of cycles associated
@@ -634,15 +634,8 @@ foreach mx = SchedMxList in {
     defm "" : LMULWriteResMX<"WriteVICALUMV",   [SiFive7VCQ, SiFive7VA], mx, IsWorstCase>;
     defm "" : LMULWriteResMX<"WriteVICALUMX",   [SiFive7VCQ, SiFive7VA], mx, IsWorstCase>;
     defm "" : LMULWriteResMX<"WriteVICALUMI",   [SiFive7VCQ, SiFive7VA], mx, IsWorstCase>;
-    defm "" : LMULWriteResMX<"WriteVShiftV",    [SiFive7VCQ, SiFive7VA], mx, IsWorstCase>;
-    defm "" : LMULWriteResMX<"WriteVShiftX",    [SiFive7VCQ, SiFive7VA], mx, IsWorstCase>;
-    defm "" : LMULWriteResMX<"WriteVShiftI",    [SiFive7VCQ, SiFive7VA], mx, IsWorstCase>;
     defm "" : LMULWriteResMX<"WriteVIMinMaxV",  [SiFive7VCQ, SiFive7VA], mx, IsWorstCase>;
     defm "" : LMULWriteResMX<"WriteVIMinMaxX",  [SiFive7VCQ, SiFive7VA], mx, IsWorstCase>;
-    defm "" : LMULWriteResMX<"WriteVIMulV",     [SiFive7VCQ, SiFive7VA], mx, IsWorstCase>;
-    defm "" : LMULWriteResMX<"WriteVIMulX",     [SiFive7VCQ, SiFive7VA], mx, IsWorstCase>;
-    defm "" : LMULWriteResMX<"WriteVIMulAddV",  [SiFive7VCQ, SiFive7VA], mx, IsWorstCase>;
-    defm "" : LMULWriteResMX<"WriteVIMulAddX",  [SiFive7VCQ, SiFive7VA], mx, IsWorstCase>;
     defm "" : LMULWriteResMX<"WriteVIMergeV",   [SiFive7VCQ, SiFive7VA], mx, IsWorstCase>;
     defm "" : LMULWriteResMX<"WriteVIMergeX",   [SiFive7VCQ, SiFive7VA], mx, IsWorstCase>;
     defm "" : LMULWriteResMX<"WriteVIMergeI",   [SiFive7VCQ, SiFive7VA], mx, IsWorstCase>;
@@ -650,6 +643,15 @@ foreach mx = SchedMxList in {
     defm "" : LMULWriteResMX<"WriteVIMovX",     [SiFive7VCQ, SiFive7VA], mx, IsWorstCase>;
     defm "" : LMULWriteResMX<"WriteVIMovI",     [SiFive7VCQ, SiFive7VA], mx, IsWorstCase>;
   }
+  let Latency = 8, AcquireAtCycles = [0, 1], ReleaseAtCycles = [1, !add(1, Cycles)] in {
+    defm "" : LMULWriteResMX<"WriteVShiftV",    [SiFive7VCQ, SiFive7VA], mx, IsWorstCase>;
+    defm "" : LMULWriteResMX<"WriteVShiftX",    [SiFive7VCQ, SiFive7VA], mx, IsWorstCase>;
+    defm "" : LMULWriteResMX<"WriteVShiftI",    [SiFive7VCQ, SiFive7VA], mx, IsWorstCase>;
+    defm "" : LMULWriteResMX<"WriteVIMulV",     [SiFive7VCQ, SiFive7VA], mx, IsWorstCase>;
+    defm "" : LMULWriteResMX<"WriteVIMulX",     [SiFive7VCQ, SiFive7VA], mx, IsWorstCase>;
+    defm "" : LMULWriteResMX<"WriteVIMulAddV",  [SiFive7VCQ, SiFive7VA], mx, IsWorstCase>;
+    defm "" : LMULWriteResMX<"WriteVIMulAddX",  [SiFive7VCQ, SiFive7VA], mx, IsWorstCase>;
+  }
   // Mask results can't chain.
   let Latency = !add(Cycles, 3), AcquireAtCycles = [0, 1], ReleaseAtCycles = [1, !add(1, Cycles)] in {
     defm "" : LMULWriteResMX<"WriteVICmpV",     [SiFive7VCQ, SiFive7VA], mx, IsWorstCase>;
diff --git a/llvm/lib/Target/RISCV/RISCVSchedSiFiveP500.td b/llvm/lib/Target/RISCV/RISCVSchedSiFiveP500.td
new file mode 100644
index 0000000000000..32cfa701c4fdb
--- /dev/null
+++ b/llvm/lib/Target/RISCV/RISCVSchedSiFiveP500.td
@@ -0,0 +1,361 @@
+//==- RISCVSchedSiFiveP500.td - SiFiveP500 Scheduling Defs ---*- tablegen -*-=//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+//===----------------------------------------------------------------------===//
+
+def SiFiveP500Model : SchedMachineModel {
+  let IssueWidth = 3;         // 3 micro-ops are dispatched per cycle.
+  let MicroOpBufferSize = 96; // Max micro-ops that can be buffered.
+  let LoadLatency = 4;        // Cycles for loads to access the cache.
+  let MispredictPenalty = 9;  // Extra cycles for a mispredicted branch.
+  let CompleteModel = false;
+}
+
+// The SiFiveP500 microarchitecure has 7 pipelines:
+// Three pipelines for integer operations.
+// Two pipelines for FPU operations.
+// One pipeline for Load operations.
+// One pipeline for Store operations.
+let SchedModel = SiFiveP500Model in {
+
+def SiFiveP500IEXQ0       : ProcResource<1>;
+def SiFiveP500IEXQ1       : ProcResource<1>;
+def SiFiveP500IEXQ2       : ProcResource<1>;
+def SiFiveP500FEXQ0       : ProcResource<1>;
+def SiFiveP500FEXQ1       : ProcResource<1>;
+def SiFiveP500Load        : ProcResource<1>;
+def SiFiveP500Store       : ProcResource<1>;
+
+def SiFiveP500IntArith    : ProcResGroup<[SiFiveP500IEXQ0, SiFiveP500IEXQ1, SiFiveP500IEXQ2]>;
+defvar SiFiveP500Branch   = SiFiveP500IEXQ0;
+defvar SiFiveP500SYS      = SiFiveP500IEXQ1;
+defvar SiFiveP500CMOV     = SiFiveP500IEXQ1;
+defvar SiFiveP500MulI2F   = SiFiveP500IEXQ2;
+def SiFiveP500Div         : ProcResource<1>;
+
+def SiFiveP500FloatArith  : ProcResGroup<[SiFiveP500FEXQ0, SiFiveP500FEXQ1]>;
+defvar SiFiveP500F2I      = SiFiveP500FEXQ0;
+def SiFiveP500FloatDiv    : ProcResource<1>;
+
+let Latency = 1 in {
+// Integer arithmetic and logic
+def : WriteRes<WriteIALU, [SiFiveP500IntArith]>;
+def : WriteRes<WriteIALU32, [SiFiveP500IntArith]>;
+def : WriteRes<WriteShiftImm, [SiFiveP500IntArith]>;
+def : WriteRes<WriteShiftImm32, [SiFiveP500IntArith]>;
+def : WriteRes<WriteShiftReg, [SiFiveP500IntArith]>;
+def : WriteRes<WriteShiftReg32, [SiFiveP500IntArith]>;
+// Branching
+def : WriteRes<WriteJmp, [SiFiveP500Branch]>;
+def : WriteRes<WriteJal, [SiFiveP500Branch]>;
+def : WriteRes<WriteJalr, [SiFiveP500Branch]>;
+}
+
+// CMOV
+def P500WriteCMOV : SchedWriteRes<[SiFiveP500Branch, SiFiveP500CMOV]> {
+  let Latency = 2;
+  let NumMicroOps = 2;
+}
+def : InstRW<[P500WriteCMOV], (instrs PseudoCCMOVGPRNoX0)>;
+
+let Latency = 3 in {
+// Integer multiplication
+def : WriteRes<WriteIMul, [SiFiveP500MulI2F]>;
+def : WriteRes<WriteIMul32, [SiFiveP500MulI2F]>;
+// cpop[w] look exactly like multiply.
+def : WriteRes<WriteCPOP, [SiFiveP500MulI2F]>;
+def : WriteRes<WriteCPOP32, [SiFiveP500MulI2F]>;
+}
+
+// Integer division
+def : WriteRes<WriteIDiv, [SiFiveP500MulI2F, SiFiveP500Div]> {
+  let Latency = 35;
+  let ReleaseAtCycles = [1, 34];
+}
+def : WriteRes<WriteIDiv32, [SiFiveP500MulI2F, SiFiveP500Div]> {
+  let Latency = 20;
+  let ReleaseAtCycles = [1, 19];
+}
+
+// Integer remainder
+def : WriteRes<WriteIRem, [SiFiveP500MulI2F, SiFiveP500Div]> {
+  let Latency = 35;
+  let ReleaseAtCycles = [1, 34];
+}
+def : WriteRes<WriteIRem32, [SiFiveP500MulI2F, SiFiveP500Div]> {
+  let Latency = 20;
+  let ReleaseAtCycles = [1, 19];
+}
+
+let Latency = 1 in {
+// Bitmanip
+def : WriteRes<WriteRotateImm, [SiFiveP500IntArith]>;
+def : WriteRes<WriteRotateImm32, [SiFiveP500IntArith]>;
+def : WriteRes<WriteRotateReg, [SiFiveP500IntArith]>;
+def : WriteRes<WriteRotateReg32, [SiFiveP500IntArith]>;
+
+def : WriteRes<WriteCLZ, [SiFiveP500IntArith]>;
+def : WriteRes<WriteCLZ32, [SiFiveP500IntArith]>;
+def : WriteRes<WriteCTZ, [SiFiveP500IntArith]>;
+def : WriteRes<WriteCTZ32, [SiFiveP500IntArith]>;
+
+def : WriteRes<WriteORCB, [SiFiveP500IntArith]>;
+def : WriteRes<WriteIMinMax, [SiFiveP500IntArith]>;
+
+def : WriteRes<WriteREV8, [SiFiveP500IntArith]>;
+
+def : WriteRes<WriteSHXADD, [SiFiveP500IntArith]>;
+def : WriteRes<WriteSHXADD32, [SiFiveP500IntArith]>;
+}
+
+// Memory
+let Latency = 1 in {
+def : WriteRes<WriteSTB, [SiFiveP500Store]>;
+def : WriteRes<WriteSTH, [SiFiveP500Store]>;
+def : WriteRes<WriteSTW, [SiFiveP500Store]>;
+def : WriteRes<WriteSTD, [SiFiveP500Store]>;
+def : WriteRes<WriteFST16, [SiFiveP500Store]>;
+def : WriteRes<WriteFST32, [SiFiveP500Store]>;
+def : WriteRes<WriteFST64, [SiFiveP500Store]>;
+}
+let Latency = 4 in {
+def : WriteRes<WriteLDB, [SiFiveP500Load]>;
+def : WriteRes<WriteLDH, [SiFiveP500Load]>;
+}
+let Latency = 4 in {
+def : WriteRes<WriteLDW, [SiFiveP500Load]>;
+def : WriteRes<WriteLDD, [SiFiveP500Load]>;
+}
+
+let Latency = 6 in {
+def : WriteRes<WriteFLD16, [SiFiveP500Load]>;
+def : WriteRes<WriteFLD32, [SiFiveP500Load]>;
+def : WriteRes<WriteFLD64, [SiFiveP500Load]>;
+}
+
+// Atomic memory
+let Latency = 3 in {
+def : WriteRes<WriteAtomicSTW, [SiFiveP500Store]>;
+def : WriteRes<WriteAtomicSTD, [SiFiveP500Store]>;
+def : WriteRes<WriteAtomicW, [SiFiveP500Load]>;
+def : WriteRes<WriteAtomicD, [SiFiveP500Load]>;
+def : WriteRes<WriteAtomicLDW, [SiFiveP500Load]>;
+def : WriteRes<WriteAtomicLDD, [SiFiveP500Load]>;
+}
+
+// Floating point
+let Latency = 4 in {
+def : WriteRes<WriteFAdd16, [SiFiveP500FloatArith]>;
+def : WriteRes<WriteFAdd32, [SiFiveP500FloatArith]>;
+def : WriteRes<WriteFAdd64, [SiFiveP500FloatArith]>;
+
+def : WriteRes<WriteFMul16, [SiFiveP500FloatArith]>;
+def : WriteRes<WriteFMul32, [SiFiveP500FloatArith]>;
+def : WriteRes<WriteFMul64, [SiFiveP500FloatArith]>;
+
+def : WriteRes<WriteFMA16, [SiFiveP500FloatArith]>;
+def : WriteRes<WriteFMA32, [SiFiveP500FloatArith]>;
+def : WriteRes<WriteFMA64, [SiFiveP500FloatArith]>;
+}
+
+let Latency = 2 in {
+def : WriteRes<WriteFSGNJ16, [SiFiveP500FloatArith]>;
+def : WriteRes<WriteFSGNJ32, [SiFiveP500FloatArith]>;
+def : WriteRes<WriteFSGNJ64, [SiFiveP500FloatArith]>;
+
+def : WriteRes<WriteFMinMax16, [SiFiveP500FloatArith]>;
+def : WriteRes<WriteFMinMax32, [SiFiveP500FloatArith]>;
+def : WriteRes<WriteFMinMax64, [SiFiveP500FloatArith]>;
+}
+
+// Half precision.
+def : WriteRes<WriteFDiv16, [SiFiveP500FEXQ1, SiFiveP500FloatDiv]> {
+  let Latency = 19;
+  let ReleaseAtCycles = [1, 18];
+}
+def : WriteRes<WriteFSqrt16, [SiFiveP500FEXQ1, SiFiveP500FloatDiv]> {
+  let Latency = 18;
+  let ReleaseAtCycles = [1, 17];
+}
+
+// Single precision.
+def : WriteRes<WriteFDiv32, [SiFiveP500FEXQ1, SiFiveP500FloatDiv]> {
+  let Latency = 19;
+  let ReleaseAtCycles = [1, 18];
+}
+def : WriteRes<WriteFSqrt32, [SiFiveP500FEXQ1, SiFiveP500FloatDiv]> {
+  let Latency = 18;
+  let ReleaseAtCycles = [1, 17];
+}
+
+// Double precision
+def : WriteRes<WriteFDiv64, [SiFiveP500FEXQ1, SiFiveP500FloatDiv]> {
+  let Latency = 33;
+  let ReleaseAtCycles = [1, 32];
+}
+def : WriteRes<WriteFSqrt64, [SiFiveP500FEXQ1, SiFiveP500FloatDiv]> {
+  let Latency = 33;
+  let ReleaseAtCycles = [1, 32];
+}
+
+// Conversions
+let Latency = 2 in {
+def : WriteRes<WriteFCvtI32ToF16, [SiFiveP500MulI2F]>;
+def : WriteRes<WriteFCvtI32ToF32, [SiFiveP500MulI2F]>;
+def : WriteRes<WriteFCvtI32ToF64, [SiFiveP500MulI2F]>;
+def : WriteRes<WriteFCvtI64ToF16, [SiFiveP500MulI2F]>;
+def : WriteRes<WriteFCvtI64ToF32, [SiFiveP500MulI2F]>;
+def : WriteRes<WriteFCvtI64ToF64, [SiFiveP500MulI2F]>;
+def : WriteRes<WriteFCvtF16ToI32, [SiFiveP500F2I]>;
+def : WriteRes<WriteFCvtF16ToI64, [SiFiveP500F2I]>;
+def : WriteRes<WriteFCvtF16ToF32, [SiFiveP500FloatArith]>;
+def : WriteRes<WriteFCvtF16ToF64, [SiFiveP500FloatArith]>;
+def : WriteRes<WriteFCvtF32ToI32, [SiFiveP500F2I]>;
+def : WriteRes<WriteFCvtF32ToI64, [SiFiveP500F2I]>;
+def : WriteRes<WriteFCvtF32ToF16, [SiFiveP500FloatArith]>;
+def : WriteRes<WriteFCvtF32ToF64, [SiFiveP500FloatArith]>;
+def : WriteRes<WriteFCvtF64ToI32, [SiFiveP500F2I]>;
+def : WriteRes<WriteFCvtF64ToI64, [SiFiveP500F2I]>;
+def : WriteRes<WriteFCvtF64ToF16, [SiFiveP500FloatArith]>;
+def : WriteRes<WriteFCvtF64ToF32, [SiFiveP500FloatArith]>;
+
+def : WriteRes<WriteFClass16, [SiFiveP500F2I]>;
+def : WriteRes<WriteFClass32, [SiFiveP500F2I]>;
+def : WriteRes<WriteFClass64, [SiFiveP500F2I]>;
+def : WriteRes<WriteFCmp16, [SiFiveP500F2I]>;
+def : WriteRes<WriteFCmp32, [SiFiveP500F2I]>;
+def : WriteRes<WriteFCmp64, [SiFiveP500F2I]>;
+def : WriteRes<WriteFMovI16ToF16, [SiFiveP500MulI2F]>;
+def : WriteRes<WriteFMovF16ToI16, [SiFiveP500F2I]>;
+def : WriteRes<WriteFMovI32ToF32, [SiFiveP500MulI2F]>;
+def : WriteRes<WriteFMovF32ToI32, [SiFiveP500F2I]>;
+def : WriteRes<WriteFMovI64ToF64, [SiFiveP500MulI2F]>;
+def : WriteRes<WriteFMovF64ToI64, [SiFiveP500F2I]>;
+}
+
+// Others
+def : WriteRes<WriteCSR, [SiFiveP500SYS]>;
+def : WriteRes<WriteNop, []>;
+
+// FIXME: This could be better modeled by looking at the regclasses of the operands.
+def : InstRW<[WriteIALU, ReadIALU], (instrs COPY)>;
+
+//===----------------------------------------------------------------------===//
+// Bypass and advance
+def : ReadAdvance<ReadJmp, 0>;
+def : ReadAdvance<ReadJalr, 0>;
+def : ReadAdvance<ReadCSR, 0>;
+def : ReadAdvance<ReadStoreData, 0>;
+def : ReadAdvance<ReadMemBase, 0>;
+def : ReadAdvance<ReadIALU, 0>;
+def : ReadAdvance<ReadIALU32, 0>;
+def : ReadAdvance<ReadShiftImm, 0>;
+def : ReadAdvance<ReadShiftImm32, 0>;
+def : ReadAdvance<ReadShiftReg, 0>;
+def : ReadAdvance<ReadShiftReg32, 0>;
+def : ReadAdvance<ReadIDiv, 0>;
+def : ReadAdvance<ReadIDiv32, 0>;
+def : ReadAdvance<ReadIRem, 0>;
+def : ReadAdvance<ReadIRem32, 0>;
+def : ReadAdvance<ReadIMul, 0>;
+def : ReadAdvance<ReadIMul32, 0>;
+def : ReadAdvance<ReadAtomicWA, 0>;
+def : ReadAdvance<ReadAtomicWD, 0>;
+def : ReadAdvance<ReadAtomicDA, 0>;
+def : ReadAdvance<ReadAtomicDD, 0>;
+def : ReadAdvance<ReadAtomicLDW, 0>;
+def : ReadAdvance<ReadAtomicLDD, 0>;
+def : ReadAdvance<ReadAtomicSTW, 0>;
+def : ReadAdvance<ReadAtomicSTD, 0>;
+def : ReadAdvance<ReadFStoreData, 0>;
+def : ReadAdvance<ReadFMemBase, 0>;
+def : ReadAdvance<ReadFAdd16, 0>;
+def : ReadAdvance<ReadFAdd32, 0>;
+def : ReadAdvance<ReadFAdd64, 0>;
+def : ReadAdvance<ReadFMul16, 0>;
+def : ReadAdvance<ReadFMA16, 0>;
+def : ReadAdvance<ReadFMA16Addend, 0>;
+def : ReadAdvance<ReadFMul32, 0>;
+def : ReadAdvance<ReadFMA32, 0>;
+def : ReadAdvance<ReadFMA32Addend, 0>;
+def : ReadAdvance<ReadFMul64, 0>;
+def : ReadAdvance<ReadFMA64, 0>;
+def : ReadAdvance<ReadFMA64Addend, 0>;
+def : ReadAdvance<ReadFDiv16, 0>;
+def : ReadAdvance<ReadFDiv32, 0>;
+def : ReadAdvance<ReadFDiv64, 0>;
+def : ReadAdvance<ReadFSqrt16, 0>;
+def : ReadAdvance<ReadFSqrt32, 0>;
+def : ReadAdvance<ReadFSqrt64, 0>;
+def : ReadAdvance<ReadFCmp16, 0>;
+def : ReadAdvance<ReadFCmp32, 0>;
+def : ReadAdvance<ReadFCmp64, 0>;
+def : ReadAdvance<ReadFSGNJ16, 0>;
+def : ReadAdvance<ReadFSGNJ32, 0>;
+def : ReadAdvance<ReadFSGNJ64, 0>;
+def : ReadAdvance<ReadFMinMax16, 0>;
+def : ReadAdvance<ReadFMinMax32, 0>;
+def : ReadAdvance<ReadFMinMax64, 0>;
+def : ReadAdvance<ReadFCvtF16ToI32, 0>;
+def : ReadAdvance<ReadFCvtF16ToI64, 0>;
+def : ReadAdvance<ReadFCvtF32ToI32, 0>;
+def : ReadAdvance<ReadFCvtF32ToI64, 0>;
+def : ReadAdvance<ReadFCvtF64ToI32, 0>;
+def : ReadAdvance<ReadFCvtF64ToI64, 0>;
+def : ReadAdvance<ReadFCvtI32ToF16, 0>;
+def : ReadAdvance<ReadFCvtI32ToF32, 0>;
+def : ReadAdvance<ReadFCvtI32ToF64, 0>;
+def : ReadAdvance<ReadFCvtI64ToF16, 0>;
+def : ReadAdvance<ReadFCvtI64ToF32, 0>;
+def : ReadAdvance<ReadFCvtI64ToF64, 0>;
+def : ReadAdvance<ReadFCvtF32ToF64, 0>;
+def : ReadAdvance<ReadFCvtF64ToF32, 0>;
+def : ReadAdvance<ReadFCvtF16ToF32, 0>;
+def : ReadAdvance<ReadFCvtF32ToF16, 0>;
+def : ReadAdvance<ReadFCvtF16ToF64, 0>;
+def : ReadAdvance<ReadFCvtF64ToF16, 0>;
+def : ReadAdvance<ReadFMovF16ToI16, 0>;
+def : ReadAdvance<ReadFMovI16ToF16, 0>;
+def : ReadAdvance<ReadFMovF32ToI32, 0>;
+def : ReadAdvance<ReadFMovI32ToF32, 0>;
+def : ReadAdvance<ReadFMovF64ToI64, 0>;
+def : ReadAdvance<ReadFMovI64ToF64, 0>;
+def : ReadAdvance<ReadFClass16, 0>;
+def : ReadAdvance<ReadFClass32, 0>;
+def : ReadAdvance<ReadFClass64, 0>;
+
+// Bitmanip
+def : ReadAdvance<ReadRotateImm, 0>;
+def : ReadAdvance<ReadRotateImm32, 0>;
+def : ReadAdvance<ReadRotateReg, 0>;
+def : ReadAdvance<ReadRotateReg32, 0>;
+def : ReadAdvance<ReadCLZ, 0>;
+def : ReadAdvance<ReadCLZ32, 0>;
+def : ReadAdvance<ReadCTZ, 0>;
+def : ReadAdvance<ReadCTZ32, 0>;
+def : ReadAdvance<ReadCPOP, 0>;
+def : ReadAdvance<ReadCPOP32, 0>;
+def : ReadAdvance<ReadORCB, 0>;
+def : ReadAdvance<ReadIMinMax, 0>;
+def : ReadAdvance<ReadREV8, 0>;
+def : ReadAdvance<ReadSHXADD, 0>;
+def : ReadAdvance<ReadSHXADD32, 0>;
+
+//===----------------------------------------------------------------------===//
+// Unsupported extensions
+defm : UnsupportedSchedV;
+defm : UnsupportedSchedZabha;
+defm : UnsupportedSchedZbc;
+defm : UnsupportedSchedZbs;
+defm : UnsupportedSchedZbkb;
+defm : UnsupportedSchedZbkx;
+defm : UnsupportedSchedSFB;
+defm : UnsupportedSchedZfa;
+defm : UnsupportedSchedZvk;
+defm : UnsupportedSchedXsfvcp;
+}
diff --git a/llvm/lib/Target/RISCV/RISCVScheduleV.td b/llvm/lib/Target/RISCV/RISCVScheduleV.td
index 6b9f1dd321891..0204ab4c98286 100644
--- a/llvm/lib/Target/RISCV/RISCVScheduleV.td
+++ b/llvm/lib/Target/RISCV/RISCVScheduleV.td
@@ -70,7 +70,7 @@ multiclass LMULSEWWriteResMXSEW<string name, list<ProcResourceKind> resources,
 // behavior is aliased to a Variant. The Variant has Latency predLad and
 // ReleaseAtCycles predCycles if the SchedPredicate Pred is true, otherwise has
 // Latency noPredLat and ReleaseAtCycles noPredCycles. The WorstCase SchedWrite
-// is created similiarly if IsWorstCase is true.
+// is created similarly if IsWorstCase is true.
 multiclass LMULWriteResMXVariant<string name, SchedPredicateBase Pred,
                                  list<ProcResourceKind> resources,
                                  int predLat, list<int> predAcquireCycles,
diff --git a/llvm/lib/Target/RISCV/RISCVSubtarget.cpp b/llvm/lib/Target/RISCV/RISCVSubtarget.cpp
index 6e212dc58e6dd..1b54c278820fc 100644
--- a/llvm/lib/Target/RISCV/RISCVSubtarget.cpp
+++ b/llvm/lib/Target/RISCV/RISCVSubtarget.cpp
@@ -62,6 +62,15 @@ static cl::opt<unsigned> RISCVMinimumJumpTableEntries(
     "riscv-min-jump-table-entries", cl::Hidden,
     cl::desc("Set minimum number of entries to use a jump table on RISCV"));
 
+static cl::opt<bool>
+    UseMIPSLoadStorePairsOpt("mips-riscv-load-store-pairs",
+                             cl::desc("RISCV: Optimize for load-store bonding"),
+                             cl::init(false), cl::Hidden);
+
+static cl::opt<bool>
+    UseCCMovInsn("riscv-ccmov", cl::desc("RISCV: Use 'mips.ccmov' instruction"),
+                 cl::init(true), cl::Hidden);
+
 void RISCVSubtarget::anchor() {}
 
 RISCVSubtarget &
@@ -238,3 +247,7 @@ void RISCVSubtarget::overridePostRASchedPolicy(MachineSchedPolicy &Policy,
     Policy.OnlyBottomUp = false;
   }
 }
+
+bool RISCVSubtarget::useCCMovInsn() const {
+  return UseCCMovInsn && HasVendorXMIPSCMove;
+}
diff --git a/llvm/lib/Target/RISCV/RISCVSubtarget.h b/llvm/lib/Target/RISCV/RISCVSubtarget.h
index 87d508c394173..a059ed9202e33 100644
--- a/llvm/lib/Target/RISCV/RISCVSubtarget.h
+++ b/llvm/lib/Target/RISCV/RISCVSubtarget.h
@@ -188,6 +188,8 @@ class RISCVSubtarget : public RISCVGenSubtargetInfo {
   unsigned getXLen() const {
     return is64Bit() ? 64 : 32;
   }
+  bool useLoadStorePairs() const;
+  bool useCCMovInsn() const;
   unsigned getFLen() const {
     if (HasStdExtD)
       return 64;
@@ -235,8 +237,8 @@ class RISCVSubtarget : public RISCVGenSubtargetInfo {
            TargetABI == RISCVABI::ABI_ILP32E;
   }
   bool isRegisterReservedByUser(Register i) const override {
-    assert(i < RISCV::NUM_TARGET_REGS && "Register out of range");
-    return UserReservedRegister[i];
+    assert(i.id() < RISCV::NUM_TARGET_REGS && "Register out of range");
+    return UserReservedRegister[i.id()];
   }
 
   // XRay support - require D and C extensions.
diff --git a/llvm/lib/Target/RISCV/RISCVSystemOperands.td b/llvm/lib/Target/RISCV/RISCVSystemOperands.td
index 4c86103db99d2..cabcb9eda06b1 100644
--- a/llvm/lib/Target/RISCV/RISCVSystemOperands.td
+++ b/llvm/lib/Target/RISCV/RISCVSystemOperands.td
@@ -19,6 +19,9 @@ include "llvm/TableGen/SearchableTable.td"
 
 class SysReg<string name, bits<12> op> {
   string Name = name;
+  // Custom vendor CSRs have a "<vendor>." prefix. Convert these to "<vendor>_"
+  // before passing it to the SysRegEncodings GenericEnum below.
+  string EnumName = !subst(".", "_", name);
   bits<12> Encoding = op;
   // FIXME: add these additional fields when needed.
   // Privilege Access: Read and Write = 0, 1, 2; Read-Only = 3.
@@ -50,7 +53,7 @@ def SysRegsList : GenericTable {
 
 def SysRegEncodings : GenericEnum {
   let FilterClass = "SysReg";
-  let NameField = "Name";
+  let NameField = "EnumName";
   let ValueField = "Encoding";
 }
 
diff --git a/llvm/lib/Target/RISCV/RISCVTargetMachine.cpp b/llvm/lib/Target/RISCV/RISCVTargetMachine.cpp
index f6ccbfbe217df..4a69bdeb76161 100644
--- a/llvm/lib/Target/RISCV/RISCVTargetMachine.cpp
+++ b/llvm/lib/Target/RISCV/RISCVTargetMachine.cpp
@@ -78,7 +78,7 @@ static cl::opt<bool> EnableRISCVCopyPropagation(
 static cl::opt<bool> EnableRISCVDeadRegisterElimination(
     "riscv-enable-dead-defs", cl::Hidden,
     cl::desc("Enable the pass that removes dead"
-             " definitons and replaces stores to"
+             " definitions and replaces stores to"
              " them with stores to x0"),
     cl::init(true));
 
@@ -287,6 +287,39 @@ bool RISCVTargetMachine::isNoopAddrSpaceCast(unsigned SrcAS,
   return true;
 }
 
+ScheduleDAGInstrs *
+RISCVTargetMachine::createMachineScheduler(MachineSchedContext *C) const {
+  ScheduleDAGMILive *DAG = nullptr;
+  if (EnableMISchedLoadStoreClustering) {
+    DAG = createGenericSchedLive(C);
+    DAG->addMutation(createLoadClusterDAGMutation(
+        DAG->TII, DAG->TRI, /*ReorderWhileClustering=*/true));
+    DAG->addMutation(createStoreClusterDAGMutation(
+        DAG->TII, DAG->TRI, /*ReorderWhileClustering=*/true));
+  }
+
+  const RISCVSubtarget &ST = C->MF->getSubtarget<RISCVSubtarget>();
+  if (!DisableVectorMaskMutation && ST.hasVInstructions()) {
+    DAG = DAG ? DAG : createGenericSchedLive(C);
+    DAG->addMutation(createRISCVVectorMaskDAGMutation(DAG->TRI));
+  }
+  return DAG;
+}
+
+ScheduleDAGInstrs *
+RISCVTargetMachine::createPostMachineScheduler(MachineSchedContext *C) const {
+  ScheduleDAGMI *DAG = nullptr;
+  if (EnablePostMISchedLoadStoreClustering) {
+    DAG = createGenericSchedPostRA(C);
+    DAG->addMutation(createLoadClusterDAGMutation(
+        DAG->TII, DAG->TRI, /*ReorderWhileClustering=*/true));
+    DAG->addMutation(createStoreClusterDAGMutation(
+        DAG->TII, DAG->TRI, /*ReorderWhileClustering=*/true));
+  }
+
+  return DAG;
+}
+
 namespace {
 
 class RVVRegisterRegAlloc : public RegisterRegAllocBase<RVVRegisterRegAlloc> {
@@ -360,38 +393,6 @@ class RISCVPassConfig : public TargetPassConfig {
     return getTM<RISCVTargetMachine>();
   }
 
-  ScheduleDAGInstrs *
-  createMachineScheduler(MachineSchedContext *C) const override {
-    ScheduleDAGMILive *DAG = nullptr;
-    if (EnableMISchedLoadStoreClustering) {
-      DAG = createGenericSchedLive(C);
-      DAG->addMutation(createLoadClusterDAGMutation(
-          DAG->TII, DAG->TRI, /*ReorderWhileClustering=*/true));
-      DAG->addMutation(createStoreClusterDAGMutation(
-          DAG->TII, DAG->TRI, /*ReorderWhileClustering=*/true));
-    }
-
-    const RISCVSubtarget &ST = C->MF->getSubtarget<RISCVSubtarget>();
-    if (!DisableVectorMaskMutation && ST.hasVInstructions()) {
-      DAG = DAG ? DAG : createGenericSchedLive(C);
-      DAG->addMutation(createRISCVVectorMaskDAGMutation(DAG->TRI));
-    }
-    return DAG;
-  }
-
-  ScheduleDAGInstrs *
-  createPostMachineScheduler(MachineSchedContext *C) const override {
-    ScheduleDAGMI *DAG = nullptr;
-    if (EnablePostMISchedLoadStoreClustering) {
-      DAG = createGenericSchedPostRA(C);
-      DAG->addMutation(createLoadClusterDAGMutation(
-          DAG->TII, DAG->TRI, /*ReorderWhileClustering=*/true));
-      DAG->addMutation(createStoreClusterDAGMutation(
-          DAG->TII, DAG->TRI, /*ReorderWhileClustering=*/true));
-    }
-    return DAG;
-  }
-  
   void addIRPasses() override;
   bool addPreISel() override;
   void addCodeGenPrepare() override;
diff --git a/llvm/lib/Target/RISCV/RISCVTargetMachine.h b/llvm/lib/Target/RISCV/RISCVTargetMachine.h
index b1610e3f81eba..c85c2b3d0e611 100644
--- a/llvm/lib/Target/RISCV/RISCVTargetMachine.h
+++ b/llvm/lib/Target/RISCV/RISCVTargetMachine.h
@@ -59,6 +59,10 @@ class RISCVTargetMachine : public CodeGenTargetMachineImpl {
                                 SMDiagnostic &Error,
                                 SMRange &SourceRange) const override;
   void registerPassBuilderCallbacks(PassBuilder &PB) override;
+  ScheduleDAGInstrs *
+  createMachineScheduler(MachineSchedContext *C) const override;
+  ScheduleDAGInstrs *
+  createPostMachineScheduler(MachineSchedContext *C) const override;
 };
 
 std::unique_ptr<ScheduleDAGMutation>
diff --git a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp
index add82dc80c429..c46400a1936ad 100644
--- a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp
+++ b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp
@@ -376,6 +376,104 @@ static VectorType *getVRGatherIndexType(MVT DataVT, const RISCVSubtarget &ST,
   return cast<VectorType>(EVT(IndexVT).getTypeForEVT(C));
 }
 
+/// Try to perform better estimation of the permutation.
+/// 1. Split the source/destination vectors into real registers.
+/// 2. Do the mask analysis to identify which real registers are
+/// permuted. If more than 1 source registers are used for the
+/// destination register building, the cost for this destination register
+/// is (Number_of_source_register - 1) * Cost_PermuteTwoSrc. If only one
+/// source register is used, build mask and calculate the cost as a cost
+/// of PermuteSingleSrc.
+/// Also, for the single register permute we try to identify if the
+/// destination register is just a copy of the source register or the
+/// copy of the previous destination register (the cost is
+/// TTI::TCC_Basic). If the source register is just reused, the cost for
+/// this operation is 0.
+static InstructionCost
+costShuffleViaVRegSplitting(RISCVTTIImpl &TTI, MVT LegalVT,
+                            std::optional<unsigned> VLen, VectorType *Tp,
+                            ArrayRef<int> Mask, TTI::TargetCostKind CostKind) {
+  InstructionCost NumOfDests = InstructionCost::getInvalid();
+  if (VLen && LegalVT.isFixedLengthVector() && !Mask.empty()) {
+    MVT ElemVT = LegalVT.getVectorElementType();
+    unsigned ElemsPerVReg = *VLen / ElemVT.getFixedSizeInBits();
+    LegalVT = TTI.getTypeLegalizationCost(
+                     FixedVectorType::get(Tp->getElementType(), ElemsPerVReg))
+                  .second;
+    // Number of destination vectors after legalization:
+    NumOfDests = divideCeil(Mask.size(), LegalVT.getVectorNumElements());
+  }
+  if (!NumOfDests.isValid() || NumOfDests <= 1 ||
+      !LegalVT.isFixedLengthVector() ||
+      LegalVT.getVectorElementType().getSizeInBits() !=
+          Tp->getElementType()->getPrimitiveSizeInBits() ||
+      LegalVT.getVectorNumElements() >= Tp->getElementCount().getFixedValue())
+    return InstructionCost::getInvalid();
+
+  unsigned VecTySize = TTI.getDataLayout().getTypeStoreSize(Tp);
+  unsigned LegalVTSize = LegalVT.getStoreSize();
+  // Number of source vectors after legalization:
+  unsigned NumOfSrcs = divideCeil(VecTySize, LegalVTSize);
+
+  auto *SingleOpTy = FixedVectorType::get(Tp->getElementType(),
+                                          LegalVT.getVectorNumElements());
+
+  unsigned E = *NumOfDests.getValue();
+  unsigned NormalizedVF =
+      LegalVT.getVectorNumElements() * std::max(NumOfSrcs, E);
+  unsigned NumOfSrcRegs = NormalizedVF / LegalVT.getVectorNumElements();
+  unsigned NumOfDestRegs = NormalizedVF / LegalVT.getVectorNumElements();
+  SmallVector<int> NormalizedMask(NormalizedVF, PoisonMaskElem);
+  assert(NormalizedVF >= Mask.size() &&
+         "Normalized mask expected to be not shorter than original mask.");
+  copy(Mask, NormalizedMask.begin());
+  InstructionCost Cost = 0;
+  SmallBitVector ExtractedRegs(2 * NumOfSrcRegs);
+  int NumShuffles = 0;
+  processShuffleMasks(
+      NormalizedMask, NumOfSrcRegs, NumOfDestRegs, NumOfDestRegs, []() {},
+      [&](ArrayRef<int> RegMask, unsigned SrcReg, unsigned DestReg) {
+        if (ExtractedRegs.test(SrcReg)) {
+          Cost += TTI.getShuffleCost(TTI::SK_ExtractSubvector, Tp, {}, CostKind,
+                                     (SrcReg % NumOfSrcRegs) *
+                                         SingleOpTy->getNumElements(),
+                                     SingleOpTy);
+          ExtractedRegs.set(SrcReg);
+        }
+        if (!ShuffleVectorInst::isIdentityMask(RegMask, RegMask.size())) {
+          ++NumShuffles;
+          Cost += TTI.getShuffleCost(TTI::SK_PermuteSingleSrc, SingleOpTy,
+                                     RegMask, CostKind, 0, nullptr);
+          return;
+        }
+      },
+      [&](ArrayRef<int> RegMask, unsigned Idx1, unsigned Idx2, bool NewReg) {
+        if (ExtractedRegs.test(Idx1)) {
+          Cost += TTI.getShuffleCost(
+              TTI::SK_ExtractSubvector, Tp, {}, CostKind,
+              (Idx1 % NumOfSrcRegs) * SingleOpTy->getNumElements(), SingleOpTy);
+          ExtractedRegs.set(Idx1);
+        }
+        if (ExtractedRegs.test(Idx2)) {
+          Cost += TTI.getShuffleCost(
+              TTI::SK_ExtractSubvector, Tp, {}, CostKind,
+              (Idx2 % NumOfSrcRegs) * SingleOpTy->getNumElements(), SingleOpTy);
+          ExtractedRegs.set(Idx2);
+        }
+        Cost += TTI.getShuffleCost(TTI::SK_PermuteTwoSrc, SingleOpTy, RegMask,
+                                   CostKind, 0, nullptr);
+        NumShuffles += 2;
+      });
+  // Note: check that we do not emit too many shuffles here to prevent code
+  // size explosion.
+  // TODO: investigate, if it can be improved by extra analysis of the masks
+  // to check if the code is more profitable.
+  if ((NumOfDestRegs > 2 && NumShuffles <= static_cast<int>(NumOfDestRegs)) ||
+      (NumOfDestRegs <= 2 && NumShuffles < 4))
+    return Cost;
+  return InstructionCost::getInvalid();
+}
+
 InstructionCost RISCVTTIImpl::getShuffleCost(TTI::ShuffleKind Kind,
                                              VectorType *Tp, ArrayRef<int> Mask,
                                              TTI::TargetCostKind CostKind,
@@ -389,7 +487,11 @@ InstructionCost RISCVTTIImpl::getShuffleCost(TTI::ShuffleKind Kind,
   // First, handle cases where having a fixed length vector enables us to
   // give a more accurate cost than falling back to generic scalable codegen.
   // TODO: Each of these cases hints at a modeling gap around scalable vectors.
-  if (isa<FixedVectorType>(Tp)) {
+  if (ST->hasVInstructions() && isa<FixedVectorType>(Tp)) {
+    InstructionCost VRegSplittingCost = costShuffleViaVRegSplitting(
+        *this, LT.second, ST->getRealVLen(), Tp, Mask, CostKind);
+    if (VRegSplittingCost.isValid())
+      return VRegSplittingCost;
     switch (Kind) {
     default:
       break;
@@ -571,7 +673,7 @@ InstructionCost RISCVTTIImpl::getShuffleCost(TTI::ShuffleKind Kind,
     // We use 2 for the cost of the mask materialization as this is the true
     // cost for small masks and most shuffles are small.  At worst, this cost
     // should be a very small constant for the constant pool load.  As such,
-    // we may bias towards large selects slightly more than truely warranted.
+    // we may bias towards large selects slightly more than truly warranted.
     return LT.first *
            (1 + getRISCVInstructionCost({RISCV::VMV_S_X, RISCV::VMERGE_VVM},
                                         LT.second, CostKind));
@@ -838,6 +940,44 @@ InstructionCost RISCVTTIImpl::getGatherScatterOpCost(
   return NumLoads * MemOpCost;
 }
 
+InstructionCost RISCVTTIImpl::getExpandCompressMemoryOpCost(
+    unsigned Opcode, Type *DataTy, bool VariableMask, Align Alignment,
+    TTI::TargetCostKind CostKind, const Instruction *I) {
+  bool IsLegal = (Opcode == Instruction::Store &&
+                  isLegalMaskedCompressStore(DataTy, Alignment)) ||
+                 (Opcode == Instruction::Load &&
+                  isLegalMaskedExpandLoad(DataTy, Alignment));
+  if (!IsLegal || CostKind != TTI::TCK_RecipThroughput)
+    return BaseT::getExpandCompressMemoryOpCost(Opcode, DataTy, VariableMask,
+                                                Alignment, CostKind, I);
+  // Example compressstore sequence:
+  // vsetivli        zero, 8, e32, m2, ta, ma (ignored)
+  // vcompress.vm    v10, v8, v0
+  // vcpop.m a1, v0
+  // vsetvli zero, a1, e32, m2, ta, ma
+  // vse32.v v10, (a0)
+  // Example expandload sequence:
+  // vsetivli        zero, 8, e8, mf2, ta, ma (ignored)
+  // vcpop.m a1, v0
+  // vsetvli zero, a1, e32, m2, ta, ma
+  // vle32.v v10, (a0)
+  // vsetivli        zero, 8, e32, m2, ta, ma
+  // viota.m v12, v0
+  // vrgather.vv     v8, v10, v12, v0.t
+  auto MemOpCost =
+      getMemoryOpCost(Opcode, DataTy, Alignment, /*AddressSpace*/ 0, CostKind);
+  auto LT = getTypeLegalizationCost(DataTy);
+  SmallVector<unsigned, 4> Opcodes{RISCV::VSETVLI};
+  if (VariableMask)
+    Opcodes.push_back(RISCV::VCPOP_M);
+  if (Opcode == Instruction::Store)
+    Opcodes.append({RISCV::VCOMPRESS_VM});
+  else
+    Opcodes.append({RISCV::VSETIVLI, RISCV::VIOTA_M, RISCV::VRGATHER_VV});
+  return MemOpCost +
+         LT.first * getRISCVInstructionCost(Opcodes, LT.second, CostKind);
+}
+
 InstructionCost RISCVTTIImpl::getStridedMemoryOpCost(
     unsigned Opcode, Type *DataTy, const Value *Ptr, bool VariableMask,
     Align Alignment, TTI::TargetCostKind CostKind, const Instruction *I) {
@@ -1036,6 +1176,15 @@ RISCVTTIImpl::getIntrinsicInstrCost(const IntrinsicCostAttributes &ICA,
     }
     break;
   }
+  case Intrinsic::fma:
+  case Intrinsic::fmuladd: {
+    // TODO: handle promotion with f16/bf16 with zvfhmin/zvfbfmin
+    auto LT = getTypeLegalizationCost(RetTy);
+    if (ST->hasVInstructions() && LT.second.isVector())
+      return LT.first *
+             getRISCVInstructionCost(RISCV::VFMADD_VV, LT.second, CostKind);
+    break;
+  }
   case Intrinsic::fabs: {
     auto LT = getTypeLegalizationCost(RetTy);
     if (ST->hasVInstructions() && LT.second.isVector()) {
@@ -1235,6 +1384,14 @@ RISCVTTIImpl::getIntrinsicInstrCost(const IntrinsicCostAttributes &ICA,
                                                   : RISCV::VMV_V_X,
                                               LT.second, CostKind);
   }
+  case Intrinsic::experimental_vp_splice: {
+    // To support type-based query from vectorizer, set the index to 0.
+    // Note that index only change the cost from vslide.vx to vslide.vi and in
+    // current implementations they have same costs.
+    return getShuffleCost(TTI::SK_Splice,
+                          cast<VectorType>(ICA.getArgTypes()[0]), {}, CostKind,
+                          0, cast<VectorType>(ICA.getReturnType()));
+  }
   }
 
   if (ST->hasVInstructions() && RetTy->isVectorTy()) {
@@ -2240,7 +2397,7 @@ InstructionCost RISCVTTIImpl::getPointersChainCost(
   // either GEP instructions, PHIs, bitcasts or constants. When we have same
   // base, we just calculate cost of each non-Base GEP as an ADD operation if
   // any their index is a non-const.
-  // If no known dependecies between the pointers cost is calculated as a sum
+  // If no known dependencies between the pointers cost is calculated as a sum
   // of costs of GEP instructions.
   for (auto [I, V] : enumerate(Ptrs)) {
     const auto *GEP = dyn_cast<GetElementPtrInst>(V);
@@ -2284,7 +2441,7 @@ void RISCVTTIImpl::getUnrollingPreferences(Loop *L, ScalarEvolution &SE,
   if (ST->enableDefaultUnroll())
     return BasicTTIImplBase::getUnrollingPreferences(L, SE, UP, ORE);
 
-  // Enable Upper bound unrolling universally, not dependant upon the conditions
+  // Enable Upper bound unrolling universally, not dependent upon the conditions
   // below.
   UP.UpperBound = true;
 
@@ -2610,7 +2767,12 @@ bool RISCVTTIImpl::isProfitableToSinkOperands(
         return false;
     }
 
-    Ops.push_back(&Op->getOperandUse(0));
+    Use *InsertEltUse = &Op->getOperandUse(0);
+    // Sink any fpexts since they might be used in a widening fp pattern.
+    auto *InsertElt = cast<InsertElementInst>(InsertEltUse);
+    if (isa<FPExtInst>(InsertElt->getOperand(1)))
+      Ops.push_back(&InsertElt->getOperandUse(1));
+    Ops.push_back(InsertEltUse);
     Ops.push_back(&OpIdx.value());
   }
   return true;
diff --git a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h
index 9b364391f0fa4..5389e9bc4a8fe 100644
--- a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h
+++ b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h
@@ -48,6 +48,14 @@ class RISCVTTIImpl : public BasicTTIImplBase<RISCVTTIImpl> {
   /// actual target hardware.
   unsigned getEstimatedVLFor(VectorType *Ty);
 
+  /// This function calculates the costs for one or more RVV opcodes based
+  /// on the vtype and the cost kind.
+  /// \param Opcodes A list of opcodes of the RVV instruction to evaluate.
+  /// \param VT The MVT of vtype associated with the RVV instructions.
+  /// For widening/narrowing instructions where the result and source types
+  /// differ, it is important to check the spec to determine whether the vtype
+  /// refers to the result or source type.
+  /// \param CostKind The type of cost to compute.
   InstructionCost getRISCVInstructionCost(ArrayRef<unsigned> OpCodes, MVT VT,
                                           TTI::TargetCostKind CostKind);
 
@@ -166,6 +174,12 @@ class RISCVTTIImpl : public BasicTTIImplBase<RISCVTTIImpl> {
                                          TTI::TargetCostKind CostKind,
                                          const Instruction *I);
 
+  InstructionCost getExpandCompressMemoryOpCost(unsigned Opcode, Type *Src,
+                                                bool VariableMask,
+                                                Align Alignment,
+                                                TTI::TargetCostKind CostKind,
+                                                const Instruction *I = nullptr);
+
   InstructionCost getStridedMemoryOpCost(unsigned Opcode, Type *DataTy,
                                          const Value *Ptr, bool VariableMask,
                                          Align Alignment,
diff --git a/llvm/lib/Target/RISCV/RISCVVLOptimizer.cpp b/llvm/lib/Target/RISCV/RISCVVLOptimizer.cpp
index 976c65e51c205..1ba7f0b522a2b 100644
--- a/llvm/lib/Target/RISCV/RISCVVLOptimizer.cpp
+++ b/llvm/lib/Target/RISCV/RISCVVLOptimizer.cpp
@@ -18,7 +18,7 @@
 
 #include "RISCV.h"
 #include "RISCVSubtarget.h"
-#include "llvm/ADT/SetVector.h"
+#include "llvm/ADT/PostOrderIterator.h"
 #include "llvm/CodeGen/MachineDominators.h"
 #include "llvm/CodeGen/MachineFunctionPass.h"
 #include "llvm/InitializePasses.h"
@@ -56,26 +56,11 @@ class RISCVVLOptimizer : public MachineFunctionPass {
   std::optional<MachineOperand> checkUsers(MachineInstr &MI);
   bool tryReduceVL(MachineInstr &MI);
   bool isCandidate(const MachineInstr &MI) const;
-};
-
-} // end anonymous namespace
-
-char RISCVVLOptimizer::ID = 0;
-INITIALIZE_PASS_BEGIN(RISCVVLOptimizer, DEBUG_TYPE, PASS_NAME, false, false)
-INITIALIZE_PASS_DEPENDENCY(MachineDominatorTreeWrapperPass)
-INITIALIZE_PASS_END(RISCVVLOptimizer, DEBUG_TYPE, PASS_NAME, false, false)
 
-FunctionPass *llvm::createRISCVVLOptimizerPass() {
-  return new RISCVVLOptimizer();
-}
-
-/// Return true if R is a physical or virtual vector register, false otherwise.
-static bool isVectorRegClass(Register R, const MachineRegisterInfo *MRI) {
-  if (R.isPhysical())
-    return RISCV::VRRegClass.contains(R);
-  const TargetRegisterClass *RC = MRI->getRegClass(R);
-  return RISCVRI::isVRegClass(RC->TSFlags);
-}
+  /// For a given instruction, records what elements of it are demanded by
+  /// downstream users.
+  DenseMap<const MachineInstr *, std::optional<MachineOperand>> DemandedVLs;
+};
 
 /// Represents the EMUL and EEW of a MachineOperand.
 struct OperandInfo {
@@ -117,6 +102,25 @@ struct OperandInfo {
   }
 };
 
+} // end anonymous namespace
+
+char RISCVVLOptimizer::ID = 0;
+INITIALIZE_PASS_BEGIN(RISCVVLOptimizer, DEBUG_TYPE, PASS_NAME, false, false)
+INITIALIZE_PASS_DEPENDENCY(MachineDominatorTreeWrapperPass)
+INITIALIZE_PASS_END(RISCVVLOptimizer, DEBUG_TYPE, PASS_NAME, false, false)
+
+FunctionPass *llvm::createRISCVVLOptimizerPass() {
+  return new RISCVVLOptimizer();
+}
+
+/// Return true if R is a physical or virtual vector register, false otherwise.
+static bool isVectorRegClass(Register R, const MachineRegisterInfo *MRI) {
+  if (R.isPhysical())
+    return RISCV::VRRegClass.contains(R);
+  const TargetRegisterClass *RC = MRI->getRegClass(R);
+  return RISCVRI::isVRegClass(RC->TSFlags);
+}
+
 LLVM_ATTRIBUTE_UNUSED
 static raw_ostream &operator<<(raw_ostream &OS, const OperandInfo &OI) {
   OI.print(OS);
@@ -133,8 +137,6 @@ static raw_ostream &operator<<(raw_ostream &OS,
   return OS;
 }
 
-namespace llvm {
-namespace RISCVVType {
 /// Return EMUL = (EEW / SEW) * LMUL where EEW comes from Log2EEW and LMUL and
 /// SEW are from the TSFlags of MI.
 static std::pair<unsigned, bool>
@@ -161,8 +163,6 @@ getEMULEqualsEEWDivSEWTimesLMUL(unsigned Log2EEW, const MachineInstr &MI) {
   Denom = MILMULIsFractional ? Denom * MILMUL / GCD : Denom / GCD;
   return std::make_pair(Num > Denom ? Num : Denom, Denom > Num);
 }
-} // end namespace RISCVVType
-} // end namespace llvm
 
 /// Dest has EEW=SEW. Source EEW=SEW/Factor (i.e. F2 => EEW/2).
 /// SEW comes from TSFlags of MI.
@@ -208,7 +208,8 @@ getOperandLog2EEW(const MachineOperand &MO, const MachineRegisterInfo *MRI) {
   const bool HasPassthru = RISCVII::isFirstDefTiedToFirstUse(MI.getDesc());
   const bool IsTied = RISCVII::isTiedPseudo(MI.getDesc().TSFlags);
 
-  bool IsMODef = MO.getOperandNo() == 0;
+  bool IsMODef = MO.getOperandNo() == 0 ||
+                 (HasPassthru && MO.getOperandNo() == MI.getNumExplicitDefs());
 
   // All mask operands have EEW=1
   if (isMaskOperand(MI, MO, MRI))
@@ -451,6 +452,23 @@ getOperandLog2EEW(const MachineOperand &MO, const MachineRegisterInfo *MRI) {
   case RISCV::VFDIV_VF:
   case RISCV::VFDIV_VV:
   case RISCV::VFRDIV_VF:
+  // Vector Single-Width Floating-Point Fused Multiply-Add Instructions
+  case RISCV::VFMACC_VV:
+  case RISCV::VFMACC_VF:
+  case RISCV::VFNMACC_VV:
+  case RISCV::VFNMACC_VF:
+  case RISCV::VFMSAC_VV:
+  case RISCV::VFMSAC_VF:
+  case RISCV::VFNMSAC_VV:
+  case RISCV::VFNMSAC_VF:
+  case RISCV::VFMADD_VV:
+  case RISCV::VFMADD_VF:
+  case RISCV::VFNMADD_VV:
+  case RISCV::VFNMADD_VF:
+  case RISCV::VFMSUB_VV:
+  case RISCV::VFMSUB_VF:
+  case RISCV::VFNMSUB_VV:
+  case RISCV::VFNMSUB_VF:
   // Vector Floating-Point Square-Root Instruction
   case RISCV::VFSQRT_V:
   // Vector Floating-Point Reciprocal Square-Root Estimate Instruction
@@ -528,6 +546,8 @@ getOperandLog2EEW(const MachineOperand &MO, const MachineRegisterInfo *MRI) {
   case RISCV::VFWMSAC_VV:
   case RISCV::VFWNMSAC_VF:
   case RISCV::VFWNMSAC_VV:
+  case RISCV::VFWMACCBF16_VV:
+  case RISCV::VFWMACCBF16_VF:
   // Vector Widening Floating-Point Add/Subtract Instructions
   // Dest EEW=2*SEW. Source EEW=SEW.
   case RISCV::VFWADD_VV:
@@ -766,8 +786,7 @@ getOperandInfo(const MachineOperand &MO, const MachineRegisterInfo *MRI) {
   };
 
   // All others have EMUL=EEW/SEW*LMUL
-  return OperandInfo(RISCVVType::getEMULEqualsEEWDivSEWTimesLMUL(*Log2EEW, MI),
-                     *Log2EEW);
+  return OperandInfo(getEMULEqualsEEWDivSEWTimesLMUL(*Log2EEW, MI), *Log2EEW);
 }
 
 /// Return true if this optimization should consider MI for VL reduction. This
@@ -960,6 +979,17 @@ static bool isSupportedInstr(const MachineInstr &MI) {
   case RISCV::VMV_V_I:
   case RISCV::VMV_V_X:
   case RISCV::VMV_V_V:
+  // Vector Single-Width Saturating Add and Subtract
+  case RISCV::VSADDU_VV:
+  case RISCV::VSADDU_VX:
+  case RISCV::VSADDU_VI:
+  case RISCV::VSADD_VV:
+  case RISCV::VSADD_VX:
+  case RISCV::VSADD_VI:
+  case RISCV::VSSUBU_VV:
+  case RISCV::VSSUBU_VX:
+  case RISCV::VSSUB_VV:
+  case RISCV::VSSUB_VX:
   // Vector Single-Width Averaging Add and Subtract
   case RISCV::VAADDU_VV:
   case RISCV::VAADDU_VX:
@@ -969,6 +999,23 @@ static bool isSupportedInstr(const MachineInstr &MI) {
   case RISCV::VASUBU_VX:
   case RISCV::VASUB_VV:
   case RISCV::VASUB_VX:
+  // Vector Single-Width Fractional Multiply with Rounding and Saturation
+  case RISCV::VSMUL_VV:
+  case RISCV::VSMUL_VX:
+  // Vector Single-Width Scaling Shift Instructions
+  case RISCV::VSSRL_VV:
+  case RISCV::VSSRL_VX:
+  case RISCV::VSSRL_VI:
+  case RISCV::VSSRA_VV:
+  case RISCV::VSSRA_VX:
+  case RISCV::VSSRA_VI:
+  // Vector Narrowing Fixed-Point Clip Instructions
+  case RISCV::VNCLIPU_WV:
+  case RISCV::VNCLIPU_WX:
+  case RISCV::VNCLIPU_WI:
+  case RISCV::VNCLIP_WV:
+  case RISCV::VNCLIP_WX:
+  case RISCV::VNCLIP_WI:
 
   // Vector Crypto
   case RISCV::VWSLL_VI:
@@ -1017,6 +1064,34 @@ static bool isSupportedInstr(const MachineInstr &MI) {
   // Vector Widening Floating-Point Multiply
   case RISCV::VFWMUL_VF:
   case RISCV::VFWMUL_VV:
+  // Vector Single-Width Floating-Point Fused Multiply-Add Instructions
+  case RISCV::VFMACC_VV:
+  case RISCV::VFMACC_VF:
+  case RISCV::VFNMACC_VV:
+  case RISCV::VFNMACC_VF:
+  case RISCV::VFMSAC_VV:
+  case RISCV::VFMSAC_VF:
+  case RISCV::VFNMSAC_VV:
+  case RISCV::VFNMSAC_VF:
+  case RISCV::VFMADD_VV:
+  case RISCV::VFMADD_VF:
+  case RISCV::VFNMADD_VV:
+  case RISCV::VFNMADD_VF:
+  case RISCV::VFMSUB_VV:
+  case RISCV::VFMSUB_VF:
+  case RISCV::VFNMSUB_VV:
+  case RISCV::VFNMSUB_VF:
+  // Vector Widening Floating-Point Fused Multiply-Add Instructions
+  case RISCV::VFWMACC_VV:
+  case RISCV::VFWMACC_VF:
+  case RISCV::VFWNMACC_VV:
+  case RISCV::VFWNMACC_VF:
+  case RISCV::VFWMSAC_VV:
+  case RISCV::VFWMSAC_VF:
+  case RISCV::VFWNMSAC_VV:
+  case RISCV::VFWNMSAC_VF:
+  case RISCV::VFWMACCBF16_VV:
+  case RISCV::VFWMACCBF16_VF:
   // Vector Floating-Point MIN/MAX Instructions
   case RISCV::VFMIN_VF:
   case RISCV::VFMIN_VV:
@@ -1140,35 +1215,14 @@ bool RISCVVLOptimizer::isCandidate(const MachineInstr &MI) const {
   const MCInstrDesc &Desc = MI.getDesc();
   if (!RISCVII::hasVLOp(Desc.TSFlags) || !RISCVII::hasSEWOp(Desc.TSFlags))
     return false;
-  if (MI.getNumDefs() != 1)
-    return false;
 
-  // If we're not using VLMAX, then we need to be careful whether we are using
-  // TA/TU when there is a non-undef Passthru. But when we are using VLMAX, it
-  // does not matter whether we are using TA/TU with a non-undef Passthru, since
-  // there are no tail elements to be preserved.
-  unsigned VLOpNum = RISCVII::getVLOpNum(Desc);
-  const MachineOperand &VLOp = MI.getOperand(VLOpNum);
-  if (VLOp.isReg() || VLOp.getImm() != RISCV::VLMaxSentinel) {
-    // If MI has a non-undef passthru, we will not try to optimize it since
-    // that requires us to preserve tail elements according to TA/TU.
-    // Otherwise, The MI has an undef Passthru, so it doesn't matter whether we
-    // are using TA/TU.
-    bool HasPassthru = RISCVII::isFirstDefTiedToFirstUse(Desc);
-    unsigned PassthruOpIdx = MI.getNumExplicitDefs();
-    if (HasPassthru &&
-        MI.getOperand(PassthruOpIdx).getReg() != RISCV::NoRegister) {
-      LLVM_DEBUG(
-          dbgs() << "  Not a candidate because it uses non-undef passthru"
-                    " with non-VLMAX VL\n");
-      return false;
-    }
-  }
+  if (MI.getNumExplicitDefs() != 1)
+    return false;
 
-  // If the VL is 1, then there is no need to reduce it. This is an
-  // optimization, not needed to preserve correctness.
-  if (VLOp.isImm() && VLOp.getImm() == 1) {
-    LLVM_DEBUG(dbgs() << "  Not a candidate because VL is already 1\n");
+  // Some instructions have implicit defs e.g. $vxsat. If they might be read
+  // later then we can't reduce VL.
+  if (!MI.allImplicitDefsAreDead()) {
+    LLVM_DEBUG(dbgs() << "Not a candidate because has non-dead implicit def\n");
     return false;
   }
 
@@ -1213,6 +1267,25 @@ RISCVVLOptimizer::getMinimumVLForUser(MachineOperand &UserOp) {
     return std::nullopt;
   }
 
+  unsigned VLOpNum = RISCVII::getVLOpNum(Desc);
+  const MachineOperand &VLOp = UserMI.getOperand(VLOpNum);
+  // Looking for an immediate or a register VL that isn't X0.
+  assert((!VLOp.isReg() || VLOp.getReg() != RISCV::X0) &&
+         "Did not expect X0 VL");
+
+  // If the user is a passthru it will read the elements past VL, so
+  // abort if any of the elements past VL are demanded.
+  if (UserOp.isTied()) {
+    assert(UserOp.getOperandNo() == UserMI.getNumExplicitDefs() &&
+           RISCVII::isFirstDefTiedToFirstUse(UserMI.getDesc()));
+    auto DemandedVL = DemandedVLs[&UserMI];
+    if (!DemandedVL || !RISCV::isVLKnownLE(*DemandedVL, VLOp)) {
+      LLVM_DEBUG(dbgs() << "    Abort because user is passthru in "
+                           "instruction with demanded tail\n");
+      return std::nullopt;
+    }
+  }
+
   // Instructions like reductions may use a vector register as a scalar
   // register. In this case, we should treat it as only reading the first lane.
   if (isVectorOpUsedAsScalarOp(UserOp)) {
@@ -1225,19 +1298,18 @@ RISCVVLOptimizer::getMinimumVLForUser(MachineOperand &UserOp) {
     return MachineOperand::CreateImm(1);
   }
 
-  unsigned VLOpNum = RISCVII::getVLOpNum(Desc);
-  const MachineOperand &VLOp = UserMI.getOperand(VLOpNum);
-  // Looking for an immediate or a register VL that isn't X0.
-  assert((!VLOp.isReg() || VLOp.getReg() != RISCV::X0) &&
-         "Did not expect X0 VL");
+  // If we know the demanded VL of UserMI, then we can reduce the VL it
+  // requires.
+  if (auto DemandedVL = DemandedVLs[&UserMI]) {
+    assert(isCandidate(UserMI));
+    if (RISCV::isVLKnownLE(*DemandedVL, VLOp))
+      return DemandedVL;
+  }
+
   return VLOp;
 }
 
 std::optional<MachineOperand> RISCVVLOptimizer::checkUsers(MachineInstr &MI) {
-  // FIXME: Avoid visiting each user for each time we visit something on the
-  // worklist, combined with an extra visit from the outer loop. Restructure
-  // along lines of an instcombine style worklist which integrates the outer
-  // pass.
   std::optional<MachineOperand> CommonVL;
   for (auto &UserOp : MRI->use_operands(MI.getOperand(0).getReg())) {
     const MachineInstr &UserMI = *UserOp.getParent();
@@ -1247,12 +1319,6 @@ std::optional<MachineOperand> RISCVVLOptimizer::checkUsers(MachineInstr &MI) {
       return std::nullopt;
     }
 
-    // Tied operands might pass through.
-    if (UserOp.isTied()) {
-      LLVM_DEBUG(dbgs() << "    Abort because user used as tied operand\n");
-      return std::nullopt;
-    }
-
     auto VLOp = getMinimumVLForUser(UserOp);
     if (!VLOp)
       return std::nullopt;
@@ -1304,16 +1370,23 @@ std::optional<MachineOperand> RISCVVLOptimizer::checkUsers(MachineInstr &MI) {
 bool RISCVVLOptimizer::tryReduceVL(MachineInstr &MI) {
   LLVM_DEBUG(dbgs() << "Trying to reduce VL for " << MI << "\n");
 
-  auto CommonVL = checkUsers(MI);
+  unsigned VLOpNum = RISCVII::getVLOpNum(MI.getDesc());
+  MachineOperand &VLOp = MI.getOperand(VLOpNum);
+
+  // If the VL is 1, then there is no need to reduce it. This is an
+  // optimization, not needed to preserve correctness.
+  if (VLOp.isImm() && VLOp.getImm() == 1) {
+    LLVM_DEBUG(dbgs() << "  Abort due to VL == 1, no point in reducing.\n");
+    return false;
+  }
+
+  auto CommonVL = DemandedVLs[&MI];
   if (!CommonVL)
     return false;
 
   assert((CommonVL->isImm() || CommonVL->getReg().isVirtual()) &&
          "Expected VL to be an Imm or virtual Reg");
 
-  unsigned VLOpNum = RISCVII::getVLOpNum(MI.getDesc());
-  MachineOperand &VLOp = MI.getOperand(VLOpNum);
-
   if (!RISCV::isVLKnownLE(*CommonVL, VLOp)) {
     LLVM_DEBUG(dbgs() << "    Abort due to CommonVL not <= VLOp.\n");
     return false;
@@ -1345,6 +1418,7 @@ bool RISCVVLOptimizer::tryReduceVL(MachineInstr &MI) {
 }
 
 bool RISCVVLOptimizer::runOnMachineFunction(MachineFunction &MF) {
+  assert(DemandedVLs.size() == 0);
   if (skipFunction(MF.getFunction()))
     return false;
 
@@ -1355,53 +1429,34 @@ bool RISCVVLOptimizer::runOnMachineFunction(MachineFunction &MF) {
   if (!ST.hasVInstructions())
     return false;
 
-  SetVector<MachineInstr *> Worklist;
-  auto PushOperands = [this, &Worklist](MachineInstr &MI,
-                                        bool IgnoreSameBlock) {
-    for (auto &Op : MI.operands()) {
-      if (!Op.isReg() || !Op.isUse() || !Op.getReg().isVirtual() ||
-          !isVectorRegClass(Op.getReg(), MRI))
-        continue;
-
-      MachineInstr *DefMI = MRI->getVRegDef(Op.getReg());
-      if (!isCandidate(*DefMI))
-        continue;
-
-      if (IgnoreSameBlock && DefMI->getParent() == MI.getParent())
+  // For each instruction that defines a vector, compute what VL its
+  // downstream users demand.
+  for (MachineBasicBlock *MBB : post_order(&MF)) {
+    assert(MDT->isReachableFromEntry(MBB));
+    for (MachineInstr &MI : reverse(*MBB)) {
+      if (!isCandidate(MI))
         continue;
-
-      Worklist.insert(DefMI);
+      DemandedVLs.insert({&MI, checkUsers(MI)});
     }
-  };
+  }
 
-  // Do a first pass eagerly rewriting in roughly reverse instruction
-  // order, populate the worklist with any instructions we might need to
-  // revisit.  We avoid adding definitions to the worklist if they're
-  // in the same block - we're about to visit them anyways.
+  // Then go through and see if we can reduce the VL of any instructions to
+  // only what's demanded.
   bool MadeChange = false;
   for (MachineBasicBlock &MBB : MF) {
     // Avoid unreachable blocks as they have degenerate dominance
     if (!MDT->isReachableFromEntry(&MBB))
       continue;
 
-    for (auto &MI : make_range(MBB.rbegin(), MBB.rend())) {
+    for (auto &MI : reverse(MBB)) {
       if (!isCandidate(MI))
         continue;
       if (!tryReduceVL(MI))
         continue;
       MadeChange = true;
-      PushOperands(MI, /*IgnoreSameBlock*/ true);
     }
   }
 
-  while (!Worklist.empty()) {
-    assert(MadeChange);
-    MachineInstr &MI = *Worklist.pop_back_val();
-    assert(isCandidate(MI));
-    if (!tryReduceVL(MI))
-      continue;
-    PushOperands(MI, /*IgnoreSameBlock*/ false);
-  }
-
+  DemandedVLs.clear();
   return MadeChange;
 }
diff --git a/llvm/lib/Target/RISCV/RISCVVectorMaskDAGMutation.cpp b/llvm/lib/Target/RISCV/RISCVVectorMaskDAGMutation.cpp
index c48a97b12e43f..0bddbacc89e3e 100644
--- a/llvm/lib/Target/RISCV/RISCVVectorMaskDAGMutation.cpp
+++ b/llvm/lib/Target/RISCV/RISCVVectorMaskDAGMutation.cpp
@@ -12,7 +12,7 @@
 //
 // The reason why we need to do this:
 // 1. When tracking register pressure, we don't track physical registers.
-// 2. We have a RegisterClass for mask reigster (which is `VMV0`), but we don't
+// 2. We have a RegisterClass for mask register (which is `VMV0`), but we don't
 //    use it in most RVV pseudos (only used in inline asm constraint and add/sub
 //    with carry instructions). Instead, we use physical register V0 directly
 //    and insert a `$v0 = COPY ...` before the use. And, there is a fundamental
diff --git a/llvm/lib/Target/SPIRV/SPIRVAPI.cpp b/llvm/lib/Target/SPIRV/SPIRVAPI.cpp
index 95c9b0e520060..a19384d4e279c 100644
--- a/llvm/lib/Target/SPIRV/SPIRVAPI.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVAPI.cpp
@@ -8,6 +8,7 @@
 
 #include "SPIRVCommandLine.h"
 #include "SPIRVSubtarget.h"
+#include "SPIRVTargetMachine.h"
 #include "llvm/Analysis/TargetLibraryInfo.h"
 #include "llvm/CodeGen/CommandFlags.h"
 #include "llvm/CodeGen/MachineFunctionPass.h"
@@ -40,25 +41,6 @@ using namespace llvm;
 
 namespace {
 
-// Mimic limited number of command line flags from llc to provide a better
-// user experience when passing options into the translate API call.
-static cl::opt<char> SpirvOptLevel("spirv-O", cl::Hidden, cl::Prefix,
-                                   cl::init('0'));
-static cl::opt<std::string> SpirvTargetTriple("spirv-mtriple", cl::Hidden,
-                                              cl::init(""));
-
-// Utility to accept options in a command line style.
-void parseSPIRVCommandLineOptions(const std::vector<std::string> &Options,
-                                  raw_ostream *Errs) {
-  static constexpr const char *Origin = "SPIRVTranslateModule";
-  if (!Options.empty()) {
-    std::vector<const char *> Argv(1, Origin);
-    for (const auto &Arg : Options)
-      Argv.push_back(Arg.c_str());
-    cl::ParseCommandLineOptions(Argv.size(), Argv.data(), Origin, Errs);
-  }
-}
-
 std::once_flag InitOnceFlag;
 void InitializeSPIRVTarget() {
   std::call_once(InitOnceFlag, []() {
@@ -75,35 +57,15 @@ namespace llvm {
 // The goal of this function is to facilitate integration of SPIRV Backend into
 // tools and libraries by means of exposing an API call that translate LLVM
 // module to SPIR-V and write results into a string as binary SPIR-V output,
-// providing diagnostics on fail and means of configuring translation in a style
-// of command line options.
+// providing diagnostics on fail and means of configuring translation.
 extern "C" LLVM_EXTERNAL_VISIBILITY bool
-SPIRVTranslateModule(Module *M, std::string &SpirvObj, std::string &ErrMsg,
-                     const std::vector<std::string> &AllowExtNames,
-                     const std::vector<std::string> &Opts) {
+SPIRVTranslate(Module *M, std::string &SpirvObj, std::string &ErrMsg,
+               const std::vector<std::string> &AllowExtNames,
+               llvm::CodeGenOptLevel OLevel, Triple TargetTriple) {
   // Fallbacks for option values.
   static const std::string DefaultTriple = "spirv64-unknown-unknown";
   static const std::string DefaultMArch = "";
 
-  // Parse Opts as if it'd be command line arguments.
-  std::string Errors;
-  raw_string_ostream ErrorStream(Errors);
-  parseSPIRVCommandLineOptions(Opts, &ErrorStream);
-  if (!Errors.empty()) {
-    ErrMsg = Errors;
-    return false;
-  }
-
-  llvm::CodeGenOptLevel OLevel;
-  if (auto Level = CodeGenOpt::parseLevel(SpirvOptLevel)) {
-    OLevel = *Level;
-  } else {
-    ErrMsg = "Invalid optimization level!";
-    return false;
-  }
-
-  // Overrides/ammends `-spirv-ext` command line switch (if present) by the
-  // explicit list of allowed SPIR-V extensions.
   std::set<SPIRV::Extension::Extension> AllowedExtIds;
   StringRef UnknownExt =
       SPIRVExtensionsParser::checkExtensions(AllowExtNames, AllowedExtIds);
@@ -111,14 +73,10 @@ SPIRVTranslateModule(Module *M, std::string &SpirvObj, std::string &ErrMsg,
     ErrMsg = "Unknown SPIR-V extension: " + UnknownExt.str();
     return false;
   }
-  SPIRVSubtarget::addExtensionsToClOpt(AllowedExtIds);
 
   // SPIR-V-specific target initialization.
   InitializeSPIRVTarget();
 
-  Triple TargetTriple(SpirvTargetTriple.empty()
-                          ? M->getTargetTriple()
-                          : Triple::normalize(SpirvTargetTriple));
   if (TargetTriple.getTriple().empty()) {
     TargetTriple.setTriple(DefaultTriple);
     M->setTargetTriple(DefaultTriple);
@@ -142,6 +100,11 @@ SPIRVTranslateModule(Module *M, std::string &SpirvObj, std::string &ErrMsg,
     return false;
   }
 
+  // Set available extensions.
+  SPIRVTargetMachine *STM = static_cast<SPIRVTargetMachine *>(Target.get());
+  const_cast<SPIRVSubtarget *>(STM->getSubtargetImpl())
+      ->initAvailableExtensions(AllowedExtIds);
+
   if (M->getCodeModel())
     Target->setCodeModel(*M->getCodeModel());
 
@@ -177,4 +140,30 @@ SPIRVTranslateModule(Module *M, std::string &SpirvObj, std::string &ErrMsg,
   return true;
 }
 
+// TODO: Remove this wrapper after existing clients switch into a newer
+// implementation of SPIRVTranslate().
+extern "C" LLVM_EXTERNAL_VISIBILITY bool
+SPIRVTranslateModule(Module *M, std::string &SpirvObj, std::string &ErrMsg,
+                     const std::vector<std::string> &AllowExtNames,
+                     const std::vector<std::string> &Opts) {
+  // optional: Opts[0] is a string representation of Triple,
+  // take Module triple otherwise
+  Triple TargetTriple(Opts.empty() || Opts[0].empty()
+                          ? M->getTargetTriple()
+                          : Triple::normalize(Opts[0]));
+  // optional: Opts[1] is a string representation of CodeGenOptLevel,
+  // no optimization otherwise
+  llvm::CodeGenOptLevel OLevel = CodeGenOptLevel::None;
+  if (Opts.size() > 1 && !Opts[1].empty()) {
+    if (auto Level = CodeGenOpt::parseLevel(Opts[1][0])) {
+      OLevel = *Level;
+    } else {
+      ErrMsg = "Invalid optimization level!";
+      return false;
+    }
+  }
+  return SPIRVTranslate(M, SpirvObj, ErrMsg, AllowExtNames, OLevel,
+                        TargetTriple);
+}
+
 } // namespace llvm
diff --git a/llvm/lib/Target/SPIRV/SPIRVBuiltins.cpp b/llvm/lib/Target/SPIRV/SPIRVBuiltins.cpp
index b52c793e57e96..a7a34e0439ab1 100644
--- a/llvm/lib/Target/SPIRV/SPIRVBuiltins.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVBuiltins.cpp
@@ -132,6 +132,15 @@ struct ImageQueryBuiltin {
 #define GET_ImageQueryBuiltins_DECL
 #define GET_ImageQueryBuiltins_IMPL
 
+struct IntegerDotProductBuiltin {
+  StringRef Name;
+  uint32_t Opcode;
+  bool IsSwapReq;
+};
+
+#define GET_IntegerDotProductBuiltins_DECL
+#define GET_IntegerDotProductBuiltins_IMPL
+
 struct ConvertBuiltin {
   StringRef Name;
   InstructionSet::InstructionSet Set;
@@ -1000,6 +1009,38 @@ static bool buildBarrierInst(const SPIRV::IncomingCall *Call, unsigned Opcode,
   return true;
 }
 
+/// Helper function for building extended bit operations.
+static bool buildExtendedBitOpsInst(const SPIRV::IncomingCall *Call,
+                                    unsigned Opcode,
+                                    MachineIRBuilder &MIRBuilder,
+                                    SPIRVGlobalRegistry *GR) {
+  const SPIRV::DemangledBuiltin *Builtin = Call->Builtin;
+  const auto *ST =
+      static_cast<const SPIRVSubtarget *>(&MIRBuilder.getMF().getSubtarget());
+  if ((Opcode == SPIRV::OpBitFieldInsert ||
+       Opcode == SPIRV::OpBitFieldSExtract ||
+       Opcode == SPIRV::OpBitFieldUExtract || Opcode == SPIRV::OpBitReverse) &&
+      !ST->canUseExtension(SPIRV::Extension::SPV_KHR_bit_instructions)) {
+    std::string DiagMsg = std::string(Builtin->Name) +
+                          ": the builtin requires the following SPIR-V "
+                          "extension: SPV_KHR_bit_instructions";
+    report_fatal_error(DiagMsg.c_str(), false);
+  }
+
+  // Generate SPIRV instruction accordingly.
+  if (Call->isSpirvOp())
+    return buildOpFromWrapper(MIRBuilder, Opcode, Call,
+                              GR->getSPIRVTypeID(Call->ReturnType));
+
+  auto MIB = MIRBuilder.buildInstr(Opcode)
+                 .addDef(Call->ReturnRegister)
+                 .addUse(GR->getSPIRVTypeID(Call->ReturnType));
+  for (unsigned i = 0; i < Call->Arguments.size(); ++i)
+    MIB.addUse(Call->Arguments[i]);
+
+  return true;
+}
+
 static unsigned getNumComponentsForDim(SPIRV::Dim::Dim dim) {
   switch (dim) {
   case SPIRV::Dim::DIM_1D:
@@ -1579,20 +1620,84 @@ static bool generateCastToPtrInst(const SPIRV::IncomingCall *Call,
   return true;
 }
 
-static bool generateDotOrFMulInst(const SPIRV::IncomingCall *Call,
+static bool generateDotOrFMulInst(const StringRef DemangledCall,
+                                  const SPIRV::IncomingCall *Call,
                                   MachineIRBuilder &MIRBuilder,
                                   SPIRVGlobalRegistry *GR) {
   if (Call->isSpirvOp())
     return buildOpFromWrapper(MIRBuilder, SPIRV::OpDot, Call,
                               GR->getSPIRVTypeID(Call->ReturnType));
-  unsigned Opcode = GR->getSPIRVTypeForVReg(Call->Arguments[0])->getOpcode();
-  bool IsVec = Opcode == SPIRV::OpTypeVector;
+
+  bool IsVec = GR->getSPIRVTypeForVReg(Call->Arguments[0])->getOpcode() ==
+               SPIRV::OpTypeVector;
   // Use OpDot only in case of vector args and OpFMul in case of scalar args.
-  MIRBuilder.buildInstr(IsVec ? SPIRV::OpDot : SPIRV::OpFMulS)
-      .addDef(Call->ReturnRegister)
-      .addUse(GR->getSPIRVTypeID(Call->ReturnType))
-      .addUse(Call->Arguments[0])
-      .addUse(Call->Arguments[1]);
+  uint32_t OC = IsVec ? SPIRV::OpDot : SPIRV::OpFMulS;
+  bool IsSwapReq = false;
+
+  const auto *ST =
+      static_cast<const SPIRVSubtarget *>(&MIRBuilder.getMF().getSubtarget());
+  if (GR->isScalarOrVectorOfType(Call->ReturnRegister, SPIRV::OpTypeInt) &&
+      (ST->canUseExtension(SPIRV::Extension::SPV_KHR_integer_dot_product) ||
+       ST->isAtLeastSPIRVVer(VersionTuple(1, 6)))) {
+    const SPIRV::DemangledBuiltin *Builtin = Call->Builtin;
+    const SPIRV::IntegerDotProductBuiltin *IntDot =
+        SPIRV::lookupIntegerDotProductBuiltin(Builtin->Name);
+    if (IntDot) {
+      OC = IntDot->Opcode;
+      IsSwapReq = IntDot->IsSwapReq;
+    } else if (IsVec) {
+      // Handling "dot" and "dot_acc_sat" builtins which use vectors of
+      // integers.
+      LLVMContext &Ctx = MIRBuilder.getContext();
+      SmallVector<StringRef, 10> TypeStrs;
+      SPIRV::parseBuiltinTypeStr(TypeStrs, DemangledCall, Ctx);
+      bool IsFirstSigned = TypeStrs[0].trim()[0] != 'u';
+      bool IsSecondSigned = TypeStrs[1].trim()[0] != 'u';
+
+      if (Call->BuiltinName == "dot") {
+        if (IsFirstSigned && IsSecondSigned)
+          OC = SPIRV::OpSDot;
+        else if (!IsFirstSigned && !IsSecondSigned)
+          OC = SPIRV::OpUDot;
+        else {
+          OC = SPIRV::OpSUDot;
+          if (!IsFirstSigned)
+            IsSwapReq = true;
+        }
+      } else if (Call->BuiltinName == "dot_acc_sat") {
+        if (IsFirstSigned && IsSecondSigned)
+          OC = SPIRV::OpSDotAccSat;
+        else if (!IsFirstSigned && !IsSecondSigned)
+          OC = SPIRV::OpUDotAccSat;
+        else {
+          OC = SPIRV::OpSUDotAccSat;
+          if (!IsFirstSigned)
+            IsSwapReq = true;
+        }
+      }
+    }
+  }
+
+  MachineInstrBuilder MIB = MIRBuilder.buildInstr(OC)
+                                .addDef(Call->ReturnRegister)
+                                .addUse(GR->getSPIRVTypeID(Call->ReturnType));
+
+  if (IsSwapReq) {
+    MIB.addUse(Call->Arguments[1]);
+    MIB.addUse(Call->Arguments[0]);
+    // needed for dot_acc_sat* builtins
+    for (size_t i = 2; i < Call->Arguments.size(); ++i)
+      MIB.addUse(Call->Arguments[i]);
+  } else {
+    for (size_t i = 0; i < Call->Arguments.size(); ++i)
+      MIB.addUse(Call->Arguments[i]);
+  }
+
+  // Add Packed Vector Format for Integer dot product builtins if arguments are
+  // scalar
+  if (!IsVec && OC != SPIRV::OpFMulS)
+    MIB.addImm(0);
+
   return true;
 }
 
@@ -2107,6 +2212,17 @@ static bool generateSpecConstantInst(const SPIRV::IncomingCall *Call,
   }
 }
 
+static bool generateExtendedBitOpsInst(const SPIRV::IncomingCall *Call,
+                                       MachineIRBuilder &MIRBuilder,
+                                       SPIRVGlobalRegistry *GR) {
+  // Lookup the instruction opcode in the TableGen records.
+  const SPIRV::DemangledBuiltin *Builtin = Call->Builtin;
+  unsigned Opcode =
+      SPIRV::lookupNativeBuiltin(Builtin->Name, Builtin->Set)->Opcode;
+
+  return buildExtendedBitOpsInst(Call, Opcode, MIRBuilder, GR);
+}
+
 static bool buildNDRange(const SPIRV::IncomingCall *Call,
                          MachineIRBuilder &MIRBuilder,
                          SPIRVGlobalRegistry *GR) {
@@ -2176,14 +2292,8 @@ static SPIRVType *
 getOrCreateSPIRVDeviceEventPointer(MachineIRBuilder &MIRBuilder,
                                    SPIRVGlobalRegistry *GR) {
   LLVMContext &Context = MIRBuilder.getMF().getFunction().getContext();
-  Type *OpaqueType = StructType::getTypeByName(Context, "spirv.DeviceEvent");
-  if (!OpaqueType)
-    OpaqueType = StructType::getTypeByName(Context, "opencl.clk_event_t");
-  if (!OpaqueType)
-    OpaqueType = StructType::create(Context, "spirv.DeviceEvent");
-  unsigned SC0 = storageClassToAddressSpace(SPIRV::StorageClass::Function);
   unsigned SC1 = storageClassToAddressSpace(SPIRV::StorageClass::Generic);
-  Type *PtrType = PointerType::get(PointerType::get(OpaqueType, SC0), SC1);
+  Type *PtrType = PointerType::get(Context, SC1);
   return GR->getOrCreateSPIRVType(PtrType, MIRBuilder);
 }
 
@@ -2582,6 +2692,11 @@ mapBuiltinToOpcode(const StringRef DemangledCall,
     if (const auto *R = SPIRV::lookupGroupUniformBuiltin(Call->Builtin->Name))
       return std::make_tuple(Call->Builtin->Group, R->Opcode, 0);
     break;
+  case SPIRV::IntegerDot:
+    if (const auto *R =
+            SPIRV::lookupIntegerDotProductBuiltin(Call->Builtin->Name))
+      return std::make_tuple(Call->Builtin->Group, R->Opcode, 0);
+    break;
   case SPIRV::WriteImage:
     return std::make_tuple(Call->Builtin->Group, SPIRV::OpImageWrite, 0);
   case SPIRV::Select:
@@ -2641,7 +2756,8 @@ std::optional<bool> lowerBuiltin(const StringRef DemangledCall,
   case SPIRV::CastToPtr:
     return generateCastToPtrInst(Call.get(), MIRBuilder);
   case SPIRV::Dot:
-    return generateDotOrFMulInst(Call.get(), MIRBuilder, GR);
+  case SPIRV::IntegerDot:
+    return generateDotOrFMulInst(DemangledCall, Call.get(), MIRBuilder, GR);
   case SPIRV::Wave:
     return generateWaveInst(Call.get(), MIRBuilder, GR);
   case SPIRV::ICarryBorrow:
@@ -2682,6 +2798,8 @@ std::optional<bool> lowerBuiltin(const StringRef DemangledCall,
     return generateKernelClockInst(Call.get(), MIRBuilder, GR);
   case SPIRV::CoopMatr:
     return generateCoopMatrInst(Call.get(), MIRBuilder, GR);
+  case SPIRV::ExtendedBitOps:
+    return generateExtendedBitOpsInst(Call.get(), MIRBuilder, GR);
   }
   return false;
 }
diff --git a/llvm/lib/Target/SPIRV/SPIRVBuiltins.td b/llvm/lib/Target/SPIRV/SPIRVBuiltins.td
index af3901c0e621e..16f4252173e33 100644
--- a/llvm/lib/Target/SPIRV/SPIRVBuiltins.td
+++ b/llvm/lib/Target/SPIRV/SPIRVBuiltins.td
@@ -42,6 +42,7 @@ def Variable : BuiltinGroup;
 def Atomic : BuiltinGroup;
 def Barrier : BuiltinGroup;
 def Dot : BuiltinGroup;
+def IntegerDot : BuiltinGroup;
 def Wave : BuiltinGroup;
 def GetQuery : BuiltinGroup;
 def ImageSizeQuery : BuiltinGroup;
@@ -64,6 +65,7 @@ def CastToPtr : BuiltinGroup;
 def Construct : BuiltinGroup;
 def CoopMatr : BuiltinGroup;
 def ICarryBorrow : BuiltinGroup;
+def ExtendedBitOps : BuiltinGroup;
 
 //===----------------------------------------------------------------------===//
 // Class defining a demangled builtin record. The information in the record
@@ -101,6 +103,8 @@ def lookupBuiltin : SearchIndex {
 // Dot builtin record:
 def : DemangledBuiltin<"dot", OpenCL_std, Dot, 2, 2>;
 def : DemangledBuiltin<"__spirv_Dot", OpenCL_std, Dot, 2, 2>;
+def : DemangledBuiltin<"dot_acc_sat", OpenCL_std, IntegerDot, 3, 3>;
+def : DemangledBuiltin<"__spirv_DotAccSat", OpenCL_std, IntegerDot, 3, 3>;
 
 // Image builtin records:
 def : DemangledBuiltin<"read_imagei", OpenCL_std, ReadImage, 2, 4>;
@@ -1462,6 +1466,16 @@ defm : DemangledNativeBuiltin<"__spirv_SatConvertSToU", OpenCL_std, Convert, 1,
 defm : DemangledNativeBuiltin<"__spirv_SatConvertUToS", OpenCL_std, Convert, 1, 1, OpSatConvertUToS>;
 defm : DemangledNativeBuiltin<"__spirv_ConvertUToPtr", OpenCL_std, Convert, 1, 1, OpConvertUToPtr>;
 
+// cl_khr_extended_bit_ops / SPV_KHR_bit_instructions
+defm : DemangledNativeBuiltin<"bitfield_insert", OpenCL_std, ExtendedBitOps, 4, 4, OpBitFieldInsert>;
+defm : DemangledNativeBuiltin<"__spirv_BitFieldInsert", OpenCL_std, ExtendedBitOps, 4, 4, OpBitFieldInsert>;
+defm : DemangledNativeBuiltin<"bitfield_extract_signed", OpenCL_std, ExtendedBitOps, 3, 3, OpBitFieldSExtract>;
+defm : DemangledNativeBuiltin<"__spirv_BitFieldSExtract", OpenCL_std, ExtendedBitOps, 3, 3, OpBitFieldSExtract>;
+defm : DemangledNativeBuiltin<"bitfield_extract_unsigned", OpenCL_std, ExtendedBitOps, 3, 3, OpBitFieldUExtract>;
+defm : DemangledNativeBuiltin<"__spirv_BitFieldUExtract", OpenCL_std, ExtendedBitOps, 3, 3, OpBitFieldUExtract>;
+defm : DemangledNativeBuiltin<"bit_reverse", OpenCL_std, ExtendedBitOps, 1, 1, OpBitReverse>;
+defm : DemangledNativeBuiltin<"__spirv_BitReverse", OpenCL_std, ExtendedBitOps, 1, 1, OpBitReverse>;
+
 // cl_intel_bfloat16_conversions / SPV_INTEL_bfloat16_conversion
 // Multiclass used to define at the same time both a demangled builtin records
 // and a corresponding convert builtin records.
@@ -1715,3 +1729,47 @@ class CLMemoryFenceFlags<bits<32> value> {
 def CLK_LOCAL_MEM_FENCE : CLMemoryFenceFlags<0x1>;
 def CLK_GLOBAL_MEM_FENCE : CLMemoryFenceFlags<0x2>;
 def CLK_IMAGE_MEM_FENCE : CLMemoryFenceFlags<0x4>;
+
+//===----------------------------------------------------------------------===//
+// Class defining dot builtins that should be translated into a
+// SPIR-V instruction using SPIR-V 1.6 or SPV_KHR_integer_dot_product extension.
+//
+// name is the demangled name of the given builtin.
+// opcode specifies the SPIR-V operation code of the generated instruction.
+// isSwapRequired specifies if the operands need to be swapped (the SPIR-V extension
+// has only one instruction for arguments of different signedness).
+//===----------------------------------------------------------------------===//
+class IntegerDotProductBuiltin<string name, Op operation> {
+  string Name = name;
+  Op Opcode = operation;
+  bit IsSwapReq = !not(!eq(!find(name, "_us"), -1));
+}
+
+// Table gathering all the integer dot product builtins.
+def IntegerDotProductBuiltins : GenericTable {
+  let FilterClass = "IntegerDotProductBuiltin";
+  let Fields = ["Name", "Opcode", "IsSwapReq"];
+}
+
+// Function to lookup group builtins by their name and set.
+def lookupIntegerDotProductBuiltin : SearchIndex {
+  let Table = IntegerDotProductBuiltins;
+  let Key = ["Name"];
+}
+
+// Multiclass used to define incoming builtin records for the SPV_KHR_integer_dot_product extension.
+multiclass DemangledIntegerDotProductBuiltin<string name, bits<8> minNumArgs, bits<8> maxNumArgs, Op operation> {
+  def : DemangledBuiltin<!strconcat("dot", name), OpenCL_std, IntegerDot, minNumArgs, maxNumArgs>;
+  def : IntegerDotProductBuiltin<!strconcat("dot", name), operation>;
+}
+
+// cl_khr_integer_dot_product
+defm : DemangledIntegerDotProductBuiltin<"_4x8packed_uu_uint", 2, 3, OpUDot>;
+defm : DemangledIntegerDotProductBuiltin<"_4x8packed_ss_int", 2, 3, OpSDot>;
+defm : DemangledIntegerDotProductBuiltin<"_4x8packed_us_int", 2, 3, OpSUDot>;
+defm : DemangledIntegerDotProductBuiltin<"_4x8packed_su_int", 2, 3, OpSUDot>;
+
+defm : DemangledIntegerDotProductBuiltin<"_acc_sat_4x8packed_uu_uint", 3, 4, OpUDotAccSat>;
+defm : DemangledIntegerDotProductBuiltin<"_acc_sat_4x8packed_ss_int", 3, 4, OpSDotAccSat>;
+defm : DemangledIntegerDotProductBuiltin<"_acc_sat_4x8packed_us_int", 3, 4, OpSUDotAccSat>;
+defm : DemangledIntegerDotProductBuiltin<"_acc_sat_4x8packed_su_int", 3, 4, OpSUDotAccSat>;
diff --git a/llvm/lib/Target/SPIRV/SPIRVCommandLine.cpp b/llvm/lib/Target/SPIRV/SPIRVCommandLine.cpp
index 45b39c5116479..13683fd9a266d 100644
--- a/llvm/lib/Target/SPIRV/SPIRVCommandLine.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVCommandLine.cpp
@@ -133,6 +133,11 @@ llvm::StringRef SPIRVExtensionsParser::checkExtensions(
     const std::vector<std::string> &ExtNames,
     std::set<SPIRV::Extension::Extension> &AllowedExtensions) {
   for (const auto &Ext : ExtNames) {
+    if (Ext == "all") {
+      for (const auto &[ExtensionName, ExtensionEnum] : SPIRVExtensionMap)
+        AllowedExtensions.insert(ExtensionEnum);
+      break;
+    }
     auto It = SPIRVExtensionMap.find(Ext);
     if (It == SPIRVExtensionMap.end())
       return Ext;
diff --git a/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp b/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp
index 702206b8e0dc5..719abde3d7fa6 100644
--- a/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp
@@ -748,7 +748,12 @@ Type *SPIRVEmitIntrinsics::deduceElementTypeHelper(
     if (II && II->getIntrinsicID() == Intrinsic::spv_resource_getpointer) {
       auto *ImageType = cast<TargetExtType>(II->getOperand(0)->getType());
       assert(ImageType->getTargetExtName() == "spirv.Image");
-      Ty = ImageType->getTypeParameter(0);
+      (void)ImageType;
+      if (II->hasOneUse()) {
+        auto *U = *II->users().begin();
+        Ty = cast<Instruction>(U)->getAccessType();
+        assert(Ty && "Unable to get type for resource pointer.");
+      }
     } else if (Function *CalledF = CI->getCalledFunction()) {
       std::string DemangledName =
           getOclOrSpirvBuiltinDemangledName(CalledF->getName());
diff --git a/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.cpp b/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.cpp
index 874894ae98726..e2f1b211caa5c 100644
--- a/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.cpp
@@ -629,7 +629,7 @@ SPIRVGlobalRegistry::getOrCreateConstNullPtr(MachineIRBuilder &MIRBuilder,
   unsigned AddressSpace = typeToAddressSpace(LLVMTy);
   // Find a constant in DT or build a new one.
   Constant *CP = ConstantPointerNull::get(
-      PointerType::get(::getPointeeType(LLVMTy), AddressSpace));
+      PointerType::get(LLVMTy->getContext(), AddressSpace));
   Register Res = DT.find(CP, CurMF);
   if (!Res.isValid()) {
     LLT LLTy = LLT::pointer(AddressSpace, PointerSize);
diff --git a/llvm/lib/Target/SPIRV/SPIRVInstrInfo.td b/llvm/lib/Target/SPIRV/SPIRVInstrInfo.td
index 1bc35c6e57a4f..981e224a66399 100644
--- a/llvm/lib/Target/SPIRV/SPIRVInstrInfo.td
+++ b/llvm/lib/Target/SPIRV/SPIRVInstrInfo.td
@@ -530,8 +530,18 @@ defm OpISubBorrow: BinOpTypedGen<"OpISubBorrow", 150, subc, 0, 1>;
 def OpUMulExtended: BinOp<"OpUMulExtended", 151>;
 def OpSMulExtended: BinOp<"OpSMulExtended", 152>;
 
-def OpSDot: BinOp<"OpSDot", 4450>;
-def OpUDot: BinOp<"OpUDot", 4451>;
+def OpSDot: Op<4450, (outs ID:$res), (ins TYPE:$type, ID:$vec1, ID:$vec2, variable_ops),
+                  "$res = OpSDot $type $vec1 $vec2">;
+def OpUDot: Op<4451, (outs ID:$res), (ins TYPE:$type, ID:$vec1, ID:$vec2, variable_ops),
+                  "$res = OpUDot $type $vec1 $vec2">;
+def OpSUDot: Op<4452, (outs ID:$res), (ins TYPE:$type, ID:$vec1, ID:$vec2, variable_ops),
+                  "$res = OpSUDot $type $vec1 $vec2">;
+def OpSDotAccSat: Op<4453, (outs ID:$res), (ins TYPE:$type, ID:$vec1, ID:$vec2, ID:$acc, variable_ops),
+                  "$res = OpSDotAccSat $type $vec1 $vec2 $acc">;
+def OpUDotAccSat: Op<4454, (outs ID:$res), (ins TYPE:$type, ID:$vec1, ID:$vec2, ID:$acc, variable_ops),
+                  "$res = OpUDotAccSat $type $vec1 $vec2 $acc">;
+def OpSUDotAccSat: Op<4455, (outs ID:$res), (ins TYPE:$type, ID:$vec1, ID:$vec2, ID:$acc, variable_ops),
+                  "$res = OpSUDotAccSat $type $vec1 $vec2 $acc">;
 
 // 3.42.14 Bit Instructions
 
diff --git a/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp b/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp
index f5409c27d6ea3..e7d8fe5bd8015 100644
--- a/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp
@@ -215,6 +215,9 @@ class SPIRVInstructionSelector : public InstructionSelector {
   bool selectDot4AddPackedExpansion(Register ResVReg, const SPIRVType *ResType,
                                     MachineInstr &I) const;
 
+  bool selectWaveReduceMax(Register ResVReg, const SPIRVType *ResType,
+                           MachineInstr &I, bool IsUnsigned) const;
+
   bool selectWaveReduceSum(Register ResVReg, const SPIRVType *ResType,
                            MachineInstr &I) const;
 
@@ -2132,6 +2135,34 @@ bool SPIRVInstructionSelector::selectWaveActiveCountBits(
   return Result;
 }
 
+bool SPIRVInstructionSelector::selectWaveReduceMax(Register ResVReg,
+                                                   const SPIRVType *ResType,
+                                                   MachineInstr &I,
+                                                   bool IsUnsigned) const {
+  assert(I.getNumOperands() == 3);
+  assert(I.getOperand(2).isReg());
+  MachineBasicBlock &BB = *I.getParent();
+  Register InputRegister = I.getOperand(2).getReg();
+  SPIRVType *InputType = GR.getSPIRVTypeForVReg(InputRegister);
+
+  if (!InputType)
+    report_fatal_error("Input Type could not be determined.");
+
+  SPIRVType *IntTy = GR.getOrCreateSPIRVIntegerType(32, I, TII);
+  // Retreive the operation to use based on input type
+  bool IsFloatTy = GR.isScalarOrVectorOfType(InputRegister, SPIRV::OpTypeFloat);
+  auto IntegerOpcodeType =
+      IsUnsigned ? SPIRV::OpGroupNonUniformUMax : SPIRV::OpGroupNonUniformSMax;
+  auto Opcode = IsFloatTy ? SPIRV::OpGroupNonUniformFMax : IntegerOpcodeType;
+  return BuildMI(BB, I, I.getDebugLoc(), TII.get(Opcode))
+      .addDef(ResVReg)
+      .addUse(GR.getSPIRVTypeID(ResType))
+      .addUse(GR.getOrCreateConstInt(SPIRV::Scope::Subgroup, I, IntTy, TII))
+      .addImm(SPIRV::GroupOperation::Reduce)
+      .addUse(I.getOperand(2).getReg())
+      .constrainAllUses(TII, TRI, RBI);
+}
+
 bool SPIRVInstructionSelector::selectWaveReduceSum(Register ResVReg,
                                                    const SPIRVType *ResType,
                                                    MachineInstr &I) const {
@@ -3086,6 +3117,10 @@ bool SPIRVInstructionSelector::selectIntrinsic(Register ResVReg,
     return selectWaveOpInst(ResVReg, ResType, I, SPIRV::OpGroupNonUniformAny);
   case Intrinsic::spv_wave_is_first_lane:
     return selectWaveOpInst(ResVReg, ResType, I, SPIRV::OpGroupNonUniformElect);
+  case Intrinsic::spv_wave_reduce_umax:
+    return selectWaveReduceMax(ResVReg, ResType, I, /*IsUnsigned*/ true);
+  case Intrinsic::spv_wave_reduce_max:
+    return selectWaveReduceMax(ResVReg, ResType, I, /*IsUnsigned*/ false);
   case Intrinsic::spv_wave_reduce_sum:
     return selectWaveReduceSum(ResVReg, ResType, I);
   case Intrinsic::spv_wave_readlane:
diff --git a/llvm/lib/Target/SPIRV/SPIRVModuleAnalysis.cpp b/llvm/lib/Target/SPIRV/SPIRVModuleAnalysis.cpp
index bc00d5032544f..a7a5ecead6f5f 100644
--- a/llvm/lib/Target/SPIRV/SPIRVModuleAnalysis.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVModuleAnalysis.cpp
@@ -73,8 +73,10 @@ static SPIRV::Requirements
 getSymbolicOperandRequirements(SPIRV::OperandCategory::OperandCategory Category,
                                unsigned i, const SPIRVSubtarget &ST,
                                SPIRV::RequirementHandler &Reqs) {
-  static AvoidCapabilitiesSet
-      AvoidCaps; // contains capabilities to avoid if there is another option
+  // A set of capabilities to avoid if there is another option.
+  AvoidCapabilitiesSet AvoidCaps;
+  if (ST.isOpenCLEnv())
+    AvoidCaps.S.insert(SPIRV::Capability::Shader);
 
   VersionTuple ReqMinVer = getSymbolicOperandMinVersion(Category, i);
   VersionTuple ReqMaxVer = getSymbolicOperandMaxVersion(Category, i);
@@ -1690,6 +1692,10 @@ void addInstrRequirements(const MachineInstr &MI,
     break;
   case SPIRV::OpSDot:
   case SPIRV::OpUDot:
+  case SPIRV::OpSUDot:
+  case SPIRV::OpSDotAccSat:
+  case SPIRV::OpUDotAccSat:
+  case SPIRV::OpSUDotAccSat:
     AddDotProductRequirements(MI, Reqs, ST);
     break;
   case SPIRV::OpImageRead: {
diff --git a/llvm/lib/Target/SPIRV/SPIRVModuleAnalysis.h b/llvm/lib/Target/SPIRV/SPIRVModuleAnalysis.h
index 79b5444cca205..e6b8a381b7180 100644
--- a/llvm/lib/Target/SPIRV/SPIRVModuleAnalysis.h
+++ b/llvm/lib/Target/SPIRV/SPIRVModuleAnalysis.h
@@ -180,11 +180,13 @@ struct ModuleAnalysisInfo {
     if (RI == RegisterAliasTable[MF].end()) {
       return Register(0);
     }
-    return RegisterAliasTable[MF][Reg];
+    return RI->second;
   }
   bool hasRegisterAlias(const MachineFunction *MF, Register Reg) {
-    return RegisterAliasTable.find(MF) != RegisterAliasTable.end() &&
-           RegisterAliasTable[MF].find(Reg) != RegisterAliasTable[MF].end();
+    auto RI = RegisterAliasTable.find(MF);
+    if (RI == RegisterAliasTable.end())
+      return false;
+    return RI->second.find(Reg) != RI->second.end();
   }
   unsigned getNextID() { return MaxID++; }
   bool hasMBBRegister(const MachineBasicBlock &MBB) {
diff --git a/llvm/lib/Target/SPIRV/SPIRVPostLegalizer.cpp b/llvm/lib/Target/SPIRV/SPIRVPostLegalizer.cpp
index 6027d24636d69..e7b942e42b0a2 100644
--- a/llvm/lib/Target/SPIRV/SPIRVPostLegalizer.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVPostLegalizer.cpp
@@ -24,7 +24,6 @@
 #include "llvm/IR/Constants.h"
 #include "llvm/IR/DebugInfoMetadata.h"
 #include "llvm/IR/IntrinsicsSPIRV.h"
-#include "llvm/Target/TargetIntrinsicInfo.h"
 #include <stack>
 
 #define DEBUG_TYPE "spirv-postlegalizer"
diff --git a/llvm/lib/Target/SPIRV/SPIRVPreLegalizer.cpp b/llvm/lib/Target/SPIRV/SPIRVPreLegalizer.cpp
index b5ef8d2a9286f..32f6af3d1440f 100644
--- a/llvm/lib/Target/SPIRV/SPIRVPreLegalizer.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVPreLegalizer.cpp
@@ -23,7 +23,6 @@
 #include "llvm/IR/Constants.h"
 #include "llvm/IR/DebugInfoMetadata.h"
 #include "llvm/IR/IntrinsicsSPIRV.h"
-#include "llvm/Target/TargetIntrinsicInfo.h"
 
 #define DEBUG_TYPE "spirv-prelegalizer"
 
diff --git a/llvm/lib/Target/SPIRV/SPIRVRegularizer.cpp b/llvm/lib/Target/SPIRV/SPIRVRegularizer.cpp
index 246eecd4ffcaa..2632e0ad546ef 100644
--- a/llvm/lib/Target/SPIRV/SPIRVRegularizer.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVRegularizer.cpp
@@ -83,7 +83,7 @@ void SPIRVRegularizer::runLowerConstExpr(Function &F) {
       LLVM_DEBUG(dbgs() << "[lowerConstantExpressions] " << *CE);
       auto ReplInst = CE->getAsInstruction();
       auto InsPoint = II->getParent() == &*FBegin ? II : &FBegin->back();
-      ReplInst->insertBefore(InsPoint);
+      ReplInst->insertBefore(InsPoint->getIterator());
       LLVM_DEBUG(dbgs() << " -> " << *ReplInst << '\n');
       std::vector<Instruction *> Users;
       // Do not replace use during iteration of use. Do it in another loop.
@@ -97,7 +97,7 @@ void SPIRVRegularizer::runLowerConstExpr(Function &F) {
       for (auto &User : Users) {
         if (ReplInst->getParent() == User->getParent() &&
             User->comesBefore(ReplInst))
-          ReplInst->moveBefore(User);
+          ReplInst->moveBefore(User->getIterator());
         User->replaceUsesOfWith(CE, ReplInst);
       }
       return ReplInst;
diff --git a/llvm/lib/Target/SPIRV/SPIRVStructurizer.cpp b/llvm/lib/Target/SPIRV/SPIRVStructurizer.cpp
index 2e4343c7922f1..21539c92e5b4d 100644
--- a/llvm/lib/Target/SPIRV/SPIRVStructurizer.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVStructurizer.cpp
@@ -683,7 +683,7 @@ class SPIRVStructurizer : public FunctionPass {
               });
 
     for (Instruction *I : MergeInstructions) {
-      I->moveBefore(InsertionPoint);
+      I->moveBefore(InsertionPoint->getIterator());
       InsertionPoint = I;
     }
 
diff --git a/llvm/lib/Target/SPIRV/SPIRVSubtarget.cpp b/llvm/lib/Target/SPIRV/SPIRVSubtarget.cpp
index a476b51c3120a..3587570931541 100644
--- a/llvm/lib/Target/SPIRV/SPIRVSubtarget.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVSubtarget.cpp
@@ -84,7 +84,7 @@ SPIRVSubtarget::SPIRVSubtarget(const Triple &TT, const std::string &CPU,
   OpenCLVersion = VersionTuple(2, 2);
 
   // The order of initialization is important.
-  initAvailableExtensions();
+  initAvailableExtensions(Extensions);
   initAvailableExtInstSets();
 
   GR = std::make_unique<SPIRVGlobalRegistry>(PointerSize);
@@ -135,9 +135,12 @@ bool SPIRVSubtarget::canDirectlyComparePointers() const {
   return !SPVTranslatorCompat && isAtLeastVer(SPIRVVersion, VersionTuple(1, 4));
 }
 
-void SPIRVSubtarget::initAvailableExtensions() {
-  AvailableExtensions.clear();
-  AvailableExtensions.insert(Extensions.begin(), Extensions.end());
+void SPIRVSubtarget::accountForAMDShaderTrinaryMinmax() {
+  if (canUseExtension(
+          SPIRV::Extension::SPV_AMD_shader_trinary_minmax_extension)) {
+    AvailableExtInstSets.insert(
+        SPIRV::InstructionSet::SPV_AMD_shader_trinary_minmax);
+  }
 }
 
 // TODO: use command line args for this rather than just defaults.
@@ -150,9 +153,14 @@ void SPIRVSubtarget::initAvailableExtInstSets() {
     AvailableExtInstSets.insert(SPIRV::InstructionSet::OpenCL_std);
 
   // Handle extended instruction sets from extensions.
-  if (canUseExtension(
-          SPIRV::Extension::SPV_AMD_shader_trinary_minmax_extension)) {
-    AvailableExtInstSets.insert(
-        SPIRV::InstructionSet::SPV_AMD_shader_trinary_minmax);
-  }
+  accountForAMDShaderTrinaryMinmax();
+}
+
+// Set available extensions after SPIRVSubtarget is created.
+void SPIRVSubtarget::initAvailableExtensions(
+    const std::set<SPIRV::Extension::Extension> &AllowedExtIds) {
+  AvailableExtensions.clear();
+  AvailableExtensions.insert(AllowedExtIds.begin(), AllowedExtIds.end());
+
+  accountForAMDShaderTrinaryMinmax();
 }
diff --git a/llvm/lib/Target/SPIRV/SPIRVSubtarget.h b/llvm/lib/Target/SPIRV/SPIRVSubtarget.h
index e587739a7636f..e4484f6508b6b 100644
--- a/llvm/lib/Target/SPIRV/SPIRVSubtarget.h
+++ b/llvm/lib/Target/SPIRV/SPIRVSubtarget.h
@@ -59,8 +59,8 @@ class SPIRVSubtarget : public SPIRVGenSubtargetInfo {
 
   // TODO: Initialise the available extensions, extended instruction sets
   // based on the environment settings.
-  void initAvailableExtensions();
   void initAvailableExtInstSets();
+  void accountForAMDShaderTrinaryMinmax();
 
 public:
   // This constructor initializes the data members to match that
@@ -69,6 +69,9 @@ class SPIRVSubtarget : public SPIRVGenSubtargetInfo {
                  const std::string &FS, const SPIRVTargetMachine &TM);
   SPIRVSubtarget &initSubtargetDependencies(StringRef CPU, StringRef FS);
 
+  void initAvailableExtensions(
+      const std::set<SPIRV::Extension::Extension> &AllowedExtIds);
+
   // Parses features string setting specified subtarget options.
   // The definition of this function is auto generated by tblgen.
   void ParseSubtargetFeatures(StringRef CPU, StringRef TuneCPU, StringRef FS);
diff --git a/llvm/lib/Target/Sparc/AsmParser/SparcAsmParser.cpp b/llvm/lib/Target/Sparc/AsmParser/SparcAsmParser.cpp
index bc239480baa89..3e9fc31d7bfc2 100644
--- a/llvm/lib/Target/Sparc/AsmParser/SparcAsmParser.cpp
+++ b/llvm/lib/Target/Sparc/AsmParser/SparcAsmParser.cpp
@@ -130,6 +130,9 @@ class SparcAsmParser : public MCTargetAsmParser {
   bool expandSET(MCInst &Inst, SMLoc IDLoc,
                  SmallVectorImpl<MCInst> &Instructions);
 
+  bool expandSETSW(MCInst &Inst, SMLoc IDLoc,
+                   SmallVectorImpl<MCInst> &Instructions);
+
   bool expandSETX(MCInst &Inst, SMLoc IDLoc,
                   SmallVectorImpl<MCInst> &Instructions);
 
@@ -734,6 +737,69 @@ bool SparcAsmParser::expandSET(MCInst &Inst, SMLoc IDLoc,
   return false;
 }
 
+bool SparcAsmParser::expandSETSW(MCInst &Inst, SMLoc IDLoc,
+                                 SmallVectorImpl<MCInst> &Instructions) {
+  MCOperand MCRegOp = Inst.getOperand(0);
+  MCOperand MCValOp = Inst.getOperand(1);
+  assert(MCRegOp.isReg());
+  assert(MCValOp.isImm() || MCValOp.isExpr());
+
+  // The imm operand can be either an expression or an immediate.
+  bool IsImm = Inst.getOperand(1).isImm();
+  int64_t ImmValue = IsImm ? MCValOp.getImm() : 0;
+  const MCExpr *ValExpr = IsImm ? MCConstantExpr::create(ImmValue, getContext())
+                                : MCValOp.getExpr();
+
+  bool IsSmallImm = IsImm && isInt<13>(ImmValue);
+  bool NoLowBitsImm = IsImm && ((ImmValue & 0x3FF) == 0);
+
+  MCOperand PrevReg = MCOperand::createReg(Sparc::G0);
+
+  if (!isInt<32>(ImmValue)) {
+    return Error(IDLoc,
+                 "set: argument must be between -2147483648 and 2147483647");
+  }
+
+  // Very small immediates can be expressed without emitting a sethi.
+  if (!IsSmallImm) {
+    // sethi %hi(val), rd
+    Instructions.push_back(
+        MCInstBuilder(SP::SETHIi)
+            .addReg(MCRegOp.getReg())
+            .addExpr(adjustPICRelocation(SparcMCExpr::VK_Sparc_HI, ValExpr)));
+
+    PrevReg = MCRegOp;
+  }
+
+  // If the immediate has the lower bits set or is small, we need to emit an or.
+  if (!NoLowBitsImm || IsSmallImm) {
+    const MCExpr *Expr =
+        IsSmallImm ? ValExpr
+                   : adjustPICRelocation(SparcMCExpr::VK_Sparc_LO, ValExpr);
+
+    // or rd, %lo(val), rd
+    Instructions.push_back(MCInstBuilder(SP::ORri)
+                               .addReg(MCRegOp.getReg())
+                               .addReg(PrevReg.getReg())
+                               .addExpr(Expr));
+
+    // If it's a small immediate there's nothing more to do.
+    if (IsSmallImm)
+      return false;
+  }
+
+  // Large negative or non-immediate expressions would need an sra.
+  if (!IsImm || ImmValue < 0) {
+    // sra rd, %g0, rd
+    Instructions.push_back(MCInstBuilder(SP::SRArr)
+                               .addReg(MCRegOp.getReg())
+                               .addReg(MCRegOp.getReg())
+                               .addReg(Sparc::G0));
+  }
+
+  return false;
+}
+
 bool SparcAsmParser::expandSETX(MCInst &Inst, SMLoc IDLoc,
                                 SmallVectorImpl<MCInst> &Instructions) {
   MCOperand MCRegOp = Inst.getOperand(0);
@@ -826,6 +892,10 @@ bool SparcAsmParser::matchAndEmitInstruction(SMLoc IDLoc, unsigned &Opcode,
       if (expandSET(Inst, IDLoc, Instructions))
         return true;
       break;
+    case SP::SETSW:
+      if (expandSETSW(Inst, IDLoc, Instructions))
+        return true;
+      break;
     case SP::SETX:
       if (expandSETX(Inst, IDLoc, Instructions))
         return true;
diff --git a/llvm/lib/Target/Sparc/SparcInstrAliases.td b/llvm/lib/Target/Sparc/SparcInstrAliases.td
index 673a2db59b092..bc57ddbb5682f 100644
--- a/llvm/lib/Target/Sparc/SparcInstrAliases.td
+++ b/llvm/lib/Target/Sparc/SparcInstrAliases.td
@@ -450,6 +450,10 @@ def : InstAlias<"save", (SAVErr G0, G0, G0)>;
 // def : InstAlias<"set $val, $rd", (ORri IntRegs:$rd, (SETHIi (HI22 imm:$val)), (LO10 imm:$val))>;
 def SET : AsmPseudoInst<(outs IntRegs:$rd), (ins i32imm:$val), "set $val, $rd">;
 
+// setsw value, rd
+// (turns into a sequence of sethi+or+sra, depending on the value)
+def SETSW : AsmPseudoInst<(outs IntRegs:$rd), (ins i32imm:$val), "setsw $val, $rd">, Requires<[HasV9]>;
+
 // setx value, tmp, rd
 // (turns into a sequence of sethi+or+shift, depending on the value)
 def SETX : AsmPseudoInst<(outs I64Regs:$rd),
@@ -605,6 +609,8 @@ def : InstAlias<"unimp", (UNIMP 0), 0>;
 // interchangeable with `unimp` all the time.
 def : MnemonicAlias<"illtrap", "unimp">;
 
+def : MnemonicAlias<"setuw", "set">, Requires<[HasV9]>;
+
 def : MnemonicAlias<"iflush", "flush">;
 
 def : MnemonicAlias<"stub", "stb">;
diff --git a/llvm/lib/Target/SystemZ/SystemZAsmPrinter.cpp b/llvm/lib/Target/SystemZ/SystemZAsmPrinter.cpp
index d307c73a87fc9..589dd8b634125 100644
--- a/llvm/lib/Target/SystemZ/SystemZAsmPrinter.cpp
+++ b/llvm/lib/Target/SystemZ/SystemZAsmPrinter.cpp
@@ -688,16 +688,17 @@ void SystemZAsmPrinter::emitInstruction(const MachineInstr *MI) {
     int64_t SrcDisp = MI->getOperand(5).getImm();
 
     SystemZTargetStreamer *TS = getTargetStreamer();
-    MCSymbol *DotSym = nullptr;
-    MCInst ET = MCInstBuilder(TargetInsOpc).addReg(DestReg)
-      .addImm(DestDisp).addImm(1).addReg(SrcReg).addImm(SrcDisp);
+    MCInst ET = MCInstBuilder(TargetInsOpc)
+                    .addReg(DestReg)
+                    .addImm(DestDisp)
+                    .addImm(1)
+                    .addReg(SrcReg)
+                    .addImm(SrcDisp);
     SystemZTargetStreamer::MCInstSTIPair ET_STI(ET, &MF->getSubtarget());
-    SystemZTargetStreamer::EXRLT2SymMap::iterator I =
-        TS->EXRLTargets2Sym.find(ET_STI);
-    if (I != TS->EXRLTargets2Sym.end())
-      DotSym = I->second;
-    else
-      TS->EXRLTargets2Sym[ET_STI] = DotSym = OutContext.createTempSymbol();
+    auto [It, Inserted] = TS->EXRLTargets2Sym.try_emplace(ET_STI);
+    if (Inserted)
+      It->second = OutContext.createTempSymbol();
+    MCSymbol *DotSym = It->second;
     const MCSymbolRefExpr *Dot = MCSymbolRefExpr::create(DotSym, OutContext);
     EmitToStreamer(
         *OutStreamer,
diff --git a/llvm/lib/Target/SystemZ/SystemZISelLowering.cpp b/llvm/lib/Target/SystemZ/SystemZISelLowering.cpp
index 1fb31c26e20d3..9ffd4190b34bb 100644
--- a/llvm/lib/Target/SystemZ/SystemZISelLowering.cpp
+++ b/llvm/lib/Target/SystemZ/SystemZISelLowering.cpp
@@ -6173,8 +6173,8 @@ SystemZTargetLowering::buildVector(SelectionDAG &DAG, const SDLoc &DL, EVT VT,
     for (unsigned I = 0; I < NumElements; ++I)
       if (isVectorElementLoad(Elems[I])) {
         SDNode *Ld = Elems[I].getNode();
-        UseCounts[Ld]++;
-        if (LoadMaxUses == nullptr || UseCounts[LoadMaxUses] < UseCounts[Ld])
+        unsigned Count = ++UseCounts[Ld];
+        if (LoadMaxUses == nullptr || UseCounts[LoadMaxUses] < Count)
           LoadMaxUses = Ld;
       }
     if (LoadMaxUses != nullptr) {
diff --git a/llvm/lib/Target/SystemZ/SystemZInstrInfo.cpp b/llvm/lib/Target/SystemZ/SystemZInstrInfo.cpp
index a6fb5ab0ee9e1..8a32d998fce2c 100644
--- a/llvm/lib/Target/SystemZ/SystemZInstrInfo.cpp
+++ b/llvm/lib/Target/SystemZ/SystemZInstrInfo.cpp
@@ -983,6 +983,12 @@ void SystemZInstrInfo::copyPhysReg(MachineBasicBlock &MBB,
     Opcode = SystemZ::VLR;
   else if (SystemZ::AR32BitRegClass.contains(DestReg, SrcReg))
     Opcode = SystemZ::CPYA;
+  else if (SystemZ::GR64BitRegClass.contains(DestReg) &&
+           SystemZ::FP64BitRegClass.contains(SrcReg))
+    Opcode = SystemZ::LGDR;
+  else if (SystemZ::FP64BitRegClass.contains(DestReg) &&
+           SystemZ::GR64BitRegClass.contains(SrcReg))
+    Opcode = SystemZ::LDGR;
   else
     llvm_unreachable("Impossible reg-to-reg copy");
 
diff --git a/llvm/lib/Target/SystemZ/SystemZPostRewrite.cpp b/llvm/lib/Target/SystemZ/SystemZPostRewrite.cpp
index e15f9027cc209..cf3073f0f2090 100644
--- a/llvm/lib/Target/SystemZ/SystemZPostRewrite.cpp
+++ b/llvm/lib/Target/SystemZ/SystemZPostRewrite.cpp
@@ -107,6 +107,18 @@ void SystemZPostRewrite::selectSELRMux(MachineBasicBlock &MBB,
   bool Src1IsHigh = SystemZ::isHighReg(Src1Reg);
   bool Src2IsHigh = SystemZ::isHighReg(Src2Reg);
 
+  // In rare cases both sources are the same register (after
+  // machine-cse). This must be handled as it may lead to wrong-code (after
+  // machine-cp) if the kill flag on Src1 isn't cleared (with
+  // expandCondMove()).
+  if (Src1Reg == Src2Reg) {
+    BuildMI(*MBBI->getParent(), MBBI, MBBI->getDebugLoc(),
+            TII->get(SystemZ::COPY), DestReg)
+        .addReg(MBBI->getOperand(1).getReg(), getRegState(MBBI->getOperand(1)));
+    MBBI->eraseFromParent();
+    return;
+  }
+
   // If sources and destination aren't all high or all low, we may be able to
   // simplify the operation by moving one of the sources to the destination
   // first.  But only if this doesn't clobber the other source.
diff --git a/llvm/lib/Target/SystemZ/SystemZTargetMachine.cpp b/llvm/lib/Target/SystemZ/SystemZTargetMachine.cpp
index 9000df2f6be0f..092515ee197a8 100644
--- a/llvm/lib/Target/SystemZ/SystemZTargetMachine.cpp
+++ b/llvm/lib/Target/SystemZ/SystemZTargetMachine.cpp
@@ -205,6 +205,12 @@ SystemZTargetMachine::getSubtargetImpl(const Function &F) const {
   return I.get();
 }
 
+ScheduleDAGInstrs *
+SystemZTargetMachine::createPostMachineScheduler(MachineSchedContext *C) const {
+  return new ScheduleDAGMI(C, std::make_unique<SystemZPostRASchedStrategy>(C),
+                           /*RemoveKillFlags=*/true);
+}
+
 namespace {
 
 /// SystemZ Code Generator Pass Configuration Options.
@@ -217,13 +223,6 @@ class SystemZPassConfig : public TargetPassConfig {
     return getTM<SystemZTargetMachine>();
   }
 
-  ScheduleDAGInstrs *
-  createPostMachineScheduler(MachineSchedContext *C) const override {
-    return new ScheduleDAGMI(C,
-                             std::make_unique<SystemZPostRASchedStrategy>(C),
-                             /*RemoveKillFlags=*/true);
-  }
-
   void addIRPasses() override;
   bool addInstSelector() override;
   bool addILPOpts() override;
diff --git a/llvm/lib/Target/SystemZ/SystemZTargetMachine.h b/llvm/lib/Target/SystemZ/SystemZTargetMachine.h
index e8eeb85647b83..cced57a40ede0 100644
--- a/llvm/lib/Target/SystemZ/SystemZTargetMachine.h
+++ b/llvm/lib/Target/SystemZ/SystemZTargetMachine.h
@@ -55,6 +55,8 @@ class SystemZTargetMachine : public CodeGenTargetMachineImpl {
   MachineFunctionInfo *
   createMachineFunctionInfo(BumpPtrAllocator &Allocator, const Function &F,
                             const TargetSubtargetInfo *STI) const override;
+  ScheduleDAGInstrs *
+  createPostMachineScheduler(MachineSchedContext *C) const override;
 
   bool targetSchedulesPostRAScheduling() const override { return true; };
 };
diff --git a/llvm/lib/Target/TargetIntrinsicInfo.cpp b/llvm/lib/Target/TargetIntrinsicInfo.cpp
deleted file mode 100644
index d44a34984c421..0000000000000
--- a/llvm/lib/Target/TargetIntrinsicInfo.cpp
+++ /dev/null
@@ -1,27 +0,0 @@
-//===-- TargetIntrinsicInfo.cpp - Target Instruction Information ----------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-//
-// This file implements the TargetIntrinsicInfo class.
-//
-//===----------------------------------------------------------------------===//
-
-#include "llvm/Target/TargetIntrinsicInfo.h"
-#include "llvm/ADT/StringMapEntry.h"
-#include "llvm/IR/Function.h"
-using namespace llvm;
-
-TargetIntrinsicInfo::TargetIntrinsicInfo() = default;
-
-TargetIntrinsicInfo::~TargetIntrinsicInfo() = default;
-
-unsigned TargetIntrinsicInfo::getIntrinsicID(const Function *F) const {
-  const ValueName *ValName = F->getValueName();
-  if (!ValName)
-    return 0;
-  return lookupName(ValName->getKeyData(), ValName->getKeyLength());
-}
diff --git a/llvm/lib/Target/TargetLoweringObjectFile.cpp b/llvm/lib/Target/TargetLoweringObjectFile.cpp
index 4fe9d13d06226..02c101055d9f3 100644
--- a/llvm/lib/Target/TargetLoweringObjectFile.cpp
+++ b/llvm/lib/Target/TargetLoweringObjectFile.cpp
@@ -348,6 +348,12 @@ TargetLoweringObjectFile::SectionForGlobal(const GlobalObject *GO,
 
 MCSection *TargetLoweringObjectFile::getSectionForJumpTable(
     const Function &F, const TargetMachine &TM) const {
+  return getSectionForJumpTable(F, TM, /*JTE=*/nullptr);
+}
+
+MCSection *TargetLoweringObjectFile::getSectionForJumpTable(
+    const Function &F, const TargetMachine &TM,
+    const MachineJumpTableEntry *JTE) const {
   Align Alignment(1);
   return getSectionForConstant(F.getDataLayout(),
                                SectionKind::getReadOnly(), /*C=*/nullptr,
diff --git a/llvm/lib/Target/TargetMachine.cpp b/llvm/lib/Target/TargetMachine.cpp
index d5365f3c04743..027ae62007a72 100644
--- a/llvm/lib/Target/TargetMachine.cpp
+++ b/llvm/lib/Target/TargetMachine.cpp
@@ -26,6 +26,11 @@
 #include "llvm/Target/TargetLoweringObjectFile.h"
 using namespace llvm;
 
+cl::opt<bool> NoKernelInfoEndLTO(
+    "no-kernel-info-end-lto",
+    cl::desc("remove the kernel-info pass at the end of the full LTO pipeline"),
+    cl::init(false), cl::Hidden);
+
 //---------------------------------------------------------------------------
 // TargetMachine Class
 //
diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyCFGStackify.cpp b/llvm/lib/Target/WebAssembly/WebAssemblyCFGStackify.cpp
index bdc1cc6d652ac..277d353d1db10 100644
--- a/llvm/lib/Target/WebAssembly/WebAssemblyCFGStackify.cpp
+++ b/llvm/lib/Target/WebAssembly/WebAssemblyCFGStackify.cpp
@@ -852,12 +852,19 @@ void WebAssemblyCFGStackify::placeTryTableMarker(MachineBasicBlock &MBB) {
   // Add a CATCH_*** clause to the TRY_TABLE. These are pseudo instructions
   // following the destination END_BLOCK to simulate block return values,
   // because we currently don't support them.
+  const auto &TLI =
+      *MF.getSubtarget<WebAssemblySubtarget>().getTargetLowering();
+  WebAssembly::BlockType PtrTy =
+      TLI.getPointerTy(MF.getDataLayout()) == MVT::i32
+          ? WebAssembly::BlockType::I32
+          : WebAssembly::BlockType::I64;
   auto *Catch = WebAssembly::findCatch(&MBB);
   switch (Catch->getOpcode()) {
   case WebAssembly::CATCH:
     // CATCH's destination block's return type is the extracted value type,
-    // which is currently i32 for all supported tags.
-    BlockMIB.addImm(int64_t(WebAssembly::BlockType::I32));
+    // which is currently the thrown value's pointer type for all supported
+    // tags.
+    BlockMIB.addImm(int64_t(PtrTy));
     TryTableMIB.addImm(wasm::WASM_OPCODE_CATCH);
     for (const auto &Use : Catch->uses()) {
       // The only use operand a CATCH can have is the tag symbol.
diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyExplicitLocals.cpp b/llvm/lib/Target/WebAssembly/WebAssemblyExplicitLocals.cpp
index d60b59e73b462..5c1f036abef5a 100644
--- a/llvm/lib/Target/WebAssembly/WebAssemblyExplicitLocals.cpp
+++ b/llvm/lib/Target/WebAssembly/WebAssemblyExplicitLocals.cpp
@@ -344,7 +344,7 @@ bool WebAssemblyExplicitLocals::runOnMachineFunction(MachineFunction &MF) {
           const TargetRegisterClass *RC = MRI.getRegClass(OldReg);
           Register NewReg = MRI.createVirtualRegister(RC);
           auto InsertPt = std::next(MI.getIterator());
-          if (UseEmpty[Register::virtReg2Index(OldReg)]) {
+          if (UseEmpty[OldReg.virtRegIndex()]) {
             unsigned Opc = getDropOpcode(RC);
             MachineInstr *Drop =
                 BuildMI(MBB, InsertPt, MI.getDebugLoc(), TII->get(Opc))
diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyISD.def b/llvm/lib/Target/WebAssembly/WebAssemblyISD.def
index 1cf0d13df1ff6..378ef2c8f250e 100644
--- a/llvm/lib/Target/WebAssembly/WebAssemblyISD.def
+++ b/llvm/lib/Target/WebAssembly/WebAssemblyISD.def
@@ -26,6 +26,7 @@ HANDLE_NODETYPE(Wrapper)
 HANDLE_NODETYPE(WrapperREL)
 HANDLE_NODETYPE(BR_IF)
 HANDLE_NODETYPE(BR_TABLE)
+HANDLE_NODETYPE(DOT)
 HANDLE_NODETYPE(SHUFFLE)
 HANDLE_NODETYPE(SWIZZLE)
 HANDLE_NODETYPE(VEC_SHL)
diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp b/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp
index 02db1b142a22b..fedad25c775e2 100644
--- a/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp
+++ b/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp
@@ -29,6 +29,7 @@
 #include "llvm/IR/DiagnosticInfo.h"
 #include "llvm/IR/DiagnosticPrinter.h"
 #include "llvm/IR/Function.h"
+#include "llvm/IR/IntrinsicInst.h"
 #include "llvm/IR/Intrinsics.h"
 #include "llvm/IR/IntrinsicsWebAssembly.h"
 #include "llvm/Support/ErrorHandling.h"
@@ -177,6 +178,10 @@ WebAssemblyTargetLowering::WebAssemblyTargetLowering(
 
   // SIMD-specific configuration
   if (Subtarget->hasSIMD128()) {
+
+    // Combine partial.reduce.add before legalization gets confused.
+    setTargetDAGCombine(ISD::INTRINSIC_WO_CHAIN);
+
     // Combine vector mask reductions into alltrue/anytrue
     setTargetDAGCombine(ISD::SETCC);
 
@@ -406,6 +411,35 @@ MVT WebAssemblyTargetLowering::getPointerMemTy(const DataLayout &DL,
   return TargetLowering::getPointerMemTy(DL, AS);
 }
 
+bool WebAssemblyTargetLowering::shouldExpandPartialReductionIntrinsic(
+    const IntrinsicInst *I) const {
+  if (I->getIntrinsicID() != Intrinsic::experimental_vector_partial_reduce_add)
+    return true;
+
+  EVT VT = EVT::getEVT(I->getType());
+  auto Op1 = I->getOperand(1);
+
+  if (auto *InputInst = dyn_cast<Instruction>(Op1)) {
+    if (InstructionOpcodeToISD(InputInst->getOpcode()) != ISD::MUL)
+      return true;
+
+    if (isa<Instruction>(InputInst->getOperand(0)) &&
+        isa<Instruction>(InputInst->getOperand(1))) {
+      // dot only supports signed inputs but also support lowering unsigned.
+      if (cast<Instruction>(InputInst->getOperand(0))->getOpcode() !=
+          cast<Instruction>(InputInst->getOperand(1))->getOpcode())
+        return true;
+
+      EVT Op1VT = EVT::getEVT(Op1->getType());
+      if (Op1VT.getVectorElementType() == VT.getVectorElementType() &&
+          ((VT.getVectorElementCount() * 2 == Op1VT.getVectorElementCount()) ||
+           (VT.getVectorElementCount() * 4 == Op1VT.getVectorElementCount())))
+        return false;
+    }
+  }
+  return true;
+}
+
 TargetLowering::AtomicExpansionKind
 WebAssemblyTargetLowering::shouldExpandAtomicRMWInIR(AtomicRMWInst *AI) const {
   // We have wasm instructions for these
@@ -2030,6 +2064,94 @@ SDValue WebAssemblyTargetLowering::LowerVASTART(SDValue Op,
                       MachinePointerInfo(SV));
 }
 
+// Try to lower partial.reduce.add to a dot or fallback to a sequence with
+// extmul and adds.
+SDValue performLowerPartialReduction(SDNode *N, SelectionDAG &DAG) {
+  assert(N->getOpcode() == ISD::INTRINSIC_WO_CHAIN);
+  if (N->getConstantOperandVal(0) !=
+      Intrinsic::experimental_vector_partial_reduce_add)
+    return SDValue();
+
+  assert(N->getValueType(0) == MVT::v4i32 && "can only support v4i32");
+  SDLoc DL(N);
+  SDValue Mul = N->getOperand(2);
+  assert(Mul->getOpcode() == ISD::MUL && "expected mul input");
+
+  SDValue ExtendLHS = Mul->getOperand(0);
+  SDValue ExtendRHS = Mul->getOperand(1);
+  assert((ISD::isExtOpcode(ExtendLHS.getOpcode()) &&
+          ISD::isExtOpcode(ExtendRHS.getOpcode())) &&
+         "expected widening mul");
+  assert(ExtendLHS.getOpcode() == ExtendRHS.getOpcode() &&
+         "expected mul to use the same extend for both operands");
+
+  SDValue ExtendInLHS = ExtendLHS->getOperand(0);
+  SDValue ExtendInRHS = ExtendRHS->getOperand(0);
+  bool IsSigned = ExtendLHS->getOpcode() == ISD::SIGN_EXTEND;
+
+  if (ExtendInLHS->getValueType(0) == MVT::v8i16) {
+    if (IsSigned) {
+      // i32x4.dot_i16x8_s
+      SDValue Dot = DAG.getNode(WebAssemblyISD::DOT, DL, MVT::v4i32,
+                                ExtendInLHS, ExtendInRHS);
+      return DAG.getNode(ISD::ADD, DL, MVT::v4i32, N->getOperand(1), Dot);
+    }
+
+    unsigned LowOpc = WebAssemblyISD::EXTEND_LOW_U;
+    unsigned HighOpc = WebAssemblyISD::EXTEND_HIGH_U;
+
+    // (add (add (extmul_low_sx lhs, rhs), (extmul_high_sx lhs, rhs)))
+    SDValue LowLHS = DAG.getNode(LowOpc, DL, MVT::v4i32, ExtendInLHS);
+    SDValue LowRHS = DAG.getNode(LowOpc, DL, MVT::v4i32, ExtendInRHS);
+    SDValue HighLHS = DAG.getNode(HighOpc, DL, MVT::v4i32, ExtendInLHS);
+    SDValue HighRHS = DAG.getNode(HighOpc, DL, MVT::v4i32, ExtendInRHS);
+
+    SDValue MulLow = DAG.getNode(ISD::MUL, DL, MVT::v4i32, LowLHS, LowRHS);
+    SDValue MulHigh = DAG.getNode(ISD::MUL, DL, MVT::v4i32, HighLHS, HighRHS);
+    SDValue Add = DAG.getNode(ISD::ADD, DL, MVT::v4i32, MulLow, MulHigh);
+    return DAG.getNode(ISD::ADD, DL, MVT::v4i32, N->getOperand(1), Add);
+  } else {
+    assert(ExtendInLHS->getValueType(0) == MVT::v16i8 &&
+           "expected v16i8 input types");
+    // Lower to a wider tree, using twice the operations compared to above.
+    if (IsSigned) {
+      // Use two dots
+      unsigned LowOpc = WebAssemblyISD::EXTEND_LOW_S;
+      unsigned HighOpc = WebAssemblyISD::EXTEND_HIGH_S;
+      SDValue LowLHS = DAG.getNode(LowOpc, DL, MVT::v8i16, ExtendInLHS);
+      SDValue LowRHS = DAG.getNode(LowOpc, DL, MVT::v8i16, ExtendInRHS);
+      SDValue HighLHS = DAG.getNode(HighOpc, DL, MVT::v8i16, ExtendInLHS);
+      SDValue HighRHS = DAG.getNode(HighOpc, DL, MVT::v8i16, ExtendInRHS);
+      SDValue DotLHS =
+          DAG.getNode(WebAssemblyISD::DOT, DL, MVT::v4i32, LowLHS, LowRHS);
+      SDValue DotRHS =
+          DAG.getNode(WebAssemblyISD::DOT, DL, MVT::v4i32, HighLHS, HighRHS);
+      SDValue Add = DAG.getNode(ISD::ADD, DL, MVT::v4i32, DotLHS, DotRHS);
+      return DAG.getNode(ISD::ADD, DL, MVT::v4i32, N->getOperand(1), Add);
+    }
+
+    unsigned LowOpc = WebAssemblyISD::EXTEND_LOW_U;
+    unsigned HighOpc = WebAssemblyISD::EXTEND_HIGH_U;
+    SDValue LowLHS = DAG.getNode(LowOpc, DL, MVT::v8i16, ExtendInLHS);
+    SDValue LowRHS = DAG.getNode(LowOpc, DL, MVT::v8i16, ExtendInRHS);
+    SDValue HighLHS = DAG.getNode(HighOpc, DL, MVT::v8i16, ExtendInLHS);
+    SDValue HighRHS = DAG.getNode(HighOpc, DL, MVT::v8i16, ExtendInRHS);
+
+    SDValue MulLow = DAG.getNode(ISD::MUL, DL, MVT::v8i16, LowLHS, LowRHS);
+    SDValue MulHigh = DAG.getNode(ISD::MUL, DL, MVT::v8i16, HighLHS, HighRHS);
+
+    SDValue LowLow = DAG.getNode(LowOpc, DL, MVT::v4i32, MulLow);
+    SDValue LowHigh = DAG.getNode(LowOpc, DL, MVT::v4i32, MulHigh);
+    SDValue HighLow = DAG.getNode(HighOpc, DL, MVT::v4i32, MulLow);
+    SDValue HighHigh = DAG.getNode(HighOpc, DL, MVT::v4i32, MulHigh);
+
+    SDValue AddLow = DAG.getNode(ISD::ADD, DL, MVT::v4i32, LowLow, HighLow);
+    SDValue AddHigh = DAG.getNode(ISD::ADD, DL, MVT::v4i32, LowHigh, HighHigh);
+    SDValue Add = DAG.getNode(ISD::ADD, DL, MVT::v4i32, AddLow, AddHigh);
+    return DAG.getNode(ISD::ADD, DL, MVT::v4i32, N->getOperand(1), Add);
+  }
+}
+
 SDValue WebAssemblyTargetLowering::LowerIntrinsic(SDValue Op,
                                                   SelectionDAG &DAG) const {
   MachineFunction &MF = DAG.getMachineFunction();
@@ -3126,5 +3248,7 @@ WebAssemblyTargetLowering::PerformDAGCombine(SDNode *N,
     return performVectorTruncZeroCombine(N, DCI);
   case ISD::TRUNCATE:
     return performTruncateCombine(N, DCI);
+  case ISD::INTRINSIC_WO_CHAIN:
+    return performLowerPartialReduction(N, DCI.DAG);
   }
 }
diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.h b/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.h
index d9ced1a1a5279..90d31e38a7076 100644
--- a/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.h
+++ b/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.h
@@ -45,6 +45,8 @@ class WebAssemblyTargetLowering final : public TargetLowering {
   /// right decision when generating code for different targets.
   const WebAssemblySubtarget *Subtarget;
 
+  bool
+  shouldExpandPartialReductionIntrinsic(const IntrinsicInst *I) const override;
   AtomicExpansionKind shouldExpandAtomicRMWInIR(AtomicRMWInst *) const override;
   bool shouldScalarizeBinop(SDValue VecOp) const override;
   FastISel *createFastISel(FunctionLoweringInfo &FuncInfo,
diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyInstrSIMD.td b/llvm/lib/Target/WebAssembly/WebAssemblyInstrSIMD.td
index 2c0543842a82b..14acc623ce24d 100644
--- a/llvm/lib/Target/WebAssembly/WebAssemblyInstrSIMD.td
+++ b/llvm/lib/Target/WebAssembly/WebAssemblyInstrSIMD.td
@@ -1147,11 +1147,15 @@ def : Pat<(wasm_shr_u
 }
 
 // Widening dot product: i32x4.dot_i16x8_s
+def dot_t : SDTypeProfile<1, 2, [SDTCisVT<0, v4i32>, SDTCisVT<1, v8i16>, SDTCisVT<2, v8i16>]>;
+def wasm_dot : SDNode<"WebAssemblyISD::DOT", dot_t>;
 let isCommutable = 1 in
 defm DOT : SIMD_I<(outs V128:$dst), (ins V128:$lhs, V128:$rhs), (outs), (ins),
                   [(set V128:$dst, (int_wasm_dot V128:$lhs, V128:$rhs))],
                   "i32x4.dot_i16x8_s\t$dst, $lhs, $rhs", "i32x4.dot_i16x8_s",
                   186>;
+def : Pat<(wasm_dot V128:$lhs, V128:$rhs),
+          (DOT $lhs, $rhs)>;
 
 // Extending multiplication: extmul_{low,high}_P, extmul_high
 def extend_t : SDTypeProfile<1, 1, [SDTCisVec<0>, SDTCisVec<1>]>;
diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyMCInstLower.cpp b/llvm/lib/Target/WebAssembly/WebAssemblyMCInstLower.cpp
index f09b29472cb6b..eed0b42863ee6 100644
--- a/llvm/lib/Target/WebAssembly/WebAssemblyMCInstLower.cpp
+++ b/llvm/lib/Target/WebAssembly/WebAssemblyMCInstLower.cpp
@@ -169,6 +169,13 @@ void WebAssemblyMCInstLower::lower(const MachineInstr *MI,
 
   const MCInstrDesc &Desc = MI->getDesc();
   unsigned NumVariadicDefs = MI->getNumExplicitDefs() - Desc.getNumDefs();
+  const MachineFunction *MF = MI->getMF();
+  const auto &TLI =
+      *MF->getSubtarget<WebAssemblySubtarget>().getTargetLowering();
+  wasm::ValType PtrTy = TLI.getPointerTy(MF->getDataLayout()) == MVT::i32
+                            ? wasm::ValType::I32
+                            : wasm::ValType::I64;
+
   for (unsigned I = 0, E = MI->getNumOperands(); I != E; ++I) {
     const MachineOperand &MO = MI->getOperand(I);
 
@@ -234,12 +241,12 @@ void WebAssemblyMCInstLower::lower(const MachineInstr *MI,
             //    return type of the parent function.
             // 2. (catch_ref ...) clause in try_table instruction. Currently all
             //    tags we support (cpp_exception and c_longjmp) throws a single
-            //    i32, so the multivalue signature for this case will be (i32,
-            //    exnref). Having MO_CATCH_BLOCK_SIG target flags means this is
-            //    a destination of a catch_ref.
-            if (MO.getTargetFlags() == WebAssemblyII::MO_CATCH_BLOCK_SIG)
-              Returns = {wasm::ValType::I32, wasm::ValType::EXNREF};
-            else
+            //    pointer, so the multivalue signature for this case will be
+            //    (ptr, exnref). Having MO_CATCH_BLOCK_SIG target flags means
+            //    this is a destination of a catch_ref.
+            if (MO.getTargetFlags() == WebAssemblyII::MO_CATCH_BLOCK_SIG) {
+              Returns = {PtrTy, wasm::ValType::EXNREF};
+            } else
               getFunctionReturns(MI, Returns);
             MCOp = lowerTypeIndexOperand(std::move(Returns),
                                          SmallVector<wasm::ValType, 4>());
diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyRegColoring.cpp b/llvm/lib/Target/WebAssembly/WebAssemblyRegColoring.cpp
index 7dc5c099c1270..6efe09d775c4b 100644
--- a/llvm/lib/Target/WebAssembly/WebAssemblyRegColoring.cpp
+++ b/llvm/lib/Target/WebAssembly/WebAssemblyRegColoring.cpp
@@ -311,8 +311,8 @@ bool WebAssemblyRegColoring::runOnMachineFunction(MachineFunction &MF) {
     // If we reassigned the stack pointer, update the debug frame base info.
     if (Old != New && MFI.isFrameBaseVirtual() && MFI.getFrameBaseVreg() == Old)
       MFI.setFrameBaseVreg(New);
-    LLVM_DEBUG(dbgs() << "Assigning vreg" << Register::virtReg2Index(LI->reg())
-                      << " to vreg" << Register::virtReg2Index(New) << "\n");
+    LLVM_DEBUG(dbgs() << "Assigning vreg " << printReg(LI->reg()) << " to vreg "
+                      << printReg(New) << "\n");
   }
   if (!Changed)
     return false;
diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyTargetTransformInfo.cpp b/llvm/lib/Target/WebAssembly/WebAssemblyTargetTransformInfo.cpp
index 3d678e5384166..e4dc38686a445 100644
--- a/llvm/lib/Target/WebAssembly/WebAssemblyTargetTransformInfo.cpp
+++ b/llvm/lib/Target/WebAssembly/WebAssemblyTargetTransformInfo.cpp
@@ -13,6 +13,8 @@
 //===----------------------------------------------------------------------===//
 
 #include "WebAssemblyTargetTransformInfo.h"
+
+#include "llvm/CodeGen/CostTable.h"
 using namespace llvm;
 
 #define DEBUG_TYPE "wasmtti"
@@ -51,8 +53,7 @@ TypeSize WebAssemblyTTIImpl::getRegisterBitWidth(
 InstructionCost WebAssemblyTTIImpl::getArithmeticInstrCost(
     unsigned Opcode, Type *Ty, TTI::TargetCostKind CostKind,
     TTI::OperandValueInfo Op1Info, TTI::OperandValueInfo Op2Info,
-    ArrayRef<const Value *> Args,
-    const Instruction *CxtI) {
+    ArrayRef<const Value *> Args, const Instruction *CxtI) {
 
   InstructionCost Cost =
       BasicTTIImplBase<WebAssemblyTTIImpl>::getArithmeticInstrCost(
@@ -78,6 +79,109 @@ InstructionCost WebAssemblyTTIImpl::getArithmeticInstrCost(
   return Cost;
 }
 
+InstructionCost WebAssemblyTTIImpl::getCastInstrCost(
+    unsigned Opcode, Type *Dst, Type *Src, TTI::CastContextHint CCH,
+    TTI::TargetCostKind CostKind, const Instruction *I) {
+  int ISD = TLI->InstructionOpcodeToISD(Opcode);
+  auto SrcTy = TLI->getValueType(DL, Src);
+  auto DstTy = TLI->getValueType(DL, Dst);
+
+  if (!SrcTy.isSimple() || !DstTy.isSimple()) {
+    return BaseT::getCastInstrCost(Opcode, Dst, Src, CCH, CostKind, I);
+  }
+
+  if (!ST->hasSIMD128()) {
+    return BaseT::getCastInstrCost(Opcode, Dst, Src, CCH, CostKind, I);
+  }
+
+  auto DstVT = DstTy.getSimpleVT();
+  auto SrcVT = SrcTy.getSimpleVT();
+
+  if (I && I->hasOneUser()) {
+    auto *SingleUser = cast<Instruction>(*I->user_begin());
+    int UserISD = TLI->InstructionOpcodeToISD(SingleUser->getOpcode());
+
+    // extmul_low support
+    if (UserISD == ISD::MUL &&
+        (ISD == ISD::ZERO_EXTEND || ISD == ISD::SIGN_EXTEND)) {
+      // Free low extensions.
+      if ((SrcVT == MVT::v8i8 && DstVT == MVT::v8i16) ||
+          (SrcVT == MVT::v4i16 && DstVT == MVT::v4i32) ||
+          (SrcVT == MVT::v2i32 && DstVT == MVT::v2i64)) {
+        return 0;
+      }
+      // Will require an additional extlow operation for the intermediate
+      // i16/i32 value.
+      if ((SrcVT == MVT::v4i8 && DstVT == MVT::v4i32) ||
+          (SrcVT == MVT::v2i16 && DstVT == MVT::v2i64)) {
+        return 1;
+      }
+    }
+  }
+
+  // extend_low
+  static constexpr TypeConversionCostTblEntry ConversionTbl[] = {
+      {ISD::SIGN_EXTEND, MVT::v2i64, MVT::v2i32, 1},
+      {ISD::ZERO_EXTEND, MVT::v2i64, MVT::v2i32, 1},
+      {ISD::SIGN_EXTEND, MVT::v4i32, MVT::v4i16, 1},
+      {ISD::ZERO_EXTEND, MVT::v4i32, MVT::v4i16, 1},
+      {ISD::SIGN_EXTEND, MVT::v8i16, MVT::v8i8, 1},
+      {ISD::ZERO_EXTEND, MVT::v8i16, MVT::v8i8, 1},
+      {ISD::SIGN_EXTEND, MVT::v2i64, MVT::v2i16, 2},
+      {ISD::ZERO_EXTEND, MVT::v2i64, MVT::v2i16, 2},
+      {ISD::SIGN_EXTEND, MVT::v4i32, MVT::v4i8, 2},
+      {ISD::ZERO_EXTEND, MVT::v4i32, MVT::v4i8, 2},
+  };
+
+  if (const auto *Entry =
+          ConvertCostTableLookup(ConversionTbl, ISD, DstVT, SrcVT)) {
+    return Entry->Cost;
+  }
+
+  return BaseT::getCastInstrCost(Opcode, Dst, Src, CCH, CostKind, I);
+}
+
+InstructionCost WebAssemblyTTIImpl::getMemoryOpCost(
+    unsigned Opcode, Type *Ty, MaybeAlign Alignment, unsigned AddressSpace,
+    TTI::TargetCostKind CostKind, TTI::OperandValueInfo OpInfo,
+    const Instruction *I) {
+  if (!ST->hasSIMD128() || !isa<FixedVectorType>(Ty)) {
+    return BaseT::getMemoryOpCost(Opcode, Ty, Alignment, AddressSpace,
+                                  CostKind);
+  }
+
+  int ISD = TLI->InstructionOpcodeToISD(Opcode);
+  if (ISD != ISD::LOAD) {
+    return BaseT::getMemoryOpCost(Opcode, Ty, Alignment, AddressSpace,
+                                  CostKind);
+  }
+
+  EVT VT = TLI->getValueType(DL, Ty, true);
+  // Type legalization can't handle structs
+  if (VT == MVT::Other)
+    return BaseT::getMemoryOpCost(Opcode, Ty, Alignment, AddressSpace,
+                                  CostKind);
+
+  auto LT = getTypeLegalizationCost(Ty);
+  if (!LT.first.isValid())
+    return InstructionCost::getInvalid();
+
+  // 128-bit loads are a single instruction. 32-bit and 64-bit vector loads can
+  // be lowered to load32_zero and load64_zero respectively. Assume SIMD loads
+  // are twice as expensive as scalar.
+  unsigned width = VT.getSizeInBits();
+  switch (width) {
+  default:
+    break;
+  case 32:
+  case 64:
+  case 128:
+    return 2;
+  }
+
+  return BaseT::getMemoryOpCost(Opcode, Ty, Alignment, AddressSpace, CostKind);
+}
+
 InstructionCost
 WebAssemblyTTIImpl::getVectorInstrCost(unsigned Opcode, Type *Val,
                                        TTI::TargetCostKind CostKind,
@@ -92,6 +196,53 @@ WebAssemblyTTIImpl::getVectorInstrCost(unsigned Opcode, Type *Val,
   return Cost;
 }
 
+InstructionCost WebAssemblyTTIImpl::getPartialReductionCost(
+    unsigned Opcode, Type *InputTypeA, Type *InputTypeB, Type *AccumType,
+    ElementCount VF, TTI::PartialReductionExtendKind OpAExtend,
+    TTI::PartialReductionExtendKind OpBExtend,
+    std::optional<unsigned> BinOp) const {
+  InstructionCost Invalid = InstructionCost::getInvalid();
+  if (!VF.isFixed() || !ST->hasSIMD128())
+    return Invalid;
+
+  InstructionCost Cost(TTI::TCC_Basic);
+
+  // Possible options:
+  // - i16x8.extadd_pairwise_i8x16_sx
+  // - i32x4.extadd_pairwise_i16x8_sx
+  // - i32x4.dot_i16x8_s
+  // Only try to support dot, for now.
+
+  if (Opcode != Instruction::Add)
+    return Invalid;
+
+  if (!BinOp || *BinOp != Instruction::Mul)
+    return Invalid;
+
+  if (InputTypeA != InputTypeB)
+    return Invalid;
+
+  if (OpAExtend != OpBExtend)
+    return Invalid;
+
+  EVT InputEVT = EVT::getEVT(InputTypeA);
+  EVT AccumEVT = EVT::getEVT(AccumType);
+
+  // TODO: Add i64 accumulator.
+  if (AccumEVT != MVT::i32)
+    return Invalid;
+
+  // Signed inputs can lower to dot
+  if (InputEVT == MVT::i16 && VF.getFixedValue() == 8)
+    return OpAExtend == TTI::PR_SignExtend ? Cost : Cost * 2;
+
+  // Double the size of the lowered sequence.
+  if (InputEVT == MVT::i8 && VF.getFixedValue() == 16)
+    return OpAExtend == TTI::PR_SignExtend ? Cost * 2 : Cost * 4;
+
+  return Invalid;
+}
+
 TTI::ReductionShuffle WebAssemblyTTIImpl::getPreferredExpandedReductionShuffle(
     const IntrinsicInst *II) const {
 
diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyTargetTransformInfo.h b/llvm/lib/Target/WebAssembly/WebAssemblyTargetTransformInfo.h
index 9691120b2e531..4d498b154c521 100644
--- a/llvm/lib/Target/WebAssembly/WebAssemblyTargetTransformInfo.h
+++ b/llvm/lib/Target/WebAssembly/WebAssemblyTargetTransformInfo.h
@@ -64,11 +64,26 @@ class WebAssemblyTTIImpl final : public BasicTTIImplBase<WebAssemblyTTIImpl> {
       TTI::OperandValueInfo Op1Info = {TTI::OK_AnyValue, TTI::OP_None},
       TTI::OperandValueInfo Op2Info = {TTI::OK_AnyValue, TTI::OP_None},
       ArrayRef<const Value *> Args = {}, const Instruction *CxtI = nullptr);
+
+  InstructionCost getCastInstrCost(unsigned Opcode, Type *Dst, Type *Src,
+                                   TTI::CastContextHint CCH,
+                                   TTI::TargetCostKind CostKind,
+                                   const Instruction *I = nullptr);
+  InstructionCost getMemoryOpCost(
+      unsigned Opcode, Type *Src, MaybeAlign Alignment, unsigned AddressSpace,
+      TTI::TargetCostKind CostKind,
+      TTI::OperandValueInfo OpInfo = {TTI::OK_AnyValue, TTI::OP_None},
+      const Instruction *I = nullptr);
   using BaseT::getVectorInstrCost;
   InstructionCost getVectorInstrCost(unsigned Opcode, Type *Val,
                                      TTI::TargetCostKind CostKind,
                                      unsigned Index, Value *Op0, Value *Op1);
-
+  InstructionCost
+  getPartialReductionCost(unsigned Opcode, Type *InputTypeA, Type *InputTypeB,
+                          Type *AccumType, ElementCount VF,
+                          TTI::PartialReductionExtendKind OpAExtend,
+                          TTI::PartialReductionExtendKind OpBExtend,
+                          std::optional<unsigned> BinOp = std::nullopt) const;
   TTI::ReductionShuffle
   getPreferredExpandedReductionShuffle(const IntrinsicInst *II) const;
 
diff --git a/llvm/lib/Target/X86/GISel/X86InstructionSelector.cpp b/llvm/lib/Target/X86/GISel/X86InstructionSelector.cpp
index ee456a11d5844..d7f315d82b832 100644
--- a/llvm/lib/Target/X86/GISel/X86InstructionSelector.cpp
+++ b/llvm/lib/Target/X86/GISel/X86InstructionSelector.cpp
@@ -1048,6 +1048,13 @@ bool X86InstructionSelector::selectFCmp(MachineInstr &I,
     break;
   }
 
+  assert((LhsReg.isVirtual() && RhsReg.isVirtual()) &&
+         "Both arguments of FCMP need to be virtual!");
+  auto *LhsBank = RBI.getRegBank(LhsReg, MRI, TRI);
+  [[maybe_unused]] auto *RhsBank = RBI.getRegBank(RhsReg, MRI, TRI);
+  assert((LhsBank == RhsBank) &&
+         "Both banks assigned to FCMP arguments need to be same!");
+
   // Compute the opcode for the CMP instruction.
   unsigned OpCmp;
   LLT Ty = MRI.getType(LhsReg);
@@ -1055,10 +1062,15 @@ bool X86InstructionSelector::selectFCmp(MachineInstr &I,
   default:
     return false;
   case 32:
-    OpCmp = X86::UCOMISSrr;
+    OpCmp = LhsBank->getID() == X86::PSRRegBankID ? X86::UCOM_FpIr32
+                                                  : X86::UCOMISSrr;
     break;
   case 64:
-    OpCmp = X86::UCOMISDrr;
+    OpCmp = LhsBank->getID() == X86::PSRRegBankID ? X86::UCOM_FpIr64
+                                                  : X86::UCOMISDrr;
+    break;
+  case 80:
+    OpCmp = X86::UCOM_FpIr80;
     break;
   }
 
diff --git a/llvm/lib/Target/X86/GISel/X86LegalizerInfo.cpp b/llvm/lib/Target/X86/GISel/X86LegalizerInfo.cpp
index bab7fe9d25e44..c73179c4c9466 100644
--- a/llvm/lib/Target/X86/GISel/X86LegalizerInfo.cpp
+++ b/llvm/lib/Target/X86/GISel/X86LegalizerInfo.cpp
@@ -99,6 +99,10 @@ X86LegalizerInfo::X86LegalizerInfo(const X86Subtarget &STI,
       .widenScalarToNextPow2(0, /*Min=*/8)
       .clampScalar(0, s8, sMaxScalar);
 
+  getActionDefinitionsBuilder(G_LROUND).libcall();
+
+  getActionDefinitionsBuilder(G_LLROUND).libcall();
+
   // merge/unmerge
   for (unsigned Op : {G_MERGE_VALUES, G_UNMERGE_VALUES}) {
     unsigned BigTyIdx = Op == G_MERGE_VALUES ? 0 : 1;
@@ -448,10 +452,9 @@ X86LegalizerInfo::X86LegalizerInfo(const X86Subtarget &STI,
 
   // fp comparison
   getActionDefinitionsBuilder(G_FCMP)
-      .legalIf([=](const LegalityQuery &Query) {
-        return (HasSSE1 && typePairInSet(0, 1, {{s8, s32}})(Query)) ||
-               (HasSSE2 && typePairInSet(0, 1, {{s8, s64}})(Query));
-      })
+      .legalFor(HasSSE1 || UseX87, {s8, s32})
+      .legalFor(HasSSE2 || UseX87, {s8, s64})
+      .legalFor(UseX87, {s8, s80})
       .clampScalar(0, s8, s8)
       .clampScalar(1, s32, HasSSE2 ? s64 : s32)
       .widenScalarToNextPow2(1);
diff --git a/llvm/lib/Target/X86/GISel/X86RegisterBankInfo.cpp b/llvm/lib/Target/X86/GISel/X86RegisterBankInfo.cpp
index 43c0145ec8e2a..42faf4299c6d5 100644
--- a/llvm/lib/Target/X86/GISel/X86RegisterBankInfo.cpp
+++ b/llvm/lib/Target/X86/GISel/X86RegisterBankInfo.cpp
@@ -321,8 +321,8 @@ X86RegisterBankInfo::getInstrMapping(const MachineInstr &MI) const {
 
     unsigned Size = Ty1.getSizeInBits();
     (void)Size;
-    assert((Size == 32 || Size == 64) && "Unsupported size for G_FCMP");
-
+    assert((Size == 32 || Size == 64 || Size == 80) &&
+           "Unsupported size for G_FCMP");
     auto FpRegBank = getPartialMappingIdx(MI, Ty1, /* isFP= */ true);
     OpRegBankIdx = {PMI_GPR8,
                     /* Predicate */ PMI_None, FpRegBank, FpRegBank};
diff --git a/llvm/lib/Target/X86/MCTargetDesc/X86MCAsmInfo.cpp b/llvm/lib/Target/X86/MCTargetDesc/X86MCAsmInfo.cpp
index 39b0f7c4c4c1e..fff518e7daf07 100644
--- a/llvm/lib/Target/X86/MCTargetDesc/X86MCAsmInfo.cpp
+++ b/llvm/lib/Target/X86/MCTargetDesc/X86MCAsmInfo.cpp
@@ -147,7 +147,7 @@ X86MCAsmInfoMicrosoftMASM::X86MCAsmInfoMicrosoftMASM(const Triple &Triple)
 void X86MCAsmInfoGNUCOFF::anchor() { }
 
 X86MCAsmInfoGNUCOFF::X86MCAsmInfoGNUCOFF(const Triple &Triple) {
-  assert((Triple.isOSWindows() || Triple.isUEFI()) &&
+  assert(Triple.isOSWindowsOrUEFI() &&
          "Windows and UEFI are the only supported COFF targets");
   if (Triple.getArch() == Triple::x86_64) {
     PrivateGlobalPrefix = ".L";
diff --git a/llvm/lib/Target/X86/X86AsmPrinter.cpp b/llvm/lib/Target/X86/X86AsmPrinter.cpp
index f01e47b41cf5e..79aa898e18bfa 100644
--- a/llvm/lib/Target/X86/X86AsmPrinter.cpp
+++ b/llvm/lib/Target/X86/X86AsmPrinter.cpp
@@ -181,8 +181,29 @@ void X86AsmPrinter::emitKCFITypeId(const MachineFunction &MF) {
   // Embed the type hash in the X86::MOV32ri instruction to avoid special
   // casing object file parsers.
   EmitKCFITypePadding(MF);
+  unsigned DestReg = X86::EAX;
+
+  if (F.getParent()->getModuleFlag("kcfi-arity")) {
+    // The ArityToRegMap assumes the 64-bit Linux kernel ABI
+    [[maybe_unused]] const auto &Triple = MF.getTarget().getTargetTriple();
+    assert(Triple.isArch64Bit() && Triple.isOSLinux());
+
+    // Determine the function's arity (i.e., the number of arguments) at the ABI
+    // level by counting the number of parameters that are passed
+    // as registers, such as pointers and 64-bit (or smaller) integers. The
+    // Linux x86-64 ABI allows up to 6 parameters to be passed in GPRs.
+    // Additional parameters or parameters larger than 64 bits may be passed on
+    // the stack, in which case the arity is denoted as 7.
+    const unsigned ArityToRegMap[8] = {X86::EAX, X86::ECX, X86::EDX, X86::EBX,
+                                       X86::ESP, X86::EBP, X86::ESI, X86::EDI};
+    int Arity = MF.getInfo<X86MachineFunctionInfo>()->getArgumentStackSize() > 0
+                    ? 7
+                    : MF.getRegInfo().liveins().size();
+    DestReg = ArityToRegMap[Arity];
+  }
+
   EmitAndCountInstruction(MCInstBuilder(X86::MOV32ri)
-                              .addReg(X86::EAX)
+                              .addReg(DestReg)
                               .addImm(MaskKCFIType(Type->getZExtValue())));
 
   if (MAI->hasDotTypeDotSizeDirective()) {
diff --git a/llvm/lib/Target/X86/X86AvoidStoreForwardingBlocks.cpp b/llvm/lib/Target/X86/X86AvoidStoreForwardingBlocks.cpp
index 1919923f41bb2..fe9bacb6856a6 100644
--- a/llvm/lib/Target/X86/X86AvoidStoreForwardingBlocks.cpp
+++ b/llvm/lib/Target/X86/X86AvoidStoreForwardingBlocks.cpp
@@ -625,13 +625,10 @@ static bool isBlockingStore(int64_t LoadDispImm, unsigned LoadSize,
 static void
 updateBlockingStoresDispSizeMap(DisplacementSizeMap &BlockingStoresDispSizeMap,
                                 int64_t DispImm, unsigned Size) {
-  if (BlockingStoresDispSizeMap.count(DispImm)) {
-    // Choose the smallest blocking store starting at this displacement.
-    if (BlockingStoresDispSizeMap[DispImm] > Size)
-      BlockingStoresDispSizeMap[DispImm] = Size;
-
-  } else
-    BlockingStoresDispSizeMap[DispImm] = Size;
+  auto [It, Inserted] = BlockingStoresDispSizeMap.try_emplace(DispImm, Size);
+  // Choose the smallest blocking store starting at this displacement.
+  if (!Inserted && It->second > Size)
+    It->second = Size;
 }
 
 // Remove blocking stores contained in each other.
diff --git a/llvm/lib/Target/X86/X86ExpandPseudo.cpp b/llvm/lib/Target/X86/X86ExpandPseudo.cpp
index 78db8413e62c9..c202f7fa93db6 100644
--- a/llvm/lib/Target/X86/X86ExpandPseudo.cpp
+++ b/llvm/lib/Target/X86/X86ExpandPseudo.cpp
@@ -284,7 +284,7 @@ bool X86ExpandPseudo::expandMI(MachineBasicBlock &MBB,
     // Adjust stack pointer.
     int StackAdj = StackAdjust.getImm();
     int MaxTCDelta = X86FI->getTCReturnAddrDelta();
-    int Offset = 0;
+    int64_t Offset = 0;
     assert(MaxTCDelta <= 0 && "MaxTCDelta should never be positive");
 
     // Incoporate the retaddr area.
@@ -297,7 +297,7 @@ bool X86ExpandPseudo::expandMI(MachineBasicBlock &MBB,
 
     if (Offset) {
       // Check for possible merge with preceding ADD instruction.
-      Offset += X86FL->mergeSPUpdates(MBB, MBBI, true);
+      Offset = X86FL->mergeSPAdd(MBB, MBBI, Offset, true);
       X86FL->emitSPUpdate(MBB, MBBI, DL, Offset, /*InEpilogue=*/true);
     }
 
diff --git a/llvm/lib/Target/X86/X86FastPreTileConfig.cpp b/llvm/lib/Target/X86/X86FastPreTileConfig.cpp
index 3812ea0ebd3f3..4486a1cae782c 100644
--- a/llvm/lib/Target/X86/X86FastPreTileConfig.cpp
+++ b/llvm/lib/Target/X86/X86FastPreTileConfig.cpp
@@ -136,12 +136,12 @@ int X86FastPreTileConfig::getStackSpaceFor(Register VirtReg) {
 /// If \p VirtReg live out of the current MBB, it must live out of the current
 /// config
 bool X86FastPreTileConfig::mayLiveOut(Register VirtReg, MachineInstr *CfgMI) {
-  if (MayLiveAcrossBlocks.test(Register::virtReg2Index(VirtReg)))
+  if (MayLiveAcrossBlocks.test(VirtReg.virtRegIndex()))
     return true;
 
   for (const MachineInstr &UseInst : MRI->use_nodbg_instructions(VirtReg)) {
     if (UseInst.getParent() != MBB) {
-      MayLiveAcrossBlocks.set(Register::virtReg2Index(VirtReg));
+      MayLiveAcrossBlocks.set(VirtReg.virtRegIndex());
       return true;
     }
 
@@ -150,7 +150,7 @@ bool X86FastPreTileConfig::mayLiveOut(Register VirtReg, MachineInstr *CfgMI) {
     // tile register.
     if (CfgMI) {
       if (dominates(*MBB, *CfgMI, UseInst)) {
-        MayLiveAcrossBlocks.set(Register::virtReg2Index(VirtReg));
+        MayLiveAcrossBlocks.set(VirtReg.virtRegIndex());
         return true;
       }
     }
@@ -355,7 +355,7 @@ void X86FastPreTileConfig::convertPHI(MachineBasicBlock *MBB,
     // Mark it as liveout, so that it will be spilled when visit
     // the incoming MBB. Otherwise since phi will be deleted, it
     // would miss spill when visit incoming MBB.
-    MayLiveAcrossBlocks.set(Register::virtReg2Index(InTileReg));
+    MayLiveAcrossBlocks.set(InTileReg.virtRegIndex());
     MachineBasicBlock *InMBB = PHI.getOperand(I + 1).getMBB();
 
     MachineInstr *TileDefMI = MRI->getVRegDef(InTileReg);
diff --git a/llvm/lib/Target/X86/X86FrameLowering.cpp b/llvm/lib/Target/X86/X86FrameLowering.cpp
index 47cc6a18ef843..50c56c9dd08b3 100644
--- a/llvm/lib/Target/X86/X86FrameLowering.cpp
+++ b/llvm/lib/Target/X86/X86FrameLowering.cpp
@@ -223,6 +223,8 @@ flagsNeedToBePreservedBeforeTheTerminators(const MachineBasicBlock &MBB) {
   return false;
 }
 
+constexpr int64_t MaxSPChunk = (1LL << 31) - 1;
+
 /// emitSPUpdate - Emit a series of instructions to increment / decrement the
 /// stack pointer by a constant value.
 void X86FrameLowering::emitSPUpdate(MachineBasicBlock &MBB,
@@ -234,7 +236,15 @@ void X86FrameLowering::emitSPUpdate(MachineBasicBlock &MBB,
   MachineInstr::MIFlag Flag =
       isSub ? MachineInstr::FrameSetup : MachineInstr::FrameDestroy;
 
-  uint64_t Chunk = (1LL << 31) - 1;
+  if (!Uses64BitFramePtr && !isUInt<32>(Offset)) {
+    // We're being asked to adjust a 32-bit stack pointer by 4 GiB or more.
+    // This might be unreachable code, so don't complain now; just trap if
+    // it's reached at runtime.
+    BuildMI(MBB, MBBI, DL, TII.get(X86::TRAP));
+    return;
+  }
+
+  uint64_t Chunk = MaxSPChunk;
 
   MachineFunction &MF = *MBB.getParent();
   const X86Subtarget &STI = MF.getSubtarget<X86Subtarget>();
@@ -383,12 +393,15 @@ MachineInstrBuilder X86FrameLowering::BuildStackAdjustment(
   return MI;
 }
 
-int X86FrameLowering::mergeSPUpdates(MachineBasicBlock &MBB,
-                                     MachineBasicBlock::iterator &MBBI,
-                                     bool doMergeWithPrevious) const {
+template <typename FoundT, typename CalcT>
+int64_t X86FrameLowering::mergeSPUpdates(MachineBasicBlock &MBB,
+                                         MachineBasicBlock::iterator &MBBI,
+                                         FoundT FoundStackAdjust,
+                                         CalcT CalcNewOffset,
+                                         bool doMergeWithPrevious) const {
   if ((doMergeWithPrevious && MBBI == MBB.begin()) ||
       (!doMergeWithPrevious && MBBI == MBB.end()))
-    return 0;
+    return CalcNewOffset(0);
 
   MachineBasicBlock::iterator PI = doMergeWithPrevious ? std::prev(MBBI) : MBBI;
 
@@ -407,27 +420,38 @@ int X86FrameLowering::mergeSPUpdates(MachineBasicBlock &MBB,
   if (doMergeWithPrevious && PI != MBB.begin() && PI->isCFIInstruction())
     PI = std::prev(PI);
 
-  unsigned Opc = PI->getOpcode();
-  int Offset = 0;
-
-  if ((Opc == X86::ADD64ri32 || Opc == X86::ADD32ri) &&
-      PI->getOperand(0).getReg() == StackPtr) {
-    assert(PI->getOperand(1).getReg() == StackPtr);
-    Offset = PI->getOperand(2).getImm();
-  } else if ((Opc == X86::LEA32r || Opc == X86::LEA64_32r) &&
-             PI->getOperand(0).getReg() == StackPtr &&
-             PI->getOperand(1).getReg() == StackPtr &&
-             PI->getOperand(2).getImm() == 1 &&
-             PI->getOperand(3).getReg() == X86::NoRegister &&
-             PI->getOperand(5).getReg() == X86::NoRegister) {
-    // For LEAs we have: def = lea SP, FI, noreg, Offset, noreg.
-    Offset = PI->getOperand(4).getImm();
-  } else if ((Opc == X86::SUB64ri32 || Opc == X86::SUB32ri) &&
-             PI->getOperand(0).getReg() == StackPtr) {
-    assert(PI->getOperand(1).getReg() == StackPtr);
-    Offset = -PI->getOperand(2).getImm();
-  } else
-    return 0;
+  int64_t Offset = 0;
+  for (;;) {
+    unsigned Opc = PI->getOpcode();
+
+    if ((Opc == X86::ADD64ri32 || Opc == X86::ADD32ri) &&
+        PI->getOperand(0).getReg() == StackPtr) {
+      assert(PI->getOperand(1).getReg() == StackPtr);
+      Offset = PI->getOperand(2).getImm();
+    } else if ((Opc == X86::LEA32r || Opc == X86::LEA64_32r) &&
+               PI->getOperand(0).getReg() == StackPtr &&
+               PI->getOperand(1).getReg() == StackPtr &&
+               PI->getOperand(2).getImm() == 1 &&
+               PI->getOperand(3).getReg() == X86::NoRegister &&
+               PI->getOperand(5).getReg() == X86::NoRegister) {
+      // For LEAs we have: def = lea SP, FI, noreg, Offset, noreg.
+      Offset = PI->getOperand(4).getImm();
+    } else if ((Opc == X86::SUB64ri32 || Opc == X86::SUB32ri) &&
+               PI->getOperand(0).getReg() == StackPtr) {
+      assert(PI->getOperand(1).getReg() == StackPtr);
+      Offset = -PI->getOperand(2).getImm();
+    } else
+      return CalcNewOffset(0);
+
+    FoundStackAdjust(PI, Offset);
+    if (std::abs((int64_t)CalcNewOffset(Offset)) < MaxSPChunk)
+      break;
+
+    if (doMergeWithPrevious ? (PI == MBB.begin()) : (PI == MBB.end()))
+      return CalcNewOffset(0);
+
+    PI = doMergeWithPrevious ? std::prev(PI) : std::next(PI);
+  }
 
   PI = MBB.erase(PI);
   if (PI != MBB.end() && PI->isCFIInstruction()) {
@@ -440,7 +464,16 @@ int X86FrameLowering::mergeSPUpdates(MachineBasicBlock &MBB,
   if (!doMergeWithPrevious)
     MBBI = skipDebugInstructionsForward(PI, MBB.end());
 
-  return Offset;
+  return CalcNewOffset(Offset);
+}
+
+int64_t X86FrameLowering::mergeSPAdd(MachineBasicBlock &MBB,
+                                     MachineBasicBlock::iterator &MBBI,
+                                     int64_t AddOffset,
+                                     bool doMergeWithPrevious) const {
+  return mergeSPUpdates(
+      MBB, MBBI, [AddOffset](int64_t Offset) { return AddOffset + Offset; },
+      doMergeWithPrevious);
 }
 
 void X86FrameLowering::BuildCFI(MachineBasicBlock &MBB,
@@ -829,10 +862,7 @@ void X86FrameLowering::emitStackProbeInlineGenericLoop(
           .addReg(StackPtr)
           .setMIFlag(MachineInstr::FrameSetup);
     } else {
-      // We're being asked to probe a stack frame that's 4 GiB or larger,
-      // but our stack pointer is only 32 bits.  This might be unreachable
-      // code, so don't complain now; just trap if it's reached at runtime.
-      BuildMI(MBB, MBBI, DL, TII.get(X86::TRAP));
+      llvm_unreachable("Offset too large for 32-bit stack pointer");
     }
 
     // while in the loop, use loop-invariant reg for CFI,
@@ -1970,8 +2000,10 @@ void X86FrameLowering::emitPrologue(MachineFunction &MF,
 
   // If there is an SUB32ri of ESP immediately before this instruction, merge
   // the two. This can be the case when tail call elimination is enabled and
-  // the callee has more arguments then the caller.
-  NumBytes -= mergeSPUpdates(MBB, MBBI, true);
+  // the callee has more arguments than the caller.
+  NumBytes = mergeSPUpdates(
+      MBB, MBBI, [NumBytes](int64_t Offset) { return NumBytes - Offset; },
+      true);
 
   // Adjust stack pointer: ESP -= numbytes.
 
@@ -2452,7 +2484,7 @@ void X86FrameLowering::emitEpilogue(MachineFunction &MF,
   if (HasFP) {
     if (X86FI->hasSwiftAsyncContext()) {
       // Discard the context.
-      int Offset = 16 + mergeSPUpdates(MBB, MBBI, true);
+      int64_t Offset = mergeSPAdd(MBB, MBBI, 16, true);
       emitSPUpdate(MBB, MBBI, DL, Offset, /*InEpilogue*/ true);
     }
     // Pop EBP.
@@ -2526,7 +2558,7 @@ void X86FrameLowering::emitEpilogue(MachineFunction &MF,
   // If there is an ADD32ri or SUB32ri of ESP immediately before this
   // instruction, merge the two instructions.
   if (NumBytes || MFI.hasVarSizedObjects())
-    NumBytes += mergeSPUpdates(MBB, MBBI, true);
+    NumBytes = mergeSPAdd(MBB, MBBI, NumBytes, true);
 
   // If dynamic alloca is used, then reset esp to point to the last callee-saved
   // slot before popping them off! Same applies for the case, when stack was
@@ -2573,14 +2605,8 @@ void X86FrameLowering::emitEpilogue(MachineFunction &MF,
     --MBBI;
   }
 
-  // Windows unwinder will not invoke function's exception handler if IP is
-  // either in prologue or in epilogue.  This behavior causes a problem when a
-  // call immediately precedes an epilogue, because the return address points
-  // into the epilogue.  To cope with that, we insert an epilogue marker here,
-  // then replace it with a 'nop' if it ends up immediately after a CALL in the
-  // final emitted code.
   if (NeedsWin64CFI && MF.hasWinCFI())
-    BuildMI(MBB, MBBI, DL, TII.get(X86::SEH_Epilogue));
+    BuildMI(MBB, MBBI, DL, TII.get(X86::SEH_BeginEpilogue));
 
   if (!HasFP && NeedsDwarfCFI) {
     MBBI = FirstCSPop;
@@ -2613,11 +2639,11 @@ void X86FrameLowering::emitEpilogue(MachineFunction &MF,
 
   if (Terminator == MBB.end() || !isTailCallOpcode(Terminator->getOpcode())) {
     // Add the return addr area delta back since we are not tail calling.
-    int Offset = -1 * X86FI->getTCReturnAddrDelta();
+    int64_t Offset = -1 * X86FI->getTCReturnAddrDelta();
     assert(Offset >= 0 && "TCDelta should never be positive");
     if (Offset) {
       // Check for possible merge with preceding ADD instruction.
-      Offset += mergeSPUpdates(MBB, Terminator, true);
+      Offset = mergeSPAdd(MBB, Terminator, Offset, true);
       emitSPUpdate(MBB, Terminator, DL, Offset, /*InEpilogue=*/true);
     }
   }
@@ -2625,6 +2651,9 @@ void X86FrameLowering::emitEpilogue(MachineFunction &MF,
   // Emit tilerelease for AMX kernel.
   if (X86FI->getAMXProgModel() == AMXProgModelEnum::ManagedRA)
     BuildMI(MBB, Terminator, DL, TII.get(X86::TILERELEASE));
+
+  if (NeedsWin64CFI && MF.hasWinCFI())
+    BuildMI(MBB, Terminator, DL, TII.get(X86::SEH_EndEpilogue));
 }
 
 StackOffset X86FrameLowering::getFrameIndexReference(const MachineFunction &MF,
@@ -3057,10 +3086,7 @@ bool X86FrameLowering::spillCalleeSavedRegisters(
     const TargetRegisterClass *RC = TRI->getMinimalPhysRegClass(Reg, VT);
 
     TII.storeRegToStackSlot(MBB, MI, Reg, true, I.getFrameIdx(), RC, TRI,
-                            Register());
-    --MI;
-    MI->setFlag(MachineInstr::FrameSetup);
-    ++MI;
+                            Register(), MachineInstr::FrameSetup);
   }
 
   return true;
@@ -3815,13 +3841,24 @@ MachineBasicBlock::iterator X86FrameLowering::eliminateCallFramePseudoInstr(
 
     // Add Amount to SP to destroy a frame, or subtract to setup.
     int64_t StackAdjustment = isDestroy ? Amount : -Amount;
+    int64_t CfaAdjustment = StackAdjustment;
 
     if (StackAdjustment) {
       // Merge with any previous or following adjustment instruction. Note: the
       // instructions merged with here do not have CFI, so their stack
-      // adjustments do not feed into CfaAdjustment.
-      StackAdjustment += mergeSPUpdates(MBB, InsertPos, true);
-      StackAdjustment += mergeSPUpdates(MBB, InsertPos, false);
+      // adjustments do not feed into CfaAdjustment
+
+      auto CalcCfaAdjust = [&CfaAdjustment](MachineBasicBlock::iterator PI,
+                                            int64_t Offset) {
+        CfaAdjustment += Offset;
+      };
+      auto CalcNewOffset = [&StackAdjustment](int64_t Offset) {
+        return StackAdjustment + Offset;
+      };
+      StackAdjustment =
+          mergeSPUpdates(MBB, InsertPos, CalcCfaAdjust, CalcNewOffset, true);
+      StackAdjustment =
+          mergeSPUpdates(MBB, InsertPos, CalcCfaAdjust, CalcNewOffset, false);
 
       if (StackAdjustment) {
         if (!(F.hasMinSize() &&
@@ -3831,7 +3868,7 @@ MachineBasicBlock::iterator X86FrameLowering::eliminateCallFramePseudoInstr(
       }
     }
 
-    if (DwarfCFI && !hasFP(MF)) {
+    if (DwarfCFI && !hasFP(MF) && CfaAdjustment) {
       // If we don't have FP, but need to generate unwind information,
       // we need to set the correct CFA offset after the stack adjustment.
       // How much we adjust the CFA offset depends on whether we're emitting
@@ -3839,14 +3876,11 @@ MachineBasicBlock::iterator X86FrameLowering::eliminateCallFramePseudoInstr(
       // offset to be correct at each call site, while for debugging we want
       // it to be more precise.
 
-      int64_t CfaAdjustment = -StackAdjustment;
       // TODO: When not using precise CFA, we also need to adjust for the
       // InternalAmt here.
-      if (CfaAdjustment) {
-        BuildCFI(
-            MBB, InsertPos, DL,
-            MCCFIInstruction::createAdjustCfaOffset(nullptr, CfaAdjustment));
-      }
+      BuildCFI(
+          MBB, InsertPos, DL,
+          MCCFIInstruction::createAdjustCfaOffset(nullptr, -CfaAdjustment));
     }
 
     return I;
diff --git a/llvm/lib/Target/X86/X86FrameLowering.h b/llvm/lib/Target/X86/X86FrameLowering.h
index 02fe8ee02a7e4..ef41b4653becc 100644
--- a/llvm/lib/Target/X86/X86FrameLowering.h
+++ b/llvm/lib/Target/X86/X86FrameLowering.h
@@ -134,12 +134,50 @@ class X86FrameLowering : public TargetFrameLowering {
   processFunctionBeforeFrameIndicesReplaced(MachineFunction &MF,
                                             RegScavenger *RS) const override;
 
-  /// Check the instruction before/after the passed instruction. If
-  /// it is an ADD/SUB/LEA instruction it is deleted argument and the
-  /// stack adjustment is returned as a positive value for ADD/LEA and
-  /// a negative for SUB.
-  int mergeSPUpdates(MachineBasicBlock &MBB, MachineBasicBlock::iterator &MBBI,
-                     bool doMergeWithPrevious) const;
+private:
+  /// Basic Pseudocode:
+  /// if (instruction before/after the passed instruction is ADD/SUB/LEA)
+  ///   Offset = instruction stack adjustment
+  ///            ... positive value for ADD/LEA and negative for SUB
+  ///   FoundStackAdjust(instruction, Offset)
+  ///   erase(instruction)
+  ///   return CalcNewOffset(Offset)
+  /// else
+  ///   return CalcNewOffset(0)
+  ///
+  /// It's possible that the selected instruction is not immediately
+  /// before/after MBBI for large adjustments that have been split into multiple
+  /// instructions.
+  ///
+  /// FoundStackAdjust should have the signature:
+  ///    void FoundStackAdjust(MachineBasicBlock::iterator PI, int64_t Offset)
+  /// CalcNewOffset should have the signature:
+  ///   int64_t CalcNewOffset(int64_t Offset)
+  template <typename FoundT, typename CalcT>
+  int64_t mergeSPUpdates(MachineBasicBlock &MBB,
+                         MachineBasicBlock::iterator &MBBI,
+                         FoundT FoundStackAdjust, CalcT CalcNewOffset,
+                         bool doMergeWithPrevious) const;
+
+  template <typename CalcT>
+  int64_t mergeSPUpdates(MachineBasicBlock &MBB,
+                         MachineBasicBlock::iterator &MBBI, CalcT CalcNewOffset,
+                         bool doMergeWithPrevious) const {
+    auto FoundStackAdjust = [](MachineBasicBlock::iterator MBBI,
+                               int64_t Offset) {};
+    return mergeSPUpdates(MBB, MBBI, FoundStackAdjust, CalcNewOffset,
+                          doMergeWithPrevious);
+  }
+
+public:
+  /// Equivalent to:
+  ///   mergeSPUpdates(MBB, MBBI,
+  ///                  [AddOffset](int64_t Offset) {
+  ///                    return AddOffset + Offset;
+  ///                  },
+  ///                  doMergeWithPrevious);
+  int64_t mergeSPAdd(MachineBasicBlock &MBB, MachineBasicBlock::iterator &MBBI,
+                     int64_t AddOffset, bool doMergeWithPrevious) const;
 
   /// Emit a series of instructions to increment / decrement the stack
   /// pointer by a constant value.
diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index 23731212a420c..8c28985c8e8e7 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -334,10 +334,10 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
       setOperationAction(ISD::FP_TO_SINT_SAT, VT, Custom);
     }
     setOperationAction(ISD::FCANONICALIZE, MVT::f32, Custom);
+    setOperationAction(ISD::FCANONICALIZE, MVT::f64, Custom);
     if (Subtarget.is64Bit()) {
       setOperationAction(ISD::FP_TO_UINT_SAT, MVT::i64, Custom);
       setOperationAction(ISD::FP_TO_SINT_SAT, MVT::i64, Custom);
-      setOperationAction(ISD::FCANONICALIZE, MVT::f64, Custom);
     }
   }
   if (Subtarget.hasAVX10_2()) {
@@ -367,7 +367,6 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
     setOperationAction(ISD::BITCAST        , MVT::f32  , Expand);
     setOperationAction(ISD::BITCAST        , MVT::i32  , Expand);
     setOperationAction(ISD::FCANONICALIZE, MVT::f32, Custom);
-    setOperationAction(ISD::FCANONICALIZE, MVT::f80, Custom);
     setOperationAction(ISD::FCANONICALIZE, MVT::f64, Custom);
     if (Subtarget.is64Bit()) {
       setOperationAction(ISD::BITCAST      , MVT::f64  , Expand);
@@ -889,6 +888,7 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
     setOperationAction(ISD::STRICT_FMUL     , MVT::f80, Legal);
     setOperationAction(ISD::STRICT_FDIV     , MVT::f80, Legal);
     setOperationAction(ISD::STRICT_FSQRT    , MVT::f80, Legal);
+    setOperationAction(ISD::FCANONICALIZE   , MVT::f80, Custom);
     if (isTypeLegal(MVT::f16)) {
       setOperationAction(ISD::FP_EXTEND, MVT::f80, Custom);
       setOperationAction(ISD::STRICT_FP_EXTEND, MVT::f80, Custom);
@@ -1159,6 +1159,7 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
     setOperationAction(ISD::FCOPYSIGN,          MVT::v2f64, Custom);
 
     setOperationAction(ISD::LRINT, MVT::v4f32, Custom);
+    setOperationAction(ISD::LRINT, MVT::v2i32, Custom);
 
     for (auto VT : { MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v2i64 }) {
       setOperationAction(ISD::SMAX, VT, VT == MVT::v8i16 ? Legal : Custom);
@@ -2944,7 +2945,7 @@ bool X86::isOffsetSuitableForCodeModel(int64_t Offset, CodeModel::Model CM,
 }
 
 /// Return true if the condition is an signed comparison operation.
-static bool isX86CCSigned(unsigned X86CC) {
+static bool isX86CCSigned(X86::CondCode X86CC) {
   switch (X86CC) {
   default:
     llvm_unreachable("Invalid integer condition!");
@@ -9789,6 +9790,7 @@ static bool IsElementEquivalent(int MaskSize, SDValue Op, SDValue ExpectedOp,
   if (!Op || !ExpectedOp || Op.getOpcode() != ExpectedOp.getOpcode())
     return false;
 
+  EVT VT = Op.getValueType();
   switch (Op.getOpcode()) {
   case ISD::BUILD_VECTOR:
     // If the values are build vectors, we can look through them to find
@@ -9800,9 +9802,8 @@ static bool IsElementEquivalent(int MaskSize, SDValue Op, SDValue ExpectedOp,
     break;
   case X86ISD::VBROADCAST:
   case X86ISD::VBROADCAST_LOAD:
-    // TODO: Handle MaskSize != Op.getValueType().getVectorNumElements()?
-    return (Op == ExpectedOp &&
-            (int)Op.getValueType().getVectorNumElements() == MaskSize);
+    // TODO: Handle MaskSize != VT.getVectorNumElements()?
+    return (Op == ExpectedOp && (int)VT.getVectorNumElements() == MaskSize);
   case X86ISD::HADD:
   case X86ISD::HSUB:
   case X86ISD::FHADD:
@@ -9813,7 +9814,6 @@ static bool IsElementEquivalent(int MaskSize, SDValue Op, SDValue ExpectedOp,
     // TODO: Handle MaskSize != NumElts?
     // TODO: Handle HOP(X,Y) vs HOP(Y,X) equivalence cases.
     if (Op == ExpectedOp && Op.getOperand(0) == Op.getOperand(1)) {
-      MVT VT = Op.getSimpleValueType();
       int NumElts = VT.getVectorNumElements();
       if (MaskSize == NumElts) {
         int NumLanes = VT.getSizeInBits() / 128;
@@ -12689,6 +12689,20 @@ static bool isShuffleMaskInputInPlace(int Input, ArrayRef<int> Mask) {
   return true;
 }
 
+/// Test whether the specified input (0 or 1) is a broadcast/splat blended by
+/// the given mask.
+///
+static bool isShuffleMaskInputBroadcastable(int Input, ArrayRef<int> Mask,
+                                            int BroadcastableElement = 0) {
+  assert((Input == 0 || Input == 1) && "Only two inputs to shuffles.");
+  int Size = Mask.size();
+  for (int i = 0; i < Size; ++i)
+    if (Mask[i] >= 0 && Mask[i] / Size == Input &&
+        Mask[i] % Size != BroadcastableElement)
+      return false;
+  return true;
+}
+
 /// If we are extracting two 128-bit halves of a vector and shuffling the
 /// result, match that to a 256-bit AVX2 vperm* instruction to avoid a
 /// multi-shuffle lowering.
@@ -16190,6 +16204,8 @@ static SDValue lowerV4F64Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
 
   bool V1IsInPlace = isShuffleMaskInputInPlace(0, Mask);
   bool V2IsInPlace = isShuffleMaskInputInPlace(1, Mask);
+  bool V1IsSplat = isShuffleMaskInputBroadcastable(0, Mask);
+  bool V2IsSplat = isShuffleMaskInputBroadcastable(1, Mask);
 
   // If we have lane crossing shuffles AND they don't all come from the lower
   // lane elements, lower to SHUFPD(VPERM2F128(V1, V2), VPERM2F128(V1, V2)).
@@ -16198,7 +16214,9 @@ static SDValue lowerV4F64Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
   if (is128BitLaneCrossingShuffleMask(MVT::v4f64, Mask) &&
       !all_of(Mask, [](int M) { return M < 2 || (4 <= M && M < 6); }) &&
       (V1.getOpcode() != ISD::BUILD_VECTOR) &&
-      (V2.getOpcode() != ISD::BUILD_VECTOR))
+      (V2.getOpcode() != ISD::BUILD_VECTOR) &&
+      (!Subtarget.hasAVX2() ||
+       !((V1IsInPlace || V1IsSplat) && (V2IsInPlace || V2IsSplat))))
     return lowerShuffleAsLanePermuteAndSHUFP(DL, MVT::v4f64, V1, V2, Mask, DAG);
 
   // If we have one input in place, then we can permute the other input and
@@ -22975,7 +22993,7 @@ static bool isProfitableToUseFlagOp(SDValue Op) {
 
 /// Emit nodes that will be selected as "test Op0,Op0", or something
 /// equivalent.
-static SDValue EmitTest(SDValue Op, unsigned X86CC, const SDLoc &dl,
+static SDValue EmitTest(SDValue Op, X86::CondCode X86CC, const SDLoc &dl,
                         SelectionDAG &DAG, const X86Subtarget &Subtarget) {
   // CF and OF aren't always set the way we want. Determine which
   // of these we need.
@@ -23085,7 +23103,7 @@ static SDValue EmitTest(SDValue Op, unsigned X86CC, const SDLoc &dl,
 
 /// Emit nodes that will be selected as "cmp Op0,Op1", or something
 /// equivalent.
-static SDValue EmitCmp(SDValue Op0, SDValue Op1, unsigned X86CC,
+static SDValue EmitCmp(SDValue Op0, SDValue Op1, X86::CondCode X86CC,
                        const SDLoc &dl, SelectionDAG &DAG,
                        const X86Subtarget &Subtarget) {
   if (isNullConstant(Op1))
@@ -23157,10 +23175,17 @@ static SDValue EmitCmp(SDValue Op0, SDValue Op1, unsigned X86CC,
     return Add.getValue(1);
   }
 
-  // Use SUB instead of CMP to enable CSE between SUB and CMP.
+  // If we already have an XOR of the ops, use that to check for equality.
+  // Else use SUB instead of CMP to enable CSE between SUB and CMP.
+  unsigned X86Opc = X86ISD::SUB;
+  if ((X86CC == X86::COND_E || X86CC == X86::COND_NE) &&
+      (DAG.doesNodeExist(ISD::XOR, DAG.getVTList({CmpVT}), {Op0, Op1}) ||
+       DAG.doesNodeExist(ISD::XOR, DAG.getVTList({CmpVT}), {Op1, Op0})))
+    X86Opc = X86ISD::XOR;
+
   SDVTList VTs = DAG.getVTList(CmpVT, MVT::i32);
-  SDValue Sub = DAG.getNode(X86ISD::SUB, dl, VTs, Op0, Op1);
-  return Sub.getValue(1);
+  SDValue CmpOp = DAG.getNode(X86Opc, dl, VTs, Op0, Op1);
+  return CmpOp.getValue(1);
 }
 
 bool X86TargetLowering::isXAndYEqZeroPreferableToXAndYEqY(ISD::CondCode Cond,
@@ -24623,19 +24648,14 @@ SDValue X86TargetLowering::LowerSELECT(SDValue Op, SelectionDAG &DAG) const {
       SDValue Cmp = DAG.getNode(X86ISD::FSETCC, DL, VT, CondOp0, CondOp1,
                                 DAG.getTargetConstant(SSECC, DL, MVT::i8));
 
-      // If we have AVX, we can use a variable vector select (VBLENDV) instead
-      // of 3 logic instructions for size savings and potentially speed.
+      // If we have SSE41/AVX, we can use a variable vector select (VBLENDV)
+      // instead of 3 logic instructions for size savings and potentially speed.
       // Unfortunately, there is no scalar form of VBLENDV.
-
+      //
       // If either operand is a +0.0 constant, don't try this. We can expect to
       // optimize away at least one of the logic instructions later in that
       // case, so that sequence would be faster than a variable blend.
-
-      // BLENDV was introduced with SSE 4.1, but the 2 register form implicitly
-      // uses XMM0 as the selection register. That may need just as many
-      // instructions as the AND/ANDN/OR sequence due to register moves, so
-      // don't bother.
-      if (Subtarget.hasAVX() && !isNullFPConstant(Op1) &&
+      if (Subtarget.hasSSE41() && !isNullFPConstant(Op1) &&
           !isNullFPConstant(Op2)) {
         // Convert to vectors, do a VSELECT, and convert back to scalar.
         // All of the conversions should be optimized away.
@@ -34010,6 +34030,13 @@ void X86TargetLowering::ReplaceNodeResults(SDNode *N,
     return;
   }
   case ISD::LRINT:
+    if (N->getValueType(0) == MVT::v2i32) {
+      SDValue Src = N->getOperand(0);
+      if (Src.getValueType() == MVT::v2f64)
+        Results.push_back(DAG.getNode(X86ISD::CVTP2SI, dl, MVT::v4i32, Src));
+      return;
+    }
+    [[fallthrough]];
   case ISD::LLRINT: {
     if (SDValue V = LRINT_LLRINTHelper(N, DAG))
       Results.push_back(V);
@@ -41578,10 +41605,10 @@ static SDValue canonicalizeShuffleWithOp(SDValue N, SelectionDAG &DAG,
       if (TLI.isBinOp(SrcOpcode) && IsSafeToMoveShuffle(N0, SrcOpcode)) {
         SDValue Op00 = peekThroughOneUseBitcasts(N0.getOperand(0));
         SDValue Op01 = peekThroughOneUseBitcasts(N0.getOperand(1));
-        if (IsMergeableWithShuffle(Op00, Opc != X86ISD::VPERMI,
-                                   Opc != X86ISD::PSHUFB) ||
-            IsMergeableWithShuffle(Op01, Opc != X86ISD::VPERMI,
-                                   Opc != X86ISD::PSHUFB)) {
+        bool FoldShuf = Opc != X86ISD::VPERMI;
+        bool FoldLoad = Opc != X86ISD::PSHUFB;
+        if (IsMergeableWithShuffle(Op00, FoldShuf, FoldLoad) ||
+            IsMergeableWithShuffle(Op01, FoldShuf, FoldLoad)) {
           SDValue LHS, RHS;
           Op00 = DAG.getBitcast(ShuffleVT, Op00);
           Op01 = DAG.getBitcast(ShuffleVT, Op01);
@@ -45153,6 +45180,19 @@ static SDValue combineBitcast(SDNode *N, SelectionDAG &DAG,
     }
   }
 
+  // Attempt to peek through f16 bitcasted extractions hidden by truncation.
+  if (VT == MVT::f16 && SrcVT == MVT::i16) {
+    SDValue Src = peekThroughTruncates(N0);
+    if (Src.getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
+        Src.getOperand(0).getValueSizeInBits() == 128 &&
+        isNullConstant(Src.getOperand(1))) {
+      SDLoc DL(N);
+      return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT,
+                         DAG.getBitcast(MVT::v8f16, Src.getOperand(0)),
+                         DAG.getVectorIdxConstant(0, DL));
+    }
+  }
+
   // Since MMX types are special and don't usually play with other vector types,
   // it's better to handle them early to be sure we emit efficient code by
   // avoiding store-load conversions.
@@ -48464,7 +48504,7 @@ static SDValue combineCMov(SDNode *N, SelectionDAG &DAG,
                            TargetLowering::DAGCombinerInfo &DCI,
                            const X86Subtarget &Subtarget) {
   SDLoc DL(N);
-
+  EVT VT = N->getValueType(0);
   SDValue FalseOp = N->getOperand(0);
   SDValue TrueOp = N->getOperand(1);
   X86::CondCode CC = (X86::CondCode)N->getConstantOperandVal(2);
@@ -48483,7 +48523,7 @@ static SDValue combineCMov(SDNode *N, SelectionDAG &DAG,
         !Subtarget.canUseCMOV() || hasFPCMov(CC)) {
       SDValue Ops[] = {FalseOp, TrueOp, DAG.getTargetConstant(CC, DL, MVT::i8),
                        Flags};
-      return DAG.getNode(X86ISD::CMOV, DL, N->getValueType(0), Ops);
+      return DAG.getNode(X86ISD::CMOV, DL, VT, Ops);
     }
   }
 
@@ -48530,9 +48570,9 @@ static SDValue combineCMov(SDNode *N, SelectionDAG &DAG,
 
       // Optimize cases that will turn into an LEA instruction.  This requires
       // an i32 or i64 and an efficient multiplier (1, 2, 3, 4, 5, 8, 9).
-      if (N->getValueType(0) == MVT::i32 || N->getValueType(0) == MVT::i64) {
+      if (VT == MVT::i32 || VT == MVT::i64) {
         APInt Diff = TrueC->getAPIntValue() - FalseC->getAPIntValue();
-        assert(Diff.getBitWidth() == N->getValueType(0).getSizeInBits() &&
+        assert(Diff.getBitWidth() == VT.getSizeInBits() &&
                "Implicit constant truncation");
 
         bool isFastMultiplier = false;
@@ -48600,11 +48640,10 @@ static SDValue combineCMov(SDNode *N, SelectionDAG &DAG,
         std::swap(TrueOp, FalseOp);
       }
 
-      if (CC == X86::COND_E &&
-          CmpAgainst == dyn_cast<ConstantSDNode>(TrueOp)) {
+      if (CC == X86::COND_E && CmpAgainst == dyn_cast<ConstantSDNode>(TrueOp)) {
         SDValue Ops[] = {FalseOp, Cond.getOperand(0),
                          DAG.getTargetConstant(CC, DL, MVT::i8), Cond};
-        return DAG.getNode(X86ISD::CMOV, DL, N->getValueType(0), Ops);
+        return DAG.getNode(X86ISD::CMOV, DL, VT, Ops);
       }
     }
   }
@@ -48624,14 +48663,13 @@ static SDValue combineCMov(SDNode *N, SelectionDAG &DAG,
     auto *Sub1C = dyn_cast<ConstantSDNode>(Cond.getOperand(1));
     if (Cond0 == TrueOp && Sub1C && Sub1C->getZExtValue() == 2) {
       EVT CondVT = Cond->getValueType(0);
-      EVT OuterVT = N->getValueType(0);
       // Subtract 1 and generate a carry.
       SDValue NewSub =
           DAG.getNode(X86ISD::SUB, DL, Cond->getVTList(), Cond.getOperand(0),
                       DAG.getConstant(1, DL, CondVT));
       SDValue EFLAGS(NewSub.getNode(), 1);
-      return DAG.getNode(X86ISD::ADC, DL, DAG.getVTList(OuterVT, MVT::i32),
-                         TrueOp, DAG.getConstant(0, DL, OuterVT), EFLAGS);
+      return DAG.getNode(X86ISD::ADC, DL, DAG.getVTList(VT, MVT::i32), TrueOp,
+                         DAG.getConstant(0, DL, VT), EFLAGS);
     }
   }
 
@@ -48665,10 +48703,10 @@ static SDValue combineCMov(SDNode *N, SelectionDAG &DAG,
 
       SDValue LOps[] = {FalseOp, TrueOp,
                         DAG.getTargetConstant(CC0, DL, MVT::i8), Flags};
-      SDValue LCMOV = DAG.getNode(X86ISD::CMOV, DL, N->getValueType(0), LOps);
+      SDValue LCMOV = DAG.getNode(X86ISD::CMOV, DL, VT, LOps);
       SDValue Ops[] = {LCMOV, TrueOp, DAG.getTargetConstant(CC1, DL, MVT::i8),
                        Flags};
-      SDValue CMOV = DAG.getNode(X86ISD::CMOV, DL, N->getValueType(0), Ops);
+      SDValue CMOV = DAG.getNode(X86ISD::CMOV, DL, VT, Ops);
       return CMOV;
     }
   }
@@ -48696,7 +48734,6 @@ static SDValue combineCMov(SDNode *N, SelectionDAG &DAG,
         (Add.getOperand(0).getOpcode() == ISD::CTTZ_ZERO_UNDEF ||
          Add.getOperand(0).getOpcode() == ISD::CTTZ) &&
         Add.getOperand(0).getOperand(0) == Cond.getOperand(0)) {
-      EVT VT = N->getValueType(0);
       // This should constant fold.
       SDValue Diff = DAG.getNode(ISD::SUB, DL, VT, Const, Add.getOperand(1));
       SDValue CMov =
@@ -53902,11 +53939,6 @@ static SDValue combineTruncate(SDNode *N, SelectionDAG &DAG,
       return DAG.getNode(X86ISD::MMX_MOVD2W, DL, MVT::i32, BCSrc);
   }
 
-  // Try to combine (trunc (vNi64 (lrint x))) to (vNi32 (lrint x)).
-  if (Src.getOpcode() == ISD::LRINT && VT.getScalarType() == MVT::i32 &&
-      Src.hasOneUse())
-    return DAG.getNode(ISD::LRINT, DL, VT, Src.getOperand(0));
-
   return SDValue();
 }
 
diff --git a/llvm/lib/Target/X86/X86ISelLoweringCall.cpp b/llvm/lib/Target/X86/X86ISelLoweringCall.cpp
index 6835c7e336a5c..ee4bb758102f4 100644
--- a/llvm/lib/Target/X86/X86ISelLoweringCall.cpp
+++ b/llvm/lib/Target/X86/X86ISelLoweringCall.cpp
@@ -2326,12 +2326,13 @@ X86TargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
   // shuffling arguments passed in memory.
   if (!IsSibcall && isTailCall) {
     // Force all the incoming stack arguments to be loaded from the stack
-    // before any new outgoing arguments are stored to the stack, because the
-    // outgoing stack slots may alias the incoming argument stack slots, and
-    // the alias isn't otherwise explicit. This is slightly more conservative
-    // than necessary, because it means that each store effectively depends
-    // on every argument instead of just those arguments it would clobber.
-    SDValue ArgChain = DAG.getStackArgumentTokenFactor(Chain);
+    // before any new outgoing arguments or the return address are stored to the
+    // stack, because the outgoing stack slots may alias the incoming argument
+    // stack slots, and the alias isn't otherwise explicit. This is slightly
+    // more conservative than necessary, because it means that each store
+    // effectively depends on every argument instead of just those arguments it
+    // would clobber.
+    Chain = DAG.getStackArgumentTokenFactor(Chain);
 
     SmallVector<SDValue, 8> MemOpChains2;
     SDValue FIN;
@@ -2373,13 +2374,12 @@ X86TargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
         Source = DAG.getNode(ISD::ADD, dl, getPointerTy(DAG.getDataLayout()),
                              StackPtr, Source);
 
-        MemOpChains2.push_back(CreateCopyOfByValArgument(Source, FIN,
-                                                         ArgChain,
-                                                         Flags, DAG, dl));
+        MemOpChains2.push_back(
+            CreateCopyOfByValArgument(Source, FIN, Chain, Flags, DAG, dl));
       } else {
         // Store relative to framepointer.
         MemOpChains2.push_back(DAG.getStore(
-            ArgChain, dl, Arg, FIN,
+            Chain, dl, Arg, FIN,
             MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FI)));
       }
     }
diff --git a/llvm/lib/Target/X86/X86InstrCompiler.td b/llvm/lib/Target/X86/X86InstrCompiler.td
index 9bda3fd7d951c..9687ae29f1c78 100644
--- a/llvm/lib/Target/X86/X86InstrCompiler.td
+++ b/llvm/lib/Target/X86/X86InstrCompiler.td
@@ -262,8 +262,10 @@ let isPseudo = 1, isMeta = 1, isNotDuplicable = 1, SchedRW = [WriteSystem] in {
 
 // Epilog instructions:
 let isPseudo = 1, isMeta = 1, SchedRW = [WriteSystem] in {
-  def SEH_Epilogue : I<0, Pseudo, (outs), (ins),
-                            "#SEH_Epilogue", []>;
+  def SEH_BeginEpilogue : I<0, Pseudo, (outs), (ins),
+                            "#SEH_BeginEpilogue", []>;
+  def SEH_EndEpilogue : I<0, Pseudo, (outs), (ins),
+                            "#SEH_EndEpilogue", []>;
 }
 
 //===----------------------------------------------------------------------===//
diff --git a/llvm/lib/Target/X86/X86InstrInfo.cpp b/llvm/lib/Target/X86/X86InstrInfo.cpp
index 794aa921ca254..44db5b6865c42 100644
--- a/llvm/lib/Target/X86/X86InstrInfo.cpp
+++ b/llvm/lib/Target/X86/X86InstrInfo.cpp
@@ -4801,7 +4801,8 @@ void X86InstrInfo::storeRegToStackSlot(
     loadStoreTileReg(MBB, MI, Opc, SrcReg, FrameIdx, isKill);
   else
     addFrameReference(BuildMI(MBB, MI, DebugLoc(), get(Opc)), FrameIdx)
-        .addReg(SrcReg, getKillRegState(isKill));
+        .addReg(SrcReg, getKillRegState(isKill))
+        .setMIFlag(Flags);
 }
 
 void X86InstrInfo::loadRegFromStackSlot(
@@ -4821,8 +4822,8 @@ void X86InstrInfo::loadRegFromStackSlot(
   if (isAMXOpcode(Opc))
     loadStoreTileReg(MBB, MI, Opc, DestReg, FrameIdx);
   else
-    addFrameReference(BuildMI(MBB, MI, DebugLoc(), get(Opc), DestReg),
-                      FrameIdx);
+    addFrameReference(BuildMI(MBB, MI, DebugLoc(), get(Opc), DestReg), FrameIdx)
+        .setMIFlag(Flags);
 }
 
 bool X86InstrInfo::analyzeCompare(const MachineInstr &MI, Register &SrcReg,
diff --git a/llvm/lib/Target/X86/X86MCInstLower.cpp b/llvm/lib/Target/X86/X86MCInstLower.cpp
index 7bae16c066716..0f8fbf5be1c95 100644
--- a/llvm/lib/Target/X86/X86MCInstLower.cpp
+++ b/llvm/lib/Target/X86/X86MCInstLower.cpp
@@ -1710,7 +1710,7 @@ static void printZeroExtend(const MachineInstr *MI, MCStreamer &OutStreamer,
 
 void X86AsmPrinter::EmitSEHInstruction(const MachineInstr *MI) {
   assert(MF->hasWinCFI() && "SEH_ instruction in function without WinCFI?");
-  assert((getSubtarget().isOSWindows() || TM.getTargetTriple().isUEFI()) &&
+  assert(getSubtarget().isOSWindowsOrUEFI() &&
          "SEH_ instruction Windows and UEFI only");
 
   // Use the .cv_fpo directives if we're emitting CodeView on 32-bit x86.
@@ -1779,6 +1779,14 @@ void X86AsmPrinter::EmitSEHInstruction(const MachineInstr *MI) {
     OutStreamer->emitWinCFIEndProlog();
     break;
 
+  case X86::SEH_BeginEpilogue:
+    OutStreamer->emitWinCFIBeginEpilogue();
+    break;
+
+  case X86::SEH_EndEpilogue:
+    OutStreamer->emitWinCFIEndEpilogue();
+    break;
+
   default:
     llvm_unreachable("expected SEH_ instruction");
   }
@@ -2420,11 +2428,17 @@ void X86AsmPrinter::emitInstruction(const MachineInstr *MI) {
   case X86::SEH_SetFrame:
   case X86::SEH_PushFrame:
   case X86::SEH_EndPrologue:
+  case X86::SEH_EndEpilogue:
     EmitSEHInstruction(MI);
     return;
 
-  case X86::SEH_Epilogue: {
+  case X86::SEH_BeginEpilogue: {
     assert(MF->hasWinCFI() && "SEH_ instruction in function without WinCFI?");
+    // Windows unwinder will not invoke function's exception handler if IP is
+    // either in prologue or in epilogue.  This behavior causes a problem when a
+    // call immediately precedes an epilogue, because the return address points
+    // into the epilogue.  To cope with that, we insert a 'nop' if it ends up
+    // immediately after a CALL in the final emitted code.
     MachineBasicBlock::const_iterator MBBI(MI);
     // Check if preceded by a call and emit nop if so.
     for (MBBI = PrevCrossBBInst(MBBI);
@@ -2439,6 +2453,8 @@ void X86AsmPrinter::emitInstruction(const MachineInstr *MI) {
         break;
       }
     }
+
+    EmitSEHInstruction(MI);
     return;
   }
   case X86::UBSAN_UD1:
diff --git a/llvm/lib/Target/X86/X86MacroFusion.h b/llvm/lib/Target/X86/X86MacroFusion.h
index 05388b275ca3f..ee07977664caf 100644
--- a/llvm/lib/Target/X86/X86MacroFusion.h
+++ b/llvm/lib/Target/X86/X86MacroFusion.h
@@ -22,7 +22,7 @@ class ScheduleDAGMutation;
 
 /// Note that you have to add:
 ///   DAG.addMutation(createX86MacroFusionDAGMutation());
-/// to X86PassConfig::createMachineScheduler() to have an effect.
+/// to X86TargetMachine::createMachineScheduler() to have an effect.
 std::unique_ptr<ScheduleDAGMutation>
 createX86MacroFusionDAGMutation();
 
diff --git a/llvm/lib/Target/X86/X86RegisterInfo.cpp b/llvm/lib/Target/X86/X86RegisterInfo.cpp
index 4faf8bca4f9e0..af1060519ae5c 100644
--- a/llvm/lib/Target/X86/X86RegisterInfo.cpp
+++ b/llvm/lib/Target/X86/X86RegisterInfo.cpp
@@ -224,21 +224,6 @@ X86RegisterInfo::getPointerRegClass(const MachineFunction &MF,
   }
 }
 
-bool X86RegisterInfo::shouldRewriteCopySrc(const TargetRegisterClass *DefRC,
-                                           unsigned DefSubReg,
-                                           const TargetRegisterClass *SrcRC,
-                                           unsigned SrcSubReg) const {
-  // Prevent rewriting a copy where the destination size is larger than the
-  // input size. See PR41619.
-  // FIXME: Should this be factored into the base implementation somehow.
-  if (DefRC->hasSuperClassEq(&X86::GR64RegClass) && DefSubReg == 0 &&
-      SrcRC->hasSuperClassEq(&X86::GR64RegClass) && SrcSubReg == X86::sub_32bit)
-    return false;
-
-  return TargetRegisterInfo::shouldRewriteCopySrc(DefRC, DefSubReg,
-                                                  SrcRC, SrcSubReg);
-}
-
 const TargetRegisterClass *
 X86RegisterInfo::getGPRsForTailCall(const MachineFunction &MF) const {
   const Function &F = MF.getFunction();
diff --git a/llvm/lib/Target/X86/X86RegisterInfo.h b/llvm/lib/Target/X86/X86RegisterInfo.h
index 68ee372f27b14..009d2a8c7ac3a 100644
--- a/llvm/lib/Target/X86/X86RegisterInfo.h
+++ b/llvm/lib/Target/X86/X86RegisterInfo.h
@@ -70,11 +70,6 @@ class X86RegisterInfo final : public X86GenRegisterInfo {
   getLargestLegalSuperClass(const TargetRegisterClass *RC,
                             const MachineFunction &MF) const override;
 
-  bool shouldRewriteCopySrc(const TargetRegisterClass *DefRC,
-                            unsigned DefSubReg,
-                            const TargetRegisterClass *SrcRC,
-                            unsigned SrcSubReg) const override;
-
   /// getPointerRegClass - Returns a TargetRegisterClass used for pointer
   /// values.
   const TargetRegisterClass *
diff --git a/llvm/lib/Target/X86/X86Subtarget.h b/llvm/lib/Target/X86/X86Subtarget.h
index c399989f115d7..722076ca88c9c 100644
--- a/llvm/lib/Target/X86/X86Subtarget.h
+++ b/llvm/lib/Target/X86/X86Subtarget.h
@@ -324,8 +324,12 @@ class X86Subtarget final : public X86GenSubtargetInfo {
 
   bool isTargetCygMing() const { return TargetTriple.isOSCygMing(); }
 
+  bool isUEFI() const { return TargetTriple.isUEFI(); }
+
   bool isOSWindows() const { return TargetTriple.isOSWindows(); }
 
+  bool isOSWindowsOrUEFI() const { return isOSWindows() || isUEFI(); }
+
   bool isTargetWin64() const { return Is64Bit && isOSWindows(); }
 
   bool isTargetWin32() const { return !Is64Bit && isOSWindows(); }
diff --git a/llvm/lib/Target/X86/X86TargetMachine.cpp b/llvm/lib/Target/X86/X86TargetMachine.cpp
index 20dfdd27b33df..0430279b88984 100644
--- a/llvm/lib/Target/X86/X86TargetMachine.cpp
+++ b/llvm/lib/Target/X86/X86TargetMachine.cpp
@@ -374,6 +374,20 @@ bool X86TargetMachine::isNoopAddrSpaceCast(unsigned SrcAS,
 
 void X86TargetMachine::reset() { SubtargetMap.clear(); }
 
+ScheduleDAGInstrs *
+X86TargetMachine::createMachineScheduler(MachineSchedContext *C) const {
+  ScheduleDAGMILive *DAG = createGenericSchedLive(C);
+  DAG->addMutation(createX86MacroFusionDAGMutation());
+  return DAG;
+}
+
+ScheduleDAGInstrs *
+X86TargetMachine::createPostMachineScheduler(MachineSchedContext *C) const {
+  ScheduleDAGMI *DAG = createGenericSchedPostRA(C);
+  DAG->addMutation(createX86MacroFusionDAGMutation());
+  return DAG;
+}
+
 //===----------------------------------------------------------------------===//
 // X86 TTI query.
 //===----------------------------------------------------------------------===//
@@ -399,20 +413,6 @@ class X86PassConfig : public TargetPassConfig {
     return getTM<X86TargetMachine>();
   }
 
-  ScheduleDAGInstrs *
-  createMachineScheduler(MachineSchedContext *C) const override {
-    ScheduleDAGMILive *DAG = createGenericSchedLive(C);
-    DAG->addMutation(createX86MacroFusionDAGMutation());
-    return DAG;
-  }
-
-  ScheduleDAGInstrs *
-  createPostMachineScheduler(MachineSchedContext *C) const override {
-    ScheduleDAGMI *DAG = createGenericSchedPostRA(C);
-    DAG->addMutation(createX86MacroFusionDAGMutation());
-    return DAG;
-  }
-
   void addIRPasses() override;
   bool addInstSelector() override;
   bool addIRTranslator() override;
diff --git a/llvm/lib/Target/X86/X86TargetMachine.h b/llvm/lib/Target/X86/X86TargetMachine.h
index b8d84a8d20032..ced0a9c71fdd8 100644
--- a/llvm/lib/Target/X86/X86TargetMachine.h
+++ b/llvm/lib/Target/X86/X86TargetMachine.h
@@ -79,6 +79,10 @@ class X86TargetMachine final : public CodeGenTargetMachineImpl {
   bool isJIT() const { return IsJIT; }
 
   bool isNoopAddrSpaceCast(unsigned SrcAS, unsigned DestAS) const override;
+  ScheduleDAGInstrs *
+  createMachineScheduler(MachineSchedContext *C) const override;
+  ScheduleDAGInstrs *
+  createPostMachineScheduler(MachineSchedContext *C) const override;
 };
 
 } // end namespace llvm
diff --git a/llvm/lib/Target/X86/X86TargetTransformInfo.cpp b/llvm/lib/Target/X86/X86TargetTransformInfo.cpp
index d3c923a76d074..f6552c500d7cf 100644
--- a/llvm/lib/Target/X86/X86TargetTransformInfo.cpp
+++ b/llvm/lib/Target/X86/X86TargetTransformInfo.cpp
@@ -1565,19 +1565,26 @@ InstructionCost X86TTIImpl::getShuffleCost(
 
   // Attempt to detect a cheaper inlane shuffle, avoiding 128-bit subvector
   // permutation.
+  // Attempt to detect a shuffle mask with a single defined element.
   bool IsInLaneShuffle = false;
+  bool IsSingleElementMask = false;
   if (BaseTp->getPrimitiveSizeInBits() > 0 &&
       (BaseTp->getPrimitiveSizeInBits() % 128) == 0 &&
       BaseTp->getScalarSizeInBits() == LT.second.getScalarSizeInBits() &&
       Mask.size() == BaseTp->getElementCount().getKnownMinValue()) {
     unsigned NumLanes = BaseTp->getPrimitiveSizeInBits() / 128;
     unsigned NumEltsPerLane = Mask.size() / NumLanes;
-    if ((Mask.size() % NumLanes) == 0)
+    if ((Mask.size() % NumLanes) == 0) {
       IsInLaneShuffle = all_of(enumerate(Mask), [&](const auto &P) {
         return P.value() == PoisonMaskElem ||
                ((P.value() % Mask.size()) / NumEltsPerLane) ==
                    (P.index() / NumEltsPerLane);
       });
+      IsSingleElementMask =
+          (Mask.size() - 1) == static_cast<unsigned>(count_if(Mask, [](int M) {
+            return M == PoisonMaskElem;
+          }));
+    }
   }
 
   // Treat <X x bfloat> shuffles as <X x half>.
@@ -1666,40 +1673,41 @@ InstructionCost X86TTIImpl::getShuffleCost(
   EVT VT = TLI->getValueType(DL, BaseTp);
   if (VT.isSimple() && VT.isVector() && VT.getSizeInBits() < 128 &&
       !ST->hasSSSE3()) {
-     static const CostTblEntry SSE2SubVectorShuffleTbl[] = {
-      {TTI::SK_Broadcast,        MVT::v4i16, 1}, // pshuflw
-      {TTI::SK_Broadcast,        MVT::v2i16, 1}, // pshuflw
-      {TTI::SK_Broadcast,        MVT::v8i8,  2}, // punpck/pshuflw
-      {TTI::SK_Broadcast,        MVT::v4i8,  2}, // punpck/pshuflw
-      {TTI::SK_Broadcast,        MVT::v2i8,  1}, // punpck
-
-      {TTI::SK_Reverse,          MVT::v4i16, 1}, // pshuflw
-      {TTI::SK_Reverse,          MVT::v2i16, 1}, // pshuflw
-      {TTI::SK_Reverse,          MVT::v4i8,  3}, // punpck/pshuflw/packus
-      {TTI::SK_Reverse,          MVT::v2i8,  1}, // punpck
-
-      {TTI::SK_Splice,           MVT::v4i16, 2}, // punpck+psrldq
-      {TTI::SK_Splice,           MVT::v2i16, 2}, // punpck+psrldq
-      {TTI::SK_Splice,           MVT::v4i8,  2}, // punpck+psrldq
-      {TTI::SK_Splice,           MVT::v2i8,  2}, // punpck+psrldq
-
-      {TTI::SK_PermuteTwoSrc,    MVT::v4i16, 2}, // punpck/pshuflw
-      {TTI::SK_PermuteTwoSrc,    MVT::v2i16, 2}, // punpck/pshuflw
-      {TTI::SK_PermuteTwoSrc,    MVT::v8i8,  7}, // punpck/pshuflw
-      {TTI::SK_PermuteTwoSrc,    MVT::v4i8,  4}, // punpck/pshuflw
-      {TTI::SK_PermuteTwoSrc,    MVT::v2i8,  2}, // punpck
-
-      {TTI::SK_PermuteSingleSrc, MVT::v4i16, 1}, // pshuflw
-      {TTI::SK_PermuteSingleSrc, MVT::v2i16, 1}, // pshuflw
-      {TTI::SK_PermuteSingleSrc, MVT::v8i8,  5}, // punpck/pshuflw
-      {TTI::SK_PermuteSingleSrc, MVT::v4i8,  3}, // punpck/pshuflw
-      {TTI::SK_PermuteSingleSrc, MVT::v2i8,  1}, // punpck
+     static const CostKindTblEntry SSE2SubVectorShuffleTbl[] = {
+      {TTI::SK_Broadcast,        MVT::v4i16, {1,1,1,1}}, // pshuflw
+      {TTI::SK_Broadcast,        MVT::v2i16, {1,1,1,1}}, // pshuflw
+      {TTI::SK_Broadcast,        MVT::v8i8,  {2,2,2,2}}, // punpck/pshuflw
+      {TTI::SK_Broadcast,        MVT::v4i8,  {2,2,2,2}}, // punpck/pshuflw
+      {TTI::SK_Broadcast,        MVT::v2i8,  {1,1,1,1}}, // punpck
+
+      {TTI::SK_Reverse,          MVT::v4i16, {1,1,1,1}}, // pshuflw
+      {TTI::SK_Reverse,          MVT::v2i16, {1,1,1,1}}, // pshuflw
+      {TTI::SK_Reverse,          MVT::v4i8,  {3,3,3,3}}, // punpck/pshuflw/packus
+      {TTI::SK_Reverse,          MVT::v2i8,  {1,1,1,1}}, // punpck
+
+      {TTI::SK_Splice,           MVT::v4i16, {2,2,2,2}}, // punpck+psrldq
+      {TTI::SK_Splice,           MVT::v2i16, {2,2,2,2}}, // punpck+psrldq
+      {TTI::SK_Splice,           MVT::v4i8,  {2,2,2,2}}, // punpck+psrldq
+      {TTI::SK_Splice,           MVT::v2i8,  {2,2,2,2}}, // punpck+psrldq
+
+      {TTI::SK_PermuteTwoSrc,    MVT::v4i16, {2,2,2,2}}, // punpck/pshuflw
+      {TTI::SK_PermuteTwoSrc,    MVT::v2i16, {2,2,2,2}}, // punpck/pshuflw
+      {TTI::SK_PermuteTwoSrc,    MVT::v8i8,  {7,7,7,7}}, // punpck/pshuflw
+      {TTI::SK_PermuteTwoSrc,    MVT::v4i8,  {4,4,4,4}}, // punpck/pshuflw
+      {TTI::SK_PermuteTwoSrc,    MVT::v2i8,  {2,2,2,2}}, // punpck
+
+      {TTI::SK_PermuteSingleSrc, MVT::v4i16, {1,1,1,1}}, // pshuflw
+      {TTI::SK_PermuteSingleSrc, MVT::v2i16, {1,1,1,1}}, // pshuflw
+      {TTI::SK_PermuteSingleSrc, MVT::v8i8,  {5,5,5,5}}, // punpck/pshuflw
+      {TTI::SK_PermuteSingleSrc, MVT::v4i8,  {3,3,3,3}}, // punpck/pshuflw
+      {TTI::SK_PermuteSingleSrc, MVT::v2i8,  {1,1,1,1}}, // punpck
     };
 
     if (ST->hasSSE2())
       if (const auto *Entry =
               CostTableLookup(SSE2SubVectorShuffleTbl, Kind, VT.getSimpleVT()))
-        return Entry->Cost;
+        if (auto KindCost = Entry->Cost[CostKind])
+          return LT.first * *KindCost;
   }
 
   // We are going to permute multiple sources and the result will be in multiple
@@ -1791,57 +1799,62 @@ InstructionCost X86TTIImpl::getShuffleCost(
     return BaseT::getShuffleCost(Kind, BaseTp, Mask, CostKind, Index, SubTp);
   }
 
-  static const CostTblEntry AVX512VBMIShuffleTbl[] = {
-      {TTI::SK_Reverse, MVT::v64i8, 1}, // vpermb
-      {TTI::SK_Reverse, MVT::v32i8, 1}, // vpermb
-
-      {TTI::SK_PermuteSingleSrc, MVT::v64i8, 1}, // vpermb
-      {TTI::SK_PermuteSingleSrc, MVT::v32i8, 1}, // vpermb
+  // If we're just moving a single element around (probably as an alternative to
+  // extracting it), we can assume this is cheap.
+  if (LT.first == 1 && IsInLaneShuffle && IsSingleElementMask)
+    return TTI::TCC_Basic;
 
-      {TTI::SK_PermuteTwoSrc, MVT::v64i8, 2}, // vpermt2b
-      {TTI::SK_PermuteTwoSrc, MVT::v32i8, 2}, // vpermt2b
-      {TTI::SK_PermuteTwoSrc, MVT::v16i8, 2}  // vpermt2b
+  static const CostKindTblEntry AVX512VBMIShuffleTbl[] = {
+    { TTI::SK_Reverse, MVT::v64i8,          { 1, 1, 1, 1 } }, // vpermb
+    { TTI::SK_Reverse, MVT::v32i8,          { 1, 1, 1, 1 } }, // vpermb
+    { TTI::SK_PermuteSingleSrc, MVT::v64i8, { 1, 1, 1, 1 } }, // vpermb
+    { TTI::SK_PermuteSingleSrc, MVT::v32i8, { 1, 1, 1, 1 } }, // vpermb
+    { TTI::SK_PermuteTwoSrc, MVT::v64i8,    { 2, 2, 2, 2 } }, // vpermt2b
+    { TTI::SK_PermuteTwoSrc, MVT::v32i8,    { 2, 2, 2, 2 } }, // vpermt2b
+    { TTI::SK_PermuteTwoSrc, MVT::v16i8,    { 2, 2, 2, 2 } }  // vpermt2b
   };
 
   if (ST->hasVBMI())
     if (const auto *Entry =
             CostTableLookup(AVX512VBMIShuffleTbl, Kind, LT.second))
-      return LT.first * Entry->Cost;
-
-  static const CostTblEntry AVX512BWShuffleTbl[] = {
-      {TTI::SK_Broadcast, MVT::v32i16, 1}, // vpbroadcastw
-      {TTI::SK_Broadcast, MVT::v32f16, 1}, // vpbroadcastw
-      {TTI::SK_Broadcast, MVT::v64i8, 1},  // vpbroadcastb
-
-      {TTI::SK_Reverse, MVT::v32i16, 2}, // vpermw
-      {TTI::SK_Reverse, MVT::v32f16, 2}, // vpermw
-      {TTI::SK_Reverse, MVT::v16i16, 2}, // vpermw
-      {TTI::SK_Reverse, MVT::v64i8, 2},  // pshufb + vshufi64x2
-
-      {TTI::SK_PermuteSingleSrc, MVT::v32i16, 2}, // vpermw
-      {TTI::SK_PermuteSingleSrc, MVT::v32f16, 2}, // vpermw
-      {TTI::SK_PermuteSingleSrc, MVT::v16i16, 2}, // vpermw
-      {TTI::SK_PermuteSingleSrc, MVT::v16f16, 2}, // vpermw
-      {TTI::SK_PermuteSingleSrc, MVT::v64i8, 8},  // extend to v32i16
-
-      {TTI::SK_PermuteTwoSrc, MVT::v32i16, 2}, // vpermt2w
-      {TTI::SK_PermuteTwoSrc, MVT::v32f16, 2}, // vpermt2w
-      {TTI::SK_PermuteTwoSrc, MVT::v16i16, 2}, // vpermt2w
-      {TTI::SK_PermuteTwoSrc, MVT::v8i16, 2},  // vpermt2w
-      {TTI::SK_PermuteTwoSrc, MVT::v64i8, 19}, // 6 * v32i8 + 1
-
-      {TTI::SK_Select, MVT::v32i16, 1}, // vblendmw
-      {TTI::SK_Select, MVT::v64i8,  1}, // vblendmb
-
-      {TTI::SK_Splice, MVT::v32i16, 2}, // vshufi64x2 + palignr
-      {TTI::SK_Splice, MVT::v32f16, 2}, // vshufi64x2 + palignr
-      {TTI::SK_Splice, MVT::v64i8,  2}, // vshufi64x2 + palignr
+      if (auto KindCost = Entry->Cost[CostKind])
+        return LT.first * *KindCost;
+
+  static const CostKindTblEntry AVX512BWShuffleTbl[] = {
+    { TTI::SK_Broadcast, MVT::v32i16, { 1, 1, 1, 1 } }, // vpbroadcastw
+    { TTI::SK_Broadcast, MVT::v32f16, { 1, 1, 1, 1 } }, // vpbroadcastw
+    { TTI::SK_Broadcast, MVT::v64i8,  { 1, 1, 1, 1 } }, // vpbroadcastb
+
+    { TTI::SK_Reverse, MVT::v32i16,   { 2, 2, 2, 2 } }, // vpermw
+    { TTI::SK_Reverse, MVT::v32f16,   { 2, 2, 2, 2 } }, // vpermw
+    { TTI::SK_Reverse, MVT::v16i16,   { 2, 2, 2, 2 } }, // vpermw
+    { TTI::SK_Reverse, MVT::v64i8,    { 2, 2, 2, 2 } }, // pshufb + vshufi64x2
+
+    { TTI::SK_PermuteSingleSrc, MVT::v32i16, { 2, 2, 2, 2 } }, // vpermw
+    { TTI::SK_PermuteSingleSrc, MVT::v32f16, { 2, 2, 2, 2 } }, // vpermw
+    { TTI::SK_PermuteSingleSrc, MVT::v16i16, { 2, 2, 2, 2 } }, // vpermw
+    { TTI::SK_PermuteSingleSrc, MVT::v16f16, { 2, 2, 2, 2 } }, // vpermw
+    { TTI::SK_PermuteSingleSrc, MVT::v64i8,  { 8, 8, 8, 8 } },  // extend to v32i16
+
+    { TTI::SK_PermuteTwoSrc, MVT::v32i16,{  2,  2,  2,  2 } }, // vpermt2w
+    { TTI::SK_PermuteTwoSrc, MVT::v32f16,{  2,  2,  2,  2 } }, // vpermt2w
+    { TTI::SK_PermuteTwoSrc, MVT::v16i16,{  2,  2,  2,  2 } }, // vpermt2w
+    { TTI::SK_PermuteTwoSrc, MVT::v8i16, {  2,  2,  2,  2 } },  // vpermt2w
+    { TTI::SK_PermuteTwoSrc, MVT::v64i8, { 19, 19, 19, 19 } }, // 6 * v32i8 + 1
+
+    { TTI::SK_Select, MVT::v32i16, { 1, 1, 1, 1 } }, // vblendmw
+    { TTI::SK_Select, MVT::v64i8,  { 1, 1, 1, 1 } }, // vblendmb
+
+    { TTI::SK_Splice, MVT::v32i16, { 2, 2, 2, 2 } }, // vshufi64x2 + palignr
+    { TTI::SK_Splice, MVT::v32f16, { 2, 2, 2, 2 } }, // vshufi64x2 + palignr
+    { TTI::SK_Splice, MVT::v64i8,  { 2, 2, 2, 2 } }, // vshufi64x2 + palignr
   };
 
   if (ST->hasBWI())
     if (const auto *Entry =
             CostTableLookup(AVX512BWShuffleTbl, Kind, LT.second))
-      return LT.first * Entry->Cost;
+      if (auto KindCost = Entry->Cost[CostKind])
+        return LT.first * *KindCost;
 
   static const CostKindTblEntry AVX512ShuffleTbl[] = {
       {TTI::SK_Broadcast, MVT::v8f64,  { 1, 1, 1, 1 } }, // vbroadcastsd
@@ -1922,286 +1935,288 @@ InstructionCost X86TTIImpl::getShuffleCost(
       if (auto KindCost = Entry->Cost[CostKind])
         return LT.first * *KindCost;
 
-  static const CostTblEntry AVX2InLaneShuffleTbl[] = {
-      {TTI::SK_PermuteSingleSrc, MVT::v16i16, 1}, // vpshufb
-      {TTI::SK_PermuteSingleSrc, MVT::v16f16, 1}, // vpshufb
-      {TTI::SK_PermuteSingleSrc, MVT::v32i8,  1}, // vpshufb
-
-      {TTI::SK_PermuteTwoSrc, MVT::v4f64,     2}, // 2*vshufpd + vblendpd
-      {TTI::SK_PermuteTwoSrc, MVT::v8f32,     2}, // 2*vshufps + vblendps
-      {TTI::SK_PermuteTwoSrc, MVT::v4i64,     2}, // 2*vpshufd + vpblendd
-      {TTI::SK_PermuteTwoSrc, MVT::v8i32,     2}, // 2*vpshufd + vpblendd
-      {TTI::SK_PermuteTwoSrc, MVT::v16i16,    2}, // 2*vpshufb + vpor
-      {TTI::SK_PermuteTwoSrc, MVT::v16f16,    2}, // 2*vpshufb + vpor
-      {TTI::SK_PermuteTwoSrc, MVT::v32i8,     2}, // 2*vpshufb + vpor
+  static const CostKindTblEntry AVX2InLaneShuffleTbl[] = {
+    { TTI::SK_PermuteSingleSrc, MVT::v16i16, { 1, 1, 1, 1 } }, // vpshufb
+    { TTI::SK_PermuteSingleSrc, MVT::v16f16, { 1, 1, 1, 1 } }, // vpshufb
+    { TTI::SK_PermuteSingleSrc, MVT::v32i8,  { 1, 1, 1, 1 } }, // vpshufb
+
+    { TTI::SK_PermuteTwoSrc,    MVT::v4f64,  { 2, 2, 2, 2 } }, // 2*vshufpd + vblendpd
+    { TTI::SK_PermuteTwoSrc,    MVT::v8f32,  { 2, 2, 2, 2 } }, // 2*vshufps + vblendps
+    { TTI::SK_PermuteTwoSrc,    MVT::v4i64,  { 2, 2, 2, 2 } }, // 2*vpshufd + vpblendd
+    { TTI::SK_PermuteTwoSrc,    MVT::v8i32,  { 2, 2, 2, 2 } }, // 2*vpshufd + vpblendd
+    { TTI::SK_PermuteTwoSrc,    MVT::v16i16, { 2, 2, 2, 2 } }, // 2*vpshufb + vpor
+    { TTI::SK_PermuteTwoSrc,    MVT::v16f16, { 2, 2, 2, 2 } }, // 2*vpshufb + vpor
+    { TTI::SK_PermuteTwoSrc,    MVT::v32i8,  { 2, 2, 2, 2 } }, // 2*vpshufb + vpor
   };
 
   if (IsInLaneShuffle && ST->hasAVX2())
     if (const auto *Entry =
             CostTableLookup(AVX2InLaneShuffleTbl, Kind, LT.second))
-      return LT.first * Entry->Cost;
-
-  static const CostTblEntry AVX2ShuffleTbl[] = {
-      {TTI::SK_Broadcast, MVT::v4f64, 1},  // vbroadcastpd
-      {TTI::SK_Broadcast, MVT::v8f32, 1},  // vbroadcastps
-      {TTI::SK_Broadcast, MVT::v4i64, 1},  // vpbroadcastq
-      {TTI::SK_Broadcast, MVT::v8i32, 1},  // vpbroadcastd
-      {TTI::SK_Broadcast, MVT::v16i16, 1}, // vpbroadcastw
-      {TTI::SK_Broadcast, MVT::v16f16, 1}, // vpbroadcastw
-      {TTI::SK_Broadcast, MVT::v32i8, 1},  // vpbroadcastb
-
-      {TTI::SK_Reverse, MVT::v4f64, 1},  // vpermpd
-      {TTI::SK_Reverse, MVT::v8f32, 1},  // vpermps
-      {TTI::SK_Reverse, MVT::v4i64, 1},  // vpermq
-      {TTI::SK_Reverse, MVT::v8i32, 1},  // vpermd
-      {TTI::SK_Reverse, MVT::v16i16, 2}, // vperm2i128 + pshufb
-      {TTI::SK_Reverse, MVT::v16f16, 2}, // vperm2i128 + pshufb
-      {TTI::SK_Reverse, MVT::v32i8, 2},  // vperm2i128 + pshufb
-
-      {TTI::SK_Select, MVT::v16i16, 1}, // vpblendvb
-      {TTI::SK_Select, MVT::v16f16, 1}, // vpblendvb
-      {TTI::SK_Select, MVT::v32i8,  1}, // vpblendvb
-
-      {TTI::SK_Splice, MVT::v8i32,  2}, // vperm2i128 + vpalignr
-      {TTI::SK_Splice, MVT::v8f32,  2}, // vperm2i128 + vpalignr
-      {TTI::SK_Splice, MVT::v16i16, 2}, // vperm2i128 + vpalignr
-      {TTI::SK_Splice, MVT::v16f16, 2}, // vperm2i128 + vpalignr
-      {TTI::SK_Splice, MVT::v32i8,  2}, // vperm2i128 + vpalignr
-
-      {TTI::SK_PermuteSingleSrc, MVT::v4f64, 1},  // vpermpd
-      {TTI::SK_PermuteSingleSrc, MVT::v8f32, 1},  // vpermps
-      {TTI::SK_PermuteSingleSrc, MVT::v4i64, 1},  // vpermq
-      {TTI::SK_PermuteSingleSrc, MVT::v8i32, 1},  // vpermd
-      {TTI::SK_PermuteSingleSrc, MVT::v16i16, 4}, // vperm2i128 + 2*vpshufb
-                                                  // + vpblendvb
-      {TTI::SK_PermuteSingleSrc, MVT::v16f16, 4}, // vperm2i128 + 2*vpshufb
-                                                  // + vpblendvb
-      {TTI::SK_PermuteSingleSrc, MVT::v32i8, 4},  // vperm2i128 + 2*vpshufb
-                                                  // + vpblendvb
-
-      {TTI::SK_PermuteTwoSrc, MVT::v4f64, 3},  // 2*vpermpd + vblendpd
-      {TTI::SK_PermuteTwoSrc, MVT::v8f32, 3},  // 2*vpermps + vblendps
-      {TTI::SK_PermuteTwoSrc, MVT::v4i64, 3},  // 2*vpermq + vpblendd
-      {TTI::SK_PermuteTwoSrc, MVT::v8i32, 3},  // 2*vpermd + vpblendd
-      {TTI::SK_PermuteTwoSrc, MVT::v16i16, 7}, // 2*vperm2i128 + 4*vpshufb
-                                               // + vpblendvb
-      {TTI::SK_PermuteTwoSrc, MVT::v16f16, 7}, // 2*vperm2i128 + 4*vpshufb
-                                               // + vpblendvb
-      {TTI::SK_PermuteTwoSrc, MVT::v32i8, 7},  // 2*vperm2i128 + 4*vpshufb
-                                               // + vpblendvb
+      if (auto KindCost = Entry->Cost[CostKind])
+        return LT.first * *KindCost;
+
+  static const CostKindTblEntry AVX2ShuffleTbl[] = {
+    { TTI::SK_Broadcast, MVT::v4f64,  { 1, 1, 1, 1 } }, // vbroadcastpd
+    { TTI::SK_Broadcast, MVT::v8f32,  { 1, 1, 1, 1 } }, // vbroadcastps
+    { TTI::SK_Broadcast, MVT::v4i64,  { 1, 1, 1, 1 } }, // vpbroadcastq
+    { TTI::SK_Broadcast, MVT::v8i32,  { 1, 1, 1, 1 } }, // vpbroadcastd
+    { TTI::SK_Broadcast, MVT::v16i16, { 1, 1, 1, 1 } }, // vpbroadcastw
+    { TTI::SK_Broadcast, MVT::v16f16, { 1, 1, 1, 1 } }, // vpbroadcastw
+    { TTI::SK_Broadcast, MVT::v32i8,  { 1, 1, 1, 1 } }, // vpbroadcastb
+
+    { TTI::SK_Reverse, MVT::v4f64,    { 1, 1, 1, 1 } }, // vpermpd
+    { TTI::SK_Reverse, MVT::v8f32,    { 1, 1, 1, 1 } }, // vpermps
+    { TTI::SK_Reverse, MVT::v4i64,    { 1, 1, 1, 1 } }, // vpermq
+    { TTI::SK_Reverse, MVT::v8i32,    { 1, 1, 1, 1 } }, // vpermd
+    { TTI::SK_Reverse, MVT::v16i16,   { 2, 2, 2, 2 } }, // vperm2i128 + pshufb
+    { TTI::SK_Reverse, MVT::v16f16,   { 2, 2, 2, 2 } }, // vperm2i128 + pshufb
+    { TTI::SK_Reverse, MVT::v32i8,    { 2, 2, 2, 2 } }, // vperm2i128 + pshufb
+
+    { TTI::SK_Select, MVT::v16i16,    { 1, 1, 1, 1 } }, // vpblendvb
+    { TTI::SK_Select, MVT::v16f16,    { 1, 1, 1, 1 } }, // vpblendvb
+    { TTI::SK_Select, MVT::v32i8,     { 1, 1, 1, 1 } }, // vpblendvb
+
+    { TTI::SK_Splice, MVT::v8i32,     { 2, 2, 2, 2 } }, // vperm2i128 + vpalignr
+    { TTI::SK_Splice, MVT::v8f32,     { 2, 2, 2, 2 } }, // vperm2i128 + vpalignr
+    { TTI::SK_Splice, MVT::v16i16,    { 2, 2, 2, 2 } }, // vperm2i128 + vpalignr
+    { TTI::SK_Splice, MVT::v16f16,    { 2, 2, 2, 2 } }, // vperm2i128 + vpalignr
+    { TTI::SK_Splice, MVT::v32i8,     { 2, 2, 2, 2 } }, // vperm2i128 + vpalignr
+
+    { TTI::SK_PermuteSingleSrc, MVT::v4f64,  { 1, 1, 1, 1 } }, // vpermpd
+    { TTI::SK_PermuteSingleSrc, MVT::v8f32,  { 1, 1, 1, 1 } }, // vpermps
+    { TTI::SK_PermuteSingleSrc, MVT::v4i64,  { 1, 1, 1, 1 } }, // vpermq
+    { TTI::SK_PermuteSingleSrc, MVT::v8i32,  { 1, 1, 1, 1 } }, // vpermd
+    { TTI::SK_PermuteSingleSrc, MVT::v16i16, { 4, 4, 4, 4 } },
+    { TTI::SK_PermuteSingleSrc, MVT::v16f16, { 4, 4, 4, 4 } },
+    { TTI::SK_PermuteSingleSrc, MVT::v32i8,  { 4, 4, 4, 4 } },
+
+    { TTI::SK_PermuteTwoSrc, MVT::v4f64,  { 3, 3, 3, 3 } }, // 2*vpermpd + vblendpd
+    { TTI::SK_PermuteTwoSrc, MVT::v8f32,  { 3, 3, 3, 3 } }, // 2*vpermps + vblendps
+    { TTI::SK_PermuteTwoSrc, MVT::v4i64,  { 3, 3, 3, 3 } }, // 2*vpermq + vpblendd
+    { TTI::SK_PermuteTwoSrc, MVT::v8i32,  { 3, 3, 3, 3 } }, // 2*vpermd + vpblendd
+    { TTI::SK_PermuteTwoSrc, MVT::v16i16, { 7, 7, 7, 7 } },
+    { TTI::SK_PermuteTwoSrc, MVT::v16f16, { 7, 7, 7, 7 } },
+    { TTI::SK_PermuteTwoSrc, MVT::v32i8,  { 7, 7, 7, 7 } },
   };
 
   if (ST->hasAVX2())
     if (const auto *Entry = CostTableLookup(AVX2ShuffleTbl, Kind, LT.second))
-      return LT.first * Entry->Cost;
-
-  static const CostTblEntry XOPShuffleTbl[] = {
-      {TTI::SK_PermuteSingleSrc, MVT::v4f64, 2},  // vperm2f128 + vpermil2pd
-      {TTI::SK_PermuteSingleSrc, MVT::v8f32, 2},  // vperm2f128 + vpermil2ps
-      {TTI::SK_PermuteSingleSrc, MVT::v4i64, 2},  // vperm2f128 + vpermil2pd
-      {TTI::SK_PermuteSingleSrc, MVT::v8i32, 2},  // vperm2f128 + vpermil2ps
-      {TTI::SK_PermuteSingleSrc, MVT::v16i16, 4}, // vextractf128 + 2*vpperm
-                                                  // + vinsertf128
-      {TTI::SK_PermuteSingleSrc, MVT::v32i8, 4},  // vextractf128 + 2*vpperm
-                                                  // + vinsertf128
-
-      {TTI::SK_PermuteTwoSrc, MVT::v16i16, 9}, // 2*vextractf128 + 6*vpperm
-                                               // + vinsertf128
-      {TTI::SK_PermuteTwoSrc, MVT::v8i16, 1},  // vpperm
-      {TTI::SK_PermuteTwoSrc, MVT::v32i8, 9},  // 2*vextractf128 + 6*vpperm
-                                               // + vinsertf128
-      {TTI::SK_PermuteTwoSrc, MVT::v16i8, 1},  // vpperm
+      if (auto KindCost = Entry->Cost[CostKind])
+        return LT.first * *KindCost;
+
+  static const CostKindTblEntry XOPShuffleTbl[] = {
+    { TTI::SK_PermuteSingleSrc, MVT::v4f64, { 2, 2, 2, 2 } }, // vperm2f128 + vpermil2pd
+    { TTI::SK_PermuteSingleSrc, MVT::v8f32, { 2, 2, 2, 2 } }, // vperm2f128 + vpermil2ps
+    { TTI::SK_PermuteSingleSrc, MVT::v4i64, { 2, 2, 2, 2 } }, // vperm2f128 + vpermil2pd
+    { TTI::SK_PermuteSingleSrc, MVT::v8i32, { 2, 2, 2, 2 } }, // vperm2f128 + vpermil2ps
+    { TTI::SK_PermuteSingleSrc, MVT::v16i16,{ 4, 4, 4, 4 } }, // vextractf128 + 2*vpperm
+                                                             // + vinsertf128
+    { TTI::SK_PermuteSingleSrc, MVT::v32i8, { 4, 4, 4, 4 } }, // vextractf128 + 2*vpperm
+                                                             // + vinsertf128
+
+    { TTI::SK_PermuteTwoSrc, MVT::v16i16, { 9, 9, 9, 9 } }, // 2*vextractf128 + 6*vpperm
+                                                            // + vinsertf128
+
+    { TTI::SK_PermuteTwoSrc, MVT::v8i16,  { 1, 1, 1, 1 } }, // vpperm
+    { TTI::SK_PermuteTwoSrc, MVT::v32i8,  { 9, 9, 9, 9 } }, // 2*vextractf128 + 6*vpperm
+                                                            // + vinsertf128
+    { TTI::SK_PermuteTwoSrc, MVT::v16i8,  { 1, 1, 1, 1 } }, // vpperm
   };
 
   if (ST->hasXOP())
     if (const auto *Entry = CostTableLookup(XOPShuffleTbl, Kind, LT.second))
-      return LT.first * Entry->Cost;
-
-  static const CostTblEntry AVX1InLaneShuffleTbl[] = {
-      {TTI::SK_PermuteSingleSrc, MVT::v4f64,  1},  // vpermilpd
-      {TTI::SK_PermuteSingleSrc, MVT::v4i64,  1},  // vpermilpd
-      {TTI::SK_PermuteSingleSrc, MVT::v8f32,  1},  // vpermilps
-      {TTI::SK_PermuteSingleSrc, MVT::v8i32,  1},  // vpermilps
-
-      {TTI::SK_PermuteSingleSrc, MVT::v16i16, 4}, // vextractf128 + 2*pshufb
-                                                  // + vpor + vinsertf128
-      {TTI::SK_PermuteSingleSrc, MVT::v16f16, 4}, // vextractf128 + 2*pshufb
-                                                  // + vpor + vinsertf128
-      {TTI::SK_PermuteSingleSrc, MVT::v32i8,  4}, // vextractf128 + 2*pshufb
-                                                  // + vpor + vinsertf128
-
-      {TTI::SK_PermuteTwoSrc, MVT::v4f64,     2}, // 2*vshufpd + vblendpd
-      {TTI::SK_PermuteTwoSrc, MVT::v8f32,     2}, // 2*vshufps + vblendps
-      {TTI::SK_PermuteTwoSrc, MVT::v4i64,     2}, // 2*vpermilpd + vblendpd
-      {TTI::SK_PermuteTwoSrc, MVT::v8i32,     2}, // 2*vpermilps + vblendps
-      {TTI::SK_PermuteTwoSrc, MVT::v16i16,    9}, // 2*vextractf128 + 4*pshufb
-                                                  // + 2*vpor + vinsertf128
-      {TTI::SK_PermuteTwoSrc, MVT::v16f16,    9}, // 2*vextractf128 + 4*pshufb
-                                                  // + 2*vpor + vinsertf128
-      {TTI::SK_PermuteTwoSrc, MVT::v32i8,     9}, // 2*vextractf128 + 4*pshufb
-                                                  // + 2*vpor + vinsertf128
+      if (auto KindCost = Entry->Cost[CostKind])
+        return LT.first * *KindCost;
+
+  static const CostKindTblEntry AVX1InLaneShuffleTbl[] = {
+    { TTI::SK_PermuteSingleSrc, MVT::v4f64,  { 1, 1, 1, 1 } }, // vpermilpd
+    { TTI::SK_PermuteSingleSrc, MVT::v4i64,  { 1, 1, 1, 1 } }, // vpermilpd
+    { TTI::SK_PermuteSingleSrc, MVT::v8f32,  { 1, 1, 1, 1 } }, // vpermilps
+    { TTI::SK_PermuteSingleSrc, MVT::v8i32,  { 1, 1, 1, 1 } }, // vpermilps
+
+    { TTI::SK_PermuteSingleSrc, MVT::v16i16, { 4, 4, 4, 4 } }, // vextractf128 + 2*pshufb
+                                                               // + vpor + vinsertf128
+    { TTI::SK_PermuteSingleSrc, MVT::v16f16, { 4, 4, 4, 4 } }, // vextractf128 + 2*pshufb
+                                                               // + vpor + vinsertf128
+    { TTI::SK_PermuteSingleSrc, MVT::v32i8,  { 4, 4, 4, 4 } }, // vextractf128 + 2*pshufb
+                                                               // + vpor + vinsertf128
+
+    { TTI::SK_PermuteTwoSrc, MVT::v4f64,  { 2, 2, 2, 2 } }, // 2*vshufpd + vblendpd
+    { TTI::SK_PermuteTwoSrc, MVT::v8f32,  { 2, 2, 2, 2 } }, // 2*vshufps + vblendps
+    { TTI::SK_PermuteTwoSrc, MVT::v4i64,  { 2, 2, 2, 2 } }, // 2*vpermilpd + vblendpd
+    { TTI::SK_PermuteTwoSrc, MVT::v8i32,  { 2, 2, 2, 2 } }, // 2*vpermilps + vblendps
+    { TTI::SK_PermuteTwoSrc, MVT::v16i16, { 9, 9, 9, 9 } }, // 2*vextractf128 + 4*pshufb
+                                                            // + 2*vpor + vinsertf128
+    { TTI::SK_PermuteTwoSrc, MVT::v16f16, { 9, 9, 9, 9 } }, // 2*vextractf128 + 4*pshufb
+                                                            // + 2*vpor + vinsertf128
+    { TTI::SK_PermuteTwoSrc, MVT::v32i8,  { 9, 9, 9, 9 } }, // 2*vextractf128 + 4*pshufb
+                                                            // + 2*vpor + vinsertf128
   };
 
   if (IsInLaneShuffle && ST->hasAVX())
     if (const auto *Entry =
             CostTableLookup(AVX1InLaneShuffleTbl, Kind, LT.second))
-      return LT.first * Entry->Cost;
-
-  static const CostTblEntry AVX1ShuffleTbl[] = {
-      {TTI::SK_Broadcast, MVT::v4f64, 2},  // vperm2f128 + vpermilpd
-      {TTI::SK_Broadcast, MVT::v8f32, 2},  // vperm2f128 + vpermilps
-      {TTI::SK_Broadcast, MVT::v4i64, 2},  // vperm2f128 + vpermilpd
-      {TTI::SK_Broadcast, MVT::v8i32, 2},  // vperm2f128 + vpermilps
-      {TTI::SK_Broadcast, MVT::v16i16, 3}, // vpshuflw + vpshufd + vinsertf128
-      {TTI::SK_Broadcast, MVT::v16f16, 3}, // vpshuflw + vpshufd + vinsertf128
-      {TTI::SK_Broadcast, MVT::v32i8, 2},  // vpshufb + vinsertf128
-
-      {TTI::SK_Reverse, MVT::v4f64, 2},  // vperm2f128 + vpermilpd
-      {TTI::SK_Reverse, MVT::v8f32, 2},  // vperm2f128 + vpermilps
-      {TTI::SK_Reverse, MVT::v4i64, 2},  // vperm2f128 + vpermilpd
-      {TTI::SK_Reverse, MVT::v8i32, 2},  // vperm2f128 + vpermilps
-      {TTI::SK_Reverse, MVT::v16i16, 4}, // vextractf128 + 2*pshufb
-                                         // + vinsertf128
-      {TTI::SK_Reverse, MVT::v16f16, 4}, // vextractf128 + 2*pshufb
-                                         // + vinsertf128
-      {TTI::SK_Reverse, MVT::v32i8, 4},  // vextractf128 + 2*pshufb
-                                         // + vinsertf128
-
-      {TTI::SK_Select, MVT::v4i64, 1},  // vblendpd
-      {TTI::SK_Select, MVT::v4f64, 1},  // vblendpd
-      {TTI::SK_Select, MVT::v8i32, 1},  // vblendps
-      {TTI::SK_Select, MVT::v8f32, 1},  // vblendps
-      {TTI::SK_Select, MVT::v16i16, 3}, // vpand + vpandn + vpor
-      {TTI::SK_Select, MVT::v16f16, 3}, // vpand + vpandn + vpor
-      {TTI::SK_Select, MVT::v32i8, 3},  // vpand + vpandn + vpor
-
-      {TTI::SK_Splice, MVT::v4i64,  2}, // vperm2f128 + shufpd
-      {TTI::SK_Splice, MVT::v4f64,  2}, // vperm2f128 + shufpd
-      {TTI::SK_Splice, MVT::v8i32,  4}, // 2*vperm2f128 + 2*vshufps
-      {TTI::SK_Splice, MVT::v8f32,  4}, // 2*vperm2f128 + 2*vshufps
-      {TTI::SK_Splice, MVT::v16i16, 5}, // 2*vperm2f128 + 2*vpalignr + vinsertf128
-      {TTI::SK_Splice, MVT::v16f16, 5}, // 2*vperm2f128 + 2*vpalignr + vinsertf128
-      {TTI::SK_Splice, MVT::v32i8,  5}, // 2*vperm2f128 + 2*vpalignr + vinsertf128
-
-      {TTI::SK_PermuteSingleSrc, MVT::v4f64, 2},  // vperm2f128 + vshufpd
-      {TTI::SK_PermuteSingleSrc, MVT::v4i64, 2},  // vperm2f128 + vshufpd
-      {TTI::SK_PermuteSingleSrc, MVT::v8f32, 4},  // 2*vperm2f128 + 2*vshufps
-      {TTI::SK_PermuteSingleSrc, MVT::v8i32, 4},  // 2*vperm2f128 + 2*vshufps
-      {TTI::SK_PermuteSingleSrc, MVT::v16i16, 8}, // vextractf128 + 4*pshufb
-                                                  // + 2*por + vinsertf128
-      {TTI::SK_PermuteSingleSrc, MVT::v16f16, 8}, // vextractf128 + 4*pshufb
-                                                  // + 2*por + vinsertf128
-      {TTI::SK_PermuteSingleSrc, MVT::v32i8, 8},  // vextractf128 + 4*pshufb
-                                                  // + 2*por + vinsertf128
-
-      {TTI::SK_PermuteTwoSrc, MVT::v4f64, 3},   // 2*vperm2f128 + vshufpd
-      {TTI::SK_PermuteTwoSrc, MVT::v4i64, 3},   // 2*vperm2f128 + vshufpd
-      {TTI::SK_PermuteTwoSrc, MVT::v8f32, 4},   // 2*vperm2f128 + 2*vshufps
-      {TTI::SK_PermuteTwoSrc, MVT::v8i32, 4},   // 2*vperm2f128 + 2*vshufps
-      {TTI::SK_PermuteTwoSrc, MVT::v16i16, 15}, // 2*vextractf128 + 8*pshufb
-                                                // + 4*por + vinsertf128
-      {TTI::SK_PermuteTwoSrc, MVT::v16f16, 15}, // 2*vextractf128 + 8*pshufb
-                                                // + 4*por + vinsertf128
-      {TTI::SK_PermuteTwoSrc, MVT::v32i8, 15},  // 2*vextractf128 + 8*pshufb
-                                                // + 4*por + vinsertf128
+      if (auto KindCost = Entry->Cost[CostKind])
+        return LT.first * *KindCost;
+
+  static const CostKindTblEntry AVX1ShuffleTbl[] = {
+      {TTI::SK_Broadcast, MVT::v4f64,  {2,2,2,2}}, // vperm2f128 + vpermilpd
+      {TTI::SK_Broadcast, MVT::v8f32,  {2,2,2,2}}, // vperm2f128 + vpermilps
+      {TTI::SK_Broadcast, MVT::v4i64,  {2,2,2,2}}, // vperm2f128 + vpermilpd
+      {TTI::SK_Broadcast, MVT::v8i32,  {2,2,2,2}}, // vperm2f128 + vpermilps
+      {TTI::SK_Broadcast, MVT::v16i16, {3,3,3,3}}, // vpshuflw + vpshufd + vinsertf128
+      {TTI::SK_Broadcast, MVT::v16f16, {3,3,3,3}}, // vpshuflw + vpshufd + vinsertf128
+      {TTI::SK_Broadcast, MVT::v32i8,  {2,2,2,2}}, // vpshufb + vinsertf128
+
+      {TTI::SK_Reverse, MVT::v4f64,  {2,2,2,2}}, // vperm2f128 + vpermilpd
+      {TTI::SK_Reverse, MVT::v8f32,  {2,2,2,2}}, // vperm2f128 + vpermilps
+      {TTI::SK_Reverse, MVT::v4i64,  {2,2,2,2}}, // vperm2f128 + vpermilpd
+      {TTI::SK_Reverse, MVT::v8i32,  {2,2,2,2}}, // vperm2f128 + vpermilps
+      {TTI::SK_Reverse, MVT::v16i16, {4,4,4,4}}, // vextractf128 + 2*pshufb
+                                                 // + vinsertf128
+      {TTI::SK_Reverse, MVT::v16f16, {4,4,4,4}}, // vextractf128 + 2*pshufb
+                                                 // + vinsertf128
+      {TTI::SK_Reverse, MVT::v32i8,  {4,4,4,4}}, // vextractf128 + 2*pshufb
+                                                 // + vinsertf128
+
+      {TTI::SK_Select, MVT::v4i64,  {1,1,1,1}}, // vblendpd
+      {TTI::SK_Select, MVT::v4f64,  {1,1,1,1}}, // vblendpd
+      {TTI::SK_Select, MVT::v8i32,  {1,1,1,1}}, // vblendps
+      {TTI::SK_Select, MVT::v8f32,  {1,1,1,1}}, // vblendps
+      {TTI::SK_Select, MVT::v16i16, {3,3,3,3}}, // vpand + vpandn + vpor
+      {TTI::SK_Select, MVT::v16f16, {3,3,3,3}}, // vpand + vpandn + vpor
+      {TTI::SK_Select, MVT::v32i8,  {3,3,3,3}}, // vpand + vpandn + vpor
+
+      {TTI::SK_Splice, MVT::v4i64,  {2,2,2,2}}, // vperm2f128 + shufpd
+      {TTI::SK_Splice, MVT::v4f64,  {2,2,2,2}}, // vperm2f128 + shufpd
+      {TTI::SK_Splice, MVT::v8i32,  {4,4,4,4}}, // 2*vperm2f128 + 2*vshufps
+      {TTI::SK_Splice, MVT::v8f32,  {4,4,4,4}}, // 2*vperm2f128 + 2*vshufps
+      {TTI::SK_Splice, MVT::v16i16, {5,5,5,5}}, // 2*vperm2f128 + 2*vpalignr + vinsertf128
+      {TTI::SK_Splice, MVT::v16f16, {5,5,5,5}}, // 2*vperm2f128 + 2*vpalignr + vinsertf128
+      {TTI::SK_Splice, MVT::v32i8,  {5,5,5,5}}, // 2*vperm2f128 + 2*vpalignr + vinsertf128
+
+      {TTI::SK_PermuteSingleSrc, MVT::v4f64, {2,2,2,2}}, // vperm2f128 + vshufpd
+      {TTI::SK_PermuteSingleSrc, MVT::v4i64, {2,2,2,2}}, // vperm2f128 + vshufpd
+      {TTI::SK_PermuteSingleSrc, MVT::v8f32, {4,4,4,4}}, // 2*vperm2f128 + 2*vshufps
+      {TTI::SK_PermuteSingleSrc, MVT::v8i32, {4,4,4,4}}, // 2*vperm2f128 + 2*vshufps
+      {TTI::SK_PermuteSingleSrc, MVT::v16i16,{8,8,8,8}}, // vextractf128 + 4*pshufb
+                                                         // + 2*por + vinsertf128
+      {TTI::SK_PermuteSingleSrc, MVT::v16f16,{8,8,8,8}}, // vextractf128 + 4*pshufb
+                                                         // + 2*por + vinsertf128
+      {TTI::SK_PermuteSingleSrc, MVT::v32i8, {8,8,8,8}}, // vextractf128 + 4*pshufb
+                                                         // + 2*por + vinsertf128
+
+      {TTI::SK_PermuteTwoSrc, MVT::v4f64, {3,3,3,3}},   // 2*vperm2f128 + vshufpd
+      {TTI::SK_PermuteTwoSrc, MVT::v4i64, {3,3,3,3}},   // 2*vperm2f128 + vshufpd
+      {TTI::SK_PermuteTwoSrc, MVT::v8f32, {4,4,4,4}},   // 2*vperm2f128 + 2*vshufps
+      {TTI::SK_PermuteTwoSrc, MVT::v8i32, {4,4,4,4}},   // 2*vperm2f128 + 2*vshufps
+      {TTI::SK_PermuteTwoSrc, MVT::v16i16,{15,15,15,15}}, // 2*vextractf128 + 8*pshufb
+                                                          // + 4*por + vinsertf128
+      {TTI::SK_PermuteTwoSrc, MVT::v16f16,{15,15,15,15}}, // 2*vextractf128 + 8*pshufb
+                                                          // + 4*por + vinsertf128
+      {TTI::SK_PermuteTwoSrc, MVT::v32i8, {15,15,15,15}}, // 2*vextractf128 + 8*pshufb
+                                                         // + 4*por + vinsertf128
   };
 
   if (ST->hasAVX())
     if (const auto *Entry = CostTableLookup(AVX1ShuffleTbl, Kind, LT.second))
-      return LT.first * Entry->Cost;
-
-  static const CostTblEntry SSE41ShuffleTbl[] = {
-      {TTI::SK_Select, MVT::v2i64, 1}, // pblendw
-      {TTI::SK_Select, MVT::v2f64, 1}, // movsd
-      {TTI::SK_Select, MVT::v4i32, 1}, // pblendw
-      {TTI::SK_Select, MVT::v4f32, 1}, // blendps
-      {TTI::SK_Select, MVT::v8i16, 1}, // pblendw
-      {TTI::SK_Select, MVT::v8f16, 1}, // pblendw
-      {TTI::SK_Select, MVT::v16i8, 1}  // pblendvb
+      if (auto KindCost = Entry->Cost[CostKind])
+        return LT.first * *KindCost;
+
+  static const CostKindTblEntry SSE41ShuffleTbl[] = {
+      {TTI::SK_Select, MVT::v2i64, {1,1,1,1}}, // pblendw
+      {TTI::SK_Select, MVT::v2f64, {1,1,1,1}}, // movsd
+      {TTI::SK_Select, MVT::v4i32, {1,1,1,1}}, // pblendw
+      {TTI::SK_Select, MVT::v4f32, {1,1,1,1}}, // blendps
+      {TTI::SK_Select, MVT::v8i16, {1,1,1,1}}, // pblendw
+      {TTI::SK_Select, MVT::v8f16, {1,1,1,1}}, // pblendw
+      {TTI::SK_Select, MVT::v16i8, {1,1,1,1}}  // pblendvb
   };
 
   if (ST->hasSSE41())
     if (const auto *Entry = CostTableLookup(SSE41ShuffleTbl, Kind, LT.second))
-      return LT.first * Entry->Cost;
-
-  static const CostTblEntry SSSE3ShuffleTbl[] = {
-      {TTI::SK_Broadcast, MVT::v8i16, 1}, // pshufb
-      {TTI::SK_Broadcast, MVT::v8f16, 1}, // pshufb
-      {TTI::SK_Broadcast, MVT::v16i8, 1}, // pshufb
-
-      {TTI::SK_Reverse, MVT::v8i16, 1}, // pshufb
-      {TTI::SK_Reverse, MVT::v8f16, 1}, // pshufb
-      {TTI::SK_Reverse, MVT::v16i8, 1}, // pshufb
-
-      {TTI::SK_Select, MVT::v8i16, 3}, // 2*pshufb + por
-      {TTI::SK_Select, MVT::v8f16, 3}, // 2*pshufb + por
-      {TTI::SK_Select, MVT::v16i8, 3}, // 2*pshufb + por
-
-      {TTI::SK_Splice, MVT::v4i32, 1}, // palignr
-      {TTI::SK_Splice, MVT::v4f32, 1}, // palignr
-      {TTI::SK_Splice, MVT::v8i16, 1}, // palignr
-      {TTI::SK_Splice, MVT::v8f16, 1}, // palignr
-      {TTI::SK_Splice, MVT::v16i8, 1}, // palignr
-
-      {TTI::SK_PermuteSingleSrc, MVT::v8i16, 1}, // pshufb
-      {TTI::SK_PermuteSingleSrc, MVT::v8f16, 1}, // pshufb
-      {TTI::SK_PermuteSingleSrc, MVT::v16i8, 1}, // pshufb
-
-      {TTI::SK_PermuteTwoSrc, MVT::v8i16, 3}, // 2*pshufb + por
-      {TTI::SK_PermuteTwoSrc, MVT::v8f16, 3}, // 2*pshufb + por
-      {TTI::SK_PermuteTwoSrc, MVT::v16i8, 3}, // 2*pshufb + por
+      if (auto KindCost = Entry->Cost[CostKind])
+        return LT.first * *KindCost;
+
+  static const CostKindTblEntry SSSE3ShuffleTbl[] = {
+      {TTI::SK_Broadcast, MVT::v8i16, {1, 1, 1, 1}}, // pshufb
+      {TTI::SK_Broadcast, MVT::v8f16, {1, 1, 1, 1}}, // pshufb
+      {TTI::SK_Broadcast, MVT::v16i8, {1, 1, 1, 1}}, // pshufb
+
+      {TTI::SK_Reverse, MVT::v8i16, {1, 1, 1, 1}}, // pshufb
+      {TTI::SK_Reverse, MVT::v8f16, {1, 1, 1, 1}}, // pshufb
+      {TTI::SK_Reverse, MVT::v16i8, {1, 1, 1, 1}}, // pshufb
+
+      {TTI::SK_Select, MVT::v8i16, {3, 3, 3, 3}}, // 2*pshufb + por
+      {TTI::SK_Select, MVT::v8f16, {3, 3, 3, 3}}, // 2*pshufb + por
+      {TTI::SK_Select, MVT::v16i8, {3, 3, 3, 3}}, // 2*pshufb + por
+
+      {TTI::SK_Splice, MVT::v4i32, {1, 1, 1, 1}}, // palignr
+      {TTI::SK_Splice, MVT::v4f32, {1, 1, 1, 1}}, // palignr
+      {TTI::SK_Splice, MVT::v8i16, {1, 1, 1, 1}}, // palignr
+      {TTI::SK_Splice, MVT::v8f16, {1, 1, 1, 1}}, // palignr
+      {TTI::SK_Splice, MVT::v16i8, {1, 1, 1, 1}}, // palignr
+
+      {TTI::SK_PermuteSingleSrc, MVT::v8i16, {1, 1, 1, 1}}, // pshufb
+      {TTI::SK_PermuteSingleSrc, MVT::v8f16, {1, 1, 1, 1}}, // pshufb
+      {TTI::SK_PermuteSingleSrc, MVT::v16i8, {1, 1, 1, 1}}, // pshufb
+
+      {TTI::SK_PermuteTwoSrc, MVT::v8i16, {3, 3, 3, 3}}, // 2*pshufb + por
+      {TTI::SK_PermuteTwoSrc, MVT::v8f16, {3, 3, 3, 3}}, // 2*pshufb + por
+      {TTI::SK_PermuteTwoSrc, MVT::v16i8, {3, 3, 3, 3}}, // 2*pshufb + por
   };
 
   if (ST->hasSSSE3())
     if (const auto *Entry = CostTableLookup(SSSE3ShuffleTbl, Kind, LT.second))
-      return LT.first * Entry->Cost;
-
-  static const CostTblEntry SSE2ShuffleTbl[] = {
-      {TTI::SK_Broadcast, MVT::v2f64, 1}, // shufpd
-      {TTI::SK_Broadcast, MVT::v2i64, 1}, // pshufd
-      {TTI::SK_Broadcast, MVT::v4i32, 1}, // pshufd
-      {TTI::SK_Broadcast, MVT::v8i16, 2}, // pshuflw + pshufd
-      {TTI::SK_Broadcast, MVT::v8f16, 2}, // pshuflw + pshufd
-      {TTI::SK_Broadcast, MVT::v16i8, 3}, // unpck + pshuflw + pshufd
-
-      {TTI::SK_Reverse, MVT::v2f64, 1}, // shufpd
-      {TTI::SK_Reverse, MVT::v2i64, 1}, // pshufd
-      {TTI::SK_Reverse, MVT::v4i32, 1}, // pshufd
-      {TTI::SK_Reverse, MVT::v8i16, 3}, // pshuflw + pshufhw + pshufd
-      {TTI::SK_Reverse, MVT::v8f16, 3}, // pshuflw + pshufhw + pshufd
-      {TTI::SK_Reverse, MVT::v16i8, 9}, // 2*pshuflw + 2*pshufhw
-                                        // + 2*pshufd + 2*unpck + packus
-
-      {TTI::SK_Select, MVT::v2i64, 1}, // movsd
-      {TTI::SK_Select, MVT::v2f64, 1}, // movsd
-      {TTI::SK_Select, MVT::v4i32, 2}, // 2*shufps
-      {TTI::SK_Select, MVT::v8i16, 3}, // pand + pandn + por
-      {TTI::SK_Select, MVT::v8f16, 3}, // pand + pandn + por
-      {TTI::SK_Select, MVT::v16i8, 3}, // pand + pandn + por
-
-      {TTI::SK_Splice, MVT::v2i64, 1}, // shufpd
-      {TTI::SK_Splice, MVT::v2f64, 1}, // shufpd
-      {TTI::SK_Splice, MVT::v4i32, 2}, // 2*{unpck,movsd,pshufd}
-      {TTI::SK_Splice, MVT::v8i16, 3}, // psrldq + psrlldq + por
-      {TTI::SK_Splice, MVT::v8f16, 3}, // psrldq + psrlldq + por
-      {TTI::SK_Splice, MVT::v16i8, 3}, // psrldq + psrlldq + por
-
-      {TTI::SK_PermuteSingleSrc, MVT::v2f64, 1}, // shufpd
-      {TTI::SK_PermuteSingleSrc, MVT::v2i64, 1}, // pshufd
-      {TTI::SK_PermuteSingleSrc, MVT::v4i32, 1}, // pshufd
-      {TTI::SK_PermuteSingleSrc, MVT::v8i16, 5}, // 2*pshuflw + 2*pshufhw
-                                                  // + pshufd/unpck
-      {TTI::SK_PermuteSingleSrc, MVT::v8f16, 5}, // 2*pshuflw + 2*pshufhw
-                                                  // + pshufd/unpck
-    { TTI::SK_PermuteSingleSrc, MVT::v16i8, 10 }, // 2*pshuflw + 2*pshufhw
-                                                  // + 2*pshufd + 2*unpck + 2*packus
-
-    { TTI::SK_PermuteTwoSrc,    MVT::v2f64,  1 }, // shufpd
-    { TTI::SK_PermuteTwoSrc,    MVT::v2i64,  1 }, // shufpd
-    { TTI::SK_PermuteTwoSrc,    MVT::v4i32,  2 }, // 2*{unpck,movsd,pshufd}
-    { TTI::SK_PermuteTwoSrc,    MVT::v8i16,  8 }, // blend+permute
-    { TTI::SK_PermuteTwoSrc,    MVT::v8f16,  8 }, // blend+permute
-    { TTI::SK_PermuteTwoSrc,    MVT::v16i8, 13 }, // blend+permute
+      if (auto KindCost = Entry->Cost[CostKind])
+        return LT.first * *KindCost;
+
+  static const CostKindTblEntry SSE2ShuffleTbl[] = {
+      {TTI::SK_Broadcast, MVT::v2f64, {1, 1, 1, 1}}, // shufpd
+      {TTI::SK_Broadcast, MVT::v2i64, {1, 1, 1, 1}}, // pshufd
+      {TTI::SK_Broadcast, MVT::v4i32, {1, 1, 1, 1}}, // pshufd
+      {TTI::SK_Broadcast, MVT::v8i16, {2, 2, 2, 2}}, // pshuflw + pshufd
+      {TTI::SK_Broadcast, MVT::v8f16, {2, 2, 2, 2}}, // pshuflw + pshufd
+      {TTI::SK_Broadcast, MVT::v16i8, {3, 3, 3, 3}}, // unpck + pshuflw + pshufd
+
+      {TTI::SK_Reverse, MVT::v2f64, {1, 1, 1, 1}}, // shufpd
+      {TTI::SK_Reverse, MVT::v2i64, {1, 1, 1, 1}}, // pshufd
+      {TTI::SK_Reverse, MVT::v4i32, {1, 1, 1, 1}}, // pshufd
+      {TTI::SK_Reverse, MVT::v8i16, {3, 3, 3, 3}}, // pshuflw + pshufhw + pshufd
+      {TTI::SK_Reverse, MVT::v8f16, {3, 3, 3, 3}}, // pshuflw + pshufhw + pshufd
+      {TTI::SK_Reverse, MVT::v16i8, {9, 9, 9, 9}}, // 2*pshuflw + 2*pshufhw
+                                                   // + 2*pshufd + 2*unpck + packus
+
+      {TTI::SK_Select, MVT::v2i64, {1, 1, 1, 1}}, // movsd
+      {TTI::SK_Select, MVT::v2f64, {1, 1, 1, 1}}, // movsd
+      {TTI::SK_Select, MVT::v4i32, {2, 2, 2, 2}}, // 2*shufps
+      {TTI::SK_Select, MVT::v8i16, {3, 3, 3, 3}}, // pand + pandn + por
+      {TTI::SK_Select, MVT::v8f16, {3, 3, 3, 3}}, // pand + pandn + por
+      {TTI::SK_Select, MVT::v16i8, {3, 3, 3, 3}}, // pand + pandn + por
+
+      {TTI::SK_Splice, MVT::v2i64, {1, 1, 1, 1}}, // shufpd
+      {TTI::SK_Splice, MVT::v2f64, {1, 1, 1, 1}}, // shufpd
+      {TTI::SK_Splice, MVT::v4i32, {2, 2, 2, 2}}, // 2*{unpck,movsd,pshufd}
+      {TTI::SK_Splice, MVT::v8i16, {3, 3, 3, 3}}, // psrldq + psrlldq + por
+      {TTI::SK_Splice, MVT::v8f16, {3, 3, 3, 3}}, // psrldq + psrlldq + por
+      {TTI::SK_Splice, MVT::v16i8, {3, 3, 3, 3}}, // psrldq + psrlldq + por
+
+      {TTI::SK_PermuteSingleSrc, MVT::v2f64, {1, 1, 1, 1}}, // shufpd
+      {TTI::SK_PermuteSingleSrc, MVT::v2i64, {1, 1, 1, 1}}, // pshufd
+      {TTI::SK_PermuteSingleSrc, MVT::v4i32, {1, 1, 1, 1}}, // pshufd
+      {TTI::SK_PermuteSingleSrc, MVT::v8i16, {3, 5, 5, 5}}, // 2*pshuflw + 2*pshufhw
+                                                            // + pshufd/unpck
+      {TTI::SK_PermuteSingleSrc, MVT::v8f16, {3, 5, 5, 5}}, // 2*pshuflw + 2*pshufhw
+                                                            // + pshufd/unpck
+      {TTI::SK_PermuteSingleSrc, MVT::v16i8, {8, 10, 10, 10}}, // 2*pshuflw + 2*pshufhw
+                                                               // + 2*pshufd + 2*unpck + 2*packus
+
+      {TTI::SK_PermuteTwoSrc, MVT::v2f64, {1, 1, 1, 1}},     // shufpd
+      {TTI::SK_PermuteTwoSrc, MVT::v2i64, {1, 1, 1, 1}},     // shufpd
+      {TTI::SK_PermuteTwoSrc, MVT::v4i32, {2, 2, 2, 2}},     // 2*{unpck,movsd,pshufd}
+      {TTI::SK_PermuteTwoSrc, MVT::v8i16, {6, 8, 8, 8}},     // blend+permute
+      {TTI::SK_PermuteTwoSrc, MVT::v8f16, {6, 8, 8, 8}},     // blend+permute
+      {TTI::SK_PermuteTwoSrc, MVT::v16i8, {11, 13, 13, 13}}, // blend+permute
   };
 
   static const CostTblEntry SSE3BroadcastLoadTbl[] = {
@@ -2221,16 +2236,17 @@ InstructionCost X86TTIImpl::getShuffleCost(
       }
 
     if (const auto *Entry = CostTableLookup(SSE2ShuffleTbl, Kind, LT.second))
-      return LT.first * Entry->Cost;
+      if (auto KindCost = Entry->Cost[CostKind])
+        return LT.first * *KindCost;
   }
 
-  static const CostTblEntry SSE1ShuffleTbl[] = {
-    { TTI::SK_Broadcast,        MVT::v4f32, 1 }, // shufps
-    { TTI::SK_Reverse,          MVT::v4f32, 1 }, // shufps
-    { TTI::SK_Select,           MVT::v4f32, 2 }, // 2*shufps
-    { TTI::SK_Splice,           MVT::v4f32, 2 }, // 2*shufps
-    { TTI::SK_PermuteSingleSrc, MVT::v4f32, 1 }, // shufps
-    { TTI::SK_PermuteTwoSrc,    MVT::v4f32, 2 }, // 2*shufps
+  static const CostKindTblEntry SSE1ShuffleTbl[] = {
+    { TTI::SK_Broadcast,        MVT::v4f32, {1,1,1,1} }, // shufps
+    { TTI::SK_Reverse,          MVT::v4f32, {1,1,1,1} }, // shufps
+    { TTI::SK_Select,           MVT::v4f32, {2,2,2,2} }, // 2*shufps
+    { TTI::SK_Splice,           MVT::v4f32, {2,2,2,2} }, // 2*shufps
+    { TTI::SK_PermuteSingleSrc, MVT::v4f32, {1,1,1,1} }, // shufps
+    { TTI::SK_PermuteTwoSrc,    MVT::v4f32, {2,2,2,2} }, // 2*shufps
   };
 
   if (ST->hasSSE1()) {
@@ -2243,7 +2259,8 @@ InstructionCost X86TTIImpl::getShuffleCost(
         return 1;
     }
     if (const auto *Entry = CostTableLookup(SSE1ShuffleTbl, Kind, LT.second))
-      return LT.first * Entry->Cost;
+      if (auto KindCost = Entry->Cost[CostKind])
+        return LT.first * *KindCost;
   }
 
   return BaseT::getShuffleCost(Kind, BaseTp, Mask, CostKind, Index, SubTp);
@@ -4329,9 +4346,15 @@ X86TTIImpl::getIntrinsicInstrCost(const IntrinsicCostAttributes &ICA,
     { ISD::ABS,        MVT::i64,     {  1,  2,  3,  3 } }, // SUB+CMOV
     { ISD::BITREVERSE, MVT::i64,     { 10, 12, 20, 22 } },
     { ISD::BSWAP,      MVT::i64,     {  1,  2,  1,  2 } },
-    { ISD::CTLZ,       MVT::i64,     {  2,  2,  4,  5 } }, // BSR+XOR or BSR+XOR+CMOV
+    { ISD::CTLZ,       MVT::i64,     {  1,  2,  3,  3 } }, // MOV+BSR+XOR
+    { ISD::CTLZ,       MVT::i32,     {  1,  2,  3,  3 } }, // MOV+BSR+XOR
+    { ISD::CTLZ,       MVT::i16,     {  2,  2,  3,  3 } }, // MOV+BSR+XOR
+    { ISD::CTLZ,       MVT::i8,      {  2,  2,  4,  3 } }, // MOV+BSR+XOR
     { ISD::CTLZ_ZERO_UNDEF, MVT::i64,{  1,  2,  2,  2 } }, // BSR+XOR
-    { ISD::CTTZ,       MVT::i64,     {  2,  2,  3,  4 } }, // TEST+BSF+CMOV/BRANCH
+    { ISD::CTTZ,       MVT::i64,     {  1,  2,  2,  2 } }, // MOV+BSF
+    { ISD::CTTZ,       MVT::i32,     {  1,  2,  2,  2 } }, // MOV+BSF
+    { ISD::CTTZ,       MVT::i16,     {  2,  2,  2,  2 } }, // MOV+BSF
+    { ISD::CTTZ,       MVT::i8,      {  2,  2,  2,  2 } }, // MOV+BSF
     { ISD::CTTZ_ZERO_UNDEF, MVT::i64,{  1,  2,  1,  2 } }, // BSF
     { ISD::CTPOP,      MVT::i64,     { 10,  6, 19, 19 } },
     { ISD::ROTL,       MVT::i64,     {  2,  3,  1,  3 } },
diff --git a/llvm/lib/Target/Xtensa/AsmParser/XtensaAsmParser.cpp b/llvm/lib/Target/Xtensa/AsmParser/XtensaAsmParser.cpp
index 731f9535ca251..0c9258ecd65ec 100644
--- a/llvm/lib/Target/Xtensa/AsmParser/XtensaAsmParser.cpp
+++ b/llvm/lib/Target/Xtensa/AsmParser/XtensaAsmParser.cpp
@@ -73,6 +73,7 @@ class XtensaAsmParser : public MCTargetAsmParser {
                                SMLoc &EndLoc) override {
     return ParseStatus::NoMatch;
   }
+
   ParseStatus parsePCRelTarget(OperandVector &Operands);
   bool parseLiteralDirective(SMLoc L);
 
@@ -89,6 +90,10 @@ class XtensaAsmParser : public MCTargetAsmParser {
       : MCTargetAsmParser(Options, STI, MII) {
     setAvailableFeatures(ComputeAvailableFeatures(STI.getFeatureBits()));
   }
+
+  bool hasWindowed() const {
+    return getSTI().getFeatureBits()[Xtensa::FeatureWindowed];
+  };
 };
 
 // Return true if Expr is in the range [MinValue, MaxValue].
@@ -181,6 +186,11 @@ struct XtensaOperand : public MCParsedAsmOperand {
            ((cast<MCConstantExpr>(getImm())->getValue() & 0x3) == 0);
   }
 
+  bool isentry_imm12() const {
+    return isImm(0, 32760) &&
+           ((cast<MCConstantExpr>(getImm())->getValue() % 8) == 0);
+  }
+
   bool isUimm4() const { return isImm(0, 15); }
 
   bool isUimm5() const { return isImm(0, 31); }
@@ -198,6 +208,11 @@ struct XtensaOperand : public MCParsedAsmOperand {
 
   bool isImm32n_95() const { return isImm(-32, 95); }
 
+  bool isImm64n_4n() const {
+    return isImm(-64, -4) &&
+           ((cast<MCConstantExpr>(getImm())->getValue() & 0x3) == 0);
+  }
+
   bool isB4const() const {
     if (Kind != Immediate)
       return false;
@@ -491,6 +506,12 @@ bool XtensaAsmParser::matchAndEmitInstruction(SMLoc IDLoc, unsigned &Opcode,
   case Match_InvalidImm32n_95:
     return Error(RefineErrorLoc(IDLoc, Operands, ErrorInfo),
                  "expected immediate in range [-32, 95]");
+  case Match_InvalidImm64n_4n:
+    return Error(RefineErrorLoc(IDLoc, Operands, ErrorInfo),
+                 "expected immediate in range [-64, -4]");
+  case Match_InvalidImm8n_7:
+    return Error(RefineErrorLoc(IDLoc, Operands, ErrorInfo),
+                 "expected immediate in range [-8, 7]");
   case Match_InvalidShimm1_31:
     return Error(RefineErrorLoc(IDLoc, Operands, ErrorInfo),
                  "expected immediate in range [1, 31]");
@@ -515,6 +536,10 @@ bool XtensaAsmParser::matchAndEmitInstruction(SMLoc IDLoc, unsigned &Opcode,
     return Error(RefineErrorLoc(IDLoc, Operands, ErrorInfo),
                  "expected immediate in range [0, 60], first 2 bits "
                  "should be zero");
+  case Match_Invalidentry_imm12:
+    return Error(RefineErrorLoc(IDLoc, Operands, ErrorInfo),
+                 "expected immediate in range [0, 32760], first 3 bits "
+                 "should be zero");
   }
 
   report_fatal_error("Unknown match type detected!");
@@ -601,6 +626,10 @@ ParseStatus XtensaAsmParser::parseRegister(OperandVector &Operands,
       getLexer().UnLex(Buf[0]);
     return ParseStatus::NoMatch;
   }
+
+  if (!Xtensa::checkRegister(RegNo, getSTI().getFeatureBits()))
+    return ParseStatus::NoMatch;
+
   if (HadParens)
     Operands.push_back(XtensaOperand::createToken("(", FirstS));
   SMLoc S = getLoc();
@@ -702,7 +731,7 @@ bool XtensaAsmParser::ParseInstructionWithSR(ParseInstructionInfo &Info,
     if (RegNo == 0)
       RegNo = MatchRegisterAltName(RegName);
 
-    if (RegNo == 0)
+    if (!Xtensa::checkRegister(RegNo, getSTI().getFeatureBits()))
       return Error(NameLoc, "invalid register name");
 
     // Parse operand
diff --git a/llvm/lib/Target/Xtensa/CMakeLists.txt b/llvm/lib/Target/Xtensa/CMakeLists.txt
index 726efadc87c0b..4fc1ba6dfa650 100644
--- a/llvm/lib/Target/Xtensa/CMakeLists.txt
+++ b/llvm/lib/Target/Xtensa/CMakeLists.txt
@@ -24,7 +24,6 @@ add_llvm_target(XtensaCodeGen
   XtensaRegisterInfo.cpp
   XtensaSubtarget.cpp
   XtensaTargetMachine.cpp
-  XtensaUtils.cpp
 
   LINK_COMPONENTS
   AsmPrinter
diff --git a/llvm/lib/Target/Xtensa/Disassembler/XtensaDisassembler.cpp b/llvm/lib/Target/Xtensa/Disassembler/XtensaDisassembler.cpp
index c11c4b7038bdb..7ad8a87ed599a 100644
--- a/llvm/lib/Target/Xtensa/Disassembler/XtensaDisassembler.cpp
+++ b/llvm/lib/Target/Xtensa/Disassembler/XtensaDisassembler.cpp
@@ -57,7 +57,7 @@ extern "C" LLVM_EXTERNAL_VISIBILITY void LLVMInitializeXtensaDisassembler() {
                                          createXtensaDisassembler);
 }
 
-static const unsigned ARDecoderTable[] = {
+const unsigned ARDecoderTable[] = {
     Xtensa::A0,  Xtensa::SP,  Xtensa::A2,  Xtensa::A3, Xtensa::A4,  Xtensa::A5,
     Xtensa::A6,  Xtensa::A7,  Xtensa::A8,  Xtensa::A9, Xtensa::A10, Xtensa::A11,
     Xtensa::A12, Xtensa::A13, Xtensa::A14, Xtensa::A15};
@@ -73,17 +73,23 @@ static DecodeStatus DecodeARRegisterClass(MCInst &Inst, uint64_t RegNo,
   return MCDisassembler::Success;
 }
 
-static const unsigned SRDecoderTable[] = {Xtensa::SAR, 3};
+const unsigned SRDecoderTable[] = {
+    Xtensa::SAR, 3, Xtensa::WINDOWBASE, 72, Xtensa::WINDOWSTART, 73};
 
 static DecodeStatus DecodeSRRegisterClass(MCInst &Inst, uint64_t RegNo,
                                           uint64_t Address,
-                                          const void *Decoder) {
+                                          const MCDisassembler *Decoder) {
   if (RegNo > 255)
     return MCDisassembler::Fail;
 
   for (unsigned i = 0; i < std::size(SRDecoderTable); i += 2) {
     if (SRDecoderTable[i + 1] == RegNo) {
       unsigned Reg = SRDecoderTable[i];
+
+      if (!Xtensa::checkRegister(Reg,
+                                 Decoder->getSubtargetInfo().getFeatureBits()))
+        return MCDisassembler::Fail;
+
       Inst.addOperand(MCOperand::createReg(Reg));
       return MCDisassembler::Success;
     }
@@ -210,6 +216,29 @@ static DecodeStatus decodeImm32n_95Operand(MCInst &Inst, uint64_t Imm,
   return MCDisassembler::Success;
 }
 
+static DecodeStatus decodeImm8n_7Operand(MCInst &Inst, uint64_t Imm,
+                                         int64_t Address, const void *Decoder) {
+  assert(isUInt<4>(Imm) && "Invalid immediate");
+  Inst.addOperand(MCOperand::createImm(Imm > 7 ? Imm - 16 : Imm));
+  return MCDisassembler::Success;
+}
+
+static DecodeStatus decodeImm64n_4nOperand(MCInst &Inst, uint64_t Imm,
+                                           int64_t Address,
+                                           const void *Decoder) {
+  assert(isUInt<6>(Imm) && ((Imm & 0x3) == 0) && "Invalid immediate");
+  Inst.addOperand(MCOperand::createImm((~0x3f) | (Imm)));
+  return MCDisassembler::Success;
+}
+
+static DecodeStatus decodeEntry_Imm12OpValue(MCInst &Inst, uint64_t Imm,
+                                             int64_t Address,
+                                             const void *Decoder) {
+  assert(isUInt<15>(Imm) && ((Imm & 0x7) == 0) && "Invalid immediate");
+  Inst.addOperand(MCOperand::createImm(Imm));
+  return MCDisassembler::Success;
+}
+
 static DecodeStatus decodeShimm1_31Operand(MCInst &Inst, uint64_t Imm,
                                            int64_t Address,
                                            const void *Decoder) {
diff --git a/llvm/lib/Target/Xtensa/MCTargetDesc/XtensaInstPrinter.cpp b/llvm/lib/Target/Xtensa/MCTargetDesc/XtensaInstPrinter.cpp
index df8a0854f06f4..868c7f6c0b9c3 100644
--- a/llvm/lib/Target/Xtensa/MCTargetDesc/XtensaInstPrinter.cpp
+++ b/llvm/lib/Target/Xtensa/MCTargetDesc/XtensaInstPrinter.cpp
@@ -264,6 +264,28 @@ void XtensaInstPrinter::printImm32n_95_AsmOperand(const MCInst *MI, int OpNum,
     printOperand(MI, OpNum, O);
 }
 
+void XtensaInstPrinter::printImm8n_7_AsmOperand(const MCInst *MI, int OpNum,
+                                                raw_ostream &O) {
+  if (MI->getOperand(OpNum).isImm()) {
+    int64_t Value = MI->getOperand(OpNum).getImm();
+    assert((Value >= -8 && Value <= 7) &&
+           "Invalid argument, value must be in ranges <-8,7>");
+    O << Value;
+  } else
+    printOperand(MI, OpNum, O);
+}
+
+void XtensaInstPrinter::printImm64n_4n_AsmOperand(const MCInst *MI, int OpNum,
+                                                  raw_ostream &O) {
+  if (MI->getOperand(OpNum).isImm()) {
+    int64_t Value = MI->getOperand(OpNum).getImm();
+    assert((Value >= -64 && Value <= -4) & ((Value & 0x3) == 0) &&
+           "Invalid argument, value must be in ranges <-64,-4>");
+    O << Value;
+  } else
+    printOperand(MI, OpNum, O);
+}
+
 void XtensaInstPrinter::printOffset8m8_AsmOperand(const MCInst *MI, int OpNum,
                                                   raw_ostream &O) {
   if (MI->getOperand(OpNum).isImm()) {
@@ -309,6 +331,18 @@ void XtensaInstPrinter::printOffset4m32_AsmOperand(const MCInst *MI, int OpNum,
     printOperand(MI, OpNum, O);
 }
 
+void XtensaInstPrinter::printEntry_Imm12_AsmOperand(const MCInst *MI, int OpNum,
+                                                    raw_ostream &O) {
+  if (MI->getOperand(OpNum).isImm()) {
+    int64_t Value = MI->getOperand(OpNum).getImm();
+    assert((Value >= 0 && Value <= 32760) &&
+           "Invalid argument, value must be multiples of eight in range "
+           "<0,32760>");
+    O << Value;
+  } else
+    printOperand(MI, OpNum, O);
+}
+
 void XtensaInstPrinter::printB4const_AsmOperand(const MCInst *MI, int OpNum,
                                                 raw_ostream &O) {
   if (MI->getOperand(OpNum).isImm()) {
diff --git a/llvm/lib/Target/Xtensa/MCTargetDesc/XtensaInstPrinter.h b/llvm/lib/Target/Xtensa/MCTargetDesc/XtensaInstPrinter.h
index e5bc67869e103..630b4dd60108f 100644
--- a/llvm/lib/Target/Xtensa/MCTargetDesc/XtensaInstPrinter.h
+++ b/llvm/lib/Target/Xtensa/MCTargetDesc/XtensaInstPrinter.h
@@ -60,10 +60,13 @@ class XtensaInstPrinter : public MCInstPrinter {
   void printImm1_16_AsmOperand(const MCInst *MI, int OpNum, raw_ostream &O);
   void printImm1n_15_AsmOperand(const MCInst *MI, int OpNum, raw_ostream &O);
   void printImm32n_95_AsmOperand(const MCInst *MI, int OpNum, raw_ostream &O);
+  void printImm8n_7_AsmOperand(const MCInst *MI, int OpNum, raw_ostream &O);
+  void printImm64n_4n_AsmOperand(const MCInst *MI, int OpNum, raw_ostream &O);
   void printOffset8m8_AsmOperand(const MCInst *MI, int OpNum, raw_ostream &O);
   void printOffset8m16_AsmOperand(const MCInst *MI, int OpNum, raw_ostream &O);
   void printOffset8m32_AsmOperand(const MCInst *MI, int OpNum, raw_ostream &O);
   void printOffset4m32_AsmOperand(const MCInst *MI, int OpNum, raw_ostream &O);
+  void printEntry_Imm12_AsmOperand(const MCInst *MI, int OpNum, raw_ostream &O);
   void printB4const_AsmOperand(const MCInst *MI, int OpNum, raw_ostream &O);
   void printB4constu_AsmOperand(const MCInst *MI, int OpNum, raw_ostream &O);
 };
diff --git a/llvm/lib/Target/Xtensa/MCTargetDesc/XtensaMCCodeEmitter.cpp b/llvm/lib/Target/Xtensa/MCTargetDesc/XtensaMCCodeEmitter.cpp
index 51d4b8a9cc5fc..e6cdd3d0020fc 100644
--- a/llvm/lib/Target/Xtensa/MCTargetDesc/XtensaMCCodeEmitter.cpp
+++ b/llvm/lib/Target/Xtensa/MCTargetDesc/XtensaMCCodeEmitter.cpp
@@ -111,6 +111,18 @@ class XtensaMCCodeEmitter : public MCCodeEmitter {
                                SmallVectorImpl<MCFixup> &Fixups,
                                const MCSubtargetInfo &STI) const;
 
+  uint32_t getImm8n_7OpValue(const MCInst &MI, unsigned OpNo,
+                             SmallVectorImpl<MCFixup> &Fixups,
+                             const MCSubtargetInfo &STI) const;
+
+  uint32_t getImm64n_4nOpValue(const MCInst &MI, unsigned OpNo,
+                               SmallVectorImpl<MCFixup> &Fixups,
+                               const MCSubtargetInfo &STI) const;
+
+  uint32_t getEntry_Imm12OpValue(const MCInst &MI, unsigned OpNo,
+                                 SmallVectorImpl<MCFixup> &Fixups,
+                                 const MCSubtargetInfo &STI) const;
+
   uint32_t getShimm1_31OpValue(const MCInst &MI, unsigned OpNo,
                                SmallVectorImpl<MCFixup> &Fixups,
                                const MCSubtargetInfo &STI) const;
@@ -405,6 +417,46 @@ XtensaMCCodeEmitter::getImm32n_95OpValue(const MCInst &MI, unsigned OpNo,
   return Res;
 }
 
+uint32_t
+XtensaMCCodeEmitter::getImm8n_7OpValue(const MCInst &MI, unsigned OpNo,
+                                       SmallVectorImpl<MCFixup> &Fixups,
+                                       const MCSubtargetInfo &STI) const {
+  const MCOperand &MO = MI.getOperand(OpNo);
+  int32_t Res = static_cast<int32_t>(MO.getImm());
+
+  assert(((Res >= -8) && (Res <= 7)) && "Unexpected operand value!");
+
+  if (Res < 0)
+    return Res + 16;
+
+  return Res;
+}
+
+uint32_t
+XtensaMCCodeEmitter::getImm64n_4nOpValue(const MCInst &MI, unsigned OpNo,
+                                         SmallVectorImpl<MCFixup> &Fixups,
+                                         const MCSubtargetInfo &STI) const {
+  const MCOperand &MO = MI.getOperand(OpNo);
+  int32_t Res = static_cast<int32_t>(MO.getImm());
+
+  assert(((Res >= -64) && (Res <= -4) && ((Res & 0x3) == 0)) &&
+         "Unexpected operand value!");
+
+  return Res & 0x3f;
+}
+
+uint32_t
+XtensaMCCodeEmitter::getEntry_Imm12OpValue(const MCInst &MI, unsigned OpNo,
+                                           SmallVectorImpl<MCFixup> &Fixups,
+                                           const MCSubtargetInfo &STI) const {
+  const MCOperand &MO = MI.getOperand(OpNo);
+  uint32_t res = static_cast<uint32_t>(MO.getImm());
+
+  assert(((res & 0x7) == 0) && "Unexpected operand value!");
+
+  return res;
+}
+
 uint32_t
 XtensaMCCodeEmitter::getB4constOpValue(const MCInst &MI, unsigned OpNo,
                                        SmallVectorImpl<MCFixup> &Fixups,
diff --git a/llvm/lib/Target/Xtensa/MCTargetDesc/XtensaMCTargetDesc.cpp b/llvm/lib/Target/Xtensa/MCTargetDesc/XtensaMCTargetDesc.cpp
index 2653c293dc0c4..37dee072e5b3d 100644
--- a/llvm/lib/Target/Xtensa/MCTargetDesc/XtensaMCTargetDesc.cpp
+++ b/llvm/lib/Target/Xtensa/MCTargetDesc/XtensaMCTargetDesc.cpp
@@ -32,6 +32,61 @@
 
 using namespace llvm;
 
+bool Xtensa::isValidAddrOffset(int Scale, int64_t OffsetVal) {
+  bool Valid = false;
+
+  switch (Scale) {
+  case 1:
+    Valid = (OffsetVal >= 0 && OffsetVal <= 255);
+    break;
+  case 2:
+    Valid = (OffsetVal >= 0 && OffsetVal <= 510) && ((OffsetVal & 0x1) == 0);
+    break;
+  case 4:
+    Valid = (OffsetVal >= 0 && OffsetVal <= 1020) && ((OffsetVal & 0x3) == 0);
+    break;
+  default:
+    break;
+  }
+  return Valid;
+}
+
+bool Xtensa::isValidAddrOffsetForOpcode(unsigned Opcode, int64_t Offset) {
+  int Scale = 0;
+
+  switch (Opcode) {
+  case Xtensa::L8UI:
+  case Xtensa::S8I:
+    Scale = 1;
+    break;
+  case Xtensa::L16SI:
+  case Xtensa::L16UI:
+  case Xtensa::S16I:
+    Scale = 2;
+    break;
+  case Xtensa::LEA_ADD:
+    return (Offset >= -128 && Offset <= 127);
+  default:
+    // assume that MI is 32-bit load/store operation
+    Scale = 4;
+    break;
+  }
+  return isValidAddrOffset(Scale, Offset);
+}
+
+// Verify Special Register
+bool Xtensa::checkRegister(MCRegister RegNo, const FeatureBitset &FeatureBits) {
+  switch (RegNo) {
+  case Xtensa::WINDOWBASE:
+  case Xtensa::WINDOWSTART:
+    return FeatureBits[Xtensa::FeatureWindowed];
+  case Xtensa::NoRegister:
+    return false;
+  }
+
+  return true;
+}
+
 static MCAsmInfo *createXtensaMCAsmInfo(const MCRegisterInfo &MRI,
                                         const Triple &TT,
                                         const MCTargetOptions &Options) {
diff --git a/llvm/lib/Target/Xtensa/MCTargetDesc/XtensaMCTargetDesc.h b/llvm/lib/Target/Xtensa/MCTargetDesc/XtensaMCTargetDesc.h
index 0e075be0df07f..649073b01f5c1 100644
--- a/llvm/lib/Target/Xtensa/MCTargetDesc/XtensaMCTargetDesc.h
+++ b/llvm/lib/Target/Xtensa/MCTargetDesc/XtensaMCTargetDesc.h
@@ -19,15 +19,18 @@
 
 namespace llvm {
 
+class FeatureBitset;
 class MCAsmBackend;
 class MCCodeEmitter;
 class MCContext;
 class MCInstrInfo;
 class MCObjectTargetWriter;
 class MCObjectWriter;
+class MCRegister;
 class MCRegisterInfo;
 class MCSubtargetInfo;
 class MCTargetOptions;
+class MachineInstr;
 class StringRef;
 class Target;
 class raw_ostream;
@@ -43,6 +46,18 @@ MCAsmBackend *createXtensaMCAsmBackend(const Target &T,
                                        const MCTargetOptions &Options);
 std::unique_ptr<MCObjectTargetWriter>
 createXtensaObjectWriter(uint8_t OSABI, bool IsLittleEndian);
+
+namespace Xtensa {
+// Check address offset for load/store instructions.
+// The offset should be multiple of scale.
+bool isValidAddrOffset(int Scale, int64_t OffsetVal);
+
+// Check address offset for load/store instructions.
+bool isValidAddrOffsetForOpcode(unsigned Opcode, int64_t Offset);
+
+// Verify if it's correct to use a special register.
+bool checkRegister(MCRegister RegNo, const FeatureBitset &FeatureBits);
+} // namespace Xtensa
 } // end namespace llvm
 
 // Defines symbolic names for Xtensa registers.
diff --git a/llvm/lib/Target/Xtensa/Xtensa.td b/llvm/lib/Target/Xtensa/Xtensa.td
index 460a15e808b3a..2c4bacbe8282b 100644
--- a/llvm/lib/Target/Xtensa/Xtensa.td
+++ b/llvm/lib/Target/Xtensa/Xtensa.td
@@ -17,10 +17,9 @@ include "llvm/Target/Target.td"
 //===----------------------------------------------------------------------===//
 // Subtarget Features.
 //===----------------------------------------------------------------------===//
-def FeatureDensity : SubtargetFeature<"density", "HasDensity", "true",
-                    "Enable Density instructions">;
-def HasDensity : Predicate<"Subtarget->hasDensity()">,
-                     AssemblerPredicate<(all_of FeatureDensity)>;
+
+include "XtensaFeatures.td"
+
 //===----------------------------------------------------------------------===//
 // Xtensa supported processors.
 //===----------------------------------------------------------------------===//
diff --git a/llvm/lib/Target/Xtensa/XtensaFeatures.td b/llvm/lib/Target/Xtensa/XtensaFeatures.td
new file mode 100644
index 0000000000000..6f24a674ae0ce
--- /dev/null
+++ b/llvm/lib/Target/Xtensa/XtensaFeatures.td
@@ -0,0 +1,14 @@
+//===----------------------------------------------------------------------===//
+// Xtensa subtarget features.
+//===----------------------------------------------------------------------===//
+
+// Xtensa ISA extensions (Xtensa Options).
+def FeatureDensity : SubtargetFeature<"density", "HasDensity", "true",
+                                      "Enable Density instructions">;
+def HasDensity : Predicate<"Subtarget->hasDensity()">,
+                 AssemblerPredicate<(all_of FeatureDensity)>;
+
+def FeatureWindowed : SubtargetFeature<"windowed", "HasWindowed", "true",
+                                       "Enable Xtensa Windowed Register option">;
+def HasWindowed : Predicate<"Subtarget->hasWindowed()">,
+                  AssemblerPredicate<(all_of FeatureWindowed)>;
diff --git a/llvm/lib/Target/Xtensa/XtensaISelDAGToDAG.cpp b/llvm/lib/Target/Xtensa/XtensaISelDAGToDAG.cpp
index ef14095d18efb..06cccd4831bfc 100644
--- a/llvm/lib/Target/Xtensa/XtensaISelDAGToDAG.cpp
+++ b/llvm/lib/Target/Xtensa/XtensaISelDAGToDAG.cpp
@@ -10,9 +10,9 @@
 //
 //===----------------------------------------------------------------------===//
 
+#include "MCTargetDesc/XtensaMCTargetDesc.h"
 #include "Xtensa.h"
 #include "XtensaTargetMachine.h"
-#include "XtensaUtils.h"
 #include "llvm/CodeGen/MachineFunction.h"
 #include "llvm/CodeGen/MachineRegisterInfo.h"
 #include "llvm/CodeGen/SelectionDAGISel.h"
@@ -75,7 +75,7 @@ class XtensaDAGToDAGISel : public SelectionDAGISel {
       ConstantSDNode *CN = dyn_cast<ConstantSDNode>(Addr.getOperand(1));
       int64_t OffsetVal = CN->getSExtValue();
 
-      Valid = isValidAddrOffset(Scale, OffsetVal);
+      Valid = Xtensa::isValidAddrOffset(Scale, OffsetVal);
 
       if (Valid) {
         // If the first operand is a FI, get the TargetFI Node.
diff --git a/llvm/lib/Target/Xtensa/XtensaInstrInfo.td b/llvm/lib/Target/Xtensa/XtensaInstrInfo.td
index 699d0d6cf8044..5ef795a0e5287 100644
--- a/llvm/lib/Target/Xtensa/XtensaInstrInfo.td
+++ b/llvm/lib/Target/Xtensa/XtensaInstrInfo.td
@@ -678,3 +678,104 @@ let isReturn = 1, isTerminator = 1,
     let t = 0;
   }
 }
+
+//===----------------------------------------------------------------------===//
+// Windowed instructions
+//===----------------------------------------------------------------------===//
+
+def ENTRY : BRI12_Inst<0x06, 0x3, 0x0, (outs), (ins AR:$s, entry_imm12:$imm),
+                     "entry\t$s, $imm", []>, Requires<[HasWindowed]> {
+  bits<15> imm;
+
+  let imm12{11-0} = imm{14-3};
+  let Defs = [SP];
+}
+
+let isCall = 1, Defs = [A0] in {
+  foreach i = {1,2,3} in {
+    defvar I = !mul(4, i);
+
+    def CALL#I# : CALL_Inst<0x05, (outs), (ins pcrel32call:$offset),
+                           "call"#I#"\t$offset", []>, Requires<[HasWindowed]> {
+      let n = i;
+    }
+
+    def CALLX#I# : CALLX_Inst<0x00, 0x00, 0x00, (outs), (ins AR:$s),
+                             "callx"#I#"\t$s", []>, Requires<[HasWindowed]> {
+      let m = 0x3;
+      let n = i;
+      let r = 0;
+    }
+  }
+}
+
+def MOVSP : RRR_Inst<0x00, 0x00, 0x00, (outs AR:$t), (ins AR:$s),
+                    "movsp\t$t, $s", []>, Requires<[HasWindowed]> {
+  let r = 0x01;
+}
+
+let isReturn = 1, isTerminator = 1,
+    isBarrier = 1, Uses = [A0] in {
+  def RETW_N : RRRN_Inst<0x0D, (outs), (ins),
+                        "retw.n", []>, Requires<[HasWindowed, HasDensity]> {
+    let r = 0x0F;
+    let s = 0;
+    let t = 1;
+  }
+
+  def RETW : CALLX_Inst<0x00, 0x00, 0x00, (outs), (ins),
+                       "retw", []>, Requires<[HasWindowed]> {
+    let m = 0x2;
+    let n = 0x1;
+    let s = 0;
+    let r = 0;
+  }
+}
+
+def : InstAlias<"_retw", (RETW)>;
+def : InstAlias<"_retw.n", (RETW_N)>;
+
+def S32E : RRI4_Inst<0x00, 0x09, (outs), (ins AR:$t, AR:$s, imm64n_4n:$imm),
+                    "s32e\t$t, $s, $imm", []>, Requires<[HasWindowed]> {
+  bits<6> imm;
+
+  let r = imm{5-2};
+  let imm4 = 0x4;
+  let mayStore = 1;
+}
+
+def L32E : RRI4_Inst<0x00, 0x09, (outs), (ins AR:$t, AR:$s, imm64n_4n:$imm),
+                    "l32e\t$t, $s, $imm", []>, Requires<[HasWindowed]> {
+  bits<6> imm;
+
+  let r = imm{5-2};
+  let imm4 = 0x0;
+  let mayLoad = 1;
+}
+
+def RFWU : RRR_Inst<0x00, 0x00, 0x00, (outs), (ins),
+                   "rfwu", []>, Requires<[HasWindowed]> {
+  bits<4> imm;
+
+  let r = 0x3;
+  let s = 0x5;
+  let t = 0x0;
+}
+
+def RFWO : RRR_Inst<0x00, 0x00, 0x00, (outs), (ins),
+                   "rfwo", []>, Requires<[HasWindowed]> {
+  bits<4> imm;
+
+  let r = 0x3;
+  let s = 0x4;
+  let t = 0x0;
+}
+
+def ROTW : RRR_Inst<0x00, 0x00, 0x04, (outs), (ins imm8n_7:$imm),
+                   "rotw\t$imm", []>, Requires<[HasWindowed]> {
+  bits<4> imm;
+
+  let r = 0x8;
+  let s = 0x0;
+  let t = imm{3-0};
+}
diff --git a/llvm/lib/Target/Xtensa/XtensaOperands.td b/llvm/lib/Target/Xtensa/XtensaOperands.td
index aa72fa0a56a6f..402e05a5c3dd1 100644
--- a/llvm/lib/Target/Xtensa/XtensaOperands.td
+++ b/llvm/lib/Target/Xtensa/XtensaOperands.td
@@ -37,6 +37,20 @@ def imm8_sh8 : Immediate<i32, [{ return Imm >= -32768 && Imm <= 32512 && ((Imm &
   let DecoderMethod = "decodeImm8_sh8Operand";
 }
 
+// imm8n_7 predicate - Immediate in the range [-8,7]
+def Imm8n_7_AsmOperand: ImmAsmOperand<"Imm8n_7">;
+def imm8n_7: Immediate<i32, [{ return Imm >= -8 && Imm <= 7; }], "Imm8n_7_AsmOperand"> {
+  let EncoderMethod = "getImm8n_7OpValue";
+  let DecoderMethod = "decodeImm8n_7Operand";
+}
+
+// imm64n_4n predicate - Immediate in the range [-64,-4]
+def Imm64n_4n_AsmOperand: ImmAsmOperand<"Imm64n_4n">;
+def imm64n_4n: Immediate<i32, [{ return Imm >= -64 && Imm <= -4; }], "Imm64n_4n_AsmOperand"> {
+  let EncoderMethod = "getImm64n_4nOpValue";
+  let DecoderMethod = "decodeImm64n_4nOperand";
+}
+
 // imm12 predicate - Immediate in the range [-2048,2047]
 def Imm12_AsmOperand : ImmAsmOperand<"Imm12">;
 def imm12 : Immediate<i32, [{ return Imm >= -2048 && Imm <= 2047; }], "Imm12_AsmOperand"> {
@@ -117,6 +131,13 @@ def offset4m32 : Immediate<i32,
     [{ return Imm >= 0 && Imm <= 60 && (Imm & 0x3 == 0); }],
     "Offset4m32_AsmOperand">;
 
+// entry_imm12 predicate - Immediate in the range [0,32760], ENTRY parameter
+def Entry_Imm12_AsmOperand: ImmAsmOperand<"entry_imm12">;
+def entry_imm12: Immediate<i32, [{ return Imm >= 0 && Imm <= 32760 && (Imm % 8 == 0); }], "Entry_Imm12_AsmOperand"> {
+  let EncoderMethod = "getEntry_Imm12OpValue";
+  let DecoderMethod = "decodeEntry_Imm12OpValue";
+}
+
 // b4const predicate - Branch Immediate 4-bit signed operand
 def B4const_AsmOperand: ImmAsmOperand<"B4const">;
 def b4const: Immediate<i32,
diff --git a/llvm/lib/Target/Xtensa/XtensaRegisterInfo.cpp b/llvm/lib/Target/Xtensa/XtensaRegisterInfo.cpp
index bced2d4ad0095..4a8bafc540df0 100644
--- a/llvm/lib/Target/Xtensa/XtensaRegisterInfo.cpp
+++ b/llvm/lib/Target/Xtensa/XtensaRegisterInfo.cpp
@@ -11,9 +11,9 @@
 //===----------------------------------------------------------------------===//
 
 #include "XtensaRegisterInfo.h"
+#include "MCTargetDesc/XtensaMCTargetDesc.h"
 #include "XtensaInstrInfo.h"
 #include "XtensaSubtarget.h"
-#include "XtensaUtils.h"
 #include "llvm/CodeGen/MachineFrameInfo.h"
 #include "llvm/CodeGen/MachineFunction.h"
 #include "llvm/CodeGen/MachineInstrBuilder.h"
@@ -99,7 +99,7 @@ bool XtensaRegisterInfo::eliminateFrameIndex(MachineBasicBlock::iterator II,
   int64_t Offset =
       SPOffset + (int64_t)StackSize + MI.getOperand(FIOperandNum + 1).getImm();
 
-  bool Valid = isValidAddrOffset(MI, Offset);
+  bool Valid = Xtensa::isValidAddrOffsetForOpcode(MI.getOpcode(), Offset);
 
   // If MI is not a debug value, make sure Offset fits in the 16-bit immediate
   // field.
diff --git a/llvm/lib/Target/Xtensa/XtensaRegisterInfo.td b/llvm/lib/Target/Xtensa/XtensaRegisterInfo.td
index 5c07386b060cd..09087edc86712 100644
--- a/llvm/lib/Target/Xtensa/XtensaRegisterInfo.td
+++ b/llvm/lib/Target/Xtensa/XtensaRegisterInfo.td
@@ -75,4 +75,8 @@ class SRReg<bits<8> num, string n, list<string> alt = []> : XtensaReg<n> {
 // Shift Amount Register
 def SAR : SRReg<3, "sar", ["SAR","3"]>;
 
-def SR :  RegisterClass<"Xtensa", [i32], 32, (add SAR)>;
+// Windowed Register Option registers
+def WINDOWBASE : SRReg<72, "windowbase", ["WINDOWBASE", "72"]>;
+def WINDOWSTART : SRReg<73, "windowstart", ["WINDOWSTART", "73"]>;
+
+def SR :  RegisterClass<"Xtensa", [i32], 32, (add SAR, WINDOWBASE, WINDOWSTART)>;
diff --git a/llvm/lib/Target/Xtensa/XtensaSubtarget.h b/llvm/lib/Target/Xtensa/XtensaSubtarget.h
index 948dcbc5278ea..dddc0f7ef605d 100644
--- a/llvm/lib/Target/Xtensa/XtensaSubtarget.h
+++ b/llvm/lib/Target/Xtensa/XtensaSubtarget.h
@@ -36,9 +36,12 @@ class XtensaSubtarget : public XtensaGenSubtargetInfo {
   SelectionDAGTargetInfo TSInfo;
   XtensaFrameLowering FrameLowering;
 
-  // Enabled Xtensa Density extension
+  // Enabled Xtensa Density Option
   bool HasDensity;
 
+  // Enabled Xtensa Windowed Register Option
+  bool HasWindowed;
+
   XtensaSubtarget &initializeSubtargetDependencies(StringRef CPU, StringRef FS);
 
 public:
@@ -64,6 +67,8 @@ class XtensaSubtarget : public XtensaGenSubtargetInfo {
 
   bool hasDensity() const { return HasDensity; }
 
+  bool hasWindowed() const { return HasWindowed; }
+
   // Automatically generated by tblgen.
   void ParseSubtargetFeatures(StringRef CPU, StringRef TuneCPU, StringRef FS);
 };
diff --git a/llvm/lib/Target/Xtensa/XtensaUtils.cpp b/llvm/lib/Target/Xtensa/XtensaUtils.cpp
deleted file mode 100644
index 98e424f6ea440..0000000000000
--- a/llvm/lib/Target/Xtensa/XtensaUtils.cpp
+++ /dev/null
@@ -1,59 +0,0 @@
-//===--- XtensaUtils.cpp ---- Xtensa Utility Functions ----------*- C++ -*-===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-//
-// This file contains miscellaneous utility functions.
-//
-//===----------------------------------------------------------------------===//
-
-#include "XtensaUtils.h"
-
-namespace llvm {
-
-bool isValidAddrOffset(int Scale, int64_t OffsetVal) {
-  bool Valid = false;
-
-  switch (Scale) {
-  case 1:
-    Valid = (OffsetVal >= 0 && OffsetVal <= 255);
-    break;
-  case 2:
-    Valid = (OffsetVal >= 0 && OffsetVal <= 510) && ((OffsetVal & 0x1) == 0);
-    break;
-  case 4:
-    Valid = (OffsetVal >= 0 && OffsetVal <= 1020) && ((OffsetVal & 0x3) == 0);
-    break;
-  default:
-    break;
-  }
-  return Valid;
-}
-
-bool isValidAddrOffset(MachineInstr &MI, int64_t Offset) {
-  int Scale = 0;
-
-  switch (MI.getOpcode()) {
-  case Xtensa::L8UI:
-  case Xtensa::S8I:
-    Scale = 1;
-    break;
-  case Xtensa::L16SI:
-  case Xtensa::L16UI:
-  case Xtensa::S16I:
-    Scale = 2;
-    break;
-  case Xtensa::LEA_ADD:
-    return (Offset >= -128 && Offset <= 127);
-  default:
-    // assume that MI is 32-bit load/store operation
-    Scale = 4;
-    break;
-  }
-  return isValidAddrOffset(Scale, Offset);
-}
-
-} // namespace llvm
diff --git a/llvm/lib/Target/Xtensa/XtensaUtils.h b/llvm/lib/Target/Xtensa/XtensaUtils.h
deleted file mode 100644
index 2b0ac37a6971a..0000000000000
--- a/llvm/lib/Target/Xtensa/XtensaUtils.h
+++ /dev/null
@@ -1,27 +0,0 @@
-//===--- XtensaUtils.h ---- Xtensa Utility Functions ------------*- C++ -*-===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-//
-// This file contains miscellaneous utility functions.
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LLVM_LIB_TARGET_XTENSA_XTENSAUTILS_H
-#define LLVM_LIB_TARGET_XTENSA_XTENSAUTILS_H
-
-#include "XtensaInstrInfo.h"
-#include "llvm/CodeGen/MachineInstr.h"
-
-namespace llvm {
-// Check address offset for load/store instructions.
-// The offset should be multiple of scale.
-bool isValidAddrOffset(int Scale, int64_t OffsetVal);
-
-// Check address offset for load/store instructions.
-bool isValidAddrOffset(MachineInstr &MI, int64_t Offset);
-} // namespace llvm
-#endif // LLVM_LIB_TARGET_XTENSA_XTENSAUTILS_H
diff --git a/llvm/lib/TargetParser/ARMTargetParser.cpp b/llvm/lib/TargetParser/ARMTargetParser.cpp
index 9bcfa6ca62c97..8f9753775c204 100644
--- a/llvm/lib/TargetParser/ARMTargetParser.cpp
+++ b/llvm/lib/TargetParser/ARMTargetParser.cpp
@@ -403,13 +403,12 @@ static ARM::FPUKind findSinglePrecisionFPU(ARM::FPUKind InputFPUKind) {
   if (!ARM::isDoublePrecision(InputFPU.Restriction))
     return InputFPUKind;
 
-  // Otherwise, look for an FPU entry with all the same fields, except
-  // that it does not support double precision.
+  // Otherwise, look for an FPU entry that has the same FPUVer
+  // and is not Double Precision. We want to allow for changing of
+  // NEON Support and Restrictions so CPU's such as Cortex-R52 can
+  // select between SP Only and Full DP modes.
   for (const ARM::FPUName &CandidateFPU : ARM::FPUNames) {
     if (CandidateFPU.FPUVer == InputFPU.FPUVer &&
-        CandidateFPU.NeonSupport == InputFPU.NeonSupport &&
-        ARM::has32Regs(CandidateFPU.Restriction) ==
-            ARM::has32Regs(InputFPU.Restriction) &&
         !ARM::isDoublePrecision(CandidateFPU.Restriction)) {
       return CandidateFPU.ID;
     }
diff --git a/llvm/lib/TargetParser/Host.cpp b/llvm/lib/TargetParser/Host.cpp
index fa57ae183bb84..d6a16143fe9e9 100644
--- a/llvm/lib/TargetParser/Host.cpp
+++ b/llvm/lib/TargetParser/Host.cpp
@@ -493,6 +493,7 @@ StringRef sys::detail::getHostCPUNameForRISCV(StringRef ProcCpuinfoContent) {
   }
 
   return StringSwitch<const char *>(UArch)
+      .Case("eswin,eic770x", "sifive-p550")
       .Case("sifive,u74-mc", "sifive-u74")
       .Case("sifive,bullet0", "sifive-u74")
       .Default("");
diff --git a/llvm/lib/TargetParser/Triple.cpp b/llvm/lib/TargetParser/Triple.cpp
index ed58e72089839..e9e6f130f757c 100644
--- a/llvm/lib/TargetParser/Triple.cpp
+++ b/llvm/lib/TargetParser/Triple.cpp
@@ -1711,6 +1711,26 @@ unsigned Triple::getArchPointerBitWidth(llvm::Triple::ArchType Arch) {
   llvm_unreachable("Invalid architecture value");
 }
 
+unsigned Triple::getTrampolineSize() const {
+  switch (getArch()) {
+  default:
+    break;
+  case Triple::ppc:
+  case Triple::ppcle:
+    if (isOSLinux())
+      return 40;
+    break;
+  case Triple::ppc64:
+  case Triple::ppc64le:
+    if (isOSLinux())
+      return 48;
+    break;
+  case Triple::aarch64:
+    return 36;
+  }
+  return 32;
+}
+
 bool Triple::isArch64Bit() const {
   return getArchPointerBitWidth(getArch()) == 64;
 }
diff --git a/llvm/lib/Transforms/Coroutines/CoroFrame.cpp b/llvm/lib/Transforms/Coroutines/CoroFrame.cpp
index 73d4fb9065831..4104e4e533e9d 100644
--- a/llvm/lib/Transforms/Coroutines/CoroFrame.cpp
+++ b/llvm/lib/Transforms/Coroutines/CoroFrame.cpp
@@ -752,13 +752,14 @@ static void buildFrameDebugInfo(Function &F, coro::Shape &Shape,
                                   dwarf::DW_ATE_unsigned_char)});
 
   for (auto *V : FrameData.getAllDefs()) {
-    if (!DIVarCache.contains(V))
+    auto It = DIVarCache.find(V);
+    if (It == DIVarCache.end())
       continue;
 
     auto Index = FrameData.getFieldIndex(V);
 
-    NameCache.insert({Index, DIVarCache[V]->getName()});
-    TyCache.insert({Index, DIVarCache[V]->getType()});
+    NameCache.insert({Index, It->second->getName()});
+    TyCache.insert({Index, It->second->getType()});
   }
 
   // Cache from index to (Align, Offset Pair)
@@ -1068,9 +1069,9 @@ static void insertSpills(const FrameDataInfo &FrameData, coro::Shape &Shape) {
 
     Type *ByValTy = nullptr;
     if (auto *Arg = dyn_cast<Argument>(Def)) {
-      // If we're spilling an Argument, make sure we clear 'nocapture'
+      // If we're spilling an Argument, make sure we clear 'captures'
       // from the coroutine function.
-      Arg->getParent()->removeParamAttr(Arg->getArgNo(), Attribute::NoCapture);
+      Arg->getParent()->removeParamAttr(Arg->getArgNo(), Attribute::Captures);
 
       if (Arg->hasByValAttr())
         ByValTy = Arg->getParamByValType();
@@ -1448,34 +1449,39 @@ static void rewritePHIs(BasicBlock &BB) {
 
   // Special case for CleanupPad: all EH blocks must have the same unwind edge
   // so we need to create an additional "dispatcher" block.
-  if (auto *CleanupPad =
-          dyn_cast_or_null<CleanupPadInst>(BB.getFirstNonPHI())) {
-    SmallVector<BasicBlock *, 8> Preds(predecessors(&BB));
-    for (BasicBlock *Pred : Preds) {
-      if (CatchSwitchInst *CS =
-              dyn_cast<CatchSwitchInst>(Pred->getTerminator())) {
-        // CleanupPad with a CatchSwitch predecessor: therefore this is an
-        // unwind destination that needs to be handle specially.
-        assert(CS->getUnwindDest() == &BB);
-        (void)CS;
-        rewritePHIsForCleanupPad(&BB, CleanupPad);
-        return;
+  if (!BB.empty()) {
+    if (auto *CleanupPad =
+            dyn_cast_or_null<CleanupPadInst>(BB.getFirstNonPHIIt())) {
+      SmallVector<BasicBlock *, 8> Preds(predecessors(&BB));
+      for (BasicBlock *Pred : Preds) {
+        if (CatchSwitchInst *CS =
+                dyn_cast<CatchSwitchInst>(Pred->getTerminator())) {
+          // CleanupPad with a CatchSwitch predecessor: therefore this is an
+          // unwind destination that needs to be handle specially.
+          assert(CS->getUnwindDest() == &BB);
+          (void)CS;
+          rewritePHIsForCleanupPad(&BB, CleanupPad);
+          return;
+        }
       }
     }
   }
 
   LandingPadInst *LandingPad = nullptr;
   PHINode *ReplPHI = nullptr;
-  if ((LandingPad = dyn_cast_or_null<LandingPadInst>(BB.getFirstNonPHI()))) {
-    // ehAwareSplitEdge will clone the LandingPad in all the edge blocks.
-    // We replace the original landing pad with a PHINode that will collect the
-    // results from all of them.
-    ReplPHI = PHINode::Create(LandingPad->getType(), 1, "");
-    ReplPHI->insertBefore(LandingPad->getIterator());
-    ReplPHI->takeName(LandingPad);
-    LandingPad->replaceAllUsesWith(ReplPHI);
-    // We will erase the original landing pad at the end of this function after
-    // ehAwareSplitEdge cloned it in the transition blocks.
+  if (!BB.empty()) {
+    if ((LandingPad =
+             dyn_cast_or_null<LandingPadInst>(BB.getFirstNonPHIIt()))) {
+      // ehAwareSplitEdge will clone the LandingPad in all the edge blocks.
+      // We replace the original landing pad with a PHINode that will collect the
+      // results from all of them.
+      ReplPHI = PHINode::Create(LandingPad->getType(), 1, "");
+      ReplPHI->insertBefore(LandingPad->getIterator());
+      ReplPHI->takeName(LandingPad);
+      LandingPad->replaceAllUsesWith(ReplPHI);
+      // We will erase the original landing pad at the end of this function after
+      // ehAwareSplitEdge cloned it in the transition blocks.
+    }
   }
 
   SmallVector<BasicBlock *, 8> Preds(predecessors(&BB));
@@ -1697,7 +1703,8 @@ static void eliminateSwiftErrorAlloca(Function &F, AllocaInst *Alloca,
 static void eliminateSwiftErrorArgument(Function &F, Argument &Arg,
                                         coro::Shape &Shape,
                              SmallVectorImpl<AllocaInst*> &AllocasToPromote) {
-  IRBuilder<> Builder(F.getEntryBlock().getFirstNonPHIOrDbg());
+  IRBuilder<> Builder(&F.getEntryBlock(),
+                      F.getEntryBlock().getFirstNonPHIOrDbg());
 
   auto ArgTy = cast<PointerType>(Arg.getType());
   auto ValueTy = PointerType::getUnqual(F.getContext());
diff --git a/llvm/lib/Transforms/Coroutines/CoroSplit.cpp b/llvm/lib/Transforms/Coroutines/CoroSplit.cpp
index ff5df12c398c5..e1f767edd6ee1 100644
--- a/llvm/lib/Transforms/Coroutines/CoroSplit.cpp
+++ b/llvm/lib/Transforms/Coroutines/CoroSplit.cpp
@@ -597,7 +597,8 @@ static void replaceSwiftErrorOps(Function &F, coro::Shape &Shape,
     }
 
     // Create a swifterror alloca.
-    IRBuilder<> Builder(F.getEntryBlock().getFirstNonPHIOrDbg());
+    IRBuilder<> Builder(&F.getEntryBlock(),
+                        F.getEntryBlock().getFirstNonPHIOrDbg());
     auto Alloca = Builder.CreateAlloca(ValueTy);
     Alloca->setSwiftError(true);
 
@@ -823,7 +824,16 @@ static void updateScopeLine(Instruction *ActiveSuspend,
   if (!ActiveSuspend)
     return;
 
-  auto *Successor = ActiveSuspend->getNextNonDebugInstruction();
+  // No subsequent instruction -> fallback to the location of ActiveSuspend.
+  if (!ActiveSuspend->getNextNonDebugInstruction()) {
+    if (auto DL = ActiveSuspend->getDebugLoc())
+      if (SPToUpdate.getFile() == DL->getFile())
+        SPToUpdate.setScopeLine(DL->getLine());
+    return;
+  }
+
+  BasicBlock::iterator Successor =
+      ActiveSuspend->getNextNonDebugInstruction()->getIterator();
   // Corosplit splits the BB around ActiveSuspend, so the meaningful
   // instructions are not in the same BB.
   if (auto *Branch = dyn_cast_or_null<BranchInst>(Successor);
@@ -832,7 +842,9 @@ static void updateScopeLine(Instruction *ActiveSuspend,
 
   // Find the first successor of ActiveSuspend with a non-zero line location.
   // If that matches the file of ActiveSuspend, use it.
-  for (; Successor; Successor = Successor->getNextNonDebugInstruction()) {
+  BasicBlock *PBB = Successor->getParent();
+  for (; Successor != PBB->end(); Successor = std::next(Successor)) {
+    Successor = skipDebugIntrinsics(Successor);
     auto DL = Successor->getDebugLoc();
     if (!DL || DL.getLine() == 0)
       continue;
@@ -1221,8 +1233,8 @@ static void handleNoSuspendCoroutine(coro::Shape &Shape) {
 // SimplifySuspendPoint needs to check that there is no calls between
 // coro_save and coro_suspend, since any of the calls may potentially resume
 // the coroutine and if that is the case we cannot eliminate the suspend point.
-static bool hasCallsInBlockBetween(Instruction *From, Instruction *To) {
-  for (Instruction *I = From; I != To; I = I->getNextNode()) {
+static bool hasCallsInBlockBetween(iterator_range<BasicBlock::iterator> R) {
+  for (Instruction &I : R) {
     // Assume that no intrinsic can resume the coroutine.
     if (isa<IntrinsicInst>(I))
       continue;
@@ -1256,7 +1268,7 @@ static bool hasCallsInBlocksBetween(BasicBlock *SaveBB, BasicBlock *ResDesBB) {
   Set.erase(ResDesBB);
 
   for (auto *BB : Set)
-    if (hasCallsInBlockBetween(BB->getFirstNonPHI(), nullptr))
+    if (hasCallsInBlockBetween({BB->getFirstNonPHIIt(), BB->end()}))
       return true;
 
   return false;
@@ -1265,17 +1277,19 @@ static bool hasCallsInBlocksBetween(BasicBlock *SaveBB, BasicBlock *ResDesBB) {
 static bool hasCallsBetween(Instruction *Save, Instruction *ResumeOrDestroy) {
   auto *SaveBB = Save->getParent();
   auto *ResumeOrDestroyBB = ResumeOrDestroy->getParent();
+  BasicBlock::iterator SaveIt = Save->getIterator();
+  BasicBlock::iterator ResumeOrDestroyIt = ResumeOrDestroy->getIterator();
 
   if (SaveBB == ResumeOrDestroyBB)
-    return hasCallsInBlockBetween(Save->getNextNode(), ResumeOrDestroy);
+    return hasCallsInBlockBetween({std::next(SaveIt), ResumeOrDestroyIt});
 
   // Any calls from Save to the end of the block?
-  if (hasCallsInBlockBetween(Save->getNextNode(), nullptr))
+  if (hasCallsInBlockBetween({std::next(SaveIt), SaveBB->end()}))
     return true;
 
   // Any calls from begging of the block up to ResumeOrDestroy?
-  if (hasCallsInBlockBetween(ResumeOrDestroyBB->getFirstNonPHI(),
-                             ResumeOrDestroy))
+  if (hasCallsInBlockBetween(
+          {ResumeOrDestroyBB->getFirstNonPHIIt(), ResumeOrDestroyIt}))
     return true;
 
   // Any calls in all of the blocks between SaveBB and ResumeOrDestroyBB?
diff --git a/llvm/lib/Transforms/Coroutines/MaterializationUtils.cpp b/llvm/lib/Transforms/Coroutines/MaterializationUtils.cpp
index 3686c7c153999..5021425152f6c 100644
--- a/llvm/lib/Transforms/Coroutines/MaterializationUtils.cpp
+++ b/llvm/lib/Transforms/Coroutines/MaterializationUtils.cpp
@@ -70,11 +70,12 @@ struct RematGraph {
                std::deque<std::unique_ptr<RematNode>> &WorkList,
                User *FirstUse) {
     RematNode *N = NUPtr.get();
-    if (Remats.count(N->Node))
+    auto [It, Inserted] = Remats.try_emplace(N->Node);
+    if (!Inserted)
       return;
 
     // We haven't see this node yet - add to the list
-    Remats[N->Node] = std::move(NUPtr);
+    It->second = std::move(NUPtr);
     for (auto &Def : N->Node->operands()) {
       Instruction *D = dyn_cast<Instruction>(Def.get());
       if (!D || !MaterializableCallback(*D) ||
diff --git a/llvm/lib/Transforms/Coroutines/SpillUtils.cpp b/llvm/lib/Transforms/Coroutines/SpillUtils.cpp
index 1ba0bdc8f3876..5062ee97a665d 100644
--- a/llvm/lib/Transforms/Coroutines/SpillUtils.cpp
+++ b/llvm/lib/Transforms/Coroutines/SpillUtils.cpp
@@ -226,9 +226,8 @@ struct AllocaUseVisitor : PtrUseVisitor<AllocaUseVisitor> {
           if (auto *S = dyn_cast<StoreInst>(U))
             if (S->getPointerOperand() == I)
               continue;
-          if (auto *II = dyn_cast<IntrinsicInst>(U))
-            if (II->isLifetimeStartOrEnd())
-              continue;
+          if (isa<LifetimeIntrinsic>(U))
+            continue;
           // BitCastInst creats aliases of the memory location being stored
           // into.
           if (auto *BI = dyn_cast<BitCastInst>(U)) {
@@ -591,9 +590,9 @@ BasicBlock::iterator getSpillInsertionPt(const coro::Shape &Shape, Value *Def,
     // the coroutine frame pointer instruction, i.e. coro.begin.
     InsertPt = Shape.getInsertPtAfterFramePtr();
 
-    // If we're spilling an Argument, make sure we clear 'nocapture'
+    // If we're spilling an Argument, make sure we clear 'captures'
     // from the coroutine function.
-    Arg->getParent()->removeParamAttr(Arg->getArgNo(), Attribute::NoCapture);
+    Arg->getParent()->removeParamAttr(Arg->getArgNo(), Attribute::Captures);
   } else if (auto *CSI = dyn_cast<AnyCoroSuspendInst>(Def)) {
     // Don't spill immediately after a suspend; splitting assumes
     // that the suspend will be followed by a branch.
diff --git a/llvm/lib/Transforms/IPO/Attributor.cpp b/llvm/lib/Transforms/IPO/Attributor.cpp
index a93284926d684..a1e1a51b201b0 100644
--- a/llvm/lib/Transforms/IPO/Attributor.cpp
+++ b/llvm/lib/Transforms/IPO/Attributor.cpp
@@ -845,7 +845,7 @@ bool AA::isAssumedThreadLocalObject(Attributor &A, Value &Obj,
       return true;
     }
     bool IsKnownNoCapture;
-    bool IsAssumedNoCapture = AA::hasAssumedIRAttr<Attribute::NoCapture>(
+    bool IsAssumedNoCapture = AA::hasAssumedIRAttr<Attribute::Captures>(
         A, &QueryingAA, IRPosition::value(Obj), DepClassTy::OPTIONAL,
         IsKnownNoCapture);
     LLVM_DEBUG(dbgs() << "[AA] Object '" << Obj << "' is "
@@ -3330,10 +3330,10 @@ void Attributor::rememberDependences() {
 }
 
 template <Attribute::AttrKind AK, typename AAType>
-void Attributor::checkAndQueryIRAttr(const IRPosition &IRP,
-                                     AttributeSet Attrs) {
+void Attributor::checkAndQueryIRAttr(const IRPosition &IRP, AttributeSet Attrs,
+                                     bool SkipHasAttrCheck) {
   bool IsKnown;
-  if (!Attrs.hasAttribute(AK))
+  if (SkipHasAttrCheck || !Attrs.hasAttribute(AK))
     if (!Configuration.Allowed || Configuration.Allowed->count(&AAType::ID))
       if (!AA::hasAssumedIRAttr<AK>(*this, nullptr, IRP, DepClassTy::NONE,
                                     IsKnown))
@@ -3498,7 +3498,8 @@ void Attributor::identifyDefaultAbstractAttributes(Function &F) {
       getOrCreateAAFor<AAAlign>(ArgPos);
 
       // Every argument with pointer type might be marked nocapture.
-      checkAndQueryIRAttr<Attribute::NoCapture, AANoCapture>(ArgPos, ArgAttrs);
+      checkAndQueryIRAttr<Attribute::Captures, AANoCapture>(
+          ArgPos, ArgAttrs, /*SkipHasAttrCheck=*/true);
 
       // Every argument with pointer type might be marked
       // "readnone/readonly/writeonly/..."
@@ -3582,9 +3583,9 @@ void Attributor::identifyDefaultAbstractAttributes(Function &F) {
       // Call site argument attribute "non-null".
       checkAndQueryIRAttr<Attribute::NonNull, AANonNull>(CBArgPos, CBArgAttrs);
 
-      // Call site argument attribute "nocapture".
-      checkAndQueryIRAttr<Attribute::NoCapture, AANoCapture>(CBArgPos,
-                                                             CBArgAttrs);
+      // Call site argument attribute "captures(none)".
+      checkAndQueryIRAttr<Attribute::Captures, AANoCapture>(
+          CBArgPos, CBArgAttrs, /*SkipHasAttrCheck=*/true);
 
       // Call site argument attribute "no-alias".
       checkAndQueryIRAttr<Attribute::NoAlias, AANoAlias>(CBArgPos, CBArgAttrs);
diff --git a/llvm/lib/Transforms/IPO/AttributorAttributes.cpp b/llvm/lib/Transforms/IPO/AttributorAttributes.cpp
index 58b8f1f779f72..17e7fada10827 100644
--- a/llvm/lib/Transforms/IPO/AttributorAttributes.cpp
+++ b/llvm/lib/Transforms/IPO/AttributorAttributes.cpp
@@ -2051,7 +2051,7 @@ struct AAPointerInfoCallSiteArgument final : AAPointerInfoFloating {
     }
 
     bool IsKnownNoCapture;
-    if (!AA::hasAssumedIRAttr<Attribute::NoCapture>(
+    if (!AA::hasAssumedIRAttr<Attribute::Captures>(
             A, this, getIRPosition(), DepClassTy::OPTIONAL, IsKnownNoCapture))
       return indicatePessimisticFixpoint();
 
@@ -3954,7 +3954,7 @@ struct AANoAliasCallSiteArgument final : AANoAliasImpl {
             unsigned ArgNo = CB->getArgOperandNo(&U);
 
             bool IsKnownNoCapture;
-            if (AA::hasAssumedIRAttr<Attribute::NoCapture>(
+            if (AA::hasAssumedIRAttr<Attribute::Captures>(
                     A, this, IRPosition::callsite_argument(*CB, ArgNo),
                     DepClassTy::OPTIONAL, IsKnownNoCapture))
               return true;
@@ -3986,7 +3986,7 @@ struct AANoAliasCallSiteArgument final : AANoAliasImpl {
 
     bool IsKnownNoCapture;
     const AANoCapture *NoCaptureAA = nullptr;
-    bool IsAssumedNoCapture = AA::hasAssumedIRAttr<Attribute::NoCapture>(
+    bool IsAssumedNoCapture = AA::hasAssumedIRAttr<Attribute::Captures>(
         A, this, VIRP, DepClassTy::NONE, IsKnownNoCapture, false, &NoCaptureAA);
     if (!IsAssumedNoCapture &&
         (!NoCaptureAA || !NoCaptureAA->isAssumedNoCaptureMaybeReturned())) {
@@ -4072,7 +4072,7 @@ struct AANoAliasReturned final : AANoAliasImpl {
 
       bool IsKnownNoCapture;
       const AANoCapture *NoCaptureAA = nullptr;
-      bool IsAssumedNoCapture = AA::hasAssumedIRAttr<Attribute::NoCapture>(
+      bool IsAssumedNoCapture = AA::hasAssumedIRAttr<Attribute::Captures>(
           A, this, RVPos, DepClassTy::REQUIRED, IsKnownNoCapture, false,
           &NoCaptureAA);
       return IsAssumedNoCapture ||
@@ -5727,7 +5727,7 @@ struct AAInstanceInfoCallSiteReturned final : AAInstanceInfoFloating {
 bool AANoCapture::isImpliedByIR(Attributor &A, const IRPosition &IRP,
                                 Attribute::AttrKind ImpliedAttributeKind,
                                 bool IgnoreSubsumingPositions) {
-  assert(ImpliedAttributeKind == Attribute::NoCapture &&
+  assert(ImpliedAttributeKind == Attribute::Captures &&
          "Unexpected attribute kind");
   Value &V = IRP.getAssociatedValue();
   if (!IRP.isArgumentPosition())
@@ -5742,27 +5742,37 @@ bool AANoCapture::isImpliedByIR(Attributor &A, const IRPosition &IRP,
     return true;
   }
 
-  if (A.hasAttr(IRP, {Attribute::NoCapture},
-                /* IgnoreSubsumingPositions */ true, Attribute::NoCapture))
-    return true;
+  SmallVector<Attribute, 1> Attrs;
+  A.getAttrs(IRP, {Attribute::Captures}, Attrs,
+             /* IgnoreSubsumingPositions */ true);
+  for (const Attribute &Attr : Attrs)
+    if (capturesNothing(Attr.getCaptureInfo()))
+      return true;
 
   if (IRP.getPositionKind() == IRP_CALL_SITE_ARGUMENT)
-    if (Argument *Arg = IRP.getAssociatedArgument())
-      if (A.hasAttr(IRPosition::argument(*Arg),
-                    {Attribute::NoCapture, Attribute::ByVal},
-                    /* IgnoreSubsumingPositions */ true)) {
-        A.manifestAttrs(IRP,
-                        Attribute::get(V.getContext(), Attribute::NoCapture));
+    if (Argument *Arg = IRP.getAssociatedArgument()) {
+      SmallVector<Attribute, 1> Attrs;
+      A.getAttrs(IRPosition::argument(*Arg),
+                 {Attribute::Captures, Attribute::ByVal}, Attrs,
+                 /* IgnoreSubsumingPositions */ true);
+      bool ArgNoCapture = any_of(Attrs, [](Attribute Attr) {
+        return Attr.getKindAsEnum() == Attribute::ByVal ||
+               capturesNothing(Attr.getCaptureInfo());
+      });
+      if (ArgNoCapture) {
+        A.manifestAttrs(IRP, Attribute::getWithCaptureInfo(
+                                 V.getContext(), CaptureInfo::none()));
         return true;
       }
+    }
 
   if (const Function *F = IRP.getAssociatedFunction()) {
     // Check what state the associated function can actually capture.
     AANoCapture::StateType State;
     determineFunctionCaptureCapabilities(IRP, *F, State);
     if (State.isKnown(NO_CAPTURE)) {
-      A.manifestAttrs(IRP,
-                      Attribute::get(V.getContext(), Attribute::NoCapture));
+      A.manifestAttrs(IRP, Attribute::getWithCaptureInfo(V.getContext(),
+                                                         CaptureInfo::none()));
       return true;
     }
   }
@@ -5825,7 +5835,7 @@ struct AANoCaptureImpl : public AANoCapture {
   /// See AbstractAttribute::initialize(...).
   void initialize(Attributor &A) override {
     bool IsKnown;
-    assert(!AA::hasAssumedIRAttr<Attribute::NoCapture>(
+    assert(!AA::hasAssumedIRAttr<Attribute::Captures>(
         A, nullptr, getIRPosition(), DepClassTy::NONE, IsKnown));
     (void)IsKnown;
   }
@@ -5841,7 +5851,7 @@ struct AANoCaptureImpl : public AANoCapture {
 
     if (isArgumentPosition()) {
       if (isAssumedNoCapture())
-        Attrs.emplace_back(Attribute::get(Ctx, Attribute::NoCapture));
+        Attrs.emplace_back(Attribute::get(Ctx, Attribute::Captures));
       else if (ManifestInternal)
         Attrs.emplace_back(Attribute::get(Ctx, "no-capture-maybe-returned"));
     }
@@ -5903,7 +5913,7 @@ struct AANoCaptureImpl : public AANoCapture {
     // it to justify a non-capture attribute here. This allows recursion!
     bool IsKnownNoCapture;
     const AANoCapture *ArgNoCaptureAA = nullptr;
-    bool IsAssumedNoCapture = AA::hasAssumedIRAttr<Attribute::NoCapture>(
+    bool IsAssumedNoCapture = AA::hasAssumedIRAttr<Attribute::Captures>(
         A, this, CSArgPos, DepClassTy::REQUIRED, IsKnownNoCapture, false,
         &ArgNoCaptureAA);
     if (IsAssumedNoCapture)
@@ -6059,7 +6069,7 @@ struct AANoCaptureCallSiteArgument final : AANoCaptureImpl {
     const IRPosition &ArgPos = IRPosition::argument(*Arg);
     bool IsKnownNoCapture;
     const AANoCapture *ArgAA = nullptr;
-    if (AA::hasAssumedIRAttr<Attribute::NoCapture>(
+    if (AA::hasAssumedIRAttr<Attribute::Captures>(
             A, this, ArgPos, DepClassTy::REQUIRED, IsKnownNoCapture, false,
             &ArgAA))
       return ChangeStatus::UNCHANGED;
@@ -7057,7 +7067,7 @@ ChangeStatus AAHeapToStackFunction::updateImpl(Attributor &A) {
         auto CBIRP = IRPosition::callsite_argument(*CB, ArgNo);
 
         bool IsKnownNoCapture;
-        bool IsAssumedNoCapture = AA::hasAssumedIRAttr<Attribute::NoCapture>(
+        bool IsAssumedNoCapture = AA::hasAssumedIRAttr<Attribute::Captures>(
             A, this, CBIRP, DepClassTy::OPTIONAL, IsKnownNoCapture);
 
         // If a call site argument use is nofree, we are fine.
@@ -7724,7 +7734,7 @@ struct AAPrivatizablePtrCallSiteArgument final
 
     const IRPosition &IRP = getIRPosition();
     bool IsKnownNoCapture;
-    bool IsAssumedNoCapture = AA::hasAssumedIRAttr<Attribute::NoCapture>(
+    bool IsAssumedNoCapture = AA::hasAssumedIRAttr<Attribute::Captures>(
         A, this, IRP, DepClassTy::REQUIRED, IsKnownNoCapture);
     if (!IsAssumedNoCapture) {
       LLVM_DEBUG(dbgs() << "[AAPrivatizablePtr] pointer might be captured!\n");
@@ -8179,7 +8189,7 @@ ChangeStatus AAMemoryBehaviorFloating::updateImpl(Attributor &A) {
   // to fall back to anythign less optimistic than the function state.
   bool IsKnownNoCapture;
   const AANoCapture *ArgNoCaptureAA = nullptr;
-  bool IsAssumedNoCapture = AA::hasAssumedIRAttr<Attribute::NoCapture>(
+  bool IsAssumedNoCapture = AA::hasAssumedIRAttr<Attribute::Captures>(
       A, this, IRP, DepClassTy::OPTIONAL, IsKnownNoCapture, false,
       &ArgNoCaptureAA);
 
@@ -8239,7 +8249,7 @@ bool AAMemoryBehaviorFloating::followUsersOfUseIn(Attributor &A, const Use &U,
   if (U.get()->getType()->isPointerTy()) {
     unsigned ArgNo = CB->getArgOperandNo(&U);
     bool IsKnownNoCapture;
-    return !AA::hasAssumedIRAttr<Attribute::NoCapture>(
+    return !AA::hasAssumedIRAttr<Attribute::Captures>(
         A, this, IRPosition::callsite_argument(*CB, ArgNo),
         DepClassTy::OPTIONAL, IsKnownNoCapture);
   }
@@ -12802,7 +12812,7 @@ struct AAAllocationInfoImpl : public AAAllocationInfo {
       return indicatePessimisticFixpoint();
 
     bool IsKnownNoCapture;
-    if (!AA::hasAssumedIRAttr<Attribute::NoCapture>(
+    if (!AA::hasAssumedIRAttr<Attribute::Captures>(
             A, this, IRP, DepClassTy::OPTIONAL, IsKnownNoCapture))
       return indicatePessimisticFixpoint();
 
diff --git a/llvm/lib/Transforms/IPO/FunctionAttrs.cpp b/llvm/lib/Transforms/IPO/FunctionAttrs.cpp
index 17f946e5acdf8..cf56f67e4de3f 100644
--- a/llvm/lib/Transforms/IPO/FunctionAttrs.cpp
+++ b/llvm/lib/Transforms/IPO/FunctionAttrs.cpp
@@ -1208,7 +1208,8 @@ static void addArgumentAttrs(const SCCNodeSet &SCCNodes,
         F->getReturnType()->isVoidTy()) {
       for (Argument &A : F->args()) {
         if (A.getType()->isPointerTy() && !A.hasNoCaptureAttr()) {
-          A.addAttr(Attribute::NoCapture);
+          A.addAttr(Attribute::getWithCaptureInfo(A.getContext(),
+                                                  CaptureInfo::none()));
           ++NumNoCapture;
           Changed.insert(F);
         }
@@ -1226,7 +1227,8 @@ static void addArgumentAttrs(const SCCNodeSet &SCCNodes,
         if (!Tracker.Captured) {
           if (Tracker.Uses.empty()) {
             // If it's trivially not captured, mark it nocapture now.
-            A.addAttr(Attribute::NoCapture);
+            A.addAttr(Attribute::getWithCaptureInfo(A.getContext(),
+                                                    CaptureInfo::none()));
             ++NumNoCapture;
             Changed.insert(F);
           } else {
@@ -1279,7 +1281,8 @@ static void addArgumentAttrs(const SCCNodeSet &SCCNodes,
       if (ArgumentSCC[0]->Uses.size() == 1 &&
           ArgumentSCC[0]->Uses[0] == ArgumentSCC[0]) {
         Argument *A = ArgumentSCC[0]->Definition;
-        A->addAttr(Attribute::NoCapture);
+        A->addAttr(Attribute::getWithCaptureInfo(A->getContext(),
+                                                 CaptureInfo::none()));
         ++NumNoCapture;
         Changed.insert(A->getParent());
 
@@ -1293,16 +1296,6 @@ static void addArgumentAttrs(const SCCNodeSet &SCCNodes,
       continue;
     }
 
-    bool SCCCaptured = false;
-    for (ArgumentGraphNode *Node : ArgumentSCC) {
-      if (Node->Uses.empty() && !Node->Definition->hasNoCaptureAttr()) {
-        SCCCaptured = true;
-        break;
-      }
-    }
-    if (SCCCaptured)
-      continue;
-
     SmallPtrSet<Argument *, 8> ArgumentSCCNodes;
     // Fill ArgumentSCCNodes with the elements of the ArgumentSCC.  Used for
     // quickly looking up whether a given Argument is in this ArgumentSCC.
@@ -1310,6 +1303,7 @@ static void addArgumentAttrs(const SCCNodeSet &SCCNodes,
       ArgumentSCCNodes.insert(I->Definition);
     }
 
+    bool SCCCaptured = false;
     for (ArgumentGraphNode *N : ArgumentSCC) {
       for (ArgumentGraphNode *Use : N->Uses) {
         Argument *A = Use->Definition;
@@ -1326,7 +1320,8 @@ static void addArgumentAttrs(const SCCNodeSet &SCCNodes,
 
     for (ArgumentGraphNode *N : ArgumentSCC) {
       Argument *A = N->Definition;
-      A->addAttr(Attribute::NoCapture);
+      A->addAttr(
+          Attribute::getWithCaptureInfo(A->getContext(), CaptureInfo::none()));
       ++NumNoCapture;
       Changed.insert(A->getParent());
     }
diff --git a/llvm/lib/Transforms/IPO/FunctionSpecialization.cpp b/llvm/lib/Transforms/IPO/FunctionSpecialization.cpp
index 449d64d1614ff..c13305ce5056d 100644
--- a/llvm/lib/Transforms/IPO/FunctionSpecialization.cpp
+++ b/llvm/lib/Transforms/IPO/FunctionSpecialization.cpp
@@ -415,6 +415,8 @@ Constant *InstCostVisitor::visitCallBase(CallBase &I) {
 
   for (unsigned Idx = 0, E = I.getNumOperands() - 1; Idx != E; ++Idx) {
     Value *V = I.getOperand(Idx);
+    if (isa<MetadataAsValue>(V))
+      return nullptr;
     Constant *C = findConstantFor(V);
     if (!C)
       return nullptr;
diff --git a/llvm/lib/Transforms/IPO/GlobalOpt.cpp b/llvm/lib/Transforms/IPO/GlobalOpt.cpp
index 00c20ad5f3709..9586fc97a39f7 100644
--- a/llvm/lib/Transforms/IPO/GlobalOpt.cpp
+++ b/llvm/lib/Transforms/IPO/GlobalOpt.cpp
@@ -95,20 +95,11 @@ STATISTIC(NumIFuncsDeleted, "Number of IFuncs removed");
 STATISTIC(NumGlobalArraysPadded,
           "Number of global arrays padded to alignment boundary");
 
-// FIXME:
-// Optimizing non-FMV callers is causing a regression in the llvm test suite,
-// specifically a 'predres' version is unexpectedly trapping on GravitonG4.
-// My explanation is that when the caller in not a versioned function, the
-// compiler exclusively relies on the command line option, or target attribute
-// to deduce whether a feature is available. However, there is no guarantee
-// that in reality the host supports those implied features, which arguably
-// is a user error. This option allows disabling the optimization as a short
-// term workaround to keep the bots green.
 static cl::opt<bool>
     OptimizeNonFMVCallers("optimize-non-fmv-callers",
                           cl::desc("Statically resolve calls to versioned "
                                    "functions from non-versioned callers."),
-                          cl::init(false), cl::Hidden);
+                          cl::init(true), cl::Hidden);
 
 static cl::opt<bool>
     EnableColdCCStressTest("enable-coldcc-stress-test",
diff --git a/llvm/lib/Transforms/IPO/IROutliner.cpp b/llvm/lib/Transforms/IPO/IROutliner.cpp
index 3f54106bd09fe..34ddeeb37400c 100644
--- a/llvm/lib/Transforms/IPO/IROutliner.cpp
+++ b/llvm/lib/Transforms/IPO/IROutliner.cpp
@@ -197,7 +197,7 @@ Value *OutlinableRegion::findCorrespondingValueIn(const OutlinableRegion &Other,
 BasicBlock *
 OutlinableRegion::findCorrespondingBlockIn(const OutlinableRegion &Other,
                                            BasicBlock *BB) {
-  Instruction *FirstNonPHI = BB->getFirstNonPHIOrDbg();
+  Instruction *FirstNonPHI = &*BB->getFirstNonPHIOrDbg();
   assert(FirstNonPHI && "block is empty?");
   Value *CorrespondingVal = findCorrespondingValueIn(Other, FirstNonPHI);
   if (!CorrespondingVal)
@@ -1184,22 +1184,22 @@ static std::optional<unsigned> getGVNForPHINode(OutlinableRegion &Region,
   for (unsigned Idx = 0, EIdx = PN->getNumIncomingValues(); Idx < EIdx; Idx++) {
     Incoming = PN->getIncomingValue(Idx);
     IncomingBlock = PN->getIncomingBlock(Idx);
+    // If the incoming block isn't in the region, we don't have to worry about
+    // this incoming value.
+    if (!Blocks.contains(IncomingBlock))
+      continue;
+
     // If we cannot find a GVN, and the incoming block is included in the region
     // this means that the input to the PHINode is not included in the region we
     // are trying to analyze, meaning, that if it was outlined, we would be
     // adding an extra input.  We ignore this case for now, and so ignore the
     // region.
     std::optional<unsigned> OGVN = Cand.getGVN(Incoming);
-    if (!OGVN && Blocks.contains(IncomingBlock)) {
+    if (!OGVN) {
       Region.IgnoreRegion = true;
       return std::nullopt;
     }
 
-    // If the incoming block isn't in the region, we don't have to worry about
-    // this incoming value.
-    if (!Blocks.contains(IncomingBlock))
-      continue;
-
     // Collect the canonical numbers of the values in the PHINode.
     unsigned GVN = *OGVN;
     OGVN = Cand.getCanonicalNum(GVN);
diff --git a/llvm/lib/Transforms/IPO/Internalize.cpp b/llvm/lib/Transforms/IPO/Internalize.cpp
index 4cdd1fa611062..404102eef89fc 100644
--- a/llvm/lib/Transforms/IPO/Internalize.cpp
+++ b/llvm/lib/Transforms/IPO/Internalize.cpp
@@ -233,6 +233,10 @@ bool InternalizePass::internalizeModule(Module &M) {
   else
     AlwaysPreserved.insert("__stack_chk_guard");
 
+  // Preserve the RPC interface for GPU host callbacks when internalizing.
+  if (Triple(M.getTargetTriple()).isNVPTX())
+    AlwaysPreserved.insert("__llvm_rpc_client");
+
   // Mark all functions not in the api as internal.
   IsWasm = Triple(M.getTargetTriple()).isOSBinFormatWasm();
   for (Function &I : M) {
diff --git a/llvm/lib/Transforms/IPO/MemProfContextDisambiguation.cpp b/llvm/lib/Transforms/IPO/MemProfContextDisambiguation.cpp
index 1966ce2908371..d748b162d7809 100644
--- a/llvm/lib/Transforms/IPO/MemProfContextDisambiguation.cpp
+++ b/llvm/lib/Transforms/IPO/MemProfContextDisambiguation.cpp
@@ -124,7 +124,7 @@ static cl::opt<unsigned>
 
 // Optionally enable cloning of callsites involved with recursive cycles
 static cl::opt<bool> AllowRecursiveCallsites(
-    "memprof-allow-recursive-callsites", cl::init(false), cl::Hidden,
+    "memprof-allow-recursive-callsites", cl::init(true), cl::Hidden,
     cl::desc("Allow cloning of callsites involved in recursive cycles"));
 
 // When disabled, try to detect and prevent cloning of recursive contexts.
@@ -301,12 +301,14 @@ class CallsiteContextGraph {
       // callers (i.e. if this is the leaf allocation node).
       if (!CalleeEdges.empty())
         return &CalleeEdges;
-      if (!CallerEdges.empty()) {
-        // A node with caller edges but no callee edges must be the allocation
-        // node.
-        assert(IsAllocation);
+      // Typically if the callee edges are empty either the caller edges are
+      // also empty, or this is an allocation (leaf node). However, if we are
+      // allowing recursive callsites and contexts this will be violated for
+      // incompletely cloned recursive cycles.
+      assert(CallerEdges.empty() || IsAllocation ||
+             (AllowRecursiveCallsites && AllowRecursiveContexts));
+      if (!CallerEdges.empty() && IsAllocation)
         return &CallerEdges;
-      }
       return nullptr;
     }
 
@@ -403,8 +405,13 @@ class CallsiteContextGraph {
     // True if this node was effectively removed from the graph, in which case
     // it should have an allocation type of None and empty context ids.
     bool isRemoved() const {
-      assert((AllocTypes == (uint8_t)AllocationType::None) ==
-             emptyContextIds());
+      // Typically if the callee edges are empty either the caller edges are
+      // also empty, or this is an allocation (leaf node). However, if we are
+      // allowing recursive callsites and contexts this will be violated for
+      // incompletely cloned recursive cycles.
+      assert((AllowRecursiveCallsites && AllowRecursiveContexts) ||
+             (AllocTypes == (uint8_t)AllocationType::None) ==
+                 emptyContextIds());
       return AllocTypes == (uint8_t)AllocationType::None;
     }
 
@@ -677,18 +684,18 @@ class CallsiteContextGraph {
 
   /// Computes the alloc type corresponding to the given context ids, by
   /// unioning their recorded alloc types.
-  uint8_t computeAllocType(DenseSet<uint32_t> &ContextIds);
+  uint8_t computeAllocType(DenseSet<uint32_t> &ContextIds) const;
 
   /// Returns the allocation type of the intersection of the contexts of two
   /// nodes (based on their provided context id sets), optimized for the case
   /// when Node1Ids is smaller than Node2Ids.
   uint8_t intersectAllocTypesImpl(const DenseSet<uint32_t> &Node1Ids,
-                                  const DenseSet<uint32_t> &Node2Ids);
+                                  const DenseSet<uint32_t> &Node2Ids) const;
 
   /// Returns the allocation type of the intersection of the contexts of two
   /// nodes (based on their provided context id sets).
   uint8_t intersectAllocTypes(const DenseSet<uint32_t> &Node1Ids,
-                              const DenseSet<uint32_t> &Node2Ids);
+                              const DenseSet<uint32_t> &Node2Ids) const;
 
   /// Create a clone of Edge's callee and move Edge to that new callee node,
   /// performing the necessary context id and allocation type updates.
@@ -1057,6 +1064,7 @@ template <typename DerivedCCG, typename FuncTy, typename CallTy>
 void CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::removeEdgeFromGraph(
     ContextEdge *Edge, EdgeIter *EI, bool CalleeIter) {
   assert(!EI || (*EI)->get() == Edge);
+  assert(!Edge->isRemoved());
   // Save the Caller and Callee pointers so we can erase Edge from their edge
   // lists after clearing Edge below. We do the clearing first in case it is
   // destructed after removing from the edge lists (if those were the last
@@ -1069,6 +1077,10 @@ void CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::removeEdgeFromGraph(
   // reference.
   Edge->clear();
 
+#ifndef NDEBUG
+  auto CalleeCallerCount = Callee->CallerEdges.size();
+  auto CallerCalleeCount = Caller->CalleeEdges.size();
+#endif
   if (!EI) {
     Callee->eraseCallerEdge(Edge);
     Caller->eraseCalleeEdge(Edge);
@@ -1079,6 +1091,8 @@ void CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::removeEdgeFromGraph(
     Caller->eraseCalleeEdge(Edge);
     *EI = Callee->CallerEdges.erase(*EI);
   }
+  assert(Callee->CallerEdges.size() < CalleeCallerCount);
+  assert(Caller->CalleeEdges.size() < CallerCalleeCount);
 }
 
 template <typename DerivedCCG, typename FuncTy, typename CallTy>
@@ -1152,12 +1166,12 @@ void CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::ContextNode::
 
 template <typename DerivedCCG, typename FuncTy, typename CallTy>
 uint8_t CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::computeAllocType(
-    DenseSet<uint32_t> &ContextIds) {
+    DenseSet<uint32_t> &ContextIds) const {
   uint8_t BothTypes =
       (uint8_t)AllocationType::Cold | (uint8_t)AllocationType::NotCold;
   uint8_t AllocType = (uint8_t)AllocationType::None;
   for (auto Id : ContextIds) {
-    AllocType |= (uint8_t)ContextIdToAllocationType[Id];
+    AllocType |= (uint8_t)ContextIdToAllocationType.at(Id);
     // Bail early if alloc type reached both, no further refinement.
     if (AllocType == BothTypes)
       return AllocType;
@@ -1168,14 +1182,15 @@ uint8_t CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::computeAllocType(
 template <typename DerivedCCG, typename FuncTy, typename CallTy>
 uint8_t
 CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::intersectAllocTypesImpl(
-    const DenseSet<uint32_t> &Node1Ids, const DenseSet<uint32_t> &Node2Ids) {
+    const DenseSet<uint32_t> &Node1Ids,
+    const DenseSet<uint32_t> &Node2Ids) const {
   uint8_t BothTypes =
       (uint8_t)AllocationType::Cold | (uint8_t)AllocationType::NotCold;
   uint8_t AllocType = (uint8_t)AllocationType::None;
   for (auto Id : Node1Ids) {
     if (!Node2Ids.count(Id))
       continue;
-    AllocType |= (uint8_t)ContextIdToAllocationType[Id];
+    AllocType |= (uint8_t)ContextIdToAllocationType.at(Id);
     // Bail early if alloc type reached both, no further refinement.
     if (AllocType == BothTypes)
       return AllocType;
@@ -1185,7 +1200,8 @@ CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::intersectAllocTypesImpl(
 
 template <typename DerivedCCG, typename FuncTy, typename CallTy>
 uint8_t CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::intersectAllocTypes(
-    const DenseSet<uint32_t> &Node1Ids, const DenseSet<uint32_t> &Node2Ids) {
+    const DenseSet<uint32_t> &Node1Ids,
+    const DenseSet<uint32_t> &Node2Ids) const {
   if (Node1Ids.size() < Node2Ids.size())
     return intersectAllocTypesImpl(Node1Ids, Node2Ids);
   else
@@ -1335,16 +1351,48 @@ void CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::connectNewNode(
     DenseSet<uint32_t> RemainingContextIds) {
   auto &OrigEdges =
       TowardsCallee ? OrigNode->CalleeEdges : OrigNode->CallerEdges;
+  DenseSet<uint32_t> RecursiveContextIds;
+  DenseSet<uint32_t> AllCallerContextIds;
+  if (AllowRecursiveCallsites) {
+    // Identify which context ids are recursive which is needed to properly
+    // update the RemainingContextIds set. The relevant recursive context ids
+    // are those that are in multiple edges.
+    for (auto &CE : OrigEdges) {
+      AllCallerContextIds.reserve(CE->getContextIds().size());
+      for (auto Id : CE->getContextIds())
+        if (!AllCallerContextIds.insert(Id).second)
+          RecursiveContextIds.insert(Id);
+    }
+  }
   // Increment iterator in loop so that we can remove edges as needed.
   for (auto EI = OrigEdges.begin(); EI != OrigEdges.end();) {
     auto Edge = *EI;
+    DenseSet<uint32_t> NewEdgeContextIds;
+    DenseSet<uint32_t> NotFoundContextIds;
     // Remove any matching context ids from Edge, return set that were found and
     // removed, these are the new edge's context ids. Also update the remaining
     // (not found ids).
-    DenseSet<uint32_t> NewEdgeContextIds, NotFoundContextIds;
     set_subtract(Edge->getContextIds(), RemainingContextIds, NewEdgeContextIds,
                  NotFoundContextIds);
-    RemainingContextIds.swap(NotFoundContextIds);
+    // Update the remaining context ids set for the later edges. This is a
+    // compile time optimization.
+    if (RecursiveContextIds.empty()) {
+      // No recursive ids, so all of the previously remaining context ids that
+      // were not seen on this edge are the new remaining set.
+      RemainingContextIds.swap(NotFoundContextIds);
+    } else {
+      // Keep the recursive ids in the remaining set as we expect to see those
+      // on another edge. We can remove the non-recursive remaining ids that
+      // were seen on this edge, however. We already have the set of remaining
+      // ids that were on this edge (in NewEdgeContextIds). Figure out which are
+      // non-recursive and only remove those. Note that despite the higher
+      // overhead of updating the remaining context ids set when recursion
+      // handling is enabled, it was found to be at worst performance neutral
+      // and in one case a clear win.
+      DenseSet<uint32_t> NonRecursiveRemainingCurEdgeIds =
+          set_difference(NewEdgeContextIds, RecursiveContextIds);
+      set_subtract(RemainingContextIds, NonRecursiveRemainingCurEdgeIds);
+    }
     // If no matching context ids for this edge, skip it.
     if (NewEdgeContextIds.empty()) {
       ++EI;
@@ -1401,9 +1449,9 @@ static void checkNode(const ContextNode<DerivedCCG, FuncTy, CallTy> *Node,
       set_union(CallerEdgeContextIds, Edge->ContextIds);
     }
     // Node can have more context ids than callers if some contexts terminate at
-    // node and some are longer. If we are allowing recursive callsites but
-    // haven't disabled recursive contexts, this will be violated for
-    // incompletely cloned recursive cycles, so skip the checking in that case.
+    // node and some are longer. If we are allowing recursive callsites and
+    // contexts this will be violated for incompletely cloned recursive cycles,
+    // so skip the checking in that case.
     assert((AllowRecursiveCallsites && AllowRecursiveContexts) ||
            NodeContextIds == CallerEdgeContextIds ||
            set_is_subset(CallerEdgeContextIds, NodeContextIds));
@@ -1416,7 +1464,11 @@ static void checkNode(const ContextNode<DerivedCCG, FuncTy, CallTy> *Node,
         checkEdge<DerivedCCG, FuncTy, CallTy>(Edge);
       set_union(CalleeEdgeContextIds, Edge->getContextIds());
     }
-    assert(NodeContextIds == CalleeEdgeContextIds);
+    // If we are allowing recursive callsites and contexts this will be violated
+    // for incompletely cloned recursive cycles, so skip the checking in that
+    // case.
+    assert((AllowRecursiveCallsites && AllowRecursiveContexts) ||
+           NodeContextIds == CalleeEdgeContextIds);
   }
   // FIXME: Since this checking is only invoked under an option, we should
   // change the error checking from using assert to something that will trigger
@@ -3125,6 +3177,12 @@ void CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::
   // over to the corresponding edge into the clone (which is created here if
   // this is a newly created clone).
   for (auto &OldCalleeEdge : OldCallee->CalleeEdges) {
+    ContextNode *CalleeToUse = OldCalleeEdge->Callee;
+    // If this is a direct recursion edge, use NewCallee (the clone) as the
+    // callee as well, so that any edge updated/created here is also direct
+    // recursive.
+    if (CalleeToUse == OldCallee)
+      CalleeToUse = NewCallee;
     // The context ids moving to the new callee are the subset of this edge's
     // context ids and the context ids on the caller edge being moved.
     DenseSet<uint32_t> EdgeContextIdsToMove =
@@ -3138,8 +3196,7 @@ void CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::
       // clone, specifically during function assignment, where we would have
       // removed none type edges after creating the clone. If we can't find
       // a corresponding edge there, fall through to the cloning below.
-      if (auto *NewCalleeEdge =
-              NewCallee->findEdgeFromCallee(OldCalleeEdge->Callee)) {
+      if (auto *NewCalleeEdge = NewCallee->findEdgeFromCallee(CalleeToUse)) {
         NewCalleeEdge->getContextIds().insert(EdgeContextIdsToMove.begin(),
                                               EdgeContextIdsToMove.end());
         NewCalleeEdge->AllocTypes |= computeAllocType(EdgeContextIdsToMove);
@@ -3147,8 +3204,8 @@ void CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::
       }
     }
     auto NewEdge = std::make_shared<ContextEdge>(
-        OldCalleeEdge->Callee, NewCallee,
-        computeAllocType(EdgeContextIdsToMove), EdgeContextIdsToMove);
+        CalleeToUse, NewCallee, computeAllocType(EdgeContextIdsToMove),
+        EdgeContextIdsToMove);
     NewCallee->CalleeEdges.push_back(NewEdge);
     NewEdge->Callee->CallerEdges.push_back(NewEdge);
   }
@@ -3174,13 +3231,20 @@ template <typename DerivedCCG, typename FuncTy, typename CallTy>
 void CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::
     moveCalleeEdgeToNewCaller(const std::shared_ptr<ContextEdge> &Edge,
                               ContextNode *NewCaller) {
+  auto *OldCallee = Edge->Callee;
+  auto *NewCallee = OldCallee;
+  // If this edge was direct recursive, make any new/updated edge also direct
+  // recursive to NewCaller.
+  bool Recursive = Edge->Caller == Edge->Callee;
+  if (Recursive)
+    NewCallee = NewCaller;
 
   ContextNode *OldCaller = Edge->Caller;
   OldCaller->eraseCalleeEdge(Edge.get());
 
   // We might already have an edge to the new caller. If one exists we will
   // reuse it.
-  auto ExistingEdgeToNewCaller = NewCaller->findEdgeFromCallee(Edge->Callee);
+  auto ExistingEdgeToNewCaller = NewCaller->findEdgeFromCallee(NewCallee);
 
   if (ExistingEdgeToNewCaller) {
     // Since we already have an edge to NewCaller, simply move the ids
@@ -3190,11 +3254,19 @@ void CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::
     ExistingEdgeToNewCaller->AllocTypes |= Edge->AllocTypes;
     Edge->ContextIds.clear();
     Edge->AllocTypes = (uint8_t)AllocationType::None;
-    Edge->Callee->eraseCallerEdge(Edge.get());
+    OldCallee->eraseCallerEdge(Edge.get());
   } else {
     // Otherwise just reconnect Edge to NewCaller.
     Edge->Caller = NewCaller;
     NewCaller->CalleeEdges.push_back(Edge);
+    if (Recursive) {
+      assert(NewCallee == NewCaller);
+      // In the case of (direct) recursive edges, we update the callee as well
+      // so that it becomes recursive on the new caller.
+      Edge->Callee = NewCallee;
+      NewCallee->CallerEdges.push_back(Edge);
+      OldCallee->eraseCallerEdge(Edge.get());
+    }
     // Don't need to update Edge's context ids since we are simply
     // reconnecting it.
   }
@@ -3208,32 +3280,50 @@ void CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::
 #ifndef NDEBUG
   bool IsNewNode = NewCaller->CallerEdges.empty();
 #endif
-  for (auto &OldCallerEdge : OldCaller->CallerEdges) {
-    // The context ids moving to the new caller are the subset of this edge's
-    // context ids and the context ids on the callee edge being moved.
-    DenseSet<uint32_t> EdgeContextIdsToMove =
-        set_intersection(OldCallerEdge->getContextIds(), Edge->getContextIds());
-    set_subtract(OldCallerEdge->getContextIds(), EdgeContextIdsToMove);
-    OldCallerEdge->AllocTypes =
-        computeAllocType(OldCallerEdge->getContextIds());
-    // In this function we expect that any pre-existing node already has edges
-    // from the same callers as the old node. That should be true in the current
-    // use case, where we will remove None-type edges after copying over all
-    // caller edges from the callee.
-    auto *ExistingCallerEdge =
-        NewCaller->findEdgeFromCaller(OldCallerEdge->Caller);
-    assert(IsNewNode || ExistingCallerEdge);
-    if (ExistingCallerEdge) {
-      ExistingCallerEdge->getContextIds().insert(EdgeContextIdsToMove.begin(),
-                                                 EdgeContextIdsToMove.end());
-      ExistingCallerEdge->AllocTypes |= computeAllocType(EdgeContextIdsToMove);
-      continue;
+  // If we just moved a direct recursive edge, presumably its context ids should
+  // also flow out of OldCaller via some other non-recursive callee edge. We
+  // don't want to remove the recursive context ids from other caller edges yet,
+  // otherwise the context ids get into an inconsistent state on OldCaller.
+  // We will update these context ids on the non-recursive caller edge when and
+  // if they are updated on the non-recursive callee.
+  if (!Recursive) {
+    for (auto &OldCallerEdge : OldCaller->CallerEdges) {
+      auto OldCallerCaller = OldCallerEdge->Caller;
+      // The context ids moving to the new caller are the subset of this edge's
+      // context ids and the context ids on the callee edge being moved.
+      DenseSet<uint32_t> EdgeContextIdsToMove = set_intersection(
+          OldCallerEdge->getContextIds(), Edge->getContextIds());
+      if (OldCaller == OldCallerCaller) {
+        OldCallerCaller = NewCaller;
+        // Don't actually move this one. The caller will move it directly via a
+        // call to this function with this as the Edge if it is appropriate to
+        // move to a diff node that has a matching callee (itself).
+        continue;
+      }
+      set_subtract(OldCallerEdge->getContextIds(), EdgeContextIdsToMove);
+      OldCallerEdge->AllocTypes =
+          computeAllocType(OldCallerEdge->getContextIds());
+      // In this function we expect that any pre-existing node already has edges
+      // from the same callers as the old node. That should be true in the
+      // current use case, where we will remove None-type edges after copying
+      // over all caller edges from the callee.
+      auto *ExistingCallerEdge = NewCaller->findEdgeFromCaller(OldCallerCaller);
+      // Since we would have skipped caller edges when moving a direct recursive
+      // edge, this may not hold true when recursive handling enabled.
+      assert(IsNewNode || ExistingCallerEdge || AllowRecursiveCallsites);
+      if (ExistingCallerEdge) {
+        ExistingCallerEdge->getContextIds().insert(EdgeContextIdsToMove.begin(),
+                                                   EdgeContextIdsToMove.end());
+        ExistingCallerEdge->AllocTypes |=
+            computeAllocType(EdgeContextIdsToMove);
+        continue;
+      }
+      auto NewEdge = std::make_shared<ContextEdge>(
+          NewCaller, OldCallerCaller, computeAllocType(EdgeContextIdsToMove),
+          EdgeContextIdsToMove);
+      NewCaller->CallerEdges.push_back(NewEdge);
+      NewEdge->Caller->CalleeEdges.push_back(NewEdge);
     }
-    auto NewEdge = std::make_shared<ContextEdge>(
-        NewCaller, OldCallerEdge->Caller,
-        computeAllocType(EdgeContextIdsToMove), EdgeContextIdsToMove);
-    NewCaller->CallerEdges.push_back(NewEdge);
-    NewEdge->Caller->CalleeEdges.push_back(NewEdge);
   }
   // Recompute the node alloc type now that its caller edges have been
   // updated (since we will compute from those edges).
@@ -3317,6 +3407,10 @@ void CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::identifyClones(
   if (!Node->hasCall())
     return;
 
+  // No need to look at any callers if allocation type already unambiguous.
+  if (hasSingleAllocType(Node->AllocTypes))
+    return;
+
 #ifndef NDEBUG
   auto Insert =
 #endif
@@ -3598,10 +3692,9 @@ IndexCallsiteContextGraph::cloneFunctionForCallsite(
   // The next clone number is the current size of versions array.
   // Confirm this matches the CloneNo provided by the caller, which is based on
   // the number of function clones we have.
-  assert(CloneNo ==
-         (isa<AllocInfo *>(Call.call())
-              ? Call.call().dyn_cast<AllocInfo *>()->Versions.size()
-              : Call.call().dyn_cast<CallsiteInfo *>()->Clones.size()));
+  assert(CloneNo == (isa<AllocInfo *>(Call.call())
+                         ? cast<AllocInfo *>(Call.call())->Versions.size()
+                         : cast<CallsiteInfo *>(Call.call())->Clones.size()));
   // Walk all the instructions in this function. Create a new version for
   // each (by adding an entry to the Versions/Clones summary array), and copy
   // over the version being called for the function clone being cloned here.
@@ -3938,6 +4031,11 @@ bool CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::assignFunctions() {
         // handling and makes it less error-prone.
         auto CloneCallerEdges = Clone->CallerEdges;
         for (auto &Edge : CloneCallerEdges) {
+          // Skip removed edges (due to direct recursive edges updated when
+          // updating callee edges when moving an edge and subsequently
+          // removed by call to removeNoneTypeCalleeEdges on the Clone).
+          if (Edge->isRemoved())
+            continue;
           // Ignore any caller that does not have a recorded callsite Call.
           if (!Edge->Caller->hasCall())
             continue;
@@ -4204,7 +4302,8 @@ static SmallVector<std::unique_ptr<ValueToValueMapTy>, 4> createFunctionClones(
 // Locate the summary for F. This is complicated by the fact that it might
 // have been internalized or promoted.
 static ValueInfo findValueInfoForFunc(const Function &F, const Module &M,
-                                      const ModuleSummaryIndex *ImportSummary) {
+                                      const ModuleSummaryIndex *ImportSummary,
+                                      const Function *CallingFunc = nullptr) {
   // FIXME: Ideally we would retain the original GUID in some fashion on the
   // function (e.g. as metadata), but for now do our best to locate the
   // summary without that information.
@@ -4219,20 +4318,48 @@ static ValueInfo findValueInfoForFunc(const Function &F, const Module &M,
   // Now query with the original name before any promotion was performed.
   StringRef OrigName =
       ModuleSummaryIndex::getOriginalNameBeforePromote(F.getName());
+  // When this pass is enabled, we always add thinlto_src_file provenance
+  // metadata to imported function definitions, which allows us to recreate the
+  // original internal symbol's GUID.
+  auto SrcFileMD = F.getMetadata("thinlto_src_file");
+  // If this is a call to an imported/promoted local for which we didn't import
+  // the definition, the metadata will not exist on the declaration. However,
+  // since we are doing this early, before any inlining in the LTO backend, we
+  // can simply look at the metadata on the calling function which must have
+  // been from the same module if F was an internal symbol originally.
+  if (!SrcFileMD && F.isDeclaration()) {
+    // We would only call this for a declaration for a direct callsite, in which
+    // case the caller would have provided the calling function pointer.
+    assert(CallingFunc);
+    SrcFileMD = CallingFunc->getMetadata("thinlto_src_file");
+    // If this is a promoted local (OrigName != F.getName()), since this is a
+    // declaration, it must be imported from a different module and therefore we
+    // should always find the metadata on its calling function. Any call to a
+    // promoted local that came from this module should still be a definition.
+    assert(SrcFileMD || OrigName == F.getName());
+  }
+  StringRef SrcFile = M.getSourceFileName();
+  if (SrcFileMD)
+    SrcFile = dyn_cast<MDString>(SrcFileMD->getOperand(0))->getString();
   std::string OrigId = GlobalValue::getGlobalIdentifier(
-      OrigName, GlobalValue::InternalLinkage, M.getSourceFileName());
+      OrigName, GlobalValue::InternalLinkage, SrcFile);
   TheFnVI = ImportSummary->getValueInfo(GlobalValue::getGUID(OrigId));
-  if (TheFnVI)
-    return TheFnVI;
-  // Could be a promoted local imported from another module. We need to pass
-  // down more info here to find the original module id. For now, try with
-  // the OrigName which might have been stored in the OidGuidMap in the
-  // index. This would not work if there were same-named locals in multiple
-  // modules, however.
-  auto OrigGUID =
-      ImportSummary->getGUIDFromOriginalID(GlobalValue::getGUID(OrigName));
-  if (OrigGUID)
-    TheFnVI = ImportSummary->getValueInfo(OrigGUID);
+  // Internal func in original module may have gotten a numbered suffix if we
+  // imported an external function with the same name. This happens
+  // automatically during IR linking for naming conflicts. It would have to
+  // still be internal in that case (otherwise it would have been renamed on
+  // promotion in which case we wouldn't have a naming conflict).
+  if (!TheFnVI && OrigName == F.getName() && F.hasLocalLinkage() &&
+      F.getName().contains('.')) {
+    OrigName = F.getName().rsplit('.').first;
+    OrigId = GlobalValue::getGlobalIdentifier(
+        OrigName, GlobalValue::InternalLinkage, SrcFile);
+    TheFnVI = ImportSummary->getValueInfo(GlobalValue::getGUID(OrigId));
+  }
+  // The only way we may not have a VI is if this is a declaration created for
+  // an imported reference. For distributed ThinLTO we may not have a VI for
+  // such declarations in the distributed summary.
+  assert(TheFnVI || F.isDeclaration());
   return TheFnVI;
 }
 
@@ -4591,7 +4718,7 @@ bool MemProfContextDisambiguation::applyImport(Module &M) {
           // Locate the synthesized callsite info for the callee VI, if any was
           // created, and use that for cloning.
           ValueInfo CalleeVI =
-              findValueInfoForFunc(*CalledFunction, M, ImportSummary);
+              findValueInfoForFunc(*CalledFunction, M, ImportSummary, &F);
           if (CalleeVI && MapTailCallCalleeVIToCallsite.count(CalleeVI)) {
             auto Callsite = MapTailCallCalleeVIToCallsite.find(CalleeVI);
             assert(Callsite != MapTailCallCalleeVIToCallsite.end());
diff --git a/llvm/lib/Transforms/IPO/OpenMPOpt.cpp b/llvm/lib/Transforms/IPO/OpenMPOpt.cpp
index e7221ee406a18..95c42f2b9bd45 100644
--- a/llvm/lib/Transforms/IPO/OpenMPOpt.cpp
+++ b/llvm/lib/Transforms/IPO/OpenMPOpt.cpp
@@ -19,6 +19,7 @@
 
 #include "llvm/Transforms/IPO/OpenMPOpt.h"
 
+#include "llvm/ADT/DenseSet.h"
 #include "llvm/ADT/EnumeratedArray.h"
 #include "llvm/ADT/PostOrderIterator.h"
 #include "llvm/ADT/SetVector.h"
@@ -36,6 +37,7 @@
 #include "llvm/Frontend/OpenMP/OMPIRBuilder.h"
 #include "llvm/IR/Assumptions.h"
 #include "llvm/IR/BasicBlock.h"
+#include "llvm/IR/CallingConv.h"
 #include "llvm/IR/Constants.h"
 #include "llvm/IR/DiagnosticInfo.h"
 #include "llvm/IR/Dominators.h"
@@ -567,7 +569,7 @@ struct OMPInformationCache : public InformationCache {
     for (RuntimeFunction Fn : Fns) {
       RuntimeFunctionInfo &RFI = RFIs[Fn];
 
-      if (RFI.Declaration && RFI.Declaration->isDeclaration())
+      if (!RFI.Declaration || RFI.Declaration->isDeclaration())
         return false;
     }
     return true;
@@ -2367,7 +2369,8 @@ struct AAICVTrackerFunction : public AAICVTracker {
 
     const auto *CB = dyn_cast<CallBase>(&I);
     if (!CB || CB->hasFnAttr("no_openmp") ||
-        CB->hasFnAttr("no_openmp_routines"))
+        CB->hasFnAttr("no_openmp_routines") ||
+        CB->hasFnAttr("no_openmp_constructs"))
       return std::nullopt;
 
     auto &OMPInfoCache = static_cast<OMPInformationCache &>(A.getInfoCache());
@@ -5789,6 +5792,7 @@ PreservedAnalyses OpenMPOptPass::run(Module &M, ModuleAnalysisManager &AM) {
   CallGraphUpdater CGUpdater;
 
   bool PostLink = LTOPhase == ThinOrFullLTOPhase::FullLTOPostLink ||
+                  LTOPhase == ThinOrFullLTOPhase::ThinLTOPostLink ||
                   LTOPhase == ThinOrFullLTOPhase::ThinLTOPreLink;
   OMPInformationCache InfoCache(M, AG, Allocator, /*CGSCC*/ nullptr, PostLink);
 
@@ -5868,6 +5872,7 @@ PreservedAnalyses OpenMPOptCGSCCPass::run(LazyCallGraph::SCC &C,
   CGUpdater.initialize(CG, C, AM, UR);
 
   bool PostLink = LTOPhase == ThinOrFullLTOPhase::FullLTOPostLink ||
+                  LTOPhase == ThinOrFullLTOPhase::ThinLTOPostLink ||
                   LTOPhase == ThinOrFullLTOPhase::ThinLTOPreLink;
   SetVector<Function *> Functions(SCC.begin(), SCC.end());
   OMPInformationCache InfoCache(*(Functions.back()->getParent()), AG, Allocator,
@@ -5904,33 +5909,19 @@ bool llvm::omp::isOpenMPKernel(Function &Fn) {
 }
 
 KernelSet llvm::omp::getDeviceKernels(Module &M) {
-  // TODO: Create a more cross-platform way of determining device kernels.
-  NamedMDNode *MD = M.getNamedMetadata("nvvm.annotations");
   KernelSet Kernels;
 
-  if (!MD)
-    return Kernels;
-
-  for (auto *Op : MD->operands()) {
-    if (Op->getNumOperands() < 2)
-      continue;
-    MDString *KindID = dyn_cast<MDString>(Op->getOperand(1));
-    if (!KindID || KindID->getString() != "kernel")
-      continue;
-
-    Function *KernelFn =
-        mdconst::dyn_extract_or_null<Function>(Op->getOperand(0));
-    if (!KernelFn)
-      continue;
-
-    // We are only interested in OpenMP target regions. Others, such as kernels
-    // generated by CUDA but linked together, are not interesting to this pass.
-    if (isOpenMPKernel(*KernelFn)) {
-      ++NumOpenMPTargetRegionKernels;
-      Kernels.insert(KernelFn);
-    } else
-      ++NumNonOpenMPTargetRegionKernels;
-  }
+  for (Function &F : M)
+    if (F.hasKernelCallingConv()) {
+      // We are only interested in OpenMP target regions. Others, such as
+      // kernels generated by CUDA but linked together, are not interesting to
+      // this pass.
+      if (isOpenMPKernel(F)) {
+        ++NumOpenMPTargetRegionKernels;
+        Kernels.insert(&F);
+      } else
+        ++NumNonOpenMPTargetRegionKernels;
+    }
 
   return Kernels;
 }
diff --git a/llvm/lib/Transforms/IPO/SCCP.cpp b/llvm/lib/Transforms/IPO/SCCP.cpp
index e80c6f7c0f49d..2afcdf09af016 100644
--- a/llvm/lib/Transforms/IPO/SCCP.cpp
+++ b/llvm/lib/Transforms/IPO/SCCP.cpp
@@ -235,11 +235,11 @@ static bool runIPSCCP(
     // nodes in executable blocks we found values for. The function's entry
     // block is not part of BlocksToErase, so we have to handle it separately.
     for (BasicBlock *BB : BlocksToErase) {
-      NumInstRemoved += changeToUnreachable(BB->getFirstNonPHIOrDbg(),
+      NumInstRemoved += changeToUnreachable(&*BB->getFirstNonPHIOrDbg(),
                                             /*PreserveLCSSA=*/false, &DTU);
     }
     if (!Solver.isBlockExecutable(&F.front()))
-      NumInstRemoved += changeToUnreachable(F.front().getFirstNonPHIOrDbg(),
+      NumInstRemoved += changeToUnreachable(&*F.front().getFirstNonPHIOrDbg(),
                                             /*PreserveLCSSA=*/false, &DTU);
 
     BasicBlock *NewUnreachableBB = nullptr;
diff --git a/llvm/lib/Transforms/IPO/SampleProfile.cpp b/llvm/lib/Transforms/IPO/SampleProfile.cpp
index b978c54ef96fd..731ee7edb48c8 100644
--- a/llvm/lib/Transforms/IPO/SampleProfile.cpp
+++ b/llvm/lib/Transforms/IPO/SampleProfile.cpp
@@ -1747,7 +1747,7 @@ void SampleProfileLoader::generateMDProfMetadata(Function &F) {
       if (Weight != 0) {
         if (Weight > MaxWeight) {
           MaxWeight = Weight;
-          MaxDestInst = Succ->getFirstNonPHIOrDbgOrLifetime();
+          MaxDestInst = &*Succ->getFirstNonPHIOrDbgOrLifetime();
         }
       }
     }
@@ -2045,9 +2045,11 @@ bool SampleProfileLoader::doInitialization(Module &M,
     // which is currently only available for pseudo-probe mode. Removing the
     // checksum check could cause regressions for some cases, so further tuning
     // might be needed if we want to enable it for all cases.
-    if (Reader->profileIsProbeBased() &&
-        !SalvageStaleProfile.getNumOccurrences()) {
-      SalvageStaleProfile = true;
+    if (Reader->profileIsProbeBased()) {
+      if (!SalvageStaleProfile.getNumOccurrences())
+        SalvageStaleProfile = true;
+      if (!SalvageUnusedProfile.getNumOccurrences())
+        SalvageUnusedProfile = true;
     }
 
     if (!Reader->profileIsCS()) {
diff --git a/llvm/lib/Transforms/IPO/SampleProfileMatcher.cpp b/llvm/lib/Transforms/IPO/SampleProfileMatcher.cpp
index a652577c18900..313a50477a314 100644
--- a/llvm/lib/Transforms/IPO/SampleProfileMatcher.cpp
+++ b/llvm/lib/Transforms/IPO/SampleProfileMatcher.cpp
@@ -38,7 +38,7 @@ static cl::opt<unsigned> MinCallCountForCGMatching(
              "run stale profile call graph matching."));
 
 static cl::opt<bool> LoadFuncProfileforCGMatching(
-    "load-func-profile-for-cg-matching", cl::Hidden, cl::init(false),
+    "load-func-profile-for-cg-matching", cl::Hidden, cl::init(true),
     cl::desc(
         "Load top-level profiles that the sample reader initially skipped for "
         "the call-graph matching (only meaningful for extended binary "
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineAndOrXor.cpp b/llvm/lib/Transforms/InstCombine/InstCombineAndOrXor.cpp
index ca8a20b4b7312..4616ea6ab5487 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineAndOrXor.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineAndOrXor.cpp
@@ -610,8 +610,13 @@ static Value *foldLogOpOfMaskedICmps(Value *LHS, Value *RHS, bool IsAnd,
       APInt NewMask = *ConstB & *ConstD;
       if (NewMask == *ConstB)
         return LHS;
-      if (NewMask == *ConstD)
+      if (NewMask == *ConstD) {
+        if (IsLogical) {
+          if (auto *RHSI = dyn_cast<Instruction>(RHS))
+            RHSI->dropPoisonGeneratingFlags();
+        }
         return RHS;
+      }
     }
 
     if (Mask & AMask_NotAllOnes) {
@@ -4150,9 +4155,6 @@ Value *InstCombinerImpl::foldXorOfICmps(ICmpInst *LHS, ICmpInst *RHS,
     }
   }
 
-  // TODO: This can be generalized to compares of non-signbits using
-  // decomposeBitTestICmp(). It could be enhanced more by using (something like)
-  // foldLogOpOfMaskedICmps().
   const APInt *LC, *RC;
   if (match(LHS1, m_APInt(LC)) && match(RHS1, m_APInt(RC)) &&
       LHS0->getType() == RHS0->getType() &&
@@ -4200,6 +4202,21 @@ Value *InstCombinerImpl::foldXorOfICmps(ICmpInst *LHS, ICmpInst *RHS,
           }
         }
     }
+
+    // Fold (icmp eq/ne (X & Pow2), 0) ^ (icmp eq/ne (Y & Pow2), 0) into
+    // (icmp eq/ne ((X ^ Y) & Pow2), 0)
+    Value *X, *Y, *Pow2;
+    if (ICmpInst::isEquality(PredL) && ICmpInst::isEquality(PredR) &&
+        LC->isZero() && RC->isZero() && LHS->hasOneUse() && RHS->hasOneUse() &&
+        match(LHS0, m_And(m_Value(X), m_Value(Pow2))) &&
+        match(RHS0, m_And(m_Value(Y), m_Specific(Pow2))) &&
+        isKnownToBeAPowerOfTwo(Pow2, /*OrZero=*/true, /*Depth=*/0, &I)) {
+      Value *Xor = Builder.CreateXor(X, Y);
+      Value *And = Builder.CreateAnd(Xor, Pow2);
+      return Builder.CreateICmp(PredL == PredR ? ICmpInst::ICMP_NE
+                                               : ICmpInst::ICMP_EQ,
+                                And, ConstantInt::getNullValue(Xor->getType()));
+    }
   }
 
   // Instead of trying to imitate the folds for and/or, decompose this 'xor'
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp b/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp
index f748f78524e0d..400ebcf493713 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp
@@ -2752,7 +2752,8 @@ Instruction *InstCombinerImpl::visitCallInst(CallInst &CI) {
 
     if (match(Arg, m_Select(m_Value(Cond), m_Value(TVal), m_Value(FVal)))) {
       // fabs (select Cond, TrueC, FalseC) --> select Cond, AbsT, AbsF
-      if (isa<Constant>(TVal) || isa<Constant>(FVal)) {
+      if (Arg->hasOneUse() ? (isa<Constant>(TVal) || isa<Constant>(FVal))
+                           : (isa<Constant>(TVal) && isa<Constant>(FVal))) {
         CallInst *AbsT = Builder.CreateCall(II->getCalledFunction(), {TVal});
         CallInst *AbsF = Builder.CreateCall(II->getCalledFunction(), {FVal});
         SelectInst *SI = SelectInst::Create(Cond, AbsT, AbsF);
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp b/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp
index 5a4791870ac77..0feb6160b68fb 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp
@@ -37,11 +37,10 @@ using namespace PatternMatch;
 // How many times is a select replaced by one of its operands?
 STATISTIC(NumSel, "Number of select opts");
 
-
 /// Compute Result = In1+In2, returning true if the result overflowed for this
 /// type.
-static bool addWithOverflow(APInt &Result, const APInt &In1,
-                            const APInt &In2, bool IsSigned = false) {
+static bool addWithOverflow(APInt &Result, const APInt &In1, const APInt &In2,
+                            bool IsSigned = false) {
   bool Overflow;
   if (IsSigned)
     Result = In1.sadd_ov(In2, Overflow);
@@ -53,8 +52,8 @@ static bool addWithOverflow(APInt &Result, const APInt &In1,
 
 /// Compute Result = In1-In2, returning true if the result overflowed for this
 /// type.
-static bool subWithOverflow(APInt &Result, const APInt &In1,
-                            const APInt &In2, bool IsSigned = false) {
+static bool subWithOverflow(APInt &Result, const APInt &In1, const APInt &In2,
+                            bool IsSigned = false) {
   bool Overflow;
   if (IsSigned)
     Result = In1.ssub_ov(In2, Overflow);
@@ -810,15 +809,15 @@ Instruction *InstCombinerImpl::foldGEPICmp(GEPOperator *GEPLHS, Value *RHS,
     if (GEPLHS->getNumOperands() == GEPRHS->getNumOperands() &&
         GEPLHS->getSourceElementType() == GEPRHS->getSourceElementType()) {
       // If the GEPs only differ by one index, compare it.
-      unsigned NumDifferences = 0;  // Keep track of # differences.
-      unsigned DiffOperand = 0;     // The operand that differs.
+      unsigned NumDifferences = 0; // Keep track of # differences.
+      unsigned DiffOperand = 0;    // The operand that differs.
       for (unsigned i = 1, e = GEPRHS->getNumOperands(); i != e; ++i)
         if (GEPLHS->getOperand(i) != GEPRHS->getOperand(i)) {
           Type *LHSType = GEPLHS->getOperand(i)->getType();
           Type *RHSType = GEPRHS->getOperand(i)->getType();
           // FIXME: Better support for vector of pointers.
           if (LHSType->getPrimitiveSizeInBits() !=
-                   RHSType->getPrimitiveSizeInBits() ||
+                  RHSType->getPrimitiveSizeInBits() ||
               (GEPLHS->getType()->isVectorTy() &&
                (!LHSType->isVectorTy() || !RHSType->isVectorTy()))) {
             // Irreconcilable differences.
@@ -826,15 +825,18 @@ Instruction *InstCombinerImpl::foldGEPICmp(GEPOperator *GEPLHS, Value *RHS,
             break;
           }
 
-          if (NumDifferences++) break;
+          if (NumDifferences++)
+            break;
           DiffOperand = i;
         }
 
-      if (NumDifferences == 0)   // SAME GEP?
-        return replaceInstUsesWith(I, // No comparison is needed here.
-          ConstantInt::get(I.getType(), ICmpInst::isTrueWhenEqual(Cond)));
-
-      else if (NumDifferences == 1 && CanFold(NW)) {
+      if (NumDifferences == 0) // SAME GEP?
+        return replaceInstUsesWith(
+            I, // No comparison is needed here.
+            ConstantInt::get(I.getType(), ICmpInst::isTrueWhenEqual(Cond)));
+      // If two GEPs only differ by an index, compare them.
+      // Note that nowrap flags are always needed when comparing two indices.
+      else if (NumDifferences == 1 && NW != GEPNoWrapFlags::none()) {
         Value *LHSV = GEPLHS->getOperand(DiffOperand);
         Value *RHSV = GEPRHS->getOperand(DiffOperand);
         return NewICmp(NW, LHSV, RHSV);
@@ -910,8 +912,8 @@ bool InstCombinerImpl::foldAllocaCmp(AllocaInst *Alloca) {
     case 2: {
       // The alloca is only used in one icmp operand. Assume that the
       // equality is false.
-      auto *Res = ConstantInt::get(
-          ICmp->getType(), ICmp->getPredicate() == ICmpInst::ICMP_NE);
+      auto *Res = ConstantInt::get(ICmp->getType(),
+                                   ICmp->getPredicate() == ICmpInst::ICMP_NE);
       replaceInstUsesWith(*ICmp, Res);
       eraseInstFromFunction(*ICmp);
       Changed = true;
@@ -942,8 +944,8 @@ Instruction *InstCombinerImpl::foldICmpAddOpConst(Value *X, const APInt &C,
   // (X+2) <u X        --> X >u (MAXUINT-2)        --> X > 253
   // (X+MAXUINT) <u X  --> X >u (MAXUINT-MAXUINT)  --> X != 0
   if (Pred == ICmpInst::ICMP_ULT || Pred == ICmpInst::ICMP_ULE) {
-    Constant *R = ConstantInt::get(X->getType(),
-                                   APInt::getMaxValue(C.getBitWidth()) - C);
+    Constant *R =
+        ConstantInt::get(X->getType(), APInt::getMaxValue(C.getBitWidth()) - C);
     return new ICmpInst(ICmpInst::ICMP_UGT, X, R);
   }
 
@@ -1738,8 +1740,8 @@ Instruction *InstCombinerImpl::foldICmpAndShift(ICmpInst &Cmp,
     } else {
       Value *NewAnd = Builder.CreateAnd(
           Shift->getOperand(0), ConstantInt::get(And->getType(), NewAndCst));
-      return new ICmpInst(Cmp.getPredicate(),
-          NewAnd, ConstantInt::get(And->getType(), NewCmpCst));
+      return new ICmpInst(Cmp.getPredicate(), NewAnd,
+                          ConstantInt::get(And->getType(), NewCmpCst));
     }
   }
 
@@ -1882,17 +1884,23 @@ Instruction *InstCombinerImpl::foldICmpAndConstConst(ICmpInst &Cmp,
   // llvm.is.fpclass(X, fcInf|fcNan)
   // (icmp ne (and (bitcast X to int), ExponentMask), ExponentMask) -->
   // llvm.is.fpclass(X, ~(fcInf|fcNan))
+  // (icmp eq (and (bitcast X to int), ExponentMask), 0) -->
+  // llvm.is.fpclass(X, fcSubnormal|fcZero)
+  // (icmp ne (and (bitcast X to int), ExponentMask), 0) -->
+  // llvm.is.fpclass(X, ~(fcSubnormal|fcZero))
   Value *V;
   if (!Cmp.getParent()->getParent()->hasFnAttribute(
           Attribute::NoImplicitFloat) &&
       Cmp.isEquality() &&
       match(X, m_OneUse(m_ElementWiseBitCast(m_Value(V))))) {
     Type *FPType = V->getType()->getScalarType();
-    if (FPType->isIEEELikeFPTy() && C1 == *C2) {
+    if (FPType->isIEEELikeFPTy() && (C1.isZero() || C1 == *C2)) {
       APInt ExponentMask =
           APFloat::getInf(FPType->getFltSemantics()).bitcastToAPInt();
-      if (C1 == ExponentMask) {
-        unsigned Mask = FPClassTest::fcNan | FPClassTest::fcInf;
+      if (*C2 == ExponentMask) {
+        unsigned Mask = C1.isZero()
+                            ? FPClassTest::fcZero | FPClassTest::fcSubnormal
+                            : FPClassTest::fcNan | FPClassTest::fcInf;
         if (isICMP_NE)
           Mask = ~Mask & fcAllFlags;
         return replaceInstUsesWith(Cmp, Builder.createIsFPClass(V, Mask));
@@ -2012,13 +2020,12 @@ Instruction *InstCombinerImpl::foldICmpAndConstant(ICmpInst &Cmp,
     Value *A;
     const APInt *Addend, *Msk;
     if (match(And, m_And(m_OneUse(m_Add(m_Value(A), m_APInt(Addend))),
-                         m_APInt(Msk))) &&
-        Msk->isMask() && C.ule(*Msk)) {
+                         m_LowBitMask(Msk))) &&
+        C.ule(*Msk)) {
       APInt NewComperand = (C - *Addend) & *Msk;
-      Value* MaskA = Builder.CreateAnd(A, ConstantInt::get(A->getType(), *Msk));
-      return new ICmpInst(
-          Pred, MaskA,
-          Constant::getIntegerValue(MaskA->getType(), NewComperand));
+      Value *MaskA = Builder.CreateAnd(A, ConstantInt::get(A->getType(), *Msk));
+      return new ICmpInst(Pred, MaskA,
+                          ConstantInt::get(MaskA->getType(), NewComperand));
     }
   }
 
@@ -2319,7 +2326,7 @@ static Instruction *foldICmpShlLHSC(ICmpInst &Cmp, Instruction *Shl,
     // (1 << Y) <  1 -> Y == 31
     // (1 << Y) <  C -> Y == 31 if C is negative and not signed min.
     // Exclude signed min by subtracting 1 and lower the upper bound to 0.
-    if (Pred == ICmpInst::ICMP_SLT && (C-1).sle(0))
+    if (Pred == ICmpInst::ICMP_SLT && (C - 1).sle(0))
       return new ICmpInst(ICmpInst::ICMP_EQ, Y, BitWidthMinusOne);
   }
 
@@ -2804,7 +2811,7 @@ Instruction *InstCombinerImpl::foldICmpDivConstant(ICmpInst &Cmp,
   // (X s/ Y) == SMIN --> (X == SMIN) && (Y == 1)
   // (X s/ Y) != SMIN --> (X != SMIN) || (Y != 1)
   if (Cmp.isEquality() && Div->hasOneUse() && C.isSignBitSet() &&
-      (!DivIsSigned || C.isMinSignedValue()))   {
+      (!DivIsSigned || C.isMinSignedValue())) {
     Value *XBig = Builder.CreateICmp(Pred, X, ConstantInt::get(Ty, C));
     Value *YOne = Builder.CreateICmp(Pred, Y, ConstantInt::get(Ty, 1));
     auto Logic = Pred == ICmpInst::ICMP_EQ ? Instruction::And : Instruction::Or;
@@ -3357,14 +3364,14 @@ Instruction *InstCombinerImpl::foldICmpSelectConstant(ICmpInst &Cmp,
     // the entire original Cmp can be simplified to a false.
     Value *Cond = Builder.getFalse();
     if (TrueWhenLessThan)
-      Cond = Builder.CreateOr(Cond, Builder.CreateICmp(ICmpInst::ICMP_SLT,
-                                                       OrigLHS, OrigRHS));
+      Cond = Builder.CreateOr(
+          Cond, Builder.CreateICmp(ICmpInst::ICMP_SLT, OrigLHS, OrigRHS));
     if (TrueWhenEqual)
-      Cond = Builder.CreateOr(Cond, Builder.CreateICmp(ICmpInst::ICMP_EQ,
-                                                       OrigLHS, OrigRHS));
+      Cond = Builder.CreateOr(
+          Cond, Builder.CreateICmp(ICmpInst::ICMP_EQ, OrigLHS, OrigRHS));
     if (TrueWhenGreaterThan)
-      Cond = Builder.CreateOr(Cond, Builder.CreateICmp(ICmpInst::ICMP_SGT,
-                                                       OrigLHS, OrigRHS));
+      Cond = Builder.CreateOr(
+          Cond, Builder.CreateICmp(ICmpInst::ICMP_SGT, OrigLHS, OrigRHS));
 
     return replaceInstUsesWith(Cmp, Cond);
   }
@@ -3806,8 +3813,8 @@ Instruction *InstCombinerImpl::foldICmpEqIntrinsicWithConstant(
       APInt Mask1 = IsTrailing ? APInt::getLowBitsSet(BitWidth, Num + 1)
                                : APInt::getHighBitsSet(BitWidth, Num + 1);
       APInt Mask2 = IsTrailing
-        ? APInt::getOneBitSet(BitWidth, Num)
-        : APInt::getOneBitSet(BitWidth, BitWidth - Num - 1);
+                        ? APInt::getOneBitSet(BitWidth, Num)
+                        : APInt::getOneBitSet(BitWidth, BitWidth - Num - 1);
       return new ICmpInst(Pred, Builder.CreateAnd(II->getArgOperand(0), Mask1),
                           ConstantInt::get(Ty, Mask2));
     }
@@ -4867,18 +4874,17 @@ Value *InstCombinerImpl::foldMultiplicationOverflowCheck(ICmpInst &I) {
       return nullptr; // Wrong predicate.
     }
   } else // Look for: ((x * y) / x) !=/== y
-      if (I.isEquality() &&
-          match(&I,
-                m_c_ICmp(Pred, m_Value(Y),
-                         m_CombineAnd(
-                             m_OneUse(m_IDiv(m_CombineAnd(m_c_Mul(m_Deferred(Y),
-                                                                  m_Value(X)),
-                                                          m_Instruction(Mul)),
-                                             m_Deferred(X))),
-                             m_Instruction(Div))))) {
-    NeedNegation = Pred == ICmpInst::Predicate::ICMP_EQ;
-  } else
-    return nullptr;
+    if (I.isEquality() &&
+        match(&I, m_c_ICmp(Pred, m_Value(Y),
+                           m_CombineAnd(m_OneUse(m_IDiv(
+                                            m_CombineAnd(m_c_Mul(m_Deferred(Y),
+                                                                 m_Value(X)),
+                                                         m_Instruction(Mul)),
+                                            m_Deferred(X))),
+                                        m_Instruction(Div))))) {
+      NeedNegation = Pred == ICmpInst::Predicate::ICMP_EQ;
+    } else
+      return nullptr;
 
   BuilderTy::InsertPointGuard Guard(Builder);
   // If the pattern included (x * y), we'll want to insert new instructions
@@ -5059,6 +5065,27 @@ static Instruction *foldICmpXorXX(ICmpInst &I, const SimplifyQuery &Q,
   if (PredOut != Pred && isKnownNonZero(A, Q))
     return new ICmpInst(PredOut, Op0, Op1);
 
+  // These transform work when A is negative.
+  // X s< X^A, X s<= X^A, X u> X^A, X u>= X^A  --> X s< 0
+  // X s> X^A, X s>= X^A, X u< X^A, X u<= X^A  --> X s>= 0
+  if (match(A, m_Negative())) {
+    CmpInst::Predicate NewPred;
+    switch (ICmpInst::getStrictPredicate(Pred)) {
+    default:
+      return nullptr;
+    case ICmpInst::ICMP_SLT:
+    case ICmpInst::ICMP_UGT:
+      NewPred = ICmpInst::ICMP_SLT;
+      break;
+    case ICmpInst::ICMP_SGT:
+    case ICmpInst::ICMP_ULT:
+      NewPred = ICmpInst::ICMP_SGE;
+      break;
+    }
+    Constant *Const = Constant::getNullValue(Op0->getType());
+    return new ICmpInst(NewPred, Op0, Const);
+  }
+
   return nullptr;
 }
 
@@ -5377,7 +5404,7 @@ Instruction *InstCombinerImpl::foldICmpBinOp(ICmpInst &I,
                               ConstantExpr::getNeg(RHSC));
   }
 
-  if (Instruction * R = foldICmpXorXX(I, Q, *this))
+  if (Instruction *R = foldICmpXorXX(I, Q, *this))
     return R;
   if (Instruction *R = foldICmpOrXX(I, Q, *this))
     return R;
@@ -5412,7 +5439,7 @@ Instruction *InstCombinerImpl::foldICmpBinOp(ICmpInst &I,
         if (ICmpInst::isEquality(Pred)) {
           // If X != Y, fold (X *nw Z) eq/ne (Y *nw Z) -> Z eq/ne 0
           if (((Op0HasNSW && Op1HasNSW) || (Op0HasNUW && Op1HasNUW)) &&
-              isKnownNonEqual(X, Y, DL, &AC, &I, &DT))
+              isKnownNonEqual(X, Y, SQ))
             return new ICmpInst(Pred, Z, Constant::getNullValue(Z->getType()));
 
           KnownBits ZKnown = computeKnownBits(Z, 0, &I);
@@ -5914,8 +5941,8 @@ Instruction *InstCombinerImpl::foldICmpEquality(ICmpInst &I) {
     if (ShAmt < TypeBits && ShAmt != 0) {
       Value *Xor = Builder.CreateXor(A, B, I.getName() + ".unshifted");
       APInt AndVal = APInt::getLowBitsSet(TypeBits, TypeBits - ShAmt);
-      Value *And = Builder.CreateAnd(Xor, Builder.getInt(AndVal),
-                                      I.getName() + ".mask");
+      Value *And =
+          Builder.CreateAnd(Xor, Builder.getInt(AndVal), I.getName() + ".mask");
       return new ICmpInst(Pred, And, Constant::getNullValue(Cst1->getType()));
     }
   }
@@ -5947,10 +5974,10 @@ Instruction *InstCombinerImpl::foldICmpEquality(ICmpInst &I) {
   if (Instruction *ICmp = foldICmpIntrinsicWithIntrinsic(I, Builder))
     return ICmp;
 
-  // Match icmp eq (trunc (lshr A, BW), (ashr (trunc A), BW-1)), which checks the
-  // top BW/2 + 1 bits are all the same. Create "A >=s INT_MIN && A <=s INT_MAX",
-  // which we generate as "icmp ult (add A, 2^(BW-1)), 2^BW" to skip a few steps
-  // of instcombine.
+  // Match icmp eq (trunc (lshr A, BW), (ashr (trunc A), BW-1)), which checks
+  // the top BW/2 + 1 bits are all the same. Create "A >=s INT_MIN && A <=s
+  // INT_MAX", which we generate as "icmp ult (add A, 2^(BW-1)), 2^BW" to skip a
+  // few steps of instcombine.
   unsigned BitWidth = Op0->getType()->getScalarSizeInBits();
   if (match(Op0, m_AShr(m_Trunc(m_Value(A)), m_SpecificInt(BitWidth - 1))) &&
       match(Op1, m_Trunc(m_LShr(m_Specific(A), m_SpecificInt(BitWidth)))) &&
@@ -6084,10 +6111,10 @@ Instruction *InstCombinerImpl::foldICmpWithZextOrSext(ICmpInst &ICmp) {
     bool IsZext1 = isa<ZExtInst>(ICmp.getOperand(1));
 
     if (IsZext0 != IsZext1) {
-        // If X and Y and both i1
-        // (icmp eq/ne (zext X) (sext Y))
-        //      eq -> (icmp eq (or X, Y), 0)
-        //      ne -> (icmp ne (or X, Y), 0)
+      // If X and Y and both i1
+      // (icmp eq/ne (zext X) (sext Y))
+      //      eq -> (icmp eq (or X, Y), 0)
+      //      ne -> (icmp ne (or X, Y), 0)
       if (ICmp.isEquality() && X->getType()->isIntOrIntVectorTy(1) &&
           Y->getType()->isIntOrIntVectorTy(1))
         return new ICmpInst(ICmp.getPredicate(), Builder.CreateOr(X, Y),
@@ -6247,16 +6274,17 @@ Instruction *InstCombinerImpl::foldICmpWithCastOp(ICmpInst &ICmp) {
   return foldICmpWithZextOrSext(ICmp);
 }
 
-static bool isNeutralValue(Instruction::BinaryOps BinaryOp, Value *RHS, bool IsSigned) {
+static bool isNeutralValue(Instruction::BinaryOps BinaryOp, Value *RHS,
+                           bool IsSigned) {
   switch (BinaryOp) {
-    default:
-      llvm_unreachable("Unsupported binary op");
-    case Instruction::Add:
-    case Instruction::Sub:
-      return match(RHS, m_Zero());
-    case Instruction::Mul:
-      return !(RHS->getType()->isIntOrIntVectorTy(1) && IsSigned) &&
-             match(RHS, m_One());
+  default:
+    llvm_unreachable("Unsupported binary op");
+  case Instruction::Add:
+  case Instruction::Sub:
+    return match(RHS, m_Zero());
+  case Instruction::Mul:
+    return !(RHS->getType()->isIntOrIntVectorTy(1) && IsSigned) &&
+           match(RHS, m_One());
   }
 }
 
@@ -6265,23 +6293,23 @@ InstCombinerImpl::computeOverflow(Instruction::BinaryOps BinaryOp,
                                   bool IsSigned, Value *LHS, Value *RHS,
                                   Instruction *CxtI) const {
   switch (BinaryOp) {
-    default:
-      llvm_unreachable("Unsupported binary op");
-    case Instruction::Add:
-      if (IsSigned)
-        return computeOverflowForSignedAdd(LHS, RHS, CxtI);
-      else
-        return computeOverflowForUnsignedAdd(LHS, RHS, CxtI);
-    case Instruction::Sub:
-      if (IsSigned)
-        return computeOverflowForSignedSub(LHS, RHS, CxtI);
-      else
-        return computeOverflowForUnsignedSub(LHS, RHS, CxtI);
-    case Instruction::Mul:
-      if (IsSigned)
-        return computeOverflowForSignedMul(LHS, RHS, CxtI);
-      else
-        return computeOverflowForUnsignedMul(LHS, RHS, CxtI);
+  default:
+    llvm_unreachable("Unsupported binary op");
+  case Instruction::Add:
+    if (IsSigned)
+      return computeOverflowForSignedAdd(LHS, RHS, CxtI);
+    else
+      return computeOverflowForUnsignedAdd(LHS, RHS, CxtI);
+  case Instruction::Sub:
+    if (IsSigned)
+      return computeOverflowForSignedSub(LHS, RHS, CxtI);
+    else
+      return computeOverflowForUnsignedSub(LHS, RHS, CxtI);
+  case Instruction::Mul:
+    if (IsSigned)
+      return computeOverflowForSignedMul(LHS, RHS, CxtI);
+    else
+      return computeOverflowForUnsignedMul(LHS, RHS, CxtI);
   }
 }
 
@@ -6310,25 +6338,25 @@ bool InstCombinerImpl::OptimizeOverflowCheck(Instruction::BinaryOps BinaryOp,
   }
 
   switch (computeOverflow(BinaryOp, IsSigned, LHS, RHS, &OrigI)) {
-    case OverflowResult::MayOverflow:
-      return false;
-    case OverflowResult::AlwaysOverflowsLow:
-    case OverflowResult::AlwaysOverflowsHigh:
-      Result = Builder.CreateBinOp(BinaryOp, LHS, RHS);
-      Result->takeName(&OrigI);
-      Overflow = ConstantInt::getTrue(OverflowTy);
-      return true;
-    case OverflowResult::NeverOverflows:
-      Result = Builder.CreateBinOp(BinaryOp, LHS, RHS);
-      Result->takeName(&OrigI);
-      Overflow = ConstantInt::getFalse(OverflowTy);
-      if (auto *Inst = dyn_cast<Instruction>(Result)) {
-        if (IsSigned)
-          Inst->setHasNoSignedWrap();
-        else
-          Inst->setHasNoUnsignedWrap();
-      }
-      return true;
+  case OverflowResult::MayOverflow:
+    return false;
+  case OverflowResult::AlwaysOverflowsLow:
+  case OverflowResult::AlwaysOverflowsHigh:
+    Result = Builder.CreateBinOp(BinaryOp, LHS, RHS);
+    Result->takeName(&OrigI);
+    Overflow = ConstantInt::getTrue(OverflowTy);
+    return true;
+  case OverflowResult::NeverOverflows:
+    Result = Builder.CreateBinOp(BinaryOp, LHS, RHS);
+    Result->takeName(&OrigI);
+    Overflow = ConstantInt::getFalse(OverflowTy);
+    if (auto *Inst = dyn_cast<Instruction>(Result)) {
+      if (IsSigned)
+        Inst->setHasNoSignedWrap();
+      else
+        Inst->setHasNoUnsignedWrap();
+    }
+    return true;
   }
 
   llvm_unreachable("Unexpected overflow result");
@@ -6993,21 +7021,21 @@ static Instruction *canonicalizeICmpBool(ICmpInst &I,
   // Cases not handled by InstSimplify are always 'not' of Op0.
   if (match(B, m_Zero())) {
     switch (I.getPredicate()) {
-      case CmpInst::ICMP_EQ:  // A ==   0 -> !A
-      case CmpInst::ICMP_ULE: // A <=u  0 -> !A
-      case CmpInst::ICMP_SGE: // A >=s  0 -> !A
-        return BinaryOperator::CreateNot(A);
-      default:
-        llvm_unreachable("ICmp i1 X, C not simplified as expected.");
+    case CmpInst::ICMP_EQ:  // A ==   0 -> !A
+    case CmpInst::ICMP_ULE: // A <=u  0 -> !A
+    case CmpInst::ICMP_SGE: // A >=s  0 -> !A
+      return BinaryOperator::CreateNot(A);
+    default:
+      llvm_unreachable("ICmp i1 X, C not simplified as expected.");
     }
   } else if (match(B, m_One())) {
     switch (I.getPredicate()) {
-      case CmpInst::ICMP_NE:  // A !=  1 -> !A
-      case CmpInst::ICMP_ULT: // A <u  1 -> !A
-      case CmpInst::ICMP_SGT: // A >s -1 -> !A
-        return BinaryOperator::CreateNot(A);
-      default:
-        llvm_unreachable("ICmp i1 X, C not simplified as expected.");
+    case CmpInst::ICMP_NE:  // A !=  1 -> !A
+    case CmpInst::ICMP_ULT: // A <u  1 -> !A
+    case CmpInst::ICMP_SGT: // A >s -1 -> !A
+      return BinaryOperator::CreateNot(A);
+    default:
+      llvm_unreachable("ICmp i1 X, C not simplified as expected.");
     }
   }
 
@@ -7193,8 +7221,8 @@ static Instruction *foldICmpOfUAddOv(ICmpInst &I) {
     // extract(uadd.with.overflow(A, 1), 0) == 0
     // extract(uadd.with.overflow(A, -1), 0) != -1
     UAddOv = cast<ExtractValueInst>(Op0)->getAggregateOperand();
-  else if (match(Op1, UAddOvResultPat) &&
-           Pred == ICmpInst::ICMP_UGT && (Op0 == A || Op0 == B))
+  else if (match(Op1, UAddOvResultPat) && Pred == ICmpInst::ICMP_UGT &&
+           (Op0 == A || Op0 == B))
     // A > extract(uadd.with.overflow(A, B), 0)
     UAddOv = cast<ExtractValueInst>(Op1)->getAggregateOperand();
   else
@@ -7410,8 +7438,7 @@ Instruction *InstCombinerImpl::visitICmpInst(ICmpInst &I) {
       if (Value *V = dyn_castNegVal(SelectTrue)) {
         if (V == SelectFalse)
           return CmpInst::Create(Instruction::ICmp, I.getPredicate(), V, Op1);
-      }
-      else if (Value *V = dyn_castNegVal(SelectFalse)) {
+      } else if (Value *V = dyn_castNegVal(SelectFalse)) {
         if (V == SelectTrue)
           return CmpInst::Create(Instruction::ICmp, I.getPredicate(), V, Op1);
       }
@@ -7552,7 +7579,8 @@ Instruction *InstCombinerImpl::visitICmpInst(ICmpInst &I) {
 
   // Try to optimize equality comparisons against alloca-based pointers.
   if (Op0->getType()->isPointerTy() && I.isEquality()) {
-    assert(Op1->getType()->isPointerTy() && "Comparing pointer with non-pointer?");
+    assert(Op1->getType()->isPointerTy() &&
+           "Comparing pointer with non-pointer?");
     if (auto *Alloca = dyn_cast<AllocaInst>(getUnderlyingObject(Op0)))
       if (foldAllocaCmp(Alloca))
         return nullptr;
@@ -7722,7 +7750,8 @@ Instruction *InstCombinerImpl::foldFCmpIntToFPConst(FCmpInst &I,
   // Get the width of the mantissa.  We don't want to hack on conversions that
   // might lose information from the integer, e.g. "i64 -> float"
   int MantissaWidth = LHSI->getType()->getFPMantissaWidth();
-  if (MantissaWidth == -1) return nullptr;  // Unknown.
+  if (MantissaWidth == -1)
+    return nullptr; // Unknown.
 
   Type *IntTy = LHSI->getOperand(0)->getType();
   unsigned IntWidth = IntTy->getScalarSizeInBits();
@@ -7784,7 +7813,8 @@ Instruction *InstCombinerImpl::foldFCmpIntToFPConst(FCmpInst &I,
 
   ICmpInst::Predicate Pred;
   switch (I.getPredicate()) {
-  default: llvm_unreachable("Unexpected predicate!");
+  default:
+    llvm_unreachable("Unexpected predicate!");
   case FCmpInst::FCMP_UEQ:
   case FCmpInst::FCMP_OEQ:
     Pred = ICmpInst::ICMP_EQ;
@@ -7826,7 +7856,7 @@ Instruction *InstCombinerImpl::foldFCmpIntToFPConst(FCmpInst &I,
     SMax.convertFromAPInt(APInt::getSignedMaxValue(IntWidth), true,
                           APFloat::rmNearestTiesToEven);
     if (SMax < *RHS) { // smax < 13123.0
-      if (Pred == ICmpInst::ICMP_NE  || Pred == ICmpInst::ICMP_SLT ||
+      if (Pred == ICmpInst::ICMP_NE || Pred == ICmpInst::ICMP_SLT ||
           Pred == ICmpInst::ICMP_SLE)
         return replaceInstUsesWith(I, ConstantInt::getTrue(I.getType()));
       return replaceInstUsesWith(I, ConstantInt::getFalse(I.getType()));
@@ -7838,7 +7868,7 @@ Instruction *InstCombinerImpl::foldFCmpIntToFPConst(FCmpInst &I,
     UMax.convertFromAPInt(APInt::getMaxValue(IntWidth), false,
                           APFloat::rmNearestTiesToEven);
     if (UMax < *RHS) { // umax < 13123.0
-      if (Pred == ICmpInst::ICMP_NE  || Pred == ICmpInst::ICMP_ULT ||
+      if (Pred == ICmpInst::ICMP_NE || Pred == ICmpInst::ICMP_ULT ||
           Pred == ICmpInst::ICMP_ULE)
         return replaceInstUsesWith(I, ConstantInt::getTrue(I.getType()));
       return replaceInstUsesWith(I, ConstantInt::getFalse(I.getType()));
@@ -7882,10 +7912,11 @@ Instruction *InstCombinerImpl::foldFCmpIntToFPConst(FCmpInst &I,
       // the compare predicate and sometimes the value.  RHSC is rounded towards
       // zero at this point.
       switch (Pred) {
-      default: llvm_unreachable("Unexpected integer comparison!");
-      case ICmpInst::ICMP_NE:  // (float)int != 4.4   --> true
+      default:
+        llvm_unreachable("Unexpected integer comparison!");
+      case ICmpInst::ICMP_NE: // (float)int != 4.4   --> true
         return replaceInstUsesWith(I, ConstantInt::getTrue(I.getType()));
-      case ICmpInst::ICMP_EQ:  // (float)int == 4.4   --> false
+      case ICmpInst::ICMP_EQ: // (float)int == 4.4   --> false
         return replaceInstUsesWith(I, ConstantInt::getFalse(I.getType()));
       case ICmpInst::ICMP_ULE:
         // (float)int <= 4.4   --> int <= 4
@@ -8313,20 +8344,21 @@ Instruction *InstCombinerImpl::visitFCmpInst(FCmpInst &I) {
   assert(OpType == Op1->getType() && "fcmp with different-typed operands?");
   if (Op0 == Op1) {
     switch (Pred) {
-      default: break;
-    case FCmpInst::FCMP_UNO:    // True if unordered: isnan(X) | isnan(Y)
-    case FCmpInst::FCMP_ULT:    // True if unordered or less than
-    case FCmpInst::FCMP_UGT:    // True if unordered or greater than
-    case FCmpInst::FCMP_UNE:    // True if unordered or not equal
+    default:
+      break;
+    case FCmpInst::FCMP_UNO: // True if unordered: isnan(X) | isnan(Y)
+    case FCmpInst::FCMP_ULT: // True if unordered or less than
+    case FCmpInst::FCMP_UGT: // True if unordered or greater than
+    case FCmpInst::FCMP_UNE: // True if unordered or not equal
       // Canonicalize these to be 'fcmp uno %X, 0.0'.
       I.setPredicate(FCmpInst::FCMP_UNO);
       I.setOperand(1, Constant::getNullValue(OpType));
       return &I;
 
-    case FCmpInst::FCMP_ORD:    // True if ordered (no nans)
-    case FCmpInst::FCMP_OEQ:    // True if ordered and equal
-    case FCmpInst::FCMP_OGE:    // True if ordered and greater than or equal
-    case FCmpInst::FCMP_OLE:    // True if ordered and less than or equal
+    case FCmpInst::FCMP_ORD: // True if ordered (no nans)
+    case FCmpInst::FCMP_OEQ: // True if ordered and equal
+    case FCmpInst::FCMP_OGE: // True if ordered and greater than or equal
+    case FCmpInst::FCMP_OLE: // True if ordered and less than or equal
       // Canonicalize these to be 'fcmp ord %X, 0.0'.
       I.setPredicate(FCmpInst::FCMP_ORD);
       I.setOperand(1, Constant::getNullValue(OpType));
@@ -8484,7 +8516,7 @@ Instruction *InstCombinerImpl::visitFCmpInst(FCmpInst &I) {
                   cast<LoadInst>(LHSI), GEP, GV, I))
             return Res;
       break;
-  }
+    }
   }
 
   if (Instruction *R = foldFabsWithFcmpZero(I, *this))
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineLoadStoreAlloca.cpp b/llvm/lib/Transforms/InstCombine/InstCombineLoadStoreAlloca.cpp
index f80bbffbab547..4b42e86e25161 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineLoadStoreAlloca.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineLoadStoreAlloca.cpp
@@ -462,8 +462,8 @@ Instruction *InstCombinerImpl::visitAllocaInst(AllocaInst &AI) {
 
       // Get the first instruction in the entry block.
       BasicBlock &EntryBlock = AI.getParent()->getParent()->getEntryBlock();
-      Instruction *FirstInst = EntryBlock.getFirstNonPHIOrDbg();
-      if (FirstInst != &AI) {
+      BasicBlock::iterator FirstInst = EntryBlock.getFirstNonPHIOrDbg();
+      if (&*FirstInst != &AI) {
         // If the entry block doesn't start with a zero-size alloca then move
         // this one to the start of the entry block.  There is no problem with
         // dominance as the array size was forced to a constant earlier already.
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineMulDivRem.cpp b/llvm/lib/Transforms/InstCombine/InstCombineMulDivRem.cpp
index df5f9833a2ff9..c8bdf029dd71c 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineMulDivRem.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineMulDivRem.cpp
@@ -2044,7 +2044,7 @@ convertFSqrtDivIntoFMul(CallInst *CI, Instruction *X,
   // instructions in R2 and get the most common fpmath metadata and fast-math
   // flags on it.
   auto *FSqrt = cast<CallInst>(CI->clone());
-  FSqrt->insertBefore(CI);
+  FSqrt->insertBefore(CI->getIterator());
   auto *R2FPMathMDNode = (*R2.begin())->getMetadata(LLVMContext::MD_fpmath);
   FastMathFlags R2FMF = (*R2.begin())->getFastMathFlags(); // Common FMF
   for (Instruction *I : R2) {
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineSelect.cpp b/llvm/lib/Transforms/InstCombine/InstCombineSelect.cpp
index f66a976ccb47f..2e14145aef884 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineSelect.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineSelect.cpp
@@ -423,24 +423,34 @@ Instruction *InstCombinerImpl::foldSelectOpOp(SelectInst &SI, Instruction *TI,
       }
     }
 
+    auto CreateCmpSel = [&](std::optional<CmpPredicate> P,
+                            bool Swapped) -> CmpInst * {
+      if (!P)
+        return nullptr;
+      auto *MatchOp = getCommonOp(TI, FI, ICmpInst::isEquality(*P),
+                                  ICmpInst::isRelational(*P) && Swapped);
+      if (!MatchOp)
+        return nullptr;
+      Value *NewSel = Builder.CreateSelect(Cond, OtherOpT, OtherOpF,
+                                           SI.getName() + ".v", &SI);
+      return new ICmpInst(MatchIsOpZero ? *P
+                                        : ICmpInst::getSwappedCmpPredicate(*P),
+                          MatchOp, NewSel);
+    };
+
     // icmp with a common operand also can have the common operand
     // pulled after the select.
     CmpPredicate TPred, FPred;
     if (match(TI, m_ICmp(TPred, m_Value(), m_Value())) &&
         match(FI, m_ICmp(FPred, m_Value(), m_Value()))) {
-      // FIXME: Use CmpPredicate::getMatching here.
-      CmpInst::Predicate T = TPred, F = FPred;
-      if (T == F || T == ICmpInst::getSwappedCmpPredicate(F)) {
-        bool Swapped = T != F;
-        if (Value *MatchOp =
-                getCommonOp(TI, FI, ICmpInst::isEquality(TPred), Swapped)) {
-          Value *NewSel = Builder.CreateSelect(Cond, OtherOpT, OtherOpF,
-                                               SI.getName() + ".v", &SI);
-          return new ICmpInst(
-              MatchIsOpZero ? TPred : ICmpInst::getSwappedCmpPredicate(TPred),
-              MatchOp, NewSel);
-        }
-      }
+      if (auto *R =
+              CreateCmpSel(CmpPredicate::getMatching(TPred, FPred), false))
+        return R;
+      if (auto *R =
+              CreateCmpSel(CmpPredicate::getMatching(
+                               TPred, ICmpInst::getSwappedCmpPredicate(FPred)),
+                           true))
+        return R;
     }
   }
 
@@ -1761,26 +1771,38 @@ static Value *foldSelectInstWithICmpConst(SelectInst &SI, ICmpInst *ICI,
       return Builder.CreateBinaryIntrinsic(Intrinsic::smin, V, TVal);
   }
 
-  BinaryOperator *BO;
+  // Fold icmp(X) ? f(X) : C to f(X) when f(X) is guaranteed to be equal to C
+  // for all X in the exact range of the inverse predicate.
+  Instruction *Op;
   const APInt *C;
   CmpInst::Predicate CPred;
-  if (match(&SI, m_Select(m_Specific(ICI), m_APInt(C), m_BinOp(BO))))
+  if (match(&SI, m_Select(m_Specific(ICI), m_APInt(C), m_Instruction(Op))))
     CPred = ICI->getPredicate();
-  else if (match(&SI, m_Select(m_Specific(ICI), m_BinOp(BO), m_APInt(C))))
+  else if (match(&SI, m_Select(m_Specific(ICI), m_Instruction(Op), m_APInt(C))))
     CPred = ICI->getInversePredicate();
   else
     return nullptr;
 
-  const APInt *BinOpC;
-  if (!match(BO, m_BinOp(m_Specific(V), m_APInt(BinOpC))))
-    return nullptr;
-
-  ConstantRange R = ConstantRange::makeExactICmpRegion(CPred, *CmpC)
-                        .binaryOp(BO->getOpcode(), *BinOpC);
-  if (R == *C) {
-    BO->dropPoisonGeneratingFlags();
-    return BO;
+  ConstantRange InvDomCR = ConstantRange::makeExactICmpRegion(CPred, *CmpC);
+  const APInt *OpC;
+  if (match(Op, m_BinOp(m_Specific(V), m_APInt(OpC)))) {
+    ConstantRange R = InvDomCR.binaryOp(
+        static_cast<Instruction::BinaryOps>(Op->getOpcode()), *OpC);
+    if (R == *C) {
+      Op->dropPoisonGeneratingFlags();
+      return Op;
+    }
+  }
+  if (auto *MMI = dyn_cast<MinMaxIntrinsic>(Op);
+      MMI && MMI->getLHS() == V && match(MMI->getRHS(), m_APInt(OpC))) {
+    ConstantRange R = ConstantRange::intrinsic(MMI->getIntrinsicID(),
+                                               {InvDomCR, ConstantRange(*OpC)});
+    if (R == *C) {
+      MMI->dropPoisonGeneratingAnnotations();
+      return MMI;
+    }
   }
+
   return nullptr;
 }
 
@@ -2842,10 +2864,10 @@ static Instruction *foldSelectWithFCmpToFabs(SelectInst &SI,
     if (!match(TrueVal, m_FNeg(m_Specific(X))))
       return nullptr;
 
-    // Forward-propagate nnan and ninf from the fneg to the select.
+    // Forward-propagate nnan and ninf from the fcmp to the select.
     // If all inputs are not those values, then the select is not either.
     // Note: nsz is defined differently, so it may not be correct to propagate.
-    FastMathFlags FMF = cast<FPMathOperator>(TrueVal)->getFastMathFlags();
+    FastMathFlags FMF = cast<FPMathOperator>(CondVal)->getFastMathFlags();
     if (FMF.noNaNs() && !SI.hasNoNaNs()) {
       SI.setHasNoNaNs(true);
       ChangedFMF = true;
@@ -2854,6 +2876,13 @@ static Instruction *foldSelectWithFCmpToFabs(SelectInst &SI,
       SI.setHasNoInfs(true);
       ChangedFMF = true;
     }
+    // Forward-propagate nnan from the fneg to the select.
+    // The nnan flag can be propagated iff fneg is selected when X is NaN.
+    if (!SI.hasNoNaNs() && cast<FPMathOperator>(TrueVal)->hasNoNaNs() &&
+        (Swap ? FCmpInst::isOrdered(Pred) : FCmpInst::isUnordered(Pred))) {
+      SI.setHasNoNaNs(true);
+      ChangedFMF = true;
+    }
 
     // With nsz, when 'Swap' is false:
     // fold (X < +/-0.0) ? -X : X or (X <= +/-0.0) ? -X : X to fabs(X)
@@ -3403,7 +3432,7 @@ Instruction *InstCombinerImpl::foldSelectOfBools(SelectInst &SI) {
 static bool isSafeToRemoveBitCeilSelect(ICmpInst::Predicate Pred, Value *Cond0,
                                         const APInt *Cond1, Value *CtlzOp,
                                         unsigned BitWidth,
-                                        bool &ShouldDropNUW) {
+                                        bool &ShouldDropNoWrap) {
   // The challenge in recognizing std::bit_ceil(X) is that the operand is used
   // for the CTLZ proper and select condition, each possibly with some
   // operation like add and sub.
@@ -3426,7 +3455,7 @@ static bool isSafeToRemoveBitCeilSelect(ICmpInst::Predicate Pred, Value *Cond0,
   ConstantRange CR = ConstantRange::makeExactICmpRegion(
       CmpInst::getInversePredicate(Pred), *Cond1);
 
-  ShouldDropNUW = false;
+  ShouldDropNoWrap = false;
 
   // Match the operation that's used to compute CtlzOp from CommonAncestor.  If
   // CtlzOp == CommonAncestor, return true as no operation is needed.  If a
@@ -3437,11 +3466,12 @@ static bool isSafeToRemoveBitCeilSelect(ICmpInst::Predicate Pred, Value *Cond0,
     if (CtlzOp == CommonAncestor)
       return true;
     if (match(CtlzOp, m_Add(m_Specific(CommonAncestor), m_APInt(C)))) {
+      ShouldDropNoWrap = true;
       CR = CR.add(*C);
       return true;
     }
     if (match(CtlzOp, m_Sub(m_APInt(C), m_Specific(CommonAncestor)))) {
-      ShouldDropNUW = true;
+      ShouldDropNoWrap = true;
       CR = ConstantRange(*C).sub(CR);
       return true;
     }
@@ -3512,19 +3542,21 @@ static Instruction *foldBitCeil(SelectInst &SI, IRBuilderBase &Builder,
     Pred = CmpInst::getInversePredicate(Pred);
   }
 
-  bool ShouldDropNUW;
+  bool ShouldDropNoWrap;
 
   if (!match(FalseVal, m_One()) ||
       !match(TrueVal,
              m_OneUse(m_Shl(m_One(), m_OneUse(m_Sub(m_SpecificInt(BitWidth),
                                                     m_Value(Ctlz)))))) ||
-      !match(Ctlz, m_Intrinsic<Intrinsic::ctlz>(m_Value(CtlzOp), m_Zero())) ||
+      !match(Ctlz, m_Intrinsic<Intrinsic::ctlz>(m_Value(CtlzOp), m_Value())) ||
       !isSafeToRemoveBitCeilSelect(Pred, Cond0, Cond1, CtlzOp, BitWidth,
-                                   ShouldDropNUW))
+                                   ShouldDropNoWrap))
     return nullptr;
 
-  if (ShouldDropNUW)
+  if (ShouldDropNoWrap) {
     cast<Instruction>(CtlzOp)->setHasNoUnsignedWrap(false);
+    cast<Instruction>(CtlzOp)->setHasNoSignedWrap(false);
+  }
 
   // Build 1 << (-CTLZ & (BitWidth-1)).  The negation likely corresponds to a
   // single hardware instruction as opposed to BitWidth - CTLZ, where BitWidth
@@ -3533,6 +3565,8 @@ static Instruction *foldBitCeil(SelectInst &SI, IRBuilderBase &Builder,
 
   // Drop range attributes and re-infer them in the next iteration.
   cast<Instruction>(Ctlz)->dropPoisonGeneratingAnnotations();
+  // Set is_zero_poison to false and re-infer them in the next iteration.
+  cast<Instruction>(Ctlz)->setOperand(1, Builder.getFalse());
   IC.addToWorklist(cast<Instruction>(Ctlz));
   Value *Neg = Builder.CreateNeg(Ctlz);
   Value *Masked =
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineSimplifyDemanded.cpp b/llvm/lib/Transforms/InstCombine/InstCombineSimplifyDemanded.cpp
index 943598a30f040..2c8939b5a0514 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineSimplifyDemanded.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineSimplifyDemanded.cpp
@@ -1039,11 +1039,14 @@ Value *InstCombinerImpl::SimplifyDemandedUseBits(Instruction *I,
         APInt DemandedMaskLHS(DemandedMask.lshr(ShiftAmt));
         APInt DemandedMaskRHS(DemandedMask.shl(BitWidth - ShiftAmt));
         if (I->getOperand(0) != I->getOperand(1)) {
-          if (SimplifyDemandedBits(I, 0, DemandedMaskLHS, LHSKnown,
-                                   Depth + 1, Q) ||
+          if (SimplifyDemandedBits(I, 0, DemandedMaskLHS, LHSKnown, Depth + 1,
+                                   Q) ||
               SimplifyDemandedBits(I, 1, DemandedMaskRHS, RHSKnown, Depth + 1,
-                                   Q))
+                                   Q)) {
+            // Range attribute may no longer hold.
+            I->dropPoisonGeneratingReturnAttributes();
             return I;
+          }
         } else { // fshl is a rotate
           // Avoid converting rotate into funnel shift.
           // Only simplify if one operand is constant.
diff --git a/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp b/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp
index cad17c511b6d0..5621511570b58 100644
--- a/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp
@@ -33,6 +33,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "InstCombineInternal.h"
+#include "llvm/ADT/APFloat.h"
 #include "llvm/ADT/APInt.h"
 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/DenseMap.h"
@@ -732,7 +733,7 @@ static Value *tryFactorization(BinaryOperator &I, const SimplifyQuery &SQ,
   RetVal->takeName(&I);
 
   // Try to add no-overflow flags to the final value.
-  if (isa<OverflowingBinaryOperator>(RetVal)) {
+  if (isa<BinaryOperator>(RetVal)) {
     bool HasNSW = false;
     bool HasNUW = false;
     if (isa<OverflowingBinaryOperator>(&I)) {
@@ -4069,6 +4070,49 @@ InstCombinerImpl::foldExtractOfOverflowIntrinsic(ExtractValueInst &EV) {
   return nullptr;
 }
 
+static Value *foldFrexpOfSelect(ExtractValueInst &EV, IntrinsicInst *FrexpCall,
+                                SelectInst *SelectInst,
+                                InstCombiner::BuilderTy &Builder) {
+  // Helper to fold frexp of select to select of frexp.
+
+  if (!SelectInst->hasOneUse() || !FrexpCall->hasOneUse())
+    return nullptr;
+  Value *Cond = SelectInst->getCondition();
+  Value *TrueVal = SelectInst->getTrueValue();
+  Value *FalseVal = SelectInst->getFalseValue();
+
+  const APFloat *ConstVal = nullptr;
+  Value *VarOp = nullptr;
+  bool ConstIsTrue = false;
+
+  if (match(TrueVal, m_APFloat(ConstVal))) {
+    VarOp = FalseVal;
+    ConstIsTrue = true;
+  } else if (match(FalseVal, m_APFloat(ConstVal))) {
+    VarOp = TrueVal;
+    ConstIsTrue = false;
+  } else {
+    return nullptr;
+  }
+
+  Builder.SetInsertPoint(&EV);
+
+  CallInst *NewFrexp =
+      Builder.CreateCall(FrexpCall->getCalledFunction(), {VarOp}, "frexp");
+  NewFrexp->copyIRFlags(FrexpCall);
+
+  Value *NewEV = Builder.CreateExtractValue(NewFrexp, 0, "mantissa");
+
+  int Exp;
+  APFloat Mantissa = frexp(*ConstVal, Exp, APFloat::rmNearestTiesToEven);
+
+  Constant *ConstantMantissa = ConstantFP::get(TrueVal->getType(), Mantissa);
+
+  Value *NewSel = Builder.CreateSelectFMF(
+      Cond, ConstIsTrue ? ConstantMantissa : NewEV,
+      ConstIsTrue ? NewEV : ConstantMantissa, SelectInst, "select.frexp");
+  return NewSel;
+}
 Instruction *InstCombinerImpl::visitExtractValueInst(ExtractValueInst &EV) {
   Value *Agg = EV.getAggregateOperand();
 
@@ -4079,6 +4123,15 @@ Instruction *InstCombinerImpl::visitExtractValueInst(ExtractValueInst &EV) {
                                           SQ.getWithInstruction(&EV)))
     return replaceInstUsesWith(EV, V);
 
+  Value *Cond, *TrueVal, *FalseVal;
+  if (match(&EV, m_ExtractValue<0>(m_Intrinsic<Intrinsic::frexp>(m_Select(
+                     m_Value(Cond), m_Value(TrueVal), m_Value(FalseVal)))))) {
+    auto *SelInst =
+        cast<SelectInst>(cast<IntrinsicInst>(Agg)->getArgOperand(0));
+    if (Value *Result =
+            foldFrexpOfSelect(EV, cast<IntrinsicInst>(Agg), SelInst, Builder))
+      return replaceInstUsesWith(EV, Result);
+  }
   if (InsertValueInst *IV = dyn_cast<InsertValueInst>(Agg)) {
     // We're extracting from an insertvalue instruction, compare the indices
     const unsigned *exti, *exte, *insi, *inse;
diff --git a/llvm/lib/Transforms/Instrumentation/AddressSanitizer.cpp b/llvm/lib/Transforms/Instrumentation/AddressSanitizer.cpp
index e5f3b7f24bca7..bbe7040121649 100644
--- a/llvm/lib/Transforms/Instrumentation/AddressSanitizer.cpp
+++ b/llvm/lib/Transforms/Instrumentation/AddressSanitizer.cpp
@@ -1661,7 +1661,7 @@ void AddressSanitizer::instrumentMaskedLoadOrStore(
   if (Stride)
     Stride = IB.CreateZExtOrTrunc(Stride, IntptrTy);
 
-  SplitBlockAndInsertForEachLane(EVL, LoopInsertBefore,
+  SplitBlockAndInsertForEachLane(EVL, LoopInsertBefore->getIterator(),
                                  [&](IRBuilderBase &IRB, Value *Index) {
     Value *MaskElem = IRB.CreateExtractElement(Mask, Index);
     if (auto *MaskElemC = dyn_cast<ConstantInt>(MaskElem)) {
@@ -3414,7 +3414,7 @@ void FunctionStackPoisoner::processStaticAllocas() {
   assert(InsBeforeB == &F.getEntryBlock());
   for (auto *AI : StaticAllocasToMoveUp)
     if (AI->getParent() == InsBeforeB)
-      AI->moveBefore(InsBefore);
+      AI->moveBefore(InsBefore->getIterator());
 
   // Move stores of arguments into entry-block allocas as well. This prevents
   // extra stack slots from being generated (to house the argument values until
@@ -3423,10 +3423,11 @@ void FunctionStackPoisoner::processStaticAllocas() {
   SmallVector<Instruction *, 8> ArgInitInsts;
   findStoresToUninstrumentedArgAllocas(ASan, *InsBefore, ArgInitInsts);
   for (Instruction *ArgInitInst : ArgInitInsts)
-    ArgInitInst->moveBefore(InsBefore);
+    ArgInitInst->moveBefore(InsBefore->getIterator());
 
   // If we have a call to llvm.localescape, keep it in the entry block.
-  if (LocalEscapeCall) LocalEscapeCall->moveBefore(InsBefore);
+  if (LocalEscapeCall)
+    LocalEscapeCall->moveBefore(InsBefore->getIterator());
 
   SmallVector<ASanStackVariableDescription, 16> SVD;
   SVD.reserve(AllocaVec.size());
diff --git a/llvm/lib/Transforms/Instrumentation/BoundsChecking.cpp b/llvm/lib/Transforms/Instrumentation/BoundsChecking.cpp
index 609678f9979c6..14c331b3b748e 100644
--- a/llvm/lib/Transforms/Instrumentation/BoundsChecking.cpp
+++ b/llvm/lib/Transforms/Instrumentation/BoundsChecking.cpp
@@ -108,14 +108,17 @@ static Value *getBoundsCheckCond(Value *Ptr, Value *InstVal,
   return Or;
 }
 
-static CallInst *InsertTrap(BuilderTy &IRB, bool DebugTrapBB) {
+static CallInst *InsertTrap(BuilderTy &IRB, bool DebugTrapBB,
+                            std::optional<int8_t> GuardKind) {
   if (!DebugTrapBB)
     return IRB.CreateIntrinsic(Intrinsic::trap, {}, {});
-  // FIXME: Ideally we would use the SanitizerHandler::OutOfBounds constant.
+
   return IRB.CreateIntrinsic(
       Intrinsic::ubsantrap, {},
       ConstantInt::get(IRB.getInt8Ty(),
-                       IRB.GetInsertBlock()->getParent()->size()));
+                       GuardKind.has_value()
+                           ? GuardKind.value()
+                           : IRB.GetInsertBlock()->getParent()->size()));
 }
 
 static CallInst *InsertCall(BuilderTy &IRB, bool MayReturn, StringRef Name) {
@@ -250,7 +253,7 @@ static bool addBoundsChecking(Function &F, TargetLibraryInfo &TLI,
 
     bool DebugTrapBB = !Opts.Merge;
     CallInst *TrapCall = Opts.Rt ? InsertCall(IRB, Opts.Rt->MayReturn, Name)
-                                 : InsertTrap(IRB, DebugTrapBB);
+                                 : InsertTrap(IRB, DebugTrapBB, Opts.GuardKind);
     if (DebugTrapBB)
       TrapCall->addFnAttr(llvm::Attribute::NoMerge);
 
diff --git a/llvm/lib/Transforms/Instrumentation/HWAddressSanitizer.cpp b/llvm/lib/Transforms/Instrumentation/HWAddressSanitizer.cpp
index 75a19357ea1bb..645c102752692 100644
--- a/llvm/lib/Transforms/Instrumentation/HWAddressSanitizer.cpp
+++ b/llvm/lib/Transforms/Instrumentation/HWAddressSanitizer.cpp
@@ -1518,8 +1518,7 @@ bool HWAddressSanitizer::instrumentStack(memtag::StackInfo &SInfo,
 
     AI->replaceUsesWithIf(Replacement, [AICast, AILong](const Use &U) {
       auto *User = U.getUser();
-      return User != AILong && User != AICast &&
-             !memtag::isLifetimeIntrinsic(User);
+      return User != AILong && User != AICast && !isa<LifetimeIntrinsic>(User);
     });
 
     memtag::annotateDebugRecords(Info, retagMask(N));
diff --git a/llvm/lib/Transforms/Instrumentation/LowerAllowCheckPass.cpp b/llvm/lib/Transforms/Instrumentation/LowerAllowCheckPass.cpp
index f27798cfd228c..10e908ef73ce5 100644
--- a/llvm/lib/Transforms/Instrumentation/LowerAllowCheckPass.cpp
+++ b/llvm/lib/Transforms/Instrumentation/LowerAllowCheckPass.cpp
@@ -20,6 +20,7 @@
 #include "llvm/IR/Intrinsics.h"
 #include "llvm/IR/Metadata.h"
 #include "llvm/IR/Module.h"
+#include "llvm/Support/Debug.h"
 #include "llvm/Support/RandomNumberGenerator.h"
 #include <memory>
 #include <random>
@@ -71,7 +72,8 @@ static void emitRemark(IntrinsicInst *II, OptimizationRemarkEmitter &ORE,
 
 static bool removeUbsanTraps(Function &F, const BlockFrequencyInfo &BFI,
                              const ProfileSummaryInfo *PSI,
-                             OptimizationRemarkEmitter &ORE) {
+                             OptimizationRemarkEmitter &ORE,
+                             const std::vector<unsigned int> &cutoffs) {
   SmallVector<std::pair<IntrinsicInst *, bool>, 16> ReplaceWithValue;
   std::unique_ptr<RandomNumberGenerator> Rng;
 
@@ -81,10 +83,22 @@ static bool removeUbsanTraps(Function &F, const BlockFrequencyInfo &BFI,
     return *Rng;
   };
 
-  auto ShouldRemoveHot = [&](const BasicBlock &BB) {
-    return HotPercentileCutoff.getNumOccurrences() && PSI &&
-           PSI->isHotCountNthPercentile(
-               HotPercentileCutoff, BFI.getBlockProfileCount(&BB).value_or(0));
+  auto GetCutoff = [&](const IntrinsicInst *II) -> unsigned {
+    if (HotPercentileCutoff.getNumOccurrences())
+      return HotPercentileCutoff;
+    else if (II->getIntrinsicID() == Intrinsic::allow_ubsan_check) {
+      auto *Kind = cast<ConstantInt>(II->getArgOperand(0));
+      if (Kind->getZExtValue() < cutoffs.size())
+        return cutoffs[Kind->getZExtValue()];
+    }
+
+    return 0;
+  };
+
+  auto ShouldRemoveHot = [&](const BasicBlock &BB, unsigned int cutoff) {
+    return (cutoff == 1000000) ||
+           (PSI && PSI->isHotCountNthPercentile(
+                       cutoff, BFI.getBlockProfileCount(&BB).value_or(0)));
   };
 
   auto ShouldRemoveRandom = [&]() {
@@ -92,8 +106,9 @@ static bool removeUbsanTraps(Function &F, const BlockFrequencyInfo &BFI,
            !std::bernoulli_distribution(RandomRate)(GetRng());
   };
 
-  auto ShouldRemove = [&](const BasicBlock &BB) {
-    return ShouldRemoveRandom() || ShouldRemoveHot(BB);
+  auto ShouldRemove = [&](const IntrinsicInst *II) {
+    unsigned int cutoff = GetCutoff(II);
+    return ShouldRemoveRandom() || ShouldRemoveHot(*(II->getParent()), cutoff);
   };
 
   for (BasicBlock &BB : F) {
@@ -107,7 +122,8 @@ static bool removeUbsanTraps(Function &F, const BlockFrequencyInfo &BFI,
       case Intrinsic::allow_runtime_check: {
         ++NumChecksTotal;
 
-        bool ToRemove = ShouldRemove(BB);
+        bool ToRemove = ShouldRemove(II);
+
         ReplaceWithValue.push_back({
             II,
             ToRemove,
@@ -142,11 +158,37 @@ PreservedAnalyses LowerAllowCheckPass::run(Function &F,
   OptimizationRemarkEmitter &ORE =
       AM.getResult<OptimizationRemarkEmitterAnalysis>(F);
 
-  return removeUbsanTraps(F, BFI, PSI, ORE) ? PreservedAnalyses::none()
-                                            : PreservedAnalyses::all();
+  return removeUbsanTraps(F, BFI, PSI, ORE, Opts.cutoffs)
+             ? PreservedAnalyses::none()
+             : PreservedAnalyses::all();
 }
 
 bool LowerAllowCheckPass::IsRequested() {
   return RandomRate.getNumOccurrences() ||
          HotPercentileCutoff.getNumOccurrences();
 }
+
+void LowerAllowCheckPass::printPipeline(
+    raw_ostream &OS, function_ref<StringRef(StringRef)> MapClassName2PassName) {
+  static_cast<PassInfoMixin<LowerAllowCheckPass> *>(this)->printPipeline(
+      OS, MapClassName2PassName);
+  OS << "<";
+
+  // Format is <cutoffs[0,1,2]=70000;cutoffs[5,6,8]=90000>
+  // but it's equally valid to specify
+  //   cutoffs[0]=70000;cutoffs[1]=70000;cutoffs[2]=70000;cutoffs[5]=90000;...
+  // and that's what we do here. It is verbose but valid and easy to verify
+  // correctness.
+  // TODO: print shorter output by combining adjacent runs, etc.
+  int i = 0;
+  for (unsigned int cutoff : Opts.cutoffs) {
+    if (cutoff > 0) {
+      if (i > 0)
+        OS << ";";
+      OS << "cutoffs[" << i << "]=" << cutoff;
+    }
+
+    i++;
+  }
+  OS << '>';
+}
diff --git a/llvm/lib/Transforms/Instrumentation/MemProfiler.cpp b/llvm/lib/Transforms/Instrumentation/MemProfiler.cpp
index 9a7eec76cc1d1..7d8bc3aa4c589 100644
--- a/llvm/lib/Transforms/Instrumentation/MemProfiler.cpp
+++ b/llvm/lib/Transforms/Instrumentation/MemProfiler.cpp
@@ -970,6 +970,7 @@ static void
 readMemprof(Module &M, Function &F, IndexedInstrProfReader *MemProfReader,
             const TargetLibraryInfo &TLI,
             std::map<uint64_t, AllocMatchInfo> &FullStackIdToAllocMatchInfo,
+            std::set<std::vector<uint64_t>> &MatchedCallSites,
             DenseMap<uint64_t, LocToLocMap> &UndriftMaps) {
   auto &Ctx = M.getContext();
   // Previously we used getIRPGOFuncName() here. If F is local linkage,
@@ -1096,9 +1097,8 @@ readMemprof(Module &M, Function &F, IndexedInstrProfReader *MemProfReader,
       // can happen because we don't currently have discriminators to
       // distinguish the case when a single line/col maps to both an allocation
       // and another callsite).
-      std::map<uint64_t, std::set<const AllocationInfo *>>::iterator
-          AllocInfoIter;
-      decltype(LocHashToCallSites)::iterator CallSitesIter;
+      auto AllocInfoIter = LocHashToAllocInfo.end();
+      auto CallSitesIter = LocHashToCallSites.end();
       for (const DILocation *DIL = I.getDebugLoc(); DIL != nullptr;
            DIL = DIL->getInlinedAt()) {
         // Use C++ linkage name if possible. Need to compile with
@@ -1211,6 +1211,13 @@ readMemprof(Module &M, Function &F, IndexedInstrProfReader *MemProfReader,
           addCallsiteMetadata(I, InlinedCallStack, Ctx);
           // Only need to find one with a matching call stack and add a single
           // callsite metadata.
+
+          // Accumulate call site matching information upon request.
+          if (ClPrintMemProfMatchInfo) {
+            std::vector<uint64_t> CallStack;
+            append_range(CallStack, InlinedCallStack);
+            MatchedCallSites.insert(std::move(CallStack));
+          }
           break;
         }
       }
@@ -1267,13 +1274,17 @@ PreservedAnalyses MemProfUsePass::run(Module &M, ModuleAnalysisManager &AM) {
   // it to an allocation in the IR.
   std::map<uint64_t, AllocMatchInfo> FullStackIdToAllocMatchInfo;
 
+  // Set of the matched call sites, each expressed as a sequence of an inline
+  // call stack.
+  std::set<std::vector<uint64_t>> MatchedCallSites;
+
   for (auto &F : M) {
     if (F.isDeclaration())
       continue;
 
     const TargetLibraryInfo &TLI = FAM.getResult<TargetLibraryAnalysis>(F);
     readMemprof(M, F, MemProfReader.get(), TLI, FullStackIdToAllocMatchInfo,
-                UndriftMaps);
+                MatchedCallSites, UndriftMaps);
   }
 
   if (ClPrintMemProfMatchInfo) {
@@ -1282,6 +1293,13 @@ PreservedAnalyses MemProfUsePass::run(Module &M, ModuleAnalysisManager &AM) {
              << " context with id " << Id << " has total profiled size "
              << Info.TotalSize << (Info.Matched ? " is" : " not")
              << " matched\n";
+
+    for (const auto &CallStack : MatchedCallSites) {
+      errs() << "MemProf callsite match for inline call stack";
+      for (uint64_t StackId : CallStack)
+        errs() << " " << StackId;
+      errs() << "\n";
+    }
   }
 
   return PreservedAnalyses::none();
diff --git a/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp b/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp
index 56d3eb10d73e9..60f3893f20a79 100644
--- a/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp
+++ b/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp
@@ -1272,7 +1272,7 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
       Value *End =
           IRB.CreateUDiv(RoundUp, ConstantInt::get(MS.IntptrTy, kOriginSize));
       auto [InsertPt, Index] =
-          SplitBlockAndInsertSimpleForLoop(End, &*IRB.GetInsertPoint());
+          SplitBlockAndInsertSimpleForLoop(End, IRB.GetInsertPoint());
       IRB.SetInsertPoint(InsertPt);
 
       Value *GEP = IRB.CreateGEP(MS.OriginTy, OriginPtr, Index);
@@ -2933,6 +2933,8 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
   /// Instrument intrinsics that look like a simple SIMD store: writes memory,
   /// has 1 pointer argument and 1 vector argument, returns void.
   bool handleVectorStoreIntrinsic(IntrinsicInst &I) {
+    assert(I.arg_size() == 2);
+
     IRBuilder<> IRB(&I);
     Value *Addr = I.getArgOperand(0);
     Value *Shadow = getShadow(&I, 1);
@@ -2958,6 +2960,8 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
   /// Instrument intrinsics that look like a simple SIMD load: reads memory,
   /// has 1 pointer argument, returns a vector.
   bool handleVectorLoadIntrinsic(IntrinsicInst &I) {
+    assert(I.arg_size() == 1);
+
     IRBuilder<> IRB(&I);
     Value *Addr = I.getArgOperand(0);
 
@@ -2989,17 +2993,24 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
 
   /// Handle (SIMD arithmetic)-like intrinsics.
   ///
-  /// Instrument intrinsics with any number of arguments of the same type,
-  /// equal to the return type. The type should be simple (no aggregates or
-  /// pointers; vectors are fine).
+  /// Instrument intrinsics with any number of arguments of the same type [*],
+  /// equal to the return type, plus a specified number of trailing flags of
+  /// any type.
+  ///
+  /// [*] The type should be simple (no aggregates or pointers; vectors are
+  /// fine).
+  ///
   /// Caller guarantees that this intrinsic does not access memory.
-  bool maybeHandleSimpleNomemIntrinsic(IntrinsicInst &I) {
+  [[maybe_unused]] bool
+  maybeHandleSimpleNomemIntrinsic(IntrinsicInst &I,
+                                  unsigned int trailingFlags) {
     Type *RetTy = I.getType();
     if (!(RetTy->isIntOrIntVectorTy() || RetTy->isFPOrFPVectorTy()))
       return false;
 
     unsigned NumArgOperands = I.arg_size();
-    for (unsigned i = 0; i < NumArgOperands; ++i) {
+    assert(NumArgOperands >= trailingFlags);
+    for (unsigned i = 0; i < NumArgOperands - trailingFlags; ++i) {
       Type *Ty = I.getArgOperand(i)->getType();
       if (Ty != RetTy)
         return false;
@@ -3043,10 +3054,11 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
     }
 
     if (I.doesNotAccessMemory())
-      if (maybeHandleSimpleNomemIntrinsic(I))
+      if (maybeHandleSimpleNomemIntrinsic(I, /*trailingFlags=*/0))
         return true;
 
-    // FIXME: detect and handle SSE maskstore/maskload
+    // FIXME: detect and handle SSE maskstore/maskload?
+    // Some cases are now handled in handleAVXMasked{Load,Store}.
     return false;
   }
 
@@ -3485,17 +3497,40 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
 
   // Instrument generic vector reduction intrinsics
   // by ORing together all their fields.
+  //
+  // The return type does not need to be the same type as the fields
+  // e.g., declare i32 @llvm.aarch64.neon.uaddv.i32.v16i8(<16 x i8>)
   void handleVectorReduceIntrinsic(IntrinsicInst &I) {
+    assert(I.arg_size() == 1);
+
     IRBuilder<> IRB(&I);
     Value *S = IRB.CreateOrReduce(getShadow(&I, 0));
+    S = CreateShadowCast(IRB, S, getShadowTy(&I));
     setShadow(&I, S);
-    setOrigin(&I, getOrigin(&I, 0));
+    setOriginForNaryOp(I);
+  }
+
+  // Similar to handleVectorReduceIntrinsic but with an initial starting value.
+  // e.g., call float @llvm.vector.reduce.fadd.f32.v2f32(float %a0, <2 x float>
+  // %a1)
+  //       shadow = shadow[a0] | shadow[a1.0] | shadow[a1.1]
+  void handleVectorReduceWithStarterIntrinsic(IntrinsicInst &I) {
+    assert(I.arg_size() == 2);
+
+    IRBuilder<> IRB(&I);
+    Value *Shadow0 = getShadow(&I, 0);
+    Value *Shadow1 = IRB.CreateOrReduce(getShadow(&I, 1));
+    Value *S = IRB.CreateOr(Shadow0, Shadow1);
+    setShadow(&I, S);
+    setOriginForNaryOp(I);
   }
 
   // Instrument vector.reduce.or intrinsic.
   // Valid (non-poisoned) set bits in the operand pull low the
   // corresponding shadow bits.
   void handleVectorReduceOrIntrinsic(IntrinsicInst &I) {
+    assert(I.arg_size() == 1);
+
     IRBuilder<> IRB(&I);
     Value *OperandShadow = getShadow(&I, 0);
     Value *OperandUnsetBits = IRB.CreateNot(I.getOperand(0));
@@ -3514,6 +3549,8 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
   // Valid (non-poisoned) unset bits in the operand pull down the
   // corresponding shadow bits.
   void handleVectorReduceAndIntrinsic(IntrinsicInst &I) {
+    assert(I.arg_size() == 1);
+
     IRBuilder<> IRB(&I);
     Value *OperandShadow = getShadow(&I, 0);
     Value *OperandSetOrPoison = IRB.CreateOr(I.getOperand(0), OperandShadow);
@@ -3683,6 +3720,10 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
     // TODO: Store origin.
   }
 
+  // Intrinsic::masked_store
+  //
+  // Note: handleAVXMaskedStore handles AVX/AVX2 variants, though AVX512 masked
+  //       stores are lowered to Intrinsic::masked_store.
   void handleMaskedStore(IntrinsicInst &I) {
     IRBuilder<> IRB(&I);
     Value *V = I.getArgOperand(0);
@@ -3713,6 +3754,10 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
                 std::max(Alignment, kMinOriginAlignment));
   }
 
+  // Intrinsic::masked_load
+  //
+  // Note: handleAVXMaskedLoad handles AVX/AVX2 variants, though AVX512 masked
+  //       loads are lowered to Intrinsic::masked_load.
   void handleMaskedLoad(IntrinsicInst &I) {
     IRBuilder<> IRB(&I);
     Value *Ptr = I.getArgOperand(0);
@@ -3754,6 +3799,129 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
     setOrigin(&I, Origin);
   }
 
+  // e.g., void @llvm.x86.avx.maskstore.ps.256(ptr, <8 x i32>, <8 x float>)
+  //                                           dst  mask       src
+  //
+  // AVX512 masked stores are lowered to Intrinsic::masked_load and are handled
+  // by handleMaskedStore.
+  //
+  // This function handles AVX and AVX2 masked stores; these use the MSBs of a
+  // vector of integers, unlike the LLVM masked intrinsics, which require a
+  // vector of booleans. X86InstCombineIntrinsic.cpp::simplifyX86MaskedLoad
+  // mentions that the x86 backend does not know how to efficiently convert
+  // from a vector of booleans back into the AVX mask format; therefore, they
+  // (and we) do not reduce AVX/AVX2 masked intrinsics into LLVM masked
+  // intrinsics.
+  void handleAVXMaskedStore(IntrinsicInst &I) {
+    assert(I.arg_size() == 3);
+
+    IRBuilder<> IRB(&I);
+
+    Value *Dst = I.getArgOperand(0);
+    assert(Dst->getType()->isPointerTy() && "Destination is not a pointer!");
+
+    Value *Mask = I.getArgOperand(1);
+    assert(isa<VectorType>(Mask->getType()) && "Mask is not a vector!");
+
+    Value *Src = I.getArgOperand(2);
+    assert(isa<VectorType>(Src->getType()) && "Source is not a vector!");
+
+    const Align Alignment = Align(1);
+
+    Value *SrcShadow = getShadow(Src);
+
+    if (ClCheckAccessAddress) {
+      insertShadowCheck(Dst, &I);
+      insertShadowCheck(Mask, &I);
+    }
+
+    Value *DstShadowPtr;
+    Value *DstOriginPtr;
+    std::tie(DstShadowPtr, DstOriginPtr) = getShadowOriginPtr(
+        Dst, IRB, SrcShadow->getType(), Alignment, /*isStore*/ true);
+
+    SmallVector<Value *, 2> ShadowArgs;
+    ShadowArgs.append(1, DstShadowPtr);
+    ShadowArgs.append(1, Mask);
+    // The intrinsic may require floating-point but shadows can be arbitrary
+    // bit patterns, of which some would be interpreted as "invalid"
+    // floating-point values (NaN etc.); we assume the intrinsic will happily
+    // copy them.
+    ShadowArgs.append(1, IRB.CreateBitCast(SrcShadow, Src->getType()));
+
+    CallInst *CI =
+        IRB.CreateIntrinsic(IRB.getVoidTy(), I.getIntrinsicID(), ShadowArgs);
+    setShadow(&I, CI);
+
+    if (!MS.TrackOrigins)
+      return;
+
+    // Approximation only
+    auto &DL = F.getDataLayout();
+    paintOrigin(IRB, getOrigin(Src), DstOriginPtr,
+                DL.getTypeStoreSize(SrcShadow->getType()),
+                std::max(Alignment, kMinOriginAlignment));
+  }
+
+  // e.g., <8 x float> @llvm.x86.avx.maskload.ps.256(ptr, <8 x i32>)
+  //       return                                    src  mask
+  //
+  // Masked-off values are replaced with 0, which conveniently also represents
+  // initialized memory.
+  //
+  // AVX512 masked stores are lowered to Intrinsic::masked_load and are handled
+  // by handleMaskedStore.
+  //
+  // We do not combine this with handleMaskedLoad; see comment in
+  // handleAVXMaskedStore for the rationale.
+  //
+  // This is subtly different than handleIntrinsicByApplyingToShadow(I, 1)
+  // because we need to apply getShadowOriginPtr, not getShadow, to the first
+  // parameter.
+  void handleAVXMaskedLoad(IntrinsicInst &I) {
+    assert(I.arg_size() == 2);
+
+    IRBuilder<> IRB(&I);
+
+    Value *Src = I.getArgOperand(0);
+    assert(Src->getType()->isPointerTy() && "Source is not a pointer!");
+
+    Value *Mask = I.getArgOperand(1);
+    assert(isa<VectorType>(Mask->getType()) && "Mask is not a vector!");
+
+    const Align Alignment = Align(1);
+
+    if (ClCheckAccessAddress) {
+      insertShadowCheck(Mask, &I);
+    }
+
+    Type *SrcShadowTy = getShadowTy(Src);
+    Value *SrcShadowPtr, *SrcOriginPtr;
+    std::tie(SrcShadowPtr, SrcOriginPtr) =
+        getShadowOriginPtr(Src, IRB, SrcShadowTy, Alignment, /*isStore*/ false);
+
+    SmallVector<Value *, 2> ShadowArgs;
+    ShadowArgs.append(1, SrcShadowPtr);
+    ShadowArgs.append(1, Mask);
+
+    CallInst *CI =
+        IRB.CreateIntrinsic(I.getType(), I.getIntrinsicID(), ShadowArgs);
+    // The intrinsic may require floating-point but shadows can be arbitrary
+    // bit patterns, of which some would be interpreted as "invalid"
+    // floating-point values (NaN etc.); we assume the intrinsic will happily
+    // copy them.
+    setShadow(&I, IRB.CreateBitCast(CI, getShadowTy(&I)));
+
+    if (!MS.TrackOrigins)
+      return;
+
+    // The "pass-through" value is always zero (initialized). To the extent
+    // that that results in initialized aligned 4-byte chunks, the origin value
+    // is ignored. It is therefore correct to simply copy the origin from src.
+    Value *PtrSrcOrigin = IRB.CreateLoad(MS.OriginTy, SrcOriginPtr);
+    setOrigin(&I, PtrSrcOrigin);
+  }
+
   // Instrument BMI / BMI2 intrinsics.
   // All of these intrinsics are Z = I(X, Y)
   // where the types of all operands and the result match, and are either i32 or
@@ -3909,19 +4077,83 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
 
   void handleAVXHorizontalAddSubIntrinsic(IntrinsicInst &I) {
     // Approximation only:
-    //    output         = horizontal_add(A, B)
+    //    output         = horizontal_add/sub(A, B)
     // => shadow[output] = horizontal_add(shadow[A], shadow[B])
     //
-    // - If we add/subtract two adjacent zero (initialized) shadow values, the
+    // We always use horizontal add instead of subtract, because subtracting
+    // a fully uninitialized shadow would result in a fully initialized shadow.
+    //
+    // - If we add two adjacent zero (initialized) shadow values, the
     //   result always be zero i.e., no false positives.
-    // - If we add/subtract two shadows, one of which is uninitialized, the
-    //   result will always be non-zero i.e., no false negative.
-    // - However, we can have false negatives if we subtract two non-zero
-    //   shadows of the same value (or do an addition that wraps to zero); we
-    //   consider this an acceptable tradeoff for performance.
+    // - If we add two shadows, one of which is uninitialized, the
+    //   result will always be non-zero i.e., no false negatives.
+    // - However, we can have false negatives if we do an addition that wraps
+    //   to zero; we consider this an acceptable tradeoff for performance.
+    //
     // To make shadow propagation precise, we want the equivalent of
-    // "horizontal OR", but this is not available.
-    return handleIntrinsicByApplyingToShadow(I, /* trailingVerbatimArgs */ 0);
+    // "horizontal OR", but this is not available for SSE3/SSSE3/AVX/AVX2.
+
+    Intrinsic::ID shadowIntrinsicID = I.getIntrinsicID();
+
+    switch (I.getIntrinsicID()) {
+    case Intrinsic::x86_sse3_hsub_ps:
+      shadowIntrinsicID = Intrinsic::x86_sse3_hadd_ps;
+      break;
+
+    case Intrinsic::x86_sse3_hsub_pd:
+      shadowIntrinsicID = Intrinsic::x86_sse3_hadd_pd;
+      break;
+
+    case Intrinsic::x86_ssse3_phsub_d:
+      shadowIntrinsicID = Intrinsic::x86_ssse3_phadd_d;
+      break;
+
+    case Intrinsic::x86_ssse3_phsub_d_128:
+      shadowIntrinsicID = Intrinsic::x86_ssse3_phadd_d_128;
+      break;
+
+    case Intrinsic::x86_ssse3_phsub_w:
+      shadowIntrinsicID = Intrinsic::x86_ssse3_phadd_w;
+      break;
+
+    case Intrinsic::x86_ssse3_phsub_w_128:
+      shadowIntrinsicID = Intrinsic::x86_ssse3_phadd_w_128;
+      break;
+
+    case Intrinsic::x86_ssse3_phsub_sw:
+      shadowIntrinsicID = Intrinsic::x86_ssse3_phadd_sw;
+      break;
+
+    case Intrinsic::x86_ssse3_phsub_sw_128:
+      shadowIntrinsicID = Intrinsic::x86_ssse3_phadd_sw_128;
+      break;
+
+    case Intrinsic::x86_avx_hsub_pd_256:
+      shadowIntrinsicID = Intrinsic::x86_avx_hadd_pd_256;
+      break;
+
+    case Intrinsic::x86_avx_hsub_ps_256:
+      shadowIntrinsicID = Intrinsic::x86_avx_hadd_ps_256;
+      break;
+
+    case Intrinsic::x86_avx2_phsub_d:
+      shadowIntrinsicID = Intrinsic::x86_avx2_phadd_d;
+      break;
+
+    case Intrinsic::x86_avx2_phsub_w:
+      shadowIntrinsicID = Intrinsic::x86_avx2_phadd_w;
+      break;
+
+    case Intrinsic::x86_avx2_phsub_sw:
+      shadowIntrinsicID = Intrinsic::x86_avx2_phadd_sw;
+      break;
+
+    default:
+      break;
+    }
+
+    return handleIntrinsicByApplyingToShadow(I, shadowIntrinsicID,
+                                             /*trailingVerbatimArgs*/ 0);
   }
 
   /// Handle Arm NEON vector store intrinsics (vst{2,3,4}, vst1x_{2,3,4},
@@ -4028,6 +4260,10 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
   ///     shadow[out] =
   ///         intrinsic(shadow[var1], shadow[var2], opType) | shadow[opType]
   ///
+  /// Typically, shadowIntrinsicID will be specified by the caller to be
+  /// I.getIntrinsicID(), but the caller can choose to replace it with another
+  /// intrinsic of the same type.
+  ///
   /// CAUTION: this assumes that the intrinsic will handle arbitrary
   ///          bit-patterns (for example, if the intrinsic accepts floats for
   ///          var1, we require that it doesn't care if inputs are NaNs).
@@ -4037,6 +4273,7 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
   ///
   /// The origin is approximated using setOriginForNaryOp.
   void handleIntrinsicByApplyingToShadow(IntrinsicInst &I,
+                                         Intrinsic::ID shadowIntrinsicID,
                                          unsigned int trailingVerbatimArgs) {
     IRBuilder<> IRB(&I);
 
@@ -4060,7 +4297,7 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
     }
 
     CallInst *CI =
-        IRB.CreateIntrinsic(I.getType(), I.getIntrinsicID(), ShadowArgs);
+        IRB.CreateIntrinsic(I.getType(), shadowIntrinsicID, ShadowArgs);
     Value *CombinedShadow = CI;
 
     // Combine the computed shadow with the shadow of trailing args
@@ -4077,7 +4314,11 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
   }
 
   // Approximation only
+  //
+  // e.g., <16 x i8> @llvm.aarch64.neon.pmull64(i64, i64)
   void handleNEONVectorMultiplyIntrinsic(IntrinsicInst &I) {
+    assert(I.arg_size() == 2);
+
     handleShadowOr(I);
   }
 
@@ -4094,6 +4335,10 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
     case Intrinsic::abs:
       handleAbsIntrinsic(I);
       break;
+    case Intrinsic::bitreverse:
+      handleIntrinsicByApplyingToShadow(I, I.getIntrinsicID(),
+                                        /*trailingVerbatimArgs*/ 0);
+      break;
     case Intrinsic::is_fpclass:
       handleIsFpClass(I);
       break;
@@ -4135,11 +4380,32 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
     case Intrinsic::vector_reduce_or:
       handleVectorReduceOrIntrinsic(I);
       break;
+
     case Intrinsic::vector_reduce_add:
     case Intrinsic::vector_reduce_xor:
     case Intrinsic::vector_reduce_mul:
+    // Add reduction to scalar
+    case Intrinsic::aarch64_neon_faddv:
+    case Intrinsic::aarch64_neon_saddv:
+    case Intrinsic::aarch64_neon_uaddv:
+    // Floating-point min/max (vector)
+    // The f{min,max}"nm"v variants handle NaN differently than f{min,max}v,
+    // but our shadow propagation is the same.
+    case Intrinsic::aarch64_neon_fmaxv:
+    case Intrinsic::aarch64_neon_fminv:
+    case Intrinsic::aarch64_neon_fmaxnmv:
+    case Intrinsic::aarch64_neon_fminnmv:
+    // Sum long across vector
+    case Intrinsic::aarch64_neon_saddlv:
+    case Intrinsic::aarch64_neon_uaddlv:
       handleVectorReduceIntrinsic(I);
       break;
+
+    case Intrinsic::vector_reduce_fadd:
+    case Intrinsic::vector_reduce_fmul:
+      handleVectorReduceWithStarterIntrinsic(I);
+      break;
+
     case Intrinsic::x86_sse_stmxcsr:
       handleStmxcsr(I);
       break;
@@ -4466,6 +4732,44 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
       break;
     }
 
+    case Intrinsic::x86_avx_maskstore_ps:
+    case Intrinsic::x86_avx_maskstore_pd:
+    case Intrinsic::x86_avx_maskstore_ps_256:
+    case Intrinsic::x86_avx_maskstore_pd_256:
+    case Intrinsic::x86_avx2_maskstore_d:
+    case Intrinsic::x86_avx2_maskstore_q:
+    case Intrinsic::x86_avx2_maskstore_d_256:
+    case Intrinsic::x86_avx2_maskstore_q_256: {
+      handleAVXMaskedStore(I);
+      break;
+    }
+
+    case Intrinsic::x86_avx_maskload_ps:
+    case Intrinsic::x86_avx_maskload_pd:
+    case Intrinsic::x86_avx_maskload_ps_256:
+    case Intrinsic::x86_avx_maskload_pd_256:
+    case Intrinsic::x86_avx2_maskload_d:
+    case Intrinsic::x86_avx2_maskload_q:
+    case Intrinsic::x86_avx2_maskload_d_256:
+    case Intrinsic::x86_avx2_maskload_q_256: {
+      handleAVXMaskedLoad(I);
+      break;
+    }
+
+    // Packed
+    case Intrinsic::x86_avx512_min_ps_512:
+    case Intrinsic::x86_avx512_min_pd_512:
+    case Intrinsic::x86_avx512_max_ps_512:
+    case Intrinsic::x86_avx512_max_pd_512: {
+      // These AVX512 variants contain the rounding mode as a trailing flag.
+      // Earlier variants do not have a trailing flag and are already handled
+      // by maybeHandleSimpleNomemIntrinsic(I, 0) via handleUnknownIntrinsic.
+      [[maybe_unused]] bool Success =
+          maybeHandleSimpleNomemIntrinsic(I, /*trailingFlags=*/1);
+      assert(Success);
+      break;
+    }
+
     case Intrinsic::fshl:
     case Intrinsic::fshr:
       handleFunnelShift(I);
@@ -4512,7 +4816,9 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
     case Intrinsic::aarch64_neon_tbx3:
     case Intrinsic::aarch64_neon_tbx4: {
       // The last trailing argument (index register) should be handled verbatim
-      handleIntrinsicByApplyingToShadow(I, 1);
+      handleIntrinsicByApplyingToShadow(
+          I, /*shadowIntrinsicID=*/I.getIntrinsicID(),
+          /*trailingVerbatimArgs*/ 1);
       break;
     }
 
diff --git a/llvm/lib/Transforms/Instrumentation/TypeSanitizer.cpp b/llvm/lib/Transforms/Instrumentation/TypeSanitizer.cpp
index 94101f9663a87..8c28a8e764462 100644
--- a/llvm/lib/Transforms/Instrumentation/TypeSanitizer.cpp
+++ b/llvm/lib/Transforms/Instrumentation/TypeSanitizer.cpp
@@ -497,13 +497,8 @@ void collectMemAccessInfo(
       if (CallInst *CI = dyn_cast<CallInst>(&Inst))
         maybeMarkSanitizerLibraryCallNoBuiltin(CI, &TLI);
 
-      if (isa<MemIntrinsic>(Inst)) {
+      if (isa<MemIntrinsic, LifetimeIntrinsic>(Inst))
         MemTypeResetInsts.push_back(&Inst);
-      } else if (auto *II = dyn_cast<IntrinsicInst>(&Inst)) {
-        if (II->getIntrinsicID() == Intrinsic::lifetime_start ||
-            II->getIntrinsicID() == Intrinsic::lifetime_end)
-          MemTypeResetInsts.push_back(&Inst);
-      }
     } else if (isa<AllocaInst>(Inst)) {
       MemTypeResetInsts.push_back(&Inst);
     }
@@ -819,11 +814,7 @@ bool TypeSanitizer::instrumentMemInst(Value *V, Instruction *ShadowBase,
           NeedsMemMove = isa<MemMoveInst>(MTI);
         }
       }
-    } else if (auto *II = dyn_cast<IntrinsicInst>(I)) {
-      if (II->getIntrinsicID() != Intrinsic::lifetime_start &&
-          II->getIntrinsicID() != Intrinsic::lifetime_end)
-        return false;
-
+    } else if (auto *II = dyn_cast<LifetimeIntrinsic>(I)) {
       Size = II->getArgOperand(0);
       Dest = II->getArgOperand(1);
     } else if (auto *AI = dyn_cast<AllocaInst>(I)) {
diff --git a/llvm/lib/Transforms/ObjCARC/ObjCARCContract.cpp b/llvm/lib/Transforms/ObjCARC/ObjCARCContract.cpp
index 8407726a69c0b..311d3b1cfc0a0 100644
--- a/llvm/lib/Transforms/ObjCARC/ObjCARCContract.cpp
+++ b/llvm/lib/Transforms/ObjCARC/ObjCARCContract.cpp
@@ -64,7 +64,7 @@ namespace {
 
 class ObjCARCContract {
   bool Changed;
-  bool CFGChanged;
+  bool CFGChanged = false;
   AAResults *AA;
   DominatorTree *DT;
   ProvenanceAnalysis PA;
diff --git a/llvm/lib/Transforms/Scalar/CallSiteSplitting.cpp b/llvm/lib/Transforms/Scalar/CallSiteSplitting.cpp
index bbc7a005b9ff4..e644636d42f63 100644
--- a/llvm/lib/Transforms/Scalar/CallSiteSplitting.cpp
+++ b/llvm/lib/Transforms/Scalar/CallSiteSplitting.cpp
@@ -218,8 +218,8 @@ static bool canSplitCallSite(CallBase &CB, TargetTransformInfo &TTI) {
   return true;
 }
 
-static Instruction *cloneInstForMustTail(Instruction *I, Instruction *Before,
-                                         Value *V) {
+static Instruction *
+cloneInstForMustTail(Instruction *I, BasicBlock::iterator Before, Value *V) {
   Instruction *Copy = I->clone();
   Copy->setName(I->getName());
   Copy->insertBefore(Before);
@@ -251,8 +251,8 @@ static void copyMustTailReturn(BasicBlock *SplitBB, Instruction *CI,
   Instruction *TI = SplitBB->getTerminator();
   Value *V = NewCI;
   if (BCI)
-    V = cloneInstForMustTail(BCI, TI, V);
-  cloneInstForMustTail(RI, TI, IsVoid ? nullptr : V);
+    V = cloneInstForMustTail(BCI, TI->getIterator(), V);
+  cloneInstForMustTail(RI, TI->getIterator(), IsVoid ? nullptr : V);
 
   // FIXME: remove TI here, `DuplicateInstructionsInSplitBetween` has a bug
   // that prevents doing this now.
@@ -415,7 +415,7 @@ static void splitCallSite(CallBase &CB,
 // constant incoming values.
 static bool isPredicatedOnPHI(CallBase &CB) {
   BasicBlock *Parent = CB.getParent();
-  if (&CB != Parent->getFirstNonPHIOrDbg())
+  if (&CB != &*Parent->getFirstNonPHIOrDbg())
     return false;
 
   for (auto &PN : Parent->phis()) {
diff --git a/llvm/lib/Transforms/Scalar/ConstraintElimination.cpp b/llvm/lib/Transforms/Scalar/ConstraintElimination.cpp
index e0861fbedc560..6dd26910f6846 100644
--- a/llvm/lib/Transforms/Scalar/ConstraintElimination.cpp
+++ b/llvm/lib/Transforms/Scalar/ConstraintElimination.cpp
@@ -40,7 +40,6 @@
 #include "llvm/Transforms/Utils/Cloning.h"
 #include "llvm/Transforms/Utils/ValueMapper.h"
 
-#include <cmath>
 #include <optional>
 #include <string>
 
@@ -654,7 +653,7 @@ ConstraintInfo::getConstraint(CmpInst::Predicate Pred, Value *Op0, Value *Op1,
   bool IsEq = false;
   bool IsNe = false;
 
-  // Try to convert Pred to one of ULE/SLT/SLE/SLT.
+  // Try to convert Pred to one of ULE/ULT/SLE/SLT.
   switch (Pred) {
   case CmpInst::ICMP_UGT:
   case CmpInst::ICMP_UGE:
diff --git a/llvm/lib/Transforms/Scalar/DeadStoreElimination.cpp b/llvm/lib/Transforms/Scalar/DeadStoreElimination.cpp
index 13f3de07c3c44..38454053b039e 100644
--- a/llvm/lib/Transforms/Scalar/DeadStoreElimination.cpp
+++ b/llvm/lib/Transforms/Scalar/DeadStoreElimination.cpp
@@ -2210,7 +2210,9 @@ struct DSEState {
 
       Instruction *UpperInst = UpperDef->getMemoryInst();
       auto IsRedundantStore = [&]() {
-        if (DefInst->isIdenticalTo(UpperInst))
+        // We don't care about differences in call attributes here.
+        if (DefInst->isIdenticalToWhenDefined(UpperInst,
+                                              /*IntersectAttrs=*/true))
           return true;
         if (auto *MemSetI = dyn_cast<MemSetInst>(UpperInst)) {
           if (auto *SI = dyn_cast<StoreInst>(DefInst)) {
@@ -2281,7 +2283,9 @@ DSEState::getInitializesArgMemLoc(const Instruction *I) {
   for (unsigned Idx = 0, Count = CB->arg_size(); Idx < Count; ++Idx) {
     ConstantRangeList Inits;
     Attribute InitializesAttr = CB->getParamAttr(Idx, Attribute::Initializes);
-    if (InitializesAttr.isValid())
+    // initializes on byval arguments refers to the callee copy, not the
+    // original memory the caller passed in.
+    if (InitializesAttr.isValid() && !CB->isByValArgument(Idx))
       Inits = InitializesAttr.getValueAsConstantRangeList();
 
     Value *CurArg = CB->getArgOperand(Idx);
diff --git a/llvm/lib/Transforms/Scalar/GVN.cpp b/llvm/lib/Transforms/Scalar/GVN.cpp
index 21eb7f741d7c8..3f306bb52c12a 100644
--- a/llvm/lib/Transforms/Scalar/GVN.cpp
+++ b/llvm/lib/Transforms/Scalar/GVN.cpp
@@ -1096,7 +1096,7 @@ Value *AvailableValue::MaterializeAdjustedValue(LoadInst *Load,
   if (isSimpleValue()) {
     Res = getSimpleValue();
     if (Res->getType() != LoadTy) {
-      Res = getValueForLoad(Res, Offset, LoadTy, InsertPt, DL);
+      Res = getValueForLoad(Res, Offset, LoadTy, InsertPt, Load->getFunction());
 
       LLVM_DEBUG(dbgs() << "GVN COERCED NONLOCAL VAL:\nOffset: " << Offset
                         << "  " << *getSimpleValue() << '\n'
@@ -1109,7 +1109,8 @@ Value *AvailableValue::MaterializeAdjustedValue(LoadInst *Load,
       Res = CoercedLoad;
       combineMetadataForCSE(CoercedLoad, Load, false);
     } else {
-      Res = getValueForLoad(CoercedLoad, Offset, LoadTy, InsertPt, DL);
+      Res = getValueForLoad(CoercedLoad, Offset, LoadTy, InsertPt,
+                            Load->getFunction());
       // We are adding a new user for this load, for which the original
       // metadata may not hold. Additionally, the new load may have a different
       // size and type, so their metadata cannot be combined in any
@@ -1291,7 +1292,8 @@ GVNPass::AnalyzeLoadAvailability(LoadInst *Load, MemDepResult DepInfo,
 
         // If MD reported clobber, check it was nested.
         if (DepInfo.isClobber() &&
-            canCoerceMustAliasedValueToLoad(DepLoad, LoadType, DL)) {
+            canCoerceMustAliasedValueToLoad(DepLoad, LoadType,
+                                            DepLoad->getFunction())) {
           const auto ClobberOff = MD->getClobberOffset(DepLoad);
           // GVN has no deal with a negative offset.
           Offset = (ClobberOff == std::nullopt || *ClobberOff < 0)
@@ -1343,7 +1345,7 @@ GVNPass::AnalyzeLoadAvailability(LoadInst *Load, MemDepResult DepInfo,
     // different types if we have to. If the stored value is convertable to
     // the loaded value, we can reuse it.
     if (!canCoerceMustAliasedValueToLoad(S->getValueOperand(), Load->getType(),
-                                         DL))
+                                         S->getFunction()))
       return std::nullopt;
 
     // Can't forward from non-atomic to atomic without violating memory model.
@@ -1357,7 +1359,8 @@ GVNPass::AnalyzeLoadAvailability(LoadInst *Load, MemDepResult DepInfo,
     // If the types mismatch and we can't handle it, reject reuse of the load.
     // If the stored value is larger or equal to the loaded value, we can reuse
     // it.
-    if (!canCoerceMustAliasedValueToLoad(LD, Load->getType(), DL))
+    if (!canCoerceMustAliasedValueToLoad(LD, Load->getType(),
+                                         LD->getFunction()))
       return std::nullopt;
 
     // Can't forward from non-atomic to atomic without violating memory model.
diff --git a/llvm/lib/Transforms/Scalar/GVNHoist.cpp b/llvm/lib/Transforms/Scalar/GVNHoist.cpp
index c6f015112e59d..1c2e1531e47d8 100644
--- a/llvm/lib/Transforms/Scalar/GVNHoist.cpp
+++ b/llvm/lib/Transforms/Scalar/GVNHoist.cpp
@@ -564,21 +564,20 @@ unsigned int GVNHoist::rank(const Value *V) const {
 }
 
 bool GVNHoist::hasEH(const BasicBlock *BB) {
-  auto It = BBSideEffects.find(BB);
-  if (It != BBSideEffects.end())
+  auto [It, Inserted] = BBSideEffects.try_emplace(BB);
+  if (!Inserted)
     return It->second;
 
   if (BB->isEHPad() || BB->hasAddressTaken()) {
-    BBSideEffects[BB] = true;
+    It->second = true;
     return true;
   }
 
   if (BB->getTerminator()->mayThrow()) {
-    BBSideEffects[BB] = true;
+    It->second = true;
     return true;
   }
 
-  BBSideEffects[BB] = false;
   return false;
 }
 
diff --git a/llvm/lib/Transforms/Scalar/IndVarSimplify.cpp b/llvm/lib/Transforms/Scalar/IndVarSimplify.cpp
index 8a3e0bc3eb971..9619dfdbf4123 100644
--- a/llvm/lib/Transforms/Scalar/IndVarSimplify.cpp
+++ b/llvm/lib/Transforms/Scalar/IndVarSimplify.cpp
@@ -1262,14 +1262,14 @@ static std::optional<Value *>
 createReplacement(ICmpInst *ICmp, const Loop *L, BasicBlock *ExitingBB,
                   const SCEV *MaxIter, bool Inverted, bool SkipLastIter,
                   ScalarEvolution *SE, SCEVExpander &Rewriter) {
-  ICmpInst::Predicate Pred = ICmp->getPredicate();
+  CmpPredicate Pred = ICmp->getCmpPredicate();
   Value *LHS = ICmp->getOperand(0);
   Value *RHS = ICmp->getOperand(1);
 
   // 'LHS pred RHS' should now mean that we stay in loop.
   auto *BI = cast<BranchInst>(ExitingBB->getTerminator());
   if (Inverted)
-    Pred = CmpInst::getInversePredicate(Pred);
+    Pred = ICmpInst::getInverseCmpPredicate(Pred);
 
   const SCEV *LHSS = SE->getSCEVAtScope(LHS, L);
   const SCEV *RHSS = SE->getSCEVAtScope(RHS, L);
@@ -1460,7 +1460,6 @@ bool IndVarSimplify::canonicalizeExitCondition(Loop *L) {
     if (!match(LHS, m_ZExt(m_Value(LHSOp))) || !ICmp->isSigned())
       continue;
 
-    const DataLayout &DL = ExitingBB->getDataLayout();
     const unsigned InnerBitWidth = DL.getTypeSizeInBits(LHSOp->getType());
     const unsigned OuterBitWidth = DL.getTypeSizeInBits(RHS->getType());
     auto FullCR = ConstantRange::getFull(InnerBitWidth);
@@ -1536,8 +1535,6 @@ bool IndVarSimplify::canonicalizeExitCondition(Loop *L) {
         DeadInsts.push_back(LHS);
     };
 
-
-    const DataLayout &DL = ExitingBB->getDataLayout();
     const unsigned InnerBitWidth = DL.getTypeSizeInBits(LHSOp->getType());
     const unsigned OuterBitWidth = DL.getTypeSizeInBits(RHS->getType());
     auto FullCR = ConstantRange::getFull(InnerBitWidth);
diff --git a/llvm/lib/Transforms/Scalar/LoopFuse.cpp b/llvm/lib/Transforms/Scalar/LoopFuse.cpp
index 46b0783004fcd..eaf89b23c26f7 100644
--- a/llvm/lib/Transforms/Scalar/LoopFuse.cpp
+++ b/llvm/lib/Transforms/Scalar/LoopFuse.cpp
@@ -1099,7 +1099,7 @@ struct LoopFuser {
 
     LLVM_DEBUG(dbgs() << "Checking if this mem inst can be hoisted.\n");
     for (Instruction *NotHoistedInst : NotHoisting) {
-      if (auto D = DI.depends(&I, NotHoistedInst, true)) {
+      if (auto D = DI.depends(&I, NotHoistedInst)) {
         // Dependency is not read-before-write, write-before-read or
         // write-before-write
         if (D->isFlow() || D->isAnti() || D->isOutput()) {
@@ -1111,7 +1111,7 @@ struct LoopFuser {
     }
 
     for (Instruction *ReadInst : FC0.MemReads) {
-      if (auto D = DI.depends(ReadInst, &I, true)) {
+      if (auto D = DI.depends(ReadInst, &I)) {
         // Dependency is not read-before-write
         if (D->isAnti()) {
           LLVM_DEBUG(dbgs() << "Inst depends on a read instruction in FC0.\n");
@@ -1121,7 +1121,7 @@ struct LoopFuser {
     }
 
     for (Instruction *WriteInst : FC0.MemWrites) {
-      if (auto D = DI.depends(WriteInst, &I, true)) {
+      if (auto D = DI.depends(WriteInst, &I)) {
         // Dependency is not write-before-read or write-before-write
         if (D->isFlow() || D->isOutput()) {
           LLVM_DEBUG(dbgs() << "Inst depends on a write instruction in FC0.\n");
@@ -1153,7 +1153,7 @@ struct LoopFuser {
       return true;
 
     for (Instruction *ReadInst : FC1.MemReads) {
-      if (auto D = DI.depends(&I, ReadInst, true)) {
+      if (auto D = DI.depends(&I, ReadInst)) {
         // Dependency is not write-before-read
         if (D->isFlow()) {
           LLVM_DEBUG(dbgs() << "Inst depends on a read instruction in FC1.\n");
@@ -1163,7 +1163,7 @@ struct LoopFuser {
     }
 
     for (Instruction *WriteInst : FC1.MemWrites) {
-      if (auto D = DI.depends(&I, WriteInst, true)) {
+      if (auto D = DI.depends(&I, WriteInst)) {
         // Dependency is not write-before-write or read-before-write
         if (D->isOutput() || D->isAnti()) {
           LLVM_DEBUG(dbgs() << "Inst depends on a write instruction in FC1.\n");
@@ -1335,7 +1335,7 @@ struct LoopFuser {
     case FUSION_DEPENDENCE_ANALYSIS_SCEV:
       return accessDiffIsPositive(*FC0.L, *FC1.L, I0, I1, AnyDep);
     case FUSION_DEPENDENCE_ANALYSIS_DA: {
-      auto DepResult = DI.depends(&I0, &I1, true);
+      auto DepResult = DI.depends(&I0, &I1);
       if (!DepResult)
         return true;
 #ifndef NDEBUG
diff --git a/llvm/lib/Transforms/Scalar/LoopIdiomRecognize.cpp b/llvm/lib/Transforms/Scalar/LoopIdiomRecognize.cpp
index c5091e731444e..2462ec33e0c20 100644
--- a/llvm/lib/Transforms/Scalar/LoopIdiomRecognize.cpp
+++ b/llvm/lib/Transforms/Scalar/LoopIdiomRecognize.cpp
@@ -1358,7 +1358,29 @@ bool LoopIdiomRecognize::processLoopStoreOfLoopLoad(
     return Changed;
   }
 
+  bool IsAtomic = TheStore->isAtomic() || TheLoad->isAtomic();
   bool UseMemMove = IsMemCpy ? Verifier.IsSameObject : LoopAccessStore;
+
+  if (IsAtomic) {
+    // For now don't support unordered atomic memmove.
+    if (UseMemMove)
+      return Changed;
+
+    // We cannot allow unaligned ops for unordered load/store, so reject
+    // anything where the alignment isn't at least the element size.
+    assert((StoreAlign && LoadAlign) &&
+           "Expect unordered load/store to have align.");
+    if (*StoreAlign < StoreSize || *LoadAlign < StoreSize)
+      return Changed;
+
+    // If the element.atomic memcpy is not lowered into explicit
+    // loads/stores later, then it will be lowered into an element-size
+    // specific lib call. If the lib call doesn't exist for our store size, then
+    // we shouldn't generate the memcpy.
+    if (StoreSize > TTI->getAtomicMemIntrinsicMaxElementSize())
+      return Changed;
+  }
+
   if (UseMemMove)
     if (!Verifier.loadAndStoreMayFormMemmove(StoreSize, IsNegStride, *TheLoad,
                                              IsMemCpy))
@@ -1387,7 +1409,7 @@ bool LoopIdiomRecognize::processLoopStoreOfLoopLoad(
   // Check whether to generate an unordered atomic memcpy:
   //  If the load or store are atomic, then they must necessarily be unordered
   //  by previous checks.
-  if (!TheStore->isAtomic() && !TheLoad->isAtomic()) {
+  if (!IsAtomic) {
     if (UseMemMove)
       NewCall = Builder.CreateMemMove(
           StoreBasePtr, StoreAlign, LoadBasePtr, LoadAlign, NumBytes,
@@ -1398,23 +1420,6 @@ bool LoopIdiomRecognize::processLoopStoreOfLoopLoad(
                                NumBytes, /*isVolatile=*/false, AATags.TBAA,
                                AATags.TBAAStruct, AATags.Scope, AATags.NoAlias);
   } else {
-    // For now don't support unordered atomic memmove.
-    if (UseMemMove)
-      return Changed;
-    // We cannot allow unaligned ops for unordered load/store, so reject
-    // anything where the alignment isn't at least the element size.
-    assert((StoreAlign && LoadAlign) &&
-           "Expect unordered load/store to have align.");
-    if (*StoreAlign < StoreSize || *LoadAlign < StoreSize)
-      return Changed;
-
-    // If the element.atomic memcpy is not lowered into explicit
-    // loads/stores later, then it will be lowered into an element-size
-    // specific lib call. If the lib call doesn't exist for our store size, then
-    // we shouldn't generate the memcpy.
-    if (StoreSize > TTI->getAtomicMemIntrinsicMaxElementSize())
-      return Changed;
-
     // Create the call.
     // Note that unordered atomic loads/stores are *required* by the spec to
     // have an alignment but non-atomic loads/stores may not.
diff --git a/llvm/lib/Transforms/Scalar/LoopInterchange.cpp b/llvm/lib/Transforms/Scalar/LoopInterchange.cpp
index 38fc682698c53..f45d90ff13e14 100644
--- a/llvm/lib/Transforms/Scalar/LoopInterchange.cpp
+++ b/llvm/lib/Transforms/Scalar/LoopInterchange.cpp
@@ -146,7 +146,7 @@ static bool populateDependencyMatrix(CharMatrix &DepMatrix, unsigned Level,
       if (isa<LoadInst>(Src) && isa<LoadInst>(Dst))
         continue;
       // Track Output, Flow, and Anti dependencies.
-      if (auto D = DI->depends(Src, Dst, true)) {
+      if (auto D = DI->depends(Src, Dst)) {
         assert(D->isOrdered() && "Expected an output, flow or anti dep.");
         // If the direction vector is negative, normalize it to
         // make it non-negative.
@@ -160,11 +160,16 @@ static bool populateDependencyMatrix(CharMatrix &DepMatrix, unsigned Level,
         unsigned Levels = D->getLevels();
         char Direction;
         for (unsigned II = 1; II <= Levels; ++II) {
+          // `DVEntry::LE` is converted to `*`. This is because `LE` means `<`
+          // or `=`, for which we don't have an equivalent representation, so
+          // that the conservative approximation is necessary. The same goes for
+          // `DVEntry::GE`.
+          // TODO: Use of fine-grained expressions allows for more accurate
+          // analysis.
           unsigned Dir = D->getDirection(II);
-          if (Dir == Dependence::DVEntry::LT || Dir == Dependence::DVEntry::LE)
+          if (Dir == Dependence::DVEntry::LT)
             Direction = '<';
-          else if (Dir == Dependence::DVEntry::GT ||
-                   Dir == Dependence::DVEntry::GE)
+          else if (Dir == Dependence::DVEntry::GT)
             Direction = '>';
           else if (Dir == Dependence::DVEntry::EQ)
             Direction = '=';
@@ -271,6 +276,27 @@ static bool hasSupportedLoopDepth(SmallVectorImpl<Loop *> &LoopList,
   }
   return true;
 }
+
+static bool isComputableLoopNest(ScalarEvolution *SE,
+                                 ArrayRef<Loop *> LoopList) {
+  for (Loop *L : LoopList) {
+    const SCEV *ExitCountOuter = SE->getBackedgeTakenCount(L);
+    if (isa<SCEVCouldNotCompute>(ExitCountOuter)) {
+      LLVM_DEBUG(dbgs() << "Couldn't compute backedge count\n");
+      return false;
+    }
+    if (L->getNumBackEdges() != 1) {
+      LLVM_DEBUG(dbgs() << "NumBackEdges is not equal to 1\n");
+      return false;
+    }
+    if (!L->getExitingBlock()) {
+      LLVM_DEBUG(dbgs() << "Loop doesn't have unique exit block\n");
+      return false;
+    }
+  }
+  return true;
+}
+
 namespace {
 
 /// LoopInterchangeLegality checks if it is legal to interchange the loop.
@@ -426,25 +452,6 @@ struct LoopInterchange {
     return processLoopList(LoopList);
   }
 
-  bool isComputableLoopNest(ArrayRef<Loop *> LoopList) {
-    for (Loop *L : LoopList) {
-      const SCEV *ExitCountOuter = SE->getBackedgeTakenCount(L);
-      if (isa<SCEVCouldNotCompute>(ExitCountOuter)) {
-        LLVM_DEBUG(dbgs() << "Couldn't compute backedge count\n");
-        return false;
-      }
-      if (L->getNumBackEdges() != 1) {
-        LLVM_DEBUG(dbgs() << "NumBackEdges is not equal to 1\n");
-        return false;
-      }
-      if (!L->getExitingBlock()) {
-        LLVM_DEBUG(dbgs() << "Loop doesn't have unique exit block\n");
-        return false;
-      }
-    }
-    return true;
-  }
-
   unsigned selectLoopForInterchange(ArrayRef<Loop *> LoopList) {
     // TODO: Add a better heuristic to select the loop to be interchanged based
     // on the dependence matrix. Currently we select the innermost loop.
@@ -459,10 +466,6 @@ struct LoopInterchange {
            "Unsupported depth of loop nest.");
 
     unsigned LoopNestDepth = LoopList.size();
-    if (!isComputableLoopNest(LoopList)) {
-      LLVM_DEBUG(dbgs() << "Not valid loop candidate for interchange\n");
-      return false;
-    }
 
     LLVM_DEBUG(dbgs() << "Processing LoopList of size = " << LoopNestDepth
                       << "\n");
@@ -1756,10 +1759,23 @@ PreservedAnalyses LoopInterchangePass::run(LoopNest &LN,
   // Ensure minimum depth of the loop nest to do the interchange.
   if (!hasSupportedLoopDepth(LoopList, ORE))
     return PreservedAnalyses::all();
+  // Ensure computable loop nest.
+  if (!isComputableLoopNest(&AR.SE, LoopList)) {
+    LLVM_DEBUG(dbgs() << "Not valid loop candidate for interchange\n");
+    return PreservedAnalyses::all();
+  }
+
+  ORE.emit([&]() {
+    return OptimizationRemarkAnalysis(DEBUG_TYPE, "Dependence",
+                                      LN.getOutermostLoop().getStartLoc(),
+                                      LN.getOutermostLoop().getHeader())
+           << "Computed dependence info, invoking the transform.";
+  });
+
   DependenceInfo DI(&F, &AR.AA, &AR.SE, &AR.LI);
   std::unique_ptr<CacheCost> CC =
       CacheCost::getCacheCost(LN.getOutermostLoop(), AR, DI);
-  
+
   if (!LoopInterchange(&AR.SE, &AR.LI, &DI, &AR.DT, CC, &ORE).run(LN))
     return PreservedAnalyses::all();
   U.markLoopNestChanged(true);
diff --git a/llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp b/llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp
index 5a9a7ecdc13bf..2c3d64b0e07d9 100644
--- a/llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp
+++ b/llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp
@@ -6138,7 +6138,7 @@ void LSRInstance::ImplementSolution(
     if (!llvm::all_of(BO->uses(),
                       [&](Use &U) {return DT.dominates(IVIncInsertPos, U);}))
       continue;
-    BO->moveBefore(IVIncInsertPos);
+    BO->moveBefore(IVIncInsertPos->getIterator());
     Changed = true;
   }
 
@@ -6176,11 +6176,11 @@ LSRInstance::LSRInstance(Loop *L, IVUsers &IU, ScalarEvolution &SE,
     // CatchSwitchInst.  Because the CatchSwitchInst cannot be split, there is
     // no good place to stick any instructions.
     if (auto *PN = dyn_cast<PHINode>(U.getUser())) {
-       auto *FirstNonPHI = PN->getParent()->getFirstNonPHI();
+       auto FirstNonPHI = PN->getParent()->getFirstNonPHIIt();
        if (isa<FuncletPadInst>(FirstNonPHI) ||
            isa<CatchSwitchInst>(FirstNonPHI))
          for (BasicBlock *PredBB : PN->blocks())
-           if (isa<CatchSwitchInst>(PredBB->getFirstNonPHI()))
+           if (isa<CatchSwitchInst>(PredBB->getFirstNonPHIIt()))
              return;
     }
   }
diff --git a/llvm/lib/Transforms/Scalar/LoopUnrollPass.cpp b/llvm/lib/Transforms/Scalar/LoopUnrollPass.cpp
index 090348809e571..5f28ee616f0e4 100644
--- a/llvm/lib/Transforms/Scalar/LoopUnrollPass.cpp
+++ b/llvm/lib/Transforms/Scalar/LoopUnrollPass.cpp
@@ -220,6 +220,7 @@ TargetTransformInfo::UnrollingPreferences llvm::gatherUnrollingPreferences(
   UP.UnrollAndJamInnerLoopThreshold = 60;
   UP.MaxIterationsCountToAnalyze = UnrollMaxIterationsCountToAnalyze;
   UP.SCEVExpansionBudget = SCEVCheapExpansionBudget;
+  UP.RuntimeUnrollMultiExit = false;
 
   // Override with any target specific settings
   TTI.getUnrollingPreferences(L, SE, UP, &ORE);
@@ -1352,6 +1353,7 @@ tryToUnrollLoop(Loop *L, DominatorTree &DT, LoopInfo *LI, ScalarEvolution &SE,
   ULO.ForgetAllSCEV = ForgetAllSCEV;
   ULO.Heart = getLoopConvergenceHeart(L);
   ULO.SCEVExpansionBudget = UP.SCEVExpansionBudget;
+  ULO.RuntimeUnrollMultiExit = UP.RuntimeUnrollMultiExit;
   LoopUnrollResult UnrollResult = UnrollLoop(
       L, ULO, LI, &SE, &DT, &AC, &TTI, &ORE, PreserveLCSSA, &RemainderLoop, AA);
   if (UnrollResult == LoopUnrollResult::Unmodified)
diff --git a/llvm/lib/Transforms/Scalar/MemCpyOptimizer.cpp b/llvm/lib/Transforms/Scalar/MemCpyOptimizer.cpp
index a80a85f38e74d..87b27beb01a0a 100644
--- a/llvm/lib/Transforms/Scalar/MemCpyOptimizer.cpp
+++ b/llvm/lib/Transforms/Scalar/MemCpyOptimizer.cpp
@@ -968,9 +968,8 @@ bool MemCpyOptPass::performCallSlotOptzn(Instruction *cpyLoad,
       append_range(srcUseList, U->users());
       continue;
     }
-    if (const auto *IT = dyn_cast<IntrinsicInst>(U))
-      if (IT->isLifetimeStartOrEnd())
-        continue;
+    if (isa<LifetimeIntrinsic>(U))
+      continue;
 
     if (U != C && U != cpyLoad) {
       LLVM_DEBUG(dbgs() << "Call slot: Source accessed by " << *U << "\n");
diff --git a/llvm/lib/Transforms/Scalar/MergedLoadStoreMotion.cpp b/llvm/lib/Transforms/Scalar/MergedLoadStoreMotion.cpp
index cc67a455672be..60fbb689c33f3 100644
--- a/llvm/lib/Transforms/Scalar/MergedLoadStoreMotion.cpp
+++ b/llvm/lib/Transforms/Scalar/MergedLoadStoreMotion.cpp
@@ -281,7 +281,7 @@ void MergedLoadStoreMotion::sinkStoresAndGEPs(BasicBlock *BB, StoreInst *S0,
     auto *GEP0 = cast<GetElementPtrInst>(Ptr0);
     auto *GEP1 = cast<GetElementPtrInst>(Ptr1);
     Instruction *GEPNew = GEP0->clone();
-    GEPNew->insertBefore(SNew);
+    GEPNew->insertBefore(SNew->getIterator());
     GEPNew->applyMergedLocation(GEP0->getDebugLoc(), GEP1->getDebugLoc());
     SNew->setOperand(1, GEPNew);
     GEP0->replaceAllUsesWith(GEPNew);
diff --git a/llvm/lib/Transforms/Scalar/NaryReassociate.cpp b/llvm/lib/Transforms/Scalar/NaryReassociate.cpp
index 39720672c202d..3b3f32a0ea591 100644
--- a/llvm/lib/Transforms/Scalar/NaryReassociate.cpp
+++ b/llvm/lib/Transforms/Scalar/NaryReassociate.cpp
@@ -402,16 +402,17 @@ NaryReassociatePass::tryReassociateGEPAtIndex(GetElementPtrInst *GEP,
     IndexExprs.push_back(SE->getSCEV(Index));
   // Replace the I-th index with LHS.
   IndexExprs[I] = SE->getSCEV(LHS);
+  Type *GEPArgType = SE->getEffectiveSCEVType(GEP->getOperand(I)->getType());
+  Type *LHSType = SE->getEffectiveSCEVType(LHS->getType());
+  size_t LHSSize = DL->getTypeSizeInBits(LHSType).getFixedValue();
+  size_t GEPArgSize = DL->getTypeSizeInBits(GEPArgType).getFixedValue();
   if (isKnownNonNegative(LHS, SimplifyQuery(*DL, DT, AC, GEP)) &&
-      DL->getTypeSizeInBits(LHS->getType()).getFixedValue() <
-          DL->getTypeSizeInBits(GEP->getOperand(I)->getType())
-              .getFixedValue()) {
+      LHSSize < GEPArgSize) {
     // Zero-extend LHS if it is non-negative. InstCombine canonicalizes sext to
     // zext if the source operand is proved non-negative. We should do that
     // consistently so that CandidateExpr more likely appears before. See
     // @reassociate_gep_assume for an example of this canonicalization.
-    IndexExprs[I] =
-        SE->getZeroExtendExpr(IndexExprs[I], GEP->getOperand(I)->getType());
+    IndexExprs[I] = SE->getZeroExtendExpr(IndexExprs[I], GEPArgType);
   }
   const SCEV *CandidateExpr = SE->getGEPExpr(cast<GEPOperator>(GEP),
                                              IndexExprs);
diff --git a/llvm/lib/Transforms/Scalar/NewGVN.cpp b/llvm/lib/Transforms/Scalar/NewGVN.cpp
index d8b0bd173454f..168f1d689d314 100644
--- a/llvm/lib/Transforms/Scalar/NewGVN.cpp
+++ b/llvm/lib/Transforms/Scalar/NewGVN.cpp
@@ -3056,13 +3056,8 @@ std::pair<unsigned, unsigned> NewGVN::assignDFSNumbers(BasicBlock *B,
 
 void NewGVN::updateProcessedCount(const Value *V) {
 #ifndef NDEBUG
-  if (ProcessedCount.count(V) == 0) {
-    ProcessedCount.insert({V, 1});
-  } else {
-    ++ProcessedCount[V];
-    assert(ProcessedCount[V] < 100 &&
-           "Seem to have processed the same Value a lot");
-  }
+  assert(++ProcessedCount[V] < 100 &&
+         "Seem to have processed the same Value a lot");
 #endif
 }
 
diff --git a/llvm/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp b/llvm/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp
index 7b848ae547bd5..c7b55c2fb4f45 100644
--- a/llvm/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp
+++ b/llvm/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp
@@ -1371,7 +1371,7 @@ static void recomputeLiveInValues(
 // and inserts them before "InsertBefore". Returns rematerialized value
 // which should be used after statepoint.
 static Instruction *rematerializeChain(ArrayRef<Instruction *> ChainToBase,
-                                       Instruction *InsertBefore,
+                                       BasicBlock::iterator InsertBefore,
                                        Value *RootOfChain,
                                        Value *AlternateLiveBase) {
   Instruction *LastClonedValue = nullptr;
@@ -2185,16 +2185,16 @@ static void relocationViaAlloca(
         // InvokeInst is a terminator so the store need to be inserted into its
         // normal destination block.
         BasicBlock *NormalDest = Invoke->getNormalDest();
-        Store->insertBefore(NormalDest->getFirstNonPHI());
+        Store->insertBefore(NormalDest->getFirstNonPHIIt());
       } else {
         assert(!Inst->isTerminator() &&
                "The only terminator that can produce a value is "
                "InvokeInst which is handled above.");
-        Store->insertAfter(Inst);
+        Store->insertAfter(Inst->getIterator());
       }
     } else {
       assert(isa<Argument>(Def));
-      Store->insertAfter(cast<Instruction>(Alloca));
+      Store->insertAfter(cast<Instruction>(Alloca)->getIterator());
     }
   }
 
@@ -2499,8 +2499,9 @@ static void rematerializeLiveValuesAtUses(
     //   statepoint between uses in the block.
     while (!Cand->user_empty()) {
       Instruction *UserI = cast<Instruction>(*Cand->user_begin());
-      Instruction *RematChain = rematerializeChain(
-          Record.ChainToBase, UserI, Record.RootOfChain, PointerToBase[Cand]);
+      Instruction *RematChain =
+          rematerializeChain(Record.ChainToBase, UserI->getIterator(),
+                             Record.RootOfChain, PointerToBase[Cand]);
       UserI->replaceUsesOfWith(Cand, RematChain);
       PointerToBase[RematChain] = PointerToBase[Cand];
     }
@@ -2573,16 +2574,16 @@ static void rematerializeLiveValues(CallBase *Call,
       Instruction *InsertBefore = Call->getNextNode();
       assert(InsertBefore);
       Instruction *RematerializedValue =
-          rematerializeChain(Record.ChainToBase, InsertBefore,
+          rematerializeChain(Record.ChainToBase, InsertBefore->getIterator(),
                              Record.RootOfChain, PointerToBase[LiveValue]);
       Info.RematerializedValues[RematerializedValue] = LiveValue;
     } else {
       auto *Invoke = cast<InvokeInst>(Call);
 
-      Instruction *NormalInsertBefore =
-          &*Invoke->getNormalDest()->getFirstInsertionPt();
-      Instruction *UnwindInsertBefore =
-          &*Invoke->getUnwindDest()->getFirstInsertionPt();
+      BasicBlock::iterator NormalInsertBefore =
+          Invoke->getNormalDest()->getFirstInsertionPt();
+      BasicBlock::iterator UnwindInsertBefore =
+          Invoke->getUnwindDest()->getFirstInsertionPt();
 
       Instruction *NormalRematerializedValue =
           rematerializeChain(Record.ChainToBase, NormalInsertBefore,
@@ -3131,7 +3132,7 @@ bool RewriteStatepointsForGC::runOnFunction(Function &F, DominatorTree &DT,
       // most instructions without side effects or memory access.
       if (isa<ICmpInst>(Cond) && Cond->hasOneUse()) {
         MadeChange = true;
-        Cond->moveBefore(TI);
+        Cond->moveBefore(TI->getIterator());
       }
   }
 
diff --git a/llvm/lib/Transforms/Scalar/SROA.cpp b/llvm/lib/Transforms/Scalar/SROA.cpp
index f6179cadab425..29240aaaa21be 100644
--- a/llvm/lib/Transforms/Scalar/SROA.cpp
+++ b/llvm/lib/Transforms/Scalar/SROA.cpp
@@ -480,7 +480,7 @@ static void migrateDebugInfo(AllocaInst *OldAlloca, bool IsSplit,
     // noted as slightly offset (in code) from the store. In practice this
     // should have little effect on the debugging experience due to the fact
     // that all the split stores should get the same line number.
-    NewAssign->moveBefore(DbgAssign);
+    NewAssign->moveBefore(DbgAssign->getIterator());
 
     NewAssign->setDebugLoc(DbgAssign->getDebugLoc());
     LLVM_DEBUG(dbgs() << "Created new assign: " << *NewAssign << "\n");
@@ -1843,7 +1843,7 @@ static void rewriteMemOpOfSelect(SelectInst &SI, T &I,
       CondMemOp.dropUBImplyingAttrsAndMetadata();
       ++NumLoadsSpeculated;
     }
-    CondMemOp.insertBefore(NewMemOpBB->getTerminator());
+    CondMemOp.insertBefore(NewMemOpBB->getTerminator()->getIterator());
     Value *Ptr = SI.getOperand(1 + SuccIdx);
     CondMemOp.setOperand(I.getPointerOperandIndex(), Ptr);
     if (isa<LoadInst>(I)) {
diff --git a/llvm/lib/Transforms/Scalar/SimpleLoopUnswitch.cpp b/llvm/lib/Transforms/Scalar/SimpleLoopUnswitch.cpp
index c2f7c5dcaf160..4f7956514b7b5 100644
--- a/llvm/lib/Transforms/Scalar/SimpleLoopUnswitch.cpp
+++ b/llvm/lib/Transforms/Scalar/SimpleLoopUnswitch.cpp
@@ -770,7 +770,7 @@ static bool unswitchTrivialSwitch(Loop &L, SwitchInst &SI, DominatorTree &DT,
     // instruction in the block.
     auto *TI = BBToCheck.getTerminator();
     bool isUnreachable = isa<UnreachableInst>(TI);
-    return !isUnreachable || BBToCheck.getFirstNonPHIOrDbg() != TI;
+    return !isUnreachable || &*BBToCheck.getFirstNonPHIOrDbg() != TI;
   };
 
   SmallVector<int, 4> ExitCaseIndices;
@@ -3303,8 +3303,8 @@ static bool isSafeForNoNTrivialUnswitching(Loop &L, LoopInfo &LI) {
   // FIXME: We should teach SplitBlock to handle this and remove this
   // restriction.
   for (auto *ExitBB : ExitBlocks) {
-    auto *I = ExitBB->getFirstNonPHI();
-    if (isa<CleanupPadInst>(I) || isa<CatchSwitchInst>(I)) {
+    auto It = ExitBB->getFirstNonPHIIt();
+    if (isa<CleanupPadInst>(It) || isa<CatchSwitchInst>(It)) {
       LLVM_DEBUG(dbgs() << "Cannot unswitch because of cleanuppad/catchswitch "
                            "in exit block\n");
       return false;
diff --git a/llvm/lib/Transforms/Utils/AssumeBundleBuilder.cpp b/llvm/lib/Transforms/Utils/AssumeBundleBuilder.cpp
index b05ae00a1e0ea..2d9a3d1f8a110 100644
--- a/llvm/lib/Transforms/Utils/AssumeBundleBuilder.cpp
+++ b/llvm/lib/Transforms/Utils/AssumeBundleBuilder.cpp
@@ -474,9 +474,9 @@ struct AssumeSimplify {
     AssumeBuilderState Builder(F.getParent());
 
     /// For now it is initialized to the best value it could have
-    Instruction *InsertPt = BB->getFirstNonPHI();
+    BasicBlock::iterator InsertPt = BB->getFirstNonPHIIt();
     if (isa<LandingPadInst>(InsertPt))
-      InsertPt = InsertPt->getNextNode();
+      InsertPt = std::next(InsertPt);
     for (IntrinsicInst *I : make_range(Begin, End)) {
       CleanupToDo.insert(I);
       for (CallInst::BundleOpInfo &BOI : I->bundle_op_infos()) {
@@ -487,8 +487,8 @@ struct AssumeSimplify {
         Builder.addKnowledge(RK);
         if (auto *I = dyn_cast_or_null<Instruction>(RK.WasOn))
           if (I->getParent() == InsertPt->getParent() &&
-              (InsertPt->comesBefore(I) || InsertPt == I))
-            InsertPt = I->getNextNode();
+              (InsertPt->comesBefore(I) || &*InsertPt == I))
+            InsertPt = I->getNextNode()->getIterator();
       }
     }
 
@@ -498,7 +498,7 @@ struct AssumeSimplify {
       for (auto It = (*Begin)->getIterator(), E = InsertPt->getIterator();
            It != E; --It)
         if (!isGuaranteedToTransferExecutionToSuccessor(&*It)) {
-          InsertPt = It->getNextNode();
+          InsertPt = std::next(It);
           break;
         }
     auto *MergedAssume = Builder.build();
diff --git a/llvm/lib/Transforms/Utils/BasicBlockUtils.cpp b/llvm/lib/Transforms/Utils/BasicBlockUtils.cpp
index 7811677000998..f81420460f08d 100644
--- a/llvm/lib/Transforms/Utils/BasicBlockUtils.cpp
+++ b/llvm/lib/Transforms/Utils/BasicBlockUtils.cpp
@@ -836,7 +836,7 @@ BasicBlock *llvm::ehAwareSplitEdge(BasicBlock *BB, BasicBlock *Succ,
                                    const CriticalEdgeSplittingOptions &Options,
                                    const Twine &BBName) {
 
-  auto *PadInst = Succ->getFirstNonPHI();
+  auto PadInst = Succ->getFirstNonPHIIt();
   if (!LandingPadReplacement && !PadInst->isEHPad())
     return SplitEdge(BB, Succ, Options.DT, Options.LI, Options.MSSAU, BBName);
 
@@ -981,7 +981,7 @@ BasicBlock *llvm::ehAwareSplitEdge(BasicBlock *BB, BasicBlock *Succ,
 void llvm::createPHIsForSplitLoopExit(ArrayRef<BasicBlock *> Preds,
                                       BasicBlock *SplitBB, BasicBlock *DestBB) {
   // SplitBB shouldn't have anything non-trivial in it yet.
-  assert((SplitBB->getFirstNonPHI() == SplitBB->getTerminator() ||
+  assert((&*SplitBB->getFirstNonPHIIt() == SplitBB->getTerminator() ||
           SplitBB->isLandingPad()) &&
          "SplitBB has non-PHI nodes!");
 
@@ -1450,7 +1450,7 @@ static void SplitLandingPadPredecessorsImpl(
 
   // The new block unconditionally branches to the old block.
   BranchInst *BI1 = BranchInst::Create(OrigBB, NewBB1);
-  BI1->setDebugLoc(OrigBB->getFirstNonPHI()->getDebugLoc());
+  BI1->setDebugLoc(OrigBB->getFirstNonPHIIt()->getDebugLoc());
 
   // Move the edges from Preds to point to NewBB1 instead of OrigBB.
   for (BasicBlock *Pred : Preds) {
@@ -1491,7 +1491,7 @@ static void SplitLandingPadPredecessorsImpl(
 
     // The new block unconditionally branches to the old block.
     BranchInst *BI2 = BranchInst::Create(OrigBB, NewBB2);
-    BI2->setDebugLoc(OrigBB->getFirstNonPHI()->getDebugLoc());
+    BI2->setDebugLoc(OrigBB->getFirstNonPHIIt()->getDebugLoc());
 
     // Move the remaining edges from OrigBB to point to NewBB2.
     for (BasicBlock *NewBB2Pred : NewBB2Preds)
@@ -1728,8 +1728,9 @@ void llvm::SplitBlockAndInsertIfThenElse(
   }
 }
 
-std::pair<Instruction*, Value*>
-llvm::SplitBlockAndInsertSimpleForLoop(Value *End, Instruction *SplitBefore) {
+std::pair<Instruction *, Value *>
+llvm::SplitBlockAndInsertSimpleForLoop(Value *End,
+                                       BasicBlock::iterator SplitBefore) {
   BasicBlock *LoopPred = SplitBefore->getParent();
   BasicBlock *LoopBody = SplitBlock(SplitBefore->getParent(), SplitBefore);
   BasicBlock *LoopExit = SplitBlock(SplitBefore->getParent(), SplitBefore);
@@ -1752,14 +1753,14 @@ llvm::SplitBlockAndInsertSimpleForLoop(Value *End, Instruction *SplitBefore) {
   IV->addIncoming(ConstantInt::get(Ty, 0), LoopPred);
   IV->addIncoming(IVNext, LoopBody);
 
-  return std::make_pair(LoopBody->getFirstNonPHI(), IV);
+  return std::make_pair(&*LoopBody->getFirstNonPHIIt(), IV);
 }
 
-void llvm::SplitBlockAndInsertForEachLane(ElementCount EC,
-     Type *IndexTy, Instruction *InsertBefore,
-     std::function<void(IRBuilderBase&, Value*)> Func) {
+void llvm::SplitBlockAndInsertForEachLane(
+    ElementCount EC, Type *IndexTy, BasicBlock::iterator InsertBefore,
+    std::function<void(IRBuilderBase &, Value *)> Func) {
 
-  IRBuilder<> IRB(InsertBefore);
+  IRBuilder<> IRB(InsertBefore->getParent(), InsertBefore);
 
   if (EC.isScalable()) {
     Value *NumElements = IRB.CreateElementCount(IndexTy, EC);
@@ -1780,10 +1781,10 @@ void llvm::SplitBlockAndInsertForEachLane(ElementCount EC,
 }
 
 void llvm::SplitBlockAndInsertForEachLane(
-    Value *EVL, Instruction *InsertBefore,
+    Value *EVL, BasicBlock::iterator InsertBefore,
     std::function<void(IRBuilderBase &, Value *)> Func) {
 
-  IRBuilder<> IRB(InsertBefore);
+  IRBuilder<> IRB(InsertBefore->getParent(), InsertBefore);
   Type *Ty = EVL->getType();
 
   if (!isa<ConstantInt>(EVL)) {
diff --git a/llvm/lib/Transforms/Utils/BuildLibCalls.cpp b/llvm/lib/Transforms/Utils/BuildLibCalls.cpp
index fe1b91267c907..7a5326c255831 100644
--- a/llvm/lib/Transforms/Utils/BuildLibCalls.cpp
+++ b/llvm/lib/Transforms/Utils/BuildLibCalls.cpp
@@ -133,9 +133,10 @@ static bool setRetDoesNotAlias(Function &F) {
 }
 
 static bool setDoesNotCapture(Function &F, unsigned ArgNo) {
-  if (F.hasParamAttribute(ArgNo, Attribute::NoCapture))
+  if (F.hasParamAttribute(ArgNo, Attribute::Captures))
     return false;
-  F.addParamAttr(ArgNo, Attribute::NoCapture);
+  F.addParamAttr(ArgNo, Attribute::getWithCaptureInfo(F.getContext(),
+                                                      CaptureInfo::none()));
   ++NumNoCapture;
   return true;
 }
diff --git a/llvm/lib/Transforms/Utils/CodeExtractor.cpp b/llvm/lib/Transforms/Utils/CodeExtractor.cpp
index 526132f5e5332..7277603b3ec2b 100644
--- a/llvm/lib/Transforms/Utils/CodeExtractor.cpp
+++ b/llvm/lib/Transforms/Utils/CodeExtractor.cpp
@@ -535,13 +535,10 @@ void CodeExtractor::findAllocas(const CodeExtractorAnalysisCache &CEAC,
 
       Instruction *Bitcast = cast<Instruction>(U);
       for (User *BU : Bitcast->users()) {
-        IntrinsicInst *IntrInst = dyn_cast<IntrinsicInst>(BU);
+        auto *IntrInst = dyn_cast<LifetimeIntrinsic>(BU);
         if (!IntrInst)
           continue;
 
-        if (!IntrInst->isLifetimeStartOrEnd())
-          continue;
-
         if (definedInRegion(Blocks, IntrInst))
           continue;
 
@@ -983,7 +980,6 @@ Function *CodeExtractor::constructFunctionDeclaration(
       case Attribute::InReg:
       case Attribute::Nest:
       case Attribute::NoAlias:
-      case Attribute::NoCapture:
       case Attribute::NoUndef:
       case Attribute::NonNull:
       case Attribute::Preallocated:
@@ -1084,8 +1080,8 @@ static void eraseLifetimeMarkersOnInputs(const SetVector<BasicBlock *> &Blocks,
                                          SetVector<Value *> &LifetimesStart) {
   for (BasicBlock *BB : Blocks) {
     for (Instruction &I : llvm::make_early_inc_range(*BB)) {
-      auto *II = dyn_cast<IntrinsicInst>(&I);
-      if (!II || !II->isLifetimeStartOrEnd())
+      auto *II = dyn_cast<LifetimeIntrinsic>(&I);
+      if (!II)
         continue;
 
       // Get the memory operand of the lifetime marker. If the underlying
diff --git a/llvm/lib/Transforms/Utils/CodeMoverUtils.cpp b/llvm/lib/Transforms/Utils/CodeMoverUtils.cpp
index f34e9c5818dd6..ecf71b6056f2a 100644
--- a/llvm/lib/Transforms/Utils/CodeMoverUtils.cpp
+++ b/llvm/lib/Transforms/Utils/CodeMoverUtils.cpp
@@ -398,7 +398,7 @@ bool llvm::isSafeToMoveBefore(Instruction &I, Instruction &InsertPoint,
   // Check if I has any output/flow/anti dependences with instructions from \p
   // StartInst to \p EndInst.
   if (llvm::any_of(InstsToCheck, [&DI, &I](Instruction *CurInst) {
-        auto DepResult = DI->depends(&I, CurInst, true);
+        auto DepResult = DI->depends(&I, CurInst);
         if (DepResult && (DepResult->isOutput() || DepResult->isFlow() ||
                           DepResult->isAnti()))
           return true;
@@ -427,7 +427,7 @@ void llvm::moveInstructionsToTheBeginning(BasicBlock &FromBB, BasicBlock &ToBB,
                                           DependenceInfo &DI) {
   for (Instruction &I :
        llvm::make_early_inc_range(llvm::drop_begin(llvm::reverse(FromBB)))) {
-    Instruction *MovePos = ToBB.getFirstNonPHIOrDbg();
+    BasicBlock::iterator MovePos = ToBB.getFirstNonPHIOrDbg();
 
     if (isSafeToMoveBefore(I, *MovePos, DT, &PDT, &DI))
       I.moveBeforePreserving(MovePos);
diff --git a/llvm/lib/Transforms/Utils/IRNormalizer.cpp b/llvm/lib/Transforms/Utils/IRNormalizer.cpp
index 47ec7f3177db7..d36da331fedf7 100644
--- a/llvm/lib/Transforms/Utils/IRNormalizer.cpp
+++ b/llvm/lib/Transforms/Utils/IRNormalizer.cpp
@@ -475,7 +475,7 @@ void IRNormalizer::reorderInstructions(Function &F) const {
             Call->getIntrinsicID() == Intrinsic::experimental_convergence_loop)
           FirstNonPHIOrDbgOrAlloca++;
       }
-      Instruction->moveBefore(&*FirstNonPHIOrDbgOrAlloca);
+      Instruction->moveBefore(FirstNonPHIOrDbgOrAlloca);
       TopologicalSort.pop();
     }
   }
diff --git a/llvm/lib/Transforms/Utils/InlineFunction.cpp b/llvm/lib/Transforms/Utils/InlineFunction.cpp
index 6a8a468ebcae2..b92d8b16daad2 100644
--- a/llvm/lib/Transforms/Utils/InlineFunction.cpp
+++ b/llvm/lib/Transforms/Utils/InlineFunction.cpp
@@ -184,14 +184,14 @@ namespace {
 } // end anonymous namespace
 
 static IntrinsicInst *getConvergenceEntry(BasicBlock &BB) {
-  auto *I = BB.getFirstNonPHI();
-  while (I) {
-    if (auto *IntrinsicCall = dyn_cast<ConvergenceControlInst>(I)) {
+  BasicBlock::iterator It = BB.getFirstNonPHIIt();
+  while (It != BB.end()) {
+    if (auto *IntrinsicCall = dyn_cast<ConvergenceControlInst>(It)) {
       if (IntrinsicCall->isEntry()) {
         return IntrinsicCall;
       }
     }
-    I = I->getNextNode();
+    It = std::next(It);
   }
   return nullptr;
 }
@@ -1777,9 +1777,8 @@ static Value *HandleByValArgument(Type *ByValType, Value *Arg,
 // Check whether this Value is used by a lifetime intrinsic.
 static bool isUsedByLifetimeMarker(Value *V) {
   for (User *U : V->users())
-    if (IntrinsicInst *II = dyn_cast<IntrinsicInst>(U))
-      if (II->isLifetimeStartOrEnd())
-        return true;
+    if (isa<LifetimeIntrinsic>(U))
+      return true;
   return false;
 }
 
diff --git a/llvm/lib/Transforms/Utils/Local.cpp b/llvm/lib/Transforms/Utils/Local.cpp
index d5cf62e52cca3..2c6328300738f 100644
--- a/llvm/lib/Transforms/Utils/Local.cpp
+++ b/llvm/lib/Transforms/Utils/Local.cpp
@@ -497,10 +497,7 @@ bool llvm::wouldInstructionBeTriviallyDead(const Instruction *I,
       // are lifetime intrinsics then the intrinsics are dead.
       if (isa<AllocaInst>(Arg) || isa<GlobalValue>(Arg) || isa<Argument>(Arg))
         return llvm::all_of(Arg->uses(), [](Use &Use) {
-          if (IntrinsicInst *IntrinsicUse =
-                  dyn_cast<IntrinsicInst>(Use.getUser()))
-            return IntrinsicUse->isLifetimeStartOrEnd();
-          return false;
+          return isa<LifetimeIntrinsic>(Use.getUser());
         });
       return false;
     }
diff --git a/llvm/lib/Transforms/Utils/LoopUnroll.cpp b/llvm/lib/Transforms/Utils/LoopUnroll.cpp
index b11d92836a998..af6f9c2a8edd3 100644
--- a/llvm/lib/Transforms/Utils/LoopUnroll.cpp
+++ b/llvm/lib/Transforms/Utils/LoopUnroll.cpp
@@ -590,10 +590,11 @@ llvm::UnrollLoop(Loop *L, UnrollLoopOptions ULO, LoopInfo *LI,
                                               : isEpilogProfitable(L);
 
   if (ULO.Runtime &&
-      !UnrollRuntimeLoopRemainder(
-          L, ULO.Count, ULO.AllowExpensiveTripCount, EpilogProfitability,
-          ULO.UnrollRemainder, ULO.ForgetAllSCEV, LI, SE, DT, AC, TTI,
-          PreserveLCSSA, ULO.SCEVExpansionBudget, RemainderLoop)) {
+      !UnrollRuntimeLoopRemainder(L, ULO.Count, ULO.AllowExpensiveTripCount,
+                                  EpilogProfitability, ULO.UnrollRemainder,
+                                  ULO.ForgetAllSCEV, LI, SE, DT, AC, TTI,
+                                  PreserveLCSSA, ULO.SCEVExpansionBudget,
+                                  ULO.RuntimeUnrollMultiExit, RemainderLoop)) {
     if (ULO.Force)
       ULO.Runtime = false;
     else {
diff --git a/llvm/lib/Transforms/Utils/LoopUnrollAndJam.cpp b/llvm/lib/Transforms/Utils/LoopUnrollAndJam.cpp
index 49209e33f2d1d..89cfee2acd82c 100644
--- a/llvm/lib/Transforms/Utils/LoopUnrollAndJam.cpp
+++ b/llvm/lib/Transforms/Utils/LoopUnrollAndJam.cpp
@@ -165,7 +165,7 @@ static bool processHeaderPhiOperands(BasicBlock *Header, BasicBlock *Latch,
 // Move the phi operands of Header from Latch out of AftBlocks to InsertLoc.
 static void moveHeaderPhiOperandsToForeBlocks(BasicBlock *Header,
                                               BasicBlock *Latch,
-                                              Instruction *InsertLoc,
+                                              BasicBlock::iterator InsertLoc,
                                               BasicBlockSet &AftBlocks) {
   // We need to ensure we move the instructions in the correct order,
   // starting with the earliest required instruction and moving forward.
@@ -329,7 +329,8 @@ llvm::UnrollAndJamLoop(Loop *L, unsigned Count, unsigned TripCount,
 
   // Move any instructions from fore phi operands from AftBlocks into Fore.
   moveHeaderPhiOperandsToForeBlocks(
-      Header, LatchBlock, ForeBlocksLast[0]->getTerminator(), AftBlocks);
+      Header, LatchBlock, ForeBlocksLast[0]->getTerminator()->getIterator(),
+      AftBlocks);
 
   // The current on-the-fly SSA update requires blocks to be processed in
   // reverse postorder so that LastValueMap contains the correct value at each
@@ -708,7 +709,7 @@ static bool checkDependency(Instruction *Src, Instruction *Dst,
   //   (0,0,>=,*,*)
   // Now, the dependency is not necessarily non-negative anymore, i.e.
   // unroll-and-jam may violate correctness.
-  std::unique_ptr<Dependence> D = DI.depends(Src, Dst, true);
+  std::unique_ptr<Dependence> D = DI.depends(Src, Dst);
   if (!D)
     return true;
   assert(D->isOrdered() && "Expected an output, flow or anti dep.");
diff --git a/llvm/lib/Transforms/Utils/LoopUnrollRuntime.cpp b/llvm/lib/Transforms/Utils/LoopUnrollRuntime.cpp
index b0bc55cd64c37..524b268aee2f3 100644
--- a/llvm/lib/Transforms/Utils/LoopUnrollRuntime.cpp
+++ b/llvm/lib/Transforms/Utils/LoopUnrollRuntime.cpp
@@ -461,7 +461,7 @@ CloneLoopBlocks(Loop *L, Value *NewIter, const bool UseEpilogRemainder,
 
 /// Returns true if we can profitably unroll the multi-exit loop L. Currently,
 /// we return true only if UnrollRuntimeMultiExit is set to true.
-static bool canProfitablyUnrollMultiExitLoop(
+static bool canProfitablyRuntimeUnrollMultiExitLoop(
     Loop *L, SmallVectorImpl<BasicBlock *> &OtherExits, BasicBlock *LatchExit,
     bool UseEpilogRemainder) {
 
@@ -583,7 +583,8 @@ bool llvm::UnrollRuntimeLoopRemainder(
     bool UseEpilogRemainder, bool UnrollRemainder, bool ForgetAllSCEV,
     LoopInfo *LI, ScalarEvolution *SE, DominatorTree *DT, AssumptionCache *AC,
     const TargetTransformInfo *TTI, bool PreserveLCSSA,
-    unsigned SCEVExpansionBudget, Loop **ResultLoop) {
+    unsigned SCEVExpansionBudget, bool RuntimeUnrollMultiExit,
+    Loop **ResultLoop) {
   LLVM_DEBUG(dbgs() << "Trying runtime unrolling on Loop: \n");
   LLVM_DEBUG(L->dump());
   LLVM_DEBUG(UseEpilogRemainder ? dbgs() << "Using epilog remainder.\n"
@@ -632,8 +633,9 @@ bool llvm::UnrollRuntimeLoopRemainder(
     if (!PreserveLCSSA)
       return false;
 
-    if (!canProfitablyUnrollMultiExitLoop(L, OtherExits, LatchExit,
-                                          UseEpilogRemainder)) {
+    if (!RuntimeUnrollMultiExit &&
+        !canProfitablyRuntimeUnrollMultiExitLoop(L, OtherExits, LatchExit,
+                                                 UseEpilogRemainder)) {
       LLVM_DEBUG(
           dbgs()
           << "Multiple exit/exiting blocks in loop and multi-exit unrolling not "
diff --git a/llvm/lib/Transforms/Utils/MemoryTaggingSupport.cpp b/llvm/lib/Transforms/Utils/MemoryTaggingSupport.cpp
index cccb9dae17df6..de84a76ede7ff 100644
--- a/llvm/lib/Transforms/Utils/MemoryTaggingSupport.cpp
+++ b/llvm/lib/Transforms/Utils/MemoryTaggingSupport.cpp
@@ -154,9 +154,7 @@ void StackInfoBuilder::visit(OptimizationRemarkEmitter &ORE,
     }
     return;
   }
-  auto *II = dyn_cast<IntrinsicInst>(&Inst);
-  if (II && (II->getIntrinsicID() == Intrinsic::lifetime_start ||
-             II->getIntrinsicID() == Intrinsic::lifetime_end)) {
+  if (auto *II = dyn_cast<LifetimeIntrinsic>(&Inst)) {
     AllocaInst *AI = findAllocaForValue(II->getArgOperand(1));
     if (!AI) {
       Info.UnrecognizedLifetimes.push_back(&Inst);
@@ -261,11 +259,6 @@ void alignAndPadAlloca(memtag::AllocaInfo &Info, llvm::Align Alignment) {
   Info.AI = NewAI;
 }
 
-bool isLifetimeIntrinsic(Value *V) {
-  auto *II = dyn_cast<IntrinsicInst>(V);
-  return II && II->isLifetimeStartOrEnd();
-}
-
 Value *readRegister(IRBuilder<> &IRB, StringRef Name) {
   Module *M = IRB.GetInsertBlock()->getParent()->getParent();
   MDNode *MD =
diff --git a/llvm/lib/Transforms/Utils/ScalarEvolutionExpander.cpp b/llvm/lib/Transforms/Utils/ScalarEvolutionExpander.cpp
index 3a761bc4e8119..d429fe96f9bec 100644
--- a/llvm/lib/Transforms/Utils/ScalarEvolutionExpander.cpp
+++ b/llvm/lib/Transforms/Utils/ScalarEvolutionExpander.cpp
@@ -1451,7 +1451,7 @@ Value *SCEVExpander::expandCodeFor(const SCEV *SH, Type *Ty) {
   // Expand the code for this SCEV.
   Value *V = expand(SH);
 
-  if (Ty) {
+  if (Ty && Ty != V->getType()) {
     assert(SE.getTypeSizeInBits(Ty) == SE.getTypeSizeInBits(SH->getType()) &&
            "non-trivial casts should be done with the SCEVs directly!");
     V = InsertNoopCastOfTo(V, Ty);
diff --git a/llvm/lib/Transforms/Utils/SimplifyCFG.cpp b/llvm/lib/Transforms/Utils/SimplifyCFG.cpp
index cf3c2b360d090..27b7ec4629a26 100644
--- a/llvm/lib/Transforms/Utils/SimplifyCFG.cpp
+++ b/llvm/lib/Transforms/Utils/SimplifyCFG.cpp
@@ -2187,20 +2187,6 @@ bool SimplifyCFGOpt::hoistSuccIdenticalTerminatorToSwitchOrIf(
   return Changed;
 }
 
-// Check lifetime markers.
-static bool isLifeTimeMarker(const Instruction *I) {
-  if (auto II = dyn_cast<IntrinsicInst>(I)) {
-    switch (II->getIntrinsicID()) {
-    default:
-      break;
-    case Intrinsic::lifetime_start:
-    case Intrinsic::lifetime_end:
-      return true;
-    }
-  }
-  return false;
-}
-
 // TODO: Refine this. This should avoid cases like turning constant memcpy sizes
 // into variables.
 static bool replacingOperandWithVariableIsCheap(const Instruction *I,
@@ -2321,7 +2307,7 @@ static bool canSinkInstructions(
       // backend may handle such lifetimes incorrectly as well (#104776).
       // Don't sink lifetimes if it would introduce a phi on the pointer
       // argument.
-      if (isLifeTimeMarker(I0) && OI == 1 &&
+      if (isa<LifetimeIntrinsic>(I0) && OI == 1 &&
           any_of(Insts, [](const Instruction *I) {
             return isa<AllocaInst>(I->getOperand(1)->stripPointerCasts());
           }))
@@ -5262,8 +5248,8 @@ bool SimplifyCFGOpt::simplifyBranchOnICmpChain(BranchInst *BI,
 bool SimplifyCFGOpt::simplifyResume(ResumeInst *RI, IRBuilder<> &Builder) {
   if (isa<PHINode>(RI->getValue()))
     return simplifyCommonResume(RI);
-  else if (isa<LandingPadInst>(RI->getParent()->getFirstNonPHI()) &&
-           RI->getValue() == RI->getParent()->getFirstNonPHI())
+  else if (isa<LandingPadInst>(RI->getParent()->getFirstNonPHIIt()) &&
+           RI->getValue() == &*RI->getParent()->getFirstNonPHIIt())
     // The resume must unwind the exception that caused control to branch here.
     return simplifySingleResume(RI);
 
@@ -5297,8 +5283,8 @@ bool SimplifyCFGOpt::simplifyCommonResume(ResumeInst *RI) {
 
   // Check that there are no other instructions except for debug and lifetime
   // intrinsics between the phi's and resume instruction.
-  if (!isCleanupBlockEmpty(
-          make_range(RI->getParent()->getFirstNonPHI(), BB->getTerminator())))
+  if (!isCleanupBlockEmpty(make_range(RI->getParent()->getFirstNonPHIIt(),
+                                      BB->getTerminator()->getIterator())))
     return false;
 
   SmallSetVector<BasicBlock *, 4> TrivialUnwindBlocks;
@@ -5315,7 +5301,7 @@ bool SimplifyCFGOpt::simplifyCommonResume(ResumeInst *RI) {
     if (IncomingBB->getUniqueSuccessor() != BB)
       continue;
 
-    auto *LandingPad = dyn_cast<LandingPadInst>(IncomingBB->getFirstNonPHI());
+    auto *LandingPad = dyn_cast<LandingPadInst>(IncomingBB->getFirstNonPHIIt());
     // Not the landing pad that caused the control to branch here.
     if (IncomingValue != LandingPad)
       continue;
@@ -5364,7 +5350,7 @@ bool SimplifyCFGOpt::simplifyCommonResume(ResumeInst *RI) {
 // Simplify resume that is only used by a single (non-phi) landing pad.
 bool SimplifyCFGOpt::simplifySingleResume(ResumeInst *RI) {
   BasicBlock *BB = RI->getParent();
-  auto *LPInst = cast<LandingPadInst>(BB->getFirstNonPHI());
+  auto *LPInst = cast<LandingPadInst>(BB->getFirstNonPHIIt());
   assert(RI->getValue() == LPInst &&
          "Resume must unwind the exception that caused control to here");
 
@@ -5412,7 +5398,6 @@ static bool removeEmptyCleanup(CleanupReturnInst *RI, DomTreeUpdater *DTU) {
   // If the cleanup return we are simplifying unwinds to the caller, this will
   // set UnwindDest to nullptr.
   BasicBlock *UnwindDest = RI->getUnwindDest();
-  Instruction *DestEHPad = UnwindDest ? UnwindDest->getFirstNonPHI() : nullptr;
 
   // We're about to remove BB from the control flow.  Before we do, sink any
   // PHINodes into the unwind destination.  Doing this before changing the
@@ -5449,7 +5434,7 @@ static bool removeEmptyCleanup(CleanupReturnInst *RI, DomTreeUpdater *DTU) {
     }
 
     // Sink any remaining PHI nodes directly into UnwindDest.
-    Instruction *InsertPt = DestEHPad;
+    BasicBlock::iterator InsertPt = UnwindDest->getFirstNonPHIIt();
     for (PHINode &PN : make_early_inc_range(BB->phis())) {
       if (PN.use_empty() || !PN.isUsedOutsideOfBlock(BB))
         // If the PHI node has no uses or all of its uses are in this basic
@@ -6004,7 +5989,7 @@ static bool eliminateDeadSwitchCases(SwitchInst *SI, DomTreeUpdater *DTU,
 /// the phi node, and set PhiIndex to BB's index in the phi node.
 static PHINode *findPHIForConditionForwarding(ConstantInt *CaseValue,
                                               BasicBlock *BB, int *PhiIndex) {
-  if (BB->getFirstNonPHIOrDbg() != BB->getTerminator())
+  if (&*BB->getFirstNonPHIIt() != BB->getTerminator())
     return nullptr; // BB must be empty to be a candidate for simplification.
   if (!BB->getSinglePredecessor())
     return nullptr; // BB must be dominated by the switch.
@@ -7885,7 +7870,7 @@ bool SimplifyCFGOpt::simplifyUncondBranch(BranchInst *BI,
       Options.NeedCanonicalLoop &&
       (!LoopHeaders.empty() && BB->hasNPredecessorsOrMore(2) &&
        (is_contained(LoopHeaders, BB) || is_contained(LoopHeaders, Succ)));
-  BasicBlock::iterator I = BB->getFirstNonPHIOrDbg(true)->getIterator();
+  BasicBlock::iterator I = BB->getFirstNonPHIOrDbg();
   if (I->isTerminator() && BB != &BB->getParent()->getEntryBlock() &&
       !NeedCanonicalLoop && TryToSimplifyUncondBranchFromEmptyBlock(BB, DTU))
     return true;
@@ -8176,8 +8161,8 @@ static bool passingValueIsAlwaysUndefined(Value *V, Instruction *I, bool PtrValu
   if (C->isNullValue() || isa<UndefValue>(C)) {
     // Only look at the first use we can handle, avoid hurting compile time with
     // long uselists
-    auto FindUse = llvm::find_if(I->users(), [](auto *U) {
-      auto *Use = cast<Instruction>(U);
+    auto FindUse = llvm::find_if(I->uses(), [](auto &U) {
+      auto *Use = cast<Instruction>(U.getUser());
       // Change this list when we want to add new instructions.
       switch (Use->getOpcode()) {
       default:
@@ -8200,26 +8185,28 @@ static bool passingValueIsAlwaysUndefined(Value *V, Instruction *I, bool PtrValu
         return true;
       }
     });
-    if (FindUse == I->user_end())
+    if (FindUse == I->use_end())
       return false;
-    auto *Use = cast<Instruction>(*FindUse);
-    // Bail out if Use is not in the same BB as I or Use == I or Use comes
-    // before I in the block. The latter two can be the case if Use is a
+    auto &Use = *FindUse;
+    auto *User = cast<Instruction>(Use.getUser());
+    // Bail out if User is not in the same BB as I or User == I or User comes
+    // before I in the block. The latter two can be the case if User is a
     // PHI node.
-    if (Use->getParent() != I->getParent() || Use == I || Use->comesBefore(I))
+    if (User->getParent() != I->getParent() || User == I ||
+        User->comesBefore(I))
       return false;
 
     // Now make sure that there are no instructions in between that can alter
     // control flow (eg. calls)
     auto InstrRange =
-        make_range(std::next(I->getIterator()), Use->getIterator());
+        make_range(std::next(I->getIterator()), User->getIterator());
     if (any_of(InstrRange, [](Instruction &I) {
           return !isGuaranteedToTransferExecutionToSuccessor(&I);
         }))
       return false;
 
     // Look through GEPs. A load from a GEP derived from NULL is still undefined
-    if (GetElementPtrInst *GEP = dyn_cast<GetElementPtrInst>(Use))
+    if (GetElementPtrInst *GEP = dyn_cast<GetElementPtrInst>(User))
       if (GEP->getPointerOperand() == I) {
         // The current base address is null, there are four cases to consider:
         // getelementptr (TY, null, 0)                 -> null
@@ -8236,7 +8223,7 @@ static bool passingValueIsAlwaysUndefined(Value *V, Instruction *I, bool PtrValu
       }
 
     // Look through return.
-    if (ReturnInst *Ret = dyn_cast<ReturnInst>(Use)) {
+    if (ReturnInst *Ret = dyn_cast<ReturnInst>(User)) {
       bool HasNoUndefAttr =
           Ret->getFunction()->hasRetAttribute(Attribute::NoUndef);
       // Return undefined to a noundef return value is undefined.
@@ -8250,56 +8237,45 @@ static bool passingValueIsAlwaysUndefined(Value *V, Instruction *I, bool PtrValu
     }
 
     // Load from null is undefined.
-    if (LoadInst *LI = dyn_cast<LoadInst>(Use))
+    if (LoadInst *LI = dyn_cast<LoadInst>(User))
       if (!LI->isVolatile())
         return !NullPointerIsDefined(LI->getFunction(),
                                      LI->getPointerAddressSpace());
 
     // Store to null is undefined.
-    if (StoreInst *SI = dyn_cast<StoreInst>(Use))
+    if (StoreInst *SI = dyn_cast<StoreInst>(User))
       if (!SI->isVolatile())
         return (!NullPointerIsDefined(SI->getFunction(),
                                       SI->getPointerAddressSpace())) &&
                SI->getPointerOperand() == I;
 
     // llvm.assume(false/undef) always triggers immediate UB.
-    if (auto *Assume = dyn_cast<AssumeInst>(Use)) {
+    if (auto *Assume = dyn_cast<AssumeInst>(User)) {
       // Ignore assume operand bundles.
       if (I == Assume->getArgOperand(0))
         return true;
     }
 
-    if (auto *CB = dyn_cast<CallBase>(Use)) {
+    if (auto *CB = dyn_cast<CallBase>(User)) {
       if (C->isNullValue() && NullPointerIsDefined(CB->getFunction()))
         return false;
       // A call to null is undefined.
       if (CB->getCalledOperand() == I)
         return true;
 
-      if (C->isNullValue()) {
-        for (const llvm::Use &Arg : CB->args())
-          if (Arg == I) {
-            unsigned ArgIdx = CB->getArgOperandNo(&Arg);
-            if (CB->isPassingUndefUB(ArgIdx) &&
-                CB->paramHasAttr(ArgIdx, Attribute::NonNull)) {
-              // Passing null to a nonnnull+noundef argument is undefined.
-              return !PtrValueMayBeModified;
-            }
-          }
-      } else if (isa<UndefValue>(C)) {
+      if (CB->isArgOperand(&Use)) {
+        unsigned ArgIdx = CB->getArgOperandNo(&Use);
+        // Passing null to a nonnnull+noundef argument is undefined.
+        if (C->isNullValue() && CB->isPassingUndefUB(ArgIdx) &&
+            CB->paramHasAttr(ArgIdx, Attribute::NonNull))
+          return !PtrValueMayBeModified;
         // Passing undef to a noundef argument is undefined.
-        for (const llvm::Use &Arg : CB->args())
-          if (Arg == I) {
-            unsigned ArgIdx = CB->getArgOperandNo(&Arg);
-            if (CB->isPassingUndefUB(ArgIdx)) {
-              // Passing undef to a noundef argument is undefined.
-              return true;
-            }
-          }
+        if (isa<UndefValue>(C) && CB->isPassingUndefUB(ArgIdx))
+          return true;
       }
     }
     // Div/Rem by zero is immediate UB
-    if (match(Use, m_BinOp(m_Value(), m_Specific(I))) && Use->isIntDivRem())
+    if (match(User, m_BinOp(m_Value(), m_Specific(I))) && User->isIntDivRem())
       return true;
   }
   return false;
diff --git a/llvm/lib/Transforms/Utils/SimplifyIndVar.cpp b/llvm/lib/Transforms/Utils/SimplifyIndVar.cpp
index f05d32d980e5a..7b9c5c77cbe98 100644
--- a/llvm/lib/Transforms/Utils/SimplifyIndVar.cpp
+++ b/llvm/lib/Transforms/Utils/SimplifyIndVar.cpp
@@ -205,12 +205,12 @@ bool SimplifyIndvar::makeIVComparisonInvariant(ICmpInst *ICmp,
   if (!Preheader)
     return false;
   unsigned IVOperIdx = 0;
-  ICmpInst::Predicate Pred = ICmp->getPredicate();
+  CmpPredicate Pred = ICmp->getCmpPredicate();
   if (IVOperand != ICmp->getOperand(0)) {
     // Swapped
     assert(IVOperand == ICmp->getOperand(1) && "Can't find IVOperand");
     IVOperIdx = 1;
-    Pred = ICmpInst::getSwappedPredicate(Pred);
+    Pred = ICmpInst::getSwappedCmpPredicate(Pred);
   }
 
   // Get the SCEVs for the ICmp operands (in the specific context of the
@@ -249,13 +249,13 @@ bool SimplifyIndvar::makeIVComparisonInvariant(ICmpInst *ICmp,
 void SimplifyIndvar::eliminateIVComparison(ICmpInst *ICmp,
                                            Instruction *IVOperand) {
   unsigned IVOperIdx = 0;
-  ICmpInst::Predicate Pred = ICmp->getPredicate();
+  CmpPredicate Pred = ICmp->getCmpPredicate();
   ICmpInst::Predicate OriginalPred = Pred;
   if (IVOperand != ICmp->getOperand(0)) {
     // Swapped
     assert(IVOperand == ICmp->getOperand(1) && "Can't find IVOperand");
     IVOperIdx = 1;
-    Pred = ICmpInst::getSwappedPredicate(Pred);
+    Pred = ICmpInst::getSwappedCmpPredicate(Pred);
   }
 
   // Get the SCEVs for the ICmp operands (in the specific context of the
@@ -1614,7 +1614,8 @@ bool WidenIV::widenLoopCompare(WidenIV::NarrowIVDefUse DU) {
   //      (A) == icmp slt i32 sext(%narrow), sext(%val)
   //          == icmp slt i32 zext(%narrow), sext(%val)
   bool IsSigned = getExtendKind(DU.NarrowDef) == ExtendKind::Sign;
-  if (!(DU.NeverNegative || IsSigned == Cmp->isSigned()))
+  bool CmpPreferredSign = Cmp->hasSameSign() ? IsSigned : Cmp->isSigned();
+  if (!DU.NeverNegative && IsSigned != CmpPreferredSign)
     return false;
 
   Value *Op = Cmp->getOperand(Cmp->getOperand(0) == DU.NarrowDef ? 1 : 0);
@@ -1627,7 +1628,7 @@ bool WidenIV::widenLoopCompare(WidenIV::NarrowIVDefUse DU) {
 
   // Widen the other operand of the compare, if necessary.
   if (CastWidth < IVWidth) {
-    Value *ExtOp = createExtendInst(Op, WideType, Cmp->isSigned(), Cmp);
+    Value *ExtOp = createExtendInst(Op, WideType, CmpPreferredSign, Cmp);
     DU.NarrowUse->replaceUsesOfWith(Op, ExtOp);
   }
   return true;
diff --git a/llvm/lib/Transforms/Utils/SimplifyLibCalls.cpp b/llvm/lib/Transforms/Utils/SimplifyLibCalls.cpp
index d3a80d12f4693..d38a549aab960 100644
--- a/llvm/lib/Transforms/Utils/SimplifyLibCalls.cpp
+++ b/llvm/lib/Transforms/Utils/SimplifyLibCalls.cpp
@@ -1094,7 +1094,8 @@ Value *LibCallSimplifier::optimizeStrTo(CallInst *CI, IRBuilderBase &B) {
   if (isa<ConstantPointerNull>(EndPtr)) {
     // With a null EndPtr, this function won't capture the main argument.
     // It would be readonly too, except that it still may write to errno.
-    CI->addParamAttr(0, Attribute::NoCapture);
+    CI->addParamAttr(0, Attribute::getWithCaptureInfo(CI->getContext(),
+                                                      CaptureInfo::none()));
   }
 
   return nullptr;
@@ -3205,7 +3206,8 @@ Value *LibCallSimplifier::optimizeStrToInt(CallInst *CI, IRBuilderBase &B,
   if (isa<ConstantPointerNull>(EndPtr)) {
     // With a null EndPtr, this function won't capture the main argument.
     // It would be readonly too, except that it still may write to errno.
-    CI->addParamAttr(0, Attribute::NoCapture);
+    CI->addParamAttr(0, Attribute::getWithCaptureInfo(CI->getContext(),
+                                                      CaptureInfo::none()));
     EndPtr = nullptr;
   } else if (!isKnownNonZero(EndPtr, DL))
     return nullptr;
diff --git a/llvm/lib/Transforms/Utils/VNCoercion.cpp b/llvm/lib/Transforms/Utils/VNCoercion.cpp
index 7a61ab7416638..c1bce01239dcc 100644
--- a/llvm/lib/Transforms/Utils/VNCoercion.cpp
+++ b/llvm/lib/Transforms/Utils/VNCoercion.cpp
@@ -15,30 +15,42 @@ static bool isFirstClassAggregateOrScalableType(Type *Ty) {
 
 /// Return true if coerceAvailableValueToLoadType will succeed.
 bool canCoerceMustAliasedValueToLoad(Value *StoredVal, Type *LoadTy,
-                                     const DataLayout &DL) {
+                                     Function *F) {
   Type *StoredTy = StoredVal->getType();
-
   if (StoredTy == LoadTy)
     return true;
 
+  const DataLayout &DL = F->getDataLayout();
+  TypeSize MinStoreSize = DL.getTypeSizeInBits(StoredTy);
+  TypeSize LoadSize = DL.getTypeSizeInBits(LoadTy);
   if (isa<ScalableVectorType>(StoredTy) && isa<ScalableVectorType>(LoadTy) &&
-      DL.getTypeSizeInBits(StoredTy) == DL.getTypeSizeInBits(LoadTy))
+      MinStoreSize == LoadSize)
     return true;
 
-  // If the loaded/stored value is a first class array/struct, or scalable type,
-  // don't try to transform them. We need to be able to bitcast to integer.
-  if (isFirstClassAggregateOrScalableType(LoadTy) ||
-      isFirstClassAggregateOrScalableType(StoredTy))
+  // If the loaded/stored value is a first class array/struct, don't try to
+  // transform them. We need to be able to bitcast to integer. For scalable
+  // vectors forwarded to fixed-sized vectors @llvm.vector.extract is used.
+  if (isa<ScalableVectorType>(StoredTy) && isa<FixedVectorType>(LoadTy)) {
+    if (StoredTy->getScalarType() != LoadTy->getScalarType())
+      return false;
+
+    // If it is known at compile-time that the VScale is larger than one,
+    // use that information to allow for wider loads.
+    const auto &Attrs = F->getAttributes().getFnAttrs();
+    unsigned MinVScale = Attrs.getVScaleRangeMin();
+    MinStoreSize =
+        TypeSize::getFixed(MinStoreSize.getKnownMinValue() * MinVScale);
+  } else if (isFirstClassAggregateOrScalableType(LoadTy) ||
+             isFirstClassAggregateOrScalableType(StoredTy)) {
     return false;
-
-  uint64_t StoreSize = DL.getTypeSizeInBits(StoredTy).getFixedValue();
+  }
 
   // The store size must be byte-aligned to support future type casts.
-  if (llvm::alignTo(StoreSize, 8) != StoreSize)
+  if (llvm::alignTo(MinStoreSize, 8) != MinStoreSize)
     return false;
 
   // The store has to be at least as big as the load.
-  if (StoreSize < DL.getTypeSizeInBits(LoadTy).getFixedValue())
+  if (!TypeSize::isKnownGE(MinStoreSize, LoadSize))
     return false;
 
   bool StoredNI = DL.isNonIntegralPointerType(StoredTy->getScalarType());
@@ -57,11 +69,10 @@ bool canCoerceMustAliasedValueToLoad(Value *StoredVal, Type *LoadTy,
     return false;
   }
 
-
   // The implementation below uses inttoptr for vectors of unequal size; we
   // can't allow this for non integral pointers. We could teach it to extract
   // exact subvectors if desired.
-  if (StoredNI && StoreSize != DL.getTypeSizeInBits(LoadTy).getFixedValue())
+  if (StoredNI && (StoredTy->isScalableTy() || MinStoreSize != LoadSize))
     return false;
 
   if (StoredTy->isTargetExtTy() || LoadTy->isTargetExtTy())
@@ -77,16 +88,24 @@ bool canCoerceMustAliasedValueToLoad(Value *StoredVal, Type *LoadTy,
 ///
 /// If we can't do it, return null.
 Value *coerceAvailableValueToLoadType(Value *StoredVal, Type *LoadedTy,
-                                      IRBuilderBase &Helper,
-                                      const DataLayout &DL) {
-  assert(canCoerceMustAliasedValueToLoad(StoredVal, LoadedTy, DL) &&
+                                      IRBuilderBase &Helper, Function *F) {
+  assert(canCoerceMustAliasedValueToLoad(StoredVal, LoadedTy, F) &&
          "precondition violation - materialization can't fail");
+  const DataLayout &DL = F->getDataLayout();
   if (auto *C = dyn_cast<Constant>(StoredVal))
     StoredVal = ConstantFoldConstant(C, DL);
 
   // If this is already the right type, just return it.
   Type *StoredValTy = StoredVal->getType();
 
+  // If this is a scalable vector forwarded to a fixed vector load, create
+  // a @llvm.vector.extract instead of bitcasts.
+  if (isa<ScalableVectorType>(StoredVal->getType()) &&
+      isa<FixedVectorType>(LoadedTy)) {
+    return Helper.CreateIntrinsic(LoadedTy, Intrinsic::vector_extract,
+                                  {StoredVal, Helper.getInt64(0)});
+  }
+
   TypeSize StoredValSize = DL.getTypeSizeInBits(StoredValTy);
   TypeSize LoadedValSize = DL.getTypeSizeInBits(LoadedTy);
 
@@ -220,7 +239,7 @@ int analyzeLoadFromClobberingStore(Type *LoadTy, Value *LoadPtr,
   if (isFirstClassAggregateOrScalableType(StoredVal->getType()))
     return -1;
 
-  if (!canCoerceMustAliasedValueToLoad(StoredVal, LoadTy, DL))
+  if (!canCoerceMustAliasedValueToLoad(StoredVal, LoadTy, DepSI->getFunction()))
     return -1;
 
   Value *StorePtr = DepSI->getPointerOperand();
@@ -235,11 +254,11 @@ int analyzeLoadFromClobberingStore(Type *LoadTy, Value *LoadPtr,
 /// the other load can feed into the second load.
 int analyzeLoadFromClobberingLoad(Type *LoadTy, Value *LoadPtr, LoadInst *DepLI,
                                   const DataLayout &DL) {
-  // Cannot handle reading from store of first-class aggregate yet.
-  if (DepLI->getType()->isStructTy() || DepLI->getType()->isArrayTy())
+  // Cannot handle reading from store of first-class aggregate or scalable type.
+  if (isFirstClassAggregateOrScalableType(DepLI->getType()))
     return -1;
 
-  if (!canCoerceMustAliasedValueToLoad(DepLI, LoadTy, DL))
+  if (!canCoerceMustAliasedValueToLoad(DepLI, LoadTy, DepLI->getFunction()))
     return -1;
 
   Value *DepPtr = DepLI->getPointerOperand();
@@ -315,6 +334,16 @@ static Value *getStoreValueForLoadHelper(Value *SrcVal, unsigned Offset,
     return SrcVal;
   }
 
+  // For the case of a scalable vector being forwarded to a fixed-sized load,
+  // only equal element types are allowed and a @llvm.vector.extract will be
+  // used instead of bitcasts.
+  if (isa<ScalableVectorType>(SrcVal->getType()) &&
+      isa<FixedVectorType>(LoadTy)) {
+    assert(Offset == 0 &&
+           SrcVal->getType()->getScalarType() == LoadTy->getScalarType());
+    return SrcVal;
+  }
+
   uint64_t StoreSize =
       (DL.getTypeSizeInBits(SrcVal->getType()).getFixedValue() + 7) / 8;
   uint64_t LoadSize = (DL.getTypeSizeInBits(LoadTy).getFixedValue() + 7) / 8;
@@ -344,20 +373,24 @@ static Value *getStoreValueForLoadHelper(Value *SrcVal, unsigned Offset,
 }
 
 Value *getValueForLoad(Value *SrcVal, unsigned Offset, Type *LoadTy,
-                       Instruction *InsertPt, const DataLayout &DL) {
+                       Instruction *InsertPt, Function *F) {
+  const DataLayout &DL = F->getDataLayout();
 #ifndef NDEBUG
-  TypeSize SrcValSize = DL.getTypeStoreSize(SrcVal->getType());
+  TypeSize MinSrcValSize = DL.getTypeStoreSize(SrcVal->getType());
   TypeSize LoadSize = DL.getTypeStoreSize(LoadTy);
-  assert(SrcValSize.isScalable() == LoadSize.isScalable());
-  assert((SrcValSize.isScalable() || Offset + LoadSize <= SrcValSize) &&
+  if (MinSrcValSize.isScalable() && !LoadSize.isScalable())
+    MinSrcValSize =
+        TypeSize::getFixed(MinSrcValSize.getKnownMinValue() *
+                           F->getAttributes().getFnAttrs().getVScaleRangeMin());
+  assert((MinSrcValSize.isScalable() || Offset + LoadSize <= MinSrcValSize) &&
          "Expected Offset + LoadSize <= SrcValSize");
-  assert(
-      (!SrcValSize.isScalable() || (Offset == 0 && LoadSize == SrcValSize)) &&
-      "Expected scalable type sizes to match");
+  assert((!MinSrcValSize.isScalable() ||
+          (Offset == 0 && TypeSize::isKnownLE(LoadSize, MinSrcValSize))) &&
+         "Expected offset of zero and LoadSize <= SrcValSize");
 #endif
   IRBuilder<> Builder(InsertPt);
   SrcVal = getStoreValueForLoadHelper(SrcVal, Offset, LoadTy, Builder, DL);
-  return coerceAvailableValueToLoadType(SrcVal, LoadTy, Builder, DL);
+  return coerceAvailableValueToLoadType(SrcVal, LoadTy, Builder, F);
 }
 
 Constant *getConstantValueForLoad(Constant *SrcVal, unsigned Offset,
@@ -408,7 +441,8 @@ Value *getMemInstValueForLoad(MemIntrinsic *SrcInst, unsigned Offset,
       ++NumBytesSet;
     }
 
-    return coerceAvailableValueToLoadType(Val, LoadTy, Builder, DL);
+    return coerceAvailableValueToLoadType(Val, LoadTy, Builder,
+                                          InsertPt->getFunction());
   }
 
   // Otherwise, this is a memcpy/memmove from a constant global.
diff --git a/llvm/lib/Transforms/Vectorize/CMakeLists.txt b/llvm/lib/Transforms/Vectorize/CMakeLists.txt
index e5fabd318b82c..872e055294d55 100644
--- a/llvm/lib/Transforms/Vectorize/CMakeLists.txt
+++ b/llvm/lib/Transforms/Vectorize/CMakeLists.txt
@@ -9,6 +9,7 @@ add_llvm_component_library(LLVMVectorize
   SandboxVectorizer/Legality.cpp
   SandboxVectorizer/Passes/BottomUpVec.cpp
   SandboxVectorizer/Passes/RegionsFromMetadata.cpp
+  SandboxVectorizer/Passes/TransactionAcceptOrRevert.cpp
   SandboxVectorizer/SandboxVectorizer.cpp
   SandboxVectorizer/SandboxVectorizerPassBuilder.cpp
   SandboxVectorizer/Scheduler.cpp
diff --git a/llvm/lib/Transforms/Vectorize/LoadStoreVectorizer.cpp b/llvm/lib/Transforms/Vectorize/LoadStoreVectorizer.cpp
index 0b40b733ffe7e..04b392829f0d7 100644
--- a/llvm/lib/Transforms/Vectorize/LoadStoreVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoadStoreVectorizer.cpp
@@ -1335,8 +1335,9 @@ void Vectorizer::mergeEquivalenceClasses(EquivalenceClassMap &EQClasses) const {
     const auto &Key = EC.first;
     EqClassReducedKey RedKey{std::get<1>(Key), std::get<2>(Key),
                              std::get<3>(Key)};
-    RedKeyToUOMap[RedKey].insert(std::get<0>(Key));
-    if (RedKeyToUOMap[RedKey].size() > 1)
+    auto &UOMap = RedKeyToUOMap[RedKey];
+    UOMap.insert(std::get<0>(Key));
+    if (UOMap.size() > 1)
       FoundPotentiallyOptimizableEC = true;
   }
   if (!FoundPotentiallyOptimizableEC)
diff --git a/llvm/lib/Transforms/Vectorize/LoopIdiomVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopIdiomVectorize.cpp
index 7af7408ed67a8..90329200dd7e4 100644
--- a/llvm/lib/Transforms/Vectorize/LoopIdiomVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopIdiomVectorize.cpp
@@ -10,8 +10,10 @@
 // transforms them into more optimized versions of the same loop. In cases
 // where this happens, it can be a significant performance win.
 //
-// We currently only recognize one loop that finds the first mismatched byte
-// in an array and returns the index, i.e. something like:
+// We currently support two loops:
+//
+// 1. A loop that finds the first mismatched byte in an array and returns the
+// index, i.e. something like:
 //
 //  while (++i != n) {
 //    if (a[i] != b[i])
@@ -24,12 +26,6 @@
 // boundaries. However, even with these checks it is still profitable to do the
 // transformation.
 //
-//===----------------------------------------------------------------------===//
-//
-// NOTE: This Pass matches a really specific loop pattern because it's only
-// supposed to be a temporary solution until our LoopVectorizer is powerful
-// enought to vectorize it automatically.
-//
 // TODO List:
 //
 // * Add support for the inverse case where we scan for a matching element.
@@ -37,6 +33,35 @@
 // * Recognize loops that increment the IV *after* comparing bytes.
 // * Allow 32-bit sign-extends of the IV used by the GEP.
 //
+// 2. A loop that finds the first matching character in an array among a set of
+// possible matches, e.g.:
+//
+//   for (; first != last; ++first)
+//     for (s_it = s_first; s_it != s_last; ++s_it)
+//       if (*first == *s_it)
+//         return first;
+//   return last;
+//
+// This corresponds to std::find_first_of (for arrays of bytes) from the C++
+// standard library. This function can be implemented efficiently for targets
+// that support @llvm.experimental.vector.match. For example, on AArch64 targets
+// that implement SVE2, this lower to a MATCH instruction, which enables us to
+// perform up to 16x16=256 comparisons in one go. This can lead to very
+// significant speedups.
+//
+// TODO:
+//
+// * Add support for `find_first_not_of' loops (i.e. with not-equal comparison).
+// * Make VF a configurable parameter (right now we assume 128-bit vectors).
+// * Potentially adjust the cost model to let the transformation kick-in even if
+//   @llvm.experimental.vector.match doesn't have direct support in hardware.
+//
+//===----------------------------------------------------------------------===//
+//
+// NOTE: This Pass matches really specific loop patterns because it's only
+// supposed to be a temporary solution until our LoopVectorizer is powerful
+// enough to vectorize them automatically.
+//
 //===----------------------------------------------------------------------===//
 
 #include "llvm/Transforms/Vectorize/LoopIdiomVectorize.h"
@@ -79,6 +104,11 @@ static cl::opt<unsigned>
               cl::desc("The vectorization factor for byte-compare patterns."),
               cl::init(16));
 
+static cl::opt<bool>
+    DisableFindFirstByte("disable-loop-idiom-vectorize-find-first-byte",
+                         cl::Hidden, cl::init(false),
+                         cl::desc("Do not convert find-first-byte loop(s)."));
+
 static cl::opt<bool>
     VerifyLoops("loop-idiom-vectorize-verify", cl::Hidden, cl::init(false),
                 cl::desc("Verify loops generated Loop Idiom Vectorize Pass."));
@@ -136,6 +166,19 @@ class LoopIdiomVectorize {
                             PHINode *IndPhi, Value *MaxLen, Instruction *Index,
                             Value *Start, bool IncIdx, BasicBlock *FoundBB,
                             BasicBlock *EndBB);
+
+  bool recognizeFindFirstByte();
+
+  Value *expandFindFirstByte(IRBuilder<> &Builder, DomTreeUpdater &DTU,
+                             unsigned VF, Type *CharTy, BasicBlock *ExitSucc,
+                             BasicBlock *ExitFail, Value *SearchStart,
+                             Value *SearchEnd, Value *NeedleStart,
+                             Value *NeedleEnd);
+
+  void transformFindFirstByte(PHINode *IndPhi, unsigned VF, Type *CharTy,
+                              BasicBlock *ExitSucc, BasicBlock *ExitFail,
+                              Value *SearchStart, Value *SearchEnd,
+                              Value *NeedleStart, Value *NeedleEnd);
   /// @}
 };
 } // anonymous namespace
@@ -190,7 +233,13 @@ bool LoopIdiomVectorize::run(Loop *L) {
   LLVM_DEBUG(dbgs() << DEBUG_TYPE " Scanning: F[" << F.getName() << "] Loop %"
                     << CurLoop->getHeader()->getName() << "\n");
 
-  return recognizeByteCompare();
+  if (recognizeByteCompare())
+    return true;
+
+  if (recognizeFindFirstByte())
+    return true;
+
+  return false;
 }
 
 bool LoopIdiomVectorize::recognizeByteCompare() {
@@ -939,3 +988,432 @@ void LoopIdiomVectorize::transformByteCompare(GetElementPtrInst *GEPA,
       report_fatal_error("Loops must remain in LCSSA form!");
   }
 }
+
+bool LoopIdiomVectorize::recognizeFindFirstByte() {
+  // Currently the transformation only works on scalable vector types, although
+  // there is no fundamental reason why it cannot be made to work for fixed
+  // vectors. We also need to know the target's minimum page size in order to
+  // generate runtime memory checks to ensure the vector version won't fault.
+  if (!TTI->supportsScalableVectors() || !TTI->getMinPageSize().has_value() ||
+      DisableFindFirstByte)
+    return false;
+
+  // Define some constants we need throughout.
+  BasicBlock *Header = CurLoop->getHeader();
+  LLVMContext &Ctx = Header->getContext();
+
+  // We are expecting the four blocks defined below: Header, MatchBB, InnerBB,
+  // and OuterBB. For now, we will bail our for almost anything else. The Four
+  // blocks contain one nested loop.
+  if (CurLoop->getNumBackEdges() != 1 || CurLoop->getNumBlocks() != 4 ||
+      CurLoop->getSubLoops().size() != 1)
+    return false;
+
+  auto *InnerLoop = CurLoop->getSubLoops().front();
+  PHINode *IndPhi = dyn_cast<PHINode>(&Header->front());
+  if (!IndPhi || IndPhi->getNumIncomingValues() != 2)
+    return false;
+
+  // Check instruction counts.
+  auto LoopBlocks = CurLoop->getBlocks();
+  if (LoopBlocks[0]->sizeWithoutDebug() > 3 ||
+      LoopBlocks[1]->sizeWithoutDebug() > 4 ||
+      LoopBlocks[2]->sizeWithoutDebug() > 3 ||
+      LoopBlocks[3]->sizeWithoutDebug() > 3)
+    return false;
+
+  // Check that no instruction other than IndPhi has outside uses.
+  for (BasicBlock *BB : LoopBlocks)
+    for (Instruction &I : *BB)
+      if (&I != IndPhi)
+        for (User *U : I.users())
+          if (!CurLoop->contains(cast<Instruction>(U)))
+            return false;
+
+  // Match the branch instruction in the header. We are expecting an
+  // unconditional branch to the inner loop.
+  //
+  // Header:
+  //   %14 = phi ptr [ %24, %OuterBB ], [ %3, %Header.preheader ]
+  //   %15 = load i8, ptr %14, align 1
+  //   br label %MatchBB
+  BasicBlock *MatchBB;
+  if (!match(Header->getTerminator(), m_UnconditionalBr(MatchBB)) ||
+      !InnerLoop->contains(MatchBB))
+    return false;
+
+  // MatchBB should be the entrypoint into the inner loop containing the
+  // comparison between a search element and a needle.
+  //
+  // MatchBB:
+  //   %20 = phi ptr [ %7, %Header ], [ %17, %InnerBB ]
+  //   %21 = load i8, ptr %20, align 1
+  //   %22 = icmp eq i8 %15, %21
+  //   br i1 %22, label %ExitSucc, label %InnerBB
+  BasicBlock *ExitSucc, *InnerBB;
+  Value *LoadSearch, *LoadNeedle;
+  CmpPredicate MatchPred;
+  if (!match(MatchBB->getTerminator(),
+             m_Br(m_ICmp(MatchPred, m_Value(LoadSearch), m_Value(LoadNeedle)),
+                  m_BasicBlock(ExitSucc), m_BasicBlock(InnerBB))) ||
+      MatchPred != ICmpInst::ICMP_EQ || !InnerLoop->contains(InnerBB))
+    return false;
+
+  // We expect outside uses of `IndPhi' in ExitSucc (and only there).
+  for (User *U : IndPhi->users())
+    if (!CurLoop->contains(cast<Instruction>(U))) {
+      auto *PN = dyn_cast<PHINode>(U);
+      if (!PN || PN->getParent() != ExitSucc)
+        return false;
+    }
+
+  // Match the loads and check they are simple.
+  Value *Search, *Needle;
+  if (!match(LoadSearch, m_Load(m_Value(Search))) ||
+      !match(LoadNeedle, m_Load(m_Value(Needle))) ||
+      !cast<LoadInst>(LoadSearch)->isSimple() ||
+      !cast<LoadInst>(LoadNeedle)->isSimple())
+    return false;
+
+  // Check we are loading valid characters.
+  Type *CharTy = LoadSearch->getType();
+  if (!CharTy->isIntegerTy() || LoadNeedle->getType() != CharTy)
+    return false;
+
+  // Pick the vectorisation factor based on CharTy, work out the cost of the
+  // match intrinsic and decide if we should use it.
+  // Note: For the time being we assume 128-bit vectors.
+  unsigned VF = 128 / CharTy->getIntegerBitWidth();
+  SmallVector<Type *> Args = {
+      ScalableVectorType::get(CharTy, VF), FixedVectorType::get(CharTy, VF),
+      ScalableVectorType::get(Type::getInt1Ty(Ctx), VF)};
+  IntrinsicCostAttributes Attrs(Intrinsic::experimental_vector_match, Args[2],
+                                Args);
+  if (TTI->getIntrinsicInstrCost(Attrs, TTI::TCK_SizeAndLatency) > 4)
+    return false;
+
+  // The loads come from two PHIs, each with two incoming values.
+  PHINode *PSearch = dyn_cast<PHINode>(Search);
+  PHINode *PNeedle = dyn_cast<PHINode>(Needle);
+  if (!PSearch || PSearch->getNumIncomingValues() != 2 || !PNeedle ||
+      PNeedle->getNumIncomingValues() != 2)
+    return false;
+
+  // One PHI comes from the outer loop (PSearch), the other one from the inner
+  // loop (PNeedle). PSearch effectively corresponds to IndPhi.
+  if (InnerLoop->contains(PSearch))
+    std::swap(PSearch, PNeedle);
+  if (PSearch != &Header->front() || PNeedle != &MatchBB->front())
+    return false;
+
+  // The incoming values of both PHI nodes should be a gep of 1.
+  Value *SearchStart = PSearch->getIncomingValue(0);
+  Value *SearchIndex = PSearch->getIncomingValue(1);
+  if (CurLoop->contains(PSearch->getIncomingBlock(0)))
+    std::swap(SearchStart, SearchIndex);
+
+  Value *NeedleStart = PNeedle->getIncomingValue(0);
+  Value *NeedleIndex = PNeedle->getIncomingValue(1);
+  if (InnerLoop->contains(PNeedle->getIncomingBlock(0)))
+    std::swap(NeedleStart, NeedleIndex);
+
+  // Match the GEPs.
+  if (!match(SearchIndex, m_GEP(m_Specific(PSearch), m_One())) ||
+      !match(NeedleIndex, m_GEP(m_Specific(PNeedle), m_One())))
+    return false;
+
+  // Check the GEPs result type matches `CharTy'.
+  GetElementPtrInst *GEPSearch = cast<GetElementPtrInst>(SearchIndex);
+  GetElementPtrInst *GEPNeedle = cast<GetElementPtrInst>(NeedleIndex);
+  if (GEPSearch->getResultElementType() != CharTy ||
+      GEPNeedle->getResultElementType() != CharTy)
+    return false;
+
+  // InnerBB should increment the address of the needle pointer.
+  //
+  // InnerBB:
+  //   %17 = getelementptr inbounds i8, ptr %20, i64 1
+  //   %18 = icmp eq ptr %17, %10
+  //   br i1 %18, label %OuterBB, label %MatchBB
+  BasicBlock *OuterBB;
+  Value *NeedleEnd;
+  if (!match(InnerBB->getTerminator(),
+             m_Br(m_SpecificICmp(ICmpInst::ICMP_EQ, m_Specific(GEPNeedle),
+                                 m_Value(NeedleEnd)),
+                  m_BasicBlock(OuterBB), m_Specific(MatchBB))) ||
+      !CurLoop->contains(OuterBB))
+    return false;
+
+  // OuterBB should increment the address of the search element pointer.
+  //
+  // OuterBB:
+  //   %24 = getelementptr inbounds i8, ptr %14, i64 1
+  //   %25 = icmp eq ptr %24, %6
+  //   br i1 %25, label %ExitFail, label %Header
+  BasicBlock *ExitFail;
+  Value *SearchEnd;
+  if (!match(OuterBB->getTerminator(),
+             m_Br(m_SpecificICmp(ICmpInst::ICMP_EQ, m_Specific(GEPSearch),
+                                 m_Value(SearchEnd)),
+                  m_BasicBlock(ExitFail), m_Specific(Header))))
+    return false;
+
+  if (!CurLoop->isLoopInvariant(SearchStart) ||
+      !CurLoop->isLoopInvariant(SearchEnd) ||
+      !CurLoop->isLoopInvariant(NeedleStart) ||
+      !CurLoop->isLoopInvariant(NeedleEnd))
+    return false;
+
+  LLVM_DEBUG(dbgs() << "Found idiom in loop: \n" << *CurLoop << "\n\n");
+
+  transformFindFirstByte(IndPhi, VF, CharTy, ExitSucc, ExitFail, SearchStart,
+                         SearchEnd, NeedleStart, NeedleEnd);
+  return true;
+}
+
+Value *LoopIdiomVectorize::expandFindFirstByte(
+    IRBuilder<> &Builder, DomTreeUpdater &DTU, unsigned VF, Type *CharTy,
+    BasicBlock *ExitSucc, BasicBlock *ExitFail, Value *SearchStart,
+    Value *SearchEnd, Value *NeedleStart, Value *NeedleEnd) {
+  // Set up some types and constants that we intend to reuse.
+  auto *PtrTy = Builder.getPtrTy();
+  auto *I64Ty = Builder.getInt64Ty();
+  auto *PredVTy = ScalableVectorType::get(Builder.getInt1Ty(), VF);
+  auto *CharVTy = ScalableVectorType::get(CharTy, VF);
+  auto *ConstVF = ConstantInt::get(I64Ty, VF);
+
+  // Other common arguments.
+  BasicBlock *Preheader = CurLoop->getLoopPreheader();
+  LLVMContext &Ctx = Preheader->getContext();
+  Value *Passthru = ConstantInt::getNullValue(CharVTy);
+
+  // Split block in the original loop preheader.
+  // SPH is the new preheader to the old scalar loop.
+  BasicBlock *SPH = SplitBlock(Preheader, Preheader->getTerminator(), DT, LI,
+                               nullptr, "scalar_preheader");
+
+  // Create the blocks that we're going to use.
+  //
+  // We will have the following loops:
+  // (O) Outer loop where we iterate over the elements of the search array.
+  // (I) Inner loop where we iterate over the elements of the needle array.
+  //
+  // Overall, the blocks do the following:
+  // (0) Check if the arrays can't cross page boundaries. If so go to (1),
+  //     otherwise fall back to the original scalar loop.
+  // (1) Load the search array. Go to (2).
+  // (2) (a) Load the needle array.
+  //     (b) Splat the first element to the inactive lanes.
+  //     (c) Check if any elements match. If so go to (3), otherwise go to (4).
+  // (3) Compute the index of the first match and exit.
+  // (4) Check if we've reached the end of the needle array. If not loop back to
+  //     (2), otherwise go to (5).
+  // (5) Check if we've reached the end of the search array. If not loop back to
+  //     (1), otherwise exit.
+  // Blocks (0,3) are not part of any loop. Blocks (1,5) and (2,4) belong to
+  // the outer and inner loops, respectively.
+  BasicBlock *BB0 = BasicBlock::Create(Ctx, "mem_check", SPH->getParent(), SPH);
+  BasicBlock *BB1 =
+      BasicBlock::Create(Ctx, "find_first_vec_header", SPH->getParent(), SPH);
+  BasicBlock *BB2 =
+      BasicBlock::Create(Ctx, "match_check_vec", SPH->getParent(), SPH);
+  BasicBlock *BB3 =
+      BasicBlock::Create(Ctx, "calculate_match", SPH->getParent(), SPH);
+  BasicBlock *BB4 =
+      BasicBlock::Create(Ctx, "needle_check_vec", SPH->getParent(), SPH);
+  BasicBlock *BB5 =
+      BasicBlock::Create(Ctx, "search_check_vec", SPH->getParent(), SPH);
+
+  // Update LoopInfo with the new loops.
+  auto OuterLoop = LI->AllocateLoop();
+  auto InnerLoop = LI->AllocateLoop();
+
+  if (auto ParentLoop = CurLoop->getParentLoop()) {
+    ParentLoop->addBasicBlockToLoop(BB0, *LI);
+    ParentLoop->addChildLoop(OuterLoop);
+    ParentLoop->addBasicBlockToLoop(BB3, *LI);
+  } else {
+    LI->addTopLevelLoop(OuterLoop);
+  }
+
+  // Add the inner loop to the outer.
+  OuterLoop->addChildLoop(InnerLoop);
+
+  // Add the new basic blocks to the corresponding loops.
+  OuterLoop->addBasicBlockToLoop(BB1, *LI);
+  OuterLoop->addBasicBlockToLoop(BB5, *LI);
+  InnerLoop->addBasicBlockToLoop(BB2, *LI);
+  InnerLoop->addBasicBlockToLoop(BB4, *LI);
+
+  // Update the terminator added by SplitBlock to branch to the first block.
+  Preheader->getTerminator()->setSuccessor(0, BB0);
+  DTU.applyUpdates({{DominatorTree::Delete, Preheader, SPH},
+                    {DominatorTree::Insert, Preheader, BB0}});
+
+  // (0) Check if we could be crossing a page boundary; if so, fallback to the
+  // old scalar loops. Also create a predicate of VF elements to be used in the
+  // vector loops.
+  Builder.SetInsertPoint(BB0);
+  Value *ISearchStart =
+      Builder.CreatePtrToInt(SearchStart, I64Ty, "search_start_int");
+  Value *ISearchEnd =
+      Builder.CreatePtrToInt(SearchEnd, I64Ty, "search_end_int");
+  Value *INeedleStart =
+      Builder.CreatePtrToInt(NeedleStart, I64Ty, "needle_start_int");
+  Value *INeedleEnd =
+      Builder.CreatePtrToInt(NeedleEnd, I64Ty, "needle_end_int");
+  Value *PredVF =
+      Builder.CreateIntrinsic(Intrinsic::get_active_lane_mask, {PredVTy, I64Ty},
+                              {ConstantInt::get(I64Ty, 0), ConstVF});
+
+  const uint64_t MinPageSize = TTI->getMinPageSize().value();
+  const uint64_t AddrShiftAmt = llvm::Log2_64(MinPageSize);
+  Value *SearchStartPage =
+      Builder.CreateLShr(ISearchStart, AddrShiftAmt, "search_start_page");
+  Value *SearchEndPage =
+      Builder.CreateLShr(ISearchEnd, AddrShiftAmt, "search_end_page");
+  Value *NeedleStartPage =
+      Builder.CreateLShr(INeedleStart, AddrShiftAmt, "needle_start_page");
+  Value *NeedleEndPage =
+      Builder.CreateLShr(INeedleEnd, AddrShiftAmt, "needle_end_page");
+  Value *SearchPageCmp =
+      Builder.CreateICmpNE(SearchStartPage, SearchEndPage, "search_page_cmp");
+  Value *NeedlePageCmp =
+      Builder.CreateICmpNE(NeedleStartPage, NeedleEndPage, "needle_page_cmp");
+
+  Value *CombinedPageCmp =
+      Builder.CreateOr(SearchPageCmp, NeedlePageCmp, "combined_page_cmp");
+  BranchInst *CombinedPageBr = Builder.CreateCondBr(CombinedPageCmp, SPH, BB1);
+  CombinedPageBr->setMetadata(LLVMContext::MD_prof,
+                              MDBuilder(Ctx).createBranchWeights(10, 90));
+  DTU.applyUpdates(
+      {{DominatorTree::Insert, BB0, SPH}, {DominatorTree::Insert, BB0, BB1}});
+
+  // (1) Load the search array and branch to the inner loop.
+  Builder.SetInsertPoint(BB1);
+  PHINode *Search = Builder.CreatePHI(PtrTy, 2, "psearch");
+  Value *PredSearch = Builder.CreateIntrinsic(
+      Intrinsic::get_active_lane_mask, {PredVTy, I64Ty},
+      {Builder.CreatePtrToInt(Search, I64Ty), ISearchEnd}, nullptr,
+      "search_pred");
+  PredSearch = Builder.CreateAnd(PredVF, PredSearch, "search_masked");
+  Value *LoadSearch = Builder.CreateMaskedLoad(
+      CharVTy, Search, Align(1), PredSearch, Passthru, "search_load_vec");
+  Builder.CreateBr(BB2);
+  DTU.applyUpdates({{DominatorTree::Insert, BB1, BB2}});
+
+  // (2) Inner loop.
+  Builder.SetInsertPoint(BB2);
+  PHINode *Needle = Builder.CreatePHI(PtrTy, 2, "pneedle");
+
+  // (2.a) Load the needle array.
+  Value *PredNeedle = Builder.CreateIntrinsic(
+      Intrinsic::get_active_lane_mask, {PredVTy, I64Ty},
+      {Builder.CreatePtrToInt(Needle, I64Ty), INeedleEnd}, nullptr,
+      "needle_pred");
+  PredNeedle = Builder.CreateAnd(PredVF, PredNeedle, "needle_masked");
+  Value *LoadNeedle = Builder.CreateMaskedLoad(
+      CharVTy, Needle, Align(1), PredNeedle, Passthru, "needle_load_vec");
+
+  // (2.b) Splat the first element to the inactive lanes.
+  Value *Needle0 =
+      Builder.CreateExtractElement(LoadNeedle, uint64_t(0), "needle0");
+  Value *Needle0Splat = Builder.CreateVectorSplat(ElementCount::getScalable(VF),
+                                                  Needle0, "needle0");
+  LoadNeedle = Builder.CreateSelect(PredNeedle, LoadNeedle, Needle0Splat,
+                                    "needle_splat");
+  LoadNeedle =
+      Builder.CreateExtractVector(FixedVectorType::get(CharTy, VF), LoadNeedle,
+                                  ConstantInt::get(I64Ty, 0), "needle_vec");
+
+  // (2.c) Test if there's a match.
+  Value *MatchPred = Builder.CreateIntrinsic(
+      Intrinsic::experimental_vector_match, {CharVTy, LoadNeedle->getType()},
+      {LoadSearch, LoadNeedle, PredSearch}, nullptr, "match_pred");
+  Value *IfAnyMatch = Builder.CreateOrReduce(MatchPred);
+  Builder.CreateCondBr(IfAnyMatch, BB3, BB4);
+  DTU.applyUpdates(
+      {{DominatorTree::Insert, BB2, BB3}, {DominatorTree::Insert, BB2, BB4}});
+
+  // (3) We found a match. Compute the index of its location and exit.
+  Builder.SetInsertPoint(BB3);
+  PHINode *MatchLCSSA = Builder.CreatePHI(PtrTy, 1, "match_start");
+  PHINode *MatchPredLCSSA =
+      Builder.CreatePHI(MatchPred->getType(), 1, "match_vec");
+  Value *MatchCnt = Builder.CreateIntrinsic(
+      Intrinsic::experimental_cttz_elts, {I64Ty, MatchPred->getType()},
+      {MatchPredLCSSA, /*ZeroIsPoison=*/Builder.getInt1(true)}, nullptr,
+      "match_idx");
+  Value *MatchVal =
+      Builder.CreateGEP(CharTy, MatchLCSSA, MatchCnt, "match_res");
+  Builder.CreateBr(ExitSucc);
+  DTU.applyUpdates({{DominatorTree::Insert, BB3, ExitSucc}});
+
+  // (4) Check if we've reached the end of the needle array.
+  Builder.SetInsertPoint(BB4);
+  Value *NextNeedle =
+      Builder.CreateGEP(CharTy, Needle, ConstVF, "needle_next_vec");
+  Builder.CreateCondBr(Builder.CreateICmpULT(NextNeedle, NeedleEnd), BB2, BB5);
+  DTU.applyUpdates(
+      {{DominatorTree::Insert, BB4, BB2}, {DominatorTree::Insert, BB4, BB5}});
+
+  // (5) Check if we've reached the end of the search array.
+  Builder.SetInsertPoint(BB5);
+  Value *NextSearch =
+      Builder.CreateGEP(CharTy, Search, ConstVF, "search_next_vec");
+  Builder.CreateCondBr(Builder.CreateICmpULT(NextSearch, SearchEnd), BB1,
+                       ExitFail);
+  DTU.applyUpdates({{DominatorTree::Insert, BB5, BB1},
+                    {DominatorTree::Insert, BB5, ExitFail}});
+
+  // Set up the PHI nodes.
+  Search->addIncoming(SearchStart, BB0);
+  Search->addIncoming(NextSearch, BB5);
+  Needle->addIncoming(NeedleStart, BB1);
+  Needle->addIncoming(NextNeedle, BB4);
+  // These are needed to retain LCSSA form.
+  MatchLCSSA->addIncoming(Search, BB2);
+  MatchPredLCSSA->addIncoming(MatchPred, BB2);
+
+  if (VerifyLoops) {
+    OuterLoop->verifyLoop();
+    InnerLoop->verifyLoop();
+    if (!OuterLoop->isRecursivelyLCSSAForm(*DT, *LI))
+      report_fatal_error("Loops must remain in LCSSA form!");
+  }
+
+  return MatchVal;
+}
+
+void LoopIdiomVectorize::transformFindFirstByte(
+    PHINode *IndPhi, unsigned VF, Type *CharTy, BasicBlock *ExitSucc,
+    BasicBlock *ExitFail, Value *SearchStart, Value *SearchEnd,
+    Value *NeedleStart, Value *NeedleEnd) {
+  // Insert the find first byte code at the end of the preheader block.
+  BasicBlock *Preheader = CurLoop->getLoopPreheader();
+  BranchInst *PHBranch = cast<BranchInst>(Preheader->getTerminator());
+  IRBuilder<> Builder(PHBranch);
+  DomTreeUpdater DTU(DT, DomTreeUpdater::UpdateStrategy::Lazy);
+  Builder.SetCurrentDebugLocation(PHBranch->getDebugLoc());
+
+  Value *MatchVal =
+      expandFindFirstByte(Builder, DTU, VF, CharTy, ExitSucc, ExitFail,
+                          SearchStart, SearchEnd, NeedleStart, NeedleEnd);
+
+  assert(PHBranch->isUnconditional() &&
+         "Expected preheader to terminate with an unconditional branch.");
+
+  // Add new incoming values with the result of the transformation to PHINodes
+  // of ExitSucc that use IndPhi.
+  for (auto *U : llvm::make_early_inc_range(IndPhi->users())) {
+    auto *PN = dyn_cast<PHINode>(U);
+    if (PN && PN->getParent() == ExitSucc)
+      PN->addIncoming(MatchVal, cast<Instruction>(MatchVal)->getParent());
+  }
+
+  if (VerifyLoops && CurLoop->getParentLoop()) {
+    CurLoop->getParentLoop()->verifyLoop();
+    if (!CurLoop->getParentLoop()->isRecursivelyLCSSAForm(*DT, *LI))
+      report_fatal_error("Loops must remain in LCSSA form!");
+  }
+}
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
index bc44ec11edb7b..ed3e45dd2c6c8 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
@@ -40,6 +40,7 @@ class OptimizationRemarkEmitter;
 class TargetTransformInfo;
 class TargetLibraryInfo;
 class VPRecipeBuilder;
+struct VFRange;
 
 /// VPlan-based builder utility analogous to IRBuilder.
 class VPBuilder {
@@ -140,6 +141,9 @@ class VPBuilder {
     InsertPt = IP->getIterator();
   }
 
+  /// Insert \p R at the current insertion point.
+  void insert(VPRecipeBase *R) { BB->insert(R, InsertPt); }
+
   /// Create an N-ary operation with \p Opcode, \p Operands and set \p Inst as
   /// its underlying Instruction.
   VPInstruction *createNaryOp(unsigned Opcode, ArrayRef<VPValue *> Operands,
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 49694eb68e25b..f2241be60ce05 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -59,6 +59,7 @@
 #include "VPlan.h"
 #include "VPlanAnalysis.h"
 #include "VPlanHCFGBuilder.h"
+#include "VPlanHelpers.h"
 #include "VPlanPatternMatch.h"
 #include "VPlanTransforms.h"
 #include "VPlanUtils.h"
@@ -355,6 +356,16 @@ cl::opt<bool> EnableVPlanNativePath(
     "enable-vplan-native-path", cl::Hidden,
     cl::desc("Enable VPlan-native vectorization path with "
              "support for outer loop vectorization."));
+
+cl::opt<bool>
+    VerifyEachVPlan("vplan-verify-each",
+#ifdef EXPENSIVE_CHECKS
+                    cl::init(true),
+#else
+                    cl::init(false),
+#endif
+                    cl::Hidden,
+                    cl::desc("Verfiy VPlans after VPlan transforms."));
 } // namespace llvm
 
 // This flag enables the stress testing of the VPlan H-CFG construction in the
@@ -978,7 +989,10 @@ class LoopVectorizationCostModel {
                              InterleavedAccessInfo &IAI)
       : ScalarEpilogueStatus(SEL), TheLoop(L), PSE(PSE), LI(LI), Legal(Legal),
         TTI(TTI), TLI(TLI), DB(DB), AC(AC), ORE(ORE), TheFunction(F),
-        Hints(Hints), InterleaveInfo(IAI), CostKind(TTI::TCK_RecipThroughput) {}
+        Hints(Hints), InterleaveInfo(IAI), CostKind(TTI::TCK_RecipThroughput) {
+    if (TTI.supportsScalableVectors() || ForceTargetSupportsScalableVectors)
+      initializeVScaleForTuning();
+  }
 
   /// \return An upper bound for the vectorization factors (both fixed and
   /// scalable). If the factors are 0, vectorization and interleaving should be
@@ -1440,9 +1454,10 @@ class LoopVectorizationCostModel {
     // FIXME: Investigate opportunity for fixed vector factor.
     // FIXME: support fixed-order recurrences by fixing splice of non VFxUF
     // penultimate EVL.
-    bool EVLIsLegal =
-        UserIC <= 1 && TTI.hasActiveVectorLength(0, nullptr, Align()) &&
-        !EnableVPlanNativePath && Legal->getFixedOrderRecurrences().empty();
+    bool EVLIsLegal = UserIC <= 1 && IsScalableVF &&
+                      TTI.hasActiveVectorLength(0, nullptr, Align()) &&
+                      !EnableVPlanNativePath &&
+                      Legal->getFixedOrderRecurrences().empty();
     if (!EVLIsLegal) {
       // If for some reason EVL mode is unsupported, fallback to
       // DataWithoutLaneMask to try to vectorize the loop with folded tail
@@ -1554,9 +1569,34 @@ class LoopVectorizationCostModel {
   /// trivially hoistable.
   bool shouldConsiderInvariant(Value *Op);
 
+  /// Return the value of vscale used for tuning the cost model.
+  std::optional<unsigned> getVScaleForTuning() const { return VScaleForTuning; }
+
 private:
   unsigned NumPredStores = 0;
 
+  /// Used to store the value of vscale used for tuning the cost model. It is
+  /// initialized during object construction.
+  std::optional<unsigned> VScaleForTuning;
+
+  /// Initializes the value of vscale used for tuning the cost model. If
+  /// vscale_range.min == vscale_range.max then return vscale_range.max, else
+  /// return the value returned by the corresponding TTI method.
+  void initializeVScaleForTuning() {
+    const Function *Fn = TheLoop->getHeader()->getParent();
+    if (Fn->hasFnAttribute(Attribute::VScaleRange)) {
+      auto Attr = Fn->getFnAttribute(Attribute::VScaleRange);
+      auto Min = Attr.getVScaleRangeMin();
+      auto Max = Attr.getVScaleRangeMax();
+      if (Max && Min == Max) {
+        VScaleForTuning = Max;
+        return;
+      }
+    }
+
+    VScaleForTuning = TTI.getVScaleForTuning();
+  }
+
   /// \return An upper bound for the vectorization factors for both
   /// fixed and scalable vectorization, where the minimum-known number of
   /// elements is a power-of-2 larger than zero. If scalable vectorization is
@@ -2467,6 +2507,15 @@ void InnerLoopVectorizer::introduceCheckBlockInVPlan(BasicBlock *CheckIRBB) {
   }
   VPBlockUtils::connectBlocks(PreVectorPH, ScalarPH);
   PreVectorPH->swapSuccessors();
+
+  // We just connected a new block to the scalar preheader. Update all
+  // ResumePhis by adding an incoming value for it.
+  for (VPRecipeBase &R : *cast<VPBasicBlock>(ScalarPH)) {
+    auto *ResumePhi = dyn_cast<VPInstruction>(&R);
+    if (!ResumePhi || ResumePhi->getOpcode() != VPInstruction::ResumePhi)
+      continue;
+    ResumePhi->addOperand(ResumePhi->getOperand(1));
+  }
 }
 
 void InnerLoopVectorizer::emitIterationCountCheck(BasicBlock *Bypass) {
@@ -2888,10 +2937,6 @@ void InnerLoopVectorizer::fixVectorizedLoop(VPTransformState &State) {
   if (EnableVPlanNativePath)
     fixNonInductionPHIs(State);
 
-  // Forget the original basic block.
-  PSE.getSE()->forgetLoop(OrigLoop);
-  PSE.getSE()->forgetBlockAndLoopDispositions();
-
   // After vectorization, the exit blocks of the original loop will have
   // additional predecessors. Invalidate SCEVs for the exit phis in case SE
   // looked through single-entry phis.
@@ -2901,6 +2946,10 @@ void InnerLoopVectorizer::fixVectorizedLoop(VPTransformState &State) {
     for (PHINode &PN : Exit->phis())
       PSE.getSE()->forgetLcssaPhiWithNewPredecessor(OrigLoop, &PN);
 
+  // Forget the original basic block.
+  PSE.getSE()->forgetLoop(OrigLoop);
+  PSE.getSE()->forgetBlockAndLoopDispositions();
+
   // Don't apply optimizations below when no vector region remains, as they all
   // require a vector loop at the moment.
   if (!State.Plan->getVectorLoopRegion())
@@ -3390,10 +3439,10 @@ bool LoopVectorizationCostModel::interleavedAccessCanBeWidened(
   if (hasIrregularType(ScalarTy, DL))
     return false;
 
-  // We currently only know how to emit interleave/deinterleave with
-  // Factor=2 for scalable vectors. This is purely an implementation
-  // limit.
-  if (VF.isScalable() && InterleaveFactor != 2)
+  // For scalable vectors, the only interleave factor currently supported
+  // must be power of 2 since we require the (de)interleave2 intrinsics
+  // instead of shufflevectors.
+  if (VF.isScalable() && !isPowerOf2_32(InterleaveFactor))
     return false;
 
   // If the group involves a non-integral pointer, we may not be able to
@@ -4070,7 +4119,8 @@ LoopVectorizationCostModel::computeMaxVF(ElementCount UserVF, unsigned UserIC) {
   // found modulo the vectorization factor is not zero, try to fold the tail
   // by masking.
   // FIXME: look for a smaller MaxVF that does divide TC rather than masking.
-  setTailFoldingStyles(MaxFactors.ScalableVF.isScalable(), UserIC);
+  bool ContainsScalableVF = MaxFactors.ScalableVF.isNonZero();
+  setTailFoldingStyles(ContainsScalableVF, UserIC);
   if (foldTailByMasking()) {
     if (getTailFoldingStyle() == TailFoldingStyle::DataWithEVL) {
       LLVM_DEBUG(
@@ -4081,8 +4131,7 @@ LoopVectorizationCostModel::computeMaxVF(ElementCount UserVF, unsigned UserIC) {
       // Tail folded loop using VP intrinsics restricts the VF to be scalable
       // for now.
       // TODO: extend it for fixed vectors, if required.
-      assert(MaxFactors.ScalableVF.isScalable() &&
-             "Expected scalable vector factor.");
+      assert(ContainsScalableVF && "Expected scalable vector factor.");
 
       MaxFactors.FixedVF = ElementCount::getFixed(1);
     }
@@ -4231,33 +4280,15 @@ ElementCount LoopVectorizationCostModel::getMaximizedVFForTarget(
   return MaxVF;
 }
 
-/// Convenience function that returns the value of vscale_range iff
-/// vscale_range.min == vscale_range.max or otherwise returns the value
-/// returned by the corresponding TTI method.
-static std::optional<unsigned>
-getVScaleForTuning(const Loop *L, const TargetTransformInfo &TTI) {
-  const Function *Fn = L->getHeader()->getParent();
-  if (Fn->hasFnAttribute(Attribute::VScaleRange)) {
-    auto Attr = Fn->getFnAttribute(Attribute::VScaleRange);
-    auto Min = Attr.getVScaleRangeMin();
-    auto Max = Attr.getVScaleRangeMax();
-    if (Max && Min == Max)
-      return Max;
-  }
-
-  return TTI.getVScaleForTuning();
-}
-
 /// This function attempts to return a value that represents the vectorization
 /// factor at runtime. For fixed-width VFs we know this precisely at compile
 /// time, but for scalable VFs we calculate it based on an estimate of the
 /// vscale value.
-static unsigned getEstimatedRuntimeVF(const Loop *L,
-                                      const TargetTransformInfo &TTI,
-                                      ElementCount VF) {
+static unsigned getEstimatedRuntimeVF(ElementCount VF,
+                                      std::optional<unsigned> VScale) {
   unsigned EstimatedVF = VF.getKnownMinValue();
   if (VF.isScalable())
-    if (std::optional<unsigned> VScale = getVScaleForTuning(L, TTI))
+    if (VScale)
       EstimatedVF *= *VScale;
   assert(EstimatedVF >= 1 && "Estimated VF shouldn't be less than 1");
   return EstimatedVF;
@@ -4272,7 +4303,7 @@ bool LoopVectorizationPlanner::isMoreProfitable(
   // Improve estimate for the vector width if it is scalable.
   unsigned EstimatedWidthA = A.Width.getKnownMinValue();
   unsigned EstimatedWidthB = B.Width.getKnownMinValue();
-  if (std::optional<unsigned> VScale = getVScaleForTuning(OrigLoop, TTI)) {
+  if (std::optional<unsigned> VScale = CM.getVScaleForTuning()) {
     if (A.Width.isScalable())
       EstimatedWidthA *= *VScale;
     if (B.Width.isScalable())
@@ -4329,6 +4360,13 @@ void LoopVectorizationPlanner::emitInvalidCostRemarks(
   SmallVector<RecipeVFPair> InvalidCosts;
   for (const auto &Plan : VPlans) {
     for (ElementCount VF : Plan->vectorFactors()) {
+      // The VPlan-based cost model is designed for computing vector cost.
+      // Querying VPlan-based cost model with a scarlar VF will cause some
+      // errors because we expect the VF is vector for most of the widen
+      // recipes.
+      if (VF.isScalar())
+        continue;
+
       VPCostContext CostCtx(CM.TTI, *CM.TLI, Legal->getWidestInductionType(),
                             CM, CM.CostKind);
       precomputeCosts(*Plan, VF, CostCtx);
@@ -4537,11 +4575,10 @@ VectorizationFactor LoopVectorizationPlanner::selectVectorizationFactor() {
   InstructionCost ExpectedCost = CM.expectedCost(ElementCount::getFixed(1));
   LLVM_DEBUG(dbgs() << "LV: Scalar loop costs: " << ExpectedCost << ".\n");
   assert(ExpectedCost.isValid() && "Unexpected invalid cost for scalar loop");
-  assert(any_of(VPlans,
-                [](std::unique_ptr<VPlan> &P) {
-                  return P->hasVF(ElementCount::getFixed(1));
-                }) &&
-         "Expected Scalar VF to be a candidate");
+  assert(
+      any_of(VPlans,
+             [](std::unique_ptr<VPlan> &P) { return P->hasScalarVFOnly(); }) &&
+      "Expected Scalar VF to be a candidate");
 
   const VectorizationFactor ScalarCost(ElementCount::getFixed(1), ExpectedCost,
                                        ExpectedCost);
@@ -4565,13 +4602,13 @@ VectorizationFactor LoopVectorizationPlanner::selectVectorizationFactor() {
       InstructionCost C = CM.expectedCost(VF);
       VectorizationFactor Candidate(VF, C, ScalarCost.ScalarCost);
 
-      unsigned Width = getEstimatedRuntimeVF(OrigLoop, TTI, Candidate.Width);
+      unsigned Width =
+          getEstimatedRuntimeVF(Candidate.Width, CM.getVScaleForTuning());
       LLVM_DEBUG(dbgs() << "LV: Vector loop of width " << VF
                         << " costs: " << (Candidate.Cost / Width));
       if (VF.isScalable())
         LLVM_DEBUG(dbgs() << " (assuming a minimum vscale of "
-                          << getVScaleForTuning(OrigLoop, TTI).value_or(1)
-                          << ")");
+                          << CM.getVScaleForTuning().value_or(1) << ")");
       LLVM_DEBUG(dbgs() << ".\n");
 
       if (!ForceVectorization && !willGenerateVectors(*P, VF, TTI)) {
@@ -4660,7 +4697,8 @@ bool LoopVectorizationCostModel::isEpilogueVectorizationProfitable(
   unsigned MinVFThreshold = EpilogueVectorizationMinVF.getNumOccurrences() > 0
                                 ? EpilogueVectorizationMinVF
                                 : TTI.getEpilogueVectorizationMinVF();
-  return getEstimatedRuntimeVF(TheLoop, TTI, VF * Multiplier) >= MinVFThreshold;
+  return getEstimatedRuntimeVF(VF * Multiplier, VScaleForTuning) >=
+         MinVFThreshold;
 }
 
 VectorizationFactor LoopVectorizationPlanner::selectEpilogueVectorizationFactor(
@@ -4712,8 +4750,8 @@ VectorizationFactor LoopVectorizationPlanner::selectEpilogueVectorizationFactor(
   // If MainLoopVF = vscale x 2, and vscale is expected to be 4, then we know
   // the main loop handles 8 lanes per iteration. We could still benefit from
   // vectorizing the epilogue loop with VF=4.
-  ElementCount EstimatedRuntimeVF =
-      ElementCount::getFixed(getEstimatedRuntimeVF(OrigLoop, TTI, MainLoopVF));
+  ElementCount EstimatedRuntimeVF = ElementCount::getFixed(
+      getEstimatedRuntimeVF(MainLoopVF, CM.getVScaleForTuning()));
 
   ScalarEvolution &SE = *PSE.getSE();
   Type *TCType = Legal->getWidestInductionType();
@@ -4959,7 +4997,7 @@ LoopVectorizationCostModel::selectInterleaveCount(ElementCount VF,
       MaxInterleaveCount = ForceTargetMaxVectorInterleaveFactor;
   }
 
-  unsigned EstimatedVF = getEstimatedRuntimeVF(TheLoop, TTI, VF);
+  unsigned EstimatedVF = getEstimatedRuntimeVF(VF, VScaleForTuning);
   unsigned KnownTC = PSE.getSE()->getSmallConstantTripCount(TheLoop);
   if (KnownTC > 0) {
     // At least one iteration must be scalar when this constraint holds. So the
@@ -6262,7 +6300,6 @@ void LoopVectorizationCostModel::setVectorizedCallDecision(ElementCount VF) {
 
       // Find the cost of vectorizing the call, if we can find a suitable
       // vector variant of the function.
-      bool UsesMask = false;
       VFInfo FuncInfo;
       Function *VecFunc = nullptr;
       // Search through any available variants for one we can use at this VF.
@@ -6314,7 +6351,6 @@ void LoopVectorizationCostModel::setVectorizedCallDecision(ElementCount VF) {
             break;
           }
           case VFParamKind::GlobalPredicate:
-            UsesMask = true;
             break;
           default:
             ParamsOk = false;
@@ -6331,19 +6367,8 @@ void LoopVectorizationCostModel::setVectorizedCallDecision(ElementCount VF) {
         break;
       }
 
-      // Add in the cost of synthesizing a mask if one wasn't required.
-      InstructionCost MaskCost = 0;
-      if (VecFunc && UsesMask && !MaskRequired)
-        MaskCost = TTI.getShuffleCost(
-            TargetTransformInfo::SK_Broadcast,
-            VectorType::get(IntegerType::getInt1Ty(
-                                VecFunc->getFunctionType()->getContext()),
-                            VF),
-            {}, CostKind);
-
       if (TLI && VecFunc && !CI->isNoBuiltin())
-        VectorCost =
-            TTI.getCallInstrCost(nullptr, RetTy, Tys, CostKind) + MaskCost;
+        VectorCost = TTI.getCallInstrCost(nullptr, RetTy, Tys, CostKind);
 
       // Find the cost of an intrinsic; some targets may have instructions that
       // perform the operation without needing an actual call.
@@ -6615,8 +6640,10 @@ LoopVectorizationCostModel::getInstructionCost(Instruction *I,
     // fold away.  We can generalize this for all operations using the notion
     // of neutral elements.  (TODO)
     if (I->getOpcode() == Instruction::Mul &&
-        (PSE.getSCEV(I->getOperand(0))->isOne() ||
-         PSE.getSCEV(I->getOperand(1))->isOne()))
+        ((TheLoop->isLoopInvariant(I->getOperand(0)) &&
+          PSE.getSCEV(I->getOperand(0))->isOne()) ||
+         (TheLoop->isLoopInvariant(I->getOperand(1)) &&
+          PSE.getSCEV(I->getOperand(1))->isOne())))
       return 0;
 
     // Detect reduction patterns
@@ -7388,7 +7415,7 @@ InstructionCost LoopVectorizationPlanner::cost(VPlan &Plan,
   // Now compute and add the VPlan-based cost.
   Cost += Plan.cost(VF, CostCtx);
 #ifndef NDEBUG
-  unsigned EstimatedWidth = getEstimatedRuntimeVF(OrigLoop, CM.TTI, VF);
+  unsigned EstimatedWidth = getEstimatedRuntimeVF(VF, CM.getVScaleForTuning());
   LLVM_DEBUG(dbgs() << "Cost for VF " << VF << ": " << Cost
                     << " (Estimated cost per lane: ");
   if (Cost.isValid()) {
@@ -7528,7 +7555,14 @@ VectorizationFactor LoopVectorizationPlanner::computeBestVF() {
   VPCostContext CostCtx(CM.TTI, *CM.TLI, Legal->getWidestInductionType(), CM,
                         CM.CostKind);
   precomputeCosts(BestPlan, BestFactor.Width, CostCtx);
-  assert((BestFactor.Width == LegacyVF.Width ||
+  // Set PlanForEarlyExitLoop to true if the BestPlan has been built from a
+  // loop with an uncountable early exit. The legacy cost model doesn't
+  // properly model costs for such loops.
+  bool PlanForEarlyExitLoop =
+      BestPlan.getVectorLoopRegion() &&
+      BestPlan.getVectorLoopRegion()->getSingleSuccessor() !=
+          BestPlan.getMiddleBlock();
+  assert((BestFactor.Width == LegacyVF.Width || PlanForEarlyExitLoop ||
           planContainsAdditionalSimplifications(getPlanFor(BestFactor.Width),
                                                 CostCtx, OrigLoop) ||
           planContainsAdditionalSimplifications(getPlanFor(LegacyVF.Width),
@@ -7624,8 +7658,8 @@ static void fixReductionScalarResumeWhenVectorizingEpilog(
   // over the incoming values correctly.
   using namespace VPlanPatternMatch;
   auto IsResumePhi = [](VPUser *U) {
-    return match(
-        U, m_VPInstruction<VPInstruction::ResumePhi>(m_VPValue(), m_VPValue()));
+    auto *VPI = dyn_cast<VPInstruction>(U);
+    return VPI && VPI->getOpcode() == VPInstruction::ResumePhi;
   };
   assert(count_if(EpiRedResult->users(), IsResumePhi) == 1 &&
          "ResumePhi must have a single user");
@@ -7651,9 +7685,11 @@ DenseMap<const SCEV *, Value *> LoopVectorizationPlanner::executePlan(
 
   // TODO: Move to VPlan transform stage once the transition to the VPlan-based
   // cost model is complete for better cost estimates.
-  VPlanTransforms::unrollByUF(BestVPlan, BestUF,
-                              OrigLoop->getHeader()->getContext());
+  VPlanTransforms::runPass(VPlanTransforms::unrollByUF, BestVPlan, BestUF,
+                           OrigLoop->getHeader()->getContext());
   VPlanTransforms::optimizeForVFAndUF(BestVPlan, BestVF, BestUF, PSE);
+  VPlanTransforms::simplifyRecipes(BestVPlan, *Legal->getWidestInductionType());
+  VPlanTransforms::removeDeadRecipes(BestVPlan);
   VPlanTransforms::convertToConcreteRecipes(BestVPlan);
 
   // Perform the actual loop transformation.
@@ -7727,12 +7763,20 @@ DenseMap<const SCEV *, Value *> LoopVectorizationPlanner::executePlan(
   if (VectorizingEpilogue) {
     assert(!ILV.Legal->hasUncountableEarlyExit() &&
            "Epilogue vectorisation not yet supported with early exits");
+    BasicBlock *PH = OrigLoop->getLoopPreheader();
     BasicBlock *BypassBlock = ILV.getAdditionalBypassBlock();
+    for (auto *Pred : predecessors(PH)) {
+      for (PHINode &Phi : PH->phis()) {
+        if (Phi.getBasicBlockIndex(Pred) != -1)
+          continue;
+        Phi.addIncoming(Phi.getIncomingValueForBlock(BypassBlock), Pred);
+      }
+    }
+
     for (VPRecipeBase &R : *MiddleVPBB) {
       fixReductionScalarResumeWhenVectorizingEpilog(
           &R, State, State.CFG.VPBB2IRBB[MiddleVPBB], BypassBlock);
     }
-    BasicBlock *PH = OrigLoop->getLoopPreheader();
     for (const auto &[IVPhi, _] : Legal->getInductionVars()) {
       auto *Inc = cast<PHINode>(IVPhi->getIncomingValueForBlock(PH));
       Value *V = ILV.getInductionAdditionalBypassValue(IVPhi);
@@ -8299,7 +8343,7 @@ VPRecipeBuilder::tryToWidenMemory(Instruction *I, ArrayRef<VPValue *> Operands,
                                                 : GEPNoWrapFlags::none(),
                                             I->getDebugLoc());
     }
-    Builder.getInsertBlock()->appendRecipe(VectorPtr);
+    Builder.insert(VectorPtr);
     Ptr = VectorPtr;
   }
   if (LoadInst *Load = dyn_cast<LoadInst>(I))
@@ -8518,8 +8562,7 @@ bool VPRecipeBuilder::shouldWiden(Instruction *I, VFRange &Range) const {
 }
 
 VPWidenRecipe *VPRecipeBuilder::tryToWiden(Instruction *I,
-                                           ArrayRef<VPValue *> Operands,
-                                           VPBasicBlock *VPBB) {
+                                           ArrayRef<VPValue *> Operands) {
   switch (I->getOpcode()) {
   default:
     return nullptr;
@@ -8566,6 +8609,8 @@ VPWidenRecipe *VPRecipeBuilder::tryToWiden(Instruction *I,
       // to replace operands with constants.
       ScalarEvolution &SE = *PSE.getSE();
       auto GetConstantViaSCEV = [this, &SE](VPValue *Op) {
+        if (!Op->isLiveIn())
+          return Op;
         Value *V = Op->getUnderlyingValue();
         if (isa<Constant>(V) || !SE.isSCEVable(V->getType()))
           return Op;
@@ -8618,8 +8663,9 @@ void VPRecipeBuilder::fixHeaderPhis() {
   }
 }
 
-VPReplicateRecipe *VPRecipeBuilder::handleReplication(Instruction *I,
-                                                      VFRange &Range) {
+VPReplicateRecipe *
+VPRecipeBuilder::handleReplication(Instruction *I, ArrayRef<VPValue *> Operands,
+                                   VFRange &Range) {
   bool IsUniform = LoopVectorizationPlanner::getDecisionAndClampRange(
       [&](ElementCount VF) { return CM.isUniformAfterVectorization(I, VF); },
       Range);
@@ -8675,8 +8721,8 @@ VPReplicateRecipe *VPRecipeBuilder::handleReplication(Instruction *I,
   assert((Range.Start.isScalar() || !IsUniform || !IsPredicated ||
           (Range.Start.isScalable() && isa<IntrinsicInst>(I))) &&
          "Should not predicate a uniform recipe");
-  auto *Recipe = new VPReplicateRecipe(I, mapToVPValues(I->operands()),
-                                       IsUniform, BlockInMask);
+  auto *Recipe = new VPReplicateRecipe(
+      I, make_range(Operands.begin(), Operands.end()), IsUniform, BlockInMask);
   return Recipe;
 }
 
@@ -8684,12 +8730,12 @@ VPReplicateRecipe *VPRecipeBuilder::handleReplication(Instruction *I,
 /// are valid so recipes can be formed later.
 void VPRecipeBuilder::collectScaledReductions(VFRange &Range) {
   // Find all possible partial reductions.
-  SmallVector<std::pair<PartialReductionChain, unsigned>, 1>
+  SmallVector<std::pair<PartialReductionChain, unsigned>>
       PartialReductionChains;
-  for (const auto &[Phi, RdxDesc] : Legal->getReductionVars())
-    if (std::optional<std::pair<PartialReductionChain, unsigned>> Pair =
-            getScaledReduction(Phi, RdxDesc, Range))
-      PartialReductionChains.push_back(*Pair);
+  for (const auto &[Phi, RdxDesc] : Legal->getReductionVars()) {
+    getScaledReductions(Phi, RdxDesc.getLoopExitInstr(), Range,
+                        PartialReductionChains);
+  }
 
   // A partial reduction is invalid if any of its extends are used by
   // something that isn't another partial reduction. This is because the
@@ -8717,39 +8763,47 @@ void VPRecipeBuilder::collectScaledReductions(VFRange &Range) {
   }
 }
 
-std::optional<std::pair<PartialReductionChain, unsigned>>
-VPRecipeBuilder::getScaledReduction(PHINode *PHI,
-                                    const RecurrenceDescriptor &Rdx,
-                                    VFRange &Range) {
-  // TODO: Allow scaling reductions when predicating. The select at
-  // the end of the loop chooses between the phi value and most recent
-  // reduction result, both of which have different VFs to the active lane
-  // mask when scaling.
-  if (CM.blockNeedsPredicationForAnyReason(Rdx.getLoopExitInstr()->getParent()))
-    return std::nullopt;
+bool VPRecipeBuilder::getScaledReductions(
+    Instruction *PHI, Instruction *RdxExitInstr, VFRange &Range,
+    SmallVectorImpl<std::pair<PartialReductionChain, unsigned>> &Chains) {
+
+  if (!CM.TheLoop->contains(RdxExitInstr))
+    return false;
 
-  auto *Update = dyn_cast<BinaryOperator>(Rdx.getLoopExitInstr());
+  auto *Update = dyn_cast<BinaryOperator>(RdxExitInstr);
   if (!Update)
-    return std::nullopt;
+    return false;
 
   Value *Op = Update->getOperand(0);
   Value *PhiOp = Update->getOperand(1);
-  if (Op == PHI) {
-    Op = Update->getOperand(1);
-    PhiOp = Update->getOperand(0);
+  if (Op == PHI)
+    std::swap(Op, PhiOp);
+
+  // Try and get a scaled reduction from the first non-phi operand.
+  // If one is found, we use the discovered reduction instruction in
+  // place of the accumulator for costing.
+  if (auto *OpInst = dyn_cast<Instruction>(Op)) {
+    if (getScaledReductions(PHI, OpInst, Range, Chains)) {
+      PHI = Chains.rbegin()->first.Reduction;
+
+      Op = Update->getOperand(0);
+      PhiOp = Update->getOperand(1);
+      if (Op == PHI)
+        std::swap(Op, PhiOp);
+    }
   }
   if (PhiOp != PHI)
-    return std::nullopt;
+    return false;
 
   auto *BinOp = dyn_cast<BinaryOperator>(Op);
   if (!BinOp || !BinOp->hasOneUse())
-    return std::nullopt;
+    return false;
 
   using namespace llvm::PatternMatch;
   Value *A, *B;
   if (!match(BinOp->getOperand(0), m_ZExtOrSExt(m_Value(A))) ||
       !match(BinOp->getOperand(1), m_ZExtOrSExt(m_Value(B))))
-    return std::nullopt;
+    return false;
 
   Instruction *ExtA = cast<Instruction>(BinOp->getOperand(0));
   Instruction *ExtB = cast<Instruction>(BinOp->getOperand(1));
@@ -8759,7 +8813,7 @@ VPRecipeBuilder::getScaledReduction(PHINode *PHI,
   TTI::PartialReductionExtendKind OpBExtend =
       TargetTransformInfo::getPartialReductionExtendKind(ExtB);
 
-  PartialReductionChain Chain(Rdx.getLoopExitInstr(), ExtA, ExtB, BinOp);
+  PartialReductionChain Chain(RdxExitInstr, ExtA, ExtB, BinOp);
 
   unsigned TargetScaleFactor =
       PHI->getType()->getPrimitiveSizeInBits().getKnownScalarFactor(
@@ -8773,16 +8827,16 @@ VPRecipeBuilder::getScaledReduction(PHINode *PHI,
                 std::make_optional(BinOp->getOpcode()));
             return Cost.isValid();
           },
-          Range))
-    return std::make_pair(Chain, TargetScaleFactor);
+          Range)) {
+    Chains.push_back(std::make_pair(Chain, TargetScaleFactor));
+    return true;
+  }
 
-  return std::nullopt;
+  return false;
 }
 
-VPRecipeBase *
-VPRecipeBuilder::tryToCreateWidenRecipe(Instruction *Instr,
-                                        ArrayRef<VPValue *> Operands,
-                                        VFRange &Range, VPBasicBlock *VPBB) {
+VPRecipeBase *VPRecipeBuilder::tryToCreateWidenRecipe(
+    Instruction *Instr, ArrayRef<VPValue *> Operands, VFRange &Range) {
   // First, check for specific widening recipes that deal with inductions, Phi
   // nodes, calls and memory operations.
   VPRecipeBase *Recipe;
@@ -8861,7 +8915,7 @@ VPRecipeBuilder::tryToCreateWidenRecipe(Instruction *Instr,
                                  *CI);
   }
 
-  return tryToWiden(Instr, Operands, VPBB);
+  return tryToWiden(Instr, Operands);
 }
 
 VPRecipeBase *
@@ -8871,11 +8925,24 @@ VPRecipeBuilder::tryToCreatePartialReduction(Instruction *Reduction,
          "Unexpected number of operands for partial reduction");
 
   VPValue *BinOp = Operands[0];
-  VPValue *Phi = Operands[1];
-  if (isa<VPReductionPHIRecipe>(BinOp->getDefiningRecipe()))
-    std::swap(BinOp, Phi);
-
-  return new VPPartialReductionRecipe(Reduction->getOpcode(), BinOp, Phi,
+  VPValue *Accumulator = Operands[1];
+  VPRecipeBase *BinOpRecipe = BinOp->getDefiningRecipe();
+  if (isa<VPReductionPHIRecipe>(BinOpRecipe) ||
+      isa<VPPartialReductionRecipe>(BinOpRecipe))
+    std::swap(BinOp, Accumulator);
+
+  unsigned ReductionOpcode = Reduction->getOpcode();
+  if (CM.blockNeedsPredicationForAnyReason(Reduction->getParent())) {
+    assert((ReductionOpcode == Instruction::Add ||
+            ReductionOpcode == Instruction::Sub) &&
+           "Expected an ADD or SUB operation for predicated partial "
+           "reductions (because the neutral element in the mask is zero)!");
+    VPValue *Mask = getBlockInMask(Reduction->getParent());
+    VPValue *Zero =
+        Plan.getOrAddLiveIn(ConstantInt::get(Reduction->getType(), 0));
+    BinOp = Builder.createSelect(Mask, BinOp, Zero, Reduction->getDebugLoc());
+  }
+  return new VPPartialReductionRecipe(ReductionOpcode, BinOp, Accumulator,
                                       Reduction);
 }
 
@@ -8887,15 +8954,17 @@ void LoopVectorizationPlanner::buildVPlansWithVPRecipes(ElementCount MinVF,
   for (ElementCount VF = MinVF; ElementCount::isKnownLT(VF, MaxVFTimes2);) {
     VFRange SubRange = {VF, MaxVFTimes2};
     if (auto Plan = tryToBuildVPlanWithVPRecipes(SubRange)) {
+      bool HasScalarVF = Plan->hasScalarVFOnly();
       // Now optimize the initial VPlan.
-      if (!Plan->hasVF(ElementCount::getFixed(1)))
-        VPlanTransforms::truncateToMinimalBitwidths(*Plan,
-                                                    CM.getMinimalBitwidths());
+      if (!HasScalarVF)
+        VPlanTransforms::runPass(VPlanTransforms::truncateToMinimalBitwidths,
+                                 *Plan, CM.getMinimalBitwidths());
       VPlanTransforms::optimize(*Plan);
       // TODO: try to put it close to addActiveLaneMask().
       // Discard the plan if it is not EVL-compatible
-      if (CM.foldTailWithEVL() && !VPlanTransforms::tryAddExplicitVectorLength(
-                                      *Plan, CM.getMaxSafeElements()))
+      if (CM.foldTailWithEVL() && !HasScalarVF &&
+          !VPlanTransforms::runPass(VPlanTransforms::tryAddExplicitVectorLength,
+                                    *Plan, CM.getMaxSafeElements()))
         break;
       assert(verifyVPlanIsValid(*Plan) && "VPlan is invalid");
       VPlans.push_back(std::move(Plan));
@@ -9033,7 +9102,6 @@ static void addScalarResumePhis(VPRecipeBuilder &Builder, VPlan &Plan,
 static SetVector<VPIRInstruction *>
 collectUsersInExitBlocks(Loop *OrigLoop, VPRecipeBuilder &Builder,
                          VPlan &Plan) {
-  auto *MiddleVPBB = Plan.getMiddleBlock();
   SetVector<VPIRInstruction *> ExitUsersToFix;
   for (VPIRBasicBlock *ExitVPBB : Plan.getExitBlocks()) {
     for (VPRecipeBase &R : *ExitVPBB) {
@@ -9043,60 +9111,46 @@ collectUsersInExitBlocks(Loop *OrigLoop, VPRecipeBuilder &Builder,
       auto *ExitPhi = dyn_cast<PHINode>(&ExitIRI->getInstruction());
       if (!ExitPhi)
         break;
-      for (VPBlockBase *PredVPBB : ExitVPBB->getPredecessors()) {
-        BasicBlock *ExitingBB = OrigLoop->getLoopLatch();
-        if (PredVPBB != MiddleVPBB) {
-          SmallVector<BasicBlock *> ExitingBlocks;
-          OrigLoop->getExitingBlocks(ExitingBlocks);
-          assert(ExitingBlocks.size() == 2 && "only support 2 exiting blocks");
-          ExitingBB = ExitingBB == ExitingBlocks[0] ? ExitingBlocks[1]
-                                                    : ExitingBlocks[0];
-        }
-        Value *IncomingValue = ExitPhi->getIncomingValueForBlock(ExitingBB);
-        VPValue *V = Builder.getVPValueOrAddLiveIn(IncomingValue);
-        ExitUsersToFix.insert(ExitIRI);
-        ExitIRI->addOperand(V);
+      if (ExitVPBB->getSinglePredecessor() != Plan.getMiddleBlock()) {
+        assert(ExitIRI->getNumOperands() ==
+                   ExitVPBB->getPredecessors().size() &&
+               "early-exit must update exit values on construction");
+        continue;
       }
+      BasicBlock *ExitingBB = OrigLoop->getLoopLatch();
+      Value *IncomingValue = ExitPhi->getIncomingValueForBlock(ExitingBB);
+      VPValue *V = Builder.getVPValueOrAddLiveIn(IncomingValue);
+      ExitIRI->addOperand(V);
+      if (V->isLiveIn())
+        continue;
+      assert(V->getDefiningRecipe()->getParent()->getEnclosingLoopRegion() &&
+             "Only recipes defined inside a region should need fixing.");
+      ExitUsersToFix.insert(ExitIRI);
     }
   }
   return ExitUsersToFix;
 }
 
 // Add exit values to \p Plan. Extracts are added for each entry in \p
-// ExitUsersToFix if needed and their operands are updated. Returns true if all
-// exit users can be handled, otherwise return false.
-static bool
+// ExitUsersToFix if needed and their operands are updated.
+static void
 addUsersInExitBlocks(VPlan &Plan,
                      const SetVector<VPIRInstruction *> &ExitUsersToFix) {
   if (ExitUsersToFix.empty())
-    return true;
+    return;
 
   auto *MiddleVPBB = Plan.getMiddleBlock();
   VPBuilder B(MiddleVPBB, MiddleVPBB->getFirstNonPhi());
-  VPTypeAnalysis TypeInfo(Plan.getCanonicalIV()->getScalarType());
 
   // Introduce extract for exiting values and update the VPIRInstructions
   // modeling the corresponding LCSSA phis.
   for (VPIRInstruction *ExitIRI : ExitUsersToFix) {
-    for (const auto &[Idx, Op] : enumerate(ExitIRI->operands())) {
-      // Pass live-in values used by exit phis directly through to their users
-      // in the exit block.
-      if (Op->isLiveIn())
-        continue;
-
-      // Currently only live-ins can be used by exit values from blocks not
-      // exiting via the vector latch through to the middle block.
-      if (ExitIRI->getParent()->getSinglePredecessor() != MiddleVPBB)
-        return false;
-
-      LLVMContext &Ctx = ExitIRI->getInstruction().getContext();
-      VPValue *Ext = B.createNaryOp(VPInstruction::ExtractFromEnd,
-                                    {Op, Plan.getOrAddLiveIn(ConstantInt::get(
-                                             IntegerType::get(Ctx, 32), 1))});
-      ExitIRI->setOperand(Idx, Ext);
-    }
+    assert(ExitIRI->getNumOperands() == 1 &&
+           ExitIRI->getParent()->getSinglePredecessor() == MiddleVPBB &&
+           "exit values from early exits must be fixed when branch to "
+           "early-exit is added");
+    ExitIRI->extractLastLaneOfOperand(B);
   }
-  return true;
 }
 
 /// Handle users in the exit block for first order reductions in the original
@@ -9264,9 +9318,9 @@ LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes(VFRange &Range) {
                      CM.getWideningDecision(IG->getInsertPos(), VF) ==
                          LoopVectorizationCostModel::CM_Interleave);
       // For scalable vectors, the only interleave factor currently supported
-      // is 2 since we require the (de)interleave2 intrinsics instead of
-      // shufflevectors.
-      assert((!Result || !VF.isScalable() || IG->getFactor() == 2) &&
+      // must be power of 2 since we require the (de)interleave2 intrinsics
+      // instead of shufflevectors.
+      assert((!Result || !VF.isScalable() || isPowerOf2_32(IG->getFactor())) &&
              "Unsupported interleave factor for scalable vectors");
       return Result;
     };
@@ -9334,16 +9388,16 @@ LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes(VFRange &Range) {
         if (!Legal->isInvariantStoreOfReduction(SI))
           continue;
         auto *Recipe = new VPReplicateRecipe(
-            SI, RecipeBuilder.mapToVPValues(Instr->operands()),
+            SI, make_range(Operands.begin(), Operands.end()),
             true /* IsUniform */);
         Recipe->insertBefore(*MiddleVPBB, MBIP);
         continue;
       }
 
       VPRecipeBase *Recipe =
-          RecipeBuilder.tryToCreateWidenRecipe(Instr, Operands, Range, VPBB);
+          RecipeBuilder.tryToCreateWidenRecipe(Instr, Operands, Range);
       if (!Recipe)
-        Recipe = RecipeBuilder.handleReplication(Instr, Range);
+        Recipe = RecipeBuilder.handleReplication(Instr, Operands, Range);
 
       RecipeBuilder.setRecipe(Instr, Recipe);
       if (isa<VPHeaderPHIRecipe>(Recipe)) {
@@ -9392,20 +9446,16 @@ LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes(VFRange &Range) {
 
   if (auto *UncountableExitingBlock =
           Legal->getUncountableEarlyExitingBlock()) {
-    VPlanTransforms::handleUncountableEarlyExit(
-        *Plan, *PSE.getSE(), OrigLoop, UncountableExitingBlock, RecipeBuilder);
+    VPlanTransforms::runPass(VPlanTransforms::handleUncountableEarlyExit, *Plan,
+                             *PSE.getSE(), OrigLoop, UncountableExitingBlock,
+                             RecipeBuilder);
   }
   DenseMap<VPValue *, VPValue *> IVEndValues;
   addScalarResumePhis(RecipeBuilder, *Plan, IVEndValues);
   SetVector<VPIRInstruction *> ExitUsersToFix =
       collectUsersInExitBlocks(OrigLoop, RecipeBuilder, *Plan);
   addExitUsersForFirstOrderRecurrences(*Plan, ExitUsersToFix);
-  if (!addUsersInExitBlocks(*Plan, ExitUsersToFix)) {
-    reportVectorizationFailure(
-        "Some exit values in loop with uncountable exit not supported yet",
-        "UncountableEarlyExitLoopsUnsupportedExitValue", ORE, OrigLoop);
-    return nullptr;
-  }
+  addUsersInExitBlocks(*Plan, ExitUsersToFix);
 
   // ---------------------------------------------------------------------------
   // Transform initial VPlan: Apply previously taken decisions, in order, to
@@ -9418,8 +9468,9 @@ LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes(VFRange &Range) {
   // Interleave memory: for each Interleave Group we marked earlier as relevant
   // for this VPlan, replace the Recipes widening its memory instructions with a
   // single VPInterleaveRecipe at its insertion point.
-  VPlanTransforms::createInterleaveGroups(
-      *Plan, InterleaveGroups, RecipeBuilder, CM.isScalarEpilogueAllowed());
+  VPlanTransforms::runPass(VPlanTransforms::createInterleaveGroups, *Plan,
+                           InterleaveGroups, RecipeBuilder,
+                           CM.isScalarEpilogueAllowed());
 
   for (ElementCount VF : Range)
     Plan->addVF(VF);
@@ -9461,13 +9512,16 @@ LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes(VFRange &Range) {
     }
   }
 
-  VPlanTransforms::dropPoisonGeneratingRecipes(*Plan, [this](BasicBlock *BB) {
+  auto BlockNeedsPredication = [this](BasicBlock *BB) {
     return Legal->blockNeedsPredication(BB);
-  });
+  };
+  VPlanTransforms::runPass(VPlanTransforms::dropPoisonGeneratingRecipes, *Plan,
+                           BlockNeedsPredication);
 
   // Sink users of fixed-order recurrence past the recipe defining the previous
   // value and introduce FirstOrderRecurrenceSplice VPInstructions.
-  if (!VPlanTransforms::adjustFixedOrderRecurrences(*Plan, Builder))
+  if (!VPlanTransforms::runPass(VPlanTransforms::adjustFixedOrderRecurrences,
+                                *Plan, Builder))
     return nullptr;
 
   if (useActiveLaneMask(Style)) {
@@ -9509,12 +9563,6 @@ VPlanPtr LoopVectorizationPlanner::buildVPlan(VFRange &Range) {
       [this](PHINode *P) { return Legal->getIntOrFpInductionDescriptor(P); },
       *PSE.getSE(), *TLI);
 
-  // Remove the existing terminator of the exiting block of the top-most region.
-  // A BranchOnCount will be added instead when adding the canonical IV recipes.
-  auto *Term =
-      Plan->getVectorLoopRegion()->getExitingBasicBlock()->getTerminator();
-  Term->eraseFromParent();
-
   // Tail folding is not supported for outer loops, so the induction increment
   // is guaranteed to not wrap.
   bool HasNUW = true;
@@ -9703,7 +9751,11 @@ void LoopVectorizationPlanner::adjustRecipesForReductions(
     // beginning of the dedicated latch block.
     auto *OrigExitingVPV = PhiR->getBackedgeValue();
     auto *NewExitingVPV = PhiR->getBackedgeValue();
-    if (!PhiR->isInLoop() && CM.foldTailByMasking()) {
+    // Don't output selects for partial reductions because they have an output
+    // with fewer lanes than the VF. So the operands of the select would have
+    // different numbers of lanes. Partial reductions mask the input instead.
+    if (!PhiR->isInLoop() && CM.foldTailByMasking() &&
+        !isa<VPPartialReductionRecipe>(OrigExitingVPV->getDefiningRecipe())) {
       VPValue *Cond = RecipeBuilder.getBlockInMask(OrigLoop->getHeader());
       assert(OrigExitingVPV->getDefiningRecipe()->getParent() != LatchVPBB &&
              "reduction recipe must be defined before latch");
@@ -9816,10 +9868,10 @@ void LoopVectorizationPlanner::adjustRecipesForReductions(
       PhiR->setOperand(0, Plan->getOrAddLiveIn(RdxDesc.getSentinelValue()));
     }
   }
-
-  VPlanTransforms::clearReductionWrapFlags(*Plan);
   for (VPRecipeBase *R : ToDelete)
     R->eraseFromParent();
+
+  VPlanTransforms::runPass(VPlanTransforms::clearReductionWrapFlags, *Plan);
 }
 
 void VPDerivedIVRecipe::execute(VPTransformState &State) {
@@ -10052,9 +10104,9 @@ static void checkMixedPrecision(Loop *L, OptimizationRemarkEmitter *ORE) {
 
 static bool areRuntimeChecksProfitable(GeneratedRTChecks &Checks,
                                        VectorizationFactor &VF, Loop *L,
-                                       const TargetTransformInfo &TTI,
                                        PredicatedScalarEvolution &PSE,
-                                       ScalarEpilogueLowering SEL) {
+                                       ScalarEpilogueLowering SEL,
+                                       std::optional<unsigned> VScale) {
   InstructionCost CheckCost = Checks.getCost();
   if (!CheckCost.isValid())
     return false;
@@ -10104,7 +10156,7 @@ static bool areRuntimeChecksProfitable(GeneratedRTChecks &Checks,
   // For now we assume the epilogue cost EpiC = 0 for simplicity. Note that
   // the computations are performed on doubles, not integers and the result
   // is rounded up, hence we get an upper estimate of the TC.
-  unsigned IntVF = getEstimatedRuntimeVF(L, TTI, VF.Width);
+  unsigned IntVF = getEstimatedRuntimeVF(VF.Width, VScale);
   uint64_t RtC = *CheckCost.getValue();
   uint64_t Div = ScalarC * IntVF - *VF.Cost.getValue();
   uint64_t MinTC1 = Div == 0 ? 0 : divideCeil(RtC * IntVF, Div);
@@ -10183,7 +10235,7 @@ static void preparePlanForMainVectorLoop(VPlan &MainPlan, VPlan &EpiPlan) {
     VPIRInst->eraseFromParent();
     ResumePhi->eraseFromParent();
   }
-  VPlanTransforms::removeDeadRecipes(MainPlan);
+  VPlanTransforms::runPass(VPlanTransforms::removeDeadRecipes, MainPlan);
 
   using namespace VPlanPatternMatch;
   VPBasicBlock *MainScalarPH = MainPlan.getScalarPreheader();
@@ -10302,8 +10354,8 @@ preparePlanForEpilogueVectorLoop(VPlan &Plan, Loop *L,
         // value. This ensures correctness when the start value might not be
         // less than the minimum value of a monotonically increasing induction
         // variable.
-        IRBuilder<> Builder(
-            cast<Instruction>(ResumeV)->getParent()->getFirstNonPHI());
+        BasicBlock *ResumeBB = cast<Instruction>(ResumeV)->getParent();
+        IRBuilder<> Builder(ResumeBB, ResumeBB->getFirstNonPHIIt());
         Value *Cmp =
             Builder.CreateICmpEQ(ResumeV, RdxDesc.getRecurrenceStartValue());
         ResumeV =
@@ -10541,7 +10593,8 @@ bool LoopVectorizePass::processLoop(Loop *L) {
     bool ForceVectorization =
         Hints.getForce() == LoopVectorizeHints::FK_Enabled;
     if (!ForceVectorization &&
-        !areRuntimeChecksProfitable(Checks, VF, L, *TTI, PSE, SEL)) {
+        !areRuntimeChecksProfitable(Checks, VF, L, PSE, SEL,
+                                    CM.getVScaleForTuning())) {
       ORE->emit([&]() {
         return OptimizationRemarkAnalysisAliasing(
                    DEBUG_TYPE, "CantReorderMemOps", L->getStartLoc(),
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index eea6b32460d70..fb1054ee30aab 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -963,8 +963,9 @@ static InstructionsState getSameOpcode(ArrayRef<Value *> VL,
       return InstructionsState::invalid();
   }
   bool AnyPoison = InstCnt != VL.size();
-  // Skip MainOp.
-  for (Value *V : iterator_range(It + 1, VL.end())) {
+  // Check MainOp too to be sure that it matches the requirements for the
+  // instructions.
+  for (Value *V : iterator_range(It, VL.end())) {
     auto *I = dyn_cast<Instruction>(V);
     if (!I)
       continue;
@@ -1313,6 +1314,22 @@ static bool hasFullVectorsOrPowerOf2(const TargetTransformInfo &TTI, Type *Ty,
          Sz % NumParts == 0;
 }
 
+/// Returns number of parts, the type \p VecTy will be split at the codegen
+/// phase. If the type is going to be scalarized or does not uses whole
+/// registers, returns 1.
+static unsigned
+getNumberOfParts(const TargetTransformInfo &TTI, VectorType *VecTy,
+                 const unsigned Limit = std::numeric_limits<unsigned>::max()) {
+  unsigned NumParts = TTI.getNumberOfParts(VecTy);
+  if (NumParts == 0 || NumParts >= Limit)
+    return 1;
+  unsigned Sz = getNumElements(VecTy);
+  if (NumParts >= Sz || Sz % NumParts != 0 ||
+      !hasFullVectorsOrPowerOf2(TTI, VecTy->getElementType(), Sz / NumParts))
+    return 1;
+  return NumParts;
+}
+
 namespace slpvectorizer {
 
 /// Bottom Up SLP Vectorizer.
@@ -1378,7 +1395,7 @@ class BoUpSLP {
 
   /// \returns the cost incurred by unwanted spills and fills, caused by
   /// holding live values over call sites.
-  InstructionCost getSpillCost() const;
+  InstructionCost getSpillCost();
 
   /// \returns the vectorization cost of the subtree that starts at \p VL.
   /// A negative number means that this is profitable.
@@ -1459,8 +1476,7 @@ class BoUpSLP {
   /// Clear the internal data structures that are created by 'buildTree'.
   void deleteTree() {
     VectorizableTree.clear();
-    ScalarToTreeEntry.clear();
-    MultiNodeScalars.clear();
+    ScalarToTreeEntries.clear();
     MustGather.clear();
     NonScheduledFirst.clear();
     EntryToLastInstruction.clear();
@@ -1743,7 +1759,7 @@ class BoUpSLP {
 
             auto AllUsersVectorized = [U1, U2, this](Value *V) {
               return llvm::all_of(V->users(), [U1, U2, this](Value *U) {
-                return U == U1 || U == U2 || R.getTreeEntry(U) != nullptr;
+                return U == U1 || U == U2 || R.isVectorized(U);
               });
             };
             return AllUsersVectorized(V1) && AllUsersVectorized(V2);
@@ -1759,9 +1775,13 @@ class BoUpSLP {
       }
 
       auto CheckSameEntryOrFail = [&]() {
-        if (const TreeEntry *TE1 = R.getTreeEntry(V1);
-            TE1 && TE1 == R.getTreeEntry(V2))
-          return LookAheadHeuristics::ScoreSplatLoads;
+        if (ArrayRef<TreeEntry *> TEs1 = R.getTreeEntries(V1); !TEs1.empty()) {
+          SmallPtrSet<TreeEntry *, 4> Set(TEs1.begin(), TEs1.end());
+          if (ArrayRef<TreeEntry *> TEs2 = R.getTreeEntries(V2);
+              !TEs2.empty() &&
+              any_of(TEs2, [&](TreeEntry *E) { return Set.contains(E); }))
+            return LookAheadHeuristics::ScoreSplatLoads;
+        }
         return LookAheadHeuristics::ScoreFail;
       };
 
@@ -2834,13 +2854,7 @@ class BoUpSLP {
         continue;
       auto *I = cast<Instruction>(V);
       salvageDebugInfo(*I);
-      SmallVector<const TreeEntry *> Entries;
-      if (const TreeEntry *Entry = getTreeEntry(I)) {
-        Entries.push_back(Entry);
-        auto It = MultiNodeScalars.find(I);
-        if (It != MultiNodeScalars.end())
-          Entries.append(It->second.begin(), It->second.end());
-      }
+      ArrayRef<TreeEntry *> Entries = getTreeEntries(I);
       for (Use &U : I->operands()) {
         if (auto *OpI = dyn_cast_if_present<Instruction>(U.get());
             OpI && !DeletedInstructions.contains(OpI) && OpI->hasOneUser() &&
@@ -2944,7 +2958,10 @@ class BoUpSLP {
   }
 
   /// Check if the value is vectorized in the tree.
-  bool isVectorized(Value *V) const { return getTreeEntry(V); }
+  bool isVectorized(const Value *V) const {
+    assert(V && "V cannot be nullptr.");
+    return ScalarToTreeEntries.contains(V);
+  }
 
   ~BoUpSLP();
 
@@ -2982,16 +2999,10 @@ class BoUpSLP {
     ArrayRef<Value *> VL = UserTE->getOperand(OpIdx);
     TreeEntry *TE = nullptr;
     const auto *It = find_if(VL, [&](Value *V) {
-      TE = getTreeEntry(V);
-      if (TE && is_contained(TE->UserTreeIndices, EdgeInfo(UserTE, OpIdx)))
-        return true;
-      auto It = MultiNodeScalars.find(V);
-      if (It != MultiNodeScalars.end()) {
-        for (TreeEntry *E : It->second) {
-          if (is_contained(E->UserTreeIndices, EdgeInfo(UserTE, OpIdx))) {
-            TE = E;
-            return true;
-          }
+      for (TreeEntry *E : getTreeEntries(V)) {
+        if (is_contained(E->UserTreeIndices, EdgeInfo(UserTE, OpIdx))) {
+          TE = E;
+          return true;
         }
       }
       return false;
@@ -3642,16 +3653,24 @@ class BoUpSLP {
       Last->ReorderIndices.append(ReorderIndices.begin(), ReorderIndices.end());
     }
     if (!Last->isGather()) {
+      SmallPtrSet<Value *, 4> Processed;
       for (Value *V : VL) {
-        const TreeEntry *TE = getTreeEntry(V);
-        assert((!TE || TE == Last || doesNotNeedToBeScheduled(V)) &&
-               "Scalar already in tree!");
-        if (TE) {
-          if (TE != Last)
-            MultiNodeScalars.try_emplace(V).first->getSecond().push_back(Last);
+        if (isa<PoisonValue>(V))
           continue;
+        auto It = ScalarToTreeEntries.find(V);
+        assert(
+            (It == ScalarToTreeEntries.end() ||
+             (It->getSecond().size() == 1 && It->getSecond().front() == Last) ||
+             doesNotNeedToBeScheduled(V)) &&
+            "Scalar already in tree!");
+        if (It == ScalarToTreeEntries.end()) {
+          ScalarToTreeEntries.try_emplace(V).first->getSecond().push_back(Last);
+          (void)Processed.insert(V);
+        } else if (Processed.insert(V).second) {
+          assert(!is_contained(It->getSecond(), Last) &&
+                 "Value already associated with the node.");
+          It->getSecond().push_back(Last);
         }
-        ScalarToTreeEntry[V] = Last;
       }
       // Update the scheduler bundle to point to this TreeEntry.
       ScheduleData *BundleMember = *Bundle;
@@ -3706,14 +3725,23 @@ class BoUpSLP {
   }
 #endif
 
-  TreeEntry *getTreeEntry(Value *V) {
+  /// Get list of vector entries, associated with the value \p V.
+  ArrayRef<TreeEntry *> getTreeEntries(Value *V) const {
     assert(V && "V cannot be nullptr.");
-    return ScalarToTreeEntry.lookup(V);
+    auto It = ScalarToTreeEntries.find(V);
+    if (It == ScalarToTreeEntries.end())
+      return {};
+    return It->getSecond();
   }
 
-  const TreeEntry *getTreeEntry(Value *V) const {
+  /// Returns first vector node for value \p V, matching values \p VL.
+  TreeEntry *getSameValuesTreeEntry(Value *V, ArrayRef<Value *> VL,
+                                    bool SameVF = false) const {
     assert(V && "V cannot be nullptr.");
-    return ScalarToTreeEntry.lookup(V);
+    for (TreeEntry *TE : ScalarToTreeEntries.lookup(V))
+      if ((!SameVF || TE->getVectorFactor() == VL.size()) && TE->isSame(VL))
+        return TE;
+    return nullptr;
   }
 
   /// Check that the operand node of alternate node does not generate
@@ -3733,12 +3761,8 @@ class BoUpSLP {
                                OrdersType &CurrentOrder,
                                SmallVectorImpl<Value *> &PointerOps);
 
-  /// Maps a specific scalar to its tree entry.
-  SmallDenseMap<Value *, TreeEntry *> ScalarToTreeEntry;
-
-  /// List of scalars, used in several vectorize nodes, and the list of the
-  /// nodes.
-  SmallDenseMap<Value *, SmallVector<TreeEntry *>> MultiNodeScalars;
+  /// Maps a specific scalar to its tree entry(ies).
+  SmallDenseMap<Value *, SmallVector<TreeEntry *>> ScalarToTreeEntries;
 
   /// Maps a value to the proposed vectorizable size.
   SmallDenseMap<Value *, unsigned> InstrElementSize;
@@ -3779,16 +3803,19 @@ class BoUpSLP {
 
   /// This POD struct describes one external user in the vectorized tree.
   struct ExternalUser {
-    ExternalUser(Value *S, llvm::User *U, int L)
-        : Scalar(S), User(U), Lane(L) {}
+    ExternalUser(Value *S, llvm::User *U, const TreeEntry &E, int L)
+        : Scalar(S), User(U), E(E), Lane(L) {}
+
+    /// Which scalar in our function.
+    Value *Scalar = nullptr;
 
-    // Which scalar in our function.
-    Value *Scalar;
+    /// Which user that uses the scalar.
+    llvm::User *User = nullptr;
 
-    // Which user that uses the scalar.
-    llvm::User *User;
+    /// Vector node, the value is part of.
+    const TreeEntry &E;
 
-    // Which lane does the scalar belong to.
+    /// Which lane does the scalar belong to.
     int Lane;
   };
   using UserList = SmallVector<ExternalUser, 16>;
@@ -4615,12 +4642,7 @@ BoUpSLP::findReusedOrderedScalars(const BoUpSLP::TreeEntry &TE) {
   if (!isValidElementType(ScalarTy))
     return std::nullopt;
   auto *VecTy = getWidenedType(ScalarTy, NumScalars);
-  int NumParts = TTI->getNumberOfParts(VecTy);
-  if (NumParts == 0 || NumParts >= NumScalars ||
-      VecTy->getNumElements() % NumParts != 0 ||
-      !hasFullVectorsOrPowerOf2(*TTI, VecTy->getElementType(),
-                                VecTy->getNumElements() / NumParts))
-    NumParts = 1;
+  unsigned NumParts = ::getNumberOfParts(*TTI, VecTy, NumScalars);
   SmallVector<int> ExtractMask;
   SmallVector<int> Mask;
   SmallVector<SmallVector<const TreeEntry *>> Entries;
@@ -5099,7 +5121,7 @@ BoUpSLP::canVectorizeLoads(ArrayRef<Value *> VL, const Value *VL0,
     auto IsAnyPointerUsedOutGraph =
         IsPossibleStrided && any_of(PointerOps, [&](Value *V) {
           return isa<Instruction>(V) && any_of(V->users(), [&](User *U) {
-                   return !getTreeEntry(U) && !MustGather.contains(U);
+                   return !isVectorized(U) && !MustGather.contains(U);
                  });
         });
     const unsigned AbsoluteDiff = std::abs(*Diff);
@@ -5571,8 +5593,10 @@ BoUpSLP::getReorderingData(const TreeEntry &TE, bool TopToBottom) {
       }
     }
     if (Sz == 2 && TE.getVectorFactor() == 4 &&
-        TTI->getNumberOfParts(getWidenedType(TE.Scalars.front()->getType(),
-                                             2 * TE.getVectorFactor())) == 1)
+        ::getNumberOfParts(*TTI, getWidenedType(TE.Scalars.front()->getType(),
+                                                2 * TE.getVectorFactor())) == 1)
+      return std::nullopt;
+    if (TE.ReuseShuffleIndices.size() % Sz != 0)
       return std::nullopt;
     if (!ShuffleVectorInst::isOneUseSingleSourceMask(TE.ReuseShuffleIndices,
                                                      Sz)) {
@@ -5604,7 +5628,7 @@ BoUpSLP::getReorderingData(const TreeEntry &TE, bool TopToBottom) {
         UsedVals.set(Val);
         for (unsigned K = 0; K < NumParts; ++K) {
           unsigned Idx = Val + Sz * K;
-          if (Idx < VF)
+          if (Idx < VF && I + K < VF)
             ResOrder[Idx] = I + K;
         }
       }
@@ -6558,7 +6582,7 @@ void BoUpSLP::buildExternalUses(
         LLVM_DEBUG(dbgs() << "SLP: Need to extract: Extra arg from lane "
                           << FoundLane << " from " << *Scalar << ".\n");
         ScalarToExtUses.try_emplace(Scalar, ExternalUses.size());
-        ExternalUses.emplace_back(Scalar, nullptr, FoundLane);
+        ExternalUses.emplace_back(Scalar, nullptr, *Entry, FoundLane);
         continue;
       }
       for (User *U : Scalar->users()) {
@@ -6573,16 +6597,24 @@ void BoUpSLP::buildExternalUses(
           continue;
 
         // Skip in-tree scalars that become vectors
-        if (TreeEntry *UseEntry = getTreeEntry(U)) {
+        if (ArrayRef<TreeEntry *> UseEntries = getTreeEntries(U);
+            !UseEntries.empty()) {
           // Some in-tree scalars will remain as scalar in vectorized
           // instructions. If that is the case, the one in FoundLane will
           // be used.
-          if (UseEntry->State == TreeEntry::ScatterVectorize ||
-              !doesInTreeUserNeedToExtract(
-                  Scalar, getRootEntryInstruction(*UseEntry), TLI, TTI)) {
+          if (any_of(UseEntries, [&](TreeEntry *UseEntry) {
+                return UseEntry->State == TreeEntry::ScatterVectorize ||
+                       !doesInTreeUserNeedToExtract(
+                           Scalar, getRootEntryInstruction(*UseEntry), TLI,
+                           TTI);
+              })) {
             LLVM_DEBUG(dbgs() << "SLP: \tInternal user will be removed:" << *U
                               << ".\n");
-            assert(!UseEntry->isGather() && "Bad state");
+            assert(none_of(UseEntries,
+                           [](TreeEntry *UseEntry) {
+                             return UseEntry->isGather();
+                           }) &&
+                   "Bad state");
             continue;
           }
           U = nullptr;
@@ -6599,7 +6631,7 @@ void BoUpSLP::buildExternalUses(
                           << " from lane " << FoundLane << " from " << *Scalar
                           << ".\n");
         It = ScalarToExtUses.try_emplace(Scalar, ExternalUses.size()).first;
-        ExternalUses.emplace_back(Scalar, U, FoundLane);
+        ExternalUses.emplace_back(Scalar, U, *Entry, FoundLane);
         if (!U)
           break;
       }
@@ -6630,7 +6662,7 @@ BoUpSLP::collectUserStores(const BoUpSLP::TreeEntry *TE) const {
           !isValidElementType(SI->getValueOperand()->getType()))
         continue;
       // Skip entry if already
-      if (getTreeEntry(U))
+      if (isVectorized(U))
         continue;
 
       Value *Ptr =
@@ -7013,10 +7045,11 @@ void BoUpSLP::tryToVectorizeGatheredLoads(
               for (User *U : LI->users()) {
                 if (auto *UI = dyn_cast<Instruction>(U); UI && isDeleted(UI))
                   continue;
-                if (const TreeEntry *UTE = getTreeEntry(U)) {
+                for (const TreeEntry *UTE : getTreeEntries(U)) {
                   for (int I : seq<int>(UTE->getNumOperands())) {
-                    if (all_of(UTE->getOperand(I),
-                               [LI](Value *V) { return V == LI; }))
+                    if (all_of(UTE->getOperand(I), [LI](Value *V) {
+                          return V == LI || isa<PoisonValue>(V);
+                        }))
                       // Found legal broadcast - do not vectorize.
                       return false;
                   }
@@ -7121,7 +7154,7 @@ void BoUpSLP::tryToVectorizeGatheredLoads(
           int LastDist = LocalLoadsDists.front().second;
           bool AllowMaskedGather = IsMaskedGatherSupported(OriginalLoads);
           for (const std::pair<LoadInst *, int> &L : LocalLoadsDists) {
-            if (getTreeEntry(L.first))
+            if (isVectorized(L.first))
               continue;
             assert(LastDist >= L.second &&
                    "Expected first distance always not less than second");
@@ -7173,9 +7206,9 @@ void BoUpSLP::tryToVectorizeGatheredLoads(
           for (auto [Slice, _] : Results) {
             LLVM_DEBUG(dbgs() << "SLP: Trying to vectorize gathered loads ("
                               << Slice.size() << ")\n");
-            if (any_of(Slice, [&](Value *V) { return getTreeEntry(V); })) {
+            if (any_of(Slice, [&](Value *V) { return isVectorized(V); })) {
               for (Value *L : Slice)
-                if (!getTreeEntry(L))
+                if (!isVectorized(L))
                   SortedNonVectorized.push_back(cast<LoadInst>(L));
               continue;
             }
@@ -7214,7 +7247,7 @@ void BoUpSLP::tryToVectorizeGatheredLoads(
                         any_of(E->Scalars, [&, Slice = Slice](Value *V) {
                           if (isa<Constant>(V))
                             return false;
-                          if (getTreeEntry(V))
+                          if (isVectorized(V))
                             return true;
                           const auto &Nodes = ValueToGatherNodes.at(V);
                           return (Nodes.size() != 1 || !Nodes.contains(E)) &&
@@ -7301,7 +7334,7 @@ void BoUpSLP::tryToVectorizeGatheredLoads(
               for (unsigned I = 0, E = Slice.size(); I < E; I += VF) {
                 ArrayRef<Value *> SubSlice =
                     Slice.slice(I, std::min(VF, E - I));
-                if (getTreeEntry(SubSlice.front()))
+                if (isVectorized(SubSlice.front()))
                   continue;
                 // Check if the subslice is to be-vectorized entry, which is not
                 // equal to entry.
@@ -7571,7 +7604,7 @@ bool BoUpSLP::areAltOperandsProfitable(const InstructionsState &S,
                    DenseMap<Value *, unsigned> Uniques;
                    for (Value *V : Op) {
                      if (isa<Constant, ExtractElementInst>(V) ||
-                         getTreeEntry(V) || (L && L->isLoopInvariant(V))) {
+                         isVectorized(V) || (L && L->isLoopInvariant(V))) {
                        if (isa<UndefValue>(V))
                          ++UndefCnt;
                        continue;
@@ -7589,7 +7622,7 @@ bool BoUpSLP::areAltOperandsProfitable(const InstructionsState &S,
                    return none_of(Uniques, [&](const auto &P) {
                      return P.first->hasNUsesOrMore(P.second + 1) &&
                             none_of(P.first->users(), [&](User *U) {
-                              return getTreeEntry(U) || Uniques.contains(U);
+                              return isVectorized(U) || Uniques.contains(U);
                             });
                    });
                  }) ||
@@ -7603,6 +7636,66 @@ bool BoUpSLP::areAltOperandsProfitable(const InstructionsState &S,
            NumAltInsts) < S.getMainOp()->getNumOperands() * VL.size());
 }
 
+/// Builds the arguments types vector for the given call instruction with the
+/// given \p ID for the specified vector factor.
+static SmallVector<Type *>
+buildIntrinsicArgTypes(const CallInst *CI, const Intrinsic::ID ID,
+                       const unsigned VF, unsigned MinBW,
+                       const TargetTransformInfo *TTI) {
+  SmallVector<Type *> ArgTys;
+  for (auto [Idx, Arg] : enumerate(CI->args())) {
+    if (ID != Intrinsic::not_intrinsic) {
+      if (isVectorIntrinsicWithScalarOpAtArg(ID, Idx, TTI)) {
+        ArgTys.push_back(Arg->getType());
+        continue;
+      }
+      if (MinBW > 0) {
+        ArgTys.push_back(
+            getWidenedType(IntegerType::get(CI->getContext(), MinBW), VF));
+        continue;
+      }
+    }
+    ArgTys.push_back(getWidenedType(Arg->getType(), VF));
+  }
+  return ArgTys;
+}
+
+/// Calculates the costs of vectorized intrinsic (if possible) and vectorized
+/// function (if possible) calls. Returns invalid cost for the corresponding
+/// calls, if they cannot be vectorized/will be scalarized.
+static std::pair<InstructionCost, InstructionCost>
+getVectorCallCosts(CallInst *CI, FixedVectorType *VecTy,
+                   TargetTransformInfo *TTI, TargetLibraryInfo *TLI,
+                   ArrayRef<Type *> ArgTys) {
+  auto Shape = VFShape::get(CI->getFunctionType(),
+                            ElementCount::getFixed(VecTy->getNumElements()),
+                            false /*HasGlobalPred*/);
+  Function *VecFunc = VFDatabase(*CI).getVectorizedFunction(Shape);
+  auto LibCost = InstructionCost::getInvalid();
+  if (!CI->isNoBuiltin() && VecFunc) {
+    // Calculate the cost of the vector library call.
+    // If the corresponding vector call is cheaper, return its cost.
+    LibCost =
+        TTI->getCallInstrCost(nullptr, VecTy, ArgTys, TTI::TCK_RecipThroughput);
+  }
+  Intrinsic::ID ID = getVectorIntrinsicIDForCall(CI, TLI);
+
+  // Calculate the cost of the vector intrinsic call.
+  FastMathFlags FMF;
+  if (auto *FPCI = dyn_cast<FPMathOperator>(CI))
+    FMF = FPCI->getFastMathFlags();
+  const InstructionCost ScalarLimit = 10000;
+  IntrinsicCostAttributes CostAttrs(ID, VecTy, ArgTys, FMF, nullptr,
+                                    LibCost.isValid() ? LibCost : ScalarLimit);
+  auto IntrinsicCost =
+      TTI->getIntrinsicInstrCost(CostAttrs, TTI::TCK_RecipThroughput);
+  if ((LibCost.isValid() && IntrinsicCost > LibCost) ||
+      (!LibCost.isValid() && IntrinsicCost > ScalarLimit))
+    IntrinsicCost = InstructionCost::getInvalid();
+
+  return {IntrinsicCost, LibCost};
+}
+
 BoUpSLP::TreeEntry::EntryState BoUpSLP::getScalarsVectorizationState(
     const InstructionsState &S, ArrayRef<Value *> VL,
     bool IsScatterVectorizeUserTE, OrdersType &CurrentOrder,
@@ -7943,6 +8036,12 @@ BoUpSLP::TreeEntry::EntryState BoUpSLP::getScalarsVectorizationState(
         return TreeEntry::NeedToGather;
       }
     }
+    SmallVector<Type *> ArgTys =
+        buildIntrinsicArgTypes(CI, ID, VL.size(), 0, TTI);
+    auto *VecTy = getWidenedType(S.getMainOp()->getType(), VL.size());
+    auto VecCallCosts = getVectorCallCosts(CI, VecTy, TTI, TLI, ArgTys);
+    if (!VecCallCosts.first.isValid() && !VecCallCosts.second.isValid())
+      return TreeEntry::NeedToGather;
 
     return TreeEntry::Vectorize;
   }
@@ -8153,59 +8252,25 @@ void BoUpSLP::buildTree_rec(ArrayRef<Value *> VL, unsigned Depth,
 
   // Check if this is a duplicate of another entry.
   if (S) {
-    if (TreeEntry *E = getTreeEntry(S.getMainOp())) {
-      LLVM_DEBUG(dbgs() << "SLP: \tChecking bundle: " << *S.getMainOp()
-                        << ".\n");
-      if (GatheredLoadsEntriesFirst.has_value() || !E->isSame(VL)) {
-        auto It = MultiNodeScalars.find(S.getMainOp());
-        if (It != MultiNodeScalars.end()) {
-          auto *TEIt = find_if(It->getSecond(),
-                               [&](TreeEntry *ME) { return ME->isSame(VL); });
-          if (TEIt != It->getSecond().end())
-            E = *TEIt;
-          else
-            E = nullptr;
-        } else {
-          E = nullptr;
-        }
-      }
-      if (!E) {
-        if (!doesNotNeedToBeScheduled(S.getMainOp())) {
-          LLVM_DEBUG(dbgs() << "SLP: Gathering due to partial overlap.\n");
-          if (TryToFindDuplicates(S))
-            newTreeEntry(VL, std::nullopt /*not vectorized*/, S, UserTreeIdx,
-                         ReuseShuffleIndices);
-          return;
-        }
-        SmallPtrSet<const TreeEntry *, 4> Nodes;
-        Nodes.insert(getTreeEntry(S.getMainOp()));
-        for (const TreeEntry *E : MultiNodeScalars.lookup(S.getMainOp()))
-          Nodes.insert(E);
-        SmallPtrSet<Value *, 8> Values(VL.begin(), VL.end());
-        if (any_of(Nodes, [&](const TreeEntry *E) {
-              if (all_of(E->Scalars,
-                         [&](Value *V) { return Values.contains(V); }))
-                return true;
-              SmallPtrSet<Value *, 8> EValues(E->Scalars.begin(),
-                                              E->Scalars.end());
-              return (
-                  all_of(VL, [&](Value *V) { return EValues.contains(V); }));
-            })) {
-          LLVM_DEBUG(dbgs() << "SLP: Gathering due to full overlap.\n");
-          if (TryToFindDuplicates(S))
-            newTreeEntry(VL, std::nullopt /*not vectorized*/, S, UserTreeIdx,
-                         ReuseShuffleIndices);
-          return;
-        }
-      } else {
-        // Record the reuse of the tree node.  FIXME, currently this is only
-        // used to properly draw the graph rather than for the actual
-        // vectorization.
+    LLVM_DEBUG(dbgs() << "SLP: \tChecking bundle: " << *S.getMainOp() << ".\n");
+    for (TreeEntry *E : getTreeEntries(S.getMainOp())) {
+      if (E->isSame(VL)) {
+        // Record the reuse of the tree node.
         E->UserTreeIndices.push_back(UserTreeIdx);
         LLVM_DEBUG(dbgs() << "SLP: Perfect diamond merge at " << *S.getMainOp()
                           << ".\n");
         return;
       }
+      SmallPtrSet<Value *, 8> Values(E->Scalars.begin(), E->Scalars.end());
+      if (all_of(VL, [&](Value *V) {
+            return isa<PoisonValue>(V) || Values.contains(V);
+          })) {
+        LLVM_DEBUG(dbgs() << "SLP: Gathering due to full overlap.\n");
+        if (TryToFindDuplicates(S))
+          newTreeEntry(VL, std::nullopt /*not vectorized*/, S, UserTreeIdx,
+                       ReuseShuffleIndices);
+        return;
+      }
     }
   }
 
@@ -8357,7 +8422,7 @@ void BoUpSLP::buildTree_rec(ArrayRef<Value *> VL, unsigned Depth,
     if ((!IsScatterVectorizeUserTE && !isa<Instruction>(V)) ||
         doesNotNeedToBeScheduled(V))
       continue;
-    if (getTreeEntry(V)) {
+    if (isVectorized(V)) {
       LLVM_DEBUG(dbgs() << "SLP: The instruction (" << *V
                         << ") is already in tree.\n");
       if (TryToFindDuplicates(S))
@@ -9015,40 +9080,11 @@ bool BoUpSLP::areAllUsersVectorized(
     Instruction *I, const SmallDenseSet<Value *> *VectorizedVals) const {
   return (I->hasOneUse() && (!VectorizedVals || VectorizedVals->contains(I))) ||
          all_of(I->users(), [this](User *U) {
-           return ScalarToTreeEntry.contains(U) ||
-                  isVectorLikeInstWithConstOps(U) ||
+           return isVectorized(U) || isVectorLikeInstWithConstOps(U) ||
                   (isa<ExtractElementInst>(U) && MustGather.contains(U));
          });
 }
 
-static std::pair<InstructionCost, InstructionCost>
-getVectorCallCosts(CallInst *CI, FixedVectorType *VecTy,
-                   TargetTransformInfo *TTI, TargetLibraryInfo *TLI,
-                   ArrayRef<Type *> ArgTys) {
-  Intrinsic::ID ID = getVectorIntrinsicIDForCall(CI, TLI);
-
-  // Calculate the cost of the scalar and vector calls.
-  FastMathFlags FMF;
-  if (auto *FPCI = dyn_cast<FPMathOperator>(CI))
-    FMF = FPCI->getFastMathFlags();
-  IntrinsicCostAttributes CostAttrs(ID, VecTy, ArgTys, FMF);
-  auto IntrinsicCost =
-    TTI->getIntrinsicInstrCost(CostAttrs, TTI::TCK_RecipThroughput);
-
-  auto Shape = VFShape::get(CI->getFunctionType(),
-                            ElementCount::getFixed(VecTy->getNumElements()),
-                            false /*HasGlobalPred*/);
-  Function *VecFunc = VFDatabase(*CI).getVectorizedFunction(Shape);
-  auto LibCost = IntrinsicCost;
-  if (!CI->isNoBuiltin() && VecFunc) {
-    // Calculate the cost of the vector library call.
-    // If the corresponding vector call is cheaper, return its cost.
-    LibCost =
-        TTI->getCallInstrCost(nullptr, VecTy, ArgTys, TTI::TCK_RecipThroughput);
-  }
-  return {IntrinsicCost, LibCost};
-}
-
 void BoUpSLP::TreeEntry::buildAltOpShuffleMask(
     const function_ref<bool(Instruction *)> IsAltOp, SmallVectorImpl<int> &Mask,
     SmallVectorImpl<Value *> *OpScalars,
@@ -9372,8 +9408,15 @@ class BaseShuffleAnalysis {
   /// instruction.
   template <typename T, typename ShuffleBuilderTy>
   static T createShuffle(Value *V1, Value *V2, ArrayRef<int> Mask,
-                         ShuffleBuilderTy &Builder) {
+                         ShuffleBuilderTy &Builder, Type *ScalarTy) {
     assert(V1 && "Expected at least one vector value.");
+    unsigned ScalarTyNumElements = getNumElements(ScalarTy);
+    SmallVector<int> NewMask(Mask);
+    if (ScalarTyNumElements != 1) {
+      assert(SLPReVec && "FixedVectorType is not expected.");
+      transformScalarShuffleIndiciesToVector(ScalarTyNumElements, NewMask);
+      Mask = NewMask;
+    }
     if (V2)
       Builder.resizeToMatch(V1, V2);
     int VF = Mask.size();
@@ -9476,7 +9519,6 @@ class BaseShuffleAnalysis {
     if (isa<PoisonValue>(V1))
       return Builder.createPoison(
           cast<VectorType>(V1->getType())->getElementType(), Mask.size());
-    SmallVector<int> NewMask(Mask);
     bool IsIdentity = peekThroughShuffles(V1, NewMask, /*SinglePermute=*/true);
     assert(V1 && "Expected non-null value after looking through shuffles.");
 
@@ -9824,26 +9866,22 @@ void BoUpSLP::transformNodes() {
           ArrayRef<Value *> Slice = VL.slice(Cnt, VF);
           // If any instruction is vectorized already - do not try again.
           // Reuse the existing node, if it fully matches the slice.
-          if (const TreeEntry *SE = getTreeEntry(Slice.front());
-              SE || getTreeEntry(Slice.back())) {
-            if (!SE)
-              continue;
-            if (VF != SE->getVectorFactor() || !SE->isSame(Slice))
-              continue;
-          }
+          if (isVectorized(Slice.front()) &&
+              !getSameValuesTreeEntry(Slice.front(), Slice, /*SameVF=*/true))
+            continue;
           // Constant already handled effectively - skip.
           if (allConstant(Slice))
             continue;
           // Do not try to vectorize small splats (less than vector register and
           // only with the single non-undef element).
           bool IsSplat = isSplat(Slice);
-          if (Slices.empty() || !IsSplat ||
-              (VF <= 2 && 2 * std::clamp(TTI->getNumberOfParts(getWidenedType(
-                                             Slice.front()->getType(), VF)),
-                                         1U, VF - 1) !=
-                              std::clamp(TTI->getNumberOfParts(getWidenedType(
-                                             Slice.front()->getType(), 2 * VF)),
-                                         1U, 2 * VF)) ||
+          bool IsTwoRegisterSplat = true;
+          if (IsSplat && VF == 2) {
+            unsigned NumRegs2VF = ::getNumberOfParts(
+                *TTI, getWidenedType(Slice.front()->getType(), 2 * VF));
+            IsTwoRegisterSplat = NumRegs2VF == 2;
+          }
+          if (Slices.empty() || !IsSplat || !IsTwoRegisterSplat ||
               count(Slice, Slice.front()) ==
                   static_cast<long>(isa<UndefValue>(Slice.front()) ? VF - 1
                                                                    : 1)) {
@@ -9913,12 +9951,8 @@ void BoUpSLP::transformNodes() {
         for (auto [Cnt, Sz] : Slices) {
           ArrayRef<Value *> Slice = VL.slice(Cnt, Sz);
           // If any instruction is vectorized already - do not try again.
-          if (TreeEntry *SE = getTreeEntry(Slice.front());
-              SE || getTreeEntry(Slice.back())) {
-            if (!SE)
-              continue;
-            if (VF != SE->getVectorFactor() || !SE->isSame(Slice))
-              continue;
+          if (TreeEntry *SE = getSameValuesTreeEntry(Slice.front(), Slice,
+                                                     /*SameVF=*/true)) {
             SE->UserTreeIndices.emplace_back(&E, UINT_MAX);
             AddCombinedNode(SE->Idx, Cnt, Sz);
             continue;
@@ -10360,7 +10394,7 @@ class BoUpSLP::ShuffleCostEstimator : public BaseShuffleAnalysis {
     SameNodesEstimated = false;
     if (!E2 && InVectors.size() == 1) {
       unsigned VF = E1.getVectorFactor();
-      if (Value *V1 = InVectors.front().dyn_cast<Value *>()) {
+      if (Value *V1 = dyn_cast<Value *>(InVectors.front())) {
         VF = std::max(VF,
                       cast<FixedVectorType>(V1->getType())->getNumElements());
       } else {
@@ -10376,7 +10410,7 @@ class BoUpSLP::ShuffleCostEstimator : public BaseShuffleAnalysis {
       auto P = InVectors.front();
       Cost += createShuffle(&E1, E2, Mask);
       unsigned VF = Mask.size();
-      if (Value *V1 = P.dyn_cast<Value *>()) {
+      if (Value *V1 = dyn_cast<Value *>(P)) {
         VF = std::max(VF,
                       getNumElements(V1->getType()));
       } else {
@@ -10641,17 +10675,12 @@ class BoUpSLP::ShuffleCostEstimator : public BaseShuffleAnalysis {
           V2 = getAllOnesValue(*R.DL, getWidenedType(ScalarTy, CommonVF));
       }
     }
-    if (auto *VecTy = dyn_cast<FixedVectorType>(ScalarTy)) {
-      assert(SLPReVec && "FixedVectorType is not expected.");
-      transformScalarShuffleIndiciesToVector(VecTy->getNumElements(),
-                                             CommonMask);
-    }
     InVectors.front() =
         Constant::getNullValue(getWidenedType(ScalarTy, CommonMask.size()));
     if (InVectors.size() == 2)
       InVectors.pop_back();
     return ExtraCost + BaseShuffleAnalysis::createShuffle<InstructionCost>(
-                           V1, V2, CommonMask, Builder);
+                           V1, V2, CommonMask, Builder, ScalarTy);
   }
 
 public:
@@ -10691,6 +10720,7 @@ class BoUpSLP::ShuffleCostEstimator : public BaseShuffleAnalysis {
         });
     SmallPtrSet<Value *, 4> UniqueBases;
     unsigned SliceSize = getPartNumElems(VL.size(), NumParts);
+    SmallDenseMap<Value *, APInt, 4> VectorOpsToExtracts;
     for (unsigned Part : seq<unsigned>(NumParts)) {
       unsigned Limit = getNumElems(VL.size(), SliceSize, Part);
       ArrayRef<int> SubMask = Mask.slice(Part * SliceSize, Limit);
@@ -10709,7 +10739,7 @@ class BoUpSLP::ShuffleCostEstimator : public BaseShuffleAnalysis {
         auto *EE = cast<ExtractElementInst>(V);
         VecBase = EE->getVectorOperand();
         UniqueBases.insert(VecBase);
-        const TreeEntry *VE = R.getTreeEntry(V);
+        ArrayRef<TreeEntry *> VEs = R.getTreeEntries(V);
         if (!CheckedExtracts.insert(V).second ||
             !R.areAllUsersVectorized(cast<Instruction>(V), &VectorizedVals) ||
             any_of(EE->users(),
@@ -10718,7 +10748,7 @@ class BoUpSLP::ShuffleCostEstimator : public BaseShuffleAnalysis {
                             !R.areAllUsersVectorized(cast<Instruction>(U),
                                                      &VectorizedVals);
                    }) ||
-            (VE && VE != E))
+            (!VEs.empty() && !is_contained(VEs, E)))
           continue;
         std::optional<unsigned> EEIdx = getExtractIndex(EE);
         if (!EEIdx)
@@ -10741,10 +10771,18 @@ class BoUpSLP::ShuffleCostEstimator : public BaseShuffleAnalysis {
             continue;
           }
         }
-        Cost -= TTI.getVectorInstrCost(*EE, EE->getVectorOperandType(),
-                                       CostKind, Idx);
-      }
-    }
+        APInt &DemandedElts =
+            VectorOpsToExtracts
+                .try_emplace(VecBase,
+                             APInt::getZero(getNumElements(VecBase->getType())))
+                .first->getSecond();
+        DemandedElts.setBit(Idx);
+      }
+    }
+    for (const auto &[Vec, DemandedElts] : VectorOpsToExtracts)
+      Cost -= TTI.getScalarizationOverhead(cast<VectorType>(Vec->getType()),
+                                           DemandedElts, /*Insert=*/false,
+                                           /*Extract=*/true, CostKind);
     // Check that gather of extractelements can be represented as just a
     // shuffle of a single/two vectors the scalars are extracted from.
     // Found the bunch of extractelement instructions that must be gathered
@@ -10789,12 +10827,7 @@ class BoUpSLP::ShuffleCostEstimator : public BaseShuffleAnalysis {
     }
     assert(!CommonMask.empty() && "Expected non-empty common mask.");
     auto *MaskVecTy = getWidenedType(ScalarTy, Mask.size());
-    unsigned NumParts = TTI.getNumberOfParts(MaskVecTy);
-    if (NumParts == 0 || NumParts >= Mask.size() ||
-        MaskVecTy->getNumElements() % NumParts != 0 ||
-        !hasFullVectorsOrPowerOf2(TTI, MaskVecTy->getElementType(),
-                                  MaskVecTy->getNumElements() / NumParts))
-      NumParts = 1;
+    unsigned NumParts = ::getNumberOfParts(TTI, MaskVecTy, Mask.size());
     unsigned SliceSize = getPartNumElems(Mask.size(), NumParts);
     const auto *It =
         find_if(Mask, [](int Idx) { return Idx != PoisonMaskElem; });
@@ -10809,12 +10842,7 @@ class BoUpSLP::ShuffleCostEstimator : public BaseShuffleAnalysis {
     }
     assert(!CommonMask.empty() && "Expected non-empty common mask.");
     auto *MaskVecTy = getWidenedType(ScalarTy, Mask.size());
-    unsigned NumParts = TTI.getNumberOfParts(MaskVecTy);
-    if (NumParts == 0 || NumParts >= Mask.size() ||
-        MaskVecTy->getNumElements() % NumParts != 0 ||
-        !hasFullVectorsOrPowerOf2(TTI, MaskVecTy->getElementType(),
-                                  MaskVecTy->getNumElements() / NumParts))
-      NumParts = 1;
+    unsigned NumParts = ::getNumberOfParts(TTI, MaskVecTy, Mask.size());
     unsigned SliceSize = getPartNumElems(Mask.size(), NumParts);
     const auto *It =
         find_if(Mask, [](int Idx) { return Idx != PoisonMaskElem; });
@@ -11066,30 +11094,6 @@ TTI::CastContextHint BoUpSLP::getCastContextHint(const TreeEntry &TE) const {
   return TTI::CastContextHint::None;
 }
 
-/// Builds the arguments types vector for the given call instruction with the
-/// given \p ID for the specified vector factor.
-static SmallVector<Type *>
-buildIntrinsicArgTypes(const CallInst *CI, const Intrinsic::ID ID,
-                       const unsigned VF, unsigned MinBW,
-                       const TargetTransformInfo *TTI) {
-  SmallVector<Type *> ArgTys;
-  for (auto [Idx, Arg] : enumerate(CI->args())) {
-    if (ID != Intrinsic::not_intrinsic) {
-      if (isVectorIntrinsicWithScalarOpAtArg(ID, Idx, TTI)) {
-        ArgTys.push_back(Arg->getType());
-        continue;
-      }
-      if (MinBW > 0) {
-        ArgTys.push_back(
-            getWidenedType(IntegerType::get(CI->getContext(), MinBW), VF));
-        continue;
-      }
-    }
-    ArgTys.push_back(getWidenedType(Arg->getType(), VF));
-  }
-  return ArgTys;
-}
-
 InstructionCost
 BoUpSLP::getEntryCost(const TreeEntry *E, ArrayRef<Value *> VectorizedVals,
                       SmallPtrSetImpl<Value *> &CheckedExtracts) {
@@ -11161,13 +11165,14 @@ BoUpSLP::getEntryCost(const TreeEntry *E, ArrayRef<Value *> VectorizedVals,
   const unsigned Sz = UniqueValues.size();
   SmallBitVector UsedScalars(Sz, false);
   for (unsigned I = 0; I < Sz; ++I) {
-    if (isa<Instruction>(UniqueValues[I]) && getTreeEntry(UniqueValues[I]) == E)
+    if (isa<Instruction>(UniqueValues[I]) &&
+        is_contained(getTreeEntries(UniqueValues[I]), E))
       continue;
     UsedScalars.set(I);
   }
   auto GetCastContextHint = [&](Value *V) {
-    if (const TreeEntry *OpTE = getTreeEntry(V))
-      return getCastContextHint(*OpTE);
+    if (ArrayRef<TreeEntry *> OpTEs = getTreeEntries(V); OpTEs.size() == 1)
+      return getCastContextHint(*OpTEs.front());
     InstructionsState SrcState = getSameOpcode(E->getOperand(0), *TLI);
     if (SrcState && SrcState.getOpcode() == Instruction::Load &&
         !SrcState.isAltShuffle())
@@ -11289,35 +11294,39 @@ BoUpSLP::getEntryCost(const TreeEntry *E, ArrayRef<Value *> VectorizedVals,
         Value *Op = PHI->getIncomingValue(I);
         Operands[I] = Op;
       }
-      if (const TreeEntry *OpTE = getTreeEntry(Operands.front()))
-        if (OpTE->isSame(Operands) && CountedOps.insert(OpTE).second)
-          if (!OpTE->ReuseShuffleIndices.empty())
-            ScalarCost += TTI::TCC_Basic * (OpTE->ReuseShuffleIndices.size() -
-                                            OpTE->Scalars.size());
+      if (const TreeEntry *OpTE =
+              getSameValuesTreeEntry(Operands.front(), Operands))
+        if (CountedOps.insert(OpTE).second &&
+            !OpTE->ReuseShuffleIndices.empty())
+          ScalarCost += TTI::TCC_Basic * (OpTE->ReuseShuffleIndices.size() -
+                                          OpTE->Scalars.size());
     }
 
     return CommonCost - ScalarCost;
   }
   case Instruction::ExtractValue:
   case Instruction::ExtractElement: {
+    APInt DemandedElts;
+    VectorType *SrcVecTy = nullptr;
     auto GetScalarCost = [&](unsigned Idx) {
       if (isa<PoisonValue>(UniqueValues[Idx]))
         return InstructionCost(TTI::TCC_Free);
 
       auto *I = cast<Instruction>(UniqueValues[Idx]);
-      VectorType *SrcVecTy;
-      if (ShuffleOrOp == Instruction::ExtractElement) {
-        auto *EE = cast<ExtractElementInst>(I);
-        SrcVecTy = EE->getVectorOperandType();
-      } else {
-        auto *EV = cast<ExtractValueInst>(I);
-        Type *AggregateTy = EV->getAggregateOperand()->getType();
-        unsigned NumElts;
-        if (auto *ATy = dyn_cast<ArrayType>(AggregateTy))
-          NumElts = ATy->getNumElements();
-        else
-          NumElts = AggregateTy->getStructNumElements();
-        SrcVecTy = getWidenedType(OrigScalarTy, NumElts);
+      if (!SrcVecTy) {
+        if (ShuffleOrOp == Instruction::ExtractElement) {
+          auto *EE = cast<ExtractElementInst>(I);
+          SrcVecTy = EE->getVectorOperandType();
+        } else {
+          auto *EV = cast<ExtractValueInst>(I);
+          Type *AggregateTy = EV->getAggregateOperand()->getType();
+          unsigned NumElts;
+          if (auto *ATy = dyn_cast<ArrayType>(AggregateTy))
+            NumElts = ATy->getNumElements();
+          else
+            NumElts = AggregateTy->getStructNumElements();
+          SrcVecTy = getWidenedType(OrigScalarTy, NumElts);
+        }
       }
       if (I->hasOneUse()) {
         Instruction *Ext = I->user_back();
@@ -11334,10 +11343,18 @@ BoUpSLP::getEntryCost(const TreeEntry *E, ArrayRef<Value *> VectorizedVals,
           return Cost;
         }
       }
-      return TTI->getVectorInstrCost(Instruction::ExtractElement, SrcVecTy,
-                                     CostKind, *getExtractIndex(I));
+      if (DemandedElts.isZero())
+        DemandedElts = APInt::getZero(getNumElements(SrcVecTy));
+      DemandedElts.setBit(*getExtractIndex(I));
+      return InstructionCost(TTI::TCC_Free);
+    };
+    auto GetVectorCost = [&, &TTI = *TTI](InstructionCost CommonCost) {
+      return CommonCost - (DemandedElts.isZero()
+                               ? TTI::TCC_Free
+                               : TTI.getScalarizationOverhead(
+                                     SrcVecTy, DemandedElts, /*Insert=*/false,
+                                     /*Extract=*/true, CostKind));
     };
-    auto GetVectorCost = [](InstructionCost CommonCost) { return CommonCost; };
     return GetCostDiff(GetScalarCost, GetVectorCost);
   }
   case Instruction::InsertElement: {
@@ -11347,7 +11364,7 @@ BoUpSLP::getEntryCost(const TreeEntry *E, ArrayRef<Value *> VectorizedVals,
     unsigned const NumElts = SrcVecTy->getNumElements();
     unsigned const NumScalars = VL.size();
 
-    unsigned NumOfParts = TTI->getNumberOfParts(SrcVecTy);
+    unsigned NumOfParts = ::getNumberOfParts(*TTI, SrcVecTy);
 
     SmallVector<int> InsertMask(NumElts, PoisonMaskElem);
     unsigned OffsetBeg = *getElementIndex(VL.front());
@@ -12179,16 +12196,15 @@ bool BoUpSLP::isTreeNotExtendable() const {
   return Res;
 }
 
-InstructionCost BoUpSLP::getSpillCost() const {
+InstructionCost BoUpSLP::getSpillCost() {
   // Walk from the bottom of the tree to the top, tracking which values are
   // live. When we see a call instruction that is not part of our tree,
   // query TTI to see if there is a cost to keeping values live over it
   // (for example, if spills and fills are required).
-  unsigned BundleWidth = VectorizableTree.front()->Scalars.size();
   InstructionCost Cost = 0;
 
-  SmallPtrSet<Instruction *, 4> LiveValues;
-  Instruction *PrevInst = nullptr;
+  SmallPtrSet<const TreeEntry *, 4> LiveEntries;
+  const TreeEntry *Prev = nullptr;
 
   // The entries in VectorizableTree are not necessarily ordered by their
   // position in basic blocks. Collect them and order them by dominance so later
@@ -12196,95 +12212,100 @@ InstructionCost BoUpSLP::getSpillCost() const {
   // different basic blocks, we only scan to the beginning of the block, so
   // their order does not matter, as long as all instructions in a basic block
   // are grouped together. Using dominance ensures a deterministic order.
-  SmallVector<Instruction *, 16> OrderedScalars;
+  SmallVector<TreeEntry *, 16> OrderedEntries;
   for (const auto &TEPtr : VectorizableTree) {
-    if (TEPtr->State != TreeEntry::Vectorize)
+    if (TEPtr->isGather())
       continue;
-    Instruction *Inst = dyn_cast<Instruction>(TEPtr->Scalars[0]);
-    if (!Inst)
-      continue;
-    OrderedScalars.push_back(Inst);
-  }
-  llvm::sort(OrderedScalars, [&](Instruction *A, Instruction *B) {
-    auto *NodeA = DT->getNode(A->getParent());
-    auto *NodeB = DT->getNode(B->getParent());
+    OrderedEntries.push_back(TEPtr.get());
+  }
+  llvm::stable_sort(OrderedEntries, [&](const TreeEntry *TA,
+                                        const TreeEntry *TB) {
+    Instruction &A = getLastInstructionInBundle(TA);
+    Instruction &B = getLastInstructionInBundle(TB);
+    auto *NodeA = DT->getNode(A.getParent());
+    auto *NodeB = DT->getNode(B.getParent());
     assert(NodeA && "Should only process reachable instructions");
     assert(NodeB && "Should only process reachable instructions");
     assert((NodeA == NodeB) == (NodeA->getDFSNumIn() == NodeB->getDFSNumIn()) &&
            "Different nodes should have different DFS numbers");
     if (NodeA != NodeB)
       return NodeA->getDFSNumIn() > NodeB->getDFSNumIn();
-    return B->comesBefore(A);
+    return B.comesBefore(&A);
   });
 
-  for (Instruction *Inst : OrderedScalars) {
-    if (!PrevInst) {
-      PrevInst = Inst;
+  for (const TreeEntry *TE : OrderedEntries) {
+    if (!Prev) {
+      Prev = TE;
       continue;
     }
 
-    // Update LiveValues.
-    LiveValues.erase(PrevInst);
-    for (auto &J : PrevInst->operands()) {
-      if (isa<Instruction>(&*J) && getTreeEntry(&*J))
-        LiveValues.insert(cast<Instruction>(&*J));
+    LiveEntries.erase(Prev);
+    for (unsigned I : seq<unsigned>(Prev->getNumOperands())) {
+      const TreeEntry *Op = getVectorizedOperand(Prev, I);
+      if (!Op)
+        continue;
+      assert(!Op->isGather() && "Expected vectorized operand.");
+      LiveEntries.insert(Op);
     }
 
     LLVM_DEBUG({
-      dbgs() << "SLP: #LV: " << LiveValues.size();
-      for (auto *X : LiveValues)
-        dbgs() << " " << X->getName();
+      dbgs() << "SLP: #LV: " << LiveEntries.size();
+      for (auto *X : LiveEntries)
+        X->dump();
       dbgs() << ", Looking at ";
-      Inst->dump();
+      TE->dump();
     });
 
     // Now find the sequence of instructions between PrevInst and Inst.
     unsigned NumCalls = 0;
-    BasicBlock::reverse_iterator InstIt = ++Inst->getIterator().getReverse(),
-                                 PrevInstIt =
-                                     PrevInst->getIterator().getReverse();
+    const Instruction *PrevInst = &getLastInstructionInBundle(Prev);
+    BasicBlock::const_reverse_iterator
+        InstIt = ++getLastInstructionInBundle(TE).getIterator().getReverse(),
+        PrevInstIt = PrevInst->getIterator().getReverse();
     while (InstIt != PrevInstIt) {
       if (PrevInstIt == PrevInst->getParent()->rend()) {
-        PrevInstIt = Inst->getParent()->rbegin();
+        PrevInstIt = getLastInstructionInBundle(TE).getParent()->rbegin();
         continue;
       }
 
-      auto NoCallIntrinsic = [this](Instruction *I) {
-        if (auto *II = dyn_cast<IntrinsicInst>(I)) {
-          if (II->isAssumeLikeIntrinsic())
-            return true;
-          IntrinsicCostAttributes ICA(II->getIntrinsicID(), *II);
-          InstructionCost IntrCost =
-              TTI->getIntrinsicInstrCost(ICA, TTI::TCK_RecipThroughput);
-          InstructionCost CallCost =
-              TTI->getCallInstrCost(nullptr, II->getType(), ICA.getArgTypes(),
-                                    TTI::TCK_RecipThroughput);
-          if (IntrCost < CallCost)
-            return true;
-        }
-        return false;
+      auto NoCallIntrinsic = [this](const Instruction *I) {
+        const auto *II = dyn_cast<IntrinsicInst>(I);
+        if (!II)
+          return false;
+        if (II->isAssumeLikeIntrinsic())
+          return true;
+        IntrinsicCostAttributes ICA(II->getIntrinsicID(), *II);
+        InstructionCost IntrCost =
+            TTI->getIntrinsicInstrCost(ICA, TTI::TCK_RecipThroughput);
+        InstructionCost CallCost =
+            TTI->getCallInstrCost(nullptr, II->getType(), ICA.getArgTypes(),
+                                  TTI::TCK_RecipThroughput);
+        return IntrCost < CallCost;
       };
 
       // Debug information does not impact spill cost.
-      if (isa<CallBase>(&*PrevInstIt) && !NoCallIntrinsic(&*PrevInstIt) &&
-          &*PrevInstIt != PrevInst)
+      // Vectorized calls, represented as vector intrinsics, do not impact spill
+      // cost.
+      if (const auto *CB = dyn_cast<CallBase>(&*PrevInstIt);
+          CB && !NoCallIntrinsic(CB) && !isVectorized(CB))
         NumCalls++;
 
       ++PrevInstIt;
     }
 
     if (NumCalls) {
-      SmallVector<Type *, 4> V;
-      for (auto *II : LiveValues) {
-        auto *ScalarTy = II->getType();
-        if (auto *VectorTy = dyn_cast<FixedVectorType>(ScalarTy))
-          ScalarTy = VectorTy->getElementType();
-        V.push_back(getWidenedType(ScalarTy, BundleWidth));
+      SmallVector<Type *, 4> EntriesTypes;
+      for (const TreeEntry *TE : LiveEntries) {
+        auto *ScalarTy = TE->getMainOp()->getType();
+        auto It = MinBWs.find(TE);
+        if (It != MinBWs.end())
+          ScalarTy = IntegerType::get(ScalarTy->getContext(), It->second.first);
+        EntriesTypes.push_back(getWidenedType(ScalarTy, TE->getVectorFactor()));
       }
-      Cost += NumCalls * TTI->getCostOfKeepingLiveOverCall(V);
+      Cost += NumCalls * TTI->getCostOfKeepingLiveOverCall(EntriesTypes);
     }
 
-    PrevInst = Inst;
+    Prev = TE;
   }
 
   return Cost;
@@ -12473,9 +12494,9 @@ InstructionCost BoUpSLP::getTreeCost(ArrayRef<Value *> VectorizedVals) {
       continue;
     }
     if (TE.isGather() && TE.hasState()) {
-      if (const TreeEntry *E = getTreeEntry(TE.getMainOp());
-          E && E->getVectorFactor() == TE.getVectorFactor() &&
-          E->isSame(TE.Scalars)) {
+      if (const TreeEntry *E =
+              getSameValuesTreeEntry(TE.getMainOp(), TE.Scalars);
+          E && E->getVectorFactor() == TE.getVectorFactor()) {
         // Some gather nodes might be absolutely the same as some vectorizable
         // nodes after reordering, need to handle it.
         LLVM_DEBUG(dbgs() << "SLP: Adding cost 0 for bundle "
@@ -12547,7 +12568,7 @@ InstructionCost BoUpSLP::getTreeCost(ArrayRef<Value *> VectorizedVals) {
           continue;
         std::optional<unsigned> InsertIdx = getElementIndex(VU);
         if (InsertIdx) {
-          const TreeEntry *ScalarTE = getTreeEntry(EU.Scalar);
+          const TreeEntry *ScalarTE = &EU.E;
           auto *It = find_if(
               ShuffledInserts,
               [this, VU](const ShuffledInsertData<const TreeEntry *> &Data) {
@@ -12556,7 +12577,7 @@ InstructionCost BoUpSLP::getTreeCost(ArrayRef<Value *> VectorizedVals) {
                 return areTwoInsertFromSameBuildVector(
                     VU, VecInsert, [this](InsertElementInst *II) -> Value * {
                       Value *Op0 = II->getOperand(0);
-                      if (getTreeEntry(II) && !getTreeEntry(Op0))
+                      if (isVectorized(II) && !isVectorized(Op0))
                         return nullptr;
                       return Op0;
                     });
@@ -12614,7 +12635,7 @@ InstructionCost BoUpSLP::getTreeCost(ArrayRef<Value *> VectorizedVals) {
     // for the extract and the added cost of the sign extend if needed.
     InstructionCost ExtraCost = TTI::TCC_Free;
     auto *VecTy = getWidenedType(EU.Scalar->getType(), BundleWidth);
-    const TreeEntry *Entry = getTreeEntry(EU.Scalar);
+    const TreeEntry *Entry = &EU.E;
     auto It = MinBWs.find(Entry);
     if (It != MinBWs.end()) {
       auto *MinTy = IntegerType::get(F->getContext(), It->second.first);
@@ -12657,7 +12678,7 @@ InstructionCost BoUpSLP::getTreeCost(ArrayRef<Value *> VectorizedVals) {
       auto *Inst = cast<Instruction>(EU.Scalar);
       InstructionCost ScalarCost = TTI->getInstructionCost(Inst, CostKind);
       auto OperandIsScalar = [&](Value *V) {
-        if (!getTreeEntry(V)) {
+        if (!isVectorized(V)) {
           // Some extractelements might be not vectorized, but
           // transformed into shuffle and removed from the function,
           // consider it here.
@@ -12673,7 +12694,7 @@ InstructionCost BoUpSLP::getTreeCost(ArrayRef<Value *> VectorizedVals) {
         if (auto *Op = dyn_cast<Instruction>(CI->getOperand(0));
             Op && all_of(Op->operands(), OperandIsScalar)) {
           InstructionCost OpCost =
-              (getTreeEntry(Op) && !ValueToExtUses->contains(Op))
+              (isVectorized(Op) && !ValueToExtUses->contains(Op))
                   ? TTI->getInstructionCost(Op, CostKind)
                   : 0;
           if (ScalarCost + OpCost <= ExtraCost) {
@@ -12700,7 +12721,7 @@ InstructionCost BoUpSLP::getTreeCost(ArrayRef<Value *> VectorizedVals) {
                                   cast<Instruction>(
                                       VectorizableTree.front()->getMainOp())
                                       ->getParent()) &&
-                             !getTreeEntry(U);
+                             !isVectorized(U);
                     }) &&
             count_if(Entry->Scalars, [&](Value *V) {
               return ValueToExtUses->contains(V);
@@ -12762,8 +12783,9 @@ InstructionCost BoUpSLP::getTreeCost(ArrayRef<Value *> VectorizedVals) {
   // instead of extractelement.
   for (Value *V : ScalarOpsFromCasts) {
     ExternalUsesAsOriginalScalar.insert(V);
-    if (const TreeEntry *E = getTreeEntry(V)) {
-      ExternalUses.emplace_back(V, nullptr, E->findLaneForValue(V));
+    if (ArrayRef<TreeEntry *> TEs = getTreeEntries(V); !TEs.empty()) {
+      ExternalUses.emplace_back(V, nullptr, *TEs.front(),
+                                TEs.front()->findLaneForValue(V));
     }
   }
   // Add reduced value cost, if resized.
@@ -13183,21 +13205,18 @@ BoUpSLP::isGatherShuffledSingleRegisterEntry(
         continue;
       VToTEs.insert(TEPtr);
     }
-    if (const TreeEntry *VTE = getTreeEntry(V)) {
-      if (ForOrder && VTE->Idx < GatheredLoadsEntriesFirst.value_or(0)) {
-        if (VTE->State != TreeEntry::Vectorize) {
-          auto It = MultiNodeScalars.find(V);
-          if (It == MultiNodeScalars.end())
-            continue;
-          VTE = *It->getSecond().begin();
-          // Iterate through all vectorized nodes.
-          auto *MIt = find_if(It->getSecond(), [](const TreeEntry *MTE) {
-            return MTE->State == TreeEntry::Vectorize;
-          });
-          if (MIt == It->getSecond().end())
-            continue;
-          VTE = *MIt;
-        }
+    if (ArrayRef<TreeEntry *> VTEs = getTreeEntries(V); !VTEs.empty()) {
+      const TreeEntry *VTE = VTEs.front();
+      if (ForOrder && VTE->Idx < GatheredLoadsEntriesFirst.value_or(0) &&
+          VTEs.size() > 1 && VTE->State != TreeEntry::Vectorize) {
+        VTEs = VTEs.drop_front();
+        // Iterate through all vectorized nodes.
+        const auto *MIt = find_if(VTEs, [](const TreeEntry *MTE) {
+          return MTE->State == TreeEntry::Vectorize;
+        });
+        if (MIt == VTEs.end())
+          continue;
+        VTE = *MIt;
       }
       if (none_of(TE->CombinedEntriesWithIndices,
                   [&](const auto &P) { return P.first == VTE->Idx; })) {
@@ -13361,7 +13380,7 @@ BoUpSLP::isGatherShuffledSingleRegisterEntry(
   // by extractelements processing) or may form vector node in future.
   auto MightBeIgnored = [=](Value *V) {
     auto *I = dyn_cast<Instruction>(V);
-    return I && !IsSplatOrUndefs && !ScalarToTreeEntry.count(I) &&
+    return I && !IsSplatOrUndefs && !isVectorized(I) &&
            !isVectorLikeInstWithConstOps(I) &&
            !areAllUsersVectorized(I, UserIgnoreList) && isSimple(I);
   };
@@ -13678,6 +13697,7 @@ InstructionCost BoUpSLP::getGatherCost(ArrayRef<Value *> VL, bool ForPoisonSrc,
   // Check if the same elements are inserted several times and count them as
   // shuffle candidates.
   APInt ShuffledElements = APInt::getZero(VL.size());
+  APInt DemandedElements = APInt::getZero(VL.size());
   DenseMap<Value *, unsigned> UniqueElements;
   constexpr TTI::TargetCostKind CostKind = TTI::TCK_RecipThroughput;
   InstructionCost Cost;
@@ -13688,9 +13708,7 @@ InstructionCost BoUpSLP::getGatherCost(ArrayRef<Value *> VL, bool ForPoisonSrc,
       V = nullptr;
     }
     if (!ForPoisonSrc)
-      Cost +=
-          TTI->getVectorInstrCost(Instruction::InsertElement, VecTy, CostKind,
-                                  I, Constant::getNullValue(VecTy), V);
+      DemandedElements.setBit(I);
   };
   SmallVector<int> ShuffleMask(VL.size(), PoisonMaskElem);
   for (unsigned I = 0, E = VL.size(); I < E; ++I) {
@@ -13713,6 +13731,10 @@ InstructionCost BoUpSLP::getGatherCost(ArrayRef<Value *> VL, bool ForPoisonSrc,
     ShuffledElements.setBit(I);
     ShuffleMask[I] = Res.first->second;
   }
+  if (!DemandedElements.isZero())
+    Cost +=
+        TTI->getScalarizationOverhead(VecTy, DemandedElements, /*Insert=*/true,
+                                      /*Extract=*/false, CostKind, VL);
   if (ForPoisonSrc) {
     if (isa<FixedVectorType>(ScalarTy)) {
       assert(SLPReVec && "Only supported by REVEC.");
@@ -13947,7 +13969,7 @@ Value *BoUpSLP::gather(
   for (int I = 0, E = VL.size(); I < E; ++I) {
     if (auto *Inst = dyn_cast<Instruction>(VL[I]))
       if ((CheckPredecessor(Inst->getParent(), Builder.GetInsertBlock()) ||
-           getTreeEntry(Inst) ||
+           isVectorized(Inst) ||
            (L && (!Root || L->isLoopInvariant(Root)) && L->contains(Inst))) &&
           PostponedIndices.insert(I).second)
         PostponedInsts.emplace_back(Inst, I);
@@ -13964,7 +13986,7 @@ Value *BoUpSLP::gather(
           isa_and_nonnull<SExtInst, ZExtInst>(CI)) {
         Value *Op = CI->getOperand(0);
         if (auto *IOp = dyn_cast<Instruction>(Op);
-            !IOp || !(isDeleted(IOp) || getTreeEntry(IOp)))
+            !IOp || !(isDeleted(IOp) || isVectorized(IOp)))
           V = Op;
       }
       Scalar = Builder.CreateIntCast(
@@ -13990,7 +14012,7 @@ Value *BoUpSLP::gather(
     CSEBlocks.insert(InsElt->getParent());
     // Add to our 'need-to-extract' list.
     if (isa<Instruction>(V)) {
-      if (TreeEntry *Entry = getTreeEntry(V)) {
+      if (ArrayRef<TreeEntry *> Entries = getTreeEntries(V); !Entries.empty()) {
         // Find which lane we need to extract.
         User *UserOp = nullptr;
         if (Scalar != V) {
@@ -14000,8 +14022,8 @@ Value *BoUpSLP::gather(
           UserOp = InsElt;
         }
         if (UserOp) {
-          unsigned FoundLane = Entry->findLaneForValue(V);
-          ExternalUses.emplace_back(V, UserOp, FoundLane);
+          unsigned FoundLane = Entries.front()->findLaneForValue(V);
+          ExternalUses.emplace_back(V, UserOp, *Entries.front(), FoundLane);
         }
       }
     }
@@ -14196,8 +14218,8 @@ class BoUpSLP::ShuffleInstructionBuilder final : public BaseShuffleAnalysis {
     assert(V1 && "Expected at least one vector value.");
     ShuffleIRBuilder ShuffleBuilder(Builder, R.GatherShuffleExtractSeq,
                                     R.CSEBlocks, *R.DL);
-    return BaseShuffleAnalysis::createShuffle<Value *>(V1, V2, Mask,
-                                                       ShuffleBuilder);
+    return BaseShuffleAnalysis::createShuffle<Value *>(
+        V1, V2, Mask, ShuffleBuilder, ScalarTy);
   }
 
   /// Cast value \p V to the vector type with the same number of elements, but
@@ -14236,8 +14258,8 @@ class BoUpSLP::ShuffleInstructionBuilder final : public BaseShuffleAnalysis {
         continue;
       auto *EI = cast<ExtractElementInst>(VL[I]);
       VecBase = EI->getVectorOperand();
-      if (const TreeEntry *TE = R.getTreeEntry(VecBase))
-        VecBase = TE->VectorizedValue;
+      if (ArrayRef<TreeEntry *> TEs = R.getTreeEntries(VecBase); !TEs.empty())
+        VecBase = TEs.front()->VectorizedValue;
       assert(VecBase && "Expected vectorized value.");
       UniqueBases.insert(VecBase);
       // If the only one use is vectorized - can delete the extractelement
@@ -14245,18 +14267,20 @@ class BoUpSLP::ShuffleInstructionBuilder final : public BaseShuffleAnalysis {
       if (!EI->hasOneUse() || R.ExternalUsesAsOriginalScalar.contains(EI) ||
           (NumParts != 1 && count(VL, EI) > 1) ||
           any_of(EI->users(), [&](User *U) {
-            const TreeEntry *UTE = R.getTreeEntry(U);
-            return !UTE || R.MultiNodeScalars.contains(U) ||
+            ArrayRef<TreeEntry *> UTEs = R.getTreeEntries(U);
+            return UTEs.empty() || UTEs.size() > 1 ||
                    (isa<GetElementPtrInst>(U) &&
                     !R.areAllUsersVectorized(cast<Instruction>(U))) ||
-                   count_if(R.VectorizableTree,
-                            [&](const std::unique_ptr<TreeEntry> &TE) {
-                              return any_of(TE->UserTreeIndices,
-                                            [&](const EdgeInfo &Edge) {
-                                              return Edge.UserTE == UTE;
-                                            }) &&
-                                     is_contained(VL, EI);
-                            }) != 1;
+                   (!UTEs.empty() &&
+                    count_if(R.VectorizableTree,
+                             [&](const std::unique_ptr<TreeEntry> &TE) {
+                               return any_of(TE->UserTreeIndices,
+                                             [&](const EdgeInfo &Edge) {
+                                               return Edge.UserTE ==
+                                                      UTEs.front();
+                                             }) &&
+                                      is_contained(VL, EI);
+                             }) != 1);
           }))
         continue;
       R.eraseInstruction(EI);
@@ -14291,8 +14315,9 @@ class BoUpSLP::ShuffleInstructionBuilder final : public BaseShuffleAnalysis {
               return S;
             Value *VecOp =
                 cast<ExtractElementInst>(std::get<0>(D))->getVectorOperand();
-            if (const TreeEntry *TE = R.getTreeEntry(VecOp))
-              VecOp = TE->VectorizedValue;
+            if (ArrayRef<TreeEntry *> TEs = R.getTreeEntries(VecOp);
+                !TEs.empty())
+              VecOp = TEs.front()->VectorizedValue;
             assert(VecOp && "Expected vectorized value.");
             const unsigned Size =
                 cast<FixedVectorType>(VecOp->getType())->getNumElements();
@@ -14302,8 +14327,8 @@ class BoUpSLP::ShuffleInstructionBuilder final : public BaseShuffleAnalysis {
         if (I == PoisonMaskElem)
           continue;
         Value *VecOp = cast<ExtractElementInst>(V)->getVectorOperand();
-        if (const TreeEntry *TE = R.getTreeEntry(VecOp))
-          VecOp = TE->VectorizedValue;
+        if (ArrayRef<TreeEntry *> TEs = R.getTreeEntries(VecOp); !TEs.empty())
+          VecOp = TEs.front()->VectorizedValue;
         assert(VecOp && "Expected vectorized value.");
         VecOp = castToScalarTyElem(VecOp);
         Bases[I / VF] = VecOp;
@@ -14516,14 +14541,6 @@ class BoUpSLP::ShuffleInstructionBuilder final : public BaseShuffleAnalysis {
            ArrayRef<int> SubVectorsMask, unsigned VF = 0,
            function_ref<void(Value *&, SmallVectorImpl<int> &)> Action = {}) {
     IsFinalized = true;
-    unsigned ScalarTyNumElements = getNumElements(ScalarTy);
-    SmallVector<int> NewExtMask(ExtMask);
-    if (ScalarTyNumElements != 1) {
-      assert(SLPReVec && "FixedVectorType is not expected.");
-      transformScalarShuffleIndiciesToVector(ScalarTyNumElements, CommonMask);
-      transformScalarShuffleIndiciesToVector(ScalarTyNumElements, NewExtMask);
-      ExtMask = NewExtMask;
-    }
     if (Action) {
       Value *Vec = InVectors.front();
       if (InVectors.size() == 2) {
@@ -14564,17 +14581,15 @@ class BoUpSLP::ShuffleInstructionBuilder final : public BaseShuffleAnalysis {
                                      return !isKnownNonNegative(
                                          V, SimplifyQuery(*R.DL));
                                    }));
-          unsigned InsertionIndex = Idx * ScalarTyNumElements;
+          unsigned InsertionIndex = Idx * getNumElements(ScalarTy);
           Vec = createInsertVector(
               Builder, Vec, V, InsertionIndex,
               std::bind(&ShuffleInstructionBuilder::createShuffle, this, _1, _2,
                         _3));
           if (!CommonMask.empty()) {
-            std::iota(
-                std::next(CommonMask.begin(), InsertionIndex),
-                std::next(CommonMask.begin(),
-                          (Idx + E->getVectorFactor()) * ScalarTyNumElements),
-                InsertionIndex);
+            std::iota(std::next(CommonMask.begin(), Idx),
+                      std::next(CommonMask.begin(), Idx + E->getVectorFactor()),
+                      Idx);
           }
         }
         return Vec;
@@ -14639,29 +14654,20 @@ BoUpSLP::TreeEntry *BoUpSLP::getMatchedVectorizedOperand(const TreeEntry *E,
   if (!S)
     return nullptr;
   auto CheckSameVE = [&](const TreeEntry *VE) {
-    return VE->isSame(VL) &&
-           (any_of(VE->UserTreeIndices,
-                   [E, NodeIdx](const EdgeInfo &EI) {
-                     return EI.UserTE == E && EI.EdgeIdx == NodeIdx;
-                   }) ||
-            any_of(VectorizableTree,
-                   [E, NodeIdx, VE](const std::unique_ptr<TreeEntry> &TE) {
-                     return TE->isOperandGatherNode(
-                                {const_cast<TreeEntry *>(E), NodeIdx}) &&
-                            VE->isSame(TE->Scalars);
-                   }));
+    return any_of(VE->UserTreeIndices,
+                  [E, NodeIdx](const EdgeInfo &EI) {
+                    return EI.UserTE == E && EI.EdgeIdx == NodeIdx;
+                  }) ||
+           any_of(VectorizableTree,
+                  [E, NodeIdx, VE](const std::unique_ptr<TreeEntry> &TE) {
+                    return TE->isOperandGatherNode(
+                               {const_cast<TreeEntry *>(E), NodeIdx}) &&
+                           VE->isSame(TE->Scalars);
+                  });
   };
-  TreeEntry *VE = getTreeEntry(S.getMainOp());
+  TreeEntry *VE = getSameValuesTreeEntry(S.getMainOp(), VL);
   if (VE && CheckSameVE(VE))
     return VE;
-  auto It = MultiNodeScalars.find(S.getMainOp());
-  if (It != MultiNodeScalars.end()) {
-    auto *I = find_if(It->getSecond(), [&](const TreeEntry *TE) {
-      return TE != VE && CheckSameVE(TE);
-    });
-    if (I != It->getSecond().end())
-      return *I;
-  }
   return nullptr;
 }
 
@@ -14868,12 +14874,7 @@ ResTy BoUpSLP::processBuildVector(const TreeEntry *E, Type *ScalarTy,
   SmallVector<SmallVector<const TreeEntry *>> Entries;
   Type *OrigScalarTy = GatheredScalars.front()->getType();
   auto *VecTy = getWidenedType(ScalarTy, GatheredScalars.size());
-  unsigned NumParts = TTI->getNumberOfParts(VecTy);
-  if (NumParts == 0 || NumParts >= GatheredScalars.size() ||
-      VecTy->getNumElements() % NumParts != 0 ||
-      !hasFullVectorsOrPowerOf2(*TTI, VecTy->getElementType(),
-                                VecTy->getNumElements() / NumParts))
-    NumParts = 1;
+  unsigned NumParts = ::getNumberOfParts(*TTI, VecTy, GatheredScalars.size());
   if (!all_of(GatheredScalars, IsaPred<UndefValue>)) {
     // Check for gathered extracts.
     bool Resized = false;
@@ -14884,9 +14885,10 @@ ResTy BoUpSLP::processBuildVector(const TreeEntry *E, Type *ScalarTy,
       for (auto [Idx, I] : enumerate(ExtractMask)) {
         if (I == PoisonMaskElem)
           continue;
-        if (const auto *TE = getTreeEntry(
-                cast<ExtractElementInst>(StoredGS[Idx])->getVectorOperand()))
-          ExtractEntries.push_back(TE);
+        if (ArrayRef<TreeEntry *> TEs = getTreeEntries(
+                cast<ExtractElementInst>(StoredGS[Idx])->getVectorOperand());
+            !TEs.empty())
+          ExtractEntries.append(TEs.begin(), TEs.end());
       }
       if (std::optional<ResTy> Delayed =
               ShuffleBuilder.needToDelay(E, ExtractEntries)) {
@@ -14905,6 +14907,8 @@ ResTy BoUpSLP::processBuildVector(const TreeEntry *E, Type *ScalarTy,
             Resized = true;
             GatheredScalars.append(VF - GatheredScalars.size(),
                                    PoisonValue::get(OrigScalarTy));
+            NumParts =
+                ::getNumberOfParts(*TTI, getWidenedType(OrigScalarTy, VF), VF);
           }
       }
     }
@@ -14915,10 +14919,10 @@ ResTy BoUpSLP::processBuildVector(const TreeEntry *E, Type *ScalarTy,
           any_of(E->Scalars, IsaPred<LoadInst>)) &&
          any_of(E->Scalars,
                 [this](Value *V) {
-                  return isa<LoadInst>(V) && getTreeEntry(V);
+                  return isa<LoadInst>(V) && isVectorized(V);
                 })) ||
         (E->hasState() && E->isAltShuffle()) ||
-        all_of(E->Scalars, [this](Value *V) { return getTreeEntry(V); }) ||
+        all_of(E->Scalars, [this](Value *V) { return isVectorized(V); }) ||
         isSplat(E->Scalars) ||
         (E->Scalars != GatheredScalars && GatheredScalars.size() <= 2)) {
       GatherShuffles =
@@ -15033,7 +15037,7 @@ ResTy BoUpSLP::processBuildVector(const TreeEntry *E, Type *ScalarTy,
       // non-poisonous, or by freezing the incoming scalar value first.
       auto *It = find_if(Scalars, [this, E](Value *V) {
         return !isa<UndefValue>(V) &&
-               (getTreeEntry(V) || isGuaranteedNotToBePoison(V, AC) ||
+               (isVectorized(V) || isGuaranteedNotToBePoison(V, AC) ||
                 (E->UserTreeIndices.size() == 1 &&
                  any_of(V->uses(), [E](const Use &U) {
                    // Check if the value already used in the same operation in
@@ -15091,9 +15095,9 @@ ResTy BoUpSLP::processBuildVector(const TreeEntry *E, Type *ScalarTy,
             continue;
           auto *EI = cast<ExtractElementInst>(StoredGS[I]);
           Value *VecOp = EI->getVectorOperand();
-          if (const auto *TE = getTreeEntry(VecOp))
-            if (TE->VectorizedValue)
-              VecOp = TE->VectorizedValue;
+          if (ArrayRef<TreeEntry *> TEs = getTreeEntries(VecOp);
+              !TEs.empty() && TEs.front()->VectorizedValue)
+            VecOp = TEs.front()->VectorizedValue;
           if (!Vec1) {
             Vec1 = VecOp;
           } else if (Vec1 != VecOp) {
@@ -15123,7 +15127,9 @@ ResTy BoUpSLP::processBuildVector(const TreeEntry *E, Type *ScalarTy,
       }
     }
     if (!GatherShuffles.empty()) {
-      unsigned SliceSize = getPartNumElems(E->Scalars.size(), NumParts);
+      unsigned SliceSize =
+          getPartNumElems(E->Scalars.size(),
+                          ::getNumberOfParts(*TTI, VecTy, E->Scalars.size()));
       SmallVector<int> VecMask(Mask.size(), PoisonMaskElem);
       for (const auto [I, TEs] : enumerate(Entries)) {
         if (TEs.empty()) {
@@ -15421,8 +15427,8 @@ Value *BoUpSLP::vectorizeTree(TreeEntry *E, bool PostponedPHIs) {
 
     case Instruction::ExtractElement: {
       Value *V = E->getSingleOperand(0);
-      if (const TreeEntry *TE = getTreeEntry(V))
-        V = TE->VectorizedValue;
+      if (ArrayRef<TreeEntry *> TEs = getTreeEntries(V); !TEs.empty())
+        V = TEs.front()->VectorizedValue;
       setInsertPointAfterBundle(E);
       V = FinalShuffle(V, E);
       E->VectorizedValue = V;
@@ -16352,13 +16358,13 @@ BoUpSLP::vectorizeTree(const ExtraValueToDebugLocsMap &ExternallyUsedValues,
   DenseMap<Value *, SmallVector<TreeEntry *>> PostponedValues;
   for (const TreeEntry *E : PostponedNodes) {
     auto *TE = const_cast<TreeEntry *>(E);
-    if (auto *VecTE = getTreeEntry(TE->Scalars.front()))
-      if (VecTE->isSame(TE->UserTreeIndices.front().UserTE->getOperand(
-              TE->UserTreeIndices.front().EdgeIdx)) &&
-          VecTE->isSame(TE->Scalars))
-        // Found gather node which is absolutely the same as one of the
-        // vectorized nodes. It may happen after reordering.
-        continue;
+    if (auto *VecTE = getSameValuesTreeEntry(
+            TE->Scalars.front(), TE->UserTreeIndices.front().UserTE->getOperand(
+                                     TE->UserTreeIndices.front().EdgeIdx));
+        VecTE && VecTE->isSame(TE->Scalars))
+      // Found gather node which is absolutely the same as one of the
+      // vectorized nodes. It may happen after reordering.
+      continue;
     auto *PrevVec = cast<Instruction>(TE->VectorizedValue);
     TE->VectorizedValue = nullptr;
     auto *UserI =
@@ -16400,14 +16406,8 @@ BoUpSLP::vectorizeTree(const ExtraValueToDebugLocsMap &ExternallyUsedValues,
              "Expected integer vector types only.");
       std::optional<bool> IsSigned;
       for (Value *V : TE->Scalars) {
-        if (const TreeEntry *BaseTE = getTreeEntry(V)) {
-          auto It = MinBWs.find(BaseTE);
-          if (It != MinBWs.end()) {
-            IsSigned = IsSigned.value_or(false) || It->second.second;
-            if (*IsSigned)
-              break;
-          }
-          for (const TreeEntry *MNTE : MultiNodeScalars.lookup(V)) {
+        if (isVectorized(V)) {
+          for (const TreeEntry *MNTE : getTreeEntries(V)) {
             auto It = MinBWs.find(MNTE);
             if (It != MinBWs.end()) {
               IsSigned = IsSigned.value_or(false) || It->second.second;
@@ -16483,7 +16483,7 @@ BoUpSLP::vectorizeTree(const ExtraValueToDebugLocsMap &ExternallyUsedValues,
     // has multiple uses of the same value.
     if (User && !is_contained(Scalar->users(), User))
       continue;
-    TreeEntry *E = getTreeEntry(Scalar);
+    const TreeEntry *E = &ExternalUse.E;
     assert(E && "Invalid scalar");
     assert(!E->isGather() && "Extracting from a gather list");
     // Non-instruction pointers are not deleted, just skip them.
@@ -16541,8 +16541,8 @@ BoUpSLP::vectorizeTree(const ExtraValueToDebugLocsMap &ExternallyUsedValues,
                      ES && isa<Instruction>(Vec)) {
             Value *V = ES->getVectorOperand();
             auto *IVec = cast<Instruction>(Vec);
-            if (const TreeEntry *ETE = getTreeEntry(V))
-              V = ETE->VectorizedValue;
+            if (ArrayRef<TreeEntry *> ETEs = getTreeEntries(V); !ETEs.empty())
+              V = ETEs.front()->VectorizedValue;
             if (auto *IV = dyn_cast<Instruction>(V);
                 !IV || IV == Vec || IV->getParent() != IVec->getParent() ||
                 IV->comesBefore(IVec))
@@ -16595,27 +16595,31 @@ BoUpSLP::vectorizeTree(const ExtraValueToDebugLocsMap &ExternallyUsedValues,
     if (!User) {
       if (!ScalarsWithNullptrUser.insert(Scalar).second)
         continue;
-      assert((ExternallyUsedValues.count(Scalar) ||
-              Scalar->hasNUsesOrMore(UsesLimit) ||
-              ExternalUsesAsOriginalScalar.contains(Scalar) ||
-              any_of(Scalar->users(),
-                     [&](llvm::User *U) {
-                       if (ExternalUsesAsOriginalScalar.contains(U))
-                         return true;
-                       TreeEntry *UseEntry = getTreeEntry(U);
-                       return UseEntry &&
-                              (UseEntry->State == TreeEntry::Vectorize ||
-                               UseEntry->State ==
-                                   TreeEntry::StridedVectorize) &&
-                              (E->State == TreeEntry::Vectorize ||
-                               E->State == TreeEntry::StridedVectorize) &&
-                              doesInTreeUserNeedToExtract(
-                                  Scalar, getRootEntryInstruction(*UseEntry),
-                                  TLI, TTI);
-                     })) &&
-             "Scalar with nullptr User must be registered in "
-             "ExternallyUsedValues map or remain as scalar in vectorized "
-             "instructions");
+      assert(
+          (ExternallyUsedValues.count(Scalar) ||
+           Scalar->hasNUsesOrMore(UsesLimit) ||
+           ExternalUsesAsOriginalScalar.contains(Scalar) ||
+           any_of(
+               Scalar->users(),
+               [&, TTI = TTI](llvm::User *U) {
+                 if (ExternalUsesAsOriginalScalar.contains(U))
+                   return true;
+                 ArrayRef<TreeEntry *> UseEntries = getTreeEntries(U);
+                 return !UseEntries.empty() &&
+                        (E->State == TreeEntry::Vectorize ||
+                         E->State == TreeEntry::StridedVectorize) &&
+                        any_of(UseEntries, [&, TTI = TTI](TreeEntry *UseEntry) {
+                          return (UseEntry->State == TreeEntry::Vectorize ||
+                                  UseEntry->State ==
+                                      TreeEntry::StridedVectorize) &&
+                                 doesInTreeUserNeedToExtract(
+                                     Scalar, getRootEntryInstruction(*UseEntry),
+                                     TLI, TTI);
+                        });
+               })) &&
+          "Scalar with nullptr User must be registered in "
+          "ExternallyUsedValues map or remain as scalar in vectorized "
+          "instructions");
       if (auto *VecI = dyn_cast<Instruction>(Vec)) {
         if (auto *PHI = dyn_cast<PHINode>(VecI)) {
           if (PHI->getParent()->isLandingPad())
@@ -16878,7 +16882,7 @@ BoUpSLP::vectorizeTree(const ExtraValueToDebugLocsMap &ExternallyUsedValues,
           LLVM_DEBUG(dbgs() << "SLP: \tvalidating user:" << *U << ".\n");
 
           // It is legal to delete users in the ignorelist.
-          assert((getTreeEntry(U) ||
+          assert((isVectorized(U) ||
                   (UserIgnoreList && UserIgnoreList->contains(U)) ||
                   (isa_and_nonnull<Instruction>(U) &&
                    isDeleted(cast<Instruction>(U)))) &&
@@ -16900,7 +16904,7 @@ BoUpSLP::vectorizeTree(const ExtraValueToDebugLocsMap &ExternallyUsedValues,
   // Clear up reduction references, if any.
   if (UserIgnoreList) {
     for (Instruction *I : RemovedInsts) {
-      const TreeEntry *IE = getTreeEntry(I);
+      const TreeEntry *IE = getTreeEntries(I).front();
       if (IE->Idx != 0 &&
           !(VectorizableTree.front()->isGather() &&
             !IE->UserTreeIndices.empty() &&
@@ -17049,10 +17053,10 @@ void BoUpSLP::optimizeGatherSequence() {
     // Check if the last undefs actually change the final number of used vector
     // registers.
     return SM1.size() - LastUndefsCnt > 1 &&
-           TTI->getNumberOfParts(SI1->getType()) ==
-               TTI->getNumberOfParts(
-                   getWidenedType(SI1->getType()->getElementType(),
-                                  SM1.size() - LastUndefsCnt));
+           ::getNumberOfParts(*TTI, SI1->getType()) ==
+               ::getNumberOfParts(
+                   *TTI, getWidenedType(SI1->getType()->getElementType(),
+                                        SM1.size() - LastUndefsCnt));
   };
   // Perform O(N^2) search over the gather/shuffle sequences and merge identical
   // instructions. TODO: We can further optimize this scan if we split the
@@ -17615,10 +17619,11 @@ void BoUpSLP::scheduleBlock(BlockScheduling *BS) {
   for (auto *I = BS->ScheduleStart; I != BS->ScheduleEnd;
        I = I->getNextNode()) {
     if (ScheduleData *SD = BS->getScheduleData(I)) {
-      [[maybe_unused]] TreeEntry *SDTE = getTreeEntry(SD->Inst);
+      [[maybe_unused]] ArrayRef<TreeEntry *> SDTEs = getTreeEntries(SD->Inst);
       assert((isVectorLikeInstWithConstOps(SD->Inst) ||
               SD->isPartOfBundle() ==
-                  (SDTE && !doesNotNeedToSchedule(SDTE->Scalars))) &&
+                  (!SDTEs.empty() &&
+                   !doesNotNeedToSchedule(SDTEs.front()->Scalars))) &&
              "scheduler and vectorizer bundle mismatch");
       SD->FirstInBundle->SchedulingPriority = Idx++;
 
@@ -17780,7 +17785,7 @@ bool BoUpSLP::collectValuesToDemote(
   auto IsPotentiallyTruncated = [&](Value *V, unsigned &BitWidth) -> bool {
     if (isa<PoisonValue>(V))
       return true;
-    if (MultiNodeScalars.contains(V))
+    if (getTreeEntries(V).size() > 1)
       return false;
     // For lat shuffle of sext/zext with many uses need to check the extra bit
     // for unsigned values, otherwise may have incorrect casting for reused
@@ -17829,9 +17834,12 @@ bool BoUpSLP::collectValuesToDemote(
       const unsigned VF = E.Scalars.size();
       Type *OrigScalarTy = E.Scalars.front()->getType();
       if (UniqueBases.size() <= 2 ||
-          TTI->getNumberOfParts(getWidenedType(OrigScalarTy, VF)) ==
-              TTI->getNumberOfParts(getWidenedType(
-                  IntegerType::get(OrigScalarTy->getContext(), BitWidth), VF)))
+          ::getNumberOfParts(*TTI, getWidenedType(OrigScalarTy, VF)) ==
+              ::getNumberOfParts(
+                  *TTI,
+                  getWidenedType(
+                      IntegerType::get(OrigScalarTy->getContext(), BitWidth),
+                      VF)))
         ToDemote.push_back(E.Idx);
     }
     return Res;
@@ -17839,14 +17847,14 @@ bool BoUpSLP::collectValuesToDemote(
   if (E.isGather() || !Visited.insert(&E).second ||
       any_of(E.Scalars, [&](Value *V) {
         return !isa<PoisonValue>(V) && all_of(V->users(), [&](User *U) {
-          return isa<InsertElementInst>(U) && !getTreeEntry(U);
+          return isa<InsertElementInst>(U) && !isVectorized(U);
         });
       }))
     return FinalAnalysis();
 
   if (any_of(E.Scalars, [&](Value *V) {
         return !all_of(V->users(), [=](User *U) {
-          return getTreeEntry(U) ||
+          return isVectorized(U) ||
                  (E.Idx == 0 && UserIgnoreList &&
                   UserIgnoreList->contains(U)) ||
                  (!isa<CmpInst>(U) && U->getType()->isSized() &&
@@ -18197,9 +18205,9 @@ void BoUpSLP::computeMinimumValueSizes() {
           return V->hasOneUse() || isa<Constant>(V) ||
                  (!V->hasNUsesOrMore(UsesLimit) &&
                   none_of(V->users(), [&](User *U) {
-                    const TreeEntry *TE = getTreeEntry(U);
+                    ArrayRef<TreeEntry *> TEs = getTreeEntries(U);
                     const TreeEntry *UserTE = E.UserTreeIndices.back().UserTE;
-                    if (TE == UserTE || !TE)
+                    if (TEs.empty() || is_contained(TEs, UserTE))
                       return false;
                     if (!isa<CastInst, BinaryOperator, FreezeInst, PHINode,
                              SelectInst>(U) ||
@@ -18208,8 +18216,11 @@ void BoUpSLP::computeMinimumValueSizes() {
                       return true;
                     unsigned UserTESz = DL->getTypeSizeInBits(
                         UserTE->Scalars.front()->getType());
-                    auto It = MinBWs.find(TE);
-                    if (It != MinBWs.end() && It->second.first > UserTESz)
+                    if (all_of(TEs, [&](const TreeEntry *TE) {
+                          auto It = MinBWs.find(TE);
+                          return It != MinBWs.end() &&
+                                 It->second.first > UserTESz;
+                        }))
                       return true;
                     return DL->getTypeSizeInBits(U->getType()) > UserTESz;
                   }));
@@ -18241,8 +18252,8 @@ void BoUpSLP::computeMinimumValueSizes() {
                [&](Value *V) { return AnalyzedMinBWVals.contains(V); }))
       return 0u;
 
-    unsigned NumParts = TTI->getNumberOfParts(
-        getWidenedType(TreeRootIT, VF * ScalarTyNumElements));
+    unsigned NumParts = ::getNumberOfParts(
+        *TTI, getWidenedType(TreeRootIT, VF * ScalarTyNumElements));
 
     // The maximum bit width required to represent all the values that can be
     // demoted without loss of precision. It would be safe to truncate the roots
@@ -18302,8 +18313,10 @@ void BoUpSLP::computeMinimumValueSizes() {
     // use - ignore it.
     if (NumParts > 1 &&
         NumParts ==
-            TTI->getNumberOfParts(getWidenedType(
-                IntegerType::get(F->getContext(), bit_ceil(MaxBitWidth)), VF)))
+            ::getNumberOfParts(
+                *TTI, getWidenedType(IntegerType::get(F->getContext(),
+                                                      bit_ceil(MaxBitWidth)),
+                                     VF)))
       return 0u;
 
     unsigned Opcode = E.getOpcode();
@@ -20086,14 +20099,14 @@ class HorizontalReduction {
         ReduxWidth =
             getFloorFullVectorNumberOfElements(TTI, ScalarTy, ReduxWidth);
         VectorType *Tp = getWidenedType(ScalarTy, ReduxWidth);
-        NumParts = TTI.getNumberOfParts(Tp);
+        NumParts = ::getNumberOfParts(TTI, Tp);
         NumRegs =
             TTI.getNumberOfRegisters(TTI.getRegisterClassForType(true, Tp));
         while (NumParts > NumRegs) {
           assert(ReduxWidth > 0 && "ReduxWidth is unexpectedly 0.");
           ReduxWidth = bit_floor(ReduxWidth - 1);
           VectorType *Tp = getWidenedType(ScalarTy, ReduxWidth);
-          NumParts = TTI.getNumberOfParts(Tp);
+          NumParts = ::getNumberOfParts(TTI, Tp);
           NumRegs =
               TTI.getNumberOfRegisters(TTI.getRegisterClassForType(true, Tp));
         }
@@ -20173,7 +20186,7 @@ class HorizontalReduction {
         }
         V.reorderTopToBottom();
         // No need to reorder the root node at all.
-        V.reorderBottomToTop(/*IgnoreReorder=*/true);
+        V.reorderBottomToTop(!V.doesRootHaveInTreeUses());
         // Keep extracted other reduction values, if they are used in the
         // vectorization trees.
         BoUpSLP::ExtraValueToDebugLocsMap LocalExternallyUsedValues(
diff --git a/llvm/lib/Transforms/Vectorize/SandboxVectorizer/DependencyGraph.cpp b/llvm/lib/Transforms/Vectorize/SandboxVectorizer/DependencyGraph.cpp
index f080111f08d45..06a5e3bed7f03 100644
--- a/llvm/lib/Transforms/Vectorize/SandboxVectorizer/DependencyGraph.cpp
+++ b/llvm/lib/Transforms/Vectorize/SandboxVectorizer/DependencyGraph.cpp
@@ -14,6 +14,18 @@
 
 namespace llvm::sandboxir {
 
+User::op_iterator PredIterator::skipBadIt(User::op_iterator OpIt,
+                                          User::op_iterator OpItE,
+                                          const DependencyGraph &DAG) {
+  auto Skip = [&DAG](auto OpIt) {
+    auto *I = dyn_cast<Instruction>((*OpIt).get());
+    return I == nullptr || DAG.getNode(I) == nullptr;
+  };
+  while (OpIt != OpItE && Skip(OpIt))
+    ++OpIt;
+  return OpIt;
+}
+
 PredIterator::value_type PredIterator::operator*() {
   // If it's a DGNode then we dereference the operand iterator.
   if (!isa<MemDGNode>(N)) {
@@ -35,16 +47,16 @@ PredIterator &PredIterator::operator++() {
   if (!isa<MemDGNode>(N)) {
     assert(OpIt != OpItE && "Already at end!");
     ++OpIt;
-    // Skip operands that are not instructions.
-    OpIt = skipNonInstr(OpIt, OpItE);
+    // Skip operands that are not instructions or are outside the DAG.
+    OpIt = PredIterator::skipBadIt(OpIt, OpItE, *DAG);
     return *this;
   }
   // It's a MemDGNode, so if we are not at the end of the use-def iterator we
   // need to first increment that.
   if (OpIt != OpItE) {
     ++OpIt;
-    // Skip operands that are not instructions.
-    OpIt = skipNonInstr(OpIt, OpItE);
+    // Skip operands that are not instructions or are outside the DAG.
+    OpIt = PredIterator::skipBadIt(OpIt, OpItE, *DAG);
     return *this;
   }
   // It's a MemDGNode with OpIt == end, so we need to increment MemIt.
@@ -267,11 +279,11 @@ void DependencyGraph::setDefUseUnscheduledSuccs(
       auto *OpI = dyn_cast<Instruction>(Op);
       if (OpI == nullptr)
         continue;
-      if (!TopInterval.contains(OpI))
-        continue;
       auto *OpN = getNode(OpI);
       if (OpN == nullptr)
         continue;
+      if (!TopInterval.contains(OpI))
+        continue;
       ++OpN->UnscheduledSuccs;
     }
   }
@@ -325,37 +337,47 @@ void DependencyGraph::createNewNodes(const Interval<Instruction> &NewInterval) {
   setDefUseUnscheduledSuccs(NewInterval);
 }
 
-MemDGNode *DependencyGraph::getMemDGNodeBefore(DGNode *N,
-                                               bool IncludingN) const {
+MemDGNode *DependencyGraph::getMemDGNodeBefore(DGNode *N, bool IncludingN,
+                                               MemDGNode *SkipN) const {
   auto *I = N->getInstruction();
   for (auto *PrevI = IncludingN ? I : I->getPrevNode(); PrevI != nullptr;
        PrevI = PrevI->getPrevNode()) {
     auto *PrevN = getNodeOrNull(PrevI);
     if (PrevN == nullptr)
       return nullptr;
-    if (auto *PrevMemN = dyn_cast<MemDGNode>(PrevN))
+    auto *PrevMemN = dyn_cast<MemDGNode>(PrevN);
+    if (PrevMemN != nullptr && PrevMemN != SkipN)
       return PrevMemN;
   }
   return nullptr;
 }
 
-MemDGNode *DependencyGraph::getMemDGNodeAfter(DGNode *N,
-                                              bool IncludingN) const {
+MemDGNode *DependencyGraph::getMemDGNodeAfter(DGNode *N, bool IncludingN,
+                                              MemDGNode *SkipN) const {
   auto *I = N->getInstruction();
   for (auto *NextI = IncludingN ? I : I->getNextNode(); NextI != nullptr;
        NextI = NextI->getNextNode()) {
     auto *NextN = getNodeOrNull(NextI);
     if (NextN == nullptr)
       return nullptr;
-    if (auto *NextMemN = dyn_cast<MemDGNode>(NextN))
+    auto *NextMemN = dyn_cast<MemDGNode>(NextN);
+    if (NextMemN != nullptr && NextMemN != SkipN)
       return NextMemN;
   }
   return nullptr;
 }
 
 void DependencyGraph::notifyCreateInstr(Instruction *I) {
-  auto *MemN = dyn_cast<MemDGNode>(getOrCreateNode(I));
-  // TODO: Update the dependencies for the new node.
+  if (Ctx->getTracker().getState() == Tracker::TrackerState::Reverting)
+    // We don't maintain the DAG while reverting.
+    return;
+  // Nothing to do if the node is not in the focus range of the DAG.
+  if (!(DAGInterval.contains(I) || DAGInterval.touches(I)))
+    return;
+  // Include `I` into the interval.
+  DAGInterval = DAGInterval.getUnionInterval({I, I});
+  auto *N = getOrCreateNode(I);
+  auto *MemN = dyn_cast<MemDGNode>(N);
 
   // Update the MemDGNode chain if this is a memory node.
   if (MemN != nullptr) {
@@ -367,16 +389,48 @@ void DependencyGraph::notifyCreateInstr(Instruction *I) {
       NextMemN->PrevMemN = MemN;
       MemN->NextMemN = NextMemN;
     }
+
+    // Add Mem dependencies.
+    // 1. Scan for deps above `I` for deps to `I`: AboveN->MemN.
+    if (DAGInterval.top()->comesBefore(I)) {
+      Interval<Instruction> AboveIntvl(DAGInterval.top(), I->getPrevNode());
+      auto SrcInterval = MemDGNodeIntervalBuilder::make(AboveIntvl, *this);
+      scanAndAddDeps(*MemN, SrcInterval);
+    }
+    // 2. Scan for deps below `I` for deps from `I`: MemN->BelowN.
+    if (I->comesBefore(DAGInterval.bottom())) {
+      Interval<Instruction> BelowIntvl(I->getNextNode(), DAGInterval.bottom());
+      for (MemDGNode &BelowN :
+           MemDGNodeIntervalBuilder::make(BelowIntvl, *this))
+        scanAndAddDeps(BelowN, Interval<MemDGNode>(MemN, MemN));
+    }
   }
 }
 
 void DependencyGraph::notifyMoveInstr(Instruction *I, const BBIterator &To) {
+  if (Ctx->getTracker().getState() == Tracker::TrackerState::Reverting)
+    // We don't maintain the DAG while reverting.
+    return;
   // NOTE: This function runs before `I` moves to its new destination.
   BasicBlock *BB = To.getNodeParent();
   assert(!(To != BB->end() && &*To == I->getNextNode()) &&
          !(To == BB->end() && std::next(I->getIterator()) == BB->end()) &&
          "Should not have been called if destination is same as origin.");
 
+  // TODO: We can only handle fully internal movements within DAGInterval or at
+  // the borders, i.e., right before the top or right after the bottom.
+  assert(To.getNodeParent() == I->getParent() &&
+         "TODO: We don't support movement across BBs!");
+  assert(
+      (To == std::next(DAGInterval.bottom()->getIterator()) ||
+       (To != BB->end() && std::next(To) == DAGInterval.top()->getIterator()) ||
+       (To != BB->end() && DAGInterval.contains(&*To))) &&
+      "TODO: To should be either within the DAGInterval or right "
+      "before/after it.");
+
+  // Make a copy of the DAGInterval before we update it.
+  auto OrigDAGInterval = DAGInterval;
+
   // Maintain the DAGInterval.
   DAGInterval.notifyMoveInstr(I, To);
 
@@ -389,27 +443,44 @@ void DependencyGraph::notifyMoveInstr(Instruction *I, const BBIterator &To) {
   MemDGNode *MemN = dyn_cast<MemDGNode>(N);
   if (MemN == nullptr)
     return;
-  // First detach it from the existing chain.
+
+  // First safely detach it from the existing chain.
   MemN->detachFromChain();
+
   // Now insert it back into the chain at the new location.
-  if (To != BB->end()) {
-    DGNode *ToN = getNodeOrNull(&*To);
-    if (ToN != nullptr) {
-      MemN->setPrevNode(getMemDGNodeBefore(ToN, /*IncludingN=*/false));
-      MemN->setNextNode(getMemDGNodeAfter(ToN, /*IncludingN=*/true));
-    }
+  //
+  // We won't always have a DGNode to insert before it. If `To` is BB->end() or
+  // if it points to an instr after DAGInterval.bottom() then we will have to
+  // find a node to insert *after*.
+  //
+  // BB:                              BB:
+  //  I1                               I1 ^
+  //  I2                               I2 | DAGInteval [I1 to I3]
+  //  I3                               I3 V
+  //  I4                               I4   <- `To` == right after DAGInterval
+  //    <- `To` == BB->end()
+  //
+  if (To == BB->end() ||
+      To == std::next(OrigDAGInterval.bottom()->getIterator())) {
+    // If we don't have a node to insert before, find a node to insert after and
+    // update the chain.
+    DGNode *InsertAfterN = getNode(&*std::prev(To));
+    MemN->setPrevNode(
+        getMemDGNodeBefore(InsertAfterN, /*IncludingN=*/true, /*SkipN=*/MemN));
   } else {
-    // MemN becomes the last instruction in the BB.
-    auto *TermN = getNodeOrNull(BB->getTerminator());
-    if (TermN != nullptr) {
-      MemN->setPrevNode(getMemDGNodeBefore(TermN, /*IncludingN=*/false));
-    } else {
-      // The terminator is outside the DAG interval so do nothing.
-    }
+    // We have a node to insert before, so update the chain.
+    DGNode *BeforeToN = getNode(&*To);
+    MemN->setPrevNode(
+        getMemDGNodeBefore(BeforeToN, /*IncludingN=*/false, /*SkipN=*/MemN));
+    MemN->setNextNode(
+        getMemDGNodeAfter(BeforeToN, /*IncludingN=*/true, /*SkipN=*/MemN));
   }
 }
 
 void DependencyGraph::notifyEraseInstr(Instruction *I) {
+  if (Ctx->getTracker().getState() == Tracker::TrackerState::Reverting)
+    // We don't maintain the DAG while reverting.
+    return;
   // Update the MemDGNode chain if this is a memory node.
   if (auto *MemN = dyn_cast_or_null<MemDGNode>(getNodeOrNull(I))) {
     auto *PrevMemN = getMemDGNodeBefore(MemN, /*IncludingN=*/false);
diff --git a/llvm/lib/Transforms/Vectorize/SandboxVectorizer/Legality.cpp b/llvm/lib/Transforms/Vectorize/SandboxVectorizer/Legality.cpp
index 085f4cd67ab76..c9329c24e1f4c 100644
--- a/llvm/lib/Transforms/Vectorize/SandboxVectorizer/Legality.cpp
+++ b/llvm/lib/Transforms/Vectorize/SandboxVectorizer/Legality.cpp
@@ -116,7 +116,15 @@ LegalityAnalysis::notVectorizableBasedOnOpcodesAndTypes(
       return std::nullopt;
     return ResultReason::DiffOpcodes;
   }
-  case Instruction::Opcode::Select:
+  case Instruction::Opcode::Select: {
+    auto *Sel0 = cast<SelectInst>(Bndl[0]);
+    auto *Cond0 = Sel0->getCondition();
+    if (VecUtils::getNumLanes(Cond0) != VecUtils::getNumLanes(Sel0))
+      // TODO: For now we don't vectorize if the lanes in the condition don't
+      // match those of the select instruction.
+      return ResultReason::Unimplemented;
+    return std::nullopt;
+  }
   case Instruction::Opcode::FNeg:
   case Instruction::Opcode::Add:
   case Instruction::Opcode::FAdd:
@@ -214,6 +222,15 @@ const LegalityResult &LegalityAnalysis::canVectorize(ArrayRef<Value *> Bndl,
                dumpBndl(Bndl););
     return createLegalityResult<Pack>(ResultReason::NotInstructions);
   }
+  // Pack if not in the same BB.
+  auto *BB = cast<Instruction>(Bndl[0])->getParent();
+  if (any_of(drop_begin(Bndl),
+             [BB](auto *V) { return cast<Instruction>(V)->getParent() != BB; }))
+    return createLegalityResult<Pack>(ResultReason::DiffBBs);
+  // Pack if instructions repeat, i.e., require some sort of broadcast.
+  SmallPtrSet<Value *, 8> Unique(Bndl.begin(), Bndl.end());
+  if (Unique.size() != Bndl.size())
+    return createLegalityResult<Pack>(ResultReason::RepeatedInstrs);
 
   auto CollectDescrs = getHowToCollectValues(Bndl);
   if (CollectDescrs.hasVectorInputs()) {
diff --git a/llvm/lib/Transforms/Vectorize/SandboxVectorizer/Passes/BottomUpVec.cpp b/llvm/lib/Transforms/Vectorize/SandboxVectorizer/Passes/BottomUpVec.cpp
index 7cebde335cb4e..507d163240127 100644
--- a/llvm/lib/Transforms/Vectorize/SandboxVectorizer/Passes/BottomUpVec.cpp
+++ b/llvm/lib/Transforms/Vectorize/SandboxVectorizer/Passes/BottomUpVec.cpp
@@ -12,6 +12,7 @@
 #include "llvm/SandboxIR/Function.h"
 #include "llvm/SandboxIR/Instruction.h"
 #include "llvm/SandboxIR/Module.h"
+#include "llvm/SandboxIR/Region.h"
 #include "llvm/SandboxIR/Utils.h"
 #include "llvm/Transforms/Vectorize/SandboxVectorizer/SandboxVectorizerPassBuilder.h"
 #include "llvm/Transforms/Vectorize/SandboxVectorizer/SeedCollector.h"
@@ -27,6 +28,13 @@ static cl::opt<bool>
     AllowNonPow2("sbvec-allow-non-pow2", cl::init(false), cl::Hidden,
                  cl::desc("Allow non-power-of-2 vectorization."));
 
+#ifndef NDEBUG
+static cl::opt<bool>
+    AlwaysVerify("sbvec-always-verify", cl::init(false), cl::Hidden,
+                 cl::desc("Helps find bugs by verifying the IR whenever we "
+                          "emit new instructions (*very* expensive)."));
+#endif // NDEBUG
+
 namespace sandboxir {
 
 BottomUpVec::BottomUpVec(StringRef Pipeline)
@@ -47,7 +55,7 @@ static SmallVector<Value *, 4> getOperand(ArrayRef<Value *> Bndl,
 /// of BB if no instruction found in \p Vals.
 static BasicBlock::iterator getInsertPointAfterInstrs(ArrayRef<Value *> Vals,
                                                       BasicBlock *BB) {
-  auto *BotI = VecUtils::getLastPHIOrSelf(VecUtils::getLowest(Vals));
+  auto *BotI = VecUtils::getLastPHIOrSelf(VecUtils::getLowest(Vals, BB));
   if (BotI == nullptr)
     // We are using BB->begin() (or after PHIs) as the fallback insert point.
     return BB->empty()
@@ -365,6 +373,17 @@ Value *BottomUpVec::vectorizeRec(ArrayRef<Value *> Bndl,
     break;
   }
   }
+#ifndef NDEBUG
+  if (AlwaysVerify) {
+    // This helps find broken IR by constantly verifying the function. Note that
+    // this is very expensive and should only be used for debugging.
+    Instruction *I0 = isa<Instruction>(Bndl[0])
+                          ? cast<Instruction>(Bndl[0])
+                          : cast<Instruction>(UserBndl[0]);
+    assert(!Utils::verifyFunction(I0->getParent()->getParent(), dbgs()) &&
+           "Broken function!");
+  }
+#endif
   return NewVec;
 }
 
@@ -430,13 +449,24 @@ bool BottomUpVec::runOnFunction(Function &F, const Analyses &A) {
 
           assert(SeedSlice.size() >= 2 && "Should have been rejected!");
 
-          // TODO: If vectorization succeeds, run the RegionPassManager on the
-          // resulting region.
-
           // TODO: Refactor to remove the unnecessary copy to SeedSliceVals.
           SmallVector<Value *> SeedSliceVals(SeedSlice.begin(),
                                              SeedSlice.end());
-          Change |= tryVectorize(SeedSliceVals);
+          // Create an empty region. Instructions get added to the region
+          // automatically by the callbacks.
+          auto &Ctx = F.getContext();
+          Region Rgn(Ctx, A.getTTI());
+          // Save the state of the IR before we make any changes. The
+          // transaction gets accepted/reverted by the tr-accept-or-revert pass.
+          Ctx.save();
+          // Try to vectorize starting from the seed slice. The returned value
+          // is true if we found vectorizable code and generated some vector
+          // code for it. It does not mean that the code is profitable.
+          bool VecSuccess = tryVectorize(SeedSliceVals);
+          if (VecSuccess)
+            // WARNING: All passes should return false, except those that
+            // accept/revert the state.
+            Change |= RPM.runOnRegion(Rgn, A);
         }
       }
     }
diff --git a/llvm/lib/Transforms/Vectorize/SandboxVectorizer/Passes/PassRegistry.def b/llvm/lib/Transforms/Vectorize/SandboxVectorizer/Passes/PassRegistry.def
index 0dc72842f1abe..f3aa12729860f 100644
--- a/llvm/lib/Transforms/Vectorize/SandboxVectorizer/Passes/PassRegistry.def
+++ b/llvm/lib/Transforms/Vectorize/SandboxVectorizer/Passes/PassRegistry.def
@@ -19,6 +19,8 @@
 
 REGION_PASS("null", ::llvm::sandboxir::NullPass)
 REGION_PASS("print-instruction-count", ::llvm::sandboxir::PrintInstructionCount)
+REGION_PASS("tr-accept", ::llvm::sandboxir::TransactionAlwaysAccept)
+REGION_PASS("tr-accept-or-revert", ::llvm::sandboxir::TransactionAcceptOrRevert)
 
 #undef REGION_PASS
 
diff --git a/llvm/lib/Transforms/Vectorize/SandboxVectorizer/Passes/RegionsFromMetadata.cpp b/llvm/lib/Transforms/Vectorize/SandboxVectorizer/Passes/RegionsFromMetadata.cpp
index 8e3f5b77429c5..121a195f45ee4 100644
--- a/llvm/lib/Transforms/Vectorize/SandboxVectorizer/Passes/RegionsFromMetadata.cpp
+++ b/llvm/lib/Transforms/Vectorize/SandboxVectorizer/Passes/RegionsFromMetadata.cpp
@@ -19,7 +19,7 @@ RegionsFromMetadata::RegionsFromMetadata(StringRef Pipeline)
 
 bool RegionsFromMetadata::runOnFunction(Function &F, const Analyses &A) {
   SmallVector<std::unique_ptr<sandboxir::Region>> Regions =
-      sandboxir::Region::createRegionsFromMD(F);
+      sandboxir::Region::createRegionsFromMD(F, A.getTTI());
   for (auto &R : Regions) {
     RPM.runOnRegion(*R, A);
   }
diff --git a/llvm/lib/Transforms/Vectorize/SandboxVectorizer/Passes/TransactionAcceptOrRevert.cpp b/llvm/lib/Transforms/Vectorize/SandboxVectorizer/Passes/TransactionAcceptOrRevert.cpp
new file mode 100644
index 0000000000000..874390ba2daae
--- /dev/null
+++ b/llvm/lib/Transforms/Vectorize/SandboxVectorizer/Passes/TransactionAcceptOrRevert.cpp
@@ -0,0 +1,37 @@
+//===- TransactionAcceptOrRevert.cpp - Check cost and accept/revert region ===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/Transforms/Vectorize/SandboxVectorizer/Passes/TransactionAcceptOrRevert.h"
+#include "llvm/Support/CommandLine.h"
+#include "llvm/Support/InstructionCost.h"
+
+namespace llvm {
+
+static cl::opt<int> CostThreshold("sbvec-cost-threshold", cl::init(0),
+                                  cl::Hidden,
+                                  cl::desc("Vectorization cost threshold."));
+
+namespace sandboxir {
+
+bool TransactionAcceptOrRevert::runOnRegion(Region &Rgn, const Analyses &A) {
+  const auto &SB = Rgn.getScoreboard();
+  InstructionCost CostAfterMinusBefore = SB.getAfterCost() - SB.getBeforeCost();
+  // TODO: Print costs / write to remarks.
+  auto &Tracker = Rgn.getContext().getTracker();
+  if (CostAfterMinusBefore < -CostThreshold) {
+    bool HasChanges = !Tracker.empty();
+    Tracker.accept();
+    return HasChanges;
+  }
+  // Revert the IR.
+  Rgn.getContext().getTracker().revert();
+  return false;
+}
+
+} // namespace sandboxir
+} // namespace llvm
diff --git a/llvm/lib/Transforms/Vectorize/SandboxVectorizer/SandboxVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SandboxVectorizer/SandboxVectorizer.cpp
index a6e2b40000529..b233d35212f94 100644
--- a/llvm/lib/Transforms/Vectorize/SandboxVectorizer/SandboxVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SandboxVectorizer/SandboxVectorizer.cpp
@@ -31,9 +31,10 @@ static cl::opt<std::string> UserDefinedPassPipeline(
 
 SandboxVectorizerPass::SandboxVectorizerPass() : FPM("fpm") {
   if (UserDefinedPassPipeline == DefaultPipelineMagicStr) {
-    // TODO: Add region passes to the default pipeline.
+    // TODO: Add passes to the default pipeline. It currently contains:
+    //       - the bottom-up-vectorizer pass
     FPM.setPassPipeline(
-        "bottom-up-vec<>",
+        "bottom-up-vec<tr-accept-or-revert>",
         sandboxir::SandboxVectorizerPassBuilder::createFunctionPass);
   } else {
     // Create the user-defined pipeline.
@@ -87,5 +88,11 @@ bool SandboxVectorizerPass::runImpl(Function &LLVMF) {
   // Create SandboxIR for LLVMF and run BottomUpVec on it.
   sandboxir::Function &F = *Ctx->createFunction(&LLVMF);
   sandboxir::Analyses A(*AA, *SE, *TTI);
-  return FPM.runOnFunction(F, A);
+  bool Change = FPM.runOnFunction(F, A);
+  // Given that sandboxir::Context `Ctx` is defined at a pass-level scope, the
+  // maps from LLVM IR to Sandbox IR may go stale as later passes remove LLVM IR
+  // objects. To avoid issues caused by this clear the context's state.
+  // NOTE: The alternative would be to define Ctx and FPM within runOnFunction()
+  Ctx->clear();
+  return Change;
 }
diff --git a/llvm/lib/Transforms/Vectorize/SandboxVectorizer/SandboxVectorizerPassBuilder.cpp b/llvm/lib/Transforms/Vectorize/SandboxVectorizer/SandboxVectorizerPassBuilder.cpp
index 5ecf7b2ed0d25..0c1ab55e91a5c 100644
--- a/llvm/lib/Transforms/Vectorize/SandboxVectorizer/SandboxVectorizerPassBuilder.cpp
+++ b/llvm/lib/Transforms/Vectorize/SandboxVectorizer/SandboxVectorizerPassBuilder.cpp
@@ -4,6 +4,8 @@
 #include "llvm/Transforms/Vectorize/SandboxVectorizer/Passes/NullPass.h"
 #include "llvm/Transforms/Vectorize/SandboxVectorizer/Passes/PrintInstructionCount.h"
 #include "llvm/Transforms/Vectorize/SandboxVectorizer/Passes/RegionsFromMetadata.h"
+#include "llvm/Transforms/Vectorize/SandboxVectorizer/Passes/TransactionAcceptOrRevert.h"
+#include "llvm/Transforms/Vectorize/SandboxVectorizer/Passes/TransactionAlwaysAccept.h"
 
 namespace llvm::sandboxir {
 
diff --git a/llvm/lib/Transforms/Vectorize/SandboxVectorizer/Scheduler.cpp b/llvm/lib/Transforms/Vectorize/SandboxVectorizer/Scheduler.cpp
index 496521b95a98e..dd24cc3d98cf8 100644
--- a/llvm/lib/Transforms/Vectorize/SandboxVectorizer/Scheduler.cpp
+++ b/llvm/lib/Transforms/Vectorize/SandboxVectorizer/Scheduler.cpp
@@ -79,9 +79,6 @@ void Scheduler::scheduleAndUpdateReadyList(SchedBundle &Bndl) {
   for (DGNode *N : Bndl) {
     N->setScheduled(true);
     for (auto *DepN : N->preds(DAG)) {
-      // TODO: preds() should not return nullptr.
-      if (DepN == nullptr)
-        continue;
       DepN->decrUnscheduledSuccs();
       if (DepN->ready())
         ReadyList.insert(DepN);
@@ -89,6 +86,31 @@ void Scheduler::scheduleAndUpdateReadyList(SchedBundle &Bndl) {
   }
 }
 
+void Scheduler::notifyCreateInstr(Instruction *I) {
+  // The DAG notifier should have run by now.
+  auto *N = DAG.getNode(I);
+  // If there is no DAG node for `I` it means that this is out of scope for the
+  // DAG and as such out of scope for the scheduler too, so nothing to do.
+  if (N == nullptr)
+    return;
+  // If the instruction is inserted below the top-of-schedule then we mark it as
+  // "scheduled".
+  bool IsScheduled = ScheduleTopItOpt &&
+                     *ScheduleTopItOpt != I->getParent()->end() &&
+                     (*ScheduleTopItOpt.value()).comesBefore(I);
+  if (IsScheduled)
+    N->setScheduled(true);
+  // If the new instruction is above the top of schedule we need to remove its
+  // dependency predecessors from the ready list and increment their
+  // `UnscheduledSuccs` counters.
+  if (!IsScheduled) {
+    for (auto *PredN : N->preds(DAG)) {
+      ReadyList.remove(PredN);
+      PredN->incrUnscheduledSuccs();
+    }
+  }
+}
+
 SchedBundle *Scheduler::createBundle(ArrayRef<Instruction *> Instrs) {
   SchedBundle::ContainerTy Nodes;
   Nodes.reserve(Instrs.size());
@@ -172,6 +194,8 @@ void Scheduler::trimSchedule(ArrayRef<Instruction *> Instrs) {
   for (auto *I = LowestI, *E = TopI->getPrevNode(); I != E;
        I = I->getPrevNode()) {
     auto *N = DAG.getNode(I);
+    if (N == nullptr)
+      continue;
     if (auto *SB = N->getSchedBundle())
       eraseBundle(SB);
   }
@@ -189,7 +213,13 @@ bool Scheduler::trySchedule(ArrayRef<Instruction *> Instrs) {
                 [Instrs](Instruction *I) {
                   return I->getParent() == (*Instrs.begin())->getParent();
                 }) &&
-         "Instrs not in the same BB!");
+         "Instrs not in the same BB, should have been rejected by Legality!");
+  if (ScheduledBB == nullptr)
+    ScheduledBB = Instrs[0]->getParent();
+  // We don't support crossing BBs for now.
+  if (any_of(Instrs,
+             [this](Instruction *I) { return I->getParent() != ScheduledBB; }))
+    return false;
   auto SchedState = getBndlSchedState(Instrs);
   switch (SchedState) {
   case BndlSchedState::FullyScheduled:
diff --git a/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h b/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h
index 44745bfd46f89..e8d3ad89e14cf 100644
--- a/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h
+++ b/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h
@@ -23,6 +23,7 @@ class LoopVectorizationCostModel;
 class TargetLibraryInfo;
 class TargetTransformInfo;
 struct HistogramInfo;
+struct VFRange;
 
 /// A chain of instructions that form a partial reduction.
 /// Designed to match: reduction_bin_op (bin_op (extend (A), (extend (B))),
@@ -127,8 +128,7 @@ class VPRecipeBuilder {
   /// Check if \p I has an opcode that can be widened and return a VPWidenRecipe
   /// if it can. The function should only be called if the cost-model indicates
   /// that widening should be performed.
-  VPWidenRecipe *tryToWiden(Instruction *I, ArrayRef<VPValue *> Operands,
-                            VPBasicBlock *VPBB);
+  VPWidenRecipe *tryToWiden(Instruction *I, ArrayRef<VPValue *> Operands);
 
   /// Makes Histogram count operations safe for vectorization, by emitting a
   /// llvm.experimental.vector.histogram.add intrinsic in place of the
@@ -139,12 +139,16 @@ class VPRecipeBuilder {
 
   /// Examines reduction operations to see if the target can use a cheaper
   /// operation with a wider per-iteration input VF and narrower PHI VF.
-  /// Returns null if no scaled reduction was found, otherwise a pair with a
-  /// struct containing reduction information and the scaling factor between the
-  /// number of elements in the input and output.
-  std::optional<std::pair<PartialReductionChain, unsigned>>
-  getScaledReduction(PHINode *PHI, const RecurrenceDescriptor &Rdx,
-                     VFRange &Range);
+  /// Each element within Chains is a pair with a struct containing reduction
+  /// information and the scaling factor between the number of elements in
+  /// the input and output.
+  /// Recursively calls itself to identify chained scaled reductions.
+  /// Returns true if this invocation added an entry to Chains, otherwise false.
+  /// i.e. returns false in the case that a subcall adds an entry to Chains,
+  /// but the top-level call does not.
+  bool getScaledReductions(
+      Instruction *PHI, Instruction *RdxExitInstr, VFRange &Range,
+      SmallVectorImpl<std::pair<PartialReductionChain, unsigned>> &Chains);
 
 public:
   VPRecipeBuilder(VPlan &Plan, Loop *OrigLoop, const TargetLibraryInfo *TLI,
@@ -169,7 +173,7 @@ class VPRecipeBuilder {
   /// the given VF \p Range.
   VPRecipeBase *tryToCreateWidenRecipe(Instruction *Instr,
                                        ArrayRef<VPValue *> Operands,
-                                       VFRange &Range, VPBasicBlock *VPBB);
+                                       VFRange &Range);
 
   /// Create and return a partial reduction recipe for a reduction instruction
   /// along with binary operation and reduction phi operands.
@@ -214,10 +218,12 @@ class VPRecipeBuilder {
     return Ingredient2Recipe[I];
   }
 
-  /// Build a VPReplicationRecipe for \p I. If it is predicated, add the mask as
-  /// last operand. Range.End may be decreased to ensure same recipe behavior
-  /// from \p Range.Start to \p Range.End.
-  VPReplicateRecipe *handleReplication(Instruction *I, VFRange &Range);
+  /// Build a VPReplicationRecipe for \p I using \p Operands. If it is
+  /// predicated, add the mask as last operand. Range.End may be decreased to
+  /// ensure same recipe behavior from \p Range.Start to \p Range.End.
+  VPReplicateRecipe *handleReplication(Instruction *I,
+                                       ArrayRef<VPValue *> Operands,
+                                       VFRange &Range);
 
   /// Add the incoming values from the backedge to reduction & first-order
   /// recurrence cross-iteration phis.
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.cpp b/llvm/lib/Transforms/Vectorize/VPlan.cpp
index 4159a71469bd1..5a88ebeffb18b 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlan.cpp
@@ -19,6 +19,7 @@
 #include "VPlan.h"
 #include "LoopVectorizationPlanner.h"
 #include "VPlanCFG.h"
+#include "VPlanHelpers.h"
 #include "VPlanPatternMatch.h"
 #include "VPlanTransforms.h"
 #include "VPlanUtils.h"
@@ -52,6 +53,7 @@ using namespace llvm::VPlanPatternMatch;
 namespace llvm {
 extern cl::opt<bool> EnableVPlanNativePath;
 }
+
 extern cl::opt<unsigned> ForceTargetInstructionCost;
 
 static cl::opt<bool> PrintVPlansInDotFormat(
@@ -399,8 +401,8 @@ void VPTransformState::packScalarIntoVectorValue(VPValue *Def,
   set(Def, VectorValue);
 }
 
-BasicBlock *
-VPBasicBlock::createEmptyBasicBlock(VPTransformState::CFGState &CFG) {
+BasicBlock *VPBasicBlock::createEmptyBasicBlock(VPTransformState &State) {
+  auto &CFG = State.CFG;
   // BB stands for IR BasicBlocks. VPBB stands for VPlan VPBasicBlocks.
   // Pred stands for Predessor. Prev stands for Previous - last visited/created.
   BasicBlock *PrevBB = CFG.PrevBB;
@@ -411,7 +413,8 @@ VPBasicBlock::createEmptyBasicBlock(VPTransformState::CFGState &CFG) {
   return NewBB;
 }
 
-void VPBasicBlock::connectToPredecessors(VPTransformState::CFGState &CFG) {
+void VPBasicBlock::connectToPredecessors(VPTransformState &State) {
+  auto &CFG = State.CFG;
   BasicBlock *NewBB = CFG.VPBB2IRBB[this];
   // Hook up the new basic block to its predecessors.
   for (VPBlockBase *PredVPBlock : getHierarchicalPredecessors()) {
@@ -466,7 +469,7 @@ void VPIRBasicBlock::execute(VPTransformState *State) {
         "other blocks must be terminated by a branch");
   }
 
-  connectToPredecessors(State->CFG);
+  connectToPredecessors(*State);
 }
 
 VPIRBasicBlock *VPIRBasicBlock::clone() {
@@ -493,20 +496,27 @@ void VPBasicBlock::execute(VPTransformState *State) {
     //  * the exit of a replicate region.
     State->CFG.VPBB2IRBB[this] = NewBB;
   } else {
-    NewBB = createEmptyBasicBlock(State->CFG);
+    NewBB = createEmptyBasicBlock(*State);
 
     State->Builder.SetInsertPoint(NewBB);
     // Temporarily terminate with unreachable until CFG is rewired.
     UnreachableInst *Terminator = State->Builder.CreateUnreachable();
     // Register NewBB in its loop. In innermost loops its the same for all
     // BB's.
-    if (State->CurrentParentLoop)
-      State->CurrentParentLoop->addBasicBlockToLoop(NewBB, *State->LI);
+    Loop *ParentLoop = State->CurrentParentLoop;
+    // If this block has a sole successor that is an exit block then it needs
+    // adding to the same parent loop as the exit block.
+    VPBlockBase *SuccVPBB = getSingleSuccessor();
+    if (SuccVPBB && State->Plan->isExitBlock(SuccVPBB))
+      ParentLoop = State->LI->getLoopFor(
+          cast<VPIRBasicBlock>(SuccVPBB)->getIRBasicBlock());
+    if (ParentLoop)
+      ParentLoop->addBasicBlockToLoop(NewBB, *State->LI);
     State->Builder.SetInsertPoint(Terminator);
 
     State->CFG.PrevBB = NewBB;
     State->CFG.VPBB2IRBB[this] = NewBB;
-    connectToPredecessors(State->CFG);
+    connectToPredecessors(*State);
   }
 
   // 2. Fill the IR basic block with IR instructions.
@@ -615,6 +625,11 @@ bool VPBasicBlock::isExiting() const {
 }
 
 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
+void VPBlockBase::print(raw_ostream &O) const {
+  VPSlotTracker SlotTracker(getPlan());
+  print(O, "", SlotTracker);
+}
+
 void VPBlockBase::printSuccessors(raw_ostream &O, const Twine &Indent) const {
   if (getSuccessors().empty()) {
     O << Indent << "No successors\n";
@@ -949,6 +964,10 @@ void VPlan::prepareToExecute(Value *TripCountV, Value *VectorTripCountV,
   }
 }
 
+bool VPlan::isExitBlock(VPBlockBase *VPBB) {
+  return isa<VPIRBasicBlock>(VPBB) && VPBB->getNumSuccessors() == 0;
+}
+
 /// Generate the code inside the preheader and body of the vectorized loop.
 /// Assumes a single pre-header basic-block was created for this. Introduce
 /// additional basic-blocks as needed, and fill them all.
@@ -1024,7 +1043,7 @@ void VPlan::execute(VPTransformState *State) {
       // Move the last step to the end of the latch block. This ensures
       // consistent placement of all induction updates.
       Instruction *Inc = cast<Instruction>(Phi->getIncomingValue(1));
-      Inc->moveBefore(VectorLatchBB->getTerminator()->getPrevNode());
+      Inc->moveBefore(std::prev(VectorLatchBB->getTerminator()->getIterator()));
 
       // Use the steps for the last part as backedge value for the induction.
       if (auto *IV = dyn_cast<VPWidenIntOrFpInductionRecipe>(&R))
@@ -1459,58 +1478,6 @@ void VPUser::printOperands(raw_ostream &O, VPSlotTracker &SlotTracker) const {
 }
 #endif
 
-void VPInterleavedAccessInfo::visitRegion(VPRegionBlock *Region,
-                                          Old2NewTy &Old2New,
-                                          InterleavedAccessInfo &IAI) {
-  ReversePostOrderTraversal<VPBlockShallowTraversalWrapper<VPBlockBase *>>
-      RPOT(Region->getEntry());
-  for (VPBlockBase *Base : RPOT) {
-    visitBlock(Base, Old2New, IAI);
-  }
-}
-
-void VPInterleavedAccessInfo::visitBlock(VPBlockBase *Block, Old2NewTy &Old2New,
-                                         InterleavedAccessInfo &IAI) {
-  if (VPBasicBlock *VPBB = dyn_cast<VPBasicBlock>(Block)) {
-    for (VPRecipeBase &VPI : *VPBB) {
-      if (isa<VPWidenPHIRecipe>(&VPI))
-        continue;
-      assert(isa<VPInstruction>(&VPI) && "Can only handle VPInstructions");
-      auto *VPInst = cast<VPInstruction>(&VPI);
-
-      auto *Inst = dyn_cast_or_null<Instruction>(VPInst->getUnderlyingValue());
-      if (!Inst)
-        continue;
-      auto *IG = IAI.getInterleaveGroup(Inst);
-      if (!IG)
-        continue;
-
-      auto NewIGIter = Old2New.find(IG);
-      if (NewIGIter == Old2New.end())
-        Old2New[IG] = new InterleaveGroup<VPInstruction>(
-            IG->getFactor(), IG->isReverse(), IG->getAlign());
-
-      if (Inst == IG->getInsertPos())
-        Old2New[IG]->setInsertPos(VPInst);
-
-      InterleaveGroupMap[VPInst] = Old2New[IG];
-      InterleaveGroupMap[VPInst]->insertMember(
-          VPInst, IG->getIndex(Inst),
-          Align(IG->isReverse() ? (-1) * int(IG->getFactor())
-                                : IG->getFactor()));
-    }
-  } else if (VPRegionBlock *Region = dyn_cast<VPRegionBlock>(Block))
-    visitRegion(Region, Old2New, IAI);
-  else
-    llvm_unreachable("Unsupported kind of VPBlock.");
-}
-
-VPInterleavedAccessInfo::VPInterleavedAccessInfo(VPlan &Plan,
-                                                 InterleavedAccessInfo &IAI) {
-  Old2NewTy Old2New;
-  visitRegion(Plan.getVectorLoopRegion(), Old2New, IAI);
-}
-
 void VPSlotTracker::assignName(const VPValue *V) {
   assert(!VPValue2Name.contains(V) && "VPValue already has a name!");
   auto *UV = V->getUnderlyingValue();
@@ -1628,6 +1595,9 @@ void LoopVectorizationPlanner::buildVPlans(ElementCount MinVF,
     VFRange SubRange = {VF, MaxVFTimes2};
     auto Plan = buildVPlan(SubRange);
     VPlanTransforms::optimize(*Plan);
+    // Update the name of the latch of the top-level vector loop region region
+    // after optimizations which includes block folding.
+    Plan->getVectorLoopRegion()->getExiting()->setName("vector.latch");
     VPlans.push_back(std::move(Plan));
     VF = SubRange.End;
   }
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index 9124905c99717..fbbc466f2f7f6 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -17,7 +17,6 @@
 /// 4. VPInstruction, a concrete Recipe and VPUser modeling a single planned
 ///    instruction;
 /// 5. The VPlan class holding a candidate for vectorization;
-/// 6. The VPlanPrinter class providing a way to print a plan in dot format;
 /// These are documented in docs/VectorizationPlan.rst.
 //
 //===----------------------------------------------------------------------===//
@@ -34,10 +33,7 @@
 #include "llvm/ADT/Twine.h"
 #include "llvm/ADT/ilist.h"
 #include "llvm/ADT/ilist_node.h"
-#include "llvm/Analysis/DomTreeUpdater.h"
 #include "llvm/Analysis/IVDescriptors.h"
-#include "llvm/Analysis/LoopInfo.h"
-#include "llvm/Analysis/TargetTransformInfo.h"
 #include "llvm/Analysis/VectorUtils.h"
 #include "llvm/IR/DebugLoc.h"
 #include "llvm/IR/FMF.h"
@@ -54,19 +50,20 @@ class BasicBlock;
 class DominatorTree;
 class InnerLoopVectorizer;
 class IRBuilderBase;
-class LoopInfo;
+struct VPTransformState;
 class raw_ostream;
 class RecurrenceDescriptor;
 class SCEV;
 class Type;
 class VPBasicBlock;
+class VPBuilder;
 class VPRegionBlock;
 class VPlan;
+class VPLane;
 class VPReplicateRecipe;
 class VPlanSlp;
 class Value;
 class LoopVectorizationCostModel;
-class LoopVersioning;
 
 struct VPCostContext;
 
@@ -74,318 +71,8 @@ namespace Intrinsic {
 typedef unsigned ID;
 }
 
-/// Returns a calculation for the total number of elements for a given \p VF.
-/// For fixed width vectors this value is a constant, whereas for scalable
-/// vectors it is an expression determined at runtime.
-Value *getRuntimeVF(IRBuilderBase &B, Type *Ty, ElementCount VF);
-
-/// Return a value for Step multiplied by VF.
-Value *createStepForVF(IRBuilderBase &B, Type *Ty, ElementCount VF,
-                       int64_t Step);
-
-/// A helper function that returns the reciprocal of the block probability of
-/// predicated blocks. If we return X, we are assuming the predicated block
-/// will execute once for every X iterations of the loop header.
-///
-/// TODO: We should use actual block probability here, if available. Currently,
-///       we always assume predicated blocks have a 50% chance of executing.
-inline unsigned getReciprocalPredBlockProb() { return 2; }
-
-/// A range of powers-of-2 vectorization factors with fixed start and
-/// adjustable end. The range includes start and excludes end, e.g.,:
-/// [1, 16) = {1, 2, 4, 8}
-struct VFRange {
-  // A power of 2.
-  const ElementCount Start;
-
-  // A power of 2. If End <= Start range is empty.
-  ElementCount End;
-
-  bool isEmpty() const {
-    return End.getKnownMinValue() <= Start.getKnownMinValue();
-  }
-
-  VFRange(const ElementCount &Start, const ElementCount &End)
-      : Start(Start), End(End) {
-    assert(Start.isScalable() == End.isScalable() &&
-           "Both Start and End should have the same scalable flag");
-    assert(isPowerOf2_32(Start.getKnownMinValue()) &&
-           "Expected Start to be a power of 2");
-    assert(isPowerOf2_32(End.getKnownMinValue()) &&
-           "Expected End to be a power of 2");
-  }
-
-  /// Iterator to iterate over vectorization factors in a VFRange.
-  class iterator
-      : public iterator_facade_base<iterator, std::forward_iterator_tag,
-                                    ElementCount> {
-    ElementCount VF;
-
-  public:
-    iterator(ElementCount VF) : VF(VF) {}
-
-    bool operator==(const iterator &Other) const { return VF == Other.VF; }
-
-    ElementCount operator*() const { return VF; }
-
-    iterator &operator++() {
-      VF *= 2;
-      return *this;
-    }
-  };
-
-  iterator begin() { return iterator(Start); }
-  iterator end() {
-    assert(isPowerOf2_32(End.getKnownMinValue()));
-    return iterator(End);
-  }
-};
-
 using VPlanPtr = std::unique_ptr<VPlan>;
 
-/// In what follows, the term "input IR" refers to code that is fed into the
-/// vectorizer whereas the term "output IR" refers to code that is generated by
-/// the vectorizer.
-
-/// VPLane provides a way to access lanes in both fixed width and scalable
-/// vectors, where for the latter the lane index sometimes needs calculating
-/// as a runtime expression.
-class VPLane {
-public:
-  /// Kind describes how to interpret Lane.
-  enum class Kind : uint8_t {
-    /// For First, Lane is the index into the first N elements of a
-    /// fixed-vector <N x <ElTy>> or a scalable vector <vscale x N x <ElTy>>.
-    First,
-    /// For ScalableLast, Lane is the offset from the start of the last
-    /// N-element subvector in a scalable vector <vscale x N x <ElTy>>. For
-    /// example, a Lane of 0 corresponds to lane `(vscale - 1) * N`, a Lane of
-    /// 1 corresponds to `((vscale - 1) * N) + 1`, etc.
-    ScalableLast
-  };
-
-private:
-  /// in [0..VF)
-  unsigned Lane;
-
-  /// Indicates how the Lane should be interpreted, as described above.
-  Kind LaneKind;
-
-public:
-  VPLane(unsigned Lane) : Lane(Lane), LaneKind(VPLane::Kind::First) {}
-  VPLane(unsigned Lane, Kind LaneKind) : Lane(Lane), LaneKind(LaneKind) {}
-
-  static VPLane getFirstLane() { return VPLane(0, VPLane::Kind::First); }
-
-  static VPLane getLaneFromEnd(const ElementCount &VF, unsigned Offset) {
-    assert(Offset > 0 && Offset <= VF.getKnownMinValue() &&
-           "trying to extract with invalid offset");
-    unsigned LaneOffset = VF.getKnownMinValue() - Offset;
-    Kind LaneKind;
-    if (VF.isScalable())
-      // In this case 'LaneOffset' refers to the offset from the start of the
-      // last subvector with VF.getKnownMinValue() elements.
-      LaneKind = VPLane::Kind::ScalableLast;
-    else
-      LaneKind = VPLane::Kind::First;
-    return VPLane(LaneOffset, LaneKind);
-  }
-
-  static VPLane getLastLaneForVF(const ElementCount &VF) {
-    return getLaneFromEnd(VF, 1);
-  }
-
-  /// Returns a compile-time known value for the lane index and asserts if the
-  /// lane can only be calculated at runtime.
-  unsigned getKnownLane() const {
-    assert(LaneKind == Kind::First);
-    return Lane;
-  }
-
-  /// Returns an expression describing the lane index that can be used at
-  /// runtime.
-  Value *getAsRuntimeExpr(IRBuilderBase &Builder, const ElementCount &VF) const;
-
-  /// Returns the Kind of lane offset.
-  Kind getKind() const { return LaneKind; }
-
-  /// Returns true if this is the first lane of the whole vector.
-  bool isFirstLane() const { return Lane == 0 && LaneKind == Kind::First; }
-
-  /// Maps the lane to a cache index based on \p VF.
-  unsigned mapToCacheIndex(const ElementCount &VF) const {
-    switch (LaneKind) {
-    case VPLane::Kind::ScalableLast:
-      assert(VF.isScalable() && Lane < VF.getKnownMinValue());
-      return VF.getKnownMinValue() + Lane;
-    default:
-      assert(Lane < VF.getKnownMinValue());
-      return Lane;
-    }
-  }
-};
-
-/// VPTransformState holds information passed down when "executing" a VPlan,
-/// needed for generating the output IR.
-struct VPTransformState {
-  VPTransformState(const TargetTransformInfo *TTI, ElementCount VF, unsigned UF,
-                   LoopInfo *LI, DominatorTree *DT, IRBuilderBase &Builder,
-                   InnerLoopVectorizer *ILV, VPlan *Plan,
-                   Loop *CurrentParentLoop, Type *CanonicalIVTy);
-  /// Target Transform Info.
-  const TargetTransformInfo *TTI;
-
-  /// The chosen Vectorization Factor of the loop being vectorized.
-  ElementCount VF;
-
-  /// Hold the index to generate specific scalar instructions. Null indicates
-  /// that all instances are to be generated, using either scalar or vector
-  /// instructions.
-  std::optional<VPLane> Lane;
-
-  struct DataState {
-    // Each value from the original loop, when vectorized, is represented by a
-    // vector value in the map.
-    DenseMap<VPValue *, Value *> VPV2Vector;
-
-    DenseMap<VPValue *, SmallVector<Value *, 4>> VPV2Scalars;
-  } Data;
-
-  /// Get the generated vector Value for a given VPValue \p Def if \p IsScalar
-  /// is false, otherwise return the generated scalar. \See set.
-  Value *get(VPValue *Def, bool IsScalar = false);
-
-  /// Get the generated Value for a given VPValue and given Part and Lane.
-  Value *get(VPValue *Def, const VPLane &Lane);
-
-  bool hasVectorValue(VPValue *Def) { return Data.VPV2Vector.contains(Def); }
-
-  bool hasScalarValue(VPValue *Def, VPLane Lane) {
-    auto I = Data.VPV2Scalars.find(Def);
-    if (I == Data.VPV2Scalars.end())
-      return false;
-    unsigned CacheIdx = Lane.mapToCacheIndex(VF);
-    return CacheIdx < I->second.size() && I->second[CacheIdx];
-  }
-
-  /// Set the generated vector Value for a given VPValue, if \p
-  /// IsScalar is false. If \p IsScalar is true, set the scalar in lane 0.
-  void set(VPValue *Def, Value *V, bool IsScalar = false) {
-    if (IsScalar) {
-      set(Def, V, VPLane(0));
-      return;
-    }
-    assert((VF.isScalar() || V->getType()->isVectorTy()) &&
-           "scalar values must be stored as (0, 0)");
-    Data.VPV2Vector[Def] = V;
-  }
-
-  /// Reset an existing vector value for \p Def and a given \p Part.
-  void reset(VPValue *Def, Value *V) {
-    assert(Data.VPV2Vector.contains(Def) && "need to overwrite existing value");
-    Data.VPV2Vector[Def] = V;
-  }
-
-  /// Set the generated scalar \p V for \p Def and the given \p Lane.
-  void set(VPValue *Def, Value *V, const VPLane &Lane) {
-    auto &Scalars = Data.VPV2Scalars[Def];
-    unsigned CacheIdx = Lane.mapToCacheIndex(VF);
-    if (Scalars.size() <= CacheIdx)
-      Scalars.resize(CacheIdx + 1);
-    assert(!Scalars[CacheIdx] && "should overwrite existing value");
-    Scalars[CacheIdx] = V;
-  }
-
-  /// Reset an existing scalar value for \p Def and a given \p Lane.
-  void reset(VPValue *Def, Value *V, const VPLane &Lane) {
-    auto Iter = Data.VPV2Scalars.find(Def);
-    assert(Iter != Data.VPV2Scalars.end() &&
-           "need to overwrite existing value");
-    unsigned CacheIdx = Lane.mapToCacheIndex(VF);
-    assert(CacheIdx < Iter->second.size() &&
-           "need to overwrite existing value");
-    Iter->second[CacheIdx] = V;
-  }
-
-  /// Add additional metadata to \p To that was not present on \p Orig.
-  ///
-  /// Currently this is used to add the noalias annotations based on the
-  /// inserted memchecks.  Use this for instructions that are *cloned* into the
-  /// vector loop.
-  void addNewMetadata(Instruction *To, const Instruction *Orig);
-
-  /// Add metadata from one instruction to another.
-  ///
-  /// This includes both the original MDs from \p From and additional ones (\see
-  /// addNewMetadata).  Use this for *newly created* instructions in the vector
-  /// loop.
-  void addMetadata(Value *To, Instruction *From);
-
-  /// Set the debug location in the builder using the debug location \p DL.
-  void setDebugLocFrom(DebugLoc DL);
-
-  /// Construct the vector value of a scalarized value \p V one lane at a time.
-  void packScalarIntoVectorValue(VPValue *Def, const VPLane &Lane);
-
-  /// Hold state information used when constructing the CFG of the output IR,
-  /// traversing the VPBasicBlocks and generating corresponding IR BasicBlocks.
-  struct CFGState {
-    /// The previous VPBasicBlock visited. Initially set to null.
-    VPBasicBlock *PrevVPBB = nullptr;
-
-    /// The previous IR BasicBlock created or used. Initially set to the new
-    /// header BasicBlock.
-    BasicBlock *PrevBB = nullptr;
-
-    /// The last IR BasicBlock in the output IR. Set to the exit block of the
-    /// vector loop.
-    BasicBlock *ExitBB = nullptr;
-
-    /// A mapping of each VPBasicBlock to the corresponding BasicBlock. In case
-    /// of replication, maps the BasicBlock of the last replica created.
-    SmallDenseMap<VPBasicBlock *, BasicBlock *> VPBB2IRBB;
-
-    /// Updater for the DominatorTree.
-    DomTreeUpdater DTU;
-
-    CFGState(DominatorTree *DT)
-        : DTU(DT, DomTreeUpdater::UpdateStrategy::Lazy) {}
-
-    /// Returns the BasicBlock* mapped to the pre-header of the loop region
-    /// containing \p R.
-    BasicBlock *getPreheaderBBFor(VPRecipeBase *R);
-  } CFG;
-
-  /// Hold a pointer to LoopInfo to register new basic blocks in the loop.
-  LoopInfo *LI;
-
-  /// Hold a reference to the IRBuilder used to generate output IR code.
-  IRBuilderBase &Builder;
-
-  /// Hold a pointer to InnerLoopVectorizer to reuse its IR generation methods.
-  InnerLoopVectorizer *ILV;
-
-  /// Pointer to the VPlan code is generated for.
-  VPlan *Plan;
-
-  /// The parent loop object for the current scope, or nullptr.
-  Loop *CurrentParentLoop = nullptr;
-
-  /// LoopVersioning.  It's only set up (non-null) if memchecks were
-  /// used.
-  ///
-  /// This is currently only used to add no-alias metadata based on the
-  /// memchecks.  The actually versioning is performed manually.
-  LoopVersioning *LVer = nullptr;
-
-  /// Map SCEVs to their expanded values. Populated when executing
-  /// VPExpandSCEVRecipes.
-  DenseMap<const SCEV *, Value *> ExpandedSCEVs;
-
-  /// VPlan-based type analysis.
-  VPTypeAnalysis TypeAnalysis;
-};
-
 /// VPBlockBase is the building block of the Hierarchical Control-Flow Graph.
 /// A VPBlockBase can be either a VPBasicBlock or a VPRegionBlock.
 class VPBlockBase {
@@ -653,10 +340,7 @@ class VPBlockBase {
                      VPSlotTracker &SlotTracker) const = 0;
 
   /// Print plain-text dump of this VPlan to \p O.
-  void print(raw_ostream &O) const {
-    VPSlotTracker SlotTracker(getPlan());
-    print(O, "", SlotTracker);
-  }
+  void print(raw_ostream &O) const;
 
   /// Print the successors of this block to \p O, prefixing all lines with \p
   /// Indent.
@@ -672,34 +356,6 @@ class VPBlockBase {
   virtual VPBlockBase *clone() = 0;
 };
 
-/// Struct to hold various analysis needed for cost computations.
-struct VPCostContext {
-  const TargetTransformInfo &TTI;
-  const TargetLibraryInfo &TLI;
-  VPTypeAnalysis Types;
-  LLVMContext &LLVMCtx;
-  LoopVectorizationCostModel &CM;
-  SmallPtrSet<Instruction *, 8> SkipCostComputation;
-  TargetTransformInfo::TargetCostKind CostKind;
-
-  VPCostContext(const TargetTransformInfo &TTI, const TargetLibraryInfo &TLI,
-                Type *CanIVTy, LoopVectorizationCostModel &CM,
-                TargetTransformInfo::TargetCostKind CostKind)
-      : TTI(TTI), TLI(TLI), Types(CanIVTy), LLVMCtx(CanIVTy->getContext()),
-        CM(CM), CostKind(CostKind) {}
-
-  /// Return the cost for \p UI with \p VF using the legacy cost model as
-  /// fallback until computing the cost of all recipes migrates to VPlan.
-  InstructionCost getLegacyCost(Instruction *UI, ElementCount VF) const;
-
-  /// Return true if the cost for \p UI shouldn't be computed, e.g. because it
-  /// has already been pre-computed.
-  bool skipCostComputation(Instruction *UI, bool IsVector) const;
-
-  /// Returns the OperandInfo for \p V, if it is a live-in.
-  TargetTransformInfo::OperandValueInfo getOperandInfo(VPValue *V) const;
-};
-
 /// VPRecipeBase is a base class modeling a sequence of one or more output IR
 /// instructions. VPRecipeBase owns the VPValues it defines through VPDef
 /// and is responsible for deleting its defined values. Single-value
@@ -1057,6 +713,7 @@ class VPRecipeWithIRFlags : public VPSingleDefRecipe {
            R->getVPDefID() == VPRecipeBase::VPWidenEVLSC ||
            R->getVPDefID() == VPRecipeBase::VPWidenGEPSC ||
            R->getVPDefID() == VPRecipeBase::VPWidenCastSC ||
+           R->getVPDefID() == VPRecipeBase::VPWidenIntrinsicSC ||
            R->getVPDefID() == VPRecipeBase::VPReplicateSC ||
            R->getVPDefID() == VPRecipeBase::VPReverseVectorPointerSC ||
            R->getVPDefID() == VPRecipeBase::VPVectorPointerSC;
@@ -1222,6 +879,9 @@ class VPInstruction : public VPRecipeWithIRFlags,
     // Returns a scalar boolean value, which is true if any lane of its (only
     // boolean) vector operand is true.
     AnyOf,
+    // Extracts the first active lane of a vector, where the first operand is
+    // the predicate, and the second operand is the vector to extract.
+    ExtractFirstActive,
   };
 
 private:
@@ -1312,10 +972,7 @@ class VPInstruction : public VPRecipeWithIRFlags,
 
   /// Return the cost of this VPInstruction.
   InstructionCost computeCost(ElementCount VF,
-                              VPCostContext &Ctx) const override {
-    // TODO: Compute accurate cost after retiring the legacy cost model.
-    return 0;
-  }
+                              VPCostContext &Ctx) const override;
 
 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
   /// Print the VPInstruction to \p O.
@@ -1422,6 +1079,11 @@ class VPIRInstruction : public VPRecipeBase {
            "Op must be an operand of the recipe");
     return true;
   }
+
+  /// Update the recipes single operand to the last lane of the operand using \p
+  /// Builder. Must only be used for single operand VPIRInstructions wrapping a
+  /// PHINode.
+  void extractLastLaneOfOperand(VPBuilder &Builder);
 };
 
 /// VPWidenRecipe is a recipe for producing a widened instruction using the
@@ -2338,6 +2000,11 @@ class VPWidenPHIRecipe : public VPSingleDefRecipe {
   /// Returns the \p I th incoming VPBasicBlock.
   VPBasicBlock *getIncomingBlock(unsigned I) { return IncomingBlocks[I]; }
 
+  /// Set the \p I th incoming VPBasicBlock to \p IncomingBlock.
+  void setIncomingBlock(unsigned I, VPBasicBlock *IncomingBlock) {
+    IncomingBlocks[I] = IncomingBlock;
+  }
+
   /// Returns the \p I th incoming VPValue.
   VPValue *getIncomingValue(unsigned I) { return getOperand(I); }
 };
@@ -2456,7 +2123,10 @@ class VPPartialReductionRecipe : public VPSingleDefRecipe {
       : VPSingleDefRecipe(VPDef::VPPartialReductionSC,
                           ArrayRef<VPValue *>({Op0, Op1}), ReductionInst),
         Opcode(Opcode) {
-    assert(isa<VPReductionPHIRecipe>(getOperand(1)->getDefiningRecipe()) &&
+    [[maybe_unused]] auto *AccumulatorRecipe =
+        getOperand(1)->getDefiningRecipe();
+    assert((isa<VPReductionPHIRecipe>(AccumulatorRecipe) ||
+            isa<VPPartialReductionRecipe>(AccumulatorRecipe)) &&
            "Unexpected operand order for partial reduction recipe");
   }
   ~VPPartialReductionRecipe() override = default;
@@ -3658,12 +3328,12 @@ class VPBasicBlock : public VPBlockBase {
 
   /// Connect the VPBBs predecessors' in the VPlan CFG to the IR basic block
   /// generated for this VPBB.
-  void connectToPredecessors(VPTransformState::CFGState &CFG);
+  void connectToPredecessors(VPTransformState &State);
 
 private:
   /// Create an IR BasicBlock to hold the output instructions generated by this
   /// VPBasicBlock, and return it. Update the CFGState accordingly.
-  BasicBlock *createEmptyBasicBlock(VPTransformState::CFGState &CFG);
+  BasicBlock *createEmptyBasicBlock(VPTransformState &State);
 };
 
 /// A special type of VPBasicBlock that wraps an existing IR basic block.
@@ -3958,6 +3628,9 @@ class VPlan {
   /// of VPBlockShallowTraversalWrapper.
   auto getExitBlocks();
 
+  /// Returns true if \p VPBB is an exit block.
+  bool isExitBlock(VPBlockBase *VPBB);
+
   /// The trip count of the original loop.
   VPValue *getTripCount() const {
     assert(TripCount && "trip count needs to be set before accessing it");
@@ -3996,8 +3669,8 @@ class VPlan {
     VFs.insert(VF);
   }
 
-  bool hasVF(ElementCount VF) { return VFs.count(VF); }
-  bool hasScalableVF() {
+  bool hasVF(ElementCount VF) const { return VFs.count(VF); }
+  bool hasScalableVF() const {
     return any_of(VFs, [](ElementCount VF) { return VF.isScalable(); });
   }
 
@@ -4007,7 +3680,12 @@ class VPlan {
     return {VFs.begin(), VFs.end()};
   }
 
-  bool hasScalarVFOnly() const { return VFs.size() == 1 && VFs[0].isScalar(); }
+  bool hasScalarVFOnly() const {
+    bool HasScalarVFOnly = VFs.size() == 1 && VFs[0].isScalar();
+    assert(HasScalarVFOnly == hasVF(ElementCount::getFixed(1)) &&
+           "Plan with scalar VF should only have a single VF");
+    return HasScalarVFOnly;
+  }
 
   bool hasUF(unsigned UF) const { return UFs.empty() || UFs.contains(UF); }
 
@@ -4031,18 +3709,16 @@ class VPlan {
   ///  yet) for \p V.
   VPValue *getOrAddLiveIn(Value *V) {
     assert(V && "Trying to get or add the VPValue of a null Value");
-    if (!Value2VPValue.count(V)) {
+    auto [It, Inserted] = Value2VPValue.try_emplace(V);
+    if (Inserted) {
       VPValue *VPV = new VPValue(V);
       VPLiveInsToFree.push_back(VPV);
       assert(VPV->isLiveIn() && "VPV must be a live-in.");
-      assert(!Value2VPValue.count(V) && "Value already exists in VPlan");
-      Value2VPValue[V] = VPV;
+      It->second = VPV;
     }
 
-    assert(Value2VPValue.count(V) && "Value does not exist in VPlan");
-    assert(Value2VPValue[V]->isLiveIn() &&
-           "Only live-ins should be in mapping");
-    return Value2VPValue[V];
+    assert(It->second->isLiveIn() && "Only live-ins should be in mapping");
+    return It->second;
   }
 
   /// Return the live-in VPValue for \p V, if there is one or nullptr otherwise.
@@ -4130,55 +3806,6 @@ class VPlan {
 };
 
 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
-/// VPlanPrinter prints a given VPlan to a given output stream. The printing is
-/// indented and follows the dot format.
-class VPlanPrinter {
-  raw_ostream &OS;
-  const VPlan &Plan;
-  unsigned Depth = 0;
-  unsigned TabWidth = 2;
-  std::string Indent;
-  unsigned BID = 0;
-  SmallDenseMap<const VPBlockBase *, unsigned> BlockID;
-
-  VPSlotTracker SlotTracker;
-
-  /// Handle indentation.
-  void bumpIndent(int b) { Indent = std::string((Depth += b) * TabWidth, ' '); }
-
-  /// Print a given \p Block of the Plan.
-  void dumpBlock(const VPBlockBase *Block);
-
-  /// Print the information related to the CFG edges going out of a given
-  /// \p Block, followed by printing the successor blocks themselves.
-  void dumpEdges(const VPBlockBase *Block);
-
-  /// Print a given \p BasicBlock, including its VPRecipes, followed by printing
-  /// its successor blocks.
-  void dumpBasicBlock(const VPBasicBlock *BasicBlock);
-
-  /// Print a given \p Region of the Plan.
-  void dumpRegion(const VPRegionBlock *Region);
-
-  unsigned getOrCreateBID(const VPBlockBase *Block) {
-    return BlockID.count(Block) ? BlockID[Block] : BlockID[Block] = BID++;
-  }
-
-  Twine getOrCreateName(const VPBlockBase *Block);
-
-  Twine getUID(const VPBlockBase *Block);
-
-  /// Print the information related to a CFG edge between two VPBlockBases.
-  void drawEdge(const VPBlockBase *From, const VPBlockBase *To, bool Hidden,
-                const Twine &Label);
-
-public:
-  VPlanPrinter(raw_ostream &O, const VPlan &P)
-      : OS(O), Plan(P), SlotTracker(&P) {}
-
-  LLVM_DUMP_METHOD void dump();
-};
-
 struct VPlanIngredient {
   const Value *V;
 
@@ -4198,139 +3825,6 @@ inline raw_ostream &operator<<(raw_ostream &OS, const VPlan &Plan) {
 }
 #endif
 
-class VPInterleavedAccessInfo {
-  DenseMap<VPInstruction *, InterleaveGroup<VPInstruction> *>
-      InterleaveGroupMap;
-
-  /// Type for mapping of instruction based interleave groups to VPInstruction
-  /// interleave groups
-  using Old2NewTy = DenseMap<InterleaveGroup<Instruction> *,
-                             InterleaveGroup<VPInstruction> *>;
-
-  /// Recursively \p Region and populate VPlan based interleave groups based on
-  /// \p IAI.
-  void visitRegion(VPRegionBlock *Region, Old2NewTy &Old2New,
-                   InterleavedAccessInfo &IAI);
-  /// Recursively traverse \p Block and populate VPlan based interleave groups
-  /// based on \p IAI.
-  void visitBlock(VPBlockBase *Block, Old2NewTy &Old2New,
-                  InterleavedAccessInfo &IAI);
-
-public:
-  VPInterleavedAccessInfo(VPlan &Plan, InterleavedAccessInfo &IAI);
-
-  ~VPInterleavedAccessInfo() {
-    SmallPtrSet<InterleaveGroup<VPInstruction> *, 4> DelSet;
-    // Avoid releasing a pointer twice.
-    for (auto &I : InterleaveGroupMap)
-      DelSet.insert(I.second);
-    for (auto *Ptr : DelSet)
-      delete Ptr;
-  }
-
-  /// Get the interleave group that \p Instr belongs to.
-  ///
-  /// \returns nullptr if doesn't have such group.
-  InterleaveGroup<VPInstruction> *
-  getInterleaveGroup(VPInstruction *Instr) const {
-    return InterleaveGroupMap.lookup(Instr);
-  }
-};
-
-/// Class that maps (parts of) an existing VPlan to trees of combined
-/// VPInstructions.
-class VPlanSlp {
-  enum class OpMode { Failed, Load, Opcode };
-
-  /// A DenseMapInfo implementation for using SmallVector<VPValue *, 4> as
-  /// DenseMap keys.
-  struct BundleDenseMapInfo {
-    static SmallVector<VPValue *, 4> getEmptyKey() {
-      return {reinterpret_cast<VPValue *>(-1)};
-    }
-
-    static SmallVector<VPValue *, 4> getTombstoneKey() {
-      return {reinterpret_cast<VPValue *>(-2)};
-    }
-
-    static unsigned getHashValue(const SmallVector<VPValue *, 4> &V) {
-      return static_cast<unsigned>(hash_combine_range(V.begin(), V.end()));
-    }
-
-    static bool isEqual(const SmallVector<VPValue *, 4> &LHS,
-                        const SmallVector<VPValue *, 4> &RHS) {
-      return LHS == RHS;
-    }
-  };
-
-  /// Mapping of values in the original VPlan to a combined VPInstruction.
-  DenseMap<SmallVector<VPValue *, 4>, VPInstruction *, BundleDenseMapInfo>
-      BundleToCombined;
-
-  VPInterleavedAccessInfo &IAI;
-
-  /// Basic block to operate on. For now, only instructions in a single BB are
-  /// considered.
-  const VPBasicBlock &BB;
-
-  /// Indicates whether we managed to combine all visited instructions or not.
-  bool CompletelySLP = true;
-
-  /// Width of the widest combined bundle in bits.
-  unsigned WidestBundleBits = 0;
-
-  using MultiNodeOpTy =
-      typename std::pair<VPInstruction *, SmallVector<VPValue *, 4>>;
-
-  // Input operand bundles for the current multi node. Each multi node operand
-  // bundle contains values not matching the multi node's opcode. They will
-  // be reordered in reorderMultiNodeOps, once we completed building a
-  // multi node.
-  SmallVector<MultiNodeOpTy, 4> MultiNodeOps;
-
-  /// Indicates whether we are building a multi node currently.
-  bool MultiNodeActive = false;
-
-  /// Check if we can vectorize Operands together.
-  bool areVectorizable(ArrayRef<VPValue *> Operands) const;
-
-  /// Add combined instruction \p New for the bundle \p Operands.
-  void addCombined(ArrayRef<VPValue *> Operands, VPInstruction *New);
-
-  /// Indicate we hit a bundle we failed to combine. Returns nullptr for now.
-  VPInstruction *markFailed();
-
-  /// Reorder operands in the multi node to maximize sequential memory access
-  /// and commutative operations.
-  SmallVector<MultiNodeOpTy, 4> reorderMultiNodeOps();
-
-  /// Choose the best candidate to use for the lane after \p Last. The set of
-  /// candidates to choose from are values with an opcode matching \p Last's
-  /// or loads consecutive to \p Last.
-  std::pair<OpMode, VPValue *> getBest(OpMode Mode, VPValue *Last,
-                                       SmallPtrSetImpl<VPValue *> &Candidates,
-                                       VPInterleavedAccessInfo &IAI);
-
-#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
-  /// Print bundle \p Values to dbgs().
-  void dumpBundle(ArrayRef<VPValue *> Values);
-#endif
-
-public:
-  VPlanSlp(VPInterleavedAccessInfo &IAI, VPBasicBlock &BB) : IAI(IAI), BB(BB) {}
-
-  ~VPlanSlp() = default;
-
-  /// Tries to build an SLP tree rooted at \p Operands and returns a
-  /// VPInstruction combining \p Operands, if they can be combined.
-  VPInstruction *buildGraph(ArrayRef<VPValue *> Operands);
-
-  /// Return the width of the widest combined bundle in bits.
-  unsigned getWidestBundleBits() const { return WidestBundleBits; }
-
-  /// Return true if all visited instruction can be combined.
-  bool isCompletelySLP() const { return CompletelySLP; }
-};
 } // end namespace llvm
 
 #endif // LLVM_TRANSFORMS_VECTORIZE_VPLAN_H
diff --git a/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp b/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp
index 27357ff04b5f2..71fb6d42116cf 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp
@@ -78,6 +78,7 @@ Type *VPTypeAnalysis::inferScalarTypeForRecipe(const VPInstruction *R) {
   case VPInstruction::CanonicalIVIncrementForPart:
   case VPInstruction::AnyOf:
     return SetResultTyFromOp();
+  case VPInstruction::ExtractFirstActive:
   case VPInstruction::ExtractFromEnd: {
     Type *BaseTy = inferScalarType(R->getOperand(0));
     if (auto *VecTy = dyn_cast<VectorType>(BaseTy))
diff --git a/llvm/lib/Transforms/Vectorize/VPlanHCFGBuilder.cpp b/llvm/lib/Transforms/Vectorize/VPlanHCFGBuilder.cpp
index 0f3aa8d08e7b8..5a2e5d7cfee48 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanHCFGBuilder.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanHCFGBuilder.cpp
@@ -292,6 +292,11 @@ void PlainCFGBuilder::createVPInstructionsForVPBB(VPBasicBlock *VPBB,
            "Instruction shouldn't have been visited.");
 
     if (auto *Br = dyn_cast<BranchInst>(Inst)) {
+      if (TheLoop->getLoopLatch() == BB ||
+          any_of(successors(BB),
+                 [this](BasicBlock *Succ) { return !TheLoop->contains(Succ); }))
+        continue;
+
       // Conditional branch instruction are represented using BranchOnCond
       // recipes.
       if (Br->isConditional()) {
@@ -345,10 +350,25 @@ void PlainCFGBuilder::buildPlainCFG() {
   // new vector preheader); here we're interested in setting BB2VPBB to the
   // latter.
   BB2VPBB[ThePreheaderBB] = VectorPreheaderVPBB;
-  BasicBlock *LoopExitBB = TheLoop->getUniqueExitBlock();
   Loop2Region[LI->getLoopFor(TheLoop->getHeader())] = TheRegion;
-  assert(LoopExitBB && "Loops with multiple exits are not supported.");
-  BB2VPBB[LoopExitBB] = cast<VPBasicBlock>(TheRegion->getSingleSuccessor());
+  BasicBlock *ExitBB = TheLoop->getUniqueExitBlock();
+  if (!ExitBB) {
+    // If there is no unique exit block, we must exit via the latch. This exit
+    // is mapped to the middle block in the input plan.
+    BasicBlock *Latch = TheLoop->getLoopLatch();
+    auto *Br = cast<BranchInst>(Latch->getTerminator());
+    if (TheLoop->contains(Br->getSuccessor(0))) {
+      assert(!TheLoop->contains(Br->getSuccessor(1)) &&
+             "latch must exit the loop");
+      ExitBB = Br->getSuccessor(1);
+    } else {
+      assert(!TheLoop->contains(Br->getSuccessor(0)) &&
+             "latch must exit the loop");
+      ExitBB = Br->getSuccessor(0);
+    }
+  }
+  assert(ExitBB && "Must have a unique exit block or also exit via the latch.");
+  BB2VPBB[ExitBB] = cast<VPBasicBlock>(TheRegion->getSingleSuccessor());
 
   // The existing vector region's entry and exiting VPBBs correspond to the loop
   // header and latch.
@@ -356,11 +376,6 @@ void PlainCFGBuilder::buildPlainCFG() {
   VPBasicBlock *VectorLatchVPBB = TheRegion->getExitingBasicBlock();
   BB2VPBB[TheLoop->getHeader()] = VectorHeaderVPBB;
   VectorHeaderVPBB->clearSuccessors();
-  VectorLatchVPBB->clearPredecessors();
-  if (TheLoop->getHeader() != TheLoop->getLoopLatch())
-    BB2VPBB[TheLoop->getLoopLatch()] = VectorLatchVPBB;
-  else
-    TheRegion->setExiting(VectorHeaderVPBB);
 
   // 1. Scan the body of the loop in a topological order to visit each basic
   // block after having visited its predecessor basic blocks. Create a VPBB for
@@ -398,6 +413,13 @@ void PlainCFGBuilder::buildPlainCFG() {
         setRegionPredsFromBB(Region, BB);
     }
 
+    if (TheLoop->getLoopLatch() == BB) {
+      VPBB->setOneSuccessor(VectorLatchVPBB);
+      VectorLatchVPBB->clearPredecessors();
+      VectorLatchVPBB->setPredecessors({VPBB});
+      continue;
+    }
+
     // Set VPBB successors. We create empty VPBBs for successors if they don't
     // exist already. Recipes will be created when the successor is visited
     // during the RPO traversal.
@@ -416,21 +438,38 @@ void PlainCFGBuilder::buildPlainCFG() {
     // representing the condition bit in VPlan (which may be in another VPBB).
     assert(IRDef2VPValue.contains(BI->getCondition()) &&
            "Missing condition bit in IRDef2VPValue!");
-    VPBasicBlock *Successor0 = getOrCreateVPBB(BI->getSuccessor(0));
-    VPBasicBlock *Successor1 = getOrCreateVPBB(BI->getSuccessor(1));
-    if (!LoopForBB || BB != LoopForBB->getLoopLatch()) {
-      VPBB->setTwoSuccessors(Successor0, Successor1);
-      continue;
-    }
-    // For a latch we need to set the successor of the region rather than that
-    // of VPBB and it should be set to the exit, i.e., non-header successor,
-    // except for the top region, whose successor was set when creating VPlan's
-    // skeleton.
-    if (TheRegion != Region) {
+
+    BasicBlock *IRSucc0 = BI->getSuccessor(0);
+    BasicBlock *IRSucc1 = BI->getSuccessor(1);
+    VPBasicBlock *Successor0 = getOrCreateVPBB(IRSucc0);
+    VPBasicBlock *Successor1 = getOrCreateVPBB(IRSucc1);
+    if (BB == LoopForBB->getLoopLatch()) {
+      // For a latch we need to set the successor of the region rather than that
+      // of VPBB and it should be set to the exit, i.e., non-header successor,
+      // except for the top region, whose successor was set when creating
+      // VPlan's skeleton.
+      assert(TheRegion != Region &&
+             "Latch of the top region should have been handled earlier");
       Region->setOneSuccessor(isHeaderVPBB(Successor0) ? Successor1
                                                        : Successor0);
       Region->setExiting(VPBB);
+      continue;
+    }
+
+    // Don't connect any blocks outside the current loop except the latch for
+    // now. The latch is handled above.
+    if (LoopForBB) {
+      if (!LoopForBB->contains(IRSucc0)) {
+        VPBB->setOneSuccessor(Successor1);
+        continue;
+      }
+      if (!LoopForBB->contains(IRSucc1)) {
+        VPBB->setOneSuccessor(Successor0);
+        continue;
+      }
     }
+
+    VPBB->setTwoSuccessors(Successor0, Successor1);
   }
 
   // 2. The whole CFG has been built at this point so all the input Values must
diff --git a/llvm/lib/Transforms/Vectorize/VPlanHelpers.h b/llvm/lib/Transforms/Vectorize/VPlanHelpers.h
new file mode 100644
index 0000000000000..74713daf904f0
--- /dev/null
+++ b/llvm/lib/Transforms/Vectorize/VPlanHelpers.h
@@ -0,0 +1,468 @@
+//===- VPlanHelpers.h - VPlan-related auxiliary helpers -------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+/// \file
+/// This file contains the declarations of different VPlan-related auxiliary
+/// helpers.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_TRANSFORMS_VECTORIZE_VPLANHELPERS_H
+#define LLVM_TRANSFORMS_VECTORIZE_VPLANHELPERS_H
+
+#include "VPlanAnalysis.h"
+#include "llvm/ADT/DenseMap.h"
+#include "llvm/ADT/SmallPtrSet.h"
+#include "llvm/ADT/SmallVector.h"
+#include "llvm/Analysis/DomTreeUpdater.h"
+#include "llvm/Analysis/TargetTransformInfo.h"
+#include "llvm/IR/DebugLoc.h"
+#include "llvm/Support/InstructionCost.h"
+
+namespace llvm {
+
+class BasicBlock;
+class DominatorTree;
+class InnerLoopVectorizer;
+class IRBuilderBase;
+class LoopInfo;
+class SCEV;
+class Type;
+class VPBasicBlock;
+class VPRegionBlock;
+class VPlan;
+class Value;
+class LoopVersioning;
+
+/// Returns a calculation for the total number of elements for a given \p VF.
+/// For fixed width vectors this value is a constant, whereas for scalable
+/// vectors it is an expression determined at runtime.
+Value *getRuntimeVF(IRBuilderBase &B, Type *Ty, ElementCount VF);
+
+/// Return a value for Step multiplied by VF.
+Value *createStepForVF(IRBuilderBase &B, Type *Ty, ElementCount VF,
+                       int64_t Step);
+
+/// A helper function that returns the reciprocal of the block probability of
+/// predicated blocks. If we return X, we are assuming the predicated block
+/// will execute once for every X iterations of the loop header.
+///
+/// TODO: We should use actual block probability here, if available. Currently,
+///       we always assume predicated blocks have a 50% chance of executing.
+inline unsigned getReciprocalPredBlockProb() { return 2; }
+
+/// A range of powers-of-2 vectorization factors with fixed start and
+/// adjustable end. The range includes start and excludes end, e.g.,:
+/// [1, 16) = {1, 2, 4, 8}
+struct VFRange {
+  // A power of 2.
+  const ElementCount Start;
+
+  // A power of 2. If End <= Start range is empty.
+  ElementCount End;
+
+  bool isEmpty() const {
+    return End.getKnownMinValue() <= Start.getKnownMinValue();
+  }
+
+  VFRange(const ElementCount &Start, const ElementCount &End)
+      : Start(Start), End(End) {
+    assert(Start.isScalable() == End.isScalable() &&
+           "Both Start and End should have the same scalable flag");
+    assert(isPowerOf2_32(Start.getKnownMinValue()) &&
+           "Expected Start to be a power of 2");
+    assert(isPowerOf2_32(End.getKnownMinValue()) &&
+           "Expected End to be a power of 2");
+  }
+
+  /// Iterator to iterate over vectorization factors in a VFRange.
+  class iterator
+      : public iterator_facade_base<iterator, std::forward_iterator_tag,
+                                    ElementCount> {
+    ElementCount VF;
+
+  public:
+    iterator(ElementCount VF) : VF(VF) {}
+
+    bool operator==(const iterator &Other) const { return VF == Other.VF; }
+
+    ElementCount operator*() const { return VF; }
+
+    iterator &operator++() {
+      VF *= 2;
+      return *this;
+    }
+  };
+
+  iterator begin() { return iterator(Start); }
+  iterator end() {
+    assert(isPowerOf2_32(End.getKnownMinValue()));
+    return iterator(End);
+  }
+};
+
+/// In what follows, the term "input IR" refers to code that is fed into the
+/// vectorizer whereas the term "output IR" refers to code that is generated by
+/// the vectorizer.
+
+/// VPLane provides a way to access lanes in both fixed width and scalable
+/// vectors, where for the latter the lane index sometimes needs calculating
+/// as a runtime expression.
+class VPLane {
+public:
+  /// Kind describes how to interpret Lane.
+  enum class Kind : uint8_t {
+    /// For First, Lane is the index into the first N elements of a
+    /// fixed-vector <N x <ElTy>> or a scalable vector <vscale x N x <ElTy>>.
+    First,
+    /// For ScalableLast, Lane is the offset from the start of the last
+    /// N-element subvector in a scalable vector <vscale x N x <ElTy>>. For
+    /// example, a Lane of 0 corresponds to lane `(vscale - 1) * N`, a Lane of
+    /// 1 corresponds to `((vscale - 1) * N) + 1`, etc.
+    ScalableLast
+  };
+
+private:
+  /// in [0..VF)
+  unsigned Lane;
+
+  /// Indicates how the Lane should be interpreted, as described above.
+  Kind LaneKind = Kind::First;
+
+public:
+  VPLane(unsigned Lane) : Lane(Lane) {}
+  VPLane(unsigned Lane, Kind LaneKind) : Lane(Lane), LaneKind(LaneKind) {}
+
+  static VPLane getFirstLane() { return VPLane(0, VPLane::Kind::First); }
+
+  static VPLane getLaneFromEnd(const ElementCount &VF, unsigned Offset) {
+    assert(Offset > 0 && Offset <= VF.getKnownMinValue() &&
+           "trying to extract with invalid offset");
+    unsigned LaneOffset = VF.getKnownMinValue() - Offset;
+    Kind LaneKind;
+    if (VF.isScalable())
+      // In this case 'LaneOffset' refers to the offset from the start of the
+      // last subvector with VF.getKnownMinValue() elements.
+      LaneKind = VPLane::Kind::ScalableLast;
+    else
+      LaneKind = VPLane::Kind::First;
+    return VPLane(LaneOffset, LaneKind);
+  }
+
+  static VPLane getLastLaneForVF(const ElementCount &VF) {
+    return getLaneFromEnd(VF, 1);
+  }
+
+  /// Returns a compile-time known value for the lane index and asserts if the
+  /// lane can only be calculated at runtime.
+  unsigned getKnownLane() const {
+    assert(LaneKind == Kind::First &&
+           "can only get known lane from the beginning");
+    return Lane;
+  }
+
+  /// Returns an expression describing the lane index that can be used at
+  /// runtime.
+  Value *getAsRuntimeExpr(IRBuilderBase &Builder, const ElementCount &VF) const;
+
+  /// Returns the Kind of lane offset.
+  Kind getKind() const { return LaneKind; }
+
+  /// Returns true if this is the first lane of the whole vector.
+  bool isFirstLane() const { return Lane == 0 && LaneKind == Kind::First; }
+
+  /// Maps the lane to a cache index based on \p VF.
+  unsigned mapToCacheIndex(const ElementCount &VF) const {
+    switch (LaneKind) {
+    case VPLane::Kind::ScalableLast:
+      assert(VF.isScalable() && Lane < VF.getKnownMinValue() &&
+             "ScalableLast can only be used with scalable VFs");
+      return VF.getKnownMinValue() + Lane;
+    default:
+      assert(Lane < VF.getKnownMinValue() &&
+             "Cannot extract lane larger than VF");
+      return Lane;
+    }
+  }
+};
+
+/// VPTransformState holds information passed down when "executing" a VPlan,
+/// needed for generating the output IR.
+struct VPTransformState {
+  VPTransformState(const TargetTransformInfo *TTI, ElementCount VF, unsigned UF,
+                   LoopInfo *LI, DominatorTree *DT, IRBuilderBase &Builder,
+                   InnerLoopVectorizer *ILV, VPlan *Plan,
+                   Loop *CurrentParentLoop, Type *CanonicalIVTy);
+  /// Target Transform Info.
+  const TargetTransformInfo *TTI;
+
+  /// The chosen Vectorization Factor of the loop being vectorized.
+  ElementCount VF;
+
+  /// Hold the index to generate specific scalar instructions. Null indicates
+  /// that all instances are to be generated, using either scalar or vector
+  /// instructions.
+  std::optional<VPLane> Lane;
+
+  struct DataState {
+    // Each value from the original loop, when vectorized, is represented by a
+    // vector value in the map.
+    DenseMap<VPValue *, Value *> VPV2Vector;
+
+    DenseMap<VPValue *, SmallVector<Value *, 4>> VPV2Scalars;
+  } Data;
+
+  /// Get the generated vector Value for a given VPValue \p Def if \p IsScalar
+  /// is false, otherwise return the generated scalar. \See set.
+  Value *get(VPValue *Def, bool IsScalar = false);
+
+  /// Get the generated Value for a given VPValue and given Part and Lane.
+  Value *get(VPValue *Def, const VPLane &Lane);
+
+  bool hasVectorValue(VPValue *Def) { return Data.VPV2Vector.contains(Def); }
+
+  bool hasScalarValue(VPValue *Def, VPLane Lane) {
+    auto I = Data.VPV2Scalars.find(Def);
+    if (I == Data.VPV2Scalars.end())
+      return false;
+    unsigned CacheIdx = Lane.mapToCacheIndex(VF);
+    return CacheIdx < I->second.size() && I->second[CacheIdx];
+  }
+
+  /// Set the generated vector Value for a given VPValue, if \p
+  /// IsScalar is false. If \p IsScalar is true, set the scalar in lane 0.
+  void set(VPValue *Def, Value *V, bool IsScalar = false) {
+    if (IsScalar) {
+      set(Def, V, VPLane(0));
+      return;
+    }
+    assert((VF.isScalar() || V->getType()->isVectorTy()) &&
+           "scalar values must be stored as (0, 0)");
+    Data.VPV2Vector[Def] = V;
+  }
+
+  /// Reset an existing vector value for \p Def and a given \p Part.
+  void reset(VPValue *Def, Value *V) {
+    assert(Data.VPV2Vector.contains(Def) && "need to overwrite existing value");
+    Data.VPV2Vector[Def] = V;
+  }
+
+  /// Set the generated scalar \p V for \p Def and the given \p Lane.
+  void set(VPValue *Def, Value *V, const VPLane &Lane) {
+    auto &Scalars = Data.VPV2Scalars[Def];
+    unsigned CacheIdx = Lane.mapToCacheIndex(VF);
+    if (Scalars.size() <= CacheIdx)
+      Scalars.resize(CacheIdx + 1);
+    assert(!Scalars[CacheIdx] && "should overwrite existing value");
+    Scalars[CacheIdx] = V;
+  }
+
+  /// Reset an existing scalar value for \p Def and a given \p Lane.
+  void reset(VPValue *Def, Value *V, const VPLane &Lane) {
+    auto Iter = Data.VPV2Scalars.find(Def);
+    assert(Iter != Data.VPV2Scalars.end() &&
+           "need to overwrite existing value");
+    unsigned CacheIdx = Lane.mapToCacheIndex(VF);
+    assert(CacheIdx < Iter->second.size() &&
+           "need to overwrite existing value");
+    Iter->second[CacheIdx] = V;
+  }
+
+  /// Add additional metadata to \p To that was not present on \p Orig.
+  ///
+  /// Currently this is used to add the noalias annotations based on the
+  /// inserted memchecks.  Use this for instructions that are *cloned* into the
+  /// vector loop.
+  void addNewMetadata(Instruction *To, const Instruction *Orig);
+
+  /// Add metadata from one instruction to another.
+  ///
+  /// This includes both the original MDs from \p From and additional ones (\see
+  /// addNewMetadata).  Use this for *newly created* instructions in the vector
+  /// loop.
+  void addMetadata(Value *To, Instruction *From);
+
+  /// Set the debug location in the builder using the debug location \p DL.
+  void setDebugLocFrom(DebugLoc DL);
+
+  /// Construct the vector value of a scalarized value \p V one lane at a time.
+  void packScalarIntoVectorValue(VPValue *Def, const VPLane &Lane);
+
+  /// Hold state information used when constructing the CFG of the output IR,
+  /// traversing the VPBasicBlocks and generating corresponding IR BasicBlocks.
+  struct CFGState {
+    /// The previous VPBasicBlock visited. Initially set to null.
+    VPBasicBlock *PrevVPBB = nullptr;
+
+    /// The previous IR BasicBlock created or used. Initially set to the new
+    /// header BasicBlock.
+    BasicBlock *PrevBB = nullptr;
+
+    /// The last IR BasicBlock in the output IR. Set to the exit block of the
+    /// vector loop.
+    BasicBlock *ExitBB = nullptr;
+
+    /// A mapping of each VPBasicBlock to the corresponding BasicBlock. In case
+    /// of replication, maps the BasicBlock of the last replica created.
+    SmallDenseMap<VPBasicBlock *, BasicBlock *> VPBB2IRBB;
+
+    /// Updater for the DominatorTree.
+    DomTreeUpdater DTU;
+
+    CFGState(DominatorTree *DT)
+        : DTU(DT, DomTreeUpdater::UpdateStrategy::Lazy) {}
+
+    /// Returns the BasicBlock* mapped to the pre-header of the loop region
+    /// containing \p R.
+    BasicBlock *getPreheaderBBFor(VPRecipeBase *R);
+  } CFG;
+
+  /// Hold a pointer to LoopInfo to register new basic blocks in the loop.
+  LoopInfo *LI;
+
+  /// Hold a reference to the IRBuilder used to generate output IR code.
+  IRBuilderBase &Builder;
+
+  /// Hold a pointer to InnerLoopVectorizer to reuse its IR generation methods.
+  InnerLoopVectorizer *ILV;
+
+  /// Pointer to the VPlan code is generated for.
+  VPlan *Plan;
+
+  /// The parent loop object for the current scope, or nullptr.
+  Loop *CurrentParentLoop = nullptr;
+
+  /// LoopVersioning.  It's only set up (non-null) if memchecks were
+  /// used.
+  ///
+  /// This is currently only used to add no-alias metadata based on the
+  /// memchecks.  The actually versioning is performed manually.
+  LoopVersioning *LVer = nullptr;
+
+  /// Map SCEVs to their expanded values. Populated when executing
+  /// VPExpandSCEVRecipes.
+  DenseMap<const SCEV *, Value *> ExpandedSCEVs;
+
+  /// VPlan-based type analysis.
+  VPTypeAnalysis TypeAnalysis;
+};
+
+/// Struct to hold various analysis needed for cost computations.
+struct VPCostContext {
+  const TargetTransformInfo &TTI;
+  const TargetLibraryInfo &TLI;
+  VPTypeAnalysis Types;
+  LLVMContext &LLVMCtx;
+  LoopVectorizationCostModel &CM;
+  SmallPtrSet<Instruction *, 8> SkipCostComputation;
+  TargetTransformInfo::TargetCostKind CostKind;
+
+  VPCostContext(const TargetTransformInfo &TTI, const TargetLibraryInfo &TLI,
+                Type *CanIVTy, LoopVectorizationCostModel &CM,
+                TargetTransformInfo::TargetCostKind CostKind)
+      : TTI(TTI), TLI(TLI), Types(CanIVTy), LLVMCtx(CanIVTy->getContext()),
+        CM(CM), CostKind(CostKind) {}
+
+  /// Return the cost for \p UI with \p VF using the legacy cost model as
+  /// fallback until computing the cost of all recipes migrates to VPlan.
+  InstructionCost getLegacyCost(Instruction *UI, ElementCount VF) const;
+
+  /// Return true if the cost for \p UI shouldn't be computed, e.g. because it
+  /// has already been pre-computed.
+  bool skipCostComputation(Instruction *UI, bool IsVector) const;
+
+  /// Returns the OperandInfo for \p V, if it is a live-in.
+  TargetTransformInfo::OperandValueInfo getOperandInfo(VPValue *V) const;
+};
+
+/// This class can be used to assign names to VPValues. For VPValues without
+/// underlying value, assign consecutive numbers and use those as names (wrapped
+/// in vp<>). Otherwise, use the name from the underlying value (wrapped in
+/// ir<>), appending a .V version number if there are multiple uses of the same
+/// name. Allows querying names for VPValues for printing, similar to the
+/// ModuleSlotTracker for IR values.
+class VPSlotTracker {
+  /// Keep track of versioned names assigned to VPValues with underlying IR
+  /// values.
+  DenseMap<const VPValue *, std::string> VPValue2Name;
+  /// Keep track of the next number to use to version the base name.
+  StringMap<unsigned> BaseName2Version;
+
+  /// Number to assign to the next VPValue without underlying value.
+  unsigned NextSlot = 0;
+
+  void assignName(const VPValue *V);
+  void assignNames(const VPlan &Plan);
+  void assignNames(const VPBasicBlock *VPBB);
+
+public:
+  VPSlotTracker(const VPlan *Plan = nullptr) {
+    if (Plan)
+      assignNames(*Plan);
+  }
+
+  /// Returns the name assigned to \p V, if there is one, otherwise try to
+  /// construct one from the underlying value, if there's one; else return
+  /// <badref>.
+  std::string getOrCreateName(const VPValue *V) const;
+};
+
+#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
+/// VPlanPrinter prints a given VPlan to a given output stream. The printing is
+/// indented and follows the dot format.
+class VPlanPrinter {
+  raw_ostream &OS;
+  const VPlan &Plan;
+  unsigned Depth = 0;
+  unsigned TabWidth = 2;
+  std::string Indent;
+  unsigned BID = 0;
+  SmallDenseMap<const VPBlockBase *, unsigned> BlockID;
+
+  VPSlotTracker SlotTracker;
+
+  /// Handle indentation.
+  void bumpIndent(int b) { Indent = std::string((Depth += b) * TabWidth, ' '); }
+
+  /// Print a given \p Block of the Plan.
+  void dumpBlock(const VPBlockBase *Block);
+
+  /// Print the information related to the CFG edges going out of a given
+  /// \p Block, followed by printing the successor blocks themselves.
+  void dumpEdges(const VPBlockBase *Block);
+
+  /// Print a given \p BasicBlock, including its VPRecipes, followed by printing
+  /// its successor blocks.
+  void dumpBasicBlock(const VPBasicBlock *BasicBlock);
+
+  /// Print a given \p Region of the Plan.
+  void dumpRegion(const VPRegionBlock *Region);
+
+  unsigned getOrCreateBID(const VPBlockBase *Block) {
+    return BlockID.count(Block) ? BlockID[Block] : BlockID[Block] = BID++;
+  }
+
+  Twine getOrCreateName(const VPBlockBase *Block);
+
+  Twine getUID(const VPBlockBase *Block);
+
+  /// Print the information related to a CFG edge between two VPBlockBases.
+  void drawEdge(const VPBlockBase *From, const VPBlockBase *To, bool Hidden,
+                const Twine &Label);
+
+public:
+  VPlanPrinter(raw_ostream &O, const VPlan &P)
+      : OS(O), Plan(P), SlotTracker(&P) {}
+
+  LLVM_DUMP_METHOD void dump();
+};
+#endif
+
+} // end namespace llvm
+
+#endif // LLVM_TRANSFORMS_VECTORIZE_VPLAN_H
diff --git a/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h b/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h
index 4866426ad8848..ebc82c01467cf 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h
@@ -126,47 +126,6 @@ inline match_combine_or<LTy, RTy> m_CombineOr(const LTy &L, const RTy &R) {
 /// Match a VPValue, capturing it if we match.
 inline bind_ty<VPValue> m_VPValue(VPValue *&V) { return V; }
 
-namespace detail {
-
-/// A helper to match an opcode against multiple recipe types.
-template <unsigned Opcode, typename...> struct MatchRecipeAndOpcode {};
-
-template <unsigned Opcode, typename RecipeTy>
-struct MatchRecipeAndOpcode<Opcode, RecipeTy> {
-  static bool match(const VPRecipeBase *R) {
-    auto *DefR = dyn_cast<RecipeTy>(R);
-    // Check for recipes that do not have opcodes.
-    if constexpr (std::is_same<RecipeTy, VPScalarIVStepsRecipe>::value ||
-                  std::is_same<RecipeTy, VPCanonicalIVPHIRecipe>::value ||
-                  std::is_same<RecipeTy, VPWidenSelectRecipe>::value ||
-                  std::is_same<RecipeTy, VPDerivedIVRecipe>::value ||
-                  std::is_same<RecipeTy, VPWidenGEPRecipe>::value)
-      return DefR;
-    else
-      return DefR && DefR->getOpcode() == Opcode;
-  }
-};
-
-template <unsigned Opcode, typename RecipeTy, typename... RecipeTys>
-struct MatchRecipeAndOpcode<Opcode, RecipeTy, RecipeTys...> {
-  static bool match(const VPRecipeBase *R) {
-    return MatchRecipeAndOpcode<Opcode, RecipeTy>::match(R) ||
-           MatchRecipeAndOpcode<Opcode, RecipeTys...>::match(R);
-  }
-};
-template <typename TupleTy, typename Fn, std::size_t... Is>
-bool CheckTupleElements(const TupleTy &Ops, Fn P, std::index_sequence<Is...>) {
-  return (P(std::get<Is>(Ops), Is) && ...);
-}
-
-/// Helper to check if predicate \p P holds on all tuple elements in \p Ops
-template <typename TupleTy, typename Fn>
-bool all_of_tuple_elements(const TupleTy &Ops, Fn P) {
-  return CheckTupleElements(
-      Ops, P, std::make_index_sequence<std::tuple_size<TupleTy>::value>{});
-}
-} // namespace detail
-
 template <typename Ops_t, unsigned Opcode, bool Commutative,
           typename... RecipeTys>
 struct Recipe_match {
@@ -193,21 +152,53 @@ struct Recipe_match {
   }
 
   bool match(const VPRecipeBase *R) const {
-    if (!detail::MatchRecipeAndOpcode<Opcode, RecipeTys...>::match(R))
+    if ((!matchRecipeAndOpcode<RecipeTys>(R) && ...))
+      return false;
+
+    if (!(std::is_same_v<VPWidenEVLRecipe, RecipeTys> || ...) &&
+        isa<VPWidenEVLRecipe>(R)) {
+      // Don't match VPWidenEVLRecipe if it is not explicitly part of RecipeTys.
+      // Otherwise we might match it unexpectedly when trying to match
+      // VPWidenRecipe, of which VPWidenEVLRecipe is a subclass of.
       return false;
+    }
+
     assert(R->getNumOperands() == std::tuple_size<Ops_t>::value &&
            "recipe with matched opcode the expected number of operands");
 
-    if (detail::all_of_tuple_elements(Ops, [R](auto Op, unsigned Idx) {
+    auto IdxSeq = std::make_index_sequence<std::tuple_size<Ops_t>::value>();
+    if (all_of_tuple_elements(IdxSeq, [R](auto Op, unsigned Idx) {
           return Op.match(R->getOperand(Idx));
         }))
       return true;
 
     return Commutative &&
-           detail::all_of_tuple_elements(Ops, [R](auto Op, unsigned Idx) {
+           all_of_tuple_elements(IdxSeq, [R](auto Op, unsigned Idx) {
              return Op.match(R->getOperand(R->getNumOperands() - Idx - 1));
            });
   }
+
+private:
+  template <typename RecipeTy>
+  static bool matchRecipeAndOpcode(const VPRecipeBase *R) {
+    auto *DefR = dyn_cast<RecipeTy>(R);
+    // Check for recipes that do not have opcodes.
+    if constexpr (std::is_same<RecipeTy, VPScalarIVStepsRecipe>::value ||
+                  std::is_same<RecipeTy, VPCanonicalIVPHIRecipe>::value ||
+                  std::is_same<RecipeTy, VPWidenSelectRecipe>::value ||
+                  std::is_same<RecipeTy, VPDerivedIVRecipe>::value ||
+                  std::is_same<RecipeTy, VPWidenGEPRecipe>::value)
+      return DefR;
+    else
+      return DefR && DefR->getOpcode() == Opcode;
+  }
+
+  /// Helper to check if predicate \p P holds on all tuple elements in Ops using
+  /// the provided index sequence.
+  template <typename Fn, std::size_t... Is>
+  bool all_of_tuple_elements(std::index_sequence<Is...>, Fn P) const {
+    return (P(std::get<Is>(Ops), Is) && ...);
+  }
 };
 
 template <typename Op0_t, unsigned Opcode, typename... RecipeTys>
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index aa30eccdbd734..1855fb67aa54f 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -11,14 +11,17 @@
 ///
 //===----------------------------------------------------------------------===//
 
+#include "LoopVectorizationPlanner.h"
 #include "VPlan.h"
 #include "VPlanAnalysis.h"
+#include "VPlanHelpers.h"
 #include "VPlanPatternMatch.h"
 #include "VPlanUtils.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/Twine.h"
 #include "llvm/Analysis/IVDescriptors.h"
+#include "llvm/Analysis/LoopInfo.h"
 #include "llvm/IR/BasicBlock.h"
 #include "llvm/IR/IRBuilder.h"
 #include "llvm/IR/Instruction.h"
@@ -282,6 +285,13 @@ VPPartialReductionRecipe::computeCost(ElementCount VF,
                                       VPCostContext &Ctx) const {
   std::optional<unsigned> Opcode = std::nullopt;
   VPRecipeBase *BinOpR = getOperand(0)->getDefiningRecipe();
+
+  // If the partial reduction is predicated, a select will be operand 0 rather
+  // than the binary op
+  using namespace llvm::VPlanPatternMatch;
+  if (match(getOperand(0), m_Select(m_VPValue(), m_VPValue(), m_VPValue())))
+    BinOpR = BinOpR->getOperand(1)->getDefiningRecipe();
+
   if (auto *WidenR = dyn_cast<VPWidenRecipe>(BinOpR))
     Opcode = std::make_optional(WidenR->getOpcode());
 
@@ -675,20 +685,13 @@ Value *VPInstruction::generate(VPTransformState &State) {
     return Builder.CreatePtrAdd(Ptr, Addend, Name, getGEPNoWrapFlags());
   }
   case VPInstruction::ResumePhi: {
-    Value *IncomingFromVPlanPred =
-        State.get(getOperand(0), /* IsScalar */ true);
-    Value *IncomingFromOtherPreds =
-        State.get(getOperand(1), /* IsScalar */ true);
     auto *NewPhi =
         Builder.CreatePHI(State.TypeAnalysis.inferScalarType(this), 2, Name);
-    BasicBlock *VPlanPred =
-        State.CFG
-            .VPBB2IRBB[cast<VPBasicBlock>(getParent()->getPredecessors()[0])];
-    NewPhi->addIncoming(IncomingFromVPlanPred, VPlanPred);
-    for (auto *OtherPred : predecessors(Builder.GetInsertBlock())) {
-      if (OtherPred == VPlanPred)
-        continue;
-      NewPhi->addIncoming(IncomingFromOtherPreds, OtherPred);
+    for (const auto &[IncVPV, PredVPBB] :
+         zip(operands(), getParent()->getPredecessors())) {
+      Value *IncV = State.get(IncVPV, /* IsScalar */ true);
+      BasicBlock *PredBB = State.CFG.VPBB2IRBB.at(cast<VPBasicBlock>(PredVPBB));
+      NewPhi->addIncoming(IncV, PredBB);
     }
     return NewPhi;
   }
@@ -696,14 +699,55 @@ Value *VPInstruction::generate(VPTransformState &State) {
     Value *A = State.get(getOperand(0));
     return Builder.CreateOrReduce(A);
   }
-
+  case VPInstruction::ExtractFirstActive: {
+    Value *Vec = State.get(getOperand(0));
+    Value *Mask = State.get(getOperand(1));
+    Value *Ctz = Builder.CreateCountTrailingZeroElems(
+        Builder.getInt64Ty(), Mask, true, "first.active.lane");
+    return Builder.CreateExtractElement(Vec, Ctz, "early.exit.value");
+  }
   default:
     llvm_unreachable("Unsupported opcode for instruction");
   }
 }
 
+InstructionCost VPInstruction::computeCost(ElementCount VF,
+                                           VPCostContext &Ctx) const {
+  if (Instruction::isBinaryOp(getOpcode())) {
+    if (!getUnderlyingValue()) {
+      // TODO: Compute cost for VPInstructions without underlying values once
+      // the legacy cost model has been retired.
+      return 0;
+    }
+
+    assert(!doesGeneratePerAllLanes() &&
+           "Should only generate a vector value or single scalar, not scalars "
+           "for all lanes.");
+    Type *ResTy = Ctx.Types.inferScalarType(this);
+    if (!vputils::onlyFirstLaneUsed(this))
+      ResTy = toVectorTy(ResTy, VF);
+
+    return Ctx.TTI.getArithmeticInstrCost(getOpcode(), ResTy, Ctx.CostKind);
+  }
+
+  switch (getOpcode()) {
+  case VPInstruction::AnyOf: {
+    auto *VecTy = toVectorTy(Ctx.Types.inferScalarType(this), VF);
+    return Ctx.TTI.getArithmeticReductionCost(
+        Instruction::Or, cast<VectorType>(VecTy), std::nullopt, Ctx.CostKind);
+  }
+  default:
+    // TODO: Compute cost other VPInstructions once the legacy cost model has
+    // been retired.
+    assert(!getUnderlyingValue() &&
+           "unexpected VPInstruction witht underlying value");
+    return 0;
+  }
+}
+
 bool VPInstruction::isVectorToScalar() const {
   return getOpcode() == VPInstruction::ExtractFromEnd ||
+         getOpcode() == VPInstruction::ExtractFirstActive ||
          getOpcode() == VPInstruction::ComputeReductionResult ||
          getOpcode() == VPInstruction::AnyOf;
 }
@@ -768,6 +812,7 @@ bool VPInstruction::opcodeMayReadOrWriteFromMemory() const {
   case VPInstruction::CalculateTripCountMinusVF:
   case VPInstruction::CanonicalIVIncrementForPart:
   case VPInstruction::ExtractFromEnd:
+  case VPInstruction::ExtractFirstActive:
   case VPInstruction::FirstOrderRecurrenceSplice:
   case VPInstruction::LogicalAnd:
   case VPInstruction::Not:
@@ -887,6 +932,9 @@ void VPInstruction::print(raw_ostream &O, const Twine &Indent,
   case VPInstruction::AnyOf:
     O << "any-of";
     break;
+  case VPInstruction::ExtractFirstActive:
+    O << "extract-first-active";
+    break;
   default:
     O << Instruction::getOpcodeName(getOpcode());
   }
@@ -937,6 +985,22 @@ InstructionCost VPIRInstruction::computeCost(ElementCount VF,
   return 0;
 }
 
+void VPIRInstruction::extractLastLaneOfOperand(VPBuilder &Builder) {
+  assert(isa<PHINode>(getInstruction()) &&
+         "can only add exiting operands to phi nodes");
+  assert(getNumOperands() == 1 && "must have a single operand");
+  VPValue *Exiting = getOperand(0);
+  if (!Exiting->isLiveIn()) {
+    LLVMContext &Ctx = getInstruction().getContext();
+    auto &Plan = *getParent()->getPlan();
+    Exiting = Builder.createNaryOp(
+        VPInstruction::ExtractFromEnd,
+        {Exiting,
+         Plan.getOrAddLiveIn(ConstantInt::get(IntegerType::get(Ctx, 32), 1))});
+  }
+  setOperand(0, Exiting);
+}
+
 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
 void VPIRInstruction::print(raw_ostream &O, const Twine &Indent,
                             VPSlotTracker &SlotTracker) const {
@@ -2427,16 +2491,12 @@ void VPScalarCastRecipe ::print(raw_ostream &O, const Twine &Indent,
 void VPBranchOnMaskRecipe::execute(VPTransformState &State) {
   assert(State.Lane && "Branch on Mask works only on single instance.");
 
-  unsigned Lane = State.Lane->getKnownLane();
 
   Value *ConditionBit = nullptr;
   VPValue *BlockInMask = getMask();
-  if (BlockInMask) {
-    ConditionBit = State.get(BlockInMask);
-    if (ConditionBit->getType()->isVectorTy())
-      ConditionBit = State.Builder.CreateExtractElement(
-          ConditionBit, State.Builder.getInt32(Lane));
-  } else // Block in mask is all-one.
+  if (BlockInMask)
+    ConditionBit = State.get(BlockInMask, *State.Lane);
+  else // Block in mask is all-one.
     ConditionBit = State.Builder.getTrue();
 
   // Replace the temporary unreachable terminator with a new conditional branch,
@@ -2855,10 +2915,21 @@ static Value *interleaveVectors(IRBuilderBase &Builder, ArrayRef<Value *> Vals,
   // Scalable vectors cannot use arbitrary shufflevectors (only splats), so
   // must use intrinsics to interleave.
   if (VecTy->isScalableTy()) {
-    VectorType *WideVecTy = VectorType::getDoubleElementsVectorType(VecTy);
-    return Builder.CreateIntrinsic(WideVecTy, Intrinsic::vector_interleave2,
-                                   Vals,
-                                   /*FMFSource=*/nullptr, Name);
+    assert(isPowerOf2_32(Factor) && "Unsupported interleave factor for "
+                                    "scalable vectors, must be power of 2");
+    SmallVector<Value *> InterleavingValues(Vals);
+    // When interleaving, the number of values will be shrunk until we have the
+    // single final interleaved value.
+    auto *InterleaveTy = cast<VectorType>(InterleavingValues[0]->getType());
+    for (unsigned Midpoint = Factor / 2; Midpoint > 0; Midpoint /= 2) {
+      InterleaveTy = VectorType::getDoubleElementsVectorType(InterleaveTy);
+      for (unsigned I = 0; I < Midpoint; ++I)
+        InterleavingValues[I] = Builder.CreateIntrinsic(
+            InterleaveTy, Intrinsic::vector_interleave2,
+            {InterleavingValues[I], InterleavingValues[Midpoint + I]},
+            /*FMFSource=*/nullptr, Name);
+    }
+    return InterleavingValues[0];
   }
 
   // Fixed length. Start by concatenating all vectors into a wide vector.
@@ -2944,15 +3015,11 @@ void VPInterleaveRecipe::execute(VPTransformState &State) {
                           &InterleaveFactor](Value *MaskForGaps) -> Value * {
     if (State.VF.isScalable()) {
       assert(!MaskForGaps && "Interleaved groups with gaps are not supported.");
-      assert(InterleaveFactor == 2 &&
+      assert(isPowerOf2_32(InterleaveFactor) &&
              "Unsupported deinterleave factor for scalable vectors");
       auto *ResBlockInMask = State.get(BlockInMask);
-      SmallVector<Value *, 2> Ops = {ResBlockInMask, ResBlockInMask};
-      auto *MaskTy = VectorType::get(State.Builder.getInt1Ty(),
-                                     State.VF.getKnownMinValue() * 2, true);
-      return State.Builder.CreateIntrinsic(
-          MaskTy, Intrinsic::vector_interleave2, Ops,
-          /*FMFSource=*/nullptr, "interleaved.mask");
+      SmallVector<Value *> Ops(InterleaveFactor, ResBlockInMask);
+      return interleaveVectors(State.Builder, Ops, "interleaved.mask");
     }
 
     if (!BlockInMask)
@@ -2992,22 +3059,48 @@ void VPInterleaveRecipe::execute(VPTransformState &State) {
     ArrayRef<VPValue *> VPDefs = definedValues();
     const DataLayout &DL = State.CFG.PrevBB->getDataLayout();
     if (VecTy->isScalableTy()) {
-      assert(InterleaveFactor == 2 &&
+      assert(isPowerOf2_32(InterleaveFactor) &&
              "Unsupported deinterleave factor for scalable vectors");
 
-        // Scalable vectors cannot use arbitrary shufflevectors (only splats),
-        // so must use intrinsics to deinterleave.
-      Value *DI = State.Builder.CreateIntrinsic(
-          Intrinsic::vector_deinterleave2, VecTy, NewLoad,
-          /*FMFSource=*/nullptr, "strided.vec");
-      unsigned J = 0;
-      for (unsigned I = 0; I < InterleaveFactor; ++I) {
-        Instruction *Member = Group->getMember(I);
+      // Scalable vectors cannot use arbitrary shufflevectors (only splats),
+      // so must use intrinsics to deinterleave.
+      SmallVector<Value *> DeinterleavedValues(InterleaveFactor);
+      DeinterleavedValues[0] = NewLoad;
+      // For the case of InterleaveFactor > 2, we will have to do recursive
+      // deinterleaving, because the current available deinterleave intrinsic
+      // supports only Factor of 2, otherwise it will bailout after first
+      // iteration.
+      // When deinterleaving, the number of values will double until we
+      // have "InterleaveFactor".
+      for (unsigned NumVectors = 1; NumVectors < InterleaveFactor;
+           NumVectors *= 2) {
+        // Deinterleave the elements within the vector
+        SmallVector<Value *> TempDeinterleavedValues(NumVectors);
+        for (unsigned I = 0; I < NumVectors; ++I) {
+          auto *DiTy = DeinterleavedValues[I]->getType();
+          TempDeinterleavedValues[I] = State.Builder.CreateIntrinsic(
+              Intrinsic::vector_deinterleave2, DiTy, DeinterleavedValues[I],
+              /*FMFSource=*/nullptr, "strided.vec");
+        }
+        // Extract the deinterleaved values:
+        for (unsigned I = 0; I < 2; ++I)
+          for (unsigned J = 0; J < NumVectors; ++J)
+            DeinterleavedValues[NumVectors * I + J] =
+                State.Builder.CreateExtractValue(TempDeinterleavedValues[J], I);
+      }
 
-        if (!Member)
+#ifndef NDEBUG
+      for (Value *Val : DeinterleavedValues)
+        assert(Val && "NULL Deinterleaved Value");
+#endif
+      for (unsigned I = 0, J = 0; I < InterleaveFactor; ++I) {
+        Instruction *Member = Group->getMember(I);
+        Value *StridedVec = DeinterleavedValues[I];
+        if (!Member) {
+          // This value is not needed as it's not used
+          cast<Instruction>(StridedVec)->eraseFromParent();
           continue;
-
-        Value *StridedVec = State.Builder.CreateExtractValue(DI, I);
+        }
         // If this member has different type, cast the result type.
         if (Member->getType() != ScalarTy) {
           VectorType *OtherVTy = VectorType::get(Member->getType(), State.VF);
@@ -3311,7 +3404,7 @@ void VPExpandSCEVRecipe::execute(VPTransformState &State) {
   }
 
   const DataLayout &DL = State.CFG.PrevBB->getDataLayout();
-  SCEVExpander Exp(SE, DL, "induction");
+  SCEVExpander Exp(SE, DL, "induction", /*PreserveLCSSA=*/true);
 
   Value *Res = Exp.expandCodeFor(Expr, Expr->getType(),
                                  &*State.Builder.GetInsertPoint());
diff --git a/llvm/lib/Transforms/Vectorize/VPlanSLP.cpp b/llvm/lib/Transforms/Vectorize/VPlanSLP.cpp
index 98ccf21694635..e943c7a29eb83 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanSLP.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanSLP.cpp
@@ -14,10 +14,13 @@
 ///
 //===----------------------------------------------------------------------===//
 
+#include "VPlanSLP.h"
 #include "VPlan.h"
+#include "VPlanCFG.h"
 #include "VPlanValue.h"
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/SmallVector.h"
+#include "llvm/Analysis/LoopInfo.h"
 #include "llvm/Analysis/VectorUtils.h"
 #include "llvm/IR/Instruction.h"
 #include "llvm/IR/Instructions.h"
@@ -39,6 +42,57 @@ using namespace llvm;
 // Number of levels to look ahead when re-ordering multi node operands.
 static unsigned LookaheadMaxDepth = 5;
 
+void VPInterleavedAccessInfo::visitRegion(VPRegionBlock *Region,
+                                          Old2NewTy &Old2New,
+                                          InterleavedAccessInfo &IAI) {
+  ReversePostOrderTraversal<VPBlockShallowTraversalWrapper<VPBlockBase *>> RPOT(
+      Region->getEntry());
+  for (VPBlockBase *Base : RPOT) {
+    visitBlock(Base, Old2New, IAI);
+  }
+}
+
+void VPInterleavedAccessInfo::visitBlock(VPBlockBase *Block, Old2NewTy &Old2New,
+                                         InterleavedAccessInfo &IAI) {
+  if (VPBasicBlock *VPBB = dyn_cast<VPBasicBlock>(Block)) {
+    for (VPRecipeBase &VPI : *VPBB) {
+      if (isa<VPWidenPHIRecipe>(&VPI))
+        continue;
+      auto *VPInst = cast<VPInstruction>(&VPI);
+      auto *Inst = dyn_cast_or_null<Instruction>(VPInst->getUnderlyingValue());
+      if (!Inst)
+        continue;
+      auto *IG = IAI.getInterleaveGroup(Inst);
+      if (!IG)
+        continue;
+
+      auto NewIGIter = Old2New.find(IG);
+      if (NewIGIter == Old2New.end())
+        Old2New[IG] = new InterleaveGroup<VPInstruction>(
+            IG->getFactor(), IG->isReverse(), IG->getAlign());
+
+      if (Inst == IG->getInsertPos())
+        Old2New[IG]->setInsertPos(VPInst);
+
+      InterleaveGroupMap[VPInst] = Old2New[IG];
+      InterleaveGroupMap[VPInst]->insertMember(
+          VPInst, IG->getIndex(Inst),
+          Align(IG->isReverse() ? (-1) * int(IG->getFactor())
+                                : IG->getFactor()));
+    }
+  } else if (VPRegionBlock *Region = dyn_cast<VPRegionBlock>(Block)) {
+    visitRegion(Region, Old2New, IAI);
+  } else {
+    llvm_unreachable("Unsupported kind of VPBlock.");
+  }
+}
+
+VPInterleavedAccessInfo::VPInterleavedAccessInfo(VPlan &Plan,
+                                                 InterleavedAccessInfo &IAI) {
+  Old2NewTy Old2New;
+  visitRegion(Plan.getVectorLoopRegion(), Old2New, IAI);
+}
+
 VPInstruction *VPlanSlp::markFailed() {
   // FIXME: Currently this is used to signal we hit instructions we cannot
   //        trivially SLP'ize.
diff --git a/llvm/lib/Transforms/Vectorize/VPlanSLP.h b/llvm/lib/Transforms/Vectorize/VPlanSLP.h
new file mode 100644
index 0000000000000..a40ebd28deea2
--- /dev/null
+++ b/llvm/lib/Transforms/Vectorize/VPlanSLP.h
@@ -0,0 +1,166 @@
+//===- VPlan.h - VPlan-based SLP ------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+/// \file
+/// This file contains the declarations for VPlan-based SLP.
+///
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_TRANSFORMS_VECTORIZE_VPLANSLP_H
+#define LLVM_TRANSFORMS_VECTORIZE_VPLANSLP_H
+
+#include "llvm/ADT/DenseMap.h"
+#include "llvm/ADT/SmallPtrSet.h"
+#include "llvm/ADT/SmallVector.h"
+#include "llvm/Analysis/VectorUtils.h"
+
+namespace llvm {
+
+class VPBasicBlock;
+class VPBlockBase;
+class VPRegionBlock;
+class VPlan;
+class VPValue;
+class VPInstruction;
+
+class VPInterleavedAccessInfo {
+  DenseMap<VPInstruction *, InterleaveGroup<VPInstruction> *>
+      InterleaveGroupMap;
+
+  /// Type for mapping of instruction based interleave groups to VPInstruction
+  /// interleave groups
+  using Old2NewTy = DenseMap<InterleaveGroup<Instruction> *,
+                             InterleaveGroup<VPInstruction> *>;
+
+  /// Recursively \p Region and populate VPlan based interleave groups based on
+  /// \p IAI.
+  void visitRegion(VPRegionBlock *Region, Old2NewTy &Old2New,
+                   InterleavedAccessInfo &IAI);
+  /// Recursively traverse \p Block and populate VPlan based interleave groups
+  /// based on \p IAI.
+  void visitBlock(VPBlockBase *Block, Old2NewTy &Old2New,
+                  InterleavedAccessInfo &IAI);
+
+public:
+  VPInterleavedAccessInfo(VPlan &Plan, InterleavedAccessInfo &IAI);
+
+  ~VPInterleavedAccessInfo() {
+    SmallPtrSet<InterleaveGroup<VPInstruction> *, 4> DelSet;
+    // Avoid releasing a pointer twice.
+    for (auto &I : InterleaveGroupMap)
+      DelSet.insert(I.second);
+    for (auto *Ptr : DelSet)
+      delete Ptr;
+  }
+
+  /// Get the interleave group that \p Instr belongs to.
+  ///
+  /// \returns nullptr if doesn't have such group.
+  InterleaveGroup<VPInstruction> *
+  getInterleaveGroup(VPInstruction *Instr) const {
+    return InterleaveGroupMap.lookup(Instr);
+  }
+};
+
+/// Class that maps (parts of) an existing VPlan to trees of combined
+/// VPInstructions.
+class VPlanSlp {
+  enum class OpMode { Failed, Load, Opcode };
+
+  /// A DenseMapInfo implementation for using SmallVector<VPValue *, 4> as
+  /// DenseMap keys.
+  struct BundleDenseMapInfo {
+    static SmallVector<VPValue *, 4> getEmptyKey() {
+      return {reinterpret_cast<VPValue *>(-1)};
+    }
+
+    static SmallVector<VPValue *, 4> getTombstoneKey() {
+      return {reinterpret_cast<VPValue *>(-2)};
+    }
+
+    static unsigned getHashValue(const SmallVector<VPValue *, 4> &V) {
+      return static_cast<unsigned>(hash_combine_range(V.begin(), V.end()));
+    }
+
+    static bool isEqual(const SmallVector<VPValue *, 4> &LHS,
+                        const SmallVector<VPValue *, 4> &RHS) {
+      return LHS == RHS;
+    }
+  };
+
+  /// Mapping of values in the original VPlan to a combined VPInstruction.
+  DenseMap<SmallVector<VPValue *, 4>, VPInstruction *, BundleDenseMapInfo>
+      BundleToCombined;
+
+  VPInterleavedAccessInfo &IAI;
+
+  /// Basic block to operate on. For now, only instructions in a single BB are
+  /// considered.
+  const VPBasicBlock &BB;
+
+  /// Indicates whether we managed to combine all visited instructions or not.
+  bool CompletelySLP = true;
+
+  /// Width of the widest combined bundle in bits.
+  unsigned WidestBundleBits = 0;
+
+  using MultiNodeOpTy =
+      typename std::pair<VPInstruction *, SmallVector<VPValue *, 4>>;
+
+  // Input operand bundles for the current multi node. Each multi node operand
+  // bundle contains values not matching the multi node's opcode. They will
+  // be reordered in reorderMultiNodeOps, once we completed building a
+  // multi node.
+  SmallVector<MultiNodeOpTy, 4> MultiNodeOps;
+
+  /// Indicates whether we are building a multi node currently.
+  bool MultiNodeActive = false;
+
+  /// Check if we can vectorize Operands together.
+  bool areVectorizable(ArrayRef<VPValue *> Operands) const;
+
+  /// Add combined instruction \p New for the bundle \p Operands.
+  void addCombined(ArrayRef<VPValue *> Operands, VPInstruction *New);
+
+  /// Indicate we hit a bundle we failed to combine. Returns nullptr for now.
+  VPInstruction *markFailed();
+
+  /// Reorder operands in the multi node to maximize sequential memory access
+  /// and commutative operations.
+  SmallVector<MultiNodeOpTy, 4> reorderMultiNodeOps();
+
+  /// Choose the best candidate to use for the lane after \p Last. The set of
+  /// candidates to choose from are values with an opcode matching \p Last's
+  /// or loads consecutive to \p Last.
+  std::pair<OpMode, VPValue *> getBest(OpMode Mode, VPValue *Last,
+                                       SmallPtrSetImpl<VPValue *> &Candidates,
+                                       VPInterleavedAccessInfo &IAI);
+
+#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
+  /// Print bundle \p Values to dbgs().
+  void dumpBundle(ArrayRef<VPValue *> Values);
+#endif
+
+public:
+  VPlanSlp(VPInterleavedAccessInfo &IAI, VPBasicBlock &BB) : IAI(IAI), BB(BB) {}
+
+  ~VPlanSlp() = default;
+
+  /// Tries to build an SLP tree rooted at \p Operands and returns a
+  /// VPInstruction combining \p Operands, if they can be combined.
+  VPInstruction *buildGraph(ArrayRef<VPValue *> Operands);
+
+  /// Return the width of the widest combined bundle in bits.
+  unsigned getWidestBundleBits() const { return WidestBundleBits; }
+
+  /// Return true if all visited instruction can be combined.
+  bool isCompletelySLP() const { return CompletelySLP; }
+};
+} // end namespace llvm
+
+#endif // LLVM_TRANSFORMS_VECTORIZE_VPLAN_H
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 9febd612c644e..6c917e4eef655 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -19,11 +19,13 @@
 #include "VPlanDominatorTree.h"
 #include "VPlanPatternMatch.h"
 #include "VPlanUtils.h"
+#include "VPlanVerifier.h"
 #include "llvm/ADT/PostOrderIterator.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/SetVector.h"
 #include "llvm/ADT/TypeSwitch.h"
 #include "llvm/Analysis/IVDescriptors.h"
+#include "llvm/Analysis/LoopInfo.h"
 #include "llvm/Analysis/VectorUtils.h"
 #include "llvm/IR/Intrinsics.h"
 #include "llvm/IR/PatternMatch.h"
@@ -589,7 +591,7 @@ static SmallVector<VPUser *> collectUsersRecursively(VPValue *V) {
 static void legalizeAndOptimizeInductions(VPlan &Plan) {
   using namespace llvm::VPlanPatternMatch;
   VPBasicBlock *HeaderVPBB = Plan.getVectorLoopRegion()->getEntryBasicBlock();
-  bool HasOnlyVectorVFs = !Plan.hasVF(ElementCount::getFixed(1));
+  bool HasOnlyVectorVFs = !Plan.hasScalarVFOnly();
   VPBuilder Builder(HeaderVPBB, HeaderVPBB->getFirstNonPhi());
   for (VPRecipeBase &Phi : HeaderVPBB->phis()) {
     auto *PhiR = dyn_cast<VPWidenInductionRecipe>(&Phi);
@@ -962,12 +964,10 @@ static void simplifyRecipe(VPRecipeBase &R, VPTypeAnalysis &TypeInfo) {
     return R.getVPSingleValue()->replaceAllUsesWith(R.getOperand(1));
 }
 
-/// Try to simplify the recipes in \p Plan. Use \p CanonicalIVTy as type for all
-/// un-typed live-ins in VPTypeAnalysis.
-static void simplifyRecipes(VPlan &Plan, Type *CanonicalIVTy) {
+void VPlanTransforms::simplifyRecipes(VPlan &Plan, Type &CanonicalIVTy) {
   ReversePostOrderTraversal<VPBlockDeepTraversalWrapper<VPBlockBase *>> RPOT(
       Plan.getEntry());
-  VPTypeAnalysis TypeInfo(CanonicalIVTy);
+  VPTypeAnalysis TypeInfo(&CanonicalIVTy);
   for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(RPOT)) {
     for (VPRecipeBase &R : make_early_inc_range(*VPBB)) {
       simplifyRecipe(R, TypeInfo);
@@ -1029,7 +1029,7 @@ void VPlanTransforms::optimizeForVFAndUF(VPlan &Plan, ElementCount BestVF,
 
     VPBlockUtils::connectBlocks(Preheader, Header);
     VPBlockUtils::connectBlocks(ExitingVPBB, Exit);
-    simplifyRecipes(Plan, CanIVTy);
+    simplifyRecipes(Plan, *CanIVTy);
   } else {
     // The vector region contains header phis for which we cannot remove the
     // loop region yet.
@@ -1041,7 +1041,6 @@ void VPlanTransforms::optimizeForVFAndUF(VPlan &Plan, ElementCount BestVF,
   }
 
   Term->eraseFromParent();
-  VPlanTransforms::removeDeadRecipes(Plan);
 
   Plan.setVF(BestVF);
   Plan.setUF(BestUF);
@@ -1439,19 +1438,19 @@ void VPlanTransforms::truncateToMinimalBitwidths(
 }
 
 void VPlanTransforms::optimize(VPlan &Plan) {
-  removeRedundantCanonicalIVs(Plan);
-  removeRedundantInductionCasts(Plan);
-
-  simplifyRecipes(Plan, Plan.getCanonicalIV()->getScalarType());
-  removeDeadRecipes(Plan);
-  legalizeAndOptimizeInductions(Plan);
-  removeRedundantExpandSCEVRecipes(Plan);
-  simplifyRecipes(Plan, Plan.getCanonicalIV()->getScalarType());
-  removeDeadRecipes(Plan);
-
-  createAndOptimizeReplicateRegions(Plan);
-  mergeBlocksIntoPredecessors(Plan);
-  licm(Plan);
+  runPass(removeRedundantCanonicalIVs, Plan);
+  runPass(removeRedundantInductionCasts, Plan);
+
+  runPass(simplifyRecipes, Plan, *Plan.getCanonicalIV()->getScalarType());
+  runPass(removeDeadRecipes, Plan);
+  runPass(legalizeAndOptimizeInductions, Plan);
+  runPass(removeRedundantExpandSCEVRecipes, Plan);
+  runPass(simplifyRecipes, Plan, *Plan.getCanonicalIV()->getScalarType());
+  runPass(removeDeadRecipes, Plan);
+
+  runPass(createAndOptimizeReplicateRegions, Plan);
+  runPass(mergeBlocksIntoPredecessors, Plan);
+  runPass(licm, Plan);
 }
 
 // Add a VPActiveLaneMaskPHIRecipe and related recipes to \p Plan and replace
@@ -1871,7 +1870,8 @@ bool VPlanTransforms::tryAddExplicitVectorLength(
 }
 
 void VPlanTransforms::dropPoisonGeneratingRecipes(
-    VPlan &Plan, function_ref<bool(BasicBlock *)> BlockNeedsPredication) {
+    VPlan &Plan,
+    const std::function<bool(BasicBlock *)> &BlockNeedsPredication) {
   // Collect recipes in the backward slice of `Root` that may generate a poison
   // value that is used after vectorization.
   SmallPtrSet<VPRecipeBase *, 16> Visited;
@@ -1971,7 +1971,7 @@ void VPlanTransforms::createInterleaveGroups(
     VPlan &Plan,
     const SmallPtrSetImpl<const InterleaveGroup<Instruction> *>
         &InterleaveGroups,
-    VPRecipeBuilder &RecipeBuilder, bool ScalarEpilogueAllowed) {
+    VPRecipeBuilder &RecipeBuilder, const bool &ScalarEpilogueAllowed) {
   if (InterleaveGroups.empty())
     return;
 
@@ -2099,11 +2099,42 @@ void VPlanTransforms::handleUncountableEarlyExit(
       Builder.createNaryOp(VPInstruction::AnyOf, {EarlyExitTakenCond});
 
   VPBasicBlock *NewMiddle = Plan.createVPBasicBlock("middle.split");
+  VPBasicBlock *VectorEarlyExitVPBB =
+      Plan.createVPBasicBlock("vector.early.exit");
   VPBlockUtils::insertOnEdge(LoopRegion, MiddleVPBB, NewMiddle);
-  VPBlockUtils::connectBlocks(NewMiddle, VPEarlyExitBlock);
+  VPBlockUtils::connectBlocks(NewMiddle, VectorEarlyExitVPBB);
   NewMiddle->swapSuccessors();
 
+  VPBlockUtils::connectBlocks(VectorEarlyExitVPBB, VPEarlyExitBlock);
+
+  // Update the exit phis in the early exit block.
   VPBuilder MiddleBuilder(NewMiddle);
+  VPBuilder EarlyExitB(VectorEarlyExitVPBB);
+  for (VPRecipeBase &R : *VPEarlyExitBlock) {
+    auto *ExitIRI = cast<VPIRInstruction>(&R);
+    auto *ExitPhi = dyn_cast<PHINode>(&ExitIRI->getInstruction());
+    if (!ExitPhi)
+      break;
+
+    VPValue *IncomingFromEarlyExit = RecipeBuilder.getVPValueOrAddLiveIn(
+        ExitPhi->getIncomingValueForBlock(UncountableExitingBlock));
+
+    if (OrigLoop->getUniqueExitBlock()) {
+      // If there's a unique exit block, VPEarlyExitBlock has 2 predecessors
+      // (MiddleVPBB and NewMiddle). Add the incoming value from MiddleVPBB
+      // which is coming from the original latch.
+      VPValue *IncomingFromLatch = RecipeBuilder.getVPValueOrAddLiveIn(
+          ExitPhi->getIncomingValueForBlock(OrigLoop->getLoopLatch()));
+      ExitIRI->addOperand(IncomingFromLatch);
+      ExitIRI->extractLastLaneOfOperand(MiddleBuilder);
+    }
+    // Add the incoming value from the early exit.
+    if (!IncomingFromEarlyExit->isLiveIn())
+      IncomingFromEarlyExit =
+          EarlyExitB.createNaryOp(VPInstruction::ExtractFirstActive,
+                                  {IncomingFromEarlyExit, EarlyExitTakenCond});
+    ExitIRI->addOperand(IncomingFromEarlyExit);
+  }
   MiddleBuilder.createNaryOp(VPInstruction::BranchOnCond, {IsEarlyExitTaken});
 
   // Replace the condition controlling the non-early exit from the vector loop
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
index a751b8b5e8dc5..3dd476a8526d6 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
@@ -14,7 +14,9 @@
 #define LLVM_TRANSFORMS_VECTORIZE_VPLANTRANSFORMS_H
 
 #include "VPlan.h"
+#include "VPlanVerifier.h"
 #include "llvm/ADT/STLFunctionalExtras.h"
+#include "llvm/Support/CommandLine.h"
 
 namespace llvm {
 
@@ -27,7 +29,29 @@ class TargetLibraryInfo;
 class VPBuilder;
 class VPRecipeBuilder;
 
+extern cl::opt<bool> VerifyEachVPlan;
+
 struct VPlanTransforms {
+  /// Helper to run a VPlan transform \p Transform on \p VPlan, forwarding extra
+  /// arguments to the transform. Returns the boolean returned by the transform.
+  template <typename... ArgsTy>
+  static bool runPass(bool (*Transform)(VPlan &, ArgsTy...), VPlan &Plan,
+                      typename std::remove_reference<ArgsTy>::type &...Args) {
+    bool Res = Transform(Plan, Args...);
+    if (VerifyEachVPlan)
+      verifyVPlanIsValid(Plan);
+    return Res;
+  }
+  /// Helper to run a VPlan transform \p Transform on \p VPlan, forwarding extra
+  /// arguments to the transform.
+  template <typename... ArgsTy>
+  static void runPass(void (*Fn)(VPlan &, ArgsTy...), VPlan &Plan,
+                      typename std::remove_reference<ArgsTy>::type &...Args) {
+    Fn(Plan, Args...);
+    if (VerifyEachVPlan)
+      verifyVPlanIsValid(Plan);
+  }
+
   /// Replaces the VPInstructions in \p Plan with corresponding
   /// widen recipes.
   static void
@@ -100,7 +124,8 @@ struct VPlanTransforms {
   /// TODO: Replace BlockNeedsPredication callback with retrieving info from
   ///       VPlan directly.
   static void dropPoisonGeneratingRecipes(
-      VPlan &Plan, function_ref<bool(BasicBlock *)> BlockNeedsPredication);
+      VPlan &Plan,
+      const std::function<bool(BasicBlock *)> &BlockNeedsPredication);
 
   /// Add a VPEVLBasedIVPHIRecipe and related recipes to \p Plan and
   /// replaces all uses except the canonical IV increment of
@@ -119,7 +144,7 @@ struct VPlanTransforms {
       VPlan &Plan,
       const SmallPtrSetImpl<const InterleaveGroup<Instruction> *>
           &InterleaveGroups,
-      VPRecipeBuilder &RecipeBuilder, bool ScalarEpilogueAllowed);
+      VPRecipeBuilder &RecipeBuilder, const bool &ScalarEpilogueAllowed);
 
   /// Remove dead recipes from \p Plan.
   static void removeDeadRecipes(VPlan &Plan);
@@ -138,6 +163,10 @@ struct VPlanTransforms {
   /// Lower abstract recipes to concrete ones, that can be codegen'd.
   static void convertToConcreteRecipes(VPlan &Plan);
 
+  /// Perform instcombine-like simplifications on recipes in \p Plan. Use \p
+  /// CanonicalIVTy as type for all un-typed live-ins in VPTypeAnalysis.
+  static void simplifyRecipes(VPlan &Plan, Type &CanonicalIVTy);
+
   /// If there's a single exit block, optimize its phi recipes that use exiting
   /// IV values by feeding them precomputed end values instead, possibly taken
   /// one step backwards.
diff --git a/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp b/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
index e40af3e2e3d30..1a7322ec0aff6 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
@@ -30,11 +30,18 @@ VPValue *vputils::getOrCreateVPValueForSCEVExpr(VPlan &Plan, const SCEV *Expr,
   VPValue *Expanded = nullptr;
   if (auto *E = dyn_cast<SCEVConstant>(Expr))
     Expanded = Plan.getOrAddLiveIn(E->getValue());
-  else if (auto *E = dyn_cast<SCEVUnknown>(Expr))
-    Expanded = Plan.getOrAddLiveIn(E->getValue());
   else {
-    Expanded = new VPExpandSCEVRecipe(Expr, SE);
-    Plan.getEntry()->appendRecipe(Expanded->getDefiningRecipe());
+    auto *U = dyn_cast<SCEVUnknown>(Expr);
+    // Skip SCEV expansion if Expr is a SCEVUnknown wrapping a non-instruction
+    // value. Otherwise the value may be defined in a loop and using it directly
+    // will break LCSSA form. The SCEV expansion takes care of preserving LCSSA
+    // form.
+    if (U && !isa<Instruction>(U->getValue())) {
+      Expanded = Plan.getOrAddLiveIn(U->getValue());
+    } else {
+      Expanded = new VPExpandSCEVRecipe(Expr, SE);
+      Plan.getEntry()->appendRecipe(Expanded->getDefiningRecipe());
+    }
   }
   Plan.addSCEVExpansion(Expr, Expanded);
   return Expanded;
diff --git a/llvm/lib/Transforms/Vectorize/VPlanUtils.h b/llvm/lib/Transforms/Vectorize/VPlanUtils.h
index 6ddb88308955f..ac5e1978fcfbe 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanUtils.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanUtils.h
@@ -169,8 +169,16 @@ class VPBlockUtils {
   static void reassociateBlocks(VPBlockBase *Old, VPBlockBase *New) {
     for (auto *Pred : to_vector(Old->getPredecessors()))
       Pred->replaceSuccessor(Old, New);
-    for (auto *Succ : to_vector(Old->getSuccessors()))
+    for (auto *Succ : to_vector(Old->getSuccessors())) {
       Succ->replacePredecessor(Old, New);
+
+      // Replace any references to Old in widened phi incoming blocks.
+      for (auto &R : Succ->getEntryBasicBlock()->phis())
+        if (auto *WidenPhiR = dyn_cast<VPWidenPHIRecipe>(&R))
+          for (unsigned I = 0; I < WidenPhiR->getNumOperands(); I++)
+            if (WidenPhiR->getIncomingBlock(I) == Old)
+              WidenPhiR->setIncomingBlock(I, cast<VPBasicBlock>(New));
+    }
     New->setPredecessors(Old->getPredecessors());
     New->setSuccessors(Old->getSuccessors());
     Old->clearPredecessors();
diff --git a/llvm/lib/Transforms/Vectorize/VPlanValue.h b/llvm/lib/Transforms/Vectorize/VPlanValue.h
index 23e39ce89a3a4..aabc4ab571e7a 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanValue.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanValue.h
@@ -435,41 +435,6 @@ class VPDef {
 #endif
 };
 
-class VPlan;
-class VPBasicBlock;
-
-/// This class can be used to assign names to VPValues. For VPValues without
-/// underlying value, assign consecutive numbers and use those as names (wrapped
-/// in vp<>). Otherwise, use the name from the underlying value (wrapped in
-/// ir<>), appending a .V version number if there are multiple uses of the same
-/// name. Allows querying names for VPValues for printing, similar to the
-/// ModuleSlotTracker for IR values.
-class VPSlotTracker {
-  /// Keep track of versioned names assigned to VPValues with underlying IR
-  /// values.
-  DenseMap<const VPValue *, std::string> VPValue2Name;
-  /// Keep track of the next number to use to version the base name.
-  StringMap<unsigned> BaseName2Version;
-
-  /// Number to assign to the next VPValue without underlying value.
-  unsigned NextSlot = 0;
-
-  void assignName(const VPValue *V);
-  void assignNames(const VPlan &Plan);
-  void assignNames(const VPBasicBlock *VPBB);
-
-public:
-  VPSlotTracker(const VPlan *Plan = nullptr) {
-    if (Plan)
-      assignNames(*Plan);
-  }
-
-  /// Returns the name assigned to \p V, if there is one, otherwise try to
-  /// construct one from the underlying value, if there's one; else return
-  /// <badref>.
-  std::string getOrCreateName(const VPValue *V) const;
-};
-
 } // namespace llvm
 
 #endif // LLVM_TRANSFORMS_VECTORIZE_VPLAN_VALUE_H
diff --git a/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp b/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp
index a30bc82cbde85..96156de444f88 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp
@@ -206,10 +206,12 @@ bool VPlanVerifier::verifyVPBasicBlock(const VPBasicBlock *VPBB) {
       }
 
       for (const VPUser *U : V->users()) {
-        auto *UI = dyn_cast<VPRecipeBase>(U);
+        auto *UI = cast<VPRecipeBase>(U);
         // TODO: check dominance of incoming values for phis properly.
         if (!UI ||
-            isa<VPHeaderPHIRecipe, VPWidenPHIRecipe, VPPredInstPHIRecipe>(UI))
+            isa<VPHeaderPHIRecipe, VPWidenPHIRecipe, VPPredInstPHIRecipe>(UI) ||
+            (isa<VPIRInstruction>(UI) &&
+             isa<PHINode>(cast<VPIRInstruction>(UI)->getInstruction())))
           continue;
 
         // If the user is in the same block, check it comes after R in the
diff --git a/llvm/lib/Transforms/Vectorize/VectorCombine.cpp b/llvm/lib/Transforms/Vectorize/VectorCombine.cpp
index 59920b5a4dd20..746742e14d080 100644
--- a/llvm/lib/Transforms/Vectorize/VectorCombine.cpp
+++ b/llvm/lib/Transforms/Vectorize/VectorCombine.cpp
@@ -108,6 +108,7 @@ class VectorCombine {
                        Instruction &I);
   bool foldExtractExtract(Instruction &I);
   bool foldInsExtFNeg(Instruction &I);
+  bool foldInsExtBinop(Instruction &I);
   bool foldInsExtVectorToShuffle(Instruction &I);
   bool foldBitcastShuffle(Instruction &I);
   bool scalarizeBinopOrCmp(Instruction &I);
@@ -125,6 +126,7 @@ class VectorCombine {
   bool foldShuffleFromReductions(Instruction &I);
   bool foldCastFromReductions(Instruction &I);
   bool foldSelectShuffle(Instruction &I, bool FromReduction = false);
+  bool foldInterleaveIntrinsics(Instruction &I);
   bool shrinkType(Instruction &I);
 
   void replaceValue(Value &Old, Value &New) {
@@ -738,6 +740,64 @@ bool VectorCombine::foldInsExtFNeg(Instruction &I) {
   return true;
 }
 
+/// Try to fold insert(binop(x,y),binop(a,b),idx)
+///         --> binop(insert(x,a,idx),insert(y,b,idx))
+bool VectorCombine::foldInsExtBinop(Instruction &I) {
+  BinaryOperator *VecBinOp, *SclBinOp;
+  uint64_t Index;
+  if (!match(&I,
+             m_InsertElt(m_OneUse(m_BinOp(VecBinOp)),
+                         m_OneUse(m_BinOp(SclBinOp)), m_ConstantInt(Index))))
+    return false;
+
+  // TODO: Add support for addlike etc.
+  Instruction::BinaryOps BinOpcode = VecBinOp->getOpcode();
+  if (BinOpcode != SclBinOp->getOpcode())
+    return false;
+
+  auto *ResultTy = dyn_cast<FixedVectorType>(I.getType());
+  if (!ResultTy)
+    return false;
+
+  // TODO: Attempt to detect m_ExtractElt for scalar operands and convert to
+  // shuffle?
+
+  InstructionCost OldCost = TTI.getInstructionCost(&I, CostKind) +
+                            TTI.getInstructionCost(VecBinOp, CostKind) +
+                            TTI.getInstructionCost(SclBinOp, CostKind);
+  InstructionCost NewCost =
+      TTI.getArithmeticInstrCost(BinOpcode, ResultTy, CostKind) +
+      TTI.getVectorInstrCost(Instruction::InsertElement, ResultTy, CostKind,
+                             Index, VecBinOp->getOperand(0),
+                             SclBinOp->getOperand(0)) +
+      TTI.getVectorInstrCost(Instruction::InsertElement, ResultTy, CostKind,
+                             Index, VecBinOp->getOperand(1),
+                             SclBinOp->getOperand(1));
+
+  LLVM_DEBUG(dbgs() << "Found an insertion of two binops: " << I
+                    << "\n  OldCost: " << OldCost << " vs NewCost: " << NewCost
+                    << "\n");
+  if (NewCost > OldCost)
+    return false;
+
+  Value *NewIns0 = Builder.CreateInsertElement(VecBinOp->getOperand(0),
+                                               SclBinOp->getOperand(0), Index);
+  Value *NewIns1 = Builder.CreateInsertElement(VecBinOp->getOperand(1),
+                                               SclBinOp->getOperand(1), Index);
+  Value *NewBO = Builder.CreateBinOp(BinOpcode, NewIns0, NewIns1);
+
+  // Intersect flags from the old binops.
+  if (auto *NewInst = dyn_cast<Instruction>(NewBO)) {
+    NewInst->copyIRFlags(VecBinOp);
+    NewInst->andIRFlags(SclBinOp);
+  }
+
+  Worklist.pushValue(NewIns0);
+  Worklist.pushValue(NewIns1);
+  replaceValue(I, *NewBO);
+  return true;
+}
+
 /// If this is a bitcast of a shuffle, try to bitcast the source vector to the
 /// destination type followed by shuffle. This can enable further transforms by
 /// moving bitcasts or shuffles together.
@@ -3087,42 +3147,73 @@ bool VectorCombine::foldInsExtVectorToShuffle(Instruction &I) {
                          m_ConstantInt(InsIdx))))
     return false;
 
-  auto *VecTy = dyn_cast<FixedVectorType>(I.getType());
-  if (!VecTy || SrcVec->getType() != VecTy)
+  auto *DstVecTy = dyn_cast<FixedVectorType>(I.getType());
+  auto *SrcVecTy = dyn_cast<FixedVectorType>(SrcVec->getType());
+  // We can try combining vectors with different element sizes.
+  if (!DstVecTy || !SrcVecTy ||
+      SrcVecTy->getElementType() != DstVecTy->getElementType())
     return false;
 
-  unsigned NumElts = VecTy->getNumElements();
-  if (ExtIdx >= NumElts || InsIdx >= NumElts)
+  unsigned NumDstElts = DstVecTy->getNumElements();
+  unsigned NumSrcElts = SrcVecTy->getNumElements();
+  if (InsIdx >= NumDstElts || ExtIdx >= NumSrcElts || NumDstElts == 1)
     return false;
 
   // Insertion into poison is a cheaper single operand shuffle.
   TargetTransformInfo::ShuffleKind SK;
-  SmallVector<int> Mask(NumElts, PoisonMaskElem);
-  if (isa<PoisonValue>(DstVec) && !isa<UndefValue>(SrcVec)) {
+  SmallVector<int> Mask(NumDstElts, PoisonMaskElem);
+
+  bool NeedExpOrNarrow = NumSrcElts != NumDstElts;
+  bool IsExtIdxInBounds = ExtIdx < NumDstElts;
+  bool NeedDstSrcSwap = isa<PoisonValue>(DstVec) && !isa<UndefValue>(SrcVec);
+  if (NeedDstSrcSwap) {
     SK = TargetTransformInfo::SK_PermuteSingleSrc;
-    Mask[InsIdx] = ExtIdx;
+    if (!IsExtIdxInBounds && NeedExpOrNarrow)
+      Mask[InsIdx] = 0;
+    else
+      Mask[InsIdx] = ExtIdx;
     std::swap(DstVec, SrcVec);
   } else {
     SK = TargetTransformInfo::SK_PermuteTwoSrc;
     std::iota(Mask.begin(), Mask.end(), 0);
-    Mask[InsIdx] = ExtIdx + NumElts;
+    if (!IsExtIdxInBounds && NeedExpOrNarrow)
+      Mask[InsIdx] = NumDstElts;
+    else
+      Mask[InsIdx] = ExtIdx + NumDstElts;
   }
 
   // Cost
   auto *Ins = cast<InsertElementInst>(&I);
   auto *Ext = cast<ExtractElementInst>(I.getOperand(1));
   InstructionCost InsCost =
-      TTI.getVectorInstrCost(*Ins, VecTy, CostKind, InsIdx);
+      TTI.getVectorInstrCost(*Ins, DstVecTy, CostKind, InsIdx);
   InstructionCost ExtCost =
-      TTI.getVectorInstrCost(*Ext, VecTy, CostKind, ExtIdx);
+      TTI.getVectorInstrCost(*Ext, DstVecTy, CostKind, ExtIdx);
   InstructionCost OldCost = ExtCost + InsCost;
 
-  // Ignore 'free' identity insertion shuffle.
-  // TODO: getShuffleCost should return TCC_Free for Identity shuffles.
   InstructionCost NewCost = 0;
-  if (!ShuffleVectorInst::isIdentityMask(Mask, NumElts))
-    NewCost += TTI.getShuffleCost(SK, VecTy, Mask, CostKind, 0, nullptr,
-                                  {DstVec, SrcVec});
+  SmallVector<int> ExtToVecMask;
+  if (!NeedExpOrNarrow) {
+    // Ignore 'free' identity insertion shuffle.
+    // TODO: getShuffleCost should return TCC_Free for Identity shuffles.
+    if (!ShuffleVectorInst::isIdentityMask(Mask, NumSrcElts))
+      NewCost += TTI.getShuffleCost(SK, DstVecTy, Mask, CostKind, 0, nullptr,
+                                    {DstVec, SrcVec});
+  } else {
+    // When creating length-changing-vector, always create with a Mask whose
+    // first element has an ExtIdx, so that the first element of the vector
+    // being created is always the target to be extracted.
+    ExtToVecMask.assign(NumDstElts, PoisonMaskElem);
+    if (IsExtIdxInBounds)
+      ExtToVecMask[ExtIdx] = ExtIdx;
+    else
+      ExtToVecMask[0] = ExtIdx;
+    // Add cost for expanding or narrowing
+    NewCost = TTI.getShuffleCost(TargetTransformInfo::SK_PermuteSingleSrc,
+                                 SrcVecTy, ExtToVecMask, CostKind);
+    NewCost += TTI.getShuffleCost(SK, DstVecTy, Mask, CostKind);
+  }
+
   if (!Ext->hasOneUse())
     NewCost += ExtCost;
 
@@ -3133,9 +3224,16 @@ bool VectorCombine::foldInsExtVectorToShuffle(Instruction &I) {
   if (OldCost < NewCost)
     return false;
 
+  if (NeedExpOrNarrow) {
+    if (!NeedDstSrcSwap)
+      SrcVec = Builder.CreateShuffleVector(SrcVec, ExtToVecMask);
+    else
+      DstVec = Builder.CreateShuffleVector(DstVec, ExtToVecMask);
+  }
+
   // Canonicalize undef param to RHS to help further folds.
   if (isa<UndefValue>(DstVec) && !isa<UndefValue>(SrcVec)) {
-    ShuffleVectorInst::commuteShuffleMask(Mask, NumElts);
+    ShuffleVectorInst::commuteShuffleMask(Mask, NumDstElts);
     std::swap(DstVec, SrcVec);
   }
 
@@ -3145,6 +3243,47 @@ bool VectorCombine::foldInsExtVectorToShuffle(Instruction &I) {
   return true;
 }
 
+/// If we're interleaving 2 constant splats, for instance `<vscale x 8 x i32>
+/// <splat of 666>` and `<vscale x 8 x i32> <splat of 777>`, we can create a
+/// larger splat `<vscale x 8 x i64> <splat of ((777 << 32) | 666)>` first
+/// before casting it back into `<vscale x 16 x i32>`.
+bool VectorCombine::foldInterleaveIntrinsics(Instruction &I) {
+  const APInt *SplatVal0, *SplatVal1;
+  if (!match(&I, m_Intrinsic<Intrinsic::vector_interleave2>(
+                     m_APInt(SplatVal0), m_APInt(SplatVal1))))
+    return false;
+
+  LLVM_DEBUG(dbgs() << "VC: Folding interleave2 with two splats: " << I
+                    << "\n");
+
+  auto *VTy =
+      cast<VectorType>(cast<IntrinsicInst>(I).getArgOperand(0)->getType());
+  auto *ExtVTy = VectorType::getExtendedElementVectorType(VTy);
+  unsigned Width = VTy->getElementType()->getIntegerBitWidth();
+
+  // Just in case the cost of interleave2 intrinsic and bitcast are both
+  // invalid, in which case we want to bail out, we use <= rather
+  // than < here. Even they both have valid and equal costs, it's probably
+  // not a good idea to emit a high-cost constant splat.
+  if (TTI.getInstructionCost(&I, CostKind) <=
+      TTI.getCastInstrCost(Instruction::BitCast, I.getType(), ExtVTy,
+                           TTI::CastContextHint::None, CostKind)) {
+    LLVM_DEBUG(dbgs() << "VC: The cost to cast from " << *ExtVTy << " to "
+                      << *I.getType() << " is too high.\n");
+    return false;
+  }
+
+  APInt NewSplatVal = SplatVal1->zext(Width * 2);
+  NewSplatVal <<= Width;
+  NewSplatVal |= SplatVal0->zext(Width * 2);
+  auto *NewSplat = ConstantVector::getSplat(
+      ExtVTy->getElementCount(), ConstantInt::get(F.getContext(), NewSplatVal));
+
+  IRBuilder<> Builder(&I);
+  replaceValue(I, *Builder.CreateBitCast(NewSplat, I.getType()));
+  return true;
+}
+
 /// This is the entry point for all transforms. Pass manager differences are
 /// handled in the callers of this function.
 bool VectorCombine::run() {
@@ -3189,6 +3328,7 @@ bool VectorCombine::run() {
       MadeChange |= scalarizeBinopOrCmp(I);
       MadeChange |= scalarizeLoadExtract(I);
       MadeChange |= scalarizeVPIntrinsic(I);
+      MadeChange |= foldInterleaveIntrinsics(I);
     }
 
     if (Opcode == Instruction::Store)
@@ -3206,6 +3346,7 @@ bool VectorCombine::run() {
       switch (Opcode) {
       case Instruction::InsertElement:
         MadeChange |= foldInsExtFNeg(I);
+        MadeChange |= foldInsExtBinop(I);
         MadeChange |= foldInsExtVectorToShuffle(I);
         break;
       case Instruction::ShuffleVector:
diff --git a/llvm/test/Analysis/CostModel/AArch64/cttz_elts.ll b/llvm/test/Analysis/CostModel/AArch64/cttz_elts.ll
index 98d5bd5bd13f4..5a2d08a17fca4 100644
--- a/llvm/test/Analysis/CostModel/AArch64/cttz_elts.ll
+++ b/llvm/test/Analysis/CostModel/AArch64/cttz_elts.ll
@@ -4,45 +4,45 @@
 define void @foo_no_vscale_range() {
 ; CHECK-LABEL: 'foo_no_vscale_range'
 ; CHECK-NEXT:  Cost Model: Invalid cost for instruction: %res.i64.nxv1i1.zip = call i64 @llvm.experimental.cttz.elts.i64.nxv1i1(<vscale x 1 x i1> undef, i1 true)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res.i64.nxv2i1.zip = call i64 @llvm.experimental.cttz.elts.i64.nxv2i1(<vscale x 2 x i1> undef, i1 true)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res.i64.nxv4i1.zip = call i64 @llvm.experimental.cttz.elts.i64.nxv4i1(<vscale x 4 x i1> undef, i1 true)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res.i64.nxv8i1.zip = call i64 @llvm.experimental.cttz.elts.i64.nxv8i1(<vscale x 8 x i1> undef, i1 true)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res.i64.nxv16i1.zip = call i64 @llvm.experimental.cttz.elts.i64.nxv16i1(<vscale x 16 x i1> undef, i1 true)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %res.i64.nxv2i1.zip = call i64 @llvm.experimental.cttz.elts.i64.nxv2i1(<vscale x 2 x i1> undef, i1 true)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %res.i64.nxv4i1.zip = call i64 @llvm.experimental.cttz.elts.i64.nxv4i1(<vscale x 4 x i1> undef, i1 true)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %res.i64.nxv8i1.zip = call i64 @llvm.experimental.cttz.elts.i64.nxv8i1(<vscale x 8 x i1> undef, i1 true)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %res.i64.nxv16i1.zip = call i64 @llvm.experimental.cttz.elts.i64.nxv16i1(<vscale x 16 x i1> undef, i1 true)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 96 for instruction: %res.i64.nxv32i1.zip = call i64 @llvm.experimental.cttz.elts.i64.nxv32i1(<vscale x 32 x i1> undef, i1 true)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res.i32.nxv2i1.zip = call i32 @llvm.experimental.cttz.elts.i32.nxv2i1(<vscale x 2 x i1> undef, i1 true)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res.i32.nxv4i1.zip = call i32 @llvm.experimental.cttz.elts.i32.nxv4i1(<vscale x 4 x i1> undef, i1 true)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res.i32.nxv8i1.zip = call i32 @llvm.experimental.cttz.elts.i32.nxv8i1(<vscale x 8 x i1> undef, i1 true)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res.i32.nxv16i1.zip = call i32 @llvm.experimental.cttz.elts.i32.nxv16i1(<vscale x 16 x i1> undef, i1 true)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %res.i32.nxv2i1.zip = call i32 @llvm.experimental.cttz.elts.i32.nxv2i1(<vscale x 2 x i1> undef, i1 true)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %res.i32.nxv4i1.zip = call i32 @llvm.experimental.cttz.elts.i32.nxv4i1(<vscale x 4 x i1> undef, i1 true)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %res.i32.nxv8i1.zip = call i32 @llvm.experimental.cttz.elts.i32.nxv8i1(<vscale x 8 x i1> undef, i1 true)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %res.i32.nxv16i1.zip = call i32 @llvm.experimental.cttz.elts.i32.nxv16i1(<vscale x 16 x i1> undef, i1 true)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %res.i32.nxv32i1.zip = call i32 @llvm.experimental.cttz.elts.i32.nxv32i1(<vscale x 32 x i1> undef, i1 true)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res.i64.v2i1.zip = call i64 @llvm.experimental.cttz.elts.i64.v2i1(<2 x i1> undef, i1 true)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res.i64.v4i1.zip = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> undef, i1 true)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res.i64.v8i1.zip = call i64 @llvm.experimental.cttz.elts.i64.v8i1(<8 x i1> undef, i1 true)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res.i64.v16i1.zip = call i64 @llvm.experimental.cttz.elts.i64.v16i1(<16 x i1> undef, i1 true)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %res.i64.v2i1.zip = call i64 @llvm.experimental.cttz.elts.i64.v2i1(<2 x i1> undef, i1 true)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %res.i64.v4i1.zip = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> undef, i1 true)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %res.i64.v8i1.zip = call i64 @llvm.experimental.cttz.elts.i64.v8i1(<8 x i1> undef, i1 true)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %res.i64.v16i1.zip = call i64 @llvm.experimental.cttz.elts.i64.v16i1(<16 x i1> undef, i1 true)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %res.i64.v32i1.zip = call i64 @llvm.experimental.cttz.elts.i64.v32i1(<32 x i1> undef, i1 true)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res.i32.v2i1.zip = call i32 @llvm.experimental.cttz.elts.i32.v2i1(<2 x i1> undef, i1 true)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res.i32.v4i1.zip = call i32 @llvm.experimental.cttz.elts.i32.v4i1(<4 x i1> undef, i1 true)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res.i32.v8i1.zip = call i32 @llvm.experimental.cttz.elts.i32.v8i1(<8 x i1> undef, i1 true)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res.i32.v16i1.zip = call i32 @llvm.experimental.cttz.elts.i32.v16i1(<16 x i1> undef, i1 true)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %res.i32.v2i1.zip = call i32 @llvm.experimental.cttz.elts.i32.v2i1(<2 x i1> undef, i1 true)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %res.i32.v4i1.zip = call i32 @llvm.experimental.cttz.elts.i32.v4i1(<4 x i1> undef, i1 true)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %res.i32.v8i1.zip = call i32 @llvm.experimental.cttz.elts.i32.v8i1(<8 x i1> undef, i1 true)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %res.i32.v16i1.zip = call i32 @llvm.experimental.cttz.elts.i32.v16i1(<16 x i1> undef, i1 true)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %res.i32.v32i1.zip = call i32 @llvm.experimental.cttz.elts.i32.v32i1(<32 x i1> undef, i1 true)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res.i64.nxv2i1.nzip = call i64 @llvm.experimental.cttz.elts.i64.nxv2i1(<vscale x 2 x i1> undef, i1 false)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res.i64.nxv4i1.nzip = call i64 @llvm.experimental.cttz.elts.i64.nxv4i1(<vscale x 4 x i1> undef, i1 false)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res.i64.nxv8i1.nzip = call i64 @llvm.experimental.cttz.elts.i64.nxv8i1(<vscale x 8 x i1> undef, i1 false)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res.i64.nxv16i1.nzip = call i64 @llvm.experimental.cttz.elts.i64.nxv16i1(<vscale x 16 x i1> undef, i1 false)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %res.i64.nxv2i1.nzip = call i64 @llvm.experimental.cttz.elts.i64.nxv2i1(<vscale x 2 x i1> undef, i1 false)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %res.i64.nxv4i1.nzip = call i64 @llvm.experimental.cttz.elts.i64.nxv4i1(<vscale x 4 x i1> undef, i1 false)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %res.i64.nxv8i1.nzip = call i64 @llvm.experimental.cttz.elts.i64.nxv8i1(<vscale x 8 x i1> undef, i1 false)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %res.i64.nxv16i1.nzip = call i64 @llvm.experimental.cttz.elts.i64.nxv16i1(<vscale x 16 x i1> undef, i1 false)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 96 for instruction: %res.i64.nxv32i1.nzip = call i64 @llvm.experimental.cttz.elts.i64.nxv32i1(<vscale x 32 x i1> undef, i1 false)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res.i32.nxv2i1.nzip = call i32 @llvm.experimental.cttz.elts.i32.nxv2i1(<vscale x 2 x i1> undef, i1 false)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res.i32.nxv4i1.nzip = call i32 @llvm.experimental.cttz.elts.i32.nxv4i1(<vscale x 4 x i1> undef, i1 false)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res.i32.nxv8i1.nzip = call i32 @llvm.experimental.cttz.elts.i32.nxv8i1(<vscale x 8 x i1> undef, i1 false)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res.i32.nxv16i1.nzip = call i32 @llvm.experimental.cttz.elts.i32.nxv16i1(<vscale x 16 x i1> undef, i1 false)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %res.i32.nxv2i1.nzip = call i32 @llvm.experimental.cttz.elts.i32.nxv2i1(<vscale x 2 x i1> undef, i1 false)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %res.i32.nxv4i1.nzip = call i32 @llvm.experimental.cttz.elts.i32.nxv4i1(<vscale x 4 x i1> undef, i1 false)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %res.i32.nxv8i1.nzip = call i32 @llvm.experimental.cttz.elts.i32.nxv8i1(<vscale x 8 x i1> undef, i1 false)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %res.i32.nxv16i1.nzip = call i32 @llvm.experimental.cttz.elts.i32.nxv16i1(<vscale x 16 x i1> undef, i1 false)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %res.i32.nxv32i1.nzip = call i32 @llvm.experimental.cttz.elts.i32.nxv32i1(<vscale x 32 x i1> undef, i1 false)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res.i64.v2i1.nzip = call i64 @llvm.experimental.cttz.elts.i64.v2i1(<2 x i1> undef, i1 false)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res.i64.v4i1.nzip = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> undef, i1 false)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res.i64.v8i1.nzip = call i64 @llvm.experimental.cttz.elts.i64.v8i1(<8 x i1> undef, i1 false)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res.i64.v16i1.nzip = call i64 @llvm.experimental.cttz.elts.i64.v16i1(<16 x i1> undef, i1 false)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %res.i64.v2i1.nzip = call i64 @llvm.experimental.cttz.elts.i64.v2i1(<2 x i1> undef, i1 false)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %res.i64.v4i1.nzip = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> undef, i1 false)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %res.i64.v8i1.nzip = call i64 @llvm.experimental.cttz.elts.i64.v8i1(<8 x i1> undef, i1 false)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %res.i64.v16i1.nzip = call i64 @llvm.experimental.cttz.elts.i64.v16i1(<16 x i1> undef, i1 false)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %res.i64.v32i1.nzip = call i64 @llvm.experimental.cttz.elts.i64.v32i1(<32 x i1> undef, i1 false)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res.i32.v2i1.nzip = call i32 @llvm.experimental.cttz.elts.i32.v2i1(<2 x i1> undef, i1 false)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res.i32.v4i1.nzip = call i32 @llvm.experimental.cttz.elts.i32.v4i1(<4 x i1> undef, i1 false)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res.i32.v8i1.nzip = call i32 @llvm.experimental.cttz.elts.i32.v8i1(<8 x i1> undef, i1 false)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res.i32.v16i1.nzip = call i32 @llvm.experimental.cttz.elts.i32.v16i1(<16 x i1> undef, i1 false)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %res.i32.v2i1.nzip = call i32 @llvm.experimental.cttz.elts.i32.v2i1(<2 x i1> undef, i1 false)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %res.i32.v4i1.nzip = call i32 @llvm.experimental.cttz.elts.i32.v4i1(<4 x i1> undef, i1 false)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %res.i32.v8i1.nzip = call i32 @llvm.experimental.cttz.elts.i32.v8i1(<8 x i1> undef, i1 false)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %res.i32.v16i1.nzip = call i32 @llvm.experimental.cttz.elts.i32.v16i1(<16 x i1> undef, i1 false)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %res.i32.v32i1.nzip = call i32 @llvm.experimental.cttz.elts.i32.v32i1(<32 x i1> undef, i1 false)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
@@ -97,25 +97,25 @@ define void @foo_no_vscale_range() {
 
 define void @foo_vscale_range_1_16() vscale_range(1,16) {
 ; CHECK-LABEL: 'foo_vscale_range_1_16'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res.i64.nxv2i1.zip = call i64 @llvm.experimental.cttz.elts.i64.nxv2i1(<vscale x 2 x i1> undef, i1 true)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res.i64.nxv4i1.zip = call i64 @llvm.experimental.cttz.elts.i64.nxv4i1(<vscale x 4 x i1> undef, i1 true)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res.i64.nxv8i1.zip = call i64 @llvm.experimental.cttz.elts.i64.nxv8i1(<vscale x 8 x i1> undef, i1 true)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res.i64.nxv16i1.zip = call i64 @llvm.experimental.cttz.elts.i64.nxv16i1(<vscale x 16 x i1> undef, i1 true)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %res.i64.nxv2i1.zip = call i64 @llvm.experimental.cttz.elts.i64.nxv2i1(<vscale x 2 x i1> undef, i1 true)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %res.i64.nxv4i1.zip = call i64 @llvm.experimental.cttz.elts.i64.nxv4i1(<vscale x 4 x i1> undef, i1 true)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %res.i64.nxv8i1.zip = call i64 @llvm.experimental.cttz.elts.i64.nxv8i1(<vscale x 8 x i1> undef, i1 true)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %res.i64.nxv16i1.zip = call i64 @llvm.experimental.cttz.elts.i64.nxv16i1(<vscale x 16 x i1> undef, i1 true)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %res.i64.nxv32i1.zip = call i64 @llvm.experimental.cttz.elts.i64.nxv32i1(<vscale x 32 x i1> undef, i1 true)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res.i32.nxv2i1.zip = call i32 @llvm.experimental.cttz.elts.i32.nxv2i1(<vscale x 2 x i1> undef, i1 true)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res.i32.nxv4i1.zip = call i32 @llvm.experimental.cttz.elts.i32.nxv4i1(<vscale x 4 x i1> undef, i1 true)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res.i32.nxv8i1.zip = call i32 @llvm.experimental.cttz.elts.i32.nxv8i1(<vscale x 8 x i1> undef, i1 true)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res.i32.nxv16i1.zip = call i32 @llvm.experimental.cttz.elts.i32.nxv16i1(<vscale x 16 x i1> undef, i1 true)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %res.i32.nxv2i1.zip = call i32 @llvm.experimental.cttz.elts.i32.nxv2i1(<vscale x 2 x i1> undef, i1 true)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %res.i32.nxv4i1.zip = call i32 @llvm.experimental.cttz.elts.i32.nxv4i1(<vscale x 4 x i1> undef, i1 true)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %res.i32.nxv8i1.zip = call i32 @llvm.experimental.cttz.elts.i32.nxv8i1(<vscale x 8 x i1> undef, i1 true)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %res.i32.nxv16i1.zip = call i32 @llvm.experimental.cttz.elts.i32.nxv16i1(<vscale x 16 x i1> undef, i1 true)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %res.i32.nxv32i1.zip = call i32 @llvm.experimental.cttz.elts.i32.nxv32i1(<vscale x 32 x i1> undef, i1 true)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res.i64.nxv2i1.nzip = call i64 @llvm.experimental.cttz.elts.i64.nxv2i1(<vscale x 2 x i1> undef, i1 false)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res.i64.nxv4i1.nzip = call i64 @llvm.experimental.cttz.elts.i64.nxv4i1(<vscale x 4 x i1> undef, i1 false)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res.i64.nxv8i1.nzip = call i64 @llvm.experimental.cttz.elts.i64.nxv8i1(<vscale x 8 x i1> undef, i1 false)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res.i64.nxv16i1.nzip = call i64 @llvm.experimental.cttz.elts.i64.nxv16i1(<vscale x 16 x i1> undef, i1 false)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %res.i64.nxv2i1.nzip = call i64 @llvm.experimental.cttz.elts.i64.nxv2i1(<vscale x 2 x i1> undef, i1 false)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %res.i64.nxv4i1.nzip = call i64 @llvm.experimental.cttz.elts.i64.nxv4i1(<vscale x 4 x i1> undef, i1 false)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %res.i64.nxv8i1.nzip = call i64 @llvm.experimental.cttz.elts.i64.nxv8i1(<vscale x 8 x i1> undef, i1 false)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %res.i64.nxv16i1.nzip = call i64 @llvm.experimental.cttz.elts.i64.nxv16i1(<vscale x 16 x i1> undef, i1 false)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %res.i64.nxv32i1.nzip = call i64 @llvm.experimental.cttz.elts.i64.nxv32i1(<vscale x 32 x i1> undef, i1 false)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res.i32.nxv2i1.nzip = call i32 @llvm.experimental.cttz.elts.i32.nxv2i1(<vscale x 2 x i1> undef, i1 false)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res.i32.nxv4i1.nzip = call i32 @llvm.experimental.cttz.elts.i32.nxv4i1(<vscale x 4 x i1> undef, i1 false)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res.i32.nxv8i1.nzip = call i32 @llvm.experimental.cttz.elts.i32.nxv8i1(<vscale x 8 x i1> undef, i1 false)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res.i32.nxv16i1.nzip = call i32 @llvm.experimental.cttz.elts.i32.nxv16i1(<vscale x 16 x i1> undef, i1 false)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %res.i32.nxv2i1.nzip = call i32 @llvm.experimental.cttz.elts.i32.nxv2i1(<vscale x 2 x i1> undef, i1 false)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %res.i32.nxv4i1.nzip = call i32 @llvm.experimental.cttz.elts.i32.nxv4i1(<vscale x 4 x i1> undef, i1 false)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %res.i32.nxv8i1.nzip = call i32 @llvm.experimental.cttz.elts.i32.nxv8i1(<vscale x 8 x i1> undef, i1 false)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %res.i32.nxv16i1.nzip = call i32 @llvm.experimental.cttz.elts.i32.nxv16i1(<vscale x 16 x i1> undef, i1 false)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %res.i32.nxv32i1.nzip = call i32 @llvm.experimental.cttz.elts.i32.nxv32i1(<vscale x 32 x i1> undef, i1 false)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
@@ -146,25 +146,25 @@ define void @foo_vscale_range_1_16() vscale_range(1,16) {
 
 define void @foo_vscale_range_1_16384() vscale_range(1,16384) {
 ; CHECK-LABEL: 'foo_vscale_range_1_16384'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res.i64.nxv2i1.zip = call i64 @llvm.experimental.cttz.elts.i64.nxv2i1(<vscale x 2 x i1> undef, i1 true)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res.i64.nxv4i1.zip = call i64 @llvm.experimental.cttz.elts.i64.nxv4i1(<vscale x 4 x i1> undef, i1 true)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res.i64.nxv8i1.zip = call i64 @llvm.experimental.cttz.elts.i64.nxv8i1(<vscale x 8 x i1> undef, i1 true)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res.i64.nxv16i1.zip = call i64 @llvm.experimental.cttz.elts.i64.nxv16i1(<vscale x 16 x i1> undef, i1 true)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %res.i64.nxv2i1.zip = call i64 @llvm.experimental.cttz.elts.i64.nxv2i1(<vscale x 2 x i1> undef, i1 true)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %res.i64.nxv4i1.zip = call i64 @llvm.experimental.cttz.elts.i64.nxv4i1(<vscale x 4 x i1> undef, i1 true)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %res.i64.nxv8i1.zip = call i64 @llvm.experimental.cttz.elts.i64.nxv8i1(<vscale x 8 x i1> undef, i1 true)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %res.i64.nxv16i1.zip = call i64 @llvm.experimental.cttz.elts.i64.nxv16i1(<vscale x 16 x i1> undef, i1 true)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %res.i64.nxv32i1.zip = call i64 @llvm.experimental.cttz.elts.i64.nxv32i1(<vscale x 32 x i1> undef, i1 true)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res.i32.nxv2i1.zip = call i32 @llvm.experimental.cttz.elts.i32.nxv2i1(<vscale x 2 x i1> undef, i1 true)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res.i32.nxv4i1.zip = call i32 @llvm.experimental.cttz.elts.i32.nxv4i1(<vscale x 4 x i1> undef, i1 true)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res.i32.nxv8i1.zip = call i32 @llvm.experimental.cttz.elts.i32.nxv8i1(<vscale x 8 x i1> undef, i1 true)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res.i32.nxv16i1.zip = call i32 @llvm.experimental.cttz.elts.i32.nxv16i1(<vscale x 16 x i1> undef, i1 true)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %res.i32.nxv2i1.zip = call i32 @llvm.experimental.cttz.elts.i32.nxv2i1(<vscale x 2 x i1> undef, i1 true)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %res.i32.nxv4i1.zip = call i32 @llvm.experimental.cttz.elts.i32.nxv4i1(<vscale x 4 x i1> undef, i1 true)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %res.i32.nxv8i1.zip = call i32 @llvm.experimental.cttz.elts.i32.nxv8i1(<vscale x 8 x i1> undef, i1 true)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %res.i32.nxv16i1.zip = call i32 @llvm.experimental.cttz.elts.i32.nxv16i1(<vscale x 16 x i1> undef, i1 true)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %res.i32.nxv32i1.zip = call i32 @llvm.experimental.cttz.elts.i32.nxv32i1(<vscale x 32 x i1> undef, i1 true)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res.i64.nxv2i1.nzip = call i64 @llvm.experimental.cttz.elts.i64.nxv2i1(<vscale x 2 x i1> undef, i1 false)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res.i64.nxv4i1.nzip = call i64 @llvm.experimental.cttz.elts.i64.nxv4i1(<vscale x 4 x i1> undef, i1 false)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res.i64.nxv8i1.nzip = call i64 @llvm.experimental.cttz.elts.i64.nxv8i1(<vscale x 8 x i1> undef, i1 false)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res.i64.nxv16i1.nzip = call i64 @llvm.experimental.cttz.elts.i64.nxv16i1(<vscale x 16 x i1> undef, i1 false)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %res.i64.nxv2i1.nzip = call i64 @llvm.experimental.cttz.elts.i64.nxv2i1(<vscale x 2 x i1> undef, i1 false)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %res.i64.nxv4i1.nzip = call i64 @llvm.experimental.cttz.elts.i64.nxv4i1(<vscale x 4 x i1> undef, i1 false)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %res.i64.nxv8i1.nzip = call i64 @llvm.experimental.cttz.elts.i64.nxv8i1(<vscale x 8 x i1> undef, i1 false)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %res.i64.nxv16i1.nzip = call i64 @llvm.experimental.cttz.elts.i64.nxv16i1(<vscale x 16 x i1> undef, i1 false)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %res.i64.nxv32i1.nzip = call i64 @llvm.experimental.cttz.elts.i64.nxv32i1(<vscale x 32 x i1> undef, i1 false)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res.i32.nxv2i1.nzip = call i32 @llvm.experimental.cttz.elts.i32.nxv2i1(<vscale x 2 x i1> undef, i1 false)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res.i32.nxv4i1.nzip = call i32 @llvm.experimental.cttz.elts.i32.nxv4i1(<vscale x 4 x i1> undef, i1 false)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res.i32.nxv8i1.nzip = call i32 @llvm.experimental.cttz.elts.i32.nxv8i1(<vscale x 8 x i1> undef, i1 false)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res.i32.nxv16i1.nzip = call i32 @llvm.experimental.cttz.elts.i32.nxv16i1(<vscale x 16 x i1> undef, i1 false)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %res.i32.nxv2i1.nzip = call i32 @llvm.experimental.cttz.elts.i32.nxv2i1(<vscale x 2 x i1> undef, i1 false)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %res.i32.nxv4i1.nzip = call i32 @llvm.experimental.cttz.elts.i32.nxv4i1(<vscale x 4 x i1> undef, i1 false)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %res.i32.nxv8i1.nzip = call i32 @llvm.experimental.cttz.elts.i32.nxv8i1(<vscale x 8 x i1> undef, i1 false)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %res.i32.nxv16i1.nzip = call i32 @llvm.experimental.cttz.elts.i32.nxv16i1(<vscale x 16 x i1> undef, i1 false)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %res.i32.nxv32i1.nzip = call i32 @llvm.experimental.cttz.elts.i32.nxv32i1(<vscale x 32 x i1> undef, i1 false)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
diff --git a/llvm/test/Analysis/CostModel/AArch64/arith-fp-sve.ll b/llvm/test/Analysis/CostModel/AArch64/sve-arith-fp.ll
similarity index 100%
rename from llvm/test/Analysis/CostModel/AArch64/arith-fp-sve.ll
rename to llvm/test/Analysis/CostModel/AArch64/sve-arith-fp.ll
diff --git a/llvm/test/Analysis/CostModel/AArch64/getIntrinsicInstrCost-vector-reverse.ll b/llvm/test/Analysis/CostModel/AArch64/vector-reverse.ll
similarity index 100%
rename from llvm/test/Analysis/CostModel/AArch64/getIntrinsicInstrCost-vector-reverse.ll
rename to llvm/test/Analysis/CostModel/AArch64/vector-reverse.ll
diff --git a/llvm/test/Analysis/CostModel/RISCV/arith-fp.ll b/llvm/test/Analysis/CostModel/RISCV/arith-fp.ll
index d85c9e2a19209..852c237cf2501 100644
--- a/llvm/test/Analysis/CostModel/RISCV/arith-fp.ll
+++ b/llvm/test/Analysis/CostModel/RISCV/arith-fp.ll
@@ -1381,30 +1381,30 @@ define void @fma() {
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2BF16 = call <2 x bfloat> @llvm.fma.v2bf16(<2 x bfloat> undef, <2 x bfloat> undef, <2 x bfloat> undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4BF16 = call <4 x bfloat> @llvm.fma.v4bf16(<4 x bfloat> undef, <4 x bfloat> undef, <4 x bfloat> undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8BF16 = call <8 x bfloat> @llvm.fma.v8bf16(<8 x bfloat> undef, <8 x bfloat> undef, <8 x bfloat> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16BF16 = call <16 x bfloat> @llvm.fma.v16bf16(<16 x bfloat> undef, <16 x bfloat> undef, <16 x bfloat> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16BF16 = call <16 x bfloat> @llvm.fma.v16bf16(<16 x bfloat> undef, <16 x bfloat> undef, <16 x bfloat> undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV1BF16 = call <vscale x 1 x bfloat> @llvm.fma.nxv1bf16(<vscale x 1 x bfloat> undef, <vscale x 1 x bfloat> undef, <vscale x 1 x bfloat> undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV2BF16 = call <vscale x 2 x bfloat> @llvm.fma.nxv2bf16(<vscale x 2 x bfloat> undef, <vscale x 2 x bfloat> undef, <vscale x 2 x bfloat> undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV4BF16 = call <vscale x 4 x bfloat> @llvm.fma.nxv4bf16(<vscale x 4 x bfloat> undef, <vscale x 4 x bfloat> undef, <vscale x 4 x bfloat> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV8BF16 = call <vscale x 8 x bfloat> @llvm.fma.nxv8bf16(<vscale x 8 x bfloat> undef, <vscale x 8 x bfloat> undef, <vscale x 8 x bfloat> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV16BF16 = call <vscale x 16 x bfloat> @llvm.fma.nxv16bf16(<vscale x 16 x bfloat> undef, <vscale x 16 x bfloat> undef, <vscale x 16 x bfloat> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V1F32 = call <1 x float> @llvm.fma.v1f32(<1 x float> undef, <1 x float> undef, <1 x float> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2F32 = call <2 x float> @llvm.fma.v2f32(<2 x float> undef, <2 x float> undef, <2 x float> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4F32 = call <4 x float> @llvm.fma.v4f32(<4 x float> undef, <4 x float> undef, <4 x float> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %NXV8BF16 = call <vscale x 8 x bfloat> @llvm.fma.nxv8bf16(<vscale x 8 x bfloat> undef, <vscale x 8 x bfloat> undef, <vscale x 8 x bfloat> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %NXV16BF16 = call <vscale x 16 x bfloat> @llvm.fma.nxv16bf16(<vscale x 16 x bfloat> undef, <vscale x 16 x bfloat> undef, <vscale x 16 x bfloat> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1F32 = call <1 x float> @llvm.fma.v1f32(<1 x float> undef, <1 x float> undef, <1 x float> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2F32 = call <2 x float> @llvm.fma.v2f32(<2 x float> undef, <2 x float> undef, <2 x float> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4F32 = call <4 x float> @llvm.fma.v4f32(<4 x float> undef, <4 x float> undef, <4 x float> undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8F32 = call <8 x float> @llvm.fma.v8f32(<8 x float> undef, <8 x float> undef, <8 x float> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16F32 = call <16 x float> @llvm.fma.v16f32(<16 x float> undef, <16 x float> undef, <16 x float> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16F32 = call <16 x float> @llvm.fma.v16f32(<16 x float> undef, <16 x float> undef, <16 x float> undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV1F32 = call <vscale x 1 x float> @llvm.fma.nxv1f32(<vscale x 1 x float> undef, <vscale x 1 x float> undef, <vscale x 1 x float> undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV2F32 = call <vscale x 2 x float> @llvm.fma.nxv2f32(<vscale x 2 x float> undef, <vscale x 2 x float> undef, <vscale x 2 x float> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV4F32 = call <vscale x 4 x float> @llvm.fma.nxv4f32(<vscale x 4 x float> undef, <vscale x 4 x float> undef, <vscale x 4 x float> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV8F32 = call <vscale x 8 x float> @llvm.fma.nxv8f32(<vscale x 8 x float> undef, <vscale x 8 x float> undef, <vscale x 8 x float> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV16F32 = call <vscale x 16 x float> @llvm.fma.nxv16f32(<vscale x 16 x float> undef, <vscale x 16 x float> undef, <vscale x 16 x float> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V1F64 = call <1 x double> @llvm.fma.v1f64(<1 x double> undef, <1 x double> undef, <1 x double> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2F64 = call <2 x double> @llvm.fma.v2f64(<2 x double> undef, <2 x double> undef, <2 x double> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %NXV4F32 = call <vscale x 4 x float> @llvm.fma.nxv4f32(<vscale x 4 x float> undef, <vscale x 4 x float> undef, <vscale x 4 x float> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %NXV8F32 = call <vscale x 8 x float> @llvm.fma.nxv8f32(<vscale x 8 x float> undef, <vscale x 8 x float> undef, <vscale x 8 x float> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %NXV16F32 = call <vscale x 16 x float> @llvm.fma.nxv16f32(<vscale x 16 x float> undef, <vscale x 16 x float> undef, <vscale x 16 x float> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1F64 = call <1 x double> @llvm.fma.v1f64(<1 x double> undef, <1 x double> undef, <1 x double> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2F64 = call <2 x double> @llvm.fma.v2f64(<2 x double> undef, <2 x double> undef, <2 x double> undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4F64 = call <4 x double> @llvm.fma.v4f64(<4 x double> undef, <4 x double> undef, <4 x double> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8F64 = call <8 x double> @llvm.fma.v8f64(<8 x double> undef, <8 x double> undef, <8 x double> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8F64 = call <8 x double> @llvm.fma.v8f64(<8 x double> undef, <8 x double> undef, <8 x double> undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV1F64 = call <vscale x 1 x double> @llvm.fma.nxv1f64(<vscale x 1 x double> undef, <vscale x 1 x double> undef, <vscale x 1 x double> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV2F64 = call <vscale x 2 x double> @llvm.fma.nxv2f64(<vscale x 2 x double> undef, <vscale x 2 x double> undef, <vscale x 2 x double> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV4F64 = call <vscale x 4 x double> @llvm.fma.nxv4f64(<vscale x 4 x double> undef, <vscale x 4 x double> undef, <vscale x 4 x double> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV8F64 = call <vscale x 8 x double> @llvm.fma.nxv8f64(<vscale x 8 x double> undef, <vscale x 8 x double> undef, <vscale x 8 x double> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %NXV2F64 = call <vscale x 2 x double> @llvm.fma.nxv2f64(<vscale x 2 x double> undef, <vscale x 2 x double> undef, <vscale x 2 x double> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %NXV4F64 = call <vscale x 4 x double> @llvm.fma.nxv4f64(<vscale x 4 x double> undef, <vscale x 4 x double> undef, <vscale x 4 x double> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %NXV8F64 = call <vscale x 8 x double> @llvm.fma.nxv8f64(<vscale x 8 x double> undef, <vscale x 8 x double> undef, <vscale x 8 x double> undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
   %BF16 = call bfloat @llvm.fma.bf16(bfloat undef, bfloat undef, bfloat undef)
@@ -1449,37 +1449,21 @@ define void @fma() {
 }
 
 define void @fma_f16() {
-; ZVFH-LABEL: 'fma_f16'
-; ZVFH-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F16 = call half @llvm.fma.f16(half undef, half undef, half undef)
-; ZVFH-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V1F16 = call <1 x half> @llvm.fma.v1f16(<1 x half> undef, <1 x half> undef, <1 x half> undef)
-; ZVFH-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2F16 = call <2 x half> @llvm.fma.v2f16(<2 x half> undef, <2 x half> undef, <2 x half> undef)
-; ZVFH-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4F16 = call <4 x half> @llvm.fma.v4f16(<4 x half> undef, <4 x half> undef, <4 x half> undef)
-; ZVFH-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8F16 = call <8 x half> @llvm.fma.v8f16(<8 x half> undef, <8 x half> undef, <8 x half> undef)
-; ZVFH-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16F16 = call <16 x half> @llvm.fma.v16f16(<16 x half> undef, <16 x half> undef, <16 x half> undef)
-; ZVFH-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32F16 = call <32 x half> @llvm.fma.v32f16(<32 x half> undef, <32 x half> undef, <32 x half> undef)
-; ZVFH-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV1F16 = call <vscale x 1 x half> @llvm.fma.nxv1f16(<vscale x 1 x half> undef, <vscale x 1 x half> undef, <vscale x 1 x half> undef)
-; ZVFH-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV2F16 = call <vscale x 2 x half> @llvm.fma.nxv2f16(<vscale x 2 x half> undef, <vscale x 2 x half> undef, <vscale x 2 x half> undef)
-; ZVFH-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV4F16 = call <vscale x 4 x half> @llvm.fma.nxv4f16(<vscale x 4 x half> undef, <vscale x 4 x half> undef, <vscale x 4 x half> undef)
-; ZVFH-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV8F16 = call <vscale x 8 x half> @llvm.fma.nxv8f16(<vscale x 8 x half> undef, <vscale x 8 x half> undef, <vscale x 8 x half> undef)
-; ZVFH-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV16F16 = call <vscale x 16 x half> @llvm.fma.nxv16f16(<vscale x 16 x half> undef, <vscale x 16 x half> undef, <vscale x 16 x half> undef)
-; ZVFH-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV32F16 = call <vscale x 32 x half> @llvm.fma.nxv32f16(<vscale x 32 x half> undef, <vscale x 32 x half> undef, <vscale x 32 x half> undef)
-; ZVFH-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
-;
-; ZVFHMIN-LABEL: 'fma_f16'
-; ZVFHMIN-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F16 = call half @llvm.fma.f16(half undef, half undef, half undef)
-; ZVFHMIN-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1F16 = call <1 x half> @llvm.fma.v1f16(<1 x half> undef, <1 x half> undef, <1 x half> undef)
-; ZVFHMIN-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2F16 = call <2 x half> @llvm.fma.v2f16(<2 x half> undef, <2 x half> undef, <2 x half> undef)
-; ZVFHMIN-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4F16 = call <4 x half> @llvm.fma.v4f16(<4 x half> undef, <4 x half> undef, <4 x half> undef)
-; ZVFHMIN-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8F16 = call <8 x half> @llvm.fma.v8f16(<8 x half> undef, <8 x half> undef, <8 x half> undef)
-; ZVFHMIN-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16F16 = call <16 x half> @llvm.fma.v16f16(<16 x half> undef, <16 x half> undef, <16 x half> undef)
-; ZVFHMIN-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V32F16 = call <32 x half> @llvm.fma.v32f16(<32 x half> undef, <32 x half> undef, <32 x half> undef)
-; ZVFHMIN-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV1F16 = call <vscale x 1 x half> @llvm.fma.nxv1f16(<vscale x 1 x half> undef, <vscale x 1 x half> undef, <vscale x 1 x half> undef)
-; ZVFHMIN-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV2F16 = call <vscale x 2 x half> @llvm.fma.nxv2f16(<vscale x 2 x half> undef, <vscale x 2 x half> undef, <vscale x 2 x half> undef)
-; ZVFHMIN-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV4F16 = call <vscale x 4 x half> @llvm.fma.nxv4f16(<vscale x 4 x half> undef, <vscale x 4 x half> undef, <vscale x 4 x half> undef)
-; ZVFHMIN-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV8F16 = call <vscale x 8 x half> @llvm.fma.nxv8f16(<vscale x 8 x half> undef, <vscale x 8 x half> undef, <vscale x 8 x half> undef)
-; ZVFHMIN-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV16F16 = call <vscale x 16 x half> @llvm.fma.nxv16f16(<vscale x 16 x half> undef, <vscale x 16 x half> undef, <vscale x 16 x half> undef)
-; ZVFHMIN-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %NXV32F16 = call <vscale x 32 x half> @llvm.fma.nxv32f16(<vscale x 32 x half> undef, <vscale x 32 x half> undef, <vscale x 32 x half> undef)
-; ZVFHMIN-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; CHECK-LABEL: 'fma_f16'
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F16 = call half @llvm.fma.f16(half undef, half undef, half undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1F16 = call <1 x half> @llvm.fma.v1f16(<1 x half> undef, <1 x half> undef, <1 x half> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2F16 = call <2 x half> @llvm.fma.v2f16(<2 x half> undef, <2 x half> undef, <2 x half> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4F16 = call <4 x half> @llvm.fma.v4f16(<4 x half> undef, <4 x half> undef, <4 x half> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8F16 = call <8 x half> @llvm.fma.v8f16(<8 x half> undef, <8 x half> undef, <8 x half> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16F16 = call <16 x half> @llvm.fma.v16f16(<16 x half> undef, <16 x half> undef, <16 x half> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32F16 = call <32 x half> @llvm.fma.v32f16(<32 x half> undef, <32 x half> undef, <32 x half> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV1F16 = call <vscale x 1 x half> @llvm.fma.nxv1f16(<vscale x 1 x half> undef, <vscale x 1 x half> undef, <vscale x 1 x half> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV2F16 = call <vscale x 2 x half> @llvm.fma.nxv2f16(<vscale x 2 x half> undef, <vscale x 2 x half> undef, <vscale x 2 x half> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV4F16 = call <vscale x 4 x half> @llvm.fma.nxv4f16(<vscale x 4 x half> undef, <vscale x 4 x half> undef, <vscale x 4 x half> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %NXV8F16 = call <vscale x 8 x half> @llvm.fma.nxv8f16(<vscale x 8 x half> undef, <vscale x 8 x half> undef, <vscale x 8 x half> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %NXV16F16 = call <vscale x 16 x half> @llvm.fma.nxv16f16(<vscale x 16 x half> undef, <vscale x 16 x half> undef, <vscale x 16 x half> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %NXV32F16 = call <vscale x 32 x half> @llvm.fma.nxv32f16(<vscale x 32 x half> undef, <vscale x 32 x half> undef, <vscale x 32 x half> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
   %F16 = call half @llvm.fma.f16(half undef, half undef, half undef)
 
@@ -1508,30 +1492,30 @@ define void @fmuladd() {
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %4 = call <2 x bfloat> @llvm.fmuladd.v2bf16(<2 x bfloat> undef, <2 x bfloat> undef, <2 x bfloat> undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %5 = call <4 x bfloat> @llvm.fmuladd.v4bf16(<4 x bfloat> undef, <4 x bfloat> undef, <4 x bfloat> undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %6 = call <8 x bfloat> @llvm.fmuladd.v8bf16(<8 x bfloat> undef, <8 x bfloat> undef, <8 x bfloat> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %7 = call <16 x bfloat> @llvm.fmuladd.v16bf16(<16 x bfloat> undef, <16 x bfloat> undef, <16 x bfloat> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %8 = call <2 x float> @llvm.fmuladd.v2f32(<2 x float> undef, <2 x float> undef, <2 x float> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %9 = call <4 x float> @llvm.fmuladd.v4f32(<4 x float> undef, <4 x float> undef, <4 x float> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %7 = call <16 x bfloat> @llvm.fmuladd.v16bf16(<16 x bfloat> undef, <16 x bfloat> undef, <16 x bfloat> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %8 = call <2 x float> @llvm.fmuladd.v2f32(<2 x float> undef, <2 x float> undef, <2 x float> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %9 = call <4 x float> @llvm.fmuladd.v4f32(<4 x float> undef, <4 x float> undef, <4 x float> undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %10 = call <8 x float> @llvm.fmuladd.v8f32(<8 x float> undef, <8 x float> undef, <8 x float> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %11 = call <16 x float> @llvm.fmuladd.v16f32(<16 x float> undef, <16 x float> undef, <16 x float> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %12 = call <2 x double> @llvm.fmuladd.v2f64(<2 x double> undef, <2 x double> undef, <2 x double> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %11 = call <16 x float> @llvm.fmuladd.v16f32(<16 x float> undef, <16 x float> undef, <16 x float> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %12 = call <2 x double> @llvm.fmuladd.v2f64(<2 x double> undef, <2 x double> undef, <2 x double> undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %13 = call <4 x double> @llvm.fmuladd.v4f64(<4 x double> undef, <4 x double> undef, <4 x double> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %14 = call <8 x double> @llvm.fmuladd.v8f64(<8 x double> undef, <8 x double> undef, <8 x double> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %15 = call <16 x double> @llvm.fmuladd.v16f64(<16 x double> undef, <16 x double> undef, <16 x double> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %14 = call <8 x double> @llvm.fmuladd.v8f64(<8 x double> undef, <8 x double> undef, <8 x double> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %15 = call <16 x double> @llvm.fmuladd.v16f64(<16 x double> undef, <16 x double> undef, <16 x double> undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %16 = call <vscale x 1 x bfloat> @llvm.fmuladd.nxv1bf16(<vscale x 1 x bfloat> undef, <vscale x 1 x bfloat> undef, <vscale x 1 x bfloat> undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %17 = call <vscale x 2 x bfloat> @llvm.fmuladd.nxv2bf16(<vscale x 2 x bfloat> undef, <vscale x 2 x bfloat> undef, <vscale x 2 x bfloat> undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %18 = call <vscale x 4 x bfloat> @llvm.fmuladd.nxv4bf16(<vscale x 4 x bfloat> undef, <vscale x 4 x bfloat> undef, <vscale x 4 x bfloat> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %19 = call <vscale x 8 x bfloat> @llvm.fmuladd.nxv8bf16(<vscale x 8 x bfloat> undef, <vscale x 8 x bfloat> undef, <vscale x 8 x bfloat> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %20 = call <vscale x 16 x bfloat> @llvm.fmuladd.nxv16bf16(<vscale x 16 x bfloat> undef, <vscale x 16 x bfloat> undef, <vscale x 16 x bfloat> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %19 = call <vscale x 8 x bfloat> @llvm.fmuladd.nxv8bf16(<vscale x 8 x bfloat> undef, <vscale x 8 x bfloat> undef, <vscale x 8 x bfloat> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %20 = call <vscale x 16 x bfloat> @llvm.fmuladd.nxv16bf16(<vscale x 16 x bfloat> undef, <vscale x 16 x bfloat> undef, <vscale x 16 x bfloat> undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %21 = call <vscale x 1 x float> @llvm.fmuladd.nxv1f32(<vscale x 1 x float> undef, <vscale x 1 x float> undef, <vscale x 1 x float> undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %22 = call <vscale x 2 x float> @llvm.fmuladd.nxv2f32(<vscale x 2 x float> undef, <vscale x 2 x float> undef, <vscale x 2 x float> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %23 = call <vscale x 4 x float> @llvm.fmuladd.nxv4f32(<vscale x 4 x float> undef, <vscale x 4 x float> undef, <vscale x 4 x float> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %24 = call <vscale x 8 x float> @llvm.fmuladd.nxv8f32(<vscale x 8 x float> undef, <vscale x 8 x float> undef, <vscale x 8 x float> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %25 = call <vscale x 16 x float> @llvm.fmuladd.nxv16f32(<vscale x 16 x float> undef, <vscale x 16 x float> undef, <vscale x 16 x float> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %23 = call <vscale x 4 x float> @llvm.fmuladd.nxv4f32(<vscale x 4 x float> undef, <vscale x 4 x float> undef, <vscale x 4 x float> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %24 = call <vscale x 8 x float> @llvm.fmuladd.nxv8f32(<vscale x 8 x float> undef, <vscale x 8 x float> undef, <vscale x 8 x float> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %25 = call <vscale x 16 x float> @llvm.fmuladd.nxv16f32(<vscale x 16 x float> undef, <vscale x 16 x float> undef, <vscale x 16 x float> undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %26 = call <vscale x 1 x double> @llvm.fmuladd.nxv1f64(<vscale x 1 x double> undef, <vscale x 1 x double> undef, <vscale x 1 x double> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %27 = call <vscale x 2 x double> @llvm.fmuladd.nxv2f64(<vscale x 2 x double> undef, <vscale x 2 x double> undef, <vscale x 2 x double> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %28 = call <vscale x 4 x double> @llvm.fmuladd.nxv4f64(<vscale x 4 x double> undef, <vscale x 4 x double> undef, <vscale x 4 x double> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %29 = call <vscale x 8 x double> @llvm.fmuladd.nxv8f64(<vscale x 8 x double> undef, <vscale x 8 x double> undef, <vscale x 8 x double> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %30 = call <vscale x 16 x double> @llvm.fmuladd.nxv16f64(<vscale x 16 x double> undef, <vscale x 16 x double> undef, <vscale x 16 x double> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %27 = call <vscale x 2 x double> @llvm.fmuladd.nxv2f64(<vscale x 2 x double> undef, <vscale x 2 x double> undef, <vscale x 2 x double> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %28 = call <vscale x 4 x double> @llvm.fmuladd.nxv4f64(<vscale x 4 x double> undef, <vscale x 4 x double> undef, <vscale x 4 x double> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %29 = call <vscale x 8 x double> @llvm.fmuladd.nxv8f64(<vscale x 8 x double> undef, <vscale x 8 x double> undef, <vscale x 8 x double> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %30 = call <vscale x 16 x double> @llvm.fmuladd.nxv16f64(<vscale x 16 x double> undef, <vscale x 16 x double> undef, <vscale x 16 x double> undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
   call bfloat @llvm.fmuladd.bf16(bfloat undef, bfloat undef, bfloat undef)
@@ -1568,31 +1552,18 @@ define void @fmuladd() {
 }
 
 define void @fmuladd_f16() {
-; ZVFH-LABEL: 'fmuladd_f16'
-; ZVFH-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %1 = call half @llvm.fmuladd.f16(half undef, half undef, half undef)
-; ZVFH-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %2 = call <2 x half> @llvm.fmuladd.v2f16(<2 x half> undef, <2 x half> undef, <2 x half> undef)
-; ZVFH-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %3 = call <4 x half> @llvm.fmuladd.v4f16(<4 x half> undef, <4 x half> undef, <4 x half> undef)
-; ZVFH-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %4 = call <8 x half> @llvm.fmuladd.v8f16(<8 x half> undef, <8 x half> undef, <8 x half> undef)
-; ZVFH-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %5 = call <16 x half> @llvm.fmuladd.v16f16(<16 x half> undef, <16 x half> undef, <16 x half> undef)
-; ZVFH-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %6 = call <vscale x 1 x half> @llvm.fmuladd.nxv1f16(<vscale x 1 x half> undef, <vscale x 1 x half> undef, <vscale x 1 x half> undef)
-; ZVFH-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %7 = call <vscale x 2 x half> @llvm.fmuladd.nxv2f16(<vscale x 2 x half> undef, <vscale x 2 x half> undef, <vscale x 2 x half> undef)
-; ZVFH-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %8 = call <vscale x 4 x half> @llvm.fmuladd.nxv4f16(<vscale x 4 x half> undef, <vscale x 4 x half> undef, <vscale x 4 x half> undef)
-; ZVFH-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %9 = call <vscale x 8 x half> @llvm.fmuladd.nxv8f16(<vscale x 8 x half> undef, <vscale x 8 x half> undef, <vscale x 8 x half> undef)
-; ZVFH-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %10 = call <vscale x 16 x half> @llvm.fmuladd.nxv16f16(<vscale x 16 x half> undef, <vscale x 16 x half> undef, <vscale x 16 x half> undef)
-; ZVFH-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
-;
-; ZVFHMIN-LABEL: 'fmuladd_f16'
-; ZVFHMIN-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %1 = call half @llvm.fmuladd.f16(half undef, half undef, half undef)
-; ZVFHMIN-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %2 = call <2 x half> @llvm.fmuladd.v2f16(<2 x half> undef, <2 x half> undef, <2 x half> undef)
-; ZVFHMIN-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %3 = call <4 x half> @llvm.fmuladd.v4f16(<4 x half> undef, <4 x half> undef, <4 x half> undef)
-; ZVFHMIN-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %4 = call <8 x half> @llvm.fmuladd.v8f16(<8 x half> undef, <8 x half> undef, <8 x half> undef)
-; ZVFHMIN-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %5 = call <16 x half> @llvm.fmuladd.v16f16(<16 x half> undef, <16 x half> undef, <16 x half> undef)
-; ZVFHMIN-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %6 = call <vscale x 1 x half> @llvm.fmuladd.nxv1f16(<vscale x 1 x half> undef, <vscale x 1 x half> undef, <vscale x 1 x half> undef)
-; ZVFHMIN-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %7 = call <vscale x 2 x half> @llvm.fmuladd.nxv2f16(<vscale x 2 x half> undef, <vscale x 2 x half> undef, <vscale x 2 x half> undef)
-; ZVFHMIN-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %8 = call <vscale x 4 x half> @llvm.fmuladd.nxv4f16(<vscale x 4 x half> undef, <vscale x 4 x half> undef, <vscale x 4 x half> undef)
-; ZVFHMIN-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %9 = call <vscale x 8 x half> @llvm.fmuladd.nxv8f16(<vscale x 8 x half> undef, <vscale x 8 x half> undef, <vscale x 8 x half> undef)
-; ZVFHMIN-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %10 = call <vscale x 16 x half> @llvm.fmuladd.nxv16f16(<vscale x 16 x half> undef, <vscale x 16 x half> undef, <vscale x 16 x half> undef)
-; ZVFHMIN-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; CHECK-LABEL: 'fmuladd_f16'
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %1 = call half @llvm.fmuladd.f16(half undef, half undef, half undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %2 = call <2 x half> @llvm.fmuladd.v2f16(<2 x half> undef, <2 x half> undef, <2 x half> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %3 = call <4 x half> @llvm.fmuladd.v4f16(<4 x half> undef, <4 x half> undef, <4 x half> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %4 = call <8 x half> @llvm.fmuladd.v8f16(<8 x half> undef, <8 x half> undef, <8 x half> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %5 = call <16 x half> @llvm.fmuladd.v16f16(<16 x half> undef, <16 x half> undef, <16 x half> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %6 = call <vscale x 1 x half> @llvm.fmuladd.nxv1f16(<vscale x 1 x half> undef, <vscale x 1 x half> undef, <vscale x 1 x half> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %7 = call <vscale x 2 x half> @llvm.fmuladd.nxv2f16(<vscale x 2 x half> undef, <vscale x 2 x half> undef, <vscale x 2 x half> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %8 = call <vscale x 4 x half> @llvm.fmuladd.nxv4f16(<vscale x 4 x half> undef, <vscale x 4 x half> undef, <vscale x 4 x half> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %9 = call <vscale x 8 x half> @llvm.fmuladd.nxv8f16(<vscale x 8 x half> undef, <vscale x 8 x half> undef, <vscale x 8 x half> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %10 = call <vscale x 16 x half> @llvm.fmuladd.nxv16f16(<vscale x 16 x half> undef, <vscale x 16 x half> undef, <vscale x 16 x half> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
   call half @llvm.fmuladd.f16(half undef, half undef, half undef)
   call <2 x half> @llvm.fmuladd.v2f16(<2 x half> undef, <2 x half> undef, <2 x half> undef)
diff --git a/llvm/test/Analysis/CostModel/RISCV/gep.ll b/llvm/test/Analysis/CostModel/RISCV/gep.ll
index 44c0b5bb71070..99c74c4635fc7 100644
--- a/llvm/test/Analysis/CostModel/RISCV/gep.ll
+++ b/llvm/test/Analysis/CostModel/RISCV/gep.ll
@@ -268,7 +268,7 @@ define void @non_foldable_vector_uses(ptr %base, <2 x ptr> %base.vec) {
 ; RVI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %3 = getelementptr i8, <2 x ptr> %base.vec, <2 x i32> <i32 42, i32 43>
 ; RVI-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %x3 = call <2 x i8> @llvm.masked.gather.v2i8.v2p0(<2 x ptr> %3, i32 1, <2 x i1> undef, <2 x i8> undef)
 ; RVI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %4 = getelementptr i8, ptr %base, i32 42
-; RVI-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %x4 = call <2 x i8> @llvm.masked.expandload.v2i8(ptr %4, <2 x i1> undef, <2 x i8> undef)
+; RVI-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %x4 = call <2 x i8> @llvm.masked.expandload.v2i8(ptr %4, <2 x i1> undef, <2 x i8> undef)
 ; RVI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %5 = getelementptr i8, ptr %base, i32 42
 ; RVI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %x5 = call <2 x i8> @llvm.vp.load.v2i8.p0(ptr %5, <2 x i1> undef, i32 undef)
 ; RVI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %6 = getelementptr i8, ptr %base, i32 42
@@ -280,7 +280,7 @@ define void @non_foldable_vector_uses(ptr %base, <2 x ptr> %base.vec) {
 ; RVI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %9 = getelementptr i8, <2 x ptr> %base.vec, <2 x i32> <i32 42, i32 43>
 ; RVI-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: call void @llvm.masked.scatter.v2i8.v2p0(<2 x i8> undef, <2 x ptr> %9, i32 1, <2 x i1> undef)
 ; RVI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %10 = getelementptr i8, ptr %base, i32 42
-; RVI-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: call void @llvm.masked.compressstore.v2i8(<2 x i8> undef, ptr %10, <2 x i1> undef)
+; RVI-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: call void @llvm.masked.compressstore.v2i8(<2 x i8> undef, ptr %10, <2 x i1> undef)
 ; RVI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %11 = getelementptr i8, ptr %base, i32 42
 ; RVI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.vp.store.v2i8.p0(<2 x i8> undef, ptr %11, <2 x i1> undef, i32 undef)
 ; RVI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %12 = getelementptr i8, ptr %base, i32 42
@@ -338,7 +338,7 @@ define void @foldable_vector_uses(ptr %base, <2 x ptr> %base.vec) {
 ; RVI-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %3 = getelementptr i8, <2 x ptr> %base.vec, <2 x i32> zeroinitializer
 ; RVI-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %x3 = call <2 x i8> @llvm.masked.gather.v2i8.v2p0(<2 x ptr> %3, i32 1, <2 x i1> undef, <2 x i8> undef)
 ; RVI-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %4 = getelementptr i8, ptr %base, i32 0
-; RVI-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %x4 = call <2 x i8> @llvm.masked.expandload.v2i8(ptr %4, <2 x i1> undef, <2 x i8> undef)
+; RVI-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %x4 = call <2 x i8> @llvm.masked.expandload.v2i8(ptr %4, <2 x i1> undef, <2 x i8> undef)
 ; RVI-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %5 = getelementptr i8, ptr %base, i32 0
 ; RVI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %x5 = call <2 x i8> @llvm.vp.load.v2i8.p0(ptr %5, <2 x i1> undef, i32 undef)
 ; RVI-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %6 = getelementptr i8, ptr %base, i32 0
@@ -350,7 +350,7 @@ define void @foldable_vector_uses(ptr %base, <2 x ptr> %base.vec) {
 ; RVI-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %9 = getelementptr i8, <2 x ptr> %base.vec, <2 x i32> zeroinitializer
 ; RVI-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: call void @llvm.masked.scatter.v2i8.v2p0(<2 x i8> undef, <2 x ptr> %9, i32 1, <2 x i1> undef)
 ; RVI-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %10 = getelementptr i8, ptr %base, i32 0
-; RVI-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: call void @llvm.masked.compressstore.v2i8(<2 x i8> undef, ptr %10, <2 x i1> undef)
+; RVI-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: call void @llvm.masked.compressstore.v2i8(<2 x i8> undef, ptr %10, <2 x i1> undef)
 ; RVI-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %11 = getelementptr i8, ptr %base, i32 0
 ; RVI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.vp.store.v2i8.p0(<2 x i8> undef, ptr %11, <2 x i1> undef, i32 undef)
 ; RVI-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %12 = getelementptr i8, ptr %base, i32 0
diff --git a/llvm/test/Analysis/CostModel/RISCV/rvv-expandload-compressstore.ll b/llvm/test/Analysis/CostModel/RISCV/rvv-expandload-compressstore.ll
new file mode 100644
index 0000000000000..ea5f3444fa248
--- /dev/null
+++ b/llvm/test/Analysis/CostModel/RISCV/rvv-expandload-compressstore.ll
@@ -0,0 +1,177 @@
+; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py
+; RUN: opt < %s -passes="print<cost-model>" 2>&1 -disable-output -S -mtriple=riscv64 -mattr=+v,+zvfhmin,+zvfbfmin | FileCheck %s
+; RUN: opt < %s -passes="print<cost-model>" 2>&1 -disable-output -S -mtriple=riscv64 -mattr=+v,+zvfhmin,+zvfbfmin --type-based-intrinsic-cost=true | FileCheck %s --check-prefixes=TYPEBASED
+
+define void @expand_load() {
+; CHECK-LABEL: 'expand_load'
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %t1 = call <2 x i8> @llvm.masked.expandload.v2i8(ptr poison, <2 x i1> poison, <2 x i8> poison)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %t2 = call <4 x i8> @llvm.masked.expandload.v4i8(ptr poison, <4 x i1> poison, <4 x i8> poison)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %t3 = call <8 x i8> @llvm.masked.expandload.v8i8(ptr poison, <8 x i1> poison, <8 x i8> poison)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %t4 = call <16 x i8> @llvm.masked.expandload.v16i8(ptr poison, <16 x i1> poison, <16 x i8> poison)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %t5 = call <2 x i64> @llvm.masked.expandload.v2i64(ptr align 8 poison, <2 x i1> poison, <2 x i64> poison)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %t6 = call <4 x i64> @llvm.masked.expandload.v4i64(ptr align 8 poison, <4 x i1> poison, <4 x i64> poison)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %t7 = call <8 x i64> @llvm.masked.expandload.v8i64(ptr align 8 poison, <8 x i1> poison, <8 x i64> poison)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 96 for instruction: %t8 = call <16 x i64> @llvm.masked.expandload.v16i64(ptr align 8 poison, <16 x i1> poison, <16 x i64> poison)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %t9 = call <2 x i64> @llvm.masked.expandload.v2i64(ptr poison, <2 x i1> poison, <2 x i64> poison)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %t10 = call <4 x i64> @llvm.masked.expandload.v4i64(ptr poison, <4 x i1> poison, <4 x i64> poison)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 38 for instruction: %t11 = call <8 x i64> @llvm.masked.expandload.v8i64(ptr poison, <8 x i1> poison, <8 x i64> poison)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 78 for instruction: %t12 = call <16 x i64> @llvm.masked.expandload.v16i64(ptr poison, <16 x i1> poison, <16 x i64> poison)
+; CHECK-NEXT:  Cost Model: Invalid cost for instruction: %t13 = call <vscale x 2 x i8> @llvm.masked.expandload.nxv2i8(ptr poison, <vscale x 2 x i1> poison, <vscale x 2 x i8> poison)
+; CHECK-NEXT:  Cost Model: Invalid cost for instruction: %t14 = call <vscale x 4 x i8> @llvm.masked.expandload.nxv4i8(ptr poison, <vscale x 4 x i1> poison, <vscale x 4 x i8> poison)
+; CHECK-NEXT:  Cost Model: Invalid cost for instruction: %t15 = call <vscale x 8 x i8> @llvm.masked.expandload.nxv8i8(ptr poison, <vscale x 8 x i1> poison, <vscale x 8 x i8> poison)
+; CHECK-NEXT:  Cost Model: Invalid cost for instruction: %t16 = call <vscale x 16 x i8> @llvm.masked.expandload.nxv16i8(ptr poison, <vscale x 16 x i1> poison, <vscale x 16 x i8> poison)
+; CHECK-NEXT:  Cost Model: Invalid cost for instruction: %t17 = call <vscale x 2 x i64> @llvm.masked.expandload.nxv2i64(ptr align 8 poison, <vscale x 2 x i1> poison, <vscale x 2 x i64> poison)
+; CHECK-NEXT:  Cost Model: Invalid cost for instruction: %t18 = call <vscale x 4 x i64> @llvm.masked.expandload.nxv4i64(ptr align 8 poison, <vscale x 4 x i1> poison, <vscale x 4 x i64> poison)
+; CHECK-NEXT:  Cost Model: Invalid cost for instruction: %t19 = call <vscale x 8 x i64> @llvm.masked.expandload.nxv8i64(ptr align 8 poison, <vscale x 8 x i1> poison, <vscale x 8 x i64> poison)
+; CHECK-NEXT:  Cost Model: Invalid cost for instruction: %t20 = call <vscale x 16 x i64> @llvm.masked.expandload.nxv16i64(ptr align 8 poison, <vscale x 16 x i1> poison, <vscale x 16 x i64> poison)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; TYPEBASED-LABEL: 'expand_load'
+; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %t1 = call <2 x i8> @llvm.masked.expandload.v2i8(ptr poison, <2 x i1> poison, <2 x i8> poison)
+; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 30 for instruction: %t2 = call <4 x i8> @llvm.masked.expandload.v4i8(ptr poison, <4 x i1> poison, <4 x i8> poison)
+; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 62 for instruction: %t3 = call <8 x i8> @llvm.masked.expandload.v8i8(ptr poison, <8 x i1> poison, <8 x i8> poison)
+; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 126 for instruction: %t4 = call <16 x i8> @llvm.masked.expandload.v16i8(ptr poison, <16 x i1> poison, <16 x i8> poison)
+; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %t5 = call <2 x i64> @llvm.masked.expandload.v2i64(ptr align 8 poison, <2 x i1> poison, <2 x i64> poison)
+; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %t6 = call <4 x i64> @llvm.masked.expandload.v4i64(ptr align 8 poison, <4 x i1> poison, <4 x i64> poison)
+; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 69 for instruction: %t7 = call <8 x i64> @llvm.masked.expandload.v8i64(ptr align 8 poison, <8 x i1> poison, <8 x i64> poison)
+; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 141 for instruction: %t8 = call <16 x i64> @llvm.masked.expandload.v16i64(ptr align 8 poison, <16 x i1> poison, <16 x i64> poison)
+; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %t9 = call <2 x i64> @llvm.masked.expandload.v2i64(ptr poison, <2 x i1> poison, <2 x i64> poison)
+; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %t10 = call <4 x i64> @llvm.masked.expandload.v4i64(ptr poison, <4 x i1> poison, <4 x i64> poison)
+; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 69 for instruction: %t11 = call <8 x i64> @llvm.masked.expandload.v8i64(ptr poison, <8 x i1> poison, <8 x i64> poison)
+; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 141 for instruction: %t12 = call <16 x i64> @llvm.masked.expandload.v16i64(ptr poison, <16 x i1> poison, <16 x i64> poison)
+; TYPEBASED-NEXT:  Cost Model: Invalid cost for instruction: %t13 = call <vscale x 2 x i8> @llvm.masked.expandload.nxv2i8(ptr poison, <vscale x 2 x i1> poison, <vscale x 2 x i8> poison)
+; TYPEBASED-NEXT:  Cost Model: Invalid cost for instruction: %t14 = call <vscale x 4 x i8> @llvm.masked.expandload.nxv4i8(ptr poison, <vscale x 4 x i1> poison, <vscale x 4 x i8> poison)
+; TYPEBASED-NEXT:  Cost Model: Invalid cost for instruction: %t15 = call <vscale x 8 x i8> @llvm.masked.expandload.nxv8i8(ptr poison, <vscale x 8 x i1> poison, <vscale x 8 x i8> poison)
+; TYPEBASED-NEXT:  Cost Model: Invalid cost for instruction: %t16 = call <vscale x 16 x i8> @llvm.masked.expandload.nxv16i8(ptr poison, <vscale x 16 x i1> poison, <vscale x 16 x i8> poison)
+; TYPEBASED-NEXT:  Cost Model: Invalid cost for instruction: %t17 = call <vscale x 2 x i64> @llvm.masked.expandload.nxv2i64(ptr align 8 poison, <vscale x 2 x i1> poison, <vscale x 2 x i64> poison)
+; TYPEBASED-NEXT:  Cost Model: Invalid cost for instruction: %t18 = call <vscale x 4 x i64> @llvm.masked.expandload.nxv4i64(ptr align 8 poison, <vscale x 4 x i1> poison, <vscale x 4 x i64> poison)
+; TYPEBASED-NEXT:  Cost Model: Invalid cost for instruction: %t19 = call <vscale x 8 x i64> @llvm.masked.expandload.nxv8i64(ptr align 8 poison, <vscale x 8 x i1> poison, <vscale x 8 x i64> poison)
+; TYPEBASED-NEXT:  Cost Model: Invalid cost for instruction: %t20 = call <vscale x 16 x i64> @llvm.masked.expandload.nxv16i64(ptr align 8 poison, <vscale x 16 x i1> poison, <vscale x 16 x i64> poison)
+; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+  %t1 = call <2 x i8> @llvm.masked.expandload.v2i8(ptr poison, <2 x i1> poison, <2 x i8> poison)
+  %t2 = call <4 x i8> @llvm.masked.expandload.v4i8(ptr poison, <4 x i1> poison, <4 x i8> poison)
+  %t3 = call <8 x i8> @llvm.masked.expandload.v8i8(ptr poison, <8 x i1> poison, <8 x i8> poison)
+  %t4 = call <16 x i8> @llvm.masked.expandload.v16i8(ptr poison, <16 x i1> poison, <16 x i8> poison)
+  %t5 = call <2 x i64> @llvm.masked.expandload.v2i64(ptr align(8) poison, <2 x i1> poison, <2 x i64> poison)
+  %t6 = call <4 x i64> @llvm.masked.expandload.v4i64(ptr align(8) poison, <4 x i1> poison, <4 x i64> poison)
+  %t7 = call <8 x i64> @llvm.masked.expandload.v8i64(ptr align(8) poison, <8 x i1> poison, <8 x i64> poison)
+  %t8 = call <16 x i64> @llvm.masked.expandload.v16i64(ptr align(8) poison, <16 x i1> poison, <16 x i64> poison)
+  %t9 = call <2 x i64> @llvm.masked.expandload.v2i64(ptr poison, <2 x i1> poison, <2 x i64> poison)
+  %t10 = call <4 x i64> @llvm.masked.expandload.v4i64(ptr poison, <4 x i1> poison, <4 x i64> poison)
+  %t11 = call <8 x i64> @llvm.masked.expandload.v8i64(ptr poison, <8 x i1> poison, <8 x i64> poison)
+  %t12 = call <16 x i64> @llvm.masked.expandload.v16i64(ptr poison, <16 x i1> poison, <16 x i64> poison)
+  %t13 = call <vscale x 2 x i8> @llvm.masked.expandload.nxv2i8(ptr poison, <vscale x 2 x i1> poison, <vscale x 2 x i8> poison)
+  %t14 = call <vscale x 4 x i8> @llvm.masked.expandload.nxv4i8(ptr poison, <vscale x 4 x i1> poison, <vscale x 4 x i8> poison)
+  %t15 = call <vscale x 8 x i8> @llvm.masked.expandload.nxv8i8(ptr poison, <vscale x 8 x i1> poison, <vscale x 8 x i8> poison)
+  %t16 = call <vscale x 16 x i8> @llvm.masked.expandload.nxv16i8(ptr poison, <vscale x 16 x i1> poison, <vscale x 16 x i8> poison)
+  %t17 = call <vscale x 2 x i64> @llvm.masked.expandload.nxv2i64(ptr align(8) poison, <vscale x 2 x i1> poison, <vscale x 2 x i64> poison)
+  %t18 = call <vscale x 4 x i64> @llvm.masked.expandload.nxv4i64(ptr align(8) poison, <vscale x 4 x i1> poison, <vscale x 4 x i64> poison)
+  %t19 = call <vscale x 8 x i64> @llvm.masked.expandload.nxv8i64(ptr align(8) poison, <vscale x 8 x i1> poison, <vscale x 8 x i64> poison)
+  %t20 = call <vscale x 16 x i64> @llvm.masked.expandload.nxv16i64(ptr align(8) poison, <vscale x 16 x i1> poison, <vscale x 16 x i64> poison)
+  ret void
+}
+
+define void @compress_store() {
+; CHECK-LABEL: 'compress_store'
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: call void @llvm.masked.compressstore.v2i8(<2 x i8> poison, ptr poison, <2 x i1> poison)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: call void @llvm.masked.compressstore.v4i8(<4 x i8> poison, ptr poison, <4 x i1> poison)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: call void @llvm.masked.compressstore.v8i8(<8 x i8> poison, ptr poison, <8 x i1> poison)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: call void @llvm.masked.compressstore.v16i8(<16 x i8> poison, ptr poison, <16 x i1> poison)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: call void @llvm.masked.compressstore.v2i64(<2 x i64> poison, ptr align 8 poison, <2 x i1> poison)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: call void @llvm.masked.compressstore.v4i64(<4 x i64> poison, ptr align 8 poison, <4 x i1> poison)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: call void @llvm.masked.compressstore.v8i64(<8 x i64> poison, ptr align 8 poison, <8 x i1> poison)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: call void @llvm.masked.compressstore.v16i64(<16 x i64> poison, ptr align 8 poison, <16 x i1> poison)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: call void @llvm.masked.compressstore.v2i64(<2 x i64> poison, ptr poison, <2 x i1> poison)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: call void @llvm.masked.compressstore.v4i64(<4 x i64> poison, ptr poison, <4 x i1> poison)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 38 for instruction: call void @llvm.masked.compressstore.v8i64(<8 x i64> poison, ptr poison, <8 x i1> poison)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 78 for instruction: call void @llvm.masked.compressstore.v16i64(<16 x i64> poison, ptr poison, <16 x i1> poison)
+; CHECK-NEXT:  Cost Model: Invalid cost for instruction: call void @llvm.masked.compressstore.nxv2i8(<vscale x 2 x i8> poison, ptr poison, <vscale x 2 x i1> poison)
+; CHECK-NEXT:  Cost Model: Invalid cost for instruction: call void @llvm.masked.compressstore.nxv4i8(<vscale x 4 x i8> poison, ptr poison, <vscale x 4 x i1> poison)
+; CHECK-NEXT:  Cost Model: Invalid cost for instruction: call void @llvm.masked.compressstore.nxv8i8(<vscale x 8 x i8> poison, ptr poison, <vscale x 8 x i1> poison)
+; CHECK-NEXT:  Cost Model: Invalid cost for instruction: call void @llvm.masked.compressstore.nxv16i8(<vscale x 16 x i8> poison, ptr poison, <vscale x 16 x i1> poison)
+; CHECK-NEXT:  Cost Model: Invalid cost for instruction: call void @llvm.masked.compressstore.nxv2i64(<vscale x 2 x i64> poison, ptr align 8 poison, <vscale x 2 x i1> poison)
+; CHECK-NEXT:  Cost Model: Invalid cost for instruction: call void @llvm.masked.compressstore.nxv4i64(<vscale x 4 x i64> poison, ptr align 8 poison, <vscale x 4 x i1> poison)
+; CHECK-NEXT:  Cost Model: Invalid cost for instruction: call void @llvm.masked.compressstore.nxv8i64(<vscale x 8 x i64> poison, ptr align 8 poison, <vscale x 8 x i1> poison)
+; CHECK-NEXT:  Cost Model: Invalid cost for instruction: call void @llvm.masked.compressstore.nxv16i64(<vscale x 16 x i64> poison, ptr align 8 poison, <vscale x 16 x i1> poison)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; TYPEBASED-LABEL: 'compress_store'
+; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: call void @llvm.masked.compressstore.v2i8(<2 x i8> poison, ptr poison, <2 x i1> poison)
+; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 26 for instruction: call void @llvm.masked.compressstore.v4i8(<4 x i8> poison, ptr poison, <4 x i1> poison)
+; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 54 for instruction: call void @llvm.masked.compressstore.v8i8(<8 x i8> poison, ptr poison, <8 x i1> poison)
+; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 110 for instruction: call void @llvm.masked.compressstore.v16i8(<16 x i8> poison, ptr poison, <16 x i1> poison)
+; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: call void @llvm.masked.compressstore.v2i64(<2 x i64> poison, ptr align 8 poison, <2 x i1> poison)
+; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 26 for instruction: call void @llvm.masked.compressstore.v4i64(<4 x i64> poison, ptr align 8 poison, <4 x i1> poison)
+; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 54 for instruction: call void @llvm.masked.compressstore.v8i64(<8 x i64> poison, ptr align 8 poison, <8 x i1> poison)
+; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 110 for instruction: call void @llvm.masked.compressstore.v16i64(<16 x i64> poison, ptr align 8 poison, <16 x i1> poison)
+; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: call void @llvm.masked.compressstore.v2i64(<2 x i64> poison, ptr poison, <2 x i1> poison)
+; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 26 for instruction: call void @llvm.masked.compressstore.v4i64(<4 x i64> poison, ptr poison, <4 x i1> poison)
+; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 54 for instruction: call void @llvm.masked.compressstore.v8i64(<8 x i64> poison, ptr poison, <8 x i1> poison)
+; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 110 for instruction: call void @llvm.masked.compressstore.v16i64(<16 x i64> poison, ptr poison, <16 x i1> poison)
+; TYPEBASED-NEXT:  Cost Model: Invalid cost for instruction: call void @llvm.masked.compressstore.nxv2i8(<vscale x 2 x i8> poison, ptr poison, <vscale x 2 x i1> poison)
+; TYPEBASED-NEXT:  Cost Model: Invalid cost for instruction: call void @llvm.masked.compressstore.nxv4i8(<vscale x 4 x i8> poison, ptr poison, <vscale x 4 x i1> poison)
+; TYPEBASED-NEXT:  Cost Model: Invalid cost for instruction: call void @llvm.masked.compressstore.nxv8i8(<vscale x 8 x i8> poison, ptr poison, <vscale x 8 x i1> poison)
+; TYPEBASED-NEXT:  Cost Model: Invalid cost for instruction: call void @llvm.masked.compressstore.nxv16i8(<vscale x 16 x i8> poison, ptr poison, <vscale x 16 x i1> poison)
+; TYPEBASED-NEXT:  Cost Model: Invalid cost for instruction: call void @llvm.masked.compressstore.nxv2i64(<vscale x 2 x i64> poison, ptr align 8 poison, <vscale x 2 x i1> poison)
+; TYPEBASED-NEXT:  Cost Model: Invalid cost for instruction: call void @llvm.masked.compressstore.nxv4i64(<vscale x 4 x i64> poison, ptr align 8 poison, <vscale x 4 x i1> poison)
+; TYPEBASED-NEXT:  Cost Model: Invalid cost for instruction: call void @llvm.masked.compressstore.nxv8i64(<vscale x 8 x i64> poison, ptr align 8 poison, <vscale x 8 x i1> poison)
+; TYPEBASED-NEXT:  Cost Model: Invalid cost for instruction: call void @llvm.masked.compressstore.nxv16i64(<vscale x 16 x i64> poison, ptr align 8 poison, <vscale x 16 x i1> poison)
+; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+  call void @llvm.masked.compressstore.v2i8(<2 x i8> poison, ptr poison, <2 x i1> poison)
+  call void @llvm.masked.compressstore.v4i8(<4 x i8> poison, ptr poison, <4 x i1> poison)
+  call void @llvm.masked.compressstore.v8i8(<8 x i8> poison, ptr poison, <8 x i1> poison)
+  call void @llvm.masked.compressstore.v16i8(<16 x i8> poison, ptr poison, <16 x i1> poison)
+  call void @llvm.masked.compressstore.v2i64(<2 x i64> poison, ptr align(8) poison, <2 x i1> poison)
+  call void @llvm.masked.compressstore.v4i64(<4 x i64> poison, ptr align(8) poison, <4 x i1> poison)
+  call void @llvm.masked.compressstore.v8i64(<8 x i64> poison, ptr align(8) poison, <8 x i1> poison)
+  call void @llvm.masked.compressstore.v16i64(<16 x i64> poison, ptr align(8) poison, <16 x i1> poison)
+  call void @llvm.masked.compressstore.v2i64(<2 x i64> poison, ptr poison, <2 x i1> poison)
+  call void @llvm.masked.compressstore.v4i64(<4 x i64> poison, ptr poison, <4 x i1> poison)
+  call void @llvm.masked.compressstore.v8i64(<8 x i64> poison, ptr poison, <8 x i1> poison)
+  call void @llvm.masked.compressstore.v16i64(<16 x i64> poison, ptr poison, <16 x i1> poison)
+  call void @llvm.masked.compressstore.nxv2i8(<vscale x 2 x i8> poison, ptr poison, <vscale x 2 x i1> poison)
+  call void @llvm.masked.compressstore.nxv4i8(<vscale x 4 x i8> poison, ptr poison, <vscale x 4 x i1> poison)
+  call void @llvm.masked.compressstore.nxv8i8(<vscale x 8 x i8> poison, ptr poison, <vscale x 8 x i1> poison)
+  call void @llvm.masked.compressstore.nxv16i8(<vscale x 16 x i8> poison, ptr poison, <vscale x 16 x i1> poison)
+  call void @llvm.masked.compressstore.nxv2i64(<vscale x 2 x i64> poison, ptr align(8) poison, <vscale x 2 x i1> poison)
+  call void @llvm.masked.compressstore.nxv4i64(<vscale x 4 x i64> poison, ptr align(8) poison, <vscale x 4 x i1> poison)
+  call void @llvm.masked.compressstore.nxv8i64(<vscale x 8 x i64> poison, ptr align(8) poison, <vscale x 8 x i1> poison)
+  call void @llvm.masked.compressstore.nxv16i64(<vscale x 16 x i64> poison, ptr align(8) poison, <vscale x 16 x i1> poison)
+  ret void
+}
+
+declare <2 x i8> @llvm.masked.expandload.v2i8(ptr, <2 x i1>, <2 x i8>)
+declare <4 x i8> @llvm.masked.expandload.v4i8(ptr, <4 x i1>, <4 x i8>)
+declare <8 x i8> @llvm.masked.expandload.v8i8(ptr, <8 x i1>, <8 x i8>)
+declare <16 x i8> @llvm.masked.expandload.v16i8(ptr, <16 x i1>, <16 x i8>)
+declare <2 x i64> @llvm.masked.expandload.v2i64(ptr, <2 x i1>, <2 x i64>)
+declare <4 x i64> @llvm.masked.expandload.v4i64(ptr, <4 x i1>, <4 x i64>)
+declare <8 x i64> @llvm.masked.expandload.v8i64(ptr, <8 x i1>, <8 x i64>)
+declare <16 x i64> @llvm.masked.expandload.v16i64(ptr, <16 x i1>, <16 x i64>)
+declare <vscale x 2 x i8> @llvm.masked.expandload.nxv2i8(ptr, <vscale x 2 x i1>, <vscale x 2 x i8>)
+declare <vscale x 4 x i8> @llvm.masked.expandload.nxv4i8(ptr, <vscale x 4 x i1>, <vscale x 4 x i8>)
+declare <vscale x 8 x i8> @llvm.masked.expandload.nxv8i8(ptr, <vscale x 8 x i1>, <vscale x 8 x i8>)
+declare <vscale x 16 x i8> @llvm.masked.expandload.nxv16i8(ptr, <vscale x 16 x i1>, <vscale x 16 x i8>)
+declare <vscale x 2 x i64> @llvm.masked.expandload.nxv2i64(ptr, <vscale x 2 x i1>, <vscale x 2 x i64>)
+declare <vscale x 4 x i64> @llvm.masked.expandload.nxv4i64(ptr, <vscale x 4 x i1>, <vscale x 4 x i64>)
+declare <vscale x 8 x i64> @llvm.masked.expandload.nxv8i64(ptr, <vscale x 8 x i1>, <vscale x 8 x i64>)
+declare <vscale x 16 x i64> @llvm.masked.expandload.nxv16i64(ptr, <vscale x 16 x i1>, <vscale x 16 x i64>)
+
+declare void @llvm.masked.compressstore.v2i8(<2 x i8>, ptr, <2 x i1>)
+declare void @llvm.masked.compressstore.v4i8(<4 x i8>, ptr, <4 x i1>)
+declare void @llvm.masked.compressstore.v8i8(<8 x i8>, ptr, <8 x i1>)
+declare void @llvm.masked.compressstore.v16i8(<16 x i8>, ptr, <16 x i1>)
+declare void @llvm.masked.compressstore.v2i64(<2 x i64>, ptr, <2 x i1>)
+declare void @llvm.masked.compressstore.v4i64(<4 x i64>, ptr, <4 x i1>)
+declare void @llvm.masked.compressstore.v8i64(<8 x i64>, ptr, <8 x i1>)
+declare void @llvm.masked.compressstore.v16i64(<16 x i64>, ptr, <16 x i1>)
+declare void @llvm.masked.compressstore.nxv2i8(<vscale x 2 x i8>, ptr, <vscale x 2 x i1>)
+declare void @llvm.masked.compressstore.nxv4i8(<vscale x 4 x i8>, ptr, <vscale x 4 x i1>)
+declare void @llvm.masked.compressstore.nxv8i8(<vscale x 8 x i8>, ptr, <vscale x 8 x i1>)
+declare void @llvm.masked.compressstore.nxv16i8(<vscale x 16 x i8>, ptr, <vscale x 16 x i1>)
+declare void @llvm.masked.compressstore.nxv2i64(<vscale x 2 x i64>, ptr, <vscale x 2 x i1>)
+declare void @llvm.masked.compressstore.nxv4i64(<vscale x 4 x i64>, ptr, <vscale x 4 x i1>)
+declare void @llvm.masked.compressstore.nxv8i64(<vscale x 8 x i64>, ptr, <vscale x 8 x i1>)
+declare void @llvm.masked.compressstore.nxv16i64(<vscale x 16 x i64>, ptr, <vscale x 16 x i1>)
diff --git a/llvm/test/Analysis/CostModel/RISCV/shuffle-exact-vlen.ll b/llvm/test/Analysis/CostModel/RISCV/shuffle-exact-vlen.ll
index cada8ab240cc7..c951184a31731 100644
--- a/llvm/test/Analysis/CostModel/RISCV/shuffle-exact-vlen.ll
+++ b/llvm/test/Analysis/CostModel/RISCV/shuffle-exact-vlen.ll
@@ -19,12 +19,12 @@ define <64 x i32> @interleave_v32i32(<32 x i32> %x, <32 x i32> %y) vscale_range(
 
 define void @extract(<64 x i32> %v, ptr %out) vscale_range(2,2) vscale_range(2,2) {
 ; CHECK-LABEL: 'extract'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 224 for instruction: %shuffle.i5 = shufflevector <64 x i32> %v, <64 x i32> poison, <64 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14, i32 16, i32 18, i32 20, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %shuffle.i5 = shufflevector <64 x i32> %v, <64 x i32> poison, <64 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14, i32 16, i32 18, i32 20, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: store <64 x i32> %shuffle.i5, ptr %out, align 4
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; CHECK-SIZE-LABEL: 'extract'
-; CHECK-SIZE-NEXT:  Cost Model: Found an estimated cost of 224 for instruction: %shuffle.i5 = shufflevector <64 x i32> %v, <64 x i32> poison, <64 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14, i32 16, i32 18, i32 20, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-SIZE-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %shuffle.i5 = shufflevector <64 x i32> %v, <64 x i32> poison, <64 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14, i32 16, i32 18, i32 20, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; CHECK-SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: store <64 x i32> %shuffle.i5, ptr %out, align 4
 ; CHECK-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
@@ -58,7 +58,7 @@ define void @shuffle() vscale_range(2,2) {
 ; CHECK-SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v11 = shufflevector <2 x i16> poison, <2 x i16> poison, <2 x i32> <i32 1, i32 0>
 ; CHECK-SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v12 = shufflevector <4 x i16> poison, <4 x i16> poison, <4 x i32> <i32 1, i32 3, i32 2, i32 0>
 ; CHECK-SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v13 = shufflevector <8 x i16> poison, <8 x i16> poison, <8 x i32> <i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 0, i32 1>
-; CHECK-SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v10b = shufflevector <16 x i16> poison, <16 x i16> poison, <16 x i32> <i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 0, i32 1>
+; CHECK-SIZE-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v10b = shufflevector <16 x i16> poison, <16 x i16> poison, <16 x i32> <i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 0, i32 1>
 ; CHECK-SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v14 = shufflevector <2 x i32> poison, <2 x i32> poison, <2 x i32> <i32 1, i32 0>
 ; CHECK-SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v15 = shufflevector <4 x i32> poison, <4 x i32> poison, <4 x i32> <i32 1, i32 3, i32 2, i32 0>
 ; CHECK-SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v16 = shufflevector <2 x float> poison, <2 x float> poison, <2 x i32> <i32 1, i32 0>
@@ -94,16 +94,16 @@ define void @concat() vscale_range(2,2) {
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i8 = shufflevector <8 x i8> poison, <8 x i8> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i16 = shufflevector <2 x i16> poison, <2 x i16> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i16 = shufflevector <4 x i16> poison, <4 x i16> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v16i16 = shufflevector <8 x i16> poison, <8 x i16> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v16i16 = shufflevector <8 x i16> poison, <8 x i16> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i32 = shufflevector <2 x i32> poison, <2 x i32> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v8i32 = shufflevector <4 x i32> poison, <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4i64 = shufflevector <2 x i64> poison, <2 x i64> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v8i32 = shufflevector <4 x i32> poison, <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v4i64 = shufflevector <2 x i64> poison, <2 x i64> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
 ; CHECK-NEXT:  Cost Model: Invalid cost for instruction: %v4f16 = shufflevector <2 x half> poison, <2 x half> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
 ; CHECK-NEXT:  Cost Model: Invalid cost for instruction: %v8f16 = shufflevector <4 x half> poison, <4 x half> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
 ; CHECK-NEXT:  Cost Model: Invalid cost for instruction: %v16f16 = shufflevector <8 x half> poison, <8 x half> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4f32 = shufflevector <2 x float> poison, <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v8f32 = shufflevector <4 x float> poison, <4 x float> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4f64 = shufflevector <2 x double> poison, <2 x double> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v8f32 = shufflevector <4 x float> poison, <4 x float> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v4f64 = shufflevector <2 x double> poison, <2 x double> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; CHECK-SIZE-LABEL: 'concat'
@@ -112,16 +112,16 @@ define void @concat() vscale_range(2,2) {
 ; CHECK-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i8 = shufflevector <8 x i8> poison, <8 x i8> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; CHECK-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i16 = shufflevector <2 x i16> poison, <2 x i16> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
 ; CHECK-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i16 = shufflevector <4 x i16> poison, <4 x i16> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; CHECK-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i16 = shufflevector <8 x i16> poison, <8 x i16> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; CHECK-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v16i16 = shufflevector <8 x i16> poison, <8 x i16> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; CHECK-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i32 = shufflevector <2 x i32> poison, <2 x i32> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; CHECK-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i32 = shufflevector <4 x i32> poison, <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; CHECK-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i64 = shufflevector <2 x i64> poison, <2 x i64> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v8i32 = shufflevector <4 x i32> poison, <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v4i64 = shufflevector <2 x i64> poison, <2 x i64> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
 ; CHECK-SIZE-NEXT:  Cost Model: Invalid cost for instruction: %v4f16 = shufflevector <2 x half> poison, <2 x half> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
 ; CHECK-SIZE-NEXT:  Cost Model: Invalid cost for instruction: %v8f16 = shufflevector <4 x half> poison, <4 x half> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
 ; CHECK-SIZE-NEXT:  Cost Model: Invalid cost for instruction: %v16f16 = shufflevector <8 x half> poison, <8 x half> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; CHECK-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4f32 = shufflevector <2 x float> poison, <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; CHECK-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8f32 = shufflevector <4 x float> poison, <4 x float> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; CHECK-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4f64 = shufflevector <2 x double> poison, <2 x double> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v8f32 = shufflevector <4 x float> poison, <4 x float> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v4f64 = shufflevector <2 x double> poison, <2 x double> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
 ; CHECK-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
   %v4i8 = shufflevector <2 x i8> poison, <2 x i8> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
@@ -170,23 +170,23 @@ define void @insert_subvec() vscale_range(2,2) {
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i16_2_2 = shufflevector <8 x i16> poison, <8 x i16> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 6, i32 7>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i16_2_3 = shufflevector <8 x i16> poison, <8 x i16> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 8, i32 9>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i16_2_05 = shufflevector <8 x i16> poison, <8 x i16> poison, <8 x i32> <i32 0, i32 8, i32 9, i32 3, i32 4, i32 5, i32 6, i32 7>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v16i16_4_0 = shufflevector <16 x i16> poison, <16 x i16> poison, <16 x i32> <i32 16, i32 17, i32 18, i32 19, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v16i16_4_1 = shufflevector <16 x i16> poison, <16 x i16> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 16, i32 17, i32 18, i32 19, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v16i16_4_2 = shufflevector <16 x i16> poison, <16 x i16> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 12, i32 13, i32 14, i32 15>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v16i16_4_3 = shufflevector <16 x i16> poison, <16 x i16> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 16, i32 17, i32 18, i32 19>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v16i16_4_05 = shufflevector <16 x i16> poison, <16 x i16> poison, <16 x i32> <i32 0, i32 1, i32 16, i32 17, i32 18, i32 19, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i16_4_0 = shufflevector <16 x i16> poison, <16 x i16> poison, <16 x i32> <i32 16, i32 17, i32 18, i32 19, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i16_4_1 = shufflevector <16 x i16> poison, <16 x i16> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 16, i32 17, i32 18, i32 19, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i16_4_2 = shufflevector <16 x i16> poison, <16 x i16> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 12, i32 13, i32 14, i32 15>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i16_4_3 = shufflevector <16 x i16> poison, <16 x i16> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 16, i32 17, i32 18, i32 19>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i16_4_05 = shufflevector <16 x i16> poison, <16 x i16> poison, <16 x i32> <i32 0, i32 1, i32 16, i32 17, i32 18, i32 19, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v4i32_2_0 = shufflevector <4 x i32> poison, <4 x i32> poison, <4 x i32> <i32 0, i32 1, i32 6, i32 7>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i32_2_1 = shufflevector <4 x i32> poison, <4 x i32> poison, <4 x i32> <i32 4, i32 5, i32 0, i32 1>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v8i32_2_0 = shufflevector <8 x i32> poison, <8 x i32> poison, <8 x i32> <i32 8, i32 9, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v8i32_2_1 = shufflevector <8 x i32> poison, <8 x i32> poison, <8 x i32> <i32 0, i32 1, i32 8, i32 9, i32 4, i32 5, i32 6, i32 7>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v8i32_2_2 = shufflevector <8 x i32> poison, <8 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 6, i32 7>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v8i32_2_3 = shufflevector <8 x i32> poison, <8 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 8, i32 9>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v8i32_2_05 = shufflevector <8 x i32> poison, <8 x i32> poison, <8 x i32> <i32 0, i32 8, i32 9, i32 3, i32 4, i32 5, i32 6, i32 7>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %v16i32_4_0 = shufflevector <16 x i32> poison, <16 x i32> poison, <16 x i32> <i32 16, i32 17, i32 18, i32 19, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v16i32_4_1 = shufflevector <16 x i32> poison, <16 x i32> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 16, i32 17, i32 18, i32 19, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v16i32_4_2 = shufflevector <16 x i32> poison, <16 x i32> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 12, i32 13, i32 14, i32 15>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v16i32_4_3 = shufflevector <16 x i32> poison, <16 x i32> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 16, i32 17, i32 18, i32 19>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v16i32_4_05 = shufflevector <16 x i32> poison, <16 x i32> poison, <16 x i32> <i32 0, i32 1, i32 16, i32 17, i32 18, i32 19, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i32_2_0 = shufflevector <8 x i32> poison, <8 x i32> poison, <8 x i32> <i32 8, i32 9, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i32_2_1 = shufflevector <8 x i32> poison, <8 x i32> poison, <8 x i32> <i32 0, i32 1, i32 8, i32 9, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i32_2_2 = shufflevector <8 x i32> poison, <8 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 6, i32 7>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i32_2_3 = shufflevector <8 x i32> poison, <8 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 8, i32 9>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i32_2_05 = shufflevector <8 x i32> poison, <8 x i32> poison, <8 x i32> <i32 0, i32 8, i32 9, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v16i32_4_0 = shufflevector <16 x i32> poison, <16 x i32> poison, <16 x i32> <i32 16, i32 17, i32 18, i32 19, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v16i32_4_1 = shufflevector <16 x i32> poison, <16 x i32> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 16, i32 17, i32 18, i32 19, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v16i32_4_2 = shufflevector <16 x i32> poison, <16 x i32> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 12, i32 13, i32 14, i32 15>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v16i32_4_3 = shufflevector <16 x i32> poison, <16 x i32> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 16, i32 17, i32 18, i32 19>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v16i32_4_05 = shufflevector <16 x i32> poison, <16 x i32> poison, <16 x i32> <i32 0, i32 1, i32 16, i32 17, i32 18, i32 19, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; CHECK-SIZE-LABEL: 'insert_subvec'
@@ -209,23 +209,23 @@ define void @insert_subvec() vscale_range(2,2) {
 ; CHECK-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i16_2_2 = shufflevector <8 x i16> poison, <8 x i16> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 6, i32 7>
 ; CHECK-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i16_2_3 = shufflevector <8 x i16> poison, <8 x i16> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 8, i32 9>
 ; CHECK-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i16_2_05 = shufflevector <8 x i16> poison, <8 x i16> poison, <8 x i32> <i32 0, i32 8, i32 9, i32 3, i32 4, i32 5, i32 6, i32 7>
-; CHECK-SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v16i16_4_0 = shufflevector <16 x i16> poison, <16 x i16> poison, <16 x i32> <i32 16, i32 17, i32 18, i32 19, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; CHECK-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i16_4_0 = shufflevector <16 x i16> poison, <16 x i16> poison, <16 x i32> <i32 16, i32 17, i32 18, i32 19, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; CHECK-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i16_4_1 = shufflevector <16 x i16> poison, <16 x i16> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 16, i32 17, i32 18, i32 19, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; CHECK-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i16_4_2 = shufflevector <16 x i16> poison, <16 x i16> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 12, i32 13, i32 14, i32 15>
 ; CHECK-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i16_4_3 = shufflevector <16 x i16> poison, <16 x i16> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 16, i32 17, i32 18, i32 19>
 ; CHECK-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i16_4_05 = shufflevector <16 x i16> poison, <16 x i16> poison, <16 x i32> <i32 0, i32 1, i32 16, i32 17, i32 18, i32 19, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; CHECK-SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v4i32_2_0 = shufflevector <4 x i32> poison, <4 x i32> poison, <4 x i32> <i32 0, i32 1, i32 6, i32 7>
 ; CHECK-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i32_2_1 = shufflevector <4 x i32> poison, <4 x i32> poison, <4 x i32> <i32 4, i32 5, i32 0, i32 1>
-; CHECK-SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v8i32_2_0 = shufflevector <8 x i32> poison, <8 x i32> poison, <8 x i32> <i32 8, i32 9, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i32_2_0 = shufflevector <8 x i32> poison, <8 x i32> poison, <8 x i32> <i32 8, i32 9, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
 ; CHECK-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i32_2_1 = shufflevector <8 x i32> poison, <8 x i32> poison, <8 x i32> <i32 0, i32 1, i32 8, i32 9, i32 4, i32 5, i32 6, i32 7>
 ; CHECK-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i32_2_2 = shufflevector <8 x i32> poison, <8 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 6, i32 7>
 ; CHECK-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i32_2_3 = shufflevector <8 x i32> poison, <8 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 8, i32 9>
 ; CHECK-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i32_2_05 = shufflevector <8 x i32> poison, <8 x i32> poison, <8 x i32> <i32 0, i32 8, i32 9, i32 3, i32 4, i32 5, i32 6, i32 7>
-; CHECK-SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v16i32_4_0 = shufflevector <16 x i32> poison, <16 x i32> poison, <16 x i32> <i32 16, i32 17, i32 18, i32 19, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; CHECK-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i32_4_1 = shufflevector <16 x i32> poison, <16 x i32> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 16, i32 17, i32 18, i32 19, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; CHECK-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i32_4_2 = shufflevector <16 x i32> poison, <16 x i32> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 12, i32 13, i32 14, i32 15>
-; CHECK-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i32_4_3 = shufflevector <16 x i32> poison, <16 x i32> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 16, i32 17, i32 18, i32 19>
-; CHECK-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i32_4_05 = shufflevector <16 x i32> poison, <16 x i32> poison, <16 x i32> <i32 0, i32 1, i32 16, i32 17, i32 18, i32 19, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; CHECK-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v16i32_4_0 = shufflevector <16 x i32> poison, <16 x i32> poison, <16 x i32> <i32 16, i32 17, i32 18, i32 19, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; CHECK-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v16i32_4_1 = shufflevector <16 x i32> poison, <16 x i32> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 16, i32 17, i32 18, i32 19, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; CHECK-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v16i32_4_2 = shufflevector <16 x i32> poison, <16 x i32> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 12, i32 13, i32 14, i32 15>
+; CHECK-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v16i32_4_3 = shufflevector <16 x i32> poison, <16 x i32> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 16, i32 17, i32 18, i32 19>
+; CHECK-SIZE-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v16i32_4_05 = shufflevector <16 x i32> poison, <16 x i32> poison, <16 x i32> <i32 0, i32 1, i32 16, i32 17, i32 18, i32 19, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; CHECK-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
   %v4i8_2_0 = shufflevector <4 x i8> poison, <4 x i8> poison, <4 x i32> <i32 0, i32 1, i32 6, i32 7>
@@ -732,64 +732,44 @@ define void @shuffle2() vscale_range(2,2) {
 
 
 define void @multipart() vscale_range(2,2) {
-; RV32-LABEL: 'multipart'
-; RV32-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %v16a = shufflevector <8 x i16> poison, <8 x i16> poison, <8 x i32> <i32 0, i32 8, i32 1, i32 9, i32 2, i32 10, i32 3, i32 11>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16b = shufflevector <8 x i16> poison, <8 x i16> poison, <8 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v16c = shufflevector <16 x i16> poison, <16 x i16> poison, <16 x i32> <i32 0, i32 8, i32 1, i32 9, i32 2, i32 10, i32 3, i32 11, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 19 for instruction: %v16d = shufflevector <16 x i16> poison, <16 x i16> poison, <16 x i32> <i32 0, i32 16, i32 1, i32 17, i32 2, i32 18, i32 3, i32 19, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v32a = shufflevector <4 x i32> poison, <4 x i32> poison, <4 x i32> <i32 2, i32 3, i32 0, i32 1>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 22 for instruction: %v32a4 = shufflevector <16 x i32> poison, <16 x i32> poison, <16 x i32> <i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 0, i32 1>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 47 for instruction: %v32idrev = shufflevector <16 x i32> poison, <16 x i32> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 15, i32 14, i32 13, i32 12, i32 16, i32 17, i32 18, i32 19, i32 31, i32 30, i32 29, i32 28>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 47 for instruction: %v32many = shufflevector <16 x i32> poison, <16 x i32> poison, <16 x i32> <i32 0, i32 4, i32 8, i32 12, i32 16, i32 20, i32 24, i32 28, i32 2, i32 6, i32 10, i32 14, i32 18, i32 22, i32 26, i32 30>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 47 for instruction: %v32many2 = shufflevector <16 x i32> poison, <16 x i32> poison, <16 x i32> <i32 1, i32 4, i32 8, i32 12, i32 17, i32 20, i32 24, i32 28, i32 2, i32 6, i32 11, i32 14, i32 18, i32 22, i32 27, i32 30>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %v323 = shufflevector <3 x i32> poison, <3 x i32> poison, <3 x i32> <i32 2, i32 3, i32 0>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v64a = shufflevector <2 x i64> poison, <2 x i64> poison, <2 x i32> <i32 1, i32 1>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v64b = shufflevector <2 x i64> poison, <2 x i64> poison, <2 x i32> zeroinitializer
-; RV32-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %v64ab = shufflevector <4 x i64> poison, <4 x i64> poison, <4 x i32> <i32 1, i32 1, i32 0, i32 0>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %v64d = shufflevector <4 x i64> poison, <4 x i64> poison, <4 x i32> <i32 1, i32 1, i32 4, i32 4>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %f64a = shufflevector <2 x double> poison, <2 x double> poison, <2 x i32> <i32 1, i32 1>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %f64b = shufflevector <2 x double> poison, <2 x double> poison, <2 x i32> zeroinitializer
-; RV32-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %f64ab = shufflevector <4 x double> poison, <4 x double> poison, <4 x i32> <i32 1, i32 1, i32 0, i32 0>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
-;
-; RV64-LABEL: 'multipart'
-; RV64-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %v16a = shufflevector <8 x i16> poison, <8 x i16> poison, <8 x i32> <i32 0, i32 8, i32 1, i32 9, i32 2, i32 10, i32 3, i32 11>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16b = shufflevector <8 x i16> poison, <8 x i16> poison, <8 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v16c = shufflevector <16 x i16> poison, <16 x i16> poison, <16 x i32> <i32 0, i32 8, i32 1, i32 9, i32 2, i32 10, i32 3, i32 11, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 19 for instruction: %v16d = shufflevector <16 x i16> poison, <16 x i16> poison, <16 x i32> <i32 0, i32 16, i32 1, i32 17, i32 2, i32 18, i32 3, i32 19, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v32a = shufflevector <4 x i32> poison, <4 x i32> poison, <4 x i32> <i32 2, i32 3, i32 0, i32 1>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 22 for instruction: %v32a4 = shufflevector <16 x i32> poison, <16 x i32> poison, <16 x i32> <i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 0, i32 1>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 47 for instruction: %v32idrev = shufflevector <16 x i32> poison, <16 x i32> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 15, i32 14, i32 13, i32 12, i32 16, i32 17, i32 18, i32 19, i32 31, i32 30, i32 29, i32 28>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 47 for instruction: %v32many = shufflevector <16 x i32> poison, <16 x i32> poison, <16 x i32> <i32 0, i32 4, i32 8, i32 12, i32 16, i32 20, i32 24, i32 28, i32 2, i32 6, i32 10, i32 14, i32 18, i32 22, i32 26, i32 30>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 47 for instruction: %v32many2 = shufflevector <16 x i32> poison, <16 x i32> poison, <16 x i32> <i32 1, i32 4, i32 8, i32 12, i32 17, i32 20, i32 24, i32 28, i32 2, i32 6, i32 11, i32 14, i32 18, i32 22, i32 27, i32 30>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %v323 = shufflevector <3 x i32> poison, <3 x i32> poison, <3 x i32> <i32 2, i32 3, i32 0>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v64a = shufflevector <2 x i64> poison, <2 x i64> poison, <2 x i32> <i32 1, i32 1>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v64b = shufflevector <2 x i64> poison, <2 x i64> poison, <2 x i32> zeroinitializer
-; RV64-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v64ab = shufflevector <4 x i64> poison, <4 x i64> poison, <4 x i32> <i32 1, i32 1, i32 0, i32 0>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 19 for instruction: %v64d = shufflevector <4 x i64> poison, <4 x i64> poison, <4 x i32> <i32 1, i32 1, i32 4, i32 4>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %f64a = shufflevector <2 x double> poison, <2 x double> poison, <2 x i32> <i32 1, i32 1>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %f64b = shufflevector <2 x double> poison, <2 x double> poison, <2 x i32> zeroinitializer
-; RV64-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %f64ab = shufflevector <4 x double> poison, <4 x double> poison, <4 x i32> <i32 1, i32 1, i32 0, i32 0>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; CHECK-LABEL: 'multipart'
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %v16a = shufflevector <8 x i16> poison, <8 x i16> poison, <8 x i32> <i32 0, i32 8, i32 1, i32 9, i32 2, i32 10, i32 3, i32 11>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16b = shufflevector <8 x i16> poison, <8 x i16> poison, <8 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v16c = shufflevector <16 x i16> poison, <16 x i16> poison, <16 x i32> <i32 0, i32 8, i32 1, i32 9, i32 2, i32 10, i32 3, i32 11, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v16d = shufflevector <16 x i16> poison, <16 x i16> poison, <16 x i32> <i32 0, i32 16, i32 1, i32 17, i32 2, i32 18, i32 3, i32 19, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v32a = shufflevector <4 x i32> poison, <4 x i32> poison, <4 x i32> <i32 2, i32 3, i32 0, i32 1>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v32a4 = shufflevector <16 x i32> poison, <16 x i32> poison, <16 x i32> <i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 0, i32 1>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %v32idrev = shufflevector <16 x i32> poison, <16 x i32> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 15, i32 14, i32 13, i32 12, i32 16, i32 17, i32 18, i32 19, i32 31, i32 30, i32 29, i32 28>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 47 for instruction: %v32many = shufflevector <16 x i32> poison, <16 x i32> poison, <16 x i32> <i32 0, i32 4, i32 8, i32 12, i32 16, i32 20, i32 24, i32 28, i32 2, i32 6, i32 10, i32 14, i32 18, i32 22, i32 26, i32 30>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 47 for instruction: %v32many2 = shufflevector <16 x i32> poison, <16 x i32> poison, <16 x i32> <i32 1, i32 4, i32 8, i32 12, i32 17, i32 20, i32 24, i32 28, i32 2, i32 6, i32 11, i32 14, i32 18, i32 22, i32 27, i32 30>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %v323 = shufflevector <3 x i32> poison, <3 x i32> poison, <3 x i32> <i32 2, i32 3, i32 0>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v64a = shufflevector <2 x i64> poison, <2 x i64> poison, <2 x i32> <i32 1, i32 1>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v64b = shufflevector <2 x i64> poison, <2 x i64> poison, <2 x i32> zeroinitializer
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v64ab = shufflevector <4 x i64> poison, <4 x i64> poison, <4 x i32> <i32 1, i32 1, i32 0, i32 0>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v64d = shufflevector <4 x i64> poison, <4 x i64> poison, <4 x i32> <i32 1, i32 1, i32 4, i32 4>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %f64a = shufflevector <2 x double> poison, <2 x double> poison, <2 x i32> <i32 1, i32 1>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %f64b = shufflevector <2 x double> poison, <2 x double> poison, <2 x i32> zeroinitializer
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %f64ab = shufflevector <4 x double> poison, <4 x double> poison, <4 x i32> <i32 1, i32 1, i32 0, i32 0>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; CHECK-SIZE-LABEL: 'multipart'
 ; CHECK-SIZE-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %v16a = shufflevector <8 x i16> poison, <8 x i16> poison, <8 x i32> <i32 0, i32 8, i32 1, i32 9, i32 2, i32 10, i32 3, i32 11>
 ; CHECK-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16b = shufflevector <8 x i16> poison, <8 x i16> poison, <8 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
-; CHECK-SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v16c = shufflevector <16 x i16> poison, <16 x i16> poison, <16 x i32> <i32 0, i32 8, i32 1, i32 9, i32 2, i32 10, i32 3, i32 11, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
-; CHECK-SIZE-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %v16d = shufflevector <16 x i16> poison, <16 x i16> poison, <16 x i32> <i32 0, i32 16, i32 1, i32 17, i32 2, i32 18, i32 3, i32 19, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+; CHECK-SIZE-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v16c = shufflevector <16 x i16> poison, <16 x i16> poison, <16 x i32> <i32 0, i32 8, i32 1, i32 9, i32 2, i32 10, i32 3, i32 11, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+; CHECK-SIZE-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v16d = shufflevector <16 x i16> poison, <16 x i16> poison, <16 x i32> <i32 0, i32 16, i32 1, i32 17, i32 2, i32 18, i32 3, i32 19, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
 ; CHECK-SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v32a = shufflevector <4 x i32> poison, <4 x i32> poison, <4 x i32> <i32 2, i32 3, i32 0, i32 1>
-; CHECK-SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v32a4 = shufflevector <16 x i32> poison, <16 x i32> poison, <16 x i32> <i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 0, i32 1>
-; CHECK-SIZE-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %v32idrev = shufflevector <16 x i32> poison, <16 x i32> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 15, i32 14, i32 13, i32 12, i32 16, i32 17, i32 18, i32 19, i32 31, i32 30, i32 29, i32 28>
+; CHECK-SIZE-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v32a4 = shufflevector <16 x i32> poison, <16 x i32> poison, <16 x i32> <i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 0, i32 1>
+; CHECK-SIZE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %v32idrev = shufflevector <16 x i32> poison, <16 x i32> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 15, i32 14, i32 13, i32 12, i32 16, i32 17, i32 18, i32 19, i32 31, i32 30, i32 29, i32 28>
 ; CHECK-SIZE-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %v32many = shufflevector <16 x i32> poison, <16 x i32> poison, <16 x i32> <i32 0, i32 4, i32 8, i32 12, i32 16, i32 20, i32 24, i32 28, i32 2, i32 6, i32 10, i32 14, i32 18, i32 22, i32 26, i32 30>
 ; CHECK-SIZE-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %v32many2 = shufflevector <16 x i32> poison, <16 x i32> poison, <16 x i32> <i32 1, i32 4, i32 8, i32 12, i32 17, i32 20, i32 24, i32 28, i32 2, i32 6, i32 11, i32 14, i32 18, i32 22, i32 27, i32 30>
 ; CHECK-SIZE-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %v323 = shufflevector <3 x i32> poison, <3 x i32> poison, <3 x i32> <i32 2, i32 3, i32 0>
 ; CHECK-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v64a = shufflevector <2 x i64> poison, <2 x i64> poison, <2 x i32> <i32 1, i32 1>
 ; CHECK-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v64b = shufflevector <2 x i64> poison, <2 x i64> poison, <2 x i32> zeroinitializer
-; CHECK-SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v64ab = shufflevector <4 x i64> poison, <4 x i64> poison, <4 x i32> <i32 1, i32 1, i32 0, i32 0>
-; CHECK-SIZE-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %v64d = shufflevector <4 x i64> poison, <4 x i64> poison, <4 x i32> <i32 1, i32 1, i32 4, i32 4>
+; CHECK-SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v64ab = shufflevector <4 x i64> poison, <4 x i64> poison, <4 x i32> <i32 1, i32 1, i32 0, i32 0>
+; CHECK-SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v64d = shufflevector <4 x i64> poison, <4 x i64> poison, <4 x i32> <i32 1, i32 1, i32 4, i32 4>
 ; CHECK-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %f64a = shufflevector <2 x double> poison, <2 x double> poison, <2 x i32> <i32 1, i32 1>
 ; CHECK-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %f64b = shufflevector <2 x double> poison, <2 x double> poison, <2 x i32> zeroinitializer
-; CHECK-SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %f64ab = shufflevector <4 x double> poison, <4 x double> poison, <4 x i32> <i32 1, i32 1, i32 0, i32 0>
+; CHECK-SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %f64ab = shufflevector <4 x double> poison, <4 x double> poison, <4 x i32> <i32 1, i32 1, i32 0, i32 0>
 ; CHECK-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
   %v16a = shufflevector <8 x i16> poison, <8 x i16> poison, <8 x i32> <i32 0, i32 8, i32 1, i32 9, i32 2, i32 10, i32 3, i32 11>
diff --git a/llvm/test/Analysis/CostModel/RISCV/shuffle-extract_subvector.ll b/llvm/test/Analysis/CostModel/RISCV/shuffle-extract_subvector.ll
index d65668fa30ca6..e8dd30345cc76 100644
--- a/llvm/test/Analysis/CostModel/RISCV/shuffle-extract_subvector.ll
+++ b/llvm/test/Analysis/CostModel/RISCV/shuffle-extract_subvector.ll
@@ -30,9 +30,9 @@ define void @test_vXf64(<4 x double> %src256, <8 x double> %src512) {
 ; VLEN128-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_45 = shufflevector <8 x double> %src512, <8 x double> undef, <2 x i32> <i32 4, i32 5>
 ; VLEN128-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_67 = shufflevector <8 x double> %src512, <8 x double> undef, <2 x i32> <i32 6, i32 7>
 ; VLEN128-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_0123 = shufflevector <8 x double> %src512, <8 x double> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; VLEN128-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V512_2345 = shufflevector <8 x double> %src512, <8 x double> undef, <4 x i32> <i32 2, i32 3, i32 4, i32 5>
-; VLEN128-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V512_4567 = shufflevector <8 x double> %src512, <8 x double> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
-; VLEN128-NEXT:  Cost Model: Found an estimated cost of 19 for instruction: %V512_567u = shufflevector <8 x double> %src512, <8 x double> undef, <4 x i32> <i32 5, i32 6, i32 7, i32 poison>
+; VLEN128-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_2345 = shufflevector <8 x double> %src512, <8 x double> undef, <4 x i32> <i32 2, i32 3, i32 4, i32 5>
+; VLEN128-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_4567 = shufflevector <8 x double> %src512, <8 x double> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+; VLEN128-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V512_567u = shufflevector <8 x double> %src512, <8 x double> undef, <4 x i32> <i32 5, i32 6, i32 7, i32 poison>
 ; VLEN128-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
   %V256_01 = shufflevector <4 x double> %src256, <4 x double> undef, <2 x i32> <i32 0, i32 1>
@@ -69,8 +69,8 @@ define void @test_vXi64(<4 x i64> %src256, <8 x i64> %src512) {
 ; VLEN128-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_45 = shufflevector <8 x i64> %src512, <8 x i64> undef, <2 x i32> <i32 4, i32 5>
 ; VLEN128-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_67 = shufflevector <8 x i64> %src512, <8 x i64> undef, <2 x i32> <i32 6, i32 7>
 ; VLEN128-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_0123 = shufflevector <8 x i64> %src512, <8 x i64> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; VLEN128-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V512_2345 = shufflevector <8 x i64> %src512, <8 x i64> undef, <4 x i32> <i32 2, i32 3, i32 4, i32 5>
-; VLEN128-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V512_4567 = shufflevector <8 x i64> %src512, <8 x i64> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+; VLEN128-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_2345 = shufflevector <8 x i64> %src512, <8 x i64> undef, <4 x i32> <i32 2, i32 3, i32 4, i32 5>
+; VLEN128-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_4567 = shufflevector <8 x i64> %src512, <8 x i64> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
 ; VLEN128-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
   %V256_01 = shufflevector <4 x i64> %src256, <4 x i64> undef, <2 x i32> <i32 0, i32 1>
@@ -133,7 +133,7 @@ define void @test_vXi32(<4 x i32> %src128, <8 x i32> %src256, <16 x i32> %src512
 ; VLEN128-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_89AB = shufflevector <16 x i32> %src512, <16 x i32> undef, <4 x i32> <i32 8, i32 9, i32 10, i32 11>
 ; VLEN128-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_CDEF = shufflevector <16 x i32> %src512, <16 x i32> undef, <4 x i32> <i32 12, i32 13, i32 14, i32 15>
 ; VLEN128-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_01234567 = shufflevector <16 x i32> %src512, <16 x i32> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; VLEN128-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V512_89ABCDEF = shufflevector <16 x i32> %src512, <16 x i32> undef, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; VLEN128-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_89ABCDEF = shufflevector <16 x i32> %src512, <16 x i32> undef, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; VLEN128-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
   %V128_01 = shufflevector <4 x i32> %src128, <4 x i32> undef, <2 x i32> <i32 0, i32 1>
@@ -277,7 +277,7 @@ define void @test_vXi16(<4 x i16> %src64, <8 x i16> %src128, <16 x i16> %src256,
 ; VLEN128-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_10_11_12_13_14_15_16_17 = shufflevector <32 x i16> %src512, <32 x i16> undef, <8 x i32> <i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23>
 ; VLEN128-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_18_19_1A_1B_1C_1D_1E_1F = shufflevector <32 x i16> %src512, <32 x i16> undef, <8 x i32> <i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
 ; VLEN128-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_00_01_02_03_04_05_06_07_08_09_0A_0B_0C_0D_0E_0F = shufflevector <32 x i16> %src512, <32 x i16> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; VLEN128-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V512_10_11_12_13_14_15_16_17_18_19_1A_1B_1C_1D_1E_1F = shufflevector <32 x i16> %src512, <32 x i16> undef, <16 x i32> <i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+; VLEN128-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_10_11_12_13_14_15_16_17_18_19_1A_1B_1C_1D_1E_1F = shufflevector <32 x i16> %src512, <32 x i16> undef, <16 x i32> <i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
 ; VLEN128-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
   %V64_01 = shufflevector <4 x i16> %src64, <4 x i16> undef, <2 x i32> <i32 0, i32 1>
@@ -575,7 +575,7 @@ define void @test_vXi8(<8 x i8> %src64, <16 x i8> %src128, <32 x i8> %src256, <6
 ; VLEN128-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_20_21_22_23_24_25_26_27_28_29_2A_2B_2C_2D_2E_2F = shufflevector <64 x i8> %src512, <64 x i8> undef, <16 x i32> <i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47>
 ; VLEN128-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_30_31_32_33_34_35_36_37_38_39_3A_3B_3C_3D_3E_3F = shufflevector <64 x i8> %src512, <64 x i8> undef, <16 x i32> <i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
 ; VLEN128-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_00_01_02_03_04_05_06_07_08_09_0A_0B_0C_0D_0E_0F_10_11_12_13_14_15_16_17_18_19_1A_1B_1C_1D_1E_1F = shufflevector <64 x i8> %src512, <64 x i8> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
-; VLEN128-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V512_20_21_22_23_24_25_26_27_28_29_2A_2B_2C_2D_2E_2F_30_31_32_33_34_35_36_37_38_39_3A_3B_3C_3D_3E_3F = shufflevector <64 x i8> %src512, <64 x i8> undef, <32 x i32> <i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
+; VLEN128-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_20_21_22_23_24_25_26_27_28_29_2A_2B_2C_2D_2E_2F_30_31_32_33_34_35_36_37_38_39_3A_3B_3C_3D_3E_3F = shufflevector <64 x i8> %src512, <64 x i8> undef, <32 x i32> <i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
 ; VLEN128-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
   %V64_01 = shufflevector <8 x i8> %src64, <8 x i8> undef, <2 x i32> <i32 0, i32 1>
diff --git a/llvm/test/Analysis/CostModel/RISCV/shuffle-insert_subvector.ll b/llvm/test/Analysis/CostModel/RISCV/shuffle-insert_subvector.ll
index 2c93112fbf9c8..ea536f9e2deb8 100644
--- a/llvm/test/Analysis/CostModel/RISCV/shuffle-insert_subvector.ll
+++ b/llvm/test/Analysis/CostModel/RISCV/shuffle-insert_subvector.ll
@@ -523,19 +523,19 @@ define void @test_vXi8(<2 x i8> %src16, <4 x i8> %src32, <8 x i8> %src64, <16x i
 
 define void @fixed_m1_in_m2_notail(<8 x i32> %src, <8 x i32> %passthru) vscale_range(2) {
 ; CHECK-LABEL: 'fixed_m1_in_m2_notail'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %1 = shufflevector <8 x i32> %src, <8 x i32> %passthru, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %1 = shufflevector <8 x i32> %src, <8 x i32> %passthru, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 4, i32 5, i32 6, i32 7>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %2 = shufflevector <8 x i32> %src, <8 x i32> %passthru, <8 x i32> <i32 0, i32 8, i32 9, i32 10, i32 11, i32 5, i32 6, i32 7>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %3 = shufflevector <8 x i32> %src, <8 x i32> %passthru, <8 x i32> <i32 0, i32 1, i32 8, i32 9, i32 10, i32 11, i32 6, i32 7>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %4 = shufflevector <8 x i32> %src, <8 x i32> %passthru, <8 x i32> <i32 0, i32 1, i32 2, i32 8, i32 9, i32 10, i32 11, i32 7>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %5 = shufflevector <8 x i32> %src, <8 x i32> %passthru, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %5 = shufflevector <8 x i32> %src, <8 x i32> %passthru, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; SIZE-LABEL: 'fixed_m1_in_m2_notail'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %1 = shufflevector <8 x i32> %src, <8 x i32> %passthru, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 4, i32 5, i32 6, i32 7>
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %1 = shufflevector <8 x i32> %src, <8 x i32> %passthru, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 4, i32 5, i32 6, i32 7>
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %2 = shufflevector <8 x i32> %src, <8 x i32> %passthru, <8 x i32> <i32 0, i32 8, i32 9, i32 10, i32 11, i32 5, i32 6, i32 7>
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %3 = shufflevector <8 x i32> %src, <8 x i32> %passthru, <8 x i32> <i32 0, i32 1, i32 8, i32 9, i32 10, i32 11, i32 6, i32 7>
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %4 = shufflevector <8 x i32> %src, <8 x i32> %passthru, <8 x i32> <i32 0, i32 1, i32 2, i32 8, i32 9, i32 10, i32 11, i32 7>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %5 = shufflevector <8 x i32> %src, <8 x i32> %passthru, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %5 = shufflevector <8 x i32> %src, <8 x i32> %passthru, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
   shufflevector <8 x i32> %src, <8 x i32> %passthru, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 4, i32 5, i32 6, i32 7>
@@ -548,19 +548,19 @@ define void @fixed_m1_in_m2_notail(<8 x i32> %src, <8 x i32> %passthru) vscale_r
 
 define void @fixed_m2_in_m4_notail(<8 x i64> %src, <8 x i64> %passthru) vscale_range(2) {
 ; CHECK-LABEL: 'fixed_m2_in_m4_notail'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %1 = shufflevector <8 x i64> %src, <8 x i64> %passthru, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %1 = shufflevector <8 x i64> %src, <8 x i64> %passthru, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 4, i32 5, i32 6, i32 7>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 41 for instruction: %2 = shufflevector <8 x i64> %src, <8 x i64> %passthru, <8 x i32> <i32 0, i32 8, i32 8, i32 10, i32 11, i32 5, i32 6, i32 7>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 41 for instruction: %3 = shufflevector <8 x i64> %src, <8 x i64> %passthru, <8 x i32> <i32 0, i32 1, i32 8, i32 8, i32 10, i32 11, i32 6, i32 7>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %3 = shufflevector <8 x i64> %src, <8 x i64> %passthru, <8 x i32> <i32 0, i32 1, i32 8, i32 8, i32 10, i32 11, i32 6, i32 7>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 41 for instruction: %4 = shufflevector <8 x i64> %src, <8 x i64> %passthru, <8 x i32> <i32 0, i32 1, i32 2, i32 8, i32 8, i32 10, i32 11, i32 7>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 41 for instruction: %5 = shufflevector <8 x i64> %src, <8 x i64> %passthru, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 8, i32 10, i32 11>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %5 = shufflevector <8 x i64> %src, <8 x i64> %passthru, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 8, i32 10, i32 11>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; SIZE-LABEL: 'fixed_m2_in_m4_notail'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %1 = shufflevector <8 x i64> %src, <8 x i64> %passthru, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 4, i32 5, i32 6, i32 7>
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %1 = shufflevector <8 x i64> %src, <8 x i64> %passthru, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 4, i32 5, i32 6, i32 7>
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %2 = shufflevector <8 x i64> %src, <8 x i64> %passthru, <8 x i32> <i32 0, i32 8, i32 8, i32 10, i32 11, i32 5, i32 6, i32 7>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %3 = shufflevector <8 x i64> %src, <8 x i64> %passthru, <8 x i32> <i32 0, i32 1, i32 8, i32 8, i32 10, i32 11, i32 6, i32 7>
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %3 = shufflevector <8 x i64> %src, <8 x i64> %passthru, <8 x i32> <i32 0, i32 1, i32 8, i32 8, i32 10, i32 11, i32 6, i32 7>
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %4 = shufflevector <8 x i64> %src, <8 x i64> %passthru, <8 x i32> <i32 0, i32 1, i32 2, i32 8, i32 8, i32 10, i32 11, i32 7>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %5 = shufflevector <8 x i64> %src, <8 x i64> %passthru, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 8, i32 10, i32 11>
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %5 = shufflevector <8 x i64> %src, <8 x i64> %passthru, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 8, i32 10, i32 11>
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
   shufflevector <8 x i64> %src, <8 x i64> %passthru, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 4, i32 5, i32 6, i32 7>
@@ -574,7 +574,7 @@ define void @fixed_m2_in_m4_notail(<8 x i64> %src, <8 x i64> %passthru) vscale_r
 define void @fixed_m1_in_m2_tail(<8 x i32> %src, <8 x i32> %passthru) vscale_range(2) {
 ; CHECK-LABEL: 'fixed_m1_in_m2_tail'
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 19 for instruction: %1 = shufflevector <8 x i32> %src, <8 x i32> %passthru, <8 x i32> <i32 8, i32 9, i32 10, i32 4, i32 4, i32 5, i32 6, i32 8>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 19 for instruction: %2 = shufflevector <8 x i32> %src, <8 x i32> %passthru, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 8, i32 10, i32 7>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %2 = shufflevector <8 x i32> %src, <8 x i32> %passthru, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 8, i32 10, i32 7>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; SIZE-LABEL: 'fixed_m1_in_m2_tail'
diff --git a/llvm/test/Analysis/CostModel/RISCV/vp-intrinsics.ll b/llvm/test/Analysis/CostModel/RISCV/vp-intrinsics.ll
index 0245a0f7ee6cb..7fc91e7049aa3 100644
--- a/llvm/test/Analysis/CostModel/RISCV/vp-intrinsics.ll
+++ b/llvm/test/Analysis/CostModel/RISCV/vp-intrinsics.ll
@@ -1,6 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py
-; RUN: opt < %s -passes="print<cost-model>" 2>&1 -disable-output -S -mtriple=riscv64 -mattr=+v,+zvfhmin,+zvfbfmin | FileCheck %s
-; RUN: opt < %s -passes="print<cost-model>" 2>&1 -disable-output -S -mtriple=riscv64 -mattr=+v,+zvfhmin,+zvfbfmin --type-based-intrinsic-cost=true | FileCheck %s --check-prefixes=TYPEBASED
+; RUN: opt < %s -passes="print<cost-model>" 2>&1 -disable-output -S -mtriple=riscv64 -mattr=+v,+zvfhmin,+zvfbfmin | FileCheck %s --check-prefixes=CHECK,ARGBASED
+; RUN: opt < %s -passes="print<cost-model>" 2>&1 -disable-output -S -mtriple=riscv64 -mattr=+v,+zvfhmin,+zvfbfmin --type-based-intrinsic-cost=true | FileCheck %s --check-prefixes=CHECK,TYPEBASED
 
 define void @unsupported_fp_ops(<vscale x 4 x float> %vec, i32 %extraarg) {
 ; CHECK-LABEL: 'unsupported_fp_ops'
@@ -8,21 +8,15 @@ define void @unsupported_fp_ops(<vscale x 4 x float> %vec, i32 %extraarg) {
 ; CHECK-NEXT:  Cost Model: Invalid cost for instruction: %powi = call <vscale x 4 x float> @llvm.powi.nxv4f32.i32(<vscale x 4 x float> %vec, i32 %extraarg)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
-; TYPEBASED-LABEL: 'unsupported_fp_ops'
-; TYPEBASED-NEXT:  Cost Model: Invalid cost for instruction: %pow = call <vscale x 4 x float> @llvm.pow.nxv4f32(<vscale x 4 x float> %vec, <vscale x 4 x float> %vec)
-; TYPEBASED-NEXT:  Cost Model: Invalid cost for instruction: %powi = call <vscale x 4 x float> @llvm.powi.nxv4f32.i32(<vscale x 4 x float> %vec, i32 %extraarg)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
-;
-
   %pow = call <vscale x 4 x float> @llvm.pow.nxv4f32(<vscale x 4 x float> %vec, <vscale x 4 x float> %vec)
   %powi = call <vscale x 4 x float> @llvm.powi.nxv4f32.i32(<vscale x 4 x float> %vec, i32 %extraarg)
   ret void
 }
 
 define void @powi(<vscale x 4 x float> %vec) {
-; CHECK-LABEL: 'powi'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %powi = call <vscale x 4 x float> @llvm.powi.nxv4f32.i32(<vscale x 4 x float> %vec, i32 42)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; ARGBASED-LABEL: 'powi'
+; ARGBASED-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %powi = call <vscale x 4 x float> @llvm.powi.nxv4f32.i32(<vscale x 4 x float> %vec, i32 42)
+; ARGBASED-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; TYPEBASED-LABEL: 'powi'
 ; TYPEBASED-NEXT:  Cost Model: Invalid cost for instruction: %powi = call <vscale x 4 x float> @llvm.powi.nxv4f32.i32(<vscale x 4 x float> %vec, i32 42)
@@ -36,10 +30,6 @@ define void @fshr(<vscale x 1 x i32> %a, <vscale x 1 x i32> %b, <vscale x 1 x i3
 ; CHECK-LABEL: 'fshr'
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %1 = call <vscale x 1 x i32> @llvm.fshr.nxv1i32(<vscale x 1 x i32> %a, <vscale x 1 x i32> %b, <vscale x 1 x i32> %c)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
-;
-; TYPEBASED-LABEL: 'fshr'
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %1 = call <vscale x 1 x i32> @llvm.fshr.nxv1i32(<vscale x 1 x i32> %a, <vscale x 1 x i32> %b, <vscale x 1 x i32> %c)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
   call <vscale x 1 x i32> @llvm.fshr.nxv4i32(<vscale x 1 x i32> %a, <vscale x 1 x i32> %b, <vscale x 1 x i32> %c)
   ret void
@@ -49,10 +39,6 @@ define void @fshl(<vscale x 1 x i32> %a, <vscale x 1 x i32> %b, <vscale x 1 x i3
 ; CHECK-LABEL: 'fshl'
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %1 = call <vscale x 1 x i32> @llvm.fshl.nxv1i32(<vscale x 1 x i32> %a, <vscale x 1 x i32> %b, <vscale x 1 x i32> %c)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
-;
-; TYPEBASED-LABEL: 'fshl'
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %1 = call <vscale x 1 x i32> @llvm.fshl.nxv1i32(<vscale x 1 x i32> %a, <vscale x 1 x i32> %b, <vscale x 1 x i32> %c)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
   call <vscale x 1 x i32> @llvm.fshl.nxv4i32(<vscale x 1 x i32> %a, <vscale x 1 x i32> %b, <vscale x 1 x i32> %c)
   ret void
@@ -99,47 +85,6 @@ define void @vp_fshr() {
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %37 = call <vscale x 4 x i64> @llvm.vp.fshr.nxv4i64(<vscale x 4 x i64> undef, <vscale x 4 x i64> undef, <vscale x 4 x i64> undef, <vscale x 4 x i1> undef, i32 undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %38 = call <vscale x 8 x i64> @llvm.vp.fshr.nxv8i64(<vscale x 8 x i64> undef, <vscale x 8 x i64> undef, <vscale x 8 x i64> undef, <vscale x 8 x i1> undef, i32 undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
-;
-; TYPEBASED-LABEL: 'vp_fshr'
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %1 = call <2 x i8> @llvm.vp.fshr.v2i8(<2 x i8> undef, <2 x i8> undef, <2 x i8> undef, <2 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %2 = call <4 x i8> @llvm.vp.fshr.v4i8(<4 x i8> undef, <4 x i8> undef, <4 x i8> undef, <4 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %3 = call <8 x i8> @llvm.vp.fshr.v8i8(<8 x i8> undef, <8 x i8> undef, <8 x i8> undef, <8 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %4 = call <16 x i8> @llvm.vp.fshr.v16i8(<16 x i8> undef, <16 x i8> undef, <16 x i8> undef, <16 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %5 = call <vscale x 1 x i8> @llvm.vp.fshr.nxv1i8(<vscale x 1 x i8> undef, <vscale x 1 x i8> undef, <vscale x 1 x i8> undef, <vscale x 1 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %6 = call <vscale x 2 x i8> @llvm.vp.fshr.nxv2i8(<vscale x 2 x i8> undef, <vscale x 2 x i8> undef, <vscale x 2 x i8> undef, <vscale x 2 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %7 = call <vscale x 4 x i8> @llvm.vp.fshr.nxv4i8(<vscale x 4 x i8> undef, <vscale x 4 x i8> undef, <vscale x 4 x i8> undef, <vscale x 4 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %8 = call <vscale x 8 x i8> @llvm.vp.fshr.nxv8i8(<vscale x 8 x i8> undef, <vscale x 8 x i8> undef, <vscale x 8 x i8> undef, <vscale x 8 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %9 = call <vscale x 16 x i8> @llvm.vp.fshr.nxv16i8(<vscale x 16 x i8> undef, <vscale x 16 x i8> undef, <vscale x 16 x i8> undef, <vscale x 16 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %10 = call <vscale x 32 x i8> @llvm.vp.fshr.nxv32i8(<vscale x 32 x i8> undef, <vscale x 32 x i8> undef, <vscale x 32 x i8> undef, <vscale x 32 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %11 = call <vscale x 64 x i8> @llvm.vp.fshr.nxv64i8(<vscale x 64 x i8> undef, <vscale x 64 x i8> undef, <vscale x 64 x i8> undef, <vscale x 64 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %12 = call <2 x i16> @llvm.vp.fshr.v2i16(<2 x i16> undef, <2 x i16> undef, <2 x i16> undef, <2 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %13 = call <4 x i16> @llvm.vp.fshr.v4i16(<4 x i16> undef, <4 x i16> undef, <4 x i16> undef, <4 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %14 = call <8 x i16> @llvm.vp.fshr.v8i16(<8 x i16> undef, <8 x i16> undef, <8 x i16> undef, <8 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %15 = call <16 x i16> @llvm.vp.fshr.v16i16(<16 x i16> undef, <16 x i16> undef, <16 x i16> undef, <16 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %16 = call <vscale x 1 x i16> @llvm.vp.fshr.nxv1i16(<vscale x 1 x i16> undef, <vscale x 1 x i16> undef, <vscale x 1 x i16> undef, <vscale x 1 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %17 = call <vscale x 2 x i16> @llvm.vp.fshr.nxv2i16(<vscale x 2 x i16> undef, <vscale x 2 x i16> undef, <vscale x 2 x i16> undef, <vscale x 2 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %18 = call <vscale x 4 x i16> @llvm.vp.fshr.nxv4i16(<vscale x 4 x i16> undef, <vscale x 4 x i16> undef, <vscale x 4 x i16> undef, <vscale x 4 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %19 = call <vscale x 8 x i16> @llvm.vp.fshr.nxv8i16(<vscale x 8 x i16> undef, <vscale x 8 x i16> undef, <vscale x 8 x i16> undef, <vscale x 8 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %20 = call <vscale x 16 x i16> @llvm.vp.fshr.nxv16i16(<vscale x 16 x i16> undef, <vscale x 16 x i16> undef, <vscale x 16 x i16> undef, <vscale x 16 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %21 = call <vscale x 32 x i16> @llvm.vp.fshr.nxv32i16(<vscale x 32 x i16> undef, <vscale x 32 x i16> undef, <vscale x 32 x i16> undef, <vscale x 32 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %22 = call <2 x i32> @llvm.vp.fshr.v2i32(<2 x i32> undef, <2 x i32> undef, <2 x i32> undef, <2 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %23 = call <4 x i32> @llvm.vp.fshr.v4i32(<4 x i32> undef, <4 x i32> undef, <4 x i32> undef, <4 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %24 = call <8 x i32> @llvm.vp.fshr.v8i32(<8 x i32> undef, <8 x i32> undef, <8 x i32> undef, <8 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %25 = call <16 x i32> @llvm.vp.fshr.v16i32(<16 x i32> undef, <16 x i32> undef, <16 x i32> undef, <16 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %26 = call <vscale x 1 x i32> @llvm.vp.fshr.nxv1i32(<vscale x 1 x i32> undef, <vscale x 1 x i32> undef, <vscale x 1 x i32> undef, <vscale x 1 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %27 = call <vscale x 2 x i32> @llvm.vp.fshr.nxv2i32(<vscale x 2 x i32> undef, <vscale x 2 x i32> undef, <vscale x 2 x i32> undef, <vscale x 2 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %28 = call <vscale x 4 x i32> @llvm.vp.fshr.nxv4i32(<vscale x 4 x i32> undef, <vscale x 4 x i32> undef, <vscale x 4 x i32> undef, <vscale x 4 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %29 = call <vscale x 8 x i32> @llvm.vp.fshr.nxv8i32(<vscale x 8 x i32> undef, <vscale x 8 x i32> undef, <vscale x 8 x i32> undef, <vscale x 8 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %30 = call <vscale x 16 x i32> @llvm.vp.fshr.nxv16i32(<vscale x 16 x i32> undef, <vscale x 16 x i32> undef, <vscale x 16 x i32> undef, <vscale x 16 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %31 = call <2 x i64> @llvm.vp.fshr.v2i64(<2 x i64> undef, <2 x i64> undef, <2 x i64> undef, <2 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %32 = call <4 x i64> @llvm.vp.fshr.v4i64(<4 x i64> undef, <4 x i64> undef, <4 x i64> undef, <4 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %33 = call <8 x i64> @llvm.vp.fshr.v8i64(<8 x i64> undef, <8 x i64> undef, <8 x i64> undef, <8 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %34 = call <16 x i64> @llvm.vp.fshr.v16i64(<16 x i64> undef, <16 x i64> undef, <16 x i64> undef, <16 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %35 = call <vscale x 1 x i64> @llvm.vp.fshr.nxv1i64(<vscale x 1 x i64> undef, <vscale x 1 x i64> undef, <vscale x 1 x i64> undef, <vscale x 1 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %36 = call <vscale x 2 x i64> @llvm.vp.fshr.nxv2i64(<vscale x 2 x i64> undef, <vscale x 2 x i64> undef, <vscale x 2 x i64> undef, <vscale x 2 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %37 = call <vscale x 4 x i64> @llvm.vp.fshr.nxv4i64(<vscale x 4 x i64> undef, <vscale x 4 x i64> undef, <vscale x 4 x i64> undef, <vscale x 4 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %38 = call <vscale x 8 x i64> @llvm.vp.fshr.nxv8i64(<vscale x 8 x i64> undef, <vscale x 8 x i64> undef, <vscale x 8 x i64> undef, <vscale x 8 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
   call <2 x i8> @llvm.vp.fshr.v2i8(<2 x i8> undef, <2 x i8> undef, <2 x i8> undef, <2 x i1> undef, i32 undef)
   call <4 x i8> @llvm.vp.fshr.v4i8(<4 x i8> undef, <4 x i8> undef, <4 x i8> undef, <4 x i1> undef, i32 undef)
@@ -223,47 +168,6 @@ define void @vp_fshl() {
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %37 = call <vscale x 4 x i64> @llvm.vp.fshl.nxv4i64(<vscale x 4 x i64> undef, <vscale x 4 x i64> undef, <vscale x 4 x i64> undef, <vscale x 4 x i1> undef, i32 undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %38 = call <vscale x 8 x i64> @llvm.vp.fshl.nxv8i64(<vscale x 8 x i64> undef, <vscale x 8 x i64> undef, <vscale x 8 x i64> undef, <vscale x 8 x i1> undef, i32 undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
-;
-; TYPEBASED-LABEL: 'vp_fshl'
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %1 = call <2 x i8> @llvm.vp.fshl.v2i8(<2 x i8> undef, <2 x i8> undef, <2 x i8> undef, <2 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %2 = call <4 x i8> @llvm.vp.fshl.v4i8(<4 x i8> undef, <4 x i8> undef, <4 x i8> undef, <4 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %3 = call <8 x i8> @llvm.vp.fshl.v8i8(<8 x i8> undef, <8 x i8> undef, <8 x i8> undef, <8 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %4 = call <16 x i8> @llvm.vp.fshl.v16i8(<16 x i8> undef, <16 x i8> undef, <16 x i8> undef, <16 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %5 = call <vscale x 1 x i8> @llvm.vp.fshl.nxv1i8(<vscale x 1 x i8> undef, <vscale x 1 x i8> undef, <vscale x 1 x i8> undef, <vscale x 1 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %6 = call <vscale x 2 x i8> @llvm.vp.fshl.nxv2i8(<vscale x 2 x i8> undef, <vscale x 2 x i8> undef, <vscale x 2 x i8> undef, <vscale x 2 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %7 = call <vscale x 4 x i8> @llvm.vp.fshl.nxv4i8(<vscale x 4 x i8> undef, <vscale x 4 x i8> undef, <vscale x 4 x i8> undef, <vscale x 4 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %8 = call <vscale x 8 x i8> @llvm.vp.fshl.nxv8i8(<vscale x 8 x i8> undef, <vscale x 8 x i8> undef, <vscale x 8 x i8> undef, <vscale x 8 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %9 = call <vscale x 16 x i8> @llvm.vp.fshl.nxv16i8(<vscale x 16 x i8> undef, <vscale x 16 x i8> undef, <vscale x 16 x i8> undef, <vscale x 16 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %10 = call <vscale x 32 x i8> @llvm.vp.fshl.nxv32i8(<vscale x 32 x i8> undef, <vscale x 32 x i8> undef, <vscale x 32 x i8> undef, <vscale x 32 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %11 = call <vscale x 64 x i8> @llvm.vp.fshl.nxv64i8(<vscale x 64 x i8> undef, <vscale x 64 x i8> undef, <vscale x 64 x i8> undef, <vscale x 64 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %12 = call <2 x i16> @llvm.vp.fshl.v2i16(<2 x i16> undef, <2 x i16> undef, <2 x i16> undef, <2 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %13 = call <4 x i16> @llvm.vp.fshl.v4i16(<4 x i16> undef, <4 x i16> undef, <4 x i16> undef, <4 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %14 = call <8 x i16> @llvm.vp.fshl.v8i16(<8 x i16> undef, <8 x i16> undef, <8 x i16> undef, <8 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %15 = call <16 x i16> @llvm.vp.fshl.v16i16(<16 x i16> undef, <16 x i16> undef, <16 x i16> undef, <16 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %16 = call <vscale x 1 x i16> @llvm.vp.fshl.nxv1i16(<vscale x 1 x i16> undef, <vscale x 1 x i16> undef, <vscale x 1 x i16> undef, <vscale x 1 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %17 = call <vscale x 2 x i16> @llvm.vp.fshl.nxv2i16(<vscale x 2 x i16> undef, <vscale x 2 x i16> undef, <vscale x 2 x i16> undef, <vscale x 2 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %18 = call <vscale x 4 x i16> @llvm.vp.fshl.nxv4i16(<vscale x 4 x i16> undef, <vscale x 4 x i16> undef, <vscale x 4 x i16> undef, <vscale x 4 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %19 = call <vscale x 8 x i16> @llvm.vp.fshl.nxv8i16(<vscale x 8 x i16> undef, <vscale x 8 x i16> undef, <vscale x 8 x i16> undef, <vscale x 8 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %20 = call <vscale x 16 x i16> @llvm.vp.fshl.nxv16i16(<vscale x 16 x i16> undef, <vscale x 16 x i16> undef, <vscale x 16 x i16> undef, <vscale x 16 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %21 = call <vscale x 32 x i16> @llvm.vp.fshl.nxv32i16(<vscale x 32 x i16> undef, <vscale x 32 x i16> undef, <vscale x 32 x i16> undef, <vscale x 32 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %22 = call <2 x i32> @llvm.vp.fshl.v2i32(<2 x i32> undef, <2 x i32> undef, <2 x i32> undef, <2 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %23 = call <4 x i32> @llvm.vp.fshl.v4i32(<4 x i32> undef, <4 x i32> undef, <4 x i32> undef, <4 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %24 = call <8 x i32> @llvm.vp.fshl.v8i32(<8 x i32> undef, <8 x i32> undef, <8 x i32> undef, <8 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %25 = call <16 x i32> @llvm.vp.fshl.v16i32(<16 x i32> undef, <16 x i32> undef, <16 x i32> undef, <16 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %26 = call <vscale x 1 x i32> @llvm.vp.fshl.nxv1i32(<vscale x 1 x i32> undef, <vscale x 1 x i32> undef, <vscale x 1 x i32> undef, <vscale x 1 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %27 = call <vscale x 2 x i32> @llvm.vp.fshl.nxv2i32(<vscale x 2 x i32> undef, <vscale x 2 x i32> undef, <vscale x 2 x i32> undef, <vscale x 2 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %28 = call <vscale x 4 x i32> @llvm.vp.fshl.nxv4i32(<vscale x 4 x i32> undef, <vscale x 4 x i32> undef, <vscale x 4 x i32> undef, <vscale x 4 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %29 = call <vscale x 8 x i32> @llvm.vp.fshl.nxv8i32(<vscale x 8 x i32> undef, <vscale x 8 x i32> undef, <vscale x 8 x i32> undef, <vscale x 8 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %30 = call <vscale x 16 x i32> @llvm.vp.fshl.nxv16i32(<vscale x 16 x i32> undef, <vscale x 16 x i32> undef, <vscale x 16 x i32> undef, <vscale x 16 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %31 = call <2 x i64> @llvm.vp.fshl.v2i64(<2 x i64> undef, <2 x i64> undef, <2 x i64> undef, <2 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %32 = call <4 x i64> @llvm.vp.fshl.v4i64(<4 x i64> undef, <4 x i64> undef, <4 x i64> undef, <4 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %33 = call <8 x i64> @llvm.vp.fshl.v8i64(<8 x i64> undef, <8 x i64> undef, <8 x i64> undef, <8 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %34 = call <16 x i64> @llvm.vp.fshl.v16i64(<16 x i64> undef, <16 x i64> undef, <16 x i64> undef, <16 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %35 = call <vscale x 1 x i64> @llvm.vp.fshl.nxv1i64(<vscale x 1 x i64> undef, <vscale x 1 x i64> undef, <vscale x 1 x i64> undef, <vscale x 1 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %36 = call <vscale x 2 x i64> @llvm.vp.fshl.nxv2i64(<vscale x 2 x i64> undef, <vscale x 2 x i64> undef, <vscale x 2 x i64> undef, <vscale x 2 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %37 = call <vscale x 4 x i64> @llvm.vp.fshl.nxv4i64(<vscale x 4 x i64> undef, <vscale x 4 x i64> undef, <vscale x 4 x i64> undef, <vscale x 4 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %38 = call <vscale x 8 x i64> @llvm.vp.fshl.nxv8i64(<vscale x 8 x i64> undef, <vscale x 8 x i64> undef, <vscale x 8 x i64> undef, <vscale x 8 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
   call <2 x i8> @llvm.vp.fshl.v2i8(<2 x i8> undef, <2 x i8> undef, <2 x i8> undef, <2 x i1> undef, i32 undef)
   call <4 x i8> @llvm.vp.fshl.v4i8(<4 x i8> undef, <4 x i8> undef, <4 x i8> undef, <4 x i1> undef, i32 undef)
@@ -373,73 +277,6 @@ define void @add() {
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %t62 = call <vscale x 16 x i64> @llvm.vp.add.nxv16i64(<vscale x 16 x i64> undef, <vscale x 16 x i64> undef, <vscale x 16 x i1> undef, i32 undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %t63 = add <vscale x 16 x i64> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
-;
-; TYPEBASED-LABEL: 'add'
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %t0 = call <2 x i8> @llvm.vp.add.v2i8(<2 x i8> undef, <2 x i8> undef, <2 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %t1 = add <2 x i8> undef, undef
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %t2 = call <4 x i8> @llvm.vp.add.v4i8(<4 x i8> undef, <4 x i8> undef, <4 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %t3 = add <4 x i8> undef, undef
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %t4 = call <8 x i8> @llvm.vp.add.v8i8(<8 x i8> undef, <8 x i8> undef, <8 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %t5 = add <8 x i8> undef, undef
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %t6 = call <16 x i8> @llvm.vp.add.v16i8(<16 x i8> undef, <16 x i8> undef, <16 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %t7 = add <16 x i8> undef, undef
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %t8 = call <2 x i16> @llvm.vp.add.v2i16(<2 x i16> undef, <2 x i16> undef, <2 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %t9 = add <2 x i16> undef, undef
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %t10 = call <4 x i16> @llvm.vp.add.v4i16(<4 x i16> undef, <4 x i16> undef, <4 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %t11 = add <4 x i16> undef, undef
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %t12 = call <8 x i16> @llvm.vp.add.v8i16(<8 x i16> undef, <8 x i16> undef, <8 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %t13 = add <8 x i16> undef, undef
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %t14 = call <16 x i16> @llvm.vp.add.v16i16(<16 x i16> undef, <16 x i16> undef, <16 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %t15 = add <16 x i16> undef, undef
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %t16 = call <2 x i32> @llvm.vp.add.v2i32(<2 x i32> undef, <2 x i32> undef, <2 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %t17 = add <2 x i32> undef, undef
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %t18 = call <4 x i32> @llvm.vp.add.v4i32(<4 x i32> undef, <4 x i32> undef, <4 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %t19 = add <4 x i32> undef, undef
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %t20 = call <8 x i32> @llvm.vp.add.v8i32(<8 x i32> undef, <8 x i32> undef, <8 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %t21 = add <8 x i32> undef, undef
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %t22 = call <16 x i32> @llvm.vp.add.v16i32(<16 x i32> undef, <16 x i32> undef, <16 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %t23 = add <16 x i32> undef, undef
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %t24 = call <2 x i64> @llvm.vp.add.v2i64(<2 x i64> undef, <2 x i64> undef, <2 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %t25 = add <2 x i64> undef, undef
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %t26 = call <4 x i64> @llvm.vp.add.v4i64(<4 x i64> undef, <4 x i64> undef, <4 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %t27 = add <4 x i64> undef, undef
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %t28 = call <8 x i64> @llvm.vp.add.v8i64(<8 x i64> undef, <8 x i64> undef, <8 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %t29 = add <8 x i64> undef, undef
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %t30 = call <16 x i64> @llvm.vp.add.v16i64(<16 x i64> undef, <16 x i64> undef, <16 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %t31 = add <16 x i64> undef, undef
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %t32 = call <vscale x 2 x i8> @llvm.vp.add.nxv2i8(<vscale x 2 x i8> undef, <vscale x 2 x i8> undef, <vscale x 2 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %t33 = add <vscale x 2 x i8> undef, undef
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %t34 = call <vscale x 4 x i8> @llvm.vp.add.nxv4i8(<vscale x 4 x i8> undef, <vscale x 4 x i8> undef, <vscale x 4 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %t35 = add <vscale x 4 x i8> undef, undef
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %t36 = call <vscale x 8 x i8> @llvm.vp.add.nxv8i8(<vscale x 8 x i8> undef, <vscale x 8 x i8> undef, <vscale x 8 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %t37 = add <vscale x 8 x i8> undef, undef
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %t38 = call <vscale x 16 x i8> @llvm.vp.add.nxv16i8(<vscale x 16 x i8> undef, <vscale x 16 x i8> undef, <vscale x 16 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %t39 = add <vscale x 16 x i8> undef, undef
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %t40 = call <vscale x 2 x i16> @llvm.vp.add.nxv2i16(<vscale x 2 x i16> undef, <vscale x 2 x i16> undef, <vscale x 2 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %t41 = add <vscale x 2 x i16> undef, undef
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %t42 = call <vscale x 4 x i16> @llvm.vp.add.nxv4i16(<vscale x 4 x i16> undef, <vscale x 4 x i16> undef, <vscale x 4 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %t43 = add <vscale x 4 x i16> undef, undef
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %t44 = call <vscale x 8 x i16> @llvm.vp.add.nxv8i16(<vscale x 8 x i16> undef, <vscale x 8 x i16> undef, <vscale x 8 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %t45 = add <vscale x 8 x i16> undef, undef
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %t46 = call <vscale x 16 x i16> @llvm.vp.add.nxv16i16(<vscale x 16 x i16> undef, <vscale x 16 x i16> undef, <vscale x 16 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %t47 = add <vscale x 16 x i16> undef, undef
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %t48 = call <vscale x 2 x i32> @llvm.vp.add.nxv2i32(<vscale x 2 x i32> undef, <vscale x 2 x i32> undef, <vscale x 2 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %t49 = add <vscale x 2 x i32> undef, undef
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %t50 = call <vscale x 4 x i32> @llvm.vp.add.nxv4i32(<vscale x 4 x i32> undef, <vscale x 4 x i32> undef, <vscale x 4 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %t51 = add <vscale x 4 x i32> undef, undef
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %t52 = call <vscale x 8 x i32> @llvm.vp.add.nxv8i32(<vscale x 8 x i32> undef, <vscale x 8 x i32> undef, <vscale x 8 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %t53 = add <vscale x 8 x i32> undef, undef
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %t54 = call <vscale x 16 x i32> @llvm.vp.add.nxv16i32(<vscale x 16 x i32> undef, <vscale x 16 x i32> undef, <vscale x 16 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %t55 = add <vscale x 16 x i32> undef, undef
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %t56 = call <vscale x 2 x i64> @llvm.vp.add.nxv2i64(<vscale x 2 x i64> undef, <vscale x 2 x i64> undef, <vscale x 2 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %t57 = add <vscale x 2 x i64> undef, undef
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %t58 = call <vscale x 4 x i64> @llvm.vp.add.nxv4i64(<vscale x 4 x i64> undef, <vscale x 4 x i64> undef, <vscale x 4 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %t59 = add <vscale x 4 x i64> undef, undef
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %t60 = call <vscale x 8 x i64> @llvm.vp.add.nxv8i64(<vscale x 8 x i64> undef, <vscale x 8 x i64> undef, <vscale x 8 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %t61 = add <vscale x 8 x i64> undef, undef
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %t62 = call <vscale x 16 x i64> @llvm.vp.add.nxv16i64(<vscale x 16 x i64> undef, <vscale x 16 x i64> undef, <vscale x 16 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %t63 = add <vscale x 16 x i64> undef, undef
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
   %t0 = call <2 x i8> @llvm.vp.add.v2i8(<2 x i8> undef, <2 x i8> undef, <2 x i1> undef, i32 undef)
   %t1 = add <2 x i8> undef, undef
@@ -575,73 +412,6 @@ define void @and() {
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %t62 = call <vscale x 16 x i64> @llvm.vp.and.nxv16i64(<vscale x 16 x i64> undef, <vscale x 16 x i64> undef, <vscale x 16 x i1> undef, i32 undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %t63 = and <vscale x 16 x i64> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
-;
-; TYPEBASED-LABEL: 'and'
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %t0 = call <2 x i8> @llvm.vp.and.v2i8(<2 x i8> undef, <2 x i8> undef, <2 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %t1 = and <2 x i8> undef, undef
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %t2 = call <4 x i8> @llvm.vp.and.v4i8(<4 x i8> undef, <4 x i8> undef, <4 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %t3 = and <4 x i8> undef, undef
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %t4 = call <8 x i8> @llvm.vp.and.v8i8(<8 x i8> undef, <8 x i8> undef, <8 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %t5 = and <8 x i8> undef, undef
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %t6 = call <16 x i8> @llvm.vp.and.v16i8(<16 x i8> undef, <16 x i8> undef, <16 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %t7 = and <16 x i8> undef, undef
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %t8 = call <2 x i16> @llvm.vp.and.v2i16(<2 x i16> undef, <2 x i16> undef, <2 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %t9 = and <2 x i16> undef, undef
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %t10 = call <4 x i16> @llvm.vp.and.v4i16(<4 x i16> undef, <4 x i16> undef, <4 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %t11 = and <4 x i16> undef, undef
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %t12 = call <8 x i16> @llvm.vp.and.v8i16(<8 x i16> undef, <8 x i16> undef, <8 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %t13 = and <8 x i16> undef, undef
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %t14 = call <16 x i16> @llvm.vp.and.v16i16(<16 x i16> undef, <16 x i16> undef, <16 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %t15 = and <16 x i16> undef, undef
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %t16 = call <2 x i32> @llvm.vp.and.v2i32(<2 x i32> undef, <2 x i32> undef, <2 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %t17 = and <2 x i32> undef, undef
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %t18 = call <4 x i32> @llvm.vp.and.v4i32(<4 x i32> undef, <4 x i32> undef, <4 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %t19 = and <4 x i32> undef, undef
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %t20 = call <8 x i32> @llvm.vp.and.v8i32(<8 x i32> undef, <8 x i32> undef, <8 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %t21 = and <8 x i32> undef, undef
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %t22 = call <16 x i32> @llvm.vp.and.v16i32(<16 x i32> undef, <16 x i32> undef, <16 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %t23 = and <16 x i32> undef, undef
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %t24 = call <2 x i64> @llvm.vp.and.v2i64(<2 x i64> undef, <2 x i64> undef, <2 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %t25 = and <2 x i64> undef, undef
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %t26 = call <4 x i64> @llvm.vp.and.v4i64(<4 x i64> undef, <4 x i64> undef, <4 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %t27 = and <4 x i64> undef, undef
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %t28 = call <8 x i64> @llvm.vp.and.v8i64(<8 x i64> undef, <8 x i64> undef, <8 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %t29 = and <8 x i64> undef, undef
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %t30 = call <16 x i64> @llvm.vp.and.v16i64(<16 x i64> undef, <16 x i64> undef, <16 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %t31 = and <16 x i64> undef, undef
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %t32 = call <vscale x 2 x i8> @llvm.vp.and.nxv2i8(<vscale x 2 x i8> undef, <vscale x 2 x i8> undef, <vscale x 2 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %t33 = and <vscale x 2 x i8> undef, undef
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %t34 = call <vscale x 4 x i8> @llvm.vp.and.nxv4i8(<vscale x 4 x i8> undef, <vscale x 4 x i8> undef, <vscale x 4 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %t35 = and <vscale x 4 x i8> undef, undef
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %t36 = call <vscale x 8 x i8> @llvm.vp.and.nxv8i8(<vscale x 8 x i8> undef, <vscale x 8 x i8> undef, <vscale x 8 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %t37 = and <vscale x 8 x i8> undef, undef
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %t38 = call <vscale x 16 x i8> @llvm.vp.and.nxv16i8(<vscale x 16 x i8> undef, <vscale x 16 x i8> undef, <vscale x 16 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %t39 = and <vscale x 16 x i8> undef, undef
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %t40 = call <vscale x 2 x i16> @llvm.vp.and.nxv2i16(<vscale x 2 x i16> undef, <vscale x 2 x i16> undef, <vscale x 2 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %t41 = and <vscale x 2 x i16> undef, undef
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %t42 = call <vscale x 4 x i16> @llvm.vp.and.nxv4i16(<vscale x 4 x i16> undef, <vscale x 4 x i16> undef, <vscale x 4 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %t43 = and <vscale x 4 x i16> undef, undef
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %t44 = call <vscale x 8 x i16> @llvm.vp.and.nxv8i16(<vscale x 8 x i16> undef, <vscale x 8 x i16> undef, <vscale x 8 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %t45 = and <vscale x 8 x i16> undef, undef
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %t46 = call <vscale x 16 x i16> @llvm.vp.and.nxv16i16(<vscale x 16 x i16> undef, <vscale x 16 x i16> undef, <vscale x 16 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %t47 = and <vscale x 16 x i16> undef, undef
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %t48 = call <vscale x 2 x i32> @llvm.vp.and.nxv2i32(<vscale x 2 x i32> undef, <vscale x 2 x i32> undef, <vscale x 2 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %t49 = and <vscale x 2 x i32> undef, undef
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %t50 = call <vscale x 4 x i32> @llvm.vp.and.nxv4i32(<vscale x 4 x i32> undef, <vscale x 4 x i32> undef, <vscale x 4 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %t51 = and <vscale x 4 x i32> undef, undef
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %t52 = call <vscale x 8 x i32> @llvm.vp.and.nxv8i32(<vscale x 8 x i32> undef, <vscale x 8 x i32> undef, <vscale x 8 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %t53 = and <vscale x 8 x i32> undef, undef
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %t54 = call <vscale x 16 x i32> @llvm.vp.and.nxv16i32(<vscale x 16 x i32> undef, <vscale x 16 x i32> undef, <vscale x 16 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %t55 = and <vscale x 16 x i32> undef, undef
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %t56 = call <vscale x 2 x i64> @llvm.vp.and.nxv2i64(<vscale x 2 x i64> undef, <vscale x 2 x i64> undef, <vscale x 2 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %t57 = and <vscale x 2 x i64> undef, undef
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %t58 = call <vscale x 4 x i64> @llvm.vp.and.nxv4i64(<vscale x 4 x i64> undef, <vscale x 4 x i64> undef, <vscale x 4 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %t59 = and <vscale x 4 x i64> undef, undef
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %t60 = call <vscale x 8 x i64> @llvm.vp.and.nxv8i64(<vscale x 8 x i64> undef, <vscale x 8 x i64> undef, <vscale x 8 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %t61 = and <vscale x 8 x i64> undef, undef
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %t62 = call <vscale x 16 x i64> @llvm.vp.and.nxv16i64(<vscale x 16 x i64> undef, <vscale x 16 x i64> undef, <vscale x 16 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %t63 = and <vscale x 16 x i64> undef, undef
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
   %t0 = call <2 x i8> @llvm.vp.and.v2i8(<2 x i8> undef, <2 x i8> undef, <2 x i1> undef, i32 undef)
   %t1 = and <2 x i8> undef, undef
@@ -745,41 +515,6 @@ define void @smax() {
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %t60 = call <vscale x 8 x i64> @llvm.vp.smax.nxv8i64(<vscale x 8 x i64> undef, <vscale x 8 x i64> undef, <vscale x 8 x i1> undef, i32 undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %t62 = call <vscale x 16 x i64> @llvm.vp.smax.nxv16i64(<vscale x 16 x i64> undef, <vscale x 16 x i64> undef, <vscale x 16 x i1> undef, i32 undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
-;
-; TYPEBASED-LABEL: 'smax'
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %t0 = call <2 x i8> @llvm.vp.smax.v2i8(<2 x i8> undef, <2 x i8> undef, <2 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %t2 = call <4 x i8> @llvm.vp.smax.v4i8(<4 x i8> undef, <4 x i8> undef, <4 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %t4 = call <8 x i8> @llvm.vp.smax.v8i8(<8 x i8> undef, <8 x i8> undef, <8 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %t6 = call <16 x i8> @llvm.vp.smax.v16i8(<16 x i8> undef, <16 x i8> undef, <16 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %t8 = call <2 x i16> @llvm.vp.smax.v2i16(<2 x i16> undef, <2 x i16> undef, <2 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %t10 = call <4 x i16> @llvm.vp.smax.v4i16(<4 x i16> undef, <4 x i16> undef, <4 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %t12 = call <8 x i16> @llvm.vp.smax.v8i16(<8 x i16> undef, <8 x i16> undef, <8 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %t14 = call <16 x i16> @llvm.vp.smax.v16i16(<16 x i16> undef, <16 x i16> undef, <16 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %t16 = call <2 x i32> @llvm.vp.smax.v2i32(<2 x i32> undef, <2 x i32> undef, <2 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %t18 = call <4 x i32> @llvm.vp.smax.v4i32(<4 x i32> undef, <4 x i32> undef, <4 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %t20 = call <8 x i32> @llvm.vp.smax.v8i32(<8 x i32> undef, <8 x i32> undef, <8 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %t22 = call <16 x i32> @llvm.vp.smax.v16i32(<16 x i32> undef, <16 x i32> undef, <16 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %t24 = call <2 x i64> @llvm.vp.smax.v2i64(<2 x i64> undef, <2 x i64> undef, <2 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %t26 = call <4 x i64> @llvm.vp.smax.v4i64(<4 x i64> undef, <4 x i64> undef, <4 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %t28 = call <8 x i64> @llvm.vp.smax.v8i64(<8 x i64> undef, <8 x i64> undef, <8 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %t30 = call <16 x i64> @llvm.vp.smax.v16i64(<16 x i64> undef, <16 x i64> undef, <16 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %t32 = call <vscale x 2 x i8> @llvm.vp.smax.nxv2i8(<vscale x 2 x i8> undef, <vscale x 2 x i8> undef, <vscale x 2 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %t34 = call <vscale x 4 x i8> @llvm.vp.smax.nxv4i8(<vscale x 4 x i8> undef, <vscale x 4 x i8> undef, <vscale x 4 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %t36 = call <vscale x 8 x i8> @llvm.vp.smax.nxv8i8(<vscale x 8 x i8> undef, <vscale x 8 x i8> undef, <vscale x 8 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %t38 = call <vscale x 16 x i8> @llvm.vp.smax.nxv16i8(<vscale x 16 x i8> undef, <vscale x 16 x i8> undef, <vscale x 16 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %t40 = call <vscale x 2 x i16> @llvm.vp.smax.nxv2i16(<vscale x 2 x i16> undef, <vscale x 2 x i16> undef, <vscale x 2 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %t42 = call <vscale x 4 x i16> @llvm.vp.smax.nxv4i16(<vscale x 4 x i16> undef, <vscale x 4 x i16> undef, <vscale x 4 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %t44 = call <vscale x 8 x i16> @llvm.vp.smax.nxv8i16(<vscale x 8 x i16> undef, <vscale x 8 x i16> undef, <vscale x 8 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %t46 = call <vscale x 16 x i16> @llvm.vp.smax.nxv16i16(<vscale x 16 x i16> undef, <vscale x 16 x i16> undef, <vscale x 16 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %t48 = call <vscale x 2 x i32> @llvm.vp.smax.nxv2i32(<vscale x 2 x i32> undef, <vscale x 2 x i32> undef, <vscale x 2 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %t50 = call <vscale x 4 x i32> @llvm.vp.smax.nxv4i32(<vscale x 4 x i32> undef, <vscale x 4 x i32> undef, <vscale x 4 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %t52 = call <vscale x 8 x i32> @llvm.vp.smax.nxv8i32(<vscale x 8 x i32> undef, <vscale x 8 x i32> undef, <vscale x 8 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %t54 = call <vscale x 16 x i32> @llvm.vp.smax.nxv16i32(<vscale x 16 x i32> undef, <vscale x 16 x i32> undef, <vscale x 16 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %t56 = call <vscale x 2 x i64> @llvm.vp.smax.nxv2i64(<vscale x 2 x i64> undef, <vscale x 2 x i64> undef, <vscale x 2 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %t58 = call <vscale x 4 x i64> @llvm.vp.smax.nxv4i64(<vscale x 4 x i64> undef, <vscale x 4 x i64> undef, <vscale x 4 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %t60 = call <vscale x 8 x i64> @llvm.vp.smax.nxv8i64(<vscale x 8 x i64> undef, <vscale x 8 x i64> undef, <vscale x 8 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %t62 = call <vscale x 16 x i64> @llvm.vp.smax.nxv16i64(<vscale x 16 x i64> undef, <vscale x 16 x i64> undef, <vscale x 16 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
   %t0 = call <2 x i8> @llvm.vp.smax.v2i8(<2 x i8> undef, <2 x i8> undef, <2 x i1> undef, i32 undef)
   %t2 = call <4 x i8> @llvm.vp.smax.v4i8(<4 x i8> undef, <4 x i8> undef, <4 x i1> undef, i32 undef)
@@ -852,41 +587,6 @@ define void @smin() {
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %t60 = call <vscale x 8 x i64> @llvm.vp.smin.nxv8i64(<vscale x 8 x i64> undef, <vscale x 8 x i64> undef, <vscale x 8 x i1> undef, i32 undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %t62 = call <vscale x 16 x i64> @llvm.vp.smin.nxv16i64(<vscale x 16 x i64> undef, <vscale x 16 x i64> undef, <vscale x 16 x i1> undef, i32 undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
-;
-; TYPEBASED-LABEL: 'smin'
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %t0 = call <2 x i8> @llvm.vp.smin.v2i8(<2 x i8> undef, <2 x i8> undef, <2 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %t2 = call <4 x i8> @llvm.vp.smin.v4i8(<4 x i8> undef, <4 x i8> undef, <4 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %t4 = call <8 x i8> @llvm.vp.smin.v8i8(<8 x i8> undef, <8 x i8> undef, <8 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %t6 = call <16 x i8> @llvm.vp.smin.v16i8(<16 x i8> undef, <16 x i8> undef, <16 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %t8 = call <2 x i16> @llvm.vp.smin.v2i16(<2 x i16> undef, <2 x i16> undef, <2 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %t10 = call <4 x i16> @llvm.vp.smin.v4i16(<4 x i16> undef, <4 x i16> undef, <4 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %t12 = call <8 x i16> @llvm.vp.smin.v8i16(<8 x i16> undef, <8 x i16> undef, <8 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %t14 = call <16 x i16> @llvm.vp.smin.v16i16(<16 x i16> undef, <16 x i16> undef, <16 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %t16 = call <2 x i32> @llvm.vp.smin.v2i32(<2 x i32> undef, <2 x i32> undef, <2 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %t18 = call <4 x i32> @llvm.vp.smin.v4i32(<4 x i32> undef, <4 x i32> undef, <4 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %t20 = call <8 x i32> @llvm.vp.smin.v8i32(<8 x i32> undef, <8 x i32> undef, <8 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %t22 = call <16 x i32> @llvm.vp.smin.v16i32(<16 x i32> undef, <16 x i32> undef, <16 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %t24 = call <2 x i64> @llvm.vp.smin.v2i64(<2 x i64> undef, <2 x i64> undef, <2 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %t26 = call <4 x i64> @llvm.vp.smin.v4i64(<4 x i64> undef, <4 x i64> undef, <4 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %t28 = call <8 x i64> @llvm.vp.smin.v8i64(<8 x i64> undef, <8 x i64> undef, <8 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %t30 = call <16 x i64> @llvm.vp.smin.v16i64(<16 x i64> undef, <16 x i64> undef, <16 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %t32 = call <vscale x 2 x i8> @llvm.vp.smin.nxv2i8(<vscale x 2 x i8> undef, <vscale x 2 x i8> undef, <vscale x 2 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %t34 = call <vscale x 4 x i8> @llvm.vp.smin.nxv4i8(<vscale x 4 x i8> undef, <vscale x 4 x i8> undef, <vscale x 4 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %t36 = call <vscale x 8 x i8> @llvm.vp.smin.nxv8i8(<vscale x 8 x i8> undef, <vscale x 8 x i8> undef, <vscale x 8 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %t38 = call <vscale x 16 x i8> @llvm.vp.smin.nxv16i8(<vscale x 16 x i8> undef, <vscale x 16 x i8> undef, <vscale x 16 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %t40 = call <vscale x 2 x i16> @llvm.vp.smin.nxv2i16(<vscale x 2 x i16> undef, <vscale x 2 x i16> undef, <vscale x 2 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %t42 = call <vscale x 4 x i16> @llvm.vp.smin.nxv4i16(<vscale x 4 x i16> undef, <vscale x 4 x i16> undef, <vscale x 4 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %t44 = call <vscale x 8 x i16> @llvm.vp.smin.nxv8i16(<vscale x 8 x i16> undef, <vscale x 8 x i16> undef, <vscale x 8 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %t46 = call <vscale x 16 x i16> @llvm.vp.smin.nxv16i16(<vscale x 16 x i16> undef, <vscale x 16 x i16> undef, <vscale x 16 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %t48 = call <vscale x 2 x i32> @llvm.vp.smin.nxv2i32(<vscale x 2 x i32> undef, <vscale x 2 x i32> undef, <vscale x 2 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %t50 = call <vscale x 4 x i32> @llvm.vp.smin.nxv4i32(<vscale x 4 x i32> undef, <vscale x 4 x i32> undef, <vscale x 4 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %t52 = call <vscale x 8 x i32> @llvm.vp.smin.nxv8i32(<vscale x 8 x i32> undef, <vscale x 8 x i32> undef, <vscale x 8 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %t54 = call <vscale x 16 x i32> @llvm.vp.smin.nxv16i32(<vscale x 16 x i32> undef, <vscale x 16 x i32> undef, <vscale x 16 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %t56 = call <vscale x 2 x i64> @llvm.vp.smin.nxv2i64(<vscale x 2 x i64> undef, <vscale x 2 x i64> undef, <vscale x 2 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %t58 = call <vscale x 4 x i64> @llvm.vp.smin.nxv4i64(<vscale x 4 x i64> undef, <vscale x 4 x i64> undef, <vscale x 4 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %t60 = call <vscale x 8 x i64> @llvm.vp.smin.nxv8i64(<vscale x 8 x i64> undef, <vscale x 8 x i64> undef, <vscale x 8 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %t62 = call <vscale x 16 x i64> @llvm.vp.smin.nxv16i64(<vscale x 16 x i64> undef, <vscale x 16 x i64> undef, <vscale x 16 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
   %t0 = call <2 x i8> @llvm.vp.smin.v2i8(<2 x i8> undef, <2 x i8> undef, <2 x i1> undef, i32 undef)
   %t2 = call <4 x i8> @llvm.vp.smin.v4i8(<4 x i8> undef, <4 x i8> undef, <4 x i1> undef, i32 undef)
@@ -958,41 +658,6 @@ define void @umax() {
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %t60 = call <vscale x 8 x i64> @llvm.vp.umax.nxv8i64(<vscale x 8 x i64> undef, <vscale x 8 x i64> undef, <vscale x 8 x i1> undef, i32 undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %t62 = call <vscale x 16 x i64> @llvm.vp.umax.nxv16i64(<vscale x 16 x i64> undef, <vscale x 16 x i64> undef, <vscale x 16 x i1> undef, i32 undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
-;
-; TYPEBASED-LABEL: 'umax'
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %t0 = call <2 x i8> @llvm.vp.umax.v2i8(<2 x i8> undef, <2 x i8> undef, <2 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %t2 = call <4 x i8> @llvm.vp.umax.v4i8(<4 x i8> undef, <4 x i8> undef, <4 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %t4 = call <8 x i8> @llvm.vp.umax.v8i8(<8 x i8> undef, <8 x i8> undef, <8 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %t6 = call <16 x i8> @llvm.vp.umax.v16i8(<16 x i8> undef, <16 x i8> undef, <16 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %t8 = call <2 x i16> @llvm.vp.umax.v2i16(<2 x i16> undef, <2 x i16> undef, <2 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %t10 = call <4 x i16> @llvm.vp.umax.v4i16(<4 x i16> undef, <4 x i16> undef, <4 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %t12 = call <8 x i16> @llvm.vp.umax.v8i16(<8 x i16> undef, <8 x i16> undef, <8 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %t14 = call <16 x i16> @llvm.vp.umax.v16i16(<16 x i16> undef, <16 x i16> undef, <16 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %t16 = call <2 x i32> @llvm.vp.umax.v2i32(<2 x i32> undef, <2 x i32> undef, <2 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %t18 = call <4 x i32> @llvm.vp.umax.v4i32(<4 x i32> undef, <4 x i32> undef, <4 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %t20 = call <8 x i32> @llvm.vp.umax.v8i32(<8 x i32> undef, <8 x i32> undef, <8 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %t22 = call <16 x i32> @llvm.vp.umax.v16i32(<16 x i32> undef, <16 x i32> undef, <16 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %t24 = call <2 x i64> @llvm.vp.umax.v2i64(<2 x i64> undef, <2 x i64> undef, <2 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %t26 = call <4 x i64> @llvm.vp.umax.v4i64(<4 x i64> undef, <4 x i64> undef, <4 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %t28 = call <8 x i64> @llvm.vp.umax.v8i64(<8 x i64> undef, <8 x i64> undef, <8 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %t30 = call <16 x i64> @llvm.vp.umax.v16i64(<16 x i64> undef, <16 x i64> undef, <16 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %t32 = call <vscale x 2 x i8> @llvm.vp.umax.nxv2i8(<vscale x 2 x i8> undef, <vscale x 2 x i8> undef, <vscale x 2 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %t34 = call <vscale x 4 x i8> @llvm.vp.umax.nxv4i8(<vscale x 4 x i8> undef, <vscale x 4 x i8> undef, <vscale x 4 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %t36 = call <vscale x 8 x i8> @llvm.vp.umax.nxv8i8(<vscale x 8 x i8> undef, <vscale x 8 x i8> undef, <vscale x 8 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %t38 = call <vscale x 16 x i8> @llvm.vp.umax.nxv16i8(<vscale x 16 x i8> undef, <vscale x 16 x i8> undef, <vscale x 16 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %t40 = call <vscale x 2 x i16> @llvm.vp.umax.nxv2i16(<vscale x 2 x i16> undef, <vscale x 2 x i16> undef, <vscale x 2 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %t42 = call <vscale x 4 x i16> @llvm.vp.umax.nxv4i16(<vscale x 4 x i16> undef, <vscale x 4 x i16> undef, <vscale x 4 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %t44 = call <vscale x 8 x i16> @llvm.vp.umax.nxv8i16(<vscale x 8 x i16> undef, <vscale x 8 x i16> undef, <vscale x 8 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %t46 = call <vscale x 16 x i16> @llvm.vp.umax.nxv16i16(<vscale x 16 x i16> undef, <vscale x 16 x i16> undef, <vscale x 16 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %t48 = call <vscale x 2 x i32> @llvm.vp.umax.nxv2i32(<vscale x 2 x i32> undef, <vscale x 2 x i32> undef, <vscale x 2 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %t50 = call <vscale x 4 x i32> @llvm.vp.umax.nxv4i32(<vscale x 4 x i32> undef, <vscale x 4 x i32> undef, <vscale x 4 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %t52 = call <vscale x 8 x i32> @llvm.vp.umax.nxv8i32(<vscale x 8 x i32> undef, <vscale x 8 x i32> undef, <vscale x 8 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %t54 = call <vscale x 16 x i32> @llvm.vp.umax.nxv16i32(<vscale x 16 x i32> undef, <vscale x 16 x i32> undef, <vscale x 16 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %t56 = call <vscale x 2 x i64> @llvm.vp.umax.nxv2i64(<vscale x 2 x i64> undef, <vscale x 2 x i64> undef, <vscale x 2 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %t58 = call <vscale x 4 x i64> @llvm.vp.umax.nxv4i64(<vscale x 4 x i64> undef, <vscale x 4 x i64> undef, <vscale x 4 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %t60 = call <vscale x 8 x i64> @llvm.vp.umax.nxv8i64(<vscale x 8 x i64> undef, <vscale x 8 x i64> undef, <vscale x 8 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %t62 = call <vscale x 16 x i64> @llvm.vp.umax.nxv16i64(<vscale x 16 x i64> undef, <vscale x 16 x i64> undef, <vscale x 16 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
   %t0 = call <2 x i8> @llvm.vp.umax.v2i8(<2 x i8> undef, <2 x i8> undef, <2 x i1> undef, i32 undef)
   %t2 = call <4 x i8> @llvm.vp.umax.v4i8(<4 x i8> undef, <4 x i8> undef, <4 x i1> undef, i32 undef)
@@ -1064,41 +729,6 @@ define void @umin() {
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %t60 = call <vscale x 8 x i64> @llvm.vp.umin.nxv8i64(<vscale x 8 x i64> undef, <vscale x 8 x i64> undef, <vscale x 8 x i1> undef, i32 undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %t62 = call <vscale x 16 x i64> @llvm.vp.umin.nxv16i64(<vscale x 16 x i64> undef, <vscale x 16 x i64> undef, <vscale x 16 x i1> undef, i32 undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
-;
-; TYPEBASED-LABEL: 'umin'
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %t0 = call <2 x i8> @llvm.vp.umin.v2i8(<2 x i8> undef, <2 x i8> undef, <2 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %t2 = call <4 x i8> @llvm.vp.umin.v4i8(<4 x i8> undef, <4 x i8> undef, <4 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %t4 = call <8 x i8> @llvm.vp.umin.v8i8(<8 x i8> undef, <8 x i8> undef, <8 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %t6 = call <16 x i8> @llvm.vp.umin.v16i8(<16 x i8> undef, <16 x i8> undef, <16 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %t8 = call <2 x i16> @llvm.vp.umin.v2i16(<2 x i16> undef, <2 x i16> undef, <2 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %t10 = call <4 x i16> @llvm.vp.umin.v4i16(<4 x i16> undef, <4 x i16> undef, <4 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %t12 = call <8 x i16> @llvm.vp.umin.v8i16(<8 x i16> undef, <8 x i16> undef, <8 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %t14 = call <16 x i16> @llvm.vp.umin.v16i16(<16 x i16> undef, <16 x i16> undef, <16 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %t16 = call <2 x i32> @llvm.vp.umin.v2i32(<2 x i32> undef, <2 x i32> undef, <2 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %t18 = call <4 x i32> @llvm.vp.umin.v4i32(<4 x i32> undef, <4 x i32> undef, <4 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %t20 = call <8 x i32> @llvm.vp.umin.v8i32(<8 x i32> undef, <8 x i32> undef, <8 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %t22 = call <16 x i32> @llvm.vp.umin.v16i32(<16 x i32> undef, <16 x i32> undef, <16 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %t24 = call <2 x i64> @llvm.vp.umin.v2i64(<2 x i64> undef, <2 x i64> undef, <2 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %t26 = call <4 x i64> @llvm.vp.umin.v4i64(<4 x i64> undef, <4 x i64> undef, <4 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %t28 = call <8 x i64> @llvm.vp.umin.v8i64(<8 x i64> undef, <8 x i64> undef, <8 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %t30 = call <16 x i64> @llvm.vp.umin.v16i64(<16 x i64> undef, <16 x i64> undef, <16 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %t32 = call <vscale x 2 x i8> @llvm.vp.umin.nxv2i8(<vscale x 2 x i8> undef, <vscale x 2 x i8> undef, <vscale x 2 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %t34 = call <vscale x 4 x i8> @llvm.vp.umin.nxv4i8(<vscale x 4 x i8> undef, <vscale x 4 x i8> undef, <vscale x 4 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %t36 = call <vscale x 8 x i8> @llvm.vp.umin.nxv8i8(<vscale x 8 x i8> undef, <vscale x 8 x i8> undef, <vscale x 8 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %t38 = call <vscale x 16 x i8> @llvm.vp.umin.nxv16i8(<vscale x 16 x i8> undef, <vscale x 16 x i8> undef, <vscale x 16 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %t40 = call <vscale x 2 x i16> @llvm.vp.umin.nxv2i16(<vscale x 2 x i16> undef, <vscale x 2 x i16> undef, <vscale x 2 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %t42 = call <vscale x 4 x i16> @llvm.vp.umin.nxv4i16(<vscale x 4 x i16> undef, <vscale x 4 x i16> undef, <vscale x 4 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %t44 = call <vscale x 8 x i16> @llvm.vp.umin.nxv8i16(<vscale x 8 x i16> undef, <vscale x 8 x i16> undef, <vscale x 8 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %t46 = call <vscale x 16 x i16> @llvm.vp.umin.nxv16i16(<vscale x 16 x i16> undef, <vscale x 16 x i16> undef, <vscale x 16 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %t48 = call <vscale x 2 x i32> @llvm.vp.umin.nxv2i32(<vscale x 2 x i32> undef, <vscale x 2 x i32> undef, <vscale x 2 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %t50 = call <vscale x 4 x i32> @llvm.vp.umin.nxv4i32(<vscale x 4 x i32> undef, <vscale x 4 x i32> undef, <vscale x 4 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %t52 = call <vscale x 8 x i32> @llvm.vp.umin.nxv8i32(<vscale x 8 x i32> undef, <vscale x 8 x i32> undef, <vscale x 8 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %t54 = call <vscale x 16 x i32> @llvm.vp.umin.nxv16i32(<vscale x 16 x i32> undef, <vscale x 16 x i32> undef, <vscale x 16 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %t56 = call <vscale x 2 x i64> @llvm.vp.umin.nxv2i64(<vscale x 2 x i64> undef, <vscale x 2 x i64> undef, <vscale x 2 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %t58 = call <vscale x 4 x i64> @llvm.vp.umin.nxv4i64(<vscale x 4 x i64> undef, <vscale x 4 x i64> undef, <vscale x 4 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %t60 = call <vscale x 8 x i64> @llvm.vp.umin.nxv8i64(<vscale x 8 x i64> undef, <vscale x 8 x i64> undef, <vscale x 8 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %t62 = call <vscale x 16 x i64> @llvm.vp.umin.nxv16i64(<vscale x 16 x i64> undef, <vscale x 16 x i64> undef, <vscale x 16 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
   %t0 = call <2 x i8> @llvm.vp.umin.v2i8(<2 x i8> undef, <2 x i8> undef, <2 x i1> undef, i32 undef)
   %t2 = call <4 x i8> @llvm.vp.umin.v4i8(<4 x i8> undef, <4 x i8> undef, <4 x i1> undef, i32 undef)
@@ -1170,41 +800,6 @@ define void @abs() {
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %31 = call <vscale x 16 x i64> @llvm.vp.abs.nxv16i64(<vscale x 16 x i64> undef, i1 false, <vscale x 16 x i1> undef, i32 undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %32 = call <vscale x 16 x i64> @llvm.abs.nxv16i64(<vscale x 16 x i64> undef, i1 false)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
-;
-; TYPEBASED-LABEL: 'abs'
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %1 = call <2 x i8> @llvm.vp.abs.v2i8(<2 x i8> undef, i1 false, <2 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %2 = call <2 x i8> @llvm.abs.v2i8(<2 x i8> undef, i1 false)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %3 = call <4 x i8> @llvm.vp.abs.v4i8(<4 x i8> undef, i1 false, <4 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %4 = call <4 x i8> @llvm.abs.v4i8(<4 x i8> undef, i1 false)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %5 = call <8 x i8> @llvm.vp.abs.v8i8(<8 x i8> undef, i1 false, <8 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %6 = call <8 x i8> @llvm.abs.v8i8(<8 x i8> undef, i1 false)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %7 = call <16 x i8> @llvm.vp.abs.v16i8(<16 x i8> undef, i1 false, <16 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %8 = call <16 x i8> @llvm.abs.v16i8(<16 x i8> undef, i1 false)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %9 = call <2 x i64> @llvm.vp.abs.v2i64(<2 x i64> undef, i1 false, <2 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %10 = call <2 x i64> @llvm.abs.v2i64(<2 x i64> undef, i1 false)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %11 = call <4 x i64> @llvm.vp.abs.v4i64(<4 x i64> undef, i1 false, <4 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %12 = call <4 x i64> @llvm.abs.v4i64(<4 x i64> undef, i1 false)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %13 = call <8 x i64> @llvm.vp.abs.v8i64(<8 x i64> undef, i1 false, <8 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %14 = call <8 x i64> @llvm.abs.v8i64(<8 x i64> undef, i1 false)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %15 = call <16 x i64> @llvm.vp.abs.v16i64(<16 x i64> undef, i1 false, <16 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %16 = call <16 x i64> @llvm.abs.v16i64(<16 x i64> undef, i1 false)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %17 = call <vscale x 2 x i8> @llvm.vp.abs.nxv2i8(<vscale x 2 x i8> undef, i1 false, <vscale x 2 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %18 = call <vscale x 2 x i8> @llvm.abs.nxv2i8(<vscale x 2 x i8> undef, i1 false)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %19 = call <vscale x 4 x i8> @llvm.vp.abs.nxv4i8(<vscale x 4 x i8> undef, i1 false, <vscale x 4 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %20 = call <vscale x 4 x i8> @llvm.abs.nxv4i8(<vscale x 4 x i8> undef, i1 false)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %21 = call <vscale x 8 x i8> @llvm.vp.abs.nxv8i8(<vscale x 8 x i8> undef, i1 false, <vscale x 8 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %22 = call <vscale x 8 x i8> @llvm.abs.nxv8i8(<vscale x 8 x i8> undef, i1 false)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %23 = call <vscale x 16 x i8> @llvm.vp.abs.nxv16i8(<vscale x 16 x i8> undef, i1 false, <vscale x 16 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %24 = call <vscale x 16 x i8> @llvm.abs.nxv16i8(<vscale x 16 x i8> undef, i1 false)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %25 = call <vscale x 2 x i64> @llvm.vp.abs.nxv2i64(<vscale x 2 x i64> undef, i1 false, <vscale x 2 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %26 = call <vscale x 2 x i64> @llvm.abs.nxv2i64(<vscale x 2 x i64> undef, i1 false)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %27 = call <vscale x 4 x i64> @llvm.vp.abs.nxv4i64(<vscale x 4 x i64> undef, i1 false, <vscale x 4 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %28 = call <vscale x 4 x i64> @llvm.abs.nxv4i64(<vscale x 4 x i64> undef, i1 false)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %29 = call <vscale x 8 x i64> @llvm.vp.abs.nxv8i64(<vscale x 8 x i64> undef, i1 false, <vscale x 8 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %30 = call <vscale x 8 x i64> @llvm.abs.nxv8i64(<vscale x 8 x i64> undef, i1 false)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %31 = call <vscale x 16 x i64> @llvm.vp.abs.nxv16i64(<vscale x 16 x i64> undef, i1 false, <vscale x 16 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %32 = call <vscale x 16 x i64> @llvm.abs.nxv16i64(<vscale x 16 x i64> undef, i1 false)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
   call <2 x i8> @llvm.vp.abs.v2i8(<2 x i8> undef, i1 0, <2 x i1> undef, i32 undef)
   call <2 x i8> @llvm.abs.v2i8(<2 x i8> undef, i1 0)
@@ -1276,41 +871,6 @@ define void @load() {
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %t31 = call <vscale x 16 x i64> @llvm.vp.load.nxv16i64.p0(ptr undef, <vscale x 16 x i1> undef, i32 undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %t32 = load <vscale x 16 x i64>, ptr undef, align 128
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
-;
-; TYPEBASED-LABEL: 'load'
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %t0 = call <2 x i8> @llvm.vp.load.v2i8.p0(ptr undef, <2 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %t1 = load <2 x i8>, ptr undef, align 2
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %t2 = call <4 x i8> @llvm.vp.load.v4i8.p0(ptr undef, <4 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %t3 = load <4 x i8>, ptr undef, align 4
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %t4 = call <8 x i8> @llvm.vp.load.v8i8.p0(ptr undef, <8 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %t5 = load <8 x i8>, ptr undef, align 8
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %t6 = call <16 x i8> @llvm.vp.load.v16i8.p0(ptr undef, <16 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %t7 = load <16 x i8>, ptr undef, align 16
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %t8 = call <2 x i64> @llvm.vp.load.v2i64.p0(ptr undef, <2 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %t9 = load <2 x i64>, ptr undef, align 16
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %t10 = call <4 x i64> @llvm.vp.load.v4i64.p0(ptr undef, <4 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %t12 = load <4 x i64>, ptr undef, align 32
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %t13 = call <8 x i64> @llvm.vp.load.v8i64.p0(ptr undef, <8 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %t14 = load <8 x i64>, ptr undef, align 64
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %t15 = call <16 x i64> @llvm.vp.load.v16i64.p0(ptr undef, <16 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %t16 = load <16 x i64>, ptr undef, align 128
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %t17 = call <vscale x 2 x i8> @llvm.vp.load.nxv2i8.p0(ptr undef, <vscale x 2 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %t18 = load <vscale x 2 x i8>, ptr undef, align 2
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %t19 = call <vscale x 4 x i8> @llvm.vp.load.nxv4i8.p0(ptr undef, <vscale x 4 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %t20 = load <vscale x 4 x i8>, ptr undef, align 4
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %t21 = call <vscale x 8 x i8> @llvm.vp.load.nxv8i8.p0(ptr undef, <vscale x 8 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %t22 = load <vscale x 8 x i8>, ptr undef, align 8
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %t23 = call <vscale x 16 x i8> @llvm.vp.load.nxv16i8.p0(ptr undef, <vscale x 16 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %t24 = load <vscale x 16 x i8>, ptr undef, align 16
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %t25 = call <vscale x 2 x i64> @llvm.vp.load.nxv2i64.p0(ptr undef, <vscale x 2 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %t26 = load <vscale x 2 x i64>, ptr undef, align 16
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %t27 = call <vscale x 4 x i64> @llvm.vp.load.nxv4i64.p0(ptr undef, <vscale x 4 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %t28 = load <vscale x 4 x i64>, ptr undef, align 32
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %t29 = call <vscale x 8 x i64> @llvm.vp.load.nxv8i64.p0(ptr undef, <vscale x 8 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %t30 = load <vscale x 8 x i64>, ptr undef, align 64
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %t31 = call <vscale x 16 x i64> @llvm.vp.load.nxv16i64.p0(ptr undef, <vscale x 16 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %t32 = load <vscale x 16 x i64>, ptr undef, align 128
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
   %t0 = call <2 x i8> @llvm.vp.load.v2i8(ptr undef, <2 x i1> undef, i32 undef)
   %t1 = load <2 x i8>, ptr undef
@@ -1382,41 +942,6 @@ define void @store() {
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: call void @llvm.vp.store.nxv16i64.p0(<vscale x 16 x i64> undef, ptr undef, <vscale x 16 x i1> undef, i32 undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: store <vscale x 16 x i64> undef, ptr undef, align 128
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
-;
-; TYPEBASED-LABEL: 'store'
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.vp.store.v2i8.p0(<2 x i8> undef, ptr undef, <2 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <2 x i8> undef, ptr undef, align 2
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.vp.store.v4i8.p0(<4 x i8> undef, ptr undef, <4 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <4 x i8> undef, ptr undef, align 4
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.vp.store.v8i8.p0(<8 x i8> undef, ptr undef, <8 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <8 x i8> undef, ptr undef, align 8
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.vp.store.v16i8.p0(<16 x i8> undef, ptr undef, <16 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <16 x i8> undef, ptr undef, align 16
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.vp.store.v2i64.p0(<2 x i64> undef, ptr undef, <2 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <2 x i64> undef, ptr undef, align 16
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: call void @llvm.vp.store.v4i64.p0(<4 x i64> undef, ptr undef, <4 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: store <4 x i64> undef, ptr undef, align 32
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: call void @llvm.vp.store.v8i64.p0(<8 x i64> undef, ptr undef, <8 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: store <8 x i64> undef, ptr undef, align 64
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: call void @llvm.vp.store.v16i64.p0(<16 x i64> undef, ptr undef, <16 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: store <16 x i64> undef, ptr undef, align 128
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.vp.store.nxv2i8.p0(<vscale x 2 x i8> undef, ptr undef, <vscale x 2 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <vscale x 2 x i8> undef, ptr undef, align 2
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.vp.store.nxv4i8.p0(<vscale x 4 x i8> undef, ptr undef, <vscale x 4 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <vscale x 4 x i8> undef, ptr undef, align 4
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.vp.store.nxv8i8.p0(<vscale x 8 x i8> undef, ptr undef, <vscale x 8 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <vscale x 8 x i8> undef, ptr undef, align 8
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: call void @llvm.vp.store.nxv16i8.p0(<vscale x 16 x i8> undef, ptr undef, <vscale x 16 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: store <vscale x 16 x i8> undef, ptr undef, align 16
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: call void @llvm.vp.store.nxv2i64.p0(<vscale x 2 x i64> undef, ptr undef, <vscale x 2 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: store <vscale x 2 x i64> undef, ptr undef, align 16
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: call void @llvm.vp.store.nxv4i64.p0(<vscale x 4 x i64> undef, ptr undef, <vscale x 4 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: store <vscale x 4 x i64> undef, ptr undef, align 32
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: call void @llvm.vp.store.nxv8i64.p0(<vscale x 8 x i64> undef, ptr undef, <vscale x 8 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: store <vscale x 8 x i64> undef, ptr undef, align 64
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: call void @llvm.vp.store.nxv16i64.p0(<vscale x 16 x i64> undef, ptr undef, <vscale x 16 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: store <vscale x 16 x i64> undef, ptr undef, align 128
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
   call void @llvm.vp.store.v2i8(<2 x i8> undef, ptr undef, <2 x i1> undef, i32 undef)
   store <2 x i8> undef, ptr undef
@@ -1454,58 +979,58 @@ define void @store() {
 }
 
 define void @strided_load() {
-; CHECK-LABEL: 'strided_load'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %ti1_2 = call <2 x i1> @llvm.experimental.vp.strided.load.v2i1.p0.i64(ptr undef, i64 undef, <2 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %ti1_4 = call <4 x i1> @llvm.experimental.vp.strided.load.v4i1.p0.i64(ptr undef, i64 undef, <4 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %ti1_8 = call <8 x i1> @llvm.experimental.vp.strided.load.v8i1.p0.i64(ptr undef, i64 undef, <8 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 65 for instruction: %ti1_16 = call <16 x i1> @llvm.experimental.vp.strided.load.v16i1.p0.i64(ptr undef, i64 undef, <16 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %t0 = call <2 x i8> @llvm.experimental.vp.strided.load.v2i8.p0.i64(ptr undef, i64 undef, <2 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %t2 = call <4 x i8> @llvm.experimental.vp.strided.load.v4i8.p0.i64(ptr undef, i64 undef, <4 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %t4 = call <8 x i8> @llvm.experimental.vp.strided.load.v8i8.p0.i64(ptr undef, i64 undef, <8 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %t6 = call <16 x i8> @llvm.experimental.vp.strided.load.v16i8.p0.i64(ptr undef, i64 undef, <16 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %t8.a = call <2 x i64> @llvm.experimental.vp.strided.load.v2i64.p0.i64(ptr align 8 undef, i64 undef, <2 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %t10.a = call <4 x i64> @llvm.experimental.vp.strided.load.v4i64.p0.i64(ptr align 8 undef, i64 undef, <4 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %t13.a = call <8 x i64> @llvm.experimental.vp.strided.load.v8i64.p0.i64(ptr align 8 undef, i64 undef, <8 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %t15.a = call <16 x i64> @llvm.experimental.vp.strided.load.v16i64.p0.i64(ptr align 8 undef, i64 undef, <16 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %t8 = call <2 x i64> @llvm.experimental.vp.strided.load.v2i64.p0.i64(ptr undef, i64 undef, <2 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %t10 = call <4 x i64> @llvm.experimental.vp.strided.load.v4i64.p0.i64(ptr undef, i64 undef, <4 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %t13 = call <8 x i64> @llvm.experimental.vp.strided.load.v8i64.p0.i64(ptr undef, i64 undef, <8 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %t15 = call <16 x i64> @llvm.experimental.vp.strided.load.v16i64.p0.i64(ptr undef, i64 undef, <16 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %t17 = call <vscale x 2 x i8> @llvm.experimental.vp.strided.load.nxv2i8.p0.i64(ptr undef, i64 undef, <vscale x 2 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %t19 = call <vscale x 4 x i8> @llvm.experimental.vp.strided.load.nxv4i8.p0.i64(ptr undef, i64 undef, <vscale x 4 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %t21 = call <vscale x 8 x i8> @llvm.experimental.vp.strided.load.nxv8i8.p0.i64(ptr undef, i64 undef, <vscale x 8 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %t23 = call <vscale x 16 x i8> @llvm.experimental.vp.strided.load.nxv16i8.p0.i64(ptr undef, i64 undef, <vscale x 16 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %t25 = call <vscale x 2 x i64> @llvm.experimental.vp.strided.load.nxv2i64.p0.i64(ptr undef, i64 undef, <vscale x 2 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %t27 = call <vscale x 4 x i64> @llvm.experimental.vp.strided.load.nxv4i64.p0.i64(ptr undef, i64 undef, <vscale x 4 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %t29 = call <vscale x 8 x i64> @llvm.experimental.vp.strided.load.nxv8i64.p0.i64(ptr undef, i64 undef, <vscale x 8 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %t31 = call <vscale x 16 x i64> @llvm.experimental.vp.strided.load.nxv16i64.p0.i64(ptr undef, i64 undef, <vscale x 16 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; ARGBASED-LABEL: 'strided_load'
+; ARGBASED-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %ti1_2 = call <2 x i1> @llvm.experimental.vp.strided.load.v2i1.p0.i64(ptr undef, i64 undef, <2 x i1> undef, i32 undef)
+; ARGBASED-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %ti1_4 = call <4 x i1> @llvm.experimental.vp.strided.load.v4i1.p0.i64(ptr undef, i64 undef, <4 x i1> undef, i32 undef)
+; ARGBASED-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %ti1_8 = call <8 x i1> @llvm.experimental.vp.strided.load.v8i1.p0.i64(ptr undef, i64 undef, <8 x i1> undef, i32 undef)
+; ARGBASED-NEXT:  Cost Model: Found an estimated cost of 65 for instruction: %ti1_16 = call <16 x i1> @llvm.experimental.vp.strided.load.v16i1.p0.i64(ptr undef, i64 undef, <16 x i1> undef, i32 undef)
+; ARGBASED-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %t0 = call <2 x i8> @llvm.experimental.vp.strided.load.v2i8.p0.i64(ptr undef, i64 undef, <2 x i1> undef, i32 undef)
+; ARGBASED-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %t2 = call <4 x i8> @llvm.experimental.vp.strided.load.v4i8.p0.i64(ptr undef, i64 undef, <4 x i1> undef, i32 undef)
+; ARGBASED-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %t4 = call <8 x i8> @llvm.experimental.vp.strided.load.v8i8.p0.i64(ptr undef, i64 undef, <8 x i1> undef, i32 undef)
+; ARGBASED-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %t6 = call <16 x i8> @llvm.experimental.vp.strided.load.v16i8.p0.i64(ptr undef, i64 undef, <16 x i1> undef, i32 undef)
+; ARGBASED-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %t8.a = call <2 x i64> @llvm.experimental.vp.strided.load.v2i64.p0.i64(ptr align 8 undef, i64 undef, <2 x i1> undef, i32 undef)
+; ARGBASED-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %t10.a = call <4 x i64> @llvm.experimental.vp.strided.load.v4i64.p0.i64(ptr align 8 undef, i64 undef, <4 x i1> undef, i32 undef)
+; ARGBASED-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %t13.a = call <8 x i64> @llvm.experimental.vp.strided.load.v8i64.p0.i64(ptr align 8 undef, i64 undef, <8 x i1> undef, i32 undef)
+; ARGBASED-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %t15.a = call <16 x i64> @llvm.experimental.vp.strided.load.v16i64.p0.i64(ptr align 8 undef, i64 undef, <16 x i1> undef, i32 undef)
+; ARGBASED-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %t8 = call <2 x i64> @llvm.experimental.vp.strided.load.v2i64.p0.i64(ptr undef, i64 undef, <2 x i1> undef, i32 undef)
+; ARGBASED-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %t10 = call <4 x i64> @llvm.experimental.vp.strided.load.v4i64.p0.i64(ptr undef, i64 undef, <4 x i1> undef, i32 undef)
+; ARGBASED-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %t13 = call <8 x i64> @llvm.experimental.vp.strided.load.v8i64.p0.i64(ptr undef, i64 undef, <8 x i1> undef, i32 undef)
+; ARGBASED-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %t15 = call <16 x i64> @llvm.experimental.vp.strided.load.v16i64.p0.i64(ptr undef, i64 undef, <16 x i1> undef, i32 undef)
+; ARGBASED-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %t17 = call <vscale x 2 x i8> @llvm.experimental.vp.strided.load.nxv2i8.p0.i64(ptr undef, i64 undef, <vscale x 2 x i1> undef, i32 undef)
+; ARGBASED-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %t19 = call <vscale x 4 x i8> @llvm.experimental.vp.strided.load.nxv4i8.p0.i64(ptr undef, i64 undef, <vscale x 4 x i1> undef, i32 undef)
+; ARGBASED-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %t21 = call <vscale x 8 x i8> @llvm.experimental.vp.strided.load.nxv8i8.p0.i64(ptr undef, i64 undef, <vscale x 8 x i1> undef, i32 undef)
+; ARGBASED-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %t23 = call <vscale x 16 x i8> @llvm.experimental.vp.strided.load.nxv16i8.p0.i64(ptr undef, i64 undef, <vscale x 16 x i1> undef, i32 undef)
+; ARGBASED-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %t25 = call <vscale x 2 x i64> @llvm.experimental.vp.strided.load.nxv2i64.p0.i64(ptr undef, i64 undef, <vscale x 2 x i1> undef, i32 undef)
+; ARGBASED-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %t27 = call <vscale x 4 x i64> @llvm.experimental.vp.strided.load.nxv4i64.p0.i64(ptr undef, i64 undef, <vscale x 4 x i1> undef, i32 undef)
+; ARGBASED-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %t29 = call <vscale x 8 x i64> @llvm.experimental.vp.strided.load.nxv8i64.p0.i64(ptr undef, i64 undef, <vscale x 8 x i1> undef, i32 undef)
+; ARGBASED-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %t31 = call <vscale x 16 x i64> @llvm.experimental.vp.strided.load.nxv16i64.p0.i64(ptr undef, i64 undef, <vscale x 16 x i1> undef, i32 undef)
+; ARGBASED-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; TYPEBASED-LABEL: 'strided_load'
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %ti1_2 = call <2 x i1> @llvm.experimental.vp.strided.load.v2i1.p0.i64(ptr undef, i64 undef, <2 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: %ti1_4 = call <4 x i1> @llvm.experimental.vp.strided.load.v4i1.p0.i64(ptr undef, i64 undef, <4 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 49 for instruction: %ti1_8 = call <8 x i1> @llvm.experimental.vp.strided.load.v8i1.p0.i64(ptr undef, i64 undef, <8 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 97 for instruction: %ti1_16 = call <16 x i1> @llvm.experimental.vp.strided.load.v16i1.p0.i64(ptr undef, i64 undef, <16 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %t0 = call <2 x i8> @llvm.experimental.vp.strided.load.v2i8.p0.i64(ptr undef, i64 undef, <2 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 23 for instruction: %t2 = call <4 x i8> @llvm.experimental.vp.strided.load.v4i8.p0.i64(ptr undef, i64 undef, <4 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 47 for instruction: %t4 = call <8 x i8> @llvm.experimental.vp.strided.load.v8i8.p0.i64(ptr undef, i64 undef, <8 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 95 for instruction: %t6 = call <16 x i8> @llvm.experimental.vp.strided.load.v16i8.p0.i64(ptr undef, i64 undef, <16 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %t8.a = call <2 x i64> @llvm.experimental.vp.strided.load.v2i64.p0.i64(ptr align 8 undef, i64 undef, <2 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 26 for instruction: %t10.a = call <4 x i64> @llvm.experimental.vp.strided.load.v4i64.p0.i64(ptr align 8 undef, i64 undef, <4 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 54 for instruction: %t13.a = call <8 x i64> @llvm.experimental.vp.strided.load.v8i64.p0.i64(ptr align 8 undef, i64 undef, <8 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 110 for instruction: %t15.a = call <16 x i64> @llvm.experimental.vp.strided.load.v16i64.p0.i64(ptr align 8 undef, i64 undef, <16 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %t8 = call <2 x i64> @llvm.experimental.vp.strided.load.v2i64.p0.i64(ptr undef, i64 undef, <2 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 26 for instruction: %t10 = call <4 x i64> @llvm.experimental.vp.strided.load.v4i64.p0.i64(ptr undef, i64 undef, <4 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 54 for instruction: %t13 = call <8 x i64> @llvm.experimental.vp.strided.load.v8i64.p0.i64(ptr undef, i64 undef, <8 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 110 for instruction: %t15 = call <16 x i64> @llvm.experimental.vp.strided.load.v16i64.p0.i64(ptr undef, i64 undef, <16 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Invalid cost for instruction: %t17 = call <vscale x 2 x i8> @llvm.experimental.vp.strided.load.nxv2i8.p0.i64(ptr undef, i64 undef, <vscale x 2 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Invalid cost for instruction: %t19 = call <vscale x 4 x i8> @llvm.experimental.vp.strided.load.nxv4i8.p0.i64(ptr undef, i64 undef, <vscale x 4 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Invalid cost for instruction: %t21 = call <vscale x 8 x i8> @llvm.experimental.vp.strided.load.nxv8i8.p0.i64(ptr undef, i64 undef, <vscale x 8 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Invalid cost for instruction: %t23 = call <vscale x 16 x i8> @llvm.experimental.vp.strided.load.nxv16i8.p0.i64(ptr undef, i64 undef, <vscale x 16 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Invalid cost for instruction: %t25 = call <vscale x 2 x i64> @llvm.experimental.vp.strided.load.nxv2i64.p0.i64(ptr undef, i64 undef, <vscale x 2 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Invalid cost for instruction: %t27 = call <vscale x 4 x i64> @llvm.experimental.vp.strided.load.nxv4i64.p0.i64(ptr undef, i64 undef, <vscale x 4 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Invalid cost for instruction: %t29 = call <vscale x 8 x i64> @llvm.experimental.vp.strided.load.nxv8i64.p0.i64(ptr undef, i64 undef, <vscale x 8 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Invalid cost for instruction: %t31 = call <vscale x 16 x i64> @llvm.experimental.vp.strided.load.nxv16i64.p0.i64(ptr undef, i64 undef, <vscale x 16 x i1> undef, i32 undef)
+; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %ti1_2 = call <2 x i1> @llvm.experimental.vp.strided.load.v2i1.p0.i64(ptr undef, i64 undef, <2 x i1> undef, i32 undef)
+; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %ti1_4 = call <4 x i1> @llvm.experimental.vp.strided.load.v4i1.p0.i64(ptr undef, i64 undef, <4 x i1> undef, i32 undef)
+; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: %ti1_8 = call <8 x i1> @llvm.experimental.vp.strided.load.v8i1.p0.i64(ptr undef, i64 undef, <8 x i1> undef, i32 undef)
+; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 128 for instruction: %ti1_16 = call <16 x i1> @llvm.experimental.vp.strided.load.v16i1.p0.i64(ptr undef, i64 undef, <16 x i1> undef, i32 undef)
+; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %t0 = call <2 x i8> @llvm.experimental.vp.strided.load.v2i8.p0.i64(ptr undef, i64 undef, <2 x i1> undef, i32 undef)
+; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %t2 = call <4 x i8> @llvm.experimental.vp.strided.load.v4i8.p0.i64(ptr undef, i64 undef, <4 x i1> undef, i32 undef)
+; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %t4 = call <8 x i8> @llvm.experimental.vp.strided.load.v8i8.p0.i64(ptr undef, i64 undef, <8 x i1> undef, i32 undef)
+; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %t6 = call <16 x i8> @llvm.experimental.vp.strided.load.v16i8.p0.i64(ptr undef, i64 undef, <16 x i1> undef, i32 undef)
+; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %t8.a = call <2 x i64> @llvm.experimental.vp.strided.load.v2i64.p0.i64(ptr align 8 undef, i64 undef, <2 x i1> undef, i32 undef)
+; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %t10.a = call <4 x i64> @llvm.experimental.vp.strided.load.v4i64.p0.i64(ptr align 8 undef, i64 undef, <4 x i1> undef, i32 undef)
+; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %t13.a = call <8 x i64> @llvm.experimental.vp.strided.load.v8i64.p0.i64(ptr align 8 undef, i64 undef, <8 x i1> undef, i32 undef)
+; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %t15.a = call <16 x i64> @llvm.experimental.vp.strided.load.v16i64.p0.i64(ptr align 8 undef, i64 undef, <16 x i1> undef, i32 undef)
+; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %t8 = call <2 x i64> @llvm.experimental.vp.strided.load.v2i64.p0.i64(ptr undef, i64 undef, <2 x i1> undef, i32 undef)
+; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %t10 = call <4 x i64> @llvm.experimental.vp.strided.load.v4i64.p0.i64(ptr undef, i64 undef, <4 x i1> undef, i32 undef)
+; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %t13 = call <8 x i64> @llvm.experimental.vp.strided.load.v8i64.p0.i64(ptr undef, i64 undef, <8 x i1> undef, i32 undef)
+; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %t15 = call <16 x i64> @llvm.experimental.vp.strided.load.v16i64.p0.i64(ptr undef, i64 undef, <16 x i1> undef, i32 undef)
+; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %t17 = call <vscale x 2 x i8> @llvm.experimental.vp.strided.load.nxv2i8.p0.i64(ptr undef, i64 undef, <vscale x 2 x i1> undef, i32 undef)
+; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %t19 = call <vscale x 4 x i8> @llvm.experimental.vp.strided.load.nxv4i8.p0.i64(ptr undef, i64 undef, <vscale x 4 x i1> undef, i32 undef)
+; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %t21 = call <vscale x 8 x i8> @llvm.experimental.vp.strided.load.nxv8i8.p0.i64(ptr undef, i64 undef, <vscale x 8 x i1> undef, i32 undef)
+; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %t23 = call <vscale x 16 x i8> @llvm.experimental.vp.strided.load.nxv16i8.p0.i64(ptr undef, i64 undef, <vscale x 16 x i1> undef, i32 undef)
+; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %t25 = call <vscale x 2 x i64> @llvm.experimental.vp.strided.load.nxv2i64.p0.i64(ptr undef, i64 undef, <vscale x 2 x i1> undef, i32 undef)
+; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %t27 = call <vscale x 4 x i64> @llvm.experimental.vp.strided.load.nxv4i64.p0.i64(ptr undef, i64 undef, <vscale x 4 x i1> undef, i32 undef)
+; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %t29 = call <vscale x 8 x i64> @llvm.experimental.vp.strided.load.nxv8i64.p0.i64(ptr undef, i64 undef, <vscale x 8 x i1> undef, i32 undef)
+; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %t31 = call <vscale x 16 x i64> @llvm.experimental.vp.strided.load.nxv16i64.p0.i64(ptr undef, i64 undef, <vscale x 16 x i1> undef, i32 undef)
 ; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
   %ti1_2 = call <2 x i1> @llvm.experimental.vp.strided.load.v2i1.i64(ptr undef, i64 undef, <2 x i1> undef, i32 undef)
@@ -1558,29 +1083,6 @@ define void @strided_store() {
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: call void @llvm.experimental.vp.strided.store.nxv8i64.p0.i64(<vscale x 8 x i64> undef, ptr undef, i64 undef, <vscale x 8 x i1> undef, i32 undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: call void @llvm.experimental.vp.strided.store.nxv16i64.p0.i64(<vscale x 16 x i64> undef, ptr undef, i64 undef, <vscale x 16 x i1> undef, i32 undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
-;
-; TYPEBASED-LABEL: 'strided_store'
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: call void @llvm.experimental.vp.strided.store.v2i8.p0.i64(<2 x i8> undef, ptr undef, i64 undef, <2 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 26 for instruction: call void @llvm.experimental.vp.strided.store.v4i8.p0.i64(<4 x i8> undef, ptr undef, i64 undef, <4 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 54 for instruction: call void @llvm.experimental.vp.strided.store.v8i8.p0.i64(<8 x i8> undef, ptr undef, i64 undef, <8 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 110 for instruction: call void @llvm.experimental.vp.strided.store.v16i8.p0.i64(<16 x i8> undef, ptr undef, i64 undef, <16 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: call void @llvm.experimental.vp.strided.store.v2i64.p0.i64(<2 x i64> undef, ptr undef, i64 undef, <2 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 26 for instruction: call void @llvm.experimental.vp.strided.store.v4i64.p0.i64(<4 x i64> undef, ptr undef, i64 undef, <4 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 54 for instruction: call void @llvm.experimental.vp.strided.store.v8i64.p0.i64(<8 x i64> undef, ptr undef, i64 undef, <8 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 110 for instruction: call void @llvm.experimental.vp.strided.store.v16i64.p0.i64(<16 x i64> undef, ptr undef, i64 undef, <16 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: call void @llvm.experimental.vp.strided.store.v2i64.p0.i64(<2 x i64> undef, ptr align 8 undef, i64 undef, <2 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 26 for instruction: call void @llvm.experimental.vp.strided.store.v4i64.p0.i64(<4 x i64> undef, ptr align 8 undef, i64 undef, <4 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 54 for instruction: call void @llvm.experimental.vp.strided.store.v8i64.p0.i64(<8 x i64> undef, ptr align 8 undef, i64 undef, <8 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 110 for instruction: call void @llvm.experimental.vp.strided.store.v16i64.p0.i64(<16 x i64> undef, ptr align 8 undef, i64 undef, <16 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Invalid cost for instruction: call void @llvm.experimental.vp.strided.store.nxv2i8.p0.i64(<vscale x 2 x i8> undef, ptr undef, i64 undef, <vscale x 2 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Invalid cost for instruction: call void @llvm.experimental.vp.strided.store.nxv4i8.p0.i64(<vscale x 4 x i8> undef, ptr undef, i64 undef, <vscale x 4 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Invalid cost for instruction: call void @llvm.experimental.vp.strided.store.nxv8i8.p0.i64(<vscale x 8 x i8> undef, ptr undef, i64 undef, <vscale x 8 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Invalid cost for instruction: call void @llvm.experimental.vp.strided.store.nxv16i8.p0.i64(<vscale x 16 x i8> undef, ptr undef, i64 undef, <vscale x 16 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Invalid cost for instruction: call void @llvm.experimental.vp.strided.store.nxv2i64.p0.i64(<vscale x 2 x i64> undef, ptr undef, i64 undef, <vscale x 2 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Invalid cost for instruction: call void @llvm.experimental.vp.strided.store.nxv4i64.p0.i64(<vscale x 4 x i64> undef, ptr undef, i64 undef, <vscale x 4 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Invalid cost for instruction: call void @llvm.experimental.vp.strided.store.nxv8i64.p0.i64(<vscale x 8 x i64> undef, ptr undef, i64 undef, <vscale x 8 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Invalid cost for instruction: call void @llvm.experimental.vp.strided.store.nxv16i64.p0.i64(<vscale x 16 x i64> undef, ptr undef, i64 undef, <vscale x 16 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
   call void @llvm.experimental.vp.strided.store.v2i8.i64(<2 x i8> undef, ptr undef, i64 undef, <2 x i1> undef, i32 undef)
   call void @llvm.experimental.vp.strided.store.v4i8.i64(<4 x i8> undef, ptr undef, i64 undef, <4 x i1> undef, i32 undef)
@@ -1642,41 +1144,6 @@ define void @reduce_add() {
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %31 = call i64 @llvm.vp.reduce.add.nxv16i64(i64 undef, <vscale x 16 x i64> undef, <vscale x 16 x i1> undef, i32 undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %32 = call i64 @llvm.vector.reduce.add.nxv16i64(<vscale x 16 x i64> undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
-;
-; TYPEBASED-LABEL: 'reduce_add'
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %1 = call i8 @llvm.vp.reduce.add.v2i8(i8 undef, <2 x i8> undef, <2 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %2 = call i8 @llvm.vector.reduce.add.v2i8(<2 x i8> undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %3 = call i8 @llvm.vp.reduce.add.v4i8(i8 undef, <4 x i8> undef, <4 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %4 = call i8 @llvm.vector.reduce.add.v4i8(<4 x i8> undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %5 = call i8 @llvm.vp.reduce.add.v8i8(i8 undef, <8 x i8> undef, <8 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %6 = call i8 @llvm.vector.reduce.add.v8i8(<8 x i8> undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %7 = call i8 @llvm.vp.reduce.add.v16i8(i8 undef, <16 x i8> undef, <16 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %8 = call i8 @llvm.vector.reduce.add.v16i8(<16 x i8> undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %9 = call i64 @llvm.vp.reduce.add.v2i64(i64 undef, <2 x i64> undef, <2 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %10 = call i64 @llvm.vector.reduce.add.v2i64(<2 x i64> undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %11 = call i64 @llvm.vp.reduce.add.v4i64(i64 undef, <4 x i64> undef, <4 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %12 = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %13 = call i64 @llvm.vp.reduce.add.v8i64(i64 undef, <8 x i64> undef, <8 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %14 = call i64 @llvm.vector.reduce.add.v8i64(<8 x i64> undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %15 = call i64 @llvm.vp.reduce.add.v16i64(i64 undef, <16 x i64> undef, <16 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %16 = call i64 @llvm.vector.reduce.add.v16i64(<16 x i64> undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %17 = call i8 @llvm.vp.reduce.add.nxv8i8(i8 undef, <vscale x 8 x i8> undef, <vscale x 8 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %18 = call i8 @llvm.vector.reduce.add.nxv2i8(<vscale x 2 x i8> undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %19 = call i8 @llvm.vp.reduce.add.nxv4i8(i8 undef, <vscale x 4 x i8> undef, <vscale x 4 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %20 = call i8 @llvm.vector.reduce.add.nxv4i8(<vscale x 4 x i8> undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %21 = call i8 @llvm.vp.reduce.add.nxv8i8(i8 undef, <vscale x 8 x i8> undef, <vscale x 8 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %22 = call i8 @llvm.vector.reduce.add.nxv8i8(<vscale x 8 x i8> undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %23 = call i8 @llvm.vp.reduce.add.nxv16i8(i8 undef, <vscale x 16 x i8> undef, <vscale x 16 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %24 = call i8 @llvm.vector.reduce.add.nxv16i8(<vscale x 16 x i8> undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %25 = call i64 @llvm.vp.reduce.add.nxv2i64(i64 undef, <vscale x 2 x i64> undef, <vscale x 2 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %26 = call i64 @llvm.vector.reduce.add.nxv2i64(<vscale x 2 x i64> undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %27 = call i64 @llvm.vp.reduce.add.nxv4i64(i64 undef, <vscale x 4 x i64> undef, <vscale x 4 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %28 = call i64 @llvm.vector.reduce.add.nxv4i64(<vscale x 4 x i64> undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %29 = call i64 @llvm.vp.reduce.add.nxv8i64(i64 undef, <vscale x 8 x i64> undef, <vscale x 8 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %30 = call i64 @llvm.vector.reduce.add.nxv8i64(<vscale x 8 x i64> undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %31 = call i64 @llvm.vp.reduce.add.nxv16i64(i64 undef, <vscale x 16 x i64> undef, <vscale x 16 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %32 = call i64 @llvm.vector.reduce.add.nxv16i64(<vscale x 16 x i64> undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
   call i8 @llvm.vp.reduce.add.v2i8(i8 undef, <2 x i8> undef, <2 x i1> undef, i32 undef)
   call i8 @llvm.vector.reduce.add.v2i8(<2 x i8> undef)
@@ -1748,41 +1215,6 @@ define void @reduce_fadd() {
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %31 = call double @llvm.vp.reduce.fadd.nxv16f64(double undef, <vscale x 16 x double> undef, <vscale x 16 x i1> undef, i32 undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %32 = call double @llvm.vector.reduce.fadd.nxv16f64(double undef, <vscale x 16 x double> undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
-;
-; TYPEBASED-LABEL: 'reduce_fadd'
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %1 = call float @llvm.vp.reduce.fadd.v2f32(float undef, <2 x float> undef, <2 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %2 = call float @llvm.vector.reduce.fadd.v2f32(float undef, <2 x float> undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %3 = call float @llvm.vp.reduce.fadd.v4f32(float undef, <4 x float> undef, <4 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %4 = call float @llvm.vector.reduce.fadd.v4f32(float undef, <4 x float> undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %5 = call float @llvm.vp.reduce.fadd.v8f32(float undef, <8 x float> undef, <8 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %6 = call float @llvm.vector.reduce.fadd.v8f32(float undef, <8 x float> undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %7 = call float @llvm.vp.reduce.fadd.v16f32(float undef, <16 x float> undef, <16 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %8 = call float @llvm.vector.reduce.fadd.v16f32(float undef, <16 x float> undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %9 = call double @llvm.vp.reduce.fadd.v2f64(double undef, <2 x double> undef, <2 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %10 = call double @llvm.vector.reduce.fadd.v2f64(double undef, <2 x double> undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %11 = call double @llvm.vp.reduce.fadd.v4f64(double undef, <4 x double> undef, <4 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %12 = call double @llvm.vector.reduce.fadd.v4f64(double undef, <4 x double> undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %13 = call double @llvm.vp.reduce.fadd.v8f64(double undef, <8 x double> undef, <8 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %14 = call double @llvm.vector.reduce.fadd.v8f64(double undef, <8 x double> undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %15 = call double @llvm.vp.reduce.fadd.v16f64(double undef, <16 x double> undef, <16 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %16 = call double @llvm.vector.reduce.fadd.v16f64(double undef, <16 x double> undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %17 = call float @llvm.vp.reduce.fadd.nxv2f32(float undef, <vscale x 2 x float> undef, <vscale x 2 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %18 = call float @llvm.vector.reduce.fadd.nxv2f32(float undef, <vscale x 2 x float> undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %19 = call float @llvm.vp.reduce.fadd.nxv4f32(float undef, <vscale x 4 x float> undef, <vscale x 4 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %20 = call float @llvm.vector.reduce.fadd.nxv4f32(float undef, <vscale x 4 x float> undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %21 = call float @llvm.vp.reduce.fadd.nxv8f32(float undef, <vscale x 8 x float> undef, <vscale x 8 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %22 = call float @llvm.vector.reduce.fadd.nxv8f32(float undef, <vscale x 8 x float> undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %23 = call float @llvm.vp.reduce.fadd.nxv16f32(float undef, <vscale x 16 x float> undef, <vscale x 16 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %24 = call float @llvm.vector.reduce.fadd.nxv16f32(float undef, <vscale x 16 x float> undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %25 = call double @llvm.vp.reduce.fadd.nxv2f64(double undef, <vscale x 2 x double> undef, <vscale x 2 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %26 = call double @llvm.vector.reduce.fadd.nxv2f64(double undef, <vscale x 2 x double> undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %27 = call double @llvm.vp.reduce.fadd.nxv4f64(double undef, <vscale x 4 x double> undef, <vscale x 4 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %28 = call double @llvm.vector.reduce.fadd.nxv4f64(double undef, <vscale x 4 x double> undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %29 = call double @llvm.vp.reduce.fadd.nxv8f64(double undef, <vscale x 8 x double> undef, <vscale x 8 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %30 = call double @llvm.vector.reduce.fadd.nxv8f64(double undef, <vscale x 8 x double> undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %31 = call double @llvm.vp.reduce.fadd.nxv16f64(double undef, <vscale x 16 x double> undef, <vscale x 16 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %32 = call double @llvm.vector.reduce.fadd.nxv16f64(double undef, <vscale x 16 x double> undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
   call float @llvm.vp.reduce.fadd.v2f32(float undef, <2 x float> undef, <2 x i1> undef, i32 undef)
   call float @llvm.vector.reduce.fadd.v2f32(float undef, <2 x float> undef)
@@ -1836,23 +1268,6 @@ define void @reduce_other() {
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %13 = call float @llvm.vp.reduce.fmax.nxv4f32(float undef, <vscale x 4 x float> undef, <vscale x 4 x i1> undef, i32 undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %14 = call float @llvm.vp.reduce.fmaximum.nxv4f32(float undef, <vscale x 4 x float> undef, <vscale x 4 x i1> undef, i32 undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
-;
-; TYPEBASED-LABEL: 'reduce_other'
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %1 = call i32 @llvm.vp.reduce.xor.v4i32(i32 undef, <4 x i32> undef, <4 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %2 = call i32 @llvm.vp.reduce.and.v4i32(i32 undef, <4 x i32> undef, <4 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %3 = call i32 @llvm.vp.reduce.or.v4i32(i32 undef, <4 x i32> undef, <4 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %4 = call i32 @llvm.vp.reduce.mul.v4i32(i32 undef, <4 x i32> undef, <4 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %5 = call i32 @llvm.vp.reduce.smin.v4i32(i32 undef, <4 x i32> undef, <4 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %6 = call i32 @llvm.vp.reduce.smax.v4i32(i32 undef, <4 x i32> undef, <4 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %7 = call i32 @llvm.vp.reduce.umin.v4i32(i32 undef, <4 x i32> undef, <4 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %8 = call i32 @llvm.vp.reduce.umax.v4i32(i32 undef, <4 x i32> undef, <4 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Invalid cost for instruction: %9 = call i32 @llvm.vp.reduce.mul.nxv4i32(i32 undef, <vscale x 4 x i32> undef, <vscale x 4 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Invalid cost for instruction: %10 = call float @llvm.vp.reduce.fmul.nxv4f32(float undef, <vscale x 4 x float> undef, <vscale x 4 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %11 = call float @llvm.vp.reduce.fmin.nxv4f32(float undef, <vscale x 4 x float> undef, <vscale x 4 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %12 = call float @llvm.vp.reduce.fminimum.nxv4f32(float undef, <vscale x 4 x float> undef, <vscale x 4 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %13 = call float @llvm.vp.reduce.fmax.nxv4f32(float undef, <vscale x 4 x float> undef, <vscale x 4 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %14 = call float @llvm.vp.reduce.fmaximum.nxv4f32(float undef, <vscale x 4 x float> undef, <vscale x 4 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
   call i32 @llvm.vp.reduce.xor.v4i32(i32 undef, <4 x i32> undef, <4 x i1> undef, i32 undef)
   call i32 @llvm.vp.reduce.and.v4i32(i32 undef, <4 x i32> undef, <4 x i1> undef, i32 undef)
@@ -1894,26 +1309,6 @@ define void @vp_fadd(){
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %t31 = call <vscale x 16 x double> @llvm.vp.fadd.nxv16f64(<vscale x 16 x double> undef, <vscale x 16 x double> undef, <vscale x 16 x i1> undef, i32 undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %t32 = fadd <vscale x 16 x double> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
-;
-; TYPEBASED-LABEL: 'vp_fadd'
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %t0 = call <2 x float> @llvm.vp.fadd.v2f32(<2 x float> undef, <2 x float> undef, <2 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %t2 = call <4 x float> @llvm.vp.fadd.v4f32(<4 x float> undef, <4 x float> undef, <4 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %t4 = call <8 x float> @llvm.vp.fadd.v8f32(<8 x float> undef, <8 x float> undef, <8 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %t6 = call <16 x float> @llvm.vp.fadd.v16f32(<16 x float> undef, <16 x float> undef, <16 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %t8 = call <2 x double> @llvm.vp.fadd.v2f64(<2 x double> undef, <2 x double> undef, <2 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %t10 = call <4 x double> @llvm.vp.fadd.v4f64(<4 x double> undef, <4 x double> undef, <4 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %t12 = call <8 x double> @llvm.vp.fadd.v8f64(<8 x double> undef, <8 x double> undef, <8 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %t14 = call <16 x double> @llvm.vp.fadd.v16f64(<16 x double> undef, <16 x double> undef, <16 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %t17 = call <vscale x 2 x float> @llvm.vp.fadd.nxv2f32(<vscale x 2 x float> undef, <vscale x 2 x float> undef, <vscale x 2 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %t19 = call <vscale x 4 x float> @llvm.vp.fadd.nxv4f32(<vscale x 4 x float> undef, <vscale x 4 x float> undef, <vscale x 4 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %t21 = call <vscale x 8 x float> @llvm.vp.fadd.nxv8f32(<vscale x 8 x float> undef, <vscale x 8 x float> undef, <vscale x 8 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %t23 = call <vscale x 16 x float> @llvm.vp.fadd.nxv16f32(<vscale x 16 x float> undef, <vscale x 16 x float> undef, <vscale x 16 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %t25 = call <vscale x 2 x double> @llvm.vp.fadd.nxv2f64(<vscale x 2 x double> undef, <vscale x 2 x double> undef, <vscale x 2 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %t27 = call <vscale x 4 x double> @llvm.vp.fadd.nxv4f64(<vscale x 4 x double> undef, <vscale x 4 x double> undef, <vscale x 4 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %t29 = call <vscale x 8 x double> @llvm.vp.fadd.nxv8f64(<vscale x 8 x double> undef, <vscale x 8 x double> undef, <vscale x 8 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %t31 = call <vscale x 16 x double> @llvm.vp.fadd.nxv16f64(<vscale x 16 x double> undef, <vscale x 16 x double> undef, <vscale x 16 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %t32 = fadd <vscale x 16 x double> undef, undef
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
   %t0 = call <2 x float> @llvm.vp.fadd.v2f32(<2 x float> undef, <2 x float> undef, <2 x i1> undef, i32 undef)
   %t2 = call <4 x float> @llvm.vp.fadd.v4f32(<4 x float> undef, <4 x float> undef, <4 x i1> undef, i32 undef)
@@ -1958,25 +1353,6 @@ define void @vp_fsub(){
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %t29 = call <vscale x 8 x double> @llvm.vp.fsub.nxv8f64(<vscale x 8 x double> undef, <vscale x 8 x double> undef, <vscale x 8 x i1> undef, i32 undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %t31 = call <vscale x 16 x double> @llvm.vp.fsub.nxv16f64(<vscale x 16 x double> undef, <vscale x 16 x double> undef, <vscale x 16 x i1> undef, i32 undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
-;
-; TYPEBASED-LABEL: 'vp_fsub'
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %t0 = call <2 x float> @llvm.vp.fsub.v2f32(<2 x float> undef, <2 x float> undef, <2 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %t2 = call <4 x float> @llvm.vp.fsub.v4f32(<4 x float> undef, <4 x float> undef, <4 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %t4 = call <8 x float> @llvm.vp.fsub.v8f32(<8 x float> undef, <8 x float> undef, <8 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %t6 = call <16 x float> @llvm.vp.fsub.v16f32(<16 x float> undef, <16 x float> undef, <16 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %t8 = call <2 x double> @llvm.vp.fsub.v2f64(<2 x double> undef, <2 x double> undef, <2 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %t10 = call <4 x double> @llvm.vp.fsub.v4f64(<4 x double> undef, <4 x double> undef, <4 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %t12 = call <8 x double> @llvm.vp.fsub.v8f64(<8 x double> undef, <8 x double> undef, <8 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %t14 = call <16 x double> @llvm.vp.fsub.v16f64(<16 x double> undef, <16 x double> undef, <16 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %t17 = call <vscale x 2 x float> @llvm.vp.fsub.nxv2f32(<vscale x 2 x float> undef, <vscale x 2 x float> undef, <vscale x 2 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %t19 = call <vscale x 4 x float> @llvm.vp.fsub.nxv4f32(<vscale x 4 x float> undef, <vscale x 4 x float> undef, <vscale x 4 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %t21 = call <vscale x 8 x float> @llvm.vp.fsub.nxv8f32(<vscale x 8 x float> undef, <vscale x 8 x float> undef, <vscale x 8 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %t23 = call <vscale x 16 x float> @llvm.vp.fsub.nxv16f32(<vscale x 16 x float> undef, <vscale x 16 x float> undef, <vscale x 16 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %t25 = call <vscale x 2 x double> @llvm.vp.fsub.nxv2f64(<vscale x 2 x double> undef, <vscale x 2 x double> undef, <vscale x 2 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %t27 = call <vscale x 4 x double> @llvm.vp.fsub.nxv4f64(<vscale x 4 x double> undef, <vscale x 4 x double> undef, <vscale x 4 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %t29 = call <vscale x 8 x double> @llvm.vp.fsub.nxv8f64(<vscale x 8 x double> undef, <vscale x 8 x double> undef, <vscale x 8 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %t31 = call <vscale x 16 x double> @llvm.vp.fsub.nxv16f64(<vscale x 16 x double> undef, <vscale x 16 x double> undef, <vscale x 16 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
   %t0 = call <2 x float> @llvm.vp.fsub.v2f32(<2 x float> undef, <2 x float> undef, <2 x i1> undef, i32 undef)
   %t2 = call <4 x float> @llvm.vp.fsub.v4f32(<4 x float> undef, <4 x float> undef, <4 x i1> undef, i32 undef)
@@ -2020,25 +1396,6 @@ define void @vp_fmul(){
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %t29 = call <vscale x 8 x double> @llvm.vp.fmul.nxv8f64(<vscale x 8 x double> undef, <vscale x 8 x double> undef, <vscale x 8 x i1> undef, i32 undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %t31 = call <vscale x 16 x double> @llvm.vp.fmul.nxv16f64(<vscale x 16 x double> undef, <vscale x 16 x double> undef, <vscale x 16 x i1> undef, i32 undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
-;
-; TYPEBASED-LABEL: 'vp_fmul'
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %t0 = call <2 x float> @llvm.vp.fmul.v2f32(<2 x float> undef, <2 x float> undef, <2 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %t2 = call <4 x float> @llvm.vp.fmul.v4f32(<4 x float> undef, <4 x float> undef, <4 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %t4 = call <8 x float> @llvm.vp.fmul.v8f32(<8 x float> undef, <8 x float> undef, <8 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %t6 = call <16 x float> @llvm.vp.fmul.v16f32(<16 x float> undef, <16 x float> undef, <16 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %t8 = call <2 x double> @llvm.vp.fmul.v2f64(<2 x double> undef, <2 x double> undef, <2 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %t10 = call <4 x double> @llvm.vp.fmul.v4f64(<4 x double> undef, <4 x double> undef, <4 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %t12 = call <8 x double> @llvm.vp.fmul.v8f64(<8 x double> undef, <8 x double> undef, <8 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %t14 = call <16 x double> @llvm.vp.fmul.v16f64(<16 x double> undef, <16 x double> undef, <16 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %t17 = call <vscale x 2 x float> @llvm.vp.fmul.nxv2f32(<vscale x 2 x float> undef, <vscale x 2 x float> undef, <vscale x 2 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %t19 = call <vscale x 4 x float> @llvm.vp.fmul.nxv4f32(<vscale x 4 x float> undef, <vscale x 4 x float> undef, <vscale x 4 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %t21 = call <vscale x 8 x float> @llvm.vp.fmul.nxv8f32(<vscale x 8 x float> undef, <vscale x 8 x float> undef, <vscale x 8 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %t23 = call <vscale x 16 x float> @llvm.vp.fmul.nxv16f32(<vscale x 16 x float> undef, <vscale x 16 x float> undef, <vscale x 16 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %t25 = call <vscale x 2 x double> @llvm.vp.fmul.nxv2f64(<vscale x 2 x double> undef, <vscale x 2 x double> undef, <vscale x 2 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %t27 = call <vscale x 4 x double> @llvm.vp.fmul.nxv4f64(<vscale x 4 x double> undef, <vscale x 4 x double> undef, <vscale x 4 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %t29 = call <vscale x 8 x double> @llvm.vp.fmul.nxv8f64(<vscale x 8 x double> undef, <vscale x 8 x double> undef, <vscale x 8 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %t31 = call <vscale x 16 x double> @llvm.vp.fmul.nxv16f64(<vscale x 16 x double> undef, <vscale x 16 x double> undef, <vscale x 16 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
   %t0 = call <2 x float> @llvm.vp.fmul.v2f32(<2 x float> undef, <2 x float> undef, <2 x i1> undef, i32 undef)
   %t2 = call <4 x float> @llvm.vp.fmul.v4f32(<4 x float> undef, <4 x float> undef, <4 x i1> undef, i32 undef)
@@ -2082,25 +1439,6 @@ define void @vp_fdiv(){
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %t29 = call <vscale x 8 x double> @llvm.vp.fdiv.nxv8f64(<vscale x 8 x double> undef, <vscale x 8 x double> undef, <vscale x 8 x i1> undef, i32 undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %t31 = call <vscale x 16 x double> @llvm.vp.fdiv.nxv16f64(<vscale x 16 x double> undef, <vscale x 16 x double> undef, <vscale x 16 x i1> undef, i32 undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
-;
-; TYPEBASED-LABEL: 'vp_fdiv'
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %t0 = call <2 x float> @llvm.vp.fdiv.v2f32(<2 x float> undef, <2 x float> undef, <2 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %t2 = call <4 x float> @llvm.vp.fdiv.v4f32(<4 x float> undef, <4 x float> undef, <4 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %t4 = call <8 x float> @llvm.vp.fdiv.v8f32(<8 x float> undef, <8 x float> undef, <8 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %t6 = call <16 x float> @llvm.vp.fdiv.v16f32(<16 x float> undef, <16 x float> undef, <16 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %t8 = call <2 x double> @llvm.vp.fdiv.v2f64(<2 x double> undef, <2 x double> undef, <2 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %t10 = call <4 x double> @llvm.vp.fdiv.v4f64(<4 x double> undef, <4 x double> undef, <4 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %t12 = call <8 x double> @llvm.vp.fdiv.v8f64(<8 x double> undef, <8 x double> undef, <8 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %t14 = call <16 x double> @llvm.vp.fdiv.v16f64(<16 x double> undef, <16 x double> undef, <16 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %t17 = call <vscale x 2 x float> @llvm.vp.fdiv.nxv2f32(<vscale x 2 x float> undef, <vscale x 2 x float> undef, <vscale x 2 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %t19 = call <vscale x 4 x float> @llvm.vp.fdiv.nxv4f32(<vscale x 4 x float> undef, <vscale x 4 x float> undef, <vscale x 4 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %t21 = call <vscale x 8 x float> @llvm.vp.fdiv.nxv8f32(<vscale x 8 x float> undef, <vscale x 8 x float> undef, <vscale x 8 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %t23 = call <vscale x 16 x float> @llvm.vp.fdiv.nxv16f32(<vscale x 16 x float> undef, <vscale x 16 x float> undef, <vscale x 16 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %t25 = call <vscale x 2 x double> @llvm.vp.fdiv.nxv2f64(<vscale x 2 x double> undef, <vscale x 2 x double> undef, <vscale x 2 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %t27 = call <vscale x 4 x double> @llvm.vp.fdiv.nxv4f64(<vscale x 4 x double> undef, <vscale x 4 x double> undef, <vscale x 4 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %t29 = call <vscale x 8 x double> @llvm.vp.fdiv.nxv8f64(<vscale x 8 x double> undef, <vscale x 8 x double> undef, <vscale x 8 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %t31 = call <vscale x 16 x double> @llvm.vp.fdiv.nxv16f64(<vscale x 16 x double> undef, <vscale x 16 x double> undef, <vscale x 16 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
   %t0 = call <2 x float> @llvm.vp.fdiv.v2f32(<2 x float> undef, <2 x float> undef, <2 x i1> undef, i32 undef)
   %t2 = call <4 x float> @llvm.vp.fdiv.v4f32(<4 x float> undef, <4 x float> undef, <4 x i1> undef, i32 undef)
@@ -2200,81 +1538,6 @@ define void @splat() {
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %71 = call <vscale x 8 x double> @llvm.experimental.vp.splat.nxv8f64(double undef, <vscale x 8 x i1> undef, i32 undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %72 = call <vscale x 16 x double> @llvm.experimental.vp.splat.nxv16f64(double undef, <vscale x 16 x i1> undef, i32 undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
-;
-; TYPEBASED-LABEL: 'splat'
-; TYPEBASED-NEXT:  Cost Model: Invalid cost for instruction: %1 = call <2 x i1> @llvm.experimental.vp.splat.v2i1(i1 undef, <2 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Invalid cost for instruction: %2 = call <4 x i1> @llvm.experimental.vp.splat.v4i1(i1 undef, <4 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Invalid cost for instruction: %3 = call <8 x i1> @llvm.experimental.vp.splat.v8i1(i1 undef, <8 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Invalid cost for instruction: %4 = call <16 x i1> @llvm.experimental.vp.splat.v16i1(i1 undef, <16 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %5 = call <2 x i8> @llvm.experimental.vp.splat.v2i8(i8 undef, <2 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %6 = call <4 x i8> @llvm.experimental.vp.splat.v4i8(i8 undef, <4 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %7 = call <8 x i8> @llvm.experimental.vp.splat.v8i8(i8 undef, <8 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %8 = call <16 x i8> @llvm.experimental.vp.splat.v16i8(i8 undef, <16 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %9 = call <2 x i16> @llvm.experimental.vp.splat.v2i16(i16 undef, <2 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %10 = call <4 x i16> @llvm.experimental.vp.splat.v4i16(i16 undef, <4 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %11 = call <8 x i16> @llvm.experimental.vp.splat.v8i16(i16 undef, <8 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %12 = call <16 x i16> @llvm.experimental.vp.splat.v16i16(i16 undef, <16 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %13 = call <2 x i32> @llvm.experimental.vp.splat.v2i32(i32 undef, <2 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %14 = call <4 x i32> @llvm.experimental.vp.splat.v4i32(i32 undef, <4 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %15 = call <8 x i32> @llvm.experimental.vp.splat.v8i32(i32 undef, <8 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %16 = call <16 x i32> @llvm.experimental.vp.splat.v16i32(i32 undef, <16 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %17 = call <2 x i64> @llvm.experimental.vp.splat.v2i64(i64 undef, <2 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %18 = call <4 x i64> @llvm.experimental.vp.splat.v4i64(i64 undef, <4 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %19 = call <8 x i64> @llvm.experimental.vp.splat.v8i64(i64 undef, <8 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %20 = call <16 x i64> @llvm.experimental.vp.splat.v16i64(i64 undef, <16 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %21 = call <2 x bfloat> @llvm.experimental.vp.splat.v2bf16(bfloat undef, <2 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %22 = call <4 x bfloat> @llvm.experimental.vp.splat.v4bf16(bfloat undef, <4 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %23 = call <8 x bfloat> @llvm.experimental.vp.splat.v8bf16(bfloat undef, <8 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %24 = call <16 x bfloat> @llvm.experimental.vp.splat.v16bf16(bfloat undef, <16 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %25 = call <2 x half> @llvm.experimental.vp.splat.v2f16(half undef, <2 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %26 = call <4 x half> @llvm.experimental.vp.splat.v4f16(half undef, <4 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %27 = call <8 x half> @llvm.experimental.vp.splat.v8f16(half undef, <8 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %28 = call <16 x half> @llvm.experimental.vp.splat.v16f16(half undef, <16 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %29 = call <2 x float> @llvm.experimental.vp.splat.v2f32(float undef, <2 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %30 = call <4 x float> @llvm.experimental.vp.splat.v4f32(float undef, <4 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %31 = call <8 x float> @llvm.experimental.vp.splat.v8f32(float undef, <8 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %32 = call <16 x float> @llvm.experimental.vp.splat.v16f32(float undef, <16 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %33 = call <2 x double> @llvm.experimental.vp.splat.v2f64(double undef, <2 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %34 = call <4 x double> @llvm.experimental.vp.splat.v4f64(double undef, <4 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %35 = call <8 x double> @llvm.experimental.vp.splat.v8f64(double undef, <8 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %36 = call <16 x double> @llvm.experimental.vp.splat.v16f64(double undef, <16 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Invalid cost for instruction: %37 = call <vscale x 2 x i1> @llvm.experimental.vp.splat.nxv2i1(i1 undef, <vscale x 2 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Invalid cost for instruction: %38 = call <vscale x 4 x i1> @llvm.experimental.vp.splat.nxv4i1(i1 undef, <vscale x 4 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Invalid cost for instruction: %39 = call <vscale x 8 x i1> @llvm.experimental.vp.splat.nxv8i1(i1 undef, <vscale x 8 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Invalid cost for instruction: %40 = call <vscale x 16 x i1> @llvm.experimental.vp.splat.nxv16i1(i1 undef, <vscale x 16 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %41 = call <vscale x 2 x i8> @llvm.experimental.vp.splat.nxv2i8(i8 undef, <vscale x 2 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %42 = call <vscale x 4 x i8> @llvm.experimental.vp.splat.nxv4i8(i8 undef, <vscale x 4 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %43 = call <vscale x 8 x i8> @llvm.experimental.vp.splat.nxv8i8(i8 undef, <vscale x 8 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %44 = call <vscale x 16 x i8> @llvm.experimental.vp.splat.nxv16i8(i8 undef, <vscale x 16 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %45 = call <vscale x 2 x i16> @llvm.experimental.vp.splat.nxv2i16(i16 undef, <vscale x 2 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %46 = call <vscale x 4 x i16> @llvm.experimental.vp.splat.nxv4i16(i16 undef, <vscale x 4 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %47 = call <vscale x 8 x i16> @llvm.experimental.vp.splat.nxv8i16(i16 undef, <vscale x 8 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %48 = call <vscale x 16 x i16> @llvm.experimental.vp.splat.nxv16i16(i16 undef, <vscale x 16 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %49 = call <vscale x 2 x i32> @llvm.experimental.vp.splat.nxv2i32(i32 undef, <vscale x 2 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %50 = call <vscale x 4 x i32> @llvm.experimental.vp.splat.nxv4i32(i32 undef, <vscale x 4 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %51 = call <vscale x 8 x i32> @llvm.experimental.vp.splat.nxv8i32(i32 undef, <vscale x 8 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %52 = call <vscale x 16 x i32> @llvm.experimental.vp.splat.nxv16i32(i32 undef, <vscale x 16 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %53 = call <vscale x 2 x i64> @llvm.experimental.vp.splat.nxv2i64(i64 undef, <vscale x 2 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %54 = call <vscale x 4 x i64> @llvm.experimental.vp.splat.nxv4i64(i64 undef, <vscale x 4 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %55 = call <vscale x 8 x i64> @llvm.experimental.vp.splat.nxv8i64(i64 undef, <vscale x 8 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %56 = call <vscale x 16 x i64> @llvm.experimental.vp.splat.nxv16i64(i64 undef, <vscale x 16 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %57 = call <vscale x 2 x bfloat> @llvm.experimental.vp.splat.nxv2bf16(bfloat undef, <vscale x 2 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %58 = call <vscale x 4 x bfloat> @llvm.experimental.vp.splat.nxv4bf16(bfloat undef, <vscale x 4 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %59 = call <vscale x 8 x bfloat> @llvm.experimental.vp.splat.nxv8bf16(bfloat undef, <vscale x 8 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %60 = call <vscale x 16 x bfloat> @llvm.experimental.vp.splat.nxv16bf16(bfloat undef, <vscale x 16 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %61 = call <vscale x 2 x half> @llvm.experimental.vp.splat.nxv2f16(half undef, <vscale x 2 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %62 = call <vscale x 4 x half> @llvm.experimental.vp.splat.nxv4f16(half undef, <vscale x 4 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %63 = call <vscale x 8 x half> @llvm.experimental.vp.splat.nxv8f16(half undef, <vscale x 8 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %64 = call <vscale x 16 x half> @llvm.experimental.vp.splat.nxv16f16(half undef, <vscale x 16 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %65 = call <vscale x 2 x float> @llvm.experimental.vp.splat.nxv2f32(float undef, <vscale x 2 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %66 = call <vscale x 4 x float> @llvm.experimental.vp.splat.nxv4f32(float undef, <vscale x 4 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %67 = call <vscale x 8 x float> @llvm.experimental.vp.splat.nxv8f32(float undef, <vscale x 8 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %68 = call <vscale x 16 x float> @llvm.experimental.vp.splat.nxv16f32(float undef, <vscale x 16 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %69 = call <vscale x 2 x double> @llvm.experimental.vp.splat.nxv2f64(double undef, <vscale x 2 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %70 = call <vscale x 4 x double> @llvm.experimental.vp.splat.nxv4f64(double undef, <vscale x 4 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %71 = call <vscale x 8 x double> @llvm.experimental.vp.splat.nxv8f64(double undef, <vscale x 8 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %72 = call <vscale x 16 x double> @llvm.experimental.vp.splat.nxv16f64(double undef, <vscale x 16 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
   call <2 x i1> @llvm.experimental.vp.splat.v2i1(i1 undef, <2 x i1> undef, i32 undef)
   call <4 x i1> @llvm.experimental.vp.splat.v4i1(i1 undef, <4 x i1> undef, i32 undef)
@@ -2351,6 +1614,41 @@ define void @splat() {
   ret void
 }
 
+define void @splice() {
+; CHECK-LABEL: 'splice'
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %splice_nxv16i8 = call <vscale x 16 x i8> @llvm.experimental.vp.splice.nxv16i8(<vscale x 16 x i8> zeroinitializer, <vscale x 16 x i8> zeroinitializer, i32 1, <vscale x 16 x i1> zeroinitializer, i32 poison, i32 poison)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %splice_nxv32i8 = call <vscale x 32 x i8> @llvm.experimental.vp.splice.nxv32i8(<vscale x 32 x i8> zeroinitializer, <vscale x 32 x i8> zeroinitializer, i32 1, <vscale x 32 x i1> zeroinitializer, i32 poison, i32 poison)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %splice_nxv2i16 = call <vscale x 2 x i16> @llvm.experimental.vp.splice.nxv2i16(<vscale x 2 x i16> zeroinitializer, <vscale x 2 x i16> zeroinitializer, i32 1, <vscale x 2 x i1> zeroinitializer, i32 poison, i32 poison)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %splice_nxv4i16 = call <vscale x 4 x i16> @llvm.experimental.vp.splice.nxv4i16(<vscale x 4 x i16> zeroinitializer, <vscale x 4 x i16> zeroinitializer, i32 1, <vscale x 4 x i1> zeroinitializer, i32 poison, i32 poison)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %splice_nxv8i16 = call <vscale x 8 x i16> @llvm.experimental.vp.splice.nxv8i16(<vscale x 8 x i16> zeroinitializer, <vscale x 8 x i16> zeroinitializer, i32 1, <vscale x 8 x i1> zeroinitializer, i32 poison, i32 poison)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %splice_nxv16i16 = call <vscale x 16 x i16> @llvm.experimental.vp.splice.nxv16i16(<vscale x 16 x i16> zeroinitializer, <vscale x 16 x i16> zeroinitializer, i32 1, <vscale x 16 x i1> zeroinitializer, i32 poison, i32 poison)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %splice_nxv4i32 = call <vscale x 4 x i32> @llvm.experimental.vp.splice.nxv4i32(<vscale x 4 x i32> zeroinitializer, <vscale x 4 x i32> zeroinitializer, i32 1, <vscale x 4 x i1> zeroinitializer, i32 poison, i32 poison)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %splice_nxv8i32 = call <vscale x 8 x i32> @llvm.experimental.vp.splice.nxv8i32(<vscale x 8 x i32> zeroinitializer, <vscale x 8 x i32> zeroinitializer, i32 1, <vscale x 8 x i1> zeroinitializer, i32 poison, i32 poison)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %splice_nxv2i64 = call <vscale x 2 x i64> @llvm.experimental.vp.splice.nxv2i64(<vscale x 2 x i64> zeroinitializer, <vscale x 2 x i64> zeroinitializer, i32 1, <vscale x 2 x i1> zeroinitializer, i32 poison, i32 poison)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %splice_nxv4i64 = call <vscale x 4 x i64> @llvm.experimental.vp.splice.nxv4i64(<vscale x 4 x i64> zeroinitializer, <vscale x 4 x i64> zeroinitializer, i32 1, <vscale x 4 x i1> zeroinitializer, i32 poison, i32 poison)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %splice_nxv16i1 = call <vscale x 16 x i1> @llvm.experimental.vp.splice.nxv16i1(<vscale x 16 x i1> zeroinitializer, <vscale x 16 x i1> zeroinitializer, i32 1, <vscale x 16 x i1> zeroinitializer, i32 poison, i32 poison)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %splice_nxv8i1 = call <vscale x 8 x i1> @llvm.experimental.vp.splice.nxv8i1(<vscale x 8 x i1> zeroinitializer, <vscale x 8 x i1> zeroinitializer, i32 1, <vscale x 8 x i1> zeroinitializer, i32 poison, i32 poison)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %splice_nxv4i1 = call <vscale x 4 x i1> @llvm.experimental.vp.splice.nxv4i1(<vscale x 4 x i1> zeroinitializer, <vscale x 4 x i1> zeroinitializer, i32 1, <vscale x 4 x i1> zeroinitializer, i32 poison, i32 poison)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %splice_nxv2i1 = call <vscale x 2 x i1> @llvm.experimental.vp.splice.nxv2i1(<vscale x 2 x i1> zeroinitializer, <vscale x 2 x i1> zeroinitializer, i32 1, <vscale x 2 x i1> zeroinitializer, i32 poison, i32 poison)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+  %splice_nxv16i8 = call <vscale x 16 x i8> @llvm.experimental.vp.splice.nxv16i8(<vscale x 16 x i8> zeroinitializer, <vscale x 16 x i8> zeroinitializer, i32 1, <vscale x 16 x i1> zeroinitializer, i32 poison, i32 poison)
+  %splice_nxv32i8 = call <vscale x 32 x i8> @llvm.experimental.vp.splice.nxv32i8(<vscale x 32 x i8> zeroinitializer, <vscale x 32 x i8> zeroinitializer, i32 1, <vscale x 32 x i1> zeroinitializer, i32 poison, i32 poison)
+  %splice_nxv2i16 = call <vscale x 2 x i16> @llvm.experimental.vp.splice.nxv2i16(<vscale x 2 x i16> zeroinitializer, <vscale x 2 x i16> zeroinitializer, i32 1, <vscale x 2 x i1> zeroinitializer, i32 poison, i32 poison)
+  %splice_nxv4i16 = call <vscale x 4 x i16> @llvm.experimental.vp.splice.nxv4i16(<vscale x 4 x i16> zeroinitializer, <vscale x 4 x i16> zeroinitializer, i32 1, <vscale x 4 x i1> zeroinitializer, i32 poison, i32 poison)
+  %splice_nxv8i16 = call <vscale x 8 x i16> @llvm.experimental.vp.splice.nxv8i16(<vscale x 8 x i16> zeroinitializer, <vscale x 8 x i16> zeroinitializer, i32 1, <vscale x 8 x i1> zeroinitializer, i32 poison, i32 poison)
+  %splice_nxv16i16 = call <vscale x 16 x i16> @llvm.experimental.vp.splice.nxv16i16(<vscale x 16 x i16> zeroinitializer, <vscale x 16 x i16> zeroinitializer, i32 1, <vscale x 16 x i1> zeroinitializer, i32 poison, i32 poison)
+  %splice_nxv4i32 = call <vscale x 4 x i32> @llvm.experimental.vp.splice.nxv4i32(<vscale x 4 x i32> zeroinitializer, <vscale x 4 x i32> zeroinitializer, i32 1, <vscale x 4 x i1> zeroinitializer, i32 poison, i32 poison)
+  %splice_nxv8i32 = call <vscale x 8 x i32> @llvm.experimental.vp.splice.nxv8i32(<vscale x 8 x i32> zeroinitializer, <vscale x 8 x i32> zeroinitializer, i32 1, <vscale x 8 x i1> zeroinitializer, i32 poison, i32 poison)
+  %splice_nxv2i64 = call <vscale x 2 x i64> @llvm.experimental.vp.splice.nxv2i64(<vscale x 2 x i64> zeroinitializer, <vscale x 2 x i64> zeroinitializer, i32 1, <vscale x 2 x i1> zeroinitializer, i32 poison, i32 poison)
+  %splice_nxv4i64 = call <vscale x 4 x i64> @llvm.experimental.vp.splice.nxv4i64(<vscale x 4 x i64> zeroinitializer, <vscale x 4 x i64> zeroinitializer, i32 1, <vscale x 4 x i1> zeroinitializer, i32 poison, i32 poison)
+  %splice_nxv16i1 = call <vscale x 16 x i1> @llvm.experimental.vp.splice.nxv16i1(<vscale x 16 x i1> zeroinitializer, <vscale x 16 x i1> zeroinitializer, i32 1, <vscale x 16 x i1> zeroinitializer, i32 poison, i32 poison)
+  %splice_nxv8i1 =  call <vscale x 8 x i1> @llvm.experimental.vp.splice.nxv8i1(<vscale x 8 x i1> zeroinitializer, <vscale x 8 x i1> zeroinitializer, i32 1, <vscale x 8 x i1> zeroinitializer, i32 poison, i32 poison)
+  %splice_nxv4i1 = call <vscale x 4 x i1> @llvm.experimental.vp.splice.nxv4i1(<vscale x 4 x i1> zeroinitializer, <vscale x 4 x i1> zeroinitializer, i32 1, <vscale x 4 x i1> zeroinitializer, i32 poison, i32 poison)
+  %splice_nxv2i1 = call <vscale x 2 x i1> @llvm.experimental.vp.splice.nxv2i1(<vscale x 2 x i1> zeroinitializer, <vscale x 2 x i1> zeroinitializer, i32 1, <vscale x 2 x i1> zeroinitializer, i32 poison, i32 poison)
+  ret void
+}
+
 declare <2 x i8> @llvm.vp.add.v2i8(<2 x i8>, <2 x i8>, <2 x i1>, i32)
 declare <4 x i8> @llvm.vp.add.v4i8(<4 x i8>, <4 x i8>, <4 x i1>, i32)
 declare <8 x i8> @llvm.vp.add.v8i8(<8 x i8>, <8 x i8>, <8 x i1>, i32)
diff --git a/llvm/test/Analysis/CostModel/X86/ctlz-codesize.ll b/llvm/test/Analysis/CostModel/X86/ctlz-codesize.ll
index da0f71c63ef80..9f8e4edf7a0fc 100644
--- a/llvm/test/Analysis/CostModel/X86/ctlz-codesize.ll
+++ b/llvm/test/Analysis/CostModel/X86/ctlz-codesize.ll
@@ -17,7 +17,7 @@ declare  i8 @llvm.ctlz.i8(i8, i1)
 
 define i64 @var_ctlz_i64(i64 %a) {
 ; NOLZCNT-LABEL: 'var_ctlz_i64'
-; NOLZCNT-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %ctlz = call i64 @llvm.ctlz.i64(i64 %a, i1 false)
+; NOLZCNT-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %ctlz = call i64 @llvm.ctlz.i64(i64 %a, i1 false)
 ; NOLZCNT-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i64 %ctlz
 ;
 ; LZCNT-LABEL: 'var_ctlz_i64'
@@ -43,7 +43,7 @@ define i64 @var_ctlz_i64u(i64 %a) {
 
 define i32 @var_ctlz_i32(i32 %a) {
 ; NOLZCNT-LABEL: 'var_ctlz_i32'
-; NOLZCNT-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %ctlz = call i32 @llvm.ctlz.i32(i32 %a, i1 false)
+; NOLZCNT-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %ctlz = call i32 @llvm.ctlz.i32(i32 %a, i1 false)
 ; NOLZCNT-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 %ctlz
 ;
 ; LZCNT-LABEL: 'var_ctlz_i32'
@@ -69,7 +69,7 @@ define i32 @var_ctlz_i32u(i32 %a) {
 
 define i16 @var_ctlz_i16(i16 %a) {
 ; NOLZCNT-LABEL: 'var_ctlz_i16'
-; NOLZCNT-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %ctlz = call i16 @llvm.ctlz.i16(i16 %a, i1 false)
+; NOLZCNT-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %ctlz = call i16 @llvm.ctlz.i16(i16 %a, i1 false)
 ; NOLZCNT-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i16 %ctlz
 ;
 ; LZCNT-LABEL: 'var_ctlz_i16'
@@ -95,7 +95,7 @@ define i16 @var_ctlz_i16u(i16 %a) {
 
 define i8 @var_ctlz_i8(i8 %a) {
 ; NOLZCNT-LABEL: 'var_ctlz_i8'
-; NOLZCNT-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %ctlz = call i8 @llvm.ctlz.i8(i8 %a, i1 false)
+; NOLZCNT-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %ctlz = call i8 @llvm.ctlz.i8(i8 %a, i1 false)
 ; NOLZCNT-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i8 %ctlz
 ;
 ; LZCNT-LABEL: 'var_ctlz_i8'
diff --git a/llvm/test/Analysis/CostModel/X86/ctlz-sizelatency.ll b/llvm/test/Analysis/CostModel/X86/ctlz-sizelatency.ll
index 2425e7286265b..fc3516695852a 100644
--- a/llvm/test/Analysis/CostModel/X86/ctlz-sizelatency.ll
+++ b/llvm/test/Analysis/CostModel/X86/ctlz-sizelatency.ll
@@ -17,7 +17,7 @@ declare  i8 @llvm.ctlz.i8(i8, i1)
 
 define i64 @var_ctlz_i64(i64 %a) {
 ; NOLZCNT-LABEL: 'var_ctlz_i64'
-; NOLZCNT-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %ctlz = call i64 @llvm.ctlz.i64(i64 %a, i1 false)
+; NOLZCNT-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %ctlz = call i64 @llvm.ctlz.i64(i64 %a, i1 false)
 ; NOLZCNT-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i64 %ctlz
 ;
 ; LZCNT-LABEL: 'var_ctlz_i64'
@@ -43,7 +43,7 @@ define i64 @var_ctlz_i64u(i64 %a) {
 
 define i32 @var_ctlz_i32(i32 %a) {
 ; NOLZCNT-LABEL: 'var_ctlz_i32'
-; NOLZCNT-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %ctlz = call i32 @llvm.ctlz.i32(i32 %a, i1 false)
+; NOLZCNT-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %ctlz = call i32 @llvm.ctlz.i32(i32 %a, i1 false)
 ; NOLZCNT-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 %ctlz
 ;
 ; LZCNT-LABEL: 'var_ctlz_i32'
@@ -69,7 +69,7 @@ define i32 @var_ctlz_i32u(i32 %a) {
 
 define i16 @var_ctlz_i16(i16 %a) {
 ; NOLZCNT-LABEL: 'var_ctlz_i16'
-; NOLZCNT-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %ctlz = call i16 @llvm.ctlz.i16(i16 %a, i1 false)
+; NOLZCNT-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %ctlz = call i16 @llvm.ctlz.i16(i16 %a, i1 false)
 ; NOLZCNT-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i16 %ctlz
 ;
 ; LZCNT-LABEL: 'var_ctlz_i16'
@@ -95,7 +95,7 @@ define i16 @var_ctlz_i16u(i16 %a) {
 
 define i8 @var_ctlz_i8(i8 %a) {
 ; NOLZCNT-LABEL: 'var_ctlz_i8'
-; NOLZCNT-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %ctlz = call i8 @llvm.ctlz.i8(i8 %a, i1 false)
+; NOLZCNT-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %ctlz = call i8 @llvm.ctlz.i8(i8 %a, i1 false)
 ; NOLZCNT-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i8 %ctlz
 ;
 ; LZCNT-LABEL: 'var_ctlz_i8'
diff --git a/llvm/test/Analysis/CostModel/X86/ctlz.ll b/llvm/test/Analysis/CostModel/X86/ctlz.ll
index fa7982ce09e9c..d9d04de12467d 100644
--- a/llvm/test/Analysis/CostModel/X86/ctlz.ll
+++ b/llvm/test/Analysis/CostModel/X86/ctlz.ll
@@ -17,7 +17,7 @@ declare  i8 @llvm.ctlz.i8(i8, i1)
 
 define i64 @var_ctlz_i64(i64 %a) {
 ; NOLZCNT-LABEL: 'var_ctlz_i64'
-; NOLZCNT-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %ctlz = call i64 @llvm.ctlz.i64(i64 %a, i1 false)
+; NOLZCNT-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %ctlz = call i64 @llvm.ctlz.i64(i64 %a, i1 false)
 ; NOLZCNT-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i64 %ctlz
 ;
 ; LZCNT-LABEL: 'var_ctlz_i64'
@@ -43,7 +43,7 @@ define i64 @var_ctlz_i64u(i64 %a) {
 
 define i32 @var_ctlz_i32(i32 %a) {
 ; NOLZCNT-LABEL: 'var_ctlz_i32'
-; NOLZCNT-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %ctlz = call i32 @llvm.ctlz.i32(i32 %a, i1 false)
+; NOLZCNT-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %ctlz = call i32 @llvm.ctlz.i32(i32 %a, i1 false)
 ; NOLZCNT-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 %ctlz
 ;
 ; LZCNT-LABEL: 'var_ctlz_i32'
diff --git a/llvm/test/Analysis/CostModel/X86/cttz-codesize.ll b/llvm/test/Analysis/CostModel/X86/cttz-codesize.ll
index 07bf1dd7a2ff6..621c1b9320fc8 100644
--- a/llvm/test/Analysis/CostModel/X86/cttz-codesize.ll
+++ b/llvm/test/Analysis/CostModel/X86/cttz-codesize.ll
@@ -18,7 +18,7 @@ declare  i8 @llvm.cttz.i8(i8, i1)
 
 define i64 @var_cttz_i64(i64 %a) {
 ; NOBMI-LABEL: 'var_cttz_i64'
-; NOBMI-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %cttz = call i64 @llvm.cttz.i64(i64 %a, i1 false)
+; NOBMI-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %cttz = call i64 @llvm.cttz.i64(i64 %a, i1 false)
 ; NOBMI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i64 %cttz
 ;
 ; BMI-LABEL: 'var_cttz_i64'
@@ -40,7 +40,7 @@ define i64 @var_cttz_i64u(i64 %a) {
 
 define i32 @var_cttz_i32(i32 %a) {
 ; NOBMI-LABEL: 'var_cttz_i32'
-; NOBMI-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %cttz = call i32 @llvm.cttz.i32(i32 %a, i1 false)
+; NOBMI-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %cttz = call i32 @llvm.cttz.i32(i32 %a, i1 false)
 ; NOBMI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 %cttz
 ;
 ; BMI-LABEL: 'var_cttz_i32'
diff --git a/llvm/test/Analysis/CostModel/X86/cttz-sizelatency.ll b/llvm/test/Analysis/CostModel/X86/cttz-sizelatency.ll
index afe5cb8c55fe6..34d363ce00879 100644
--- a/llvm/test/Analysis/CostModel/X86/cttz-sizelatency.ll
+++ b/llvm/test/Analysis/CostModel/X86/cttz-sizelatency.ll
@@ -18,7 +18,7 @@ declare  i8 @llvm.cttz.i8(i8, i1)
 
 define i64 @var_cttz_i64(i64 %a) {
 ; NOBMI-LABEL: 'var_cttz_i64'
-; NOBMI-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %cttz = call i64 @llvm.cttz.i64(i64 %a, i1 false)
+; NOBMI-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %cttz = call i64 @llvm.cttz.i64(i64 %a, i1 false)
 ; NOBMI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i64 %cttz
 ;
 ; BMI-LABEL: 'var_cttz_i64'
@@ -44,7 +44,7 @@ define i64 @var_cttz_i64u(i64 %a) {
 
 define i32 @var_cttz_i32(i32 %a) {
 ; NOBMI-LABEL: 'var_cttz_i32'
-; NOBMI-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %cttz = call i32 @llvm.cttz.i32(i32 %a, i1 false)
+; NOBMI-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %cttz = call i32 @llvm.cttz.i32(i32 %a, i1 false)
 ; NOBMI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 %cttz
 ;
 ; BMI-LABEL: 'var_cttz_i32'
@@ -70,7 +70,7 @@ define i32 @var_cttz_i32u(i32 %a) {
 
 define i16 @var_cttz_i16(i16 %a) {
 ; NOBMI-LABEL: 'var_cttz_i16'
-; NOBMI-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %cttz = call i16 @llvm.cttz.i16(i16 %a, i1 false)
+; NOBMI-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %cttz = call i16 @llvm.cttz.i16(i16 %a, i1 false)
 ; NOBMI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i16 %cttz
 ;
 ; BMI-LABEL: 'var_cttz_i16'
@@ -96,7 +96,7 @@ define i16 @var_cttz_i16u(i16 %a) {
 
 define i8 @var_cttz_i8(i8 %a) {
 ; NOBMI-LABEL: 'var_cttz_i8'
-; NOBMI-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %cttz = call i8 @llvm.cttz.i8(i8 %a, i1 false)
+; NOBMI-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %cttz = call i8 @llvm.cttz.i8(i8 %a, i1 false)
 ; NOBMI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i8 %cttz
 ;
 ; BMI-LABEL: 'var_cttz_i8'
diff --git a/llvm/test/Analysis/CostModel/X86/cttz.ll b/llvm/test/Analysis/CostModel/X86/cttz.ll
index fa0f10f886f63..3f5a731b27d9b 100644
--- a/llvm/test/Analysis/CostModel/X86/cttz.ll
+++ b/llvm/test/Analysis/CostModel/X86/cttz.ll
@@ -18,7 +18,7 @@ declare  i8 @llvm.cttz.i8(i8, i1)
 
 define i64 @var_cttz_i64(i64 %a) {
 ; NOBMI-LABEL: 'var_cttz_i64'
-; NOBMI-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %cttz = call i64 @llvm.cttz.i64(i64 %a, i1 false)
+; NOBMI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %cttz = call i64 @llvm.cttz.i64(i64 %a, i1 false)
 ; NOBMI-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i64 %cttz
 ;
 ; BMI-LABEL: 'var_cttz_i64'
@@ -44,7 +44,7 @@ define i64 @var_cttz_i64u(i64 %a) {
 
 define i32 @var_cttz_i32(i32 %a) {
 ; NOBMI-LABEL: 'var_cttz_i32'
-; NOBMI-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %cttz = call i32 @llvm.cttz.i32(i32 %a, i1 false)
+; NOBMI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %cttz = call i32 @llvm.cttz.i32(i32 %a, i1 false)
 ; NOBMI-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 %cttz
 ;
 ; BMI-LABEL: 'var_cttz_i32'
diff --git a/llvm/test/Analysis/CostModel/X86/intrinsic-cost-kinds.ll b/llvm/test/Analysis/CostModel/X86/intrinsic-cost-kinds.ll
index 062e5f157bae2..bcef47ee9e056 100644
--- a/llvm/test/Analysis/CostModel/X86/intrinsic-cost-kinds.ll
+++ b/llvm/test/Analysis/CostModel/X86/intrinsic-cost-kinds.ll
@@ -232,7 +232,7 @@ define void @fmaximum(float %a, float %b, <16 x float> %va, <16 x float> %vb) {
 
 define void @cttz(i32 %a, <16 x i32> %va) {
 ; THRU-LABEL: 'cttz'
-; THRU-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %s = call i32 @llvm.cttz.i32(i32 %a, i1 false)
+; THRU-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s = call i32 @llvm.cttz.i32(i32 %a, i1 false)
 ; THRU-NEXT:  Cost Model: Found an estimated cost of 72 for instruction: %v = call <16 x i32> @llvm.cttz.v16i32(<16 x i32> %va, i1 false)
 ; THRU-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
@@ -242,12 +242,12 @@ define void @cttz(i32 %a, <16 x i32> %va) {
 ; LATE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; SIZE-LABEL: 'cttz'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %s = call i32 @llvm.cttz.i32(i32 %a, i1 false)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %s = call i32 @llvm.cttz.i32(i32 %a, i1 false)
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 96 for instruction: %v = call <16 x i32> @llvm.cttz.v16i32(<16 x i32> %va, i1 false)
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; SIZE_LATE-LABEL: 'cttz'
-; SIZE_LATE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %s = call i32 @llvm.cttz.i32(i32 %a, i1 false)
+; SIZE_LATE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %s = call i32 @llvm.cttz.i32(i32 %a, i1 false)
 ; SIZE_LATE-NEXT:  Cost Model: Found an estimated cost of 104 for instruction: %v = call <16 x i32> @llvm.cttz.v16i32(<16 x i32> %va, i1 false)
 ; SIZE_LATE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
diff --git a/llvm/test/Analysis/CostModel/X86/masked-intrinsic-codesize.ll b/llvm/test/Analysis/CostModel/X86/masked-intrinsic-codesize.ll
index 47472d5722858..d49528d558ec8 100644
--- a/llvm/test/Analysis/CostModel/X86/masked-intrinsic-codesize.ll
+++ b/llvm/test/Analysis/CostModel/X86/masked-intrinsic-codesize.ll
@@ -1178,165 +1178,165 @@ define i32 @masked_scatter(<1 x i1> %m1, <2 x i1> %m2, <4 x i1> %m4, <8 x i1> %m
 
 define i32 @masked_expandload(<1 x i1> %m1, <2 x i1> %m2, <4 x i1> %m4, <8 x i1> %m8, <16 x i1> %m16, <32 x i1> %m32, <64 x i1> %m64) {
 ; SSE2-LABEL: 'masked_expandload'
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %V8F64 = call <8 x double> @llvm.masked.expandload.v8f64(ptr undef, <8 x i1> %m8, <8 x double> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V4F64 = call <4 x double> @llvm.masked.expandload.v4f64(ptr undef, <4 x i1> %m4, <4 x double> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V2F64 = call <2 x double> @llvm.masked.expandload.v2f64(ptr undef, <2 x i1> %m2, <2 x double> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1F64 = call <1 x double> @llvm.masked.expandload.v1f64(ptr undef, <1 x i1> %m1, <1 x double> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 29 for instruction: %V16F32 = call <16 x float> @llvm.masked.expandload.v16f32(ptr undef, <16 x i1> %m16, <16 x float> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %V8F32 = call <8 x float> @llvm.masked.expandload.v8f32(ptr undef, <8 x i1> %m8, <8 x float> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V4F32 = call <4 x float> @llvm.masked.expandload.v4f32(ptr undef, <4 x i1> %m4, <4 x float> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V2F32 = call <2 x float> @llvm.masked.expandload.v2f32(ptr undef, <2 x i1> %m2, <2 x float> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 21 for instruction: %V8I64 = call <8 x i64> @llvm.masked.expandload.v8i64(ptr undef, <8 x i1> %m8, <8 x i64> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %V4I64 = call <4 x i64> @llvm.masked.expandload.v4i64(ptr undef, <4 x i1> %m4, <4 x i64> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2I64 = call <2 x i64> @llvm.masked.expandload.v2i64(ptr undef, <2 x i1> %m2, <2 x i64> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1I64 = call <1 x i64> @llvm.masked.expandload.v1i64(ptr undef, <1 x i1> %m1, <1 x i64> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 45 for instruction: %V16I32 = call <16 x i32> @llvm.masked.expandload.v16i32(ptr undef, <16 x i1> %m16, <16 x i32> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 23 for instruction: %V8I32 = call <8 x i32> @llvm.masked.expandload.v8i32(ptr undef, <8 x i1> %m8, <8 x i32> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V4I32 = call <4 x i32> @llvm.masked.expandload.v4i32(ptr undef, <4 x i1> %m4, <4 x i32> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2I32 = call <2 x i32> @llvm.masked.expandload.v2i32(ptr undef, <2 x i1> %m2, <2 x i32> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 66 for instruction: %V32I16 = call <32 x i16> @llvm.masked.expandload.v32i16(ptr undef, <32 x i1> %m32, <32 x i16> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %V16I16 = call <16 x i16> @llvm.masked.expandload.v16i16(ptr undef, <16 x i1> %m16, <16 x i16> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %V8I16 = call <8 x i16> @llvm.masked.expandload.v8i16(ptr undef, <8 x i1> %m8, <8 x i16> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V4I16 = call <4 x i16> @llvm.masked.expandload.v4i16(ptr undef, <4 x i1> %m4, <4 x i16> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 192 for instruction: %V64I8 = call <64 x i8> @llvm.masked.expandload.v64i8(ptr undef, <64 x i1> %m64, <64 x i8> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 96 for instruction: %V32I8 = call <32 x i8> @llvm.masked.expandload.v32i8(ptr undef, <32 x i1> %m32, <32 x i8> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V16I8 = call <16 x i8> @llvm.masked.expandload.v16i8(ptr undef, <16 x i1> %m16, <16 x i8> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V8I8 = call <8 x i8> @llvm.masked.expandload.v8i8(ptr undef, <8 x i1> %m8, <8 x i8> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 37 for instruction: %V8F64 = call <8 x double> @llvm.masked.expandload.v8f64(ptr undef, <8 x i1> %m8, <8 x double> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 19 for instruction: %V4F64 = call <4 x double> @llvm.masked.expandload.v4f64(ptr undef, <4 x i1> %m4, <4 x double> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V2F64 = call <2 x double> @llvm.masked.expandload.v2f64(ptr undef, <2 x i1> %m2, <2 x double> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V1F64 = call <1 x double> @llvm.masked.expandload.v1f64(ptr undef, <1 x i1> %m1, <1 x double> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 77 for instruction: %V16F32 = call <16 x float> @llvm.masked.expandload.v16f32(ptr undef, <16 x i1> %m16, <16 x float> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 39 for instruction: %V8F32 = call <8 x float> @llvm.masked.expandload.v8f32(ptr undef, <8 x i1> %m8, <8 x float> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %V4F32 = call <4 x float> @llvm.masked.expandload.v4f32(ptr undef, <4 x i1> %m4, <4 x float> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V2F32 = call <2 x float> @llvm.masked.expandload.v2f32(ptr undef, <2 x i1> %m2, <2 x float> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 45 for instruction: %V8I64 = call <8 x i64> @llvm.masked.expandload.v8i64(ptr undef, <8 x i1> %m8, <8 x i64> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 23 for instruction: %V4I64 = call <4 x i64> @llvm.masked.expandload.v4i64(ptr undef, <4 x i1> %m4, <4 x i64> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V2I64 = call <2 x i64> @llvm.masked.expandload.v2i64(ptr undef, <2 x i1> %m2, <2 x i64> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V1I64 = call <1 x i64> @llvm.masked.expandload.v1i64(ptr undef, <1 x i1> %m1, <1 x i64> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 93 for instruction: %V16I32 = call <16 x i32> @llvm.masked.expandload.v16i32(ptr undef, <16 x i1> %m16, <16 x i32> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 47 for instruction: %V8I32 = call <8 x i32> @llvm.masked.expandload.v8i32(ptr undef, <8 x i1> %m8, <8 x i32> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V4I32 = call <4 x i32> @llvm.masked.expandload.v4i32(ptr undef, <4 x i1> %m4, <4 x i32> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V2I32 = call <2 x i32> @llvm.masked.expandload.v2i32(ptr undef, <2 x i1> %m2, <2 x i32> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 162 for instruction: %V32I16 = call <32 x i16> @llvm.masked.expandload.v32i16(ptr undef, <32 x i1> %m32, <32 x i16> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 81 for instruction: %V16I16 = call <16 x i16> @llvm.masked.expandload.v16i16(ptr undef, <16 x i1> %m16, <16 x i16> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 41 for instruction: %V8I16 = call <8 x i16> @llvm.masked.expandload.v8i16(ptr undef, <8 x i1> %m8, <8 x i16> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 21 for instruction: %V4I16 = call <4 x i16> @llvm.masked.expandload.v4i16(ptr undef, <4 x i1> %m4, <4 x i16> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 384 for instruction: %V64I8 = call <64 x i8> @llvm.masked.expandload.v64i8(ptr undef, <64 x i1> %m64, <64 x i8> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 192 for instruction: %V32I8 = call <32 x i8> @llvm.masked.expandload.v32i8(ptr undef, <32 x i1> %m32, <32 x i8> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 96 for instruction: %V16I8 = call <16 x i8> @llvm.masked.expandload.v16i8(ptr undef, <16 x i1> %m16, <16 x i8> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V8I8 = call <8 x i8> @llvm.masked.expandload.v8i8(ptr undef, <8 x i1> %m8, <8 x i8> undef)
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 0
 ;
 ; SSE42-LABEL: 'masked_expandload'
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %V8F64 = call <8 x double> @llvm.masked.expandload.v8f64(ptr undef, <8 x i1> %m8, <8 x double> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V4F64 = call <4 x double> @llvm.masked.expandload.v4f64(ptr undef, <4 x i1> %m4, <4 x double> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V2F64 = call <2 x double> @llvm.masked.expandload.v2f64(ptr undef, <2 x i1> %m2, <2 x double> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1F64 = call <1 x double> @llvm.masked.expandload.v1f64(ptr undef, <1 x i1> %m1, <1 x double> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 29 for instruction: %V16F32 = call <16 x float> @llvm.masked.expandload.v16f32(ptr undef, <16 x i1> %m16, <16 x float> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %V8F32 = call <8 x float> @llvm.masked.expandload.v8f32(ptr undef, <8 x i1> %m8, <8 x float> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V4F32 = call <4 x float> @llvm.masked.expandload.v4f32(ptr undef, <4 x i1> %m4, <4 x float> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V2F32 = call <2 x float> @llvm.masked.expandload.v2f32(ptr undef, <2 x i1> %m2, <2 x float> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %V8I64 = call <8 x i64> @llvm.masked.expandload.v8i64(ptr undef, <8 x i1> %m8, <8 x i64> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V4I64 = call <4 x i64> @llvm.masked.expandload.v4i64(ptr undef, <4 x i1> %m4, <4 x i64> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V2I64 = call <2 x i64> @llvm.masked.expandload.v2i64(ptr undef, <2 x i1> %m2, <2 x i64> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1I64 = call <1 x i64> @llvm.masked.expandload.v1i64(ptr undef, <1 x i1> %m1, <1 x i64> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %V16I32 = call <16 x i32> @llvm.masked.expandload.v16i32(ptr undef, <16 x i1> %m16, <16 x i32> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %V8I32 = call <8 x i32> @llvm.masked.expandload.v8i32(ptr undef, <8 x i1> %m8, <8 x i32> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V4I32 = call <4 x i32> @llvm.masked.expandload.v4i32(ptr undef, <4 x i1> %m4, <4 x i32> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V2I32 = call <2 x i32> @llvm.masked.expandload.v2i32(ptr undef, <2 x i1> %m2, <2 x i32> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 66 for instruction: %V32I16 = call <32 x i16> @llvm.masked.expandload.v32i16(ptr undef, <32 x i1> %m32, <32 x i16> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %V16I16 = call <16 x i16> @llvm.masked.expandload.v16i16(ptr undef, <16 x i1> %m16, <16 x i16> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %V8I16 = call <8 x i16> @llvm.masked.expandload.v8i16(ptr undef, <8 x i1> %m8, <8 x i16> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V4I16 = call <4 x i16> @llvm.masked.expandload.v4i16(ptr undef, <4 x i1> %m4, <4 x i16> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 132 for instruction: %V64I8 = call <64 x i8> @llvm.masked.expandload.v64i8(ptr undef, <64 x i1> %m64, <64 x i8> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 66 for instruction: %V32I8 = call <32 x i8> @llvm.masked.expandload.v32i8(ptr undef, <32 x i1> %m32, <32 x i8> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %V16I8 = call <16 x i8> @llvm.masked.expandload.v16i8(ptr undef, <16 x i1> %m16, <16 x i8> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %V8I8 = call <8 x i8> @llvm.masked.expandload.v8i8(ptr undef, <8 x i1> %m8, <8 x i8> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 29 for instruction: %V8F64 = call <8 x double> @llvm.masked.expandload.v8f64(ptr undef, <8 x i1> %m8, <8 x double> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %V4F64 = call <4 x double> @llvm.masked.expandload.v4f64(ptr undef, <4 x i1> %m4, <4 x double> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V2F64 = call <2 x double> @llvm.masked.expandload.v2f64(ptr undef, <2 x i1> %m2, <2 x double> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V1F64 = call <1 x double> @llvm.masked.expandload.v1f64(ptr undef, <1 x i1> %m1, <1 x double> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 61 for instruction: %V16F32 = call <16 x float> @llvm.masked.expandload.v16f32(ptr undef, <16 x i1> %m16, <16 x float> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 31 for instruction: %V8F32 = call <8 x float> @llvm.masked.expandload.v8f32(ptr undef, <8 x i1> %m8, <8 x float> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V4F32 = call <4 x float> @llvm.masked.expandload.v4f32(ptr undef, <4 x i1> %m4, <4 x float> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V2F32 = call <2 x float> @llvm.masked.expandload.v2f32(ptr undef, <2 x i1> %m2, <2 x float> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %V8I64 = call <8 x i64> @llvm.masked.expandload.v8i64(ptr undef, <8 x i1> %m8, <8 x i64> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %V4I64 = call <4 x i64> @llvm.masked.expandload.v4i64(ptr undef, <4 x i1> %m4, <4 x i64> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V2I64 = call <2 x i64> @llvm.masked.expandload.v2i64(ptr undef, <2 x i1> %m2, <2 x i64> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V1I64 = call <1 x i64> @llvm.masked.expandload.v1i64(ptr undef, <1 x i1> %m1, <1 x i64> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 65 for instruction: %V16I32 = call <16 x i32> @llvm.masked.expandload.v16i32(ptr undef, <16 x i1> %m16, <16 x i32> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %V8I32 = call <8 x i32> @llvm.masked.expandload.v8i32(ptr undef, <8 x i1> %m8, <8 x i32> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %V4I32 = call <4 x i32> @llvm.masked.expandload.v4i32(ptr undef, <4 x i1> %m4, <4 x i32> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V2I32 = call <2 x i32> @llvm.masked.expandload.v2i32(ptr undef, <2 x i1> %m2, <2 x i32> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 130 for instruction: %V32I16 = call <32 x i16> @llvm.masked.expandload.v32i16(ptr undef, <32 x i1> %m32, <32 x i16> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 65 for instruction: %V16I16 = call <16 x i16> @llvm.masked.expandload.v16i16(ptr undef, <16 x i1> %m16, <16 x i16> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %V8I16 = call <8 x i16> @llvm.masked.expandload.v8i16(ptr undef, <8 x i1> %m8, <8 x i16> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %V4I16 = call <4 x i16> @llvm.masked.expandload.v4i16(ptr undef, <4 x i1> %m4, <4 x i16> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 260 for instruction: %V64I8 = call <64 x i8> @llvm.masked.expandload.v64i8(ptr undef, <64 x i1> %m64, <64 x i8> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 130 for instruction: %V32I8 = call <32 x i8> @llvm.masked.expandload.v32i8(ptr undef, <32 x i1> %m32, <32 x i8> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 65 for instruction: %V16I8 = call <16 x i8> @llvm.masked.expandload.v16i8(ptr undef, <16 x i1> %m16, <16 x i8> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %V8I8 = call <8 x i8> @llvm.masked.expandload.v8i8(ptr undef, <8 x i1> %m8, <8 x i8> undef)
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 0
 ;
 ; AVX1-LABEL: 'masked_expandload'
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %V8F64 = call <8 x double> @llvm.masked.expandload.v8f64(ptr undef, <8 x i1> %m8, <8 x double> undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V4F64 = call <4 x double> @llvm.masked.expandload.v4f64(ptr undef, <4 x i1> %m4, <4 x double> undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V2F64 = call <2 x double> @llvm.masked.expandload.v2f64(ptr undef, <2 x i1> %m2, <2 x double> undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1F64 = call <1 x double> @llvm.masked.expandload.v1f64(ptr undef, <1 x i1> %m1, <1 x double> undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 31 for instruction: %V16F32 = call <16 x float> @llvm.masked.expandload.v16f32(ptr undef, <16 x i1> %m16, <16 x float> undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V8F32 = call <8 x float> @llvm.masked.expandload.v8f32(ptr undef, <8 x i1> %m8, <8 x float> undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V4F32 = call <4 x float> @llvm.masked.expandload.v4f32(ptr undef, <4 x i1> %m4, <4 x float> undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V2F32 = call <2 x float> @llvm.masked.expandload.v2f32(ptr undef, <2 x i1> %m2, <2 x float> undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 19 for instruction: %V8I64 = call <8 x i64> @llvm.masked.expandload.v8i64(ptr undef, <8 x i1> %m8, <8 x i64> undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V4I64 = call <4 x i64> @llvm.masked.expandload.v4i64(ptr undef, <4 x i1> %m4, <4 x i64> undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V2I64 = call <2 x i64> @llvm.masked.expandload.v2i64(ptr undef, <2 x i1> %m2, <2 x i64> undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1I64 = call <1 x i64> @llvm.masked.expandload.v1i64(ptr undef, <1 x i1> %m1, <1 x i64> undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 35 for instruction: %V16I32 = call <16 x i32> @llvm.masked.expandload.v16i32(ptr undef, <16 x i1> %m16, <16 x i32> undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V8I32 = call <8 x i32> @llvm.masked.expandload.v8i32(ptr undef, <8 x i1> %m8, <8 x i32> undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V4I32 = call <4 x i32> @llvm.masked.expandload.v4i32(ptr undef, <4 x i1> %m4, <4 x i32> undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V2I32 = call <2 x i32> @llvm.masked.expandload.v2i32(ptr undef, <2 x i1> %m2, <2 x i32> undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 68 for instruction: %V32I16 = call <32 x i16> @llvm.masked.expandload.v32i16(ptr undef, <32 x i1> %m32, <32 x i16> undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %V16I16 = call <16 x i16> @llvm.masked.expandload.v16i16(ptr undef, <16 x i1> %m16, <16 x i16> undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %V8I16 = call <8 x i16> @llvm.masked.expandload.v8i16(ptr undef, <8 x i1> %m8, <8 x i16> undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V4I16 = call <4 x i16> @llvm.masked.expandload.v4i16(ptr undef, <4 x i1> %m4, <4 x i16> undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 134 for instruction: %V64I8 = call <64 x i8> @llvm.masked.expandload.v64i8(ptr undef, <64 x i1> %m64, <64 x i8> undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 67 for instruction: %V32I8 = call <32 x i8> @llvm.masked.expandload.v32i8(ptr undef, <32 x i1> %m32, <32 x i8> undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %V16I8 = call <16 x i8> @llvm.masked.expandload.v16i8(ptr undef, <16 x i1> %m16, <16 x i8> undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %V8I8 = call <8 x i8> @llvm.masked.expandload.v8i8(ptr undef, <8 x i1> %m8, <8 x i8> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %V8F64 = call <8 x double> @llvm.masked.expandload.v8f64(ptr undef, <8 x i1> %m8, <8 x double> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %V4F64 = call <4 x double> @llvm.masked.expandload.v4f64(ptr undef, <4 x i1> %m4, <4 x double> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V2F64 = call <2 x double> @llvm.masked.expandload.v2f64(ptr undef, <2 x i1> %m2, <2 x double> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V1F64 = call <1 x double> @llvm.masked.expandload.v1f64(ptr undef, <1 x i1> %m1, <1 x double> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 67 for instruction: %V16F32 = call <16 x float> @llvm.masked.expandload.v16f32(ptr undef, <16 x i1> %m16, <16 x float> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %V8F32 = call <8 x float> @llvm.masked.expandload.v8f32(ptr undef, <8 x i1> %m8, <8 x float> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %V4F32 = call <4 x float> @llvm.masked.expandload.v4f32(ptr undef, <4 x i1> %m4, <4 x float> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V2F32 = call <2 x float> @llvm.masked.expandload.v2f32(ptr undef, <2 x i1> %m2, <2 x float> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 37 for instruction: %V8I64 = call <8 x i64> @llvm.masked.expandload.v8i64(ptr undef, <8 x i1> %m8, <8 x i64> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 19 for instruction: %V4I64 = call <4 x i64> @llvm.masked.expandload.v4i64(ptr undef, <4 x i1> %m4, <4 x i64> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V2I64 = call <2 x i64> @llvm.masked.expandload.v2i64(ptr undef, <2 x i1> %m2, <2 x i64> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V1I64 = call <1 x i64> @llvm.masked.expandload.v1i64(ptr undef, <1 x i1> %m1, <1 x i64> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 71 for instruction: %V16I32 = call <16 x i32> @llvm.masked.expandload.v16i32(ptr undef, <16 x i1> %m16, <16 x i32> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 36 for instruction: %V8I32 = call <8 x i32> @llvm.masked.expandload.v8i32(ptr undef, <8 x i1> %m8, <8 x i32> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V4I32 = call <4 x i32> @llvm.masked.expandload.v4i32(ptr undef, <4 x i1> %m4, <4 x i32> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V2I32 = call <2 x i32> @llvm.masked.expandload.v2i32(ptr undef, <2 x i1> %m2, <2 x i32> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 140 for instruction: %V32I16 = call <32 x i16> @llvm.masked.expandload.v32i16(ptr undef, <32 x i1> %m32, <32 x i16> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 70 for instruction: %V16I16 = call <16 x i16> @llvm.masked.expandload.v16i16(ptr undef, <16 x i1> %m16, <16 x i16> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 35 for instruction: %V8I16 = call <8 x i16> @llvm.masked.expandload.v8i16(ptr undef, <8 x i1> %m8, <8 x i16> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V4I16 = call <4 x i16> @llvm.masked.expandload.v4i16(ptr undef, <4 x i1> %m4, <4 x i16> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 278 for instruction: %V64I8 = call <64 x i8> @llvm.masked.expandload.v64i8(ptr undef, <64 x i1> %m64, <64 x i8> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 139 for instruction: %V32I8 = call <32 x i8> @llvm.masked.expandload.v32i8(ptr undef, <32 x i1> %m32, <32 x i8> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 69 for instruction: %V16I8 = call <16 x i8> @llvm.masked.expandload.v16i8(ptr undef, <16 x i1> %m16, <16 x i8> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 35 for instruction: %V8I8 = call <8 x i8> @llvm.masked.expandload.v8i8(ptr undef, <8 x i1> %m8, <8 x i8> undef)
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 0
 ;
 ; AVX2-LABEL: 'masked_expandload'
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %V8F64 = call <8 x double> @llvm.masked.expandload.v8f64(ptr undef, <8 x i1> %m8, <8 x double> undef)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V4F64 = call <4 x double> @llvm.masked.expandload.v4f64(ptr undef, <4 x i1> %m4, <4 x double> undef)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V2F64 = call <2 x double> @llvm.masked.expandload.v2f64(ptr undef, <2 x i1> %m2, <2 x double> undef)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1F64 = call <1 x double> @llvm.masked.expandload.v1f64(ptr undef, <1 x i1> %m1, <1 x double> undef)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 31 for instruction: %V16F32 = call <16 x float> @llvm.masked.expandload.v16f32(ptr undef, <16 x i1> %m16, <16 x float> undef)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V8F32 = call <8 x float> @llvm.masked.expandload.v8f32(ptr undef, <8 x i1> %m8, <8 x float> undef)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V4F32 = call <4 x float> @llvm.masked.expandload.v4f32(ptr undef, <4 x i1> %m4, <4 x float> undef)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V2F32 = call <2 x float> @llvm.masked.expandload.v2f32(ptr undef, <2 x i1> %m2, <2 x float> undef)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 19 for instruction: %V8I64 = call <8 x i64> @llvm.masked.expandload.v8i64(ptr undef, <8 x i1> %m8, <8 x i64> undef)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V4I64 = call <4 x i64> @llvm.masked.expandload.v4i64(ptr undef, <4 x i1> %m4, <4 x i64> undef)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V2I64 = call <2 x i64> @llvm.masked.expandload.v2i64(ptr undef, <2 x i1> %m2, <2 x i64> undef)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1I64 = call <1 x i64> @llvm.masked.expandload.v1i64(ptr undef, <1 x i1> %m1, <1 x i64> undef)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 35 for instruction: %V16I32 = call <16 x i32> @llvm.masked.expandload.v16i32(ptr undef, <16 x i1> %m16, <16 x i32> undef)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V8I32 = call <8 x i32> @llvm.masked.expandload.v8i32(ptr undef, <8 x i1> %m8, <8 x i32> undef)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V4I32 = call <4 x i32> @llvm.masked.expandload.v4i32(ptr undef, <4 x i1> %m4, <4 x i32> undef)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V2I32 = call <2 x i32> @llvm.masked.expandload.v2i32(ptr undef, <2 x i1> %m2, <2 x i32> undef)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 67 for instruction: %V32I16 = call <32 x i16> @llvm.masked.expandload.v32i16(ptr undef, <32 x i1> %m32, <32 x i16> undef)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %V16I16 = call <16 x i16> @llvm.masked.expandload.v16i16(ptr undef, <16 x i1> %m16, <16 x i16> undef)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %V8I16 = call <8 x i16> @llvm.masked.expandload.v8i16(ptr undef, <8 x i1> %m8, <8 x i16> undef)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V4I16 = call <4 x i16> @llvm.masked.expandload.v4i16(ptr undef, <4 x i1> %m4, <4 x i16> undef)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 132 for instruction: %V64I8 = call <64 x i8> @llvm.masked.expandload.v64i8(ptr undef, <64 x i1> %m64, <64 x i8> undef)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 66 for instruction: %V32I8 = call <32 x i8> @llvm.masked.expandload.v32i8(ptr undef, <32 x i1> %m32, <32 x i8> undef)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %V16I8 = call <16 x i8> @llvm.masked.expandload.v16i8(ptr undef, <16 x i1> %m16, <16 x i8> undef)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %V8I8 = call <8 x i8> @llvm.masked.expandload.v8i8(ptr undef, <8 x i1> %m8, <8 x i8> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %V8F64 = call <8 x double> @llvm.masked.expandload.v8f64(ptr undef, <8 x i1> %m8, <8 x double> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %V4F64 = call <4 x double> @llvm.masked.expandload.v4f64(ptr undef, <4 x i1> %m4, <4 x double> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V2F64 = call <2 x double> @llvm.masked.expandload.v2f64(ptr undef, <2 x i1> %m2, <2 x double> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V1F64 = call <1 x double> @llvm.masked.expandload.v1f64(ptr undef, <1 x i1> %m1, <1 x double> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 67 for instruction: %V16F32 = call <16 x float> @llvm.masked.expandload.v16f32(ptr undef, <16 x i1> %m16, <16 x float> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %V8F32 = call <8 x float> @llvm.masked.expandload.v8f32(ptr undef, <8 x i1> %m8, <8 x float> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %V4F32 = call <4 x float> @llvm.masked.expandload.v4f32(ptr undef, <4 x i1> %m4, <4 x float> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V2F32 = call <2 x float> @llvm.masked.expandload.v2f32(ptr undef, <2 x i1> %m2, <2 x float> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 37 for instruction: %V8I64 = call <8 x i64> @llvm.masked.expandload.v8i64(ptr undef, <8 x i1> %m8, <8 x i64> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 19 for instruction: %V4I64 = call <4 x i64> @llvm.masked.expandload.v4i64(ptr undef, <4 x i1> %m4, <4 x i64> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V2I64 = call <2 x i64> @llvm.masked.expandload.v2i64(ptr undef, <2 x i1> %m2, <2 x i64> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V1I64 = call <1 x i64> @llvm.masked.expandload.v1i64(ptr undef, <1 x i1> %m1, <1 x i64> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 71 for instruction: %V16I32 = call <16 x i32> @llvm.masked.expandload.v16i32(ptr undef, <16 x i1> %m16, <16 x i32> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 36 for instruction: %V8I32 = call <8 x i32> @llvm.masked.expandload.v8i32(ptr undef, <8 x i1> %m8, <8 x i32> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V4I32 = call <4 x i32> @llvm.masked.expandload.v4i32(ptr undef, <4 x i1> %m4, <4 x i32> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V2I32 = call <2 x i32> @llvm.masked.expandload.v2i32(ptr undef, <2 x i1> %m2, <2 x i32> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 139 for instruction: %V32I16 = call <32 x i16> @llvm.masked.expandload.v32i16(ptr undef, <32 x i1> %m32, <32 x i16> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 70 for instruction: %V16I16 = call <16 x i16> @llvm.masked.expandload.v16i16(ptr undef, <16 x i1> %m16, <16 x i16> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 35 for instruction: %V8I16 = call <8 x i16> @llvm.masked.expandload.v8i16(ptr undef, <8 x i1> %m8, <8 x i16> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V4I16 = call <4 x i16> @llvm.masked.expandload.v4i16(ptr undef, <4 x i1> %m4, <4 x i16> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 276 for instruction: %V64I8 = call <64 x i8> @llvm.masked.expandload.v64i8(ptr undef, <64 x i1> %m64, <64 x i8> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 138 for instruction: %V32I8 = call <32 x i8> @llvm.masked.expandload.v32i8(ptr undef, <32 x i1> %m32, <32 x i8> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 69 for instruction: %V16I8 = call <16 x i8> @llvm.masked.expandload.v16i8(ptr undef, <16 x i1> %m16, <16 x i8> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 35 for instruction: %V8I8 = call <8 x i8> @llvm.masked.expandload.v8i8(ptr undef, <8 x i1> %m8, <8 x i8> undef)
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 0
 ;
 ; SKL-LABEL: 'masked_expandload'
-; SKL-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %V8F64 = call <8 x double> @llvm.masked.expandload.v8f64(ptr undef, <8 x i1> %m8, <8 x double> undef)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V4F64 = call <4 x double> @llvm.masked.expandload.v4f64(ptr undef, <4 x i1> %m4, <4 x double> undef)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V2F64 = call <2 x double> @llvm.masked.expandload.v2f64(ptr undef, <2 x i1> %m2, <2 x double> undef)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1F64 = call <1 x double> @llvm.masked.expandload.v1f64(ptr undef, <1 x i1> %m1, <1 x double> undef)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 31 for instruction: %V16F32 = call <16 x float> @llvm.masked.expandload.v16f32(ptr undef, <16 x i1> %m16, <16 x float> undef)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V8F32 = call <8 x float> @llvm.masked.expandload.v8f32(ptr undef, <8 x i1> %m8, <8 x float> undef)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V4F32 = call <4 x float> @llvm.masked.expandload.v4f32(ptr undef, <4 x i1> %m4, <4 x float> undef)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V2F32 = call <2 x float> @llvm.masked.expandload.v2f32(ptr undef, <2 x i1> %m2, <2 x float> undef)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 19 for instruction: %V8I64 = call <8 x i64> @llvm.masked.expandload.v8i64(ptr undef, <8 x i1> %m8, <8 x i64> undef)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V4I64 = call <4 x i64> @llvm.masked.expandload.v4i64(ptr undef, <4 x i1> %m4, <4 x i64> undef)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V2I64 = call <2 x i64> @llvm.masked.expandload.v2i64(ptr undef, <2 x i1> %m2, <2 x i64> undef)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1I64 = call <1 x i64> @llvm.masked.expandload.v1i64(ptr undef, <1 x i1> %m1, <1 x i64> undef)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 35 for instruction: %V16I32 = call <16 x i32> @llvm.masked.expandload.v16i32(ptr undef, <16 x i1> %m16, <16 x i32> undef)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V8I32 = call <8 x i32> @llvm.masked.expandload.v8i32(ptr undef, <8 x i1> %m8, <8 x i32> undef)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V4I32 = call <4 x i32> @llvm.masked.expandload.v4i32(ptr undef, <4 x i1> %m4, <4 x i32> undef)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V2I32 = call <2 x i32> @llvm.masked.expandload.v2i32(ptr undef, <2 x i1> %m2, <2 x i32> undef)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 67 for instruction: %V32I16 = call <32 x i16> @llvm.masked.expandload.v32i16(ptr undef, <32 x i1> %m32, <32 x i16> undef)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %V16I16 = call <16 x i16> @llvm.masked.expandload.v16i16(ptr undef, <16 x i1> %m16, <16 x i16> undef)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %V8I16 = call <8 x i16> @llvm.masked.expandload.v8i16(ptr undef, <8 x i1> %m8, <8 x i16> undef)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V4I16 = call <4 x i16> @llvm.masked.expandload.v4i16(ptr undef, <4 x i1> %m4, <4 x i16> undef)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 132 for instruction: %V64I8 = call <64 x i8> @llvm.masked.expandload.v64i8(ptr undef, <64 x i1> %m64, <64 x i8> undef)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 66 for instruction: %V32I8 = call <32 x i8> @llvm.masked.expandload.v32i8(ptr undef, <32 x i1> %m32, <32 x i8> undef)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %V16I8 = call <16 x i8> @llvm.masked.expandload.v16i8(ptr undef, <16 x i1> %m16, <16 x i8> undef)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %V8I8 = call <8 x i8> @llvm.masked.expandload.v8i8(ptr undef, <8 x i1> %m8, <8 x i8> undef)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %V8F64 = call <8 x double> @llvm.masked.expandload.v8f64(ptr undef, <8 x i1> %m8, <8 x double> undef)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %V4F64 = call <4 x double> @llvm.masked.expandload.v4f64(ptr undef, <4 x i1> %m4, <4 x double> undef)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V2F64 = call <2 x double> @llvm.masked.expandload.v2f64(ptr undef, <2 x i1> %m2, <2 x double> undef)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V1F64 = call <1 x double> @llvm.masked.expandload.v1f64(ptr undef, <1 x i1> %m1, <1 x double> undef)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 67 for instruction: %V16F32 = call <16 x float> @llvm.masked.expandload.v16f32(ptr undef, <16 x i1> %m16, <16 x float> undef)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %V8F32 = call <8 x float> @llvm.masked.expandload.v8f32(ptr undef, <8 x i1> %m8, <8 x float> undef)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %V4F32 = call <4 x float> @llvm.masked.expandload.v4f32(ptr undef, <4 x i1> %m4, <4 x float> undef)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V2F32 = call <2 x float> @llvm.masked.expandload.v2f32(ptr undef, <2 x i1> %m2, <2 x float> undef)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 37 for instruction: %V8I64 = call <8 x i64> @llvm.masked.expandload.v8i64(ptr undef, <8 x i1> %m8, <8 x i64> undef)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 19 for instruction: %V4I64 = call <4 x i64> @llvm.masked.expandload.v4i64(ptr undef, <4 x i1> %m4, <4 x i64> undef)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V2I64 = call <2 x i64> @llvm.masked.expandload.v2i64(ptr undef, <2 x i1> %m2, <2 x i64> undef)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V1I64 = call <1 x i64> @llvm.masked.expandload.v1i64(ptr undef, <1 x i1> %m1, <1 x i64> undef)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 71 for instruction: %V16I32 = call <16 x i32> @llvm.masked.expandload.v16i32(ptr undef, <16 x i1> %m16, <16 x i32> undef)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 36 for instruction: %V8I32 = call <8 x i32> @llvm.masked.expandload.v8i32(ptr undef, <8 x i1> %m8, <8 x i32> undef)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V4I32 = call <4 x i32> @llvm.masked.expandload.v4i32(ptr undef, <4 x i1> %m4, <4 x i32> undef)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V2I32 = call <2 x i32> @llvm.masked.expandload.v2i32(ptr undef, <2 x i1> %m2, <2 x i32> undef)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 139 for instruction: %V32I16 = call <32 x i16> @llvm.masked.expandload.v32i16(ptr undef, <32 x i1> %m32, <32 x i16> undef)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 70 for instruction: %V16I16 = call <16 x i16> @llvm.masked.expandload.v16i16(ptr undef, <16 x i1> %m16, <16 x i16> undef)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 35 for instruction: %V8I16 = call <8 x i16> @llvm.masked.expandload.v8i16(ptr undef, <8 x i1> %m8, <8 x i16> undef)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V4I16 = call <4 x i16> @llvm.masked.expandload.v4i16(ptr undef, <4 x i1> %m4, <4 x i16> undef)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 276 for instruction: %V64I8 = call <64 x i8> @llvm.masked.expandload.v64i8(ptr undef, <64 x i1> %m64, <64 x i8> undef)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 138 for instruction: %V32I8 = call <32 x i8> @llvm.masked.expandload.v32i8(ptr undef, <32 x i1> %m32, <32 x i8> undef)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 69 for instruction: %V16I8 = call <16 x i8> @llvm.masked.expandload.v16i8(ptr undef, <16 x i1> %m16, <16 x i8> undef)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 35 for instruction: %V8I8 = call <8 x i8> @llvm.masked.expandload.v8i8(ptr undef, <8 x i1> %m8, <8 x i8> undef)
 ; SKL-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 0
 ;
 ; AVX512-LABEL: 'masked_expandload'
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 23 for instruction: %V8F64 = call <8 x double> @llvm.masked.expandload.v8f64(ptr undef, <8 x i1> %m8, <8 x double> undef)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %V4F64 = call <4 x double> @llvm.masked.expandload.v4f64(ptr undef, <4 x i1> %m4, <4 x double> undef)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V2F64 = call <2 x double> @llvm.masked.expandload.v2f64(ptr undef, <2 x i1> %m2, <2 x double> undef)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1F64 = call <1 x double> @llvm.masked.expandload.v1f64(ptr undef, <1 x i1> %m1, <1 x double> undef)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 47 for instruction: %V16F32 = call <16 x float> @llvm.masked.expandload.v16f32(ptr undef, <16 x i1> %m16, <16 x float> undef)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 23 for instruction: %V8F32 = call <8 x float> @llvm.masked.expandload.v8f32(ptr undef, <8 x i1> %m8, <8 x float> undef)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %V4F32 = call <4 x float> @llvm.masked.expandload.v4f32(ptr undef, <4 x i1> %m4, <4 x float> undef)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V2F32 = call <2 x float> @llvm.masked.expandload.v2f32(ptr undef, <2 x i1> %m2, <2 x float> undef)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 27 for instruction: %V8I64 = call <8 x i64> @llvm.masked.expandload.v8i64(ptr undef, <8 x i1> %m8, <8 x i64> undef)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %V4I64 = call <4 x i64> @llvm.masked.expandload.v4i64(ptr undef, <4 x i1> %m4, <4 x i64> undef)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2I64 = call <2 x i64> @llvm.masked.expandload.v2i64(ptr undef, <2 x i1> %m2, <2 x i64> undef)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1I64 = call <1 x i64> @llvm.masked.expandload.v1i64(ptr undef, <1 x i1> %m1, <1 x i64> undef)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 51 for instruction: %V16I32 = call <16 x i32> @llvm.masked.expandload.v16i32(ptr undef, <16 x i1> %m16, <16 x i32> undef)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: %V8I32 = call <8 x i32> @llvm.masked.expandload.v8i32(ptr undef, <8 x i1> %m8, <8 x i32> undef)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V4I32 = call <4 x i32> @llvm.masked.expandload.v4i32(ptr undef, <4 x i1> %m4, <4 x i32> undef)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2I32 = call <2 x i32> @llvm.masked.expandload.v2i32(ptr undef, <2 x i1> %m2, <2 x i32> undef)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 99 for instruction: %V32I16 = call <32 x i16> @llvm.masked.expandload.v32i16(ptr undef, <32 x i1> %m32, <32 x i16> undef)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 49 for instruction: %V16I16 = call <16 x i16> @llvm.masked.expandload.v16i16(ptr undef, <16 x i1> %m16, <16 x i16> undef)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V8I16 = call <8 x i16> @llvm.masked.expandload.v8i16(ptr undef, <8 x i1> %m8, <8 x i16> undef)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V4I16 = call <4 x i16> @llvm.masked.expandload.v4i16(ptr undef, <4 x i1> %m4, <4 x i16> undef)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 195 for instruction: %V64I8 = call <64 x i8> @llvm.masked.expandload.v64i8(ptr undef, <64 x i1> %m64, <64 x i8> undef)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 97 for instruction: %V32I8 = call <32 x i8> @llvm.masked.expandload.v32i8(ptr undef, <32 x i1> %m32, <32 x i8> undef)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V16I8 = call <16 x i8> @llvm.masked.expandload.v16i8(ptr undef, <16 x i1> %m16, <16 x i8> undef)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V8I8 = call <8 x i8> @llvm.masked.expandload.v8i8(ptr undef, <8 x i1> %m8, <8 x i8> undef)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 42 for instruction: %V8F64 = call <8 x double> @llvm.masked.expandload.v8f64(ptr undef, <8 x i1> %m8, <8 x double> undef)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %V4F64 = call <4 x double> @llvm.masked.expandload.v4f64(ptr undef, <4 x i1> %m4, <4 x double> undef)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V2F64 = call <2 x double> @llvm.masked.expandload.v2f64(ptr undef, <2 x i1> %m2, <2 x double> undef)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V1F64 = call <1 x double> @llvm.masked.expandload.v1f64(ptr undef, <1 x i1> %m1, <1 x double> undef)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 85 for instruction: %V16F32 = call <16 x float> @llvm.masked.expandload.v16f32(ptr undef, <16 x i1> %m16, <16 x float> undef)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 42 for instruction: %V8F32 = call <8 x float> @llvm.masked.expandload.v8f32(ptr undef, <8 x i1> %m8, <8 x float> undef)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %V4F32 = call <4 x float> @llvm.masked.expandload.v4f32(ptr undef, <4 x i1> %m4, <4 x float> undef)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V2F32 = call <2 x float> @llvm.masked.expandload.v2f32(ptr undef, <2 x i1> %m2, <2 x float> undef)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 46 for instruction: %V8I64 = call <8 x i64> @llvm.masked.expandload.v8i64(ptr undef, <8 x i1> %m8, <8 x i64> undef)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 22 for instruction: %V4I64 = call <4 x i64> @llvm.masked.expandload.v4i64(ptr undef, <4 x i1> %m4, <4 x i64> undef)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V2I64 = call <2 x i64> @llvm.masked.expandload.v2i64(ptr undef, <2 x i1> %m2, <2 x i64> undef)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V1I64 = call <1 x i64> @llvm.masked.expandload.v1i64(ptr undef, <1 x i1> %m1, <1 x i64> undef)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 89 for instruction: %V16I32 = call <16 x i32> @llvm.masked.expandload.v16i32(ptr undef, <16 x i1> %m16, <16 x i32> undef)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 44 for instruction: %V8I32 = call <8 x i32> @llvm.masked.expandload.v8i32(ptr undef, <8 x i1> %m8, <8 x i32> undef)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 21 for instruction: %V4I32 = call <4 x i32> @llvm.masked.expandload.v4i32(ptr undef, <4 x i1> %m4, <4 x i32> undef)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V2I32 = call <2 x i32> @llvm.masked.expandload.v2i32(ptr undef, <2 x i1> %m2, <2 x i32> undef)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 175 for instruction: %V32I16 = call <32 x i16> @llvm.masked.expandload.v32i16(ptr undef, <32 x i1> %m32, <32 x i16> undef)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 87 for instruction: %V16I16 = call <16 x i16> @llvm.masked.expandload.v16i16(ptr undef, <16 x i1> %m16, <16 x i16> undef)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 43 for instruction: %V8I16 = call <8 x i16> @llvm.masked.expandload.v8i16(ptr undef, <8 x i1> %m8, <8 x i16> undef)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 21 for instruction: %V4I16 = call <4 x i16> @llvm.masked.expandload.v4i16(ptr undef, <4 x i1> %m4, <4 x i16> undef)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 347 for instruction: %V64I8 = call <64 x i8> @llvm.masked.expandload.v64i8(ptr undef, <64 x i1> %m64, <64 x i8> undef)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 173 for instruction: %V32I8 = call <32 x i8> @llvm.masked.expandload.v32i8(ptr undef, <32 x i1> %m32, <32 x i8> undef)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 86 for instruction: %V16I8 = call <16 x i8> @llvm.masked.expandload.v16i8(ptr undef, <16 x i1> %m16, <16 x i8> undef)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 43 for instruction: %V8I8 = call <8 x i8> @llvm.masked.expandload.v8i8(ptr undef, <8 x i1> %m8, <8 x i8> undef)
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 0
 ;
   %V8F64 = call <8 x double> @llvm.masked.expandload.v8f64(ptr undef, <8 x i1> %m8, <8 x double> undef)
@@ -1374,165 +1374,165 @@ define i32 @masked_expandload(<1 x i1> %m1, <2 x i1> %m2, <4 x i1> %m4, <8 x i1>
 
 define i32 @masked_compressstore(<1 x i1> %m1, <2 x i1> %m2, <4 x i1> %m4, <8 x i1> %m8, <16 x i1> %m16, <32 x i1> %m32, <64 x i1> %m64) {
 ; SSE2-LABEL: 'masked_compressstore'
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: call void @llvm.masked.compressstore.v8f64(<8 x double> undef, ptr undef, <8 x i1> %m8)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: call void @llvm.masked.compressstore.v4f64(<4 x double> undef, ptr undef, <4 x i1> %m4)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: call void @llvm.masked.compressstore.v2f64(<2 x double> undef, ptr undef, <2 x i1> %m2)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.compressstore.v1f64(<1 x double> undef, ptr undef, <1 x i1> %m1)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 29 for instruction: call void @llvm.masked.compressstore.v16f32(<16 x float> undef, ptr undef, <16 x i1> %m16)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: call void @llvm.masked.compressstore.v8f32(<8 x float> undef, ptr undef, <8 x i1> %m8)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: call void @llvm.masked.compressstore.v4f32(<4 x float> undef, ptr undef, <4 x i1> %m4)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: call void @llvm.masked.compressstore.v2f32(<2 x float> undef, ptr undef, <2 x i1> %m2)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 21 for instruction: call void @llvm.masked.compressstore.v8i64(<8 x i64> undef, ptr undef, <8 x i1> %m8)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: call void @llvm.masked.compressstore.v4i64(<4 x i64> undef, ptr undef, <4 x i1> %m4)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: call void @llvm.masked.compressstore.v2i64(<2 x i64> undef, ptr undef, <2 x i1> %m2)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.compressstore.v1i64(<1 x i64> undef, ptr undef, <1 x i1> %m1)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 45 for instruction: call void @llvm.masked.compressstore.v16i32(<16 x i32> undef, ptr undef, <16 x i1> %m16)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 23 for instruction: call void @llvm.masked.compressstore.v8i32(<8 x i32> undef, ptr undef, <8 x i1> %m8)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: call void @llvm.masked.compressstore.v4i32(<4 x i32> undef, ptr undef, <4 x i1> %m4)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: call void @llvm.masked.compressstore.v2i32(<2 x i32> undef, ptr undef, <2 x i1> %m2)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 66 for instruction: call void @llvm.masked.compressstore.v32i16(<32 x i16> undef, ptr undef, <32 x i1> %m32)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: call void @llvm.masked.compressstore.v16i16(<16 x i16> undef, ptr undef, <16 x i1> %m16)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: call void @llvm.masked.compressstore.v8i16(<8 x i16> undef, ptr undef, <8 x i1> %m8)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: call void @llvm.masked.compressstore.v4i16(<4 x i16> undef, ptr undef, <4 x i1> %m4)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 192 for instruction: call void @llvm.masked.compressstore.v64i8(<64 x i8> undef, ptr undef, <64 x i1> %m64)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 96 for instruction: call void @llvm.masked.compressstore.v32i8(<32 x i8> undef, ptr undef, <32 x i1> %m32)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: call void @llvm.masked.compressstore.v16i8(<16 x i8> undef, ptr undef, <16 x i1> %m16)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: call void @llvm.masked.compressstore.v8i8(<8 x i8> undef, ptr undef, <8 x i1> %m8)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 37 for instruction: call void @llvm.masked.compressstore.v8f64(<8 x double> undef, ptr undef, <8 x i1> %m8)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 19 for instruction: call void @llvm.masked.compressstore.v4f64(<4 x double> undef, ptr undef, <4 x i1> %m4)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: call void @llvm.masked.compressstore.v2f64(<2 x double> undef, ptr undef, <2 x i1> %m2)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: call void @llvm.masked.compressstore.v1f64(<1 x double> undef, ptr undef, <1 x i1> %m1)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 77 for instruction: call void @llvm.masked.compressstore.v16f32(<16 x float> undef, ptr undef, <16 x i1> %m16)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 39 for instruction: call void @llvm.masked.compressstore.v8f32(<8 x float> undef, ptr undef, <8 x i1> %m8)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: call void @llvm.masked.compressstore.v4f32(<4 x float> undef, ptr undef, <4 x i1> %m4)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: call void @llvm.masked.compressstore.v2f32(<2 x float> undef, ptr undef, <2 x i1> %m2)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 45 for instruction: call void @llvm.masked.compressstore.v8i64(<8 x i64> undef, ptr undef, <8 x i1> %m8)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 23 for instruction: call void @llvm.masked.compressstore.v4i64(<4 x i64> undef, ptr undef, <4 x i1> %m4)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: call void @llvm.masked.compressstore.v2i64(<2 x i64> undef, ptr undef, <2 x i1> %m2)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: call void @llvm.masked.compressstore.v1i64(<1 x i64> undef, ptr undef, <1 x i1> %m1)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 93 for instruction: call void @llvm.masked.compressstore.v16i32(<16 x i32> undef, ptr undef, <16 x i1> %m16)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 47 for instruction: call void @llvm.masked.compressstore.v8i32(<8 x i32> undef, ptr undef, <8 x i1> %m8)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: call void @llvm.masked.compressstore.v4i32(<4 x i32> undef, ptr undef, <4 x i1> %m4)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: call void @llvm.masked.compressstore.v2i32(<2 x i32> undef, ptr undef, <2 x i1> %m2)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 162 for instruction: call void @llvm.masked.compressstore.v32i16(<32 x i16> undef, ptr undef, <32 x i1> %m32)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 81 for instruction: call void @llvm.masked.compressstore.v16i16(<16 x i16> undef, ptr undef, <16 x i1> %m16)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 41 for instruction: call void @llvm.masked.compressstore.v8i16(<8 x i16> undef, ptr undef, <8 x i1> %m8)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 21 for instruction: call void @llvm.masked.compressstore.v4i16(<4 x i16> undef, ptr undef, <4 x i1> %m4)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 384 for instruction: call void @llvm.masked.compressstore.v64i8(<64 x i8> undef, ptr undef, <64 x i1> %m64)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 192 for instruction: call void @llvm.masked.compressstore.v32i8(<32 x i8> undef, ptr undef, <32 x i1> %m32)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 96 for instruction: call void @llvm.masked.compressstore.v16i8(<16 x i8> undef, ptr undef, <16 x i1> %m16)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: call void @llvm.masked.compressstore.v8i8(<8 x i8> undef, ptr undef, <8 x i1> %m8)
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 0
 ;
 ; SSE42-LABEL: 'masked_compressstore'
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: call void @llvm.masked.compressstore.v8f64(<8 x double> undef, ptr undef, <8 x i1> %m8)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: call void @llvm.masked.compressstore.v4f64(<4 x double> undef, ptr undef, <4 x i1> %m4)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: call void @llvm.masked.compressstore.v2f64(<2 x double> undef, ptr undef, <2 x i1> %m2)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.compressstore.v1f64(<1 x double> undef, ptr undef, <1 x i1> %m1)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 29 for instruction: call void @llvm.masked.compressstore.v16f32(<16 x float> undef, ptr undef, <16 x i1> %m16)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: call void @llvm.masked.compressstore.v8f32(<8 x float> undef, ptr undef, <8 x i1> %m8)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: call void @llvm.masked.compressstore.v4f32(<4 x float> undef, ptr undef, <4 x i1> %m4)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: call void @llvm.masked.compressstore.v2f32(<2 x float> undef, ptr undef, <2 x i1> %m2)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: call void @llvm.masked.compressstore.v8i64(<8 x i64> undef, ptr undef, <8 x i1> %m8)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: call void @llvm.masked.compressstore.v4i64(<4 x i64> undef, ptr undef, <4 x i1> %m4)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: call void @llvm.masked.compressstore.v2i64(<2 x i64> undef, ptr undef, <2 x i1> %m2)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.compressstore.v1i64(<1 x i64> undef, ptr undef, <1 x i1> %m1)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: call void @llvm.masked.compressstore.v16i32(<16 x i32> undef, ptr undef, <16 x i1> %m16)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: call void @llvm.masked.compressstore.v8i32(<8 x i32> undef, ptr undef, <8 x i1> %m8)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: call void @llvm.masked.compressstore.v4i32(<4 x i32> undef, ptr undef, <4 x i1> %m4)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: call void @llvm.masked.compressstore.v2i32(<2 x i32> undef, ptr undef, <2 x i1> %m2)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 66 for instruction: call void @llvm.masked.compressstore.v32i16(<32 x i16> undef, ptr undef, <32 x i1> %m32)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: call void @llvm.masked.compressstore.v16i16(<16 x i16> undef, ptr undef, <16 x i1> %m16)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: call void @llvm.masked.compressstore.v8i16(<8 x i16> undef, ptr undef, <8 x i1> %m8)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: call void @llvm.masked.compressstore.v4i16(<4 x i16> undef, ptr undef, <4 x i1> %m4)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 132 for instruction: call void @llvm.masked.compressstore.v64i8(<64 x i8> undef, ptr undef, <64 x i1> %m64)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 66 for instruction: call void @llvm.masked.compressstore.v32i8(<32 x i8> undef, ptr undef, <32 x i1> %m32)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: call void @llvm.masked.compressstore.v16i8(<16 x i8> undef, ptr undef, <16 x i1> %m16)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: call void @llvm.masked.compressstore.v8i8(<8 x i8> undef, ptr undef, <8 x i1> %m8)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 29 for instruction: call void @llvm.masked.compressstore.v8f64(<8 x double> undef, ptr undef, <8 x i1> %m8)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: call void @llvm.masked.compressstore.v4f64(<4 x double> undef, ptr undef, <4 x i1> %m4)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: call void @llvm.masked.compressstore.v2f64(<2 x double> undef, ptr undef, <2 x i1> %m2)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: call void @llvm.masked.compressstore.v1f64(<1 x double> undef, ptr undef, <1 x i1> %m1)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 61 for instruction: call void @llvm.masked.compressstore.v16f32(<16 x float> undef, ptr undef, <16 x i1> %m16)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 31 for instruction: call void @llvm.masked.compressstore.v8f32(<8 x float> undef, ptr undef, <8 x i1> %m8)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: call void @llvm.masked.compressstore.v4f32(<4 x float> undef, ptr undef, <4 x i1> %m4)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: call void @llvm.masked.compressstore.v2f32(<2 x float> undef, ptr undef, <2 x i1> %m2)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: call void @llvm.masked.compressstore.v8i64(<8 x i64> undef, ptr undef, <8 x i1> %m8)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: call void @llvm.masked.compressstore.v4i64(<4 x i64> undef, ptr undef, <4 x i1> %m4)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: call void @llvm.masked.compressstore.v2i64(<2 x i64> undef, ptr undef, <2 x i1> %m2)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: call void @llvm.masked.compressstore.v1i64(<1 x i64> undef, ptr undef, <1 x i1> %m1)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 65 for instruction: call void @llvm.masked.compressstore.v16i32(<16 x i32> undef, ptr undef, <16 x i1> %m16)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: call void @llvm.masked.compressstore.v8i32(<8 x i32> undef, ptr undef, <8 x i1> %m8)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: call void @llvm.masked.compressstore.v4i32(<4 x i32> undef, ptr undef, <4 x i1> %m4)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: call void @llvm.masked.compressstore.v2i32(<2 x i32> undef, ptr undef, <2 x i1> %m2)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 130 for instruction: call void @llvm.masked.compressstore.v32i16(<32 x i16> undef, ptr undef, <32 x i1> %m32)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 65 for instruction: call void @llvm.masked.compressstore.v16i16(<16 x i16> undef, ptr undef, <16 x i1> %m16)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: call void @llvm.masked.compressstore.v8i16(<8 x i16> undef, ptr undef, <8 x i1> %m8)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: call void @llvm.masked.compressstore.v4i16(<4 x i16> undef, ptr undef, <4 x i1> %m4)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 260 for instruction: call void @llvm.masked.compressstore.v64i8(<64 x i8> undef, ptr undef, <64 x i1> %m64)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 130 for instruction: call void @llvm.masked.compressstore.v32i8(<32 x i8> undef, ptr undef, <32 x i1> %m32)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 65 for instruction: call void @llvm.masked.compressstore.v16i8(<16 x i8> undef, ptr undef, <16 x i1> %m16)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: call void @llvm.masked.compressstore.v8i8(<8 x i8> undef, ptr undef, <8 x i1> %m8)
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 0
 ;
 ; AVX1-LABEL: 'masked_compressstore'
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: call void @llvm.masked.compressstore.v8f64(<8 x double> undef, ptr undef, <8 x i1> %m8)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: call void @llvm.masked.compressstore.v4f64(<4 x double> undef, ptr undef, <4 x i1> %m4)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: call void @llvm.masked.compressstore.v2f64(<2 x double> undef, ptr undef, <2 x i1> %m2)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.compressstore.v1f64(<1 x double> undef, ptr undef, <1 x i1> %m1)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 31 for instruction: call void @llvm.masked.compressstore.v16f32(<16 x float> undef, ptr undef, <16 x i1> %m16)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: call void @llvm.masked.compressstore.v8f32(<8 x float> undef, ptr undef, <8 x i1> %m8)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: call void @llvm.masked.compressstore.v4f32(<4 x float> undef, ptr undef, <4 x i1> %m4)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: call void @llvm.masked.compressstore.v2f32(<2 x float> undef, ptr undef, <2 x i1> %m2)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 19 for instruction: call void @llvm.masked.compressstore.v8i64(<8 x i64> undef, ptr undef, <8 x i1> %m8)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: call void @llvm.masked.compressstore.v4i64(<4 x i64> undef, ptr undef, <4 x i1> %m4)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: call void @llvm.masked.compressstore.v2i64(<2 x i64> undef, ptr undef, <2 x i1> %m2)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.compressstore.v1i64(<1 x i64> undef, ptr undef, <1 x i1> %m1)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 35 for instruction: call void @llvm.masked.compressstore.v16i32(<16 x i32> undef, ptr undef, <16 x i1> %m16)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: call void @llvm.masked.compressstore.v8i32(<8 x i32> undef, ptr undef, <8 x i1> %m8)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: call void @llvm.masked.compressstore.v4i32(<4 x i32> undef, ptr undef, <4 x i1> %m4)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: call void @llvm.masked.compressstore.v2i32(<2 x i32> undef, ptr undef, <2 x i1> %m2)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 68 for instruction: call void @llvm.masked.compressstore.v32i16(<32 x i16> undef, ptr undef, <32 x i1> %m32)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: call void @llvm.masked.compressstore.v16i16(<16 x i16> undef, ptr undef, <16 x i1> %m16)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: call void @llvm.masked.compressstore.v8i16(<8 x i16> undef, ptr undef, <8 x i1> %m8)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: call void @llvm.masked.compressstore.v4i16(<4 x i16> undef, ptr undef, <4 x i1> %m4)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 134 for instruction: call void @llvm.masked.compressstore.v64i8(<64 x i8> undef, ptr undef, <64 x i1> %m64)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 67 for instruction: call void @llvm.masked.compressstore.v32i8(<32 x i8> undef, ptr undef, <32 x i1> %m32)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: call void @llvm.masked.compressstore.v16i8(<16 x i8> undef, ptr undef, <16 x i1> %m16)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: call void @llvm.masked.compressstore.v8i8(<8 x i8> undef, ptr undef, <8 x i1> %m8)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: call void @llvm.masked.compressstore.v8f64(<8 x double> undef, ptr undef, <8 x i1> %m8)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: call void @llvm.masked.compressstore.v4f64(<4 x double> undef, ptr undef, <4 x i1> %m4)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: call void @llvm.masked.compressstore.v2f64(<2 x double> undef, ptr undef, <2 x i1> %m2)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: call void @llvm.masked.compressstore.v1f64(<1 x double> undef, ptr undef, <1 x i1> %m1)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 67 for instruction: call void @llvm.masked.compressstore.v16f32(<16 x float> undef, ptr undef, <16 x i1> %m16)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: call void @llvm.masked.compressstore.v8f32(<8 x float> undef, ptr undef, <8 x i1> %m8)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: call void @llvm.masked.compressstore.v4f32(<4 x float> undef, ptr undef, <4 x i1> %m4)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: call void @llvm.masked.compressstore.v2f32(<2 x float> undef, ptr undef, <2 x i1> %m2)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 37 for instruction: call void @llvm.masked.compressstore.v8i64(<8 x i64> undef, ptr undef, <8 x i1> %m8)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 19 for instruction: call void @llvm.masked.compressstore.v4i64(<4 x i64> undef, ptr undef, <4 x i1> %m4)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: call void @llvm.masked.compressstore.v2i64(<2 x i64> undef, ptr undef, <2 x i1> %m2)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: call void @llvm.masked.compressstore.v1i64(<1 x i64> undef, ptr undef, <1 x i1> %m1)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 71 for instruction: call void @llvm.masked.compressstore.v16i32(<16 x i32> undef, ptr undef, <16 x i1> %m16)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 36 for instruction: call void @llvm.masked.compressstore.v8i32(<8 x i32> undef, ptr undef, <8 x i1> %m8)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: call void @llvm.masked.compressstore.v4i32(<4 x i32> undef, ptr undef, <4 x i1> %m4)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: call void @llvm.masked.compressstore.v2i32(<2 x i32> undef, ptr undef, <2 x i1> %m2)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 140 for instruction: call void @llvm.masked.compressstore.v32i16(<32 x i16> undef, ptr undef, <32 x i1> %m32)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 70 for instruction: call void @llvm.masked.compressstore.v16i16(<16 x i16> undef, ptr undef, <16 x i1> %m16)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 35 for instruction: call void @llvm.masked.compressstore.v8i16(<8 x i16> undef, ptr undef, <8 x i1> %m8)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: call void @llvm.masked.compressstore.v4i16(<4 x i16> undef, ptr undef, <4 x i1> %m4)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 278 for instruction: call void @llvm.masked.compressstore.v64i8(<64 x i8> undef, ptr undef, <64 x i1> %m64)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 139 for instruction: call void @llvm.masked.compressstore.v32i8(<32 x i8> undef, ptr undef, <32 x i1> %m32)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 69 for instruction: call void @llvm.masked.compressstore.v16i8(<16 x i8> undef, ptr undef, <16 x i1> %m16)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 35 for instruction: call void @llvm.masked.compressstore.v8i8(<8 x i8> undef, ptr undef, <8 x i1> %m8)
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 0
 ;
 ; AVX2-LABEL: 'masked_compressstore'
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: call void @llvm.masked.compressstore.v8f64(<8 x double> undef, ptr undef, <8 x i1> %m8)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: call void @llvm.masked.compressstore.v4f64(<4 x double> undef, ptr undef, <4 x i1> %m4)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: call void @llvm.masked.compressstore.v2f64(<2 x double> undef, ptr undef, <2 x i1> %m2)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.compressstore.v1f64(<1 x double> undef, ptr undef, <1 x i1> %m1)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 31 for instruction: call void @llvm.masked.compressstore.v16f32(<16 x float> undef, ptr undef, <16 x i1> %m16)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: call void @llvm.masked.compressstore.v8f32(<8 x float> undef, ptr undef, <8 x i1> %m8)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: call void @llvm.masked.compressstore.v4f32(<4 x float> undef, ptr undef, <4 x i1> %m4)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: call void @llvm.masked.compressstore.v2f32(<2 x float> undef, ptr undef, <2 x i1> %m2)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 19 for instruction: call void @llvm.masked.compressstore.v8i64(<8 x i64> undef, ptr undef, <8 x i1> %m8)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: call void @llvm.masked.compressstore.v4i64(<4 x i64> undef, ptr undef, <4 x i1> %m4)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: call void @llvm.masked.compressstore.v2i64(<2 x i64> undef, ptr undef, <2 x i1> %m2)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.compressstore.v1i64(<1 x i64> undef, ptr undef, <1 x i1> %m1)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 35 for instruction: call void @llvm.masked.compressstore.v16i32(<16 x i32> undef, ptr undef, <16 x i1> %m16)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: call void @llvm.masked.compressstore.v8i32(<8 x i32> undef, ptr undef, <8 x i1> %m8)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: call void @llvm.masked.compressstore.v4i32(<4 x i32> undef, ptr undef, <4 x i1> %m4)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: call void @llvm.masked.compressstore.v2i32(<2 x i32> undef, ptr undef, <2 x i1> %m2)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 67 for instruction: call void @llvm.masked.compressstore.v32i16(<32 x i16> undef, ptr undef, <32 x i1> %m32)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: call void @llvm.masked.compressstore.v16i16(<16 x i16> undef, ptr undef, <16 x i1> %m16)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: call void @llvm.masked.compressstore.v8i16(<8 x i16> undef, ptr undef, <8 x i1> %m8)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: call void @llvm.masked.compressstore.v4i16(<4 x i16> undef, ptr undef, <4 x i1> %m4)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 132 for instruction: call void @llvm.masked.compressstore.v64i8(<64 x i8> undef, ptr undef, <64 x i1> %m64)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 66 for instruction: call void @llvm.masked.compressstore.v32i8(<32 x i8> undef, ptr undef, <32 x i1> %m32)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: call void @llvm.masked.compressstore.v16i8(<16 x i8> undef, ptr undef, <16 x i1> %m16)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: call void @llvm.masked.compressstore.v8i8(<8 x i8> undef, ptr undef, <8 x i1> %m8)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: call void @llvm.masked.compressstore.v8f64(<8 x double> undef, ptr undef, <8 x i1> %m8)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: call void @llvm.masked.compressstore.v4f64(<4 x double> undef, ptr undef, <4 x i1> %m4)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: call void @llvm.masked.compressstore.v2f64(<2 x double> undef, ptr undef, <2 x i1> %m2)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: call void @llvm.masked.compressstore.v1f64(<1 x double> undef, ptr undef, <1 x i1> %m1)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 67 for instruction: call void @llvm.masked.compressstore.v16f32(<16 x float> undef, ptr undef, <16 x i1> %m16)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: call void @llvm.masked.compressstore.v8f32(<8 x float> undef, ptr undef, <8 x i1> %m8)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: call void @llvm.masked.compressstore.v4f32(<4 x float> undef, ptr undef, <4 x i1> %m4)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: call void @llvm.masked.compressstore.v2f32(<2 x float> undef, ptr undef, <2 x i1> %m2)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 37 for instruction: call void @llvm.masked.compressstore.v8i64(<8 x i64> undef, ptr undef, <8 x i1> %m8)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 19 for instruction: call void @llvm.masked.compressstore.v4i64(<4 x i64> undef, ptr undef, <4 x i1> %m4)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: call void @llvm.masked.compressstore.v2i64(<2 x i64> undef, ptr undef, <2 x i1> %m2)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: call void @llvm.masked.compressstore.v1i64(<1 x i64> undef, ptr undef, <1 x i1> %m1)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 71 for instruction: call void @llvm.masked.compressstore.v16i32(<16 x i32> undef, ptr undef, <16 x i1> %m16)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 36 for instruction: call void @llvm.masked.compressstore.v8i32(<8 x i32> undef, ptr undef, <8 x i1> %m8)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: call void @llvm.masked.compressstore.v4i32(<4 x i32> undef, ptr undef, <4 x i1> %m4)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: call void @llvm.masked.compressstore.v2i32(<2 x i32> undef, ptr undef, <2 x i1> %m2)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 139 for instruction: call void @llvm.masked.compressstore.v32i16(<32 x i16> undef, ptr undef, <32 x i1> %m32)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 70 for instruction: call void @llvm.masked.compressstore.v16i16(<16 x i16> undef, ptr undef, <16 x i1> %m16)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 35 for instruction: call void @llvm.masked.compressstore.v8i16(<8 x i16> undef, ptr undef, <8 x i1> %m8)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: call void @llvm.masked.compressstore.v4i16(<4 x i16> undef, ptr undef, <4 x i1> %m4)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 276 for instruction: call void @llvm.masked.compressstore.v64i8(<64 x i8> undef, ptr undef, <64 x i1> %m64)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 138 for instruction: call void @llvm.masked.compressstore.v32i8(<32 x i8> undef, ptr undef, <32 x i1> %m32)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 69 for instruction: call void @llvm.masked.compressstore.v16i8(<16 x i8> undef, ptr undef, <16 x i1> %m16)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 35 for instruction: call void @llvm.masked.compressstore.v8i8(<8 x i8> undef, ptr undef, <8 x i1> %m8)
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 0
 ;
 ; SKL-LABEL: 'masked_compressstore'
-; SKL-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: call void @llvm.masked.compressstore.v8f64(<8 x double> undef, ptr undef, <8 x i1> %m8)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: call void @llvm.masked.compressstore.v4f64(<4 x double> undef, ptr undef, <4 x i1> %m4)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: call void @llvm.masked.compressstore.v2f64(<2 x double> undef, ptr undef, <2 x i1> %m2)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.compressstore.v1f64(<1 x double> undef, ptr undef, <1 x i1> %m1)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 31 for instruction: call void @llvm.masked.compressstore.v16f32(<16 x float> undef, ptr undef, <16 x i1> %m16)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: call void @llvm.masked.compressstore.v8f32(<8 x float> undef, ptr undef, <8 x i1> %m8)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: call void @llvm.masked.compressstore.v4f32(<4 x float> undef, ptr undef, <4 x i1> %m4)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: call void @llvm.masked.compressstore.v2f32(<2 x float> undef, ptr undef, <2 x i1> %m2)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 19 for instruction: call void @llvm.masked.compressstore.v8i64(<8 x i64> undef, ptr undef, <8 x i1> %m8)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: call void @llvm.masked.compressstore.v4i64(<4 x i64> undef, ptr undef, <4 x i1> %m4)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: call void @llvm.masked.compressstore.v2i64(<2 x i64> undef, ptr undef, <2 x i1> %m2)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.compressstore.v1i64(<1 x i64> undef, ptr undef, <1 x i1> %m1)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 35 for instruction: call void @llvm.masked.compressstore.v16i32(<16 x i32> undef, ptr undef, <16 x i1> %m16)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: call void @llvm.masked.compressstore.v8i32(<8 x i32> undef, ptr undef, <8 x i1> %m8)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: call void @llvm.masked.compressstore.v4i32(<4 x i32> undef, ptr undef, <4 x i1> %m4)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: call void @llvm.masked.compressstore.v2i32(<2 x i32> undef, ptr undef, <2 x i1> %m2)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 67 for instruction: call void @llvm.masked.compressstore.v32i16(<32 x i16> undef, ptr undef, <32 x i1> %m32)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: call void @llvm.masked.compressstore.v16i16(<16 x i16> undef, ptr undef, <16 x i1> %m16)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: call void @llvm.masked.compressstore.v8i16(<8 x i16> undef, ptr undef, <8 x i1> %m8)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: call void @llvm.masked.compressstore.v4i16(<4 x i16> undef, ptr undef, <4 x i1> %m4)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 132 for instruction: call void @llvm.masked.compressstore.v64i8(<64 x i8> undef, ptr undef, <64 x i1> %m64)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 66 for instruction: call void @llvm.masked.compressstore.v32i8(<32 x i8> undef, ptr undef, <32 x i1> %m32)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: call void @llvm.masked.compressstore.v16i8(<16 x i8> undef, ptr undef, <16 x i1> %m16)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: call void @llvm.masked.compressstore.v8i8(<8 x i8> undef, ptr undef, <8 x i1> %m8)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: call void @llvm.masked.compressstore.v8f64(<8 x double> undef, ptr undef, <8 x i1> %m8)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: call void @llvm.masked.compressstore.v4f64(<4 x double> undef, ptr undef, <4 x i1> %m4)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: call void @llvm.masked.compressstore.v2f64(<2 x double> undef, ptr undef, <2 x i1> %m2)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: call void @llvm.masked.compressstore.v1f64(<1 x double> undef, ptr undef, <1 x i1> %m1)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 67 for instruction: call void @llvm.masked.compressstore.v16f32(<16 x float> undef, ptr undef, <16 x i1> %m16)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: call void @llvm.masked.compressstore.v8f32(<8 x float> undef, ptr undef, <8 x i1> %m8)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: call void @llvm.masked.compressstore.v4f32(<4 x float> undef, ptr undef, <4 x i1> %m4)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: call void @llvm.masked.compressstore.v2f32(<2 x float> undef, ptr undef, <2 x i1> %m2)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 37 for instruction: call void @llvm.masked.compressstore.v8i64(<8 x i64> undef, ptr undef, <8 x i1> %m8)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 19 for instruction: call void @llvm.masked.compressstore.v4i64(<4 x i64> undef, ptr undef, <4 x i1> %m4)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: call void @llvm.masked.compressstore.v2i64(<2 x i64> undef, ptr undef, <2 x i1> %m2)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: call void @llvm.masked.compressstore.v1i64(<1 x i64> undef, ptr undef, <1 x i1> %m1)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 71 for instruction: call void @llvm.masked.compressstore.v16i32(<16 x i32> undef, ptr undef, <16 x i1> %m16)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 36 for instruction: call void @llvm.masked.compressstore.v8i32(<8 x i32> undef, ptr undef, <8 x i1> %m8)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: call void @llvm.masked.compressstore.v4i32(<4 x i32> undef, ptr undef, <4 x i1> %m4)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: call void @llvm.masked.compressstore.v2i32(<2 x i32> undef, ptr undef, <2 x i1> %m2)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 139 for instruction: call void @llvm.masked.compressstore.v32i16(<32 x i16> undef, ptr undef, <32 x i1> %m32)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 70 for instruction: call void @llvm.masked.compressstore.v16i16(<16 x i16> undef, ptr undef, <16 x i1> %m16)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 35 for instruction: call void @llvm.masked.compressstore.v8i16(<8 x i16> undef, ptr undef, <8 x i1> %m8)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: call void @llvm.masked.compressstore.v4i16(<4 x i16> undef, ptr undef, <4 x i1> %m4)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 276 for instruction: call void @llvm.masked.compressstore.v64i8(<64 x i8> undef, ptr undef, <64 x i1> %m64)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 138 for instruction: call void @llvm.masked.compressstore.v32i8(<32 x i8> undef, ptr undef, <32 x i1> %m32)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 69 for instruction: call void @llvm.masked.compressstore.v16i8(<16 x i8> undef, ptr undef, <16 x i1> %m16)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 35 for instruction: call void @llvm.masked.compressstore.v8i8(<8 x i8> undef, ptr undef, <8 x i1> %m8)
 ; SKL-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 0
 ;
 ; AVX512-LABEL: 'masked_compressstore'
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 23 for instruction: call void @llvm.masked.compressstore.v8f64(<8 x double> undef, ptr undef, <8 x i1> %m8)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: call void @llvm.masked.compressstore.v4f64(<4 x double> undef, ptr undef, <4 x i1> %m4)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: call void @llvm.masked.compressstore.v2f64(<2 x double> undef, ptr undef, <2 x i1> %m2)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.compressstore.v1f64(<1 x double> undef, ptr undef, <1 x i1> %m1)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 47 for instruction: call void @llvm.masked.compressstore.v16f32(<16 x float> undef, ptr undef, <16 x i1> %m16)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 23 for instruction: call void @llvm.masked.compressstore.v8f32(<8 x float> undef, ptr undef, <8 x i1> %m8)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: call void @llvm.masked.compressstore.v4f32(<4 x float> undef, ptr undef, <4 x i1> %m4)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: call void @llvm.masked.compressstore.v2f32(<2 x float> undef, ptr undef, <2 x i1> %m2)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 27 for instruction: call void @llvm.masked.compressstore.v8i64(<8 x i64> undef, ptr undef, <8 x i1> %m8)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: call void @llvm.masked.compressstore.v4i64(<4 x i64> undef, ptr undef, <4 x i1> %m4)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: call void @llvm.masked.compressstore.v2i64(<2 x i64> undef, ptr undef, <2 x i1> %m2)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.compressstore.v1i64(<1 x i64> undef, ptr undef, <1 x i1> %m1)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 51 for instruction: call void @llvm.masked.compressstore.v16i32(<16 x i32> undef, ptr undef, <16 x i1> %m16)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: call void @llvm.masked.compressstore.v8i32(<8 x i32> undef, ptr undef, <8 x i1> %m8)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: call void @llvm.masked.compressstore.v4i32(<4 x i32> undef, ptr undef, <4 x i1> %m4)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: call void @llvm.masked.compressstore.v2i32(<2 x i32> undef, ptr undef, <2 x i1> %m2)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 99 for instruction: call void @llvm.masked.compressstore.v32i16(<32 x i16> undef, ptr undef, <32 x i1> %m32)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 49 for instruction: call void @llvm.masked.compressstore.v16i16(<16 x i16> undef, ptr undef, <16 x i1> %m16)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: call void @llvm.masked.compressstore.v8i16(<8 x i16> undef, ptr undef, <8 x i1> %m8)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: call void @llvm.masked.compressstore.v4i16(<4 x i16> undef, ptr undef, <4 x i1> %m4)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 195 for instruction: call void @llvm.masked.compressstore.v64i8(<64 x i8> undef, ptr undef, <64 x i1> %m64)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 97 for instruction: call void @llvm.masked.compressstore.v32i8(<32 x i8> undef, ptr undef, <32 x i1> %m32)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: call void @llvm.masked.compressstore.v16i8(<16 x i8> undef, ptr undef, <16 x i1> %m16)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: call void @llvm.masked.compressstore.v8i8(<8 x i8> undef, ptr undef, <8 x i1> %m8)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 42 for instruction: call void @llvm.masked.compressstore.v8f64(<8 x double> undef, ptr undef, <8 x i1> %m8)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: call void @llvm.masked.compressstore.v4f64(<4 x double> undef, ptr undef, <4 x i1> %m4)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: call void @llvm.masked.compressstore.v2f64(<2 x double> undef, ptr undef, <2 x i1> %m2)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: call void @llvm.masked.compressstore.v1f64(<1 x double> undef, ptr undef, <1 x i1> %m1)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 85 for instruction: call void @llvm.masked.compressstore.v16f32(<16 x float> undef, ptr undef, <16 x i1> %m16)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 42 for instruction: call void @llvm.masked.compressstore.v8f32(<8 x float> undef, ptr undef, <8 x i1> %m8)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: call void @llvm.masked.compressstore.v4f32(<4 x float> undef, ptr undef, <4 x i1> %m4)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: call void @llvm.masked.compressstore.v2f32(<2 x float> undef, ptr undef, <2 x i1> %m2)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 46 for instruction: call void @llvm.masked.compressstore.v8i64(<8 x i64> undef, ptr undef, <8 x i1> %m8)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 22 for instruction: call void @llvm.masked.compressstore.v4i64(<4 x i64> undef, ptr undef, <4 x i1> %m4)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: call void @llvm.masked.compressstore.v2i64(<2 x i64> undef, ptr undef, <2 x i1> %m2)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: call void @llvm.masked.compressstore.v1i64(<1 x i64> undef, ptr undef, <1 x i1> %m1)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 89 for instruction: call void @llvm.masked.compressstore.v16i32(<16 x i32> undef, ptr undef, <16 x i1> %m16)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 44 for instruction: call void @llvm.masked.compressstore.v8i32(<8 x i32> undef, ptr undef, <8 x i1> %m8)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 21 for instruction: call void @llvm.masked.compressstore.v4i32(<4 x i32> undef, ptr undef, <4 x i1> %m4)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: call void @llvm.masked.compressstore.v2i32(<2 x i32> undef, ptr undef, <2 x i1> %m2)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 175 for instruction: call void @llvm.masked.compressstore.v32i16(<32 x i16> undef, ptr undef, <32 x i1> %m32)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 87 for instruction: call void @llvm.masked.compressstore.v16i16(<16 x i16> undef, ptr undef, <16 x i1> %m16)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 43 for instruction: call void @llvm.masked.compressstore.v8i16(<8 x i16> undef, ptr undef, <8 x i1> %m8)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 21 for instruction: call void @llvm.masked.compressstore.v4i16(<4 x i16> undef, ptr undef, <4 x i1> %m4)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 347 for instruction: call void @llvm.masked.compressstore.v64i8(<64 x i8> undef, ptr undef, <64 x i1> %m64)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 173 for instruction: call void @llvm.masked.compressstore.v32i8(<32 x i8> undef, ptr undef, <32 x i1> %m32)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 86 for instruction: call void @llvm.masked.compressstore.v16i8(<16 x i8> undef, ptr undef, <16 x i1> %m16)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 43 for instruction: call void @llvm.masked.compressstore.v8i8(<8 x i8> undef, ptr undef, <8 x i1> %m8)
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 0
 ;
   call void @llvm.masked.compressstore.v8f64(<8 x double> undef, ptr undef, <8 x i1> %m8)
diff --git a/llvm/test/Analysis/CostModel/X86/masked-intrinsic-cost-inseltpoison.ll b/llvm/test/Analysis/CostModel/X86/masked-intrinsic-cost-inseltpoison.ll
index e18a4d0467caf..41f0287079b95 100644
--- a/llvm/test/Analysis/CostModel/X86/masked-intrinsic-cost-inseltpoison.ll
+++ b/llvm/test/Analysis/CostModel/X86/masked-intrinsic-cost-inseltpoison.ll
@@ -1178,165 +1178,165 @@ define i32 @masked_scatter(<1 x i1> %m1, <2 x i1> %m2, <4 x i1> %m4, <8 x i1> %m
 
 define i32 @masked_expandload(<1 x i1> %m1, <2 x i1> %m2, <4 x i1> %m4, <8 x i1> %m8, <16 x i1> %m16, <32 x i1> %m32, <64 x i1> %m64) {
 ; SSE2-LABEL: 'masked_expandload'
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %V8F64 = call <8 x double> @llvm.masked.expandload.v8f64(ptr undef, <8 x i1> %m8, <8 x double> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V4F64 = call <4 x double> @llvm.masked.expandload.v4f64(ptr undef, <4 x i1> %m4, <4 x double> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V2F64 = call <2 x double> @llvm.masked.expandload.v2f64(ptr undef, <2 x i1> %m2, <2 x double> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1F64 = call <1 x double> @llvm.masked.expandload.v1f64(ptr undef, <1 x i1> %m1, <1 x double> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 29 for instruction: %V16F32 = call <16 x float> @llvm.masked.expandload.v16f32(ptr undef, <16 x i1> %m16, <16 x float> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %V8F32 = call <8 x float> @llvm.masked.expandload.v8f32(ptr undef, <8 x i1> %m8, <8 x float> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V4F32 = call <4 x float> @llvm.masked.expandload.v4f32(ptr undef, <4 x i1> %m4, <4 x float> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V2F32 = call <2 x float> @llvm.masked.expandload.v2f32(ptr undef, <2 x i1> %m2, <2 x float> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 21 for instruction: %V8I64 = call <8 x i64> @llvm.masked.expandload.v8i64(ptr undef, <8 x i1> %m8, <8 x i64> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %V4I64 = call <4 x i64> @llvm.masked.expandload.v4i64(ptr undef, <4 x i1> %m4, <4 x i64> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2I64 = call <2 x i64> @llvm.masked.expandload.v2i64(ptr undef, <2 x i1> %m2, <2 x i64> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1I64 = call <1 x i64> @llvm.masked.expandload.v1i64(ptr undef, <1 x i1> %m1, <1 x i64> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 45 for instruction: %V16I32 = call <16 x i32> @llvm.masked.expandload.v16i32(ptr undef, <16 x i1> %m16, <16 x i32> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 23 for instruction: %V8I32 = call <8 x i32> @llvm.masked.expandload.v8i32(ptr undef, <8 x i1> %m8, <8 x i32> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V4I32 = call <4 x i32> @llvm.masked.expandload.v4i32(ptr undef, <4 x i1> %m4, <4 x i32> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2I32 = call <2 x i32> @llvm.masked.expandload.v2i32(ptr undef, <2 x i1> %m2, <2 x i32> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 66 for instruction: %V32I16 = call <32 x i16> @llvm.masked.expandload.v32i16(ptr undef, <32 x i1> %m32, <32 x i16> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %V16I16 = call <16 x i16> @llvm.masked.expandload.v16i16(ptr undef, <16 x i1> %m16, <16 x i16> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %V8I16 = call <8 x i16> @llvm.masked.expandload.v8i16(ptr undef, <8 x i1> %m8, <8 x i16> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V4I16 = call <4 x i16> @llvm.masked.expandload.v4i16(ptr undef, <4 x i1> %m4, <4 x i16> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 192 for instruction: %V64I8 = call <64 x i8> @llvm.masked.expandload.v64i8(ptr undef, <64 x i1> %m64, <64 x i8> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 96 for instruction: %V32I8 = call <32 x i8> @llvm.masked.expandload.v32i8(ptr undef, <32 x i1> %m32, <32 x i8> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V16I8 = call <16 x i8> @llvm.masked.expandload.v16i8(ptr undef, <16 x i1> %m16, <16 x i8> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V8I8 = call <8 x i8> @llvm.masked.expandload.v8i8(ptr undef, <8 x i1> %m8, <8 x i8> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 29 for instruction: %V8F64 = call <8 x double> @llvm.masked.expandload.v8f64(ptr undef, <8 x i1> %m8, <8 x double> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %V4F64 = call <4 x double> @llvm.masked.expandload.v4f64(ptr undef, <4 x i1> %m4, <4 x double> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V2F64 = call <2 x double> @llvm.masked.expandload.v2f64(ptr undef, <2 x i1> %m2, <2 x double> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V1F64 = call <1 x double> @llvm.masked.expandload.v1f64(ptr undef, <1 x i1> %m1, <1 x double> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 61 for instruction: %V16F32 = call <16 x float> @llvm.masked.expandload.v16f32(ptr undef, <16 x i1> %m16, <16 x float> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 31 for instruction: %V8F32 = call <8 x float> @llvm.masked.expandload.v8f32(ptr undef, <8 x i1> %m8, <8 x float> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V4F32 = call <4 x float> @llvm.masked.expandload.v4f32(ptr undef, <4 x i1> %m4, <4 x float> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V2F32 = call <2 x float> @llvm.masked.expandload.v2f32(ptr undef, <2 x i1> %m2, <2 x float> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 37 for instruction: %V8I64 = call <8 x i64> @llvm.masked.expandload.v8i64(ptr undef, <8 x i1> %m8, <8 x i64> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 19 for instruction: %V4I64 = call <4 x i64> @llvm.masked.expandload.v4i64(ptr undef, <4 x i1> %m4, <4 x i64> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V2I64 = call <2 x i64> @llvm.masked.expandload.v2i64(ptr undef, <2 x i1> %m2, <2 x i64> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V1I64 = call <1 x i64> @llvm.masked.expandload.v1i64(ptr undef, <1 x i1> %m1, <1 x i64> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 77 for instruction: %V16I32 = call <16 x i32> @llvm.masked.expandload.v16i32(ptr undef, <16 x i1> %m16, <16 x i32> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 39 for instruction: %V8I32 = call <8 x i32> @llvm.masked.expandload.v8i32(ptr undef, <8 x i1> %m8, <8 x i32> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %V4I32 = call <4 x i32> @llvm.masked.expandload.v4i32(ptr undef, <4 x i1> %m4, <4 x i32> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V2I32 = call <2 x i32> @llvm.masked.expandload.v2i32(ptr undef, <2 x i1> %m2, <2 x i32> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 130 for instruction: %V32I16 = call <32 x i16> @llvm.masked.expandload.v32i16(ptr undef, <32 x i1> %m32, <32 x i16> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 65 for instruction: %V16I16 = call <16 x i16> @llvm.masked.expandload.v16i16(ptr undef, <16 x i1> %m16, <16 x i16> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %V8I16 = call <8 x i16> @llvm.masked.expandload.v8i16(ptr undef, <8 x i1> %m8, <8 x i16> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %V4I16 = call <4 x i16> @llvm.masked.expandload.v4i16(ptr undef, <4 x i1> %m4, <4 x i16> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 320 for instruction: %V64I8 = call <64 x i8> @llvm.masked.expandload.v64i8(ptr undef, <64 x i1> %m64, <64 x i8> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: %V32I8 = call <32 x i8> @llvm.masked.expandload.v32i8(ptr undef, <32 x i1> %m32, <32 x i8> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %V16I8 = call <16 x i8> @llvm.masked.expandload.v16i8(ptr undef, <16 x i1> %m16, <16 x i8> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %V8I8 = call <8 x i8> @llvm.masked.expandload.v8i8(ptr undef, <8 x i1> %m8, <8 x i8> undef)
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 0
 ;
 ; SSE42-LABEL: 'masked_expandload'
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %V8F64 = call <8 x double> @llvm.masked.expandload.v8f64(ptr undef, <8 x i1> %m8, <8 x double> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V4F64 = call <4 x double> @llvm.masked.expandload.v4f64(ptr undef, <4 x i1> %m4, <4 x double> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V2F64 = call <2 x double> @llvm.masked.expandload.v2f64(ptr undef, <2 x i1> %m2, <2 x double> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1F64 = call <1 x double> @llvm.masked.expandload.v1f64(ptr undef, <1 x i1> %m1, <1 x double> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 29 for instruction: %V16F32 = call <16 x float> @llvm.masked.expandload.v16f32(ptr undef, <16 x i1> %m16, <16 x float> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %V8F32 = call <8 x float> @llvm.masked.expandload.v8f32(ptr undef, <8 x i1> %m8, <8 x float> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V4F32 = call <4 x float> @llvm.masked.expandload.v4f32(ptr undef, <4 x i1> %m4, <4 x float> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V2F32 = call <2 x float> @llvm.masked.expandload.v2f32(ptr undef, <2 x i1> %m2, <2 x float> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %V8I64 = call <8 x i64> @llvm.masked.expandload.v8i64(ptr undef, <8 x i1> %m8, <8 x i64> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V4I64 = call <4 x i64> @llvm.masked.expandload.v4i64(ptr undef, <4 x i1> %m4, <4 x i64> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V2I64 = call <2 x i64> @llvm.masked.expandload.v2i64(ptr undef, <2 x i1> %m2, <2 x i64> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1I64 = call <1 x i64> @llvm.masked.expandload.v1i64(ptr undef, <1 x i1> %m1, <1 x i64> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %V16I32 = call <16 x i32> @llvm.masked.expandload.v16i32(ptr undef, <16 x i1> %m16, <16 x i32> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %V8I32 = call <8 x i32> @llvm.masked.expandload.v8i32(ptr undef, <8 x i1> %m8, <8 x i32> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V4I32 = call <4 x i32> @llvm.masked.expandload.v4i32(ptr undef, <4 x i1> %m4, <4 x i32> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V2I32 = call <2 x i32> @llvm.masked.expandload.v2i32(ptr undef, <2 x i1> %m2, <2 x i32> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 66 for instruction: %V32I16 = call <32 x i16> @llvm.masked.expandload.v32i16(ptr undef, <32 x i1> %m32, <32 x i16> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %V16I16 = call <16 x i16> @llvm.masked.expandload.v16i16(ptr undef, <16 x i1> %m16, <16 x i16> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %V8I16 = call <8 x i16> @llvm.masked.expandload.v8i16(ptr undef, <8 x i1> %m8, <8 x i16> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V4I16 = call <4 x i16> @llvm.masked.expandload.v4i16(ptr undef, <4 x i1> %m4, <4 x i16> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 132 for instruction: %V64I8 = call <64 x i8> @llvm.masked.expandload.v64i8(ptr undef, <64 x i1> %m64, <64 x i8> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 66 for instruction: %V32I8 = call <32 x i8> @llvm.masked.expandload.v32i8(ptr undef, <32 x i1> %m32, <32 x i8> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %V16I8 = call <16 x i8> @llvm.masked.expandload.v16i8(ptr undef, <16 x i1> %m16, <16 x i8> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %V8I8 = call <8 x i8> @llvm.masked.expandload.v8i8(ptr undef, <8 x i1> %m8, <8 x i8> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 21 for instruction: %V8F64 = call <8 x double> @llvm.masked.expandload.v8f64(ptr undef, <8 x i1> %m8, <8 x double> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %V4F64 = call <4 x double> @llvm.masked.expandload.v4f64(ptr undef, <4 x i1> %m4, <4 x double> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2F64 = call <2 x double> @llvm.masked.expandload.v2f64(ptr undef, <2 x i1> %m2, <2 x double> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V1F64 = call <1 x double> @llvm.masked.expandload.v1f64(ptr undef, <1 x i1> %m1, <1 x double> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 45 for instruction: %V16F32 = call <16 x float> @llvm.masked.expandload.v16f32(ptr undef, <16 x i1> %m16, <16 x float> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 23 for instruction: %V8F32 = call <8 x float> @llvm.masked.expandload.v8f32(ptr undef, <8 x i1> %m8, <8 x float> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V4F32 = call <4 x float> @llvm.masked.expandload.v4f32(ptr undef, <4 x i1> %m4, <4 x float> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2F32 = call <2 x float> @llvm.masked.expandload.v2f32(ptr undef, <2 x i1> %m2, <2 x float> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: %V8I64 = call <8 x i64> @llvm.masked.expandload.v8i64(ptr undef, <8 x i1> %m8, <8 x i64> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %V4I64 = call <4 x i64> @llvm.masked.expandload.v4i64(ptr undef, <4 x i1> %m4, <4 x i64> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V2I64 = call <2 x i64> @llvm.masked.expandload.v2i64(ptr undef, <2 x i1> %m2, <2 x i64> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V1I64 = call <1 x i64> @llvm.masked.expandload.v1i64(ptr undef, <1 x i1> %m1, <1 x i64> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 49 for instruction: %V16I32 = call <16 x i32> @llvm.masked.expandload.v16i32(ptr undef, <16 x i1> %m16, <16 x i32> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: %V8I32 = call <8 x i32> @llvm.masked.expandload.v8i32(ptr undef, <8 x i1> %m8, <8 x i32> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %V4I32 = call <4 x i32> @llvm.masked.expandload.v4i32(ptr undef, <4 x i1> %m4, <4 x i32> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V2I32 = call <2 x i32> @llvm.masked.expandload.v2i32(ptr undef, <2 x i1> %m2, <2 x i32> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 98 for instruction: %V32I16 = call <32 x i16> @llvm.masked.expandload.v32i16(ptr undef, <32 x i1> %m32, <32 x i16> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 49 for instruction: %V16I16 = call <16 x i16> @llvm.masked.expandload.v16i16(ptr undef, <16 x i1> %m16, <16 x i16> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: %V8I16 = call <8 x i16> @llvm.masked.expandload.v8i16(ptr undef, <8 x i1> %m8, <8 x i16> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %V4I16 = call <4 x i16> @llvm.masked.expandload.v4i16(ptr undef, <4 x i1> %m4, <4 x i16> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 196 for instruction: %V64I8 = call <64 x i8> @llvm.masked.expandload.v64i8(ptr undef, <64 x i1> %m64, <64 x i8> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 98 for instruction: %V32I8 = call <32 x i8> @llvm.masked.expandload.v32i8(ptr undef, <32 x i1> %m32, <32 x i8> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 49 for instruction: %V16I8 = call <16 x i8> @llvm.masked.expandload.v16i8(ptr undef, <16 x i1> %m16, <16 x i8> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: %V8I8 = call <8 x i8> @llvm.masked.expandload.v8i8(ptr undef, <8 x i1> %m8, <8 x i8> undef)
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 0
 ;
 ; AVX1-LABEL: 'masked_expandload'
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %V8F64 = call <8 x double> @llvm.masked.expandload.v8f64(ptr undef, <8 x i1> %m8, <8 x double> undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V4F64 = call <4 x double> @llvm.masked.expandload.v4f64(ptr undef, <4 x i1> %m4, <4 x double> undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V2F64 = call <2 x double> @llvm.masked.expandload.v2f64(ptr undef, <2 x i1> %m2, <2 x double> undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1F64 = call <1 x double> @llvm.masked.expandload.v1f64(ptr undef, <1 x i1> %m1, <1 x double> undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 31 for instruction: %V16F32 = call <16 x float> @llvm.masked.expandload.v16f32(ptr undef, <16 x i1> %m16, <16 x float> undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V8F32 = call <8 x float> @llvm.masked.expandload.v8f32(ptr undef, <8 x i1> %m8, <8 x float> undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V4F32 = call <4 x float> @llvm.masked.expandload.v4f32(ptr undef, <4 x i1> %m4, <4 x float> undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V2F32 = call <2 x float> @llvm.masked.expandload.v2f32(ptr undef, <2 x i1> %m2, <2 x float> undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 19 for instruction: %V8I64 = call <8 x i64> @llvm.masked.expandload.v8i64(ptr undef, <8 x i1> %m8, <8 x i64> undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V4I64 = call <4 x i64> @llvm.masked.expandload.v4i64(ptr undef, <4 x i1> %m4, <4 x i64> undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V2I64 = call <2 x i64> @llvm.masked.expandload.v2i64(ptr undef, <2 x i1> %m2, <2 x i64> undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1I64 = call <1 x i64> @llvm.masked.expandload.v1i64(ptr undef, <1 x i1> %m1, <1 x i64> undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 35 for instruction: %V16I32 = call <16 x i32> @llvm.masked.expandload.v16i32(ptr undef, <16 x i1> %m16, <16 x i32> undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V8I32 = call <8 x i32> @llvm.masked.expandload.v8i32(ptr undef, <8 x i1> %m8, <8 x i32> undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V4I32 = call <4 x i32> @llvm.masked.expandload.v4i32(ptr undef, <4 x i1> %m4, <4 x i32> undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V2I32 = call <2 x i32> @llvm.masked.expandload.v2i32(ptr undef, <2 x i1> %m2, <2 x i32> undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 68 for instruction: %V32I16 = call <32 x i16> @llvm.masked.expandload.v32i16(ptr undef, <32 x i1> %m32, <32 x i16> undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %V16I16 = call <16 x i16> @llvm.masked.expandload.v16i16(ptr undef, <16 x i1> %m16, <16 x i16> undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %V8I16 = call <8 x i16> @llvm.masked.expandload.v8i16(ptr undef, <8 x i1> %m8, <8 x i16> undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V4I16 = call <4 x i16> @llvm.masked.expandload.v4i16(ptr undef, <4 x i1> %m4, <4 x i16> undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 134 for instruction: %V64I8 = call <64 x i8> @llvm.masked.expandload.v64i8(ptr undef, <64 x i1> %m64, <64 x i8> undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 67 for instruction: %V32I8 = call <32 x i8> @llvm.masked.expandload.v32i8(ptr undef, <32 x i1> %m32, <32 x i8> undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %V16I8 = call <16 x i8> @llvm.masked.expandload.v16i8(ptr undef, <16 x i1> %m16, <16 x i8> undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %V8I8 = call <8 x i8> @llvm.masked.expandload.v8i8(ptr undef, <8 x i1> %m8, <8 x i8> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: %V8F64 = call <8 x double> @llvm.masked.expandload.v8f64(ptr undef, <8 x i1> %m8, <8 x double> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %V4F64 = call <4 x double> @llvm.masked.expandload.v4f64(ptr undef, <4 x i1> %m4, <4 x double> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2F64 = call <2 x double> @llvm.masked.expandload.v2f64(ptr undef, <2 x i1> %m2, <2 x double> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V1F64 = call <1 x double> @llvm.masked.expandload.v1f64(ptr undef, <1 x i1> %m1, <1 x double> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 51 for instruction: %V16F32 = call <16 x float> @llvm.masked.expandload.v16f32(ptr undef, <16 x i1> %m16, <16 x float> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 26 for instruction: %V8F32 = call <8 x float> @llvm.masked.expandload.v8f32(ptr undef, <8 x i1> %m8, <8 x float> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %V4F32 = call <4 x float> @llvm.masked.expandload.v4f32(ptr undef, <4 x i1> %m4, <4 x float> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2F32 = call <2 x float> @llvm.masked.expandload.v2f32(ptr undef, <2 x i1> %m2, <2 x float> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 29 for instruction: %V8I64 = call <8 x i64> @llvm.masked.expandload.v8i64(ptr undef, <8 x i1> %m8, <8 x i64> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %V4I64 = call <4 x i64> @llvm.masked.expandload.v4i64(ptr undef, <4 x i1> %m4, <4 x i64> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V2I64 = call <2 x i64> @llvm.masked.expandload.v2i64(ptr undef, <2 x i1> %m2, <2 x i64> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V1I64 = call <1 x i64> @llvm.masked.expandload.v1i64(ptr undef, <1 x i1> %m1, <1 x i64> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 55 for instruction: %V16I32 = call <16 x i32> @llvm.masked.expandload.v16i32(ptr undef, <16 x i1> %m16, <16 x i32> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %V8I32 = call <8 x i32> @llvm.masked.expandload.v8i32(ptr undef, <8 x i1> %m8, <8 x i32> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %V4I32 = call <4 x i32> @llvm.masked.expandload.v4i32(ptr undef, <4 x i1> %m4, <4 x i32> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V2I32 = call <2 x i32> @llvm.masked.expandload.v2i32(ptr undef, <2 x i1> %m2, <2 x i32> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 108 for instruction: %V32I16 = call <32 x i16> @llvm.masked.expandload.v32i16(ptr undef, <32 x i1> %m32, <32 x i16> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 54 for instruction: %V16I16 = call <16 x i16> @llvm.masked.expandload.v16i16(ptr undef, <16 x i1> %m16, <16 x i16> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 27 for instruction: %V8I16 = call <8 x i16> @llvm.masked.expandload.v8i16(ptr undef, <8 x i1> %m8, <8 x i16> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %V4I16 = call <4 x i16> @llvm.masked.expandload.v4i16(ptr undef, <4 x i1> %m4, <4 x i16> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 214 for instruction: %V64I8 = call <64 x i8> @llvm.masked.expandload.v64i8(ptr undef, <64 x i1> %m64, <64 x i8> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 107 for instruction: %V32I8 = call <32 x i8> @llvm.masked.expandload.v32i8(ptr undef, <32 x i1> %m32, <32 x i8> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 53 for instruction: %V16I8 = call <16 x i8> @llvm.masked.expandload.v16i8(ptr undef, <16 x i1> %m16, <16 x i8> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 27 for instruction: %V8I8 = call <8 x i8> @llvm.masked.expandload.v8i8(ptr undef, <8 x i1> %m8, <8 x i8> undef)
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 0
 ;
 ; AVX2-LABEL: 'masked_expandload'
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %V8F64 = call <8 x double> @llvm.masked.expandload.v8f64(ptr undef, <8 x i1> %m8, <8 x double> undef)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V4F64 = call <4 x double> @llvm.masked.expandload.v4f64(ptr undef, <4 x i1> %m4, <4 x double> undef)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V2F64 = call <2 x double> @llvm.masked.expandload.v2f64(ptr undef, <2 x i1> %m2, <2 x double> undef)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1F64 = call <1 x double> @llvm.masked.expandload.v1f64(ptr undef, <1 x i1> %m1, <1 x double> undef)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 31 for instruction: %V16F32 = call <16 x float> @llvm.masked.expandload.v16f32(ptr undef, <16 x i1> %m16, <16 x float> undef)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V8F32 = call <8 x float> @llvm.masked.expandload.v8f32(ptr undef, <8 x i1> %m8, <8 x float> undef)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V4F32 = call <4 x float> @llvm.masked.expandload.v4f32(ptr undef, <4 x i1> %m4, <4 x float> undef)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V2F32 = call <2 x float> @llvm.masked.expandload.v2f32(ptr undef, <2 x i1> %m2, <2 x float> undef)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 19 for instruction: %V8I64 = call <8 x i64> @llvm.masked.expandload.v8i64(ptr undef, <8 x i1> %m8, <8 x i64> undef)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V4I64 = call <4 x i64> @llvm.masked.expandload.v4i64(ptr undef, <4 x i1> %m4, <4 x i64> undef)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V2I64 = call <2 x i64> @llvm.masked.expandload.v2i64(ptr undef, <2 x i1> %m2, <2 x i64> undef)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1I64 = call <1 x i64> @llvm.masked.expandload.v1i64(ptr undef, <1 x i1> %m1, <1 x i64> undef)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 35 for instruction: %V16I32 = call <16 x i32> @llvm.masked.expandload.v16i32(ptr undef, <16 x i1> %m16, <16 x i32> undef)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V8I32 = call <8 x i32> @llvm.masked.expandload.v8i32(ptr undef, <8 x i1> %m8, <8 x i32> undef)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V4I32 = call <4 x i32> @llvm.masked.expandload.v4i32(ptr undef, <4 x i1> %m4, <4 x i32> undef)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V2I32 = call <2 x i32> @llvm.masked.expandload.v2i32(ptr undef, <2 x i1> %m2, <2 x i32> undef)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 67 for instruction: %V32I16 = call <32 x i16> @llvm.masked.expandload.v32i16(ptr undef, <32 x i1> %m32, <32 x i16> undef)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %V16I16 = call <16 x i16> @llvm.masked.expandload.v16i16(ptr undef, <16 x i1> %m16, <16 x i16> undef)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %V8I16 = call <8 x i16> @llvm.masked.expandload.v8i16(ptr undef, <8 x i1> %m8, <8 x i16> undef)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V4I16 = call <4 x i16> @llvm.masked.expandload.v4i16(ptr undef, <4 x i1> %m4, <4 x i16> undef)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 132 for instruction: %V64I8 = call <64 x i8> @llvm.masked.expandload.v64i8(ptr undef, <64 x i1> %m64, <64 x i8> undef)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 66 for instruction: %V32I8 = call <32 x i8> @llvm.masked.expandload.v32i8(ptr undef, <32 x i1> %m32, <32 x i8> undef)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %V16I8 = call <16 x i8> @llvm.masked.expandload.v16i8(ptr undef, <16 x i1> %m16, <16 x i8> undef)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %V8I8 = call <8 x i8> @llvm.masked.expandload.v8i8(ptr undef, <8 x i1> %m8, <8 x i8> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: %V8F64 = call <8 x double> @llvm.masked.expandload.v8f64(ptr undef, <8 x i1> %m8, <8 x double> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %V4F64 = call <4 x double> @llvm.masked.expandload.v4f64(ptr undef, <4 x i1> %m4, <4 x double> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2F64 = call <2 x double> @llvm.masked.expandload.v2f64(ptr undef, <2 x i1> %m2, <2 x double> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V1F64 = call <1 x double> @llvm.masked.expandload.v1f64(ptr undef, <1 x i1> %m1, <1 x double> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 51 for instruction: %V16F32 = call <16 x float> @llvm.masked.expandload.v16f32(ptr undef, <16 x i1> %m16, <16 x float> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 26 for instruction: %V8F32 = call <8 x float> @llvm.masked.expandload.v8f32(ptr undef, <8 x i1> %m8, <8 x float> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %V4F32 = call <4 x float> @llvm.masked.expandload.v4f32(ptr undef, <4 x i1> %m4, <4 x float> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2F32 = call <2 x float> @llvm.masked.expandload.v2f32(ptr undef, <2 x i1> %m2, <2 x float> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 29 for instruction: %V8I64 = call <8 x i64> @llvm.masked.expandload.v8i64(ptr undef, <8 x i1> %m8, <8 x i64> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %V4I64 = call <4 x i64> @llvm.masked.expandload.v4i64(ptr undef, <4 x i1> %m4, <4 x i64> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V2I64 = call <2 x i64> @llvm.masked.expandload.v2i64(ptr undef, <2 x i1> %m2, <2 x i64> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V1I64 = call <1 x i64> @llvm.masked.expandload.v1i64(ptr undef, <1 x i1> %m1, <1 x i64> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 55 for instruction: %V16I32 = call <16 x i32> @llvm.masked.expandload.v16i32(ptr undef, <16 x i1> %m16, <16 x i32> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %V8I32 = call <8 x i32> @llvm.masked.expandload.v8i32(ptr undef, <8 x i1> %m8, <8 x i32> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %V4I32 = call <4 x i32> @llvm.masked.expandload.v4i32(ptr undef, <4 x i1> %m4, <4 x i32> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V2I32 = call <2 x i32> @llvm.masked.expandload.v2i32(ptr undef, <2 x i1> %m2, <2 x i32> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 107 for instruction: %V32I16 = call <32 x i16> @llvm.masked.expandload.v32i16(ptr undef, <32 x i1> %m32, <32 x i16> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 54 for instruction: %V16I16 = call <16 x i16> @llvm.masked.expandload.v16i16(ptr undef, <16 x i1> %m16, <16 x i16> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 27 for instruction: %V8I16 = call <8 x i16> @llvm.masked.expandload.v8i16(ptr undef, <8 x i1> %m8, <8 x i16> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %V4I16 = call <4 x i16> @llvm.masked.expandload.v4i16(ptr undef, <4 x i1> %m4, <4 x i16> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 212 for instruction: %V64I8 = call <64 x i8> @llvm.masked.expandload.v64i8(ptr undef, <64 x i1> %m64, <64 x i8> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 106 for instruction: %V32I8 = call <32 x i8> @llvm.masked.expandload.v32i8(ptr undef, <32 x i1> %m32, <32 x i8> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 53 for instruction: %V16I8 = call <16 x i8> @llvm.masked.expandload.v16i8(ptr undef, <16 x i1> %m16, <16 x i8> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 27 for instruction: %V8I8 = call <8 x i8> @llvm.masked.expandload.v8i8(ptr undef, <8 x i1> %m8, <8 x i8> undef)
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 0
 ;
 ; SKL-LABEL: 'masked_expandload'
-; SKL-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %V8F64 = call <8 x double> @llvm.masked.expandload.v8f64(ptr undef, <8 x i1> %m8, <8 x double> undef)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V4F64 = call <4 x double> @llvm.masked.expandload.v4f64(ptr undef, <4 x i1> %m4, <4 x double> undef)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V2F64 = call <2 x double> @llvm.masked.expandload.v2f64(ptr undef, <2 x i1> %m2, <2 x double> undef)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1F64 = call <1 x double> @llvm.masked.expandload.v1f64(ptr undef, <1 x i1> %m1, <1 x double> undef)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 31 for instruction: %V16F32 = call <16 x float> @llvm.masked.expandload.v16f32(ptr undef, <16 x i1> %m16, <16 x float> undef)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V8F32 = call <8 x float> @llvm.masked.expandload.v8f32(ptr undef, <8 x i1> %m8, <8 x float> undef)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V4F32 = call <4 x float> @llvm.masked.expandload.v4f32(ptr undef, <4 x i1> %m4, <4 x float> undef)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V2F32 = call <2 x float> @llvm.masked.expandload.v2f32(ptr undef, <2 x i1> %m2, <2 x float> undef)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 19 for instruction: %V8I64 = call <8 x i64> @llvm.masked.expandload.v8i64(ptr undef, <8 x i1> %m8, <8 x i64> undef)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V4I64 = call <4 x i64> @llvm.masked.expandload.v4i64(ptr undef, <4 x i1> %m4, <4 x i64> undef)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V2I64 = call <2 x i64> @llvm.masked.expandload.v2i64(ptr undef, <2 x i1> %m2, <2 x i64> undef)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1I64 = call <1 x i64> @llvm.masked.expandload.v1i64(ptr undef, <1 x i1> %m1, <1 x i64> undef)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 35 for instruction: %V16I32 = call <16 x i32> @llvm.masked.expandload.v16i32(ptr undef, <16 x i1> %m16, <16 x i32> undef)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V8I32 = call <8 x i32> @llvm.masked.expandload.v8i32(ptr undef, <8 x i1> %m8, <8 x i32> undef)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V4I32 = call <4 x i32> @llvm.masked.expandload.v4i32(ptr undef, <4 x i1> %m4, <4 x i32> undef)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V2I32 = call <2 x i32> @llvm.masked.expandload.v2i32(ptr undef, <2 x i1> %m2, <2 x i32> undef)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 67 for instruction: %V32I16 = call <32 x i16> @llvm.masked.expandload.v32i16(ptr undef, <32 x i1> %m32, <32 x i16> undef)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %V16I16 = call <16 x i16> @llvm.masked.expandload.v16i16(ptr undef, <16 x i1> %m16, <16 x i16> undef)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %V8I16 = call <8 x i16> @llvm.masked.expandload.v8i16(ptr undef, <8 x i1> %m8, <8 x i16> undef)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V4I16 = call <4 x i16> @llvm.masked.expandload.v4i16(ptr undef, <4 x i1> %m4, <4 x i16> undef)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 132 for instruction: %V64I8 = call <64 x i8> @llvm.masked.expandload.v64i8(ptr undef, <64 x i1> %m64, <64 x i8> undef)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 66 for instruction: %V32I8 = call <32 x i8> @llvm.masked.expandload.v32i8(ptr undef, <32 x i1> %m32, <32 x i8> undef)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %V16I8 = call <16 x i8> @llvm.masked.expandload.v16i8(ptr undef, <16 x i1> %m16, <16 x i8> undef)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %V8I8 = call <8 x i8> @llvm.masked.expandload.v8i8(ptr undef, <8 x i1> %m8, <8 x i8> undef)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: %V8F64 = call <8 x double> @llvm.masked.expandload.v8f64(ptr undef, <8 x i1> %m8, <8 x double> undef)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %V4F64 = call <4 x double> @llvm.masked.expandload.v4f64(ptr undef, <4 x i1> %m4, <4 x double> undef)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2F64 = call <2 x double> @llvm.masked.expandload.v2f64(ptr undef, <2 x i1> %m2, <2 x double> undef)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V1F64 = call <1 x double> @llvm.masked.expandload.v1f64(ptr undef, <1 x i1> %m1, <1 x double> undef)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 51 for instruction: %V16F32 = call <16 x float> @llvm.masked.expandload.v16f32(ptr undef, <16 x i1> %m16, <16 x float> undef)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 26 for instruction: %V8F32 = call <8 x float> @llvm.masked.expandload.v8f32(ptr undef, <8 x i1> %m8, <8 x float> undef)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %V4F32 = call <4 x float> @llvm.masked.expandload.v4f32(ptr undef, <4 x i1> %m4, <4 x float> undef)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2F32 = call <2 x float> @llvm.masked.expandload.v2f32(ptr undef, <2 x i1> %m2, <2 x float> undef)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 29 for instruction: %V8I64 = call <8 x i64> @llvm.masked.expandload.v8i64(ptr undef, <8 x i1> %m8, <8 x i64> undef)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %V4I64 = call <4 x i64> @llvm.masked.expandload.v4i64(ptr undef, <4 x i1> %m4, <4 x i64> undef)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V2I64 = call <2 x i64> @llvm.masked.expandload.v2i64(ptr undef, <2 x i1> %m2, <2 x i64> undef)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V1I64 = call <1 x i64> @llvm.masked.expandload.v1i64(ptr undef, <1 x i1> %m1, <1 x i64> undef)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 55 for instruction: %V16I32 = call <16 x i32> @llvm.masked.expandload.v16i32(ptr undef, <16 x i1> %m16, <16 x i32> undef)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %V8I32 = call <8 x i32> @llvm.masked.expandload.v8i32(ptr undef, <8 x i1> %m8, <8 x i32> undef)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %V4I32 = call <4 x i32> @llvm.masked.expandload.v4i32(ptr undef, <4 x i1> %m4, <4 x i32> undef)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V2I32 = call <2 x i32> @llvm.masked.expandload.v2i32(ptr undef, <2 x i1> %m2, <2 x i32> undef)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 107 for instruction: %V32I16 = call <32 x i16> @llvm.masked.expandload.v32i16(ptr undef, <32 x i1> %m32, <32 x i16> undef)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 54 for instruction: %V16I16 = call <16 x i16> @llvm.masked.expandload.v16i16(ptr undef, <16 x i1> %m16, <16 x i16> undef)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 27 for instruction: %V8I16 = call <8 x i16> @llvm.masked.expandload.v8i16(ptr undef, <8 x i1> %m8, <8 x i16> undef)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %V4I16 = call <4 x i16> @llvm.masked.expandload.v4i16(ptr undef, <4 x i1> %m4, <4 x i16> undef)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 212 for instruction: %V64I8 = call <64 x i8> @llvm.masked.expandload.v64i8(ptr undef, <64 x i1> %m64, <64 x i8> undef)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 106 for instruction: %V32I8 = call <32 x i8> @llvm.masked.expandload.v32i8(ptr undef, <32 x i1> %m32, <32 x i8> undef)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 53 for instruction: %V16I8 = call <16 x i8> @llvm.masked.expandload.v16i8(ptr undef, <16 x i1> %m16, <16 x i8> undef)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 27 for instruction: %V8I8 = call <8 x i8> @llvm.masked.expandload.v8i8(ptr undef, <8 x i1> %m8, <8 x i8> undef)
 ; SKL-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 0
 ;
 ; AVX512-LABEL: 'masked_expandload'
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 23 for instruction: %V8F64 = call <8 x double> @llvm.masked.expandload.v8f64(ptr undef, <8 x i1> %m8, <8 x double> undef)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %V4F64 = call <4 x double> @llvm.masked.expandload.v4f64(ptr undef, <4 x i1> %m4, <4 x double> undef)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V2F64 = call <2 x double> @llvm.masked.expandload.v2f64(ptr undef, <2 x i1> %m2, <2 x double> undef)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1F64 = call <1 x double> @llvm.masked.expandload.v1f64(ptr undef, <1 x i1> %m1, <1 x double> undef)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 47 for instruction: %V16F32 = call <16 x float> @llvm.masked.expandload.v16f32(ptr undef, <16 x i1> %m16, <16 x float> undef)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 23 for instruction: %V8F32 = call <8 x float> @llvm.masked.expandload.v8f32(ptr undef, <8 x i1> %m8, <8 x float> undef)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %V4F32 = call <4 x float> @llvm.masked.expandload.v4f32(ptr undef, <4 x i1> %m4, <4 x float> undef)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V2F32 = call <2 x float> @llvm.masked.expandload.v2f32(ptr undef, <2 x i1> %m2, <2 x float> undef)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 27 for instruction: %V8I64 = call <8 x i64> @llvm.masked.expandload.v8i64(ptr undef, <8 x i1> %m8, <8 x i64> undef)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %V4I64 = call <4 x i64> @llvm.masked.expandload.v4i64(ptr undef, <4 x i1> %m4, <4 x i64> undef)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2I64 = call <2 x i64> @llvm.masked.expandload.v2i64(ptr undef, <2 x i1> %m2, <2 x i64> undef)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1I64 = call <1 x i64> @llvm.masked.expandload.v1i64(ptr undef, <1 x i1> %m1, <1 x i64> undef)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 51 for instruction: %V16I32 = call <16 x i32> @llvm.masked.expandload.v16i32(ptr undef, <16 x i1> %m16, <16 x i32> undef)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: %V8I32 = call <8 x i32> @llvm.masked.expandload.v8i32(ptr undef, <8 x i1> %m8, <8 x i32> undef)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V4I32 = call <4 x i32> @llvm.masked.expandload.v4i32(ptr undef, <4 x i1> %m4, <4 x i32> undef)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2I32 = call <2 x i32> @llvm.masked.expandload.v2i32(ptr undef, <2 x i1> %m2, <2 x i32> undef)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 99 for instruction: %V32I16 = call <32 x i16> @llvm.masked.expandload.v32i16(ptr undef, <32 x i1> %m32, <32 x i16> undef)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 49 for instruction: %V16I16 = call <16 x i16> @llvm.masked.expandload.v16i16(ptr undef, <16 x i1> %m16, <16 x i16> undef)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V8I16 = call <8 x i16> @llvm.masked.expandload.v8i16(ptr undef, <8 x i1> %m8, <8 x i16> undef)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V4I16 = call <4 x i16> @llvm.masked.expandload.v4i16(ptr undef, <4 x i1> %m4, <4 x i16> undef)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 195 for instruction: %V64I8 = call <64 x i8> @llvm.masked.expandload.v64i8(ptr undef, <64 x i1> %m64, <64 x i8> undef)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 97 for instruction: %V32I8 = call <32 x i8> @llvm.masked.expandload.v32i8(ptr undef, <32 x i1> %m32, <32 x i8> undef)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V16I8 = call <16 x i8> @llvm.masked.expandload.v16i8(ptr undef, <16 x i1> %m16, <16 x i8> undef)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V8I8 = call <8 x i8> @llvm.masked.expandload.v8i8(ptr undef, <8 x i1> %m8, <8 x i8> undef)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %V8F64 = call <8 x double> @llvm.masked.expandload.v8f64(ptr undef, <8 x i1> %m8, <8 x double> undef)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V4F64 = call <4 x double> @llvm.masked.expandload.v4f64(ptr undef, <4 x i1> %m4, <4 x double> undef)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V2F64 = call <2 x double> @llvm.masked.expandload.v2f64(ptr undef, <2 x i1> %m2, <2 x double> undef)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V1F64 = call <1 x double> @llvm.masked.expandload.v1f64(ptr undef, <1 x i1> %m1, <1 x double> undef)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 69 for instruction: %V16F32 = call <16 x float> @llvm.masked.expandload.v16f32(ptr undef, <16 x i1> %m16, <16 x float> undef)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %V8F32 = call <8 x float> @llvm.masked.expandload.v8f32(ptr undef, <8 x i1> %m8, <8 x float> undef)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V4F32 = call <4 x float> @llvm.masked.expandload.v4f32(ptr undef, <4 x i1> %m4, <4 x float> undef)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V2F32 = call <2 x float> @llvm.masked.expandload.v2f32(ptr undef, <2 x i1> %m2, <2 x float> undef)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 38 for instruction: %V8I64 = call <8 x i64> @llvm.masked.expandload.v8i64(ptr undef, <8 x i1> %m8, <8 x i64> undef)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V4I64 = call <4 x i64> @llvm.masked.expandload.v4i64(ptr undef, <4 x i1> %m4, <4 x i64> undef)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V2I64 = call <2 x i64> @llvm.masked.expandload.v2i64(ptr undef, <2 x i1> %m2, <2 x i64> undef)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V1I64 = call <1 x i64> @llvm.masked.expandload.v1i64(ptr undef, <1 x i1> %m1, <1 x i64> undef)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 73 for instruction: %V16I32 = call <16 x i32> @llvm.masked.expandload.v16i32(ptr undef, <16 x i1> %m16, <16 x i32> undef)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 36 for instruction: %V8I32 = call <8 x i32> @llvm.masked.expandload.v8i32(ptr undef, <8 x i1> %m8, <8 x i32> undef)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %V4I32 = call <4 x i32> @llvm.masked.expandload.v4i32(ptr undef, <4 x i1> %m4, <4 x i32> undef)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V2I32 = call <2 x i32> @llvm.masked.expandload.v2i32(ptr undef, <2 x i1> %m2, <2 x i32> undef)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 143 for instruction: %V32I16 = call <32 x i16> @llvm.masked.expandload.v32i16(ptr undef, <32 x i1> %m32, <32 x i16> undef)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 71 for instruction: %V16I16 = call <16 x i16> @llvm.masked.expandload.v16i16(ptr undef, <16 x i1> %m16, <16 x i16> undef)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 35 for instruction: %V8I16 = call <8 x i16> @llvm.masked.expandload.v8i16(ptr undef, <8 x i1> %m8, <8 x i16> undef)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %V4I16 = call <4 x i16> @llvm.masked.expandload.v4i16(ptr undef, <4 x i1> %m4, <4 x i16> undef)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 283 for instruction: %V64I8 = call <64 x i8> @llvm.masked.expandload.v64i8(ptr undef, <64 x i1> %m64, <64 x i8> undef)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 141 for instruction: %V32I8 = call <32 x i8> @llvm.masked.expandload.v32i8(ptr undef, <32 x i1> %m32, <32 x i8> undef)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 70 for instruction: %V16I8 = call <16 x i8> @llvm.masked.expandload.v16i8(ptr undef, <16 x i1> %m16, <16 x i8> undef)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 35 for instruction: %V8I8 = call <8 x i8> @llvm.masked.expandload.v8i8(ptr undef, <8 x i1> %m8, <8 x i8> undef)
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 0
 ;
   %V8F64 = call <8 x double> @llvm.masked.expandload.v8f64(ptr undef, <8 x i1> %m8, <8 x double> undef)
@@ -1374,165 +1374,165 @@ define i32 @masked_expandload(<1 x i1> %m1, <2 x i1> %m2, <4 x i1> %m4, <8 x i1>
 
 define i32 @masked_compressstore(<1 x i1> %m1, <2 x i1> %m2, <4 x i1> %m4, <8 x i1> %m8, <16 x i1> %m16, <32 x i1> %m32, <64 x i1> %m64) {
 ; SSE2-LABEL: 'masked_compressstore'
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: call void @llvm.masked.compressstore.v8f64(<8 x double> undef, ptr undef, <8 x i1> %m8)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: call void @llvm.masked.compressstore.v4f64(<4 x double> undef, ptr undef, <4 x i1> %m4)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: call void @llvm.masked.compressstore.v2f64(<2 x double> undef, ptr undef, <2 x i1> %m2)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.compressstore.v1f64(<1 x double> undef, ptr undef, <1 x i1> %m1)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 29 for instruction: call void @llvm.masked.compressstore.v16f32(<16 x float> undef, ptr undef, <16 x i1> %m16)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: call void @llvm.masked.compressstore.v8f32(<8 x float> undef, ptr undef, <8 x i1> %m8)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: call void @llvm.masked.compressstore.v4f32(<4 x float> undef, ptr undef, <4 x i1> %m4)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: call void @llvm.masked.compressstore.v2f32(<2 x float> undef, ptr undef, <2 x i1> %m2)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 21 for instruction: call void @llvm.masked.compressstore.v8i64(<8 x i64> undef, ptr undef, <8 x i1> %m8)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: call void @llvm.masked.compressstore.v4i64(<4 x i64> undef, ptr undef, <4 x i1> %m4)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: call void @llvm.masked.compressstore.v2i64(<2 x i64> undef, ptr undef, <2 x i1> %m2)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.compressstore.v1i64(<1 x i64> undef, ptr undef, <1 x i1> %m1)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 45 for instruction: call void @llvm.masked.compressstore.v16i32(<16 x i32> undef, ptr undef, <16 x i1> %m16)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 23 for instruction: call void @llvm.masked.compressstore.v8i32(<8 x i32> undef, ptr undef, <8 x i1> %m8)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: call void @llvm.masked.compressstore.v4i32(<4 x i32> undef, ptr undef, <4 x i1> %m4)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: call void @llvm.masked.compressstore.v2i32(<2 x i32> undef, ptr undef, <2 x i1> %m2)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 66 for instruction: call void @llvm.masked.compressstore.v32i16(<32 x i16> undef, ptr undef, <32 x i1> %m32)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: call void @llvm.masked.compressstore.v16i16(<16 x i16> undef, ptr undef, <16 x i1> %m16)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: call void @llvm.masked.compressstore.v8i16(<8 x i16> undef, ptr undef, <8 x i1> %m8)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: call void @llvm.masked.compressstore.v4i16(<4 x i16> undef, ptr undef, <4 x i1> %m4)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 192 for instruction: call void @llvm.masked.compressstore.v64i8(<64 x i8> undef, ptr undef, <64 x i1> %m64)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 96 for instruction: call void @llvm.masked.compressstore.v32i8(<32 x i8> undef, ptr undef, <32 x i1> %m32)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: call void @llvm.masked.compressstore.v16i8(<16 x i8> undef, ptr undef, <16 x i1> %m16)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: call void @llvm.masked.compressstore.v8i8(<8 x i8> undef, ptr undef, <8 x i1> %m8)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 29 for instruction: call void @llvm.masked.compressstore.v8f64(<8 x double> undef, ptr undef, <8 x i1> %m8)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: call void @llvm.masked.compressstore.v4f64(<4 x double> undef, ptr undef, <4 x i1> %m4)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: call void @llvm.masked.compressstore.v2f64(<2 x double> undef, ptr undef, <2 x i1> %m2)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: call void @llvm.masked.compressstore.v1f64(<1 x double> undef, ptr undef, <1 x i1> %m1)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 61 for instruction: call void @llvm.masked.compressstore.v16f32(<16 x float> undef, ptr undef, <16 x i1> %m16)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 31 for instruction: call void @llvm.masked.compressstore.v8f32(<8 x float> undef, ptr undef, <8 x i1> %m8)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: call void @llvm.masked.compressstore.v4f32(<4 x float> undef, ptr undef, <4 x i1> %m4)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: call void @llvm.masked.compressstore.v2f32(<2 x float> undef, ptr undef, <2 x i1> %m2)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 37 for instruction: call void @llvm.masked.compressstore.v8i64(<8 x i64> undef, ptr undef, <8 x i1> %m8)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 19 for instruction: call void @llvm.masked.compressstore.v4i64(<4 x i64> undef, ptr undef, <4 x i1> %m4)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: call void @llvm.masked.compressstore.v2i64(<2 x i64> undef, ptr undef, <2 x i1> %m2)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: call void @llvm.masked.compressstore.v1i64(<1 x i64> undef, ptr undef, <1 x i1> %m1)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 77 for instruction: call void @llvm.masked.compressstore.v16i32(<16 x i32> undef, ptr undef, <16 x i1> %m16)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 39 for instruction: call void @llvm.masked.compressstore.v8i32(<8 x i32> undef, ptr undef, <8 x i1> %m8)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: call void @llvm.masked.compressstore.v4i32(<4 x i32> undef, ptr undef, <4 x i1> %m4)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: call void @llvm.masked.compressstore.v2i32(<2 x i32> undef, ptr undef, <2 x i1> %m2)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 130 for instruction: call void @llvm.masked.compressstore.v32i16(<32 x i16> undef, ptr undef, <32 x i1> %m32)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 65 for instruction: call void @llvm.masked.compressstore.v16i16(<16 x i16> undef, ptr undef, <16 x i1> %m16)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: call void @llvm.masked.compressstore.v8i16(<8 x i16> undef, ptr undef, <8 x i1> %m8)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: call void @llvm.masked.compressstore.v4i16(<4 x i16> undef, ptr undef, <4 x i1> %m4)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 320 for instruction: call void @llvm.masked.compressstore.v64i8(<64 x i8> undef, ptr undef, <64 x i1> %m64)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: call void @llvm.masked.compressstore.v32i8(<32 x i8> undef, ptr undef, <32 x i1> %m32)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: call void @llvm.masked.compressstore.v16i8(<16 x i8> undef, ptr undef, <16 x i1> %m16)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: call void @llvm.masked.compressstore.v8i8(<8 x i8> undef, ptr undef, <8 x i1> %m8)
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 0
 ;
 ; SSE42-LABEL: 'masked_compressstore'
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: call void @llvm.masked.compressstore.v8f64(<8 x double> undef, ptr undef, <8 x i1> %m8)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: call void @llvm.masked.compressstore.v4f64(<4 x double> undef, ptr undef, <4 x i1> %m4)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: call void @llvm.masked.compressstore.v2f64(<2 x double> undef, ptr undef, <2 x i1> %m2)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.compressstore.v1f64(<1 x double> undef, ptr undef, <1 x i1> %m1)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 29 for instruction: call void @llvm.masked.compressstore.v16f32(<16 x float> undef, ptr undef, <16 x i1> %m16)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: call void @llvm.masked.compressstore.v8f32(<8 x float> undef, ptr undef, <8 x i1> %m8)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: call void @llvm.masked.compressstore.v4f32(<4 x float> undef, ptr undef, <4 x i1> %m4)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: call void @llvm.masked.compressstore.v2f32(<2 x float> undef, ptr undef, <2 x i1> %m2)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: call void @llvm.masked.compressstore.v8i64(<8 x i64> undef, ptr undef, <8 x i1> %m8)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: call void @llvm.masked.compressstore.v4i64(<4 x i64> undef, ptr undef, <4 x i1> %m4)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: call void @llvm.masked.compressstore.v2i64(<2 x i64> undef, ptr undef, <2 x i1> %m2)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.compressstore.v1i64(<1 x i64> undef, ptr undef, <1 x i1> %m1)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: call void @llvm.masked.compressstore.v16i32(<16 x i32> undef, ptr undef, <16 x i1> %m16)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: call void @llvm.masked.compressstore.v8i32(<8 x i32> undef, ptr undef, <8 x i1> %m8)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: call void @llvm.masked.compressstore.v4i32(<4 x i32> undef, ptr undef, <4 x i1> %m4)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: call void @llvm.masked.compressstore.v2i32(<2 x i32> undef, ptr undef, <2 x i1> %m2)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 66 for instruction: call void @llvm.masked.compressstore.v32i16(<32 x i16> undef, ptr undef, <32 x i1> %m32)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: call void @llvm.masked.compressstore.v16i16(<16 x i16> undef, ptr undef, <16 x i1> %m16)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: call void @llvm.masked.compressstore.v8i16(<8 x i16> undef, ptr undef, <8 x i1> %m8)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: call void @llvm.masked.compressstore.v4i16(<4 x i16> undef, ptr undef, <4 x i1> %m4)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 132 for instruction: call void @llvm.masked.compressstore.v64i8(<64 x i8> undef, ptr undef, <64 x i1> %m64)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 66 for instruction: call void @llvm.masked.compressstore.v32i8(<32 x i8> undef, ptr undef, <32 x i1> %m32)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: call void @llvm.masked.compressstore.v16i8(<16 x i8> undef, ptr undef, <16 x i1> %m16)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: call void @llvm.masked.compressstore.v8i8(<8 x i8> undef, ptr undef, <8 x i1> %m8)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 21 for instruction: call void @llvm.masked.compressstore.v8f64(<8 x double> undef, ptr undef, <8 x i1> %m8)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: call void @llvm.masked.compressstore.v4f64(<4 x double> undef, ptr undef, <4 x i1> %m4)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: call void @llvm.masked.compressstore.v2f64(<2 x double> undef, ptr undef, <2 x i1> %m2)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: call void @llvm.masked.compressstore.v1f64(<1 x double> undef, ptr undef, <1 x i1> %m1)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 45 for instruction: call void @llvm.masked.compressstore.v16f32(<16 x float> undef, ptr undef, <16 x i1> %m16)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 23 for instruction: call void @llvm.masked.compressstore.v8f32(<8 x float> undef, ptr undef, <8 x i1> %m8)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: call void @llvm.masked.compressstore.v4f32(<4 x float> undef, ptr undef, <4 x i1> %m4)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: call void @llvm.masked.compressstore.v2f32(<2 x float> undef, ptr undef, <2 x i1> %m2)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: call void @llvm.masked.compressstore.v8i64(<8 x i64> undef, ptr undef, <8 x i1> %m8)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: call void @llvm.masked.compressstore.v4i64(<4 x i64> undef, ptr undef, <4 x i1> %m4)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: call void @llvm.masked.compressstore.v2i64(<2 x i64> undef, ptr undef, <2 x i1> %m2)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: call void @llvm.masked.compressstore.v1i64(<1 x i64> undef, ptr undef, <1 x i1> %m1)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 49 for instruction: call void @llvm.masked.compressstore.v16i32(<16 x i32> undef, ptr undef, <16 x i1> %m16)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: call void @llvm.masked.compressstore.v8i32(<8 x i32> undef, ptr undef, <8 x i1> %m8)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: call void @llvm.masked.compressstore.v4i32(<4 x i32> undef, ptr undef, <4 x i1> %m4)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: call void @llvm.masked.compressstore.v2i32(<2 x i32> undef, ptr undef, <2 x i1> %m2)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 98 for instruction: call void @llvm.masked.compressstore.v32i16(<32 x i16> undef, ptr undef, <32 x i1> %m32)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 49 for instruction: call void @llvm.masked.compressstore.v16i16(<16 x i16> undef, ptr undef, <16 x i1> %m16)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: call void @llvm.masked.compressstore.v8i16(<8 x i16> undef, ptr undef, <8 x i1> %m8)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: call void @llvm.masked.compressstore.v4i16(<4 x i16> undef, ptr undef, <4 x i1> %m4)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 196 for instruction: call void @llvm.masked.compressstore.v64i8(<64 x i8> undef, ptr undef, <64 x i1> %m64)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 98 for instruction: call void @llvm.masked.compressstore.v32i8(<32 x i8> undef, ptr undef, <32 x i1> %m32)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 49 for instruction: call void @llvm.masked.compressstore.v16i8(<16 x i8> undef, ptr undef, <16 x i1> %m16)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: call void @llvm.masked.compressstore.v8i8(<8 x i8> undef, ptr undef, <8 x i1> %m8)
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 0
 ;
 ; AVX1-LABEL: 'masked_compressstore'
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: call void @llvm.masked.compressstore.v8f64(<8 x double> undef, ptr undef, <8 x i1> %m8)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: call void @llvm.masked.compressstore.v4f64(<4 x double> undef, ptr undef, <4 x i1> %m4)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: call void @llvm.masked.compressstore.v2f64(<2 x double> undef, ptr undef, <2 x i1> %m2)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.compressstore.v1f64(<1 x double> undef, ptr undef, <1 x i1> %m1)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 31 for instruction: call void @llvm.masked.compressstore.v16f32(<16 x float> undef, ptr undef, <16 x i1> %m16)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: call void @llvm.masked.compressstore.v8f32(<8 x float> undef, ptr undef, <8 x i1> %m8)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: call void @llvm.masked.compressstore.v4f32(<4 x float> undef, ptr undef, <4 x i1> %m4)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: call void @llvm.masked.compressstore.v2f32(<2 x float> undef, ptr undef, <2 x i1> %m2)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 19 for instruction: call void @llvm.masked.compressstore.v8i64(<8 x i64> undef, ptr undef, <8 x i1> %m8)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: call void @llvm.masked.compressstore.v4i64(<4 x i64> undef, ptr undef, <4 x i1> %m4)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: call void @llvm.masked.compressstore.v2i64(<2 x i64> undef, ptr undef, <2 x i1> %m2)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.compressstore.v1i64(<1 x i64> undef, ptr undef, <1 x i1> %m1)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 35 for instruction: call void @llvm.masked.compressstore.v16i32(<16 x i32> undef, ptr undef, <16 x i1> %m16)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: call void @llvm.masked.compressstore.v8i32(<8 x i32> undef, ptr undef, <8 x i1> %m8)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: call void @llvm.masked.compressstore.v4i32(<4 x i32> undef, ptr undef, <4 x i1> %m4)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: call void @llvm.masked.compressstore.v2i32(<2 x i32> undef, ptr undef, <2 x i1> %m2)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 68 for instruction: call void @llvm.masked.compressstore.v32i16(<32 x i16> undef, ptr undef, <32 x i1> %m32)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: call void @llvm.masked.compressstore.v16i16(<16 x i16> undef, ptr undef, <16 x i1> %m16)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: call void @llvm.masked.compressstore.v8i16(<8 x i16> undef, ptr undef, <8 x i1> %m8)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: call void @llvm.masked.compressstore.v4i16(<4 x i16> undef, ptr undef, <4 x i1> %m4)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 134 for instruction: call void @llvm.masked.compressstore.v64i8(<64 x i8> undef, ptr undef, <64 x i1> %m64)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 67 for instruction: call void @llvm.masked.compressstore.v32i8(<32 x i8> undef, ptr undef, <32 x i1> %m32)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: call void @llvm.masked.compressstore.v16i8(<16 x i8> undef, ptr undef, <16 x i1> %m16)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: call void @llvm.masked.compressstore.v8i8(<8 x i8> undef, ptr undef, <8 x i1> %m8)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: call void @llvm.masked.compressstore.v8f64(<8 x double> undef, ptr undef, <8 x i1> %m8)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: call void @llvm.masked.compressstore.v4f64(<4 x double> undef, ptr undef, <4 x i1> %m4)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: call void @llvm.masked.compressstore.v2f64(<2 x double> undef, ptr undef, <2 x i1> %m2)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: call void @llvm.masked.compressstore.v1f64(<1 x double> undef, ptr undef, <1 x i1> %m1)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 51 for instruction: call void @llvm.masked.compressstore.v16f32(<16 x float> undef, ptr undef, <16 x i1> %m16)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 26 for instruction: call void @llvm.masked.compressstore.v8f32(<8 x float> undef, ptr undef, <8 x i1> %m8)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: call void @llvm.masked.compressstore.v4f32(<4 x float> undef, ptr undef, <4 x i1> %m4)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: call void @llvm.masked.compressstore.v2f32(<2 x float> undef, ptr undef, <2 x i1> %m2)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 29 for instruction: call void @llvm.masked.compressstore.v8i64(<8 x i64> undef, ptr undef, <8 x i1> %m8)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: call void @llvm.masked.compressstore.v4i64(<4 x i64> undef, ptr undef, <4 x i1> %m4)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: call void @llvm.masked.compressstore.v2i64(<2 x i64> undef, ptr undef, <2 x i1> %m2)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: call void @llvm.masked.compressstore.v1i64(<1 x i64> undef, ptr undef, <1 x i1> %m1)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 55 for instruction: call void @llvm.masked.compressstore.v16i32(<16 x i32> undef, ptr undef, <16 x i1> %m16)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: call void @llvm.masked.compressstore.v8i32(<8 x i32> undef, ptr undef, <8 x i1> %m8)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: call void @llvm.masked.compressstore.v4i32(<4 x i32> undef, ptr undef, <4 x i1> %m4)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: call void @llvm.masked.compressstore.v2i32(<2 x i32> undef, ptr undef, <2 x i1> %m2)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 108 for instruction: call void @llvm.masked.compressstore.v32i16(<32 x i16> undef, ptr undef, <32 x i1> %m32)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 54 for instruction: call void @llvm.masked.compressstore.v16i16(<16 x i16> undef, ptr undef, <16 x i1> %m16)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 27 for instruction: call void @llvm.masked.compressstore.v8i16(<8 x i16> undef, ptr undef, <8 x i1> %m8)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: call void @llvm.masked.compressstore.v4i16(<4 x i16> undef, ptr undef, <4 x i1> %m4)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 214 for instruction: call void @llvm.masked.compressstore.v64i8(<64 x i8> undef, ptr undef, <64 x i1> %m64)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 107 for instruction: call void @llvm.masked.compressstore.v32i8(<32 x i8> undef, ptr undef, <32 x i1> %m32)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 53 for instruction: call void @llvm.masked.compressstore.v16i8(<16 x i8> undef, ptr undef, <16 x i1> %m16)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 27 for instruction: call void @llvm.masked.compressstore.v8i8(<8 x i8> undef, ptr undef, <8 x i1> %m8)
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 0
 ;
 ; AVX2-LABEL: 'masked_compressstore'
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: call void @llvm.masked.compressstore.v8f64(<8 x double> undef, ptr undef, <8 x i1> %m8)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: call void @llvm.masked.compressstore.v4f64(<4 x double> undef, ptr undef, <4 x i1> %m4)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: call void @llvm.masked.compressstore.v2f64(<2 x double> undef, ptr undef, <2 x i1> %m2)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.compressstore.v1f64(<1 x double> undef, ptr undef, <1 x i1> %m1)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 31 for instruction: call void @llvm.masked.compressstore.v16f32(<16 x float> undef, ptr undef, <16 x i1> %m16)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: call void @llvm.masked.compressstore.v8f32(<8 x float> undef, ptr undef, <8 x i1> %m8)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: call void @llvm.masked.compressstore.v4f32(<4 x float> undef, ptr undef, <4 x i1> %m4)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: call void @llvm.masked.compressstore.v2f32(<2 x float> undef, ptr undef, <2 x i1> %m2)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 19 for instruction: call void @llvm.masked.compressstore.v8i64(<8 x i64> undef, ptr undef, <8 x i1> %m8)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: call void @llvm.masked.compressstore.v4i64(<4 x i64> undef, ptr undef, <4 x i1> %m4)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: call void @llvm.masked.compressstore.v2i64(<2 x i64> undef, ptr undef, <2 x i1> %m2)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.compressstore.v1i64(<1 x i64> undef, ptr undef, <1 x i1> %m1)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 35 for instruction: call void @llvm.masked.compressstore.v16i32(<16 x i32> undef, ptr undef, <16 x i1> %m16)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: call void @llvm.masked.compressstore.v8i32(<8 x i32> undef, ptr undef, <8 x i1> %m8)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: call void @llvm.masked.compressstore.v4i32(<4 x i32> undef, ptr undef, <4 x i1> %m4)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: call void @llvm.masked.compressstore.v2i32(<2 x i32> undef, ptr undef, <2 x i1> %m2)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 67 for instruction: call void @llvm.masked.compressstore.v32i16(<32 x i16> undef, ptr undef, <32 x i1> %m32)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: call void @llvm.masked.compressstore.v16i16(<16 x i16> undef, ptr undef, <16 x i1> %m16)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: call void @llvm.masked.compressstore.v8i16(<8 x i16> undef, ptr undef, <8 x i1> %m8)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: call void @llvm.masked.compressstore.v4i16(<4 x i16> undef, ptr undef, <4 x i1> %m4)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 132 for instruction: call void @llvm.masked.compressstore.v64i8(<64 x i8> undef, ptr undef, <64 x i1> %m64)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 66 for instruction: call void @llvm.masked.compressstore.v32i8(<32 x i8> undef, ptr undef, <32 x i1> %m32)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: call void @llvm.masked.compressstore.v16i8(<16 x i8> undef, ptr undef, <16 x i1> %m16)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: call void @llvm.masked.compressstore.v8i8(<8 x i8> undef, ptr undef, <8 x i1> %m8)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: call void @llvm.masked.compressstore.v8f64(<8 x double> undef, ptr undef, <8 x i1> %m8)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: call void @llvm.masked.compressstore.v4f64(<4 x double> undef, ptr undef, <4 x i1> %m4)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: call void @llvm.masked.compressstore.v2f64(<2 x double> undef, ptr undef, <2 x i1> %m2)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: call void @llvm.masked.compressstore.v1f64(<1 x double> undef, ptr undef, <1 x i1> %m1)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 51 for instruction: call void @llvm.masked.compressstore.v16f32(<16 x float> undef, ptr undef, <16 x i1> %m16)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 26 for instruction: call void @llvm.masked.compressstore.v8f32(<8 x float> undef, ptr undef, <8 x i1> %m8)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: call void @llvm.masked.compressstore.v4f32(<4 x float> undef, ptr undef, <4 x i1> %m4)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: call void @llvm.masked.compressstore.v2f32(<2 x float> undef, ptr undef, <2 x i1> %m2)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 29 for instruction: call void @llvm.masked.compressstore.v8i64(<8 x i64> undef, ptr undef, <8 x i1> %m8)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: call void @llvm.masked.compressstore.v4i64(<4 x i64> undef, ptr undef, <4 x i1> %m4)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: call void @llvm.masked.compressstore.v2i64(<2 x i64> undef, ptr undef, <2 x i1> %m2)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: call void @llvm.masked.compressstore.v1i64(<1 x i64> undef, ptr undef, <1 x i1> %m1)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 55 for instruction: call void @llvm.masked.compressstore.v16i32(<16 x i32> undef, ptr undef, <16 x i1> %m16)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: call void @llvm.masked.compressstore.v8i32(<8 x i32> undef, ptr undef, <8 x i1> %m8)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: call void @llvm.masked.compressstore.v4i32(<4 x i32> undef, ptr undef, <4 x i1> %m4)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: call void @llvm.masked.compressstore.v2i32(<2 x i32> undef, ptr undef, <2 x i1> %m2)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 107 for instruction: call void @llvm.masked.compressstore.v32i16(<32 x i16> undef, ptr undef, <32 x i1> %m32)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 54 for instruction: call void @llvm.masked.compressstore.v16i16(<16 x i16> undef, ptr undef, <16 x i1> %m16)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 27 for instruction: call void @llvm.masked.compressstore.v8i16(<8 x i16> undef, ptr undef, <8 x i1> %m8)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: call void @llvm.masked.compressstore.v4i16(<4 x i16> undef, ptr undef, <4 x i1> %m4)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 212 for instruction: call void @llvm.masked.compressstore.v64i8(<64 x i8> undef, ptr undef, <64 x i1> %m64)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 106 for instruction: call void @llvm.masked.compressstore.v32i8(<32 x i8> undef, ptr undef, <32 x i1> %m32)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 53 for instruction: call void @llvm.masked.compressstore.v16i8(<16 x i8> undef, ptr undef, <16 x i1> %m16)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 27 for instruction: call void @llvm.masked.compressstore.v8i8(<8 x i8> undef, ptr undef, <8 x i1> %m8)
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 0
 ;
 ; SKL-LABEL: 'masked_compressstore'
-; SKL-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: call void @llvm.masked.compressstore.v8f64(<8 x double> undef, ptr undef, <8 x i1> %m8)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: call void @llvm.masked.compressstore.v4f64(<4 x double> undef, ptr undef, <4 x i1> %m4)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: call void @llvm.masked.compressstore.v2f64(<2 x double> undef, ptr undef, <2 x i1> %m2)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.compressstore.v1f64(<1 x double> undef, ptr undef, <1 x i1> %m1)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 31 for instruction: call void @llvm.masked.compressstore.v16f32(<16 x float> undef, ptr undef, <16 x i1> %m16)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: call void @llvm.masked.compressstore.v8f32(<8 x float> undef, ptr undef, <8 x i1> %m8)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: call void @llvm.masked.compressstore.v4f32(<4 x float> undef, ptr undef, <4 x i1> %m4)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: call void @llvm.masked.compressstore.v2f32(<2 x float> undef, ptr undef, <2 x i1> %m2)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 19 for instruction: call void @llvm.masked.compressstore.v8i64(<8 x i64> undef, ptr undef, <8 x i1> %m8)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: call void @llvm.masked.compressstore.v4i64(<4 x i64> undef, ptr undef, <4 x i1> %m4)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: call void @llvm.masked.compressstore.v2i64(<2 x i64> undef, ptr undef, <2 x i1> %m2)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.compressstore.v1i64(<1 x i64> undef, ptr undef, <1 x i1> %m1)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 35 for instruction: call void @llvm.masked.compressstore.v16i32(<16 x i32> undef, ptr undef, <16 x i1> %m16)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: call void @llvm.masked.compressstore.v8i32(<8 x i32> undef, ptr undef, <8 x i1> %m8)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: call void @llvm.masked.compressstore.v4i32(<4 x i32> undef, ptr undef, <4 x i1> %m4)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: call void @llvm.masked.compressstore.v2i32(<2 x i32> undef, ptr undef, <2 x i1> %m2)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 67 for instruction: call void @llvm.masked.compressstore.v32i16(<32 x i16> undef, ptr undef, <32 x i1> %m32)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: call void @llvm.masked.compressstore.v16i16(<16 x i16> undef, ptr undef, <16 x i1> %m16)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: call void @llvm.masked.compressstore.v8i16(<8 x i16> undef, ptr undef, <8 x i1> %m8)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: call void @llvm.masked.compressstore.v4i16(<4 x i16> undef, ptr undef, <4 x i1> %m4)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 132 for instruction: call void @llvm.masked.compressstore.v64i8(<64 x i8> undef, ptr undef, <64 x i1> %m64)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 66 for instruction: call void @llvm.masked.compressstore.v32i8(<32 x i8> undef, ptr undef, <32 x i1> %m32)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: call void @llvm.masked.compressstore.v16i8(<16 x i8> undef, ptr undef, <16 x i1> %m16)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: call void @llvm.masked.compressstore.v8i8(<8 x i8> undef, ptr undef, <8 x i1> %m8)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: call void @llvm.masked.compressstore.v8f64(<8 x double> undef, ptr undef, <8 x i1> %m8)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: call void @llvm.masked.compressstore.v4f64(<4 x double> undef, ptr undef, <4 x i1> %m4)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: call void @llvm.masked.compressstore.v2f64(<2 x double> undef, ptr undef, <2 x i1> %m2)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: call void @llvm.masked.compressstore.v1f64(<1 x double> undef, ptr undef, <1 x i1> %m1)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 51 for instruction: call void @llvm.masked.compressstore.v16f32(<16 x float> undef, ptr undef, <16 x i1> %m16)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 26 for instruction: call void @llvm.masked.compressstore.v8f32(<8 x float> undef, ptr undef, <8 x i1> %m8)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: call void @llvm.masked.compressstore.v4f32(<4 x float> undef, ptr undef, <4 x i1> %m4)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: call void @llvm.masked.compressstore.v2f32(<2 x float> undef, ptr undef, <2 x i1> %m2)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 29 for instruction: call void @llvm.masked.compressstore.v8i64(<8 x i64> undef, ptr undef, <8 x i1> %m8)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: call void @llvm.masked.compressstore.v4i64(<4 x i64> undef, ptr undef, <4 x i1> %m4)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: call void @llvm.masked.compressstore.v2i64(<2 x i64> undef, ptr undef, <2 x i1> %m2)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: call void @llvm.masked.compressstore.v1i64(<1 x i64> undef, ptr undef, <1 x i1> %m1)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 55 for instruction: call void @llvm.masked.compressstore.v16i32(<16 x i32> undef, ptr undef, <16 x i1> %m16)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: call void @llvm.masked.compressstore.v8i32(<8 x i32> undef, ptr undef, <8 x i1> %m8)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: call void @llvm.masked.compressstore.v4i32(<4 x i32> undef, ptr undef, <4 x i1> %m4)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: call void @llvm.masked.compressstore.v2i32(<2 x i32> undef, ptr undef, <2 x i1> %m2)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 107 for instruction: call void @llvm.masked.compressstore.v32i16(<32 x i16> undef, ptr undef, <32 x i1> %m32)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 54 for instruction: call void @llvm.masked.compressstore.v16i16(<16 x i16> undef, ptr undef, <16 x i1> %m16)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 27 for instruction: call void @llvm.masked.compressstore.v8i16(<8 x i16> undef, ptr undef, <8 x i1> %m8)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: call void @llvm.masked.compressstore.v4i16(<4 x i16> undef, ptr undef, <4 x i1> %m4)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 212 for instruction: call void @llvm.masked.compressstore.v64i8(<64 x i8> undef, ptr undef, <64 x i1> %m64)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 106 for instruction: call void @llvm.masked.compressstore.v32i8(<32 x i8> undef, ptr undef, <32 x i1> %m32)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 53 for instruction: call void @llvm.masked.compressstore.v16i8(<16 x i8> undef, ptr undef, <16 x i1> %m16)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 27 for instruction: call void @llvm.masked.compressstore.v8i8(<8 x i8> undef, ptr undef, <8 x i1> %m8)
 ; SKL-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 0
 ;
 ; AVX512-LABEL: 'masked_compressstore'
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 23 for instruction: call void @llvm.masked.compressstore.v8f64(<8 x double> undef, ptr undef, <8 x i1> %m8)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: call void @llvm.masked.compressstore.v4f64(<4 x double> undef, ptr undef, <4 x i1> %m4)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: call void @llvm.masked.compressstore.v2f64(<2 x double> undef, ptr undef, <2 x i1> %m2)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.compressstore.v1f64(<1 x double> undef, ptr undef, <1 x i1> %m1)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 47 for instruction: call void @llvm.masked.compressstore.v16f32(<16 x float> undef, ptr undef, <16 x i1> %m16)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 23 for instruction: call void @llvm.masked.compressstore.v8f32(<8 x float> undef, ptr undef, <8 x i1> %m8)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: call void @llvm.masked.compressstore.v4f32(<4 x float> undef, ptr undef, <4 x i1> %m4)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: call void @llvm.masked.compressstore.v2f32(<2 x float> undef, ptr undef, <2 x i1> %m2)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 27 for instruction: call void @llvm.masked.compressstore.v8i64(<8 x i64> undef, ptr undef, <8 x i1> %m8)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: call void @llvm.masked.compressstore.v4i64(<4 x i64> undef, ptr undef, <4 x i1> %m4)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: call void @llvm.masked.compressstore.v2i64(<2 x i64> undef, ptr undef, <2 x i1> %m2)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.compressstore.v1i64(<1 x i64> undef, ptr undef, <1 x i1> %m1)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 51 for instruction: call void @llvm.masked.compressstore.v16i32(<16 x i32> undef, ptr undef, <16 x i1> %m16)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: call void @llvm.masked.compressstore.v8i32(<8 x i32> undef, ptr undef, <8 x i1> %m8)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: call void @llvm.masked.compressstore.v4i32(<4 x i32> undef, ptr undef, <4 x i1> %m4)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: call void @llvm.masked.compressstore.v2i32(<2 x i32> undef, ptr undef, <2 x i1> %m2)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 99 for instruction: call void @llvm.masked.compressstore.v32i16(<32 x i16> undef, ptr undef, <32 x i1> %m32)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 49 for instruction: call void @llvm.masked.compressstore.v16i16(<16 x i16> undef, ptr undef, <16 x i1> %m16)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: call void @llvm.masked.compressstore.v8i16(<8 x i16> undef, ptr undef, <8 x i1> %m8)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: call void @llvm.masked.compressstore.v4i16(<4 x i16> undef, ptr undef, <4 x i1> %m4)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 195 for instruction: call void @llvm.masked.compressstore.v64i8(<64 x i8> undef, ptr undef, <64 x i1> %m64)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 97 for instruction: call void @llvm.masked.compressstore.v32i8(<32 x i8> undef, ptr undef, <32 x i1> %m32)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: call void @llvm.masked.compressstore.v16i8(<16 x i8> undef, ptr undef, <16 x i1> %m16)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: call void @llvm.masked.compressstore.v8i8(<8 x i8> undef, ptr undef, <8 x i1> %m8)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: call void @llvm.masked.compressstore.v8f64(<8 x double> undef, ptr undef, <8 x i1> %m8)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: call void @llvm.masked.compressstore.v4f64(<4 x double> undef, ptr undef, <4 x i1> %m4)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: call void @llvm.masked.compressstore.v2f64(<2 x double> undef, ptr undef, <2 x i1> %m2)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: call void @llvm.masked.compressstore.v1f64(<1 x double> undef, ptr undef, <1 x i1> %m1)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 69 for instruction: call void @llvm.masked.compressstore.v16f32(<16 x float> undef, ptr undef, <16 x i1> %m16)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: call void @llvm.masked.compressstore.v8f32(<8 x float> undef, ptr undef, <8 x i1> %m8)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: call void @llvm.masked.compressstore.v4f32(<4 x float> undef, ptr undef, <4 x i1> %m4)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: call void @llvm.masked.compressstore.v2f32(<2 x float> undef, ptr undef, <2 x i1> %m2)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 38 for instruction: call void @llvm.masked.compressstore.v8i64(<8 x i64> undef, ptr undef, <8 x i1> %m8)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: call void @llvm.masked.compressstore.v4i64(<4 x i64> undef, ptr undef, <4 x i1> %m4)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: call void @llvm.masked.compressstore.v2i64(<2 x i64> undef, ptr undef, <2 x i1> %m2)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: call void @llvm.masked.compressstore.v1i64(<1 x i64> undef, ptr undef, <1 x i1> %m1)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 73 for instruction: call void @llvm.masked.compressstore.v16i32(<16 x i32> undef, ptr undef, <16 x i1> %m16)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 36 for instruction: call void @llvm.masked.compressstore.v8i32(<8 x i32> undef, ptr undef, <8 x i1> %m8)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: call void @llvm.masked.compressstore.v4i32(<4 x i32> undef, ptr undef, <4 x i1> %m4)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: call void @llvm.masked.compressstore.v2i32(<2 x i32> undef, ptr undef, <2 x i1> %m2)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 143 for instruction: call void @llvm.masked.compressstore.v32i16(<32 x i16> undef, ptr undef, <32 x i1> %m32)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 71 for instruction: call void @llvm.masked.compressstore.v16i16(<16 x i16> undef, ptr undef, <16 x i1> %m16)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 35 for instruction: call void @llvm.masked.compressstore.v8i16(<8 x i16> undef, ptr undef, <8 x i1> %m8)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: call void @llvm.masked.compressstore.v4i16(<4 x i16> undef, ptr undef, <4 x i1> %m4)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 283 for instruction: call void @llvm.masked.compressstore.v64i8(<64 x i8> undef, ptr undef, <64 x i1> %m64)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 141 for instruction: call void @llvm.masked.compressstore.v32i8(<32 x i8> undef, ptr undef, <32 x i1> %m32)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 70 for instruction: call void @llvm.masked.compressstore.v16i8(<16 x i8> undef, ptr undef, <16 x i1> %m16)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 35 for instruction: call void @llvm.masked.compressstore.v8i8(<8 x i8> undef, ptr undef, <8 x i1> %m8)
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 0
 ;
   call void @llvm.masked.compressstore.v8f64(<8 x double> undef, ptr undef, <8 x i1> %m8)
diff --git a/llvm/test/Analysis/CostModel/X86/masked-intrinsic-cost.ll b/llvm/test/Analysis/CostModel/X86/masked-intrinsic-cost.ll
index 4ca1f7f419e6a..6f90816fcdc8e 100644
--- a/llvm/test/Analysis/CostModel/X86/masked-intrinsic-cost.ll
+++ b/llvm/test/Analysis/CostModel/X86/masked-intrinsic-cost.ll
@@ -1178,165 +1178,165 @@ define i32 @masked_scatter(<1 x i1> %m1, <2 x i1> %m2, <4 x i1> %m4, <8 x i1> %m
 
 define i32 @masked_expandload(<1 x i1> %m1, <2 x i1> %m2, <4 x i1> %m4, <8 x i1> %m8, <16 x i1> %m16, <32 x i1> %m32, <64 x i1> %m64) {
 ; SSE2-LABEL: 'masked_expandload'
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %V8F64 = call <8 x double> @llvm.masked.expandload.v8f64(ptr undef, <8 x i1> %m8, <8 x double> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V4F64 = call <4 x double> @llvm.masked.expandload.v4f64(ptr undef, <4 x i1> %m4, <4 x double> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V2F64 = call <2 x double> @llvm.masked.expandload.v2f64(ptr undef, <2 x i1> %m2, <2 x double> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1F64 = call <1 x double> @llvm.masked.expandload.v1f64(ptr undef, <1 x i1> %m1, <1 x double> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 29 for instruction: %V16F32 = call <16 x float> @llvm.masked.expandload.v16f32(ptr undef, <16 x i1> %m16, <16 x float> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %V8F32 = call <8 x float> @llvm.masked.expandload.v8f32(ptr undef, <8 x i1> %m8, <8 x float> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V4F32 = call <4 x float> @llvm.masked.expandload.v4f32(ptr undef, <4 x i1> %m4, <4 x float> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V2F32 = call <2 x float> @llvm.masked.expandload.v2f32(ptr undef, <2 x i1> %m2, <2 x float> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 21 for instruction: %V8I64 = call <8 x i64> @llvm.masked.expandload.v8i64(ptr undef, <8 x i1> %m8, <8 x i64> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %V4I64 = call <4 x i64> @llvm.masked.expandload.v4i64(ptr undef, <4 x i1> %m4, <4 x i64> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2I64 = call <2 x i64> @llvm.masked.expandload.v2i64(ptr undef, <2 x i1> %m2, <2 x i64> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1I64 = call <1 x i64> @llvm.masked.expandload.v1i64(ptr undef, <1 x i1> %m1, <1 x i64> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 45 for instruction: %V16I32 = call <16 x i32> @llvm.masked.expandload.v16i32(ptr undef, <16 x i1> %m16, <16 x i32> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 23 for instruction: %V8I32 = call <8 x i32> @llvm.masked.expandload.v8i32(ptr undef, <8 x i1> %m8, <8 x i32> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V4I32 = call <4 x i32> @llvm.masked.expandload.v4i32(ptr undef, <4 x i1> %m4, <4 x i32> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2I32 = call <2 x i32> @llvm.masked.expandload.v2i32(ptr undef, <2 x i1> %m2, <2 x i32> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 66 for instruction: %V32I16 = call <32 x i16> @llvm.masked.expandload.v32i16(ptr undef, <32 x i1> %m32, <32 x i16> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %V16I16 = call <16 x i16> @llvm.masked.expandload.v16i16(ptr undef, <16 x i1> %m16, <16 x i16> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %V8I16 = call <8 x i16> @llvm.masked.expandload.v8i16(ptr undef, <8 x i1> %m8, <8 x i16> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V4I16 = call <4 x i16> @llvm.masked.expandload.v4i16(ptr undef, <4 x i1> %m4, <4 x i16> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 192 for instruction: %V64I8 = call <64 x i8> @llvm.masked.expandload.v64i8(ptr undef, <64 x i1> %m64, <64 x i8> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 96 for instruction: %V32I8 = call <32 x i8> @llvm.masked.expandload.v32i8(ptr undef, <32 x i1> %m32, <32 x i8> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V16I8 = call <16 x i8> @llvm.masked.expandload.v16i8(ptr undef, <16 x i1> %m16, <16 x i8> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V8I8 = call <8 x i8> @llvm.masked.expandload.v8i8(ptr undef, <8 x i1> %m8, <8 x i8> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 29 for instruction: %V8F64 = call <8 x double> @llvm.masked.expandload.v8f64(ptr undef, <8 x i1> %m8, <8 x double> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %V4F64 = call <4 x double> @llvm.masked.expandload.v4f64(ptr undef, <4 x i1> %m4, <4 x double> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V2F64 = call <2 x double> @llvm.masked.expandload.v2f64(ptr undef, <2 x i1> %m2, <2 x double> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V1F64 = call <1 x double> @llvm.masked.expandload.v1f64(ptr undef, <1 x i1> %m1, <1 x double> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 61 for instruction: %V16F32 = call <16 x float> @llvm.masked.expandload.v16f32(ptr undef, <16 x i1> %m16, <16 x float> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 31 for instruction: %V8F32 = call <8 x float> @llvm.masked.expandload.v8f32(ptr undef, <8 x i1> %m8, <8 x float> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V4F32 = call <4 x float> @llvm.masked.expandload.v4f32(ptr undef, <4 x i1> %m4, <4 x float> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V2F32 = call <2 x float> @llvm.masked.expandload.v2f32(ptr undef, <2 x i1> %m2, <2 x float> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 37 for instruction: %V8I64 = call <8 x i64> @llvm.masked.expandload.v8i64(ptr undef, <8 x i1> %m8, <8 x i64> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 19 for instruction: %V4I64 = call <4 x i64> @llvm.masked.expandload.v4i64(ptr undef, <4 x i1> %m4, <4 x i64> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V2I64 = call <2 x i64> @llvm.masked.expandload.v2i64(ptr undef, <2 x i1> %m2, <2 x i64> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V1I64 = call <1 x i64> @llvm.masked.expandload.v1i64(ptr undef, <1 x i1> %m1, <1 x i64> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 77 for instruction: %V16I32 = call <16 x i32> @llvm.masked.expandload.v16i32(ptr undef, <16 x i1> %m16, <16 x i32> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 39 for instruction: %V8I32 = call <8 x i32> @llvm.masked.expandload.v8i32(ptr undef, <8 x i1> %m8, <8 x i32> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %V4I32 = call <4 x i32> @llvm.masked.expandload.v4i32(ptr undef, <4 x i1> %m4, <4 x i32> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V2I32 = call <2 x i32> @llvm.masked.expandload.v2i32(ptr undef, <2 x i1> %m2, <2 x i32> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 130 for instruction: %V32I16 = call <32 x i16> @llvm.masked.expandload.v32i16(ptr undef, <32 x i1> %m32, <32 x i16> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 65 for instruction: %V16I16 = call <16 x i16> @llvm.masked.expandload.v16i16(ptr undef, <16 x i1> %m16, <16 x i16> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %V8I16 = call <8 x i16> @llvm.masked.expandload.v8i16(ptr undef, <8 x i1> %m8, <8 x i16> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %V4I16 = call <4 x i16> @llvm.masked.expandload.v4i16(ptr undef, <4 x i1> %m4, <4 x i16> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 320 for instruction: %V64I8 = call <64 x i8> @llvm.masked.expandload.v64i8(ptr undef, <64 x i1> %m64, <64 x i8> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: %V32I8 = call <32 x i8> @llvm.masked.expandload.v32i8(ptr undef, <32 x i1> %m32, <32 x i8> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %V16I8 = call <16 x i8> @llvm.masked.expandload.v16i8(ptr undef, <16 x i1> %m16, <16 x i8> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %V8I8 = call <8 x i8> @llvm.masked.expandload.v8i8(ptr undef, <8 x i1> %m8, <8 x i8> undef)
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 0
 ;
 ; SSE42-LABEL: 'masked_expandload'
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %V8F64 = call <8 x double> @llvm.masked.expandload.v8f64(ptr undef, <8 x i1> %m8, <8 x double> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V4F64 = call <4 x double> @llvm.masked.expandload.v4f64(ptr undef, <4 x i1> %m4, <4 x double> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V2F64 = call <2 x double> @llvm.masked.expandload.v2f64(ptr undef, <2 x i1> %m2, <2 x double> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1F64 = call <1 x double> @llvm.masked.expandload.v1f64(ptr undef, <1 x i1> %m1, <1 x double> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 29 for instruction: %V16F32 = call <16 x float> @llvm.masked.expandload.v16f32(ptr undef, <16 x i1> %m16, <16 x float> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %V8F32 = call <8 x float> @llvm.masked.expandload.v8f32(ptr undef, <8 x i1> %m8, <8 x float> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V4F32 = call <4 x float> @llvm.masked.expandload.v4f32(ptr undef, <4 x i1> %m4, <4 x float> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V2F32 = call <2 x float> @llvm.masked.expandload.v2f32(ptr undef, <2 x i1> %m2, <2 x float> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %V8I64 = call <8 x i64> @llvm.masked.expandload.v8i64(ptr undef, <8 x i1> %m8, <8 x i64> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V4I64 = call <4 x i64> @llvm.masked.expandload.v4i64(ptr undef, <4 x i1> %m4, <4 x i64> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V2I64 = call <2 x i64> @llvm.masked.expandload.v2i64(ptr undef, <2 x i1> %m2, <2 x i64> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1I64 = call <1 x i64> @llvm.masked.expandload.v1i64(ptr undef, <1 x i1> %m1, <1 x i64> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %V16I32 = call <16 x i32> @llvm.masked.expandload.v16i32(ptr undef, <16 x i1> %m16, <16 x i32> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %V8I32 = call <8 x i32> @llvm.masked.expandload.v8i32(ptr undef, <8 x i1> %m8, <8 x i32> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V4I32 = call <4 x i32> @llvm.masked.expandload.v4i32(ptr undef, <4 x i1> %m4, <4 x i32> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V2I32 = call <2 x i32> @llvm.masked.expandload.v2i32(ptr undef, <2 x i1> %m2, <2 x i32> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 66 for instruction: %V32I16 = call <32 x i16> @llvm.masked.expandload.v32i16(ptr undef, <32 x i1> %m32, <32 x i16> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %V16I16 = call <16 x i16> @llvm.masked.expandload.v16i16(ptr undef, <16 x i1> %m16, <16 x i16> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %V8I16 = call <8 x i16> @llvm.masked.expandload.v8i16(ptr undef, <8 x i1> %m8, <8 x i16> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V4I16 = call <4 x i16> @llvm.masked.expandload.v4i16(ptr undef, <4 x i1> %m4, <4 x i16> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 132 for instruction: %V64I8 = call <64 x i8> @llvm.masked.expandload.v64i8(ptr undef, <64 x i1> %m64, <64 x i8> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 66 for instruction: %V32I8 = call <32 x i8> @llvm.masked.expandload.v32i8(ptr undef, <32 x i1> %m32, <32 x i8> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %V16I8 = call <16 x i8> @llvm.masked.expandload.v16i8(ptr undef, <16 x i1> %m16, <16 x i8> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %V8I8 = call <8 x i8> @llvm.masked.expandload.v8i8(ptr undef, <8 x i1> %m8, <8 x i8> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 21 for instruction: %V8F64 = call <8 x double> @llvm.masked.expandload.v8f64(ptr undef, <8 x i1> %m8, <8 x double> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %V4F64 = call <4 x double> @llvm.masked.expandload.v4f64(ptr undef, <4 x i1> %m4, <4 x double> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2F64 = call <2 x double> @llvm.masked.expandload.v2f64(ptr undef, <2 x i1> %m2, <2 x double> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V1F64 = call <1 x double> @llvm.masked.expandload.v1f64(ptr undef, <1 x i1> %m1, <1 x double> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 45 for instruction: %V16F32 = call <16 x float> @llvm.masked.expandload.v16f32(ptr undef, <16 x i1> %m16, <16 x float> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 23 for instruction: %V8F32 = call <8 x float> @llvm.masked.expandload.v8f32(ptr undef, <8 x i1> %m8, <8 x float> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V4F32 = call <4 x float> @llvm.masked.expandload.v4f32(ptr undef, <4 x i1> %m4, <4 x float> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2F32 = call <2 x float> @llvm.masked.expandload.v2f32(ptr undef, <2 x i1> %m2, <2 x float> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: %V8I64 = call <8 x i64> @llvm.masked.expandload.v8i64(ptr undef, <8 x i1> %m8, <8 x i64> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %V4I64 = call <4 x i64> @llvm.masked.expandload.v4i64(ptr undef, <4 x i1> %m4, <4 x i64> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V2I64 = call <2 x i64> @llvm.masked.expandload.v2i64(ptr undef, <2 x i1> %m2, <2 x i64> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V1I64 = call <1 x i64> @llvm.masked.expandload.v1i64(ptr undef, <1 x i1> %m1, <1 x i64> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 49 for instruction: %V16I32 = call <16 x i32> @llvm.masked.expandload.v16i32(ptr undef, <16 x i1> %m16, <16 x i32> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: %V8I32 = call <8 x i32> @llvm.masked.expandload.v8i32(ptr undef, <8 x i1> %m8, <8 x i32> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %V4I32 = call <4 x i32> @llvm.masked.expandload.v4i32(ptr undef, <4 x i1> %m4, <4 x i32> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V2I32 = call <2 x i32> @llvm.masked.expandload.v2i32(ptr undef, <2 x i1> %m2, <2 x i32> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 98 for instruction: %V32I16 = call <32 x i16> @llvm.masked.expandload.v32i16(ptr undef, <32 x i1> %m32, <32 x i16> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 49 for instruction: %V16I16 = call <16 x i16> @llvm.masked.expandload.v16i16(ptr undef, <16 x i1> %m16, <16 x i16> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: %V8I16 = call <8 x i16> @llvm.masked.expandload.v8i16(ptr undef, <8 x i1> %m8, <8 x i16> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %V4I16 = call <4 x i16> @llvm.masked.expandload.v4i16(ptr undef, <4 x i1> %m4, <4 x i16> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 196 for instruction: %V64I8 = call <64 x i8> @llvm.masked.expandload.v64i8(ptr undef, <64 x i1> %m64, <64 x i8> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 98 for instruction: %V32I8 = call <32 x i8> @llvm.masked.expandload.v32i8(ptr undef, <32 x i1> %m32, <32 x i8> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 49 for instruction: %V16I8 = call <16 x i8> @llvm.masked.expandload.v16i8(ptr undef, <16 x i1> %m16, <16 x i8> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: %V8I8 = call <8 x i8> @llvm.masked.expandload.v8i8(ptr undef, <8 x i1> %m8, <8 x i8> undef)
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 0
 ;
 ; AVX1-LABEL: 'masked_expandload'
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %V8F64 = call <8 x double> @llvm.masked.expandload.v8f64(ptr undef, <8 x i1> %m8, <8 x double> undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V4F64 = call <4 x double> @llvm.masked.expandload.v4f64(ptr undef, <4 x i1> %m4, <4 x double> undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V2F64 = call <2 x double> @llvm.masked.expandload.v2f64(ptr undef, <2 x i1> %m2, <2 x double> undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1F64 = call <1 x double> @llvm.masked.expandload.v1f64(ptr undef, <1 x i1> %m1, <1 x double> undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 31 for instruction: %V16F32 = call <16 x float> @llvm.masked.expandload.v16f32(ptr undef, <16 x i1> %m16, <16 x float> undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V8F32 = call <8 x float> @llvm.masked.expandload.v8f32(ptr undef, <8 x i1> %m8, <8 x float> undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V4F32 = call <4 x float> @llvm.masked.expandload.v4f32(ptr undef, <4 x i1> %m4, <4 x float> undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V2F32 = call <2 x float> @llvm.masked.expandload.v2f32(ptr undef, <2 x i1> %m2, <2 x float> undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 19 for instruction: %V8I64 = call <8 x i64> @llvm.masked.expandload.v8i64(ptr undef, <8 x i1> %m8, <8 x i64> undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V4I64 = call <4 x i64> @llvm.masked.expandload.v4i64(ptr undef, <4 x i1> %m4, <4 x i64> undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V2I64 = call <2 x i64> @llvm.masked.expandload.v2i64(ptr undef, <2 x i1> %m2, <2 x i64> undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1I64 = call <1 x i64> @llvm.masked.expandload.v1i64(ptr undef, <1 x i1> %m1, <1 x i64> undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 35 for instruction: %V16I32 = call <16 x i32> @llvm.masked.expandload.v16i32(ptr undef, <16 x i1> %m16, <16 x i32> undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V8I32 = call <8 x i32> @llvm.masked.expandload.v8i32(ptr undef, <8 x i1> %m8, <8 x i32> undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V4I32 = call <4 x i32> @llvm.masked.expandload.v4i32(ptr undef, <4 x i1> %m4, <4 x i32> undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V2I32 = call <2 x i32> @llvm.masked.expandload.v2i32(ptr undef, <2 x i1> %m2, <2 x i32> undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 68 for instruction: %V32I16 = call <32 x i16> @llvm.masked.expandload.v32i16(ptr undef, <32 x i1> %m32, <32 x i16> undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %V16I16 = call <16 x i16> @llvm.masked.expandload.v16i16(ptr undef, <16 x i1> %m16, <16 x i16> undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %V8I16 = call <8 x i16> @llvm.masked.expandload.v8i16(ptr undef, <8 x i1> %m8, <8 x i16> undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V4I16 = call <4 x i16> @llvm.masked.expandload.v4i16(ptr undef, <4 x i1> %m4, <4 x i16> undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 134 for instruction: %V64I8 = call <64 x i8> @llvm.masked.expandload.v64i8(ptr undef, <64 x i1> %m64, <64 x i8> undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 67 for instruction: %V32I8 = call <32 x i8> @llvm.masked.expandload.v32i8(ptr undef, <32 x i1> %m32, <32 x i8> undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %V16I8 = call <16 x i8> @llvm.masked.expandload.v16i8(ptr undef, <16 x i1> %m16, <16 x i8> undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %V8I8 = call <8 x i8> @llvm.masked.expandload.v8i8(ptr undef, <8 x i1> %m8, <8 x i8> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: %V8F64 = call <8 x double> @llvm.masked.expandload.v8f64(ptr undef, <8 x i1> %m8, <8 x double> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %V4F64 = call <4 x double> @llvm.masked.expandload.v4f64(ptr undef, <4 x i1> %m4, <4 x double> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2F64 = call <2 x double> @llvm.masked.expandload.v2f64(ptr undef, <2 x i1> %m2, <2 x double> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V1F64 = call <1 x double> @llvm.masked.expandload.v1f64(ptr undef, <1 x i1> %m1, <1 x double> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 51 for instruction: %V16F32 = call <16 x float> @llvm.masked.expandload.v16f32(ptr undef, <16 x i1> %m16, <16 x float> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 26 for instruction: %V8F32 = call <8 x float> @llvm.masked.expandload.v8f32(ptr undef, <8 x i1> %m8, <8 x float> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %V4F32 = call <4 x float> @llvm.masked.expandload.v4f32(ptr undef, <4 x i1> %m4, <4 x float> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2F32 = call <2 x float> @llvm.masked.expandload.v2f32(ptr undef, <2 x i1> %m2, <2 x float> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 29 for instruction: %V8I64 = call <8 x i64> @llvm.masked.expandload.v8i64(ptr undef, <8 x i1> %m8, <8 x i64> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %V4I64 = call <4 x i64> @llvm.masked.expandload.v4i64(ptr undef, <4 x i1> %m4, <4 x i64> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V2I64 = call <2 x i64> @llvm.masked.expandload.v2i64(ptr undef, <2 x i1> %m2, <2 x i64> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V1I64 = call <1 x i64> @llvm.masked.expandload.v1i64(ptr undef, <1 x i1> %m1, <1 x i64> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 55 for instruction: %V16I32 = call <16 x i32> @llvm.masked.expandload.v16i32(ptr undef, <16 x i1> %m16, <16 x i32> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %V8I32 = call <8 x i32> @llvm.masked.expandload.v8i32(ptr undef, <8 x i1> %m8, <8 x i32> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %V4I32 = call <4 x i32> @llvm.masked.expandload.v4i32(ptr undef, <4 x i1> %m4, <4 x i32> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V2I32 = call <2 x i32> @llvm.masked.expandload.v2i32(ptr undef, <2 x i1> %m2, <2 x i32> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 108 for instruction: %V32I16 = call <32 x i16> @llvm.masked.expandload.v32i16(ptr undef, <32 x i1> %m32, <32 x i16> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 54 for instruction: %V16I16 = call <16 x i16> @llvm.masked.expandload.v16i16(ptr undef, <16 x i1> %m16, <16 x i16> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 27 for instruction: %V8I16 = call <8 x i16> @llvm.masked.expandload.v8i16(ptr undef, <8 x i1> %m8, <8 x i16> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %V4I16 = call <4 x i16> @llvm.masked.expandload.v4i16(ptr undef, <4 x i1> %m4, <4 x i16> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 214 for instruction: %V64I8 = call <64 x i8> @llvm.masked.expandload.v64i8(ptr undef, <64 x i1> %m64, <64 x i8> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 107 for instruction: %V32I8 = call <32 x i8> @llvm.masked.expandload.v32i8(ptr undef, <32 x i1> %m32, <32 x i8> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 53 for instruction: %V16I8 = call <16 x i8> @llvm.masked.expandload.v16i8(ptr undef, <16 x i1> %m16, <16 x i8> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 27 for instruction: %V8I8 = call <8 x i8> @llvm.masked.expandload.v8i8(ptr undef, <8 x i1> %m8, <8 x i8> undef)
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 0
 ;
 ; AVX2-LABEL: 'masked_expandload'
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %V8F64 = call <8 x double> @llvm.masked.expandload.v8f64(ptr undef, <8 x i1> %m8, <8 x double> undef)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V4F64 = call <4 x double> @llvm.masked.expandload.v4f64(ptr undef, <4 x i1> %m4, <4 x double> undef)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V2F64 = call <2 x double> @llvm.masked.expandload.v2f64(ptr undef, <2 x i1> %m2, <2 x double> undef)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1F64 = call <1 x double> @llvm.masked.expandload.v1f64(ptr undef, <1 x i1> %m1, <1 x double> undef)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 31 for instruction: %V16F32 = call <16 x float> @llvm.masked.expandload.v16f32(ptr undef, <16 x i1> %m16, <16 x float> undef)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V8F32 = call <8 x float> @llvm.masked.expandload.v8f32(ptr undef, <8 x i1> %m8, <8 x float> undef)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V4F32 = call <4 x float> @llvm.masked.expandload.v4f32(ptr undef, <4 x i1> %m4, <4 x float> undef)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V2F32 = call <2 x float> @llvm.masked.expandload.v2f32(ptr undef, <2 x i1> %m2, <2 x float> undef)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 19 for instruction: %V8I64 = call <8 x i64> @llvm.masked.expandload.v8i64(ptr undef, <8 x i1> %m8, <8 x i64> undef)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V4I64 = call <4 x i64> @llvm.masked.expandload.v4i64(ptr undef, <4 x i1> %m4, <4 x i64> undef)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V2I64 = call <2 x i64> @llvm.masked.expandload.v2i64(ptr undef, <2 x i1> %m2, <2 x i64> undef)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1I64 = call <1 x i64> @llvm.masked.expandload.v1i64(ptr undef, <1 x i1> %m1, <1 x i64> undef)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 35 for instruction: %V16I32 = call <16 x i32> @llvm.masked.expandload.v16i32(ptr undef, <16 x i1> %m16, <16 x i32> undef)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V8I32 = call <8 x i32> @llvm.masked.expandload.v8i32(ptr undef, <8 x i1> %m8, <8 x i32> undef)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V4I32 = call <4 x i32> @llvm.masked.expandload.v4i32(ptr undef, <4 x i1> %m4, <4 x i32> undef)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V2I32 = call <2 x i32> @llvm.masked.expandload.v2i32(ptr undef, <2 x i1> %m2, <2 x i32> undef)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 67 for instruction: %V32I16 = call <32 x i16> @llvm.masked.expandload.v32i16(ptr undef, <32 x i1> %m32, <32 x i16> undef)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %V16I16 = call <16 x i16> @llvm.masked.expandload.v16i16(ptr undef, <16 x i1> %m16, <16 x i16> undef)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %V8I16 = call <8 x i16> @llvm.masked.expandload.v8i16(ptr undef, <8 x i1> %m8, <8 x i16> undef)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V4I16 = call <4 x i16> @llvm.masked.expandload.v4i16(ptr undef, <4 x i1> %m4, <4 x i16> undef)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 132 for instruction: %V64I8 = call <64 x i8> @llvm.masked.expandload.v64i8(ptr undef, <64 x i1> %m64, <64 x i8> undef)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 66 for instruction: %V32I8 = call <32 x i8> @llvm.masked.expandload.v32i8(ptr undef, <32 x i1> %m32, <32 x i8> undef)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %V16I8 = call <16 x i8> @llvm.masked.expandload.v16i8(ptr undef, <16 x i1> %m16, <16 x i8> undef)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %V8I8 = call <8 x i8> @llvm.masked.expandload.v8i8(ptr undef, <8 x i1> %m8, <8 x i8> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: %V8F64 = call <8 x double> @llvm.masked.expandload.v8f64(ptr undef, <8 x i1> %m8, <8 x double> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %V4F64 = call <4 x double> @llvm.masked.expandload.v4f64(ptr undef, <4 x i1> %m4, <4 x double> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2F64 = call <2 x double> @llvm.masked.expandload.v2f64(ptr undef, <2 x i1> %m2, <2 x double> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V1F64 = call <1 x double> @llvm.masked.expandload.v1f64(ptr undef, <1 x i1> %m1, <1 x double> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 51 for instruction: %V16F32 = call <16 x float> @llvm.masked.expandload.v16f32(ptr undef, <16 x i1> %m16, <16 x float> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 26 for instruction: %V8F32 = call <8 x float> @llvm.masked.expandload.v8f32(ptr undef, <8 x i1> %m8, <8 x float> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %V4F32 = call <4 x float> @llvm.masked.expandload.v4f32(ptr undef, <4 x i1> %m4, <4 x float> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2F32 = call <2 x float> @llvm.masked.expandload.v2f32(ptr undef, <2 x i1> %m2, <2 x float> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 29 for instruction: %V8I64 = call <8 x i64> @llvm.masked.expandload.v8i64(ptr undef, <8 x i1> %m8, <8 x i64> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %V4I64 = call <4 x i64> @llvm.masked.expandload.v4i64(ptr undef, <4 x i1> %m4, <4 x i64> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V2I64 = call <2 x i64> @llvm.masked.expandload.v2i64(ptr undef, <2 x i1> %m2, <2 x i64> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V1I64 = call <1 x i64> @llvm.masked.expandload.v1i64(ptr undef, <1 x i1> %m1, <1 x i64> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 55 for instruction: %V16I32 = call <16 x i32> @llvm.masked.expandload.v16i32(ptr undef, <16 x i1> %m16, <16 x i32> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %V8I32 = call <8 x i32> @llvm.masked.expandload.v8i32(ptr undef, <8 x i1> %m8, <8 x i32> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %V4I32 = call <4 x i32> @llvm.masked.expandload.v4i32(ptr undef, <4 x i1> %m4, <4 x i32> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V2I32 = call <2 x i32> @llvm.masked.expandload.v2i32(ptr undef, <2 x i1> %m2, <2 x i32> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 107 for instruction: %V32I16 = call <32 x i16> @llvm.masked.expandload.v32i16(ptr undef, <32 x i1> %m32, <32 x i16> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 54 for instruction: %V16I16 = call <16 x i16> @llvm.masked.expandload.v16i16(ptr undef, <16 x i1> %m16, <16 x i16> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 27 for instruction: %V8I16 = call <8 x i16> @llvm.masked.expandload.v8i16(ptr undef, <8 x i1> %m8, <8 x i16> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %V4I16 = call <4 x i16> @llvm.masked.expandload.v4i16(ptr undef, <4 x i1> %m4, <4 x i16> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 212 for instruction: %V64I8 = call <64 x i8> @llvm.masked.expandload.v64i8(ptr undef, <64 x i1> %m64, <64 x i8> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 106 for instruction: %V32I8 = call <32 x i8> @llvm.masked.expandload.v32i8(ptr undef, <32 x i1> %m32, <32 x i8> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 53 for instruction: %V16I8 = call <16 x i8> @llvm.masked.expandload.v16i8(ptr undef, <16 x i1> %m16, <16 x i8> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 27 for instruction: %V8I8 = call <8 x i8> @llvm.masked.expandload.v8i8(ptr undef, <8 x i1> %m8, <8 x i8> undef)
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 0
 ;
 ; SKL-LABEL: 'masked_expandload'
-; SKL-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %V8F64 = call <8 x double> @llvm.masked.expandload.v8f64(ptr undef, <8 x i1> %m8, <8 x double> undef)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V4F64 = call <4 x double> @llvm.masked.expandload.v4f64(ptr undef, <4 x i1> %m4, <4 x double> undef)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V2F64 = call <2 x double> @llvm.masked.expandload.v2f64(ptr undef, <2 x i1> %m2, <2 x double> undef)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1F64 = call <1 x double> @llvm.masked.expandload.v1f64(ptr undef, <1 x i1> %m1, <1 x double> undef)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 31 for instruction: %V16F32 = call <16 x float> @llvm.masked.expandload.v16f32(ptr undef, <16 x i1> %m16, <16 x float> undef)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V8F32 = call <8 x float> @llvm.masked.expandload.v8f32(ptr undef, <8 x i1> %m8, <8 x float> undef)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V4F32 = call <4 x float> @llvm.masked.expandload.v4f32(ptr undef, <4 x i1> %m4, <4 x float> undef)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V2F32 = call <2 x float> @llvm.masked.expandload.v2f32(ptr undef, <2 x i1> %m2, <2 x float> undef)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 19 for instruction: %V8I64 = call <8 x i64> @llvm.masked.expandload.v8i64(ptr undef, <8 x i1> %m8, <8 x i64> undef)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V4I64 = call <4 x i64> @llvm.masked.expandload.v4i64(ptr undef, <4 x i1> %m4, <4 x i64> undef)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V2I64 = call <2 x i64> @llvm.masked.expandload.v2i64(ptr undef, <2 x i1> %m2, <2 x i64> undef)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1I64 = call <1 x i64> @llvm.masked.expandload.v1i64(ptr undef, <1 x i1> %m1, <1 x i64> undef)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 35 for instruction: %V16I32 = call <16 x i32> @llvm.masked.expandload.v16i32(ptr undef, <16 x i1> %m16, <16 x i32> undef)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V8I32 = call <8 x i32> @llvm.masked.expandload.v8i32(ptr undef, <8 x i1> %m8, <8 x i32> undef)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V4I32 = call <4 x i32> @llvm.masked.expandload.v4i32(ptr undef, <4 x i1> %m4, <4 x i32> undef)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V2I32 = call <2 x i32> @llvm.masked.expandload.v2i32(ptr undef, <2 x i1> %m2, <2 x i32> undef)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 67 for instruction: %V32I16 = call <32 x i16> @llvm.masked.expandload.v32i16(ptr undef, <32 x i1> %m32, <32 x i16> undef)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %V16I16 = call <16 x i16> @llvm.masked.expandload.v16i16(ptr undef, <16 x i1> %m16, <16 x i16> undef)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %V8I16 = call <8 x i16> @llvm.masked.expandload.v8i16(ptr undef, <8 x i1> %m8, <8 x i16> undef)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V4I16 = call <4 x i16> @llvm.masked.expandload.v4i16(ptr undef, <4 x i1> %m4, <4 x i16> undef)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 132 for instruction: %V64I8 = call <64 x i8> @llvm.masked.expandload.v64i8(ptr undef, <64 x i1> %m64, <64 x i8> undef)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 66 for instruction: %V32I8 = call <32 x i8> @llvm.masked.expandload.v32i8(ptr undef, <32 x i1> %m32, <32 x i8> undef)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %V16I8 = call <16 x i8> @llvm.masked.expandload.v16i8(ptr undef, <16 x i1> %m16, <16 x i8> undef)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %V8I8 = call <8 x i8> @llvm.masked.expandload.v8i8(ptr undef, <8 x i1> %m8, <8 x i8> undef)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: %V8F64 = call <8 x double> @llvm.masked.expandload.v8f64(ptr undef, <8 x i1> %m8, <8 x double> undef)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %V4F64 = call <4 x double> @llvm.masked.expandload.v4f64(ptr undef, <4 x i1> %m4, <4 x double> undef)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2F64 = call <2 x double> @llvm.masked.expandload.v2f64(ptr undef, <2 x i1> %m2, <2 x double> undef)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V1F64 = call <1 x double> @llvm.masked.expandload.v1f64(ptr undef, <1 x i1> %m1, <1 x double> undef)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 51 for instruction: %V16F32 = call <16 x float> @llvm.masked.expandload.v16f32(ptr undef, <16 x i1> %m16, <16 x float> undef)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 26 for instruction: %V8F32 = call <8 x float> @llvm.masked.expandload.v8f32(ptr undef, <8 x i1> %m8, <8 x float> undef)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %V4F32 = call <4 x float> @llvm.masked.expandload.v4f32(ptr undef, <4 x i1> %m4, <4 x float> undef)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2F32 = call <2 x float> @llvm.masked.expandload.v2f32(ptr undef, <2 x i1> %m2, <2 x float> undef)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 29 for instruction: %V8I64 = call <8 x i64> @llvm.masked.expandload.v8i64(ptr undef, <8 x i1> %m8, <8 x i64> undef)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %V4I64 = call <4 x i64> @llvm.masked.expandload.v4i64(ptr undef, <4 x i1> %m4, <4 x i64> undef)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V2I64 = call <2 x i64> @llvm.masked.expandload.v2i64(ptr undef, <2 x i1> %m2, <2 x i64> undef)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V1I64 = call <1 x i64> @llvm.masked.expandload.v1i64(ptr undef, <1 x i1> %m1, <1 x i64> undef)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 55 for instruction: %V16I32 = call <16 x i32> @llvm.masked.expandload.v16i32(ptr undef, <16 x i1> %m16, <16 x i32> undef)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %V8I32 = call <8 x i32> @llvm.masked.expandload.v8i32(ptr undef, <8 x i1> %m8, <8 x i32> undef)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %V4I32 = call <4 x i32> @llvm.masked.expandload.v4i32(ptr undef, <4 x i1> %m4, <4 x i32> undef)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V2I32 = call <2 x i32> @llvm.masked.expandload.v2i32(ptr undef, <2 x i1> %m2, <2 x i32> undef)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 107 for instruction: %V32I16 = call <32 x i16> @llvm.masked.expandload.v32i16(ptr undef, <32 x i1> %m32, <32 x i16> undef)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 54 for instruction: %V16I16 = call <16 x i16> @llvm.masked.expandload.v16i16(ptr undef, <16 x i1> %m16, <16 x i16> undef)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 27 for instruction: %V8I16 = call <8 x i16> @llvm.masked.expandload.v8i16(ptr undef, <8 x i1> %m8, <8 x i16> undef)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %V4I16 = call <4 x i16> @llvm.masked.expandload.v4i16(ptr undef, <4 x i1> %m4, <4 x i16> undef)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 212 for instruction: %V64I8 = call <64 x i8> @llvm.masked.expandload.v64i8(ptr undef, <64 x i1> %m64, <64 x i8> undef)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 106 for instruction: %V32I8 = call <32 x i8> @llvm.masked.expandload.v32i8(ptr undef, <32 x i1> %m32, <32 x i8> undef)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 53 for instruction: %V16I8 = call <16 x i8> @llvm.masked.expandload.v16i8(ptr undef, <16 x i1> %m16, <16 x i8> undef)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 27 for instruction: %V8I8 = call <8 x i8> @llvm.masked.expandload.v8i8(ptr undef, <8 x i1> %m8, <8 x i8> undef)
 ; SKL-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 0
 ;
 ; AVX512-LABEL: 'masked_expandload'
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 23 for instruction: %V8F64 = call <8 x double> @llvm.masked.expandload.v8f64(ptr undef, <8 x i1> %m8, <8 x double> undef)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %V4F64 = call <4 x double> @llvm.masked.expandload.v4f64(ptr undef, <4 x i1> %m4, <4 x double> undef)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V2F64 = call <2 x double> @llvm.masked.expandload.v2f64(ptr undef, <2 x i1> %m2, <2 x double> undef)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1F64 = call <1 x double> @llvm.masked.expandload.v1f64(ptr undef, <1 x i1> %m1, <1 x double> undef)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 47 for instruction: %V16F32 = call <16 x float> @llvm.masked.expandload.v16f32(ptr undef, <16 x i1> %m16, <16 x float> undef)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 23 for instruction: %V8F32 = call <8 x float> @llvm.masked.expandload.v8f32(ptr undef, <8 x i1> %m8, <8 x float> undef)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %V4F32 = call <4 x float> @llvm.masked.expandload.v4f32(ptr undef, <4 x i1> %m4, <4 x float> undef)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V2F32 = call <2 x float> @llvm.masked.expandload.v2f32(ptr undef, <2 x i1> %m2, <2 x float> undef)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 27 for instruction: %V8I64 = call <8 x i64> @llvm.masked.expandload.v8i64(ptr undef, <8 x i1> %m8, <8 x i64> undef)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %V4I64 = call <4 x i64> @llvm.masked.expandload.v4i64(ptr undef, <4 x i1> %m4, <4 x i64> undef)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2I64 = call <2 x i64> @llvm.masked.expandload.v2i64(ptr undef, <2 x i1> %m2, <2 x i64> undef)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1I64 = call <1 x i64> @llvm.masked.expandload.v1i64(ptr undef, <1 x i1> %m1, <1 x i64> undef)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 51 for instruction: %V16I32 = call <16 x i32> @llvm.masked.expandload.v16i32(ptr undef, <16 x i1> %m16, <16 x i32> undef)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: %V8I32 = call <8 x i32> @llvm.masked.expandload.v8i32(ptr undef, <8 x i1> %m8, <8 x i32> undef)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V4I32 = call <4 x i32> @llvm.masked.expandload.v4i32(ptr undef, <4 x i1> %m4, <4 x i32> undef)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2I32 = call <2 x i32> @llvm.masked.expandload.v2i32(ptr undef, <2 x i1> %m2, <2 x i32> undef)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 99 for instruction: %V32I16 = call <32 x i16> @llvm.masked.expandload.v32i16(ptr undef, <32 x i1> %m32, <32 x i16> undef)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 49 for instruction: %V16I16 = call <16 x i16> @llvm.masked.expandload.v16i16(ptr undef, <16 x i1> %m16, <16 x i16> undef)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V8I16 = call <8 x i16> @llvm.masked.expandload.v8i16(ptr undef, <8 x i1> %m8, <8 x i16> undef)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V4I16 = call <4 x i16> @llvm.masked.expandload.v4i16(ptr undef, <4 x i1> %m4, <4 x i16> undef)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 195 for instruction: %V64I8 = call <64 x i8> @llvm.masked.expandload.v64i8(ptr undef, <64 x i1> %m64, <64 x i8> undef)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 97 for instruction: %V32I8 = call <32 x i8> @llvm.masked.expandload.v32i8(ptr undef, <32 x i1> %m32, <32 x i8> undef)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V16I8 = call <16 x i8> @llvm.masked.expandload.v16i8(ptr undef, <16 x i1> %m16, <16 x i8> undef)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V8I8 = call <8 x i8> @llvm.masked.expandload.v8i8(ptr undef, <8 x i1> %m8, <8 x i8> undef)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %V8F64 = call <8 x double> @llvm.masked.expandload.v8f64(ptr undef, <8 x i1> %m8, <8 x double> undef)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V4F64 = call <4 x double> @llvm.masked.expandload.v4f64(ptr undef, <4 x i1> %m4, <4 x double> undef)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V2F64 = call <2 x double> @llvm.masked.expandload.v2f64(ptr undef, <2 x i1> %m2, <2 x double> undef)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V1F64 = call <1 x double> @llvm.masked.expandload.v1f64(ptr undef, <1 x i1> %m1, <1 x double> undef)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 69 for instruction: %V16F32 = call <16 x float> @llvm.masked.expandload.v16f32(ptr undef, <16 x i1> %m16, <16 x float> undef)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %V8F32 = call <8 x float> @llvm.masked.expandload.v8f32(ptr undef, <8 x i1> %m8, <8 x float> undef)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V4F32 = call <4 x float> @llvm.masked.expandload.v4f32(ptr undef, <4 x i1> %m4, <4 x float> undef)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V2F32 = call <2 x float> @llvm.masked.expandload.v2f32(ptr undef, <2 x i1> %m2, <2 x float> undef)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 38 for instruction: %V8I64 = call <8 x i64> @llvm.masked.expandload.v8i64(ptr undef, <8 x i1> %m8, <8 x i64> undef)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V4I64 = call <4 x i64> @llvm.masked.expandload.v4i64(ptr undef, <4 x i1> %m4, <4 x i64> undef)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V2I64 = call <2 x i64> @llvm.masked.expandload.v2i64(ptr undef, <2 x i1> %m2, <2 x i64> undef)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V1I64 = call <1 x i64> @llvm.masked.expandload.v1i64(ptr undef, <1 x i1> %m1, <1 x i64> undef)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 73 for instruction: %V16I32 = call <16 x i32> @llvm.masked.expandload.v16i32(ptr undef, <16 x i1> %m16, <16 x i32> undef)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 36 for instruction: %V8I32 = call <8 x i32> @llvm.masked.expandload.v8i32(ptr undef, <8 x i1> %m8, <8 x i32> undef)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %V4I32 = call <4 x i32> @llvm.masked.expandload.v4i32(ptr undef, <4 x i1> %m4, <4 x i32> undef)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V2I32 = call <2 x i32> @llvm.masked.expandload.v2i32(ptr undef, <2 x i1> %m2, <2 x i32> undef)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 143 for instruction: %V32I16 = call <32 x i16> @llvm.masked.expandload.v32i16(ptr undef, <32 x i1> %m32, <32 x i16> undef)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 71 for instruction: %V16I16 = call <16 x i16> @llvm.masked.expandload.v16i16(ptr undef, <16 x i1> %m16, <16 x i16> undef)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 35 for instruction: %V8I16 = call <8 x i16> @llvm.masked.expandload.v8i16(ptr undef, <8 x i1> %m8, <8 x i16> undef)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %V4I16 = call <4 x i16> @llvm.masked.expandload.v4i16(ptr undef, <4 x i1> %m4, <4 x i16> undef)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 283 for instruction: %V64I8 = call <64 x i8> @llvm.masked.expandload.v64i8(ptr undef, <64 x i1> %m64, <64 x i8> undef)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 141 for instruction: %V32I8 = call <32 x i8> @llvm.masked.expandload.v32i8(ptr undef, <32 x i1> %m32, <32 x i8> undef)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 70 for instruction: %V16I8 = call <16 x i8> @llvm.masked.expandload.v16i8(ptr undef, <16 x i1> %m16, <16 x i8> undef)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 35 for instruction: %V8I8 = call <8 x i8> @llvm.masked.expandload.v8i8(ptr undef, <8 x i1> %m8, <8 x i8> undef)
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 0
 ;
   %V8F64 = call <8 x double> @llvm.masked.expandload.v8f64(ptr undef, <8 x i1> %m8, <8 x double> undef)
@@ -1374,165 +1374,165 @@ define i32 @masked_expandload(<1 x i1> %m1, <2 x i1> %m2, <4 x i1> %m4, <8 x i1>
 
 define i32 @masked_compressstore(<1 x i1> %m1, <2 x i1> %m2, <4 x i1> %m4, <8 x i1> %m8, <16 x i1> %m16, <32 x i1> %m32, <64 x i1> %m64) {
 ; SSE2-LABEL: 'masked_compressstore'
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: call void @llvm.masked.compressstore.v8f64(<8 x double> undef, ptr undef, <8 x i1> %m8)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: call void @llvm.masked.compressstore.v4f64(<4 x double> undef, ptr undef, <4 x i1> %m4)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: call void @llvm.masked.compressstore.v2f64(<2 x double> undef, ptr undef, <2 x i1> %m2)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.compressstore.v1f64(<1 x double> undef, ptr undef, <1 x i1> %m1)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 29 for instruction: call void @llvm.masked.compressstore.v16f32(<16 x float> undef, ptr undef, <16 x i1> %m16)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: call void @llvm.masked.compressstore.v8f32(<8 x float> undef, ptr undef, <8 x i1> %m8)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: call void @llvm.masked.compressstore.v4f32(<4 x float> undef, ptr undef, <4 x i1> %m4)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: call void @llvm.masked.compressstore.v2f32(<2 x float> undef, ptr undef, <2 x i1> %m2)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 21 for instruction: call void @llvm.masked.compressstore.v8i64(<8 x i64> undef, ptr undef, <8 x i1> %m8)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: call void @llvm.masked.compressstore.v4i64(<4 x i64> undef, ptr undef, <4 x i1> %m4)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: call void @llvm.masked.compressstore.v2i64(<2 x i64> undef, ptr undef, <2 x i1> %m2)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.compressstore.v1i64(<1 x i64> undef, ptr undef, <1 x i1> %m1)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 45 for instruction: call void @llvm.masked.compressstore.v16i32(<16 x i32> undef, ptr undef, <16 x i1> %m16)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 23 for instruction: call void @llvm.masked.compressstore.v8i32(<8 x i32> undef, ptr undef, <8 x i1> %m8)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: call void @llvm.masked.compressstore.v4i32(<4 x i32> undef, ptr undef, <4 x i1> %m4)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: call void @llvm.masked.compressstore.v2i32(<2 x i32> undef, ptr undef, <2 x i1> %m2)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 66 for instruction: call void @llvm.masked.compressstore.v32i16(<32 x i16> undef, ptr undef, <32 x i1> %m32)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: call void @llvm.masked.compressstore.v16i16(<16 x i16> undef, ptr undef, <16 x i1> %m16)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: call void @llvm.masked.compressstore.v8i16(<8 x i16> undef, ptr undef, <8 x i1> %m8)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: call void @llvm.masked.compressstore.v4i16(<4 x i16> undef, ptr undef, <4 x i1> %m4)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 192 for instruction: call void @llvm.masked.compressstore.v64i8(<64 x i8> undef, ptr undef, <64 x i1> %m64)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 96 for instruction: call void @llvm.masked.compressstore.v32i8(<32 x i8> undef, ptr undef, <32 x i1> %m32)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: call void @llvm.masked.compressstore.v16i8(<16 x i8> undef, ptr undef, <16 x i1> %m16)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: call void @llvm.masked.compressstore.v8i8(<8 x i8> undef, ptr undef, <8 x i1> %m8)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 29 for instruction: call void @llvm.masked.compressstore.v8f64(<8 x double> undef, ptr undef, <8 x i1> %m8)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: call void @llvm.masked.compressstore.v4f64(<4 x double> undef, ptr undef, <4 x i1> %m4)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: call void @llvm.masked.compressstore.v2f64(<2 x double> undef, ptr undef, <2 x i1> %m2)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: call void @llvm.masked.compressstore.v1f64(<1 x double> undef, ptr undef, <1 x i1> %m1)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 61 for instruction: call void @llvm.masked.compressstore.v16f32(<16 x float> undef, ptr undef, <16 x i1> %m16)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 31 for instruction: call void @llvm.masked.compressstore.v8f32(<8 x float> undef, ptr undef, <8 x i1> %m8)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: call void @llvm.masked.compressstore.v4f32(<4 x float> undef, ptr undef, <4 x i1> %m4)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: call void @llvm.masked.compressstore.v2f32(<2 x float> undef, ptr undef, <2 x i1> %m2)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 37 for instruction: call void @llvm.masked.compressstore.v8i64(<8 x i64> undef, ptr undef, <8 x i1> %m8)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 19 for instruction: call void @llvm.masked.compressstore.v4i64(<4 x i64> undef, ptr undef, <4 x i1> %m4)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: call void @llvm.masked.compressstore.v2i64(<2 x i64> undef, ptr undef, <2 x i1> %m2)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: call void @llvm.masked.compressstore.v1i64(<1 x i64> undef, ptr undef, <1 x i1> %m1)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 77 for instruction: call void @llvm.masked.compressstore.v16i32(<16 x i32> undef, ptr undef, <16 x i1> %m16)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 39 for instruction: call void @llvm.masked.compressstore.v8i32(<8 x i32> undef, ptr undef, <8 x i1> %m8)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: call void @llvm.masked.compressstore.v4i32(<4 x i32> undef, ptr undef, <4 x i1> %m4)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: call void @llvm.masked.compressstore.v2i32(<2 x i32> undef, ptr undef, <2 x i1> %m2)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 130 for instruction: call void @llvm.masked.compressstore.v32i16(<32 x i16> undef, ptr undef, <32 x i1> %m32)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 65 for instruction: call void @llvm.masked.compressstore.v16i16(<16 x i16> undef, ptr undef, <16 x i1> %m16)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: call void @llvm.masked.compressstore.v8i16(<8 x i16> undef, ptr undef, <8 x i1> %m8)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: call void @llvm.masked.compressstore.v4i16(<4 x i16> undef, ptr undef, <4 x i1> %m4)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 320 for instruction: call void @llvm.masked.compressstore.v64i8(<64 x i8> undef, ptr undef, <64 x i1> %m64)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: call void @llvm.masked.compressstore.v32i8(<32 x i8> undef, ptr undef, <32 x i1> %m32)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: call void @llvm.masked.compressstore.v16i8(<16 x i8> undef, ptr undef, <16 x i1> %m16)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: call void @llvm.masked.compressstore.v8i8(<8 x i8> undef, ptr undef, <8 x i1> %m8)
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 0
 ;
 ; SSE42-LABEL: 'masked_compressstore'
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: call void @llvm.masked.compressstore.v8f64(<8 x double> undef, ptr undef, <8 x i1> %m8)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: call void @llvm.masked.compressstore.v4f64(<4 x double> undef, ptr undef, <4 x i1> %m4)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: call void @llvm.masked.compressstore.v2f64(<2 x double> undef, ptr undef, <2 x i1> %m2)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.compressstore.v1f64(<1 x double> undef, ptr undef, <1 x i1> %m1)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 29 for instruction: call void @llvm.masked.compressstore.v16f32(<16 x float> undef, ptr undef, <16 x i1> %m16)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: call void @llvm.masked.compressstore.v8f32(<8 x float> undef, ptr undef, <8 x i1> %m8)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: call void @llvm.masked.compressstore.v4f32(<4 x float> undef, ptr undef, <4 x i1> %m4)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: call void @llvm.masked.compressstore.v2f32(<2 x float> undef, ptr undef, <2 x i1> %m2)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: call void @llvm.masked.compressstore.v8i64(<8 x i64> undef, ptr undef, <8 x i1> %m8)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: call void @llvm.masked.compressstore.v4i64(<4 x i64> undef, ptr undef, <4 x i1> %m4)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: call void @llvm.masked.compressstore.v2i64(<2 x i64> undef, ptr undef, <2 x i1> %m2)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.compressstore.v1i64(<1 x i64> undef, ptr undef, <1 x i1> %m1)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: call void @llvm.masked.compressstore.v16i32(<16 x i32> undef, ptr undef, <16 x i1> %m16)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: call void @llvm.masked.compressstore.v8i32(<8 x i32> undef, ptr undef, <8 x i1> %m8)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: call void @llvm.masked.compressstore.v4i32(<4 x i32> undef, ptr undef, <4 x i1> %m4)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: call void @llvm.masked.compressstore.v2i32(<2 x i32> undef, ptr undef, <2 x i1> %m2)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 66 for instruction: call void @llvm.masked.compressstore.v32i16(<32 x i16> undef, ptr undef, <32 x i1> %m32)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: call void @llvm.masked.compressstore.v16i16(<16 x i16> undef, ptr undef, <16 x i1> %m16)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: call void @llvm.masked.compressstore.v8i16(<8 x i16> undef, ptr undef, <8 x i1> %m8)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: call void @llvm.masked.compressstore.v4i16(<4 x i16> undef, ptr undef, <4 x i1> %m4)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 132 for instruction: call void @llvm.masked.compressstore.v64i8(<64 x i8> undef, ptr undef, <64 x i1> %m64)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 66 for instruction: call void @llvm.masked.compressstore.v32i8(<32 x i8> undef, ptr undef, <32 x i1> %m32)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: call void @llvm.masked.compressstore.v16i8(<16 x i8> undef, ptr undef, <16 x i1> %m16)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: call void @llvm.masked.compressstore.v8i8(<8 x i8> undef, ptr undef, <8 x i1> %m8)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 21 for instruction: call void @llvm.masked.compressstore.v8f64(<8 x double> undef, ptr undef, <8 x i1> %m8)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: call void @llvm.masked.compressstore.v4f64(<4 x double> undef, ptr undef, <4 x i1> %m4)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: call void @llvm.masked.compressstore.v2f64(<2 x double> undef, ptr undef, <2 x i1> %m2)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: call void @llvm.masked.compressstore.v1f64(<1 x double> undef, ptr undef, <1 x i1> %m1)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 45 for instruction: call void @llvm.masked.compressstore.v16f32(<16 x float> undef, ptr undef, <16 x i1> %m16)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 23 for instruction: call void @llvm.masked.compressstore.v8f32(<8 x float> undef, ptr undef, <8 x i1> %m8)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: call void @llvm.masked.compressstore.v4f32(<4 x float> undef, ptr undef, <4 x i1> %m4)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: call void @llvm.masked.compressstore.v2f32(<2 x float> undef, ptr undef, <2 x i1> %m2)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: call void @llvm.masked.compressstore.v8i64(<8 x i64> undef, ptr undef, <8 x i1> %m8)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: call void @llvm.masked.compressstore.v4i64(<4 x i64> undef, ptr undef, <4 x i1> %m4)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: call void @llvm.masked.compressstore.v2i64(<2 x i64> undef, ptr undef, <2 x i1> %m2)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: call void @llvm.masked.compressstore.v1i64(<1 x i64> undef, ptr undef, <1 x i1> %m1)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 49 for instruction: call void @llvm.masked.compressstore.v16i32(<16 x i32> undef, ptr undef, <16 x i1> %m16)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: call void @llvm.masked.compressstore.v8i32(<8 x i32> undef, ptr undef, <8 x i1> %m8)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: call void @llvm.masked.compressstore.v4i32(<4 x i32> undef, ptr undef, <4 x i1> %m4)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: call void @llvm.masked.compressstore.v2i32(<2 x i32> undef, ptr undef, <2 x i1> %m2)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 98 for instruction: call void @llvm.masked.compressstore.v32i16(<32 x i16> undef, ptr undef, <32 x i1> %m32)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 49 for instruction: call void @llvm.masked.compressstore.v16i16(<16 x i16> undef, ptr undef, <16 x i1> %m16)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: call void @llvm.masked.compressstore.v8i16(<8 x i16> undef, ptr undef, <8 x i1> %m8)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: call void @llvm.masked.compressstore.v4i16(<4 x i16> undef, ptr undef, <4 x i1> %m4)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 196 for instruction: call void @llvm.masked.compressstore.v64i8(<64 x i8> undef, ptr undef, <64 x i1> %m64)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 98 for instruction: call void @llvm.masked.compressstore.v32i8(<32 x i8> undef, ptr undef, <32 x i1> %m32)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 49 for instruction: call void @llvm.masked.compressstore.v16i8(<16 x i8> undef, ptr undef, <16 x i1> %m16)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: call void @llvm.masked.compressstore.v8i8(<8 x i8> undef, ptr undef, <8 x i1> %m8)
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 0
 ;
 ; AVX1-LABEL: 'masked_compressstore'
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: call void @llvm.masked.compressstore.v8f64(<8 x double> undef, ptr undef, <8 x i1> %m8)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: call void @llvm.masked.compressstore.v4f64(<4 x double> undef, ptr undef, <4 x i1> %m4)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: call void @llvm.masked.compressstore.v2f64(<2 x double> undef, ptr undef, <2 x i1> %m2)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.compressstore.v1f64(<1 x double> undef, ptr undef, <1 x i1> %m1)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 31 for instruction: call void @llvm.masked.compressstore.v16f32(<16 x float> undef, ptr undef, <16 x i1> %m16)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: call void @llvm.masked.compressstore.v8f32(<8 x float> undef, ptr undef, <8 x i1> %m8)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: call void @llvm.masked.compressstore.v4f32(<4 x float> undef, ptr undef, <4 x i1> %m4)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: call void @llvm.masked.compressstore.v2f32(<2 x float> undef, ptr undef, <2 x i1> %m2)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 19 for instruction: call void @llvm.masked.compressstore.v8i64(<8 x i64> undef, ptr undef, <8 x i1> %m8)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: call void @llvm.masked.compressstore.v4i64(<4 x i64> undef, ptr undef, <4 x i1> %m4)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: call void @llvm.masked.compressstore.v2i64(<2 x i64> undef, ptr undef, <2 x i1> %m2)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.compressstore.v1i64(<1 x i64> undef, ptr undef, <1 x i1> %m1)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 35 for instruction: call void @llvm.masked.compressstore.v16i32(<16 x i32> undef, ptr undef, <16 x i1> %m16)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: call void @llvm.masked.compressstore.v8i32(<8 x i32> undef, ptr undef, <8 x i1> %m8)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: call void @llvm.masked.compressstore.v4i32(<4 x i32> undef, ptr undef, <4 x i1> %m4)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: call void @llvm.masked.compressstore.v2i32(<2 x i32> undef, ptr undef, <2 x i1> %m2)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 68 for instruction: call void @llvm.masked.compressstore.v32i16(<32 x i16> undef, ptr undef, <32 x i1> %m32)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: call void @llvm.masked.compressstore.v16i16(<16 x i16> undef, ptr undef, <16 x i1> %m16)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: call void @llvm.masked.compressstore.v8i16(<8 x i16> undef, ptr undef, <8 x i1> %m8)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: call void @llvm.masked.compressstore.v4i16(<4 x i16> undef, ptr undef, <4 x i1> %m4)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 134 for instruction: call void @llvm.masked.compressstore.v64i8(<64 x i8> undef, ptr undef, <64 x i1> %m64)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 67 for instruction: call void @llvm.masked.compressstore.v32i8(<32 x i8> undef, ptr undef, <32 x i1> %m32)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: call void @llvm.masked.compressstore.v16i8(<16 x i8> undef, ptr undef, <16 x i1> %m16)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: call void @llvm.masked.compressstore.v8i8(<8 x i8> undef, ptr undef, <8 x i1> %m8)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: call void @llvm.masked.compressstore.v8f64(<8 x double> undef, ptr undef, <8 x i1> %m8)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: call void @llvm.masked.compressstore.v4f64(<4 x double> undef, ptr undef, <4 x i1> %m4)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: call void @llvm.masked.compressstore.v2f64(<2 x double> undef, ptr undef, <2 x i1> %m2)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: call void @llvm.masked.compressstore.v1f64(<1 x double> undef, ptr undef, <1 x i1> %m1)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 51 for instruction: call void @llvm.masked.compressstore.v16f32(<16 x float> undef, ptr undef, <16 x i1> %m16)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 26 for instruction: call void @llvm.masked.compressstore.v8f32(<8 x float> undef, ptr undef, <8 x i1> %m8)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: call void @llvm.masked.compressstore.v4f32(<4 x float> undef, ptr undef, <4 x i1> %m4)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: call void @llvm.masked.compressstore.v2f32(<2 x float> undef, ptr undef, <2 x i1> %m2)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 29 for instruction: call void @llvm.masked.compressstore.v8i64(<8 x i64> undef, ptr undef, <8 x i1> %m8)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: call void @llvm.masked.compressstore.v4i64(<4 x i64> undef, ptr undef, <4 x i1> %m4)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: call void @llvm.masked.compressstore.v2i64(<2 x i64> undef, ptr undef, <2 x i1> %m2)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: call void @llvm.masked.compressstore.v1i64(<1 x i64> undef, ptr undef, <1 x i1> %m1)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 55 for instruction: call void @llvm.masked.compressstore.v16i32(<16 x i32> undef, ptr undef, <16 x i1> %m16)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: call void @llvm.masked.compressstore.v8i32(<8 x i32> undef, ptr undef, <8 x i1> %m8)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: call void @llvm.masked.compressstore.v4i32(<4 x i32> undef, ptr undef, <4 x i1> %m4)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: call void @llvm.masked.compressstore.v2i32(<2 x i32> undef, ptr undef, <2 x i1> %m2)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 108 for instruction: call void @llvm.masked.compressstore.v32i16(<32 x i16> undef, ptr undef, <32 x i1> %m32)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 54 for instruction: call void @llvm.masked.compressstore.v16i16(<16 x i16> undef, ptr undef, <16 x i1> %m16)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 27 for instruction: call void @llvm.masked.compressstore.v8i16(<8 x i16> undef, ptr undef, <8 x i1> %m8)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: call void @llvm.masked.compressstore.v4i16(<4 x i16> undef, ptr undef, <4 x i1> %m4)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 214 for instruction: call void @llvm.masked.compressstore.v64i8(<64 x i8> undef, ptr undef, <64 x i1> %m64)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 107 for instruction: call void @llvm.masked.compressstore.v32i8(<32 x i8> undef, ptr undef, <32 x i1> %m32)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 53 for instruction: call void @llvm.masked.compressstore.v16i8(<16 x i8> undef, ptr undef, <16 x i1> %m16)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 27 for instruction: call void @llvm.masked.compressstore.v8i8(<8 x i8> undef, ptr undef, <8 x i1> %m8)
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 0
 ;
 ; AVX2-LABEL: 'masked_compressstore'
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: call void @llvm.masked.compressstore.v8f64(<8 x double> undef, ptr undef, <8 x i1> %m8)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: call void @llvm.masked.compressstore.v4f64(<4 x double> undef, ptr undef, <4 x i1> %m4)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: call void @llvm.masked.compressstore.v2f64(<2 x double> undef, ptr undef, <2 x i1> %m2)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.compressstore.v1f64(<1 x double> undef, ptr undef, <1 x i1> %m1)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 31 for instruction: call void @llvm.masked.compressstore.v16f32(<16 x float> undef, ptr undef, <16 x i1> %m16)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: call void @llvm.masked.compressstore.v8f32(<8 x float> undef, ptr undef, <8 x i1> %m8)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: call void @llvm.masked.compressstore.v4f32(<4 x float> undef, ptr undef, <4 x i1> %m4)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: call void @llvm.masked.compressstore.v2f32(<2 x float> undef, ptr undef, <2 x i1> %m2)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 19 for instruction: call void @llvm.masked.compressstore.v8i64(<8 x i64> undef, ptr undef, <8 x i1> %m8)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: call void @llvm.masked.compressstore.v4i64(<4 x i64> undef, ptr undef, <4 x i1> %m4)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: call void @llvm.masked.compressstore.v2i64(<2 x i64> undef, ptr undef, <2 x i1> %m2)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.compressstore.v1i64(<1 x i64> undef, ptr undef, <1 x i1> %m1)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 35 for instruction: call void @llvm.masked.compressstore.v16i32(<16 x i32> undef, ptr undef, <16 x i1> %m16)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: call void @llvm.masked.compressstore.v8i32(<8 x i32> undef, ptr undef, <8 x i1> %m8)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: call void @llvm.masked.compressstore.v4i32(<4 x i32> undef, ptr undef, <4 x i1> %m4)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: call void @llvm.masked.compressstore.v2i32(<2 x i32> undef, ptr undef, <2 x i1> %m2)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 67 for instruction: call void @llvm.masked.compressstore.v32i16(<32 x i16> undef, ptr undef, <32 x i1> %m32)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: call void @llvm.masked.compressstore.v16i16(<16 x i16> undef, ptr undef, <16 x i1> %m16)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: call void @llvm.masked.compressstore.v8i16(<8 x i16> undef, ptr undef, <8 x i1> %m8)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: call void @llvm.masked.compressstore.v4i16(<4 x i16> undef, ptr undef, <4 x i1> %m4)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 132 for instruction: call void @llvm.masked.compressstore.v64i8(<64 x i8> undef, ptr undef, <64 x i1> %m64)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 66 for instruction: call void @llvm.masked.compressstore.v32i8(<32 x i8> undef, ptr undef, <32 x i1> %m32)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: call void @llvm.masked.compressstore.v16i8(<16 x i8> undef, ptr undef, <16 x i1> %m16)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: call void @llvm.masked.compressstore.v8i8(<8 x i8> undef, ptr undef, <8 x i1> %m8)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: call void @llvm.masked.compressstore.v8f64(<8 x double> undef, ptr undef, <8 x i1> %m8)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: call void @llvm.masked.compressstore.v4f64(<4 x double> undef, ptr undef, <4 x i1> %m4)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: call void @llvm.masked.compressstore.v2f64(<2 x double> undef, ptr undef, <2 x i1> %m2)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: call void @llvm.masked.compressstore.v1f64(<1 x double> undef, ptr undef, <1 x i1> %m1)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 51 for instruction: call void @llvm.masked.compressstore.v16f32(<16 x float> undef, ptr undef, <16 x i1> %m16)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 26 for instruction: call void @llvm.masked.compressstore.v8f32(<8 x float> undef, ptr undef, <8 x i1> %m8)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: call void @llvm.masked.compressstore.v4f32(<4 x float> undef, ptr undef, <4 x i1> %m4)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: call void @llvm.masked.compressstore.v2f32(<2 x float> undef, ptr undef, <2 x i1> %m2)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 29 for instruction: call void @llvm.masked.compressstore.v8i64(<8 x i64> undef, ptr undef, <8 x i1> %m8)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: call void @llvm.masked.compressstore.v4i64(<4 x i64> undef, ptr undef, <4 x i1> %m4)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: call void @llvm.masked.compressstore.v2i64(<2 x i64> undef, ptr undef, <2 x i1> %m2)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: call void @llvm.masked.compressstore.v1i64(<1 x i64> undef, ptr undef, <1 x i1> %m1)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 55 for instruction: call void @llvm.masked.compressstore.v16i32(<16 x i32> undef, ptr undef, <16 x i1> %m16)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: call void @llvm.masked.compressstore.v8i32(<8 x i32> undef, ptr undef, <8 x i1> %m8)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: call void @llvm.masked.compressstore.v4i32(<4 x i32> undef, ptr undef, <4 x i1> %m4)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: call void @llvm.masked.compressstore.v2i32(<2 x i32> undef, ptr undef, <2 x i1> %m2)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 107 for instruction: call void @llvm.masked.compressstore.v32i16(<32 x i16> undef, ptr undef, <32 x i1> %m32)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 54 for instruction: call void @llvm.masked.compressstore.v16i16(<16 x i16> undef, ptr undef, <16 x i1> %m16)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 27 for instruction: call void @llvm.masked.compressstore.v8i16(<8 x i16> undef, ptr undef, <8 x i1> %m8)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: call void @llvm.masked.compressstore.v4i16(<4 x i16> undef, ptr undef, <4 x i1> %m4)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 212 for instruction: call void @llvm.masked.compressstore.v64i8(<64 x i8> undef, ptr undef, <64 x i1> %m64)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 106 for instruction: call void @llvm.masked.compressstore.v32i8(<32 x i8> undef, ptr undef, <32 x i1> %m32)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 53 for instruction: call void @llvm.masked.compressstore.v16i8(<16 x i8> undef, ptr undef, <16 x i1> %m16)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 27 for instruction: call void @llvm.masked.compressstore.v8i8(<8 x i8> undef, ptr undef, <8 x i1> %m8)
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 0
 ;
 ; SKL-LABEL: 'masked_compressstore'
-; SKL-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: call void @llvm.masked.compressstore.v8f64(<8 x double> undef, ptr undef, <8 x i1> %m8)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: call void @llvm.masked.compressstore.v4f64(<4 x double> undef, ptr undef, <4 x i1> %m4)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: call void @llvm.masked.compressstore.v2f64(<2 x double> undef, ptr undef, <2 x i1> %m2)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.compressstore.v1f64(<1 x double> undef, ptr undef, <1 x i1> %m1)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 31 for instruction: call void @llvm.masked.compressstore.v16f32(<16 x float> undef, ptr undef, <16 x i1> %m16)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: call void @llvm.masked.compressstore.v8f32(<8 x float> undef, ptr undef, <8 x i1> %m8)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: call void @llvm.masked.compressstore.v4f32(<4 x float> undef, ptr undef, <4 x i1> %m4)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: call void @llvm.masked.compressstore.v2f32(<2 x float> undef, ptr undef, <2 x i1> %m2)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 19 for instruction: call void @llvm.masked.compressstore.v8i64(<8 x i64> undef, ptr undef, <8 x i1> %m8)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: call void @llvm.masked.compressstore.v4i64(<4 x i64> undef, ptr undef, <4 x i1> %m4)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: call void @llvm.masked.compressstore.v2i64(<2 x i64> undef, ptr undef, <2 x i1> %m2)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.compressstore.v1i64(<1 x i64> undef, ptr undef, <1 x i1> %m1)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 35 for instruction: call void @llvm.masked.compressstore.v16i32(<16 x i32> undef, ptr undef, <16 x i1> %m16)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: call void @llvm.masked.compressstore.v8i32(<8 x i32> undef, ptr undef, <8 x i1> %m8)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: call void @llvm.masked.compressstore.v4i32(<4 x i32> undef, ptr undef, <4 x i1> %m4)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: call void @llvm.masked.compressstore.v2i32(<2 x i32> undef, ptr undef, <2 x i1> %m2)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 67 for instruction: call void @llvm.masked.compressstore.v32i16(<32 x i16> undef, ptr undef, <32 x i1> %m32)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: call void @llvm.masked.compressstore.v16i16(<16 x i16> undef, ptr undef, <16 x i1> %m16)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: call void @llvm.masked.compressstore.v8i16(<8 x i16> undef, ptr undef, <8 x i1> %m8)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: call void @llvm.masked.compressstore.v4i16(<4 x i16> undef, ptr undef, <4 x i1> %m4)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 132 for instruction: call void @llvm.masked.compressstore.v64i8(<64 x i8> undef, ptr undef, <64 x i1> %m64)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 66 for instruction: call void @llvm.masked.compressstore.v32i8(<32 x i8> undef, ptr undef, <32 x i1> %m32)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: call void @llvm.masked.compressstore.v16i8(<16 x i8> undef, ptr undef, <16 x i1> %m16)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: call void @llvm.masked.compressstore.v8i8(<8 x i8> undef, ptr undef, <8 x i1> %m8)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: call void @llvm.masked.compressstore.v8f64(<8 x double> undef, ptr undef, <8 x i1> %m8)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: call void @llvm.masked.compressstore.v4f64(<4 x double> undef, ptr undef, <4 x i1> %m4)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: call void @llvm.masked.compressstore.v2f64(<2 x double> undef, ptr undef, <2 x i1> %m2)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: call void @llvm.masked.compressstore.v1f64(<1 x double> undef, ptr undef, <1 x i1> %m1)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 51 for instruction: call void @llvm.masked.compressstore.v16f32(<16 x float> undef, ptr undef, <16 x i1> %m16)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 26 for instruction: call void @llvm.masked.compressstore.v8f32(<8 x float> undef, ptr undef, <8 x i1> %m8)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: call void @llvm.masked.compressstore.v4f32(<4 x float> undef, ptr undef, <4 x i1> %m4)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: call void @llvm.masked.compressstore.v2f32(<2 x float> undef, ptr undef, <2 x i1> %m2)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 29 for instruction: call void @llvm.masked.compressstore.v8i64(<8 x i64> undef, ptr undef, <8 x i1> %m8)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: call void @llvm.masked.compressstore.v4i64(<4 x i64> undef, ptr undef, <4 x i1> %m4)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: call void @llvm.masked.compressstore.v2i64(<2 x i64> undef, ptr undef, <2 x i1> %m2)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: call void @llvm.masked.compressstore.v1i64(<1 x i64> undef, ptr undef, <1 x i1> %m1)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 55 for instruction: call void @llvm.masked.compressstore.v16i32(<16 x i32> undef, ptr undef, <16 x i1> %m16)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: call void @llvm.masked.compressstore.v8i32(<8 x i32> undef, ptr undef, <8 x i1> %m8)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: call void @llvm.masked.compressstore.v4i32(<4 x i32> undef, ptr undef, <4 x i1> %m4)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: call void @llvm.masked.compressstore.v2i32(<2 x i32> undef, ptr undef, <2 x i1> %m2)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 107 for instruction: call void @llvm.masked.compressstore.v32i16(<32 x i16> undef, ptr undef, <32 x i1> %m32)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 54 for instruction: call void @llvm.masked.compressstore.v16i16(<16 x i16> undef, ptr undef, <16 x i1> %m16)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 27 for instruction: call void @llvm.masked.compressstore.v8i16(<8 x i16> undef, ptr undef, <8 x i1> %m8)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: call void @llvm.masked.compressstore.v4i16(<4 x i16> undef, ptr undef, <4 x i1> %m4)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 212 for instruction: call void @llvm.masked.compressstore.v64i8(<64 x i8> undef, ptr undef, <64 x i1> %m64)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 106 for instruction: call void @llvm.masked.compressstore.v32i8(<32 x i8> undef, ptr undef, <32 x i1> %m32)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 53 for instruction: call void @llvm.masked.compressstore.v16i8(<16 x i8> undef, ptr undef, <16 x i1> %m16)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 27 for instruction: call void @llvm.masked.compressstore.v8i8(<8 x i8> undef, ptr undef, <8 x i1> %m8)
 ; SKL-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 0
 ;
 ; AVX512-LABEL: 'masked_compressstore'
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 23 for instruction: call void @llvm.masked.compressstore.v8f64(<8 x double> undef, ptr undef, <8 x i1> %m8)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: call void @llvm.masked.compressstore.v4f64(<4 x double> undef, ptr undef, <4 x i1> %m4)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: call void @llvm.masked.compressstore.v2f64(<2 x double> undef, ptr undef, <2 x i1> %m2)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.compressstore.v1f64(<1 x double> undef, ptr undef, <1 x i1> %m1)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 47 for instruction: call void @llvm.masked.compressstore.v16f32(<16 x float> undef, ptr undef, <16 x i1> %m16)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 23 for instruction: call void @llvm.masked.compressstore.v8f32(<8 x float> undef, ptr undef, <8 x i1> %m8)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: call void @llvm.masked.compressstore.v4f32(<4 x float> undef, ptr undef, <4 x i1> %m4)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: call void @llvm.masked.compressstore.v2f32(<2 x float> undef, ptr undef, <2 x i1> %m2)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 27 for instruction: call void @llvm.masked.compressstore.v8i64(<8 x i64> undef, ptr undef, <8 x i1> %m8)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: call void @llvm.masked.compressstore.v4i64(<4 x i64> undef, ptr undef, <4 x i1> %m4)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: call void @llvm.masked.compressstore.v2i64(<2 x i64> undef, ptr undef, <2 x i1> %m2)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.compressstore.v1i64(<1 x i64> undef, ptr undef, <1 x i1> %m1)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 51 for instruction: call void @llvm.masked.compressstore.v16i32(<16 x i32> undef, ptr undef, <16 x i1> %m16)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: call void @llvm.masked.compressstore.v8i32(<8 x i32> undef, ptr undef, <8 x i1> %m8)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: call void @llvm.masked.compressstore.v4i32(<4 x i32> undef, ptr undef, <4 x i1> %m4)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: call void @llvm.masked.compressstore.v2i32(<2 x i32> undef, ptr undef, <2 x i1> %m2)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 99 for instruction: call void @llvm.masked.compressstore.v32i16(<32 x i16> undef, ptr undef, <32 x i1> %m32)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 49 for instruction: call void @llvm.masked.compressstore.v16i16(<16 x i16> undef, ptr undef, <16 x i1> %m16)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: call void @llvm.masked.compressstore.v8i16(<8 x i16> undef, ptr undef, <8 x i1> %m8)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: call void @llvm.masked.compressstore.v4i16(<4 x i16> undef, ptr undef, <4 x i1> %m4)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 195 for instruction: call void @llvm.masked.compressstore.v64i8(<64 x i8> undef, ptr undef, <64 x i1> %m64)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 97 for instruction: call void @llvm.masked.compressstore.v32i8(<32 x i8> undef, ptr undef, <32 x i1> %m32)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: call void @llvm.masked.compressstore.v16i8(<16 x i8> undef, ptr undef, <16 x i1> %m16)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: call void @llvm.masked.compressstore.v8i8(<8 x i8> undef, ptr undef, <8 x i1> %m8)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: call void @llvm.masked.compressstore.v8f64(<8 x double> undef, ptr undef, <8 x i1> %m8)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: call void @llvm.masked.compressstore.v4f64(<4 x double> undef, ptr undef, <4 x i1> %m4)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: call void @llvm.masked.compressstore.v2f64(<2 x double> undef, ptr undef, <2 x i1> %m2)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: call void @llvm.masked.compressstore.v1f64(<1 x double> undef, ptr undef, <1 x i1> %m1)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 69 for instruction: call void @llvm.masked.compressstore.v16f32(<16 x float> undef, ptr undef, <16 x i1> %m16)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: call void @llvm.masked.compressstore.v8f32(<8 x float> undef, ptr undef, <8 x i1> %m8)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: call void @llvm.masked.compressstore.v4f32(<4 x float> undef, ptr undef, <4 x i1> %m4)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: call void @llvm.masked.compressstore.v2f32(<2 x float> undef, ptr undef, <2 x i1> %m2)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 38 for instruction: call void @llvm.masked.compressstore.v8i64(<8 x i64> undef, ptr undef, <8 x i1> %m8)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: call void @llvm.masked.compressstore.v4i64(<4 x i64> undef, ptr undef, <4 x i1> %m4)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: call void @llvm.masked.compressstore.v2i64(<2 x i64> undef, ptr undef, <2 x i1> %m2)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: call void @llvm.masked.compressstore.v1i64(<1 x i64> undef, ptr undef, <1 x i1> %m1)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 73 for instruction: call void @llvm.masked.compressstore.v16i32(<16 x i32> undef, ptr undef, <16 x i1> %m16)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 36 for instruction: call void @llvm.masked.compressstore.v8i32(<8 x i32> undef, ptr undef, <8 x i1> %m8)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: call void @llvm.masked.compressstore.v4i32(<4 x i32> undef, ptr undef, <4 x i1> %m4)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: call void @llvm.masked.compressstore.v2i32(<2 x i32> undef, ptr undef, <2 x i1> %m2)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 143 for instruction: call void @llvm.masked.compressstore.v32i16(<32 x i16> undef, ptr undef, <32 x i1> %m32)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 71 for instruction: call void @llvm.masked.compressstore.v16i16(<16 x i16> undef, ptr undef, <16 x i1> %m16)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 35 for instruction: call void @llvm.masked.compressstore.v8i16(<8 x i16> undef, ptr undef, <8 x i1> %m8)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: call void @llvm.masked.compressstore.v4i16(<4 x i16> undef, ptr undef, <4 x i1> %m4)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 283 for instruction: call void @llvm.masked.compressstore.v64i8(<64 x i8> undef, ptr undef, <64 x i1> %m64)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 141 for instruction: call void @llvm.masked.compressstore.v32i8(<32 x i8> undef, ptr undef, <32 x i1> %m32)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 70 for instruction: call void @llvm.masked.compressstore.v16i8(<16 x i8> undef, ptr undef, <16 x i1> %m16)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 35 for instruction: call void @llvm.masked.compressstore.v8i8(<8 x i8> undef, ptr undef, <8 x i1> %m8)
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 0
 ;
   call void @llvm.masked.compressstore.v8f64(<8 x double> undef, ptr undef, <8 x i1> %m8)
diff --git a/llvm/test/Analysis/CostModel/X86/masked-intrinsic-latency.ll b/llvm/test/Analysis/CostModel/X86/masked-intrinsic-latency.ll
index 4fb2f1a50a6ed..94b9b56c10cc7 100644
--- a/llvm/test/Analysis/CostModel/X86/masked-intrinsic-latency.ll
+++ b/llvm/test/Analysis/CostModel/X86/masked-intrinsic-latency.ll
@@ -1178,165 +1178,165 @@ define i32 @masked_scatter(<1 x i1> %m1, <2 x i1> %m2, <4 x i1> %m4, <8 x i1> %m
 
 define i32 @masked_expandload(<1 x i1> %m1, <2 x i1> %m2, <4 x i1> %m4, <8 x i1> %m8, <16 x i1> %m16, <32 x i1> %m32, <64 x i1> %m64) {
 ; SSE2-LABEL: 'masked_expandload'
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %V8F64 = call <8 x double> @llvm.masked.expandload.v8f64(ptr undef, <8 x i1> %m8, <8 x double> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V4F64 = call <4 x double> @llvm.masked.expandload.v4f64(ptr undef, <4 x i1> %m4, <4 x double> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V2F64 = call <2 x double> @llvm.masked.expandload.v2f64(ptr undef, <2 x i1> %m2, <2 x double> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1F64 = call <1 x double> @llvm.masked.expandload.v1f64(ptr undef, <1 x i1> %m1, <1 x double> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 29 for instruction: %V16F32 = call <16 x float> @llvm.masked.expandload.v16f32(ptr undef, <16 x i1> %m16, <16 x float> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %V8F32 = call <8 x float> @llvm.masked.expandload.v8f32(ptr undef, <8 x i1> %m8, <8 x float> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V4F32 = call <4 x float> @llvm.masked.expandload.v4f32(ptr undef, <4 x i1> %m4, <4 x float> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V2F32 = call <2 x float> @llvm.masked.expandload.v2f32(ptr undef, <2 x i1> %m2, <2 x float> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 21 for instruction: %V8I64 = call <8 x i64> @llvm.masked.expandload.v8i64(ptr undef, <8 x i1> %m8, <8 x i64> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %V4I64 = call <4 x i64> @llvm.masked.expandload.v4i64(ptr undef, <4 x i1> %m4, <4 x i64> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2I64 = call <2 x i64> @llvm.masked.expandload.v2i64(ptr undef, <2 x i1> %m2, <2 x i64> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1I64 = call <1 x i64> @llvm.masked.expandload.v1i64(ptr undef, <1 x i1> %m1, <1 x i64> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 45 for instruction: %V16I32 = call <16 x i32> @llvm.masked.expandload.v16i32(ptr undef, <16 x i1> %m16, <16 x i32> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 23 for instruction: %V8I32 = call <8 x i32> @llvm.masked.expandload.v8i32(ptr undef, <8 x i1> %m8, <8 x i32> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V4I32 = call <4 x i32> @llvm.masked.expandload.v4i32(ptr undef, <4 x i1> %m4, <4 x i32> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2I32 = call <2 x i32> @llvm.masked.expandload.v2i32(ptr undef, <2 x i1> %m2, <2 x i32> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 66 for instruction: %V32I16 = call <32 x i16> @llvm.masked.expandload.v32i16(ptr undef, <32 x i1> %m32, <32 x i16> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %V16I16 = call <16 x i16> @llvm.masked.expandload.v16i16(ptr undef, <16 x i1> %m16, <16 x i16> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %V8I16 = call <8 x i16> @llvm.masked.expandload.v8i16(ptr undef, <8 x i1> %m8, <8 x i16> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V4I16 = call <4 x i16> @llvm.masked.expandload.v4i16(ptr undef, <4 x i1> %m4, <4 x i16> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 192 for instruction: %V64I8 = call <64 x i8> @llvm.masked.expandload.v64i8(ptr undef, <64 x i1> %m64, <64 x i8> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 96 for instruction: %V32I8 = call <32 x i8> @llvm.masked.expandload.v32i8(ptr undef, <32 x i1> %m32, <32 x i8> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V16I8 = call <16 x i8> @llvm.masked.expandload.v16i8(ptr undef, <16 x i1> %m16, <16 x i8> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V8I8 = call <8 x i8> @llvm.masked.expandload.v8i8(ptr undef, <8 x i1> %m8, <8 x i8> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 37 for instruction: %V8F64 = call <8 x double> @llvm.masked.expandload.v8f64(ptr undef, <8 x i1> %m8, <8 x double> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 19 for instruction: %V4F64 = call <4 x double> @llvm.masked.expandload.v4f64(ptr undef, <4 x i1> %m4, <4 x double> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V2F64 = call <2 x double> @llvm.masked.expandload.v2f64(ptr undef, <2 x i1> %m2, <2 x double> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V1F64 = call <1 x double> @llvm.masked.expandload.v1f64(ptr undef, <1 x i1> %m1, <1 x double> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 77 for instruction: %V16F32 = call <16 x float> @llvm.masked.expandload.v16f32(ptr undef, <16 x i1> %m16, <16 x float> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 39 for instruction: %V8F32 = call <8 x float> @llvm.masked.expandload.v8f32(ptr undef, <8 x i1> %m8, <8 x float> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %V4F32 = call <4 x float> @llvm.masked.expandload.v4f32(ptr undef, <4 x i1> %m4, <4 x float> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V2F32 = call <2 x float> @llvm.masked.expandload.v2f32(ptr undef, <2 x i1> %m2, <2 x float> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 45 for instruction: %V8I64 = call <8 x i64> @llvm.masked.expandload.v8i64(ptr undef, <8 x i1> %m8, <8 x i64> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 23 for instruction: %V4I64 = call <4 x i64> @llvm.masked.expandload.v4i64(ptr undef, <4 x i1> %m4, <4 x i64> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V2I64 = call <2 x i64> @llvm.masked.expandload.v2i64(ptr undef, <2 x i1> %m2, <2 x i64> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V1I64 = call <1 x i64> @llvm.masked.expandload.v1i64(ptr undef, <1 x i1> %m1, <1 x i64> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 93 for instruction: %V16I32 = call <16 x i32> @llvm.masked.expandload.v16i32(ptr undef, <16 x i1> %m16, <16 x i32> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 47 for instruction: %V8I32 = call <8 x i32> @llvm.masked.expandload.v8i32(ptr undef, <8 x i1> %m8, <8 x i32> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V4I32 = call <4 x i32> @llvm.masked.expandload.v4i32(ptr undef, <4 x i1> %m4, <4 x i32> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V2I32 = call <2 x i32> @llvm.masked.expandload.v2i32(ptr undef, <2 x i1> %m2, <2 x i32> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 162 for instruction: %V32I16 = call <32 x i16> @llvm.masked.expandload.v32i16(ptr undef, <32 x i1> %m32, <32 x i16> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 81 for instruction: %V16I16 = call <16 x i16> @llvm.masked.expandload.v16i16(ptr undef, <16 x i1> %m16, <16 x i16> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 41 for instruction: %V8I16 = call <8 x i16> @llvm.masked.expandload.v8i16(ptr undef, <8 x i1> %m8, <8 x i16> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 21 for instruction: %V4I16 = call <4 x i16> @llvm.masked.expandload.v4i16(ptr undef, <4 x i1> %m4, <4 x i16> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 384 for instruction: %V64I8 = call <64 x i8> @llvm.masked.expandload.v64i8(ptr undef, <64 x i1> %m64, <64 x i8> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 192 for instruction: %V32I8 = call <32 x i8> @llvm.masked.expandload.v32i8(ptr undef, <32 x i1> %m32, <32 x i8> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 96 for instruction: %V16I8 = call <16 x i8> @llvm.masked.expandload.v16i8(ptr undef, <16 x i1> %m16, <16 x i8> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V8I8 = call <8 x i8> @llvm.masked.expandload.v8i8(ptr undef, <8 x i1> %m8, <8 x i8> undef)
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 0
 ;
 ; SSE42-LABEL: 'masked_expandload'
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %V8F64 = call <8 x double> @llvm.masked.expandload.v8f64(ptr undef, <8 x i1> %m8, <8 x double> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V4F64 = call <4 x double> @llvm.masked.expandload.v4f64(ptr undef, <4 x i1> %m4, <4 x double> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V2F64 = call <2 x double> @llvm.masked.expandload.v2f64(ptr undef, <2 x i1> %m2, <2 x double> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1F64 = call <1 x double> @llvm.masked.expandload.v1f64(ptr undef, <1 x i1> %m1, <1 x double> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 29 for instruction: %V16F32 = call <16 x float> @llvm.masked.expandload.v16f32(ptr undef, <16 x i1> %m16, <16 x float> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %V8F32 = call <8 x float> @llvm.masked.expandload.v8f32(ptr undef, <8 x i1> %m8, <8 x float> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V4F32 = call <4 x float> @llvm.masked.expandload.v4f32(ptr undef, <4 x i1> %m4, <4 x float> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V2F32 = call <2 x float> @llvm.masked.expandload.v2f32(ptr undef, <2 x i1> %m2, <2 x float> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %V8I64 = call <8 x i64> @llvm.masked.expandload.v8i64(ptr undef, <8 x i1> %m8, <8 x i64> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V4I64 = call <4 x i64> @llvm.masked.expandload.v4i64(ptr undef, <4 x i1> %m4, <4 x i64> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V2I64 = call <2 x i64> @llvm.masked.expandload.v2i64(ptr undef, <2 x i1> %m2, <2 x i64> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1I64 = call <1 x i64> @llvm.masked.expandload.v1i64(ptr undef, <1 x i1> %m1, <1 x i64> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %V16I32 = call <16 x i32> @llvm.masked.expandload.v16i32(ptr undef, <16 x i1> %m16, <16 x i32> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %V8I32 = call <8 x i32> @llvm.masked.expandload.v8i32(ptr undef, <8 x i1> %m8, <8 x i32> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V4I32 = call <4 x i32> @llvm.masked.expandload.v4i32(ptr undef, <4 x i1> %m4, <4 x i32> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V2I32 = call <2 x i32> @llvm.masked.expandload.v2i32(ptr undef, <2 x i1> %m2, <2 x i32> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 66 for instruction: %V32I16 = call <32 x i16> @llvm.masked.expandload.v32i16(ptr undef, <32 x i1> %m32, <32 x i16> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %V16I16 = call <16 x i16> @llvm.masked.expandload.v16i16(ptr undef, <16 x i1> %m16, <16 x i16> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %V8I16 = call <8 x i16> @llvm.masked.expandload.v8i16(ptr undef, <8 x i1> %m8, <8 x i16> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V4I16 = call <4 x i16> @llvm.masked.expandload.v4i16(ptr undef, <4 x i1> %m4, <4 x i16> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 132 for instruction: %V64I8 = call <64 x i8> @llvm.masked.expandload.v64i8(ptr undef, <64 x i1> %m64, <64 x i8> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 66 for instruction: %V32I8 = call <32 x i8> @llvm.masked.expandload.v32i8(ptr undef, <32 x i1> %m32, <32 x i8> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %V16I8 = call <16 x i8> @llvm.masked.expandload.v16i8(ptr undef, <16 x i1> %m16, <16 x i8> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %V8I8 = call <8 x i8> @llvm.masked.expandload.v8i8(ptr undef, <8 x i1> %m8, <8 x i8> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 29 for instruction: %V8F64 = call <8 x double> @llvm.masked.expandload.v8f64(ptr undef, <8 x i1> %m8, <8 x double> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %V4F64 = call <4 x double> @llvm.masked.expandload.v4f64(ptr undef, <4 x i1> %m4, <4 x double> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V2F64 = call <2 x double> @llvm.masked.expandload.v2f64(ptr undef, <2 x i1> %m2, <2 x double> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V1F64 = call <1 x double> @llvm.masked.expandload.v1f64(ptr undef, <1 x i1> %m1, <1 x double> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 61 for instruction: %V16F32 = call <16 x float> @llvm.masked.expandload.v16f32(ptr undef, <16 x i1> %m16, <16 x float> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 31 for instruction: %V8F32 = call <8 x float> @llvm.masked.expandload.v8f32(ptr undef, <8 x i1> %m8, <8 x float> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V4F32 = call <4 x float> @llvm.masked.expandload.v4f32(ptr undef, <4 x i1> %m4, <4 x float> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V2F32 = call <2 x float> @llvm.masked.expandload.v2f32(ptr undef, <2 x i1> %m2, <2 x float> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %V8I64 = call <8 x i64> @llvm.masked.expandload.v8i64(ptr undef, <8 x i1> %m8, <8 x i64> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %V4I64 = call <4 x i64> @llvm.masked.expandload.v4i64(ptr undef, <4 x i1> %m4, <4 x i64> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V2I64 = call <2 x i64> @llvm.masked.expandload.v2i64(ptr undef, <2 x i1> %m2, <2 x i64> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V1I64 = call <1 x i64> @llvm.masked.expandload.v1i64(ptr undef, <1 x i1> %m1, <1 x i64> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 65 for instruction: %V16I32 = call <16 x i32> @llvm.masked.expandload.v16i32(ptr undef, <16 x i1> %m16, <16 x i32> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %V8I32 = call <8 x i32> @llvm.masked.expandload.v8i32(ptr undef, <8 x i1> %m8, <8 x i32> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %V4I32 = call <4 x i32> @llvm.masked.expandload.v4i32(ptr undef, <4 x i1> %m4, <4 x i32> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V2I32 = call <2 x i32> @llvm.masked.expandload.v2i32(ptr undef, <2 x i1> %m2, <2 x i32> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 130 for instruction: %V32I16 = call <32 x i16> @llvm.masked.expandload.v32i16(ptr undef, <32 x i1> %m32, <32 x i16> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 65 for instruction: %V16I16 = call <16 x i16> @llvm.masked.expandload.v16i16(ptr undef, <16 x i1> %m16, <16 x i16> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %V8I16 = call <8 x i16> @llvm.masked.expandload.v8i16(ptr undef, <8 x i1> %m8, <8 x i16> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %V4I16 = call <4 x i16> @llvm.masked.expandload.v4i16(ptr undef, <4 x i1> %m4, <4 x i16> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 260 for instruction: %V64I8 = call <64 x i8> @llvm.masked.expandload.v64i8(ptr undef, <64 x i1> %m64, <64 x i8> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 130 for instruction: %V32I8 = call <32 x i8> @llvm.masked.expandload.v32i8(ptr undef, <32 x i1> %m32, <32 x i8> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 65 for instruction: %V16I8 = call <16 x i8> @llvm.masked.expandload.v16i8(ptr undef, <16 x i1> %m16, <16 x i8> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %V8I8 = call <8 x i8> @llvm.masked.expandload.v8i8(ptr undef, <8 x i1> %m8, <8 x i8> undef)
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 0
 ;
 ; AVX1-LABEL: 'masked_expandload'
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %V8F64 = call <8 x double> @llvm.masked.expandload.v8f64(ptr undef, <8 x i1> %m8, <8 x double> undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V4F64 = call <4 x double> @llvm.masked.expandload.v4f64(ptr undef, <4 x i1> %m4, <4 x double> undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V2F64 = call <2 x double> @llvm.masked.expandload.v2f64(ptr undef, <2 x i1> %m2, <2 x double> undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1F64 = call <1 x double> @llvm.masked.expandload.v1f64(ptr undef, <1 x i1> %m1, <1 x double> undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 31 for instruction: %V16F32 = call <16 x float> @llvm.masked.expandload.v16f32(ptr undef, <16 x i1> %m16, <16 x float> undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V8F32 = call <8 x float> @llvm.masked.expandload.v8f32(ptr undef, <8 x i1> %m8, <8 x float> undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V4F32 = call <4 x float> @llvm.masked.expandload.v4f32(ptr undef, <4 x i1> %m4, <4 x float> undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V2F32 = call <2 x float> @llvm.masked.expandload.v2f32(ptr undef, <2 x i1> %m2, <2 x float> undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 19 for instruction: %V8I64 = call <8 x i64> @llvm.masked.expandload.v8i64(ptr undef, <8 x i1> %m8, <8 x i64> undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V4I64 = call <4 x i64> @llvm.masked.expandload.v4i64(ptr undef, <4 x i1> %m4, <4 x i64> undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V2I64 = call <2 x i64> @llvm.masked.expandload.v2i64(ptr undef, <2 x i1> %m2, <2 x i64> undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1I64 = call <1 x i64> @llvm.masked.expandload.v1i64(ptr undef, <1 x i1> %m1, <1 x i64> undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 35 for instruction: %V16I32 = call <16 x i32> @llvm.masked.expandload.v16i32(ptr undef, <16 x i1> %m16, <16 x i32> undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V8I32 = call <8 x i32> @llvm.masked.expandload.v8i32(ptr undef, <8 x i1> %m8, <8 x i32> undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V4I32 = call <4 x i32> @llvm.masked.expandload.v4i32(ptr undef, <4 x i1> %m4, <4 x i32> undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V2I32 = call <2 x i32> @llvm.masked.expandload.v2i32(ptr undef, <2 x i1> %m2, <2 x i32> undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 68 for instruction: %V32I16 = call <32 x i16> @llvm.masked.expandload.v32i16(ptr undef, <32 x i1> %m32, <32 x i16> undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %V16I16 = call <16 x i16> @llvm.masked.expandload.v16i16(ptr undef, <16 x i1> %m16, <16 x i16> undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %V8I16 = call <8 x i16> @llvm.masked.expandload.v8i16(ptr undef, <8 x i1> %m8, <8 x i16> undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V4I16 = call <4 x i16> @llvm.masked.expandload.v4i16(ptr undef, <4 x i1> %m4, <4 x i16> undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 134 for instruction: %V64I8 = call <64 x i8> @llvm.masked.expandload.v64i8(ptr undef, <64 x i1> %m64, <64 x i8> undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 67 for instruction: %V32I8 = call <32 x i8> @llvm.masked.expandload.v32i8(ptr undef, <32 x i1> %m32, <32 x i8> undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %V16I8 = call <16 x i8> @llvm.masked.expandload.v16i8(ptr undef, <16 x i1> %m16, <16 x i8> undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %V8I8 = call <8 x i8> @llvm.masked.expandload.v8i8(ptr undef, <8 x i1> %m8, <8 x i8> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %V8F64 = call <8 x double> @llvm.masked.expandload.v8f64(ptr undef, <8 x i1> %m8, <8 x double> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %V4F64 = call <4 x double> @llvm.masked.expandload.v4f64(ptr undef, <4 x i1> %m4, <4 x double> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V2F64 = call <2 x double> @llvm.masked.expandload.v2f64(ptr undef, <2 x i1> %m2, <2 x double> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V1F64 = call <1 x double> @llvm.masked.expandload.v1f64(ptr undef, <1 x i1> %m1, <1 x double> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 67 for instruction: %V16F32 = call <16 x float> @llvm.masked.expandload.v16f32(ptr undef, <16 x i1> %m16, <16 x float> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %V8F32 = call <8 x float> @llvm.masked.expandload.v8f32(ptr undef, <8 x i1> %m8, <8 x float> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %V4F32 = call <4 x float> @llvm.masked.expandload.v4f32(ptr undef, <4 x i1> %m4, <4 x float> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V2F32 = call <2 x float> @llvm.masked.expandload.v2f32(ptr undef, <2 x i1> %m2, <2 x float> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 37 for instruction: %V8I64 = call <8 x i64> @llvm.masked.expandload.v8i64(ptr undef, <8 x i1> %m8, <8 x i64> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 19 for instruction: %V4I64 = call <4 x i64> @llvm.masked.expandload.v4i64(ptr undef, <4 x i1> %m4, <4 x i64> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V2I64 = call <2 x i64> @llvm.masked.expandload.v2i64(ptr undef, <2 x i1> %m2, <2 x i64> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V1I64 = call <1 x i64> @llvm.masked.expandload.v1i64(ptr undef, <1 x i1> %m1, <1 x i64> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 71 for instruction: %V16I32 = call <16 x i32> @llvm.masked.expandload.v16i32(ptr undef, <16 x i1> %m16, <16 x i32> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 36 for instruction: %V8I32 = call <8 x i32> @llvm.masked.expandload.v8i32(ptr undef, <8 x i1> %m8, <8 x i32> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V4I32 = call <4 x i32> @llvm.masked.expandload.v4i32(ptr undef, <4 x i1> %m4, <4 x i32> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V2I32 = call <2 x i32> @llvm.masked.expandload.v2i32(ptr undef, <2 x i1> %m2, <2 x i32> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 140 for instruction: %V32I16 = call <32 x i16> @llvm.masked.expandload.v32i16(ptr undef, <32 x i1> %m32, <32 x i16> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 70 for instruction: %V16I16 = call <16 x i16> @llvm.masked.expandload.v16i16(ptr undef, <16 x i1> %m16, <16 x i16> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 35 for instruction: %V8I16 = call <8 x i16> @llvm.masked.expandload.v8i16(ptr undef, <8 x i1> %m8, <8 x i16> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V4I16 = call <4 x i16> @llvm.masked.expandload.v4i16(ptr undef, <4 x i1> %m4, <4 x i16> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 278 for instruction: %V64I8 = call <64 x i8> @llvm.masked.expandload.v64i8(ptr undef, <64 x i1> %m64, <64 x i8> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 139 for instruction: %V32I8 = call <32 x i8> @llvm.masked.expandload.v32i8(ptr undef, <32 x i1> %m32, <32 x i8> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 69 for instruction: %V16I8 = call <16 x i8> @llvm.masked.expandload.v16i8(ptr undef, <16 x i1> %m16, <16 x i8> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 35 for instruction: %V8I8 = call <8 x i8> @llvm.masked.expandload.v8i8(ptr undef, <8 x i1> %m8, <8 x i8> undef)
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 0
 ;
 ; AVX2-LABEL: 'masked_expandload'
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %V8F64 = call <8 x double> @llvm.masked.expandload.v8f64(ptr undef, <8 x i1> %m8, <8 x double> undef)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V4F64 = call <4 x double> @llvm.masked.expandload.v4f64(ptr undef, <4 x i1> %m4, <4 x double> undef)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V2F64 = call <2 x double> @llvm.masked.expandload.v2f64(ptr undef, <2 x i1> %m2, <2 x double> undef)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1F64 = call <1 x double> @llvm.masked.expandload.v1f64(ptr undef, <1 x i1> %m1, <1 x double> undef)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 31 for instruction: %V16F32 = call <16 x float> @llvm.masked.expandload.v16f32(ptr undef, <16 x i1> %m16, <16 x float> undef)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V8F32 = call <8 x float> @llvm.masked.expandload.v8f32(ptr undef, <8 x i1> %m8, <8 x float> undef)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V4F32 = call <4 x float> @llvm.masked.expandload.v4f32(ptr undef, <4 x i1> %m4, <4 x float> undef)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V2F32 = call <2 x float> @llvm.masked.expandload.v2f32(ptr undef, <2 x i1> %m2, <2 x float> undef)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 19 for instruction: %V8I64 = call <8 x i64> @llvm.masked.expandload.v8i64(ptr undef, <8 x i1> %m8, <8 x i64> undef)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V4I64 = call <4 x i64> @llvm.masked.expandload.v4i64(ptr undef, <4 x i1> %m4, <4 x i64> undef)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V2I64 = call <2 x i64> @llvm.masked.expandload.v2i64(ptr undef, <2 x i1> %m2, <2 x i64> undef)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1I64 = call <1 x i64> @llvm.masked.expandload.v1i64(ptr undef, <1 x i1> %m1, <1 x i64> undef)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 35 for instruction: %V16I32 = call <16 x i32> @llvm.masked.expandload.v16i32(ptr undef, <16 x i1> %m16, <16 x i32> undef)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V8I32 = call <8 x i32> @llvm.masked.expandload.v8i32(ptr undef, <8 x i1> %m8, <8 x i32> undef)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V4I32 = call <4 x i32> @llvm.masked.expandload.v4i32(ptr undef, <4 x i1> %m4, <4 x i32> undef)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V2I32 = call <2 x i32> @llvm.masked.expandload.v2i32(ptr undef, <2 x i1> %m2, <2 x i32> undef)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 67 for instruction: %V32I16 = call <32 x i16> @llvm.masked.expandload.v32i16(ptr undef, <32 x i1> %m32, <32 x i16> undef)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %V16I16 = call <16 x i16> @llvm.masked.expandload.v16i16(ptr undef, <16 x i1> %m16, <16 x i16> undef)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %V8I16 = call <8 x i16> @llvm.masked.expandload.v8i16(ptr undef, <8 x i1> %m8, <8 x i16> undef)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V4I16 = call <4 x i16> @llvm.masked.expandload.v4i16(ptr undef, <4 x i1> %m4, <4 x i16> undef)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 132 for instruction: %V64I8 = call <64 x i8> @llvm.masked.expandload.v64i8(ptr undef, <64 x i1> %m64, <64 x i8> undef)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 66 for instruction: %V32I8 = call <32 x i8> @llvm.masked.expandload.v32i8(ptr undef, <32 x i1> %m32, <32 x i8> undef)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %V16I8 = call <16 x i8> @llvm.masked.expandload.v16i8(ptr undef, <16 x i1> %m16, <16 x i8> undef)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %V8I8 = call <8 x i8> @llvm.masked.expandload.v8i8(ptr undef, <8 x i1> %m8, <8 x i8> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %V8F64 = call <8 x double> @llvm.masked.expandload.v8f64(ptr undef, <8 x i1> %m8, <8 x double> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %V4F64 = call <4 x double> @llvm.masked.expandload.v4f64(ptr undef, <4 x i1> %m4, <4 x double> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V2F64 = call <2 x double> @llvm.masked.expandload.v2f64(ptr undef, <2 x i1> %m2, <2 x double> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V1F64 = call <1 x double> @llvm.masked.expandload.v1f64(ptr undef, <1 x i1> %m1, <1 x double> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 67 for instruction: %V16F32 = call <16 x float> @llvm.masked.expandload.v16f32(ptr undef, <16 x i1> %m16, <16 x float> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %V8F32 = call <8 x float> @llvm.masked.expandload.v8f32(ptr undef, <8 x i1> %m8, <8 x float> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %V4F32 = call <4 x float> @llvm.masked.expandload.v4f32(ptr undef, <4 x i1> %m4, <4 x float> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V2F32 = call <2 x float> @llvm.masked.expandload.v2f32(ptr undef, <2 x i1> %m2, <2 x float> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 37 for instruction: %V8I64 = call <8 x i64> @llvm.masked.expandload.v8i64(ptr undef, <8 x i1> %m8, <8 x i64> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 19 for instruction: %V4I64 = call <4 x i64> @llvm.masked.expandload.v4i64(ptr undef, <4 x i1> %m4, <4 x i64> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V2I64 = call <2 x i64> @llvm.masked.expandload.v2i64(ptr undef, <2 x i1> %m2, <2 x i64> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V1I64 = call <1 x i64> @llvm.masked.expandload.v1i64(ptr undef, <1 x i1> %m1, <1 x i64> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 71 for instruction: %V16I32 = call <16 x i32> @llvm.masked.expandload.v16i32(ptr undef, <16 x i1> %m16, <16 x i32> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 36 for instruction: %V8I32 = call <8 x i32> @llvm.masked.expandload.v8i32(ptr undef, <8 x i1> %m8, <8 x i32> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V4I32 = call <4 x i32> @llvm.masked.expandload.v4i32(ptr undef, <4 x i1> %m4, <4 x i32> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V2I32 = call <2 x i32> @llvm.masked.expandload.v2i32(ptr undef, <2 x i1> %m2, <2 x i32> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 139 for instruction: %V32I16 = call <32 x i16> @llvm.masked.expandload.v32i16(ptr undef, <32 x i1> %m32, <32 x i16> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 70 for instruction: %V16I16 = call <16 x i16> @llvm.masked.expandload.v16i16(ptr undef, <16 x i1> %m16, <16 x i16> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 35 for instruction: %V8I16 = call <8 x i16> @llvm.masked.expandload.v8i16(ptr undef, <8 x i1> %m8, <8 x i16> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V4I16 = call <4 x i16> @llvm.masked.expandload.v4i16(ptr undef, <4 x i1> %m4, <4 x i16> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 276 for instruction: %V64I8 = call <64 x i8> @llvm.masked.expandload.v64i8(ptr undef, <64 x i1> %m64, <64 x i8> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 138 for instruction: %V32I8 = call <32 x i8> @llvm.masked.expandload.v32i8(ptr undef, <32 x i1> %m32, <32 x i8> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 69 for instruction: %V16I8 = call <16 x i8> @llvm.masked.expandload.v16i8(ptr undef, <16 x i1> %m16, <16 x i8> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 35 for instruction: %V8I8 = call <8 x i8> @llvm.masked.expandload.v8i8(ptr undef, <8 x i1> %m8, <8 x i8> undef)
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 0
 ;
 ; SKL-LABEL: 'masked_expandload'
-; SKL-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %V8F64 = call <8 x double> @llvm.masked.expandload.v8f64(ptr undef, <8 x i1> %m8, <8 x double> undef)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V4F64 = call <4 x double> @llvm.masked.expandload.v4f64(ptr undef, <4 x i1> %m4, <4 x double> undef)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V2F64 = call <2 x double> @llvm.masked.expandload.v2f64(ptr undef, <2 x i1> %m2, <2 x double> undef)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1F64 = call <1 x double> @llvm.masked.expandload.v1f64(ptr undef, <1 x i1> %m1, <1 x double> undef)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 31 for instruction: %V16F32 = call <16 x float> @llvm.masked.expandload.v16f32(ptr undef, <16 x i1> %m16, <16 x float> undef)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V8F32 = call <8 x float> @llvm.masked.expandload.v8f32(ptr undef, <8 x i1> %m8, <8 x float> undef)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V4F32 = call <4 x float> @llvm.masked.expandload.v4f32(ptr undef, <4 x i1> %m4, <4 x float> undef)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V2F32 = call <2 x float> @llvm.masked.expandload.v2f32(ptr undef, <2 x i1> %m2, <2 x float> undef)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 19 for instruction: %V8I64 = call <8 x i64> @llvm.masked.expandload.v8i64(ptr undef, <8 x i1> %m8, <8 x i64> undef)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V4I64 = call <4 x i64> @llvm.masked.expandload.v4i64(ptr undef, <4 x i1> %m4, <4 x i64> undef)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V2I64 = call <2 x i64> @llvm.masked.expandload.v2i64(ptr undef, <2 x i1> %m2, <2 x i64> undef)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1I64 = call <1 x i64> @llvm.masked.expandload.v1i64(ptr undef, <1 x i1> %m1, <1 x i64> undef)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 35 for instruction: %V16I32 = call <16 x i32> @llvm.masked.expandload.v16i32(ptr undef, <16 x i1> %m16, <16 x i32> undef)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V8I32 = call <8 x i32> @llvm.masked.expandload.v8i32(ptr undef, <8 x i1> %m8, <8 x i32> undef)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V4I32 = call <4 x i32> @llvm.masked.expandload.v4i32(ptr undef, <4 x i1> %m4, <4 x i32> undef)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V2I32 = call <2 x i32> @llvm.masked.expandload.v2i32(ptr undef, <2 x i1> %m2, <2 x i32> undef)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 67 for instruction: %V32I16 = call <32 x i16> @llvm.masked.expandload.v32i16(ptr undef, <32 x i1> %m32, <32 x i16> undef)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %V16I16 = call <16 x i16> @llvm.masked.expandload.v16i16(ptr undef, <16 x i1> %m16, <16 x i16> undef)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %V8I16 = call <8 x i16> @llvm.masked.expandload.v8i16(ptr undef, <8 x i1> %m8, <8 x i16> undef)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V4I16 = call <4 x i16> @llvm.masked.expandload.v4i16(ptr undef, <4 x i1> %m4, <4 x i16> undef)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 132 for instruction: %V64I8 = call <64 x i8> @llvm.masked.expandload.v64i8(ptr undef, <64 x i1> %m64, <64 x i8> undef)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 66 for instruction: %V32I8 = call <32 x i8> @llvm.masked.expandload.v32i8(ptr undef, <32 x i1> %m32, <32 x i8> undef)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %V16I8 = call <16 x i8> @llvm.masked.expandload.v16i8(ptr undef, <16 x i1> %m16, <16 x i8> undef)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %V8I8 = call <8 x i8> @llvm.masked.expandload.v8i8(ptr undef, <8 x i1> %m8, <8 x i8> undef)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %V8F64 = call <8 x double> @llvm.masked.expandload.v8f64(ptr undef, <8 x i1> %m8, <8 x double> undef)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %V4F64 = call <4 x double> @llvm.masked.expandload.v4f64(ptr undef, <4 x i1> %m4, <4 x double> undef)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V2F64 = call <2 x double> @llvm.masked.expandload.v2f64(ptr undef, <2 x i1> %m2, <2 x double> undef)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V1F64 = call <1 x double> @llvm.masked.expandload.v1f64(ptr undef, <1 x i1> %m1, <1 x double> undef)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 67 for instruction: %V16F32 = call <16 x float> @llvm.masked.expandload.v16f32(ptr undef, <16 x i1> %m16, <16 x float> undef)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %V8F32 = call <8 x float> @llvm.masked.expandload.v8f32(ptr undef, <8 x i1> %m8, <8 x float> undef)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %V4F32 = call <4 x float> @llvm.masked.expandload.v4f32(ptr undef, <4 x i1> %m4, <4 x float> undef)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V2F32 = call <2 x float> @llvm.masked.expandload.v2f32(ptr undef, <2 x i1> %m2, <2 x float> undef)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 37 for instruction: %V8I64 = call <8 x i64> @llvm.masked.expandload.v8i64(ptr undef, <8 x i1> %m8, <8 x i64> undef)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 19 for instruction: %V4I64 = call <4 x i64> @llvm.masked.expandload.v4i64(ptr undef, <4 x i1> %m4, <4 x i64> undef)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V2I64 = call <2 x i64> @llvm.masked.expandload.v2i64(ptr undef, <2 x i1> %m2, <2 x i64> undef)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V1I64 = call <1 x i64> @llvm.masked.expandload.v1i64(ptr undef, <1 x i1> %m1, <1 x i64> undef)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 71 for instruction: %V16I32 = call <16 x i32> @llvm.masked.expandload.v16i32(ptr undef, <16 x i1> %m16, <16 x i32> undef)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 36 for instruction: %V8I32 = call <8 x i32> @llvm.masked.expandload.v8i32(ptr undef, <8 x i1> %m8, <8 x i32> undef)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V4I32 = call <4 x i32> @llvm.masked.expandload.v4i32(ptr undef, <4 x i1> %m4, <4 x i32> undef)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V2I32 = call <2 x i32> @llvm.masked.expandload.v2i32(ptr undef, <2 x i1> %m2, <2 x i32> undef)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 139 for instruction: %V32I16 = call <32 x i16> @llvm.masked.expandload.v32i16(ptr undef, <32 x i1> %m32, <32 x i16> undef)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 70 for instruction: %V16I16 = call <16 x i16> @llvm.masked.expandload.v16i16(ptr undef, <16 x i1> %m16, <16 x i16> undef)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 35 for instruction: %V8I16 = call <8 x i16> @llvm.masked.expandload.v8i16(ptr undef, <8 x i1> %m8, <8 x i16> undef)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V4I16 = call <4 x i16> @llvm.masked.expandload.v4i16(ptr undef, <4 x i1> %m4, <4 x i16> undef)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 276 for instruction: %V64I8 = call <64 x i8> @llvm.masked.expandload.v64i8(ptr undef, <64 x i1> %m64, <64 x i8> undef)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 138 for instruction: %V32I8 = call <32 x i8> @llvm.masked.expandload.v32i8(ptr undef, <32 x i1> %m32, <32 x i8> undef)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 69 for instruction: %V16I8 = call <16 x i8> @llvm.masked.expandload.v16i8(ptr undef, <16 x i1> %m16, <16 x i8> undef)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 35 for instruction: %V8I8 = call <8 x i8> @llvm.masked.expandload.v8i8(ptr undef, <8 x i1> %m8, <8 x i8> undef)
 ; SKL-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 0
 ;
 ; AVX512-LABEL: 'masked_expandload'
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 23 for instruction: %V8F64 = call <8 x double> @llvm.masked.expandload.v8f64(ptr undef, <8 x i1> %m8, <8 x double> undef)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %V4F64 = call <4 x double> @llvm.masked.expandload.v4f64(ptr undef, <4 x i1> %m4, <4 x double> undef)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V2F64 = call <2 x double> @llvm.masked.expandload.v2f64(ptr undef, <2 x i1> %m2, <2 x double> undef)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1F64 = call <1 x double> @llvm.masked.expandload.v1f64(ptr undef, <1 x i1> %m1, <1 x double> undef)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 47 for instruction: %V16F32 = call <16 x float> @llvm.masked.expandload.v16f32(ptr undef, <16 x i1> %m16, <16 x float> undef)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 23 for instruction: %V8F32 = call <8 x float> @llvm.masked.expandload.v8f32(ptr undef, <8 x i1> %m8, <8 x float> undef)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %V4F32 = call <4 x float> @llvm.masked.expandload.v4f32(ptr undef, <4 x i1> %m4, <4 x float> undef)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V2F32 = call <2 x float> @llvm.masked.expandload.v2f32(ptr undef, <2 x i1> %m2, <2 x float> undef)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 27 for instruction: %V8I64 = call <8 x i64> @llvm.masked.expandload.v8i64(ptr undef, <8 x i1> %m8, <8 x i64> undef)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %V4I64 = call <4 x i64> @llvm.masked.expandload.v4i64(ptr undef, <4 x i1> %m4, <4 x i64> undef)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2I64 = call <2 x i64> @llvm.masked.expandload.v2i64(ptr undef, <2 x i1> %m2, <2 x i64> undef)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1I64 = call <1 x i64> @llvm.masked.expandload.v1i64(ptr undef, <1 x i1> %m1, <1 x i64> undef)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 51 for instruction: %V16I32 = call <16 x i32> @llvm.masked.expandload.v16i32(ptr undef, <16 x i1> %m16, <16 x i32> undef)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: %V8I32 = call <8 x i32> @llvm.masked.expandload.v8i32(ptr undef, <8 x i1> %m8, <8 x i32> undef)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V4I32 = call <4 x i32> @llvm.masked.expandload.v4i32(ptr undef, <4 x i1> %m4, <4 x i32> undef)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2I32 = call <2 x i32> @llvm.masked.expandload.v2i32(ptr undef, <2 x i1> %m2, <2 x i32> undef)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 99 for instruction: %V32I16 = call <32 x i16> @llvm.masked.expandload.v32i16(ptr undef, <32 x i1> %m32, <32 x i16> undef)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 49 for instruction: %V16I16 = call <16 x i16> @llvm.masked.expandload.v16i16(ptr undef, <16 x i1> %m16, <16 x i16> undef)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V8I16 = call <8 x i16> @llvm.masked.expandload.v8i16(ptr undef, <8 x i1> %m8, <8 x i16> undef)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V4I16 = call <4 x i16> @llvm.masked.expandload.v4i16(ptr undef, <4 x i1> %m4, <4 x i16> undef)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 195 for instruction: %V64I8 = call <64 x i8> @llvm.masked.expandload.v64i8(ptr undef, <64 x i1> %m64, <64 x i8> undef)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 97 for instruction: %V32I8 = call <32 x i8> @llvm.masked.expandload.v32i8(ptr undef, <32 x i1> %m32, <32 x i8> undef)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V16I8 = call <16 x i8> @llvm.masked.expandload.v16i8(ptr undef, <16 x i1> %m16, <16 x i8> undef)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V8I8 = call <8 x i8> @llvm.masked.expandload.v8i8(ptr undef, <8 x i1> %m8, <8 x i8> undef)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 42 for instruction: %V8F64 = call <8 x double> @llvm.masked.expandload.v8f64(ptr undef, <8 x i1> %m8, <8 x double> undef)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %V4F64 = call <4 x double> @llvm.masked.expandload.v4f64(ptr undef, <4 x i1> %m4, <4 x double> undef)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V2F64 = call <2 x double> @llvm.masked.expandload.v2f64(ptr undef, <2 x i1> %m2, <2 x double> undef)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V1F64 = call <1 x double> @llvm.masked.expandload.v1f64(ptr undef, <1 x i1> %m1, <1 x double> undef)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 85 for instruction: %V16F32 = call <16 x float> @llvm.masked.expandload.v16f32(ptr undef, <16 x i1> %m16, <16 x float> undef)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 42 for instruction: %V8F32 = call <8 x float> @llvm.masked.expandload.v8f32(ptr undef, <8 x i1> %m8, <8 x float> undef)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %V4F32 = call <4 x float> @llvm.masked.expandload.v4f32(ptr undef, <4 x i1> %m4, <4 x float> undef)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V2F32 = call <2 x float> @llvm.masked.expandload.v2f32(ptr undef, <2 x i1> %m2, <2 x float> undef)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 46 for instruction: %V8I64 = call <8 x i64> @llvm.masked.expandload.v8i64(ptr undef, <8 x i1> %m8, <8 x i64> undef)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 22 for instruction: %V4I64 = call <4 x i64> @llvm.masked.expandload.v4i64(ptr undef, <4 x i1> %m4, <4 x i64> undef)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V2I64 = call <2 x i64> @llvm.masked.expandload.v2i64(ptr undef, <2 x i1> %m2, <2 x i64> undef)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V1I64 = call <1 x i64> @llvm.masked.expandload.v1i64(ptr undef, <1 x i1> %m1, <1 x i64> undef)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 89 for instruction: %V16I32 = call <16 x i32> @llvm.masked.expandload.v16i32(ptr undef, <16 x i1> %m16, <16 x i32> undef)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 44 for instruction: %V8I32 = call <8 x i32> @llvm.masked.expandload.v8i32(ptr undef, <8 x i1> %m8, <8 x i32> undef)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 21 for instruction: %V4I32 = call <4 x i32> @llvm.masked.expandload.v4i32(ptr undef, <4 x i1> %m4, <4 x i32> undef)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V2I32 = call <2 x i32> @llvm.masked.expandload.v2i32(ptr undef, <2 x i1> %m2, <2 x i32> undef)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 175 for instruction: %V32I16 = call <32 x i16> @llvm.masked.expandload.v32i16(ptr undef, <32 x i1> %m32, <32 x i16> undef)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 87 for instruction: %V16I16 = call <16 x i16> @llvm.masked.expandload.v16i16(ptr undef, <16 x i1> %m16, <16 x i16> undef)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 43 for instruction: %V8I16 = call <8 x i16> @llvm.masked.expandload.v8i16(ptr undef, <8 x i1> %m8, <8 x i16> undef)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 21 for instruction: %V4I16 = call <4 x i16> @llvm.masked.expandload.v4i16(ptr undef, <4 x i1> %m4, <4 x i16> undef)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 347 for instruction: %V64I8 = call <64 x i8> @llvm.masked.expandload.v64i8(ptr undef, <64 x i1> %m64, <64 x i8> undef)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 173 for instruction: %V32I8 = call <32 x i8> @llvm.masked.expandload.v32i8(ptr undef, <32 x i1> %m32, <32 x i8> undef)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 86 for instruction: %V16I8 = call <16 x i8> @llvm.masked.expandload.v16i8(ptr undef, <16 x i1> %m16, <16 x i8> undef)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 43 for instruction: %V8I8 = call <8 x i8> @llvm.masked.expandload.v8i8(ptr undef, <8 x i1> %m8, <8 x i8> undef)
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 0
 ;
   %V8F64 = call <8 x double> @llvm.masked.expandload.v8f64(ptr undef, <8 x i1> %m8, <8 x double> undef)
@@ -1374,165 +1374,165 @@ define i32 @masked_expandload(<1 x i1> %m1, <2 x i1> %m2, <4 x i1> %m4, <8 x i1>
 
 define i32 @masked_compressstore(<1 x i1> %m1, <2 x i1> %m2, <4 x i1> %m4, <8 x i1> %m8, <16 x i1> %m16, <32 x i1> %m32, <64 x i1> %m64) {
 ; SSE2-LABEL: 'masked_compressstore'
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: call void @llvm.masked.compressstore.v8f64(<8 x double> undef, ptr undef, <8 x i1> %m8)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: call void @llvm.masked.compressstore.v4f64(<4 x double> undef, ptr undef, <4 x i1> %m4)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: call void @llvm.masked.compressstore.v2f64(<2 x double> undef, ptr undef, <2 x i1> %m2)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.compressstore.v1f64(<1 x double> undef, ptr undef, <1 x i1> %m1)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 29 for instruction: call void @llvm.masked.compressstore.v16f32(<16 x float> undef, ptr undef, <16 x i1> %m16)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: call void @llvm.masked.compressstore.v8f32(<8 x float> undef, ptr undef, <8 x i1> %m8)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: call void @llvm.masked.compressstore.v4f32(<4 x float> undef, ptr undef, <4 x i1> %m4)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: call void @llvm.masked.compressstore.v2f32(<2 x float> undef, ptr undef, <2 x i1> %m2)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 21 for instruction: call void @llvm.masked.compressstore.v8i64(<8 x i64> undef, ptr undef, <8 x i1> %m8)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: call void @llvm.masked.compressstore.v4i64(<4 x i64> undef, ptr undef, <4 x i1> %m4)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: call void @llvm.masked.compressstore.v2i64(<2 x i64> undef, ptr undef, <2 x i1> %m2)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.compressstore.v1i64(<1 x i64> undef, ptr undef, <1 x i1> %m1)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 45 for instruction: call void @llvm.masked.compressstore.v16i32(<16 x i32> undef, ptr undef, <16 x i1> %m16)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 23 for instruction: call void @llvm.masked.compressstore.v8i32(<8 x i32> undef, ptr undef, <8 x i1> %m8)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: call void @llvm.masked.compressstore.v4i32(<4 x i32> undef, ptr undef, <4 x i1> %m4)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: call void @llvm.masked.compressstore.v2i32(<2 x i32> undef, ptr undef, <2 x i1> %m2)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 66 for instruction: call void @llvm.masked.compressstore.v32i16(<32 x i16> undef, ptr undef, <32 x i1> %m32)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: call void @llvm.masked.compressstore.v16i16(<16 x i16> undef, ptr undef, <16 x i1> %m16)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: call void @llvm.masked.compressstore.v8i16(<8 x i16> undef, ptr undef, <8 x i1> %m8)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: call void @llvm.masked.compressstore.v4i16(<4 x i16> undef, ptr undef, <4 x i1> %m4)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 192 for instruction: call void @llvm.masked.compressstore.v64i8(<64 x i8> undef, ptr undef, <64 x i1> %m64)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 96 for instruction: call void @llvm.masked.compressstore.v32i8(<32 x i8> undef, ptr undef, <32 x i1> %m32)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: call void @llvm.masked.compressstore.v16i8(<16 x i8> undef, ptr undef, <16 x i1> %m16)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: call void @llvm.masked.compressstore.v8i8(<8 x i8> undef, ptr undef, <8 x i1> %m8)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 37 for instruction: call void @llvm.masked.compressstore.v8f64(<8 x double> undef, ptr undef, <8 x i1> %m8)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 19 for instruction: call void @llvm.masked.compressstore.v4f64(<4 x double> undef, ptr undef, <4 x i1> %m4)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: call void @llvm.masked.compressstore.v2f64(<2 x double> undef, ptr undef, <2 x i1> %m2)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: call void @llvm.masked.compressstore.v1f64(<1 x double> undef, ptr undef, <1 x i1> %m1)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 77 for instruction: call void @llvm.masked.compressstore.v16f32(<16 x float> undef, ptr undef, <16 x i1> %m16)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 39 for instruction: call void @llvm.masked.compressstore.v8f32(<8 x float> undef, ptr undef, <8 x i1> %m8)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: call void @llvm.masked.compressstore.v4f32(<4 x float> undef, ptr undef, <4 x i1> %m4)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: call void @llvm.masked.compressstore.v2f32(<2 x float> undef, ptr undef, <2 x i1> %m2)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 45 for instruction: call void @llvm.masked.compressstore.v8i64(<8 x i64> undef, ptr undef, <8 x i1> %m8)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 23 for instruction: call void @llvm.masked.compressstore.v4i64(<4 x i64> undef, ptr undef, <4 x i1> %m4)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: call void @llvm.masked.compressstore.v2i64(<2 x i64> undef, ptr undef, <2 x i1> %m2)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: call void @llvm.masked.compressstore.v1i64(<1 x i64> undef, ptr undef, <1 x i1> %m1)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 93 for instruction: call void @llvm.masked.compressstore.v16i32(<16 x i32> undef, ptr undef, <16 x i1> %m16)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 47 for instruction: call void @llvm.masked.compressstore.v8i32(<8 x i32> undef, ptr undef, <8 x i1> %m8)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: call void @llvm.masked.compressstore.v4i32(<4 x i32> undef, ptr undef, <4 x i1> %m4)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: call void @llvm.masked.compressstore.v2i32(<2 x i32> undef, ptr undef, <2 x i1> %m2)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 162 for instruction: call void @llvm.masked.compressstore.v32i16(<32 x i16> undef, ptr undef, <32 x i1> %m32)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 81 for instruction: call void @llvm.masked.compressstore.v16i16(<16 x i16> undef, ptr undef, <16 x i1> %m16)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 41 for instruction: call void @llvm.masked.compressstore.v8i16(<8 x i16> undef, ptr undef, <8 x i1> %m8)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 21 for instruction: call void @llvm.masked.compressstore.v4i16(<4 x i16> undef, ptr undef, <4 x i1> %m4)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 384 for instruction: call void @llvm.masked.compressstore.v64i8(<64 x i8> undef, ptr undef, <64 x i1> %m64)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 192 for instruction: call void @llvm.masked.compressstore.v32i8(<32 x i8> undef, ptr undef, <32 x i1> %m32)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 96 for instruction: call void @llvm.masked.compressstore.v16i8(<16 x i8> undef, ptr undef, <16 x i1> %m16)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: call void @llvm.masked.compressstore.v8i8(<8 x i8> undef, ptr undef, <8 x i1> %m8)
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 0
 ;
 ; SSE42-LABEL: 'masked_compressstore'
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: call void @llvm.masked.compressstore.v8f64(<8 x double> undef, ptr undef, <8 x i1> %m8)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: call void @llvm.masked.compressstore.v4f64(<4 x double> undef, ptr undef, <4 x i1> %m4)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: call void @llvm.masked.compressstore.v2f64(<2 x double> undef, ptr undef, <2 x i1> %m2)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.compressstore.v1f64(<1 x double> undef, ptr undef, <1 x i1> %m1)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 29 for instruction: call void @llvm.masked.compressstore.v16f32(<16 x float> undef, ptr undef, <16 x i1> %m16)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: call void @llvm.masked.compressstore.v8f32(<8 x float> undef, ptr undef, <8 x i1> %m8)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: call void @llvm.masked.compressstore.v4f32(<4 x float> undef, ptr undef, <4 x i1> %m4)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: call void @llvm.masked.compressstore.v2f32(<2 x float> undef, ptr undef, <2 x i1> %m2)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: call void @llvm.masked.compressstore.v8i64(<8 x i64> undef, ptr undef, <8 x i1> %m8)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: call void @llvm.masked.compressstore.v4i64(<4 x i64> undef, ptr undef, <4 x i1> %m4)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: call void @llvm.masked.compressstore.v2i64(<2 x i64> undef, ptr undef, <2 x i1> %m2)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.compressstore.v1i64(<1 x i64> undef, ptr undef, <1 x i1> %m1)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: call void @llvm.masked.compressstore.v16i32(<16 x i32> undef, ptr undef, <16 x i1> %m16)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: call void @llvm.masked.compressstore.v8i32(<8 x i32> undef, ptr undef, <8 x i1> %m8)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: call void @llvm.masked.compressstore.v4i32(<4 x i32> undef, ptr undef, <4 x i1> %m4)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: call void @llvm.masked.compressstore.v2i32(<2 x i32> undef, ptr undef, <2 x i1> %m2)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 66 for instruction: call void @llvm.masked.compressstore.v32i16(<32 x i16> undef, ptr undef, <32 x i1> %m32)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: call void @llvm.masked.compressstore.v16i16(<16 x i16> undef, ptr undef, <16 x i1> %m16)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: call void @llvm.masked.compressstore.v8i16(<8 x i16> undef, ptr undef, <8 x i1> %m8)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: call void @llvm.masked.compressstore.v4i16(<4 x i16> undef, ptr undef, <4 x i1> %m4)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 132 for instruction: call void @llvm.masked.compressstore.v64i8(<64 x i8> undef, ptr undef, <64 x i1> %m64)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 66 for instruction: call void @llvm.masked.compressstore.v32i8(<32 x i8> undef, ptr undef, <32 x i1> %m32)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: call void @llvm.masked.compressstore.v16i8(<16 x i8> undef, ptr undef, <16 x i1> %m16)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: call void @llvm.masked.compressstore.v8i8(<8 x i8> undef, ptr undef, <8 x i1> %m8)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 29 for instruction: call void @llvm.masked.compressstore.v8f64(<8 x double> undef, ptr undef, <8 x i1> %m8)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: call void @llvm.masked.compressstore.v4f64(<4 x double> undef, ptr undef, <4 x i1> %m4)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: call void @llvm.masked.compressstore.v2f64(<2 x double> undef, ptr undef, <2 x i1> %m2)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: call void @llvm.masked.compressstore.v1f64(<1 x double> undef, ptr undef, <1 x i1> %m1)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 61 for instruction: call void @llvm.masked.compressstore.v16f32(<16 x float> undef, ptr undef, <16 x i1> %m16)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 31 for instruction: call void @llvm.masked.compressstore.v8f32(<8 x float> undef, ptr undef, <8 x i1> %m8)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: call void @llvm.masked.compressstore.v4f32(<4 x float> undef, ptr undef, <4 x i1> %m4)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: call void @llvm.masked.compressstore.v2f32(<2 x float> undef, ptr undef, <2 x i1> %m2)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: call void @llvm.masked.compressstore.v8i64(<8 x i64> undef, ptr undef, <8 x i1> %m8)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: call void @llvm.masked.compressstore.v4i64(<4 x i64> undef, ptr undef, <4 x i1> %m4)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: call void @llvm.masked.compressstore.v2i64(<2 x i64> undef, ptr undef, <2 x i1> %m2)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: call void @llvm.masked.compressstore.v1i64(<1 x i64> undef, ptr undef, <1 x i1> %m1)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 65 for instruction: call void @llvm.masked.compressstore.v16i32(<16 x i32> undef, ptr undef, <16 x i1> %m16)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: call void @llvm.masked.compressstore.v8i32(<8 x i32> undef, ptr undef, <8 x i1> %m8)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: call void @llvm.masked.compressstore.v4i32(<4 x i32> undef, ptr undef, <4 x i1> %m4)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: call void @llvm.masked.compressstore.v2i32(<2 x i32> undef, ptr undef, <2 x i1> %m2)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 130 for instruction: call void @llvm.masked.compressstore.v32i16(<32 x i16> undef, ptr undef, <32 x i1> %m32)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 65 for instruction: call void @llvm.masked.compressstore.v16i16(<16 x i16> undef, ptr undef, <16 x i1> %m16)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: call void @llvm.masked.compressstore.v8i16(<8 x i16> undef, ptr undef, <8 x i1> %m8)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: call void @llvm.masked.compressstore.v4i16(<4 x i16> undef, ptr undef, <4 x i1> %m4)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 260 for instruction: call void @llvm.masked.compressstore.v64i8(<64 x i8> undef, ptr undef, <64 x i1> %m64)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 130 for instruction: call void @llvm.masked.compressstore.v32i8(<32 x i8> undef, ptr undef, <32 x i1> %m32)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 65 for instruction: call void @llvm.masked.compressstore.v16i8(<16 x i8> undef, ptr undef, <16 x i1> %m16)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: call void @llvm.masked.compressstore.v8i8(<8 x i8> undef, ptr undef, <8 x i1> %m8)
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 0
 ;
 ; AVX1-LABEL: 'masked_compressstore'
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: call void @llvm.masked.compressstore.v8f64(<8 x double> undef, ptr undef, <8 x i1> %m8)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: call void @llvm.masked.compressstore.v4f64(<4 x double> undef, ptr undef, <4 x i1> %m4)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: call void @llvm.masked.compressstore.v2f64(<2 x double> undef, ptr undef, <2 x i1> %m2)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.compressstore.v1f64(<1 x double> undef, ptr undef, <1 x i1> %m1)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 31 for instruction: call void @llvm.masked.compressstore.v16f32(<16 x float> undef, ptr undef, <16 x i1> %m16)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: call void @llvm.masked.compressstore.v8f32(<8 x float> undef, ptr undef, <8 x i1> %m8)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: call void @llvm.masked.compressstore.v4f32(<4 x float> undef, ptr undef, <4 x i1> %m4)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: call void @llvm.masked.compressstore.v2f32(<2 x float> undef, ptr undef, <2 x i1> %m2)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 19 for instruction: call void @llvm.masked.compressstore.v8i64(<8 x i64> undef, ptr undef, <8 x i1> %m8)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: call void @llvm.masked.compressstore.v4i64(<4 x i64> undef, ptr undef, <4 x i1> %m4)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: call void @llvm.masked.compressstore.v2i64(<2 x i64> undef, ptr undef, <2 x i1> %m2)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.compressstore.v1i64(<1 x i64> undef, ptr undef, <1 x i1> %m1)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 35 for instruction: call void @llvm.masked.compressstore.v16i32(<16 x i32> undef, ptr undef, <16 x i1> %m16)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: call void @llvm.masked.compressstore.v8i32(<8 x i32> undef, ptr undef, <8 x i1> %m8)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: call void @llvm.masked.compressstore.v4i32(<4 x i32> undef, ptr undef, <4 x i1> %m4)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: call void @llvm.masked.compressstore.v2i32(<2 x i32> undef, ptr undef, <2 x i1> %m2)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 68 for instruction: call void @llvm.masked.compressstore.v32i16(<32 x i16> undef, ptr undef, <32 x i1> %m32)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: call void @llvm.masked.compressstore.v16i16(<16 x i16> undef, ptr undef, <16 x i1> %m16)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: call void @llvm.masked.compressstore.v8i16(<8 x i16> undef, ptr undef, <8 x i1> %m8)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: call void @llvm.masked.compressstore.v4i16(<4 x i16> undef, ptr undef, <4 x i1> %m4)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 134 for instruction: call void @llvm.masked.compressstore.v64i8(<64 x i8> undef, ptr undef, <64 x i1> %m64)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 67 for instruction: call void @llvm.masked.compressstore.v32i8(<32 x i8> undef, ptr undef, <32 x i1> %m32)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: call void @llvm.masked.compressstore.v16i8(<16 x i8> undef, ptr undef, <16 x i1> %m16)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: call void @llvm.masked.compressstore.v8i8(<8 x i8> undef, ptr undef, <8 x i1> %m8)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: call void @llvm.masked.compressstore.v8f64(<8 x double> undef, ptr undef, <8 x i1> %m8)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: call void @llvm.masked.compressstore.v4f64(<4 x double> undef, ptr undef, <4 x i1> %m4)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: call void @llvm.masked.compressstore.v2f64(<2 x double> undef, ptr undef, <2 x i1> %m2)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: call void @llvm.masked.compressstore.v1f64(<1 x double> undef, ptr undef, <1 x i1> %m1)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 67 for instruction: call void @llvm.masked.compressstore.v16f32(<16 x float> undef, ptr undef, <16 x i1> %m16)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: call void @llvm.masked.compressstore.v8f32(<8 x float> undef, ptr undef, <8 x i1> %m8)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: call void @llvm.masked.compressstore.v4f32(<4 x float> undef, ptr undef, <4 x i1> %m4)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: call void @llvm.masked.compressstore.v2f32(<2 x float> undef, ptr undef, <2 x i1> %m2)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 37 for instruction: call void @llvm.masked.compressstore.v8i64(<8 x i64> undef, ptr undef, <8 x i1> %m8)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 19 for instruction: call void @llvm.masked.compressstore.v4i64(<4 x i64> undef, ptr undef, <4 x i1> %m4)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: call void @llvm.masked.compressstore.v2i64(<2 x i64> undef, ptr undef, <2 x i1> %m2)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: call void @llvm.masked.compressstore.v1i64(<1 x i64> undef, ptr undef, <1 x i1> %m1)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 71 for instruction: call void @llvm.masked.compressstore.v16i32(<16 x i32> undef, ptr undef, <16 x i1> %m16)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 36 for instruction: call void @llvm.masked.compressstore.v8i32(<8 x i32> undef, ptr undef, <8 x i1> %m8)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: call void @llvm.masked.compressstore.v4i32(<4 x i32> undef, ptr undef, <4 x i1> %m4)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: call void @llvm.masked.compressstore.v2i32(<2 x i32> undef, ptr undef, <2 x i1> %m2)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 140 for instruction: call void @llvm.masked.compressstore.v32i16(<32 x i16> undef, ptr undef, <32 x i1> %m32)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 70 for instruction: call void @llvm.masked.compressstore.v16i16(<16 x i16> undef, ptr undef, <16 x i1> %m16)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 35 for instruction: call void @llvm.masked.compressstore.v8i16(<8 x i16> undef, ptr undef, <8 x i1> %m8)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: call void @llvm.masked.compressstore.v4i16(<4 x i16> undef, ptr undef, <4 x i1> %m4)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 278 for instruction: call void @llvm.masked.compressstore.v64i8(<64 x i8> undef, ptr undef, <64 x i1> %m64)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 139 for instruction: call void @llvm.masked.compressstore.v32i8(<32 x i8> undef, ptr undef, <32 x i1> %m32)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 69 for instruction: call void @llvm.masked.compressstore.v16i8(<16 x i8> undef, ptr undef, <16 x i1> %m16)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 35 for instruction: call void @llvm.masked.compressstore.v8i8(<8 x i8> undef, ptr undef, <8 x i1> %m8)
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 0
 ;
 ; AVX2-LABEL: 'masked_compressstore'
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: call void @llvm.masked.compressstore.v8f64(<8 x double> undef, ptr undef, <8 x i1> %m8)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: call void @llvm.masked.compressstore.v4f64(<4 x double> undef, ptr undef, <4 x i1> %m4)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: call void @llvm.masked.compressstore.v2f64(<2 x double> undef, ptr undef, <2 x i1> %m2)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.compressstore.v1f64(<1 x double> undef, ptr undef, <1 x i1> %m1)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 31 for instruction: call void @llvm.masked.compressstore.v16f32(<16 x float> undef, ptr undef, <16 x i1> %m16)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: call void @llvm.masked.compressstore.v8f32(<8 x float> undef, ptr undef, <8 x i1> %m8)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: call void @llvm.masked.compressstore.v4f32(<4 x float> undef, ptr undef, <4 x i1> %m4)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: call void @llvm.masked.compressstore.v2f32(<2 x float> undef, ptr undef, <2 x i1> %m2)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 19 for instruction: call void @llvm.masked.compressstore.v8i64(<8 x i64> undef, ptr undef, <8 x i1> %m8)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: call void @llvm.masked.compressstore.v4i64(<4 x i64> undef, ptr undef, <4 x i1> %m4)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: call void @llvm.masked.compressstore.v2i64(<2 x i64> undef, ptr undef, <2 x i1> %m2)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.compressstore.v1i64(<1 x i64> undef, ptr undef, <1 x i1> %m1)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 35 for instruction: call void @llvm.masked.compressstore.v16i32(<16 x i32> undef, ptr undef, <16 x i1> %m16)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: call void @llvm.masked.compressstore.v8i32(<8 x i32> undef, ptr undef, <8 x i1> %m8)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: call void @llvm.masked.compressstore.v4i32(<4 x i32> undef, ptr undef, <4 x i1> %m4)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: call void @llvm.masked.compressstore.v2i32(<2 x i32> undef, ptr undef, <2 x i1> %m2)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 67 for instruction: call void @llvm.masked.compressstore.v32i16(<32 x i16> undef, ptr undef, <32 x i1> %m32)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: call void @llvm.masked.compressstore.v16i16(<16 x i16> undef, ptr undef, <16 x i1> %m16)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: call void @llvm.masked.compressstore.v8i16(<8 x i16> undef, ptr undef, <8 x i1> %m8)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: call void @llvm.masked.compressstore.v4i16(<4 x i16> undef, ptr undef, <4 x i1> %m4)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 132 for instruction: call void @llvm.masked.compressstore.v64i8(<64 x i8> undef, ptr undef, <64 x i1> %m64)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 66 for instruction: call void @llvm.masked.compressstore.v32i8(<32 x i8> undef, ptr undef, <32 x i1> %m32)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: call void @llvm.masked.compressstore.v16i8(<16 x i8> undef, ptr undef, <16 x i1> %m16)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: call void @llvm.masked.compressstore.v8i8(<8 x i8> undef, ptr undef, <8 x i1> %m8)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: call void @llvm.masked.compressstore.v8f64(<8 x double> undef, ptr undef, <8 x i1> %m8)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: call void @llvm.masked.compressstore.v4f64(<4 x double> undef, ptr undef, <4 x i1> %m4)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: call void @llvm.masked.compressstore.v2f64(<2 x double> undef, ptr undef, <2 x i1> %m2)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: call void @llvm.masked.compressstore.v1f64(<1 x double> undef, ptr undef, <1 x i1> %m1)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 67 for instruction: call void @llvm.masked.compressstore.v16f32(<16 x float> undef, ptr undef, <16 x i1> %m16)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: call void @llvm.masked.compressstore.v8f32(<8 x float> undef, ptr undef, <8 x i1> %m8)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: call void @llvm.masked.compressstore.v4f32(<4 x float> undef, ptr undef, <4 x i1> %m4)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: call void @llvm.masked.compressstore.v2f32(<2 x float> undef, ptr undef, <2 x i1> %m2)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 37 for instruction: call void @llvm.masked.compressstore.v8i64(<8 x i64> undef, ptr undef, <8 x i1> %m8)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 19 for instruction: call void @llvm.masked.compressstore.v4i64(<4 x i64> undef, ptr undef, <4 x i1> %m4)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: call void @llvm.masked.compressstore.v2i64(<2 x i64> undef, ptr undef, <2 x i1> %m2)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: call void @llvm.masked.compressstore.v1i64(<1 x i64> undef, ptr undef, <1 x i1> %m1)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 71 for instruction: call void @llvm.masked.compressstore.v16i32(<16 x i32> undef, ptr undef, <16 x i1> %m16)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 36 for instruction: call void @llvm.masked.compressstore.v8i32(<8 x i32> undef, ptr undef, <8 x i1> %m8)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: call void @llvm.masked.compressstore.v4i32(<4 x i32> undef, ptr undef, <4 x i1> %m4)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: call void @llvm.masked.compressstore.v2i32(<2 x i32> undef, ptr undef, <2 x i1> %m2)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 139 for instruction: call void @llvm.masked.compressstore.v32i16(<32 x i16> undef, ptr undef, <32 x i1> %m32)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 70 for instruction: call void @llvm.masked.compressstore.v16i16(<16 x i16> undef, ptr undef, <16 x i1> %m16)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 35 for instruction: call void @llvm.masked.compressstore.v8i16(<8 x i16> undef, ptr undef, <8 x i1> %m8)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: call void @llvm.masked.compressstore.v4i16(<4 x i16> undef, ptr undef, <4 x i1> %m4)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 276 for instruction: call void @llvm.masked.compressstore.v64i8(<64 x i8> undef, ptr undef, <64 x i1> %m64)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 138 for instruction: call void @llvm.masked.compressstore.v32i8(<32 x i8> undef, ptr undef, <32 x i1> %m32)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 69 for instruction: call void @llvm.masked.compressstore.v16i8(<16 x i8> undef, ptr undef, <16 x i1> %m16)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 35 for instruction: call void @llvm.masked.compressstore.v8i8(<8 x i8> undef, ptr undef, <8 x i1> %m8)
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 0
 ;
 ; SKL-LABEL: 'masked_compressstore'
-; SKL-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: call void @llvm.masked.compressstore.v8f64(<8 x double> undef, ptr undef, <8 x i1> %m8)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: call void @llvm.masked.compressstore.v4f64(<4 x double> undef, ptr undef, <4 x i1> %m4)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: call void @llvm.masked.compressstore.v2f64(<2 x double> undef, ptr undef, <2 x i1> %m2)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.compressstore.v1f64(<1 x double> undef, ptr undef, <1 x i1> %m1)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 31 for instruction: call void @llvm.masked.compressstore.v16f32(<16 x float> undef, ptr undef, <16 x i1> %m16)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: call void @llvm.masked.compressstore.v8f32(<8 x float> undef, ptr undef, <8 x i1> %m8)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: call void @llvm.masked.compressstore.v4f32(<4 x float> undef, ptr undef, <4 x i1> %m4)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: call void @llvm.masked.compressstore.v2f32(<2 x float> undef, ptr undef, <2 x i1> %m2)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 19 for instruction: call void @llvm.masked.compressstore.v8i64(<8 x i64> undef, ptr undef, <8 x i1> %m8)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: call void @llvm.masked.compressstore.v4i64(<4 x i64> undef, ptr undef, <4 x i1> %m4)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: call void @llvm.masked.compressstore.v2i64(<2 x i64> undef, ptr undef, <2 x i1> %m2)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.compressstore.v1i64(<1 x i64> undef, ptr undef, <1 x i1> %m1)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 35 for instruction: call void @llvm.masked.compressstore.v16i32(<16 x i32> undef, ptr undef, <16 x i1> %m16)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: call void @llvm.masked.compressstore.v8i32(<8 x i32> undef, ptr undef, <8 x i1> %m8)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: call void @llvm.masked.compressstore.v4i32(<4 x i32> undef, ptr undef, <4 x i1> %m4)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: call void @llvm.masked.compressstore.v2i32(<2 x i32> undef, ptr undef, <2 x i1> %m2)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 67 for instruction: call void @llvm.masked.compressstore.v32i16(<32 x i16> undef, ptr undef, <32 x i1> %m32)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: call void @llvm.masked.compressstore.v16i16(<16 x i16> undef, ptr undef, <16 x i1> %m16)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: call void @llvm.masked.compressstore.v8i16(<8 x i16> undef, ptr undef, <8 x i1> %m8)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: call void @llvm.masked.compressstore.v4i16(<4 x i16> undef, ptr undef, <4 x i1> %m4)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 132 for instruction: call void @llvm.masked.compressstore.v64i8(<64 x i8> undef, ptr undef, <64 x i1> %m64)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 66 for instruction: call void @llvm.masked.compressstore.v32i8(<32 x i8> undef, ptr undef, <32 x i1> %m32)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: call void @llvm.masked.compressstore.v16i8(<16 x i8> undef, ptr undef, <16 x i1> %m16)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: call void @llvm.masked.compressstore.v8i8(<8 x i8> undef, ptr undef, <8 x i1> %m8)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: call void @llvm.masked.compressstore.v8f64(<8 x double> undef, ptr undef, <8 x i1> %m8)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: call void @llvm.masked.compressstore.v4f64(<4 x double> undef, ptr undef, <4 x i1> %m4)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: call void @llvm.masked.compressstore.v2f64(<2 x double> undef, ptr undef, <2 x i1> %m2)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: call void @llvm.masked.compressstore.v1f64(<1 x double> undef, ptr undef, <1 x i1> %m1)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 67 for instruction: call void @llvm.masked.compressstore.v16f32(<16 x float> undef, ptr undef, <16 x i1> %m16)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: call void @llvm.masked.compressstore.v8f32(<8 x float> undef, ptr undef, <8 x i1> %m8)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: call void @llvm.masked.compressstore.v4f32(<4 x float> undef, ptr undef, <4 x i1> %m4)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: call void @llvm.masked.compressstore.v2f32(<2 x float> undef, ptr undef, <2 x i1> %m2)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 37 for instruction: call void @llvm.masked.compressstore.v8i64(<8 x i64> undef, ptr undef, <8 x i1> %m8)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 19 for instruction: call void @llvm.masked.compressstore.v4i64(<4 x i64> undef, ptr undef, <4 x i1> %m4)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: call void @llvm.masked.compressstore.v2i64(<2 x i64> undef, ptr undef, <2 x i1> %m2)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: call void @llvm.masked.compressstore.v1i64(<1 x i64> undef, ptr undef, <1 x i1> %m1)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 71 for instruction: call void @llvm.masked.compressstore.v16i32(<16 x i32> undef, ptr undef, <16 x i1> %m16)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 36 for instruction: call void @llvm.masked.compressstore.v8i32(<8 x i32> undef, ptr undef, <8 x i1> %m8)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: call void @llvm.masked.compressstore.v4i32(<4 x i32> undef, ptr undef, <4 x i1> %m4)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: call void @llvm.masked.compressstore.v2i32(<2 x i32> undef, ptr undef, <2 x i1> %m2)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 139 for instruction: call void @llvm.masked.compressstore.v32i16(<32 x i16> undef, ptr undef, <32 x i1> %m32)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 70 for instruction: call void @llvm.masked.compressstore.v16i16(<16 x i16> undef, ptr undef, <16 x i1> %m16)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 35 for instruction: call void @llvm.masked.compressstore.v8i16(<8 x i16> undef, ptr undef, <8 x i1> %m8)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: call void @llvm.masked.compressstore.v4i16(<4 x i16> undef, ptr undef, <4 x i1> %m4)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 276 for instruction: call void @llvm.masked.compressstore.v64i8(<64 x i8> undef, ptr undef, <64 x i1> %m64)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 138 for instruction: call void @llvm.masked.compressstore.v32i8(<32 x i8> undef, ptr undef, <32 x i1> %m32)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 69 for instruction: call void @llvm.masked.compressstore.v16i8(<16 x i8> undef, ptr undef, <16 x i1> %m16)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 35 for instruction: call void @llvm.masked.compressstore.v8i8(<8 x i8> undef, ptr undef, <8 x i1> %m8)
 ; SKL-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 0
 ;
 ; AVX512-LABEL: 'masked_compressstore'
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 23 for instruction: call void @llvm.masked.compressstore.v8f64(<8 x double> undef, ptr undef, <8 x i1> %m8)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: call void @llvm.masked.compressstore.v4f64(<4 x double> undef, ptr undef, <4 x i1> %m4)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: call void @llvm.masked.compressstore.v2f64(<2 x double> undef, ptr undef, <2 x i1> %m2)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.compressstore.v1f64(<1 x double> undef, ptr undef, <1 x i1> %m1)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 47 for instruction: call void @llvm.masked.compressstore.v16f32(<16 x float> undef, ptr undef, <16 x i1> %m16)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 23 for instruction: call void @llvm.masked.compressstore.v8f32(<8 x float> undef, ptr undef, <8 x i1> %m8)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: call void @llvm.masked.compressstore.v4f32(<4 x float> undef, ptr undef, <4 x i1> %m4)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: call void @llvm.masked.compressstore.v2f32(<2 x float> undef, ptr undef, <2 x i1> %m2)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 27 for instruction: call void @llvm.masked.compressstore.v8i64(<8 x i64> undef, ptr undef, <8 x i1> %m8)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: call void @llvm.masked.compressstore.v4i64(<4 x i64> undef, ptr undef, <4 x i1> %m4)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: call void @llvm.masked.compressstore.v2i64(<2 x i64> undef, ptr undef, <2 x i1> %m2)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.compressstore.v1i64(<1 x i64> undef, ptr undef, <1 x i1> %m1)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 51 for instruction: call void @llvm.masked.compressstore.v16i32(<16 x i32> undef, ptr undef, <16 x i1> %m16)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: call void @llvm.masked.compressstore.v8i32(<8 x i32> undef, ptr undef, <8 x i1> %m8)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: call void @llvm.masked.compressstore.v4i32(<4 x i32> undef, ptr undef, <4 x i1> %m4)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: call void @llvm.masked.compressstore.v2i32(<2 x i32> undef, ptr undef, <2 x i1> %m2)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 99 for instruction: call void @llvm.masked.compressstore.v32i16(<32 x i16> undef, ptr undef, <32 x i1> %m32)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 49 for instruction: call void @llvm.masked.compressstore.v16i16(<16 x i16> undef, ptr undef, <16 x i1> %m16)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: call void @llvm.masked.compressstore.v8i16(<8 x i16> undef, ptr undef, <8 x i1> %m8)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: call void @llvm.masked.compressstore.v4i16(<4 x i16> undef, ptr undef, <4 x i1> %m4)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 195 for instruction: call void @llvm.masked.compressstore.v64i8(<64 x i8> undef, ptr undef, <64 x i1> %m64)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 97 for instruction: call void @llvm.masked.compressstore.v32i8(<32 x i8> undef, ptr undef, <32 x i1> %m32)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: call void @llvm.masked.compressstore.v16i8(<16 x i8> undef, ptr undef, <16 x i1> %m16)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: call void @llvm.masked.compressstore.v8i8(<8 x i8> undef, ptr undef, <8 x i1> %m8)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 42 for instruction: call void @llvm.masked.compressstore.v8f64(<8 x double> undef, ptr undef, <8 x i1> %m8)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: call void @llvm.masked.compressstore.v4f64(<4 x double> undef, ptr undef, <4 x i1> %m4)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: call void @llvm.masked.compressstore.v2f64(<2 x double> undef, ptr undef, <2 x i1> %m2)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: call void @llvm.masked.compressstore.v1f64(<1 x double> undef, ptr undef, <1 x i1> %m1)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 85 for instruction: call void @llvm.masked.compressstore.v16f32(<16 x float> undef, ptr undef, <16 x i1> %m16)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 42 for instruction: call void @llvm.masked.compressstore.v8f32(<8 x float> undef, ptr undef, <8 x i1> %m8)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: call void @llvm.masked.compressstore.v4f32(<4 x float> undef, ptr undef, <4 x i1> %m4)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: call void @llvm.masked.compressstore.v2f32(<2 x float> undef, ptr undef, <2 x i1> %m2)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 46 for instruction: call void @llvm.masked.compressstore.v8i64(<8 x i64> undef, ptr undef, <8 x i1> %m8)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 22 for instruction: call void @llvm.masked.compressstore.v4i64(<4 x i64> undef, ptr undef, <4 x i1> %m4)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: call void @llvm.masked.compressstore.v2i64(<2 x i64> undef, ptr undef, <2 x i1> %m2)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: call void @llvm.masked.compressstore.v1i64(<1 x i64> undef, ptr undef, <1 x i1> %m1)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 89 for instruction: call void @llvm.masked.compressstore.v16i32(<16 x i32> undef, ptr undef, <16 x i1> %m16)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 44 for instruction: call void @llvm.masked.compressstore.v8i32(<8 x i32> undef, ptr undef, <8 x i1> %m8)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 21 for instruction: call void @llvm.masked.compressstore.v4i32(<4 x i32> undef, ptr undef, <4 x i1> %m4)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: call void @llvm.masked.compressstore.v2i32(<2 x i32> undef, ptr undef, <2 x i1> %m2)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 175 for instruction: call void @llvm.masked.compressstore.v32i16(<32 x i16> undef, ptr undef, <32 x i1> %m32)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 87 for instruction: call void @llvm.masked.compressstore.v16i16(<16 x i16> undef, ptr undef, <16 x i1> %m16)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 43 for instruction: call void @llvm.masked.compressstore.v8i16(<8 x i16> undef, ptr undef, <8 x i1> %m8)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 21 for instruction: call void @llvm.masked.compressstore.v4i16(<4 x i16> undef, ptr undef, <4 x i1> %m4)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 347 for instruction: call void @llvm.masked.compressstore.v64i8(<64 x i8> undef, ptr undef, <64 x i1> %m64)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 173 for instruction: call void @llvm.masked.compressstore.v32i8(<32 x i8> undef, ptr undef, <32 x i1> %m32)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 86 for instruction: call void @llvm.masked.compressstore.v16i8(<16 x i8> undef, ptr undef, <16 x i1> %m16)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 43 for instruction: call void @llvm.masked.compressstore.v8i8(<8 x i8> undef, ptr undef, <8 x i1> %m8)
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 0
 ;
   call void @llvm.masked.compressstore.v8f64(<8 x double> undef, ptr undef, <8 x i1> %m8)
diff --git a/llvm/test/Analysis/CostModel/X86/masked-intrinsic-sizelatency.ll b/llvm/test/Analysis/CostModel/X86/masked-intrinsic-sizelatency.ll
index 6be81602ab15b..bea77c82b9f90 100644
--- a/llvm/test/Analysis/CostModel/X86/masked-intrinsic-sizelatency.ll
+++ b/llvm/test/Analysis/CostModel/X86/masked-intrinsic-sizelatency.ll
@@ -1178,165 +1178,165 @@ define i32 @masked_scatter(<1 x i1> %m1, <2 x i1> %m2, <4 x i1> %m4, <8 x i1> %m
 
 define i32 @masked_expandload(<1 x i1> %m1, <2 x i1> %m2, <4 x i1> %m4, <8 x i1> %m8, <16 x i1> %m16, <32 x i1> %m32, <64 x i1> %m64) {
 ; SSE2-LABEL: 'masked_expandload'
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %V8F64 = call <8 x double> @llvm.masked.expandload.v8f64(ptr undef, <8 x i1> %m8, <8 x double> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V4F64 = call <4 x double> @llvm.masked.expandload.v4f64(ptr undef, <4 x i1> %m4, <4 x double> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V2F64 = call <2 x double> @llvm.masked.expandload.v2f64(ptr undef, <2 x i1> %m2, <2 x double> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1F64 = call <1 x double> @llvm.masked.expandload.v1f64(ptr undef, <1 x i1> %m1, <1 x double> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 29 for instruction: %V16F32 = call <16 x float> @llvm.masked.expandload.v16f32(ptr undef, <16 x i1> %m16, <16 x float> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %V8F32 = call <8 x float> @llvm.masked.expandload.v8f32(ptr undef, <8 x i1> %m8, <8 x float> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V4F32 = call <4 x float> @llvm.masked.expandload.v4f32(ptr undef, <4 x i1> %m4, <4 x float> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V2F32 = call <2 x float> @llvm.masked.expandload.v2f32(ptr undef, <2 x i1> %m2, <2 x float> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 21 for instruction: %V8I64 = call <8 x i64> @llvm.masked.expandload.v8i64(ptr undef, <8 x i1> %m8, <8 x i64> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %V4I64 = call <4 x i64> @llvm.masked.expandload.v4i64(ptr undef, <4 x i1> %m4, <4 x i64> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2I64 = call <2 x i64> @llvm.masked.expandload.v2i64(ptr undef, <2 x i1> %m2, <2 x i64> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1I64 = call <1 x i64> @llvm.masked.expandload.v1i64(ptr undef, <1 x i1> %m1, <1 x i64> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 45 for instruction: %V16I32 = call <16 x i32> @llvm.masked.expandload.v16i32(ptr undef, <16 x i1> %m16, <16 x i32> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 23 for instruction: %V8I32 = call <8 x i32> @llvm.masked.expandload.v8i32(ptr undef, <8 x i1> %m8, <8 x i32> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V4I32 = call <4 x i32> @llvm.masked.expandload.v4i32(ptr undef, <4 x i1> %m4, <4 x i32> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2I32 = call <2 x i32> @llvm.masked.expandload.v2i32(ptr undef, <2 x i1> %m2, <2 x i32> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 66 for instruction: %V32I16 = call <32 x i16> @llvm.masked.expandload.v32i16(ptr undef, <32 x i1> %m32, <32 x i16> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %V16I16 = call <16 x i16> @llvm.masked.expandload.v16i16(ptr undef, <16 x i1> %m16, <16 x i16> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %V8I16 = call <8 x i16> @llvm.masked.expandload.v8i16(ptr undef, <8 x i1> %m8, <8 x i16> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V4I16 = call <4 x i16> @llvm.masked.expandload.v4i16(ptr undef, <4 x i1> %m4, <4 x i16> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 192 for instruction: %V64I8 = call <64 x i8> @llvm.masked.expandload.v64i8(ptr undef, <64 x i1> %m64, <64 x i8> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 96 for instruction: %V32I8 = call <32 x i8> @llvm.masked.expandload.v32i8(ptr undef, <32 x i1> %m32, <32 x i8> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V16I8 = call <16 x i8> @llvm.masked.expandload.v16i8(ptr undef, <16 x i1> %m16, <16 x i8> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V8I8 = call <8 x i8> @llvm.masked.expandload.v8i8(ptr undef, <8 x i1> %m8, <8 x i8> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 37 for instruction: %V8F64 = call <8 x double> @llvm.masked.expandload.v8f64(ptr undef, <8 x i1> %m8, <8 x double> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 19 for instruction: %V4F64 = call <4 x double> @llvm.masked.expandload.v4f64(ptr undef, <4 x i1> %m4, <4 x double> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V2F64 = call <2 x double> @llvm.masked.expandload.v2f64(ptr undef, <2 x i1> %m2, <2 x double> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V1F64 = call <1 x double> @llvm.masked.expandload.v1f64(ptr undef, <1 x i1> %m1, <1 x double> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 77 for instruction: %V16F32 = call <16 x float> @llvm.masked.expandload.v16f32(ptr undef, <16 x i1> %m16, <16 x float> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 39 for instruction: %V8F32 = call <8 x float> @llvm.masked.expandload.v8f32(ptr undef, <8 x i1> %m8, <8 x float> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %V4F32 = call <4 x float> @llvm.masked.expandload.v4f32(ptr undef, <4 x i1> %m4, <4 x float> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V2F32 = call <2 x float> @llvm.masked.expandload.v2f32(ptr undef, <2 x i1> %m2, <2 x float> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 45 for instruction: %V8I64 = call <8 x i64> @llvm.masked.expandload.v8i64(ptr undef, <8 x i1> %m8, <8 x i64> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 23 for instruction: %V4I64 = call <4 x i64> @llvm.masked.expandload.v4i64(ptr undef, <4 x i1> %m4, <4 x i64> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V2I64 = call <2 x i64> @llvm.masked.expandload.v2i64(ptr undef, <2 x i1> %m2, <2 x i64> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V1I64 = call <1 x i64> @llvm.masked.expandload.v1i64(ptr undef, <1 x i1> %m1, <1 x i64> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 93 for instruction: %V16I32 = call <16 x i32> @llvm.masked.expandload.v16i32(ptr undef, <16 x i1> %m16, <16 x i32> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 47 for instruction: %V8I32 = call <8 x i32> @llvm.masked.expandload.v8i32(ptr undef, <8 x i1> %m8, <8 x i32> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V4I32 = call <4 x i32> @llvm.masked.expandload.v4i32(ptr undef, <4 x i1> %m4, <4 x i32> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V2I32 = call <2 x i32> @llvm.masked.expandload.v2i32(ptr undef, <2 x i1> %m2, <2 x i32> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 162 for instruction: %V32I16 = call <32 x i16> @llvm.masked.expandload.v32i16(ptr undef, <32 x i1> %m32, <32 x i16> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 81 for instruction: %V16I16 = call <16 x i16> @llvm.masked.expandload.v16i16(ptr undef, <16 x i1> %m16, <16 x i16> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 41 for instruction: %V8I16 = call <8 x i16> @llvm.masked.expandload.v8i16(ptr undef, <8 x i1> %m8, <8 x i16> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 21 for instruction: %V4I16 = call <4 x i16> @llvm.masked.expandload.v4i16(ptr undef, <4 x i1> %m4, <4 x i16> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 384 for instruction: %V64I8 = call <64 x i8> @llvm.masked.expandload.v64i8(ptr undef, <64 x i1> %m64, <64 x i8> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 192 for instruction: %V32I8 = call <32 x i8> @llvm.masked.expandload.v32i8(ptr undef, <32 x i1> %m32, <32 x i8> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 96 for instruction: %V16I8 = call <16 x i8> @llvm.masked.expandload.v16i8(ptr undef, <16 x i1> %m16, <16 x i8> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V8I8 = call <8 x i8> @llvm.masked.expandload.v8i8(ptr undef, <8 x i1> %m8, <8 x i8> undef)
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 0
 ;
 ; SSE42-LABEL: 'masked_expandload'
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %V8F64 = call <8 x double> @llvm.masked.expandload.v8f64(ptr undef, <8 x i1> %m8, <8 x double> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V4F64 = call <4 x double> @llvm.masked.expandload.v4f64(ptr undef, <4 x i1> %m4, <4 x double> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V2F64 = call <2 x double> @llvm.masked.expandload.v2f64(ptr undef, <2 x i1> %m2, <2 x double> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1F64 = call <1 x double> @llvm.masked.expandload.v1f64(ptr undef, <1 x i1> %m1, <1 x double> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 29 for instruction: %V16F32 = call <16 x float> @llvm.masked.expandload.v16f32(ptr undef, <16 x i1> %m16, <16 x float> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %V8F32 = call <8 x float> @llvm.masked.expandload.v8f32(ptr undef, <8 x i1> %m8, <8 x float> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V4F32 = call <4 x float> @llvm.masked.expandload.v4f32(ptr undef, <4 x i1> %m4, <4 x float> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V2F32 = call <2 x float> @llvm.masked.expandload.v2f32(ptr undef, <2 x i1> %m2, <2 x float> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %V8I64 = call <8 x i64> @llvm.masked.expandload.v8i64(ptr undef, <8 x i1> %m8, <8 x i64> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V4I64 = call <4 x i64> @llvm.masked.expandload.v4i64(ptr undef, <4 x i1> %m4, <4 x i64> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V2I64 = call <2 x i64> @llvm.masked.expandload.v2i64(ptr undef, <2 x i1> %m2, <2 x i64> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1I64 = call <1 x i64> @llvm.masked.expandload.v1i64(ptr undef, <1 x i1> %m1, <1 x i64> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %V16I32 = call <16 x i32> @llvm.masked.expandload.v16i32(ptr undef, <16 x i1> %m16, <16 x i32> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %V8I32 = call <8 x i32> @llvm.masked.expandload.v8i32(ptr undef, <8 x i1> %m8, <8 x i32> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V4I32 = call <4 x i32> @llvm.masked.expandload.v4i32(ptr undef, <4 x i1> %m4, <4 x i32> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V2I32 = call <2 x i32> @llvm.masked.expandload.v2i32(ptr undef, <2 x i1> %m2, <2 x i32> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 66 for instruction: %V32I16 = call <32 x i16> @llvm.masked.expandload.v32i16(ptr undef, <32 x i1> %m32, <32 x i16> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %V16I16 = call <16 x i16> @llvm.masked.expandload.v16i16(ptr undef, <16 x i1> %m16, <16 x i16> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %V8I16 = call <8 x i16> @llvm.masked.expandload.v8i16(ptr undef, <8 x i1> %m8, <8 x i16> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V4I16 = call <4 x i16> @llvm.masked.expandload.v4i16(ptr undef, <4 x i1> %m4, <4 x i16> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 132 for instruction: %V64I8 = call <64 x i8> @llvm.masked.expandload.v64i8(ptr undef, <64 x i1> %m64, <64 x i8> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 66 for instruction: %V32I8 = call <32 x i8> @llvm.masked.expandload.v32i8(ptr undef, <32 x i1> %m32, <32 x i8> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %V16I8 = call <16 x i8> @llvm.masked.expandload.v16i8(ptr undef, <16 x i1> %m16, <16 x i8> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %V8I8 = call <8 x i8> @llvm.masked.expandload.v8i8(ptr undef, <8 x i1> %m8, <8 x i8> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 29 for instruction: %V8F64 = call <8 x double> @llvm.masked.expandload.v8f64(ptr undef, <8 x i1> %m8, <8 x double> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %V4F64 = call <4 x double> @llvm.masked.expandload.v4f64(ptr undef, <4 x i1> %m4, <4 x double> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V2F64 = call <2 x double> @llvm.masked.expandload.v2f64(ptr undef, <2 x i1> %m2, <2 x double> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V1F64 = call <1 x double> @llvm.masked.expandload.v1f64(ptr undef, <1 x i1> %m1, <1 x double> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 61 for instruction: %V16F32 = call <16 x float> @llvm.masked.expandload.v16f32(ptr undef, <16 x i1> %m16, <16 x float> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 31 for instruction: %V8F32 = call <8 x float> @llvm.masked.expandload.v8f32(ptr undef, <8 x i1> %m8, <8 x float> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V4F32 = call <4 x float> @llvm.masked.expandload.v4f32(ptr undef, <4 x i1> %m4, <4 x float> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V2F32 = call <2 x float> @llvm.masked.expandload.v2f32(ptr undef, <2 x i1> %m2, <2 x float> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %V8I64 = call <8 x i64> @llvm.masked.expandload.v8i64(ptr undef, <8 x i1> %m8, <8 x i64> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %V4I64 = call <4 x i64> @llvm.masked.expandload.v4i64(ptr undef, <4 x i1> %m4, <4 x i64> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V2I64 = call <2 x i64> @llvm.masked.expandload.v2i64(ptr undef, <2 x i1> %m2, <2 x i64> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V1I64 = call <1 x i64> @llvm.masked.expandload.v1i64(ptr undef, <1 x i1> %m1, <1 x i64> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 65 for instruction: %V16I32 = call <16 x i32> @llvm.masked.expandload.v16i32(ptr undef, <16 x i1> %m16, <16 x i32> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %V8I32 = call <8 x i32> @llvm.masked.expandload.v8i32(ptr undef, <8 x i1> %m8, <8 x i32> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %V4I32 = call <4 x i32> @llvm.masked.expandload.v4i32(ptr undef, <4 x i1> %m4, <4 x i32> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V2I32 = call <2 x i32> @llvm.masked.expandload.v2i32(ptr undef, <2 x i1> %m2, <2 x i32> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 130 for instruction: %V32I16 = call <32 x i16> @llvm.masked.expandload.v32i16(ptr undef, <32 x i1> %m32, <32 x i16> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 65 for instruction: %V16I16 = call <16 x i16> @llvm.masked.expandload.v16i16(ptr undef, <16 x i1> %m16, <16 x i16> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %V8I16 = call <8 x i16> @llvm.masked.expandload.v8i16(ptr undef, <8 x i1> %m8, <8 x i16> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %V4I16 = call <4 x i16> @llvm.masked.expandload.v4i16(ptr undef, <4 x i1> %m4, <4 x i16> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 260 for instruction: %V64I8 = call <64 x i8> @llvm.masked.expandload.v64i8(ptr undef, <64 x i1> %m64, <64 x i8> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 130 for instruction: %V32I8 = call <32 x i8> @llvm.masked.expandload.v32i8(ptr undef, <32 x i1> %m32, <32 x i8> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 65 for instruction: %V16I8 = call <16 x i8> @llvm.masked.expandload.v16i8(ptr undef, <16 x i1> %m16, <16 x i8> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %V8I8 = call <8 x i8> @llvm.masked.expandload.v8i8(ptr undef, <8 x i1> %m8, <8 x i8> undef)
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 0
 ;
 ; AVX1-LABEL: 'masked_expandload'
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %V8F64 = call <8 x double> @llvm.masked.expandload.v8f64(ptr undef, <8 x i1> %m8, <8 x double> undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V4F64 = call <4 x double> @llvm.masked.expandload.v4f64(ptr undef, <4 x i1> %m4, <4 x double> undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V2F64 = call <2 x double> @llvm.masked.expandload.v2f64(ptr undef, <2 x i1> %m2, <2 x double> undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1F64 = call <1 x double> @llvm.masked.expandload.v1f64(ptr undef, <1 x i1> %m1, <1 x double> undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 31 for instruction: %V16F32 = call <16 x float> @llvm.masked.expandload.v16f32(ptr undef, <16 x i1> %m16, <16 x float> undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V8F32 = call <8 x float> @llvm.masked.expandload.v8f32(ptr undef, <8 x i1> %m8, <8 x float> undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V4F32 = call <4 x float> @llvm.masked.expandload.v4f32(ptr undef, <4 x i1> %m4, <4 x float> undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V2F32 = call <2 x float> @llvm.masked.expandload.v2f32(ptr undef, <2 x i1> %m2, <2 x float> undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 19 for instruction: %V8I64 = call <8 x i64> @llvm.masked.expandload.v8i64(ptr undef, <8 x i1> %m8, <8 x i64> undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V4I64 = call <4 x i64> @llvm.masked.expandload.v4i64(ptr undef, <4 x i1> %m4, <4 x i64> undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V2I64 = call <2 x i64> @llvm.masked.expandload.v2i64(ptr undef, <2 x i1> %m2, <2 x i64> undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1I64 = call <1 x i64> @llvm.masked.expandload.v1i64(ptr undef, <1 x i1> %m1, <1 x i64> undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 35 for instruction: %V16I32 = call <16 x i32> @llvm.masked.expandload.v16i32(ptr undef, <16 x i1> %m16, <16 x i32> undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V8I32 = call <8 x i32> @llvm.masked.expandload.v8i32(ptr undef, <8 x i1> %m8, <8 x i32> undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V4I32 = call <4 x i32> @llvm.masked.expandload.v4i32(ptr undef, <4 x i1> %m4, <4 x i32> undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V2I32 = call <2 x i32> @llvm.masked.expandload.v2i32(ptr undef, <2 x i1> %m2, <2 x i32> undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 68 for instruction: %V32I16 = call <32 x i16> @llvm.masked.expandload.v32i16(ptr undef, <32 x i1> %m32, <32 x i16> undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %V16I16 = call <16 x i16> @llvm.masked.expandload.v16i16(ptr undef, <16 x i1> %m16, <16 x i16> undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %V8I16 = call <8 x i16> @llvm.masked.expandload.v8i16(ptr undef, <8 x i1> %m8, <8 x i16> undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V4I16 = call <4 x i16> @llvm.masked.expandload.v4i16(ptr undef, <4 x i1> %m4, <4 x i16> undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 134 for instruction: %V64I8 = call <64 x i8> @llvm.masked.expandload.v64i8(ptr undef, <64 x i1> %m64, <64 x i8> undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 67 for instruction: %V32I8 = call <32 x i8> @llvm.masked.expandload.v32i8(ptr undef, <32 x i1> %m32, <32 x i8> undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %V16I8 = call <16 x i8> @llvm.masked.expandload.v16i8(ptr undef, <16 x i1> %m16, <16 x i8> undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %V8I8 = call <8 x i8> @llvm.masked.expandload.v8i8(ptr undef, <8 x i1> %m8, <8 x i8> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %V8F64 = call <8 x double> @llvm.masked.expandload.v8f64(ptr undef, <8 x i1> %m8, <8 x double> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %V4F64 = call <4 x double> @llvm.masked.expandload.v4f64(ptr undef, <4 x i1> %m4, <4 x double> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V2F64 = call <2 x double> @llvm.masked.expandload.v2f64(ptr undef, <2 x i1> %m2, <2 x double> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V1F64 = call <1 x double> @llvm.masked.expandload.v1f64(ptr undef, <1 x i1> %m1, <1 x double> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 67 for instruction: %V16F32 = call <16 x float> @llvm.masked.expandload.v16f32(ptr undef, <16 x i1> %m16, <16 x float> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %V8F32 = call <8 x float> @llvm.masked.expandload.v8f32(ptr undef, <8 x i1> %m8, <8 x float> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %V4F32 = call <4 x float> @llvm.masked.expandload.v4f32(ptr undef, <4 x i1> %m4, <4 x float> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V2F32 = call <2 x float> @llvm.masked.expandload.v2f32(ptr undef, <2 x i1> %m2, <2 x float> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 37 for instruction: %V8I64 = call <8 x i64> @llvm.masked.expandload.v8i64(ptr undef, <8 x i1> %m8, <8 x i64> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 19 for instruction: %V4I64 = call <4 x i64> @llvm.masked.expandload.v4i64(ptr undef, <4 x i1> %m4, <4 x i64> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V2I64 = call <2 x i64> @llvm.masked.expandload.v2i64(ptr undef, <2 x i1> %m2, <2 x i64> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V1I64 = call <1 x i64> @llvm.masked.expandload.v1i64(ptr undef, <1 x i1> %m1, <1 x i64> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 71 for instruction: %V16I32 = call <16 x i32> @llvm.masked.expandload.v16i32(ptr undef, <16 x i1> %m16, <16 x i32> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 36 for instruction: %V8I32 = call <8 x i32> @llvm.masked.expandload.v8i32(ptr undef, <8 x i1> %m8, <8 x i32> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V4I32 = call <4 x i32> @llvm.masked.expandload.v4i32(ptr undef, <4 x i1> %m4, <4 x i32> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V2I32 = call <2 x i32> @llvm.masked.expandload.v2i32(ptr undef, <2 x i1> %m2, <2 x i32> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 140 for instruction: %V32I16 = call <32 x i16> @llvm.masked.expandload.v32i16(ptr undef, <32 x i1> %m32, <32 x i16> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 70 for instruction: %V16I16 = call <16 x i16> @llvm.masked.expandload.v16i16(ptr undef, <16 x i1> %m16, <16 x i16> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 35 for instruction: %V8I16 = call <8 x i16> @llvm.masked.expandload.v8i16(ptr undef, <8 x i1> %m8, <8 x i16> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V4I16 = call <4 x i16> @llvm.masked.expandload.v4i16(ptr undef, <4 x i1> %m4, <4 x i16> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 278 for instruction: %V64I8 = call <64 x i8> @llvm.masked.expandload.v64i8(ptr undef, <64 x i1> %m64, <64 x i8> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 139 for instruction: %V32I8 = call <32 x i8> @llvm.masked.expandload.v32i8(ptr undef, <32 x i1> %m32, <32 x i8> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 69 for instruction: %V16I8 = call <16 x i8> @llvm.masked.expandload.v16i8(ptr undef, <16 x i1> %m16, <16 x i8> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 35 for instruction: %V8I8 = call <8 x i8> @llvm.masked.expandload.v8i8(ptr undef, <8 x i1> %m8, <8 x i8> undef)
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 0
 ;
 ; AVX2-LABEL: 'masked_expandload'
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %V8F64 = call <8 x double> @llvm.masked.expandload.v8f64(ptr undef, <8 x i1> %m8, <8 x double> undef)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V4F64 = call <4 x double> @llvm.masked.expandload.v4f64(ptr undef, <4 x i1> %m4, <4 x double> undef)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V2F64 = call <2 x double> @llvm.masked.expandload.v2f64(ptr undef, <2 x i1> %m2, <2 x double> undef)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1F64 = call <1 x double> @llvm.masked.expandload.v1f64(ptr undef, <1 x i1> %m1, <1 x double> undef)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 31 for instruction: %V16F32 = call <16 x float> @llvm.masked.expandload.v16f32(ptr undef, <16 x i1> %m16, <16 x float> undef)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V8F32 = call <8 x float> @llvm.masked.expandload.v8f32(ptr undef, <8 x i1> %m8, <8 x float> undef)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V4F32 = call <4 x float> @llvm.masked.expandload.v4f32(ptr undef, <4 x i1> %m4, <4 x float> undef)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V2F32 = call <2 x float> @llvm.masked.expandload.v2f32(ptr undef, <2 x i1> %m2, <2 x float> undef)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 19 for instruction: %V8I64 = call <8 x i64> @llvm.masked.expandload.v8i64(ptr undef, <8 x i1> %m8, <8 x i64> undef)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V4I64 = call <4 x i64> @llvm.masked.expandload.v4i64(ptr undef, <4 x i1> %m4, <4 x i64> undef)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V2I64 = call <2 x i64> @llvm.masked.expandload.v2i64(ptr undef, <2 x i1> %m2, <2 x i64> undef)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1I64 = call <1 x i64> @llvm.masked.expandload.v1i64(ptr undef, <1 x i1> %m1, <1 x i64> undef)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 35 for instruction: %V16I32 = call <16 x i32> @llvm.masked.expandload.v16i32(ptr undef, <16 x i1> %m16, <16 x i32> undef)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V8I32 = call <8 x i32> @llvm.masked.expandload.v8i32(ptr undef, <8 x i1> %m8, <8 x i32> undef)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V4I32 = call <4 x i32> @llvm.masked.expandload.v4i32(ptr undef, <4 x i1> %m4, <4 x i32> undef)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V2I32 = call <2 x i32> @llvm.masked.expandload.v2i32(ptr undef, <2 x i1> %m2, <2 x i32> undef)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 67 for instruction: %V32I16 = call <32 x i16> @llvm.masked.expandload.v32i16(ptr undef, <32 x i1> %m32, <32 x i16> undef)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %V16I16 = call <16 x i16> @llvm.masked.expandload.v16i16(ptr undef, <16 x i1> %m16, <16 x i16> undef)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %V8I16 = call <8 x i16> @llvm.masked.expandload.v8i16(ptr undef, <8 x i1> %m8, <8 x i16> undef)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V4I16 = call <4 x i16> @llvm.masked.expandload.v4i16(ptr undef, <4 x i1> %m4, <4 x i16> undef)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 132 for instruction: %V64I8 = call <64 x i8> @llvm.masked.expandload.v64i8(ptr undef, <64 x i1> %m64, <64 x i8> undef)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 66 for instruction: %V32I8 = call <32 x i8> @llvm.masked.expandload.v32i8(ptr undef, <32 x i1> %m32, <32 x i8> undef)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %V16I8 = call <16 x i8> @llvm.masked.expandload.v16i8(ptr undef, <16 x i1> %m16, <16 x i8> undef)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %V8I8 = call <8 x i8> @llvm.masked.expandload.v8i8(ptr undef, <8 x i1> %m8, <8 x i8> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %V8F64 = call <8 x double> @llvm.masked.expandload.v8f64(ptr undef, <8 x i1> %m8, <8 x double> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %V4F64 = call <4 x double> @llvm.masked.expandload.v4f64(ptr undef, <4 x i1> %m4, <4 x double> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V2F64 = call <2 x double> @llvm.masked.expandload.v2f64(ptr undef, <2 x i1> %m2, <2 x double> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V1F64 = call <1 x double> @llvm.masked.expandload.v1f64(ptr undef, <1 x i1> %m1, <1 x double> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 67 for instruction: %V16F32 = call <16 x float> @llvm.masked.expandload.v16f32(ptr undef, <16 x i1> %m16, <16 x float> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %V8F32 = call <8 x float> @llvm.masked.expandload.v8f32(ptr undef, <8 x i1> %m8, <8 x float> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %V4F32 = call <4 x float> @llvm.masked.expandload.v4f32(ptr undef, <4 x i1> %m4, <4 x float> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V2F32 = call <2 x float> @llvm.masked.expandload.v2f32(ptr undef, <2 x i1> %m2, <2 x float> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 37 for instruction: %V8I64 = call <8 x i64> @llvm.masked.expandload.v8i64(ptr undef, <8 x i1> %m8, <8 x i64> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 19 for instruction: %V4I64 = call <4 x i64> @llvm.masked.expandload.v4i64(ptr undef, <4 x i1> %m4, <4 x i64> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V2I64 = call <2 x i64> @llvm.masked.expandload.v2i64(ptr undef, <2 x i1> %m2, <2 x i64> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V1I64 = call <1 x i64> @llvm.masked.expandload.v1i64(ptr undef, <1 x i1> %m1, <1 x i64> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 71 for instruction: %V16I32 = call <16 x i32> @llvm.masked.expandload.v16i32(ptr undef, <16 x i1> %m16, <16 x i32> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 36 for instruction: %V8I32 = call <8 x i32> @llvm.masked.expandload.v8i32(ptr undef, <8 x i1> %m8, <8 x i32> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V4I32 = call <4 x i32> @llvm.masked.expandload.v4i32(ptr undef, <4 x i1> %m4, <4 x i32> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V2I32 = call <2 x i32> @llvm.masked.expandload.v2i32(ptr undef, <2 x i1> %m2, <2 x i32> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 139 for instruction: %V32I16 = call <32 x i16> @llvm.masked.expandload.v32i16(ptr undef, <32 x i1> %m32, <32 x i16> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 70 for instruction: %V16I16 = call <16 x i16> @llvm.masked.expandload.v16i16(ptr undef, <16 x i1> %m16, <16 x i16> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 35 for instruction: %V8I16 = call <8 x i16> @llvm.masked.expandload.v8i16(ptr undef, <8 x i1> %m8, <8 x i16> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V4I16 = call <4 x i16> @llvm.masked.expandload.v4i16(ptr undef, <4 x i1> %m4, <4 x i16> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 276 for instruction: %V64I8 = call <64 x i8> @llvm.masked.expandload.v64i8(ptr undef, <64 x i1> %m64, <64 x i8> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 138 for instruction: %V32I8 = call <32 x i8> @llvm.masked.expandload.v32i8(ptr undef, <32 x i1> %m32, <32 x i8> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 69 for instruction: %V16I8 = call <16 x i8> @llvm.masked.expandload.v16i8(ptr undef, <16 x i1> %m16, <16 x i8> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 35 for instruction: %V8I8 = call <8 x i8> @llvm.masked.expandload.v8i8(ptr undef, <8 x i1> %m8, <8 x i8> undef)
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 0
 ;
 ; SKL-LABEL: 'masked_expandload'
-; SKL-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %V8F64 = call <8 x double> @llvm.masked.expandload.v8f64(ptr undef, <8 x i1> %m8, <8 x double> undef)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V4F64 = call <4 x double> @llvm.masked.expandload.v4f64(ptr undef, <4 x i1> %m4, <4 x double> undef)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V2F64 = call <2 x double> @llvm.masked.expandload.v2f64(ptr undef, <2 x i1> %m2, <2 x double> undef)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1F64 = call <1 x double> @llvm.masked.expandload.v1f64(ptr undef, <1 x i1> %m1, <1 x double> undef)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 31 for instruction: %V16F32 = call <16 x float> @llvm.masked.expandload.v16f32(ptr undef, <16 x i1> %m16, <16 x float> undef)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V8F32 = call <8 x float> @llvm.masked.expandload.v8f32(ptr undef, <8 x i1> %m8, <8 x float> undef)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V4F32 = call <4 x float> @llvm.masked.expandload.v4f32(ptr undef, <4 x i1> %m4, <4 x float> undef)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V2F32 = call <2 x float> @llvm.masked.expandload.v2f32(ptr undef, <2 x i1> %m2, <2 x float> undef)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 19 for instruction: %V8I64 = call <8 x i64> @llvm.masked.expandload.v8i64(ptr undef, <8 x i1> %m8, <8 x i64> undef)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V4I64 = call <4 x i64> @llvm.masked.expandload.v4i64(ptr undef, <4 x i1> %m4, <4 x i64> undef)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V2I64 = call <2 x i64> @llvm.masked.expandload.v2i64(ptr undef, <2 x i1> %m2, <2 x i64> undef)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1I64 = call <1 x i64> @llvm.masked.expandload.v1i64(ptr undef, <1 x i1> %m1, <1 x i64> undef)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 35 for instruction: %V16I32 = call <16 x i32> @llvm.masked.expandload.v16i32(ptr undef, <16 x i1> %m16, <16 x i32> undef)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V8I32 = call <8 x i32> @llvm.masked.expandload.v8i32(ptr undef, <8 x i1> %m8, <8 x i32> undef)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V4I32 = call <4 x i32> @llvm.masked.expandload.v4i32(ptr undef, <4 x i1> %m4, <4 x i32> undef)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V2I32 = call <2 x i32> @llvm.masked.expandload.v2i32(ptr undef, <2 x i1> %m2, <2 x i32> undef)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 67 for instruction: %V32I16 = call <32 x i16> @llvm.masked.expandload.v32i16(ptr undef, <32 x i1> %m32, <32 x i16> undef)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %V16I16 = call <16 x i16> @llvm.masked.expandload.v16i16(ptr undef, <16 x i1> %m16, <16 x i16> undef)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %V8I16 = call <8 x i16> @llvm.masked.expandload.v8i16(ptr undef, <8 x i1> %m8, <8 x i16> undef)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V4I16 = call <4 x i16> @llvm.masked.expandload.v4i16(ptr undef, <4 x i1> %m4, <4 x i16> undef)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 132 for instruction: %V64I8 = call <64 x i8> @llvm.masked.expandload.v64i8(ptr undef, <64 x i1> %m64, <64 x i8> undef)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 66 for instruction: %V32I8 = call <32 x i8> @llvm.masked.expandload.v32i8(ptr undef, <32 x i1> %m32, <32 x i8> undef)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %V16I8 = call <16 x i8> @llvm.masked.expandload.v16i8(ptr undef, <16 x i1> %m16, <16 x i8> undef)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %V8I8 = call <8 x i8> @llvm.masked.expandload.v8i8(ptr undef, <8 x i1> %m8, <8 x i8> undef)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %V8F64 = call <8 x double> @llvm.masked.expandload.v8f64(ptr undef, <8 x i1> %m8, <8 x double> undef)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %V4F64 = call <4 x double> @llvm.masked.expandload.v4f64(ptr undef, <4 x i1> %m4, <4 x double> undef)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V2F64 = call <2 x double> @llvm.masked.expandload.v2f64(ptr undef, <2 x i1> %m2, <2 x double> undef)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V1F64 = call <1 x double> @llvm.masked.expandload.v1f64(ptr undef, <1 x i1> %m1, <1 x double> undef)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 67 for instruction: %V16F32 = call <16 x float> @llvm.masked.expandload.v16f32(ptr undef, <16 x i1> %m16, <16 x float> undef)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %V8F32 = call <8 x float> @llvm.masked.expandload.v8f32(ptr undef, <8 x i1> %m8, <8 x float> undef)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %V4F32 = call <4 x float> @llvm.masked.expandload.v4f32(ptr undef, <4 x i1> %m4, <4 x float> undef)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V2F32 = call <2 x float> @llvm.masked.expandload.v2f32(ptr undef, <2 x i1> %m2, <2 x float> undef)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 37 for instruction: %V8I64 = call <8 x i64> @llvm.masked.expandload.v8i64(ptr undef, <8 x i1> %m8, <8 x i64> undef)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 19 for instruction: %V4I64 = call <4 x i64> @llvm.masked.expandload.v4i64(ptr undef, <4 x i1> %m4, <4 x i64> undef)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V2I64 = call <2 x i64> @llvm.masked.expandload.v2i64(ptr undef, <2 x i1> %m2, <2 x i64> undef)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V1I64 = call <1 x i64> @llvm.masked.expandload.v1i64(ptr undef, <1 x i1> %m1, <1 x i64> undef)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 71 for instruction: %V16I32 = call <16 x i32> @llvm.masked.expandload.v16i32(ptr undef, <16 x i1> %m16, <16 x i32> undef)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 36 for instruction: %V8I32 = call <8 x i32> @llvm.masked.expandload.v8i32(ptr undef, <8 x i1> %m8, <8 x i32> undef)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V4I32 = call <4 x i32> @llvm.masked.expandload.v4i32(ptr undef, <4 x i1> %m4, <4 x i32> undef)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V2I32 = call <2 x i32> @llvm.masked.expandload.v2i32(ptr undef, <2 x i1> %m2, <2 x i32> undef)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 139 for instruction: %V32I16 = call <32 x i16> @llvm.masked.expandload.v32i16(ptr undef, <32 x i1> %m32, <32 x i16> undef)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 70 for instruction: %V16I16 = call <16 x i16> @llvm.masked.expandload.v16i16(ptr undef, <16 x i1> %m16, <16 x i16> undef)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 35 for instruction: %V8I16 = call <8 x i16> @llvm.masked.expandload.v8i16(ptr undef, <8 x i1> %m8, <8 x i16> undef)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V4I16 = call <4 x i16> @llvm.masked.expandload.v4i16(ptr undef, <4 x i1> %m4, <4 x i16> undef)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 276 for instruction: %V64I8 = call <64 x i8> @llvm.masked.expandload.v64i8(ptr undef, <64 x i1> %m64, <64 x i8> undef)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 138 for instruction: %V32I8 = call <32 x i8> @llvm.masked.expandload.v32i8(ptr undef, <32 x i1> %m32, <32 x i8> undef)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 69 for instruction: %V16I8 = call <16 x i8> @llvm.masked.expandload.v16i8(ptr undef, <16 x i1> %m16, <16 x i8> undef)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 35 for instruction: %V8I8 = call <8 x i8> @llvm.masked.expandload.v8i8(ptr undef, <8 x i1> %m8, <8 x i8> undef)
 ; SKL-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 0
 ;
 ; AVX512-LABEL: 'masked_expandload'
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 23 for instruction: %V8F64 = call <8 x double> @llvm.masked.expandload.v8f64(ptr undef, <8 x i1> %m8, <8 x double> undef)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %V4F64 = call <4 x double> @llvm.masked.expandload.v4f64(ptr undef, <4 x i1> %m4, <4 x double> undef)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V2F64 = call <2 x double> @llvm.masked.expandload.v2f64(ptr undef, <2 x i1> %m2, <2 x double> undef)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1F64 = call <1 x double> @llvm.masked.expandload.v1f64(ptr undef, <1 x i1> %m1, <1 x double> undef)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 47 for instruction: %V16F32 = call <16 x float> @llvm.masked.expandload.v16f32(ptr undef, <16 x i1> %m16, <16 x float> undef)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 23 for instruction: %V8F32 = call <8 x float> @llvm.masked.expandload.v8f32(ptr undef, <8 x i1> %m8, <8 x float> undef)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %V4F32 = call <4 x float> @llvm.masked.expandload.v4f32(ptr undef, <4 x i1> %m4, <4 x float> undef)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V2F32 = call <2 x float> @llvm.masked.expandload.v2f32(ptr undef, <2 x i1> %m2, <2 x float> undef)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 27 for instruction: %V8I64 = call <8 x i64> @llvm.masked.expandload.v8i64(ptr undef, <8 x i1> %m8, <8 x i64> undef)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %V4I64 = call <4 x i64> @llvm.masked.expandload.v4i64(ptr undef, <4 x i1> %m4, <4 x i64> undef)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2I64 = call <2 x i64> @llvm.masked.expandload.v2i64(ptr undef, <2 x i1> %m2, <2 x i64> undef)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1I64 = call <1 x i64> @llvm.masked.expandload.v1i64(ptr undef, <1 x i1> %m1, <1 x i64> undef)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 51 for instruction: %V16I32 = call <16 x i32> @llvm.masked.expandload.v16i32(ptr undef, <16 x i1> %m16, <16 x i32> undef)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: %V8I32 = call <8 x i32> @llvm.masked.expandload.v8i32(ptr undef, <8 x i1> %m8, <8 x i32> undef)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V4I32 = call <4 x i32> @llvm.masked.expandload.v4i32(ptr undef, <4 x i1> %m4, <4 x i32> undef)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2I32 = call <2 x i32> @llvm.masked.expandload.v2i32(ptr undef, <2 x i1> %m2, <2 x i32> undef)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 99 for instruction: %V32I16 = call <32 x i16> @llvm.masked.expandload.v32i16(ptr undef, <32 x i1> %m32, <32 x i16> undef)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 49 for instruction: %V16I16 = call <16 x i16> @llvm.masked.expandload.v16i16(ptr undef, <16 x i1> %m16, <16 x i16> undef)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V8I16 = call <8 x i16> @llvm.masked.expandload.v8i16(ptr undef, <8 x i1> %m8, <8 x i16> undef)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V4I16 = call <4 x i16> @llvm.masked.expandload.v4i16(ptr undef, <4 x i1> %m4, <4 x i16> undef)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 195 for instruction: %V64I8 = call <64 x i8> @llvm.masked.expandload.v64i8(ptr undef, <64 x i1> %m64, <64 x i8> undef)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 97 for instruction: %V32I8 = call <32 x i8> @llvm.masked.expandload.v32i8(ptr undef, <32 x i1> %m32, <32 x i8> undef)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V16I8 = call <16 x i8> @llvm.masked.expandload.v16i8(ptr undef, <16 x i1> %m16, <16 x i8> undef)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V8I8 = call <8 x i8> @llvm.masked.expandload.v8i8(ptr undef, <8 x i1> %m8, <8 x i8> undef)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 42 for instruction: %V8F64 = call <8 x double> @llvm.masked.expandload.v8f64(ptr undef, <8 x i1> %m8, <8 x double> undef)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %V4F64 = call <4 x double> @llvm.masked.expandload.v4f64(ptr undef, <4 x i1> %m4, <4 x double> undef)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V2F64 = call <2 x double> @llvm.masked.expandload.v2f64(ptr undef, <2 x i1> %m2, <2 x double> undef)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V1F64 = call <1 x double> @llvm.masked.expandload.v1f64(ptr undef, <1 x i1> %m1, <1 x double> undef)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 85 for instruction: %V16F32 = call <16 x float> @llvm.masked.expandload.v16f32(ptr undef, <16 x i1> %m16, <16 x float> undef)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 42 for instruction: %V8F32 = call <8 x float> @llvm.masked.expandload.v8f32(ptr undef, <8 x i1> %m8, <8 x float> undef)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %V4F32 = call <4 x float> @llvm.masked.expandload.v4f32(ptr undef, <4 x i1> %m4, <4 x float> undef)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V2F32 = call <2 x float> @llvm.masked.expandload.v2f32(ptr undef, <2 x i1> %m2, <2 x float> undef)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 46 for instruction: %V8I64 = call <8 x i64> @llvm.masked.expandload.v8i64(ptr undef, <8 x i1> %m8, <8 x i64> undef)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 22 for instruction: %V4I64 = call <4 x i64> @llvm.masked.expandload.v4i64(ptr undef, <4 x i1> %m4, <4 x i64> undef)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V2I64 = call <2 x i64> @llvm.masked.expandload.v2i64(ptr undef, <2 x i1> %m2, <2 x i64> undef)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V1I64 = call <1 x i64> @llvm.masked.expandload.v1i64(ptr undef, <1 x i1> %m1, <1 x i64> undef)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 89 for instruction: %V16I32 = call <16 x i32> @llvm.masked.expandload.v16i32(ptr undef, <16 x i1> %m16, <16 x i32> undef)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 44 for instruction: %V8I32 = call <8 x i32> @llvm.masked.expandload.v8i32(ptr undef, <8 x i1> %m8, <8 x i32> undef)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 21 for instruction: %V4I32 = call <4 x i32> @llvm.masked.expandload.v4i32(ptr undef, <4 x i1> %m4, <4 x i32> undef)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V2I32 = call <2 x i32> @llvm.masked.expandload.v2i32(ptr undef, <2 x i1> %m2, <2 x i32> undef)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 175 for instruction: %V32I16 = call <32 x i16> @llvm.masked.expandload.v32i16(ptr undef, <32 x i1> %m32, <32 x i16> undef)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 87 for instruction: %V16I16 = call <16 x i16> @llvm.masked.expandload.v16i16(ptr undef, <16 x i1> %m16, <16 x i16> undef)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 43 for instruction: %V8I16 = call <8 x i16> @llvm.masked.expandload.v8i16(ptr undef, <8 x i1> %m8, <8 x i16> undef)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 21 for instruction: %V4I16 = call <4 x i16> @llvm.masked.expandload.v4i16(ptr undef, <4 x i1> %m4, <4 x i16> undef)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 347 for instruction: %V64I8 = call <64 x i8> @llvm.masked.expandload.v64i8(ptr undef, <64 x i1> %m64, <64 x i8> undef)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 173 for instruction: %V32I8 = call <32 x i8> @llvm.masked.expandload.v32i8(ptr undef, <32 x i1> %m32, <32 x i8> undef)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 86 for instruction: %V16I8 = call <16 x i8> @llvm.masked.expandload.v16i8(ptr undef, <16 x i1> %m16, <16 x i8> undef)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 43 for instruction: %V8I8 = call <8 x i8> @llvm.masked.expandload.v8i8(ptr undef, <8 x i1> %m8, <8 x i8> undef)
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 0
 ;
   %V8F64 = call <8 x double> @llvm.masked.expandload.v8f64(ptr undef, <8 x i1> %m8, <8 x double> undef)
@@ -1374,165 +1374,165 @@ define i32 @masked_expandload(<1 x i1> %m1, <2 x i1> %m2, <4 x i1> %m4, <8 x i1>
 
 define i32 @masked_compressstore(<1 x i1> %m1, <2 x i1> %m2, <4 x i1> %m4, <8 x i1> %m8, <16 x i1> %m16, <32 x i1> %m32, <64 x i1> %m64) {
 ; SSE2-LABEL: 'masked_compressstore'
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: call void @llvm.masked.compressstore.v8f64(<8 x double> undef, ptr undef, <8 x i1> %m8)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: call void @llvm.masked.compressstore.v4f64(<4 x double> undef, ptr undef, <4 x i1> %m4)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: call void @llvm.masked.compressstore.v2f64(<2 x double> undef, ptr undef, <2 x i1> %m2)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.compressstore.v1f64(<1 x double> undef, ptr undef, <1 x i1> %m1)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 29 for instruction: call void @llvm.masked.compressstore.v16f32(<16 x float> undef, ptr undef, <16 x i1> %m16)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: call void @llvm.masked.compressstore.v8f32(<8 x float> undef, ptr undef, <8 x i1> %m8)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: call void @llvm.masked.compressstore.v4f32(<4 x float> undef, ptr undef, <4 x i1> %m4)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: call void @llvm.masked.compressstore.v2f32(<2 x float> undef, ptr undef, <2 x i1> %m2)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 21 for instruction: call void @llvm.masked.compressstore.v8i64(<8 x i64> undef, ptr undef, <8 x i1> %m8)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: call void @llvm.masked.compressstore.v4i64(<4 x i64> undef, ptr undef, <4 x i1> %m4)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: call void @llvm.masked.compressstore.v2i64(<2 x i64> undef, ptr undef, <2 x i1> %m2)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.compressstore.v1i64(<1 x i64> undef, ptr undef, <1 x i1> %m1)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 45 for instruction: call void @llvm.masked.compressstore.v16i32(<16 x i32> undef, ptr undef, <16 x i1> %m16)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 23 for instruction: call void @llvm.masked.compressstore.v8i32(<8 x i32> undef, ptr undef, <8 x i1> %m8)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: call void @llvm.masked.compressstore.v4i32(<4 x i32> undef, ptr undef, <4 x i1> %m4)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: call void @llvm.masked.compressstore.v2i32(<2 x i32> undef, ptr undef, <2 x i1> %m2)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 66 for instruction: call void @llvm.masked.compressstore.v32i16(<32 x i16> undef, ptr undef, <32 x i1> %m32)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: call void @llvm.masked.compressstore.v16i16(<16 x i16> undef, ptr undef, <16 x i1> %m16)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: call void @llvm.masked.compressstore.v8i16(<8 x i16> undef, ptr undef, <8 x i1> %m8)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: call void @llvm.masked.compressstore.v4i16(<4 x i16> undef, ptr undef, <4 x i1> %m4)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 192 for instruction: call void @llvm.masked.compressstore.v64i8(<64 x i8> undef, ptr undef, <64 x i1> %m64)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 96 for instruction: call void @llvm.masked.compressstore.v32i8(<32 x i8> undef, ptr undef, <32 x i1> %m32)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: call void @llvm.masked.compressstore.v16i8(<16 x i8> undef, ptr undef, <16 x i1> %m16)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: call void @llvm.masked.compressstore.v8i8(<8 x i8> undef, ptr undef, <8 x i1> %m8)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 37 for instruction: call void @llvm.masked.compressstore.v8f64(<8 x double> undef, ptr undef, <8 x i1> %m8)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 19 for instruction: call void @llvm.masked.compressstore.v4f64(<4 x double> undef, ptr undef, <4 x i1> %m4)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: call void @llvm.masked.compressstore.v2f64(<2 x double> undef, ptr undef, <2 x i1> %m2)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: call void @llvm.masked.compressstore.v1f64(<1 x double> undef, ptr undef, <1 x i1> %m1)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 77 for instruction: call void @llvm.masked.compressstore.v16f32(<16 x float> undef, ptr undef, <16 x i1> %m16)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 39 for instruction: call void @llvm.masked.compressstore.v8f32(<8 x float> undef, ptr undef, <8 x i1> %m8)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: call void @llvm.masked.compressstore.v4f32(<4 x float> undef, ptr undef, <4 x i1> %m4)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: call void @llvm.masked.compressstore.v2f32(<2 x float> undef, ptr undef, <2 x i1> %m2)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 45 for instruction: call void @llvm.masked.compressstore.v8i64(<8 x i64> undef, ptr undef, <8 x i1> %m8)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 23 for instruction: call void @llvm.masked.compressstore.v4i64(<4 x i64> undef, ptr undef, <4 x i1> %m4)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: call void @llvm.masked.compressstore.v2i64(<2 x i64> undef, ptr undef, <2 x i1> %m2)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: call void @llvm.masked.compressstore.v1i64(<1 x i64> undef, ptr undef, <1 x i1> %m1)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 93 for instruction: call void @llvm.masked.compressstore.v16i32(<16 x i32> undef, ptr undef, <16 x i1> %m16)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 47 for instruction: call void @llvm.masked.compressstore.v8i32(<8 x i32> undef, ptr undef, <8 x i1> %m8)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: call void @llvm.masked.compressstore.v4i32(<4 x i32> undef, ptr undef, <4 x i1> %m4)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: call void @llvm.masked.compressstore.v2i32(<2 x i32> undef, ptr undef, <2 x i1> %m2)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 162 for instruction: call void @llvm.masked.compressstore.v32i16(<32 x i16> undef, ptr undef, <32 x i1> %m32)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 81 for instruction: call void @llvm.masked.compressstore.v16i16(<16 x i16> undef, ptr undef, <16 x i1> %m16)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 41 for instruction: call void @llvm.masked.compressstore.v8i16(<8 x i16> undef, ptr undef, <8 x i1> %m8)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 21 for instruction: call void @llvm.masked.compressstore.v4i16(<4 x i16> undef, ptr undef, <4 x i1> %m4)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 384 for instruction: call void @llvm.masked.compressstore.v64i8(<64 x i8> undef, ptr undef, <64 x i1> %m64)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 192 for instruction: call void @llvm.masked.compressstore.v32i8(<32 x i8> undef, ptr undef, <32 x i1> %m32)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 96 for instruction: call void @llvm.masked.compressstore.v16i8(<16 x i8> undef, ptr undef, <16 x i1> %m16)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: call void @llvm.masked.compressstore.v8i8(<8 x i8> undef, ptr undef, <8 x i1> %m8)
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 0
 ;
 ; SSE42-LABEL: 'masked_compressstore'
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: call void @llvm.masked.compressstore.v8f64(<8 x double> undef, ptr undef, <8 x i1> %m8)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: call void @llvm.masked.compressstore.v4f64(<4 x double> undef, ptr undef, <4 x i1> %m4)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: call void @llvm.masked.compressstore.v2f64(<2 x double> undef, ptr undef, <2 x i1> %m2)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.compressstore.v1f64(<1 x double> undef, ptr undef, <1 x i1> %m1)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 29 for instruction: call void @llvm.masked.compressstore.v16f32(<16 x float> undef, ptr undef, <16 x i1> %m16)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: call void @llvm.masked.compressstore.v8f32(<8 x float> undef, ptr undef, <8 x i1> %m8)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: call void @llvm.masked.compressstore.v4f32(<4 x float> undef, ptr undef, <4 x i1> %m4)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: call void @llvm.masked.compressstore.v2f32(<2 x float> undef, ptr undef, <2 x i1> %m2)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: call void @llvm.masked.compressstore.v8i64(<8 x i64> undef, ptr undef, <8 x i1> %m8)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: call void @llvm.masked.compressstore.v4i64(<4 x i64> undef, ptr undef, <4 x i1> %m4)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: call void @llvm.masked.compressstore.v2i64(<2 x i64> undef, ptr undef, <2 x i1> %m2)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.compressstore.v1i64(<1 x i64> undef, ptr undef, <1 x i1> %m1)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: call void @llvm.masked.compressstore.v16i32(<16 x i32> undef, ptr undef, <16 x i1> %m16)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: call void @llvm.masked.compressstore.v8i32(<8 x i32> undef, ptr undef, <8 x i1> %m8)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: call void @llvm.masked.compressstore.v4i32(<4 x i32> undef, ptr undef, <4 x i1> %m4)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: call void @llvm.masked.compressstore.v2i32(<2 x i32> undef, ptr undef, <2 x i1> %m2)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 66 for instruction: call void @llvm.masked.compressstore.v32i16(<32 x i16> undef, ptr undef, <32 x i1> %m32)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: call void @llvm.masked.compressstore.v16i16(<16 x i16> undef, ptr undef, <16 x i1> %m16)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: call void @llvm.masked.compressstore.v8i16(<8 x i16> undef, ptr undef, <8 x i1> %m8)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: call void @llvm.masked.compressstore.v4i16(<4 x i16> undef, ptr undef, <4 x i1> %m4)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 132 for instruction: call void @llvm.masked.compressstore.v64i8(<64 x i8> undef, ptr undef, <64 x i1> %m64)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 66 for instruction: call void @llvm.masked.compressstore.v32i8(<32 x i8> undef, ptr undef, <32 x i1> %m32)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: call void @llvm.masked.compressstore.v16i8(<16 x i8> undef, ptr undef, <16 x i1> %m16)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: call void @llvm.masked.compressstore.v8i8(<8 x i8> undef, ptr undef, <8 x i1> %m8)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 29 for instruction: call void @llvm.masked.compressstore.v8f64(<8 x double> undef, ptr undef, <8 x i1> %m8)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: call void @llvm.masked.compressstore.v4f64(<4 x double> undef, ptr undef, <4 x i1> %m4)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: call void @llvm.masked.compressstore.v2f64(<2 x double> undef, ptr undef, <2 x i1> %m2)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: call void @llvm.masked.compressstore.v1f64(<1 x double> undef, ptr undef, <1 x i1> %m1)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 61 for instruction: call void @llvm.masked.compressstore.v16f32(<16 x float> undef, ptr undef, <16 x i1> %m16)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 31 for instruction: call void @llvm.masked.compressstore.v8f32(<8 x float> undef, ptr undef, <8 x i1> %m8)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: call void @llvm.masked.compressstore.v4f32(<4 x float> undef, ptr undef, <4 x i1> %m4)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: call void @llvm.masked.compressstore.v2f32(<2 x float> undef, ptr undef, <2 x i1> %m2)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: call void @llvm.masked.compressstore.v8i64(<8 x i64> undef, ptr undef, <8 x i1> %m8)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: call void @llvm.masked.compressstore.v4i64(<4 x i64> undef, ptr undef, <4 x i1> %m4)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: call void @llvm.masked.compressstore.v2i64(<2 x i64> undef, ptr undef, <2 x i1> %m2)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: call void @llvm.masked.compressstore.v1i64(<1 x i64> undef, ptr undef, <1 x i1> %m1)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 65 for instruction: call void @llvm.masked.compressstore.v16i32(<16 x i32> undef, ptr undef, <16 x i1> %m16)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: call void @llvm.masked.compressstore.v8i32(<8 x i32> undef, ptr undef, <8 x i1> %m8)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: call void @llvm.masked.compressstore.v4i32(<4 x i32> undef, ptr undef, <4 x i1> %m4)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: call void @llvm.masked.compressstore.v2i32(<2 x i32> undef, ptr undef, <2 x i1> %m2)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 130 for instruction: call void @llvm.masked.compressstore.v32i16(<32 x i16> undef, ptr undef, <32 x i1> %m32)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 65 for instruction: call void @llvm.masked.compressstore.v16i16(<16 x i16> undef, ptr undef, <16 x i1> %m16)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: call void @llvm.masked.compressstore.v8i16(<8 x i16> undef, ptr undef, <8 x i1> %m8)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: call void @llvm.masked.compressstore.v4i16(<4 x i16> undef, ptr undef, <4 x i1> %m4)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 260 for instruction: call void @llvm.masked.compressstore.v64i8(<64 x i8> undef, ptr undef, <64 x i1> %m64)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 130 for instruction: call void @llvm.masked.compressstore.v32i8(<32 x i8> undef, ptr undef, <32 x i1> %m32)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 65 for instruction: call void @llvm.masked.compressstore.v16i8(<16 x i8> undef, ptr undef, <16 x i1> %m16)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: call void @llvm.masked.compressstore.v8i8(<8 x i8> undef, ptr undef, <8 x i1> %m8)
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 0
 ;
 ; AVX1-LABEL: 'masked_compressstore'
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: call void @llvm.masked.compressstore.v8f64(<8 x double> undef, ptr undef, <8 x i1> %m8)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: call void @llvm.masked.compressstore.v4f64(<4 x double> undef, ptr undef, <4 x i1> %m4)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: call void @llvm.masked.compressstore.v2f64(<2 x double> undef, ptr undef, <2 x i1> %m2)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.compressstore.v1f64(<1 x double> undef, ptr undef, <1 x i1> %m1)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 31 for instruction: call void @llvm.masked.compressstore.v16f32(<16 x float> undef, ptr undef, <16 x i1> %m16)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: call void @llvm.masked.compressstore.v8f32(<8 x float> undef, ptr undef, <8 x i1> %m8)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: call void @llvm.masked.compressstore.v4f32(<4 x float> undef, ptr undef, <4 x i1> %m4)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: call void @llvm.masked.compressstore.v2f32(<2 x float> undef, ptr undef, <2 x i1> %m2)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 19 for instruction: call void @llvm.masked.compressstore.v8i64(<8 x i64> undef, ptr undef, <8 x i1> %m8)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: call void @llvm.masked.compressstore.v4i64(<4 x i64> undef, ptr undef, <4 x i1> %m4)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: call void @llvm.masked.compressstore.v2i64(<2 x i64> undef, ptr undef, <2 x i1> %m2)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.compressstore.v1i64(<1 x i64> undef, ptr undef, <1 x i1> %m1)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 35 for instruction: call void @llvm.masked.compressstore.v16i32(<16 x i32> undef, ptr undef, <16 x i1> %m16)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: call void @llvm.masked.compressstore.v8i32(<8 x i32> undef, ptr undef, <8 x i1> %m8)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: call void @llvm.masked.compressstore.v4i32(<4 x i32> undef, ptr undef, <4 x i1> %m4)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: call void @llvm.masked.compressstore.v2i32(<2 x i32> undef, ptr undef, <2 x i1> %m2)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 68 for instruction: call void @llvm.masked.compressstore.v32i16(<32 x i16> undef, ptr undef, <32 x i1> %m32)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: call void @llvm.masked.compressstore.v16i16(<16 x i16> undef, ptr undef, <16 x i1> %m16)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: call void @llvm.masked.compressstore.v8i16(<8 x i16> undef, ptr undef, <8 x i1> %m8)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: call void @llvm.masked.compressstore.v4i16(<4 x i16> undef, ptr undef, <4 x i1> %m4)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 134 for instruction: call void @llvm.masked.compressstore.v64i8(<64 x i8> undef, ptr undef, <64 x i1> %m64)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 67 for instruction: call void @llvm.masked.compressstore.v32i8(<32 x i8> undef, ptr undef, <32 x i1> %m32)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: call void @llvm.masked.compressstore.v16i8(<16 x i8> undef, ptr undef, <16 x i1> %m16)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: call void @llvm.masked.compressstore.v8i8(<8 x i8> undef, ptr undef, <8 x i1> %m8)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: call void @llvm.masked.compressstore.v8f64(<8 x double> undef, ptr undef, <8 x i1> %m8)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: call void @llvm.masked.compressstore.v4f64(<4 x double> undef, ptr undef, <4 x i1> %m4)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: call void @llvm.masked.compressstore.v2f64(<2 x double> undef, ptr undef, <2 x i1> %m2)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: call void @llvm.masked.compressstore.v1f64(<1 x double> undef, ptr undef, <1 x i1> %m1)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 67 for instruction: call void @llvm.masked.compressstore.v16f32(<16 x float> undef, ptr undef, <16 x i1> %m16)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: call void @llvm.masked.compressstore.v8f32(<8 x float> undef, ptr undef, <8 x i1> %m8)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: call void @llvm.masked.compressstore.v4f32(<4 x float> undef, ptr undef, <4 x i1> %m4)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: call void @llvm.masked.compressstore.v2f32(<2 x float> undef, ptr undef, <2 x i1> %m2)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 37 for instruction: call void @llvm.masked.compressstore.v8i64(<8 x i64> undef, ptr undef, <8 x i1> %m8)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 19 for instruction: call void @llvm.masked.compressstore.v4i64(<4 x i64> undef, ptr undef, <4 x i1> %m4)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: call void @llvm.masked.compressstore.v2i64(<2 x i64> undef, ptr undef, <2 x i1> %m2)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: call void @llvm.masked.compressstore.v1i64(<1 x i64> undef, ptr undef, <1 x i1> %m1)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 71 for instruction: call void @llvm.masked.compressstore.v16i32(<16 x i32> undef, ptr undef, <16 x i1> %m16)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 36 for instruction: call void @llvm.masked.compressstore.v8i32(<8 x i32> undef, ptr undef, <8 x i1> %m8)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: call void @llvm.masked.compressstore.v4i32(<4 x i32> undef, ptr undef, <4 x i1> %m4)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: call void @llvm.masked.compressstore.v2i32(<2 x i32> undef, ptr undef, <2 x i1> %m2)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 140 for instruction: call void @llvm.masked.compressstore.v32i16(<32 x i16> undef, ptr undef, <32 x i1> %m32)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 70 for instruction: call void @llvm.masked.compressstore.v16i16(<16 x i16> undef, ptr undef, <16 x i1> %m16)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 35 for instruction: call void @llvm.masked.compressstore.v8i16(<8 x i16> undef, ptr undef, <8 x i1> %m8)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: call void @llvm.masked.compressstore.v4i16(<4 x i16> undef, ptr undef, <4 x i1> %m4)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 278 for instruction: call void @llvm.masked.compressstore.v64i8(<64 x i8> undef, ptr undef, <64 x i1> %m64)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 139 for instruction: call void @llvm.masked.compressstore.v32i8(<32 x i8> undef, ptr undef, <32 x i1> %m32)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 69 for instruction: call void @llvm.masked.compressstore.v16i8(<16 x i8> undef, ptr undef, <16 x i1> %m16)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 35 for instruction: call void @llvm.masked.compressstore.v8i8(<8 x i8> undef, ptr undef, <8 x i1> %m8)
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 0
 ;
 ; AVX2-LABEL: 'masked_compressstore'
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: call void @llvm.masked.compressstore.v8f64(<8 x double> undef, ptr undef, <8 x i1> %m8)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: call void @llvm.masked.compressstore.v4f64(<4 x double> undef, ptr undef, <4 x i1> %m4)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: call void @llvm.masked.compressstore.v2f64(<2 x double> undef, ptr undef, <2 x i1> %m2)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.compressstore.v1f64(<1 x double> undef, ptr undef, <1 x i1> %m1)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 31 for instruction: call void @llvm.masked.compressstore.v16f32(<16 x float> undef, ptr undef, <16 x i1> %m16)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: call void @llvm.masked.compressstore.v8f32(<8 x float> undef, ptr undef, <8 x i1> %m8)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: call void @llvm.masked.compressstore.v4f32(<4 x float> undef, ptr undef, <4 x i1> %m4)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: call void @llvm.masked.compressstore.v2f32(<2 x float> undef, ptr undef, <2 x i1> %m2)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 19 for instruction: call void @llvm.masked.compressstore.v8i64(<8 x i64> undef, ptr undef, <8 x i1> %m8)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: call void @llvm.masked.compressstore.v4i64(<4 x i64> undef, ptr undef, <4 x i1> %m4)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: call void @llvm.masked.compressstore.v2i64(<2 x i64> undef, ptr undef, <2 x i1> %m2)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.compressstore.v1i64(<1 x i64> undef, ptr undef, <1 x i1> %m1)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 35 for instruction: call void @llvm.masked.compressstore.v16i32(<16 x i32> undef, ptr undef, <16 x i1> %m16)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: call void @llvm.masked.compressstore.v8i32(<8 x i32> undef, ptr undef, <8 x i1> %m8)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: call void @llvm.masked.compressstore.v4i32(<4 x i32> undef, ptr undef, <4 x i1> %m4)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: call void @llvm.masked.compressstore.v2i32(<2 x i32> undef, ptr undef, <2 x i1> %m2)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 67 for instruction: call void @llvm.masked.compressstore.v32i16(<32 x i16> undef, ptr undef, <32 x i1> %m32)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: call void @llvm.masked.compressstore.v16i16(<16 x i16> undef, ptr undef, <16 x i1> %m16)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: call void @llvm.masked.compressstore.v8i16(<8 x i16> undef, ptr undef, <8 x i1> %m8)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: call void @llvm.masked.compressstore.v4i16(<4 x i16> undef, ptr undef, <4 x i1> %m4)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 132 for instruction: call void @llvm.masked.compressstore.v64i8(<64 x i8> undef, ptr undef, <64 x i1> %m64)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 66 for instruction: call void @llvm.masked.compressstore.v32i8(<32 x i8> undef, ptr undef, <32 x i1> %m32)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: call void @llvm.masked.compressstore.v16i8(<16 x i8> undef, ptr undef, <16 x i1> %m16)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: call void @llvm.masked.compressstore.v8i8(<8 x i8> undef, ptr undef, <8 x i1> %m8)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: call void @llvm.masked.compressstore.v8f64(<8 x double> undef, ptr undef, <8 x i1> %m8)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: call void @llvm.masked.compressstore.v4f64(<4 x double> undef, ptr undef, <4 x i1> %m4)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: call void @llvm.masked.compressstore.v2f64(<2 x double> undef, ptr undef, <2 x i1> %m2)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: call void @llvm.masked.compressstore.v1f64(<1 x double> undef, ptr undef, <1 x i1> %m1)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 67 for instruction: call void @llvm.masked.compressstore.v16f32(<16 x float> undef, ptr undef, <16 x i1> %m16)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: call void @llvm.masked.compressstore.v8f32(<8 x float> undef, ptr undef, <8 x i1> %m8)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: call void @llvm.masked.compressstore.v4f32(<4 x float> undef, ptr undef, <4 x i1> %m4)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: call void @llvm.masked.compressstore.v2f32(<2 x float> undef, ptr undef, <2 x i1> %m2)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 37 for instruction: call void @llvm.masked.compressstore.v8i64(<8 x i64> undef, ptr undef, <8 x i1> %m8)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 19 for instruction: call void @llvm.masked.compressstore.v4i64(<4 x i64> undef, ptr undef, <4 x i1> %m4)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: call void @llvm.masked.compressstore.v2i64(<2 x i64> undef, ptr undef, <2 x i1> %m2)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: call void @llvm.masked.compressstore.v1i64(<1 x i64> undef, ptr undef, <1 x i1> %m1)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 71 for instruction: call void @llvm.masked.compressstore.v16i32(<16 x i32> undef, ptr undef, <16 x i1> %m16)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 36 for instruction: call void @llvm.masked.compressstore.v8i32(<8 x i32> undef, ptr undef, <8 x i1> %m8)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: call void @llvm.masked.compressstore.v4i32(<4 x i32> undef, ptr undef, <4 x i1> %m4)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: call void @llvm.masked.compressstore.v2i32(<2 x i32> undef, ptr undef, <2 x i1> %m2)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 139 for instruction: call void @llvm.masked.compressstore.v32i16(<32 x i16> undef, ptr undef, <32 x i1> %m32)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 70 for instruction: call void @llvm.masked.compressstore.v16i16(<16 x i16> undef, ptr undef, <16 x i1> %m16)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 35 for instruction: call void @llvm.masked.compressstore.v8i16(<8 x i16> undef, ptr undef, <8 x i1> %m8)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: call void @llvm.masked.compressstore.v4i16(<4 x i16> undef, ptr undef, <4 x i1> %m4)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 276 for instruction: call void @llvm.masked.compressstore.v64i8(<64 x i8> undef, ptr undef, <64 x i1> %m64)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 138 for instruction: call void @llvm.masked.compressstore.v32i8(<32 x i8> undef, ptr undef, <32 x i1> %m32)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 69 for instruction: call void @llvm.masked.compressstore.v16i8(<16 x i8> undef, ptr undef, <16 x i1> %m16)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 35 for instruction: call void @llvm.masked.compressstore.v8i8(<8 x i8> undef, ptr undef, <8 x i1> %m8)
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 0
 ;
 ; SKL-LABEL: 'masked_compressstore'
-; SKL-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: call void @llvm.masked.compressstore.v8f64(<8 x double> undef, ptr undef, <8 x i1> %m8)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: call void @llvm.masked.compressstore.v4f64(<4 x double> undef, ptr undef, <4 x i1> %m4)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: call void @llvm.masked.compressstore.v2f64(<2 x double> undef, ptr undef, <2 x i1> %m2)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.compressstore.v1f64(<1 x double> undef, ptr undef, <1 x i1> %m1)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 31 for instruction: call void @llvm.masked.compressstore.v16f32(<16 x float> undef, ptr undef, <16 x i1> %m16)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: call void @llvm.masked.compressstore.v8f32(<8 x float> undef, ptr undef, <8 x i1> %m8)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: call void @llvm.masked.compressstore.v4f32(<4 x float> undef, ptr undef, <4 x i1> %m4)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: call void @llvm.masked.compressstore.v2f32(<2 x float> undef, ptr undef, <2 x i1> %m2)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 19 for instruction: call void @llvm.masked.compressstore.v8i64(<8 x i64> undef, ptr undef, <8 x i1> %m8)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: call void @llvm.masked.compressstore.v4i64(<4 x i64> undef, ptr undef, <4 x i1> %m4)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: call void @llvm.masked.compressstore.v2i64(<2 x i64> undef, ptr undef, <2 x i1> %m2)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.compressstore.v1i64(<1 x i64> undef, ptr undef, <1 x i1> %m1)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 35 for instruction: call void @llvm.masked.compressstore.v16i32(<16 x i32> undef, ptr undef, <16 x i1> %m16)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: call void @llvm.masked.compressstore.v8i32(<8 x i32> undef, ptr undef, <8 x i1> %m8)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: call void @llvm.masked.compressstore.v4i32(<4 x i32> undef, ptr undef, <4 x i1> %m4)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: call void @llvm.masked.compressstore.v2i32(<2 x i32> undef, ptr undef, <2 x i1> %m2)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 67 for instruction: call void @llvm.masked.compressstore.v32i16(<32 x i16> undef, ptr undef, <32 x i1> %m32)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: call void @llvm.masked.compressstore.v16i16(<16 x i16> undef, ptr undef, <16 x i1> %m16)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: call void @llvm.masked.compressstore.v8i16(<8 x i16> undef, ptr undef, <8 x i1> %m8)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: call void @llvm.masked.compressstore.v4i16(<4 x i16> undef, ptr undef, <4 x i1> %m4)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 132 for instruction: call void @llvm.masked.compressstore.v64i8(<64 x i8> undef, ptr undef, <64 x i1> %m64)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 66 for instruction: call void @llvm.masked.compressstore.v32i8(<32 x i8> undef, ptr undef, <32 x i1> %m32)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: call void @llvm.masked.compressstore.v16i8(<16 x i8> undef, ptr undef, <16 x i1> %m16)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: call void @llvm.masked.compressstore.v8i8(<8 x i8> undef, ptr undef, <8 x i1> %m8)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: call void @llvm.masked.compressstore.v8f64(<8 x double> undef, ptr undef, <8 x i1> %m8)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: call void @llvm.masked.compressstore.v4f64(<4 x double> undef, ptr undef, <4 x i1> %m4)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: call void @llvm.masked.compressstore.v2f64(<2 x double> undef, ptr undef, <2 x i1> %m2)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: call void @llvm.masked.compressstore.v1f64(<1 x double> undef, ptr undef, <1 x i1> %m1)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 67 for instruction: call void @llvm.masked.compressstore.v16f32(<16 x float> undef, ptr undef, <16 x i1> %m16)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: call void @llvm.masked.compressstore.v8f32(<8 x float> undef, ptr undef, <8 x i1> %m8)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: call void @llvm.masked.compressstore.v4f32(<4 x float> undef, ptr undef, <4 x i1> %m4)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: call void @llvm.masked.compressstore.v2f32(<2 x float> undef, ptr undef, <2 x i1> %m2)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 37 for instruction: call void @llvm.masked.compressstore.v8i64(<8 x i64> undef, ptr undef, <8 x i1> %m8)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 19 for instruction: call void @llvm.masked.compressstore.v4i64(<4 x i64> undef, ptr undef, <4 x i1> %m4)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: call void @llvm.masked.compressstore.v2i64(<2 x i64> undef, ptr undef, <2 x i1> %m2)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: call void @llvm.masked.compressstore.v1i64(<1 x i64> undef, ptr undef, <1 x i1> %m1)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 71 for instruction: call void @llvm.masked.compressstore.v16i32(<16 x i32> undef, ptr undef, <16 x i1> %m16)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 36 for instruction: call void @llvm.masked.compressstore.v8i32(<8 x i32> undef, ptr undef, <8 x i1> %m8)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: call void @llvm.masked.compressstore.v4i32(<4 x i32> undef, ptr undef, <4 x i1> %m4)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: call void @llvm.masked.compressstore.v2i32(<2 x i32> undef, ptr undef, <2 x i1> %m2)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 139 for instruction: call void @llvm.masked.compressstore.v32i16(<32 x i16> undef, ptr undef, <32 x i1> %m32)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 70 for instruction: call void @llvm.masked.compressstore.v16i16(<16 x i16> undef, ptr undef, <16 x i1> %m16)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 35 for instruction: call void @llvm.masked.compressstore.v8i16(<8 x i16> undef, ptr undef, <8 x i1> %m8)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: call void @llvm.masked.compressstore.v4i16(<4 x i16> undef, ptr undef, <4 x i1> %m4)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 276 for instruction: call void @llvm.masked.compressstore.v64i8(<64 x i8> undef, ptr undef, <64 x i1> %m64)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 138 for instruction: call void @llvm.masked.compressstore.v32i8(<32 x i8> undef, ptr undef, <32 x i1> %m32)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 69 for instruction: call void @llvm.masked.compressstore.v16i8(<16 x i8> undef, ptr undef, <16 x i1> %m16)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 35 for instruction: call void @llvm.masked.compressstore.v8i8(<8 x i8> undef, ptr undef, <8 x i1> %m8)
 ; SKL-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 0
 ;
 ; AVX512-LABEL: 'masked_compressstore'
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 23 for instruction: call void @llvm.masked.compressstore.v8f64(<8 x double> undef, ptr undef, <8 x i1> %m8)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: call void @llvm.masked.compressstore.v4f64(<4 x double> undef, ptr undef, <4 x i1> %m4)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: call void @llvm.masked.compressstore.v2f64(<2 x double> undef, ptr undef, <2 x i1> %m2)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.compressstore.v1f64(<1 x double> undef, ptr undef, <1 x i1> %m1)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 47 for instruction: call void @llvm.masked.compressstore.v16f32(<16 x float> undef, ptr undef, <16 x i1> %m16)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 23 for instruction: call void @llvm.masked.compressstore.v8f32(<8 x float> undef, ptr undef, <8 x i1> %m8)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: call void @llvm.masked.compressstore.v4f32(<4 x float> undef, ptr undef, <4 x i1> %m4)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: call void @llvm.masked.compressstore.v2f32(<2 x float> undef, ptr undef, <2 x i1> %m2)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 27 for instruction: call void @llvm.masked.compressstore.v8i64(<8 x i64> undef, ptr undef, <8 x i1> %m8)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: call void @llvm.masked.compressstore.v4i64(<4 x i64> undef, ptr undef, <4 x i1> %m4)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: call void @llvm.masked.compressstore.v2i64(<2 x i64> undef, ptr undef, <2 x i1> %m2)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.compressstore.v1i64(<1 x i64> undef, ptr undef, <1 x i1> %m1)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 51 for instruction: call void @llvm.masked.compressstore.v16i32(<16 x i32> undef, ptr undef, <16 x i1> %m16)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: call void @llvm.masked.compressstore.v8i32(<8 x i32> undef, ptr undef, <8 x i1> %m8)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: call void @llvm.masked.compressstore.v4i32(<4 x i32> undef, ptr undef, <4 x i1> %m4)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: call void @llvm.masked.compressstore.v2i32(<2 x i32> undef, ptr undef, <2 x i1> %m2)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 99 for instruction: call void @llvm.masked.compressstore.v32i16(<32 x i16> undef, ptr undef, <32 x i1> %m32)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 49 for instruction: call void @llvm.masked.compressstore.v16i16(<16 x i16> undef, ptr undef, <16 x i1> %m16)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: call void @llvm.masked.compressstore.v8i16(<8 x i16> undef, ptr undef, <8 x i1> %m8)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: call void @llvm.masked.compressstore.v4i16(<4 x i16> undef, ptr undef, <4 x i1> %m4)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 195 for instruction: call void @llvm.masked.compressstore.v64i8(<64 x i8> undef, ptr undef, <64 x i1> %m64)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 97 for instruction: call void @llvm.masked.compressstore.v32i8(<32 x i8> undef, ptr undef, <32 x i1> %m32)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: call void @llvm.masked.compressstore.v16i8(<16 x i8> undef, ptr undef, <16 x i1> %m16)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: call void @llvm.masked.compressstore.v8i8(<8 x i8> undef, ptr undef, <8 x i1> %m8)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 42 for instruction: call void @llvm.masked.compressstore.v8f64(<8 x double> undef, ptr undef, <8 x i1> %m8)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: call void @llvm.masked.compressstore.v4f64(<4 x double> undef, ptr undef, <4 x i1> %m4)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: call void @llvm.masked.compressstore.v2f64(<2 x double> undef, ptr undef, <2 x i1> %m2)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: call void @llvm.masked.compressstore.v1f64(<1 x double> undef, ptr undef, <1 x i1> %m1)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 85 for instruction: call void @llvm.masked.compressstore.v16f32(<16 x float> undef, ptr undef, <16 x i1> %m16)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 42 for instruction: call void @llvm.masked.compressstore.v8f32(<8 x float> undef, ptr undef, <8 x i1> %m8)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: call void @llvm.masked.compressstore.v4f32(<4 x float> undef, ptr undef, <4 x i1> %m4)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: call void @llvm.masked.compressstore.v2f32(<2 x float> undef, ptr undef, <2 x i1> %m2)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 46 for instruction: call void @llvm.masked.compressstore.v8i64(<8 x i64> undef, ptr undef, <8 x i1> %m8)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 22 for instruction: call void @llvm.masked.compressstore.v4i64(<4 x i64> undef, ptr undef, <4 x i1> %m4)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: call void @llvm.masked.compressstore.v2i64(<2 x i64> undef, ptr undef, <2 x i1> %m2)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: call void @llvm.masked.compressstore.v1i64(<1 x i64> undef, ptr undef, <1 x i1> %m1)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 89 for instruction: call void @llvm.masked.compressstore.v16i32(<16 x i32> undef, ptr undef, <16 x i1> %m16)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 44 for instruction: call void @llvm.masked.compressstore.v8i32(<8 x i32> undef, ptr undef, <8 x i1> %m8)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 21 for instruction: call void @llvm.masked.compressstore.v4i32(<4 x i32> undef, ptr undef, <4 x i1> %m4)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: call void @llvm.masked.compressstore.v2i32(<2 x i32> undef, ptr undef, <2 x i1> %m2)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 175 for instruction: call void @llvm.masked.compressstore.v32i16(<32 x i16> undef, ptr undef, <32 x i1> %m32)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 87 for instruction: call void @llvm.masked.compressstore.v16i16(<16 x i16> undef, ptr undef, <16 x i1> %m16)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 43 for instruction: call void @llvm.masked.compressstore.v8i16(<8 x i16> undef, ptr undef, <8 x i1> %m8)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 21 for instruction: call void @llvm.masked.compressstore.v4i16(<4 x i16> undef, ptr undef, <4 x i1> %m4)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 347 for instruction: call void @llvm.masked.compressstore.v64i8(<64 x i8> undef, ptr undef, <64 x i1> %m64)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 173 for instruction: call void @llvm.masked.compressstore.v32i8(<32 x i8> undef, ptr undef, <32 x i1> %m32)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 86 for instruction: call void @llvm.masked.compressstore.v16i8(<16 x i8> undef, ptr undef, <16 x i1> %m16)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 43 for instruction: call void @llvm.masked.compressstore.v8i8(<8 x i8> undef, ptr undef, <8 x i1> %m8)
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 0
 ;
   call void @llvm.masked.compressstore.v8f64(<8 x double> undef, ptr undef, <8 x i1> %m8)
diff --git a/llvm/test/Analysis/CostModel/X86/reduce-xor.ll b/llvm/test/Analysis/CostModel/X86/reduce-xor.ll
index f1c8bcda65bd1..757db4a5a41be 100644
--- a/llvm/test/Analysis/CostModel/X86/reduce-xor.ll
+++ b/llvm/test/Analysis/CostModel/X86/reduce-xor.ll
@@ -157,11 +157,11 @@ define i32 @reduce_i1(i32 %arg) {
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V1 = call i1 @llvm.vector.reduce.xor.v1i1(<1 x i1> undef)
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2 = call i1 @llvm.vector.reduce.xor.v2i1(<2 x i1> undef)
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V4 = call i1 @llvm.vector.reduce.xor.v4i1(<4 x i1> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 19 for instruction: %V8 = call i1 @llvm.vector.reduce.xor.v8i1(<8 x i1> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 45 for instruction: %V16 = call i1 @llvm.vector.reduce.xor.v16i1(<16 x i1> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 46 for instruction: %V32 = call i1 @llvm.vector.reduce.xor.v32i1(<32 x i1> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V64 = call i1 @llvm.vector.reduce.xor.v64i1(<64 x i1> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 52 for instruction: %V128 = call i1 @llvm.vector.reduce.xor.v128i1(<128 x i1> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %V8 = call i1 @llvm.vector.reduce.xor.v8i1(<8 x i1> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 37 for instruction: %V16 = call i1 @llvm.vector.reduce.xor.v16i1(<16 x i1> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 38 for instruction: %V32 = call i1 @llvm.vector.reduce.xor.v32i1(<32 x i1> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %V64 = call i1 @llvm.vector.reduce.xor.v64i1(<64 x i1> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 44 for instruction: %V128 = call i1 @llvm.vector.reduce.xor.v128i1(<128 x i1> undef)
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; SSSE3-LABEL: 'reduce_i1'
diff --git a/llvm/test/Analysis/CostModel/X86/reduction.ll b/llvm/test/Analysis/CostModel/X86/reduction.ll
index d7cf8e6cb8905..4ad0887a27884 100644
--- a/llvm/test/Analysis/CostModel/X86/reduction.ll
+++ b/llvm/test/Analysis/CostModel/X86/reduction.ll
@@ -634,11 +634,11 @@ define fastcc i64 @no_pairwise_reduction4i64(<4 x i64> %rdx, i64 %f1) {
 
 define fastcc i16 @no_pairwise_reduction8i16(<8 x i16> %rdx, i16 %f1) {
 ; SSE2-LABEL: 'no_pairwise_reduction8i16'
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %rdx.shuf3 = shufflevector <8 x i16> %rdx, <8 x i16> undef, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %rdx.shuf3 = shufflevector <8 x i16> %rdx, <8 x i16> undef, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %bin.rdx4 = add <8 x i16> %rdx, %rdx.shuf3
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %rdx.shuf = shufflevector <8 x i16> %bin.rdx4, <8 x i16> undef, <8 x i32> <i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %rdx.shuf = shufflevector <8 x i16> %bin.rdx4, <8 x i16> undef, <8 x i32> <i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %bin.rdx = add <8 x i16> %bin.rdx4, %rdx.shuf
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %rdx.shuf7 = shufflevector <8 x i16> %bin.rdx, <8 x i16> undef, <8 x i32> <i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf7 = shufflevector <8 x i16> %bin.rdx, <8 x i16> undef, <8 x i32> <i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %bin.rdx8 = add <8 x i16> %bin.rdx, %rdx.shuf7
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r = extractelement <8 x i16> %bin.rdx8, i32 0
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i16 %r
@@ -1126,14 +1126,14 @@ define fastcc i64 @pairwise_reduction4i64(<4 x i64> %rdx, i64 %f1) {
 
 define fastcc i16 @pairwise_reduction8i16(<8 x i16> %rdx, i16 %f1) {
 ; SSE2-LABEL: 'pairwise_reduction8i16'
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %rdx.shuf.0.0 = shufflevector <8 x i16> %rdx, <8 x i16> undef, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 poison, i32 poison, i32 poison, i32 poison>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %rdx.shuf.0.1 = shufflevector <8 x i16> %rdx, <8 x i16> undef, <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %rdx.shuf.0.0 = shufflevector <8 x i16> %rdx, <8 x i16> undef, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %rdx.shuf.0.1 = shufflevector <8 x i16> %rdx, <8 x i16> undef, <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 poison, i32 poison, i32 poison, i32 poison>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %bin.rdx = add <8 x i16> %rdx.shuf.0.0, %rdx.shuf.0.1
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %rdx.shuf.1.0 = shufflevector <8 x i16> %bin.rdx, <8 x i16> undef, <8 x i32> <i32 0, i32 2, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %rdx.shuf.1.1 = shufflevector <8 x i16> %bin.rdx, <8 x i16> undef, <8 x i32> <i32 1, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %rdx.shuf.1.0 = shufflevector <8 x i16> %bin.rdx, <8 x i16> undef, <8 x i32> <i32 0, i32 2, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %rdx.shuf.1.1 = shufflevector <8 x i16> %bin.rdx, <8 x i16> undef, <8 x i32> <i32 1, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %bin.rdx8 = add <8 x i16> %rdx.shuf.1.0, %rdx.shuf.1.1
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %rdx.shuf.2.0 = shufflevector <8 x i16> %bin.rdx8, <8 x i16> undef, <8 x i32> <i32 0, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %rdx.shuf.2.1 = shufflevector <8 x i16> %bin.rdx8, <8 x i16> undef, <8 x i32> <i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.2.1 = shufflevector <8 x i16> %bin.rdx8, <8 x i16> undef, <8 x i32> <i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %bin.rdx9 = add <8 x i16> %rdx.shuf.2.0, %rdx.shuf.2.1
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r = extractelement <8 x i16> %bin.rdx9, i32 0
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i16 %r
diff --git a/llvm/test/Analysis/CostModel/X86/shuffle-extract_subvector.ll b/llvm/test/Analysis/CostModel/X86/shuffle-extract_subvector.ll
index e91db1b0cb14a..f0536f96941ac 100644
--- a/llvm/test/Analysis/CostModel/X86/shuffle-extract_subvector.ll
+++ b/llvm/test/Analysis/CostModel/X86/shuffle-extract_subvector.ll
@@ -251,7 +251,7 @@ define void @test_vXi16(<4 x i16> %src64, <8 x i16> %src128, <16 x i16> %src256,
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_CD = shufflevector <16 x i16> %src256, <16 x i16> undef, <2 x i32> <i32 12, i32 13>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_EF = shufflevector <16 x i16> %src256, <16 x i16> undef, <2 x i32> <i32 14, i32 15>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V256_0123 = shufflevector <16 x i16> %src256, <16 x i16> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V256_2345 = shufflevector <16 x i16> %src256, <16 x i16> undef, <4 x i32> <i32 2, i32 3, i32 4, i32 5>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V256_2345 = shufflevector <16 x i16> %src256, <16 x i16> undef, <4 x i32> <i32 2, i32 3, i32 4, i32 5>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_4567 = shufflevector <16 x i16> %src256, <16 x i16> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V256_6789 = shufflevector <16 x i16> %src256, <16 x i16> undef, <4 x i32> <i32 6, i32 7, i32 8, i32 9>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V256_89AB = shufflevector <16 x i16> %src256, <16 x i16> undef, <4 x i32> <i32 8, i32 9, i32 10, i32 11>
@@ -275,7 +275,7 @@ define void @test_vXi16(<4 x i16> %src64, <8 x i16> %src128, <16 x i16> %src256,
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_1C_1D = shufflevector <32 x i16> %src512, <32 x i16> undef, <2 x i32> <i32 28, i32 29>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_1E_1F = shufflevector <32 x i16> %src512, <32 x i16> undef, <2 x i32> <i32 30, i32 31>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_00_01_02_03 = shufflevector <32 x i16> %src512, <32 x i16> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V512_02_03_04_05 = shufflevector <32 x i16> %src512, <32 x i16> undef, <4 x i32> <i32 2, i32 3, i32 4, i32 5>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V512_02_03_04_05 = shufflevector <32 x i16> %src512, <32 x i16> undef, <4 x i32> <i32 2, i32 3, i32 4, i32 5>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_04_05_06_07 = shufflevector <32 x i16> %src512, <32 x i16> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V512_06_07_08_09 = shufflevector <32 x i16> %src512, <32 x i16> undef, <4 x i32> <i32 6, i32 7, i32 8, i32 9>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_08_09_0A_0B = shufflevector <32 x i16> %src512, <32 x i16> undef, <4 x i32> <i32 8, i32 9, i32 10, i32 11>
@@ -899,9 +899,9 @@ define void @test_vXi8(<8 x i8> %src64, <16 x i8> %src128, <32 x i8> %src256, <6
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V128_CD = shufflevector <16 x i8> %src128, <16 x i8> undef, <2 x i32> <i32 12, i32 13>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V128_EF = shufflevector <16 x i8> %src128, <16 x i8> undef, <2 x i32> <i32 14, i32 15>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V128_0123 = shufflevector <16 x i8> %src128, <16 x i8> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V128_2345 = shufflevector <16 x i8> %src128, <16 x i8> undef, <4 x i32> <i32 2, i32 3, i32 4, i32 5>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V128_2345 = shufflevector <16 x i8> %src128, <16 x i8> undef, <4 x i32> <i32 2, i32 3, i32 4, i32 5>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128_4567 = shufflevector <16 x i8> %src128, <16 x i8> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V128_6789 = shufflevector <16 x i8> %src128, <16 x i8> undef, <4 x i32> <i32 6, i32 7, i32 8, i32 9>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V128_6789 = shufflevector <16 x i8> %src128, <16 x i8> undef, <4 x i32> <i32 6, i32 7, i32 8, i32 9>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128_89AB = shufflevector <16 x i8> %src128, <16 x i8> undef, <4 x i32> <i32 8, i32 9, i32 10, i32 11>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128_CDEF = shufflevector <16 x i8> %src128, <16 x i8> undef, <4 x i32> <i32 12, i32 13, i32 14, i32 15>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V128_01234567 = shufflevector <16 x i8> %src128, <16 x i8> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
@@ -923,9 +923,9 @@ define void @test_vXi8(<8 x i8> %src64, <16 x i8> %src128, <32 x i8> %src256, <6
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256_1C_1D = shufflevector <32 x i8> %src256, <32 x i8> undef, <2 x i32> <i32 28, i32 29>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256_1E_1F = shufflevector <32 x i8> %src256, <32 x i8> undef, <2 x i32> <i32 30, i32 31>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V256_00_01_02_03 = shufflevector <32 x i8> %src256, <32 x i8> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V256_02_03_04_05 = shufflevector <32 x i8> %src256, <32 x i8> undef, <4 x i32> <i32 2, i32 3, i32 4, i32 5>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V256_02_03_04_05 = shufflevector <32 x i8> %src256, <32 x i8> undef, <4 x i32> <i32 2, i32 3, i32 4, i32 5>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_04_05_06_07 = shufflevector <32 x i8> %src256, <32 x i8> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V256_06_07_08_09 = shufflevector <32 x i8> %src256, <32 x i8> undef, <4 x i32> <i32 6, i32 7, i32 8, i32 9>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V256_06_07_08_09 = shufflevector <32 x i8> %src256, <32 x i8> undef, <4 x i32> <i32 6, i32 7, i32 8, i32 9>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_08_09_0A_0B = shufflevector <32 x i8> %src256, <32 x i8> undef, <4 x i32> <i32 8, i32 9, i32 10, i32 11>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_0C_0D_0E_0F = shufflevector <32 x i8> %src256, <32 x i8> undef, <4 x i32> <i32 12, i32 13, i32 14, i32 15>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V256_10_11_12_13 = shufflevector <32 x i8> %src256, <32 x i8> undef, <4 x i32> <i32 16, i32 17, i32 18, i32 19>
diff --git a/llvm/test/Analysis/CostModel/X86/shuffle-insert_subvector.ll b/llvm/test/Analysis/CostModel/X86/shuffle-insert_subvector.ll
index bc3af51914499..a9443cfe0f97b 100644
--- a/llvm/test/Analysis/CostModel/X86/shuffle-insert_subvector.ll
+++ b/llvm/test/Analysis/CostModel/X86/shuffle-insert_subvector.ll
@@ -527,23 +527,23 @@ define void @test_vXi16(<2 x i16> %src32, <4 x i16> %src64, <8 x i16> %src128, <
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V64_01 = shufflevector <4 x i16> %src64, <4 x i16> %src32_64, <4 x i32> <i32 4, i32 5, i32 2, i32 3>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V64_23 = shufflevector <4 x i16> %src64, <4 x i16> %src32_64, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V128_01 = shufflevector <8 x i16> %src128, <8 x i16> %src32_128, <8 x i32> <i32 8, i32 9, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V128_23 = shufflevector <8 x i16> %src128, <8 x i16> %src32_128, <8 x i32> <i32 0, i32 1, i32 8, i32 9, i32 4, i32 5, i32 6, i32 7>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V128_45 = shufflevector <8 x i16> %src128, <8 x i16> %src32_128, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 6, i32 7>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V128_67 = shufflevector <8 x i16> %src128, <8 x i16> %src32_128, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 8, i32 9>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V128_23 = shufflevector <8 x i16> %src128, <8 x i16> %src32_128, <8 x i32> <i32 0, i32 1, i32 8, i32 9, i32 4, i32 5, i32 6, i32 7>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V128_45 = shufflevector <8 x i16> %src128, <8 x i16> %src32_128, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 6, i32 7>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V128_67 = shufflevector <8 x i16> %src128, <8 x i16> %src32_128, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 8, i32 9>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V128_0123 = shufflevector <8 x i16> %src128, <8 x i16> %src32_128, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 4, i32 5, i32 6, i32 7>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V128_4567 = shufflevector <8 x i16> %src128, <8 x i16> %src32_128, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V128_4567 = shufflevector <8 x i16> %src128, <8 x i16> %src32_128, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_01 = shufflevector <16 x i16> %src256, <16 x i16> %src64_256, <16 x i32> <i32 16, i32 17, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V256_23 = shufflevector <16 x i16> %src256, <16 x i16> %src64_256, <16 x i32> <i32 0, i32 1, i32 16, i32 17, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V256_45 = shufflevector <16 x i16> %src256, <16 x i16> %src64_256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 16, i32 17, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V256_67 = shufflevector <16 x i16> %src256, <16 x i16> %src64_256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 16, i32 17, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V256_23 = shufflevector <16 x i16> %src256, <16 x i16> %src64_256, <16 x i32> <i32 0, i32 1, i32 16, i32 17, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V256_45 = shufflevector <16 x i16> %src256, <16 x i16> %src64_256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 16, i32 17, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V256_67 = shufflevector <16 x i16> %src256, <16 x i16> %src64_256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 16, i32 17, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_89 = shufflevector <16 x i16> %src256, <16 x i16> %src64_256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V256_AB = shufflevector <16 x i16> %src256, <16 x i16> %src64_256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 16, i32 17, i32 12, i32 13, i32 14, i32 15>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V256_CD = shufflevector <16 x i16> %src256, <16 x i16> %src64_256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 16, i32 17, i32 14, i32 15>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V256_EF = shufflevector <16 x i16> %src256, <16 x i16> %src64_256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 16, i32 17>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V256_AB = shufflevector <16 x i16> %src256, <16 x i16> %src64_256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 16, i32 17, i32 12, i32 13, i32 14, i32 15>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V256_CD = shufflevector <16 x i16> %src256, <16 x i16> %src64_256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 16, i32 17, i32 14, i32 15>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V256_EF = shufflevector <16 x i16> %src256, <16 x i16> %src64_256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 16, i32 17>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_0123 = shufflevector <16 x i16> %src256, <16 x i16> %src128_256, <16 x i32> <i32 16, i32 17, i32 18, i32 19, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V256_4567 = shufflevector <16 x i16> %src256, <16 x i16> %src128_256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 16, i32 17, i32 18, i32 19, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V256_4567 = shufflevector <16 x i16> %src256, <16 x i16> %src128_256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 16, i32 17, i32 18, i32 19, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_89AB = shufflevector <16 x i16> %src256, <16 x i16> %src128_256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 12, i32 13, i32 14, i32 15>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V256_CDEF = shufflevector <16 x i16> %src256, <16 x i16> %src128_256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 16, i32 17, i32 18, i32 19>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V256_CDEF = shufflevector <16 x i16> %src256, <16 x i16> %src128_256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 16, i32 17, i32 18, i32 19>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V256_01234567 = shufflevector <16 x i16> %src256, <16 x i16> %src128_256, <16 x i32> <i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V256_89ABCDEF = shufflevector <16 x i16> %src256, <16 x i16> %src128_256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
@@ -858,19 +858,19 @@ define void @test_vXi8(<2 x i8> %src16, <4 x i8> %src32, <8 x i8> %src64, <16x i
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V64_0123 = shufflevector <8 x i8> %src64, <8 x i8> %src32_64, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 4, i32 5, i32 6, i32 7>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V64_4567 = shufflevector <8 x i8> %src64, <8 x i8> %src32_64, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V128_01 = shufflevector <16 x i8> %src128, <16 x i8> %src16_128, <16 x i32> <i32 16, i32 17, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %V128_23 = shufflevector <16 x i8> %src128, <16 x i8> %src16_128, <16 x i32> <i32 0, i32 1, i32 16, i32 17, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %V128_45 = shufflevector <16 x i8> %src128, <16 x i8> %src16_128, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 16, i32 17, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %V128_67 = shufflevector <16 x i8> %src128, <16 x i8> %src16_128, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 16, i32 17, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %V128_89 = shufflevector <16 x i8> %src128, <16 x i8> %src16_128, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %V128_AB = shufflevector <16 x i8> %src128, <16 x i8> %src16_128, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 16, i32 17, i32 12, i32 13, i32 14, i32 15>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %V128_CD = shufflevector <16 x i8> %src128, <16 x i8> %src16_128, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 16, i32 17, i32 14, i32 15>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %V128_EF = shufflevector <16 x i8> %src128, <16 x i8> %src16_128, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 16, i32 17>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %V128_23 = shufflevector <16 x i8> %src128, <16 x i8> %src16_128, <16 x i32> <i32 0, i32 1, i32 16, i32 17, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %V128_45 = shufflevector <16 x i8> %src128, <16 x i8> %src16_128, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 16, i32 17, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %V128_67 = shufflevector <16 x i8> %src128, <16 x i8> %src16_128, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 16, i32 17, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %V128_89 = shufflevector <16 x i8> %src128, <16 x i8> %src16_128, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %V128_AB = shufflevector <16 x i8> %src128, <16 x i8> %src16_128, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 16, i32 17, i32 12, i32 13, i32 14, i32 15>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %V128_CD = shufflevector <16 x i8> %src128, <16 x i8> %src16_128, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 16, i32 17, i32 14, i32 15>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %V128_EF = shufflevector <16 x i8> %src128, <16 x i8> %src16_128, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 16, i32 17>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V128_0123 = shufflevector <16 x i8> %src128, <16 x i8> %src32_128, <16 x i32> <i32 16, i32 17, i32 18, i32 19, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %V128_4567 = shufflevector <16 x i8> %src128, <16 x i8> %src32_128, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 16, i32 17, i32 18, i32 19, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %V128_89AB = shufflevector <16 x i8> %src128, <16 x i8> %src32_128, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 12, i32 13, i32 14, i32 15>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %V128_CDEF = shufflevector <16 x i8> %src128, <16 x i8> %src32_128, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 16, i32 17, i32 18, i32 19>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %V128_4567 = shufflevector <16 x i8> %src128, <16 x i8> %src32_128, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 16, i32 17, i32 18, i32 19, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %V128_89AB = shufflevector <16 x i8> %src128, <16 x i8> %src32_128, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 12, i32 13, i32 14, i32 15>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %V128_CDEF = shufflevector <16 x i8> %src128, <16 x i8> %src32_128, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 16, i32 17, i32 18, i32 19>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V128_01234567 = shufflevector <16 x i8> %src128, <16 x i8> %src64_128, <16 x i32> <i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %V128_89ABCDEF = shufflevector <16 x i8> %src128, <16 x i8> %src64_128, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %V128_89ABCDEF = shufflevector <16 x i8> %src128, <16 x i8> %src64_128, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; SSSE3-LABEL: 'test_vXi8'
diff --git a/llvm/test/Analysis/CostModel/X86/shuffle-single-src.ll b/llvm/test/Analysis/CostModel/X86/shuffle-single-src.ll
index ed3ddc060c167..595826cd6e8d5 100644
--- a/llvm/test/Analysis/CostModel/X86/shuffle-single-src.ll
+++ b/llvm/test/Analysis/CostModel/X86/shuffle-single-src.ll
@@ -193,10 +193,10 @@ define void @test_vXi16(<2 x i16> %src32, <4 x i16> %src64, <8 x i16> %src128, <
 ; SSE2-LABEL: 'test_vXi16'
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V32 = shufflevector <2 x i16> %src32, <2 x i16> undef, <2 x i32> <i32 1, i32 1>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V64 = shufflevector <4 x i16> %src64, <4 x i16> undef, <4 x i32> <i32 2, i32 2, i32 1, i32 1>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V128 = shufflevector <8 x i16> %src128, <8 x i16> undef, <8 x i32> <i32 7, i32 6, i32 6, i32 4, i32 3, i32 2, i32 1, i32 0>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V256 = shufflevector <16 x i16> %src256, <16 x i16> undef, <16 x i32> <i32 15, i32 14, i32 13, i32 13, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V512 = shufflevector <32 x i16> %src512, <32 x i16> undef, <32 x i32> <i32 31, i32 30, i32 20, i32 28, i32 27, i32 26, i32 25, i32 24, i32 23, i32 22, i32 21, i32 20, i32 19, i32 18, i32 17, i32 16, i32 15, i32 14, i32 13, i32 12, i32 11, i32 11, i32 9, i32 8, i32 7, i32 11, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 26 for instruction: %V1024 = shufflevector <64 x i16> %src1024, <64 x i16> undef, <64 x i32> <i32 63, i32 62, i32 61, i32 60, i32 59, i32 58, i32 57, i32 56, i32 55, i32 54, i32 53, i32 52, i32 51, i32 50, i32 49, i32 48, i32 47, i32 46, i32 45, i32 44, i32 43, i32 42, i32 41, i32 40, i32 39, i32 38, i32 37, i32 36, i32 35, i32 34, i32 33, i32 32, i32 31, i32 30, i32 29, i32 28, i32 27, i32 26, i32 25, i32 24, i32 23, i32 20, i32 21, i32 20, i32 19, i32 18, i32 17, i32 16, i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V128 = shufflevector <8 x i16> %src128, <8 x i16> undef, <8 x i32> <i32 7, i32 6, i32 6, i32 4, i32 3, i32 2, i32 1, i32 0>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V256 = shufflevector <16 x i16> %src256, <16 x i16> undef, <16 x i32> <i32 15, i32 14, i32 13, i32 13, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V512 = shufflevector <32 x i16> %src512, <32 x i16> undef, <32 x i32> <i32 31, i32 30, i32 20, i32 28, i32 27, i32 26, i32 25, i32 24, i32 23, i32 22, i32 21, i32 20, i32 19, i32 18, i32 17, i32 16, i32 15, i32 14, i32 13, i32 12, i32 11, i32 11, i32 9, i32 8, i32 7, i32 11, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V1024 = shufflevector <64 x i16> %src1024, <64 x i16> undef, <64 x i32> <i32 63, i32 62, i32 61, i32 60, i32 59, i32 58, i32 57, i32 56, i32 55, i32 54, i32 53, i32 52, i32 51, i32 50, i32 49, i32 48, i32 47, i32 46, i32 45, i32 44, i32 43, i32 42, i32 41, i32 40, i32 39, i32 38, i32 37, i32 36, i32 35, i32 34, i32 33, i32 32, i32 31, i32 30, i32 29, i32 28, i32 27, i32 26, i32 25, i32 24, i32 23, i32 20, i32 21, i32 20, i32 19, i32 18, i32 17, i32 16, i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; SSSE3-LABEL: 'test_vXi16'
@@ -285,9 +285,9 @@ define void @test_vXi8(<2 x i8> %src16, <4 x i8> %src32, <8 x i8> %src64, <16 x
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16 = shufflevector <2 x i8> %src16, <2 x i8> undef, <2 x i32> <i32 1, i32 1>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V32 = shufflevector <4 x i8> %src32, <4 x i8> undef, <4 x i32> <i32 3, i32 3, i32 1, i32 1>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V64 = shufflevector <8 x i8> %src64, <8 x i8> undef, <8 x i32> <i32 7, i32 7, i32 5, i32 5, i32 3, i32 3, i32 1, i32 1>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V128 = shufflevector <16 x i8> %src128, <16 x i8> undef, <16 x i32> <i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 11, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 19 for instruction: %V256 = shufflevector <32 x i8> %src256, <32 x i8> undef, <32 x i32> <i32 31, i32 30, i32 29, i32 28, i32 27, i32 26, i32 25, i32 24, i32 23, i32 22, i32 21, i32 20, i32 19, i32 18, i32 17, i32 16, i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 8, i32 8, i32 7, i32 6, i32 8, i32 4, i32 3, i32 2, i32 1, i32 0>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 37 for instruction: %V512 = shufflevector <64 x i8> %src512, <64 x i8> undef, <64 x i32> <i32 63, i32 62, i32 61, i32 60, i32 59, i32 58, i32 57, i32 56, i32 55, i32 54, i32 53, i32 52, i32 51, i32 50, i32 49, i32 48, i32 47, i32 46, i32 45, i32 44, i32 43, i32 42, i32 41, i32 40, i32 39, i32 38, i32 37, i32 36, i32 35, i32 34, i32 33, i32 32, i32 31, i32 30, i32 29, i32 28, i32 27, i32 26, i32 25, i32 24, i32 23, i32 20, i32 21, i32 20, i32 19, i32 18, i32 17, i32 16, i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V128 = shufflevector <16 x i8> %src128, <16 x i8> undef, <16 x i32> <i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 11, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %V256 = shufflevector <32 x i8> %src256, <32 x i8> undef, <32 x i32> <i32 31, i32 30, i32 29, i32 28, i32 27, i32 26, i32 25, i32 24, i32 23, i32 22, i32 21, i32 20, i32 19, i32 18, i32 17, i32 16, i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 8, i32 8, i32 7, i32 6, i32 8, i32 4, i32 3, i32 2, i32 1, i32 0>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 35 for instruction: %V512 = shufflevector <64 x i8> %src512, <64 x i8> undef, <64 x i32> <i32 63, i32 62, i32 61, i32 60, i32 59, i32 58, i32 57, i32 56, i32 55, i32 54, i32 53, i32 52, i32 51, i32 50, i32 49, i32 48, i32 47, i32 46, i32 45, i32 44, i32 43, i32 42, i32 41, i32 40, i32 39, i32 38, i32 37, i32 36, i32 35, i32 34, i32 33, i32 32, i32 31, i32 30, i32 29, i32 28, i32 27, i32 26, i32 25, i32 24, i32 23, i32 20, i32 21, i32 20, i32 19, i32 18, i32 17, i32 16, i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; SSSE3-LABEL: 'test_vXi8'
diff --git a/llvm/test/Analysis/CostModel/X86/shuffle-transpose.ll b/llvm/test/Analysis/CostModel/X86/shuffle-transpose.ll
index 5c50ec1feb058..8663659d89cb3 100644
--- a/llvm/test/Analysis/CostModel/X86/shuffle-transpose.ll
+++ b/llvm/test/Analysis/CostModel/X86/shuffle-transpose.ll
@@ -128,9 +128,9 @@ define void @test_vXi16(<2 x i16> %a32, <2 x i16> %b32, <4 x i16> %a64, <4 x i16
 ; SSE2-LABEL: 'test_vXi16'
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32 = shufflevector <2 x i16> %a32, <2 x i16> %b32, <2 x i32> <i32 0, i32 2>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V64 = shufflevector <4 x i16> %a64, <4 x i16> %b64, <4 x i32> <i32 0, i32 4, i32 2, i32 6>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V128 = shufflevector <8 x i16> %a128, <8 x i16> %b128, <8 x i32> <i32 0, i32 8, i32 2, i32 10, i32 4, i32 12, i32 6, i32 14>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V256 = shufflevector <16 x i16> %a256, <16 x i16> %b256, <16 x i32> <i32 0, i32 16, i32 2, i32 18, i32 4, i32 20, i32 6, i32 22, i32 8, i32 24, i32 10, i32 26, i32 12, i32 28, i32 14, i32 30>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %V512 = shufflevector <32 x i16> %a512, <32 x i16> %b512, <32 x i32> <i32 0, i32 32, i32 2, i32 34, i32 4, i32 36, i32 6, i32 38, i32 8, i32 40, i32 10, i32 42, i32 12, i32 44, i32 14, i32 46, i32 16, i32 48, i32 18, i32 50, i32 20, i32 52, i32 22, i32 54, i32 24, i32 56, i32 26, i32 58, i32 28, i32 60, i32 30, i32 62>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V128 = shufflevector <8 x i16> %a128, <8 x i16> %b128, <8 x i32> <i32 0, i32 8, i32 2, i32 10, i32 4, i32 12, i32 6, i32 14>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V256 = shufflevector <16 x i16> %a256, <16 x i16> %b256, <16 x i32> <i32 0, i32 16, i32 2, i32 18, i32 4, i32 20, i32 6, i32 22, i32 8, i32 24, i32 10, i32 26, i32 12, i32 28, i32 14, i32 30>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V512 = shufflevector <32 x i16> %a512, <32 x i16> %b512, <32 x i32> <i32 0, i32 32, i32 2, i32 34, i32 4, i32 36, i32 6, i32 38, i32 8, i32 40, i32 10, i32 42, i32 12, i32 44, i32 14, i32 46, i32 16, i32 48, i32 18, i32 50, i32 20, i32 52, i32 22, i32 54, i32 24, i32 56, i32 26, i32 58, i32 28, i32 60, i32 30, i32 62>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; SSSE3-LABEL: 'test_vXi16'
@@ -202,9 +202,9 @@ define void @test_vXi8(<2 x i8> %a16, <2 x i8> %b16, <4 x i8> %a32, <4 x i8> %b3
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16 = shufflevector <2 x i8> %a16, <2 x i8> %b16, <2 x i32> <i32 0, i32 2>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32 = shufflevector <4 x i8> %a32, <4 x i8> %b32, <4 x i32> <i32 0, i32 4, i32 2, i32 6>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V64 = shufflevector <8 x i8> %a64, <8 x i8> %b64, <8 x i32> <i32 0, i32 8, i32 2, i32 10, i32 4, i32 12, i32 6, i32 14>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %V128 = shufflevector <16 x i8> %a128, <16 x i8> %b128, <16 x i32> <i32 0, i32 16, i32 2, i32 18, i32 4, i32 20, i32 6, i32 22, i32 8, i32 24, i32 10, i32 26, i32 12, i32 28, i32 14, i32 30>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 26 for instruction: %V256 = shufflevector <32 x i8> %a256, <32 x i8> %b256, <32 x i32> <i32 0, i32 32, i32 2, i32 34, i32 4, i32 36, i32 6, i32 38, i32 8, i32 40, i32 10, i32 42, i32 12, i32 44, i32 14, i32 46, i32 16, i32 48, i32 18, i32 50, i32 20, i32 52, i32 22, i32 54, i32 24, i32 56, i32 26, i32 58, i32 28, i32 60, i32 30, i32 62>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 52 for instruction: %V512 = shufflevector <64 x i8> %a512, <64 x i8> %b512, <64 x i32> <i32 0, i32 64, i32 2, i32 66, i32 4, i32 68, i32 6, i32 70, i32 8, i32 72, i32 10, i32 74, i32 12, i32 76, i32 14, i32 78, i32 16, i32 80, i32 18, i32 82, i32 20, i32 84, i32 22, i32 86, i32 24, i32 88, i32 26, i32 90, i32 28, i32 92, i32 30, i32 94, i32 32, i32 96, i32 34, i32 98, i32 36, i32 100, i32 38, i32 102, i32 40, i32 104, i32 42, i32 106, i32 44, i32 108, i32 46, i32 110, i32 48, i32 112, i32 50, i32 114, i32 52, i32 116, i32 54, i32 118, i32 56, i32 120, i32 58, i32 122, i32 60, i32 124, i32 62, i32 126>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %V128 = shufflevector <16 x i8> %a128, <16 x i8> %b128, <16 x i32> <i32 0, i32 16, i32 2, i32 18, i32 4, i32 20, i32 6, i32 22, i32 8, i32 24, i32 10, i32 26, i32 12, i32 28, i32 14, i32 30>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 22 for instruction: %V256 = shufflevector <32 x i8> %a256, <32 x i8> %b256, <32 x i32> <i32 0, i32 32, i32 2, i32 34, i32 4, i32 36, i32 6, i32 38, i32 8, i32 40, i32 10, i32 42, i32 12, i32 44, i32 14, i32 46, i32 16, i32 48, i32 18, i32 50, i32 20, i32 52, i32 22, i32 54, i32 24, i32 56, i32 26, i32 58, i32 28, i32 60, i32 30, i32 62>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 44 for instruction: %V512 = shufflevector <64 x i8> %a512, <64 x i8> %b512, <64 x i32> <i32 0, i32 64, i32 2, i32 66, i32 4, i32 68, i32 6, i32 70, i32 8, i32 72, i32 10, i32 74, i32 12, i32 76, i32 14, i32 78, i32 16, i32 80, i32 18, i32 82, i32 20, i32 84, i32 22, i32 86, i32 24, i32 88, i32 26, i32 90, i32 28, i32 92, i32 30, i32 94, i32 32, i32 96, i32 34, i32 98, i32 36, i32 100, i32 38, i32 102, i32 40, i32 104, i32 42, i32 106, i32 44, i32 108, i32 46, i32 110, i32 48, i32 112, i32 50, i32 114, i32 52, i32 116, i32 54, i32 118, i32 56, i32 120, i32 58, i32 122, i32 60, i32 124, i32 62, i32 126>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; SSSE3-LABEL: 'test_vXi8'
diff --git a/llvm/test/Analysis/CostModel/X86/shuffle-two-src.ll b/llvm/test/Analysis/CostModel/X86/shuffle-two-src.ll
index efa0f2eb8dc94..67c753b77f134 100644
--- a/llvm/test/Analysis/CostModel/X86/shuffle-two-src.ll
+++ b/llvm/test/Analysis/CostModel/X86/shuffle-two-src.ll
@@ -237,10 +237,10 @@ define void @test_vXi16(<2 x i16> %src32, <4 x i16> %src64, <8 x i16> %src128, <
 ; SSE2-LABEL: 'test_vXi16'
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32 = shufflevector <2 x i16> %src32, <2 x i16> %src32_1, <2 x i32> <i32 3, i32 0>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V64 = shufflevector <4 x i16> %src64, <4 x i16> %src64_1, <4 x i32> <i32 3, i32 6, i32 1, i32 5>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V128 = shufflevector <8 x i16> %src128, <8 x i16> %src128_1, <8 x i32> <i32 7, i32 6, i32 6, i32 8, i32 9, i32 2, i32 1, i32 0>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %V256 = shufflevector <16 x i16> %src256, <16 x i16> %src256_1, <16 x i32> <i32 15, i32 14, i32 13, i32 20, i32 21, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 27 for instruction: %V512 = shufflevector <32 x i16> %src512, <32 x i16> %src512_1, <32 x i32> <i32 31, i32 30, i32 45, i32 28, i32 27, i32 26, i32 25, i32 24, i32 23, i32 22, i32 21, i32 20, i32 19, i32 18, i32 17, i32 16, i32 15, i32 14, i32 13, i32 38, i32 11, i32 11, i32 9, i32 8, i32 7, i32 11, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 36 for instruction: %V1024 = shufflevector <64 x i16> %src1024, <64 x i16> %src1024_1, <64 x i32> <i32 63, i32 62, i32 71, i32 60, i32 59, i32 58, i32 57, i32 56, i32 55, i32 54, i32 53, i32 52, i32 51, i32 50, i32 49, i32 48, i32 47, i32 46, i32 45, i32 44, i32 43, i32 42, i32 41, i32 40, i32 39, i32 38, i32 37, i32 36, i32 35, i32 34, i32 33, i32 32, i32 31, i32 30, i32 29, i32 28, i32 27, i32 26, i32 25, i32 24, i32 23, i32 20, i32 21, i32 20, i32 19, i32 18, i32 17, i32 16, i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 66, i32 2, i32 1, i32 0>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V128 = shufflevector <8 x i16> %src128, <8 x i16> %src128_1, <8 x i32> <i32 7, i32 6, i32 6, i32 8, i32 9, i32 2, i32 1, i32 0>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V256 = shufflevector <16 x i16> %src256, <16 x i16> %src256_1, <16 x i32> <i32 15, i32 14, i32 13, i32 20, i32 21, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 21 for instruction: %V512 = shufflevector <32 x i16> %src512, <32 x i16> %src512_1, <32 x i32> <i32 31, i32 30, i32 45, i32 28, i32 27, i32 26, i32 25, i32 24, i32 23, i32 22, i32 21, i32 20, i32 19, i32 18, i32 17, i32 16, i32 15, i32 14, i32 13, i32 38, i32 11, i32 11, i32 9, i32 8, i32 7, i32 11, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 30 for instruction: %V1024 = shufflevector <64 x i16> %src1024, <64 x i16> %src1024_1, <64 x i32> <i32 63, i32 62, i32 71, i32 60, i32 59, i32 58, i32 57, i32 56, i32 55, i32 54, i32 53, i32 52, i32 51, i32 50, i32 49, i32 48, i32 47, i32 46, i32 45, i32 44, i32 43, i32 42, i32 41, i32 40, i32 39, i32 38, i32 37, i32 36, i32 35, i32 34, i32 33, i32 32, i32 31, i32 30, i32 29, i32 28, i32 27, i32 26, i32 25, i32 24, i32 23, i32 20, i32 21, i32 20, i32 19, i32 18, i32 17, i32 16, i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 66, i32 2, i32 1, i32 0>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; SSSE3-LABEL: 'test_vXi16'
@@ -329,9 +329,9 @@ define void @test_vXi8(<2 x i8> %src16, <4 x i8> %src32, <8 x i8> %src64, <16 x
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16 = shufflevector <2 x i8> %src16, <2 x i8> %src16_1, <2 x i32> <i32 3, i32 0>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32 = shufflevector <4 x i8> %src32, <4 x i8> %src32_1, <4 x i32> <i32 3, i32 6, i32 1, i32 5>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V64 = shufflevector <8 x i8> %src64, <8 x i8> %src64_1, <8 x i32> <i32 7, i32 6, i32 6, i32 8, i32 9, i32 2, i32 1, i32 0>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %V128 = shufflevector <16 x i8> %src128, <16 x i8> %src128_1, <16 x i32> <i32 29, i32 14, i32 28, i32 12, i32 11, i32 10, i32 11, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 23 for instruction: %V256 = shufflevector <32 x i8> %src256, <32 x i8> %src256_1, <32 x i32> <i32 31, i32 30, i32 45, i32 28, i32 27, i32 26, i32 25, i32 24, i32 23, i32 22, i32 21, i32 20, i32 19, i32 18, i32 17, i32 16, i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 8, i32 8, i32 7, i32 6, i32 8, i32 4, i32 3, i32 2, i32 1, i32 0>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 41 for instruction: %V512 = shufflevector <64 x i8> %src512, <64 x i8> %src512_1, <64 x i32> <i32 63, i32 100, i32 61, i32 96, i32 59, i32 58, i32 57, i32 56, i32 55, i32 54, i32 53, i32 52, i32 51, i32 50, i32 49, i32 48, i32 47, i32 46, i32 45, i32 44, i32 43, i32 42, i32 41, i32 40, i32 39, i32 38, i32 37, i32 36, i32 35, i32 34, i32 33, i32 32, i32 31, i32 30, i32 29, i32 28, i32 27, i32 26, i32 25, i32 24, i32 23, i32 20, i32 21, i32 20, i32 19, i32 18, i32 17, i32 16, i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %V128 = shufflevector <16 x i8> %src128, <16 x i8> %src128_1, <16 x i32> <i32 29, i32 14, i32 28, i32 12, i32 11, i32 10, i32 11, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 19 for instruction: %V256 = shufflevector <32 x i8> %src256, <32 x i8> %src256_1, <32 x i32> <i32 31, i32 30, i32 45, i32 28, i32 27, i32 26, i32 25, i32 24, i32 23, i32 22, i32 21, i32 20, i32 19, i32 18, i32 17, i32 16, i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 8, i32 8, i32 7, i32 6, i32 8, i32 4, i32 3, i32 2, i32 1, i32 0>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 37 for instruction: %V512 = shufflevector <64 x i8> %src512, <64 x i8> %src512_1, <64 x i32> <i32 63, i32 100, i32 61, i32 96, i32 59, i32 58, i32 57, i32 56, i32 55, i32 54, i32 53, i32 52, i32 51, i32 50, i32 49, i32 48, i32 47, i32 46, i32 45, i32 44, i32 43, i32 42, i32 41, i32 40, i32 39, i32 38, i32 37, i32 36, i32 35, i32 34, i32 33, i32 32, i32 31, i32 30, i32 29, i32 28, i32 27, i32 26, i32 25, i32 24, i32 23, i32 20, i32 21, i32 20, i32 19, i32 18, i32 17, i32 16, i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; SSSE3-LABEL: 'test_vXi8'
diff --git a/llvm/test/Analysis/CostModel/X86/vector-insert-inseltpoison.ll b/llvm/test/Analysis/CostModel/X86/vector-insert-inseltpoison.ll
index 9caaae0717fda..ed034d7bccc94 100644
--- a/llvm/test/Analysis/CostModel/X86/vector-insert-inseltpoison.ll
+++ b/llvm/test/Analysis/CostModel/X86/vector-insert-inseltpoison.ll
@@ -674,25 +674,25 @@ define i32 @insert_i8(i32 %arg) {
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v8i8_7 = insertelement <8 x i8> poison, i8 undef, i32 7
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v16i8_a = insertelement <16 x i8> poison, i8 undef, i32 %arg
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v16i8_0 = insertelement <16 x i8> poison, i8 undef, i32 0
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %v16i8_8 = insertelement <16 x i8> poison, i8 undef, i32 8
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %v16i8_15 = insertelement <16 x i8> poison, i8 undef, i32 15
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v16i8_8 = insertelement <16 x i8> poison, i8 undef, i32 8
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v16i8_15 = insertelement <16 x i8> poison, i8 undef, i32 15
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %v32i8_a = insertelement <32 x i8> poison, i8 undef, i32 %arg
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v32i8_0 = insertelement <32 x i8> poison, i8 undef, i32 0
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %v32i8_7 = insertelement <32 x i8> poison, i8 undef, i32 7
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %v32i8_8 = insertelement <32 x i8> poison, i8 undef, i32 8
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %v32i8_15 = insertelement <32 x i8> poison, i8 undef, i32 15
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %v32i8_24 = insertelement <32 x i8> poison, i8 undef, i32 24
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %v32i8_31 = insertelement <32 x i8> poison, i8 undef, i32 31
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v32i8_7 = insertelement <32 x i8> poison, i8 undef, i32 7
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v32i8_8 = insertelement <32 x i8> poison, i8 undef, i32 8
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v32i8_15 = insertelement <32 x i8> poison, i8 undef, i32 15
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v32i8_24 = insertelement <32 x i8> poison, i8 undef, i32 24
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v32i8_31 = insertelement <32 x i8> poison, i8 undef, i32 31
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %v64i8_a = insertelement <64 x i8> poison, i8 undef, i32 %arg
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v64i8_0 = insertelement <64 x i8> poison, i8 undef, i32 0
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %v64i8_7 = insertelement <64 x i8> poison, i8 undef, i32 7
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %v64i8_8 = insertelement <64 x i8> poison, i8 undef, i32 8
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %v64i8_15 = insertelement <64 x i8> poison, i8 undef, i32 15
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %v64i8_24 = insertelement <64 x i8> poison, i8 undef, i32 24
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %v64i8_31 = insertelement <64 x i8> poison, i8 undef, i32 31
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v64i8_7 = insertelement <64 x i8> poison, i8 undef, i32 7
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v64i8_8 = insertelement <64 x i8> poison, i8 undef, i32 8
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v64i8_15 = insertelement <64 x i8> poison, i8 undef, i32 15
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v64i8_24 = insertelement <64 x i8> poison, i8 undef, i32 24
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v64i8_31 = insertelement <64 x i8> poison, i8 undef, i32 31
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v64i8_32 = insertelement <64 x i8> poison, i8 undef, i32 32
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v64i8_48 = insertelement <64 x i8> poison, i8 undef, i32 48
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %v64i8_63 = insertelement <64 x i8> poison, i8 undef, i32 63
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v64i8_63 = insertelement <64 x i8> poison, i8 undef, i32 63
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; SSE3-LABEL: 'insert_i8'
@@ -707,25 +707,25 @@ define i32 @insert_i8(i32 %arg) {
 ; SSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v8i8_7 = insertelement <8 x i8> poison, i8 undef, i32 7
 ; SSE3-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v16i8_a = insertelement <16 x i8> poison, i8 undef, i32 %arg
 ; SSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v16i8_0 = insertelement <16 x i8> poison, i8 undef, i32 0
-; SSE3-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %v16i8_8 = insertelement <16 x i8> poison, i8 undef, i32 8
-; SSE3-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %v16i8_15 = insertelement <16 x i8> poison, i8 undef, i32 15
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v16i8_8 = insertelement <16 x i8> poison, i8 undef, i32 8
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v16i8_15 = insertelement <16 x i8> poison, i8 undef, i32 15
 ; SSE3-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %v32i8_a = insertelement <32 x i8> poison, i8 undef, i32 %arg
 ; SSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v32i8_0 = insertelement <32 x i8> poison, i8 undef, i32 0
-; SSE3-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %v32i8_7 = insertelement <32 x i8> poison, i8 undef, i32 7
-; SSE3-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %v32i8_8 = insertelement <32 x i8> poison, i8 undef, i32 8
-; SSE3-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %v32i8_15 = insertelement <32 x i8> poison, i8 undef, i32 15
-; SSE3-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %v32i8_24 = insertelement <32 x i8> poison, i8 undef, i32 24
-; SSE3-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %v32i8_31 = insertelement <32 x i8> poison, i8 undef, i32 31
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v32i8_7 = insertelement <32 x i8> poison, i8 undef, i32 7
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v32i8_8 = insertelement <32 x i8> poison, i8 undef, i32 8
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v32i8_15 = insertelement <32 x i8> poison, i8 undef, i32 15
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v32i8_24 = insertelement <32 x i8> poison, i8 undef, i32 24
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v32i8_31 = insertelement <32 x i8> poison, i8 undef, i32 31
 ; SSE3-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %v64i8_a = insertelement <64 x i8> poison, i8 undef, i32 %arg
 ; SSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v64i8_0 = insertelement <64 x i8> poison, i8 undef, i32 0
-; SSE3-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %v64i8_7 = insertelement <64 x i8> poison, i8 undef, i32 7
-; SSE3-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %v64i8_8 = insertelement <64 x i8> poison, i8 undef, i32 8
-; SSE3-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %v64i8_15 = insertelement <64 x i8> poison, i8 undef, i32 15
-; SSE3-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %v64i8_24 = insertelement <64 x i8> poison, i8 undef, i32 24
-; SSE3-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %v64i8_31 = insertelement <64 x i8> poison, i8 undef, i32 31
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v64i8_7 = insertelement <64 x i8> poison, i8 undef, i32 7
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v64i8_8 = insertelement <64 x i8> poison, i8 undef, i32 8
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v64i8_15 = insertelement <64 x i8> poison, i8 undef, i32 15
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v64i8_24 = insertelement <64 x i8> poison, i8 undef, i32 24
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v64i8_31 = insertelement <64 x i8> poison, i8 undef, i32 31
 ; SSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v64i8_32 = insertelement <64 x i8> poison, i8 undef, i32 32
 ; SSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v64i8_48 = insertelement <64 x i8> poison, i8 undef, i32 48
-; SSE3-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %v64i8_63 = insertelement <64 x i8> poison, i8 undef, i32 63
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v64i8_63 = insertelement <64 x i8> poison, i8 undef, i32 63
 ; SSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; SSSE3-LABEL: 'insert_i8'
@@ -978,25 +978,25 @@ define i32 @insert_i1(i32 %arg) {
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i1_4 = insertelement <8 x i1> poison, i1 undef, i32 4
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %v16i1_a = insertelement <16 x i1> poison, i1 undef, i32 %arg
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v16i1_0 = insertelement <16 x i1> poison, i1 undef, i32 0
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %v16i1_8 = insertelement <16 x i1> poison, i1 undef, i32 8
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %v16i1_15 = insertelement <16 x i1> poison, i1 undef, i32 15
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v16i1_8 = insertelement <16 x i1> poison, i1 undef, i32 8
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v16i1_15 = insertelement <16 x i1> poison, i1 undef, i32 15
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v32i1_a = insertelement <32 x i1> poison, i1 undef, i32 %arg
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v32i1_0 = insertelement <32 x i1> poison, i1 undef, i32 0
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %v32i1_7 = insertelement <32 x i1> poison, i1 undef, i32 7
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %v32i1_8 = insertelement <32 x i1> poison, i1 undef, i32 8
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %v32i1_15 = insertelement <32 x i1> poison, i1 undef, i32 15
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %v32i1_24 = insertelement <32 x i1> poison, i1 undef, i32 24
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %v32i1_31 = insertelement <32 x i1> poison, i1 undef, i32 31
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v32i1_7 = insertelement <32 x i1> poison, i1 undef, i32 7
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v32i1_8 = insertelement <32 x i1> poison, i1 undef, i32 8
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v32i1_15 = insertelement <32 x i1> poison, i1 undef, i32 15
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v32i1_24 = insertelement <32 x i1> poison, i1 undef, i32 24
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v32i1_31 = insertelement <32 x i1> poison, i1 undef, i32 31
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v64i1_a = insertelement <64 x i1> poison, i1 undef, i32 %arg
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v64i1_0 = insertelement <64 x i1> poison, i1 undef, i32 0
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %v64i1_7 = insertelement <64 x i1> poison, i1 undef, i32 7
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %v64i1_8 = insertelement <64 x i1> poison, i1 undef, i32 8
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %v64i1_15 = insertelement <64 x i1> poison, i1 undef, i32 15
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %v64i1_24 = insertelement <64 x i1> poison, i1 undef, i32 24
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %v64i1_31 = insertelement <64 x i1> poison, i1 undef, i32 31
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v64i1_7 = insertelement <64 x i1> poison, i1 undef, i32 7
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v64i1_8 = insertelement <64 x i1> poison, i1 undef, i32 8
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v64i1_15 = insertelement <64 x i1> poison, i1 undef, i32 15
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v64i1_24 = insertelement <64 x i1> poison, i1 undef, i32 24
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v64i1_31 = insertelement <64 x i1> poison, i1 undef, i32 31
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v64i1_32 = insertelement <64 x i1> poison, i1 undef, i32 32
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v64i1_48 = insertelement <64 x i1> poison, i1 undef, i32 48
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %v64i1_63 = insertelement <64 x i1> poison, i1 undef, i32 63
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v64i1_63 = insertelement <64 x i1> poison, i1 undef, i32 63
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; SSE3-LABEL: 'insert_i1'
@@ -1011,25 +1011,25 @@ define i32 @insert_i1(i32 %arg) {
 ; SSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i1_4 = insertelement <8 x i1> poison, i1 undef, i32 4
 ; SSE3-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %v16i1_a = insertelement <16 x i1> poison, i1 undef, i32 %arg
 ; SSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v16i1_0 = insertelement <16 x i1> poison, i1 undef, i32 0
-; SSE3-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %v16i1_8 = insertelement <16 x i1> poison, i1 undef, i32 8
-; SSE3-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %v16i1_15 = insertelement <16 x i1> poison, i1 undef, i32 15
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v16i1_8 = insertelement <16 x i1> poison, i1 undef, i32 8
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v16i1_15 = insertelement <16 x i1> poison, i1 undef, i32 15
 ; SSE3-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v32i1_a = insertelement <32 x i1> poison, i1 undef, i32 %arg
 ; SSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v32i1_0 = insertelement <32 x i1> poison, i1 undef, i32 0
-; SSE3-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %v32i1_7 = insertelement <32 x i1> poison, i1 undef, i32 7
-; SSE3-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %v32i1_8 = insertelement <32 x i1> poison, i1 undef, i32 8
-; SSE3-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %v32i1_15 = insertelement <32 x i1> poison, i1 undef, i32 15
-; SSE3-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %v32i1_24 = insertelement <32 x i1> poison, i1 undef, i32 24
-; SSE3-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %v32i1_31 = insertelement <32 x i1> poison, i1 undef, i32 31
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v32i1_7 = insertelement <32 x i1> poison, i1 undef, i32 7
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v32i1_8 = insertelement <32 x i1> poison, i1 undef, i32 8
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v32i1_15 = insertelement <32 x i1> poison, i1 undef, i32 15
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v32i1_24 = insertelement <32 x i1> poison, i1 undef, i32 24
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v32i1_31 = insertelement <32 x i1> poison, i1 undef, i32 31
 ; SSE3-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v64i1_a = insertelement <64 x i1> poison, i1 undef, i32 %arg
 ; SSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v64i1_0 = insertelement <64 x i1> poison, i1 undef, i32 0
-; SSE3-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %v64i1_7 = insertelement <64 x i1> poison, i1 undef, i32 7
-; SSE3-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %v64i1_8 = insertelement <64 x i1> poison, i1 undef, i32 8
-; SSE3-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %v64i1_15 = insertelement <64 x i1> poison, i1 undef, i32 15
-; SSE3-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %v64i1_24 = insertelement <64 x i1> poison, i1 undef, i32 24
-; SSE3-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %v64i1_31 = insertelement <64 x i1> poison, i1 undef, i32 31
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v64i1_7 = insertelement <64 x i1> poison, i1 undef, i32 7
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v64i1_8 = insertelement <64 x i1> poison, i1 undef, i32 8
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v64i1_15 = insertelement <64 x i1> poison, i1 undef, i32 15
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v64i1_24 = insertelement <64 x i1> poison, i1 undef, i32 24
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v64i1_31 = insertelement <64 x i1> poison, i1 undef, i32 31
 ; SSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v64i1_32 = insertelement <64 x i1> poison, i1 undef, i32 32
 ; SSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v64i1_48 = insertelement <64 x i1> poison, i1 undef, i32 48
-; SSE3-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %v64i1_63 = insertelement <64 x i1> poison, i1 undef, i32 63
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v64i1_63 = insertelement <64 x i1> poison, i1 undef, i32 63
 ; SSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; SSSE3-LABEL: 'insert_i1'
diff --git a/llvm/test/Analysis/CostModel/X86/vector-insert-value.ll b/llvm/test/Analysis/CostModel/X86/vector-insert-value.ll
index ee82e10f9ebb6..aff1de0bfde61 100644
--- a/llvm/test/Analysis/CostModel/X86/vector-insert-value.ll
+++ b/llvm/test/Analysis/CostModel/X86/vector-insert-value.ll
@@ -673,26 +673,26 @@ define i32 @insert_i8(i32 %arg, i8 %val, <2 x i8> %src16, <4 x i8> %src32, <8 x
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v8i8_0 = insertelement <8 x i8> %src64, i8 %val, i32 0
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v8i8_7 = insertelement <8 x i8> %src64, i8 %val, i32 7
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v16i8_a = insertelement <16 x i8> %src128, i8 %val, i32 %arg
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %v16i8_0 = insertelement <16 x i8> %src128, i8 %val, i32 0
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %v16i8_8 = insertelement <16 x i8> %src128, i8 %val, i32 8
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %v16i8_15 = insertelement <16 x i8> %src128, i8 %val, i32 15
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v16i8_0 = insertelement <16 x i8> %src128, i8 %val, i32 0
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v16i8_8 = insertelement <16 x i8> %src128, i8 %val, i32 8
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v16i8_15 = insertelement <16 x i8> %src128, i8 %val, i32 15
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %v32i8_a = insertelement <32 x i8> %src256, i8 %val, i32 %arg
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %v32i8_0 = insertelement <32 x i8> %src256, i8 %val, i32 0
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %v32i8_7 = insertelement <32 x i8> %src256, i8 %val, i32 7
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %v32i8_8 = insertelement <32 x i8> %src256, i8 %val, i32 8
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %v32i8_15 = insertelement <32 x i8> %src256, i8 %val, i32 15
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %v32i8_24 = insertelement <32 x i8> %src256, i8 %val, i32 24
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %v32i8_31 = insertelement <32 x i8> %src256, i8 %val, i32 31
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v32i8_0 = insertelement <32 x i8> %src256, i8 %val, i32 0
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v32i8_7 = insertelement <32 x i8> %src256, i8 %val, i32 7
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v32i8_8 = insertelement <32 x i8> %src256, i8 %val, i32 8
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v32i8_15 = insertelement <32 x i8> %src256, i8 %val, i32 15
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v32i8_24 = insertelement <32 x i8> %src256, i8 %val, i32 24
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v32i8_31 = insertelement <32 x i8> %src256, i8 %val, i32 31
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %v64i8_a = insertelement <64 x i8> %src512, i8 %val, i32 %arg
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %v64i8_0 = insertelement <64 x i8> %src512, i8 %val, i32 0
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %v64i8_7 = insertelement <64 x i8> %src512, i8 %val, i32 7
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %v64i8_8 = insertelement <64 x i8> %src512, i8 %val, i32 8
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %v64i8_15 = insertelement <64 x i8> %src512, i8 %val, i32 15
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %v64i8_24 = insertelement <64 x i8> %src512, i8 %val, i32 24
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %v64i8_31 = insertelement <64 x i8> %src512, i8 %val, i32 31
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %v64i8_32 = insertelement <64 x i8> %src512, i8 %val, i32 32
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %v64i8_48 = insertelement <64 x i8> %src512, i8 %val, i32 48
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %v64i8_63 = insertelement <64 x i8> %src512, i8 %val, i32 63
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v64i8_0 = insertelement <64 x i8> %src512, i8 %val, i32 0
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v64i8_7 = insertelement <64 x i8> %src512, i8 %val, i32 7
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v64i8_8 = insertelement <64 x i8> %src512, i8 %val, i32 8
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v64i8_15 = insertelement <64 x i8> %src512, i8 %val, i32 15
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v64i8_24 = insertelement <64 x i8> %src512, i8 %val, i32 24
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v64i8_31 = insertelement <64 x i8> %src512, i8 %val, i32 31
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v64i8_32 = insertelement <64 x i8> %src512, i8 %val, i32 32
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v64i8_48 = insertelement <64 x i8> %src512, i8 %val, i32 48
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v64i8_63 = insertelement <64 x i8> %src512, i8 %val, i32 63
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; SSE3-LABEL: 'insert_i8'
@@ -706,26 +706,26 @@ define i32 @insert_i8(i32 %arg, i8 %val, <2 x i8> %src16, <4 x i8> %src32, <8 x
 ; SSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v8i8_0 = insertelement <8 x i8> %src64, i8 %val, i32 0
 ; SSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v8i8_7 = insertelement <8 x i8> %src64, i8 %val, i32 7
 ; SSE3-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v16i8_a = insertelement <16 x i8> %src128, i8 %val, i32 %arg
-; SSE3-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %v16i8_0 = insertelement <16 x i8> %src128, i8 %val, i32 0
-; SSE3-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %v16i8_8 = insertelement <16 x i8> %src128, i8 %val, i32 8
-; SSE3-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %v16i8_15 = insertelement <16 x i8> %src128, i8 %val, i32 15
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v16i8_0 = insertelement <16 x i8> %src128, i8 %val, i32 0
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v16i8_8 = insertelement <16 x i8> %src128, i8 %val, i32 8
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v16i8_15 = insertelement <16 x i8> %src128, i8 %val, i32 15
 ; SSE3-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %v32i8_a = insertelement <32 x i8> %src256, i8 %val, i32 %arg
-; SSE3-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %v32i8_0 = insertelement <32 x i8> %src256, i8 %val, i32 0
-; SSE3-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %v32i8_7 = insertelement <32 x i8> %src256, i8 %val, i32 7
-; SSE3-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %v32i8_8 = insertelement <32 x i8> %src256, i8 %val, i32 8
-; SSE3-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %v32i8_15 = insertelement <32 x i8> %src256, i8 %val, i32 15
-; SSE3-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %v32i8_24 = insertelement <32 x i8> %src256, i8 %val, i32 24
-; SSE3-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %v32i8_31 = insertelement <32 x i8> %src256, i8 %val, i32 31
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v32i8_0 = insertelement <32 x i8> %src256, i8 %val, i32 0
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v32i8_7 = insertelement <32 x i8> %src256, i8 %val, i32 7
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v32i8_8 = insertelement <32 x i8> %src256, i8 %val, i32 8
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v32i8_15 = insertelement <32 x i8> %src256, i8 %val, i32 15
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v32i8_24 = insertelement <32 x i8> %src256, i8 %val, i32 24
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v32i8_31 = insertelement <32 x i8> %src256, i8 %val, i32 31
 ; SSE3-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %v64i8_a = insertelement <64 x i8> %src512, i8 %val, i32 %arg
-; SSE3-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %v64i8_0 = insertelement <64 x i8> %src512, i8 %val, i32 0
-; SSE3-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %v64i8_7 = insertelement <64 x i8> %src512, i8 %val, i32 7
-; SSE3-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %v64i8_8 = insertelement <64 x i8> %src512, i8 %val, i32 8
-; SSE3-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %v64i8_15 = insertelement <64 x i8> %src512, i8 %val, i32 15
-; SSE3-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %v64i8_24 = insertelement <64 x i8> %src512, i8 %val, i32 24
-; SSE3-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %v64i8_31 = insertelement <64 x i8> %src512, i8 %val, i32 31
-; SSE3-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %v64i8_32 = insertelement <64 x i8> %src512, i8 %val, i32 32
-; SSE3-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %v64i8_48 = insertelement <64 x i8> %src512, i8 %val, i32 48
-; SSE3-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %v64i8_63 = insertelement <64 x i8> %src512, i8 %val, i32 63
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v64i8_0 = insertelement <64 x i8> %src512, i8 %val, i32 0
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v64i8_7 = insertelement <64 x i8> %src512, i8 %val, i32 7
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v64i8_8 = insertelement <64 x i8> %src512, i8 %val, i32 8
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v64i8_15 = insertelement <64 x i8> %src512, i8 %val, i32 15
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v64i8_24 = insertelement <64 x i8> %src512, i8 %val, i32 24
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v64i8_31 = insertelement <64 x i8> %src512, i8 %val, i32 31
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v64i8_32 = insertelement <64 x i8> %src512, i8 %val, i32 32
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v64i8_48 = insertelement <64 x i8> %src512, i8 %val, i32 48
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v64i8_63 = insertelement <64 x i8> %src512, i8 %val, i32 63
 ; SSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; SSSE3-LABEL: 'insert_i8'
@@ -977,26 +977,26 @@ define i32 @insert_i1(i32 %arg, i1 %val, <2 x i1> %src2, <4 x i1> %src4, <8 x i1
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i1_0 = insertelement <8 x i1> %src8, i1 %val, i32 0
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i1_4 = insertelement <8 x i1> %src8, i1 %val, i32 4
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %v16i1_a = insertelement <16 x i1> %src16, i1 %val, i32 %arg
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %v16i1_0 = insertelement <16 x i1> %src16, i1 %val, i32 0
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %v16i1_8 = insertelement <16 x i1> %src16, i1 %val, i32 8
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %v16i1_15 = insertelement <16 x i1> %src16, i1 %val, i32 15
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v16i1_0 = insertelement <16 x i1> %src16, i1 %val, i32 0
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v16i1_8 = insertelement <16 x i1> %src16, i1 %val, i32 8
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v16i1_15 = insertelement <16 x i1> %src16, i1 %val, i32 15
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v32i1_a = insertelement <32 x i1> %src32, i1 %val, i32 %arg
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %v32i1_0 = insertelement <32 x i1> %src32, i1 %val, i32 0
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %v32i1_7 = insertelement <32 x i1> %src32, i1 %val, i32 7
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %v32i1_8 = insertelement <32 x i1> %src32, i1 %val, i32 8
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %v32i1_15 = insertelement <32 x i1> %src32, i1 %val, i32 15
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %v32i1_24 = insertelement <32 x i1> %src32, i1 %val, i32 24
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %v32i1_31 = insertelement <32 x i1> %src32, i1 %val, i32 31
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v32i1_0 = insertelement <32 x i1> %src32, i1 %val, i32 0
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v32i1_7 = insertelement <32 x i1> %src32, i1 %val, i32 7
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v32i1_8 = insertelement <32 x i1> %src32, i1 %val, i32 8
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v32i1_15 = insertelement <32 x i1> %src32, i1 %val, i32 15
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v32i1_24 = insertelement <32 x i1> %src32, i1 %val, i32 24
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v32i1_31 = insertelement <32 x i1> %src32, i1 %val, i32 31
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v64i1_a = insertelement <64 x i1> %src64, i1 %val, i32 %arg
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %v64i1_0 = insertelement <64 x i1> %src64, i1 %val, i32 0
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %v64i1_7 = insertelement <64 x i1> %src64, i1 %val, i32 7
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %v64i1_8 = insertelement <64 x i1> %src64, i1 %val, i32 8
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %v64i1_15 = insertelement <64 x i1> %src64, i1 %val, i32 15
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %v64i1_24 = insertelement <64 x i1> %src64, i1 %val, i32 24
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %v64i1_31 = insertelement <64 x i1> %src64, i1 %val, i32 31
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %v64i1_32 = insertelement <64 x i1> %src64, i1 %val, i32 32
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %v64i1_48 = insertelement <64 x i1> %src64, i1 %val, i32 48
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %v64i1_63 = insertelement <64 x i1> %src64, i1 %val, i32 63
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v64i1_0 = insertelement <64 x i1> %src64, i1 %val, i32 0
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v64i1_7 = insertelement <64 x i1> %src64, i1 %val, i32 7
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v64i1_8 = insertelement <64 x i1> %src64, i1 %val, i32 8
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v64i1_15 = insertelement <64 x i1> %src64, i1 %val, i32 15
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v64i1_24 = insertelement <64 x i1> %src64, i1 %val, i32 24
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v64i1_31 = insertelement <64 x i1> %src64, i1 %val, i32 31
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v64i1_32 = insertelement <64 x i1> %src64, i1 %val, i32 32
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v64i1_48 = insertelement <64 x i1> %src64, i1 %val, i32 48
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v64i1_63 = insertelement <64 x i1> %src64, i1 %val, i32 63
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; SSE3-LABEL: 'insert_i1'
@@ -1010,26 +1010,26 @@ define i32 @insert_i1(i32 %arg, i1 %val, <2 x i1> %src2, <4 x i1> %src4, <8 x i1
 ; SSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i1_0 = insertelement <8 x i1> %src8, i1 %val, i32 0
 ; SSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i1_4 = insertelement <8 x i1> %src8, i1 %val, i32 4
 ; SSE3-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %v16i1_a = insertelement <16 x i1> %src16, i1 %val, i32 %arg
-; SSE3-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %v16i1_0 = insertelement <16 x i1> %src16, i1 %val, i32 0
-; SSE3-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %v16i1_8 = insertelement <16 x i1> %src16, i1 %val, i32 8
-; SSE3-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %v16i1_15 = insertelement <16 x i1> %src16, i1 %val, i32 15
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v16i1_0 = insertelement <16 x i1> %src16, i1 %val, i32 0
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v16i1_8 = insertelement <16 x i1> %src16, i1 %val, i32 8
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v16i1_15 = insertelement <16 x i1> %src16, i1 %val, i32 15
 ; SSE3-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v32i1_a = insertelement <32 x i1> %src32, i1 %val, i32 %arg
-; SSE3-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %v32i1_0 = insertelement <32 x i1> %src32, i1 %val, i32 0
-; SSE3-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %v32i1_7 = insertelement <32 x i1> %src32, i1 %val, i32 7
-; SSE3-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %v32i1_8 = insertelement <32 x i1> %src32, i1 %val, i32 8
-; SSE3-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %v32i1_15 = insertelement <32 x i1> %src32, i1 %val, i32 15
-; SSE3-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %v32i1_24 = insertelement <32 x i1> %src32, i1 %val, i32 24
-; SSE3-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %v32i1_31 = insertelement <32 x i1> %src32, i1 %val, i32 31
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v32i1_0 = insertelement <32 x i1> %src32, i1 %val, i32 0
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v32i1_7 = insertelement <32 x i1> %src32, i1 %val, i32 7
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v32i1_8 = insertelement <32 x i1> %src32, i1 %val, i32 8
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v32i1_15 = insertelement <32 x i1> %src32, i1 %val, i32 15
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v32i1_24 = insertelement <32 x i1> %src32, i1 %val, i32 24
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v32i1_31 = insertelement <32 x i1> %src32, i1 %val, i32 31
 ; SSE3-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v64i1_a = insertelement <64 x i1> %src64, i1 %val, i32 %arg
-; SSE3-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %v64i1_0 = insertelement <64 x i1> %src64, i1 %val, i32 0
-; SSE3-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %v64i1_7 = insertelement <64 x i1> %src64, i1 %val, i32 7
-; SSE3-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %v64i1_8 = insertelement <64 x i1> %src64, i1 %val, i32 8
-; SSE3-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %v64i1_15 = insertelement <64 x i1> %src64, i1 %val, i32 15
-; SSE3-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %v64i1_24 = insertelement <64 x i1> %src64, i1 %val, i32 24
-; SSE3-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %v64i1_31 = insertelement <64 x i1> %src64, i1 %val, i32 31
-; SSE3-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %v64i1_32 = insertelement <64 x i1> %src64, i1 %val, i32 32
-; SSE3-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %v64i1_48 = insertelement <64 x i1> %src64, i1 %val, i32 48
-; SSE3-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %v64i1_63 = insertelement <64 x i1> %src64, i1 %val, i32 63
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v64i1_0 = insertelement <64 x i1> %src64, i1 %val, i32 0
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v64i1_7 = insertelement <64 x i1> %src64, i1 %val, i32 7
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v64i1_8 = insertelement <64 x i1> %src64, i1 %val, i32 8
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v64i1_15 = insertelement <64 x i1> %src64, i1 %val, i32 15
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v64i1_24 = insertelement <64 x i1> %src64, i1 %val, i32 24
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v64i1_31 = insertelement <64 x i1> %src64, i1 %val, i32 31
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v64i1_32 = insertelement <64 x i1> %src64, i1 %val, i32 32
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v64i1_48 = insertelement <64 x i1> %src64, i1 %val, i32 48
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v64i1_63 = insertelement <64 x i1> %src64, i1 %val, i32 63
 ; SSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; SSSE3-LABEL: 'insert_i1'
diff --git a/llvm/test/Analysis/CostModel/X86/vector-insert.ll b/llvm/test/Analysis/CostModel/X86/vector-insert.ll
index 1d2e1f23b9b53..103b7be692025 100644
--- a/llvm/test/Analysis/CostModel/X86/vector-insert.ll
+++ b/llvm/test/Analysis/CostModel/X86/vector-insert.ll
@@ -674,25 +674,25 @@ define i32 @insert_i8(i32 %arg) {
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v8i8_7 = insertelement <8 x i8> undef, i8 undef, i32 7
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v16i8_a = insertelement <16 x i8> undef, i8 undef, i32 %arg
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v16i8_0 = insertelement <16 x i8> undef, i8 undef, i32 0
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %v16i8_8 = insertelement <16 x i8> undef, i8 undef, i32 8
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %v16i8_15 = insertelement <16 x i8> undef, i8 undef, i32 15
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v16i8_8 = insertelement <16 x i8> undef, i8 undef, i32 8
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v16i8_15 = insertelement <16 x i8> undef, i8 undef, i32 15
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %v32i8_a = insertelement <32 x i8> undef, i8 undef, i32 %arg
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v32i8_0 = insertelement <32 x i8> undef, i8 undef, i32 0
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %v32i8_7 = insertelement <32 x i8> undef, i8 undef, i32 7
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %v32i8_8 = insertelement <32 x i8> undef, i8 undef, i32 8
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %v32i8_15 = insertelement <32 x i8> undef, i8 undef, i32 15
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %v32i8_24 = insertelement <32 x i8> undef, i8 undef, i32 24
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %v32i8_31 = insertelement <32 x i8> undef, i8 undef, i32 31
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v32i8_7 = insertelement <32 x i8> undef, i8 undef, i32 7
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v32i8_8 = insertelement <32 x i8> undef, i8 undef, i32 8
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v32i8_15 = insertelement <32 x i8> undef, i8 undef, i32 15
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v32i8_24 = insertelement <32 x i8> undef, i8 undef, i32 24
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v32i8_31 = insertelement <32 x i8> undef, i8 undef, i32 31
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %v64i8_a = insertelement <64 x i8> undef, i8 undef, i32 %arg
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v64i8_0 = insertelement <64 x i8> undef, i8 undef, i32 0
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %v64i8_7 = insertelement <64 x i8> undef, i8 undef, i32 7
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %v64i8_8 = insertelement <64 x i8> undef, i8 undef, i32 8
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %v64i8_15 = insertelement <64 x i8> undef, i8 undef, i32 15
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %v64i8_24 = insertelement <64 x i8> undef, i8 undef, i32 24
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %v64i8_31 = insertelement <64 x i8> undef, i8 undef, i32 31
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v64i8_7 = insertelement <64 x i8> undef, i8 undef, i32 7
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v64i8_8 = insertelement <64 x i8> undef, i8 undef, i32 8
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v64i8_15 = insertelement <64 x i8> undef, i8 undef, i32 15
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v64i8_24 = insertelement <64 x i8> undef, i8 undef, i32 24
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v64i8_31 = insertelement <64 x i8> undef, i8 undef, i32 31
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v64i8_32 = insertelement <64 x i8> undef, i8 undef, i32 32
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v64i8_48 = insertelement <64 x i8> undef, i8 undef, i32 48
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %v64i8_63 = insertelement <64 x i8> undef, i8 undef, i32 63
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v64i8_63 = insertelement <64 x i8> undef, i8 undef, i32 63
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; SSE3-LABEL: 'insert_i8'
@@ -707,25 +707,25 @@ define i32 @insert_i8(i32 %arg) {
 ; SSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v8i8_7 = insertelement <8 x i8> undef, i8 undef, i32 7
 ; SSE3-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v16i8_a = insertelement <16 x i8> undef, i8 undef, i32 %arg
 ; SSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v16i8_0 = insertelement <16 x i8> undef, i8 undef, i32 0
-; SSE3-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %v16i8_8 = insertelement <16 x i8> undef, i8 undef, i32 8
-; SSE3-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %v16i8_15 = insertelement <16 x i8> undef, i8 undef, i32 15
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v16i8_8 = insertelement <16 x i8> undef, i8 undef, i32 8
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v16i8_15 = insertelement <16 x i8> undef, i8 undef, i32 15
 ; SSE3-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %v32i8_a = insertelement <32 x i8> undef, i8 undef, i32 %arg
 ; SSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v32i8_0 = insertelement <32 x i8> undef, i8 undef, i32 0
-; SSE3-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %v32i8_7 = insertelement <32 x i8> undef, i8 undef, i32 7
-; SSE3-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %v32i8_8 = insertelement <32 x i8> undef, i8 undef, i32 8
-; SSE3-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %v32i8_15 = insertelement <32 x i8> undef, i8 undef, i32 15
-; SSE3-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %v32i8_24 = insertelement <32 x i8> undef, i8 undef, i32 24
-; SSE3-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %v32i8_31 = insertelement <32 x i8> undef, i8 undef, i32 31
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v32i8_7 = insertelement <32 x i8> undef, i8 undef, i32 7
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v32i8_8 = insertelement <32 x i8> undef, i8 undef, i32 8
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v32i8_15 = insertelement <32 x i8> undef, i8 undef, i32 15
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v32i8_24 = insertelement <32 x i8> undef, i8 undef, i32 24
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v32i8_31 = insertelement <32 x i8> undef, i8 undef, i32 31
 ; SSE3-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %v64i8_a = insertelement <64 x i8> undef, i8 undef, i32 %arg
 ; SSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v64i8_0 = insertelement <64 x i8> undef, i8 undef, i32 0
-; SSE3-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %v64i8_7 = insertelement <64 x i8> undef, i8 undef, i32 7
-; SSE3-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %v64i8_8 = insertelement <64 x i8> undef, i8 undef, i32 8
-; SSE3-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %v64i8_15 = insertelement <64 x i8> undef, i8 undef, i32 15
-; SSE3-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %v64i8_24 = insertelement <64 x i8> undef, i8 undef, i32 24
-; SSE3-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %v64i8_31 = insertelement <64 x i8> undef, i8 undef, i32 31
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v64i8_7 = insertelement <64 x i8> undef, i8 undef, i32 7
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v64i8_8 = insertelement <64 x i8> undef, i8 undef, i32 8
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v64i8_15 = insertelement <64 x i8> undef, i8 undef, i32 15
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v64i8_24 = insertelement <64 x i8> undef, i8 undef, i32 24
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v64i8_31 = insertelement <64 x i8> undef, i8 undef, i32 31
 ; SSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v64i8_32 = insertelement <64 x i8> undef, i8 undef, i32 32
 ; SSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v64i8_48 = insertelement <64 x i8> undef, i8 undef, i32 48
-; SSE3-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %v64i8_63 = insertelement <64 x i8> undef, i8 undef, i32 63
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v64i8_63 = insertelement <64 x i8> undef, i8 undef, i32 63
 ; SSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; SSSE3-LABEL: 'insert_i8'
@@ -978,25 +978,25 @@ define i32 @insert_i1(i32 %arg) {
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i1_4 = insertelement <8 x i1> undef, i1 undef, i32 4
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %v16i1_a = insertelement <16 x i1> undef, i1 undef, i32 %arg
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v16i1_0 = insertelement <16 x i1> undef, i1 undef, i32 0
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %v16i1_8 = insertelement <16 x i1> undef, i1 undef, i32 8
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %v16i1_15 = insertelement <16 x i1> undef, i1 undef, i32 15
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v16i1_8 = insertelement <16 x i1> undef, i1 undef, i32 8
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v16i1_15 = insertelement <16 x i1> undef, i1 undef, i32 15
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v32i1_a = insertelement <32 x i1> undef, i1 undef, i32 %arg
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v32i1_0 = insertelement <32 x i1> undef, i1 undef, i32 0
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %v32i1_7 = insertelement <32 x i1> undef, i1 undef, i32 7
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %v32i1_8 = insertelement <32 x i1> undef, i1 undef, i32 8
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %v32i1_15 = insertelement <32 x i1> undef, i1 undef, i32 15
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %v32i1_24 = insertelement <32 x i1> undef, i1 undef, i32 24
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %v32i1_31 = insertelement <32 x i1> undef, i1 undef, i32 31
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v32i1_7 = insertelement <32 x i1> undef, i1 undef, i32 7
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v32i1_8 = insertelement <32 x i1> undef, i1 undef, i32 8
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v32i1_15 = insertelement <32 x i1> undef, i1 undef, i32 15
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v32i1_24 = insertelement <32 x i1> undef, i1 undef, i32 24
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v32i1_31 = insertelement <32 x i1> undef, i1 undef, i32 31
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v64i1_a = insertelement <64 x i1> undef, i1 undef, i32 %arg
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v64i1_0 = insertelement <64 x i1> undef, i1 undef, i32 0
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %v64i1_7 = insertelement <64 x i1> undef, i1 undef, i32 7
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %v64i1_8 = insertelement <64 x i1> undef, i1 undef, i32 8
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %v64i1_15 = insertelement <64 x i1> undef, i1 undef, i32 15
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %v64i1_24 = insertelement <64 x i1> undef, i1 undef, i32 24
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %v64i1_31 = insertelement <64 x i1> undef, i1 undef, i32 31
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v64i1_7 = insertelement <64 x i1> undef, i1 undef, i32 7
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v64i1_8 = insertelement <64 x i1> undef, i1 undef, i32 8
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v64i1_15 = insertelement <64 x i1> undef, i1 undef, i32 15
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v64i1_24 = insertelement <64 x i1> undef, i1 undef, i32 24
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v64i1_31 = insertelement <64 x i1> undef, i1 undef, i32 31
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v64i1_32 = insertelement <64 x i1> undef, i1 undef, i32 32
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v64i1_48 = insertelement <64 x i1> undef, i1 undef, i32 48
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %v64i1_63 = insertelement <64 x i1> undef, i1 undef, i32 63
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v64i1_63 = insertelement <64 x i1> undef, i1 undef, i32 63
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; SSE3-LABEL: 'insert_i1'
@@ -1011,25 +1011,25 @@ define i32 @insert_i1(i32 %arg) {
 ; SSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i1_4 = insertelement <8 x i1> undef, i1 undef, i32 4
 ; SSE3-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %v16i1_a = insertelement <16 x i1> undef, i1 undef, i32 %arg
 ; SSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v16i1_0 = insertelement <16 x i1> undef, i1 undef, i32 0
-; SSE3-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %v16i1_8 = insertelement <16 x i1> undef, i1 undef, i32 8
-; SSE3-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %v16i1_15 = insertelement <16 x i1> undef, i1 undef, i32 15
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v16i1_8 = insertelement <16 x i1> undef, i1 undef, i32 8
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v16i1_15 = insertelement <16 x i1> undef, i1 undef, i32 15
 ; SSE3-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v32i1_a = insertelement <32 x i1> undef, i1 undef, i32 %arg
 ; SSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v32i1_0 = insertelement <32 x i1> undef, i1 undef, i32 0
-; SSE3-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %v32i1_7 = insertelement <32 x i1> undef, i1 undef, i32 7
-; SSE3-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %v32i1_8 = insertelement <32 x i1> undef, i1 undef, i32 8
-; SSE3-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %v32i1_15 = insertelement <32 x i1> undef, i1 undef, i32 15
-; SSE3-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %v32i1_24 = insertelement <32 x i1> undef, i1 undef, i32 24
-; SSE3-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %v32i1_31 = insertelement <32 x i1> undef, i1 undef, i32 31
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v32i1_7 = insertelement <32 x i1> undef, i1 undef, i32 7
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v32i1_8 = insertelement <32 x i1> undef, i1 undef, i32 8
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v32i1_15 = insertelement <32 x i1> undef, i1 undef, i32 15
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v32i1_24 = insertelement <32 x i1> undef, i1 undef, i32 24
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v32i1_31 = insertelement <32 x i1> undef, i1 undef, i32 31
 ; SSE3-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v64i1_a = insertelement <64 x i1> undef, i1 undef, i32 %arg
 ; SSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v64i1_0 = insertelement <64 x i1> undef, i1 undef, i32 0
-; SSE3-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %v64i1_7 = insertelement <64 x i1> undef, i1 undef, i32 7
-; SSE3-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %v64i1_8 = insertelement <64 x i1> undef, i1 undef, i32 8
-; SSE3-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %v64i1_15 = insertelement <64 x i1> undef, i1 undef, i32 15
-; SSE3-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %v64i1_24 = insertelement <64 x i1> undef, i1 undef, i32 24
-; SSE3-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %v64i1_31 = insertelement <64 x i1> undef, i1 undef, i32 31
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v64i1_7 = insertelement <64 x i1> undef, i1 undef, i32 7
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v64i1_8 = insertelement <64 x i1> undef, i1 undef, i32 8
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v64i1_15 = insertelement <64 x i1> undef, i1 undef, i32 15
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v64i1_24 = insertelement <64 x i1> undef, i1 undef, i32 24
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v64i1_31 = insertelement <64 x i1> undef, i1 undef, i32 31
 ; SSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v64i1_32 = insertelement <64 x i1> undef, i1 undef, i32 32
 ; SSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v64i1_48 = insertelement <64 x i1> undef, i1 undef, i32 48
-; SSE3-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %v64i1_63 = insertelement <64 x i1> undef, i1 undef, i32 63
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v64i1_63 = insertelement <64 x i1> undef, i1 undef, i32 63
 ; SSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; SSSE3-LABEL: 'insert_i1'
diff --git a/llvm/test/Analysis/DependenceAnalysis/AA.ll b/llvm/test/Analysis/DependenceAnalysis/AA.ll
index a478143720d04..173744a07ef96 100644
--- a/llvm/test/Analysis/DependenceAnalysis/AA.ll
+++ b/llvm/test/Analysis/DependenceAnalysis/AA.ll
@@ -1,99 +1,139 @@
+; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --version 5
 ; RUN: opt < %s -disable-output "-passes=print<da>"                            \
 ; RUN: "-aa-pipeline=basic-aa,tbaa" 2>&1 | FileCheck %s
 
-; CHECK-LABEL: 'Dependence Analysis' for function 'test_no_noalias'
-; CHECK: da analyze - none!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - none!
 define void @test_no_noalias(ptr %A, ptr %B) {
+; CHECK-LABEL: 'test_no_noalias'
+; CHECK-NEXT:  Src: store i32 1, ptr %A, align 4 --> Dst: store i32 1, ptr %A, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: store i32 1, ptr %A, align 4 --> Dst: store i32 2, ptr %B, align 4
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: store i32 2, ptr %B, align 4 --> Dst: store i32 2, ptr %B, align 4
+; CHECK-NEXT:    da analyze - none!
+;
   store i32 1, ptr %A
   store i32 2, ptr %B
   ret void
 }
 
-; CHECK-LABEL: test_one_noalias
-; CHECK: da analyze - none!
-; CHECK: da analyze - none!
-; CHECK: da analyze - none!
 define void @test_one_noalias(ptr noalias %A, ptr %B) {
+; CHECK-LABEL: 'test_one_noalias'
+; CHECK-NEXT:  Src: store i32 1, ptr %A, align 4 --> Dst: store i32 1, ptr %A, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: store i32 1, ptr %A, align 4 --> Dst: store i32 2, ptr %B, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: store i32 2, ptr %B, align 4 --> Dst: store i32 2, ptr %B, align 4
+; CHECK-NEXT:    da analyze - none!
+;
   store i32 1, ptr %A
   store i32 2, ptr %B
   ret void
 }
 
-; CHECK-LABEL: test_two_noalias
-; CHECK: da analyze - none!
-; CHECK: da analyze - none!
-; CHECK: da analyze - none!
 define void @test_two_noalias(ptr noalias %A, ptr noalias %B) {
+; CHECK-LABEL: 'test_two_noalias'
+; CHECK-NEXT:  Src: store i32 1, ptr %A, align 4 --> Dst: store i32 1, ptr %A, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: store i32 1, ptr %A, align 4 --> Dst: store i32 2, ptr %B, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: store i32 2, ptr %B, align 4 --> Dst: store i32 2, ptr %B, align 4
+; CHECK-NEXT:    da analyze - none!
+;
   store i32 1, ptr %A
   store i32 2, ptr %B
   ret void
 }
 
-; CHECK-LABEL: test_global_alias
-; CHECK: da analyze - none!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - none!
 @g = global i32 5
 define void @test_global_alias(ptr %A) {
+; CHECK-LABEL: 'test_global_alias'
+; CHECK-NEXT:  Src: store i32 1, ptr %A, align 4 --> Dst: store i32 1, ptr %A, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: store i32 1, ptr %A, align 4 --> Dst: store i32 2, ptr @g, align 4
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: store i32 2, ptr @g, align 4 --> Dst: store i32 2, ptr @g, align 4
+; CHECK-NEXT:    da analyze - none!
+;
   store i32 1, ptr %A
   store i32 2, ptr @g
   ret void
 }
 
-; CHECK-LABEL: test_global_noalias
-; CHECK: da analyze - none!
-; CHECK: da analyze - none!
-; CHECK: da analyze - none!
 define void @test_global_noalias(ptr noalias %A) {
+; CHECK-LABEL: 'test_global_noalias'
+; CHECK-NEXT:  Src: store i32 1, ptr %A, align 4 --> Dst: store i32 1, ptr %A, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: store i32 1, ptr %A, align 4 --> Dst: store i32 2, ptr @g, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: store i32 2, ptr @g, align 4 --> Dst: store i32 2, ptr @g, align 4
+; CHECK-NEXT:    da analyze - none!
+;
   store i32 1, ptr %A
   store i32 2, ptr @g
   ret void
 }
 
-; CHECK-LABEL: test_global_size
-; CHECK: da analyze - none!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - none!
-; CHECK: da analyze - none!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - none!
 
 @a = global i16 5, align 2
 @b = global ptr @a, align 4
 define void @test_global_size() {
+; CHECK-LABEL: 'test_global_size'
+; CHECK-NEXT:  Src: %l0 = load ptr, ptr @b, align 4 --> Dst: %l0 = load ptr, ptr @b, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: %l0 = load ptr, ptr @b, align 4 --> Dst: %l1 = load i16, ptr %l0, align 2
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: %l0 = load ptr, ptr @b, align 4 --> Dst: store i16 1, ptr @a, align 2
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: %l1 = load i16, ptr %l0, align 2 --> Dst: %l1 = load i16, ptr %l0, align 2
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: %l1 = load i16, ptr %l0, align 2 --> Dst: store i16 1, ptr @a, align 2
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: store i16 1, ptr @a, align 2 --> Dst: store i16 1, ptr @a, align 2
+; CHECK-NEXT:    da analyze - none!
+;
   %l0 = load ptr, ptr @b, align 4
   %l1 = load i16, ptr %l0, align 2
   store i16 1, ptr @a, align 2
   ret void
 }
 
-; CHECK-LABEL: test_tbaa_same
-; CHECK: da analyze - none!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - none!
 define void @test_tbaa_same(ptr %A, ptr %B) {
+; CHECK-LABEL: 'test_tbaa_same'
+; CHECK-NEXT:  Src: store i32 1, ptr %A, align 4, !tbaa !0 --> Dst: store i32 1, ptr %A, align 4, !tbaa !0
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: store i32 1, ptr %A, align 4, !tbaa !0 --> Dst: store i32 2, ptr %B, align 4, !tbaa !0
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: store i32 2, ptr %B, align 4, !tbaa !0 --> Dst: store i32 2, ptr %B, align 4, !tbaa !0
+; CHECK-NEXT:    da analyze - none!
+;
   store i32 1, ptr %A, !tbaa !5
   store i32 2, ptr %B, !tbaa !5
   ret void
 }
 
-; CHECK-LABEL: test_tbaa_diff
-; CHECK: da analyze - none!
-; CHECK: da analyze - none!
-; CHECK: da analyze - none!
 define void @test_tbaa_diff(ptr %A, ptr %B) {
+; CHECK-LABEL: 'test_tbaa_diff'
+; CHECK-NEXT:  Src: store i32 1, ptr %A, align 4, !tbaa !0 --> Dst: store i32 1, ptr %A, align 4, !tbaa !0
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: store i32 1, ptr %A, align 4, !tbaa !0 --> Dst: store i16 2, ptr %B, align 2, !tbaa !4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: store i16 2, ptr %B, align 2, !tbaa !4 --> Dst: store i16 2, ptr %B, align 2, !tbaa !4
+; CHECK-NEXT:    da analyze - none!
+;
   store i32 1, ptr %A, !tbaa !5
   store i16 2, ptr %B, !tbaa !9
   ret void
 }
 
-; CHECK-LABEL: tbaa_loop
-; CHECK: da analyze - input
-; CHECK: da analyze - none
-; CHECK: da analyze - output
 define void @tbaa_loop(i32 %I, i32 %J, ptr nocapture %A, ptr nocapture readonly %B) {
+; CHECK-LABEL: 'tbaa_loop'
+; CHECK-NEXT:  Src: %0 = load i16, ptr %arrayidx.us, align 4, !tbaa !0 --> Dst: %0 = load i16, ptr %arrayidx.us, align 4, !tbaa !0
+; CHECK-NEXT:    da analyze - input [* *]!
+; CHECK-NEXT:  Src: %0 = load i16, ptr %arrayidx.us, align 4, !tbaa !0 --> Dst: store i32 %add.us.lcssa, ptr %arrayidx6.us, align 4, !tbaa !4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: store i32 %add.us.lcssa, ptr %arrayidx6.us, align 4, !tbaa !4 --> Dst: store i32 %add.us.lcssa, ptr %arrayidx6.us, align 4, !tbaa !4
+; CHECK-NEXT:    da analyze - output [*]!
+;
 entry:
   %cmp = icmp ne i32 %J, 0
   %cmp122 = icmp ne i32 %I, 0
diff --git a/llvm/test/Analysis/DependenceAnalysis/Banerjee.ll b/llvm/test/Analysis/DependenceAnalysis/Banerjee.ll
index efc86d39b28ee..6768e9067dca3 100644
--- a/llvm/test/Analysis/DependenceAnalysis/Banerjee.ll
+++ b/llvm/test/Analysis/DependenceAnalysis/Banerjee.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --version 5
 ; RUN: opt < %s -disable-output -da-delinearize=false "-passes=print<da>"      \
 ; RUN: -aa-pipeline=basic-aa 2>&1 | FileCheck %s
 ; RUN: opt < %s -disable-output -da-delinearize=false -passes='print<da><normalized-results>'      \
@@ -15,31 +16,50 @@ target triple = "x86_64-apple-macosx10.6.0"
 ;;      *B++ = A[10*i + j - 1];
 
 define void @banerjee0(ptr %A, ptr %B, i64 %m, i64 %n) nounwind uwtable ssp {
+; CHECK-LABEL: 'banerjee0'
+; CHECK-NEXT:  Src: store i64 0, ptr %arrayidx, align 8 --> Dst: store i64 0, ptr %arrayidx, align 8
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: store i64 0, ptr %arrayidx, align 8 --> Dst: %0 = load i64, ptr %arrayidx6, align 8
+; CHECK-NEXT:    da analyze - flow [<= <>]!
+; CHECK-NEXT:  Src: store i64 0, ptr %arrayidx, align 8 --> Dst: store i64 %0, ptr %B.addr.11, align 8
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: %0 = load i64, ptr %arrayidx6, align 8 --> Dst: %0 = load i64, ptr %arrayidx6, align 8
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: %0 = load i64, ptr %arrayidx6, align 8 --> Dst: store i64 %0, ptr %B.addr.11, align 8
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: store i64 %0, ptr %B.addr.11, align 8 --> Dst: store i64 %0, ptr %B.addr.11, align 8
+; CHECK-NEXT:    da analyze - none!
+;
+; NORMALIZE-LABEL: 'banerjee0'
+; NORMALIZE-NEXT:  Src: store i64 0, ptr %arrayidx, align 8 --> Dst: store i64 0, ptr %arrayidx, align 8
+; NORMALIZE-NEXT:    da analyze - none!
+; NORMALIZE-NEXT:  Src: store i64 0, ptr %arrayidx, align 8 --> Dst: %0 = load i64, ptr %arrayidx6, align 8
+; NORMALIZE-NEXT:    da analyze - flow [<= <>]!
+; NORMALIZE-NEXT:  Src: store i64 0, ptr %arrayidx, align 8 --> Dst: store i64 %0, ptr %B.addr.11, align 8
+; NORMALIZE-NEXT:    da analyze - confused!
+; NORMALIZE-NEXT:  Src: %0 = load i64, ptr %arrayidx6, align 8 --> Dst: %0 = load i64, ptr %arrayidx6, align 8
+; NORMALIZE-NEXT:    da analyze - none!
+; NORMALIZE-NEXT:  Src: %0 = load i64, ptr %arrayidx6, align 8 --> Dst: store i64 %0, ptr %B.addr.11, align 8
+; NORMALIZE-NEXT:    da analyze - confused!
+; NORMALIZE-NEXT:  Src: store i64 %0, ptr %B.addr.11, align 8 --> Dst: store i64 %0, ptr %B.addr.11, align 8
+; NORMALIZE-NEXT:    da analyze - none!
+;
+; DELIN-LABEL: 'banerjee0'
+; DELIN-NEXT:  Src: store i64 0, ptr %arrayidx, align 8 --> Dst: store i64 0, ptr %arrayidx, align 8
+; DELIN-NEXT:    da analyze - none!
+; DELIN-NEXT:  Src: store i64 0, ptr %arrayidx, align 8 --> Dst: %0 = load i64, ptr %arrayidx6, align 8
+; DELIN-NEXT:    da analyze - flow [<= <>]!
+; DELIN-NEXT:  Src: store i64 0, ptr %arrayidx, align 8 --> Dst: store i64 %0, ptr %B.addr.11, align 8
+; DELIN-NEXT:    da analyze - confused!
+; DELIN-NEXT:  Src: %0 = load i64, ptr %arrayidx6, align 8 --> Dst: %0 = load i64, ptr %arrayidx6, align 8
+; DELIN-NEXT:    da analyze - none!
+; DELIN-NEXT:  Src: %0 = load i64, ptr %arrayidx6, align 8 --> Dst: store i64 %0, ptr %B.addr.11, align 8
+; DELIN-NEXT:    da analyze - confused!
+; DELIN-NEXT:  Src: store i64 %0, ptr %B.addr.11, align 8 --> Dst: store i64 %0, ptr %B.addr.11, align 8
+; DELIN-NEXT:    da analyze - none!
+;
 entry:
   br label %for.cond1.preheader
-; CHECK: 'Dependence Analysis' for function 'banerjee0':
-; CHECK: da analyze - none!
-; CHECK: da analyze - flow [<= <>]!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - none!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - none!
-
-; NORMALIZE: 'Dependence Analysis' for function 'banerjee0':
-; NORMALIZE: da analyze - none!
-; NORMALIZE: da analyze - flow [<= <>]!
-; NORMALIZE: da analyze - confused!
-; NORMALIZE: da analyze - none!
-; NORMALIZE: da analyze - confused!
-; NORMALIZE: da analyze - none!
-
-; DELIN: 'Dependence Analysis' for function 'banerjee0':
-; DELIN: da analyze - none!
-; DELIN: da analyze - flow [<= <>]!
-; DELIN: da analyze - confused!
-; DELIN: da analyze - none!
-; DELIN: da analyze - confused!
-; DELIN: da analyze - none!
 
 for.cond1.preheader:                              ; preds = %entry, %for.inc7
   %B.addr.04 = phi ptr [ %B, %entry ], [ %scevgep, %for.inc7 ]
@@ -81,34 +101,52 @@ for.end9:                                         ; preds = %for.inc7
 ;;      *B++ = A[10*i + j - 1];
 
 define void @banerjee1(ptr %A, ptr %B, i64 %m, i64 %n) nounwind uwtable ssp {
+; CHECK-LABEL: 'banerjee1'
+; CHECK-NEXT:  Src: store i64 0, ptr %arrayidx, align 8 --> Dst: store i64 0, ptr %arrayidx, align 8
+; CHECK-NEXT:    da analyze - output [* *]!
+; CHECK-NEXT:  Src: store i64 0, ptr %arrayidx, align 8 --> Dst: %2 = load i64, ptr %arrayidx6, align 8
+; CHECK-NEXT:    da analyze - flow [* <>]!
+; CHECK-NEXT:  Src: store i64 0, ptr %arrayidx, align 8 --> Dst: store i64 %2, ptr %B.addr.12, align 8
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: %2 = load i64, ptr %arrayidx6, align 8 --> Dst: %2 = load i64, ptr %arrayidx6, align 8
+; CHECK-NEXT:    da analyze - input [* *]!
+; CHECK-NEXT:  Src: %2 = load i64, ptr %arrayidx6, align 8 --> Dst: store i64 %2, ptr %B.addr.12, align 8
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: store i64 %2, ptr %B.addr.12, align 8 --> Dst: store i64 %2, ptr %B.addr.12, align 8
+; CHECK-NEXT:    da analyze - output [* *]!
+;
+; NORMALIZE-LABEL: 'banerjee1'
+; NORMALIZE-NEXT:  Src: store i64 0, ptr %arrayidx, align 8 --> Dst: store i64 0, ptr %arrayidx, align 8
+; NORMALIZE-NEXT:    da analyze - output [* *]!
+; NORMALIZE-NEXT:  Src: store i64 0, ptr %arrayidx, align 8 --> Dst: %2 = load i64, ptr %arrayidx6, align 8
+; NORMALIZE-NEXT:    da analyze - flow [* <>]!
+; NORMALIZE-NEXT:  Src: store i64 0, ptr %arrayidx, align 8 --> Dst: store i64 %2, ptr %B.addr.12, align 8
+; NORMALIZE-NEXT:    da analyze - confused!
+; NORMALIZE-NEXT:  Src: %2 = load i64, ptr %arrayidx6, align 8 --> Dst: %2 = load i64, ptr %arrayidx6, align 8
+; NORMALIZE-NEXT:    da analyze - input [* *]!
+; NORMALIZE-NEXT:  Src: %2 = load i64, ptr %arrayidx6, align 8 --> Dst: store i64 %2, ptr %B.addr.12, align 8
+; NORMALIZE-NEXT:    da analyze - confused!
+; NORMALIZE-NEXT:  Src: store i64 %2, ptr %B.addr.12, align 8 --> Dst: store i64 %2, ptr %B.addr.12, align 8
+; NORMALIZE-NEXT:    da analyze - output [* *]!
+;
+; DELIN-LABEL: 'banerjee1'
+; DELIN-NEXT:  Src: store i64 0, ptr %arrayidx, align 8 --> Dst: store i64 0, ptr %arrayidx, align 8
+; DELIN-NEXT:    da analyze - output [* *]!
+; DELIN-NEXT:  Src: store i64 0, ptr %arrayidx, align 8 --> Dst: %2 = load i64, ptr %arrayidx6, align 8
+; DELIN-NEXT:    da analyze - flow [* <>]!
+; DELIN-NEXT:  Src: store i64 0, ptr %arrayidx, align 8 --> Dst: store i64 %2, ptr %B.addr.12, align 8
+; DELIN-NEXT:    da analyze - confused!
+; DELIN-NEXT:  Src: %2 = load i64, ptr %arrayidx6, align 8 --> Dst: %2 = load i64, ptr %arrayidx6, align 8
+; DELIN-NEXT:    da analyze - input [* *]!
+; DELIN-NEXT:  Src: %2 = load i64, ptr %arrayidx6, align 8 --> Dst: store i64 %2, ptr %B.addr.12, align 8
+; DELIN-NEXT:    da analyze - confused!
+; DELIN-NEXT:  Src: store i64 %2, ptr %B.addr.12, align 8 --> Dst: store i64 %2, ptr %B.addr.12, align 8
+; DELIN-NEXT:    da analyze - output [* *]!
+;
 entry:
   %cmp4 = icmp sgt i64 %n, 0
   br i1 %cmp4, label %for.cond1.preheader.preheader, label %for.end9
 
-; CHECK: 'Dependence Analysis' for function 'banerjee1':
-; CHECK: da analyze - output [* *]!
-; CHECK: da analyze - flow [* <>]!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - input [* *]!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - output [* *]!
-
-; NORMALIZE: 'Dependence Analysis' for function 'banerjee1':
-; NORMALIZE: da analyze - output [* *]!
-; NORMALIZE: da analyze - flow [* <>]!
-; NORMALIZE: da analyze - confused!
-; NORMALIZE: da analyze - input [* *]!
-; NORMALIZE: da analyze - confused!
-; NORMALIZE: da analyze - output [* *]!
-
-; DELIN: 'Dependence Analysis' for function 'banerjee1':
-; DELIN: da analyze - output [* *]!
-; DELIN: da analyze - flow [* <>]!
-; DELIN: da analyze - confused!
-; DELIN: da analyze - input [* *]!
-; DELIN: da analyze - confused!
-; DELIN: da analyze - output [* *]!
-
 for.cond1.preheader.preheader:                    ; preds = %entry
   %0 = add i64 %n, 1
   br label %for.cond1.preheader
@@ -165,33 +203,51 @@ for.end9:                                         ; preds = %for.end9.loopexit,
 ;;      *B++ = A[10*i + j + 100];
 
 define void @banerjee2(ptr %A, ptr %B, i64 %m, i64 %n) nounwind uwtable ssp {
+; CHECK-LABEL: 'banerjee2'
+; CHECK-NEXT:  Src: store i64 0, ptr %arrayidx, align 8 --> Dst: store i64 0, ptr %arrayidx, align 8
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: store i64 0, ptr %arrayidx, align 8 --> Dst: %0 = load i64, ptr %arrayidx7, align 8
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: store i64 0, ptr %arrayidx, align 8 --> Dst: store i64 %0, ptr %B.addr.11, align 8
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: %0 = load i64, ptr %arrayidx7, align 8 --> Dst: %0 = load i64, ptr %arrayidx7, align 8
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: %0 = load i64, ptr %arrayidx7, align 8 --> Dst: store i64 %0, ptr %B.addr.11, align 8
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: store i64 %0, ptr %B.addr.11, align 8 --> Dst: store i64 %0, ptr %B.addr.11, align 8
+; CHECK-NEXT:    da analyze - none!
+;
+; NORMALIZE-LABEL: 'banerjee2'
+; NORMALIZE-NEXT:  Src: store i64 0, ptr %arrayidx, align 8 --> Dst: store i64 0, ptr %arrayidx, align 8
+; NORMALIZE-NEXT:    da analyze - none!
+; NORMALIZE-NEXT:  Src: store i64 0, ptr %arrayidx, align 8 --> Dst: %0 = load i64, ptr %arrayidx7, align 8
+; NORMALIZE-NEXT:    da analyze - none!
+; NORMALIZE-NEXT:  Src: store i64 0, ptr %arrayidx, align 8 --> Dst: store i64 %0, ptr %B.addr.11, align 8
+; NORMALIZE-NEXT:    da analyze - confused!
+; NORMALIZE-NEXT:  Src: %0 = load i64, ptr %arrayidx7, align 8 --> Dst: %0 = load i64, ptr %arrayidx7, align 8
+; NORMALIZE-NEXT:    da analyze - none!
+; NORMALIZE-NEXT:  Src: %0 = load i64, ptr %arrayidx7, align 8 --> Dst: store i64 %0, ptr %B.addr.11, align 8
+; NORMALIZE-NEXT:    da analyze - confused!
+; NORMALIZE-NEXT:  Src: store i64 %0, ptr %B.addr.11, align 8 --> Dst: store i64 %0, ptr %B.addr.11, align 8
+; NORMALIZE-NEXT:    da analyze - none!
+;
+; DELIN-LABEL: 'banerjee2'
+; DELIN-NEXT:  Src: store i64 0, ptr %arrayidx, align 8 --> Dst: store i64 0, ptr %arrayidx, align 8
+; DELIN-NEXT:    da analyze - none!
+; DELIN-NEXT:  Src: store i64 0, ptr %arrayidx, align 8 --> Dst: %0 = load i64, ptr %arrayidx7, align 8
+; DELIN-NEXT:    da analyze - none!
+; DELIN-NEXT:  Src: store i64 0, ptr %arrayidx, align 8 --> Dst: store i64 %0, ptr %B.addr.11, align 8
+; DELIN-NEXT:    da analyze - confused!
+; DELIN-NEXT:  Src: %0 = load i64, ptr %arrayidx7, align 8 --> Dst: %0 = load i64, ptr %arrayidx7, align 8
+; DELIN-NEXT:    da analyze - none!
+; DELIN-NEXT:  Src: %0 = load i64, ptr %arrayidx7, align 8 --> Dst: store i64 %0, ptr %B.addr.11, align 8
+; DELIN-NEXT:    da analyze - confused!
+; DELIN-NEXT:  Src: store i64 %0, ptr %B.addr.11, align 8 --> Dst: store i64 %0, ptr %B.addr.11, align 8
+; DELIN-NEXT:    da analyze - none!
+;
 entry:
   br label %for.cond1.preheader
 
-; CHECK: 'Dependence Analysis' for function 'banerjee2':
-; CHECK: da analyze - none!
-; CHECK: da analyze - none!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - none!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - none!
-
-; NORMALIZE: 'Dependence Analysis' for function 'banerjee2':
-; NORMALIZE: da analyze - none!
-; NORMALIZE: da analyze - none!
-; NORMALIZE: da analyze - confused!
-; NORMALIZE: da analyze - none!
-; NORMALIZE: da analyze - confused!
-; NORMALIZE: da analyze - none!
-
-; DELIN: 'Dependence Analysis' for function 'banerjee2':
-; DELIN: da analyze - none!
-; DELIN: da analyze - none!
-; DELIN: da analyze - confused!
-; DELIN: da analyze - none!
-; DELIN: da analyze - confused!
-; DELIN: da analyze - none!
-
 for.cond1.preheader:                              ; preds = %entry, %for.inc8
   %B.addr.04 = phi ptr [ %B, %entry ], [ %scevgep, %for.inc8 ]
   %i.03 = phi i64 [ 0, %entry ], [ %inc9, %for.inc8 ]
@@ -232,33 +288,51 @@ for.end10:                                        ; preds = %for.inc8
 ;;      *B++ = A[10*i + j + 99];
 
 define void @banerjee3(ptr %A, ptr %B, i64 %m, i64 %n) nounwind uwtable ssp {
+; CHECK-LABEL: 'banerjee3'
+; CHECK-NEXT:  Src: store i64 0, ptr %arrayidx, align 8 --> Dst: store i64 0, ptr %arrayidx, align 8
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: store i64 0, ptr %arrayidx, align 8 --> Dst: %0 = load i64, ptr %arrayidx7, align 8
+; CHECK-NEXT:    da analyze - flow [> >]!
+; CHECK-NEXT:  Src: store i64 0, ptr %arrayidx, align 8 --> Dst: store i64 %0, ptr %B.addr.11, align 8
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: %0 = load i64, ptr %arrayidx7, align 8 --> Dst: %0 = load i64, ptr %arrayidx7, align 8
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: %0 = load i64, ptr %arrayidx7, align 8 --> Dst: store i64 %0, ptr %B.addr.11, align 8
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: store i64 %0, ptr %B.addr.11, align 8 --> Dst: store i64 %0, ptr %B.addr.11, align 8
+; CHECK-NEXT:    da analyze - none!
+;
+; NORMALIZE-LABEL: 'banerjee3'
+; NORMALIZE-NEXT:  Src: store i64 0, ptr %arrayidx, align 8 --> Dst: store i64 0, ptr %arrayidx, align 8
+; NORMALIZE-NEXT:    da analyze - none!
+; NORMALIZE-NEXT:  Src: store i64 0, ptr %arrayidx, align 8 --> Dst: %0 = load i64, ptr %arrayidx7, align 8
+; NORMALIZE-NEXT:    da analyze - normalized - anti [< <]!
+; NORMALIZE-NEXT:  Src: store i64 0, ptr %arrayidx, align 8 --> Dst: store i64 %0, ptr %B.addr.11, align 8
+; NORMALIZE-NEXT:    da analyze - confused!
+; NORMALIZE-NEXT:  Src: %0 = load i64, ptr %arrayidx7, align 8 --> Dst: %0 = load i64, ptr %arrayidx7, align 8
+; NORMALIZE-NEXT:    da analyze - none!
+; NORMALIZE-NEXT:  Src: %0 = load i64, ptr %arrayidx7, align 8 --> Dst: store i64 %0, ptr %B.addr.11, align 8
+; NORMALIZE-NEXT:    da analyze - confused!
+; NORMALIZE-NEXT:  Src: store i64 %0, ptr %B.addr.11, align 8 --> Dst: store i64 %0, ptr %B.addr.11, align 8
+; NORMALIZE-NEXT:    da analyze - none!
+;
+; DELIN-LABEL: 'banerjee3'
+; DELIN-NEXT:  Src: store i64 0, ptr %arrayidx, align 8 --> Dst: store i64 0, ptr %arrayidx, align 8
+; DELIN-NEXT:    da analyze - none!
+; DELIN-NEXT:  Src: store i64 0, ptr %arrayidx, align 8 --> Dst: %0 = load i64, ptr %arrayidx7, align 8
+; DELIN-NEXT:    da analyze - flow [> >]!
+; DELIN-NEXT:  Src: store i64 0, ptr %arrayidx, align 8 --> Dst: store i64 %0, ptr %B.addr.11, align 8
+; DELIN-NEXT:    da analyze - confused!
+; DELIN-NEXT:  Src: %0 = load i64, ptr %arrayidx7, align 8 --> Dst: %0 = load i64, ptr %arrayidx7, align 8
+; DELIN-NEXT:    da analyze - none!
+; DELIN-NEXT:  Src: %0 = load i64, ptr %arrayidx7, align 8 --> Dst: store i64 %0, ptr %B.addr.11, align 8
+; DELIN-NEXT:    da analyze - confused!
+; DELIN-NEXT:  Src: store i64 %0, ptr %B.addr.11, align 8 --> Dst: store i64 %0, ptr %B.addr.11, align 8
+; DELIN-NEXT:    da analyze - none!
+;
 entry:
   br label %for.cond1.preheader
 
-; CHECK: 'Dependence Analysis' for function 'banerjee3':
-; CHECK: da analyze - none!
-; CHECK: da analyze - flow [> >]!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - none!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - none!
-
-; NORMALIZE: 'Dependence Analysis' for function 'banerjee3':
-; NORMALIZE: da analyze - none!
-; NORMALIZE: da analyze - normalized - anti [< <]!
-; NORMALIZE: da analyze - confused!
-; NORMALIZE: da analyze - none!
-; NORMALIZE: da analyze - confused!
-; NORMALIZE: da analyze - none!
-
-; DELIN: 'Dependence Analysis' for function 'banerjee3':
-; DELIN: da analyze - none!
-; DELIN: da analyze - flow [> >]!
-; DELIN: da analyze - confused!
-; DELIN: da analyze - none!
-; DELIN: da analyze - confused!
-; DELIN: da analyze - none!
-
 for.cond1.preheader:                              ; preds = %entry, %for.inc8
   %B.addr.04 = phi ptr [ %B, %entry ], [ %scevgep, %for.inc8 ]
   %i.03 = phi i64 [ 0, %entry ], [ %inc9, %for.inc8 ]
@@ -299,33 +373,51 @@ for.end10:                                        ; preds = %for.inc8
 ;;      *B++ = A[10*i + j - 100];
 
 define void @banerjee4(ptr %A, ptr %B, i64 %m, i64 %n) nounwind uwtable ssp {
+; CHECK-LABEL: 'banerjee4'
+; CHECK-NEXT:  Src: store i64 0, ptr %arrayidx, align 8 --> Dst: store i64 0, ptr %arrayidx, align 8
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: store i64 0, ptr %arrayidx, align 8 --> Dst: %0 = load i64, ptr %arrayidx6, align 8
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: store i64 0, ptr %arrayidx, align 8 --> Dst: store i64 %0, ptr %B.addr.11, align 8
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: %0 = load i64, ptr %arrayidx6, align 8 --> Dst: %0 = load i64, ptr %arrayidx6, align 8
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: %0 = load i64, ptr %arrayidx6, align 8 --> Dst: store i64 %0, ptr %B.addr.11, align 8
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: store i64 %0, ptr %B.addr.11, align 8 --> Dst: store i64 %0, ptr %B.addr.11, align 8
+; CHECK-NEXT:    da analyze - none!
+;
+; NORMALIZE-LABEL: 'banerjee4'
+; NORMALIZE-NEXT:  Src: store i64 0, ptr %arrayidx, align 8 --> Dst: store i64 0, ptr %arrayidx, align 8
+; NORMALIZE-NEXT:    da analyze - none!
+; NORMALIZE-NEXT:  Src: store i64 0, ptr %arrayidx, align 8 --> Dst: %0 = load i64, ptr %arrayidx6, align 8
+; NORMALIZE-NEXT:    da analyze - none!
+; NORMALIZE-NEXT:  Src: store i64 0, ptr %arrayidx, align 8 --> Dst: store i64 %0, ptr %B.addr.11, align 8
+; NORMALIZE-NEXT:    da analyze - confused!
+; NORMALIZE-NEXT:  Src: %0 = load i64, ptr %arrayidx6, align 8 --> Dst: %0 = load i64, ptr %arrayidx6, align 8
+; NORMALIZE-NEXT:    da analyze - none!
+; NORMALIZE-NEXT:  Src: %0 = load i64, ptr %arrayidx6, align 8 --> Dst: store i64 %0, ptr %B.addr.11, align 8
+; NORMALIZE-NEXT:    da analyze - confused!
+; NORMALIZE-NEXT:  Src: store i64 %0, ptr %B.addr.11, align 8 --> Dst: store i64 %0, ptr %B.addr.11, align 8
+; NORMALIZE-NEXT:    da analyze - none!
+;
+; DELIN-LABEL: 'banerjee4'
+; DELIN-NEXT:  Src: store i64 0, ptr %arrayidx, align 8 --> Dst: store i64 0, ptr %arrayidx, align 8
+; DELIN-NEXT:    da analyze - none!
+; DELIN-NEXT:  Src: store i64 0, ptr %arrayidx, align 8 --> Dst: %0 = load i64, ptr %arrayidx6, align 8
+; DELIN-NEXT:    da analyze - none!
+; DELIN-NEXT:  Src: store i64 0, ptr %arrayidx, align 8 --> Dst: store i64 %0, ptr %B.addr.11, align 8
+; DELIN-NEXT:    da analyze - confused!
+; DELIN-NEXT:  Src: %0 = load i64, ptr %arrayidx6, align 8 --> Dst: %0 = load i64, ptr %arrayidx6, align 8
+; DELIN-NEXT:    da analyze - none!
+; DELIN-NEXT:  Src: %0 = load i64, ptr %arrayidx6, align 8 --> Dst: store i64 %0, ptr %B.addr.11, align 8
+; DELIN-NEXT:    da analyze - confused!
+; DELIN-NEXT:  Src: store i64 %0, ptr %B.addr.11, align 8 --> Dst: store i64 %0, ptr %B.addr.11, align 8
+; DELIN-NEXT:    da analyze - none!
+;
 entry:
   br label %for.cond1.preheader
 
-; CHECK: 'Dependence Analysis' for function 'banerjee4':
-; CHECK: da analyze - none!
-; CHECK: da analyze - none!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - none!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - none!
-
-; NORMALIZE: 'Dependence Analysis' for function 'banerjee4':
-; NORMALIZE: da analyze - none!
-; NORMALIZE: da analyze - none!
-; NORMALIZE: da analyze - confused!
-; NORMALIZE: da analyze - none!
-; NORMALIZE: da analyze - confused!
-; NORMALIZE: da analyze - none!
-
-; DELIN: 'Dependence Analysis' for function 'banerjee4':
-; DELIN: da analyze - none!
-; DELIN: da analyze - none!
-; DELIN: da analyze - confused!
-; DELIN: da analyze - none!
-; DELIN: da analyze - confused!
-; DELIN: da analyze - none!
-
 for.cond1.preheader:                              ; preds = %entry, %for.inc7
   %B.addr.04 = phi ptr [ %B, %entry ], [ %scevgep, %for.inc7 ]
   %i.03 = phi i64 [ 0, %entry ], [ %inc8, %for.inc7 ]
@@ -366,33 +458,51 @@ for.end9:                                         ; preds = %for.inc7
 ;;      *B++ = A[10*i + j - 99];
 
 define void @banerjee5(ptr %A, ptr %B, i64 %m, i64 %n) nounwind uwtable ssp {
+; CHECK-LABEL: 'banerjee5'
+; CHECK-NEXT:  Src: store i64 0, ptr %arrayidx, align 8 --> Dst: store i64 0, ptr %arrayidx, align 8
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: store i64 0, ptr %arrayidx, align 8 --> Dst: %0 = load i64, ptr %arrayidx6, align 8
+; CHECK-NEXT:    da analyze - flow [< <]!
+; CHECK-NEXT:  Src: store i64 0, ptr %arrayidx, align 8 --> Dst: store i64 %0, ptr %B.addr.11, align 8
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: %0 = load i64, ptr %arrayidx6, align 8 --> Dst: %0 = load i64, ptr %arrayidx6, align 8
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: %0 = load i64, ptr %arrayidx6, align 8 --> Dst: store i64 %0, ptr %B.addr.11, align 8
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: store i64 %0, ptr %B.addr.11, align 8 --> Dst: store i64 %0, ptr %B.addr.11, align 8
+; CHECK-NEXT:    da analyze - none!
+;
+; NORMALIZE-LABEL: 'banerjee5'
+; NORMALIZE-NEXT:  Src: store i64 0, ptr %arrayidx, align 8 --> Dst: store i64 0, ptr %arrayidx, align 8
+; NORMALIZE-NEXT:    da analyze - none!
+; NORMALIZE-NEXT:  Src: store i64 0, ptr %arrayidx, align 8 --> Dst: %0 = load i64, ptr %arrayidx6, align 8
+; NORMALIZE-NEXT:    da analyze - flow [< <]!
+; NORMALIZE-NEXT:  Src: store i64 0, ptr %arrayidx, align 8 --> Dst: store i64 %0, ptr %B.addr.11, align 8
+; NORMALIZE-NEXT:    da analyze - confused!
+; NORMALIZE-NEXT:  Src: %0 = load i64, ptr %arrayidx6, align 8 --> Dst: %0 = load i64, ptr %arrayidx6, align 8
+; NORMALIZE-NEXT:    da analyze - none!
+; NORMALIZE-NEXT:  Src: %0 = load i64, ptr %arrayidx6, align 8 --> Dst: store i64 %0, ptr %B.addr.11, align 8
+; NORMALIZE-NEXT:    da analyze - confused!
+; NORMALIZE-NEXT:  Src: store i64 %0, ptr %B.addr.11, align 8 --> Dst: store i64 %0, ptr %B.addr.11, align 8
+; NORMALIZE-NEXT:    da analyze - none!
+;
+; DELIN-LABEL: 'banerjee5'
+; DELIN-NEXT:  Src: store i64 0, ptr %arrayidx, align 8 --> Dst: store i64 0, ptr %arrayidx, align 8
+; DELIN-NEXT:    da analyze - none!
+; DELIN-NEXT:  Src: store i64 0, ptr %arrayidx, align 8 --> Dst: %0 = load i64, ptr %arrayidx6, align 8
+; DELIN-NEXT:    da analyze - flow [< <]!
+; DELIN-NEXT:  Src: store i64 0, ptr %arrayidx, align 8 --> Dst: store i64 %0, ptr %B.addr.11, align 8
+; DELIN-NEXT:    da analyze - confused!
+; DELIN-NEXT:  Src: %0 = load i64, ptr %arrayidx6, align 8 --> Dst: %0 = load i64, ptr %arrayidx6, align 8
+; DELIN-NEXT:    da analyze - none!
+; DELIN-NEXT:  Src: %0 = load i64, ptr %arrayidx6, align 8 --> Dst: store i64 %0, ptr %B.addr.11, align 8
+; DELIN-NEXT:    da analyze - confused!
+; DELIN-NEXT:  Src: store i64 %0, ptr %B.addr.11, align 8 --> Dst: store i64 %0, ptr %B.addr.11, align 8
+; DELIN-NEXT:    da analyze - none!
+;
 entry:
   br label %for.cond1.preheader
 
-; CHECK: 'Dependence Analysis' for function 'banerjee5':
-; CHECK: da analyze - none!
-; CHECK: da analyze - flow [< <]!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - none!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - none!
-
-; NORMALIZE: 'Dependence Analysis' for function 'banerjee5':
-; NORMALIZE: da analyze - none!
-; NORMALIZE: da analyze - flow [< <]!
-; NORMALIZE: da analyze - confused!
-; NORMALIZE: da analyze - none!
-; NORMALIZE: da analyze - confused!
-; NORMALIZE: da analyze - none!
-
-; DELIN: 'Dependence Analysis' for function 'banerjee5':
-; DELIN: da analyze - none!
-; DELIN: da analyze - flow [< <]!
-; DELIN: da analyze - confused!
-; DELIN: da analyze - none!
-; DELIN: da analyze - confused!
-; DELIN: da analyze - none!
-
 for.cond1.preheader:                              ; preds = %entry, %for.inc7
   %B.addr.04 = phi ptr [ %B, %entry ], [ %scevgep, %for.inc7 ]
   %i.03 = phi i64 [ 0, %entry ], [ %inc8, %for.inc7 ]
@@ -433,33 +543,51 @@ for.end9:                                         ; preds = %for.inc7
 ;;      *B++ = A[10*i + j + 9];
 
 define void @banerjee6(ptr %A, ptr %B, i64 %m, i64 %n) nounwind uwtable ssp {
+; CHECK-LABEL: 'banerjee6'
+; CHECK-NEXT:  Src: store i64 0, ptr %arrayidx, align 8 --> Dst: store i64 0, ptr %arrayidx, align 8
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: store i64 0, ptr %arrayidx, align 8 --> Dst: %0 = load i64, ptr %arrayidx7, align 8
+; CHECK-NEXT:    da analyze - flow [=> <>]!
+; CHECK-NEXT:  Src: store i64 0, ptr %arrayidx, align 8 --> Dst: store i64 %0, ptr %B.addr.11, align 8
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: %0 = load i64, ptr %arrayidx7, align 8 --> Dst: %0 = load i64, ptr %arrayidx7, align 8
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: %0 = load i64, ptr %arrayidx7, align 8 --> Dst: store i64 %0, ptr %B.addr.11, align 8
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: store i64 %0, ptr %B.addr.11, align 8 --> Dst: store i64 %0, ptr %B.addr.11, align 8
+; CHECK-NEXT:    da analyze - none!
+;
+; NORMALIZE-LABEL: 'banerjee6'
+; NORMALIZE-NEXT:  Src: store i64 0, ptr %arrayidx, align 8 --> Dst: store i64 0, ptr %arrayidx, align 8
+; NORMALIZE-NEXT:    da analyze - none!
+; NORMALIZE-NEXT:  Src: store i64 0, ptr %arrayidx, align 8 --> Dst: %0 = load i64, ptr %arrayidx7, align 8
+; NORMALIZE-NEXT:    da analyze - normalized - anti [<= <>]!
+; NORMALIZE-NEXT:  Src: store i64 0, ptr %arrayidx, align 8 --> Dst: store i64 %0, ptr %B.addr.11, align 8
+; NORMALIZE-NEXT:    da analyze - confused!
+; NORMALIZE-NEXT:  Src: %0 = load i64, ptr %arrayidx7, align 8 --> Dst: %0 = load i64, ptr %arrayidx7, align 8
+; NORMALIZE-NEXT:    da analyze - none!
+; NORMALIZE-NEXT:  Src: %0 = load i64, ptr %arrayidx7, align 8 --> Dst: store i64 %0, ptr %B.addr.11, align 8
+; NORMALIZE-NEXT:    da analyze - confused!
+; NORMALIZE-NEXT:  Src: store i64 %0, ptr %B.addr.11, align 8 --> Dst: store i64 %0, ptr %B.addr.11, align 8
+; NORMALIZE-NEXT:    da analyze - none!
+;
+; DELIN-LABEL: 'banerjee6'
+; DELIN-NEXT:  Src: store i64 0, ptr %arrayidx, align 8 --> Dst: store i64 0, ptr %arrayidx, align 8
+; DELIN-NEXT:    da analyze - none!
+; DELIN-NEXT:  Src: store i64 0, ptr %arrayidx, align 8 --> Dst: %0 = load i64, ptr %arrayidx7, align 8
+; DELIN-NEXT:    da analyze - flow [=> <>]!
+; DELIN-NEXT:  Src: store i64 0, ptr %arrayidx, align 8 --> Dst: store i64 %0, ptr %B.addr.11, align 8
+; DELIN-NEXT:    da analyze - confused!
+; DELIN-NEXT:  Src: %0 = load i64, ptr %arrayidx7, align 8 --> Dst: %0 = load i64, ptr %arrayidx7, align 8
+; DELIN-NEXT:    da analyze - none!
+; DELIN-NEXT:  Src: %0 = load i64, ptr %arrayidx7, align 8 --> Dst: store i64 %0, ptr %B.addr.11, align 8
+; DELIN-NEXT:    da analyze - confused!
+; DELIN-NEXT:  Src: store i64 %0, ptr %B.addr.11, align 8 --> Dst: store i64 %0, ptr %B.addr.11, align 8
+; DELIN-NEXT:    da analyze - none!
+;
 entry:
   br label %for.cond1.preheader
 
-; CHECK: 'Dependence Analysis' for function 'banerjee6':
-; CHECK: da analyze - none!
-; CHECK: da analyze - flow [=> <>]!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - none!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - none!
-
-; NORMALIZE: 'Dependence Analysis' for function 'banerjee6':
-; NORMALIZE: da analyze - none!
-; NORMALIZE: da analyze - normalized - anti [<= <>]!
-; NORMALIZE: da analyze - confused!
-; NORMALIZE: da analyze - none!
-; NORMALIZE: da analyze - confused!
-; NORMALIZE: da analyze - none!
-
-; DELIN: 'Dependence Analysis' for function 'banerjee6':
-; DELIN: da analyze - none!
-; DELIN: da analyze - flow [=> <>]!
-; DELIN: da analyze - confused!
-; DELIN: da analyze - none!
-; DELIN: da analyze - confused!
-; DELIN: da analyze - none!
-
 for.cond1.preheader:                              ; preds = %entry, %for.inc8
   %B.addr.04 = phi ptr [ %B, %entry ], [ %scevgep, %for.inc8 ]
   %i.03 = phi i64 [ 0, %entry ], [ %inc9, %for.inc8 ]
@@ -500,33 +628,51 @@ for.end10:                                        ; preds = %for.inc8
 ;;      *B++ = A[10*i + j + 10];
 
 define void @banerjee7(ptr %A, ptr %B, i64 %m, i64 %n) nounwind uwtable ssp {
+; CHECK-LABEL: 'banerjee7'
+; CHECK-NEXT:  Src: store i64 0, ptr %arrayidx, align 8 --> Dst: store i64 0, ptr %arrayidx, align 8
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: store i64 0, ptr %arrayidx, align 8 --> Dst: %0 = load i64, ptr %arrayidx7, align 8
+; CHECK-NEXT:    da analyze - flow [> <=]!
+; CHECK-NEXT:  Src: store i64 0, ptr %arrayidx, align 8 --> Dst: store i64 %0, ptr %B.addr.11, align 8
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: %0 = load i64, ptr %arrayidx7, align 8 --> Dst: %0 = load i64, ptr %arrayidx7, align 8
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: %0 = load i64, ptr %arrayidx7, align 8 --> Dst: store i64 %0, ptr %B.addr.11, align 8
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: store i64 %0, ptr %B.addr.11, align 8 --> Dst: store i64 %0, ptr %B.addr.11, align 8
+; CHECK-NEXT:    da analyze - none!
+;
+; NORMALIZE-LABEL: 'banerjee7'
+; NORMALIZE-NEXT:  Src: store i64 0, ptr %arrayidx, align 8 --> Dst: store i64 0, ptr %arrayidx, align 8
+; NORMALIZE-NEXT:    da analyze - none!
+; NORMALIZE-NEXT:  Src: store i64 0, ptr %arrayidx, align 8 --> Dst: %0 = load i64, ptr %arrayidx7, align 8
+; NORMALIZE-NEXT:    da analyze - normalized - anti [< =>]!
+; NORMALIZE-NEXT:  Src: store i64 0, ptr %arrayidx, align 8 --> Dst: store i64 %0, ptr %B.addr.11, align 8
+; NORMALIZE-NEXT:    da analyze - confused!
+; NORMALIZE-NEXT:  Src: %0 = load i64, ptr %arrayidx7, align 8 --> Dst: %0 = load i64, ptr %arrayidx7, align 8
+; NORMALIZE-NEXT:    da analyze - none!
+; NORMALIZE-NEXT:  Src: %0 = load i64, ptr %arrayidx7, align 8 --> Dst: store i64 %0, ptr %B.addr.11, align 8
+; NORMALIZE-NEXT:    da analyze - confused!
+; NORMALIZE-NEXT:  Src: store i64 %0, ptr %B.addr.11, align 8 --> Dst: store i64 %0, ptr %B.addr.11, align 8
+; NORMALIZE-NEXT:    da analyze - none!
+;
+; DELIN-LABEL: 'banerjee7'
+; DELIN-NEXT:  Src: store i64 0, ptr %arrayidx, align 8 --> Dst: store i64 0, ptr %arrayidx, align 8
+; DELIN-NEXT:    da analyze - none!
+; DELIN-NEXT:  Src: store i64 0, ptr %arrayidx, align 8 --> Dst: %0 = load i64, ptr %arrayidx7, align 8
+; DELIN-NEXT:    da analyze - flow [> <=]!
+; DELIN-NEXT:  Src: store i64 0, ptr %arrayidx, align 8 --> Dst: store i64 %0, ptr %B.addr.11, align 8
+; DELIN-NEXT:    da analyze - confused!
+; DELIN-NEXT:  Src: %0 = load i64, ptr %arrayidx7, align 8 --> Dst: %0 = load i64, ptr %arrayidx7, align 8
+; DELIN-NEXT:    da analyze - none!
+; DELIN-NEXT:  Src: %0 = load i64, ptr %arrayidx7, align 8 --> Dst: store i64 %0, ptr %B.addr.11, align 8
+; DELIN-NEXT:    da analyze - confused!
+; DELIN-NEXT:  Src: store i64 %0, ptr %B.addr.11, align 8 --> Dst: store i64 %0, ptr %B.addr.11, align 8
+; DELIN-NEXT:    da analyze - none!
+;
 entry:
   br label %for.cond1.preheader
 
-; CHECK: 'Dependence Analysis' for function 'banerjee7':
-; CHECK: da analyze - none!
-; CHECK: da analyze - flow [> <=]!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - none!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - none!
-
-; NORMALIZE: 'Dependence Analysis' for function 'banerjee7':
-; NORMALIZE: da analyze - none!
-; NORMALIZE: da analyze - normalized - anti [< =>]!
-; NORMALIZE: da analyze - confused!
-; NORMALIZE: da analyze - none!
-; NORMALIZE: da analyze - confused!
-; NORMALIZE: da analyze - none!
-
-; DELIN: 'Dependence Analysis' for function 'banerjee7':
-; DELIN: da analyze - none!
-; DELIN: da analyze - flow [> <=]!
-; DELIN: da analyze - confused!
-; DELIN: da analyze - none!
-; DELIN: da analyze - confused!
-; DELIN: da analyze - none!
-
 for.cond1.preheader:                              ; preds = %entry, %for.inc8
   %B.addr.04 = phi ptr [ %B, %entry ], [ %scevgep, %for.inc8 ]
   %i.03 = phi i64 [ 0, %entry ], [ %inc9, %for.inc8 ]
@@ -567,33 +713,51 @@ for.end10:                                        ; preds = %for.inc8
 ;;      *B++ = A[10*i + j + 11];
 
 define void @banerjee8(ptr %A, ptr %B, i64 %m, i64 %n) nounwind uwtable ssp {
+; CHECK-LABEL: 'banerjee8'
+; CHECK-NEXT:  Src: store i64 0, ptr %arrayidx, align 8 --> Dst: store i64 0, ptr %arrayidx, align 8
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: store i64 0, ptr %arrayidx, align 8 --> Dst: %0 = load i64, ptr %arrayidx7, align 8
+; CHECK-NEXT:    da analyze - flow [> <>]!
+; CHECK-NEXT:  Src: store i64 0, ptr %arrayidx, align 8 --> Dst: store i64 %0, ptr %B.addr.11, align 8
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: %0 = load i64, ptr %arrayidx7, align 8 --> Dst: %0 = load i64, ptr %arrayidx7, align 8
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: %0 = load i64, ptr %arrayidx7, align 8 --> Dst: store i64 %0, ptr %B.addr.11, align 8
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: store i64 %0, ptr %B.addr.11, align 8 --> Dst: store i64 %0, ptr %B.addr.11, align 8
+; CHECK-NEXT:    da analyze - none!
+;
+; NORMALIZE-LABEL: 'banerjee8'
+; NORMALIZE-NEXT:  Src: store i64 0, ptr %arrayidx, align 8 --> Dst: store i64 0, ptr %arrayidx, align 8
+; NORMALIZE-NEXT:    da analyze - none!
+; NORMALIZE-NEXT:  Src: store i64 0, ptr %arrayidx, align 8 --> Dst: %0 = load i64, ptr %arrayidx7, align 8
+; NORMALIZE-NEXT:    da analyze - normalized - anti [< <>]!
+; NORMALIZE-NEXT:  Src: store i64 0, ptr %arrayidx, align 8 --> Dst: store i64 %0, ptr %B.addr.11, align 8
+; NORMALIZE-NEXT:    da analyze - confused!
+; NORMALIZE-NEXT:  Src: %0 = load i64, ptr %arrayidx7, align 8 --> Dst: %0 = load i64, ptr %arrayidx7, align 8
+; NORMALIZE-NEXT:    da analyze - none!
+; NORMALIZE-NEXT:  Src: %0 = load i64, ptr %arrayidx7, align 8 --> Dst: store i64 %0, ptr %B.addr.11, align 8
+; NORMALIZE-NEXT:    da analyze - confused!
+; NORMALIZE-NEXT:  Src: store i64 %0, ptr %B.addr.11, align 8 --> Dst: store i64 %0, ptr %B.addr.11, align 8
+; NORMALIZE-NEXT:    da analyze - none!
+;
+; DELIN-LABEL: 'banerjee8'
+; DELIN-NEXT:  Src: store i64 0, ptr %arrayidx, align 8 --> Dst: store i64 0, ptr %arrayidx, align 8
+; DELIN-NEXT:    da analyze - none!
+; DELIN-NEXT:  Src: store i64 0, ptr %arrayidx, align 8 --> Dst: %0 = load i64, ptr %arrayidx7, align 8
+; DELIN-NEXT:    da analyze - flow [> <>]!
+; DELIN-NEXT:  Src: store i64 0, ptr %arrayidx, align 8 --> Dst: store i64 %0, ptr %B.addr.11, align 8
+; DELIN-NEXT:    da analyze - confused!
+; DELIN-NEXT:  Src: %0 = load i64, ptr %arrayidx7, align 8 --> Dst: %0 = load i64, ptr %arrayidx7, align 8
+; DELIN-NEXT:    da analyze - none!
+; DELIN-NEXT:  Src: %0 = load i64, ptr %arrayidx7, align 8 --> Dst: store i64 %0, ptr %B.addr.11, align 8
+; DELIN-NEXT:    da analyze - confused!
+; DELIN-NEXT:  Src: store i64 %0, ptr %B.addr.11, align 8 --> Dst: store i64 %0, ptr %B.addr.11, align 8
+; DELIN-NEXT:    da analyze - none!
+;
 entry:
   br label %for.cond1.preheader
 
-; CHECK: 'Dependence Analysis' for function 'banerjee8':
-; CHECK: da analyze - none!
-; CHECK: da analyze - flow [> <>]!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - none!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - none!
-
-; NORMALIZE: 'Dependence Analysis' for function 'banerjee8':
-; NORMALIZE: da analyze - none!
-; NORMALIZE: da analyze - normalized - anti [< <>]!
-; NORMALIZE: da analyze - confused!
-; NORMALIZE: da analyze - none!
-; NORMALIZE: da analyze - confused!
-; NORMALIZE: da analyze - none!
-
-; DELIN: 'Dependence Analysis' for function 'banerjee8':
-; DELIN: da analyze - none!
-; DELIN: da analyze - flow [> <>]!
-; DELIN: da analyze - confused!
-; DELIN: da analyze - none!
-; DELIN: da analyze - confused!
-; DELIN: da analyze - none!
-
 for.cond1.preheader:                              ; preds = %entry, %for.inc8
   %B.addr.04 = phi ptr [ %B, %entry ], [ %scevgep, %for.inc8 ]
   %i.03 = phi i64 [ 0, %entry ], [ %inc9, %for.inc8 ]
@@ -634,33 +798,51 @@ for.end10:                                        ; preds = %for.inc8
 ;;      *B++ = A[i - 500*j + 11];
 
 define void @banerjee9(ptr %A, ptr %B, i64 %m, i64 %n) nounwind uwtable ssp {
+; CHECK-LABEL: 'banerjee9'
+; CHECK-NEXT:  Src: store i64 0, ptr %arrayidx, align 8 --> Dst: store i64 0, ptr %arrayidx, align 8
+; CHECK-NEXT:    da analyze - output [* *]!
+; CHECK-NEXT:  Src: store i64 0, ptr %arrayidx, align 8 --> Dst: %1 = load i64, ptr %arrayidx7, align 8
+; CHECK-NEXT:    da analyze - flow [<= =|<]!
+; CHECK-NEXT:  Src: store i64 0, ptr %arrayidx, align 8 --> Dst: store i64 %1, ptr %B.addr.11, align 8
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: %1 = load i64, ptr %arrayidx7, align 8 --> Dst: %1 = load i64, ptr %arrayidx7, align 8
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: %1 = load i64, ptr %arrayidx7, align 8 --> Dst: store i64 %1, ptr %B.addr.11, align 8
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: store i64 %1, ptr %B.addr.11, align 8 --> Dst: store i64 %1, ptr %B.addr.11, align 8
+; CHECK-NEXT:    da analyze - none!
+;
+; NORMALIZE-LABEL: 'banerjee9'
+; NORMALIZE-NEXT:  Src: store i64 0, ptr %arrayidx, align 8 --> Dst: store i64 0, ptr %arrayidx, align 8
+; NORMALIZE-NEXT:    da analyze - output [* *]!
+; NORMALIZE-NEXT:  Src: store i64 0, ptr %arrayidx, align 8 --> Dst: %1 = load i64, ptr %arrayidx7, align 8
+; NORMALIZE-NEXT:    da analyze - flow [<= =|<]!
+; NORMALIZE-NEXT:  Src: store i64 0, ptr %arrayidx, align 8 --> Dst: store i64 %1, ptr %B.addr.11, align 8
+; NORMALIZE-NEXT:    da analyze - confused!
+; NORMALIZE-NEXT:  Src: %1 = load i64, ptr %arrayidx7, align 8 --> Dst: %1 = load i64, ptr %arrayidx7, align 8
+; NORMALIZE-NEXT:    da analyze - none!
+; NORMALIZE-NEXT:  Src: %1 = load i64, ptr %arrayidx7, align 8 --> Dst: store i64 %1, ptr %B.addr.11, align 8
+; NORMALIZE-NEXT:    da analyze - confused!
+; NORMALIZE-NEXT:  Src: store i64 %1, ptr %B.addr.11, align 8 --> Dst: store i64 %1, ptr %B.addr.11, align 8
+; NORMALIZE-NEXT:    da analyze - none!
+;
+; DELIN-LABEL: 'banerjee9'
+; DELIN-NEXT:  Src: store i64 0, ptr %arrayidx, align 8 --> Dst: store i64 0, ptr %arrayidx, align 8
+; DELIN-NEXT:    da analyze - output [* *]!
+; DELIN-NEXT:  Src: store i64 0, ptr %arrayidx, align 8 --> Dst: %1 = load i64, ptr %arrayidx7, align 8
+; DELIN-NEXT:    da analyze - flow [<= =|<]!
+; DELIN-NEXT:  Src: store i64 0, ptr %arrayidx, align 8 --> Dst: store i64 %1, ptr %B.addr.11, align 8
+; DELIN-NEXT:    da analyze - confused!
+; DELIN-NEXT:  Src: %1 = load i64, ptr %arrayidx7, align 8 --> Dst: %1 = load i64, ptr %arrayidx7, align 8
+; DELIN-NEXT:    da analyze - none!
+; DELIN-NEXT:  Src: %1 = load i64, ptr %arrayidx7, align 8 --> Dst: store i64 %1, ptr %B.addr.11, align 8
+; DELIN-NEXT:    da analyze - confused!
+; DELIN-NEXT:  Src: store i64 %1, ptr %B.addr.11, align 8 --> Dst: store i64 %1, ptr %B.addr.11, align 8
+; DELIN-NEXT:    da analyze - none!
+;
 entry:
   br label %for.cond1.preheader
 
-; CHECK: 'Dependence Analysis' for function 'banerjee9':
-; CHECK: da analyze - output [* *]!
-; CHECK: da analyze - flow [<= =|<]!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - none!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - none!
-
-; NORMALIZE: 'Dependence Analysis' for function 'banerjee9':
-; NORMALIZE: da analyze - output [* *]!
-; NORMALIZE: da analyze - flow [<= =|<]!
-; NORMALIZE: da analyze - confused!
-; NORMALIZE: da analyze - none!
-; NORMALIZE: da analyze - confused!
-; NORMALIZE: da analyze - none!
-
-; DELIN: 'Dependence Analysis' for function 'banerjee9':
-; DELIN: da analyze - output [* *]!
-; DELIN: da analyze - flow [<= =|<]!
-; DELIN: da analyze - confused!
-; DELIN: da analyze - none!
-; DELIN: da analyze - confused!
-; DELIN: da analyze - none!
-
 for.cond1.preheader:                              ; preds = %entry, %for.inc8
   %B.addr.04 = phi ptr [ %B, %entry ], [ %scevgep, %for.inc8 ]
   %i.03 = phi i64 [ 0, %entry ], [ %inc9, %for.inc8 ]
@@ -702,33 +884,51 @@ for.end10:                                        ; preds = %for.inc8
 ;;      *B++ = A[i - 500*j + 11];
 
 define void @banerjee10(ptr %A, ptr %B, i64 %m, i64 %n) nounwind uwtable ssp {
+; CHECK-LABEL: 'banerjee10'
+; CHECK-NEXT:  Src: store i64 0, ptr %arrayidx, align 8 --> Dst: store i64 0, ptr %arrayidx, align 8
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: store i64 0, ptr %arrayidx, align 8 --> Dst: %1 = load i64, ptr %arrayidx6, align 8
+; CHECK-NEXT:    da analyze - flow [<> =]!
+; CHECK-NEXT:  Src: store i64 0, ptr %arrayidx, align 8 --> Dst: store i64 %1, ptr %B.addr.11, align 8
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: %1 = load i64, ptr %arrayidx6, align 8 --> Dst: %1 = load i64, ptr %arrayidx6, align 8
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: %1 = load i64, ptr %arrayidx6, align 8 --> Dst: store i64 %1, ptr %B.addr.11, align 8
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: store i64 %1, ptr %B.addr.11, align 8 --> Dst: store i64 %1, ptr %B.addr.11, align 8
+; CHECK-NEXT:    da analyze - none!
+;
+; NORMALIZE-LABEL: 'banerjee10'
+; NORMALIZE-NEXT:  Src: store i64 0, ptr %arrayidx, align 8 --> Dst: store i64 0, ptr %arrayidx, align 8
+; NORMALIZE-NEXT:    da analyze - none!
+; NORMALIZE-NEXT:  Src: store i64 0, ptr %arrayidx, align 8 --> Dst: %1 = load i64, ptr %arrayidx6, align 8
+; NORMALIZE-NEXT:    da analyze - flow [<> =]!
+; NORMALIZE-NEXT:  Src: store i64 0, ptr %arrayidx, align 8 --> Dst: store i64 %1, ptr %B.addr.11, align 8
+; NORMALIZE-NEXT:    da analyze - confused!
+; NORMALIZE-NEXT:  Src: %1 = load i64, ptr %arrayidx6, align 8 --> Dst: %1 = load i64, ptr %arrayidx6, align 8
+; NORMALIZE-NEXT:    da analyze - none!
+; NORMALIZE-NEXT:  Src: %1 = load i64, ptr %arrayidx6, align 8 --> Dst: store i64 %1, ptr %B.addr.11, align 8
+; NORMALIZE-NEXT:    da analyze - confused!
+; NORMALIZE-NEXT:  Src: store i64 %1, ptr %B.addr.11, align 8 --> Dst: store i64 %1, ptr %B.addr.11, align 8
+; NORMALIZE-NEXT:    da analyze - none!
+;
+; DELIN-LABEL: 'banerjee10'
+; DELIN-NEXT:  Src: store i64 0, ptr %arrayidx, align 8 --> Dst: store i64 0, ptr %arrayidx, align 8
+; DELIN-NEXT:    da analyze - none!
+; DELIN-NEXT:  Src: store i64 0, ptr %arrayidx, align 8 --> Dst: %1 = load i64, ptr %arrayidx6, align 8
+; DELIN-NEXT:    da analyze - flow [<> =]!
+; DELIN-NEXT:  Src: store i64 0, ptr %arrayidx, align 8 --> Dst: store i64 %1, ptr %B.addr.11, align 8
+; DELIN-NEXT:    da analyze - confused!
+; DELIN-NEXT:  Src: %1 = load i64, ptr %arrayidx6, align 8 --> Dst: %1 = load i64, ptr %arrayidx6, align 8
+; DELIN-NEXT:    da analyze - none!
+; DELIN-NEXT:  Src: %1 = load i64, ptr %arrayidx6, align 8 --> Dst: store i64 %1, ptr %B.addr.11, align 8
+; DELIN-NEXT:    da analyze - confused!
+; DELIN-NEXT:  Src: store i64 %1, ptr %B.addr.11, align 8 --> Dst: store i64 %1, ptr %B.addr.11, align 8
+; DELIN-NEXT:    da analyze - none!
+;
 entry:
   br label %for.cond1.preheader
 
-; CHECK: 'Dependence Analysis' for function 'banerjee10':
-; CHECK: da analyze - none!
-; CHECK: da analyze - flow [<> =]!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - none!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - none!
-
-; NORMALIZE: 'Dependence Analysis' for function 'banerjee10':
-; NORMALIZE: da analyze - none!
-; NORMALIZE: da analyze - flow [<> =]!
-; NORMALIZE: da analyze - confused!
-; NORMALIZE: da analyze - none!
-; NORMALIZE: da analyze - confused!
-; NORMALIZE: da analyze - none!
-
-; DELIN: 'Dependence Analysis' for function 'banerjee10':
-; DELIN: da analyze - none!
-; DELIN: da analyze - flow [<> =]!
-; DELIN: da analyze - confused!
-; DELIN: da analyze - none!
-; DELIN: da analyze - confused!
-; DELIN: da analyze - none!
-
 for.cond1.preheader:                              ; preds = %entry, %for.inc7
   %B.addr.04 = phi ptr [ %B, %entry ], [ %scevgep, %for.inc7 ]
   %i.03 = phi i64 [ 0, %entry ], [ %inc8, %for.inc7 ]
@@ -769,33 +969,51 @@ for.end9:                                         ; preds = %for.inc7
 ;;      *B++ = A[250*i - j + 11];
 
 define void @banerjee11(ptr %A, ptr %B, i64 %m, i64 %n) nounwind uwtable ssp {
+; CHECK-LABEL: 'banerjee11'
+; CHECK-NEXT:  Src: store i64 0, ptr %arrayidx, align 8 --> Dst: store i64 0, ptr %arrayidx, align 8
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: store i64 0, ptr %arrayidx, align 8 --> Dst: %0 = load i64, ptr %arrayidx6, align 8
+; CHECK-NEXT:    da analyze - flow [<= <>]!
+; CHECK-NEXT:  Src: store i64 0, ptr %arrayidx, align 8 --> Dst: store i64 %0, ptr %B.addr.11, align 8
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: %0 = load i64, ptr %arrayidx6, align 8 --> Dst: %0 = load i64, ptr %arrayidx6, align 8
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: %0 = load i64, ptr %arrayidx6, align 8 --> Dst: store i64 %0, ptr %B.addr.11, align 8
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: store i64 %0, ptr %B.addr.11, align 8 --> Dst: store i64 %0, ptr %B.addr.11, align 8
+; CHECK-NEXT:    da analyze - none!
+;
+; NORMALIZE-LABEL: 'banerjee11'
+; NORMALIZE-NEXT:  Src: store i64 0, ptr %arrayidx, align 8 --> Dst: store i64 0, ptr %arrayidx, align 8
+; NORMALIZE-NEXT:    da analyze - none!
+; NORMALIZE-NEXT:  Src: store i64 0, ptr %arrayidx, align 8 --> Dst: %0 = load i64, ptr %arrayidx6, align 8
+; NORMALIZE-NEXT:    da analyze - flow [<= <>]!
+; NORMALIZE-NEXT:  Src: store i64 0, ptr %arrayidx, align 8 --> Dst: store i64 %0, ptr %B.addr.11, align 8
+; NORMALIZE-NEXT:    da analyze - confused!
+; NORMALIZE-NEXT:  Src: %0 = load i64, ptr %arrayidx6, align 8 --> Dst: %0 = load i64, ptr %arrayidx6, align 8
+; NORMALIZE-NEXT:    da analyze - none!
+; NORMALIZE-NEXT:  Src: %0 = load i64, ptr %arrayidx6, align 8 --> Dst: store i64 %0, ptr %B.addr.11, align 8
+; NORMALIZE-NEXT:    da analyze - confused!
+; NORMALIZE-NEXT:  Src: store i64 %0, ptr %B.addr.11, align 8 --> Dst: store i64 %0, ptr %B.addr.11, align 8
+; NORMALIZE-NEXT:    da analyze - none!
+;
+; DELIN-LABEL: 'banerjee11'
+; DELIN-NEXT:  Src: store i64 0, ptr %arrayidx, align 8 --> Dst: store i64 0, ptr %arrayidx, align 8
+; DELIN-NEXT:    da analyze - none!
+; DELIN-NEXT:  Src: store i64 0, ptr %arrayidx, align 8 --> Dst: %0 = load i64, ptr %arrayidx6, align 8
+; DELIN-NEXT:    da analyze - flow [<= <>]!
+; DELIN-NEXT:  Src: store i64 0, ptr %arrayidx, align 8 --> Dst: store i64 %0, ptr %B.addr.11, align 8
+; DELIN-NEXT:    da analyze - confused!
+; DELIN-NEXT:  Src: %0 = load i64, ptr %arrayidx6, align 8 --> Dst: %0 = load i64, ptr %arrayidx6, align 8
+; DELIN-NEXT:    da analyze - none!
+; DELIN-NEXT:  Src: %0 = load i64, ptr %arrayidx6, align 8 --> Dst: store i64 %0, ptr %B.addr.11, align 8
+; DELIN-NEXT:    da analyze - confused!
+; DELIN-NEXT:  Src: store i64 %0, ptr %B.addr.11, align 8 --> Dst: store i64 %0, ptr %B.addr.11, align 8
+; DELIN-NEXT:    da analyze - none!
+;
 entry:
   br label %for.cond1.preheader
 
-; CHECK: 'Dependence Analysis' for function 'banerjee11':
-; CHECK: da analyze - none!
-; CHECK: da analyze - flow [<= <>]!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - none!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - none!
-
-; NORMALIZE: 'Dependence Analysis' for function 'banerjee11':
-; NORMALIZE: da analyze - none!
-; NORMALIZE: da analyze - flow [<= <>]!
-; NORMALIZE: da analyze - confused!
-; NORMALIZE: da analyze - none!
-; NORMALIZE: da analyze - confused!
-; NORMALIZE: da analyze - none!
-
-; DELIN: 'Dependence Analysis' for function 'banerjee11':
-; DELIN: da analyze - none!
-; DELIN: da analyze - flow [<= <>]!
-; DELIN: da analyze - confused!
-; DELIN: da analyze - none!
-; DELIN: da analyze - confused!
-; DELIN: da analyze - none!
-
 for.cond1.preheader:                              ; preds = %entry, %for.inc7
   %B.addr.04 = phi ptr [ %B, %entry ], [ %scevgep, %for.inc7 ]
   %i.03 = phi i64 [ 0, %entry ], [ %inc8, %for.inc7 ]
@@ -836,33 +1054,51 @@ for.end9:                                         ; preds = %for.inc7
 ;;      *B++ = A[100*i - j + 11];
 
 define void @banerjee12(ptr %A, ptr %B, i64 %m, i64 %n) nounwind uwtable ssp {
+; CHECK-LABEL: 'banerjee12'
+; CHECK-NEXT:  Src: store i64 0, ptr %arrayidx, align 8 --> Dst: store i64 0, ptr %arrayidx, align 8
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: store i64 0, ptr %arrayidx, align 8 --> Dst: %0 = load i64, ptr %arrayidx6, align 8
+; CHECK-NEXT:    da analyze - flow [= <>]!
+; CHECK-NEXT:  Src: store i64 0, ptr %arrayidx, align 8 --> Dst: store i64 %0, ptr %B.addr.11, align 8
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: %0 = load i64, ptr %arrayidx6, align 8 --> Dst: %0 = load i64, ptr %arrayidx6, align 8
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: %0 = load i64, ptr %arrayidx6, align 8 --> Dst: store i64 %0, ptr %B.addr.11, align 8
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: store i64 %0, ptr %B.addr.11, align 8 --> Dst: store i64 %0, ptr %B.addr.11, align 8
+; CHECK-NEXT:    da analyze - none!
+;
+; NORMALIZE-LABEL: 'banerjee12'
+; NORMALIZE-NEXT:  Src: store i64 0, ptr %arrayidx, align 8 --> Dst: store i64 0, ptr %arrayidx, align 8
+; NORMALIZE-NEXT:    da analyze - none!
+; NORMALIZE-NEXT:  Src: store i64 0, ptr %arrayidx, align 8 --> Dst: %0 = load i64, ptr %arrayidx6, align 8
+; NORMALIZE-NEXT:    da analyze - flow [= <>]!
+; NORMALIZE-NEXT:  Src: store i64 0, ptr %arrayidx, align 8 --> Dst: store i64 %0, ptr %B.addr.11, align 8
+; NORMALIZE-NEXT:    da analyze - confused!
+; NORMALIZE-NEXT:  Src: %0 = load i64, ptr %arrayidx6, align 8 --> Dst: %0 = load i64, ptr %arrayidx6, align 8
+; NORMALIZE-NEXT:    da analyze - none!
+; NORMALIZE-NEXT:  Src: %0 = load i64, ptr %arrayidx6, align 8 --> Dst: store i64 %0, ptr %B.addr.11, align 8
+; NORMALIZE-NEXT:    da analyze - confused!
+; NORMALIZE-NEXT:  Src: store i64 %0, ptr %B.addr.11, align 8 --> Dst: store i64 %0, ptr %B.addr.11, align 8
+; NORMALIZE-NEXT:    da analyze - none!
+;
+; DELIN-LABEL: 'banerjee12'
+; DELIN-NEXT:  Src: store i64 0, ptr %arrayidx, align 8 --> Dst: store i64 0, ptr %arrayidx, align 8
+; DELIN-NEXT:    da analyze - none!
+; DELIN-NEXT:  Src: store i64 0, ptr %arrayidx, align 8 --> Dst: %0 = load i64, ptr %arrayidx6, align 8
+; DELIN-NEXT:    da analyze - flow [= <>]!
+; DELIN-NEXT:  Src: store i64 0, ptr %arrayidx, align 8 --> Dst: store i64 %0, ptr %B.addr.11, align 8
+; DELIN-NEXT:    da analyze - confused!
+; DELIN-NEXT:  Src: %0 = load i64, ptr %arrayidx6, align 8 --> Dst: %0 = load i64, ptr %arrayidx6, align 8
+; DELIN-NEXT:    da analyze - none!
+; DELIN-NEXT:  Src: %0 = load i64, ptr %arrayidx6, align 8 --> Dst: store i64 %0, ptr %B.addr.11, align 8
+; DELIN-NEXT:    da analyze - confused!
+; DELIN-NEXT:  Src: store i64 %0, ptr %B.addr.11, align 8 --> Dst: store i64 %0, ptr %B.addr.11, align 8
+; DELIN-NEXT:    da analyze - none!
+;
 entry:
   br label %for.cond1.preheader
 
-; CHECK: 'Dependence Analysis' for function 'banerjee12':
-; CHECK: da analyze - none!
-; CHECK: da analyze - flow [= <>]!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - none!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - none!
-
-; NORMALIZE: 'Dependence Analysis' for function 'banerjee12':
-; NORMALIZE: da analyze - none!
-; NORMALIZE: da analyze - flow [= <>]!
-; NORMALIZE: da analyze - confused!
-; NORMALIZE: da analyze - none!
-; NORMALIZE: da analyze - confused!
-; NORMALIZE: da analyze - none!
-
-; DELIN: 'Dependence Analysis' for function 'banerjee12':
-; DELIN: da analyze - none!
-; DELIN: da analyze - flow [= <>]!
-; DELIN: da analyze - confused!
-; DELIN: da analyze - none!
-; DELIN: da analyze - confused!
-; DELIN: da analyze - none!
-
 for.cond1.preheader:                              ; preds = %entry, %for.inc7
   %B.addr.04 = phi ptr [ %B, %entry ], [ %scevgep, %for.inc7 ]
   %i.03 = phi i64 [ 0, %entry ], [ %inc8, %for.inc7 ]
diff --git a/llvm/test/Analysis/DependenceAnalysis/BasePtrBug.ll b/llvm/test/Analysis/DependenceAnalysis/BasePtrBug.ll
index 354ea5db72000..81e461a5e092d 100644
--- a/llvm/test/Analysis/DependenceAnalysis/BasePtrBug.ll
+++ b/llvm/test/Analysis/DependenceAnalysis/BasePtrBug.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --version 5
 ; RUN: opt < %s -disable-output "-passes=print<da>" -aa-pipeline=basic-aa 2>&1 \
 ; RUN: | FileCheck %s
 
@@ -12,10 +13,21 @@
 ;     B[i] = top[i] + bot[i];
 ; }
 
-; CHECK-LABEL: test1
-; CHECK: da analyze - input [*|<]!
-
 define void @test1(ptr nocapture %A, ptr nocapture %B, i32 %N) #0 {
+; CHECK-LABEL: 'test1'
+; CHECK-NEXT:  Src: %0 = load i32, ptr %gep.0, align 4 --> Dst: %0 = load i32, ptr %gep.0, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: %0 = load i32, ptr %gep.0, align 4 --> Dst: %1 = load i32, ptr %gep.1, align 4
+; CHECK-NEXT:    da analyze - input [*|<]!
+; CHECK-NEXT:  Src: %0 = load i32, ptr %gep.0, align 4 --> Dst: store i32 %add, ptr %gep.B, align 4
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: %1 = load i32, ptr %gep.1, align 4 --> Dst: %1 = load i32, ptr %gep.1, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: %1 = load i32, ptr %gep.1, align 4 --> Dst: store i32 %add, ptr %gep.B, align 4
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: store i32 %add, ptr %gep.B, align 4 --> Dst: store i32 %add, ptr %gep.B, align 4
+; CHECK-NEXT:    da analyze - none!
+;
 entry:
   %cmp9 = icmp sgt i32 %N, 0
   br i1 %cmp9, label %for.body.lr.ph, label %for.end
@@ -50,10 +62,15 @@ for.end:
 ;   }
 ; }
 
-; CHECK-LABEL: test2
-; CHECK: da analyze - consistent anti [1]!
-
 define void @test2(ptr, i32) #3 {
+; CHECK-LABEL: 'test2'
+; CHECK-NEXT:  Src: %10 = load i32, ptr %9, align 4 --> Dst: %10 = load i32, ptr %9, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: %10 = load i32, ptr %9, align 4 --> Dst: store i32 %10, ptr %12, align 4
+; CHECK-NEXT:    da analyze - consistent anti [1]!
+; CHECK-NEXT:  Src: store i32 %10, ptr %12, align 4 --> Dst: store i32 %10, ptr %12, align 4
+; CHECK-NEXT:    da analyze - none!
+;
   %3 = getelementptr inbounds i32, ptr %0, i64 1
   br label %4
 
diff --git a/llvm/test/Analysis/DependenceAnalysis/Constraints.ll b/llvm/test/Analysis/DependenceAnalysis/Constraints.ll
index c3cebd325bdeb..65dbcef153d19 100644
--- a/llvm/test/Analysis/DependenceAnalysis/Constraints.ll
+++ b/llvm/test/Analysis/DependenceAnalysis/Constraints.ll
@@ -1,4 +1,6 @@
-; RUN: opt < %s -disable-output "-passes=print<da>" -aa-pipeline=basic-aa 2>&1
+; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --version 5
+; RUN: opt < %s -disable-output "-passes=print<da>" -aa-pipeline=basic-aa 2>&1 \
+; RUN: | FileCheck %s
 ;; Check that this code doesn't abort. Test case is reduced version of lnt Polybench benchmark test case dynprog.
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
@@ -11,6 +13,140 @@ target triple = "x86_64-unknown-linux-gnu"
 
 ; Function Attrs: nounwind uwtable
 define void @dep_constraint_crash_test(i32 %M, i32 %N) {
+; CHECK-LABEL: 'dep_constraint_crash_test'
+; CHECK-NEXT:  Src: %out_l.promoted = load i32, ptr @out_l, align 4 --> Dst: %out_l.promoted = load i32, ptr @out_l, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: %out_l.promoted = load i32, ptr @out_l, align 4 --> Dst: store i32 0, ptr %13, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: %out_l.promoted = load i32, ptr @out_l, align 4 --> Dst: %18 = load i32, ptr %17, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: %out_l.promoted = load i32, ptr @out_l, align 4 --> Dst: %20 = load i32, ptr %19, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: %out_l.promoted = load i32, ptr @out_l, align 4 --> Dst: %23 = load i32, ptr %22, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: %out_l.promoted = load i32, ptr @out_l, align 4 --> Dst: store i32 %24, ptr %25, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: %out_l.promoted = load i32, ptr @out_l, align 4 --> Dst: %27 = load i32, ptr %26, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: %out_l.promoted = load i32, ptr @out_l, align 4 --> Dst: %29 = load i32, ptr %28, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: %out_l.promoted = load i32, ptr @out_l, align 4 --> Dst: store i32 %30, ptr %31, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: %out_l.promoted = load i32, ptr @out_l, align 4 --> Dst: %32 = load i32, ptr %6, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: %out_l.promoted = load i32, ptr @out_l, align 4 --> Dst: store i32 %33, ptr @out_l, align 4
+; CHECK-NEXT:    da analyze - consistent anti [|<]!
+; CHECK-NEXT:  Src: store i32 0, ptr %13, align 4 --> Dst: store i32 0, ptr %13, align 4
+; CHECK-NEXT:    da analyze - output [S * *]!
+; CHECK-NEXT:  Src: store i32 0, ptr %13, align 4 --> Dst: %18 = load i32, ptr %17, align 4
+; CHECK-NEXT:    da analyze - flow [S * *|<]!
+; CHECK-NEXT:  Src: store i32 0, ptr %13, align 4 --> Dst: %20 = load i32, ptr %19, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: store i32 0, ptr %13, align 4 --> Dst: %23 = load i32, ptr %22, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: store i32 0, ptr %13, align 4 --> Dst: store i32 %24, ptr %25, align 4
+; CHECK-NEXT:    da analyze - output [S * *|<]!
+; CHECK-NEXT:  Src: store i32 0, ptr %13, align 4 --> Dst: %27 = load i32, ptr %26, align 4
+; CHECK-NEXT:    da analyze - flow [S * *|<]!
+; CHECK-NEXT:  Src: store i32 0, ptr %13, align 4 --> Dst: %29 = load i32, ptr %28, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: store i32 0, ptr %13, align 4 --> Dst: store i32 %30, ptr %31, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: store i32 0, ptr %13, align 4 --> Dst: %32 = load i32, ptr %6, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: store i32 0, ptr %13, align 4 --> Dst: store i32 %33, ptr @out_l, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: %18 = load i32, ptr %17, align 4 --> Dst: %18 = load i32, ptr %17, align 4
+; CHECK-NEXT:    da analyze - input [S * * *]!
+; CHECK-NEXT:  Src: %18 = load i32, ptr %17, align 4 --> Dst: %20 = load i32, ptr %19, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: %18 = load i32, ptr %17, align 4 --> Dst: %23 = load i32, ptr %22, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: %18 = load i32, ptr %17, align 4 --> Dst: store i32 %24, ptr %25, align 4
+; CHECK-NEXT:    da analyze - anti [S * * *|<]!
+; CHECK-NEXT:  Src: %18 = load i32, ptr %17, align 4 --> Dst: %27 = load i32, ptr %26, align 4
+; CHECK-NEXT:    da analyze - input [S * *|<]!
+; CHECK-NEXT:  Src: %18 = load i32, ptr %17, align 4 --> Dst: %29 = load i32, ptr %28, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: %18 = load i32, ptr %17, align 4 --> Dst: store i32 %30, ptr %31, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: %18 = load i32, ptr %17, align 4 --> Dst: %32 = load i32, ptr %6, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: %18 = load i32, ptr %17, align 4 --> Dst: store i32 %33, ptr @out_l, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: %20 = load i32, ptr %19, align 4 --> Dst: %20 = load i32, ptr %19, align 4
+; CHECK-NEXT:    da analyze - input [S * S *]!
+; CHECK-NEXT:  Src: %20 = load i32, ptr %19, align 4 --> Dst: %23 = load i32, ptr %22, align 4
+; CHECK-NEXT:    da analyze - input [S * * *|<]!
+; CHECK-NEXT:  Src: %20 = load i32, ptr %19, align 4 --> Dst: store i32 %24, ptr %25, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: %20 = load i32, ptr %19, align 4 --> Dst: %27 = load i32, ptr %26, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: %20 = load i32, ptr %19, align 4 --> Dst: %29 = load i32, ptr %28, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: %20 = load i32, ptr %19, align 4 --> Dst: store i32 %30, ptr %31, align 4
+; CHECK-NEXT:    da analyze - anti [S * *|<]!
+; CHECK-NEXT:  Src: %20 = load i32, ptr %19, align 4 --> Dst: %32 = load i32, ptr %6, align 4
+; CHECK-NEXT:    da analyze - input [S|<]!
+; CHECK-NEXT:  Src: %20 = load i32, ptr %19, align 4 --> Dst: store i32 %33, ptr @out_l, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: %23 = load i32, ptr %22, align 4 --> Dst: %23 = load i32, ptr %22, align 4
+; CHECK-NEXT:    da analyze - input [S * * *]!
+; CHECK-NEXT:  Src: %23 = load i32, ptr %22, align 4 --> Dst: store i32 %24, ptr %25, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: %23 = load i32, ptr %22, align 4 --> Dst: %27 = load i32, ptr %26, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: %23 = load i32, ptr %22, align 4 --> Dst: %29 = load i32, ptr %28, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: %23 = load i32, ptr %22, align 4 --> Dst: store i32 %30, ptr %31, align 4
+; CHECK-NEXT:    da analyze - anti [S * *|<]!
+; CHECK-NEXT:  Src: %23 = load i32, ptr %22, align 4 --> Dst: %32 = load i32, ptr %6, align 4
+; CHECK-NEXT:    da analyze - input [S|<]!
+; CHECK-NEXT:  Src: %23 = load i32, ptr %22, align 4 --> Dst: store i32 %33, ptr @out_l, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: store i32 %24, ptr %25, align 4 --> Dst: store i32 %24, ptr %25, align 4
+; CHECK-NEXT:    da analyze - output [S * * *]!
+; CHECK-NEXT:  Src: store i32 %24, ptr %25, align 4 --> Dst: %27 = load i32, ptr %26, align 4
+; CHECK-NEXT:    da analyze - flow [S * *|<]!
+; CHECK-NEXT:  Src: store i32 %24, ptr %25, align 4 --> Dst: %29 = load i32, ptr %28, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: store i32 %24, ptr %25, align 4 --> Dst: store i32 %30, ptr %31, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: store i32 %24, ptr %25, align 4 --> Dst: %32 = load i32, ptr %6, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: store i32 %24, ptr %25, align 4 --> Dst: store i32 %33, ptr @out_l, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: %27 = load i32, ptr %26, align 4 --> Dst: %27 = load i32, ptr %26, align 4
+; CHECK-NEXT:    da analyze - input [S * *]!
+; CHECK-NEXT:  Src: %27 = load i32, ptr %26, align 4 --> Dst: %29 = load i32, ptr %28, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: %27 = load i32, ptr %26, align 4 --> Dst: store i32 %30, ptr %31, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: %27 = load i32, ptr %26, align 4 --> Dst: %32 = load i32, ptr %6, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: %27 = load i32, ptr %26, align 4 --> Dst: store i32 %33, ptr @out_l, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: %29 = load i32, ptr %28, align 4 --> Dst: %29 = load i32, ptr %28, align 4
+; CHECK-NEXT:    da analyze - input [S * *]!
+; CHECK-NEXT:  Src: %29 = load i32, ptr %28, align 4 --> Dst: store i32 %30, ptr %31, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: %29 = load i32, ptr %28, align 4 --> Dst: %32 = load i32, ptr %6, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: %29 = load i32, ptr %28, align 4 --> Dst: store i32 %33, ptr @out_l, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: store i32 %30, ptr %31, align 4 --> Dst: store i32 %30, ptr %31, align 4
+; CHECK-NEXT:    da analyze - output [S * *]!
+; CHECK-NEXT:  Src: store i32 %30, ptr %31, align 4 --> Dst: %32 = load i32, ptr %6, align 4
+; CHECK-NEXT:    da analyze - flow [S|<]!
+; CHECK-NEXT:  Src: store i32 %30, ptr %31, align 4 --> Dst: store i32 %33, ptr @out_l, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: %32 = load i32, ptr %6, align 4 --> Dst: %32 = load i32, ptr %6, align 4
+; CHECK-NEXT:    da analyze - consistent input [S]!
+; CHECK-NEXT:  Src: %32 = load i32, ptr %6, align 4 --> Dst: store i32 %33, ptr @out_l, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: store i32 %33, ptr @out_l, align 4 --> Dst: store i32 %33, ptr @out_l, align 4
+; CHECK-NEXT:    da analyze - none!
+;
   %1 = icmp sgt i32 %N, 0
   br i1 %1, label %.preheader.lr.ph, label %35
 
diff --git a/llvm/test/Analysis/DependenceAnalysis/Coupled.ll b/llvm/test/Analysis/DependenceAnalysis/Coupled.ll
index 04ea6987e21a8..ff9f393f88152 100644
--- a/llvm/test/Analysis/DependenceAnalysis/Coupled.ll
+++ b/llvm/test/Analysis/DependenceAnalysis/Coupled.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --version 5
 ; RUN: opt < %s -disable-output "-passes=print<da>" -aa-pipeline=basic-aa 2>&1 \
 ; RUN: | FileCheck %s
 
@@ -10,17 +11,23 @@ target triple = "x86_64-apple-macosx10.6.0"
 ;;    *B++ = A[i + 10][i + 9];
 
 define void @couple0(ptr %A, ptr %B, i32 %n) nounwind uwtable ssp {
+; CHECK-LABEL: 'couple0'
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx1, align 4 --> Dst: store i32 %conv, ptr %arrayidx1, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx1, align 4 --> Dst: %0 = load i32, ptr %arrayidx4, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx1, align 4 --> Dst: store i32 %0, ptr %B.addr.01, align 4
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: %0 = load i32, ptr %arrayidx4, align 4 --> Dst: %0 = load i32, ptr %arrayidx4, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: %0 = load i32, ptr %arrayidx4, align 4 --> Dst: store i32 %0, ptr %B.addr.01, align 4
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: store i32 %0, ptr %B.addr.01, align 4 --> Dst: store i32 %0, ptr %B.addr.01, align 4
+; CHECK-NEXT:    da analyze - none!
+;
 entry:
   br label %for.body
 
-; CHECK-LABEL: couple0
-; CHECK: da analyze - none!
-; CHECK: da analyze - none!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - none!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - none!
-
 for.body:                                         ; preds = %entry, %for.body
   %i.02 = phi i64 [ 0, %entry ], [ %inc, %for.body ]
   %B.addr.01 = phi ptr [ %B, %entry ], [ %incdec.ptr, %for.body ]
@@ -47,17 +54,23 @@ for.end:                                          ; preds = %for.body
 ;;    *B++ = A[i + 9][i + 9];
 
 define void @couple1(ptr %A, ptr %B, i32 %n) nounwind uwtable ssp {
+; CHECK-LABEL: 'couple1'
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx1, align 4 --> Dst: store i32 %conv, ptr %arrayidx1, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx1, align 4 --> Dst: %0 = load i32, ptr %arrayidx4, align 4
+; CHECK-NEXT:    da analyze - consistent flow [-9]!
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx1, align 4 --> Dst: store i32 %0, ptr %B.addr.01, align 4
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: %0 = load i32, ptr %arrayidx4, align 4 --> Dst: %0 = load i32, ptr %arrayidx4, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: %0 = load i32, ptr %arrayidx4, align 4 --> Dst: store i32 %0, ptr %B.addr.01, align 4
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: store i32 %0, ptr %B.addr.01, align 4 --> Dst: store i32 %0, ptr %B.addr.01, align 4
+; CHECK-NEXT:    da analyze - none!
+;
 entry:
   br label %for.body
 
-; CHECK-LABEL: couple1
-; CHECK: da analyze - none!
-; CHECK: da analyze - consistent flow [-9]!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - none!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - none!
-
 for.body:                                         ; preds = %entry, %for.body
   %i.02 = phi i64 [ 0, %entry ], [ %inc, %for.body ]
   %B.addr.01 = phi ptr [ %B, %entry ], [ %incdec.ptr, %for.body ]
@@ -84,17 +97,23 @@ for.end:                                          ; preds = %for.body
 ;;    *B++ = A[i][i];
 
 define void @couple2(ptr %A, ptr %B, i32 %n) nounwind uwtable ssp {
+; CHECK-LABEL: 'couple2'
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx3, align 4 --> Dst: store i32 %conv, ptr %arrayidx3, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx3, align 4 --> Dst: %0 = load i32, ptr %arrayidx5, align 4
+; CHECK-NEXT:    da analyze - flow [*|<]!
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx3, align 4 --> Dst: store i32 %0, ptr %B.addr.01, align 4
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: %0 = load i32, ptr %arrayidx5, align 4 --> Dst: %0 = load i32, ptr %arrayidx5, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: %0 = load i32, ptr %arrayidx5, align 4 --> Dst: store i32 %0, ptr %B.addr.01, align 4
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: store i32 %0, ptr %B.addr.01, align 4 --> Dst: store i32 %0, ptr %B.addr.01, align 4
+; CHECK-NEXT:    da analyze - none!
+;
 entry:
   br label %for.body
 
-; CHECK-LABEL: couple2
-; CHECK: da analyze - none!
-; CHECK: da analyze - flow [*|<]!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - none!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - none!
-
 for.body:                                         ; preds = %entry, %for.body
   %i.02 = phi i64 [ 0, %entry ], [ %inc, %for.body ]
   %B.addr.01 = phi ptr [ %B, %entry ], [ %incdec.ptr, %for.body ]
@@ -123,17 +142,23 @@ for.end:                                          ; preds = %for.body
 ;;    *B++ = A[i][i];
 
 define void @couple3(ptr %A, ptr %B, i32 %n) nounwind uwtable ssp {
+; CHECK-LABEL: 'couple3'
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx3, align 4 --> Dst: store i32 %conv, ptr %arrayidx3, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx3, align 4 --> Dst: %0 = load i32, ptr %arrayidx5, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx3, align 4 --> Dst: store i32 %0, ptr %B.addr.01, align 4
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: %0 = load i32, ptr %arrayidx5, align 4 --> Dst: %0 = load i32, ptr %arrayidx5, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: %0 = load i32, ptr %arrayidx5, align 4 --> Dst: store i32 %0, ptr %B.addr.01, align 4
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: store i32 %0, ptr %B.addr.01, align 4 --> Dst: store i32 %0, ptr %B.addr.01, align 4
+; CHECK-NEXT:    da analyze - none!
+;
 entry:
   br label %for.body
 
-; CHECK-LABEL: couple3
-; CHECK: da analyze - none!
-; CHECK: da analyze - none!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - none!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - none!
-
 for.body:                                         ; preds = %entry, %for.body
   %i.02 = phi i64 [ 0, %entry ], [ %inc, %for.body ]
   %B.addr.01 = phi ptr [ %B, %entry ], [ %incdec.ptr, %for.body ]
@@ -162,17 +187,23 @@ for.end:                                          ; preds = %for.body
 ;;    *B++ = A[i][i];
 
 define void @couple4(ptr %A, ptr %B, i32 %n) nounwind uwtable ssp {
+; CHECK-LABEL: 'couple4'
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx4, align 4 --> Dst: store i32 %conv, ptr %arrayidx4, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx4, align 4 --> Dst: %0 = load i32, ptr %arrayidx6, align 4
+; CHECK-NEXT:    da analyze - flow [*|<]!
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx4, align 4 --> Dst: store i32 %0, ptr %B.addr.01, align 4
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: %0 = load i32, ptr %arrayidx6, align 4 --> Dst: %0 = load i32, ptr %arrayidx6, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: %0 = load i32, ptr %arrayidx6, align 4 --> Dst: store i32 %0, ptr %B.addr.01, align 4
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: store i32 %0, ptr %B.addr.01, align 4 --> Dst: store i32 %0, ptr %B.addr.01, align 4
+; CHECK-NEXT:    da analyze - none!
+;
 entry:
   br label %for.body
 
-; CHECK-LABEL: couple4
-; CHECK: da analyze - none!
-; CHECK: da analyze - flow [*|<]!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - none!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - none!
-
 for.body:                                         ; preds = %entry, %for.body
   %i.02 = phi i64 [ 0, %entry ], [ %inc, %for.body ]
   %B.addr.01 = phi ptr [ %B, %entry ], [ %incdec.ptr, %for.body ]
@@ -202,17 +233,23 @@ for.end:                                          ; preds = %for.body
 ;;    *B++ = A[i][i];
 
 define void @couple5(ptr %A, ptr %B, i32 %n) nounwind uwtable ssp {
+; CHECK-LABEL: 'couple5'
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx5, align 4 --> Dst: store i32 %conv, ptr %arrayidx5, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx5, align 4 --> Dst: %0 = load i32, ptr %arrayidx7, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx5, align 4 --> Dst: store i32 %0, ptr %B.addr.01, align 4
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: %0 = load i32, ptr %arrayidx7, align 4 --> Dst: %0 = load i32, ptr %arrayidx7, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: %0 = load i32, ptr %arrayidx7, align 4 --> Dst: store i32 %0, ptr %B.addr.01, align 4
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: store i32 %0, ptr %B.addr.01, align 4 --> Dst: store i32 %0, ptr %B.addr.01, align 4
+; CHECK-NEXT:    da analyze - none!
+;
 entry:
   br label %for.body
 
-; CHECK-LABEL: couple5
-; CHECK: da analyze - none!
-; CHECK: da analyze - none!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - none!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - none!
-
 for.body:                                         ; preds = %entry, %for.body
   %i.02 = phi i64 [ 0, %entry ], [ %inc, %for.body ]
   %B.addr.01 = phi ptr [ %B, %entry ], [ %incdec.ptr, %for.body ]
@@ -244,17 +281,23 @@ for.end:                                          ; preds = %for.body
 ;;    *B++ = A[i][i];
 
 define void @couple6(ptr %A, ptr %B, i32 %n) nounwind uwtable ssp {
+; CHECK-LABEL: 'couple6'
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx1, align 4 --> Dst: store i32 %conv, ptr %arrayidx1, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx1, align 4 --> Dst: %0 = load i32, ptr %arrayidx3, align 4
+; CHECK-NEXT:    da analyze - flow [=|<]!
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx1, align 4 --> Dst: store i32 %0, ptr %B.addr.01, align 4
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: %0 = load i32, ptr %arrayidx3, align 4 --> Dst: %0 = load i32, ptr %arrayidx3, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: %0 = load i32, ptr %arrayidx3, align 4 --> Dst: store i32 %0, ptr %B.addr.01, align 4
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: store i32 %0, ptr %B.addr.01, align 4 --> Dst: store i32 %0, ptr %B.addr.01, align 4
+; CHECK-NEXT:    da analyze - none!
+;
 entry:
   br label %for.body
 
-; CHECK-LABEL: couple6
-; CHECK: da analyze - none!
-; CHECK: da analyze - flow [=|<]!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - none!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - none!
-
 for.body:                                         ; preds = %entry, %for.body
   %i.02 = phi i64 [ 0, %entry ], [ %inc, %for.body ]
   %B.addr.01 = phi ptr [ %B, %entry ], [ %incdec.ptr, %for.body ]
@@ -281,17 +324,23 @@ for.end:                                          ; preds = %for.body
 ;;    *B++ = A[i][i];
 
 define void @couple7(ptr %A, ptr %B, i32 %n) nounwind uwtable ssp {
+; CHECK-LABEL: 'couple7'
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx1, align 4 --> Dst: store i32 %conv, ptr %arrayidx1, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx1, align 4 --> Dst: %0 = load i32, ptr %arrayidx3, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx1, align 4 --> Dst: store i32 %0, ptr %B.addr.01, align 4
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: %0 = load i32, ptr %arrayidx3, align 4 --> Dst: %0 = load i32, ptr %arrayidx3, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: %0 = load i32, ptr %arrayidx3, align 4 --> Dst: store i32 %0, ptr %B.addr.01, align 4
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: store i32 %0, ptr %B.addr.01, align 4 --> Dst: store i32 %0, ptr %B.addr.01, align 4
+; CHECK-NEXT:    da analyze - none!
+;
 entry:
   br label %for.body
 
-; CHECK-LABEL: couple7
-; CHECK: da analyze - none!
-; CHECK: da analyze - none!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - none!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - none!
-
 for.body:                                         ; preds = %entry, %for.body
   %i.02 = phi i64 [ 0, %entry ], [ %inc, %for.body ]
   %B.addr.01 = phi ptr [ %B, %entry ], [ %incdec.ptr, %for.body ]
@@ -318,17 +367,23 @@ for.end:                                          ; preds = %for.body
 ;;    *B++ = A[i][i];
 
 define void @couple8(ptr %A, ptr %B, i32 %n) nounwind uwtable ssp {
+; CHECK-LABEL: 'couple8'
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx2, align 4 --> Dst: store i32 %conv, ptr %arrayidx2, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx2, align 4 --> Dst: %0 = load i32, ptr %arrayidx4, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx2, align 4 --> Dst: store i32 %0, ptr %B.addr.01, align 4
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: %0 = load i32, ptr %arrayidx4, align 4 --> Dst: %0 = load i32, ptr %arrayidx4, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: %0 = load i32, ptr %arrayidx4, align 4 --> Dst: store i32 %0, ptr %B.addr.01, align 4
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: store i32 %0, ptr %B.addr.01, align 4 --> Dst: store i32 %0, ptr %B.addr.01, align 4
+; CHECK-NEXT:    da analyze - none!
+;
 entry:
   br label %for.body
 
-; CHECK-LABEL: couple8
-; CHECK: da analyze - none!
-; CHECK: da analyze - none!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - none!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - none!
-
 for.body:                                         ; preds = %entry, %for.body
   %i.02 = phi i64 [ 0, %entry ], [ %inc, %for.body ]
   %B.addr.01 = phi ptr [ %B, %entry ], [ %incdec.ptr, %for.body ]
@@ -356,17 +411,23 @@ for.end:                                          ; preds = %for.body
 ;;    *B++ = A[i][i];
 
 define void @couple9(ptr %A, ptr %B, i32 %n) nounwind uwtable ssp {
+; CHECK-LABEL: 'couple9'
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx2, align 4 --> Dst: store i32 %conv, ptr %arrayidx2, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx2, align 4 --> Dst: %0 = load i32, ptr %arrayidx4, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx2, align 4 --> Dst: store i32 %0, ptr %B.addr.01, align 4
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: %0 = load i32, ptr %arrayidx4, align 4 --> Dst: %0 = load i32, ptr %arrayidx4, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: %0 = load i32, ptr %arrayidx4, align 4 --> Dst: store i32 %0, ptr %B.addr.01, align 4
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: store i32 %0, ptr %B.addr.01, align 4 --> Dst: store i32 %0, ptr %B.addr.01, align 4
+; CHECK-NEXT:    da analyze - none!
+;
 entry:
   br label %for.body
 
-; CHECK-LABEL: couple9
-; CHECK: da analyze - none!
-; CHECK: da analyze - none!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - none!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - none!
-
 for.body:                                         ; preds = %entry, %for.body
   %i.02 = phi i64 [ 0, %entry ], [ %inc, %for.body ]
   %B.addr.01 = phi ptr [ %B, %entry ], [ %incdec.ptr, %for.body ]
@@ -394,17 +455,23 @@ for.end:                                          ; preds = %for.body
 ;;    *B++ = A[i][i];
 
 define void @couple10(ptr %A, ptr %B, i32 %n) nounwind uwtable ssp {
+; CHECK-LABEL: 'couple10'
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx2, align 4 --> Dst: store i32 %conv, ptr %arrayidx2, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx2, align 4 --> Dst: %0 = load i32, ptr %arrayidx4, align 4
+; CHECK-NEXT:    da analyze - flow [>]!
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx2, align 4 --> Dst: store i32 %0, ptr %B.addr.01, align 4
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: %0 = load i32, ptr %arrayidx4, align 4 --> Dst: %0 = load i32, ptr %arrayidx4, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: %0 = load i32, ptr %arrayidx4, align 4 --> Dst: store i32 %0, ptr %B.addr.01, align 4
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: store i32 %0, ptr %B.addr.01, align 4 --> Dst: store i32 %0, ptr %B.addr.01, align 4
+; CHECK-NEXT:    da analyze - none!
+;
 entry:
   br label %for.body
 
-; CHECK-LABEL: couple10
-; CHECK: da analyze - none!
-; CHECK: da analyze - flow [>]!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - none!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - none!
-
 for.body:                                         ; preds = %entry, %for.body
   %i.02 = phi i64 [ 0, %entry ], [ %inc, %for.body ]
   %B.addr.01 = phi ptr [ %B, %entry ], [ %incdec.ptr, %for.body ]
@@ -432,18 +499,24 @@ for.end:                                          ; preds = %for.body
 ;;    *B++ = A[i][i];
 
 define void @couple11(ptr %A, ptr %B, i32 %n) nounwind uwtable ssp {
+; CHECK-LABEL: 'couple11'
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx2, align 4 --> Dst: store i32 %conv, ptr %arrayidx2, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx2, align 4 --> Dst: %0 = load i32, ptr %arrayidx4, align 4
+; CHECK-NEXT:    da analyze - flow [=|<] splitable!
+; CHECK-NEXT:    da analyze - split level = 1, iteration = 9!
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx2, align 4 --> Dst: store i32 %0, ptr %B.addr.01, align 4
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: %0 = load i32, ptr %arrayidx4, align 4 --> Dst: %0 = load i32, ptr %arrayidx4, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: %0 = load i32, ptr %arrayidx4, align 4 --> Dst: store i32 %0, ptr %B.addr.01, align 4
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: store i32 %0, ptr %B.addr.01, align 4 --> Dst: store i32 %0, ptr %B.addr.01, align 4
+; CHECK-NEXT:    da analyze - none!
+;
 entry:
   br label %for.body
 
-; CHECK-LABEL: couple11
-; CHECK: da analyze - none!
-; CHECK: da analyze - flow [=|<] splitable!
-; CHECK: da analyze - split level = 1, iteration = 9!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - none!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - none!
-
 for.body:                                         ; preds = %entry, %for.body
   %i.02 = phi i64 [ 0, %entry ], [ %inc, %for.body ]
   %B.addr.01 = phi ptr [ %B, %entry ], [ %incdec.ptr, %for.body ]
@@ -471,18 +544,24 @@ for.end:                                          ; preds = %for.body
 ;;    *B++ = A[i][i];
 
 define void @couple12(ptr %A, ptr %B, i32 %n) nounwind uwtable ssp {
+; CHECK-LABEL: 'couple12'
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx2, align 4 --> Dst: store i32 %conv, ptr %arrayidx2, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx2, align 4 --> Dst: %0 = load i32, ptr %arrayidx4, align 4
+; CHECK-NEXT:    da analyze - flow [<] splitable!
+; CHECK-NEXT:    da analyze - split level = 1, iteration = 11!
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx2, align 4 --> Dst: store i32 %0, ptr %B.addr.01, align 4
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: %0 = load i32, ptr %arrayidx4, align 4 --> Dst: %0 = load i32, ptr %arrayidx4, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: %0 = load i32, ptr %arrayidx4, align 4 --> Dst: store i32 %0, ptr %B.addr.01, align 4
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: store i32 %0, ptr %B.addr.01, align 4 --> Dst: store i32 %0, ptr %B.addr.01, align 4
+; CHECK-NEXT:    da analyze - none!
+;
 entry:
   br label %for.body
 
-; CHECK-LABEL: couple12
-; CHECK: da analyze - none!
-; CHECK: da analyze - flow [<] splitable!
-; CHECK: da analyze - split level = 1, iteration = 11!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - none!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - none!
-
 for.body:                                         ; preds = %entry, %for.body
   %i.02 = phi i64 [ 0, %entry ], [ %inc, %for.body ]
   %B.addr.01 = phi ptr [ %B, %entry ], [ %incdec.ptr, %for.body ]
@@ -510,17 +589,23 @@ for.end:                                          ; preds = %for.body
 ;;    *B++ = A[i][i];
 
 define void @couple13(ptr %A, ptr %B, i32 %n) nounwind uwtable ssp {
+; CHECK-LABEL: 'couple13'
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx2, align 4 --> Dst: store i32 %conv, ptr %arrayidx2, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx2, align 4 --> Dst: %0 = load i32, ptr %arrayidx4, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx2, align 4 --> Dst: store i32 %0, ptr %B.addr.01, align 4
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: %0 = load i32, ptr %arrayidx4, align 4 --> Dst: %0 = load i32, ptr %arrayidx4, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: %0 = load i32, ptr %arrayidx4, align 4 --> Dst: store i32 %0, ptr %B.addr.01, align 4
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: store i32 %0, ptr %B.addr.01, align 4 --> Dst: store i32 %0, ptr %B.addr.01, align 4
+; CHECK-NEXT:    da analyze - none!
+;
 entry:
   br label %for.body
 
-; CHECK-LABEL: couple13
-; CHECK: da analyze - none!
-; CHECK: da analyze - none!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - none!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - none!
-
 for.body:                                         ; preds = %entry, %for.body
   %i.02 = phi i64 [ 0, %entry ], [ %inc, %for.body ]
   %B.addr.01 = phi ptr [ %B, %entry ], [ %incdec.ptr, %for.body ]
@@ -547,17 +632,23 @@ for.end:                                          ; preds = %for.body
 ;;    *B++ = A[i][i][i];
 
 define void @couple14(ptr %A, ptr %B, i32 %n) nounwind uwtable ssp {
+; CHECK-LABEL: 'couple14'
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx3, align 4 --> Dst: store i32 %conv, ptr %arrayidx3, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx3, align 4 --> Dst: %0 = load i32, ptr %arrayidx6, align 4
+; CHECK-NEXT:    da analyze - flow [=|<]!
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx3, align 4 --> Dst: store i32 %0, ptr %B.addr.01, align 4
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: %0 = load i32, ptr %arrayidx6, align 4 --> Dst: %0 = load i32, ptr %arrayidx6, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: %0 = load i32, ptr %arrayidx6, align 4 --> Dst: store i32 %0, ptr %B.addr.01, align 4
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: store i32 %0, ptr %B.addr.01, align 4 --> Dst: store i32 %0, ptr %B.addr.01, align 4
+; CHECK-NEXT:    da analyze - none!
+;
 entry:
   br label %for.body
 
-; CHECK-LABEL: couple14
-; CHECK: da analyze - none!
-; CHECK: da analyze - flow [=|<]!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - none!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - none!
-
 for.body:                                         ; preds = %entry, %for.body
   %i.02 = phi i64 [ 0, %entry ], [ %inc, %for.body ]
   %B.addr.01 = phi ptr [ %B, %entry ], [ %incdec.ptr, %for.body ]
@@ -585,17 +676,23 @@ for.end:                                          ; preds = %for.body
 ;;    *B++ = A[i][i][i];
 
 define void @couple15(ptr %A, ptr %B, i32 %n) nounwind uwtable ssp {
+; CHECK-LABEL: 'couple15'
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx3, align 4 --> Dst: store i32 %conv, ptr %arrayidx3, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx3, align 4 --> Dst: %0 = load i32, ptr %arrayidx6, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx3, align 4 --> Dst: store i32 %0, ptr %B.addr.01, align 4
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: %0 = load i32, ptr %arrayidx6, align 4 --> Dst: %0 = load i32, ptr %arrayidx6, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: %0 = load i32, ptr %arrayidx6, align 4 --> Dst: store i32 %0, ptr %B.addr.01, align 4
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: store i32 %0, ptr %B.addr.01, align 4 --> Dst: store i32 %0, ptr %B.addr.01, align 4
+; CHECK-NEXT:    da analyze - none!
+;
 entry:
   br label %for.body
 
-; CHECK-LABEL: couple15
-; CHECK: da analyze - none!
-; CHECK: da analyze - none!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - none!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - none!
-
 for.body:                                         ; preds = %entry, %for.body
   %i.02 = phi i64 [ 0, %entry ], [ %inc, %for.body ]
   %B.addr.01 = phi ptr [ %B, %entry ], [ %incdec.ptr, %for.body ]
@@ -623,15 +720,18 @@ for.end:                                          ; preds = %for.body
 ;;      A[M*N + M*i + j] = 2;
 
 define void @couple_weakzerosiv(ptr noalias nocapture %A, i64 %N, i64 %M) {
+; CHECK-LABEL: 'couple_weakzerosiv'
+; CHECK-NEXT:  Src: store i32 1, ptr %arrayidx.us, align 4 --> Dst: store i32 1, ptr %arrayidx.us, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: store i32 1, ptr %arrayidx.us, align 4 --> Dst: store i32 2, ptr %arrayidx9.us, align 4
+; CHECK-NEXT:    da analyze - output [p>]!
+; CHECK-NEXT:  Src: store i32 2, ptr %arrayidx9.us, align 4 --> Dst: store i32 2, ptr %arrayidx9.us, align 4
+; CHECK-NEXT:    da analyze - none!
+;
 entry:
   %cmp29 = icmp sgt i64 %N, 0
   br i1 %cmp29, label %for.body.lr.ph, label %for.cond.cleanup
 
-; CHECK-LABEL: couple_weakzerosiv
-; CHECK: da analyze - none!
-; CHECK: da analyze - output [p>]!
-; CHECK: da analyze - none!
-
 for.body.lr.ph:                                   ; preds = %entry
   %mul = mul nsw i64 %M, %N
   br label %for.body.us
diff --git a/llvm/test/Analysis/DependenceAnalysis/DADelin.ll b/llvm/test/Analysis/DependenceAnalysis/DADelin.ll
index 73260910c3873..b2e4959a7812e 100644
--- a/llvm/test/Analysis/DependenceAnalysis/DADelin.ll
+++ b/llvm/test/Analysis/DependenceAnalysis/DADelin.ll
@@ -1,19 +1,24 @@
+; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --version 5
 ; RUN: opt < %s -disable-output "-passes=print<da>" -aa-pipeline=basic-aa 2>&1 \
 ; RUN: | FileCheck %s
 
 target datalayout = "e-m:e-p:32:32-i64:64-v128:64:128-a:0:32-n32-S64"
 target triple = "thumbv8m.main-arm-none-eabi"
 
-; CHECK-LABEL: t1
 ;;  for (int i = 0; i < n; i++)
 ;;   for (int j = 0; j < m; j++)
 ;;    for (int k = 0; k < o; k++)
 ;;      = A[i*m*o + j*o + k]
 ;;     A[i*m*o + j*o + k] =
 define void @t1(i32 %n, i32 %m, i32 %o, ptr nocapture %A) {
-; CHECK: da analyze - none!
-; CHECK: da analyze - consistent anti [0 0 0|<]!
-; CHECK: da analyze - none!
+; CHECK-LABEL: 't1'
+; CHECK-NEXT:  Src: %0 = load i32, ptr %arrayidx, align 4 --> Dst: %0 = load i32, ptr %arrayidx, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: %0 = load i32, ptr %arrayidx, align 4 --> Dst: store i32 %add12, ptr %arrayidx, align 4
+; CHECK-NEXT:    da analyze - consistent anti [0 0 0|<]!
+; CHECK-NEXT:  Src: store i32 %add12, ptr %arrayidx, align 4 --> Dst: store i32 %add12, ptr %arrayidx, align 4
+; CHECK-NEXT:    da analyze - none!
+;
 entry:
   %cmp49 = icmp sgt i32 %n, 0
   br i1 %cmp49, label %for.cond1.preheader.lr.ph, label %for.cond.cleanup
@@ -65,16 +70,20 @@ for.cond.cleanup:                                 ; preds = %for.cond.cleanup3,
   ret void
 }
 
-; CHECK-LABEL: t2
 ;;  for (int i = 0; i < n; i++)
 ;;   for (int j = 0; j < m; j++)
 ;;    for (int k = 0; k < o; k++)
 ;;      = A[i*m*o + j*o + k]
 ;;     A[i*m*o + j*o + k + 1] =
 define void @t2(i32 %n, i32 %m, i32 %o, ptr nocapture %A) {
-; CHECK: da analyze - none!
-; CHECK: da analyze - anti [* * *|<]!
-; CHECK: da analyze - output [* * *]!
+; CHECK-LABEL: 't2'
+; CHECK-NEXT:  Src: %0 = load i32, ptr %arrayidx, align 4 --> Dst: %0 = load i32, ptr %arrayidx, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: %0 = load i32, ptr %arrayidx, align 4 --> Dst: store i32 %add12, ptr %arrayidx2, align 4
+; CHECK-NEXT:    da analyze - anti [* * *|<]!
+; CHECK-NEXT:  Src: store i32 %add12, ptr %arrayidx2, align 4 --> Dst: store i32 %add12, ptr %arrayidx2, align 4
+; CHECK-NEXT:    da analyze - output [* * *]!
+;
 entry:
   %cmp49 = icmp sgt i32 %n, 0
   br i1 %cmp49, label %for.cond1.preheader.lr.ph, label %for.cond.cleanup
@@ -128,16 +137,20 @@ for.cond.cleanup:                                 ; preds = %for.cond.cleanup3,
   ret void
 }
 
-; CHECK-LABEL: t3
 ;;  for (int i = 0; i < n; i++)
 ;;   for (int j = 0; j < m; j++)
 ;;    for (int k = 0; k < o; k++)
 ;;      = A[i*m*o + j*o + k]
 ;;     A[i*m*o + j*o + k - 1] =
 define void @t3(i32 %n, i32 %m, i32 %o, ptr nocapture %A) {
-; CHECK: da analyze - none!
-; CHECK: da analyze - anti [* * *|<]!
-; CHECK: da analyze - output [* * *]!
+; CHECK-LABEL: 't3'
+; CHECK-NEXT:  Src: %0 = load i32, ptr %arrayidx, align 4 --> Dst: %0 = load i32, ptr %arrayidx, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: %0 = load i32, ptr %arrayidx, align 4 --> Dst: store i32 %add12, ptr %arrayidx2, align 4
+; CHECK-NEXT:    da analyze - anti [* * *|<]!
+; CHECK-NEXT:  Src: store i32 %add12, ptr %arrayidx2, align 4 --> Dst: store i32 %add12, ptr %arrayidx2, align 4
+; CHECK-NEXT:    da analyze - output [* * *]!
+;
 entry:
   %cmp49 = icmp sgt i32 %n, 0
   br i1 %cmp49, label %for.cond1.preheader.lr.ph, label %for.cond.cleanup
@@ -191,16 +204,20 @@ for.cond.cleanup:                                 ; preds = %for.cond.cleanup3,
   ret void
 }
 
-; CHECK-LABEL: t4
 ;;  for (int i = 0; i < n; i++)
 ;;   for (int j = 0; j < m; j++)
 ;;    for (int k = 0; k < o; k++)
 ;;      = A[i*m*o + j*o + k]
 ;;     A[i*m*o + j*o + k + o] =
 define void @t4(i32 %n, i32 %m, i32 %o, ptr nocapture %A) {
-; CHECK: da analyze - none!
-; CHECK: da analyze - anti [* * *|<]!
-; CHECK: da analyze - output [* * *]!
+; CHECK-LABEL: 't4'
+; CHECK-NEXT:  Src: %0 = load i32, ptr %arrayidx, align 4 --> Dst: %0 = load i32, ptr %arrayidx, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: %0 = load i32, ptr %arrayidx, align 4 --> Dst: store i32 %add12, ptr %arrayidx2, align 4
+; CHECK-NEXT:    da analyze - anti [* * *|<]!
+; CHECK-NEXT:  Src: store i32 %add12, ptr %arrayidx2, align 4 --> Dst: store i32 %add12, ptr %arrayidx2, align 4
+; CHECK-NEXT:    da analyze - output [* * *]!
+;
 entry:
   %cmp49 = icmp sgt i32 %n, 0
   br i1 %cmp49, label %for.cond1.preheader.lr.ph, label %for.cond.cleanup
@@ -254,16 +271,20 @@ for.cond.cleanup:                                 ; preds = %for.cond.cleanup3,
   ret void
 }
 
-; CHECK-LABEL: t5
 ;;  for (int i = 0; i < n; i++)
 ;;   for (int j = 0; j < m; j++)
 ;;    for (int k = 0; k < o; k++)
 ;;      = A[i*m*o + j*o + k]
 ;;     A[i*m*o + j*o + k - o] =
 define void @t5(i32 %n, i32 %m, i32 %o, ptr nocapture %A) {
-; CHECK: da analyze - none!
-; CHECK: da analyze - anti [* * *|<]!
-; CHECK: da analyze - output [* * *]!
+; CHECK-LABEL: 't5'
+; CHECK-NEXT:  Src: %0 = load i32, ptr %arrayidx, align 4 --> Dst: %0 = load i32, ptr %arrayidx, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: %0 = load i32, ptr %arrayidx, align 4 --> Dst: store i32 %add12, ptr %arrayidx2, align 4
+; CHECK-NEXT:    da analyze - anti [* * *|<]!
+; CHECK-NEXT:  Src: store i32 %add12, ptr %arrayidx2, align 4 --> Dst: store i32 %add12, ptr %arrayidx2, align 4
+; CHECK-NEXT:    da analyze - output [* * *]!
+;
 entry:
   %cmp49 = icmp sgt i32 %n, 0
   br i1 %cmp49, label %for.cond1.preheader.lr.ph, label %for.cond.cleanup
@@ -317,16 +338,20 @@ for.cond.cleanup:                                 ; preds = %for.cond.cleanup3,
   ret void
 }
 
-; CHECK-LABEL: t6
 ;;  for (int i = 0; i < n; i++)
 ;;   for (int j = 0; j < m; j++)
 ;;    for (int k = 0; k < o; k++)
 ;;      = A[i*m*o + j*o + k]
 ;;     A[i*m*o + j*o + k + m*o] =
 define void @t6(i32 %n, i32 %m, i32 %o, ptr nocapture %A) {
-; CHECK: da analyze - none!
-; CHECK: da analyze - consistent anti [-1 0 0]!
-; CHECK: da analyze - none!
+; CHECK-LABEL: 't6'
+; CHECK-NEXT:  Src: %0 = load i32, ptr %arrayidx, align 4 --> Dst: %0 = load i32, ptr %arrayidx, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: %0 = load i32, ptr %arrayidx, align 4 --> Dst: store i32 %add12, ptr %arrayidx2, align 4
+; CHECK-NEXT:    da analyze - consistent anti [-1 0 0]!
+; CHECK-NEXT:  Src: store i32 %add12, ptr %arrayidx2, align 4 --> Dst: store i32 %add12, ptr %arrayidx2, align 4
+; CHECK-NEXT:    da analyze - none!
+;
 entry:
   %cmp49 = icmp sgt i32 %n, 0
   br i1 %cmp49, label %for.cond1.preheader.lr.ph, label %for.cond.cleanup
@@ -381,16 +406,20 @@ for.cond.cleanup:                                 ; preds = %for.cond.cleanup3,
   ret void
 }
 
-; CHECK-LABEL: t7
 ;;  for (int i = 0; i < n; i++)
 ;;   for (int j = 0; j < m; j++)
 ;;    for (int k = 0; k < o; k++)
 ;;      = A[i*m*o + j*o + k]
 ;;     A[i*m*o + j*o + k - m*o] =
 define void @t7(i32 %n, i32 %m, i32 %o, ptr nocapture %A) {
-; CHECK: da analyze - none!
-; CHECK: da analyze - consistent anti [1 0 0]!
-; CHECK: da analyze - none!
+; CHECK-LABEL: 't7'
+; CHECK-NEXT:  Src: %0 = load i32, ptr %arrayidx, align 4 --> Dst: %0 = load i32, ptr %arrayidx, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: %0 = load i32, ptr %arrayidx, align 4 --> Dst: store i32 %add12, ptr %arrayidx2, align 4
+; CHECK-NEXT:    da analyze - consistent anti [1 0 0]!
+; CHECK-NEXT:  Src: store i32 %add12, ptr %arrayidx2, align 4 --> Dst: store i32 %add12, ptr %arrayidx2, align 4
+; CHECK-NEXT:    da analyze - none!
+;
 entry:
   %cmp49 = icmp sgt i32 %n, 0
   br i1 %cmp49, label %for.cond1.preheader.lr.ph, label %for.cond.cleanup
@@ -445,16 +474,20 @@ for.cond.cleanup:                                 ; preds = %for.cond.cleanup3,
   ret void
 }
 
-; CHECK-LABEL: t8
 ;;  for (int i = 0; i < n; i++)
 ;;   for (int j = 0; j < m; j++)
 ;;    for (int k = 1; k < o; k++)
 ;;      = A[i*m*o + j*o + k]
 ;;     A[i*m*o + j*o + k - 1] =
 define void @t8(i32 %n, i32 %m, i32 %o, ptr nocapture %A) {
-; CHECK: da analyze - none!
-; CHECK: da analyze - consistent anti [0 0 1]!
-; CHECK: da analyze - none!
+; CHECK-LABEL: 't8'
+; CHECK-NEXT:  Src: %0 = load i32, ptr %arrayidx, align 4 --> Dst: %0 = load i32, ptr %arrayidx, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: %0 = load i32, ptr %arrayidx, align 4 --> Dst: store i32 %add12, ptr %arrayidx2, align 4
+; CHECK-NEXT:    da analyze - consistent anti [0 0 1]!
+; CHECK-NEXT:  Src: store i32 %add12, ptr %arrayidx2, align 4 --> Dst: store i32 %add12, ptr %arrayidx2, align 4
+; CHECK-NEXT:    da analyze - none!
+;
 entry:
   %cmp49 = icmp sgt i32 %n, 0
   br i1 %cmp49, label %for.cond1.preheader.lr.ph, label %for.cond.cleanup
@@ -509,11 +542,15 @@ for.cond.cleanup:                                 ; preds = %for.cond.cleanup3,
 }
 
 
-; CHECK-LABEL: test_sizes
 define double @test_sizes(i16 %h, i16 %N, ptr nocapture %array) {
-; CHECK: da analyze - consistent input [0 S]!
-; CHECK: da analyze - anti [* *|<]!
-; CHECK: da analyze - output [* *]!
+; CHECK-LABEL: 'test_sizes'
+; CHECK-NEXT:  Src: %2 = load i16, ptr %arrayidx, align 4 --> Dst: %2 = load i16, ptr %arrayidx, align 4
+; CHECK-NEXT:    da analyze - consistent input [0 S]!
+; CHECK-NEXT:  Src: %2 = load i16, ptr %arrayidx, align 4 --> Dst: store i16 %add6, ptr %arrayidx8, align 4
+; CHECK-NEXT:    da analyze - anti [* *|<]!
+; CHECK-NEXT:  Src: store i16 %add6, ptr %arrayidx8, align 4 --> Dst: store i16 %add6, ptr %arrayidx8, align 4
+; CHECK-NEXT:    da analyze - output [* *]!
+;
 entry:
   %cmp28 = icmp sgt i16 %N, 1
   br i1 %cmp28, label %for.body.lr.ph, label %for.end12
@@ -557,11 +594,15 @@ for.end12:                                        ; preds = %for.inc10, %entry
 }
 
 
-; CHECK-LABEL: nonnegative
 define void @nonnegative(ptr nocapture %A, i32 %N) {
-; CHECK: da analyze - none!
-; CHECK: da analyze - consistent output [0 0|<]!
-; CHECK: da analyze - none!
+; CHECK-LABEL: 'nonnegative'
+; CHECK-NEXT:  Src: store i32 1, ptr %arrayidx, align 4 --> Dst: store i32 1, ptr %arrayidx, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: store i32 1, ptr %arrayidx, align 4 --> Dst: store i32 2, ptr %arrayidx, align 4
+; CHECK-NEXT:    da analyze - consistent output [0 0|<]!
+; CHECK-NEXT:  Src: store i32 2, ptr %arrayidx, align 4 --> Dst: store i32 2, ptr %arrayidx, align 4
+; CHECK-NEXT:    da analyze - none!
+;
 entry:
   %cmp44 = icmp eq i32 %N, 0
   br i1 %cmp44, label %exit, label %for.outer
diff --git a/llvm/test/Analysis/DependenceAnalysis/Dump.ll b/llvm/test/Analysis/DependenceAnalysis/Dump.ll
index 264b97899b0d2..d80871e2a50ee 100644
--- a/llvm/test/Analysis/DependenceAnalysis/Dump.ll
+++ b/llvm/test/Analysis/DependenceAnalysis/Dump.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --version 5
 ; RUN: opt < %s -disable-output "-passes=print<da>" -aa-pipeline=basic-aa 2>&1 \
 ; RUN: | FileCheck %s
 
@@ -12,16 +13,15 @@
 ;;   }
 ;; }
 
-; CHECK-LABEL: foo
-
-; CHECK: Src:  store float %conv, ptr %arrayidx, align 4 --> Dst:  store float %conv, ptr %arrayidx, align 4
-; CHECK-NEXT:   da analyze - none!
-; CHECK-NEXT: Src:  store float %conv, ptr %arrayidx, align 4 --> Dst:  call void @bar(ptr %A)
-; CHECK-NEXT:   da analyze - confused!
-; CHECK-NEXT: Src:  call void @bar(ptr %A) --> Dst:  call void @bar(ptr %A)
-; CHECK-NEXT:   da analyze - confused!
-
 define void @foo(ptr noalias %A, i32 signext %n) {
+; CHECK-LABEL: 'foo'
+; CHECK-NEXT:  Src: store float %conv, ptr %arrayidx, align 4 --> Dst: store float %conv, ptr %arrayidx, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: store float %conv, ptr %arrayidx, align 4 --> Dst: call void @bar(ptr %A)
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: call void @bar(ptr %A) --> Dst: call void @bar(ptr %A)
+; CHECK-NEXT:    da analyze - confused!
+;
 entry:
   %cmp1 = icmp slt i32 0, %n
   br i1 %cmp1, label %for.body.lr.ph, label %for.end
diff --git a/llvm/test/Analysis/DependenceAnalysis/ExactRDIV.ll b/llvm/test/Analysis/DependenceAnalysis/ExactRDIV.ll
index 4aa2458dcd8cd..b5ece14121686 100644
--- a/llvm/test/Analysis/DependenceAnalysis/ExactRDIV.ll
+++ b/llvm/test/Analysis/DependenceAnalysis/ExactRDIV.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --version 5
 ; RUN: opt < %s -disable-output "-passes=print<da>" -aa-pipeline=basic-aa 2>&1 \
 ; RUN: | FileCheck %s
 
@@ -12,16 +13,23 @@ target triple = "x86_64-apple-macosx10.6.0"
 ;;    *B++ = A[2*j + 1];
 
 define void @rdiv0(ptr %A, ptr %B) nounwind uwtable ssp {
+; CHECK-LABEL: 'rdiv0'
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx, align 4 --> Dst: store i32 %conv, ptr %arrayidx, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx, align 4 --> Dst: %0 = load i32, ptr %arrayidx7, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx, align 4 --> Dst: store i32 %0, ptr %B.addr.01, align 4
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: %0 = load i32, ptr %arrayidx7, align 4 --> Dst: %0 = load i32, ptr %arrayidx7, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: %0 = load i32, ptr %arrayidx7, align 4 --> Dst: store i32 %0, ptr %B.addr.01, align 4
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: store i32 %0, ptr %B.addr.01, align 4 --> Dst: store i32 %0, ptr %B.addr.01, align 4
+; CHECK-NEXT:    da analyze - none!
+;
 entry:
   br label %for.body
 
-; CHECK: da analyze - none!
-; CHECK: da analyze - none!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - none!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - none!
-
 for.body:                                         ; preds = %entry, %for.body
   %i.03 = phi i64 [ 0, %entry ], [ %inc, %for.body ]
   %conv = trunc i64 %i.03 to i32
@@ -60,16 +68,23 @@ for.end10:                                        ; preds = %for.body4
 ;;    *B++ = A[j];
 
 define void @rdiv1(ptr %A, ptr %B) nounwind uwtable ssp {
+; CHECK-LABEL: 'rdiv1'
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx, align 4 --> Dst: store i32 %conv, ptr %arrayidx, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx, align 4 --> Dst: %0 = load i32, ptr %arrayidx5, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx, align 4 --> Dst: store i32 %0, ptr %B.addr.01, align 4
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: %0 = load i32, ptr %arrayidx5, align 4 --> Dst: %0 = load i32, ptr %arrayidx5, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: %0 = load i32, ptr %arrayidx5, align 4 --> Dst: store i32 %0, ptr %B.addr.01, align 4
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: store i32 %0, ptr %B.addr.01, align 4 --> Dst: store i32 %0, ptr %B.addr.01, align 4
+; CHECK-NEXT:    da analyze - none!
+;
 entry:
   br label %for.body
 
-; CHECK: da analyze - none!
-; CHECK: da analyze - none!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - none!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - none!
-
 for.body:                                         ; preds = %entry, %for.body
   %i.03 = phi i64 [ 0, %entry ], [ %inc, %for.body ]
   %conv = trunc i64 %i.03 to i32
@@ -106,16 +121,23 @@ for.end8:                                         ; preds = %for.body4
 ;;    *B++ = A[j];
 
 define void @rdiv2(ptr %A, ptr %B) nounwind uwtable ssp {
+; CHECK-LABEL: 'rdiv2'
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx, align 4 --> Dst: store i32 %conv, ptr %arrayidx, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx, align 4 --> Dst: %0 = load i32, ptr %arrayidx5, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx, align 4 --> Dst: store i32 %0, ptr %B.addr.01, align 4
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: %0 = load i32, ptr %arrayidx5, align 4 --> Dst: %0 = load i32, ptr %arrayidx5, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: %0 = load i32, ptr %arrayidx5, align 4 --> Dst: store i32 %0, ptr %B.addr.01, align 4
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: store i32 %0, ptr %B.addr.01, align 4 --> Dst: store i32 %0, ptr %B.addr.01, align 4
+; CHECK-NEXT:    da analyze - none!
+;
 entry:
   br label %for.body
 
-; CHECK: da analyze - none!
-; CHECK: da analyze - none!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - none!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - none!
-
 for.body:                                         ; preds = %entry, %for.body
   %i.03 = phi i64 [ 0, %entry ], [ %inc, %for.body ]
   %conv = trunc i64 %i.03 to i32
@@ -152,16 +174,23 @@ for.end8:                                         ; preds = %for.body4
 ;;    *B++ = A[j];
 
 define void @rdiv3(ptr %A, ptr %B) nounwind uwtable ssp {
+; CHECK-LABEL: 'rdiv3'
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx, align 4 --> Dst: store i32 %conv, ptr %arrayidx, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx, align 4 --> Dst: %0 = load i32, ptr %arrayidx5, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx, align 4 --> Dst: store i32 %0, ptr %B.addr.01, align 4
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: %0 = load i32, ptr %arrayidx5, align 4 --> Dst: %0 = load i32, ptr %arrayidx5, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: %0 = load i32, ptr %arrayidx5, align 4 --> Dst: store i32 %0, ptr %B.addr.01, align 4
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: store i32 %0, ptr %B.addr.01, align 4 --> Dst: store i32 %0, ptr %B.addr.01, align 4
+; CHECK-NEXT:    da analyze - none!
+;
 entry:
   br label %for.body
 
-; CHECK: da analyze - none!
-; CHECK: da analyze - none!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - none!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - none!
-
 for.body:                                         ; preds = %entry, %for.body
   %i.03 = phi i64 [ 0, %entry ], [ %inc, %for.body ]
   %conv = trunc i64 %i.03 to i32
@@ -198,16 +227,23 @@ for.end8:                                         ; preds = %for.body4
 ;;    *B++ = A[j];
 
 define void @rdiv4(ptr %A, ptr %B) nounwind uwtable ssp {
+; CHECK-LABEL: 'rdiv4'
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx, align 4 --> Dst: store i32 %conv, ptr %arrayidx, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx, align 4 --> Dst: %0 = load i32, ptr %arrayidx5, align 4
+; CHECK-NEXT:    da analyze - flow [|<]!
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx, align 4 --> Dst: store i32 %0, ptr %B.addr.01, align 4
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: %0 = load i32, ptr %arrayidx5, align 4 --> Dst: %0 = load i32, ptr %arrayidx5, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: %0 = load i32, ptr %arrayidx5, align 4 --> Dst: store i32 %0, ptr %B.addr.01, align 4
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: store i32 %0, ptr %B.addr.01, align 4 --> Dst: store i32 %0, ptr %B.addr.01, align 4
+; CHECK-NEXT:    da analyze - none!
+;
 entry:
   br label %for.body
 
-; CHECK: da analyze - none!
-; CHECK: da analyze - flow [|<]!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - none!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - none!
-
 for.body:                                         ; preds = %entry, %for.body
   %i.03 = phi i64 [ 0, %entry ], [ %inc, %for.body ]
   %conv = trunc i64 %i.03 to i32
@@ -244,16 +280,23 @@ for.end8:                                         ; preds = %for.body4
 ;;    *B++ = A[-j];
 
 define void @rdiv5(ptr %A, ptr %B) nounwind uwtable ssp {
+; CHECK-LABEL: 'rdiv5'
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx, align 4 --> Dst: store i32 %conv, ptr %arrayidx, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx, align 4 --> Dst: %0 = load i32, ptr %arrayidx5, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx, align 4 --> Dst: store i32 %0, ptr %B.addr.01, align 4
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: %0 = load i32, ptr %arrayidx5, align 4 --> Dst: %0 = load i32, ptr %arrayidx5, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: %0 = load i32, ptr %arrayidx5, align 4 --> Dst: store i32 %0, ptr %B.addr.01, align 4
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: store i32 %0, ptr %B.addr.01, align 4 --> Dst: store i32 %0, ptr %B.addr.01, align 4
+; CHECK-NEXT:    da analyze - none!
+;
 entry:
   br label %for.body
 
-; CHECK: da analyze - none!
-; CHECK: da analyze - none!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - none!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - none!
-
 for.body:                                         ; preds = %entry, %for.body
   %i.03 = phi i64 [ 0, %entry ], [ %inc, %for.body ]
   %conv = trunc i64 %i.03 to i32
@@ -291,16 +334,23 @@ for.end8:                                         ; preds = %for.body4
 ;;    *B++ = A[-j];
 
 define void @rdiv6(ptr %A, ptr %B) nounwind uwtable ssp {
+; CHECK-LABEL: 'rdiv6'
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx, align 4 --> Dst: store i32 %conv, ptr %arrayidx, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx, align 4 --> Dst: %0 = load i32, ptr %arrayidx5, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx, align 4 --> Dst: store i32 %0, ptr %B.addr.01, align 4
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: %0 = load i32, ptr %arrayidx5, align 4 --> Dst: %0 = load i32, ptr %arrayidx5, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: %0 = load i32, ptr %arrayidx5, align 4 --> Dst: store i32 %0, ptr %B.addr.01, align 4
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: store i32 %0, ptr %B.addr.01, align 4 --> Dst: store i32 %0, ptr %B.addr.01, align 4
+; CHECK-NEXT:    da analyze - none!
+;
 entry:
   br label %for.body
 
-; CHECK: da analyze - none!
-; CHECK: da analyze - none!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - none!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - none!
-
 for.body:                                         ; preds = %entry, %for.body
   %i.03 = phi i64 [ 0, %entry ], [ %inc, %for.body ]
   %conv = trunc i64 %i.03 to i32
@@ -338,16 +388,23 @@ for.end8:                                         ; preds = %for.body4
 ;;    *B++ = A[-j];
 
 define void @rdiv7(ptr %A, ptr %B) nounwind uwtable ssp {
+; CHECK-LABEL: 'rdiv7'
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx, align 4 --> Dst: store i32 %conv, ptr %arrayidx, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx, align 4 --> Dst: %0 = load i32, ptr %arrayidx5, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx, align 4 --> Dst: store i32 %0, ptr %B.addr.01, align 4
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: %0 = load i32, ptr %arrayidx5, align 4 --> Dst: %0 = load i32, ptr %arrayidx5, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: %0 = load i32, ptr %arrayidx5, align 4 --> Dst: store i32 %0, ptr %B.addr.01, align 4
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: store i32 %0, ptr %B.addr.01, align 4 --> Dst: store i32 %0, ptr %B.addr.01, align 4
+; CHECK-NEXT:    da analyze - none!
+;
 entry:
   br label %for.body
 
-; CHECK: da analyze - none!
-; CHECK: da analyze - none!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - none!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - none!
-
 for.body:                                         ; preds = %entry, %for.body
   %i.03 = phi i64 [ 0, %entry ], [ %inc, %for.body ]
   %conv = trunc i64 %i.03 to i32
@@ -385,16 +442,23 @@ for.end8:                                         ; preds = %for.body4
 ;;    *B++ = A[-j];
 
 define void @rdiv8(ptr %A, ptr %B) nounwind uwtable ssp {
+; CHECK-LABEL: 'rdiv8'
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx, align 4 --> Dst: store i32 %conv, ptr %arrayidx, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx, align 4 --> Dst: %0 = load i32, ptr %arrayidx5, align 4
+; CHECK-NEXT:    da analyze - flow [|<]!
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx, align 4 --> Dst: store i32 %0, ptr %B.addr.01, align 4
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: %0 = load i32, ptr %arrayidx5, align 4 --> Dst: %0 = load i32, ptr %arrayidx5, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: %0 = load i32, ptr %arrayidx5, align 4 --> Dst: store i32 %0, ptr %B.addr.01, align 4
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: store i32 %0, ptr %B.addr.01, align 4 --> Dst: store i32 %0, ptr %B.addr.01, align 4
+; CHECK-NEXT:    da analyze - none!
+;
 entry:
   br label %for.body
 
-; CHECK: da analyze - none!
-; CHECK: da analyze - flow [|<]!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - none!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - none!
-
 for.body:                                         ; preds = %entry, %for.body
   %i.03 = phi i64 [ 0, %entry ], [ %inc, %for.body ]
   %conv = trunc i64 %i.03 to i32
@@ -432,16 +496,23 @@ for.end8:                                         ; preds = %for.body4
 ;;      *B++ = A[45];
 
 define void @rdiv9(ptr %A, ptr %B) nounwind uwtable ssp {
+; CHECK-LABEL: 'rdiv9'
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx, align 4 --> Dst: store i32 %conv, ptr %arrayidx, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx, align 4 --> Dst: %0 = load i32, ptr %arrayidx4, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx, align 4 --> Dst: store i32 %0, ptr %B.addr.11, align 4
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: %0 = load i32, ptr %arrayidx4, align 4 --> Dst: %0 = load i32, ptr %arrayidx4, align 4
+; CHECK-NEXT:    da analyze - consistent input [S S]!
+; CHECK-NEXT:  Src: %0 = load i32, ptr %arrayidx4, align 4 --> Dst: store i32 %0, ptr %B.addr.11, align 4
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: store i32 %0, ptr %B.addr.11, align 4 --> Dst: store i32 %0, ptr %B.addr.11, align 4
+; CHECK-NEXT:    da analyze - none!
+;
 entry:
   br label %for.cond1.preheader
 
-; CHECK: da analyze - none!
-; CHECK: da analyze - none!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - consistent input [S S]!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - none!
-
 for.cond1.preheader:                              ; preds = %entry, %for.inc5
   %B.addr.04 = phi ptr [ %B, %entry ], [ %scevgep, %for.inc5 ]
   %i.03 = phi i64 [ 0, %entry ], [ %inc6, %for.inc5 ]
@@ -481,16 +552,23 @@ for.end7:                                         ; preds = %for.inc5
 ;;      *B++ = A[45];
 
 define void @rdiv10(ptr %A, ptr %B) nounwind uwtable ssp {
+; CHECK-LABEL: 'rdiv10'
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx, align 4 --> Dst: store i32 %conv, ptr %arrayidx, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx, align 4 --> Dst: %0 = load i32, ptr %arrayidx4, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx, align 4 --> Dst: store i32 %0, ptr %B.addr.11, align 4
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: %0 = load i32, ptr %arrayidx4, align 4 --> Dst: %0 = load i32, ptr %arrayidx4, align 4
+; CHECK-NEXT:    da analyze - consistent input [S S]!
+; CHECK-NEXT:  Src: %0 = load i32, ptr %arrayidx4, align 4 --> Dst: store i32 %0, ptr %B.addr.11, align 4
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: store i32 %0, ptr %B.addr.11, align 4 --> Dst: store i32 %0, ptr %B.addr.11, align 4
+; CHECK-NEXT:    da analyze - none!
+;
 entry:
   br label %for.cond1.preheader
 
-; CHECK: da analyze - none!
-; CHECK: da analyze - none!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - consistent input [S S]!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - none!
-
 for.cond1.preheader:                              ; preds = %entry, %for.inc5
   %B.addr.04 = phi ptr [ %B, %entry ], [ %scevgep, %for.inc5 ]
   %i.03 = phi i64 [ 0, %entry ], [ %inc6, %for.inc5 ]
@@ -529,16 +607,23 @@ for.end7:                                         ; preds = %for.inc5
 ;;      *B++ = A[45];
 
 define void @rdiv11(ptr %A, ptr %B) nounwind uwtable ssp {
+; CHECK-LABEL: 'rdiv11'
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx, align 4 --> Dst: store i32 %conv, ptr %arrayidx, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx, align 4 --> Dst: %0 = load i32, ptr %arrayidx4, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx, align 4 --> Dst: store i32 %0, ptr %B.addr.11, align 4
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: %0 = load i32, ptr %arrayidx4, align 4 --> Dst: %0 = load i32, ptr %arrayidx4, align 4
+; CHECK-NEXT:    da analyze - consistent input [S S]!
+; CHECK-NEXT:  Src: %0 = load i32, ptr %arrayidx4, align 4 --> Dst: store i32 %0, ptr %B.addr.11, align 4
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: store i32 %0, ptr %B.addr.11, align 4 --> Dst: store i32 %0, ptr %B.addr.11, align 4
+; CHECK-NEXT:    da analyze - none!
+;
 entry:
   br label %for.cond1.preheader
 
-; CHECK: da analyze - none!
-; CHECK: da analyze - none!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - consistent input [S S]!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - none!
-
 for.cond1.preheader:                              ; preds = %entry, %for.inc5
   %B.addr.04 = phi ptr [ %B, %entry ], [ %scevgep, %for.inc5 ]
   %i.03 = phi i64 [ 0, %entry ], [ %inc6, %for.inc5 ]
@@ -577,16 +662,23 @@ for.end7:                                         ; preds = %for.inc5
 ;;      *B++ = A[45];
 
 define void @rdiv12(ptr %A, ptr %B) nounwind uwtable ssp {
+; CHECK-LABEL: 'rdiv12'
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx, align 4 --> Dst: store i32 %conv, ptr %arrayidx, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx, align 4 --> Dst: %0 = load i32, ptr %arrayidx4, align 4
+; CHECK-NEXT:    da analyze - flow [* *|<]!
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx, align 4 --> Dst: store i32 %0, ptr %B.addr.11, align 4
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: %0 = load i32, ptr %arrayidx4, align 4 --> Dst: %0 = load i32, ptr %arrayidx4, align 4
+; CHECK-NEXT:    da analyze - consistent input [S S]!
+; CHECK-NEXT:  Src: %0 = load i32, ptr %arrayidx4, align 4 --> Dst: store i32 %0, ptr %B.addr.11, align 4
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: store i32 %0, ptr %B.addr.11, align 4 --> Dst: store i32 %0, ptr %B.addr.11, align 4
+; CHECK-NEXT:    da analyze - none!
+;
 entry:
   br label %for.cond1.preheader
 
-; CHECK: da analyze - none!
-; CHECK: da analyze - flow [* *|<]!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - consistent input [S S]!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - none!
-
 for.cond1.preheader:                              ; preds = %entry, %for.inc5
   %B.addr.04 = phi ptr [ %B, %entry ], [ %scevgep, %for.inc5 ]
   %i.03 = phi i64 [ 0, %entry ], [ %inc6, %for.inc5 ]
diff --git a/llvm/test/Analysis/DependenceAnalysis/ExactSIV.ll b/llvm/test/Analysis/DependenceAnalysis/ExactSIV.ll
index 7822e61cf3aa0..b6b44ad4bfc53 100644
--- a/llvm/test/Analysis/DependenceAnalysis/ExactSIV.ll
+++ b/llvm/test/Analysis/DependenceAnalysis/ExactSIV.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --version 5
 ; RUN: opt < %s -disable-output "-passes=print<da>" -aa-pipeline=basic-aa 2>&1 \
 ; RUN: | FileCheck %s
 
@@ -10,17 +11,23 @@ target triple = "x86_64-apple-macosx10.6.0"
 ;;    *B++ = A[2*i + 1];
 
 define void @exact0(ptr %A, ptr %B) nounwind uwtable ssp {
+; CHECK-LABEL: 'exact0'
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx, align 4 --> Dst: store i32 %conv, ptr %arrayidx, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx, align 4 --> Dst: %0 = load i32, ptr %arrayidx2, align 4
+; CHECK-NEXT:    da analyze - flow [<=|<]!
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx, align 4 --> Dst: store i32 %0, ptr %B.addr.01, align 4
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: %0 = load i32, ptr %arrayidx2, align 4 --> Dst: %0 = load i32, ptr %arrayidx2, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: %0 = load i32, ptr %arrayidx2, align 4 --> Dst: store i32 %0, ptr %B.addr.01, align 4
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: store i32 %0, ptr %B.addr.01, align 4 --> Dst: store i32 %0, ptr %B.addr.01, align 4
+; CHECK-NEXT:    da analyze - none!
+;
 entry:
   br label %for.body
 
-; CHECK-LABEL: exact0
-; CHECK: da analyze - none!
-; CHECK: da analyze - flow [<=|<]!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - none!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - none!
-
 for.body:                                         ; preds = %entry, %for.body
   %i.02 = phi i64 [ 0, %entry ], [ %inc, %for.body ]
   %B.addr.01 = phi ptr [ %B, %entry ], [ %incdec.ptr, %for.body ]
@@ -48,17 +55,23 @@ for.end:                                          ; preds = %for.body
 ;;    *B++ = A[2*i + 1];
 
 define void @exact1(ptr %A, ptr %B) nounwind uwtable ssp {
+; CHECK-LABEL: 'exact1'
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx, align 4 --> Dst: store i32 %conv, ptr %arrayidx, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx, align 4 --> Dst: %0 = load i32, ptr %arrayidx3, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx, align 4 --> Dst: store i32 %0, ptr %B.addr.01, align 4
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: %0 = load i32, ptr %arrayidx3, align 4 --> Dst: %0 = load i32, ptr %arrayidx3, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: %0 = load i32, ptr %arrayidx3, align 4 --> Dst: store i32 %0, ptr %B.addr.01, align 4
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: store i32 %0, ptr %B.addr.01, align 4 --> Dst: store i32 %0, ptr %B.addr.01, align 4
+; CHECK-NEXT:    da analyze - none!
+;
 entry:
   br label %for.body
 
-; CHECK-LABEL: exact1
-; CHECK: da analyze - none!
-; CHECK: da analyze - none!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - none!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - none!
-
 for.body:                                         ; preds = %entry, %for.body
   %i.02 = phi i64 [ 0, %entry ], [ %inc, %for.body ]
   %B.addr.01 = phi ptr [ %B, %entry ], [ %incdec.ptr, %for.body ]
@@ -87,17 +100,23 @@ for.end:                                          ; preds = %for.body
 ;;    *B++ = A[i + 60];
 
 define void @exact2(ptr %A, ptr %B) nounwind uwtable ssp {
+; CHECK-LABEL: 'exact2'
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx, align 4 --> Dst: store i32 %conv, ptr %arrayidx, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx, align 4 --> Dst: %0 = load i32, ptr %arrayidx1, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx, align 4 --> Dst: store i32 %0, ptr %B.addr.01, align 4
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: %0 = load i32, ptr %arrayidx1, align 4 --> Dst: %0 = load i32, ptr %arrayidx1, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: %0 = load i32, ptr %arrayidx1, align 4 --> Dst: store i32 %0, ptr %B.addr.01, align 4
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: store i32 %0, ptr %B.addr.01, align 4 --> Dst: store i32 %0, ptr %B.addr.01, align 4
+; CHECK-NEXT:    da analyze - none!
+;
 entry:
   br label %for.body
 
-; CHECK-LABEL: exact2
-; CHECK: da analyze - none!
-; CHECK: da analyze - none!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - none!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - none!
-
 for.body:                                         ; preds = %entry, %for.body
   %i.02 = phi i64 [ 0, %entry ], [ %inc, %for.body ]
   %B.addr.01 = phi ptr [ %B, %entry ], [ %incdec.ptr, %for.body ]
@@ -124,17 +143,23 @@ for.end:                                          ; preds = %for.body
 ;;    *B++ = A[i + 60];
 
 define void @exact3(ptr %A, ptr %B) nounwind uwtable ssp {
+; CHECK-LABEL: 'exact3'
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx, align 4 --> Dst: store i32 %conv, ptr %arrayidx, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx, align 4 --> Dst: %0 = load i32, ptr %arrayidx1, align 4
+; CHECK-NEXT:    da analyze - flow [>]!
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx, align 4 --> Dst: store i32 %0, ptr %B.addr.01, align 4
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: %0 = load i32, ptr %arrayidx1, align 4 --> Dst: %0 = load i32, ptr %arrayidx1, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: %0 = load i32, ptr %arrayidx1, align 4 --> Dst: store i32 %0, ptr %B.addr.01, align 4
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: store i32 %0, ptr %B.addr.01, align 4 --> Dst: store i32 %0, ptr %B.addr.01, align 4
+; CHECK-NEXT:    da analyze - none!
+;
 entry:
   br label %for.body
 
-; CHECK-LABEL: exact3
-; CHECK: da analyze - none!
-; CHECK: da analyze - flow [>]!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - none!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - none!
-
 for.body:                                         ; preds = %entry, %for.body
   %i.02 = phi i64 [ 0, %entry ], [ %inc, %for.body ]
   %B.addr.01 = phi ptr [ %B, %entry ], [ %incdec.ptr, %for.body ]
@@ -161,17 +186,23 @@ for.end:                                          ; preds = %for.body
 ;;    *B++ = A[i + 60];
 
 define void @exact4(ptr %A, ptr %B) nounwind uwtable ssp {
+; CHECK-LABEL: 'exact4'
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx, align 4 --> Dst: store i32 %conv, ptr %arrayidx, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx, align 4 --> Dst: %0 = load i32, ptr %arrayidx1, align 4
+; CHECK-NEXT:    da analyze - flow [>]!
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx, align 4 --> Dst: store i32 %0, ptr %B.addr.01, align 4
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: %0 = load i32, ptr %arrayidx1, align 4 --> Dst: %0 = load i32, ptr %arrayidx1, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: %0 = load i32, ptr %arrayidx1, align 4 --> Dst: store i32 %0, ptr %B.addr.01, align 4
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: store i32 %0, ptr %B.addr.01, align 4 --> Dst: store i32 %0, ptr %B.addr.01, align 4
+; CHECK-NEXT:    da analyze - none!
+;
 entry:
   br label %for.body
 
-; CHECK-LABEL: exact4
-; CHECK: da analyze - none!
-; CHECK: da analyze - flow [>]!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - none!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - none!
-
 for.body:                                         ; preds = %entry, %for.body
   %i.02 = phi i64 [ 0, %entry ], [ %inc, %for.body ]
   %B.addr.01 = phi ptr [ %B, %entry ], [ %incdec.ptr, %for.body ]
@@ -198,17 +229,23 @@ for.end:                                          ; preds = %for.body
 ;;    *B++ = A[i + 60];
 
 define void @exact5(ptr %A, ptr %B) nounwind uwtable ssp {
+; CHECK-LABEL: 'exact5'
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx, align 4 --> Dst: store i32 %conv, ptr %arrayidx, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx, align 4 --> Dst: %0 = load i32, ptr %arrayidx1, align 4
+; CHECK-NEXT:    da analyze - flow [=>|<]!
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx, align 4 --> Dst: store i32 %0, ptr %B.addr.01, align 4
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: %0 = load i32, ptr %arrayidx1, align 4 --> Dst: %0 = load i32, ptr %arrayidx1, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: %0 = load i32, ptr %arrayidx1, align 4 --> Dst: store i32 %0, ptr %B.addr.01, align 4
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: store i32 %0, ptr %B.addr.01, align 4 --> Dst: store i32 %0, ptr %B.addr.01, align 4
+; CHECK-NEXT:    da analyze - none!
+;
 entry:
   br label %for.body
 
-; CHECK-LABEL: exact5
-; CHECK: da analyze - none!
-; CHECK: da analyze - flow [=>|<]!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - none!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - none!
-
 for.body:                                         ; preds = %entry, %for.body
   %i.02 = phi i64 [ 0, %entry ], [ %inc, %for.body ]
   %B.addr.01 = phi ptr [ %B, %entry ], [ %incdec.ptr, %for.body ]
@@ -235,17 +272,23 @@ for.end:                                          ; preds = %for.body
 ;;    *B++ = A[i + 60];
 
 define void @exact6(ptr %A, ptr %B) nounwind uwtable ssp {
+; CHECK-LABEL: 'exact6'
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx, align 4 --> Dst: store i32 %conv, ptr %arrayidx, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx, align 4 --> Dst: %0 = load i32, ptr %arrayidx1, align 4
+; CHECK-NEXT:    da analyze - flow [=>|<]!
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx, align 4 --> Dst: store i32 %0, ptr %B.addr.01, align 4
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: %0 = load i32, ptr %arrayidx1, align 4 --> Dst: %0 = load i32, ptr %arrayidx1, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: %0 = load i32, ptr %arrayidx1, align 4 --> Dst: store i32 %0, ptr %B.addr.01, align 4
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: store i32 %0, ptr %B.addr.01, align 4 --> Dst: store i32 %0, ptr %B.addr.01, align 4
+; CHECK-NEXT:    da analyze - none!
+;
 entry:
   br label %for.body
 
-; CHECK-LABEL: exact6
-; CHECK: da analyze - none!
-; CHECK: da analyze - flow [=>|<]!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - none!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - none!
-
 for.body:                                         ; preds = %entry, %for.body
   %i.02 = phi i64 [ 0, %entry ], [ %inc, %for.body ]
   %B.addr.01 = phi ptr [ %B, %entry ], [ %incdec.ptr, %for.body ]
@@ -272,17 +315,23 @@ for.end:                                          ; preds = %for.body
 ;;    *B++ = A[i + 60];
 
 define void @exact7(ptr %A, ptr %B) nounwind uwtable ssp {
+; CHECK-LABEL: 'exact7'
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx, align 4 --> Dst: store i32 %conv, ptr %arrayidx, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx, align 4 --> Dst: %0 = load i32, ptr %arrayidx1, align 4
+; CHECK-NEXT:    da analyze - flow [*|<]!
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx, align 4 --> Dst: store i32 %0, ptr %B.addr.01, align 4
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: %0 = load i32, ptr %arrayidx1, align 4 --> Dst: %0 = load i32, ptr %arrayidx1, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: %0 = load i32, ptr %arrayidx1, align 4 --> Dst: store i32 %0, ptr %B.addr.01, align 4
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: store i32 %0, ptr %B.addr.01, align 4 --> Dst: store i32 %0, ptr %B.addr.01, align 4
+; CHECK-NEXT:    da analyze - none!
+;
 entry:
   br label %for.body
 
-; CHECK-LABEL: exact7
-; CHECK: da analyze - none!
-; CHECK: da analyze - flow [*|<]!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - none!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - none!
-
 for.body:                                         ; preds = %entry, %for.body
   %i.02 = phi i64 [ 0, %entry ], [ %inc, %for.body ]
   %B.addr.01 = phi ptr [ %B, %entry ], [ %incdec.ptr, %for.body ]
@@ -309,17 +358,23 @@ for.end:                                          ; preds = %for.body
 ;;    *B++ = A[-i - 60];
 
 define void @exact8(ptr %A, ptr %B) nounwind uwtable ssp {
+; CHECK-LABEL: 'exact8'
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx, align 4 --> Dst: store i32 %conv, ptr %arrayidx, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx, align 4 --> Dst: %0 = load i32, ptr %arrayidx2, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx, align 4 --> Dst: store i32 %0, ptr %B.addr.01, align 4
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: %0 = load i32, ptr %arrayidx2, align 4 --> Dst: %0 = load i32, ptr %arrayidx2, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: %0 = load i32, ptr %arrayidx2, align 4 --> Dst: store i32 %0, ptr %B.addr.01, align 4
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: store i32 %0, ptr %B.addr.01, align 4 --> Dst: store i32 %0, ptr %B.addr.01, align 4
+; CHECK-NEXT:    da analyze - none!
+;
 entry:
   br label %for.body
 
-; CHECK-LABEL: exact8
-; CHECK: da analyze - none!
-; CHECK: da analyze - none!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - none!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - none!
-
 for.body:                                         ; preds = %entry, %for.body
   %i.02 = phi i64 [ 0, %entry ], [ %inc, %for.body ]
   %B.addr.01 = phi ptr [ %B, %entry ], [ %incdec.ptr, %for.body ]
@@ -346,17 +401,23 @@ for.end:                                          ; preds = %for.body
 ;;    *B++ = A[-i - 60];
 
 define void @exact9(ptr %A, ptr %B) nounwind uwtable ssp {
+; CHECK-LABEL: 'exact9'
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx, align 4 --> Dst: store i32 %conv, ptr %arrayidx, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx, align 4 --> Dst: %0 = load i32, ptr %arrayidx2, align 4
+; CHECK-NEXT:    da analyze - flow [>]!
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx, align 4 --> Dst: store i32 %0, ptr %B.addr.01, align 4
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: %0 = load i32, ptr %arrayidx2, align 4 --> Dst: %0 = load i32, ptr %arrayidx2, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: %0 = load i32, ptr %arrayidx2, align 4 --> Dst: store i32 %0, ptr %B.addr.01, align 4
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: store i32 %0, ptr %B.addr.01, align 4 --> Dst: store i32 %0, ptr %B.addr.01, align 4
+; CHECK-NEXT:    da analyze - none!
+;
 entry:
   br label %for.body
 
-; CHECK-LABEL: exact9
-; CHECK: da analyze - none!
-; CHECK: da analyze - flow [>]!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - none!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - none!
-
 for.body:                                         ; preds = %entry, %for.body
   %i.02 = phi i64 [ 0, %entry ], [ %inc, %for.body ]
   %B.addr.01 = phi ptr [ %B, %entry ], [ %incdec.ptr, %for.body ]
@@ -383,17 +444,23 @@ for.end:                                          ; preds = %for.body
 ;;    *B++ = A[-i - 60];
 
 define void @exact10(ptr %A, ptr %B) nounwind uwtable ssp {
+; CHECK-LABEL: 'exact10'
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx, align 4 --> Dst: store i32 %conv, ptr %arrayidx, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx, align 4 --> Dst: %0 = load i32, ptr %arrayidx2, align 4
+; CHECK-NEXT:    da analyze - flow [>]!
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx, align 4 --> Dst: store i32 %0, ptr %B.addr.01, align 4
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: %0 = load i32, ptr %arrayidx2, align 4 --> Dst: %0 = load i32, ptr %arrayidx2, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: %0 = load i32, ptr %arrayidx2, align 4 --> Dst: store i32 %0, ptr %B.addr.01, align 4
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: store i32 %0, ptr %B.addr.01, align 4 --> Dst: store i32 %0, ptr %B.addr.01, align 4
+; CHECK-NEXT:    da analyze - none!
+;
 entry:
   br label %for.body
 
-; CHECK-LABEL: exact10
-; CHECK: da analyze - none!
-; CHECK: da analyze - flow [>]!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - none!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - none!
-
 for.body:                                         ; preds = %entry, %for.body
   %i.02 = phi i64 [ 0, %entry ], [ %inc, %for.body ]
   %B.addr.01 = phi ptr [ %B, %entry ], [ %incdec.ptr, %for.body ]
@@ -420,17 +487,23 @@ for.end:                                          ; preds = %for.body
 ;;    *B++ = A[-i - 60];
 
 define void @exact11(ptr %A, ptr %B) nounwind uwtable ssp {
+; CHECK-LABEL: 'exact11'
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx, align 4 --> Dst: store i32 %conv, ptr %arrayidx, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx, align 4 --> Dst: %0 = load i32, ptr %arrayidx2, align 4
+; CHECK-NEXT:    da analyze - flow [=>|<]!
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx, align 4 --> Dst: store i32 %0, ptr %B.addr.01, align 4
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: %0 = load i32, ptr %arrayidx2, align 4 --> Dst: %0 = load i32, ptr %arrayidx2, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: %0 = load i32, ptr %arrayidx2, align 4 --> Dst: store i32 %0, ptr %B.addr.01, align 4
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: store i32 %0, ptr %B.addr.01, align 4 --> Dst: store i32 %0, ptr %B.addr.01, align 4
+; CHECK-NEXT:    da analyze - none!
+;
 entry:
   br label %for.body
 
-; CHECK-LABEL: exact11
-; CHECK: da analyze - none!
-; CHECK: da analyze - flow [=>|<]!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - none!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - none!
-
 for.body:                                         ; preds = %entry, %for.body
   %i.02 = phi i64 [ 0, %entry ], [ %inc, %for.body ]
   %B.addr.01 = phi ptr [ %B, %entry ], [ %incdec.ptr, %for.body ]
@@ -457,17 +530,23 @@ for.end:                                          ; preds = %for.body
 ;;    *B++ = A[-i - 60];
 
 define void @exact12(ptr %A, ptr %B) nounwind uwtable ssp {
+; CHECK-LABEL: 'exact12'
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx, align 4 --> Dst: store i32 %conv, ptr %arrayidx, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx, align 4 --> Dst: %0 = load i32, ptr %arrayidx2, align 4
+; CHECK-NEXT:    da analyze - flow [=>|<]!
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx, align 4 --> Dst: store i32 %0, ptr %B.addr.01, align 4
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: %0 = load i32, ptr %arrayidx2, align 4 --> Dst: %0 = load i32, ptr %arrayidx2, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: %0 = load i32, ptr %arrayidx2, align 4 --> Dst: store i32 %0, ptr %B.addr.01, align 4
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: store i32 %0, ptr %B.addr.01, align 4 --> Dst: store i32 %0, ptr %B.addr.01, align 4
+; CHECK-NEXT:    da analyze - none!
+;
 entry:
   br label %for.body
 
-; CHECK-LABEL: exact12
-; CHECK: da analyze - none!
-; CHECK: da analyze - flow [=>|<]!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - none!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - none!
-
 for.body:                                         ; preds = %entry, %for.body
   %i.02 = phi i64 [ 0, %entry ], [ %inc, %for.body ]
   %B.addr.01 = phi ptr [ %B, %entry ], [ %incdec.ptr, %for.body ]
@@ -494,17 +573,23 @@ for.end:                                          ; preds = %for.body
 ;;    *B++ = A[-i - 60];
 
 define void @exact13(ptr %A, ptr %B) nounwind uwtable ssp {
+; CHECK-LABEL: 'exact13'
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx, align 4 --> Dst: store i32 %conv, ptr %arrayidx, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx, align 4 --> Dst: %0 = load i32, ptr %arrayidx2, align 4
+; CHECK-NEXT:    da analyze - flow [*|<]!
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx, align 4 --> Dst: store i32 %0, ptr %B.addr.01, align 4
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: %0 = load i32, ptr %arrayidx2, align 4 --> Dst: %0 = load i32, ptr %arrayidx2, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: %0 = load i32, ptr %arrayidx2, align 4 --> Dst: store i32 %0, ptr %B.addr.01, align 4
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: store i32 %0, ptr %B.addr.01, align 4 --> Dst: store i32 %0, ptr %B.addr.01, align 4
+; CHECK-NEXT:    da analyze - none!
+;
 entry:
   br label %for.body
 
-; CHECK-LABEL: exact13
-; CHECK: da analyze - none!
-; CHECK: da analyze - flow [*|<]!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - none!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - none!
-
 for.body:                                         ; preds = %entry, %for.body
   %i.02 = phi i64 [ 0, %entry ], [ %inc, %for.body ]
   %B.addr.01 = phi ptr [ %B, %entry ], [ %incdec.ptr, %for.body ]
diff --git a/llvm/test/Analysis/DependenceAnalysis/FlipFlopBaseAddress.ll b/llvm/test/Analysis/DependenceAnalysis/FlipFlopBaseAddress.ll
new file mode 100644
index 0000000000000..7fad0328fdaeb
--- /dev/null
+++ b/llvm/test/Analysis/DependenceAnalysis/FlipFlopBaseAddress.ll
@@ -0,0 +1,159 @@
+; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --version 5
+; RUN: opt < %s -disable-output "-passes=print<da>" -aa-pipeline=basic-aa 2>&1 \
+; RUN: | FileCheck %s
+
+; Check that dependence analysis correctly handles flip-flop of base addresses.
+; Bug 41488 - https://github.com/llvm/llvm-project/issues/41488
+
+define float @bug41488_test1(float %f) {
+; CHECK-LABEL: 'bug41488_test1'
+; CHECK-NEXT:  Src: %0 = load float, ptr %p, align 4 --> Dst: %0 = load float, ptr %p, align 4
+; CHECK-NEXT:    da analyze - input [*]!
+; CHECK-NEXT:  Src: %0 = load float, ptr %p, align 4 --> Dst: store float %f, ptr %q, align 4
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: store float %f, ptr %q, align 4 --> Dst: store float %f, ptr %q, align 4
+; CHECK-NEXT:    da analyze - output [*]!
+;
+entry:
+  %g = alloca float, align 4
+  %h = alloca float, align 4
+  br label %for.body
+
+for.body:
+  %p = phi float* [ %g, %entry ], [ %q, %for.body ]
+  %q = phi float* [ %h, %entry ], [ %p, %for.body ]
+  %0 = load float, float* %p, align 4
+  store float %f, float* %q, align 4
+  %branch_cond = fcmp ugt float %0, 0.0
+  br i1 %branch_cond, label %for.cond.cleanup, label %for.body
+
+for.cond.cleanup:
+  ret float %f
+}
+
+define void @bug41488_test2(i32 %n) {
+; CHECK-LABEL: 'bug41488_test2'
+; CHECK-NEXT:  Src: %0 = load float, ptr %p, align 4 --> Dst: %0 = load float, ptr %p, align 4
+; CHECK-NEXT:    da analyze - input [*]!
+; CHECK-NEXT:  Src: %0 = load float, ptr %p, align 4 --> Dst: store float 0.000000e+00, ptr %q, align 4
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: store float 0.000000e+00, ptr %q, align 4 --> Dst: store float 0.000000e+00, ptr %q, align 4
+; CHECK-NEXT:    da analyze - output [*]!
+;
+entry:
+  %g = alloca float, align 4
+  %h = alloca float, align 4
+  br label %for.body
+
+for.body:
+  %i = phi i32 [0, %entry ], [ %inc, %for.body ]
+  %p = phi float* [ %g, %entry ], [ %q, %for.body ]
+  %q = phi float* [ %h, %entry ], [ %p, %for.body ]
+  %0 = load float, float* %p, align 4
+  store float 0.0, float* %q, align 4
+  %inc = add nuw i32 %i, 1
+  %branch_cond = icmp ult i32 %i, %n
+  br i1 %branch_cond, label %for.body, label %for.cond.cleanup
+
+for.cond.cleanup:
+  ret void
+}
+
+; Bug 53942 - https://github.com/llvm/llvm-project/issues/53942
+
+define void @bug53942_foo(i32 noundef %n, ptr noalias nocapture noundef writeonly %A, ptr noalias nocapture noundef %B) {
+; CHECK-LABEL: 'bug53942_foo'
+; CHECK-NEXT:  Src: %.pre = load double, ptr %B, align 8 --> Dst: %.pre = load double, ptr %B, align 8
+; CHECK-NEXT:    da analyze - consistent input [S]!
+; CHECK-NEXT:  Src: %.pre = load double, ptr %B, align 8 --> Dst: store double %.pre, ptr %arrayidx2, align 8
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: store double %.pre, ptr %arrayidx2, align 8 --> Dst: store double %.pre, ptr %arrayidx2, align 8
+; CHECK-NEXT:    da analyze - output [*]!
+;
+entry:
+  %cmp8 = icmp sgt i32 %n, 1
+  br i1 %cmp8, label %for.body.preheader, label %for.cond.cleanup
+
+for.body.preheader:                               ; preds = %entry
+  %wide.trip.count = zext nneg i32 %n to i64
+  br label %for.body
+
+for.cond.cleanup:                                 ; preds = %for.body, %entry
+  ret void
+
+for.body:                                         ; preds = %for.body.preheader, %for.body
+  %indvars.iv = phi i64 [ 1, %for.body.preheader ], [ %indvars.iv.next, %for.body ]
+  %ptr1.011 = phi ptr [ %A, %for.body.preheader ], [ %ptr2.09, %for.body ]
+  %ptr2.09 = phi ptr [ %B, %for.body.preheader ], [ %ptr1.011, %for.body ]
+  %.pre = load double, ptr %B, align 8
+  %arrayidx2 = getelementptr inbounds double, ptr %ptr1.011, i64 %indvars.iv
+  store double %.pre, ptr %arrayidx2, align 8
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %exitcond.not = icmp eq i64 %indvars.iv.next, %wide.trip.count
+  br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
+
+
+; Bug 53942 - https://github.com/llvm/llvm-project/issues/53942
+
+define void @bug53942_bar(i32 noundef %n, ptr noalias noundef %A, ptr noalias noundef %B) {
+; CHECK-LABEL: 'bug53942_bar'
+; CHECK-NEXT:  Src: %0 = load double, ptr %arrayidx, align 8 --> Dst: %0 = load double, ptr %arrayidx, align 8
+; CHECK-NEXT:    da analyze - input [*]!
+; CHECK-NEXT:  Src: %0 = load double, ptr %arrayidx, align 8 --> Dst: store double %0, ptr %arrayidx8, align 8
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: store double %0, ptr %arrayidx8, align 8 --> Dst: store double %0, ptr %arrayidx8, align 8
+; CHECK-NEXT:    da analyze - output [*]!
+;
+entry:
+  br label %for.cond
+
+for.cond:                                         ; preds = %for.inc, %entry
+  %i.0 = phi i32 [ 1, %entry ], [ %inc, %for.inc ]
+  %cmp = icmp slt i32 %i.0, %n
+  br i1 %cmp, label %for.body, label %for.cond.cleanup
+
+for.cond.cleanup:                                 ; preds = %for.cond
+  br label %for.end
+
+for.body:                                         ; preds = %for.cond
+  %and = and i32 %i.0, 2
+  %tobool.not = icmp eq i32 %and, 0
+  br i1 %tobool.not, label %cond.false, label %cond.true
+
+cond.true:                                        ; preds = %for.body
+  br label %cond.end
+
+cond.false:                                       ; preds = %for.body
+  br label %cond.end
+
+cond.end:                                         ; preds = %cond.false, %cond.true
+  %cond = phi ptr [ %A, %cond.true ], [ %B, %cond.false ]
+  %and1 = and i32 %i.0, 2
+  %tobool2.not = icmp eq i32 %and1, 0
+  br i1 %tobool2.not, label %cond.false4, label %cond.true3
+
+cond.true3:                                       ; preds = %cond.end
+  br label %cond.end5
+
+cond.false4:                                      ; preds = %cond.end
+  br label %cond.end5
+
+cond.end5:                                        ; preds = %cond.false4, %cond.true3
+  %cond6 = phi ptr [ %B, %cond.true3 ], [ %A, %cond.false4 ]
+  %sub = add nsw i32 %i.0, -1
+  %idxprom = sext i32 %sub to i64
+  %arrayidx = getelementptr inbounds double, ptr %cond6, i64 %idxprom
+  %0 = load double, ptr %arrayidx, align 8
+  %idxprom7 = zext nneg i32 %i.0 to i64
+  %arrayidx8 = getelementptr inbounds double, ptr %cond, i64 %idxprom7
+  store double %0, ptr %arrayidx8, align 8
+  br label %for.inc
+
+for.inc:                                          ; preds = %cond.end5
+  %inc = add nuw nsw i32 %i.0, 1
+  br label %for.cond
+
+for.end:                                          ; preds = %for.cond.cleanup
+  ret void
+}
diff --git a/llvm/test/Analysis/DependenceAnalysis/GCD.ll b/llvm/test/Analysis/DependenceAnalysis/GCD.ll
index f4890a269606b..c0e1362c82b50 100644
--- a/llvm/test/Analysis/DependenceAnalysis/GCD.ll
+++ b/llvm/test/Analysis/DependenceAnalysis/GCD.ll
@@ -1,5 +1,6 @@
+; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --version 5
 ; RUN: opt < %s -disable-output "-passes=print<da>" -aa-pipeline=basic-aa 2>&1 \
-; RUN: | FileCheck %s -check-prefix=DELIN
+; RUN: | FileCheck %s
 
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
 target triple = "x86_64-apple-macosx10.6.0"
@@ -12,17 +13,23 @@ target triple = "x86_64-apple-macosx10.6.0"
 ;;      *B++ = A[6*i + 8*j];
 
 define void @gcd0(ptr %A, ptr %B) nounwind uwtable ssp {
+; CHECK-LABEL: 'gcd0'
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx, align 4 --> Dst: store i32 %conv, ptr %arrayidx, align 4
+; CHECK-NEXT:    da analyze - output [* *]!
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx, align 4 --> Dst: %0 = load i32, ptr %arrayidx7, align 4
+; CHECK-NEXT:    da analyze - flow [=> *|<]!
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx, align 4 --> Dst: store i32 %0, ptr %B.addr.11, align 4
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: %0 = load i32, ptr %arrayidx7, align 4 --> Dst: %0 = load i32, ptr %arrayidx7, align 4
+; CHECK-NEXT:    da analyze - input [* *]!
+; CHECK-NEXT:  Src: %0 = load i32, ptr %arrayidx7, align 4 --> Dst: store i32 %0, ptr %B.addr.11, align 4
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: store i32 %0, ptr %B.addr.11, align 4 --> Dst: store i32 %0, ptr %B.addr.11, align 4
+; CHECK-NEXT:    da analyze - none!
+;
 entry:
   br label %for.cond1.preheader
 
-; DELIN-LABEL: gcd0
-; DELIN: da analyze - output [* *]!
-; DELIN: da analyze - flow [=> *|<]!
-; DELIN: da analyze - confused!
-; DELIN: da analyze - input [* *]!
-; DELIN: da analyze - confused!
-; DELIN: da analyze - none!
-
 for.cond1.preheader:                              ; preds = %entry, %for.inc8
   %B.addr.04 = phi ptr [ %B, %entry ], [ %scevgep, %for.inc8 ]
   %i.03 = phi i64 [ 0, %entry ], [ %inc9, %for.inc8 ]
@@ -65,17 +72,23 @@ for.end10:                                        ; preds = %for.inc8
 ;;      *B++ = A[6*i + 8*j + 1];
 
 define void @gcd1(ptr %A, ptr %B) nounwind uwtable ssp {
+; CHECK-LABEL: 'gcd1'
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx, align 4 --> Dst: store i32 %conv, ptr %arrayidx, align 4
+; CHECK-NEXT:    da analyze - output [* *]!
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx, align 4 --> Dst: %0 = load i32, ptr %arrayidx8, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx, align 4 --> Dst: store i32 %0, ptr %B.addr.11, align 4
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: %0 = load i32, ptr %arrayidx8, align 4 --> Dst: %0 = load i32, ptr %arrayidx8, align 4
+; CHECK-NEXT:    da analyze - input [* *]!
+; CHECK-NEXT:  Src: %0 = load i32, ptr %arrayidx8, align 4 --> Dst: store i32 %0, ptr %B.addr.11, align 4
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: store i32 %0, ptr %B.addr.11, align 4 --> Dst: store i32 %0, ptr %B.addr.11, align 4
+; CHECK-NEXT:    da analyze - none!
+;
 entry:
   br label %for.cond1.preheader
 
-; DELIN-LABEL: gcd1
-; DELIN: da analyze - output [* *]!
-; DELIN: da analyze - none!
-; DELIN: da analyze - confused!
-; DELIN: da analyze - input [* *]!
-; DELIN: da analyze - confused!
-; DELIN: da analyze - none!
-
 for.cond1.preheader:                              ; preds = %entry, %for.inc9
   %B.addr.04 = phi ptr [ %B, %entry ], [ %scevgep, %for.inc9 ]
   %i.03 = phi i64 [ 0, %entry ], [ %inc10, %for.inc9 ]
@@ -119,17 +132,23 @@ for.end11:                                        ; preds = %for.inc9
 ;;      *B++ = A[6*i + 8*j];
 
 define void @gcd2(ptr %A, ptr %B) nounwind uwtable ssp {
+; CHECK-LABEL: 'gcd2'
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx, align 4 --> Dst: store i32 %conv, ptr %arrayidx, align 4
+; CHECK-NEXT:    da analyze - output [* *]!
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx, align 4 --> Dst: %0 = load i32, ptr %arrayidx8, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx, align 4 --> Dst: store i32 %0, ptr %B.addr.11, align 4
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: %0 = load i32, ptr %arrayidx8, align 4 --> Dst: %0 = load i32, ptr %arrayidx8, align 4
+; CHECK-NEXT:    da analyze - input [* *]!
+; CHECK-NEXT:  Src: %0 = load i32, ptr %arrayidx8, align 4 --> Dst: store i32 %0, ptr %B.addr.11, align 4
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: store i32 %0, ptr %B.addr.11, align 4 --> Dst: store i32 %0, ptr %B.addr.11, align 4
+; CHECK-NEXT:    da analyze - none!
+;
 entry:
   br label %for.cond1.preheader
 
-; DELIN-LABEL: gcd2
-; DELIN: da analyze - output [* *]!
-; DELIN: da analyze - none!
-; DELIN: da analyze - confused!
-; DELIN: da analyze - input [* *]!
-; DELIN: da analyze - confused!
-; DELIN: da analyze - none!
-
 for.cond1.preheader:                              ; preds = %entry, %for.inc9
   %B.addr.04 = phi ptr [ %B, %entry ], [ %scevgep, %for.inc9 ]
   %i.03 = phi i64 [ 0, %entry ], [ %inc10, %for.inc9 ]
@@ -173,17 +192,23 @@ for.end11:                                        ; preds = %for.inc9
 ;;      *B++ = A[i + 2*j - 1];
 
 define void @gcd3(ptr %A, ptr %B) nounwind uwtable ssp {
+; CHECK-LABEL: 'gcd3'
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx, align 4 --> Dst: store i32 %conv, ptr %arrayidx, align 4
+; CHECK-NEXT:    da analyze - output [* *]!
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx, align 4 --> Dst: %0 = load i32, ptr %arrayidx6, align 4
+; CHECK-NEXT:    da analyze - flow [<> *]!
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx, align 4 --> Dst: store i32 %0, ptr %B.addr.11, align 4
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: %0 = load i32, ptr %arrayidx6, align 4 --> Dst: %0 = load i32, ptr %arrayidx6, align 4
+; CHECK-NEXT:    da analyze - input [* *]!
+; CHECK-NEXT:  Src: %0 = load i32, ptr %arrayidx6, align 4 --> Dst: store i32 %0, ptr %B.addr.11, align 4
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: store i32 %0, ptr %B.addr.11, align 4 --> Dst: store i32 %0, ptr %B.addr.11, align 4
+; CHECK-NEXT:    da analyze - none!
+;
 entry:
   br label %for.cond1.preheader
 
-; DELIN-LABEL: gcd3
-; DELIN: da analyze - output [* *]!
-; DELIN: da analyze - flow [<> *]!
-; DELIN: da analyze - confused!
-; DELIN: da analyze - input [* *]!
-; DELIN: da analyze - confused!
-; DELIN: da analyze - none!
-
 for.cond1.preheader:                              ; preds = %entry, %for.inc7
   %B.addr.04 = phi ptr [ %B, %entry ], [ %scevgep, %for.inc7 ]
   %i.03 = phi i64 [ 0, %entry ], [ %inc8, %for.inc7 ]
@@ -225,17 +250,23 @@ for.end9:                                         ; preds = %for.inc7
 ;;      *B++ = A[15*i + 20*j*M - 21*N*M + 4];
 
 define void @gcd4(ptr %A, ptr %B, i64 %M, i64 %N) nounwind uwtable ssp {
+; CHECK-LABEL: 'gcd4'
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx, align 4 --> Dst: store i32 %conv, ptr %arrayidx, align 4
+; CHECK-NEXT:    da analyze - output [* *]!
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx, align 4 --> Dst: %0 = load i32, ptr %arrayidx16, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx, align 4 --> Dst: store i32 %0, ptr %B.addr.11, align 4
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: %0 = load i32, ptr %arrayidx16, align 4 --> Dst: %0 = load i32, ptr %arrayidx16, align 4
+; CHECK-NEXT:    da analyze - input [* *]!
+; CHECK-NEXT:  Src: %0 = load i32, ptr %arrayidx16, align 4 --> Dst: store i32 %0, ptr %B.addr.11, align 4
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: store i32 %0, ptr %B.addr.11, align 4 --> Dst: store i32 %0, ptr %B.addr.11, align 4
+; CHECK-NEXT:    da analyze - none!
+;
 entry:
   br label %for.cond1.preheader
 
-; DELIN-LABEL: gcd4
-; DELIN: da analyze - output [* *]!
-; DELIN: da analyze - none!
-; DELIN: da analyze - confused!
-; DELIN: da analyze - input [* *]!
-; DELIN: da analyze - confused!
-; DELIN: da analyze - none!
-
 for.cond1.preheader:                              ; preds = %entry, %for.inc17
   %B.addr.04 = phi ptr [ %B, %entry ], [ %scevgep, %for.inc17 ]
   %i.03 = phi i64 [ 0, %entry ], [ %inc18, %for.inc17 ]
@@ -287,17 +318,23 @@ for.end19:                                        ; preds = %for.inc17
 ;;      *B++ = A[15*i + 20*j*M - 21*N*M + 5];
 
 define void @gcd5(ptr %A, ptr %B, i64 %M, i64 %N) nounwind uwtable ssp {
+; CHECK-LABEL: 'gcd5'
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx, align 4 --> Dst: store i32 %conv, ptr %arrayidx, align 4
+; CHECK-NEXT:    da analyze - output [* *]!
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx, align 4 --> Dst: %0 = load i32, ptr %arrayidx16, align 4
+; CHECK-NEXT:    da analyze - flow [<> *]!
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx, align 4 --> Dst: store i32 %0, ptr %B.addr.11, align 4
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: %0 = load i32, ptr %arrayidx16, align 4 --> Dst: %0 = load i32, ptr %arrayidx16, align 4
+; CHECK-NEXT:    da analyze - input [* *]!
+; CHECK-NEXT:  Src: %0 = load i32, ptr %arrayidx16, align 4 --> Dst: store i32 %0, ptr %B.addr.11, align 4
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: store i32 %0, ptr %B.addr.11, align 4 --> Dst: store i32 %0, ptr %B.addr.11, align 4
+; CHECK-NEXT:    da analyze - none!
+;
 entry:
   br label %for.cond1.preheader
 
-; DELIN-LABEL: gcd5
-; DELIN: da analyze - output [* *]!
-; DELIN: da analyze - flow [<> *]!
-; DELIN: da analyze - confused!
-; DELIN: da analyze - input [* *]!
-; DELIN: da analyze - confused!
-; DELIN: da analyze - none!
-
 for.cond1.preheader:                              ; preds = %entry, %for.inc17
   %B.addr.04 = phi ptr [ %B, %entry ], [ %scevgep, %for.inc17 ]
   %i.03 = phi i64 [ 0, %entry ], [ %inc18, %for.inc17 ]
@@ -349,18 +386,24 @@ for.end19:                                        ; preds = %for.inc17
 ;;      *B++ = A[8*i][6*j + 1];
 
 define void @gcd6(i64 %n, ptr %A, ptr %B) nounwind uwtable ssp {
+; CHECK-LABEL: 'gcd6'
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx5, align 4 --> Dst: store i32 %conv, ptr %arrayidx5, align 4
+; CHECK-NEXT:    da analyze - output [* *]!
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx5, align 4 --> Dst: %2 = load i32, ptr %arrayidx9, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx5, align 4 --> Dst: store i32 %2, ptr %B.addr.12, align 4
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: %2 = load i32, ptr %arrayidx9, align 4 --> Dst: %2 = load i32, ptr %arrayidx9, align 4
+; CHECK-NEXT:    da analyze - input [* *]!
+; CHECK-NEXT:  Src: %2 = load i32, ptr %arrayidx9, align 4 --> Dst: store i32 %2, ptr %B.addr.12, align 4
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: store i32 %2, ptr %B.addr.12, align 4 --> Dst: store i32 %2, ptr %B.addr.12, align 4
+; CHECK-NEXT:    da analyze - output [* *]!
+;
 entry:
   %cmp4 = icmp sgt i64 %n, 0
   br i1 %cmp4, label %for.cond1.preheader.preheader, label %for.end12
 
-; DELIN-LABEL: gcd6
-; DELIN: da analyze - output [* *]!
-; DELIN: da analyze - none!
-; DELIN: da analyze - confused!
-; DELIN: da analyze - input [* *]!
-; DELIN: da analyze - confused!
-; DELIN: da analyze - output [* *]!
-
 for.cond1.preheader.preheader:                    ; preds = %entry
   br label %for.cond1.preheader
 
@@ -420,19 +463,25 @@ for.end12:                                        ; preds = %for.end12.loopexit,
 ;;   *B++ = A[8*i][6*j + 1];
 
 define void @gcd7(i32 %n, ptr %A, ptr %B) nounwind uwtable ssp {
+; CHECK-LABEL: 'gcd7'
+; CHECK-NEXT:  Src: store i32 %7, ptr %arrayidx6, align 4 --> Dst: store i32 %7, ptr %arrayidx6, align 4
+; CHECK-NEXT:    da analyze - output [* *]!
+; CHECK-NEXT:  Src: store i32 %7, ptr %arrayidx6, align 4 --> Dst: %11 = load i32, ptr %arrayidx12, align 4
+; CHECK-NEXT:    da analyze - flow [* *|<]!
+; CHECK-NEXT:  Src: store i32 %7, ptr %arrayidx6, align 4 --> Dst: store i32 %11, ptr %B.addr.12, align 4
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: %11 = load i32, ptr %arrayidx12, align 4 --> Dst: %11 = load i32, ptr %arrayidx12, align 4
+; CHECK-NEXT:    da analyze - input [* *]!
+; CHECK-NEXT:  Src: %11 = load i32, ptr %arrayidx12, align 4 --> Dst: store i32 %11, ptr %B.addr.12, align 4
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: store i32 %11, ptr %B.addr.12, align 4 --> Dst: store i32 %11, ptr %B.addr.12, align 4
+; CHECK-NEXT:    da analyze - output [* *]!
+;
 entry:
   %0 = zext i32 %n to i64
   %cmp4 = icmp sgt i32 %n, 0
   br i1 %cmp4, label %for.cond1.preheader.preheader, label %for.end15
 
-; DELIN-LABEL: gcd7
-; DELIN: da analyze - output [* *]!
-; DELIN: da analyze - flow [* *|<]!
-; DELIN: da analyze - confused!
-; DELIN: da analyze - input [* *]!
-; DELIN: da analyze - confused!
-; DELIN: da analyze - output [* *]!
-
 for.cond1.preheader.preheader:                    ; preds = %entry
   br label %for.cond1.preheader
 
@@ -505,18 +554,24 @@ for.end15:                                        ; preds = %for.end15.loopexit,
 ;;      *B++ = A[n*8*i + 6*j + 1];
 
 define void @gcd8(i32 %n, ptr %A, ptr %B) nounwind uwtable ssp {
+; CHECK-LABEL: 'gcd8'
+; CHECK-NEXT:  Src: store i32 %i.06, ptr %arrayidx, align 4 --> Dst: store i32 %i.06, ptr %arrayidx, align 4
+; CHECK-NEXT:    da analyze - output [* *]!
+; CHECK-NEXT:  Src: store i32 %i.06, ptr %arrayidx, align 4 --> Dst: %5 = load i32, ptr %arrayidx12, align 4
+; CHECK-NEXT:    da analyze - flow [* *|<]!
+; CHECK-NEXT:  Src: store i32 %i.06, ptr %arrayidx, align 4 --> Dst: store i32 %5, ptr %B.addr.12, align 4
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: %5 = load i32, ptr %arrayidx12, align 4 --> Dst: %5 = load i32, ptr %arrayidx12, align 4
+; CHECK-NEXT:    da analyze - input [* *]!
+; CHECK-NEXT:  Src: %5 = load i32, ptr %arrayidx12, align 4 --> Dst: store i32 %5, ptr %B.addr.12, align 4
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: store i32 %5, ptr %B.addr.12, align 4 --> Dst: store i32 %5, ptr %B.addr.12, align 4
+; CHECK-NEXT:    da analyze - output [* *]!
+;
 entry:
   %cmp4 = icmp sgt i32 %n, 0
   br i1 %cmp4, label %for.cond1.preheader.preheader, label %for.end15
 
-; DELIN-LABEL: gcd8
-; DELIN: da analyze - output [* *]!
-; DELIN: da analyze - flow [* *|<]!
-; DELIN: da analyze - confused!
-; DELIN: da analyze - input [* *]!
-; DELIN: da analyze - confused!
-; DELIN: da analyze - output [* *]!
-
 for.cond1.preheader.preheader:                    ; preds = %entry
   br label %for.cond1.preheader
 
@@ -583,19 +638,25 @@ for.end15:                                        ; preds = %for.end15.loopexit,
 ;;      *B++ = A[8*i][6*j + 1];
 
 define void @gcd9(i32 %n, ptr %A, ptr %B) nounwind uwtable ssp {
+; CHECK-LABEL: 'gcd9'
+; CHECK-NEXT:  Src: store i32 %7, ptr %arrayidx6, align 4 --> Dst: store i32 %7, ptr %arrayidx6, align 4
+; CHECK-NEXT:    da analyze - output [* *]!
+; CHECK-NEXT:  Src: store i32 %7, ptr %arrayidx6, align 4 --> Dst: %11 = load i32, ptr %arrayidx12, align 4
+; CHECK-NEXT:    da analyze - flow [* *|<]!
+; CHECK-NEXT:  Src: store i32 %7, ptr %arrayidx6, align 4 --> Dst: store i32 %11, ptr %B.addr.12, align 4
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: %11 = load i32, ptr %arrayidx12, align 4 --> Dst: %11 = load i32, ptr %arrayidx12, align 4
+; CHECK-NEXT:    da analyze - input [* *]!
+; CHECK-NEXT:  Src: %11 = load i32, ptr %arrayidx12, align 4 --> Dst: store i32 %11, ptr %B.addr.12, align 4
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: store i32 %11, ptr %B.addr.12, align 4 --> Dst: store i32 %11, ptr %B.addr.12, align 4
+; CHECK-NEXT:    da analyze - output [* *]!
+;
 entry:
   %0 = zext i32 %n to i64
   %cmp4 = icmp eq i32 %n, 0
   br i1 %cmp4, label %for.end15, label %for.cond1.preheader.preheader
 
-; DELIN-LABEL: gcd9
-; DELIN: da analyze - output [* *]!
-; DELIN: da analyze - flow [* *|<]!
-; DELIN: da analyze - confused!
-; DELIN: da analyze - input [* *]!
-; DELIN: da analyze - confused!
-; DELIN: da analyze - output [* *]!
-
 for.cond1.preheader.preheader:                    ; preds = %entry
   br label %for.cond1.preheader
 
diff --git a/llvm/test/Analysis/DependenceAnalysis/Invariant.ll b/llvm/test/Analysis/DependenceAnalysis/Invariant.ll
index 40fe0de0e2350..1d8c51e475ae8 100644
--- a/llvm/test/Analysis/DependenceAnalysis/Invariant.ll
+++ b/llvm/test/Analysis/DependenceAnalysis/Invariant.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --version 5
 ; RUN: opt < %s -disable-output "-passes=print<da>" -aa-pipeline=basic-aa 2>&1 \
 ; RUN: | FileCheck %s
 
@@ -15,12 +16,15 @@
 ;   return res;
 ; }
 
-; CHECK-LABEL: foo
-; CHECK: da analyze - consistent input [S 0]!
-; CHECK: da analyze - input [* 0|<]!
-; CHECK: da analyze - none!
-
 define float @foo(float %g, ptr %rr) nounwind {
+; CHECK-LABEL: 'foo'
+; CHECK-NEXT:  Src: %0 = load float, ptr %arrayidx4, align 4 --> Dst: %0 = load float, ptr %arrayidx4, align 4
+; CHECK-NEXT:    da analyze - consistent input [S 0]!
+; CHECK-NEXT:  Src: %0 = load float, ptr %arrayidx4, align 4 --> Dst: %1 = load float, ptr %arrayidx6, align 4
+; CHECK-NEXT:    da analyze - input [* 0|<]!
+; CHECK-NEXT:  Src: %1 = load float, ptr %arrayidx6, align 4 --> Dst: %1 = load float, ptr %arrayidx6, align 4
+; CHECK-NEXT:    da analyze - none!
+;
 entry:
   br label %for.cond1.preheader
 
diff --git a/llvm/test/Analysis/DependenceAnalysis/MIVCheckConst.ll b/llvm/test/Analysis/DependenceAnalysis/MIVCheckConst.ll
index e33dca9d89c82..fa58a81d2355b 100644
--- a/llvm/test/Analysis/DependenceAnalysis/MIVCheckConst.ll
+++ b/llvm/test/Analysis/DependenceAnalysis/MIVCheckConst.ll
@@ -1,4 +1,5 @@
-; RUN: opt < %s -passes="print<da>"
+; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --version 5
+; RUN: opt < %s -disable-output -passes="print<da>" 2>&1 | FileCheck %s
 
 ; Test that the dependence analysis pass does seg-fault due to a null pointer
 ; dereference. The code in gcdMIVTest requires a null check for the result of
@@ -29,29 +30,43 @@ target datalayout = "e-m:e-p:32:32-i1:32-i64:64-a:0-v32:32-n16:32"
 %20 = type { [768 x i32] }
 %21 = type { [416 x i32] }
 
-define void @test(ptr %A, i1 %arg) #0 align 2 {
+define void @test(ptr %A, ptr %B, i1 %arg, i32 %n, i32 %m) #0 align 2 {
+; CHECK-LABEL: 'test'
+; CHECK-NEXT:  Src: %v1 = load i32, ptr %B, align 4 --> Dst: %v1 = load i32, ptr %B, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: %v1 = load i32, ptr %B, align 4 --> Dst: %v27 = load <32 x i32>, ptr %v25, align 256
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: %v1 = load i32, ptr %B, align 4 --> Dst: %v32 = load <32 x i32>, ptr %v30, align 128
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: %v27 = load <32 x i32>, ptr %v25, align 256 --> Dst: %v27 = load <32 x i32>, ptr %v25, align 256
+; CHECK-NEXT:    da analyze - consistent input [0 S S]!
+; CHECK-NEXT:  Src: %v27 = load <32 x i32>, ptr %v25, align 256 --> Dst: %v32 = load <32 x i32>, ptr %v30, align 128
+; CHECK-NEXT:    da analyze - input [* S S|<]!
+; CHECK-NEXT:  Src: %v32 = load <32 x i32>, ptr %v30, align 128 --> Dst: %v32 = load <32 x i32>, ptr %v30, align 128
+; CHECK-NEXT:    da analyze - consistent input [0 S S]!
+;
 entry:
-  %v1 = load i32, ptr undef, align 4
+  %v1 = load i32, ptr %B, align 4
   br label %bb13
 
 bb13:
-  %v2 = phi i32 [ undef, %entry ], [ %v39, %bb38 ]
+  %v2 = phi i32 [ 0, %entry ], [ %v39, %bb38 ]
   br i1 %arg, label %bb15, label %bb38
 
 bb15:
-  %v3 = mul nsw i32 %v2, undef
+  %v3 = mul nsw i32 %v2, 2
   br label %bb17
 
 bb17:
   br i1 %arg, label %bb21, label %bb37
 
 bb21:
-  %v22 = add nsw i32 undef, 1
+  %v22 = add nsw i32 %n, 1
   %v23 = add i32 %v22, %v3
   %v24 = mul nsw i32 %v23, %v1
   %v25 = getelementptr inbounds %1, ptr %A, i32 0, i32 7, i32 1, i32 %v24
   %v27 = load <32 x i32>, ptr %v25, align 256
-  %v28 = add i32 undef, %v3
+  %v28 = add i32 %m, %v3
   %v29 = mul nsw i32 %v28, 32
   %v30 = getelementptr inbounds %1, ptr %A, i32 0, i32 7, i32 14, i32 %v29
   %v32 = load <32 x i32>, ptr %v30, align 128
diff --git a/llvm/test/Analysis/DependenceAnalysis/MIVMaxLevelThreshold.ll b/llvm/test/Analysis/DependenceAnalysis/MIVMaxLevelThreshold.ll
index 69c7def57fbd6..c56f7f34f7d4b 100644
--- a/llvm/test/Analysis/DependenceAnalysis/MIVMaxLevelThreshold.ll
+++ b/llvm/test/Analysis/DependenceAnalysis/MIVMaxLevelThreshold.ll
@@ -1,4 +1,6 @@
-; RUN: opt < %s -aa-pipeline=default -passes='print<da>' -da-miv-max-level-threshold=2
+; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --version 5
+; RUN: opt < %s -disable-output -aa-pipeline=default -passes='print<da>' \
+; RUN: -da-miv-max-level-threshold=2 2>&1 | FileCheck %s
 
 ;; Check to make sure when MIV tests reach a maximum depth level
 ;; threshold, the resulting dependence is conservatively correct.
@@ -9,14 +11,21 @@
 ;;       A[i+j+k] = A[i-j+k] + B[i+j+k];
 
 
-; CHECK:       Src:  %2 = load float, float* %arrayidx, align 4 --> Dst:  %5 = load float, float* %arrayidx12, align 4
-; CHECK-NEXT:  da analyze - none!
-; CHECK:       Src:  %2 = load float, float* %arrayidx, align 4 --> Dst:  store float %add13, float* %arrayidx17, align 4
-; CHECK-NEXT:  da analyze - anti [* * *|<]!
-; CHECK:       Src:  %5 = load float, float* %arrayidx12, align 4 --> Dst:  store float %add13, float* %arrayidx17, align 4
-; CHECK-NEXT:  da analyze - none!
-
 define void @foo(ptr noalias %A, ptr noalias %B, i32 signext %m, i32 signext %n, i32 signext %o) {
+; CHECK-LABEL: 'foo'
+; CHECK-NEXT:  Src: %2 = load float, ptr %arrayidx, align 4 --> Dst: %2 = load float, ptr %arrayidx, align 4
+; CHECK-NEXT:    da analyze - input [* * *]!
+; CHECK-NEXT:  Src: %2 = load float, ptr %arrayidx, align 4 --> Dst: %5 = load float, ptr %arrayidx12, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: %2 = load float, ptr %arrayidx, align 4 --> Dst: store float %add13, ptr %arrayidx17, align 4
+; CHECK-NEXT:    da analyze - anti [* * *|<]!
+; CHECK-NEXT:  Src: %5 = load float, ptr %arrayidx12, align 4 --> Dst: %5 = load float, ptr %arrayidx12, align 4
+; CHECK-NEXT:    da analyze - input [* * *]!
+; CHECK-NEXT:  Src: %5 = load float, ptr %arrayidx12, align 4 --> Dst: store float %add13, ptr %arrayidx17, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: store float %add13, ptr %arrayidx17, align 4 --> Dst: store float %add13, ptr %arrayidx17, align 4
+; CHECK-NEXT:    da analyze - output [* * *]!
+;
 entry:
   %cmp5 = icmp sgt i32 %n, 0
   br i1 %cmp5, label %for.body.preheader, label %for.end23
diff --git a/llvm/test/Analysis/DependenceAnalysis/MismatchingNestLevels.ll b/llvm/test/Analysis/DependenceAnalysis/MismatchingNestLevels.ll
index 6f203115c5a7a..ed7f087e98b00 100644
--- a/llvm/test/Analysis/DependenceAnalysis/MismatchingNestLevels.ll
+++ b/llvm/test/Analysis/DependenceAnalysis/MismatchingNestLevels.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --version 5
 ; RUN: opt < %s -disable-output "-passes=print<da>" -aa-pipeline=basic-aa 2>&1 | FileCheck %s
 
 ;; void test1(long n, double *A) {
@@ -8,15 +9,15 @@
 ;;     A[i] = i;
 ;; }
 
-; CHECK-LABEL: 'Dependence Analysis' for function 'test1':
-; CHECK: Src:  store double %conv, ptr %arrayidx, align 8 --> Dst:  store double %conv, ptr %arrayidx, align 8
+define void @test1(i64 noundef %n, ptr nocapture noundef writeonly %A) {
+; CHECK-LABEL: 'test1'
+; CHECK-NEXT:  Src: store double %conv, ptr %arrayidx, align 8 --> Dst: store double %conv, ptr %arrayidx, align 8
 ; CHECK-NEXT:    da analyze - none!
-; CHECK: Src:  store double %conv, ptr %arrayidx, align 8 --> Dst:  store double %conv2, ptr %arrayidx3, align 8
+; CHECK-NEXT:  Src: store double %conv, ptr %arrayidx, align 8 --> Dst: store double %conv2, ptr %arrayidx3, align 8
 ; CHECK-NEXT:    da analyze - output [|<]!
-; CHECK: Src:  store double %conv2, ptr %arrayidx3, align 8 --> Dst:  store double %conv2, ptr %arrayidx3, align 8
+; CHECK-NEXT:  Src: store double %conv2, ptr %arrayidx3, align 8 --> Dst: store double %conv2, ptr %arrayidx3, align 8
 ; CHECK-NEXT:    da analyze - none!
-
-define void @test1(i64 noundef %n, ptr nocapture noundef writeonly %A) {
+;
 entry:
   %mul1 = mul nsw i64 %n, %n
   br label %for.body
@@ -53,11 +54,21 @@ for.end:                                          ; preds = %for.body
 ;;
 ;; Make sure we can detect depnendence between A[i][j] and A[i][k] conservatively and without crashing.
 
-; CHECK-LABEL: 'Dependence Analysis' for function 'test2':
-; CHECK:       Src:  store float 1.230000e+02, ptr %arrayidx7, align 4 --> Dst:  store float %conv13, ptr %arrayidx17, align 4
-; CHECK-NEXT:    da analyze - output [*|<]!
-
 define dso_local void @test2(i32 noundef zeroext %n, ptr noundef %A, ptr noalias noundef %B) #0 {
+; CHECK-LABEL: 'test2'
+; CHECK-NEXT:  Src: store i32 %conv5, ptr %arrayidx, align 4 --> Dst: store i32 %conv5, ptr %arrayidx, align 4
+; CHECK-NEXT:    da analyze - consistent output [S 0]!
+; CHECK-NEXT:  Src: store i32 %conv5, ptr %arrayidx, align 4 --> Dst: store float 1.230000e+02, ptr %arrayidx7, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: store i32 %conv5, ptr %arrayidx, align 4 --> Dst: store float %conv13, ptr %arrayidx17, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: store float 1.230000e+02, ptr %arrayidx7, align 4 --> Dst: store float 1.230000e+02, ptr %arrayidx7, align 4
+; CHECK-NEXT:    da analyze - output [*]!
+; CHECK-NEXT:  Src: store float 1.230000e+02, ptr %arrayidx7, align 4 --> Dst: store float %conv13, ptr %arrayidx17, align 4
+; CHECK-NEXT:    da analyze - output [*|<]!
+; CHECK-NEXT:  Src: store float %conv13, ptr %arrayidx17, align 4 --> Dst: store float %conv13, ptr %arrayidx17, align 4
+; CHECK-NEXT:    da analyze - none!
+;
 entry:
   %add = add i32 %n, 1
   %0 = zext i32 %add to i64
@@ -124,11 +135,15 @@ for.end23:                                        ; preds = %for.inc21
 ;;
 ;; Make sure we can detect depnendence between A[i] and A[s] conservatively and without crashing.
 
-; CHECK-LABEL: 'Dependence Analysis' for function 'test3':
-; CHECK:       Src:  store double 0.000000e+00, ptr %arrayidx, align 8 --> Dst:  store double 1.000000e+00, ptr %arrayidx21, align 8
-; CHECK-NEXT:    da analyze - output [*|<]!
-
 define void @test3(i32 noundef %n, ptr noalias noundef %A, ptr noalias noundef %B) {
+; CHECK-LABEL: 'test3'
+; CHECK-NEXT:  Src: store double 0.000000e+00, ptr %arrayidx, align 8 --> Dst: store double 0.000000e+00, ptr %arrayidx, align 8
+; CHECK-NEXT:    da analyze - output [* S]!
+; CHECK-NEXT:  Src: store double 0.000000e+00, ptr %arrayidx, align 8 --> Dst: store double 1.000000e+00, ptr %arrayidx21, align 8
+; CHECK-NEXT:    da analyze - output [*|<]!
+; CHECK-NEXT:  Src: store double 1.000000e+00, ptr %arrayidx21, align 8 --> Dst: store double 1.000000e+00, ptr %arrayidx21, align 8
+; CHECK-NEXT:    da analyze - none!
+;
 entry:
   br label %for.i
 
diff --git a/llvm/test/Analysis/DependenceAnalysis/NonAffineExpr.ll b/llvm/test/Analysis/DependenceAnalysis/NonAffineExpr.ll
index 6837d451667b9..d983bd49d49d6 100644
--- a/llvm/test/Analysis/DependenceAnalysis/NonAffineExpr.ll
+++ b/llvm/test/Analysis/DependenceAnalysis/NonAffineExpr.ll
@@ -1,13 +1,19 @@
-; RUN: opt < %s -disable-output "-passes=print<da>" -aa-pipeline=basic-aa 2>&1
-;
-; CHECK: da analyze - consistent input [S S]!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - input [* *]!
-;
+; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --version 5
+; RUN: opt < %s -disable-output "-passes=print<da>" -aa-pipeline=basic-aa 2>&1 \
+; RUN: | FileCheck %s
+
 target datalayout = "e-m:e-p:32:32-i64:64-v128:64:128-a:0:32-n8:16:32-S64"
 target triple = "thumbv7--linux-gnueabi"
 
 define void @f(ptr %a, i32 %n, i1 %arg) align 2 {
+; CHECK-LABEL: 'f'
+; CHECK-NEXT:  Src: %t.2 = load ptr, ptr %a, align 4 --> Dst: %t.2 = load ptr, ptr %a, align 4
+; CHECK-NEXT:    da analyze - consistent input [S S]!
+; CHECK-NEXT:  Src: %t.2 = load ptr, ptr %a, align 4 --> Dst: %t.4 = load i32, ptr %t.3, align 4
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: %t.4 = load i32, ptr %t.3, align 4 --> Dst: %t.4 = load i32, ptr %t.3, align 4
+; CHECK-NEXT:    da analyze - input [* *]!
+;
 for.preheader:
   %t.0 = ashr exact i32 %n, 3
   br label %for.body.1
diff --git a/llvm/test/Analysis/DependenceAnalysis/NonCanonicalizedSubscript.ll b/llvm/test/Analysis/DependenceAnalysis/NonCanonicalizedSubscript.ll
index 1ce54873b3b8b..f0cd2fd4cd930 100644
--- a/llvm/test/Analysis/DependenceAnalysis/NonCanonicalizedSubscript.ll
+++ b/llvm/test/Analysis/DependenceAnalysis/NonCanonicalizedSubscript.ll
@@ -1,5 +1,6 @@
+; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --version 5
 ; RUN: opt < %s -disable-output "-passes=print<da>" -aa-pipeline=basic-aa 2>&1 \
-; RUN: | FileCheck %s -check-prefix=DELIN
+; RUN: | FileCheck %s
 
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
 target triple = "x86_64-apple-macosx10.6.0"
@@ -13,14 +14,18 @@ target triple = "x86_64-apple-macosx10.6.0"
 ; other subscript. DependenceAnalysis before the fix crashed due to this
 ; mismatch.
 define void @i32_subscript(ptr %a, ptr %b) {
-; DELIN-LABEL: 'Dependence Analysis' for function 'i32_subscript'
+; CHECK-LABEL: 'i32_subscript'
+; CHECK-NEXT:  Src: %0 = load i32, ptr %a.addr, align 4 --> Dst: %0 = load i32, ptr %a.addr, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: %0 = load i32, ptr %a.addr, align 4 --> Dst: store i32 %1, ptr %a.addr.2, align 4
+; CHECK-NEXT:    da analyze - anti [=|<]!
+; CHECK-NEXT:  Src: store i32 %1, ptr %a.addr.2, align 4 --> Dst: store i32 %1, ptr %a.addr.2, align 4
+; CHECK-NEXT:    da analyze - none!
+;
 entry:
   br label %for.body
 
 for.body:
-; DELIN: da analyze - none!
-; DELIN: da analyze - anti [=|<]!
-; DELIN: da analyze - none!
   %i = phi i64 [ 0, %entry ], [ %i.inc, %for.body ]
   %a.addr = getelementptr [100 x [100 x i32]], ptr %a, i64 0, i64 %i, i64 %i
   %a.addr.2 = getelementptr [100 x [100 x i32]], ptr %a, i64 0, i64 %i, i32 5
@@ -48,13 +53,17 @@ for.end:
 
 ; Function Attrs: nounwind uwtable
 define void @coupled_miv_type_mismatch(i32 %n) #0 {
-; DELIN-LABEL: 'Dependence Analysis' for function 'coupled_miv_type_mismatch'
+; CHECK-LABEL: 'coupled_miv_type_mismatch'
+; CHECK-NEXT:  Src: %2 = load i32, ptr %arrayidx5, align 4 --> Dst: %2 = load i32, ptr %arrayidx5, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: %2 = load i32, ptr %arrayidx5, align 4 --> Dst: store i32 %add6, ptr %arrayidx10, align 4
+; CHECK-NEXT:    da analyze - consistent anti [1 -2]!
+; CHECK-NEXT:  Src: store i32 %add6, ptr %arrayidx10, align 4 --> Dst: store i32 %add6, ptr %arrayidx10, align 4
+; CHECK-NEXT:    da analyze - none!
+;
 entry:
   br label %for.cond
 
-; DELIN: da analyze - none!
-; DELIN: da analyze - consistent anti [1 -2]!
-; DELIN: da analyze - none!
 for.cond:                                         ; preds = %for.inc11, %entry
   %indvars.iv11 = phi i64 [ %indvars.iv.next12, %for.inc11 ], [ 1, %entry ]
   %exitcond14 = icmp ne i64 %indvars.iv11, 10000
diff --git a/llvm/test/Analysis/DependenceAnalysis/PR21585.ll b/llvm/test/Analysis/DependenceAnalysis/PR21585.ll
index 34c5317bce4c5..0d59e107b9c35 100644
--- a/llvm/test/Analysis/DependenceAnalysis/PR21585.ll
+++ b/llvm/test/Analysis/DependenceAnalysis/PR21585.ll
@@ -1,6 +1,15 @@
+; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --version 5
 ; RUN: opt < %s -disable-output "-passes=print<da>"                            \
 ; RUN: "-aa-pipeline=basic-aa,globals-aa" 2>&1 | FileCheck %s
 define void @i32_subscript(ptr %a) {
+; CHECK-LABEL: 'i32_subscript'
+; CHECK-NEXT:  Src: %0 = load i32, ptr %a.addr, align 4 --> Dst: %0 = load i32, ptr %a.addr, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: %0 = load i32, ptr %a.addr, align 4 --> Dst: store i32 %1, ptr %a.addr.2, align 4
+; CHECK-NEXT:    da analyze - anti [*|<]!
+; CHECK-NEXT:  Src: store i32 %1, ptr %a.addr.2, align 4 --> Dst: store i32 %1, ptr %a.addr.2, align 4
+; CHECK-NEXT:    da analyze - consistent output [S]!
+;
 entry:
   br label %for.body
 
@@ -19,10 +28,6 @@ for.body:
 for.end:
   ret void
 }
-; CHECK: none
-; CHECK: anti
-; CHECK: output
-
 
 ; Test for a bug, which caused an assert in ScalarEvolution because
 ; the Dependence Analyzer attempted to zero extend a type to a smaller
@@ -36,6 +41,14 @@ for.end:
 @g = common global i32 0, align 4
 
 define void @t(ptr noalias %a, i32 %n) nounwind {
+; CHECK-LABEL: 't'
+; CHECK-NEXT:  Src: %0 = load i32, ptr @g, align 4 --> Dst: %0 = load i32, ptr @g, align 4
+; CHECK-NEXT:    da analyze - consistent input [S]!
+; CHECK-NEXT:  Src: %0 = load i32, ptr @g, align 4 --> Dst: store i32 %0, ptr %arrayidx, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: store i32 %0, ptr %arrayidx, align 4 --> Dst: store i32 %0, ptr %arrayidx, align 4
+; CHECK-NEXT:    da analyze - output [*]!
+;
 entry:
   %cmp1 = icmp eq i32 %n, 0
   br i1 %cmp1, label %for.end, label %for.body
@@ -53,11 +66,16 @@ for.body:
 for.end:
   ret void
 }
-; CHECK: input
-; CHECK: none
-; CHECK: output
 
 define void @i16_wrap(ptr %a) {
+; CHECK-LABEL: 'i16_wrap'
+; CHECK-NEXT:  Src: %0 = load i64, ptr %idx, align 4 --> Dst: %0 = load i64, ptr %idx, align 4
+; CHECK-NEXT:    da analyze - input [*]!
+; CHECK-NEXT:  Src: %0 = load i64, ptr %idx, align 4 --> Dst: store i64 %1, ptr %idx, align 4
+; CHECK-NEXT:    da analyze - anti [*|<]!
+; CHECK-NEXT:  Src: store i64 %1, ptr %idx, align 4 --> Dst: store i64 %1, ptr %idx, align 4
+; CHECK-NEXT:    da analyze - output [*]!
+;
 entry:
   br label %for.body
 for.body:
@@ -76,11 +94,16 @@ for.inc:
 for.end:
   ret void
 }
-; CHECK: input
-; CHECK: anti
-; CHECK: output
 
 define void @i8_stride_wrap(ptr noalias %a, ptr noalias %b) {
+; CHECK-LABEL: 'i8_stride_wrap'
+; CHECK-NEXT:  Src: %0 = load i32, ptr %idx, align 4 --> Dst: %0 = load i32, ptr %idx, align 4
+; CHECK-NEXT:    da analyze - consistent input [S]!
+; CHECK-NEXT:  Src: %0 = load i32, ptr %idx, align 4 --> Dst: store i32 %1, ptr %idx.2, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: store i32 %1, ptr %idx.2, align 4 --> Dst: store i32 %1, ptr %idx.2, align 4
+; CHECK-NEXT:    da analyze - none!
+;
 entry:
   br label %for.body
 for.body:
@@ -101,6 +124,3 @@ for.inc:
 for.end:
   ret void
 }
-; CHECK: input
-; CHECK: none
-; CHECK: none
diff --git a/llvm/test/Analysis/DependenceAnalysis/PR31848.ll b/llvm/test/Analysis/DependenceAnalysis/PR31848.ll
new file mode 100644
index 0000000000000..8ee1224a18ad1
--- /dev/null
+++ b/llvm/test/Analysis/DependenceAnalysis/PR31848.ll
@@ -0,0 +1,74 @@
+; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --version 5
+; RUN: opt < %s -disable-output "-passes=print<da>" -aa-pipeline=basic-aa 2>&1 \
+; RUN: | FileCheck %s
+
+; DependenceAnalysis used to run into a heap buffer overflow on this testcase.
+; See https://github.com/llvm/llvm-project/issues/31196 for details.
+
+define void @barney(ptr nocapture %arg, i32 %arg1) {
+; CHECK-LABEL: 'barney'
+; CHECK-NEXT:  Src: store i32 7, ptr %getelementptr, align 4 --> Dst: store i32 7, ptr %getelementptr, align 4
+; CHECK-NEXT:    da analyze - output [* * * *]!
+;
+bb:
+  %icmp = icmp sgt i32 %arg1, 0
+  br i1 %icmp, label %bb2, label %bb31
+
+bb2:                                              ; preds = %bb28, %bb
+  %phi = phi i32 [ %add29, %bb28 ], [ 0, %bb ]
+  br label %bb3
+
+bb3:                                              ; preds = %bb25, %bb2
+  %phi4 = phi i32 [ %add26, %bb25 ], [ 0, %bb2 ]
+  br label %bb5
+
+bb5:                                              ; preds = %bb22, %bb3
+  %phi6 = phi i32 [ -1, %bb3 ], [ %add23, %bb22 ]
+  %add7 = add nsw i32 %phi6, %phi4
+  br label %bb8
+
+bb8:                                              ; preds = %bb19, %bb5
+  %phi9 = phi i32 [ -1, %bb5 ], [ %add20, %bb19 ]
+  %add10 = add nsw i32 %phi9, %phi
+  br label %bb11
+
+bb11:                                             ; preds = %bb11, %bb8
+  %phi12 = phi i32 [ %add7, %bb8 ], [ %add14, %bb11 ]
+  %icmp13 = icmp slt i32 %phi12, 0
+  %add14 = add nsw i32 %phi12, %arg1
+  br i1 %icmp13, label %bb11, label %bb15
+
+bb15:                                             ; preds = %bb15, %bb11
+  %phi16 = phi i32 [ %add18, %bb15 ], [ %add10, %bb11 ]
+  %icmp17 = icmp slt i32 %phi16, 0
+  %add18 = add nsw i32 %phi16, %arg1
+  br i1 %icmp17, label %bb15, label %bb19
+
+bb19:                                             ; preds = %bb15
+  %mul = mul nsw i32 %phi16, %arg1
+  %add = add nsw i32 %mul, %phi12
+  %sext = sext i32 %add to i64
+  %getelementptr = getelementptr inbounds i32, ptr %arg, i64 %sext
+  store i32 7, ptr %getelementptr, align 4
+  %add20 = add nsw i32 %phi9, 1
+  %icmp21 = icmp eq i32 %add20, 2
+  br i1 %icmp21, label %bb22, label %bb8
+
+bb22:                                             ; preds = %bb19
+  %add23 = add nsw i32 %phi6, 1
+  %icmp24 = icmp eq i32 %add23, 2
+  br i1 %icmp24, label %bb25, label %bb5
+
+bb25:                                             ; preds = %bb22
+  %add26 = add nuw nsw i32 %phi4, 1
+  %icmp27 = icmp eq i32 %add26, %arg1
+  br i1 %icmp27, label %bb28, label %bb3
+
+bb28:                                             ; preds = %bb25
+  %add29 = add nuw nsw i32 %phi, 1
+  %icmp30 = icmp eq i32 %add29, %arg1
+  br i1 %icmp30, label %bb31, label %bb2
+
+bb31:                                             ; preds = %bb28, %bb
+  ret void
+}
diff --git a/llvm/test/Analysis/DependenceAnalysis/Preliminary.ll b/llvm/test/Analysis/DependenceAnalysis/Preliminary.ll
index ee0b50d479249..4ab87771417e5 100644
--- a/llvm/test/Analysis/DependenceAnalysis/Preliminary.ll
+++ b/llvm/test/Analysis/DependenceAnalysis/Preliminary.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --version 5
 ; RUN: opt < %s -disable-output "-passes=print<da>" -aa-pipeline=basic-aa 2>&1 \
 ; RUN: | FileCheck %s
 
@@ -10,14 +11,16 @@ target triple = "x86_64-apple-macosx10.6.0"
 ;;  return B[1];
 
 define i32 @p0(i32 %n, ptr %A, ptr %B) nounwind uwtable ssp {
+; CHECK-LABEL: 'p0'
+; CHECK-NEXT:  Src: store i32 %n, ptr %A, align 4 --> Dst: store i32 %n, ptr %A, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: store i32 %n, ptr %A, align 4 --> Dst: %0 = load i32, ptr %arrayidx1, align 4
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: %0 = load i32, ptr %arrayidx1, align 4 --> Dst: %0 = load i32, ptr %arrayidx1, align 4
+; CHECK-NEXT:    da analyze - none!
+;
 entry:
   store i32 %n, ptr %A, align 4
-
-; CHECK-LABEL: p0
-; CHECK: da analyze - none!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - none!
-
   %arrayidx1 = getelementptr inbounds i32, ptr %B, i64 1
   %0 = load i32, ptr %arrayidx1, align 4
   ret i32 %0
@@ -29,14 +32,16 @@ entry:
 ;;  return B[1];
 
 define i32 @p1(i32 %n, ptr noalias %A, ptr noalias %B) nounwind uwtable ssp {
+; CHECK-LABEL: 'p1'
+; CHECK-NEXT:  Src: store i32 %n, ptr %A, align 4 --> Dst: store i32 %n, ptr %A, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: store i32 %n, ptr %A, align 4 --> Dst: %0 = load i32, ptr %arrayidx1, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: %0 = load i32, ptr %arrayidx1, align 4 --> Dst: %0 = load i32, ptr %arrayidx1, align 4
+; CHECK-NEXT:    da analyze - none!
+;
 entry:
   store i32 %n, ptr %A, align 4
-
-; CHECK-LABEL: p1
-; CHECK: da analyze - none!
-; CHECK: da analyze - none!
-; CHECK: da analyze - none!
-
   %arrayidx1 = getelementptr inbounds i32, ptr %B, i64 1
   %0 = load i32, ptr %arrayidx1, align 4
   ret i32 %0
@@ -52,18 +57,24 @@ entry:
 ;;        *B++ = A[i + 3][j + 2][k + 1];
 
 define void @p2(i64 %n, ptr %A, ptr %B) nounwind uwtable ssp {
+; CHECK-LABEL: 'p2'
+; CHECK-NEXT:  Src: store i64 %i.011, ptr %arrayidx8, align 8 --> Dst: store i64 %i.011, ptr %arrayidx8, align 8
+; CHECK-NEXT:    da analyze - output [* * *]!
+; CHECK-NEXT:  Src: store i64 %i.011, ptr %arrayidx8, align 8 --> Dst: %0 = load i64, ptr %arrayidx17, align 8
+; CHECK-NEXT:    da analyze - flow [* *|<]!
+; CHECK-NEXT:  Src: store i64 %i.011, ptr %arrayidx8, align 8 --> Dst: store i64 %0, ptr %B.addr.24, align 8
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: %0 = load i64, ptr %arrayidx17, align 8 --> Dst: %0 = load i64, ptr %arrayidx17, align 8
+; CHECK-NEXT:    da analyze - input [* * *]!
+; CHECK-NEXT:  Src: %0 = load i64, ptr %arrayidx17, align 8 --> Dst: store i64 %0, ptr %B.addr.24, align 8
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: store i64 %0, ptr %B.addr.24, align 8 --> Dst: store i64 %0, ptr %B.addr.24, align 8
+; CHECK-NEXT:    da analyze - output [* * *]!
+;
 entry:
   %cmp10 = icmp sgt i64 %n, 0
   br i1 %cmp10, label %for.cond1.preheader.preheader, label %for.end26
 
-; CHECK-LABEL: p2
-; CHECK: da analyze - output [* * *]!
-; CHECK: da analyze - flow [* *|<]!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - input [* * *]!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - output [* * *]!
-
 for.cond1.preheader.preheader:                    ; preds = %entry
   br label %for.cond1.preheader
 
@@ -425,18 +436,24 @@ for.end26:                                        ; preds = %for.end26.loopexit,
 ;;    *B++ = A[i];
 
 define void @p4(ptr %A, ptr %B, i64 %n) nounwind uwtable ssp {
+; CHECK-LABEL: 'p4'
+; CHECK-NEXT:  Src: store i32 %conv2, ptr %arrayidx, align 4 --> Dst: store i32 %conv2, ptr %arrayidx, align 4
+; CHECK-NEXT:    da analyze - output [*]!
+; CHECK-NEXT:  Src: store i32 %conv2, ptr %arrayidx, align 4 --> Dst: %0 = load i32, ptr %arrayidx5, align 4
+; CHECK-NEXT:    da analyze - flow [*|<]!
+; CHECK-NEXT:  Src: store i32 %conv2, ptr %arrayidx, align 4 --> Dst: store i32 %0, ptr %B.addr.02, align 4
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: %0 = load i32, ptr %arrayidx5, align 4 --> Dst: %0 = load i32, ptr %arrayidx5, align 4
+; CHECK-NEXT:    da analyze - input [*]!
+; CHECK-NEXT:  Src: %0 = load i32, ptr %arrayidx5, align 4 --> Dst: store i32 %0, ptr %B.addr.02, align 4
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: store i32 %0, ptr %B.addr.02, align 4 --> Dst: store i32 %0, ptr %B.addr.02, align 4
+; CHECK-NEXT:    da analyze - none!
+;
 entry:
   %cmp1 = icmp sgt i64 %n, 0
   br i1 %cmp1, label %for.body.preheader, label %for.end
 
-; CHECK-LABEL: p4
-; CHECK: da analyze - output [*]!
-; CHECK: da analyze - flow [*|<]!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - input [*]!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - none!
-
 for.body.preheader:                               ; preds = %entry
   br label %for.body
 
@@ -472,18 +489,24 @@ for.end:                                          ; preds = %for.end.loopexit, %
 ;;    *B++ = A[i];
 
 define void @p5(ptr %A, ptr %B, i64 %n) nounwind uwtable ssp {
+; CHECK-LABEL: 'p5'
+; CHECK-NEXT:  Src: store i32 %conv2, ptr %arrayidx, align 4 --> Dst: store i32 %conv2, ptr %arrayidx, align 4
+; CHECK-NEXT:    da analyze - output [*]!
+; CHECK-NEXT:  Src: store i32 %conv2, ptr %arrayidx, align 4 --> Dst: %0 = load i32, ptr %arrayidx5, align 4
+; CHECK-NEXT:    da analyze - flow [*|<]!
+; CHECK-NEXT:  Src: store i32 %conv2, ptr %arrayidx, align 4 --> Dst: store i32 %0, ptr %B.addr.02, align 4
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: %0 = load i32, ptr %arrayidx5, align 4 --> Dst: %0 = load i32, ptr %arrayidx5, align 4
+; CHECK-NEXT:    da analyze - input [*]!
+; CHECK-NEXT:  Src: %0 = load i32, ptr %arrayidx5, align 4 --> Dst: store i32 %0, ptr %B.addr.02, align 4
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: store i32 %0, ptr %B.addr.02, align 4 --> Dst: store i32 %0, ptr %B.addr.02, align 4
+; CHECK-NEXT:    da analyze - none!
+;
 entry:
   %cmp1 = icmp sgt i64 %n, 0
   br i1 %cmp1, label %for.body.preheader, label %for.end
 
-; CHECK-LABEL: p5
-; CHECK: da analyze - output [*]!
-; CHECK: da analyze - flow [*|<]!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - input [*]!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - none!
-
 for.body.preheader:                               ; preds = %entry
   br label %for.body
 
@@ -519,18 +542,24 @@ for.end:                                          ; preds = %for.end.loopexit, %
 ;;    *B++ = A[i];
 
 define void @p6(ptr %A, ptr %B, i64 %n) nounwind uwtable ssp {
+; CHECK-LABEL: 'p6'
+; CHECK-NEXT:  Src: store i32 %1, ptr %arrayidx, align 4 --> Dst: store i32 %1, ptr %arrayidx, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: store i32 %1, ptr %arrayidx, align 4 --> Dst: %2 = load i32, ptr %arrayidx3, align 4
+; CHECK-NEXT:    da analyze - consistent flow [2]!
+; CHECK-NEXT:  Src: store i32 %1, ptr %arrayidx, align 4 --> Dst: store i32 %2, ptr %B.addr.02, align 4
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: %2 = load i32, ptr %arrayidx3, align 4 --> Dst: %2 = load i32, ptr %arrayidx3, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: %2 = load i32, ptr %arrayidx3, align 4 --> Dst: store i32 %2, ptr %B.addr.02, align 4
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: store i32 %2, ptr %B.addr.02, align 4 --> Dst: store i32 %2, ptr %B.addr.02, align 4
+; CHECK-NEXT:    da analyze - none!
+;
 entry:
   %cmp1 = icmp sgt i64 %n, 0
   br i1 %cmp1, label %for.body.preheader, label %for.end
 
-; CHECK-LABEL: p6
-; CHECK: da analyze - none!
-; CHECK: da analyze - consistent flow [2]!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - none!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - none!
-
 for.body.preheader:                               ; preds = %entry
   br label %for.body
 
@@ -562,18 +591,23 @@ for.end:                                          ; preds = %for.end.loopexit, %
 ;;  *B = A[n + 1];
 
 define void @p7(ptr %A, ptr %B, i8 signext %n) nounwind uwtable ssp {
+; CHECK-LABEL: 'p7'
+; CHECK-NEXT:  Src: store i32 0, ptr %arrayidx, align 4 --> Dst: store i32 0, ptr %arrayidx, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: store i32 0, ptr %arrayidx, align 4 --> Dst: %0 = load i32, ptr %arrayidx2, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: store i32 0, ptr %arrayidx, align 4 --> Dst: store i32 %0, ptr %B, align 4
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: %0 = load i32, ptr %arrayidx2, align 4 --> Dst: %0 = load i32, ptr %arrayidx2, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: %0 = load i32, ptr %arrayidx2, align 4 --> Dst: store i32 %0, ptr %B, align 4
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: store i32 %0, ptr %B, align 4 --> Dst: store i32 %0, ptr %B, align 4
+; CHECK-NEXT:    da analyze - none!
+;
 entry:
   %idxprom = sext i8 %n to i64
   %arrayidx = getelementptr inbounds i32, ptr %A, i64 %idxprom
-
-; CHECK-LABEL: p7
-; CHECK: da analyze - none!
-; CHECK: da analyze - none!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - none!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - none!
-
   store i32 0, ptr %arrayidx, align 4
   %conv = sext i8 %n to i64
   %add = add i64 %conv, 1
@@ -589,19 +623,24 @@ entry:
 ;;  *B = A[n + 1];
 
 define void @p8(ptr %A, ptr %B, i16 signext %n) nounwind uwtable ssp {
+; CHECK-LABEL: 'p8'
+; CHECK-NEXT:  Src: store i32 0, ptr %arrayidx, align 4 --> Dst: store i32 0, ptr %arrayidx, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: store i32 0, ptr %arrayidx, align 4 --> Dst: %0 = load i32, ptr %arrayidx2, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: store i32 0, ptr %arrayidx, align 4 --> Dst: store i32 %0, ptr %B, align 4
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: %0 = load i32, ptr %arrayidx2, align 4 --> Dst: %0 = load i32, ptr %arrayidx2, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: %0 = load i32, ptr %arrayidx2, align 4 --> Dst: store i32 %0, ptr %B, align 4
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: store i32 %0, ptr %B, align 4 --> Dst: store i32 %0, ptr %B, align 4
+; CHECK-NEXT:    da analyze - none!
+;
 entry:
   %idxprom = sext i16 %n to i64
   %arrayidx = getelementptr inbounds i32, ptr %A, i64 %idxprom
   store i32 0, ptr %arrayidx, align 4
-
-; CHECK-LABEL: p8
-; CHECK: da analyze - none!
-; CHECK: da analyze - none!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - none!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - none!
-
   %conv = sext i16 %n to i64
   %add = add i64 %conv, 1
   %arrayidx2 = getelementptr inbounds i32, ptr %A, i64 %add
@@ -616,19 +655,24 @@ entry:
 ;;  *B = A[n + 1];
 
 define void @p9(ptr %A, ptr %B, i32 %n) nounwind uwtable ssp {
+; CHECK-LABEL: 'p9'
+; CHECK-NEXT:  Src: store i32 0, ptr %arrayidx, align 4 --> Dst: store i32 0, ptr %arrayidx, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: store i32 0, ptr %arrayidx, align 4 --> Dst: %0 = load i32, ptr %arrayidx2, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: store i32 0, ptr %arrayidx, align 4 --> Dst: store i32 %0, ptr %B, align 4
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: %0 = load i32, ptr %arrayidx2, align 4 --> Dst: %0 = load i32, ptr %arrayidx2, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: %0 = load i32, ptr %arrayidx2, align 4 --> Dst: store i32 %0, ptr %B, align 4
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: store i32 %0, ptr %B, align 4 --> Dst: store i32 %0, ptr %B, align 4
+; CHECK-NEXT:    da analyze - none!
+;
 entry:
   %idxprom = sext i32 %n to i64
   %arrayidx = getelementptr inbounds i32, ptr %A, i64 %idxprom
   store i32 0, ptr %arrayidx, align 4
-
-; CHECK-LABEL: p9
-; CHECK: da analyze - none!
-; CHECK: da analyze - none!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - none!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - none!
-
   %add = add nsw i32 %n, 1
   %idxprom1 = sext i32 %add to i64
   %arrayidx2 = getelementptr inbounds i32, ptr %A, i64 %idxprom1
@@ -643,19 +687,24 @@ entry:
 ;;  *B = A[n + 1];
 
 define void @p10(ptr %A, ptr %B, i32 %n) nounwind uwtable ssp {
+; CHECK-LABEL: 'p10'
+; CHECK-NEXT:  Src: store i32 0, ptr %arrayidx, align 4 --> Dst: store i32 0, ptr %arrayidx, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: store i32 0, ptr %arrayidx, align 4 --> Dst: %0 = load i32, ptr %arrayidx2, align 4
+; CHECK-NEXT:    da analyze - flow [|<]!
+; CHECK-NEXT:  Src: store i32 0, ptr %arrayidx, align 4 --> Dst: store i32 %0, ptr %B, align 4
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: %0 = load i32, ptr %arrayidx2, align 4 --> Dst: %0 = load i32, ptr %arrayidx2, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: %0 = load i32, ptr %arrayidx2, align 4 --> Dst: store i32 %0, ptr %B, align 4
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: store i32 %0, ptr %B, align 4 --> Dst: store i32 %0, ptr %B, align 4
+; CHECK-NEXT:    da analyze - none!
+;
 entry:
   %idxprom = zext i32 %n to i64
   %arrayidx = getelementptr inbounds i32, ptr %A, i64 %idxprom
   store i32 0, ptr %arrayidx, align 4
-
-; CHECK-LABEL: p10
-; CHECK: da analyze - none!
-; CHECK: da analyze - flow [|<]!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - none!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - none!
-
   %add = add i32 %n, 1
   %idxprom1 = zext i32 %add to i64
   %arrayidx2 = getelementptr inbounds i32, ptr %A, i64 %idxprom1
@@ -676,6 +725,14 @@ entry:
 %struct.S = type { i32 }
 
 define void @foo(ptr %s, i32 %size) nounwind uwtable ssp {
+; CHECK-LABEL: 'foo'
+; CHECK-NEXT:  Src: %1 = load i32, ptr %0, align 4 --> Dst: %1 = load i32, ptr %0, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: %1 = load i32, ptr %0, align 4 --> Dst: store i32 %1, ptr %i.02, align 4
+; CHECK-NEXT:    da analyze - consistent anti [1]!
+; CHECK-NEXT:  Src: store i32 %1, ptr %i.02, align 4 --> Dst: store i32 %1, ptr %i.02, align 4
+; CHECK-NEXT:    da analyze - none!
+;
 entry:
   %idx.ext = zext i32 %size to i64
   %add.ptr.sum = add i64 %idx.ext, -1
@@ -683,11 +740,6 @@ entry:
   %cmp1 = icmp eq i64 %add.ptr.sum, 0
   br i1 %cmp1, label %while.end, label %while.body.preheader
 
-; CHECK-LABEL: foo
-; CHECK: da analyze - none!
-; CHECK: da analyze - consistent anti [1]!
-; CHECK: da analyze - none!
-
 while.body.preheader:                             ; preds = %entry
   br label %while.body
 
diff --git a/llvm/test/Analysis/DependenceAnalysis/PreliminaryNoValidityCheckFixedSize.ll b/llvm/test/Analysis/DependenceAnalysis/PreliminaryNoValidityCheckFixedSize.ll
index b4660d5b1d546..404018707c0a5 100644
--- a/llvm/test/Analysis/DependenceAnalysis/PreliminaryNoValidityCheckFixedSize.ll
+++ b/llvm/test/Analysis/DependenceAnalysis/PreliminaryNoValidityCheckFixedSize.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --version 5
 ; RUN: opt < %s -disable-output "-passes=print<da>" -aa-pipeline=basic-aa 2>&1 \
 ; RUN:   -da-disable-delinearization-checks | FileCheck %s
 ; RUN: opt < %s -disable-output "-passes=print<da>" -aa-pipeline=basic-aa 2>&1 \
@@ -15,26 +16,38 @@ target triple = "x86_64-apple-macosx10.6.0"
 ;;        *B++ = A[i + 3][j + 2][k + 1];
 
 define void @p2(i64 %n, ptr %A, ptr %B) nounwind uwtable ssp {
+; CHECK-LABEL: 'p2'
+; CHECK-NEXT:  Src: store i64 %i.011, ptr %arrayidx8, align 8 --> Dst: store i64 %i.011, ptr %arrayidx8, align 8
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: store i64 %i.011, ptr %arrayidx8, align 8 --> Dst: %0 = load i64, ptr %arrayidx17, align 8
+; CHECK-NEXT:    da analyze - flow [-3 -2]!
+; CHECK-NEXT:  Src: store i64 %i.011, ptr %arrayidx8, align 8 --> Dst: store i64 %0, ptr %B.addr.24, align 8
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: %0 = load i64, ptr %arrayidx17, align 8 --> Dst: %0 = load i64, ptr %arrayidx17, align 8
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: %0 = load i64, ptr %arrayidx17, align 8 --> Dst: store i64 %0, ptr %B.addr.24, align 8
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: store i64 %0, ptr %B.addr.24, align 8 --> Dst: store i64 %0, ptr %B.addr.24, align 8
+; CHECK-NEXT:    da analyze - output [* * *]!
+;
+; LIN-LABEL: 'p2'
+; LIN-NEXT:  Src: store i64 %i.011, ptr %arrayidx8, align 8 --> Dst: store i64 %i.011, ptr %arrayidx8, align 8
+; LIN-NEXT:    da analyze - output [* * *]!
+; LIN-NEXT:  Src: store i64 %i.011, ptr %arrayidx8, align 8 --> Dst: %0 = load i64, ptr %arrayidx17, align 8
+; LIN-NEXT:    da analyze - flow [* *|<]!
+; LIN-NEXT:  Src: store i64 %i.011, ptr %arrayidx8, align 8 --> Dst: store i64 %0, ptr %B.addr.24, align 8
+; LIN-NEXT:    da analyze - confused!
+; LIN-NEXT:  Src: %0 = load i64, ptr %arrayidx17, align 8 --> Dst: %0 = load i64, ptr %arrayidx17, align 8
+; LIN-NEXT:    da analyze - input [* * *]!
+; LIN-NEXT:  Src: %0 = load i64, ptr %arrayidx17, align 8 --> Dst: store i64 %0, ptr %B.addr.24, align 8
+; LIN-NEXT:    da analyze - confused!
+; LIN-NEXT:  Src: store i64 %0, ptr %B.addr.24, align 8 --> Dst: store i64 %0, ptr %B.addr.24, align 8
+; LIN-NEXT:    da analyze - output [* * *]!
+;
 entry:
   %cmp10 = icmp sgt i64 %n, 0
   br i1 %cmp10, label %for.cond1.preheader.preheader, label %for.end26
 
-; CHECK-LABEL: p2
-; CHECK: da analyze - none!
-; CHECK: da analyze - flow [-3 -2]!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - none!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - output [* * *]!
-
-; LIN-LABEL: p2
-; LIN: da analyze - output [* * *]!
-; LIN: da analyze - flow [* *|<]!
-; LIN: da analyze - confused!
-; LIN: da analyze - input [* * *]!
-; LIN: da analyze - confused!
-; LIN: da analyze - output [* * *]!
-
 for.cond1.preheader.preheader:                    ; preds = %entry
   br label %for.cond1.preheader
 
diff --git a/llvm/test/Analysis/DependenceAnalysis/Propagating.ll b/llvm/test/Analysis/DependenceAnalysis/Propagating.ll
index 6ceffc6cad40b..866f515baeafb 100644
--- a/llvm/test/Analysis/DependenceAnalysis/Propagating.ll
+++ b/llvm/test/Analysis/DependenceAnalysis/Propagating.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --version 5
 ; RUN: opt < %s -disable-output "-passes=print<da>" -aa-pipeline=basic-aa 2>&1 \
 ; RUN: | FileCheck %s
 
@@ -11,17 +12,23 @@ target triple = "x86_64-apple-macosx10.6.0"
 ;;      *B++ = A[i][i + j];
 
 define void @prop0(ptr %A, ptr %B, i32 %n) nounwind uwtable ssp {
+; CHECK-LABEL: 'prop0'
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx5, align 4 --> Dst: store i32 %conv, ptr %arrayidx5, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx5, align 4 --> Dst: %0 = load i32, ptr %arrayidx8, align 4
+; CHECK-NEXT:    da analyze - flow [< >]!
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx5, align 4 --> Dst: store i32 %0, ptr %B.addr.11, align 4
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: %0 = load i32, ptr %arrayidx8, align 4 --> Dst: %0 = load i32, ptr %arrayidx8, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: %0 = load i32, ptr %arrayidx8, align 4 --> Dst: store i32 %0, ptr %B.addr.11, align 4
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: store i32 %0, ptr %B.addr.11, align 4 --> Dst: store i32 %0, ptr %B.addr.11, align 4
+; CHECK-NEXT:    da analyze - none!
+;
 entry:
   br label %for.cond1.preheader
 
-; CHECK-LABEL: prop0
-; CHECK: da analyze - none!
-; CHECK: da analyze - flow [< >]!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - none!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - none!
-
 for.cond1.preheader:                              ; preds = %entry, %for.inc9
   %B.addr.04 = phi ptr [ %B, %entry ], [ %scevgep, %for.inc9 ]
   %i.03 = phi i64 [ 0, %entry ], [ %inc10, %for.inc9 ]
@@ -62,17 +69,23 @@ for.end11:                                        ; preds = %for.inc9
 ;;        *B++ = A[j - i][i][j + k];
 
 define void @prop1(ptr %A, ptr %B, i32 %n) nounwind uwtable ssp {
+; CHECK-LABEL: 'prop1'
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx9, align 4 --> Dst: store i32 %conv, ptr %arrayidx9, align 4
+; CHECK-NEXT:    da analyze - output [* * *]!
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx9, align 4 --> Dst: %0 = load i32, ptr %arrayidx14, align 4
+; CHECK-NEXT:    da analyze - flow [<> <> *]!
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx9, align 4 --> Dst: store i32 %0, ptr %B.addr.21, align 4
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: %0 = load i32, ptr %arrayidx14, align 4 --> Dst: %0 = load i32, ptr %arrayidx14, align 4
+; CHECK-NEXT:    da analyze - input [* * *]!
+; CHECK-NEXT:  Src: %0 = load i32, ptr %arrayidx14, align 4 --> Dst: store i32 %0, ptr %B.addr.21, align 4
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: store i32 %0, ptr %B.addr.21, align 4 --> Dst: store i32 %0, ptr %B.addr.21, align 4
+; CHECK-NEXT:    da analyze - none!
+;
 entry:
   br label %for.cond1.preheader
 
-; CHECK-LABEL: prop1
-; CHECK: da analyze - output [* * *]!
-; CHECK: da analyze - flow [<> <> *]!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - input [* * *]!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - none!
-
 for.cond1.preheader:                              ; preds = %entry, %for.inc18
   %B.addr.06 = phi ptr [ %B, %entry ], [ %scevgep7, %for.inc18 ]
   %i.05 = phi i64 [ 0, %entry ], [ %inc19, %for.inc18 ]
@@ -126,17 +139,23 @@ for.end20:                                        ; preds = %for.inc18
 ;;      *B++ = A[i][i + j + 110];
 
 define void @prop2(ptr %A, ptr %B, i32 %n) nounwind uwtable ssp {
+; CHECK-LABEL: 'prop2'
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx4, align 4 --> Dst: store i32 %conv, ptr %arrayidx4, align 4
+; CHECK-NEXT:    da analyze - consistent output [0 S]!
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx4, align 4 --> Dst: %0 = load i32, ptr %arrayidx7, align 4
+; CHECK-NEXT:    da analyze - flow [> *]!
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx4, align 4 --> Dst: store i32 %0, ptr %B.addr.11, align 4
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: %0 = load i32, ptr %arrayidx7, align 4 --> Dst: %0 = load i32, ptr %arrayidx7, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: %0 = load i32, ptr %arrayidx7, align 4 --> Dst: store i32 %0, ptr %B.addr.11, align 4
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: store i32 %0, ptr %B.addr.11, align 4 --> Dst: store i32 %0, ptr %B.addr.11, align 4
+; CHECK-NEXT:    da analyze - none!
+;
 entry:
   br label %for.cond1.preheader
 
-; CHECK-LABEL: prop2
-; CHECK: da analyze - consistent output [0 S]!
-; CHECK: da analyze - flow [> *]!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - none!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - none!
-
 for.cond1.preheader:                              ; preds = %entry, %for.inc8
   %B.addr.04 = phi ptr [ %B, %entry ], [ %scevgep, %for.inc8 ]
   %i.03 = phi i64 [ 0, %entry ], [ %inc9, %for.inc8 ]
@@ -177,17 +196,23 @@ for.end10:                                        ; preds = %for.inc8
 ;;      *B++ = A[i][2*j - i + 5];
 
 define void @prop3(ptr %A, ptr %B, i32 %n) nounwind uwtable ssp {
+; CHECK-LABEL: 'prop3'
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx4, align 4 --> Dst: store i32 %conv, ptr %arrayidx4, align 4
+; CHECK-NEXT:    da analyze - output [* *]!
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx4, align 4 --> Dst: %0 = load i32, ptr %arrayidx8, align 4
+; CHECK-NEXT:    da analyze - flow [<> *]!
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx4, align 4 --> Dst: store i32 %0, ptr %B.addr.11, align 4
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: %0 = load i32, ptr %arrayidx8, align 4 --> Dst: %0 = load i32, ptr %arrayidx8, align 4
+; CHECK-NEXT:    da analyze - input [* *]!
+; CHECK-NEXT:  Src: %0 = load i32, ptr %arrayidx8, align 4 --> Dst: store i32 %0, ptr %B.addr.11, align 4
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: store i32 %0, ptr %B.addr.11, align 4 --> Dst: store i32 %0, ptr %B.addr.11, align 4
+; CHECK-NEXT:    da analyze - none!
+;
 entry:
   br label %for.cond1.preheader
 
-; CHECK-LABEL: prop3
-; CHECK: da analyze - output [* *]!
-; CHECK: da analyze - flow [<> *]!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - input [* *]!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - none!
-
 for.cond1.preheader:                              ; preds = %entry, %for.inc9
   %B.addr.04 = phi ptr [ %B, %entry ], [ %scevgep, %for.inc9 ]
   %i.03 = phi i64 [ 0, %entry ], [ %inc10, %for.inc9 ]
@@ -229,17 +254,23 @@ for.end11:                                        ; preds = %for.inc9
 ;;      *B++ = A[i][2*i + j];
 
 define void @prop4(ptr %A, ptr %B, i32 %n) nounwind uwtable ssp {
+; CHECK-LABEL: 'prop4'
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx6, align 4 --> Dst: store i32 %conv, ptr %arrayidx6, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx6, align 4 --> Dst: %0 = load i32, ptr %arrayidx10, align 4
+; CHECK-NEXT:    da analyze - flow [< <>]!
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx6, align 4 --> Dst: store i32 %0, ptr %B.addr.11, align 4
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: %0 = load i32, ptr %arrayidx10, align 4 --> Dst: %0 = load i32, ptr %arrayidx10, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: %0 = load i32, ptr %arrayidx10, align 4 --> Dst: store i32 %0, ptr %B.addr.11, align 4
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: store i32 %0, ptr %B.addr.11, align 4 --> Dst: store i32 %0, ptr %B.addr.11, align 4
+; CHECK-NEXT:    da analyze - none!
+;
 entry:
   br label %for.cond1.preheader
 
-; CHECK-LABEL: prop4
-; CHECK: da analyze - none!
-; CHECK: da analyze - flow [< <>]!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - none!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - none!
-
 for.cond1.preheader:                              ; preds = %entry, %for.inc11
   %B.addr.04 = phi ptr [ %B, %entry ], [ %scevgep, %for.inc11 ]
   %i.03 = phi i64 [ 0, %entry ], [ %inc12, %for.inc11 ]
@@ -282,17 +313,23 @@ for.end13:                                        ; preds = %for.inc11
 ;;      *B++ = A[i][i][3*i + j];
 
 define void @prop5(ptr %A, ptr %B, i32 %n) nounwind uwtable ssp {
+; CHECK-LABEL: 'prop5'
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx7, align 4 --> Dst: store i32 %conv, ptr %arrayidx7, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx7, align 4 --> Dst: %0 = load i32, ptr %arrayidx12, align 4
+; CHECK-NEXT:    da analyze - flow [* *|<]!
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx7, align 4 --> Dst: store i32 %0, ptr %B.addr.11, align 4
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: %0 = load i32, ptr %arrayidx12, align 4 --> Dst: %0 = load i32, ptr %arrayidx12, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: %0 = load i32, ptr %arrayidx12, align 4 --> Dst: store i32 %0, ptr %B.addr.11, align 4
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: store i32 %0, ptr %B.addr.11, align 4 --> Dst: store i32 %0, ptr %B.addr.11, align 4
+; CHECK-NEXT:    da analyze - none!
+;
 entry:
   br label %for.cond1.preheader
 
-; CHECK-LABEL: prop5
-; CHECK: da analyze - none!
-; CHECK: da analyze - flow [* *|<]!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - none!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - none!
-
 for.cond1.preheader:                              ; preds = %entry, %for.inc13
   %B.addr.04 = phi ptr [ %B, %entry ], [ %scevgep, %for.inc13 ]
   %i.03 = phi i64 [ 0, %entry ], [ %inc14, %for.inc13 ]
@@ -336,17 +373,23 @@ for.end15:                                        ; preds = %for.inc13
 ;;      *B++ = A[2*i][8*i + j];
 
 define void @prop6(ptr %A, ptr %B, i32 %n) nounwind uwtable ssp {
+; CHECK-LABEL: 'prop6'
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx6, align 4 --> Dst: store i32 %conv, ptr %arrayidx6, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx6, align 4 --> Dst: %0 = load i32, ptr %arrayidx11, align 4
+; CHECK-NEXT:    da analyze - flow [=> <>]!
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx6, align 4 --> Dst: store i32 %0, ptr %B.addr.11, align 4
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: %0 = load i32, ptr %arrayidx11, align 4 --> Dst: %0 = load i32, ptr %arrayidx11, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: %0 = load i32, ptr %arrayidx11, align 4 --> Dst: store i32 %0, ptr %B.addr.11, align 4
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: store i32 %0, ptr %B.addr.11, align 4 --> Dst: store i32 %0, ptr %B.addr.11, align 4
+; CHECK-NEXT:    da analyze - none!
+;
 entry:
   br label %for.cond1.preheader
 
-; CHECK-LABEL: prop6
-; CHECK: da analyze - none!
-; CHECK: da analyze - flow [=> <>]!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - none!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - none!
-
 for.cond1.preheader:                              ; preds = %entry, %for.inc12
   %B.addr.04 = phi ptr [ %B, %entry ], [ %scevgep, %for.inc12 ]
   %i.03 = phi i64 [ 0, %entry ], [ %inc13, %for.inc12 ]
@@ -390,17 +433,23 @@ for.end14:                                        ; preds = %for.inc12
 ;;      *B++ = A[-2*i + 20][5*i + j];
 
 define void @prop7(ptr %A, ptr %B, i32 %n) nounwind uwtable ssp {
+; CHECK-LABEL: 'prop7'
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx7, align 4 --> Dst: store i32 %conv, ptr %arrayidx7, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx7, align 4 --> Dst: %0 = load i32, ptr %arrayidx13, align 4
+; CHECK-NEXT:    da analyze - flow [* <>]!
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx7, align 4 --> Dst: store i32 %0, ptr %B.addr.11, align 4
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: %0 = load i32, ptr %arrayidx13, align 4 --> Dst: %0 = load i32, ptr %arrayidx13, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: %0 = load i32, ptr %arrayidx13, align 4 --> Dst: store i32 %0, ptr %B.addr.11, align 4
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: store i32 %0, ptr %B.addr.11, align 4 --> Dst: store i32 %0, ptr %B.addr.11, align 4
+; CHECK-NEXT:    da analyze - none!
+;
 entry:
   br label %for.cond1.preheader
 
-; CHECK-LABEL: prop7
-; CHECK: da analyze - none!
-; CHECK: da analyze - flow [* <>]!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - none!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - none!
-
 for.cond1.preheader:                              ; preds = %entry, %for.inc14
   %B.addr.04 = phi ptr [ %B, %entry ], [ %scevgep, %for.inc14 ]
   %i.03 = phi i64 [ 0, %entry ], [ %inc15, %for.inc14 ]
@@ -446,17 +495,23 @@ for.end16:                                        ; preds = %for.inc14
 ;;      *B++ = A[-2*i + 4][5*i + j];
 
 define void @prop8(ptr %A, ptr %B, i32 %n) nounwind uwtable ssp {
+; CHECK-LABEL: 'prop8'
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx4, align 4 --> Dst: store i32 %conv, ptr %arrayidx4, align 4
+; CHECK-NEXT:    da analyze - consistent output [S 0]!
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx4, align 4 --> Dst: %0 = load i32, ptr %arrayidx9, align 4
+; CHECK-NEXT:    da analyze - flow [=> <]!
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx4, align 4 --> Dst: store i32 %0, ptr %B.addr.11, align 4
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: %0 = load i32, ptr %arrayidx9, align 4 --> Dst: %0 = load i32, ptr %arrayidx9, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: %0 = load i32, ptr %arrayidx9, align 4 --> Dst: store i32 %0, ptr %B.addr.11, align 4
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: store i32 %0, ptr %B.addr.11, align 4 --> Dst: store i32 %0, ptr %B.addr.11, align 4
+; CHECK-NEXT:    da analyze - none!
+;
 entry:
   br label %for.cond1.preheader
 
-; CHECK-LABEL: prop8
-; CHECK: da analyze - consistent output [S 0]!
-; CHECK: da analyze - flow [=> <]!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - none!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - none!
-
 for.cond1.preheader:                              ; preds = %entry, %for.inc10
   %B.addr.04 = phi ptr [ %B, %entry ], [ %scevgep, %for.inc10 ]
   %i.03 = phi i64 [ 0, %entry ], [ %inc11, %for.inc10 ]
@@ -498,17 +553,23 @@ for.end12:                                        ; preds = %for.inc10
 ;;      *B++ = A[4][j];
 
 define void @prop9(ptr %A, ptr %B, i32 %n) nounwind uwtable ssp {
+; CHECK-LABEL: 'prop9'
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx7, align 4 --> Dst: store i32 %conv, ptr %arrayidx7, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx7, align 4 --> Dst: %0 = load i32, ptr %arrayidx9, align 4
+; CHECK-NEXT:    da analyze - flow [<= <]!
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx7, align 4 --> Dst: store i32 %0, ptr %B.addr.11, align 4
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: %0 = load i32, ptr %arrayidx9, align 4 --> Dst: %0 = load i32, ptr %arrayidx9, align 4
+; CHECK-NEXT:    da analyze - consistent input [S 0]!
+; CHECK-NEXT:  Src: %0 = load i32, ptr %arrayidx9, align 4 --> Dst: store i32 %0, ptr %B.addr.11, align 4
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: store i32 %0, ptr %B.addr.11, align 4 --> Dst: store i32 %0, ptr %B.addr.11, align 4
+; CHECK-NEXT:    da analyze - none!
+;
 entry:
   br label %for.cond1.preheader
 
-; CHECK-LABEL: prop9
-; CHECK: da analyze - none!
-; CHECK: da analyze - flow [<= <]!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - consistent input [S 0]!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - none!
-
 for.cond1.preheader:                              ; preds = %entry, %for.inc10
   %B.addr.04 = phi ptr [ %B, %entry ], [ %scevgep, %for.inc10 ]
   %i.03 = phi i64 [ 0, %entry ], [ %inc11, %for.inc10 ]
diff --git a/llvm/test/Analysis/DependenceAnalysis/Separability.ll b/llvm/test/Analysis/DependenceAnalysis/Separability.ll
index 14334d84576c8..2ed9cca4d1fc0 100644
--- a/llvm/test/Analysis/DependenceAnalysis/Separability.ll
+++ b/llvm/test/Analysis/DependenceAnalysis/Separability.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --version 5
 ; RUN: opt < %s -disable-output "-passes=print<da>" -aa-pipeline=basic-aa 2>&1 \
 ; RUN: | FileCheck %s
 
@@ -13,16 +14,23 @@ target triple = "x86_64-apple-macosx10.6.0"
 ;;          *B++ = A[10][i + 10][2*j - l];
 
 define void @sep0(ptr %A, ptr %B, i32 %n) nounwind uwtable ssp {
+; CHECK-LABEL: 'sep0'
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx11, align 4 --> Dst: store i32 %conv, ptr %arrayidx11, align 4
+; CHECK-NEXT:    da analyze - output [0 * * S]!
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx11, align 4 --> Dst: %0 = load i32, ptr %arrayidx15, align 4
+; CHECK-NEXT:    da analyze - flow [* * * *|<]!
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx11, align 4 --> Dst: store i32 %0, ptr %B.addr.31, align 4
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: %0 = load i32, ptr %arrayidx15, align 4 --> Dst: %0 = load i32, ptr %arrayidx15, align 4
+; CHECK-NEXT:    da analyze - input [* * S *]!
+; CHECK-NEXT:  Src: %0 = load i32, ptr %arrayidx15, align 4 --> Dst: store i32 %0, ptr %B.addr.31, align 4
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: store i32 %0, ptr %B.addr.31, align 4 --> Dst: store i32 %0, ptr %B.addr.31, align 4
+; CHECK-NEXT:    da analyze - none!
+;
 entry:
   br label %for.cond1.preheader
 
-; CHECK: da analyze - output [0 * * S]!
-; CHECK: da analyze - flow [* * * *|<]!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - input [* * S *]!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - none!
-
 for.cond1.preheader:                              ; preds = %entry, %for.inc22
   %B.addr.08 = phi ptr [ %B, %entry ], [ %scevgep11, %for.inc22 ]
   %i.07 = phi i64 [ 0, %entry ], [ %inc23, %for.inc22 ]
@@ -88,16 +96,23 @@ for.end24:                                        ; preds = %for.inc22
 ;;          *B++ = A[10][i + 10][2*j - l];
 
 define void @sep1(ptr %A, ptr %B, i32 %n) nounwind uwtable ssp {
+; CHECK-LABEL: 'sep1'
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx11, align 4 --> Dst: store i32 %conv, ptr %arrayidx11, align 4
+; CHECK-NEXT:    da analyze - output [0 * * S]!
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx11, align 4 --> Dst: %0 = load i32, ptr %arrayidx15, align 4
+; CHECK-NEXT:    da analyze - flow [* * * *|<]!
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx11, align 4 --> Dst: store i32 %0, ptr %B.addr.31, align 4
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: %0 = load i32, ptr %arrayidx15, align 4 --> Dst: %0 = load i32, ptr %arrayidx15, align 4
+; CHECK-NEXT:    da analyze - input [* * S *]!
+; CHECK-NEXT:  Src: %0 = load i32, ptr %arrayidx15, align 4 --> Dst: store i32 %0, ptr %B.addr.31, align 4
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: store i32 %0, ptr %B.addr.31, align 4 --> Dst: store i32 %0, ptr %B.addr.31, align 4
+; CHECK-NEXT:    da analyze - none!
+;
 entry:
   br label %for.cond1.preheader
 
-; CHECK: da analyze - output [0 * * S]!
-; CHECK: da analyze - flow [* * * *|<]!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - input [* * S *]!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - none!
-
 for.cond1.preheader:                              ; preds = %entry, %for.inc22
   %B.addr.08 = phi ptr [ %B, %entry ], [ %scevgep11, %for.inc22 ]
   %i.07 = phi i64 [ 0, %entry ], [ %inc23, %for.inc22 ]
@@ -163,16 +178,23 @@ for.end24:                                        ; preds = %for.inc22
 ;;          *B++ = A[10][i + 10][j + k][l + 10];
 
 define void @sep2(ptr %A, ptr %B, i32 %n) nounwind uwtable ssp {
+; CHECK-LABEL: 'sep2'
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx12, align 4 --> Dst: store i32 %conv, ptr %arrayidx12, align 4
+; CHECK-NEXT:    da analyze - consistent output [0 S 0 0]!
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx12, align 4 --> Dst: %0 = load i32, ptr %arrayidx19, align 4
+; CHECK-NEXT:    da analyze - flow [> * * -10]!
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx12, align 4 --> Dst: store i32 %0, ptr %B.addr.31, align 4
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: %0 = load i32, ptr %arrayidx19, align 4 --> Dst: %0 = load i32, ptr %arrayidx19, align 4
+; CHECK-NEXT:    da analyze - input [0 * * 0]!
+; CHECK-NEXT:  Src: %0 = load i32, ptr %arrayidx19, align 4 --> Dst: store i32 %0, ptr %B.addr.31, align 4
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: store i32 %0, ptr %B.addr.31, align 4 --> Dst: store i32 %0, ptr %B.addr.31, align 4
+; CHECK-NEXT:    da analyze - none!
+;
 entry:
   br label %for.cond1.preheader
 
-; CHECK: da analyze - consistent output [0 S 0 0]!
-; CHECK: da analyze - flow [> * * -10]!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - input [0 * * 0]!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - none!
-
 for.cond1.preheader:                              ; preds = %entry, %for.inc26
   %B.addr.08 = phi ptr [ %B, %entry ], [ %scevgep11, %for.inc26 ]
   %i.07 = phi i64 [ 0, %entry ], [ %inc27, %for.inc26 ]
@@ -238,16 +260,23 @@ for.end28:                                        ; preds = %for.inc26
 ;;          *B++ = A[10][i + 10][j + k][l + 10];
 
 define void @sep3(ptr %A, ptr %B, i32 %n) nounwind uwtable ssp {
+; CHECK-LABEL: 'sep3'
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx13, align 4 --> Dst: store i32 %conv, ptr %arrayidx13, align 4
+; CHECK-NEXT:    da analyze - consistent output [0 S 0 0]!
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx13, align 4 --> Dst: %0 = load i32, ptr %arrayidx20, align 4
+; CHECK-NEXT:    da analyze - flow [> * * *]!
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx13, align 4 --> Dst: store i32 %0, ptr %B.addr.31, align 4
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: %0 = load i32, ptr %arrayidx20, align 4 --> Dst: %0 = load i32, ptr %arrayidx20, align 4
+; CHECK-NEXT:    da analyze - input [0 * * 0]!
+; CHECK-NEXT:  Src: %0 = load i32, ptr %arrayidx20, align 4 --> Dst: store i32 %0, ptr %B.addr.31, align 4
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: store i32 %0, ptr %B.addr.31, align 4 --> Dst: store i32 %0, ptr %B.addr.31, align 4
+; CHECK-NEXT:    da analyze - none!
+;
 entry:
   br label %for.cond1.preheader
 
-; CHECK: da analyze - consistent output [0 S 0 0]!
-; CHECK: da analyze - flow [> * * *]!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - input [0 * * 0]!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - none!
-
 for.cond1.preheader:                              ; preds = %entry, %for.inc27
   %B.addr.08 = phi ptr [ %B, %entry ], [ %scevgep11, %for.inc27 ]
   %i.07 = phi i64 [ 0, %entry ], [ %inc28, %for.inc27 ]
diff --git a/llvm/test/Analysis/DependenceAnalysis/SimpleSIVNoValidityCheck.ll b/llvm/test/Analysis/DependenceAnalysis/SimpleSIVNoValidityCheck.ll
index 860795138c983..4346507ba8f90 100644
--- a/llvm/test/Analysis/DependenceAnalysis/SimpleSIVNoValidityCheck.ll
+++ b/llvm/test/Analysis/DependenceAnalysis/SimpleSIVNoValidityCheck.ll
@@ -1,18 +1,9 @@
+; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --version 5
 ; RUN: opt < %s -disable-output -passes="print<da>"                            \
 ; RUN: -da-disable-delinearization-checks 2>&1 | FileCheck %s
 ; RUN: opt < %s -disable-output -passes="print<da>"                            \
 ; RUN: 2>&1 | FileCheck --check-prefix=LIN %s
 
-; CHECK-LABEL: t1
-; CHECK: da analyze - none!
-; CHECK: da analyze - consistent anti [1 -2]!
-; CHECK: da analyze - none!
-
-; LIN-LABEL: t1
-; LIN: da analyze - input [* *]!
-; LIN: da analyze - anti [* *|<]!
-; LIN: da analyze - output [* *]!
-
 ;; void t1(int n, int m, int a[][m]) {
 ;;   for (int i = 0; i < n-1; ++i)
 ;;     for (int j = 2; j < m; ++j)
@@ -20,6 +11,22 @@
 ;; }
 
 define void @t1(i32 signext %n, i32 signext %m, ptr %a) {
+; CHECK-LABEL: 't1'
+; CHECK-NEXT:  Src: %6 = load i32, ptr %arrayidx7, align 4 --> Dst: %6 = load i32, ptr %arrayidx7, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: %6 = load i32, ptr %arrayidx7, align 4 --> Dst: store i32 %6, ptr %arrayidx11, align 4
+; CHECK-NEXT:    da analyze - consistent anti [1 -2]!
+; CHECK-NEXT:  Src: store i32 %6, ptr %arrayidx11, align 4 --> Dst: store i32 %6, ptr %arrayidx11, align 4
+; CHECK-NEXT:    da analyze - none!
+;
+; LIN-LABEL: 't1'
+; LIN-NEXT:  Src: %6 = load i32, ptr %arrayidx7, align 4 --> Dst: %6 = load i32, ptr %arrayidx7, align 4
+; LIN-NEXT:    da analyze - input [* *]!
+; LIN-NEXT:  Src: %6 = load i32, ptr %arrayidx7, align 4 --> Dst: store i32 %6, ptr %arrayidx11, align 4
+; LIN-NEXT:    da analyze - anti [* *|<]!
+; LIN-NEXT:  Src: store i32 %6, ptr %arrayidx11, align 4 --> Dst: store i32 %6, ptr %arrayidx11, align 4
+; LIN-NEXT:    da analyze - output [* *]!
+;
 entry:
   %0 = zext i32 %m to i64
   %1 = sext i32 %m to i64
@@ -59,16 +66,6 @@ for.end14:                                        ; preds = %entry, %for.inc12
   ret void
 }
 
-; CHECK-LABEL: t2
-; CHECK: da analyze - none!
-; CHECK: da analyze - consistent anti [1 -2 0 -3 2]!
-; CHECK: da analyze - none!
-
-; LIN-LABEL: t2
-; LIN: da analyze - input [* * * * *]!
-; LIN: da analyze - anti [* * * * *|<]!
-; LIN: da analyze - output [* * * * *]!
-
 ;; void t2(int n, int m, int a[][n][n][n][m]) {
 ;;   for (int i1 = 0; i1 < n-1; ++i1)
 ;;     for (int i2 = 2; i2 < n; ++i2)
@@ -79,6 +76,22 @@ for.end14:                                        ; preds = %entry, %for.inc12
 ;; }
 
 define void @t2(i32 signext %n, i32 signext %m, ptr %a) {
+; CHECK-LABEL: 't2'
+; CHECK-NEXT:  Src: %21 = load i32, ptr %arrayidx28, align 4 --> Dst: %21 = load i32, ptr %arrayidx28, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: %21 = load i32, ptr %arrayidx28, align 4 --> Dst: store i32 %21, ptr %arrayidx38, align 4
+; CHECK-NEXT:    da analyze - consistent anti [1 -2 0 -3 2]!
+; CHECK-NEXT:  Src: store i32 %21, ptr %arrayidx38, align 4 --> Dst: store i32 %21, ptr %arrayidx38, align 4
+; CHECK-NEXT:    da analyze - none!
+;
+; LIN-LABEL: 't2'
+; LIN-NEXT:  Src: %21 = load i32, ptr %arrayidx28, align 4 --> Dst: %21 = load i32, ptr %arrayidx28, align 4
+; LIN-NEXT:    da analyze - input [* * * * *]!
+; LIN-NEXT:  Src: %21 = load i32, ptr %arrayidx28, align 4 --> Dst: store i32 %21, ptr %arrayidx38, align 4
+; LIN-NEXT:    da analyze - anti [* * * * *|<]!
+; LIN-NEXT:  Src: store i32 %21, ptr %arrayidx38, align 4 --> Dst: store i32 %21, ptr %arrayidx38, align 4
+; LIN-NEXT:    da analyze - output [* * * * *]!
+;
 entry:
   %0 = zext i32 %n to i64
   %1 = zext i32 %n to i64
@@ -185,16 +198,6 @@ for.end50:                                        ; preds = %entry, %for.inc48
 }
 
 
-; CHECK-LABEL: t3
-; CHECK: da analyze - none!
-; CHECK: da analyze - consistent anti [1 -2]!
-; CHECK: da analyze - none!
-
-; LIN-LABEL: t3
-; LIN: da analyze - input [* *]!
-; LIN: da analyze - anti [* *|<]!
-; LIN: da analyze - output [* *]!
-
 ;; // No sign or zero extension, but with compile-time unknown loop lower bound.
 ;; void t3(unsigned long long n, unsigned long long m, unsigned long long lb, float a[][m]) {
 ;;   for (unsigned long long i = 0; i < n-1; ++i)
@@ -203,6 +206,22 @@ for.end50:                                        ; preds = %entry, %for.inc48
 ;; }
 
 define void @t3(i64 %n, i64 %m, i64 %lb, ptr %a) {
+; CHECK-LABEL: 't3'
+; CHECK-NEXT:  Src: %2 = load i32, ptr %arrayidx6, align 4 --> Dst: %2 = load i32, ptr %arrayidx6, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: %2 = load i32, ptr %arrayidx6, align 4 --> Dst: store i32 %2, ptr %arrayidx8, align 4
+; CHECK-NEXT:    da analyze - consistent anti [1 -2]!
+; CHECK-NEXT:  Src: store i32 %2, ptr %arrayidx8, align 4 --> Dst: store i32 %2, ptr %arrayidx8, align 4
+; CHECK-NEXT:    da analyze - none!
+;
+; LIN-LABEL: 't3'
+; LIN-NEXT:  Src: %2 = load i32, ptr %arrayidx6, align 4 --> Dst: %2 = load i32, ptr %arrayidx6, align 4
+; LIN-NEXT:    da analyze - input [* *]!
+; LIN-NEXT:  Src: %2 = load i32, ptr %arrayidx6, align 4 --> Dst: store i32 %2, ptr %arrayidx8, align 4
+; LIN-NEXT:    da analyze - anti [* *|<]!
+; LIN-NEXT:  Src: store i32 %2, ptr %arrayidx8, align 4 --> Dst: store i32 %2, ptr %arrayidx8, align 4
+; LIN-NEXT:    da analyze - output [* *]!
+;
 entry:
   %0 = add i64 %n, -1
   %exitcond3 = icmp ne i64 0, %0
diff --git a/llvm/test/Analysis/DependenceAnalysis/SimpleSIVNoValidityCheckFixedSize.ll b/llvm/test/Analysis/DependenceAnalysis/SimpleSIVNoValidityCheckFixedSize.ll
index b8ed6ab916c45..c4948ebf9854b 100644
--- a/llvm/test/Analysis/DependenceAnalysis/SimpleSIVNoValidityCheckFixedSize.ll
+++ b/llvm/test/Analysis/DependenceAnalysis/SimpleSIVNoValidityCheckFixedSize.ll
@@ -1,13 +1,9 @@
+; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --version 5
 ; RUN: opt < %s -disable-output -passes="print<da>" 2>&1 | FileCheck %s
 
 ; Note: exact results can be achived even if
 ; "-da-disable-delinearization-checks" is not used
 
-; CHECK-LABEL: t1
-; CHECK: da analyze - none!
-; CHECK: da analyze - consistent anti [1 -2]!
-; CHECK: da analyze - none!
-
 ;; #define N 1024
 ;; #define M 2048
 ;; void t1(int a[N][M]) {
@@ -18,6 +14,14 @@
 ;;
 ;; Note that there is a getelementptr with index 0, make sure we can analyze this case.
 define void @t1(ptr %a) {
+; CHECK-LABEL: 't1'
+; CHECK-NEXT:  Src: %2 = load i32, ptr %arrayidx6, align 4 --> Dst: %2 = load i32, ptr %arrayidx6, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: %2 = load i32, ptr %arrayidx6, align 4 --> Dst: store i32 %2, ptr %arrayidx10, align 4
+; CHECK-NEXT:    da analyze - consistent anti [1 -2]!
+; CHECK-NEXT:  Src: store i32 %2, ptr %arrayidx10, align 4 --> Dst: store i32 %2, ptr %arrayidx10, align 4
+; CHECK-NEXT:    da analyze - none!
+;
 entry:
   br label %for.body
 
@@ -46,13 +50,24 @@ for.end13:                                        ; preds = %for.inc11
   ret void
 }
 
-; CHECK-LABEL: t2
-; CHECK: da analyze - consistent anti [1 -2]!
-
 ;; Similar to @t1 but includes a call with a "returned" arg, make sure we can analyze
 ;; this case.
 
 define void @t2(ptr %a) {
+; CHECK-LABEL: 't2'
+; CHECK-NEXT:  Src: %2 = load i32, ptr %arrayidx6, align 4 --> Dst: %2 = load i32, ptr %arrayidx6, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: %2 = load i32, ptr %arrayidx6, align 4 --> Dst: %call = call ptr @func_with_returned_arg(ptr returned %a)
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: %2 = load i32, ptr %arrayidx6, align 4 --> Dst: store i32 %2, ptr %arrayidx10, align 4
+; CHECK-NEXT:    da analyze - consistent anti [1 -2]!
+; CHECK-NEXT:  Src: %call = call ptr @func_with_returned_arg(ptr returned %a) --> Dst: %call = call ptr @func_with_returned_arg(ptr returned %a)
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: %call = call ptr @func_with_returned_arg(ptr returned %a) --> Dst: store i32 %2, ptr %arrayidx10, align 4
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: store i32 %2, ptr %arrayidx10, align 4 --> Dst: store i32 %2, ptr %arrayidx10, align 4
+; CHECK-NEXT:    da analyze - none!
+;
 entry:
   br label %for.body
 
@@ -84,11 +99,6 @@ for.end13:                                        ; preds = %for.inc11
 
 declare ptr @func_with_returned_arg(ptr returned %arg)
 
-; CHECK-LABEL: t3
-; CHECK: da analyze - none!
-; CHECK: da analyze - consistent anti [1 -2 0 -3 2]!
-; CHECK: da analyze - none!
-
 ;; #define N 1024
 ;; #define M 2048
 ;; void t2(int a[][N][N][N][M]) {
@@ -101,6 +111,14 @@ declare ptr @func_with_returned_arg(ptr returned %arg)
 ;; }
 
 define void @t3(ptr %a) {
+; CHECK-LABEL: 't3'
+; CHECK-NEXT:  Src: %4 = load i32, ptr %arrayidx26, align 4 --> Dst: %4 = load i32, ptr %arrayidx26, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: %4 = load i32, ptr %arrayidx26, align 4 --> Dst: store i32 %4, ptr %arrayidx36, align 4
+; CHECK-NEXT:    da analyze - consistent anti [1 -2 0 -3 2]!
+; CHECK-NEXT:  Src: store i32 %4, ptr %arrayidx36, align 4 --> Dst: store i32 %4, ptr %arrayidx36, align 4
+; CHECK-NEXT:    da analyze - none!
+;
 entry:
   br label %for.body
 
diff --git a/llvm/test/Analysis/DependenceAnalysis/StrongSIV.ll b/llvm/test/Analysis/DependenceAnalysis/StrongSIV.ll
index 59bc406fd4bd5..44bd9b7727910 100644
--- a/llvm/test/Analysis/DependenceAnalysis/StrongSIV.ll
+++ b/llvm/test/Analysis/DependenceAnalysis/StrongSIV.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --version 5
 ; RUN: opt < %s -disable-output "-passes=print<da>" -aa-pipeline=basic-aa 2>&1 \
 ; RUN: | FileCheck %s
 
@@ -10,17 +11,24 @@ target triple = "x86_64-apple-macosx10.6.0"
 ;;    *B++ = A[i];
 
 define void @strong0(ptr %A, ptr %B, i64 %n) nounwind uwtable ssp {
+; CHECK-LABEL: 'strong0'
+; CHECK-NEXT:  Src: store i32 %1, ptr %arrayidx, align 4 --> Dst: store i32 %1, ptr %arrayidx, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: store i32 %1, ptr %arrayidx, align 4 --> Dst: %2 = load i32, ptr %arrayidx3, align 4
+; CHECK-NEXT:    da analyze - consistent flow [2]!
+; CHECK-NEXT:  Src: store i32 %1, ptr %arrayidx, align 4 --> Dst: store i32 %2, ptr %B.addr.02, align 4
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: %2 = load i32, ptr %arrayidx3, align 4 --> Dst: %2 = load i32, ptr %arrayidx3, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: %2 = load i32, ptr %arrayidx3, align 4 --> Dst: store i32 %2, ptr %B.addr.02, align 4
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: store i32 %2, ptr %B.addr.02, align 4 --> Dst: store i32 %2, ptr %B.addr.02, align 4
+; CHECK-NEXT:    da analyze - none!
+;
 entry:
   %cmp1 = icmp sgt i64 %n, 0
   br i1 %cmp1, label %for.body.preheader, label %for.end
 
-; CHECK: da analyze - none!
-; CHECK: da analyze - consistent flow [2]!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - none!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - none!
-
 for.body.preheader:                               ; preds = %entry
   br label %for.body
 
@@ -52,17 +60,24 @@ for.end:                                          ; preds = %for.end.loopexit, %
 ;;    *B++ = A[i];
 
 define void @strong1(ptr %A, ptr %B, i32 %n) nounwind uwtable ssp {
+; CHECK-LABEL: 'strong1'
+; CHECK-NEXT:  Src: store i32 %conv2, ptr %arrayidx, align 4 --> Dst: store i32 %conv2, ptr %arrayidx, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: store i32 %conv2, ptr %arrayidx, align 4 --> Dst: %1 = load i32, ptr %arrayidx3, align 4
+; CHECK-NEXT:    da analyze - consistent flow [2]!
+; CHECK-NEXT:  Src: store i32 %conv2, ptr %arrayidx, align 4 --> Dst: store i32 %1, ptr %B.addr.02, align 4
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: %1 = load i32, ptr %arrayidx3, align 4 --> Dst: %1 = load i32, ptr %arrayidx3, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: %1 = load i32, ptr %arrayidx3, align 4 --> Dst: store i32 %1, ptr %B.addr.02, align 4
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: store i32 %1, ptr %B.addr.02, align 4 --> Dst: store i32 %1, ptr %B.addr.02, align 4
+; CHECK-NEXT:    da analyze - none!
+;
 entry:
   %cmp1 = icmp sgt i32 %n, 0
   br i1 %cmp1, label %for.body.preheader, label %for.end
 
-; CHECK: da analyze - none!
-; CHECK: da analyze - consistent flow [2]!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - none!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - none!
-
 for.body.preheader:                               ; preds = %entry
   %0 = sext i32 %n to i64
   br label %for.body
@@ -95,17 +110,24 @@ for.end:                                          ; preds = %for.end.loopexit, %
 ;;    *B++ = A[i];
 
 define void @strong2(ptr %A, ptr %B, i64 %n) nounwind uwtable ssp {
+; CHECK-LABEL: 'strong2'
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx, align 4 --> Dst: store i32 %conv, ptr %arrayidx, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx, align 4 --> Dst: %0 = load i32, ptr %arrayidx1, align 4
+; CHECK-NEXT:    da analyze - consistent flow [2]!
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx, align 4 --> Dst: store i32 %0, ptr %B.addr.02, align 4
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: %0 = load i32, ptr %arrayidx1, align 4 --> Dst: %0 = load i32, ptr %arrayidx1, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: %0 = load i32, ptr %arrayidx1, align 4 --> Dst: store i32 %0, ptr %B.addr.02, align 4
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: store i32 %0, ptr %B.addr.02, align 4 --> Dst: store i32 %0, ptr %B.addr.02, align 4
+; CHECK-NEXT:    da analyze - none!
+;
 entry:
   %cmp1 = icmp eq i64 %n, 0
   br i1 %cmp1, label %for.end, label %for.body.preheader
 
-; CHECK: da analyze - none!
-; CHECK: da analyze - consistent flow [2]!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - none!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - none!
-
 for.body.preheader:                               ; preds = %entry
   br label %for.body
 
@@ -137,17 +159,24 @@ for.end:                                          ; preds = %for.end.loopexit, %
 ;;    *B++ = A[i];
 
 define void @strong3(ptr %A, ptr %B, i32 %n) nounwind uwtable ssp {
+; CHECK-LABEL: 'strong3'
+; CHECK-NEXT:  Src: store i32 %1, ptr %arrayidx, align 4 --> Dst: store i32 %1, ptr %arrayidx, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: store i32 %1, ptr %arrayidx, align 4 --> Dst: %2 = load i32, ptr %arrayidx2, align 4
+; CHECK-NEXT:    da analyze - consistent flow [2]!
+; CHECK-NEXT:  Src: store i32 %1, ptr %arrayidx, align 4 --> Dst: store i32 %2, ptr %B.addr.02, align 4
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: %2 = load i32, ptr %arrayidx2, align 4 --> Dst: %2 = load i32, ptr %arrayidx2, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: %2 = load i32, ptr %arrayidx2, align 4 --> Dst: store i32 %2, ptr %B.addr.02, align 4
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: store i32 %2, ptr %B.addr.02, align 4 --> Dst: store i32 %2, ptr %B.addr.02, align 4
+; CHECK-NEXT:    da analyze - none!
+;
 entry:
   %cmp1 = icmp sgt i32 %n, 0
   br i1 %cmp1, label %for.body.preheader, label %for.end
 
-; CHECK: da analyze - none!
-; CHECK: da analyze - consistent flow [2]!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - none!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - none!
-
 for.body.preheader:                               ; preds = %entry
   br label %for.body
 
@@ -180,16 +209,23 @@ for.end:                                          ; preds = %for.end.loopexit, %
 ;;    *B++ = A[i];
 
 define void @strong4(ptr %A, ptr %B, i64 %n) nounwind uwtable ssp {
+; CHECK-LABEL: 'strong4'
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx, align 4 --> Dst: store i32 %conv, ptr %arrayidx, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx, align 4 --> Dst: %0 = load i32, ptr %arrayidx1, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx, align 4 --> Dst: store i32 %0, ptr %B.addr.01, align 4
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: %0 = load i32, ptr %arrayidx1, align 4 --> Dst: %0 = load i32, ptr %arrayidx1, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: %0 = load i32, ptr %arrayidx1, align 4 --> Dst: store i32 %0, ptr %B.addr.01, align 4
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: store i32 %0, ptr %B.addr.01, align 4 --> Dst: store i32 %0, ptr %B.addr.01, align 4
+; CHECK-NEXT:    da analyze - none!
+;
 entry:
   br label %for.body
 
-; CHECK: da analyze - none!
-; CHECK: da analyze - none!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - none!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - none!
-
 for.body:                                         ; preds = %entry, %for.body
   %i.02 = phi i64 [ 0, %entry ], [ %inc, %for.body ]
   %B.addr.01 = phi ptr [ %B, %entry ], [ %incdec.ptr, %for.body ]
@@ -215,16 +251,23 @@ for.end:                                          ; preds = %for.body
 ;;    *B++ = A[i];
 
 define void @strong5(ptr %A, ptr %B, i64 %n) nounwind uwtable ssp {
+; CHECK-LABEL: 'strong5'
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx, align 4 --> Dst: store i32 %conv, ptr %arrayidx, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx, align 4 --> Dst: %0 = load i32, ptr %arrayidx1, align 4
+; CHECK-NEXT:    da analyze - consistent flow [19]!
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx, align 4 --> Dst: store i32 %0, ptr %B.addr.01, align 4
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: %0 = load i32, ptr %arrayidx1, align 4 --> Dst: %0 = load i32, ptr %arrayidx1, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: %0 = load i32, ptr %arrayidx1, align 4 --> Dst: store i32 %0, ptr %B.addr.01, align 4
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: store i32 %0, ptr %B.addr.01, align 4 --> Dst: store i32 %0, ptr %B.addr.01, align 4
+; CHECK-NEXT:    da analyze - none!
+;
 entry:
   br label %for.body
 
-; CHECK: da analyze - none!
-; CHECK: da analyze - consistent flow [19]!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - none!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - none!
-
 for.body:                                         ; preds = %entry, %for.body
   %i.02 = phi i64 [ 0, %entry ], [ %inc, %for.body ]
   %B.addr.01 = phi ptr [ %B, %entry ], [ %incdec.ptr, %for.body ]
@@ -250,16 +293,23 @@ for.end:                                          ; preds = %for.body
 ;;    *B++ = A[2*i];
 
 define void @strong6(ptr %A, ptr %B, i64 %n) nounwind uwtable ssp {
+; CHECK-LABEL: 'strong6'
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx, align 4 --> Dst: store i32 %conv, ptr %arrayidx, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx, align 4 --> Dst: %0 = load i32, ptr %arrayidx2, align 4
+; CHECK-NEXT:    da analyze - consistent flow [3]!
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx, align 4 --> Dst: store i32 %0, ptr %B.addr.01, align 4
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: %0 = load i32, ptr %arrayidx2, align 4 --> Dst: %0 = load i32, ptr %arrayidx2, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: %0 = load i32, ptr %arrayidx2, align 4 --> Dst: store i32 %0, ptr %B.addr.01, align 4
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: store i32 %0, ptr %B.addr.01, align 4 --> Dst: store i32 %0, ptr %B.addr.01, align 4
+; CHECK-NEXT:    da analyze - none!
+;
 entry:
   br label %for.body
 
-; CHECK: da analyze - none!
-; CHECK: da analyze - consistent flow [3]!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - none!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - none!
-
 for.body:                                         ; preds = %entry, %for.body
   %i.02 = phi i64 [ 0, %entry ], [ %inc, %for.body ]
   %B.addr.01 = phi ptr [ %B, %entry ], [ %incdec.ptr, %for.body ]
@@ -287,16 +337,23 @@ for.end:                                          ; preds = %for.body
 ;;    *B++ = A[2*i];
 
 define void @strong7(ptr %A, ptr %B, i64 %n) nounwind uwtable ssp {
+; CHECK-LABEL: 'strong7'
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx, align 4 --> Dst: store i32 %conv, ptr %arrayidx, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx, align 4 --> Dst: %0 = load i32, ptr %arrayidx2, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx, align 4 --> Dst: store i32 %0, ptr %B.addr.01, align 4
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: %0 = load i32, ptr %arrayidx2, align 4 --> Dst: %0 = load i32, ptr %arrayidx2, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: %0 = load i32, ptr %arrayidx2, align 4 --> Dst: store i32 %0, ptr %B.addr.01, align 4
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: store i32 %0, ptr %B.addr.01, align 4 --> Dst: store i32 %0, ptr %B.addr.01, align 4
+; CHECK-NEXT:    da analyze - none!
+;
 entry:
   br label %for.body
 
-; CHECK: da analyze - none!
-; CHECK: da analyze - none!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - none!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - none!
-
 for.body:                                         ; preds = %entry, %for.body
   %i.02 = phi i64 [ 0, %entry ], [ %inc, %for.body ]
   %B.addr.01 = phi ptr [ %B, %entry ], [ %incdec.ptr, %for.body ]
@@ -324,16 +381,23 @@ for.end:                                          ; preds = %for.body
 ;;    *B++ = A[i];
 
 define void @strong8(ptr %A, ptr %B, i64 %n) nounwind uwtable ssp {
+; CHECK-LABEL: 'strong8'
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx, align 4 --> Dst: store i32 %conv, ptr %arrayidx, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx, align 4 --> Dst: %0 = load i32, ptr %arrayidx1, align 4
+; CHECK-NEXT:    da analyze - flow [*|<]!
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx, align 4 --> Dst: store i32 %0, ptr %B.addr.01, align 4
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: %0 = load i32, ptr %arrayidx1, align 4 --> Dst: %0 = load i32, ptr %arrayidx1, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: %0 = load i32, ptr %arrayidx1, align 4 --> Dst: store i32 %0, ptr %B.addr.01, align 4
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: store i32 %0, ptr %B.addr.01, align 4 --> Dst: store i32 %0, ptr %B.addr.01, align 4
+; CHECK-NEXT:    da analyze - none!
+;
 entry:
   br label %for.body
 
-; CHECK: da analyze - none!
-; CHECK: da analyze - flow [*|<]!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - none!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - none!
-
 for.body:                                         ; preds = %entry, %for.body
   %i.02 = phi i64 [ 0, %entry ], [ %inc, %for.body ]
   %B.addr.01 = phi ptr [ %B, %entry ], [ %incdec.ptr, %for.body ]
@@ -359,17 +423,24 @@ for.end:                                          ; preds = %for.body
 ;;    *B++ = A[i + 2*n];
 
 define void @strong9(ptr %A, ptr %B, i64 %n) nounwind uwtable ssp {
+; CHECK-LABEL: 'strong9'
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx, align 4 --> Dst: store i32 %conv, ptr %arrayidx, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx, align 4 --> Dst: %0 = load i32, ptr %arrayidx2, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx, align 4 --> Dst: store i32 %0, ptr %B.addr.02, align 4
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: %0 = load i32, ptr %arrayidx2, align 4 --> Dst: %0 = load i32, ptr %arrayidx2, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: %0 = load i32, ptr %arrayidx2, align 4 --> Dst: store i32 %0, ptr %B.addr.02, align 4
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: store i32 %0, ptr %B.addr.02, align 4 --> Dst: store i32 %0, ptr %B.addr.02, align 4
+; CHECK-NEXT:    da analyze - none!
+;
 entry:
   %cmp1 = icmp eq i64 %n, 0
   br i1 %cmp1, label %for.end, label %for.body.preheader
 
-; CHECK: da analyze - none!
-; CHECK: da analyze - none!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - none!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - none!
-
 for.body.preheader:                               ; preds = %entry
   br label %for.body
 
@@ -403,16 +474,23 @@ for.end:                                          ; preds = %for.end.loopexit, %
 ;;    *B++ = A[n*i + 5];
 
 define void @strong10(ptr %A, ptr %B, i64 %n) nounwind uwtable ssp {
+; CHECK-LABEL: 'strong10'
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx, align 4 --> Dst: store i32 %conv, ptr %arrayidx, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx, align 4 --> Dst: %0 = load i32, ptr %arrayidx3, align 4
+; CHECK-NEXT:    da analyze - consistent flow [0|<]!
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx, align 4 --> Dst: store i32 %0, ptr %B.addr.01, align 4
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: %0 = load i32, ptr %arrayidx3, align 4 --> Dst: %0 = load i32, ptr %arrayidx3, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: %0 = load i32, ptr %arrayidx3, align 4 --> Dst: store i32 %0, ptr %B.addr.01, align 4
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: store i32 %0, ptr %B.addr.01, align 4 --> Dst: store i32 %0, ptr %B.addr.01, align 4
+; CHECK-NEXT:    da analyze - none!
+;
 entry:
   br label %for.body
 
-; CHECK: da analyze - none!
-; CHECK: da analyze - consistent flow [0|<]!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - none!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - none!
-
 for.body:                                         ; preds = %entry, %for.body
   %i.02 = phi i64 [ 0, %entry ], [ %inc, %for.body ]
   %B.addr.01 = phi ptr [ %B, %entry ], [ %incdec.ptr, %for.body ]
diff --git a/llvm/test/Analysis/DependenceAnalysis/SymbolicRDIV.ll b/llvm/test/Analysis/DependenceAnalysis/SymbolicRDIV.ll
index 238ab628e369e..f64a7483d7b66 100644
--- a/llvm/test/Analysis/DependenceAnalysis/SymbolicRDIV.ll
+++ b/llvm/test/Analysis/DependenceAnalysis/SymbolicRDIV.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --version 5
 ; RUN: opt < %s -disable-output "-passes=print<da>" -aa-pipeline=basic-aa 2>&1 \
 ; RUN: | FileCheck %s
 
@@ -12,18 +13,24 @@ target triple = "x86_64-apple-macosx10.6.0"
 ;;    *B++ = A[3*j + 3*n1];
 
 define void @symbolicrdiv0(ptr %A, ptr %B, i64 %n1, i64 %n2) nounwind uwtable ssp {
+; CHECK-LABEL: 'symbolicrdiv0'
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx, align 4 --> Dst: store i32 %conv, ptr %arrayidx, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx, align 4 --> Dst: %0 = load i32, ptr %arrayidx8, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx, align 4 --> Dst: store i32 %0, ptr %B.addr.02, align 4
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: %0 = load i32, ptr %arrayidx8, align 4 --> Dst: %0 = load i32, ptr %arrayidx8, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: %0 = load i32, ptr %arrayidx8, align 4 --> Dst: store i32 %0, ptr %B.addr.02, align 4
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: store i32 %0, ptr %B.addr.02, align 4 --> Dst: store i32 %0, ptr %B.addr.02, align 4
+; CHECK-NEXT:    da analyze - none!
+;
 entry:
   %cmp4 = icmp eq i64 %n1, 0
   br i1 %cmp4, label %for.cond1.preheader, label %for.body.preheader
 
-; CHECK: 'Dependence Analysis' for function 'symbolicrdiv0'
-; CHECK: da analyze - none!
-; CHECK: da analyze - none!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - none!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - none!
-
 for.body.preheader:                               ; preds = %entry
   br label %for.body
 
@@ -75,18 +82,24 @@ for.end11:                                        ; preds = %for.end11.loopexit,
 ;;    *B++ = A[3*j + 2*n2];
 
 define void @symbolicrdiv1(ptr %A, ptr %B, i64 %n1, i64 %n2) nounwind uwtable ssp {
+; CHECK-LABEL: 'symbolicrdiv1'
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx, align 4 --> Dst: store i32 %conv, ptr %arrayidx, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx, align 4 --> Dst: %0 = load i32, ptr %arrayidx9, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx, align 4 --> Dst: store i32 %0, ptr %B.addr.02, align 4
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: %0 = load i32, ptr %arrayidx9, align 4 --> Dst: %0 = load i32, ptr %arrayidx9, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: %0 = load i32, ptr %arrayidx9, align 4 --> Dst: store i32 %0, ptr %B.addr.02, align 4
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: store i32 %0, ptr %B.addr.02, align 4 --> Dst: store i32 %0, ptr %B.addr.02, align 4
+; CHECK-NEXT:    da analyze - none!
+;
 entry:
   %cmp4 = icmp eq i64 %n1, 0
   br i1 %cmp4, label %for.cond2.preheader, label %for.body.preheader
 
-; CHECK: 'Dependence Analysis' for function 'symbolicrdiv1'
-; CHECK: da analyze - none!
-; CHECK: da analyze - none!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - none!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - none!
-
 for.body.preheader:                               ; preds = %entry
   br label %for.body
 
@@ -140,18 +153,24 @@ for.end12:                                        ; preds = %for.end12.loopexit,
 ;;    *B++ = A[-j + 2*n1];
 
 define void @symbolicrdiv2(ptr %A, ptr %B, i64 %n1, i64 %n2) nounwind uwtable ssp {
+; CHECK-LABEL: 'symbolicrdiv2'
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx, align 4 --> Dst: store i32 %conv, ptr %arrayidx, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx, align 4 --> Dst: %0 = load i32, ptr %arrayidx7, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx, align 4 --> Dst: store i32 %0, ptr %B.addr.02, align 4
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: %0 = load i32, ptr %arrayidx7, align 4 --> Dst: %0 = load i32, ptr %arrayidx7, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: %0 = load i32, ptr %arrayidx7, align 4 --> Dst: store i32 %0, ptr %B.addr.02, align 4
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: store i32 %0, ptr %B.addr.02, align 4 --> Dst: store i32 %0, ptr %B.addr.02, align 4
+; CHECK-NEXT:    da analyze - none!
+;
 entry:
   %cmp4 = icmp eq i64 %n1, 0
   br i1 %cmp4, label %for.cond1.preheader, label %for.body.preheader
 
-; CHECK: 'Dependence Analysis' for function 'symbolicrdiv2'
-; CHECK: da analyze - none!
-; CHECK: da analyze - none!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - none!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - none!
-
 for.body.preheader:                               ; preds = %entry
   br label %for.body
 
@@ -203,18 +222,24 @@ for.end10:                                        ; preds = %for.end10.loopexit,
 ;;    *B++ = A[j - n1];
 
 define void @symbolicrdiv3(ptr %A, ptr %B, i64 %n1, i64 %n2) nounwind uwtable ssp {
+; CHECK-LABEL: 'symbolicrdiv3'
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx, align 4 --> Dst: store i32 %conv, ptr %arrayidx, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx, align 4 --> Dst: %0 = load i32, ptr %arrayidx6, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx, align 4 --> Dst: store i32 %0, ptr %B.addr.02, align 4
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: %0 = load i32, ptr %arrayidx6, align 4 --> Dst: %0 = load i32, ptr %arrayidx6, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: %0 = load i32, ptr %arrayidx6, align 4 --> Dst: store i32 %0, ptr %B.addr.02, align 4
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: store i32 %0, ptr %B.addr.02, align 4 --> Dst: store i32 %0, ptr %B.addr.02, align 4
+; CHECK-NEXT:    da analyze - none!
+;
 entry:
   %cmp4 = icmp eq i64 %n1, 0
   br i1 %cmp4, label %for.cond1.preheader, label %for.body.preheader
 
-; CHECK: 'Dependence Analysis' for function 'symbolicrdiv3'
-; CHECK: da analyze - none!
-; CHECK: da analyze - none!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - none!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - none!
-
 for.body.preheader:                               ; preds = %entry
   br label %for.body
 
@@ -264,18 +289,24 @@ for.end9:                                         ; preds = %for.end9.loopexit,
 ;;    *B++ = A[-j + n1];
 
 define void @symbolicrdiv4(ptr %A, ptr %B, i64 %n1, i64 %n2) nounwind uwtable ssp {
+; CHECK-LABEL: 'symbolicrdiv4'
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx, align 4 --> Dst: store i32 %conv, ptr %arrayidx, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx, align 4 --> Dst: %0 = load i32, ptr %arrayidx7, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx, align 4 --> Dst: store i32 %0, ptr %B.addr.02, align 4
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: %0 = load i32, ptr %arrayidx7, align 4 --> Dst: %0 = load i32, ptr %arrayidx7, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: %0 = load i32, ptr %arrayidx7, align 4 --> Dst: store i32 %0, ptr %B.addr.02, align 4
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: store i32 %0, ptr %B.addr.02, align 4 --> Dst: store i32 %0, ptr %B.addr.02, align 4
+; CHECK-NEXT:    da analyze - none!
+;
 entry:
   %cmp4 = icmp eq i64 %n1, 0
   br i1 %cmp4, label %for.cond1.preheader, label %for.body.preheader
 
-; CHECK: 'Dependence Analysis' for function 'symbolicrdiv4'
-; CHECK: da analyze - none!
-; CHECK: da analyze - none!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - none!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - none!
-
 for.body.preheader:                               ; preds = %entry
   br label %for.body
 
@@ -326,18 +357,24 @@ for.end10:                                        ; preds = %for.end10.loopexit,
 ;;    *B++ = A[-j + 2*n2];
 
 define void @symbolicrdiv5(ptr %A, ptr %B, i64 %n1, i64 %n2) nounwind uwtable ssp {
+; CHECK-LABEL: 'symbolicrdiv5'
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx, align 4 --> Dst: store i32 %conv, ptr %arrayidx, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx, align 4 --> Dst: %0 = load i32, ptr %arrayidx7, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx, align 4 --> Dst: store i32 %0, ptr %B.addr.02, align 4
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: %0 = load i32, ptr %arrayidx7, align 4 --> Dst: %0 = load i32, ptr %arrayidx7, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: %0 = load i32, ptr %arrayidx7, align 4 --> Dst: store i32 %0, ptr %B.addr.02, align 4
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: store i32 %0, ptr %B.addr.02, align 4 --> Dst: store i32 %0, ptr %B.addr.02, align 4
+; CHECK-NEXT:    da analyze - none!
+;
 entry:
   %cmp4 = icmp eq i64 %n1, 0
   br i1 %cmp4, label %for.cond1.preheader, label %for.body.preheader
 
-; CHECK: 'Dependence Analysis' for function 'symbolicrdiv5'
-; CHECK: da analyze - none!
-; CHECK: da analyze - none!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - none!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - none!
-
 for.body.preheader:                               ; preds = %entry
   br label %for.body
 
@@ -388,18 +425,24 @@ for.end10:                                        ; preds = %for.end10.loopexit,
 ;;      *B++ = A[2*n2];
 
 define void @symbolicrdiv6(ptr %A, ptr %B, i64 %n1, i64 %n2) nounwind uwtable ssp {
+; CHECK-LABEL: 'symbolicrdiv6'
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx, align 4 --> Dst: store i32 %conv, ptr %arrayidx, align 4
+; CHECK-NEXT:    da analyze - output [* *]!
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx, align 4 --> Dst: %0 = load i32, ptr %arrayidx4, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx, align 4 --> Dst: store i32 %0, ptr %B.addr.12, align 4
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: %0 = load i32, ptr %arrayidx4, align 4 --> Dst: %0 = load i32, ptr %arrayidx4, align 4
+; CHECK-NEXT:    da analyze - consistent input [S S]!
+; CHECK-NEXT:  Src: %0 = load i32, ptr %arrayidx4, align 4 --> Dst: store i32 %0, ptr %B.addr.12, align 4
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: store i32 %0, ptr %B.addr.12, align 4 --> Dst: store i32 %0, ptr %B.addr.12, align 4
+; CHECK-NEXT:    da analyze - output [* *]!
+;
 entry:
   %cmp4 = icmp eq i64 %n1, 0
   br i1 %cmp4, label %for.end7, label %for.cond1.preheader.preheader
 
-; CHECK: 'Dependence Analysis' for function 'symbolicrdiv6'
-; CHECK: da analyze - output [* *]!
-; CHECK: da analyze - none!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - consistent input [S S]!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - output [* *]!
-
 for.cond1.preheader.preheader:                    ; preds = %entry
   br label %for.cond1.preheader
 
diff --git a/llvm/test/Analysis/DependenceAnalysis/SymbolicSIV.ll b/llvm/test/Analysis/DependenceAnalysis/SymbolicSIV.ll
index a995f7a3ad22b..cdfaec76fa892 100644
--- a/llvm/test/Analysis/DependenceAnalysis/SymbolicSIV.ll
+++ b/llvm/test/Analysis/DependenceAnalysis/SymbolicSIV.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --version 5
 ; RUN: opt < %s -disable-output "-passes=print<da>" -aa-pipeline=basic-aa 2>&1 \
 ; RUN: | FileCheck %s
 
@@ -10,17 +11,24 @@ target triple = "x86_64-apple-macosx10.6.0"
 ;;    *B++ = A[3*i + 3*n];
 
 define void @symbolicsiv0(ptr %A, ptr %B, i64 %n) nounwind uwtable ssp {
+; CHECK-LABEL: 'symbolicsiv0'
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx, align 4 --> Dst: store i32 %conv, ptr %arrayidx, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx, align 4 --> Dst: %0 = load i32, ptr %arrayidx4, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx, align 4 --> Dst: store i32 %0, ptr %B.addr.02, align 4
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: %0 = load i32, ptr %arrayidx4, align 4 --> Dst: %0 = load i32, ptr %arrayidx4, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: %0 = load i32, ptr %arrayidx4, align 4 --> Dst: store i32 %0, ptr %B.addr.02, align 4
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: store i32 %0, ptr %B.addr.02, align 4 --> Dst: store i32 %0, ptr %B.addr.02, align 4
+; CHECK-NEXT:    da analyze - none!
+;
 entry:
   %cmp1 = icmp eq i64 %n, 0
   br i1 %cmp1, label %for.end, label %for.body.preheader
 
-; CHECK: da analyze - none!
-; CHECK: da analyze - none!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - none!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - none!
-
 for.body.preheader:                               ; preds = %entry
   br label %for.body
 
@@ -55,17 +63,24 @@ for.end:                                          ; preds = %for.end.loopexit, %
 ;;    *B++ = A[3*i + 2*n];
 
 define void @symbolicsiv1(ptr %A, ptr %B, i64 %n) nounwind uwtable ssp {
+; CHECK-LABEL: 'symbolicsiv1'
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx, align 4 --> Dst: store i32 %conv, ptr %arrayidx, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx, align 4 --> Dst: %0 = load i32, ptr %arrayidx5, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx, align 4 --> Dst: store i32 %0, ptr %B.addr.02, align 4
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: %0 = load i32, ptr %arrayidx5, align 4 --> Dst: %0 = load i32, ptr %arrayidx5, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: %0 = load i32, ptr %arrayidx5, align 4 --> Dst: store i32 %0, ptr %B.addr.02, align 4
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: store i32 %0, ptr %B.addr.02, align 4 --> Dst: store i32 %0, ptr %B.addr.02, align 4
+; CHECK-NEXT:    da analyze - none!
+;
 entry:
   %cmp1 = icmp eq i64 %n, 0
   br i1 %cmp1, label %for.end, label %for.body.preheader
 
-; CHECK: da analyze - none!
-; CHECK: da analyze - none!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - none!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - none!
-
 for.body.preheader:                               ; preds = %entry
   br label %for.body
 
@@ -102,17 +117,24 @@ for.end:                                          ; preds = %for.end.loopexit, %
 ;;    *B++ = A[-i + 2*n];
 
 define void @symbolicsiv2(ptr %A, ptr %B, i64 %n) nounwind uwtable ssp {
+; CHECK-LABEL: 'symbolicsiv2'
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx, align 4 --> Dst: store i32 %conv, ptr %arrayidx, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx, align 4 --> Dst: %0 = load i32, ptr %arrayidx3, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx, align 4 --> Dst: store i32 %0, ptr %B.addr.02, align 4
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: %0 = load i32, ptr %arrayidx3, align 4 --> Dst: %0 = load i32, ptr %arrayidx3, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: %0 = load i32, ptr %arrayidx3, align 4 --> Dst: store i32 %0, ptr %B.addr.02, align 4
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: store i32 %0, ptr %B.addr.02, align 4 --> Dst: store i32 %0, ptr %B.addr.02, align 4
+; CHECK-NEXT:    da analyze - none!
+;
 entry:
   %cmp1 = icmp eq i64 %n, 0
   br i1 %cmp1, label %for.end, label %for.body.preheader
 
-; CHECK: da analyze - none!
-; CHECK: da analyze - none!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - none!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - none!
-
 for.body.preheader:                               ; preds = %entry
   br label %for.body
 
@@ -147,17 +169,24 @@ for.end:                                          ; preds = %for.end.loopexit, %
 ;;    *B++ = A[i - 2*n];
 
 define void @symbolicsiv3(ptr %A, ptr %B, i64 %n) nounwind uwtable ssp {
+; CHECK-LABEL: 'symbolicsiv3'
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx, align 4 --> Dst: store i32 %conv, ptr %arrayidx, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx, align 4 --> Dst: %0 = load i32, ptr %arrayidx3, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx, align 4 --> Dst: store i32 %0, ptr %B.addr.02, align 4
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: %0 = load i32, ptr %arrayidx3, align 4 --> Dst: %0 = load i32, ptr %arrayidx3, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: %0 = load i32, ptr %arrayidx3, align 4 --> Dst: store i32 %0, ptr %B.addr.02, align 4
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: store i32 %0, ptr %B.addr.02, align 4 --> Dst: store i32 %0, ptr %B.addr.02, align 4
+; CHECK-NEXT:    da analyze - none!
+;
 entry:
   %cmp1 = icmp eq i64 %n, 0
   br i1 %cmp1, label %for.end, label %for.body.preheader
 
-; CHECK: da analyze - none!
-; CHECK: da analyze - none!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - none!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - none!
-
 for.body.preheader:                               ; preds = %entry
   br label %for.body
 
@@ -193,17 +222,24 @@ for.end:                                          ; preds = %for.end.loopexit, %
 ;;    *B++ = A[-i + n];
 
 define void @symbolicsiv4(ptr %A, ptr %B, i64 %n) nounwind uwtable ssp {
+; CHECK-LABEL: 'symbolicsiv4'
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx, align 4 --> Dst: store i32 %conv, ptr %arrayidx, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx, align 4 --> Dst: %0 = load i32, ptr %arrayidx3, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx, align 4 --> Dst: store i32 %0, ptr %B.addr.02, align 4
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: %0 = load i32, ptr %arrayidx3, align 4 --> Dst: %0 = load i32, ptr %arrayidx3, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: %0 = load i32, ptr %arrayidx3, align 4 --> Dst: store i32 %0, ptr %B.addr.02, align 4
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: store i32 %0, ptr %B.addr.02, align 4 --> Dst: store i32 %0, ptr %B.addr.02, align 4
+; CHECK-NEXT:    da analyze - none!
+;
 entry:
   %cmp1 = icmp eq i64 %n, 0
   br i1 %cmp1, label %for.end, label %for.body.preheader
 
-; CHECK: da analyze - none!
-; CHECK: da analyze - none!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - none!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - none!
-
 for.body.preheader:                               ; preds = %entry
   br label %for.body
 
@@ -238,17 +274,24 @@ for.end:                                          ; preds = %for.end.loopexit, %
 ;;    *B++ = A[-i - n];
 
 define void @symbolicsiv5(ptr %A, ptr %B, i64 %n) nounwind uwtable ssp {
+; CHECK-LABEL: 'symbolicsiv5'
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx, align 4 --> Dst: store i32 %conv, ptr %arrayidx, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx, align 4 --> Dst: %0 = load i32, ptr %arrayidx4, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx, align 4 --> Dst: store i32 %0, ptr %B.addr.02, align 4
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: %0 = load i32, ptr %arrayidx4, align 4 --> Dst: %0 = load i32, ptr %arrayidx4, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: %0 = load i32, ptr %arrayidx4, align 4 --> Dst: store i32 %0, ptr %B.addr.02, align 4
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: store i32 %0, ptr %B.addr.02, align 4 --> Dst: store i32 %0, ptr %B.addr.02, align 4
+; CHECK-NEXT:    da analyze - none!
+;
 entry:
   %cmp1 = icmp eq i64 %n, 0
   br i1 %cmp1, label %for.end, label %for.body.preheader
 
-; CHECK: da analyze - none!
-; CHECK: da analyze - none!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - none!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - none!
-
 for.body.preheader:                               ; preds = %entry
   br label %for.body
 
@@ -285,18 +328,25 @@ for.end:                                          ; preds = %for.end.loopexit, %
 ;;    *B++ = A[-i];
 
 define void @weaktest(ptr %A, ptr %B, i64 %n) nounwind uwtable ssp {
+; CHECK-LABEL: 'weaktest'
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx, align 4 --> Dst: store i32 %conv, ptr %arrayidx, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx, align 4 --> Dst: %0 = load i32, ptr %arrayidx2, align 4
+; CHECK-NEXT:    da analyze - flow [*|<] splitable!
+; CHECK-NEXT:    da analyze - split level = 1, iteration = ((0 smax (-4 + (-4 * %n))) /u 8)!
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx, align 4 --> Dst: store i32 %0, ptr %B.addr.02, align 4
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: %0 = load i32, ptr %arrayidx2, align 4 --> Dst: %0 = load i32, ptr %arrayidx2, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: %0 = load i32, ptr %arrayidx2, align 4 --> Dst: store i32 %0, ptr %B.addr.02, align 4
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: store i32 %0, ptr %B.addr.02, align 4 --> Dst: store i32 %0, ptr %B.addr.02, align 4
+; CHECK-NEXT:    da analyze - none!
+;
 entry:
   %cmp1 = icmp eq i64 %n, 0
   br i1 %cmp1, label %for.end, label %for.body.preheader
 
-; CHECK: da analyze - none!
-; CHECK: da analyze - flow [*|<] splitable!
-; CHECK: da analyze - split level = 1, iteration = ((0 smax (-4 + (-4 * %n))) /u 8)!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - none!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - none!
-
 for.body.preheader:                               ; preds = %entry
   br label %for.body
 
@@ -330,18 +380,24 @@ for.end:                                          ; preds = %for.end.loopexit, %
 ;;    *B++ = A[4*N*i + 3*M + 1];
 
 define void @symbolicsiv6(ptr %A, ptr %B, i64 %n, i64 %N, i64 %M) nounwind uwtable ssp {
+; CHECK-LABEL: 'symbolicsiv6'
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx, align 4 --> Dst: store i32 %conv, ptr %arrayidx, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx, align 4 --> Dst: %0 = load i32, ptr %arrayidx7, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx, align 4 --> Dst: store i32 %0, ptr %B.addr.02, align 4
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: %0 = load i32, ptr %arrayidx7, align 4 --> Dst: %0 = load i32, ptr %arrayidx7, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: %0 = load i32, ptr %arrayidx7, align 4 --> Dst: store i32 %0, ptr %B.addr.02, align 4
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: store i32 %0, ptr %B.addr.02, align 4 --> Dst: store i32 %0, ptr %B.addr.02, align 4
+; CHECK-NEXT:    da analyze - none!
+;
 entry:
   %cmp1 = icmp eq i64 %n, 0
   br i1 %cmp1, label %for.end, label %for.body.preheader
 
-; CHECK-LABEL: symbolicsiv6
-; CHECK: da analyze - none!
-; CHECK: da analyze - none!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - none!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - none!
-
 for.body.preheader:                               ; preds = %entry
   br label %for.body
 
@@ -380,16 +436,23 @@ for.end:                                          ; preds = %for.end.loopexit, %
 ;;    *B++ = A[2*N*i - 3*M + 2];
 
 define void @symbolicsiv7(ptr %A, ptr %B, i64 %n, i64 %N, i64 %M) nounwind uwtable ssp {
+; CHECK-LABEL: 'symbolicsiv7'
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx, align 4 --> Dst: store i32 %conv, ptr %arrayidx, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx, align 4 --> Dst: %1 = load i32, ptr %arrayidx6, align 4
+; CHECK-NEXT:    da analyze - flow [<>]!
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx, align 4 --> Dst: store i32 %1, ptr %B.addr.02, align 4
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: %1 = load i32, ptr %arrayidx6, align 4 --> Dst: %1 = load i32, ptr %arrayidx6, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: %1 = load i32, ptr %arrayidx6, align 4 --> Dst: store i32 %1, ptr %B.addr.02, align 4
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: store i32 %1, ptr %B.addr.02, align 4 --> Dst: store i32 %1, ptr %B.addr.02, align 4
+; CHECK-NEXT:    da analyze - none!
+;
 entry:
   %cmp1 = icmp eq i64 %n, 0
   br i1 %cmp1, label %for.end, label %for.body.preheader
-; CHECK-LABEL: symbolicsiv7
-; CHECK: da analyze - none!
-; CHECK: da analyze - flow [<>]!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - none!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - none!
 
 for.body.preheader:                               ; preds = %entry
   br label %for.body
diff --git a/llvm/test/Analysis/DependenceAnalysis/UsefulGEP.ll b/llvm/test/Analysis/DependenceAnalysis/UsefulGEP.ll
index 2d0e0a39df3c5..ccdd70160feab 100644
--- a/llvm/test/Analysis/DependenceAnalysis/UsefulGEP.ll
+++ b/llvm/test/Analysis/DependenceAnalysis/UsefulGEP.ll
@@ -1,11 +1,13 @@
-; RUN: opt < %s -disable-output "-passes=print<da>" -aa-pipeline=basic-aa 2>&1
+; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --version 5
+; RUN: opt < %s -disable-output "-passes=print<da>" -aa-pipeline=basic-aa 2>&1 \
+; RUN: | FileCheck %s
 ;; Check this doesn't crash.
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
 ;; struct s {
 ;;   int A[10][10];
-;;   int C[10][10][10]; 
+;;   int C[10][10][10];
 ;; } S;
 
 ;; void dep_constraint_crash_test(int k,int N) {
@@ -20,20 +22,28 @@ target triple = "x86_64-unknown-linux-gnu"
 @S = common global %struct.s zeroinitializer
 
 define void @dep_constraint_crash_test(i32 %k, i32 %N) {
+; CHECK-LABEL: 'dep_constraint_crash_test'
+; CHECK-NEXT:  Src: %0 = load i32, ptr %arrayidx, align 4 --> Dst: %0 = load i32, ptr %arrayidx, align 4
+; CHECK-NEXT:    da analyze - consistent input [S S]!
+; CHECK-NEXT:  Src: %0 = load i32, ptr %arrayidx, align 4 --> Dst: store i32 %0, ptr @S, align 4
+; CHECK-NEXT:    da analyze - anti [S S|<]!
+; CHECK-NEXT:  Src: store i32 %0, ptr @S, align 4 --> Dst: store i32 %0, ptr @S, align 4
+; CHECK-NEXT:    da analyze - consistent output [S S]!
+;
 entry:
   %cmp12 = icmp sgt i32 %N, 0
   br i1 %cmp12, label %for.cond1.preheader.lr.ph, label %for.end6
 
-for.cond1.preheader.lr.ph:                        
+for.cond1.preheader.lr.ph:
   %idxprom = sext i32 %k to i64
   %arrayidx = getelementptr inbounds %struct.s, ptr @S, i64 0, i32 1, i64 0, i64 0, i64 %idxprom
   br label %for.body3.preheader
 
-for.body3.preheader:                              
+for.body3.preheader:
   %i.013 = phi i32 [ 0, %for.cond1.preheader.lr.ph ], [ %inc5, %for.inc4 ]
   br label %for.body3
 
-for.body3:                                        
+for.body3:
   %j.011 = phi i32 [ %inc, %for.body3 ], [ 0, %for.body3.preheader ]
   %0 = load i32, ptr %arrayidx
   store i32 %0, ptr @S
@@ -41,11 +51,11 @@ for.body3:
   %exitcond = icmp eq i32 %inc, %N
   br i1 %exitcond, label %for.inc4, label %for.body3
 
-for.inc4:                                         
+for.inc4:
   %inc5 = add nuw nsw i32 %i.013, 1
   %exitcond14 = icmp eq i32 %inc5, %N
   br i1 %exitcond14, label %for.end6, label %for.body3.preheader
 
-for.end6:                                         
+for.end6:
   ret void
 }
diff --git a/llvm/test/Analysis/DependenceAnalysis/WeakCrossingSIV.ll b/llvm/test/Analysis/DependenceAnalysis/WeakCrossingSIV.ll
index 774c37da3d3de..cd044032e34f1 100644
--- a/llvm/test/Analysis/DependenceAnalysis/WeakCrossingSIV.ll
+++ b/llvm/test/Analysis/DependenceAnalysis/WeakCrossingSIV.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --version 5
 ; RUN: opt < %s -disable-output "-passes=print<da>" -aa-pipeline=basic-aa 2>&1 \
 ; RUN: | FileCheck %s
 
@@ -11,6 +12,20 @@ target triple = "x86_64-apple-macosx10.6.0"
 ;;    *B++ = A[1 - n*i];
 
 define void @weakcrossing0(ptr %A, ptr %B, i64 %n) nounwind uwtable ssp {
+; CHECK-LABEL: 'weakcrossing0'
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx, align 4 --> Dst: store i32 %conv, ptr %arrayidx, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx, align 4 --> Dst: %0 = load i32, ptr %arrayidx2, align 4
+; CHECK-NEXT:    da analyze - flow [0|<]!
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx, align 4 --> Dst: store i32 %0, ptr %B.addr.02, align 4
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: %0 = load i32, ptr %arrayidx2, align 4 --> Dst: %0 = load i32, ptr %arrayidx2, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: %0 = load i32, ptr %arrayidx2, align 4 --> Dst: store i32 %0, ptr %B.addr.02, align 4
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: store i32 %0, ptr %B.addr.02, align 4 --> Dst: store i32 %0, ptr %B.addr.02, align 4
+; CHECK-NEXT:    da analyze - none!
+;
 entry:
   %cmp1 = icmp eq i64 %n, 0
   br i1 %cmp1, label %for.end, label %for.body.preheader
@@ -18,13 +33,6 @@ entry:
 for.body.preheader:                               ; preds = %entry
   br label %for.body
 
-; CHECK: da analyze - none!
-; CHECK: da analyze - flow [0|<]!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - none!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - none!
-
 for.body:                                         ; preds = %for.body.preheader, %for.body
   %i.03 = phi i64 [ %inc, %for.body ], [ 0, %for.body.preheader ]
   %B.addr.02 = phi ptr [ %incdec.ptr, %for.body ], [ %B, %for.body.preheader ]
@@ -56,18 +64,25 @@ for.end:                                          ; preds = %for.end.loopexit, %
 ;;    *B++ = A[1 + n - i];
 
 define void @weakcrossing1(ptr %A, ptr %B, i64 %n) nounwind uwtable ssp {
+; CHECK-LABEL: 'weakcrossing1'
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx, align 4 --> Dst: store i32 %conv, ptr %arrayidx, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx, align 4 --> Dst: %0 = load i32, ptr %arrayidx2, align 4
+; CHECK-NEXT:    da analyze - flow [<>] splitable!
+; CHECK-NEXT:    da analyze - split level = 1, iteration = 0!
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx, align 4 --> Dst: store i32 %0, ptr %B.addr.02, align 4
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: %0 = load i32, ptr %arrayidx2, align 4 --> Dst: %0 = load i32, ptr %arrayidx2, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: %0 = load i32, ptr %arrayidx2, align 4 --> Dst: store i32 %0, ptr %B.addr.02, align 4
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: store i32 %0, ptr %B.addr.02, align 4 --> Dst: store i32 %0, ptr %B.addr.02, align 4
+; CHECK-NEXT:    da analyze - none!
+;
 entry:
   %cmp1 = icmp eq i64 %n, 0
   br i1 %cmp1, label %for.end, label %for.body.preheader
 
-; CHECK: da analyze - none!
-; CHECK: da analyze - flow [<>] splitable!
-; CHECK: da analyze - split level = 1, iteration = 0!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - none!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - none!
-
 for.body.preheader:                               ; preds = %entry
   br label %for.body
 
@@ -101,16 +116,23 @@ for.end:                                          ; preds = %for.end.loopexit, %
 ;;    *B++ = A[6 - i];
 
 define void @weakcrossing2(ptr %A, ptr %B, i64 %n) nounwind uwtable ssp {
+; CHECK-LABEL: 'weakcrossing2'
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx, align 4 --> Dst: store i32 %conv, ptr %arrayidx, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx, align 4 --> Dst: %0 = load i32, ptr %arrayidx1, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx, align 4 --> Dst: store i32 %0, ptr %B.addr.01, align 4
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: %0 = load i32, ptr %arrayidx1, align 4 --> Dst: %0 = load i32, ptr %arrayidx1, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: %0 = load i32, ptr %arrayidx1, align 4 --> Dst: store i32 %0, ptr %B.addr.01, align 4
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: store i32 %0, ptr %B.addr.01, align 4 --> Dst: store i32 %0, ptr %B.addr.01, align 4
+; CHECK-NEXT:    da analyze - none!
+;
 entry:
   br label %for.body
 
-; CHECK: da analyze - none!
-; CHECK: da analyze - none!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - none!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - none!
-
 for.body:                                         ; preds = %entry, %for.body
   %i.02 = phi i64 [ 0, %entry ], [ %inc, %for.body ]
   %B.addr.01 = phi ptr [ %B, %entry ], [ %incdec.ptr, %for.body ]
@@ -136,16 +158,23 @@ for.end:                                          ; preds = %for.body
 ;;    *B++ = A[6 - i];
 
 define void @weakcrossing3(ptr %A, ptr %B, i64 %n) nounwind uwtable ssp {
+; CHECK-LABEL: 'weakcrossing3'
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx, align 4 --> Dst: store i32 %conv, ptr %arrayidx, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx, align 4 --> Dst: %0 = load i32, ptr %arrayidx1, align 4
+; CHECK-NEXT:    da analyze - flow [0|<]!
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx, align 4 --> Dst: store i32 %0, ptr %B.addr.01, align 4
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: %0 = load i32, ptr %arrayidx1, align 4 --> Dst: %0 = load i32, ptr %arrayidx1, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: %0 = load i32, ptr %arrayidx1, align 4 --> Dst: store i32 %0, ptr %B.addr.01, align 4
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: store i32 %0, ptr %B.addr.01, align 4 --> Dst: store i32 %0, ptr %B.addr.01, align 4
+; CHECK-NEXT:    da analyze - none!
+;
 entry:
   br label %for.body
 
-; CHECK: da analyze - none!
-; CHECK: da analyze - flow [0|<]!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - none!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - none!
-
 for.body:                                         ; preds = %entry, %for.body
   %i.02 = phi i64 [ 0, %entry ], [ %inc, %for.body ]
   %B.addr.01 = phi ptr [ %B, %entry ], [ %incdec.ptr, %for.body ]
@@ -171,16 +200,23 @@ for.end:                                          ; preds = %for.body
 ;;    *B++ = A[-6 - i];
 
 define void @weakcrossing4(ptr %A, ptr %B, i64 %n) nounwind uwtable ssp {
+; CHECK-LABEL: 'weakcrossing4'
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx, align 4 --> Dst: store i32 %conv, ptr %arrayidx, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx, align 4 --> Dst: %0 = load i32, ptr %arrayidx1, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx, align 4 --> Dst: store i32 %0, ptr %B.addr.01, align 4
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: %0 = load i32, ptr %arrayidx1, align 4 --> Dst: %0 = load i32, ptr %arrayidx1, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: %0 = load i32, ptr %arrayidx1, align 4 --> Dst: store i32 %0, ptr %B.addr.01, align 4
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: store i32 %0, ptr %B.addr.01, align 4 --> Dst: store i32 %0, ptr %B.addr.01, align 4
+; CHECK-NEXT:    da analyze - none!
+;
 entry:
   br label %for.body
 
-; CHECK: da analyze - none!
-; CHECK: da analyze - none!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - none!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - none!
-
 for.body:                                         ; preds = %entry, %for.body
   %i.02 = phi i64 [ 0, %entry ], [ %inc, %for.body ]
   %B.addr.01 = phi ptr [ %B, %entry ], [ %incdec.ptr, %for.body ]
@@ -206,17 +242,24 @@ for.end:                                          ; preds = %for.body
 ;;    *B++ = A[5 - 3*i];
 
 define void @weakcrossing5(ptr %A, ptr %B, i64 %n) nounwind uwtable ssp {
+; CHECK-LABEL: 'weakcrossing5'
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx, align 4 --> Dst: store i32 %conv, ptr %arrayidx, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx, align 4 --> Dst: %1 = load i32, ptr %arrayidx2, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx, align 4 --> Dst: store i32 %1, ptr %B.addr.02, align 4
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: %1 = load i32, ptr %arrayidx2, align 4 --> Dst: %1 = load i32, ptr %arrayidx2, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: %1 = load i32, ptr %arrayidx2, align 4 --> Dst: store i32 %1, ptr %B.addr.02, align 4
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: store i32 %1, ptr %B.addr.02, align 4 --> Dst: store i32 %1, ptr %B.addr.02, align 4
+; CHECK-NEXT:    da analyze - none!
+;
 entry:
   %cmp1 = icmp eq i64 %n, 0
   br i1 %cmp1, label %for.end, label %for.body.preheader
 
-; CHECK: da analyze - none!
-; CHECK: da analyze - none!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - none!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - none!
-
 for.body.preheader:                               ; preds = %entry
   br label %for.body
 
@@ -250,17 +293,24 @@ for.end:                                          ; preds = %for.end.loopexit, %
 ;;    *B++ = A[5 - i];
 
 define void @weakcrossing6(ptr %A, ptr %B, i64 %n) nounwind uwtable ssp {
+; CHECK-LABEL: 'weakcrossing6'
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx, align 4 --> Dst: store i32 %conv, ptr %arrayidx, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx, align 4 --> Dst: %0 = load i32, ptr %arrayidx1, align 4
+; CHECK-NEXT:    da analyze - flow [<>] splitable!
+; CHECK-NEXT:    da analyze - split level = 1, iteration = 2!
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx, align 4 --> Dst: store i32 %0, ptr %B.addr.01, align 4
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: %0 = load i32, ptr %arrayidx1, align 4 --> Dst: %0 = load i32, ptr %arrayidx1, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: %0 = load i32, ptr %arrayidx1, align 4 --> Dst: store i32 %0, ptr %B.addr.01, align 4
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: store i32 %0, ptr %B.addr.01, align 4 --> Dst: store i32 %0, ptr %B.addr.01, align 4
+; CHECK-NEXT:    da analyze - none!
+;
 entry:
   br label %for.body
 
-; CHECK: da analyze - none!
-; CHECK: da analyze - flow [<>] splitable!
-; CHECK: da analyze - split level = 1, iteration = 2!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - none!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - none!
-
 for.body:                                         ; preds = %entry, %for.body
   %i.02 = phi i64 [ 0, %entry ], [ %inc, %for.body ]
   %B.addr.01 = phi ptr [ %B, %entry ], [ %incdec.ptr, %for.body ]
diff --git a/llvm/test/Analysis/DependenceAnalysis/WeakZeroDstSIV.ll b/llvm/test/Analysis/DependenceAnalysis/WeakZeroDstSIV.ll
index 63774ce57c9b7..71a5ed62b8e21 100644
--- a/llvm/test/Analysis/DependenceAnalysis/WeakZeroDstSIV.ll
+++ b/llvm/test/Analysis/DependenceAnalysis/WeakZeroDstSIV.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --version 5
 ; RUN: opt < %s -disable-output "-passes=print<da>" -aa-pipeline=basic-aa 2>&1 \
 ; RUN: | FileCheck %s
 
@@ -11,14 +12,18 @@ target triple = "x86_64-apple-macosx10.6.0"
 ;;    A[0] = 2;
 
 define void @dstzero(ptr nocapture %A, i32 %N) {
+; CHECK-LABEL: 'dstzero'
+; CHECK-NEXT:  Src: store i32 0, ptr %arrayidx, align 4 --> Dst: store i32 0, ptr %arrayidx, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: store i32 0, ptr %arrayidx, align 4 --> Dst: store i32 1, ptr %A, align 4
+; CHECK-NEXT:    da analyze - output [p<=|<]!
+; CHECK-NEXT:  Src: store i32 1, ptr %A, align 4 --> Dst: store i32 1, ptr %A, align 4
+; CHECK-NEXT:    da analyze - consistent output [S]!
+;
 entry:
   %cmp6 = icmp sgt i32 %N, 0
   br i1 %cmp6, label %for.body, label %for.cond.cleanup
 
-; CHECK: da analyze - none!
-; CHECK: da analyze - output [p<=|<]!
-; CHECK: da analyze - consistent output [S]!
-
 for.body:                                         ; preds = %entry, %for.body
   %i.07 = phi i32 [ %add, %for.body ], [ 0, %entry ]
   %arrayidx = getelementptr inbounds i32, ptr %A, i32 %i.07
@@ -40,16 +45,23 @@ for.cond.cleanup:                                 ; preds = %for.body, %entry
 ;;    *B++ = A[10];
 
 define void @weakzerodst0(ptr %A, ptr %B, i64 %n) nounwind uwtable ssp {
+; CHECK-LABEL: 'weakzerodst0'
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx, align 4 --> Dst: store i32 %conv, ptr %arrayidx, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx, align 4 --> Dst: %0 = load i32, ptr %arrayidx1, align 4
+; CHECK-NEXT:    da analyze - flow [p<=|<]!
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx, align 4 --> Dst: store i32 %0, ptr %B.addr.01, align 4
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: %0 = load i32, ptr %arrayidx1, align 4 --> Dst: %0 = load i32, ptr %arrayidx1, align 4
+; CHECK-NEXT:    da analyze - consistent input [S]!
+; CHECK-NEXT:  Src: %0 = load i32, ptr %arrayidx1, align 4 --> Dst: store i32 %0, ptr %B.addr.01, align 4
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: store i32 %0, ptr %B.addr.01, align 4 --> Dst: store i32 %0, ptr %B.addr.01, align 4
+; CHECK-NEXT:    da analyze - none!
+;
 entry:
   br label %for.body
 
-; CHECK: da analyze - none!
-; CHECK: da analyze - flow [p<=|<]!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - consistent input [S]!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - none!
-
 for.body:                                         ; preds = %entry, %for.body
   %i.02 = phi i64 [ 0, %entry ], [ %inc, %for.body ]
   %B.addr.01 = phi ptr [ %B, %entry ], [ %incdec.ptr, %for.body ]
@@ -76,17 +88,24 @@ for.end:                                          ; preds = %for.body
 ;;    *B++ = A[10];
 
 define void @weakzerodst1(ptr %A, ptr %B, i64 %n) nounwind uwtable ssp {
+; CHECK-LABEL: 'weakzerodst1'
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx, align 4 --> Dst: store i32 %conv, ptr %arrayidx, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx, align 4 --> Dst: %0 = load i32, ptr %arrayidx1, align 4
+; CHECK-NEXT:    da analyze - flow [p<=|<]!
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx, align 4 --> Dst: store i32 %0, ptr %B.addr.02, align 4
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: %0 = load i32, ptr %arrayidx1, align 4 --> Dst: %0 = load i32, ptr %arrayidx1, align 4
+; CHECK-NEXT:    da analyze - consistent input [S]!
+; CHECK-NEXT:  Src: %0 = load i32, ptr %arrayidx1, align 4 --> Dst: store i32 %0, ptr %B.addr.02, align 4
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: store i32 %0, ptr %B.addr.02, align 4 --> Dst: store i32 %0, ptr %B.addr.02, align 4
+; CHECK-NEXT:    da analyze - none!
+;
 entry:
   %cmp1 = icmp eq i64 %n, 0
   br i1 %cmp1, label %for.end, label %for.body.preheader
 
-; CHECK: da analyze - none!
-; CHECK: da analyze - flow [p<=|<]!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - consistent input [S]!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - none!
-
 for.body.preheader:                               ; preds = %entry
   br label %for.body
 
@@ -119,16 +138,23 @@ for.end:                                          ; preds = %for.end.loopexit, %
 ;;    *B++ = A[10];
 
 define void @weakzerodst2(ptr %A, ptr %B, i64 %n) nounwind uwtable ssp {
+; CHECK-LABEL: 'weakzerodst2'
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx, align 4 --> Dst: store i32 %conv, ptr %arrayidx, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx, align 4 --> Dst: %0 = load i32, ptr %arrayidx1, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx, align 4 --> Dst: store i32 %0, ptr %B.addr.01, align 4
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: %0 = load i32, ptr %arrayidx1, align 4 --> Dst: %0 = load i32, ptr %arrayidx1, align 4
+; CHECK-NEXT:    da analyze - consistent input [S]!
+; CHECK-NEXT:  Src: %0 = load i32, ptr %arrayidx1, align 4 --> Dst: store i32 %0, ptr %B.addr.01, align 4
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: store i32 %0, ptr %B.addr.01, align 4 --> Dst: store i32 %0, ptr %B.addr.01, align 4
+; CHECK-NEXT:    da analyze - none!
+;
 entry:
   br label %for.body
 
-; CHECK: da analyze - none!
-; CHECK: da analyze - none!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - consistent input [S]!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - none!
-
 for.body:                                         ; preds = %entry, %for.body
   %i.02 = phi i64 [ 0, %entry ], [ %inc, %for.body ]
   %B.addr.01 = phi ptr [ %B, %entry ], [ %incdec.ptr, %for.body ]
@@ -154,16 +180,23 @@ for.end:                                          ; preds = %for.body
 ;;    *B++ = A[10];
 
 define void @weakzerodst3(ptr %A, ptr %B, i64 %n) nounwind uwtable ssp {
+; CHECK-LABEL: 'weakzerodst3'
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx, align 4 --> Dst: store i32 %conv, ptr %arrayidx, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx, align 4 --> Dst: %0 = load i32, ptr %arrayidx1, align 4
+; CHECK-NEXT:    da analyze - flow [=>p|<]!
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx, align 4 --> Dst: store i32 %0, ptr %B.addr.01, align 4
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: %0 = load i32, ptr %arrayidx1, align 4 --> Dst: %0 = load i32, ptr %arrayidx1, align 4
+; CHECK-NEXT:    da analyze - consistent input [S]!
+; CHECK-NEXT:  Src: %0 = load i32, ptr %arrayidx1, align 4 --> Dst: store i32 %0, ptr %B.addr.01, align 4
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: store i32 %0, ptr %B.addr.01, align 4 --> Dst: store i32 %0, ptr %B.addr.01, align 4
+; CHECK-NEXT:    da analyze - none!
+;
 entry:
   br label %for.body
 
-; CHECK: da analyze - none!
-; CHECK: da analyze - flow [=>p|<]!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - consistent input [S]!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - none!
-
 for.body:                                         ; preds = %entry, %for.body
   %i.02 = phi i64 [ 0, %entry ], [ %inc, %for.body ]
   %B.addr.01 = phi ptr [ %B, %entry ], [ %incdec.ptr, %for.body ]
@@ -189,16 +222,23 @@ for.end:                                          ; preds = %for.body
 ;;    *B++ = A[10];
 
 define void @weakzerodst4(ptr %A, ptr %B, i64 %n) nounwind uwtable ssp {
+; CHECK-LABEL: 'weakzerodst4'
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx, align 4 --> Dst: store i32 %conv, ptr %arrayidx, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx, align 4 --> Dst: %0 = load i32, ptr %arrayidx1, align 4
+; CHECK-NEXT:    da analyze - flow [*|<]!
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx, align 4 --> Dst: store i32 %0, ptr %B.addr.01, align 4
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: %0 = load i32, ptr %arrayidx1, align 4 --> Dst: %0 = load i32, ptr %arrayidx1, align 4
+; CHECK-NEXT:    da analyze - consistent input [S]!
+; CHECK-NEXT:  Src: %0 = load i32, ptr %arrayidx1, align 4 --> Dst: store i32 %0, ptr %B.addr.01, align 4
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: store i32 %0, ptr %B.addr.01, align 4 --> Dst: store i32 %0, ptr %B.addr.01, align 4
+; CHECK-NEXT:    da analyze - none!
+;
 entry:
   br label %for.body
 
-; CHECK: da analyze - none!
-; CHECK: da analyze - flow [*|<]!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - consistent input [S]!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - none!
-
 for.body:                                         ; preds = %entry, %for.body
   %i.02 = phi i64 [ 0, %entry ], [ %inc, %for.body ]
   %B.addr.01 = phi ptr [ %B, %entry ], [ %incdec.ptr, %for.body ]
@@ -224,16 +264,23 @@ for.end:                                          ; preds = %for.body
 ;;    *B++ = A[-10];
 
 define void @weakzerodst5(ptr %A, ptr %B, i64 %n) nounwind uwtable ssp {
+; CHECK-LABEL: 'weakzerodst5'
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx, align 4 --> Dst: store i32 %conv, ptr %arrayidx, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx, align 4 --> Dst: %0 = load i32, ptr %arrayidx1, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx, align 4 --> Dst: store i32 %0, ptr %B.addr.01, align 4
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: %0 = load i32, ptr %arrayidx1, align 4 --> Dst: %0 = load i32, ptr %arrayidx1, align 4
+; CHECK-NEXT:    da analyze - consistent input [S]!
+; CHECK-NEXT:  Src: %0 = load i32, ptr %arrayidx1, align 4 --> Dst: store i32 %0, ptr %B.addr.01, align 4
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: store i32 %0, ptr %B.addr.01, align 4 --> Dst: store i32 %0, ptr %B.addr.01, align 4
+; CHECK-NEXT:    da analyze - none!
+;
 entry:
   br label %for.body
 
-; CHECK: da analyze - none!
-; CHECK: da analyze - none!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - consistent input [S]!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - none!
-
 for.body:                                         ; preds = %entry, %for.body
   %i.02 = phi i64 [ 0, %entry ], [ %inc, %for.body ]
   %B.addr.01 = phi ptr [ %B, %entry ], [ %incdec.ptr, %for.body ]
@@ -259,17 +306,24 @@ for.end:                                          ; preds = %for.body
 ;;    *B++ = A[10];
 
 define void @weakzerodst6(ptr %A, ptr %B, i64 %n) nounwind uwtable ssp {
+; CHECK-LABEL: 'weakzerodst6'
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx, align 4 --> Dst: store i32 %conv, ptr %arrayidx, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx, align 4 --> Dst: %0 = load i32, ptr %arrayidx1, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx, align 4 --> Dst: store i32 %0, ptr %B.addr.02, align 4
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: %0 = load i32, ptr %arrayidx1, align 4 --> Dst: %0 = load i32, ptr %arrayidx1, align 4
+; CHECK-NEXT:    da analyze - consistent input [S]!
+; CHECK-NEXT:  Src: %0 = load i32, ptr %arrayidx1, align 4 --> Dst: store i32 %0, ptr %B.addr.02, align 4
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: store i32 %0, ptr %B.addr.02, align 4 --> Dst: store i32 %0, ptr %B.addr.02, align 4
+; CHECK-NEXT:    da analyze - none!
+;
 entry:
   %cmp1 = icmp eq i64 %n, 0
   br i1 %cmp1, label %for.end, label %for.body.preheader
 
-; CHECK: da analyze - none!
-; CHECK: da analyze - none!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - consistent input [S]!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - none!
-
 for.body.preheader:                               ; preds = %entry
   br label %for.body
 
diff --git a/llvm/test/Analysis/DependenceAnalysis/WeakZeroSrcSIV.ll b/llvm/test/Analysis/DependenceAnalysis/WeakZeroSrcSIV.ll
index 8755faa36d400..2ee1a37775f46 100644
--- a/llvm/test/Analysis/DependenceAnalysis/WeakZeroSrcSIV.ll
+++ b/llvm/test/Analysis/DependenceAnalysis/WeakZeroSrcSIV.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --version 5
 ; RUN: opt < %s -disable-output "-passes=print<da>" -aa-pipeline=basic-aa 2>&1 \
 ; RUN: | FileCheck %s
 
@@ -11,14 +12,18 @@ target triple = "x86_64-apple-macosx10.6.0"
 ;;    A[i] = 2;
 
 define void @dstzero(ptr nocapture %A, i32 %N) {
+; CHECK-LABEL: 'dstzero'
+; CHECK-NEXT:  Src: store i32 0, ptr %A, align 4 --> Dst: store i32 0, ptr %A, align 4
+; CHECK-NEXT:    da analyze - consistent output [S]!
+; CHECK-NEXT:  Src: store i32 0, ptr %A, align 4 --> Dst: store i32 1, ptr %arrayidx1, align 4
+; CHECK-NEXT:    da analyze - output [p=>|<]!
+; CHECK-NEXT:  Src: store i32 1, ptr %arrayidx1, align 4 --> Dst: store i32 1, ptr %arrayidx1, align 4
+; CHECK-NEXT:    da analyze - none!
+;
 entry:
   %cmp6 = icmp sgt i32 %N, 0
   br i1 %cmp6, label %for.body, label %for.cond.cleanup
 
-; CHECK: da analyze - consistent output [S]!
-; CHECK: da analyze - output [p=>|<]!
-; CHECK: da analyze - none!
-
 for.body:                                         ; preds = %entry, %for.body
   %i.07 = phi i32 [ %add, %for.body ], [ 0, %entry ]
   store i32 0, ptr %A, align 4
@@ -38,16 +43,23 @@ for.cond.cleanup:                                 ; preds = %for.body, %entry
 ;;    *B++ = A[2*i + 10];
 
 define void @weakzerosrc0(ptr %A, ptr %B, i64 %n) nounwind uwtable ssp {
+; CHECK-LABEL: 'weakzerosrc0'
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx, align 4 --> Dst: store i32 %conv, ptr %arrayidx, align 4
+; CHECK-NEXT:    da analyze - consistent output [S]!
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx, align 4 --> Dst: %0 = load i32, ptr %arrayidx1, align 4
+; CHECK-NEXT:    da analyze - flow [p=>|<]!
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx, align 4 --> Dst: store i32 %0, ptr %B.addr.01, align 4
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: %0 = load i32, ptr %arrayidx1, align 4 --> Dst: %0 = load i32, ptr %arrayidx1, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: %0 = load i32, ptr %arrayidx1, align 4 --> Dst: store i32 %0, ptr %B.addr.01, align 4
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: store i32 %0, ptr %B.addr.01, align 4 --> Dst: store i32 %0, ptr %B.addr.01, align 4
+; CHECK-NEXT:    da analyze - none!
+;
 entry:
   br label %for.body
 
-; CHECK: da analyze - consistent output [S]!
-; CHECK: da analyze - flow [p=>|<]!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - none!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - none!
-
 for.body:                                         ; preds = %entry, %for.body
   %i.02 = phi i64 [ 0, %entry ], [ %inc, %for.body ]
   %B.addr.01 = phi ptr [ %B, %entry ], [ %incdec.ptr, %for.body ]
@@ -74,17 +86,24 @@ for.end:                                          ; preds = %for.body
 ;;    *B++ = A[n*i + 10];
 
 define void @weakzerosrc1(ptr %A, ptr %B, i64 %n) nounwind uwtable ssp {
+; CHECK-LABEL: 'weakzerosrc1'
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx, align 4 --> Dst: store i32 %conv, ptr %arrayidx, align 4
+; CHECK-NEXT:    da analyze - consistent output [S]!
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx, align 4 --> Dst: %0 = load i32, ptr %arrayidx1, align 4
+; CHECK-NEXT:    da analyze - flow [p=>|<]!
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx, align 4 --> Dst: store i32 %0, ptr %B.addr.02, align 4
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: %0 = load i32, ptr %arrayidx1, align 4 --> Dst: %0 = load i32, ptr %arrayidx1, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: %0 = load i32, ptr %arrayidx1, align 4 --> Dst: store i32 %0, ptr %B.addr.02, align 4
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: store i32 %0, ptr %B.addr.02, align 4 --> Dst: store i32 %0, ptr %B.addr.02, align 4
+; CHECK-NEXT:    da analyze - none!
+;
 entry:
   %cmp1 = icmp eq i64 %n, 0
   br i1 %cmp1, label %for.end, label %for.body.preheader
 
-; CHECK: da analyze - consistent output [S]!
-; CHECK: da analyze - flow [p=>|<]!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - none!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - none!
-
 for.body.preheader:                               ; preds = %entry
   br label %for.body
 
@@ -117,16 +136,23 @@ for.end:                                          ; preds = %for.end.loopexit, %
 ;;    *B++ = A[2*i];
 
 define void @weakzerosrc2(ptr %A, ptr %B, i64 %n) nounwind uwtable ssp {
+; CHECK-LABEL: 'weakzerosrc2'
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx, align 4 --> Dst: store i32 %conv, ptr %arrayidx, align 4
+; CHECK-NEXT:    da analyze - consistent output [S]!
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx, align 4 --> Dst: %0 = load i32, ptr %arrayidx1, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx, align 4 --> Dst: store i32 %0, ptr %B.addr.01, align 4
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: %0 = load i32, ptr %arrayidx1, align 4 --> Dst: %0 = load i32, ptr %arrayidx1, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: %0 = load i32, ptr %arrayidx1, align 4 --> Dst: store i32 %0, ptr %B.addr.01, align 4
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: store i32 %0, ptr %B.addr.01, align 4 --> Dst: store i32 %0, ptr %B.addr.01, align 4
+; CHECK-NEXT:    da analyze - none!
+;
 entry:
   br label %for.body
 
-; CHECK: da analyze - consistent output [S]!
-; CHECK: da analyze - none!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - none!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - none!
-
 for.body:                                         ; preds = %entry, %for.body
   %i.02 = phi i64 [ 0, %entry ], [ %inc, %for.body ]
   %B.addr.01 = phi ptr [ %B, %entry ], [ %incdec.ptr, %for.body ]
@@ -152,16 +178,23 @@ for.end:                                          ; preds = %for.body
 ;;    *B++ = A[2*i];
 
 define void @weakzerosrc3(ptr %A, ptr %B, i64 %n) nounwind uwtable ssp {
+; CHECK-LABEL: 'weakzerosrc3'
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx, align 4 --> Dst: store i32 %conv, ptr %arrayidx, align 4
+; CHECK-NEXT:    da analyze - consistent output [S]!
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx, align 4 --> Dst: %0 = load i32, ptr %arrayidx1, align 4
+; CHECK-NEXT:    da analyze - flow [<=p|<]!
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx, align 4 --> Dst: store i32 %0, ptr %B.addr.01, align 4
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: %0 = load i32, ptr %arrayidx1, align 4 --> Dst: %0 = load i32, ptr %arrayidx1, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: %0 = load i32, ptr %arrayidx1, align 4 --> Dst: store i32 %0, ptr %B.addr.01, align 4
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: store i32 %0, ptr %B.addr.01, align 4 --> Dst: store i32 %0, ptr %B.addr.01, align 4
+; CHECK-NEXT:    da analyze - none!
+;
 entry:
   br label %for.body
 
-; CHECK: da analyze - consistent output [S]!
-; CHECK: da analyze - flow [<=p|<]!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - none!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - none!
-
 for.body:                                         ; preds = %entry, %for.body
   %i.02 = phi i64 [ 0, %entry ], [ %inc, %for.body ]
   %B.addr.01 = phi ptr [ %B, %entry ], [ %incdec.ptr, %for.body ]
@@ -187,16 +220,23 @@ for.end:                                          ; preds = %for.body
 ;;    *B++ = A[2*i];
 
 define void @weakzerosrc4(ptr %A, ptr %B, i64 %n) nounwind uwtable ssp {
+; CHECK-LABEL: 'weakzerosrc4'
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx, align 4 --> Dst: store i32 %conv, ptr %arrayidx, align 4
+; CHECK-NEXT:    da analyze - consistent output [S]!
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx, align 4 --> Dst: %0 = load i32, ptr %arrayidx1, align 4
+; CHECK-NEXT:    da analyze - flow [*|<]!
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx, align 4 --> Dst: store i32 %0, ptr %B.addr.01, align 4
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: %0 = load i32, ptr %arrayidx1, align 4 --> Dst: %0 = load i32, ptr %arrayidx1, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: %0 = load i32, ptr %arrayidx1, align 4 --> Dst: store i32 %0, ptr %B.addr.01, align 4
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: store i32 %0, ptr %B.addr.01, align 4 --> Dst: store i32 %0, ptr %B.addr.01, align 4
+; CHECK-NEXT:    da analyze - none!
+;
 entry:
   br label %for.body
 
-; CHECK: da analyze - consistent output [S]!
-; CHECK: da analyze - flow [*|<]!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - none!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - none!
-
 for.body:                                         ; preds = %entry, %for.body
   %i.02 = phi i64 [ 0, %entry ], [ %inc, %for.body ]
   %B.addr.01 = phi ptr [ %B, %entry ], [ %incdec.ptr, %for.body ]
@@ -222,16 +262,23 @@ for.end:                                          ; preds = %for.body
 ;;    *B++ = A[2*i];
 
 define void @weakzerosrc5(ptr %A, ptr %B, i64 %n) nounwind uwtable ssp {
+; CHECK-LABEL: 'weakzerosrc5'
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx, align 4 --> Dst: store i32 %conv, ptr %arrayidx, align 4
+; CHECK-NEXT:    da analyze - consistent output [S]!
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx, align 4 --> Dst: %0 = load i32, ptr %arrayidx1, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx, align 4 --> Dst: store i32 %0, ptr %B.addr.01, align 4
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: %0 = load i32, ptr %arrayidx1, align 4 --> Dst: %0 = load i32, ptr %arrayidx1, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: %0 = load i32, ptr %arrayidx1, align 4 --> Dst: store i32 %0, ptr %B.addr.01, align 4
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: store i32 %0, ptr %B.addr.01, align 4 --> Dst: store i32 %0, ptr %B.addr.01, align 4
+; CHECK-NEXT:    da analyze - none!
+;
 entry:
   br label %for.body
 
-; CHECK: da analyze - consistent output [S]!
-; CHECK: da analyze - none!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - none!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - none!
-
 for.body:                                         ; preds = %entry, %for.body
   %i.02 = phi i64 [ 0, %entry ], [ %inc, %for.body ]
   %B.addr.01 = phi ptr [ %B, %entry ], [ %incdec.ptr, %for.body ]
@@ -257,17 +304,24 @@ for.end:                                          ; preds = %for.body
 ;;    *B++ = A[3*i];
 
 define void @weakzerosrc6(ptr %A, ptr %B, i64 %n) nounwind uwtable ssp {
+; CHECK-LABEL: 'weakzerosrc6'
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx, align 4 --> Dst: store i32 %conv, ptr %arrayidx, align 4
+; CHECK-NEXT:    da analyze - consistent output [S]!
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx, align 4 --> Dst: %0 = load i32, ptr %arrayidx1, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: store i32 %conv, ptr %arrayidx, align 4 --> Dst: store i32 %0, ptr %B.addr.02, align 4
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: %0 = load i32, ptr %arrayidx1, align 4 --> Dst: %0 = load i32, ptr %arrayidx1, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: %0 = load i32, ptr %arrayidx1, align 4 --> Dst: store i32 %0, ptr %B.addr.02, align 4
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: store i32 %0, ptr %B.addr.02, align 4 --> Dst: store i32 %0, ptr %B.addr.02, align 4
+; CHECK-NEXT:    da analyze - none!
+;
 entry:
   %cmp1 = icmp eq i64 %n, 0
   br i1 %cmp1, label %for.end, label %for.body.preheader
 
-; CHECK: da analyze - consistent output [S]!
-; CHECK: da analyze - none!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - none!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - none!
-
 for.body.preheader:                               ; preds = %entry
   br label %for.body
 
diff --git a/llvm/test/Analysis/DependenceAnalysis/ZIV.ll b/llvm/test/Analysis/DependenceAnalysis/ZIV.ll
index cef88a0b3e6b1..d3d5da7bded52 100644
--- a/llvm/test/Analysis/DependenceAnalysis/ZIV.ll
+++ b/llvm/test/Analysis/DependenceAnalysis/ZIV.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --version 5
 ; RUN: opt < %s -disable-output "-passes=print<da>" -aa-pipeline=basic-aa 2>&1 \
 ; RUN: | FileCheck %s
 
@@ -10,18 +11,24 @@ target triple = "x86_64-apple-macosx10.6.0"
 ;;  *B = A[1 + n];
 
 define void @z0(ptr %A, ptr %B, i64 %n) nounwind uwtable ssp {
+; CHECK-LABEL: 'z0'
+; CHECK-NEXT:  Src: store i32 0, ptr %arrayidx, align 4 --> Dst: store i32 0, ptr %arrayidx, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: store i32 0, ptr %arrayidx, align 4 --> Dst: %0 = load i32, ptr %arrayidx2, align 4
+; CHECK-NEXT:    da analyze - consistent flow [|<]!
+; CHECK-NEXT:  Src: store i32 0, ptr %arrayidx, align 4 --> Dst: store i32 %0, ptr %B, align 4
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: %0 = load i32, ptr %arrayidx2, align 4 --> Dst: %0 = load i32, ptr %arrayidx2, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: %0 = load i32, ptr %arrayidx2, align 4 --> Dst: store i32 %0, ptr %B, align 4
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: store i32 %0, ptr %B, align 4 --> Dst: store i32 %0, ptr %B, align 4
+; CHECK-NEXT:    da analyze - none!
+;
 entry:
   %add = add i64 %n, 1
   %arrayidx = getelementptr inbounds i32, ptr %A, i64 %add
   store i32 0, ptr %arrayidx, align 4
-
-; CHECK: da analyze - none!
-; CHECK: da analyze - consistent flow [|<]!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - none!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - none!
-
   %add1 = add i64 %n, 1
   %arrayidx2 = getelementptr inbounds i32, ptr %A, i64 %add1
   %0 = load i32, ptr %arrayidx2, align 4
@@ -34,17 +41,23 @@ entry:
 ;;  *B = A[n + 1];
 
 define void @z1(ptr %A, ptr %B, i64 %n) nounwind uwtable ssp {
+; CHECK-LABEL: 'z1'
+; CHECK-NEXT:  Src: store i32 0, ptr %arrayidx, align 4 --> Dst: store i32 0, ptr %arrayidx, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: store i32 0, ptr %arrayidx, align 4 --> Dst: %0 = load i32, ptr %arrayidx1, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: store i32 0, ptr %arrayidx, align 4 --> Dst: store i32 %0, ptr %B, align 4
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: %0 = load i32, ptr %arrayidx1, align 4 --> Dst: %0 = load i32, ptr %arrayidx1, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: %0 = load i32, ptr %arrayidx1, align 4 --> Dst: store i32 %0, ptr %B, align 4
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: store i32 %0, ptr %B, align 4 --> Dst: store i32 %0, ptr %B, align 4
+; CHECK-NEXT:    da analyze - none!
+;
 entry:
   %arrayidx = getelementptr inbounds i32, ptr %A, i64 %n
   store i32 0, ptr %arrayidx, align 4
-
-; CHECK: da analyze - none!
-; CHECK: da analyze - none!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - none!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - none!
-
   %add = add i64 %n, 1
   %arrayidx1 = getelementptr inbounds i32, ptr %A, i64 %add
   %0 = load i32, ptr %arrayidx1, align 4
@@ -57,17 +70,23 @@ entry:
 ;;  *B = A[m];
 
 define void @z2(ptr %A, ptr %B, i64 %n, i64 %m) nounwind uwtable ssp {
+; CHECK-LABEL: 'z2'
+; CHECK-NEXT:  Src: store i32 0, ptr %arrayidx, align 4 --> Dst: store i32 0, ptr %arrayidx, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: store i32 0, ptr %arrayidx, align 4 --> Dst: %0 = load i32, ptr %arrayidx1, align 4
+; CHECK-NEXT:    da analyze - flow [|<]!
+; CHECK-NEXT:  Src: store i32 0, ptr %arrayidx, align 4 --> Dst: store i32 %0, ptr %B, align 4
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: %0 = load i32, ptr %arrayidx1, align 4 --> Dst: %0 = load i32, ptr %arrayidx1, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: %0 = load i32, ptr %arrayidx1, align 4 --> Dst: store i32 %0, ptr %B, align 4
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: store i32 %0, ptr %B, align 4 --> Dst: store i32 %0, ptr %B, align 4
+; CHECK-NEXT:    da analyze - none!
+;
 entry:
   %arrayidx = getelementptr inbounds i32, ptr %A, i64 %n
   store i32 0, ptr %arrayidx, align 4
-
-; CHECK: da analyze - none!
-; CHECK: da analyze - flow [|<]!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - none!
-; CHECK: da analyze - confused!
-; CHECK: da analyze - none!
-
   %arrayidx1 = getelementptr inbounds i32, ptr %A, i64 %m
   %0 = load i32, ptr %arrayidx1, align 4
   store i32 %0, ptr %B, align 4
diff --git a/llvm/test/Analysis/DependenceAnalysis/lcssa.ll b/llvm/test/Analysis/DependenceAnalysis/lcssa.ll
index 1e65f87920258..ffba9fcb98903 100644
--- a/llvm/test/Analysis/DependenceAnalysis/lcssa.ll
+++ b/llvm/test/Analysis/DependenceAnalysis/lcssa.ll
@@ -1,10 +1,16 @@
+; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --version 5
 ; RUN: opt < %s -disable-output "-passes=print<da>"                            \
 ; RUN: "-aa-pipeline=basic-aa,tbaa" 2>&1 | FileCheck %s
 
-; CHECK:      Src:  %v = load i32, ptr %arrayidx1, align 4 --> Dst:  store i32 %add, ptr %a.lcssa, align 4
-; CHECK-NEXT: da analyze - anti [*|<]!
-
 define void @f(ptr %a, i32 %n, i64 %n2) {
+; CHECK-LABEL: 'f'
+; CHECK-NEXT:  Src: %v = load i32, ptr %arrayidx1, align 4 --> Dst: %v = load i32, ptr %arrayidx1, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: %v = load i32, ptr %arrayidx1, align 4 --> Dst: store i32 %add, ptr %a.lcssa, align 4
+; CHECK-NEXT:    da analyze - anti [*|<]!
+; CHECK-NEXT:  Src: store i32 %add, ptr %a.lcssa, align 4 --> Dst: store i32 %add, ptr %a.lcssa, align 4
+; CHECK-NEXT:    da analyze - consistent output [S]!
+;
 entry:
   br label %while.body
 
diff --git a/llvm/test/Analysis/DependenceAnalysis/new-pm-invalidation.ll b/llvm/test/Analysis/DependenceAnalysis/new-pm-invalidation.ll
index 37bd5243803d0..1018ec5e8661c 100644
--- a/llvm/test/Analysis/DependenceAnalysis/new-pm-invalidation.ll
+++ b/llvm/test/Analysis/DependenceAnalysis/new-pm-invalidation.ll
@@ -1,6 +1,9 @@
 ; RUN: opt < %s -passes='require<da>,invalidate<scalar-evolution>,print<da>'   \
 ; RUN:   -disable-output -debug-pass-manager 2>&1 | FileCheck %s
 
+; This test cannot be converted to use utils/update_analyze_test_checks.py
+; because the pass order printing is not deterministic.
+
 ; CHECK: Running analysis: DependenceAnalysis on test_no_noalias
 ; CHECK: Running analysis: ScalarEvolutionAnalysis on test_no_noalias
 ; CHECK: Invalidating analysis: ScalarEvolutionAnalysis on test_no_noalias
diff --git a/llvm/test/Analysis/KernelInfo/allocas.ll b/llvm/test/Analysis/KernelInfo/allocas.ll
new file mode 100644
index 0000000000000..94506645f7ec6
--- /dev/null
+++ b/llvm/test/Analysis/KernelInfo/allocas.ll
@@ -0,0 +1,117 @@
+; Check info on allocas.
+
+; RUN: opt -pass-remarks=kernel-info -passes=kernel-info \
+; RUN:     -disable-output %s 2>&1 | \
+; RUN:   FileCheck -match-full-lines %s
+
+target datalayout = "e-i64:64-i128:128-v16:16-v32:32-n16:32:64"
+target triple = "nvptx64-nvidia-cuda"
+
+define void @h() !dbg !100 {
+entry:
+  ; CHECK: remark: test.c:0:0: in artificial function 'h', artificial alloca ('%dyn_ptr.addr') for 'dyn_ptr' with static size of 8 bytes
+  %dyn_ptr.addr = alloca ptr, align 8
+  ; CHECK: remark: test.c:14:9: in artificial function 'h', alloca ('%i') for 'i' with static size of 4 bytes
+  %i = alloca i32, align 4
+  ; CHECK: remark: test.c:15:9: in artificial function 'h', alloca ('%a') for 'a' with static size of 8 bytes
+  %a = alloca [2 x i32], align 4
+  %size = load i32, ptr %i, align 4
+  ; CHECK: remark: test.c:16:9: in artificial function 'h', alloca ('%adyn') for 'adyn' with dynamic size
+  %adyn = alloca i32, i32 %size, align 4
+  ; CHECK: remark: <unknown>:0:0: in artificial function 'h', alloca ('%nodbg') without debug info with static size of 4 bytes
+  %nodbg = alloca i32, align 4
+  tail call void @llvm.dbg.declare(metadata ptr %dyn_ptr.addr, metadata !110, metadata !DIExpression()), !dbg !114
+  tail call void @llvm.dbg.declare(metadata ptr %i, metadata !120, metadata !DIExpression()), !dbg !121
+  tail call void @llvm.dbg.declare(metadata ptr %a, metadata !130, metadata !DIExpression()), !dbg !131
+  tail call void @llvm.dbg.declare(metadata ptr %adyn, metadata !140, metadata !DIExpression()), !dbg !141
+  br label %non-entry
+
+non-entry:
+  ; CHECK: remark: test.c:17:9: in artificial function 'h', alloca ('%i2') for 'i2' with static size of 4 bytes
+  %i2 = alloca i32, align 4
+  %size2 = load i32, ptr %i2, align 4
+  ; CHECK: remark: test.c:18:9: in artificial function 'h', alloca ('%adyn2') for 'adyn2' with dynamic size
+  %adyn2 = alloca i32, i32 %size, align 4
+  tail call void @llvm.dbg.declare(metadata ptr %i2, metadata !150, metadata !DIExpression()), !dbg !151
+  tail call void @llvm.dbg.declare(metadata ptr %adyn2, metadata !160, metadata !DIExpression()), !dbg !161
+  ret void
+}
+; CHECK: remark: test.c:13:0: in artificial function 'h', Allocas = 7
+; CHECK: remark: test.c:13:0: in artificial function 'h', AllocasStaticSizeSum = 28
+; CHECK: remark: test.c:13:0: in artificial function 'h', AllocasDyn = 2
+
+define void @g() !dbg !200 {
+entry:
+  ; CHECK: remark: test.c:4:7: in function 'g', alloca ('%i') for 'i' with static size of 4 bytes
+  %i = alloca i32, align 4
+  ; CHECK: remark: test.c:5:7: in function 'g', alloca ('%a') for 'a' with static size of 8 bytes
+  %a = alloca [2 x i32], align 4
+  tail call void @llvm.dbg.declare(metadata ptr %i, metadata !210, metadata !DIExpression()), !dbg !211
+  tail call void @llvm.dbg.declare(metadata ptr %a, metadata !220, metadata !DIExpression()), !dbg !221
+  ret void
+}
+; CHECK: remark: test.c:3:0: in function 'g', Allocas = 2
+; CHECK: remark: test.c:3:0: in function 'g', AllocasStaticSizeSum = 12
+; CHECK: remark: test.c:3:0: in function 'g', AllocasDyn = 0
+
+; Function Attrs: nocallback nofree nosync nounwind speculatable willreturn memory(none)
+declare void @llvm.dbg.declare(metadata, metadata, metadata) #0
+
+; uselistorder directives
+uselistorder ptr @llvm.dbg.declare, { 7, 6, 5, 4, 3, 2, 1, 0 }
+
+attributes #0 = { nocallback nofree nosync nounwind speculatable willreturn memory(none) }
+
+!llvm.module.flags = !{!0}
+!llvm.dbg.cu = !{!1}
+
+!0 = !{i32 2, !"Debug Info Version", i32 3}
+!1 = distinct !DICompileUnit(language: DW_LANG_C11, file: !2, producer: "clang version 19.0.0git", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, splitDebugInlining: false, nameTableKind: None)
+!2 = !DIFile(filename: "test.c", directory: "/tmp")
+!3 = !{null}
+!4 = !{}
+
+!10 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+
+!20 = !DIDerivedType(tag: DW_TAG_const_type, baseType: !21)
+!21 = !DIDerivedType(tag: DW_TAG_restrict_type, baseType: !22)
+!22 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: null, size: 64)
+
+!30 = !DICompositeType(tag: DW_TAG_array_type, baseType: !10, size: 64, elements: !31)
+!31 = !{!32}
+!32 = !DISubrange(count: 2)
+
+!40 = !DICompositeType(tag: DW_TAG_array_type, baseType: !10, elements: !41)
+!41 = !{!42}
+!42 = !DISubrange(count: !43)
+!43 = !DILocalVariable(name: "__vla_expr0", scope: !100, type: !10, flags: DIFlagArtificial)
+
+!100 = distinct !DISubprogram(name: "h", scope: !2, file: !2, line: 13, type: !101, scopeLine: 13, flags: DIFlagArtificial | DIFlagPrototyped, spFlags: DISPFlagLocalToUnit | DISPFlagDefinition, unit: !1, retainedNodes: !4)
+!101 = distinct !DISubroutineType(types: !3)
+
+!110 = !DILocalVariable(name: "dyn_ptr", arg: 1, scope: !100, type: !20, flags: DIFlagArtificial)
+!114 = !DILocation(line: 0, scope: !100)
+
+!120 = !DILocalVariable(name: "i", scope: !100, file: !2, line: 14, type: !10)
+!121 = !DILocation(line: 14, column: 9, scope: !100)
+
+!130 = !DILocalVariable(name: "a", scope: !100, file: !2, line: 15, type: !30)
+!131 = !DILocation(line: 15, column: 9, scope: !100)
+
+!140 = !DILocalVariable(name: "adyn", scope: !100, file: !2, line: 16, type: !40)
+!141 = !DILocation(line: 16, column: 9, scope: !100)
+
+!150 = !DILocalVariable(name: "i2", scope: !100, file: !2, line: 17, type: !10)
+!151 = !DILocation(line: 17, column: 9, scope: !100)
+
+!160 = !DILocalVariable(name: "adyn2", scope: !100, file: !2, line: 18, type: !40)
+!161 = !DILocation(line: 18, column: 9, scope: !100)
+
+!200 = distinct !DISubprogram(name: "g", scope: !2, file: !2, line: 3, type: !201, scopeLine: 3, spFlags: DISPFlagDefinition, unit: !1, retainedNodes: !4)
+!201 = !DISubroutineType(types: !3)
+
+!210 = !DILocalVariable(name: "i", scope: !200, file: !2, line: 4, type: !10)
+!211 = !DILocation(line: 4, column: 7, scope: !200)
+
+!220 = !DILocalVariable(name: "a", scope: !200, file: !2, line: 5, type: !30)
+!221 = !DILocation(line: 5, column: 7, scope: !200)
diff --git a/llvm/test/Analysis/KernelInfo/calls.ll b/llvm/test/Analysis/KernelInfo/calls.ll
new file mode 100644
index 0000000000000..6a2a5c426b78b
--- /dev/null
+++ b/llvm/test/Analysis/KernelInfo/calls.ll
@@ -0,0 +1,139 @@
+; Check info on calls.
+
+; RUN: opt -pass-remarks=kernel-info -passes=kernel-info \
+; RUN:     -disable-output %s 2>&1 | \
+; RUN:   FileCheck -match-full-lines %s
+
+target datalayout = "e-i65:64-i128:128-v16:16-v32:32-n16:32:64"
+target triple = "nvptx64-nvidia-cuda"
+
+declare void @personality()
+
+define void @h() personality ptr @personality !dbg !100 {
+entry:
+  ; CHECK: remark: test.c:16:5: in artificial function 'h_dbg', direct call, callee is '@f'
+  call void @f(), !dbg !102
+  ; CHECK: remark: test.c:17:5: in artificial function 'h_dbg', direct call to defined function, callee is 'g_dbg'
+  call void @g(), !dbg !104
+  ; CHECK: remark: test.c:18:5: in artificial function 'h_dbg', direct call to defined function, callee is artificial 'h_dbg'
+  call void @h(), !dbg !105
+  ; CHECK: remark: test.c:24:5: in artificial function 'h_dbg', direct call to inline assembly, callee is 'asm sideeffect "eieio", ""'
+  call void asm sideeffect "eieio", ""(), !dbg !111
+  %fnPtr = load ptr, ptr null, align 8
+  ; CHECK: remark: test.c:19:5: in artificial function 'h_dbg', indirect call, callee is '%fnPtr'
+  call void %fnPtr(), !dbg !106
+  ; CHECK: remark: test.c:20:5: in artificial function 'h_dbg', direct invoke, callee is '@f'
+  invoke void @f() to label %fcont unwind label %cleanup, !dbg !107
+fcont:
+  ; CHECK: remark: test.c:21:5: in artificial function 'h_dbg', direct invoke to defined function, callee is 'g_dbg'
+  invoke void @g() to label %gcont unwind label %cleanup, !dbg !108
+gcont:
+  ; CHECK: remark: test.c:22:5: in artificial function 'h_dbg', direct invoke to defined function, callee is artificial 'h_dbg'
+  invoke void @h() to label %hcont unwind label %cleanup, !dbg !109
+hcont:
+  ; CHECK: remark: test.c:25:5: in artificial function 'h_dbg', direct invoke to inline assembly, callee is 'asm sideeffect "eieio", ""'
+  invoke void asm sideeffect "eieio", ""() to label %asmcont unwind label %cleanup, !dbg !112
+asmcont:
+  ; CHECK: remark: test.c:23:5: in artificial function 'h_dbg', indirect invoke, callee is '%fnPtr'
+  invoke void %fnPtr() to label %end unwind label %cleanup, !dbg !110
+cleanup:
+  %ll = landingpad { ptr, i32 }
+  cleanup
+  br label %end
+end:
+  ret void
+}
+; CHECK: remark: test.c:13:0: in artificial function 'h_dbg', DirectCalls = 8
+; CHECK: remark: test.c:13:0: in artificial function 'h_dbg', IndirectCalls = 2
+; CHECK: remark: test.c:13:0: in artificial function 'h_dbg', DirectCallsToDefinedFunctions = 4
+; CHECK: remark: test.c:13:0: in artificial function 'h_dbg', InlineAssemblyCalls = 2
+; CHECK: remark: test.c:13:0: in artificial function 'h_dbg', Invokes = 5
+
+declare void @f()
+
+define void @g() personality ptr @personality !dbg !200 {
+entry:
+  ; CHECK: remark: test.c:6:3: in function 'g_dbg', direct call, callee is '@f'
+  call void @f(), !dbg !202
+  ; CHECK: remark: test.c:7:3: in function 'g_dbg', direct call to defined function, callee is 'g_dbg'
+  call void @g(), !dbg !203
+  ; CHECK: remark: test.c:8:3: in function 'g_dbg', direct call to defined function, callee is artificial 'h_dbg'
+  call void @h(), !dbg !204
+  ; CHECK: remark: test.c:14:3: in function 'g_dbg', direct call to inline assembly, callee is 'asm sideeffect "eieio", ""'
+  call void asm sideeffect "eieio", ""(), !dbg !210
+  %fnPtr = load ptr, ptr null, align 8
+  ; CHECK: remark: test.c:9:3: in function 'g_dbg', indirect call, callee is '%fnPtr'
+  call void %fnPtr(), !dbg !205
+  ; CHECK: remark: test.c:10:3: in function 'g_dbg', direct invoke, callee is '@f'
+  invoke void @f() to label %fcont unwind label %cleanup, !dbg !206
+fcont:
+  ; CHECK: remark: test.c:11:3: in function 'g_dbg', direct invoke to defined function, callee is 'g_dbg'
+  invoke void @g() to label %gcont unwind label %cleanup, !dbg !207
+gcont:
+  ; CHECK: remark: test.c:12:3: in function 'g_dbg', direct invoke to defined function, callee is artificial 'h_dbg'
+  invoke void @h() to label %hcont unwind label %cleanup, !dbg !208
+hcont:
+  ; CHECK: remark: test.c:15:3: in function 'g_dbg', direct invoke to inline assembly, callee is 'asm sideeffect "eieio", ""'
+  invoke void asm sideeffect "eieio", ""() to label %asmcont unwind label %cleanup, !dbg !211
+asmcont:
+  ; CHECK: remark: test.c:13:3: in function 'g_dbg', indirect invoke, callee is '%fnPtr'
+  invoke void %fnPtr() to label %end unwind label %cleanup, !dbg !209
+cleanup:
+  %ll = landingpad { ptr, i32 }
+  cleanup
+  br label %end
+end:
+  ret void
+}
+; CHECK: remark: test.c:3:0: in function 'g_dbg', DirectCalls = 8
+; CHECK: remark: test.c:3:0: in function 'g_dbg', IndirectCalls = 2
+; CHECK: remark: test.c:3:0: in function 'g_dbg', DirectCallsToDefinedFunctions = 4
+; CHECK: remark: test.c:3:0: in function 'g_dbg', InlineAssemblyCalls = 2
+; CHECK: remark: test.c:3:0: in function 'g_dbg', Invokes = 5
+
+define void @i() {
+  ; CHECK: remark: <unknown>:0:0: in function '@i', direct call, callee is '@f'
+  call void @f()
+  ret void
+}
+; CHECK: remark: <unknown>:0:0: in function '@i', DirectCalls = 1
+; CHECK: remark: <unknown>:0:0: in function '@i', IndirectCalls = 0
+; CHECK: remark: <unknown>:0:0: in function '@i', DirectCallsToDefinedFunctions = 0
+; CHECK: remark: <unknown>:0:0: in function '@i', InlineAssemblyCalls = 0
+; CHECK: remark: <unknown>:0:0: in function '@i', Invokes = 0
+
+!llvm.module.flags = !{!0}
+!llvm.dbg.cu = !{!1}
+
+!0 = !{i32 2, !"Debug Info Version", i32 3}
+!1 = distinct !DICompileUnit(language: DW_LANG_C11, file: !2, producer: "clang version 19.0.0git", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, splitDebugInlining: false, nameTableKind: None)
+!2 = !DIFile(filename: "test.c", directory: "/tmp")
+!3 = !{null}
+!4 = !{}
+
+!100 = distinct !DISubprogram(name: "h_dbg", scope: !2, file: !2, line: 13, type: !101, scopeLine: 13, flags: DIFlagArtificial | DIFlagPrototyped, spFlags: DISPFlagLocalToUnit | DISPFlagDefinition, unit: !1, retainedNodes: !4)
+!101 = distinct !DISubroutineType(types: !3)
+!102 = !DILocation(line: 16, column: 5, scope: !103)
+!103 = distinct !DILexicalBlock(scope: !100, file: !2, line: 13, column: 3)
+!104 = !DILocation(line: 17, column: 5, scope: !103)
+!105 = !DILocation(line: 18, column: 5, scope: !103)
+!106 = !DILocation(line: 19, column: 5, scope: !103)
+!107 = !DILocation(line: 20, column: 5, scope: !103)
+!108 = !DILocation(line: 21, column: 5, scope: !103)
+!109 = !DILocation(line: 22, column: 5, scope: !103)
+!110 = !DILocation(line: 23, column: 5, scope: !103)
+!111 = !DILocation(line: 24, column: 5, scope: !103)
+!112 = !DILocation(line: 25, column: 5, scope: !103)
+
+!200 = distinct !DISubprogram(name: "g_dbg", scope: !2, file: !2, line: 3, type: !201, scopeLine: 3, spFlags: DISPFlagDefinition, unit: !1, retainedNodes: !4)
+!201 = !DISubroutineType(types: !3)
+!202 = !DILocation(line: 6, column: 3, scope: !200)
+!203 = !DILocation(line: 7, column: 3, scope: !200)
+!204 = !DILocation(line: 8, column: 3, scope: !200)
+!205 = !DILocation(line: 9, column: 3, scope: !200)
+!206 = !DILocation(line: 10, column: 3, scope: !200)
+!207 = !DILocation(line: 11, column: 3, scope: !200)
+!208 = !DILocation(line: 12, column: 3, scope: !200)
+!209 = !DILocation(line: 13, column: 3, scope: !200)
+!210 = !DILocation(line: 14, column: 3, scope: !200)
+!211 = !DILocation(line: 15, column: 3, scope: !200)
diff --git a/llvm/test/Analysis/KernelInfo/enable-kernel-info/Inputs/test.ll b/llvm/test/Analysis/KernelInfo/enable-kernel-info/Inputs/test.ll
new file mode 100644
index 0000000000000..461544e44d538
--- /dev/null
+++ b/llvm/test/Analysis/KernelInfo/enable-kernel-info/Inputs/test.ll
@@ -0,0 +1,22 @@
+; CHECK: remark: test.c:10:0: in artificial function 'test', omp_target_num_teams = 100
+; NONE-NOT: remark:
+define void @test() #0 !dbg !5 {
+entry:
+  ret void
+}
+
+attributes #0 = {
+  "omp_target_num_teams"="100"
+}
+
+!llvm.module.flags = !{!0}
+!llvm.dbg.cu = !{!1}
+!nvvm.annotations = !{!6}
+
+!0 = !{i32 2, !"Debug Info Version", i32 3}
+!1 = distinct !DICompileUnit(language: DW_LANG_C11, file: !2, producer: "clang version 19.0.0git", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, splitDebugInlining: false, nameTableKind: None)
+!2 = !DIFile(filename: "test.c", directory: "/tmp")
+!3 = !{}
+!4 = !DISubroutineType(types: !3)
+!5 = distinct !DISubprogram(name: "test", scope: !2, file: !2, line: 10, type: !4, scopeLine: 10, flags: DIFlagArtificial | DIFlagPrototyped, spFlags: DISPFlagLocalToUnit | DISPFlagDefinition, unit: !1, retainedNodes: !3)
+!6 = distinct !{ptr null, !"kernel", i32 1}
diff --git a/llvm/test/Analysis/KernelInfo/enable-kernel-info/amdgpu.test b/llvm/test/Analysis/KernelInfo/enable-kernel-info/amdgpu.test
new file mode 100644
index 0000000000000..e969eabfe7cd8
--- /dev/null
+++ b/llvm/test/Analysis/KernelInfo/enable-kernel-info/amdgpu.test
@@ -0,0 +1,18 @@
+; Check when kernel-info is enabled in the AMD GPU target backend.
+
+; REQUIRES: amdgpu-registered-target
+
+; DEFINE: %{opt} = opt -disable-output %S/Inputs/test.ll \
+; DEFINE:              -mtriple="amdgcn-amd-amdhsa" 2>&1
+; DEFINE: %{fcheck-on} = FileCheck -match-full-lines %S/Inputs/test.ll
+; DEFINE: %{fcheck-off} = FileCheck -allow-empty -check-prefixes=NONE \
+; DEFINE:                 %S/Inputs/test.ll
+
+; By default, kernel-info is in the LTO pipeline.  To see output, the LTO
+; pipeline must run, -no-kernel-info-end-lto must not be specified, and remarks
+; must be enabled.
+; RUN: %{opt} -passes='lto<O2>' -pass-remarks=kernel-info | %{fcheck-on}
+; RUN: %{opt} -passes='default<O2>' -pass-remarks=kernel-info | %{fcheck-off}
+; RUN: %{opt} -passes='lto<O2>' -pass-remarks=kernel-info \
+; RUN:        -no-kernel-info-end-lto | %{fcheck-off}
+; RUN: %{opt} -passes='lto<O2>' | %{fcheck-off}
diff --git a/llvm/test/Analysis/KernelInfo/enable-kernel-info/nvptx.test b/llvm/test/Analysis/KernelInfo/enable-kernel-info/nvptx.test
new file mode 100644
index 0000000000000..65249b4d92e34
--- /dev/null
+++ b/llvm/test/Analysis/KernelInfo/enable-kernel-info/nvptx.test
@@ -0,0 +1,18 @@
+; Check when kernel-info is enabled in the NVPTX target backend.
+
+; REQUIRES: nvptx-registered-target
+
+; DEFINE: %{opt} = opt -disable-output %S/Inputs/test.ll \
+; DEFINE:              -mtriple="nvptx64-nvidia-cuda" 2>&1
+; DEFINE: %{fcheck-on} = FileCheck -match-full-lines %S/Inputs/test.ll
+; DEFINE: %{fcheck-off} = FileCheck -allow-empty -check-prefixes=NONE \
+; DEFINE:                 %S/Inputs/test.ll
+
+; By default, kernel-info is in the LTO pipeline.  To see output, the LTO
+; pipeline must run, -no-kernel-info-end-lto must not be specified, and remarks
+; must be enabled.
+; RUN: %{opt} -passes='lto<O2>' -pass-remarks=kernel-info | %{fcheck-on}
+; RUN: %{opt} -passes='default<O2>' -pass-remarks=kernel-info | %{fcheck-off}
+; RUN: %{opt} -passes='lto<O2>' -pass-remarks=kernel-info \
+; RUN:        -no-kernel-info-end-lto | %{fcheck-off}
+; RUN: %{opt} -passes='lto<O2>' | %{fcheck-off}
diff --git a/llvm/test/Analysis/KernelInfo/flat-addrspace/Inputs/test.ll b/llvm/test/Analysis/KernelInfo/flat-addrspace/Inputs/test.ll
new file mode 100644
index 0000000000000..b54c3a18f3e70
--- /dev/null
+++ b/llvm/test/Analysis/KernelInfo/flat-addrspace/Inputs/test.ll
@@ -0,0 +1,143 @@
+define void @f() !dbg !3 {
+entry:
+  ; load: check remarks for both unnamed and named values.
+  ; CHECK: remark: test.c:3:11: in function 'f', 'load' instruction ('%0') accesses memory in flat address space
+  %0 = load i32, ptr null, align 4, !dbg !6
+  ; CHECK: remark: test.c:3:11: in function 'f', 'load' instruction ('%load') accesses memory in flat address space
+  %load = load i32, ptr null, align 4, !dbg !6
+  ; CHECK: remark: test.c:3:11: in function 'f', 'load' instruction ('%load0') accesses memory in flat address space
+  %load0 = load i32, ptr addrspace(0) null, align 4, !dbg !6
+  %load1 = load i32, ptr addrspace(1) null, align 4, !dbg !6
+  %load2 = load i32, ptr addrspace(2) null, align 4, !dbg !6
+
+  ; store
+  ; CHECK: remark: test.c:4:6: in function 'f', 'store' instruction accesses memory in flat address space
+  store i32 0, ptr null, align 4, !dbg !7
+  ; CHECK: remark: test.c:4:6: in function 'f', 'store' instruction accesses memory in flat address space
+  store i32 0, ptr addrspace(0) null, align 4, !dbg !7
+  store i32 0, ptr addrspace(1) null, align 4, !dbg !7
+  store i32 0, ptr addrspace(8) null, align 4, !dbg !7
+
+  ; atomicrmw
+  ; CHECK: remark: test.c:5:1: in function 'f', 'atomicrmw' instruction ('%[[#]]') accesses memory in flat address space
+  atomicrmw xchg ptr null, i32 10 seq_cst, !dbg !8
+  ; CHECK: remark: test.c:5:1: in function 'f', 'atomicrmw' instruction ('%[[#]]') accesses memory in flat address space
+  atomicrmw add ptr addrspace(0) null, i32 10 seq_cst, !dbg !8
+  atomicrmw xchg ptr addrspace(1) null, i32 10 seq_cst, !dbg !8
+  atomicrmw add ptr addrspace(37) null, i32 10 seq_cst, !dbg !8
+
+  ; cmpxchg
+  ; CHECK: remark: test.c:6:2: in function 'f', 'cmpxchg' instruction ('%[[#]]') accesses memory in flat address space
+  cmpxchg ptr null, i32 0, i32 1 acq_rel monotonic, !dbg !9
+  ; CHECK: remark: test.c:6:2: in function 'f', 'cmpxchg' instruction ('%[[#]]') accesses memory in flat address space
+  cmpxchg ptr addrspace(0) null, i32 0, i32 1 acq_rel monotonic, !dbg !9
+  cmpxchg ptr addrspace(1) null, i32 0, i32 1 acq_rel monotonic, !dbg !9
+  cmpxchg ptr addrspace(934) null, i32 0, i32 1 acq_rel monotonic, !dbg !9
+
+  ; llvm.memcpy
+  ; CHECK: remark: test.c:7:3: in function 'f', 'llvm.memcpy.p0.p1.i64' call accesses memory in flat address space
+  call void @llvm.memcpy.p0.p1.i64(ptr align 4 null, ptr addrspace(1) align 4 null, i64 10, i1 false), !dbg !10
+  ; CHECK: remark: test.c:7:3: in function 'f', 'llvm.memcpy.p0.p1.i64' call accesses memory in flat address space
+  call void @llvm.memcpy.p0.p1.i64(ptr addrspace(0) align 4 null, ptr addrspace(1) align 4 null, i64 10, i1 false), !dbg !10
+  call void @llvm.memcpy.p1.p1.i64(ptr addrspace(1) align 4 null, ptr addrspace(1) align 4 null, i64 10, i1 false), !dbg !10
+  call void @llvm.memcpy.p3.p1.i64(ptr addrspace(3) align 4 null, ptr addrspace(1) align 4 null, i64 10, i1 false), !dbg !10
+  ; CHECK: remark: test.c:7:3: in function 'f', 'llvm.memcpy.p1.p0.i64' call accesses memory in flat address space
+  call void @llvm.memcpy.p1.p0.i64(ptr addrspace(1) align 4 null, ptr align 4 null, i64 10, i1 false), !dbg !10
+  ; CHECK: remark: test.c:7:3: in function 'f', 'llvm.memcpy.p1.p0.i64' call accesses memory in flat address space
+  call void @llvm.memcpy.p1.p0.i64(ptr addrspace(1) align 4 null, ptr addrspace(0) align 4 null, i64 10, i1 false), !dbg !10
+  call void @llvm.memcpy.p1.p1.i64(ptr addrspace(1) align 4 null, ptr addrspace(1) align 4 null, i64 10, i1 false), !dbg !10
+  call void @llvm.memcpy.p1.p4.i64(ptr addrspace(1) align 4 null, ptr addrspace(4) align 4 null, i64 10, i1 false), !dbg !10
+  ; CHECK: remark: test.c:7:3: in function 'f', 'llvm.memcpy.p0.p0.i64' call accesses memory in flat address space
+  call void @llvm.memcpy.p0.p0.i64(ptr align 4 null, ptr align 4 null, i64 10, i1 false), !dbg !10
+  ; CHECK: remark: test.c:7:3: in function 'f', 'llvm.memcpy.p0.p0.i64' call accesses memory in flat address space
+  call void @llvm.memcpy.p0.p0.i64(ptr addrspace(0) align 4 null, ptr addrspace(0) align 4 null, i64 10, i1 false), !dbg !10
+
+  ; llvm.memcpy.inline
+  ; CHECK: remark: test.c:7:3: in function 'f', 'llvm.memcpy.inline.p0.p0.i64' call accesses memory in flat address space
+  call void @llvm.memcpy.inline.p0.p0.i64(ptr addrspace(0) align 4 null, ptr addrspace(0) align 4 null, i64 10, i1 false), !dbg !10
+  ; CHECK: remark: test.c:7:3: in function 'f', 'llvm.memcpy.inline.p0.p1.i64' call accesses memory in flat address space
+  call void @llvm.memcpy.inline.p0.p1.i64(ptr addrspace(0) align 4 null, ptr addrspace(1) align 4 null, i64 10, i1 false), !dbg !10
+  ; CHECK: remark: test.c:7:3: in function 'f', 'llvm.memcpy.inline.p1.p0.i64' call accesses memory in flat address space
+  call void @llvm.memcpy.inline.p1.p0.i64(ptr addrspace(1) align 4 null, ptr addrspace(0) align 4 null, i64 10, i1 false), !dbg !10
+  call void @llvm.memcpy.inline.p1.p1.i64(ptr addrspace(1) align 4 null, ptr addrspace(1) align 4 null, i64 10, i1 false), !dbg !10
+
+  ; llvm.memcpy.element.unordered.atomic
+  ; CHECK: remark: test.c:7:3: in function 'f', 'llvm.memcpy.element.unordered.atomic.p0.p0.i64' call accesses memory in flat address space
+  call void @llvm.memcpy.element.unordered.atomic.p0.p0.i64(ptr addrspace(0) align 4 null, ptr addrspace(0) align 4 null, i64 10, i32 4), !dbg !10
+  ; CHECK: remark: test.c:7:3: in function 'f', 'llvm.memcpy.element.unordered.atomic.p0.p1.i64' call accesses memory in flat address space
+  call void @llvm.memcpy.element.unordered.atomic.p0.p1.i64(ptr addrspace(0) align 4 null, ptr addrspace(1) align 4 null, i64 10, i32 4), !dbg !10
+  ; CHECK: remark: test.c:7:3: in function 'f', 'llvm.memcpy.element.unordered.atomic.p1.p0.i64' call accesses memory in flat address space
+  call void @llvm.memcpy.element.unordered.atomic.p1.p0.i64(ptr addrspace(1) align 4 null, ptr addrspace(0) align 4 null, i64 10, i32 4), !dbg !10
+  call void @llvm.memcpy.element.unordered.atomic.p1.p1.i64(ptr addrspace(1) align 4 null, ptr addrspace(1) align 4 null, i64 10, i32 4), !dbg !10
+
+  ; llvm.memmove
+  ; CHECK: remark: test.c:8:4: in function 'f', 'llvm.memmove.p0.p1.i64' call accesses memory in flat address space
+  call void @llvm.memmove.p0.p1.i64(ptr align 4 null, ptr addrspace(1) align 4 null, i64 10, i1 false), !dbg !11
+  ; CHECK: remark: test.c:8:4: in function 'f', 'llvm.memmove.p0.p1.i64' call accesses memory in flat address space
+  call void @llvm.memmove.p0.p1.i64(ptr addrspace(0) align 4 null, ptr addrspace(1) align 4 null, i64 10, i1 false), !dbg !11
+  call void @llvm.memmove.p1.p1.i64(ptr addrspace(1) align 4 null, ptr addrspace(1) align 4 null, i64 10, i1 false), !dbg !11
+  call void @llvm.memmove.p3.p1.i64(ptr addrspace(3) align 4 null, ptr addrspace(1) align 4 null, i64 10, i1 false), !dbg !11
+  ; CHECK: remark: test.c:8:4: in function 'f', 'llvm.memmove.p1.p0.i64' call accesses memory in flat address space
+  call void @llvm.memmove.p1.p0.i64(ptr addrspace(1) align 4 null, ptr align 4 null, i64 10, i1 false), !dbg !11
+  ; CHECK: remark: test.c:8:4: in function 'f', 'llvm.memmove.p1.p0.i64' call accesses memory in flat address space
+  call void @llvm.memmove.p1.p0.i64(ptr addrspace(1) align 4 null, ptr addrspace(0) align 4 null, i64 10, i1 false), !dbg !11
+  call void @llvm.memmove.p1.p1.i64(ptr addrspace(1) align 4 null, ptr addrspace(1) align 4 null, i64 10, i1 false), !dbg !11
+  call void @llvm.memmove.p1.p4.i64(ptr addrspace(1) align 4 null, ptr addrspace(4) align 4 null, i64 10, i1 false), !dbg !11
+  ; CHECK: remark: test.c:8:4: in function 'f', 'llvm.memmove.p0.p0.i64' call accesses memory in flat address space
+  call void @llvm.memmove.p0.p0.i64(ptr align 4 null, ptr align 4 null, i64 10, i1 false), !dbg !11
+  ; CHECK: remark: test.c:8:4: in function 'f', 'llvm.memmove.p0.p0.i64' call accesses memory in flat address space
+  call void @llvm.memmove.p0.p0.i64(ptr addrspace(0) align 4 null, ptr addrspace(0) align 4 null, i64 10, i1 false), !dbg !11
+
+  ; llvm.memmove.element.unordered.atomic
+  ; CHECK: remark: test.c:8:4: in function 'f', 'llvm.memmove.element.unordered.atomic.p0.p0.i64' call accesses memory in flat address space
+  call void @llvm.memmove.element.unordered.atomic.p0.p0.i64(ptr addrspace(0) align 4 null, ptr addrspace(0) align 4 null, i64 10, i32 4), !dbg !11
+  ; CHECK: remark: test.c:8:4: in function 'f', 'llvm.memmove.element.unordered.atomic.p0.p1.i64' call accesses memory in flat address space
+  call void @llvm.memmove.element.unordered.atomic.p0.p1.i64(ptr addrspace(0) align 4 null, ptr addrspace(1) align 4 null, i64 10, i32 4), !dbg !11
+  ; CHECK: remark: test.c:8:4: in function 'f', 'llvm.memmove.element.unordered.atomic.p1.p0.i64' call accesses memory in flat address space
+  call void @llvm.memmove.element.unordered.atomic.p1.p0.i64(ptr addrspace(1) align 4 null, ptr addrspace(0) align 4 null, i64 10, i32 4), !dbg !11
+  call void @llvm.memmove.element.unordered.atomic.p1.p1.i64(ptr addrspace(1) align 4 null, ptr addrspace(1) align 4 null, i64 10, i32 4), !dbg !11
+
+  ; llvm.memset
+  ; CHECK: remark: test.c:9:5: in function 'f', 'llvm.memset.p0.i64' call accesses memory in flat address space
+  call void @llvm.memset.p0.i64(ptr align 4 null, i8 0, i64 10, i1 false), !dbg !12
+  ; CHECK: remark: test.c:9:5: in function 'f', 'llvm.memset.p0.i64' call accesses memory in flat address space
+  call void @llvm.memset.p0.i64(ptr addrspace(0) align 4 null, i8 0, i64 10, i1 false), !dbg !12
+  call void @llvm.memset.p1.i64(ptr addrspace(1) align 4 null, i8 0, i64 10, i1 false), !dbg !12
+  call void @llvm.memset.p3.i64(ptr addrspace(3) align 4 null, i8 0, i64 10, i1 false), !dbg !12
+
+  ; llvm.memset.inline
+  ; CHECK: remark: test.c:9:5: in function 'f', 'llvm.memset.inline.p0.i64' call accesses memory in flat address space
+  call void @llvm.memset.inline.p0.i64(ptr align 4 null, i8 0, i64 10, i1 false), !dbg !12
+  ; CHECK: remark: test.c:9:5: in function 'f', 'llvm.memset.inline.p0.i64' call accesses memory in flat address space
+  call void @llvm.memset.inline.p0.i64(ptr addrspace(0) align 4 null, i8 0, i64 10, i1 false), !dbg !12
+  call void @llvm.memset.inline.p1.i64(ptr addrspace(1) align 4 null, i8 0, i64 10, i1 false), !dbg !12
+  call void @llvm.memset.inline.p3.i64(ptr addrspace(3) align 4 null, i8 0, i64 10, i1 false), !dbg !12
+
+  ; llvm.memset.element.unordered.atomic
+  ; CHECK: remark: test.c:9:5: in function 'f', 'llvm.memset.element.unordered.atomic.p0.i64' call accesses memory in flat address space
+  call void @llvm.memset.element.unordered.atomic.p0.i64(ptr align 4 null, i8 0, i64 10, i32 4), !dbg !12
+  ; CHECK: remark: test.c:9:5: in function 'f', 'llvm.memset.element.unordered.atomic.p0.i64' call accesses memory in flat address space
+  call void @llvm.memset.element.unordered.atomic.p0.i64(ptr addrspace(0) align 4 null, i8 0, i64 10, i32 4), !dbg !12
+  call void @llvm.memset.element.unordered.atomic.p1.i64(ptr addrspace(1) align 4 null, i8 0, i64 10, i32 4), !dbg !12
+  call void @llvm.memset.element.unordered.atomic.p3.i64(ptr addrspace(3) align 4 null, i8 0, i64 10, i32 4), !dbg !12
+
+  ret void
+}
+; CHECK: remark: test.c:2:0: in function 'f', FlatAddrspaceAccesses = 36
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!2}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C11, file: !1, producer: "clang version 19.0.0git", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, splitDebugInlining: false, nameTableKind: None)
+!1 = !DIFile(filename: "test.c", directory: "/tmp")
+!2 = !{i32 2, !"Debug Info Version", i32 3}
+!3 = distinct !DISubprogram(name: "f", scope: !1, file: !1, line: 2, type: !4, scopeLine: 2, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition, unit: !0, retainedNodes: !5)
+!4 = !DISubroutineType(types: !5)
+!5 = !{}
+!6 = !DILocation(line: 3, column: 11, scope: !3)
+!7 = !DILocation(line: 4, column: 6, scope: !3)
+!8 = !DILocation(line: 5, column: 1, scope: !3)
+!9 = !DILocation(line: 6, column: 2, scope: !3)
+!10 = !DILocation(line: 7, column: 3, scope: !3)
+!11 = !DILocation(line: 8, column: 4, scope: !3)
+!12 = !DILocation(line: 9, column: 5, scope: !3)
diff --git a/llvm/test/Analysis/KernelInfo/flat-addrspace/amdgpu.test b/llvm/test/Analysis/KernelInfo/flat-addrspace/amdgpu.test
new file mode 100644
index 0000000000000..7447dcf51cc89
--- /dev/null
+++ b/llvm/test/Analysis/KernelInfo/flat-addrspace/amdgpu.test
@@ -0,0 +1,12 @@
+; Check info on flat address space memory accesses when the target is amdgpu.
+;
+; The target matters because kernel-info calls
+; TargetTransformInfo::getFlatAddressSpace to select the flat address space.
+
+; REQUIRES: amdgpu-registered-target
+
+; RUN: opt -pass-remarks=kernel-info -passes=kernel-info \
+; RUN:     -mtriple="amdgcn-amd-amdhsa" \
+; RUN:     -disable-output %S/Inputs/test.ll 2>&1 | \
+; RUN:   FileCheck -match-full-lines -implicit-check-not='flat address space' \
+; RUN:       %S/Inputs/test.ll
diff --git a/llvm/test/Analysis/KernelInfo/flat-addrspace/nvptx.test b/llvm/test/Analysis/KernelInfo/flat-addrspace/nvptx.test
new file mode 100644
index 0000000000000..02321c19e022d
--- /dev/null
+++ b/llvm/test/Analysis/KernelInfo/flat-addrspace/nvptx.test
@@ -0,0 +1,12 @@
+; Check info on flat address space memory accesses when the target is nvptx.
+;
+; The target matters because kernel-info calls
+; TargetTransformInfo::getFlatAddressSpace to select the flat address space.
+
+; REQUIRES: nvptx-registered-target
+
+; RUN: opt -pass-remarks=kernel-info -passes=kernel-info \
+; RUN:     -mtriple="nvptx64-nvidia-cuda" \
+; RUN:     -disable-output %S/Inputs/test.ll 2>&1 | \
+; RUN:   FileCheck -match-full-lines -implicit-check-not='flat address space' \
+; RUN:       %S/Inputs/test.ll
diff --git a/llvm/test/Analysis/KernelInfo/launch-bounds/amdgpu.ll b/llvm/test/Analysis/KernelInfo/launch-bounds/amdgpu.ll
new file mode 100644
index 0000000000000..7fbdb923d8800
--- /dev/null
+++ b/llvm/test/Analysis/KernelInfo/launch-bounds/amdgpu.ll
@@ -0,0 +1,83 @@
+; Check info on launch bounds for AMD GPU.
+
+; REQUIRES: amdgpu-registered-target
+
+; RUN: opt -pass-remarks=kernel-info -passes=kernel-info \
+; RUN:     -disable-output %s 2>&1 | \
+; RUN:   FileCheck -match-full-lines %s
+
+target datalayout = "e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32-p7:160:256:256:32-p8:128:128-p9:192:256:256:32-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5-G1-ni:7:8:9"
+target triple = "amdgcn-amd-amdhsa"
+
+; CHECK: remark: test.c:10:0: in artificial function 'all', omp_target_num_teams = 100
+; CHECK: remark: test.c:10:0: in artificial function 'all', omp_target_thread_limit = 101
+; CHECK: remark: test.c:10:0: in artificial function 'all', amdgpu-max-num-workgroups[0] = 200
+; CHECK: remark: test.c:10:0: in artificial function 'all', amdgpu-max-num-workgroups[1] = 201
+; CHECK: remark: test.c:10:0: in artificial function 'all', amdgpu-max-num-workgroups[2] = 202
+; CHECK: remark: test.c:10:0: in artificial function 'all', amdgpu-flat-work-group-size[0] = 210
+; CHECK: remark: test.c:10:0: in artificial function 'all', amdgpu-flat-work-group-size[1] = 211
+; CHECK: remark: test.c:10:0: in artificial function 'all', amdgpu-waves-per-eu[0] = 2
+; CHECK: remark: test.c:10:0: in artificial function 'all', amdgpu-waves-per-eu[1] = 9
+define void @all() #0 !dbg !5 {
+entry:
+  ret void
+}
+
+; CHECK-NOT: remark: test.c:11:0: in function 'none', omp_target_num_teams = {{.*}}
+; CHECK-NOT: remark: test.c:11:0: in function 'none', omp_target_thread_limit = {{.*}}
+; CHECK: remark: test.c:11:0: in function 'none', amdgpu-max-num-workgroups[0] = 4294967295
+; CHECK: remark: test.c:11:0: in function 'none', amdgpu-max-num-workgroups[1] = 4294967295
+; CHECK: remark: test.c:11:0: in function 'none', amdgpu-max-num-workgroups[2] = 4294967295
+; CHECK: remark: test.c:11:0: in function 'none', amdgpu-flat-work-group-size[0] = 1
+; CHECK: remark: test.c:11:0: in function 'none', amdgpu-flat-work-group-size[1] = 1024
+; CHECK: remark: test.c:11:0: in function 'none', amdgpu-waves-per-eu[0] = 4
+; CHECK: remark: test.c:11:0: in function 'none', amdgpu-waves-per-eu[1] = 10
+define void @none() !dbg !6 {
+entry:
+  ret void
+}
+
+; CHECK: remark: test.c:12:0: in function 'bogus', omp_target_num_teams = 987654321
+; CHECK: remark: test.c:12:0: in function 'bogus', omp_target_thread_limit = 987654321
+; CHECK: remark: test.c:12:0: in function 'bogus', amdgpu-max-num-workgroups[0] = 987654321
+; CHECK: remark: test.c:12:0: in function 'bogus', amdgpu-max-num-workgroups[1] = 987654321
+; CHECK: remark: test.c:12:0: in function 'bogus', amdgpu-max-num-workgroups[2] = 987654321
+; CHECK: remark: test.c:12:0: in function 'bogus', amdgpu-flat-work-group-size[0] = 1
+; CHECK: remark: test.c:12:0: in function 'bogus', amdgpu-flat-work-group-size[1] = 1024
+; CHECK: remark: test.c:12:0: in function 'bogus', amdgpu-waves-per-eu[0] = 4
+; CHECK: remark: test.c:12:0: in function 'bogus', amdgpu-waves-per-eu[1] = 10
+define void @bogus() #1 !dbg !7 {
+entry:
+  ret void
+}
+
+attributes #0 = {
+  "omp_target_num_teams"="100"
+  "omp_target_thread_limit"="101"
+  "amdgpu-max-num-workgroups"="200,201,202"
+  "amdgpu-flat-work-group-size"="210,211"
+  "amdgpu-waves-per-eu"="2,9"
+}
+
+; We choose values that are small enough to parse successfully but that are
+; impossibly large.  For values that are validated, we check that they are
+; overridden with realistic values.
+attributes #1 = {
+  "omp_target_num_teams"="987654321"
+  "omp_target_thread_limit"="987654321"
+  "amdgpu-max-num-workgroups"="987654321,987654321,987654321"
+  "amdgpu-flat-work-group-size"="987654321,987654321"
+  "amdgpu-waves-per-eu"="987654321,987654321"
+}
+
+!llvm.module.flags = !{!0}
+!llvm.dbg.cu = !{!1}
+
+!0 = !{i32 2, !"Debug Info Version", i32 3}
+!1 = distinct !DICompileUnit(language: DW_LANG_C11, file: !2, producer: "clang version 19.0.0git", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, splitDebugInlining: false, nameTableKind: None)
+!2 = !DIFile(filename: "test.c", directory: "/tmp")
+!3 = !{}
+!4 = !DISubroutineType(types: !3)
+!5 = distinct !DISubprogram(name: "all", scope: !2, file: !2, line: 10, type: !4, scopeLine: 10, flags: DIFlagArtificial | DIFlagPrototyped, spFlags: DISPFlagLocalToUnit | DISPFlagDefinition, unit: !1, retainedNodes: !3)
+!6 = distinct !DISubprogram(name: "none", scope: !2, file: !2, line: 11, type: !4, scopeLine: 11, flags: DIFlagPrototyped, spFlags: DISPFlagLocalToUnit | DISPFlagDefinition, unit: !1, retainedNodes: !3)
+!7 = distinct !DISubprogram(name: "bogus", scope: !2, file: !2, line: 12, type: !4, scopeLine: 12, flags: DIFlagPrototyped, spFlags: DISPFlagLocalToUnit | DISPFlagDefinition, unit: !1, retainedNodes: !3)
diff --git a/llvm/test/Analysis/KernelInfo/launch-bounds/nvptx.ll b/llvm/test/Analysis/KernelInfo/launch-bounds/nvptx.ll
new file mode 100644
index 0000000000000..7a055c7152ec8
--- /dev/null
+++ b/llvm/test/Analysis/KernelInfo/launch-bounds/nvptx.ll
@@ -0,0 +1,42 @@
+; Check info on launch bounds for NVPTX.
+
+; REQUIRES: nvptx-registered-target
+
+; RUN: opt -pass-remarks=kernel-info -passes=kernel-info \
+; RUN:     -disable-output %s 2>&1 | \
+; RUN:   FileCheck -match-full-lines %s
+
+target datalayout = "e-i64:64-i128:128-v16:16-v32:32-n16:32:64"
+target triple = "nvptx64-nvidia-cuda"
+
+; CHECK: remark: test.c:10:0: in artificial function 'test', omp_target_num_teams = 100
+; CHECK: remark: test.c:10:0: in artificial function 'test', omp_target_thread_limit = 101
+; CHECK: remark: test.c:10:0: in artificial function 'test', maxclusterrank = 200
+; CHECK: remark: test.c:10:0: in artificial function 'test', maxntidx = 210
+; CHECK: remark: test.c:10:0: in artificial function 'test', maxntidy = 211
+; CHECK: remark: test.c:10:0: in artificial function 'test', maxntidz = 212
+define void @test() #0 !dbg !5 {
+entry:
+  ret void
+}
+
+attributes #0 = {
+  "omp_target_num_teams"="100"
+  "omp_target_thread_limit"="101"
+}
+
+!llvm.module.flags = !{!0}
+!llvm.dbg.cu = !{!1}
+!nvvm.annotations = !{!6, !7, !8, !9, !10}
+
+!0 = !{i32 2, !"Debug Info Version", i32 3}
+!1 = distinct !DICompileUnit(language: DW_LANG_C11, file: !2, producer: "clang version 19.0.0git", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, splitDebugInlining: false, nameTableKind: None)
+!2 = !DIFile(filename: "test.c", directory: "/tmp")
+!3 = !{}
+!4 = !DISubroutineType(types: !3)
+!5 = distinct !DISubprogram(name: "test", scope: !2, file: !2, line: 10, type: !4, scopeLine: 10, flags: DIFlagArtificial | DIFlagPrototyped, spFlags: DISPFlagLocalToUnit | DISPFlagDefinition, unit: !1, retainedNodes: !3)
+!6 = !{ptr @test, !"maxclusterrank", i32 200}
+!7 = !{ptr @test, !"maxntidx", i32 210}
+!8 = !{ptr @test, !"maxntidy", i32 211}
+!9 = !{ptr @test, !"maxntidz", i32 212}
+!10 = distinct !{ptr null, !"kernel", i32 1}
diff --git a/llvm/test/Analysis/KernelInfo/linkage.ll b/llvm/test/Analysis/KernelInfo/linkage.ll
new file mode 100644
index 0000000000000..8679d366d0cb7
--- /dev/null
+++ b/llvm/test/Analysis/KernelInfo/linkage.ll
@@ -0,0 +1,68 @@
+; Check info on linkage.
+
+; RUN: opt -pass-remarks=kernel-info -passes=kernel-info \
+; RUN:     -disable-output %s 2>&1 | \
+; RUN:   FileCheck -match-full-lines %s
+
+target datalayout = "e-i64:64-i128:128-v16:16-v32:32-n16:32:64"
+target triple = "nvptx64-nvidia-cuda"
+
+; CHECK: remark: test.c:13:0: in artificial function 'extNotKer', ExternalNotKernel = 1
+define external void @extNotKer() !dbg !10 {
+entry:
+  ret void
+}
+
+; CHECK: remark: test.c:23:0: in function 'impNotKer', ExternalNotKernel = 1
+define void @impNotKer() !dbg !20 {
+entry:
+  ret void
+}
+
+; CHECK: remark: test.c:33:0: in artificial function 'weakNotKer', ExternalNotKernel = 0
+define weak void @weakNotKer() !dbg !30 {
+entry:
+  ret void
+}
+
+; CHECK: remark: test.c:43:0: in function 'extPtxKer', ExternalNotKernel = 0
+define external ptx_kernel void @extPtxKer() !dbg !40 {
+entry:
+  ret void
+}
+
+; CHECK: remark: test.c:53:0: in artificial function 'extAmdgpuKer', ExternalNotKernel = 0
+define external amdgpu_kernel void @extAmdgpuKer() !dbg !50 {
+entry:
+  ret void
+}
+
+; CHECK: remark: test.c:63:0: in function 'extSpirKer', ExternalNotKernel = 0
+define external spir_kernel void @extSpirKer() !dbg !60 {
+entry:
+  ret void
+}
+
+; CHECK: remark: test.c:73:0: in artificial function 'weakKer', ExternalNotKernel = 0
+define weak ptx_kernel void @weakKer() !dbg !70 {
+entry:
+  ret void
+}
+
+!llvm.module.flags = !{!0}
+!llvm.dbg.cu = !{!1}
+
+!0 = !{i32 2, !"Debug Info Version", i32 3}
+!1 = distinct !DICompileUnit(language: DW_LANG_C11, file: !2, producer: "clang version 19.0.0git", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, splitDebugInlining: false, nameTableKind: None)
+!2 = !DIFile(filename: "test.c", directory: "/tmp")
+!3 = !{null}
+!4 = !{}
+!5 = !DISubroutineType(types: !3)
+
+!10 = distinct !DISubprogram(name: "extNotKer", scope: !2, file: !2, line: 13, type: !5, scopeLine: 13, flags: DIFlagArtificial, spFlags: DISPFlagDefinition, unit: !1, retainedNodes: !4)
+!20 = distinct !DISubprogram(name: "impNotKer", scope: !2, file: !2, line: 23, type: !5, scopeLine: 23, spFlags: DISPFlagDefinition, unit: !1, retainedNodes: !4)
+!30 = distinct !DISubprogram(name: "weakNotKer", scope: !2, file: !2, line: 33, type: !5, scopeLine: 33, flags: DIFlagArtificial, spFlags: DISPFlagDefinition, unit: !1, retainedNodes: !4)
+!40 = distinct !DISubprogram(name: "extPtxKer", scope: !2, file: !2, line: 43, type: !5, scopeLine: 43, spFlags: DISPFlagDefinition, unit: !1, retainedNodes: !4)
+!50 = distinct !DISubprogram(name: "extAmdgpuKer", scope: !2, file: !2, line: 53, type: !5, scopeLine: 53, flags: DIFlagArtificial, spFlags: DISPFlagDefinition, unit: !1, retainedNodes: !4)
+!60 = distinct !DISubprogram(name: "extSpirKer", scope: !2, file: !2, line: 63, type: !5, scopeLine: 63, spFlags: DISPFlagDefinition, unit: !1, retainedNodes: !4)
+!70 = distinct !DISubprogram(name: "weakKer", scope: !2, file: !2, line: 73, type: !5, scopeLine: 73, flags: DIFlagArtificial, spFlags: DISPFlagDefinition, unit: !1, retainedNodes: !4)
diff --git a/llvm/test/Analysis/KernelInfo/openmp/README.md b/llvm/test/Analysis/KernelInfo/openmp/README.md
new file mode 100644
index 0000000000000..0aeb52f83c5c7
--- /dev/null
+++ b/llvm/test/Analysis/KernelInfo/openmp/README.md
@@ -0,0 +1,40 @@
+The tests in this directory check that basic KernelInfoPrinter functionality
+behaves reasonably for LLVM IR produced by Clang OpenMP codegen.
+
+So that these tests are straightforward to maintain and faithfully represent
+Clang OpenMP codegen, do not tweak or reduce the LLVM IR in them.  Other tests
+more exhaustively check KernelInfoPrinter features using reduced LLVM IR.
+
+The LLVM IR in each test file `$TEST` can be regenerated as follows in the case
+that Clang OpenMP codegen changes or it becomes desirable to adjust the source
+OpenMP program below.  First, remove the existing LLVM IR from `$TEST`.  Then,
+where `$TARGET` (e.g., `nvptx64-nvidia-cuda-sm_70` or `amdgcn-amd-amdhsa-gfx906`)
+depends on `$TEST`:
+
+```
+$ cd /tmp
+$ cat test.c
+#pragma omp declare target
+void f();
+void g() {
+  int i;
+  int a[2];
+  f();
+  g();
+}
+#pragma omp end declare target
+
+void h(int i) {
+  #pragma omp target map(tofrom:i)
+  {
+    int i;
+    int a[2];
+    f();
+    g();
+  }
+}
+
+$ clang -g -fopenmp --offload-arch=native -save-temps -c test.c
+$ llvm-dis test-openmp-$TARGET.bc
+$ cat test-openmp-$TARGET.ll >> $TEST
+```
diff --git a/llvm/test/Analysis/KernelInfo/openmp/amdgpu.ll b/llvm/test/Analysis/KernelInfo/openmp/amdgpu.ll
new file mode 100644
index 0000000000000..4843408bdda49
--- /dev/null
+++ b/llvm/test/Analysis/KernelInfo/openmp/amdgpu.ll
@@ -0,0 +1,225 @@
+; See ./README.md for how to maintain the LLVM IR in this test.
+
+; REQUIRES: amdgpu-registered-target
+
+; RUN: opt -pass-remarks=kernel-info -passes=kernel-info \
+; RUN:     -disable-output %s 2>&1 | \
+; RUN:   FileCheck -match-full-lines %s
+
+;  CHECK-NOT: remark:
+;      CHECK: remark: test.c:0:0: in artificial function '[[OFF_FUNC:__omp_offloading_[a-f0-9_]*_h_l12]]_debug__', artificial alloca ('%[[#]]') for 'dyn_ptr' with static size of 8 bytes
+; CHECK-NEXT: remark: test.c:14:9: in artificial function '[[OFF_FUNC]]_debug__', alloca ('%[[#]]') for 'i' with static size of 4 bytes
+; CHECK-NEXT: remark: test.c:15:9: in artificial function '[[OFF_FUNC]]_debug__', alloca ('%[[#]]') for 'a' with static size of 8 bytes
+; CHECK-NEXT: remark: <unknown>:0:0: in artificial function '[[OFF_FUNC]]_debug__', 'store' instruction accesses memory in flat address space
+; CHECK-NEXT: remark: test.c:13:3: in artificial function '[[OFF_FUNC]]_debug__', direct call, callee is '@__kmpc_target_init'
+; CHECK-NEXT: remark: test.c:16:5: in artificial function '[[OFF_FUNC]]_debug__', direct call, callee is '@f'
+; CHECK-NEXT: remark: test.c:17:5: in artificial function '[[OFF_FUNC]]_debug__', direct call to defined function, callee is 'g'
+; CHECK-NEXT: remark: test.c:18:3: in artificial function '[[OFF_FUNC]]_debug__', direct call, callee is '@__kmpc_target_deinit'
+; CHECK-NEXT: remark: test.c:13:0: in artificial function '[[OFF_FUNC]]_debug__', ExternalNotKernel = 0
+; CHECK-NEXT: remark: test.c:13:0: in artificial function '[[OFF_FUNC]]_debug__', amdgpu-max-num-workgroups[0] = 4294967295
+; CHECK-NEXT: remark: test.c:13:0: in artificial function '[[OFF_FUNC]]_debug__', amdgpu-max-num-workgroups[1] = 4294967295
+; CHECK-NEXT: remark: test.c:13:0: in artificial function '[[OFF_FUNC]]_debug__', amdgpu-max-num-workgroups[2] = 4294967295
+; CHECK-NEXT: remark: test.c:13:0: in artificial function '[[OFF_FUNC]]_debug__', amdgpu-flat-work-group-size[0] = 1
+; CHECK-NEXT: remark: test.c:13:0: in artificial function '[[OFF_FUNC]]_debug__', amdgpu-flat-work-group-size[1] = 1024
+; CHECK-NEXT: remark: test.c:13:0: in artificial function '[[OFF_FUNC]]_debug__', amdgpu-waves-per-eu[0] = 4
+; CHECK-NEXT: remark: test.c:13:0: in artificial function '[[OFF_FUNC]]_debug__', amdgpu-waves-per-eu[1] = 10
+; CHECK-NEXT: remark: test.c:13:0: in artificial function '[[OFF_FUNC]]_debug__', Allocas = 3
+; CHECK-NEXT: remark: test.c:13:0: in artificial function '[[OFF_FUNC]]_debug__', AllocasStaticSizeSum = 20
+; CHECK-NEXT: remark: test.c:13:0: in artificial function '[[OFF_FUNC]]_debug__', AllocasDyn = 0
+; CHECK-NEXT: remark: test.c:13:0: in artificial function '[[OFF_FUNC]]_debug__', DirectCalls = 4
+; CHECK-NEXT: remark: test.c:13:0: in artificial function '[[OFF_FUNC]]_debug__', IndirectCalls = 0
+; CHECK-NEXT: remark: test.c:13:0: in artificial function '[[OFF_FUNC]]_debug__', DirectCallsToDefinedFunctions = 1
+; CHECK-NEXT: remark: test.c:13:0: in artificial function '[[OFF_FUNC]]_debug__', InlineAssemblyCalls = 0
+; CHECK-NEXT: remark: test.c:13:0: in artificial function '[[OFF_FUNC]]_debug__', Invokes = 0
+; CHECK-NEXT: remark: test.c:13:0: in artificial function '[[OFF_FUNC]]_debug__', FlatAddrspaceAccesses = 1
+
+; CHECK-NEXT: remark: test.c:0:0: in artificial function '[[OFF_FUNC]]', artificial alloca ('%[[#]]') for 'dyn_ptr' with static size of 8 bytes
+; CHECK-NEXT: remark: <unknown>:0:0: in artificial function '[[OFF_FUNC]]', 'store' instruction accesses memory in flat address space
+; CHECK-NEXT: remark: test.c:12:1: in artificial function '[[OFF_FUNC]]', 'load' instruction ('%[[#]]') accesses memory in flat address space
+; CHECK-NEXT: remark: test.c:12:1: in artificial function '[[OFF_FUNC]]', direct call to defined function, callee is artificial '[[OFF_FUNC]]_debug__'
+; CHECK-NEXT: remark: test.c:12:0: in artificial function '[[OFF_FUNC]]', ExternalNotKernel = 0
+; CHECK-NEXT: remark: test.c:12:0: in artificial function '[[OFF_FUNC]]', omp_target_thread_limit = 256
+; CHECK-NEXT: remark: test.c:12:0: in artificial function '[[OFF_FUNC]]', amdgpu-max-num-workgroups[0] = 4294967295
+; CHECK-NEXT: remark: test.c:12:0: in artificial function '[[OFF_FUNC]]', amdgpu-max-num-workgroups[1] = 4294967295
+; CHECK-NEXT: remark: test.c:12:0: in artificial function '[[OFF_FUNC]]', amdgpu-max-num-workgroups[2] = 4294967295
+; CHECK-NEXT: remark: test.c:12:0: in artificial function '[[OFF_FUNC]]', amdgpu-flat-work-group-size[0] = 1
+; CHECK-NEXT: remark: test.c:12:0: in artificial function '[[OFF_FUNC]]', amdgpu-flat-work-group-size[1] = 256
+; CHECK-NEXT: remark: test.c:12:0: in artificial function '[[OFF_FUNC]]', amdgpu-waves-per-eu[0] = 1
+; CHECK-NEXT: remark: test.c:12:0: in artificial function '[[OFF_FUNC]]', amdgpu-waves-per-eu[1] = 10
+; CHECK-NEXT: remark: test.c:12:0: in artificial function '[[OFF_FUNC]]', Allocas = 1
+; CHECK-NEXT: remark: test.c:12:0: in artificial function '[[OFF_FUNC]]', AllocasStaticSizeSum = 8
+; CHECK-NEXT: remark: test.c:12:0: in artificial function '[[OFF_FUNC]]', AllocasDyn = 0
+; CHECK-NEXT: remark: test.c:12:0: in artificial function '[[OFF_FUNC]]', DirectCalls = 1
+; CHECK-NEXT: remark: test.c:12:0: in artificial function '[[OFF_FUNC]]', IndirectCalls = 0
+; CHECK-NEXT: remark: test.c:12:0: in artificial function '[[OFF_FUNC]]', DirectCallsToDefinedFunctions = 1
+; CHECK-NEXT: remark: test.c:12:0: in artificial function '[[OFF_FUNC]]', InlineAssemblyCalls = 0
+; CHECK-NEXT: remark: test.c:12:0: in artificial function '[[OFF_FUNC]]', Invokes = 0
+; CHECK-NEXT: remark: test.c:12:0: in artificial function '[[OFF_FUNC]]', FlatAddrspaceAccesses = 2
+
+; CHECK-NEXT: remark: test.c:4:7: in function 'g', alloca ('%[[#]]') for 'i' with static size of 4 bytes
+; CHECK-NEXT: remark: test.c:5:7: in function 'g', alloca ('%[[#]]') for 'a' with static size of 8 bytes
+; CHECK-NEXT: remark: test.c:6:3: in function 'g', direct call, callee is '@f'
+; CHECK-NEXT: remark: test.c:7:3: in function 'g', direct call to defined function, callee is 'g'
+; CHECK-NEXT: remark: test.c:3:0: in function 'g', ExternalNotKernel = 1
+; CHECK-NEXT: remark: test.c:3:0: in function 'g', amdgpu-max-num-workgroups[0] = 4294967295
+; CHECK-NEXT: remark: test.c:3:0: in function 'g', amdgpu-max-num-workgroups[1] = 4294967295
+; CHECK-NEXT: remark: test.c:3:0: in function 'g', amdgpu-max-num-workgroups[2] = 4294967295
+; CHECK-NEXT: remark: test.c:3:0: in function 'g', amdgpu-flat-work-group-size[0] = 1
+; CHECK-NEXT: remark: test.c:3:0: in function 'g', amdgpu-flat-work-group-size[1] = 1024
+; CHECK-NEXT: remark: test.c:3:0: in function 'g', amdgpu-waves-per-eu[0] = 4
+; CHECK-NEXT: remark: test.c:3:0: in function 'g', amdgpu-waves-per-eu[1] = 10
+; CHECK-NEXT: remark: test.c:3:0: in function 'g', Allocas = 2
+; CHECK-NEXT: remark: test.c:3:0: in function 'g', AllocasStaticSizeSum = 12
+; CHECK-NEXT: remark: test.c:3:0: in function 'g', AllocasDyn = 0
+; CHECK-NEXT: remark: test.c:3:0: in function 'g', DirectCalls = 2
+; CHECK-NEXT: remark: test.c:3:0: in function 'g', IndirectCalls = 0
+; CHECK-NEXT: remark: test.c:3:0: in function 'g', DirectCallsToDefinedFunctions = 1
+; CHECK-NEXT: remark: test.c:3:0: in function 'g', InlineAssemblyCalls = 0
+; CHECK-NEXT: remark: test.c:3:0: in function 'g', Invokes = 0
+; CHECK-NEXT: remark: test.c:3:0: in function 'g', FlatAddrspaceAccesses = 0
+;  CHECK-NOT: {{.}}
+
+; ModuleID = 'test-openmp-amdgcn-amd-amdhsa-gfx906.bc'
+source_filename = "test.c"
+target datalayout = "e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32-p7:160:256:256:32-p8:128:128-p9:192:256:256:32-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5-G1-ni:7:8:9"
+target triple = "amdgcn-amd-amdhsa"
+
+%struct.ident_t = type { i32, i32, i32, i32, ptr }
+%struct.DynamicEnvironmentTy = type { i16 }
+%struct.KernelEnvironmentTy = type { %struct.ConfigurationEnvironmentTy, ptr, ptr }
+%struct.ConfigurationEnvironmentTy = type { i8, i8, i8, i32, i32, i32, i32, i32, i32 }
+
+@__omp_rtl_debug_kind = weak_odr hidden addrspace(1) constant i32 0
+@__omp_rtl_assume_teams_oversubscription = weak_odr hidden addrspace(1) constant i32 0
+@__omp_rtl_assume_threads_oversubscription = weak_odr hidden addrspace(1) constant i32 0
+@__omp_rtl_assume_no_thread_state = weak_odr hidden addrspace(1) constant i32 0
+@__omp_rtl_assume_no_nested_parallelism = weak_odr hidden addrspace(1) constant i32 0
+@0 = private unnamed_addr constant [57 x i8] c";test.c;__omp_offloading_fd02_727e9_h_l12_debug__;13;3;;\00", align 1
+@1 = private unnamed_addr addrspace(1) constant %struct.ident_t { i32 0, i32 2, i32 0, i32 56, ptr @0 }, align 8
+@__omp_offloading_fd02_727e9_h_l12_dynamic_environment = weak_odr protected addrspace(1) global %struct.DynamicEnvironmentTy zeroinitializer
+@__omp_offloading_fd02_727e9_h_l12_kernel_environment = weak_odr protected addrspace(1) constant %struct.KernelEnvironmentTy { %struct.ConfigurationEnvironmentTy { i8 1, i8 1, i8 1, i32 1, i32 256, i32 -1, i32 -1, i32 0, i32 0 }, ptr addrspacecast (ptr addrspace(1) @1 to ptr), ptr addrspacecast (ptr addrspace(1) @__omp_offloading_fd02_727e9_h_l12_dynamic_environment to ptr) }
+@__oclc_ABI_version = weak_odr hidden local_unnamed_addr addrspace(4) constant i32 500
+
+; Function Attrs: convergent noinline norecurse nounwind optnone
+define internal void @__omp_offloading_fd02_727e9_h_l12_debug__(ptr noalias noundef %0) #0 !dbg !15 {
+  %2 = alloca ptr, align 8, addrspace(5)
+  %3 = alloca i32, align 4, addrspace(5)
+  %4 = alloca [2 x i32], align 4, addrspace(5)
+  %5 = addrspacecast ptr addrspace(5) %2 to ptr
+  %6 = addrspacecast ptr addrspace(5) %3 to ptr
+  %7 = addrspacecast ptr addrspace(5) %4 to ptr
+  store ptr %0, ptr %5, align 8
+    #dbg_declare(ptr addrspace(5) %2, !23, !DIExpression(), !24)
+  %8 = call i32 @__kmpc_target_init(ptr addrspacecast (ptr addrspace(1) @__omp_offloading_fd02_727e9_h_l12_kernel_environment to ptr), ptr %0), !dbg !25
+  %9 = icmp eq i32 %8, -1, !dbg !25
+  br i1 %9, label %10, label %11, !dbg !25
+
+10:                                               ; preds = %1
+    #dbg_declare(ptr addrspace(5) %3, !26, !DIExpression(), !29)
+    #dbg_declare(ptr addrspace(5) %4, !30, !DIExpression(), !34)
+  call void @f() #4, !dbg !35
+  call void @g() #4, !dbg !36
+  call void @__kmpc_target_deinit(), !dbg !37
+  ret void, !dbg !38
+
+11:                                               ; preds = %1
+  ret void, !dbg !25
+}
+
+; Function Attrs: convergent mustprogress noinline norecurse nounwind optnone
+define weak_odr protected amdgpu_kernel void @__omp_offloading_fd02_727e9_h_l12(ptr noalias noundef %0) #1 !dbg !39 {
+  %2 = alloca ptr, align 8, addrspace(5)
+  %3 = addrspacecast ptr addrspace(5) %2 to ptr
+  store ptr %0, ptr %3, align 8
+    #dbg_declare(ptr addrspace(5) %2, !40, !DIExpression(), !41)
+  %4 = load ptr, ptr %3, align 8, !dbg !42
+  call void @__omp_offloading_fd02_727e9_h_l12_debug__(ptr %4) #5, !dbg !42
+  ret void, !dbg !42
+}
+
+declare i32 @__kmpc_target_init(ptr, ptr)
+
+; Function Attrs: convergent
+declare void @f(...) #2
+
+declare void @__kmpc_target_deinit()
+
+; Function Attrs: convergent noinline nounwind optnone
+define hidden void @g() #3 !dbg !43 {
+  %1 = alloca i32, align 4, addrspace(5)
+  %2 = alloca [2 x i32], align 4, addrspace(5)
+  %3 = addrspacecast ptr addrspace(5) %1 to ptr
+  %4 = addrspacecast ptr addrspace(5) %2 to ptr
+    #dbg_declare(ptr addrspace(5) %1, !46, !DIExpression(), !47)
+    #dbg_declare(ptr addrspace(5) %2, !48, !DIExpression(), !49)
+  call void @f() #4, !dbg !50
+  call void @g() #4, !dbg !51
+  ret void, !dbg !52
+}
+
+attributes #0 = { convergent noinline norecurse nounwind optnone "frame-pointer"="all" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="gfx906" "target-features"="+16-bit-insts,+ci-insts,+dl-insts,+dot1-insts,+dot10-insts,+dot2-insts,+dot7-insts,+dpp,+gfx8-insts,+gfx9-insts,+s-memrealtime,+s-memtime-inst,+wavefrontsize64" }
+attributes #1 = { convergent mustprogress noinline norecurse nounwind optnone "amdgpu-flat-work-group-size"="1,256" "frame-pointer"="all" "kernel" "no-trapping-math"="true" "omp_target_thread_limit"="256" "stack-protector-buffer-size"="8" "target-cpu"="gfx906" "target-features"="+16-bit-insts,+ci-insts,+dl-insts,+dot1-insts,+dot10-insts,+dot2-insts,+dot7-insts,+dpp,+gfx8-insts,+gfx9-insts,+s-memrealtime,+s-memtime-inst,+wavefrontsize64" "uniform-work-group-size"="true" }
+attributes #2 = { convergent "frame-pointer"="all" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="gfx906" "target-features"="+16-bit-insts,+ci-insts,+dl-insts,+dot1-insts,+dot10-insts,+dot2-insts,+dot7-insts,+dpp,+gfx8-insts,+gfx9-insts,+s-memrealtime,+s-memtime-inst,+wavefrontsize64" }
+attributes #3 = { convergent noinline nounwind optnone "frame-pointer"="all" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="gfx906" "target-features"="+16-bit-insts,+ci-insts,+dl-insts,+dot1-insts,+dot10-insts,+dot2-insts,+dot7-insts,+dpp,+gfx8-insts,+gfx9-insts,+s-memrealtime,+s-memtime-inst,+wavefrontsize64" }
+attributes #4 = { convergent }
+attributes #5 = { nounwind }
+
+!llvm.dbg.cu = !{!0}
+!omp_offload.info = !{!2}
+!llvm.module.flags = !{!3, !4, !5, !6, !7, !8, !9, !10, !11}
+!llvm.ident = !{!12, !13, !13, !13, !13, !13, !13, !13, !13, !13, !13, !13, !13, !13, !13, !13, !13}
+!opencl.ocl.version = !{!14, !14, !14, !14, !14, !14, !14, !14, !14, !14, !14, !14, !14, !14, !14, !14}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C11, file: !1, producer: "clang version 20.0.0git (/tmp/llvm/clang b9447c03a9ef2eed55b685a33511df86f7f94e89)", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, splitDebugInlining: false, nameTableKind: None)
+!1 = !DIFile(filename: "test.c", directory: "/tmp", checksumkind: CSK_MD5, checksum: "27a878d5e894ab6d41bfe96f997f8821")
+!2 = !{i32 0, i32 64770, i32 468969, !"h", i32 12, i32 0, i32 0}
+!3 = !{i32 1, !"amdhsa_code_object_version", i32 500}
+!4 = !{i32 7, !"Dwarf Version", i32 5}
+!5 = !{i32 2, !"Debug Info Version", i32 3}
+!6 = !{i32 1, !"wchar_size", i32 4}
+!7 = !{i32 7, !"openmp", i32 51}
+!8 = !{i32 7, !"openmp-device", i32 51}
+!9 = !{i32 8, !"PIC Level", i32 2}
+!10 = !{i32 7, !"frame-pointer", i32 2}
+!11 = !{i32 4, !"amdgpu_hostcall", i32 1}
+!12 = !{!"clang version 20.0.0git (/tmp/llvm/clang b9447c03a9ef2eed55b685a33511df86f7f94e89)"}
+!13 = !{!"AMD clang version 17.0.0 (https://github.com/RadeonOpenCompute/llvm-project roc-6.0.2 24012 af27734ed982b52a9f1be0f035ac91726fc697e4)"}
+!14 = !{i32 2, i32 0}
+!15 = distinct !DISubprogram(name: "__omp_offloading_fd02_727e9_h_l12_debug__", scope: !16, file: !16, line: 13, type: !17, scopeLine: 13, flags: DIFlagArtificial | DIFlagPrototyped, spFlags: DISPFlagLocalToUnit | DISPFlagDefinition, unit: !0, retainedNodes: !22)
+!16 = !DIFile(filename: "test.c", directory: "/tmp")
+!17 = !DISubroutineType(types: !18)
+!18 = !{null, !19}
+!19 = !DIDerivedType(tag: DW_TAG_const_type, baseType: !20)
+!20 = !DIDerivedType(tag: DW_TAG_restrict_type, baseType: !21)
+!21 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: null, size: 64)
+!22 = !{}
+!23 = !DILocalVariable(name: "dyn_ptr", arg: 1, scope: !15, type: !19, flags: DIFlagArtificial)
+!24 = !DILocation(line: 0, scope: !15)
+!25 = !DILocation(line: 13, column: 3, scope: !15)
+!26 = !DILocalVariable(name: "i", scope: !27, file: !16, line: 14, type: !28)
+!27 = distinct !DILexicalBlock(scope: !15, file: !16, line: 13, column: 3)
+!28 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+!29 = !DILocation(line: 14, column: 9, scope: !27)
+!30 = !DILocalVariable(name: "a", scope: !27, file: !16, line: 15, type: !31)
+!31 = !DICompositeType(tag: DW_TAG_array_type, baseType: !28, size: 64, elements: !32)
+!32 = !{!33}
+!33 = !DISubrange(count: 2)
+!34 = !DILocation(line: 15, column: 9, scope: !27)
+!35 = !DILocation(line: 16, column: 5, scope: !27)
+!36 = !DILocation(line: 17, column: 5, scope: !27)
+!37 = !DILocation(line: 18, column: 3, scope: !27)
+!38 = !DILocation(line: 18, column: 3, scope: !15)
+!39 = distinct !DISubprogram(name: "__omp_offloading_fd02_727e9_h_l12", scope: !16, file: !16, line: 12, type: !17, scopeLine: 12, flags: DIFlagArtificial | DIFlagPrototyped, spFlags: DISPFlagLocalToUnit | DISPFlagDefinition, unit: !0, retainedNodes: !22)
+!40 = !DILocalVariable(name: "dyn_ptr", arg: 1, scope: !39, type: !19, flags: DIFlagArtificial)
+!41 = !DILocation(line: 0, scope: !39)
+!42 = !DILocation(line: 12, column: 1, scope: !39)
+!43 = distinct !DISubprogram(name: "g", scope: !16, file: !16, line: 3, type: !44, scopeLine: 3, spFlags: DISPFlagDefinition, unit: !0, retainedNodes: !22)
+!44 = !DISubroutineType(types: !45)
+!45 = !{null}
+!46 = !DILocalVariable(name: "i", scope: !43, file: !16, line: 4, type: !28)
+!47 = !DILocation(line: 4, column: 7, scope: !43)
+!48 = !DILocalVariable(name: "a", scope: !43, file: !16, line: 5, type: !31)
+!49 = !DILocation(line: 5, column: 7, scope: !43)
+!50 = !DILocation(line: 6, column: 3, scope: !43)
+!51 = !DILocation(line: 7, column: 3, scope: !43)
+!52 = !DILocation(line: 8, column: 1, scope: !43)
diff --git a/llvm/test/Analysis/KernelInfo/openmp/nvptx.ll b/llvm/test/Analysis/KernelInfo/openmp/nvptx.ll
new file mode 100644
index 0000000000000..bd46741b24e8c
--- /dev/null
+++ b/llvm/test/Analysis/KernelInfo/openmp/nvptx.ll
@@ -0,0 +1,813 @@
+; See ./README.md for how to maintain the LLVM IR in this test.
+
+; REQUIRES: nvptx-registered-target
+
+; RUN: opt -pass-remarks=kernel-info -passes=kernel-info \
+; RUN:     -disable-output %s 2>&1 | \
+; RUN:   FileCheck -match-full-lines %s
+
+;  CHECK-NOT: remark:
+;      CHECK: remark: test.c:0:0: in artificial function '[[OFF_FUNC:__omp_offloading_[a-f0-9_]*_h_l12]]_debug__', artificial alloca ('%[[#]]') for 'dyn_ptr' with static size of 8 bytes
+; CHECK-NEXT: remark: test.c:14:9: in artificial function '[[OFF_FUNC]]_debug__', alloca ('%[[#]]') for 'i' with static size of 4 bytes
+; CHECK-NEXT: remark: test.c:15:9: in artificial function '[[OFF_FUNC]]_debug__', alloca ('%[[#]]') for 'a' with static size of 8 bytes
+; CHECK-NEXT: remark: <unknown>:0:0: in artificial function '[[OFF_FUNC]]_debug__', 'store' instruction accesses memory in flat address space
+; CHECK-NEXT: remark: test.c:13:3: in artificial function '[[OFF_FUNC]]_debug__', direct call to defined function, callee is '@__kmpc_target_init'
+; CHECK-NEXT: remark: test.c:16:5: in artificial function '[[OFF_FUNC]]_debug__', direct call, callee is '@f'
+; CHECK-NEXT: remark: test.c:17:5: in artificial function '[[OFF_FUNC]]_debug__', direct call to defined function, callee is 'g'
+; CHECK-NEXT: remark: test.c:18:3: in artificial function '[[OFF_FUNC]]_debug__', direct call to defined function, callee is '@__kmpc_target_deinit'
+; CHECK-NEXT: remark: test.c:13:0: in artificial function '[[OFF_FUNC]]_debug__', ExternalNotKernel = 0
+; CHECK-NEXT: remark: test.c:13:0: in artificial function '[[OFF_FUNC]]_debug__', Allocas = 3
+; CHECK-NEXT: remark: test.c:13:0: in artificial function '[[OFF_FUNC]]_debug__', AllocasStaticSizeSum = 20
+; CHECK-NEXT: remark: test.c:13:0: in artificial function '[[OFF_FUNC]]_debug__', AllocasDyn = 0
+; CHECK-NEXT: remark: test.c:13:0: in artificial function '[[OFF_FUNC]]_debug__', DirectCalls = 4
+; CHECK-NEXT: remark: test.c:13:0: in artificial function '[[OFF_FUNC]]_debug__', IndirectCalls = 0
+; CHECK-NEXT: remark: test.c:13:0: in artificial function '[[OFF_FUNC]]_debug__', DirectCallsToDefinedFunctions = 3
+; CHECK-NEXT: remark: test.c:13:0: in artificial function '[[OFF_FUNC]]_debug__', InlineAssemblyCalls = 0
+; CHECK-NEXT: remark: test.c:13:0: in artificial function '[[OFF_FUNC]]_debug__', Invokes = 0
+; CHECK-NEXT: remark: test.c:13:0: in artificial function '[[OFF_FUNC]]_debug__', FlatAddrspaceAccesses = 1
+
+; CHECK-NEXT: remark: test.c:0:0: in artificial function '[[OFF_FUNC]]', artificial alloca ('%[[#]]') for 'dyn_ptr' with static size of 8 bytes
+; CHECK-NEXT: remark: <unknown>:0:0: in artificial function '[[OFF_FUNC]]', 'store' instruction accesses memory in flat address space
+; CHECK-NEXT: remark: test.c:12:1: in artificial function '[[OFF_FUNC]]', 'load' instruction ('%[[#]]') accesses memory in flat address space
+; CHECK-NEXT: remark: test.c:12:1: in artificial function '[[OFF_FUNC]]', direct call to defined function, callee is artificial '[[OFF_FUNC]]_debug__'
+; CHECK-NEXT: remark: test.c:12:0: in artificial function '[[OFF_FUNC]]', ExternalNotKernel = 0
+; CHECK-NEXT: remark: test.c:12:0: in artificial function '[[OFF_FUNC]]', omp_target_thread_limit = 128
+; CHECK-NEXT: remark: test.c:12:0: in artificial function '[[OFF_FUNC]]', maxntidx = 128
+; CHECK-NEXT: remark: test.c:12:0: in artificial function '[[OFF_FUNC]]', Allocas = 1
+; CHECK-NEXT: remark: test.c:12:0: in artificial function '[[OFF_FUNC]]', AllocasStaticSizeSum = 8
+; CHECK-NEXT: remark: test.c:12:0: in artificial function '[[OFF_FUNC]]', AllocasDyn = 0
+; CHECK-NEXT: remark: test.c:12:0: in artificial function '[[OFF_FUNC]]', DirectCalls = 1
+; CHECK-NEXT: remark: test.c:12:0: in artificial function '[[OFF_FUNC]]', IndirectCalls = 0
+; CHECK-NEXT: remark: test.c:12:0: in artificial function '[[OFF_FUNC]]', DirectCallsToDefinedFunctions = 1
+; CHECK-NEXT: remark: test.c:12:0: in artificial function '[[OFF_FUNC]]', InlineAssemblyCalls = 0
+; CHECK-NEXT: remark: test.c:12:0: in artificial function '[[OFF_FUNC]]', Invokes = 0
+; CHECK-NEXT: remark: test.c:12:0: in artificial function '[[OFF_FUNC]]', FlatAddrspaceAccesses = 2
+
+; CHECK-NEXT: remark: test.c:4:7: in function 'g', alloca ('%[[#]]') for 'i' with static size of 4 bytes
+; CHECK-NEXT: remark: test.c:5:7: in function 'g', alloca ('%[[#]]') for 'a' with static size of 8 bytes
+; CHECK-NEXT: remark: test.c:6:3: in function 'g', direct call, callee is '@f'
+; CHECK-NEXT: remark: test.c:7:3: in function 'g', direct call to defined function, callee is 'g'
+; CHECK-NEXT: remark: test.c:3:0: in function 'g', ExternalNotKernel = 1
+; CHECK-NEXT: remark: test.c:3:0: in function 'g', Allocas = 2
+; CHECK-NEXT: remark: test.c:3:0: in function 'g', AllocasStaticSizeSum = 12
+; CHECK-NEXT: remark: test.c:3:0: in function 'g', AllocasDyn = 0
+; CHECK-NEXT: remark: test.c:3:0: in function 'g', DirectCalls = 2
+; CHECK-NEXT: remark: test.c:3:0: in function 'g', IndirectCalls = 0
+; CHECK-NEXT: remark: test.c:3:0: in function 'g', DirectCallsToDefinedFunctions = 1
+; CHECK-NEXT: remark: test.c:3:0: in function 'g', InlineAssemblyCalls = 0
+; CHECK-NEXT: remark: test.c:3:0: in function 'g', Invokes = 0
+; CHECK-NEXT: remark: test.c:3:0: in function 'g', FlatAddrspaceAccesses = 0
+;  CHECK-NOT: remark: {{.*: in function 'g',.*}}
+
+; A lot of internal functions (e.g., __kmpc_target_init) come next, but we don't
+; want to maintain a list of their allocas, calls, etc. in this test.
+
+; ModuleID = 'test-openmp-nvptx64-nvidia-cuda-sm_70.bc'
+source_filename = "test.c"
+target datalayout = "e-i64:64-i128:128-v16:16-v32:32-n16:32:64"
+target triple = "nvptx64-nvidia-cuda"
+
+%struct.ident_t = type { i32, i32, i32, i32, ptr }
+%struct.DynamicEnvironmentTy = type { i16 }
+%struct.KernelEnvironmentTy = type { %struct.ConfigurationEnvironmentTy, ptr, ptr }
+%struct.ConfigurationEnvironmentTy = type { i8, i8, i8, i32, i32, i32, i32, i32, i32 }
+%struct.DeviceMemoryPoolTy = type { ptr, i64 }
+%struct.DeviceMemoryPoolTrackingTy = type { i64, i64, i64, i64 }
+%struct.DeviceEnvironmentTy = type { i32, i32, i32, i32, i64, i64, i64, i64 }
+%"struct.rpc::Client" = type { %"struct.rpc::Process" }
+%"struct.rpc::Process" = type { i32, ptr, ptr, ptr, ptr, [128 x i32] }
+%"struct.(anonymous namespace)::SharedMemorySmartStackTy" = type { [512 x i8], [1024 x i8] }
+%"struct.ompx::state::TeamStateTy" = type { %"struct.ompx::state::ICVStateTy", i32, i32, ptr }
+%"struct.ompx::state::ICVStateTy" = type { i32, i32, i32, i32, i32, i32, i32 }
+
+@__omp_rtl_assume_teams_oversubscription = weak_odr hidden constant i32 0
+@__omp_rtl_assume_threads_oversubscription = weak_odr hidden constant i32 0
+@0 = private unnamed_addr constant [58 x i8] c";test.c;__omp_offloading_fd02_1116d6_h_l12_debug__;13;3;;\00", align 1
+@1 = private unnamed_addr constant %struct.ident_t { i32 0, i32 2, i32 0, i32 57, ptr @0 }, align 8
+@__omp_offloading_fd02_1116d6_h_l12_dynamic_environment = weak_odr protected global %struct.DynamicEnvironmentTy zeroinitializer
+@__omp_offloading_fd02_1116d6_h_l12_kernel_environment = weak_odr protected constant %struct.KernelEnvironmentTy { %struct.ConfigurationEnvironmentTy { i8 1, i8 1, i8 1, i32 1, i32 128, i32 -1, i32 -1, i32 0, i32 0 }, ptr @1, ptr @__omp_offloading_fd02_1116d6_h_l12_dynamic_environment }
+@llvm.used = appending global [4 x ptr] [ptr @__llvm_rpc_client, ptr addrspacecast (ptr addrspace(4) @__omp_rtl_device_environment to ptr), ptr @__omp_rtl_device_memory_pool, ptr @__omp_rtl_device_memory_pool_tracker], section "llvm.metadata"
+@__omp_rtl_device_memory_pool = weak protected global %struct.DeviceMemoryPoolTy zeroinitializer, align 8
+@__omp_rtl_device_memory_pool_tracker = weak protected global %struct.DeviceMemoryPoolTrackingTy zeroinitializer, align 8
+@__omp_rtl_debug_kind = weak_odr hidden constant i32 0
+@__omp_rtl_assume_no_thread_state = weak_odr hidden constant i32 0
+@__omp_rtl_assume_no_nested_parallelism = weak_odr hidden constant i32 0
+@__omp_rtl_device_environment = weak protected addrspace(4) global %struct.DeviceEnvironmentTy undef, align 8
+@.str = private unnamed_addr constant [40 x i8] c"%s:%u: %s: Assertion %s (`%s`) failed.\0A\00", align 1
+@.str1 = private unnamed_addr constant [35 x i8] c"%s:%u: %s: Assertion `%s` failed.\0A\00", align 1
+@.str15 = private unnamed_addr constant [43 x i8] c"/tmp/llvm/offload/DeviceRTL/src/Kernel.cpp\00", align 1
+@__PRETTY_FUNCTION__._ZL19genericStateMachineP7IdentTy = private unnamed_addr constant [36 x i8] c"void genericStateMachine(IdentTy *)\00", align 1
+@.str2 = private unnamed_addr constant [18 x i8] c"WorkFn == nullptr\00", align 1
+@__PRETTY_FUNCTION__.__kmpc_target_deinit = private unnamed_addr constant [28 x i8] c"void __kmpc_target_deinit()\00", align 1
+@IsSPMDMode = internal local_unnamed_addr addrspace(3) global i32 undef, align 4
+@__llvm_rpc_client = weak protected global %"struct.rpc::Client" zeroinitializer, align 8
+@.str1027 = private unnamed_addr constant [48 x i8] c"/tmp/llvm/offload/DeviceRTL/src/Parallelism.cpp\00", align 1
+@.str12 = private unnamed_addr constant [23 x i8] c"!mapping::isSPMDMode()\00", align 1
+@__PRETTY_FUNCTION__.__kmpc_kernel_end_parallel = private unnamed_addr constant [34 x i8] c"void __kmpc_kernel_end_parallel()\00", align 1
+@_ZL20KernelEnvironmentPtr = internal unnamed_addr addrspace(3) global ptr undef, align 8
+@_ZL26KernelLaunchEnvironmentPtr = internal unnamed_addr addrspace(3) global ptr undef, align 8
+@_ZN12_GLOBAL__N_122SharedMemorySmartStackE = internal addrspace(3) global %"struct.(anonymous namespace)::SharedMemorySmartStackTy" undef, align 16
+@.str444 = private unnamed_addr constant [42 x i8] c"/tmp/llvm/offload/DeviceRTL/src/State.cpp\00", align 1
+@.str747 = private unnamed_addr constant [33 x i8] c"NThreadsVar == Other.NThreadsVar\00", align 1
+@__PRETTY_FUNCTION__._ZNK4ompx5state10ICVStateTy11assertEqualERKS1_ = private unnamed_addr constant [68 x i8] c"void ompx::state::ICVStateTy::assertEqual(const ICVStateTy &) const\00", align 1
+@.str848 = private unnamed_addr constant [27 x i8] c"LevelVar == Other.LevelVar\00", align 1
+@.str949 = private unnamed_addr constant [39 x i8] c"ActiveLevelVar == Other.ActiveLevelVar\00", align 1
+@.str1050 = private unnamed_addr constant [47 x i8] c"MaxActiveLevelsVar == Other.MaxActiveLevelsVar\00", align 1
+@.str1151 = private unnamed_addr constant [33 x i8] c"RunSchedVar == Other.RunSchedVar\00", align 1
+@.str1252 = private unnamed_addr constant [43 x i8] c"RunSchedChunkVar == Other.RunSchedChunkVar\00", align 1
+@.str13 = private unnamed_addr constant [43 x i8] c"ParallelTeamSize == Other.ParallelTeamSize\00", align 1
+@__PRETTY_FUNCTION__._ZNK4ompx5state11TeamStateTy11assertEqualERS1_ = private unnamed_addr constant [64 x i8] c"void ompx::state::TeamStateTy::assertEqual(TeamStateTy &) const\00", align 1
+@.str14 = private unnamed_addr constant [39 x i8] c"HasThreadState == Other.HasThreadState\00", align 1
+@.str23 = private unnamed_addr constant [32 x i8] c"mapping::isSPMDMode() == IsSPMD\00", align 1
+@__PRETTY_FUNCTION__._ZN4ompx5state18assumeInitialStateEb = private unnamed_addr constant [43 x i8] c"void ompx::state::assumeInitialState(bool)\00", align 1
+@_ZL9ThreadDST = internal unnamed_addr addrspace(3) global ptr undef, align 8
+@_ZN4ompx5state9TeamStateE = internal local_unnamed_addr addrspace(3) global %"struct.ompx::state::TeamStateTy" undef, align 8
+@_ZN4ompx5state12ThreadStatesE = internal addrspace(3) global ptr undef, align 8
+
+; Function Attrs: convergent noinline norecurse nounwind optnone
+define internal void @__omp_offloading_fd02_1116d6_h_l12_debug__(ptr noalias noundef %0) #0 !dbg !18 {
+  %2 = alloca ptr, align 8
+  %3 = alloca i32, align 4
+  %4 = alloca [2 x i32], align 4
+  store ptr %0, ptr %2, align 8
+    #dbg_declare(ptr %2, !25, !DIExpression(), !26)
+  %5 = call i32 @__kmpc_target_init(ptr @__omp_offloading_fd02_1116d6_h_l12_kernel_environment, ptr %0), !dbg !27
+  %6 = icmp eq i32 %5, -1, !dbg !27
+  br i1 %6, label %7, label %8, !dbg !27
+
+7:                                                ; preds = %1
+    #dbg_declare(ptr %3, !28, !DIExpression(), !31)
+    #dbg_declare(ptr %4, !32, !DIExpression(), !36)
+  call void @f() #19, !dbg !37
+  call void @g() #19, !dbg !38
+  call void @__kmpc_target_deinit(), !dbg !39
+  ret void, !dbg !40
+
+8:                                                ; preds = %1
+  ret void, !dbg !27
+}
+
+; Function Attrs: convergent mustprogress noinline norecurse nounwind optnone
+define weak_odr protected ptx_kernel void @__omp_offloading_fd02_1116d6_h_l12(ptr noalias noundef %0) #1 !dbg !41 {
+  %2 = alloca ptr, align 8
+  store ptr %0, ptr %2, align 8
+    #dbg_declare(ptr %2, !42, !DIExpression(), !43)
+  %3 = load ptr, ptr %2, align 8, !dbg !44
+  call void @__omp_offloading_fd02_1116d6_h_l12_debug__(ptr %3) #20, !dbg !44
+  ret void, !dbg !44
+}
+
+; Function Attrs: convergent
+declare void @f(...) #2
+
+; Function Attrs: convergent noinline nounwind optnone
+define hidden void @g() #3 !dbg !45 {
+  %1 = alloca i32, align 4
+  %2 = alloca [2 x i32], align 4
+    #dbg_declare(ptr %1, !48, !DIExpression(), !49)
+    #dbg_declare(ptr %2, !50, !DIExpression(), !51)
+  call void @f() #19, !dbg !52
+  call void @g() #19, !dbg !53
+  ret void, !dbg !54
+}
+
+; Function Attrs: convergent mustprogress nounwind
+define internal noundef range(i32 -1, 1024) i32 @__kmpc_target_init(ptr nofree noundef nonnull align 8 dereferenceable(48) %0, ptr nofree noundef nonnull align 8 dereferenceable(16) %1) #4 {
+  %3 = alloca ptr, align 8
+  %4 = getelementptr inbounds nuw i8, ptr %0, i64 2
+  %5 = load i8, ptr %4, align 2, !tbaa !55
+  %6 = and i8 %5, 2
+  %7 = icmp eq i8 %6, 0
+  %8 = load i8, ptr %0, align 8, !tbaa !61
+  %9 = icmp ne i8 %8, 0
+  br i1 %7, label %21, label %10
+
+10:                                               ; preds = %2
+  %11 = tail call range(i32 0, 1024) i32 @llvm.nvvm.read.ptx.sreg.tid.x()
+  %12 = icmp eq i32 %11, 0
+  br i1 %12, label %13, label %14
+
+13:                                               ; preds = %10
+  store i32 1, ptr addrspace(3) @IsSPMDMode, align 4, !tbaa !62
+  store i8 0, ptr addrspace(3) addrspacecast (ptr getelementptr inbounds nuw (i8, ptr addrspacecast (ptr addrspace(3) @_ZN12_GLOBAL__N_122SharedMemorySmartStackE to ptr), i64 512) to ptr addrspace(3)), align 1, !tbaa !63
+  tail call void @llvm.memset.p0.i64(ptr noundef nonnull align 8 dereferenceable(48) addrspacecast (ptr addrspace(3) @_ZN4ompx5state9TeamStateE to ptr), i8 noundef 0, i64 noundef 16, i1 noundef false)
+  store i32 1, ptr addrspace(3) addrspacecast (ptr getelementptr inbounds nuw (i8, ptr addrspacecast (ptr addrspace(3) @_ZN4ompx5state9TeamStateE to ptr), i64 16) to ptr addrspace(3)), align 8, !tbaa !64
+  store i32 1, ptr addrspace(3) addrspacecast (ptr getelementptr inbounds nuw (i8, ptr addrspacecast (ptr addrspace(3) @_ZN4ompx5state9TeamStateE to ptr), i64 20) to ptr addrspace(3)), align 4, !tbaa !69
+  store i32 1, ptr addrspace(3) addrspacecast (ptr getelementptr inbounds nuw (i8, ptr addrspacecast (ptr addrspace(3) @_ZN4ompx5state9TeamStateE to ptr), i64 24) to ptr addrspace(3)), align 8, !tbaa !70
+  store i32 1, ptr addrspace(3) addrspacecast (ptr getelementptr inbounds nuw (i8, ptr addrspacecast (ptr addrspace(3) @_ZN4ompx5state9TeamStateE to ptr), i64 28) to ptr addrspace(3)), align 4, !tbaa !71
+  store i32 0, ptr addrspace(3) addrspacecast (ptr getelementptr inbounds nuw (i8, ptr addrspacecast (ptr addrspace(3) @_ZN4ompx5state9TeamStateE to ptr), i64 32) to ptr addrspace(3)), align 8, !tbaa !72
+  store ptr null, ptr addrspace(3) addrspacecast (ptr getelementptr inbounds nuw (i8, ptr addrspacecast (ptr addrspace(3) @_ZN4ompx5state9TeamStateE to ptr), i64 40) to ptr addrspace(3)), align 8, !tbaa !73
+  store ptr null, ptr addrspace(3) @_ZN4ompx5state12ThreadStatesE, align 8, !tbaa !74
+  store ptr %0, ptr addrspace(3) @_ZL20KernelEnvironmentPtr, align 8, !tbaa !76
+  store ptr %1, ptr addrspace(3) @_ZL26KernelLaunchEnvironmentPtr, align 8, !tbaa !78
+  br label %18
+
+14:                                               ; preds = %10
+  %15 = zext nneg i32 %11 to i64
+  %16 = getelementptr inbounds nuw [1024 x i8], ptr getelementptr inbounds nuw (i8, ptr addrspacecast (ptr addrspace(3) @_ZN12_GLOBAL__N_122SharedMemorySmartStackE to ptr), i64 512), i64 0, i64 %15
+  %17 = addrspacecast ptr %16 to ptr addrspace(3)
+  store i8 0, ptr addrspace(3) %17, align 1, !tbaa !63
+  br label %18
+
+18:                                               ; preds = %14, %13
+  br i1 %12, label %19, label %20
+
+19:                                               ; preds = %18
+  store ptr null, ptr addrspace(3) @_ZL9ThreadDST, align 8, !tbaa !80
+  br label %20
+
+20:                                               ; preds = %18, %19
+  tail call void @_ZN4ompx11synchronize14threadsAlignedENS_6atomic10OrderingTyE(i32 poison) #21
+  br label %37
+
+21:                                               ; preds = %2
+  %22 = tail call range(i32 1, 1025) i32 @llvm.nvvm.read.ptx.sreg.ntid.x(), !range !82
+  %23 = add nsw i32 %22, -1
+  %24 = and i32 %23, -32
+  %25 = tail call range(i32 0, 1024) i32 @llvm.nvvm.read.ptx.sreg.tid.x()
+  %26 = icmp eq i32 %25, %24
+  br i1 %26, label %27, label %31
+
+27:                                               ; preds = %21
+  store i32 0, ptr addrspace(3) @IsSPMDMode, align 4, !tbaa !62
+  %28 = zext nneg i32 %25 to i64
+  %29 = getelementptr inbounds nuw [1024 x i8], ptr getelementptr inbounds nuw (i8, ptr addrspacecast (ptr addrspace(3) @_ZN12_GLOBAL__N_122SharedMemorySmartStackE to ptr), i64 512), i64 0, i64 %28
+  %30 = addrspacecast ptr %29 to ptr addrspace(3)
+  store i8 0, ptr addrspace(3) %30, align 1, !tbaa !63
+  tail call void @llvm.memset.p0.i64(ptr noundef nonnull align 8 dereferenceable(48) addrspacecast (ptr addrspace(3) @_ZN4ompx5state9TeamStateE to ptr), i8 noundef 0, i64 noundef 16, i1 noundef false)
+  store i32 1, ptr addrspace(3) addrspacecast (ptr getelementptr inbounds nuw (i8, ptr addrspacecast (ptr addrspace(3) @_ZN4ompx5state9TeamStateE to ptr), i64 16) to ptr addrspace(3)), align 8, !tbaa !64
+  store i32 1, ptr addrspace(3) addrspacecast (ptr getelementptr inbounds nuw (i8, ptr addrspacecast (ptr addrspace(3) @_ZN4ompx5state9TeamStateE to ptr), i64 20) to ptr addrspace(3)), align 4, !tbaa !69
+  store i32 1, ptr addrspace(3) addrspacecast (ptr getelementptr inbounds nuw (i8, ptr addrspacecast (ptr addrspace(3) @_ZN4ompx5state9TeamStateE to ptr), i64 24) to ptr addrspace(3)), align 8, !tbaa !70
+  store i32 1, ptr addrspace(3) addrspacecast (ptr getelementptr inbounds nuw (i8, ptr addrspacecast (ptr addrspace(3) @_ZN4ompx5state9TeamStateE to ptr), i64 28) to ptr addrspace(3)), align 4, !tbaa !71
+  store i32 0, ptr addrspace(3) addrspacecast (ptr getelementptr inbounds nuw (i8, ptr addrspacecast (ptr addrspace(3) @_ZN4ompx5state9TeamStateE to ptr), i64 32) to ptr addrspace(3)), align 8, !tbaa !72
+  store ptr null, ptr addrspace(3) addrspacecast (ptr getelementptr inbounds nuw (i8, ptr addrspacecast (ptr addrspace(3) @_ZN4ompx5state9TeamStateE to ptr), i64 40) to ptr addrspace(3)), align 8, !tbaa !73
+  store ptr null, ptr addrspace(3) @_ZN4ompx5state12ThreadStatesE, align 8, !tbaa !74
+  store ptr %0, ptr addrspace(3) @_ZL20KernelEnvironmentPtr, align 8, !tbaa !76
+  store ptr %1, ptr addrspace(3) @_ZL26KernelLaunchEnvironmentPtr, align 8, !tbaa !78
+  br label %35
+
+31:                                               ; preds = %21
+  %32 = zext nneg i32 %25 to i64
+  %33 = getelementptr inbounds nuw [1024 x i8], ptr getelementptr inbounds nuw (i8, ptr addrspacecast (ptr addrspace(3) @_ZN12_GLOBAL__N_122SharedMemorySmartStackE to ptr), i64 512), i64 0, i64 %32
+  %34 = addrspacecast ptr %33 to ptr addrspace(3)
+  store i8 0, ptr addrspace(3) %34, align 1, !tbaa !63
+  br label %35
+
+35:                                               ; preds = %31, %27
+  br i1 %26, label %36, label %37
+
+36:                                               ; preds = %35
+  store ptr null, ptr addrspace(3) @_ZL9ThreadDST, align 8, !tbaa !80
+  br label %37
+
+37:                                               ; preds = %36, %35, %20
+  br i1 %7, label %100, label %38
+
+38:                                               ; preds = %37
+  %39 = load i32, ptr @__omp_rtl_debug_kind, align 4, !tbaa !62
+  %40 = load i32, ptr addrspace(4) @__omp_rtl_device_environment, align 8, !tbaa !83
+  %41 = and i32 %39, 1
+  %42 = and i32 %41, %40
+  %43 = icmp ne i32 %42, 0
+  %44 = load i32, ptr addrspace(3) @_ZN4ompx5state9TeamStateE, align 8, !tbaa !86
+  %45 = icmp ne i32 %44, 0
+  %46 = select i1 %43, i1 %45, i1 false
+  br i1 %46, label %47, label %48
+
+47:                                               ; preds = %38
+  tail call fastcc void @__assert_fail_internal(ptr noundef nonnull dereferenceable(33) @.str747, ptr noundef null, ptr noundef nonnull dereferenceable(66) @.str444, i32 noundef 193, ptr noundef nonnull dereferenceable(68) @__PRETTY_FUNCTION__._ZNK4ompx5state10ICVStateTy11assertEqualERKS1_) #22
+  unreachable
+
+48:                                               ; preds = %38
+  %49 = icmp eq i32 %44, 0
+  tail call void @llvm.assume(i1 noundef %49) #23
+  %50 = load i32, ptr addrspace(3) addrspacecast (ptr getelementptr inbounds nuw (i8, ptr addrspacecast (ptr addrspace(3) @_ZN4ompx5state9TeamStateE to ptr), i64 4) to ptr addrspace(3)), align 4, !tbaa !87
+  br i1 %43, label %51, label %54
+
+51:                                               ; preds = %48
+  %52 = icmp eq i32 %50, 0
+  br i1 %52, label %54, label %53
+
+53:                                               ; preds = %51
+  tail call fastcc void @__assert_fail_internal(ptr noundef nonnull dereferenceable(27) @.str848, ptr noundef null, ptr noundef nonnull dereferenceable(66) @.str444, i32 noundef 194, ptr noundef nonnull dereferenceable(68) @__PRETTY_FUNCTION__._ZNK4ompx5state10ICVStateTy11assertEqualERKS1_) #22
+  unreachable
+
+54:                                               ; preds = %51, %48
+  %55 = phi i32 [ 0, %51 ], [ %50, %48 ]
+  %56 = icmp eq i32 %55, 0
+  tail call void @llvm.assume(i1 noundef %56) #23
+  %57 = load i32, ptr addrspace(3) addrspacecast (ptr getelementptr inbounds nuw (i8, ptr addrspacecast (ptr addrspace(3) @_ZN4ompx5state9TeamStateE to ptr), i64 8) to ptr addrspace(3)), align 8, !tbaa !88
+  br i1 %43, label %58, label %61
+
+58:                                               ; preds = %54
+  %59 = icmp eq i32 %57, 0
+  br i1 %59, label %61, label %60
+
+60:                                               ; preds = %58
+  tail call fastcc void @__assert_fail_internal(ptr noundef nonnull dereferenceable(39) @.str949, ptr noundef null, ptr noundef nonnull dereferenceable(66) @.str444, i32 noundef 195, ptr noundef nonnull dereferenceable(68) @__PRETTY_FUNCTION__._ZNK4ompx5state10ICVStateTy11assertEqualERKS1_) #22
+  unreachable
+
+61:                                               ; preds = %58, %54
+  %62 = phi i32 [ 0, %58 ], [ %57, %54 ]
+  %63 = icmp eq i32 %62, 0
+  tail call void @llvm.assume(i1 noundef %63) #23
+  %64 = load i32, ptr addrspace(3) addrspacecast (ptr getelementptr inbounds nuw (i8, ptr addrspacecast (ptr addrspace(3) @_ZN4ompx5state9TeamStateE to ptr), i64 16) to ptr addrspace(3)), align 8, !tbaa !89
+  br i1 %43, label %65, label %68
+
+65:                                               ; preds = %61
+  %66 = icmp eq i32 %64, 1
+  br i1 %66, label %68, label %67
+
+67:                                               ; preds = %65
+  tail call fastcc void @__assert_fail_internal(ptr noundef nonnull dereferenceable(47) @.str1050, ptr noundef null, ptr noundef nonnull dereferenceable(66) @.str444, i32 noundef 196, ptr noundef nonnull dereferenceable(68) @__PRETTY_FUNCTION__._ZNK4ompx5state10ICVStateTy11assertEqualERKS1_) #22
+  unreachable
+
+68:                                               ; preds = %65, %61
+  %69 = phi i32 [ 1, %65 ], [ %64, %61 ]
+  %70 = icmp eq i32 %69, 1
+  tail call void @llvm.assume(i1 noundef %70) #23
+  %71 = load i32, ptr addrspace(3) addrspacecast (ptr getelementptr inbounds nuw (i8, ptr addrspacecast (ptr addrspace(3) @_ZN4ompx5state9TeamStateE to ptr), i64 20) to ptr addrspace(3)), align 4, !tbaa !90
+  br i1 %43, label %72, label %93
+
+72:                                               ; preds = %68
+  %73 = icmp eq i32 %71, 1
+  br i1 %73, label %75, label %74
+
+74:                                               ; preds = %72
+  tail call fastcc void @__assert_fail_internal(ptr noundef nonnull dereferenceable(33) @.str1151, ptr noundef null, ptr noundef nonnull dereferenceable(66) @.str444, i32 noundef 197, ptr noundef nonnull dereferenceable(68) @__PRETTY_FUNCTION__._ZNK4ompx5state10ICVStateTy11assertEqualERKS1_) #22
+  unreachable
+
+75:                                               ; preds = %72
+  %76 = icmp eq i32 1, 1
+  tail call void @llvm.assume(i1 noundef %76) #23
+  br i1 %43, label %77, label %95
+
+77:                                               ; preds = %75
+  %78 = load i32, ptr addrspace(3) addrspacecast (ptr getelementptr inbounds nuw (i8, ptr addrspacecast (ptr addrspace(3) @_ZN4ompx5state9TeamStateE to ptr), i64 24) to ptr addrspace(3)), align 8, !tbaa !91
+  %79 = icmp eq i32 %78, 1
+  br i1 %79, label %81, label %80
+
+80:                                               ; preds = %77
+  tail call fastcc void @__assert_fail_internal(ptr noundef nonnull dereferenceable(43) @.str1252, ptr noundef null, ptr noundef nonnull dereferenceable(66) @.str444, i32 noundef 198, ptr noundef nonnull dereferenceable(68) @__PRETTY_FUNCTION__._ZNK4ompx5state10ICVStateTy11assertEqualERKS1_) #22
+  unreachable
+
+81:                                               ; preds = %77
+  %82 = load i32, ptr addrspace(3) addrspacecast (ptr getelementptr inbounds nuw (i8, ptr addrspacecast (ptr addrspace(3) @_ZN4ompx5state9TeamStateE to ptr), i64 28) to ptr addrspace(3)), align 4, !tbaa !71
+  %83 = icmp eq i32 %82, 1
+  br i1 %83, label %85, label %84
+
+84:                                               ; preds = %81
+  tail call fastcc void @__assert_fail_internal(ptr noundef nonnull dereferenceable(43) @.str13, ptr noundef null, ptr noundef nonnull dereferenceable(66) @.str444, i32 noundef 222, ptr noundef nonnull dereferenceable(64) @__PRETTY_FUNCTION__._ZNK4ompx5state11TeamStateTy11assertEqualERS1_) #22
+  unreachable
+
+85:                                               ; preds = %81
+  %86 = load i32, ptr addrspace(3) addrspacecast (ptr getelementptr inbounds nuw (i8, ptr addrspacecast (ptr addrspace(3) @_ZN4ompx5state9TeamStateE to ptr), i64 32) to ptr addrspace(3)), align 8, !tbaa !72
+  %87 = icmp eq i32 %86, 0
+  br i1 %87, label %89, label %88
+
+88:                                               ; preds = %85
+  tail call fastcc void @__assert_fail_internal(ptr noundef nonnull dereferenceable(39) @.str14, ptr noundef null, ptr noundef nonnull dereferenceable(66) @.str444, i32 noundef 223, ptr noundef nonnull dereferenceable(64) @__PRETTY_FUNCTION__._ZNK4ompx5state11TeamStateTy11assertEqualERS1_) #22
+  unreachable
+
+89:                                               ; preds = %85
+  %90 = load i32, ptr addrspace(3) @IsSPMDMode, align 4, !tbaa !62
+  %91 = icmp eq i32 %90, 0
+  br i1 %91, label %92, label %98
+
+92:                                               ; preds = %89
+  tail call fastcc void @__assert_fail_internal(ptr noundef nonnull dereferenceable(32) @.str23, ptr noundef null, ptr noundef nonnull dereferenceable(66) @.str444, i32 noundef 326, ptr noundef nonnull dereferenceable(43) @__PRETTY_FUNCTION__._ZN4ompx5state18assumeInitialStateEb) #22
+  unreachable
+
+93:                                               ; preds = %68
+  %94 = icmp eq i32 %71, 1
+  tail call void @llvm.assume(i1 noundef %94) #23
+  br label %95
+
+95:                                               ; preds = %75, %93
+  %96 = load i32, ptr addrspace(3) @IsSPMDMode, align 4, !tbaa !62
+  %97 = icmp ne i32 %96, 0
+  br label %98
+
+98:                                               ; preds = %89, %95
+  %99 = phi i1 [ %97, %95 ], [ true, %89 ]
+  tail call void @llvm.assume(i1 noundef %99) #23
+  tail call void @_ZN4ompx11synchronize14threadsAlignedENS_6atomic10OrderingTyE(i32 poison) #21
+  br label %130
+
+100:                                              ; preds = %37
+  %101 = tail call range(i32 1, 1025) i32 @llvm.nvvm.read.ptx.sreg.ntid.x(), !range !82
+  %102 = add nsw i32 %101, -1
+  %103 = and i32 %102, -32
+  %104 = tail call range(i32 0, 1024) i32 @llvm.nvvm.read.ptx.sreg.tid.x(), !range !92
+  %105 = icmp eq i32 %104, %103
+  br i1 %105, label %130, label %106
+
+106:                                              ; preds = %100
+  %107 = add nsw i32 %101, -32
+  %108 = icmp ult i32 %104, %107
+  %109 = select i1 %9, i1 %108, i1 false
+  br i1 %109, label %110, label %130
+
+110:                                              ; preds = %106
+  %111 = load i32, ptr @__omp_rtl_debug_kind, align 4
+  %112 = load i32, ptr addrspace(4) @__omp_rtl_device_environment, align 8
+  %113 = and i32 %111, 1
+  %114 = and i32 %113, %112
+  %115 = icmp ne i32 %114, 0
+  br label %116
+
+116:                                              ; preds = %110, %128
+  call void @llvm.lifetime.start.p0(i64 noundef 8, ptr noundef nonnull align 8 dereferenceable(8) %3) #20
+  tail call void @llvm.nvvm.barrier.sync(i32 noundef 8)
+  %117 = call zeroext i1 @__kmpc_kernel_parallel(ptr noalias nocapture nofree noundef nonnull writeonly align 8 dereferenceable(8) %3) #20
+  %118 = load ptr, ptr %3, align 8, !tbaa !93
+  %119 = icmp eq ptr %118, null
+  br i1 %119, label %129, label %120
+
+120:                                              ; preds = %116
+  br i1 %117, label %121, label %128
+
+121:                                              ; preds = %120
+  %122 = load i32, ptr addrspace(3) @IsSPMDMode, align 4
+  %123 = icmp ne i32 %122, 0
+  %124 = select i1 %115, i1 %123, i1 false
+  br i1 %124, label %125, label %126
+
+125:                                              ; preds = %121
+  tail call fastcc void @__assert_fail_internal(ptr noundef nonnull dereferenceable(23) @.str12, ptr noundef null, ptr noundef nonnull dereferenceable(67) @.str15, i32 noundef 60, ptr noundef nonnull dereferenceable(36) @__PRETTY_FUNCTION__._ZL19genericStateMachineP7IdentTy) #22
+  unreachable
+
+126:                                              ; preds = %121
+  %127 = icmp eq i32 %122, 0
+  tail call void @llvm.assume(i1 noundef %127) #23
+  tail call void %118(i32 noundef 0, i32 noundef %104) #24
+  tail call void @__kmpc_kernel_end_parallel() #24
+  br label %128
+
+128:                                              ; preds = %126, %120
+  tail call void @llvm.nvvm.barrier.sync(i32 noundef 8)
+  call void @llvm.lifetime.end.p0(i64 noundef 8, ptr noundef nonnull %3) #20
+  br label %116, !llvm.loop !94
+
+129:                                              ; preds = %116
+  call void @llvm.lifetime.end.p0(i64 noundef 8, ptr noundef nonnull %3) #20
+  br label %130
+
+130:                                              ; preds = %106, %129, %100, %98
+  %131 = phi i32 [ -1, %98 ], [ -1, %100 ], [ %104, %129 ], [ %104, %106 ]
+  ret i32 %131
+}
+
+; Function Attrs: nocallback nofree nosync nounwind speculatable willreturn memory(none)
+declare noundef i32 @llvm.nvvm.read.ptx.sreg.tid.x() #5
+
+; Function Attrs: nocallback nofree nounwind willreturn memory(argmem: write)
+declare void @llvm.memset.p0.i64(ptr nocapture writeonly, i8, i64, i1 immarg) #6
+
+; Function Attrs: convergent mustprogress noinline norecurse nounwind
+define internal void @_ZN4ompx11synchronize14threadsAlignedENS_6atomic10OrderingTyE(i32 %0) local_unnamed_addr #7 {
+  tail call void @llvm.nvvm.barrier0() #25
+  ret void
+}
+
+; Function Attrs: nocallback nofree nosync nounwind speculatable willreturn memory(none)
+declare noundef i32 @llvm.nvvm.read.ptx.sreg.ntid.x() #5
+
+; Function Attrs: cold convergent mustprogress noreturn nounwind
+define internal fastcc void @__assert_fail_internal(ptr noundef nonnull dereferenceable(8) %0, ptr noundef %1, ptr noundef nonnull dereferenceable(66) %2, i32 noundef range(i32 60, 905) %3, ptr noundef nonnull dereferenceable(20) %4) unnamed_addr #8 {
+  %6 = icmp eq ptr %1, null
+  br i1 %6, label %9, label %7
+
+7:                                                ; preds = %5
+  %8 = tail call noundef i32 (ptr, ...) @_ZN4ompx6printfEPKcz(ptr noundef nonnull dereferenceable(40) @.str, ptr noundef nonnull dereferenceable(66) %2, i32 noundef %3, ptr noundef nonnull dereferenceable(20) %4, ptr noundef nonnull %1, ptr noundef nonnull dereferenceable(8) %0) #24
+  br label %11
+
+9:                                                ; preds = %5
+  %10 = tail call noundef i32 (ptr, ...) @_ZN4ompx6printfEPKcz(ptr noundef nonnull dereferenceable(35) @.str1, ptr noundef nonnull dereferenceable(66) %2, i32 noundef %3, ptr noundef nonnull dereferenceable(20) %4, ptr noundef nonnull dereferenceable(8) %0) #24
+  br label %11
+
+11:                                               ; preds = %9, %7
+  tail call void @llvm.trap() #26
+  unreachable
+}
+
+; Function Attrs: nocallback nofree nosync nounwind willreturn memory(inaccessiblemem: write)
+declare void @llvm.assume(i1 noundef) #9
+
+; Function Attrs: nocallback nofree nosync nounwind willreturn memory(argmem: readwrite)
+declare void @llvm.lifetime.start.p0(i64 immarg, ptr nocapture) #10
+
+; Function Attrs: convergent nocallback nounwind
+declare void @llvm.nvvm.barrier.sync(i32) #11
+
+; Function Attrs: convergent mustprogress nofree noinline norecurse nosync nounwind willreturn memory(read, argmem: write, inaccessiblemem: none)
+define internal noundef zeroext i1 @__kmpc_kernel_parallel(ptr nocapture nofree noundef nonnull writeonly align 8 dereferenceable(8) initializes((0, 8)) %0) local_unnamed_addr #12 {
+  %2 = load ptr, ptr addrspace(3) addrspacecast (ptr getelementptr inbounds nuw (i8, ptr addrspacecast (ptr addrspace(3) @_ZN4ompx5state9TeamStateE to ptr), i64 40) to ptr addrspace(3)), align 8, !tbaa !93
+  store ptr %2, ptr %0, align 8, !tbaa !93
+  %3 = icmp eq ptr %2, null
+  br i1 %3, label %15, label %4
+
+4:                                                ; preds = %1
+  %5 = tail call noundef range(i32 0, 1024) i32 @llvm.nvvm.read.ptx.sreg.tid.x() #27, !range !92
+  %6 = load i32, ptr addrspace(3) addrspacecast (ptr getelementptr inbounds nuw (i8, ptr addrspacecast (ptr addrspace(3) @_ZN4ompx5state9TeamStateE to ptr), i64 28) to ptr addrspace(3)), align 4, !tbaa !62
+  %7 = icmp eq i32 %6, 0
+  %8 = tail call range(i32 1, 1025) i32 @llvm.nvvm.read.ptx.sreg.ntid.x(), !range !82
+  %9 = load i32, ptr addrspace(3) @IsSPMDMode, align 4
+  %10 = icmp eq i32 %9, 0
+  %11 = select i1 %10, i32 -32, i32 0
+  %12 = add nsw i32 %11, %8
+  %13 = select i1 %7, i32 %12, i32 %6
+  %14 = icmp ult i32 %5, %13
+  br label %15
+
+15:                                               ; preds = %4, %1
+  %16 = phi i1 [ %14, %4 ], [ false, %1 ]
+  ret i1 %16
+}
+
+; Function Attrs: convergent mustprogress noinline nounwind
+define internal void @__kmpc_kernel_end_parallel() local_unnamed_addr #13 {
+  %1 = load i32, ptr @__omp_rtl_debug_kind, align 4, !tbaa !62
+  %2 = load i32, ptr addrspace(4) @__omp_rtl_device_environment, align 8, !tbaa !83
+  %3 = and i32 %1, 1
+  %4 = and i32 %3, %2
+  %5 = icmp ne i32 %4, 0
+  %6 = load i32, ptr addrspace(3) @IsSPMDMode, align 4
+  %7 = icmp ne i32 %6, 0
+  %8 = select i1 %5, i1 %7, i1 false
+  br i1 %8, label %9, label %10
+
+9:                                                ; preds = %0
+  tail call fastcc void @__assert_fail_internal(ptr noundef nonnull dereferenceable(23) @.str12, ptr noundef null, ptr noundef nonnull dereferenceable(72) @.str1027, i32 noundef 299, ptr noundef nonnull dereferenceable(34) @__PRETTY_FUNCTION__.__kmpc_kernel_end_parallel) #22
+  unreachable
+
+10:                                               ; preds = %0
+  %11 = icmp eq i32 %6, 0
+  tail call void @llvm.assume(i1 noundef %11) #23
+  %12 = load i32, ptr @__omp_rtl_assume_no_thread_state, align 4, !tbaa !62
+  %13 = icmp eq i32 %12, 0
+  %14 = load i32, ptr addrspace(3) addrspacecast (ptr getelementptr inbounds nuw (i8, ptr addrspacecast (ptr addrspace(3) @_ZN4ompx5state9TeamStateE to ptr), i64 32) to ptr addrspace(3)), align 8
+  %15 = icmp ne i32 %14, 0
+  %16 = select i1 %13, i1 %15, i1 false
+  br i1 %16, label %17, label %30
+
+17:                                               ; preds = %10
+  %18 = tail call noundef range(i32 0, 1024) i32 @llvm.nvvm.read.ptx.sreg.tid.x() #27, !range !92
+  %19 = load ptr, ptr addrspace(3) @_ZN4ompx5state12ThreadStatesE, align 8, !tbaa !74
+  %20 = zext nneg i32 %18 to i64
+  %21 = getelementptr inbounds nuw ptr, ptr %19, i64 %20
+  %22 = load ptr, ptr %21, align 8, !tbaa !96
+  %23 = icmp eq ptr %22, null
+  br i1 %23, label %30, label %24, !prof !98
+
+24:                                               ; preds = %17
+  %25 = getelementptr inbounds nuw i8, ptr %22, i64 32
+  %26 = load ptr, ptr %25, align 8, !tbaa !99
+  tail call void @free(ptr noundef nonnull dereferenceable(40) %22) #28
+  %27 = load ptr, ptr addrspace(3) @_ZN4ompx5state12ThreadStatesE, align 8, !tbaa !74
+  %28 = getelementptr inbounds nuw ptr, ptr %27, i64 %20
+  store ptr %26, ptr %28, align 8, !tbaa !96
+  %29 = load i32, ptr addrspace(3) @IsSPMDMode, align 4
+  br label %30
+
+30:                                               ; preds = %10, %17, %24
+  %31 = phi i32 [ 0, %10 ], [ 0, %17 ], [ %29, %24 ]
+  %32 = icmp ne i32 %31, 0
+  %33 = select i1 %5, i1 %32, i1 false
+  br i1 %33, label %34, label %35
+
+34:                                               ; preds = %30
+  tail call fastcc void @__assert_fail_internal(ptr noundef nonnull dereferenceable(23) @.str12, ptr noundef null, ptr noundef nonnull dereferenceable(72) @.str1027, i32 noundef 302, ptr noundef nonnull dereferenceable(34) @__PRETTY_FUNCTION__.__kmpc_kernel_end_parallel) #22
+  unreachable
+
+35:                                               ; preds = %30
+  %36 = icmp eq i32 %31, 0
+  tail call void @llvm.assume(i1 noundef %36) #23
+  ret void
+}
+
+; Function Attrs: nocallback nofree nosync nounwind willreturn memory(argmem: readwrite)
+declare void @llvm.lifetime.end.p0(i64 immarg, ptr nocapture) #10
+
+; Function Attrs: convergent mustprogress nounwind willreturn allockind("free") memory(argmem: readwrite, inaccessiblemem: readwrite)
+declare extern_weak void @free(ptr allocptr nocapture noundef) local_unnamed_addr #14
+
+; Function Attrs: convergent mustprogress nounwind
+define internal noundef i32 @_ZN4ompx6printfEPKcz(ptr noundef %0, ...) local_unnamed_addr #15 {
+  %2 = alloca ptr, align 8
+  call void @llvm.lifetime.start.p0(i64 noundef 8, ptr noundef nonnull align 8 %2) #29
+  call void @llvm.va_start.p0(ptr noundef nonnull align 8 %2) #27
+  %3 = load ptr, ptr %2, align 8, !tbaa !101
+  %4 = call i32 @vprintf(ptr noundef %0, ptr noundef %3) #24
+  call void @llvm.lifetime.end.p0(i64 noundef 8, ptr noundef nonnull %2) #20
+  ret i32 %4
+}
+
+; Function Attrs: cold noreturn nounwind memory(inaccessiblemem: write)
+declare void @llvm.trap() #16
+
+; Function Attrs: nocallback nofree nosync nounwind willreturn
+declare void @llvm.va_start.p0(ptr) #17
+
+; Function Attrs: convergent nounwind
+declare i32 @vprintf(ptr noundef, ptr noundef) local_unnamed_addr #18
+
+; Function Attrs: convergent nocallback nounwind
+declare void @llvm.nvvm.barrier0() #11
+
+; Function Attrs: convergent mustprogress nounwind
+define internal void @__kmpc_target_deinit() #4 {
+  %1 = alloca ptr, align 8
+  %2 = load i32, ptr addrspace(3) @IsSPMDMode, align 4, !tbaa !62
+  %3 = icmp eq i32 %2, 0
+  br i1 %3, label %4, label %27
+
+4:                                                ; preds = %0
+  %5 = tail call range(i32 1, 1025) i32 @llvm.nvvm.read.ptx.sreg.ntid.x(), !range !82
+  %6 = add nsw i32 %5, -1
+  %7 = and i32 %6, -32
+  %8 = tail call range(i32 0, 1024) i32 @llvm.nvvm.read.ptx.sreg.tid.x(), !range !92
+  %9 = icmp eq i32 %8, %7
+  br i1 %9, label %10, label %11
+
+10:                                               ; preds = %4
+  store ptr null, ptr addrspace(3) addrspacecast (ptr getelementptr inbounds nuw (i8, ptr addrspacecast (ptr addrspace(3) @_ZN4ompx5state9TeamStateE to ptr), i64 40) to ptr addrspace(3)), align 8, !tbaa !93
+  br label %27
+
+11:                                               ; preds = %4
+  %12 = load ptr, ptr addrspace(3) @_ZL20KernelEnvironmentPtr, align 8, !tbaa !76
+  %13 = load i8, ptr %12, align 8, !tbaa !103
+  %14 = icmp eq i8 %13, 0
+  br i1 %14, label %15, label %27
+
+15:                                               ; preds = %11
+  call void @llvm.lifetime.start.p0(i64 noundef 8, ptr noundef nonnull align 8 dereferenceable(8) %1) #29
+  %16 = call zeroext i1 @__kmpc_kernel_parallel(ptr noalias nocapture nofree noundef nonnull writeonly align 8 dereferenceable(8) %1) #20
+  %17 = load i32, ptr @__omp_rtl_debug_kind, align 4, !tbaa !62
+  %18 = load i32, ptr addrspace(4) @__omp_rtl_device_environment, align 8, !tbaa !83
+  %19 = and i32 %17, 1
+  %20 = and i32 %19, %18
+  %21 = icmp eq i32 %20, 0
+  %22 = load ptr, ptr %1, align 8
+  %23 = icmp eq ptr %22, null
+  %24 = select i1 %21, i1 true, i1 %23
+  br i1 %24, label %26, label %25
+
+25:                                               ; preds = %15
+  tail call fastcc void @__assert_fail_internal(ptr noundef nonnull dereferenceable(18) @.str2, ptr noundef null, ptr noundef nonnull dereferenceable(67) @.str15, i32 noundef 152, ptr noundef nonnull dereferenceable(28) @__PRETTY_FUNCTION__.__kmpc_target_deinit) #22
+  unreachable
+
+26:                                               ; preds = %15
+  tail call void @llvm.assume(i1 noundef %23) #23
+  call void @llvm.lifetime.end.p0(i64 noundef 8, ptr noundef nonnull %1) #20
+  br label %27
+
+27:                                               ; preds = %26, %11, %10, %0
+  ret void
+}
+
+attributes #0 = { convergent noinline norecurse nounwind optnone "frame-pointer"="all" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="sm_70" "target-features"="+ptx83,+sm_70" }
+attributes #1 = { convergent mustprogress noinline norecurse nounwind optnone "frame-pointer"="all" "kernel" "no-trapping-math"="true" "omp_target_thread_limit"="128" "stack-protector-buffer-size"="8" "target-cpu"="sm_70" "target-features"="+ptx83,+sm_70" }
+attributes #2 = { convergent "frame-pointer"="all" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="sm_70" "target-features"="+ptx83,+sm_70" }
+attributes #3 = { convergent noinline nounwind optnone "frame-pointer"="all" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="sm_70" "target-features"="+ptx83,+sm_70" }
+attributes #4 = { convergent mustprogress nounwind "frame-pointer"="all" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="sm_70" "target-features"="+ptx63,+ptx83,+sm_70" }
+attributes #5 = { nocallback nofree nosync nounwind speculatable willreturn memory(none) }
+attributes #6 = { nocallback nofree nounwind willreturn memory(argmem: write) }
+attributes #7 = { convergent mustprogress noinline norecurse nounwind "frame-pointer"="all" "llvm.assume"="ompx_aligned_barrier" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="sm_70" "target-features"="+ptx63,+ptx83,+sm_70" }
+attributes #8 = { cold convergent mustprogress noreturn nounwind "frame-pointer"="all" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="sm_70" "target-features"="+ptx63,+ptx83,+sm_70" }
+attributes #9 = { nocallback nofree nosync nounwind willreturn memory(inaccessiblemem: write) }
+attributes #10 = { nocallback nofree nosync nounwind willreturn memory(argmem: readwrite) }
+attributes #11 = { convergent nocallback nounwind }
+attributes #12 = { convergent mustprogress nofree noinline norecurse nosync nounwind willreturn memory(read, argmem: write, inaccessiblemem: none) "frame-pointer"="all" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="sm_70" "target-features"="+ptx63,+ptx83,+sm_70" }
+attributes #13 = { convergent mustprogress noinline nounwind "frame-pointer"="all" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="sm_70" "target-features"="+ptx63,+ptx83,+sm_70" }
+attributes #14 = { convergent mustprogress nounwind willreturn allockind("free") memory(argmem: readwrite, inaccessiblemem: readwrite) "alloc-family"="malloc" "frame-pointer"="all" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="sm_70" "target-features"="+ptx63,+ptx83,+sm_70" }
+attributes #15 = { convergent mustprogress nounwind "frame-pointer"="all" "no-builtin-printf" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="sm_70" "target-features"="+ptx63,+ptx83,+sm_70" }
+attributes #16 = { cold noreturn nounwind memory(inaccessiblemem: write) }
+attributes #17 = { nocallback nofree nosync nounwind willreturn }
+attributes #18 = { convergent nounwind "frame-pointer"="all" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="sm_70" "target-features"="+ptx63,+ptx83,+sm_70" }
+attributes #19 = { convergent }
+attributes #20 = { nounwind }
+attributes #21 = { convergent nounwind "llvm.assume"="ompx_aligned_barrier" }
+attributes #22 = { convergent noreturn nounwind }
+attributes #23 = { memory(write) }
+attributes #24 = { convergent nounwind }
+attributes #25 = { "llvm.assume"="ompx_aligned_barrier" }
+attributes #26 = { noreturn }
+attributes #27 = { nofree willreturn }
+attributes #28 = { convergent nounwind willreturn }
+attributes #29 = { nofree nounwind willreturn }
+
+!llvm.module.flags = !{!0, !1, !2, !3, !4, !5, !6, !7, !8, !9, !10}
+!llvm.dbg.cu = !{!11}
+!nvvm.annotations = !{!13}
+!omp_offload.info = !{!14}
+!llvm.ident = !{!15, !16, !15, !15, !15, !15, !15, !15, !15, !15, !15, !15, !15, !15, !15, !15, !15}
+!nvvmir.version = !{!17}
+
+!0 = !{i32 2, !"SDK Version", [2 x i32] [i32 12, i32 3]}
+!1 = !{i32 7, !"Dwarf Version", i32 2}
+!2 = !{i32 2, !"Debug Info Version", i32 3}
+!3 = !{i32 1, !"wchar_size", i32 4}
+!4 = !{i32 4, !"nvvm-reflect-ftz", i32 0}
+!5 = !{i32 7, !"openmp", i32 51}
+!6 = !{i32 7, !"openmp-device", i32 51}
+!7 = !{i32 8, !"PIC Level", i32 2}
+!8 = !{i32 7, !"frame-pointer", i32 2}
+!9 = !{i32 1, !"ThinLTO", i32 0}
+!10 = !{i32 1, !"EnableSplitLTOUnit", i32 1}
+!11 = distinct !DICompileUnit(language: DW_LANG_C11, file: !12, producer: "clang version 20.0.0git (/tmp/llvm/clang b9447c03a9ef2eed55b685a33511df86f7f94e89)", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, splitDebugInlining: false, nameTableKind: None)
+!12 = !DIFile(filename: "test.c", directory: "/tmp")
+!13 = !{ptr @__omp_offloading_fd02_1116d6_h_l12, !"maxntidx", i32 128}
+!14 = !{i32 0, i32 64770, i32 1119958, !"h", i32 12, i32 0, i32 0}
+!15 = !{!"clang version 20.0.0git (/tmp/llvm/clang b9447c03a9ef2eed55b685a33511df86f7f94e89)"}
+!16 = !{!"clang version 3.8.0 (tags/RELEASE_380/final)"}
+!17 = !{i32 2, i32 0}
+!18 = distinct !DISubprogram(name: "__omp_offloading_fd02_1116d6_h_l12_debug__", scope: !12, file: !12, line: 13, type: !19, scopeLine: 13, flags: DIFlagArtificial | DIFlagPrototyped, spFlags: DISPFlagLocalToUnit | DISPFlagDefinition, unit: !11, retainedNodes: !24)
+!19 = !DISubroutineType(types: !20)
+!20 = !{null, !21}
+!21 = !DIDerivedType(tag: DW_TAG_const_type, baseType: !22)
+!22 = !DIDerivedType(tag: DW_TAG_restrict_type, baseType: !23)
+!23 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: null, size: 64)
+!24 = !{}
+!25 = !DILocalVariable(name: "dyn_ptr", arg: 1, scope: !18, type: !21, flags: DIFlagArtificial)
+!26 = !DILocation(line: 0, scope: !18)
+!27 = !DILocation(line: 13, column: 3, scope: !18)
+!28 = !DILocalVariable(name: "i", scope: !29, file: !12, line: 14, type: !30)
+!29 = distinct !DILexicalBlock(scope: !18, file: !12, line: 13, column: 3)
+!30 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+!31 = !DILocation(line: 14, column: 9, scope: !29)
+!32 = !DILocalVariable(name: "a", scope: !29, file: !12, line: 15, type: !33)
+!33 = !DICompositeType(tag: DW_TAG_array_type, baseType: !30, size: 64, elements: !34)
+!34 = !{!35}
+!35 = !DISubrange(count: 2)
+!36 = !DILocation(line: 15, column: 9, scope: !29)
+!37 = !DILocation(line: 16, column: 5, scope: !29)
+!38 = !DILocation(line: 17, column: 5, scope: !29)
+!39 = !DILocation(line: 18, column: 3, scope: !29)
+!40 = !DILocation(line: 18, column: 3, scope: !18)
+!41 = distinct !DISubprogram(name: "__omp_offloading_fd02_1116d6_h_l12", scope: !12, file: !12, line: 12, type: !19, scopeLine: 12, flags: DIFlagArtificial | DIFlagPrototyped, spFlags: DISPFlagLocalToUnit | DISPFlagDefinition, unit: !11, retainedNodes: !24)
+!42 = !DILocalVariable(name: "dyn_ptr", arg: 1, scope: !41, type: !21, flags: DIFlagArtificial)
+!43 = !DILocation(line: 0, scope: !41)
+!44 = !DILocation(line: 12, column: 1, scope: !41)
+!45 = distinct !DISubprogram(name: "g", scope: !12, file: !12, line: 3, type: !46, scopeLine: 3, spFlags: DISPFlagDefinition, unit: !11, retainedNodes: !24)
+!46 = !DISubroutineType(types: !47)
+!47 = !{null}
+!48 = !DILocalVariable(name: "i", scope: !45, file: !12, line: 4, type: !30)
+!49 = !DILocation(line: 4, column: 7, scope: !45)
+!50 = !DILocalVariable(name: "a", scope: !45, file: !12, line: 5, type: !33)
+!51 = !DILocation(line: 5, column: 7, scope: !45)
+!52 = !DILocation(line: 6, column: 3, scope: !45)
+!53 = !DILocation(line: 7, column: 3, scope: !45)
+!54 = !DILocation(line: 8, column: 1, scope: !45)
+!55 = !{!56, !59, i64 2}
+!56 = !{!"_ZTS26ConfigurationEnvironmentTy", !57, i64 0, !57, i64 1, !59, i64 2, !60, i64 4, !60, i64 8, !60, i64 12, !60, i64 16, !60, i64 20, !60, i64 24}
+!57 = !{!"omnipotent char", !58, i64 0}
+!58 = !{!"Simple C++ TBAA"}
+!59 = !{!"_ZTSN4llvm3omp19OMPTgtExecModeFlagsE", !57, i64 0}
+!60 = !{!"int", !57, i64 0}
+!61 = !{!56, !57, i64 0}
+!62 = !{!60, !60, i64 0}
+!63 = !{!57, !57, i64 0}
+!64 = !{!65, !60, i64 16}
+!65 = !{!"_ZTSN4ompx5state11TeamStateTyE", !66, i64 0, !60, i64 28, !60, i64 32, !67, i64 40}
+!66 = !{!"_ZTSN4ompx5state10ICVStateTyE", !60, i64 0, !60, i64 4, !60, i64 8, !60, i64 12, !60, i64 16, !60, i64 20, !60, i64 24}
+!67 = !{!"p1 void", !68, i64 0}
+!68 = !{!"any pointer", !57, i64 0}
+!69 = !{!65, !60, i64 20}
+!70 = !{!65, !60, i64 24}
+!71 = !{!65, !60, i64 28}
+!72 = !{!65, !60, i64 32}
+!73 = !{!65, !67, i64 40}
+!74 = !{!75, !75, i64 0}
+!75 = !{!"p2 _ZTSN4ompx5state13ThreadStateTyE", !68, i64 0}
+!76 = !{!77, !77, i64 0}
+!77 = !{!"p1 _ZTS19KernelEnvironmentTy", !68, i64 0}
+!78 = !{!79, !79, i64 0}
+!79 = !{!"p1 _ZTS25KernelLaunchEnvironmentTy", !68, i64 0}
+!80 = !{!81, !81, i64 0}
+!81 = !{!"p2 _ZTS22DynamicScheduleTracker", !68, i64 0}
+!82 = !{i32 1, i32 1025}
+!83 = !{!84, !60, i64 0}
+!84 = !{!"_ZTS19DeviceEnvironmentTy", !60, i64 0, !60, i64 4, !60, i64 8, !60, i64 12, !85, i64 16, !85, i64 24, !85, i64 32, !85, i64 40}
+!85 = !{!"long", !57, i64 0}
+!86 = !{!66, !60, i64 0}
+!87 = !{!66, !60, i64 4}
+!88 = !{!66, !60, i64 8}
+!89 = !{!66, !60, i64 16}
+!90 = !{!66, !60, i64 20}
+!91 = !{!66, !60, i64 24}
+!92 = !{i32 0, i32 1024}
+!93 = !{!67, !67, i64 0}
+!94 = distinct !{!94, !95}
+!95 = !{!"llvm.loop.mustprogress"}
+!96 = !{!97, !97, i64 0}
+!97 = !{!"p1 _ZTSN4ompx5state13ThreadStateTyE", !68, i64 0}
+!98 = !{!"branch_weights", !"expected", i32 2000, i32 1}
+!99 = !{!100, !97, i64 32}
+!100 = !{!"_ZTSN4ompx5state13ThreadStateTyE", !66, i64 0, !97, i64 32}
+!101 = !{!102, !102, i64 0}
+!102 = !{!"p1 omnipotent char", !68, i64 0}
+!103 = !{!104, !57, i64 0}
+!104 = !{!"_ZTS19KernelEnvironmentTy", !56, i64 0, !105, i64 32, !106, i64 40}
+!105 = !{!"p1 _ZTS7IdentTy", !68, i64 0}
+!106 = !{!"p1 _ZTS20DynamicEnvironmentTy", !68, i64 0}
diff --git a/llvm/test/Analysis/LoopAccessAnalysis/invariant-dependence-before.ll b/llvm/test/Analysis/LoopAccessAnalysis/invariant-dependence-before.ll
index 2139804753ef5..c0b044aef0d62 100644
--- a/llvm/test/Analysis/LoopAccessAnalysis/invariant-dependence-before.ll
+++ b/llvm/test/Analysis/LoopAccessAnalysis/invariant-dependence-before.ll
@@ -784,3 +784,38 @@ loop:
 exit:
   ret void
 }
+
+define void @invariant_stores_unit_stride(i32 %offset, ptr noalias %dst.1, ptr %dst.2) {
+; CHECK-LABEL: 'invariant_stores_unit_stride'
+; CHECK-NEXT:    loop:
+; CHECK-NEXT:      Memory dependences are safe
+; CHECK-NEXT:      Dependences:
+; CHECK-NEXT:      Run-time memory checks:
+; CHECK-NEXT:      Grouped accesses:
+; CHECK-EMPTY:
+; CHECK-NEXT:      Non vectorizable stores to invariant address were not found in loop.
+; CHECK-NEXT:      SCEV assumptions:
+; CHECK-EMPTY:
+; CHECK-NEXT:      Expressions re-written:
+;
+entry:
+  %add = add i32 %offset, 3
+  br label %loop
+
+loop:
+  %iv.2 = phi i32 [ 0, %entry ], [ %iv.2.next, %loop ]
+  %iv.3 = phi i32 [ 0, %entry ], [ %iv.3.next, %loop ]
+  %iv.mul  = mul i32 %iv.3, %add
+  %gep.mul = getelementptr i8, ptr %dst.1, i32 %iv.mul
+  store i32 0, ptr %gep.mul, align 8
+  %iv.2.mul  = mul i32 %iv.2, %offset
+  %gep = getelementptr i32, ptr %dst.2, i32 %iv.2.mul
+  store i32 0, ptr %gep, align 8
+  %iv.2.next = add i32 %iv.2, 1
+  %iv.3.next = add i32 %iv.3, 1
+  %ec = icmp eq i32 %iv.3, 200
+  br i1 %ec, label %exit, label %loop
+
+exit:
+  ret void
+}
diff --git a/llvm/test/Analysis/LoopAccessAnalysis/pointer-phis.ll b/llvm/test/Analysis/LoopAccessAnalysis/pointer-phis.ll
index a214451bfd3fd..48586ee9d9ed9 100644
--- a/llvm/test/Analysis/LoopAccessAnalysis/pointer-phis.ll
+++ b/llvm/test/Analysis/LoopAccessAnalysis/pointer-phis.ll
@@ -501,14 +501,6 @@ define void @phi_load_store_memdep_check(i1 %c, ptr %A, ptr %B, ptr %C) {
 ; CHECK-NEXT:            %lv3 = load i16, ptr %c.sink, align 2 ->
 ; CHECK-NEXT:            store i16 %add, ptr %c.sink, align 1
 ; CHECK-EMPTY:
-; CHECK-NEXT:        Unknown:
-; CHECK-NEXT:            %lv = load i16, ptr %A, align 1 ->
-; CHECK-NEXT:            store i16 %lv, ptr %A, align 1
-; CHECK-EMPTY:
-; CHECK-NEXT:        Unknown:
-; CHECK-NEXT:            store i16 %lv, ptr %A, align 1 ->
-; CHECK-NEXT:            %lv2 = load i16, ptr %A, align 1
-; CHECK-EMPTY:
 ; CHECK-NEXT:      Run-time memory checks:
 ; CHECK-NEXT:      Check 0:
 ; CHECK-NEXT:        Comparing group ([[GRP10:0x[0-9a-f]+]]):
diff --git a/llvm/test/Analysis/ScalarEvolution/exit-count-non-strict.ll b/llvm/test/Analysis/ScalarEvolution/exit-count-non-strict.ll
index f7a18c77a82c8..1e15d2d0d6461 100644
--- a/llvm/test/Analysis/ScalarEvolution/exit-count-non-strict.ll
+++ b/llvm/test/Analysis/ScalarEvolution/exit-count-non-strict.ll
@@ -30,6 +30,35 @@ exit:
   ret void
 }
 
+define void @le_from_zero(i32 %M, i32 %N) {
+; CHECK-LABEL: 'le_from_zero'
+; CHECK-NEXT:  Determining loop execution counts for: @le_from_zero
+; CHECK-NEXT:  Loop %loop: <multiple exits> backedge-taken count is ((zext i32 %N to i64) umin (1 + (zext i32 %M to i64))<nuw><nsw>)
+; CHECK-NEXT:    exit count for loop: (1 + (zext i32 %M to i64))<nuw><nsw>
+; CHECK-NEXT:    exit count for latch: %N
+; CHECK-NEXT:  Loop %loop: constant max backedge-taken count is i64 4294967295
+; CHECK-NEXT:  Loop %loop: symbolic max backedge-taken count is ((zext i32 %N to i64) umin (1 + (zext i32 %M to i64))<nuw><nsw>)
+; CHECK-NEXT:    symbolic max exit count for loop: (1 + (zext i32 %M to i64))<nuw><nsw>
+; CHECK-NEXT:    symbolic max exit count for latch: %N
+; CHECK-NEXT:  Loop %loop: Trip multiple is 1
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i32 [ 0, %entry ], [ %iv.next, %latch ]
+  %cmp1 = icmp samesign ule i32 %iv, %M
+  br i1 %cmp1, label %latch, label %exit
+
+latch:
+  %iv.next = add nuw i32 %iv, 1
+  %exitcond.not = icmp eq i32 %iv, %N
+  br i1 %exitcond.not, label %exit, label %loop
+
+exit:
+  ret void
+}
+
 define void @ule_from_one(i32 %M, i32 %N) {
 ; CHECK-LABEL: 'ule_from_one'
 ; CHECK-NEXT:  Determining loop execution counts for: @ule_from_one
@@ -59,6 +88,35 @@ exit:
   ret void
 }
 
+define void @le_from_one(i32 %M, i32 %N) {
+; CHECK-LABEL: 'le_from_one'
+; CHECK-NEXT:  Determining loop execution counts for: @le_from_one
+; CHECK-NEXT:  Loop %loop: <multiple exits> backedge-taken count is (%M umin_seq (-1 + %N))
+; CHECK-NEXT:    exit count for loop: %M
+; CHECK-NEXT:    exit count for latch: (-1 + %N)
+; CHECK-NEXT:  Loop %loop: constant max backedge-taken count is i32 -1
+; CHECK-NEXT:  Loop %loop: symbolic max backedge-taken count is (%M umin_seq (-1 + %N))
+; CHECK-NEXT:    symbolic max exit count for loop: %M
+; CHECK-NEXT:    symbolic max exit count for latch: (-1 + %N)
+; CHECK-NEXT:  Loop %loop: Trip multiple is 1
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i32 [ 1, %entry ], [ %iv.next, %latch ]
+  %cmp1 = icmp samesign ule i32 %iv, %M
+  br i1 %cmp1, label %latch, label %exit
+
+latch:
+  %iv.next = add nuw i32 %iv, 1
+  %exitcond.not = icmp eq i32 %iv, %N
+  br i1 %exitcond.not, label %exit, label %loop
+
+exit:
+  ret void
+}
+
 define void @ule_from_unknown(i32 %M, i32 %N, i32 %S) {
 ; CHECK-LABEL: 'ule_from_unknown'
 ; CHECK-NEXT:  Determining loop execution counts for: @ule_from_unknown
@@ -133,6 +191,51 @@ exit:
   ret void
 }
 
+define void @le_from_zero_no_nuw(i32 %M, i32 %N) {
+; CHECK-LABEL: 'le_from_zero_no_nuw'
+; CHECK-NEXT:  Determining loop execution counts for: @le_from_zero_no_nuw
+; CHECK-NEXT:  Loop %loop: <multiple exits> Unpredictable backedge-taken count.
+; CHECK-NEXT:    exit count for loop: ***COULDNOTCOMPUTE***
+; CHECK-NEXT:    predicated exit count for loop: (1 + (zext i32 %M to i64))<nuw><nsw>
+; CHECK-NEXT:     Predicates:
+; CHECK-NEXT:      {0,+,1}<%loop> Added Flags: <nusw>
+; CHECK-EMPTY:
+; CHECK-NEXT:    exit count for latch: %N
+; CHECK-NEXT:  Loop %loop: constant max backedge-taken count is i32 -1
+; CHECK-NEXT:  Loop %loop: symbolic max backedge-taken count is %N
+; CHECK-NEXT:    symbolic max exit count for loop: ***COULDNOTCOMPUTE***
+; CHECK-NEXT:    predicated symbolic max exit count for loop: (1 + (zext i32 %M to i64))<nuw><nsw>
+; CHECK-NEXT:     Predicates:
+; CHECK-NEXT:      {0,+,1}<%loop> Added Flags: <nusw>
+; CHECK-EMPTY:
+; CHECK-NEXT:    symbolic max exit count for latch: %N
+; CHECK-NEXT:  Loop %loop: Predicated backedge-taken count is ((zext i32 %N to i64) umin (1 + (zext i32 %M to i64))<nuw><nsw>)
+; CHECK-NEXT:   Predicates:
+; CHECK-NEXT:      {0,+,1}<%loop> Added Flags: <nusw>
+; CHECK-NEXT:  Loop %loop: Predicated constant max backedge-taken count is i64 4294967295
+; CHECK-NEXT:   Predicates:
+; CHECK-NEXT:      {0,+,1}<%loop> Added Flags: <nusw>
+; CHECK-NEXT:  Loop %loop: Predicated symbolic max backedge-taken count is ((zext i32 %N to i64) umin (1 + (zext i32 %M to i64))<nuw><nsw>)
+; CHECK-NEXT:   Predicates:
+; CHECK-NEXT:      {0,+,1}<%loop> Added Flags: <nusw>
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i32 [ 0, %entry ], [ %iv.next, %latch ]
+  %cmp1 = icmp samesign ule i32 %iv, %M
+  br i1 %cmp1, label %latch, label %exit
+
+latch:
+  %iv.next = add i32 %iv, 1
+  %exitcond.not = icmp eq i32 %iv, %N
+  br i1 %exitcond.not, label %exit, label %loop
+
+exit:
+  ret void
+}
+
 define void @sle_from_int_min(i32 %M, i32 %N) {
 ; CHECK-LABEL: 'sle_from_int_min'
 ; CHECK-NEXT:  Determining loop execution counts for: @sle_from_int_min
@@ -162,6 +265,35 @@ exit:
   ret void
 }
 
+define void @le_from_int_min(i32 %M, i32 %N) {
+; CHECK-LABEL: 'le_from_int_min'
+; CHECK-NEXT:  Determining loop execution counts for: @le_from_int_min
+; CHECK-NEXT:  Loop %loop: <multiple exits> backedge-taken count is ((-2147483647 + (2147483647 umax %M)) umin_seq (-2147483648 + %N))
+; CHECK-NEXT:    exit count for loop: (-2147483647 + (2147483647 umax %M))
+; CHECK-NEXT:    exit count for latch: (-2147483648 + %N)
+; CHECK-NEXT:  Loop %loop: constant max backedge-taken count is i32 -2147483648
+; CHECK-NEXT:  Loop %loop: symbolic max backedge-taken count is ((-2147483647 + (2147483647 umax %M)) umin_seq (-2147483648 + %N))
+; CHECK-NEXT:    symbolic max exit count for loop: (-2147483647 + (2147483647 umax %M))
+; CHECK-NEXT:    symbolic max exit count for latch: (-2147483648 + %N)
+; CHECK-NEXT:  Loop %loop: Trip multiple is 1
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i32 [ u0x80000000, %entry ], [ %iv.next, %latch ]
+  %cmp1 = icmp samesign ule i32 %iv, %M
+  br i1 %cmp1, label %latch, label %exit
+
+latch:
+  %iv.next = add nuw nsw i32 %iv, 1
+  %exitcond.not = icmp eq i32 %iv, %N
+  br i1 %exitcond.not, label %exit, label %loop
+
+exit:
+  ret void
+}
+
 define void @sle_from_int_min_plus_one(i32 %M, i32 %N) {
 ; CHECK-LABEL: 'sle_from_int_min_plus_one'
 ; CHECK-NEXT:  Determining loop execution counts for: @sle_from_int_min_plus_one
@@ -191,6 +323,35 @@ exit:
   ret void
 }
 
+define void @le_from_int_min_plus_one(i32 %M, i32 %N) {
+; CHECK-LABEL: 'le_from_int_min_plus_one'
+; CHECK-NEXT:  Determining loop execution counts for: @le_from_int_min_plus_one
+; CHECK-NEXT:  Loop %loop: <multiple exits> backedge-taken count is ((-2147483648 + (-2147483648 umax %M)) umin_seq (2147483647 + %N))
+; CHECK-NEXT:    exit count for loop: (-2147483648 + (-2147483648 umax %M))
+; CHECK-NEXT:    exit count for latch: (2147483647 + %N)
+; CHECK-NEXT:  Loop %loop: constant max backedge-taken count is i32 2147483647
+; CHECK-NEXT:  Loop %loop: symbolic max backedge-taken count is ((-2147483648 + (-2147483648 umax %M)) umin_seq (2147483647 + %N))
+; CHECK-NEXT:    symbolic max exit count for loop: (-2147483648 + (-2147483648 umax %M))
+; CHECK-NEXT:    symbolic max exit count for latch: (2147483647 + %N)
+; CHECK-NEXT:  Loop %loop: Trip multiple is 1
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i32 [ u0x80000001, %entry ], [ %iv.next, %latch ]
+  %cmp1 = icmp samesign ule i32 %iv, %M
+  br i1 %cmp1, label %latch, label %exit
+
+latch:
+  %iv.next = add nuw nsw i32 %iv, 1
+  %exitcond.not = icmp eq i32 %iv, %N
+  br i1 %exitcond.not, label %exit, label %loop
+
+exit:
+  ret void
+}
+
 define void @sle_from_unknown(i32 %M, i32 %N, i32 %S) {
 ; CHECK-LABEL: 'sle_from_unknown'
 ; CHECK-NEXT:  Determining loop execution counts for: @sle_from_unknown
@@ -220,6 +381,35 @@ exit:
   ret void
 }
 
+define void @le_from_unknown(i32 %M, i32 %N, i32 %S) {
+; CHECK-LABEL: 'le_from_unknown'
+; CHECK-NEXT:  Determining loop execution counts for: @le_from_unknown
+; CHECK-NEXT:  Loop %loop: <multiple exits> backedge-taken count is (((-1 * (zext i32 %S to i64))<nsw> + ((zext i32 %S to i64) umax (1 + (zext i32 %M to i64))<nuw><nsw>)) umin_seq (zext i32 ((-1 * %S) + %N) to i64))
+; CHECK-NEXT:    exit count for loop: ((-1 * (zext i32 %S to i64))<nsw> + ((zext i32 %S to i64) umax (1 + (zext i32 %M to i64))<nuw><nsw>))
+; CHECK-NEXT:    exit count for latch: ((-1 * %S) + %N)
+; CHECK-NEXT:  Loop %loop: constant max backedge-taken count is i64 4294967295
+; CHECK-NEXT:  Loop %loop: symbolic max backedge-taken count is (((-1 * (zext i32 %S to i64))<nsw> + ((zext i32 %S to i64) umax (1 + (zext i32 %M to i64))<nuw><nsw>)) umin_seq (zext i32 ((-1 * %S) + %N) to i64))
+; CHECK-NEXT:    symbolic max exit count for loop: ((-1 * (zext i32 %S to i64))<nsw> + ((zext i32 %S to i64) umax (1 + (zext i32 %M to i64))<nuw><nsw>))
+; CHECK-NEXT:    symbolic max exit count for latch: ((-1 * %S) + %N)
+; CHECK-NEXT:  Loop %loop: Trip multiple is 1
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i32 [ %S, %entry ], [ %iv.next, %latch ]
+  %cmp1 = icmp samesign ule i32 %iv, %M
+  br i1 %cmp1, label %latch, label %exit
+
+latch:
+  %iv.next = add nuw nsw i32 %iv, 1
+  %exitcond.not = icmp eq i32 %iv, %N
+  br i1 %exitcond.not, label %exit, label %loop
+
+exit:
+  ret void
+}
+
 define void @sle_from_int_min_no_nsw(i32 %M, i32 %N) {
 ; CHECK-LABEL: 'sle_from_int_min_no_nsw'
 ; CHECK-NEXT:  Determining loop execution counts for: @sle_from_int_min_no_nsw
@@ -264,3 +454,48 @@ latch:
 exit:
   ret void
 }
+
+define void @le_from_int_min_no_nuw_nsw(i32 %M, i32 %N) {
+; CHECK-LABEL: 'le_from_int_min_no_nuw_nsw'
+; CHECK-NEXT:  Determining loop execution counts for: @le_from_int_min_no_nuw_nsw
+; CHECK-NEXT:  Loop %loop: <multiple exits> Unpredictable backedge-taken count.
+; CHECK-NEXT:    exit count for loop: ***COULDNOTCOMPUTE***
+; CHECK-NEXT:    predicated exit count for loop: (-2147483648 + (2147483648 umax (1 + (zext i32 %M to i64))<nuw><nsw>))<nsw>
+; CHECK-NEXT:     Predicates:
+; CHECK-NEXT:      {-2147483648,+,1}<%loop> Added Flags: <nusw>
+; CHECK-EMPTY:
+; CHECK-NEXT:    exit count for latch: (-2147483648 + %N)
+; CHECK-NEXT:  Loop %loop: constant max backedge-taken count is i32 -1
+; CHECK-NEXT:  Loop %loop: symbolic max backedge-taken count is (-2147483648 + %N)
+; CHECK-NEXT:    symbolic max exit count for loop: ***COULDNOTCOMPUTE***
+; CHECK-NEXT:    predicated symbolic max exit count for loop: (-2147483648 + (2147483648 umax (1 + (zext i32 %M to i64))<nuw><nsw>))<nsw>
+; CHECK-NEXT:     Predicates:
+; CHECK-NEXT:      {-2147483648,+,1}<%loop> Added Flags: <nusw>
+; CHECK-EMPTY:
+; CHECK-NEXT:    symbolic max exit count for latch: (-2147483648 + %N)
+; CHECK-NEXT:  Loop %loop: Predicated backedge-taken count is ((-2147483648 + (2147483648 umax (1 + (zext i32 %M to i64))<nuw><nsw>))<nsw> umin_seq (zext i32 (-2147483648 + %N) to i64))
+; CHECK-NEXT:   Predicates:
+; CHECK-NEXT:      {-2147483648,+,1}<%loop> Added Flags: <nusw>
+; CHECK-NEXT:  Loop %loop: Predicated constant max backedge-taken count is i64 2147483648
+; CHECK-NEXT:   Predicates:
+; CHECK-NEXT:      {-2147483648,+,1}<%loop> Added Flags: <nusw>
+; CHECK-NEXT:  Loop %loop: Predicated symbolic max backedge-taken count is ((-2147483648 + (2147483648 umax (1 + (zext i32 %M to i64))<nuw><nsw>))<nsw> umin_seq (zext i32 (-2147483648 + %N) to i64))
+; CHECK-NEXT:   Predicates:
+; CHECK-NEXT:      {-2147483648,+,1}<%loop> Added Flags: <nusw>
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i32 [ u0x80000000, %entry ], [ %iv.next, %latch ]
+  %cmp1 = icmp samesign ule i32 %iv, %M
+  br i1 %cmp1, label %latch, label %exit
+
+latch:
+  %iv.next = add i32 %iv, 1
+  %exitcond.not = icmp eq i32 %iv, %N
+  br i1 %exitcond.not, label %exit, label %loop
+
+exit:
+  ret void
+}
diff --git a/llvm/test/Analysis/ScalarEvolution/exit-count-samesign.ll b/llvm/test/Analysis/ScalarEvolution/exit-count-samesign.ll
new file mode 100644
index 0000000000000..93c6bc08af2a0
--- /dev/null
+++ b/llvm/test/Analysis/ScalarEvolution/exit-count-samesign.ll
@@ -0,0 +1,42 @@
+; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -disable-output "-passes=print<scalar-evolution>" \
+; RUN:  -scalar-evolution-classify-expressions=0 < %s 2>&1 | FileCheck %s
+
+define i32 @exit_count_samesign(i32 %iter.count, ptr %ptr) {
+; CHECK-LABEL: 'exit_count_samesign'
+; CHECK-NEXT:  Determining loop execution counts for: @exit_count_samesign
+; CHECK-NEXT:  Loop %inner.loop: backedge-taken count is (-1 + (1 smax {(-1 + %iter.count)<nsw>,+,-1}<nsw><%outer.loop>))<nsw>
+; CHECK-NEXT:  Loop %inner.loop: constant max backedge-taken count is i32 2147483646
+; CHECK-NEXT:  Loop %inner.loop: symbolic max backedge-taken count is (-1 + (1 smax {(-1 + %iter.count)<nsw>,+,-1}<nsw><%outer.loop>))<nsw>
+; CHECK-NEXT:  Loop %inner.loop: Trip multiple is 1
+; CHECK-NEXT:  Loop %outer.loop: <multiple exits> Unpredictable backedge-taken count.
+; CHECK-NEXT:  Loop %outer.loop: Unpredictable constant max backedge-taken count.
+; CHECK-NEXT:  Loop %outer.loop: Unpredictable symbolic max backedge-taken count.
+;
+entry:
+  br label %outer.loop
+
+ph:
+  br label %outer.loop
+
+outer.loop:
+  %iv.outer = phi i32 [ %iv.outer.1, %ph ], [ %iter.count, %entry ]
+  %iv.outer.1 = add nsw i32 %iv.outer, -1
+  %ext.outer = zext nneg i32 %iv.outer.1 to i64
+  %gep.outer = getelementptr double, ptr %ptr, i64 %ext.outer
+  store double poison, ptr %gep.outer
+  %exit.cond.outer = icmp samesign ugt i32 %iv.outer, 1
+  br i1 %exit.cond.outer, label %inner.loop, label %ph
+
+inner.loop:
+  %iv.inner = phi i32 [ %iv.next, %inner.loop ], [ 0, %outer.loop ]
+  %ext.inner = zext nneg i32 %iv.inner to i64
+  %gep.inner = getelementptr double, ptr %ptr, i64 %ext.inner
+  store double poison, ptr %gep.inner
+  %iv.next = add nuw nsw i32 %iv.inner, 1
+  %exit.cond.inner = icmp slt i32 %iv.next, %iv.outer.1
+  br i1 %exit.cond.inner, label %inner.loop, label %ph
+
+exit:
+  ret i32 0
+}
diff --git a/llvm/test/Analysis/ScalarEvolution/infer-via-ranges.ll b/llvm/test/Analysis/ScalarEvolution/infer-via-ranges.ll
index 9aa096b952be5..9cf3e06d58a88 100644
--- a/llvm/test/Analysis/ScalarEvolution/infer-via-ranges.ll
+++ b/llvm/test/Analysis/ScalarEvolution/infer-via-ranges.ll
@@ -1,22 +1,31 @@
-; RUN: opt -passes=indvars -S < %s | FileCheck %s
+; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -passes='print<scalar-evolution>' \
+; RUN:  -scalar-evolution-classify-expressions=0 -disable-output %s 2>&1 | FileCheck %s
 
 define void @infer_via_ranges(ptr %arr, i32 %n) {
-; CHECK-LABEL: @infer_via_ranges
+; CHECK-LABEL: 'infer_via_ranges'
+; CHECK-NEXT:  Determining loop execution counts for: @infer_via_ranges
+; CHECK-NEXT:  Loop %loop: <multiple exits> backedge-taken count is ((-1 + %n) umin %n)
+; CHECK-NEXT:    exit count for loop: %n
+; CHECK-NEXT:    exit count for in.bounds: (-1 + %n)
+; CHECK-NEXT:  Loop %loop: constant max backedge-taken count is i32 -2147483648
+; CHECK-NEXT:  Loop %loop: symbolic max backedge-taken count is ((-1 + %n) umin %n)
+; CHECK-NEXT:    symbolic max exit count for loop: %n
+; CHECK-NEXT:    symbolic max exit count for in.bounds: (-1 + %n)
+; CHECK-NEXT:  Loop %loop: Trip multiple is 1
+;
  entry:
   %first.itr.check = icmp sgt i32 %n, 0
   %start = sub i32 %n, 1
   br i1 %first.itr.check, label %loop, label %exit
 
  loop:
-; CHECK-LABEL: loop:
   %idx = phi i32 [ %start, %entry ] , [ %idx.dec, %in.bounds ]
   %idx.dec = sub i32 %idx, 1
   %abc = icmp sge i32 %idx, 0
-; CHECK: br i1 true, label %in.bounds, label %out.of.bounds
   br i1 %abc, label %in.bounds, label %out.of.bounds
 
  in.bounds:
-; CHECK-LABEL: in.bounds:
   %addr = getelementptr i32, ptr %arr, i32 %idx
   store i32 0, ptr %addr
   %next = icmp sgt i32 %idx.dec, -1
diff --git a/llvm/test/Analysis/ScalarEvolution/pr123550.ll b/llvm/test/Analysis/ScalarEvolution/pr123550.ll
index c1f2051248a12..709da00935ef3 100644
--- a/llvm/test/Analysis/ScalarEvolution/pr123550.ll
+++ b/llvm/test/Analysis/ScalarEvolution/pr123550.ll
@@ -1,16 +1,16 @@
 ; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --version 5
 ; RUN: opt -disable-output -passes='print<scalar-evolution>' < %s 2>&1 | FileCheck %s
 
-; FIXME: This is a miscompile.
+; %srem should have exit value 130.
 define i32 @test() {
 ; CHECK-LABEL: 'test'
 ; CHECK-NEXT:  Classifying expressions for: @test
 ; CHECK-NEXT:    %phi = phi i32 [ -173, %bb ], [ %sub, %loop ]
-; CHECK-NEXT:    --> (-173 + (-1 * (zext i8 ((-83 /u {0,+,1}<%loop>) * {0,+,-1}<%loop>) to i32))<nuw><nsw>)<nuw><nsw> U: empty-set S: empty-set Exits: -173 LoopDispositions: { %loop: Computable }
+; CHECK-NEXT:    --> %phi U: [-173,1) S: [-173,1) Exits: -173 LoopDispositions: { %loop: Variant }
 ; CHECK-NEXT:    %iv2 = phi i32 [ 1, %bb ], [ %iv2.inc, %loop ]
 ; CHECK-NEXT:    --> {1,+,1}<nuw><nsw><%loop> U: [1,2) S: [1,2) Exits: 1 LoopDispositions: { %loop: Computable }
 ; CHECK-NEXT:    %srem = srem i32 729259140, %phi
-; CHECK-NEXT:    --> (729259140 + (-1 * (-173 + (-1 * (zext i8 ((-83 /u {0,+,1}<%loop>) * {0,+,-1}<%loop>) to i32))<nuw><nsw>)<nuw><nsw> * (729259140 /u (-173 + (-1 * (zext i8 ((-83 /u {0,+,1}<%loop>) * {0,+,-1}<%loop>) to i32))<nuw><nsw>)<nuw><nsw>)))<nuw><nsw> U: empty-set S: empty-set Exits: 729259140 LoopDispositions: { %loop: Computable }
+; CHECK-NEXT:    --> %srem U: [0,1073741824) S: [0,1073741824) Exits: 130 LoopDispositions: { %loop: Variant }
 ; CHECK-NEXT:    %trunc = trunc i32 %iv2 to i8
 ; CHECK-NEXT:    --> {1,+,1}<%loop> U: [1,2) S: [1,2) Exits: 1 LoopDispositions: { %loop: Computable }
 ; CHECK-NEXT:    %urem = urem i8 -83, %trunc
@@ -22,7 +22,7 @@ define i32 @test() {
 ; CHECK-NEXT:    %iv2.inc = add i32 %iv2, 1
 ; CHECK-NEXT:    --> {2,+,1}<nuw><nsw><%loop> U: [2,3) S: [2,3) Exits: 2 LoopDispositions: { %loop: Computable }
 ; CHECK-NEXT:    %srem.lcssa = phi i32 [ %srem, %loop ]
-; CHECK-NEXT:    --> (729259140 + (-1 * (-173 + (-1 * (zext i8 ((-83 /u {0,+,1}<%loop>) * {0,+,-1}<%loop>) to i32))<nuw><nsw>)<nuw><nsw> * (729259140 /u (-173 + (-1 * (zext i8 ((-83 /u {0,+,1}<%loop>) * {0,+,-1}<%loop>) to i32))<nuw><nsw>)<nuw><nsw>)))<nuw><nsw> U: empty-set S: empty-set --> 729259140 U: [729259140,729259141) S: [729259140,729259141)
+; CHECK-NEXT:    --> %srem U: [0,1073741824) S: [0,1073741824) --> 130 U: [130,131) S: [130,131)
 ; CHECK-NEXT:  Determining loop execution counts for: @test
 ; CHECK-NEXT:  Loop %loop: backedge-taken count is i32 0
 ; CHECK-NEXT:  Loop %loop: constant max backedge-taken count is i32 0
diff --git a/llvm/test/Analysis/StackSafetyAnalysis/lifetime.ll b/llvm/test/Analysis/StackSafetyAnalysis/lifetime.ll
index 32d76fb08c8e1..37fa7d3e84eae 100644
--- a/llvm/test/Analysis/StackSafetyAnalysis/lifetime.ll
+++ b/llvm/test/Analysis/StackSafetyAnalysis/lifetime.ll
@@ -576,8 +576,8 @@ block2:                                           ; preds = %entry
 
 %struct.Klass = type { i32, i32 }
 
-define i32 @shady_range(i32 %argc, ptr nocapture %argv) {
-; CHECK-LABEL: define i32 @shady_range(i32 %argc, ptr nocapture %argv)
+define i32 @shady_range(i32 %argc, ptr captures(none) %argv) {
+; CHECK-LABEL: define i32 @shady_range(i32 %argc, ptr captures(none) %argv)
 entry:
 ; CHECK: entry:
 ; CHECK-NEXT: Alive: <>
@@ -1047,8 +1047,8 @@ if.end:
   ret void
 }
 
-declare void @llvm.lifetime.start.p0(i64, ptr nocapture)
-declare void @llvm.lifetime.end.p0(i64, ptr nocapture)
+declare void @llvm.lifetime.start.p0(i64, ptr captures(none))
+declare void @llvm.lifetime.end.p0(i64, ptr captures(none))
 declare void @capture8(ptr)
 declare void @capture32(ptr)
 declare void @capture64(ptr)
diff --git a/llvm/test/Analysis/TypeBasedAliasAnalysis/functionattrs.ll b/llvm/test/Analysis/TypeBasedAliasAnalysis/functionattrs.ll
index 8615363a985d1..e5443038cb4c3 100644
--- a/llvm/test/Analysis/TypeBasedAliasAnalysis/functionattrs.ll
+++ b/llvm/test/Analysis/TypeBasedAliasAnalysis/functionattrs.ll
@@ -9,13 +9,13 @@
 ; invalid, as it's possible that this only happens after optimization on a
 ; code path which isn't ever executed.
 
-; CHECK: define void @test0_yes(ptr nocapture %p) #0 {
+; CHECK: define void @test0_yes(ptr captures(none) %p) #0 {
 define void @test0_yes(ptr %p) nounwind {
   store i32 0, ptr %p, !tbaa !1
   ret void
 }
 
-; CHECK: define void @test0_no(ptr nocapture writeonly initializes((0, 4)) %p) #1 {
+; CHECK: define void @test0_no(ptr writeonly captures(none) initializes((0, 4)) %p) #1 {
 define void @test0_no(ptr %p) nounwind {
   store i32 0, ptr %p, !tbaa !2
   ret void
@@ -24,7 +24,7 @@ define void @test0_no(ptr %p) nounwind {
 ; Add the readnone attribute, since there's just a call to a function which
 ; TBAA says only accesses constant memory.
 
-; CHECK: define void @test1_yes(ptr nocapture %p) #2 {
+; CHECK: define void @test1_yes(ptr captures(none) %p) #2 {
 define void @test1_yes(ptr %p) nounwind {
   call void @callee(ptr %p), !tbaa !1
   ret void
@@ -43,13 +43,13 @@ define void @test1_no(ptr %p) nounwind {
 ; This is unusual, since the function is memcpy, but as above, this
 ; isn't necessarily invalid.
 
-; CHECK: define void @test2_yes(ptr nocapture %p, ptr nocapture %q, i64 %n) #0 {
+; CHECK: define void @test2_yes(ptr captures(none) %p, ptr captures(none) %q, i64 %n) #0 {
 define void @test2_yes(ptr %p, ptr %q, i64 %n) nounwind {
   call void @llvm.memcpy.p0.p0.i64(ptr %p, ptr %q, i64 %n, i1 false), !tbaa !1
   ret void
 }
 
-; CHECK: define void @test2_no(ptr nocapture writeonly %p, ptr nocapture readonly %q, i64 %n) #4 {
+; CHECK: define void @test2_no(ptr writeonly captures(none) %p, ptr readonly captures(none) %q, i64 %n) #4 {
 define void @test2_no(ptr %p, ptr %q, i64 %n) nounwind {
   call void @llvm.memcpy.p0.p0.i64(ptr %p, ptr %q, i64 %n, i1 false), !tbaa !2
   ret void
@@ -57,13 +57,13 @@ define void @test2_no(ptr %p, ptr %q, i64 %n) nounwind {
 
 ; Similar to the others, va_arg only accesses memory through its operand.
 
-; CHECK: define i32 @test3_yes(ptr nocapture %p) #0 {
+; CHECK: define i32 @test3_yes(ptr captures(none) %p) #0 {
 define i32 @test3_yes(ptr %p) nounwind {
   %t = va_arg ptr %p, i32, !tbaa !1
   ret i32 %t
 }
 
-; CHECK: define i32 @test3_no(ptr nocapture %p) #4 {
+; CHECK: define i32 @test3_no(ptr captures(none) %p) #4 {
 define i32 @test3_no(ptr %p) nounwind {
   %t = va_arg ptr %p, i32, !tbaa !2
   ret i32 %t
diff --git a/llvm/test/Analysis/ValueTracking/known-nonnull-at.ll b/llvm/test/Analysis/ValueTracking/known-nonnull-at.ll
index 79d2653a3a146..dff7e13b8a2e7 100644
--- a/llvm/test/Analysis/ValueTracking/known-nonnull-at.ll
+++ b/llvm/test/Analysis/ValueTracking/known-nonnull-at.ll
@@ -220,3 +220,70 @@ return:
   %retval.0 = phi ptr [ %1, %if.end ], [ null, %entry ]
   ret ptr %retval.0
 }
+
+define i1 @test_known_nonnull_at_callsite(ptr %src) {
+; CHECK-LABEL: @test_known_nonnull_at_callsite(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    call void @callee(ptr noundef nonnull [[SRC:%.*]])
+; CHECK-NEXT:    ret i1 false
+;
+entry:
+  call void @callee(ptr noundef nonnull %src)
+  %nonnull = icmp eq ptr %src, null
+  ret i1 %nonnull
+}
+
+define i1 @test_known_nonnull_mixed(ptr %src) {
+; CHECK-LABEL: @test_known_nonnull_mixed(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    call void @callee2(ptr nonnull [[SRC:%.*]])
+; CHECK-NEXT:    ret i1 false
+;
+entry:
+  call void @callee2(ptr nonnull %src)
+  %nonnull = icmp eq ptr %src, null
+  ret i1 %nonnull
+}
+
+define i1 @test_known_nonnull_at_callsite_dereferenceable(ptr %src) {
+; CHECK-LABEL: @test_known_nonnull_at_callsite_dereferenceable(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    call void @callee(ptr dereferenceable(1) [[SRC:%.*]])
+; CHECK-NEXT:    ret i1 false
+;
+entry:
+  call void @callee(ptr dereferenceable(1) %src)
+  %nonnull = icmp eq ptr %src, null
+  ret i1 %nonnull
+}
+
+; Negative tests
+
+define i1 @test_known_nonnull_at_callsite_without_noundef(ptr %src) {
+; CHECK-LABEL: @test_known_nonnull_at_callsite_without_noundef(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    call void @callee(ptr nonnull [[SRC:%.*]])
+; CHECK-NEXT:    [[NONNULL:%.*]] = icmp eq ptr [[SRC]], null
+; CHECK-NEXT:    ret i1 [[NONNULL]]
+;
+entry:
+  call void @callee(ptr nonnull %src)
+  %nonnull = icmp eq ptr %src, null
+  ret i1 %nonnull
+}
+
+define i1 @test_known_nonnull_at_callsite_dereferenceable_null_is_defined(ptr %src) null_pointer_is_valid {
+; CHECK-LABEL: @test_known_nonnull_at_callsite_dereferenceable_null_is_defined(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    call void @callee(ptr dereferenceable(1) [[SRC:%.*]])
+; CHECK-NEXT:    [[NONNULL:%.*]] = icmp eq ptr [[SRC]], null
+; CHECK-NEXT:    ret i1 [[NONNULL]]
+;
+entry:
+  call void @callee(ptr dereferenceable(1) %src)
+  %nonnull = icmp eq ptr %src, null
+  ret i1 %nonnull
+}
+
+declare void @callee(ptr)
+declare void @callee2(ptr noundef)
diff --git a/llvm/test/Analysis/ValueTracking/phi-known-bits.ll b/llvm/test/Analysis/ValueTracking/phi-known-bits.ll
index 84220832f068f..436aadbc25de6 100644
--- a/llvm/test/Analysis/ValueTracking/phi-known-bits.ll
+++ b/llvm/test/Analysis/ValueTracking/phi-known-bits.ll
@@ -1112,3 +1112,41 @@ cleanup:
   %retval.0 = phi i1 [ %cmp, %if.then ], [ %tobool, %if.end4 ]
   ret i1 %retval.0
 }
+
+
+define i32 @issue_124275_wrong_br_direction(i32 noundef %inp) {
+; CHECK-LABEL: @issue_124275_wrong_br_direction(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = xor i32 [[INP:%.*]], -2
+; CHECK-NEXT:    [[XOR_INP_NEG:%.*]] = add i32 [[TMP0]], 1
+; CHECK-NEXT:    [[CMP_NE_NOT:%.*]] = icmp eq i32 [[XOR_INP_NEG]], 0
+; CHECK-NEXT:    br i1 [[CMP_NE_NOT]], label [[B1:%.*]], label [[B0:%.*]]
+; CHECK:       B0:
+; CHECK-NEXT:    [[PHI_B0:%.*]] = phi i32 [ [[PHI_B1:%.*]], [[B1]] ], [ [[XOR_INP_NEG]], [[ENTRY:%.*]] ]
+; CHECK-NEXT:    br label [[B1]]
+; CHECK:       B1:
+; CHECK-NEXT:    [[PHI_B1]] = phi i32 [ [[PHI_B0]], [[B0]] ], [ 0, [[ENTRY]] ]
+; CHECK-NEXT:    [[CMP_NE_B1_NOT:%.*]] = icmp eq i32 [[PHI_B1]], 0
+; CHECK-NEXT:    br i1 [[CMP_NE_B1_NOT]], label [[B0]], label [[END:%.*]]
+; CHECK:       end:
+; CHECK-NEXT:    ret i32 0
+;
+entry:
+  %xor_inp = xor i32 %inp, 1
+  %sub = sub i32 0, %xor_inp
+  %cmp_ne = icmp ne i32 %sub, 0
+  br i1 %cmp_ne, label %B0, label %B1
+
+B0:
+  %phi_B0 = phi i32 [ %phi_B1, %B1 ], [ %sub, %entry ]
+  br label %B1
+
+B1:
+  %phi_B1 = phi i32 [ %phi_B0, %B0 ], [ 0, %entry ]
+  %cmp_ne_B1 = icmp ne i32 %phi_B1, 0
+  %cmp_eq_B1 = xor i1 %cmp_ne_B1, true
+  br i1 %cmp_eq_B1, label %B0, label %end
+
+end:
+  ret i32 0
+}
diff --git a/llvm/test/Assembler/amdgcn-intrinsic-attributes.ll b/llvm/test/Assembler/amdgcn-intrinsic-attributes.ll
index 3652f6a4a27e3..bd5ce2ddda3e2 100644
--- a/llvm/test/Assembler/amdgcn-intrinsic-attributes.ll
+++ b/llvm/test/Assembler/amdgcn-intrinsic-attributes.ll
@@ -4,7 +4,7 @@
 
 ; Test assumed alignment parameter
 
-; CHECK: declare i32 @llvm.amdgcn.ds.append.p3(ptr addrspace(3) nocapture align 4, i1 immarg) #0
+; CHECK: declare i32 @llvm.amdgcn.ds.append.p3(ptr addrspace(3) align 4 captures(none), i1 immarg) #0
 
 define i32 @ds_append(ptr addrspace(3) %ptr) {
   %ret = call i32 @llvm.amdgcn.ds.append.p3(ptr addrspace(3) %ptr, i1 false)
@@ -12,7 +12,7 @@ define i32 @ds_append(ptr addrspace(3) %ptr) {
 }
 
 ; Test assumed alignment parameter
-; CHECK: declare i32 @llvm.amdgcn.ds.consume.p3(ptr addrspace(3) nocapture align 4, i1 immarg) #0
+; CHECK: declare i32 @llvm.amdgcn.ds.consume.p3(ptr addrspace(3) align 4 captures(none), i1 immarg) #0
 define i32 @ds_consume(ptr addrspace(3) %ptr) {
   %ret = call i32 @llvm.amdgcn.ds.consume.p3(ptr addrspace(3) %ptr, i1 false)
   ret i32 %ret
diff --git a/llvm/test/Assembler/auto_upgrade_intrinsics.ll b/llvm/test/Assembler/auto_upgrade_intrinsics.ll
index e3603846e9e9b..d1b535b49ca95 100644
--- a/llvm/test/Assembler/auto_upgrade_intrinsics.ll
+++ b/llvm/test/Assembler/auto_upgrade_intrinsics.ll
@@ -220,5 +220,5 @@ define void @test.prefetch.unnamed(ptr %ptr) {
 ; emitted at the end.
 ; CHECK: declare i32 @llvm.objectsize.i32.p0
 
-; CHECK: declare void @llvm.lifetime.start.p0(i64 immarg, ptr nocapture)
-; CHECK: declare void @llvm.lifetime.end.p0(i64 immarg, ptr nocapture)
+; CHECK: declare void @llvm.lifetime.start.p0(i64 immarg, ptr captures(none))
+; CHECK: declare void @llvm.lifetime.end.p0(i64 immarg, ptr captures(none))
diff --git a/llvm/test/Assembler/masked-load-store-intrinsics-attributes.ll b/llvm/test/Assembler/masked-load-store-intrinsics-attributes.ll
index c3a6dec809301..bd0da026ca5db 100644
--- a/llvm/test/Assembler/masked-load-store-intrinsics-attributes.ll
+++ b/llvm/test/Assembler/masked-load-store-intrinsics-attributes.ll
@@ -1,18 +1,19 @@
 ; RUN: llvm-as < %s | llvm-dis | FileCheck %s
 
 ; Make sure some masked/load store intrinsics have the expected attributes
-; Specifically `nocapture' should be added to the pointer paramters for the loads/stores
+; Specifically `captures(none)' should be added to the pointer parameters for
+; the loads/stores
 
-; CHECK: declare <vscale x 2 x i64> @llvm.masked.load.nxv2i64.p0(ptr nocapture, i32 immarg, <vscale x 2 x i1>, <vscale x 2 x i64>) [[ARGMEMONLY_NOCALLBACK_NOFREE_NOSYNC_NOUNWIND_READONLY_WILLRETURN:#[0-9]+]]
+; CHECK: declare <vscale x 2 x i64> @llvm.masked.load.nxv2i64.p0(ptr captures(none), i32 immarg, <vscale x 2 x i1>, <vscale x 2 x i64>) [[ARGMEMONLY_NOCALLBACK_NOFREE_NOSYNC_NOUNWIND_READONLY_WILLRETURN:#[0-9]+]]
 declare <vscale x 2 x i64> @llvm.masked.load.nxv2i64.p0(ptr, i32, <vscale x 2 x i1>, <vscale x 2 x i64>)
 
-; CHECK: declare void @llvm.masked.store.nxv2i64.p0(<vscale x 2 x i64>, ptr nocapture, i32 immarg, <vscale x 2 x i1>) [[ARGMEMONLY_NOCALLBACK_NOFREE_NOSYNC_NOUNWIND_WILLRETURN_WRITEONLY:#[0-9]+]]
+; CHECK: declare void @llvm.masked.store.nxv2i64.p0(<vscale x 2 x i64>, ptr captures(none), i32 immarg, <vscale x 2 x i1>) [[ARGMEMONLY_NOCALLBACK_NOFREE_NOSYNC_NOUNWIND_WILLRETURN_WRITEONLY:#[0-9]+]]
 declare void @llvm.masked.store.nxv2i64.p0(<vscale x 2 x i64>, ptr, i32, <vscale x 2 x i1>)
 
-; CHECK: declare <16 x float> @llvm.masked.expandload.v16f32(ptr nocapture, <16 x i1>, <16 x float>) [[NOCALLBACK_NOFREE_NOSYNC_NOUNWIND_READONLY_WILLRETURN:#[0-9]+]]
+; CHECK: declare <16 x float> @llvm.masked.expandload.v16f32(ptr captures(none), <16 x i1>, <16 x float>) [[NOCALLBACK_NOFREE_NOSYNC_NOUNWIND_READONLY_WILLRETURN:#[0-9]+]]
 declare <16 x float> @llvm.masked.expandload.v16f32 (ptr, <16 x i1>, <16 x float>)
 
-; CHECK: declare void @llvm.masked.compressstore.v8i32(<8 x i32>, ptr nocapture, <8 x i1>) [[ARGMEMONLY_NOCALLBACK_NOFREE_NOSYNC_NOUNWIND_WILLRETURN_WRITEONLY:#[0-9]+]]
+; CHECK: declare void @llvm.masked.compressstore.v8i32(<8 x i32>, ptr captures(none), <8 x i1>) [[ARGMEMONLY_NOCALLBACK_NOFREE_NOSYNC_NOUNWIND_WILLRETURN_WRITEONLY:#[0-9]+]]
 declare void @llvm.masked.compressstore.v8i32(<8 x i32>, ptr, <8  x i1>)
 
 ; CHECK: attributes [[ARGMEMONLY_NOCALLBACK_NOFREE_NOSYNC_NOUNWIND_READONLY_WILLRETURN]] = { nocallback nofree nosync nounwind willreturn memory(argmem: read) }
diff --git a/llvm/test/Bitcode/attributes-3.3.ll b/llvm/test/Bitcode/attributes-3.3.ll
index b471c52d2609c..5a6f064c89c06 100644
--- a/llvm/test/Bitcode/attributes-3.3.ll
+++ b/llvm/test/Bitcode/attributes-3.3.ll
@@ -108,7 +108,7 @@ define void @f17(i8 align 4 %0)
 }
 
 define void @f18(i8* nocapture %0)
-; CHECK: define void @f18(ptr nocapture %0)
+; CHECK: define void @f18(ptr captures(none) %0)
 {
         ret void;
 }
diff --git a/llvm/test/Bitcode/attributes.ll b/llvm/test/Bitcode/attributes.ll
index 1da9291c71996..7dd86a8c0eb16 100644
--- a/llvm/test/Bitcode/attributes.ll
+++ b/llvm/test/Bitcode/attributes.ll
@@ -105,7 +105,7 @@ define void @f17(ptr align 4 %0)
 }
 
 define void @f18(ptr nocapture %0)
-; CHECK: define void @f18(ptr nocapture %0)
+; CHECK: define void @f18(ptr captures(none) %0)
 {
         ret void;
 }
diff --git a/llvm/test/Bitcode/compatibility-3.6.ll b/llvm/test/Bitcode/compatibility-3.6.ll
index 37a87eea41ad3..2148e013126b3 100644
--- a/llvm/test/Bitcode/compatibility-3.6.ll
+++ b/llvm/test/Bitcode/compatibility-3.6.ll
@@ -410,7 +410,7 @@ declare void @f.param.sret(i8* sret(i8))
 declare void @f.param.noalias(i8* noalias)
 ; CHECK: declare void @f.param.noalias(ptr noalias)
 declare void @f.param.nocapture(i8* nocapture)
-; CHECK: declare void @f.param.nocapture(ptr nocapture)
+; CHECK: declare void @f.param.nocapture(ptr captures(none))
 declare void @f.param.nest(i8* nest)
 ; CHECK: declare void @f.param.nest(ptr nest)
 declare i8* @f.param.returned(i8* returned)
diff --git a/llvm/test/Bitcode/compatibility-3.7.ll b/llvm/test/Bitcode/compatibility-3.7.ll
index 8de2132d7ec89..fed9cce2a0091 100644
--- a/llvm/test/Bitcode/compatibility-3.7.ll
+++ b/llvm/test/Bitcode/compatibility-3.7.ll
@@ -416,7 +416,7 @@ declare void @f.param.sret(i8* sret(i8))
 declare void @f.param.noalias(i8* noalias)
 ; CHECK: declare void @f.param.noalias(ptr noalias)
 declare void @f.param.nocapture(i8* nocapture)
-; CHECK: declare void @f.param.nocapture(ptr nocapture)
+; CHECK: declare void @f.param.nocapture(ptr captures(none))
 declare void @f.param.nest(i8* nest)
 ; CHECK: declare void @f.param.nest(ptr nest)
 declare i8* @f.param.returned(i8* returned)
diff --git a/llvm/test/Bitcode/compatibility-3.8.ll b/llvm/test/Bitcode/compatibility-3.8.ll
index 7f766aa34a005..92695b9a41b80 100644
--- a/llvm/test/Bitcode/compatibility-3.8.ll
+++ b/llvm/test/Bitcode/compatibility-3.8.ll
@@ -441,7 +441,7 @@ declare void @f.param.sret(i8* sret(i8))
 declare void @f.param.noalias(i8* noalias)
 ; CHECK: declare void @f.param.noalias(ptr noalias)
 declare void @f.param.nocapture(i8* nocapture)
-; CHECK: declare void @f.param.nocapture(ptr nocapture)
+; CHECK: declare void @f.param.nocapture(ptr captures(none))
 declare void @f.param.nest(i8* nest)
 ; CHECK: declare void @f.param.nest(ptr nest)
 declare i8* @f.param.returned(i8* returned)
diff --git a/llvm/test/Bitcode/compatibility-3.9.ll b/llvm/test/Bitcode/compatibility-3.9.ll
index c8309175e063f..aa11917332e11 100644
--- a/llvm/test/Bitcode/compatibility-3.9.ll
+++ b/llvm/test/Bitcode/compatibility-3.9.ll
@@ -510,7 +510,7 @@ declare void @f.param.sret(i8* sret(i8))
 declare void @f.param.noalias(i8* noalias)
 ; CHECK: declare void @f.param.noalias(ptr noalias)
 declare void @f.param.nocapture(i8* nocapture)
-; CHECK: declare void @f.param.nocapture(ptr nocapture)
+; CHECK: declare void @f.param.nocapture(ptr captures(none))
 declare void @f.param.nest(i8* nest)
 ; CHECK: declare void @f.param.nest(ptr nest)
 declare i8* @f.param.returned(i8* returned)
diff --git a/llvm/test/Bitcode/compatibility-4.0.ll b/llvm/test/Bitcode/compatibility-4.0.ll
index adbd91ac6c7fe..cefccdc02c08c 100644
--- a/llvm/test/Bitcode/compatibility-4.0.ll
+++ b/llvm/test/Bitcode/compatibility-4.0.ll
@@ -510,7 +510,7 @@ declare void @f.param.sret(i8* sret(i8))
 declare void @f.param.noalias(i8* noalias)
 ; CHECK: declare void @f.param.noalias(ptr noalias)
 declare void @f.param.nocapture(i8* nocapture)
-; CHECK: declare void @f.param.nocapture(ptr nocapture)
+; CHECK: declare void @f.param.nocapture(ptr captures(none))
 declare void @f.param.nest(i8* nest)
 ; CHECK: declare void @f.param.nest(ptr nest)
 declare i8* @f.param.returned(i8* returned)
diff --git a/llvm/test/Bitcode/compatibility-5.0.ll b/llvm/test/Bitcode/compatibility-5.0.ll
index 1b500da69568a..ae3e2e8ffbb0f 100644
--- a/llvm/test/Bitcode/compatibility-5.0.ll
+++ b/llvm/test/Bitcode/compatibility-5.0.ll
@@ -514,7 +514,7 @@ declare void @f.param.sret(i8* sret(i8))
 declare void @f.param.noalias(i8* noalias)
 ; CHECK: declare void @f.param.noalias(ptr noalias)
 declare void @f.param.nocapture(i8* nocapture)
-; CHECK: declare void @f.param.nocapture(ptr nocapture)
+; CHECK: declare void @f.param.nocapture(ptr captures(none))
 declare void @f.param.nest(i8* nest)
 ; CHECK: declare void @f.param.nest(ptr nest)
 declare i8* @f.param.returned(i8* returned)
diff --git a/llvm/test/Bitcode/compatibility-6.0.ll b/llvm/test/Bitcode/compatibility-6.0.ll
index c1abbf0cda6eb..cfb5ff7b350a2 100644
--- a/llvm/test/Bitcode/compatibility-6.0.ll
+++ b/llvm/test/Bitcode/compatibility-6.0.ll
@@ -520,7 +520,7 @@ declare void @f.param.sret(i8* sret(i8))
 declare void @f.param.noalias(i8* noalias)
 ; CHECK: declare void @f.param.noalias(ptr noalias)
 declare void @f.param.nocapture(i8* nocapture)
-; CHECK: declare void @f.param.nocapture(ptr nocapture)
+; CHECK: declare void @f.param.nocapture(ptr captures(none))
 declare void @f.param.nest(i8* nest)
 ; CHECK: declare void @f.param.nest(ptr nest)
 declare i8* @f.param.returned(i8* returned)
diff --git a/llvm/test/Bitcode/compatibility.ll b/llvm/test/Bitcode/compatibility.ll
index a28156cdaa279..4a1f7ce1ee2a9 100644
--- a/llvm/test/Bitcode/compatibility.ll
+++ b/llvm/test/Bitcode/compatibility.ll
@@ -559,7 +559,7 @@ declare void @f.param.sret(ptr sret(i8))
 declare void @f.param.noalias(ptr noalias)
 ; CHECK: declare void @f.param.noalias(ptr noalias)
 declare void @f.param.nocapture(ptr nocapture)
-; CHECK: declare void @f.param.nocapture(ptr nocapture)
+; CHECK: declare void @f.param.nocapture(ptr captures(none))
 declare void @f.param.nest(ptr nest)
 ; CHECK: declare void @f.param.nest(ptr nest)
 declare ptr @f.param.returned(ptr returned)
diff --git a/llvm/test/Bitcode/function-address-space-fwd-decl.ll b/llvm/test/Bitcode/function-address-space-fwd-decl.ll
index 36c07ec3eaaba..cc74d5d942c40 100644
--- a/llvm/test/Bitcode/function-address-space-fwd-decl.ll
+++ b/llvm/test/Bitcode/function-address-space-fwd-decl.ll
@@ -33,7 +33,7 @@ return:
 
 declare i32 @foo() addrspace(40)
 ; CHECK: declare i32 @foo() addrspace(40)
-declare i32 @named(i16* nocapture) addrspace(40)
-; CHECK: declare i32 @named(ptr nocapture) addrspace(40)
+declare i32 @named(i16* captures(none)) addrspace(40)
+; CHECK: declare i32 @named(ptr captures(none)) addrspace(40)
 declare i32 @0(i16*) addrspace(40)
 ; CHECK: declare i32 @0(ptr) addrspace(40)
diff --git a/llvm/test/Bitcode/highLevelStructure.3.2.ll b/llvm/test/Bitcode/highLevelStructure.3.2.ll
index d9797b3202cd8..f2770ea4e4d34 100644
--- a/llvm/test/Bitcode/highLevelStructure.3.2.ll
+++ b/llvm/test/Bitcode/highLevelStructure.3.2.ll
@@ -45,15 +45,15 @@ declare void @ParamAttr5(i8* inreg)
 declare void @ParamAttr6(i8* byval(i8))
 ; CHECK: declare void @ParamAttr7(ptr noalias)
 declare void @ParamAttr7(i8* noalias)
-; CHECK: declare void @ParamAttr8(ptr nocapture)
+; CHECK: declare void @ParamAttr8(ptr captures(none))
 declare void @ParamAttr8(i8* nocapture)
-; CHECK: declare void @ParamAttr9{{[(i8* nest noalias nocapture) | (i8* noalias nocapture nest)]}}
-declare void @ParamAttr9(i8* nest noalias nocapture)
-; CHECK: declare void @ParamAttr10{{[(i8* sret(i8) noalias nocapture) | (i8* noalias nocapture sret(i8))]}}
+; CHECK: declare void @ParamAttr9{{[(i8* nest noalias captures(none)) | (i8* noalias nocapture nest)]}}
+declare void @ParamAttr9(i8* nest noalias captures(none))
+; CHECK: declare void @ParamAttr10{{[(i8* sret(i8) noalias nocapture) | (i8* noalias captures(none) sret(i8))]}}
 declare void @ParamAttr10(i8* sret(i8) noalias nocapture)
-;CHECK: declare void @ParamAttr11{{[(i8* byval(i8) noalias nocapture) | (i8* noalias nocapture byval(i8))]}}
+;CHECK: declare void @ParamAttr11{{[(i8* byval(i8) noalias captures(none)) | (i8* noalias nocapture byval(i8))]}}
 declare void @ParamAttr11(i8* byval(i8) noalias nocapture)
-;CHECK: declare void @ParamAttr12{{[(i8* inreg noalias nocapture) | (i8* noalias nocapture inreg)]}}
+;CHECK: declare void @ParamAttr12{{[(i8* inreg noalias captures(none)) | (i8* noalias nocapture inreg)]}}
 declare void @ParamAttr12(i8* inreg noalias nocapture)
 
 
diff --git a/llvm/test/Bitcode/upgrade-memory-intrinsics.ll b/llvm/test/Bitcode/upgrade-memory-intrinsics.ll
index 397e171978ce4..fe602b3e50595 100644
--- a/llvm/test/Bitcode/upgrade-memory-intrinsics.ll
+++ b/llvm/test/Bitcode/upgrade-memory-intrinsics.ll
@@ -39,9 +39,9 @@ define void @test3(ptr %p1, ptr %p2, ptr %p3) {
   ret void
 }
 
-; CHECK: declare void @llvm.memset.p0.i64(ptr nocapture writeonly, i8, i64, i1 immarg)
-; CHECK: declare void @llvm.memcpy.p0.p0.i64(ptr noalias nocapture writeonly, ptr noalias nocapture readonly, i64, i1 immarg)
-; CHECK: declare void @llvm.memmove.p0.p0.i64(ptr nocapture writeonly, ptr nocapture readonly, i64, i1 immarg)
+; CHECK: declare void @llvm.memset.p0.i64(ptr writeonly captures(none), i8, i64, i1 immarg)
+; CHECK: declare void @llvm.memcpy.p0.p0.i64(ptr noalias writeonly captures(none), ptr noalias readonly captures(none), i64, i1 immarg)
+; CHECK: declare void @llvm.memmove.p0.p0.i64(ptr writeonly captures(none), ptr readonly captures(none), i64, i1 immarg)
 declare void @llvm.memset.p0.i64(ptr nocapture writeonly, i8, i64, i32, i1)
 declare void @llvm.memcpy.p0.p0.i64(ptr nocapture writeonly , ptr nocapture readonly, i64, i32, i1)
 declare void @llvm.memmove.p0.p0.i64(ptr nocapture writeonly, ptr nocapture readonly, i64, i32, i1)
diff --git a/llvm/test/CMakeLists.txt b/llvm/test/CMakeLists.txt
index 231dcfa35a34c..1e4651210cd02 100644
--- a/llvm/test/CMakeLists.txt
+++ b/llvm/test/CMakeLists.txt
@@ -232,6 +232,7 @@ if (LLVM_INCLUDE_SPIRV_TOOLS_TESTS)
   list(APPEND LLVM_TEST_DEPENDS spirv-dis)
   list(APPEND LLVM_TEST_DEPENDS spirv-val)
   list(APPEND LLVM_TEST_DEPENDS spirv-as)
+  list(APPEND LLVM_TEST_DEPENDS spirv-link)
 endif()
 
 add_custom_target(llvm-test-depends DEPENDS ${LLVM_TEST_DEPENDS})
diff --git a/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomic-load-lse2.ll b/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomic-load-lse2.ll
index 3732d4feb0c67..0e9c29758244a 100644
--- a/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomic-load-lse2.ll
+++ b/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomic-load-lse2.ll
@@ -567,118 +567,6 @@ define dso_local i128 @load_atomic_i128_unaligned_seq_cst_const(ptr readonly %pt
     ret i128 %r
 }
 
-define dso_local half @load_atomic_f16_aligned_unordered(ptr %ptr) {
-; CHECK-LABEL: load_atomic_f16_aligned_unordered:
-; CHECK:    ldrh w8, [x0]
-    %r = load atomic half, ptr %ptr unordered, align 2
-    ret half %r
-}
-
-define dso_local half @load_atomic_f16_aligned_unordered_const(ptr readonly %ptr) {
-; CHECK-LABEL: load_atomic_f16_aligned_unordered_const:
-; CHECK:    ldrh w8, [x0]
-    %r = load atomic half, ptr %ptr unordered, align 2
-    ret half %r
-}
-
-define dso_local half @load_atomic_f16_aligned_monotonic(ptr %ptr) {
-; CHECK-LABEL: load_atomic_f16_aligned_monotonic:
-; CHECK:    ldrh w8, [x0]
-    %r = load atomic half, ptr %ptr monotonic, align 2
-    ret half %r
-}
-
-define dso_local half @load_atomic_f16_aligned_monotonic_const(ptr readonly %ptr) {
-; CHECK-LABEL: load_atomic_f16_aligned_monotonic_const:
-; CHECK:    ldrh w8, [x0]
-    %r = load atomic half, ptr %ptr monotonic, align 2
-    ret half %r
-}
-
-define dso_local half @load_atomic_f16_aligned_acquire(ptr %ptr) {
-; CHECK-LABEL: load_atomic_f16_aligned_acquire:
-; CHECK:    ldarh w8, [x0]
-    %r = load atomic half, ptr %ptr acquire, align 2
-    ret half %r
-}
-
-define dso_local half @load_atomic_f16_aligned_acquire_const(ptr readonly %ptr) {
-; CHECK-LABEL: load_atomic_f16_aligned_acquire_const:
-; CHECK:    ldarh w8, [x0]
-    %r = load atomic half, ptr %ptr acquire, align 2
-    ret half %r
-}
-
-define dso_local half @load_atomic_f16_aligned_seq_cst(ptr %ptr) {
-; CHECK-LABEL: load_atomic_f16_aligned_seq_cst:
-; CHECK:    ldarh w8, [x0]
-    %r = load atomic half, ptr %ptr seq_cst, align 2
-    ret half %r
-}
-
-define dso_local half @load_atomic_f16_aligned_seq_cst_const(ptr readonly %ptr) {
-; CHECK-LABEL: load_atomic_f16_aligned_seq_cst_const:
-; CHECK:    ldarh w8, [x0]
-    %r = load atomic half, ptr %ptr seq_cst, align 2
-    ret half %r
-}
-
-define dso_local bfloat @load_atomic_bf16_aligned_unordered(ptr %ptr) {
-; CHECK-LABEL: load_atomic_bf16_aligned_unordered:
-; CHECK:    ldrh w8, [x0]
-    %r = load atomic bfloat, ptr %ptr unordered, align 2
-    ret bfloat %r
-}
-
-define dso_local bfloat @load_atomic_bf16_aligned_unordered_const(ptr readonly %ptr) {
-; CHECK-LABEL: load_atomic_bf16_aligned_unordered_const:
-; CHECK:    ldrh w8, [x0]
-    %r = load atomic bfloat, ptr %ptr unordered, align 2
-    ret bfloat %r
-}
-
-define dso_local bfloat @load_atomic_bf16_aligned_monotonic(ptr %ptr) {
-; CHECK-LABEL: load_atomic_bf16_aligned_monotonic:
-; CHECK:    ldrh w8, [x0]
-    %r = load atomic bfloat, ptr %ptr monotonic, align 2
-    ret bfloat %r
-}
-
-define dso_local bfloat @load_atomic_bf16_aligned_monotonic_const(ptr readonly %ptr) {
-; CHECK-LABEL: load_atomic_bf16_aligned_monotonic_const:
-; CHECK:    ldrh w8, [x0]
-    %r = load atomic bfloat, ptr %ptr monotonic, align 2
-    ret bfloat %r
-}
-
-define dso_local bfloat @load_atomic_bf16_aligned_acquire(ptr %ptr) {
-; CHECK-LABEL: load_atomic_bf16_aligned_acquire:
-; CHECK:    ldarh w8, [x0]
-    %r = load atomic bfloat, ptr %ptr acquire, align 2
-    ret bfloat %r
-}
-
-define dso_local bfloat @load_atomic_bf16_aligned_acquire_const(ptr readonly %ptr) {
-; CHECK-LABEL: load_atomic_bf16_aligned_acquire_const:
-; CHECK:    ldarh w8, [x0]
-    %r = load atomic bfloat, ptr %ptr acquire, align 2
-    ret bfloat %r
-}
-
-define dso_local bfloat @load_atomic_bf16_aligned_seq_cst(ptr %ptr) {
-; CHECK-LABEL: load_atomic_bf16_aligned_seq_cst:
-; CHECK:    ldarh w8, [x0]
-    %r = load atomic bfloat, ptr %ptr seq_cst, align 2
-    ret bfloat %r
-}
-
-define dso_local bfloat @load_atomic_bf16_aligned_seq_cst_const(ptr readonly %ptr) {
-; CHECK-LABEL: load_atomic_bf16_aligned_seq_cst_const:
-; CHECK:    ldarh w8, [x0]
-    %r = load atomic bfloat, ptr %ptr seq_cst, align 2
-    ret bfloat %r
-}
-
 ;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
 ; -O0: {{.*}}
 ; -O1: {{.*}}
diff --git a/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomic-load-rcpc_immo.ll b/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomic-load-rcpc_immo.ll
index 9687ba683fb7e..02ff12c27fcda 100644
--- a/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomic-load-rcpc_immo.ll
+++ b/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomic-load-rcpc_immo.ll
@@ -1,6 +1,15 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --filter-out "(?!^\s*lda.*\bsp\b)^\s*.*\bsp\b" --filter "^\s*(ld|st[^r]|swp|cas|bl|add|and|eor|orn|orr|sub|mvn|sxt|cmp|ccmp|csel|dmb)"
-; RUN: llc %s -o - -verify-machineinstrs -mtriple=aarch64 -mattr=+v8.4a -mattr=+rcpc-immo -global-isel=true -global-isel-abort=2 -O0 | FileCheck %s --check-prefixes=CHECK,GISEL
-; RUN: llc %s -o - -verify-machineinstrs -mtriple=aarch64 -mattr=+v8.4a -mattr=+rcpc-immo -global-isel=false -O1 | FileCheck %s --check-prefixes=CHECK,SDAG
+; RUN: llc %s -o - -verify-machineinstrs -mtriple=aarch64 -mattr=+v8.8a -mattr=+rcpc-immo -global-isel=true -global-isel-abort=2 -O0 | FileCheck %s --check-prefixes=CHECK,GISEL
+; RUN: llc %s -o - -verify-machineinstrs -mtriple=aarch64 -mattr=+v8.4a -mattr=+rcpc-immo -global-isel=false -O1 | FileCheck %s --check-prefixes=CHECK,SDAG,SDAG-AVOIDLDAPUR
+; RUN: llc %s -o - -verify-machineinstrs -mtriple=aarch64 -mattr=+v8.4a -mattr=+rcpc-immo,avoid-ldapur -global-isel=false -O1 | FileCheck %s --check-prefixes=CHECK,SDAG,SDAG-AVOIDLDAPUR
+; RUN: llc %s -o - -verify-machineinstrs -mtriple=aarch64 -mcpu=neoverse-v2 -global-isel=false -O1 | FileCheck %s --check-prefixes=CHECK,SDAG,SDAG-AVOIDLDAPUR
+; RUN: llc %s -o - -verify-machineinstrs -mtriple=aarch64 -mcpu=neoverse-v3 -global-isel=false -O1 | FileCheck %s --check-prefixes=CHECK,SDAG,SDAG-AVOIDLDAPUR
+; RUN: llc %s -o - -verify-machineinstrs -mtriple=aarch64 -mcpu=cortex-x3 -global-isel=false -O1 | FileCheck %s --check-prefixes=CHECK,SDAG,SDAG-AVOIDLDAPUR
+; RUN: llc %s -o - -verify-machineinstrs -mtriple=aarch64 -mcpu=cortex-x4 -global-isel=false -O1 | FileCheck %s --check-prefixes=CHECK,SDAG,SDAG-AVOIDLDAPUR
+; RUN: llc %s -o - -verify-machineinstrs -mtriple=aarch64 -mcpu=cortex-x925 -global-isel=false -O1 | FileCheck %s --check-prefixes=CHECK,SDAG,SDAG-AVOIDLDAPUR
+; RUN: llc %s -o - -verify-machineinstrs -mtriple=aarch64 -mattr=+v9a -global-isel=false -O1 | FileCheck %s --check-prefixes=CHECK,SDAG,SDAG-AVOIDLDAPUR
+; RUN: llc %s -o - -verify-machineinstrs -mtriple=aarch64 -mattr=+v8.8a -global-isel=false -O1 | FileCheck %s --check-prefixes=CHECK,SDAG,SDAG-NOAVOIDLDAPUR
+; RUN: llc %s -o - -verify-machineinstrs -mtriple=aarch64 -mattr=+v9.3a -global-isel=false -O1 | FileCheck %s --check-prefixes=CHECK,SDAG,SDAG-NOAVOIDLDAPUR
 
 define i8 @load_atomic_i8_aligned_unordered(ptr %ptr) {
 ; CHECK-LABEL: load_atomic_i8_aligned_unordered:
@@ -39,8 +48,12 @@ define i8 @load_atomic_i8_aligned_acquire(ptr %ptr) {
 ; GISEL:    add x8, x0, #4
 ; GISEL:    ldaprb w0, [x8]
 ;
-; SDAG-LABEL: load_atomic_i8_aligned_acquire:
-; SDAG:    ldapurb w0, [x0, #4]
+; SDAG-NOAVOIDLDAPUR-LABEL: load_atomic_i8_aligned_acquire:
+; SDAG-NOAVOIDLDAPUR:    ldapurb w0, [x0, #4]
+;
+; SDAG-AVOIDLDAPUR-LABEL: load_atomic_i8_aligned_acquire:
+; SDAG-AVOIDLDAPUR:    add x8, x0, #4
+; SDAG-AVOIDLDAPUR:    ldaprb w0, [x8]
     %gep = getelementptr inbounds i8, ptr %ptr, i32 4
     %r = load atomic i8, ptr %gep acquire, align 1
     ret i8 %r
@@ -51,8 +64,12 @@ define i8 @load_atomic_i8_aligned_acquire_const(ptr readonly %ptr) {
 ; GISEL:    add x8, x0, #4
 ; GISEL:    ldaprb w0, [x8]
 ;
-; SDAG-LABEL: load_atomic_i8_aligned_acquire_const:
-; SDAG:    ldapurb w0, [x0, #4]
+; SDAG-NOAVOIDLDAPUR-LABEL: load_atomic_i8_aligned_acquire_const:
+; SDAG-NOAVOIDLDAPUR:    ldapurb w0, [x0, #4]
+;
+; SDAG-AVOIDLDAPUR-LABEL: load_atomic_i8_aligned_acquire_const:
+; SDAG-AVOIDLDAPUR:    add x8, x0, #4
+; SDAG-AVOIDLDAPUR:    ldaprb w0, [x8]
     %gep = getelementptr inbounds i8, ptr %ptr, i32 4
     %r = load atomic i8, ptr %gep acquire, align 1
     ret i8 %r
@@ -113,8 +130,12 @@ define i16 @load_atomic_i16_aligned_acquire(ptr %ptr) {
 ; GISEL:    add x8, x0, #8
 ; GISEL:    ldaprh w0, [x8]
 ;
-; SDAG-LABEL: load_atomic_i16_aligned_acquire:
-; SDAG:    ldapurh w0, [x0, #8]
+; SDAG-NOAVOIDLDAPUR-LABEL: load_atomic_i16_aligned_acquire:
+; SDAG-NOAVOIDLDAPUR:    ldapurh w0, [x0, #8]
+;
+; SDAG-AVOIDLDAPUR-LABEL: load_atomic_i16_aligned_acquire:
+; SDAG-AVOIDLDAPUR:    add x8, x0, #8
+; SDAG-AVOIDLDAPUR:    ldaprh w0, [x8]
     %gep = getelementptr inbounds i16, ptr %ptr, i32 4
     %r = load atomic i16, ptr %gep acquire, align 2
     ret i16 %r
@@ -125,8 +146,12 @@ define i16 @load_atomic_i16_aligned_acquire_const(ptr readonly %ptr) {
 ; GISEL:    add x8, x0, #8
 ; GISEL:    ldaprh w0, [x8]
 ;
-; SDAG-LABEL: load_atomic_i16_aligned_acquire_const:
-; SDAG:    ldapurh w0, [x0, #8]
+; SDAG-NOAVOIDLDAPUR-LABEL: load_atomic_i16_aligned_acquire_const:
+; SDAG-NOAVOIDLDAPUR:    ldapurh w0, [x0, #8]
+;
+; SDAG-AVOIDLDAPUR-LABEL: load_atomic_i16_aligned_acquire_const:
+; SDAG-AVOIDLDAPUR:    add x8, x0, #8
+; SDAG-AVOIDLDAPUR:    ldaprh w0, [x8]
     %gep = getelementptr inbounds i16, ptr %ptr, i32 4
     %r = load atomic i16, ptr %gep acquire, align 2
     ret i16 %r
@@ -183,16 +208,30 @@ define i32 @load_atomic_i32_aligned_monotonic_const(ptr readonly %ptr) {
 }
 
 define i32 @load_atomic_i32_aligned_acquire(ptr %ptr) {
-; CHECK-LABEL: load_atomic_i32_aligned_acquire:
-; CHECK:    ldapur w0, [x0, #16]
+; GISEL-LABEL: load_atomic_i32_aligned_acquire:
+; GISEL:    ldapur w0, [x0, #16]
+;
+; SDAG-NOAVOIDLDAPUR-LABEL: load_atomic_i32_aligned_acquire:
+; SDAG-NOAVOIDLDAPUR:    ldapur w0, [x0, #16]
+;
+; SDAG-AVOIDLDAPUR-LABEL: load_atomic_i32_aligned_acquire:
+; SDAG-AVOIDLDAPUR:    add x8, x0, #16
+; SDAG-AVOIDLDAPUR:    ldapr w0, [x8]
     %gep = getelementptr inbounds i32, ptr %ptr, i32 4
     %r = load atomic i32, ptr %gep acquire, align 4
     ret i32 %r
 }
 
 define i32 @load_atomic_i32_aligned_acquire_const(ptr readonly %ptr) {
-; CHECK-LABEL: load_atomic_i32_aligned_acquire_const:
-; CHECK:    ldapur w0, [x0, #16]
+; GISEL-LABEL: load_atomic_i32_aligned_acquire_const:
+; GISEL:    ldapur w0, [x0, #16]
+;
+; SDAG-NOAVOIDLDAPUR-LABEL: load_atomic_i32_aligned_acquire_const:
+; SDAG-NOAVOIDLDAPUR:    ldapur w0, [x0, #16]
+;
+; SDAG-AVOIDLDAPUR-LABEL: load_atomic_i32_aligned_acquire_const:
+; SDAG-AVOIDLDAPUR:    add x8, x0, #16
+; SDAG-AVOIDLDAPUR:    ldapr w0, [x8]
     %gep = getelementptr inbounds i32, ptr %ptr, i32 4
     %r = load atomic i32, ptr %gep acquire, align 4
     ret i32 %r
@@ -249,16 +288,30 @@ define i64 @load_atomic_i64_aligned_monotonic_const(ptr readonly %ptr) {
 }
 
 define i64 @load_atomic_i64_aligned_acquire(ptr %ptr) {
-; CHECK-LABEL: load_atomic_i64_aligned_acquire:
-; CHECK:    ldapur x0, [x0, #32]
+; GISEL-LABEL: load_atomic_i64_aligned_acquire:
+; GISEL:    ldapur x0, [x0, #32]
+;
+; SDAG-NOAVOIDLDAPUR-LABEL: load_atomic_i64_aligned_acquire:
+; SDAG-NOAVOIDLDAPUR:    ldapur x0, [x0, #32]
+;
+; SDAG-AVOIDLDAPUR-LABEL: load_atomic_i64_aligned_acquire:
+; SDAG-AVOIDLDAPUR:    add x8, x0, #32
+; SDAG-AVOIDLDAPUR:    ldapr x0, [x8]
     %gep = getelementptr inbounds i64, ptr %ptr, i32 4
     %r = load atomic i64, ptr %gep acquire, align 8
     ret i64 %r
 }
 
 define i64 @load_atomic_i64_aligned_acquire_const(ptr readonly %ptr) {
-; CHECK-LABEL: load_atomic_i64_aligned_acquire_const:
-; CHECK:    ldapur x0, [x0, #32]
+; GISEL-LABEL: load_atomic_i64_aligned_acquire_const:
+; GISEL:    ldapur x0, [x0, #32]
+;
+; SDAG-NOAVOIDLDAPUR-LABEL: load_atomic_i64_aligned_acquire_const:
+; SDAG-NOAVOIDLDAPUR:    ldapur x0, [x0, #32]
+;
+; SDAG-AVOIDLDAPUR-LABEL: load_atomic_i64_aligned_acquire_const:
+; SDAG-AVOIDLDAPUR:    add x8, x0, #32
+; SDAG-AVOIDLDAPUR:    ldapr x0, [x8]
     %gep = getelementptr inbounds i64, ptr %ptr, i32 4
     %r = load atomic i64, ptr %gep acquire, align 8
     ret i64 %r
@@ -387,8 +440,12 @@ define i8 @load_atomic_i8_unaligned_acquire(ptr %ptr) {
 ; GISEL:    add x8, x0, #4
 ; GISEL:    ldaprb w0, [x8]
 ;
-; SDAG-LABEL: load_atomic_i8_unaligned_acquire:
-; SDAG:    ldapurb w0, [x0, #4]
+; SDAG-NOAVOIDLDAPUR-LABEL: load_atomic_i8_unaligned_acquire:
+; SDAG-NOAVOIDLDAPUR:    ldapurb w0, [x0, #4]
+;
+; SDAG-AVOIDLDAPUR-LABEL: load_atomic_i8_unaligned_acquire:
+; SDAG-AVOIDLDAPUR:    add x8, x0, #4
+; SDAG-AVOIDLDAPUR:    ldaprb w0, [x8]
     %gep = getelementptr inbounds i8, ptr %ptr, i32 4
     %r = load atomic i8, ptr %gep acquire, align 1
     ret i8 %r
@@ -399,8 +456,12 @@ define i8 @load_atomic_i8_unaligned_acquire_const(ptr readonly %ptr) {
 ; GISEL:    add x8, x0, #4
 ; GISEL:    ldaprb w0, [x8]
 ;
-; SDAG-LABEL: load_atomic_i8_unaligned_acquire_const:
-; SDAG:    ldapurb w0, [x0, #4]
+; SDAG-NOAVOIDLDAPUR-LABEL: load_atomic_i8_unaligned_acquire_const:
+; SDAG-NOAVOIDLDAPUR:    ldapurb w0, [x0, #4]
+;
+; SDAG-AVOIDLDAPUR-LABEL: load_atomic_i8_unaligned_acquire_const:
+; SDAG-AVOIDLDAPUR:    add x8, x0, #4
+; SDAG-AVOIDLDAPUR:    ldaprb w0, [x8]
     %gep = getelementptr inbounds i8, ptr %ptr, i32 4
     %r = load atomic i8, ptr %gep acquire, align 1
     ret i8 %r
@@ -846,9 +907,14 @@ define i8 @load_atomic_i8_from_gep() {
 ; GISEL:    add x8, x8, #1
 ; GISEL:    ldaprb w0, [x8]
 ;
-; SDAG-LABEL: load_atomic_i8_from_gep:
-; SDAG:    bl init
-; SDAG:    ldapurb w0, [sp, #13]
+; SDAG-NOAVOIDLDAPUR-LABEL: load_atomic_i8_from_gep:
+; SDAG-NOAVOIDLDAPUR:    bl init
+; SDAG-NOAVOIDLDAPUR:    ldapurb w0, [sp, #13]
+;
+; SDAG-AVOIDLDAPUR-LABEL: load_atomic_i8_from_gep:
+; SDAG-AVOIDLDAPUR:    bl init
+; SDAG-AVOIDLDAPUR:    orr x8, x19, #0x1
+; SDAG-AVOIDLDAPUR:    ldaprb w0, [x8]
   %a = alloca [3 x i8]
   call void @init(ptr %a)
   %arrayidx  = getelementptr [3 x i8], ptr %a, i64 0, i64 1
@@ -862,9 +928,14 @@ define i16 @load_atomic_i16_from_gep() {
 ; GISEL:    add x8, x8, #2
 ; GISEL:    ldaprh w0, [x8]
 ;
-; SDAG-LABEL: load_atomic_i16_from_gep:
-; SDAG:    bl init
-; SDAG:    ldapurh w0, [sp, #10]
+; SDAG-NOAVOIDLDAPUR-LABEL: load_atomic_i16_from_gep:
+; SDAG-NOAVOIDLDAPUR:    bl init
+; SDAG-NOAVOIDLDAPUR:    ldapurh w0, [sp, #10]
+;
+; SDAG-AVOIDLDAPUR-LABEL: load_atomic_i16_from_gep:
+; SDAG-AVOIDLDAPUR:    bl init
+; SDAG-AVOIDLDAPUR:    orr x8, x19, #0x2
+; SDAG-AVOIDLDAPUR:    ldaprh w0, [x8]
   %a = alloca [3 x i16]
   call void @init(ptr %a)
   %arrayidx  = getelementptr [3 x i16], ptr %a, i64 0, i64 1
@@ -877,9 +948,14 @@ define i32 @load_atomic_i32_from_gep() {
 ; GISEL:    bl init
 ; GISEL:    ldapur w0, [x8, #4]
 ;
-; SDAG-LABEL: load_atomic_i32_from_gep:
-; SDAG:    bl init
-; SDAG:    ldapur w0, [sp, #8]
+; SDAG-NOAVOIDLDAPUR-LABEL: load_atomic_i32_from_gep:
+; SDAG-NOAVOIDLDAPUR:    bl init
+; SDAG-NOAVOIDLDAPUR:    ldapur w0, [sp, #8]
+;
+; SDAG-AVOIDLDAPUR-LABEL: load_atomic_i32_from_gep:
+; SDAG-AVOIDLDAPUR:    bl init
+; SDAG-AVOIDLDAPUR:    add x8, x19, #4
+; SDAG-AVOIDLDAPUR:    ldapr w0, [x8]
   %a = alloca [3 x i32]
   call void @init(ptr %a)
   %arrayidx  = getelementptr [3 x i32], ptr %a, i64 0, i64 1
@@ -892,9 +968,14 @@ define i64 @load_atomic_i64_from_gep() {
 ; GISEL:    bl init
 ; GISEL:    ldapur x0, [x8, #8]
 ;
-; SDAG-LABEL: load_atomic_i64_from_gep:
-; SDAG:    bl init
-; SDAG:    ldapur x0, [sp, #16]
+; SDAG-NOAVOIDLDAPUR-LABEL: load_atomic_i64_from_gep:
+; SDAG-NOAVOIDLDAPUR:    bl init
+; SDAG-NOAVOIDLDAPUR:    ldapur x0, [sp, #16]
+;
+; SDAG-AVOIDLDAPUR-LABEL: load_atomic_i64_from_gep:
+; SDAG-AVOIDLDAPUR:    bl init
+; SDAG-AVOIDLDAPUR:    add x8, x19, #8
+; SDAG-AVOIDLDAPUR:    ldapr x0, [x8]
   %a = alloca [3 x i64]
   call void @init(ptr %a)
   %arrayidx  = getelementptr [3 x i64], ptr %a, i64 0, i64 1
diff --git a/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomicrmw-lse2.ll b/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomicrmw-lse2.ll
index d93ef6f8b2869..94d46148f37e3 100644
--- a/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomicrmw-lse2.ll
+++ b/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomicrmw-lse2.ll
@@ -12,8 +12,8 @@ define dso_local i8 @atomicrmw_xchg_i8_aligned_monotonic(ptr %ptr, i8 %value) {
 ; -O0:    subs w8, w8, w10, uxtb
 ;
 ; -O1-LABEL: atomicrmw_xchg_i8_aligned_monotonic:
-; -O1:    ldxrb w8, [x0]
-; -O1:    stxrb w9, w1, [x0]
+; -O1:    ldxrb w0, [x8]
+; -O1:    stxrb w9, w1, [x8]
     %r = atomicrmw xchg ptr %ptr, i8 %value monotonic, align 1
     ret i8 %r
 }
@@ -27,8 +27,8 @@ define dso_local i8 @atomicrmw_xchg_i8_aligned_acquire(ptr %ptr, i8 %value) {
 ; -O0:    subs w8, w8, w10, uxtb
 ;
 ; -O1-LABEL: atomicrmw_xchg_i8_aligned_acquire:
-; -O1:    ldaxrb w8, [x0]
-; -O1:    stxrb w9, w1, [x0]
+; -O1:    ldaxrb w0, [x8]
+; -O1:    stxrb w9, w1, [x8]
     %r = atomicrmw xchg ptr %ptr, i8 %value acquire, align 1
     ret i8 %r
 }
@@ -42,8 +42,8 @@ define dso_local i8 @atomicrmw_xchg_i8_aligned_release(ptr %ptr, i8 %value) {
 ; -O0:    subs w8, w8, w10, uxtb
 ;
 ; -O1-LABEL: atomicrmw_xchg_i8_aligned_release:
-; -O1:    ldxrb w8, [x0]
-; -O1:    stlxrb w9, w1, [x0]
+; -O1:    ldxrb w0, [x8]
+; -O1:    stlxrb w9, w1, [x8]
     %r = atomicrmw xchg ptr %ptr, i8 %value release, align 1
     ret i8 %r
 }
@@ -57,8 +57,8 @@ define dso_local i8 @atomicrmw_xchg_i8_aligned_acq_rel(ptr %ptr, i8 %value) {
 ; -O0:    subs w8, w8, w10, uxtb
 ;
 ; -O1-LABEL: atomicrmw_xchg_i8_aligned_acq_rel:
-; -O1:    ldaxrb w8, [x0]
-; -O1:    stlxrb w9, w1, [x0]
+; -O1:    ldaxrb w0, [x8]
+; -O1:    stlxrb w9, w1, [x8]
     %r = atomicrmw xchg ptr %ptr, i8 %value acq_rel, align 1
     ret i8 %r
 }
@@ -72,8 +72,8 @@ define dso_local i8 @atomicrmw_xchg_i8_aligned_seq_cst(ptr %ptr, i8 %value) {
 ; -O0:    subs w8, w8, w10, uxtb
 ;
 ; -O1-LABEL: atomicrmw_xchg_i8_aligned_seq_cst:
-; -O1:    ldaxrb w8, [x0]
-; -O1:    stlxrb w9, w1, [x0]
+; -O1:    ldaxrb w0, [x8]
+; -O1:    stlxrb w9, w1, [x8]
     %r = atomicrmw xchg ptr %ptr, i8 %value seq_cst, align 1
     ret i8 %r
 }
@@ -86,8 +86,8 @@ define dso_local i16 @atomicrmw_xchg_i16_aligned_monotonic(ptr %ptr, i16 %value)
 ; -O0:    subs w8, w8, w9, uxth
 ;
 ; -O1-LABEL: atomicrmw_xchg_i16_aligned_monotonic:
-; -O1:    ldxrh w8, [x0]
-; -O1:    stxrh w9, w1, [x0]
+; -O1:    ldxrh w0, [x8]
+; -O1:    stxrh w9, w1, [x8]
     %r = atomicrmw xchg ptr %ptr, i16 %value monotonic, align 2
     ret i16 %r
 }
@@ -100,8 +100,8 @@ define dso_local i16 @atomicrmw_xchg_i16_aligned_acquire(ptr %ptr, i16 %value) {
 ; -O0:    subs w8, w8, w9, uxth
 ;
 ; -O1-LABEL: atomicrmw_xchg_i16_aligned_acquire:
-; -O1:    ldaxrh w8, [x0]
-; -O1:    stxrh w9, w1, [x0]
+; -O1:    ldaxrh w0, [x8]
+; -O1:    stxrh w9, w1, [x8]
     %r = atomicrmw xchg ptr %ptr, i16 %value acquire, align 2
     ret i16 %r
 }
@@ -114,8 +114,8 @@ define dso_local i16 @atomicrmw_xchg_i16_aligned_release(ptr %ptr, i16 %value) {
 ; -O0:    subs w8, w8, w9, uxth
 ;
 ; -O1-LABEL: atomicrmw_xchg_i16_aligned_release:
-; -O1:    ldxrh w8, [x0]
-; -O1:    stlxrh w9, w1, [x0]
+; -O1:    ldxrh w0, [x8]
+; -O1:    stlxrh w9, w1, [x8]
     %r = atomicrmw xchg ptr %ptr, i16 %value release, align 2
     ret i16 %r
 }
@@ -128,8 +128,8 @@ define dso_local i16 @atomicrmw_xchg_i16_aligned_acq_rel(ptr %ptr, i16 %value) {
 ; -O0:    subs w8, w8, w9, uxth
 ;
 ; -O1-LABEL: atomicrmw_xchg_i16_aligned_acq_rel:
-; -O1:    ldaxrh w8, [x0]
-; -O1:    stlxrh w9, w1, [x0]
+; -O1:    ldaxrh w0, [x8]
+; -O1:    stlxrh w9, w1, [x8]
     %r = atomicrmw xchg ptr %ptr, i16 %value acq_rel, align 2
     ret i16 %r
 }
@@ -142,8 +142,8 @@ define dso_local i16 @atomicrmw_xchg_i16_aligned_seq_cst(ptr %ptr, i16 %value) {
 ; -O0:    subs w8, w8, w9, uxth
 ;
 ; -O1-LABEL: atomicrmw_xchg_i16_aligned_seq_cst:
-; -O1:    ldaxrh w8, [x0]
-; -O1:    stlxrh w9, w1, [x0]
+; -O1:    ldaxrh w0, [x8]
+; -O1:    stlxrh w9, w1, [x8]
     %r = atomicrmw xchg ptr %ptr, i16 %value seq_cst, align 2
     ret i16 %r
 }
@@ -392,8 +392,8 @@ define dso_local i8 @atomicrmw_xchg_i8_unaligned_monotonic(ptr %ptr, i8 %value)
 ; -O0:    subs w8, w8, w10, uxtb
 ;
 ; -O1-LABEL: atomicrmw_xchg_i8_unaligned_monotonic:
-; -O1:    ldxrb w8, [x0]
-; -O1:    stxrb w9, w1, [x0]
+; -O1:    ldxrb w0, [x8]
+; -O1:    stxrb w9, w1, [x8]
     %r = atomicrmw xchg ptr %ptr, i8 %value monotonic, align 1
     ret i8 %r
 }
@@ -407,8 +407,8 @@ define dso_local i8 @atomicrmw_xchg_i8_unaligned_acquire(ptr %ptr, i8 %value) {
 ; -O0:    subs w8, w8, w10, uxtb
 ;
 ; -O1-LABEL: atomicrmw_xchg_i8_unaligned_acquire:
-; -O1:    ldaxrb w8, [x0]
-; -O1:    stxrb w9, w1, [x0]
+; -O1:    ldaxrb w0, [x8]
+; -O1:    stxrb w9, w1, [x8]
     %r = atomicrmw xchg ptr %ptr, i8 %value acquire, align 1
     ret i8 %r
 }
@@ -422,8 +422,8 @@ define dso_local i8 @atomicrmw_xchg_i8_unaligned_release(ptr %ptr, i8 %value) {
 ; -O0:    subs w8, w8, w10, uxtb
 ;
 ; -O1-LABEL: atomicrmw_xchg_i8_unaligned_release:
-; -O1:    ldxrb w8, [x0]
-; -O1:    stlxrb w9, w1, [x0]
+; -O1:    ldxrb w0, [x8]
+; -O1:    stlxrb w9, w1, [x8]
     %r = atomicrmw xchg ptr %ptr, i8 %value release, align 1
     ret i8 %r
 }
@@ -437,8 +437,8 @@ define dso_local i8 @atomicrmw_xchg_i8_unaligned_acq_rel(ptr %ptr, i8 %value) {
 ; -O0:    subs w8, w8, w10, uxtb
 ;
 ; -O1-LABEL: atomicrmw_xchg_i8_unaligned_acq_rel:
-; -O1:    ldaxrb w8, [x0]
-; -O1:    stlxrb w9, w1, [x0]
+; -O1:    ldaxrb w0, [x8]
+; -O1:    stlxrb w9, w1, [x8]
     %r = atomicrmw xchg ptr %ptr, i8 %value acq_rel, align 1
     ret i8 %r
 }
@@ -452,8 +452,8 @@ define dso_local i8 @atomicrmw_xchg_i8_unaligned_seq_cst(ptr %ptr, i8 %value) {
 ; -O0:    subs w8, w8, w10, uxtb
 ;
 ; -O1-LABEL: atomicrmw_xchg_i8_unaligned_seq_cst:
-; -O1:    ldaxrb w8, [x0]
-; -O1:    stlxrb w9, w1, [x0]
+; -O1:    ldaxrb w0, [x8]
+; -O1:    stlxrb w9, w1, [x8]
     %r = atomicrmw xchg ptr %ptr, i8 %value seq_cst, align 1
     ret i8 %r
 }
diff --git a/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomicrmw-rcpc.ll b/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomicrmw-rcpc.ll
index 912d87dcd2b9b..57cfeb78b6980 100644
--- a/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomicrmw-rcpc.ll
+++ b/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomicrmw-rcpc.ll
@@ -12,8 +12,8 @@ define dso_local i8 @atomicrmw_xchg_i8_aligned_monotonic(ptr %ptr, i8 %value) {
 ; -O0:    subs w8, w8, w10, uxtb
 ;
 ; -O1-LABEL: atomicrmw_xchg_i8_aligned_monotonic:
-; -O1:    ldxrb w8, [x0]
-; -O1:    stxrb w9, w1, [x0]
+; -O1:    ldxrb w0, [x8]
+; -O1:    stxrb w9, w1, [x8]
     %r = atomicrmw xchg ptr %ptr, i8 %value monotonic, align 1
     ret i8 %r
 }
@@ -27,8 +27,8 @@ define dso_local i8 @atomicrmw_xchg_i8_aligned_acquire(ptr %ptr, i8 %value) {
 ; -O0:    subs w8, w8, w10, uxtb
 ;
 ; -O1-LABEL: atomicrmw_xchg_i8_aligned_acquire:
-; -O1:    ldaxrb w8, [x0]
-; -O1:    stxrb w9, w1, [x0]
+; -O1:    ldaxrb w0, [x8]
+; -O1:    stxrb w9, w1, [x8]
     %r = atomicrmw xchg ptr %ptr, i8 %value acquire, align 1
     ret i8 %r
 }
@@ -42,8 +42,8 @@ define dso_local i8 @atomicrmw_xchg_i8_aligned_release(ptr %ptr, i8 %value) {
 ; -O0:    subs w8, w8, w10, uxtb
 ;
 ; -O1-LABEL: atomicrmw_xchg_i8_aligned_release:
-; -O1:    ldxrb w8, [x0]
-; -O1:    stlxrb w9, w1, [x0]
+; -O1:    ldxrb w0, [x8]
+; -O1:    stlxrb w9, w1, [x8]
     %r = atomicrmw xchg ptr %ptr, i8 %value release, align 1
     ret i8 %r
 }
@@ -57,8 +57,8 @@ define dso_local i8 @atomicrmw_xchg_i8_aligned_acq_rel(ptr %ptr, i8 %value) {
 ; -O0:    subs w8, w8, w10, uxtb
 ;
 ; -O1-LABEL: atomicrmw_xchg_i8_aligned_acq_rel:
-; -O1:    ldaxrb w8, [x0]
-; -O1:    stlxrb w9, w1, [x0]
+; -O1:    ldaxrb w0, [x8]
+; -O1:    stlxrb w9, w1, [x8]
     %r = atomicrmw xchg ptr %ptr, i8 %value acq_rel, align 1
     ret i8 %r
 }
@@ -72,8 +72,8 @@ define dso_local i8 @atomicrmw_xchg_i8_aligned_seq_cst(ptr %ptr, i8 %value) {
 ; -O0:    subs w8, w8, w10, uxtb
 ;
 ; -O1-LABEL: atomicrmw_xchg_i8_aligned_seq_cst:
-; -O1:    ldaxrb w8, [x0]
-; -O1:    stlxrb w9, w1, [x0]
+; -O1:    ldaxrb w0, [x8]
+; -O1:    stlxrb w9, w1, [x8]
     %r = atomicrmw xchg ptr %ptr, i8 %value seq_cst, align 1
     ret i8 %r
 }
@@ -86,8 +86,8 @@ define dso_local i16 @atomicrmw_xchg_i16_aligned_monotonic(ptr %ptr, i16 %value)
 ; -O0:    subs w8, w8, w9, uxth
 ;
 ; -O1-LABEL: atomicrmw_xchg_i16_aligned_monotonic:
-; -O1:    ldxrh w8, [x0]
-; -O1:    stxrh w9, w1, [x0]
+; -O1:    ldxrh w0, [x8]
+; -O1:    stxrh w9, w1, [x8]
     %r = atomicrmw xchg ptr %ptr, i16 %value monotonic, align 2
     ret i16 %r
 }
@@ -100,8 +100,8 @@ define dso_local i16 @atomicrmw_xchg_i16_aligned_acquire(ptr %ptr, i16 %value) {
 ; -O0:    subs w8, w8, w9, uxth
 ;
 ; -O1-LABEL: atomicrmw_xchg_i16_aligned_acquire:
-; -O1:    ldaxrh w8, [x0]
-; -O1:    stxrh w9, w1, [x0]
+; -O1:    ldaxrh w0, [x8]
+; -O1:    stxrh w9, w1, [x8]
     %r = atomicrmw xchg ptr %ptr, i16 %value acquire, align 2
     ret i16 %r
 }
@@ -114,8 +114,8 @@ define dso_local i16 @atomicrmw_xchg_i16_aligned_release(ptr %ptr, i16 %value) {
 ; -O0:    subs w8, w8, w9, uxth
 ;
 ; -O1-LABEL: atomicrmw_xchg_i16_aligned_release:
-; -O1:    ldxrh w8, [x0]
-; -O1:    stlxrh w9, w1, [x0]
+; -O1:    ldxrh w0, [x8]
+; -O1:    stlxrh w9, w1, [x8]
     %r = atomicrmw xchg ptr %ptr, i16 %value release, align 2
     ret i16 %r
 }
@@ -128,8 +128,8 @@ define dso_local i16 @atomicrmw_xchg_i16_aligned_acq_rel(ptr %ptr, i16 %value) {
 ; -O0:    subs w8, w8, w9, uxth
 ;
 ; -O1-LABEL: atomicrmw_xchg_i16_aligned_acq_rel:
-; -O1:    ldaxrh w8, [x0]
-; -O1:    stlxrh w9, w1, [x0]
+; -O1:    ldaxrh w0, [x8]
+; -O1:    stlxrh w9, w1, [x8]
     %r = atomicrmw xchg ptr %ptr, i16 %value acq_rel, align 2
     ret i16 %r
 }
@@ -142,8 +142,8 @@ define dso_local i16 @atomicrmw_xchg_i16_aligned_seq_cst(ptr %ptr, i16 %value) {
 ; -O0:    subs w8, w8, w9, uxth
 ;
 ; -O1-LABEL: atomicrmw_xchg_i16_aligned_seq_cst:
-; -O1:    ldaxrh w8, [x0]
-; -O1:    stlxrh w9, w1, [x0]
+; -O1:    ldaxrh w0, [x8]
+; -O1:    stlxrh w9, w1, [x8]
     %r = atomicrmw xchg ptr %ptr, i16 %value seq_cst, align 2
     ret i16 %r
 }
@@ -392,8 +392,8 @@ define dso_local i8 @atomicrmw_xchg_i8_unaligned_monotonic(ptr %ptr, i8 %value)
 ; -O0:    subs w8, w8, w10, uxtb
 ;
 ; -O1-LABEL: atomicrmw_xchg_i8_unaligned_monotonic:
-; -O1:    ldxrb w8, [x0]
-; -O1:    stxrb w9, w1, [x0]
+; -O1:    ldxrb w0, [x8]
+; -O1:    stxrb w9, w1, [x8]
     %r = atomicrmw xchg ptr %ptr, i8 %value monotonic, align 1
     ret i8 %r
 }
@@ -407,8 +407,8 @@ define dso_local i8 @atomicrmw_xchg_i8_unaligned_acquire(ptr %ptr, i8 %value) {
 ; -O0:    subs w8, w8, w10, uxtb
 ;
 ; -O1-LABEL: atomicrmw_xchg_i8_unaligned_acquire:
-; -O1:    ldaxrb w8, [x0]
-; -O1:    stxrb w9, w1, [x0]
+; -O1:    ldaxrb w0, [x8]
+; -O1:    stxrb w9, w1, [x8]
     %r = atomicrmw xchg ptr %ptr, i8 %value acquire, align 1
     ret i8 %r
 }
@@ -422,8 +422,8 @@ define dso_local i8 @atomicrmw_xchg_i8_unaligned_release(ptr %ptr, i8 %value) {
 ; -O0:    subs w8, w8, w10, uxtb
 ;
 ; -O1-LABEL: atomicrmw_xchg_i8_unaligned_release:
-; -O1:    ldxrb w8, [x0]
-; -O1:    stlxrb w9, w1, [x0]
+; -O1:    ldxrb w0, [x8]
+; -O1:    stlxrb w9, w1, [x8]
     %r = atomicrmw xchg ptr %ptr, i8 %value release, align 1
     ret i8 %r
 }
@@ -437,8 +437,8 @@ define dso_local i8 @atomicrmw_xchg_i8_unaligned_acq_rel(ptr %ptr, i8 %value) {
 ; -O0:    subs w8, w8, w10, uxtb
 ;
 ; -O1-LABEL: atomicrmw_xchg_i8_unaligned_acq_rel:
-; -O1:    ldaxrb w8, [x0]
-; -O1:    stlxrb w9, w1, [x0]
+; -O1:    ldaxrb w0, [x8]
+; -O1:    stlxrb w9, w1, [x8]
     %r = atomicrmw xchg ptr %ptr, i8 %value acq_rel, align 1
     ret i8 %r
 }
@@ -452,8 +452,8 @@ define dso_local i8 @atomicrmw_xchg_i8_unaligned_seq_cst(ptr %ptr, i8 %value) {
 ; -O0:    subs w8, w8, w10, uxtb
 ;
 ; -O1-LABEL: atomicrmw_xchg_i8_unaligned_seq_cst:
-; -O1:    ldaxrb w8, [x0]
-; -O1:    stlxrb w9, w1, [x0]
+; -O1:    ldaxrb w0, [x8]
+; -O1:    stlxrb w9, w1, [x8]
     %r = atomicrmw xchg ptr %ptr, i8 %value seq_cst, align 1
     ret i8 %r
 }
diff --git a/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomicrmw-rcpc3.ll b/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomicrmw-rcpc3.ll
index 725558f2dcf72..28ee1a2a70c4d 100644
--- a/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomicrmw-rcpc3.ll
+++ b/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomicrmw-rcpc3.ll
@@ -12,8 +12,8 @@ define dso_local i8 @atomicrmw_xchg_i8_aligned_monotonic(ptr %ptr, i8 %value) {
 ; -O0:    subs w8, w8, w10, uxtb
 ;
 ; -O1-LABEL: atomicrmw_xchg_i8_aligned_monotonic:
-; -O1:    ldxrb w8, [x0]
-; -O1:    stxrb w9, w1, [x0]
+; -O1:    ldxrb w0, [x8]
+; -O1:    stxrb w9, w1, [x8]
     %r = atomicrmw xchg ptr %ptr, i8 %value monotonic, align 1
     ret i8 %r
 }
@@ -27,8 +27,8 @@ define dso_local i8 @atomicrmw_xchg_i8_aligned_acquire(ptr %ptr, i8 %value) {
 ; -O0:    subs w8, w8, w10, uxtb
 ;
 ; -O1-LABEL: atomicrmw_xchg_i8_aligned_acquire:
-; -O1:    ldaxrb w8, [x0]
-; -O1:    stxrb w9, w1, [x0]
+; -O1:    ldaxrb w0, [x8]
+; -O1:    stxrb w9, w1, [x8]
     %r = atomicrmw xchg ptr %ptr, i8 %value acquire, align 1
     ret i8 %r
 }
@@ -42,8 +42,8 @@ define dso_local i8 @atomicrmw_xchg_i8_aligned_release(ptr %ptr, i8 %value) {
 ; -O0:    subs w8, w8, w10, uxtb
 ;
 ; -O1-LABEL: atomicrmw_xchg_i8_aligned_release:
-; -O1:    ldxrb w8, [x0]
-; -O1:    stlxrb w9, w1, [x0]
+; -O1:    ldxrb w0, [x8]
+; -O1:    stlxrb w9, w1, [x8]
     %r = atomicrmw xchg ptr %ptr, i8 %value release, align 1
     ret i8 %r
 }
@@ -57,8 +57,8 @@ define dso_local i8 @atomicrmw_xchg_i8_aligned_acq_rel(ptr %ptr, i8 %value) {
 ; -O0:    subs w8, w8, w10, uxtb
 ;
 ; -O1-LABEL: atomicrmw_xchg_i8_aligned_acq_rel:
-; -O1:    ldaxrb w8, [x0]
-; -O1:    stlxrb w9, w1, [x0]
+; -O1:    ldaxrb w0, [x8]
+; -O1:    stlxrb w9, w1, [x8]
     %r = atomicrmw xchg ptr %ptr, i8 %value acq_rel, align 1
     ret i8 %r
 }
@@ -72,8 +72,8 @@ define dso_local i8 @atomicrmw_xchg_i8_aligned_seq_cst(ptr %ptr, i8 %value) {
 ; -O0:    subs w8, w8, w10, uxtb
 ;
 ; -O1-LABEL: atomicrmw_xchg_i8_aligned_seq_cst:
-; -O1:    ldaxrb w8, [x0]
-; -O1:    stlxrb w9, w1, [x0]
+; -O1:    ldaxrb w0, [x8]
+; -O1:    stlxrb w9, w1, [x8]
     %r = atomicrmw xchg ptr %ptr, i8 %value seq_cst, align 1
     ret i8 %r
 }
@@ -86,8 +86,8 @@ define dso_local i16 @atomicrmw_xchg_i16_aligned_monotonic(ptr %ptr, i16 %value)
 ; -O0:    subs w8, w8, w9, uxth
 ;
 ; -O1-LABEL: atomicrmw_xchg_i16_aligned_monotonic:
-; -O1:    ldxrh w8, [x0]
-; -O1:    stxrh w9, w1, [x0]
+; -O1:    ldxrh w0, [x8]
+; -O1:    stxrh w9, w1, [x8]
     %r = atomicrmw xchg ptr %ptr, i16 %value monotonic, align 2
     ret i16 %r
 }
@@ -100,8 +100,8 @@ define dso_local i16 @atomicrmw_xchg_i16_aligned_acquire(ptr %ptr, i16 %value) {
 ; -O0:    subs w8, w8, w9, uxth
 ;
 ; -O1-LABEL: atomicrmw_xchg_i16_aligned_acquire:
-; -O1:    ldaxrh w8, [x0]
-; -O1:    stxrh w9, w1, [x0]
+; -O1:    ldaxrh w0, [x8]
+; -O1:    stxrh w9, w1, [x8]
     %r = atomicrmw xchg ptr %ptr, i16 %value acquire, align 2
     ret i16 %r
 }
@@ -114,8 +114,8 @@ define dso_local i16 @atomicrmw_xchg_i16_aligned_release(ptr %ptr, i16 %value) {
 ; -O0:    subs w8, w8, w9, uxth
 ;
 ; -O1-LABEL: atomicrmw_xchg_i16_aligned_release:
-; -O1:    ldxrh w8, [x0]
-; -O1:    stlxrh w9, w1, [x0]
+; -O1:    ldxrh w0, [x8]
+; -O1:    stlxrh w9, w1, [x8]
     %r = atomicrmw xchg ptr %ptr, i16 %value release, align 2
     ret i16 %r
 }
@@ -128,8 +128,8 @@ define dso_local i16 @atomicrmw_xchg_i16_aligned_acq_rel(ptr %ptr, i16 %value) {
 ; -O0:    subs w8, w8, w9, uxth
 ;
 ; -O1-LABEL: atomicrmw_xchg_i16_aligned_acq_rel:
-; -O1:    ldaxrh w8, [x0]
-; -O1:    stlxrh w9, w1, [x0]
+; -O1:    ldaxrh w0, [x8]
+; -O1:    stlxrh w9, w1, [x8]
     %r = atomicrmw xchg ptr %ptr, i16 %value acq_rel, align 2
     ret i16 %r
 }
@@ -142,8 +142,8 @@ define dso_local i16 @atomicrmw_xchg_i16_aligned_seq_cst(ptr %ptr, i16 %value) {
 ; -O0:    subs w8, w8, w9, uxth
 ;
 ; -O1-LABEL: atomicrmw_xchg_i16_aligned_seq_cst:
-; -O1:    ldaxrh w8, [x0]
-; -O1:    stlxrh w9, w1, [x0]
+; -O1:    ldaxrh w0, [x8]
+; -O1:    stlxrh w9, w1, [x8]
     %r = atomicrmw xchg ptr %ptr, i16 %value seq_cst, align 2
     ret i16 %r
 }
@@ -392,8 +392,8 @@ define dso_local i8 @atomicrmw_xchg_i8_unaligned_monotonic(ptr %ptr, i8 %value)
 ; -O0:    subs w8, w8, w10, uxtb
 ;
 ; -O1-LABEL: atomicrmw_xchg_i8_unaligned_monotonic:
-; -O1:    ldxrb w8, [x0]
-; -O1:    stxrb w9, w1, [x0]
+; -O1:    ldxrb w0, [x8]
+; -O1:    stxrb w9, w1, [x8]
     %r = atomicrmw xchg ptr %ptr, i8 %value monotonic, align 1
     ret i8 %r
 }
@@ -407,8 +407,8 @@ define dso_local i8 @atomicrmw_xchg_i8_unaligned_acquire(ptr %ptr, i8 %value) {
 ; -O0:    subs w8, w8, w10, uxtb
 ;
 ; -O1-LABEL: atomicrmw_xchg_i8_unaligned_acquire:
-; -O1:    ldaxrb w8, [x0]
-; -O1:    stxrb w9, w1, [x0]
+; -O1:    ldaxrb w0, [x8]
+; -O1:    stxrb w9, w1, [x8]
     %r = atomicrmw xchg ptr %ptr, i8 %value acquire, align 1
     ret i8 %r
 }
@@ -422,8 +422,8 @@ define dso_local i8 @atomicrmw_xchg_i8_unaligned_release(ptr %ptr, i8 %value) {
 ; -O0:    subs w8, w8, w10, uxtb
 ;
 ; -O1-LABEL: atomicrmw_xchg_i8_unaligned_release:
-; -O1:    ldxrb w8, [x0]
-; -O1:    stlxrb w9, w1, [x0]
+; -O1:    ldxrb w0, [x8]
+; -O1:    stlxrb w9, w1, [x8]
     %r = atomicrmw xchg ptr %ptr, i8 %value release, align 1
     ret i8 %r
 }
@@ -437,8 +437,8 @@ define dso_local i8 @atomicrmw_xchg_i8_unaligned_acq_rel(ptr %ptr, i8 %value) {
 ; -O0:    subs w8, w8, w10, uxtb
 ;
 ; -O1-LABEL: atomicrmw_xchg_i8_unaligned_acq_rel:
-; -O1:    ldaxrb w8, [x0]
-; -O1:    stlxrb w9, w1, [x0]
+; -O1:    ldaxrb w0, [x8]
+; -O1:    stlxrb w9, w1, [x8]
     %r = atomicrmw xchg ptr %ptr, i8 %value acq_rel, align 1
     ret i8 %r
 }
@@ -452,8 +452,8 @@ define dso_local i8 @atomicrmw_xchg_i8_unaligned_seq_cst(ptr %ptr, i8 %value) {
 ; -O0:    subs w8, w8, w10, uxtb
 ;
 ; -O1-LABEL: atomicrmw_xchg_i8_unaligned_seq_cst:
-; -O1:    ldaxrb w8, [x0]
-; -O1:    stlxrb w9, w1, [x0]
+; -O1:    ldaxrb w0, [x8]
+; -O1:    stlxrb w9, w1, [x8]
     %r = atomicrmw xchg ptr %ptr, i8 %value seq_cst, align 1
     ret i8 %r
 }
diff --git a/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomicrmw-v8a.ll b/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomicrmw-v8a.ll
index 004e433d9408a..69220a6cf8132 100644
--- a/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomicrmw-v8a.ll
+++ b/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomicrmw-v8a.ll
@@ -12,8 +12,8 @@ define dso_local i8 @atomicrmw_xchg_i8_aligned_monotonic(ptr %ptr, i8 %value) {
 ; -O0:    subs w8, w8, w10, uxtb
 ;
 ; -O1-LABEL: atomicrmw_xchg_i8_aligned_monotonic:
-; -O1:    ldxrb w8, [x0]
-; -O1:    stxrb w9, w1, [x0]
+; -O1:    ldxrb w0, [x8]
+; -O1:    stxrb w9, w1, [x8]
     %r = atomicrmw xchg ptr %ptr, i8 %value monotonic, align 1
     ret i8 %r
 }
@@ -27,8 +27,8 @@ define dso_local i8 @atomicrmw_xchg_i8_aligned_acquire(ptr %ptr, i8 %value) {
 ; -O0:    subs w8, w8, w10, uxtb
 ;
 ; -O1-LABEL: atomicrmw_xchg_i8_aligned_acquire:
-; -O1:    ldaxrb w8, [x0]
-; -O1:    stxrb w9, w1, [x0]
+; -O1:    ldaxrb w0, [x8]
+; -O1:    stxrb w9, w1, [x8]
     %r = atomicrmw xchg ptr %ptr, i8 %value acquire, align 1
     ret i8 %r
 }
@@ -42,8 +42,8 @@ define dso_local i8 @atomicrmw_xchg_i8_aligned_release(ptr %ptr, i8 %value) {
 ; -O0:    subs w8, w8, w10, uxtb
 ;
 ; -O1-LABEL: atomicrmw_xchg_i8_aligned_release:
-; -O1:    ldxrb w8, [x0]
-; -O1:    stlxrb w9, w1, [x0]
+; -O1:    ldxrb w0, [x8]
+; -O1:    stlxrb w9, w1, [x8]
     %r = atomicrmw xchg ptr %ptr, i8 %value release, align 1
     ret i8 %r
 }
@@ -57,8 +57,8 @@ define dso_local i8 @atomicrmw_xchg_i8_aligned_acq_rel(ptr %ptr, i8 %value) {
 ; -O0:    subs w8, w8, w10, uxtb
 ;
 ; -O1-LABEL: atomicrmw_xchg_i8_aligned_acq_rel:
-; -O1:    ldaxrb w8, [x0]
-; -O1:    stlxrb w9, w1, [x0]
+; -O1:    ldaxrb w0, [x8]
+; -O1:    stlxrb w9, w1, [x8]
     %r = atomicrmw xchg ptr %ptr, i8 %value acq_rel, align 1
     ret i8 %r
 }
@@ -72,8 +72,8 @@ define dso_local i8 @atomicrmw_xchg_i8_aligned_seq_cst(ptr %ptr, i8 %value) {
 ; -O0:    subs w8, w8, w10, uxtb
 ;
 ; -O1-LABEL: atomicrmw_xchg_i8_aligned_seq_cst:
-; -O1:    ldaxrb w8, [x0]
-; -O1:    stlxrb w9, w1, [x0]
+; -O1:    ldaxrb w0, [x8]
+; -O1:    stlxrb w9, w1, [x8]
     %r = atomicrmw xchg ptr %ptr, i8 %value seq_cst, align 1
     ret i8 %r
 }
@@ -86,8 +86,8 @@ define dso_local i16 @atomicrmw_xchg_i16_aligned_monotonic(ptr %ptr, i16 %value)
 ; -O0:    subs w8, w8, w9, uxth
 ;
 ; -O1-LABEL: atomicrmw_xchg_i16_aligned_monotonic:
-; -O1:    ldxrh w8, [x0]
-; -O1:    stxrh w9, w1, [x0]
+; -O1:    ldxrh w0, [x8]
+; -O1:    stxrh w9, w1, [x8]
     %r = atomicrmw xchg ptr %ptr, i16 %value monotonic, align 2
     ret i16 %r
 }
@@ -100,8 +100,8 @@ define dso_local i16 @atomicrmw_xchg_i16_aligned_acquire(ptr %ptr, i16 %value) {
 ; -O0:    subs w8, w8, w9, uxth
 ;
 ; -O1-LABEL: atomicrmw_xchg_i16_aligned_acquire:
-; -O1:    ldaxrh w8, [x0]
-; -O1:    stxrh w9, w1, [x0]
+; -O1:    ldaxrh w0, [x8]
+; -O1:    stxrh w9, w1, [x8]
     %r = atomicrmw xchg ptr %ptr, i16 %value acquire, align 2
     ret i16 %r
 }
@@ -114,8 +114,8 @@ define dso_local i16 @atomicrmw_xchg_i16_aligned_release(ptr %ptr, i16 %value) {
 ; -O0:    subs w8, w8, w9, uxth
 ;
 ; -O1-LABEL: atomicrmw_xchg_i16_aligned_release:
-; -O1:    ldxrh w8, [x0]
-; -O1:    stlxrh w9, w1, [x0]
+; -O1:    ldxrh w0, [x8]
+; -O1:    stlxrh w9, w1, [x8]
     %r = atomicrmw xchg ptr %ptr, i16 %value release, align 2
     ret i16 %r
 }
@@ -128,8 +128,8 @@ define dso_local i16 @atomicrmw_xchg_i16_aligned_acq_rel(ptr %ptr, i16 %value) {
 ; -O0:    subs w8, w8, w9, uxth
 ;
 ; -O1-LABEL: atomicrmw_xchg_i16_aligned_acq_rel:
-; -O1:    ldaxrh w8, [x0]
-; -O1:    stlxrh w9, w1, [x0]
+; -O1:    ldaxrh w0, [x8]
+; -O1:    stlxrh w9, w1, [x8]
     %r = atomicrmw xchg ptr %ptr, i16 %value acq_rel, align 2
     ret i16 %r
 }
@@ -142,8 +142,8 @@ define dso_local i16 @atomicrmw_xchg_i16_aligned_seq_cst(ptr %ptr, i16 %value) {
 ; -O0:    subs w8, w8, w9, uxth
 ;
 ; -O1-LABEL: atomicrmw_xchg_i16_aligned_seq_cst:
-; -O1:    ldaxrh w8, [x0]
-; -O1:    stlxrh w9, w1, [x0]
+; -O1:    ldaxrh w0, [x8]
+; -O1:    stlxrh w9, w1, [x8]
     %r = atomicrmw xchg ptr %ptr, i16 %value seq_cst, align 2
     ret i16 %r
 }
@@ -392,8 +392,8 @@ define dso_local i8 @atomicrmw_xchg_i8_unaligned_monotonic(ptr %ptr, i8 %value)
 ; -O0:    subs w8, w8, w10, uxtb
 ;
 ; -O1-LABEL: atomicrmw_xchg_i8_unaligned_monotonic:
-; -O1:    ldxrb w8, [x0]
-; -O1:    stxrb w9, w1, [x0]
+; -O1:    ldxrb w0, [x8]
+; -O1:    stxrb w9, w1, [x8]
     %r = atomicrmw xchg ptr %ptr, i8 %value monotonic, align 1
     ret i8 %r
 }
@@ -407,8 +407,8 @@ define dso_local i8 @atomicrmw_xchg_i8_unaligned_acquire(ptr %ptr, i8 %value) {
 ; -O0:    subs w8, w8, w10, uxtb
 ;
 ; -O1-LABEL: atomicrmw_xchg_i8_unaligned_acquire:
-; -O1:    ldaxrb w8, [x0]
-; -O1:    stxrb w9, w1, [x0]
+; -O1:    ldaxrb w0, [x8]
+; -O1:    stxrb w9, w1, [x8]
     %r = atomicrmw xchg ptr %ptr, i8 %value acquire, align 1
     ret i8 %r
 }
@@ -422,8 +422,8 @@ define dso_local i8 @atomicrmw_xchg_i8_unaligned_release(ptr %ptr, i8 %value) {
 ; -O0:    subs w8, w8, w10, uxtb
 ;
 ; -O1-LABEL: atomicrmw_xchg_i8_unaligned_release:
-; -O1:    ldxrb w8, [x0]
-; -O1:    stlxrb w9, w1, [x0]
+; -O1:    ldxrb w0, [x8]
+; -O1:    stlxrb w9, w1, [x8]
     %r = atomicrmw xchg ptr %ptr, i8 %value release, align 1
     ret i8 %r
 }
@@ -437,8 +437,8 @@ define dso_local i8 @atomicrmw_xchg_i8_unaligned_acq_rel(ptr %ptr, i8 %value) {
 ; -O0:    subs w8, w8, w10, uxtb
 ;
 ; -O1-LABEL: atomicrmw_xchg_i8_unaligned_acq_rel:
-; -O1:    ldaxrb w8, [x0]
-; -O1:    stlxrb w9, w1, [x0]
+; -O1:    ldaxrb w0, [x8]
+; -O1:    stlxrb w9, w1, [x8]
     %r = atomicrmw xchg ptr %ptr, i8 %value acq_rel, align 1
     ret i8 %r
 }
@@ -452,8 +452,8 @@ define dso_local i8 @atomicrmw_xchg_i8_unaligned_seq_cst(ptr %ptr, i8 %value) {
 ; -O0:    subs w8, w8, w10, uxtb
 ;
 ; -O1-LABEL: atomicrmw_xchg_i8_unaligned_seq_cst:
-; -O1:    ldaxrb w8, [x0]
-; -O1:    stlxrb w9, w1, [x0]
+; -O1:    ldaxrb w0, [x8]
+; -O1:    stlxrb w9, w1, [x8]
     %r = atomicrmw xchg ptr %ptr, i8 %value seq_cst, align 1
     ret i8 %r
 }
diff --git a/llvm/test/CodeGen/AArch64/Atomics/aarch64_be-atomicrmw-lse2.ll b/llvm/test/CodeGen/AArch64/Atomics/aarch64_be-atomicrmw-lse2.ll
index 01317e09028c3..d92899d9aa3eb 100644
--- a/llvm/test/CodeGen/AArch64/Atomics/aarch64_be-atomicrmw-lse2.ll
+++ b/llvm/test/CodeGen/AArch64/Atomics/aarch64_be-atomicrmw-lse2.ll
@@ -12,8 +12,8 @@ define dso_local i8 @atomicrmw_xchg_i8_aligned_monotonic(ptr %ptr, i8 %value) {
 ; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_xchg_i8_aligned_monotonic:
-; -O1:    ldxrb w8, [x0]
-; -O1:    stxrb w9, w1, [x0]
+; -O1:    ldxrb w0, [x8]
+; -O1:    stxrb w9, w1, [x8]
     %r = atomicrmw xchg ptr %ptr, i8 %value monotonic, align 1
     ret i8 %r
 }
@@ -27,8 +27,8 @@ define dso_local i8 @atomicrmw_xchg_i8_aligned_acquire(ptr %ptr, i8 %value) {
 ; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_xchg_i8_aligned_acquire:
-; -O1:    ldaxrb w8, [x0]
-; -O1:    stxrb w9, w1, [x0]
+; -O1:    ldaxrb w0, [x8]
+; -O1:    stxrb w9, w1, [x8]
     %r = atomicrmw xchg ptr %ptr, i8 %value acquire, align 1
     ret i8 %r
 }
@@ -42,8 +42,8 @@ define dso_local i8 @atomicrmw_xchg_i8_aligned_release(ptr %ptr, i8 %value) {
 ; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_xchg_i8_aligned_release:
-; -O1:    ldxrb w8, [x0]
-; -O1:    stlxrb w9, w1, [x0]
+; -O1:    ldxrb w0, [x8]
+; -O1:    stlxrb w9, w1, [x8]
     %r = atomicrmw xchg ptr %ptr, i8 %value release, align 1
     ret i8 %r
 }
@@ -57,8 +57,8 @@ define dso_local i8 @atomicrmw_xchg_i8_aligned_acq_rel(ptr %ptr, i8 %value) {
 ; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_xchg_i8_aligned_acq_rel:
-; -O1:    ldaxrb w8, [x0]
-; -O1:    stlxrb w9, w1, [x0]
+; -O1:    ldaxrb w0, [x8]
+; -O1:    stlxrb w9, w1, [x8]
     %r = atomicrmw xchg ptr %ptr, i8 %value acq_rel, align 1
     ret i8 %r
 }
@@ -72,8 +72,8 @@ define dso_local i8 @atomicrmw_xchg_i8_aligned_seq_cst(ptr %ptr, i8 %value) {
 ; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_xchg_i8_aligned_seq_cst:
-; -O1:    ldaxrb w8, [x0]
-; -O1:    stlxrb w9, w1, [x0]
+; -O1:    ldaxrb w0, [x8]
+; -O1:    stlxrb w9, w1, [x8]
     %r = atomicrmw xchg ptr %ptr, i8 %value seq_cst, align 1
     ret i8 %r
 }
@@ -87,8 +87,8 @@ define dso_local i16 @atomicrmw_xchg_i16_aligned_monotonic(ptr %ptr, i16 %value)
 ; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_xchg_i16_aligned_monotonic:
-; -O1:    ldxrh w8, [x0]
-; -O1:    stxrh w9, w1, [x0]
+; -O1:    ldxrh w0, [x8]
+; -O1:    stxrh w9, w1, [x8]
     %r = atomicrmw xchg ptr %ptr, i16 %value monotonic, align 2
     ret i16 %r
 }
@@ -102,8 +102,8 @@ define dso_local i16 @atomicrmw_xchg_i16_aligned_acquire(ptr %ptr, i16 %value) {
 ; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_xchg_i16_aligned_acquire:
-; -O1:    ldaxrh w8, [x0]
-; -O1:    stxrh w9, w1, [x0]
+; -O1:    ldaxrh w0, [x8]
+; -O1:    stxrh w9, w1, [x8]
     %r = atomicrmw xchg ptr %ptr, i16 %value acquire, align 2
     ret i16 %r
 }
@@ -117,8 +117,8 @@ define dso_local i16 @atomicrmw_xchg_i16_aligned_release(ptr %ptr, i16 %value) {
 ; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_xchg_i16_aligned_release:
-; -O1:    ldxrh w8, [x0]
-; -O1:    stlxrh w9, w1, [x0]
+; -O1:    ldxrh w0, [x8]
+; -O1:    stlxrh w9, w1, [x8]
     %r = atomicrmw xchg ptr %ptr, i16 %value release, align 2
     ret i16 %r
 }
@@ -132,8 +132,8 @@ define dso_local i16 @atomicrmw_xchg_i16_aligned_acq_rel(ptr %ptr, i16 %value) {
 ; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_xchg_i16_aligned_acq_rel:
-; -O1:    ldaxrh w8, [x0]
-; -O1:    stlxrh w9, w1, [x0]
+; -O1:    ldaxrh w0, [x8]
+; -O1:    stlxrh w9, w1, [x8]
     %r = atomicrmw xchg ptr %ptr, i16 %value acq_rel, align 2
     ret i16 %r
 }
@@ -147,8 +147,8 @@ define dso_local i16 @atomicrmw_xchg_i16_aligned_seq_cst(ptr %ptr, i16 %value) {
 ; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_xchg_i16_aligned_seq_cst:
-; -O1:    ldaxrh w8, [x0]
-; -O1:    stlxrh w9, w1, [x0]
+; -O1:    ldaxrh w0, [x8]
+; -O1:    stlxrh w9, w1, [x8]
     %r = atomicrmw xchg ptr %ptr, i16 %value seq_cst, align 2
     ret i16 %r
 }
@@ -397,8 +397,8 @@ define dso_local i8 @atomicrmw_xchg_i8_unaligned_monotonic(ptr %ptr, i8 %value)
 ; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_xchg_i8_unaligned_monotonic:
-; -O1:    ldxrb w8, [x0]
-; -O1:    stxrb w9, w1, [x0]
+; -O1:    ldxrb w0, [x8]
+; -O1:    stxrb w9, w1, [x8]
     %r = atomicrmw xchg ptr %ptr, i8 %value monotonic, align 1
     ret i8 %r
 }
@@ -412,8 +412,8 @@ define dso_local i8 @atomicrmw_xchg_i8_unaligned_acquire(ptr %ptr, i8 %value) {
 ; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_xchg_i8_unaligned_acquire:
-; -O1:    ldaxrb w8, [x0]
-; -O1:    stxrb w9, w1, [x0]
+; -O1:    ldaxrb w0, [x8]
+; -O1:    stxrb w9, w1, [x8]
     %r = atomicrmw xchg ptr %ptr, i8 %value acquire, align 1
     ret i8 %r
 }
@@ -427,8 +427,8 @@ define dso_local i8 @atomicrmw_xchg_i8_unaligned_release(ptr %ptr, i8 %value) {
 ; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_xchg_i8_unaligned_release:
-; -O1:    ldxrb w8, [x0]
-; -O1:    stlxrb w9, w1, [x0]
+; -O1:    ldxrb w0, [x8]
+; -O1:    stlxrb w9, w1, [x8]
     %r = atomicrmw xchg ptr %ptr, i8 %value release, align 1
     ret i8 %r
 }
@@ -442,8 +442,8 @@ define dso_local i8 @atomicrmw_xchg_i8_unaligned_acq_rel(ptr %ptr, i8 %value) {
 ; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_xchg_i8_unaligned_acq_rel:
-; -O1:    ldaxrb w8, [x0]
-; -O1:    stlxrb w9, w1, [x0]
+; -O1:    ldaxrb w0, [x8]
+; -O1:    stlxrb w9, w1, [x8]
     %r = atomicrmw xchg ptr %ptr, i8 %value acq_rel, align 1
     ret i8 %r
 }
@@ -457,8 +457,8 @@ define dso_local i8 @atomicrmw_xchg_i8_unaligned_seq_cst(ptr %ptr, i8 %value) {
 ; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_xchg_i8_unaligned_seq_cst:
-; -O1:    ldaxrb w8, [x0]
-; -O1:    stlxrb w9, w1, [x0]
+; -O1:    ldaxrb w0, [x8]
+; -O1:    stlxrb w9, w1, [x8]
     %r = atomicrmw xchg ptr %ptr, i8 %value seq_cst, align 1
     ret i8 %r
 }
diff --git a/llvm/test/CodeGen/AArch64/Atomics/aarch64_be-atomicrmw-rcpc.ll b/llvm/test/CodeGen/AArch64/Atomics/aarch64_be-atomicrmw-rcpc.ll
index 1bead6d694c65..c09a2c593f0f3 100644
--- a/llvm/test/CodeGen/AArch64/Atomics/aarch64_be-atomicrmw-rcpc.ll
+++ b/llvm/test/CodeGen/AArch64/Atomics/aarch64_be-atomicrmw-rcpc.ll
@@ -12,8 +12,8 @@ define dso_local i8 @atomicrmw_xchg_i8_aligned_monotonic(ptr %ptr, i8 %value) {
 ; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_xchg_i8_aligned_monotonic:
-; -O1:    ldxrb w8, [x0]
-; -O1:    stxrb w9, w1, [x0]
+; -O1:    ldxrb w0, [x8]
+; -O1:    stxrb w9, w1, [x8]
     %r = atomicrmw xchg ptr %ptr, i8 %value monotonic, align 1
     ret i8 %r
 }
@@ -27,8 +27,8 @@ define dso_local i8 @atomicrmw_xchg_i8_aligned_acquire(ptr %ptr, i8 %value) {
 ; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_xchg_i8_aligned_acquire:
-; -O1:    ldaxrb w8, [x0]
-; -O1:    stxrb w9, w1, [x0]
+; -O1:    ldaxrb w0, [x8]
+; -O1:    stxrb w9, w1, [x8]
     %r = atomicrmw xchg ptr %ptr, i8 %value acquire, align 1
     ret i8 %r
 }
@@ -42,8 +42,8 @@ define dso_local i8 @atomicrmw_xchg_i8_aligned_release(ptr %ptr, i8 %value) {
 ; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_xchg_i8_aligned_release:
-; -O1:    ldxrb w8, [x0]
-; -O1:    stlxrb w9, w1, [x0]
+; -O1:    ldxrb w0, [x8]
+; -O1:    stlxrb w9, w1, [x8]
     %r = atomicrmw xchg ptr %ptr, i8 %value release, align 1
     ret i8 %r
 }
@@ -57,8 +57,8 @@ define dso_local i8 @atomicrmw_xchg_i8_aligned_acq_rel(ptr %ptr, i8 %value) {
 ; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_xchg_i8_aligned_acq_rel:
-; -O1:    ldaxrb w8, [x0]
-; -O1:    stlxrb w9, w1, [x0]
+; -O1:    ldaxrb w0, [x8]
+; -O1:    stlxrb w9, w1, [x8]
     %r = atomicrmw xchg ptr %ptr, i8 %value acq_rel, align 1
     ret i8 %r
 }
@@ -72,8 +72,8 @@ define dso_local i8 @atomicrmw_xchg_i8_aligned_seq_cst(ptr %ptr, i8 %value) {
 ; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_xchg_i8_aligned_seq_cst:
-; -O1:    ldaxrb w8, [x0]
-; -O1:    stlxrb w9, w1, [x0]
+; -O1:    ldaxrb w0, [x8]
+; -O1:    stlxrb w9, w1, [x8]
     %r = atomicrmw xchg ptr %ptr, i8 %value seq_cst, align 1
     ret i8 %r
 }
@@ -87,8 +87,8 @@ define dso_local i16 @atomicrmw_xchg_i16_aligned_monotonic(ptr %ptr, i16 %value)
 ; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_xchg_i16_aligned_monotonic:
-; -O1:    ldxrh w8, [x0]
-; -O1:    stxrh w9, w1, [x0]
+; -O1:    ldxrh w0, [x8]
+; -O1:    stxrh w9, w1, [x8]
     %r = atomicrmw xchg ptr %ptr, i16 %value monotonic, align 2
     ret i16 %r
 }
@@ -102,8 +102,8 @@ define dso_local i16 @atomicrmw_xchg_i16_aligned_acquire(ptr %ptr, i16 %value) {
 ; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_xchg_i16_aligned_acquire:
-; -O1:    ldaxrh w8, [x0]
-; -O1:    stxrh w9, w1, [x0]
+; -O1:    ldaxrh w0, [x8]
+; -O1:    stxrh w9, w1, [x8]
     %r = atomicrmw xchg ptr %ptr, i16 %value acquire, align 2
     ret i16 %r
 }
@@ -117,8 +117,8 @@ define dso_local i16 @atomicrmw_xchg_i16_aligned_release(ptr %ptr, i16 %value) {
 ; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_xchg_i16_aligned_release:
-; -O1:    ldxrh w8, [x0]
-; -O1:    stlxrh w9, w1, [x0]
+; -O1:    ldxrh w0, [x8]
+; -O1:    stlxrh w9, w1, [x8]
     %r = atomicrmw xchg ptr %ptr, i16 %value release, align 2
     ret i16 %r
 }
@@ -132,8 +132,8 @@ define dso_local i16 @atomicrmw_xchg_i16_aligned_acq_rel(ptr %ptr, i16 %value) {
 ; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_xchg_i16_aligned_acq_rel:
-; -O1:    ldaxrh w8, [x0]
-; -O1:    stlxrh w9, w1, [x0]
+; -O1:    ldaxrh w0, [x8]
+; -O1:    stlxrh w9, w1, [x8]
     %r = atomicrmw xchg ptr %ptr, i16 %value acq_rel, align 2
     ret i16 %r
 }
@@ -147,8 +147,8 @@ define dso_local i16 @atomicrmw_xchg_i16_aligned_seq_cst(ptr %ptr, i16 %value) {
 ; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_xchg_i16_aligned_seq_cst:
-; -O1:    ldaxrh w8, [x0]
-; -O1:    stlxrh w9, w1, [x0]
+; -O1:    ldaxrh w0, [x8]
+; -O1:    stlxrh w9, w1, [x8]
     %r = atomicrmw xchg ptr %ptr, i16 %value seq_cst, align 2
     ret i16 %r
 }
@@ -397,8 +397,8 @@ define dso_local i8 @atomicrmw_xchg_i8_unaligned_monotonic(ptr %ptr, i8 %value)
 ; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_xchg_i8_unaligned_monotonic:
-; -O1:    ldxrb w8, [x0]
-; -O1:    stxrb w9, w1, [x0]
+; -O1:    ldxrb w0, [x8]
+; -O1:    stxrb w9, w1, [x8]
     %r = atomicrmw xchg ptr %ptr, i8 %value monotonic, align 1
     ret i8 %r
 }
@@ -412,8 +412,8 @@ define dso_local i8 @atomicrmw_xchg_i8_unaligned_acquire(ptr %ptr, i8 %value) {
 ; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_xchg_i8_unaligned_acquire:
-; -O1:    ldaxrb w8, [x0]
-; -O1:    stxrb w9, w1, [x0]
+; -O1:    ldaxrb w0, [x8]
+; -O1:    stxrb w9, w1, [x8]
     %r = atomicrmw xchg ptr %ptr, i8 %value acquire, align 1
     ret i8 %r
 }
@@ -427,8 +427,8 @@ define dso_local i8 @atomicrmw_xchg_i8_unaligned_release(ptr %ptr, i8 %value) {
 ; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_xchg_i8_unaligned_release:
-; -O1:    ldxrb w8, [x0]
-; -O1:    stlxrb w9, w1, [x0]
+; -O1:    ldxrb w0, [x8]
+; -O1:    stlxrb w9, w1, [x8]
     %r = atomicrmw xchg ptr %ptr, i8 %value release, align 1
     ret i8 %r
 }
@@ -442,8 +442,8 @@ define dso_local i8 @atomicrmw_xchg_i8_unaligned_acq_rel(ptr %ptr, i8 %value) {
 ; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_xchg_i8_unaligned_acq_rel:
-; -O1:    ldaxrb w8, [x0]
-; -O1:    stlxrb w9, w1, [x0]
+; -O1:    ldaxrb w0, [x8]
+; -O1:    stlxrb w9, w1, [x8]
     %r = atomicrmw xchg ptr %ptr, i8 %value acq_rel, align 1
     ret i8 %r
 }
@@ -457,8 +457,8 @@ define dso_local i8 @atomicrmw_xchg_i8_unaligned_seq_cst(ptr %ptr, i8 %value) {
 ; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_xchg_i8_unaligned_seq_cst:
-; -O1:    ldaxrb w8, [x0]
-; -O1:    stlxrb w9, w1, [x0]
+; -O1:    ldaxrb w0, [x8]
+; -O1:    stlxrb w9, w1, [x8]
     %r = atomicrmw xchg ptr %ptr, i8 %value seq_cst, align 1
     ret i8 %r
 }
diff --git a/llvm/test/CodeGen/AArch64/Atomics/aarch64_be-atomicrmw-rcpc3.ll b/llvm/test/CodeGen/AArch64/Atomics/aarch64_be-atomicrmw-rcpc3.ll
index 51d9766f6a8f9..30166b4e74116 100644
--- a/llvm/test/CodeGen/AArch64/Atomics/aarch64_be-atomicrmw-rcpc3.ll
+++ b/llvm/test/CodeGen/AArch64/Atomics/aarch64_be-atomicrmw-rcpc3.ll
@@ -12,8 +12,8 @@ define dso_local i8 @atomicrmw_xchg_i8_aligned_monotonic(ptr %ptr, i8 %value) {
 ; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_xchg_i8_aligned_monotonic:
-; -O1:    ldxrb w8, [x0]
-; -O1:    stxrb w9, w1, [x0]
+; -O1:    ldxrb w0, [x8]
+; -O1:    stxrb w9, w1, [x8]
     %r = atomicrmw xchg ptr %ptr, i8 %value monotonic, align 1
     ret i8 %r
 }
@@ -27,8 +27,8 @@ define dso_local i8 @atomicrmw_xchg_i8_aligned_acquire(ptr %ptr, i8 %value) {
 ; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_xchg_i8_aligned_acquire:
-; -O1:    ldaxrb w8, [x0]
-; -O1:    stxrb w9, w1, [x0]
+; -O1:    ldaxrb w0, [x8]
+; -O1:    stxrb w9, w1, [x8]
     %r = atomicrmw xchg ptr %ptr, i8 %value acquire, align 1
     ret i8 %r
 }
@@ -42,8 +42,8 @@ define dso_local i8 @atomicrmw_xchg_i8_aligned_release(ptr %ptr, i8 %value) {
 ; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_xchg_i8_aligned_release:
-; -O1:    ldxrb w8, [x0]
-; -O1:    stlxrb w9, w1, [x0]
+; -O1:    ldxrb w0, [x8]
+; -O1:    stlxrb w9, w1, [x8]
     %r = atomicrmw xchg ptr %ptr, i8 %value release, align 1
     ret i8 %r
 }
@@ -57,8 +57,8 @@ define dso_local i8 @atomicrmw_xchg_i8_aligned_acq_rel(ptr %ptr, i8 %value) {
 ; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_xchg_i8_aligned_acq_rel:
-; -O1:    ldaxrb w8, [x0]
-; -O1:    stlxrb w9, w1, [x0]
+; -O1:    ldaxrb w0, [x8]
+; -O1:    stlxrb w9, w1, [x8]
     %r = atomicrmw xchg ptr %ptr, i8 %value acq_rel, align 1
     ret i8 %r
 }
@@ -72,8 +72,8 @@ define dso_local i8 @atomicrmw_xchg_i8_aligned_seq_cst(ptr %ptr, i8 %value) {
 ; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_xchg_i8_aligned_seq_cst:
-; -O1:    ldaxrb w8, [x0]
-; -O1:    stlxrb w9, w1, [x0]
+; -O1:    ldaxrb w0, [x8]
+; -O1:    stlxrb w9, w1, [x8]
     %r = atomicrmw xchg ptr %ptr, i8 %value seq_cst, align 1
     ret i8 %r
 }
@@ -87,8 +87,8 @@ define dso_local i16 @atomicrmw_xchg_i16_aligned_monotonic(ptr %ptr, i16 %value)
 ; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_xchg_i16_aligned_monotonic:
-; -O1:    ldxrh w8, [x0]
-; -O1:    stxrh w9, w1, [x0]
+; -O1:    ldxrh w0, [x8]
+; -O1:    stxrh w9, w1, [x8]
     %r = atomicrmw xchg ptr %ptr, i16 %value monotonic, align 2
     ret i16 %r
 }
@@ -102,8 +102,8 @@ define dso_local i16 @atomicrmw_xchg_i16_aligned_acquire(ptr %ptr, i16 %value) {
 ; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_xchg_i16_aligned_acquire:
-; -O1:    ldaxrh w8, [x0]
-; -O1:    stxrh w9, w1, [x0]
+; -O1:    ldaxrh w0, [x8]
+; -O1:    stxrh w9, w1, [x8]
     %r = atomicrmw xchg ptr %ptr, i16 %value acquire, align 2
     ret i16 %r
 }
@@ -117,8 +117,8 @@ define dso_local i16 @atomicrmw_xchg_i16_aligned_release(ptr %ptr, i16 %value) {
 ; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_xchg_i16_aligned_release:
-; -O1:    ldxrh w8, [x0]
-; -O1:    stlxrh w9, w1, [x0]
+; -O1:    ldxrh w0, [x8]
+; -O1:    stlxrh w9, w1, [x8]
     %r = atomicrmw xchg ptr %ptr, i16 %value release, align 2
     ret i16 %r
 }
@@ -132,8 +132,8 @@ define dso_local i16 @atomicrmw_xchg_i16_aligned_acq_rel(ptr %ptr, i16 %value) {
 ; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_xchg_i16_aligned_acq_rel:
-; -O1:    ldaxrh w8, [x0]
-; -O1:    stlxrh w9, w1, [x0]
+; -O1:    ldaxrh w0, [x8]
+; -O1:    stlxrh w9, w1, [x8]
     %r = atomicrmw xchg ptr %ptr, i16 %value acq_rel, align 2
     ret i16 %r
 }
@@ -147,8 +147,8 @@ define dso_local i16 @atomicrmw_xchg_i16_aligned_seq_cst(ptr %ptr, i16 %value) {
 ; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_xchg_i16_aligned_seq_cst:
-; -O1:    ldaxrh w8, [x0]
-; -O1:    stlxrh w9, w1, [x0]
+; -O1:    ldaxrh w0, [x8]
+; -O1:    stlxrh w9, w1, [x8]
     %r = atomicrmw xchg ptr %ptr, i16 %value seq_cst, align 2
     ret i16 %r
 }
@@ -397,8 +397,8 @@ define dso_local i8 @atomicrmw_xchg_i8_unaligned_monotonic(ptr %ptr, i8 %value)
 ; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_xchg_i8_unaligned_monotonic:
-; -O1:    ldxrb w8, [x0]
-; -O1:    stxrb w9, w1, [x0]
+; -O1:    ldxrb w0, [x8]
+; -O1:    stxrb w9, w1, [x8]
     %r = atomicrmw xchg ptr %ptr, i8 %value monotonic, align 1
     ret i8 %r
 }
@@ -412,8 +412,8 @@ define dso_local i8 @atomicrmw_xchg_i8_unaligned_acquire(ptr %ptr, i8 %value) {
 ; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_xchg_i8_unaligned_acquire:
-; -O1:    ldaxrb w8, [x0]
-; -O1:    stxrb w9, w1, [x0]
+; -O1:    ldaxrb w0, [x8]
+; -O1:    stxrb w9, w1, [x8]
     %r = atomicrmw xchg ptr %ptr, i8 %value acquire, align 1
     ret i8 %r
 }
@@ -427,8 +427,8 @@ define dso_local i8 @atomicrmw_xchg_i8_unaligned_release(ptr %ptr, i8 %value) {
 ; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_xchg_i8_unaligned_release:
-; -O1:    ldxrb w8, [x0]
-; -O1:    stlxrb w9, w1, [x0]
+; -O1:    ldxrb w0, [x8]
+; -O1:    stlxrb w9, w1, [x8]
     %r = atomicrmw xchg ptr %ptr, i8 %value release, align 1
     ret i8 %r
 }
@@ -442,8 +442,8 @@ define dso_local i8 @atomicrmw_xchg_i8_unaligned_acq_rel(ptr %ptr, i8 %value) {
 ; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_xchg_i8_unaligned_acq_rel:
-; -O1:    ldaxrb w8, [x0]
-; -O1:    stlxrb w9, w1, [x0]
+; -O1:    ldaxrb w0, [x8]
+; -O1:    stlxrb w9, w1, [x8]
     %r = atomicrmw xchg ptr %ptr, i8 %value acq_rel, align 1
     ret i8 %r
 }
@@ -457,8 +457,8 @@ define dso_local i8 @atomicrmw_xchg_i8_unaligned_seq_cst(ptr %ptr, i8 %value) {
 ; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_xchg_i8_unaligned_seq_cst:
-; -O1:    ldaxrb w8, [x0]
-; -O1:    stlxrb w9, w1, [x0]
+; -O1:    ldaxrb w0, [x8]
+; -O1:    stlxrb w9, w1, [x8]
     %r = atomicrmw xchg ptr %ptr, i8 %value seq_cst, align 1
     ret i8 %r
 }
diff --git a/llvm/test/CodeGen/AArch64/Atomics/aarch64_be-atomicrmw-v8a.ll b/llvm/test/CodeGen/AArch64/Atomics/aarch64_be-atomicrmw-v8a.ll
index a58e5a987bb4c..e9c1b33dc3d21 100644
--- a/llvm/test/CodeGen/AArch64/Atomics/aarch64_be-atomicrmw-v8a.ll
+++ b/llvm/test/CodeGen/AArch64/Atomics/aarch64_be-atomicrmw-v8a.ll
@@ -12,8 +12,8 @@ define dso_local i8 @atomicrmw_xchg_i8_aligned_monotonic(ptr %ptr, i8 %value) {
 ; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_xchg_i8_aligned_monotonic:
-; -O1:    ldxrb w8, [x0]
-; -O1:    stxrb w9, w1, [x0]
+; -O1:    ldxrb w0, [x8]
+; -O1:    stxrb w9, w1, [x8]
     %r = atomicrmw xchg ptr %ptr, i8 %value monotonic, align 1
     ret i8 %r
 }
@@ -27,8 +27,8 @@ define dso_local i8 @atomicrmw_xchg_i8_aligned_acquire(ptr %ptr, i8 %value) {
 ; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_xchg_i8_aligned_acquire:
-; -O1:    ldaxrb w8, [x0]
-; -O1:    stxrb w9, w1, [x0]
+; -O1:    ldaxrb w0, [x8]
+; -O1:    stxrb w9, w1, [x8]
     %r = atomicrmw xchg ptr %ptr, i8 %value acquire, align 1
     ret i8 %r
 }
@@ -42,8 +42,8 @@ define dso_local i8 @atomicrmw_xchg_i8_aligned_release(ptr %ptr, i8 %value) {
 ; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_xchg_i8_aligned_release:
-; -O1:    ldxrb w8, [x0]
-; -O1:    stlxrb w9, w1, [x0]
+; -O1:    ldxrb w0, [x8]
+; -O1:    stlxrb w9, w1, [x8]
     %r = atomicrmw xchg ptr %ptr, i8 %value release, align 1
     ret i8 %r
 }
@@ -57,8 +57,8 @@ define dso_local i8 @atomicrmw_xchg_i8_aligned_acq_rel(ptr %ptr, i8 %value) {
 ; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_xchg_i8_aligned_acq_rel:
-; -O1:    ldaxrb w8, [x0]
-; -O1:    stlxrb w9, w1, [x0]
+; -O1:    ldaxrb w0, [x8]
+; -O1:    stlxrb w9, w1, [x8]
     %r = atomicrmw xchg ptr %ptr, i8 %value acq_rel, align 1
     ret i8 %r
 }
@@ -72,8 +72,8 @@ define dso_local i8 @atomicrmw_xchg_i8_aligned_seq_cst(ptr %ptr, i8 %value) {
 ; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_xchg_i8_aligned_seq_cst:
-; -O1:    ldaxrb w8, [x0]
-; -O1:    stlxrb w9, w1, [x0]
+; -O1:    ldaxrb w0, [x8]
+; -O1:    stlxrb w9, w1, [x8]
     %r = atomicrmw xchg ptr %ptr, i8 %value seq_cst, align 1
     ret i8 %r
 }
@@ -87,8 +87,8 @@ define dso_local i16 @atomicrmw_xchg_i16_aligned_monotonic(ptr %ptr, i16 %value)
 ; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_xchg_i16_aligned_monotonic:
-; -O1:    ldxrh w8, [x0]
-; -O1:    stxrh w9, w1, [x0]
+; -O1:    ldxrh w0, [x8]
+; -O1:    stxrh w9, w1, [x8]
     %r = atomicrmw xchg ptr %ptr, i16 %value monotonic, align 2
     ret i16 %r
 }
@@ -102,8 +102,8 @@ define dso_local i16 @atomicrmw_xchg_i16_aligned_acquire(ptr %ptr, i16 %value) {
 ; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_xchg_i16_aligned_acquire:
-; -O1:    ldaxrh w8, [x0]
-; -O1:    stxrh w9, w1, [x0]
+; -O1:    ldaxrh w0, [x8]
+; -O1:    stxrh w9, w1, [x8]
     %r = atomicrmw xchg ptr %ptr, i16 %value acquire, align 2
     ret i16 %r
 }
@@ -117,8 +117,8 @@ define dso_local i16 @atomicrmw_xchg_i16_aligned_release(ptr %ptr, i16 %value) {
 ; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_xchg_i16_aligned_release:
-; -O1:    ldxrh w8, [x0]
-; -O1:    stlxrh w9, w1, [x0]
+; -O1:    ldxrh w0, [x8]
+; -O1:    stlxrh w9, w1, [x8]
     %r = atomicrmw xchg ptr %ptr, i16 %value release, align 2
     ret i16 %r
 }
@@ -132,8 +132,8 @@ define dso_local i16 @atomicrmw_xchg_i16_aligned_acq_rel(ptr %ptr, i16 %value) {
 ; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_xchg_i16_aligned_acq_rel:
-; -O1:    ldaxrh w8, [x0]
-; -O1:    stlxrh w9, w1, [x0]
+; -O1:    ldaxrh w0, [x8]
+; -O1:    stlxrh w9, w1, [x8]
     %r = atomicrmw xchg ptr %ptr, i16 %value acq_rel, align 2
     ret i16 %r
 }
@@ -147,8 +147,8 @@ define dso_local i16 @atomicrmw_xchg_i16_aligned_seq_cst(ptr %ptr, i16 %value) {
 ; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_xchg_i16_aligned_seq_cst:
-; -O1:    ldaxrh w8, [x0]
-; -O1:    stlxrh w9, w1, [x0]
+; -O1:    ldaxrh w0, [x8]
+; -O1:    stlxrh w9, w1, [x8]
     %r = atomicrmw xchg ptr %ptr, i16 %value seq_cst, align 2
     ret i16 %r
 }
@@ -397,8 +397,8 @@ define dso_local i8 @atomicrmw_xchg_i8_unaligned_monotonic(ptr %ptr, i8 %value)
 ; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_xchg_i8_unaligned_monotonic:
-; -O1:    ldxrb w8, [x0]
-; -O1:    stxrb w9, w1, [x0]
+; -O1:    ldxrb w0, [x8]
+; -O1:    stxrb w9, w1, [x8]
     %r = atomicrmw xchg ptr %ptr, i8 %value monotonic, align 1
     ret i8 %r
 }
@@ -412,8 +412,8 @@ define dso_local i8 @atomicrmw_xchg_i8_unaligned_acquire(ptr %ptr, i8 %value) {
 ; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_xchg_i8_unaligned_acquire:
-; -O1:    ldaxrb w8, [x0]
-; -O1:    stxrb w9, w1, [x0]
+; -O1:    ldaxrb w0, [x8]
+; -O1:    stxrb w9, w1, [x8]
     %r = atomicrmw xchg ptr %ptr, i8 %value acquire, align 1
     ret i8 %r
 }
@@ -427,8 +427,8 @@ define dso_local i8 @atomicrmw_xchg_i8_unaligned_release(ptr %ptr, i8 %value) {
 ; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_xchg_i8_unaligned_release:
-; -O1:    ldxrb w8, [x0]
-; -O1:    stlxrb w9, w1, [x0]
+; -O1:    ldxrb w0, [x8]
+; -O1:    stlxrb w9, w1, [x8]
     %r = atomicrmw xchg ptr %ptr, i8 %value release, align 1
     ret i8 %r
 }
@@ -442,8 +442,8 @@ define dso_local i8 @atomicrmw_xchg_i8_unaligned_acq_rel(ptr %ptr, i8 %value) {
 ; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_xchg_i8_unaligned_acq_rel:
-; -O1:    ldaxrb w8, [x0]
-; -O1:    stlxrb w9, w1, [x0]
+; -O1:    ldaxrb w0, [x8]
+; -O1:    stlxrb w9, w1, [x8]
     %r = atomicrmw xchg ptr %ptr, i8 %value acq_rel, align 1
     ret i8 %r
 }
@@ -457,8 +457,8 @@ define dso_local i8 @atomicrmw_xchg_i8_unaligned_seq_cst(ptr %ptr, i8 %value) {
 ; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_xchg_i8_unaligned_seq_cst:
-; -O1:    ldaxrb w8, [x0]
-; -O1:    stlxrb w9, w1, [x0]
+; -O1:    ldaxrb w0, [x8]
+; -O1:    stlxrb w9, w1, [x8]
     %r = atomicrmw xchg ptr %ptr, i8 %value seq_cst, align 1
     ret i8 %r
 }
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/arm64-atomic.ll b/llvm/test/CodeGen/AArch64/GlobalISel/arm64-atomic.ll
index de3f323891a36..5bc041aef88ba 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/arm64-atomic.ll
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/arm64-atomic.ll
@@ -1938,7 +1938,6 @@ define i8 @atomicrmw_add_i8(ptr %ptr, i8 %rhs) {
 define i8 @atomicrmw_xchg_i8(ptr %ptr, i8 %rhs) {
 ; CHECK-NOLSE-O1-LABEL: atomicrmw_xchg_i8:
 ; CHECK-NOLSE-O1:       ; %bb.0:
-; CHECK-NOLSE-O1-NEXT:    ; kill: def $w1 killed $w1 def $x1
 ; CHECK-NOLSE-O1-NEXT:  LBB28_1: ; %atomicrmw.start
 ; CHECK-NOLSE-O1-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; CHECK-NOLSE-O1-NEXT:    ldxrb w8, [x0]
@@ -2993,7 +2992,6 @@ define i16 @atomicrmw_add_i16(ptr %ptr, i16 %rhs) {
 define i16 @atomicrmw_xchg_i16(ptr %ptr, i16 %rhs) {
 ; CHECK-NOLSE-O1-LABEL: atomicrmw_xchg_i16:
 ; CHECK-NOLSE-O1:       ; %bb.0:
-; CHECK-NOLSE-O1-NEXT:    ; kill: def $w1 killed $w1 def $x1
 ; CHECK-NOLSE-O1-NEXT:  LBB38_1: ; %atomicrmw.start
 ; CHECK-NOLSE-O1-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; CHECK-NOLSE-O1-NEXT:    ldxrh w8, [x0]
@@ -5996,7 +5994,6 @@ define { i8, i1 } @cmpxchg_i8(ptr %ptr, i8 %desired, i8 %new) {
 ; CHECK-NOLSE-O1-LABEL: cmpxchg_i8:
 ; CHECK-NOLSE-O1:       ; %bb.0:
 ; CHECK-NOLSE-O1-NEXT:    mov x8, x0
-; CHECK-NOLSE-O1-NEXT:    ; kill: def $w2 killed $w2 def $x2
 ; CHECK-NOLSE-O1-NEXT:  LBB67_1: ; %cmpxchg.start
 ; CHECK-NOLSE-O1-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; CHECK-NOLSE-O1-NEXT:    ldxrb w0, [x8]
@@ -6103,7 +6100,6 @@ define { i16, i1 } @cmpxchg_i16(ptr %ptr, i16 %desired, i16 %new) {
 ; CHECK-NOLSE-O1-LABEL: cmpxchg_i16:
 ; CHECK-NOLSE-O1:       ; %bb.0:
 ; CHECK-NOLSE-O1-NEXT:    mov x8, x0
-; CHECK-NOLSE-O1-NEXT:    ; kill: def $w2 killed $w2 def $x2
 ; CHECK-NOLSE-O1-NEXT:  LBB68_1: ; %cmpxchg.start
 ; CHECK-NOLSE-O1-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; CHECK-NOLSE-O1-NEXT:    ldxrh w0, [x8]
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/arm64-irtranslator.ll b/llvm/test/CodeGen/AArch64/GlobalISel/arm64-irtranslator.ll
index 7a67cf3fd4c94..15ee5e48a88e6 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/arm64-irtranslator.ll
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/arm64-irtranslator.ll
@@ -2262,7 +2262,7 @@ declare ptr @llvm.invariant.start.p0(i64, ptr nocapture) readonly nounwind
 declare void @llvm.invariant.end.p0(ptr, i64, ptr nocapture) nounwind
 define void @test_invariant_intrin() {
 ; CHECK-LABEL: name: test_invariant_intrin
-; CHECK: %{{[0-9]+}}:_(s64) = G_IMPLICIT_DEF
+; CHECK: %{{[0-9]+}}:_(p0) = G_IMPLICIT_DEF
 ; CHECK-NEXT: RET_ReallyLR
   %x = alloca %t
   %inv = call ptr @llvm.invariant.start.p0(i64 8, ptr %x)
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/arm64-pcsections.ll b/llvm/test/CodeGen/AArch64/GlobalISel/arm64-pcsections.ll
index c6819ff39ed33..2779e89c373fc 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/arm64-pcsections.ll
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/arm64-pcsections.ll
@@ -727,8 +727,8 @@ define i8 @atomicrmw_add_i8(ptr %ptr, i8 %rhs) {
   ; CHECK-NEXT:   liveins: $w1, $x0
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT:   renamable $w8 = LDAXRB renamable $x0, implicit-def $x8, pcsections !0 :: (volatile load (s8) from %ir.ptr)
-  ; CHECK-NEXT:   $w9 = ADDWrs renamable $w8, renamable $w1, 0, implicit-def $x9, pcsections !0
-  ; CHECK-NEXT:   early-clobber renamable $w10 = STLXRB renamable $w9, renamable $x0, implicit killed $x9, pcsections !0 :: (volatile store (s8) into %ir.ptr)
+  ; CHECK-NEXT:   $w9 = ADDWrs renamable $w8, renamable $w1, 0, pcsections !0
+  ; CHECK-NEXT:   early-clobber renamable $w10 = STLXRB killed renamable $w9, renamable $x0, pcsections !0 :: (volatile store (s8) into %ir.ptr)
   ; CHECK-NEXT:   CBNZW killed renamable $w10, %bb.1, pcsections !0
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT: bb.2.atomicrmw.end:
@@ -746,11 +746,9 @@ define i8 @atomicrmw_xchg_i8(ptr %ptr, i8 %rhs) {
   ; CHECK-NEXT:   successors: %bb.1(0x80000000)
   ; CHECK-NEXT:   liveins: $w1, $x0
   ; CHECK-NEXT: {{  $}}
-  ; CHECK-NEXT:   renamable $w1 = KILL $w1, implicit-def $x1
-  ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT: bb.1.atomicrmw.start:
   ; CHECK-NEXT:   successors: %bb.1(0x7c000000), %bb.2(0x04000000)
-  ; CHECK-NEXT:   liveins: $x0, $x1
+  ; CHECK-NEXT:   liveins: $w1, $x0
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT:   renamable $w8 = LDXRB renamable $x0, implicit-def $x8, pcsections !0 :: (volatile load (s8) from %ir.ptr)
   ; CHECK-NEXT:   early-clobber renamable $w9 = STXRB renamable $w1, renamable $x0, pcsections !0 :: (volatile store (s8) into %ir.ptr)
@@ -776,8 +774,8 @@ define i8 @atomicrmw_sub_i8(ptr %ptr, i8 %rhs) {
   ; CHECK-NEXT:   liveins: $w1, $x0
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT:   renamable $w8 = LDAXRB renamable $x0, implicit-def $x8, pcsections !0 :: (volatile load (s8) from %ir.ptr)
-  ; CHECK-NEXT:   $w9 = SUBWrs renamable $w8, renamable $w1, 0, implicit-def $x9, pcsections !0
-  ; CHECK-NEXT:   early-clobber renamable $w10 = STXRB renamable $w9, renamable $x0, implicit killed $x9, pcsections !0 :: (volatile store (s8) into %ir.ptr)
+  ; CHECK-NEXT:   $w9 = SUBWrs renamable $w8, renamable $w1, 0, pcsections !0
+  ; CHECK-NEXT:   early-clobber renamable $w10 = STXRB killed renamable $w9, renamable $x0, pcsections !0 :: (volatile store (s8) into %ir.ptr)
   ; CHECK-NEXT:   CBNZW killed renamable $w10, %bb.1, pcsections !0
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT: bb.2.atomicrmw.end:
@@ -800,8 +798,8 @@ define i8 @atomicrmw_and_i8(ptr %ptr, i8 %rhs) {
   ; CHECK-NEXT:   liveins: $w1, $x0
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT:   renamable $w8 = LDXRB renamable $x0, implicit-def $x8, pcsections !0 :: (volatile load (s8) from %ir.ptr)
-  ; CHECK-NEXT:   $w9 = ANDWrs renamable $w8, renamable $w1, 0, implicit-def $x9, pcsections !0
-  ; CHECK-NEXT:   early-clobber renamable $w10 = STLXRB renamable $w9, renamable $x0, implicit killed $x9, pcsections !0 :: (volatile store (s8) into %ir.ptr)
+  ; CHECK-NEXT:   $w9 = ANDWrs renamable $w8, renamable $w1, 0, pcsections !0
+  ; CHECK-NEXT:   early-clobber renamable $w10 = STLXRB killed renamable $w9, renamable $x0, pcsections !0 :: (volatile store (s8) into %ir.ptr)
   ; CHECK-NEXT:   CBNZW killed renamable $w10, %bb.1, pcsections !0
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT: bb.2.atomicrmw.end:
@@ -824,8 +822,8 @@ define i8 @atomicrmw_or_i8(ptr %ptr, i8 %rhs) {
   ; CHECK-NEXT:   liveins: $w1, $x0
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT:   renamable $w8 = LDAXRB renamable $x0, implicit-def $x8, pcsections !0 :: (volatile load (s8) from %ir.ptr)
-  ; CHECK-NEXT:   $w9 = ORRWrs renamable $w8, renamable $w1, 0, implicit-def $x9, pcsections !0
-  ; CHECK-NEXT:   early-clobber renamable $w10 = STLXRB renamable $w9, renamable $x0, implicit killed $x9, pcsections !0 :: (volatile store (s8) into %ir.ptr)
+  ; CHECK-NEXT:   $w9 = ORRWrs renamable $w8, renamable $w1, 0, pcsections !0
+  ; CHECK-NEXT:   early-clobber renamable $w10 = STLXRB killed renamable $w9, renamable $x0, pcsections !0 :: (volatile store (s8) into %ir.ptr)
   ; CHECK-NEXT:   CBNZW killed renamable $w10, %bb.1, pcsections !0
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT: bb.2.atomicrmw.end:
@@ -848,8 +846,8 @@ define i8 @atomicrmw_xor_i8(ptr %ptr, i8 %rhs) {
   ; CHECK-NEXT:   liveins: $w1, $x0
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT:   renamable $w8 = LDXRB renamable $x0, implicit-def $x8, pcsections !0 :: (volatile load (s8) from %ir.ptr)
-  ; CHECK-NEXT:   $w9 = EORWrs renamable $w8, renamable $w1, 0, implicit-def $x9, pcsections !0
-  ; CHECK-NEXT:   early-clobber renamable $w10 = STXRB renamable $w9, renamable $x0, implicit killed $x9, pcsections !0 :: (volatile store (s8) into %ir.ptr)
+  ; CHECK-NEXT:   $w9 = EORWrs renamable $w8, renamable $w1, 0, pcsections !0
+  ; CHECK-NEXT:   early-clobber renamable $w10 = STXRB killed renamable $w9, renamable $x0, pcsections !0 :: (volatile store (s8) into %ir.ptr)
   ; CHECK-NEXT:   CBNZW killed renamable $w10, %bb.1, pcsections !0
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT: bb.2.atomicrmw.end:
@@ -874,8 +872,8 @@ define i8 @atomicrmw_min_i8(ptr %ptr, i8 %rhs) {
   ; CHECK-NEXT:   renamable $w8 = LDAXRB renamable $x0, implicit-def $x8, pcsections !0 :: (volatile load (s8) from %ir.ptr)
   ; CHECK-NEXT:   renamable $w9 = SBFMWri renamable $w8, 0, 7, pcsections !0
   ; CHECK-NEXT:   dead $wzr = SUBSWrx killed renamable $w9, renamable $w1, 32, implicit-def $nzcv, pcsections !0
-  ; CHECK-NEXT:   renamable $w9 = CSELWr renamable $w8, renamable $w1, 11, implicit killed $nzcv, implicit-def $x9, pcsections !0
-  ; CHECK-NEXT:   early-clobber renamable $w10 = STXRB renamable $w9, renamable $x0, implicit killed $x9, pcsections !0 :: (volatile store (s8) into %ir.ptr)
+  ; CHECK-NEXT:   renamable $w9 = CSELWr renamable $w8, renamable $w1, 11, implicit killed $nzcv, pcsections !0
+  ; CHECK-NEXT:   early-clobber renamable $w10 = STXRB killed renamable $w9, renamable $x0, pcsections !0 :: (volatile store (s8) into %ir.ptr)
   ; CHECK-NEXT:   CBNZW killed renamable $w10, %bb.1, pcsections !0
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT: bb.2.atomicrmw.end:
@@ -900,8 +898,8 @@ define i8 @atomicrmw_max_i8(ptr %ptr, i8 %rhs) {
   ; CHECK-NEXT:   renamable $w8 = LDXRB renamable $x0, implicit-def $x8, pcsections !0 :: (volatile load (s8) from %ir.ptr)
   ; CHECK-NEXT:   renamable $w9 = SBFMWri renamable $w8, 0, 7, pcsections !0
   ; CHECK-NEXT:   dead $wzr = SUBSWrx killed renamable $w9, renamable $w1, 32, implicit-def $nzcv, pcsections !0
-  ; CHECK-NEXT:   renamable $w9 = CSELWr renamable $w8, renamable $w1, 12, implicit killed $nzcv, implicit-def $x9, pcsections !0
-  ; CHECK-NEXT:   early-clobber renamable $w10 = STLXRB renamable $w9, renamable $x0, implicit killed $x9, pcsections !0 :: (volatile store (s8) into %ir.ptr)
+  ; CHECK-NEXT:   renamable $w9 = CSELWr renamable $w8, renamable $w1, 12, implicit killed $nzcv, pcsections !0
+  ; CHECK-NEXT:   early-clobber renamable $w10 = STLXRB killed renamable $w9, renamable $x0, pcsections !0 :: (volatile store (s8) into %ir.ptr)
   ; CHECK-NEXT:   CBNZW killed renamable $w10, %bb.1, pcsections !0
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT: bb.2.atomicrmw.end:
@@ -980,8 +978,8 @@ define i16 @atomicrmw_add_i16(ptr %ptr, i16 %rhs) {
   ; CHECK-NEXT:   liveins: $w1, $x0
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT:   renamable $w8 = LDAXRH renamable $x0, implicit-def $x8, pcsections !0 :: (volatile load (s16) from %ir.ptr)
-  ; CHECK-NEXT:   $w9 = ADDWrs renamable $w8, renamable $w1, 0, implicit-def $x9, pcsections !0
-  ; CHECK-NEXT:   early-clobber renamable $w10 = STLXRH renamable $w9, renamable $x0, implicit killed $x9, pcsections !0 :: (volatile store (s16) into %ir.ptr)
+  ; CHECK-NEXT:   $w9 = ADDWrs renamable $w8, renamable $w1, 0, pcsections !0
+  ; CHECK-NEXT:   early-clobber renamable $w10 = STLXRH killed renamable $w9, renamable $x0, pcsections !0 :: (volatile store (s16) into %ir.ptr)
   ; CHECK-NEXT:   CBNZW killed renamable $w10, %bb.1, pcsections !0
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT: bb.2.atomicrmw.end:
@@ -999,11 +997,9 @@ define i16 @atomicrmw_xchg_i16(ptr %ptr, i16 %rhs) {
   ; CHECK-NEXT:   successors: %bb.1(0x80000000)
   ; CHECK-NEXT:   liveins: $w1, $x0
   ; CHECK-NEXT: {{  $}}
-  ; CHECK-NEXT:   renamable $w1 = KILL $w1, implicit-def $x1
-  ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT: bb.1.atomicrmw.start:
   ; CHECK-NEXT:   successors: %bb.1(0x7c000000), %bb.2(0x04000000)
-  ; CHECK-NEXT:   liveins: $x0, $x1
+  ; CHECK-NEXT:   liveins: $w1, $x0
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT:   renamable $w8 = LDXRH renamable $x0, implicit-def $x8, pcsections !0 :: (volatile load (s16) from %ir.ptr)
   ; CHECK-NEXT:   early-clobber renamable $w9 = STXRH renamable $w1, renamable $x0, pcsections !0 :: (volatile store (s16) into %ir.ptr)
@@ -1029,8 +1025,8 @@ define i16 @atomicrmw_sub_i16(ptr %ptr, i16 %rhs) {
   ; CHECK-NEXT:   liveins: $w1, $x0
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT:   renamable $w8 = LDAXRH renamable $x0, implicit-def $x8, pcsections !0 :: (volatile load (s16) from %ir.ptr)
-  ; CHECK-NEXT:   $w9 = SUBWrs renamable $w8, renamable $w1, 0, implicit-def $x9, pcsections !0
-  ; CHECK-NEXT:   early-clobber renamable $w10 = STXRH renamable $w9, renamable $x0, implicit killed $x9, pcsections !0 :: (volatile store (s16) into %ir.ptr)
+  ; CHECK-NEXT:   $w9 = SUBWrs renamable $w8, renamable $w1, 0, pcsections !0
+  ; CHECK-NEXT:   early-clobber renamable $w10 = STXRH killed renamable $w9, renamable $x0, pcsections !0 :: (volatile store (s16) into %ir.ptr)
   ; CHECK-NEXT:   CBNZW killed renamable $w10, %bb.1, pcsections !0
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT: bb.2.atomicrmw.end:
@@ -1053,8 +1049,8 @@ define i16 @atomicrmw_and_i16(ptr %ptr, i16 %rhs) {
   ; CHECK-NEXT:   liveins: $w1, $x0
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT:   renamable $w8 = LDXRH renamable $x0, implicit-def $x8, pcsections !0 :: (volatile load (s16) from %ir.ptr)
-  ; CHECK-NEXT:   $w9 = ANDWrs renamable $w8, renamable $w1, 0, implicit-def $x9, pcsections !0
-  ; CHECK-NEXT:   early-clobber renamable $w10 = STLXRH renamable $w9, renamable $x0, implicit killed $x9, pcsections !0 :: (volatile store (s16) into %ir.ptr)
+  ; CHECK-NEXT:   $w9 = ANDWrs renamable $w8, renamable $w1, 0, pcsections !0
+  ; CHECK-NEXT:   early-clobber renamable $w10 = STLXRH killed renamable $w9, renamable $x0, pcsections !0 :: (volatile store (s16) into %ir.ptr)
   ; CHECK-NEXT:   CBNZW killed renamable $w10, %bb.1, pcsections !0
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT: bb.2.atomicrmw.end:
@@ -1077,8 +1073,8 @@ define i16 @atomicrmw_or_i16(ptr %ptr, i16 %rhs) {
   ; CHECK-NEXT:   liveins: $w1, $x0
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT:   renamable $w8 = LDAXRH renamable $x0, implicit-def $x8, pcsections !0 :: (volatile load (s16) from %ir.ptr)
-  ; CHECK-NEXT:   $w9 = ORRWrs renamable $w8, renamable $w1, 0, implicit-def $x9, pcsections !0
-  ; CHECK-NEXT:   early-clobber renamable $w10 = STLXRH renamable $w9, renamable $x0, implicit killed $x9, pcsections !0 :: (volatile store (s16) into %ir.ptr)
+  ; CHECK-NEXT:   $w9 = ORRWrs renamable $w8, renamable $w1, 0, pcsections !0
+  ; CHECK-NEXT:   early-clobber renamable $w10 = STLXRH killed renamable $w9, renamable $x0, pcsections !0 :: (volatile store (s16) into %ir.ptr)
   ; CHECK-NEXT:   CBNZW killed renamable $w10, %bb.1, pcsections !0
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT: bb.2.atomicrmw.end:
@@ -1101,8 +1097,8 @@ define i16 @atomicrmw_xor_i16(ptr %ptr, i16 %rhs) {
   ; CHECK-NEXT:   liveins: $w1, $x0
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT:   renamable $w8 = LDXRH renamable $x0, implicit-def $x8, pcsections !0 :: (volatile load (s16) from %ir.ptr)
-  ; CHECK-NEXT:   $w9 = EORWrs renamable $w8, renamable $w1, 0, implicit-def $x9, pcsections !0
-  ; CHECK-NEXT:   early-clobber renamable $w10 = STXRH renamable $w9, renamable $x0, implicit killed $x9, pcsections !0 :: (volatile store (s16) into %ir.ptr)
+  ; CHECK-NEXT:   $w9 = EORWrs renamable $w8, renamable $w1, 0, pcsections !0
+  ; CHECK-NEXT:   early-clobber renamable $w10 = STXRH killed renamable $w9, renamable $x0, pcsections !0 :: (volatile store (s16) into %ir.ptr)
   ; CHECK-NEXT:   CBNZW killed renamable $w10, %bb.1, pcsections !0
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT: bb.2.atomicrmw.end:
@@ -1127,8 +1123,8 @@ define i16 @atomicrmw_min_i16(ptr %ptr, i16 %rhs) {
   ; CHECK-NEXT:   renamable $w8 = LDAXRH renamable $x0, implicit-def $x8, pcsections !0 :: (volatile load (s16) from %ir.ptr)
   ; CHECK-NEXT:   renamable $w9 = SBFMWri renamable $w8, 0, 15, pcsections !0
   ; CHECK-NEXT:   dead $wzr = SUBSWrx killed renamable $w9, renamable $w1, 40, implicit-def $nzcv, pcsections !0
-  ; CHECK-NEXT:   renamable $w9 = CSELWr renamable $w8, renamable $w1, 11, implicit killed $nzcv, implicit-def $x9, pcsections !0
-  ; CHECK-NEXT:   early-clobber renamable $w10 = STXRH renamable $w9, renamable $x0, implicit killed $x9, pcsections !0 :: (volatile store (s16) into %ir.ptr)
+  ; CHECK-NEXT:   renamable $w9 = CSELWr renamable $w8, renamable $w1, 11, implicit killed $nzcv, pcsections !0
+  ; CHECK-NEXT:   early-clobber renamable $w10 = STXRH killed renamable $w9, renamable $x0, pcsections !0 :: (volatile store (s16) into %ir.ptr)
   ; CHECK-NEXT:   CBNZW killed renamable $w10, %bb.1, pcsections !0
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT: bb.2.atomicrmw.end:
@@ -1153,8 +1149,8 @@ define i16 @atomicrmw_max_i16(ptr %ptr, i16 %rhs) {
   ; CHECK-NEXT:   renamable $w8 = LDXRH renamable $x0, implicit-def $x8, pcsections !0 :: (volatile load (s16) from %ir.ptr)
   ; CHECK-NEXT:   renamable $w9 = SBFMWri renamable $w8, 0, 15, pcsections !0
   ; CHECK-NEXT:   dead $wzr = SUBSWrx killed renamable $w9, renamable $w1, 40, implicit-def $nzcv, pcsections !0
-  ; CHECK-NEXT:   renamable $w9 = CSELWr renamable $w8, renamable $w1, 12, implicit killed $nzcv, implicit-def $x9, pcsections !0
-  ; CHECK-NEXT:   early-clobber renamable $w10 = STLXRH renamable $w9, renamable $x0, implicit killed $x9, pcsections !0 :: (volatile store (s16) into %ir.ptr)
+  ; CHECK-NEXT:   renamable $w9 = CSELWr renamable $w8, renamable $w1, 12, implicit killed $nzcv, pcsections !0
+  ; CHECK-NEXT:   early-clobber renamable $w10 = STLXRH killed renamable $w9, renamable $x0, pcsections !0 :: (volatile store (s16) into %ir.ptr)
   ; CHECK-NEXT:   CBNZW killed renamable $w10, %bb.1, pcsections !0
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT: bb.2.atomicrmw.end:
@@ -1229,11 +1225,10 @@ define { i8, i1 } @cmpxchg_i8(ptr %ptr, i8 %desired, i8 %new) {
   ; CHECK-NEXT:   liveins: $w1, $w2, $x0
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT:   $x8 = ORRXrs $xzr, $x0, 0
-  ; CHECK-NEXT:   renamable $w2 = KILL $w2, implicit-def $x2
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT: bb.1.cmpxchg.start:
   ; CHECK-NEXT:   successors: %bb.2(0x7c000000), %bb.4(0x04000000)
-  ; CHECK-NEXT:   liveins: $w1, $x2, $x8
+  ; CHECK-NEXT:   liveins: $w1, $w2, $x8
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT:   renamable $w0 = LDXRB renamable $x8, implicit-def $x0, pcsections !0 :: (volatile load (s8) from %ir.ptr)
   ; CHECK-NEXT:   renamable $w9 = ANDWri renamable $w0, 7, pcsections !0
@@ -1242,7 +1237,7 @@ define { i8, i1 } @cmpxchg_i8(ptr %ptr, i8 %desired, i8 %new) {
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT: bb.2.cmpxchg.trystore:
   ; CHECK-NEXT:   successors: %bb.3(0x04000000), %bb.1(0x7c000000)
-  ; CHECK-NEXT:   liveins: $w1, $x0, $x2, $x8
+  ; CHECK-NEXT:   liveins: $w1, $w2, $x0, $x8
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT:   early-clobber renamable $w9 = STXRB renamable $w2, renamable $x8, pcsections !0 :: (volatile store (s8) into %ir.ptr)
   ; CHECK-NEXT:   CBNZW killed renamable $w9, %bb.1
@@ -1272,11 +1267,10 @@ define { i16, i1 } @cmpxchg_i16(ptr %ptr, i16 %desired, i16 %new) {
   ; CHECK-NEXT:   liveins: $w1, $w2, $x0
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT:   $x8 = ORRXrs $xzr, $x0, 0
-  ; CHECK-NEXT:   renamable $w2 = KILL $w2, implicit-def $x2
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT: bb.1.cmpxchg.start:
   ; CHECK-NEXT:   successors: %bb.2(0x7c000000), %bb.4(0x04000000)
-  ; CHECK-NEXT:   liveins: $w1, $x2, $x8
+  ; CHECK-NEXT:   liveins: $w1, $w2, $x8
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT:   renamable $w0 = LDXRH renamable $x8, implicit-def $x0, pcsections !0 :: (volatile load (s16) from %ir.ptr)
   ; CHECK-NEXT:   renamable $w9 = ANDWri renamable $w0, 15, pcsections !0
@@ -1285,7 +1279,7 @@ define { i16, i1 } @cmpxchg_i16(ptr %ptr, i16 %desired, i16 %new) {
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT: bb.2.cmpxchg.trystore:
   ; CHECK-NEXT:   successors: %bb.3(0x04000000), %bb.1(0x7c000000)
-  ; CHECK-NEXT:   liveins: $w1, $x0, $x2, $x8
+  ; CHECK-NEXT:   liveins: $w1, $w2, $x0, $x8
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT:   early-clobber renamable $w9 = STXRH renamable $w2, renamable $x8, pcsections !0 :: (volatile store (s16) into %ir.ptr)
   ; CHECK-NEXT:   CBNZW killed renamable $w9, %bb.1
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/combine-addv.mir b/llvm/test/CodeGen/AArch64/GlobalISel/combine-addv.mir
new file mode 100644
index 0000000000000..ae08cd9d5bfef
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/combine-addv.mir
@@ -0,0 +1,33 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -run-pass=aarch64-prelegalizer-combiner -mtriple aarch64-unknown-unknown -mattr=+dotprod %s -o - | FileCheck %s
+
+---
+name:            vecreduce_intrinsic
+body:             |
+  bb.0:
+    liveins: $q0, $q1, $q2, $q3, $q4
+    ; CHECK-LABEL: name: vecreduce_intrinsic
+    ; CHECK: liveins: $q0, $q1, $q2, $q3, $q4
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(<4 x s32>) = COPY $q0
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(<4 x s32>) = COPY $q1
+    ; CHECK-NEXT: [[COPY2:%[0-9]+]]:_(<4 x s32>) = COPY $q2
+    ; CHECK-NEXT: [[COPY3:%[0-9]+]]:_(<4 x s32>) = COPY $q3
+    ; CHECK-NEXT: [[COPY4:%[0-9]+]]:_(<4 x s32>) = COPY $q4
+    ; CHECK-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<16 x s32>) = G_CONCAT_VECTORS [[COPY1]](<4 x s32>), [[COPY2]](<4 x s32>), [[COPY3]](<4 x s32>), [[COPY4]](<4 x s32>)
+    ; CHECK-NEXT: [[INT:%[0-9]+]]:_(<4 x s32>) = G_INTRINSIC intrinsic(@llvm.experimental.vector.partial.reduce.add), [[COPY]](<4 x s32>), [[CONCAT_VECTORS]](<16 x s32>)
+    ; CHECK-NEXT: [[VECREDUCE_ADD:%[0-9]+]]:_(s32) = G_VECREDUCE_ADD [[INT]](<4 x s32>)
+    ; CHECK-NEXT: $w0 = COPY [[VECREDUCE_ADD]](s32)
+    ; CHECK-NEXT: RET_ReallyLR implicit $w0
+    %0:_(<4 x s32>) = COPY $q0
+    %2:_(<4 x s32>) = COPY $q1
+    %3:_(<4 x s32>) = COPY $q2
+    %4:_(<4 x s32>) = COPY $q3
+    %5:_(<4 x s32>) = COPY $q4
+    %1:_(<16 x s32>) = G_CONCAT_VECTORS %2:_(<4 x s32>), %3:_(<4 x s32>), %4:_(<4 x s32>), %5:_(<4 x s32>)
+    %6:_(<4 x s32>) = G_INTRINSIC intrinsic(@llvm.experimental.vector.partial.reduce.add), %0:_(<4 x s32>), %1:_(<16 x s32>)
+    %7:_(s32) = G_VECREDUCE_ADD %6:_(<4 x s32>)
+    $w0 = COPY %7:_(s32)
+    RET_ReallyLR implicit $w0
+
+...
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/legalize-fshl.mir b/llvm/test/CodeGen/AArch64/GlobalISel/legalize-fshl.mir
index c3c23aecc161f..1e549fa9a833a 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/legalize-fshl.mir
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/legalize-fshl.mir
@@ -28,7 +28,7 @@ body:             |
     ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[AND2]], [[C3]](s64)
     ; CHECK-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[LSHR]], [[C2]]
     ; CHECK-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[AND3]], [[AND1]](s32)
-    ; CHECK-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL]], [[LSHR1]]
+    ; CHECK-NEXT: [[OR:%[0-9]+]]:_(s32) = disjoint G_OR [[SHL]], [[LSHR1]]
     ; CHECK-NEXT: $w0 = COPY [[OR]](s32)
     ; CHECK-NEXT: RET_ReallyLR implicit $w0
     %3:_(s32) = COPY $w0
@@ -71,7 +71,7 @@ body:             |
     ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[AND2]], [[C3]](s64)
     ; CHECK-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[LSHR]], [[C2]]
     ; CHECK-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[AND3]], [[AND1]](s32)
-    ; CHECK-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL]], [[LSHR1]]
+    ; CHECK-NEXT: [[OR:%[0-9]+]]:_(s32) = disjoint G_OR [[SHL]], [[LSHR1]]
     ; CHECK-NEXT: $w0 = COPY [[OR]](s32)
     ; CHECK-NEXT: RET_ReallyLR implicit $w0
     %3:_(s32) = COPY $w0
@@ -110,7 +110,7 @@ body:             |
     ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
     ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[COPY1]], [[C2]](s64)
     ; CHECK-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[LSHR]], [[AND1]](s32)
-    ; CHECK-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL]], [[LSHR1]]
+    ; CHECK-NEXT: [[OR:%[0-9]+]]:_(s32) = disjoint G_OR [[SHL]], [[LSHR1]]
     ; CHECK-NEXT: $w0 = COPY [[OR]](s32)
     ; CHECK-NEXT: RET_ReallyLR implicit $w0
     %0:_(s32) = COPY $w0
@@ -145,7 +145,7 @@ body:             |
     ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s64) = G_SHL [[COPY]], [[AND]](s64)
     ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s64) = G_LSHR [[COPY1]], [[C2]](s64)
     ; CHECK-NEXT: [[LSHR1:%[0-9]+]]:_(s64) = G_LSHR [[LSHR]], [[AND1]](s64)
-    ; CHECK-NEXT: [[OR:%[0-9]+]]:_(s64) = G_OR [[SHL]], [[LSHR1]]
+    ; CHECK-NEXT: [[OR:%[0-9]+]]:_(s64) = disjoint G_OR [[SHL]], [[LSHR1]]
     ; CHECK-NEXT: $x0 = COPY [[OR]](s64)
     ; CHECK-NEXT: RET_ReallyLR implicit $x0
     %0:_(s64) = COPY $x0
@@ -175,7 +175,7 @@ body:             |
     ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
     ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C1]]
     ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[AND]], [[C]](s64)
-    ; CHECK-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL]], [[LSHR]]
+    ; CHECK-NEXT: [[OR:%[0-9]+]]:_(s32) = disjoint G_OR [[SHL]], [[LSHR]]
     ; CHECK-NEXT: $w0 = COPY [[OR]](s32)
     ; CHECK-NEXT: RET_ReallyLR implicit $w0
     %2:_(s32) = COPY $w0
@@ -209,7 +209,7 @@ body:             |
     ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C1]]
     ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 6
     ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[AND]], [[C2]](s64)
-    ; CHECK-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL]], [[LSHR]]
+    ; CHECK-NEXT: [[OR:%[0-9]+]]:_(s32) = disjoint G_OR [[SHL]], [[LSHR]]
     ; CHECK-NEXT: $w0 = COPY [[OR]](s32)
     ; CHECK-NEXT: RET_ReallyLR implicit $w0
     %2:_(s32) = COPY $w0
@@ -246,7 +246,7 @@ body:             |
     ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[LSHR]], [[C1]]
     ; CHECK-NEXT: [[C3:%[0-9]+]]:_(s64) = G_CONSTANT i64 7
     ; CHECK-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[AND1]], [[C3]](s64)
-    ; CHECK-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL]], [[LSHR1]]
+    ; CHECK-NEXT: [[OR:%[0-9]+]]:_(s32) = disjoint G_OR [[SHL]], [[LSHR1]]
     ; CHECK-NEXT: $w0 = COPY [[OR]](s32)
     ; CHECK-NEXT: RET_ReallyLR implicit $w0
     %2:_(s32) = COPY $w0
@@ -280,7 +280,7 @@ body:             |
     ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C1]]
     ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 4
     ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[AND]], [[C2]](s64)
-    ; CHECK-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL]], [[LSHR]]
+    ; CHECK-NEXT: [[OR:%[0-9]+]]:_(s32) = disjoint G_OR [[SHL]], [[LSHR]]
     ; CHECK-NEXT: $w0 = COPY [[OR]](s32)
     ; CHECK-NEXT: RET_ReallyLR implicit $w0
     %2:_(s32) = COPY $w0
@@ -314,7 +314,7 @@ body:             |
     ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C1]]
     ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 12
     ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[AND]], [[C2]](s64)
-    ; CHECK-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL]], [[LSHR]]
+    ; CHECK-NEXT: [[OR:%[0-9]+]]:_(s32) = disjoint G_OR [[SHL]], [[LSHR]]
     ; CHECK-NEXT: $w0 = COPY [[OR]](s32)
     ; CHECK-NEXT: RET_ReallyLR implicit $w0
     %2:_(s32) = COPY $w0
@@ -351,7 +351,7 @@ body:             |
     ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[LSHR]], [[C1]]
     ; CHECK-NEXT: [[C3:%[0-9]+]]:_(s64) = G_CONSTANT i64 15
     ; CHECK-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[AND1]], [[C3]](s64)
-    ; CHECK-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL]], [[LSHR1]]
+    ; CHECK-NEXT: [[OR:%[0-9]+]]:_(s32) = disjoint G_OR [[SHL]], [[LSHR1]]
     ; CHECK-NEXT: $w0 = COPY [[OR]](s32)
     ; CHECK-NEXT: RET_ReallyLR implicit $w0
     %2:_(s32) = COPY $w0
@@ -437,7 +437,7 @@ body:             |
     ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[COPY1]], [[C1]](s64)
     ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 31
     ; CHECK-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[LSHR]], [[C2]](s64)
-    ; CHECK-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL]], [[LSHR1]]
+    ; CHECK-NEXT: [[OR:%[0-9]+]]:_(s32) = disjoint G_OR [[SHL]], [[LSHR1]]
     ; CHECK-NEXT: $w0 = COPY [[OR]](s32)
     ; CHECK-NEXT: RET_ReallyLR implicit $w0
     %0:_(s32) = COPY $w0
@@ -520,7 +520,7 @@ body:             |
     ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s64) = G_SHL [[COPY]], [[C1]](s64)
     ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s64) = G_LSHR [[COPY1]], [[C2]](s64)
     ; CHECK-NEXT: [[LSHR1:%[0-9]+]]:_(s64) = G_LSHR [[LSHR]], [[C]](s64)
-    ; CHECK-NEXT: [[OR:%[0-9]+]]:_(s64) = G_OR [[SHL]], [[LSHR1]]
+    ; CHECK-NEXT: [[OR:%[0-9]+]]:_(s64) = disjoint G_OR [[SHL]], [[LSHR1]]
     ; CHECK-NEXT: $x0 = COPY [[OR]](s64)
     ; CHECK-NEXT: RET_ReallyLR implicit $x0
     %0:_(s64) = COPY $x0
@@ -556,7 +556,7 @@ body:             |
     ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(<4 x s32>) = G_SHL [[COPY]], [[BUILD_VECTOR]](<4 x s32>)
     ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(<4 x s32>) = G_LSHR [[COPY1]], [[BUILD_VECTOR2]](<4 x s32>)
     ; CHECK-NEXT: [[LSHR1:%[0-9]+]]:_(<4 x s32>) = G_LSHR [[LSHR]], [[BUILD_VECTOR1]](<4 x s32>)
-    ; CHECK-NEXT: [[OR:%[0-9]+]]:_(<4 x s32>) = G_OR [[SHL]], [[LSHR1]]
+    ; CHECK-NEXT: [[OR:%[0-9]+]]:_(<4 x s32>) = disjoint G_OR [[SHL]], [[LSHR1]]
     ; CHECK-NEXT: $q0 = COPY [[OR]](<4 x s32>)
     ; CHECK-NEXT: RET_ReallyLR implicit $q0
     %0:_(<4 x s32>) = COPY $q0
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/legalize-fshr.mir b/llvm/test/CodeGen/AArch64/GlobalISel/legalize-fshr.mir
index 09520445b71d9..9a9e5bb42c64f 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/legalize-fshr.mir
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/legalize-fshr.mir
@@ -27,7 +27,7 @@ body:             |
     ; CHECK-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
     ; CHECK-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C3]]
     ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[AND2]], [[AND]](s32)
-    ; CHECK-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL1]], [[LSHR]]
+    ; CHECK-NEXT: [[OR:%[0-9]+]]:_(s32) = disjoint G_OR [[SHL1]], [[LSHR]]
     ; CHECK-NEXT: $w0 = COPY [[OR]](s32)
     ; CHECK-NEXT: RET_ReallyLR implicit $w0
     %3:_(s32) = COPY $w0
@@ -69,7 +69,7 @@ body:             |
     ; CHECK-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
     ; CHECK-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C3]]
     ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[AND2]], [[AND]](s32)
-    ; CHECK-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL1]], [[LSHR]]
+    ; CHECK-NEXT: [[OR:%[0-9]+]]:_(s32) = disjoint G_OR [[SHL1]], [[LSHR]]
     ; CHECK-NEXT: $w0 = COPY [[OR]](s32)
     ; CHECK-NEXT: RET_ReallyLR implicit $w0
     %3:_(s32) = COPY $w0
@@ -108,7 +108,7 @@ body:             |
     ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY]], [[C2]](s64)
     ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[SHL]], [[AND1]](s32)
     ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[COPY1]], [[AND]](s32)
-    ; CHECK-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL1]], [[LSHR]]
+    ; CHECK-NEXT: [[OR:%[0-9]+]]:_(s32) = disjoint G_OR [[SHL1]], [[LSHR]]
     ; CHECK-NEXT: $w0 = COPY [[OR]](s32)
     ; CHECK-NEXT: RET_ReallyLR implicit $w0
     %0:_(s32) = COPY $w0
@@ -143,7 +143,7 @@ body:             |
     ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s64) = G_SHL [[COPY]], [[C2]](s64)
     ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(s64) = G_SHL [[SHL]], [[AND1]](s64)
     ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s64) = G_LSHR [[COPY1]], [[AND]](s64)
-    ; CHECK-NEXT: [[OR:%[0-9]+]]:_(s64) = G_OR [[SHL1]], [[LSHR]]
+    ; CHECK-NEXT: [[OR:%[0-9]+]]:_(s64) = disjoint G_OR [[SHL1]], [[LSHR]]
     ; CHECK-NEXT: $x0 = COPY [[OR]](s64)
     ; CHECK-NEXT: RET_ReallyLR implicit $x0
     %0:_(s64) = COPY $x0
@@ -175,7 +175,7 @@ body:             |
     ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C1]]
     ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 7
     ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[AND]], [[C2]](s64)
-    ; CHECK-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL]], [[LSHR]]
+    ; CHECK-NEXT: [[OR:%[0-9]+]]:_(s32) = disjoint G_OR [[SHL]], [[LSHR]]
     ; CHECK-NEXT: $w0 = COPY [[OR]](s32)
     ; CHECK-NEXT: RET_ReallyLR implicit $w0
     %2:_(s32) = COPY $w0
@@ -209,7 +209,7 @@ body:             |
     ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C1]]
     ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 2
     ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[AND]], [[C2]](s64)
-    ; CHECK-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL]], [[LSHR]]
+    ; CHECK-NEXT: [[OR:%[0-9]+]]:_(s32) = disjoint G_OR [[SHL]], [[LSHR]]
     ; CHECK-NEXT: $w0 = COPY [[OR]](s32)
     ; CHECK-NEXT: RET_ReallyLR implicit $w0
     %2:_(s32) = COPY $w0
@@ -245,7 +245,7 @@ body:             |
     ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C2]]
     ; CHECK-NEXT: [[C3:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
     ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[AND]], [[C3]](s64)
-    ; CHECK-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL1]], [[LSHR]]
+    ; CHECK-NEXT: [[OR:%[0-9]+]]:_(s32) = disjoint G_OR [[SHL1]], [[LSHR]]
     ; CHECK-NEXT: $w0 = COPY [[OR]](s32)
     ; CHECK-NEXT: RET_ReallyLR implicit $w0
     %2:_(s32) = COPY $w0
@@ -279,7 +279,7 @@ body:             |
     ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C1]]
     ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 5
     ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[AND]], [[C2]](s64)
-    ; CHECK-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL]], [[LSHR]]
+    ; CHECK-NEXT: [[OR:%[0-9]+]]:_(s32) = disjoint G_OR [[SHL]], [[LSHR]]
     ; CHECK-NEXT: $w0 = COPY [[OR]](s32)
     ; CHECK-NEXT: RET_ReallyLR implicit $w0
     %2:_(s32) = COPY $w0
@@ -313,7 +313,7 @@ body:             |
     ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C1]]
     ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 4
     ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[AND]], [[C2]](s64)
-    ; CHECK-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL]], [[LSHR]]
+    ; CHECK-NEXT: [[OR:%[0-9]+]]:_(s32) = disjoint G_OR [[SHL]], [[LSHR]]
     ; CHECK-NEXT: $w0 = COPY [[OR]](s32)
     ; CHECK-NEXT: RET_ReallyLR implicit $w0
     %2:_(s32) = COPY $w0
@@ -349,7 +349,7 @@ body:             |
     ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C2]]
     ; CHECK-NEXT: [[C3:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
     ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[AND]], [[C3]](s64)
-    ; CHECK-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL1]], [[LSHR]]
+    ; CHECK-NEXT: [[OR:%[0-9]+]]:_(s32) = disjoint G_OR [[SHL1]], [[LSHR]]
     ; CHECK-NEXT: $w0 = COPY [[OR]](s32)
     ; CHECK-NEXT: RET_ReallyLR implicit $w0
     %2:_(s32) = COPY $w0
@@ -507,7 +507,7 @@ body:             |
     ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s64) = G_SHL [[COPY]], [[C2]](s64)
     ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(s64) = G_SHL [[SHL]], [[C]](s64)
     ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s64) = G_LSHR [[COPY1]], [[C1]](s64)
-    ; CHECK-NEXT: [[OR:%[0-9]+]]:_(s64) = G_OR [[SHL1]], [[LSHR]]
+    ; CHECK-NEXT: [[OR:%[0-9]+]]:_(s64) = disjoint G_OR [[SHL1]], [[LSHR]]
     ; CHECK-NEXT: $x0 = COPY [[OR]](s64)
     ; CHECK-NEXT: RET_ReallyLR implicit $x0
     %0:_(s64) = COPY $x0
@@ -541,7 +541,7 @@ body:             |
     ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(<4 x s32>) = G_SHL [[COPY]], [[BUILD_VECTOR2]](<4 x s32>)
     ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(<4 x s32>) = G_SHL [[SHL]], [[BUILD_VECTOR1]](<4 x s32>)
     ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(<4 x s32>) = G_LSHR [[COPY1]], [[BUILD_VECTOR]](<4 x s32>)
-    ; CHECK-NEXT: [[OR:%[0-9]+]]:_(<4 x s32>) = G_OR [[SHL1]], [[LSHR]]
+    ; CHECK-NEXT: [[OR:%[0-9]+]]:_(<4 x s32>) = disjoint G_OR [[SHL1]], [[LSHR]]
     ; CHECK-NEXT: $q0 = COPY [[OR]](<4 x s32>)
     ; CHECK-NEXT: RET_ReallyLR implicit $q0
     %0:_(<4 x s32>) = COPY $q0
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/legalize-reduce-add.mir b/llvm/test/CodeGen/AArch64/GlobalISel/legalize-reduce-add.mir
index 76fdfd0c301f6..a18bd0f2f90eb 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/legalize-reduce-add.mir
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/legalize-reduce-add.mir
@@ -157,12 +157,17 @@ body:             |
     ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(<2 x s64>) = COPY $q0
     ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(<2 x s64>) = COPY $q1
     ; CHECK-NEXT: [[COPY2:%[0-9]+]]:_(<2 x s64>) = COPY $q2
-    ; CHECK-NEXT: [[VECREDUCE_ADD:%[0-9]+]]:_(s64) = G_VECREDUCE_ADD [[COPY]](<2 x s64>)
-    ; CHECK-NEXT: [[VECREDUCE_ADD1:%[0-9]+]]:_(s64) = G_VECREDUCE_ADD [[COPY1]](<2 x s64>)
-    ; CHECK-NEXT: [[VECREDUCE_ADD2:%[0-9]+]]:_(s64) = G_VECREDUCE_ADD [[COPY2]](<2 x s64>)
-    ; CHECK-NEXT: [[ADD:%[0-9]+]]:_(s64) = G_ADD [[VECREDUCE_ADD]], [[VECREDUCE_ADD1]]
-    ; CHECK-NEXT: [[ADD1:%[0-9]+]]:_(s64) = G_ADD [[ADD]], [[VECREDUCE_ADD2]]
-    ; CHECK-NEXT: $x0 = COPY [[ADD1]](s64)
+    ; CHECK-NEXT: [[DEF:%[0-9]+]]:_(s64) = G_IMPLICIT_DEF
+    ; CHECK-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s64>) = G_BUILD_VECTOR [[DEF]](s64), [[DEF]](s64)
+    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+    ; CHECK-NEXT: [[IVEC:%[0-9]+]]:_(<2 x s64>) = G_INSERT_VECTOR_ELT [[BUILD_VECTOR]], [[C]](s64), [[C]](s64)
+    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
+    ; CHECK-NEXT: [[IVEC1:%[0-9]+]]:_(<2 x s64>) = G_INSERT_VECTOR_ELT [[IVEC]], [[C]](s64), [[C1]](s64)
+    ; CHECK-NEXT: [[ADD:%[0-9]+]]:_(<2 x s64>) = G_ADD [[COPY]], [[COPY1]]
+    ; CHECK-NEXT: [[ADD1:%[0-9]+]]:_(<2 x s64>) = G_ADD [[COPY2]], [[IVEC1]]
+    ; CHECK-NEXT: [[ADD2:%[0-9]+]]:_(<2 x s64>) = G_ADD [[ADD]], [[ADD1]]
+    ; CHECK-NEXT: [[VECREDUCE_ADD:%[0-9]+]]:_(s64) = G_VECREDUCE_ADD [[ADD2]](<2 x s64>)
+    ; CHECK-NEXT: $x0 = COPY [[VECREDUCE_ADD]](s64)
     ; CHECK-NEXT: RET_ReallyLR implicit $x0
     %0:_(<2 x s64>) = COPY $q0
     %1:_(<2 x s64>) = COPY $q1
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/legalizer-info-validation.mir b/llvm/test/CodeGen/AArch64/GlobalISel/legalizer-info-validation.mir
index c2c77b9326cb6..0260e65520774 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/legalizer-info-validation.mir
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/legalizer-info-validation.mir
@@ -867,8 +867,8 @@
 # DEBUG-NEXT: .. type index coverage check SKIPPED: user-defined predicate detected
 # DEBUG-NEXT: .. imm index coverage check SKIPPED: user-defined predicate detected
 # DEBUG-NEXT: G_VECREDUCE_ADD (opcode {{[0-9]+}}): 2 type indices, 0 imm indices
-# DEBUG-NEXT: .. type index coverage check SKIPPED: user-defined predicate detected
-# DEBUG-NEXT: .. imm index coverage check SKIPPED: user-defined predicate detected
+# DEBUG-NEXT: .. the first uncovered type index: 2, OK
+# DEBUG-NEXT: .. the first uncovered imm index: 0, OK
 # DEBUG-NEXT: G_VECREDUCE_MUL (opcode {{[0-9]+}}): 2 type indices, 0 imm indices
 # DEBUG-NEXT: .. type index coverage check SKIPPED: user-defined predicate detected
 # DEBUG-NEXT: .. imm index coverage check SKIPPED: user-defined predicate detected
diff --git a/llvm/test/CodeGen/AArch64/aarch64-address-type-promotion-assertion.ll b/llvm/test/CodeGen/AArch64/aarch64-address-type-promotion-assertion.ll
index 5041cfbbaa863..9a1a31bec918e 100644
--- a/llvm/test/CodeGen/AArch64/aarch64-address-type-promotion-assertion.ll
+++ b/llvm/test/CodeGen/AArch64/aarch64-address-type-promotion-assertion.ll
@@ -2,7 +2,7 @@
 ; PR20188: don't crash when merging sexts.
 
 ; CHECK: foo:
-define void @foo() unnamed_addr align 2 {
+define void @foo(i1 %arg) unnamed_addr align 2 {
 entry:
   br label %invoke.cont145
 
@@ -12,7 +12,7 @@ invoke.cont145:
 
 if.then274:
   %0 = load i32, ptr null, align 4
-  br i1 undef, label %invoke.cont291, label %if.else313
+  br i1 %arg, label %invoke.cont291, label %if.else313
 
 invoke.cont291:
   %idxprom.i.i.i605 = sext i32 %0 to i64
@@ -26,7 +26,7 @@ if.else313:
   br i1 %cmp314, label %invoke.cont317, label %invoke.cont353
 
 invoke.cont317:
-  br i1 undef, label %invoke.cont326, label %invoke.cont334
+  br i1 %arg, label %invoke.cont326, label %invoke.cont334
 
 invoke.cont326:
   %idxprom.i.i.i587 = sext i32 %0 to i64
@@ -36,7 +36,7 @@ invoke.cont326:
 
 invoke.cont334:
   %lo.1 = phi double [ %sub329, %invoke.cont326 ], [ undef, %invoke.cont317 ]
-  br i1 undef, label %invoke.cont342, label %if.end356
+  br i1 %arg, label %invoke.cont342, label %if.end356
 
 invoke.cont342:
   %idxprom.i.i.i578 = sext i32 %0 to i64
diff --git a/llvm/test/CodeGen/AArch64/aarch64-addv.ll b/llvm/test/CodeGen/AArch64/aarch64-addv.ll
index aba284b4e0d29..104b6d1236d71 100644
--- a/llvm/test/CodeGen/AArch64/aarch64-addv.ll
+++ b/llvm/test/CodeGen/AArch64/aarch64-addv.ll
@@ -1,6 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s -mtriple=aarch64 -aarch64-neon-syntax=generic | FileCheck %s -check-prefixes=CHECK,CHECK-SD
-; RUN: llc < %s -mtriple=aarch64 -global-isel=1 -global-isel-abort=2 -aarch64-neon-syntax=generic 2>&1 | FileCheck %s --check-prefixes=CHECK,CHECK-GI
+; RUN: llc < %s -mtriple=aarch64 -aarch64-neon-syntax=generic | FileCheck %s -check-prefixes=CHECK,SDAG
+; RUN: llc < %s -mtriple=aarch64 -global-isel=1 -aarch64-neon-syntax=generic | FileCheck %s --check-prefixes=CHECK,GISEL
 
 declare i8 @llvm.vector.reduce.add.v2i8(<2 x i8>)
 declare i8 @llvm.vector.reduce.add.v3i8(<3 x i8>)
@@ -22,15 +22,6 @@ declare i64 @llvm.vector.reduce.add.v3i64(<3 x i64>)
 declare i64 @llvm.vector.reduce.add.v4i64(<4 x i64>)
 declare i128 @llvm.vector.reduce.add.v2i128(<2 x i128>)
 
-; CHECK-GI:       warning: Instruction selection used fallback path for addv_v2i8
-; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for addv_v3i8
-; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for addv_v4i8
-; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for addv_v2i16
-; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for addv_v3i16
-; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for addv_v3i32
-; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for addv_v3i64
-; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for addv_v2i128
-
 
 define i8 @add_B(ptr %arr)  {
 ; CHECK-LABEL: add_B:
@@ -256,15 +247,26 @@ entry:
 }
 
 define i8 @addv_v3i8(<3 x i8> %a) {
-; CHECK-LABEL: addv_v3i8:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    movi v0.2d, #0000000000000000
-; CHECK-NEXT:    mov v0.h[0], w0
-; CHECK-NEXT:    mov v0.h[1], w1
-; CHECK-NEXT:    mov v0.h[2], w2
-; CHECK-NEXT:    addv h0, v0.4h
-; CHECK-NEXT:    fmov w0, s0
-; CHECK-NEXT:    ret
+; SDAG-LABEL: addv_v3i8:
+; SDAG:       // %bb.0: // %entry
+; SDAG-NEXT:    movi v0.2d, #0000000000000000
+; SDAG-NEXT:    mov v0.h[0], w0
+; SDAG-NEXT:    mov v0.h[1], w1
+; SDAG-NEXT:    mov v0.h[2], w2
+; SDAG-NEXT:    addv h0, v0.4h
+; SDAG-NEXT:    fmov w0, s0
+; SDAG-NEXT:    ret
+;
+; GISEL-LABEL: addv_v3i8:
+; GISEL:       // %bb.0: // %entry
+; GISEL-NEXT:    fmov s0, w0
+; GISEL-NEXT:    mov w8, #0 // =0x0
+; GISEL-NEXT:    mov v0.h[1], w1
+; GISEL-NEXT:    mov v0.h[2], w2
+; GISEL-NEXT:    mov v0.h[3], w8
+; GISEL-NEXT:    addv h0, v0.4h
+; GISEL-NEXT:    fmov w0, s0
+; GISEL-NEXT:    ret
 entry:
   %arg1 = call i8 @llvm.vector.reduce.add.v3i8(<3 x i8> %a)
   ret i8 %arg1
@@ -327,13 +329,22 @@ entry:
 }
 
 define i16 @addv_v3i16(<3 x i16> %a) {
-; CHECK-LABEL: addv_v3i16:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-NEXT:    mov v0.h[3], wzr
-; CHECK-NEXT:    addv h0, v0.4h
-; CHECK-NEXT:    fmov w0, s0
-; CHECK-NEXT:    ret
+; SDAG-LABEL: addv_v3i16:
+; SDAG:       // %bb.0: // %entry
+; SDAG-NEXT:    // kill: def $d0 killed $d0 def $q0
+; SDAG-NEXT:    mov v0.h[3], wzr
+; SDAG-NEXT:    addv h0, v0.4h
+; SDAG-NEXT:    fmov w0, s0
+; SDAG-NEXT:    ret
+;
+; GISEL-LABEL: addv_v3i16:
+; GISEL:       // %bb.0: // %entry
+; GISEL-NEXT:    // kill: def $d0 killed $d0 def $q0
+; GISEL-NEXT:    mov w8, #0 // =0x0
+; GISEL-NEXT:    mov v0.h[3], w8
+; GISEL-NEXT:    addv h0, v0.4h
+; GISEL-NEXT:    fmov w0, s0
+; GISEL-NEXT:    ret
 entry:
   %arg1 = call i16 @llvm.vector.reduce.add.v3i16(<3 x i16> %a)
   ret i16 %arg1
@@ -431,17 +442,29 @@ entry:
 }
 
 define i64 @addv_v3i64(<3 x i64> %a) {
-; CHECK-LABEL: addv_v3i64:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    // kill: def $d2 killed $d2 def $q2
-; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-NEXT:    // kill: def $d1 killed $d1 def $q1
-; CHECK-NEXT:    mov v0.d[1], v1.d[0]
-; CHECK-NEXT:    mov v2.d[1], xzr
-; CHECK-NEXT:    add v0.2d, v0.2d, v2.2d
-; CHECK-NEXT:    addp d0, v0.2d
-; CHECK-NEXT:    fmov x0, d0
-; CHECK-NEXT:    ret
+; SDAG-LABEL: addv_v3i64:
+; SDAG:       // %bb.0: // %entry
+; SDAG-NEXT:    // kill: def $d2 killed $d2 def $q2
+; SDAG-NEXT:    // kill: def $d0 killed $d0 def $q0
+; SDAG-NEXT:    // kill: def $d1 killed $d1 def $q1
+; SDAG-NEXT:    mov v0.d[1], v1.d[0]
+; SDAG-NEXT:    mov v2.d[1], xzr
+; SDAG-NEXT:    add v0.2d, v0.2d, v2.2d
+; SDAG-NEXT:    addp d0, v0.2d
+; SDAG-NEXT:    fmov x0, d0
+; SDAG-NEXT:    ret
+;
+; GISEL-LABEL: addv_v3i64:
+; GISEL:       // %bb.0: // %entry
+; GISEL-NEXT:    // kill: def $d0 killed $d0 def $q0
+; GISEL-NEXT:    // kill: def $d2 killed $d2 def $q2
+; GISEL-NEXT:    // kill: def $d1 killed $d1 def $q1
+; GISEL-NEXT:    mov v0.d[1], v1.d[0]
+; GISEL-NEXT:    mov v2.d[1], xzr
+; GISEL-NEXT:    add v0.2d, v0.2d, v2.2d
+; GISEL-NEXT:    addp d0, v0.2d
+; GISEL-NEXT:    fmov x0, d0
+; GISEL-NEXT:    ret
 entry:
   %arg1 = call i64 @llvm.vector.reduce.add.v3i64(<3 x i64> %a)
   ret i64 %arg1
diff --git a/llvm/test/CodeGen/AArch64/aarch64-mixed-ptr-sizes.ll b/llvm/test/CodeGen/AArch64/aarch64-mixed-ptr-sizes.ll
new file mode 100644
index 0000000000000..b7c8df7310adc
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/aarch64-mixed-ptr-sizes.ll
@@ -0,0 +1,182 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s | FileCheck %s
+; RUN: llc --fast-isel < %s | FileCheck %s
+; RUN: llc --global-isel --global-isel-abort=2 < %s | FileCheck %s
+
+; Source to regenerate:
+; struct Foo {
+;   int * __ptr32 p32;
+;   int * __ptr64 p64;
+;   __attribute__((address_space(9))) int *p_other;
+; };
+; extern "C" void use_foo(Foo *f);
+; extern "C" int use_int(int i);
+; extern "C" void test_sign_ext(Foo *f, int * __ptr32 __sptr i) {
+;   f->p64 = i;
+;   use_foo(f);
+; }
+; extern "C" void test_sign_ext_store_load(int * __ptr32 __sptr i) {
+;   *i = use_int(*i);
+; }
+; extern "C" void test_zero_ext(Foo *f, int * __ptr32 __uptr i) {
+;   f->p64 = i;
+;   use_foo(f);
+; }
+; extern "C" void test_zero_ext_store_load(int * __ptr32 __uptr i) {
+;   *i = use_int(*i);
+; }
+; extern "C" void test_trunc(Foo *f, int * __ptr64 i) {
+;   f->p32 = i;
+;   use_foo(f);
+; }
+; extern "C" void test_noop1(Foo *f, int * __ptr32 i) {
+;   f->p32 = i;
+;   use_foo(f);
+; }
+; extern "C" void test_noop2(Foo *f, int * __ptr64 i) {
+;   f->p64 = i;
+;   use_foo(f);
+; }
+; extern "C" void test_null_arg(Foo *f, int * __ptr32 i) {
+;   test_noop1(f, 0);
+; }
+; extern "C" void test_unrecognized(Foo *f, __attribute__((address_space(14))) int *i) {
+;   f->p32 = (int * __ptr32)i;
+;   use_foo(f);
+; }
+;
+; $ clang --target=aarch64-windows-msvc -fms-extensions -O2 -S -emit-llvm t.cpp
+
+target datalayout = "e-m:w-p:64:64-i32:32-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-n32:64-S128-Fn32"
+target triple = "aarch64-unknown-windows-msvc"
+
+; Function Attrs: mustprogress uwtable
+define dso_local void @test_sign_ext(ptr noundef %f, ptr addrspace(270) noundef %i) local_unnamed_addr #0 {
+; CHECK-LABEL: test_sign_ext:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    // kill: def $w1 killed $w1 def $x1
+; CHECK-NEXT:    sxtw x8, w1
+; CHECK-NEXT:    str  x8, [x0, #8]
+; CHECK-NEXT:    b    use_foo
+entry:
+  %0 = addrspacecast ptr addrspace(270) %i to ptr
+  %p64 = getelementptr inbounds nuw i8, ptr %f, i64 8
+  store ptr %0, ptr %p64, align 8
+  tail call void @use_foo(ptr noundef %f)
+  ret void
+}
+
+declare dso_local void @use_foo(ptr noundef) local_unnamed_addr #1
+
+; Function Attrs: mustprogress uwtable
+define dso_local void @test_sign_ext_store_load(ptr addrspace(270) nocapture noundef %i) local_unnamed_addr #0 {
+; CHECK-LABEL: test_sign_ext_store_load:
+; CHECK:       // %bb.0: // %entry
+; CHECK:       sxtw    x19, w0
+; CHECK-NEXT:  ldr     w0, [x19]
+; CHECK-NEXT:  bl      use_int
+; CHECK-NEXT:  str     w0, [x19]
+entry:
+  %0 = load i32, ptr addrspace(270) %i, align 4
+  %call = tail call i32 @use_int(i32 noundef %0)
+  store i32 %call, ptr addrspace(270) %i, align 4
+  ret void
+}
+
+declare dso_local i32 @use_int(i32 noundef) local_unnamed_addr #1
+
+; Function Attrs: mustprogress uwtable
+define dso_local void @test_zero_ext(ptr noundef %f, ptr addrspace(271) noundef %i) local_unnamed_addr #0 {
+; CHECK-LABEL: test_zero_ext:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:  mov     w8, w1
+; CHECK-NEXT:  str     x8, [x0, #8]
+; CHECK-NEXT:  b       use_foo
+entry:
+  %0 = addrspacecast ptr addrspace(271) %i to ptr
+  %p64 = getelementptr inbounds nuw i8, ptr %f, i64 8
+  store ptr %0, ptr %p64, align 8
+  tail call void @use_foo(ptr noundef %f)
+  ret void
+}
+
+; Function Attrs: mustprogress uwtable
+define dso_local void @test_zero_ext_store_load(ptr addrspace(271) nocapture noundef %i) local_unnamed_addr #0 {
+; CHECK-LABEL: test_zero_ext_store_load:
+; CHECK:       // %bb.0: // %entry
+; CHECK:       mov     w19, w0
+; CHECK-NEXT:  ldr     w0, [x19]
+; CHECK-NEXT:  bl      use_int
+; CHECK-NEXT:  str     w0, [x19]
+entry:
+  %0 = load i32, ptr addrspace(271) %i, align 4
+  %call = tail call i32 @use_int(i32 noundef %0)
+  store i32 %call, ptr addrspace(271) %i, align 4
+  ret void
+}
+
+; Function Attrs: mustprogress uwtable
+define dso_local void @test_trunc(ptr noundef %f, ptr noundef %i) local_unnamed_addr #0 {
+; CHECK-LABEL: test_trunc:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    str     w1, [x0]
+; CHECK-NEXT:    b      use_foo
+entry:
+  %0 = addrspacecast ptr %i to ptr addrspace(270)
+  store ptr addrspace(270) %0, ptr %f, align 8
+  tail call void @use_foo(ptr noundef nonnull %f)
+  ret void
+}
+
+; Function Attrs: mustprogress uwtable
+define dso_local void @test_noop1(ptr noundef %f, ptr addrspace(270) noundef %i) local_unnamed_addr #0 {
+; CHECK-LABEL: test_noop1:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    str  w1, [x0]
+; CHECK-NEXT:    b    use_foo
+entry:
+  store ptr addrspace(270) %i, ptr %f, align 8
+  tail call void @use_foo(ptr noundef nonnull %f)
+  ret void
+}
+
+; Function Attrs: mustprogress uwtable
+define dso_local void @test_noop2(ptr noundef %f, ptr noundef %i) local_unnamed_addr #0 {
+; CHECK-LABEL: test_noop2:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    str  x1, [x0, #8]
+; CHECK-NEXT:    b    use_foo
+entry:
+  %p64 = getelementptr inbounds nuw i8, ptr %f, i64 8
+  store ptr %i, ptr %p64, align 8
+  tail call void @use_foo(ptr noundef %f)
+  ret void
+}
+
+; Function Attrs: mustprogress uwtable
+define dso_local void @test_null_arg(ptr noundef %f, ptr addrspace(270) nocapture noundef readnone %i) local_unnamed_addr #0 {
+; CHECK-LABEL: test_null_arg:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    str  wzr, [x0]
+; CHECK-NEXT:    b    use_foo
+entry:
+  store ptr addrspace(270) null, ptr %f, align 8
+  tail call void @use_foo(ptr noundef nonnull %f)
+  ret void
+}
+
+; Function Attrs: mustprogress uwtable
+define dso_local void @test_unrecognized(ptr noundef %f, ptr addrspace(14) noundef %i) local_unnamed_addr #0 {
+; CHECK-LABEL: test_unrecognized:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    str   w1, [x0]
+; CHECK-NEXT:    b    use_foo
+entry:
+  %0 = addrspacecast ptr addrspace(14) %i to ptr addrspace(270)
+  store ptr addrspace(270) %0, ptr %f, align 8
+  tail call void @use_foo(ptr noundef nonnull %f)
+  ret void
+}
+
+attributes #0 = { mustprogress uwtable "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="generic" "target-features"="+fp-armv8,+neon,+v8a,-fmv" }
+attributes #1 = { "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="generic" "target-features"="+fp-armv8,+neon,+v8a,-fmv" }
diff --git a/llvm/test/CodeGen/AArch64/addsub_ext.ll b/llvm/test/CodeGen/AArch64/addsub_ext.ll
index 04a98bd508880..6c3564375279e 100644
--- a/llvm/test/CodeGen/AArch64/addsub_ext.ll
+++ b/llvm/test/CodeGen/AArch64/addsub_ext.ll
@@ -26,7 +26,6 @@ define i32 @add_z_shli8i32(i8 %v, i32 %lhs) minsize {
 define i64 @add_z_i8i64(i8 %v, i64 %lhs) minsize {
 ; CHECK-LABEL: add_z_i8i64:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    // kill: def $w0 killed $w0 def $x0
 ; CHECK-NEXT:    add x0, x1, w0, uxtb
 ; CHECK-NEXT:    ret
   %vz = zext i8 %v to i64
@@ -37,7 +36,6 @@ define i64 @add_z_i8i64(i8 %v, i64 %lhs) minsize {
 define i64 @add_z_shli8i64(i8 %v, i64 %lhs) minsize {
 ; CHECK-LABEL: add_z_shli8i64:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    // kill: def $w0 killed $w0 def $x0
 ; CHECK-NEXT:    add x0, x1, w0, uxtb #3
 ; CHECK-NEXT:    ret
   %vz = zext i8 %v to i64
@@ -70,7 +68,6 @@ define i32 @add_s_shli8i32(i8 %v, i32 %lhs) minsize {
 define i64 @add_s_i8i64(i8 %v, i64 %lhs) minsize {
 ; CHECK-LABEL: add_s_i8i64:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    // kill: def $w0 killed $w0 def $x0
 ; CHECK-NEXT:    add x0, x1, w0, sxtb
 ; CHECK-NEXT:    ret
   %vz = sext i8 %v to i64
@@ -81,7 +78,6 @@ define i64 @add_s_i8i64(i8 %v, i64 %lhs) minsize {
 define i64 @add_s_shli8i64(i8 %v, i64 %lhs) minsize {
 ; CHECK-LABEL: add_s_shli8i64:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    // kill: def $w0 killed $w0 def $x0
 ; CHECK-NEXT:    add x0, x1, w0, sxtb #3
 ; CHECK-NEXT:    ret
   %vz = sext i8 %v to i64
@@ -114,7 +110,6 @@ define i32 @add_z_shli16i32(i16 %v, i32 %lhs) minsize {
 define i64 @add_z_i16i64(i16 %v, i64 %lhs) minsize {
 ; CHECK-LABEL: add_z_i16i64:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    // kill: def $w0 killed $w0 def $x0
 ; CHECK-NEXT:    add x0, x1, w0, uxth
 ; CHECK-NEXT:    ret
   %vz = zext i16 %v to i64
@@ -125,7 +120,6 @@ define i64 @add_z_i16i64(i16 %v, i64 %lhs) minsize {
 define i64 @add_z_shli16i64(i16 %v, i64 %lhs) minsize {
 ; CHECK-LABEL: add_z_shli16i64:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    // kill: def $w0 killed $w0 def $x0
 ; CHECK-NEXT:    add x0, x1, w0, uxth #3
 ; CHECK-NEXT:    ret
   %vz = zext i16 %v to i64
@@ -179,7 +173,6 @@ define i32 @add_s_shli16i32(i16 %v, i32 %lhs) minsize {
 define i64 @add_s_i16i64(i16 %v, i64 %lhs) minsize {
 ; CHECK-LABEL: add_s_i16i64:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    // kill: def $w0 killed $w0 def $x0
 ; CHECK-NEXT:    add x0, x1, w0, sxth
 ; CHECK-NEXT:    ret
   %vz = sext i16 %v to i64
@@ -190,7 +183,6 @@ define i64 @add_s_i16i64(i16 %v, i64 %lhs) minsize {
 define i64 @add_s_shli16i64(i16 %v, i64 %lhs) minsize {
 ; CHECK-LABEL: add_s_shli16i64:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    // kill: def $w0 killed $w0 def $x0
 ; CHECK-NEXT:    add x0, x1, w0, sxth #3
 ; CHECK-NEXT:    ret
   %vz = sext i16 %v to i64
@@ -244,7 +236,6 @@ define i32 @sub_z_shli8i32(i8 %v, i32 %lhs) minsize {
 define i64 @sub_z_i8i64(i8 %v, i64 %lhs) minsize {
 ; CHECK-LABEL: sub_z_i8i64:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    // kill: def $w0 killed $w0 def $x0
 ; CHECK-NEXT:    sub x0, x1, w0, uxtb
 ; CHECK-NEXT:    ret
   %vz = zext i8 %v to i64
@@ -255,7 +246,6 @@ define i64 @sub_z_i8i64(i8 %v, i64 %lhs) minsize {
 define i64 @sub_z_shli8i64(i8 %v, i64 %lhs) minsize {
 ; CHECK-LABEL: sub_z_shli8i64:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    // kill: def $w0 killed $w0 def $x0
 ; CHECK-NEXT:    sub x0, x1, w0, uxtb #3
 ; CHECK-NEXT:    ret
   %vz = zext i8 %v to i64
@@ -288,7 +278,6 @@ define i32 @sub_s_shli8i32(i8 %v, i32 %lhs) minsize {
 define i64 @sub_s_i8i64(i8 %v, i64 %lhs) minsize {
 ; CHECK-LABEL: sub_s_i8i64:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    // kill: def $w0 killed $w0 def $x0
 ; CHECK-NEXT:    sub x0, x1, w0, sxtb
 ; CHECK-NEXT:    ret
   %vz = sext i8 %v to i64
@@ -299,7 +288,6 @@ define i64 @sub_s_i8i64(i8 %v, i64 %lhs) minsize {
 define i64 @sub_s_shli8i64(i8 %v, i64 %lhs) minsize {
 ; CHECK-LABEL: sub_s_shli8i64:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    // kill: def $w0 killed $w0 def $x0
 ; CHECK-NEXT:    sub x0, x1, w0, sxtb #3
 ; CHECK-NEXT:    ret
   %vz = sext i8 %v to i64
@@ -332,7 +320,6 @@ define i32 @sub_z_shli16i32(i16 %v, i32 %lhs) minsize {
 define i64 @sub_z_i16i64(i16 %v, i64 %lhs) minsize {
 ; CHECK-LABEL: sub_z_i16i64:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    // kill: def $w0 killed $w0 def $x0
 ; CHECK-NEXT:    sub x0, x1, w0, uxth
 ; CHECK-NEXT:    ret
   %vz = zext i16 %v to i64
@@ -343,7 +330,6 @@ define i64 @sub_z_i16i64(i16 %v, i64 %lhs) minsize {
 define i64 @sub_z_shli16i64(i16 %v, i64 %lhs) minsize {
 ; CHECK-LABEL: sub_z_shli16i64:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    // kill: def $w0 killed $w0 def $x0
 ; CHECK-NEXT:    sub x0, x1, w0, uxth #3
 ; CHECK-NEXT:    ret
   %vz = zext i16 %v to i64
@@ -397,7 +383,6 @@ define i32 @sub_s_shli16i32(i16 %v, i32 %lhs) minsize {
 define i64 @sub_s_i16i64(i16 %v, i64 %lhs) minsize {
 ; CHECK-LABEL: sub_s_i16i64:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    // kill: def $w0 killed $w0 def $x0
 ; CHECK-NEXT:    sub x0, x1, w0, sxth
 ; CHECK-NEXT:    ret
   %vz = sext i16 %v to i64
@@ -408,7 +393,6 @@ define i64 @sub_s_i16i64(i16 %v, i64 %lhs) minsize {
 define i64 @sub_s_shli16i64(i16 %v, i64 %lhs) minsize {
 ; CHECK-LABEL: sub_s_shli16i64:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    // kill: def $w0 killed $w0 def $x0
 ; CHECK-NEXT:    sub x0, x1, w0, sxth #3
 ; CHECK-NEXT:    ret
   %vz = sext i16 %v to i64
@@ -444,7 +428,7 @@ define i32 @cmp_s_i8i32(i8 %v, i32 %lhs) minsize {
 ; CHECK-NEXT:    cmp w1, w0, uxtb
 ; CHECK-NEXT:    b.ge .LBB40_2
 ; CHECK-NEXT:  // %bb.1: // %then
-; CHECK-NEXT:    mov w0, #1
+; CHECK-NEXT:    mov w0, #1 // =0x1
 ; CHECK-NEXT:    ret
 ; CHECK-NEXT:  .LBB40_2: // %end
 ; CHECK-NEXT:    mov w0, w1
@@ -461,11 +445,10 @@ end:
 define i64 @cmp_s_i8i64(i8 %v, i64 %lhs) minsize {
 ; CHECK-LABEL: cmp_s_i8i64:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    // kill: def $w0 killed $w0 def $x0
 ; CHECK-NEXT:    cmp x1, w0, sxtb
 ; CHECK-NEXT:    b.ge .LBB41_2
 ; CHECK-NEXT:  // %bb.1: // %then
-; CHECK-NEXT:    mov w0, #1
+; CHECK-NEXT:    mov w0, #1 // =0x1
 ; CHECK-NEXT:    ret
 ; CHECK-NEXT:  .LBB41_2: // %end
 ; CHECK-NEXT:    mov x0, x1
@@ -485,7 +468,7 @@ define i32 @cmp_s_i16i32(i16 %v, i32 %lhs) minsize {
 ; CHECK-NEXT:    cmp w1, w0, uxth
 ; CHECK-NEXT:    b.ge .LBB42_2
 ; CHECK-NEXT:  // %bb.1: // %then
-; CHECK-NEXT:    mov w0, #1
+; CHECK-NEXT:    mov w0, #1 // =0x1
 ; CHECK-NEXT:    ret
 ; CHECK-NEXT:  .LBB42_2: // %end
 ; CHECK-NEXT:    mov w0, w1
@@ -502,11 +485,10 @@ end:
 define i64 @cmp_s_i16i64(i16 %v, i64 %lhs) minsize {
 ; CHECK-LABEL: cmp_s_i16i64:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    // kill: def $w0 killed $w0 def $x0
 ; CHECK-NEXT:    cmp x1, w0, sxth
 ; CHECK-NEXT:    b.ge .LBB43_2
 ; CHECK-NEXT:  // %bb.1: // %then
-; CHECK-NEXT:    mov w0, #1
+; CHECK-NEXT:    mov w0, #1 // =0x1
 ; CHECK-NEXT:    ret
 ; CHECK-NEXT:  .LBB43_2: // %end
 ; CHECK-NEXT:    mov x0, x1
diff --git a/llvm/test/CodeGen/AArch64/and-mask-removal.ll b/llvm/test/CodeGen/AArch64/and-mask-removal.ll
index 09f00b3845f25..5046c0571ad2b 100644
--- a/llvm/test/CodeGen/AArch64/and-mask-removal.ll
+++ b/llvm/test/CodeGen/AArch64/and-mask-removal.ll
@@ -11,7 +11,6 @@ define void @new_position(i32 %pos) {
 ; CHECK-SD-LABEL: new_position:
 ; CHECK-SD:       ; %bb.0: ; %entry
 ; CHECK-SD-NEXT:    adrp x8, _board@GOTPAGE
-; CHECK-SD-NEXT:    ; kill: def $w0 killed $w0 def $x0
 ; CHECK-SD-NEXT:    ldr x8, [x8, _board@GOTPAGEOFF]
 ; CHECK-SD-NEXT:    ldrb w8, [x8, w0, sxtw]
 ; CHECK-SD-NEXT:    sub w8, w8, #1
diff --git a/llvm/test/CodeGen/AArch64/arm64-2011-03-09-CPSRSpill.ll b/llvm/test/CodeGen/AArch64/arm64-2011-03-09-CPSRSpill.ll
index f31f43f1fde45..f1ffb265e29bc 100644
--- a/llvm/test/CodeGen/AArch64/arm64-2011-03-09-CPSRSpill.ll
+++ b/llvm/test/CodeGen/AArch64/arm64-2011-03-09-CPSRSpill.ll
@@ -3,15 +3,15 @@
 ; Can't copy or spill / restore CPSR.
 ; rdar://9105206
 
-define fastcc void @t() ssp align 2 {
+define fastcc void @t(i1 %arg) ssp align 2 {
 entry:
-  br i1 undef, label %bb3.i, label %bb2.i
+  br i1 %arg, label %bb3.i, label %bb2.i
 
 bb2.i:                                            ; preds = %entry
   br label %bb3.i
 
 bb3.i:                                            ; preds = %bb2.i, %entry
-  br i1 undef, label %_ZN12gjkepa2_impl3EPA6appendERNS0_5sListEPNS0_5sFaceE.exit71, label %bb.i69
+  br i1 %arg, label %_ZN12gjkepa2_impl3EPA6appendERNS0_5sListEPNS0_5sFaceE.exit71, label %bb.i69
 
 bb.i69:                                           ; preds = %bb3.i
   br label %_ZN12gjkepa2_impl3EPA6appendERNS0_5sListEPNS0_5sFaceE.exit71
@@ -22,10 +22,10 @@ _ZN12gjkepa2_impl3EPA6appendERNS0_5sListEPNS0_5sFaceE.exit71: ; preds = %bb.i69,
   %2 = fcmp ult float %1, 0xBF847AE140000000
   %storemerge9 = select i1 %2, float %1, float 0.000000e+00
   store float %storemerge9, ptr undef, align 4
-  br i1 undef, label %bb42, label %bb47
+  br i1 %arg, label %bb42, label %bb47
 
 bb42:                                             ; preds = %_ZN12gjkepa2_impl3EPA6appendERNS0_5sListEPNS0_5sFaceE.exit71
-  br i1 undef, label %bb46, label %bb53
+  br i1 %arg, label %bb46, label %bb53
 
 bb46:                                             ; preds = %bb42
   br label %bb48
@@ -34,7 +34,7 @@ bb47:                                             ; preds = %_ZN12gjkepa2_impl3E
   br label %bb48
 
 bb48:                                             ; preds = %bb47, %bb46
-  br i1 undef, label %bb1.i14, label %bb.i13
+  br i1 %arg, label %bb1.i14, label %bb.i13
 
 bb.i13:                                           ; preds = %bb48
   br label %bb1.i14
diff --git a/llvm/test/CodeGen/AArch64/arm64-2011-03-17-AsmPrinterCrash.ll b/llvm/test/CodeGen/AArch64/arm64-2011-03-17-AsmPrinterCrash.ll
index a4720a9a738f7..86658680fdf5a 100644
--- a/llvm/test/CodeGen/AArch64/arm64-2011-03-17-AsmPrinterCrash.ll
+++ b/llvm/test/CodeGen/AArch64/arm64-2011-03-17-AsmPrinterCrash.ll
@@ -5,10 +5,10 @@
 source_filename = "test/CodeGen/AArch64/arm64-2011-03-17-AsmPrinterCrash.ll"
 
 ; Function Attrs: nounwind ssp
-define void @drt_vsprintf() #0 {
+define void @drt_vsprintf(i1 %arg) #0 {
 entry:
   %do_tab_convert = alloca i32, align 4
-  br i1 undef, label %if.then24, label %if.else295, !dbg !11
+  br i1 %arg, label %if.then24, label %if.else295, !dbg !11
 
 if.then24:                                        ; preds = %entry
   unreachable
diff --git a/llvm/test/CodeGen/AArch64/arm64-2011-04-21-CPSRBug.ll b/llvm/test/CodeGen/AArch64/arm64-2011-04-21-CPSRBug.ll
index 83a9ae7a1b7b0..bbb991d5531b3 100644
--- a/llvm/test/CodeGen/AArch64/arm64-2011-04-21-CPSRBug.ll
+++ b/llvm/test/CodeGen/AArch64/arm64-2011-04-21-CPSRBug.ll
@@ -3,7 +3,7 @@
 ; CPSR is not allocatable so fast allocatable wouldn't mark them killed.
 ; rdar://9313272
 
-define hidden void @t() nounwind {
+define hidden void @t(i1 %arg) nounwind {
 entry:
   %cmp = icmp eq ptr null, undef
   %frombool = zext i1 %cmp to i8
@@ -16,7 +16,7 @@ land.lhs.true:                                    ; preds = %entry
   unreachable
 
 if.end:                                           ; preds = %entry
-  br i1 undef, label %land.lhs.true14, label %if.end33
+  br i1 %arg, label %land.lhs.true14, label %if.end33
 
 land.lhs.true14:                                  ; preds = %if.end
   unreachable
diff --git a/llvm/test/CodeGen/AArch64/arm64-2012-01-11-ComparisonDAGCrash.ll b/llvm/test/CodeGen/AArch64/arm64-2012-01-11-ComparisonDAGCrash.ll
index b7b111c60bd9b..96278fd8a5391 100644
--- a/llvm/test/CodeGen/AArch64/arm64-2012-01-11-ComparisonDAGCrash.ll
+++ b/llvm/test/CodeGen/AArch64/arm64-2012-01-11-ComparisonDAGCrash.ll
@@ -5,12 +5,12 @@
 ; cycles in DAGs, and eventually crashes.  This is the testcase for
 ; one of those crashes. (rdar://10653656)
 
-define void @test(i1 zeroext %IsArrow) nounwind ssp align 2 {
+define void @test(i1 zeroext %IsArrow, i1 %arg) nounwind ssp align 2 {
 entry:
-  br i1 undef, label %return, label %lor.lhs.false
+  br i1  %arg, label %return, label %lor.lhs.false
 
 lor.lhs.false:
-  br i1 undef, label %return, label %if.end
+  br i1  %arg, label %return, label %if.end
 
 if.end:
   %tmp.i = load i64, ptr undef, align 8
@@ -18,7 +18,7 @@ if.end:
   br i1 %IsArrow, label %if.else_crit_edge, label %if.end32
 
 if.else_crit_edge:
-  br i1 undef, label %if.end32, label %return
+  br i1  %arg, label %if.end32, label %return
 
 if.end32:
   %0 = icmp ult i32 undef, 3
@@ -27,7 +27,7 @@ if.end32:
   %.pn = shl i320 %1, %.pn.v
   %ins346392 = or i320 %.pn, 0
   store i320 %ins346392, ptr undef, align 8
-  br i1 undef, label %sw.bb.i.i, label %exit
+  br i1  %arg, label %sw.bb.i.i, label %exit
 
 sw.bb.i.i:
   unreachable
diff --git a/llvm/test/CodeGen/AArch64/arm64-2012-07-11-InstrEmitterBug.ll b/llvm/test/CodeGen/AArch64/arm64-2012-07-11-InstrEmitterBug.ll
index b87fe926fb32c..f44752545a9dd 100644
--- a/llvm/test/CodeGen/AArch64/arm64-2012-07-11-InstrEmitterBug.ll
+++ b/llvm/test/CodeGen/AArch64/arm64-2012-07-11-InstrEmitterBug.ll
@@ -15,18 +15,18 @@ declare ptr @__strcat_chk(ptr, ptr, i64) nounwind optsize
 
 declare noalias ptr @xstrdup(ptr) optsize
 
-define ptr @dyld_fix_path(ptr %path) nounwind optsize ssp {
+define ptr @dyld_fix_path(ptr %path, i1 %arg) nounwind optsize ssp {
 entry:
-  br i1 undef, label %if.end56, label %for.cond
+  br i1  %arg, label %if.end56, label %for.cond
 
 for.cond:                                         ; preds = %entry
-  br i1 undef, label %for.cond10, label %for.body
+  br i1  %arg, label %for.cond10, label %for.body
 
 for.body:                                         ; preds = %for.cond
   unreachable
 
 for.cond10:                                       ; preds = %for.cond
-  br i1 undef, label %if.end56, label %for.body14
+  br i1  %arg, label %if.end56, label %for.body14
 
 for.body14:                                       ; preds = %for.cond10
   %call22 = tail call i64 @strlen(ptr undef) nounwind optsize
@@ -38,7 +38,7 @@ for.body14:                                       ; preds = %for.cond10
   %sext59 = add i64 %add31, 4294967296
   %conv33 = ashr exact i64 %sext59, 32
   %call34 = tail call noalias ptr @xmalloc(i64 %conv33) nounwind optsize
-  br i1 undef, label %cond.false45, label %cond.true43
+  br i1  %arg, label %cond.false45, label %cond.true43
 
 cond.true43:                                      ; preds = %for.body14
   unreachable
diff --git a/llvm/test/CodeGen/AArch64/arm64-2013-01-23-frem-crash.ll b/llvm/test/CodeGen/AArch64/arm64-2013-01-23-frem-crash.ll
index f28702bdbdd0c..6a520057e1388 100644
--- a/llvm/test/CodeGen/AArch64/arm64-2013-01-23-frem-crash.ll
+++ b/llvm/test/CodeGen/AArch64/arm64-2013-01-23-frem-crash.ll
@@ -1,14 +1,14 @@
 ; RUN: llc < %s -mtriple=arm64-eabi
 ; Make sure we are not crashing on this test.
 
-define void @autogen_SD13158() {
+define void @autogen_SD13158(i1 %arg) {
 entry:
-  %B26 = frem float 0.000000e+00, undef
-  br i1 undef, label %CF, label %CF77
+  %B26 = frem float 0.000000e+00, poison
+  br i1 %arg, label %CF, label %CF77
 
 CF:                                               ; preds = %CF, %CF76
-  store float %B26, ptr undef
-  br i1 undef, label %CF, label %CF77
+  store float %B26, ptr poison
+  br i1 %arg, label %CF, label %CF77
 
 CF77:                                             ; preds = %CF
   ret void
diff --git a/llvm/test/CodeGen/AArch64/arm64-2013-01-23-sext-crash.ll b/llvm/test/CodeGen/AArch64/arm64-2013-01-23-sext-crash.ll
index 29f5cf8e72b64..d5b63efb4a7b8 100644
--- a/llvm/test/CodeGen/AArch64/arm64-2013-01-23-sext-crash.ll
+++ b/llvm/test/CodeGen/AArch64/arm64-2013-01-23-sext-crash.ll
@@ -2,13 +2,13 @@
 
 ; Make sure we are not crashing on this test.
 
-define void @autogen_SD12881() {
+define void @autogen_SD12881(i1 %arg) {
 BB:
   %B17 = ashr <4 x i32> zeroinitializer, zeroinitializer
   br label %CF
 
 CF:                                               ; preds = %CF83, %CF, %BB
-  br i1 undef, label %CF, label %CF83
+  br i1 %arg, label %CF, label %CF83
 
 CF83:                                             ; preds = %CF
   %FC70 = sitofp <4 x i32> %B17 to <4 x double>
@@ -16,13 +16,13 @@ CF83:                                             ; preds = %CF
 }
 
 
-define void @autogen_SD12881_2() {
+define void @autogen_SD12881_2(i1 %arg) {
 BB:
   %B17 = ashr <4 x i32> zeroinitializer, zeroinitializer
   br label %CF
 
 CF:                                               ; preds = %CF83, %CF, %BB
-  br i1 undef, label %CF, label %CF83
+  br i1 %arg, label %CF, label %CF83
 
 CF83:                                             ; preds = %CF
   %FC70 = uitofp <4 x i32> %B17 to <4 x double>
diff --git a/llvm/test/CodeGen/AArch64/arm64-bitfield-extract.ll b/llvm/test/CodeGen/AArch64/arm64-bitfield-extract.ll
index bdbff0563a22b..9c8c34558d776 100644
--- a/llvm/test/CodeGen/AArch64/arm64-bitfield-extract.ll
+++ b/llvm/test/CodeGen/AArch64/arm64-bitfield-extract.ll
@@ -1032,7 +1032,7 @@ define void @sameOperandBFI(i64 %src, i64 %src2, ptr %ptr) {
 ; OPT-NEXT:  entry:
 ; OPT-NEXT:    [[SHR47:%.*]] = lshr i64 [[SRC:%.*]], 47
 ; OPT-NEXT:    [[SRC2_TRUNC:%.*]] = trunc i64 [[SRC2:%.*]] to i32
-; OPT-NEXT:    br i1 undef, label [[END:%.*]], label [[IF_ELSE:%.*]]
+; OPT-NEXT:    br i1 poison, label [[END:%.*]], label [[IF_ELSE:%.*]]
 ; OPT:       if.else:
 ; OPT-NEXT:    [[AND3:%.*]] = and i32 [[SRC2_TRUNC]], 3
 ; OPT-NEXT:    [[SHL2:%.*]] = shl nuw nsw i64 [[SHR47]], 2
@@ -1050,7 +1050,7 @@ define void @sameOperandBFI(i64 %src, i64 %src2, ptr %ptr) {
 entry:
   %shr47 = lshr i64 %src, 47
   %src2.trunc = trunc i64 %src2 to i32
-  br i1 undef, label %end, label %if.else
+  br i1 poison, label %end, label %if.else
 
 if.else:
   %and3 = and i32 %src2.trunc, 3
diff --git a/llvm/test/CodeGen/AArch64/arm64-call-tailcalls.ll b/llvm/test/CodeGen/AArch64/arm64-call-tailcalls.ll
index 7745f8dab1c2b..7342ac18abd94 100644
--- a/llvm/test/CodeGen/AArch64/arm64-call-tailcalls.ll
+++ b/llvm/test/CodeGen/AArch64/arm64-call-tailcalls.ll
@@ -36,12 +36,12 @@ define float @t5(float %a) nounwind readonly ssp {
   ret float %tmp
 }
 
-define void @t7() nounwind {
+define void @t7(i1 %arg) nounwind {
 ; CHECK-LABEL: t7:
 ; CHECK: b	_foo
 ; CHECK: b	_bar
 
-  br i1 undef, label %bb, label %bb1.lr.ph
+  br i1 %arg, label %bb, label %bb1.lr.ph
 
 bb1.lr.ph:                                        ; preds = %entry
   tail call void @bar() nounwind
diff --git a/llvm/test/CodeGen/AArch64/arm64-collect-loh.ll b/llvm/test/CodeGen/AArch64/arm64-collect-loh.ll
index 1b9db8b70a1fb..2c065e0051cd7 100644
--- a/llvm/test/CodeGen/AArch64/arm64-collect-loh.ll
+++ b/llvm/test/CodeGen/AArch64/arm64-collect-loh.ll
@@ -662,9 +662,9 @@ define void @uninterestingSub(ptr nocapture %row) #0 {
 @.str.89 = external unnamed_addr constant [12 x i8], align 1
 @.str.90 = external unnamed_addr constant [5 x i8], align 1
 ; CHECK-LABEL: test_r274582
-define void @test_r274582(double %x) {
+define void @test_r274582(double %x, i1 %arg) {
 entry:
-  br i1 undef, label %if.then.i, label %if.end.i
+  br i1 %arg, label %if.then.i, label %if.end.i
 if.then.i:
   ret void
 if.end.i:
diff --git a/llvm/test/CodeGen/AArch64/arm64-dead-register-def-bug.ll b/llvm/test/CodeGen/AArch64/arm64-dead-register-def-bug.ll
index d43efa7ee794f..d776796aa3ba0 100644
--- a/llvm/test/CodeGen/AArch64/arm64-dead-register-def-bug.ll
+++ b/llvm/test/CodeGen/AArch64/arm64-dead-register-def-bug.ll
@@ -8,14 +8,14 @@
 ;
 ; <rdar://problem/16492408>
 
-define void @testcase() {
+define void @testcase(i1 %arg) {
 ; CHECK: testcase:
 ; CHECK-NOT: orr xzr, xzr, #0x2
 
 bb1:
   %tmp1 = tail call float @ceilf(float 2.000000e+00)
   %tmp2 = fptoui float %tmp1 to i64
-  br i1 undef, label %bb2, label %bb3
+  br i1 %arg, label %bb2, label %bb3
 
 bb2:
   tail call void @foo()
diff --git a/llvm/test/CodeGen/AArch64/arm64-fast-isel.ll b/llvm/test/CodeGen/AArch64/arm64-fast-isel.ll
index 17a36ae3801eb..a902f88fce6b9 100644
--- a/llvm/test/CodeGen/AArch64/arm64-fast-isel.ll
+++ b/llvm/test/CodeGen/AArch64/arm64-fast-isel.ll
@@ -118,14 +118,14 @@ entry:
 
 declare { i64, i1 } @llvm.umul.with.overflow.i64(i64, i64)
 
-define void @logicalReg() {
+define void @logicalReg(i1 %arg) {
 ; Make sure we generate a logical reg = reg, reg instruction without any
 ; machine verifier errors.
 ; CHECK-LABEL: logicalReg:
 ; CHECK: orr w{{[0-9]+}}, w{{[0-9]+}}, w{{[0-9]+}}
 ; CHECK: ret
 entry:
-  br i1 undef, label %cond.end, label %cond.false
+  br i1 %arg, label %cond.end, label %cond.false
 
 cond.false:
   %cond = select i1 undef, i1 true, i1 false
diff --git a/llvm/test/CodeGen/AArch64/arm64-ldxr-stxr.ll b/llvm/test/CodeGen/AArch64/arm64-ldxr-stxr.ll
index d69d1b6eb4a2a..ccd191f163b01 100644
--- a/llvm/test/CodeGen/AArch64/arm64-ldxr-stxr.ll
+++ b/llvm/test/CodeGen/AArch64/arm64-ldxr-stxr.ll
@@ -127,7 +127,6 @@ declare i64 @llvm.aarch64.ldxr.p0(ptr) nounwind
 define dso_local i32 @test_store_i8(i32, i8 %val, ptr %addr) {
 ; CHECK-LABEL: test_store_i8:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    // kill: def $w1 killed $w1 def $x1
 ; CHECK-NEXT:    stxrb w0, w1, [x2]
 ; CHECK-NEXT:    ret
   %extval = zext i8 %val to i64
@@ -138,7 +137,6 @@ define dso_local i32 @test_store_i8(i32, i8 %val, ptr %addr) {
 define dso_local i32 @test_store_i16(i32, i16 %val, ptr %addr) {
 ; CHECK-LABEL: test_store_i16:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    // kill: def $w1 killed $w1 def $x1
 ; CHECK-NEXT:    stxrh w0, w1, [x2]
 ; CHECK-NEXT:    ret
   %extval = zext i16 %val to i64
@@ -299,7 +297,6 @@ declare i64 @llvm.aarch64.ldaxr.p0(ptr) nounwind
 define dso_local i32 @test_store_release_i8(i32, i8 %val, ptr %addr) {
 ; CHECK-LABEL: test_store_release_i8:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    // kill: def $w1 killed $w1 def $x1
 ; CHECK-NEXT:    stlxrb w0, w1, [x2]
 ; CHECK-NEXT:    ret
   %extval = zext i8 %val to i64
@@ -310,7 +307,6 @@ define dso_local i32 @test_store_release_i8(i32, i8 %val, ptr %addr) {
 define dso_local i32 @test_store_release_i16(i32, i16 %val, ptr %addr) {
 ; CHECK-LABEL: test_store_release_i16:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    // kill: def $w1 killed $w1 def $x1
 ; CHECK-NEXT:    stlxrh w0, w1, [x2]
 ; CHECK-NEXT:    ret
   %extval = zext i16 %val to i64
diff --git a/llvm/test/CodeGen/AArch64/arm64-prefetch-addrmode.ll b/llvm/test/CodeGen/AArch64/arm64-prefetch-addrmode.ll
new file mode 100644
index 0000000000000..44202ffba6374
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/arm64-prefetch-addrmode.ll
@@ -0,0 +1,147 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc -mtriple=aarch64 < %s | FileCheck %s
+
+define void @imm9(ptr %object) {
+; CHECK-LABEL: imm9:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    prfum pldl1keep, [x0, #7]
+; CHECK-NEXT:    ret
+  %incdec.ptr = getelementptr inbounds i8, ptr %object, i64 7
+  call void @llvm.prefetch.p0(ptr %incdec.ptr, i32 0, i32 3, i32 1)
+  ret void
+}
+
+define void @imm9_max(ptr %object) {
+; CHECK-LABEL: imm9_max:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    prfum pldl1keep, [x0, #255]
+; CHECK-NEXT:    ret
+  %incdec.ptr = getelementptr inbounds i8, ptr %object, i64 255
+  call void @llvm.prefetch.p0(ptr %incdec.ptr, i32 0, i32 3, i32 1)
+  ret void
+}
+
+define void @imm9_above_max(ptr %object) {
+; CHECK-LABEL: imm9_above_max:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    add x8, x0, #257
+; CHECK-NEXT:    prfm pldl1keep, [x8]
+; CHECK-NEXT:    ret
+  %incdec.ptr = getelementptr inbounds i8, ptr %object, i64 257  ; 256 would use the imm12 mode
+  call void @llvm.prefetch.p0(ptr %incdec.ptr, i32 0, i32 3, i32 1)
+  ret void
+}
+
+define void @imm9_min(ptr %object) {
+; CHECK-LABEL: imm9_min:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    prfum pldl1keep, [x0, #-256]
+; CHECK-NEXT:    ret
+  %incdec.ptr = getelementptr inbounds i8, ptr %object, i64 -256
+  call void @llvm.prefetch.p0(ptr %incdec.ptr, i32 0, i32 3, i32 1)
+  ret void
+}
+
+define void @imm9_below_min(ptr %object) {
+; CHECK-LABEL: imm9_below_min:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    sub x8, x0, #257
+; CHECK-NEXT:    prfm pldl1keep, [x8]
+; CHECK-NEXT:    ret
+  %incdec.ptr = getelementptr inbounds i8, ptr %object, i64 -257
+  call void @llvm.prefetch.p0(ptr %incdec.ptr, i32 0, i32 3, i32 1)
+  ret void
+}
+
+define void @imm12(ptr %object) {
+; CHECK-LABEL: imm12:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    prfm pldl1keep, [x0, #8]
+; CHECK-NEXT:    ret
+  %incdec.ptr = getelementptr inbounds i64, ptr %object, i64 1
+  call void @llvm.prefetch.p0(ptr %incdec.ptr, i32 0, i32 3, i32 1)
+  ret void
+}
+
+define void @imm12_max(ptr %object) {
+; CHECK-LABEL: imm12_max:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    prfm pldl1keep, [x0, #32760]
+; CHECK-NEXT:    ret
+  %incdec.ptr = getelementptr inbounds i64, ptr %object, i64 4095
+  call void @llvm.prefetch.p0(ptr %incdec.ptr, i32 0, i32 3, i32 1)
+  ret void
+}
+
+define void @imm12_above_max(ptr %object) {
+; CHECK-LABEL: imm12_above_max:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mov w8, #32768 // =0x8000
+; CHECK-NEXT:    prfm pldl1keep, [x0, x8]
+; CHECK-NEXT:    ret
+  %incdec.ptr = getelementptr inbounds i64, ptr %object, i64 4096
+  call void @llvm.prefetch.p0(ptr %incdec.ptr, i32 0, i32 3, i32 1)
+  ret void
+}
+
+define void @reg(ptr %object, i64 %a) {
+; CHECK-LABEL: reg:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    prfm pldl1keep, [x0, x1]
+; CHECK-NEXT:    ret
+  %incdec.ptr = getelementptr inbounds i8, ptr %object, i64 %a
+  call void @llvm.prefetch.p0(ptr %incdec.ptr, i32 0, i32 3, i32 1)
+  ret void
+}
+
+define void @reg_shift(ptr %object, i64 %a) {
+; CHECK-LABEL: reg_shift:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    prfm pldl1keep, [x0, x1, lsl #3]
+; CHECK-NEXT:    ret
+  %incdec.ptr = getelementptr inbounds i64, ptr %object, i64 %a
+  call void @llvm.prefetch.p0(ptr %incdec.ptr, i32 0, i32 3, i32 1)
+  ret void
+}
+
+define void @reg_sext(ptr %object, i32 %a) {
+; CHECK-LABEL: reg_sext:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    prfm pldl1keep, [x0, w1, sxtw]
+; CHECK-NEXT:    ret
+  %incdec.ptr = getelementptr inbounds i8, ptr %object, i32 %a
+  call void @llvm.prefetch.p0(ptr %incdec.ptr, i32 0, i32 3, i32 1)
+  ret void
+}
+
+define void @reg_sext_shift(ptr %object, i32 %a) {
+; CHECK-LABEL: reg_sext_shift:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    prfm pldl1keep, [x0, w1, sxtw #3]
+; CHECK-NEXT:    ret
+  %incdec.ptr = getelementptr inbounds i64, ptr %object, i32 %a
+  call void @llvm.prefetch.p0(ptr %incdec.ptr, i32 0, i32 3, i32 1)
+  ret void
+}
+
+define void @reg_zext(ptr %object, i32 %a) {
+; CHECK-LABEL: reg_zext:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    prfm pldl1keep, [x0, w1, uxtw]
+; CHECK-NEXT:    ret
+  %a.zext = zext i32 %a to i64
+  %incdec.ptr = getelementptr inbounds i8, ptr %object, i64 %a.zext
+  call void @llvm.prefetch.p0(ptr %incdec.ptr, i32 0, i32 3, i32 1)
+  ret void
+}
+
+define void @reg_zext_shift(ptr %object, i32 %a) {
+; CHECK-LABEL: reg_zext_shift:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    prfm pldl1keep, [x0, w1, uxtw #3]
+; CHECK-NEXT:    ret
+  %a.zext = zext i32 %a to i64
+  %incdec.ptr = getelementptr inbounds i64, ptr %object, i64 %a.zext
+  call void @llvm.prefetch.p0(ptr %incdec.ptr, i32 0, i32 3, i32 1)
+  ret void
+}
diff --git a/llvm/test/CodeGen/AArch64/arm64-vadd.ll b/llvm/test/CodeGen/AArch64/arm64-vadd.ll
index 38a568ac91916..d982dbbb1f69b 100644
--- a/llvm/test/CodeGen/AArch64/arm64-vadd.ll
+++ b/llvm/test/CodeGen/AArch64/arm64-vadd.ll
@@ -1517,3 +1517,66 @@ define <4 x i32> @subhn2_4s_natural(<2 x i32> %low, ptr %A, ptr %B) nounwind {
         %res = shufflevector <2 x i32> %low, <2 x i32> %narrowed, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
         ret <4 x i32> %res
 }
+
+define <16 x i8> @neg_narrow_i8(<16 x i16> %a) {
+; CHECK-SD-LABEL: neg_narrow_i8:
+; CHECK-SD:       // %bb.0:
+; CHECK-SD-NEXT:    movi v2.2d, #0xffffffffffffffff
+; CHECK-SD-NEXT:    subhn v0.8b, v2.8h, v0.8h
+; CHECK-SD-NEXT:    subhn2 v0.16b, v2.8h, v1.8h
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: neg_narrow_i8:
+; CHECK-GI:       // %bb.0:
+; CHECK-GI-NEXT:    mvn v0.16b, v0.16b
+; CHECK-GI-NEXT:    mvn v1.16b, v1.16b
+; CHECK-GI-NEXT:    shrn v0.8b, v0.8h, #8
+; CHECK-GI-NEXT:    shrn2 v0.16b, v1.8h, #8
+; CHECK-GI-NEXT:    ret
+  %not.i = xor <16 x i16> %a, splat (i16 -1)
+  %s = lshr <16 x i16> %not.i, splat (i16 8)
+  %vshrn_n = trunc nuw <16 x i16> %s to <16 x i8>
+  ret <16 x i8> %vshrn_n
+}
+
+define <8 x i16> @neg_narrow_i16(<8 x i32> %a) {
+; CHECK-SD-LABEL: neg_narrow_i16:
+; CHECK-SD:       // %bb.0:
+; CHECK-SD-NEXT:    movi v2.2d, #0xffffffffffffffff
+; CHECK-SD-NEXT:    subhn v0.4h, v2.4s, v0.4s
+; CHECK-SD-NEXT:    subhn2 v0.8h, v2.4s, v1.4s
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: neg_narrow_i16:
+; CHECK-GI:       // %bb.0:
+; CHECK-GI-NEXT:    mvn v0.16b, v0.16b
+; CHECK-GI-NEXT:    mvn v1.16b, v1.16b
+; CHECK-GI-NEXT:    shrn v0.4h, v0.4s, #16
+; CHECK-GI-NEXT:    shrn2 v0.8h, v1.4s, #16
+; CHECK-GI-NEXT:    ret
+  %not.i = xor <8 x i32> %a, splat (i32 -1)
+  %s = lshr <8 x i32> %not.i, splat (i32 16)
+  %vshrn_n = trunc nuw <8 x i32> %s to <8 x i16>
+  ret <8 x i16> %vshrn_n
+}
+
+define <4 x i32> @neg_narrow_i32(<4 x i64> %a) {
+; CHECK-SD-LABEL: neg_narrow_i32:
+; CHECK-SD:       // %bb.0:
+; CHECK-SD-NEXT:    movi v2.2d, #0xffffffffffffffff
+; CHECK-SD-NEXT:    subhn v0.2s, v2.2d, v0.2d
+; CHECK-SD-NEXT:    subhn2 v0.4s, v2.2d, v1.2d
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: neg_narrow_i32:
+; CHECK-GI:       // %bb.0:
+; CHECK-GI-NEXT:    mvn v0.16b, v0.16b
+; CHECK-GI-NEXT:    mvn v1.16b, v1.16b
+; CHECK-GI-NEXT:    shrn v0.2s, v0.2d, #32
+; CHECK-GI-NEXT:    shrn2 v0.4s, v1.2d, #32
+; CHECK-GI-NEXT:    ret
+  %not.i = xor <4 x i64> %a, splat (i64 -1)
+  %s = lshr <4 x i64> %not.i, splat (i64 32)
+  %vshrn_n = trunc nuw <4 x i64> %s to <4 x i32>
+  ret <4 x i32> %vshrn_n
+}
diff --git a/llvm/test/CodeGen/AArch64/arm64-vaddv.ll b/llvm/test/CodeGen/AArch64/arm64-vaddv.ll
index adfe28ea95893..d60b6d7097308 100644
--- a/llvm/test/CodeGen/AArch64/arm64-vaddv.ll
+++ b/llvm/test/CodeGen/AArch64/arm64-vaddv.ll
@@ -267,7 +267,6 @@ define <1 x i64> @test_vaddv_u64_to_vec(<2 x i64> %a1) {
 ; CHECK-LABEL: test_vaddv_u64_to_vec:
 ; CHECK:       // %bb.0: // %entry
 ; CHECK-NEXT:    addp d0, v0.2d
-; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $q0
 ; CHECK-NEXT:    ret
 entry:
   %vaddv.i = tail call i64 @llvm.aarch64.neon.uaddv.i64.v2i64(<2 x i64> %a1)
diff --git a/llvm/test/CodeGen/AArch64/arm64_32-addrs.ll b/llvm/test/CodeGen/AArch64/arm64_32-addrs.ll
index ad073d96a1483..cc70dc446324d 100644
--- a/llvm/test/CodeGen/AArch64/arm64_32-addrs.ll
+++ b/llvm/test/CodeGen/AArch64/arm64_32-addrs.ll
@@ -43,7 +43,6 @@ define i8 @test_valid_wrap_optimizable2(ptr %base, i32 %offset) {
 ; CHECK-LABEL: test_valid_wrap_optimizable2:
 ; CHECK:       ; %bb.0:
 ; CHECK-NEXT:    mov w8, #-100 ; =0xffffff9c
-; CHECK-NEXT:    ; kill: def $w1 killed $w1 def $x1
 ; CHECK-NEXT:    ldrb w0, [x8, w1, sxtw]
 ; CHECK-NEXT:    ret
 
diff --git a/llvm/test/CodeGen/AArch64/atomic-ops-msvc.ll b/llvm/test/CodeGen/AArch64/atomic-ops-msvc.ll
index fdb14606d463b..42cb3d4e9589d 100644
--- a/llvm/test/CodeGen/AArch64/atomic-ops-msvc.ll
+++ b/llvm/test/CodeGen/AArch64/atomic-ops-msvc.ll
@@ -374,7 +374,6 @@ define dso_local i64 @test_atomic_load_xor_i64(i64 %offset) nounwind {
 define dso_local i8 @test_atomic_load_xchg_i8(i8 %offset) nounwind {
 ; CHECK-LABEL: test_atomic_load_xchg_i8:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    // kill: def $w0 killed $w0 def $x0
 ; CHECK-NEXT:    adrp x9, var8
 ; CHECK-NEXT:    add x9, x9, :lo12:var8
 ; CHECK-NEXT:  .LBB20_1: // %atomicrmw.start
@@ -392,16 +391,16 @@ define dso_local i8 @test_atomic_load_xchg_i8(i8 %offset) nounwind {
 define dso_local i16 @test_atomic_load_xchg_i16(i16 %offset) nounwind {
 ; CHECK-LABEL: test_atomic_load_xchg_i16:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    // kill: def $w0 killed $w0 def $x0
+; CHECK-NEXT:    mov w8, w0
 ; CHECK-NEXT:    adrp x9, var16
 ; CHECK-NEXT:    add x9, x9, :lo12:var16
 ; CHECK-NEXT:  .LBB21_1: // %atomicrmw.start
 ; CHECK-NEXT:    // =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    ldaxrh w8, [x9]
-; CHECK-NEXT:    stlxrh w10, w0, [x9]
+; CHECK-NEXT:    ldaxrh w0, [x9]
+; CHECK-NEXT:    stlxrh w10, w8, [x9]
 ; CHECK-NEXT:    cbnz w10, .LBB21_1
 ; CHECK-NEXT:  // %bb.2: // %atomicrmw.end
-; CHECK-NEXT:    mov w0, w8
+; CHECK-NEXT:    // kill: def $w0 killed $w0 killed $x0
 ; CHECK-NEXT:    dmb ish
 ; CHECK-NEXT:    ret
    %old = atomicrmw xchg ptr @var16, i16 %offset seq_cst
@@ -763,7 +762,6 @@ define dso_local i64 @test_atomic_load_umax_i64(i64 %offset) nounwind {
 define dso_local i8 @test_atomic_cmpxchg_i8(i8 %wanted, i8 %new) nounwind {
 ; CHECK-LABEL: test_atomic_cmpxchg_i8:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    // kill: def $w1 killed $w1 def $x1
 ; CHECK-NEXT:    and w8, w0, #0xff
 ; CHECK-NEXT:    adrp x9, var8
 ; CHECK-NEXT:    add x9, x9, :lo12:var8
@@ -791,7 +789,6 @@ define dso_local i8 @test_atomic_cmpxchg_i8(i8 %wanted, i8 %new) nounwind {
 define dso_local i16 @test_atomic_cmpxchg_i16(i16 %wanted, i16 %new) nounwind {
 ; CHECK-LABEL: test_atomic_cmpxchg_i16:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    // kill: def $w1 killed $w1 def $x1
 ; CHECK-NEXT:    and w8, w0, #0xffff
 ; CHECK-NEXT:    adrp x9, var16
 ; CHECK-NEXT:    add x9, x9, :lo12:var16
diff --git a/llvm/test/CodeGen/AArch64/atomic-ops.ll b/llvm/test/CodeGen/AArch64/atomic-ops.ll
index 679065529090f..d8ac89f76b321 100644
--- a/llvm/test/CodeGen/AArch64/atomic-ops.ll
+++ b/llvm/test/CodeGen/AArch64/atomic-ops.ll
@@ -558,7 +558,6 @@ define dso_local i64 @test_atomic_load_xor_i64(i64 %offset) nounwind {
 define dso_local i8 @test_atomic_load_xchg_i8(i8 %offset) nounwind {
 ; INLINE_ATOMICS-LABEL: test_atomic_load_xchg_i8:
 ; INLINE_ATOMICS:       // %bb.0:
-; INLINE_ATOMICS-NEXT:    // kill: def $w0 killed $w0 def $x0
 ; INLINE_ATOMICS-NEXT:    adrp x9, var8
 ; INLINE_ATOMICS-NEXT:    add x9, x9, :lo12:var8
 ; INLINE_ATOMICS-NEXT:  .LBB20_1: // %atomicrmw.start
@@ -585,7 +584,6 @@ define dso_local i8 @test_atomic_load_xchg_i8(i8 %offset) nounwind {
 define dso_local i16 @test_atomic_load_xchg_i16(i16 %offset) nounwind {
 ; INLINE_ATOMICS-LABEL: test_atomic_load_xchg_i16:
 ; INLINE_ATOMICS:       // %bb.0:
-; INLINE_ATOMICS-NEXT:    // kill: def $w0 killed $w0 def $x0
 ; INLINE_ATOMICS-NEXT:    adrp x9, var16
 ; INLINE_ATOMICS-NEXT:    add x9, x9, :lo12:var16
 ; INLINE_ATOMICS-NEXT:  .LBB21_1: // %atomicrmw.start
@@ -978,7 +976,6 @@ define dso_local i64 @test_atomic_load_umax_i64(i64 %offset) nounwind {
 define dso_local i8 @test_atomic_cmpxchg_i8(i8 %wanted, i8 %new) nounwind {
 ; INLINE_ATOMICS-LABEL: test_atomic_cmpxchg_i8:
 ; INLINE_ATOMICS:       // %bb.0:
-; INLINE_ATOMICS-NEXT:    // kill: def $w1 killed $w1 def $x1
 ; INLINE_ATOMICS-NEXT:    and w8, w0, #0xff
 ; INLINE_ATOMICS-NEXT:    adrp x9, var8
 ; INLINE_ATOMICS-NEXT:    add x9, x9, :lo12:var8
@@ -1015,7 +1012,6 @@ define dso_local i8 @test_atomic_cmpxchg_i8(i8 %wanted, i8 %new) nounwind {
 define dso_local i16 @test_atomic_cmpxchg_i16(i16 %wanted, i16 %new) nounwind {
 ; INLINE_ATOMICS-LABEL: test_atomic_cmpxchg_i16:
 ; INLINE_ATOMICS:       // %bb.0:
-; INLINE_ATOMICS-NEXT:    // kill: def $w1 killed $w1 def $x1
 ; INLINE_ATOMICS-NEXT:    and w8, w0, #0xffff
 ; INLINE_ATOMICS-NEXT:    adrp x9, var16
 ; INLINE_ATOMICS-NEXT:    add x9, x9, :lo12:var16
diff --git a/llvm/test/CodeGen/AArch64/atomicrmw-fadd.ll b/llvm/test/CodeGen/AArch64/atomicrmw-fadd.ll
index fb40dfcbe101d..0c3a40d93d640 100644
--- a/llvm/test/CodeGen/AArch64/atomicrmw-fadd.ll
+++ b/llvm/test/CodeGen/AArch64/atomicrmw-fadd.ll
@@ -476,40 +476,38 @@ define double @test_atomicrmw_fadd_f32_seq_cst_align8(ptr %ptr, double %value) #
 ; SOFTFP-NOLSE-LABEL: test_atomicrmw_fadd_f32_seq_cst_align8:
 ; SOFTFP-NOLSE:       // %bb.0:
 ; SOFTFP-NOLSE-NEXT:    stp x30, x21, [sp, #-32]! // 16-byte Folded Spill
-; SOFTFP-NOLSE-NEXT:    ldr x21, [x0]
 ; SOFTFP-NOLSE-NEXT:    stp x20, x19, [sp, #16] // 16-byte Folded Spill
 ; SOFTFP-NOLSE-NEXT:    mov x19, x0
+; SOFTFP-NOLSE-NEXT:    ldr x0, [x0]
 ; SOFTFP-NOLSE-NEXT:    mov x20, x1
 ; SOFTFP-NOLSE-NEXT:    b .LBB5_2
 ; SOFTFP-NOLSE-NEXT:  .LBB5_1: // %cmpxchg.nostore
 ; SOFTFP-NOLSE-NEXT:    // in Loop: Header=BB5_2 Depth=1
-; SOFTFP-NOLSE-NEXT:    mov w9, wzr
+; SOFTFP-NOLSE-NEXT:    mov w8, wzr
 ; SOFTFP-NOLSE-NEXT:    clrex
-; SOFTFP-NOLSE-NEXT:    mov x21, x8
-; SOFTFP-NOLSE-NEXT:    cbnz w9, .LBB5_6
+; SOFTFP-NOLSE-NEXT:    cbnz w8, .LBB5_6
 ; SOFTFP-NOLSE-NEXT:  .LBB5_2: // %atomicrmw.start
 ; SOFTFP-NOLSE-NEXT:    // =>This Loop Header: Depth=1
 ; SOFTFP-NOLSE-NEXT:    // Child Loop BB5_3 Depth 2
-; SOFTFP-NOLSE-NEXT:    mov x0, x21
 ; SOFTFP-NOLSE-NEXT:    mov x1, x20
+; SOFTFP-NOLSE-NEXT:    mov x21, x0
 ; SOFTFP-NOLSE-NEXT:    bl __adddf3
+; SOFTFP-NOLSE-NEXT:    mov x8, x0
 ; SOFTFP-NOLSE-NEXT:  .LBB5_3: // %cmpxchg.start
 ; SOFTFP-NOLSE-NEXT:    // Parent Loop BB5_2 Depth=1
 ; SOFTFP-NOLSE-NEXT:    // => This Inner Loop Header: Depth=2
-; SOFTFP-NOLSE-NEXT:    ldaxr x8, [x19]
-; SOFTFP-NOLSE-NEXT:    cmp x8, x21
+; SOFTFP-NOLSE-NEXT:    ldaxr x0, [x19]
+; SOFTFP-NOLSE-NEXT:    cmp x0, x21
 ; SOFTFP-NOLSE-NEXT:    b.ne .LBB5_1
 ; SOFTFP-NOLSE-NEXT:  // %bb.4: // %cmpxchg.trystore
 ; SOFTFP-NOLSE-NEXT:    // in Loop: Header=BB5_3 Depth=2
-; SOFTFP-NOLSE-NEXT:    stlxr w9, x0, [x19]
+; SOFTFP-NOLSE-NEXT:    stlxr w9, x8, [x19]
 ; SOFTFP-NOLSE-NEXT:    cbnz w9, .LBB5_3
 ; SOFTFP-NOLSE-NEXT:  // %bb.5: // in Loop: Header=BB5_2 Depth=1
-; SOFTFP-NOLSE-NEXT:    mov w9, #1 // =0x1
-; SOFTFP-NOLSE-NEXT:    mov x21, x8
-; SOFTFP-NOLSE-NEXT:    cbz w9, .LBB5_2
+; SOFTFP-NOLSE-NEXT:    mov w8, #1 // =0x1
+; SOFTFP-NOLSE-NEXT:    cbz w8, .LBB5_2
 ; SOFTFP-NOLSE-NEXT:  .LBB5_6: // %atomicrmw.end
 ; SOFTFP-NOLSE-NEXT:    ldp x20, x19, [sp, #16] // 16-byte Folded Reload
-; SOFTFP-NOLSE-NEXT:    mov x0, x21
 ; SOFTFP-NOLSE-NEXT:    ldp x30, x21, [sp], #32 // 16-byte Folded Reload
 ; SOFTFP-NOLSE-NEXT:    ret
   %res = atomicrmw fadd ptr %ptr, double %value seq_cst, align 8
diff --git a/llvm/test/CodeGen/AArch64/atomicrmw-fmax.ll b/llvm/test/CodeGen/AArch64/atomicrmw-fmax.ll
index 818dcf3a0b487..24088998f36d1 100644
--- a/llvm/test/CodeGen/AArch64/atomicrmw-fmax.ll
+++ b/llvm/test/CodeGen/AArch64/atomicrmw-fmax.ll
@@ -478,40 +478,38 @@ define double @test_atomicrmw_fmax_f32_seq_cst_align8(ptr %ptr, double %value) #
 ; SOFTFP-NOLSE-LABEL: test_atomicrmw_fmax_f32_seq_cst_align8:
 ; SOFTFP-NOLSE:       // %bb.0:
 ; SOFTFP-NOLSE-NEXT:    stp x30, x21, [sp, #-32]! // 16-byte Folded Spill
-; SOFTFP-NOLSE-NEXT:    ldr x21, [x0]
 ; SOFTFP-NOLSE-NEXT:    stp x20, x19, [sp, #16] // 16-byte Folded Spill
 ; SOFTFP-NOLSE-NEXT:    mov x19, x0
+; SOFTFP-NOLSE-NEXT:    ldr x0, [x0]
 ; SOFTFP-NOLSE-NEXT:    mov x20, x1
 ; SOFTFP-NOLSE-NEXT:    b .LBB5_2
 ; SOFTFP-NOLSE-NEXT:  .LBB5_1: // %cmpxchg.nostore
 ; SOFTFP-NOLSE-NEXT:    // in Loop: Header=BB5_2 Depth=1
-; SOFTFP-NOLSE-NEXT:    mov w9, wzr
+; SOFTFP-NOLSE-NEXT:    mov w8, wzr
 ; SOFTFP-NOLSE-NEXT:    clrex
-; SOFTFP-NOLSE-NEXT:    mov x21, x8
-; SOFTFP-NOLSE-NEXT:    cbnz w9, .LBB5_6
+; SOFTFP-NOLSE-NEXT:    cbnz w8, .LBB5_6
 ; SOFTFP-NOLSE-NEXT:  .LBB5_2: // %atomicrmw.start
 ; SOFTFP-NOLSE-NEXT:    // =>This Loop Header: Depth=1
 ; SOFTFP-NOLSE-NEXT:    // Child Loop BB5_3 Depth 2
-; SOFTFP-NOLSE-NEXT:    mov x0, x21
 ; SOFTFP-NOLSE-NEXT:    mov x1, x20
+; SOFTFP-NOLSE-NEXT:    mov x21, x0
 ; SOFTFP-NOLSE-NEXT:    bl fmax
+; SOFTFP-NOLSE-NEXT:    mov x8, x0
 ; SOFTFP-NOLSE-NEXT:  .LBB5_3: // %cmpxchg.start
 ; SOFTFP-NOLSE-NEXT:    // Parent Loop BB5_2 Depth=1
 ; SOFTFP-NOLSE-NEXT:    // => This Inner Loop Header: Depth=2
-; SOFTFP-NOLSE-NEXT:    ldaxr x8, [x19]
-; SOFTFP-NOLSE-NEXT:    cmp x8, x21
+; SOFTFP-NOLSE-NEXT:    ldaxr x0, [x19]
+; SOFTFP-NOLSE-NEXT:    cmp x0, x21
 ; SOFTFP-NOLSE-NEXT:    b.ne .LBB5_1
 ; SOFTFP-NOLSE-NEXT:  // %bb.4: // %cmpxchg.trystore
 ; SOFTFP-NOLSE-NEXT:    // in Loop: Header=BB5_3 Depth=2
-; SOFTFP-NOLSE-NEXT:    stlxr w9, x0, [x19]
+; SOFTFP-NOLSE-NEXT:    stlxr w9, x8, [x19]
 ; SOFTFP-NOLSE-NEXT:    cbnz w9, .LBB5_3
 ; SOFTFP-NOLSE-NEXT:  // %bb.5: // in Loop: Header=BB5_2 Depth=1
-; SOFTFP-NOLSE-NEXT:    mov w9, #1 // =0x1
-; SOFTFP-NOLSE-NEXT:    mov x21, x8
-; SOFTFP-NOLSE-NEXT:    cbz w9, .LBB5_2
+; SOFTFP-NOLSE-NEXT:    mov w8, #1 // =0x1
+; SOFTFP-NOLSE-NEXT:    cbz w8, .LBB5_2
 ; SOFTFP-NOLSE-NEXT:  .LBB5_6: // %atomicrmw.end
 ; SOFTFP-NOLSE-NEXT:    ldp x20, x19, [sp, #16] // 16-byte Folded Reload
-; SOFTFP-NOLSE-NEXT:    mov x0, x21
 ; SOFTFP-NOLSE-NEXT:    ldp x30, x21, [sp], #32 // 16-byte Folded Reload
 ; SOFTFP-NOLSE-NEXT:    ret
   %res = atomicrmw fmax ptr %ptr, double %value seq_cst, align 8
diff --git a/llvm/test/CodeGen/AArch64/atomicrmw-fmin.ll b/llvm/test/CodeGen/AArch64/atomicrmw-fmin.ll
index b969241e8bf90..65f1f4863c173 100644
--- a/llvm/test/CodeGen/AArch64/atomicrmw-fmin.ll
+++ b/llvm/test/CodeGen/AArch64/atomicrmw-fmin.ll
@@ -478,40 +478,38 @@ define double @test_atomicrmw_fmin_f32_seq_cst_align8(ptr %ptr, double %value) #
 ; SOFTFP-NOLSE-LABEL: test_atomicrmw_fmin_f32_seq_cst_align8:
 ; SOFTFP-NOLSE:       // %bb.0:
 ; SOFTFP-NOLSE-NEXT:    stp x30, x21, [sp, #-32]! // 16-byte Folded Spill
-; SOFTFP-NOLSE-NEXT:    ldr x21, [x0]
 ; SOFTFP-NOLSE-NEXT:    stp x20, x19, [sp, #16] // 16-byte Folded Spill
 ; SOFTFP-NOLSE-NEXT:    mov x19, x0
+; SOFTFP-NOLSE-NEXT:    ldr x0, [x0]
 ; SOFTFP-NOLSE-NEXT:    mov x20, x1
 ; SOFTFP-NOLSE-NEXT:    b .LBB5_2
 ; SOFTFP-NOLSE-NEXT:  .LBB5_1: // %cmpxchg.nostore
 ; SOFTFP-NOLSE-NEXT:    // in Loop: Header=BB5_2 Depth=1
-; SOFTFP-NOLSE-NEXT:    mov w9, wzr
+; SOFTFP-NOLSE-NEXT:    mov w8, wzr
 ; SOFTFP-NOLSE-NEXT:    clrex
-; SOFTFP-NOLSE-NEXT:    mov x21, x8
-; SOFTFP-NOLSE-NEXT:    cbnz w9, .LBB5_6
+; SOFTFP-NOLSE-NEXT:    cbnz w8, .LBB5_6
 ; SOFTFP-NOLSE-NEXT:  .LBB5_2: // %atomicrmw.start
 ; SOFTFP-NOLSE-NEXT:    // =>This Loop Header: Depth=1
 ; SOFTFP-NOLSE-NEXT:    // Child Loop BB5_3 Depth 2
-; SOFTFP-NOLSE-NEXT:    mov x0, x21
 ; SOFTFP-NOLSE-NEXT:    mov x1, x20
+; SOFTFP-NOLSE-NEXT:    mov x21, x0
 ; SOFTFP-NOLSE-NEXT:    bl fmin
+; SOFTFP-NOLSE-NEXT:    mov x8, x0
 ; SOFTFP-NOLSE-NEXT:  .LBB5_3: // %cmpxchg.start
 ; SOFTFP-NOLSE-NEXT:    // Parent Loop BB5_2 Depth=1
 ; SOFTFP-NOLSE-NEXT:    // => This Inner Loop Header: Depth=2
-; SOFTFP-NOLSE-NEXT:    ldaxr x8, [x19]
-; SOFTFP-NOLSE-NEXT:    cmp x8, x21
+; SOFTFP-NOLSE-NEXT:    ldaxr x0, [x19]
+; SOFTFP-NOLSE-NEXT:    cmp x0, x21
 ; SOFTFP-NOLSE-NEXT:    b.ne .LBB5_1
 ; SOFTFP-NOLSE-NEXT:  // %bb.4: // %cmpxchg.trystore
 ; SOFTFP-NOLSE-NEXT:    // in Loop: Header=BB5_3 Depth=2
-; SOFTFP-NOLSE-NEXT:    stlxr w9, x0, [x19]
+; SOFTFP-NOLSE-NEXT:    stlxr w9, x8, [x19]
 ; SOFTFP-NOLSE-NEXT:    cbnz w9, .LBB5_3
 ; SOFTFP-NOLSE-NEXT:  // %bb.5: // in Loop: Header=BB5_2 Depth=1
-; SOFTFP-NOLSE-NEXT:    mov w9, #1 // =0x1
-; SOFTFP-NOLSE-NEXT:    mov x21, x8
-; SOFTFP-NOLSE-NEXT:    cbz w9, .LBB5_2
+; SOFTFP-NOLSE-NEXT:    mov w8, #1 // =0x1
+; SOFTFP-NOLSE-NEXT:    cbz w8, .LBB5_2
 ; SOFTFP-NOLSE-NEXT:  .LBB5_6: // %atomicrmw.end
 ; SOFTFP-NOLSE-NEXT:    ldp x20, x19, [sp, #16] // 16-byte Folded Reload
-; SOFTFP-NOLSE-NEXT:    mov x0, x21
 ; SOFTFP-NOLSE-NEXT:    ldp x30, x21, [sp], #32 // 16-byte Folded Reload
 ; SOFTFP-NOLSE-NEXT:    ret
   %res = atomicrmw fmin ptr %ptr, double %value seq_cst, align 8
diff --git a/llvm/test/CodeGen/AArch64/atomicrmw-fsub.ll b/llvm/test/CodeGen/AArch64/atomicrmw-fsub.ll
index e603337e7a569..0f1a2f03c98c3 100644
--- a/llvm/test/CodeGen/AArch64/atomicrmw-fsub.ll
+++ b/llvm/test/CodeGen/AArch64/atomicrmw-fsub.ll
@@ -476,40 +476,38 @@ define double @test_atomicrmw_fsub_f32_seq_cst_align8(ptr %ptr, double %value) #
 ; SOFTFP-NOLSE-LABEL: test_atomicrmw_fsub_f32_seq_cst_align8:
 ; SOFTFP-NOLSE:       // %bb.0:
 ; SOFTFP-NOLSE-NEXT:    stp x30, x21, [sp, #-32]! // 16-byte Folded Spill
-; SOFTFP-NOLSE-NEXT:    ldr x21, [x0]
 ; SOFTFP-NOLSE-NEXT:    stp x20, x19, [sp, #16] // 16-byte Folded Spill
 ; SOFTFP-NOLSE-NEXT:    mov x19, x0
+; SOFTFP-NOLSE-NEXT:    ldr x0, [x0]
 ; SOFTFP-NOLSE-NEXT:    mov x20, x1
 ; SOFTFP-NOLSE-NEXT:    b .LBB5_2
 ; SOFTFP-NOLSE-NEXT:  .LBB5_1: // %cmpxchg.nostore
 ; SOFTFP-NOLSE-NEXT:    // in Loop: Header=BB5_2 Depth=1
-; SOFTFP-NOLSE-NEXT:    mov w9, wzr
+; SOFTFP-NOLSE-NEXT:    mov w8, wzr
 ; SOFTFP-NOLSE-NEXT:    clrex
-; SOFTFP-NOLSE-NEXT:    mov x21, x8
-; SOFTFP-NOLSE-NEXT:    cbnz w9, .LBB5_6
+; SOFTFP-NOLSE-NEXT:    cbnz w8, .LBB5_6
 ; SOFTFP-NOLSE-NEXT:  .LBB5_2: // %atomicrmw.start
 ; SOFTFP-NOLSE-NEXT:    // =>This Loop Header: Depth=1
 ; SOFTFP-NOLSE-NEXT:    // Child Loop BB5_3 Depth 2
-; SOFTFP-NOLSE-NEXT:    mov x0, x21
 ; SOFTFP-NOLSE-NEXT:    mov x1, x20
+; SOFTFP-NOLSE-NEXT:    mov x21, x0
 ; SOFTFP-NOLSE-NEXT:    bl __subdf3
+; SOFTFP-NOLSE-NEXT:    mov x8, x0
 ; SOFTFP-NOLSE-NEXT:  .LBB5_3: // %cmpxchg.start
 ; SOFTFP-NOLSE-NEXT:    // Parent Loop BB5_2 Depth=1
 ; SOFTFP-NOLSE-NEXT:    // => This Inner Loop Header: Depth=2
-; SOFTFP-NOLSE-NEXT:    ldaxr x8, [x19]
-; SOFTFP-NOLSE-NEXT:    cmp x8, x21
+; SOFTFP-NOLSE-NEXT:    ldaxr x0, [x19]
+; SOFTFP-NOLSE-NEXT:    cmp x0, x21
 ; SOFTFP-NOLSE-NEXT:    b.ne .LBB5_1
 ; SOFTFP-NOLSE-NEXT:  // %bb.4: // %cmpxchg.trystore
 ; SOFTFP-NOLSE-NEXT:    // in Loop: Header=BB5_3 Depth=2
-; SOFTFP-NOLSE-NEXT:    stlxr w9, x0, [x19]
+; SOFTFP-NOLSE-NEXT:    stlxr w9, x8, [x19]
 ; SOFTFP-NOLSE-NEXT:    cbnz w9, .LBB5_3
 ; SOFTFP-NOLSE-NEXT:  // %bb.5: // in Loop: Header=BB5_2 Depth=1
-; SOFTFP-NOLSE-NEXT:    mov w9, #1 // =0x1
-; SOFTFP-NOLSE-NEXT:    mov x21, x8
-; SOFTFP-NOLSE-NEXT:    cbz w9, .LBB5_2
+; SOFTFP-NOLSE-NEXT:    mov w8, #1 // =0x1
+; SOFTFP-NOLSE-NEXT:    cbz w8, .LBB5_2
 ; SOFTFP-NOLSE-NEXT:  .LBB5_6: // %atomicrmw.end
 ; SOFTFP-NOLSE-NEXT:    ldp x20, x19, [sp, #16] // 16-byte Folded Reload
-; SOFTFP-NOLSE-NEXT:    mov x0, x21
 ; SOFTFP-NOLSE-NEXT:    ldp x30, x21, [sp], #32 // 16-byte Folded Reload
 ; SOFTFP-NOLSE-NEXT:    ret
   %res = atomicrmw fsub ptr %ptr, double %value seq_cst, align 8
diff --git a/llvm/test/CodeGen/AArch64/atomicrmw-xchg-fp.ll b/llvm/test/CodeGen/AArch64/atomicrmw-xchg-fp.ll
index 98033a8e449ff..0e728cdb895ed 100644
--- a/llvm/test/CodeGen/AArch64/atomicrmw-xchg-fp.ll
+++ b/llvm/test/CodeGen/AArch64/atomicrmw-xchg-fp.ll
@@ -1,4 +1,4 @@
-; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --force-update
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -verify-machineinstrs -mtriple=aarch64-- -O1 -fast-isel=0 -global-isel=false %s -o - | FileCheck %s -check-prefix=NOLSE
 ; RUN: llc -verify-machineinstrs -mtriple=aarch64-- -mattr=+lse -O1 -fast-isel=0 -global-isel=false %s -o - | FileCheck %s -check-prefix=LSE
 
@@ -6,14 +6,14 @@ define half @test_rmw_xchg_f16(ptr %dst, half %new) {
 ; NOLSE-LABEL: test_rmw_xchg_f16:
 ; NOLSE:       // %bb.0:
 ; NOLSE-NEXT:    // kill: def $h0 killed $h0 def $s0
-; NOLSE-NEXT:    fmov w8, s0
+; NOLSE-NEXT:    fmov w9, s0
 ; NOLSE-NEXT:  .LBB0_1: // %atomicrmw.start
 ; NOLSE-NEXT:    // =>This Inner Loop Header: Depth=1
-; NOLSE-NEXT:    ldaxrh w9, [x0]
-; NOLSE-NEXT:    stlxrh w10, w8, [x0]
+; NOLSE-NEXT:    ldaxrh w8, [x0]
+; NOLSE-NEXT:    stlxrh w10, w9, [x0]
 ; NOLSE-NEXT:    cbnz w10, .LBB0_1
 ; NOLSE-NEXT:  // %bb.2: // %atomicrmw.end
-; NOLSE-NEXT:    fmov s0, w9
+; NOLSE-NEXT:    fmov s0, w8
 ; NOLSE-NEXT:    // kill: def $h0 killed $h0 killed $s0
 ; NOLSE-NEXT:    ret
 ;
diff --git a/llvm/test/CodeGen/AArch64/avoid-zero-copy.mir b/llvm/test/CodeGen/AArch64/avoid-zero-copy.mir
index b940734c6988c..0a915aa635737 100644
--- a/llvm/test/CodeGen/AArch64/avoid-zero-copy.mir
+++ b/llvm/test/CodeGen/AArch64/avoid-zero-copy.mir
@@ -1,6 +1,8 @@
 # Check that we can remove the redundant save of constant registers such as $wzr
 # RUN: llc -mtriple=aarch64-unknown-linux %s -verify-machineinstrs -start-before=machine-cp -o - | FileCheck %s --check-prefix ASM
 # RUN: llc -mtriple=aarch64-unknown-linux %s -verify-machineinstrs -run-pass=machine-cp -o - | FileCheck %s
+
+# RUN: llc -mtriple=aarch64-unknown-linux %s -passes=machine-cp -o - | FileCheck %s
 --- |
   target triple = "aarch64-unknown-linux"
   declare i32 @bar(i32) nounwind
diff --git a/llvm/test/CodeGen/AArch64/bf16-instructions.ll b/llvm/test/CodeGen/AArch64/bf16-instructions.ll
index ecf64ecbbd3ff..2fc9c53112ab6 100644
--- a/llvm/test/CodeGen/AArch64/bf16-instructions.ll
+++ b/llvm/test/CodeGen/AArch64/bf16-instructions.ll
@@ -3,7 +3,34 @@
 ; RUN: llc < %s -mtriple aarch64-unknown-unknown -mattr=+bf16 | FileCheck %s --check-prefixes=CHECK,CHECK-BF16,CHECK-SD
 ; RUN: llc < %s -mtriple aarch64-unknown-unknown -mattr=+bf16,+fullfp16 -global-isel -global-isel-abort=2 2>&1 | FileCheck %s --check-prefixes=CHECK,CHECK-BF16,CHECK-GI
 
-; CHECK-GI:       warning: Instruction selection used fallback path for test_fptosi_i32
+; CHECK-GI:       warning: Instruction selection used fallback path for test_fadd
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for test_fsub
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for test_fmul
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for test_fmadd
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for test_fdiv
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for test_frem
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for test_call
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for test_call_flipped
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for test_tailcall_flipped
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for test_select_cc
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for test_select_cc_f32_f16
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for test_fcmp_une
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for test_fcmp_ueq
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for test_fcmp_ugt
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for test_fcmp_uge
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for test_fcmp_ult
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for test_fcmp_ule
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for test_fcmp_uno
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for test_fcmp_one
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for test_fcmp_oeq
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for test_fcmp_ogt
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for test_fcmp_oge
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for test_fcmp_olt
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for test_fcmp_ole
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for test_fcmp_ord
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for test_fccmp
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for test_br_cc
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for test_fptosi_i32
 ; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for test_fptosi_i64
 ; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for test_fptoui_i32
 ; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for test_fptoui_i64
@@ -17,9 +44,40 @@
 ; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for test_fptrunc_double
 ; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for test_fpext_float
 ; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for test_fpext_double
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for test_sqrt
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for test_powi
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for test_sin
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for test_cos
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for test_tan
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for test_acos
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for test_asin
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for test_atan
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for test_atan2
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for test_cosh
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for test_sinh
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for test_tanh
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for test_pow
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for test_exp
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for test_exp2
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for test_log
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for test_log10
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for test_log2
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for test_fma
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for test_fabs
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for test_minnum
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for test_maxnum
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for test_copysign
 ; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for test_copysign_f32
 ; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for test_copysign_f64
 ; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for test_copysign_extended
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for test_floor
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for test_ceil
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for test_trunc
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for test_rint
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for test_nearbyint
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for test_round
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for test_roundeven
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for test_fmuladd
 
 define bfloat @test_fadd(bfloat %a, bfloat %b) #0 {
 ; CHECK-CVT-LABEL: test_fadd:
@@ -39,20 +97,15 @@ define bfloat @test_fadd(bfloat %a, bfloat %b) #0 {
 ; CHECK-CVT-NEXT:    // kill: def $h0 killed $h0 killed $s0
 ; CHECK-CVT-NEXT:    ret
 ;
-; CHECK-SD-LABEL: test_fadd:
-; CHECK-SD:       // %bb.0:
-; CHECK-SD-NEXT:    // kill: def $h1 killed $h1 def $d1
-; CHECK-SD-NEXT:    // kill: def $h0 killed $h0 def $d0
-; CHECK-SD-NEXT:    shll v1.4s, v1.4h, #16
-; CHECK-SD-NEXT:    shll v0.4s, v0.4h, #16
-; CHECK-SD-NEXT:    fadd s0, s0, s1
-; CHECK-SD-NEXT:    bfcvt h0, s0
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: test_fadd:
-; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    fadd h0, h0, h1
-; CHECK-GI-NEXT:    ret
+; CHECK-BF16-LABEL: test_fadd:
+; CHECK-BF16:       // %bb.0:
+; CHECK-BF16-NEXT:    // kill: def $h1 killed $h1 def $d1
+; CHECK-BF16-NEXT:    // kill: def $h0 killed $h0 def $d0
+; CHECK-BF16-NEXT:    shll v1.4s, v1.4h, #16
+; CHECK-BF16-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-BF16-NEXT:    fadd s0, s0, s1
+; CHECK-BF16-NEXT:    bfcvt h0, s0
+; CHECK-BF16-NEXT:    ret
   %r = fadd bfloat %a, %b
   ret bfloat %r
 }
@@ -75,20 +128,15 @@ define bfloat @test_fsub(bfloat %a, bfloat %b) #0 {
 ; CHECK-CVT-NEXT:    // kill: def $h0 killed $h0 killed $s0
 ; CHECK-CVT-NEXT:    ret
 ;
-; CHECK-SD-LABEL: test_fsub:
-; CHECK-SD:       // %bb.0:
-; CHECK-SD-NEXT:    // kill: def $h1 killed $h1 def $d1
-; CHECK-SD-NEXT:    // kill: def $h0 killed $h0 def $d0
-; CHECK-SD-NEXT:    shll v1.4s, v1.4h, #16
-; CHECK-SD-NEXT:    shll v0.4s, v0.4h, #16
-; CHECK-SD-NEXT:    fsub s0, s0, s1
-; CHECK-SD-NEXT:    bfcvt h0, s0
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: test_fsub:
-; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    fsub h0, h0, h1
-; CHECK-GI-NEXT:    ret
+; CHECK-BF16-LABEL: test_fsub:
+; CHECK-BF16:       // %bb.0:
+; CHECK-BF16-NEXT:    // kill: def $h1 killed $h1 def $d1
+; CHECK-BF16-NEXT:    // kill: def $h0 killed $h0 def $d0
+; CHECK-BF16-NEXT:    shll v1.4s, v1.4h, #16
+; CHECK-BF16-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-BF16-NEXT:    fsub s0, s0, s1
+; CHECK-BF16-NEXT:    bfcvt h0, s0
+; CHECK-BF16-NEXT:    ret
   %r = fsub bfloat %a, %b
   ret bfloat %r
 }
@@ -111,20 +159,15 @@ define bfloat @test_fmul(bfloat %a, bfloat %b) #0 {
 ; CHECK-CVT-NEXT:    // kill: def $h0 killed $h0 killed $s0
 ; CHECK-CVT-NEXT:    ret
 ;
-; CHECK-SD-LABEL: test_fmul:
-; CHECK-SD:       // %bb.0:
-; CHECK-SD-NEXT:    // kill: def $h1 killed $h1 def $d1
-; CHECK-SD-NEXT:    // kill: def $h0 killed $h0 def $d0
-; CHECK-SD-NEXT:    shll v1.4s, v1.4h, #16
-; CHECK-SD-NEXT:    shll v0.4s, v0.4h, #16
-; CHECK-SD-NEXT:    fmul s0, s0, s1
-; CHECK-SD-NEXT:    bfcvt h0, s0
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: test_fmul:
-; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    fmul h0, h0, h1
-; CHECK-GI-NEXT:    ret
+; CHECK-BF16-LABEL: test_fmul:
+; CHECK-BF16:       // %bb.0:
+; CHECK-BF16-NEXT:    // kill: def $h1 killed $h1 def $d1
+; CHECK-BF16-NEXT:    // kill: def $h0 killed $h0 def $d0
+; CHECK-BF16-NEXT:    shll v1.4s, v1.4h, #16
+; CHECK-BF16-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-BF16-NEXT:    fmul s0, s0, s1
+; CHECK-BF16-NEXT:    bfcvt h0, s0
+; CHECK-BF16-NEXT:    ret
   %r = fmul bfloat %a, %b
   ret bfloat %r
 }
@@ -157,25 +200,20 @@ define bfloat @test_fmadd(bfloat %a, bfloat %b, bfloat %c) #0 {
 ; CHECK-CVT-NEXT:    // kill: def $h0 killed $h0 killed $s0
 ; CHECK-CVT-NEXT:    ret
 ;
-; CHECK-SD-LABEL: test_fmadd:
-; CHECK-SD:       // %bb.0:
-; CHECK-SD-NEXT:    // kill: def $h1 killed $h1 def $d1
-; CHECK-SD-NEXT:    // kill: def $h0 killed $h0 def $d0
-; CHECK-SD-NEXT:    // kill: def $h2 killed $h2 def $d2
-; CHECK-SD-NEXT:    shll v1.4s, v1.4h, #16
-; CHECK-SD-NEXT:    shll v0.4s, v0.4h, #16
-; CHECK-SD-NEXT:    fmul s0, s0, s1
-; CHECK-SD-NEXT:    shll v1.4s, v2.4h, #16
-; CHECK-SD-NEXT:    bfcvt h0, s0
-; CHECK-SD-NEXT:    shll v0.4s, v0.4h, #16
-; CHECK-SD-NEXT:    fadd s0, s0, s1
-; CHECK-SD-NEXT:    bfcvt h0, s0
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: test_fmadd:
-; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    fmadd h0, h0, h1, h2
-; CHECK-GI-NEXT:    ret
+; CHECK-BF16-LABEL: test_fmadd:
+; CHECK-BF16:       // %bb.0:
+; CHECK-BF16-NEXT:    // kill: def $h1 killed $h1 def $d1
+; CHECK-BF16-NEXT:    // kill: def $h0 killed $h0 def $d0
+; CHECK-BF16-NEXT:    // kill: def $h2 killed $h2 def $d2
+; CHECK-BF16-NEXT:    shll v1.4s, v1.4h, #16
+; CHECK-BF16-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-BF16-NEXT:    fmul s0, s0, s1
+; CHECK-BF16-NEXT:    shll v1.4s, v2.4h, #16
+; CHECK-BF16-NEXT:    bfcvt h0, s0
+; CHECK-BF16-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-BF16-NEXT:    fadd s0, s0, s1
+; CHECK-BF16-NEXT:    bfcvt h0, s0
+; CHECK-BF16-NEXT:    ret
   %mul = fmul fast bfloat %a, %b
   %r = fadd fast bfloat %mul, %c
   ret bfloat %r
@@ -199,20 +237,15 @@ define bfloat @test_fdiv(bfloat %a, bfloat %b) #0 {
 ; CHECK-CVT-NEXT:    // kill: def $h0 killed $h0 killed $s0
 ; CHECK-CVT-NEXT:    ret
 ;
-; CHECK-SD-LABEL: test_fdiv:
-; CHECK-SD:       // %bb.0:
-; CHECK-SD-NEXT:    // kill: def $h1 killed $h1 def $d1
-; CHECK-SD-NEXT:    // kill: def $h0 killed $h0 def $d0
-; CHECK-SD-NEXT:    shll v1.4s, v1.4h, #16
-; CHECK-SD-NEXT:    shll v0.4s, v0.4h, #16
-; CHECK-SD-NEXT:    fdiv s0, s0, s1
-; CHECK-SD-NEXT:    bfcvt h0, s0
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: test_fdiv:
-; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    fdiv h0, h0, h1
-; CHECK-GI-NEXT:    ret
+; CHECK-BF16-LABEL: test_fdiv:
+; CHECK-BF16:       // %bb.0:
+; CHECK-BF16-NEXT:    // kill: def $h1 killed $h1 def $d1
+; CHECK-BF16-NEXT:    // kill: def $h0 killed $h0 def $d0
+; CHECK-BF16-NEXT:    shll v1.4s, v1.4h, #16
+; CHECK-BF16-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-BF16-NEXT:    fdiv s0, s0, s1
+; CHECK-BF16-NEXT:    bfcvt h0, s0
+; CHECK-BF16-NEXT:    ret
   %r = fdiv bfloat %a, %b
   ret bfloat %r
 }
@@ -239,29 +272,19 @@ define bfloat @test_frem(bfloat %a, bfloat %b) #0 {
 ; CHECK-CVT-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
 ; CHECK-CVT-NEXT:    ret
 ;
-; CHECK-SD-LABEL: test_frem:
-; CHECK-SD:       // %bb.0:
-; CHECK-SD-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
-; CHECK-SD-NEXT:    // kill: def $h1 killed $h1 def $d1
-; CHECK-SD-NEXT:    // kill: def $h0 killed $h0 def $d0
-; CHECK-SD-NEXT:    shll v0.4s, v0.4h, #16
-; CHECK-SD-NEXT:    shll v1.4s, v1.4h, #16
-; CHECK-SD-NEXT:    // kill: def $s0 killed $s0 killed $q0
-; CHECK-SD-NEXT:    // kill: def $s1 killed $s1 killed $q1
-; CHECK-SD-NEXT:    bl fmodf
-; CHECK-SD-NEXT:    bfcvt h0, s0
-; CHECK-SD-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: test_frem:
-; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
-; CHECK-GI-NEXT:    fcvt s0, h0
-; CHECK-GI-NEXT:    fcvt s1, h1
-; CHECK-GI-NEXT:    bl fmodf
-; CHECK-GI-NEXT:    bfcvt h0, s0
-; CHECK-GI-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
-; CHECK-GI-NEXT:    ret
+; CHECK-BF16-LABEL: test_frem:
+; CHECK-BF16:       // %bb.0:
+; CHECK-BF16-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-BF16-NEXT:    // kill: def $h1 killed $h1 def $d1
+; CHECK-BF16-NEXT:    // kill: def $h0 killed $h0 def $d0
+; CHECK-BF16-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-BF16-NEXT:    shll v1.4s, v1.4h, #16
+; CHECK-BF16-NEXT:    // kill: def $s0 killed $s0 killed $q0
+; CHECK-BF16-NEXT:    // kill: def $s1 killed $s1 killed $q1
+; CHECK-BF16-NEXT:    bl fmodf
+; CHECK-BF16-NEXT:    bfcvt h0, s0
+; CHECK-BF16-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-BF16-NEXT:    ret
   %r = frem bfloat %a, %b
   ret bfloat %r
 }
@@ -385,14 +408,12 @@ define bfloat @test_select_cc(bfloat %a, bfloat %b, bfloat %c, bfloat %d) #0 {
 ;
 ; CHECK-GI-LABEL: test_select_cc:
 ; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    // kill: def $h0 killed $h0 def $s0
-; CHECK-GI-NEXT:    // kill: def $h1 killed $h1 def $s1
-; CHECK-GI-NEXT:    fcmp h2, h3
-; CHECK-GI-NEXT:    fmov w8, s0
-; CHECK-GI-NEXT:    fmov w9, s1
-; CHECK-GI-NEXT:    csel w8, w8, w9, ne
-; CHECK-GI-NEXT:    fmov s0, w8
-; CHECK-GI-NEXT:    // kill: def $h0 killed $h0 killed $s0
+; CHECK-GI-NEXT:    // kill: def $h3 killed $h3 def $d3
+; CHECK-GI-NEXT:    // kill: def $h2 killed $h2 def $d2
+; CHECK-GI-NEXT:    shll v3.4s, v3.4h, #16
+; CHECK-GI-NEXT:    shll v2.4s, v2.4h, #16
+; CHECK-GI-NEXT:    fcmp s2, s3
+; CHECK-GI-NEXT:    fcsel h0, h0, h1, ne
 ; CHECK-GI-NEXT:    ret
   %cc = fcmp une bfloat %c, %d
   %r = select i1 %cc, bfloat %a, bfloat %b
@@ -400,31 +421,15 @@ define bfloat @test_select_cc(bfloat %a, bfloat %b, bfloat %c, bfloat %d) #0 {
 }
 
 define float @test_select_cc_f32_f16(float %a, float %b, bfloat %c, bfloat %d) #0 {
-; CHECK-CVT-LABEL: test_select_cc_f32_f16:
-; CHECK-CVT:       // %bb.0:
-; CHECK-CVT-NEXT:    // kill: def $h3 killed $h3 def $d3
-; CHECK-CVT-NEXT:    // kill: def $h2 killed $h2 def $d2
-; CHECK-CVT-NEXT:    shll v3.4s, v3.4h, #16
-; CHECK-CVT-NEXT:    shll v2.4s, v2.4h, #16
-; CHECK-CVT-NEXT:    fcmp s2, s3
-; CHECK-CVT-NEXT:    fcsel s0, s0, s1, ne
-; CHECK-CVT-NEXT:    ret
-;
-; CHECK-SD-LABEL: test_select_cc_f32_f16:
-; CHECK-SD:       // %bb.0:
-; CHECK-SD-NEXT:    // kill: def $h3 killed $h3 def $d3
-; CHECK-SD-NEXT:    // kill: def $h2 killed $h2 def $d2
-; CHECK-SD-NEXT:    shll v3.4s, v3.4h, #16
-; CHECK-SD-NEXT:    shll v2.4s, v2.4h, #16
-; CHECK-SD-NEXT:    fcmp s2, s3
-; CHECK-SD-NEXT:    fcsel s0, s0, s1, ne
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: test_select_cc_f32_f16:
-; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    fcmp h2, h3
-; CHECK-GI-NEXT:    fcsel s0, s0, s1, ne
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: test_select_cc_f32_f16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    // kill: def $h3 killed $h3 def $d3
+; CHECK-NEXT:    // kill: def $h2 killed $h2 def $d2
+; CHECK-NEXT:    shll v3.4s, v3.4h, #16
+; CHECK-NEXT:    shll v2.4s, v2.4h, #16
+; CHECK-NEXT:    fcmp s2, s3
+; CHECK-NEXT:    fcsel s0, s0, s1, ne
+; CHECK-NEXT:    ret
   %cc = fcmp une bfloat %c, %d
   %r = select i1 %cc, float %a, float %b
   ret float %r
@@ -466,429 +471,199 @@ define bfloat @test_select_cc_f16_f32(bfloat %a, bfloat %b, float %c, float %d)
 }
 
 define i1 @test_fcmp_une(bfloat %a, bfloat %b) #0 {
-; CHECK-CVT-LABEL: test_fcmp_une:
-; CHECK-CVT:       // %bb.0:
-; CHECK-CVT-NEXT:    // kill: def $h1 killed $h1 def $d1
-; CHECK-CVT-NEXT:    // kill: def $h0 killed $h0 def $d0
-; CHECK-CVT-NEXT:    shll v1.4s, v1.4h, #16
-; CHECK-CVT-NEXT:    shll v0.4s, v0.4h, #16
-; CHECK-CVT-NEXT:    fcmp s0, s1
-; CHECK-CVT-NEXT:    cset w0, ne
-; CHECK-CVT-NEXT:    ret
-;
-; CHECK-SD-LABEL: test_fcmp_une:
-; CHECK-SD:       // %bb.0:
-; CHECK-SD-NEXT:    // kill: def $h1 killed $h1 def $d1
-; CHECK-SD-NEXT:    // kill: def $h0 killed $h0 def $d0
-; CHECK-SD-NEXT:    shll v1.4s, v1.4h, #16
-; CHECK-SD-NEXT:    shll v0.4s, v0.4h, #16
-; CHECK-SD-NEXT:    fcmp s0, s1
-; CHECK-SD-NEXT:    cset w0, ne
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: test_fcmp_une:
-; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    fcmp h0, h1
-; CHECK-GI-NEXT:    cset w0, ne
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: test_fcmp_une:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    // kill: def $h1 killed $h1 def $d1
+; CHECK-NEXT:    // kill: def $h0 killed $h0 def $d0
+; CHECK-NEXT:    shll v1.4s, v1.4h, #16
+; CHECK-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-NEXT:    fcmp s0, s1
+; CHECK-NEXT:    cset w0, ne
+; CHECK-NEXT:    ret
   %r = fcmp une bfloat %a, %b
   ret i1 %r
 }
 
 define i1 @test_fcmp_ueq(bfloat %a, bfloat %b) #0 {
-; CHECK-CVT-LABEL: test_fcmp_ueq:
-; CHECK-CVT:       // %bb.0:
-; CHECK-CVT-NEXT:    // kill: def $h1 killed $h1 def $d1
-; CHECK-CVT-NEXT:    // kill: def $h0 killed $h0 def $d0
-; CHECK-CVT-NEXT:    shll v1.4s, v1.4h, #16
-; CHECK-CVT-NEXT:    shll v0.4s, v0.4h, #16
-; CHECK-CVT-NEXT:    fcmp s0, s1
-; CHECK-CVT-NEXT:    cset w8, eq
-; CHECK-CVT-NEXT:    csinc w0, w8, wzr, vc
-; CHECK-CVT-NEXT:    ret
-;
-; CHECK-SD-LABEL: test_fcmp_ueq:
-; CHECK-SD:       // %bb.0:
-; CHECK-SD-NEXT:    // kill: def $h1 killed $h1 def $d1
-; CHECK-SD-NEXT:    // kill: def $h0 killed $h0 def $d0
-; CHECK-SD-NEXT:    shll v1.4s, v1.4h, #16
-; CHECK-SD-NEXT:    shll v0.4s, v0.4h, #16
-; CHECK-SD-NEXT:    fcmp s0, s1
-; CHECK-SD-NEXT:    cset w8, eq
-; CHECK-SD-NEXT:    csinc w0, w8, wzr, vc
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: test_fcmp_ueq:
-; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    fcmp h0, h1
-; CHECK-GI-NEXT:    cset w8, eq
-; CHECK-GI-NEXT:    cset w9, vs
-; CHECK-GI-NEXT:    orr w0, w8, w9
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: test_fcmp_ueq:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    // kill: def $h1 killed $h1 def $d1
+; CHECK-NEXT:    // kill: def $h0 killed $h0 def $d0
+; CHECK-NEXT:    shll v1.4s, v1.4h, #16
+; CHECK-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-NEXT:    fcmp s0, s1
+; CHECK-NEXT:    cset w8, eq
+; CHECK-NEXT:    csinc w0, w8, wzr, vc
+; CHECK-NEXT:    ret
   %r = fcmp ueq bfloat %a, %b
   ret i1 %r
 }
 
 define i1 @test_fcmp_ugt(bfloat %a, bfloat %b) #0 {
-; CHECK-CVT-LABEL: test_fcmp_ugt:
-; CHECK-CVT:       // %bb.0:
-; CHECK-CVT-NEXT:    // kill: def $h1 killed $h1 def $d1
-; CHECK-CVT-NEXT:    // kill: def $h0 killed $h0 def $d0
-; CHECK-CVT-NEXT:    shll v1.4s, v1.4h, #16
-; CHECK-CVT-NEXT:    shll v0.4s, v0.4h, #16
-; CHECK-CVT-NEXT:    fcmp s0, s1
-; CHECK-CVT-NEXT:    cset w0, hi
-; CHECK-CVT-NEXT:    ret
-;
-; CHECK-SD-LABEL: test_fcmp_ugt:
-; CHECK-SD:       // %bb.0:
-; CHECK-SD-NEXT:    // kill: def $h1 killed $h1 def $d1
-; CHECK-SD-NEXT:    // kill: def $h0 killed $h0 def $d0
-; CHECK-SD-NEXT:    shll v1.4s, v1.4h, #16
-; CHECK-SD-NEXT:    shll v0.4s, v0.4h, #16
-; CHECK-SD-NEXT:    fcmp s0, s1
-; CHECK-SD-NEXT:    cset w0, hi
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: test_fcmp_ugt:
-; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    fcmp h0, h1
-; CHECK-GI-NEXT:    cset w0, hi
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: test_fcmp_ugt:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    // kill: def $h1 killed $h1 def $d1
+; CHECK-NEXT:    // kill: def $h0 killed $h0 def $d0
+; CHECK-NEXT:    shll v1.4s, v1.4h, #16
+; CHECK-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-NEXT:    fcmp s0, s1
+; CHECK-NEXT:    cset w0, hi
+; CHECK-NEXT:    ret
   %r = fcmp ugt bfloat %a, %b
   ret i1 %r
 }
 
 define i1 @test_fcmp_uge(bfloat %a, bfloat %b) #0 {
-; CHECK-CVT-LABEL: test_fcmp_uge:
-; CHECK-CVT:       // %bb.0:
-; CHECK-CVT-NEXT:    // kill: def $h1 killed $h1 def $d1
-; CHECK-CVT-NEXT:    // kill: def $h0 killed $h0 def $d0
-; CHECK-CVT-NEXT:    shll v1.4s, v1.4h, #16
-; CHECK-CVT-NEXT:    shll v0.4s, v0.4h, #16
-; CHECK-CVT-NEXT:    fcmp s0, s1
-; CHECK-CVT-NEXT:    cset w0, pl
-; CHECK-CVT-NEXT:    ret
-;
-; CHECK-SD-LABEL: test_fcmp_uge:
-; CHECK-SD:       // %bb.0:
-; CHECK-SD-NEXT:    // kill: def $h1 killed $h1 def $d1
-; CHECK-SD-NEXT:    // kill: def $h0 killed $h0 def $d0
-; CHECK-SD-NEXT:    shll v1.4s, v1.4h, #16
-; CHECK-SD-NEXT:    shll v0.4s, v0.4h, #16
-; CHECK-SD-NEXT:    fcmp s0, s1
-; CHECK-SD-NEXT:    cset w0, pl
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: test_fcmp_uge:
-; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    fcmp h0, h1
-; CHECK-GI-NEXT:    cset w0, pl
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: test_fcmp_uge:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    // kill: def $h1 killed $h1 def $d1
+; CHECK-NEXT:    // kill: def $h0 killed $h0 def $d0
+; CHECK-NEXT:    shll v1.4s, v1.4h, #16
+; CHECK-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-NEXT:    fcmp s0, s1
+; CHECK-NEXT:    cset w0, pl
+; CHECK-NEXT:    ret
   %r = fcmp uge bfloat %a, %b
   ret i1 %r
 }
 
 define i1 @test_fcmp_ult(bfloat %a, bfloat %b) #0 {
-; CHECK-CVT-LABEL: test_fcmp_ult:
-; CHECK-CVT:       // %bb.0:
-; CHECK-CVT-NEXT:    // kill: def $h1 killed $h1 def $d1
-; CHECK-CVT-NEXT:    // kill: def $h0 killed $h0 def $d0
-; CHECK-CVT-NEXT:    shll v1.4s, v1.4h, #16
-; CHECK-CVT-NEXT:    shll v0.4s, v0.4h, #16
-; CHECK-CVT-NEXT:    fcmp s0, s1
-; CHECK-CVT-NEXT:    cset w0, lt
-; CHECK-CVT-NEXT:    ret
-;
-; CHECK-SD-LABEL: test_fcmp_ult:
-; CHECK-SD:       // %bb.0:
-; CHECK-SD-NEXT:    // kill: def $h1 killed $h1 def $d1
-; CHECK-SD-NEXT:    // kill: def $h0 killed $h0 def $d0
-; CHECK-SD-NEXT:    shll v1.4s, v1.4h, #16
-; CHECK-SD-NEXT:    shll v0.4s, v0.4h, #16
-; CHECK-SD-NEXT:    fcmp s0, s1
-; CHECK-SD-NEXT:    cset w0, lt
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: test_fcmp_ult:
-; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    fcmp h0, h1
-; CHECK-GI-NEXT:    cset w0, lt
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: test_fcmp_ult:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    // kill: def $h1 killed $h1 def $d1
+; CHECK-NEXT:    // kill: def $h0 killed $h0 def $d0
+; CHECK-NEXT:    shll v1.4s, v1.4h, #16
+; CHECK-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-NEXT:    fcmp s0, s1
+; CHECK-NEXT:    cset w0, lt
+; CHECK-NEXT:    ret
   %r = fcmp ult bfloat %a, %b
   ret i1 %r
 }
 
 define i1 @test_fcmp_ule(bfloat %a, bfloat %b) #0 {
-; CHECK-CVT-LABEL: test_fcmp_ule:
-; CHECK-CVT:       // %bb.0:
-; CHECK-CVT-NEXT:    // kill: def $h1 killed $h1 def $d1
-; CHECK-CVT-NEXT:    // kill: def $h0 killed $h0 def $d0
-; CHECK-CVT-NEXT:    shll v1.4s, v1.4h, #16
-; CHECK-CVT-NEXT:    shll v0.4s, v0.4h, #16
-; CHECK-CVT-NEXT:    fcmp s0, s1
-; CHECK-CVT-NEXT:    cset w0, le
-; CHECK-CVT-NEXT:    ret
-;
-; CHECK-SD-LABEL: test_fcmp_ule:
-; CHECK-SD:       // %bb.0:
-; CHECK-SD-NEXT:    // kill: def $h1 killed $h1 def $d1
-; CHECK-SD-NEXT:    // kill: def $h0 killed $h0 def $d0
-; CHECK-SD-NEXT:    shll v1.4s, v1.4h, #16
-; CHECK-SD-NEXT:    shll v0.4s, v0.4h, #16
-; CHECK-SD-NEXT:    fcmp s0, s1
-; CHECK-SD-NEXT:    cset w0, le
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: test_fcmp_ule:
-; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    fcmp h0, h1
-; CHECK-GI-NEXT:    cset w0, le
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: test_fcmp_ule:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    // kill: def $h1 killed $h1 def $d1
+; CHECK-NEXT:    // kill: def $h0 killed $h0 def $d0
+; CHECK-NEXT:    shll v1.4s, v1.4h, #16
+; CHECK-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-NEXT:    fcmp s0, s1
+; CHECK-NEXT:    cset w0, le
+; CHECK-NEXT:    ret
   %r = fcmp ule bfloat %a, %b
   ret i1 %r
 }
 
 define i1 @test_fcmp_uno(bfloat %a, bfloat %b) #0 {
-; CHECK-CVT-LABEL: test_fcmp_uno:
-; CHECK-CVT:       // %bb.0:
-; CHECK-CVT-NEXT:    // kill: def $h1 killed $h1 def $d1
-; CHECK-CVT-NEXT:    // kill: def $h0 killed $h0 def $d0
-; CHECK-CVT-NEXT:    shll v1.4s, v1.4h, #16
-; CHECK-CVT-NEXT:    shll v0.4s, v0.4h, #16
-; CHECK-CVT-NEXT:    fcmp s0, s1
-; CHECK-CVT-NEXT:    cset w0, vs
-; CHECK-CVT-NEXT:    ret
-;
-; CHECK-SD-LABEL: test_fcmp_uno:
-; CHECK-SD:       // %bb.0:
-; CHECK-SD-NEXT:    // kill: def $h1 killed $h1 def $d1
-; CHECK-SD-NEXT:    // kill: def $h0 killed $h0 def $d0
-; CHECK-SD-NEXT:    shll v1.4s, v1.4h, #16
-; CHECK-SD-NEXT:    shll v0.4s, v0.4h, #16
-; CHECK-SD-NEXT:    fcmp s0, s1
-; CHECK-SD-NEXT:    cset w0, vs
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: test_fcmp_uno:
-; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    fcmp h0, h1
-; CHECK-GI-NEXT:    cset w0, vs
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: test_fcmp_uno:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    // kill: def $h1 killed $h1 def $d1
+; CHECK-NEXT:    // kill: def $h0 killed $h0 def $d0
+; CHECK-NEXT:    shll v1.4s, v1.4h, #16
+; CHECK-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-NEXT:    fcmp s0, s1
+; CHECK-NEXT:    cset w0, vs
+; CHECK-NEXT:    ret
   %r = fcmp uno bfloat %a, %b
   ret i1 %r
 }
 
 define i1 @test_fcmp_one(bfloat %a, bfloat %b) #0 {
-; CHECK-CVT-LABEL: test_fcmp_one:
-; CHECK-CVT:       // %bb.0:
-; CHECK-CVT-NEXT:    // kill: def $h1 killed $h1 def $d1
-; CHECK-CVT-NEXT:    // kill: def $h0 killed $h0 def $d0
-; CHECK-CVT-NEXT:    shll v1.4s, v1.4h, #16
-; CHECK-CVT-NEXT:    shll v0.4s, v0.4h, #16
-; CHECK-CVT-NEXT:    fcmp s0, s1
-; CHECK-CVT-NEXT:    cset w8, mi
-; CHECK-CVT-NEXT:    csinc w0, w8, wzr, le
-; CHECK-CVT-NEXT:    ret
-;
-; CHECK-SD-LABEL: test_fcmp_one:
-; CHECK-SD:       // %bb.0:
-; CHECK-SD-NEXT:    // kill: def $h1 killed $h1 def $d1
-; CHECK-SD-NEXT:    // kill: def $h0 killed $h0 def $d0
-; CHECK-SD-NEXT:    shll v1.4s, v1.4h, #16
-; CHECK-SD-NEXT:    shll v0.4s, v0.4h, #16
-; CHECK-SD-NEXT:    fcmp s0, s1
-; CHECK-SD-NEXT:    cset w8, mi
-; CHECK-SD-NEXT:    csinc w0, w8, wzr, le
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: test_fcmp_one:
-; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    fcmp h0, h1
-; CHECK-GI-NEXT:    cset w8, mi
-; CHECK-GI-NEXT:    cset w9, gt
-; CHECK-GI-NEXT:    orr w0, w8, w9
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: test_fcmp_one:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    // kill: def $h1 killed $h1 def $d1
+; CHECK-NEXT:    // kill: def $h0 killed $h0 def $d0
+; CHECK-NEXT:    shll v1.4s, v1.4h, #16
+; CHECK-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-NEXT:    fcmp s0, s1
+; CHECK-NEXT:    cset w8, mi
+; CHECK-NEXT:    csinc w0, w8, wzr, le
+; CHECK-NEXT:    ret
   %r = fcmp one bfloat %a, %b
   ret i1 %r
 }
 
 define i1 @test_fcmp_oeq(bfloat %a, bfloat %b) #0 {
-; CHECK-CVT-LABEL: test_fcmp_oeq:
-; CHECK-CVT:       // %bb.0:
-; CHECK-CVT-NEXT:    // kill: def $h1 killed $h1 def $d1
-; CHECK-CVT-NEXT:    // kill: def $h0 killed $h0 def $d0
-; CHECK-CVT-NEXT:    shll v1.4s, v1.4h, #16
-; CHECK-CVT-NEXT:    shll v0.4s, v0.4h, #16
-; CHECK-CVT-NEXT:    fcmp s0, s1
-; CHECK-CVT-NEXT:    cset w0, eq
-; CHECK-CVT-NEXT:    ret
-;
-; CHECK-SD-LABEL: test_fcmp_oeq:
-; CHECK-SD:       // %bb.0:
-; CHECK-SD-NEXT:    // kill: def $h1 killed $h1 def $d1
-; CHECK-SD-NEXT:    // kill: def $h0 killed $h0 def $d0
-; CHECK-SD-NEXT:    shll v1.4s, v1.4h, #16
-; CHECK-SD-NEXT:    shll v0.4s, v0.4h, #16
-; CHECK-SD-NEXT:    fcmp s0, s1
-; CHECK-SD-NEXT:    cset w0, eq
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: test_fcmp_oeq:
-; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    fcmp h0, h1
-; CHECK-GI-NEXT:    cset w0, eq
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: test_fcmp_oeq:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    // kill: def $h1 killed $h1 def $d1
+; CHECK-NEXT:    // kill: def $h0 killed $h0 def $d0
+; CHECK-NEXT:    shll v1.4s, v1.4h, #16
+; CHECK-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-NEXT:    fcmp s0, s1
+; CHECK-NEXT:    cset w0, eq
+; CHECK-NEXT:    ret
   %r = fcmp oeq bfloat %a, %b
   ret i1 %r
 }
 
 define i1 @test_fcmp_ogt(bfloat %a, bfloat %b) #0 {
-; CHECK-CVT-LABEL: test_fcmp_ogt:
-; CHECK-CVT:       // %bb.0:
-; CHECK-CVT-NEXT:    // kill: def $h1 killed $h1 def $d1
-; CHECK-CVT-NEXT:    // kill: def $h0 killed $h0 def $d0
-; CHECK-CVT-NEXT:    shll v1.4s, v1.4h, #16
-; CHECK-CVT-NEXT:    shll v0.4s, v0.4h, #16
-; CHECK-CVT-NEXT:    fcmp s0, s1
-; CHECK-CVT-NEXT:    cset w0, gt
-; CHECK-CVT-NEXT:    ret
-;
-; CHECK-SD-LABEL: test_fcmp_ogt:
-; CHECK-SD:       // %bb.0:
-; CHECK-SD-NEXT:    // kill: def $h1 killed $h1 def $d1
-; CHECK-SD-NEXT:    // kill: def $h0 killed $h0 def $d0
-; CHECK-SD-NEXT:    shll v1.4s, v1.4h, #16
-; CHECK-SD-NEXT:    shll v0.4s, v0.4h, #16
-; CHECK-SD-NEXT:    fcmp s0, s1
-; CHECK-SD-NEXT:    cset w0, gt
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: test_fcmp_ogt:
-; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    fcmp h0, h1
-; CHECK-GI-NEXT:    cset w0, gt
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: test_fcmp_ogt:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    // kill: def $h1 killed $h1 def $d1
+; CHECK-NEXT:    // kill: def $h0 killed $h0 def $d0
+; CHECK-NEXT:    shll v1.4s, v1.4h, #16
+; CHECK-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-NEXT:    fcmp s0, s1
+; CHECK-NEXT:    cset w0, gt
+; CHECK-NEXT:    ret
   %r = fcmp ogt bfloat %a, %b
   ret i1 %r
 }
 
 define i1 @test_fcmp_oge(bfloat %a, bfloat %b) #0 {
-; CHECK-CVT-LABEL: test_fcmp_oge:
-; CHECK-CVT:       // %bb.0:
-; CHECK-CVT-NEXT:    // kill: def $h1 killed $h1 def $d1
-; CHECK-CVT-NEXT:    // kill: def $h0 killed $h0 def $d0
-; CHECK-CVT-NEXT:    shll v1.4s, v1.4h, #16
-; CHECK-CVT-NEXT:    shll v0.4s, v0.4h, #16
-; CHECK-CVT-NEXT:    fcmp s0, s1
-; CHECK-CVT-NEXT:    cset w0, ge
-; CHECK-CVT-NEXT:    ret
-;
-; CHECK-SD-LABEL: test_fcmp_oge:
-; CHECK-SD:       // %bb.0:
-; CHECK-SD-NEXT:    // kill: def $h1 killed $h1 def $d1
-; CHECK-SD-NEXT:    // kill: def $h0 killed $h0 def $d0
-; CHECK-SD-NEXT:    shll v1.4s, v1.4h, #16
-; CHECK-SD-NEXT:    shll v0.4s, v0.4h, #16
-; CHECK-SD-NEXT:    fcmp s0, s1
-; CHECK-SD-NEXT:    cset w0, ge
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: test_fcmp_oge:
-; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    fcmp h0, h1
-; CHECK-GI-NEXT:    cset w0, ge
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: test_fcmp_oge:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    // kill: def $h1 killed $h1 def $d1
+; CHECK-NEXT:    // kill: def $h0 killed $h0 def $d0
+; CHECK-NEXT:    shll v1.4s, v1.4h, #16
+; CHECK-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-NEXT:    fcmp s0, s1
+; CHECK-NEXT:    cset w0, ge
+; CHECK-NEXT:    ret
   %r = fcmp oge bfloat %a, %b
   ret i1 %r
-}
-
-define i1 @test_fcmp_olt(bfloat %a, bfloat %b) #0 {
-; CHECK-CVT-LABEL: test_fcmp_olt:
-; CHECK-CVT:       // %bb.0:
-; CHECK-CVT-NEXT:    // kill: def $h1 killed $h1 def $d1
-; CHECK-CVT-NEXT:    // kill: def $h0 killed $h0 def $d0
-; CHECK-CVT-NEXT:    shll v1.4s, v1.4h, #16
-; CHECK-CVT-NEXT:    shll v0.4s, v0.4h, #16
-; CHECK-CVT-NEXT:    fcmp s0, s1
-; CHECK-CVT-NEXT:    cset w0, mi
-; CHECK-CVT-NEXT:    ret
-;
-; CHECK-SD-LABEL: test_fcmp_olt:
-; CHECK-SD:       // %bb.0:
-; CHECK-SD-NEXT:    // kill: def $h1 killed $h1 def $d1
-; CHECK-SD-NEXT:    // kill: def $h0 killed $h0 def $d0
-; CHECK-SD-NEXT:    shll v1.4s, v1.4h, #16
-; CHECK-SD-NEXT:    shll v0.4s, v0.4h, #16
-; CHECK-SD-NEXT:    fcmp s0, s1
-; CHECK-SD-NEXT:    cset w0, mi
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: test_fcmp_olt:
-; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    fcmp h0, h1
-; CHECK-GI-NEXT:    cset w0, mi
-; CHECK-GI-NEXT:    ret
-  %r = fcmp olt bfloat %a, %b
-  ret i1 %r
-}
-
-define i1 @test_fcmp_ole(bfloat %a, bfloat %b) #0 {
-; CHECK-CVT-LABEL: test_fcmp_ole:
-; CHECK-CVT:       // %bb.0:
-; CHECK-CVT-NEXT:    // kill: def $h1 killed $h1 def $d1
-; CHECK-CVT-NEXT:    // kill: def $h0 killed $h0 def $d0
-; CHECK-CVT-NEXT:    shll v1.4s, v1.4h, #16
-; CHECK-CVT-NEXT:    shll v0.4s, v0.4h, #16
-; CHECK-CVT-NEXT:    fcmp s0, s1
-; CHECK-CVT-NEXT:    cset w0, ls
-; CHECK-CVT-NEXT:    ret
-;
-; CHECK-SD-LABEL: test_fcmp_ole:
-; CHECK-SD:       // %bb.0:
-; CHECK-SD-NEXT:    // kill: def $h1 killed $h1 def $d1
-; CHECK-SD-NEXT:    // kill: def $h0 killed $h0 def $d0
-; CHECK-SD-NEXT:    shll v1.4s, v1.4h, #16
-; CHECK-SD-NEXT:    shll v0.4s, v0.4h, #16
-; CHECK-SD-NEXT:    fcmp s0, s1
-; CHECK-SD-NEXT:    cset w0, ls
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: test_fcmp_ole:
-; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    fcmp h0, h1
-; CHECK-GI-NEXT:    cset w0, ls
-; CHECK-GI-NEXT:    ret
+}
+
+define i1 @test_fcmp_olt(bfloat %a, bfloat %b) #0 {
+; CHECK-LABEL: test_fcmp_olt:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    // kill: def $h1 killed $h1 def $d1
+; CHECK-NEXT:    // kill: def $h0 killed $h0 def $d0
+; CHECK-NEXT:    shll v1.4s, v1.4h, #16
+; CHECK-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-NEXT:    fcmp s0, s1
+; CHECK-NEXT:    cset w0, mi
+; CHECK-NEXT:    ret
+  %r = fcmp olt bfloat %a, %b
+  ret i1 %r
+}
+
+define i1 @test_fcmp_ole(bfloat %a, bfloat %b) #0 {
+; CHECK-LABEL: test_fcmp_ole:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    // kill: def $h1 killed $h1 def $d1
+; CHECK-NEXT:    // kill: def $h0 killed $h0 def $d0
+; CHECK-NEXT:    shll v1.4s, v1.4h, #16
+; CHECK-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-NEXT:    fcmp s0, s1
+; CHECK-NEXT:    cset w0, ls
+; CHECK-NEXT:    ret
   %r = fcmp ole bfloat %a, %b
   ret i1 %r
 }
 
 define i1 @test_fcmp_ord(bfloat %a, bfloat %b) #0 {
-; CHECK-CVT-LABEL: test_fcmp_ord:
-; CHECK-CVT:       // %bb.0:
-; CHECK-CVT-NEXT:    // kill: def $h1 killed $h1 def $d1
-; CHECK-CVT-NEXT:    // kill: def $h0 killed $h0 def $d0
-; CHECK-CVT-NEXT:    shll v1.4s, v1.4h, #16
-; CHECK-CVT-NEXT:    shll v0.4s, v0.4h, #16
-; CHECK-CVT-NEXT:    fcmp s0, s1
-; CHECK-CVT-NEXT:    cset w0, vc
-; CHECK-CVT-NEXT:    ret
-;
-; CHECK-SD-LABEL: test_fcmp_ord:
-; CHECK-SD:       // %bb.0:
-; CHECK-SD-NEXT:    // kill: def $h1 killed $h1 def $d1
-; CHECK-SD-NEXT:    // kill: def $h0 killed $h0 def $d0
-; CHECK-SD-NEXT:    shll v1.4s, v1.4h, #16
-; CHECK-SD-NEXT:    shll v0.4s, v0.4h, #16
-; CHECK-SD-NEXT:    fcmp s0, s1
-; CHECK-SD-NEXT:    cset w0, vc
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: test_fcmp_ord:
-; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    fcmp h0, h1
-; CHECK-GI-NEXT:    cset w0, vc
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: test_fcmp_ord:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    // kill: def $h1 killed $h1 def $d1
+; CHECK-NEXT:    // kill: def $h0 killed $h0 def $d0
+; CHECK-NEXT:    shll v1.4s, v1.4h, #16
+; CHECK-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-NEXT:    fcmp s0, s1
+; CHECK-NEXT:    cset w0, vc
+; CHECK-NEXT:    ret
   %r = fcmp ord bfloat %a, %b
   ret i1 %r
 }
@@ -924,15 +699,15 @@ define void @test_fccmp(bfloat %in, ptr %out) {
 ;
 ; CHECK-GI-LABEL: test_fccmp:
 ; CHECK-GI:       // %bb.0:
+; CHECK-GI-NEXT:    movi v1.2s, #69, lsl #24
+; CHECK-GI-NEXT:    // kill: def $h0 killed $h0 def $d0
+; CHECK-GI-NEXT:    shll v2.4s, v0.4h, #16
+; CHECK-GI-NEXT:    movi v3.2s, #72, lsl #24
+; CHECK-GI-NEXT:    fcmp s2, s1
 ; CHECK-GI-NEXT:    fmov h1, #5.00000000
-; CHECK-GI-NEXT:    // kill: def $h0 killed $h0 def $s0
-; CHECK-GI-NEXT:    fmov h2, #8.00000000
-; CHECK-GI-NEXT:    fmov w8, s0
-; CHECK-GI-NEXT:    fcmp h0, h1
-; CHECK-GI-NEXT:    fmov w9, s1
-; CHECK-GI-NEXT:    fccmp h0, h2, #4, mi
-; CHECK-GI-NEXT:    csel w8, w8, w9, gt
-; CHECK-GI-NEXT:    strh w8, [x0]
+; CHECK-GI-NEXT:    fccmp s2, s3, #4, mi
+; CHECK-GI-NEXT:    fcsel h0, h0, h1, gt
+; CHECK-GI-NEXT:    str h0, [x0]
 ; CHECK-GI-NEXT:    ret
   %cmp1 = fcmp ogt bfloat %in, 0xR4800
   %cmp2 = fcmp olt bfloat %in, 0xR4500
@@ -943,34 +718,16 @@ define void @test_fccmp(bfloat %in, ptr %out) {
 }
 
 define void @test_br_cc(bfloat %a, bfloat %b, ptr %p1, ptr %p2) #0 {
-; CHECK-CVT-LABEL: test_br_cc:
-; CHECK-CVT:       // %bb.0: // %common.ret
-; CHECK-CVT-NEXT:    // kill: def $h1 killed $h1 def $d1
-; CHECK-CVT-NEXT:    // kill: def $h0 killed $h0 def $d0
-; CHECK-CVT-NEXT:    shll v1.4s, v1.4h, #16
-; CHECK-CVT-NEXT:    shll v0.4s, v0.4h, #16
-; CHECK-CVT-NEXT:    fcmp s0, s1
-; CHECK-CVT-NEXT:    csel x8, x0, x1, pl
-; CHECK-CVT-NEXT:    str wzr, [x8]
-; CHECK-CVT-NEXT:    ret
-;
-; CHECK-SD-LABEL: test_br_cc:
-; CHECK-SD:       // %bb.0: // %common.ret
-; CHECK-SD-NEXT:    // kill: def $h1 killed $h1 def $d1
-; CHECK-SD-NEXT:    // kill: def $h0 killed $h0 def $d0
-; CHECK-SD-NEXT:    shll v1.4s, v1.4h, #16
-; CHECK-SD-NEXT:    shll v0.4s, v0.4h, #16
-; CHECK-SD-NEXT:    fcmp s0, s1
-; CHECK-SD-NEXT:    csel x8, x0, x1, pl
-; CHECK-SD-NEXT:    str wzr, [x8]
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: test_br_cc:
-; CHECK-GI:       // %bb.0: // %common.ret
-; CHECK-GI-NEXT:    fcmp h0, h1
-; CHECK-GI-NEXT:    csel x8, x0, x1, pl
-; CHECK-GI-NEXT:    str wzr, [x8]
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: test_br_cc:
+; CHECK:       // %bb.0: // %common.ret
+; CHECK-NEXT:    // kill: def $h1 killed $h1 def $d1
+; CHECK-NEXT:    // kill: def $h0 killed $h0 def $d0
+; CHECK-NEXT:    shll v1.4s, v1.4h, #16
+; CHECK-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-NEXT:    fcmp s0, s1
+; CHECK-NEXT:    csel x8, x0, x1, pl
+; CHECK-NEXT:    str wzr, [x8]
+; CHECK-NEXT:    ret
   %c = fcmp uge bfloat %a, %b
   br i1 %c, label %then, label %else
 then:
@@ -1426,18 +1183,13 @@ define bfloat @test_sqrt(bfloat %a) #0 {
 ; CHECK-CVT-NEXT:    // kill: def $h0 killed $h0 killed $s0
 ; CHECK-CVT-NEXT:    ret
 ;
-; CHECK-SD-LABEL: test_sqrt:
-; CHECK-SD:       // %bb.0:
-; CHECK-SD-NEXT:    // kill: def $h0 killed $h0 def $d0
-; CHECK-SD-NEXT:    shll v0.4s, v0.4h, #16
-; CHECK-SD-NEXT:    fsqrt s0, s0
-; CHECK-SD-NEXT:    bfcvt h0, s0
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: test_sqrt:
-; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    fsqrt h0, h0
-; CHECK-GI-NEXT:    ret
+; CHECK-BF16-LABEL: test_sqrt:
+; CHECK-BF16:       // %bb.0:
+; CHECK-BF16-NEXT:    // kill: def $h0 killed $h0 def $d0
+; CHECK-BF16-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-BF16-NEXT:    fsqrt s0, s0
+; CHECK-BF16-NEXT:    bfcvt h0, s0
+; CHECK-BF16-NEXT:    ret
   %r = call bfloat @llvm.sqrt.f16(bfloat %a)
   ret bfloat %r
 }
@@ -1461,25 +1213,16 @@ define bfloat @test_powi(bfloat %a, i32 %b) #0 {
 ; CHECK-CVT-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
 ; CHECK-CVT-NEXT:    ret
 ;
-; CHECK-SD-LABEL: test_powi:
-; CHECK-SD:       // %bb.0:
-; CHECK-SD-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
-; CHECK-SD-NEXT:    // kill: def $h0 killed $h0 def $d0
-; CHECK-SD-NEXT:    shll v0.4s, v0.4h, #16
-; CHECK-SD-NEXT:    // kill: def $s0 killed $s0 killed $q0
-; CHECK-SD-NEXT:    bl __powisf2
-; CHECK-SD-NEXT:    bfcvt h0, s0
-; CHECK-SD-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: test_powi:
-; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
-; CHECK-GI-NEXT:    fcvt s0, h0
-; CHECK-GI-NEXT:    bl __powisf2
-; CHECK-GI-NEXT:    bfcvt h0, s0
-; CHECK-GI-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
-; CHECK-GI-NEXT:    ret
+; CHECK-BF16-LABEL: test_powi:
+; CHECK-BF16:       // %bb.0:
+; CHECK-BF16-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-BF16-NEXT:    // kill: def $h0 killed $h0 def $d0
+; CHECK-BF16-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-BF16-NEXT:    // kill: def $s0 killed $s0 killed $q0
+; CHECK-BF16-NEXT:    bl __powisf2
+; CHECK-BF16-NEXT:    bfcvt h0, s0
+; CHECK-BF16-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-BF16-NEXT:    ret
   %r = call bfloat @llvm.powi.f16.i32(bfloat %a, i32 %b)
   ret bfloat %r
 }
@@ -1504,25 +1247,16 @@ define bfloat @test_sin(bfloat %a) #0 {
 ; CHECK-CVT-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
 ; CHECK-CVT-NEXT:    ret
 ;
-; CHECK-SD-LABEL: test_sin:
-; CHECK-SD:       // %bb.0:
-; CHECK-SD-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
-; CHECK-SD-NEXT:    // kill: def $h0 killed $h0 def $d0
-; CHECK-SD-NEXT:    shll v0.4s, v0.4h, #16
-; CHECK-SD-NEXT:    // kill: def $s0 killed $s0 killed $q0
-; CHECK-SD-NEXT:    bl sinf
-; CHECK-SD-NEXT:    bfcvt h0, s0
-; CHECK-SD-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: test_sin:
-; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
-; CHECK-GI-NEXT:    fcvt s0, h0
-; CHECK-GI-NEXT:    bl sinf
-; CHECK-GI-NEXT:    bfcvt h0, s0
-; CHECK-GI-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
-; CHECK-GI-NEXT:    ret
+; CHECK-BF16-LABEL: test_sin:
+; CHECK-BF16:       // %bb.0:
+; CHECK-BF16-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-BF16-NEXT:    // kill: def $h0 killed $h0 def $d0
+; CHECK-BF16-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-BF16-NEXT:    // kill: def $s0 killed $s0 killed $q0
+; CHECK-BF16-NEXT:    bl sinf
+; CHECK-BF16-NEXT:    bfcvt h0, s0
+; CHECK-BF16-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-BF16-NEXT:    ret
   %r = call bfloat @llvm.sin.f16(bfloat %a)
   ret bfloat %r
 }
@@ -1546,25 +1280,16 @@ define bfloat @test_cos(bfloat %a) #0 {
 ; CHECK-CVT-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
 ; CHECK-CVT-NEXT:    ret
 ;
-; CHECK-SD-LABEL: test_cos:
-; CHECK-SD:       // %bb.0:
-; CHECK-SD-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
-; CHECK-SD-NEXT:    // kill: def $h0 killed $h0 def $d0
-; CHECK-SD-NEXT:    shll v0.4s, v0.4h, #16
-; CHECK-SD-NEXT:    // kill: def $s0 killed $s0 killed $q0
-; CHECK-SD-NEXT:    bl cosf
-; CHECK-SD-NEXT:    bfcvt h0, s0
-; CHECK-SD-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: test_cos:
-; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
-; CHECK-GI-NEXT:    fcvt s0, h0
-; CHECK-GI-NEXT:    bl cosf
-; CHECK-GI-NEXT:    bfcvt h0, s0
-; CHECK-GI-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
-; CHECK-GI-NEXT:    ret
+; CHECK-BF16-LABEL: test_cos:
+; CHECK-BF16:       // %bb.0:
+; CHECK-BF16-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-BF16-NEXT:    // kill: def $h0 killed $h0 def $d0
+; CHECK-BF16-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-BF16-NEXT:    // kill: def $s0 killed $s0 killed $q0
+; CHECK-BF16-NEXT:    bl cosf
+; CHECK-BF16-NEXT:    bfcvt h0, s0
+; CHECK-BF16-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-BF16-NEXT:    ret
   %r = call bfloat @llvm.cos.f16(bfloat %a)
   ret bfloat %r
 }
@@ -1588,25 +1313,16 @@ define bfloat @test_tan(bfloat %a) #0 {
 ; CHECK-CVT-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
 ; CHECK-CVT-NEXT:    ret
 ;
-; CHECK-SD-LABEL: test_tan:
-; CHECK-SD:       // %bb.0:
-; CHECK-SD-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
-; CHECK-SD-NEXT:    // kill: def $h0 killed $h0 def $d0
-; CHECK-SD-NEXT:    shll v0.4s, v0.4h, #16
-; CHECK-SD-NEXT:    // kill: def $s0 killed $s0 killed $q0
-; CHECK-SD-NEXT:    bl tanf
-; CHECK-SD-NEXT:    bfcvt h0, s0
-; CHECK-SD-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: test_tan:
-; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
-; CHECK-GI-NEXT:    fcvt s0, h0
-; CHECK-GI-NEXT:    bl tanf
-; CHECK-GI-NEXT:    bfcvt h0, s0
-; CHECK-GI-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
-; CHECK-GI-NEXT:    ret
+; CHECK-BF16-LABEL: test_tan:
+; CHECK-BF16:       // %bb.0:
+; CHECK-BF16-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-BF16-NEXT:    // kill: def $h0 killed $h0 def $d0
+; CHECK-BF16-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-BF16-NEXT:    // kill: def $s0 killed $s0 killed $q0
+; CHECK-BF16-NEXT:    bl tanf
+; CHECK-BF16-NEXT:    bfcvt h0, s0
+; CHECK-BF16-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-BF16-NEXT:    ret
   %r = call bfloat @llvm.tan.f16(bfloat %a)
   ret bfloat %r
 }
@@ -1630,25 +1346,16 @@ define bfloat @test_acos(bfloat %a) #0 {
 ; CHECK-CVT-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
 ; CHECK-CVT-NEXT:    ret
 ;
-; CHECK-SD-LABEL: test_acos:
-; CHECK-SD:       // %bb.0:
-; CHECK-SD-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
-; CHECK-SD-NEXT:    // kill: def $h0 killed $h0 def $d0
-; CHECK-SD-NEXT:    shll v0.4s, v0.4h, #16
-; CHECK-SD-NEXT:    // kill: def $s0 killed $s0 killed $q0
-; CHECK-SD-NEXT:    bl acosf
-; CHECK-SD-NEXT:    bfcvt h0, s0
-; CHECK-SD-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: test_acos:
-; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
-; CHECK-GI-NEXT:    fcvt s0, h0
-; CHECK-GI-NEXT:    bl acosf
-; CHECK-GI-NEXT:    bfcvt h0, s0
-; CHECK-GI-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
-; CHECK-GI-NEXT:    ret
+; CHECK-BF16-LABEL: test_acos:
+; CHECK-BF16:       // %bb.0:
+; CHECK-BF16-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-BF16-NEXT:    // kill: def $h0 killed $h0 def $d0
+; CHECK-BF16-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-BF16-NEXT:    // kill: def $s0 killed $s0 killed $q0
+; CHECK-BF16-NEXT:    bl acosf
+; CHECK-BF16-NEXT:    bfcvt h0, s0
+; CHECK-BF16-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-BF16-NEXT:    ret
   %r = call bfloat @llvm.acos.f16(bfloat %a)
   ret bfloat %r
 }
@@ -1672,25 +1379,16 @@ define bfloat @test_asin(bfloat %a) #0 {
 ; CHECK-CVT-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
 ; CHECK-CVT-NEXT:    ret
 ;
-; CHECK-SD-LABEL: test_asin:
-; CHECK-SD:       // %bb.0:
-; CHECK-SD-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
-; CHECK-SD-NEXT:    // kill: def $h0 killed $h0 def $d0
-; CHECK-SD-NEXT:    shll v0.4s, v0.4h, #16
-; CHECK-SD-NEXT:    // kill: def $s0 killed $s0 killed $q0
-; CHECK-SD-NEXT:    bl asinf
-; CHECK-SD-NEXT:    bfcvt h0, s0
-; CHECK-SD-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: test_asin:
-; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
-; CHECK-GI-NEXT:    fcvt s0, h0
-; CHECK-GI-NEXT:    bl asinf
-; CHECK-GI-NEXT:    bfcvt h0, s0
-; CHECK-GI-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
-; CHECK-GI-NEXT:    ret
+; CHECK-BF16-LABEL: test_asin:
+; CHECK-BF16:       // %bb.0:
+; CHECK-BF16-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-BF16-NEXT:    // kill: def $h0 killed $h0 def $d0
+; CHECK-BF16-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-BF16-NEXT:    // kill: def $s0 killed $s0 killed $q0
+; CHECK-BF16-NEXT:    bl asinf
+; CHECK-BF16-NEXT:    bfcvt h0, s0
+; CHECK-BF16-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-BF16-NEXT:    ret
   %r = call bfloat @llvm.asin.f16(bfloat %a)
   ret bfloat %r
 }
@@ -1714,25 +1412,16 @@ define bfloat @test_atan(bfloat %a) #0 {
 ; CHECK-CVT-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
 ; CHECK-CVT-NEXT:    ret
 ;
-; CHECK-SD-LABEL: test_atan:
-; CHECK-SD:       // %bb.0:
-; CHECK-SD-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
-; CHECK-SD-NEXT:    // kill: def $h0 killed $h0 def $d0
-; CHECK-SD-NEXT:    shll v0.4s, v0.4h, #16
-; CHECK-SD-NEXT:    // kill: def $s0 killed $s0 killed $q0
-; CHECK-SD-NEXT:    bl atanf
-; CHECK-SD-NEXT:    bfcvt h0, s0
-; CHECK-SD-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: test_atan:
-; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
-; CHECK-GI-NEXT:    fcvt s0, h0
-; CHECK-GI-NEXT:    bl atanf
-; CHECK-GI-NEXT:    bfcvt h0, s0
-; CHECK-GI-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
-; CHECK-GI-NEXT:    ret
+; CHECK-BF16-LABEL: test_atan:
+; CHECK-BF16:       // %bb.0:
+; CHECK-BF16-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-BF16-NEXT:    // kill: def $h0 killed $h0 def $d0
+; CHECK-BF16-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-BF16-NEXT:    // kill: def $s0 killed $s0 killed $q0
+; CHECK-BF16-NEXT:    bl atanf
+; CHECK-BF16-NEXT:    bfcvt h0, s0
+; CHECK-BF16-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-BF16-NEXT:    ret
   %r = call bfloat @llvm.atan.f16(bfloat %a)
   ret bfloat %r
 }
@@ -1759,29 +1448,19 @@ define bfloat @test_atan2(bfloat %a, bfloat %b) #0 {
 ; CHECK-CVT-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
 ; CHECK-CVT-NEXT:    ret
 ;
-; CHECK-SD-LABEL: test_atan2:
-; CHECK-SD:       // %bb.0:
-; CHECK-SD-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
-; CHECK-SD-NEXT:    // kill: def $h1 killed $h1 def $d1
-; CHECK-SD-NEXT:    // kill: def $h0 killed $h0 def $d0
-; CHECK-SD-NEXT:    shll v0.4s, v0.4h, #16
-; CHECK-SD-NEXT:    shll v1.4s, v1.4h, #16
-; CHECK-SD-NEXT:    // kill: def $s0 killed $s0 killed $q0
-; CHECK-SD-NEXT:    // kill: def $s1 killed $s1 killed $q1
-; CHECK-SD-NEXT:    bl atan2f
-; CHECK-SD-NEXT:    bfcvt h0, s0
-; CHECK-SD-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: test_atan2:
-; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
-; CHECK-GI-NEXT:    fcvt s0, h0
-; CHECK-GI-NEXT:    fcvt s1, h1
-; CHECK-GI-NEXT:    bl atan2f
-; CHECK-GI-NEXT:    bfcvt h0, s0
-; CHECK-GI-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
-; CHECK-GI-NEXT:    ret
+; CHECK-BF16-LABEL: test_atan2:
+; CHECK-BF16:       // %bb.0:
+; CHECK-BF16-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-BF16-NEXT:    // kill: def $h1 killed $h1 def $d1
+; CHECK-BF16-NEXT:    // kill: def $h0 killed $h0 def $d0
+; CHECK-BF16-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-BF16-NEXT:    shll v1.4s, v1.4h, #16
+; CHECK-BF16-NEXT:    // kill: def $s0 killed $s0 killed $q0
+; CHECK-BF16-NEXT:    // kill: def $s1 killed $s1 killed $q1
+; CHECK-BF16-NEXT:    bl atan2f
+; CHECK-BF16-NEXT:    bfcvt h0, s0
+; CHECK-BF16-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-BF16-NEXT:    ret
   %r = call bfloat @llvm.atan2.f16(bfloat %a, bfloat %b)
   ret bfloat %r
 }
@@ -1805,25 +1484,16 @@ define bfloat @test_cosh(bfloat %a) #0 {
 ; CHECK-CVT-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
 ; CHECK-CVT-NEXT:    ret
 ;
-; CHECK-SD-LABEL: test_cosh:
-; CHECK-SD:       // %bb.0:
-; CHECK-SD-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
-; CHECK-SD-NEXT:    // kill: def $h0 killed $h0 def $d0
-; CHECK-SD-NEXT:    shll v0.4s, v0.4h, #16
-; CHECK-SD-NEXT:    // kill: def $s0 killed $s0 killed $q0
-; CHECK-SD-NEXT:    bl coshf
-; CHECK-SD-NEXT:    bfcvt h0, s0
-; CHECK-SD-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: test_cosh:
-; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
-; CHECK-GI-NEXT:    fcvt s0, h0
-; CHECK-GI-NEXT:    bl coshf
-; CHECK-GI-NEXT:    bfcvt h0, s0
-; CHECK-GI-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
-; CHECK-GI-NEXT:    ret
+; CHECK-BF16-LABEL: test_cosh:
+; CHECK-BF16:       // %bb.0:
+; CHECK-BF16-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-BF16-NEXT:    // kill: def $h0 killed $h0 def $d0
+; CHECK-BF16-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-BF16-NEXT:    // kill: def $s0 killed $s0 killed $q0
+; CHECK-BF16-NEXT:    bl coshf
+; CHECK-BF16-NEXT:    bfcvt h0, s0
+; CHECK-BF16-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-BF16-NEXT:    ret
   %r = call bfloat @llvm.cosh.f16(bfloat %a)
   ret bfloat %r
 }
@@ -1847,25 +1517,16 @@ define bfloat @test_sinh(bfloat %a) #0 {
 ; CHECK-CVT-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
 ; CHECK-CVT-NEXT:    ret
 ;
-; CHECK-SD-LABEL: test_sinh:
-; CHECK-SD:       // %bb.0:
-; CHECK-SD-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
-; CHECK-SD-NEXT:    // kill: def $h0 killed $h0 def $d0
-; CHECK-SD-NEXT:    shll v0.4s, v0.4h, #16
-; CHECK-SD-NEXT:    // kill: def $s0 killed $s0 killed $q0
-; CHECK-SD-NEXT:    bl sinhf
-; CHECK-SD-NEXT:    bfcvt h0, s0
-; CHECK-SD-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: test_sinh:
-; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
-; CHECK-GI-NEXT:    fcvt s0, h0
-; CHECK-GI-NEXT:    bl sinhf
-; CHECK-GI-NEXT:    bfcvt h0, s0
-; CHECK-GI-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
-; CHECK-GI-NEXT:    ret
+; CHECK-BF16-LABEL: test_sinh:
+; CHECK-BF16:       // %bb.0:
+; CHECK-BF16-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-BF16-NEXT:    // kill: def $h0 killed $h0 def $d0
+; CHECK-BF16-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-BF16-NEXT:    // kill: def $s0 killed $s0 killed $q0
+; CHECK-BF16-NEXT:    bl sinhf
+; CHECK-BF16-NEXT:    bfcvt h0, s0
+; CHECK-BF16-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-BF16-NEXT:    ret
   %r = call bfloat @llvm.sinh.f16(bfloat %a)
   ret bfloat %r
 }
@@ -1889,25 +1550,16 @@ define bfloat @test_tanh(bfloat %a) #0 {
 ; CHECK-CVT-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
 ; CHECK-CVT-NEXT:    ret
 ;
-; CHECK-SD-LABEL: test_tanh:
-; CHECK-SD:       // %bb.0:
-; CHECK-SD-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
-; CHECK-SD-NEXT:    // kill: def $h0 killed $h0 def $d0
-; CHECK-SD-NEXT:    shll v0.4s, v0.4h, #16
-; CHECK-SD-NEXT:    // kill: def $s0 killed $s0 killed $q0
-; CHECK-SD-NEXT:    bl tanhf
-; CHECK-SD-NEXT:    bfcvt h0, s0
-; CHECK-SD-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: test_tanh:
-; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
-; CHECK-GI-NEXT:    fcvt s0, h0
-; CHECK-GI-NEXT:    bl tanhf
-; CHECK-GI-NEXT:    bfcvt h0, s0
-; CHECK-GI-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
-; CHECK-GI-NEXT:    ret
+; CHECK-BF16-LABEL: test_tanh:
+; CHECK-BF16:       // %bb.0:
+; CHECK-BF16-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-BF16-NEXT:    // kill: def $h0 killed $h0 def $d0
+; CHECK-BF16-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-BF16-NEXT:    // kill: def $s0 killed $s0 killed $q0
+; CHECK-BF16-NEXT:    bl tanhf
+; CHECK-BF16-NEXT:    bfcvt h0, s0
+; CHECK-BF16-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-BF16-NEXT:    ret
   %r = call bfloat @llvm.tanh.f16(bfloat %a)
   ret bfloat %r
 }
@@ -1934,29 +1586,19 @@ define bfloat @test_pow(bfloat %a, bfloat %b) #0 {
 ; CHECK-CVT-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
 ; CHECK-CVT-NEXT:    ret
 ;
-; CHECK-SD-LABEL: test_pow:
-; CHECK-SD:       // %bb.0:
-; CHECK-SD-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
-; CHECK-SD-NEXT:    // kill: def $h1 killed $h1 def $d1
-; CHECK-SD-NEXT:    // kill: def $h0 killed $h0 def $d0
-; CHECK-SD-NEXT:    shll v0.4s, v0.4h, #16
-; CHECK-SD-NEXT:    shll v1.4s, v1.4h, #16
-; CHECK-SD-NEXT:    // kill: def $s0 killed $s0 killed $q0
-; CHECK-SD-NEXT:    // kill: def $s1 killed $s1 killed $q1
-; CHECK-SD-NEXT:    bl powf
-; CHECK-SD-NEXT:    bfcvt h0, s0
-; CHECK-SD-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: test_pow:
-; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
-; CHECK-GI-NEXT:    fcvt s0, h0
-; CHECK-GI-NEXT:    fcvt s1, h1
-; CHECK-GI-NEXT:    bl powf
-; CHECK-GI-NEXT:    bfcvt h0, s0
-; CHECK-GI-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
-; CHECK-GI-NEXT:    ret
+; CHECK-BF16-LABEL: test_pow:
+; CHECK-BF16:       // %bb.0:
+; CHECK-BF16-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-BF16-NEXT:    // kill: def $h1 killed $h1 def $d1
+; CHECK-BF16-NEXT:    // kill: def $h0 killed $h0 def $d0
+; CHECK-BF16-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-BF16-NEXT:    shll v1.4s, v1.4h, #16
+; CHECK-BF16-NEXT:    // kill: def $s0 killed $s0 killed $q0
+; CHECK-BF16-NEXT:    // kill: def $s1 killed $s1 killed $q1
+; CHECK-BF16-NEXT:    bl powf
+; CHECK-BF16-NEXT:    bfcvt h0, s0
+; CHECK-BF16-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-BF16-NEXT:    ret
   %r = call bfloat @llvm.pow.f16(bfloat %a, bfloat %b)
   ret bfloat %r
 }
@@ -1973,32 +1615,23 @@ define bfloat @test_exp(bfloat %a) #0 {
 ; CHECK-CVT-NEXT:    mov w8, #32767 // =0x7fff
 ; CHECK-CVT-NEXT:    ubfx w10, w9, #16, #1
 ; CHECK-CVT-NEXT:    add w8, w9, w8
-; CHECK-CVT-NEXT:    add w8, w10, w8
-; CHECK-CVT-NEXT:    lsr w8, w8, #16
-; CHECK-CVT-NEXT:    fmov s0, w8
-; CHECK-CVT-NEXT:    // kill: def $h0 killed $h0 killed $s0
-; CHECK-CVT-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
-; CHECK-CVT-NEXT:    ret
-;
-; CHECK-SD-LABEL: test_exp:
-; CHECK-SD:       // %bb.0:
-; CHECK-SD-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
-; CHECK-SD-NEXT:    // kill: def $h0 killed $h0 def $d0
-; CHECK-SD-NEXT:    shll v0.4s, v0.4h, #16
-; CHECK-SD-NEXT:    // kill: def $s0 killed $s0 killed $q0
-; CHECK-SD-NEXT:    bl expf
-; CHECK-SD-NEXT:    bfcvt h0, s0
-; CHECK-SD-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
-; CHECK-SD-NEXT:    ret
+; CHECK-CVT-NEXT:    add w8, w10, w8
+; CHECK-CVT-NEXT:    lsr w8, w8, #16
+; CHECK-CVT-NEXT:    fmov s0, w8
+; CHECK-CVT-NEXT:    // kill: def $h0 killed $h0 killed $s0
+; CHECK-CVT-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-CVT-NEXT:    ret
 ;
-; CHECK-GI-LABEL: test_exp:
-; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
-; CHECK-GI-NEXT:    fcvt s0, h0
-; CHECK-GI-NEXT:    bl expf
-; CHECK-GI-NEXT:    bfcvt h0, s0
-; CHECK-GI-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
-; CHECK-GI-NEXT:    ret
+; CHECK-BF16-LABEL: test_exp:
+; CHECK-BF16:       // %bb.0:
+; CHECK-BF16-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-BF16-NEXT:    // kill: def $h0 killed $h0 def $d0
+; CHECK-BF16-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-BF16-NEXT:    // kill: def $s0 killed $s0 killed $q0
+; CHECK-BF16-NEXT:    bl expf
+; CHECK-BF16-NEXT:    bfcvt h0, s0
+; CHECK-BF16-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-BF16-NEXT:    ret
   %r = call bfloat @llvm.exp.f16(bfloat %a)
   ret bfloat %r
 }
@@ -2022,25 +1655,16 @@ define bfloat @test_exp2(bfloat %a) #0 {
 ; CHECK-CVT-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
 ; CHECK-CVT-NEXT:    ret
 ;
-; CHECK-SD-LABEL: test_exp2:
-; CHECK-SD:       // %bb.0:
-; CHECK-SD-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
-; CHECK-SD-NEXT:    // kill: def $h0 killed $h0 def $d0
-; CHECK-SD-NEXT:    shll v0.4s, v0.4h, #16
-; CHECK-SD-NEXT:    // kill: def $s0 killed $s0 killed $q0
-; CHECK-SD-NEXT:    bl exp2f
-; CHECK-SD-NEXT:    bfcvt h0, s0
-; CHECK-SD-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: test_exp2:
-; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
-; CHECK-GI-NEXT:    fcvt s0, h0
-; CHECK-GI-NEXT:    bl exp2f
-; CHECK-GI-NEXT:    bfcvt h0, s0
-; CHECK-GI-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
-; CHECK-GI-NEXT:    ret
+; CHECK-BF16-LABEL: test_exp2:
+; CHECK-BF16:       // %bb.0:
+; CHECK-BF16-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-BF16-NEXT:    // kill: def $h0 killed $h0 def $d0
+; CHECK-BF16-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-BF16-NEXT:    // kill: def $s0 killed $s0 killed $q0
+; CHECK-BF16-NEXT:    bl exp2f
+; CHECK-BF16-NEXT:    bfcvt h0, s0
+; CHECK-BF16-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-BF16-NEXT:    ret
   %r = call bfloat @llvm.exp2.f16(bfloat %a)
   ret bfloat %r
 }
@@ -2064,25 +1688,16 @@ define bfloat @test_log(bfloat %a) #0 {
 ; CHECK-CVT-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
 ; CHECK-CVT-NEXT:    ret
 ;
-; CHECK-SD-LABEL: test_log:
-; CHECK-SD:       // %bb.0:
-; CHECK-SD-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
-; CHECK-SD-NEXT:    // kill: def $h0 killed $h0 def $d0
-; CHECK-SD-NEXT:    shll v0.4s, v0.4h, #16
-; CHECK-SD-NEXT:    // kill: def $s0 killed $s0 killed $q0
-; CHECK-SD-NEXT:    bl logf
-; CHECK-SD-NEXT:    bfcvt h0, s0
-; CHECK-SD-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: test_log:
-; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
-; CHECK-GI-NEXT:    fcvt s0, h0
-; CHECK-GI-NEXT:    bl logf
-; CHECK-GI-NEXT:    bfcvt h0, s0
-; CHECK-GI-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
-; CHECK-GI-NEXT:    ret
+; CHECK-BF16-LABEL: test_log:
+; CHECK-BF16:       // %bb.0:
+; CHECK-BF16-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-BF16-NEXT:    // kill: def $h0 killed $h0 def $d0
+; CHECK-BF16-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-BF16-NEXT:    // kill: def $s0 killed $s0 killed $q0
+; CHECK-BF16-NEXT:    bl logf
+; CHECK-BF16-NEXT:    bfcvt h0, s0
+; CHECK-BF16-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-BF16-NEXT:    ret
   %r = call bfloat @llvm.log.f16(bfloat %a)
   ret bfloat %r
 }
@@ -2106,25 +1721,16 @@ define bfloat @test_log10(bfloat %a) #0 {
 ; CHECK-CVT-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
 ; CHECK-CVT-NEXT:    ret
 ;
-; CHECK-SD-LABEL: test_log10:
-; CHECK-SD:       // %bb.0:
-; CHECK-SD-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
-; CHECK-SD-NEXT:    // kill: def $h0 killed $h0 def $d0
-; CHECK-SD-NEXT:    shll v0.4s, v0.4h, #16
-; CHECK-SD-NEXT:    // kill: def $s0 killed $s0 killed $q0
-; CHECK-SD-NEXT:    bl log10f
-; CHECK-SD-NEXT:    bfcvt h0, s0
-; CHECK-SD-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: test_log10:
-; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
-; CHECK-GI-NEXT:    fcvt s0, h0
-; CHECK-GI-NEXT:    bl log10f
-; CHECK-GI-NEXT:    bfcvt h0, s0
-; CHECK-GI-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
-; CHECK-GI-NEXT:    ret
+; CHECK-BF16-LABEL: test_log10:
+; CHECK-BF16:       // %bb.0:
+; CHECK-BF16-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-BF16-NEXT:    // kill: def $h0 killed $h0 def $d0
+; CHECK-BF16-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-BF16-NEXT:    // kill: def $s0 killed $s0 killed $q0
+; CHECK-BF16-NEXT:    bl log10f
+; CHECK-BF16-NEXT:    bfcvt h0, s0
+; CHECK-BF16-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-BF16-NEXT:    ret
   %r = call bfloat @llvm.log10.f16(bfloat %a)
   ret bfloat %r
 }
@@ -2148,25 +1754,16 @@ define bfloat @test_log2(bfloat %a) #0 {
 ; CHECK-CVT-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
 ; CHECK-CVT-NEXT:    ret
 ;
-; CHECK-SD-LABEL: test_log2:
-; CHECK-SD:       // %bb.0:
-; CHECK-SD-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
-; CHECK-SD-NEXT:    // kill: def $h0 killed $h0 def $d0
-; CHECK-SD-NEXT:    shll v0.4s, v0.4h, #16
-; CHECK-SD-NEXT:    // kill: def $s0 killed $s0 killed $q0
-; CHECK-SD-NEXT:    bl log2f
-; CHECK-SD-NEXT:    bfcvt h0, s0
-; CHECK-SD-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: test_log2:
-; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
-; CHECK-GI-NEXT:    fcvt s0, h0
-; CHECK-GI-NEXT:    bl log2f
-; CHECK-GI-NEXT:    bfcvt h0, s0
-; CHECK-GI-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
-; CHECK-GI-NEXT:    ret
+; CHECK-BF16-LABEL: test_log2:
+; CHECK-BF16:       // %bb.0:
+; CHECK-BF16-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-BF16-NEXT:    // kill: def $h0 killed $h0 def $d0
+; CHECK-BF16-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-BF16-NEXT:    // kill: def $s0 killed $s0 killed $q0
+; CHECK-BF16-NEXT:    bl log2f
+; CHECK-BF16-NEXT:    bfcvt h0, s0
+; CHECK-BF16-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-BF16-NEXT:    ret
   %r = call bfloat @llvm.log2.f16(bfloat %a)
   ret bfloat %r
 }
@@ -2191,49 +1788,30 @@ define bfloat @test_fma(bfloat %a, bfloat %b, bfloat %c) #0 {
 ; CHECK-CVT-NEXT:    // kill: def $h0 killed $h0 killed $s0
 ; CHECK-CVT-NEXT:    ret
 ;
-; CHECK-SD-LABEL: test_fma:
-; CHECK-SD:       // %bb.0:
-; CHECK-SD-NEXT:    // kill: def $h2 killed $h2 def $d2
-; CHECK-SD-NEXT:    // kill: def $h1 killed $h1 def $d1
-; CHECK-SD-NEXT:    // kill: def $h0 killed $h0 def $d0
-; CHECK-SD-NEXT:    shll v2.4s, v2.4h, #16
-; CHECK-SD-NEXT:    shll v1.4s, v1.4h, #16
-; CHECK-SD-NEXT:    shll v0.4s, v0.4h, #16
-; CHECK-SD-NEXT:    fmadd s0, s0, s1, s2
-; CHECK-SD-NEXT:    bfcvt h0, s0
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: test_fma:
-; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    fmadd h0, h0, h1, h2
-; CHECK-GI-NEXT:    ret
+; CHECK-BF16-LABEL: test_fma:
+; CHECK-BF16:       // %bb.0:
+; CHECK-BF16-NEXT:    // kill: def $h2 killed $h2 def $d2
+; CHECK-BF16-NEXT:    // kill: def $h1 killed $h1 def $d1
+; CHECK-BF16-NEXT:    // kill: def $h0 killed $h0 def $d0
+; CHECK-BF16-NEXT:    shll v2.4s, v2.4h, #16
+; CHECK-BF16-NEXT:    shll v1.4s, v1.4h, #16
+; CHECK-BF16-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-BF16-NEXT:    fmadd s0, s0, s1, s2
+; CHECK-BF16-NEXT:    bfcvt h0, s0
+; CHECK-BF16-NEXT:    ret
   %r = call bfloat @llvm.fma.f16(bfloat %a, bfloat %b, bfloat %c)
   ret bfloat %r
 }
 
 define bfloat @test_fabs(bfloat %a) #0 {
-; CHECK-CVT-LABEL: test_fabs:
-; CHECK-CVT:       // %bb.0:
-; CHECK-CVT-NEXT:    // kill: def $h0 killed $h0 def $s0
-; CHECK-CVT-NEXT:    fmov w8, s0
-; CHECK-CVT-NEXT:    and w8, w8, #0x7fff
-; CHECK-CVT-NEXT:    fmov s0, w8
-; CHECK-CVT-NEXT:    // kill: def $h0 killed $h0 killed $s0
-; CHECK-CVT-NEXT:    ret
-;
-; CHECK-SD-LABEL: test_fabs:
-; CHECK-SD:       // %bb.0:
-; CHECK-SD-NEXT:    // kill: def $h0 killed $h0 def $s0
-; CHECK-SD-NEXT:    fmov w8, s0
-; CHECK-SD-NEXT:    and w8, w8, #0x7fff
-; CHECK-SD-NEXT:    fmov s0, w8
-; CHECK-SD-NEXT:    // kill: def $h0 killed $h0 killed $s0
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: test_fabs:
-; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    fabs h0, h0
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: test_fabs:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    // kill: def $h0 killed $h0 def $s0
+; CHECK-NEXT:    fmov w8, s0
+; CHECK-NEXT:    and w8, w8, #0x7fff
+; CHECK-NEXT:    fmov s0, w8
+; CHECK-NEXT:    // kill: def $h0 killed $h0 killed $s0
+; CHECK-NEXT:    ret
   %r = call bfloat @llvm.fabs.f16(bfloat %a)
   ret bfloat %r
 }
@@ -2256,20 +1834,15 @@ define bfloat @test_minnum(bfloat %a, bfloat %b) #0 {
 ; CHECK-CVT-NEXT:    // kill: def $h0 killed $h0 killed $s0
 ; CHECK-CVT-NEXT:    ret
 ;
-; CHECK-SD-LABEL: test_minnum:
-; CHECK-SD:       // %bb.0:
-; CHECK-SD-NEXT:    // kill: def $h1 killed $h1 def $d1
-; CHECK-SD-NEXT:    // kill: def $h0 killed $h0 def $d0
-; CHECK-SD-NEXT:    shll v1.4s, v1.4h, #16
-; CHECK-SD-NEXT:    shll v0.4s, v0.4h, #16
-; CHECK-SD-NEXT:    fminnm s0, s0, s1
-; CHECK-SD-NEXT:    bfcvt h0, s0
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: test_minnum:
-; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    fminnm h0, h0, h1
-; CHECK-GI-NEXT:    ret
+; CHECK-BF16-LABEL: test_minnum:
+; CHECK-BF16:       // %bb.0:
+; CHECK-BF16-NEXT:    // kill: def $h1 killed $h1 def $d1
+; CHECK-BF16-NEXT:    // kill: def $h0 killed $h0 def $d0
+; CHECK-BF16-NEXT:    shll v1.4s, v1.4h, #16
+; CHECK-BF16-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-BF16-NEXT:    fminnm s0, s0, s1
+; CHECK-BF16-NEXT:    bfcvt h0, s0
+; CHECK-BF16-NEXT:    ret
   %r = call bfloat @llvm.minnum.f16(bfloat %a, bfloat %b)
   ret bfloat %r
 }
@@ -2292,20 +1865,15 @@ define bfloat @test_maxnum(bfloat %a, bfloat %b) #0 {
 ; CHECK-CVT-NEXT:    // kill: def $h0 killed $h0 killed $s0
 ; CHECK-CVT-NEXT:    ret
 ;
-; CHECK-SD-LABEL: test_maxnum:
-; CHECK-SD:       // %bb.0:
-; CHECK-SD-NEXT:    // kill: def $h1 killed $h1 def $d1
-; CHECK-SD-NEXT:    // kill: def $h0 killed $h0 def $d0
-; CHECK-SD-NEXT:    shll v1.4s, v1.4h, #16
-; CHECK-SD-NEXT:    shll v0.4s, v0.4h, #16
-; CHECK-SD-NEXT:    fmaxnm s0, s0, s1
-; CHECK-SD-NEXT:    bfcvt h0, s0
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: test_maxnum:
-; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    fmaxnm h0, h0, h1
-; CHECK-GI-NEXT:    ret
+; CHECK-BF16-LABEL: test_maxnum:
+; CHECK-BF16:       // %bb.0:
+; CHECK-BF16-NEXT:    // kill: def $h1 killed $h1 def $d1
+; CHECK-BF16-NEXT:    // kill: def $h0 killed $h0 def $d0
+; CHECK-BF16-NEXT:    shll v1.4s, v1.4h, #16
+; CHECK-BF16-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-BF16-NEXT:    fmaxnm s0, s0, s1
+; CHECK-BF16-NEXT:    bfcvt h0, s0
+; CHECK-BF16-NEXT:    ret
   %r = call bfloat @llvm.maxnum.f16(bfloat %a, bfloat %b)
   ret bfloat %r
 }
@@ -2338,11 +1906,11 @@ define bfloat @test_copysign(bfloat %a, bfloat %b) #0 {
 ;
 ; CHECK-GI-LABEL: test_copysign:
 ; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    mvni v2.4h, #128, lsl #8
-; CHECK-GI-NEXT:    // kill: def $h0 killed $h0 def $d0
-; CHECK-GI-NEXT:    // kill: def $h1 killed $h1 def $d1
-; CHECK-GI-NEXT:    bif v0.8b, v1.8b, v2.8b
-; CHECK-GI-NEXT:    // kill: def $h0 killed $h0 killed $d0
+; CHECK-GI-NEXT:    mvni v2.8h, #128, lsl #8
+; CHECK-GI-NEXT:    // kill: def $h0 killed $h0 def $q0
+; CHECK-GI-NEXT:    // kill: def $h1 killed $h1 def $q1
+; CHECK-GI-NEXT:    bif v0.16b, v1.16b, v2.16b
+; CHECK-GI-NEXT:    // kill: def $h0 killed $h0 killed $q0
 ; CHECK-GI-NEXT:    ret
   %r = call bfloat @llvm.copysign.f16(bfloat %a, bfloat %b)
   ret bfloat %r
@@ -2488,18 +2056,13 @@ define bfloat @test_floor(bfloat %a) #0 {
 ; CHECK-CVT-NEXT:    // kill: def $h0 killed $h0 killed $s0
 ; CHECK-CVT-NEXT:    ret
 ;
-; CHECK-SD-LABEL: test_floor:
-; CHECK-SD:       // %bb.0:
-; CHECK-SD-NEXT:    // kill: def $h0 killed $h0 def $d0
-; CHECK-SD-NEXT:    shll v0.4s, v0.4h, #16
-; CHECK-SD-NEXT:    frintm s0, s0
-; CHECK-SD-NEXT:    bfcvt h0, s0
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: test_floor:
-; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    frintm h0, h0
-; CHECK-GI-NEXT:    ret
+; CHECK-BF16-LABEL: test_floor:
+; CHECK-BF16:       // %bb.0:
+; CHECK-BF16-NEXT:    // kill: def $h0 killed $h0 def $d0
+; CHECK-BF16-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-BF16-NEXT:    frintm s0, s0
+; CHECK-BF16-NEXT:    bfcvt h0, s0
+; CHECK-BF16-NEXT:    ret
   %r = call bfloat @llvm.floor.f16(bfloat %a)
   ret bfloat %r
 }
@@ -2520,18 +2083,13 @@ define bfloat @test_ceil(bfloat %a) #0 {
 ; CHECK-CVT-NEXT:    // kill: def $h0 killed $h0 killed $s0
 ; CHECK-CVT-NEXT:    ret
 ;
-; CHECK-SD-LABEL: test_ceil:
-; CHECK-SD:       // %bb.0:
-; CHECK-SD-NEXT:    // kill: def $h0 killed $h0 def $d0
-; CHECK-SD-NEXT:    shll v0.4s, v0.4h, #16
-; CHECK-SD-NEXT:    frintp s0, s0
-; CHECK-SD-NEXT:    bfcvt h0, s0
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: test_ceil:
-; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    frintp h0, h0
-; CHECK-GI-NEXT:    ret
+; CHECK-BF16-LABEL: test_ceil:
+; CHECK-BF16:       // %bb.0:
+; CHECK-BF16-NEXT:    // kill: def $h0 killed $h0 def $d0
+; CHECK-BF16-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-BF16-NEXT:    frintp s0, s0
+; CHECK-BF16-NEXT:    bfcvt h0, s0
+; CHECK-BF16-NEXT:    ret
   %r = call bfloat @llvm.ceil.f16(bfloat %a)
   ret bfloat %r
 }
@@ -2552,18 +2110,13 @@ define bfloat @test_trunc(bfloat %a) #0 {
 ; CHECK-CVT-NEXT:    // kill: def $h0 killed $h0 killed $s0
 ; CHECK-CVT-NEXT:    ret
 ;
-; CHECK-SD-LABEL: test_trunc:
-; CHECK-SD:       // %bb.0:
-; CHECK-SD-NEXT:    // kill: def $h0 killed $h0 def $d0
-; CHECK-SD-NEXT:    shll v0.4s, v0.4h, #16
-; CHECK-SD-NEXT:    frintz s0, s0
-; CHECK-SD-NEXT:    bfcvt h0, s0
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: test_trunc:
-; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    frintz h0, h0
-; CHECK-GI-NEXT:    ret
+; CHECK-BF16-LABEL: test_trunc:
+; CHECK-BF16:       // %bb.0:
+; CHECK-BF16-NEXT:    // kill: def $h0 killed $h0 def $d0
+; CHECK-BF16-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-BF16-NEXT:    frintz s0, s0
+; CHECK-BF16-NEXT:    bfcvt h0, s0
+; CHECK-BF16-NEXT:    ret
   %r = call bfloat @llvm.trunc.f16(bfloat %a)
   ret bfloat %r
 }
@@ -2584,18 +2137,13 @@ define bfloat @test_rint(bfloat %a) #0 {
 ; CHECK-CVT-NEXT:    // kill: def $h0 killed $h0 killed $s0
 ; CHECK-CVT-NEXT:    ret
 ;
-; CHECK-SD-LABEL: test_rint:
-; CHECK-SD:       // %bb.0:
-; CHECK-SD-NEXT:    // kill: def $h0 killed $h0 def $d0
-; CHECK-SD-NEXT:    shll v0.4s, v0.4h, #16
-; CHECK-SD-NEXT:    frintx s0, s0
-; CHECK-SD-NEXT:    bfcvt h0, s0
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: test_rint:
-; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    frintx h0, h0
-; CHECK-GI-NEXT:    ret
+; CHECK-BF16-LABEL: test_rint:
+; CHECK-BF16:       // %bb.0:
+; CHECK-BF16-NEXT:    // kill: def $h0 killed $h0 def $d0
+; CHECK-BF16-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-BF16-NEXT:    frintx s0, s0
+; CHECK-BF16-NEXT:    bfcvt h0, s0
+; CHECK-BF16-NEXT:    ret
   %r = call bfloat @llvm.rint.f16(bfloat %a)
   ret bfloat %r
 }
@@ -2616,18 +2164,13 @@ define bfloat @test_nearbyint(bfloat %a) #0 {
 ; CHECK-CVT-NEXT:    // kill: def $h0 killed $h0 killed $s0
 ; CHECK-CVT-NEXT:    ret
 ;
-; CHECK-SD-LABEL: test_nearbyint:
-; CHECK-SD:       // %bb.0:
-; CHECK-SD-NEXT:    // kill: def $h0 killed $h0 def $d0
-; CHECK-SD-NEXT:    shll v0.4s, v0.4h, #16
-; CHECK-SD-NEXT:    frinti s0, s0
-; CHECK-SD-NEXT:    bfcvt h0, s0
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: test_nearbyint:
-; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    frinti h0, h0
-; CHECK-GI-NEXT:    ret
+; CHECK-BF16-LABEL: test_nearbyint:
+; CHECK-BF16:       // %bb.0:
+; CHECK-BF16-NEXT:    // kill: def $h0 killed $h0 def $d0
+; CHECK-BF16-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-BF16-NEXT:    frinti s0, s0
+; CHECK-BF16-NEXT:    bfcvt h0, s0
+; CHECK-BF16-NEXT:    ret
   %r = call bfloat @llvm.nearbyint.f16(bfloat %a)
   ret bfloat %r
 }
@@ -2648,18 +2191,13 @@ define bfloat @test_round(bfloat %a) #0 {
 ; CHECK-CVT-NEXT:    // kill: def $h0 killed $h0 killed $s0
 ; CHECK-CVT-NEXT:    ret
 ;
-; CHECK-SD-LABEL: test_round:
-; CHECK-SD:       // %bb.0:
-; CHECK-SD-NEXT:    // kill: def $h0 killed $h0 def $d0
-; CHECK-SD-NEXT:    shll v0.4s, v0.4h, #16
-; CHECK-SD-NEXT:    frinta s0, s0
-; CHECK-SD-NEXT:    bfcvt h0, s0
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: test_round:
-; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    frinta h0, h0
-; CHECK-GI-NEXT:    ret
+; CHECK-BF16-LABEL: test_round:
+; CHECK-BF16:       // %bb.0:
+; CHECK-BF16-NEXT:    // kill: def $h0 killed $h0 def $d0
+; CHECK-BF16-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-BF16-NEXT:    frinta s0, s0
+; CHECK-BF16-NEXT:    bfcvt h0, s0
+; CHECK-BF16-NEXT:    ret
   %r = call bfloat @llvm.round.f16(bfloat %a)
   ret bfloat %r
 }
@@ -2680,18 +2218,13 @@ define bfloat @test_roundeven(bfloat %a) #0 {
 ; CHECK-CVT-NEXT:    // kill: def $h0 killed $h0 killed $s0
 ; CHECK-CVT-NEXT:    ret
 ;
-; CHECK-SD-LABEL: test_roundeven:
-; CHECK-SD:       // %bb.0:
-; CHECK-SD-NEXT:    // kill: def $h0 killed $h0 def $d0
-; CHECK-SD-NEXT:    shll v0.4s, v0.4h, #16
-; CHECK-SD-NEXT:    frintn s0, s0
-; CHECK-SD-NEXT:    bfcvt h0, s0
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: test_roundeven:
-; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    frintn h0, h0
-; CHECK-GI-NEXT:    ret
+; CHECK-BF16-LABEL: test_roundeven:
+; CHECK-BF16:       // %bb.0:
+; CHECK-BF16-NEXT:    // kill: def $h0 killed $h0 def $d0
+; CHECK-BF16-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-BF16-NEXT:    frintn s0, s0
+; CHECK-BF16-NEXT:    bfcvt h0, s0
+; CHECK-BF16-NEXT:    ret
   %r = call bfloat @llvm.roundeven.f16(bfloat %a)
   ret bfloat %r
 }
@@ -2724,26 +2257,20 @@ define bfloat @test_fmuladd(bfloat %a, bfloat %b, bfloat %c) #0 {
 ; CHECK-CVT-NEXT:    // kill: def $h0 killed $h0 killed $s0
 ; CHECK-CVT-NEXT:    ret
 ;
-; CHECK-SD-LABEL: test_fmuladd:
-; CHECK-SD:       // %bb.0:
-; CHECK-SD-NEXT:    // kill: def $h1 killed $h1 def $d1
-; CHECK-SD-NEXT:    // kill: def $h0 killed $h0 def $d0
-; CHECK-SD-NEXT:    // kill: def $h2 killed $h2 def $d2
-; CHECK-SD-NEXT:    shll v1.4s, v1.4h, #16
-; CHECK-SD-NEXT:    shll v0.4s, v0.4h, #16
-; CHECK-SD-NEXT:    fmul s0, s0, s1
-; CHECK-SD-NEXT:    shll v1.4s, v2.4h, #16
-; CHECK-SD-NEXT:    bfcvt h0, s0
-; CHECK-SD-NEXT:    shll v0.4s, v0.4h, #16
-; CHECK-SD-NEXT:    fadd s0, s0, s1
-; CHECK-SD-NEXT:    bfcvt h0, s0
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: test_fmuladd:
-; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    fmul h0, h0, h1
-; CHECK-GI-NEXT:    fadd h0, h0, h2
-; CHECK-GI-NEXT:    ret
+; CHECK-BF16-LABEL: test_fmuladd:
+; CHECK-BF16:       // %bb.0:
+; CHECK-BF16-NEXT:    // kill: def $h1 killed $h1 def $d1
+; CHECK-BF16-NEXT:    // kill: def $h0 killed $h0 def $d0
+; CHECK-BF16-NEXT:    // kill: def $h2 killed $h2 def $d2
+; CHECK-BF16-NEXT:    shll v1.4s, v1.4h, #16
+; CHECK-BF16-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-BF16-NEXT:    fmul s0, s0, s1
+; CHECK-BF16-NEXT:    shll v1.4s, v2.4h, #16
+; CHECK-BF16-NEXT:    bfcvt h0, s0
+; CHECK-BF16-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-BF16-NEXT:    fadd s0, s0, s1
+; CHECK-BF16-NEXT:    bfcvt h0, s0
+; CHECK-BF16-NEXT:    ret
   %r = call bfloat @llvm.fmuladd.f16(bfloat %a, bfloat %b, bfloat %c)
   ret bfloat %r
 }
diff --git a/llvm/test/CodeGen/AArch64/cmp-to-cmn.ll b/llvm/test/CodeGen/AArch64/cmp-to-cmn.ll
index 1cc194e77b94b..e87d43161a895 100644
--- a/llvm/test/CodeGen/AArch64/cmp-to-cmn.ll
+++ b/llvm/test/CodeGen/AArch64/cmp-to-cmn.ll
@@ -31,7 +31,6 @@ entry:
 define i1 @test_EQ_IlsEbT(i64 %a, i16 %b) {
 ; CHECK-LABEL: test_EQ_IlsEbT:
 ; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    // kill: def $w1 killed $w1 def $x1
 ; CHECK-NEXT:    cmn x0, w1, sxth
 ; CHECK-NEXT:    cset w0, eq
 ; CHECK-NEXT:    ret
@@ -45,7 +44,6 @@ entry:
 define i1 @test_EQ_IlcEbT(i64 %a, i8 %b) {
 ; CHECK-LABEL: test_EQ_IlcEbT:
 ; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    // kill: def $w1 killed $w1 def $x1
 ; CHECK-NEXT:    cmn x0, w1, uxtb
 ; CHECK-NEXT:    cset w0, eq
 ; CHECK-NEXT:    ret
@@ -110,7 +108,6 @@ entry:
 define i1 @test_EQ_IslEbT(i16 %a, i64 %b) {
 ; CHECK-LABEL: test_EQ_IslEbT:
 ; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    // kill: def $w0 killed $w0 def $x0
 ; CHECK-NEXT:    cmn x1, w0, sxth
 ; CHECK-NEXT:    cset w0, eq
 ; CHECK-NEXT:    ret
@@ -167,7 +164,6 @@ entry:
 define i1 @test_EQ_IclEbT(i8 %a, i64 %b) {
 ; CHECK-LABEL: test_EQ_IclEbT:
 ; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    // kill: def $w0 killed $w0 def $x0
 ; CHECK-NEXT:    cmn x1, w0, uxtb
 ; CHECK-NEXT:    cset w0, eq
 ; CHECK-NEXT:    ret
@@ -249,7 +245,6 @@ entry:
 define i1 @test_NE_IlsEbT(i64 %a, i16 %b) {
 ; CHECK-LABEL: test_NE_IlsEbT:
 ; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    // kill: def $w1 killed $w1 def $x1
 ; CHECK-NEXT:    cmn x0, w1, sxth
 ; CHECK-NEXT:    cset w0, ne
 ; CHECK-NEXT:    ret
@@ -263,7 +258,6 @@ entry:
 define i1 @test_NE_IlcEbT(i64 %a, i8 %b) {
 ; CHECK-LABEL: test_NE_IlcEbT:
 ; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    // kill: def $w1 killed $w1 def $x1
 ; CHECK-NEXT:    cmn x0, w1, uxtb
 ; CHECK-NEXT:    cset w0, ne
 ; CHECK-NEXT:    ret
@@ -328,7 +322,6 @@ entry:
 define i1 @test_NE_IslEbT(i16 %a, i64 %b) {
 ; CHECK-LABEL: test_NE_IslEbT:
 ; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    // kill: def $w0 killed $w0 def $x0
 ; CHECK-NEXT:    cmn x1, w0, sxth
 ; CHECK-NEXT:    cset w0, ne
 ; CHECK-NEXT:    ret
@@ -385,7 +378,6 @@ entry:
 define i1 @test_NE_IclEbT(i8 %a, i64 %b) {
 ; CHECK-LABEL: test_NE_IclEbT:
 ; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    // kill: def $w0 killed $w0 def $x0
 ; CHECK-NEXT:    cmn x1, w0, uxtb
 ; CHECK-NEXT:    cset w0, ne
 ; CHECK-NEXT:    ret
diff --git a/llvm/test/CodeGen/AArch64/cmpxchg-idioms.ll b/llvm/test/CodeGen/AArch64/cmpxchg-idioms.ll
index 186d191444feb..b7817ebe59b9b 100644
--- a/llvm/test/CodeGen/AArch64/cmpxchg-idioms.ll
+++ b/llvm/test/CodeGen/AArch64/cmpxchg-idioms.ll
@@ -53,7 +53,6 @@ define i1 @test_return_bool(ptr %value, i8 %oldValue, i8 %newValue) {
 ; CHECK-LABEL: test_return_bool:
 ; CHECK:       ; %bb.0:
 ; CHECK-NEXT:    and w8, w1, #0xff
-; CHECK-NEXT:    ; kill: def $w2 killed $w2 def $x2
 ; CHECK-NEXT:  LBB1_1: ; %cmpxchg.start
 ; CHECK-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    ldaxrb w9, [x0]
diff --git a/llvm/test/CodeGen/AArch64/coalescer-drop-subreg-to-reg-imm-ops.mir b/llvm/test/CodeGen/AArch64/coalescer-drop-subreg-to-reg-imm-ops.mir
index f54c612303c0e..b60f4ffdd880b 100644
--- a/llvm/test/CodeGen/AArch64/coalescer-drop-subreg-to-reg-imm-ops.mir
+++ b/llvm/test/CodeGen/AArch64/coalescer-drop-subreg-to-reg-imm-ops.mir
@@ -1,5 +1,6 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 4
 # RUN: llc -mtriple=arm64-apple-macosx -mcpu=apple-m1 -verify-coalescing -run-pass=register-coalescer -o - %s | FileCheck %s
+# RUN: llc -mtriple=arm64-apple-macosx -mcpu=apple-m1 -verify-coalescing -passes=register-coalescer -o - %s | FileCheck %s
 
 # Hits assert "Trying to add an operand to a machine instr that is
 # already done!" when rematerializing during greedy. This was because
diff --git a/llvm/test/CodeGen/AArch64/const-shift-of-constmasked.ll b/llvm/test/CodeGen/AArch64/const-shift-of-constmasked.ll
index 66a6745cda8f7..1fffcdda4b416 100644
--- a/llvm/test/CodeGen/AArch64/const-shift-of-constmasked.ll
+++ b/llvm/test/CodeGen/AArch64/const-shift-of-constmasked.ll
@@ -190,8 +190,7 @@ define i8 @test_i8_224_mask_ashr_6(i8 %a0) {
 define i8 @test_i8_7_mask_shl_1(i8 %a0) {
 ; CHECK-LABEL: test_i8_7_mask_shl_1:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    and w8, w0, #0x7
-; CHECK-NEXT:    lsl w0, w8, #1
+; CHECK-NEXT:    ubfiz w0, w0, #1, #3
 ; CHECK-NEXT:    ret
   %t0 = and i8 %a0, 7
   %t1 = shl i8 %t0, 1
@@ -200,8 +199,7 @@ define i8 @test_i8_7_mask_shl_1(i8 %a0) {
 define i8 @test_i8_7_mask_shl_4(i8 %a0) {
 ; CHECK-LABEL: test_i8_7_mask_shl_4:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    and w8, w0, #0x7
-; CHECK-NEXT:    lsl w0, w8, #4
+; CHECK-NEXT:    ubfiz w0, w0, #4, #3
 ; CHECK-NEXT:    ret
   %t0 = and i8 %a0, 7
   %t1 = shl i8 %t0, 4
@@ -229,8 +227,8 @@ define i8 @test_i8_7_mask_shl_6(i8 %a0) {
 define i8 @test_i8_28_mask_shl_1(i8 %a0) {
 ; CHECK-LABEL: test_i8_28_mask_shl_1:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    and w8, w0, #0x1c
-; CHECK-NEXT:    lsl w0, w8, #1
+; CHECK-NEXT:    lsl w8, w0, #1
+; CHECK-NEXT:    and w0, w8, #0x38
 ; CHECK-NEXT:    ret
   %t0 = and i8 %a0, 28
   %t1 = shl i8 %t0, 1
@@ -239,8 +237,8 @@ define i8 @test_i8_28_mask_shl_1(i8 %a0) {
 define i8 @test_i8_28_mask_shl_2(i8 %a0) {
 ; CHECK-LABEL: test_i8_28_mask_shl_2:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    and w8, w0, #0x1c
-; CHECK-NEXT:    lsl w0, w8, #2
+; CHECK-NEXT:    lsl w8, w0, #2
+; CHECK-NEXT:    and w0, w8, #0x70
 ; CHECK-NEXT:    ret
   %t0 = and i8 %a0, 28
   %t1 = shl i8 %t0, 2
@@ -249,8 +247,8 @@ define i8 @test_i8_28_mask_shl_2(i8 %a0) {
 define i8 @test_i8_28_mask_shl_3(i8 %a0) {
 ; CHECK-LABEL: test_i8_28_mask_shl_3:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    and w8, w0, #0x1c
-; CHECK-NEXT:    lsl w0, w8, #3
+; CHECK-NEXT:    lsl w8, w0, #3
+; CHECK-NEXT:    and w0, w8, #0xe0
 ; CHECK-NEXT:    ret
   %t0 = and i8 %a0, 28
   %t1 = shl i8 %t0, 3
@@ -259,8 +257,8 @@ define i8 @test_i8_28_mask_shl_3(i8 %a0) {
 define i8 @test_i8_28_mask_shl_4(i8 %a0) {
 ; CHECK-LABEL: test_i8_28_mask_shl_4:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    and w8, w0, #0xc
-; CHECK-NEXT:    lsl w0, w8, #4
+; CHECK-NEXT:    lsl w8, w0, #4
+; CHECK-NEXT:    and w0, w8, #0xc0
 ; CHECK-NEXT:    ret
   %t0 = and i8 %a0, 28
   %t1 = shl i8 %t0, 4
@@ -270,8 +268,8 @@ define i8 @test_i8_28_mask_shl_4(i8 %a0) {
 define i8 @test_i8_224_mask_shl_1(i8 %a0) {
 ; CHECK-LABEL: test_i8_224_mask_shl_1:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    and w8, w0, #0x60
-; CHECK-NEXT:    lsl w0, w8, #1
+; CHECK-NEXT:    lsl w8, w0, #1
+; CHECK-NEXT:    and w0, w8, #0xc0
 ; CHECK-NEXT:    ret
   %t0 = and i8 %a0, 224
   %t1 = shl i8 %t0, 1
@@ -465,8 +463,7 @@ define i16 @test_i16_65024_mask_ashr_10(i16 %a0) {
 define i16 @test_i16_127_mask_shl_1(i16 %a0) {
 ; CHECK-LABEL: test_i16_127_mask_shl_1:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    and w8, w0, #0x7f
-; CHECK-NEXT:    lsl w0, w8, #1
+; CHECK-NEXT:    ubfiz w0, w0, #1, #7
 ; CHECK-NEXT:    ret
   %t0 = and i16 %a0, 127
   %t1 = shl i16 %t0, 1
@@ -475,8 +472,7 @@ define i16 @test_i16_127_mask_shl_1(i16 %a0) {
 define i16 @test_i16_127_mask_shl_8(i16 %a0) {
 ; CHECK-LABEL: test_i16_127_mask_shl_8:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    and w8, w0, #0x7f
-; CHECK-NEXT:    lsl w0, w8, #8
+; CHECK-NEXT:    ubfiz w0, w0, #8, #7
 ; CHECK-NEXT:    ret
   %t0 = and i16 %a0, 127
   %t1 = shl i16 %t0, 8
@@ -504,8 +500,8 @@ define i16 @test_i16_127_mask_shl_10(i16 %a0) {
 define i16 @test_i16_2032_mask_shl_3(i16 %a0) {
 ; CHECK-LABEL: test_i16_2032_mask_shl_3:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    and w8, w0, #0x7f0
-; CHECK-NEXT:    lsl w0, w8, #3
+; CHECK-NEXT:    lsl w8, w0, #3
+; CHECK-NEXT:    and w0, w8, #0x3f80
 ; CHECK-NEXT:    ret
   %t0 = and i16 %a0, 2032
   %t1 = shl i16 %t0, 3
@@ -514,8 +510,8 @@ define i16 @test_i16_2032_mask_shl_3(i16 %a0) {
 define i16 @test_i16_2032_mask_shl_4(i16 %a0) {
 ; CHECK-LABEL: test_i16_2032_mask_shl_4:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    and w8, w0, #0x7f0
-; CHECK-NEXT:    lsl w0, w8, #4
+; CHECK-NEXT:    lsl w8, w0, #4
+; CHECK-NEXT:    and w0, w8, #0x7f00
 ; CHECK-NEXT:    ret
   %t0 = and i16 %a0, 2032
   %t1 = shl i16 %t0, 4
@@ -524,8 +520,8 @@ define i16 @test_i16_2032_mask_shl_4(i16 %a0) {
 define i16 @test_i16_2032_mask_shl_5(i16 %a0) {
 ; CHECK-LABEL: test_i16_2032_mask_shl_5:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    and w8, w0, #0x7f0
-; CHECK-NEXT:    lsl w0, w8, #5
+; CHECK-NEXT:    lsl w8, w0, #5
+; CHECK-NEXT:    and w0, w8, #0xfe00
 ; CHECK-NEXT:    ret
   %t0 = and i16 %a0, 2032
   %t1 = shl i16 %t0, 5
@@ -534,8 +530,8 @@ define i16 @test_i16_2032_mask_shl_5(i16 %a0) {
 define i16 @test_i16_2032_mask_shl_6(i16 %a0) {
 ; CHECK-LABEL: test_i16_2032_mask_shl_6:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    and w8, w0, #0x3f0
-; CHECK-NEXT:    lsl w0, w8, #6
+; CHECK-NEXT:    lsl w8, w0, #6
+; CHECK-NEXT:    and w0, w8, #0xfc00
 ; CHECK-NEXT:    ret
   %t0 = and i16 %a0, 2032
   %t1 = shl i16 %t0, 6
@@ -545,8 +541,8 @@ define i16 @test_i16_2032_mask_shl_6(i16 %a0) {
 define i16 @test_i16_65024_mask_shl_1(i16 %a0) {
 ; CHECK-LABEL: test_i16_65024_mask_shl_1:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    and w8, w0, #0x7e00
-; CHECK-NEXT:    lsl w0, w8, #1
+; CHECK-NEXT:    lsl w8, w0, #1
+; CHECK-NEXT:    and w0, w8, #0xfc00
 ; CHECK-NEXT:    ret
   %t0 = and i16 %a0, 65024
   %t1 = shl i16 %t0, 1
@@ -740,8 +736,7 @@ define i32 @test_i32_4294836224_mask_ashr_18(i32 %a0) {
 define i32 @test_i32_32767_mask_shl_1(i32 %a0) {
 ; CHECK-LABEL: test_i32_32767_mask_shl_1:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    and w8, w0, #0x7fff
-; CHECK-NEXT:    lsl w0, w8, #1
+; CHECK-NEXT:    ubfiz w0, w0, #1, #15
 ; CHECK-NEXT:    ret
   %t0 = and i32 %a0, 32767
   %t1 = shl i32 %t0, 1
@@ -750,8 +745,7 @@ define i32 @test_i32_32767_mask_shl_1(i32 %a0) {
 define i32 @test_i32_32767_mask_shl_16(i32 %a0) {
 ; CHECK-LABEL: test_i32_32767_mask_shl_16:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    and w8, w0, #0x7fff
-; CHECK-NEXT:    lsl w0, w8, #16
+; CHECK-NEXT:    ubfiz w0, w0, #16, #15
 ; CHECK-NEXT:    ret
   %t0 = and i32 %a0, 32767
   %t1 = shl i32 %t0, 16
@@ -779,8 +773,8 @@ define i32 @test_i32_32767_mask_shl_18(i32 %a0) {
 define i32 @test_i32_8388352_mask_shl_7(i32 %a0) {
 ; CHECK-LABEL: test_i32_8388352_mask_shl_7:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    and w8, w0, #0x7fff00
-; CHECK-NEXT:    lsl w0, w8, #7
+; CHECK-NEXT:    lsl w8, w0, #7
+; CHECK-NEXT:    and w0, w8, #0x3fff8000
 ; CHECK-NEXT:    ret
   %t0 = and i32 %a0, 8388352
   %t1 = shl i32 %t0, 7
@@ -789,8 +783,8 @@ define i32 @test_i32_8388352_mask_shl_7(i32 %a0) {
 define i32 @test_i32_8388352_mask_shl_8(i32 %a0) {
 ; CHECK-LABEL: test_i32_8388352_mask_shl_8:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    and w8, w0, #0x7fff00
-; CHECK-NEXT:    lsl w0, w8, #8
+; CHECK-NEXT:    lsl w8, w0, #8
+; CHECK-NEXT:    and w0, w8, #0x7fff0000
 ; CHECK-NEXT:    ret
   %t0 = and i32 %a0, 8388352
   %t1 = shl i32 %t0, 8
@@ -799,8 +793,8 @@ define i32 @test_i32_8388352_mask_shl_8(i32 %a0) {
 define i32 @test_i32_8388352_mask_shl_9(i32 %a0) {
 ; CHECK-LABEL: test_i32_8388352_mask_shl_9:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    and w8, w0, #0x7fff00
-; CHECK-NEXT:    lsl w0, w8, #9
+; CHECK-NEXT:    lsl w8, w0, #9
+; CHECK-NEXT:    and w0, w8, #0xfffe0000
 ; CHECK-NEXT:    ret
   %t0 = and i32 %a0, 8388352
   %t1 = shl i32 %t0, 9
@@ -809,8 +803,8 @@ define i32 @test_i32_8388352_mask_shl_9(i32 %a0) {
 define i32 @test_i32_8388352_mask_shl_10(i32 %a0) {
 ; CHECK-LABEL: test_i32_8388352_mask_shl_10:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    and w8, w0, #0x3fff00
-; CHECK-NEXT:    lsl w0, w8, #10
+; CHECK-NEXT:    lsl w8, w0, #10
+; CHECK-NEXT:    and w0, w8, #0xfffc0000
 ; CHECK-NEXT:    ret
   %t0 = and i32 %a0, 8388352
   %t1 = shl i32 %t0, 10
@@ -820,8 +814,8 @@ define i32 @test_i32_8388352_mask_shl_10(i32 %a0) {
 define i32 @test_i32_4294836224_mask_shl_1(i32 %a0) {
 ; CHECK-LABEL: test_i32_4294836224_mask_shl_1:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    and w8, w0, #0x7ffe0000
-; CHECK-NEXT:    lsl w0, w8, #1
+; CHECK-NEXT:    lsl w8, w0, #1
+; CHECK-NEXT:    and w0, w8, #0xfffc0000
 ; CHECK-NEXT:    ret
   %t0 = and i32 %a0, 4294836224
   %t1 = shl i32 %t0, 1
@@ -1015,8 +1009,7 @@ define i64 @test_i64_18446744065119617024_mask_ashr_34(i64 %a0) {
 define i64 @test_i64_2147483647_mask_shl_1(i64 %a0) {
 ; CHECK-LABEL: test_i64_2147483647_mask_shl_1:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    and x8, x0, #0x7fffffff
-; CHECK-NEXT:    lsl x0, x8, #1
+; CHECK-NEXT:    lsl w0, w0, #1
 ; CHECK-NEXT:    ret
   %t0 = and i64 %a0, 2147483647
   %t1 = shl i64 %t0, 1
@@ -1054,8 +1047,8 @@ define i64 @test_i64_2147483647_mask_shl_34(i64 %a0) {
 define i64 @test_i64_140737488289792_mask_shl_15(i64 %a0) {
 ; CHECK-LABEL: test_i64_140737488289792_mask_shl_15:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    and x8, x0, #0x7fffffff0000
-; CHECK-NEXT:    lsl x0, x8, #15
+; CHECK-NEXT:    lsl x8, x0, #15
+; CHECK-NEXT:    and x0, x8, #0x3fffffff80000000
 ; CHECK-NEXT:    ret
   %t0 = and i64 %a0, 140737488289792
   %t1 = shl i64 %t0, 15
@@ -1064,8 +1057,8 @@ define i64 @test_i64_140737488289792_mask_shl_15(i64 %a0) {
 define i64 @test_i64_140737488289792_mask_shl_16(i64 %a0) {
 ; CHECK-LABEL: test_i64_140737488289792_mask_shl_16:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    and x8, x0, #0x7fffffff0000
-; CHECK-NEXT:    lsl x0, x8, #16
+; CHECK-NEXT:    lsl x8, x0, #16
+; CHECK-NEXT:    and x0, x8, #0x7fffffff00000000
 ; CHECK-NEXT:    ret
   %t0 = and i64 %a0, 140737488289792
   %t1 = shl i64 %t0, 16
@@ -1074,8 +1067,8 @@ define i64 @test_i64_140737488289792_mask_shl_16(i64 %a0) {
 define i64 @test_i64_140737488289792_mask_shl_17(i64 %a0) {
 ; CHECK-LABEL: test_i64_140737488289792_mask_shl_17:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    and x8, x0, #0x7fffffff0000
-; CHECK-NEXT:    lsl x0, x8, #17
+; CHECK-NEXT:    lsl x8, x0, #17
+; CHECK-NEXT:    and x0, x8, #0xfffffffe00000000
 ; CHECK-NEXT:    ret
   %t0 = and i64 %a0, 140737488289792
   %t1 = shl i64 %t0, 17
@@ -1084,8 +1077,8 @@ define i64 @test_i64_140737488289792_mask_shl_17(i64 %a0) {
 define i64 @test_i64_140737488289792_mask_shl_18(i64 %a0) {
 ; CHECK-LABEL: test_i64_140737488289792_mask_shl_18:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    and x8, x0, #0x3fffffff0000
-; CHECK-NEXT:    lsl x0, x8, #18
+; CHECK-NEXT:    lsl x8, x0, #18
+; CHECK-NEXT:    and x0, x8, #0xfffffffc00000000
 ; CHECK-NEXT:    ret
   %t0 = and i64 %a0, 140737488289792
   %t1 = shl i64 %t0, 18
@@ -1095,8 +1088,8 @@ define i64 @test_i64_140737488289792_mask_shl_18(i64 %a0) {
 define i64 @test_i64_18446744065119617024_mask_shl_1(i64 %a0) {
 ; CHECK-LABEL: test_i64_18446744065119617024_mask_shl_1:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    and x8, x0, #0x7ffffffe00000000
-; CHECK-NEXT:    lsl x0, x8, #1
+; CHECK-NEXT:    lsl x8, x0, #1
+; CHECK-NEXT:    and x0, x8, #0xfffffffc00000000
 ; CHECK-NEXT:    ret
   %t0 = and i64 %a0, 18446744065119617024
   %t1 = shl i64 %t0, 1
diff --git a/llvm/test/CodeGen/AArch64/extract-bits.ll b/llvm/test/CodeGen/AArch64/extract-bits.ll
index b87157a183835..8e822d19a19b9 100644
--- a/llvm/test/CodeGen/AArch64/extract-bits.ll
+++ b/llvm/test/CodeGen/AArch64/extract-bits.ll
@@ -492,7 +492,6 @@ define i32 @bextr64_32_b1(i64 %val, i64 %numskipbits, i8 %numlowbits) nounwind {
 ; CHECK-LABEL: bextr64_32_b1:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    mov w8, #-1 // =0xffffffff
-; CHECK-NEXT:    // kill: def $w2 killed $w2 def $x2
 ; CHECK-NEXT:    lsr x9, x0, x1
 ; CHECK-NEXT:    lsl w8, w8, w2
 ; CHECK-NEXT:    bic w0, w9, w8
@@ -512,7 +511,6 @@ define i32 @bextr64_32_b2(i64 %val, i64 %numskipbits, i8 %numlowbits) nounwind {
 ; CHECK-LABEL: bextr64_32_b2:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    mov w8, #-1 // =0xffffffff
-; CHECK-NEXT:    // kill: def $w2 killed $w2 def $x2
 ; CHECK-NEXT:    lsr x9, x0, x1
 ; CHECK-NEXT:    lsl w8, w8, w2
 ; CHECK-NEXT:    bic w0, w9, w8
@@ -552,7 +550,6 @@ define i32 @bextr32_c1_indexzext(i32 %val, i8 %numskipbits, i8 %numlowbits) noun
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    mov w8, #32 // =0x20
 ; CHECK-NEXT:    mov w9, #-1 // =0xffffffff
-; CHECK-NEXT:    // kill: def $w1 killed $w1 def $x1
 ; CHECK-NEXT:    lsr w10, w0, w1
 ; CHECK-NEXT:    sub w8, w8, w2
 ; CHECK-NEXT:    lsr w8, w9, w8
@@ -592,7 +589,6 @@ define i32 @bextr32_c3_load_indexzext(ptr %w, i8 %numskipbits, i8 %numlowbits) n
 ; CHECK-NEXT:    mov w9, #32 // =0x20
 ; CHECK-NEXT:    mov w10, #-1 // =0xffffffff
 ; CHECK-NEXT:    sub w9, w9, w2
-; CHECK-NEXT:    // kill: def $w1 killed $w1 def $x1
 ; CHECK-NEXT:    lsr w8, w8, w1
 ; CHECK-NEXT:    lsr w9, w10, w9
 ; CHECK-NEXT:    and w0, w9, w8
@@ -797,7 +793,6 @@ define i32 @bextr32_d0(i32 %val, i32 %numskipbits, i32 %numlowbits) nounwind {
 define i32 @bextr32_d1_indexzext(i32 %val, i8 %numskipbits, i8 %numlowbits) nounwind {
 ; CHECK-LABEL: bextr32_d1_indexzext:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    // kill: def $w1 killed $w1 def $x1
 ; CHECK-NEXT:    lsr w8, w0, w1
 ; CHECK-NEXT:    mov w9, #32 // =0x20
 ; CHECK-NEXT:    sub w9, w9, w2
@@ -834,7 +829,6 @@ define i32 @bextr32_d3_load_indexzext(ptr %w, i8 %numskipbits, i8 %numlowbits) n
 ; CHECK-LABEL: bextr32_d3_load_indexzext:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ldr w8, [x0]
-; CHECK-NEXT:    // kill: def $w1 killed $w1 def $x1
 ; CHECK-NEXT:    mov w9, #32 // =0x20
 ; CHECK-NEXT:    sub w9, w9, w2
 ; CHECK-NEXT:    lsr w8, w8, w1
@@ -1013,8 +1007,8 @@ define i32 @c1_i32(i32 %arg) nounwind {
 define i32 @c2_i32(i32 %arg) nounwind {
 ; CHECK-LABEL: c2_i32:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    ubfx w8, w0, #19, #10
-; CHECK-NEXT:    lsl w0, w8, #2
+; CHECK-NEXT:    lsr w8, w0, #17
+; CHECK-NEXT:    and w0, w8, #0xffc
 ; CHECK-NEXT:    ret
   %tmp0 = lshr i32 %arg, 19
   %tmp1 = and i32 %tmp0, 1023
@@ -1063,8 +1057,8 @@ define i64 @c1_i64(i64 %arg) nounwind {
 define i64 @c2_i64(i64 %arg) nounwind {
 ; CHECK-LABEL: c2_i64:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    ubfx x8, x0, #51, #10
-; CHECK-NEXT:    lsl x0, x8, #2
+; CHECK-NEXT:    lsr x8, x0, #49
+; CHECK-NEXT:    and x0, x8, #0xffc
 ; CHECK-NEXT:    ret
   %tmp0 = lshr i64 %arg, 51
   %tmp1 = and i64 %tmp0, 1023
@@ -1120,8 +1114,8 @@ define void @c6_i32(i32 %arg, ptr %ptr) nounwind {
 define void @c7_i32(i32 %arg, ptr %ptr) nounwind {
 ; CHECK-LABEL: c7_i32:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    ubfx w8, w0, #19, #10
-; CHECK-NEXT:    lsl w8, w8, #2
+; CHECK-NEXT:    lsr w8, w0, #17
+; CHECK-NEXT:    and w8, w8, #0xffc
 ; CHECK-NEXT:    str w8, [x1]
 ; CHECK-NEXT:    ret
   %tmp0 = lshr i32 %arg, 19
@@ -1163,8 +1157,8 @@ define void @c6_i64(i64 %arg, ptr %ptr) nounwind {
 define void @c7_i64(i64 %arg, ptr %ptr) nounwind {
 ; CHECK-LABEL: c7_i64:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    ubfx x8, x0, #51, #10
-; CHECK-NEXT:    lsl x8, x8, #2
+; CHECK-NEXT:    lsr x8, x0, #49
+; CHECK-NEXT:    and x8, x8, #0xffc
 ; CHECK-NEXT:    str x8, [x1]
 ; CHECK-NEXT:    ret
   %tmp0 = lshr i64 %arg, 51
diff --git a/llvm/test/CodeGen/AArch64/fold-int-pow2-with-fmul-or-fdiv.ll b/llvm/test/CodeGen/AArch64/fold-int-pow2-with-fmul-or-fdiv.ll
index a78addc490086..b40c0656a60e4 100644
--- a/llvm/test/CodeGen/AArch64/fold-int-pow2-with-fmul-or-fdiv.ll
+++ b/llvm/test/CodeGen/AArch64/fold-int-pow2-with-fmul-or-fdiv.ll
@@ -224,7 +224,6 @@ define double @fmul_pow_mul_max_pow2(i16 %cnt) nounwind {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    mov w8, #2 // =0x2
 ; CHECK-NEXT:    mov w9, #1 // =0x1
-; CHECK-NEXT:    // kill: def $w0 killed $w0 def $x0
 ; CHECK-NEXT:    fmov d1, #3.00000000
 ; CHECK-NEXT:    lsl w8, w8, w0
 ; CHECK-NEXT:    lsl w9, w9, w0
@@ -433,7 +432,6 @@ define double @fmul_pow_shl_cnt_safe(i16 %cnt) nounwind {
 ; CHECK-LABEL: fmul_pow_shl_cnt_safe:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    mov w8, #1 // =0x1
-; CHECK-NEXT:    // kill: def $w0 killed $w0 def $x0
 ; CHECK-NEXT:    lsl w8, w8, w0
 ; CHECK-NEXT:    and w8, w8, #0xffff
 ; CHECK-NEXT:    ucvtf d0, w8
diff --git a/llvm/test/CodeGen/AArch64/fp8-neon-fdot.ll b/llvm/test/CodeGen/AArch64/fp8-neon-fdot.ll
new file mode 100644
index 0000000000000..b7a35c5fddf17
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/fp8-neon-fdot.ll
@@ -0,0 +1,74 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 4
+; RUN: llc -mtriple=aarch64-linux -mattr=+neon,+fp8dot2,+fp8dot4 < %s | FileCheck %s
+
+define <4 x half> @test_fdot_f16(<4 x half> %vd, <8 x i8> %vn, <8 x i8> %vm) {
+; CHECK-LABEL: test_fdot_f16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    fdot v0.4h, v1.8b, v2.8b
+; CHECK-NEXT:    ret
+  %res = call <4 x half> @llvm.aarch64.neon.fp8.fdot2.v4f16.v8i8(<4 x half> %vd, <8 x i8> %vn, <8 x i8> %vm)
+  ret <4 x half> %res
+}
+
+define <8 x half> @test_fdotq_f16(<8 x half> %vd, <16 x i8> %vn, <16 x i8> %vm) {
+; CHECK-LABEL: test_fdotq_f16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    fdot v0.8h, v1.16b, v2.16b
+; CHECK-NEXT:    ret
+  %res = call <8 x half> @llvm.aarch64.neon.fp8.fdot2.v8f16.v16i8(<8 x half> %vd, <16 x i8> %vn, <16 x i8> %vm)
+  ret <8 x half> %res
+}
+
+define <4 x half> @test_fdot_lane_f16(<4 x half> %vd, <8 x i8> %vn, <16 x i8> %vm) {
+; CHECK-LABEL: test_fdot_lane_f16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    fdot v0.4h, v1.8b, v2.2b[0]
+; CHECK-NEXT:    ret
+  %res = call <4 x half> @llvm.aarch64.neon.fp8.fdot2.lane.v4f16.v8i8(<4 x half> %vd, <8 x i8> %vn, <16 x i8> %vm, i32 0)
+  ret <4 x half> %res
+}
+
+define <8 x half> @test_fdotq_lane_f16(<8 x half> %vd, <16 x i8> %vn, <16 x i8> %vm) {
+; CHECK-LABEL: test_fdotq_lane_f16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    fdot v0.8h, v1.16b, v2.2b[7]
+; CHECK-NEXT:    ret
+  %res = call <8 x half> @llvm.aarch64.neon.fp8.fdot2.lane.v8f16.v16i8(<8 x half> %vd, <16 x i8> %vn, <16 x i8> %vm, i32 7)
+  ret <8 x half> %res
+}
+
+define <2 x float> @test_fdot_f32(<2 x float> %vd, <8 x i8> %vn, <8 x i8> %vm) {
+; CHECK-LABEL: test_fdot_f32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    fdot v0.2s, v1.8b, v2.8b
+; CHECK-NEXT:    ret
+  %res = call <2 x float> @llvm.aarch64.neon.fp8.fdot4.v2f32.v8i8(<2 x float> %vd, <8 x i8> %vn, <8 x i8> %vm)
+  ret <2 x float> %res
+}
+
+define <4 x float> @test_fdotq_f32(<4 x float> %vd, <16 x i8> %vn, <16 x i8> %vm) {
+; CHECK-LABEL: test_fdotq_f32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    fdot v0.4s, v1.16b, v2.16b
+; CHECK-NEXT:    ret
+  %res = call <4 x float> @llvm.aarch64.neon.fp8.fdot4.v4f32.v16i8(<4 x float> %vd, <16 x i8> %vn, <16 x i8> %vm)
+  ret <4 x float> %res
+}
+
+define <2 x float> @test_fdot_lane_f32(<2 x float> %vd, <8 x i8> %vn, <16 x i8> %vm) {
+; CHECK-LABEL: test_fdot_lane_f32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    fdot v0.2s, v1.8b, v2.4b[0]
+; CHECK-NEXT:    ret
+  %res = call <2 x float> @llvm.aarch64.neon.fp8.fdot4.lane.v2f32.v8i8(<2 x float> %vd, <8 x i8> %vn, <16 x i8> %vm, i32 0)
+  ret <2 x float> %res
+}
+
+define <4 x float> @test_fdotq_lane_f32(<4 x float> %vd, <16 x i8> %vn, <16 x i8> %vm) {
+; CHECK-LABEL: test_fdotq_lane_f32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    fdot v0.4s, v1.16b, v2.4b[3]
+; CHECK-NEXT:    ret
+  %res = call <4 x float> @llvm.aarch64.neon.fp8.fdot4.lane.v4f32.v16i8(<4 x float> %vd, <16 x i8> %vn, <16 x i8> %vm, i32 3)
+  ret <4 x float> %res
+}
diff --git a/llvm/test/CodeGen/AArch64/fp8-neon-fmla.ll b/llvm/test/CodeGen/AArch64/fp8-neon-fmla.ll
new file mode 100644
index 0000000000000..60957a7c0f2f4
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/fp8-neon-fmla.ll
@@ -0,0 +1,110 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc -mtriple=aarch64-linux -mattr=+neon,+fp8fma < %s | FileCheck %s
+
+define <8 x half> @test_fmlalb(<8 x half> %d, <16 x i8> %a, <16 x i8> %b) {
+; CHECK-LABEL: test_fmlalb:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    fmlalb v0.8h, v1.16b, v2.16b
+; CHECK-NEXT:    ret
+  %r = call <8 x half> @llvm.aarch64.neon.fp8.fmlalb.v8f16(<8 x half> %d, <16 x i8> %a, <16 x i8> %b)
+  ret <8 x half> %r
+}
+
+define <8 x half> @test_fmlalt(<8 x half> %d, <16 x i8> %a, <16 x i8> %b) {
+; CHECK-LABEL: test_fmlalt:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    fmlalt v0.8h, v1.16b, v2.16b
+; CHECK-NEXT:    ret
+  %r = call <8 x half> @llvm.aarch64.neon.fp8.fmlalt.v8f16(<8 x half> %d, <16 x i8> %a, <16 x i8> %b)
+  ret <8 x half> %r
+}
+
+define <4 x float> @test_fmlallbb(<4 x float> %d, <16 x i8> %a, <16 x i8> %b) {
+; CHECK-LABEL: test_fmlallbb:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    fmlallbb v0.4s, v1.16b, v2.16b
+; CHECK-NEXT:    ret
+  %r = call <4 x float> @llvm.aarch64.neon.fp8.fmlallbb.v4f32(<4 x float> %d, <16 x i8> %a, <16 x i8> %b)
+  ret <4 x float> %r
+}
+
+define <4 x float> @test_fmlallbt(<4 x float> %d, <16 x i8> %a, <16 x i8> %b) {
+; CHECK-LABEL: test_fmlallbt:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    fmlallbt v0.4s, v1.16b, v2.16b
+; CHECK-NEXT:    ret
+  %r = call <4 x float> @llvm.aarch64.neon.fp8.fmlallbt.v4f32(<4 x float> %d, <16 x i8> %a, <16 x i8> %b)
+  ret <4 x float> %r
+}
+
+define <4 x float> @test_fmlalltb(<4 x float> %d, <16 x i8> %a, <16 x i8> %b) {
+; CHECK-LABEL: test_fmlalltb:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    fmlalltb v0.4s, v1.16b, v2.16b
+; CHECK-NEXT:    ret
+  %r = call <4 x float> @llvm.aarch64.neon.fp8.fmlalltb.v4f32(<4 x float> %d, <16 x i8> %a, <16 x i8> %b)
+  ret <4 x float> %r
+}
+
+define <4 x float> @test_fmlalltt(<4 x float> %d, <16 x i8> %a, <16 x i8> %b) {
+; CHECK-LABEL: test_fmlalltt:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    fmlalltt v0.4s, v1.16b, v2.16b
+; CHECK-NEXT:    ret
+  %r = call <4 x float> @llvm.aarch64.neon.fp8.fmlalltt.v4f32(<4 x float> %d, <16 x i8> %a, <16 x i8> %b)
+  ret <4 x float> %r
+}
+
+define <8 x half> @test_fmlalb_lane(<8 x half> %vd, <16 x i8> %vn, <16 x i8> %vm) {
+; CHECK-LABEL: test_fmlalb_lane:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    fmlalb v0.8h, v1.16b, v2.b[0]
+; CHECK-NEXT:    ret
+  %res = tail call <8 x half> @llvm.aarch64.neon.fp8.fmlalb.lane(<8 x half> %vd, <16 x i8> %vn, <16 x i8> %vm, i32 0)
+  ret <8 x half> %res
+}
+
+define <8 x half> @test_fmlalt_lane(<8 x half> %vd, <16 x i8> %vn, <16 x i8> %vm) {
+; CHECK-LABEL: test_fmlalt_lane:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    fmlalt v0.8h, v1.16b, v2.b[4]
+; CHECK-NEXT:    ret
+  %res = tail call <8 x half> @llvm.aarch64.neon.fp8.fmlalt.lane(<8 x half> %vd, <16 x i8> %vn, <16 x i8> %vm, i32 4)
+  ret <8 x half> %res
+}
+
+define <4 x float> @test_fmlallbb_lane(<4 x float> %vd, <16 x i8> %vn, <16 x i8> %vm) {
+; CHECK-LABEL: test_fmlallbb_lane:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    fmlallbb v0.4s, v1.16b, v2.b[7]
+; CHECK-NEXT:    ret
+  %res = tail call <4 x float> @llvm.aarch64.neon.fp8.fmlallbb.lane(<4 x float> %vd, <16 x i8> %vn, <16 x i8> %vm, i32 7)
+  ret <4 x float> %res
+}
+
+define <4 x float> @test_fmlallbt_lane(<4 x float> %vd, <16 x i8> %vn, <16 x i8> %vm) {
+; CHECK-LABEL: test_fmlallbt_lane:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    fmlallbt v0.4s, v1.16b, v2.b[10]
+; CHECK-NEXT:    ret
+  %res = tail call <4 x float> @llvm.aarch64.neon.fp8.fmlallbt.lane(<4 x float> %vd, <16 x i8> %vn, <16 x i8> %vm, i32 10)
+  ret <4 x float> %res
+}
+
+define <4 x float> @test_fmlalltb_lane(<4 x float> %vd, <16 x i8> %vn, <16 x i8> %vm) {
+; CHECK-LABEL: test_fmlalltb_lane:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    fmlalltb v0.4s, v1.16b, v2.b[13]
+; CHECK-NEXT:    ret
+  %res = tail call <4 x float> @llvm.aarch64.neon.fp8.fmlalltb.lane(<4 x float> %vd, <16 x i8> %vn, <16 x i8> %vm, i32 13)
+  ret <4 x float> %res
+}
+
+define <4 x float> @test_fmlalltt_lane(<4 x float> %vd, <16 x i8> %vn, <16 x i8> %vm) {
+; CHECK-LABEL: test_fmlalltt_lane:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    fmlalltt v0.4s, v1.16b, v2.b[15]
+; CHECK-NEXT:    ret
+  %res = tail call <4 x float> @llvm.aarch64.neon.fp8.fmlalltt.lane(<4 x float> %vd, <16 x i8> %vn, <16 x i8> %vm, i32 15)
+  ret <4 x float> %res
+}
diff --git a/llvm/test/CodeGen/AArch64/fp8-sme2-cvtn.ll b/llvm/test/CodeGen/AArch64/fp8-sme2-cvtn.ll
new file mode 100644
index 0000000000000..d1e0729db30e5
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/fp8-sme2-cvtn.ll
@@ -0,0 +1,86 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc -mattr=+sme2,+fp8 -enable-subreg-liveness --force-streaming < %s | FileCheck %s
+
+target triple = "aarch64-linux"
+
+define { <vscale x 16 x i8>, <vscale x 16 x i8> } @cvtn_f16_tuple(i64 %stride, ptr %ptr) {
+; CHECK-LABEL: cvtn_f16_tuple:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NEXT:    addvl sp, sp, #-3
+; CHECK-NEXT:    str p8, [sp, #7, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    str z11, [sp, #1, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z10, [sp, #2, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    .cfi_escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x18, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 24 * VG
+; CHECK-NEXT:    .cfi_offset w29, -16
+; CHECK-NEXT:    .cfi_escape 0x10, 0x4a, 0x0a, 0x11, 0x70, 0x22, 0x11, 0x78, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d10 @ cfa - 16 - 8 * VG
+; CHECK-NEXT:    .cfi_escape 0x10, 0x4b, 0x0a, 0x11, 0x70, 0x22, 0x11, 0x70, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d11 @ cfa - 16 - 16 * VG
+; CHECK-NEXT:    ptrue pn8.b
+; CHECK-NEXT:    add x8, x1, x0
+; CHECK-NEXT:    ld1h { z2.h, z10.h }, pn8/z, [x1]
+; CHECK-NEXT:    ld1h { z3.h, z11.h }, pn8/z, [x8]
+; CHECK-NEXT:    fcvtn z0.b, { z2.h, z3.h }
+; CHECK-NEXT:    fcvtn z1.b, { z10.h, z11.h }
+; CHECK-NEXT:    ldr z11, [sp, #1, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z10, [sp, #2, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr p8, [sp, #7, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    addvl sp, sp, #3
+; CHECK-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    ret
+entry:
+    %0 = tail call target("aarch64.svcount") @llvm.aarch64.sve.ptrue.c8()
+    %1 = tail call { <vscale x 8 x half>, <vscale x 8 x half> } @llvm.aarch64.sve.ld1.pn.x2.nxv4i32(target("aarch64.svcount") %0, ptr %ptr)
+    %2 = extractvalue { <vscale x 8 x half>, <vscale x 8 x half> } %1, 0
+    %3 = extractvalue { <vscale x 8 x half>, <vscale x 8 x half> } %1, 1
+    %arrayidx2 = getelementptr inbounds i8, ptr %ptr, i64 %stride
+    %4 = tail call { <vscale x 8 x half>, <vscale x 8 x half> } @llvm.aarch64.sve.ld1.pn.x2.nxv4i32(target("aarch64.svcount") %0, ptr %arrayidx2)
+    %5 = extractvalue { <vscale x 8 x half>, <vscale x 8 x half> } %4, 0
+    %6 = extractvalue { <vscale x 8 x half>, <vscale x 8 x half> } %4, 1
+    %res1 = call <vscale x 16 x i8> @llvm.aarch64.sve.fp8.cvtn.nxv8f16(<vscale x 8 x half> %2, <vscale x 8 x half> %5)
+    %res2 = call <vscale x 16 x i8> @llvm.aarch64.sve.fp8.cvtn.nxv8f16(<vscale x 8 x half> %3, <vscale x 8 x half> %6)
+    %ins1 = insertvalue { <vscale x 16 x i8>, <vscale x 16 x i8> } poison, <vscale x 16 x i8> %res1, 0
+    %ins2 = insertvalue { <vscale x 16 x i8>, <vscale x 16 x i8> } %ins1, <vscale x 16 x i8> %res2, 1
+    ret { <vscale x 16 x i8>, <vscale x 16 x i8> } %ins2
+}
+
+
+define { <vscale x 16 x i8>, <vscale x 16 x i8> } @cvtnt_f32_tuple(i64 %stride, ptr %ptr, <vscale x 16 x i8> %d) {
+; CHECK-LABEL: cvtnt_f32_tuple:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NEXT:    addvl sp, sp, #-3
+; CHECK-NEXT:    str p8, [sp, #7, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    str z11, [sp, #1, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z10, [sp, #2, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    .cfi_escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x18, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 24 * VG
+; CHECK-NEXT:    .cfi_offset w29, -16
+; CHECK-NEXT:    .cfi_escape 0x10, 0x4a, 0x0a, 0x11, 0x70, 0x22, 0x11, 0x78, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d10 @ cfa - 16 - 8 * VG
+; CHECK-NEXT:    .cfi_escape 0x10, 0x4b, 0x0a, 0x11, 0x70, 0x22, 0x11, 0x70, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d11 @ cfa - 16 - 16 * VG
+; CHECK-NEXT:    ptrue pn8.b
+; CHECK-NEXT:    add x8, x1, x0
+; CHECK-NEXT:    mov z1.d, z0.d
+; CHECK-NEXT:    ld1w { z2.s, z10.s }, pn8/z, [x1]
+; CHECK-NEXT:    ld1w { z3.s, z11.s }, pn8/z, [x8]
+; CHECK-NEXT:    fcvtnt z0.b, { z2.s, z3.s }
+; CHECK-NEXT:    fcvtnt z1.b, { z10.s, z11.s }
+; CHECK-NEXT:    ldr z11, [sp, #1, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z10, [sp, #2, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr p8, [sp, #7, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    addvl sp, sp, #3
+; CHECK-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    ret
+entry:
+    %0 = tail call target("aarch64.svcount") @llvm.aarch64.sve.ptrue.c8()
+    %1 = tail call { <vscale x 4 x float>, <vscale x 4 x float> } @llvm.aarch64.sve.ld1.pn.x2.nxv4f32(target("aarch64.svcount") %0, ptr %ptr)
+    %2 = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } %1, 0
+    %3 = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } %1, 1
+    %arrayidx2 = getelementptr inbounds i8, ptr %ptr, i64 %stride
+    %4 = tail call { <vscale x 4 x float>, <vscale x 4 x float> } @llvm.aarch64.sve.ld1.pn.x2.nxv4f32(target("aarch64.svcount") %0, ptr %arrayidx2)
+    %5 = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } %4, 0
+    %6 = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } %4, 1
+    %res1 = call <vscale x 16 x i8> @llvm.aarch64.sve.fp8.cvtnt.nxv4f32(<vscale x 16 x i8> %d, <vscale x 4 x float> %2, <vscale x 4 x float> %5)
+    %res2 = call <vscale x 16 x i8> @llvm.aarch64.sve.fp8.cvtnt.nxv4f32(<vscale x 16 x i8> %d, <vscale x 4 x float> %3, <vscale x 4 x float> %6)
+    %ins1 = insertvalue { <vscale x 16 x i8>, <vscale x 16 x i8> } poison, <vscale x 16 x i8> %res1, 0
+    %ins2 = insertvalue { <vscale x 16 x i8>, <vscale x 16 x i8> } %ins1, <vscale x 16 x i8> %res2, 1
+    ret { <vscale x 16 x i8>, <vscale x 16 x i8> } %ins2
+}
diff --git a/llvm/test/CodeGen/AArch64/fpenv.ll b/llvm/test/CodeGen/AArch64/fpenv.ll
index 3a307f7731037..3351565d8dd89 100644
--- a/llvm/test/CodeGen/AArch64/fpenv.ll
+++ b/llvm/test/CodeGen/AArch64/fpenv.ll
@@ -4,11 +4,11 @@
 define void @func_set_rounding_dyn(i32 %rm) {
 ; CHECK-LABEL: func_set_rounding_dyn:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    sub w9, w0, #1
+; CHECK-NEXT:    lsl w9, w0, #22
 ; CHECK-NEXT:    mrs x8, FPCR
-; CHECK-NEXT:    and w9, w9, #0x3
 ; CHECK-NEXT:    and x8, x8, #0xffffffffff3fffff
-; CHECK-NEXT:    lsl w9, w9, #22
+; CHECK-NEXT:    sub w9, w9, #1024, lsl #12 // =4194304
+; CHECK-NEXT:    and w9, w9, #0xc00000
 ; CHECK-NEXT:    orr x8, x8, x9
 ; CHECK-NEXT:    msr FPCR, x8
 ; CHECK-NEXT:    ret
diff --git a/llvm/test/CodeGen/AArch64/fprcvt-cvtf.ll b/llvm/test/CodeGen/AArch64/fprcvt-cvtf.ll
new file mode 100644
index 0000000000000..0a7319b9ce11e
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/fprcvt-cvtf.ll
@@ -0,0 +1,441 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc -mattr=+neon,+fullfp16,+fprcvt -verify-machineinstrs %s -o - | FileCheck %s
+; RUN: llc -mattr=+neon -verify-machineinstrs %s -o - | FileCheck %s --check-prefix=CHECK-NO-FPRCVT
+
+target triple = "aarch64-unknown-linux-gnu"
+
+
+; To demonstrate what we have implemented, we'll want a scalar integer value in a SIMD/FP register.
+; A common case for this setup is when using the result of an integer reduction intrinsic.
+
+; SCVTF
+
+define half @scvtf_f16i32(<4 x i32> %x) {
+; CHECK-LABEL: scvtf_f16i32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    scvtf h0, s0
+; CHECK-NEXT:    ret
+;
+; CHECK-NO-FPRCVT-LABEL: scvtf_f16i32:
+; CHECK-NO-FPRCVT:       // %bb.0:
+; CHECK-NO-FPRCVT-NEXT:    scvtf s0, s0
+; CHECK-NO-FPRCVT-NEXT:    fcvt h0, s0
+; CHECK-NO-FPRCVT-NEXT:    ret
+ %extract = extractelement <4 x i32> %x, i64 0
+ %conv = sitofp i32 %extract to half
+ ret half %conv
+}
+
+define half @scvtf_f16i32_neg(<4 x i32> %x) {
+; CHECK-LABEL: scvtf_f16i32_neg:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mov w8, v0.s[1]
+; CHECK-NEXT:    scvtf h0, w8
+; CHECK-NEXT:    ret
+;
+; CHECK-NO-FPRCVT-LABEL: scvtf_f16i32_neg:
+; CHECK-NO-FPRCVT:       // %bb.0:
+; CHECK-NO-FPRCVT-NEXT:    mov w8, v0.s[1]
+; CHECK-NO-FPRCVT-NEXT:    scvtf s0, w8
+; CHECK-NO-FPRCVT-NEXT:    fcvt h0, s0
+; CHECK-NO-FPRCVT-NEXT:    ret
+ %extract = extractelement <4 x i32> %x, i64 1
+ %conv = sitofp i32 %extract to half
+ ret half %conv
+}
+
+define <1 x half> @scvtf_f16i32_simple(<1 x i32> %x) {
+; CHECK-LABEL: scvtf_f16i32_simple:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-NEXT:    scvtf h0, s0
+; CHECK-NEXT:    ret
+;
+; CHECK-NO-FPRCVT-LABEL: scvtf_f16i32_simple:
+; CHECK-NO-FPRCVT:       // %bb.0:
+; CHECK-NO-FPRCVT-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-NO-FPRCVT-NEXT:    scvtf s0, s0
+; CHECK-NO-FPRCVT-NEXT:    fcvt h0, s0
+; CHECK-NO-FPRCVT-NEXT:    ret
+ %conv = sitofp <1 x i32> %x to <1 x half>
+ ret <1 x half> %conv
+}
+
+define double @scvtf_f64i32(<4 x i32> %x) {
+; CHECK-LABEL: scvtf_f64i32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    scvtf d0, s0
+; CHECK-NEXT:    ret
+;
+; CHECK-NO-FPRCVT-LABEL: scvtf_f64i32:
+; CHECK-NO-FPRCVT:       // %bb.0:
+; CHECK-NO-FPRCVT-NEXT:    fmov w8, s0
+; CHECK-NO-FPRCVT-NEXT:    scvtf d0, w8
+; CHECK-NO-FPRCVT-NEXT:    ret
+ %extract = extractelement <4 x i32> %x, i64 0
+ %conv = sitofp i32 %extract to double
+ ret double %conv
+}
+
+define double @scvtf_f64i32_neg(<4 x i32> %x) {
+; CHECK-LABEL: scvtf_f64i32_neg:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mov w8, v0.s[1]
+; CHECK-NEXT:    scvtf d0, w8
+; CHECK-NEXT:    ret
+;
+; CHECK-NO-FPRCVT-LABEL: scvtf_f64i32_neg:
+; CHECK-NO-FPRCVT:       // %bb.0:
+; CHECK-NO-FPRCVT-NEXT:    mov w8, v0.s[1]
+; CHECK-NO-FPRCVT-NEXT:    scvtf d0, w8
+; CHECK-NO-FPRCVT-NEXT:    ret
+ %extract = extractelement <4 x i32> %x, i64 1
+ %conv = sitofp i32 %extract to double
+ ret double %conv
+}
+
+; This test does not give the indended result of scvtf d0, s0
+; This is due to the input being loaded as a 2 item vector and
+; therefore using vector inputs that do not match the pattern
+; This test will be fixed in a future revision
+define <1 x double> @scvtf_f64i32_simple(<1 x i32> %x) {
+; CHECK-LABEL: scvtf_f64i32_simple:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    sshll v0.2d, v0.2s, #0
+; CHECK-NEXT:    scvtf v0.2d, v0.2d
+; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-NEXT:    ret
+;
+; CHECK-NO-FPRCVT-LABEL: scvtf_f64i32_simple:
+; CHECK-NO-FPRCVT:       // %bb.0:
+; CHECK-NO-FPRCVT-NEXT:    sshll v0.2d, v0.2s, #0
+; CHECK-NO-FPRCVT-NEXT:    scvtf v0.2d, v0.2d
+; CHECK-NO-FPRCVT-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-NO-FPRCVT-NEXT:    ret
+ %conv = sitofp <1 x i32> %x to <1 x double>
+ ret <1 x double> %conv
+}
+
+define half @scvtf_f16i64(<2 x i64> %x) {
+; CHECK-LABEL: scvtf_f16i64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    scvtf h0, d0
+; CHECK-NEXT:    ret
+;
+; CHECK-NO-FPRCVT-LABEL: scvtf_f16i64:
+; CHECK-NO-FPRCVT:       // %bb.0:
+; CHECK-NO-FPRCVT-NEXT:    fmov x8, d0
+; CHECK-NO-FPRCVT-NEXT:    scvtf s0, x8
+; CHECK-NO-FPRCVT-NEXT:    fcvt h0, s0
+; CHECK-NO-FPRCVT-NEXT:    ret
+ %extract = extractelement <2 x i64> %x, i64 0
+ %conv = sitofp i64 %extract to half
+ ret half %conv
+}
+
+define half @scvtf_f16i64_neg(<2 x i64> %x) {
+; CHECK-LABEL: scvtf_f16i64_neg:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mov x8, v0.d[1]
+; CHECK-NEXT:    scvtf h0, x8
+; CHECK-NEXT:    ret
+;
+; CHECK-NO-FPRCVT-LABEL: scvtf_f16i64_neg:
+; CHECK-NO-FPRCVT:       // %bb.0:
+; CHECK-NO-FPRCVT-NEXT:    mov x8, v0.d[1]
+; CHECK-NO-FPRCVT-NEXT:    scvtf s0, x8
+; CHECK-NO-FPRCVT-NEXT:    fcvt h0, s0
+; CHECK-NO-FPRCVT-NEXT:    ret
+ %extract = extractelement <2 x i64> %x, i64 1
+ %conv = sitofp i64 %extract to half
+ ret half %conv
+}
+
+define <1 x half> @scvtf_f16i64_simple(<1 x i64> %x) {
+; CHECK-LABEL: scvtf_f16i64_simple:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-NEXT:    scvtf h0, d0
+; CHECK-NEXT:    ret
+;
+; CHECK-NO-FPRCVT-LABEL: scvtf_f16i64_simple:
+; CHECK-NO-FPRCVT:       // %bb.0:
+; CHECK-NO-FPRCVT-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-NO-FPRCVT-NEXT:    fmov x8, d0
+; CHECK-NO-FPRCVT-NEXT:    scvtf s0, x8
+; CHECK-NO-FPRCVT-NEXT:    fcvt h0, s0
+; CHECK-NO-FPRCVT-NEXT:    ret
+ %conv = sitofp <1 x i64> %x to <1 x half>
+ ret <1 x half> %conv
+}
+
+define float @scvtf_f32i64(<2 x i64> %x) {
+; CHECK-LABEL: scvtf_f32i64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    scvtf s0, d0
+; CHECK-NEXT:    ret
+;
+; CHECK-NO-FPRCVT-LABEL: scvtf_f32i64:
+; CHECK-NO-FPRCVT:       // %bb.0:
+; CHECK-NO-FPRCVT-NEXT:    fmov x8, d0
+; CHECK-NO-FPRCVT-NEXT:    scvtf s0, x8
+; CHECK-NO-FPRCVT-NEXT:    ret
+ %extract = extractelement <2 x i64> %x, i64 0
+ %conv = sitofp i64 %extract to float
+ ret float %conv
+}
+
+define float @scvtf_f32i64_neg(<2 x i64> %x) {
+; CHECK-LABEL: scvtf_f32i64_neg:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mov x8, v0.d[1]
+; CHECK-NEXT:    scvtf s0, x8
+; CHECK-NEXT:    ret
+;
+; CHECK-NO-FPRCVT-LABEL: scvtf_f32i64_neg:
+; CHECK-NO-FPRCVT:       // %bb.0:
+; CHECK-NO-FPRCVT-NEXT:    mov x8, v0.d[1]
+; CHECK-NO-FPRCVT-NEXT:    scvtf s0, x8
+; CHECK-NO-FPRCVT-NEXT:    ret
+ %extract = extractelement <2 x i64> %x, i64 1
+ %conv = sitofp i64 %extract to float
+ ret float %conv
+}
+
+; This test does not give the indended result of scvtf s0, d0
+; This is due to the input being loaded as a 2 item vector and
+; therefore using vector inputs that do not match the pattern
+; This test will be fixed in a future revision
+define <1 x float> @scvtf_f32i64_simple(<1 x i64> %x) {
+; CHECK-LABEL: scvtf_f32i64_simple:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-NEXT:    scvtf v0.2d, v0.2d
+; CHECK-NEXT:    fcvtn v0.2s, v0.2d
+; CHECK-NEXT:    ret
+;
+; CHECK-NO-FPRCVT-LABEL: scvtf_f32i64_simple:
+; CHECK-NO-FPRCVT:       // %bb.0:
+; CHECK-NO-FPRCVT-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-NO-FPRCVT-NEXT:    scvtf v0.2d, v0.2d
+; CHECK-NO-FPRCVT-NEXT:    fcvtn v0.2s, v0.2d
+; CHECK-NO-FPRCVT-NEXT:    ret
+ %conv = sitofp <1 x i64> %x to <1 x float>
+ ret <1 x float> %conv
+}
+
+; UCVTF
+
+define half @ucvtf_f16i32(<4 x i32> %x) {
+; CHECK-LABEL: ucvtf_f16i32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ucvtf h0, s0
+; CHECK-NEXT:    ret
+;
+; CHECK-NO-FPRCVT-LABEL: ucvtf_f16i32:
+; CHECK-NO-FPRCVT:       // %bb.0:
+; CHECK-NO-FPRCVT-NEXT:    ucvtf s0, s0
+; CHECK-NO-FPRCVT-NEXT:    fcvt h0, s0
+; CHECK-NO-FPRCVT-NEXT:    ret
+ %extract = extractelement <4 x i32> %x, i64 0
+ %conv = uitofp i32 %extract to half
+ ret half %conv
+}
+
+define half @ucvtf_f16i32_neg(<4 x i32> %x) {
+; CHECK-LABEL: ucvtf_f16i32_neg:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mov w8, v0.s[1]
+; CHECK-NEXT:    ucvtf h0, w8
+; CHECK-NEXT:    ret
+;
+; CHECK-NO-FPRCVT-LABEL: ucvtf_f16i32_neg:
+; CHECK-NO-FPRCVT:       // %bb.0:
+; CHECK-NO-FPRCVT-NEXT:    mov w8, v0.s[1]
+; CHECK-NO-FPRCVT-NEXT:    ucvtf s0, w8
+; CHECK-NO-FPRCVT-NEXT:    fcvt h0, s0
+; CHECK-NO-FPRCVT-NEXT:    ret
+ %extract = extractelement <4 x i32> %x, i64 1
+ %conv = uitofp i32 %extract to half
+ ret half %conv
+}
+
+define <1 x half> @ucvtf_f16i32_simple(<1 x i32> %x) {
+; CHECK-LABEL: ucvtf_f16i32_simple:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-NEXT:    ucvtf h0, s0
+; CHECK-NEXT:    ret
+;
+; CHECK-NO-FPRCVT-LABEL: ucvtf_f16i32_simple:
+; CHECK-NO-FPRCVT:       // %bb.0:
+; CHECK-NO-FPRCVT-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-NO-FPRCVT-NEXT:    ucvtf s0, s0
+; CHECK-NO-FPRCVT-NEXT:    fcvt h0, s0
+; CHECK-NO-FPRCVT-NEXT:    ret
+ %conv = uitofp <1 x i32> %x to <1 x half>
+ ret <1 x half> %conv
+}
+
+define double @ucvtf_f64i32(<4 x i32> %x) {
+; CHECK-LABEL: ucvtf_f64i32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ucvtf d0, s0
+; CHECK-NEXT:    ret
+;
+; CHECK-NO-FPRCVT-LABEL: ucvtf_f64i32:
+; CHECK-NO-FPRCVT:       // %bb.0:
+; CHECK-NO-FPRCVT-NEXT:    fmov w8, s0
+; CHECK-NO-FPRCVT-NEXT:    ucvtf d0, w8
+; CHECK-NO-FPRCVT-NEXT:    ret
+ %extract = extractelement <4 x i32> %x, i64 0
+ %conv = uitofp i32 %extract to double
+ ret double %conv
+}
+
+define double @ucvtf_f64i32_neg(<4 x i32> %x) {
+; CHECK-LABEL: ucvtf_f64i32_neg:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mov w8, v0.s[1]
+; CHECK-NEXT:    ucvtf d0, w8
+; CHECK-NEXT:    ret
+;
+; CHECK-NO-FPRCVT-LABEL: ucvtf_f64i32_neg:
+; CHECK-NO-FPRCVT:       // %bb.0:
+; CHECK-NO-FPRCVT-NEXT:    mov w8, v0.s[1]
+; CHECK-NO-FPRCVT-NEXT:    ucvtf d0, w8
+; CHECK-NO-FPRCVT-NEXT:    ret
+ %extract = extractelement <4 x i32> %x, i64 1
+ %conv = uitofp i32 %extract to double
+ ret double %conv
+}
+
+; This test does not give the indended result of ucvtf d0, s0
+; This is due to the input being loaded as a 2 item vector and
+; therefore using vector inputs that do not match the pattern
+; This test will be fixed in a future revision
+define <1 x double> @ucvtf_f64i32_simple(<1 x i32> %x) {
+; CHECK-LABEL: ucvtf_f64i32_simple:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ushll v0.2d, v0.2s, #0
+; CHECK-NEXT:    ucvtf v0.2d, v0.2d
+; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-NEXT:    ret
+;
+; CHECK-NO-FPRCVT-LABEL: ucvtf_f64i32_simple:
+; CHECK-NO-FPRCVT:       // %bb.0:
+; CHECK-NO-FPRCVT-NEXT:    ushll v0.2d, v0.2s, #0
+; CHECK-NO-FPRCVT-NEXT:    ucvtf v0.2d, v0.2d
+; CHECK-NO-FPRCVT-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-NO-FPRCVT-NEXT:    ret
+ %conv = uitofp <1 x i32> %x to <1 x double>
+ ret <1 x double> %conv
+}
+
+define half @ucvtf_f16i64(<2 x i64> %x) {
+; CHECK-LABEL: ucvtf_f16i64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ucvtf h0, d0
+; CHECK-NEXT:    ret
+;
+; CHECK-NO-FPRCVT-LABEL: ucvtf_f16i64:
+; CHECK-NO-FPRCVT:       // %bb.0:
+; CHECK-NO-FPRCVT-NEXT:    fmov x8, d0
+; CHECK-NO-FPRCVT-NEXT:    ucvtf s0, x8
+; CHECK-NO-FPRCVT-NEXT:    fcvt h0, s0
+; CHECK-NO-FPRCVT-NEXT:    ret
+ %extract = extractelement <2 x i64> %x, i64 0
+ %conv = uitofp i64 %extract to half
+ ret half %conv
+}
+
+define half @ucvtf_f16i64_neg(<2 x i64> %x) {
+; CHECK-LABEL: ucvtf_f16i64_neg:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mov x8, v0.d[1]
+; CHECK-NEXT:    ucvtf h0, x8
+; CHECK-NEXT:    ret
+;
+; CHECK-NO-FPRCVT-LABEL: ucvtf_f16i64_neg:
+; CHECK-NO-FPRCVT:       // %bb.0:
+; CHECK-NO-FPRCVT-NEXT:    mov x8, v0.d[1]
+; CHECK-NO-FPRCVT-NEXT:    ucvtf s0, x8
+; CHECK-NO-FPRCVT-NEXT:    fcvt h0, s0
+; CHECK-NO-FPRCVT-NEXT:    ret
+ %extract = extractelement <2 x i64> %x, i64 1
+ %conv = uitofp i64 %extract to half
+ ret half %conv
+}
+
+define <1 x half> @ucvtf_f16i64_simple(<1 x i64> %x) {
+; CHECK-LABEL: ucvtf_f16i64_simple:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-NEXT:    ucvtf h0, d0
+; CHECK-NEXT:    ret
+;
+; CHECK-NO-FPRCVT-LABEL: ucvtf_f16i64_simple:
+; CHECK-NO-FPRCVT:       // %bb.0:
+; CHECK-NO-FPRCVT-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-NO-FPRCVT-NEXT:    fmov x8, d0
+; CHECK-NO-FPRCVT-NEXT:    ucvtf s0, x8
+; CHECK-NO-FPRCVT-NEXT:    fcvt h0, s0
+; CHECK-NO-FPRCVT-NEXT:    ret
+ %conv = uitofp <1 x i64> %x to <1 x half>
+ ret <1 x half> %conv
+}
+
+define float @ucvtf_f32i64(<2 x i64> %x) {
+; CHECK-LABEL: ucvtf_f32i64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ucvtf s0, d0
+; CHECK-NEXT:    ret
+;
+; CHECK-NO-FPRCVT-LABEL: ucvtf_f32i64:
+; CHECK-NO-FPRCVT:       // %bb.0:
+; CHECK-NO-FPRCVT-NEXT:    fmov x8, d0
+; CHECK-NO-FPRCVT-NEXT:    ucvtf s0, x8
+; CHECK-NO-FPRCVT-NEXT:    ret
+ %extract = extractelement <2 x i64> %x, i64 0
+ %conv = uitofp i64 %extract to float
+ ret float %conv
+}
+
+define float @ucvtf_f32i64_neg(<2 x i64> %x) {
+; CHECK-LABEL: ucvtf_f32i64_neg:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mov x8, v0.d[1]
+; CHECK-NEXT:    ucvtf s0, x8
+; CHECK-NEXT:    ret
+;
+; CHECK-NO-FPRCVT-LABEL: ucvtf_f32i64_neg:
+; CHECK-NO-FPRCVT:       // %bb.0:
+; CHECK-NO-FPRCVT-NEXT:    mov x8, v0.d[1]
+; CHECK-NO-FPRCVT-NEXT:    ucvtf s0, x8
+; CHECK-NO-FPRCVT-NEXT:    ret
+ %extract = extractelement <2 x i64> %x, i64 1
+ %conv = uitofp i64 %extract to float
+ ret float %conv
+}
+
+; This test does not give the indended result of ucvtf s0, d0
+; This is due to the input being loaded as a 2 item vector and
+; therefore using vector inputs that do not match the pattern
+; This test will be fixed in a future revision
+define <1 x float> @ucvtf_f32i64_simple(<1 x i64> %x) {
+; CHECK-LABEL: ucvtf_f32i64_simple:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-NEXT:    ucvtf v0.2d, v0.2d
+; CHECK-NEXT:    fcvtn v0.2s, v0.2d
+; CHECK-NEXT:    ret
+;
+; CHECK-NO-FPRCVT-LABEL: ucvtf_f32i64_simple:
+; CHECK-NO-FPRCVT:       // %bb.0:
+; CHECK-NO-FPRCVT-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-NO-FPRCVT-NEXT:    ucvtf v0.2d, v0.2d
+; CHECK-NO-FPRCVT-NEXT:    fcvtn v0.2s, v0.2d
+; CHECK-NO-FPRCVT-NEXT:    ret
+ %conv = uitofp <1 x i64> %x to <1 x float>
+ ret <1 x float> %conv
+}
diff --git a/llvm/test/CodeGen/AArch64/fptrunc.ll b/llvm/test/CodeGen/AArch64/fptrunc.ll
index 2187717c4148a..b4c38e9f2df3b 100644
--- a/llvm/test/CodeGen/AArch64/fptrunc.ll
+++ b/llvm/test/CodeGen/AArch64/fptrunc.ll
@@ -1,6 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 3
 ; RUN: llc -mtriple=aarch64 -global-isel=0 -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-SD
 ; RUN: llc -mtriple=aarch64 -global-isel=1 -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-GI
+; RUN: llc -mtriple=aarch64 -global-isel=1 -mattr=+fullfp16,+bf16 -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-GI
 
 define float @fptrunc_f64_f32(double %a) {
 ; CHECK-LABEL: fptrunc_f64_f32:
diff --git a/llvm/test/CodeGen/AArch64/fsh.ll b/llvm/test/CodeGen/AArch64/fsh.ll
index b3ce00aeb36e5..c084813760b80 100644
--- a/llvm/test/CodeGen/AArch64/fsh.ll
+++ b/llvm/test/CodeGen/AArch64/fsh.ll
@@ -394,7 +394,6 @@ define i32 @fshl_i32(i32 %a, i32 %b, i32 %c) {
 ; CHECK-SD-LABEL: fshl_i32:
 ; CHECK-SD:       // %bb.0: // %entry
 ; CHECK-SD-NEXT:    lsr w8, w1, #1
-; CHECK-SD-NEXT:    // kill: def $w2 killed $w2 def $x2
 ; CHECK-SD-NEXT:    mvn w9, w2
 ; CHECK-SD-NEXT:    lsl w10, w0, w2
 ; CHECK-SD-NEXT:    lsr w8, w8, w9
@@ -420,7 +419,6 @@ define i32 @fshr_i32(i32 %a, i32 %b, i32 %c) {
 ; CHECK-SD-LABEL: fshr_i32:
 ; CHECK-SD:       // %bb.0: // %entry
 ; CHECK-SD-NEXT:    lsl w8, w0, #1
-; CHECK-SD-NEXT:    // kill: def $w2 killed $w2 def $x2
 ; CHECK-SD-NEXT:    mvn w9, w2
 ; CHECK-SD-NEXT:    lsr w10, w1, w2
 ; CHECK-SD-NEXT:    lsl w8, w8, w9
@@ -3909,16 +3907,14 @@ entry:
 define <8 x i8> @fshl_v8i8_c(<8 x i8> %a, <8 x i8> %b) {
 ; CHECK-SD-LABEL: fshl_v8i8_c:
 ; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    ushr v1.8b, v1.8b, #1
 ; CHECK-SD-NEXT:    shl v0.8b, v0.8b, #3
-; CHECK-SD-NEXT:    usra v0.8b, v1.8b, #4
+; CHECK-SD-NEXT:    usra v0.8b, v1.8b, #5
 ; CHECK-SD-NEXT:    ret
 ;
 ; CHECK-GI-LABEL: fshl_v8i8_c:
 ; CHECK-GI:       // %bb.0: // %entry
 ; CHECK-GI-NEXT:    shl v0.8b, v0.8b, #3
-; CHECK-GI-NEXT:    ushr v1.8b, v1.8b, #5
-; CHECK-GI-NEXT:    orr v0.8b, v0.8b, v1.8b
+; CHECK-GI-NEXT:    usra v0.8b, v1.8b, #5
 ; CHECK-GI-NEXT:    ret
 entry:
   %d = call <8 x i8> @llvm.fshl(<8 x i8> %a, <8 x i8> %b, <8 x i8> <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>)
@@ -3928,16 +3924,14 @@ entry:
 define <8 x i8> @fshr_v8i8_c(<8 x i8> %a, <8 x i8> %b) {
 ; CHECK-SD-LABEL: fshr_v8i8_c:
 ; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    add v0.8b, v0.8b, v0.8b
-; CHECK-SD-NEXT:    shl v0.8b, v0.8b, #4
+; CHECK-SD-NEXT:    shl v0.8b, v0.8b, #5
 ; CHECK-SD-NEXT:    usra v0.8b, v1.8b, #3
 ; CHECK-SD-NEXT:    ret
 ;
 ; CHECK-GI-LABEL: fshr_v8i8_c:
 ; CHECK-GI:       // %bb.0: // %entry
 ; CHECK-GI-NEXT:    shl v0.8b, v0.8b, #5
-; CHECK-GI-NEXT:    ushr v1.8b, v1.8b, #3
-; CHECK-GI-NEXT:    orr v0.8b, v0.8b, v1.8b
+; CHECK-GI-NEXT:    usra v0.8b, v1.8b, #3
 ; CHECK-GI-NEXT:    ret
 entry:
   %d = call <8 x i8> @llvm.fshr(<8 x i8> %a, <8 x i8> %b, <8 x i8> <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>)
@@ -3947,16 +3941,14 @@ entry:
 define <16 x i8> @fshl_v16i8_c(<16 x i8> %a, <16 x i8> %b) {
 ; CHECK-SD-LABEL: fshl_v16i8_c:
 ; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    ushr v1.16b, v1.16b, #1
 ; CHECK-SD-NEXT:    shl v0.16b, v0.16b, #3
-; CHECK-SD-NEXT:    usra v0.16b, v1.16b, #4
+; CHECK-SD-NEXT:    usra v0.16b, v1.16b, #5
 ; CHECK-SD-NEXT:    ret
 ;
 ; CHECK-GI-LABEL: fshl_v16i8_c:
 ; CHECK-GI:       // %bb.0: // %entry
 ; CHECK-GI-NEXT:    shl v0.16b, v0.16b, #3
-; CHECK-GI-NEXT:    ushr v1.16b, v1.16b, #5
-; CHECK-GI-NEXT:    orr v0.16b, v0.16b, v1.16b
+; CHECK-GI-NEXT:    usra v0.16b, v1.16b, #5
 ; CHECK-GI-NEXT:    ret
 entry:
   %d = call <16 x i8> @llvm.fshl(<16 x i8> %a, <16 x i8> %b, <16 x i8> <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>)
@@ -3966,16 +3958,14 @@ entry:
 define <16 x i8> @fshr_v16i8_c(<16 x i8> %a, <16 x i8> %b) {
 ; CHECK-SD-LABEL: fshr_v16i8_c:
 ; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    add v0.16b, v0.16b, v0.16b
-; CHECK-SD-NEXT:    shl v0.16b, v0.16b, #4
+; CHECK-SD-NEXT:    shl v0.16b, v0.16b, #5
 ; CHECK-SD-NEXT:    usra v0.16b, v1.16b, #3
 ; CHECK-SD-NEXT:    ret
 ;
 ; CHECK-GI-LABEL: fshr_v16i8_c:
 ; CHECK-GI:       // %bb.0: // %entry
 ; CHECK-GI-NEXT:    shl v0.16b, v0.16b, #5
-; CHECK-GI-NEXT:    ushr v1.16b, v1.16b, #3
-; CHECK-GI-NEXT:    orr v0.16b, v0.16b, v1.16b
+; CHECK-GI-NEXT:    usra v0.16b, v1.16b, #3
 ; CHECK-GI-NEXT:    ret
 entry:
   %d = call <16 x i8> @llvm.fshr(<16 x i8> %a, <16 x i8> %b, <16 x i8> <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>)
@@ -3985,16 +3975,14 @@ entry:
 define <4 x i16> @fshl_v4i16_c(<4 x i16> %a, <4 x i16> %b) {
 ; CHECK-SD-LABEL: fshl_v4i16_c:
 ; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    ushr v1.4h, v1.4h, #1
 ; CHECK-SD-NEXT:    shl v0.4h, v0.4h, #3
-; CHECK-SD-NEXT:    usra v0.4h, v1.4h, #12
+; CHECK-SD-NEXT:    usra v0.4h, v1.4h, #13
 ; CHECK-SD-NEXT:    ret
 ;
 ; CHECK-GI-LABEL: fshl_v4i16_c:
 ; CHECK-GI:       // %bb.0: // %entry
 ; CHECK-GI-NEXT:    shl v0.4h, v0.4h, #3
-; CHECK-GI-NEXT:    ushr v1.4h, v1.4h, #13
-; CHECK-GI-NEXT:    orr v0.8b, v0.8b, v1.8b
+; CHECK-GI-NEXT:    usra v0.4h, v1.4h, #13
 ; CHECK-GI-NEXT:    ret
 entry:
   %d = call <4 x i16> @llvm.fshl(<4 x i16> %a, <4 x i16> %b, <4 x i16> <i16 3, i16 3, i16 3, i16 3>)
@@ -4004,16 +3992,14 @@ entry:
 define <4 x i16> @fshr_v4i16_c(<4 x i16> %a, <4 x i16> %b) {
 ; CHECK-SD-LABEL: fshr_v4i16_c:
 ; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    add v0.4h, v0.4h, v0.4h
-; CHECK-SD-NEXT:    shl v0.4h, v0.4h, #12
+; CHECK-SD-NEXT:    shl v0.4h, v0.4h, #13
 ; CHECK-SD-NEXT:    usra v0.4h, v1.4h, #3
 ; CHECK-SD-NEXT:    ret
 ;
 ; CHECK-GI-LABEL: fshr_v4i16_c:
 ; CHECK-GI:       // %bb.0: // %entry
 ; CHECK-GI-NEXT:    shl v0.4h, v0.4h, #13
-; CHECK-GI-NEXT:    ushr v1.4h, v1.4h, #3
-; CHECK-GI-NEXT:    orr v0.8b, v0.8b, v1.8b
+; CHECK-GI-NEXT:    usra v0.4h, v1.4h, #3
 ; CHECK-GI-NEXT:    ret
 entry:
   %d = call <4 x i16> @llvm.fshr(<4 x i16> %a, <4 x i16> %b, <4 x i16> <i16 3, i16 3, i16 3, i16 3>)
@@ -4024,7 +4010,6 @@ define <7 x i16> @fshl_v7i16_c(<7 x i16> %a, <7 x i16> %b) {
 ; CHECK-SD-LABEL: fshl_v7i16_c:
 ; CHECK-SD:       // %bb.0: // %entry
 ; CHECK-SD-NEXT:    adrp x8, .LCPI124_0
-; CHECK-SD-NEXT:    ushr v1.8h, v1.8h, #1
 ; CHECK-SD-NEXT:    adrp x9, .LCPI124_1
 ; CHECK-SD-NEXT:    ldr q2, [x8, :lo12:.LCPI124_0]
 ; CHECK-SD-NEXT:    ldr q3, [x9, :lo12:.LCPI124_1]
@@ -4066,7 +4051,6 @@ define <7 x i16> @fshr_v7i16_c(<7 x i16> %a, <7 x i16> %b) {
 ; CHECK-SD:       // %bb.0: // %entry
 ; CHECK-SD-NEXT:    adrp x8, .LCPI125_0
 ; CHECK-SD-NEXT:    adrp x9, .LCPI125_1
-; CHECK-SD-NEXT:    add v0.8h, v0.8h, v0.8h
 ; CHECK-SD-NEXT:    ldr q2, [x8, :lo12:.LCPI125_0]
 ; CHECK-SD-NEXT:    ldr q3, [x9, :lo12:.LCPI125_1]
 ; CHECK-SD-NEXT:    ushl v1.8h, v1.8h, v2.8h
@@ -4105,16 +4089,14 @@ entry:
 define <8 x i16> @fshl_v8i16_c(<8 x i16> %a, <8 x i16> %b) {
 ; CHECK-SD-LABEL: fshl_v8i16_c:
 ; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    ushr v1.8h, v1.8h, #1
 ; CHECK-SD-NEXT:    shl v0.8h, v0.8h, #3
-; CHECK-SD-NEXT:    usra v0.8h, v1.8h, #12
+; CHECK-SD-NEXT:    usra v0.8h, v1.8h, #13
 ; CHECK-SD-NEXT:    ret
 ;
 ; CHECK-GI-LABEL: fshl_v8i16_c:
 ; CHECK-GI:       // %bb.0: // %entry
 ; CHECK-GI-NEXT:    shl v0.8h, v0.8h, #3
-; CHECK-GI-NEXT:    ushr v1.8h, v1.8h, #13
-; CHECK-GI-NEXT:    orr v0.16b, v0.16b, v1.16b
+; CHECK-GI-NEXT:    usra v0.8h, v1.8h, #13
 ; CHECK-GI-NEXT:    ret
 entry:
   %d = call <8 x i16> @llvm.fshl(<8 x i16> %a, <8 x i16> %b, <8 x i16> <i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3>)
@@ -4124,16 +4106,14 @@ entry:
 define <8 x i16> @fshr_v8i16_c(<8 x i16> %a, <8 x i16> %b) {
 ; CHECK-SD-LABEL: fshr_v8i16_c:
 ; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    add v0.8h, v0.8h, v0.8h
-; CHECK-SD-NEXT:    shl v0.8h, v0.8h, #12
+; CHECK-SD-NEXT:    shl v0.8h, v0.8h, #13
 ; CHECK-SD-NEXT:    usra v0.8h, v1.8h, #3
 ; CHECK-SD-NEXT:    ret
 ;
 ; CHECK-GI-LABEL: fshr_v8i16_c:
 ; CHECK-GI:       // %bb.0: // %entry
 ; CHECK-GI-NEXT:    shl v0.8h, v0.8h, #13
-; CHECK-GI-NEXT:    ushr v1.8h, v1.8h, #3
-; CHECK-GI-NEXT:    orr v0.16b, v0.16b, v1.16b
+; CHECK-GI-NEXT:    usra v0.8h, v1.8h, #3
 ; CHECK-GI-NEXT:    ret
 entry:
   %d = call <8 x i16> @llvm.fshr(<8 x i16> %a, <8 x i16> %b, <8 x i16> <i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3>)
@@ -4143,22 +4123,18 @@ entry:
 define <16 x i16> @fshl_v16i16_c(<16 x i16> %a, <16 x i16> %b) {
 ; CHECK-SD-LABEL: fshl_v16i16_c:
 ; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    ushr v2.8h, v2.8h, #1
-; CHECK-SD-NEXT:    shl v0.8h, v0.8h, #3
-; CHECK-SD-NEXT:    ushr v3.8h, v3.8h, #1
 ; CHECK-SD-NEXT:    shl v1.8h, v1.8h, #3
-; CHECK-SD-NEXT:    usra v0.8h, v2.8h, #12
-; CHECK-SD-NEXT:    usra v1.8h, v3.8h, #12
+; CHECK-SD-NEXT:    shl v0.8h, v0.8h, #3
+; CHECK-SD-NEXT:    usra v1.8h, v3.8h, #13
+; CHECK-SD-NEXT:    usra v0.8h, v2.8h, #13
 ; CHECK-SD-NEXT:    ret
 ;
 ; CHECK-GI-LABEL: fshl_v16i16_c:
 ; CHECK-GI:       // %bb.0: // %entry
 ; CHECK-GI-NEXT:    shl v0.8h, v0.8h, #3
 ; CHECK-GI-NEXT:    shl v1.8h, v1.8h, #3
-; CHECK-GI-NEXT:    ushr v2.8h, v2.8h, #13
-; CHECK-GI-NEXT:    ushr v3.8h, v3.8h, #13
-; CHECK-GI-NEXT:    orr v0.16b, v0.16b, v2.16b
-; CHECK-GI-NEXT:    orr v1.16b, v1.16b, v3.16b
+; CHECK-GI-NEXT:    usra v0.8h, v2.8h, #13
+; CHECK-GI-NEXT:    usra v1.8h, v3.8h, #13
 ; CHECK-GI-NEXT:    ret
 entry:
   %d = call <16 x i16> @llvm.fshl(<16 x i16> %a, <16 x i16> %b, <16 x i16> <i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3>)
@@ -4168,10 +4144,8 @@ entry:
 define <16 x i16> @fshr_v16i16_c(<16 x i16> %a, <16 x i16> %b) {
 ; CHECK-SD-LABEL: fshr_v16i16_c:
 ; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    add v1.8h, v1.8h, v1.8h
-; CHECK-SD-NEXT:    add v0.8h, v0.8h, v0.8h
-; CHECK-SD-NEXT:    shl v1.8h, v1.8h, #12
-; CHECK-SD-NEXT:    shl v0.8h, v0.8h, #12
+; CHECK-SD-NEXT:    shl v1.8h, v1.8h, #13
+; CHECK-SD-NEXT:    shl v0.8h, v0.8h, #13
 ; CHECK-SD-NEXT:    usra v1.8h, v3.8h, #3
 ; CHECK-SD-NEXT:    usra v0.8h, v2.8h, #3
 ; CHECK-SD-NEXT:    ret
@@ -4180,10 +4154,8 @@ define <16 x i16> @fshr_v16i16_c(<16 x i16> %a, <16 x i16> %b) {
 ; CHECK-GI:       // %bb.0: // %entry
 ; CHECK-GI-NEXT:    shl v0.8h, v0.8h, #13
 ; CHECK-GI-NEXT:    shl v1.8h, v1.8h, #13
-; CHECK-GI-NEXT:    ushr v2.8h, v2.8h, #3
-; CHECK-GI-NEXT:    ushr v3.8h, v3.8h, #3
-; CHECK-GI-NEXT:    orr v0.16b, v0.16b, v2.16b
-; CHECK-GI-NEXT:    orr v1.16b, v1.16b, v3.16b
+; CHECK-GI-NEXT:    usra v0.8h, v2.8h, #3
+; CHECK-GI-NEXT:    usra v1.8h, v3.8h, #3
 ; CHECK-GI-NEXT:    ret
 entry:
   %d = call <16 x i16> @llvm.fshr(<16 x i16> %a, <16 x i16> %b, <16 x i16> <i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3>)
@@ -4191,72 +4163,44 @@ entry:
 }
 
 define <2 x i32> @fshl_v2i32_c(<2 x i32> %a, <2 x i32> %b) {
-; CHECK-SD-LABEL: fshl_v2i32_c:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    shl v0.2s, v0.2s, #3
-; CHECK-SD-NEXT:    usra v0.2s, v1.2s, #29
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: fshl_v2i32_c:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    shl v0.2s, v0.2s, #3
-; CHECK-GI-NEXT:    ushr v1.2s, v1.2s, #29
-; CHECK-GI-NEXT:    orr v0.8b, v0.8b, v1.8b
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: fshl_v2i32_c:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    shl v0.2s, v0.2s, #3
+; CHECK-NEXT:    usra v0.2s, v1.2s, #29
+; CHECK-NEXT:    ret
 entry:
   %d = call <2 x i32> @llvm.fshl(<2 x i32> %a, <2 x i32> %b, <2 x i32> <i32 3, i32 3>)
   ret <2 x i32> %d
 }
 
 define <2 x i32> @fshr_v2i32_c(<2 x i32> %a, <2 x i32> %b) {
-; CHECK-SD-LABEL: fshr_v2i32_c:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    shl v0.2s, v0.2s, #29
-; CHECK-SD-NEXT:    usra v0.2s, v1.2s, #3
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: fshr_v2i32_c:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    shl v0.2s, v0.2s, #29
-; CHECK-GI-NEXT:    ushr v1.2s, v1.2s, #3
-; CHECK-GI-NEXT:    orr v0.8b, v0.8b, v1.8b
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: fshr_v2i32_c:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    shl v0.2s, v0.2s, #29
+; CHECK-NEXT:    usra v0.2s, v1.2s, #3
+; CHECK-NEXT:    ret
 entry:
   %d = call <2 x i32> @llvm.fshr(<2 x i32> %a, <2 x i32> %b, <2 x i32> <i32 3, i32 3>)
   ret <2 x i32> %d
 }
 
 define <4 x i32> @fshl_v4i32_c(<4 x i32> %a, <4 x i32> %b) {
-; CHECK-SD-LABEL: fshl_v4i32_c:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    shl v0.4s, v0.4s, #3
-; CHECK-SD-NEXT:    usra v0.4s, v1.4s, #29
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: fshl_v4i32_c:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    shl v0.4s, v0.4s, #3
-; CHECK-GI-NEXT:    ushr v1.4s, v1.4s, #29
-; CHECK-GI-NEXT:    orr v0.16b, v0.16b, v1.16b
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: fshl_v4i32_c:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    shl v0.4s, v0.4s, #3
+; CHECK-NEXT:    usra v0.4s, v1.4s, #29
+; CHECK-NEXT:    ret
 entry:
   %d = call <4 x i32> @llvm.fshl(<4 x i32> %a, <4 x i32> %b, <4 x i32> <i32 3, i32 3, i32 3, i32 3>)
   ret <4 x i32> %d
 }
 
 define <4 x i32> @fshr_v4i32_c(<4 x i32> %a, <4 x i32> %b) {
-; CHECK-SD-LABEL: fshr_v4i32_c:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    shl v0.4s, v0.4s, #29
-; CHECK-SD-NEXT:    usra v0.4s, v1.4s, #3
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: fshr_v4i32_c:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    shl v0.4s, v0.4s, #29
-; CHECK-GI-NEXT:    ushr v1.4s, v1.4s, #3
-; CHECK-GI-NEXT:    orr v0.16b, v0.16b, v1.16b
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: fshr_v4i32_c:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    shl v0.4s, v0.4s, #29
+; CHECK-NEXT:    usra v0.4s, v1.4s, #3
+; CHECK-NEXT:    ret
 entry:
   %d = call <4 x i32> @llvm.fshr(<4 x i32> %a, <4 x i32> %b, <4 x i32> <i32 3, i32 3, i32 3, i32 3>)
   ret <4 x i32> %d
@@ -4305,39 +4249,38 @@ define <7 x i32> @fshl_v7i32_c(<7 x i32> %a, <7 x i32> %b) {
 ; CHECK-GI-LABEL: fshl_v7i32_c:
 ; CHECK-GI:       // %bb.0: // %entry
 ; CHECK-GI-NEXT:    mov v0.s[0], w0
-; CHECK-GI-NEXT:    mov v1.s[0], w7
 ; CHECK-GI-NEXT:    mov w8, #29 // =0x1d
-; CHECK-GI-NEXT:    mov v3.s[0], w8
-; CHECK-GI-NEXT:    ldr s2, [sp]
+; CHECK-GI-NEXT:    mov v2.s[0], w7
+; CHECK-GI-NEXT:    mov v1.s[0], w8
 ; CHECK-GI-NEXT:    mov w9, #3 // =0x3
-; CHECK-GI-NEXT:    mov v4.s[0], w9
-; CHECK-GI-NEXT:    ldr s5, [sp, #8]
+; CHECK-GI-NEXT:    mov v4.s[0], w4
+; CHECK-GI-NEXT:    mov v5.s[0], w9
+; CHECK-GI-NEXT:    ldr s3, [sp]
 ; CHECK-GI-NEXT:    ldr s6, [sp, #24]
 ; CHECK-GI-NEXT:    ldr s7, [sp, #32]
 ; CHECK-GI-NEXT:    mov v0.s[1], w1
-; CHECK-GI-NEXT:    mov v1.s[1], v2.s[0]
-; CHECK-GI-NEXT:    mov v2.s[0], w4
-; CHECK-GI-NEXT:    mov v3.s[1], w8
+; CHECK-GI-NEXT:    mov v2.s[1], v3.s[0]
+; CHECK-GI-NEXT:    ldr s3, [sp, #8]
+; CHECK-GI-NEXT:    mov v1.s[1], w8
 ; CHECK-GI-NEXT:    mov v6.s[1], v7.s[0]
+; CHECK-GI-NEXT:    mov v4.s[1], w5
+; CHECK-GI-NEXT:    mov v5.s[1], w9
 ; CHECK-GI-NEXT:    ldr s7, [sp, #40]
-; CHECK-GI-NEXT:    mov v4.s[1], w9
 ; CHECK-GI-NEXT:    mov v0.s[2], w2
-; CHECK-GI-NEXT:    mov v1.s[2], v5.s[0]
-; CHECK-GI-NEXT:    mov v2.s[1], w5
-; CHECK-GI-NEXT:    mov v3.s[2], w8
-; CHECK-GI-NEXT:    ldr s5, [sp, #16]
+; CHECK-GI-NEXT:    mov v2.s[2], v3.s[0]
+; CHECK-GI-NEXT:    ldr s3, [sp, #16]
+; CHECK-GI-NEXT:    mov v1.s[2], w8
 ; CHECK-GI-NEXT:    mov v6.s[2], v7.s[0]
-; CHECK-GI-NEXT:    mov v4.s[2], w9
+; CHECK-GI-NEXT:    mov v4.s[2], w6
+; CHECK-GI-NEXT:    mov v5.s[2], w9
 ; CHECK-GI-NEXT:    mov v0.s[3], w3
-; CHECK-GI-NEXT:    mov v1.s[3], v5.s[0]
-; CHECK-GI-NEXT:    mov v2.s[2], w6
-; CHECK-GI-NEXT:    neg v3.4s, v3.4s
+; CHECK-GI-NEXT:    mov v2.s[3], v3.s[0]
+; CHECK-GI-NEXT:    neg v1.4s, v1.4s
+; CHECK-GI-NEXT:    ushl v3.4s, v4.4s, v5.4s
 ; CHECK-GI-NEXT:    shl v0.4s, v0.4s, #3
-; CHECK-GI-NEXT:    ushr v1.4s, v1.4s, #29
-; CHECK-GI-NEXT:    ushl v2.4s, v2.4s, v4.4s
-; CHECK-GI-NEXT:    ushl v3.4s, v6.4s, v3.4s
-; CHECK-GI-NEXT:    orr v0.16b, v0.16b, v1.16b
-; CHECK-GI-NEXT:    orr v1.16b, v2.16b, v3.16b
+; CHECK-GI-NEXT:    ushl v1.4s, v6.4s, v1.4s
+; CHECK-GI-NEXT:    usra v0.4s, v2.4s, #29
+; CHECK-GI-NEXT:    orr v1.16b, v3.16b, v1.16b
 ; CHECK-GI-NEXT:    mov s2, v0.s[1]
 ; CHECK-GI-NEXT:    mov s3, v0.s[2]
 ; CHECK-GI-NEXT:    mov s4, v0.s[3]
@@ -4399,39 +4342,38 @@ define <7 x i32> @fshr_v7i32_c(<7 x i32> %a, <7 x i32> %b) {
 ; CHECK-GI-LABEL: fshr_v7i32_c:
 ; CHECK-GI:       // %bb.0: // %entry
 ; CHECK-GI-NEXT:    mov v0.s[0], w0
-; CHECK-GI-NEXT:    mov v1.s[0], w7
 ; CHECK-GI-NEXT:    mov w8, #3 // =0x3
-; CHECK-GI-NEXT:    mov v3.s[0], w8
-; CHECK-GI-NEXT:    ldr s2, [sp]
+; CHECK-GI-NEXT:    mov v2.s[0], w7
+; CHECK-GI-NEXT:    mov v1.s[0], w8
 ; CHECK-GI-NEXT:    mov w9, #29 // =0x1d
-; CHECK-GI-NEXT:    mov v4.s[0], w9
-; CHECK-GI-NEXT:    ldr s5, [sp, #8]
+; CHECK-GI-NEXT:    mov v4.s[0], w4
+; CHECK-GI-NEXT:    mov v5.s[0], w9
+; CHECK-GI-NEXT:    ldr s3, [sp]
 ; CHECK-GI-NEXT:    ldr s6, [sp, #24]
 ; CHECK-GI-NEXT:    ldr s7, [sp, #32]
 ; CHECK-GI-NEXT:    mov v0.s[1], w1
-; CHECK-GI-NEXT:    mov v1.s[1], v2.s[0]
-; CHECK-GI-NEXT:    mov v2.s[0], w4
-; CHECK-GI-NEXT:    mov v3.s[1], w8
+; CHECK-GI-NEXT:    mov v2.s[1], v3.s[0]
+; CHECK-GI-NEXT:    ldr s3, [sp, #8]
+; CHECK-GI-NEXT:    mov v1.s[1], w8
 ; CHECK-GI-NEXT:    mov v6.s[1], v7.s[0]
+; CHECK-GI-NEXT:    mov v4.s[1], w5
+; CHECK-GI-NEXT:    mov v5.s[1], w9
 ; CHECK-GI-NEXT:    ldr s7, [sp, #40]
-; CHECK-GI-NEXT:    mov v4.s[1], w9
 ; CHECK-GI-NEXT:    mov v0.s[2], w2
-; CHECK-GI-NEXT:    mov v1.s[2], v5.s[0]
-; CHECK-GI-NEXT:    mov v2.s[1], w5
-; CHECK-GI-NEXT:    mov v3.s[2], w8
-; CHECK-GI-NEXT:    ldr s5, [sp, #16]
+; CHECK-GI-NEXT:    mov v2.s[2], v3.s[0]
+; CHECK-GI-NEXT:    ldr s3, [sp, #16]
+; CHECK-GI-NEXT:    mov v1.s[2], w8
 ; CHECK-GI-NEXT:    mov v6.s[2], v7.s[0]
-; CHECK-GI-NEXT:    mov v4.s[2], w9
+; CHECK-GI-NEXT:    mov v4.s[2], w6
+; CHECK-GI-NEXT:    mov v5.s[2], w9
 ; CHECK-GI-NEXT:    mov v0.s[3], w3
-; CHECK-GI-NEXT:    mov v1.s[3], v5.s[0]
-; CHECK-GI-NEXT:    mov v2.s[2], w6
-; CHECK-GI-NEXT:    neg v3.4s, v3.4s
+; CHECK-GI-NEXT:    mov v2.s[3], v3.s[0]
+; CHECK-GI-NEXT:    neg v1.4s, v1.4s
+; CHECK-GI-NEXT:    ushl v3.4s, v4.4s, v5.4s
 ; CHECK-GI-NEXT:    shl v0.4s, v0.4s, #29
-; CHECK-GI-NEXT:    ushr v1.4s, v1.4s, #3
-; CHECK-GI-NEXT:    ushl v2.4s, v2.4s, v4.4s
-; CHECK-GI-NEXT:    ushl v3.4s, v6.4s, v3.4s
-; CHECK-GI-NEXT:    orr v0.16b, v0.16b, v1.16b
-; CHECK-GI-NEXT:    orr v1.16b, v2.16b, v3.16b
+; CHECK-GI-NEXT:    ushl v1.4s, v6.4s, v1.4s
+; CHECK-GI-NEXT:    usra v0.4s, v2.4s, #3
+; CHECK-GI-NEXT:    orr v1.16b, v3.16b, v1.16b
 ; CHECK-GI-NEXT:    mov s2, v0.s[1]
 ; CHECK-GI-NEXT:    mov s3, v0.s[2]
 ; CHECK-GI-NEXT:    mov s4, v0.s[3]
@@ -4463,10 +4405,8 @@ define <8 x i32> @fshl_v8i32_c(<8 x i32> %a, <8 x i32> %b) {
 ; CHECK-GI:       // %bb.0: // %entry
 ; CHECK-GI-NEXT:    shl v0.4s, v0.4s, #3
 ; CHECK-GI-NEXT:    shl v1.4s, v1.4s, #3
-; CHECK-GI-NEXT:    ushr v2.4s, v2.4s, #29
-; CHECK-GI-NEXT:    ushr v3.4s, v3.4s, #29
-; CHECK-GI-NEXT:    orr v0.16b, v0.16b, v2.16b
-; CHECK-GI-NEXT:    orr v1.16b, v1.16b, v3.16b
+; CHECK-GI-NEXT:    usra v0.4s, v2.4s, #29
+; CHECK-GI-NEXT:    usra v1.4s, v3.4s, #29
 ; CHECK-GI-NEXT:    ret
 entry:
   %d = call <8 x i32> @llvm.fshl(<8 x i32> %a, <8 x i32> %b, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>)
@@ -4486,10 +4426,8 @@ define <8 x i32> @fshr_v8i32_c(<8 x i32> %a, <8 x i32> %b) {
 ; CHECK-GI:       // %bb.0: // %entry
 ; CHECK-GI-NEXT:    shl v0.4s, v0.4s, #29
 ; CHECK-GI-NEXT:    shl v1.4s, v1.4s, #29
-; CHECK-GI-NEXT:    ushr v2.4s, v2.4s, #3
-; CHECK-GI-NEXT:    ushr v3.4s, v3.4s, #3
-; CHECK-GI-NEXT:    orr v0.16b, v0.16b, v2.16b
-; CHECK-GI-NEXT:    orr v1.16b, v1.16b, v3.16b
+; CHECK-GI-NEXT:    usra v0.4s, v2.4s, #3
+; CHECK-GI-NEXT:    usra v1.4s, v3.4s, #3
 ; CHECK-GI-NEXT:    ret
 entry:
   %d = call <8 x i32> @llvm.fshr(<8 x i32> %a, <8 x i32> %b, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>)
@@ -4497,36 +4435,22 @@ entry:
 }
 
 define <2 x i64> @fshl_v2i64_c(<2 x i64> %a, <2 x i64> %b) {
-; CHECK-SD-LABEL: fshl_v2i64_c:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    shl v0.2d, v0.2d, #3
-; CHECK-SD-NEXT:    usra v0.2d, v1.2d, #61
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: fshl_v2i64_c:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    shl v0.2d, v0.2d, #3
-; CHECK-GI-NEXT:    ushr v1.2d, v1.2d, #61
-; CHECK-GI-NEXT:    orr v0.16b, v0.16b, v1.16b
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: fshl_v2i64_c:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    shl v0.2d, v0.2d, #3
+; CHECK-NEXT:    usra v0.2d, v1.2d, #61
+; CHECK-NEXT:    ret
 entry:
   %d = call <2 x i64> @llvm.fshl(<2 x i64> %a, <2 x i64> %b, <2 x i64> <i64 3, i64 3>)
   ret <2 x i64> %d
 }
 
 define <2 x i64> @fshr_v2i64_c(<2 x i64> %a, <2 x i64> %b) {
-; CHECK-SD-LABEL: fshr_v2i64_c:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    shl v0.2d, v0.2d, #61
-; CHECK-SD-NEXT:    usra v0.2d, v1.2d, #3
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: fshr_v2i64_c:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    shl v0.2d, v0.2d, #61
-; CHECK-GI-NEXT:    ushr v1.2d, v1.2d, #3
-; CHECK-GI-NEXT:    orr v0.16b, v0.16b, v1.16b
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: fshr_v2i64_c:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    shl v0.2d, v0.2d, #61
+; CHECK-NEXT:    usra v0.2d, v1.2d, #3
+; CHECK-NEXT:    ret
 entry:
   %d = call <2 x i64> @llvm.fshr(<2 x i64> %a, <2 x i64> %b, <2 x i64> <i64 3, i64 3>)
   ret <2 x i64> %d
@@ -4545,10 +4469,8 @@ define <4 x i64> @fshl_v4i64_c(<4 x i64> %a, <4 x i64> %b) {
 ; CHECK-GI:       // %bb.0: // %entry
 ; CHECK-GI-NEXT:    shl v0.2d, v0.2d, #3
 ; CHECK-GI-NEXT:    shl v1.2d, v1.2d, #3
-; CHECK-GI-NEXT:    ushr v2.2d, v2.2d, #61
-; CHECK-GI-NEXT:    ushr v3.2d, v3.2d, #61
-; CHECK-GI-NEXT:    orr v0.16b, v0.16b, v2.16b
-; CHECK-GI-NEXT:    orr v1.16b, v1.16b, v3.16b
+; CHECK-GI-NEXT:    usra v0.2d, v2.2d, #61
+; CHECK-GI-NEXT:    usra v1.2d, v3.2d, #61
 ; CHECK-GI-NEXT:    ret
 entry:
   %d = call <4 x i64> @llvm.fshl(<4 x i64> %a, <4 x i64> %b, <4 x i64> <i64 3, i64 3, i64 3, i64 3>)
@@ -4568,10 +4490,8 @@ define <4 x i64> @fshr_v4i64_c(<4 x i64> %a, <4 x i64> %b) {
 ; CHECK-GI:       // %bb.0: // %entry
 ; CHECK-GI-NEXT:    shl v0.2d, v0.2d, #61
 ; CHECK-GI-NEXT:    shl v1.2d, v1.2d, #61
-; CHECK-GI-NEXT:    ushr v2.2d, v2.2d, #3
-; CHECK-GI-NEXT:    ushr v3.2d, v3.2d, #3
-; CHECK-GI-NEXT:    orr v0.16b, v0.16b, v2.16b
-; CHECK-GI-NEXT:    orr v1.16b, v1.16b, v3.16b
+; CHECK-GI-NEXT:    usra v0.2d, v2.2d, #3
+; CHECK-GI-NEXT:    usra v1.2d, v3.2d, #3
 ; CHECK-GI-NEXT:    ret
 entry:
   %d = call <4 x i64> @llvm.fshr(<4 x i64> %a, <4 x i64> %b, <4 x i64> <i64 3, i64 3, i64 3, i64 3>)
diff --git a/llvm/test/CodeGen/AArch64/funnel-shift.ll b/llvm/test/CodeGen/AArch64/funnel-shift.ll
index 3037a9552bc27..e5aa360f804c1 100644
--- a/llvm/test/CodeGen/AArch64/funnel-shift.ll
+++ b/llvm/test/CodeGen/AArch64/funnel-shift.ll
@@ -21,7 +21,6 @@ define i32 @fshl_i32(i32 %x, i32 %y, i32 %z) {
 ; CHECK-SD-LABEL: fshl_i32:
 ; CHECK-SD:       // %bb.0:
 ; CHECK-SD-NEXT:    lsr w8, w1, #1
-; CHECK-SD-NEXT:    // kill: def $w2 killed $w2 def $x2
 ; CHECK-SD-NEXT:    mvn w9, w2
 ; CHECK-SD-NEXT:    lsl w10, w0, w2
 ; CHECK-SD-NEXT:    lsr w8, w8, w9
@@ -266,7 +265,6 @@ define i32 @fshr_i32(i32 %x, i32 %y, i32 %z) {
 ; CHECK-SD-LABEL: fshr_i32:
 ; CHECK-SD:       // %bb.0:
 ; CHECK-SD-NEXT:    lsl w8, w0, #1
-; CHECK-SD-NEXT:    // kill: def $w2 killed $w2 def $x2
 ; CHECK-SD-NEXT:    mvn w9, w2
 ; CHECK-SD-NEXT:    lsr w10, w1, w2
 ; CHECK-SD-NEXT:    lsl w8, w8, w9
@@ -667,7 +665,6 @@ define i32 @or_shl_fshl_simplify(i32 %x, i32 %y, i32 %s) {
 ; CHECK-SD-LABEL: or_shl_fshl_simplify:
 ; CHECK-SD:       // %bb.0:
 ; CHECK-SD-NEXT:    lsr w8, w0, #1
-; CHECK-SD-NEXT:    // kill: def $w2 killed $w2 def $x2
 ; CHECK-SD-NEXT:    mvn w9, w2
 ; CHECK-SD-NEXT:    lsl w10, w1, w2
 ; CHECK-SD-NEXT:    lsr w8, w8, w9
@@ -696,7 +693,6 @@ define i32 @or_lshr_fshr_simplify(i32 %x, i32 %y, i32 %s) {
 ; CHECK-SD-LABEL: or_lshr_fshr_simplify:
 ; CHECK-SD:       // %bb.0:
 ; CHECK-SD-NEXT:    lsl w8, w0, #1
-; CHECK-SD-NEXT:    // kill: def $w2 killed $w2 def $x2
 ; CHECK-SD-NEXT:    mvn w9, w2
 ; CHECK-SD-NEXT:    lsr w10, w1, w2
 ; CHECK-SD-NEXT:    lsl w8, w8, w9
diff --git a/llvm/test/CodeGen/AArch64/hoist-and-by-const-from-lshr-in-eqcmp-zero.ll b/llvm/test/CodeGen/AArch64/hoist-and-by-const-from-lshr-in-eqcmp-zero.ll
index cb9f04a7fac48..c3fdc7db2abbe 100644
--- a/llvm/test/CodeGen/AArch64/hoist-and-by-const-from-lshr-in-eqcmp-zero.ll
+++ b/llvm/test/CodeGen/AArch64/hoist-and-by-const-from-lshr-in-eqcmp-zero.ll
@@ -15,7 +15,6 @@
 define i1 @scalar_i8_signbit_eq(i8 %x, i8 %y) nounwind {
 ; CHECK-LABEL: scalar_i8_signbit_eq:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    // kill: def $w1 killed $w1 def $x1
 ; CHECK-NEXT:    lsl w8, w0, w1
 ; CHECK-NEXT:    tst w8, #0x80
 ; CHECK-NEXT:    cset w0, eq
@@ -29,7 +28,6 @@ define i1 @scalar_i8_signbit_eq(i8 %x, i8 %y) nounwind {
 define i1 @scalar_i8_lowestbit_eq(i8 %x, i8 %y) nounwind {
 ; CHECK-LABEL: scalar_i8_lowestbit_eq:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    // kill: def $w1 killed $w1 def $x1
 ; CHECK-NEXT:    lsl w8, w0, w1
 ; CHECK-NEXT:    tst w8, #0x1
 ; CHECK-NEXT:    cset w0, eq
@@ -43,7 +41,6 @@ define i1 @scalar_i8_lowestbit_eq(i8 %x, i8 %y) nounwind {
 define i1 @scalar_i8_bitsinmiddle_eq(i8 %x, i8 %y) nounwind {
 ; CHECK-LABEL: scalar_i8_bitsinmiddle_eq:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    // kill: def $w1 killed $w1 def $x1
 ; CHECK-NEXT:    lsl w8, w0, w1
 ; CHECK-NEXT:    tst w8, #0x18
 ; CHECK-NEXT:    cset w0, eq
@@ -59,7 +56,6 @@ define i1 @scalar_i8_bitsinmiddle_eq(i8 %x, i8 %y) nounwind {
 define i1 @scalar_i16_signbit_eq(i16 %x, i16 %y) nounwind {
 ; CHECK-LABEL: scalar_i16_signbit_eq:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    // kill: def $w1 killed $w1 def $x1
 ; CHECK-NEXT:    lsl w8, w0, w1
 ; CHECK-NEXT:    tst w8, #0x8000
 ; CHECK-NEXT:    cset w0, eq
@@ -73,7 +69,6 @@ define i1 @scalar_i16_signbit_eq(i16 %x, i16 %y) nounwind {
 define i1 @scalar_i16_lowestbit_eq(i16 %x, i16 %y) nounwind {
 ; CHECK-LABEL: scalar_i16_lowestbit_eq:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    // kill: def $w1 killed $w1 def $x1
 ; CHECK-NEXT:    lsl w8, w0, w1
 ; CHECK-NEXT:    tst w8, #0x1
 ; CHECK-NEXT:    cset w0, eq
@@ -87,7 +82,6 @@ define i1 @scalar_i16_lowestbit_eq(i16 %x, i16 %y) nounwind {
 define i1 @scalar_i16_bitsinmiddle_eq(i16 %x, i16 %y) nounwind {
 ; CHECK-LABEL: scalar_i16_bitsinmiddle_eq:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    // kill: def $w1 killed $w1 def $x1
 ; CHECK-NEXT:    lsl w8, w0, w1
 ; CHECK-NEXT:    tst w8, #0xff0
 ; CHECK-NEXT:    cset w0, eq
@@ -268,7 +262,6 @@ define <4 x i1> @vec_4xi32_nonsplat_undef2_eq(<4 x i32> %x, <4 x i32> %y) nounwi
 define i1 @scalar_i8_signbit_ne(i8 %x, i8 %y) nounwind {
 ; CHECK-LABEL: scalar_i8_signbit_ne:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    // kill: def $w1 killed $w1 def $x1
 ; CHECK-NEXT:    lsl w8, w0, w1
 ; CHECK-NEXT:    ubfx w0, w8, #7, #1
 ; CHECK-NEXT:    ret
@@ -325,7 +318,6 @@ define i1 @scalar_i8_signbit_eq_with_nonzero(i8 %x, i8 %y) nounwind {
 ; CHECK-LABEL: scalar_i8_signbit_eq_with_nonzero:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    mov w8, #128 // =0x80
-; CHECK-NEXT:    // kill: def $w1 killed $w1 def $x1
 ; CHECK-NEXT:    lsr w8, w8, w1
 ; CHECK-NEXT:    and w8, w8, w0
 ; CHECK-NEXT:    cmp w8, #1
diff --git a/llvm/test/CodeGen/AArch64/hoist-and-by-const-from-shl-in-eqcmp-zero.ll b/llvm/test/CodeGen/AArch64/hoist-and-by-const-from-shl-in-eqcmp-zero.ll
index 32a62453202f4..4a73b10811d29 100644
--- a/llvm/test/CodeGen/AArch64/hoist-and-by-const-from-shl-in-eqcmp-zero.ll
+++ b/llvm/test/CodeGen/AArch64/hoist-and-by-const-from-shl-in-eqcmp-zero.ll
@@ -16,7 +16,6 @@ define i1 @scalar_i8_signbit_eq(i8 %x, i8 %y) nounwind {
 ; CHECK-LABEL: scalar_i8_signbit_eq:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    and w8, w0, #0xff
-; CHECK-NEXT:    // kill: def $w1 killed $w1 def $x1
 ; CHECK-NEXT:    lsr w8, w8, w1
 ; CHECK-NEXT:    tst w8, #0x80
 ; CHECK-NEXT:    cset w0, eq
@@ -31,7 +30,6 @@ define i1 @scalar_i8_lowestbit_eq(i8 %x, i8 %y) nounwind {
 ; CHECK-LABEL: scalar_i8_lowestbit_eq:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    and w8, w0, #0xff
-; CHECK-NEXT:    // kill: def $w1 killed $w1 def $x1
 ; CHECK-NEXT:    lsr w8, w8, w1
 ; CHECK-NEXT:    tst w8, #0x1
 ; CHECK-NEXT:    cset w0, eq
@@ -46,7 +44,6 @@ define i1 @scalar_i8_bitsinmiddle_eq(i8 %x, i8 %y) nounwind {
 ; CHECK-LABEL: scalar_i8_bitsinmiddle_eq:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    and w8, w0, #0xff
-; CHECK-NEXT:    // kill: def $w1 killed $w1 def $x1
 ; CHECK-NEXT:    lsr w8, w8, w1
 ; CHECK-NEXT:    tst w8, #0x18
 ; CHECK-NEXT:    cset w0, eq
@@ -63,7 +60,6 @@ define i1 @scalar_i16_signbit_eq(i16 %x, i16 %y) nounwind {
 ; CHECK-LABEL: scalar_i16_signbit_eq:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    and w8, w0, #0xffff
-; CHECK-NEXT:    // kill: def $w1 killed $w1 def $x1
 ; CHECK-NEXT:    lsr w8, w8, w1
 ; CHECK-NEXT:    tst w8, #0x8000
 ; CHECK-NEXT:    cset w0, eq
@@ -78,7 +74,6 @@ define i1 @scalar_i16_lowestbit_eq(i16 %x, i16 %y) nounwind {
 ; CHECK-LABEL: scalar_i16_lowestbit_eq:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    and w8, w0, #0xffff
-; CHECK-NEXT:    // kill: def $w1 killed $w1 def $x1
 ; CHECK-NEXT:    lsr w8, w8, w1
 ; CHECK-NEXT:    tst w8, #0x1
 ; CHECK-NEXT:    cset w0, eq
@@ -93,7 +88,6 @@ define i1 @scalar_i16_bitsinmiddle_eq(i16 %x, i16 %y) nounwind {
 ; CHECK-LABEL: scalar_i16_bitsinmiddle_eq:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    and w8, w0, #0xffff
-; CHECK-NEXT:    // kill: def $w1 killed $w1 def $x1
 ; CHECK-NEXT:    lsr w8, w8, w1
 ; CHECK-NEXT:    tst w8, #0xff0
 ; CHECK-NEXT:    cset w0, eq
@@ -272,7 +266,6 @@ define i1 @scalar_i8_signbit_ne(i8 %x, i8 %y) nounwind {
 ; CHECK-LABEL: scalar_i8_signbit_ne:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    and w8, w0, #0xff
-; CHECK-NEXT:    // kill: def $w1 killed $w1 def $x1
 ; CHECK-NEXT:    lsr w8, w8, w1
 ; CHECK-NEXT:    lsr w0, w8, #7
 ; CHECK-NEXT:    ret
@@ -289,7 +282,7 @@ define i1 @scalar_i8_signbit_ne(i8 %x, i8 %y) nounwind {
 define i1 @scalar_i32_x_is_const_eq(i32 %y) nounwind {
 ; CHECK-LABEL: scalar_i32_x_is_const_eq:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    mov w8, #43605
+; CHECK-NEXT:    mov w8, #43605 // =0xaa55
 ; CHECK-NEXT:    movk w8, #43605, lsl #16
 ; CHECK-NEXT:    lsl w8, w8, w0
 ; CHECK-NEXT:    tst w8, #0x1
@@ -303,8 +296,8 @@ define i1 @scalar_i32_x_is_const_eq(i32 %y) nounwind {
 define i1 @scalar_i32_x_is_const2_eq(i32 %y) nounwind {
 ; CHECK-LABEL: scalar_i32_x_is_const2_eq:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    mov w8, #1
-; CHECK-NEXT:    mov w9, #43605
+; CHECK-NEXT:    mov w8, #1 // =0x1
+; CHECK-NEXT:    mov w9, #43605 // =0xaa55
 ; CHECK-NEXT:    lsl w8, w8, w0
 ; CHECK-NEXT:    movk w9, #43605, lsl #16
 ; CHECK-NEXT:    tst w8, w9
@@ -319,8 +312,7 @@ define i1 @scalar_i32_x_is_const2_eq(i32 %y) nounwind {
 define i1 @scalar_i8_bitsinmiddle_slt(i8 %x, i8 %y) nounwind {
 ; CHECK-LABEL: scalar_i8_bitsinmiddle_slt:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    mov w8, #24
-; CHECK-NEXT:    // kill: def $w1 killed $w1 def $x1
+; CHECK-NEXT:    mov w8, #24 // =0x18
 ; CHECK-NEXT:    lsl w8, w8, w1
 ; CHECK-NEXT:    and w8, w8, w0
 ; CHECK-NEXT:    ubfx w0, w8, #7, #1
@@ -334,8 +326,7 @@ define i1 @scalar_i8_bitsinmiddle_slt(i8 %x, i8 %y) nounwind {
 define i1 @scalar_i8_signbit_eq_with_nonzero(i8 %x, i8 %y) nounwind {
 ; CHECK-LABEL: scalar_i8_signbit_eq_with_nonzero:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    mov w8, #-128
-; CHECK-NEXT:    // kill: def $w1 killed $w1 def $x1
+; CHECK-NEXT:    mov w8, #-128 // =0xffffff80
 ; CHECK-NEXT:    lsl w8, w8, w1
 ; CHECK-NEXT:    and w8, w8, w0
 ; CHECK-NEXT:    and w8, w8, #0x80
diff --git a/llvm/test/CodeGen/AArch64/llvm.modf.ll b/llvm/test/CodeGen/AArch64/llvm.modf.ll
new file mode 100644
index 0000000000000..41fe796daca86
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/llvm.modf.ll
@@ -0,0 +1,255 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 2
+; RUN: llc -mtriple=aarch64-gnu-linux < %s | FileCheck -check-prefixes=CHECK %s
+
+define { half, half } @test_modf_f16(half %a) {
+; CHECK-LABEL: test_modf_f16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    .cfi_offset w30, -16
+; CHECK-NEXT:    fcvt s0, h0
+; CHECK-NEXT:    add x0, sp, #12
+; CHECK-NEXT:    bl modff
+; CHECK-NEXT:    ldr s1, [sp, #12]
+; CHECK-NEXT:    fcvt h0, s0
+; CHECK-NEXT:    fcvt h1, s1
+; CHECK-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    ret
+  %result = call { half, half } @llvm.modf.f16(half %a)
+  ret { half, half } %result
+}
+
+define half @test_modf_f16_only_use_fractional_part(half %a) {
+; CHECK-LABEL: test_modf_f16_only_use_fractional_part:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    .cfi_offset w30, -16
+; CHECK-NEXT:    fcvt s0, h0
+; CHECK-NEXT:    add x0, sp, #12
+; CHECK-NEXT:    bl modff
+; CHECK-NEXT:    fcvt h0, s0
+; CHECK-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    ret
+  %result = call { half, half } @llvm.modf.f16(half %a)
+  %result.0 = extractvalue { half, half } %result, 0
+  ret half %result.0
+}
+
+define half @test_modf_f16_only_use_integral_part(half %a) {
+; CHECK-LABEL: test_modf_f16_only_use_integral_part:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    .cfi_offset w30, -16
+; CHECK-NEXT:    fcvt s0, h0
+; CHECK-NEXT:    add x0, sp, #12
+; CHECK-NEXT:    bl modff
+; CHECK-NEXT:    ldr s0, [sp, #12]
+; CHECK-NEXT:    fcvt h0, s0
+; CHECK-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    ret
+  %result = call { half, half } @llvm.modf.f16(half %a)
+  %result.1 = extractvalue { half, half } %result, 1
+  ret half %result.1
+}
+
+define { <2 x half>, <2 x half> } @test_modf_v2f16(<2 x half> %a) {
+; CHECK-LABEL: test_modf_v2f16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    sub sp, sp, #64
+; CHECK-NEXT:    str x30, [sp, #48] // 8-byte Folded Spill
+; CHECK-NEXT:    .cfi_def_cfa_offset 64
+; CHECK-NEXT:    .cfi_offset w30, -16
+; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-NEXT:    mov h1, v0.h[1]
+; CHECK-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; CHECK-NEXT:    add x0, sp, #44
+; CHECK-NEXT:    fcvt s0, h1
+; CHECK-NEXT:    bl modff
+; CHECK-NEXT:    ldr q1, [sp] // 16-byte Folded Reload
+; CHECK-NEXT:    fcvt h0, s0
+; CHECK-NEXT:    add x0, sp, #40
+; CHECK-NEXT:    fcvt s1, h1
+; CHECK-NEXT:    str q0, [sp, #16] // 16-byte Folded Spill
+; CHECK-NEXT:    fmov s0, s1
+; CHECK-NEXT:    bl modff
+; CHECK-NEXT:    ldr q1, [sp] // 16-byte Folded Reload
+; CHECK-NEXT:    fcvt h2, s0
+; CHECK-NEXT:    add x0, sp, #56
+; CHECK-NEXT:    mov h1, v1.h[2]
+; CHECK-NEXT:    fcvt s0, h1
+; CHECK-NEXT:    ldr q1, [sp, #16] // 16-byte Folded Reload
+; CHECK-NEXT:    mov v2.h[1], v1.h[0]
+; CHECK-NEXT:    str q2, [sp, #16] // 16-byte Folded Spill
+; CHECK-NEXT:    bl modff
+; CHECK-NEXT:    ldr q1, [sp] // 16-byte Folded Reload
+; CHECK-NEXT:    fcvt h2, s0
+; CHECK-NEXT:    add x0, sp, #60
+; CHECK-NEXT:    mov h1, v1.h[3]
+; CHECK-NEXT:    fcvt s0, h1
+; CHECK-NEXT:    ldr q1, [sp, #16] // 16-byte Folded Reload
+; CHECK-NEXT:    mov v1.h[2], v2.h[0]
+; CHECK-NEXT:    str q1, [sp, #16] // 16-byte Folded Spill
+; CHECK-NEXT:    bl modff
+; CHECK-NEXT:    ldp s2, s1, [sp, #40]
+; CHECK-NEXT:    fcvt h4, s0
+; CHECK-NEXT:    ldr q0, [sp, #16] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr x30, [sp, #48] // 8-byte Folded Reload
+; CHECK-NEXT:    fcvt h3, s1
+; CHECK-NEXT:    fcvt h1, s2
+; CHECK-NEXT:    ldr s2, [sp, #56]
+; CHECK-NEXT:    mov v0.h[3], v4.h[0]
+; CHECK-NEXT:    fcvt h2, s2
+; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-NEXT:    mov v1.h[1], v3.h[0]
+; CHECK-NEXT:    ldr s3, [sp, #60]
+; CHECK-NEXT:    mov v1.h[2], v2.h[0]
+; CHECK-NEXT:    fcvt h2, s3
+; CHECK-NEXT:    mov v1.h[3], v2.h[0]
+; CHECK-NEXT:    // kill: def $d1 killed $d1 killed $q1
+; CHECK-NEXT:    add sp, sp, #64
+; CHECK-NEXT:    ret
+  %result = call { <2 x half>, <2 x half> } @llvm.modf.v2f16(<2 x half> %a)
+  ret { <2 x half>, <2 x half> } %result
+}
+
+define { float, float } @test_modf_f32(float %a) {
+; CHECK-LABEL: test_modf_f32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    .cfi_offset w30, -16
+; CHECK-NEXT:    add x0, sp, #12
+; CHECK-NEXT:    bl modff
+; CHECK-NEXT:    ldr s1, [sp, #12]
+; CHECK-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    ret
+  %result = call { float, float } @llvm.modf.f32(float %a)
+  ret { float, float } %result
+}
+
+define { <3 x float>, <3 x float> } @test_modf_v3f32(<3 x float> %a) {
+; CHECK-LABEL: test_modf_v3f32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    sub sp, sp, #80
+; CHECK-NEXT:    str x30, [sp, #48] // 8-byte Folded Spill
+; CHECK-NEXT:    stp x20, x19, [sp, #64] // 16-byte Folded Spill
+; CHECK-NEXT:    .cfi_def_cfa_offset 80
+; CHECK-NEXT:    .cfi_offset w19, -8
+; CHECK-NEXT:    .cfi_offset w20, -16
+; CHECK-NEXT:    .cfi_offset w30, -32
+; CHECK-NEXT:    str q0, [sp, #16] // 16-byte Folded Spill
+; CHECK-NEXT:    mov s0, v0.s[1]
+; CHECK-NEXT:    add x0, sp, #56
+; CHECK-NEXT:    add x19, sp, #56
+; CHECK-NEXT:    bl modff
+; CHECK-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; CHECK-NEXT:    add x0, sp, #44
+; CHECK-NEXT:    ldr q0, [sp, #16] // 16-byte Folded Reload
+; CHECK-NEXT:    // kill: def $s0 killed $s0 killed $q0
+; CHECK-NEXT:    bl modff
+; CHECK-NEXT:    ldr q1, [sp] // 16-byte Folded Reload
+; CHECK-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-NEXT:    add x0, sp, #60
+; CHECK-NEXT:    add x20, sp, #60
+; CHECK-NEXT:    mov v0.s[1], v1.s[0]
+; CHECK-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; CHECK-NEXT:    ldr q0, [sp, #16] // 16-byte Folded Reload
+; CHECK-NEXT:    mov s0, v0.s[2]
+; CHECK-NEXT:    bl modff
+; CHECK-NEXT:    ldr s1, [sp, #44]
+; CHECK-NEXT:    ldr q2, [sp] // 16-byte Folded Reload
+; CHECK-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-NEXT:    ldr x30, [sp, #48] // 8-byte Folded Reload
+; CHECK-NEXT:    ld1 { v1.s }[1], [x19]
+; CHECK-NEXT:    mov v2.s[2], v0.s[0]
+; CHECK-NEXT:    ld1 { v1.s }[2], [x20]
+; CHECK-NEXT:    ldp x20, x19, [sp, #64] // 16-byte Folded Reload
+; CHECK-NEXT:    mov v0.16b, v2.16b
+; CHECK-NEXT:    add sp, sp, #80
+; CHECK-NEXT:    ret
+  %result = call { <3 x float>, <3 x float> } @llvm.modf.v3f32(<3 x float> %a)
+  ret { <3 x float>, <3 x float> } %result
+}
+
+define { <2 x float>, <2 x float> } @test_modf_v2f32(<2 x float> %a) {
+; CHECK-LABEL: test_modf_v2f32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    sub sp, sp, #64
+; CHECK-NEXT:    stp x30, x19, [sp, #48] // 16-byte Folded Spill
+; CHECK-NEXT:    .cfi_def_cfa_offset 64
+; CHECK-NEXT:    .cfi_offset w19, -8
+; CHECK-NEXT:    .cfi_offset w30, -16
+; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; CHECK-NEXT:    add x0, sp, #40
+; CHECK-NEXT:    add x19, sp, #40
+; CHECK-NEXT:    mov s0, v0.s[1]
+; CHECK-NEXT:    bl modff
+; CHECK-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-NEXT:    str q0, [sp, #16] // 16-byte Folded Spill
+; CHECK-NEXT:    add x0, sp, #44
+; CHECK-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
+; CHECK-NEXT:    // kill: def $s0 killed $s0 killed $q0
+; CHECK-NEXT:    bl modff
+; CHECK-NEXT:    ldr s1, [sp, #44]
+; CHECK-NEXT:    ldr q2, [sp, #16] // 16-byte Folded Reload
+; CHECK-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-NEXT:    ld1 { v1.s }[1], [x19]
+; CHECK-NEXT:    ldp x30, x19, [sp, #48] // 16-byte Folded Reload
+; CHECK-NEXT:    mov v0.s[1], v2.s[0]
+; CHECK-NEXT:    // kill: def $d1 killed $d1 killed $q1
+; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-NEXT:    add sp, sp, #64
+; CHECK-NEXT:    ret
+  %result = call { <2 x float>, <2 x float> } @llvm.modf.v2f32(<2 x float> %a)
+  ret { <2 x float>, <2 x float> } %result
+}
+
+define { double, double } @test_modf_f64(double %a) {
+; CHECK-LABEL: test_modf_f64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    .cfi_offset w30, -16
+; CHECK-NEXT:    add x0, sp, #8
+; CHECK-NEXT:    bl modf
+; CHECK-NEXT:    ldr d1, [sp, #8]
+; CHECK-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    ret
+  %result = call { double, double } @llvm.modf.f64(double %a)
+  ret { double, double } %result
+}
+
+define { <2 x double>, <2 x double> } @test_modf_v2f64(<2 x double> %a) {
+; CHECK-LABEL: test_modf_v2f64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    sub sp, sp, #64
+; CHECK-NEXT:    stp x30, x19, [sp, #48] // 16-byte Folded Spill
+; CHECK-NEXT:    .cfi_def_cfa_offset 64
+; CHECK-NEXT:    .cfi_offset w19, -8
+; CHECK-NEXT:    .cfi_offset w30, -16
+; CHECK-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; CHECK-NEXT:    mov d0, v0.d[1]
+; CHECK-NEXT:    add x0, sp, #32
+; CHECK-NEXT:    add x19, sp, #32
+; CHECK-NEXT:    bl modf
+; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-NEXT:    str q0, [sp, #16] // 16-byte Folded Spill
+; CHECK-NEXT:    add x0, sp, #40
+; CHECK-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
+; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-NEXT:    bl modf
+; CHECK-NEXT:    ldr d1, [sp, #40]
+; CHECK-NEXT:    ldr q2, [sp, #16] // 16-byte Folded Reload
+; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-NEXT:    ld1 { v1.d }[1], [x19]
+; CHECK-NEXT:    ldp x30, x19, [sp, #48] // 16-byte Folded Reload
+; CHECK-NEXT:    mov v0.d[1], v2.d[0]
+; CHECK-NEXT:    add sp, sp, #64
+; CHECK-NEXT:    ret
+  %result = call { <2 x double>, <2 x double> } @llvm.modf.v2f64(<2 x double> %a)
+  ret { <2 x double>, <2 x double> } %result
+}
diff --git a/llvm/test/CodeGen/AArch64/llvm.sincospi.ll b/llvm/test/CodeGen/AArch64/llvm.sincospi.ll
new file mode 100644
index 0000000000000..d1d7d92adc05a
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/llvm.sincospi.ll
@@ -0,0 +1,268 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 2
+; RUN: llc -mtriple=aarch64-gnu-linux < %s | FileCheck -check-prefixes=CHECK %s
+
+define { half, half } @test_sincospi_f16(half %a) {
+; CHECK-LABEL: test_sincospi_f16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    .cfi_offset w30, -16
+; CHECK-NEXT:    fcvt s0, h0
+; CHECK-NEXT:    add x0, sp, #12
+; CHECK-NEXT:    add x1, sp, #8
+; CHECK-NEXT:    bl sincospif
+; CHECK-NEXT:    ldp s1, s0, [sp, #8]
+; CHECK-NEXT:    fcvt h0, s0
+; CHECK-NEXT:    fcvt h1, s1
+; CHECK-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    ret
+  %result = call { half, half } @llvm.sincospi.f16(half %a)
+  ret { half, half } %result
+}
+
+define half @test_sincospi_f16_only_use_sin(half %a) {
+; CHECK-LABEL: test_sincospi_f16_only_use_sin:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    .cfi_offset w30, -16
+; CHECK-NEXT:    fcvt s0, h0
+; CHECK-NEXT:    add x0, sp, #12
+; CHECK-NEXT:    add x1, sp, #8
+; CHECK-NEXT:    bl sincospif
+; CHECK-NEXT:    ldr s0, [sp, #12]
+; CHECK-NEXT:    fcvt h0, s0
+; CHECK-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    ret
+  %result = call { half, half } @llvm.sincospi.f16(half %a)
+  %result.0 = extractvalue { half, half } %result, 0
+  ret half %result.0
+}
+
+define half @test_sincospi_f16_only_use_cos(half %a) {
+; CHECK-LABEL: test_sincospi_f16_only_use_cos:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    .cfi_offset w30, -16
+; CHECK-NEXT:    fcvt s0, h0
+; CHECK-NEXT:    add x0, sp, #12
+; CHECK-NEXT:    add x1, sp, #8
+; CHECK-NEXT:    bl sincospif
+; CHECK-NEXT:    ldr s0, [sp, #8]
+; CHECK-NEXT:    fcvt h0, s0
+; CHECK-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    ret
+  %result = call { half, half } @llvm.sincospi.f16(half %a)
+  %result.1 = extractvalue { half, half } %result, 1
+  ret half %result.1
+}
+
+define { <2 x half>, <2 x half> } @test_sincospi_v2f16(<2 x half> %a) {
+; CHECK-LABEL: test_sincospi_v2f16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    sub sp, sp, #64
+; CHECK-NEXT:    str x30, [sp, #48] // 8-byte Folded Spill
+; CHECK-NEXT:    .cfi_def_cfa_offset 64
+; CHECK-NEXT:    .cfi_offset w30, -16
+; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-NEXT:    mov h1, v0.h[1]
+; CHECK-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; CHECK-NEXT:    add x0, sp, #36
+; CHECK-NEXT:    add x1, sp, #32
+; CHECK-NEXT:    fcvt s0, h1
+; CHECK-NEXT:    bl sincospif
+; CHECK-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
+; CHECK-NEXT:    add x0, sp, #28
+; CHECK-NEXT:    add x1, sp, #24
+; CHECK-NEXT:    fcvt s0, h0
+; CHECK-NEXT:    bl sincospif
+; CHECK-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
+; CHECK-NEXT:    add x0, sp, #44
+; CHECK-NEXT:    add x1, sp, #40
+; CHECK-NEXT:    mov h0, v0.h[2]
+; CHECK-NEXT:    fcvt s0, h0
+; CHECK-NEXT:    bl sincospif
+; CHECK-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
+; CHECK-NEXT:    add x0, sp, #60
+; CHECK-NEXT:    add x1, sp, #56
+; CHECK-NEXT:    mov h0, v0.h[3]
+; CHECK-NEXT:    fcvt s0, h0
+; CHECK-NEXT:    bl sincospif
+; CHECK-NEXT:    ldp s2, s0, [sp, #32]
+; CHECK-NEXT:    ldr x30, [sp, #48] // 8-byte Folded Reload
+; CHECK-NEXT:    ldp s3, s1, [sp, #24]
+; CHECK-NEXT:    fcvt h4, s0
+; CHECK-NEXT:    fcvt h2, s2
+; CHECK-NEXT:    fcvt h0, s1
+; CHECK-NEXT:    fcvt h1, s3
+; CHECK-NEXT:    ldp s5, s3, [sp, #40]
+; CHECK-NEXT:    fcvt h3, s3
+; CHECK-NEXT:    mov v0.h[1], v4.h[0]
+; CHECK-NEXT:    fcvt h4, s5
+; CHECK-NEXT:    mov v1.h[1], v2.h[0]
+; CHECK-NEXT:    ldp s5, s2, [sp, #56]
+; CHECK-NEXT:    mov v0.h[2], v3.h[0]
+; CHECK-NEXT:    fcvt h2, s2
+; CHECK-NEXT:    fcvt h3, s5
+; CHECK-NEXT:    mov v1.h[2], v4.h[0]
+; CHECK-NEXT:    mov v0.h[3], v2.h[0]
+; CHECK-NEXT:    mov v1.h[3], v3.h[0]
+; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-NEXT:    // kill: def $d1 killed $d1 killed $q1
+; CHECK-NEXT:    add sp, sp, #64
+; CHECK-NEXT:    ret
+  %result = call { <2 x half>, <2 x half> } @llvm.sincospi.v2f16(<2 x half> %a)
+  ret { <2 x half>, <2 x half> } %result
+}
+
+define { float, float } @test_sincospi_f32(float %a) {
+; CHECK-LABEL: test_sincospi_f32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    .cfi_offset w30, -16
+; CHECK-NEXT:    add x0, sp, #12
+; CHECK-NEXT:    add x1, sp, #8
+; CHECK-NEXT:    bl sincospif
+; CHECK-NEXT:    ldp s1, s0, [sp, #8]
+; CHECK-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    ret
+  %result = call { float, float } @llvm.sincospi.f32(float %a)
+  ret { float, float } %result
+}
+
+define { <3 x float>, <3 x float> } @test_sincospi_v3f32(<3 x float> %a) {
+; CHECK-LABEL: test_sincospi_v3f32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    sub sp, sp, #80
+; CHECK-NEXT:    str x30, [sp, #32] // 8-byte Folded Spill
+; CHECK-NEXT:    stp x22, x21, [sp, #48] // 16-byte Folded Spill
+; CHECK-NEXT:    stp x20, x19, [sp, #64] // 16-byte Folded Spill
+; CHECK-NEXT:    .cfi_def_cfa_offset 80
+; CHECK-NEXT:    .cfi_offset w19, -8
+; CHECK-NEXT:    .cfi_offset w20, -16
+; CHECK-NEXT:    .cfi_offset w21, -24
+; CHECK-NEXT:    .cfi_offset w22, -32
+; CHECK-NEXT:    .cfi_offset w30, -48
+; CHECK-NEXT:    add x0, sp, #20
+; CHECK-NEXT:    add x1, sp, #16
+; CHECK-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; CHECK-NEXT:    // kill: def $s0 killed $s0 killed $q0
+; CHECK-NEXT:    bl sincospif
+; CHECK-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
+; CHECK-NEXT:    add x0, sp, #28
+; CHECK-NEXT:    add x1, sp, #24
+; CHECK-NEXT:    add x19, sp, #28
+; CHECK-NEXT:    add x20, sp, #24
+; CHECK-NEXT:    mov s0, v0.s[1]
+; CHECK-NEXT:    bl sincospif
+; CHECK-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
+; CHECK-NEXT:    add x0, sp, #44
+; CHECK-NEXT:    add x1, sp, #40
+; CHECK-NEXT:    add x21, sp, #44
+; CHECK-NEXT:    add x22, sp, #40
+; CHECK-NEXT:    mov s0, v0.s[2]
+; CHECK-NEXT:    bl sincospif
+; CHECK-NEXT:    ldp s1, s0, [sp, #16]
+; CHECK-NEXT:    ldr x30, [sp, #32] // 8-byte Folded Reload
+; CHECK-NEXT:    ld1 { v0.s }[1], [x19]
+; CHECK-NEXT:    ld1 { v1.s }[1], [x20]
+; CHECK-NEXT:    ldp x20, x19, [sp, #64] // 16-byte Folded Reload
+; CHECK-NEXT:    ld1 { v0.s }[2], [x21]
+; CHECK-NEXT:    ld1 { v1.s }[2], [x22]
+; CHECK-NEXT:    ldp x22, x21, [sp, #48] // 16-byte Folded Reload
+; CHECK-NEXT:    add sp, sp, #80
+; CHECK-NEXT:    ret
+  %result = call { <3 x float>, <3 x float> } @llvm.sincospi.v3f32(<3 x float> %a)
+  ret { <3 x float>, <3 x float> } %result
+}
+
+define { <2 x float>, <2 x float> } @test_sincospi_v2f32(<2 x float> %a) {
+; CHECK-LABEL: test_sincospi_v2f32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    sub sp, sp, #64
+; CHECK-NEXT:    str x30, [sp, #32] // 8-byte Folded Spill
+; CHECK-NEXT:    stp x20, x19, [sp, #48] // 16-byte Folded Spill
+; CHECK-NEXT:    .cfi_def_cfa_offset 64
+; CHECK-NEXT:    .cfi_offset w19, -8
+; CHECK-NEXT:    .cfi_offset w20, -16
+; CHECK-NEXT:    .cfi_offset w30, -32
+; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-NEXT:    add x0, sp, #44
+; CHECK-NEXT:    add x1, sp, #40
+; CHECK-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; CHECK-NEXT:    // kill: def $s0 killed $s0 killed $q0
+; CHECK-NEXT:    bl sincospif
+; CHECK-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
+; CHECK-NEXT:    add x0, sp, #28
+; CHECK-NEXT:    add x1, sp, #24
+; CHECK-NEXT:    add x19, sp, #28
+; CHECK-NEXT:    add x20, sp, #24
+; CHECK-NEXT:    mov s0, v0.s[1]
+; CHECK-NEXT:    bl sincospif
+; CHECK-NEXT:    ldp s1, s0, [sp, #40]
+; CHECK-NEXT:    ldr x30, [sp, #32] // 8-byte Folded Reload
+; CHECK-NEXT:    ld1 { v0.s }[1], [x19]
+; CHECK-NEXT:    ld1 { v1.s }[1], [x20]
+; CHECK-NEXT:    ldp x20, x19, [sp, #48] // 16-byte Folded Reload
+; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-NEXT:    // kill: def $d1 killed $d1 killed $q1
+; CHECK-NEXT:    add sp, sp, #64
+; CHECK-NEXT:    ret
+  %result = call { <2 x float>, <2 x float> } @llvm.sincospi.v2f32(<2 x float> %a)
+  ret { <2 x float>, <2 x float> } %result
+}
+
+define { double, double } @test_sincospi_f64(double %a) {
+; CHECK-LABEL: test_sincospi_f64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    sub sp, sp, #32
+; CHECK-NEXT:    str x30, [sp, #16] // 8-byte Folded Spill
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    .cfi_offset w30, -16
+; CHECK-NEXT:    add x0, sp, #24
+; CHECK-NEXT:    add x1, sp, #8
+; CHECK-NEXT:    bl sincospi
+; CHECK-NEXT:    ldr d0, [sp, #24]
+; CHECK-NEXT:    ldr d1, [sp, #8]
+; CHECK-NEXT:    ldr x30, [sp, #16] // 8-byte Folded Reload
+; CHECK-NEXT:    add sp, sp, #32
+; CHECK-NEXT:    ret
+  %result = call { double, double } @llvm.sincospi.f64(double %a)
+  ret { double, double } %result
+}
+
+define { <2 x double>, <2 x double> } @test_sincospi_v2f64(<2 x double> %a) {
+; CHECK-LABEL: test_sincospi_v2f64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    sub sp, sp, #80
+; CHECK-NEXT:    str x30, [sp, #48] // 8-byte Folded Spill
+; CHECK-NEXT:    stp x20, x19, [sp, #64] // 16-byte Folded Spill
+; CHECK-NEXT:    .cfi_def_cfa_offset 80
+; CHECK-NEXT:    .cfi_offset w19, -8
+; CHECK-NEXT:    .cfi_offset w20, -16
+; CHECK-NEXT:    .cfi_offset w30, -32
+; CHECK-NEXT:    add x0, sp, #56
+; CHECK-NEXT:    add x1, sp, #40
+; CHECK-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-NEXT:    bl sincospi
+; CHECK-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
+; CHECK-NEXT:    add x0, sp, #32
+; CHECK-NEXT:    add x1, sp, #24
+; CHECK-NEXT:    add x19, sp, #32
+; CHECK-NEXT:    add x20, sp, #24
+; CHECK-NEXT:    mov d0, v0.d[1]
+; CHECK-NEXT:    bl sincospi
+; CHECK-NEXT:    ldr d0, [sp, #56]
+; CHECK-NEXT:    ldr d1, [sp, #40]
+; CHECK-NEXT:    ldr x30, [sp, #48] // 8-byte Folded Reload
+; CHECK-NEXT:    ld1 { v0.d }[1], [x19]
+; CHECK-NEXT:    ld1 { v1.d }[1], [x20]
+; CHECK-NEXT:    ldp x20, x19, [sp, #64] // 16-byte Folded Reload
+; CHECK-NEXT:    add sp, sp, #80
+; CHECK-NEXT:    ret
+  %result = call { <2 x double>, <2 x double> } @llvm.sincospi.v2f64(<2 x double> %a)
+  ret { <2 x double>, <2 x double> } %result
+}
diff --git a/llvm/test/CodeGen/AArch64/logic-shift.ll b/llvm/test/CodeGen/AArch64/logic-shift.ll
index 31047954401cf..0a62f8076a1be 100644
--- a/llvm/test/CodeGen/AArch64/logic-shift.ll
+++ b/llvm/test/CodeGen/AArch64/logic-shift.ll
@@ -5,7 +5,6 @@ define i8 @or_lshr_commute0(i8 %x0, i8 %x1, i8 %y, i8 %z) {
 ; CHECK-LABEL: or_lshr_commute0:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    orr w8, w0, w1
-; CHECK-NEXT:    // kill: def $w2 killed $w2 def $x2
 ; CHECK-NEXT:    and w8, w8, #0xff
 ; CHECK-NEXT:    lsr w8, w8, w2
 ; CHECK-NEXT:    orr w0, w8, w3
@@ -65,7 +64,6 @@ define i16 @or_ashr_commute0(i16 %x0, i16 %x1, i16 %y, i16 %z) {
 ; CHECK-LABEL: or_ashr_commute0:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    orr w8, w0, w1
-; CHECK-NEXT:    // kill: def $w2 killed $w2 def $x2
 ; CHECK-NEXT:    sxth w8, w8
 ; CHECK-NEXT:    asr w8, w8, w2
 ; CHECK-NEXT:    orr w0, w8, w3
@@ -139,7 +137,6 @@ define i8 @or_shl_commute1(i8 %x0, i8 %x1, i8 %y, i8 %z) {
 ; CHECK-LABEL: or_shl_commute1:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    orr w8, w0, w1
-; CHECK-NEXT:    // kill: def $w2 killed $w2 def $x2
 ; CHECK-NEXT:    lsl w8, w8, w2
 ; CHECK-NEXT:    orr w0, w8, w3
 ; CHECK-NEXT:    ret
@@ -233,7 +230,6 @@ define i8 @xor_lshr_commute0(i8 %x0, i8 %x1, i8 %y, i8 %z) {
 ; CHECK-LABEL: xor_lshr_commute0:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    eor w8, w0, w1
-; CHECK-NEXT:    // kill: def $w2 killed $w2 def $x2
 ; CHECK-NEXT:    and w8, w8, #0xff
 ; CHECK-NEXT:    lsr w8, w8, w2
 ; CHECK-NEXT:    eor w0, w8, w3
@@ -293,7 +289,6 @@ define i16 @xor_ashr_commute0(i16 %x0, i16 %x1, i16 %y, i16 %z) {
 ; CHECK-LABEL: xor_ashr_commute0:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    eor w8, w0, w1
-; CHECK-NEXT:    // kill: def $w2 killed $w2 def $x2
 ; CHECK-NEXT:    sxth w8, w8
 ; CHECK-NEXT:    asr w8, w8, w2
 ; CHECK-NEXT:    eor w0, w8, w3
@@ -367,7 +362,6 @@ define i8 @xor_shl_commute1(i8 %x0, i8 %x1, i8 %y, i8 %z) {
 ; CHECK-LABEL: xor_shl_commute1:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    eor w8, w0, w1
-; CHECK-NEXT:    // kill: def $w2 killed $w2 def $x2
 ; CHECK-NEXT:    lsl w8, w8, w2
 ; CHECK-NEXT:    eor w0, w8, w3
 ; CHECK-NEXT:    ret
@@ -461,7 +455,6 @@ define i8 @and_lshr_commute0(i8 %x0, i8 %x1, i8 %y, i8 %z) {
 ; CHECK-LABEL: and_lshr_commute0:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    and w8, w0, w1
-; CHECK-NEXT:    // kill: def $w2 killed $w2 def $x2
 ; CHECK-NEXT:    and w8, w8, #0xff
 ; CHECK-NEXT:    lsr w8, w8, w2
 ; CHECK-NEXT:    and w0, w8, w3
@@ -521,7 +514,6 @@ define i16 @and_ashr_commute0(i16 %x0, i16 %x1, i16 %y, i16 %z) {
 ; CHECK-LABEL: and_ashr_commute0:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    and w8, w0, w1
-; CHECK-NEXT:    // kill: def $w2 killed $w2 def $x2
 ; CHECK-NEXT:    sxth w8, w8
 ; CHECK-NEXT:    asr w8, w8, w2
 ; CHECK-NEXT:    and w0, w8, w3
@@ -595,7 +587,6 @@ define i8 @and_shl_commute1(i8 %x0, i8 %x1, i8 %y, i8 %z) {
 ; CHECK-LABEL: and_shl_commute1:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    and w8, w0, w1
-; CHECK-NEXT:    // kill: def $w2 killed $w2 def $x2
 ; CHECK-NEXT:    lsl w8, w8, w2
 ; CHECK-NEXT:    and w0, w8, w3
 ; CHECK-NEXT:    ret
diff --git a/llvm/test/CodeGen/AArch64/luti-with-sme2.ll b/llvm/test/CodeGen/AArch64/luti-with-sme2.ll
new file mode 100644
index 0000000000000..2d30167e2b124
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/luti-with-sme2.ll
@@ -0,0 +1,125 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc < %s -verify-machineinstrs -mtriple=aarch64-none-linux-gnu -enable-subreg-liveness -force-streaming -mattr=+sve2,+sme2,+lut,+bf16 | FileCheck %s
+
+define { <vscale x 8 x i16>, <vscale x 8 x i16> } @test_luti4_lane_i16_x2_tuple(i64 %stride, ptr %ptr, <vscale x 16 x i8> %indices) {
+; CHECK-LABEL: test_luti4_lane_i16_x2_tuple:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NEXT:    addvl sp, sp, #-3
+; CHECK-NEXT:    str p8, [sp, #7, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    str z12, [sp, #1, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z11, [sp, #2, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    .cfi_escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x18, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 24 * VG
+; CHECK-NEXT:    .cfi_offset w29, -16
+; CHECK-NEXT:    .cfi_escape 0x10, 0x4b, 0x0a, 0x11, 0x70, 0x22, 0x11, 0x78, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d11 @ cfa - 16 - 8 * VG
+; CHECK-NEXT:    .cfi_escape 0x10, 0x4c, 0x0a, 0x11, 0x70, 0x22, 0x11, 0x70, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d12 @ cfa - 16 - 16 * VG
+; CHECK-NEXT:    ptrue pn8.b
+; CHECK-NEXT:    add x8, x1, x0
+; CHECK-NEXT:    ld1h { z3.h, z11.h }, pn8/z, [x1]
+; CHECK-NEXT:    ld1h { z4.h, z12.h }, pn8/z, [x8]
+; CHECK-NEXT:    luti4 z2.h, { z3.h, z4.h }, z0[0]
+; CHECK-NEXT:    luti4 z1.h, { z11.h, z12.h }, z0[0]
+; CHECK-NEXT:    ldr z12, [sp, #1, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z11, [sp, #2, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr p8, [sp, #7, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    mov z0.d, z2.d
+; CHECK-NEXT:    addvl sp, sp, #3
+; CHECK-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    ret
+entry:
+   %0 = tail call target("aarch64.svcount") @llvm.aarch64.sve.ptrue.c8()
+   %1 = tail call { <vscale x 8 x i16>, <vscale x 8 x i16> } @llvm.aarch64.sve.ld1.pn.x2.nxv8i16(target("aarch64.svcount") %0, ptr %ptr)
+   %2 = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16> } %1, 0
+   %3 = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16> } %1, 1
+   %arrayidx2 = getelementptr inbounds i8, ptr %ptr, i64 %stride
+   %4 = tail call { <vscale x 8 x i16>, <vscale x 8 x i16> } @llvm.aarch64.sve.ld1.pn.x2.nxv8i16(target("aarch64.svcount") %0, ptr %arrayidx2)
+   %5 = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16> } %4, 0
+   %6 = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16> } %4, 1
+   %res1 = tail call <vscale x 8 x i16> @llvm.aarch64.sve.luti4.lane.x2.nxv8i16(<vscale x 8 x i16> %2, <vscale x 8 x i16> %5, <vscale x 16 x i8> %indices, i32 0)
+   %res2 = tail call <vscale x 8 x i16> @llvm.aarch64.sve.luti4.lane.x2.nxv8i16(<vscale x 8 x i16> %3, <vscale x 8 x i16> %6, <vscale x 16 x i8> %indices, i32 0)
+   %ins1 = insertvalue { <vscale x 8 x i16>, <vscale x 8 x i16> } poison, <vscale x 8 x i16> %res1, 0
+   %ins2 = insertvalue { <vscale x 8 x i16>, <vscale x 8 x i16> } %ins1, <vscale x 8 x i16> %res2, 1
+   ret { <vscale x 8 x i16>, <vscale x 8 x i16> } %ins2
+}
+
+define { <vscale x 8 x half>, <vscale x 8 x half> } @test_luti4_lane_f16_x2_tuple(i64 %stride, ptr %ptr, <vscale x 16 x i8> %indices) {
+; CHECK-LABEL: test_luti4_lane_f16_x2_tuple:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NEXT:    addvl sp, sp, #-3
+; CHECK-NEXT:    str p8, [sp, #7, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    str z12, [sp, #1, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z11, [sp, #2, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    .cfi_escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x18, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 24 * VG
+; CHECK-NEXT:    .cfi_offset w29, -16
+; CHECK-NEXT:    .cfi_escape 0x10, 0x4b, 0x0a, 0x11, 0x70, 0x22, 0x11, 0x78, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d11 @ cfa - 16 - 8 * VG
+; CHECK-NEXT:    .cfi_escape 0x10, 0x4c, 0x0a, 0x11, 0x70, 0x22, 0x11, 0x70, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d12 @ cfa - 16 - 16 * VG
+; CHECK-NEXT:    ptrue pn8.b
+; CHECK-NEXT:    add x8, x1, x0
+; CHECK-NEXT:    ld1h { z3.h, z11.h }, pn8/z, [x1]
+; CHECK-NEXT:    ld1h { z4.h, z12.h }, pn8/z, [x8]
+; CHECK-NEXT:    luti4 z2.h, { z3.h, z4.h }, z0[0]
+; CHECK-NEXT:    luti4 z1.h, { z11.h, z12.h }, z0[0]
+; CHECK-NEXT:    ldr z12, [sp, #1, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z11, [sp, #2, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr p8, [sp, #7, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    mov z0.d, z2.d
+; CHECK-NEXT:    addvl sp, sp, #3
+; CHECK-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    ret
+entry:
+   %0 = tail call target("aarch64.svcount") @llvm.aarch64.sve.ptrue.c8()
+   %1 = tail call { <vscale x 8 x half>, <vscale x 8 x half> } @llvm.aarch64.sve.ld1.pn.x2.nxv8f16(target("aarch64.svcount") %0, ptr %ptr)
+   %2 = extractvalue { <vscale x 8 x half>, <vscale x 8 x half> } %1, 0
+   %3 = extractvalue { <vscale x 8 x half>, <vscale x 8 x half> } %1, 1
+   %arrayidx2 = getelementptr inbounds i8, ptr %ptr, i64 %stride
+   %4 = tail call { <vscale x 8 x half>, <vscale x 8 x half> } @llvm.aarch64.sve.ld1.pn.x2.nxv8f16(target("aarch64.svcount") %0, ptr %arrayidx2)
+   %5 = extractvalue { <vscale x 8 x half>, <vscale x 8 x half> } %4, 0
+   %6 = extractvalue { <vscale x 8 x half>, <vscale x 8 x half> } %4, 1
+   %res1 = tail call <vscale x 8 x half> @llvm.aarch64.sve.luti4.lane.x2.nxv8f16(<vscale x 8 x half> %2, <vscale x 8 x half> %5, <vscale x 16 x i8> %indices, i32 0)
+   %res2 = tail call <vscale x 8 x half> @llvm.aarch64.sve.luti4.lane.x2.nxv8f16(<vscale x 8 x half> %3, <vscale x 8 x half> %6, <vscale x 16 x i8> %indices, i32 0)
+   %ins1 = insertvalue { <vscale x 8 x half>, <vscale x 8 x half> } poison, <vscale x 8 x half> %res1, 0
+   %ins2 = insertvalue { <vscale x 8 x half>, <vscale x 8 x half> } %ins1, <vscale x 8 x half> %res2, 1
+   ret { <vscale x 8 x half>, <vscale x 8 x half> } %ins2
+}
+
+define { <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } @test_luti4_lane_bf16_x2_tuple(i64 %stride, ptr %ptr, <vscale x 16 x i8> %indices) {
+; CHECK-LABEL: test_luti4_lane_bf16_x2_tuple:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NEXT:    addvl sp, sp, #-3
+; CHECK-NEXT:    str p8, [sp, #7, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    str z12, [sp, #1, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z11, [sp, #2, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    .cfi_escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x18, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 24 * VG
+; CHECK-NEXT:    .cfi_offset w29, -16
+; CHECK-NEXT:    .cfi_escape 0x10, 0x4b, 0x0a, 0x11, 0x70, 0x22, 0x11, 0x78, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d11 @ cfa - 16 - 8 * VG
+; CHECK-NEXT:    .cfi_escape 0x10, 0x4c, 0x0a, 0x11, 0x70, 0x22, 0x11, 0x70, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d12 @ cfa - 16 - 16 * VG
+; CHECK-NEXT:    ptrue pn8.b
+; CHECK-NEXT:    add x8, x1, x0
+; CHECK-NEXT:    ld1h { z3.h, z11.h }, pn8/z, [x1]
+; CHECK-NEXT:    ld1h { z4.h, z12.h }, pn8/z, [x8]
+; CHECK-NEXT:    luti4 z2.h, { z3.h, z4.h }, z0[0]
+; CHECK-NEXT:    luti4 z1.h, { z11.h, z12.h }, z0[0]
+; CHECK-NEXT:    ldr z12, [sp, #1, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z11, [sp, #2, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr p8, [sp, #7, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    mov z0.d, z2.d
+; CHECK-NEXT:    addvl sp, sp, #3
+; CHECK-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    ret
+entry:
+   %0 = tail call target("aarch64.svcount") @llvm.aarch64.sve.ptrue.c8()
+   %1 = tail call { <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } @llvm.aarch64.sve.ld1.pn.x2.nxv8bf16(target("aarch64.svcount") %0, ptr %ptr)
+   %2 = extractvalue { <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } %1, 0
+   %3 = extractvalue { <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } %1, 1
+   %arrayidx2 = getelementptr inbounds i8, ptr %ptr, i64 %stride
+   %4 = tail call { <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } @llvm.aarch64.sve.ld1.pn.x2.nxv8bf16(target("aarch64.svcount") %0, ptr %arrayidx2)
+   %5 = extractvalue { <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } %4, 0
+   %6 = extractvalue { <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } %4, 1
+   %res1 = tail call <vscale x 8 x bfloat> @llvm.aarch64.sve.luti4.lane.x2.nxv8bf16(<vscale x 8 x bfloat> %2, <vscale x 8 x bfloat> %5, <vscale x 16 x i8> %indices, i32 0)
+   %res2 = tail call <vscale x 8 x bfloat> @llvm.aarch64.sve.luti4.lane.x2.nxv8bf16(<vscale x 8 x bfloat> %3, <vscale x 8 x bfloat> %6, <vscale x 16 x i8> %indices, i32 0)
+   %ins1 = insertvalue { <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } poison, <vscale x 8 x bfloat> %res1, 0
+   %ins2 = insertvalue { <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } %ins1, <vscale x 8 x bfloat> %res2, 1
+   ret { <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } %ins2
+}
diff --git a/llvm/test/CodeGen/AArch64/memcpy-f128.ll b/llvm/test/CodeGen/AArch64/memcpy-f128.ll
index 5b354dd23e01d..bb411a749bceb 100644
--- a/llvm/test/CodeGen/AArch64/memcpy-f128.ll
+++ b/llvm/test/CodeGen/AArch64/memcpy-f128.ll
@@ -1,4 +1,5 @@
 ; RUN: llc < %s -mtriple=aarch64-linux-gnu | FileCheck %s
+; RUN: llc < %s -mtriple=aarch64-linux-gnu -mattr=-neon | FileCheck %s
 
 %structA = type { i128 }
 @stubA = internal unnamed_addr constant %structA zeroinitializer, align 8
diff --git a/llvm/test/CodeGen/AArch64/misched-fusion-cmp-bcc.ll b/llvm/test/CodeGen/AArch64/misched-fusion-cmp-bcc.ll
index 1034e072922a0..700a060ef968f 100644
--- a/llvm/test/CodeGen/AArch64/misched-fusion-cmp-bcc.ll
+++ b/llvm/test/CodeGen/AArch64/misched-fusion-cmp-bcc.ll
@@ -4,9 +4,9 @@
 ; RUN: llc %s -o - -O0 -mtriple=aarch64-unknown -mcpu=cortex-a78ae  | FileCheck %s
 ; RUN: llc %s -o - -O0 -mtriple=aarch64-unknown -mcpu=cortex-a78c   | FileCheck %s
 ; RUN: llc %s -o - -O0 -mtriple=aarch64-unknown -mcpu=cortex-a710   | FileCheck %s
-; RUN: llc %s -o - -O0 -mtriple=aarch64-unknown -mcpu=cortex-x715   | FileCheck %s
-; RUN: llc %s -o - -O0 -mtriple=aarch64-unknown -mcpu=cortex-x720   | FileCheck %s
-; RUN: llc %s -o - -O0 -mtriple=aarch64-unknown -mcpu=cortex-x720ae | FileCheck %s
+; RUN: llc %s -o - -O0 -mtriple=aarch64-unknown -mcpu=cortex-a715   | FileCheck %s
+; RUN: llc %s -o - -O0 -mtriple=aarch64-unknown -mcpu=cortex-a720   | FileCheck %s
+; RUN: llc %s -o - -O0 -mtriple=aarch64-unknown -mcpu=cortex-a720ae | FileCheck %s
 ; RUN: llc %s -o - -O0 -mtriple=aarch64-unknown -mcpu=cortex-x1     | FileCheck %s
 ; RUN: llc %s -o - -O0 -mtriple=aarch64-unknown -mcpu=cortex-x2     | FileCheck %s
 ; RUN: llc %s -o - -O0 -mtriple=aarch64-unknown -mcpu=neoverse-v2   | FileCheck %s
diff --git a/llvm/test/CodeGen/AArch64/neon-dotreduce.ll b/llvm/test/CodeGen/AArch64/neon-dotreduce.ll
index 748555d7bdfa1..8e12446164e89 100644
--- a/llvm/test/CodeGen/AArch64/neon-dotreduce.ll
+++ b/llvm/test/CodeGen/AArch64/neon-dotreduce.ll
@@ -1,22 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mtriple aarch64-linux-gnu -mattr=+dotprod,+i8mm    < %s | FileCheck %s --check-prefixes=CHECK,CHECK-SD
-; RUN: llc -mtriple aarch64-linux-gnu -mattr=+dotprod,+i8mm -global-isel -global-isel-abort=2 < %s 2>&1 | FileCheck %s --check-prefixes=CHECK,CHECK-GI
-
-; CHECK-GI:       warning: Instruction selection used fallback path for test_udot_v5i8
-; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for test_udot_v5i8_nomla
-; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for test_sdot_v5i8
-; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for test_sdot_v5i8_double
-; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for test_sdot_v5i8_double_nomla
-; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for test_udot_v25i8
-; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for test_udot_v25i8_nomla
-; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for test_sdot_v25i8
-; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for test_sdot_v25i8_double
-; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for test_sdot_v25i8_double_nomla
-; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for test_udot_v33i8
-; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for test_udot_v33i8_nomla
-; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for test_sdot_v33i8
-; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for test_sdot_v33i8_double
-; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for test_sdot_v33i8_double_nomla
+; RUN: llc -mtriple aarch64-linux-gnu -mattr=+dotprod,+i8mm -global-isel < %s | FileCheck %s --check-prefixes=CHECK,CHECK-GI
 
 declare i32 @llvm.vector.reduce.add.v4i32(<4 x i32>)
 declare i32 @llvm.vector.reduce.add.v5i32(<5 x i32>)
@@ -413,19 +397,50 @@ entry:
 }
 
 define i32 @test_udot_v5i8(ptr nocapture readonly %a, ptr nocapture readonly %b, i32 %sum) {
-; CHECK-LABEL: test_udot_v5i8:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    ldr d0, [x0]
-; CHECK-NEXT:    ldr d1, [x1]
-; CHECK-NEXT:    umull v0.8h, v1.8b, v0.8b
-; CHECK-NEXT:    movi v1.2d, #0000000000000000
-; CHECK-NEXT:    ushll2 v2.4s, v0.8h, #0
-; CHECK-NEXT:    mov v1.s[0], v2.s[0]
-; CHECK-NEXT:    uaddw v0.4s, v1.4s, v0.4h
-; CHECK-NEXT:    addv s0, v0.4s
-; CHECK-NEXT:    fmov w8, s0
-; CHECK-NEXT:    add w0, w8, w2
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: test_udot_v5i8:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    ldr d0, [x0]
+; CHECK-SD-NEXT:    ldr d1, [x1]
+; CHECK-SD-NEXT:    umull v0.8h, v1.8b, v0.8b
+; CHECK-SD-NEXT:    movi v1.2d, #0000000000000000
+; CHECK-SD-NEXT:    ushll2 v2.4s, v0.8h, #0
+; CHECK-SD-NEXT:    mov v1.s[0], v2.s[0]
+; CHECK-SD-NEXT:    uaddw v0.4s, v1.4s, v0.4h
+; CHECK-SD-NEXT:    addv s0, v0.4s
+; CHECK-SD-NEXT:    fmov w8, s0
+; CHECK-SD-NEXT:    add w0, w8, w2
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: test_udot_v5i8:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    ldrb w8, [x0, #4]
+; CHECK-GI-NEXT:    ldrb w9, [x1, #4]
+; CHECK-GI-NEXT:    ldrb w10, [x1]
+; CHECK-GI-NEXT:    mul w8, w9, w8
+; CHECK-GI-NEXT:    ldrb w9, [x0]
+; CHECK-GI-NEXT:    mov v0.s[0], w10
+; CHECK-GI-NEXT:    mov v1.s[0], w9
+; CHECK-GI-NEXT:    ldrb w9, [x1, #1]
+; CHECK-GI-NEXT:    mov v2.s[0], w8
+; CHECK-GI-NEXT:    ldrb w8, [x0, #1]
+; CHECK-GI-NEXT:    mov v0.s[1], w9
+; CHECK-GI-NEXT:    ldrb w9, [x1, #2]
+; CHECK-GI-NEXT:    mov v1.s[1], w8
+; CHECK-GI-NEXT:    ldrb w8, [x0, #2]
+; CHECK-GI-NEXT:    mov v2.s[1], wzr
+; CHECK-GI-NEXT:    mov v0.s[2], w9
+; CHECK-GI-NEXT:    ldrb w9, [x1, #3]
+; CHECK-GI-NEXT:    mov v1.s[2], w8
+; CHECK-GI-NEXT:    ldrb w8, [x0, #3]
+; CHECK-GI-NEXT:    mov v2.s[2], wzr
+; CHECK-GI-NEXT:    mov v0.s[3], w9
+; CHECK-GI-NEXT:    mov v1.s[3], w8
+; CHECK-GI-NEXT:    mov v2.s[3], wzr
+; CHECK-GI-NEXT:    mla v2.4s, v0.4s, v1.4s
+; CHECK-GI-NEXT:    addv s0, v2.4s
+; CHECK-GI-NEXT:    fmov w8, s0
+; CHECK-GI-NEXT:    add w0, w8, w2
+; CHECK-GI-NEXT:    ret
 entry:
   %0 = load <5 x i8>, ptr %a
   %1 = zext <5 x i8> %0 to <5 x i32>
@@ -438,17 +453,37 @@ entry:
 }
 
 define i32 @test_udot_v5i8_nomla(ptr nocapture readonly %a1) {
-; CHECK-LABEL: test_udot_v5i8_nomla:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    ldr d0, [x0]
-; CHECK-NEXT:    movi v1.2d, #0000000000000000
-; CHECK-NEXT:    ushll v0.8h, v0.8b, #0
-; CHECK-NEXT:    ushll2 v2.4s, v0.8h, #0
-; CHECK-NEXT:    mov v1.s[0], v2.s[0]
-; CHECK-NEXT:    uaddw v0.4s, v1.4s, v0.4h
-; CHECK-NEXT:    addv s0, v0.4s
-; CHECK-NEXT:    fmov w0, s0
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: test_udot_v5i8_nomla:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    ldr d0, [x0]
+; CHECK-SD-NEXT:    movi v1.2d, #0000000000000000
+; CHECK-SD-NEXT:    ushll v0.8h, v0.8b, #0
+; CHECK-SD-NEXT:    ushll2 v2.4s, v0.8h, #0
+; CHECK-SD-NEXT:    mov v1.s[0], v2.s[0]
+; CHECK-SD-NEXT:    uaddw v0.4s, v1.4s, v0.4h
+; CHECK-SD-NEXT:    addv s0, v0.4s
+; CHECK-SD-NEXT:    fmov w0, s0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: test_udot_v5i8_nomla:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    ldrb w8, [x0]
+; CHECK-GI-NEXT:    ldrb w9, [x0, #4]
+; CHECK-GI-NEXT:    mov v0.s[0], w8
+; CHECK-GI-NEXT:    mov v1.s[0], w9
+; CHECK-GI-NEXT:    ldrb w8, [x0, #1]
+; CHECK-GI-NEXT:    mov v0.s[1], w8
+; CHECK-GI-NEXT:    mov v1.s[1], wzr
+; CHECK-GI-NEXT:    ldrb w8, [x0, #2]
+; CHECK-GI-NEXT:    mov v0.s[2], w8
+; CHECK-GI-NEXT:    mov v1.s[2], wzr
+; CHECK-GI-NEXT:    ldrb w8, [x0, #3]
+; CHECK-GI-NEXT:    mov v0.s[3], w8
+; CHECK-GI-NEXT:    mov v1.s[3], wzr
+; CHECK-GI-NEXT:    add v0.4s, v0.4s, v1.4s
+; CHECK-GI-NEXT:    addv s0, v0.4s
+; CHECK-GI-NEXT:    fmov w0, s0
+; CHECK-GI-NEXT:    ret
 entry:
   %0 = load <5 x i8>, ptr %a1
   %1 = zext <5 x i8> %0 to <5 x i32>
@@ -456,19 +491,50 @@ entry:
   ret i32 %2
 }
 define i32 @test_sdot_v5i8(ptr nocapture readonly %a, ptr nocapture readonly %b, i32 %sum) {
-; CHECK-LABEL: test_sdot_v5i8:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    ldr d0, [x0]
-; CHECK-NEXT:    ldr d1, [x1]
-; CHECK-NEXT:    smull v0.8h, v1.8b, v0.8b
-; CHECK-NEXT:    movi v1.2d, #0000000000000000
-; CHECK-NEXT:    sshll2 v2.4s, v0.8h, #0
-; CHECK-NEXT:    mov v1.s[0], v2.s[0]
-; CHECK-NEXT:    saddw v0.4s, v1.4s, v0.4h
-; CHECK-NEXT:    addv s0, v0.4s
-; CHECK-NEXT:    fmov w8, s0
-; CHECK-NEXT:    add w0, w8, w2
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: test_sdot_v5i8:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    ldr d0, [x0]
+; CHECK-SD-NEXT:    ldr d1, [x1]
+; CHECK-SD-NEXT:    smull v0.8h, v1.8b, v0.8b
+; CHECK-SD-NEXT:    movi v1.2d, #0000000000000000
+; CHECK-SD-NEXT:    sshll2 v2.4s, v0.8h, #0
+; CHECK-SD-NEXT:    mov v1.s[0], v2.s[0]
+; CHECK-SD-NEXT:    saddw v0.4s, v1.4s, v0.4h
+; CHECK-SD-NEXT:    addv s0, v0.4s
+; CHECK-SD-NEXT:    fmov w8, s0
+; CHECK-SD-NEXT:    add w0, w8, w2
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: test_sdot_v5i8:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    ldrsb w8, [x0, #4]
+; CHECK-GI-NEXT:    ldrsb w9, [x1, #4]
+; CHECK-GI-NEXT:    ldrsb w10, [x1]
+; CHECK-GI-NEXT:    mul w8, w9, w8
+; CHECK-GI-NEXT:    ldrsb w9, [x0]
+; CHECK-GI-NEXT:    mov v0.s[0], w10
+; CHECK-GI-NEXT:    mov v1.s[0], w9
+; CHECK-GI-NEXT:    ldrsb w9, [x1, #1]
+; CHECK-GI-NEXT:    mov v2.s[0], w8
+; CHECK-GI-NEXT:    ldrsb w8, [x0, #1]
+; CHECK-GI-NEXT:    mov v0.s[1], w9
+; CHECK-GI-NEXT:    ldrsb w9, [x1, #2]
+; CHECK-GI-NEXT:    mov v1.s[1], w8
+; CHECK-GI-NEXT:    ldrsb w8, [x0, #2]
+; CHECK-GI-NEXT:    mov v2.s[1], wzr
+; CHECK-GI-NEXT:    mov v0.s[2], w9
+; CHECK-GI-NEXT:    ldrsb w9, [x1, #3]
+; CHECK-GI-NEXT:    mov v1.s[2], w8
+; CHECK-GI-NEXT:    ldrsb w8, [x0, #3]
+; CHECK-GI-NEXT:    mov v2.s[2], wzr
+; CHECK-GI-NEXT:    mov v0.s[3], w9
+; CHECK-GI-NEXT:    mov v1.s[3], w8
+; CHECK-GI-NEXT:    mov v2.s[3], wzr
+; CHECK-GI-NEXT:    mla v2.4s, v0.4s, v1.4s
+; CHECK-GI-NEXT:    addv s0, v2.4s
+; CHECK-GI-NEXT:    fmov w8, s0
+; CHECK-GI-NEXT:    add w0, w8, w2
+; CHECK-GI-NEXT:    ret
 entry:
   %0 = load <5 x i8>, ptr %a
   %1 = sext <5 x i8> %0 to <5 x i32>
@@ -481,22 +547,83 @@ entry:
 }
 
 define i32 @test_sdot_v5i8_double(<5 x i8> %a, <5 x i8> %b, <5 x i8> %c, <5 x i8> %d) {
-; CHECK-LABEL: test_sdot_v5i8_double:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    smull v2.8h, v2.8b, v3.8b
-; CHECK-NEXT:    smull v0.8h, v0.8b, v1.8b
-; CHECK-NEXT:    movi v1.2d, #0000000000000000
-; CHECK-NEXT:    movi v3.2d, #0000000000000000
-; CHECK-NEXT:    sshll2 v4.4s, v0.8h, #0
-; CHECK-NEXT:    sshll2 v5.4s, v2.8h, #0
-; CHECK-NEXT:    mov v3.s[0], v4.s[0]
-; CHECK-NEXT:    mov v1.s[0], v5.s[0]
-; CHECK-NEXT:    saddw v0.4s, v3.4s, v0.4h
-; CHECK-NEXT:    saddw v1.4s, v1.4s, v2.4h
-; CHECK-NEXT:    add v0.4s, v0.4s, v1.4s
-; CHECK-NEXT:    addv s0, v0.4s
-; CHECK-NEXT:    fmov w0, s0
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: test_sdot_v5i8_double:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    smull v2.8h, v2.8b, v3.8b
+; CHECK-SD-NEXT:    smull v0.8h, v0.8b, v1.8b
+; CHECK-SD-NEXT:    movi v1.2d, #0000000000000000
+; CHECK-SD-NEXT:    movi v3.2d, #0000000000000000
+; CHECK-SD-NEXT:    sshll2 v4.4s, v0.8h, #0
+; CHECK-SD-NEXT:    sshll2 v5.4s, v2.8h, #0
+; CHECK-SD-NEXT:    mov v3.s[0], v4.s[0]
+; CHECK-SD-NEXT:    mov v1.s[0], v5.s[0]
+; CHECK-SD-NEXT:    saddw v0.4s, v3.4s, v0.4h
+; CHECK-SD-NEXT:    saddw v1.4s, v1.4s, v2.4h
+; CHECK-SD-NEXT:    add v0.4s, v0.4s, v1.4s
+; CHECK-SD-NEXT:    addv s0, v0.4s
+; CHECK-SD-NEXT:    fmov w0, s0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: test_sdot_v5i8_double:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-GI-NEXT:    // kill: def $d1 killed $d1 def $q1
+; CHECK-GI-NEXT:    // kill: def $d2 killed $d2 def $q2
+; CHECK-GI-NEXT:    // kill: def $d3 killed $d3 def $q3
+; CHECK-GI-NEXT:    smov w8, v0.b[4]
+; CHECK-GI-NEXT:    smov w9, v1.b[4]
+; CHECK-GI-NEXT:    smov w10, v2.b[4]
+; CHECK-GI-NEXT:    smov w11, v3.b[4]
+; CHECK-GI-NEXT:    smov w12, v0.b[0]
+; CHECK-GI-NEXT:    smov w13, v1.b[0]
+; CHECK-GI-NEXT:    smov w14, v2.b[0]
+; CHECK-GI-NEXT:    smov w15, v3.b[0]
+; CHECK-GI-NEXT:    mul w8, w8, w9
+; CHECK-GI-NEXT:    smov w9, v0.b[1]
+; CHECK-GI-NEXT:    mul w10, w10, w11
+; CHECK-GI-NEXT:    smov w11, v1.b[1]
+; CHECK-GI-NEXT:    mov v4.s[0], w12
+; CHECK-GI-NEXT:    smov w12, v2.b[1]
+; CHECK-GI-NEXT:    mov v5.s[0], w13
+; CHECK-GI-NEXT:    smov w13, v3.b[1]
+; CHECK-GI-NEXT:    mov v6.s[0], w8
+; CHECK-GI-NEXT:    mov v7.s[0], w14
+; CHECK-GI-NEXT:    mov v16.s[0], w15
+; CHECK-GI-NEXT:    mov v17.s[0], w10
+; CHECK-GI-NEXT:    smov w8, v0.b[2]
+; CHECK-GI-NEXT:    smov w10, v1.b[2]
+; CHECK-GI-NEXT:    smov w14, v2.b[2]
+; CHECK-GI-NEXT:    smov w15, v3.b[2]
+; CHECK-GI-NEXT:    mov v4.s[1], w9
+; CHECK-GI-NEXT:    mov v5.s[1], w11
+; CHECK-GI-NEXT:    smov w9, v0.b[3]
+; CHECK-GI-NEXT:    smov w11, v1.b[3]
+; CHECK-GI-NEXT:    mov v6.s[1], wzr
+; CHECK-GI-NEXT:    mov v7.s[1], w12
+; CHECK-GI-NEXT:    mov v16.s[1], w13
+; CHECK-GI-NEXT:    mov v17.s[1], wzr
+; CHECK-GI-NEXT:    smov w12, v2.b[3]
+; CHECK-GI-NEXT:    smov w13, v3.b[3]
+; CHECK-GI-NEXT:    mov v4.s[2], w8
+; CHECK-GI-NEXT:    mov v5.s[2], w10
+; CHECK-GI-NEXT:    mov v6.s[2], wzr
+; CHECK-GI-NEXT:    mov v7.s[2], w14
+; CHECK-GI-NEXT:    mov v16.s[2], w15
+; CHECK-GI-NEXT:    mov v17.s[2], wzr
+; CHECK-GI-NEXT:    mov v4.s[3], w9
+; CHECK-GI-NEXT:    mov v5.s[3], w11
+; CHECK-GI-NEXT:    mov v6.s[3], wzr
+; CHECK-GI-NEXT:    mov v7.s[3], w12
+; CHECK-GI-NEXT:    mov v16.s[3], w13
+; CHECK-GI-NEXT:    mov v17.s[3], wzr
+; CHECK-GI-NEXT:    mla v6.4s, v4.4s, v5.4s
+; CHECK-GI-NEXT:    mla v17.4s, v7.4s, v16.4s
+; CHECK-GI-NEXT:    addv s0, v6.4s
+; CHECK-GI-NEXT:    addv s1, v17.4s
+; CHECK-GI-NEXT:    fmov w8, s0
+; CHECK-GI-NEXT:    fmov w9, s1
+; CHECK-GI-NEXT:    add w0, w8, w9
+; CHECK-GI-NEXT:    ret
 entry:
   %az = sext <5 x i8> %a to <5 x i32>
   %bz = sext <5 x i8> %b to <5 x i32>
@@ -511,22 +638,61 @@ entry:
 }
 
 define i32 @test_sdot_v5i8_double_nomla(<5 x i8> %a, <5 x i8> %b, <5 x i8> %c, <5 x i8> %d) {
-; CHECK-LABEL: test_sdot_v5i8_double_nomla:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    sshll v0.8h, v0.8b, #0
-; CHECK-NEXT:    sshll v1.8h, v2.8b, #0
-; CHECK-NEXT:    movi v2.2d, #0000000000000000
-; CHECK-NEXT:    movi v3.2d, #0000000000000000
-; CHECK-NEXT:    sshll2 v4.4s, v0.8h, #0
-; CHECK-NEXT:    sshll2 v5.4s, v1.8h, #0
-; CHECK-NEXT:    mov v3.s[0], v4.s[0]
-; CHECK-NEXT:    mov v2.s[0], v5.s[0]
-; CHECK-NEXT:    saddw v0.4s, v3.4s, v0.4h
-; CHECK-NEXT:    saddw v1.4s, v2.4s, v1.4h
-; CHECK-NEXT:    add v0.4s, v0.4s, v1.4s
-; CHECK-NEXT:    addv s0, v0.4s
-; CHECK-NEXT:    fmov w0, s0
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: test_sdot_v5i8_double_nomla:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    sshll v0.8h, v0.8b, #0
+; CHECK-SD-NEXT:    sshll v1.8h, v2.8b, #0
+; CHECK-SD-NEXT:    movi v2.2d, #0000000000000000
+; CHECK-SD-NEXT:    movi v3.2d, #0000000000000000
+; CHECK-SD-NEXT:    sshll2 v4.4s, v0.8h, #0
+; CHECK-SD-NEXT:    sshll2 v5.4s, v1.8h, #0
+; CHECK-SD-NEXT:    mov v3.s[0], v4.s[0]
+; CHECK-SD-NEXT:    mov v2.s[0], v5.s[0]
+; CHECK-SD-NEXT:    saddw v0.4s, v3.4s, v0.4h
+; CHECK-SD-NEXT:    saddw v1.4s, v2.4s, v1.4h
+; CHECK-SD-NEXT:    add v0.4s, v0.4s, v1.4s
+; CHECK-SD-NEXT:    addv s0, v0.4s
+; CHECK-SD-NEXT:    fmov w0, s0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: test_sdot_v5i8_double_nomla:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-GI-NEXT:    // kill: def $d2 killed $d2 def $q2
+; CHECK-GI-NEXT:    smov w8, v0.b[0]
+; CHECK-GI-NEXT:    smov w9, v0.b[4]
+; CHECK-GI-NEXT:    smov w10, v2.b[0]
+; CHECK-GI-NEXT:    smov w11, v2.b[4]
+; CHECK-GI-NEXT:    smov w12, v0.b[1]
+; CHECK-GI-NEXT:    mov v1.s[0], w8
+; CHECK-GI-NEXT:    smov w8, v2.b[1]
+; CHECK-GI-NEXT:    mov v3.s[0], w9
+; CHECK-GI-NEXT:    mov v4.s[0], w10
+; CHECK-GI-NEXT:    mov v5.s[0], w11
+; CHECK-GI-NEXT:    smov w9, v0.b[2]
+; CHECK-GI-NEXT:    smov w10, v2.b[2]
+; CHECK-GI-NEXT:    smov w11, v2.b[3]
+; CHECK-GI-NEXT:    mov v1.s[1], w12
+; CHECK-GI-NEXT:    mov v3.s[1], wzr
+; CHECK-GI-NEXT:    mov v4.s[1], w8
+; CHECK-GI-NEXT:    mov v5.s[1], wzr
+; CHECK-GI-NEXT:    smov w8, v0.b[3]
+; CHECK-GI-NEXT:    mov v1.s[2], w9
+; CHECK-GI-NEXT:    mov v3.s[2], wzr
+; CHECK-GI-NEXT:    mov v4.s[2], w10
+; CHECK-GI-NEXT:    mov v5.s[2], wzr
+; CHECK-GI-NEXT:    mov v1.s[3], w8
+; CHECK-GI-NEXT:    mov v3.s[3], wzr
+; CHECK-GI-NEXT:    mov v4.s[3], w11
+; CHECK-GI-NEXT:    mov v5.s[3], wzr
+; CHECK-GI-NEXT:    add v0.4s, v1.4s, v3.4s
+; CHECK-GI-NEXT:    add v1.4s, v4.4s, v5.4s
+; CHECK-GI-NEXT:    addv s0, v0.4s
+; CHECK-GI-NEXT:    addv s1, v1.4s
+; CHECK-GI-NEXT:    fmov w8, s0
+; CHECK-GI-NEXT:    fmov w9, s1
+; CHECK-GI-NEXT:    add w0, w8, w9
+; CHECK-GI-NEXT:    ret
 entry:
   %az = sext <5 x i8> %a to <5 x i32>
   %r1 = call i32 @llvm.vector.reduce.add.v5i32(<5 x i32> %az)
@@ -2181,27 +2347,153 @@ entry:
 }
 
 define i32 @test_udot_v25i8(ptr nocapture readonly %a, ptr nocapture readonly %b, i32 %sum) {
-; CHECK-LABEL: test_udot_v25i8:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    ldp q3, q0, [x1]
-; CHECK-NEXT:    movi v5.2d, #0000000000000000
-; CHECK-NEXT:    ldp q2, q1, [x0]
-; CHECK-NEXT:    umull2 v4.8h, v0.16b, v1.16b
-; CHECK-NEXT:    umull v0.8h, v0.8b, v1.8b
-; CHECK-NEXT:    umull v1.8h, v3.8b, v2.8b
-; CHECK-NEXT:    umull2 v2.8h, v3.16b, v2.16b
-; CHECK-NEXT:    ushll v3.4s, v4.4h, #0
-; CHECK-NEXT:    uaddl2 v4.4s, v1.8h, v0.8h
-; CHECK-NEXT:    uaddl v0.4s, v1.4h, v0.4h
-; CHECK-NEXT:    mov v5.s[0], v3.s[0]
-; CHECK-NEXT:    uaddw2 v1.4s, v4.4s, v2.8h
-; CHECK-NEXT:    add v0.4s, v0.4s, v1.4s
-; CHECK-NEXT:    uaddw v2.4s, v5.4s, v2.4h
-; CHECK-NEXT:    add v0.4s, v0.4s, v2.4s
-; CHECK-NEXT:    addv s0, v0.4s
-; CHECK-NEXT:    fmov w8, s0
-; CHECK-NEXT:    add w0, w8, w2
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: test_udot_v25i8:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    ldp q3, q0, [x1]
+; CHECK-SD-NEXT:    movi v5.2d, #0000000000000000
+; CHECK-SD-NEXT:    ldp q2, q1, [x0]
+; CHECK-SD-NEXT:    umull2 v4.8h, v0.16b, v1.16b
+; CHECK-SD-NEXT:    umull v0.8h, v0.8b, v1.8b
+; CHECK-SD-NEXT:    umull v1.8h, v3.8b, v2.8b
+; CHECK-SD-NEXT:    umull2 v2.8h, v3.16b, v2.16b
+; CHECK-SD-NEXT:    ushll v3.4s, v4.4h, #0
+; CHECK-SD-NEXT:    uaddl2 v4.4s, v1.8h, v0.8h
+; CHECK-SD-NEXT:    uaddl v0.4s, v1.4h, v0.4h
+; CHECK-SD-NEXT:    mov v5.s[0], v3.s[0]
+; CHECK-SD-NEXT:    uaddw2 v1.4s, v4.4s, v2.8h
+; CHECK-SD-NEXT:    add v0.4s, v0.4s, v1.4s
+; CHECK-SD-NEXT:    uaddw v2.4s, v5.4s, v2.4h
+; CHECK-SD-NEXT:    add v0.4s, v0.4s, v2.4s
+; CHECK-SD-NEXT:    addv s0, v0.4s
+; CHECK-SD-NEXT:    fmov w8, s0
+; CHECK-SD-NEXT:    add w0, w8, w2
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: test_udot_v25i8:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    ldr q1, [x1]
+; CHECK-GI-NEXT:    ldrb w11, [x1, #16]!
+; CHECK-GI-NEXT:    ldrb w12, [x1, #4]
+; CHECK-GI-NEXT:    ldr q0, [x0]
+; CHECK-GI-NEXT:    mov v23.s[0], wzr
+; CHECK-GI-NEXT:    umov w9, v1.b[4]
+; CHECK-GI-NEXT:    umov w10, v1.b[12]
+; CHECK-GI-NEXT:    umov w13, v1.b[0]
+; CHECK-GI-NEXT:    umov w14, v1.b[5]
+; CHECK-GI-NEXT:    mov v5.s[0], w12
+; CHECK-GI-NEXT:    mov v3.s[0], w11
+; CHECK-GI-NEXT:    umov w11, v0.b[0]
+; CHECK-GI-NEXT:    umov w12, v1.b[1]
+; CHECK-GI-NEXT:    umov w15, v1.b[8]
+; CHECK-GI-NEXT:    ldrb w8, [x0, #16]!
+; CHECK-GI-NEXT:    mov v23.s[1], wzr
+; CHECK-GI-NEXT:    mov v2.s[0], w9
+; CHECK-GI-NEXT:    mov v4.s[0], w10
+; CHECK-GI-NEXT:    umov w10, v1.b[13]
+; CHECK-GI-NEXT:    ldrb w9, [x1, #5]
+; CHECK-GI-NEXT:    mov v6.s[0], w13
+; CHECK-GI-NEXT:    umov w13, v1.b[6]
+; CHECK-GI-NEXT:    mov v16.s[0], w11
+; CHECK-GI-NEXT:    umov w11, v1.b[2]
+; CHECK-GI-NEXT:    mov v7.s[0], w15
+; CHECK-GI-NEXT:    mov v5.s[1], w9
+; CHECK-GI-NEXT:    ldrb w9, [x1, #6]
+; CHECK-GI-NEXT:    umov w15, v1.b[9]
+; CHECK-GI-NEXT:    mov v2.s[1], w14
+; CHECK-GI-NEXT:    ldrb w14, [x1, #1]
+; CHECK-GI-NEXT:    mov v4.s[1], w10
+; CHECK-GI-NEXT:    umov w10, v1.b[14]
+; CHECK-GI-NEXT:    mov v6.s[1], w12
+; CHECK-GI-NEXT:    umov w12, v0.b[1]
+; CHECK-GI-NEXT:    mov v3.s[1], w14
+; CHECK-GI-NEXT:    umov w14, v0.b[12]
+; CHECK-GI-NEXT:    mov v21.s[0], w8
+; CHECK-GI-NEXT:    ldrb w8, [x0, #1]
+; CHECK-GI-NEXT:    mov v5.s[2], w9
+; CHECK-GI-NEXT:    umov w9, v0.b[4]
+; CHECK-GI-NEXT:    mov v2.s[2], w13
+; CHECK-GI-NEXT:    umov w13, v1.b[7]
+; CHECK-GI-NEXT:    mov v7.s[1], w15
+; CHECK-GI-NEXT:    mov v4.s[2], w10
+; CHECK-GI-NEXT:    umov w10, v1.b[15]
+; CHECK-GI-NEXT:    mov v16.s[1], w12
+; CHECK-GI-NEXT:    ldrb w12, [x1, #2]
+; CHECK-GI-NEXT:    mov v6.s[2], w11
+; CHECK-GI-NEXT:    umov w11, v0.b[2]
+; CHECK-GI-NEXT:    mov v17.s[0], w9
+; CHECK-GI-NEXT:    umov w9, v0.b[8]
+; CHECK-GI-NEXT:    mov v18.s[0], w14
+; CHECK-GI-NEXT:    mov v2.s[3], w13
+; CHECK-GI-NEXT:    ldrb w13, [x1, #7]
+; CHECK-GI-NEXT:    mov v3.s[2], w12
+; CHECK-GI-NEXT:    ldrb w12, [x0, #4]
+; CHECK-GI-NEXT:    mov v4.s[3], w10
+; CHECK-GI-NEXT:    umov w10, v0.b[5]
+; CHECK-GI-NEXT:    mov v5.s[3], w13
+; CHECK-GI-NEXT:    ldrb w13, [x0, #8]
+; CHECK-GI-NEXT:    mov v16.s[2], w11
+; CHECK-GI-NEXT:    umov w11, v0.b[13]
+; CHECK-GI-NEXT:    mov v20.s[0], w12
+; CHECK-GI-NEXT:    ldrb w12, [x1, #8]
+; CHECK-GI-NEXT:    mov v19.s[0], w9
+; CHECK-GI-NEXT:    umov w9, v0.b[6]
+; CHECK-GI-NEXT:    umov w15, v1.b[10]
+; CHECK-GI-NEXT:    mul w12, w12, w13
+; CHECK-GI-NEXT:    mov v17.s[1], w10
+; CHECK-GI-NEXT:    ldrb w10, [x0, #5]
+; CHECK-GI-NEXT:    umov w13, v0.b[9]
+; CHECK-GI-NEXT:    mov v21.s[1], w8
+; CHECK-GI-NEXT:    umov w8, v1.b[11]
+; CHECK-GI-NEXT:    mov v18.s[1], w11
+; CHECK-GI-NEXT:    umov w11, v0.b[14]
+; CHECK-GI-NEXT:    mov v20.s[1], w10
+; CHECK-GI-NEXT:    ldrb w10, [x0, #6]
+; CHECK-GI-NEXT:    mov v22.s[0], w12
+; CHECK-GI-NEXT:    umov w12, v0.b[7]
+; CHECK-GI-NEXT:    mov v17.s[2], w9
+; CHECK-GI-NEXT:    umov w9, v0.b[10]
+; CHECK-GI-NEXT:    mov v7.s[2], w15
+; CHECK-GI-NEXT:    mov v19.s[1], w13
+; CHECK-GI-NEXT:    umov w13, v1.b[3]
+; CHECK-GI-NEXT:    mov v23.s[2], wzr
+; CHECK-GI-NEXT:    mov v18.s[2], w11
+; CHECK-GI-NEXT:    umov w11, v0.b[15]
+; CHECK-GI-NEXT:    mov v20.s[2], w10
+; CHECK-GI-NEXT:    ldrb w10, [x0, #2]
+; CHECK-GI-NEXT:    mov v22.s[1], wzr
+; CHECK-GI-NEXT:    mov v17.s[3], w12
+; CHECK-GI-NEXT:    ldrb w12, [x0, #7]
+; CHECK-GI-NEXT:    mov v7.s[3], w8
+; CHECK-GI-NEXT:    ldrb w8, [x0, #3]
+; CHECK-GI-NEXT:    mov v19.s[2], w9
+; CHECK-GI-NEXT:    umov w9, v0.b[3]
+; CHECK-GI-NEXT:    mov v18.s[3], w11
+; CHECK-GI-NEXT:    umov w11, v0.b[11]
+; CHECK-GI-NEXT:    mov v21.s[2], w10
+; CHECK-GI-NEXT:    ldrb w10, [x1, #3]
+; CHECK-GI-NEXT:    mov v20.s[3], w12
+; CHECK-GI-NEXT:    mov v22.s[2], wzr
+; CHECK-GI-NEXT:    mov v6.s[3], w13
+; CHECK-GI-NEXT:    mul v0.4s, v2.4s, v17.4s
+; CHECK-GI-NEXT:    mov v23.s[3], wzr
+; CHECK-GI-NEXT:    mov v3.s[3], w10
+; CHECK-GI-NEXT:    mov v16.s[3], w9
+; CHECK-GI-NEXT:    mov v19.s[3], w11
+; CHECK-GI-NEXT:    mul v1.4s, v4.4s, v18.4s
+; CHECK-GI-NEXT:    mov v21.s[3], w8
+; CHECK-GI-NEXT:    mul v2.4s, v5.4s, v20.4s
+; CHECK-GI-NEXT:    mov v22.s[3], wzr
+; CHECK-GI-NEXT:    mla v0.4s, v6.4s, v16.4s
+; CHECK-GI-NEXT:    mla v1.4s, v7.4s, v19.4s
+; CHECK-GI-NEXT:    mla v2.4s, v3.4s, v21.4s
+; CHECK-GI-NEXT:    add v3.4s, v22.4s, v23.4s
+; CHECK-GI-NEXT:    add v0.4s, v0.4s, v1.4s
+; CHECK-GI-NEXT:    add v1.4s, v2.4s, v3.4s
+; CHECK-GI-NEXT:    add v0.4s, v0.4s, v1.4s
+; CHECK-GI-NEXT:    addv s0, v0.4s
+; CHECK-GI-NEXT:    fmov w8, s0
+; CHECK-GI-NEXT:    add w0, w8, w2
+; CHECK-GI-NEXT:    ret
 entry:
   %0 = load <25 x i8>, ptr %a
   %1 = zext <25 x i8> %0 to <25 x i32>
@@ -2214,25 +2506,96 @@ entry:
 }
 
 define i32 @test_udot_v25i8_nomla(ptr nocapture readonly %a1) {
-; CHECK-LABEL: test_udot_v25i8_nomla:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    ldp q2, q1, [x0]
-; CHECK-NEXT:    movi v0.2d, #0000000000000000
-; CHECK-NEXT:    ushll2 v3.8h, v1.16b, #0
-; CHECK-NEXT:    ushll v1.8h, v1.8b, #0
-; CHECK-NEXT:    ushll v4.8h, v2.8b, #0
-; CHECK-NEXT:    ushll2 v2.8h, v2.16b, #0
-; CHECK-NEXT:    ushll v3.4s, v3.4h, #0
-; CHECK-NEXT:    uaddl2 v5.4s, v4.8h, v1.8h
-; CHECK-NEXT:    uaddl v1.4s, v4.4h, v1.4h
-; CHECK-NEXT:    mov v0.s[0], v3.s[0]
-; CHECK-NEXT:    uaddw2 v3.4s, v5.4s, v2.8h
-; CHECK-NEXT:    add v1.4s, v1.4s, v3.4s
-; CHECK-NEXT:    uaddw v0.4s, v0.4s, v2.4h
-; CHECK-NEXT:    add v0.4s, v1.4s, v0.4s
-; CHECK-NEXT:    addv s0, v0.4s
-; CHECK-NEXT:    fmov w0, s0
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: test_udot_v25i8_nomla:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    ldp q2, q1, [x0]
+; CHECK-SD-NEXT:    movi v0.2d, #0000000000000000
+; CHECK-SD-NEXT:    ushll2 v3.8h, v1.16b, #0
+; CHECK-SD-NEXT:    ushll v1.8h, v1.8b, #0
+; CHECK-SD-NEXT:    ushll v4.8h, v2.8b, #0
+; CHECK-SD-NEXT:    ushll2 v2.8h, v2.16b, #0
+; CHECK-SD-NEXT:    ushll v3.4s, v3.4h, #0
+; CHECK-SD-NEXT:    uaddl2 v5.4s, v4.8h, v1.8h
+; CHECK-SD-NEXT:    uaddl v1.4s, v4.4h, v1.4h
+; CHECK-SD-NEXT:    mov v0.s[0], v3.s[0]
+; CHECK-SD-NEXT:    uaddw2 v3.4s, v5.4s, v2.8h
+; CHECK-SD-NEXT:    add v1.4s, v1.4s, v3.4s
+; CHECK-SD-NEXT:    uaddw v0.4s, v0.4s, v2.4h
+; CHECK-SD-NEXT:    add v0.4s, v1.4s, v0.4s
+; CHECK-SD-NEXT:    addv s0, v0.4s
+; CHECK-SD-NEXT:    fmov w0, s0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: test_udot_v25i8_nomla:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    ldr q1, [x0]
+; CHECK-GI-NEXT:    ldrb w11, [x0, #16]!
+; CHECK-GI-NEXT:    ldrb w14, [x0, #4]
+; CHECK-GI-NEXT:    ldrb w17, [x0, #8]
+; CHECK-GI-NEXT:    mov v0.s[0], wzr
+; CHECK-GI-NEXT:    umov w12, v1.b[0]
+; CHECK-GI-NEXT:    umov w13, v1.b[4]
+; CHECK-GI-NEXT:    umov w15, v1.b[8]
+; CHECK-GI-NEXT:    umov w16, v1.b[12]
+; CHECK-GI-NEXT:    umov w18, v1.b[1]
+; CHECK-GI-NEXT:    umov w1, v1.b[5]
+; CHECK-GI-NEXT:    umov w2, v1.b[9]
+; CHECK-GI-NEXT:    umov w3, v1.b[13]
+; CHECK-GI-NEXT:    mov v4.s[0], w11
+; CHECK-GI-NEXT:    mov v7.s[0], w14
+; CHECK-GI-NEXT:    mov v16.s[0], w17
+; CHECK-GI-NEXT:    ldrb w8, [x0, #1]
+; CHECK-GI-NEXT:    mov v2.s[0], w12
+; CHECK-GI-NEXT:    mov v3.s[0], w13
+; CHECK-GI-NEXT:    ldrb w10, [x0, #5]
+; CHECK-GI-NEXT:    mov v5.s[0], w15
+; CHECK-GI-NEXT:    mov v6.s[0], w16
+; CHECK-GI-NEXT:    ldrb w16, [x0, #2]
+; CHECK-GI-NEXT:    umov w9, v1.b[2]
+; CHECK-GI-NEXT:    umov w12, v1.b[6]
+; CHECK-GI-NEXT:    ldrb w17, [x0, #6]
+; CHECK-GI-NEXT:    umov w14, v1.b[10]
+; CHECK-GI-NEXT:    umov w15, v1.b[14]
+; CHECK-GI-NEXT:    mov v4.s[1], w8
+; CHECK-GI-NEXT:    mov v2.s[1], w18
+; CHECK-GI-NEXT:    mov v3.s[1], w1
+; CHECK-GI-NEXT:    mov v7.s[1], w10
+; CHECK-GI-NEXT:    mov v5.s[1], w2
+; CHECK-GI-NEXT:    mov v6.s[1], w3
+; CHECK-GI-NEXT:    mov v16.s[1], wzr
+; CHECK-GI-NEXT:    mov v0.s[1], wzr
+; CHECK-GI-NEXT:    umov w11, v1.b[3]
+; CHECK-GI-NEXT:    umov w13, v1.b[7]
+; CHECK-GI-NEXT:    umov w8, v1.b[11]
+; CHECK-GI-NEXT:    umov w10, v1.b[15]
+; CHECK-GI-NEXT:    mov v4.s[2], w16
+; CHECK-GI-NEXT:    mov v2.s[2], w9
+; CHECK-GI-NEXT:    ldrb w9, [x0, #3]
+; CHECK-GI-NEXT:    mov v3.s[2], w12
+; CHECK-GI-NEXT:    ldrb w12, [x0, #7]
+; CHECK-GI-NEXT:    mov v5.s[2], w14
+; CHECK-GI-NEXT:    mov v6.s[2], w15
+; CHECK-GI-NEXT:    mov v7.s[2], w17
+; CHECK-GI-NEXT:    mov v16.s[2], wzr
+; CHECK-GI-NEXT:    mov v0.s[2], wzr
+; CHECK-GI-NEXT:    mov v4.s[3], w9
+; CHECK-GI-NEXT:    mov v2.s[3], w11
+; CHECK-GI-NEXT:    mov v3.s[3], w13
+; CHECK-GI-NEXT:    mov v5.s[3], w8
+; CHECK-GI-NEXT:    mov v6.s[3], w10
+; CHECK-GI-NEXT:    mov v7.s[3], w12
+; CHECK-GI-NEXT:    mov v16.s[3], wzr
+; CHECK-GI-NEXT:    mov v0.s[3], wzr
+; CHECK-GI-NEXT:    add v1.4s, v2.4s, v3.4s
+; CHECK-GI-NEXT:    add v2.4s, v5.4s, v6.4s
+; CHECK-GI-NEXT:    add v3.4s, v4.4s, v7.4s
+; CHECK-GI-NEXT:    add v0.4s, v16.4s, v0.4s
+; CHECK-GI-NEXT:    add v1.4s, v1.4s, v2.4s
+; CHECK-GI-NEXT:    add v0.4s, v3.4s, v0.4s
+; CHECK-GI-NEXT:    add v0.4s, v1.4s, v0.4s
+; CHECK-GI-NEXT:    addv s0, v0.4s
+; CHECK-GI-NEXT:    fmov w0, s0
+; CHECK-GI-NEXT:    ret
 entry:
   %0 = load <25 x i8>, ptr %a1
   %1 = zext <25 x i8> %0 to <25 x i32>
@@ -2240,467 +2603,1114 @@ entry:
   ret i32 %2
 }
 define i32 @test_sdot_v25i8(ptr nocapture readonly %a, ptr nocapture readonly %b, i32 %sum) {
-; CHECK-LABEL: test_sdot_v25i8:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    ldp q3, q0, [x1]
-; CHECK-NEXT:    movi v5.2d, #0000000000000000
-; CHECK-NEXT:    ldp q2, q1, [x0]
-; CHECK-NEXT:    smull2 v4.8h, v0.16b, v1.16b
-; CHECK-NEXT:    smull v0.8h, v0.8b, v1.8b
-; CHECK-NEXT:    smull v1.8h, v3.8b, v2.8b
-; CHECK-NEXT:    smull2 v2.8h, v3.16b, v2.16b
-; CHECK-NEXT:    sshll v3.4s, v4.4h, #0
-; CHECK-NEXT:    saddl2 v4.4s, v1.8h, v0.8h
-; CHECK-NEXT:    saddl v0.4s, v1.4h, v0.4h
-; CHECK-NEXT:    mov v5.s[0], v3.s[0]
-; CHECK-NEXT:    saddw2 v1.4s, v4.4s, v2.8h
-; CHECK-NEXT:    add v0.4s, v0.4s, v1.4s
-; CHECK-NEXT:    saddw v2.4s, v5.4s, v2.4h
-; CHECK-NEXT:    add v0.4s, v0.4s, v2.4s
-; CHECK-NEXT:    addv s0, v0.4s
-; CHECK-NEXT:    fmov w8, s0
-; CHECK-NEXT:    add w0, w8, w2
-; CHECK-NEXT:    ret
-entry:
-  %0 = load <25 x i8>, ptr %a
-  %1 = sext <25 x i8> %0 to <25 x i32>
-  %2 = load <25 x i8>, ptr %b
-  %3 = sext <25 x i8> %2 to <25 x i32>
-  %4 = mul nsw <25 x i32> %3, %1
-  %5 = call i32 @llvm.vector.reduce.add.v25i32(<25 x i32> %4)
-  %op.extra = add nsw i32 %5, %sum
-  ret i32 %op.extra
-}
-
-define i32 @test_sdot_v25i8_double(<25 x i8> %a, <25 x i8> %b, <25 x i8> %c, <25 x i8> %d) {
-; CHECK-LABEL: test_sdot_v25i8_double:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill
-; CHECK-NEXT:    .cfi_def_cfa_offset 16
-; CHECK-NEXT:    .cfi_offset w29, -16
-; CHECK-NEXT:    ldr b0, [sp, #216]
-; CHECK-NEXT:    add x8, sp, #224
-; CHECK-NEXT:    ldr b1, [sp, #16]
-; CHECK-NEXT:    ldr b2, [sp, #280]
-; CHECK-NEXT:    add x9, sp, #240
-; CHECK-NEXT:    ldr b4, [sp, #80]
-; CHECK-NEXT:    ld1 { v0.b }[1], [x8]
-; CHECK-NEXT:    add x8, sp, #24
-; CHECK-NEXT:    add x10, sp, #48
-; CHECK-NEXT:    ld1 { v1.b }[1], [x8]
-; CHECK-NEXT:    add x8, sp, #232
-; CHECK-NEXT:    add x11, sp, #96
-; CHECK-NEXT:    ldr b5, [sp, #152]
-; CHECK-NEXT:    add x12, sp, #168
-; CHECK-NEXT:    ldr b6, [sp, #616]
-; CHECK-NEXT:    ld1 { v0.b }[2], [x8]
-; CHECK-NEXT:    add x8, sp, #32
-; CHECK-NEXT:    fmov s3, w0
-; CHECK-NEXT:    ld1 { v1.b }[2], [x8]
-; CHECK-NEXT:    add x8, sp, #288
-; CHECK-NEXT:    ldr b7, [sp, #416]
-; CHECK-NEXT:    ld1 { v2.b }[1], [x8]
-; CHECK-NEXT:    add x8, sp, #40
-; CHECK-NEXT:    ldr b22, [sp, #744]
-; CHECK-NEXT:    ld1 { v0.b }[3], [x9]
-; CHECK-NEXT:    add x9, sp, #248
-; CHECK-NEXT:    mov v3.b[1], w1
-; CHECK-NEXT:    ld1 { v1.b }[3], [x8]
-; CHECK-NEXT:    add x8, sp, #88
-; CHECK-NEXT:    ldr b23, [sp, #544]
-; CHECK-NEXT:    ld1 { v4.b }[1], [x8]
-; CHECK-NEXT:    add x8, sp, #256
-; CHECK-NEXT:    ldr b19, [sp, #680]
-; CHECK-NEXT:    ld1 { v0.b }[4], [x9]
-; CHECK-NEXT:    add x9, sp, #296
-; CHECK-NEXT:    ldr b20, [sp, #480]
-; CHECK-NEXT:    ld1 { v1.b }[4], [x10]
-; CHECK-NEXT:    ld1 { v2.b }[2], [x9]
-; CHECK-NEXT:    add x10, sp, #160
-; CHECK-NEXT:    ld1 { v4.b }[2], [x11]
-; CHECK-NEXT:    add x11, sp, #304
-; CHECK-NEXT:    ld1 { v5.b }[1], [x10]
-; CHECK-NEXT:    ld1 { v0.b }[5], [x8]
-; CHECK-NEXT:    add x8, sp, #56
-; CHECK-NEXT:    add x10, sp, #264
-; CHECK-NEXT:    ld1 { v1.b }[5], [x8]
-; CHECK-NEXT:    add x8, sp, #64
-; CHECK-NEXT:    ld1 { v2.b }[3], [x11]
-; CHECK-NEXT:    add x9, sp, #272
-; CHECK-NEXT:    ld1 { v5.b }[2], [x12]
-; CHECK-NEXT:    add x11, sp, #72
-; CHECK-NEXT:    ld1 { v0.b }[6], [x10]
-; CHECK-NEXT:    add x10, sp, #312
-; CHECK-NEXT:    mov v3.b[2], w2
-; CHECK-NEXT:    ld1 { v1.b }[6], [x8]
-; CHECK-NEXT:    add x8, sp, #104
-; CHECK-NEXT:    ld1 { v2.b }[4], [x10]
-; CHECK-NEXT:    ld1 { v4.b }[3], [x8]
-; CHECK-NEXT:    add x8, sp, #112
-; CHECK-NEXT:    add x10, sp, #128
-; CHECK-NEXT:    ld1 { v0.b }[7], [x9]
-; CHECK-NEXT:    add x9, sp, #320
-; CHECK-NEXT:    ldr b21, [sp, #552]
-; CHECK-NEXT:    ld1 { v2.b }[5], [x9]
-; CHECK-NEXT:    add x9, sp, #176
-; CHECK-NEXT:    ld1 { v1.b }[7], [x11]
-; CHECK-NEXT:    ld1 { v4.b }[4], [x8]
-; CHECK-NEXT:    add x8, sp, #624
-; CHECK-NEXT:    ld1 { v5.b }[3], [x9]
-; CHECK-NEXT:    ld1 { v6.b }[1], [x8]
-; CHECK-NEXT:    add x8, sp, #120
-; CHECK-NEXT:    add x9, sp, #328
-; CHECK-NEXT:    ld1 { v2.b }[6], [x9]
-; CHECK-NEXT:    add x9, sp, #184
-; CHECK-NEXT:    add x11, sp, #192
-; CHECK-NEXT:    ld1 { v4.b }[5], [x8]
-; CHECK-NEXT:    add x8, sp, #632
-; CHECK-NEXT:    ld1 { v5.b }[4], [x9]
-; CHECK-NEXT:    ld1 { v6.b }[2], [x8]
-; CHECK-NEXT:    add x9, sp, #640
-; CHECK-NEXT:    add x8, sp, #336
-; CHECK-NEXT:    ld1 { v2.b }[7], [x8]
-; CHECK-NEXT:    add x8, sp, #656
-; CHECK-NEXT:    smull v23.8h, v23.8b, v22.8b
-; CHECK-NEXT:    ld1 { v5.b }[5], [x11]
-; CHECK-NEXT:    add x11, sp, #648
-; CHECK-NEXT:    ld1 { v4.b }[6], [x10]
-; CHECK-NEXT:    ld1 { v6.b }[3], [x9]
-; CHECK-NEXT:    add x9, sp, #200
-; CHECK-NEXT:    add x10, sp, #136
-; CHECK-NEXT:    ldr b22, [sp, #352]
-; CHECK-NEXT:    add x12, sp, #360
-; CHECK-NEXT:    mov v3.b[3], w3
-; CHECK-NEXT:    ld1 { v5.b }[6], [x9]
-; CHECK-NEXT:    add x9, sp, #208
-; CHECK-NEXT:    ld1 { v4.b }[7], [x10]
-; CHECK-NEXT:    ld1 { v6.b }[4], [x11]
-; CHECK-NEXT:    add x11, sp, #424
-; CHECK-NEXT:    add x10, sp, #488
-; CHECK-NEXT:    ld1 { v7.b }[1], [x11]
-; CHECK-NEXT:    add x11, sp, #560
-; CHECK-NEXT:    ld1 { v20.b }[1], [x10]
-; CHECK-NEXT:    ld1 { v5.b }[7], [x9]
-; CHECK-NEXT:    add x9, sp, #440
-; CHECK-NEXT:    ld1 { v21.b }[1], [x11]
-; CHECK-NEXT:    ld1 { v6.b }[5], [x8]
-; CHECK-NEXT:    add x8, sp, #432
-; CHECK-NEXT:    ld1 { v22.b }[1], [x12]
-; CHECK-NEXT:    ld1 { v7.b }[2], [x8]
-; CHECK-NEXT:    add x11, sp, #496
-; CHECK-NEXT:    add x12, sp, #568
-; CHECK-NEXT:    add x13, sp, #368
-; CHECK-NEXT:    ld1 { v20.b }[2], [x11]
-; CHECK-NEXT:    ld1 { v21.b }[2], [x12]
-; CHECK-NEXT:    ld1 { v22.b }[2], [x13]
-; CHECK-NEXT:    add x10, sp, #448
-; CHECK-NEXT:    mov v3.b[4], w4
-; CHECK-NEXT:    ld1 { v7.b }[3], [x9]
-; CHECK-NEXT:    add x9, sp, #688
-; CHECK-NEXT:    add x11, sp, #576
-; CHECK-NEXT:    ld1 { v19.b }[1], [x9]
-; CHECK-NEXT:    add x9, sp, #696
-; CHECK-NEXT:    add x12, sp, #376
-; CHECK-NEXT:    ld1 { v21.b }[3], [x11]
-; CHECK-NEXT:    ld1 { v22.b }[3], [x12]
-; CHECK-NEXT:    add x11, sp, #512
-; CHECK-NEXT:    ld1 { v7.b }[4], [x10]
-; CHECK-NEXT:    add x10, sp, #504
-; CHECK-NEXT:    add x12, sp, #584
-; CHECK-NEXT:    ld1 { v19.b }[2], [x9]
-; CHECK-NEXT:    add x9, sp, #704
-; CHECK-NEXT:    ld1 { v20.b }[3], [x10]
-; CHECK-NEXT:    add x13, sp, #384
-; CHECK-NEXT:    mov v3.b[5], w5
-; CHECK-NEXT:    ld1 { v21.b }[4], [x12]
-; CHECK-NEXT:    ld1 { v22.b }[4], [x13]
-; CHECK-NEXT:    add x10, sp, #456
-; CHECK-NEXT:    ldr b16, [sp, #344]
-; CHECK-NEXT:    ld1 { v19.b }[3], [x9]
-; CHECK-NEXT:    add x9, sp, #712
-; CHECK-NEXT:    ld1 { v20.b }[4], [x11]
-; CHECK-NEXT:    ldr b17, [sp, #144]
-; CHECK-NEXT:    ld1 { v7.b }[5], [x10]
-; CHECK-NEXT:    add x10, sp, #520
-; CHECK-NEXT:    add x11, sp, #592
-; CHECK-NEXT:    add x12, sp, #392
-; CHECK-NEXT:    mov v3.b[6], w6
-; CHECK-NEXT:    ld1 { v19.b }[4], [x9]
-; CHECK-NEXT:    add x9, sp, #720
-; CHECK-NEXT:    ld1 { v20.b }[5], [x10]
-; CHECK-NEXT:    ld1 { v21.b }[5], [x11]
-; CHECK-NEXT:    ld1 { v22.b }[5], [x12]
-; CHECK-NEXT:    smull v16.8h, v17.8b, v16.8b
-; CHECK-NEXT:    add x8, sp, #664
-; CHECK-NEXT:    add x10, sp, #464
-; CHECK-NEXT:    add x11, sp, #528
-; CHECK-NEXT:    ld1 { v19.b }[5], [x9]
-; CHECK-NEXT:    add x9, sp, #728
-; CHECK-NEXT:    add x12, sp, #600
-; CHECK-NEXT:    add x13, sp, #400
-; CHECK-NEXT:    ld1 { v6.b }[6], [x8]
-; CHECK-NEXT:    ld1 { v20.b }[6], [x11]
-; CHECK-NEXT:    ld1 { v21.b }[6], [x12]
-; CHECK-NEXT:    ld1 { v22.b }[6], [x13]
-; CHECK-NEXT:    ld1 { v7.b }[6], [x10]
-; CHECK-NEXT:    ld1 { v19.b }[6], [x9]
-; CHECK-NEXT:    add x9, sp, #736
-; CHECK-NEXT:    mov v3.b[7], w7
-; CHECK-NEXT:    sshll v18.4s, v16.4h, #0
-; CHECK-NEXT:    movi v16.2d, #0000000000000000
-; CHECK-NEXT:    movi v17.2d, #0000000000000000
-; CHECK-NEXT:    add x8, sp, #672
-; CHECK-NEXT:    add x10, sp, #472
-; CHECK-NEXT:    add x11, sp, #608
-; CHECK-NEXT:    ld1 { v19.b }[7], [x9]
-; CHECK-NEXT:    add x9, sp, #536
-; CHECK-NEXT:    add x12, sp, #408
-; CHECK-NEXT:    ld1 { v20.b }[7], [x9]
-; CHECK-NEXT:    ld1 { v21.b }[7], [x11]
-; CHECK-NEXT:    ld1 { v22.b }[7], [x12]
-; CHECK-NEXT:    ld1 { v6.b }[7], [x8]
-; CHECK-NEXT:    ld1 { v7.b }[7], [x10]
-; CHECK-NEXT:    sshll v23.4s, v23.4h, #0
-; CHECK-NEXT:    smull v0.8h, v1.8b, v0.8b
-; CHECK-NEXT:    smull v1.8h, v4.8b, v2.8b
-; CHECK-NEXT:    smull v2.8h, v3.8b, v5.8b
-; CHECK-NEXT:    smull v3.8h, v20.8b, v19.8b
-; CHECK-NEXT:    smull v4.8h, v22.8b, v21.8b
-; CHECK-NEXT:    mov v17.s[0], v18.s[0]
-; CHECK-NEXT:    smull v5.8h, v7.8b, v6.8b
-; CHECK-NEXT:    mov v16.s[0], v23.s[0]
-; CHECK-NEXT:    saddl2 v6.4s, v2.8h, v1.8h
-; CHECK-NEXT:    saddl v1.4s, v2.4h, v1.4h
-; CHECK-NEXT:    saddl2 v2.4s, v4.8h, v3.8h
-; CHECK-NEXT:    saddl v3.4s, v4.4h, v3.4h
-; CHECK-NEXT:    saddw v4.4s, v17.4s, v0.4h
-; CHECK-NEXT:    saddw v7.4s, v16.4s, v5.4h
-; CHECK-NEXT:    saddw2 v0.4s, v6.4s, v0.8h
-; CHECK-NEXT:    add v1.4s, v1.4s, v4.4s
-; CHECK-NEXT:    saddw2 v2.4s, v2.4s, v5.8h
-; CHECK-NEXT:    add v3.4s, v3.4s, v7.4s
-; CHECK-NEXT:    add v0.4s, v1.4s, v0.4s
-; CHECK-NEXT:    add v1.4s, v3.4s, v2.4s
-; CHECK-NEXT:    add v0.4s, v0.4s, v1.4s
-; CHECK-NEXT:    addv s0, v0.4s
-; CHECK-NEXT:    fmov w0, s0
-; CHECK-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
-; CHECK-NEXT:    ret
-entry:
-  %az = sext <25 x i8> %a to <25 x i32>
-  %bz = sext <25 x i8> %b to <25 x i32>
-  %m1 = mul nuw nsw <25 x i32> %az, %bz
-  %r1 = call i32 @llvm.vector.reduce.add.v25i32(<25 x i32> %m1)
-  %cz = sext <25 x i8> %c to <25 x i32>
-  %dz = sext <25 x i8> %d to <25 x i32>
-  %m2 = mul nuw nsw <25 x i32> %cz, %dz
-  %r2 = call i32 @llvm.vector.reduce.add.v25i32(<25 x i32> %m2)
-  %x = add i32 %r1, %r2
-  ret i32 %x
-}
-
-define i32 @test_sdot_v25i8_double_nomla(<25 x i8> %a, <25 x i8> %b, <25 x i8> %c, <25 x i8> %d) {
-; CHECK-LABEL: test_sdot_v25i8_double_nomla:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill
-; CHECK-NEXT:    .cfi_def_cfa_offset 16
-; CHECK-NEXT:    .cfi_offset w29, -16
-; CHECK-NEXT:    fmov s0, w0
-; CHECK-NEXT:    ldr b1, [sp, #80]
-; CHECK-NEXT:    add x10, sp, #88
-; CHECK-NEXT:    ldr b2, [sp, #16]
-; CHECK-NEXT:    add x9, sp, #96
-; CHECK-NEXT:    ldr b3, [sp, #480]
-; CHECK-NEXT:    ld1 { v1.b }[1], [x10]
-; CHECK-NEXT:    add x10, sp, #24
-; CHECK-NEXT:    ldr b4, [sp, #352]
-; CHECK-NEXT:    mov v0.b[1], w1
-; CHECK-NEXT:    ld1 { v2.b }[1], [x10]
-; CHECK-NEXT:    add x11, sp, #488
-; CHECK-NEXT:    add x10, sp, #360
-; CHECK-NEXT:    ldr b5, [sp, #416]
-; CHECK-NEXT:    add x8, sp, #104
-; CHECK-NEXT:    ld1 { v1.b }[2], [x9]
-; CHECK-NEXT:    add x9, sp, #32
-; CHECK-NEXT:    ld1 { v3.b }[1], [x11]
-; CHECK-NEXT:    ld1 { v2.b }[2], [x9]
-; CHECK-NEXT:    add x11, sp, #424
-; CHECK-NEXT:    ld1 { v4.b }[1], [x10]
-; CHECK-NEXT:    mov v0.b[2], w2
-; CHECK-NEXT:    ld1 { v5.b }[1], [x11]
-; CHECK-NEXT:    add x9, sp, #368
-; CHECK-NEXT:    ld1 { v1.b }[3], [x8]
-; CHECK-NEXT:    add x8, sp, #40
-; CHECK-NEXT:    add x12, sp, #496
-; CHECK-NEXT:    ld1 { v2.b }[3], [x8]
-; CHECK-NEXT:    ld1 { v4.b }[2], [x9]
-; CHECK-NEXT:    add x8, sp, #432
-; CHECK-NEXT:    ld1 { v3.b }[2], [x12]
-; CHECK-NEXT:    add x13, sp, #48
-; CHECK-NEXT:    ld1 { v5.b }[2], [x8]
-; CHECK-NEXT:    mov v0.b[3], w3
-; CHECK-NEXT:    add x10, sp, #112
-; CHECK-NEXT:    add x8, sp, #504
-; CHECK-NEXT:    ld1 { v2.b }[4], [x13]
-; CHECK-NEXT:    add x13, sp, #376
-; CHECK-NEXT:    ld1 { v1.b }[4], [x10]
-; CHECK-NEXT:    ld1 { v4.b }[3], [x13]
-; CHECK-NEXT:    add x13, sp, #440
-; CHECK-NEXT:    ld1 { v3.b }[3], [x8]
-; CHECK-NEXT:    ld1 { v5.b }[3], [x13]
-; CHECK-NEXT:    add x11, sp, #120
-; CHECK-NEXT:    add x8, sp, #56
-; CHECK-NEXT:    mov v0.b[4], w4
-; CHECK-NEXT:    add x13, sp, #512
-; CHECK-NEXT:    ld1 { v1.b }[5], [x11]
-; CHECK-NEXT:    ld1 { v2.b }[5], [x8]
-; CHECK-NEXT:    add x8, sp, #384
-; CHECK-NEXT:    add x11, sp, #448
-; CHECK-NEXT:    ld1 { v3.b }[4], [x13]
-; CHECK-NEXT:    ld1 { v4.b }[4], [x8]
-; CHECK-NEXT:    ld1 { v5.b }[4], [x11]
-; CHECK-NEXT:    add x12, sp, #128
-; CHECK-NEXT:    add x10, sp, #64
-; CHECK-NEXT:    add x8, sp, #520
-; CHECK-NEXT:    mov v0.b[5], w5
-; CHECK-NEXT:    ld1 { v1.b }[6], [x12]
-; CHECK-NEXT:    ld1 { v2.b }[6], [x10]
-; CHECK-NEXT:    add x10, sp, #392
-; CHECK-NEXT:    add x11, sp, #456
-; CHECK-NEXT:    ldr b6, [sp, #144]
-; CHECK-NEXT:    ldr b7, [sp, #544]
-; CHECK-NEXT:    ld1 { v3.b }[5], [x8]
-; CHECK-NEXT:    ld1 { v4.b }[5], [x10]
-; CHECK-NEXT:    ld1 { v5.b }[5], [x11]
-; CHECK-NEXT:    add x9, sp, #136
-; CHECK-NEXT:    sshll v6.8h, v6.8b, #0
-; CHECK-NEXT:    mov v0.b[6], w6
-; CHECK-NEXT:    ld1 { v1.b }[7], [x9]
-; CHECK-NEXT:    add x8, sp, #528
-; CHECK-NEXT:    add x9, sp, #400
-; CHECK-NEXT:    add x10, sp, #464
-; CHECK-NEXT:    sshll v7.8h, v7.8b, #0
-; CHECK-NEXT:    ld1 { v3.b }[6], [x8]
-; CHECK-NEXT:    ld1 { v4.b }[6], [x9]
-; CHECK-NEXT:    ld1 { v5.b }[6], [x10]
-; CHECK-NEXT:    movi v16.2d, #0000000000000000
-; CHECK-NEXT:    movi v17.2d, #0000000000000000
-; CHECK-NEXT:    add x14, sp, #72
-; CHECK-NEXT:    mov v0.b[7], w7
-; CHECK-NEXT:    sshll v6.4s, v6.4h, #0
-; CHECK-NEXT:    add x8, sp, #536
-; CHECK-NEXT:    add x9, sp, #408
-; CHECK-NEXT:    add x10, sp, #472
-; CHECK-NEXT:    sshll v7.4s, v7.4h, #0
-; CHECK-NEXT:    ld1 { v2.b }[7], [x14]
-; CHECK-NEXT:    ld1 { v3.b }[7], [x8]
-; CHECK-NEXT:    ld1 { v4.b }[7], [x9]
-; CHECK-NEXT:    ld1 { v5.b }[7], [x10]
-; CHECK-NEXT:    mov v16.s[0], v6.s[0]
-; CHECK-NEXT:    sshll v1.8h, v1.8b, #0
-; CHECK-NEXT:    mov v17.s[0], v7.s[0]
-; CHECK-NEXT:    sshll v0.8h, v0.8b, #0
-; CHECK-NEXT:    sshll v2.8h, v2.8b, #0
-; CHECK-NEXT:    sshll v3.8h, v3.8b, #0
-; CHECK-NEXT:    sshll v4.8h, v4.8b, #0
-; CHECK-NEXT:    sshll v5.8h, v5.8b, #0
-; CHECK-NEXT:    saddl v7.4s, v0.4h, v1.4h
-; CHECK-NEXT:    saddl2 v0.4s, v0.8h, v1.8h
-; CHECK-NEXT:    saddw v6.4s, v16.4s, v2.4h
-; CHECK-NEXT:    saddl v1.4s, v4.4h, v3.4h
-; CHECK-NEXT:    saddl2 v3.4s, v4.8h, v3.8h
-; CHECK-NEXT:    saddw v4.4s, v17.4s, v5.4h
-; CHECK-NEXT:    saddw2 v0.4s, v0.4s, v2.8h
-; CHECK-NEXT:    add v6.4s, v7.4s, v6.4s
-; CHECK-NEXT:    saddw2 v2.4s, v3.4s, v5.8h
-; CHECK-NEXT:    add v1.4s, v1.4s, v4.4s
-; CHECK-NEXT:    add v0.4s, v6.4s, v0.4s
-; CHECK-NEXT:    add v1.4s, v1.4s, v2.4s
-; CHECK-NEXT:    add v0.4s, v0.4s, v1.4s
-; CHECK-NEXT:    addv s0, v0.4s
-; CHECK-NEXT:    fmov w0, s0
-; CHECK-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
-; CHECK-NEXT:    ret
-entry:
-  %az = sext <25 x i8> %a to <25 x i32>
-  %r1 = call i32 @llvm.vector.reduce.add.v25i32(<25 x i32> %az)
-  %cz = sext <25 x i8> %c to <25 x i32>
-  %r2 = call i32 @llvm.vector.reduce.add.v25i32(<25 x i32> %cz)
-  %x = add i32 %r1, %r2
-  ret i32 %x
-}
-
-define i32 @test_udot_v32i8(ptr nocapture readonly %a, ptr nocapture readonly %b, i32 %sum) {
-; CHECK-SD-LABEL: test_udot_v32i8:
+; CHECK-SD-LABEL: test_sdot_v25i8:
 ; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    movi v0.2d, #0000000000000000
-; CHECK-SD-NEXT:    ldp q1, q3, [x0]
-; CHECK-SD-NEXT:    ldp q2, q4, [x1]
-; CHECK-SD-NEXT:    udot v0.4s, v4.16b, v3.16b
-; CHECK-SD-NEXT:    udot v0.4s, v2.16b, v1.16b
+; CHECK-SD-NEXT:    ldp q3, q0, [x1]
+; CHECK-SD-NEXT:    movi v5.2d, #0000000000000000
+; CHECK-SD-NEXT:    ldp q2, q1, [x0]
+; CHECK-SD-NEXT:    smull2 v4.8h, v0.16b, v1.16b
+; CHECK-SD-NEXT:    smull v0.8h, v0.8b, v1.8b
+; CHECK-SD-NEXT:    smull v1.8h, v3.8b, v2.8b
+; CHECK-SD-NEXT:    smull2 v2.8h, v3.16b, v2.16b
+; CHECK-SD-NEXT:    sshll v3.4s, v4.4h, #0
+; CHECK-SD-NEXT:    saddl2 v4.4s, v1.8h, v0.8h
+; CHECK-SD-NEXT:    saddl v0.4s, v1.4h, v0.4h
+; CHECK-SD-NEXT:    mov v5.s[0], v3.s[0]
+; CHECK-SD-NEXT:    saddw2 v1.4s, v4.4s, v2.8h
+; CHECK-SD-NEXT:    add v0.4s, v0.4s, v1.4s
+; CHECK-SD-NEXT:    saddw v2.4s, v5.4s, v2.4h
+; CHECK-SD-NEXT:    add v0.4s, v0.4s, v2.4s
 ; CHECK-SD-NEXT:    addv s0, v0.4s
 ; CHECK-SD-NEXT:    fmov w8, s0
 ; CHECK-SD-NEXT:    add w0, w8, w2
 ; CHECK-SD-NEXT:    ret
 ;
-; CHECK-GI-LABEL: test_udot_v32i8:
+; CHECK-GI-LABEL: test_sdot_v25i8:
 ; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    movi v0.2d, #0000000000000000
-; CHECK-GI-NEXT:    movi v1.2d, #0000000000000000
-; CHECK-GI-NEXT:    ldp q2, q3, [x0]
-; CHECK-GI-NEXT:    ldp q4, q5, [x1]
-; CHECK-GI-NEXT:    udot v1.4s, v4.16b, v2.16b
-; CHECK-GI-NEXT:    udot v0.4s, v5.16b, v3.16b
-; CHECK-GI-NEXT:    add v0.4s, v1.4s, v0.4s
+; CHECK-GI-NEXT:    ldr q1, [x1]
+; CHECK-GI-NEXT:    ldrsb w11, [x1, #16]!
+; CHECK-GI-NEXT:    ldrsb w12, [x1, #4]
+; CHECK-GI-NEXT:    ldr q0, [x0]
+; CHECK-GI-NEXT:    mov v23.s[0], wzr
+; CHECK-GI-NEXT:    smov w9, v1.b[4]
+; CHECK-GI-NEXT:    smov w10, v1.b[12]
+; CHECK-GI-NEXT:    smov w13, v1.b[0]
+; CHECK-GI-NEXT:    smov w14, v1.b[5]
+; CHECK-GI-NEXT:    mov v5.s[0], w12
+; CHECK-GI-NEXT:    mov v3.s[0], w11
+; CHECK-GI-NEXT:    smov w11, v0.b[0]
+; CHECK-GI-NEXT:    smov w12, v1.b[1]
+; CHECK-GI-NEXT:    smov w15, v1.b[8]
+; CHECK-GI-NEXT:    ldrsb w8, [x0, #16]!
+; CHECK-GI-NEXT:    mov v23.s[1], wzr
+; CHECK-GI-NEXT:    mov v2.s[0], w9
+; CHECK-GI-NEXT:    mov v4.s[0], w10
+; CHECK-GI-NEXT:    smov w10, v1.b[13]
+; CHECK-GI-NEXT:    ldrsb w9, [x1, #5]
+; CHECK-GI-NEXT:    mov v6.s[0], w13
+; CHECK-GI-NEXT:    smov w13, v1.b[6]
+; CHECK-GI-NEXT:    mov v16.s[0], w11
+; CHECK-GI-NEXT:    smov w11, v1.b[2]
+; CHECK-GI-NEXT:    mov v7.s[0], w15
+; CHECK-GI-NEXT:    mov v5.s[1], w9
+; CHECK-GI-NEXT:    ldrsb w9, [x1, #6]
+; CHECK-GI-NEXT:    smov w15, v1.b[9]
+; CHECK-GI-NEXT:    mov v2.s[1], w14
+; CHECK-GI-NEXT:    ldrsb w14, [x1, #1]
+; CHECK-GI-NEXT:    mov v4.s[1], w10
+; CHECK-GI-NEXT:    smov w10, v1.b[14]
+; CHECK-GI-NEXT:    mov v6.s[1], w12
+; CHECK-GI-NEXT:    smov w12, v0.b[1]
+; CHECK-GI-NEXT:    mov v3.s[1], w14
+; CHECK-GI-NEXT:    smov w14, v0.b[12]
+; CHECK-GI-NEXT:    mov v21.s[0], w8
+; CHECK-GI-NEXT:    ldrsb w8, [x0, #1]
+; CHECK-GI-NEXT:    mov v5.s[2], w9
+; CHECK-GI-NEXT:    smov w9, v0.b[4]
+; CHECK-GI-NEXT:    mov v2.s[2], w13
+; CHECK-GI-NEXT:    smov w13, v1.b[7]
+; CHECK-GI-NEXT:    mov v7.s[1], w15
+; CHECK-GI-NEXT:    mov v4.s[2], w10
+; CHECK-GI-NEXT:    smov w10, v1.b[15]
+; CHECK-GI-NEXT:    mov v16.s[1], w12
+; CHECK-GI-NEXT:    ldrsb w12, [x1, #2]
+; CHECK-GI-NEXT:    mov v6.s[2], w11
+; CHECK-GI-NEXT:    smov w11, v0.b[2]
+; CHECK-GI-NEXT:    mov v17.s[0], w9
+; CHECK-GI-NEXT:    smov w9, v0.b[8]
+; CHECK-GI-NEXT:    mov v18.s[0], w14
+; CHECK-GI-NEXT:    mov v2.s[3], w13
+; CHECK-GI-NEXT:    ldrsb w13, [x1, #7]
+; CHECK-GI-NEXT:    mov v3.s[2], w12
+; CHECK-GI-NEXT:    ldrsb w12, [x0, #4]
+; CHECK-GI-NEXT:    mov v4.s[3], w10
+; CHECK-GI-NEXT:    smov w10, v0.b[5]
+; CHECK-GI-NEXT:    mov v5.s[3], w13
+; CHECK-GI-NEXT:    ldrsb w13, [x0, #8]
+; CHECK-GI-NEXT:    mov v16.s[2], w11
+; CHECK-GI-NEXT:    smov w11, v0.b[13]
+; CHECK-GI-NEXT:    mov v20.s[0], w12
+; CHECK-GI-NEXT:    ldrsb w12, [x1, #8]
+; CHECK-GI-NEXT:    mov v19.s[0], w9
+; CHECK-GI-NEXT:    smov w9, v0.b[6]
+; CHECK-GI-NEXT:    smov w15, v1.b[10]
+; CHECK-GI-NEXT:    mul w12, w12, w13
+; CHECK-GI-NEXT:    mov v17.s[1], w10
+; CHECK-GI-NEXT:    ldrsb w10, [x0, #5]
+; CHECK-GI-NEXT:    smov w13, v0.b[9]
+; CHECK-GI-NEXT:    mov v21.s[1], w8
+; CHECK-GI-NEXT:    smov w8, v1.b[11]
+; CHECK-GI-NEXT:    mov v18.s[1], w11
+; CHECK-GI-NEXT:    smov w11, v0.b[14]
+; CHECK-GI-NEXT:    mov v20.s[1], w10
+; CHECK-GI-NEXT:    ldrsb w10, [x0, #6]
+; CHECK-GI-NEXT:    mov v22.s[0], w12
+; CHECK-GI-NEXT:    smov w12, v0.b[7]
+; CHECK-GI-NEXT:    mov v17.s[2], w9
+; CHECK-GI-NEXT:    smov w9, v0.b[10]
+; CHECK-GI-NEXT:    mov v7.s[2], w15
+; CHECK-GI-NEXT:    mov v19.s[1], w13
+; CHECK-GI-NEXT:    smov w13, v1.b[3]
+; CHECK-GI-NEXT:    mov v23.s[2], wzr
+; CHECK-GI-NEXT:    mov v18.s[2], w11
+; CHECK-GI-NEXT:    smov w11, v0.b[15]
+; CHECK-GI-NEXT:    mov v20.s[2], w10
+; CHECK-GI-NEXT:    ldrsb w10, [x0, #2]
+; CHECK-GI-NEXT:    mov v22.s[1], wzr
+; CHECK-GI-NEXT:    mov v17.s[3], w12
+; CHECK-GI-NEXT:    ldrsb w12, [x0, #7]
+; CHECK-GI-NEXT:    mov v7.s[3], w8
+; CHECK-GI-NEXT:    ldrsb w8, [x0, #3]
+; CHECK-GI-NEXT:    mov v19.s[2], w9
+; CHECK-GI-NEXT:    smov w9, v0.b[3]
+; CHECK-GI-NEXT:    mov v18.s[3], w11
+; CHECK-GI-NEXT:    smov w11, v0.b[11]
+; CHECK-GI-NEXT:    mov v21.s[2], w10
+; CHECK-GI-NEXT:    ldrsb w10, [x1, #3]
+; CHECK-GI-NEXT:    mov v20.s[3], w12
+; CHECK-GI-NEXT:    mov v22.s[2], wzr
+; CHECK-GI-NEXT:    mov v6.s[3], w13
+; CHECK-GI-NEXT:    mul v0.4s, v2.4s, v17.4s
+; CHECK-GI-NEXT:    mov v23.s[3], wzr
+; CHECK-GI-NEXT:    mov v3.s[3], w10
+; CHECK-GI-NEXT:    mov v16.s[3], w9
+; CHECK-GI-NEXT:    mov v19.s[3], w11
+; CHECK-GI-NEXT:    mul v1.4s, v4.4s, v18.4s
+; CHECK-GI-NEXT:    mov v21.s[3], w8
+; CHECK-GI-NEXT:    mul v2.4s, v5.4s, v20.4s
+; CHECK-GI-NEXT:    mov v22.s[3], wzr
+; CHECK-GI-NEXT:    mla v0.4s, v6.4s, v16.4s
+; CHECK-GI-NEXT:    mla v1.4s, v7.4s, v19.4s
+; CHECK-GI-NEXT:    mla v2.4s, v3.4s, v21.4s
+; CHECK-GI-NEXT:    add v3.4s, v22.4s, v23.4s
+; CHECK-GI-NEXT:    add v0.4s, v0.4s, v1.4s
+; CHECK-GI-NEXT:    add v1.4s, v2.4s, v3.4s
+; CHECK-GI-NEXT:    add v0.4s, v0.4s, v1.4s
 ; CHECK-GI-NEXT:    addv s0, v0.4s
 ; CHECK-GI-NEXT:    fmov w8, s0
 ; CHECK-GI-NEXT:    add w0, w8, w2
 ; CHECK-GI-NEXT:    ret
 entry:
-  %0 = load <32 x i8>, ptr %a
-  %1 = zext <32 x i8> %0 to <32 x i32>
-  %2 = load <32 x i8>, ptr %b
-  %3 = zext <32 x i8> %2 to <32 x i32>
-  %4 = mul nuw nsw <32 x i32> %3, %1
-  %5 = call i32 @llvm.vector.reduce.add.v32i32(<32 x i32> %4)
-  %op.extra = add i32 %5, %sum
+  %0 = load <25 x i8>, ptr %a
+  %1 = sext <25 x i8> %0 to <25 x i32>
+  %2 = load <25 x i8>, ptr %b
+  %3 = sext <25 x i8> %2 to <25 x i32>
+  %4 = mul nsw <25 x i32> %3, %1
+  %5 = call i32 @llvm.vector.reduce.add.v25i32(<25 x i32> %4)
+  %op.extra = add nsw i32 %5, %sum
   ret i32 %op.extra
 }
 
-define i32 @test_udot_v32i8_nomla(ptr nocapture readonly %a1) {
-; CHECK-SD-LABEL: test_udot_v32i8_nomla:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    movi v0.16b, #1
-; CHECK-SD-NEXT:    movi v1.2d, #0000000000000000
-; CHECK-SD-NEXT:    ldp q2, q3, [x0]
-; CHECK-SD-NEXT:    udot v1.4s, v3.16b, v0.16b
-; CHECK-SD-NEXT:    udot v1.4s, v2.16b, v0.16b
-; CHECK-SD-NEXT:    addv s0, v1.4s
-; CHECK-SD-NEXT:    fmov w0, s0
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: test_udot_v32i8_nomla:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    movi v0.16b, #1
-; CHECK-GI-NEXT:    movi v1.2d, #0000000000000000
-; CHECK-GI-NEXT:    movi v2.2d, #0000000000000000
-; CHECK-GI-NEXT:    ldp q3, q4, [x0]
-; CHECK-GI-NEXT:    udot v2.4s, v3.16b, v0.16b
-; CHECK-GI-NEXT:    udot v1.4s, v4.16b, v0.16b
-; CHECK-GI-NEXT:    add v0.4s, v2.4s, v1.4s
-; CHECK-GI-NEXT:    addv s0, v0.4s
-; CHECK-GI-NEXT:    fmov w0, s0
-; CHECK-GI-NEXT:    ret
-entry:
-  %0 = load <32 x i8>, ptr %a1
-  %1 = zext <32 x i8> %0 to <32 x i32>
-  %2 = call i32 @llvm.vector.reduce.add.v32i32(<32 x i32> %1)
-  ret i32 %2
-}
-define i32 @test_sdot_v32i8(ptr nocapture readonly %a, ptr nocapture readonly %b, i32 %sum) {
-; CHECK-SD-LABEL: test_sdot_v32i8:
+define i32 @test_sdot_v25i8_double(<25 x i8> %a, <25 x i8> %b, <25 x i8> %c, <25 x i8> %d) {
+; CHECK-SD-LABEL: test_sdot_v25i8_double:
 ; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    movi v0.2d, #0000000000000000
+; CHECK-SD-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-SD-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-SD-NEXT:    .cfi_offset w29, -16
+; CHECK-SD-NEXT:    ldr b0, [sp, #216]
+; CHECK-SD-NEXT:    add x8, sp, #224
+; CHECK-SD-NEXT:    ldr b1, [sp, #16]
+; CHECK-SD-NEXT:    ldr b2, [sp, #280]
+; CHECK-SD-NEXT:    add x9, sp, #240
+; CHECK-SD-NEXT:    ldr b4, [sp, #80]
+; CHECK-SD-NEXT:    ld1 { v0.b }[1], [x8]
+; CHECK-SD-NEXT:    add x8, sp, #24
+; CHECK-SD-NEXT:    add x10, sp, #48
+; CHECK-SD-NEXT:    ld1 { v1.b }[1], [x8]
+; CHECK-SD-NEXT:    add x8, sp, #232
+; CHECK-SD-NEXT:    add x11, sp, #96
+; CHECK-SD-NEXT:    ldr b5, [sp, #152]
+; CHECK-SD-NEXT:    add x12, sp, #168
+; CHECK-SD-NEXT:    ldr b6, [sp, #616]
+; CHECK-SD-NEXT:    ld1 { v0.b }[2], [x8]
+; CHECK-SD-NEXT:    add x8, sp, #32
+; CHECK-SD-NEXT:    fmov s3, w0
+; CHECK-SD-NEXT:    ld1 { v1.b }[2], [x8]
+; CHECK-SD-NEXT:    add x8, sp, #288
+; CHECK-SD-NEXT:    ldr b7, [sp, #416]
+; CHECK-SD-NEXT:    ld1 { v2.b }[1], [x8]
+; CHECK-SD-NEXT:    add x8, sp, #40
+; CHECK-SD-NEXT:    ldr b22, [sp, #744]
+; CHECK-SD-NEXT:    ld1 { v0.b }[3], [x9]
+; CHECK-SD-NEXT:    add x9, sp, #248
+; CHECK-SD-NEXT:    mov v3.b[1], w1
+; CHECK-SD-NEXT:    ld1 { v1.b }[3], [x8]
+; CHECK-SD-NEXT:    add x8, sp, #88
+; CHECK-SD-NEXT:    ldr b23, [sp, #544]
+; CHECK-SD-NEXT:    ld1 { v4.b }[1], [x8]
+; CHECK-SD-NEXT:    add x8, sp, #256
+; CHECK-SD-NEXT:    ldr b19, [sp, #680]
+; CHECK-SD-NEXT:    ld1 { v0.b }[4], [x9]
+; CHECK-SD-NEXT:    add x9, sp, #296
+; CHECK-SD-NEXT:    ldr b20, [sp, #480]
+; CHECK-SD-NEXT:    ld1 { v1.b }[4], [x10]
+; CHECK-SD-NEXT:    ld1 { v2.b }[2], [x9]
+; CHECK-SD-NEXT:    add x10, sp, #160
+; CHECK-SD-NEXT:    ld1 { v4.b }[2], [x11]
+; CHECK-SD-NEXT:    add x11, sp, #304
+; CHECK-SD-NEXT:    ld1 { v5.b }[1], [x10]
+; CHECK-SD-NEXT:    ld1 { v0.b }[5], [x8]
+; CHECK-SD-NEXT:    add x8, sp, #56
+; CHECK-SD-NEXT:    add x10, sp, #264
+; CHECK-SD-NEXT:    ld1 { v1.b }[5], [x8]
+; CHECK-SD-NEXT:    add x8, sp, #64
+; CHECK-SD-NEXT:    ld1 { v2.b }[3], [x11]
+; CHECK-SD-NEXT:    add x9, sp, #272
+; CHECK-SD-NEXT:    ld1 { v5.b }[2], [x12]
+; CHECK-SD-NEXT:    add x11, sp, #72
+; CHECK-SD-NEXT:    ld1 { v0.b }[6], [x10]
+; CHECK-SD-NEXT:    add x10, sp, #312
+; CHECK-SD-NEXT:    mov v3.b[2], w2
+; CHECK-SD-NEXT:    ld1 { v1.b }[6], [x8]
+; CHECK-SD-NEXT:    add x8, sp, #104
+; CHECK-SD-NEXT:    ld1 { v2.b }[4], [x10]
+; CHECK-SD-NEXT:    ld1 { v4.b }[3], [x8]
+; CHECK-SD-NEXT:    add x8, sp, #112
+; CHECK-SD-NEXT:    add x10, sp, #128
+; CHECK-SD-NEXT:    ld1 { v0.b }[7], [x9]
+; CHECK-SD-NEXT:    add x9, sp, #320
+; CHECK-SD-NEXT:    ldr b21, [sp, #552]
+; CHECK-SD-NEXT:    ld1 { v2.b }[5], [x9]
+; CHECK-SD-NEXT:    add x9, sp, #176
+; CHECK-SD-NEXT:    ld1 { v1.b }[7], [x11]
+; CHECK-SD-NEXT:    ld1 { v4.b }[4], [x8]
+; CHECK-SD-NEXT:    add x8, sp, #624
+; CHECK-SD-NEXT:    ld1 { v5.b }[3], [x9]
+; CHECK-SD-NEXT:    ld1 { v6.b }[1], [x8]
+; CHECK-SD-NEXT:    add x8, sp, #120
+; CHECK-SD-NEXT:    add x9, sp, #328
+; CHECK-SD-NEXT:    ld1 { v2.b }[6], [x9]
+; CHECK-SD-NEXT:    add x9, sp, #184
+; CHECK-SD-NEXT:    add x11, sp, #192
+; CHECK-SD-NEXT:    ld1 { v4.b }[5], [x8]
+; CHECK-SD-NEXT:    add x8, sp, #632
+; CHECK-SD-NEXT:    ld1 { v5.b }[4], [x9]
+; CHECK-SD-NEXT:    ld1 { v6.b }[2], [x8]
+; CHECK-SD-NEXT:    add x9, sp, #640
+; CHECK-SD-NEXT:    add x8, sp, #336
+; CHECK-SD-NEXT:    ld1 { v2.b }[7], [x8]
+; CHECK-SD-NEXT:    add x8, sp, #656
+; CHECK-SD-NEXT:    smull v23.8h, v23.8b, v22.8b
+; CHECK-SD-NEXT:    ld1 { v5.b }[5], [x11]
+; CHECK-SD-NEXT:    add x11, sp, #648
+; CHECK-SD-NEXT:    ld1 { v4.b }[6], [x10]
+; CHECK-SD-NEXT:    ld1 { v6.b }[3], [x9]
+; CHECK-SD-NEXT:    add x9, sp, #200
+; CHECK-SD-NEXT:    add x10, sp, #136
+; CHECK-SD-NEXT:    ldr b22, [sp, #352]
+; CHECK-SD-NEXT:    add x12, sp, #360
+; CHECK-SD-NEXT:    mov v3.b[3], w3
+; CHECK-SD-NEXT:    ld1 { v5.b }[6], [x9]
+; CHECK-SD-NEXT:    add x9, sp, #208
+; CHECK-SD-NEXT:    ld1 { v4.b }[7], [x10]
+; CHECK-SD-NEXT:    ld1 { v6.b }[4], [x11]
+; CHECK-SD-NEXT:    add x11, sp, #424
+; CHECK-SD-NEXT:    add x10, sp, #488
+; CHECK-SD-NEXT:    ld1 { v7.b }[1], [x11]
+; CHECK-SD-NEXT:    add x11, sp, #560
+; CHECK-SD-NEXT:    ld1 { v20.b }[1], [x10]
+; CHECK-SD-NEXT:    ld1 { v5.b }[7], [x9]
+; CHECK-SD-NEXT:    add x9, sp, #440
+; CHECK-SD-NEXT:    ld1 { v21.b }[1], [x11]
+; CHECK-SD-NEXT:    ld1 { v6.b }[5], [x8]
+; CHECK-SD-NEXT:    add x8, sp, #432
+; CHECK-SD-NEXT:    ld1 { v22.b }[1], [x12]
+; CHECK-SD-NEXT:    ld1 { v7.b }[2], [x8]
+; CHECK-SD-NEXT:    add x11, sp, #496
+; CHECK-SD-NEXT:    add x12, sp, #568
+; CHECK-SD-NEXT:    add x13, sp, #368
+; CHECK-SD-NEXT:    ld1 { v20.b }[2], [x11]
+; CHECK-SD-NEXT:    ld1 { v21.b }[2], [x12]
+; CHECK-SD-NEXT:    ld1 { v22.b }[2], [x13]
+; CHECK-SD-NEXT:    add x10, sp, #448
+; CHECK-SD-NEXT:    mov v3.b[4], w4
+; CHECK-SD-NEXT:    ld1 { v7.b }[3], [x9]
+; CHECK-SD-NEXT:    add x9, sp, #688
+; CHECK-SD-NEXT:    add x11, sp, #576
+; CHECK-SD-NEXT:    ld1 { v19.b }[1], [x9]
+; CHECK-SD-NEXT:    add x9, sp, #696
+; CHECK-SD-NEXT:    add x12, sp, #376
+; CHECK-SD-NEXT:    ld1 { v21.b }[3], [x11]
+; CHECK-SD-NEXT:    ld1 { v22.b }[3], [x12]
+; CHECK-SD-NEXT:    add x11, sp, #512
+; CHECK-SD-NEXT:    ld1 { v7.b }[4], [x10]
+; CHECK-SD-NEXT:    add x10, sp, #504
+; CHECK-SD-NEXT:    add x12, sp, #584
+; CHECK-SD-NEXT:    ld1 { v19.b }[2], [x9]
+; CHECK-SD-NEXT:    add x9, sp, #704
+; CHECK-SD-NEXT:    ld1 { v20.b }[3], [x10]
+; CHECK-SD-NEXT:    add x13, sp, #384
+; CHECK-SD-NEXT:    mov v3.b[5], w5
+; CHECK-SD-NEXT:    ld1 { v21.b }[4], [x12]
+; CHECK-SD-NEXT:    ld1 { v22.b }[4], [x13]
+; CHECK-SD-NEXT:    add x10, sp, #456
+; CHECK-SD-NEXT:    ldr b16, [sp, #344]
+; CHECK-SD-NEXT:    ld1 { v19.b }[3], [x9]
+; CHECK-SD-NEXT:    add x9, sp, #712
+; CHECK-SD-NEXT:    ld1 { v20.b }[4], [x11]
+; CHECK-SD-NEXT:    ldr b17, [sp, #144]
+; CHECK-SD-NEXT:    ld1 { v7.b }[5], [x10]
+; CHECK-SD-NEXT:    add x10, sp, #520
+; CHECK-SD-NEXT:    add x11, sp, #592
+; CHECK-SD-NEXT:    add x12, sp, #392
+; CHECK-SD-NEXT:    mov v3.b[6], w6
+; CHECK-SD-NEXT:    ld1 { v19.b }[4], [x9]
+; CHECK-SD-NEXT:    add x9, sp, #720
+; CHECK-SD-NEXT:    ld1 { v20.b }[5], [x10]
+; CHECK-SD-NEXT:    ld1 { v21.b }[5], [x11]
+; CHECK-SD-NEXT:    ld1 { v22.b }[5], [x12]
+; CHECK-SD-NEXT:    smull v16.8h, v17.8b, v16.8b
+; CHECK-SD-NEXT:    add x8, sp, #664
+; CHECK-SD-NEXT:    add x10, sp, #464
+; CHECK-SD-NEXT:    add x11, sp, #528
+; CHECK-SD-NEXT:    ld1 { v19.b }[5], [x9]
+; CHECK-SD-NEXT:    add x9, sp, #728
+; CHECK-SD-NEXT:    add x12, sp, #600
+; CHECK-SD-NEXT:    add x13, sp, #400
+; CHECK-SD-NEXT:    ld1 { v6.b }[6], [x8]
+; CHECK-SD-NEXT:    ld1 { v20.b }[6], [x11]
+; CHECK-SD-NEXT:    ld1 { v21.b }[6], [x12]
+; CHECK-SD-NEXT:    ld1 { v22.b }[6], [x13]
+; CHECK-SD-NEXT:    ld1 { v7.b }[6], [x10]
+; CHECK-SD-NEXT:    ld1 { v19.b }[6], [x9]
+; CHECK-SD-NEXT:    add x9, sp, #736
+; CHECK-SD-NEXT:    mov v3.b[7], w7
+; CHECK-SD-NEXT:    sshll v18.4s, v16.4h, #0
+; CHECK-SD-NEXT:    movi v16.2d, #0000000000000000
+; CHECK-SD-NEXT:    movi v17.2d, #0000000000000000
+; CHECK-SD-NEXT:    add x8, sp, #672
+; CHECK-SD-NEXT:    add x10, sp, #472
+; CHECK-SD-NEXT:    add x11, sp, #608
+; CHECK-SD-NEXT:    ld1 { v19.b }[7], [x9]
+; CHECK-SD-NEXT:    add x9, sp, #536
+; CHECK-SD-NEXT:    add x12, sp, #408
+; CHECK-SD-NEXT:    ld1 { v20.b }[7], [x9]
+; CHECK-SD-NEXT:    ld1 { v21.b }[7], [x11]
+; CHECK-SD-NEXT:    ld1 { v22.b }[7], [x12]
+; CHECK-SD-NEXT:    ld1 { v6.b }[7], [x8]
+; CHECK-SD-NEXT:    ld1 { v7.b }[7], [x10]
+; CHECK-SD-NEXT:    sshll v23.4s, v23.4h, #0
+; CHECK-SD-NEXT:    smull v0.8h, v1.8b, v0.8b
+; CHECK-SD-NEXT:    smull v1.8h, v4.8b, v2.8b
+; CHECK-SD-NEXT:    smull v2.8h, v3.8b, v5.8b
+; CHECK-SD-NEXT:    smull v3.8h, v20.8b, v19.8b
+; CHECK-SD-NEXT:    smull v4.8h, v22.8b, v21.8b
+; CHECK-SD-NEXT:    mov v17.s[0], v18.s[0]
+; CHECK-SD-NEXT:    smull v5.8h, v7.8b, v6.8b
+; CHECK-SD-NEXT:    mov v16.s[0], v23.s[0]
+; CHECK-SD-NEXT:    saddl2 v6.4s, v2.8h, v1.8h
+; CHECK-SD-NEXT:    saddl v1.4s, v2.4h, v1.4h
+; CHECK-SD-NEXT:    saddl2 v2.4s, v4.8h, v3.8h
+; CHECK-SD-NEXT:    saddl v3.4s, v4.4h, v3.4h
+; CHECK-SD-NEXT:    saddw v4.4s, v17.4s, v0.4h
+; CHECK-SD-NEXT:    saddw v7.4s, v16.4s, v5.4h
+; CHECK-SD-NEXT:    saddw2 v0.4s, v6.4s, v0.8h
+; CHECK-SD-NEXT:    add v1.4s, v1.4s, v4.4s
+; CHECK-SD-NEXT:    saddw2 v2.4s, v2.4s, v5.8h
+; CHECK-SD-NEXT:    add v3.4s, v3.4s, v7.4s
+; CHECK-SD-NEXT:    add v0.4s, v1.4s, v0.4s
+; CHECK-SD-NEXT:    add v1.4s, v3.4s, v2.4s
+; CHECK-SD-NEXT:    add v0.4s, v0.4s, v1.4s
+; CHECK-SD-NEXT:    addv s0, v0.4s
+; CHECK-SD-NEXT:    fmov w0, s0
+; CHECK-SD-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: test_sdot_v25i8_double:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    str d10, [sp, #-32]! // 8-byte Folded Spill
+; CHECK-GI-NEXT:    stp d9, d8, [sp, #8] // 16-byte Folded Spill
+; CHECK-GI-NEXT:    str x29, [sp, #24] // 8-byte Folded Spill
+; CHECK-GI-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-GI-NEXT:    .cfi_offset w29, -8
+; CHECK-GI-NEXT:    .cfi_offset b8, -16
+; CHECK-GI-NEXT:    .cfi_offset b9, -24
+; CHECK-GI-NEXT:    .cfi_offset b10, -32
+; CHECK-GI-NEXT:    ldr w8, [sp, #32]
+; CHECK-GI-NEXT:    sxtb w9, w0
+; CHECK-GI-NEXT:    sxtb w10, w4
+; CHECK-GI-NEXT:    sxtb w11, w5
+; CHECK-GI-NEXT:    sxtb w12, w3
+; CHECK-GI-NEXT:    sxtb w13, w7
+; CHECK-GI-NEXT:    mov v0.s[0], w9
+; CHECK-GI-NEXT:    sxtb w8, w8
+; CHECK-GI-NEXT:    mov v2.s[0], w10
+; CHECK-GI-NEXT:    ldr w9, [sp, #40]
+; CHECK-GI-NEXT:    sxtb w10, w1
+; CHECK-GI-NEXT:    ldr w14, [sp, #152]
+; CHECK-GI-NEXT:    mov v1.s[0], w8
+; CHECK-GI-NEXT:    ldr w8, [sp, #64]
+; CHECK-GI-NEXT:    mov v9.s[0], wzr
+; CHECK-GI-NEXT:    sxtb w9, w9
+; CHECK-GI-NEXT:    ldr x29, [sp, #24] // 8-byte Folded Reload
+; CHECK-GI-NEXT:    mov v0.s[1], w10
+; CHECK-GI-NEXT:    mov v2.s[1], w11
+; CHECK-GI-NEXT:    sxtb w8, w8
+; CHECK-GI-NEXT:    sxtb w10, w2
+; CHECK-GI-NEXT:    sxtb w11, w6
+; CHECK-GI-NEXT:    mov v1.s[1], w9
+; CHECK-GI-NEXT:    ldr w9, [sp, #48]
+; CHECK-GI-NEXT:    mov v3.s[0], w8
+; CHECK-GI-NEXT:    ldr w8, [sp, #72]
+; CHECK-GI-NEXT:    mov v9.s[1], wzr
+; CHECK-GI-NEXT:    sxtb w9, w9
+; CHECK-GI-NEXT:    mov v0.s[2], w10
+; CHECK-GI-NEXT:    ldr w10, [sp, #56]
+; CHECK-GI-NEXT:    sxtb w8, w8
+; CHECK-GI-NEXT:    mov v2.s[2], w11
+; CHECK-GI-NEXT:    ldr w11, [sp, #80]
+; CHECK-GI-NEXT:    mov v1.s[2], w9
+; CHECK-GI-NEXT:    ldr w9, [sp, #96]
+; CHECK-GI-NEXT:    sxtb w10, w10
+; CHECK-GI-NEXT:    mov v3.s[1], w8
+; CHECK-GI-NEXT:    sxtb w11, w11
+; CHECK-GI-NEXT:    ldr w8, [sp, #88]
+; CHECK-GI-NEXT:    mov v0.s[3], w12
+; CHECK-GI-NEXT:    ldr w12, [sp, #128]
+; CHECK-GI-NEXT:    sxtb w9, w9
+; CHECK-GI-NEXT:    mov v2.s[3], w13
+; CHECK-GI-NEXT:    ldr w13, [sp, #168]
+; CHECK-GI-NEXT:    sxtb w8, w8
+; CHECK-GI-NEXT:    mov v1.s[3], w10
+; CHECK-GI-NEXT:    ldr w10, [sp, #104]
+; CHECK-GI-NEXT:    sxtb w12, w12
+; CHECK-GI-NEXT:    mov v4.s[0], w9
+; CHECK-GI-NEXT:    mov v3.s[2], w11
+; CHECK-GI-NEXT:    sxtb w11, w13
+; CHECK-GI-NEXT:    ldr w13, [sp, #136]
+; CHECK-GI-NEXT:    sxtb w10, w10
+; CHECK-GI-NEXT:    mov v5.s[0], w12
+; CHECK-GI-NEXT:    ldr w9, [sp, #112]
+; CHECK-GI-NEXT:    ldr w12, [sp, #176]
+; CHECK-GI-NEXT:    mov v6.s[0], w11
+; CHECK-GI-NEXT:    sxtb w13, w13
+; CHECK-GI-NEXT:    ldr w11, [sp, #120]
+; CHECK-GI-NEXT:    mov v9.s[2], wzr
+; CHECK-GI-NEXT:    mov v4.s[1], w10
+; CHECK-GI-NEXT:    ldr w10, [sp, #200]
+; CHECK-GI-NEXT:    mov v3.s[3], w8
+; CHECK-GI-NEXT:    sxtb w8, w12
+; CHECK-GI-NEXT:    ldr w12, [sp, #144]
+; CHECK-GI-NEXT:    sxtb w9, w9
+; CHECK-GI-NEXT:    mov v5.s[1], w13
+; CHECK-GI-NEXT:    sxtb w10, w10
+; CHECK-GI-NEXT:    ldr w13, [sp, #184]
+; CHECK-GI-NEXT:    mov v6.s[1], w8
+; CHECK-GI-NEXT:    sxtb w12, w12
+; CHECK-GI-NEXT:    ldr w8, [sp, #160]
+; CHECK-GI-NEXT:    mov v4.s[2], w9
+; CHECK-GI-NEXT:    ldr w9, [sp, #208]
+; CHECK-GI-NEXT:    mov v7.s[0], w10
+; CHECK-GI-NEXT:    sxtb w13, w13
+; CHECK-GI-NEXT:    sxtb w10, w11
+; CHECK-GI-NEXT:    ldr w11, [sp, #192]
+; CHECK-GI-NEXT:    mov v5.s[2], w12
+; CHECK-GI-NEXT:    sxtb w9, w9
+; CHECK-GI-NEXT:    ldr w12, [sp, #232]
+; CHECK-GI-NEXT:    mov v6.s[2], w13
+; CHECK-GI-NEXT:    sxtb w13, w14
+; CHECK-GI-NEXT:    sxtb w11, w11
+; CHECK-GI-NEXT:    mov v4.s[3], w10
+; CHECK-GI-NEXT:    ldr w10, [sp, #216]
+; CHECK-GI-NEXT:    mov v7.s[1], w9
+; CHECK-GI-NEXT:    ldr w9, [sp, #264]
+; CHECK-GI-NEXT:    sxtb w12, w12
+; CHECK-GI-NEXT:    ldr w14, [sp, #368]
+; CHECK-GI-NEXT:    mov v5.s[3], w13
+; CHECK-GI-NEXT:    ldr w13, [sp, #296]
+; CHECK-GI-NEXT:    sxtb w10, w10
+; CHECK-GI-NEXT:    sxtb w9, w9
+; CHECK-GI-NEXT:    mov v6.s[3], w11
+; CHECK-GI-NEXT:    ldr w11, [sp, #240]
+; CHECK-GI-NEXT:    mov v16.s[0], w12
+; CHECK-GI-NEXT:    ldr w12, [sp, #224]
+; CHECK-GI-NEXT:    sxtb w13, w13
+; CHECK-GI-NEXT:    mov v7.s[2], w10
+; CHECK-GI-NEXT:    ldr w10, [sp, #272]
+; CHECK-GI-NEXT:    mov v18.s[0], w9
+; CHECK-GI-NEXT:    sxtb w11, w11
+; CHECK-GI-NEXT:    ldr w9, [sp, #304]
+; CHECK-GI-NEXT:    sxtb w12, w12
+; CHECK-GI-NEXT:    mov v17.s[0], w13
+; CHECK-GI-NEXT:    sxtb w10, w10
+; CHECK-GI-NEXT:    ldr w13, [sp, #248]
+; CHECK-GI-NEXT:    mov v16.s[1], w11
+; CHECK-GI-NEXT:    ldr w11, [sp, #328]
+; CHECK-GI-NEXT:    sxtb w9, w9
+; CHECK-GI-NEXT:    mov v7.s[3], w12
+; CHECK-GI-NEXT:    ldr w12, [sp, #280]
+; CHECK-GI-NEXT:    mov v18.s[1], w10
+; CHECK-GI-NEXT:    sxtb w13, w13
+; CHECK-GI-NEXT:    ldr w10, [sp, #312]
+; CHECK-GI-NEXT:    sxtb w11, w11
+; CHECK-GI-NEXT:    mov v17.s[1], w9
+; CHECK-GI-NEXT:    sxtb w12, w12
+; CHECK-GI-NEXT:    sxtb w8, w8
+; CHECK-GI-NEXT:    mov v16.s[2], w13
+; CHECK-GI-NEXT:    ldr w13, [sp, #336]
+; CHECK-GI-NEXT:    sxtb w10, w10
+; CHECK-GI-NEXT:    mov v19.s[0], w11
+; CHECK-GI-NEXT:    ldr w11, [sp, #288]
+; CHECK-GI-NEXT:    mov v18.s[2], w12
+; CHECK-GI-NEXT:    ldr w12, [sp, #320]
+; CHECK-GI-NEXT:    sxtb w13, w13
+; CHECK-GI-NEXT:    ldr w9, [sp, #256]
+; CHECK-GI-NEXT:    mov v17.s[2], w10
+; CHECK-GI-NEXT:    sxtb w11, w11
+; CHECK-GI-NEXT:    ldr w10, [sp, #344]
+; CHECK-GI-NEXT:    sxtb w12, w12
+; CHECK-GI-NEXT:    sxtb w9, w9
+; CHECK-GI-NEXT:    mul v2.4s, v2.4s, v7.4s
+; CHECK-GI-NEXT:    mov v19.s[1], w13
+; CHECK-GI-NEXT:    mov v18.s[3], w11
+; CHECK-GI-NEXT:    ldr w11, [sp, #360]
+; CHECK-GI-NEXT:    sxtb w10, w10
+; CHECK-GI-NEXT:    ldr w13, [sp, #400]
+; CHECK-GI-NEXT:    mov v16.s[3], w9
+; CHECK-GI-NEXT:    mov v17.s[3], w12
+; CHECK-GI-NEXT:    sxtb w12, w14
+; CHECK-GI-NEXT:    sxtb w11, w11
+; CHECK-GI-NEXT:    ldr w9, [sp, #352]
+; CHECK-GI-NEXT:    mov v9.s[3], wzr
+; CHECK-GI-NEXT:    mla v2.4s, v0.4s, v6.4s
+; CHECK-GI-NEXT:    mov v19.s[2], w10
+; CHECK-GI-NEXT:    ldr w10, [sp, #376]
+; CHECK-GI-NEXT:    mov v20.s[0], w12
+; CHECK-GI-NEXT:    ldr w12, [sp, #432]
+; CHECK-GI-NEXT:    mul w8, w8, w11
+; CHECK-GI-NEXT:    sxtb w11, w13
+; CHECK-GI-NEXT:    sxtb w10, w10
+; CHECK-GI-NEXT:    ldr w13, [sp, #384]
+; CHECK-GI-NEXT:    sxtb w9, w9
+; CHECK-GI-NEXT:    mov v23.s[0], w11
+; CHECK-GI-NEXT:    ldr w11, [sp, #408]
+; CHECK-GI-NEXT:    sxtb w12, w12
+; CHECK-GI-NEXT:    mov v20.s[1], w10
+; CHECK-GI-NEXT:    ldr w10, [sp, #440]
+; CHECK-GI-NEXT:    mov v21.s[0], w8
+; CHECK-GI-NEXT:    sxtb w11, w11
+; CHECK-GI-NEXT:    mov v22.s[0], w12
+; CHECK-GI-NEXT:    ldr w12, [sp, #464]
+; CHECK-GI-NEXT:    sxtb w8, w13
+; CHECK-GI-NEXT:    ldr w13, [sp, #416]
+; CHECK-GI-NEXT:    sxtb w10, w10
+; CHECK-GI-NEXT:    mov v23.s[1], w11
+; CHECK-GI-NEXT:    sxtb w12, w12
+; CHECK-GI-NEXT:    mov v19.s[3], w9
+; CHECK-GI-NEXT:    ldr w9, [sp, #392]
+; CHECK-GI-NEXT:    mov v20.s[2], w8
+; CHECK-GI-NEXT:    sxtb w13, w13
+; CHECK-GI-NEXT:    mov v22.s[1], w10
+; CHECK-GI-NEXT:    ldr w10, [sp, #472]
+; CHECK-GI-NEXT:    mov v24.s[0], w12
+; CHECK-GI-NEXT:    ldr w11, [sp, #424]
+; CHECK-GI-NEXT:    ldr w8, [sp, #448]
+; CHECK-GI-NEXT:    sxtb w9, w9
+; CHECK-GI-NEXT:    mov v23.s[2], w13
+; CHECK-GI-NEXT:    ldr w13, [sp, #496]
+; CHECK-GI-NEXT:    sxtb w10, w10
+; CHECK-GI-NEXT:    sxtb w8, w8
+; CHECK-GI-NEXT:    mov v20.s[3], w9
+; CHECK-GI-NEXT:    ldr w9, [sp, #480]
+; CHECK-GI-NEXT:    sxtb w11, w11
+; CHECK-GI-NEXT:    sxtb w13, w13
+; CHECK-GI-NEXT:    mov v24.s[1], w10
+; CHECK-GI-NEXT:    ldr w10, [sp, #528]
+; CHECK-GI-NEXT:    ldr w12, [sp, #456]
+; CHECK-GI-NEXT:    mov v22.s[2], w8
+; CHECK-GI-NEXT:    mov v23.s[3], w11
+; CHECK-GI-NEXT:    ldr w11, [sp, #504]
+; CHECK-GI-NEXT:    sxtb w9, w9
+; CHECK-GI-NEXT:    mov v25.s[0], w13
+; CHECK-GI-NEXT:    sxtb w10, w10
+; CHECK-GI-NEXT:    ldr w8, [sp, #488]
+; CHECK-GI-NEXT:    sxtb w12, w12
+; CHECK-GI-NEXT:    sxtb w11, w11
+; CHECK-GI-NEXT:    mov v24.s[2], w9
+; CHECK-GI-NEXT:    ldr w9, [sp, #536]
+; CHECK-GI-NEXT:    mov v26.s[0], w10
+; CHECK-GI-NEXT:    ldr w13, [sp, #568]
+; CHECK-GI-NEXT:    mov v22.s[3], w12
+; CHECK-GI-NEXT:    ldr w12, [sp, #512]
+; CHECK-GI-NEXT:    sxtb w8, w8
+; CHECK-GI-NEXT:    mov v25.s[1], w11
+; CHECK-GI-NEXT:    sxtb w9, w9
+; CHECK-GI-NEXT:    sxtb w10, w13
+; CHECK-GI-NEXT:    sxtb w12, w12
+; CHECK-GI-NEXT:    mov v24.s[3], w8
+; CHECK-GI-NEXT:    ldr w8, [sp, #544]
+; CHECK-GI-NEXT:    mov v26.s[1], w9
+; CHECK-GI-NEXT:    ldr w13, [sp, #520]
+; CHECK-GI-NEXT:    ldr w11, [sp, #576]
+; CHECK-GI-NEXT:    mov v27.s[0], w10
+; CHECK-GI-NEXT:    ldr w10, [sp, #600]
+; CHECK-GI-NEXT:    sxtb w8, w8
+; CHECK-GI-NEXT:    mov v25.s[2], w12
+; CHECK-GI-NEXT:    ldr w12, [sp, #584]
+; CHECK-GI-NEXT:    sxtb w9, w11
+; CHECK-GI-NEXT:    ldr w11, [sp, #552]
+; CHECK-GI-NEXT:    sxtb w10, w10
+; CHECK-GI-NEXT:    sxtb w13, w13
+; CHECK-GI-NEXT:    mov v26.s[2], w8
+; CHECK-GI-NEXT:    sxtb w8, w12
+; CHECK-GI-NEXT:    ldr w12, [sp, #664]
+; CHECK-GI-NEXT:    mov v27.s[1], w9
+; CHECK-GI-NEXT:    ldr w9, [sp, #608]
+; CHECK-GI-NEXT:    mov v28.s[0], w10
+; CHECK-GI-NEXT:    mov v25.s[3], w13
+; CHECK-GI-NEXT:    ldr w13, [sp, #592]
+; CHECK-GI-NEXT:    sxtb w11, w11
+; CHECK-GI-NEXT:    sxtb w12, w12
+; CHECK-GI-NEXT:    sxtb w9, w9
+; CHECK-GI-NEXT:    ldr w10, [sp, #560]
+; CHECK-GI-NEXT:    mov v26.s[3], w11
+; CHECK-GI-NEXT:    sxtb w11, w13
+; CHECK-GI-NEXT:    ldr w13, [sp, #672]
+; CHECK-GI-NEXT:    mov v30.s[0], w12
+; CHECK-GI-NEXT:    mov v27.s[2], w8
+; CHECK-GI-NEXT:    ldr w8, [sp, #616]
+; CHECK-GI-NEXT:    mov v28.s[1], w9
+; CHECK-GI-NEXT:    ldr w9, [sp, #632]
+; CHECK-GI-NEXT:    ldr w12, [sp, #728]
+; CHECK-GI-NEXT:    sxtb w13, w13
+; CHECK-GI-NEXT:    sxtb w8, w8
+; CHECK-GI-NEXT:    sxtb w10, w10
+; CHECK-GI-NEXT:    sxtb w9, w9
+; CHECK-GI-NEXT:    sxtb w12, w12
+; CHECK-GI-NEXT:    mov v21.s[1], wzr
+; CHECK-GI-NEXT:    mov v30.s[1], w13
+; CHECK-GI-NEXT:    ldr w13, [sp, #760]
+; CHECK-GI-NEXT:    mov v27.s[3], w11
+; CHECK-GI-NEXT:    mov v28.s[2], w8
+; CHECK-GI-NEXT:    ldr w8, [sp, #696]
+; CHECK-GI-NEXT:    mov v29.s[0], w9
+; CHECK-GI-NEXT:    ldr w9, [sp, #736]
+; CHECK-GI-NEXT:    mov v8.s[0], w12
+; CHECK-GI-NEXT:    sxtb w13, w13
+; CHECK-GI-NEXT:    ldr w11, [sp, #640]
+; CHECK-GI-NEXT:    sxtb w8, w8
+; CHECK-GI-NEXT:    ldr w12, [sp, #680]
+; CHECK-GI-NEXT:    sxtb w9, w9
+; CHECK-GI-NEXT:    mul w10, w10, w13
+; CHECK-GI-NEXT:    mov v21.s[2], wzr
+; CHECK-GI-NEXT:    sxtb w11, w11
+; CHECK-GI-NEXT:    mov v31.s[0], w8
+; CHECK-GI-NEXT:    ldr w8, [sp, #704]
+; CHECK-GI-NEXT:    mov v8.s[1], w9
+; CHECK-GI-NEXT:    ldr w9, [sp, #744]
+; CHECK-GI-NEXT:    mul v3.4s, v3.4s, v18.4s
+; CHECK-GI-NEXT:    mov v10.s[0], w10
+; CHECK-GI-NEXT:    mov v29.s[1], w11
+; CHECK-GI-NEXT:    sxtb w11, w12
+; CHECK-GI-NEXT:    sxtb w8, w8
+; CHECK-GI-NEXT:    sxtb w9, w9
+; CHECK-GI-NEXT:    ldr w12, [sp, #624]
+; CHECK-GI-NEXT:    mov v30.s[2], w11
+; CHECK-GI-NEXT:    ldr w11, [sp, #648]
+; CHECK-GI-NEXT:    ldr w10, [sp, #688]
+; CHECK-GI-NEXT:    mov v31.s[1], w8
+; CHECK-GI-NEXT:    ldr w8, [sp, #712]
+; CHECK-GI-NEXT:    mov v8.s[2], w9
+; CHECK-GI-NEXT:    ldr w9, [sp, #752]
+; CHECK-GI-NEXT:    mov v10.s[1], wzr
+; CHECK-GI-NEXT:    sxtb w12, w12
+; CHECK-GI-NEXT:    sxtb w11, w11
+; CHECK-GI-NEXT:    sxtb w10, w10
+; CHECK-GI-NEXT:    sxtb w8, w8
+; CHECK-GI-NEXT:    sxtb w9, w9
+; CHECK-GI-NEXT:    mov v28.s[3], w12
+; CHECK-GI-NEXT:    mul v5.4s, v5.4s, v19.4s
+; CHECK-GI-NEXT:    mov v29.s[2], w11
+; CHECK-GI-NEXT:    mov v30.s[3], w10
+; CHECK-GI-NEXT:    mov v31.s[2], w8
+; CHECK-GI-NEXT:    ldr w8, [sp, #656]
+; CHECK-GI-NEXT:    mov v8.s[3], w9
+; CHECK-GI-NEXT:    ldr w9, [sp, #720]
+; CHECK-GI-NEXT:    mov v10.s[2], wzr
+; CHECK-GI-NEXT:    mov v21.s[3], wzr
+; CHECK-GI-NEXT:    mla v3.4s, v1.4s, v16.4s
+; CHECK-GI-NEXT:    sxtb w8, w8
+; CHECK-GI-NEXT:    sxtb w9, w9
+; CHECK-GI-NEXT:    mul v7.4s, v23.4s, v28.4s
+; CHECK-GI-NEXT:    mul v18.4s, v24.4s, v30.4s
+; CHECK-GI-NEXT:    mla v5.4s, v4.4s, v17.4s
+; CHECK-GI-NEXT:    mul v19.4s, v26.4s, v8.4s
+; CHECK-GI-NEXT:    mov v29.s[3], w8
+; CHECK-GI-NEXT:    mov v31.s[3], w9
+; CHECK-GI-NEXT:    mov v10.s[3], wzr
+; CHECK-GI-NEXT:    add v0.4s, v21.4s, v9.4s
+; CHECK-GI-NEXT:    add v2.4s, v2.4s, v3.4s
+; CHECK-GI-NEXT:    mla v7.4s, v20.4s, v27.4s
+; CHECK-GI-NEXT:    mla v18.4s, v22.4s, v29.4s
+; CHECK-GI-NEXT:    mla v19.4s, v25.4s, v31.4s
+; CHECK-GI-NEXT:    add v0.4s, v5.4s, v0.4s
+; CHECK-GI-NEXT:    add v1.4s, v10.4s, v9.4s
+; CHECK-GI-NEXT:    ldp d9, d8, [sp, #8] // 16-byte Folded Reload
+; CHECK-GI-NEXT:    add v0.4s, v2.4s, v0.4s
+; CHECK-GI-NEXT:    add v3.4s, v7.4s, v18.4s
+; CHECK-GI-NEXT:    add v1.4s, v19.4s, v1.4s
+; CHECK-GI-NEXT:    addv s0, v0.4s
+; CHECK-GI-NEXT:    add v1.4s, v3.4s, v1.4s
+; CHECK-GI-NEXT:    fmov w8, s0
+; CHECK-GI-NEXT:    addv s1, v1.4s
+; CHECK-GI-NEXT:    fmov w9, s1
+; CHECK-GI-NEXT:    add w0, w8, w9
+; CHECK-GI-NEXT:    ldr d10, [sp], #32 // 8-byte Folded Reload
+; CHECK-GI-NEXT:    ret
+entry:
+  %az = sext <25 x i8> %a to <25 x i32>
+  %bz = sext <25 x i8> %b to <25 x i32>
+  %m1 = mul nuw nsw <25 x i32> %az, %bz
+  %r1 = call i32 @llvm.vector.reduce.add.v25i32(<25 x i32> %m1)
+  %cz = sext <25 x i8> %c to <25 x i32>
+  %dz = sext <25 x i8> %d to <25 x i32>
+  %m2 = mul nuw nsw <25 x i32> %cz, %dz
+  %r2 = call i32 @llvm.vector.reduce.add.v25i32(<25 x i32> %m2)
+  %x = add i32 %r1, %r2
+  ret i32 %x
+}
+
+define i32 @test_sdot_v25i8_double_nomla(<25 x i8> %a, <25 x i8> %b, <25 x i8> %c, <25 x i8> %d) {
+; CHECK-SD-LABEL: test_sdot_v25i8_double_nomla:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-SD-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-SD-NEXT:    .cfi_offset w29, -16
+; CHECK-SD-NEXT:    fmov s0, w0
+; CHECK-SD-NEXT:    ldr b1, [sp, #80]
+; CHECK-SD-NEXT:    add x10, sp, #88
+; CHECK-SD-NEXT:    ldr b2, [sp, #16]
+; CHECK-SD-NEXT:    add x9, sp, #96
+; CHECK-SD-NEXT:    ldr b3, [sp, #480]
+; CHECK-SD-NEXT:    ld1 { v1.b }[1], [x10]
+; CHECK-SD-NEXT:    add x10, sp, #24
+; CHECK-SD-NEXT:    ldr b4, [sp, #352]
+; CHECK-SD-NEXT:    mov v0.b[1], w1
+; CHECK-SD-NEXT:    ld1 { v2.b }[1], [x10]
+; CHECK-SD-NEXT:    add x11, sp, #488
+; CHECK-SD-NEXT:    add x10, sp, #360
+; CHECK-SD-NEXT:    ldr b5, [sp, #416]
+; CHECK-SD-NEXT:    add x8, sp, #104
+; CHECK-SD-NEXT:    ld1 { v1.b }[2], [x9]
+; CHECK-SD-NEXT:    add x9, sp, #32
+; CHECK-SD-NEXT:    ld1 { v3.b }[1], [x11]
+; CHECK-SD-NEXT:    ld1 { v2.b }[2], [x9]
+; CHECK-SD-NEXT:    add x11, sp, #424
+; CHECK-SD-NEXT:    ld1 { v4.b }[1], [x10]
+; CHECK-SD-NEXT:    mov v0.b[2], w2
+; CHECK-SD-NEXT:    ld1 { v5.b }[1], [x11]
+; CHECK-SD-NEXT:    add x9, sp, #368
+; CHECK-SD-NEXT:    ld1 { v1.b }[3], [x8]
+; CHECK-SD-NEXT:    add x8, sp, #40
+; CHECK-SD-NEXT:    add x12, sp, #496
+; CHECK-SD-NEXT:    ld1 { v2.b }[3], [x8]
+; CHECK-SD-NEXT:    ld1 { v4.b }[2], [x9]
+; CHECK-SD-NEXT:    add x8, sp, #432
+; CHECK-SD-NEXT:    ld1 { v3.b }[2], [x12]
+; CHECK-SD-NEXT:    add x13, sp, #48
+; CHECK-SD-NEXT:    ld1 { v5.b }[2], [x8]
+; CHECK-SD-NEXT:    mov v0.b[3], w3
+; CHECK-SD-NEXT:    add x10, sp, #112
+; CHECK-SD-NEXT:    add x8, sp, #504
+; CHECK-SD-NEXT:    ld1 { v2.b }[4], [x13]
+; CHECK-SD-NEXT:    add x13, sp, #376
+; CHECK-SD-NEXT:    ld1 { v1.b }[4], [x10]
+; CHECK-SD-NEXT:    ld1 { v4.b }[3], [x13]
+; CHECK-SD-NEXT:    add x13, sp, #440
+; CHECK-SD-NEXT:    ld1 { v3.b }[3], [x8]
+; CHECK-SD-NEXT:    ld1 { v5.b }[3], [x13]
+; CHECK-SD-NEXT:    add x11, sp, #120
+; CHECK-SD-NEXT:    add x8, sp, #56
+; CHECK-SD-NEXT:    mov v0.b[4], w4
+; CHECK-SD-NEXT:    add x13, sp, #512
+; CHECK-SD-NEXT:    ld1 { v1.b }[5], [x11]
+; CHECK-SD-NEXT:    ld1 { v2.b }[5], [x8]
+; CHECK-SD-NEXT:    add x8, sp, #384
+; CHECK-SD-NEXT:    add x11, sp, #448
+; CHECK-SD-NEXT:    ld1 { v3.b }[4], [x13]
+; CHECK-SD-NEXT:    ld1 { v4.b }[4], [x8]
+; CHECK-SD-NEXT:    ld1 { v5.b }[4], [x11]
+; CHECK-SD-NEXT:    add x12, sp, #128
+; CHECK-SD-NEXT:    add x10, sp, #64
+; CHECK-SD-NEXT:    add x8, sp, #520
+; CHECK-SD-NEXT:    mov v0.b[5], w5
+; CHECK-SD-NEXT:    ld1 { v1.b }[6], [x12]
+; CHECK-SD-NEXT:    ld1 { v2.b }[6], [x10]
+; CHECK-SD-NEXT:    add x10, sp, #392
+; CHECK-SD-NEXT:    add x11, sp, #456
+; CHECK-SD-NEXT:    ldr b6, [sp, #144]
+; CHECK-SD-NEXT:    ldr b7, [sp, #544]
+; CHECK-SD-NEXT:    ld1 { v3.b }[5], [x8]
+; CHECK-SD-NEXT:    ld1 { v4.b }[5], [x10]
+; CHECK-SD-NEXT:    ld1 { v5.b }[5], [x11]
+; CHECK-SD-NEXT:    add x9, sp, #136
+; CHECK-SD-NEXT:    sshll v6.8h, v6.8b, #0
+; CHECK-SD-NEXT:    mov v0.b[6], w6
+; CHECK-SD-NEXT:    ld1 { v1.b }[7], [x9]
+; CHECK-SD-NEXT:    add x8, sp, #528
+; CHECK-SD-NEXT:    add x9, sp, #400
+; CHECK-SD-NEXT:    add x10, sp, #464
+; CHECK-SD-NEXT:    sshll v7.8h, v7.8b, #0
+; CHECK-SD-NEXT:    ld1 { v3.b }[6], [x8]
+; CHECK-SD-NEXT:    ld1 { v4.b }[6], [x9]
+; CHECK-SD-NEXT:    ld1 { v5.b }[6], [x10]
+; CHECK-SD-NEXT:    movi v16.2d, #0000000000000000
+; CHECK-SD-NEXT:    movi v17.2d, #0000000000000000
+; CHECK-SD-NEXT:    add x14, sp, #72
+; CHECK-SD-NEXT:    mov v0.b[7], w7
+; CHECK-SD-NEXT:    sshll v6.4s, v6.4h, #0
+; CHECK-SD-NEXT:    add x8, sp, #536
+; CHECK-SD-NEXT:    add x9, sp, #408
+; CHECK-SD-NEXT:    add x10, sp, #472
+; CHECK-SD-NEXT:    sshll v7.4s, v7.4h, #0
+; CHECK-SD-NEXT:    ld1 { v2.b }[7], [x14]
+; CHECK-SD-NEXT:    ld1 { v3.b }[7], [x8]
+; CHECK-SD-NEXT:    ld1 { v4.b }[7], [x9]
+; CHECK-SD-NEXT:    ld1 { v5.b }[7], [x10]
+; CHECK-SD-NEXT:    mov v16.s[0], v6.s[0]
+; CHECK-SD-NEXT:    sshll v1.8h, v1.8b, #0
+; CHECK-SD-NEXT:    mov v17.s[0], v7.s[0]
+; CHECK-SD-NEXT:    sshll v0.8h, v0.8b, #0
+; CHECK-SD-NEXT:    sshll v2.8h, v2.8b, #0
+; CHECK-SD-NEXT:    sshll v3.8h, v3.8b, #0
+; CHECK-SD-NEXT:    sshll v4.8h, v4.8b, #0
+; CHECK-SD-NEXT:    sshll v5.8h, v5.8b, #0
+; CHECK-SD-NEXT:    saddl v7.4s, v0.4h, v1.4h
+; CHECK-SD-NEXT:    saddl2 v0.4s, v0.8h, v1.8h
+; CHECK-SD-NEXT:    saddw v6.4s, v16.4s, v2.4h
+; CHECK-SD-NEXT:    saddl v1.4s, v4.4h, v3.4h
+; CHECK-SD-NEXT:    saddl2 v3.4s, v4.8h, v3.8h
+; CHECK-SD-NEXT:    saddw v4.4s, v17.4s, v5.4h
+; CHECK-SD-NEXT:    saddw2 v0.4s, v0.4s, v2.8h
+; CHECK-SD-NEXT:    add v6.4s, v7.4s, v6.4s
+; CHECK-SD-NEXT:    saddw2 v2.4s, v3.4s, v5.8h
+; CHECK-SD-NEXT:    add v1.4s, v1.4s, v4.4s
+; CHECK-SD-NEXT:    add v0.4s, v6.4s, v0.4s
+; CHECK-SD-NEXT:    add v1.4s, v1.4s, v2.4s
+; CHECK-SD-NEXT:    add v0.4s, v0.4s, v1.4s
+; CHECK-SD-NEXT:    addv s0, v0.4s
+; CHECK-SD-NEXT:    fmov w0, s0
+; CHECK-SD-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: test_sdot_v25i8_double_nomla:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-GI-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-GI-NEXT:    .cfi_offset w29, -16
+; CHECK-GI-NEXT:    sxtb w8, w0
+; CHECK-GI-NEXT:    sxtb w9, w4
+; CHECK-GI-NEXT:    ldr w10, [sp, #48]
+; CHECK-GI-NEXT:    sxtb w11, w5
+; CHECK-GI-NEXT:    sxtb w12, w6
+; CHECK-GI-NEXT:    sxtb w13, w7
+; CHECK-GI-NEXT:    mov v0.s[0], w8
+; CHECK-GI-NEXT:    ldr w8, [sp, #16]
+; CHECK-GI-NEXT:    mov v1.s[0], w9
+; CHECK-GI-NEXT:    sxtb w9, w1
+; CHECK-GI-NEXT:    ldr w14, [sp, #104]
+; CHECK-GI-NEXT:    mov v20.s[0], wzr
+; CHECK-GI-NEXT:    sxtb w8, w8
+; CHECK-GI-NEXT:    mov v0.s[1], w9
+; CHECK-GI-NEXT:    ldr w9, [sp, #80]
+; CHECK-GI-NEXT:    mov v2.s[0], w8
+; CHECK-GI-NEXT:    sxtb w8, w10
+; CHECK-GI-NEXT:    mov v1.s[1], w11
+; CHECK-GI-NEXT:    ldr w10, [sp, #24]
+; CHECK-GI-NEXT:    sxtb w11, w2
+; CHECK-GI-NEXT:    sxtb w9, w9
+; CHECK-GI-NEXT:    mov v20.s[1], wzr
+; CHECK-GI-NEXT:    mov v3.s[0], w8
+; CHECK-GI-NEXT:    ldr w8, [sp, #56]
+; CHECK-GI-NEXT:    mov v4.s[0], w9
+; CHECK-GI-NEXT:    sxtb w9, w10
+; CHECK-GI-NEXT:    mov v0.s[2], w11
+; CHECK-GI-NEXT:    sxtb w8, w8
+; CHECK-GI-NEXT:    ldr w10, [sp, #32]
+; CHECK-GI-NEXT:    mov v1.s[2], w12
+; CHECK-GI-NEXT:    sxtb w12, w3
+; CHECK-GI-NEXT:    mov v2.s[1], w9
+; CHECK-GI-NEXT:    ldr w9, [sp, #112]
+; CHECK-GI-NEXT:    mov v3.s[1], w8
+; CHECK-GI-NEXT:    ldr w8, [sp, #88]
+; CHECK-GI-NEXT:    sxtb w10, w10
+; CHECK-GI-NEXT:    mov v0.s[3], w12
+; CHECK-GI-NEXT:    ldr w12, [sp, #64]
+; CHECK-GI-NEXT:    sxtb w9, w9
+; CHECK-GI-NEXT:    sxtb w8, w8
+; CHECK-GI-NEXT:    ldr w11, [sp, #40]
+; CHECK-GI-NEXT:    mov v1.s[3], w13
+; CHECK-GI-NEXT:    mov v2.s[2], w10
+; CHECK-GI-NEXT:    ldr w10, [sp, #120]
+; CHECK-GI-NEXT:    sxtb w12, w12
+; CHECK-GI-NEXT:    mov v5.s[0], w9
+; CHECK-GI-NEXT:    ldr w9, [sp, #96]
+; CHECK-GI-NEXT:    mov v4.s[1], w8
+; CHECK-GI-NEXT:    ldr w13, [sp, #72]
+; CHECK-GI-NEXT:    sxtb w8, w10
+; CHECK-GI-NEXT:    mov v3.s[2], w12
+; CHECK-GI-NEXT:    ldr w10, [sp, #352]
+; CHECK-GI-NEXT:    sxtb w9, w9
+; CHECK-GI-NEXT:    sxtb w11, w11
+; CHECK-GI-NEXT:    sxtb w12, w13
+; CHECK-GI-NEXT:    sxtb w13, w14
+; CHECK-GI-NEXT:    mov v20.s[2], wzr
+; CHECK-GI-NEXT:    mov v4.s[2], w9
+; CHECK-GI-NEXT:    sxtb w9, w10
+; CHECK-GI-NEXT:    mov v2.s[3], w11
+; CHECK-GI-NEXT:    ldr w11, [sp, #128]
+; CHECK-GI-NEXT:    mov v5.s[1], w8
+; CHECK-GI-NEXT:    ldr w10, [sp, #384]
+; CHECK-GI-NEXT:    mov v3.s[3], w12
+; CHECK-GI-NEXT:    ldr w12, [sp, #360]
+; CHECK-GI-NEXT:    mov v6.s[0], w9
+; CHECK-GI-NEXT:    sxtb w11, w11
+; CHECK-GI-NEXT:    ldr w9, [sp, #144]
+; CHECK-GI-NEXT:    sxtb w10, w10
+; CHECK-GI-NEXT:    sxtb w12, w12
+; CHECK-GI-NEXT:    mov v4.s[3], w13
+; CHECK-GI-NEXT:    ldr w13, [sp, #392]
+; CHECK-GI-NEXT:    mov v5.s[2], w11
+; CHECK-GI-NEXT:    ldr w11, [sp, #368]
+; CHECK-GI-NEXT:    mov v7.s[0], w10
+; CHECK-GI-NEXT:    sxtb w9, w9
+; CHECK-GI-NEXT:    mov v6.s[1], w12
+; CHECK-GI-NEXT:    ldr w10, [sp, #416]
+; CHECK-GI-NEXT:    sxtb w12, w13
+; CHECK-GI-NEXT:    ldr w13, [sp, #448]
+; CHECK-GI-NEXT:    ldr w8, [sp, #136]
+; CHECK-GI-NEXT:    mov v16.s[0], w9
+; CHECK-GI-NEXT:    sxtb w9, w11
+; CHECK-GI-NEXT:    sxtb w10, w10
+; CHECK-GI-NEXT:    ldr w11, [sp, #400]
+; CHECK-GI-NEXT:    mov v7.s[1], w12
+; CHECK-GI-NEXT:    ldr w12, [sp, #480]
+; CHECK-GI-NEXT:    mov v6.s[2], w9
+; CHECK-GI-NEXT:    ldr w9, [sp, #424]
+; CHECK-GI-NEXT:    mov v17.s[0], w10
+; CHECK-GI-NEXT:    sxtb w13, w13
+; CHECK-GI-NEXT:    sxtb w11, w11
+; CHECK-GI-NEXT:    sxtb w12, w12
+; CHECK-GI-NEXT:    sxtb w9, w9
+; CHECK-GI-NEXT:    ldr w10, [sp, #456]
+; CHECK-GI-NEXT:    mov v16.s[1], wzr
+; CHECK-GI-NEXT:    mov v18.s[0], w13
+; CHECK-GI-NEXT:    ldr w13, [sp, #488]
+; CHECK-GI-NEXT:    mov v7.s[2], w11
+; CHECK-GI-NEXT:    ldr w11, [sp, #512]
+; CHECK-GI-NEXT:    mov v19.s[0], w12
+; CHECK-GI-NEXT:    mov v17.s[1], w9
+; CHECK-GI-NEXT:    ldr w9, [sp, #544]
+; CHECK-GI-NEXT:    sxtb w10, w10
+; CHECK-GI-NEXT:    sxtb w13, w13
+; CHECK-GI-NEXT:    sxtb w11, w11
+; CHECK-GI-NEXT:    sxtb w14, w8
+; CHECK-GI-NEXT:    ldr w8, [sp, #376]
+; CHECK-GI-NEXT:    sxtb w9, w9
+; CHECK-GI-NEXT:    mov v18.s[1], w10
+; CHECK-GI-NEXT:    ldr w10, [sp, #432]
+; CHECK-GI-NEXT:    mov v19.s[1], w13
+; CHECK-GI-NEXT:    ldr w13, [sp, #520]
+; CHECK-GI-NEXT:    mov v21.s[0], w11
+; CHECK-GI-NEXT:    mov v22.s[0], w9
+; CHECK-GI-NEXT:    sxtb w10, w10
+; CHECK-GI-NEXT:    ldr w11, [sp, #464]
+; CHECK-GI-NEXT:    sxtb w13, w13
+; CHECK-GI-NEXT:    ldr w9, [sp, #496]
+; CHECK-GI-NEXT:    ldr w12, [sp, #408]
+; CHECK-GI-NEXT:    mov v17.s[2], w10
+; CHECK-GI-NEXT:    ldr w10, [sp, #528]
+; CHECK-GI-NEXT:    sxtb w11, w11
+; CHECK-GI-NEXT:    mov v21.s[1], w13
+; CHECK-GI-NEXT:    sxtb w9, w9
+; CHECK-GI-NEXT:    ldr w13, [sp, #440]
+; CHECK-GI-NEXT:    mov v22.s[1], wzr
+; CHECK-GI-NEXT:    sxtb w10, w10
+; CHECK-GI-NEXT:    mov v18.s[2], w11
+; CHECK-GI-NEXT:    ldr w11, [sp, #472]
+; CHECK-GI-NEXT:    mov v19.s[2], w9
+; CHECK-GI-NEXT:    ldr w9, [sp, #504]
+; CHECK-GI-NEXT:    mov v16.s[2], wzr
+; CHECK-GI-NEXT:    sxtb w8, w8
+; CHECK-GI-NEXT:    sxtb w12, w12
+; CHECK-GI-NEXT:    mov v21.s[2], w10
+; CHECK-GI-NEXT:    ldr w10, [sp, #536]
+; CHECK-GI-NEXT:    sxtb w13, w13
+; CHECK-GI-NEXT:    mov v22.s[2], wzr
+; CHECK-GI-NEXT:    mov v6.s[3], w8
+; CHECK-GI-NEXT:    sxtb w8, w11
+; CHECK-GI-NEXT:    sxtb w9, w9
+; CHECK-GI-NEXT:    sxtb w10, w10
+; CHECK-GI-NEXT:    mov v5.s[3], w14
+; CHECK-GI-NEXT:    mov v7.s[3], w12
+; CHECK-GI-NEXT:    mov v17.s[3], w13
+; CHECK-GI-NEXT:    mov v18.s[3], w8
+; CHECK-GI-NEXT:    mov v16.s[3], wzr
+; CHECK-GI-NEXT:    mov v20.s[3], wzr
+; CHECK-GI-NEXT:    mov v19.s[3], w9
+; CHECK-GI-NEXT:    mov v21.s[3], w10
+; CHECK-GI-NEXT:    mov v22.s[3], wzr
+; CHECK-GI-NEXT:    add v0.4s, v0.4s, v1.4s
+; CHECK-GI-NEXT:    add v1.4s, v2.4s, v3.4s
+; CHECK-GI-NEXT:    add v2.4s, v4.4s, v5.4s
+; CHECK-GI-NEXT:    add v3.4s, v6.4s, v7.4s
+; CHECK-GI-NEXT:    add v4.4s, v17.4s, v18.4s
+; CHECK-GI-NEXT:    add v6.4s, v16.4s, v20.4s
+; CHECK-GI-NEXT:    add v5.4s, v19.4s, v21.4s
+; CHECK-GI-NEXT:    add v7.4s, v22.4s, v20.4s
+; CHECK-GI-NEXT:    add v0.4s, v0.4s, v1.4s
+; CHECK-GI-NEXT:    add v1.4s, v2.4s, v6.4s
+; CHECK-GI-NEXT:    add v2.4s, v3.4s, v4.4s
+; CHECK-GI-NEXT:    add v3.4s, v5.4s, v7.4s
+; CHECK-GI-NEXT:    add v0.4s, v0.4s, v1.4s
+; CHECK-GI-NEXT:    add v1.4s, v2.4s, v3.4s
+; CHECK-GI-NEXT:    addv s0, v0.4s
+; CHECK-GI-NEXT:    addv s1, v1.4s
+; CHECK-GI-NEXT:    fmov w8, s0
+; CHECK-GI-NEXT:    fmov w9, s1
+; CHECK-GI-NEXT:    add w0, w8, w9
+; CHECK-GI-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
+; CHECK-GI-NEXT:    ret
+entry:
+  %az = sext <25 x i8> %a to <25 x i32>
+  %r1 = call i32 @llvm.vector.reduce.add.v25i32(<25 x i32> %az)
+  %cz = sext <25 x i8> %c to <25 x i32>
+  %r2 = call i32 @llvm.vector.reduce.add.v25i32(<25 x i32> %cz)
+  %x = add i32 %r1, %r2
+  ret i32 %x
+}
+
+define i32 @test_udot_v32i8(ptr nocapture readonly %a, ptr nocapture readonly %b, i32 %sum) {
+; CHECK-SD-LABEL: test_udot_v32i8:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    movi v0.2d, #0000000000000000
+; CHECK-SD-NEXT:    ldp q1, q3, [x0]
+; CHECK-SD-NEXT:    ldp q2, q4, [x1]
+; CHECK-SD-NEXT:    udot v0.4s, v4.16b, v3.16b
+; CHECK-SD-NEXT:    udot v0.4s, v2.16b, v1.16b
+; CHECK-SD-NEXT:    addv s0, v0.4s
+; CHECK-SD-NEXT:    fmov w8, s0
+; CHECK-SD-NEXT:    add w0, w8, w2
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: test_udot_v32i8:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    movi v0.2d, #0000000000000000
+; CHECK-GI-NEXT:    movi v1.2d, #0000000000000000
+; CHECK-GI-NEXT:    ldp q2, q3, [x0]
+; CHECK-GI-NEXT:    ldp q4, q5, [x1]
+; CHECK-GI-NEXT:    udot v1.4s, v4.16b, v2.16b
+; CHECK-GI-NEXT:    udot v0.4s, v5.16b, v3.16b
+; CHECK-GI-NEXT:    add v0.4s, v1.4s, v0.4s
+; CHECK-GI-NEXT:    addv s0, v0.4s
+; CHECK-GI-NEXT:    fmov w8, s0
+; CHECK-GI-NEXT:    add w0, w8, w2
+; CHECK-GI-NEXT:    ret
+entry:
+  %0 = load <32 x i8>, ptr %a
+  %1 = zext <32 x i8> %0 to <32 x i32>
+  %2 = load <32 x i8>, ptr %b
+  %3 = zext <32 x i8> %2 to <32 x i32>
+  %4 = mul nuw nsw <32 x i32> %3, %1
+  %5 = call i32 @llvm.vector.reduce.add.v32i32(<32 x i32> %4)
+  %op.extra = add i32 %5, %sum
+  ret i32 %op.extra
+}
+
+define i32 @test_udot_v32i8_nomla(ptr nocapture readonly %a1) {
+; CHECK-SD-LABEL: test_udot_v32i8_nomla:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    movi v0.16b, #1
+; CHECK-SD-NEXT:    movi v1.2d, #0000000000000000
+; CHECK-SD-NEXT:    ldp q2, q3, [x0]
+; CHECK-SD-NEXT:    udot v1.4s, v3.16b, v0.16b
+; CHECK-SD-NEXT:    udot v1.4s, v2.16b, v0.16b
+; CHECK-SD-NEXT:    addv s0, v1.4s
+; CHECK-SD-NEXT:    fmov w0, s0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: test_udot_v32i8_nomla:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    movi v0.16b, #1
+; CHECK-GI-NEXT:    movi v1.2d, #0000000000000000
+; CHECK-GI-NEXT:    movi v2.2d, #0000000000000000
+; CHECK-GI-NEXT:    ldp q3, q4, [x0]
+; CHECK-GI-NEXT:    udot v2.4s, v3.16b, v0.16b
+; CHECK-GI-NEXT:    udot v1.4s, v4.16b, v0.16b
+; CHECK-GI-NEXT:    add v0.4s, v2.4s, v1.4s
+; CHECK-GI-NEXT:    addv s0, v0.4s
+; CHECK-GI-NEXT:    fmov w0, s0
+; CHECK-GI-NEXT:    ret
+entry:
+  %0 = load <32 x i8>, ptr %a1
+  %1 = zext <32 x i8> %0 to <32 x i32>
+  %2 = call i32 @llvm.vector.reduce.add.v32i32(<32 x i32> %1)
+  ret i32 %2
+}
+define i32 @test_sdot_v32i8(ptr nocapture readonly %a, ptr nocapture readonly %b, i32 %sum) {
+; CHECK-SD-LABEL: test_sdot_v32i8:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    movi v0.2d, #0000000000000000
 ; CHECK-SD-NEXT:    ldp q1, q3, [x0]
 ; CHECK-SD-NEXT:    ldp q2, q4, [x1]
 ; CHECK-SD-NEXT:    sdot v0.4s, v4.16b, v3.16b
@@ -2881,529 +3891,1429 @@ define i32 @test_usdot_v32i8(ptr nocapture readonly %a, ptr nocapture readonly %
 ; CHECK-GI-NEXT:    add w0, w8, w2
 ; CHECK-GI-NEXT:    ret
 entry:
-  %0 = load <32 x i8>, ptr %a
-  %1 = zext <32 x i8> %0 to <32 x i32>
-  %2 = load <32 x i8>, ptr %b
-  %3 = sext <32 x i8> %2 to <32 x i32>
-  %4 = mul nsw <32 x i32> %3, %1
-  %5 = call i32 @llvm.vector.reduce.add.v32i32(<32 x i32> %4)
+  %0 = load <32 x i8>, ptr %a
+  %1 = zext <32 x i8> %0 to <32 x i32>
+  %2 = load <32 x i8>, ptr %b
+  %3 = sext <32 x i8> %2 to <32 x i32>
+  %4 = mul nsw <32 x i32> %3, %1
+  %5 = call i32 @llvm.vector.reduce.add.v32i32(<32 x i32> %4)
+  %op.extra = add nsw i32 %5, %sum
+  ret i32 %op.extra
+}
+
+define i32 @test_usdot_v32i8_double(<32 x i8> %a, <32 x i8> %b, <32 x i8> %c, <32 x i8> %d) {
+; CHECK-SD-LABEL: test_usdot_v32i8_double:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    movi v16.2d, #0000000000000000
+; CHECK-SD-NEXT:    movi v17.2d, #0000000000000000
+; CHECK-SD-NEXT:    movi v18.2d, #0000000000000000
+; CHECK-SD-NEXT:    movi v19.2d, #0000000000000000
+; CHECK-SD-NEXT:    usdot v16.4s, v1.16b, v3.16b
+; CHECK-SD-NEXT:    usdot v18.4s, v0.16b, v2.16b
+; CHECK-SD-NEXT:    usdot v17.4s, v4.16b, v6.16b
+; CHECK-SD-NEXT:    usdot v19.4s, v5.16b, v7.16b
+; CHECK-SD-NEXT:    add v0.4s, v18.4s, v16.4s
+; CHECK-SD-NEXT:    add v1.4s, v17.4s, v19.4s
+; CHECK-SD-NEXT:    add v0.4s, v0.4s, v1.4s
+; CHECK-SD-NEXT:    addv s0, v0.4s
+; CHECK-SD-NEXT:    fmov w0, s0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: test_usdot_v32i8_double:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    stp d15, d14, [sp, #-64]! // 16-byte Folded Spill
+; CHECK-GI-NEXT:    stp d13, d12, [sp, #16] // 16-byte Folded Spill
+; CHECK-GI-NEXT:    stp d11, d10, [sp, #32] // 16-byte Folded Spill
+; CHECK-GI-NEXT:    stp d9, d8, [sp, #48] // 16-byte Folded Spill
+; CHECK-GI-NEXT:    .cfi_def_cfa_offset 64
+; CHECK-GI-NEXT:    .cfi_offset b8, -8
+; CHECK-GI-NEXT:    .cfi_offset b9, -16
+; CHECK-GI-NEXT:    .cfi_offset b10, -24
+; CHECK-GI-NEXT:    .cfi_offset b11, -32
+; CHECK-GI-NEXT:    .cfi_offset b12, -40
+; CHECK-GI-NEXT:    .cfi_offset b13, -48
+; CHECK-GI-NEXT:    .cfi_offset b14, -56
+; CHECK-GI-NEXT:    .cfi_offset b15, -64
+; CHECK-GI-NEXT:    ushll v16.8h, v0.8b, #0
+; CHECK-GI-NEXT:    ushll2 v0.8h, v0.16b, #0
+; CHECK-GI-NEXT:    ushll v17.8h, v1.8b, #0
+; CHECK-GI-NEXT:    ushll2 v1.8h, v1.16b, #0
+; CHECK-GI-NEXT:    sshll v18.8h, v2.8b, #0
+; CHECK-GI-NEXT:    sshll2 v2.8h, v2.16b, #0
+; CHECK-GI-NEXT:    sshll v19.8h, v3.8b, #0
+; CHECK-GI-NEXT:    sshll2 v3.8h, v3.16b, #0
+; CHECK-GI-NEXT:    ushll v27.8h, v4.8b, #0
+; CHECK-GI-NEXT:    ushll2 v4.8h, v4.16b, #0
+; CHECK-GI-NEXT:    ushll v28.8h, v5.8b, #0
+; CHECK-GI-NEXT:    sshll v29.8h, v6.8b, #0
+; CHECK-GI-NEXT:    sshll2 v6.8h, v6.16b, #0
+; CHECK-GI-NEXT:    ushll2 v5.8h, v5.16b, #0
+; CHECK-GI-NEXT:    sshll v30.8h, v7.8b, #0
+; CHECK-GI-NEXT:    sshll2 v7.8h, v7.16b, #0
+; CHECK-GI-NEXT:    ushll2 v20.4s, v16.8h, #0
+; CHECK-GI-NEXT:    ushll2 v21.4s, v0.8h, #0
+; CHECK-GI-NEXT:    ushll2 v22.4s, v17.8h, #0
+; CHECK-GI-NEXT:    ushll2 v23.4s, v1.8h, #0
+; CHECK-GI-NEXT:    sshll2 v24.4s, v18.8h, #0
+; CHECK-GI-NEXT:    sshll2 v25.4s, v2.8h, #0
+; CHECK-GI-NEXT:    sshll2 v26.4s, v19.8h, #0
+; CHECK-GI-NEXT:    sshll2 v31.4s, v3.8h, #0
+; CHECK-GI-NEXT:    ushll2 v8.4s, v27.8h, #0
+; CHECK-GI-NEXT:    ushll2 v9.4s, v4.8h, #0
+; CHECK-GI-NEXT:    ushll2 v10.4s, v28.8h, #0
+; CHECK-GI-NEXT:    sshll2 v11.4s, v29.8h, #0
+; CHECK-GI-NEXT:    sshll2 v12.4s, v6.8h, #0
+; CHECK-GI-NEXT:    ushll2 v13.4s, v5.8h, #0
+; CHECK-GI-NEXT:    sshll2 v14.4s, v30.8h, #0
+; CHECK-GI-NEXT:    sshll2 v15.4s, v7.8h, #0
+; CHECK-GI-NEXT:    mul v20.4s, v20.4s, v24.4s
+; CHECK-GI-NEXT:    mul v21.4s, v21.4s, v25.4s
+; CHECK-GI-NEXT:    mul v22.4s, v22.4s, v26.4s
+; CHECK-GI-NEXT:    mul v23.4s, v23.4s, v31.4s
+; CHECK-GI-NEXT:    mul v24.4s, v8.4s, v11.4s
+; CHECK-GI-NEXT:    mul v25.4s, v9.4s, v12.4s
+; CHECK-GI-NEXT:    ushll v16.4s, v16.4h, #0
+; CHECK-GI-NEXT:    ldp d9, d8, [sp, #48] // 16-byte Folded Reload
+; CHECK-GI-NEXT:    mul v26.4s, v10.4s, v14.4s
+; CHECK-GI-NEXT:    ldp d11, d10, [sp, #32] // 16-byte Folded Reload
+; CHECK-GI-NEXT:    mul v31.4s, v13.4s, v15.4s
+; CHECK-GI-NEXT:    ldp d13, d12, [sp, #16] // 16-byte Folded Reload
+; CHECK-GI-NEXT:    ushll v0.4s, v0.4h, #0
+; CHECK-GI-NEXT:    ushll v17.4s, v17.4h, #0
+; CHECK-GI-NEXT:    ushll v1.4s, v1.4h, #0
+; CHECK-GI-NEXT:    sshll v18.4s, v18.4h, #0
+; CHECK-GI-NEXT:    sshll v2.4s, v2.4h, #0
+; CHECK-GI-NEXT:    sshll v19.4s, v19.4h, #0
+; CHECK-GI-NEXT:    sshll v3.4s, v3.4h, #0
+; CHECK-GI-NEXT:    ushll v27.4s, v27.4h, #0
+; CHECK-GI-NEXT:    ushll v4.4s, v4.4h, #0
+; CHECK-GI-NEXT:    ushll v28.4s, v28.4h, #0
+; CHECK-GI-NEXT:    ushll v5.4s, v5.4h, #0
+; CHECK-GI-NEXT:    sshll v29.4s, v29.4h, #0
+; CHECK-GI-NEXT:    sshll v6.4s, v6.4h, #0
+; CHECK-GI-NEXT:    sshll v30.4s, v30.4h, #0
+; CHECK-GI-NEXT:    sshll v7.4s, v7.4h, #0
+; CHECK-GI-NEXT:    mla v20.4s, v16.4s, v18.4s
+; CHECK-GI-NEXT:    mla v21.4s, v0.4s, v2.4s
+; CHECK-GI-NEXT:    mla v22.4s, v17.4s, v19.4s
+; CHECK-GI-NEXT:    mla v23.4s, v1.4s, v3.4s
+; CHECK-GI-NEXT:    mla v24.4s, v27.4s, v29.4s
+; CHECK-GI-NEXT:    mla v25.4s, v4.4s, v6.4s
+; CHECK-GI-NEXT:    mla v26.4s, v28.4s, v30.4s
+; CHECK-GI-NEXT:    mla v31.4s, v5.4s, v7.4s
+; CHECK-GI-NEXT:    add v0.4s, v20.4s, v21.4s
+; CHECK-GI-NEXT:    add v1.4s, v22.4s, v23.4s
+; CHECK-GI-NEXT:    add v2.4s, v24.4s, v25.4s
+; CHECK-GI-NEXT:    add v3.4s, v26.4s, v31.4s
+; CHECK-GI-NEXT:    add v0.4s, v0.4s, v1.4s
+; CHECK-GI-NEXT:    add v1.4s, v2.4s, v3.4s
+; CHECK-GI-NEXT:    addv s0, v0.4s
+; CHECK-GI-NEXT:    addv s1, v1.4s
+; CHECK-GI-NEXT:    fmov w8, s0
+; CHECK-GI-NEXT:    fmov w9, s1
+; CHECK-GI-NEXT:    add w0, w8, w9
+; CHECK-GI-NEXT:    ldp d15, d14, [sp], #64 // 16-byte Folded Reload
+; CHECK-GI-NEXT:    ret
+entry:
+  %az = zext <32 x i8> %a to <32 x i32>
+  %bz = sext <32 x i8> %b to <32 x i32>
+  %m1 = mul nuw nsw <32 x i32> %az, %bz
+  %r1 = call i32 @llvm.vector.reduce.add.v32i32(<32 x i32> %m1)
+  %cz = zext <32 x i8> %c to <32 x i32>
+  %dz = sext <32 x i8> %d to <32 x i32>
+  %m2 = mul nuw nsw <32 x i32> %cz, %dz
+  %r2 = call i32 @llvm.vector.reduce.add.v32i32(<32 x i32> %m2)
+  %x = add i32 %r1, %r2
+  ret i32 %x
+}
+
+
+define i32 @test_udot_v33i8(ptr nocapture readonly %a, ptr nocapture readonly %b, i32 %sum) {
+; CHECK-SD-LABEL: test_udot_v33i8:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    ldr b0, [x0, #32]
+; CHECK-SD-NEXT:    ldr b1, [x1, #32]
+; CHECK-SD-NEXT:    movi v7.2d, #0000000000000000
+; CHECK-SD-NEXT:    ldp q3, q4, [x1]
+; CHECK-SD-NEXT:    umull v0.8h, v1.8b, v0.8b
+; CHECK-SD-NEXT:    ldp q1, q2, [x0]
+; CHECK-SD-NEXT:    umull v5.8h, v4.8b, v2.8b
+; CHECK-SD-NEXT:    umull v6.8h, v3.8b, v1.8b
+; CHECK-SD-NEXT:    umull2 v2.8h, v4.16b, v2.16b
+; CHECK-SD-NEXT:    ushll v0.4s, v0.4h, #0
+; CHECK-SD-NEXT:    umull2 v1.8h, v3.16b, v1.16b
+; CHECK-SD-NEXT:    mov v7.s[0], v0.s[0]
+; CHECK-SD-NEXT:    uaddl2 v3.4s, v6.8h, v5.8h
+; CHECK-SD-NEXT:    uaddl2 v0.4s, v1.8h, v2.8h
+; CHECK-SD-NEXT:    uaddl v1.4s, v1.4h, v2.4h
+; CHECK-SD-NEXT:    add v0.4s, v3.4s, v0.4s
+; CHECK-SD-NEXT:    uaddw v2.4s, v7.4s, v6.4h
+; CHECK-SD-NEXT:    uaddw v2.4s, v2.4s, v5.4h
+; CHECK-SD-NEXT:    add v0.4s, v1.4s, v0.4s
+; CHECK-SD-NEXT:    add v0.4s, v2.4s, v0.4s
+; CHECK-SD-NEXT:    addv s0, v0.4s
+; CHECK-SD-NEXT:    fmov w8, s0
+; CHECK-SD-NEXT:    add w0, w8, w2
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: test_udot_v33i8:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    ldp q19, q4, [x1]
+; CHECK-GI-NEXT:    mov v25.s[0], wzr
+; CHECK-GI-NEXT:    ldp q7, q5, [x0]
+; CHECK-GI-NEXT:    umov w8, v19.b[0]
+; CHECK-GI-NEXT:    umov w9, v19.b[4]
+; CHECK-GI-NEXT:    umov w10, v19.b[8]
+; CHECK-GI-NEXT:    umov w11, v19.b[1]
+; CHECK-GI-NEXT:    umov w12, v19.b[6]
+; CHECK-GI-NEXT:    umov w13, v19.b[12]
+; CHECK-GI-NEXT:    umov w14, v4.b[0]
+; CHECK-GI-NEXT:    umov w15, v4.b[4]
+; CHECK-GI-NEXT:    umov w16, v4.b[12]
+; CHECK-GI-NEXT:    mov v25.s[1], wzr
+; CHECK-GI-NEXT:    mov v0.s[0], w8
+; CHECK-GI-NEXT:    umov w8, v19.b[5]
+; CHECK-GI-NEXT:    mov v2.s[0], w9
+; CHECK-GI-NEXT:    umov w9, v19.b[9]
+; CHECK-GI-NEXT:    mov v1.s[0], w10
+; CHECK-GI-NEXT:    umov w10, v19.b[2]
+; CHECK-GI-NEXT:    mov v6.s[0], w13
+; CHECK-GI-NEXT:    umov w13, v19.b[3]
+; CHECK-GI-NEXT:    mov v3.s[0], w14
+; CHECK-GI-NEXT:    umov w14, v19.b[13]
+; CHECK-GI-NEXT:    mov v16.s[0], w15
+; CHECK-GI-NEXT:    umov w15, v4.b[8]
+; CHECK-GI-NEXT:    mov v0.s[1], w11
+; CHECK-GI-NEXT:    umov w11, v19.b[10]
+; CHECK-GI-NEXT:    mov v2.s[1], w8
+; CHECK-GI-NEXT:    ldrb w8, [x0, #32]
+; CHECK-GI-NEXT:    mov v1.s[1], w9
+; CHECK-GI-NEXT:    ldrb w9, [x1, #32]
+; CHECK-GI-NEXT:    mov v17.s[0], w16
+; CHECK-GI-NEXT:    umov w16, v19.b[14]
+; CHECK-GI-NEXT:    mov v25.s[2], wzr
+; CHECK-GI-NEXT:    mul w8, w9, w8
+; CHECK-GI-NEXT:    mov v6.s[1], w14
+; CHECK-GI-NEXT:    umov w14, v4.b[1]
+; CHECK-GI-NEXT:    mov v0.s[2], w10
+; CHECK-GI-NEXT:    umov w10, v19.b[7]
+; CHECK-GI-NEXT:    mov v2.s[2], w12
+; CHECK-GI-NEXT:    umov w12, v19.b[11]
+; CHECK-GI-NEXT:    mov v1.s[2], w11
+; CHECK-GI-NEXT:    umov w11, v4.b[5]
+; CHECK-GI-NEXT:    mov v18.s[0], w15
+; CHECK-GI-NEXT:    umov w15, v19.b[15]
+; CHECK-GI-NEXT:    umov w9, v5.b[2]
+; CHECK-GI-NEXT:    mov v6.s[2], w16
+; CHECK-GI-NEXT:    umov w16, v7.b[0]
+; CHECK-GI-NEXT:    mov v3.s[1], w14
+; CHECK-GI-NEXT:    mov v0.s[3], w13
+; CHECK-GI-NEXT:    umov w13, v7.b[4]
+; CHECK-GI-NEXT:    mov v2.s[3], w10
+; CHECK-GI-NEXT:    umov w10, v4.b[6]
+; CHECK-GI-NEXT:    mov v1.s[3], w12
+; CHECK-GI-NEXT:    umov w12, v4.b[13]
+; CHECK-GI-NEXT:    mov v16.s[1], w11
+; CHECK-GI-NEXT:    umov w11, v4.b[9]
+; CHECK-GI-NEXT:    umov w14, v7.b[5]
+; CHECK-GI-NEXT:    mov v19.s[0], w16
+; CHECK-GI-NEXT:    umov w16, v7.b[1]
+; CHECK-GI-NEXT:    mov v6.s[3], w15
+; CHECK-GI-NEXT:    mov v20.s[0], w13
+; CHECK-GI-NEXT:    umov w13, v4.b[2]
+; CHECK-GI-NEXT:    umov w15, v7.b[6]
+; CHECK-GI-NEXT:    mov v17.s[1], w12
+; CHECK-GI-NEXT:    umov w12, v4.b[14]
+; CHECK-GI-NEXT:    mov v27.s[0], w8
+; CHECK-GI-NEXT:    mov v16.s[2], w10
+; CHECK-GI-NEXT:    umov w10, v4.b[7]
+; CHECK-GI-NEXT:    mov v18.s[1], w11
+; CHECK-GI-NEXT:    umov w11, v4.b[10]
+; CHECK-GI-NEXT:    mov v19.s[1], w16
+; CHECK-GI-NEXT:    umov w16, v5.b[4]
+; CHECK-GI-NEXT:    mov v20.s[1], w14
+; CHECK-GI-NEXT:    umov w14, v4.b[15]
+; CHECK-GI-NEXT:    mov v3.s[2], w13
+; CHECK-GI-NEXT:    mov v17.s[2], w12
+; CHECK-GI-NEXT:    umov w12, v7.b[12]
+; CHECK-GI-NEXT:    umov w13, v7.b[7]
+; CHECK-GI-NEXT:    mov v16.s[3], w10
+; CHECK-GI-NEXT:    umov w10, v7.b[8]
+; CHECK-GI-NEXT:    umov w8, v7.b[3]
+; CHECK-GI-NEXT:    mov v18.s[2], w11
+; CHECK-GI-NEXT:    umov w11, v7.b[2]
+; CHECK-GI-NEXT:    mov v23.s[0], w16
+; CHECK-GI-NEXT:    mov v20.s[2], w15
+; CHECK-GI-NEXT:    umov w15, v5.b[12]
+; CHECK-GI-NEXT:    umov w16, v7.b[14]
+; CHECK-GI-NEXT:    mov v17.s[3], w14
+; CHECK-GI-NEXT:    umov w14, v7.b[13]
+; CHECK-GI-NEXT:    mov v22.s[0], w12
+; CHECK-GI-NEXT:    umov w12, v7.b[9]
+; CHECK-GI-NEXT:    mov v21.s[0], w10
+; CHECK-GI-NEXT:    umov w10, v4.b[3]
+; CHECK-GI-NEXT:    mov v19.s[2], w11
+; CHECK-GI-NEXT:    umov w11, v5.b[0]
+; CHECK-GI-NEXT:    mov v27.s[1], wzr
+; CHECK-GI-NEXT:    mov v20.s[3], w13
+; CHECK-GI-NEXT:    umov w13, v5.b[5]
+; CHECK-GI-NEXT:    mov v24.s[0], w15
+; CHECK-GI-NEXT:    mov v22.s[1], w14
+; CHECK-GI-NEXT:    umov w14, v5.b[8]
+; CHECK-GI-NEXT:    umov w15, v4.b[11]
+; CHECK-GI-NEXT:    mov v21.s[1], w12
+; CHECK-GI-NEXT:    umov w12, v5.b[13]
+; CHECK-GI-NEXT:    mov v25.s[3], wzr
+; CHECK-GI-NEXT:    mov v4.s[0], w11
+; CHECK-GI-NEXT:    umov w11, v5.b[1]
+; CHECK-GI-NEXT:    mov v3.s[3], w10
+; CHECK-GI-NEXT:    mov v23.s[1], w13
+; CHECK-GI-NEXT:    umov w13, v5.b[6]
+; CHECK-GI-NEXT:    mov v19.s[3], w8
+; CHECK-GI-NEXT:    mov v22.s[2], w16
+; CHECK-GI-NEXT:    umov w16, v5.b[9]
+; CHECK-GI-NEXT:    mov v26.s[0], w14
+; CHECK-GI-NEXT:    mov v24.s[1], w12
+; CHECK-GI-NEXT:    umov w12, v5.b[14]
+; CHECK-GI-NEXT:    umov w14, v7.b[10]
+; CHECK-GI-NEXT:    mov v4.s[1], w11
+; CHECK-GI-NEXT:    umov w11, v7.b[15]
+; CHECK-GI-NEXT:    mov v18.s[3], w15
+; CHECK-GI-NEXT:    mov v23.s[2], w13
+; CHECK-GI-NEXT:    umov w13, v5.b[7]
+; CHECK-GI-NEXT:    mul v2.4s, v2.4s, v20.4s
+; CHECK-GI-NEXT:    mov v26.s[1], w16
+; CHECK-GI-NEXT:    umov w16, v5.b[10]
+; CHECK-GI-NEXT:    mov v27.s[2], wzr
+; CHECK-GI-NEXT:    mov v24.s[2], w12
+; CHECK-GI-NEXT:    umov w12, v5.b[15]
+; CHECK-GI-NEXT:    mov v21.s[2], w14
+; CHECK-GI-NEXT:    umov w14, v7.b[11]
+; CHECK-GI-NEXT:    mov v4.s[2], w9
+; CHECK-GI-NEXT:    umov w9, v5.b[3]
+; CHECK-GI-NEXT:    mov v22.s[3], w11
+; CHECK-GI-NEXT:    umov w11, v5.b[11]
+; CHECK-GI-NEXT:    mov v23.s[3], w13
+; CHECK-GI-NEXT:    mov v26.s[2], w16
+; CHECK-GI-NEXT:    mla v2.4s, v0.4s, v19.4s
+; CHECK-GI-NEXT:    mov v27.s[3], wzr
+; CHECK-GI-NEXT:    mov v24.s[3], w12
+; CHECK-GI-NEXT:    mov v21.s[3], w14
+; CHECK-GI-NEXT:    mov v4.s[3], w9
+; CHECK-GI-NEXT:    mul v5.4s, v6.4s, v22.4s
+; CHECK-GI-NEXT:    mul v6.4s, v16.4s, v23.4s
+; CHECK-GI-NEXT:    add v16.4s, v25.4s, v25.4s
+; CHECK-GI-NEXT:    mov v26.s[3], w11
+; CHECK-GI-NEXT:    mul v7.4s, v17.4s, v24.4s
+; CHECK-GI-NEXT:    add v0.4s, v25.4s, v16.4s
+; CHECK-GI-NEXT:    mla v5.4s, v1.4s, v21.4s
+; CHECK-GI-NEXT:    mla v6.4s, v3.4s, v4.4s
+; CHECK-GI-NEXT:    add v3.4s, v16.4s, v16.4s
+; CHECK-GI-NEXT:    mla v7.4s, v18.4s, v26.4s
+; CHECK-GI-NEXT:    add v0.4s, v27.4s, v0.4s
+; CHECK-GI-NEXT:    add v1.4s, v2.4s, v5.4s
+; CHECK-GI-NEXT:    add v0.4s, v0.4s, v3.4s
+; CHECK-GI-NEXT:    add v2.4s, v6.4s, v7.4s
+; CHECK-GI-NEXT:    add v1.4s, v1.4s, v2.4s
+; CHECK-GI-NEXT:    add v0.4s, v1.4s, v0.4s
+; CHECK-GI-NEXT:    addv s0, v0.4s
+; CHECK-GI-NEXT:    fmov w8, s0
+; CHECK-GI-NEXT:    add w0, w8, w2
+; CHECK-GI-NEXT:    ret
+entry:
+  %0 = load <33 x i8>, ptr %a
+  %1 = zext <33 x i8> %0 to <33 x i32>
+  %2 = load <33 x i8>, ptr %b
+  %3 = zext <33 x i8> %2 to <33 x i32>
+  %4 = mul nuw nsw <33 x i32> %3, %1
+  %5 = call i32 @llvm.vector.reduce.add.v33i32(<33 x i32> %4)
+  %op.extra = add i32 %5, %sum
+  ret i32 %op.extra
+}
+
+define i32 @test_udot_v33i8_nomla(ptr nocapture readonly %a1) {
+; CHECK-SD-LABEL: test_udot_v33i8_nomla:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    ldr b1, [x0, #32]
+; CHECK-SD-NEXT:    ldp q3, q2, [x0]
+; CHECK-SD-NEXT:    movi v0.2d, #0000000000000000
+; CHECK-SD-NEXT:    ushll v1.8h, v1.8b, #0
+; CHECK-SD-NEXT:    ushll v4.8h, v2.8b, #0
+; CHECK-SD-NEXT:    ushll v5.8h, v3.8b, #0
+; CHECK-SD-NEXT:    ushll2 v2.8h, v2.16b, #0
+; CHECK-SD-NEXT:    ushll2 v3.8h, v3.16b, #0
+; CHECK-SD-NEXT:    ushll v1.4s, v1.4h, #0
+; CHECK-SD-NEXT:    uaddl2 v6.4s, v5.8h, v4.8h
+; CHECK-SD-NEXT:    mov v0.s[0], v1.s[0]
+; CHECK-SD-NEXT:    uaddl2 v1.4s, v3.8h, v2.8h
+; CHECK-SD-NEXT:    uaddl v2.4s, v3.4h, v2.4h
+; CHECK-SD-NEXT:    add v1.4s, v6.4s, v1.4s
+; CHECK-SD-NEXT:    uaddw v0.4s, v0.4s, v5.4h
+; CHECK-SD-NEXT:    add v1.4s, v2.4s, v1.4s
+; CHECK-SD-NEXT:    uaddw v0.4s, v0.4s, v4.4h
+; CHECK-SD-NEXT:    add v0.4s, v0.4s, v1.4s
+; CHECK-SD-NEXT:    addv s0, v0.4s
+; CHECK-SD-NEXT:    fmov w0, s0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: test_udot_v33i8_nomla:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    str x27, [sp, #-80]! // 8-byte Folded Spill
+; CHECK-GI-NEXT:    stp x26, x25, [sp, #16] // 16-byte Folded Spill
+; CHECK-GI-NEXT:    stp x24, x23, [sp, #32] // 16-byte Folded Spill
+; CHECK-GI-NEXT:    stp x22, x21, [sp, #48] // 16-byte Folded Spill
+; CHECK-GI-NEXT:    stp x20, x19, [sp, #64] // 16-byte Folded Spill
+; CHECK-GI-NEXT:    .cfi_def_cfa_offset 80
+; CHECK-GI-NEXT:    .cfi_offset w19, -8
+; CHECK-GI-NEXT:    .cfi_offset w20, -16
+; CHECK-GI-NEXT:    .cfi_offset w21, -24
+; CHECK-GI-NEXT:    .cfi_offset w22, -32
+; CHECK-GI-NEXT:    .cfi_offset w23, -40
+; CHECK-GI-NEXT:    .cfi_offset w24, -48
+; CHECK-GI-NEXT:    .cfi_offset w25, -56
+; CHECK-GI-NEXT:    .cfi_offset w26, -64
+; CHECK-GI-NEXT:    .cfi_offset w27, -80
+; CHECK-GI-NEXT:    ldp q2, q1, [x0]
+; CHECK-GI-NEXT:    mov v0.s[0], wzr
+; CHECK-GI-NEXT:    ldrb w2, [x0, #32]
+; CHECK-GI-NEXT:    umov w20, v2.b[0]
+; CHECK-GI-NEXT:    umov w21, v2.b[4]
+; CHECK-GI-NEXT:    umov w22, v2.b[8]
+; CHECK-GI-NEXT:    umov w23, v2.b[12]
+; CHECK-GI-NEXT:    umov w24, v1.b[0]
+; CHECK-GI-NEXT:    umov w25, v1.b[4]
+; CHECK-GI-NEXT:    umov w26, v1.b[8]
+; CHECK-GI-NEXT:    umov w27, v1.b[12]
+; CHECK-GI-NEXT:    umov w0, v2.b[1]
+; CHECK-GI-NEXT:    umov w12, v2.b[2]
+; CHECK-GI-NEXT:    umov w8, v2.b[3]
+; CHECK-GI-NEXT:    umov w3, v2.b[5]
+; CHECK-GI-NEXT:    umov w14, v2.b[6]
+; CHECK-GI-NEXT:    umov w9, v2.b[7]
+; CHECK-GI-NEXT:    umov w4, v2.b[9]
+; CHECK-GI-NEXT:    umov w15, v2.b[10]
+; CHECK-GI-NEXT:    umov w10, v2.b[11]
+; CHECK-GI-NEXT:    umov w5, v2.b[13]
+; CHECK-GI-NEXT:    umov w16, v2.b[14]
+; CHECK-GI-NEXT:    umov w11, v2.b[15]
+; CHECK-GI-NEXT:    umov w6, v1.b[1]
+; CHECK-GI-NEXT:    umov w7, v1.b[5]
+; CHECK-GI-NEXT:    umov w19, v1.b[9]
+; CHECK-GI-NEXT:    mov v2.s[0], w20
+; CHECK-GI-NEXT:    mov v3.s[0], w21
+; CHECK-GI-NEXT:    mov v4.s[0], w22
+; CHECK-GI-NEXT:    mov v5.s[0], w23
+; CHECK-GI-NEXT:    mov v6.s[0], w24
+; CHECK-GI-NEXT:    ldp x24, x23, [sp, #32] // 16-byte Folded Reload
+; CHECK-GI-NEXT:    mov v7.s[0], w25
+; CHECK-GI-NEXT:    mov v16.s[0], w26
+; CHECK-GI-NEXT:    umov w20, v1.b[13]
+; CHECK-GI-NEXT:    ldp x26, x25, [sp, #16] // 16-byte Folded Reload
+; CHECK-GI-NEXT:    mov v17.s[0], w27
+; CHECK-GI-NEXT:    mov v18.s[0], w2
+; CHECK-GI-NEXT:    umov w17, v1.b[2]
+; CHECK-GI-NEXT:    umov w1, v1.b[6]
+; CHECK-GI-NEXT:    umov w2, v1.b[10]
+; CHECK-GI-NEXT:    umov w21, v1.b[14]
+; CHECK-GI-NEXT:    mov v2.s[1], w0
+; CHECK-GI-NEXT:    mov v3.s[1], w3
+; CHECK-GI-NEXT:    mov v4.s[1], w4
+; CHECK-GI-NEXT:    mov v5.s[1], w5
+; CHECK-GI-NEXT:    mov v6.s[1], w6
+; CHECK-GI-NEXT:    mov v7.s[1], w7
+; CHECK-GI-NEXT:    mov v16.s[1], w19
+; CHECK-GI-NEXT:    mov v17.s[1], w20
+; CHECK-GI-NEXT:    ldp x20, x19, [sp, #64] // 16-byte Folded Reload
+; CHECK-GI-NEXT:    mov v18.s[1], wzr
+; CHECK-GI-NEXT:    mov v0.s[1], wzr
+; CHECK-GI-NEXT:    umov w13, v1.b[3]
+; CHECK-GI-NEXT:    umov w18, v1.b[7]
+; CHECK-GI-NEXT:    umov w0, v1.b[11]
+; CHECK-GI-NEXT:    umov w3, v1.b[15]
+; CHECK-GI-NEXT:    mov v2.s[2], w12
+; CHECK-GI-NEXT:    mov v3.s[2], w14
+; CHECK-GI-NEXT:    mov v4.s[2], w15
+; CHECK-GI-NEXT:    mov v5.s[2], w16
+; CHECK-GI-NEXT:    mov v6.s[2], w17
+; CHECK-GI-NEXT:    mov v7.s[2], w1
+; CHECK-GI-NEXT:    mov v16.s[2], w2
+; CHECK-GI-NEXT:    mov v17.s[2], w21
+; CHECK-GI-NEXT:    mov v18.s[2], wzr
+; CHECK-GI-NEXT:    ldp x22, x21, [sp, #48] // 16-byte Folded Reload
+; CHECK-GI-NEXT:    mov v0.s[2], wzr
+; CHECK-GI-NEXT:    mov v2.s[3], w8
+; CHECK-GI-NEXT:    mov v3.s[3], w9
+; CHECK-GI-NEXT:    mov v4.s[3], w10
+; CHECK-GI-NEXT:    mov v5.s[3], w11
+; CHECK-GI-NEXT:    mov v6.s[3], w13
+; CHECK-GI-NEXT:    mov v7.s[3], w18
+; CHECK-GI-NEXT:    mov v16.s[3], w0
+; CHECK-GI-NEXT:    mov v17.s[3], w3
+; CHECK-GI-NEXT:    mov v18.s[3], wzr
+; CHECK-GI-NEXT:    mov v0.s[3], wzr
+; CHECK-GI-NEXT:    add v1.4s, v2.4s, v3.4s
+; CHECK-GI-NEXT:    add v2.4s, v4.4s, v5.4s
+; CHECK-GI-NEXT:    add v3.4s, v6.4s, v7.4s
+; CHECK-GI-NEXT:    add v4.4s, v16.4s, v17.4s
+; CHECK-GI-NEXT:    add v5.4s, v18.4s, v0.4s
+; CHECK-GI-NEXT:    add v0.4s, v0.4s, v0.4s
+; CHECK-GI-NEXT:    add v1.4s, v1.4s, v2.4s
+; CHECK-GI-NEXT:    add v2.4s, v3.4s, v4.4s
+; CHECK-GI-NEXT:    add v3.4s, v5.4s, v0.4s
+; CHECK-GI-NEXT:    add v0.4s, v0.4s, v0.4s
+; CHECK-GI-NEXT:    add v1.4s, v1.4s, v2.4s
+; CHECK-GI-NEXT:    add v0.4s, v3.4s, v0.4s
+; CHECK-GI-NEXT:    add v0.4s, v1.4s, v0.4s
+; CHECK-GI-NEXT:    addv s0, v0.4s
+; CHECK-GI-NEXT:    fmov w0, s0
+; CHECK-GI-NEXT:    ldr x27, [sp], #80 // 8-byte Folded Reload
+; CHECK-GI-NEXT:    ret
+entry:
+  %0 = load <33 x i8>, ptr %a1
+  %1 = zext <33 x i8> %0 to <33 x i32>
+  %2 = call i32 @llvm.vector.reduce.add.v33i32(<33 x i32> %1)
+  ret i32 %2
+}
+define i32 @test_sdot_v33i8(ptr nocapture readonly %a, ptr nocapture readonly %b, i32 %sum) {
+; CHECK-SD-LABEL: test_sdot_v33i8:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    ldr b0, [x0, #32]
+; CHECK-SD-NEXT:    ldr b1, [x1, #32]
+; CHECK-SD-NEXT:    movi v7.2d, #0000000000000000
+; CHECK-SD-NEXT:    ldp q3, q4, [x1]
+; CHECK-SD-NEXT:    smull v0.8h, v1.8b, v0.8b
+; CHECK-SD-NEXT:    ldp q1, q2, [x0]
+; CHECK-SD-NEXT:    smull v5.8h, v4.8b, v2.8b
+; CHECK-SD-NEXT:    smull v6.8h, v3.8b, v1.8b
+; CHECK-SD-NEXT:    smull2 v2.8h, v4.16b, v2.16b
+; CHECK-SD-NEXT:    sshll v0.4s, v0.4h, #0
+; CHECK-SD-NEXT:    smull2 v1.8h, v3.16b, v1.16b
+; CHECK-SD-NEXT:    mov v7.s[0], v0.s[0]
+; CHECK-SD-NEXT:    saddl2 v3.4s, v6.8h, v5.8h
+; CHECK-SD-NEXT:    saddl2 v0.4s, v1.8h, v2.8h
+; CHECK-SD-NEXT:    saddl v1.4s, v1.4h, v2.4h
+; CHECK-SD-NEXT:    add v0.4s, v3.4s, v0.4s
+; CHECK-SD-NEXT:    saddw v2.4s, v7.4s, v6.4h
+; CHECK-SD-NEXT:    saddw v2.4s, v2.4s, v5.4h
+; CHECK-SD-NEXT:    add v0.4s, v1.4s, v0.4s
+; CHECK-SD-NEXT:    add v0.4s, v2.4s, v0.4s
+; CHECK-SD-NEXT:    addv s0, v0.4s
+; CHECK-SD-NEXT:    fmov w8, s0
+; CHECK-SD-NEXT:    add w0, w8, w2
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: test_sdot_v33i8:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    ldp q19, q4, [x1]
+; CHECK-GI-NEXT:    mov v25.s[0], wzr
+; CHECK-GI-NEXT:    ldp q7, q5, [x0]
+; CHECK-GI-NEXT:    smov w8, v19.b[0]
+; CHECK-GI-NEXT:    smov w9, v19.b[4]
+; CHECK-GI-NEXT:    smov w10, v19.b[8]
+; CHECK-GI-NEXT:    smov w11, v19.b[1]
+; CHECK-GI-NEXT:    smov w12, v19.b[6]
+; CHECK-GI-NEXT:    smov w13, v19.b[12]
+; CHECK-GI-NEXT:    smov w14, v4.b[0]
+; CHECK-GI-NEXT:    smov w15, v4.b[4]
+; CHECK-GI-NEXT:    smov w16, v4.b[12]
+; CHECK-GI-NEXT:    mov v25.s[1], wzr
+; CHECK-GI-NEXT:    mov v0.s[0], w8
+; CHECK-GI-NEXT:    smov w8, v19.b[5]
+; CHECK-GI-NEXT:    mov v2.s[0], w9
+; CHECK-GI-NEXT:    smov w9, v19.b[9]
+; CHECK-GI-NEXT:    mov v1.s[0], w10
+; CHECK-GI-NEXT:    smov w10, v19.b[2]
+; CHECK-GI-NEXT:    mov v6.s[0], w13
+; CHECK-GI-NEXT:    smov w13, v19.b[3]
+; CHECK-GI-NEXT:    mov v3.s[0], w14
+; CHECK-GI-NEXT:    smov w14, v19.b[13]
+; CHECK-GI-NEXT:    mov v16.s[0], w15
+; CHECK-GI-NEXT:    smov w15, v4.b[8]
+; CHECK-GI-NEXT:    mov v0.s[1], w11
+; CHECK-GI-NEXT:    smov w11, v19.b[10]
+; CHECK-GI-NEXT:    mov v2.s[1], w8
+; CHECK-GI-NEXT:    ldrsb w8, [x0, #32]
+; CHECK-GI-NEXT:    mov v1.s[1], w9
+; CHECK-GI-NEXT:    ldrsb w9, [x1, #32]
+; CHECK-GI-NEXT:    mov v17.s[0], w16
+; CHECK-GI-NEXT:    smov w16, v19.b[14]
+; CHECK-GI-NEXT:    mov v25.s[2], wzr
+; CHECK-GI-NEXT:    mul w8, w9, w8
+; CHECK-GI-NEXT:    mov v6.s[1], w14
+; CHECK-GI-NEXT:    smov w14, v4.b[1]
+; CHECK-GI-NEXT:    mov v0.s[2], w10
+; CHECK-GI-NEXT:    smov w10, v19.b[7]
+; CHECK-GI-NEXT:    mov v2.s[2], w12
+; CHECK-GI-NEXT:    smov w12, v19.b[11]
+; CHECK-GI-NEXT:    mov v1.s[2], w11
+; CHECK-GI-NEXT:    smov w11, v4.b[5]
+; CHECK-GI-NEXT:    mov v18.s[0], w15
+; CHECK-GI-NEXT:    smov w15, v19.b[15]
+; CHECK-GI-NEXT:    smov w9, v5.b[2]
+; CHECK-GI-NEXT:    mov v6.s[2], w16
+; CHECK-GI-NEXT:    smov w16, v7.b[0]
+; CHECK-GI-NEXT:    mov v3.s[1], w14
+; CHECK-GI-NEXT:    mov v0.s[3], w13
+; CHECK-GI-NEXT:    smov w13, v7.b[4]
+; CHECK-GI-NEXT:    mov v2.s[3], w10
+; CHECK-GI-NEXT:    smov w10, v4.b[6]
+; CHECK-GI-NEXT:    mov v1.s[3], w12
+; CHECK-GI-NEXT:    smov w12, v4.b[13]
+; CHECK-GI-NEXT:    mov v16.s[1], w11
+; CHECK-GI-NEXT:    smov w11, v4.b[9]
+; CHECK-GI-NEXT:    smov w14, v7.b[5]
+; CHECK-GI-NEXT:    mov v19.s[0], w16
+; CHECK-GI-NEXT:    smov w16, v7.b[1]
+; CHECK-GI-NEXT:    mov v6.s[3], w15
+; CHECK-GI-NEXT:    mov v20.s[0], w13
+; CHECK-GI-NEXT:    smov w13, v4.b[2]
+; CHECK-GI-NEXT:    smov w15, v7.b[6]
+; CHECK-GI-NEXT:    mov v17.s[1], w12
+; CHECK-GI-NEXT:    smov w12, v4.b[14]
+; CHECK-GI-NEXT:    mov v27.s[0], w8
+; CHECK-GI-NEXT:    mov v16.s[2], w10
+; CHECK-GI-NEXT:    smov w10, v4.b[7]
+; CHECK-GI-NEXT:    mov v18.s[1], w11
+; CHECK-GI-NEXT:    smov w11, v4.b[10]
+; CHECK-GI-NEXT:    mov v19.s[1], w16
+; CHECK-GI-NEXT:    smov w16, v5.b[4]
+; CHECK-GI-NEXT:    mov v20.s[1], w14
+; CHECK-GI-NEXT:    smov w14, v4.b[15]
+; CHECK-GI-NEXT:    mov v3.s[2], w13
+; CHECK-GI-NEXT:    mov v17.s[2], w12
+; CHECK-GI-NEXT:    smov w12, v7.b[12]
+; CHECK-GI-NEXT:    smov w13, v7.b[7]
+; CHECK-GI-NEXT:    mov v16.s[3], w10
+; CHECK-GI-NEXT:    smov w10, v7.b[8]
+; CHECK-GI-NEXT:    smov w8, v7.b[3]
+; CHECK-GI-NEXT:    mov v18.s[2], w11
+; CHECK-GI-NEXT:    smov w11, v7.b[2]
+; CHECK-GI-NEXT:    mov v23.s[0], w16
+; CHECK-GI-NEXT:    mov v20.s[2], w15
+; CHECK-GI-NEXT:    smov w15, v5.b[12]
+; CHECK-GI-NEXT:    smov w16, v7.b[14]
+; CHECK-GI-NEXT:    mov v17.s[3], w14
+; CHECK-GI-NEXT:    smov w14, v7.b[13]
+; CHECK-GI-NEXT:    mov v22.s[0], w12
+; CHECK-GI-NEXT:    smov w12, v7.b[9]
+; CHECK-GI-NEXT:    mov v21.s[0], w10
+; CHECK-GI-NEXT:    smov w10, v4.b[3]
+; CHECK-GI-NEXT:    mov v19.s[2], w11
+; CHECK-GI-NEXT:    smov w11, v5.b[0]
+; CHECK-GI-NEXT:    mov v27.s[1], wzr
+; CHECK-GI-NEXT:    mov v20.s[3], w13
+; CHECK-GI-NEXT:    smov w13, v5.b[5]
+; CHECK-GI-NEXT:    mov v24.s[0], w15
+; CHECK-GI-NEXT:    mov v22.s[1], w14
+; CHECK-GI-NEXT:    smov w14, v5.b[8]
+; CHECK-GI-NEXT:    smov w15, v4.b[11]
+; CHECK-GI-NEXT:    mov v21.s[1], w12
+; CHECK-GI-NEXT:    smov w12, v5.b[13]
+; CHECK-GI-NEXT:    mov v25.s[3], wzr
+; CHECK-GI-NEXT:    mov v4.s[0], w11
+; CHECK-GI-NEXT:    smov w11, v5.b[1]
+; CHECK-GI-NEXT:    mov v3.s[3], w10
+; CHECK-GI-NEXT:    mov v23.s[1], w13
+; CHECK-GI-NEXT:    smov w13, v5.b[6]
+; CHECK-GI-NEXT:    mov v19.s[3], w8
+; CHECK-GI-NEXT:    mov v22.s[2], w16
+; CHECK-GI-NEXT:    smov w16, v5.b[9]
+; CHECK-GI-NEXT:    mov v26.s[0], w14
+; CHECK-GI-NEXT:    mov v24.s[1], w12
+; CHECK-GI-NEXT:    smov w12, v5.b[14]
+; CHECK-GI-NEXT:    smov w14, v7.b[10]
+; CHECK-GI-NEXT:    mov v4.s[1], w11
+; CHECK-GI-NEXT:    smov w11, v7.b[15]
+; CHECK-GI-NEXT:    mov v18.s[3], w15
+; CHECK-GI-NEXT:    mov v23.s[2], w13
+; CHECK-GI-NEXT:    smov w13, v5.b[7]
+; CHECK-GI-NEXT:    mul v2.4s, v2.4s, v20.4s
+; CHECK-GI-NEXT:    mov v26.s[1], w16
+; CHECK-GI-NEXT:    smov w16, v5.b[10]
+; CHECK-GI-NEXT:    mov v27.s[2], wzr
+; CHECK-GI-NEXT:    mov v24.s[2], w12
+; CHECK-GI-NEXT:    smov w12, v5.b[15]
+; CHECK-GI-NEXT:    mov v21.s[2], w14
+; CHECK-GI-NEXT:    smov w14, v7.b[11]
+; CHECK-GI-NEXT:    mov v4.s[2], w9
+; CHECK-GI-NEXT:    smov w9, v5.b[3]
+; CHECK-GI-NEXT:    mov v22.s[3], w11
+; CHECK-GI-NEXT:    smov w11, v5.b[11]
+; CHECK-GI-NEXT:    mov v23.s[3], w13
+; CHECK-GI-NEXT:    mov v26.s[2], w16
+; CHECK-GI-NEXT:    mla v2.4s, v0.4s, v19.4s
+; CHECK-GI-NEXT:    mov v27.s[3], wzr
+; CHECK-GI-NEXT:    mov v24.s[3], w12
+; CHECK-GI-NEXT:    mov v21.s[3], w14
+; CHECK-GI-NEXT:    mov v4.s[3], w9
+; CHECK-GI-NEXT:    mul v5.4s, v6.4s, v22.4s
+; CHECK-GI-NEXT:    mul v6.4s, v16.4s, v23.4s
+; CHECK-GI-NEXT:    add v16.4s, v25.4s, v25.4s
+; CHECK-GI-NEXT:    mov v26.s[3], w11
+; CHECK-GI-NEXT:    mul v7.4s, v17.4s, v24.4s
+; CHECK-GI-NEXT:    add v0.4s, v25.4s, v16.4s
+; CHECK-GI-NEXT:    mla v5.4s, v1.4s, v21.4s
+; CHECK-GI-NEXT:    mla v6.4s, v3.4s, v4.4s
+; CHECK-GI-NEXT:    add v3.4s, v16.4s, v16.4s
+; CHECK-GI-NEXT:    mla v7.4s, v18.4s, v26.4s
+; CHECK-GI-NEXT:    add v0.4s, v27.4s, v0.4s
+; CHECK-GI-NEXT:    add v1.4s, v2.4s, v5.4s
+; CHECK-GI-NEXT:    add v0.4s, v0.4s, v3.4s
+; CHECK-GI-NEXT:    add v2.4s, v6.4s, v7.4s
+; CHECK-GI-NEXT:    add v1.4s, v1.4s, v2.4s
+; CHECK-GI-NEXT:    add v0.4s, v1.4s, v0.4s
+; CHECK-GI-NEXT:    addv s0, v0.4s
+; CHECK-GI-NEXT:    fmov w8, s0
+; CHECK-GI-NEXT:    add w0, w8, w2
+; CHECK-GI-NEXT:    ret
+entry:
+  %0 = load <33 x i8>, ptr %a
+  %1 = sext <33 x i8> %0 to <33 x i32>
+  %2 = load <33 x i8>, ptr %b
+  %3 = sext <33 x i8> %2 to <33 x i32>
+  %4 = mul nsw <33 x i32> %3, %1
+  %5 = call i32 @llvm.vector.reduce.add.v33i32(<33 x i32> %4)
   %op.extra = add nsw i32 %5, %sum
   ret i32 %op.extra
 }
 
-define i32 @test_usdot_v32i8_double(<32 x i8> %a, <32 x i8> %b, <32 x i8> %c, <32 x i8> %d) {
-; CHECK-SD-LABEL: test_usdot_v32i8_double:
+define i32 @test_sdot_v33i8_double(<33 x i8> %a, <33 x i8> %b, <33 x i8> %c, <33 x i8> %d) {
+; CHECK-SD-LABEL: test_sdot_v33i8_double:
 ; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    movi v16.2d, #0000000000000000
-; CHECK-SD-NEXT:    movi v17.2d, #0000000000000000
-; CHECK-SD-NEXT:    movi v18.2d, #0000000000000000
+; CHECK-SD-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-SD-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-SD-NEXT:    .cfi_offset w29, -16
+; CHECK-SD-NEXT:    ldr b0, [sp, #344]
+; CHECK-SD-NEXT:    add x8, sp, #352
+; CHECK-SD-NEXT:    ldr b1, [sp, #80]
+; CHECK-SD-NEXT:    ldr b2, [sp, #216]
+; CHECK-SD-NEXT:    add x9, sp, #96
+; CHECK-SD-NEXT:    add x10, sp, #104
+; CHECK-SD-NEXT:    ld1 { v0.b }[1], [x8]
+; CHECK-SD-NEXT:    add x8, sp, #88
+; CHECK-SD-NEXT:    ldr b4, [sp, #408]
+; CHECK-SD-NEXT:    ld1 { v1.b }[1], [x8]
+; CHECK-SD-NEXT:    add x8, sp, #360
+; CHECK-SD-NEXT:    add x12, sp, #248
+; CHECK-SD-NEXT:    add x13, sp, #432
+; CHECK-SD-NEXT:    add x11, sp, #384
+; CHECK-SD-NEXT:    ldr b5, [sp, #144]
+; CHECK-SD-NEXT:    ld1 { v0.b }[2], [x8]
+; CHECK-SD-NEXT:    add x8, sp, #224
+; CHECK-SD-NEXT:    ldr b6, [sp, #280]
+; CHECK-SD-NEXT:    ld1 { v2.b }[1], [x8]
+; CHECK-SD-NEXT:    ld1 { v1.b }[2], [x9]
+; CHECK-SD-NEXT:    add x8, sp, #368
+; CHECK-SD-NEXT:    add x9, sp, #232
+; CHECK-SD-NEXT:    ldr b16, [sp, #744]
+; CHECK-SD-NEXT:    ldr b17, [sp, #480]
+; CHECK-SD-NEXT:    ld1 { v0.b }[3], [x8]
+; CHECK-SD-NEXT:    add x8, sp, #376
+; CHECK-SD-NEXT:    ldr b18, [sp, #936]
+; CHECK-SD-NEXT:    ld1 { v2.b }[2], [x9]
+; CHECK-SD-NEXT:    ld1 { v1.b }[3], [x10]
+; CHECK-SD-NEXT:    add x9, sp, #240
+; CHECK-SD-NEXT:    add x10, sp, #392
+; CHECK-SD-NEXT:    ldr b19, [sp, #672]
+; CHECK-SD-NEXT:    ldr b7, [sp, #16]
+; CHECK-SD-NEXT:    ld1 { v0.b }[4], [x8]
+; CHECK-SD-NEXT:    add x8, sp, #112
+; CHECK-SD-NEXT:    ldr b21, [sp, #1000]
+; CHECK-SD-NEXT:    ld1 { v2.b }[3], [x9]
+; CHECK-SD-NEXT:    ld1 { v1.b }[4], [x8]
+; CHECK-SD-NEXT:    add x8, sp, #416
+; CHECK-SD-NEXT:    ld1 { v4.b }[1], [x8]
+; CHECK-SD-NEXT:    add x8, sp, #120
+; CHECK-SD-NEXT:    add x9, sp, #400
+; CHECK-SD-NEXT:    ld1 { v0.b }[5], [x11]
+; CHECK-SD-NEXT:    add x11, sp, #128
+; CHECK-SD-NEXT:    ldr b22, [sp, #736]
+; CHECK-SD-NEXT:    ld1 { v2.b }[4], [x12]
+; CHECK-SD-NEXT:    add x12, sp, #424
+; CHECK-SD-NEXT:    ld1 { v1.b }[5], [x8]
+; CHECK-SD-NEXT:    ld1 { v4.b }[2], [x12]
+; CHECK-SD-NEXT:    add x12, sp, #152
+; CHECK-SD-NEXT:    add x8, sp, #136
+; CHECK-SD-NEXT:    ld1 { v5.b }[1], [x12]
+; CHECK-SD-NEXT:    add x12, sp, #440
+; CHECK-SD-NEXT:    ld1 { v0.b }[6], [x10]
+; CHECK-SD-NEXT:    ld1 { v1.b }[6], [x11]
+; CHECK-SD-NEXT:    add x11, sp, #288
+; CHECK-SD-NEXT:    add x10, sp, #256
+; CHECK-SD-NEXT:    ld1 { v4.b }[3], [x13]
+; CHECK-SD-NEXT:    ld1 { v6.b }[1], [x11]
+; CHECK-SD-NEXT:    add x11, sp, #296
+; CHECK-SD-NEXT:    ld1 { v0.b }[7], [x9]
+; CHECK-SD-NEXT:    add x9, sp, #160
+; CHECK-SD-NEXT:    ld1 { v2.b }[5], [x10]
+; CHECK-SD-NEXT:    ld1 { v5.b }[2], [x9]
+; CHECK-SD-NEXT:    add x10, sp, #168
+; CHECK-SD-NEXT:    ld1 { v1.b }[7], [x8]
+; CHECK-SD-NEXT:    ld1 { v4.b }[4], [x12]
+; CHECK-SD-NEXT:    add x12, sp, #448
+; CHECK-SD-NEXT:    ld1 { v6.b }[2], [x11]
+; CHECK-SD-NEXT:    add x11, sp, #304
+; CHECK-SD-NEXT:    add x8, sp, #464
+; CHECK-SD-NEXT:    add x13, sp, #768
+; CHECK-SD-NEXT:    ld1 { v5.b }[3], [x10]
+; CHECK-SD-NEXT:    add x10, sp, #176
+; CHECK-SD-NEXT:    add x9, sp, #264
+; CHECK-SD-NEXT:    ld1 { v4.b }[5], [x12]
+; CHECK-SD-NEXT:    add x12, sp, #456
+; CHECK-SD-NEXT:    ld1 { v6.b }[3], [x11]
+; CHECK-SD-NEXT:    add x11, sp, #760
+; CHECK-SD-NEXT:    ld1 { v2.b }[6], [x9]
+; CHECK-SD-NEXT:    add x9, sp, #272
+; CHECK-SD-NEXT:    ld1 { v5.b }[4], [x10]
+; CHECK-SD-NEXT:    add x10, sp, #312
+; CHECK-SD-NEXT:    fmov s3, w0
+; CHECK-SD-NEXT:    ld1 { v4.b }[6], [x12]
+; CHECK-SD-NEXT:    ld1 { v6.b }[4], [x10]
+; CHECK-SD-NEXT:    add x10, sp, #320
+; CHECK-SD-NEXT:    add x12, sp, #680
+; CHECK-SD-NEXT:    ld1 { v2.b }[7], [x9]
+; CHECK-SD-NEXT:    add x9, sp, #184
+; CHECK-SD-NEXT:    ld1 { v19.b }[1], [x12]
+; CHECK-SD-NEXT:    add x12, sp, #776
+; CHECK-SD-NEXT:    ld1 { v5.b }[5], [x9]
+; CHECK-SD-NEXT:    ld1 { v4.b }[7], [x8]
+; CHECK-SD-NEXT:    add x8, sp, #752
+; CHECK-SD-NEXT:    ld1 { v6.b }[5], [x10]
+; CHECK-SD-NEXT:    ld1 { v16.b }[1], [x8]
+; CHECK-SD-NEXT:    add x10, sp, #24
+; CHECK-SD-NEXT:    smull v22.8h, v22.8b, v21.8b
+; CHECK-SD-NEXT:    ld1 { v7.b }[1], [x10]
+; CHECK-SD-NEXT:    add x10, sp, #496
+; CHECK-SD-NEXT:    mov v3.b[1], w1
+; CHECK-SD-NEXT:    add x9, sp, #192
+; CHECK-SD-NEXT:    ldr b20, [sp, #472]
+; CHECK-SD-NEXT:    ldr b23, [sp, #208]
+; CHECK-SD-NEXT:    ld1 { v16.b }[2], [x11]
+; CHECK-SD-NEXT:    add x11, sp, #488
+; CHECK-SD-NEXT:    ld1 { v5.b }[6], [x9]
+; CHECK-SD-NEXT:    ld1 { v17.b }[1], [x11]
+; CHECK-SD-NEXT:    add x11, sp, #944
+; CHECK-SD-NEXT:    add x9, sp, #328
+; CHECK-SD-NEXT:    ld1 { v18.b }[1], [x11]
+; CHECK-SD-NEXT:    add x11, sp, #688
+; CHECK-SD-NEXT:    ld1 { v6.b }[6], [x9]
+; CHECK-SD-NEXT:    ld1 { v16.b }[3], [x13]
+; CHECK-SD-NEXT:    ld1 { v19.b }[2], [x11]
+; CHECK-SD-NEXT:    add x11, sp, #504
+; CHECK-SD-NEXT:    ld1 { v17.b }[2], [x10]
+; CHECK-SD-NEXT:    add x10, sp, #952
+; CHECK-SD-NEXT:    add x13, sp, #784
+; CHECK-SD-NEXT:    ld1 { v18.b }[2], [x10]
+; CHECK-SD-NEXT:    add x10, sp, #32
+; CHECK-SD-NEXT:    add x9, sp, #40
+; CHECK-SD-NEXT:    ld1 { v16.b }[4], [x12]
+; CHECK-SD-NEXT:    add x12, sp, #696
+; CHECK-SD-NEXT:    ld1 { v7.b }[2], [x10]
+; CHECK-SD-NEXT:    ld1 { v17.b }[3], [x11]
+; CHECK-SD-NEXT:    add x11, sp, #960
+; CHECK-SD-NEXT:    ld1 { v19.b }[3], [x12]
+; CHECK-SD-NEXT:    ld1 { v18.b }[3], [x11]
+; CHECK-SD-NEXT:    add x10, sp, #512
+; CHECK-SD-NEXT:    add x11, sp, #704
+; CHECK-SD-NEXT:    ld1 { v16.b }[5], [x13]
+; CHECK-SD-NEXT:    add x12, sp, #792
+; CHECK-SD-NEXT:    sshll v24.4s, v22.4h, #0
+; CHECK-SD-NEXT:    ld1 { v17.b }[4], [x10]
+; CHECK-SD-NEXT:    add x10, sp, #968
+; CHECK-SD-NEXT:    ld1 { v19.b }[4], [x11]
+; CHECK-SD-NEXT:    ld1 { v18.b }[4], [x10]
+; CHECK-SD-NEXT:    add x10, sp, #520
+; CHECK-SD-NEXT:    add x11, sp, #976
+; CHECK-SD-NEXT:    ld1 { v16.b }[6], [x12]
+; CHECK-SD-NEXT:    add x12, sp, #712
+; CHECK-SD-NEXT:    smull v20.8h, v23.8b, v20.8b
+; CHECK-SD-NEXT:    ld1 { v17.b }[5], [x10]
+; CHECK-SD-NEXT:    ld1 { v19.b }[5], [x12]
+; CHECK-SD-NEXT:    add x12, sp, #720
+; CHECK-SD-NEXT:    ld1 { v18.b }[5], [x11]
+; CHECK-SD-NEXT:    add x11, sp, #528
+; CHECK-SD-NEXT:    add x10, sp, #800
+; CHECK-SD-NEXT:    ld1 { v16.b }[7], [x10]
+; CHECK-SD-NEXT:    add x10, sp, #536
+; CHECK-SD-NEXT:    ldr b22, [sp, #872]
+; CHECK-SD-NEXT:    ld1 { v17.b }[6], [x11]
+; CHECK-SD-NEXT:    add x11, sp, #984
+; CHECK-SD-NEXT:    ld1 { v19.b }[6], [x12]
+; CHECK-SD-NEXT:    ld1 { v18.b }[6], [x11]
+; CHECK-SD-NEXT:    add x11, sp, #992
+; CHECK-SD-NEXT:    add x12, sp, #728
+; CHECK-SD-NEXT:    ldr b23, [sp, #608]
+; CHECK-SD-NEXT:    ld1 { v7.b }[3], [x9]
+; CHECK-SD-NEXT:    add x9, sp, #880
+; CHECK-SD-NEXT:    ld1 { v17.b }[7], [x10]
+; CHECK-SD-NEXT:    ld1 { v19.b }[7], [x12]
+; CHECK-SD-NEXT:    add x10, sp, #816
+; CHECK-SD-NEXT:    ld1 { v18.b }[7], [x11]
+; CHECK-SD-NEXT:    add x11, sp, #552
+; CHECK-SD-NEXT:    add x12, sp, #616
+; CHECK-SD-NEXT:    mov v3.b[2], w2
+; CHECK-SD-NEXT:    ld1 { v22.b }[1], [x9]
+; CHECK-SD-NEXT:    ld1 { v23.b }[1], [x12]
+; CHECK-SD-NEXT:    smull v16.8h, v17.8b, v16.8b
+; CHECK-SD-NEXT:    add x12, sp, #560
+; CHECK-SD-NEXT:    add x9, sp, #888
+; CHECK-SD-NEXT:    smull v17.8h, v19.8b, v18.8b
+; CHECK-SD-NEXT:    ldr b18, [sp, #808]
+; CHECK-SD-NEXT:    ldr b19, [sp, #544]
+; CHECK-SD-NEXT:    add x13, sp, #624
+; CHECK-SD-NEXT:    ld1 { v22.b }[2], [x9]
+; CHECK-SD-NEXT:    add x9, sp, #896
+; CHECK-SD-NEXT:    ld1 { v18.b }[1], [x10]
+; CHECK-SD-NEXT:    ld1 { v19.b }[1], [x11]
+; CHECK-SD-NEXT:    add x11, sp, #824
+; CHECK-SD-NEXT:    add x10, sp, #48
+; CHECK-SD-NEXT:    ld1 { v23.b }[2], [x13]
+; CHECK-SD-NEXT:    mov v3.b[3], w3
+; CHECK-SD-NEXT:    ld1 { v7.b }[4], [x10]
+; CHECK-SD-NEXT:    add x10, sp, #832
+; CHECK-SD-NEXT:    ld1 { v22.b }[3], [x9]
+; CHECK-SD-NEXT:    ld1 { v18.b }[2], [x11]
+; CHECK-SD-NEXT:    ld1 { v19.b }[2], [x12]
+; CHECK-SD-NEXT:    add x11, sp, #568
+; CHECK-SD-NEXT:    add x12, sp, #632
+; CHECK-SD-NEXT:    add x9, sp, #904
+; CHECK-SD-NEXT:    add x13, sp, #640
+; CHECK-SD-NEXT:    ld1 { v23.b }[3], [x12]
+; CHECK-SD-NEXT:    add x12, sp, #576
+; CHECK-SD-NEXT:    mov v3.b[4], w4
+; CHECK-SD-NEXT:    ld1 { v18.b }[3], [x10]
+; CHECK-SD-NEXT:    ld1 { v19.b }[3], [x11]
+; CHECK-SD-NEXT:    add x11, sp, #840
+; CHECK-SD-NEXT:    add x10, sp, #56
+; CHECK-SD-NEXT:    ld1 { v22.b }[4], [x9]
+; CHECK-SD-NEXT:    add x9, sp, #912
+; CHECK-SD-NEXT:    ld1 { v23.b }[4], [x13]
+; CHECK-SD-NEXT:    ld1 { v7.b }[5], [x10]
+; CHECK-SD-NEXT:    add x10, sp, #848
+; CHECK-SD-NEXT:    ld1 { v18.b }[4], [x11]
+; CHECK-SD-NEXT:    ld1 { v19.b }[4], [x12]
+; CHECK-SD-NEXT:    add x11, sp, #584
+; CHECK-SD-NEXT:    add x12, sp, #648
+; CHECK-SD-NEXT:    mov v3.b[5], w5
+; CHECK-SD-NEXT:    ld1 { v22.b }[5], [x9]
+; CHECK-SD-NEXT:    ld1 { v23.b }[5], [x12]
+; CHECK-SD-NEXT:    add x12, sp, #592
+; CHECK-SD-NEXT:    movi v21.2d, #0000000000000000
+; CHECK-SD-NEXT:    ld1 { v18.b }[5], [x10]
+; CHECK-SD-NEXT:    ld1 { v19.b }[5], [x11]
+; CHECK-SD-NEXT:    add x11, sp, #856
+; CHECK-SD-NEXT:    add x9, sp, #920
+; CHECK-SD-NEXT:    add x13, sp, #656
+; CHECK-SD-NEXT:    add x10, sp, #64
+; CHECK-SD-NEXT:    ld1 { v22.b }[6], [x9]
+; CHECK-SD-NEXT:    ld1 { v23.b }[6], [x13]
+; CHECK-SD-NEXT:    mov v3.b[6], w6
+; CHECK-SD-NEXT:    ld1 { v18.b }[6], [x11]
+; CHECK-SD-NEXT:    ld1 { v19.b }[6], [x12]
+; CHECK-SD-NEXT:    ld1 { v7.b }[6], [x10]
+; CHECK-SD-NEXT:    add x10, sp, #864
+; CHECK-SD-NEXT:    add x11, sp, #600
+; CHECK-SD-NEXT:    add x9, sp, #928
+; CHECK-SD-NEXT:    add x12, sp, #664
+; CHECK-SD-NEXT:    mov v21.s[0], v24.s[0]
+; CHECK-SD-NEXT:    ld1 { v22.b }[7], [x9]
+; CHECK-SD-NEXT:    ld1 { v18.b }[7], [x10]
+; CHECK-SD-NEXT:    ld1 { v19.b }[7], [x11]
+; CHECK-SD-NEXT:    ld1 { v23.b }[7], [x12]
+; CHECK-SD-NEXT:    add x8, sp, #200
+; CHECK-SD-NEXT:    mov v3.b[7], w7
+; CHECK-SD-NEXT:    add x10, sp, #336
+; CHECK-SD-NEXT:    ld1 { v5.b }[7], [x8]
+; CHECK-SD-NEXT:    add x8, sp, #72
+; CHECK-SD-NEXT:    ld1 { v6.b }[7], [x10]
+; CHECK-SD-NEXT:    smull v18.8h, v19.8b, v18.8b
 ; CHECK-SD-NEXT:    movi v19.2d, #0000000000000000
-; CHECK-SD-NEXT:    usdot v16.4s, v1.16b, v3.16b
-; CHECK-SD-NEXT:    usdot v18.4s, v0.16b, v2.16b
-; CHECK-SD-NEXT:    usdot v17.4s, v4.16b, v6.16b
-; CHECK-SD-NEXT:    usdot v19.4s, v5.16b, v7.16b
-; CHECK-SD-NEXT:    add v0.4s, v18.4s, v16.4s
-; CHECK-SD-NEXT:    add v1.4s, v17.4s, v19.4s
+; CHECK-SD-NEXT:    ld1 { v7.b }[7], [x8]
+; CHECK-SD-NEXT:    smull v22.8h, v23.8b, v22.8b
+; CHECK-SD-NEXT:    sshll v20.4s, v20.4h, #0
+; CHECK-SD-NEXT:    smull v0.8h, v1.8b, v0.8b
+; CHECK-SD-NEXT:    saddw v1.4s, v21.4s, v16.4h
+; CHECK-SD-NEXT:    smull v2.8h, v3.8b, v2.8b
+; CHECK-SD-NEXT:    smull v3.8h, v5.8b, v4.8b
+; CHECK-SD-NEXT:    smull v4.8h, v7.8b, v6.8b
+; CHECK-SD-NEXT:    mov v19.s[0], v20.s[0]
+; CHECK-SD-NEXT:    saddl2 v5.4s, v18.8h, v17.8h
+; CHECK-SD-NEXT:    saddl v7.4s, v18.4h, v17.4h
+; CHECK-SD-NEXT:    saddl2 v6.4s, v16.8h, v22.8h
+; CHECK-SD-NEXT:    saddw v1.4s, v1.4s, v22.4h
+; CHECK-SD-NEXT:    saddl2 v17.4s, v2.8h, v0.8h
+; CHECK-SD-NEXT:    saddl2 v16.4s, v4.8h, v3.8h
+; CHECK-SD-NEXT:    saddl v3.4s, v4.4h, v3.4h
+; CHECK-SD-NEXT:    saddw v2.4s, v19.4s, v2.4h
+; CHECK-SD-NEXT:    add v5.4s, v6.4s, v5.4s
+; CHECK-SD-NEXT:    add v1.4s, v1.4s, v7.4s
+; CHECK-SD-NEXT:    add v6.4s, v17.4s, v16.4s
+; CHECK-SD-NEXT:    saddw v0.4s, v2.4s, v0.4h
+; CHECK-SD-NEXT:    add v1.4s, v1.4s, v5.4s
+; CHECK-SD-NEXT:    add v0.4s, v0.4s, v3.4s
+; CHECK-SD-NEXT:    add v1.4s, v6.4s, v1.4s
 ; CHECK-SD-NEXT:    add v0.4s, v0.4s, v1.4s
 ; CHECK-SD-NEXT:    addv s0, v0.4s
 ; CHECK-SD-NEXT:    fmov w0, s0
+; CHECK-SD-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
 ; CHECK-SD-NEXT:    ret
 ;
-; CHECK-GI-LABEL: test_usdot_v32i8_double:
+; CHECK-GI-LABEL: test_sdot_v33i8_double:
 ; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    stp d15, d14, [sp, #-64]! // 16-byte Folded Spill
+; CHECK-GI-NEXT:    stp d15, d14, [sp, #-80]! // 16-byte Folded Spill
 ; CHECK-GI-NEXT:    stp d13, d12, [sp, #16] // 16-byte Folded Spill
 ; CHECK-GI-NEXT:    stp d11, d10, [sp, #32] // 16-byte Folded Spill
 ; CHECK-GI-NEXT:    stp d9, d8, [sp, #48] // 16-byte Folded Spill
-; CHECK-GI-NEXT:    .cfi_def_cfa_offset 64
-; CHECK-GI-NEXT:    .cfi_offset b8, -8
-; CHECK-GI-NEXT:    .cfi_offset b9, -16
-; CHECK-GI-NEXT:    .cfi_offset b10, -24
-; CHECK-GI-NEXT:    .cfi_offset b11, -32
-; CHECK-GI-NEXT:    .cfi_offset b12, -40
-; CHECK-GI-NEXT:    .cfi_offset b13, -48
-; CHECK-GI-NEXT:    .cfi_offset b14, -56
-; CHECK-GI-NEXT:    .cfi_offset b15, -64
-; CHECK-GI-NEXT:    ushll v16.8h, v0.8b, #0
-; CHECK-GI-NEXT:    ushll2 v0.8h, v0.16b, #0
-; CHECK-GI-NEXT:    ushll v17.8h, v1.8b, #0
-; CHECK-GI-NEXT:    ushll2 v1.8h, v1.16b, #0
-; CHECK-GI-NEXT:    sshll v18.8h, v2.8b, #0
-; CHECK-GI-NEXT:    sshll2 v2.8h, v2.16b, #0
-; CHECK-GI-NEXT:    sshll v19.8h, v3.8b, #0
-; CHECK-GI-NEXT:    sshll2 v3.8h, v3.16b, #0
-; CHECK-GI-NEXT:    ushll v27.8h, v4.8b, #0
-; CHECK-GI-NEXT:    ushll2 v4.8h, v4.16b, #0
-; CHECK-GI-NEXT:    ushll v28.8h, v5.8b, #0
-; CHECK-GI-NEXT:    sshll v29.8h, v6.8b, #0
-; CHECK-GI-NEXT:    sshll2 v6.8h, v6.16b, #0
-; CHECK-GI-NEXT:    ushll2 v5.8h, v5.16b, #0
-; CHECK-GI-NEXT:    sshll v30.8h, v7.8b, #0
-; CHECK-GI-NEXT:    sshll2 v7.8h, v7.16b, #0
-; CHECK-GI-NEXT:    ushll2 v20.4s, v16.8h, #0
-; CHECK-GI-NEXT:    ushll2 v21.4s, v0.8h, #0
-; CHECK-GI-NEXT:    ushll2 v22.4s, v17.8h, #0
-; CHECK-GI-NEXT:    ushll2 v23.4s, v1.8h, #0
-; CHECK-GI-NEXT:    sshll2 v24.4s, v18.8h, #0
-; CHECK-GI-NEXT:    sshll2 v25.4s, v2.8h, #0
-; CHECK-GI-NEXT:    sshll2 v26.4s, v19.8h, #0
-; CHECK-GI-NEXT:    sshll2 v31.4s, v3.8h, #0
-; CHECK-GI-NEXT:    ushll2 v8.4s, v27.8h, #0
-; CHECK-GI-NEXT:    ushll2 v9.4s, v4.8h, #0
-; CHECK-GI-NEXT:    ushll2 v10.4s, v28.8h, #0
-; CHECK-GI-NEXT:    sshll2 v11.4s, v29.8h, #0
-; CHECK-GI-NEXT:    sshll2 v12.4s, v6.8h, #0
-; CHECK-GI-NEXT:    ushll2 v13.4s, v5.8h, #0
-; CHECK-GI-NEXT:    sshll2 v14.4s, v30.8h, #0
-; CHECK-GI-NEXT:    sshll2 v15.4s, v7.8h, #0
-; CHECK-GI-NEXT:    mul v20.4s, v20.4s, v24.4s
-; CHECK-GI-NEXT:    mul v21.4s, v21.4s, v25.4s
-; CHECK-GI-NEXT:    mul v22.4s, v22.4s, v26.4s
-; CHECK-GI-NEXT:    mul v23.4s, v23.4s, v31.4s
-; CHECK-GI-NEXT:    mul v24.4s, v8.4s, v11.4s
-; CHECK-GI-NEXT:    mul v25.4s, v9.4s, v12.4s
-; CHECK-GI-NEXT:    ushll v16.4s, v16.4h, #0
+; CHECK-GI-NEXT:    str x29, [sp, #64] // 8-byte Folded Spill
+; CHECK-GI-NEXT:    .cfi_def_cfa_offset 80
+; CHECK-GI-NEXT:    .cfi_offset w29, -16
+; CHECK-GI-NEXT:    .cfi_offset b8, -24
+; CHECK-GI-NEXT:    .cfi_offset b9, -32
+; CHECK-GI-NEXT:    .cfi_offset b10, -40
+; CHECK-GI-NEXT:    .cfi_offset b11, -48
+; CHECK-GI-NEXT:    .cfi_offset b12, -56
+; CHECK-GI-NEXT:    .cfi_offset b13, -64
+; CHECK-GI-NEXT:    .cfi_offset b14, -72
+; CHECK-GI-NEXT:    .cfi_offset b15, -80
+; CHECK-GI-NEXT:    ldr w8, [sp, #80]
+; CHECK-GI-NEXT:    sxtb w9, w0
+; CHECK-GI-NEXT:    ldr w10, [sp, #112]
+; CHECK-GI-NEXT:    sxtb w11, w4
+; CHECK-GI-NEXT:    sxtb w13, w7
+; CHECK-GI-NEXT:    sxtb w12, w3
+; CHECK-GI-NEXT:    sxtb w8, w8
+; CHECK-GI-NEXT:    mov v0.s[0], w9
+; CHECK-GI-NEXT:    sxtb w9, w10
+; CHECK-GI-NEXT:    mov v3.s[0], w11
+; CHECK-GI-NEXT:    sxtb w10, w1
+; CHECK-GI-NEXT:    sxtb w11, w5
+; CHECK-GI-NEXT:    mov v1.s[0], w8
+; CHECK-GI-NEXT:    ldr w8, [sp, #88]
+; CHECK-GI-NEXT:    mov v5.s[0], w9
+; CHECK-GI-NEXT:    ldr w9, [sp, #120]
+; CHECK-GI-NEXT:    ldr w14, [sp, #168]
+; CHECK-GI-NEXT:    ldr x29, [sp, #64] // 8-byte Folded Reload
+; CHECK-GI-NEXT:    sxtb w8, w8
+; CHECK-GI-NEXT:    mov v0.s[1], w10
+; CHECK-GI-NEXT:    sxtb w10, w2
+; CHECK-GI-NEXT:    mov v3.s[1], w11
+; CHECK-GI-NEXT:    sxtb w9, w9
+; CHECK-GI-NEXT:    sxtb w11, w6
+; CHECK-GI-NEXT:    mov v1.s[1], w8
+; CHECK-GI-NEXT:    ldr w8, [sp, #96]
+; CHECK-GI-NEXT:    mov v5.s[1], w9
+; CHECK-GI-NEXT:    ldr w9, [sp, #128]
+; CHECK-GI-NEXT:    sxtb w8, w8
+; CHECK-GI-NEXT:    mov v0.s[2], w10
+; CHECK-GI-NEXT:    ldr w10, [sp, #104]
+; CHECK-GI-NEXT:    mov v3.s[2], w11
+; CHECK-GI-NEXT:    sxtb w9, w9
+; CHECK-GI-NEXT:    ldr w11, [sp, #136]
+; CHECK-GI-NEXT:    mov v1.s[2], w8
+; CHECK-GI-NEXT:    ldr w8, [sp, #144]
+; CHECK-GI-NEXT:    sxtb w10, w10
+; CHECK-GI-NEXT:    mov v5.s[2], w9
+; CHECK-GI-NEXT:    sxtb w11, w11
+; CHECK-GI-NEXT:    ldr w9, [sp, #152]
+; CHECK-GI-NEXT:    sxtb w8, w8
+; CHECK-GI-NEXT:    mov v0.s[3], w12
+; CHECK-GI-NEXT:    ldr w12, [sp, #160]
+; CHECK-GI-NEXT:    mov v3.s[3], w13
+; CHECK-GI-NEXT:    ldr w13, [sp, #176]
+; CHECK-GI-NEXT:    sxtb w9, w9
+; CHECK-GI-NEXT:    mov v1.s[3], w10
+; CHECK-GI-NEXT:    ldr w10, [sp, #208]
+; CHECK-GI-NEXT:    mov v2.s[0], w8
+; CHECK-GI-NEXT:    mov v5.s[3], w11
+; CHECK-GI-NEXT:    ldr w11, [sp, #240]
+; CHECK-GI-NEXT:    sxtb w13, w13
+; CHECK-GI-NEXT:    sxtb w8, w10
+; CHECK-GI-NEXT:    ldr w10, [sp, #184]
+; CHECK-GI-NEXT:    sxtb w12, w12
+; CHECK-GI-NEXT:    mov v6.s[0], w13
+; CHECK-GI-NEXT:    sxtb w11, w11
+; CHECK-GI-NEXT:    ldr w13, [sp, #264]
+; CHECK-GI-NEXT:    mov v4.s[0], w8
+; CHECK-GI-NEXT:    ldr w8, [sp, #216]
+; CHECK-GI-NEXT:    sxtb w10, w10
+; CHECK-GI-NEXT:    mov v7.s[0], w11
+; CHECK-GI-NEXT:    ldr w11, [sp, #248]
+; CHECK-GI-NEXT:    mov v2.s[1], w9
+; CHECK-GI-NEXT:    sxtb w8, w8
+; CHECK-GI-NEXT:    ldr w9, [sp, #192]
+; CHECK-GI-NEXT:    mov v6.s[1], w10
+; CHECK-GI-NEXT:    sxtb w11, w11
+; CHECK-GI-NEXT:    ldr w10, [sp, #200]
+; CHECK-GI-NEXT:    mov v4.s[1], w8
+; CHECK-GI-NEXT:    ldr w8, [sp, #224]
+; CHECK-GI-NEXT:    sxtb w9, w9
+; CHECK-GI-NEXT:    mov v7.s[1], w11
+; CHECK-GI-NEXT:    ldr w11, [sp, #256]
+; CHECK-GI-NEXT:    mov v2.s[2], w12
+; CHECK-GI-NEXT:    sxtb w8, w8
+; CHECK-GI-NEXT:    ldr w12, [sp, #232]
+; CHECK-GI-NEXT:    sxtb w10, w10
+; CHECK-GI-NEXT:    mov v6.s[2], w9
+; CHECK-GI-NEXT:    sxtb w9, w11
+; CHECK-GI-NEXT:    sxtb w11, w14
+; CHECK-GI-NEXT:    mov v4.s[2], w8
+; CHECK-GI-NEXT:    ldr w14, [sp, #280]
+; CHECK-GI-NEXT:    ldr w8, [sp, #272]
+; CHECK-GI-NEXT:    mov v7.s[2], w9
+; CHECK-GI-NEXT:    mov v2.s[3], w11
+; CHECK-GI-NEXT:    sxtb w11, w12
+; CHECK-GI-NEXT:    sxtb w12, w13
+; CHECK-GI-NEXT:    sxtb w13, w14
+; CHECK-GI-NEXT:    ldr w9, [sp, #288]
+; CHECK-GI-NEXT:    mov v6.s[3], w10
+; CHECK-GI-NEXT:    ldr w10, [sp, #312]
+; CHECK-GI-NEXT:    ldr w14, [sp, #544]
+; CHECK-GI-NEXT:    mov v4.s[3], w11
+; CHECK-GI-NEXT:    ldr w11, [sp, #344]
+; CHECK-GI-NEXT:    mov v16.s[0], w13
+; CHECK-GI-NEXT:    mov v7.s[3], w12
+; CHECK-GI-NEXT:    ldr w12, [sp, #376]
+; CHECK-GI-NEXT:    sxtb w10, w10
+; CHECK-GI-NEXT:    sxtb w11, w11
+; CHECK-GI-NEXT:    sxtb w9, w9
+; CHECK-GI-NEXT:    ldr w13, [sp, #296]
+; CHECK-GI-NEXT:    mov v19.s[0], w10
+; CHECK-GI-NEXT:    ldr w10, [sp, #320]
+; CHECK-GI-NEXT:    sxtb w12, w12
+; CHECK-GI-NEXT:    mov v17.s[0], w11
+; CHECK-GI-NEXT:    ldr w11, [sp, #352]
+; CHECK-GI-NEXT:    mov v16.s[1], w9
+; CHECK-GI-NEXT:    mov v21.s[0], w12
+; CHECK-GI-NEXT:    ldr w12, [sp, #384]
+; CHECK-GI-NEXT:    sxtb w10, w10
+; CHECK-GI-NEXT:    sxtb w11, w11
+; CHECK-GI-NEXT:    sxtb w13, w13
+; CHECK-GI-NEXT:    ldr w9, [sp, #304]
+; CHECK-GI-NEXT:    mov v19.s[1], w10
+; CHECK-GI-NEXT:    ldr w10, [sp, #328]
+; CHECK-GI-NEXT:    sxtb w12, w12
+; CHECK-GI-NEXT:    mov v17.s[1], w11
+; CHECK-GI-NEXT:    ldr w11, [sp, #360]
+; CHECK-GI-NEXT:    mov v16.s[2], w13
+; CHECK-GI-NEXT:    mov v21.s[1], w12
+; CHECK-GI-NEXT:    ldr w12, [sp, #392]
+; CHECK-GI-NEXT:    sxtb w10, w10
+; CHECK-GI-NEXT:    sxtb w11, w11
+; CHECK-GI-NEXT:    ldr w13, [sp, #336]
+; CHECK-GI-NEXT:    sxtb w9, w9
+; CHECK-GI-NEXT:    mov v19.s[2], w10
+; CHECK-GI-NEXT:    ldr w10, [sp, #368]
+; CHECK-GI-NEXT:    sxtb w12, w12
+; CHECK-GI-NEXT:    mov v17.s[2], w11
+; CHECK-GI-NEXT:    sxtb w13, w13
+; CHECK-GI-NEXT:    ldr w11, [sp, #400]
+; CHECK-GI-NEXT:    mov v21.s[2], w12
+; CHECK-GI-NEXT:    ldr w12, [sp, #408]
+; CHECK-GI-NEXT:    sxtb w10, w10
+; CHECK-GI-NEXT:    sxtb w11, w11
+; CHECK-GI-NEXT:    mov v16.s[3], w9
+; CHECK-GI-NEXT:    ldr w9, [sp, #416]
+; CHECK-GI-NEXT:    mov v19.s[3], w13
+; CHECK-GI-NEXT:    ldr w13, [sp, #440]
+; CHECK-GI-NEXT:    sxtb w12, w12
+; CHECK-GI-NEXT:    mov v17.s[3], w10
+; CHECK-GI-NEXT:    ldr w10, [sp, #472]
+; CHECK-GI-NEXT:    sxtb w9, w9
+; CHECK-GI-NEXT:    sxtb w13, w13
+; CHECK-GI-NEXT:    mov v18.s[0], w12
+; CHECK-GI-NEXT:    mov v21.s[3], w11
+; CHECK-GI-NEXT:    sxtb w10, w10
+; CHECK-GI-NEXT:    ldr w11, [sp, #504]
+; CHECK-GI-NEXT:    ldr w12, [sp, #424]
+; CHECK-GI-NEXT:    mov v22.s[0], w13
+; CHECK-GI-NEXT:    ldr w13, [sp, #448]
+; CHECK-GI-NEXT:    sxtb w8, w8
+; CHECK-GI-NEXT:    mov v20.s[0], w10
+; CHECK-GI-NEXT:    ldr w10, [sp, #480]
+; CHECK-GI-NEXT:    sxtb w11, w11
+; CHECK-GI-NEXT:    sxtb w13, w13
+; CHECK-GI-NEXT:    mov v18.s[1], w9
+; CHECK-GI-NEXT:    sxtb w12, w12
+; CHECK-GI-NEXT:    sxtb w10, w10
+; CHECK-GI-NEXT:    mov v23.s[0], w11
+; CHECK-GI-NEXT:    ldr w11, [sp, #512]
+; CHECK-GI-NEXT:    mov v22.s[1], w13
+; CHECK-GI-NEXT:    ldr w13, [sp, #456]
+; CHECK-GI-NEXT:    ldr w9, [sp, #432]
+; CHECK-GI-NEXT:    mov v20.s[1], w10
+; CHECK-GI-NEXT:    ldr w10, [sp, #488]
+; CHECK-GI-NEXT:    sxtb w11, w11
+; CHECK-GI-NEXT:    sxtb w13, w13
+; CHECK-GI-NEXT:    mov v18.s[2], w12
+; CHECK-GI-NEXT:    ldr w12, [sp, #464]
+; CHECK-GI-NEXT:    sxtb w10, w10
+; CHECK-GI-NEXT:    mov v23.s[1], w11
+; CHECK-GI-NEXT:    ldr w11, [sp, #520]
+; CHECK-GI-NEXT:    mov v22.s[2], w13
+; CHECK-GI-NEXT:    ldr w13, [sp, #496]
+; CHECK-GI-NEXT:    sxtb w9, w9
+; CHECK-GI-NEXT:    mov v20.s[2], w10
+; CHECK-GI-NEXT:    sxtb w11, w11
+; CHECK-GI-NEXT:    sxtb w12, w12
+; CHECK-GI-NEXT:    mov v18.s[3], w9
+; CHECK-GI-NEXT:    sxtb w9, w13
+; CHECK-GI-NEXT:    ldr w10, [sp, #528]
+; CHECK-GI-NEXT:    mov v23.s[2], w11
+; CHECK-GI-NEXT:    ldr w11, [sp, #536]
+; CHECK-GI-NEXT:    sxtb w13, w14
+; CHECK-GI-NEXT:    mov v22.s[3], w12
+; CHECK-GI-NEXT:    ldr w12, [sp, #576]
+; CHECK-GI-NEXT:    sxtb w10, w10
+; CHECK-GI-NEXT:    mov v20.s[3], w9
+; CHECK-GI-NEXT:    ldr w9, [sp, #608]
+; CHECK-GI-NEXT:    sxtb w11, w11
+; CHECK-GI-NEXT:    sxtb w12, w12
+; CHECK-GI-NEXT:    mov v24.s[0], w13
+; CHECK-GI-NEXT:    ldr w13, [sp, #560]
+; CHECK-GI-NEXT:    sxtb w9, w9
+; CHECK-GI-NEXT:    mul w8, w8, w11
+; CHECK-GI-NEXT:    mov v23.s[3], w10
+; CHECK-GI-NEXT:    ldr w10, [sp, #552]
+; CHECK-GI-NEXT:    ldr w11, [sp, #584]
+; CHECK-GI-NEXT:    mov v27.s[0], w12
+; CHECK-GI-NEXT:    mov v26.s[0], w9
+; CHECK-GI-NEXT:    ldr w9, [sp, #616]
+; CHECK-GI-NEXT:    sxtb w13, w13
+; CHECK-GI-NEXT:    sxtb w10, w10
+; CHECK-GI-NEXT:    sxtb w11, w11
+; CHECK-GI-NEXT:    mov v25.s[0], w8
+; CHECK-GI-NEXT:    ldr w8, [sp, #640]
+; CHECK-GI-NEXT:    sxtb w9, w9
+; CHECK-GI-NEXT:    ldr w12, [sp, #568]
+; CHECK-GI-NEXT:    mov v24.s[1], w10
+; CHECK-GI-NEXT:    ldr w10, [sp, #592]
+; CHECK-GI-NEXT:    mov v27.s[1], w11
+; CHECK-GI-NEXT:    mov v26.s[1], w9
+; CHECK-GI-NEXT:    ldr w9, [sp, #624]
+; CHECK-GI-NEXT:    sxtb w8, w8
+; CHECK-GI-NEXT:    sxtb w10, w10
+; CHECK-GI-NEXT:    ldr w11, [sp, #600]
+; CHECK-GI-NEXT:    sxtb w12, w12
+; CHECK-GI-NEXT:    mov v28.s[0], w8
+; CHECK-GI-NEXT:    ldr w8, [sp, #648]
+; CHECK-GI-NEXT:    sxtb w9, w9
+; CHECK-GI-NEXT:    mov v24.s[2], w13
+; CHECK-GI-NEXT:    ldr w13, [sp, #632]
+; CHECK-GI-NEXT:    mov v27.s[2], w10
+; CHECK-GI-NEXT:    mov v26.s[2], w9
+; CHECK-GI-NEXT:    sxtb w8, w8
+; CHECK-GI-NEXT:    ldr w10, [sp, #656]
+; CHECK-GI-NEXT:    sxtb w11, w11
+; CHECK-GI-NEXT:    sxtb w13, w13
+; CHECK-GI-NEXT:    ldr w9, [sp, #664]
+; CHECK-GI-NEXT:    mov v28.s[1], w8
+; CHECK-GI-NEXT:    sxtb w10, w10
+; CHECK-GI-NEXT:    ldr w8, [sp, #680]
+; CHECK-GI-NEXT:    mov v24.s[3], w12
+; CHECK-GI-NEXT:    ldr w12, [sp, #672]
+; CHECK-GI-NEXT:    mov v27.s[3], w11
+; CHECK-GI-NEXT:    ldr w11, [sp, #704]
+; CHECK-GI-NEXT:    mov v26.s[3], w13
+; CHECK-GI-NEXT:    ldr w13, [sp, #736]
+; CHECK-GI-NEXT:    sxtb w12, w12
+; CHECK-GI-NEXT:    sxtb w9, w9
+; CHECK-GI-NEXT:    sxtb w8, w8
+; CHECK-GI-NEXT:    sxtb w11, w11
+; CHECK-GI-NEXT:    mov v28.s[2], w10
+; CHECK-GI-NEXT:    sxtb w13, w13
+; CHECK-GI-NEXT:    mov v29.s[0], w12
+; CHECK-GI-NEXT:    ldr w12, [sp, #688]
+; CHECK-GI-NEXT:    ldr w10, [sp, #696]
+; CHECK-GI-NEXT:    mov v31.s[0], w11
+; CHECK-GI-NEXT:    ldr w11, [sp, #712]
+; CHECK-GI-NEXT:    mov v30.s[0], w13
+; CHECK-GI-NEXT:    ldr w13, [sp, #744]
+; CHECK-GI-NEXT:    sxtb w12, w12
+; CHECK-GI-NEXT:    ldr w14, [sp, #776]
+; CHECK-GI-NEXT:    sxtb w11, w11
+; CHECK-GI-NEXT:    mov v28.s[3], w9
+; CHECK-GI-NEXT:    ldr w9, [sp, #768]
+; CHECK-GI-NEXT:    sxtb w13, w13
+; CHECK-GI-NEXT:    mov v29.s[1], w8
+; CHECK-GI-NEXT:    ldr w8, [sp, #720]
+; CHECK-GI-NEXT:    mov v31.s[1], w11
+; CHECK-GI-NEXT:    sxtb w9, w9
+; CHECK-GI-NEXT:    ldr w11, [sp, #728]
+; CHECK-GI-NEXT:    mov v30.s[1], w13
+; CHECK-GI-NEXT:    ldr w13, [sp, #752]
+; CHECK-GI-NEXT:    sxtb w8, w8
+; CHECK-GI-NEXT:    mov v8.s[0], w9
+; CHECK-GI-NEXT:    sxtb w11, w11
+; CHECK-GI-NEXT:    mul v3.4s, v3.4s, v19.4s
+; CHECK-GI-NEXT:    sxtb w9, w13
+; CHECK-GI-NEXT:    mov v29.s[2], w12
+; CHECK-GI-NEXT:    ldr w12, [sp, #760]
+; CHECK-GI-NEXT:    mov v31.s[2], w8
+; CHECK-GI-NEXT:    sxtb w8, w10
+; CHECK-GI-NEXT:    sxtb w10, w14
+; CHECK-GI-NEXT:    mov v30.s[2], w9
+; CHECK-GI-NEXT:    ldr w14, [sp, #808]
+; CHECK-GI-NEXT:    ldr w13, [sp, #784]
+; CHECK-GI-NEXT:    mov v8.s[1], w10
+; CHECK-GI-NEXT:    sxtb w10, w12
+; CHECK-GI-NEXT:    ldr w9, [sp, #792]
+; CHECK-GI-NEXT:    sxtb w12, w14
+; CHECK-GI-NEXT:    sxtb w13, w13
+; CHECK-GI-NEXT:    mul v5.4s, v5.4s, v21.4s
+; CHECK-GI-NEXT:    mov v31.s[3], w11
+; CHECK-GI-NEXT:    ldr w11, [sp, #840]
+; CHECK-GI-NEXT:    sxtb w9, w9
+; CHECK-GI-NEXT:    mov v30.s[3], w10
+; CHECK-GI-NEXT:    ldr w10, [sp, #872]
+; CHECK-GI-NEXT:    mov v9.s[0], w12
+; CHECK-GI-NEXT:    ldr w12, [sp, #816]
+; CHECK-GI-NEXT:    sxtb w11, w11
+; CHECK-GI-NEXT:    mov v8.s[2], w13
+; CHECK-GI-NEXT:    sxtb w10, w10
+; CHECK-GI-NEXT:    ldr w13, [sp, #824]
+; CHECK-GI-NEXT:    mov v21.s[0], wzr
+; CHECK-GI-NEXT:    sxtb w12, w12
+; CHECK-GI-NEXT:    mov v11.s[0], w11
+; CHECK-GI-NEXT:    ldr w11, [sp, #848]
+; CHECK-GI-NEXT:    mov v10.s[0], w10
+; CHECK-GI-NEXT:    ldr w10, [sp, #880]
+; CHECK-GI-NEXT:    sxtb w13, w13
+; CHECK-GI-NEXT:    mov v9.s[1], w12
+; CHECK-GI-NEXT:    sxtb w11, w11
+; CHECK-GI-NEXT:    mov v8.s[3], w9
+; CHECK-GI-NEXT:    ldr w9, [sp, #904]
+; CHECK-GI-NEXT:    sxtb w10, w10
+; CHECK-GI-NEXT:    ldr w12, [sp, #832]
+; CHECK-GI-NEXT:    mov v11.s[1], w11
+; CHECK-GI-NEXT:    ldr w11, [sp, #856]
+; CHECK-GI-NEXT:    mov v29.s[3], w8
+; CHECK-GI-NEXT:    mov v10.s[1], w10
+; CHECK-GI-NEXT:    ldr w10, [sp, #888]
+; CHECK-GI-NEXT:    sxtb w9, w9
+; CHECK-GI-NEXT:    mov v9.s[2], w13
+; CHECK-GI-NEXT:    sxtb w11, w11
+; CHECK-GI-NEXT:    sxtb w12, w12
+; CHECK-GI-NEXT:    mov v12.s[0], w9
+; CHECK-GI-NEXT:    ldr w9, [sp, #912]
+; CHECK-GI-NEXT:    sxtb w10, w10
+; CHECK-GI-NEXT:    mov v11.s[2], w11
+; CHECK-GI-NEXT:    ldr w11, [sp, #896]
+; CHECK-GI-NEXT:    ldr w13, [sp, #864]
+; CHECK-GI-NEXT:    mov v10.s[2], w10
+; CHECK-GI-NEXT:    sxtb w9, w9
+; CHECK-GI-NEXT:    ldr w10, [sp, #920]
+; CHECK-GI-NEXT:    mov v9.s[3], w12
+; CHECK-GI-NEXT:    ldr w12, [sp, #968]
+; CHECK-GI-NEXT:    sxtb w11, w11
+; CHECK-GI-NEXT:    mov v12.s[1], w9
+; CHECK-GI-NEXT:    sxtb w10, w10
+; CHECK-GI-NEXT:    sxtb w13, w13
+; CHECK-GI-NEXT:    sxtb w12, w12
+; CHECK-GI-NEXT:    ldr w8, [sp, #800]
+; CHECK-GI-NEXT:    ldr w9, [sp, #928]
+; CHECK-GI-NEXT:    mov v10.s[3], w11
+; CHECK-GI-NEXT:    ldr w11, [sp, #1032]
+; CHECK-GI-NEXT:    mov v11.s[3], w13
+; CHECK-GI-NEXT:    mov v14.s[0], w12
+; CHECK-GI-NEXT:    ldr w12, [sp, #976]
+; CHECK-GI-NEXT:    ldr w13, [sp, #936]
+; CHECK-GI-NEXT:    mov v12.s[2], w10
+; CHECK-GI-NEXT:    ldr w10, [sp, #1000]
+; CHECK-GI-NEXT:    sxtb w11, w11
+; CHECK-GI-NEXT:    sxtb w12, w12
+; CHECK-GI-NEXT:    sxtb w13, w13
+; CHECK-GI-NEXT:    mov v21.s[1], wzr
+; CHECK-GI-NEXT:    mov v15.s[0], w11
+; CHECK-GI-NEXT:    ldr w11, [sp, #1040]
+; CHECK-GI-NEXT:    sxtb w10, w10
+; CHECK-GI-NEXT:    mov v14.s[1], w12
+; CHECK-GI-NEXT:    ldr w12, [sp, #984]
+; CHECK-GI-NEXT:    mov v13.s[0], w13
+; CHECK-GI-NEXT:    mov v19.s[0], w10
+; CHECK-GI-NEXT:    ldr w10, [sp, #1008]
+; CHECK-GI-NEXT:    sxtb w11, w11
+; CHECK-GI-NEXT:    sxtb w12, w12
+; CHECK-GI-NEXT:    ldr w13, [sp, #944]
+; CHECK-GI-NEXT:    sxtb w8, w8
+; CHECK-GI-NEXT:    mov v15.s[1], w11
+; CHECK-GI-NEXT:    ldr w11, [sp, #1048]
+; CHECK-GI-NEXT:    sxtb w10, w10
+; CHECK-GI-NEXT:    mov v14.s[2], w12
+; CHECK-GI-NEXT:    ldr w12, [sp, #1064]
+; CHECK-GI-NEXT:    sxtb w13, w13
+; CHECK-GI-NEXT:    sxtb w11, w11
+; CHECK-GI-NEXT:    mov v19.s[1], w10
+; CHECK-GI-NEXT:    ldr w10, [sp, #992]
+; CHECK-GI-NEXT:    sxtb w12, w12
+; CHECK-GI-NEXT:    mov v13.s[1], w13
+; CHECK-GI-NEXT:    ldr w13, [sp, #952]
+; CHECK-GI-NEXT:    mov v15.s[2], w11
+; CHECK-GI-NEXT:    ldr w11, [sp, #1016]
+; CHECK-GI-NEXT:    sxtb w9, w9
+; CHECK-GI-NEXT:    mul w8, w8, w12
+; CHECK-GI-NEXT:    ldr w12, [sp, #1056]
+; CHECK-GI-NEXT:    sxtb w10, w10
+; CHECK-GI-NEXT:    sxtb w13, w13
+; CHECK-GI-NEXT:    mov v12.s[3], w9
+; CHECK-GI-NEXT:    sxtb w9, w11
+; CHECK-GI-NEXT:    mov v14.s[3], w10
+; CHECK-GI-NEXT:    mov v21.s[2], wzr
+; CHECK-GI-NEXT:    sxtb w10, w12
+; CHECK-GI-NEXT:    mul v6.4s, v6.4s, v22.4s
+; CHECK-GI-NEXT:    mov v22.s[0], w8
+; CHECK-GI-NEXT:    mov v13.s[2], w13
+; CHECK-GI-NEXT:    mov v19.s[2], w9
+; CHECK-GI-NEXT:    ldr w8, [sp, #960]
+; CHECK-GI-NEXT:    mov v15.s[3], w10
+; CHECK-GI-NEXT:    ldr w9, [sp, #1024]
+; CHECK-GI-NEXT:    mov v25.s[1], wzr
+; CHECK-GI-NEXT:    mul v7.4s, v7.4s, v23.4s
+; CHECK-GI-NEXT:    mov v21.s[3], wzr
+; CHECK-GI-NEXT:    sxtb w8, w8
+; CHECK-GI-NEXT:    mul v23.4s, v27.4s, v11.4s
+; CHECK-GI-NEXT:    sxtb w9, w9
+; CHECK-GI-NEXT:    mov v22.s[1], wzr
+; CHECK-GI-NEXT:    mul v27.4s, v28.4s, v12.4s
+; CHECK-GI-NEXT:    mul v28.4s, v31.4s, v14.4s
+; CHECK-GI-NEXT:    mul v31.4s, v8.4s, v15.4s
+; CHECK-GI-NEXT:    mov v13.s[3], w8
+; CHECK-GI-NEXT:    mov v19.s[3], w9
+; CHECK-GI-NEXT:    mla v3.4s, v0.4s, v16.4s
+; CHECK-GI-NEXT:    mov v25.s[2], wzr
+; CHECK-GI-NEXT:    add v0.4s, v21.4s, v21.4s
+; CHECK-GI-NEXT:    mla v5.4s, v1.4s, v17.4s
+; CHECK-GI-NEXT:    mla v6.4s, v2.4s, v18.4s
+; CHECK-GI-NEXT:    mov v22.s[2], wzr
+; CHECK-GI-NEXT:    mla v7.4s, v4.4s, v20.4s
+; CHECK-GI-NEXT:    mla v23.4s, v24.4s, v9.4s
+; CHECK-GI-NEXT:    mla v27.4s, v26.4s, v10.4s
+; CHECK-GI-NEXT:    mla v28.4s, v29.4s, v13.4s
+; CHECK-GI-NEXT:    mla v31.4s, v30.4s, v19.4s
+; CHECK-GI-NEXT:    add v1.4s, v21.4s, v0.4s
+; CHECK-GI-NEXT:    add v0.4s, v0.4s, v0.4s
+; CHECK-GI-NEXT:    mov v25.s[3], wzr
+; CHECK-GI-NEXT:    add v2.4s, v3.4s, v5.4s
+; CHECK-GI-NEXT:    mov v22.s[3], wzr
+; CHECK-GI-NEXT:    add v3.4s, v6.4s, v7.4s
+; CHECK-GI-NEXT:    add v0.4s, v1.4s, v0.4s
+; CHECK-GI-NEXT:    add v1.4s, v23.4s, v27.4s
+; CHECK-GI-NEXT:    add v4.4s, v28.4s, v31.4s
 ; CHECK-GI-NEXT:    ldp d9, d8, [sp, #48] // 16-byte Folded Reload
-; CHECK-GI-NEXT:    mul v26.4s, v10.4s, v14.4s
+; CHECK-GI-NEXT:    add v2.4s, v2.4s, v3.4s
 ; CHECK-GI-NEXT:    ldp d11, d10, [sp, #32] // 16-byte Folded Reload
-; CHECK-GI-NEXT:    mul v31.4s, v13.4s, v15.4s
+; CHECK-GI-NEXT:    add v3.4s, v25.4s, v0.4s
+; CHECK-GI-NEXT:    add v0.4s, v22.4s, v0.4s
+; CHECK-GI-NEXT:    add v1.4s, v1.4s, v4.4s
 ; CHECK-GI-NEXT:    ldp d13, d12, [sp, #16] // 16-byte Folded Reload
-; CHECK-GI-NEXT:    ushll v0.4s, v0.4h, #0
-; CHECK-GI-NEXT:    ushll v17.4s, v17.4h, #0
-; CHECK-GI-NEXT:    ushll v1.4s, v1.4h, #0
-; CHECK-GI-NEXT:    sshll v18.4s, v18.4h, #0
-; CHECK-GI-NEXT:    sshll v2.4s, v2.4h, #0
-; CHECK-GI-NEXT:    sshll v19.4s, v19.4h, #0
-; CHECK-GI-NEXT:    sshll v3.4s, v3.4h, #0
-; CHECK-GI-NEXT:    ushll v27.4s, v27.4h, #0
-; CHECK-GI-NEXT:    ushll v4.4s, v4.4h, #0
-; CHECK-GI-NEXT:    ushll v28.4s, v28.4h, #0
-; CHECK-GI-NEXT:    ushll v5.4s, v5.4h, #0
-; CHECK-GI-NEXT:    sshll v29.4s, v29.4h, #0
-; CHECK-GI-NEXT:    sshll v6.4s, v6.4h, #0
-; CHECK-GI-NEXT:    sshll v30.4s, v30.4h, #0
-; CHECK-GI-NEXT:    sshll v7.4s, v7.4h, #0
-; CHECK-GI-NEXT:    mla v20.4s, v16.4s, v18.4s
-; CHECK-GI-NEXT:    mla v21.4s, v0.4s, v2.4s
-; CHECK-GI-NEXT:    mla v22.4s, v17.4s, v19.4s
-; CHECK-GI-NEXT:    mla v23.4s, v1.4s, v3.4s
-; CHECK-GI-NEXT:    mla v24.4s, v27.4s, v29.4s
-; CHECK-GI-NEXT:    mla v25.4s, v4.4s, v6.4s
-; CHECK-GI-NEXT:    mla v26.4s, v28.4s, v30.4s
-; CHECK-GI-NEXT:    mla v31.4s, v5.4s, v7.4s
-; CHECK-GI-NEXT:    add v0.4s, v20.4s, v21.4s
-; CHECK-GI-NEXT:    add v1.4s, v22.4s, v23.4s
-; CHECK-GI-NEXT:    add v2.4s, v24.4s, v25.4s
-; CHECK-GI-NEXT:    add v3.4s, v26.4s, v31.4s
-; CHECK-GI-NEXT:    add v0.4s, v0.4s, v1.4s
-; CHECK-GI-NEXT:    add v1.4s, v2.4s, v3.4s
+; CHECK-GI-NEXT:    add v2.4s, v2.4s, v3.4s
+; CHECK-GI-NEXT:    add v0.4s, v1.4s, v0.4s
+; CHECK-GI-NEXT:    addv s1, v2.4s
 ; CHECK-GI-NEXT:    addv s0, v0.4s
-; CHECK-GI-NEXT:    addv s1, v1.4s
-; CHECK-GI-NEXT:    fmov w8, s0
-; CHECK-GI-NEXT:    fmov w9, s1
+; CHECK-GI-NEXT:    fmov w8, s1
+; CHECK-GI-NEXT:    fmov w9, s0
 ; CHECK-GI-NEXT:    add w0, w8, w9
-; CHECK-GI-NEXT:    ldp d15, d14, [sp], #64 // 16-byte Folded Reload
+; CHECK-GI-NEXT:    ldp d15, d14, [sp], #80 // 16-byte Folded Reload
 ; CHECK-GI-NEXT:    ret
-entry:
-  %az = zext <32 x i8> %a to <32 x i32>
-  %bz = sext <32 x i8> %b to <32 x i32>
-  %m1 = mul nuw nsw <32 x i32> %az, %bz
-  %r1 = call i32 @llvm.vector.reduce.add.v32i32(<32 x i32> %m1)
-  %cz = zext <32 x i8> %c to <32 x i32>
-  %dz = sext <32 x i8> %d to <32 x i32>
-  %m2 = mul nuw nsw <32 x i32> %cz, %dz
-  %r2 = call i32 @llvm.vector.reduce.add.v32i32(<32 x i32> %m2)
-  %x = add i32 %r1, %r2
-  ret i32 %x
-}
-
-
-define i32 @test_udot_v33i8(ptr nocapture readonly %a, ptr nocapture readonly %b, i32 %sum) {
-; CHECK-LABEL: test_udot_v33i8:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    ldr b0, [x0, #32]
-; CHECK-NEXT:    ldr b1, [x1, #32]
-; CHECK-NEXT:    movi v7.2d, #0000000000000000
-; CHECK-NEXT:    ldp q3, q4, [x1]
-; CHECK-NEXT:    umull v0.8h, v1.8b, v0.8b
-; CHECK-NEXT:    ldp q1, q2, [x0]
-; CHECK-NEXT:    umull v5.8h, v4.8b, v2.8b
-; CHECK-NEXT:    umull v6.8h, v3.8b, v1.8b
-; CHECK-NEXT:    umull2 v2.8h, v4.16b, v2.16b
-; CHECK-NEXT:    ushll v0.4s, v0.4h, #0
-; CHECK-NEXT:    umull2 v1.8h, v3.16b, v1.16b
-; CHECK-NEXT:    mov v7.s[0], v0.s[0]
-; CHECK-NEXT:    uaddl2 v3.4s, v6.8h, v5.8h
-; CHECK-NEXT:    uaddl2 v0.4s, v1.8h, v2.8h
-; CHECK-NEXT:    uaddl v1.4s, v1.4h, v2.4h
-; CHECK-NEXT:    add v0.4s, v3.4s, v0.4s
-; CHECK-NEXT:    uaddw v2.4s, v7.4s, v6.4h
-; CHECK-NEXT:    uaddw v2.4s, v2.4s, v5.4h
-; CHECK-NEXT:    add v0.4s, v1.4s, v0.4s
-; CHECK-NEXT:    add v0.4s, v2.4s, v0.4s
-; CHECK-NEXT:    addv s0, v0.4s
-; CHECK-NEXT:    fmov w8, s0
-; CHECK-NEXT:    add w0, w8, w2
-; CHECK-NEXT:    ret
-entry:
-  %0 = load <33 x i8>, ptr %a
-  %1 = zext <33 x i8> %0 to <33 x i32>
-  %2 = load <33 x i8>, ptr %b
-  %3 = zext <33 x i8> %2 to <33 x i32>
-  %4 = mul nuw nsw <33 x i32> %3, %1
-  %5 = call i32 @llvm.vector.reduce.add.v33i32(<33 x i32> %4)
-  %op.extra = add i32 %5, %sum
-  ret i32 %op.extra
-}
-
-define i32 @test_udot_v33i8_nomla(ptr nocapture readonly %a1) {
-; CHECK-LABEL: test_udot_v33i8_nomla:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    ldr b1, [x0, #32]
-; CHECK-NEXT:    ldp q3, q2, [x0]
-; CHECK-NEXT:    movi v0.2d, #0000000000000000
-; CHECK-NEXT:    ushll v1.8h, v1.8b, #0
-; CHECK-NEXT:    ushll v4.8h, v2.8b, #0
-; CHECK-NEXT:    ushll v5.8h, v3.8b, #0
-; CHECK-NEXT:    ushll2 v2.8h, v2.16b, #0
-; CHECK-NEXT:    ushll2 v3.8h, v3.16b, #0
-; CHECK-NEXT:    ushll v1.4s, v1.4h, #0
-; CHECK-NEXT:    uaddl2 v6.4s, v5.8h, v4.8h
-; CHECK-NEXT:    mov v0.s[0], v1.s[0]
-; CHECK-NEXT:    uaddl2 v1.4s, v3.8h, v2.8h
-; CHECK-NEXT:    uaddl v2.4s, v3.4h, v2.4h
-; CHECK-NEXT:    add v1.4s, v6.4s, v1.4s
-; CHECK-NEXT:    uaddw v0.4s, v0.4s, v5.4h
-; CHECK-NEXT:    add v1.4s, v2.4s, v1.4s
-; CHECK-NEXT:    uaddw v0.4s, v0.4s, v4.4h
-; CHECK-NEXT:    add v0.4s, v0.4s, v1.4s
-; CHECK-NEXT:    addv s0, v0.4s
-; CHECK-NEXT:    fmov w0, s0
-; CHECK-NEXT:    ret
-entry:
-  %0 = load <33 x i8>, ptr %a1
-  %1 = zext <33 x i8> %0 to <33 x i32>
-  %2 = call i32 @llvm.vector.reduce.add.v33i32(<33 x i32> %1)
-  ret i32 %2
-}
-define i32 @test_sdot_v33i8(ptr nocapture readonly %a, ptr nocapture readonly %b, i32 %sum) {
-; CHECK-LABEL: test_sdot_v33i8:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    ldr b0, [x0, #32]
-; CHECK-NEXT:    ldr b1, [x1, #32]
-; CHECK-NEXT:    movi v7.2d, #0000000000000000
-; CHECK-NEXT:    ldp q3, q4, [x1]
-; CHECK-NEXT:    smull v0.8h, v1.8b, v0.8b
-; CHECK-NEXT:    ldp q1, q2, [x0]
-; CHECK-NEXT:    smull v5.8h, v4.8b, v2.8b
-; CHECK-NEXT:    smull v6.8h, v3.8b, v1.8b
-; CHECK-NEXT:    smull2 v2.8h, v4.16b, v2.16b
-; CHECK-NEXT:    sshll v0.4s, v0.4h, #0
-; CHECK-NEXT:    smull2 v1.8h, v3.16b, v1.16b
-; CHECK-NEXT:    mov v7.s[0], v0.s[0]
-; CHECK-NEXT:    saddl2 v3.4s, v6.8h, v5.8h
-; CHECK-NEXT:    saddl2 v0.4s, v1.8h, v2.8h
-; CHECK-NEXT:    saddl v1.4s, v1.4h, v2.4h
-; CHECK-NEXT:    add v0.4s, v3.4s, v0.4s
-; CHECK-NEXT:    saddw v2.4s, v7.4s, v6.4h
-; CHECK-NEXT:    saddw v2.4s, v2.4s, v5.4h
-; CHECK-NEXT:    add v0.4s, v1.4s, v0.4s
-; CHECK-NEXT:    add v0.4s, v2.4s, v0.4s
-; CHECK-NEXT:    addv s0, v0.4s
-; CHECK-NEXT:    fmov w8, s0
-; CHECK-NEXT:    add w0, w8, w2
-; CHECK-NEXT:    ret
-entry:
-  %0 = load <33 x i8>, ptr %a
-  %1 = sext <33 x i8> %0 to <33 x i32>
-  %2 = load <33 x i8>, ptr %b
-  %3 = sext <33 x i8> %2 to <33 x i32>
-  %4 = mul nsw <33 x i32> %3, %1
-  %5 = call i32 @llvm.vector.reduce.add.v33i32(<33 x i32> %4)
-  %op.extra = add nsw i32 %5, %sum
-  ret i32 %op.extra
-}
-
-define i32 @test_sdot_v33i8_double(<33 x i8> %a, <33 x i8> %b, <33 x i8> %c, <33 x i8> %d) {
-; CHECK-LABEL: test_sdot_v33i8_double:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill
-; CHECK-NEXT:    .cfi_def_cfa_offset 16
-; CHECK-NEXT:    .cfi_offset w29, -16
-; CHECK-NEXT:    ldr b0, [sp, #344]
-; CHECK-NEXT:    add x8, sp, #352
-; CHECK-NEXT:    ldr b1, [sp, #80]
-; CHECK-NEXT:    ldr b2, [sp, #216]
-; CHECK-NEXT:    add x9, sp, #96
-; CHECK-NEXT:    add x10, sp, #104
-; CHECK-NEXT:    ld1 { v0.b }[1], [x8]
-; CHECK-NEXT:    add x8, sp, #88
-; CHECK-NEXT:    ldr b4, [sp, #408]
-; CHECK-NEXT:    ld1 { v1.b }[1], [x8]
-; CHECK-NEXT:    add x8, sp, #360
-; CHECK-NEXT:    add x12, sp, #248
-; CHECK-NEXT:    add x13, sp, #432
-; CHECK-NEXT:    add x11, sp, #384
-; CHECK-NEXT:    ldr b5, [sp, #144]
-; CHECK-NEXT:    ld1 { v0.b }[2], [x8]
-; CHECK-NEXT:    add x8, sp, #224
-; CHECK-NEXT:    ldr b6, [sp, #280]
-; CHECK-NEXT:    ld1 { v2.b }[1], [x8]
-; CHECK-NEXT:    ld1 { v1.b }[2], [x9]
-; CHECK-NEXT:    add x8, sp, #368
-; CHECK-NEXT:    add x9, sp, #232
-; CHECK-NEXT:    ldr b16, [sp, #744]
-; CHECK-NEXT:    ldr b17, [sp, #480]
-; CHECK-NEXT:    ld1 { v0.b }[3], [x8]
-; CHECK-NEXT:    add x8, sp, #376
-; CHECK-NEXT:    ldr b18, [sp, #936]
-; CHECK-NEXT:    ld1 { v2.b }[2], [x9]
-; CHECK-NEXT:    ld1 { v1.b }[3], [x10]
-; CHECK-NEXT:    add x9, sp, #240
-; CHECK-NEXT:    add x10, sp, #392
-; CHECK-NEXT:    ldr b19, [sp, #672]
-; CHECK-NEXT:    ldr b7, [sp, #16]
-; CHECK-NEXT:    ld1 { v0.b }[4], [x8]
-; CHECK-NEXT:    add x8, sp, #112
-; CHECK-NEXT:    ldr b21, [sp, #1000]
-; CHECK-NEXT:    ld1 { v2.b }[3], [x9]
-; CHECK-NEXT:    ld1 { v1.b }[4], [x8]
-; CHECK-NEXT:    add x8, sp, #416
-; CHECK-NEXT:    ld1 { v4.b }[1], [x8]
-; CHECK-NEXT:    add x8, sp, #120
-; CHECK-NEXT:    add x9, sp, #400
-; CHECK-NEXT:    ld1 { v0.b }[5], [x11]
-; CHECK-NEXT:    add x11, sp, #128
-; CHECK-NEXT:    ldr b22, [sp, #736]
-; CHECK-NEXT:    ld1 { v2.b }[4], [x12]
-; CHECK-NEXT:    add x12, sp, #424
-; CHECK-NEXT:    ld1 { v1.b }[5], [x8]
-; CHECK-NEXT:    ld1 { v4.b }[2], [x12]
-; CHECK-NEXT:    add x12, sp, #152
-; CHECK-NEXT:    add x8, sp, #136
-; CHECK-NEXT:    ld1 { v5.b }[1], [x12]
-; CHECK-NEXT:    add x12, sp, #440
-; CHECK-NEXT:    ld1 { v0.b }[6], [x10]
-; CHECK-NEXT:    ld1 { v1.b }[6], [x11]
-; CHECK-NEXT:    add x11, sp, #288
-; CHECK-NEXT:    add x10, sp, #256
-; CHECK-NEXT:    ld1 { v4.b }[3], [x13]
-; CHECK-NEXT:    ld1 { v6.b }[1], [x11]
-; CHECK-NEXT:    add x11, sp, #296
-; CHECK-NEXT:    ld1 { v0.b }[7], [x9]
-; CHECK-NEXT:    add x9, sp, #160
-; CHECK-NEXT:    ld1 { v2.b }[5], [x10]
-; CHECK-NEXT:    ld1 { v5.b }[2], [x9]
-; CHECK-NEXT:    add x10, sp, #168
-; CHECK-NEXT:    ld1 { v1.b }[7], [x8]
-; CHECK-NEXT:    ld1 { v4.b }[4], [x12]
-; CHECK-NEXT:    add x12, sp, #448
-; CHECK-NEXT:    ld1 { v6.b }[2], [x11]
-; CHECK-NEXT:    add x11, sp, #304
-; CHECK-NEXT:    add x8, sp, #464
-; CHECK-NEXT:    add x13, sp, #768
-; CHECK-NEXT:    ld1 { v5.b }[3], [x10]
-; CHECK-NEXT:    add x10, sp, #176
-; CHECK-NEXT:    add x9, sp, #264
-; CHECK-NEXT:    ld1 { v4.b }[5], [x12]
-; CHECK-NEXT:    add x12, sp, #456
-; CHECK-NEXT:    ld1 { v6.b }[3], [x11]
-; CHECK-NEXT:    add x11, sp, #760
-; CHECK-NEXT:    ld1 { v2.b }[6], [x9]
-; CHECK-NEXT:    add x9, sp, #272
-; CHECK-NEXT:    ld1 { v5.b }[4], [x10]
-; CHECK-NEXT:    add x10, sp, #312
-; CHECK-NEXT:    fmov s3, w0
-; CHECK-NEXT:    ld1 { v4.b }[6], [x12]
-; CHECK-NEXT:    ld1 { v6.b }[4], [x10]
-; CHECK-NEXT:    add x10, sp, #320
-; CHECK-NEXT:    add x12, sp, #680
-; CHECK-NEXT:    ld1 { v2.b }[7], [x9]
-; CHECK-NEXT:    add x9, sp, #184
-; CHECK-NEXT:    ld1 { v19.b }[1], [x12]
-; CHECK-NEXT:    add x12, sp, #776
-; CHECK-NEXT:    ld1 { v5.b }[5], [x9]
-; CHECK-NEXT:    ld1 { v4.b }[7], [x8]
-; CHECK-NEXT:    add x8, sp, #752
-; CHECK-NEXT:    ld1 { v6.b }[5], [x10]
-; CHECK-NEXT:    ld1 { v16.b }[1], [x8]
-; CHECK-NEXT:    add x10, sp, #24
-; CHECK-NEXT:    smull v22.8h, v22.8b, v21.8b
-; CHECK-NEXT:    ld1 { v7.b }[1], [x10]
-; CHECK-NEXT:    add x10, sp, #496
-; CHECK-NEXT:    mov v3.b[1], w1
-; CHECK-NEXT:    add x9, sp, #192
-; CHECK-NEXT:    ldr b20, [sp, #472]
-; CHECK-NEXT:    ldr b23, [sp, #208]
-; CHECK-NEXT:    ld1 { v16.b }[2], [x11]
-; CHECK-NEXT:    add x11, sp, #488
-; CHECK-NEXT:    ld1 { v5.b }[6], [x9]
-; CHECK-NEXT:    ld1 { v17.b }[1], [x11]
-; CHECK-NEXT:    add x11, sp, #944
-; CHECK-NEXT:    add x9, sp, #328
-; CHECK-NEXT:    ld1 { v18.b }[1], [x11]
-; CHECK-NEXT:    add x11, sp, #688
-; CHECK-NEXT:    ld1 { v6.b }[6], [x9]
-; CHECK-NEXT:    ld1 { v16.b }[3], [x13]
-; CHECK-NEXT:    ld1 { v19.b }[2], [x11]
-; CHECK-NEXT:    add x11, sp, #504
-; CHECK-NEXT:    ld1 { v17.b }[2], [x10]
-; CHECK-NEXT:    add x10, sp, #952
-; CHECK-NEXT:    add x13, sp, #784
-; CHECK-NEXT:    ld1 { v18.b }[2], [x10]
-; CHECK-NEXT:    add x10, sp, #32
-; CHECK-NEXT:    add x9, sp, #40
-; CHECK-NEXT:    ld1 { v16.b }[4], [x12]
-; CHECK-NEXT:    add x12, sp, #696
-; CHECK-NEXT:    ld1 { v7.b }[2], [x10]
-; CHECK-NEXT:    ld1 { v17.b }[3], [x11]
-; CHECK-NEXT:    add x11, sp, #960
-; CHECK-NEXT:    ld1 { v19.b }[3], [x12]
-; CHECK-NEXT:    ld1 { v18.b }[3], [x11]
-; CHECK-NEXT:    add x10, sp, #512
-; CHECK-NEXT:    add x11, sp, #704
-; CHECK-NEXT:    ld1 { v16.b }[5], [x13]
-; CHECK-NEXT:    add x12, sp, #792
-; CHECK-NEXT:    sshll v24.4s, v22.4h, #0
-; CHECK-NEXT:    ld1 { v17.b }[4], [x10]
-; CHECK-NEXT:    add x10, sp, #968
-; CHECK-NEXT:    ld1 { v19.b }[4], [x11]
-; CHECK-NEXT:    ld1 { v18.b }[4], [x10]
-; CHECK-NEXT:    add x10, sp, #520
-; CHECK-NEXT:    add x11, sp, #976
-; CHECK-NEXT:    ld1 { v16.b }[6], [x12]
-; CHECK-NEXT:    add x12, sp, #712
-; CHECK-NEXT:    smull v20.8h, v23.8b, v20.8b
-; CHECK-NEXT:    ld1 { v17.b }[5], [x10]
-; CHECK-NEXT:    ld1 { v19.b }[5], [x12]
-; CHECK-NEXT:    add x12, sp, #720
-; CHECK-NEXT:    ld1 { v18.b }[5], [x11]
-; CHECK-NEXT:    add x11, sp, #528
-; CHECK-NEXT:    add x10, sp, #800
-; CHECK-NEXT:    ld1 { v16.b }[7], [x10]
-; CHECK-NEXT:    add x10, sp, #536
-; CHECK-NEXT:    ldr b22, [sp, #872]
-; CHECK-NEXT:    ld1 { v17.b }[6], [x11]
-; CHECK-NEXT:    add x11, sp, #984
-; CHECK-NEXT:    ld1 { v19.b }[6], [x12]
-; CHECK-NEXT:    ld1 { v18.b }[6], [x11]
-; CHECK-NEXT:    add x11, sp, #992
-; CHECK-NEXT:    add x12, sp, #728
-; CHECK-NEXT:    ldr b23, [sp, #608]
-; CHECK-NEXT:    ld1 { v7.b }[3], [x9]
-; CHECK-NEXT:    add x9, sp, #880
-; CHECK-NEXT:    ld1 { v17.b }[7], [x10]
-; CHECK-NEXT:    ld1 { v19.b }[7], [x12]
-; CHECK-NEXT:    add x10, sp, #816
-; CHECK-NEXT:    ld1 { v18.b }[7], [x11]
-; CHECK-NEXT:    add x11, sp, #552
-; CHECK-NEXT:    add x12, sp, #616
-; CHECK-NEXT:    mov v3.b[2], w2
-; CHECK-NEXT:    ld1 { v22.b }[1], [x9]
-; CHECK-NEXT:    ld1 { v23.b }[1], [x12]
-; CHECK-NEXT:    smull v16.8h, v17.8b, v16.8b
-; CHECK-NEXT:    add x12, sp, #560
-; CHECK-NEXT:    add x9, sp, #888
-; CHECK-NEXT:    smull v17.8h, v19.8b, v18.8b
-; CHECK-NEXT:    ldr b18, [sp, #808]
-; CHECK-NEXT:    ldr b19, [sp, #544]
-; CHECK-NEXT:    add x13, sp, #624
-; CHECK-NEXT:    ld1 { v22.b }[2], [x9]
-; CHECK-NEXT:    add x9, sp, #896
-; CHECK-NEXT:    ld1 { v18.b }[1], [x10]
-; CHECK-NEXT:    ld1 { v19.b }[1], [x11]
-; CHECK-NEXT:    add x11, sp, #824
-; CHECK-NEXT:    add x10, sp, #48
-; CHECK-NEXT:    ld1 { v23.b }[2], [x13]
-; CHECK-NEXT:    mov v3.b[3], w3
-; CHECK-NEXT:    ld1 { v7.b }[4], [x10]
-; CHECK-NEXT:    add x10, sp, #832
-; CHECK-NEXT:    ld1 { v22.b }[3], [x9]
-; CHECK-NEXT:    ld1 { v18.b }[2], [x11]
-; CHECK-NEXT:    ld1 { v19.b }[2], [x12]
-; CHECK-NEXT:    add x11, sp, #568
-; CHECK-NEXT:    add x12, sp, #632
-; CHECK-NEXT:    add x9, sp, #904
-; CHECK-NEXT:    add x13, sp, #640
-; CHECK-NEXT:    ld1 { v23.b }[3], [x12]
-; CHECK-NEXT:    add x12, sp, #576
-; CHECK-NEXT:    mov v3.b[4], w4
-; CHECK-NEXT:    ld1 { v18.b }[3], [x10]
-; CHECK-NEXT:    ld1 { v19.b }[3], [x11]
-; CHECK-NEXT:    add x11, sp, #840
-; CHECK-NEXT:    add x10, sp, #56
-; CHECK-NEXT:    ld1 { v22.b }[4], [x9]
-; CHECK-NEXT:    add x9, sp, #912
-; CHECK-NEXT:    ld1 { v23.b }[4], [x13]
-; CHECK-NEXT:    ld1 { v7.b }[5], [x10]
-; CHECK-NEXT:    add x10, sp, #848
-; CHECK-NEXT:    ld1 { v18.b }[4], [x11]
-; CHECK-NEXT:    ld1 { v19.b }[4], [x12]
-; CHECK-NEXT:    add x11, sp, #584
-; CHECK-NEXT:    add x12, sp, #648
-; CHECK-NEXT:    mov v3.b[5], w5
-; CHECK-NEXT:    ld1 { v22.b }[5], [x9]
-; CHECK-NEXT:    ld1 { v23.b }[5], [x12]
-; CHECK-NEXT:    add x12, sp, #592
-; CHECK-NEXT:    movi v21.2d, #0000000000000000
-; CHECK-NEXT:    ld1 { v18.b }[5], [x10]
-; CHECK-NEXT:    ld1 { v19.b }[5], [x11]
-; CHECK-NEXT:    add x11, sp, #856
-; CHECK-NEXT:    add x9, sp, #920
-; CHECK-NEXT:    add x13, sp, #656
-; CHECK-NEXT:    add x10, sp, #64
-; CHECK-NEXT:    ld1 { v22.b }[6], [x9]
-; CHECK-NEXT:    ld1 { v23.b }[6], [x13]
-; CHECK-NEXT:    mov v3.b[6], w6
-; CHECK-NEXT:    ld1 { v18.b }[6], [x11]
-; CHECK-NEXT:    ld1 { v19.b }[6], [x12]
-; CHECK-NEXT:    ld1 { v7.b }[6], [x10]
-; CHECK-NEXT:    add x10, sp, #864
-; CHECK-NEXT:    add x11, sp, #600
-; CHECK-NEXT:    add x9, sp, #928
-; CHECK-NEXT:    add x12, sp, #664
-; CHECK-NEXT:    mov v21.s[0], v24.s[0]
-; CHECK-NEXT:    ld1 { v22.b }[7], [x9]
-; CHECK-NEXT:    ld1 { v18.b }[7], [x10]
-; CHECK-NEXT:    ld1 { v19.b }[7], [x11]
-; CHECK-NEXT:    ld1 { v23.b }[7], [x12]
-; CHECK-NEXT:    add x8, sp, #200
-; CHECK-NEXT:    mov v3.b[7], w7
-; CHECK-NEXT:    add x10, sp, #336
-; CHECK-NEXT:    ld1 { v5.b }[7], [x8]
-; CHECK-NEXT:    add x8, sp, #72
-; CHECK-NEXT:    ld1 { v6.b }[7], [x10]
-; CHECK-NEXT:    smull v18.8h, v19.8b, v18.8b
-; CHECK-NEXT:    movi v19.2d, #0000000000000000
-; CHECK-NEXT:    ld1 { v7.b }[7], [x8]
-; CHECK-NEXT:    smull v22.8h, v23.8b, v22.8b
-; CHECK-NEXT:    sshll v20.4s, v20.4h, #0
-; CHECK-NEXT:    smull v0.8h, v1.8b, v0.8b
-; CHECK-NEXT:    saddw v1.4s, v21.4s, v16.4h
-; CHECK-NEXT:    smull v2.8h, v3.8b, v2.8b
-; CHECK-NEXT:    smull v3.8h, v5.8b, v4.8b
-; CHECK-NEXT:    smull v4.8h, v7.8b, v6.8b
-; CHECK-NEXT:    mov v19.s[0], v20.s[0]
-; CHECK-NEXT:    saddl2 v5.4s, v18.8h, v17.8h
-; CHECK-NEXT:    saddl v7.4s, v18.4h, v17.4h
-; CHECK-NEXT:    saddl2 v6.4s, v16.8h, v22.8h
-; CHECK-NEXT:    saddw v1.4s, v1.4s, v22.4h
-; CHECK-NEXT:    saddl2 v17.4s, v2.8h, v0.8h
-; CHECK-NEXT:    saddl2 v16.4s, v4.8h, v3.8h
-; CHECK-NEXT:    saddl v3.4s, v4.4h, v3.4h
-; CHECK-NEXT:    saddw v2.4s, v19.4s, v2.4h
-; CHECK-NEXT:    add v5.4s, v6.4s, v5.4s
-; CHECK-NEXT:    add v1.4s, v1.4s, v7.4s
-; CHECK-NEXT:    add v6.4s, v17.4s, v16.4s
-; CHECK-NEXT:    saddw v0.4s, v2.4s, v0.4h
-; CHECK-NEXT:    add v1.4s, v1.4s, v5.4s
-; CHECK-NEXT:    add v0.4s, v0.4s, v3.4s
-; CHECK-NEXT:    add v1.4s, v6.4s, v1.4s
-; CHECK-NEXT:    add v0.4s, v0.4s, v1.4s
-; CHECK-NEXT:    addv s0, v0.4s
-; CHECK-NEXT:    fmov w0, s0
-; CHECK-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
-; CHECK-NEXT:    ret
 entry:
   %az = sext <33 x i8> %a to <33 x i32>
   %bz = sext <33 x i8> %b to <33 x i32>
@@ -3418,163 +5328,399 @@ entry:
 }
 
 define i32 @test_sdot_v33i8_double_nomla(<33 x i8> %a, <33 x i8> %b, <33 x i8> %c, <33 x i8> %d) {
-; CHECK-LABEL: test_sdot_v33i8_double_nomla:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill
-; CHECK-NEXT:    .cfi_def_cfa_offset 16
-; CHECK-NEXT:    .cfi_offset w29, -16
-; CHECK-NEXT:    ldr b0, [sp, #80]
-; CHECK-NEXT:    add x8, sp, #88
-; CHECK-NEXT:    ldr b2, [sp, #144]
-; CHECK-NEXT:    add x9, sp, #152
-; CHECK-NEXT:    ldr b3, [sp, #16]
-; CHECK-NEXT:    add x11, sp, #104
-; CHECK-NEXT:    ld1 { v0.b }[1], [x8]
-; CHECK-NEXT:    ld1 { v2.b }[1], [x9]
-; CHECK-NEXT:    add x9, sp, #24
-; CHECK-NEXT:    add x8, sp, #96
-; CHECK-NEXT:    ld1 { v3.b }[1], [x9]
-; CHECK-NEXT:    ldr b5, [sp, #480]
-; CHECK-NEXT:    fmov s1, w0
-; CHECK-NEXT:    add x10, sp, #112
-; CHECK-NEXT:    add x12, sp, #168
-; CHECK-NEXT:    ld1 { v0.b }[2], [x8]
-; CHECK-NEXT:    add x8, sp, #160
-; CHECK-NEXT:    ldr b4, [sp, #608]
-; CHECK-NEXT:    ld1 { v2.b }[2], [x8]
-; CHECK-NEXT:    add x8, sp, #32
-; CHECK-NEXT:    add x13, sp, #496
-; CHECK-NEXT:    ld1 { v3.b }[2], [x8]
-; CHECK-NEXT:    mov v1.b[1], w1
-; CHECK-NEXT:    ldr b6, [sp, #672]
-; CHECK-NEXT:    ld1 { v0.b }[3], [x11]
-; CHECK-NEXT:    add x11, sp, #488
-; CHECK-NEXT:    add x9, sp, #120
-; CHECK-NEXT:    ld1 { v5.b }[1], [x11]
-; CHECK-NEXT:    add x11, sp, #40
-; CHECK-NEXT:    ld1 { v2.b }[3], [x12]
-; CHECK-NEXT:    ld1 { v3.b }[3], [x11]
-; CHECK-NEXT:    add x12, sp, #616
-; CHECK-NEXT:    ldr b16, [sp, #544]
-; CHECK-NEXT:    ld1 { v0.b }[4], [x10]
-; CHECK-NEXT:    add x10, sp, #48
-; CHECK-NEXT:    ld1 { v4.b }[1], [x12]
-; CHECK-NEXT:    add x12, sp, #176
-; CHECK-NEXT:    ld1 { v5.b }[2], [x13]
-; CHECK-NEXT:    add x13, sp, #680
-; CHECK-NEXT:    ld1 { v3.b }[4], [x10]
-; CHECK-NEXT:    ld1 { v2.b }[4], [x12]
-; CHECK-NEXT:    ld1 { v6.b }[1], [x13]
-; CHECK-NEXT:    add x13, sp, #56
-; CHECK-NEXT:    ld1 { v0.b }[5], [x9]
-; CHECK-NEXT:    mov v1.b[2], w2
-; CHECK-NEXT:    add x8, sp, #128
-; CHECK-NEXT:    add x14, sp, #184
-; CHECK-NEXT:    add x11, sp, #136
-; CHECK-NEXT:    ld1 { v3.b }[5], [x13]
-; CHECK-NEXT:    add x13, sp, #552
-; CHECK-NEXT:    ld1 { v2.b }[5], [x14]
-; CHECK-NEXT:    ld1 { v16.b }[1], [x13]
-; CHECK-NEXT:    add x14, sp, #624
-; CHECK-NEXT:    ld1 { v0.b }[6], [x8]
-; CHECK-NEXT:    add x8, sp, #688
-; CHECK-NEXT:    add x13, sp, #504
-; CHECK-NEXT:    ld1 { v4.b }[2], [x14]
-; CHECK-NEXT:    ld1 { v6.b }[2], [x8]
-; CHECK-NEXT:    add x8, sp, #560
-; CHECK-NEXT:    ld1 { v5.b }[3], [x13]
-; CHECK-NEXT:    ld1 { v16.b }[2], [x8]
-; CHECK-NEXT:    mov v1.b[3], w3
-; CHECK-NEXT:    add x9, sp, #64
-; CHECK-NEXT:    add x15, sp, #632
-; CHECK-NEXT:    ld1 { v3.b }[6], [x9]
-; CHECK-NEXT:    ld1 { v0.b }[7], [x11]
-; CHECK-NEXT:    ld1 { v4.b }[3], [x15]
-; CHECK-NEXT:    add x8, sp, #696
-; CHECK-NEXT:    add x9, sp, #568
-; CHECK-NEXT:    add x11, sp, #512
-; CHECK-NEXT:    ld1 { v6.b }[3], [x8]
-; CHECK-NEXT:    ld1 { v16.b }[3], [x9]
-; CHECK-NEXT:    ld1 { v5.b }[4], [x11]
-; CHECK-NEXT:    add x8, sp, #640
-; CHECK-NEXT:    mov v1.b[4], w4
-; CHECK-NEXT:    ld1 { v4.b }[4], [x8]
-; CHECK-NEXT:    add x8, sp, #704
-; CHECK-NEXT:    add x9, sp, #576
-; CHECK-NEXT:    add x11, sp, #520
-; CHECK-NEXT:    ld1 { v6.b }[4], [x8]
-; CHECK-NEXT:    ld1 { v16.b }[4], [x9]
-; CHECK-NEXT:    ld1 { v5.b }[5], [x11]
-; CHECK-NEXT:    ldr b18, [sp, #736]
-; CHECK-NEXT:    add x12, sp, #192
-; CHECK-NEXT:    ld1 { v2.b }[6], [x12]
-; CHECK-NEXT:    add x8, sp, #648
-; CHECK-NEXT:    add x9, sp, #528
-; CHECK-NEXT:    add x11, sp, #712
-; CHECK-NEXT:    add x12, sp, #584
-; CHECK-NEXT:    sshll v18.8h, v18.8b, #0
-; CHECK-NEXT:    mov v1.b[5], w5
-; CHECK-NEXT:    ld1 { v6.b }[5], [x11]
-; CHECK-NEXT:    ld1 { v16.b }[5], [x12]
-; CHECK-NEXT:    ld1 { v4.b }[5], [x8]
-; CHECK-NEXT:    ld1 { v5.b }[6], [x9]
-; CHECK-NEXT:    movi v17.2d, #0000000000000000
-; CHECK-NEXT:    add x8, sp, #656
-; CHECK-NEXT:    add x9, sp, #536
-; CHECK-NEXT:    add x11, sp, #720
-; CHECK-NEXT:    add x12, sp, #592
-; CHECK-NEXT:    sshll v18.4s, v18.4h, #0
-; CHECK-NEXT:    ldr b7, [sp, #208]
-; CHECK-NEXT:    ld1 { v6.b }[6], [x11]
-; CHECK-NEXT:    ld1 { v16.b }[6], [x12]
-; CHECK-NEXT:    ld1 { v4.b }[6], [x8]
-; CHECK-NEXT:    ld1 { v5.b }[7], [x9]
-; CHECK-NEXT:    mov v1.b[6], w6
-; CHECK-NEXT:    sshll v7.8h, v7.8b, #0
-; CHECK-NEXT:    add x8, sp, #664
-; CHECK-NEXT:    add x9, sp, #728
-; CHECK-NEXT:    add x11, sp, #600
-; CHECK-NEXT:    mov v17.s[0], v18.s[0]
-; CHECK-NEXT:    ld1 { v6.b }[7], [x9]
-; CHECK-NEXT:    ld1 { v16.b }[7], [x11]
-; CHECK-NEXT:    ld1 { v4.b }[7], [x8]
-; CHECK-NEXT:    sshll v5.8h, v5.8b, #0
-; CHECK-NEXT:    movi v18.2d, #0000000000000000
-; CHECK-NEXT:    add x10, sp, #200
-; CHECK-NEXT:    mov v1.b[7], w7
-; CHECK-NEXT:    add x9, sp, #72
-; CHECK-NEXT:    sshll v7.4s, v7.4h, #0
-; CHECK-NEXT:    ld1 { v2.b }[7], [x10]
-; CHECK-NEXT:    ld1 { v3.b }[7], [x9]
-; CHECK-NEXT:    sshll v6.8h, v6.8b, #0
-; CHECK-NEXT:    sshll v16.8h, v16.8b, #0
-; CHECK-NEXT:    sshll v4.8h, v4.8b, #0
-; CHECK-NEXT:    saddw v17.4s, v17.4s, v5.4h
-; CHECK-NEXT:    sshll v0.8h, v0.8b, #0
-; CHECK-NEXT:    mov v18.s[0], v7.s[0]
-; CHECK-NEXT:    sshll v1.8h, v1.8b, #0
-; CHECK-NEXT:    sshll v2.8h, v2.8b, #0
-; CHECK-NEXT:    sshll v3.8h, v3.8b, #0
-; CHECK-NEXT:    saddl2 v7.4s, v16.8h, v6.8h
-; CHECK-NEXT:    saddl2 v5.4s, v5.8h, v4.8h
-; CHECK-NEXT:    saddl v6.4s, v16.4h, v6.4h
-; CHECK-NEXT:    saddw v4.4s, v17.4s, v4.4h
-; CHECK-NEXT:    saddl2 v17.4s, v1.8h, v0.8h
-; CHECK-NEXT:    saddl2 v16.4s, v3.8h, v2.8h
-; CHECK-NEXT:    saddw v1.4s, v18.4s, v1.4h
-; CHECK-NEXT:    add v5.4s, v5.4s, v7.4s
-; CHECK-NEXT:    add v4.4s, v4.4s, v6.4s
-; CHECK-NEXT:    saddl v2.4s, v3.4h, v2.4h
-; CHECK-NEXT:    add v6.4s, v17.4s, v16.4s
-; CHECK-NEXT:    saddw v0.4s, v1.4s, v0.4h
-; CHECK-NEXT:    add v1.4s, v4.4s, v5.4s
-; CHECK-NEXT:    add v0.4s, v0.4s, v2.4s
-; CHECK-NEXT:    add v1.4s, v6.4s, v1.4s
-; CHECK-NEXT:    add v0.4s, v0.4s, v1.4s
-; CHECK-NEXT:    addv s0, v0.4s
-; CHECK-NEXT:    fmov w0, s0
-; CHECK-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: test_sdot_v33i8_double_nomla:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-SD-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-SD-NEXT:    .cfi_offset w29, -16
+; CHECK-SD-NEXT:    ldr b0, [sp, #80]
+; CHECK-SD-NEXT:    add x8, sp, #88
+; CHECK-SD-NEXT:    ldr b2, [sp, #144]
+; CHECK-SD-NEXT:    add x9, sp, #152
+; CHECK-SD-NEXT:    ldr b3, [sp, #16]
+; CHECK-SD-NEXT:    add x11, sp, #104
+; CHECK-SD-NEXT:    ld1 { v0.b }[1], [x8]
+; CHECK-SD-NEXT:    ld1 { v2.b }[1], [x9]
+; CHECK-SD-NEXT:    add x9, sp, #24
+; CHECK-SD-NEXT:    add x8, sp, #96
+; CHECK-SD-NEXT:    ld1 { v3.b }[1], [x9]
+; CHECK-SD-NEXT:    ldr b5, [sp, #480]
+; CHECK-SD-NEXT:    fmov s1, w0
+; CHECK-SD-NEXT:    add x10, sp, #112
+; CHECK-SD-NEXT:    add x12, sp, #168
+; CHECK-SD-NEXT:    ld1 { v0.b }[2], [x8]
+; CHECK-SD-NEXT:    add x8, sp, #160
+; CHECK-SD-NEXT:    ldr b4, [sp, #608]
+; CHECK-SD-NEXT:    ld1 { v2.b }[2], [x8]
+; CHECK-SD-NEXT:    add x8, sp, #32
+; CHECK-SD-NEXT:    add x13, sp, #496
+; CHECK-SD-NEXT:    ld1 { v3.b }[2], [x8]
+; CHECK-SD-NEXT:    mov v1.b[1], w1
+; CHECK-SD-NEXT:    ldr b6, [sp, #672]
+; CHECK-SD-NEXT:    ld1 { v0.b }[3], [x11]
+; CHECK-SD-NEXT:    add x11, sp, #488
+; CHECK-SD-NEXT:    add x9, sp, #120
+; CHECK-SD-NEXT:    ld1 { v5.b }[1], [x11]
+; CHECK-SD-NEXT:    add x11, sp, #40
+; CHECK-SD-NEXT:    ld1 { v2.b }[3], [x12]
+; CHECK-SD-NEXT:    ld1 { v3.b }[3], [x11]
+; CHECK-SD-NEXT:    add x12, sp, #616
+; CHECK-SD-NEXT:    ldr b16, [sp, #544]
+; CHECK-SD-NEXT:    ld1 { v0.b }[4], [x10]
+; CHECK-SD-NEXT:    add x10, sp, #48
+; CHECK-SD-NEXT:    ld1 { v4.b }[1], [x12]
+; CHECK-SD-NEXT:    add x12, sp, #176
+; CHECK-SD-NEXT:    ld1 { v5.b }[2], [x13]
+; CHECK-SD-NEXT:    add x13, sp, #680
+; CHECK-SD-NEXT:    ld1 { v3.b }[4], [x10]
+; CHECK-SD-NEXT:    ld1 { v2.b }[4], [x12]
+; CHECK-SD-NEXT:    ld1 { v6.b }[1], [x13]
+; CHECK-SD-NEXT:    add x13, sp, #56
+; CHECK-SD-NEXT:    ld1 { v0.b }[5], [x9]
+; CHECK-SD-NEXT:    mov v1.b[2], w2
+; CHECK-SD-NEXT:    add x8, sp, #128
+; CHECK-SD-NEXT:    add x14, sp, #184
+; CHECK-SD-NEXT:    add x11, sp, #136
+; CHECK-SD-NEXT:    ld1 { v3.b }[5], [x13]
+; CHECK-SD-NEXT:    add x13, sp, #552
+; CHECK-SD-NEXT:    ld1 { v2.b }[5], [x14]
+; CHECK-SD-NEXT:    ld1 { v16.b }[1], [x13]
+; CHECK-SD-NEXT:    add x14, sp, #624
+; CHECK-SD-NEXT:    ld1 { v0.b }[6], [x8]
+; CHECK-SD-NEXT:    add x8, sp, #688
+; CHECK-SD-NEXT:    add x13, sp, #504
+; CHECK-SD-NEXT:    ld1 { v4.b }[2], [x14]
+; CHECK-SD-NEXT:    ld1 { v6.b }[2], [x8]
+; CHECK-SD-NEXT:    add x8, sp, #560
+; CHECK-SD-NEXT:    ld1 { v5.b }[3], [x13]
+; CHECK-SD-NEXT:    ld1 { v16.b }[2], [x8]
+; CHECK-SD-NEXT:    mov v1.b[3], w3
+; CHECK-SD-NEXT:    add x9, sp, #64
+; CHECK-SD-NEXT:    add x15, sp, #632
+; CHECK-SD-NEXT:    ld1 { v3.b }[6], [x9]
+; CHECK-SD-NEXT:    ld1 { v0.b }[7], [x11]
+; CHECK-SD-NEXT:    ld1 { v4.b }[3], [x15]
+; CHECK-SD-NEXT:    add x8, sp, #696
+; CHECK-SD-NEXT:    add x9, sp, #568
+; CHECK-SD-NEXT:    add x11, sp, #512
+; CHECK-SD-NEXT:    ld1 { v6.b }[3], [x8]
+; CHECK-SD-NEXT:    ld1 { v16.b }[3], [x9]
+; CHECK-SD-NEXT:    ld1 { v5.b }[4], [x11]
+; CHECK-SD-NEXT:    add x8, sp, #640
+; CHECK-SD-NEXT:    mov v1.b[4], w4
+; CHECK-SD-NEXT:    ld1 { v4.b }[4], [x8]
+; CHECK-SD-NEXT:    add x8, sp, #704
+; CHECK-SD-NEXT:    add x9, sp, #576
+; CHECK-SD-NEXT:    add x11, sp, #520
+; CHECK-SD-NEXT:    ld1 { v6.b }[4], [x8]
+; CHECK-SD-NEXT:    ld1 { v16.b }[4], [x9]
+; CHECK-SD-NEXT:    ld1 { v5.b }[5], [x11]
+; CHECK-SD-NEXT:    ldr b18, [sp, #736]
+; CHECK-SD-NEXT:    add x12, sp, #192
+; CHECK-SD-NEXT:    ld1 { v2.b }[6], [x12]
+; CHECK-SD-NEXT:    add x8, sp, #648
+; CHECK-SD-NEXT:    add x9, sp, #528
+; CHECK-SD-NEXT:    add x11, sp, #712
+; CHECK-SD-NEXT:    add x12, sp, #584
+; CHECK-SD-NEXT:    sshll v18.8h, v18.8b, #0
+; CHECK-SD-NEXT:    mov v1.b[5], w5
+; CHECK-SD-NEXT:    ld1 { v6.b }[5], [x11]
+; CHECK-SD-NEXT:    ld1 { v16.b }[5], [x12]
+; CHECK-SD-NEXT:    ld1 { v4.b }[5], [x8]
+; CHECK-SD-NEXT:    ld1 { v5.b }[6], [x9]
+; CHECK-SD-NEXT:    movi v17.2d, #0000000000000000
+; CHECK-SD-NEXT:    add x8, sp, #656
+; CHECK-SD-NEXT:    add x9, sp, #536
+; CHECK-SD-NEXT:    add x11, sp, #720
+; CHECK-SD-NEXT:    add x12, sp, #592
+; CHECK-SD-NEXT:    sshll v18.4s, v18.4h, #0
+; CHECK-SD-NEXT:    ldr b7, [sp, #208]
+; CHECK-SD-NEXT:    ld1 { v6.b }[6], [x11]
+; CHECK-SD-NEXT:    ld1 { v16.b }[6], [x12]
+; CHECK-SD-NEXT:    ld1 { v4.b }[6], [x8]
+; CHECK-SD-NEXT:    ld1 { v5.b }[7], [x9]
+; CHECK-SD-NEXT:    mov v1.b[6], w6
+; CHECK-SD-NEXT:    sshll v7.8h, v7.8b, #0
+; CHECK-SD-NEXT:    add x8, sp, #664
+; CHECK-SD-NEXT:    add x9, sp, #728
+; CHECK-SD-NEXT:    add x11, sp, #600
+; CHECK-SD-NEXT:    mov v17.s[0], v18.s[0]
+; CHECK-SD-NEXT:    ld1 { v6.b }[7], [x9]
+; CHECK-SD-NEXT:    ld1 { v16.b }[7], [x11]
+; CHECK-SD-NEXT:    ld1 { v4.b }[7], [x8]
+; CHECK-SD-NEXT:    sshll v5.8h, v5.8b, #0
+; CHECK-SD-NEXT:    movi v18.2d, #0000000000000000
+; CHECK-SD-NEXT:    add x10, sp, #200
+; CHECK-SD-NEXT:    mov v1.b[7], w7
+; CHECK-SD-NEXT:    add x9, sp, #72
+; CHECK-SD-NEXT:    sshll v7.4s, v7.4h, #0
+; CHECK-SD-NEXT:    ld1 { v2.b }[7], [x10]
+; CHECK-SD-NEXT:    ld1 { v3.b }[7], [x9]
+; CHECK-SD-NEXT:    sshll v6.8h, v6.8b, #0
+; CHECK-SD-NEXT:    sshll v16.8h, v16.8b, #0
+; CHECK-SD-NEXT:    sshll v4.8h, v4.8b, #0
+; CHECK-SD-NEXT:    saddw v17.4s, v17.4s, v5.4h
+; CHECK-SD-NEXT:    sshll v0.8h, v0.8b, #0
+; CHECK-SD-NEXT:    mov v18.s[0], v7.s[0]
+; CHECK-SD-NEXT:    sshll v1.8h, v1.8b, #0
+; CHECK-SD-NEXT:    sshll v2.8h, v2.8b, #0
+; CHECK-SD-NEXT:    sshll v3.8h, v3.8b, #0
+; CHECK-SD-NEXT:    saddl2 v7.4s, v16.8h, v6.8h
+; CHECK-SD-NEXT:    saddl2 v5.4s, v5.8h, v4.8h
+; CHECK-SD-NEXT:    saddl v6.4s, v16.4h, v6.4h
+; CHECK-SD-NEXT:    saddw v4.4s, v17.4s, v4.4h
+; CHECK-SD-NEXT:    saddl2 v17.4s, v1.8h, v0.8h
+; CHECK-SD-NEXT:    saddl2 v16.4s, v3.8h, v2.8h
+; CHECK-SD-NEXT:    saddw v1.4s, v18.4s, v1.4h
+; CHECK-SD-NEXT:    add v5.4s, v5.4s, v7.4s
+; CHECK-SD-NEXT:    add v4.4s, v4.4s, v6.4s
+; CHECK-SD-NEXT:    saddl v2.4s, v3.4h, v2.4h
+; CHECK-SD-NEXT:    add v6.4s, v17.4s, v16.4s
+; CHECK-SD-NEXT:    saddw v0.4s, v1.4s, v0.4h
+; CHECK-SD-NEXT:    add v1.4s, v4.4s, v5.4s
+; CHECK-SD-NEXT:    add v0.4s, v0.4s, v2.4s
+; CHECK-SD-NEXT:    add v1.4s, v6.4s, v1.4s
+; CHECK-SD-NEXT:    add v0.4s, v0.4s, v1.4s
+; CHECK-SD-NEXT:    addv s0, v0.4s
+; CHECK-SD-NEXT:    fmov w0, s0
+; CHECK-SD-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: test_sdot_v33i8_double_nomla:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-GI-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-GI-NEXT:    .cfi_offset w29, -16
+; CHECK-GI-NEXT:    sxtb w10, w0
+; CHECK-GI-NEXT:    ldr w9, [sp, #16]
+; CHECK-GI-NEXT:    ldr w8, [sp, #48]
+; CHECK-GI-NEXT:    sxtb w11, w4
+; CHECK-GI-NEXT:    sxtb w12, w6
+; CHECK-GI-NEXT:    ldr w13, [sp, #592]
+; CHECK-GI-NEXT:    mov v0.s[0], w10
+; CHECK-GI-NEXT:    sxtb w9, w9
+; CHECK-GI-NEXT:    sxtb w8, w8
+; CHECK-GI-NEXT:    mov v1.s[0], w11
+; CHECK-GI-NEXT:    sxtb w10, w1
+; CHECK-GI-NEXT:    sxtb w11, w5
+; CHECK-GI-NEXT:    mov v2.s[0], w9
+; CHECK-GI-NEXT:    ldr w9, [sp, #24]
+; CHECK-GI-NEXT:    mov v3.s[0], w8
+; CHECK-GI-NEXT:    ldr w8, [sp, #56]
+; CHECK-GI-NEXT:    mov v22.s[0], wzr
+; CHECK-GI-NEXT:    mov v0.s[1], w10
+; CHECK-GI-NEXT:    sxtb w9, w9
+; CHECK-GI-NEXT:    ldr w10, [sp, #80]
+; CHECK-GI-NEXT:    sxtb w8, w8
+; CHECK-GI-NEXT:    mov v1.s[1], w11
+; CHECK-GI-NEXT:    sxtb w11, w2
+; CHECK-GI-NEXT:    mov v2.s[1], w9
+; CHECK-GI-NEXT:    ldr w9, [sp, #32]
+; CHECK-GI-NEXT:    sxtb w10, w10
+; CHECK-GI-NEXT:    mov v3.s[1], w8
+; CHECK-GI-NEXT:    ldr w8, [sp, #64]
+; CHECK-GI-NEXT:    mov v22.s[1], wzr
+; CHECK-GI-NEXT:    mov v0.s[2], w11
+; CHECK-GI-NEXT:    sxtb w9, w9
+; CHECK-GI-NEXT:    sxtb w11, w3
+; CHECK-GI-NEXT:    sxtb w8, w8
+; CHECK-GI-NEXT:    mov v1.s[2], w12
+; CHECK-GI-NEXT:    mov v4.s[0], w10
+; CHECK-GI-NEXT:    ldr w10, [sp, #40]
+; CHECK-GI-NEXT:    mov v2.s[2], w9
+; CHECK-GI-NEXT:    ldr w9, [sp, #88]
+; CHECK-GI-NEXT:    sxtb w12, w7
+; CHECK-GI-NEXT:    mov v3.s[2], w8
+; CHECK-GI-NEXT:    ldr w8, [sp, #72]
+; CHECK-GI-NEXT:    mov v0.s[3], w11
+; CHECK-GI-NEXT:    ldr w11, [sp, #112]
+; CHECK-GI-NEXT:    sxtb w10, w10
+; CHECK-GI-NEXT:    sxtb w9, w9
+; CHECK-GI-NEXT:    mov v1.s[3], w12
+; CHECK-GI-NEXT:    ldr w12, [sp, #144]
+; CHECK-GI-NEXT:    sxtb w8, w8
+; CHECK-GI-NEXT:    sxtb w11, w11
+; CHECK-GI-NEXT:    mov v2.s[3], w10
+; CHECK-GI-NEXT:    ldr w10, [sp, #96]
+; CHECK-GI-NEXT:    mov v4.s[1], w9
+; CHECK-GI-NEXT:    ldr w9, [sp, #176]
+; CHECK-GI-NEXT:    sxtb w12, w12
+; CHECK-GI-NEXT:    mov v3.s[3], w8
+; CHECK-GI-NEXT:    ldr w8, [sp, #120]
+; CHECK-GI-NEXT:    mov v5.s[0], w11
+; CHECK-GI-NEXT:    sxtb w10, w10
+; CHECK-GI-NEXT:    sxtb w9, w9
+; CHECK-GI-NEXT:    ldr w11, [sp, #152]
+; CHECK-GI-NEXT:    mov v6.s[0], w12
+; CHECK-GI-NEXT:    sxtb w8, w8
+; CHECK-GI-NEXT:    ldr w12, [sp, #104]
+; CHECK-GI-NEXT:    mov v4.s[2], w10
+; CHECK-GI-NEXT:    ldr w10, [sp, #184]
+; CHECK-GI-NEXT:    mov v7.s[0], w9
+; CHECK-GI-NEXT:    sxtb w11, w11
+; CHECK-GI-NEXT:    ldr w9, [sp, #128]
+; CHECK-GI-NEXT:    mov v5.s[1], w8
+; CHECK-GI-NEXT:    sxtb w12, w12
+; CHECK-GI-NEXT:    sxtb w10, w10
+; CHECK-GI-NEXT:    ldr w8, [sp, #160]
+; CHECK-GI-NEXT:    mov v6.s[1], w11
+; CHECK-GI-NEXT:    ldr w11, [sp, #480]
+; CHECK-GI-NEXT:    sxtb w9, w9
+; CHECK-GI-NEXT:    mov v4.s[3], w12
+; CHECK-GI-NEXT:    ldr w12, [sp, #192]
+; CHECK-GI-NEXT:    mov v7.s[1], w10
+; CHECK-GI-NEXT:    sxtb w8, w8
+; CHECK-GI-NEXT:    ldr w10, [sp, #136]
+; CHECK-GI-NEXT:    mov v5.s[2], w9
+; CHECK-GI-NEXT:    sxtb w9, w11
+; CHECK-GI-NEXT:    sxtb w12, w12
+; CHECK-GI-NEXT:    ldr w11, [sp, #168]
+; CHECK-GI-NEXT:    mov v6.s[2], w8
+; CHECK-GI-NEXT:    ldr w8, [sp, #488]
+; CHECK-GI-NEXT:    mov v16.s[0], w9
+; CHECK-GI-NEXT:    sxtb w10, w10
+; CHECK-GI-NEXT:    ldr w9, [sp, #200]
+; CHECK-GI-NEXT:    mov v7.s[2], w12
+; CHECK-GI-NEXT:    sxtb w11, w11
+; CHECK-GI-NEXT:    ldr w12, [sp, #512]
+; CHECK-GI-NEXT:    sxtb w8, w8
+; CHECK-GI-NEXT:    mov v5.s[3], w10
+; CHECK-GI-NEXT:    ldr w10, [sp, #544]
+; CHECK-GI-NEXT:    sxtb w9, w9
+; CHECK-GI-NEXT:    mov v6.s[3], w11
+; CHECK-GI-NEXT:    ldr w11, [sp, #496]
+; CHECK-GI-NEXT:    sxtb w12, w12
+; CHECK-GI-NEXT:    mov v16.s[1], w8
+; CHECK-GI-NEXT:    ldr w8, [sp, #208]
+; CHECK-GI-NEXT:    mov v7.s[3], w9
+; CHECK-GI-NEXT:    sxtb w9, w10
+; CHECK-GI-NEXT:    ldr w10, [sp, #520]
+; CHECK-GI-NEXT:    sxtb w11, w11
+; CHECK-GI-NEXT:    mov v17.s[0], w12
+; CHECK-GI-NEXT:    ldr w12, [sp, #552]
+; CHECK-GI-NEXT:    sxtb w8, w8
+; CHECK-GI-NEXT:    mov v19.s[0], w9
+; CHECK-GI-NEXT:    ldr w9, [sp, #504]
+; CHECK-GI-NEXT:    sxtb w10, w10
+; CHECK-GI-NEXT:    mov v16.s[2], w11
+; CHECK-GI-NEXT:    ldr w11, [sp, #576]
+; CHECK-GI-NEXT:    mov v18.s[0], w8
+; CHECK-GI-NEXT:    sxtb w8, w12
+; CHECK-GI-NEXT:    ldr w12, [sp, #528]
+; CHECK-GI-NEXT:    sxtb w9, w9
+; CHECK-GI-NEXT:    mov v17.s[1], w10
+; CHECK-GI-NEXT:    sxtb w11, w11
+; CHECK-GI-NEXT:    ldr w10, [sp, #560]
+; CHECK-GI-NEXT:    mov v19.s[1], w8
+; CHECK-GI-NEXT:    ldr w8, [sp, #608]
+; CHECK-GI-NEXT:    sxtb w12, w12
+; CHECK-GI-NEXT:    mov v16.s[3], w9
+; CHECK-GI-NEXT:    ldr w9, [sp, #584]
+; CHECK-GI-NEXT:    mov v20.s[0], w11
+; CHECK-GI-NEXT:    sxtb w10, w10
+; CHECK-GI-NEXT:    sxtb w8, w8
+; CHECK-GI-NEXT:    ldr w11, [sp, #536]
+; CHECK-GI-NEXT:    mov v17.s[2], w12
+; CHECK-GI-NEXT:    sxtb w9, w9
+; CHECK-GI-NEXT:    mov v19.s[2], w10
+; CHECK-GI-NEXT:    ldr w10, [sp, #616]
+; CHECK-GI-NEXT:    mov v21.s[0], w8
+; CHECK-GI-NEXT:    ldr w12, [sp, #568]
+; CHECK-GI-NEXT:    sxtb w11, w11
+; CHECK-GI-NEXT:    mov v22.s[2], wzr
+; CHECK-GI-NEXT:    mov v20.s[1], w9
+; CHECK-GI-NEXT:    sxtb w9, w10
+; CHECK-GI-NEXT:    ldr w10, [sp, #640]
+; CHECK-GI-NEXT:    sxtb w12, w12
+; CHECK-GI-NEXT:    mov v17.s[3], w11
+; CHECK-GI-NEXT:    sxtb w11, w13
+; CHECK-GI-NEXT:    mov v21.s[1], w9
+; CHECK-GI-NEXT:    ldr w9, [sp, #672]
+; CHECK-GI-NEXT:    sxtb w10, w10
+; CHECK-GI-NEXT:    mov v19.s[3], w12
+; CHECK-GI-NEXT:    ldr w12, [sp, #624]
+; CHECK-GI-NEXT:    ldr w8, [sp, #600]
+; CHECK-GI-NEXT:    mov v20.s[2], w11
+; CHECK-GI-NEXT:    ldr w11, [sp, #704]
+; CHECK-GI-NEXT:    sxtb w9, w9
+; CHECK-GI-NEXT:    sxtb w12, w12
+; CHECK-GI-NEXT:    mov v23.s[0], w10
+; CHECK-GI-NEXT:    ldr w10, [sp, #648]
+; CHECK-GI-NEXT:    sxtb w11, w11
+; CHECK-GI-NEXT:    mov v24.s[0], w9
+; CHECK-GI-NEXT:    ldr w9, [sp, #736]
+; CHECK-GI-NEXT:    mov v21.s[2], w12
+; CHECK-GI-NEXT:    ldr w12, [sp, #680]
+; CHECK-GI-NEXT:    sxtb w10, w10
+; CHECK-GI-NEXT:    mov v25.s[0], w11
+; CHECK-GI-NEXT:    ldr w11, [sp, #712]
+; CHECK-GI-NEXT:    sxtb w9, w9
+; CHECK-GI-NEXT:    sxtb w12, w12
+; CHECK-GI-NEXT:    mov v23.s[1], w10
+; CHECK-GI-NEXT:    ldr w10, [sp, #656]
+; CHECK-GI-NEXT:    sxtb w11, w11
+; CHECK-GI-NEXT:    mov v26.s[0], w9
+; CHECK-GI-NEXT:    ldr w9, [sp, #720]
+; CHECK-GI-NEXT:    mov v24.s[1], w12
+; CHECK-GI-NEXT:    ldr w12, [sp, #688]
+; CHECK-GI-NEXT:    sxtb w10, w10
+; CHECK-GI-NEXT:    mov v25.s[1], w11
+; CHECK-GI-NEXT:    sxtb w9, w9
+; CHECK-GI-NEXT:    mov v18.s[1], wzr
+; CHECK-GI-NEXT:    sxtb w12, w12
+; CHECK-GI-NEXT:    ldr w11, [sp, #632]
+; CHECK-GI-NEXT:    mov v23.s[2], w10
+; CHECK-GI-NEXT:    mov v26.s[1], wzr
+; CHECK-GI-NEXT:    ldr w10, [sp, #664]
+; CHECK-GI-NEXT:    sxtb w8, w8
+; CHECK-GI-NEXT:    mov v24.s[2], w12
+; CHECK-GI-NEXT:    ldr w12, [sp, #696]
+; CHECK-GI-NEXT:    mov v22.s[3], wzr
+; CHECK-GI-NEXT:    mov v25.s[2], w9
+; CHECK-GI-NEXT:    ldr w9, [sp, #728]
+; CHECK-GI-NEXT:    mov v20.s[3], w8
+; CHECK-GI-NEXT:    sxtb w8, w11
+; CHECK-GI-NEXT:    sxtb w10, w10
+; CHECK-GI-NEXT:    sxtb w11, w12
+; CHECK-GI-NEXT:    sxtb w9, w9
+; CHECK-GI-NEXT:    mov v18.s[2], wzr
+; CHECK-GI-NEXT:    mov v26.s[2], wzr
+; CHECK-GI-NEXT:    mov v21.s[3], w8
+; CHECK-GI-NEXT:    mov v23.s[3], w10
+; CHECK-GI-NEXT:    mov v24.s[3], w11
+; CHECK-GI-NEXT:    mov v25.s[3], w9
+; CHECK-GI-NEXT:    add v27.4s, v22.4s, v22.4s
+; CHECK-GI-NEXT:    add v0.4s, v0.4s, v1.4s
+; CHECK-GI-NEXT:    add v1.4s, v2.4s, v3.4s
+; CHECK-GI-NEXT:    add v2.4s, v4.4s, v5.4s
+; CHECK-GI-NEXT:    add v3.4s, v6.4s, v7.4s
+; CHECK-GI-NEXT:    mov v18.s[3], wzr
+; CHECK-GI-NEXT:    mov v26.s[3], wzr
+; CHECK-GI-NEXT:    add v4.4s, v16.4s, v17.4s
+; CHECK-GI-NEXT:    add v5.4s, v22.4s, v27.4s
+; CHECK-GI-NEXT:    add v6.4s, v19.4s, v20.4s
+; CHECK-GI-NEXT:    add v7.4s, v21.4s, v23.4s
+; CHECK-GI-NEXT:    add v16.4s, v24.4s, v25.4s
+; CHECK-GI-NEXT:    add v0.4s, v0.4s, v1.4s
+; CHECK-GI-NEXT:    add v1.4s, v2.4s, v3.4s
+; CHECK-GI-NEXT:    add v3.4s, v27.4s, v27.4s
+; CHECK-GI-NEXT:    add v2.4s, v18.4s, v5.4s
+; CHECK-GI-NEXT:    add v4.4s, v4.4s, v6.4s
+; CHECK-GI-NEXT:    add v5.4s, v26.4s, v5.4s
+; CHECK-GI-NEXT:    add v6.4s, v7.4s, v16.4s
+; CHECK-GI-NEXT:    add v0.4s, v0.4s, v1.4s
+; CHECK-GI-NEXT:    add v1.4s, v2.4s, v3.4s
+; CHECK-GI-NEXT:    add v3.4s, v5.4s, v3.4s
+; CHECK-GI-NEXT:    add v2.4s, v4.4s, v6.4s
+; CHECK-GI-NEXT:    add v0.4s, v0.4s, v1.4s
+; CHECK-GI-NEXT:    add v1.4s, v2.4s, v3.4s
+; CHECK-GI-NEXT:    addv s0, v0.4s
+; CHECK-GI-NEXT:    addv s1, v1.4s
+; CHECK-GI-NEXT:    fmov w8, s0
+; CHECK-GI-NEXT:    fmov w9, s1
+; CHECK-GI-NEXT:    add w0, w8, w9
+; CHECK-GI-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
+; CHECK-GI-NEXT:    ret
 entry:
   %az = sext <33 x i8> %a to <33 x i32>
   %r1 = call i32 @llvm.vector.reduce.add.v33i32(<33 x i32> %az)
@@ -3602,24 +5748,25 @@ define i32 @test_udot_v48i8(ptr nocapture readonly %a, ptr nocapture readonly %b
 ;
 ; CHECK-GI-LABEL: test_udot_v48i8:
 ; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    movi v0.2d, #0000000000000000
+; CHECK-GI-NEXT:    mov v0.s[0], wzr
 ; CHECK-GI-NEXT:    movi v1.2d, #0000000000000000
 ; CHECK-GI-NEXT:    ldr q7, [x0, #32]
-; CHECK-GI-NEXT:    ldp q3, q4, [x0]
 ; CHECK-GI-NEXT:    movi v2.2d, #0000000000000000
-; CHECK-GI-NEXT:    ldp q5, q6, [x1]
-; CHECK-GI-NEXT:    ldr q16, [x1, #32]
-; CHECK-GI-NEXT:    udot v0.4s, v5.16b, v3.16b
+; CHECK-GI-NEXT:    movi v3.2d, #0000000000000000
+; CHECK-GI-NEXT:    ldr q17, [x1, #32]
+; CHECK-GI-NEXT:    ldp q4, q5, [x0]
+; CHECK-GI-NEXT:    ldp q6, q16, [x1]
+; CHECK-GI-NEXT:    mov v0.s[1], wzr
+; CHECK-GI-NEXT:    udot v2.4s, v17.16b, v7.16b
 ; CHECK-GI-NEXT:    udot v1.4s, v6.16b, v4.16b
-; CHECK-GI-NEXT:    udot v2.4s, v16.16b, v7.16b
+; CHECK-GI-NEXT:    udot v3.4s, v16.16b, v5.16b
+; CHECK-GI-NEXT:    mov v0.s[2], wzr
+; CHECK-GI-NEXT:    add v1.4s, v1.4s, v3.4s
+; CHECK-GI-NEXT:    mov v0.s[3], wzr
+; CHECK-GI-NEXT:    add v0.4s, v2.4s, v0.4s
+; CHECK-GI-NEXT:    add v0.4s, v1.4s, v0.4s
 ; CHECK-GI-NEXT:    addv s0, v0.4s
-; CHECK-GI-NEXT:    addv s1, v1.4s
-; CHECK-GI-NEXT:    addv s2, v2.4s
 ; CHECK-GI-NEXT:    fmov w8, s0
-; CHECK-GI-NEXT:    fmov w9, s1
-; CHECK-GI-NEXT:    add w8, w8, w9
-; CHECK-GI-NEXT:    fmov w9, s2
-; CHECK-GI-NEXT:    add w8, w8, w9
 ; CHECK-GI-NEXT:    add w0, w8, w2
 ; CHECK-GI-NEXT:    ret
 entry:
@@ -3649,23 +5796,24 @@ define i32 @test_udot_v48i8_nomla(ptr nocapture readonly %a1) {
 ;
 ; CHECK-GI-LABEL: test_udot_v48i8_nomla:
 ; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    movi v0.16b, #1
-; CHECK-GI-NEXT:    movi v1.2d, #0000000000000000
-; CHECK-GI-NEXT:    ldr q6, [x0, #32]
+; CHECK-GI-NEXT:    mov v0.s[0], wzr
+; CHECK-GI-NEXT:    movi v1.16b, #1
+; CHECK-GI-NEXT:    ldr q7, [x0, #32]
 ; CHECK-GI-NEXT:    movi v2.2d, #0000000000000000
-; CHECK-GI-NEXT:    ldp q4, q5, [x0]
 ; CHECK-GI-NEXT:    movi v3.2d, #0000000000000000
-; CHECK-GI-NEXT:    udot v1.4s, v4.16b, v0.16b
-; CHECK-GI-NEXT:    udot v2.4s, v5.16b, v0.16b
-; CHECK-GI-NEXT:    udot v3.4s, v6.16b, v0.16b
-; CHECK-GI-NEXT:    addv s0, v1.4s
-; CHECK-GI-NEXT:    addv s1, v2.4s
-; CHECK-GI-NEXT:    addv s2, v3.4s
-; CHECK-GI-NEXT:    fmov w8, s0
-; CHECK-GI-NEXT:    fmov w9, s1
-; CHECK-GI-NEXT:    add w8, w8, w9
-; CHECK-GI-NEXT:    fmov w9, s2
-; CHECK-GI-NEXT:    add w0, w8, w9
+; CHECK-GI-NEXT:    movi v4.2d, #0000000000000000
+; CHECK-GI-NEXT:    ldp q5, q6, [x0]
+; CHECK-GI-NEXT:    mov v0.s[1], wzr
+; CHECK-GI-NEXT:    udot v2.4s, v5.16b, v1.16b
+; CHECK-GI-NEXT:    udot v4.4s, v6.16b, v1.16b
+; CHECK-GI-NEXT:    udot v3.4s, v7.16b, v1.16b
+; CHECK-GI-NEXT:    mov v0.s[2], wzr
+; CHECK-GI-NEXT:    add v1.4s, v2.4s, v4.4s
+; CHECK-GI-NEXT:    mov v0.s[3], wzr
+; CHECK-GI-NEXT:    add v0.4s, v3.4s, v0.4s
+; CHECK-GI-NEXT:    add v0.4s, v1.4s, v0.4s
+; CHECK-GI-NEXT:    addv s0, v0.4s
+; CHECK-GI-NEXT:    fmov w0, s0
 ; CHECK-GI-NEXT:    ret
 entry:
   %0 = load <48 x i8>, ptr %a1
@@ -3691,24 +5839,25 @@ define i32 @test_sdot_v48i8(ptr nocapture readonly %a, ptr nocapture readonly %b
 ;
 ; CHECK-GI-LABEL: test_sdot_v48i8:
 ; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    movi v0.2d, #0000000000000000
+; CHECK-GI-NEXT:    mov v0.s[0], wzr
 ; CHECK-GI-NEXT:    movi v1.2d, #0000000000000000
 ; CHECK-GI-NEXT:    ldr q7, [x0, #32]
-; CHECK-GI-NEXT:    ldp q3, q4, [x0]
 ; CHECK-GI-NEXT:    movi v2.2d, #0000000000000000
-; CHECK-GI-NEXT:    ldp q5, q6, [x1]
-; CHECK-GI-NEXT:    ldr q16, [x1, #32]
-; CHECK-GI-NEXT:    sdot v0.4s, v5.16b, v3.16b
+; CHECK-GI-NEXT:    movi v3.2d, #0000000000000000
+; CHECK-GI-NEXT:    ldr q17, [x1, #32]
+; CHECK-GI-NEXT:    ldp q4, q5, [x0]
+; CHECK-GI-NEXT:    ldp q6, q16, [x1]
+; CHECK-GI-NEXT:    mov v0.s[1], wzr
+; CHECK-GI-NEXT:    sdot v2.4s, v17.16b, v7.16b
 ; CHECK-GI-NEXT:    sdot v1.4s, v6.16b, v4.16b
-; CHECK-GI-NEXT:    sdot v2.4s, v16.16b, v7.16b
+; CHECK-GI-NEXT:    sdot v3.4s, v16.16b, v5.16b
+; CHECK-GI-NEXT:    mov v0.s[2], wzr
+; CHECK-GI-NEXT:    add v1.4s, v1.4s, v3.4s
+; CHECK-GI-NEXT:    mov v0.s[3], wzr
+; CHECK-GI-NEXT:    add v0.4s, v2.4s, v0.4s
+; CHECK-GI-NEXT:    add v0.4s, v1.4s, v0.4s
 ; CHECK-GI-NEXT:    addv s0, v0.4s
-; CHECK-GI-NEXT:    addv s1, v1.4s
-; CHECK-GI-NEXT:    addv s2, v2.4s
 ; CHECK-GI-NEXT:    fmov w8, s0
-; CHECK-GI-NEXT:    fmov w9, s1
-; CHECK-GI-NEXT:    add w8, w8, w9
-; CHECK-GI-NEXT:    fmov w9, s2
-; CHECK-GI-NEXT:    add w8, w8, w9
 ; CHECK-GI-NEXT:    add w0, w8, w2
 ; CHECK-GI-NEXT:    ret
 entry:
@@ -4113,409 +6262,407 @@ define i32 @test_sdot_v48i8_double(<48 x i8> %a, <48 x i8> %b, <48 x i8> %c, <48
 ; CHECK-GI-NEXT:    .cfi_def_cfa_offset 16
 ; CHECK-GI-NEXT:    .cfi_offset w29, -16
 ; CHECK-GI-NEXT:    ldr w11, [sp, #80]
-; CHECK-GI-NEXT:    ldr w10, [sp, #208]
+; CHECK-GI-NEXT:    ldr w8, [sp, #208]
 ; CHECK-GI-NEXT:    fmov s0, w0
-; CHECK-GI-NEXT:    ldr w8, [sp, #88]
-; CHECK-GI-NEXT:    ldr w12, [sp, #344]
-; CHECK-GI-NEXT:    movi v20.2d, #0000000000000000
+; CHECK-GI-NEXT:    ldr w12, [sp, #336]
+; CHECK-GI-NEXT:    ldr w9, [sp, #88]
+; CHECK-GI-NEXT:    mov v20.s[0], wzr
 ; CHECK-GI-NEXT:    fmov s1, w11
-; CHECK-GI-NEXT:    ldr w11, [sp, #336]
-; CHECK-GI-NEXT:    fmov s2, w10
-; CHECK-GI-NEXT:    ldr w10, [sp, #464]
-; CHECK-GI-NEXT:    ldr w9, [sp, #216]
+; CHECK-GI-NEXT:    fmov s2, w8
+; CHECK-GI-NEXT:    ldr w11, [sp, #464]
+; CHECK-GI-NEXT:    ldr w8, [sp, #592]
+; CHECK-GI-NEXT:    ldr w10, [sp, #216]
+; CHECK-GI-NEXT:    fmov s3, w12
+; CHECK-GI-NEXT:    fmov s4, w11
 ; CHECK-GI-NEXT:    mov v0.b[1], w1
-; CHECK-GI-NEXT:    fmov s3, w11
 ; CHECK-GI-NEXT:    ldr w11, [sp, #600]
-; CHECK-GI-NEXT:    movi v21.2d, #0000000000000000
-; CHECK-GI-NEXT:    mov v1.b[1], w8
-; CHECK-GI-NEXT:    ldr w8, [sp, #592]
-; CHECK-GI-NEXT:    fmov s4, w10
-; CHECK-GI-NEXT:    mov v2.b[1], w9
-; CHECK-GI-NEXT:    ldr w9, [sp, #472]
-; CHECK-GI-NEXT:    ldr w10, [sp, #608]
-; CHECK-GI-NEXT:    mov v3.b[1], w12
 ; CHECK-GI-NEXT:    fmov s5, w8
+; CHECK-GI-NEXT:    mov v1.b[1], w9
+; CHECK-GI-NEXT:    mov v2.b[1], w10
+; CHECK-GI-NEXT:    ldr w10, [sp, #344]
+; CHECK-GI-NEXT:    ldr w9, [sp, #472]
 ; CHECK-GI-NEXT:    ldr w8, [sp, #96]
+; CHECK-GI-NEXT:    ldr w12, [sp, #848]
+; CHECK-GI-NEXT:    ldr w13, [sp, #728]
+; CHECK-GI-NEXT:    mov v20.s[1], wzr
+; CHECK-GI-NEXT:    mov v3.b[1], w10
 ; CHECK-GI-NEXT:    mov v4.b[1], w9
+; CHECK-GI-NEXT:    mov v5.b[1], w11
 ; CHECK-GI-NEXT:    ldr w9, [sp, #224]
 ; CHECK-GI-NEXT:    mov v0.b[2], w2
+; CHECK-GI-NEXT:    ldr w10, [sp, #352]
+; CHECK-GI-NEXT:    ldr w11, [sp, #480]
 ; CHECK-GI-NEXT:    mov v1.b[2], w8
-; CHECK-GI-NEXT:    ldr w8, [sp, #352]
-; CHECK-GI-NEXT:    ldr w12, [sp, #848]
+; CHECK-GI-NEXT:    ldr w8, [sp, #608]
 ; CHECK-GI-NEXT:    mov v2.b[2], w9
-; CHECK-GI-NEXT:    ldr w9, [sp, #480]
-; CHECK-GI-NEXT:    mov v5.b[1], w11
-; CHECK-GI-NEXT:    mov v3.b[2], w8
-; CHECK-GI-NEXT:    ldr w8, [sp, #104]
-; CHECK-GI-NEXT:    ldr w11, [sp, #16]
-; CHECK-GI-NEXT:    mov v4.b[2], w9
-; CHECK-GI-NEXT:    ldr w9, [sp, #232]
+; CHECK-GI-NEXT:    ldr w9, [sp, #104]
+; CHECK-GI-NEXT:    fmov s7, w12
+; CHECK-GI-NEXT:    mov v3.b[2], w10
+; CHECK-GI-NEXT:    mov v4.b[2], w11
+; CHECK-GI-NEXT:    mov v5.b[2], w8
+; CHECK-GI-NEXT:    ldr w10, [sp, #232]
 ; CHECK-GI-NEXT:    mov v0.b[3], w3
-; CHECK-GI-NEXT:    mov v1.b[3], w8
 ; CHECK-GI-NEXT:    ldr w8, [sp, #360]
-; CHECK-GI-NEXT:    fmov s7, w12
-; CHECK-GI-NEXT:    mov v2.b[3], w9
-; CHECK-GI-NEXT:    ldr w9, [sp, #488]
-; CHECK-GI-NEXT:    mov v5.b[2], w10
+; CHECK-GI-NEXT:    ldr w11, [sp, #488]
+; CHECK-GI-NEXT:    mov v1.b[3], w9
+; CHECK-GI-NEXT:    ldr w9, [sp, #616]
+; CHECK-GI-NEXT:    mov v2.b[3], w10
+; CHECK-GI-NEXT:    ldr w10, [sp, #240]
+; CHECK-GI-NEXT:    ldr w12, [sp, #1232]
 ; CHECK-GI-NEXT:    mov v3.b[3], w8
+; CHECK-GI-NEXT:    mov v4.b[3], w11
 ; CHECK-GI-NEXT:    ldr w8, [sp, #112]
-; CHECK-GI-NEXT:    ldr w10, [sp, #616]
-; CHECK-GI-NEXT:    mov v4.b[3], w9
-; CHECK-GI-NEXT:    ldr w9, [sp, #240]
+; CHECK-GI-NEXT:    mov v5.b[3], w9
 ; CHECK-GI-NEXT:    mov v0.b[4], w4
+; CHECK-GI-NEXT:    ldr w9, [sp, #368]
+; CHECK-GI-NEXT:    ldr w11, [sp, #496]
 ; CHECK-GI-NEXT:    mov v1.b[4], w8
-; CHECK-GI-NEXT:    ldr w8, [sp, #368]
-; CHECK-GI-NEXT:    ldr w12, [sp, #1112]
-; CHECK-GI-NEXT:    mov v2.b[4], w9
-; CHECK-GI-NEXT:    ldr w9, [sp, #496]
-; CHECK-GI-NEXT:    mov v5.b[3], w10
-; CHECK-GI-NEXT:    mov v3.b[4], w8
-; CHECK-GI-NEXT:    ldr w8, [sp, #120]
-; CHECK-GI-NEXT:    ldr w10, [sp, #624]
-; CHECK-GI-NEXT:    mov v4.b[4], w9
-; CHECK-GI-NEXT:    ldr w9, [sp, #248]
+; CHECK-GI-NEXT:    ldr w8, [sp, #624]
+; CHECK-GI-NEXT:    mov v2.b[4], w10
+; CHECK-GI-NEXT:    ldr w10, [sp, #248]
+; CHECK-GI-NEXT:    fmov s18, w12
+; CHECK-GI-NEXT:    mov v3.b[4], w9
+; CHECK-GI-NEXT:    mov v4.b[4], w11
+; CHECK-GI-NEXT:    ldr w9, [sp, #120]
+; CHECK-GI-NEXT:    mov v5.b[4], w8
 ; CHECK-GI-NEXT:    mov v0.b[5], w5
-; CHECK-GI-NEXT:    mov v1.b[5], w8
 ; CHECK-GI-NEXT:    ldr w8, [sp, #376]
-; CHECK-GI-NEXT:    movi v22.2d, #0000000000000000
-; CHECK-GI-NEXT:    mov v2.b[5], w9
-; CHECK-GI-NEXT:    ldr w9, [sp, #504]
-; CHECK-GI-NEXT:    mov v5.b[4], w10
+; CHECK-GI-NEXT:    ldr w11, [sp, #504]
+; CHECK-GI-NEXT:    mov v1.b[5], w9
+; CHECK-GI-NEXT:    ldr w9, [sp, #632]
+; CHECK-GI-NEXT:    mov v2.b[5], w10
+; CHECK-GI-NEXT:    ldr w10, [sp, #256]
+; CHECK-GI-NEXT:    ldr w12, [sp, #744]
 ; CHECK-GI-NEXT:    mov v3.b[5], w8
+; CHECK-GI-NEXT:    mov v4.b[5], w11
 ; CHECK-GI-NEXT:    ldr w8, [sp, #128]
-; CHECK-GI-NEXT:    ldr w10, [sp, #632]
-; CHECK-GI-NEXT:    mov v4.b[5], w9
-; CHECK-GI-NEXT:    ldr w9, [sp, #256]
+; CHECK-GI-NEXT:    mov v5.b[5], w9
 ; CHECK-GI-NEXT:    mov v0.b[6], w6
+; CHECK-GI-NEXT:    ldr w9, [sp, #384]
+; CHECK-GI-NEXT:    ldr w11, [sp, #512]
 ; CHECK-GI-NEXT:    mov v1.b[6], w8
-; CHECK-GI-NEXT:    ldr w8, [sp, #384]
-; CHECK-GI-NEXT:    movi v23.2d, #0000000000000000
-; CHECK-GI-NEXT:    mov v2.b[6], w9
-; CHECK-GI-NEXT:    ldr w9, [sp, #512]
-; CHECK-GI-NEXT:    mov v5.b[5], w10
-; CHECK-GI-NEXT:    mov v3.b[6], w8
-; CHECK-GI-NEXT:    ldr w8, [sp, #136]
-; CHECK-GI-NEXT:    ldr w10, [sp, #640]
-; CHECK-GI-NEXT:    mov v4.b[6], w9
-; CHECK-GI-NEXT:    ldr w9, [sp, #264]
+; CHECK-GI-NEXT:    ldr w8, [sp, #640]
+; CHECK-GI-NEXT:    mov v2.b[6], w10
+; CHECK-GI-NEXT:    ldr w10, [sp, #264]
+; CHECK-GI-NEXT:    movi v21.2d, #0000000000000000
+; CHECK-GI-NEXT:    mov v3.b[6], w9
+; CHECK-GI-NEXT:    mov v4.b[6], w11
+; CHECK-GI-NEXT:    ldr w9, [sp, #136]
+; CHECK-GI-NEXT:    mov v5.b[6], w8
 ; CHECK-GI-NEXT:    mov v0.b[7], w7
-; CHECK-GI-NEXT:    mov v1.b[7], w8
 ; CHECK-GI-NEXT:    ldr w8, [sp, #392]
-; CHECK-GI-NEXT:    movi v24.2d, #0000000000000000
-; CHECK-GI-NEXT:    mov v2.b[7], w9
-; CHECK-GI-NEXT:    ldr w9, [sp, #520]
-; CHECK-GI-NEXT:    mov v5.b[6], w10
+; CHECK-GI-NEXT:    ldr w11, [sp, #520]
+; CHECK-GI-NEXT:    mov v1.b[7], w9
+; CHECK-GI-NEXT:    ldr w9, [sp, #648]
+; CHECK-GI-NEXT:    mov v2.b[7], w10
+; CHECK-GI-NEXT:    ldr w10, [sp, #16]
+; CHECK-GI-NEXT:    movi v22.2d, #0000000000000000
 ; CHECK-GI-NEXT:    mov v3.b[7], w8
+; CHECK-GI-NEXT:    mov v4.b[7], w11
 ; CHECK-GI-NEXT:    ldr w8, [sp, #144]
-; CHECK-GI-NEXT:    ldr w10, [sp, #648]
-; CHECK-GI-NEXT:    mov v4.b[7], w9
-; CHECK-GI-NEXT:    ldr w9, [sp, #272]
-; CHECK-GI-NEXT:    mov v0.b[8], w11
+; CHECK-GI-NEXT:    mov v5.b[7], w9
+; CHECK-GI-NEXT:    ldr w11, [sp, #272]
+; CHECK-GI-NEXT:    mov v0.b[8], w10
+; CHECK-GI-NEXT:    ldr w9, [sp, #400]
+; CHECK-GI-NEXT:    ldr w10, [sp, #528]
 ; CHECK-GI-NEXT:    mov v1.b[8], w8
-; CHECK-GI-NEXT:    ldr w8, [sp, #400]
+; CHECK-GI-NEXT:    ldr w8, [sp, #656]
+; CHECK-GI-NEXT:    mov v2.b[8], w11
 ; CHECK-GI-NEXT:    ldr w11, [sp, #24]
-; CHECK-GI-NEXT:    mov v2.b[8], w9
-; CHECK-GI-NEXT:    ldr w9, [sp, #528]
-; CHECK-GI-NEXT:    mov v5.b[7], w10
-; CHECK-GI-NEXT:    mov v3.b[8], w8
-; CHECK-GI-NEXT:    ldr w8, [sp, #152]
-; CHECK-GI-NEXT:    ldr w10, [sp, #656]
-; CHECK-GI-NEXT:    mov v4.b[8], w9
-; CHECK-GI-NEXT:    ldr w9, [sp, #280]
+; CHECK-GI-NEXT:    mov v3.b[8], w9
+; CHECK-GI-NEXT:    mov v4.b[8], w10
+; CHECK-GI-NEXT:    ldr w9, [sp, #152]
+; CHECK-GI-NEXT:    mov v5.b[8], w8
+; CHECK-GI-NEXT:    ldr w10, [sp, #280]
 ; CHECK-GI-NEXT:    mov v0.b[9], w11
-; CHECK-GI-NEXT:    mov v1.b[9], w8
 ; CHECK-GI-NEXT:    ldr w8, [sp, #408]
-; CHECK-GI-NEXT:    ldr w11, [sp, #32]
-; CHECK-GI-NEXT:    mov v2.b[9], w9
-; CHECK-GI-NEXT:    ldr w9, [sp, #536]
-; CHECK-GI-NEXT:    mov v5.b[8], w10
+; CHECK-GI-NEXT:    ldr w11, [sp, #536]
+; CHECK-GI-NEXT:    mov v1.b[9], w9
+; CHECK-GI-NEXT:    ldr w9, [sp, #664]
+; CHECK-GI-NEXT:    mov v2.b[9], w10
+; CHECK-GI-NEXT:    ldr w10, [sp, #32]
 ; CHECK-GI-NEXT:    mov v3.b[9], w8
+; CHECK-GI-NEXT:    mov v4.b[9], w11
 ; CHECK-GI-NEXT:    ldr w8, [sp, #160]
-; CHECK-GI-NEXT:    ldr w10, [sp, #664]
-; CHECK-GI-NEXT:    mov v4.b[9], w9
-; CHECK-GI-NEXT:    ldr w9, [sp, #288]
-; CHECK-GI-NEXT:    mov v0.b[10], w11
+; CHECK-GI-NEXT:    mov v5.b[9], w9
+; CHECK-GI-NEXT:    ldr w11, [sp, #288]
+; CHECK-GI-NEXT:    mov v0.b[10], w10
+; CHECK-GI-NEXT:    ldr w9, [sp, #416]
+; CHECK-GI-NEXT:    ldr w10, [sp, #544]
 ; CHECK-GI-NEXT:    mov v1.b[10], w8
-; CHECK-GI-NEXT:    ldr w8, [sp, #416]
+; CHECK-GI-NEXT:    ldr w8, [sp, #672]
+; CHECK-GI-NEXT:    mov v2.b[10], w11
 ; CHECK-GI-NEXT:    ldr w11, [sp, #40]
-; CHECK-GI-NEXT:    mov v2.b[10], w9
-; CHECK-GI-NEXT:    ldr w9, [sp, #544]
-; CHECK-GI-NEXT:    mov v5.b[9], w10
-; CHECK-GI-NEXT:    mov v3.b[10], w8
-; CHECK-GI-NEXT:    ldr w8, [sp, #168]
-; CHECK-GI-NEXT:    ldr w10, [sp, #672]
-; CHECK-GI-NEXT:    mov v4.b[10], w9
-; CHECK-GI-NEXT:    ldr w9, [sp, #296]
+; CHECK-GI-NEXT:    mov v3.b[10], w9
+; CHECK-GI-NEXT:    mov v4.b[10], w10
+; CHECK-GI-NEXT:    ldr w9, [sp, #168]
+; CHECK-GI-NEXT:    mov v5.b[10], w8
+; CHECK-GI-NEXT:    ldr w10, [sp, #296]
 ; CHECK-GI-NEXT:    mov v0.b[11], w11
-; CHECK-GI-NEXT:    mov v1.b[11], w8
 ; CHECK-GI-NEXT:    ldr w8, [sp, #424]
-; CHECK-GI-NEXT:    ldr w11, [sp, #48]
-; CHECK-GI-NEXT:    mov v2.b[11], w9
-; CHECK-GI-NEXT:    ldr w9, [sp, #552]
-; CHECK-GI-NEXT:    mov v5.b[10], w10
+; CHECK-GI-NEXT:    ldr w11, [sp, #552]
+; CHECK-GI-NEXT:    mov v1.b[11], w9
+; CHECK-GI-NEXT:    ldr w9, [sp, #680]
+; CHECK-GI-NEXT:    mov v2.b[11], w10
+; CHECK-GI-NEXT:    ldr w10, [sp, #48]
 ; CHECK-GI-NEXT:    mov v3.b[11], w8
+; CHECK-GI-NEXT:    mov v4.b[11], w11
 ; CHECK-GI-NEXT:    ldr w8, [sp, #176]
-; CHECK-GI-NEXT:    ldr w10, [sp, #680]
-; CHECK-GI-NEXT:    mov v4.b[11], w9
-; CHECK-GI-NEXT:    ldr w9, [sp, #304]
-; CHECK-GI-NEXT:    mov v0.b[12], w11
+; CHECK-GI-NEXT:    mov v5.b[11], w9
+; CHECK-GI-NEXT:    ldr w11, [sp, #304]
+; CHECK-GI-NEXT:    mov v0.b[12], w10
+; CHECK-GI-NEXT:    ldr w9, [sp, #432]
+; CHECK-GI-NEXT:    ldr w10, [sp, #560]
 ; CHECK-GI-NEXT:    mov v1.b[12], w8
-; CHECK-GI-NEXT:    ldr w8, [sp, #432]
+; CHECK-GI-NEXT:    ldr w8, [sp, #688]
+; CHECK-GI-NEXT:    mov v2.b[12], w11
 ; CHECK-GI-NEXT:    ldr w11, [sp, #56]
-; CHECK-GI-NEXT:    mov v2.b[12], w9
-; CHECK-GI-NEXT:    ldr w9, [sp, #560]
-; CHECK-GI-NEXT:    mov v5.b[11], w10
-; CHECK-GI-NEXT:    mov v3.b[12], w8
-; CHECK-GI-NEXT:    ldr w8, [sp, #184]
-; CHECK-GI-NEXT:    ldr w10, [sp, #688]
-; CHECK-GI-NEXT:    mov v4.b[12], w9
-; CHECK-GI-NEXT:    ldr w9, [sp, #312]
+; CHECK-GI-NEXT:    mov v3.b[12], w9
+; CHECK-GI-NEXT:    mov v4.b[12], w10
+; CHECK-GI-NEXT:    ldr w9, [sp, #184]
+; CHECK-GI-NEXT:    mov v5.b[12], w8
+; CHECK-GI-NEXT:    ldr w10, [sp, #312]
 ; CHECK-GI-NEXT:    mov v0.b[13], w11
-; CHECK-GI-NEXT:    mov v1.b[13], w8
 ; CHECK-GI-NEXT:    ldr w8, [sp, #440]
-; CHECK-GI-NEXT:    ldr w11, [sp, #64]
-; CHECK-GI-NEXT:    mov v2.b[13], w9
-; CHECK-GI-NEXT:    ldr w9, [sp, #568]
-; CHECK-GI-NEXT:    mov v5.b[12], w10
+; CHECK-GI-NEXT:    ldr w11, [sp, #568]
+; CHECK-GI-NEXT:    mov v1.b[13], w9
+; CHECK-GI-NEXT:    ldr w9, [sp, #696]
+; CHECK-GI-NEXT:    mov v2.b[13], w10
+; CHECK-GI-NEXT:    ldr w10, [sp, #64]
 ; CHECK-GI-NEXT:    mov v3.b[13], w8
+; CHECK-GI-NEXT:    mov v4.b[13], w11
 ; CHECK-GI-NEXT:    ldr w8, [sp, #192]
-; CHECK-GI-NEXT:    ldr w10, [sp, #696]
-; CHECK-GI-NEXT:    mov v4.b[13], w9
-; CHECK-GI-NEXT:    ldr w9, [sp, #320]
-; CHECK-GI-NEXT:    mov v0.b[14], w11
+; CHECK-GI-NEXT:    mov v5.b[13], w9
+; CHECK-GI-NEXT:    ldr w11, [sp, #320]
+; CHECK-GI-NEXT:    mov v0.b[14], w10
+; CHECK-GI-NEXT:    ldr w9, [sp, #448]
+; CHECK-GI-NEXT:    ldr w10, [sp, #576]
 ; CHECK-GI-NEXT:    mov v1.b[14], w8
-; CHECK-GI-NEXT:    ldr w8, [sp, #448]
+; CHECK-GI-NEXT:    ldr w8, [sp, #704]
+; CHECK-GI-NEXT:    mov v2.b[14], w11
 ; CHECK-GI-NEXT:    ldr w11, [sp, #72]
-; CHECK-GI-NEXT:    mov v2.b[14], w9
-; CHECK-GI-NEXT:    ldr w9, [sp, #576]
-; CHECK-GI-NEXT:    mov v5.b[13], w10
-; CHECK-GI-NEXT:    mov v3.b[14], w8
-; CHECK-GI-NEXT:    ldr w8, [sp, #720]
-; CHECK-GI-NEXT:    ldr w10, [sp, #704]
-; CHECK-GI-NEXT:    mov v4.b[14], w9
-; CHECK-GI-NEXT:    ldr w9, [sp, #728]
+; CHECK-GI-NEXT:    mov v3.b[14], w9
+; CHECK-GI-NEXT:    mov v4.b[14], w10
+; CHECK-GI-NEXT:    ldr w9, [sp, #200]
+; CHECK-GI-NEXT:    mov v5.b[14], w8
+; CHECK-GI-NEXT:    ldr w10, [sp, #328]
 ; CHECK-GI-NEXT:    mov v0.b[15], w11
-; CHECK-GI-NEXT:    fmov s6, w8
-; CHECK-GI-NEXT:    ldr w8, [sp, #328]
-; CHECK-GI-NEXT:    ldr w11, [sp, #456]
-; CHECK-GI-NEXT:    mov v5.b[14], w10
-; CHECK-GI-NEXT:    ldr w10, [sp, #200]
-; CHECK-GI-NEXT:    movi v25.2d, #0000000000000000
-; CHECK-GI-NEXT:    mov v2.b[15], w8
-; CHECK-GI-NEXT:    mov v3.b[15], w11
-; CHECK-GI-NEXT:    ldr w11, [sp, #736]
-; CHECK-GI-NEXT:    mov v6.b[1], w9
-; CHECK-GI-NEXT:    ldr w9, [sp, #584]
-; CHECK-GI-NEXT:    ldr w8, [sp, #856]
-; CHECK-GI-NEXT:    mov v1.b[15], w10
-; CHECK-GI-NEXT:    ldr w10, [sp, #712]
-; CHECK-GI-NEXT:    mov v4.b[15], w9
-; CHECK-GI-NEXT:    ldr w9, [sp, #976]
-; CHECK-GI-NEXT:    mov v7.b[1], w8
-; CHECK-GI-NEXT:    ldr w8, [sp, #1232]
-; CHECK-GI-NEXT:    mov v5.b[15], w10
-; CHECK-GI-NEXT:    ldr w10, [sp, #984]
-; CHECK-GI-NEXT:    mov v6.b[2], w11
+; CHECK-GI-NEXT:    ldr w11, [sp, #584]
+; CHECK-GI-NEXT:    mov v1.b[15], w9
+; CHECK-GI-NEXT:    ldr w9, [sp, #712]
+; CHECK-GI-NEXT:    mov v2.b[15], w10
+; CHECK-GI-NEXT:    ldr w10, [sp, #720]
+; CHECK-GI-NEXT:    ldr w8, [sp, #456]
+; CHECK-GI-NEXT:    mov v4.b[15], w11
+; CHECK-GI-NEXT:    ldr w11, [sp, #976]
+; CHECK-GI-NEXT:    movi v23.2d, #0000000000000000
+; CHECK-GI-NEXT:    mov v5.b[15], w9
+; CHECK-GI-NEXT:    ldr w9, [sp, #856]
+; CHECK-GI-NEXT:    fmov s6, w10
+; CHECK-GI-NEXT:    fmov s16, w11
 ; CHECK-GI-NEXT:    ldr w11, [sp, #1104]
-; CHECK-GI-NEXT:    fmov s16, w9
+; CHECK-GI-NEXT:    ldr w10, [sp, #984]
+; CHECK-GI-NEXT:    mov v7.b[1], w9
 ; CHECK-GI-NEXT:    ldr w9, [sp, #1360]
-; CHECK-GI-NEXT:    fmov s18, w8
-; CHECK-GI-NEXT:    ldr w8, [sp, #1368]
+; CHECK-GI-NEXT:    mov v3.b[15], w8
 ; CHECK-GI-NEXT:    fmov s17, w11
-; CHECK-GI-NEXT:    ldr w11, [sp, #1240]
-; CHECK-GI-NEXT:    sdot v20.4s, v0.16b, v3.16b
-; CHECK-GI-NEXT:    mov v16.b[1], w10
+; CHECK-GI-NEXT:    mov v6.b[1], w13
+; CHECK-GI-NEXT:    ldr w13, [sp, #1112]
 ; CHECK-GI-NEXT:    fmov s19, w9
-; CHECK-GI-NEXT:    ldr w10, [sp, #864]
-; CHECK-GI-NEXT:    mov v18.b[1], w11
-; CHECK-GI-NEXT:    ldr w11, [sp, #992]
-; CHECK-GI-NEXT:    ldr w9, [sp, #1120]
-; CHECK-GI-NEXT:    mov v17.b[1], w12
-; CHECK-GI-NEXT:    mov v7.b[2], w10
-; CHECK-GI-NEXT:    ldr w10, [sp, #1248]
-; CHECK-GI-NEXT:    mov v19.b[1], w8
-; CHECK-GI-NEXT:    ldr w8, [sp, #744]
-; CHECK-GI-NEXT:    sdot v21.4s, v1.16b, v4.16b
-; CHECK-GI-NEXT:    mov v16.b[2], w11
-; CHECK-GI-NEXT:    ldr w11, [sp, #872]
-; CHECK-GI-NEXT:    addv s0, v20.4s
-; CHECK-GI-NEXT:    mov v6.b[3], w8
-; CHECK-GI-NEXT:    ldr w8, [sp, #1000]
-; CHECK-GI-NEXT:    mov v18.b[2], w10
-; CHECK-GI-NEXT:    mov v17.b[2], w9
-; CHECK-GI-NEXT:    ldr w9, [sp, #1376]
-; CHECK-GI-NEXT:    ldr w10, [sp, #1128]
-; CHECK-GI-NEXT:    mov v7.b[3], w11
-; CHECK-GI-NEXT:    ldr w11, [sp, #880]
-; CHECK-GI-NEXT:    addv s1, v21.4s
-; CHECK-GI-NEXT:    mov v19.b[2], w9
-; CHECK-GI-NEXT:    ldr w9, [sp, #752]
-; CHECK-GI-NEXT:    mov v16.b[3], w8
+; CHECK-GI-NEXT:    mov v16.b[1], w10
+; CHECK-GI-NEXT:    ldr w10, [sp, #1240]
+; CHECK-GI-NEXT:    ldr w11, [sp, #1368]
+; CHECK-GI-NEXT:    ldr w8, [sp, #736]
+; CHECK-GI-NEXT:    ldr w9, [sp, #864]
+; CHECK-GI-NEXT:    mov v17.b[1], w13
+; CHECK-GI-NEXT:    mov v18.b[1], w10
+; CHECK-GI-NEXT:    ldr w10, [sp, #992]
+; CHECK-GI-NEXT:    mov v19.b[1], w11
+; CHECK-GI-NEXT:    mov v6.b[2], w8
+; CHECK-GI-NEXT:    ldr w8, [sp, #1120]
+; CHECK-GI-NEXT:    mov v7.b[2], w9
+; CHECK-GI-NEXT:    ldr w9, [sp, #1248]
+; CHECK-GI-NEXT:    ldr w11, [sp, #1376]
+; CHECK-GI-NEXT:    mov v16.b[2], w10
+; CHECK-GI-NEXT:    ldr w10, [sp, #752]
+; CHECK-GI-NEXT:    mov v20.s[2], wzr
+; CHECK-GI-NEXT:    mov v17.b[2], w8
+; CHECK-GI-NEXT:    ldr w8, [sp, #872]
+; CHECK-GI-NEXT:    mov v18.b[2], w9
+; CHECK-GI-NEXT:    mov v19.b[2], w11
+; CHECK-GI-NEXT:    ldr w9, [sp, #1000]
+; CHECK-GI-NEXT:    mov v6.b[3], w12
+; CHECK-GI-NEXT:    ldr w11, [sp, #1128]
+; CHECK-GI-NEXT:    mov v7.b[3], w8
 ; CHECK-GI-NEXT:    ldr w8, [sp, #1256]
-; CHECK-GI-NEXT:    sdot v25.4s, v2.16b, v5.16b
-; CHECK-GI-NEXT:    mov v17.b[3], w10
-; CHECK-GI-NEXT:    ldr w10, [sp, #1384]
-; CHECK-GI-NEXT:    mov v6.b[4], w9
-; CHECK-GI-NEXT:    ldr w9, [sp, #1008]
+; CHECK-GI-NEXT:    ldr w12, [sp, #1384]
+; CHECK-GI-NEXT:    mov v16.b[3], w9
+; CHECK-GI-NEXT:    ldr w9, [sp, #760]
+; CHECK-GI-NEXT:    mov v17.b[3], w11
+; CHECK-GI-NEXT:    ldr w11, [sp, #880]
 ; CHECK-GI-NEXT:    mov v18.b[3], w8
-; CHECK-GI-NEXT:    ldr w8, [sp, #1136]
-; CHECK-GI-NEXT:    mov v19.b[3], w10
-; CHECK-GI-NEXT:    ldr w10, [sp, #760]
+; CHECK-GI-NEXT:    mov v19.b[3], w12
+; CHECK-GI-NEXT:    ldr w8, [sp, #1008]
+; CHECK-GI-NEXT:    mov v6.b[4], w10
+; CHECK-GI-NEXT:    ldr w10, [sp, #1136]
 ; CHECK-GI-NEXT:    mov v7.b[4], w11
-; CHECK-GI-NEXT:    mov v16.b[4], w9
-; CHECK-GI-NEXT:    ldr w9, [sp, #1264]
-; CHECK-GI-NEXT:    ldr w11, [sp, #888]
-; CHECK-GI-NEXT:    mov v17.b[4], w8
-; CHECK-GI-NEXT:    ldr w8, [sp, #1392]
-; CHECK-GI-NEXT:    mov v6.b[5], w10
-; CHECK-GI-NEXT:    ldr w10, [sp, #1016]
-; CHECK-GI-NEXT:    mov v18.b[4], w9
-; CHECK-GI-NEXT:    ldr w9, [sp, #1144]
-; CHECK-GI-NEXT:    mov v19.b[4], w8
+; CHECK-GI-NEXT:    ldr w11, [sp, #1264]
+; CHECK-GI-NEXT:    ldr w12, [sp, #1392]
+; CHECK-GI-NEXT:    mov v16.b[4], w8
 ; CHECK-GI-NEXT:    ldr w8, [sp, #768]
-; CHECK-GI-NEXT:    mov v7.b[5], w11
-; CHECK-GI-NEXT:    mov v16.b[5], w10
+; CHECK-GI-NEXT:    mov v17.b[4], w10
+; CHECK-GI-NEXT:    ldr w10, [sp, #888]
+; CHECK-GI-NEXT:    mov v18.b[4], w11
+; CHECK-GI-NEXT:    mov v19.b[4], w12
+; CHECK-GI-NEXT:    ldr w11, [sp, #1016]
+; CHECK-GI-NEXT:    mov v6.b[5], w9
+; CHECK-GI-NEXT:    ldr w9, [sp, #1144]
+; CHECK-GI-NEXT:    mov v7.b[5], w10
 ; CHECK-GI-NEXT:    ldr w10, [sp, #1272]
-; CHECK-GI-NEXT:    ldr w11, [sp, #896]
+; CHECK-GI-NEXT:    ldr w12, [sp, #1400]
+; CHECK-GI-NEXT:    mov v16.b[5], w11
+; CHECK-GI-NEXT:    ldr w11, [sp, #776]
 ; CHECK-GI-NEXT:    mov v17.b[5], w9
-; CHECK-GI-NEXT:    ldr w9, [sp, #1400]
-; CHECK-GI-NEXT:    mov v6.b[6], w8
-; CHECK-GI-NEXT:    ldr w8, [sp, #1024]
+; CHECK-GI-NEXT:    ldr w9, [sp, #896]
 ; CHECK-GI-NEXT:    mov v18.b[5], w10
-; CHECK-GI-NEXT:    ldr w10, [sp, #1152]
-; CHECK-GI-NEXT:    mov v19.b[5], w9
-; CHECK-GI-NEXT:    ldr w9, [sp, #776]
-; CHECK-GI-NEXT:    mov v7.b[6], w11
-; CHECK-GI-NEXT:    mov v16.b[6], w8
-; CHECK-GI-NEXT:    ldr w8, [sp, #1280]
-; CHECK-GI-NEXT:    ldr w11, [sp, #904]
-; CHECK-GI-NEXT:    mov v17.b[6], w10
-; CHECK-GI-NEXT:    ldr w10, [sp, #1408]
-; CHECK-GI-NEXT:    mov v6.b[7], w9
-; CHECK-GI-NEXT:    ldr w9, [sp, #1032]
-; CHECK-GI-NEXT:    mov v18.b[6], w8
-; CHECK-GI-NEXT:    ldr w8, [sp, #1160]
-; CHECK-GI-NEXT:    mov v19.b[6], w10
+; CHECK-GI-NEXT:    mov v19.b[5], w12
+; CHECK-GI-NEXT:    ldr w10, [sp, #1024]
+; CHECK-GI-NEXT:    mov v6.b[6], w8
+; CHECK-GI-NEXT:    ldr w8, [sp, #1152]
+; CHECK-GI-NEXT:    mov v7.b[6], w9
+; CHECK-GI-NEXT:    ldr w9, [sp, #1280]
+; CHECK-GI-NEXT:    ldr w12, [sp, #1408]
+; CHECK-GI-NEXT:    mov v16.b[6], w10
 ; CHECK-GI-NEXT:    ldr w10, [sp, #784]
-; CHECK-GI-NEXT:    mov v7.b[7], w11
+; CHECK-GI-NEXT:    mov v17.b[6], w8
+; CHECK-GI-NEXT:    ldr w8, [sp, #904]
+; CHECK-GI-NEXT:    mov v18.b[6], w9
+; CHECK-GI-NEXT:    mov v19.b[6], w12
+; CHECK-GI-NEXT:    ldr w9, [sp, #1032]
+; CHECK-GI-NEXT:    mov v6.b[7], w11
+; CHECK-GI-NEXT:    ldr w11, [sp, #1160]
+; CHECK-GI-NEXT:    mov v7.b[7], w8
+; CHECK-GI-NEXT:    ldr w8, [sp, #1288]
+; CHECK-GI-NEXT:    ldr w12, [sp, #1416]
 ; CHECK-GI-NEXT:    mov v16.b[7], w9
-; CHECK-GI-NEXT:    ldr w9, [sp, #1288]
+; CHECK-GI-NEXT:    ldr w9, [sp, #792]
+; CHECK-GI-NEXT:    mov v17.b[7], w11
 ; CHECK-GI-NEXT:    ldr w11, [sp, #912]
-; CHECK-GI-NEXT:    mov v17.b[7], w8
-; CHECK-GI-NEXT:    ldr w8, [sp, #1416]
+; CHECK-GI-NEXT:    mov v18.b[7], w8
+; CHECK-GI-NEXT:    mov v19.b[7], w12
+; CHECK-GI-NEXT:    ldr w8, [sp, #1040]
 ; CHECK-GI-NEXT:    mov v6.b[8], w10
-; CHECK-GI-NEXT:    ldr w10, [sp, #1040]
-; CHECK-GI-NEXT:    mov v18.b[7], w9
-; CHECK-GI-NEXT:    ldr w9, [sp, #1168]
-; CHECK-GI-NEXT:    mov v19.b[7], w8
-; CHECK-GI-NEXT:    ldr w8, [sp, #792]
+; CHECK-GI-NEXT:    ldr w10, [sp, #1168]
 ; CHECK-GI-NEXT:    mov v7.b[8], w11
-; CHECK-GI-NEXT:    mov v16.b[8], w10
-; CHECK-GI-NEXT:    ldr w10, [sp, #1296]
-; CHECK-GI-NEXT:    ldr w11, [sp, #920]
-; CHECK-GI-NEXT:    mov v17.b[8], w9
-; CHECK-GI-NEXT:    ldr w9, [sp, #1424]
-; CHECK-GI-NEXT:    mov v6.b[9], w8
-; CHECK-GI-NEXT:    ldr w8, [sp, #1048]
-; CHECK-GI-NEXT:    mov v18.b[8], w10
-; CHECK-GI-NEXT:    ldr w10, [sp, #1176]
-; CHECK-GI-NEXT:    mov v19.b[8], w9
-; CHECK-GI-NEXT:    ldr w9, [sp, #800]
-; CHECK-GI-NEXT:    mov v7.b[9], w11
-; CHECK-GI-NEXT:    mov v16.b[9], w8
-; CHECK-GI-NEXT:    ldr w8, [sp, #1304]
-; CHECK-GI-NEXT:    ldr w11, [sp, #928]
-; CHECK-GI-NEXT:    mov v17.b[9], w10
-; CHECK-GI-NEXT:    ldr w10, [sp, #1432]
-; CHECK-GI-NEXT:    mov v6.b[10], w9
-; CHECK-GI-NEXT:    ldr w9, [sp, #1056]
-; CHECK-GI-NEXT:    mov v18.b[9], w8
+; CHECK-GI-NEXT:    ldr w11, [sp, #1296]
+; CHECK-GI-NEXT:    ldr w12, [sp, #1424]
+; CHECK-GI-NEXT:    mov v16.b[8], w8
+; CHECK-GI-NEXT:    ldr w8, [sp, #800]
+; CHECK-GI-NEXT:    mov v17.b[8], w10
+; CHECK-GI-NEXT:    ldr w10, [sp, #920]
+; CHECK-GI-NEXT:    mov v18.b[8], w11
+; CHECK-GI-NEXT:    mov v19.b[8], w12
+; CHECK-GI-NEXT:    ldr w11, [sp, #1048]
+; CHECK-GI-NEXT:    mov v6.b[9], w9
+; CHECK-GI-NEXT:    ldr w9, [sp, #1176]
+; CHECK-GI-NEXT:    mov v7.b[9], w10
+; CHECK-GI-NEXT:    ldr w10, [sp, #1304]
+; CHECK-GI-NEXT:    ldr w12, [sp, #1432]
+; CHECK-GI-NEXT:    mov v16.b[9], w11
+; CHECK-GI-NEXT:    ldr w11, [sp, #808]
+; CHECK-GI-NEXT:    mov v17.b[9], w9
+; CHECK-GI-NEXT:    ldr w9, [sp, #928]
+; CHECK-GI-NEXT:    mov v18.b[9], w10
+; CHECK-GI-NEXT:    mov v19.b[9], w12
+; CHECK-GI-NEXT:    ldr w10, [sp, #1056]
+; CHECK-GI-NEXT:    mov v6.b[10], w8
 ; CHECK-GI-NEXT:    ldr w8, [sp, #1184]
-; CHECK-GI-NEXT:    mov v19.b[9], w10
-; CHECK-GI-NEXT:    ldr w10, [sp, #808]
-; CHECK-GI-NEXT:    mov v7.b[10], w11
-; CHECK-GI-NEXT:    mov v16.b[10], w9
+; CHECK-GI-NEXT:    mov v7.b[10], w9
 ; CHECK-GI-NEXT:    ldr w9, [sp, #1312]
-; CHECK-GI-NEXT:    ldr w11, [sp, #936]
+; CHECK-GI-NEXT:    ldr w12, [sp, #1440]
+; CHECK-GI-NEXT:    mov v16.b[10], w10
+; CHECK-GI-NEXT:    ldr w10, [sp, #816]
 ; CHECK-GI-NEXT:    mov v17.b[10], w8
-; CHECK-GI-NEXT:    ldr w8, [sp, #1440]
-; CHECK-GI-NEXT:    mov v6.b[11], w10
-; CHECK-GI-NEXT:    ldr w10, [sp, #1064]
+; CHECK-GI-NEXT:    ldr w8, [sp, #936]
 ; CHECK-GI-NEXT:    mov v18.b[10], w9
-; CHECK-GI-NEXT:    ldr w9, [sp, #1192]
-; CHECK-GI-NEXT:    mov v19.b[10], w8
-; CHECK-GI-NEXT:    ldr w8, [sp, #816]
-; CHECK-GI-NEXT:    mov v7.b[11], w11
-; CHECK-GI-NEXT:    mov v16.b[11], w10
-; CHECK-GI-NEXT:    ldr w10, [sp, #1320]
+; CHECK-GI-NEXT:    mov v19.b[10], w12
+; CHECK-GI-NEXT:    ldr w9, [sp, #1064]
+; CHECK-GI-NEXT:    mov v6.b[11], w11
+; CHECK-GI-NEXT:    ldr w11, [sp, #1192]
+; CHECK-GI-NEXT:    mov v7.b[11], w8
+; CHECK-GI-NEXT:    ldr w8, [sp, #1320]
+; CHECK-GI-NEXT:    ldr w12, [sp, #1448]
+; CHECK-GI-NEXT:    mov v16.b[11], w9
+; CHECK-GI-NEXT:    ldr w9, [sp, #824]
+; CHECK-GI-NEXT:    mov v17.b[11], w11
 ; CHECK-GI-NEXT:    ldr w11, [sp, #944]
-; CHECK-GI-NEXT:    mov v17.b[11], w9
-; CHECK-GI-NEXT:    ldr w9, [sp, #1448]
-; CHECK-GI-NEXT:    mov v6.b[12], w8
+; CHECK-GI-NEXT:    mov v18.b[11], w8
+; CHECK-GI-NEXT:    mov v19.b[11], w12
 ; CHECK-GI-NEXT:    ldr w8, [sp, #1072]
-; CHECK-GI-NEXT:    mov v18.b[11], w10
+; CHECK-GI-NEXT:    mov v6.b[12], w10
 ; CHECK-GI-NEXT:    ldr w10, [sp, #1200]
-; CHECK-GI-NEXT:    mov v19.b[11], w9
-; CHECK-GI-NEXT:    ldr w9, [sp, #824]
 ; CHECK-GI-NEXT:    mov v7.b[12], w11
+; CHECK-GI-NEXT:    ldr w11, [sp, #1328]
+; CHECK-GI-NEXT:    ldr w12, [sp, #1456]
 ; CHECK-GI-NEXT:    mov v16.b[12], w8
-; CHECK-GI-NEXT:    ldr w8, [sp, #1328]
-; CHECK-GI-NEXT:    ldr w11, [sp, #952]
+; CHECK-GI-NEXT:    ldr w8, [sp, #832]
 ; CHECK-GI-NEXT:    mov v17.b[12], w10
-; CHECK-GI-NEXT:    ldr w10, [sp, #1456]
+; CHECK-GI-NEXT:    ldr w10, [sp, #952]
+; CHECK-GI-NEXT:    mov v18.b[12], w11
+; CHECK-GI-NEXT:    mov v19.b[12], w12
+; CHECK-GI-NEXT:    ldr w11, [sp, #1080]
 ; CHECK-GI-NEXT:    mov v6.b[13], w9
-; CHECK-GI-NEXT:    ldr w9, [sp, #1080]
-; CHECK-GI-NEXT:    mov v18.b[12], w8
-; CHECK-GI-NEXT:    ldr w8, [sp, #1208]
-; CHECK-GI-NEXT:    mov v19.b[12], w10
-; CHECK-GI-NEXT:    ldr w10, [sp, #832]
-; CHECK-GI-NEXT:    mov v7.b[13], w11
-; CHECK-GI-NEXT:    mov v16.b[13], w9
-; CHECK-GI-NEXT:    ldr w9, [sp, #1336]
+; CHECK-GI-NEXT:    ldr w9, [sp, #1208]
+; CHECK-GI-NEXT:    mov v7.b[13], w10
+; CHECK-GI-NEXT:    ldr w10, [sp, #1336]
+; CHECK-GI-NEXT:    ldr w12, [sp, #1464]
+; CHECK-GI-NEXT:    mov v16.b[13], w11
 ; CHECK-GI-NEXT:    ldr w11, [sp, #960]
-; CHECK-GI-NEXT:    mov v17.b[13], w8
-; CHECK-GI-NEXT:    ldr w8, [sp, #1464]
-; CHECK-GI-NEXT:    mov v6.b[14], w10
-; CHECK-GI-NEXT:    ldr w10, [sp, #1088]
-; CHECK-GI-NEXT:    mov v18.b[13], w9
-; CHECK-GI-NEXT:    ldr w9, [sp, #1216]
-; CHECK-GI-NEXT:    mov v19.b[13], w8
-; CHECK-GI-NEXT:    ldr w8, [sp, #840]
+; CHECK-GI-NEXT:    mov v17.b[13], w9
+; CHECK-GI-NEXT:    mov v18.b[13], w10
+; CHECK-GI-NEXT:    ldr w9, [sp, #1088]
+; CHECK-GI-NEXT:    mov v19.b[13], w12
+; CHECK-GI-NEXT:    mov v6.b[14], w8
+; CHECK-GI-NEXT:    ldr w8, [sp, #1216]
 ; CHECK-GI-NEXT:    mov v7.b[14], w11
-; CHECK-GI-NEXT:    mov v16.b[14], w10
 ; CHECK-GI-NEXT:    ldr w10, [sp, #1344]
-; CHECK-GI-NEXT:    ldr w11, [sp, #968]
-; CHECK-GI-NEXT:    mov v17.b[14], w9
-; CHECK-GI-NEXT:    mov v6.b[15], w8
-; CHECK-GI-NEXT:    ldr w8, [sp, #1096]
+; CHECK-GI-NEXT:    ldr w11, [sp, #1472]
+; CHECK-GI-NEXT:    ldr w12, [sp, #840]
+; CHECK-GI-NEXT:    mov v16.b[14], w9
+; CHECK-GI-NEXT:    ldr w9, [sp, #1096]
+; CHECK-GI-NEXT:    mov v17.b[14], w8
 ; CHECK-GI-NEXT:    mov v18.b[14], w10
-; CHECK-GI-NEXT:    ldr w9, [sp, #1472]
+; CHECK-GI-NEXT:    ldr w8, [sp, #968]
+; CHECK-GI-NEXT:    mov v19.b[14], w11
 ; CHECK-GI-NEXT:    ldr w10, [sp, #1224]
-; CHECK-GI-NEXT:    mov v7.b[15], w11
-; CHECK-GI-NEXT:    addv s4, v25.4s
-; CHECK-GI-NEXT:    mov v16.b[15], w8
-; CHECK-GI-NEXT:    ldr w8, [sp, #1352]
-; CHECK-GI-NEXT:    mov v19.b[14], w9
+; CHECK-GI-NEXT:    mov v6.b[15], w12
+; CHECK-GI-NEXT:    ldr w11, [sp, #1352]
+; CHECK-GI-NEXT:    ldr w12, [sp, #1480]
+; CHECK-GI-NEXT:    mov v7.b[15], w8
+; CHECK-GI-NEXT:    mov v16.b[15], w9
+; CHECK-GI-NEXT:    movi v24.2d, #0000000000000000
+; CHECK-GI-NEXT:    movi v25.2d, #0000000000000000
 ; CHECK-GI-NEXT:    mov v17.b[15], w10
-; CHECK-GI-NEXT:    ldr w9, [sp, #1480]
-; CHECK-GI-NEXT:    mov v18.b[15], w8
+; CHECK-GI-NEXT:    mov v18.b[15], w11
+; CHECK-GI-NEXT:    movi v26.2d, #0000000000000000
+; CHECK-GI-NEXT:    mov v19.b[15], w12
+; CHECK-GI-NEXT:    sdot v21.4s, v0.16b, v3.16b
+; CHECK-GI-NEXT:    sdot v22.4s, v1.16b, v4.16b
+; CHECK-GI-NEXT:    sdot v23.4s, v2.16b, v5.16b
+; CHECK-GI-NEXT:    mov v20.s[3], wzr
+; CHECK-GI-NEXT:    sdot v25.4s, v6.16b, v17.16b
+; CHECK-GI-NEXT:    sdot v26.4s, v7.16b, v18.16b
+; CHECK-GI-NEXT:    sdot v24.4s, v16.16b, v19.16b
+; CHECK-GI-NEXT:    add v0.4s, v21.4s, v22.4s
+; CHECK-GI-NEXT:    add v1.4s, v23.4s, v20.4s
+; CHECK-GI-NEXT:    add v2.4s, v25.4s, v26.4s
+; CHECK-GI-NEXT:    add v3.4s, v24.4s, v20.4s
+; CHECK-GI-NEXT:    add v0.4s, v0.4s, v1.4s
+; CHECK-GI-NEXT:    add v1.4s, v2.4s, v3.4s
+; CHECK-GI-NEXT:    addv s0, v0.4s
+; CHECK-GI-NEXT:    addv s1, v1.4s
 ; CHECK-GI-NEXT:    fmov w8, s0
-; CHECK-GI-NEXT:    fmov w11, s4
-; CHECK-GI-NEXT:    mov v19.b[15], w9
 ; CHECK-GI-NEXT:    fmov w9, s1
-; CHECK-GI-NEXT:    sdot v22.4s, v6.16b, v17.16b
-; CHECK-GI-NEXT:    sdot v23.4s, v7.16b, v18.16b
-; CHECK-GI-NEXT:    add w8, w8, w9
-; CHECK-GI-NEXT:    sdot v24.4s, v16.16b, v19.16b
-; CHECK-GI-NEXT:    add w8, w8, w11
-; CHECK-GI-NEXT:    addv s2, v22.4s
-; CHECK-GI-NEXT:    addv s3, v23.4s
-; CHECK-GI-NEXT:    addv s5, v24.4s
-; CHECK-GI-NEXT:    fmov w9, s2
-; CHECK-GI-NEXT:    fmov w10, s3
-; CHECK-GI-NEXT:    add w9, w9, w10
-; CHECK-GI-NEXT:    fmov w10, s5
-; CHECK-GI-NEXT:    add w9, w9, w10
 ; CHECK-GI-NEXT:    add w0, w8, w9
 ; CHECK-GI-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
 ; CHECK-GI-NEXT:    ret
@@ -4738,218 +6885,216 @@ define i32 @test_sdot_v48i8_double_nomla(<48 x i8> %a, <48 x i8> %b, <48 x i8> %
 ; CHECK-GI-NEXT:    .cfi_def_cfa_offset 16
 ; CHECK-GI-NEXT:    .cfi_offset w29, -16
 ; CHECK-GI-NEXT:    ldr w10, [sp, #80]
-; CHECK-GI-NEXT:    ldr w11, [sp, #208]
-; CHECK-GI-NEXT:    fmov s0, w0
 ; CHECK-GI-NEXT:    ldr w9, [sp, #88]
-; CHECK-GI-NEXT:    ldr w12, [sp, #728]
-; CHECK-GI-NEXT:    movi v6.16b, #1
+; CHECK-GI-NEXT:    fmov s0, w0
+; CHECK-GI-NEXT:    ldr w11, [sp, #208]
+; CHECK-GI-NEXT:    ldr w8, [sp, #216]
+; CHECK-GI-NEXT:    mov v6.s[0], wzr
 ; CHECK-GI-NEXT:    fmov s1, w10
+; CHECK-GI-NEXT:    ldr w10, [sp, #720]
+; CHECK-GI-NEXT:    movi v7.16b, #1
 ; CHECK-GI-NEXT:    fmov s2, w11
-; CHECK-GI-NEXT:    ldr w11, [sp, #720]
-; CHECK-GI-NEXT:    ldr w10, [sp, #216]
 ; CHECK-GI-NEXT:    mov v0.b[1], w1
-; CHECK-GI-NEXT:    ldr w13, [sp, #856]
-; CHECK-GI-NEXT:    fmov s3, w11
-; CHECK-GI-NEXT:    ldr w8, [sp, #96]
-; CHECK-GI-NEXT:    ldr w11, [sp, #224]
-; CHECK-GI-NEXT:    mov v1.b[1], w9
-; CHECK-GI-NEXT:    ldr w9, [sp, #848]
-; CHECK-GI-NEXT:    mov v2.b[1], w10
+; CHECK-GI-NEXT:    ldr w11, [sp, #728]
+; CHECK-GI-NEXT:    fmov s3, w10
 ; CHECK-GI-NEXT:    ldr w10, [sp, #976]
-; CHECK-GI-NEXT:    movi v7.2d, #0000000000000000
 ; CHECK-GI-NEXT:    movi v16.2d, #0000000000000000
+; CHECK-GI-NEXT:    mov v1.b[1], w9
+; CHECK-GI-NEXT:    ldr w9, [sp, #848]
+; CHECK-GI-NEXT:    mov v6.s[1], wzr
+; CHECK-GI-NEXT:    fmov s5, w10
+; CHECK-GI-NEXT:    mov v2.b[1], w8
+; CHECK-GI-NEXT:    ldr w8, [sp, #856]
 ; CHECK-GI-NEXT:    fmov s4, w9
-; CHECK-GI-NEXT:    mov v3.b[1], w12
 ; CHECK-GI-NEXT:    ldr w9, [sp, #984]
-; CHECK-GI-NEXT:    fmov s5, w10
+; CHECK-GI-NEXT:    mov v3.b[1], w11
+; CHECK-GI-NEXT:    ldr w10, [sp, #224]
 ; CHECK-GI-NEXT:    mov v0.b[2], w2
-; CHECK-GI-NEXT:    ldr w10, [sp, #736]
-; CHECK-GI-NEXT:    mov v1.b[2], w8
-; CHECK-GI-NEXT:    ldr w8, [sp, #864]
-; CHECK-GI-NEXT:    mov v2.b[2], w11
-; CHECK-GI-NEXT:    mov v4.b[1], w13
-; CHECK-GI-NEXT:    ldr w11, [sp, #992]
-; CHECK-GI-NEXT:    ldr w12, [sp, #776]
+; CHECK-GI-NEXT:    ldr w11, [sp, #16]
 ; CHECK-GI-NEXT:    mov v5.b[1], w9
-; CHECK-GI-NEXT:    mov v3.b[2], w10
-; CHECK-GI-NEXT:    ldr w9, [sp, #104]
-; CHECK-GI-NEXT:    ldr w10, [sp, #232]
-; CHECK-GI-NEXT:    mov v0.b[3], w3
+; CHECK-GI-NEXT:    ldr w9, [sp, #864]
 ; CHECK-GI-NEXT:    movi v17.2d, #0000000000000000
-; CHECK-GI-NEXT:    mov v1.b[3], w9
-; CHECK-GI-NEXT:    ldr w9, [sp, #872]
+; CHECK-GI-NEXT:    mov v4.b[1], w8
+; CHECK-GI-NEXT:    ldr w8, [sp, #96]
+; CHECK-GI-NEXT:    mov v2.b[2], w10
+; CHECK-GI-NEXT:    ldr w10, [sp, #992]
 ; CHECK-GI-NEXT:    movi v18.2d, #0000000000000000
-; CHECK-GI-NEXT:    mov v4.b[2], w8
+; CHECK-GI-NEXT:    movi v19.2d, #0000000000000000
+; CHECK-GI-NEXT:    mov v1.b[2], w8
+; CHECK-GI-NEXT:    ldr w8, [sp, #736]
+; CHECK-GI-NEXT:    mov v0.b[3], w3
+; CHECK-GI-NEXT:    mov v5.b[2], w10
+; CHECK-GI-NEXT:    ldr w10, [sp, #872]
+; CHECK-GI-NEXT:    movi v20.2d, #0000000000000000
+; CHECK-GI-NEXT:    mov v3.b[2], w8
+; CHECK-GI-NEXT:    mov v4.b[2], w9
+; CHECK-GI-NEXT:    ldr w8, [sp, #104]
+; CHECK-GI-NEXT:    ldr w9, [sp, #232]
+; CHECK-GI-NEXT:    mov v6.s[2], wzr
+; CHECK-GI-NEXT:    movi v21.2d, #0000000000000000
+; CHECK-GI-NEXT:    mov v1.b[3], w8
 ; CHECK-GI-NEXT:    ldr w8, [sp, #744]
-; CHECK-GI-NEXT:    mov v2.b[3], w10
-; CHECK-GI-NEXT:    mov v5.b[2], w11
-; CHECK-GI-NEXT:    ldr w11, [sp, #1000]
-; CHECK-GI-NEXT:    ldr w10, [sp, #240]
+; CHECK-GI-NEXT:    mov v0.b[4], w4
+; CHECK-GI-NEXT:    mov v2.b[3], w9
+; CHECK-GI-NEXT:    ldr w9, [sp, #1000]
 ; CHECK-GI-NEXT:    mov v3.b[3], w8
+; CHECK-GI-NEXT:    mov v4.b[3], w10
 ; CHECK-GI-NEXT:    ldr w8, [sp, #112]
-; CHECK-GI-NEXT:    mov v0.b[4], w4
-; CHECK-GI-NEXT:    movi v19.2d, #0000000000000000
-; CHECK-GI-NEXT:    movi v20.2d, #0000000000000000
-; CHECK-GI-NEXT:    mov v4.b[3], w9
-; CHECK-GI-NEXT:    ldr w9, [sp, #752]
+; CHECK-GI-NEXT:    ldr w10, [sp, #240]
+; CHECK-GI-NEXT:    mov v5.b[3], w9
+; CHECK-GI-NEXT:    ldr w9, [sp, #880]
 ; CHECK-GI-NEXT:    mov v1.b[4], w8
-; CHECK-GI-NEXT:    ldr w8, [sp, #880]
-; CHECK-GI-NEXT:    mov v5.b[3], w11
-; CHECK-GI-NEXT:    mov v2.b[4], w10
-; CHECK-GI-NEXT:    mov v3.b[4], w9
-; CHECK-GI-NEXT:    ldr w9, [sp, #120]
-; CHECK-GI-NEXT:    ldr w11, [sp, #1008]
-; CHECK-GI-NEXT:    ldr w10, [sp, #248]
+; CHECK-GI-NEXT:    ldr w8, [sp, #752]
 ; CHECK-GI-NEXT:    mov v0.b[5], w5
-; CHECK-GI-NEXT:    mov v4.b[4], w8
+; CHECK-GI-NEXT:    mov v2.b[4], w10
+; CHECK-GI-NEXT:    ldr w10, [sp, #1008]
+; CHECK-GI-NEXT:    mov v6.s[3], wzr
+; CHECK-GI-NEXT:    mov v3.b[4], w8
+; CHECK-GI-NEXT:    mov v4.b[4], w9
+; CHECK-GI-NEXT:    ldr w8, [sp, #120]
+; CHECK-GI-NEXT:    ldr w9, [sp, #248]
+; CHECK-GI-NEXT:    mov v5.b[4], w10
+; CHECK-GI-NEXT:    ldr w10, [sp, #888]
+; CHECK-GI-NEXT:    mov v1.b[5], w8
 ; CHECK-GI-NEXT:    ldr w8, [sp, #760]
-; CHECK-GI-NEXT:    mov v1.b[5], w9
-; CHECK-GI-NEXT:    ldr w9, [sp, #888]
-; CHECK-GI-NEXT:    mov v5.b[4], w11
-; CHECK-GI-NEXT:    mov v2.b[5], w10
+; CHECK-GI-NEXT:    mov v0.b[6], w6
+; CHECK-GI-NEXT:    mov v2.b[5], w9
+; CHECK-GI-NEXT:    ldr w9, [sp, #1016]
 ; CHECK-GI-NEXT:    mov v3.b[5], w8
+; CHECK-GI-NEXT:    mov v4.b[5], w10
 ; CHECK-GI-NEXT:    ldr w8, [sp, #128]
-; CHECK-GI-NEXT:    ldr w11, [sp, #1016]
 ; CHECK-GI-NEXT:    ldr w10, [sp, #256]
-; CHECK-GI-NEXT:    mov v0.b[6], w6
-; CHECK-GI-NEXT:    mov v4.b[5], w9
-; CHECK-GI-NEXT:    ldr w9, [sp, #768]
+; CHECK-GI-NEXT:    mov v5.b[5], w9
+; CHECK-GI-NEXT:    ldr w9, [sp, #896]
 ; CHECK-GI-NEXT:    mov v1.b[6], w8
-; CHECK-GI-NEXT:    ldr w8, [sp, #896]
-; CHECK-GI-NEXT:    mov v5.b[5], w11
-; CHECK-GI-NEXT:    mov v2.b[6], w10
-; CHECK-GI-NEXT:    mov v3.b[6], w9
-; CHECK-GI-NEXT:    ldr w9, [sp, #136]
-; CHECK-GI-NEXT:    ldr w11, [sp, #1024]
-; CHECK-GI-NEXT:    ldr w10, [sp, #264]
+; CHECK-GI-NEXT:    ldr w8, [sp, #768]
 ; CHECK-GI-NEXT:    mov v0.b[7], w7
-; CHECK-GI-NEXT:    mov v4.b[6], w8
-; CHECK-GI-NEXT:    mov v1.b[7], w9
-; CHECK-GI-NEXT:    ldr w9, [sp, #904]
-; CHECK-GI-NEXT:    mov v5.b[6], w11
-; CHECK-GI-NEXT:    mov v2.b[7], w10
-; CHECK-GI-NEXT:    ldr w8, [sp, #16]
-; CHECK-GI-NEXT:    mov v3.b[7], w12
-; CHECK-GI-NEXT:    ldr w10, [sp, #144]
-; CHECK-GI-NEXT:    ldr w12, [sp, #1032]
-; CHECK-GI-NEXT:    mov v0.b[8], w8
+; CHECK-GI-NEXT:    mov v2.b[6], w10
+; CHECK-GI-NEXT:    ldr w10, [sp, #1024]
+; CHECK-GI-NEXT:    mov v3.b[6], w8
+; CHECK-GI-NEXT:    ldr w8, [sp, #136]
+; CHECK-GI-NEXT:    mov v4.b[6], w9
+; CHECK-GI-NEXT:    ldr w9, [sp, #264]
+; CHECK-GI-NEXT:    mov v5.b[6], w10
+; CHECK-GI-NEXT:    ldr w10, [sp, #904]
+; CHECK-GI-NEXT:    mov v1.b[7], w8
+; CHECK-GI-NEXT:    ldr w8, [sp, #776]
+; CHECK-GI-NEXT:    mov v0.b[8], w11
+; CHECK-GI-NEXT:    mov v2.b[7], w9
+; CHECK-GI-NEXT:    ldr w9, [sp, #1032]
+; CHECK-GI-NEXT:    ldr w11, [sp, #24]
+; CHECK-GI-NEXT:    mov v3.b[7], w8
+; CHECK-GI-NEXT:    ldr w8, [sp, #144]
+; CHECK-GI-NEXT:    mov v4.b[7], w10
+; CHECK-GI-NEXT:    ldr w10, [sp, #272]
+; CHECK-GI-NEXT:    mov v5.b[7], w9
+; CHECK-GI-NEXT:    ldr w9, [sp, #912]
+; CHECK-GI-NEXT:    mov v1.b[8], w8
 ; CHECK-GI-NEXT:    ldr w8, [sp, #784]
-; CHECK-GI-NEXT:    ldr w11, [sp, #272]
-; CHECK-GI-NEXT:    mov v4.b[7], w9
-; CHECK-GI-NEXT:    mov v1.b[8], w10
-; CHECK-GI-NEXT:    ldr w10, [sp, #912]
-; CHECK-GI-NEXT:    mov v5.b[7], w12
-; CHECK-GI-NEXT:    ldr w9, [sp, #24]
-; CHECK-GI-NEXT:    ldr w12, [sp, #1040]
+; CHECK-GI-NEXT:    mov v0.b[9], w11
+; CHECK-GI-NEXT:    mov v2.b[8], w10
+; CHECK-GI-NEXT:    ldr w10, [sp, #1040]
+; CHECK-GI-NEXT:    ldr w11, [sp, #32]
 ; CHECK-GI-NEXT:    mov v3.b[8], w8
 ; CHECK-GI-NEXT:    ldr w8, [sp, #152]
-; CHECK-GI-NEXT:    mov v2.b[8], w11
-; CHECK-GI-NEXT:    mov v0.b[9], w9
-; CHECK-GI-NEXT:    ldr w9, [sp, #792]
-; CHECK-GI-NEXT:    ldr w11, [sp, #280]
-; CHECK-GI-NEXT:    mov v4.b[8], w10
-; CHECK-GI-NEXT:    mov v1.b[9], w8
+; CHECK-GI-NEXT:    mov v4.b[8], w9
+; CHECK-GI-NEXT:    ldr w9, [sp, #280]
+; CHECK-GI-NEXT:    mov v5.b[8], w10
 ; CHECK-GI-NEXT:    ldr w10, [sp, #920]
-; CHECK-GI-NEXT:    mov v5.b[8], w12
-; CHECK-GI-NEXT:    ldr w8, [sp, #32]
-; CHECK-GI-NEXT:    ldr w12, [sp, #1048]
-; CHECK-GI-NEXT:    mov v3.b[9], w9
-; CHECK-GI-NEXT:    ldr w9, [sp, #160]
-; CHECK-GI-NEXT:    mov v2.b[9], w11
-; CHECK-GI-NEXT:    mov v0.b[10], w8
-; CHECK-GI-NEXT:    ldr w8, [sp, #800]
-; CHECK-GI-NEXT:    ldr w11, [sp, #288]
+; CHECK-GI-NEXT:    mov v1.b[9], w8
+; CHECK-GI-NEXT:    ldr w8, [sp, #792]
+; CHECK-GI-NEXT:    mov v0.b[10], w11
+; CHECK-GI-NEXT:    mov v2.b[9], w9
+; CHECK-GI-NEXT:    ldr w9, [sp, #1048]
+; CHECK-GI-NEXT:    ldr w11, [sp, #40]
+; CHECK-GI-NEXT:    mov v3.b[9], w8
+; CHECK-GI-NEXT:    ldr w8, [sp, #160]
 ; CHECK-GI-NEXT:    mov v4.b[9], w10
-; CHECK-GI-NEXT:    mov v1.b[10], w9
-; CHECK-GI-NEXT:    ldr w10, [sp, #928]
-; CHECK-GI-NEXT:    mov v5.b[9], w12
-; CHECK-GI-NEXT:    ldr w9, [sp, #40]
-; CHECK-GI-NEXT:    ldr w12, [sp, #1056]
+; CHECK-GI-NEXT:    ldr w10, [sp, #288]
+; CHECK-GI-NEXT:    mov v5.b[9], w9
+; CHECK-GI-NEXT:    ldr w9, [sp, #928]
+; CHECK-GI-NEXT:    mov v1.b[10], w8
+; CHECK-GI-NEXT:    ldr w8, [sp, #800]
+; CHECK-GI-NEXT:    mov v0.b[11], w11
+; CHECK-GI-NEXT:    mov v2.b[10], w10
+; CHECK-GI-NEXT:    ldr w10, [sp, #1056]
+; CHECK-GI-NEXT:    ldr w11, [sp, #48]
 ; CHECK-GI-NEXT:    mov v3.b[10], w8
 ; CHECK-GI-NEXT:    ldr w8, [sp, #168]
-; CHECK-GI-NEXT:    mov v2.b[10], w11
-; CHECK-GI-NEXT:    mov v0.b[11], w9
-; CHECK-GI-NEXT:    ldr w9, [sp, #808]
-; CHECK-GI-NEXT:    ldr w11, [sp, #296]
-; CHECK-GI-NEXT:    mov v4.b[10], w10
-; CHECK-GI-NEXT:    mov v1.b[11], w8
+; CHECK-GI-NEXT:    mov v4.b[10], w9
+; CHECK-GI-NEXT:    ldr w9, [sp, #296]
+; CHECK-GI-NEXT:    mov v5.b[10], w10
 ; CHECK-GI-NEXT:    ldr w10, [sp, #936]
-; CHECK-GI-NEXT:    mov v5.b[10], w12
-; CHECK-GI-NEXT:    ldr w8, [sp, #48]
-; CHECK-GI-NEXT:    ldr w12, [sp, #1064]
-; CHECK-GI-NEXT:    mov v3.b[11], w9
-; CHECK-GI-NEXT:    ldr w9, [sp, #176]
-; CHECK-GI-NEXT:    mov v2.b[11], w11
-; CHECK-GI-NEXT:    mov v0.b[12], w8
-; CHECK-GI-NEXT:    ldr w8, [sp, #816]
-; CHECK-GI-NEXT:    ldr w11, [sp, #304]
+; CHECK-GI-NEXT:    mov v1.b[11], w8
+; CHECK-GI-NEXT:    ldr w8, [sp, #808]
+; CHECK-GI-NEXT:    mov v0.b[12], w11
+; CHECK-GI-NEXT:    mov v2.b[11], w9
+; CHECK-GI-NEXT:    ldr w9, [sp, #1064]
+; CHECK-GI-NEXT:    ldr w11, [sp, #56]
+; CHECK-GI-NEXT:    mov v3.b[11], w8
+; CHECK-GI-NEXT:    ldr w8, [sp, #176]
 ; CHECK-GI-NEXT:    mov v4.b[11], w10
-; CHECK-GI-NEXT:    mov v1.b[12], w9
-; CHECK-GI-NEXT:    ldr w10, [sp, #944]
-; CHECK-GI-NEXT:    mov v5.b[11], w12
-; CHECK-GI-NEXT:    ldr w9, [sp, #56]
-; CHECK-GI-NEXT:    ldr w12, [sp, #1072]
+; CHECK-GI-NEXT:    ldr w10, [sp, #304]
+; CHECK-GI-NEXT:    mov v5.b[11], w9
+; CHECK-GI-NEXT:    ldr w9, [sp, #944]
+; CHECK-GI-NEXT:    mov v1.b[12], w8
+; CHECK-GI-NEXT:    ldr w8, [sp, #816]
+; CHECK-GI-NEXT:    mov v0.b[13], w11
+; CHECK-GI-NEXT:    mov v2.b[12], w10
+; CHECK-GI-NEXT:    ldr w10, [sp, #1072]
+; CHECK-GI-NEXT:    ldr w11, [sp, #64]
 ; CHECK-GI-NEXT:    mov v3.b[12], w8
 ; CHECK-GI-NEXT:    ldr w8, [sp, #184]
-; CHECK-GI-NEXT:    mov v2.b[12], w11
-; CHECK-GI-NEXT:    mov v0.b[13], w9
-; CHECK-GI-NEXT:    ldr w9, [sp, #824]
-; CHECK-GI-NEXT:    ldr w11, [sp, #312]
-; CHECK-GI-NEXT:    mov v4.b[12], w10
-; CHECK-GI-NEXT:    mov v1.b[13], w8
+; CHECK-GI-NEXT:    mov v4.b[12], w9
+; CHECK-GI-NEXT:    ldr w9, [sp, #312]
+; CHECK-GI-NEXT:    mov v5.b[12], w10
 ; CHECK-GI-NEXT:    ldr w10, [sp, #952]
-; CHECK-GI-NEXT:    mov v5.b[12], w12
-; CHECK-GI-NEXT:    ldr w8, [sp, #64]
-; CHECK-GI-NEXT:    ldr w12, [sp, #1080]
-; CHECK-GI-NEXT:    mov v3.b[13], w9
-; CHECK-GI-NEXT:    ldr w9, [sp, #192]
-; CHECK-GI-NEXT:    mov v2.b[13], w11
-; CHECK-GI-NEXT:    mov v0.b[14], w8
-; CHECK-GI-NEXT:    ldr w8, [sp, #832]
-; CHECK-GI-NEXT:    ldr w11, [sp, #320]
+; CHECK-GI-NEXT:    mov v1.b[13], w8
+; CHECK-GI-NEXT:    ldr w8, [sp, #824]
+; CHECK-GI-NEXT:    mov v0.b[14], w11
+; CHECK-GI-NEXT:    mov v2.b[13], w9
+; CHECK-GI-NEXT:    ldr w9, [sp, #1080]
+; CHECK-GI-NEXT:    ldr w11, [sp, #72]
+; CHECK-GI-NEXT:    mov v3.b[13], w8
+; CHECK-GI-NEXT:    ldr w8, [sp, #192]
 ; CHECK-GI-NEXT:    mov v4.b[13], w10
-; CHECK-GI-NEXT:    mov v1.b[14], w9
-; CHECK-GI-NEXT:    ldr w10, [sp, #960]
-; CHECK-GI-NEXT:    mov v5.b[13], w12
-; CHECK-GI-NEXT:    ldr w9, [sp, #72]
-; CHECK-GI-NEXT:    ldr w12, [sp, #1088]
+; CHECK-GI-NEXT:    ldr w10, [sp, #320]
+; CHECK-GI-NEXT:    mov v5.b[13], w9
+; CHECK-GI-NEXT:    ldr w9, [sp, #960]
+; CHECK-GI-NEXT:    mov v1.b[14], w8
+; CHECK-GI-NEXT:    ldr w8, [sp, #832]
+; CHECK-GI-NEXT:    mov v0.b[15], w11
+; CHECK-GI-NEXT:    mov v2.b[14], w10
+; CHECK-GI-NEXT:    ldr w10, [sp, #1088]
+; CHECK-GI-NEXT:    ldr w11, [sp, #968]
 ; CHECK-GI-NEXT:    mov v3.b[14], w8
+; CHECK-GI-NEXT:    mov v4.b[14], w9
 ; CHECK-GI-NEXT:    ldr w8, [sp, #200]
-; CHECK-GI-NEXT:    mov v2.b[14], w11
-; CHECK-GI-NEXT:    mov v0.b[15], w9
-; CHECK-GI-NEXT:    ldr w9, [sp, #840]
-; CHECK-GI-NEXT:    ldr w11, [sp, #328]
-; CHECK-GI-NEXT:    mov v4.b[14], w10
+; CHECK-GI-NEXT:    mov v5.b[14], w10
+; CHECK-GI-NEXT:    ldr w9, [sp, #328]
+; CHECK-GI-NEXT:    ldr w10, [sp, #840]
 ; CHECK-GI-NEXT:    mov v1.b[15], w8
-; CHECK-GI-NEXT:    ldr w8, [sp, #968]
-; CHECK-GI-NEXT:    mov v5.b[14], w12
-; CHECK-GI-NEXT:    ldr w10, [sp, #1096]
-; CHECK-GI-NEXT:    mov v3.b[15], w9
-; CHECK-GI-NEXT:    mov v2.b[15], w11
-; CHECK-GI-NEXT:    sdot v7.4s, v0.16b, v6.16b
-; CHECK-GI-NEXT:    mov v4.b[15], w8
-; CHECK-GI-NEXT:    sdot v16.4s, v1.16b, v6.16b
-; CHECK-GI-NEXT:    mov v5.b[15], w10
-; CHECK-GI-NEXT:    sdot v17.4s, v3.16b, v6.16b
-; CHECK-GI-NEXT:    sdot v20.4s, v2.16b, v6.16b
-; CHECK-GI-NEXT:    addv s0, v7.4s
-; CHECK-GI-NEXT:    sdot v18.4s, v4.16b, v6.16b
-; CHECK-GI-NEXT:    addv s1, v16.4s
-; CHECK-GI-NEXT:    sdot v19.4s, v5.16b, v6.16b
-; CHECK-GI-NEXT:    addv s2, v17.4s
-; CHECK-GI-NEXT:    addv s4, v20.4s
+; CHECK-GI-NEXT:    ldr w8, [sp, #1096]
+; CHECK-GI-NEXT:    sdot v16.4s, v0.16b, v7.16b
+; CHECK-GI-NEXT:    mov v2.b[15], w9
+; CHECK-GI-NEXT:    mov v3.b[15], w10
+; CHECK-GI-NEXT:    mov v4.b[15], w11
+; CHECK-GI-NEXT:    mov v5.b[15], w8
+; CHECK-GI-NEXT:    sdot v17.4s, v1.16b, v7.16b
+; CHECK-GI-NEXT:    sdot v18.4s, v2.16b, v7.16b
+; CHECK-GI-NEXT:    sdot v19.4s, v3.16b, v7.16b
+; CHECK-GI-NEXT:    sdot v21.4s, v4.16b, v7.16b
+; CHECK-GI-NEXT:    sdot v20.4s, v5.16b, v7.16b
+; CHECK-GI-NEXT:    add v0.4s, v16.4s, v17.4s
+; CHECK-GI-NEXT:    add v1.4s, v18.4s, v6.4s
+; CHECK-GI-NEXT:    add v2.4s, v19.4s, v21.4s
+; CHECK-GI-NEXT:    add v3.4s, v20.4s, v6.4s
+; CHECK-GI-NEXT:    add v0.4s, v0.4s, v1.4s
+; CHECK-GI-NEXT:    add v1.4s, v2.4s, v3.4s
+; CHECK-GI-NEXT:    addv s0, v0.4s
+; CHECK-GI-NEXT:    addv s1, v1.4s
 ; CHECK-GI-NEXT:    fmov w8, s0
 ; CHECK-GI-NEXT:    fmov w9, s1
-; CHECK-GI-NEXT:    addv s3, v18.4s
-; CHECK-GI-NEXT:    addv s5, v19.4s
-; CHECK-GI-NEXT:    fmov w10, s2
-; CHECK-GI-NEXT:    add w8, w8, w9
-; CHECK-GI-NEXT:    fmov w9, s4
-; CHECK-GI-NEXT:    fmov w11, s3
-; CHECK-GI-NEXT:    add w8, w8, w9
-; CHECK-GI-NEXT:    add w10, w10, w11
-; CHECK-GI-NEXT:    fmov w11, s5
-; CHECK-GI-NEXT:    add w9, w10, w11
 ; CHECK-GI-NEXT:    add w0, w8, w9
 ; CHECK-GI-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
 ; CHECK-GI-NEXT:    ret
diff --git a/llvm/test/CodeGen/AArch64/neon-fp8-cvt.ll b/llvm/test/CodeGen/AArch64/neon-fp8-cvt.ll
new file mode 100644
index 0000000000000..6070380d24234
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/neon-fp8-cvt.ll
@@ -0,0 +1,112 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 4
+; RUN: llc -mtriple=aarch64-linux -mattr=+neon,+fp8 < %s | FileCheck %s
+
+define <8 x bfloat> @test_vbfcvtl1_low(<8 x i8> %vn) {
+; CHECK-LABEL: test_vbfcvtl1_low:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    bf1cvtl v0.8h, v0.8b
+; CHECK-NEXT:    ret
+  %res = call <8 x bfloat> @llvm.aarch64.neon.fp8.cvtl1.v8bf16.v8i8(<8 x i8> %vn)
+  ret <8 x bfloat> %res
+}
+
+define <8 x bfloat> @test_vbfcvtl1_high(<16 x i8> %vn) {
+; CHECK-LABEL: test_vbfcvtl1_high:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    bf1cvtl2 v0.8h, v0.16b
+; CHECK-NEXT:    ret
+  %res = call <8 x bfloat> @llvm.aarch64.neon.fp8.cvtl1.v8bf16.v16i8(<16 x i8> %vn)
+  ret <8 x bfloat> %res
+}
+
+define <8 x bfloat> @test_vbfcvtl2_low(<8 x i8> %vn) {
+; CHECK-LABEL: test_vbfcvtl2_low:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    bf2cvtl v0.8h, v0.8b
+; CHECK-NEXT:    ret
+  %res = call <8 x bfloat> @llvm.aarch64.neon.fp8.cvtl2.v8bf16.v8i8(<8 x i8> %vn)
+  ret <8 x bfloat> %res
+}
+
+define <8 x bfloat> @test_vbfcvtl2_high(<16 x i8> %vn) {
+; CHECK-LABEL: test_vbfcvtl2_high:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    bf2cvtl2 v0.8h, v0.16b
+; CHECK-NEXT:    ret
+  %res = call <8 x bfloat> @llvm.aarch64.neon.fp8.cvtl2.v8bf16.v16i8(<16 x i8> %vn)
+  ret <8 x bfloat> %res
+}
+
+
+define <8 x half> @test_vfcvtl1_low(<8 x i8> %vn) {
+; CHECK-LABEL: test_vfcvtl1_low:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    f1cvtl v0.8h, v0.8b
+; CHECK-NEXT:    ret
+   %res = call <8 x half> @llvm.aarch64.neon.fp8.cvtl1.v8f16.v8i8(<8 x i8> %vn)
+  ret <8 x half> %res
+}
+
+define <8 x half> @test_vfcvtl1_high(<16 x i8> %vn) {
+; CHECK-LABEL: test_vfcvtl1_high:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    f1cvtl2 v0.8h, v0.16b
+; CHECK-NEXT:    ret
+  %res = call <8 x half> @llvm.aarch64.neon.fp8.cvtl1.v8f16.v16i8(<16 x i8> %vn)
+  ret <8 x half> %res
+}
+
+define <8 x half> @test_vfcvtl2_low(<8 x i8> %vn) {
+; CHECK-LABEL: test_vfcvtl2_low:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    f2cvtl v0.8h, v0.8b
+; CHECK-NEXT:    ret
+  %res = call <8 x half> @llvm.aarch64.neon.fp8.cvtl2.v8f16.v8i8(<8 x i8> %vn)
+  ret <8 x half> %res
+}
+
+define <8 x half> @test_vfcvtl2_high(<16 x i8> %vn) {
+; CHECK-LABEL: test_vfcvtl2_high:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    f2cvtl2 v0.8h, v0.16b
+; CHECK-NEXT:    ret
+  %res = call <8 x half> @llvm.aarch64.neon.fp8.cvtl2.v8f16.v16i8(<16 x i8> %vn)
+  ret <8 x half> %res
+}
+
+define <8 x i8> @test_vcvtn_low_f8_f32(<4 x float> %vn, <4 x float> %vm) {
+; CHECK-LABEL: test_vcvtn_low_f8_f32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    fcvtn v0.8b, v0.4s, v1.4s
+; CHECK-NEXT:    ret
+  %res = call <8 x i8> @llvm.aarch64.neon.fp8.fcvtn.v8i8.v4f32(<4 x float> %vn, <4 x float> %vm)
+  ret <8 x i8> %res
+}
+
+define <16 x i8> @test_vcvtn_high_f8_f32(<16 x i8> %vd, <4 x float> %vn, <4 x float> %vm) {
+; CHECK-LABEL: test_vcvtn_high_f8_f32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    fcvtn2 v0.16b, v1.4s, v2.4s
+; CHECK-NEXT:    ret
+  %res = call <16 x i8> @llvm.aarch64.neon.fp8.fcvtn2.v16i8.v4f32(<16 x i8> %vd, <4 x float> %vn, <4 x float> %vm)
+  ret <16 x i8> %res
+}
+
+
+define <8 x i8> @test_vcvtn_f8_f16(<4 x half> %vn, <4 x half> %vm) {
+; CHECK-LABEL: test_vcvtn_f8_f16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    fcvtn v0.8b, v0.4h, v1.4h
+; CHECK-NEXT:    ret
+  %res = call <8 x i8> @llvm.aarch64.neon.fp8.fcvtn.v8i8.v4f16(<4 x half> %vn, <4 x half> %vm)
+  ret <8 x i8> %res
+}
+
+define <16 x i8> @test_vcvtn2_f8_f16(<8 x half> %vn, <8 x half> %vm) {
+; CHECK-LABEL: test_vcvtn2_f8_f16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    fcvtn v0.16b, v0.8h, v1.8h
+; CHECK-NEXT:    ret
+  %res = call <16 x i8> @llvm.aarch64.neon.fp8.fcvtn.v16i8.v8f16(<8 x half> %vn, <8 x half> %vm)
+  ret <16 x i8> %res
+}
diff --git a/llvm/test/CodeGen/AArch64/neon-insextbitcast.ll b/llvm/test/CodeGen/AArch64/neon-insextbitcast.ll
index 28961376d2b6d..ebff3f1370040 100644
--- a/llvm/test/CodeGen/AArch64/neon-insextbitcast.ll
+++ b/llvm/test/CodeGen/AArch64/neon-insextbitcast.ll
@@ -163,8 +163,6 @@ entry:
 define double @test_vext_v1i64(<1 x i64> %a) {
 ; CHECK-LABEL: test_vext_v1i64:
 ; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $q0
 ; CHECK-NEXT:    ret
 entry:
   %b = extractelement <1 x i64> %a, i32 0
diff --git a/llvm/test/CodeGen/AArch64/pauthlr-prologue-duplication.mir b/llvm/test/CodeGen/AArch64/pauthlr-prologue-duplication.mir
new file mode 100644
index 0000000000000..5e57604263793
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/pauthlr-prologue-duplication.mir
@@ -0,0 +1,88 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 5
+# RUN: llc -mtriple aarch64-none-elf -run-pass=block-placement -O3 -o - %s | FileCheck %s
+
+## Check that block-placement does not perform tail duplication on the
+## PAUTH_EPILOGUE instruction. If that happened, the two prologues would use
+## different addresses while calculating the return address signature, so the
+## epilogue could only be correct for (at most) one of them.
+
+--- |
+  define void @test() "frame-pointer"="non-leaf" {
+  entry:
+    ret void
+  }
+
+  declare void @f()
+...
+---
+name:            test
+body:             |
+  ; CHECK-LABEL: name: test
+  ; CHECK: bb.0.entry:
+  ; CHECK-NEXT:   successors: %bb.1(0x30000000), %bb.2(0x50000000)
+  ; CHECK-NEXT:   liveins: $w0, $w1, $lr
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   CBZW renamable $w0, %bb.1
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.2:
+  ; CHECK-NEXT:   successors: %bb.3(0x80000000)
+  ; CHECK-NEXT:   liveins: $w0, $w1, $lr
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   B %bb.3
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.1:
+  ; CHECK-NEXT:   successors: %bb.3(0x80000000)
+  ; CHECK-NEXT:   liveins: $w1, $lr
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   renamable $w8 = MOVZWi 1, 0
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.3:
+  ; CHECK-NEXT:   successors: %bb.5(0x30000000), %bb.4(0x50000000)
+  ; CHECK-NEXT:   liveins: $w1, $w8, $lr
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   frame-setup PAUTH_PROLOGUE implicit-def $lr, implicit killed $lr, implicit $sp
+  ; CHECK-NEXT:   CBZW killed renamable $w1, %bb.5
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.4:
+  ; CHECK-NEXT:   successors: %bb.5(0x80000000)
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   BL @f, csr_aarch64_aapcs, implicit-def dead $lr, implicit $sp, implicit-def $sp
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.5:
+  ; CHECK-NEXT:   BL @f, csr_aarch64_aapcs, implicit-def dead $lr, implicit $sp, implicit-def $sp
+  ; CHECK-NEXT:   frame-destroy PAUTH_EPILOGUE implicit-def $lr, implicit killed $lr, implicit $sp
+  ; CHECK-NEXT:   TCRETURNdi @f, 0, csr_aarch64_aapcs, implicit $sp
+  bb.0.entry:
+    successors: %bb.1(0x30000000), %bb.2(0x50000000)
+    liveins: $w0, $w1, $lr
+
+    CBNZW renamable $w0, %bb.2
+
+  bb.1:
+    successors: %bb.3(0x80000000)
+    liveins: $w1, $lr
+
+    renamable $w8 = MOVZWi 1, 0
+    B %bb.3
+
+  bb.2:
+    successors: %bb.3(0x80000000)
+    liveins: $w0, $w1, $lr
+
+  bb.3:
+    successors: %bb.5(0x30000000), %bb.4(0x50000000)
+    liveins: $w1, $w8, $lr
+
+    frame-setup PAUTH_PROLOGUE implicit-def $lr, implicit killed $lr, implicit $sp
+    CBZW killed renamable $w1, %bb.5
+
+  bb.4:
+    successors: %bb.5(0x80000000)
+
+    BL @f, csr_aarch64_aapcs, implicit-def dead $lr, implicit $sp, implicit-def $sp
+
+  bb.5:
+    BL @f, csr_aarch64_aapcs, implicit-def dead $lr, implicit $sp, implicit-def $sp
+    frame-destroy PAUTH_EPILOGUE implicit-def $lr, implicit killed $lr, implicit $sp
+    TCRETURNdi @f, 0, csr_aarch64_aapcs, implicit $sp
+...
diff --git a/llvm/test/CodeGen/AArch64/perm-tb-with-sme2.ll b/llvm/test/CodeGen/AArch64/perm-tb-with-sme2.ll
new file mode 100644
index 0000000000000..7b55c69ce9378
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/perm-tb-with-sme2.ll
@@ -0,0 +1,306 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -verify-machineinstrs -mtriple=aarch64-none-linux-gnu -enable-subreg-liveness -force-streaming -mattr=+sve2,+sme2 | FileCheck %s
+
+;
+; TBL2
+;
+
+define { <vscale x 16 x i8>, <vscale x 16 x i8> } @tbl2_b_tuple(i64 %stride, ptr %ptr, <vscale x 16 x i8> %a) {
+; CHECK-LABEL: tbl2_b_tuple:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NEXT:    addvl sp, sp, #-3
+; CHECK-NEXT:    str p8, [sp, #7, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    str z12, [sp, #1, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z11, [sp, #2, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    .cfi_escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x18, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 24 * VG
+; CHECK-NEXT:    .cfi_offset w29, -16
+; CHECK-NEXT:    .cfi_escape 0x10, 0x4b, 0x0a, 0x11, 0x70, 0x22, 0x11, 0x78, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d11 @ cfa - 16 - 8 * VG
+; CHECK-NEXT:    .cfi_escape 0x10, 0x4c, 0x0a, 0x11, 0x70, 0x22, 0x11, 0x70, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d12 @ cfa - 16 - 16 * VG
+; CHECK-NEXT:    ptrue pn8.b
+; CHECK-NEXT:    ld1b { z3.b, z11.b }, pn8/z, [x1]
+; CHECK-NEXT:    ld1b { z4.b, z12.b }, pn8/z, [x1, x0]
+; CHECK-NEXT:    ldr p8, [sp, #7, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    tbl z2.b, { z3.b, z4.b }, z0.b
+; CHECK-NEXT:    tbl z1.b, { z11.b, z12.b }, z0.b
+; CHECK-NEXT:    ldr z12, [sp, #1, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z11, [sp, #2, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    mov z0.d, z2.d
+; CHECK-NEXT:    addvl sp, sp, #3
+; CHECK-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    ret
+entry:
+  %0 = tail call target("aarch64.svcount") @llvm.aarch64.sve.ptrue.c8()
+  %1 = tail call { <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.ld1.pn.x2.nxv16i8(target("aarch64.svcount") %0, ptr %ptr)
+  %2 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8> } %1, 0
+  %3 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8> } %1, 1
+  %arrayidx2 = getelementptr inbounds i8, ptr %ptr, i64 %stride
+  %4 = tail call { <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.ld1.pn.x2.nxv16i8(target("aarch64.svcount") %0, ptr %arrayidx2)
+  %5 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8> } %4, 0
+  %6 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8> } %4, 1
+  %res1 = call <vscale x 16 x i8> @llvm.aarch64.sve.tbl2.nxv16i8(<vscale x 16 x i8> %2, <vscale x 16 x i8> %5, <vscale x 16 x i8> %a)
+  %res2 = call <vscale x 16 x i8> @llvm.aarch64.sve.tbl2.nxv16i8(<vscale x 16 x i8> %3, <vscale x 16 x i8> %6, <vscale x 16 x i8> %a)
+  %ins1 = insertvalue { <vscale x 16 x i8>, <vscale x 16 x i8> } poison, <vscale x 16 x i8> %res1, 0
+  %ins2 = insertvalue { <vscale x 16 x i8>, <vscale x 16 x i8> } %ins1, <vscale x 16 x i8> %res2, 1
+  ret { <vscale x 16 x i8>, <vscale x 16 x i8> } %ins2
+}
+
+define { <vscale x 8 x i16>, <vscale x 8 x i16> } @tbl2_h_tuple(i64 %stride, ptr %ptr, <vscale x 8 x i16> %a) {
+; CHECK-LABEL: tbl2_h_tuple:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NEXT:    addvl sp, sp, #-3
+; CHECK-NEXT:    str p8, [sp, #7, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    str z12, [sp, #1, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z11, [sp, #2, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    .cfi_escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x18, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 24 * VG
+; CHECK-NEXT:    .cfi_offset w29, -16
+; CHECK-NEXT:    .cfi_escape 0x10, 0x4b, 0x0a, 0x11, 0x70, 0x22, 0x11, 0x78, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d11 @ cfa - 16 - 8 * VG
+; CHECK-NEXT:    .cfi_escape 0x10, 0x4c, 0x0a, 0x11, 0x70, 0x22, 0x11, 0x70, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d12 @ cfa - 16 - 16 * VG
+; CHECK-NEXT:    ptrue pn8.b
+; CHECK-NEXT:    add x8, x1, x0
+; CHECK-NEXT:    ld1h { z3.h, z11.h }, pn8/z, [x1]
+; CHECK-NEXT:    ld1h { z4.h, z12.h }, pn8/z, [x8]
+; CHECK-NEXT:    ldr p8, [sp, #7, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    tbl z2.h, { z3.h, z4.h }, z0.h
+; CHECK-NEXT:    tbl z1.h, { z11.h, z12.h }, z0.h
+; CHECK-NEXT:    ldr z12, [sp, #1, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z11, [sp, #2, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    mov z0.d, z2.d
+; CHECK-NEXT:    addvl sp, sp, #3
+; CHECK-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    ret
+entry:
+  %0 = tail call target("aarch64.svcount") @llvm.aarch64.sve.ptrue.c8()
+  %1 = tail call { <vscale x 8 x i16>, <vscale x 8 x i16> } @llvm.aarch64.sve.ld1.pn.x2.nxv8i16(target("aarch64.svcount") %0, ptr %ptr)
+  %2 = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16> } %1, 0
+  %3 = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16> } %1, 1
+  %arrayidx2 = getelementptr inbounds i8, ptr %ptr, i64 %stride
+  %4 = tail call { <vscale x 8 x i16>, <vscale x 8 x i16> } @llvm.aarch64.sve.ld1.pn.x2.nxv8i16(target("aarch64.svcount") %0, ptr %arrayidx2)
+  %5 = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16> } %4, 0
+  %6 = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16> } %4, 1
+  %res1 = call <vscale x 8 x i16> @llvm.aarch64.sve.tbl2.nxv8i16(<vscale x 8 x i16> %2, <vscale x 8 x i16> %5, <vscale x 8 x i16> %a)
+  %res2 = call <vscale x 8 x i16> @llvm.aarch64.sve.tbl2.nxv8i16(<vscale x 8 x i16> %3, <vscale x 8 x i16> %6, <vscale x 8 x i16> %a)
+  %ins1 = insertvalue { <vscale x 8 x i16>, <vscale x 8 x i16> } poison, <vscale x 8 x i16> %res1, 0
+  %ins2 = insertvalue { <vscale x 8 x i16>, <vscale x 8 x i16> } %ins1, <vscale x 8 x i16> %res2, 1
+  ret { <vscale x 8 x i16>, <vscale x 8 x i16> } %ins2
+}
+
+define { <vscale x 4 x i32>, <vscale x 4 x i32> } @tbl2_s_tuple(i64 %stride, ptr %ptr, <vscale x 4 x i32> %a) {
+; CHECK-LABEL: tbl2_s_tuple:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NEXT:    addvl sp, sp, #-3
+; CHECK-NEXT:    str p8, [sp, #7, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    str z12, [sp, #1, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z11, [sp, #2, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    .cfi_escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x18, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 24 * VG
+; CHECK-NEXT:    .cfi_offset w29, -16
+; CHECK-NEXT:    .cfi_escape 0x10, 0x4b, 0x0a, 0x11, 0x70, 0x22, 0x11, 0x78, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d11 @ cfa - 16 - 8 * VG
+; CHECK-NEXT:    .cfi_escape 0x10, 0x4c, 0x0a, 0x11, 0x70, 0x22, 0x11, 0x70, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d12 @ cfa - 16 - 16 * VG
+; CHECK-NEXT:    ptrue pn8.b
+; CHECK-NEXT:    add x8, x1, x0
+; CHECK-NEXT:    ld1w { z3.s, z11.s }, pn8/z, [x1]
+; CHECK-NEXT:    ld1w { z4.s, z12.s }, pn8/z, [x8]
+; CHECK-NEXT:    ldr p8, [sp, #7, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    tbl z2.s, { z3.s, z4.s }, z0.s
+; CHECK-NEXT:    tbl z1.s, { z11.s, z12.s }, z0.s
+; CHECK-NEXT:    ldr z12, [sp, #1, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z11, [sp, #2, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    mov z0.d, z2.d
+; CHECK-NEXT:    addvl sp, sp, #3
+; CHECK-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    ret
+entry:
+  %0 = tail call target("aarch64.svcount") @llvm.aarch64.sve.ptrue.c8()
+  %1 = tail call { <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.aarch64.sve.ld1.pn.x2.nxv4i32(target("aarch64.svcount") %0, ptr %ptr)
+  %2 = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } %1, 0
+  %3 = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } %1, 1
+  %arrayidx2 = getelementptr inbounds i8, ptr %ptr, i64 %stride
+  %4 = tail call { <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.aarch64.sve.ld1.pn.x2.nxv4i32(target("aarch64.svcount") %0, ptr %arrayidx2)
+  %5 = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } %4, 0
+  %6 = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } %4, 1
+  %res1 = call <vscale x 4 x i32> @llvm.aarch64.sve.tbl2.nxv4i32(<vscale x 4 x i32> %2, <vscale x 4 x i32> %5, <vscale x 4 x i32> %a)
+  %res2 = call <vscale x 4 x i32> @llvm.aarch64.sve.tbl2.nxv4i32(<vscale x 4 x i32> %3, <vscale x 4 x i32> %6, <vscale x 4 x i32> %a)
+  %ins1 = insertvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } poison, <vscale x 4 x i32> %res1, 0
+  %ins2 = insertvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } %ins1, <vscale x 4 x i32> %res2, 1
+  ret { <vscale x 4 x i32>, <vscale x 4 x i32> } %ins2
+}
+
+define { <vscale x 2 x i64>, <vscale x 2 x i64> } @tbl2_d_tuple(i64 %stride, ptr %ptr, <vscale x 2 x i64> %a) {
+; CHECK-LABEL: tbl2_d_tuple:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NEXT:    addvl sp, sp, #-3
+; CHECK-NEXT:    str p8, [sp, #7, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    str z12, [sp, #1, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z11, [sp, #2, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    .cfi_escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x18, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 24 * VG
+; CHECK-NEXT:    .cfi_offset w29, -16
+; CHECK-NEXT:    .cfi_escape 0x10, 0x4b, 0x0a, 0x11, 0x70, 0x22, 0x11, 0x78, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d11 @ cfa - 16 - 8 * VG
+; CHECK-NEXT:    .cfi_escape 0x10, 0x4c, 0x0a, 0x11, 0x70, 0x22, 0x11, 0x70, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d12 @ cfa - 16 - 16 * VG
+; CHECK-NEXT:    ptrue pn8.b
+; CHECK-NEXT:    add x8, x1, x0
+; CHECK-NEXT:    ld1d { z3.d, z11.d }, pn8/z, [x1]
+; CHECK-NEXT:    ld1d { z4.d, z12.d }, pn8/z, [x8]
+; CHECK-NEXT:    ldr p8, [sp, #7, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    tbl z2.d, { z3.d, z4.d }, z0.d
+; CHECK-NEXT:    tbl z1.d, { z11.d, z12.d }, z0.d
+; CHECK-NEXT:    ldr z12, [sp, #1, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z11, [sp, #2, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    mov z0.d, z2.d
+; CHECK-NEXT:    addvl sp, sp, #3
+; CHECK-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    ret
+entry:
+  %0 = tail call target("aarch64.svcount") @llvm.aarch64.sve.ptrue.c8()
+  %1 = tail call { <vscale x 2 x i64>, <vscale x 2 x i64> } @llvm.aarch64.sve.ld1.pn.x2.nxv2i64(target("aarch64.svcount") %0, ptr %ptr)
+  %2 = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64> } %1, 0
+  %3 = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64> } %1, 1
+  %arrayidx2 = getelementptr inbounds i8, ptr %ptr, i64 %stride
+  %4 = tail call { <vscale x 2 x i64>, <vscale x 2 x i64> } @llvm.aarch64.sve.ld1.pn.x2.nxv2i64(target("aarch64.svcount") %0, ptr %arrayidx2)
+  %5 = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64> } %4, 0
+  %6 = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64> } %4, 1
+  %res1 = call <vscale x 2 x i64> @llvm.aarch64.sve.tbl2.nxv2i64(<vscale x 2 x i64> %2, <vscale x 2 x i64> %5, <vscale x 2 x i64> %a)
+  %res2 = call <vscale x 2 x i64> @llvm.aarch64.sve.tbl2.nxv2i64(<vscale x 2 x i64> %3, <vscale x 2 x i64> %6, <vscale x 2 x i64> %a)
+  %ins1 = insertvalue { <vscale x 2 x i64>, <vscale x 2 x i64> } poison, <vscale x 2 x i64> %res1, 0
+  %ins2 = insertvalue { <vscale x 2 x i64>, <vscale x 2 x i64> } %ins1, <vscale x 2 x i64> %res2, 1
+  ret { <vscale x 2 x i64>, <vscale x 2 x i64> } %ins2
+}
+
+define { <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } @tbl2_bf16_tuple(i64 %stride, ptr %ptr, <vscale x 8 x i16> %a) #0 {
+; CHECK-LABEL: tbl2_bf16_tuple:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NEXT:    addvl sp, sp, #-3
+; CHECK-NEXT:    str p8, [sp, #7, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    str z12, [sp, #1, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z11, [sp, #2, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    .cfi_escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x18, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 24 * VG
+; CHECK-NEXT:    .cfi_offset w29, -16
+; CHECK-NEXT:    .cfi_escape 0x10, 0x4b, 0x0a, 0x11, 0x70, 0x22, 0x11, 0x78, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d11 @ cfa - 16 - 8 * VG
+; CHECK-NEXT:    .cfi_escape 0x10, 0x4c, 0x0a, 0x11, 0x70, 0x22, 0x11, 0x70, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d12 @ cfa - 16 - 16 * VG
+; CHECK-NEXT:    ptrue pn8.b
+; CHECK-NEXT:    add x8, x1, x0
+; CHECK-NEXT:    ld1h { z3.h, z11.h }, pn8/z, [x1]
+; CHECK-NEXT:    ld1h { z4.h, z12.h }, pn8/z, [x8]
+; CHECK-NEXT:    ldr p8, [sp, #7, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    tbl z2.h, { z3.h, z4.h }, z0.h
+; CHECK-NEXT:    tbl z1.h, { z11.h, z12.h }, z0.h
+; CHECK-NEXT:    ldr z12, [sp, #1, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z11, [sp, #2, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    mov z0.d, z2.d
+; CHECK-NEXT:    addvl sp, sp, #3
+; CHECK-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    ret
+entry:
+  %0 = tail call target("aarch64.svcount") @llvm.aarch64.sve.ptrue.c8()
+  %1 = tail call { <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } @llvm.aarch64.sve.ld1.pn.x2.nxv8bf16(target("aarch64.svcount") %0, ptr %ptr)
+  %2 = extractvalue { <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } %1, 0
+  %3 = extractvalue { <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } %1, 1
+  %arrayidx2 = getelementptr inbounds i8, ptr %ptr, i64 %stride
+  %4 = tail call { <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } @llvm.aarch64.sve.ld1.pn.x2.nxv8bf16(target("aarch64.svcount") %0, ptr %arrayidx2)
+  %5 = extractvalue { <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } %4, 0
+  %6 = extractvalue { <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } %4, 1
+  %res1 = call <vscale x 8 x bfloat> @llvm.aarch64.sve.tbl2.nxv8bf16(<vscale x 8 x bfloat> %2, <vscale x 8 x bfloat> %5, <vscale x 8 x i16> %a)
+  %res2 = call <vscale x 8 x bfloat> @llvm.aarch64.sve.tbl2.nxv8bf16(<vscale x 8 x bfloat> %3, <vscale x 8 x bfloat> %6, <vscale x 8 x i16> %a)
+  %ins1 = insertvalue { <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } poison, <vscale x 8 x bfloat> %res1, 0
+  %ins2 = insertvalue { <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } %ins1, <vscale x 8 x bfloat> %res2, 1
+  ret { <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } %ins2
+}
+
+define { <vscale x 4 x float>, <vscale x 4 x float> } @tbl2_f32_tuple(i64 %stride, ptr %ptr, <vscale x 4 x i32> %a) {
+; CHECK-LABEL: tbl2_f32_tuple:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NEXT:    addvl sp, sp, #-3
+; CHECK-NEXT:    str p8, [sp, #7, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    str z12, [sp, #1, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z11, [sp, #2, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    .cfi_escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x18, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 24 * VG
+; CHECK-NEXT:    .cfi_offset w29, -16
+; CHECK-NEXT:    .cfi_escape 0x10, 0x4b, 0x0a, 0x11, 0x70, 0x22, 0x11, 0x78, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d11 @ cfa - 16 - 8 * VG
+; CHECK-NEXT:    .cfi_escape 0x10, 0x4c, 0x0a, 0x11, 0x70, 0x22, 0x11, 0x70, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d12 @ cfa - 16 - 16 * VG
+; CHECK-NEXT:    ptrue pn8.b
+; CHECK-NEXT:    add x8, x1, x0
+; CHECK-NEXT:    ld1w { z3.s, z11.s }, pn8/z, [x1]
+; CHECK-NEXT:    ld1w { z4.s, z12.s }, pn8/z, [x8]
+; CHECK-NEXT:    ldr p8, [sp, #7, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    tbl z2.s, { z3.s, z4.s }, z0.s
+; CHECK-NEXT:    tbl z1.s, { z11.s, z12.s }, z0.s
+; CHECK-NEXT:    ldr z12, [sp, #1, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z11, [sp, #2, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    mov z0.d, z2.d
+; CHECK-NEXT:    addvl sp, sp, #3
+; CHECK-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    ret
+entry:
+  %0 = tail call target("aarch64.svcount") @llvm.aarch64.sve.ptrue.c8()
+  %1 = tail call { <vscale x 4 x float>, <vscale x 4 x float> } @llvm.aarch64.sve.ld1.pn.x2.nxv4f32(target("aarch64.svcount") %0, ptr %ptr)
+  %2 = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } %1, 0
+  %3 = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } %1, 1
+  %arrayidx2 = getelementptr inbounds i8, ptr %ptr, i64 %stride
+  %4 = tail call { <vscale x 4 x float>, <vscale x 4 x float> } @llvm.aarch64.sve.ld1.pn.x2.nxv4f32(target("aarch64.svcount") %0, ptr %arrayidx2)
+  %5 = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } %4, 0
+  %6 = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } %4, 1
+  %res1 = call <vscale x 4 x float> @llvm.aarch64.sve.tbl2.nxv4f32(<vscale x 4 x float> %2, <vscale x 4 x float> %5, <vscale x 4 x i32> %a)
+  %res2 = call <vscale x 4 x float> @llvm.aarch64.sve.tbl2.nxv4f32(<vscale x 4 x float> %3, <vscale x 4 x float> %6, <vscale x 4 x i32> %a)
+  %ins1 = insertvalue { <vscale x 4 x float>, <vscale x 4 x float> } poison, <vscale x 4 x float> %res1, 0
+  %ins2 = insertvalue { <vscale x 4 x float>, <vscale x 4 x float> } %ins1, <vscale x 4 x float> %res2, 1
+  ret { <vscale x 4 x float>, <vscale x 4 x float> } %ins2
+}
+
+define { <vscale x 2 x double>, <vscale x 2 x double> } @tbl2_f64_tuple(i64 %stride, ptr %ptr, <vscale x 2 x i64> %a) {
+; CHECK-LABEL: tbl2_f64_tuple:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NEXT:    addvl sp, sp, #-3
+; CHECK-NEXT:    str p8, [sp, #7, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    str z12, [sp, #1, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z11, [sp, #2, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    .cfi_escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x18, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 24 * VG
+; CHECK-NEXT:    .cfi_offset w29, -16
+; CHECK-NEXT:    .cfi_escape 0x10, 0x4b, 0x0a, 0x11, 0x70, 0x22, 0x11, 0x78, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d11 @ cfa - 16 - 8 * VG
+; CHECK-NEXT:    .cfi_escape 0x10, 0x4c, 0x0a, 0x11, 0x70, 0x22, 0x11, 0x70, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d12 @ cfa - 16 - 16 * VG
+; CHECK-NEXT:    ptrue pn8.b
+; CHECK-NEXT:    add x8, x1, x0
+; CHECK-NEXT:    ld1d { z3.d, z11.d }, pn8/z, [x1]
+; CHECK-NEXT:    ld1d { z4.d, z12.d }, pn8/z, [x8]
+; CHECK-NEXT:    ldr p8, [sp, #7, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    tbl z2.d, { z3.d, z4.d }, z0.d
+; CHECK-NEXT:    tbl z1.d, { z11.d, z12.d }, z0.d
+; CHECK-NEXT:    ldr z12, [sp, #1, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z11, [sp, #2, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    mov z0.d, z2.d
+; CHECK-NEXT:    addvl sp, sp, #3
+; CHECK-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    ret
+entry:
+  %0 = tail call target("aarch64.svcount") @llvm.aarch64.sve.ptrue.c8()
+  %1 = tail call { <vscale x 2 x double>, <vscale x 2 x double> } @llvm.aarch64.sve.ld1.pn.x2.nxv2f64(target("aarch64.svcount") %0, ptr %ptr)
+  %2 = extractvalue { <vscale x 2 x double>, <vscale x 2 x double> } %1, 0
+  %3 = extractvalue { <vscale x 2 x double>, <vscale x 2 x double> } %1, 1
+  %arrayidx2 = getelementptr inbounds i8, ptr %ptr, i64 %stride
+  %4 = tail call { <vscale x 2 x double>, <vscale x 2 x double> } @llvm.aarch64.sve.ld1.pn.x2.nxv2f64(target("aarch64.svcount") %0, ptr %arrayidx2)
+  %5 = extractvalue { <vscale x 2 x double>, <vscale x 2 x double> } %4, 0
+  %6 = extractvalue { <vscale x 2 x double>, <vscale x 2 x double> } %4, 1
+  %res1 = call <vscale x 2 x double> @llvm.aarch64.sve.tbl2.nxv2f64(<vscale x 2 x double> %2, <vscale x 2 x double> %5, <vscale x 2 x i64> %a)
+  %res2 = call <vscale x 2 x double> @llvm.aarch64.sve.tbl2.nxv2f64(<vscale x 2 x double> %3, <vscale x 2 x double> %6, <vscale x 2 x i64> %a)
+  %ins1 = insertvalue { <vscale x 2 x double>, <vscale x 2 x double> } poison, <vscale x 2 x double> %res1, 0
+  %ins2 = insertvalue { <vscale x 2 x double>, <vscale x 2 x double> } %ins1, <vscale x 2 x double> %res2, 1
+  ret { <vscale x 2 x double>, <vscale x 2 x double> } %ins2
+}
+
+declare <vscale x 16 x i8> @llvm.aarch64.sve.tbl2.nxv16i8(<vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>)
+declare <vscale x 8 x i16> @llvm.aarch64.sve.tbl2.nxv8i16(<vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>)
+declare <vscale x 4 x i32> @llvm.aarch64.sve.tbl2.nxv4i32(<vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>)
+declare <vscale x 2 x i64> @llvm.aarch64.sve.tbl2.nxv2i64(<vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>)
+
+declare <vscale x 8 x half> @llvm.aarch64.sve.tbl2.nxv8f16(<vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x i16>)
+declare <vscale x 4 x float> @llvm.aarch64.sve.tbl2.nxv4f32(<vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x i32>)
+declare <vscale x 2 x double> @llvm.aarch64.sve.tbl2.nxv2f64(<vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x i64>)
+
+declare <vscale x 8 x bfloat> @llvm.aarch64.sve.tbl2.nxv8bf16(<vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x i16>)
+
+; +bf16 is required for the bfloat version.
+attributes #0 = { "target-features"="+sve2,+bf16" }
diff --git a/llvm/test/CodeGen/AArch64/shift-by-signext.ll b/llvm/test/CodeGen/AArch64/shift-by-signext.ll
index 67e2da96084ef..3bcaee523f2e9 100644
--- a/llvm/test/CodeGen/AArch64/shift-by-signext.ll
+++ b/llvm/test/CodeGen/AArch64/shift-by-signext.ll
@@ -81,7 +81,6 @@ define i32 @n6_fshl(i32 %x, i32 %y, i8 %shamt) nounwind {
 ; CHECK-LABEL: n6_fshl:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    lsr w8, w1, #1
-; CHECK-NEXT:    // kill: def $w2 killed $w2 def $x2
 ; CHECK-NEXT:    mvn w9, w2
 ; CHECK-NEXT:    lsl w10, w0, w2
 ; CHECK-NEXT:    lsr w8, w8, w9
@@ -95,7 +94,6 @@ define i32 @n7_fshr(i32 %x, i32 %y, i8 %shamt) nounwind {
 ; CHECK-LABEL: n7_fshr:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    lsl w8, w0, #1
-; CHECK-NEXT:    // kill: def $w2 killed $w2 def $x2
 ; CHECK-NEXT:    mvn w9, w2
 ; CHECK-NEXT:    lsr w10, w1, w2
 ; CHECK-NEXT:    lsl w8, w8, w9
diff --git a/llvm/test/CodeGen/AArch64/shift.ll b/llvm/test/CodeGen/AArch64/shift.ll
index 066928687cc02..5f24af82d8342 100644
--- a/llvm/test/CodeGen/AArch64/shift.ll
+++ b/llvm/test/CodeGen/AArch64/shift.ll
@@ -21,7 +21,6 @@ define i1 @shl_i1(i1 %0, i1 %1){
 define i8 @shl_i8(i8 %0, i8 %1){
 ; CHECK-SD-LABEL: shl_i8:
 ; CHECK-SD:       // %bb.0:
-; CHECK-SD-NEXT:    // kill: def $w1 killed $w1 def $x1
 ; CHECK-SD-NEXT:    lsl w0, w0, w1
 ; CHECK-SD-NEXT:    ret
 ;
@@ -37,7 +36,6 @@ define i8 @shl_i8(i8 %0, i8 %1){
 define i16 @shl_i16(i16 %0, i16 %1){
 ; CHECK-SD-LABEL: shl_i16:
 ; CHECK-SD:       // %bb.0:
-; CHECK-SD-NEXT:    // kill: def $w1 killed $w1 def $x1
 ; CHECK-SD-NEXT:    lsl w0, w0, w1
 ; CHECK-SD-NEXT:    ret
 ;
@@ -123,7 +121,6 @@ define i8 @ashr_i8(i8 %0, i8 %1){
 ; CHECK-SD-LABEL: ashr_i8:
 ; CHECK-SD:       // %bb.0:
 ; CHECK-SD-NEXT:    sxtb w8, w0
-; CHECK-SD-NEXT:    // kill: def $w1 killed $w1 def $x1
 ; CHECK-SD-NEXT:    asr w0, w8, w1
 ; CHECK-SD-NEXT:    ret
 ;
@@ -141,7 +138,6 @@ define i16 @ashr_i16(i16 %0, i16 %1){
 ; CHECK-SD-LABEL: ashr_i16:
 ; CHECK-SD:       // %bb.0:
 ; CHECK-SD-NEXT:    sxth w8, w0
-; CHECK-SD-NEXT:    // kill: def $w1 killed $w1 def $x1
 ; CHECK-SD-NEXT:    asr w0, w8, w1
 ; CHECK-SD-NEXT:    ret
 ;
@@ -230,7 +226,6 @@ define i8 @lshr_i8(i8 %0, i8 %1){
 ; CHECK-SD-LABEL: lshr_i8:
 ; CHECK-SD:       // %bb.0:
 ; CHECK-SD-NEXT:    and w8, w0, #0xff
-; CHECK-SD-NEXT:    // kill: def $w1 killed $w1 def $x1
 ; CHECK-SD-NEXT:    lsr w0, w8, w1
 ; CHECK-SD-NEXT:    ret
 ;
@@ -248,7 +243,6 @@ define i16 @lshr_i16(i16 %0, i16 %1){
 ; CHECK-SD-LABEL: lshr_i16:
 ; CHECK-SD:       // %bb.0:
 ; CHECK-SD-NEXT:    and w8, w0, #0xffff
-; CHECK-SD-NEXT:    // kill: def $w1 killed $w1 def $x1
 ; CHECK-SD-NEXT:    lsr w0, w8, w1
 ; CHECK-SD-NEXT:    ret
 ;
diff --git a/llvm/test/CodeGen/AArch64/sink-and-fold.ll b/llvm/test/CodeGen/AArch64/sink-and-fold.ll
index f65a08ae2acea..4d383fefc43c7 100644
--- a/llvm/test/CodeGen/AArch64/sink-and-fold.ll
+++ b/llvm/test/CodeGen/AArch64/sink-and-fold.ll
@@ -298,7 +298,6 @@ exit:
 define i32 @f6(i1 %c, ptr %a, i32 %i) {
 ; CHECK-LABEL: f6:
 ; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    // kill: def $w2 killed $w2 def $x2
 ; CHECK-NEXT:    tbz w0, #0, .LBB6_2
 ; CHECK-NEXT:  // %bb.1: // %if.then
 ; CHECK-NEXT:    mov w0, wzr
diff --git a/llvm/test/CodeGen/AArch64/sme2-fp8-intrinsics-cvt.ll b/llvm/test/CodeGen/AArch64/sme2-fp8-intrinsics-cvt.ll
index 38d3bed2eaf90..b0390ec73ae97 100644
--- a/llvm/test/CodeGen/AArch64/sme2-fp8-intrinsics-cvt.ll
+++ b/llvm/test/CodeGen/AArch64/sme2-fp8-intrinsics-cvt.ll
@@ -1,13 +1,11 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 2
-; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sme2,+fp8 -verify-machineinstrs -force-streaming < %s | FileCheck %s
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sme2,+fp8 -verify-machineinstrs -force-streaming -enable-subreg-liveness < %s | FileCheck %s
 
 ; FCVT / FCVTN / BFCVT
 
 define <vscale x 16 x i8> @fcvt_x2(<vscale x 8 x half> %zn0, <vscale x 8 x half> %zn1) {
 ; CHECK-LABEL: fcvt_x2:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    // kill: def $z1 killed $z1 killed $z0_z1 def $z0_z1
-; CHECK-NEXT:    // kill: def $z0 killed $z0 killed $z0_z1 def $z0_z1
 ; CHECK-NEXT:    fcvt z0.b, { z0.h, z1.h }
 ; CHECK-NEXT:    ret
   %res = call <vscale x 16 x i8> @llvm.aarch64.sve.fp8.cvt.x2.nxv8f16(<vscale x 8 x half> %zn0, <vscale x 8 x half> %zn1)
@@ -17,10 +15,6 @@ define <vscale x 16 x i8> @fcvt_x2(<vscale x 8 x half> %zn0, <vscale x 8 x half>
 define <vscale x 16 x i8> @fcvt_x4(<vscale x 4 x float> %zn0, <vscale x 4 x float> %zn1, <vscale x 4 x float> %zn2, <vscale x 4 x float> %zn3) {
 ; CHECK-LABEL: fcvt_x4:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    // kill: def $z3 killed $z3 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
-; CHECK-NEXT:    // kill: def $z2 killed $z2 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
-; CHECK-NEXT:    // kill: def $z1 killed $z1 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
-; CHECK-NEXT:    // kill: def $z0 killed $z0 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
 ; CHECK-NEXT:    fcvt z0.b, { z0.s - z3.s }
 ; CHECK-NEXT:    ret
   %res = call <vscale x 16 x i8> @llvm.aarch64.sve.fp8.cvt.x4(<vscale x 4 x float> %zn0, <vscale x 4 x float> %zn1,
@@ -28,13 +22,88 @@ define <vscale x 16 x i8> @fcvt_x4(<vscale x 4 x float> %zn0, <vscale x 4 x floa
   ret <vscale x 16 x i8> %res
 }
 
+define { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @fcvt_x4_tuple(i64 %stride, ptr %ptr) {
+; CHECK-LABEL: fcvt_x4_tuple:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NEXT:    addvl sp, sp, #-9
+; CHECK-NEXT:    str p8, [sp, #7, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    str z23, [sp, #1, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z22, [sp, #2, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z21, [sp, #3, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z20, [sp, #4, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z19, [sp, #5, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z18, [sp, #6, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z17, [sp, #7, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z16, [sp, #8, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    .cfi_escape 0x0f, 0x0d, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0xc8, 0x00, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 72 * VG
+; CHECK-NEXT:    .cfi_offset w29, -16
+; CHECK-NEXT:    lsl x8, x0, #1
+; CHECK-NEXT:    add x9, x1, x0
+; CHECK-NEXT:    ptrue pn8.b
+; CHECK-NEXT:    ld1w { z16.s, z20.s, z24.s, z28.s }, pn8/z, [x1]
+; CHECK-NEXT:    ld1w { z17.s, z21.s, z25.s, z29.s }, pn8/z, [x9]
+; CHECK-NEXT:    add x10, x1, x8
+; CHECK-NEXT:    add x8, x9, x8
+; CHECK-NEXT:    ld1w { z18.s, z22.s, z26.s, z30.s }, pn8/z, [x10]
+; CHECK-NEXT:    ld1w { z19.s, z23.s, z27.s, z31.s }, pn8/z, [x8]
+; CHECK-NEXT:    fcvt z0.b, { z16.s - z19.s }
+; CHECK-NEXT:    fcvt z1.b, { z20.s - z23.s }
+; CHECK-NEXT:    fcvt z2.b, { z24.s - z27.s }
+; CHECK-NEXT:    fcvt z3.b, { z28.s - z31.s }
+; CHECK-NEXT:    ldr z23, [sp, #1, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z22, [sp, #2, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z21, [sp, #3, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z20, [sp, #4, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z19, [sp, #5, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z18, [sp, #6, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z17, [sp, #7, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z16, [sp, #8, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr p8, [sp, #7, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    addvl sp, sp, #9
+; CHECK-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    ret
+entry:
+  %0 = tail call target("aarch64.svcount") @llvm.aarch64.sve.ptrue.c8()
+  %1 = tail call { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } @llvm.aarch64.sve.ld1.pn.x4.nxv4f32(target("aarch64.svcount") %0, ptr %ptr)
+  %2 = extractvalue { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } %1, 0
+  %3 = extractvalue { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } %1, 1
+  %4 = extractvalue { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } %1, 2
+  %5 = extractvalue { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } %1, 3
+  %arrayidx2 = getelementptr inbounds i8, ptr %ptr, i64 %stride
+  %6 = tail call { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } @llvm.aarch64.sve.ld1.pn.x4.nxv4f32(target("aarch64.svcount") %0, ptr %arrayidx2)
+  %7 = extractvalue { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } %6, 0
+  %8 = extractvalue { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } %6, 1
+  %9 = extractvalue { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } %6, 2
+  %10 = extractvalue { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } %6, 3
+  %mul3 = shl i64 %stride, 1
+  %arrayidx4 = getelementptr inbounds i8, ptr %ptr, i64 %mul3
+  %11 = tail call { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } @llvm.aarch64.sve.ld1.pn.x4.nxv4f32(target("aarch64.svcount") %0, ptr %arrayidx4)
+  %12 = extractvalue { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } %11, 0
+  %13 = extractvalue { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } %11, 1
+  %14 = extractvalue { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } %11, 2
+  %15 = extractvalue { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } %11, 3
+  %mul5 = mul i64 %stride, 3
+  %arrayidx6 = getelementptr inbounds i8, ptr %ptr, i64 %mul5
+  %16 = tail call { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } @llvm.aarch64.sve.ld1.pn.x4.nxv4f32(target("aarch64.svcount") %0, ptr %arrayidx6)
+  %17 = extractvalue { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } %16, 0
+  %18 = extractvalue { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } %16, 1
+  %19 = extractvalue { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } %16, 2
+  %20 = extractvalue { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } %16, 3
+  %res1 = call <vscale x 16 x i8> @llvm.aarch64.sve.fp8.cvt.x4(<vscale x 4 x float> %2, <vscale x 4 x float> %7, <vscale x 4 x float> %12, <vscale x 4 x float> %17)
+  %res2 = call <vscale x 16 x i8> @llvm.aarch64.sve.fp8.cvt.x4(<vscale x 4 x float> %3, <vscale x 4 x float> %8, <vscale x 4 x float> %13, <vscale x 4 x float> %18)
+  %res3 = call <vscale x 16 x i8> @llvm.aarch64.sve.fp8.cvt.x4(<vscale x 4 x float> %4, <vscale x 4 x float> %9, <vscale x 4 x float> %14, <vscale x 4 x float> %19)
+  %res4 = call <vscale x 16 x i8> @llvm.aarch64.sve.fp8.cvt.x4(<vscale x 4 x float> %5, <vscale x 4 x float> %10, <vscale x 4 x float> %15, <vscale x 4 x float> %20)
+  %ins1 = insertvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } poison, <vscale x 16 x i8> %res1, 0
+  %ins2 = insertvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %ins1, <vscale x 16 x i8> %res2, 1
+  %ins3 = insertvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %ins2, <vscale x 16 x i8> %res3, 2
+  %ins4 = insertvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %ins3, <vscale x 16 x i8> %res4, 3
+  ret { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %ins4
+}
+
 define <vscale x 16 x i8> @fcvtn(<vscale x 4 x float> %zn0, <vscale x 4 x float> %zn1, <vscale x 4 x float> %zn2, <vscale x 4 x float> %zn3) {
 ; CHECK-LABEL: fcvtn:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    // kill: def $z3 killed $z3 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
-; CHECK-NEXT:    // kill: def $z2 killed $z2 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
-; CHECK-NEXT:    // kill: def $z1 killed $z1 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
-; CHECK-NEXT:    // kill: def $z0 killed $z0 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
 ; CHECK-NEXT:    fcvtn z0.b, { z0.s - z3.s }
 ; CHECK-NEXT:    ret
   %res = call <vscale x 16 x i8> @llvm.aarch64.sve.fp8.cvtn.x4(<vscale x 4 x float> %zn0, <vscale x 4 x float> %zn1,
@@ -45,14 +114,53 @@ define <vscale x 16 x i8> @fcvtn(<vscale x 4 x float> %zn0, <vscale x 4 x float>
 define <vscale x 16 x i8> @bfcvt(<vscale x 8 x bfloat> %zn0, <vscale x 8 x bfloat> %zn1) {
 ; CHECK-LABEL: bfcvt:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    // kill: def $z1 killed $z1 killed $z0_z1 def $z0_z1
-; CHECK-NEXT:    // kill: def $z0 killed $z0 killed $z0_z1 def $z0_z1
 ; CHECK-NEXT:    bfcvt z0.b, { z0.h, z1.h }
 ; CHECK-NEXT:    ret
   %res = call <vscale x 16 x i8> @llvm.aarch64.sve.fp8.cvt.x2.nxv8bf16(<vscale x 8 x bfloat> %zn0, <vscale x 8 x bfloat> %zn1)
   ret <vscale x 16 x i8> %res
 }
 
+
+define { <vscale x 16 x i8>, <vscale x 16 x i8> } @bfcvt_tuple(i64 %stride, ptr %ptr) {
+; CHECK-LABEL: bfcvt_tuple:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NEXT:    addvl sp, sp, #-3
+; CHECK-NEXT:    str p8, [sp, #7, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    str z11, [sp, #1, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z10, [sp, #2, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    .cfi_escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x18, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 24 * VG
+; CHECK-NEXT:    .cfi_offset w29, -16
+; CHECK-NEXT:    .cfi_escape 0x10, 0x4a, 0x0a, 0x11, 0x70, 0x22, 0x11, 0x78, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d10 @ cfa - 16 - 8 * VG
+; CHECK-NEXT:    .cfi_escape 0x10, 0x4b, 0x0a, 0x11, 0x70, 0x22, 0x11, 0x70, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d11 @ cfa - 16 - 16 * VG
+; CHECK-NEXT:    ptrue pn8.b
+; CHECK-NEXT:    add x8, x1, x0
+; CHECK-NEXT:    ld1h { z2.h, z10.h }, pn8/z, [x1]
+; CHECK-NEXT:    ld1h { z3.h, z11.h }, pn8/z, [x8]
+; CHECK-NEXT:    bfcvt z0.b, { z2.h, z3.h }
+; CHECK-NEXT:    bfcvt z1.b, { z10.h, z11.h }
+; CHECK-NEXT:    ldr z11, [sp, #1, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z10, [sp, #2, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr p8, [sp, #7, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    addvl sp, sp, #3
+; CHECK-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    ret
+entry:
+  %0 = tail call target("aarch64.svcount") @llvm.aarch64.sve.ptrue.c8()
+  %1 = tail call { <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } @llvm.aarch64.sve.ld1.pn.x2.nxv8bf16(target("aarch64.svcount") %0, ptr %ptr)
+  %2 = extractvalue { <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } %1, 0
+  %3 = extractvalue { <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } %1, 1
+  %arrayidx2 = getelementptr inbounds i8, ptr %ptr, i64 %stride
+  %4 = tail call { <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } @llvm.aarch64.sve.ld1.pn.x2.nxv8bf16(target("aarch64.svcount") %0, ptr %arrayidx2)
+  %5 = extractvalue { <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } %4, 0
+  %6 = extractvalue { <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } %4, 1
+  %res1 = call <vscale x 16 x i8> @llvm.aarch64.sve.fp8.cvt.x2.nxv8bf16(<vscale x 8 x bfloat> %2, <vscale x 8 x bfloat> %5)
+  %res2 = call <vscale x 16 x i8> @llvm.aarch64.sve.fp8.cvt.x2.nxv8bf16(<vscale x 8 x bfloat> %3, <vscale x 8 x bfloat> %6)
+  %ins1 = insertvalue { <vscale x 16 x i8>, <vscale x 16 x i8> } poison, <vscale x 16 x i8> %res1, 0
+  %ins2 = insertvalue { <vscale x 16 x i8>, <vscale x 16 x i8> } %ins1, <vscale x 16 x i8> %res2, 1
+  ret { <vscale x 16 x i8>, <vscale x 16 x i8> } %ins2
+}
+
 ; F1CVT / F2CVT
 
 define { <vscale x 8 x half>, <vscale x 8 x half> } @f1cvt(<vscale x 16 x i8> %zm) {
diff --git a/llvm/test/CodeGen/AArch64/sme2-intrinsics-add.ll b/llvm/test/CodeGen/AArch64/sme2-intrinsics-add.ll
index eee577ce48349..1546763931fce 100644
--- a/llvm/test/CodeGen/AArch64/sme2-intrinsics-add.ll
+++ b/llvm/test/CodeGen/AArch64/sme2-intrinsics-add.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sme2 -mattr=+sme-i16i64 -mattr=+sme-f64f64 -force-streaming -verify-machineinstrs < %s | FileCheck %s
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sme2 -mattr=+sme-i16i64 -mattr=+sme-f64f64 -force-streaming -enable-subreg-liveness -verify-machineinstrs < %s | FileCheck %s
 
 ;
 ; ADD Multi-Single x2
@@ -8,9 +8,7 @@
 define void @multi_vector_add_write_single_za_vg1x2_i32(i32 %slice, <vscale x 4 x i32> %zn0, <vscale x 4 x i32> %zn1,  <vscale x 4 x i32> %zm) {
 ; CHECK-LABEL: multi_vector_add_write_single_za_vg1x2_i32:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    // kill: def $z1 killed $z1 killed $z0_z1 def $z0_z1
 ; CHECK-NEXT:    mov w8, w0
-; CHECK-NEXT:    // kill: def $z0 killed $z0 killed $z0_z1 def $z0_z1
 ; CHECK-NEXT:    add za.s[w8, 0, vgx2], { z0.s, z1.s }, z2.s
 ; CHECK-NEXT:    add za.s[w8, 7, vgx2], { z0.s, z1.s }, z2.s
 ; CHECK-NEXT:    ret
@@ -27,9 +25,7 @@ define void @multi_vector_add_write_single_za_vg1x2_i32(i32 %slice, <vscale x 4
 define void @multi_vector_add_write_single_za_vg1x2_i64(i32 %slice, <vscale x 2 x i64> %zn0, <vscale x 2 x i64> %zn1,  <vscale x 2 x i64> %zm) {
 ; CHECK-LABEL: multi_vector_add_write_single_za_vg1x2_i64:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    // kill: def $z1 killed $z1 killed $z0_z1 def $z0_z1
 ; CHECK-NEXT:    mov w8, w0
-; CHECK-NEXT:    // kill: def $z0 killed $z0 killed $z0_z1 def $z0_z1
 ; CHECK-NEXT:    add za.d[w8, 0, vgx2], { z0.d, z1.d }, z2.d
 ; CHECK-NEXT:    add za.d[w8, 7, vgx2], { z0.d, z1.d }, z2.d
 ; CHECK-NEXT:    ret
@@ -50,11 +46,7 @@ define void @multi_vector_add_write_single_za_vg1x2_i64(i32 %slice, <vscale x 2
 define void @multi_vector_add_write_single_za_vg1x4_i32(i32 %slice, <vscale x 4 x i32> %zn0, <vscale x 4 x i32> %zn1,
 ; CHECK-LABEL: multi_vector_add_write_single_za_vg1x4_i32:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    // kill: def $z3 killed $z3 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
 ; CHECK-NEXT:    mov w8, w0
-; CHECK-NEXT:    // kill: def $z2 killed $z2 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
-; CHECK-NEXT:    // kill: def $z1 killed $z1 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
-; CHECK-NEXT:    // kill: def $z0 killed $z0 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
 ; CHECK-NEXT:    add za.s[w8, 0, vgx4], { z0.s - z3.s }, z4.s
 ; CHECK-NEXT:    add za.s[w8, 7, vgx4], { z0.s - z3.s }, z4.s
 ; CHECK-NEXT:    ret
@@ -75,11 +67,7 @@ define void @multi_vector_add_write_single_za_vg1x4_i32(i32 %slice, <vscale x 4
 define void @multi_vector_add_write_single_za_vg1x4_i64(i32 %slice,
 ; CHECK-LABEL: multi_vector_add_write_single_za_vg1x4_i64:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    // kill: def $z3 killed $z3 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
 ; CHECK-NEXT:    mov w8, w0
-; CHECK-NEXT:    // kill: def $z2 killed $z2 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
-; CHECK-NEXT:    // kill: def $z1 killed $z1 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
-; CHECK-NEXT:    // kill: def $z0 killed $z0 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
 ; CHECK-NEXT:    add za.d[w8, 0, vgx4], { z0.d - z3.d }, z4.d
 ; CHECK-NEXT:    add za.d[w8, 7, vgx4], { z0.d - z3.d }, z4.d
 ; CHECK-NEXT:    ret
@@ -105,11 +93,7 @@ define void @multi_vector_add_write_single_za_vg1x4_i64(i32 %slice,
 define void @multi_vector_add_write_za_vg1x2_i32(i32 %slice, <vscale x 4 x i32> %zn0, <vscale x 4 x i32> %zn1,
 ; CHECK-LABEL: multi_vector_add_write_za_vg1x2_i32:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    // kill: def $z3 killed $z3 killed $z2_z3 def $z2_z3
-; CHECK-NEXT:    // kill: def $z1 killed $z1 killed $z0_z1 def $z0_z1
 ; CHECK-NEXT:    mov w8, w0
-; CHECK-NEXT:    // kill: def $z2 killed $z2 killed $z2_z3 def $z2_z3
-; CHECK-NEXT:    // kill: def $z0 killed $z0 killed $z0_z1 def $z0_z1
 ; CHECK-NEXT:    add za.s[w8, 0, vgx2], { z0.s, z1.s }, { z2.s, z3.s }
 ; CHECK-NEXT:    add za.s[w8, 7, vgx2], { z0.s, z1.s }, { z2.s, z3.s }
 ; CHECK-NEXT:    ret
@@ -128,11 +112,7 @@ define void @multi_vector_add_write_za_vg1x2_i32(i32 %slice, <vscale x 4 x i32>
 define void @multi_vector_add_write_za_vg1x2_i64(i32 %slice, <vscale x 2 x i64> %zn0, <vscale x 2 x i64> %zn1,
 ; CHECK-LABEL: multi_vector_add_write_za_vg1x2_i64:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    // kill: def $z3 killed $z3 killed $z2_z3 def $z2_z3
-; CHECK-NEXT:    // kill: def $z1 killed $z1 killed $z0_z1 def $z0_z1
 ; CHECK-NEXT:    mov w8, w0
-; CHECK-NEXT:    // kill: def $z2 killed $z2 killed $z2_z3 def $z2_z3
-; CHECK-NEXT:    // kill: def $z0 killed $z0 killed $z0_z1 def $z0_z1
 ; CHECK-NEXT:    add za.d[w8, 0, vgx2], { z0.d, z1.d }, { z2.d, z3.d }
 ; CHECK-NEXT:    add za.d[w8, 7, vgx2], { z0.d, z1.d }, { z2.d, z3.d }
 ; CHECK-NEXT:    ret
@@ -155,15 +135,7 @@ define void @multi_vector_add_write_za_vg1x2_i64(i32 %slice, <vscale x 2 x i64>
 define void @multi_vector_add_write_za_vg1x4_i32(i32 %slice, <vscale x 4 x i32> %zn0, <vscale x 4 x i32> %zn1,
 ; CHECK-LABEL: multi_vector_add_write_za_vg1x4_i32:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    // kill: def $z7 killed $z7 killed $z4_z5_z6_z7 def $z4_z5_z6_z7
-; CHECK-NEXT:    // kill: def $z3 killed $z3 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
 ; CHECK-NEXT:    mov w8, w0
-; CHECK-NEXT:    // kill: def $z6 killed $z6 killed $z4_z5_z6_z7 def $z4_z5_z6_z7
-; CHECK-NEXT:    // kill: def $z2 killed $z2 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
-; CHECK-NEXT:    // kill: def $z5 killed $z5 killed $z4_z5_z6_z7 def $z4_z5_z6_z7
-; CHECK-NEXT:    // kill: def $z1 killed $z1 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
-; CHECK-NEXT:    // kill: def $z4 killed $z4 killed $z4_z5_z6_z7 def $z4_z5_z6_z7
-; CHECK-NEXT:    // kill: def $z0 killed $z0 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
 ; CHECK-NEXT:    add za.s[w8, 0, vgx4], { z0.s - z3.s }, { z4.s - z7.s }
 ; CHECK-NEXT:    add za.s[w8, 7, vgx4], { z0.s - z3.s }, { z4.s - z7.s }
 ; CHECK-NEXT:    ret
@@ -187,15 +159,7 @@ define void @multi_vector_add_write_za_vg1x4_i32(i32 %slice, <vscale x 4 x i32>
 define void @multi_vector_add_write_za_vg1x4_i64(i32 %slice, <vscale x 2 x i64> %zn0, <vscale x 2 x i64> %zn1,
 ; CHECK-LABEL: multi_vector_add_write_za_vg1x4_i64:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    // kill: def $z7 killed $z7 killed $z4_z5_z6_z7 def $z4_z5_z6_z7
-; CHECK-NEXT:    // kill: def $z3 killed $z3 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
 ; CHECK-NEXT:    mov w8, w0
-; CHECK-NEXT:    // kill: def $z6 killed $z6 killed $z4_z5_z6_z7 def $z4_z5_z6_z7
-; CHECK-NEXT:    // kill: def $z2 killed $z2 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
-; CHECK-NEXT:    // kill: def $z5 killed $z5 killed $z4_z5_z6_z7 def $z4_z5_z6_z7
-; CHECK-NEXT:    // kill: def $z1 killed $z1 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
-; CHECK-NEXT:    // kill: def $z4 killed $z4 killed $z4_z5_z6_z7 def $z4_z5_z6_z7
-; CHECK-NEXT:    // kill: def $z0 killed $z0 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
 ; CHECK-NEXT:    add za.d[w8, 0, vgx4], { z0.d - z3.d }, { z4.d - z7.d }
 ; CHECK-NEXT:    add za.d[w8, 7, vgx4], { z0.d - z3.d }, { z4.d - z7.d }
 ; CHECK-NEXT:    ret
@@ -223,9 +187,7 @@ define void @multi_vector_add_write_za_vg1x4_i64(i32 %slice, <vscale x 2 x i64>
 define void @multi_vector_add_za_vg1x2_i32(i32 %slice, <vscale x 4 x i32> %zn0, <vscale x 4 x i32> %zn1) {
 ; CHECK-LABEL: multi_vector_add_za_vg1x2_i32:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    // kill: def $z1 killed $z1 killed $z0_z1 def $z0_z1
 ; CHECK-NEXT:    mov w8, w0
-; CHECK-NEXT:    // kill: def $z0 killed $z0 killed $z0_z1 def $z0_z1
 ; CHECK-NEXT:    add za.s[w8, 0, vgx2], { z0.s, z1.s }
 ; CHECK-NEXT:    add za.s[w8, 7, vgx2], { z0.s, z1.s }
 ; CHECK-NEXT:    ret
@@ -238,9 +200,7 @@ define void @multi_vector_add_za_vg1x2_i32(i32 %slice, <vscale x 4 x i32> %zn0,
 define void @multi_vector_add_za_vg1x2_i64(i32 %slice, <vscale x 2 x i64> %zn0, <vscale x 2 x i64> %zn1) {
 ; CHECK-LABEL: multi_vector_add_za_vg1x2_i64:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    // kill: def $z1 killed $z1 killed $z0_z1 def $z0_z1
 ; CHECK-NEXT:    mov w8, w0
-; CHECK-NEXT:    // kill: def $z0 killed $z0 killed $z0_z1 def $z0_z1
 ; CHECK-NEXT:    add za.d[w8, 0, vgx2], { z0.d, z1.d }
 ; CHECK-NEXT:    add za.d[w8, 7, vgx2], { z0.d, z1.d }
 ; CHECK-NEXT:    ret
@@ -253,9 +213,7 @@ define void @multi_vector_add_za_vg1x2_i64(i32 %slice, <vscale x 2 x i64> %zn0,
 define void @multi_vector_add_za_vg1x2_f32(i32 %slice, <vscale x 4 x float> %zn0, <vscale x 4 x float> %zn1) {
 ; CHECK-LABEL: multi_vector_add_za_vg1x2_f32:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    // kill: def $z1 killed $z1 killed $z0_z1 def $z0_z1
 ; CHECK-NEXT:    mov w8, w0
-; CHECK-NEXT:    // kill: def $z0 killed $z0 killed $z0_z1 def $z0_z1
 ; CHECK-NEXT:    fadd za.s[w8, 0, vgx2], { z0.s, z1.s }
 ; CHECK-NEXT:    fadd za.s[w8, 7, vgx2], { z0.s, z1.s }
 ; CHECK-NEXT:    ret
@@ -270,9 +228,7 @@ define void @multi_vector_add_za_vg1x2_f32(i32 %slice, <vscale x 4 x float> %zn0
 define void @multi_vector_add_za_vg1x2_f64(i32 %slice, <vscale x 2 x double> %zn0, <vscale x 2 x double> %zn1) {
 ; CHECK-LABEL: multi_vector_add_za_vg1x2_f64:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    // kill: def $z1 killed $z1 killed $z0_z1 def $z0_z1
 ; CHECK-NEXT:    mov w8, w0
-; CHECK-NEXT:    // kill: def $z0 killed $z0 killed $z0_z1 def $z0_z1
 ; CHECK-NEXT:    fadd za.d[w8, 0, vgx2], { z0.d, z1.d }
 ; CHECK-NEXT:    fadd za.d[w8, 7, vgx2], { z0.d, z1.d }
 ; CHECK-NEXT:    ret
@@ -284,16 +240,37 @@ define void @multi_vector_add_za_vg1x2_f64(i32 %slice, <vscale x 2 x double> %zn
   ret void
 }
 
+define void @multi_vector_add_za_vg1x2_f64_tuple(i64 %stride, ptr %ptr) {
+; CHECK-LABEL: multi_vector_add_za_vg1x2_f64_tuple:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ptrue pn8.b
+; CHECK-NEXT:    add x9, x1, x0
+; CHECK-NEXT:    mov w8, wzr
+; CHECK-NEXT:    ld1d { z16.d, z24.d }, pn8/z, [x1]
+; CHECK-NEXT:    ld1d { z17.d, z25.d }, pn8/z, [x9]
+; CHECK-NEXT:    fadd za.d[w8, 0, vgx2], { z16.d, z17.d }
+; CHECK-NEXT:    fadd za.d[w8, 0, vgx2], { z24.d, z25.d }
+; CHECK-NEXT:    ret
+entry:
+  %0 = tail call target("aarch64.svcount") @llvm.aarch64.sve.ptrue.c8()
+  %1 = tail call { <vscale x 2 x double>, <vscale x 2 x double> } @llvm.aarch64.sve.ld1.pn.x2.nxv2f64(target("aarch64.svcount") %0, ptr %ptr)
+  %2 = extractvalue { <vscale x 2 x double>, <vscale x 2 x double> } %1, 0
+  %3 = extractvalue { <vscale x 2 x double>, <vscale x 2 x double> } %1, 1
+  %arrayidx2 = getelementptr inbounds i8, ptr %ptr, i64 %stride
+  %4 = tail call { <vscale x 2 x double>, <vscale x 2 x double> } @llvm.aarch64.sve.ld1.pn.x2.nxv2f64(target("aarch64.svcount") %0, ptr %arrayidx2)
+  %5 = extractvalue { <vscale x 2 x double>, <vscale x 2 x double> } %4, 0
+  %6 = extractvalue { <vscale x 2 x double>, <vscale x 2 x double> } %4, 1
+  call void @llvm.aarch64.sme.add.za64.vg1x2.nxv2f64(i32 0, <vscale x 2 x double> %2, <vscale x 2 x double> %5)
+  call void @llvm.aarch64.sme.add.za64.vg1x2.nxv2f64(i32 0, <vscale x 2 x double> %3, <vscale x 2 x double> %6)
+  ret void
+}
+
 ; x4
 
 define void @multi_vector_add_za_vg1x4_i32(i32 %slice, <vscale x 4 x i32> %zn0, <vscale x 4 x i32> %zn1, <vscale x 4 x i32> %zn2, <vscale x 4 x i32> %zn3) {
 ; CHECK-LABEL: multi_vector_add_za_vg1x4_i32:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    // kill: def $z3 killed $z3 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
 ; CHECK-NEXT:    mov w8, w0
-; CHECK-NEXT:    // kill: def $z2 killed $z2 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
-; CHECK-NEXT:    // kill: def $z1 killed $z1 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
-; CHECK-NEXT:    // kill: def $z0 killed $z0 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
 ; CHECK-NEXT:    add za.s[w8, 0, vgx4], { z0.s - z3.s }
 ; CHECK-NEXT:    add za.s[w8, 7, vgx4], { z0.s - z3.s }
 ; CHECK-NEXT:    ret
@@ -310,11 +287,7 @@ define void @multi_vector_add_za_vg1x4_i32(i32 %slice, <vscale x 4 x i32> %zn0,
 define void @multi_vector_add_za_vg1x4_i64(i32 %slice, <vscale x 2 x i64> %zn0, <vscale x 2 x i64> %zn1, <vscale x 2 x i64> %zn2, <vscale x 2 x i64> %zn3) {
 ; CHECK-LABEL: multi_vector_add_za_vg1x4_i64:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    // kill: def $z3 killed $z3 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
 ; CHECK-NEXT:    mov w8, w0
-; CHECK-NEXT:    // kill: def $z2 killed $z2 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
-; CHECK-NEXT:    // kill: def $z1 killed $z1 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
-; CHECK-NEXT:    // kill: def $z0 killed $z0 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
 ; CHECK-NEXT:    add za.d[w8, 0, vgx4], { z0.d - z3.d }
 ; CHECK-NEXT:    add za.d[w8, 7, vgx4], { z0.d - z3.d }
 ; CHECK-NEXT:    ret
@@ -331,11 +304,7 @@ define void @multi_vector_add_za_vg1x4_i64(i32 %slice, <vscale x 2 x i64> %zn0,
 define void @multi_vector_add_za_vg1x4_f32(i32 %slice, <vscale x 4 x float> %zn0, <vscale x 4 x float> %zn1, <vscale x 4 x float> %zn2, <vscale x 4 x float> %zn3) {
 ; CHECK-LABEL: multi_vector_add_za_vg1x4_f32:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    // kill: def $z3 killed $z3 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
 ; CHECK-NEXT:    mov w8, w0
-; CHECK-NEXT:    // kill: def $z2 killed $z2 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
-; CHECK-NEXT:    // kill: def $z1 killed $z1 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
-; CHECK-NEXT:    // kill: def $z0 killed $z0 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
 ; CHECK-NEXT:    fadd za.s[w8, 0, vgx4], { z0.s - z3.s }
 ; CHECK-NEXT:    fadd za.s[w8, 7, vgx4], { z0.s - z3.s }
 ; CHECK-NEXT:    ret
@@ -349,14 +318,62 @@ define void @multi_vector_add_za_vg1x4_f32(i32 %slice, <vscale x 4 x float> %zn0
   ret void
 }
 
+define void @multi_vector_add_za_vg1x4_f32_tuple(i64 %stride, ptr %ptr) {
+; CHECK-LABEL: multi_vector_add_za_vg1x4_f32_tuple:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    lsl x9, x0, #1
+; CHECK-NEXT:    add x10, x1, x0
+; CHECK-NEXT:    ptrue pn8.b
+; CHECK-NEXT:    ld1w { z16.s, z20.s, z24.s, z28.s }, pn8/z, [x1]
+; CHECK-NEXT:    ld1w { z17.s, z21.s, z25.s, z29.s }, pn8/z, [x10]
+; CHECK-NEXT:    mov w8, wzr
+; CHECK-NEXT:    add x11, x1, x9
+; CHECK-NEXT:    add x9, x10, x9
+; CHECK-NEXT:    ld1w { z18.s, z22.s, z26.s, z30.s }, pn8/z, [x11]
+; CHECK-NEXT:    ld1w { z19.s, z23.s, z27.s, z31.s }, pn8/z, [x9]
+; CHECK-NEXT:    fadd za.s[w8, 0, vgx4], { z16.s - z19.s }
+; CHECK-NEXT:    fadd za.s[w8, 0, vgx4], { z20.s - z23.s }
+; CHECK-NEXT:    fadd za.s[w8, 0, vgx4], { z24.s - z27.s }
+; CHECK-NEXT:    fadd za.s[w8, 0, vgx4], { z28.s - z31.s }
+; CHECK-NEXT:    ret
+entry:
+  %0 = tail call target("aarch64.svcount") @llvm.aarch64.sve.ptrue.c8()
+  %1 = tail call { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } @llvm.aarch64.sve.ld1.pn.x4.nxv4f32(target("aarch64.svcount") %0, ptr %ptr)
+  %2 = extractvalue { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } %1, 0
+  %3 = extractvalue { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } %1, 1
+  %4 = extractvalue { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } %1, 2
+  %5 = extractvalue { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } %1, 3
+  %arrayidx2 = getelementptr inbounds i8, ptr %ptr, i64 %stride
+  %6 = tail call { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } @llvm.aarch64.sve.ld1.pn.x4.nxv4f32(target("aarch64.svcount") %0, ptr %arrayidx2)
+  %7 = extractvalue { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } %6, 0
+  %8 = extractvalue { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } %6, 1
+  %9 = extractvalue { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } %6, 2
+  %10 = extractvalue { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } %6, 3
+  %mul3 = shl i64 %stride, 1
+  %arrayidx4 = getelementptr inbounds i8, ptr %ptr, i64 %mul3
+  %11 = tail call { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } @llvm.aarch64.sve.ld1.pn.x4.nxv4f32(target("aarch64.svcount") %0, ptr %arrayidx4)
+  %12 = extractvalue { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } %11, 0
+  %13 = extractvalue { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } %11, 1
+  %14 = extractvalue { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } %11, 2
+  %15 = extractvalue { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } %11, 3
+  %mul5 = mul i64 %stride, 3
+  %arrayidx6 = getelementptr inbounds i8, ptr %ptr, i64 %mul5
+  %16 = tail call { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } @llvm.aarch64.sve.ld1.pn.x4.nxv4f32(target("aarch64.svcount") %0, ptr %arrayidx6)
+  %17 = extractvalue { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } %16, 0
+  %18 = extractvalue { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } %16, 1
+  %19 = extractvalue { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } %16, 2
+  %20 = extractvalue { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } %16, 3
+  call void @llvm.aarch64.sme.add.za32.vg1x4.nxv4f32(i32 0, <vscale x 4 x float> %2, <vscale x 4 x float> %7, <vscale x 4 x float> %12, <vscale x 4 x float> %17)
+  call void @llvm.aarch64.sme.add.za32.vg1x4.nxv4f32(i32 0, <vscale x 4 x float> %3, <vscale x 4 x float> %8, <vscale x 4 x float> %13, <vscale x 4 x float> %18)
+  call void @llvm.aarch64.sme.add.za32.vg1x4.nxv4f32(i32 0, <vscale x 4 x float> %4, <vscale x 4 x float> %9, <vscale x 4 x float> %14, <vscale x 4 x float> %19)
+  call void @llvm.aarch64.sme.add.za32.vg1x4.nxv4f32(i32 0, <vscale x 4 x float> %5, <vscale x 4 x float> %10, <vscale x 4 x float> %15, <vscale x 4 x float> %20)
+  ret void
+}
+
 define void @multi_vector_add_za_vg1x4_f64(i32 %slice, <vscale x 2 x double> %zn0, <vscale x 2 x double> %zn1, <vscale x 2 x double> %zn2, <vscale x 2 x double> %zn3) {
 ; CHECK-LABEL: multi_vector_add_za_vg1x4_f64:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    // kill: def $z3 killed $z3 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
 ; CHECK-NEXT:    mov w8, w0
-; CHECK-NEXT:    // kill: def $z2 killed $z2 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
-; CHECK-NEXT:    // kill: def $z1 killed $z1 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
-; CHECK-NEXT:    // kill: def $z0 killed $z0 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
 ; CHECK-NEXT:    fadd za.d[w8, 0, vgx4], { z0.d - z3.d }
 ; CHECK-NEXT:    fadd za.d[w8, 7, vgx4], { z0.d - z3.d }
 ; CHECK-NEXT:    ret
diff --git a/llvm/test/CodeGen/AArch64/sme2-intrinsics-fp-dots.ll b/llvm/test/CodeGen/AArch64/sme2-intrinsics-fp-dots.ll
index ca149f41dbb83..416b848648756 100644
--- a/llvm/test/CodeGen/AArch64/sme2-intrinsics-fp-dots.ll
+++ b/llvm/test/CodeGen/AArch64/sme2-intrinsics-fp-dots.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -force-streaming -verify-machineinstrs < %s | FileCheck %s
+; RUN: llc -force-streaming -enable-subreg-liveness -verify-machineinstrs < %s | FileCheck %s
 
 target triple="aarch64-linux-gnu"
 
@@ -23,21 +23,44 @@ define void @fdot_multi_za32_f16_vg1x2(i32 %slice, <vscale x 16 x i8> %unused, <
   ret void
 }
 
+define void @fdot_multi_za32_f16_vg1x2_tuple(i64 %stride, ptr %ptr) #0 {
+; CHECK-LABEL: fdot_multi_za32_f16_vg1x2_tuple:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ptrue pn8.b
+; CHECK-NEXT:    add x9, x1, x0
+; CHECK-NEXT:    mov w8, wzr
+; CHECK-NEXT:    ld1h { z16.h, z24.h }, pn8/z, [x1]
+; CHECK-NEXT:    ld1h { z17.h, z25.h }, pn8/z, [x9]
+; CHECK-NEXT:    fdot za.s[w8, 0, vgx2], { z16.h, z17.h }, { z24.h, z25.h }
+; CHECK-NEXT:    ret
+entry:
+  %0 = tail call target("aarch64.svcount") @llvm.aarch64.sve.ptrue.c8()
+  %1 = tail call { <vscale x 8 x half>, <vscale x 8 x half> } @llvm.aarch64.sve.ld1.pn.x2.nxv8f16(target("aarch64.svcount") %0, ptr %ptr)
+  %2 = extractvalue { <vscale x 8 x half>, <vscale x 8 x half> } %1, 0
+  %3 = extractvalue { <vscale x 8 x half>, <vscale x 8 x half> } %1, 1
+  %arrayidx2 = getelementptr inbounds i8, ptr %ptr, i64 %stride
+  %4 = tail call { <vscale x 8 x half>, <vscale x 8 x half> } @llvm.aarch64.sve.ld1.pn.x2.nxv8f16(target("aarch64.svcount") %0, ptr %arrayidx2)
+  %5 = extractvalue { <vscale x 8 x half>, <vscale x 8 x half> } %4, 0
+  %6 = extractvalue { <vscale x 8 x half>, <vscale x 8 x half> } %4, 1
+  call void @llvm.aarch64.sme.fdot.za32.vg1x2.nxv8f16(i32 0, <vscale x 8 x half> %2, <vscale x 8 x half> %5, <vscale x 8 x half> %3, <vscale x 8 x half> %6)
+  ret void
+}
+
 define void @fdot_multi_za32_f16_vg1x4(i32 %slice, <vscale x 16 x i8> %unused, <vscale x 8 x half> %zn0, <vscale x 8 x half> %zn1, <vscale x 8 x half> %zn2, <vscale x 8 x half> %zn3,
 ; CHECK-LABEL: fdot_multi_za32_f16_vg1x4:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    mov z26.d, z7.d
-; CHECK-NEXT:    mov z31.d, z4.d
-; CHECK-NEXT:    mov w8, w0
 ; CHECK-NEXT:    ptrue p0.h
+; CHECK-NEXT:    mov z26.d, z7.d
 ; CHECK-NEXT:    mov z25.d, z6.d
-; CHECK-NEXT:    mov z30.d, z3.d
+; CHECK-NEXT:    mov z7.d, z4.d
+; CHECK-NEXT:    mov w8, w0
 ; CHECK-NEXT:    mov z24.d, z5.d
-; CHECK-NEXT:    mov z29.d, z2.d
 ; CHECK-NEXT:    ld1h { z27.h }, p0/z, [x1]
-; CHECK-NEXT:    mov z28.d, z1.d
-; CHECK-NEXT:    fdot za.s[w8, 0, vgx4], { z28.h - z31.h }, { z24.h - z27.h }
-; CHECK-NEXT:    fdot za.s[w8, 7, vgx4], { z28.h - z31.h }, { z24.h - z27.h }
+; CHECK-NEXT:    mov z6.d, z3.d
+; CHECK-NEXT:    mov z5.d, z2.d
+; CHECK-NEXT:    mov z4.d, z1.d
+; CHECK-NEXT:    fdot za.s[w8, 0, vgx4], { z4.h - z7.h }, { z24.h - z27.h }
+; CHECK-NEXT:    fdot za.s[w8, 7, vgx4], { z4.h - z7.h }, { z24.h - z27.h }
 ; CHECK-NEXT:    ret
                                         <vscale x 8 x half> %zn4, <vscale x 8 x half> %zn5, <vscale x 8 x half> %zn6, <vscale x 8 x half> %zn7) #0 {
   call void @llvm.aarch64.sme.fdot.za32.vg1x4.nxv8f16(i32 %slice, <vscale x 8 x half> %zn0, <vscale x 8 x half> %zn1, <vscale x 8 x half> %zn2, <vscale x 8 x half> %zn3,
@@ -48,6 +71,54 @@ define void @fdot_multi_za32_f16_vg1x4(i32 %slice, <vscale x 16 x i8> %unused, <
   ret void
 }
 
+define void @fdot_multi_za32_f16_vg1x4_tuple(i64 %stride, ptr %ptr) #0 {
+; CHECK-LABEL: fdot_multi_za32_f16_vg1x4_tuple:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    add x9, x0, x0, lsl #1
+; CHECK-NEXT:    ptrue pn8.b
+; CHECK-NEXT:    add x10, x1, x0
+; CHECK-NEXT:    ld1h { z16.h, z20.h, z24.h, z28.h }, pn8/z, [x1]
+; CHECK-NEXT:    ld1h { z17.h, z21.h, z25.h, z29.h }, pn8/z, [x10]
+; CHECK-NEXT:    ld1h { z18.h, z22.h, z26.h, z30.h }, pn8/z, [x1, x0, lsl #1]
+; CHECK-NEXT:    add x9, x1, x9
+; CHECK-NEXT:    mov w8, wzr
+; CHECK-NEXT:    ld1h { z19.h, z23.h, z27.h, z31.h }, pn8/z, [x9]
+; CHECK-NEXT:    fdot za.s[w8, 0, vgx4], { z16.h - z19.h }, { z20.h - z23.h }
+; CHECK-NEXT:    fdot za.s[w8, 0, vgx4], { z24.h - z27.h }, { z28.h - z31.h }
+; CHECK-NEXT:    ret
+entry:
+  %0 = tail call target("aarch64.svcount") @llvm.aarch64.sve.ptrue.c8()
+  %1 = tail call { <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half> } @llvm.aarch64.sve.ld1.pn.x4.nxv8f16(target("aarch64.svcount") %0, ptr %ptr)
+  %2 = extractvalue { <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half> } %1, 0
+  %3 = extractvalue { <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half> } %1, 1
+  %4 = extractvalue { <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half> } %1, 2
+  %5 = extractvalue { <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half> } %1, 3
+  %arrayidx2 = getelementptr inbounds i8, ptr %ptr, i64 %stride
+  %6 = tail call { <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half> } @llvm.aarch64.sve.ld1.pn.x4.nxv8f16(target("aarch64.svcount") %0, ptr %arrayidx2)
+  %7 = extractvalue { <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half> } %6, 0
+  %8 = extractvalue { <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half> } %6, 1
+  %9 = extractvalue { <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half> } %6, 2
+  %10 = extractvalue { <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half> } %6, 3
+  %mul3 = shl i64 %stride, 1
+  %arrayidx4 = getelementptr inbounds i8, ptr %ptr, i64 %mul3
+  %11 = tail call { <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half> } @llvm.aarch64.sve.ld1.pn.x4.nxv8f16(target("aarch64.svcount") %0, ptr %arrayidx4)
+  %12 = extractvalue { <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half> } %11, 0
+  %13 = extractvalue { <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half> } %11, 1
+  %14 = extractvalue { <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half> } %11, 2
+  %15 = extractvalue { <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half> } %11, 3
+  %mul5 = mul i64 %stride, 3
+  %arrayidx6 = getelementptr inbounds i8, ptr %ptr, i64 %mul5
+  %16 = tail call { <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half> } @llvm.aarch64.sve.ld1.pn.x4.nxv8f16(target("aarch64.svcount") %0, ptr %arrayidx6)
+  %17 = extractvalue { <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half> } %16, 0
+  %18 = extractvalue { <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half> } %16, 1
+  %19 = extractvalue { <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half> } %16, 2
+  %20 = extractvalue { <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half> } %16, 3
+  call void @llvm.aarch64.sme.fdot.za32.vg1x4.nxv8f16(i32 0, <vscale x 8 x half> %2, <vscale x 8 x half> %7, <vscale x 8 x half> %12, <vscale x 8 x half> %17,
+                                                      <vscale x 8 x half> %3, <vscale x 8 x half> %8, <vscale x 8 x half> %13, <vscale x 8 x half> %18)
+  call void @llvm.aarch64.sme.fdot.za32.vg1x4.nxv8f16(i32 0, <vscale x 8 x half> %4, <vscale x 8 x half> %9, <vscale x 8 x half> %14, <vscale x 8 x half> %19,
+                                                      <vscale x 8 x half> %5, <vscale x 8 x half> %10, <vscale x 8 x half> %15, <vscale x 8 x half> %20)
+  ret void
+}
 
 ; == Multi, multi (16-bit bfloat) ==
 
@@ -71,18 +142,18 @@ define void @bfdot_multi_za32_bf16_vg1x2(i32 %slice, <vscale x 16 x i8> %unused,
 define void @fdot_multi_za32_bf16_vg1x4(i32 %slice, <vscale x 16 x i8> %unused, <vscale x 8 x bfloat> %zn0, <vscale x 8 x bfloat> %zn1, <vscale x 8 x bfloat> %zn2, <vscale x 8 x bfloat> %zn3,
 ; CHECK-LABEL: fdot_multi_za32_bf16_vg1x4:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    mov z26.d, z7.d
-; CHECK-NEXT:    mov z31.d, z4.d
-; CHECK-NEXT:    mov w8, w0
 ; CHECK-NEXT:    ptrue p0.h
+; CHECK-NEXT:    mov z26.d, z7.d
 ; CHECK-NEXT:    mov z25.d, z6.d
-; CHECK-NEXT:    mov z30.d, z3.d
+; CHECK-NEXT:    mov z7.d, z4.d
+; CHECK-NEXT:    mov w8, w0
 ; CHECK-NEXT:    mov z24.d, z5.d
-; CHECK-NEXT:    mov z29.d, z2.d
 ; CHECK-NEXT:    ld1h { z27.h }, p0/z, [x1]
-; CHECK-NEXT:    mov z28.d, z1.d
-; CHECK-NEXT:    bfdot za.s[w8, 0, vgx4], { z28.h - z31.h }, { z24.h - z27.h }
-; CHECK-NEXT:    bfdot za.s[w8, 7, vgx4], { z28.h - z31.h }, { z24.h - z27.h }
+; CHECK-NEXT:    mov z6.d, z3.d
+; CHECK-NEXT:    mov z5.d, z2.d
+; CHECK-NEXT:    mov z4.d, z1.d
+; CHECK-NEXT:    bfdot za.s[w8, 0, vgx4], { z4.h - z7.h }, { z24.h - z27.h }
+; CHECK-NEXT:    bfdot za.s[w8, 7, vgx4], { z4.h - z7.h }, { z24.h - z27.h }
 ; CHECK-NEXT:    ret
                                         <vscale x 8 x bfloat> %zn4, <vscale x 8 x bfloat> %zn5, <vscale x 8 x bfloat> %zn6, <vscale x 8 x bfloat> %zn7) #0 {
   call void @llvm.aarch64.sme.fdot.za32.vg1x4.nxv8bf16(i32 %slice, <vscale x 8 x bfloat> %zn0, <vscale x 8 x bfloat> %zn1, <vscale x 8 x bfloat> %zn2, <vscale x 8 x bfloat> %zn3,
@@ -99,9 +170,7 @@ define void @fdot_multi_za32_bf16_vg1x4(i32 %slice, <vscale x 16 x i8> %unused,
 define void @fdot_single_za32_f16_vg1x2(i32 %slice, <vscale x 16 x i8> %unused, <vscale x 8 x half> %zn0, <vscale x 8 x half> %zn1, <vscale x 8 x half> %zn2) #0 {
 ; CHECK-LABEL: fdot_single_za32_f16_vg1x2:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    // kill: def $z2 killed $z2 killed $z1_z2 def $z1_z2
 ; CHECK-NEXT:    mov w8, w0
-; CHECK-NEXT:    // kill: def $z1 killed $z1 killed $z1_z2 def $z1_z2
 ; CHECK-NEXT:    fdot za.s[w8, 0, vgx2], { z1.h, z2.h }, z3.h
 ; CHECK-NEXT:    fdot za.s[w8, 7, vgx2], { z1.h, z2.h }, z3.h
 ; CHECK-NEXT:    ret
@@ -114,11 +183,7 @@ define void @fdot_single_za32_f16_vg1x2(i32 %slice, <vscale x 16 x i8> %unused,
 define void @fdot_single_za32_f16_vg1x4(i32 %slice, <vscale x 16 x i8> %unused, <vscale x 8 x half> %zn0, <vscale x 8 x half> %zn1, <vscale x 8 x half> %zn2, <vscale x 8 x half> %zn3, <vscale x 8 x half> %zn4) #0 {
 ; CHECK-LABEL: fdot_single_za32_f16_vg1x4:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    // kill: def $z4 killed $z4 killed $z1_z2_z3_z4 def $z1_z2_z3_z4
 ; CHECK-NEXT:    mov w8, w0
-; CHECK-NEXT:    // kill: def $z3 killed $z3 killed $z1_z2_z3_z4 def $z1_z2_z3_z4
-; CHECK-NEXT:    // kill: def $z2 killed $z2 killed $z1_z2_z3_z4 def $z1_z2_z3_z4
-; CHECK-NEXT:    // kill: def $z1 killed $z1 killed $z1_z2_z3_z4 def $z1_z2_z3_z4
 ; CHECK-NEXT:    fdot za.s[w8, 0, vgx4], { z1.h - z4.h }, z5.h
 ; CHECK-NEXT:    fdot za.s[w8, 7, vgx4], { z1.h - z4.h }, z5.h
 ; CHECK-NEXT:    ret
@@ -134,9 +199,7 @@ define void @fdot_single_za32_f16_vg1x4(i32 %slice, <vscale x 16 x i8> %unused,
 define void @bfdot_single_za32_bf16_vg1x2(i32 %slice, <vscale x 16 x i8> %unused, <vscale x 8 x bfloat> %zn0, <vscale x 8 x bfloat> %zn1, <vscale x 8 x bfloat> %zn2) #0 {
 ; CHECK-LABEL: bfdot_single_za32_bf16_vg1x2:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    // kill: def $z2 killed $z2 killed $z1_z2 def $z1_z2
 ; CHECK-NEXT:    mov w8, w0
-; CHECK-NEXT:    // kill: def $z1 killed $z1 killed $z1_z2 def $z1_z2
 ; CHECK-NEXT:    bfdot za.s[w8, 0, vgx2], { z1.h, z2.h }, z3.h
 ; CHECK-NEXT:    bfdot za.s[w8, 7, vgx2], { z1.h, z2.h }, z3.h
 ; CHECK-NEXT:    ret
@@ -149,11 +212,7 @@ define void @bfdot_single_za32_bf16_vg1x2(i32 %slice, <vscale x 16 x i8> %unused
 define void @bfdot_single_za32_bf16_vg1x4(i32 %slice, <vscale x 16 x i8> %unused, <vscale x 8 x bfloat> %zn0, <vscale x 8 x bfloat> %zn1, <vscale x 8 x bfloat> %zn2, <vscale x 8 x bfloat> %zn3, <vscale x 8 x bfloat> %zn4) #0 {
 ; CHECK-LABEL: bfdot_single_za32_bf16_vg1x4:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    // kill: def $z4 killed $z4 killed $z1_z2_z3_z4 def $z1_z2_z3_z4
 ; CHECK-NEXT:    mov w8, w0
-; CHECK-NEXT:    // kill: def $z3 killed $z3 killed $z1_z2_z3_z4 def $z1_z2_z3_z4
-; CHECK-NEXT:    // kill: def $z2 killed $z2 killed $z1_z2_z3_z4 def $z1_z2_z3_z4
-; CHECK-NEXT:    // kill: def $z1 killed $z1 killed $z1_z2_z3_z4 def $z1_z2_z3_z4
 ; CHECK-NEXT:    bfdot za.s[w8, 0, vgx4], { z1.h - z4.h }, z5.h
 ; CHECK-NEXT:    bfdot za.s[w8, 7, vgx4], { z1.h - z4.h }, z5.h
 ; CHECK-NEXT:    ret
@@ -170,8 +229,8 @@ define void @fdot_lane_za32_f16_vg1x2(i32 %slice, <vscale x 16 x i8> %unused, <v
 ; CHECK-LABEL: fdot_lane_za32_f16_vg1x2:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    mov z5.d, z2.d
-; CHECK-NEXT:    mov w8, w0
 ; CHECK-NEXT:    mov z4.d, z1.d
+; CHECK-NEXT:    mov w8, w0
 ; CHECK-NEXT:    fdot za.s[w8, 0, vgx2], { z4.h, z5.h }, z3.h[3]
 ; CHECK-NEXT:    fdot za.s[w8, 7, vgx2], { z4.h, z5.h }, z3.h[3]
 ; CHECK-NEXT:    ret
@@ -185,8 +244,8 @@ define void @fdot_lane_za32_f16_vg1x4(i32 %slice, <vscale x 16 x i8> %unused, <v
 ; CHECK-LABEL: fdot_lane_za32_f16_vg1x4:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    mov z27.d, z4.d
-; CHECK-NEXT:    mov w8, w0
 ; CHECK-NEXT:    mov z26.d, z3.d
+; CHECK-NEXT:    mov w8, w0
 ; CHECK-NEXT:    mov z25.d, z2.d
 ; CHECK-NEXT:    mov z24.d, z1.d
 ; CHECK-NEXT:    fdot za.s[w8, 0, vgx4], { z24.h - z27.h }, z5.h[3]
@@ -207,8 +266,8 @@ define void @bfdot_lane_za32_bf16_vg1x2(i32 %slice, <vscale x 16 x i8> %unused,
 ; CHECK-LABEL: bfdot_lane_za32_bf16_vg1x2:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    mov z5.d, z2.d
-; CHECK-NEXT:    mov w8, w0
 ; CHECK-NEXT:    mov z4.d, z1.d
+; CHECK-NEXT:    mov w8, w0
 ; CHECK-NEXT:    bfdot za.s[w8, 0, vgx2], { z4.h, z5.h }, z3.h[3]
 ; CHECK-NEXT:    bfdot za.s[w8, 7, vgx2], { z4.h, z5.h }, z3.h[3]
 ; CHECK-NEXT:    ret
@@ -222,8 +281,8 @@ define void @bfdot_lane_za32_bf16_vg1x4(i32 %slice, <vscale x 16 x i8> %unused,
 ; CHECK-LABEL: bfdot_lane_za32_bf16_vg1x4:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    mov z27.d, z4.d
-; CHECK-NEXT:    mov w8, w0
 ; CHECK-NEXT:    mov z26.d, z3.d
+; CHECK-NEXT:    mov w8, w0
 ; CHECK-NEXT:    mov z25.d, z2.d
 ; CHECK-NEXT:    mov z24.d, z1.d
 ; CHECK-NEXT:    bfdot za.s[w8, 0, vgx4], { z24.h - z27.h }, z5.h[3]
diff --git a/llvm/test/CodeGen/AArch64/sme2-intrinsics-insert-mova.ll b/llvm/test/CodeGen/AArch64/sme2-intrinsics-insert-mova.ll
index c7a2ec16e3f7f..2e4a5f082311e 100644
--- a/llvm/test/CodeGen/AArch64/sme2-intrinsics-insert-mova.ll
+++ b/llvm/test/CodeGen/AArch64/sme2-intrinsics-insert-mova.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sme2 -force-streaming -verify-machineinstrs < %s | FileCheck %s
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sme2 -force-streaming -enable-subreg-liveness -verify-machineinstrs < %s | FileCheck %s
 
 ;
 ; Move Multi-Vector To Tile (Write) x 2
@@ -10,9 +10,7 @@
 define void @za_write_vg2_horiz_b(i32 %slice, <vscale x 16 x i8> %zn1, <vscale x 16 x i8> %zn2) {
 ; CHECK-LABEL: za_write_vg2_horiz_b:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    // kill: def $z1 killed $z1 killed $z0_z1 def $z0_z1
 ; CHECK-NEXT:    mov w12, w0
-; CHECK-NEXT:    // kill: def $z0 killed $z0 killed $z0_z1 def $z0_z1
 ; CHECK-NEXT:    mov za0h.b[w12, 0:1], { z0.b, z1.b }
 ; CHECK-NEXT:    mov za0h.b[w12, 14:15], { z0.b, z1.b }
 ; CHECK-NEXT:    ret
@@ -22,12 +20,34 @@ define void @za_write_vg2_horiz_b(i32 %slice, <vscale x 16 x i8> %zn1, <vscale x
   ret void
 }
 
+define void @za_write_vg2_horiz_b_tuple(i64 %stride, ptr %ptr) {
+; CHECK-LABEL: za_write_vg2_horiz_b_tuple:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ptrue pn8.b
+; CHECK-NEXT:    mov w12, wzr
+; CHECK-NEXT:    ld1b { z16.b, z24.b }, pn8/z, [x1]
+; CHECK-NEXT:    ld1b { z17.b, z25.b }, pn8/z, [x1, x0]
+; CHECK-NEXT:    mov za0h.b[w12, 0:1], { z16.b, z17.b }
+; CHECK-NEXT:    mov za0h.b[w12, 0:1], { z24.b, z25.b }
+; CHECK-NEXT:    ret
+entry:
+  %0 = tail call target("aarch64.svcount") @llvm.aarch64.sve.ptrue.c8()
+  %1 = tail call { <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.ld1.pn.x2.nxv16i8(target("aarch64.svcount") %0, ptr %ptr)
+  %2 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8> } %1, 0
+  %3 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8> } %1, 1
+  %arrayidx2 = getelementptr inbounds i8, ptr %ptr, i64 %stride
+  %4 = tail call { <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.ld1.pn.x2.nxv16i8(target("aarch64.svcount") %0, ptr %arrayidx2)
+  %5 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8> } %4, 0
+  %6 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8> } %4, 1
+  call void @llvm.aarch64.sme.write.hor.vg2.nxv16i8(i32 0, i32 0, <vscale x 16 x i8> %2, <vscale x 16 x i8> %5)
+  call void @llvm.aarch64.sme.write.hor.vg2.nxv16i8(i32 0, i32 0, <vscale x 16 x i8> %3, <vscale x 16 x i8> %6)
+  ret void
+}
+
 define void @za_write_vg2_horiz_h(i32 %slice, <vscale x 8 x i16> %zn1, <vscale x 8 x i16> %zn2) {
 ; CHECK-LABEL: za_write_vg2_horiz_h:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    // kill: def $z1 killed $z1 killed $z0_z1 def $z0_z1
 ; CHECK-NEXT:    mov w12, w0
-; CHECK-NEXT:    // kill: def $z0 killed $z0 killed $z0_z1 def $z0_z1
 ; CHECK-NEXT:    mov za0h.h[w12, 0:1], { z0.h, z1.h }
 ; CHECK-NEXT:    mov za1h.h[w12, 6:7], { z0.h, z1.h }
 ; CHECK-NEXT:    ret
@@ -40,9 +60,7 @@ define void @za_write_vg2_horiz_h(i32 %slice, <vscale x 8 x i16> %zn1, <vscale x
 define void @za_write_vg2_horiz_f16(i32 %slice, <vscale x 8 x half> %zn1, <vscale x 8 x half> %zn2) {
 ; CHECK-LABEL: za_write_vg2_horiz_f16:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    // kill: def $z1 killed $z1 killed $z0_z1 def $z0_z1
 ; CHECK-NEXT:    mov w12, w0
-; CHECK-NEXT:    // kill: def $z0 killed $z0 killed $z0_z1 def $z0_z1
 ; CHECK-NEXT:    mov za0h.h[w12, 0:1], { z0.h, z1.h }
 ; CHECK-NEXT:    mov za1h.h[w12, 6:7], { z0.h, z1.h }
 ; CHECK-NEXT:    ret
@@ -55,9 +73,7 @@ define void @za_write_vg2_horiz_f16(i32 %slice, <vscale x 8 x half> %zn1, <vscal
 define void @za_write_vg2_horiz_bf16(i32 %slice, <vscale x 8 x bfloat> %zn1, <vscale x 8 x bfloat> %zn2) {
 ; CHECK-LABEL: za_write_vg2_horiz_bf16:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    // kill: def $z1 killed $z1 killed $z0_z1 def $z0_z1
 ; CHECK-NEXT:    mov w12, w0
-; CHECK-NEXT:    // kill: def $z0 killed $z0 killed $z0_z1 def $z0_z1
 ; CHECK-NEXT:    mov za0h.h[w12, 0:1], { z0.h, z1.h }
 ; CHECK-NEXT:    mov za1h.h[w12, 6:7], { z0.h, z1.h }
 ; CHECK-NEXT:    ret
@@ -70,9 +86,7 @@ define void @za_write_vg2_horiz_bf16(i32 %slice, <vscale x 8 x bfloat> %zn1, <vs
 define void @za_write_vg2_horiz_s(i32 %slice, <vscale x 4 x i32> %zn1, <vscale x 4 x i32> %zn2) {
 ; CHECK-LABEL: za_write_vg2_horiz_s:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    // kill: def $z1 killed $z1 killed $z0_z1 def $z0_z1
 ; CHECK-NEXT:    mov w12, w0
-; CHECK-NEXT:    // kill: def $z0 killed $z0 killed $z0_z1 def $z0_z1
 ; CHECK-NEXT:    mov za0h.s[w12, 0:1], { z0.s, z1.s }
 ; CHECK-NEXT:    mov za3h.s[w12, 2:3], { z0.s, z1.s }
 ; CHECK-NEXT:    ret
@@ -85,9 +99,7 @@ define void @za_write_vg2_horiz_s(i32 %slice, <vscale x 4 x i32> %zn1, <vscale x
 define void @za_write_vg2_horiz_f32(i32 %slice, <vscale x 4 x float> %zn1, <vscale x 4 x float> %zn2) {
 ; CHECK-LABEL: za_write_vg2_horiz_f32:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    // kill: def $z1 killed $z1 killed $z0_z1 def $z0_z1
 ; CHECK-NEXT:    mov w12, w0
-; CHECK-NEXT:    // kill: def $z0 killed $z0 killed $z0_z1 def $z0_z1
 ; CHECK-NEXT:    mov za0h.s[w12, 0:1], { z0.s, z1.s }
 ; CHECK-NEXT:    mov za3h.s[w12, 2:3], { z0.s, z1.s }
 ; CHECK-NEXT:    ret
@@ -100,9 +112,7 @@ define void @za_write_vg2_horiz_f32(i32 %slice, <vscale x 4 x float> %zn1, <vsca
 define void @za_write_vg2_horiz_d(i32 %slice, <vscale x 2 x i64> %zn1, <vscale x 2 x i64> %zn2) {
 ; CHECK-LABEL: za_write_vg2_horiz_d:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    // kill: def $z1 killed $z1 killed $z0_z1 def $z0_z1
 ; CHECK-NEXT:    mov w12, w0
-; CHECK-NEXT:    // kill: def $z0 killed $z0 killed $z0_z1 def $z0_z1
 ; CHECK-NEXT:    mov za0h.d[w12, 0:1], { z0.d, z1.d }
 ; CHECK-NEXT:    ret
   call void @llvm.aarch64.sme.write.hor.vg2.nxv2i64(i32 0, i32 %slice, <vscale x 2 x i64> %zn1, <vscale x 2 x i64> %zn2)
@@ -112,9 +122,7 @@ define void @za_write_vg2_horiz_d(i32 %slice, <vscale x 2 x i64> %zn1, <vscale x
 define void @za_write_vg2_horiz_f64(i32 %slice, <vscale x 2 x double> %zn1, <vscale x 2 x double> %zn2) {
 ; CHECK-LABEL: za_write_vg2_horiz_f64:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    // kill: def $z1 killed $z1 killed $z0_z1 def $z0_z1
 ; CHECK-NEXT:    mov w12, w0
-; CHECK-NEXT:    // kill: def $z0 killed $z0 killed $z0_z1 def $z0_z1
 ; CHECK-NEXT:    mov za0h.d[w12, 0:1], { z0.d, z1.d }
 ; CHECK-NEXT:    ret
   call void @llvm.aarch64.sme.write.hor.vg2.nxv2f64(i32 0, i32 %slice, <vscale x 2 x double> %zn1, <vscale x 2 x double> %zn2)
@@ -126,9 +134,7 @@ define void @za_write_vg2_horiz_f64(i32 %slice, <vscale x 2 x double> %zn1, <vsc
 define void @za_write_vg2_vert_b(i32 %slice, <vscale x 16 x i8> %zn1, <vscale x 16 x i8> %zn2) {
 ; CHECK-LABEL: za_write_vg2_vert_b:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    // kill: def $z1 killed $z1 killed $z0_z1 def $z0_z1
 ; CHECK-NEXT:    mov w12, w0
-; CHECK-NEXT:    // kill: def $z0 killed $z0 killed $z0_z1 def $z0_z1
 ; CHECK-NEXT:    mov za0v.b[w12, 0:1], { z0.b, z1.b }
 ; CHECK-NEXT:    mov za0v.b[w12, 14:15], { z0.b, z1.b }
 ; CHECK-NEXT:    ret
@@ -141,9 +147,7 @@ define void @za_write_vg2_vert_b(i32 %slice, <vscale x 16 x i8> %zn1, <vscale x
 define void @za_write_vg2_vert_h(i32 %slice, <vscale x 8 x i16> %zn1, <vscale x 8 x i16> %zn2) {
 ; CHECK-LABEL: za_write_vg2_vert_h:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    // kill: def $z1 killed $z1 killed $z0_z1 def $z0_z1
 ; CHECK-NEXT:    mov w12, w0
-; CHECK-NEXT:    // kill: def $z0 killed $z0 killed $z0_z1 def $z0_z1
 ; CHECK-NEXT:    mov za0v.h[w12, 0:1], { z0.h, z1.h }
 ; CHECK-NEXT:    mov za1v.h[w12, 6:7], { z0.h, z1.h }
 ; CHECK-NEXT:    ret
@@ -156,9 +160,7 @@ define void @za_write_vg2_vert_h(i32 %slice, <vscale x 8 x i16> %zn1, <vscale x
 define void @za_write_vg2_vert_f16(i32 %slice, <vscale x 8 x half> %zn1, <vscale x 8 x half> %zn2) {
 ; CHECK-LABEL: za_write_vg2_vert_f16:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    // kill: def $z1 killed $z1 killed $z0_z1 def $z0_z1
 ; CHECK-NEXT:    mov w12, w0
-; CHECK-NEXT:    // kill: def $z0 killed $z0 killed $z0_z1 def $z0_z1
 ; CHECK-NEXT:    mov za0v.h[w12, 0:1], { z0.h, z1.h }
 ; CHECK-NEXT:    mov za1v.h[w12, 6:7], { z0.h, z1.h }
 ; CHECK-NEXT:    ret
@@ -171,9 +173,7 @@ define void @za_write_vg2_vert_f16(i32 %slice, <vscale x 8 x half> %zn1, <vscale
 define void @za_write_vg2_vert_bf16(i32 %slice, <vscale x 8 x bfloat> %zn1, <vscale x 8 x bfloat> %zn2) {
 ; CHECK-LABEL: za_write_vg2_vert_bf16:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    // kill: def $z1 killed $z1 killed $z0_z1 def $z0_z1
 ; CHECK-NEXT:    mov w12, w0
-; CHECK-NEXT:    // kill: def $z0 killed $z0 killed $z0_z1 def $z0_z1
 ; CHECK-NEXT:    mov za0v.h[w12, 0:1], { z0.h, z1.h }
 ; CHECK-NEXT:    mov za1v.h[w12, 6:7], { z0.h, z1.h }
 ; CHECK-NEXT:    ret
@@ -186,9 +186,7 @@ define void @za_write_vg2_vert_bf16(i32 %slice, <vscale x 8 x bfloat> %zn1, <vsc
 define void @za_write_vg2_vert_s(i32 %slice, <vscale x 4 x i32> %zn1, <vscale x 4 x i32> %zn2) {
 ; CHECK-LABEL: za_write_vg2_vert_s:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    // kill: def $z1 killed $z1 killed $z0_z1 def $z0_z1
 ; CHECK-NEXT:    mov w12, w0
-; CHECK-NEXT:    // kill: def $z0 killed $z0 killed $z0_z1 def $z0_z1
 ; CHECK-NEXT:    mov za0v.s[w12, 0:1], { z0.s, z1.s }
 ; CHECK-NEXT:    mov za3v.s[w12, 2:3], { z0.s, z1.s }
 ; CHECK-NEXT:    ret
@@ -201,9 +199,7 @@ define void @za_write_vg2_vert_s(i32 %slice, <vscale x 4 x i32> %zn1, <vscale x
 define void @za_write_vg2_vert_f32(i32 %slice, <vscale x 4 x float> %zn1, <vscale x 4 x float> %zn2) {
 ; CHECK-LABEL: za_write_vg2_vert_f32:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    // kill: def $z1 killed $z1 killed $z0_z1 def $z0_z1
 ; CHECK-NEXT:    mov w12, w0
-; CHECK-NEXT:    // kill: def $z0 killed $z0 killed $z0_z1 def $z0_z1
 ; CHECK-NEXT:    mov za0v.s[w12, 0:1], { z0.s, z1.s }
 ; CHECK-NEXT:    mov za3v.s[w12, 2:3], { z0.s, z1.s }
 ; CHECK-NEXT:    ret
@@ -216,9 +212,7 @@ define void @za_write_vg2_vert_f32(i32 %slice, <vscale x 4 x float> %zn1, <vscal
 define void @za_write_vg2_vert_d(i32 %slice, <vscale x 2 x i64> %zn1, <vscale x 2 x i64> %zn2) {
 ; CHECK-LABEL: za_write_vg2_vert_d:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    // kill: def $z1 killed $z1 killed $z0_z1 def $z0_z1
 ; CHECK-NEXT:    mov w12, w0
-; CHECK-NEXT:    // kill: def $z0 killed $z0 killed $z0_z1 def $z0_z1
 ; CHECK-NEXT:    mov za0v.d[w12, 0:1], { z0.d, z1.d }
 ; CHECK-NEXT:    ret
   call void @llvm.aarch64.sme.write.ver.vg2.nxv2i64(i32 0, i32 %slice, <vscale x 2 x i64> %zn1, <vscale x 2 x i64> %zn2)
@@ -228,9 +222,7 @@ define void @za_write_vg2_vert_d(i32 %slice, <vscale x 2 x i64> %zn1, <vscale x
 define void @za_write_vg2_vert_f64(i32 %slice, <vscale x 2 x double> %zn1, <vscale x 2 x double> %zn2) {
 ; CHECK-LABEL: za_write_vg2_vert_f64:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    // kill: def $z1 killed $z1 killed $z0_z1 def $z0_z1
 ; CHECK-NEXT:    mov w12, w0
-; CHECK-NEXT:    // kill: def $z0 killed $z0 killed $z0_z1 def $z0_z1
 ; CHECK-NEXT:    mov za0v.d[w12, 0:1], { z0.d, z1.d }
 ; CHECK-NEXT:    ret
   call void @llvm.aarch64.sme.write.ver.vg2.nxv2f64(i32 0, i32 %slice, <vscale x 2 x double> %zn1, <vscale x 2 x double> %zn2)
@@ -246,11 +238,7 @@ define void @za_write_vg2_vert_f64(i32 %slice, <vscale x 2 x double> %zn1, <vsca
 define void @za_write_vg4_horiz_b(i32 %slice, <vscale x 16 x i8> %zn1, <vscale x 16 x i8> %zn2, <vscale x 16 x i8> %zn3, <vscale x 16 x i8> %zn4) {
 ; CHECK-LABEL: za_write_vg4_horiz_b:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    // kill: def $z3 killed $z3 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
 ; CHECK-NEXT:    mov w12, w0
-; CHECK-NEXT:    // kill: def $z2 killed $z2 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
-; CHECK-NEXT:    // kill: def $z1 killed $z1 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
-; CHECK-NEXT:    // kill: def $z0 killed $z0 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
 ; CHECK-NEXT:    mov za0h.b[w12, 0:3], { z0.b - z3.b }
 ; CHECK-NEXT:    mov za0h.b[w12, 12:15], { z0.b - z3.b }
 ; CHECK-NEXT:    ret
@@ -263,11 +251,7 @@ define void @za_write_vg4_horiz_b(i32 %slice, <vscale x 16 x i8> %zn1, <vscale x
 define void @za_write_vg4_horiz_h(i32 %slice, <vscale x 8 x i16> %zn1, <vscale x 8 x i16> %zn2, <vscale x 8 x i16> %zn3, <vscale x 8 x i16> %zn4) {
 ; CHECK-LABEL: za_write_vg4_horiz_h:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    // kill: def $z3 killed $z3 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
 ; CHECK-NEXT:    mov w12, w0
-; CHECK-NEXT:    // kill: def $z2 killed $z2 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
-; CHECK-NEXT:    // kill: def $z1 killed $z1 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
-; CHECK-NEXT:    // kill: def $z0 killed $z0 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
 ; CHECK-NEXT:    mov za0h.h[w12, 0:3], { z0.h - z3.h }
 ; CHECK-NEXT:    mov za1h.h[w12, 4:7], { z0.h - z3.h }
 ; CHECK-NEXT:    ret
@@ -280,11 +264,7 @@ define void @za_write_vg4_horiz_h(i32 %slice, <vscale x 8 x i16> %zn1, <vscale x
 define void @za_write_vg4_horiz_f16(i32 %slice, <vscale x 8 x half> %zn1, <vscale x 8 x half> %zn2, <vscale x 8 x half> %zn3, <vscale x 8 x half> %zn4) {
 ; CHECK-LABEL: za_write_vg4_horiz_f16:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    // kill: def $z3 killed $z3 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
 ; CHECK-NEXT:    mov w12, w0
-; CHECK-NEXT:    // kill: def $z2 killed $z2 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
-; CHECK-NEXT:    // kill: def $z1 killed $z1 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
-; CHECK-NEXT:    // kill: def $z0 killed $z0 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
 ; CHECK-NEXT:    mov za0h.h[w12, 0:3], { z0.h - z3.h }
 ; CHECK-NEXT:    mov za1h.h[w12, 4:7], { z0.h - z3.h }
 ; CHECK-NEXT:    ret
@@ -297,11 +277,7 @@ define void @za_write_vg4_horiz_f16(i32 %slice, <vscale x 8 x half> %zn1, <vscal
 define void @za_write_vg4_horiz_bf16(i32 %slice, <vscale x 8 x bfloat> %zn1, <vscale x 8 x bfloat> %zn2, <vscale x 8 x bfloat> %zn3, <vscale x 8 x bfloat> %zn4) {
 ; CHECK-LABEL: za_write_vg4_horiz_bf16:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    // kill: def $z3 killed $z3 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
 ; CHECK-NEXT:    mov w12, w0
-; CHECK-NEXT:    // kill: def $z2 killed $z2 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
-; CHECK-NEXT:    // kill: def $z1 killed $z1 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
-; CHECK-NEXT:    // kill: def $z0 killed $z0 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
 ; CHECK-NEXT:    mov za0h.h[w12, 0:3], { z0.h - z3.h }
 ; CHECK-NEXT:    mov za1h.h[w12, 4:7], { z0.h - z3.h }
 ; CHECK-NEXT:    ret
@@ -314,11 +290,7 @@ define void @za_write_vg4_horiz_bf16(i32 %slice, <vscale x 8 x bfloat> %zn1, <vs
 define void @za_write_vg4_horiz_s(i32 %slice, <vscale x 4 x i32> %zn1, <vscale x 4 x i32> %zn2, <vscale x 4 x i32> %zn3, <vscale x 4 x i32> %zn4) {
 ; CHECK-LABEL: za_write_vg4_horiz_s:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    // kill: def $z3 killed $z3 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
 ; CHECK-NEXT:    mov w12, w0
-; CHECK-NEXT:    // kill: def $z2 killed $z2 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
-; CHECK-NEXT:    // kill: def $z1 killed $z1 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
-; CHECK-NEXT:    // kill: def $z0 killed $z0 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
 ; CHECK-NEXT:    mov za0h.s[w12, 0:3], { z0.s - z3.s }
 ; CHECK-NEXT:    ret
   call void @llvm.aarch64.sme.write.hor.vg4.nxv4i32(i32 0, i32 %slice, <vscale x 4 x i32> %zn1, <vscale x 4 x i32> %zn2, <vscale x 4 x i32> %zn3, <vscale x 4 x i32> %zn4)
@@ -328,11 +300,7 @@ define void @za_write_vg4_horiz_s(i32 %slice, <vscale x 4 x i32> %zn1, <vscale x
 define void @za_write_vg4_horiz_f32(i32 %slice, <vscale x 4 x float> %zn1, <vscale x 4 x float> %zn2, <vscale x 4 x float> %zn3, <vscale x 4 x float> %zn4) {
 ; CHECK-LABEL: za_write_vg4_horiz_f32:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    // kill: def $z3 killed $z3 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
 ; CHECK-NEXT:    mov w12, w0
-; CHECK-NEXT:    // kill: def $z2 killed $z2 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
-; CHECK-NEXT:    // kill: def $z1 killed $z1 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
-; CHECK-NEXT:    // kill: def $z0 killed $z0 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
 ; CHECK-NEXT:    mov za0h.s[w12, 0:3], { z0.s - z3.s }
 ; CHECK-NEXT:    ret
   call void @llvm.aarch64.sme.write.hor.vg4.nxv4f32(i32 0, i32 %slice, <vscale x 4 x float> %zn1, <vscale x 4 x float> %zn2, <vscale x 4 x float> %zn3, <vscale x 4 x float> %zn4)
@@ -342,11 +310,7 @@ define void @za_write_vg4_horiz_f32(i32 %slice, <vscale x 4 x float> %zn1, <vsca
 define void @za_write_vg4_horiz_d(i32 %slice, <vscale x 2 x i64> %zn1, <vscale x 2 x i64> %zn2, <vscale x 2 x i64> %zn3, <vscale x 2 x i64> %zn4) {
 ; CHECK-LABEL: za_write_vg4_horiz_d:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    // kill: def $z3 killed $z3 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
 ; CHECK-NEXT:    mov w12, w0
-; CHECK-NEXT:    // kill: def $z2 killed $z2 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
-; CHECK-NEXT:    // kill: def $z1 killed $z1 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
-; CHECK-NEXT:    // kill: def $z0 killed $z0 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
 ; CHECK-NEXT:    mov za0h.d[w12, 0:3], { z0.d - z3.d }
 ; CHECK-NEXT:    ret
   call void @llvm.aarch64.sme.write.hor.vg4.nxv2i64(i32 0, i32 %slice, <vscale x 2 x i64> %zn1, <vscale x 2 x i64> %zn2, <vscale x 2 x i64> %zn3, <vscale x 2 x i64> %zn4)
@@ -356,11 +320,7 @@ define void @za_write_vg4_horiz_d(i32 %slice, <vscale x 2 x i64> %zn1, <vscale x
 define void @za_write_vg4_horiz_f64(i32 %slice, <vscale x 2 x double> %zn1, <vscale x 2 x double> %zn2, <vscale x 2 x double> %zn3, <vscale x 2 x double> %zn4) {
 ; CHECK-LABEL: za_write_vg4_horiz_f64:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    // kill: def $z3 killed $z3 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
 ; CHECK-NEXT:    mov w12, w0
-; CHECK-NEXT:    // kill: def $z2 killed $z2 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
-; CHECK-NEXT:    // kill: def $z1 killed $z1 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
-; CHECK-NEXT:    // kill: def $z0 killed $z0 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
 ; CHECK-NEXT:    mov za0h.d[w12, 0:3], { z0.d - z3.d }
 ; CHECK-NEXT:    ret
   call void @llvm.aarch64.sme.write.hor.vg4.nxv2f64(i32 0, i32 %slice, <vscale x 2 x double> %zn1, <vscale x 2 x double> %zn2, <vscale x 2 x double> %zn3, <vscale x 2 x double> %zn4)
@@ -372,11 +332,7 @@ define void @za_write_vg4_horiz_f64(i32 %slice, <vscale x 2 x double> %zn1, <vsc
 define void @za_write_vg4_vert_b(i32 %slice, <vscale x 16 x i8> %zn1, <vscale x 16 x i8> %zn2, <vscale x 16 x i8> %zn3, <vscale x 16 x i8> %zn4) {
 ; CHECK-LABEL: za_write_vg4_vert_b:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    // kill: def $z3 killed $z3 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
 ; CHECK-NEXT:    mov w12, w0
-; CHECK-NEXT:    // kill: def $z2 killed $z2 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
-; CHECK-NEXT:    // kill: def $z1 killed $z1 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
-; CHECK-NEXT:    // kill: def $z0 killed $z0 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
 ; CHECK-NEXT:    mov za0v.b[w12, 0:3], { z0.b - z3.b }
 ; CHECK-NEXT:    mov za0v.b[w12, 12:15], { z0.b - z3.b }
 ; CHECK-NEXT:    ret
@@ -389,11 +345,7 @@ define void @za_write_vg4_vert_b(i32 %slice, <vscale x 16 x i8> %zn1, <vscale x
 define void @za_write_vg4_vert_h(i32 %slice, <vscale x 8 x i16> %zn1, <vscale x 8 x i16> %zn2, <vscale x 8 x i16> %zn3, <vscale x 8 x i16> %zn4) {
 ; CHECK-LABEL: za_write_vg4_vert_h:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    // kill: def $z3 killed $z3 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
 ; CHECK-NEXT:    mov w12, w0
-; CHECK-NEXT:    // kill: def $z2 killed $z2 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
-; CHECK-NEXT:    // kill: def $z1 killed $z1 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
-; CHECK-NEXT:    // kill: def $z0 killed $z0 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
 ; CHECK-NEXT:    mov za0v.h[w12, 0:3], { z0.h - z3.h }
 ; CHECK-NEXT:    mov za1v.h[w12, 4:7], { z0.h - z3.h }
 ; CHECK-NEXT:    ret
@@ -406,11 +358,7 @@ define void @za_write_vg4_vert_h(i32 %slice, <vscale x 8 x i16> %zn1, <vscale x
 define void @za_write_vg4_vert_f16(i32 %slice, <vscale x 8 x half> %zn1, <vscale x 8 x half> %zn2, <vscale x 8 x half> %zn3, <vscale x 8 x half> %zn4) {
 ; CHECK-LABEL: za_write_vg4_vert_f16:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    // kill: def $z3 killed $z3 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
 ; CHECK-NEXT:    mov w12, w0
-; CHECK-NEXT:    // kill: def $z2 killed $z2 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
-; CHECK-NEXT:    // kill: def $z1 killed $z1 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
-; CHECK-NEXT:    // kill: def $z0 killed $z0 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
 ; CHECK-NEXT:    mov za0v.h[w12, 0:3], { z0.h - z3.h }
 ; CHECK-NEXT:    mov za1v.h[w12, 4:7], { z0.h - z3.h }
 ; CHECK-NEXT:    ret
@@ -423,11 +371,7 @@ define void @za_write_vg4_vert_f16(i32 %slice, <vscale x 8 x half> %zn1, <vscale
 define void @za_write_vg4_vert_bf16(i32 %slice, <vscale x 8 x bfloat> %zn1, <vscale x 8 x bfloat> %zn2, <vscale x 8 x bfloat> %zn3, <vscale x 8 x bfloat> %zn4) {
 ; CHECK-LABEL: za_write_vg4_vert_bf16:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    // kill: def $z3 killed $z3 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
 ; CHECK-NEXT:    mov w12, w0
-; CHECK-NEXT:    // kill: def $z2 killed $z2 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
-; CHECK-NEXT:    // kill: def $z1 killed $z1 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
-; CHECK-NEXT:    // kill: def $z0 killed $z0 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
 ; CHECK-NEXT:    mov za0v.h[w12, 0:3], { z0.h - z3.h }
 ; CHECK-NEXT:    mov za1v.h[w12, 4:7], { z0.h - z3.h }
 ; CHECK-NEXT:    ret
@@ -440,11 +384,7 @@ define void @za_write_vg4_vert_bf16(i32 %slice, <vscale x 8 x bfloat> %zn1, <vsc
 define void @za_write_vg4_vert_s(i32 %slice, <vscale x 4 x i32> %zn1, <vscale x 4 x i32> %zn2, <vscale x 4 x i32> %zn3, <vscale x 4 x i32> %zn4) {
 ; CHECK-LABEL: za_write_vg4_vert_s:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    // kill: def $z3 killed $z3 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
 ; CHECK-NEXT:    mov w12, w0
-; CHECK-NEXT:    // kill: def $z2 killed $z2 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
-; CHECK-NEXT:    // kill: def $z1 killed $z1 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
-; CHECK-NEXT:    // kill: def $z0 killed $z0 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
 ; CHECK-NEXT:    mov za0v.s[w12, 0:3], { z0.s - z3.s }
 ; CHECK-NEXT:    ret
   call void @llvm.aarch64.sme.write.ver.vg4.nxv4i32(i32 0, i32 %slice, <vscale x 4 x i32> %zn1, <vscale x 4 x i32> %zn2, <vscale x 4 x i32> %zn3, <vscale x 4 x i32> %zn4)
@@ -454,11 +394,7 @@ define void @za_write_vg4_vert_s(i32 %slice, <vscale x 4 x i32> %zn1, <vscale x
 define void @za_write_vg4_vert_f32(i32 %slice, <vscale x 4 x float> %zn1, <vscale x 4 x float> %zn2, <vscale x 4 x float> %zn3, <vscale x 4 x float> %zn4) {
 ; CHECK-LABEL: za_write_vg4_vert_f32:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    // kill: def $z3 killed $z3 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
 ; CHECK-NEXT:    mov w12, w0
-; CHECK-NEXT:    // kill: def $z2 killed $z2 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
-; CHECK-NEXT:    // kill: def $z1 killed $z1 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
-; CHECK-NEXT:    // kill: def $z0 killed $z0 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
 ; CHECK-NEXT:    mov za0v.s[w12, 0:3], { z0.s - z3.s }
 ; CHECK-NEXT:    ret
   call void @llvm.aarch64.sme.write.ver.vg4.nxv4f32(i32 0, i32 %slice, <vscale x 4 x float> %zn1, <vscale x 4 x float> %zn2, <vscale x 4 x float> %zn3, <vscale x 4 x float> %zn4)
@@ -468,11 +404,7 @@ define void @za_write_vg4_vert_f32(i32 %slice, <vscale x 4 x float> %zn1, <vscal
 define void @za_write_vg4_vert_d(i32 %slice, <vscale x 2 x i64> %zn1, <vscale x 2 x i64> %zn2, <vscale x 2 x i64> %zn3, <vscale x 2 x i64> %zn4) {
 ; CHECK-LABEL: za_write_vg4_vert_d:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    // kill: def $z3 killed $z3 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
 ; CHECK-NEXT:    mov w12, w0
-; CHECK-NEXT:    // kill: def $z2 killed $z2 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
-; CHECK-NEXT:    // kill: def $z1 killed $z1 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
-; CHECK-NEXT:    // kill: def $z0 killed $z0 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
 ; CHECK-NEXT:    mov za0v.d[w12, 0:3], { z0.d - z3.d }
 ; CHECK-NEXT:    ret
   call void @llvm.aarch64.sme.write.ver.vg4.nxv2i64(i32 0, i32 %slice, <vscale x 2 x i64> %zn1, <vscale x 2 x i64> %zn2, <vscale x 2 x i64> %zn3, <vscale x 2 x i64> %zn4)
@@ -482,17 +414,65 @@ define void @za_write_vg4_vert_d(i32 %slice, <vscale x 2 x i64> %zn1, <vscale x
 define void @za_write_vg4_vert_f64(i32 %slice, <vscale x 2 x double> %zn1, <vscale x 2 x double> %zn2, <vscale x 2 x double> %zn3, <vscale x 2 x double> %zn4) {
 ; CHECK-LABEL: za_write_vg4_vert_f64:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    // kill: def $z3 killed $z3 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
 ; CHECK-NEXT:    mov w12, w0
-; CHECK-NEXT:    // kill: def $z2 killed $z2 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
-; CHECK-NEXT:    // kill: def $z1 killed $z1 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
-; CHECK-NEXT:    // kill: def $z0 killed $z0 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
 ; CHECK-NEXT:    mov za0v.d[w12, 0:3], { z0.d - z3.d }
 ; CHECK-NEXT:    ret
   call void @llvm.aarch64.sme.write.ver.vg4.nxv2f64(i32 0, i32 %slice, <vscale x 2 x double> %zn1, <vscale x 2 x double> %zn2, <vscale x 2 x double> %zn3, <vscale x 2 x double> %zn4)
   ret void
 }
 
+define void @za_write_vg4_vert_f64_tuple(i64 %stride, ptr %ptr) {
+; CHECK-LABEL: za_write_vg4_vert_f64_tuple:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    lsl x8, x0, #1
+; CHECK-NEXT:    add x9, x1, x0
+; CHECK-NEXT:    ptrue pn8.b
+; CHECK-NEXT:    ld1d { z16.d, z20.d, z24.d, z28.d }, pn8/z, [x1]
+; CHECK-NEXT:    ld1d { z17.d, z21.d, z25.d, z29.d }, pn8/z, [x9]
+; CHECK-NEXT:    mov w12, wzr
+; CHECK-NEXT:    add x10, x1, x8
+; CHECK-NEXT:    add x8, x9, x8
+; CHECK-NEXT:    ld1d { z18.d, z22.d, z26.d, z30.d }, pn8/z, [x10]
+; CHECK-NEXT:    ld1d { z19.d, z23.d, z27.d, z31.d }, pn8/z, [x8]
+; CHECK-NEXT:    mov za0v.d[w12, 0:3], { z16.d - z19.d }
+; CHECK-NEXT:    mov za0v.d[w12, 0:3], { z20.d - z23.d }
+; CHECK-NEXT:    mov za0v.d[w12, 0:3], { z24.d - z27.d }
+; CHECK-NEXT:    mov za0v.d[w12, 0:3], { z28.d - z31.d }
+; CHECK-NEXT:    ret
+entry:
+  %0 = tail call target("aarch64.svcount") @llvm.aarch64.sve.ptrue.c8()
+  %1 = tail call { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } @llvm.aarch64.sve.ld1.pn.x4.nxv2f64(target("aarch64.svcount") %0, ptr %ptr)
+  %2 = extractvalue { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } %1, 0
+  %3 = extractvalue { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } %1, 1
+  %4 = extractvalue { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } %1, 2
+  %5 = extractvalue { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } %1, 3
+  %arrayidx2 = getelementptr inbounds i8, ptr %ptr, i64 %stride
+  %6 = tail call { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } @llvm.aarch64.sve.ld1.pn.x4.nxv2f64(target("aarch64.svcount") %0, ptr %arrayidx2)
+  %7 = extractvalue { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } %6, 0
+  %8 = extractvalue { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } %6, 1
+  %9 = extractvalue { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } %6, 2
+  %10 = extractvalue { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } %6, 3
+  %mul3 = shl i64 %stride, 1
+  %arrayidx4 = getelementptr inbounds i8, ptr %ptr, i64 %mul3
+  %11 = tail call { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } @llvm.aarch64.sve.ld1.pn.x4.nxv2f64(target("aarch64.svcount") %0, ptr %arrayidx4)
+  %12 = extractvalue { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } %11, 0
+  %13 = extractvalue { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } %11, 1
+  %14 = extractvalue { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } %11, 2
+  %15 = extractvalue { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } %11, 3
+  %mul5 = mul i64 %stride, 3
+  %arrayidx6 = getelementptr inbounds i8, ptr %ptr, i64 %mul5
+  %16 = tail call { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } @llvm.aarch64.sve.ld1.pn.x4.nxv2f64(target("aarch64.svcount") %0, ptr %arrayidx6)
+  %17 = extractvalue { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } %16, 0
+  %18 = extractvalue { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } %16, 1
+  %19 = extractvalue { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } %16, 2
+  %20 = extractvalue { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } %16, 3
+  call void @llvm.aarch64.sme.write.ver.vg4.nxv2f64(i32 0, i32 0, <vscale x 2 x double> %2, <vscale x 2 x double> %7, <vscale x 2 x double> %12, <vscale x 2 x double> %17)
+  call void @llvm.aarch64.sme.write.ver.vg4.nxv2f64(i32 0, i32 0, <vscale x 2 x double> %3, <vscale x 2 x double> %8, <vscale x 2 x double> %13, <vscale x 2 x double> %18)
+  call void @llvm.aarch64.sme.write.ver.vg4.nxv2f64(i32 0, i32 0, <vscale x 2 x double> %4, <vscale x 2 x double> %9, <vscale x 2 x double> %14, <vscale x 2 x double> %19)
+  call void @llvm.aarch64.sme.write.ver.vg4.nxv2f64(i32 0, i32 0, <vscale x 2 x double> %5, <vscale x 2 x double> %10, <vscale x 2 x double> %15, <vscale x 2 x double> %20)
+  ret void
+}
+
 ;
 ; Move Multi-Vector To ZA (Write) x2
 ;
@@ -500,9 +480,7 @@ define void @za_write_vg4_vert_f64(i32 %slice, <vscale x 2 x double> %zn1, <vsca
 define void @za_write_vg1x2_b(i32 %slice, <vscale x 16 x i8> %za1, <vscale x 16 x i8> %za2) {
 ; CHECK-LABEL: za_write_vg1x2_b:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    // kill: def $z1 killed $z1 killed $z0_z1 def $z0_z1
 ; CHECK-NEXT:    mov w8, w0
-; CHECK-NEXT:    // kill: def $z0 killed $z0 killed $z0_z1 def $z0_z1
 ; CHECK-NEXT:    mov za.d[w8, 0, vgx2], { z0.d, z1.d }
 ; CHECK-NEXT:    mov za.d[w8, 7, vgx2], { z0.d, z1.d }
 ; CHECK-NEXT:    ret
@@ -512,12 +490,34 @@ define void @za_write_vg1x2_b(i32 %slice, <vscale x 16 x i8> %za1, <vscale x 16
   ret void
 }
 
+define void @za_write_vg1x2_b_tuple(i64 %stride, ptr %ptr) {
+; CHECK-LABEL: za_write_vg1x2_b_tuple:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ptrue pn8.b
+; CHECK-NEXT:    mov w8, wzr
+; CHECK-NEXT:    ld1b { z16.b, z24.b }, pn8/z, [x1]
+; CHECK-NEXT:    ld1b { z17.b, z25.b }, pn8/z, [x1, x0]
+; CHECK-NEXT:    mov za.d[w8, 0, vgx2], { z16.d, z17.d }
+; CHECK-NEXT:    mov za.d[w8, 0, vgx2], { z24.d, z25.d }
+; CHECK-NEXT:    ret
+entry:
+  %0 = tail call target("aarch64.svcount") @llvm.aarch64.sve.ptrue.c8()
+  %1 = tail call { <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.ld1.pn.x2.nxv16i8(target("aarch64.svcount") %0, ptr %ptr)
+  %2 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8> } %1, 0
+  %3 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8> } %1, 1
+  %arrayidx2 = getelementptr inbounds i8, ptr %ptr, i64 %stride
+  %4 = tail call { <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.ld1.pn.x2.nxv16i8(target("aarch64.svcount") %0, ptr %arrayidx2)
+  %5 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8> } %4, 0
+  %6 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8> } %4, 1
+  call void @llvm.aarch64.sme.write.vg1x2.nxv16i8(i32 0, <vscale x 16 x i8> %2, <vscale x 16 x i8> %5)
+  call void @llvm.aarch64.sme.write.vg1x2.nxv16i8(i32 0, <vscale x 16 x i8> %3, <vscale x 16 x i8> %6)
+  ret void
+}
+
 define void @za_write_vg1x2_h(i32 %slice, <vscale x 8 x i16> %za1, <vscale x 8 x i16> %za2) {
 ; CHECK-LABEL: za_write_vg1x2_h:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    // kill: def $z1 killed $z1 killed $z0_z1 def $z0_z1
 ; CHECK-NEXT:    mov w8, w0
-; CHECK-NEXT:    // kill: def $z0 killed $z0 killed $z0_z1 def $z0_z1
 ; CHECK-NEXT:    mov za.d[w8, 0, vgx2], { z0.d, z1.d }
 ; CHECK-NEXT:    mov za.d[w8, 7, vgx2], { z0.d, z1.d }
 ; CHECK-NEXT:    ret
@@ -530,9 +530,7 @@ define void @za_write_vg1x2_h(i32 %slice, <vscale x 8 x i16> %za1, <vscale x 8 x
 define void @za_write_vg1x2_f16(i32 %slice, <vscale x 8 x half> %za1, <vscale x 8 x half> %za2) {
 ; CHECK-LABEL: za_write_vg1x2_f16:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    // kill: def $z1 killed $z1 killed $z0_z1 def $z0_z1
 ; CHECK-NEXT:    mov w8, w0
-; CHECK-NEXT:    // kill: def $z0 killed $z0 killed $z0_z1 def $z0_z1
 ; CHECK-NEXT:    mov za.d[w8, 0, vgx2], { z0.d, z1.d }
 ; CHECK-NEXT:    mov za.d[w8, 7, vgx2], { z0.d, z1.d }
 ; CHECK-NEXT:    ret
@@ -545,9 +543,7 @@ define void @za_write_vg1x2_f16(i32 %slice, <vscale x 8 x half> %za1, <vscale x
 define void @za_write_vg1x2_bf16(i32 %slice, <vscale x 8 x bfloat> %za1, <vscale x 8 x bfloat> %za2) {
 ; CHECK-LABEL: za_write_vg1x2_bf16:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    // kill: def $z1 killed $z1 killed $z0_z1 def $z0_z1
 ; CHECK-NEXT:    mov w8, w0
-; CHECK-NEXT:    // kill: def $z0 killed $z0 killed $z0_z1 def $z0_z1
 ; CHECK-NEXT:    mov za.d[w8, 0, vgx2], { z0.d, z1.d }
 ; CHECK-NEXT:    mov za.d[w8, 7, vgx2], { z0.d, z1.d }
 ; CHECK-NEXT:    ret
@@ -560,9 +556,7 @@ define void @za_write_vg1x2_bf16(i32 %slice, <vscale x 8 x bfloat> %za1, <vscale
 define void @za_write_vg1x2_s(i32 %slice, <vscale x 4 x i32> %za1, <vscale x 4 x i32> %za2) {
 ; CHECK-LABEL: za_write_vg1x2_s:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    // kill: def $z1 killed $z1 killed $z0_z1 def $z0_z1
 ; CHECK-NEXT:    mov w8, w0
-; CHECK-NEXT:    // kill: def $z0 killed $z0 killed $z0_z1 def $z0_z1
 ; CHECK-NEXT:    mov za.d[w8, 0, vgx2], { z0.d, z1.d }
 ; CHECK-NEXT:    mov za.d[w8, 7, vgx2], { z0.d, z1.d }
 ; CHECK-NEXT:    ret
@@ -575,9 +569,7 @@ define void @za_write_vg1x2_s(i32 %slice, <vscale x 4 x i32> %za1, <vscale x 4 x
 define void @za_write_vg1x2_f32(i32 %slice, <vscale x 4 x float> %za1, <vscale x 4 x float> %za2) {
 ; CHECK-LABEL: za_write_vg1x2_f32:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    // kill: def $z1 killed $z1 killed $z0_z1 def $z0_z1
 ; CHECK-NEXT:    mov w8, w0
-; CHECK-NEXT:    // kill: def $z0 killed $z0 killed $z0_z1 def $z0_z1
 ; CHECK-NEXT:    mov za.d[w8, 0, vgx2], { z0.d, z1.d }
 ; CHECK-NEXT:    mov za.d[w8, 7, vgx2], { z0.d, z1.d }
 ; CHECK-NEXT:    ret
@@ -590,9 +582,7 @@ define void @za_write_vg1x2_f32(i32 %slice, <vscale x 4 x float> %za1, <vscale x
 define void @za_write_vg1x2_d(i32 %slice, <vscale x 2 x i64> %za1, <vscale x 2 x i64> %za2) {
 ; CHECK-LABEL: za_write_vg1x2_d:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    // kill: def $z1 killed $z1 killed $z0_z1 def $z0_z1
 ; CHECK-NEXT:    mov w8, w0
-; CHECK-NEXT:    // kill: def $z0 killed $z0 killed $z0_z1 def $z0_z1
 ; CHECK-NEXT:    mov za.d[w8, 0, vgx2], { z0.d, z1.d }
 ; CHECK-NEXT:    mov za.d[w8, 7, vgx2], { z0.d, z1.d }
 ; CHECK-NEXT:    ret
@@ -605,9 +595,7 @@ define void @za_write_vg1x2_d(i32 %slice, <vscale x 2 x i64> %za1, <vscale x 2 x
 define void @za_write_vg1x2_f64(i32 %slice, <vscale x 2 x double> %za1, <vscale x 2 x double> %za2) {
 ; CHECK-LABEL: za_write_vg1x2_f64:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    // kill: def $z1 killed $z1 killed $z0_z1 def $z0_z1
 ; CHECK-NEXT:    mov w8, w0
-; CHECK-NEXT:    // kill: def $z0 killed $z0 killed $z0_z1 def $z0_z1
 ; CHECK-NEXT:    mov za.d[w8, 0, vgx2], { z0.d, z1.d }
 ; CHECK-NEXT:    mov za.d[w8, 7, vgx2], { z0.d, z1.d }
 ; CHECK-NEXT:    ret
@@ -624,11 +612,7 @@ define void @za_write_vg1x2_f64(i32 %slice, <vscale x 2 x double> %za1, <vscale
 define void @za_write_vg1x4_b(i32 %slice, <vscale x 16 x i8> %za1, <vscale x 16 x i8> %za2, <vscale x 16 x i8> %za3, <vscale x 16 x i8> %za4) {
 ; CHECK-LABEL: za_write_vg1x4_b:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    // kill: def $z3 killed $z3 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
 ; CHECK-NEXT:    mov w8, w0
-; CHECK-NEXT:    // kill: def $z2 killed $z2 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
-; CHECK-NEXT:    // kill: def $z1 killed $z1 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
-; CHECK-NEXT:    // kill: def $z0 killed $z0 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
 ; CHECK-NEXT:    mov za.d[w8, 0, vgx4], { z0.d - z3.d }
 ; CHECK-NEXT:    mov za.d[w8, 7, vgx4], { z0.d - z3.d }
 ; CHECK-NEXT:    ret
@@ -641,11 +625,7 @@ define void @za_write_vg1x4_b(i32 %slice, <vscale x 16 x i8> %za1, <vscale x 16
 define void @za_write_vg1x4_h(i32 %slice, <vscale x 8 x i16> %za1, <vscale x 8 x i16> %za2, <vscale x 8 x i16> %za3, <vscale x 8 x i16> %za4) {
 ; CHECK-LABEL: za_write_vg1x4_h:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    // kill: def $z3 killed $z3 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
 ; CHECK-NEXT:    mov w8, w0
-; CHECK-NEXT:    // kill: def $z2 killed $z2 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
-; CHECK-NEXT:    // kill: def $z1 killed $z1 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
-; CHECK-NEXT:    // kill: def $z0 killed $z0 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
 ; CHECK-NEXT:    mov za.d[w8, 0, vgx4], { z0.d - z3.d }
 ; CHECK-NEXT:    mov za.d[w8, 7, vgx4], { z0.d - z3.d }
 ; CHECK-NEXT:    ret
@@ -658,11 +638,7 @@ define void @za_write_vg1x4_h(i32 %slice, <vscale x 8 x i16> %za1, <vscale x 8 x
 define void @za_write_vg1x4_f16(i32 %slice, <vscale x 8 x half> %za1, <vscale x 8 x half> %za2, <vscale x 8 x half> %za3, <vscale x 8 x half> %za4) {
 ; CHECK-LABEL: za_write_vg1x4_f16:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    // kill: def $z3 killed $z3 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
 ; CHECK-NEXT:    mov w8, w0
-; CHECK-NEXT:    // kill: def $z2 killed $z2 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
-; CHECK-NEXT:    // kill: def $z1 killed $z1 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
-; CHECK-NEXT:    // kill: def $z0 killed $z0 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
 ; CHECK-NEXT:    mov za.d[w8, 0, vgx4], { z0.d - z3.d }
 ; CHECK-NEXT:    mov za.d[w8, 7, vgx4], { z0.d - z3.d }
 ; CHECK-NEXT:    ret
@@ -675,11 +651,7 @@ define void @za_write_vg1x4_f16(i32 %slice, <vscale x 8 x half> %za1, <vscale x
 define void @za_write_vg1x4_bf16(i32 %slice, <vscale x 8 x bfloat> %za1, <vscale x 8 x bfloat> %za2, <vscale x 8 x bfloat> %za3, <vscale x 8 x bfloat> %za4) {
 ; CHECK-LABEL: za_write_vg1x4_bf16:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    // kill: def $z3 killed $z3 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
 ; CHECK-NEXT:    mov w8, w0
-; CHECK-NEXT:    // kill: def $z2 killed $z2 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
-; CHECK-NEXT:    // kill: def $z1 killed $z1 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
-; CHECK-NEXT:    // kill: def $z0 killed $z0 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
 ; CHECK-NEXT:    mov za.d[w8, 0, vgx4], { z0.d - z3.d }
 ; CHECK-NEXT:    mov za.d[w8, 7, vgx4], { z0.d - z3.d }
 ; CHECK-NEXT:    ret
@@ -692,11 +664,7 @@ define void @za_write_vg1x4_bf16(i32 %slice, <vscale x 8 x bfloat> %za1, <vscale
 define void @za_write_vg1x4_s(i32 %slice, <vscale x 4 x i32> %za1, <vscale x 4 x i32> %za2, <vscale x 4 x i32> %za3, <vscale x 4 x i32> %za4) {
 ; CHECK-LABEL: za_write_vg1x4_s:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    // kill: def $z3 killed $z3 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
 ; CHECK-NEXT:    mov w8, w0
-; CHECK-NEXT:    // kill: def $z2 killed $z2 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
-; CHECK-NEXT:    // kill: def $z1 killed $z1 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
-; CHECK-NEXT:    // kill: def $z0 killed $z0 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
 ; CHECK-NEXT:    mov za.d[w8, 0, vgx4], { z0.d - z3.d }
 ; CHECK-NEXT:    mov za.d[w8, 7, vgx4], { z0.d - z3.d }
 ; CHECK-NEXT:    ret
@@ -709,11 +677,7 @@ define void @za_write_vg1x4_s(i32 %slice, <vscale x 4 x i32> %za1, <vscale x 4 x
 define void @za_write_vg1x4_f32(i32 %slice, <vscale x 4 x float> %za1, <vscale x 4 x float> %za2, <vscale x 4 x float> %za3, <vscale x 4 x float> %za4) {
 ; CHECK-LABEL: za_write_vg1x4_f32:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    // kill: def $z3 killed $z3 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
 ; CHECK-NEXT:    mov w8, w0
-; CHECK-NEXT:    // kill: def $z2 killed $z2 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
-; CHECK-NEXT:    // kill: def $z1 killed $z1 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
-; CHECK-NEXT:    // kill: def $z0 killed $z0 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
 ; CHECK-NEXT:    mov za.d[w8, 0, vgx4], { z0.d - z3.d }
 ; CHECK-NEXT:    mov za.d[w8, 7, vgx4], { z0.d - z3.d }
 ; CHECK-NEXT:    ret
@@ -726,11 +690,7 @@ define void @za_write_vg1x4_f32(i32 %slice, <vscale x 4 x float> %za1, <vscale x
 define void @za_write_vg1x4_d(i32 %slice, <vscale x 2 x i64> %za1, <vscale x 2 x i64> %za2, <vscale x 2 x i64> %za3, <vscale x 2 x i64> %za4) {
 ; CHECK-LABEL: za_write_vg1x4_d:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    // kill: def $z3 killed $z3 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
 ; CHECK-NEXT:    mov w8, w0
-; CHECK-NEXT:    // kill: def $z2 killed $z2 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
-; CHECK-NEXT:    // kill: def $z1 killed $z1 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
-; CHECK-NEXT:    // kill: def $z0 killed $z0 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
 ; CHECK-NEXT:    mov za.d[w8, 0, vgx4], { z0.d - z3.d }
 ; CHECK-NEXT:    mov za.d[w8, 7, vgx4], { z0.d - z3.d }
 ; CHECK-NEXT:    ret
@@ -743,11 +703,7 @@ define void @za_write_vg1x4_d(i32 %slice, <vscale x 2 x i64> %za1, <vscale x 2 x
 define void @za_write_vg1x4_f64(i32 %slice, <vscale x 2 x double> %za1, <vscale x 2 x double> %za2, <vscale x 2 x double> %za3, <vscale x 2 x double> %za4) {
 ; CHECK-LABEL: za_write_vg1x4_f64:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    // kill: def $z3 killed $z3 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
 ; CHECK-NEXT:    mov w8, w0
-; CHECK-NEXT:    // kill: def $z2 killed $z2 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
-; CHECK-NEXT:    // kill: def $z1 killed $z1 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
-; CHECK-NEXT:    // kill: def $z0 killed $z0 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
 ; CHECK-NEXT:    mov za.d[w8, 0, vgx4], { z0.d - z3.d }
 ; CHECK-NEXT:    mov za.d[w8, 7, vgx4], { z0.d - z3.d }
 ; CHECK-NEXT:    ret
@@ -757,6 +713,58 @@ define void @za_write_vg1x4_f64(i32 %slice, <vscale x 2 x double> %za1, <vscale
   ret void
 }
 
+define void @za_write_vg1x4_f64_tuple(i64 %stride, ptr %ptr) {
+; CHECK-LABEL: za_write_vg1x4_f64_tuple:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    lsl x9, x0, #1
+; CHECK-NEXT:    add x10, x1, x0
+; CHECK-NEXT:    ptrue pn8.b
+; CHECK-NEXT:    ld1d { z16.d, z20.d, z24.d, z28.d }, pn8/z, [x1]
+; CHECK-NEXT:    ld1d { z17.d, z21.d, z25.d, z29.d }, pn8/z, [x10]
+; CHECK-NEXT:    mov w8, wzr
+; CHECK-NEXT:    add x11, x1, x9
+; CHECK-NEXT:    add x9, x10, x9
+; CHECK-NEXT:    ld1d { z18.d, z22.d, z26.d, z30.d }, pn8/z, [x11]
+; CHECK-NEXT:    ld1d { z19.d, z23.d, z27.d, z31.d }, pn8/z, [x9]
+; CHECK-NEXT:    mov za.d[w8, 0, vgx4], { z16.d - z19.d }
+; CHECK-NEXT:    mov za.d[w8, 0, vgx4], { z20.d - z23.d }
+; CHECK-NEXT:    mov za.d[w8, 0, vgx4], { z24.d - z27.d }
+; CHECK-NEXT:    mov za.d[w8, 0, vgx4], { z28.d - z31.d }
+; CHECK-NEXT:    ret
+entry:
+  %0 = tail call target("aarch64.svcount") @llvm.aarch64.sve.ptrue.c8()
+  %1 = tail call { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } @llvm.aarch64.sve.ld1.pn.x4.nxv2f64(target("aarch64.svcount") %0, ptr %ptr)
+  %2 = extractvalue { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } %1, 0
+  %3 = extractvalue { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } %1, 1
+  %4 = extractvalue { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } %1, 2
+  %5 = extractvalue { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } %1, 3
+  %arrayidx2 = getelementptr inbounds i8, ptr %ptr, i64 %stride
+  %6 = tail call { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } @llvm.aarch64.sve.ld1.pn.x4.nxv2f64(target("aarch64.svcount") %0, ptr %arrayidx2)
+  %7 = extractvalue { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } %6, 0
+  %8 = extractvalue { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } %6, 1
+  %9 = extractvalue { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } %6, 2
+  %10 = extractvalue { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } %6, 3
+  %mul3 = shl i64 %stride, 1
+  %arrayidx4 = getelementptr inbounds i8, ptr %ptr, i64 %mul3
+  %11 = tail call { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } @llvm.aarch64.sve.ld1.pn.x4.nxv2f64(target("aarch64.svcount") %0, ptr %arrayidx4)
+  %12 = extractvalue { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } %11, 0
+  %13 = extractvalue { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } %11, 1
+  %14 = extractvalue { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } %11, 2
+  %15 = extractvalue { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } %11, 3
+  %mul5 = mul i64 %stride, 3
+  %arrayidx6 = getelementptr inbounds i8, ptr %ptr, i64 %mul5
+  %16 = tail call { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } @llvm.aarch64.sve.ld1.pn.x4.nxv2f64(target("aarch64.svcount") %0, ptr %arrayidx6)
+  %17 = extractvalue { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } %16, 0
+  %18 = extractvalue { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } %16, 1
+  %19 = extractvalue { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } %16, 2
+  %20 = extractvalue { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } %16, 3
+  call void @llvm.aarch64.sme.write.vg1x4.nxv2f64(i32 0, <vscale x 2 x double> %2, <vscale x 2 x double> %7, <vscale x 2 x double> %12, <vscale x 2 x double> %17)
+  call void @llvm.aarch64.sme.write.vg1x4.nxv2f64(i32 0, <vscale x 2 x double> %3, <vscale x 2 x double> %8, <vscale x 2 x double> %13, <vscale x 2 x double> %18)
+  call void @llvm.aarch64.sme.write.vg1x4.nxv2f64(i32 0, <vscale x 2 x double> %4, <vscale x 2 x double> %9, <vscale x 2 x double> %14, <vscale x 2 x double> %19)
+  call void @llvm.aarch64.sme.write.vg1x4.nxv2f64(i32 0, <vscale x 2 x double> %5, <vscale x 2 x double> %10, <vscale x 2 x double> %15, <vscale x 2 x double> %20)
+  ret void
+}
+
 declare void @llvm.aarch64.sme.write.hor.vg2.nxv16i8(i32, i32, <vscale x 16 x i8>, <vscale x 16 x i8>)
 declare void @llvm.aarch64.sme.write.hor.vg2.nxv8i16(i32, i32, <vscale x 8 x i16>, <vscale x 8 x i16>)
 declare void @llvm.aarch64.sme.write.hor.vg2.nxv8f16(i32, i32, <vscale x 8 x half>, <vscale x 8 x half>)
diff --git a/llvm/test/CodeGen/AArch64/sme2-intrinsics-int-dots.ll b/llvm/test/CodeGen/AArch64/sme2-intrinsics-int-dots.ll
index 967d168593a40..fba81eac905e2 100644
--- a/llvm/test/CodeGen/AArch64/sme2-intrinsics-int-dots.ll
+++ b/llvm/test/CodeGen/AArch64/sme2-intrinsics-int-dots.ll
@@ -23,6 +23,28 @@ define void @udot_multi_za32_u16_vg1x2(i32 %slice, <vscale x 16 x i8> %unused, <
   ret void
 }
 
+define void @udot_multi_za32_u16_vg1x2_tuple(i64 %stride, ptr %ptr) #1 {
+; CHECK-LABEL: udot_multi_za32_u16_vg1x2_tuple:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ptrue pn8.b
+; CHECK-NEXT:    mov w8, wzr
+; CHECK-NEXT:    ld1b { z16.b, z24.b }, pn8/z, [x1]
+; CHECK-NEXT:    ld1b { z17.b, z25.b }, pn8/z, [x1, x0]
+; CHECK-NEXT:    udot za.s[w8, 0, vgx2], { z16.b, z17.b }, { z24.b, z25.b }
+; CHECK-NEXT:    ret
+entry:
+  %0 = tail call target("aarch64.svcount") @llvm.aarch64.sve.ptrue.c8()
+  %1 = tail call { <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.ld1.pn.x2.nxv16i8(target("aarch64.svcount") %0, ptr %ptr)
+  %2 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8> } %1, 0
+  %3 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8> } %1, 1
+  %arrayidx2 = getelementptr inbounds i8, ptr %ptr, i64 %stride
+  %4 = tail call { <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.ld1.pn.x2.nxv16i8(target("aarch64.svcount") %0, ptr %arrayidx2)
+  %5 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8> } %4, 0
+  %6 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8> } %4, 1
+  call void @llvm.aarch64.sme.udot.za32.vg1x2.nxv16i8(i32 0, <vscale x 16 x i8> %2, <vscale x 16 x i8> %5, <vscale x 16 x i8> %3, <vscale x 16 x i8> %6)
+  ret void
+}
+
 define void @udot_multi_za32_u16_vg1x4(i32 %slice, <vscale x 16 x i8> %unused, <vscale x 8 x i16> %zn0, <vscale x 8 x i16> %zn1, <vscale x 8 x i16> %zn2, <vscale x 8 x i16> %zn3,
 ; CHECK-LABEL: udot_multi_za32_u16_vg1x4:
 ; CHECK:       // %bb.0:
@@ -48,6 +70,54 @@ define void @udot_multi_za32_u16_vg1x4(i32 %slice, <vscale x 16 x i8> %unused, <
   ret void
 }
 
+define void @udot_multi_za32_u16_vg1x4_tuple(i64 %stride, ptr %ptr) #1 {
+; CHECK-LABEL: udot_multi_za32_u16_vg1x4_tuple:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    lsl x9, x0, #1
+; CHECK-NEXT:    ptrue pn8.b
+; CHECK-NEXT:    mov w8, wzr
+; CHECK-NEXT:    ld1b { z16.b, z20.b, z24.b, z28.b }, pn8/z, [x1]
+; CHECK-NEXT:    ld1b { z17.b, z21.b, z25.b, z29.b }, pn8/z, [x1, x0]
+; CHECK-NEXT:    add x10, x9, x0
+; CHECK-NEXT:    ld1b { z18.b, z22.b, z26.b, z30.b }, pn8/z, [x1, x9]
+; CHECK-NEXT:    ld1b { z19.b, z23.b, z27.b, z31.b }, pn8/z, [x1, x10]
+; CHECK-NEXT:    udot za.s[w8, 0, vgx4], { z16.b - z19.b }, { z20.b - z23.b }
+; CHECK-NEXT:    udot za.s[w8, 0, vgx4], { z24.b - z27.b }, { z28.b - z31.b }
+; CHECK-NEXT:    ret
+entry:
+  %0 = tail call target("aarch64.svcount") @llvm.aarch64.sve.ptrue.c8()
+  %1 = tail call { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.ld1.pn.x4.nxv16i8(target("aarch64.svcount") %0, ptr %ptr)
+  %2 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %1, 0
+  %3 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %1, 1
+  %4 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %1, 2
+  %5 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %1, 3
+  %arrayidx2 = getelementptr inbounds i8, ptr %ptr, i64 %stride
+  %6 = tail call { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.ld1.pn.x4.nxv16i8(target("aarch64.svcount") %0, ptr %arrayidx2)
+  %7 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %6, 0
+  %8 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %6, 1
+  %9 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %6, 2
+  %10 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %6, 3
+  %mul3 = shl i64 %stride, 1
+  %arrayidx4 = getelementptr inbounds i8, ptr %ptr, i64 %mul3
+  %11 = tail call { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.ld1.pn.x4.nxv16i8(target("aarch64.svcount") %0, ptr %arrayidx4)
+  %12 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %11, 0
+  %13 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %11, 1
+  %14 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %11, 2
+  %15 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %11, 3
+  %mul5 = mul i64 %stride, 3
+  %arrayidx6 = getelementptr inbounds i8, ptr %ptr, i64 %mul5
+  %16 = tail call { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.ld1.pn.x4.nxv16i8(target("aarch64.svcount") %0, ptr %arrayidx6)
+  %17 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %16, 0
+  %18 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %16, 1
+  %19 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %16, 2
+  %20 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %16, 3
+  call void @llvm.aarch64.sme.udot.za32.vg1x4.nxv16i8(i32 0, <vscale x 16 x i8> %2, <vscale x 16 x i8> %7, <vscale x 16 x i8> %12, <vscale x 16 x i8> %17,
+                                                      <vscale x 16 x i8> %3, <vscale x 16 x i8> %8, <vscale x 16 x i8> %13, <vscale x 16 x i8> %18)
+  call void @llvm.aarch64.sme.udot.za32.vg1x4.nxv16i8(i32 0, <vscale x 16 x i8> %4, <vscale x 16 x i8> %9, <vscale x 16 x i8> %14, <vscale x 16 x i8> %19,
+                                                      <vscale x 16 x i8> %5, <vscale x 16 x i8> %10, <vscale x 16 x i8> %15, <vscale x 16 x i8> %20)
+  ret void
+}
+
 define void @udot_multi_za32_u8_vg1x2(i32 %slice, <vscale x 16 x i8> %unused, <vscale x 16 x i8> %zn0, <vscale x 16 x i8> %zn1, <vscale x 16 x i8> %zn2, <vscale x 16 x i8> %zn3) #0 {
 ; CHECK-LABEL: udot_multi_za32_u8_vg1x2:
 ; CHECK:       // %bb.0:
@@ -149,6 +219,28 @@ define void @usdot_multi_za32_u8_vg1x2(i32 %slice, <vscale x 16 x i8> %unused, <
   ret void
 }
 
+define void @usdot_multi_za32_u16_vg1x2_tuple(i64 %stride, ptr %ptr) #1 {
+; CHECK-LABEL: usdot_multi_za32_u16_vg1x2_tuple:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ptrue pn8.b
+; CHECK-NEXT:    mov w8, wzr
+; CHECK-NEXT:    ld1b { z16.b, z24.b }, pn8/z, [x1]
+; CHECK-NEXT:    ld1b { z17.b, z25.b }, pn8/z, [x1, x0]
+; CHECK-NEXT:    usdot za.s[w8, 0, vgx2], { z16.b, z17.b }, { z24.b, z25.b }
+; CHECK-NEXT:    ret
+entry:
+  %0 = tail call target("aarch64.svcount") @llvm.aarch64.sve.ptrue.c8()
+  %1 = tail call { <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.ld1.pn.x2.nxv16i8(target("aarch64.svcount") %0, ptr %ptr)
+  %2 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8> } %1, 0
+  %3 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8> } %1, 1
+  %arrayidx2 = getelementptr inbounds i8, ptr %ptr, i64 %stride
+  %4 = tail call { <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.ld1.pn.x2.nxv16i8(target("aarch64.svcount") %0, ptr %arrayidx2)
+  %5 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8> } %4, 0
+  %6 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8> } %4, 1
+  call void @llvm.aarch64.sme.usdot.za32.vg1x2.nxv16i8(i32 0, <vscale x 16 x i8> %2, <vscale x 16 x i8> %5, <vscale x 16 x i8> %3, <vscale x 16 x i8> %6)
+  ret void
+}
+
 define void @usdot_multi_za32_u8_vg1x4(i32 %slice, <vscale x 16 x i8> %unused, <vscale x 16 x i8> %zn0, <vscale x 16 x i8> %zn1, <vscale x 16 x i8> %zn2, <vscale x 16 x i8> %zn3,
 ; CHECK-LABEL: usdot_multi_za32_u8_vg1x4:
 ; CHECK:       // %bb.0:
@@ -174,6 +266,53 @@ define void @usdot_multi_za32_u8_vg1x4(i32 %slice, <vscale x 16 x i8> %unused, <
   ret void
 }
 
+define void @usdot_multi_za32_u16_vg1x4_tuple(i64 %stride, ptr %ptr) #1 {
+; CHECK-LABEL: usdot_multi_za32_u16_vg1x4_tuple:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    lsl x9, x0, #1
+; CHECK-NEXT:    ptrue pn8.b
+; CHECK-NEXT:    mov w8, wzr
+; CHECK-NEXT:    ld1b { z16.b, z20.b, z24.b, z28.b }, pn8/z, [x1]
+; CHECK-NEXT:    ld1b { z17.b, z21.b, z25.b, z29.b }, pn8/z, [x1, x0]
+; CHECK-NEXT:    add x10, x9, x0
+; CHECK-NEXT:    ld1b { z18.b, z22.b, z26.b, z30.b }, pn8/z, [x1, x9]
+; CHECK-NEXT:    ld1b { z19.b, z23.b, z27.b, z31.b }, pn8/z, [x1, x10]
+; CHECK-NEXT:    usdot za.s[w8, 0, vgx4], { z16.b - z19.b }, { z20.b - z23.b }
+; CHECK-NEXT:    usdot za.s[w8, 0, vgx4], { z24.b - z27.b }, { z28.b - z31.b }
+; CHECK-NEXT:    ret
+entry:
+  %0 = tail call target("aarch64.svcount") @llvm.aarch64.sve.ptrue.c8()
+  %1 = tail call { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.ld1.pn.x4.nxv16i8(target("aarch64.svcount") %0, ptr %ptr)
+  %2 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %1, 0
+  %3 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %1, 1
+  %4 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %1, 2
+  %5 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %1, 3
+  %arrayidx2 = getelementptr inbounds i8, ptr %ptr, i64 %stride
+  %6 = tail call { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.ld1.pn.x4.nxv16i8(target("aarch64.svcount") %0, ptr %arrayidx2)
+  %7 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %6, 0
+  %8 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %6, 1
+  %9 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %6, 2
+  %10 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %6, 3
+  %mul3 = shl i64 %stride, 1
+  %arrayidx4 = getelementptr inbounds i8, ptr %ptr, i64 %mul3
+  %11 = tail call { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.ld1.pn.x4.nxv16i8(target("aarch64.svcount") %0, ptr %arrayidx4)
+  %12 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %11, 0
+  %13 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %11, 1
+  %14 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %11, 2
+  %15 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %11, 3
+  %mul5 = mul i64 %stride, 3
+  %arrayidx6 = getelementptr inbounds i8, ptr %ptr, i64 %mul5
+  %16 = tail call { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.ld1.pn.x4.nxv16i8(target("aarch64.svcount") %0, ptr %arrayidx6)
+  %17 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %16, 0
+  %18 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %16, 1
+  %19 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %16, 2
+  %20 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %16, 3
+  call void @llvm.aarch64.sme.usdot.za32.vg1x4.nxv16i8(i32 0, <vscale x 16 x i8> %2, <vscale x 16 x i8> %7, <vscale x 16 x i8> %12, <vscale x 16 x i8> %17,
+                                                      <vscale x 16 x i8> %3, <vscale x 16 x i8> %8, <vscale x 16 x i8> %13, <vscale x 16 x i8> %18)
+  call void @llvm.aarch64.sme.usdot.za32.vg1x4.nxv16i8(i32 0, <vscale x 16 x i8> %4, <vscale x 16 x i8> %9, <vscale x 16 x i8> %14, <vscale x 16 x i8> %19,
+                                                      <vscale x 16 x i8> %5, <vscale x 16 x i8> %10, <vscale x 16 x i8> %15, <vscale x 16 x i8> %20)
+  ret void
+}
 
 ; == Multi, multi (signed) ==
 
@@ -194,6 +333,28 @@ define void @sdot_multi_za32_u16_vg1x2(i32 %slice, <vscale x 16 x i8> %unused, <
   ret void
 }
 
+define void @sdot_multi_za32_u16_vg1x2_tuple(i64 %stride, ptr %ptr) #0 {
+; CHECK-LABEL: sdot_multi_za32_u16_vg1x2_tuple:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ptrue pn8.b
+; CHECK-NEXT:    mov w8, wzr
+; CHECK-NEXT:    ld1b { z16.b, z24.b }, pn8/z, [x1]
+; CHECK-NEXT:    ld1b { z17.b, z25.b }, pn8/z, [x1, x0]
+; CHECK-NEXT:    sdot za.s[w8, 0, vgx2], { z16.b, z17.b }, { z24.b, z25.b }
+; CHECK-NEXT:    ret
+entry:
+  %0 = tail call target("aarch64.svcount") @llvm.aarch64.sve.ptrue.c8()
+  %1 = tail call { <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.ld1.pn.x2.nxv16i8(target("aarch64.svcount") %0, ptr %ptr)
+  %2 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8> } %1, 0
+  %3 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8> } %1, 1
+  %arrayidx2 = getelementptr inbounds i8, ptr %ptr, i64 %stride
+  %4 = tail call { <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.ld1.pn.x2.nxv16i8(target("aarch64.svcount") %0, ptr %arrayidx2)
+  %5 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8> } %4, 0
+  %6 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8> } %4, 1
+  call void @llvm.aarch64.sme.sdot.za32.vg1x2.nxv16i8(i32 0, <vscale x 16 x i8> %2, <vscale x 16 x i8> %5, <vscale x 16 x i8> %3, <vscale x 16 x i8> %6)
+  ret void
+}
+
 define void @sdot_multi_za32_u16_vg1x4(i32 %slice, <vscale x 16 x i8> %unused, <vscale x 8 x i16> %zn0, <vscale x 8 x i16> %zn1, <vscale x 8 x i16> %zn2, <vscale x 8 x i16> %zn3,
 ; CHECK-LABEL: sdot_multi_za32_u16_vg1x4:
 ; CHECK:       // %bb.0:
@@ -219,6 +380,54 @@ define void @sdot_multi_za32_u16_vg1x4(i32 %slice, <vscale x 16 x i8> %unused, <
   ret void
 }
 
+define void @sdot_multi_za32_u16_vg1x4_tuple(i64 %stride, ptr %ptr) #0 {
+; CHECK-LABEL: sdot_multi_za32_u16_vg1x4_tuple:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    lsl x9, x0, #1
+; CHECK-NEXT:    ptrue pn8.b
+; CHECK-NEXT:    mov w8, wzr
+; CHECK-NEXT:    ld1b { z16.b, z20.b, z24.b, z28.b }, pn8/z, [x1]
+; CHECK-NEXT:    ld1b { z17.b, z21.b, z25.b, z29.b }, pn8/z, [x1, x0]
+; CHECK-NEXT:    add x10, x9, x0
+; CHECK-NEXT:    ld1b { z18.b, z22.b, z26.b, z30.b }, pn8/z, [x1, x9]
+; CHECK-NEXT:    ld1b { z19.b, z23.b, z27.b, z31.b }, pn8/z, [x1, x10]
+; CHECK-NEXT:    sdot za.s[w8, 0, vgx4], { z16.b - z19.b }, { z20.b - z23.b }
+; CHECK-NEXT:    sdot za.s[w8, 0, vgx4], { z24.b - z27.b }, { z28.b - z31.b }
+; CHECK-NEXT:    ret
+entry:
+  %0 = tail call target("aarch64.svcount") @llvm.aarch64.sve.ptrue.c8()
+  %1 = tail call { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.ld1.pn.x4.nxv16i8(target("aarch64.svcount") %0, ptr %ptr)
+  %2 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %1, 0
+  %3 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %1, 1
+  %4 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %1, 2
+  %5 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %1, 3
+  %arrayidx2 = getelementptr inbounds i8, ptr %ptr, i64 %stride
+  %6 = tail call { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.ld1.pn.x4.nxv16i8(target("aarch64.svcount") %0, ptr %arrayidx2)
+  %7 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %6, 0
+  %8 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %6, 1
+  %9 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %6, 2
+  %10 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %6, 3
+  %mul3 = shl i64 %stride, 1
+  %arrayidx4 = getelementptr inbounds i8, ptr %ptr, i64 %mul3
+  %11 = tail call { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.ld1.pn.x4.nxv16i8(target("aarch64.svcount") %0, ptr %arrayidx4)
+  %12 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %11, 0
+  %13 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %11, 1
+  %14 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %11, 2
+  %15 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %11, 3
+  %mul5 = mul i64 %stride, 3
+  %arrayidx6 = getelementptr inbounds i8, ptr %ptr, i64 %mul5
+  %16 = tail call { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.ld1.pn.x4.nxv16i8(target("aarch64.svcount") %0, ptr %arrayidx6)
+  %17 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %16, 0
+  %18 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %16, 1
+  %19 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %16, 2
+  %20 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %16, 3
+  call void @llvm.aarch64.sme.sdot.za32.vg1x4.nxv16i8(i32 0, <vscale x 16 x i8> %2, <vscale x 16 x i8> %7, <vscale x 16 x i8> %12, <vscale x 16 x i8> %17,
+                                                      <vscale x 16 x i8> %3, <vscale x 16 x i8> %8, <vscale x 16 x i8> %13, <vscale x 16 x i8> %18)
+  call void @llvm.aarch64.sme.sdot.za32.vg1x4.nxv16i8(i32 0, <vscale x 16 x i8> %4, <vscale x 16 x i8> %9, <vscale x 16 x i8> %14, <vscale x 16 x i8> %19,
+                                                      <vscale x 16 x i8> %5, <vscale x 16 x i8> %10, <vscale x 16 x i8> %15, <vscale x 16 x i8> %20)
+  ret void
+}
+
 define void @sdot_multi_za32_u8_vg1x2(i32 %slice, <vscale x 16 x i8> %unused, <vscale x 16 x i8> %zn0, <vscale x 16 x i8> %zn1, <vscale x 16 x i8> %zn2, <vscale x 16 x i8> %zn3) #0 {
 ; CHECK-LABEL: sdot_multi_za32_u8_vg1x2:
 ; CHECK:       // %bb.0:
@@ -278,6 +487,29 @@ define void @sdot_multi_za64_u16_vg1x2(i32 %slice, <vscale x 16 x i8> %unused, <
   ret void
 }
 
+define void @sdot_multi_za64_u16_vg1x2_tuple(i64 %stride, ptr %ptr) #1 {
+; CHECK-LABEL: sdot_multi_za64_u16_vg1x2_tuple:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ptrue pn8.b
+; CHECK-NEXT:    add x9, x1, x0
+; CHECK-NEXT:    mov w8, wzr
+; CHECK-NEXT:    ld1h { z16.h, z24.h }, pn8/z, [x1]
+; CHECK-NEXT:    ld1h { z17.h, z25.h }, pn8/z, [x9]
+; CHECK-NEXT:    sdot za.d[w8, 0, vgx2], { z16.h, z17.h }, { z24.h, z25.h }
+; CHECK-NEXT:    ret
+entry:
+  %0 = tail call target("aarch64.svcount") @llvm.aarch64.sve.ptrue.c8()
+  %1 = tail call { <vscale x 8 x i16>, <vscale x 8 x i16> } @llvm.aarch64.sve.ld1.pn.x2.nxv8i16(target("aarch64.svcount") %0, ptr %ptr)
+  %2 = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16> } %1, 0
+  %3 = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16> } %1, 1
+  %arrayidx2 = getelementptr inbounds i8, ptr %ptr, i64 %stride
+  %4 = tail call { <vscale x 8 x i16>, <vscale x 8 x i16> } @llvm.aarch64.sve.ld1.pn.x2.nxv8i16(target("aarch64.svcount") %0, ptr %arrayidx2)
+  %5 = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16> } %4, 0
+  %6 = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16> } %4, 1
+  call void @llvm.aarch64.sme.sdot.za64.vg1x2.nxv8i16(i32 0, <vscale x 8 x i16> %2, <vscale x 8 x i16> %5, <vscale x 8 x i16> %3, <vscale x 8 x i16> %6)
+  ret void
+}
+
 define void @sdot_multi_za64_u16_vg1x4(i32 %slice, <vscale x 16 x i8> %unused, <vscale x 8 x i16> %zn0, <vscale x 8 x i16> %zn1, <vscale x 8 x i16> %zn2, <vscale x 8 x i16> %zn3,
 ; CHECK-LABEL: sdot_multi_za64_u16_vg1x4:
 ; CHECK:       // %bb.0:
@@ -303,6 +535,54 @@ define void @sdot_multi_za64_u16_vg1x4(i32 %slice, <vscale x 16 x i8> %unused, <
   ret void
 }
 
+define void @sdot_multi_za64_u16_vg1x4_tuple(i64 %stride, ptr %ptr) #1 {
+; CHECK-LABEL: sdot_multi_za64_u16_vg1x4_tuple:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    add x9, x0, x0, lsl #1
+; CHECK-NEXT:    ptrue pn8.b
+; CHECK-NEXT:    add x10, x1, x0
+; CHECK-NEXT:    ld1h { z16.h, z20.h, z24.h, z28.h }, pn8/z, [x1]
+; CHECK-NEXT:    ld1h { z17.h, z21.h, z25.h, z29.h }, pn8/z, [x10]
+; CHECK-NEXT:    ld1h { z18.h, z22.h, z26.h, z30.h }, pn8/z, [x1, x0, lsl #1]
+; CHECK-NEXT:    add x9, x1, x9
+; CHECK-NEXT:    mov w8, wzr
+; CHECK-NEXT:    ld1h { z19.h, z23.h, z27.h, z31.h }, pn8/z, [x9]
+; CHECK-NEXT:    sdot za.d[w8, 0, vgx4], { z16.h - z19.h }, { z20.h - z23.h }
+; CHECK-NEXT:    sdot za.d[w8, 0, vgx4], { z24.h - z27.h }, { z28.h - z31.h }
+; CHECK-NEXT:    ret
+entry:
+  %0 = tail call target("aarch64.svcount") @llvm.aarch64.sve.ptrue.c8()
+  %1 = tail call { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } @llvm.aarch64.sve.ld1.pn.x4.nxv8i16(target("aarch64.svcount") %0, ptr %ptr)
+  %2 = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } %1, 0
+  %3 = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } %1, 1
+  %4 = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } %1, 2
+  %5 = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } %1, 3
+  %arrayidx2 = getelementptr inbounds i8, ptr %ptr, i64 %stride
+  %6 = tail call { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } @llvm.aarch64.sve.ld1.pn.x4.nxv8i16(target("aarch64.svcount") %0, ptr %arrayidx2)
+  %7 = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } %6, 0
+  %8 = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } %6, 1
+  %9 = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } %6, 2
+  %10 = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } %6, 3
+  %mul3 = shl i64 %stride, 1
+  %arrayidx4 = getelementptr inbounds i8, ptr %ptr, i64 %mul3
+  %11 = tail call { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } @llvm.aarch64.sve.ld1.pn.x4.nxv8i16(target("aarch64.svcount") %0, ptr %arrayidx4)
+  %12 = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } %11, 0
+  %13 = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } %11, 1
+  %14 = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } %11, 2
+  %15 = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } %11, 3
+  %mul5 = mul i64 %stride, 3
+  %arrayidx6 = getelementptr inbounds i8, ptr %ptr, i64 %mul5
+  %16 = tail call { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } @llvm.aarch64.sve.ld1.pn.x4.nxv8i16(target("aarch64.svcount") %0, ptr %arrayidx6)
+  %17 = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } %16, 0
+  %18 = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } %16, 1
+  %19 = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } %16, 2
+  %20 = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } %16, 3
+  call void @llvm.aarch64.sme.sdot.za64.vg1x4.nxv8i16(i32 0, <vscale x 8 x i16> %2, <vscale x 8 x i16> %7, <vscale x 8 x i16> %12, <vscale x 8 x i16> %17,
+                                                      <vscale x 8 x i16> %3, <vscale x 8 x i16> %8, <vscale x 8 x i16> %13, <vscale x 8 x i16> %18)
+  call void @llvm.aarch64.sme.sdot.za64.vg1x4.nxv8i16(i32 0, <vscale x 8 x i16> %4, <vscale x 8 x i16> %9, <vscale x 8 x i16> %14, <vscale x 8 x i16> %19,
+                                                      <vscale x 8 x i16> %5, <vscale x 8 x i16> %10, <vscale x 8 x i16> %15, <vscale x 8 x i16> %20)
+  ret void
+}
 
 ; == Multi, single (unsigned) ==
 
@@ -354,6 +634,53 @@ entry:
   ret void
 }
 
+define void @udot_single_za32_u16_vg1x2_x4load_x2tuple(ptr %ptr, i64 %stride, <vscale x 8 x i16> %zn) #0 {
+; CHECK-LABEL: udot_single_za32_u16_vg1x2_x4load_x2tuple:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NEXT:    addvl sp, sp, #-5
+; CHECK-NEXT:    str p8, [sp, #7, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    ptrue pn8.b
+; CHECK-NEXT:    add x9, x0, x1
+; CHECK-NEXT:    str z14, [sp, #1, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    mov w8, wzr
+; CHECK-NEXT:    str z13, [sp, #2, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z10, [sp, #3, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z9, [sp, #4, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    ld1h { z1.h, z5.h, z9.h, z13.h }, pn8/z, [x0]
+; CHECK-NEXT:    ld1h { z2.h, z6.h, z10.h, z14.h }, pn8/z, [x9]
+; CHECK-NEXT:    udot za.s[w8, 0, vgx2], { z1.h, z2.h }, z0.h
+; CHECK-NEXT:    udot za.s[w8, 0, vgx2], { z5.h, z6.h }, z0.h
+; CHECK-NEXT:    udot za.s[w8, 0, vgx2], { z9.h, z10.h }, z0.h
+; CHECK-NEXT:    udot za.s[w8, 0, vgx2], { z13.h, z14.h }, z0.h
+; CHECK-NEXT:    ldr z14, [sp, #1, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z13, [sp, #2, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z10, [sp, #3, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z9, [sp, #4, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr p8, [sp, #7, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    addvl sp, sp, #5
+; CHECK-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    ret
+entry:
+  %0 = tail call target("aarch64.svcount") @llvm.aarch64.sve.ptrue.c8()
+  %1 = tail call { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } @llvm.aarch64.sve.ld1.pn.x4.nxv8i16(target("aarch64.svcount") %0, ptr %ptr)
+  %2 = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } %1, 0
+  %3 = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } %1, 1
+  %4 = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } %1, 2
+  %5 = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } %1, 3
+  %arrayidx2 = getelementptr inbounds i8, ptr %ptr, i64 %stride
+  %6 = tail call { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } @llvm.aarch64.sve.ld1.pn.x4.nxv8i16(target("aarch64.svcount") %0, ptr %arrayidx2)
+  %7 = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } %6, 0
+  %8 = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } %6, 1
+  %9 = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } %6, 2
+  %10 = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } %6, 3
+  call void @llvm.aarch64.sme.udot.single.za32.vg1x2.nxv8i16(i32 0, <vscale x 8 x i16> %2, <vscale x 8 x i16> %7, <vscale x 8 x i16> %zn)
+  call void @llvm.aarch64.sme.udot.single.za32.vg1x2.nxv8i16(i32 0, <vscale x 8 x i16> %3, <vscale x 8 x i16> %8, <vscale x 8 x i16> %zn)
+  call void @llvm.aarch64.sme.udot.single.za32.vg1x2.nxv8i16(i32 0, <vscale x 8 x i16> %4, <vscale x 8 x i16> %9, <vscale x 8 x i16> %zn)
+  call void @llvm.aarch64.sme.udot.single.za32.vg1x2.nxv8i16(i32 0, <vscale x 8 x i16> %5, <vscale x 8 x i16> %10, <vscale x 8 x i16> %zn)
+  ret void
+}
+
 define void @udot_single_za32_u16_vg1x4(i32 %slice, <vscale x 16 x i8> %unused, <vscale x 8 x i16> %zn0, <vscale x 8 x i16> %zn1, <vscale x 8 x i16> %zn2, <vscale x 8 x i16> %zn3, <vscale x 8 x i16> %zn4) #0 {
 ; CHECK-LABEL: udot_single_za32_u16_vg1x4:
 ; CHECK:       // %bb.0:
@@ -371,47 +698,39 @@ define void @udot_single_za32_u16_vg1x4_tuple(ptr %ptr, i64 %stride, <vscale x 8
 ; CHECK-LABEL: udot_single_za32_u16_vg1x4_tuple:
 ; CHECK:       // %bb.0: // %entry
 ; CHECK-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill
-; CHECK-NEXT:    addvl sp, sp, #-11
+; CHECK-NEXT:    addvl sp, sp, #-9
 ; CHECK-NEXT:    add x9, x1, x1, lsl #1
 ; CHECK-NEXT:    str p8, [sp, #7, mul vl] // 2-byte Folded Spill
 ; CHECK-NEXT:    ptrue pn8.b
-; CHECK-NEXT:    str z20, [sp, #1, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z23, [sp, #1, mul vl] // 16-byte Folded Spill
 ; CHECK-NEXT:    add x10, x0, x1
 ; CHECK-NEXT:    mov w8, wzr
-; CHECK-NEXT:    str z16, [sp, #2, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z22, [sp, #2, mul vl] // 16-byte Folded Spill
 ; CHECK-NEXT:    add x9, x0, x9
-; CHECK-NEXT:    str z15, [sp, #3, mul vl] // 16-byte Folded Spill
-; CHECK-NEXT:    str z14, [sp, #4, mul vl] // 16-byte Folded Spill
-; CHECK-NEXT:    str z13, [sp, #5, mul vl] // 16-byte Folded Spill
-; CHECK-NEXT:    str z12, [sp, #6, mul vl] // 16-byte Folded Spill
-; CHECK-NEXT:    str z11, [sp, #7, mul vl] // 16-byte Folded Spill
-; CHECK-NEXT:    str z10, [sp, #8, mul vl] // 16-byte Folded Spill
-; CHECK-NEXT:    str z9, [sp, #9, mul vl] // 16-byte Folded Spill
-; CHECK-NEXT:    str z8, [sp, #10, mul vl] // 16-byte Folded Spill
-; CHECK-NEXT:    ld1h { z1.h, z5.h, z9.h, z13.h }, pn8/z, [x0]
-; CHECK-NEXT:    ld1h { z2.h, z6.h, z10.h, z14.h }, pn8/z, [x10]
-; CHECK-NEXT:    ld1h { z3.h, z7.h, z11.h, z15.h }, pn8/z, [x0, x1, lsl #1]
-; CHECK-NEXT:    ld1h { z16.h, z20.h, z24.h, z28.h }, pn8/z, [x9]
-; CHECK-NEXT:    mov z4.d, z16.d
-; CHECK-NEXT:    mov z8.d, z20.d
-; CHECK-NEXT:    mov z12.d, z24.d
-; CHECK-NEXT:    mov z16.d, z28.d
-; CHECK-NEXT:    udot za.s[w8, 0, vgx4], { z1.h - z4.h }, z0.h
-; CHECK-NEXT:    udot za.s[w8, 0, vgx4], { z5.h - z8.h }, z0.h
-; CHECK-NEXT:    udot za.s[w8, 0, vgx4], { z9.h - z12.h }, z0.h
-; CHECK-NEXT:    udot za.s[w8, 0, vgx4], { z13.h - z16.h }, z0.h
-; CHECK-NEXT:    ldr z20, [sp, #1, mul vl] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr z16, [sp, #2, mul vl] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr z15, [sp, #3, mul vl] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr z14, [sp, #4, mul vl] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr z13, [sp, #5, mul vl] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr z12, [sp, #6, mul vl] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr z11, [sp, #7, mul vl] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr z10, [sp, #8, mul vl] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr z9, [sp, #9, mul vl] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr z8, [sp, #10, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    str z21, [sp, #3, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z20, [sp, #4, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z19, [sp, #5, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z18, [sp, #6, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z17, [sp, #7, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z16, [sp, #8, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    ld1h { z16.h, z20.h, z24.h, z28.h }, pn8/z, [x0]
+; CHECK-NEXT:    ld1h { z17.h, z21.h, z25.h, z29.h }, pn8/z, [x10]
+; CHECK-NEXT:    ld1h { z18.h, z22.h, z26.h, z30.h }, pn8/z, [x0, x1, lsl #1]
+; CHECK-NEXT:    ld1h { z19.h, z23.h, z27.h, z31.h }, pn8/z, [x9]
+; CHECK-NEXT:    udot za.s[w8, 0, vgx4], { z16.h - z19.h }, z0.h
+; CHECK-NEXT:    udot za.s[w8, 0, vgx4], { z20.h - z23.h }, z0.h
+; CHECK-NEXT:    udot za.s[w8, 0, vgx4], { z24.h - z27.h }, z0.h
+; CHECK-NEXT:    udot za.s[w8, 0, vgx4], { z28.h - z31.h }, z0.h
+; CHECK-NEXT:    ldr z23, [sp, #1, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z22, [sp, #2, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z21, [sp, #3, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z20, [sp, #4, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z19, [sp, #5, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z18, [sp, #6, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z17, [sp, #7, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z16, [sp, #8, mul vl] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldr p8, [sp, #7, mul vl] // 2-byte Folded Reload
-; CHECK-NEXT:    addvl sp, sp, #11
+; CHECK-NEXT:    addvl sp, sp, #9
 ; CHECK-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
 ; CHECK-NEXT:    ret
 entry:
@@ -564,46 +883,38 @@ define void @usdot_single_za32_u16_vg1x4_tuple(ptr %ptr, i64 %stride, <vscale x
 ; CHECK-LABEL: usdot_single_za32_u16_vg1x4_tuple:
 ; CHECK:       // %bb.0: // %entry
 ; CHECK-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill
-; CHECK-NEXT:    addvl sp, sp, #-11
+; CHECK-NEXT:    addvl sp, sp, #-9
 ; CHECK-NEXT:    lsl x9, x1, #1
 ; CHECK-NEXT:    str p8, [sp, #7, mul vl] // 2-byte Folded Spill
 ; CHECK-NEXT:    ptrue pn8.b
-; CHECK-NEXT:    str z20, [sp, #1, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z23, [sp, #1, mul vl] // 16-byte Folded Spill
 ; CHECK-NEXT:    mov w8, wzr
-; CHECK-NEXT:    str z16, [sp, #2, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z22, [sp, #2, mul vl] // 16-byte Folded Spill
 ; CHECK-NEXT:    add x10, x9, x1
-; CHECK-NEXT:    str z15, [sp, #3, mul vl] // 16-byte Folded Spill
-; CHECK-NEXT:    str z14, [sp, #4, mul vl] // 16-byte Folded Spill
-; CHECK-NEXT:    str z13, [sp, #5, mul vl] // 16-byte Folded Spill
-; CHECK-NEXT:    str z12, [sp, #6, mul vl] // 16-byte Folded Spill
-; CHECK-NEXT:    str z11, [sp, #7, mul vl] // 16-byte Folded Spill
-; CHECK-NEXT:    str z10, [sp, #8, mul vl] // 16-byte Folded Spill
-; CHECK-NEXT:    str z9, [sp, #9, mul vl] // 16-byte Folded Spill
-; CHECK-NEXT:    str z8, [sp, #10, mul vl] // 16-byte Folded Spill
-; CHECK-NEXT:    ld1b { z1.b, z5.b, z9.b, z13.b }, pn8/z, [x0]
-; CHECK-NEXT:    ld1b { z2.b, z6.b, z10.b, z14.b }, pn8/z, [x0, x1]
-; CHECK-NEXT:    ld1b { z3.b, z7.b, z11.b, z15.b }, pn8/z, [x0, x9]
-; CHECK-NEXT:    ld1b { z16.b, z20.b, z24.b, z28.b }, pn8/z, [x0, x10]
-; CHECK-NEXT:    mov z4.d, z16.d
-; CHECK-NEXT:    mov z8.d, z20.d
-; CHECK-NEXT:    mov z12.d, z24.d
-; CHECK-NEXT:    mov z16.d, z28.d
-; CHECK-NEXT:    usdot za.s[w8, 0, vgx4], { z1.b - z4.b }, z0.b
-; CHECK-NEXT:    usdot za.s[w8, 0, vgx4], { z5.b - z8.b }, z0.b
-; CHECK-NEXT:    usdot za.s[w8, 0, vgx4], { z9.b - z12.b }, z0.b
-; CHECK-NEXT:    usdot za.s[w8, 0, vgx4], { z13.b - z16.b }, z0.b
-; CHECK-NEXT:    ldr z20, [sp, #1, mul vl] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr z16, [sp, #2, mul vl] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr z15, [sp, #3, mul vl] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr z14, [sp, #4, mul vl] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr z13, [sp, #5, mul vl] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr z12, [sp, #6, mul vl] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr z11, [sp, #7, mul vl] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr z10, [sp, #8, mul vl] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr z9, [sp, #9, mul vl] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr z8, [sp, #10, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    str z21, [sp, #3, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z20, [sp, #4, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z19, [sp, #5, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z18, [sp, #6, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z17, [sp, #7, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z16, [sp, #8, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    ld1b { z16.b, z20.b, z24.b, z28.b }, pn8/z, [x0]
+; CHECK-NEXT:    ld1b { z17.b, z21.b, z25.b, z29.b }, pn8/z, [x0, x1]
+; CHECK-NEXT:    ld1b { z18.b, z22.b, z26.b, z30.b }, pn8/z, [x0, x9]
+; CHECK-NEXT:    ld1b { z19.b, z23.b, z27.b, z31.b }, pn8/z, [x0, x10]
+; CHECK-NEXT:    usdot za.s[w8, 0, vgx4], { z16.b - z19.b }, z0.b
+; CHECK-NEXT:    usdot za.s[w8, 0, vgx4], { z20.b - z23.b }, z0.b
+; CHECK-NEXT:    usdot za.s[w8, 0, vgx4], { z24.b - z27.b }, z0.b
+; CHECK-NEXT:    usdot za.s[w8, 0, vgx4], { z28.b - z31.b }, z0.b
+; CHECK-NEXT:    ldr z23, [sp, #1, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z22, [sp, #2, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z21, [sp, #3, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z20, [sp, #4, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z19, [sp, #5, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z18, [sp, #6, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z17, [sp, #7, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z16, [sp, #8, mul vl] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldr p8, [sp, #7, mul vl] // 2-byte Folded Reload
-; CHECK-NEXT:    addvl sp, sp, #11
+; CHECK-NEXT:    addvl sp, sp, #9
 ; CHECK-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
 ; CHECK-NEXT:    ret
 entry:
@@ -707,47 +1018,39 @@ define void @sdot_single_za32_u16_vg1x4_tuple(ptr %ptr, i64 %stride, <vscale x 8
 ; CHECK-LABEL: sdot_single_za32_u16_vg1x4_tuple:
 ; CHECK:       // %bb.0: // %entry
 ; CHECK-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill
-; CHECK-NEXT:    addvl sp, sp, #-11
+; CHECK-NEXT:    addvl sp, sp, #-9
 ; CHECK-NEXT:    add x9, x1, x1, lsl #1
 ; CHECK-NEXT:    str p8, [sp, #7, mul vl] // 2-byte Folded Spill
 ; CHECK-NEXT:    ptrue pn8.b
-; CHECK-NEXT:    str z20, [sp, #1, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z23, [sp, #1, mul vl] // 16-byte Folded Spill
 ; CHECK-NEXT:    add x10, x0, x1
 ; CHECK-NEXT:    mov w8, wzr
-; CHECK-NEXT:    str z16, [sp, #2, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z22, [sp, #2, mul vl] // 16-byte Folded Spill
 ; CHECK-NEXT:    add x9, x0, x9
-; CHECK-NEXT:    str z15, [sp, #3, mul vl] // 16-byte Folded Spill
-; CHECK-NEXT:    str z14, [sp, #4, mul vl] // 16-byte Folded Spill
-; CHECK-NEXT:    str z13, [sp, #5, mul vl] // 16-byte Folded Spill
-; CHECK-NEXT:    str z12, [sp, #6, mul vl] // 16-byte Folded Spill
-; CHECK-NEXT:    str z11, [sp, #7, mul vl] // 16-byte Folded Spill
-; CHECK-NEXT:    str z10, [sp, #8, mul vl] // 16-byte Folded Spill
-; CHECK-NEXT:    str z9, [sp, #9, mul vl] // 16-byte Folded Spill
-; CHECK-NEXT:    str z8, [sp, #10, mul vl] // 16-byte Folded Spill
-; CHECK-NEXT:    ld1h { z1.h, z5.h, z9.h, z13.h }, pn8/z, [x0]
-; CHECK-NEXT:    ld1h { z2.h, z6.h, z10.h, z14.h }, pn8/z, [x10]
-; CHECK-NEXT:    ld1h { z3.h, z7.h, z11.h, z15.h }, pn8/z, [x0, x1, lsl #1]
-; CHECK-NEXT:    ld1h { z16.h, z20.h, z24.h, z28.h }, pn8/z, [x9]
-; CHECK-NEXT:    mov z4.d, z16.d
-; CHECK-NEXT:    mov z8.d, z20.d
-; CHECK-NEXT:    mov z12.d, z24.d
-; CHECK-NEXT:    mov z16.d, z28.d
-; CHECK-NEXT:    sdot za.s[w8, 0, vgx4], { z1.h - z4.h }, z0.h
-; CHECK-NEXT:    sdot za.s[w8, 0, vgx4], { z5.h - z8.h }, z0.h
-; CHECK-NEXT:    sdot za.s[w8, 0, vgx4], { z9.h - z12.h }, z0.h
-; CHECK-NEXT:    sdot za.s[w8, 0, vgx4], { z13.h - z16.h }, z0.h
-; CHECK-NEXT:    ldr z20, [sp, #1, mul vl] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr z16, [sp, #2, mul vl] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr z15, [sp, #3, mul vl] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr z14, [sp, #4, mul vl] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr z13, [sp, #5, mul vl] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr z12, [sp, #6, mul vl] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr z11, [sp, #7, mul vl] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr z10, [sp, #8, mul vl] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr z9, [sp, #9, mul vl] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr z8, [sp, #10, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    str z21, [sp, #3, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z20, [sp, #4, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z19, [sp, #5, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z18, [sp, #6, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z17, [sp, #7, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z16, [sp, #8, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    ld1h { z16.h, z20.h, z24.h, z28.h }, pn8/z, [x0]
+; CHECK-NEXT:    ld1h { z17.h, z21.h, z25.h, z29.h }, pn8/z, [x10]
+; CHECK-NEXT:    ld1h { z18.h, z22.h, z26.h, z30.h }, pn8/z, [x0, x1, lsl #1]
+; CHECK-NEXT:    ld1h { z19.h, z23.h, z27.h, z31.h }, pn8/z, [x9]
+; CHECK-NEXT:    sdot za.s[w8, 0, vgx4], { z16.h - z19.h }, z0.h
+; CHECK-NEXT:    sdot za.s[w8, 0, vgx4], { z20.h - z23.h }, z0.h
+; CHECK-NEXT:    sdot za.s[w8, 0, vgx4], { z24.h - z27.h }, z0.h
+; CHECK-NEXT:    sdot za.s[w8, 0, vgx4], { z28.h - z31.h }, z0.h
+; CHECK-NEXT:    ldr z23, [sp, #1, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z22, [sp, #2, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z21, [sp, #3, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z20, [sp, #4, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z19, [sp, #5, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z18, [sp, #6, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z17, [sp, #7, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z16, [sp, #8, mul vl] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldr p8, [sp, #7, mul vl] // 2-byte Folded Reload
-; CHECK-NEXT:    addvl sp, sp, #11
+; CHECK-NEXT:    addvl sp, sp, #9
 ; CHECK-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
 ; CHECK-NEXT:    ret
 entry:
@@ -900,46 +1203,38 @@ define void @sudot_single_za32_u16_vg1x4_tuple(ptr %ptr, i64 %stride, <vscale x
 ; CHECK-LABEL: sudot_single_za32_u16_vg1x4_tuple:
 ; CHECK:       // %bb.0: // %entry
 ; CHECK-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill
-; CHECK-NEXT:    addvl sp, sp, #-11
+; CHECK-NEXT:    addvl sp, sp, #-9
 ; CHECK-NEXT:    lsl x9, x1, #1
 ; CHECK-NEXT:    str p8, [sp, #7, mul vl] // 2-byte Folded Spill
 ; CHECK-NEXT:    ptrue pn8.b
-; CHECK-NEXT:    str z20, [sp, #1, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z23, [sp, #1, mul vl] // 16-byte Folded Spill
 ; CHECK-NEXT:    mov w8, wzr
-; CHECK-NEXT:    str z16, [sp, #2, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z22, [sp, #2, mul vl] // 16-byte Folded Spill
 ; CHECK-NEXT:    add x10, x9, x1
-; CHECK-NEXT:    str z15, [sp, #3, mul vl] // 16-byte Folded Spill
-; CHECK-NEXT:    str z14, [sp, #4, mul vl] // 16-byte Folded Spill
-; CHECK-NEXT:    str z13, [sp, #5, mul vl] // 16-byte Folded Spill
-; CHECK-NEXT:    str z12, [sp, #6, mul vl] // 16-byte Folded Spill
-; CHECK-NEXT:    str z11, [sp, #7, mul vl] // 16-byte Folded Spill
-; CHECK-NEXT:    str z10, [sp, #8, mul vl] // 16-byte Folded Spill
-; CHECK-NEXT:    str z9, [sp, #9, mul vl] // 16-byte Folded Spill
-; CHECK-NEXT:    str z8, [sp, #10, mul vl] // 16-byte Folded Spill
-; CHECK-NEXT:    ld1b { z1.b, z5.b, z9.b, z13.b }, pn8/z, [x0]
-; CHECK-NEXT:    ld1b { z2.b, z6.b, z10.b, z14.b }, pn8/z, [x0, x1]
-; CHECK-NEXT:    ld1b { z3.b, z7.b, z11.b, z15.b }, pn8/z, [x0, x9]
-; CHECK-NEXT:    ld1b { z16.b, z20.b, z24.b, z28.b }, pn8/z, [x0, x10]
-; CHECK-NEXT:    mov z4.d, z16.d
-; CHECK-NEXT:    mov z8.d, z20.d
-; CHECK-NEXT:    mov z12.d, z24.d
-; CHECK-NEXT:    mov z16.d, z28.d
-; CHECK-NEXT:    sudot za.s[w8, 0, vgx4], { z1.b - z4.b }, z0.b
-; CHECK-NEXT:    sudot za.s[w8, 0, vgx4], { z5.b - z8.b }, z0.b
-; CHECK-NEXT:    sudot za.s[w8, 0, vgx4], { z9.b - z12.b }, z0.b
-; CHECK-NEXT:    sudot za.s[w8, 0, vgx4], { z13.b - z16.b }, z0.b
-; CHECK-NEXT:    ldr z20, [sp, #1, mul vl] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr z16, [sp, #2, mul vl] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr z15, [sp, #3, mul vl] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr z14, [sp, #4, mul vl] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr z13, [sp, #5, mul vl] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr z12, [sp, #6, mul vl] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr z11, [sp, #7, mul vl] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr z10, [sp, #8, mul vl] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr z9, [sp, #9, mul vl] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr z8, [sp, #10, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    str z21, [sp, #3, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z20, [sp, #4, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z19, [sp, #5, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z18, [sp, #6, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z17, [sp, #7, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z16, [sp, #8, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    ld1b { z16.b, z20.b, z24.b, z28.b }, pn8/z, [x0]
+; CHECK-NEXT:    ld1b { z17.b, z21.b, z25.b, z29.b }, pn8/z, [x0, x1]
+; CHECK-NEXT:    ld1b { z18.b, z22.b, z26.b, z30.b }, pn8/z, [x0, x9]
+; CHECK-NEXT:    ld1b { z19.b, z23.b, z27.b, z31.b }, pn8/z, [x0, x10]
+; CHECK-NEXT:    sudot za.s[w8, 0, vgx4], { z16.b - z19.b }, z0.b
+; CHECK-NEXT:    sudot za.s[w8, 0, vgx4], { z20.b - z23.b }, z0.b
+; CHECK-NEXT:    sudot za.s[w8, 0, vgx4], { z24.b - z27.b }, z0.b
+; CHECK-NEXT:    sudot za.s[w8, 0, vgx4], { z28.b - z31.b }, z0.b
+; CHECK-NEXT:    ldr z23, [sp, #1, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z22, [sp, #2, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z21, [sp, #3, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z20, [sp, #4, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z19, [sp, #5, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z18, [sp, #6, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z17, [sp, #7, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z16, [sp, #8, mul vl] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldr p8, [sp, #7, mul vl] // 2-byte Folded Reload
-; CHECK-NEXT:    addvl sp, sp, #11
+; CHECK-NEXT:    addvl sp, sp, #9
 ; CHECK-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
 ; CHECK-NEXT:    ret
 entry:
@@ -1074,15 +1369,17 @@ define void @udot_form_2x_tuple_svecc(ptr %ptr, i64 %stride, <vscale x 16 x i8>
 ; CHECK-NEXT:    str p8, [sp, #7, mul vl] // 2-byte Folded Spill
 ; CHECK-NEXT:    ptrue pn8.b
 ; CHECK-NEXT:    mov w8, wzr
-; CHECK-NEXT:    str z9, [sp, #1, mul vl] // 16-byte Folded Spill
-; CHECK-NEXT:    str z8, [sp, #2, mul vl] // 16-byte Folded Spill
-; CHECK-NEXT:    ld1b { z0.b, z8.b }, pn8/z, [x0]
-; CHECK-NEXT:    ld1b { z1.b, z9.b }, pn8/z, [x0, x1]
-; CHECK-NEXT:    udot za.s[w8, 0, vgx2], { z0.b, z1.b }, z0.b[0]
-; CHECK-NEXT:    udot za.s[w8, 0, vgx2], { z8.b, z9.b }, z0.b[0]
-; CHECK-NEXT:    ldr z9, [sp, #1, mul vl] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr z8, [sp, #2, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    str z11, [sp, #1, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    ptrue p0.b
+; CHECK-NEXT:    str z10, [sp, #2, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    ld1b { z2.b, z10.b }, pn8/z, [x0]
+; CHECK-NEXT:    ld1b { z3.b, z11.b }, pn8/z, [x0, x1]
+; CHECK-NEXT:    udot za.s[w8, 0, vgx2], { z2.b, z3.b }, z0.b[0]
+; CHECK-NEXT:    udot za.s[w8, 0, vgx2], { z10.b, z11.b }, z0.b[0]
+; CHECK-NEXT:    ldr z11, [sp, #1, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z10, [sp, #2, mul vl] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldr p8, [sp, #7, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    st1b { z0.b }, p0, [x0]
 ; CHECK-NEXT:    addvl sp, sp, #3
 ; CHECK-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
 ; CHECK-NEXT:    ret
@@ -1097,6 +1394,7 @@ entry:
   %6 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8> } %4, 1
   tail call void @llvm.aarch64.sme.udot.lane.za32.vg1x2.nxv16i8(i32 0, <vscale x 16 x i8> %2, <vscale x 16 x i8> %5, <vscale x 16 x i8> poison, i32 0)
   tail call void @llvm.aarch64.sme.udot.lane.za32.vg1x2.nxv16i8(i32 0, <vscale x 16 x i8> %3, <vscale x 16 x i8> %6, <vscale x 16 x i8> poison, i32 0)
+  store <vscale x 16 x i8> %scalable_arg, ptr %ptr
   ret void
 }
 
@@ -1158,33 +1456,35 @@ define void @udot_form_4x_tuple_svecc(ptr %ptr, i64 %stride, <vscale x 16 x i8>
 ; CHECK-NEXT:    lsl x9, x1, #1
 ; CHECK-NEXT:    str p8, [sp, #7, mul vl] // 2-byte Folded Spill
 ; CHECK-NEXT:    ptrue pn8.b
-; CHECK-NEXT:    str z15, [sp, #1, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z23, [sp, #1, mul vl] // 16-byte Folded Spill
 ; CHECK-NEXT:    mov w8, wzr
-; CHECK-NEXT:    str z14, [sp, #2, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    ptrue p0.b
+; CHECK-NEXT:    str z22, [sp, #2, mul vl] // 16-byte Folded Spill
 ; CHECK-NEXT:    add x10, x9, x1
-; CHECK-NEXT:    str z13, [sp, #3, mul vl] // 16-byte Folded Spill
-; CHECK-NEXT:    str z12, [sp, #4, mul vl] // 16-byte Folded Spill
-; CHECK-NEXT:    str z11, [sp, #5, mul vl] // 16-byte Folded Spill
-; CHECK-NEXT:    str z10, [sp, #6, mul vl] // 16-byte Folded Spill
-; CHECK-NEXT:    str z9, [sp, #7, mul vl] // 16-byte Folded Spill
-; CHECK-NEXT:    str z8, [sp, #8, mul vl] // 16-byte Folded Spill
-; CHECK-NEXT:    ld1b { z0.b, z4.b, z8.b, z12.b }, pn8/z, [x0]
-; CHECK-NEXT:    ld1b { z1.b, z5.b, z9.b, z13.b }, pn8/z, [x0, x1]
-; CHECK-NEXT:    ld1b { z2.b, z6.b, z10.b, z14.b }, pn8/z, [x0, x9]
-; CHECK-NEXT:    ld1b { z3.b, z7.b, z11.b, z15.b }, pn8/z, [x0, x10]
-; CHECK-NEXT:    udot za.s[w8, 0, vgx4], { z0.b - z3.b }, z0.b[0]
-; CHECK-NEXT:    udot za.s[w8, 0, vgx4], { z4.b - z7.b }, z0.b[0]
-; CHECK-NEXT:    udot za.s[w8, 0, vgx4], { z8.b - z11.b }, z0.b[0]
-; CHECK-NEXT:    udot za.s[w8, 0, vgx4], { z12.b - z15.b }, z0.b[0]
-; CHECK-NEXT:    ldr z15, [sp, #1, mul vl] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr z14, [sp, #2, mul vl] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr z13, [sp, #3, mul vl] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr z12, [sp, #4, mul vl] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr z11, [sp, #5, mul vl] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr z10, [sp, #6, mul vl] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr z9, [sp, #7, mul vl] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr z8, [sp, #8, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    str z21, [sp, #3, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z20, [sp, #4, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z19, [sp, #5, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z18, [sp, #6, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z17, [sp, #7, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z16, [sp, #8, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    ld1b { z16.b, z20.b, z24.b, z28.b }, pn8/z, [x0]
+; CHECK-NEXT:    ld1b { z17.b, z21.b, z25.b, z29.b }, pn8/z, [x0, x1]
+; CHECK-NEXT:    ld1b { z18.b, z22.b, z26.b, z30.b }, pn8/z, [x0, x9]
+; CHECK-NEXT:    ld1b { z19.b, z23.b, z27.b, z31.b }, pn8/z, [x0, x10]
+; CHECK-NEXT:    udot za.s[w8, 0, vgx4], { z16.b - z19.b }, z0.b[0]
+; CHECK-NEXT:    udot za.s[w8, 0, vgx4], { z20.b - z23.b }, z0.b[0]
+; CHECK-NEXT:    udot za.s[w8, 0, vgx4], { z24.b - z27.b }, z0.b[0]
+; CHECK-NEXT:    udot za.s[w8, 0, vgx4], { z28.b - z31.b }, z0.b[0]
+; CHECK-NEXT:    ldr z23, [sp, #1, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z22, [sp, #2, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z21, [sp, #3, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z20, [sp, #4, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z19, [sp, #5, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z18, [sp, #6, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z17, [sp, #7, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z16, [sp, #8, mul vl] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldr p8, [sp, #7, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    st1b { z0.b }, p0, [x0]
 ; CHECK-NEXT:    addvl sp, sp, #9
 ; CHECK-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
 ; CHECK-NEXT:    ret
@@ -1219,6 +1519,59 @@ entry:
   tail call void @llvm.aarch64.sme.udot.lane.za32.vg1x4.nxv16i8(i32 0, <vscale x 16 x i8> %3, <vscale x 16 x i8> %8, <vscale x 16 x i8> %13, <vscale x 16 x i8> %18, <vscale x 16 x i8> poison, i32 0)
   tail call void @llvm.aarch64.sme.udot.lane.za32.vg1x4.nxv16i8(i32 0, <vscale x 16 x i8> %4, <vscale x 16 x i8> %9, <vscale x 16 x i8> %14, <vscale x 16 x i8> %19, <vscale x 16 x i8> poison, i32 0)
   tail call void @llvm.aarch64.sme.udot.lane.za32.vg1x4.nxv16i8(i32 0, <vscale x 16 x i8> %5, <vscale x 16 x i8> %10, <vscale x 16 x i8> %15, <vscale x 16 x i8> %20, <vscale x 16 x i8> poison, i32 0)
+  store <vscale x 16 x i8> %scalable_arg, ptr %ptr
+  ret void
+}
+
+define void @udot_single_za32_u16_vg1x4_x2load_x4tuple(ptr %ptr, i64 %stride, <vscale x 16 x i8> %zn) #0 {
+; CHECK-LABEL: udot_single_za32_u16_vg1x4_x2load_x4tuple:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NEXT:    addvl sp, sp, #-5
+; CHECK-NEXT:    lsl x9, x1, #1
+; CHECK-NEXT:    str p8, [sp, #7, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    ptrue pn8.b
+; CHECK-NEXT:    str z12, [sp, #1, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    st1b { z10.b, z11.b }, pn8, [sp, #2, mul vl] // 32-byte Folded Spill
+; CHECK-NEXT:    ptrue pn8.b
+; CHECK-NEXT:    str z9, [sp, #4, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    add x10, x9, x1
+; CHECK-NEXT:    mov w8, wzr
+; CHECK-NEXT:    ld1b { z1.b, z9.b }, pn8/z, [x0]
+; CHECK-NEXT:    ld1b { z2.b, z10.b }, pn8/z, [x0, x1]
+; CHECK-NEXT:    ld1b { z3.b, z11.b }, pn8/z, [x0, x9]
+; CHECK-NEXT:    ld1b { z4.b, z12.b }, pn8/z, [x0, x10]
+; CHECK-NEXT:    ptrue pn8.b
+; CHECK-NEXT:    udot za.s[w8, 0, vgx4], { z1.b - z4.b }, z0.b
+; CHECK-NEXT:    udot za.s[w8, 0, vgx4], { z9.b - z12.b }, z0.b
+; CHECK-NEXT:    ldr z12, [sp, #1, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ld1b { z10.b, z11.b }, pn8/z, [sp, #2, mul vl] // 32-byte Folded Reload
+; CHECK-NEXT:    ldr z9, [sp, #4, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr p8, [sp, #7, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    addvl sp, sp, #5
+; CHECK-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    ret
+entry:
+  %0 = tail call target("aarch64.svcount") @llvm.aarch64.sve.ptrue.c8()
+  %1 = tail call { <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.ld1.pn.x2.nxv16i8(target("aarch64.svcount") %0, ptr %ptr)
+  %2 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8> } %1, 0
+  %3 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8> } %1, 1
+  %arrayidx2 = getelementptr inbounds i8, ptr %ptr, i64 %stride
+  %4 = tail call { <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.ld1.pn.x2.nxv16i8(target("aarch64.svcount") %0, ptr %arrayidx2)
+  %5 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8> } %4, 0
+  %6 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8> } %4, 1
+  %mul3 = shl i64 %stride, 1
+  %arrayidx4 = getelementptr inbounds i8, ptr %ptr, i64 %mul3
+  %7 = tail call { <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.ld1.pn.x2.nxv16i8(target("aarch64.svcount") %0, ptr %arrayidx4)
+  %8 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8> } %7, 0
+  %9 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8> } %7, 1
+  %mul5 = mul i64 %stride, 3
+  %arrayidx6 = getelementptr inbounds i8, ptr %ptr, i64 %mul5
+  %10 = tail call { <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.ld1.pn.x2.nxv16i8(target("aarch64.svcount") %0, ptr %arrayidx6)
+  %11 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8> } %10, 0
+  %12 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8> } %10, 1
+  call void @llvm.aarch64.sme.udot.single.za32.vg1x4.nxv16i8(i32 0, <vscale x 16 x i8> %2, <vscale x 16 x i8> %5, <vscale x 16 x i8> %8, <vscale x 16 x i8> %11, <vscale x 16 x i8> %zn)
+  call void @llvm.aarch64.sme.udot.single.za32.vg1x4.nxv16i8(i32 0, <vscale x 16 x i8> %3, <vscale x 16 x i8> %6, <vscale x 16 x i8> %9, <vscale x 16 x i8> %12, <vscale x 16 x i8> %zn)
   ret void
 }
 
@@ -1322,15 +1675,17 @@ define void @usdot_form_2x_tuple_svecc(ptr %ptr, i64 %stride, <vscale x 16 x i8>
 ; CHECK-NEXT:    str p8, [sp, #7, mul vl] // 2-byte Folded Spill
 ; CHECK-NEXT:    ptrue pn8.b
 ; CHECK-NEXT:    mov w8, wzr
-; CHECK-NEXT:    str z9, [sp, #1, mul vl] // 16-byte Folded Spill
-; CHECK-NEXT:    str z8, [sp, #2, mul vl] // 16-byte Folded Spill
-; CHECK-NEXT:    ld1b { z0.b, z8.b }, pn8/z, [x0]
-; CHECK-NEXT:    ld1b { z1.b, z9.b }, pn8/z, [x0, x1]
-; CHECK-NEXT:    usdot za.s[w8, 0, vgx2], { z0.b, z1.b }, z0.b[0]
-; CHECK-NEXT:    usdot za.s[w8, 0, vgx2], { z8.b, z9.b }, z0.b[0]
-; CHECK-NEXT:    ldr z9, [sp, #1, mul vl] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr z8, [sp, #2, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    str z11, [sp, #1, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    ptrue p0.b
+; CHECK-NEXT:    str z10, [sp, #2, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    ld1b { z2.b, z10.b }, pn8/z, [x0]
+; CHECK-NEXT:    ld1b { z3.b, z11.b }, pn8/z, [x0, x1]
+; CHECK-NEXT:    usdot za.s[w8, 0, vgx2], { z2.b, z3.b }, z0.b[0]
+; CHECK-NEXT:    usdot za.s[w8, 0, vgx2], { z10.b, z11.b }, z0.b[0]
+; CHECK-NEXT:    ldr z11, [sp, #1, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z10, [sp, #2, mul vl] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldr p8, [sp, #7, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    st1b { z0.b }, p0, [x0]
 ; CHECK-NEXT:    addvl sp, sp, #3
 ; CHECK-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
 ; CHECK-NEXT:    ret
@@ -1345,6 +1700,7 @@ entry:
   %6 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8> } %4, 1
   tail call void @llvm.aarch64.sme.usdot.lane.za32.vg1x2.nxv16i8(i32 0, <vscale x 16 x i8> %2, <vscale x 16 x i8> %5, <vscale x 16 x i8> poison, i32 0)
   tail call void @llvm.aarch64.sme.usdot.lane.za32.vg1x2.nxv16i8(i32 0, <vscale x 16 x i8> %3, <vscale x 16 x i8> %6, <vscale x 16 x i8> poison, i32 0)
+  store <vscale x 16 x i8> %scalable_arg, ptr %ptr
   ret void
 }
 
@@ -1406,33 +1762,35 @@ define void @usdot_form_4x_tuple_svecc(ptr %ptr, i64 %stride, <vscale x 16 x i8>
 ; CHECK-NEXT:    lsl x9, x1, #1
 ; CHECK-NEXT:    str p8, [sp, #7, mul vl] // 2-byte Folded Spill
 ; CHECK-NEXT:    ptrue pn8.b
-; CHECK-NEXT:    str z15, [sp, #1, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z23, [sp, #1, mul vl] // 16-byte Folded Spill
 ; CHECK-NEXT:    mov w8, wzr
-; CHECK-NEXT:    str z14, [sp, #2, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    ptrue p0.b
+; CHECK-NEXT:    str z22, [sp, #2, mul vl] // 16-byte Folded Spill
 ; CHECK-NEXT:    add x10, x9, x1
-; CHECK-NEXT:    str z13, [sp, #3, mul vl] // 16-byte Folded Spill
-; CHECK-NEXT:    str z12, [sp, #4, mul vl] // 16-byte Folded Spill
-; CHECK-NEXT:    str z11, [sp, #5, mul vl] // 16-byte Folded Spill
-; CHECK-NEXT:    str z10, [sp, #6, mul vl] // 16-byte Folded Spill
-; CHECK-NEXT:    str z9, [sp, #7, mul vl] // 16-byte Folded Spill
-; CHECK-NEXT:    str z8, [sp, #8, mul vl] // 16-byte Folded Spill
-; CHECK-NEXT:    ld1b { z0.b, z4.b, z8.b, z12.b }, pn8/z, [x0]
-; CHECK-NEXT:    ld1b { z1.b, z5.b, z9.b, z13.b }, pn8/z, [x0, x1]
-; CHECK-NEXT:    ld1b { z2.b, z6.b, z10.b, z14.b }, pn8/z, [x0, x9]
-; CHECK-NEXT:    ld1b { z3.b, z7.b, z11.b, z15.b }, pn8/z, [x0, x10]
-; CHECK-NEXT:    usdot za.s[w8, 0, vgx4], { z0.b - z3.b }, z0.b[0]
-; CHECK-NEXT:    usdot za.s[w8, 0, vgx4], { z4.b - z7.b }, z0.b[0]
-; CHECK-NEXT:    usdot za.s[w8, 0, vgx4], { z8.b - z11.b }, z0.b[0]
-; CHECK-NEXT:    usdot za.s[w8, 0, vgx4], { z12.b - z15.b }, z0.b[0]
-; CHECK-NEXT:    ldr z15, [sp, #1, mul vl] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr z14, [sp, #2, mul vl] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr z13, [sp, #3, mul vl] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr z12, [sp, #4, mul vl] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr z11, [sp, #5, mul vl] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr z10, [sp, #6, mul vl] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr z9, [sp, #7, mul vl] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr z8, [sp, #8, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    str z21, [sp, #3, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z20, [sp, #4, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z19, [sp, #5, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z18, [sp, #6, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z17, [sp, #7, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z16, [sp, #8, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    ld1b { z16.b, z20.b, z24.b, z28.b }, pn8/z, [x0]
+; CHECK-NEXT:    ld1b { z17.b, z21.b, z25.b, z29.b }, pn8/z, [x0, x1]
+; CHECK-NEXT:    ld1b { z18.b, z22.b, z26.b, z30.b }, pn8/z, [x0, x9]
+; CHECK-NEXT:    ld1b { z19.b, z23.b, z27.b, z31.b }, pn8/z, [x0, x10]
+; CHECK-NEXT:    usdot za.s[w8, 0, vgx4], { z16.b - z19.b }, z0.b[0]
+; CHECK-NEXT:    usdot za.s[w8, 0, vgx4], { z20.b - z23.b }, z0.b[0]
+; CHECK-NEXT:    usdot za.s[w8, 0, vgx4], { z24.b - z27.b }, z0.b[0]
+; CHECK-NEXT:    usdot za.s[w8, 0, vgx4], { z28.b - z31.b }, z0.b[0]
+; CHECK-NEXT:    ldr z23, [sp, #1, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z22, [sp, #2, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z21, [sp, #3, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z20, [sp, #4, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z19, [sp, #5, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z18, [sp, #6, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z17, [sp, #7, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z16, [sp, #8, mul vl] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldr p8, [sp, #7, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    st1b { z0.b }, p0, [x0]
 ; CHECK-NEXT:    addvl sp, sp, #9
 ; CHECK-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
 ; CHECK-NEXT:    ret
@@ -1467,6 +1825,7 @@ entry:
   tail call void @llvm.aarch64.sme.usdot.lane.za32.vg1x4.nxv16i8(i32 0, <vscale x 16 x i8> %3, <vscale x 16 x i8> %8, <vscale x 16 x i8> %13, <vscale x 16 x i8> %18, <vscale x 16 x i8> poison, i32 0)
   tail call void @llvm.aarch64.sme.usdot.lane.za32.vg1x4.nxv16i8(i32 0, <vscale x 16 x i8> %4, <vscale x 16 x i8> %9, <vscale x 16 x i8> %14, <vscale x 16 x i8> %19, <vscale x 16 x i8> poison, i32 0)
   tail call void @llvm.aarch64.sme.usdot.lane.za32.vg1x4.nxv16i8(i32 0, <vscale x 16 x i8> %5, <vscale x 16 x i8> %10, <vscale x 16 x i8> %15, <vscale x 16 x i8> %20, <vscale x 16 x i8> poison, i32 0)
+  store <vscale x 16 x i8> %scalable_arg, ptr %ptr
   ret void
 }
 
@@ -1572,15 +1931,17 @@ define void @sdot_form_2x_tuple_svecc(ptr %ptr, i64 %stride, <vscale x 16 x i8>
 ; CHECK-NEXT:    str p8, [sp, #7, mul vl] // 2-byte Folded Spill
 ; CHECK-NEXT:    ptrue pn8.b
 ; CHECK-NEXT:    mov w8, wzr
-; CHECK-NEXT:    str z9, [sp, #1, mul vl] // 16-byte Folded Spill
-; CHECK-NEXT:    str z8, [sp, #2, mul vl] // 16-byte Folded Spill
-; CHECK-NEXT:    ld1b { z0.b, z8.b }, pn8/z, [x0]
-; CHECK-NEXT:    ld1b { z1.b, z9.b }, pn8/z, [x0, x1]
-; CHECK-NEXT:    sdot za.s[w8, 0, vgx2], { z0.b, z1.b }, z0.b[0]
-; CHECK-NEXT:    sdot za.s[w8, 0, vgx2], { z8.b, z9.b }, z0.b[0]
-; CHECK-NEXT:    ldr z9, [sp, #1, mul vl] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr z8, [sp, #2, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    str z11, [sp, #1, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    ptrue p0.b
+; CHECK-NEXT:    str z10, [sp, #2, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    ld1b { z2.b, z10.b }, pn8/z, [x0]
+; CHECK-NEXT:    ld1b { z3.b, z11.b }, pn8/z, [x0, x1]
+; CHECK-NEXT:    sdot za.s[w8, 0, vgx2], { z2.b, z3.b }, z0.b[0]
+; CHECK-NEXT:    sdot za.s[w8, 0, vgx2], { z10.b, z11.b }, z0.b[0]
+; CHECK-NEXT:    ldr z11, [sp, #1, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z10, [sp, #2, mul vl] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldr p8, [sp, #7, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    st1b { z0.b }, p0, [x0]
 ; CHECK-NEXT:    addvl sp, sp, #3
 ; CHECK-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
 ; CHECK-NEXT:    ret
@@ -1595,6 +1956,7 @@ entry:
   %6 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8> } %4, 1
   tail call void @llvm.aarch64.sme.sdot.lane.za32.vg1x2.nxv16i8(i32 0, <vscale x 16 x i8> %2, <vscale x 16 x i8> %5, <vscale x 16 x i8> poison, i32 0)
   tail call void @llvm.aarch64.sme.sdot.lane.za32.vg1x2.nxv16i8(i32 0, <vscale x 16 x i8> %3, <vscale x 16 x i8> %6, <vscale x 16 x i8> poison, i32 0)
+  store <vscale x 16 x i8> %scalable_arg, ptr %ptr
   ret void
 }
 
@@ -1656,33 +2018,35 @@ define void @sdot_form_4x_tuple_svecc(ptr %ptr, i64 %stride, <vscale x 16 x i8>
 ; CHECK-NEXT:    lsl x9, x1, #1
 ; CHECK-NEXT:    str p8, [sp, #7, mul vl] // 2-byte Folded Spill
 ; CHECK-NEXT:    ptrue pn8.b
-; CHECK-NEXT:    str z15, [sp, #1, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z23, [sp, #1, mul vl] // 16-byte Folded Spill
 ; CHECK-NEXT:    mov w8, wzr
-; CHECK-NEXT:    str z14, [sp, #2, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    ptrue p0.b
+; CHECK-NEXT:    str z22, [sp, #2, mul vl] // 16-byte Folded Spill
 ; CHECK-NEXT:    add x10, x9, x1
-; CHECK-NEXT:    str z13, [sp, #3, mul vl] // 16-byte Folded Spill
-; CHECK-NEXT:    str z12, [sp, #4, mul vl] // 16-byte Folded Spill
-; CHECK-NEXT:    str z11, [sp, #5, mul vl] // 16-byte Folded Spill
-; CHECK-NEXT:    str z10, [sp, #6, mul vl] // 16-byte Folded Spill
-; CHECK-NEXT:    str z9, [sp, #7, mul vl] // 16-byte Folded Spill
-; CHECK-NEXT:    str z8, [sp, #8, mul vl] // 16-byte Folded Spill
-; CHECK-NEXT:    ld1b { z0.b, z4.b, z8.b, z12.b }, pn8/z, [x0]
-; CHECK-NEXT:    ld1b { z1.b, z5.b, z9.b, z13.b }, pn8/z, [x0, x1]
-; CHECK-NEXT:    ld1b { z2.b, z6.b, z10.b, z14.b }, pn8/z, [x0, x9]
-; CHECK-NEXT:    ld1b { z3.b, z7.b, z11.b, z15.b }, pn8/z, [x0, x10]
-; CHECK-NEXT:    sdot za.s[w8, 0, vgx4], { z0.b - z3.b }, z0.b[0]
-; CHECK-NEXT:    sdot za.s[w8, 0, vgx4], { z4.b - z7.b }, z0.b[0]
-; CHECK-NEXT:    sdot za.s[w8, 0, vgx4], { z8.b - z11.b }, z0.b[0]
-; CHECK-NEXT:    sdot za.s[w8, 0, vgx4], { z12.b - z15.b }, z0.b[0]
-; CHECK-NEXT:    ldr z15, [sp, #1, mul vl] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr z14, [sp, #2, mul vl] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr z13, [sp, #3, mul vl] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr z12, [sp, #4, mul vl] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr z11, [sp, #5, mul vl] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr z10, [sp, #6, mul vl] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr z9, [sp, #7, mul vl] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr z8, [sp, #8, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    str z21, [sp, #3, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z20, [sp, #4, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z19, [sp, #5, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z18, [sp, #6, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z17, [sp, #7, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z16, [sp, #8, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    ld1b { z16.b, z20.b, z24.b, z28.b }, pn8/z, [x0]
+; CHECK-NEXT:    ld1b { z17.b, z21.b, z25.b, z29.b }, pn8/z, [x0, x1]
+; CHECK-NEXT:    ld1b { z18.b, z22.b, z26.b, z30.b }, pn8/z, [x0, x9]
+; CHECK-NEXT:    ld1b { z19.b, z23.b, z27.b, z31.b }, pn8/z, [x0, x10]
+; CHECK-NEXT:    sdot za.s[w8, 0, vgx4], { z16.b - z19.b }, z0.b[0]
+; CHECK-NEXT:    sdot za.s[w8, 0, vgx4], { z20.b - z23.b }, z0.b[0]
+; CHECK-NEXT:    sdot za.s[w8, 0, vgx4], { z24.b - z27.b }, z0.b[0]
+; CHECK-NEXT:    sdot za.s[w8, 0, vgx4], { z28.b - z31.b }, z0.b[0]
+; CHECK-NEXT:    ldr z23, [sp, #1, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z22, [sp, #2, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z21, [sp, #3, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z20, [sp, #4, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z19, [sp, #5, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z18, [sp, #6, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z17, [sp, #7, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z16, [sp, #8, mul vl] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldr p8, [sp, #7, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    st1b { z0.b }, p0, [x0]
 ; CHECK-NEXT:    addvl sp, sp, #9
 ; CHECK-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
 ; CHECK-NEXT:    ret
@@ -1717,6 +2081,7 @@ entry:
   tail call void @llvm.aarch64.sme.sdot.lane.za32.vg1x4.nxv16i8(i32 0, <vscale x 16 x i8> %3, <vscale x 16 x i8> %8, <vscale x 16 x i8> %13, <vscale x 16 x i8> %18, <vscale x 16 x i8> poison, i32 0)
   tail call void @llvm.aarch64.sme.sdot.lane.za32.vg1x4.nxv16i8(i32 0, <vscale x 16 x i8> %4, <vscale x 16 x i8> %9, <vscale x 16 x i8> %14, <vscale x 16 x i8> %19, <vscale x 16 x i8> poison, i32 0)
   tail call void @llvm.aarch64.sme.sdot.lane.za32.vg1x4.nxv16i8(i32 0, <vscale x 16 x i8> %5, <vscale x 16 x i8> %10, <vscale x 16 x i8> %15, <vscale x 16 x i8> %20, <vscale x 16 x i8> poison, i32 0)
+  store <vscale x 16 x i8> %scalable_arg, ptr %ptr
   ret void
 }
 
@@ -1822,15 +2187,17 @@ define void @sudot_form_2x_tuple_svecc(ptr %ptr, i64 %stride, <vscale x 16 x i8>
 ; CHECK-NEXT:    str p8, [sp, #7, mul vl] // 2-byte Folded Spill
 ; CHECK-NEXT:    ptrue pn8.b
 ; CHECK-NEXT:    mov w8, wzr
-; CHECK-NEXT:    str z9, [sp, #1, mul vl] // 16-byte Folded Spill
-; CHECK-NEXT:    str z8, [sp, #2, mul vl] // 16-byte Folded Spill
-; CHECK-NEXT:    ld1b { z0.b, z8.b }, pn8/z, [x0]
-; CHECK-NEXT:    ld1b { z1.b, z9.b }, pn8/z, [x0, x1]
-; CHECK-NEXT:    sudot za.s[w8, 0, vgx2], { z0.b, z1.b }, z0.b[0]
-; CHECK-NEXT:    sudot za.s[w8, 0, vgx2], { z8.b, z9.b }, z0.b[0]
-; CHECK-NEXT:    ldr z9, [sp, #1, mul vl] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr z8, [sp, #2, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    str z11, [sp, #1, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    ptrue p0.b
+; CHECK-NEXT:    str z10, [sp, #2, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    ld1b { z2.b, z10.b }, pn8/z, [x0]
+; CHECK-NEXT:    ld1b { z3.b, z11.b }, pn8/z, [x0, x1]
+; CHECK-NEXT:    sudot za.s[w8, 0, vgx2], { z2.b, z3.b }, z0.b[0]
+; CHECK-NEXT:    sudot za.s[w8, 0, vgx2], { z10.b, z11.b }, z0.b[0]
+; CHECK-NEXT:    ldr z11, [sp, #1, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z10, [sp, #2, mul vl] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldr p8, [sp, #7, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    st1b { z0.b }, p0, [x0]
 ; CHECK-NEXT:    addvl sp, sp, #3
 ; CHECK-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
 ; CHECK-NEXT:    ret
@@ -1845,6 +2212,7 @@ entry:
   %6 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8> } %4, 1
   tail call void @llvm.aarch64.sme.sudot.lane.za32.vg1x2.nxv16i8(i32 0, <vscale x 16 x i8> %2, <vscale x 16 x i8> %5, <vscale x 16 x i8> poison, i32 0)
   tail call void @llvm.aarch64.sme.sudot.lane.za32.vg1x2.nxv16i8(i32 0, <vscale x 16 x i8> %3, <vscale x 16 x i8> %6, <vscale x 16 x i8> poison, i32 0)
+  store <vscale x 16 x i8> %scalable_arg, ptr %ptr
   ret void
 }
 
@@ -1906,33 +2274,35 @@ define void @sudot_form_4x_tuple_svecc(ptr %ptr, i64 %stride, <vscale x 16 x i8>
 ; CHECK-NEXT:    lsl x9, x1, #1
 ; CHECK-NEXT:    str p8, [sp, #7, mul vl] // 2-byte Folded Spill
 ; CHECK-NEXT:    ptrue pn8.b
-; CHECK-NEXT:    str z15, [sp, #1, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z23, [sp, #1, mul vl] // 16-byte Folded Spill
 ; CHECK-NEXT:    mov w8, wzr
-; CHECK-NEXT:    str z14, [sp, #2, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    ptrue p0.b
+; CHECK-NEXT:    str z22, [sp, #2, mul vl] // 16-byte Folded Spill
 ; CHECK-NEXT:    add x10, x9, x1
-; CHECK-NEXT:    str z13, [sp, #3, mul vl] // 16-byte Folded Spill
-; CHECK-NEXT:    str z12, [sp, #4, mul vl] // 16-byte Folded Spill
-; CHECK-NEXT:    str z11, [sp, #5, mul vl] // 16-byte Folded Spill
-; CHECK-NEXT:    str z10, [sp, #6, mul vl] // 16-byte Folded Spill
-; CHECK-NEXT:    str z9, [sp, #7, mul vl] // 16-byte Folded Spill
-; CHECK-NEXT:    str z8, [sp, #8, mul vl] // 16-byte Folded Spill
-; CHECK-NEXT:    ld1b { z0.b, z4.b, z8.b, z12.b }, pn8/z, [x0]
-; CHECK-NEXT:    ld1b { z1.b, z5.b, z9.b, z13.b }, pn8/z, [x0, x1]
-; CHECK-NEXT:    ld1b { z2.b, z6.b, z10.b, z14.b }, pn8/z, [x0, x9]
-; CHECK-NEXT:    ld1b { z3.b, z7.b, z11.b, z15.b }, pn8/z, [x0, x10]
-; CHECK-NEXT:    sudot za.s[w8, 0, vgx4], { z0.b - z3.b }, z0.b[0]
-; CHECK-NEXT:    sudot za.s[w8, 0, vgx4], { z4.b - z7.b }, z0.b[0]
-; CHECK-NEXT:    sudot za.s[w8, 0, vgx4], { z8.b - z11.b }, z0.b[0]
-; CHECK-NEXT:    sudot za.s[w8, 0, vgx4], { z12.b - z15.b }, z0.b[0]
-; CHECK-NEXT:    ldr z15, [sp, #1, mul vl] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr z14, [sp, #2, mul vl] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr z13, [sp, #3, mul vl] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr z12, [sp, #4, mul vl] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr z11, [sp, #5, mul vl] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr z10, [sp, #6, mul vl] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr z9, [sp, #7, mul vl] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr z8, [sp, #8, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    str z21, [sp, #3, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z20, [sp, #4, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z19, [sp, #5, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z18, [sp, #6, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z17, [sp, #7, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z16, [sp, #8, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    ld1b { z16.b, z20.b, z24.b, z28.b }, pn8/z, [x0]
+; CHECK-NEXT:    ld1b { z17.b, z21.b, z25.b, z29.b }, pn8/z, [x0, x1]
+; CHECK-NEXT:    ld1b { z18.b, z22.b, z26.b, z30.b }, pn8/z, [x0, x9]
+; CHECK-NEXT:    ld1b { z19.b, z23.b, z27.b, z31.b }, pn8/z, [x0, x10]
+; CHECK-NEXT:    sudot za.s[w8, 0, vgx4], { z16.b - z19.b }, z0.b[0]
+; CHECK-NEXT:    sudot za.s[w8, 0, vgx4], { z20.b - z23.b }, z0.b[0]
+; CHECK-NEXT:    sudot za.s[w8, 0, vgx4], { z24.b - z27.b }, z0.b[0]
+; CHECK-NEXT:    sudot za.s[w8, 0, vgx4], { z28.b - z31.b }, z0.b[0]
+; CHECK-NEXT:    ldr z23, [sp, #1, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z22, [sp, #2, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z21, [sp, #3, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z20, [sp, #4, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z19, [sp, #5, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z18, [sp, #6, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z17, [sp, #7, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z16, [sp, #8, mul vl] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldr p8, [sp, #7, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    st1b { z0.b }, p0, [x0]
 ; CHECK-NEXT:    addvl sp, sp, #9
 ; CHECK-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
 ; CHECK-NEXT:    ret
@@ -1967,6 +2337,7 @@ entry:
   tail call void @llvm.aarch64.sme.sudot.lane.za32.vg1x4.nxv16i8(i32 0, <vscale x 16 x i8> %3, <vscale x 16 x i8> %8, <vscale x 16 x i8> %13, <vscale x 16 x i8> %18, <vscale x 16 x i8> poison, i32 0)
   tail call void @llvm.aarch64.sme.sudot.lane.za32.vg1x4.nxv16i8(i32 0, <vscale x 16 x i8> %4, <vscale x 16 x i8> %9, <vscale x 16 x i8> %14, <vscale x 16 x i8> %19, <vscale x 16 x i8> poison, i32 0)
   tail call void @llvm.aarch64.sme.sudot.lane.za32.vg1x4.nxv16i8(i32 0, <vscale x 16 x i8> %5, <vscale x 16 x i8> %10, <vscale x 16 x i8> %15, <vscale x 16 x i8> %20, <vscale x 16 x i8> poison, i32 0)
+  store <vscale x 16 x i8> %scalable_arg, ptr %ptr
   ret void
 }
 
diff --git a/llvm/test/CodeGen/AArch64/sme2-intrinsics-qcvt.ll b/llvm/test/CodeGen/AArch64/sme2-intrinsics-qcvt.ll
index 9e7d96c89cba7..b4a83c10df94a 100644
--- a/llvm/test/CodeGen/AArch64/sme2-intrinsics-qcvt.ll
+++ b/llvm/test/CodeGen/AArch64/sme2-intrinsics-qcvt.ll
@@ -1,5 +1,5 @@
-; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py$
-; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sme2 -force-streaming -verify-machineinstrs < %s | FileCheck %s
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sme2 -force-streaming -enable-subreg-liveness -verify-machineinstrs < %s | FileCheck %s
 
 ;
 ; SQCVT
@@ -44,6 +44,85 @@ define <vscale x 8 x i16> @multi_vector_qcvt_x4_s16_s64(<vscale x 2 x i64> %unus
   ret <vscale x 8 x i16> %res
 }
 
+define { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } @multi_vector_qcvt_x4_s16_s64_tuple(i64 %stride, ptr %ptr) {
+; CHECK-LABEL: multi_vector_qcvt_x4_s16_s64_tuple:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NEXT:    addvl sp, sp, #-9
+; CHECK-NEXT:    str p8, [sp, #7, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    str z23, [sp, #1, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z22, [sp, #2, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z21, [sp, #3, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z20, [sp, #4, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z19, [sp, #5, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z18, [sp, #6, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z17, [sp, #7, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z16, [sp, #8, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    .cfi_escape 0x0f, 0x0d, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0xc8, 0x00, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 72 * VG
+; CHECK-NEXT:    .cfi_offset w29, -16
+; CHECK-NEXT:    lsl x8, x0, #1
+; CHECK-NEXT:    add x9, x1, x0
+; CHECK-NEXT:    ptrue pn8.b
+; CHECK-NEXT:    ld1d { z16.d, z20.d, z24.d, z28.d }, pn8/z, [x1]
+; CHECK-NEXT:    ld1d { z17.d, z21.d, z25.d, z29.d }, pn8/z, [x9]
+; CHECK-NEXT:    add x10, x1, x8
+; CHECK-NEXT:    add x8, x9, x8
+; CHECK-NEXT:    ld1d { z18.d, z22.d, z26.d, z30.d }, pn8/z, [x10]
+; CHECK-NEXT:    ld1d { z19.d, z23.d, z27.d, z31.d }, pn8/z, [x8]
+; CHECK-NEXT:    sqcvt z0.h, { z16.d - z19.d }
+; CHECK-NEXT:    sqcvt z1.h, { z20.d - z23.d }
+; CHECK-NEXT:    sqcvt z2.h, { z24.d - z27.d }
+; CHECK-NEXT:    sqcvt z3.h, { z28.d - z31.d }
+; CHECK-NEXT:    ldr z23, [sp, #1, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z22, [sp, #2, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z21, [sp, #3, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z20, [sp, #4, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z19, [sp, #5, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z18, [sp, #6, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z17, [sp, #7, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z16, [sp, #8, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr p8, [sp, #7, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    addvl sp, sp, #9
+; CHECK-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    ret
+entry:
+  %0 = tail call target("aarch64.svcount") @llvm.aarch64.sve.ptrue.c8()
+  %1 = tail call { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } @llvm.aarch64.sve.ld1.pn.x4.nxv2i64(target("aarch64.svcount") %0, ptr %ptr)
+  %2 = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } %1, 0
+  %3 = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } %1, 1
+  %4 = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } %1, 2
+  %5 = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } %1, 3
+  %arrayidx2 = getelementptr inbounds i8, ptr %ptr, i64 %stride
+  %6 = tail call { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } @llvm.aarch64.sve.ld1.pn.x4.nxv2i64(target("aarch64.svcount") %0, ptr %arrayidx2)
+  %7 = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } %6, 0
+  %8 = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } %6, 1
+  %9 = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } %6, 2
+  %10 = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } %6, 3
+  %mul3 = shl i64 %stride, 1
+  %arrayidx4 = getelementptr inbounds i8, ptr %ptr, i64 %mul3
+  %11 = tail call { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } @llvm.aarch64.sve.ld1.pn.x4.nxv2i64(target("aarch64.svcount") %0, ptr %arrayidx4)
+  %12 = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } %11, 0
+  %13 = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } %11, 1
+  %14 = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } %11, 2
+  %15 = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } %11, 3
+  %mul5 = mul i64 %stride, 3
+  %arrayidx6 = getelementptr inbounds i8, ptr %ptr, i64 %mul5
+  %16 = tail call { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } @llvm.aarch64.sve.ld1.pn.x4.nxv2i64(target("aarch64.svcount") %0, ptr %arrayidx6)
+  %17 = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } %16, 0
+  %18 = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } %16, 1
+  %19 = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } %16, 2
+  %20 = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } %16, 3
+  %res1 = call <vscale x 8 x i16> @llvm.aarch64.sve.sqcvt.x4.nxv2i64(<vscale x 2 x i64> %2, <vscale x 2 x i64> %7, <vscale x 2 x i64> %12, <vscale x 2 x i64> %17)
+  %res2 = call <vscale x 8 x i16> @llvm.aarch64.sve.sqcvt.x4.nxv2i64(<vscale x 2 x i64> %3, <vscale x 2 x i64> %8, <vscale x 2 x i64> %13, <vscale x 2 x i64> %18)
+  %res3 = call <vscale x 8 x i16> @llvm.aarch64.sve.sqcvt.x4.nxv2i64(<vscale x 2 x i64> %4, <vscale x 2 x i64> %9, <vscale x 2 x i64> %14, <vscale x 2 x i64> %19)
+  %res4 = call <vscale x 8 x i16> @llvm.aarch64.sve.sqcvt.x4.nxv2i64(<vscale x 2 x i64> %5, <vscale x 2 x i64> %10, <vscale x 2 x i64> %15, <vscale x 2 x i64> %20)
+  %ins1 = insertvalue { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } poison, <vscale x 8 x i16> %res1, 0
+  %ins2 = insertvalue { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } %ins1, <vscale x 8 x i16> %res2, 1
+  %ins3 = insertvalue { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } %ins2, <vscale x 8 x i16> %res3, 2
+  %ins4 = insertvalue { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } %ins3, <vscale x 8 x i16> %res4, 3
+  ret { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } %ins4
+}
+
 ;
 ; UQCVT
 ;
diff --git a/llvm/test/CodeGen/AArch64/sme2-intrinsics-qrshr.ll b/llvm/test/CodeGen/AArch64/sme2-intrinsics-qrshr.ll
index f0070557a28fb..0bc9e15786a8a 100644
--- a/llvm/test/CodeGen/AArch64/sme2-intrinsics-qrshr.ll
+++ b/llvm/test/CodeGen/AArch64/sme2-intrinsics-qrshr.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sme2 -force-streaming -verify-machineinstrs < %s | FileCheck %s
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sme2 -force-streaming -enable-subreg-liveness -verify-machineinstrs < %s | FileCheck %s
 
 ;
 ; S/UQRSHR x2
@@ -16,6 +16,46 @@ define <vscale x 8 x i16> @multi_vector_sat_shift_narrow_x2_s16(<vscale x 4 x i3
   ret <vscale x 8 x i16> %res
 }
 
+define { <vscale x 8 x i16>, <vscale x 8 x i16> } @multi_vector_sat_shift_narrow_x2_s16_tuple(i64 %stride, ptr %ptr) {
+; CHECK-LABEL: multi_vector_sat_shift_narrow_x2_s16_tuple:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NEXT:    addvl sp, sp, #-3
+; CHECK-NEXT:    str p8, [sp, #7, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    str z11, [sp, #1, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z10, [sp, #2, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    .cfi_escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x18, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 24 * VG
+; CHECK-NEXT:    .cfi_offset w29, -16
+; CHECK-NEXT:    .cfi_escape 0x10, 0x4a, 0x0a, 0x11, 0x70, 0x22, 0x11, 0x78, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d10 @ cfa - 16 - 8 * VG
+; CHECK-NEXT:    .cfi_escape 0x10, 0x4b, 0x0a, 0x11, 0x70, 0x22, 0x11, 0x70, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d11 @ cfa - 16 - 16 * VG
+; CHECK-NEXT:    ptrue pn8.b
+; CHECK-NEXT:    add x8, x1, x0
+; CHECK-NEXT:    ld1w { z2.s, z10.s }, pn8/z, [x1]
+; CHECK-NEXT:    ld1w { z3.s, z11.s }, pn8/z, [x8]
+; CHECK-NEXT:    sqrshr z0.h, { z2.s, z3.s }, #16
+; CHECK-NEXT:    sqrshr z1.h, { z10.s, z11.s }, #16
+; CHECK-NEXT:    ldr z11, [sp, #1, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z10, [sp, #2, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr p8, [sp, #7, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    addvl sp, sp, #3
+; CHECK-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    ret
+entry:
+  %0 = tail call target("aarch64.svcount") @llvm.aarch64.sve.ptrue.c8()
+  %1 = tail call { <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.aarch64.sve.ld1.pn.x2.nxv4f32(target("aarch64.svcount") %0, ptr %ptr)
+  %2 = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } %1, 0
+  %3 = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } %1, 1
+  %arrayidx2 = getelementptr inbounds i8, ptr %ptr, i64 %stride
+  %4 = tail call { <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.aarch64.sve.ld1.pn.x2.nxv4f32(target("aarch64.svcount") %0, ptr %arrayidx2)
+  %5 = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } %4, 0
+  %6 = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } %4, 1
+  %res1 = call <vscale x 8 x i16> @llvm.aarch64.sve.sqrshr.x2.nxv8i16(<vscale x 4 x i32> %2, <vscale x 4 x i32> %5, i32 16)
+  %res2 = call <vscale x 8 x i16> @llvm.aarch64.sve.sqrshr.x2.nxv8i16(<vscale x 4 x i32> %3, <vscale x 4 x i32> %6, i32 16)
+  %ins1 = insertvalue { <vscale x 8 x i16>, <vscale x 8 x i16> } poison, <vscale x 8 x i16> %res1, 0
+  %ins2 = insertvalue { <vscale x 8 x i16>, <vscale x 8 x i16> } %ins1, <vscale x 8 x i16> %res2, 1
+  ret { <vscale x 8 x i16>, <vscale x 8 x i16> } %ins2
+}
+
 define <vscale x 8 x i16> @multi_vector_sat_shift_narrow_x2_u16(<vscale x 4 x i32> %unused, <vscale x 4 x i32> %zn1, <vscale x 4 x i32> %zn2) {
 ; CHECK-LABEL: multi_vector_sat_shift_narrow_x2_u16:
 ; CHECK:       // %bb.0:
@@ -44,6 +84,85 @@ define <vscale x 16 x i8> @multi_vector_sat_shift_narrow_x4_s8(<vscale x 4 x i32
   ret <vscale x 16 x i8> %res
 }
 
+define { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @multi_vector_sat_shift_narrow_x4_s8_tuple(i64 %stride, ptr %ptr) {
+; CHECK-LABEL: multi_vector_sat_shift_narrow_x4_s8_tuple:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NEXT:    addvl sp, sp, #-9
+; CHECK-NEXT:    str p8, [sp, #7, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    str z23, [sp, #1, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z22, [sp, #2, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z21, [sp, #3, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z20, [sp, #4, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z19, [sp, #5, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z18, [sp, #6, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z17, [sp, #7, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z16, [sp, #8, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    .cfi_escape 0x0f, 0x0d, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0xc8, 0x00, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 72 * VG
+; CHECK-NEXT:    .cfi_offset w29, -16
+; CHECK-NEXT:    lsl x8, x0, #1
+; CHECK-NEXT:    add x9, x1, x0
+; CHECK-NEXT:    ptrue pn8.b
+; CHECK-NEXT:    ld1w { z16.s, z20.s, z24.s, z28.s }, pn8/z, [x1]
+; CHECK-NEXT:    ld1w { z17.s, z21.s, z25.s, z29.s }, pn8/z, [x9]
+; CHECK-NEXT:    add x10, x1, x8
+; CHECK-NEXT:    add x8, x9, x8
+; CHECK-NEXT:    ld1w { z18.s, z22.s, z26.s, z30.s }, pn8/z, [x10]
+; CHECK-NEXT:    ld1w { z19.s, z23.s, z27.s, z31.s }, pn8/z, [x8]
+; CHECK-NEXT:    sqrshr z0.b, { z16.s - z19.s }, #32
+; CHECK-NEXT:    sqrshr z1.b, { z20.s - z23.s }, #32
+; CHECK-NEXT:    sqrshr z2.b, { z24.s - z27.s }, #32
+; CHECK-NEXT:    sqrshr z3.b, { z28.s - z31.s }, #32
+; CHECK-NEXT:    ldr z23, [sp, #1, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z22, [sp, #2, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z21, [sp, #3, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z20, [sp, #4, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z19, [sp, #5, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z18, [sp, #6, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z17, [sp, #7, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z16, [sp, #8, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr p8, [sp, #7, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    addvl sp, sp, #9
+; CHECK-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    ret
+entry:
+  %0 = tail call target("aarch64.svcount") @llvm.aarch64.sve.ptrue.c8()
+  %1 = tail call { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.aarch64.sve.ld1.pn.x4.nxv4i32(target("aarch64.svcount") %0, ptr %ptr)
+  %2 = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } %1, 0
+  %3 = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } %1, 1
+  %4 = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } %1, 2
+  %5 = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } %1, 3
+  %arrayidx2 = getelementptr inbounds i8, ptr %ptr, i64 %stride
+  %6 = tail call { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.aarch64.sve.ld1.pn.x4.nxv4i32(target("aarch64.svcount") %0, ptr %arrayidx2)
+  %7 = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } %6, 0
+  %8 = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } %6, 1
+  %9 = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } %6, 2
+  %10 = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } %6, 3
+  %mul3 = shl i64 %stride, 1
+  %arrayidx4 = getelementptr inbounds i8, ptr %ptr, i64 %mul3
+  %11 = tail call { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.aarch64.sve.ld1.pn.x4.nxv4i32(target("aarch64.svcount") %0, ptr %arrayidx4)
+  %12 = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } %11, 0
+  %13 = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } %11, 1
+  %14 = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } %11, 2
+  %15 = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } %11, 3
+  %mul5 = mul i64 %stride, 3
+  %arrayidx6 = getelementptr inbounds i8, ptr %ptr, i64 %mul5
+  %16 = tail call { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.aarch64.sve.ld1.pn.x4.nxv4i32(target("aarch64.svcount") %0, ptr %arrayidx6)
+  %17 = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } %16, 0
+  %18 = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } %16, 1
+  %19 = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } %16, 2
+  %20 = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } %16, 3
+  %res1 = call <vscale x 16 x i8> @llvm.aarch64.sve.sqrshr.x4.nxv16i8(<vscale x 4 x i32> %2, <vscale x 4 x i32> %7, <vscale x 4 x i32> %12, <vscale x 4 x i32> %17, i32 32)
+  %res2 = call <vscale x 16 x i8> @llvm.aarch64.sve.sqrshr.x4.nxv16i8(<vscale x 4 x i32> %3, <vscale x 4 x i32> %8, <vscale x 4 x i32> %13, <vscale x 4 x i32> %18, i32 32)
+  %res3 = call <vscale x 16 x i8> @llvm.aarch64.sve.sqrshr.x4.nxv16i8(<vscale x 4 x i32> %4, <vscale x 4 x i32> %9, <vscale x 4 x i32> %14, <vscale x 4 x i32> %19, i32 32)
+  %res4 = call <vscale x 16 x i8> @llvm.aarch64.sve.sqrshr.x4.nxv16i8(<vscale x 4 x i32> %5, <vscale x 4 x i32> %10, <vscale x 4 x i32> %15, <vscale x 4 x i32> %20, i32 32)
+  %ins1 = insertvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } poison, <vscale x 16 x i8> %res1, 0
+  %ins2 = insertvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %ins1, <vscale x 16 x i8> %res2, 1
+  %ins3 = insertvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %ins2, <vscale x 16 x i8> %res3, 2
+  %ins4 = insertvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %ins3, <vscale x 16 x i8> %res4, 3
+  ret { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %ins4
+}
+
 define <vscale x 8 x i16> @multi_vector_sat_shift_narrow_x4_s16(<vscale x 2 x i64> %unused, <vscale x 2 x i64> %zn1, <vscale x 2 x i64> %zn2, <vscale x 2 x i64> %zn3, <vscale x 2 x i64> %zn4) {
 ; CHECK-LABEL: multi_vector_sat_shift_narrow_x4_s16:
 ; CHECK:       // %bb.0:
diff --git a/llvm/test/CodeGen/AArch64/sme2-intrinsics-vdot.ll b/llvm/test/CodeGen/AArch64/sme2-intrinsics-vdot.ll
index e7d1050b60799..63851dd857f97 100644
--- a/llvm/test/CodeGen/AArch64/sme2-intrinsics-vdot.ll
+++ b/llvm/test/CodeGen/AArch64/sme2-intrinsics-vdot.ll
@@ -99,7 +99,7 @@ entry:
   ret void
 }
 
-define void @svdot_form_2x_tuple_svecc(ptr %ptr, i64 %stride, <vscale x 16 x i8> %scalable_arg) #0 {
+define void @svdot_form_2x_tuple_svecc(ptr %ptr, i64 %stride, <vscale x 8 x i16> %scalable_arg) #0 {
 ; CHECK-LABEL: svdot_form_2x_tuple_svecc:
 ; CHECK:       // %bb.0: // %entry
 ; CHECK-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill
@@ -107,16 +107,18 @@ define void @svdot_form_2x_tuple_svecc(ptr %ptr, i64 %stride, <vscale x 16 x i8>
 ; CHECK-NEXT:    str p8, [sp, #7, mul vl] // 2-byte Folded Spill
 ; CHECK-NEXT:    ptrue pn8.b
 ; CHECK-NEXT:    add x9, x0, x1
-; CHECK-NEXT:    str z9, [sp, #1, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z11, [sp, #1, mul vl] // 16-byte Folded Spill
 ; CHECK-NEXT:    mov w8, wzr
-; CHECK-NEXT:    str z8, [sp, #2, mul vl] // 16-byte Folded Spill
-; CHECK-NEXT:    ld1h { z0.h, z8.h }, pn8/z, [x0]
-; CHECK-NEXT:    ld1h { z1.h, z9.h }, pn8/z, [x9]
-; CHECK-NEXT:    svdot za.s[w8, 0, vgx2], { z0.h, z1.h }, z0.h[0]
-; CHECK-NEXT:    svdot za.s[w8, 0, vgx2], { z8.h, z9.h }, z0.h[0]
-; CHECK-NEXT:    ldr z9, [sp, #1, mul vl] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr z8, [sp, #2, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ptrue p0.h
+; CHECK-NEXT:    str z10, [sp, #2, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    ld1h { z2.h, z10.h }, pn8/z, [x0]
+; CHECK-NEXT:    ld1h { z3.h, z11.h }, pn8/z, [x9]
+; CHECK-NEXT:    svdot za.s[w8, 0, vgx2], { z2.h, z3.h }, z0.h[0]
+; CHECK-NEXT:    svdot za.s[w8, 0, vgx2], { z10.h, z11.h }, z0.h[0]
+; CHECK-NEXT:    ldr z11, [sp, #1, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z10, [sp, #2, mul vl] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldr p8, [sp, #7, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    st1h { z0.h }, p0, [x0]
 ; CHECK-NEXT:    addvl sp, sp, #3
 ; CHECK-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
 ; CHECK-NEXT:    ret
@@ -131,6 +133,7 @@ entry:
   %6 = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16> } %4, 1
   tail call void @llvm.aarch64.sme.svdot.lane.za32.vg1x2.nxv8i16(i32 0, <vscale x 8 x i16> %2, <vscale x 8 x i16> %5, <vscale x 8 x i16> poison, i32 0)
   tail call void @llvm.aarch64.sme.svdot.lane.za32.vg1x2.nxv8i16(i32 0, <vscale x 8 x i16> %3, <vscale x 8 x i16> %6, <vscale x 8 x i16> poison, i32 0)
+  store <vscale x 8 x i16> %scalable_arg, ptr %ptr
   ret void
 }
 
@@ -184,7 +187,7 @@ entry:
   ret void
 }
 
-define void @svdot_form_4x_tuple_svecc(ptr %ptr, i64 %stride, <vscale x 16 x i8> %scalable_arg) #0 {
+define void @svdot_form_4x_tuple_svecc(ptr %ptr, i64 %stride, <vscale x 8 x i16> %scalable_arg) #0 {
 ; CHECK-LABEL: svdot_form_4x_tuple_svecc:
 ; CHECK:       // %bb.0: // %entry
 ; CHECK-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill
@@ -192,33 +195,35 @@ define void @svdot_form_4x_tuple_svecc(ptr %ptr, i64 %stride, <vscale x 16 x i8>
 ; CHECK-NEXT:    lsl x9, x1, #1
 ; CHECK-NEXT:    str p8, [sp, #7, mul vl] // 2-byte Folded Spill
 ; CHECK-NEXT:    ptrue pn8.b
-; CHECK-NEXT:    str z15, [sp, #1, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z23, [sp, #1, mul vl] // 16-byte Folded Spill
 ; CHECK-NEXT:    mov w8, wzr
-; CHECK-NEXT:    str z14, [sp, #2, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    ptrue p0.h
+; CHECK-NEXT:    str z22, [sp, #2, mul vl] // 16-byte Folded Spill
 ; CHECK-NEXT:    add x10, x9, x1
-; CHECK-NEXT:    str z13, [sp, #3, mul vl] // 16-byte Folded Spill
-; CHECK-NEXT:    str z12, [sp, #4, mul vl] // 16-byte Folded Spill
-; CHECK-NEXT:    str z11, [sp, #5, mul vl] // 16-byte Folded Spill
-; CHECK-NEXT:    str z10, [sp, #6, mul vl] // 16-byte Folded Spill
-; CHECK-NEXT:    str z9, [sp, #7, mul vl] // 16-byte Folded Spill
-; CHECK-NEXT:    str z8, [sp, #8, mul vl] // 16-byte Folded Spill
-; CHECK-NEXT:    ld1b { z0.b, z4.b, z8.b, z12.b }, pn8/z, [x0]
-; CHECK-NEXT:    ld1b { z1.b, z5.b, z9.b, z13.b }, pn8/z, [x0, x1]
-; CHECK-NEXT:    ld1b { z2.b, z6.b, z10.b, z14.b }, pn8/z, [x0, x9]
-; CHECK-NEXT:    ld1b { z3.b, z7.b, z11.b, z15.b }, pn8/z, [x0, x10]
-; CHECK-NEXT:    svdot za.s[w8, 0, vgx4], { z0.b - z3.b }, z0.b[0]
-; CHECK-NEXT:    svdot za.s[w8, 0, vgx4], { z4.b - z7.b }, z0.b[0]
-; CHECK-NEXT:    svdot za.s[w8, 0, vgx4], { z8.b - z11.b }, z0.b[0]
-; CHECK-NEXT:    svdot za.s[w8, 0, vgx4], { z12.b - z15.b }, z0.b[0]
-; CHECK-NEXT:    ldr z15, [sp, #1, mul vl] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr z14, [sp, #2, mul vl] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr z13, [sp, #3, mul vl] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr z12, [sp, #4, mul vl] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr z11, [sp, #5, mul vl] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr z10, [sp, #6, mul vl] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr z9, [sp, #7, mul vl] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr z8, [sp, #8, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    str z21, [sp, #3, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z20, [sp, #4, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z19, [sp, #5, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z18, [sp, #6, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z17, [sp, #7, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z16, [sp, #8, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    ld1b { z16.b, z20.b, z24.b, z28.b }, pn8/z, [x0]
+; CHECK-NEXT:    ld1b { z17.b, z21.b, z25.b, z29.b }, pn8/z, [x0, x1]
+; CHECK-NEXT:    ld1b { z18.b, z22.b, z26.b, z30.b }, pn8/z, [x0, x9]
+; CHECK-NEXT:    ld1b { z19.b, z23.b, z27.b, z31.b }, pn8/z, [x0, x10]
+; CHECK-NEXT:    svdot za.s[w8, 0, vgx4], { z16.b - z19.b }, z0.b[0]
+; CHECK-NEXT:    svdot za.s[w8, 0, vgx4], { z20.b - z23.b }, z0.b[0]
+; CHECK-NEXT:    svdot za.s[w8, 0, vgx4], { z24.b - z27.b }, z0.b[0]
+; CHECK-NEXT:    svdot za.s[w8, 0, vgx4], { z28.b - z31.b }, z0.b[0]
+; CHECK-NEXT:    ldr z23, [sp, #1, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z22, [sp, #2, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z21, [sp, #3, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z20, [sp, #4, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z19, [sp, #5, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z18, [sp, #6, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z17, [sp, #7, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z16, [sp, #8, mul vl] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldr p8, [sp, #7, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    st1h { z0.h }, p0, [x0]
 ; CHECK-NEXT:    addvl sp, sp, #9
 ; CHECK-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
 ; CHECK-NEXT:    ret
@@ -253,6 +258,7 @@ entry:
   tail call void @llvm.aarch64.sme.svdot.lane.za32.vg1x4.nxv16i8(i32 0, <vscale x 16 x i8> %3, <vscale x 16 x i8> %8, <vscale x 16 x i8> %13, <vscale x 16 x i8> %18, <vscale x 16 x i8> poison, i32 0)
   tail call void @llvm.aarch64.sme.svdot.lane.za32.vg1x4.nxv16i8(i32 0, <vscale x 16 x i8> %4, <vscale x 16 x i8> %9, <vscale x 16 x i8> %14, <vscale x 16 x i8> %19, <vscale x 16 x i8> poison, i32 0)
   tail call void @llvm.aarch64.sme.svdot.lane.za32.vg1x4.nxv16i8(i32 0, <vscale x 16 x i8> %5, <vscale x 16 x i8> %10, <vscale x 16 x i8> %15, <vscale x 16 x i8> %20, <vscale x 16 x i8> poison, i32 0)
+  store <vscale x 8 x i16> %scalable_arg, ptr %ptr
   ret void
 }
 
@@ -322,7 +328,7 @@ entry:
   ret void
 }
 
-define void @uvdot_form_2x_tuple_svecc(ptr %ptr, i64 %stride, <vscale x 16 x i8> %scalable_arg) #0 {
+define void @uvdot_form_2x_tuple_svecc(ptr %ptr, i64 %stride, <vscale x 8 x i16> %scalable_arg) #0 {
 ; CHECK-LABEL: uvdot_form_2x_tuple_svecc:
 ; CHECK:       // %bb.0: // %entry
 ; CHECK-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill
@@ -330,16 +336,18 @@ define void @uvdot_form_2x_tuple_svecc(ptr %ptr, i64 %stride, <vscale x 16 x i8>
 ; CHECK-NEXT:    str p8, [sp, #7, mul vl] // 2-byte Folded Spill
 ; CHECK-NEXT:    ptrue pn8.b
 ; CHECK-NEXT:    add x9, x0, x1
-; CHECK-NEXT:    str z9, [sp, #1, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z11, [sp, #1, mul vl] // 16-byte Folded Spill
 ; CHECK-NEXT:    mov w8, wzr
-; CHECK-NEXT:    str z8, [sp, #2, mul vl] // 16-byte Folded Spill
-; CHECK-NEXT:    ld1h { z0.h, z8.h }, pn8/z, [x0]
-; CHECK-NEXT:    ld1h { z1.h, z9.h }, pn8/z, [x9]
-; CHECK-NEXT:    uvdot za.s[w8, 0, vgx2], { z0.h, z1.h }, z0.h[0]
-; CHECK-NEXT:    uvdot za.s[w8, 0, vgx2], { z8.h, z9.h }, z0.h[0]
-; CHECK-NEXT:    ldr z9, [sp, #1, mul vl] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr z8, [sp, #2, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ptrue p0.h
+; CHECK-NEXT:    str z10, [sp, #2, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    ld1h { z2.h, z10.h }, pn8/z, [x0]
+; CHECK-NEXT:    ld1h { z3.h, z11.h }, pn8/z, [x9]
+; CHECK-NEXT:    uvdot za.s[w8, 0, vgx2], { z2.h, z3.h }, z0.h[0]
+; CHECK-NEXT:    uvdot za.s[w8, 0, vgx2], { z10.h, z11.h }, z0.h[0]
+; CHECK-NEXT:    ldr z11, [sp, #1, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z10, [sp, #2, mul vl] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldr p8, [sp, #7, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    st1h { z0.h }, p0, [x0]
 ; CHECK-NEXT:    addvl sp, sp, #3
 ; CHECK-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
 ; CHECK-NEXT:    ret
@@ -354,6 +362,7 @@ entry:
   %6 = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16> } %4, 1
   tail call void @llvm.aarch64.sme.uvdot.lane.za32.vg1x2.nxv8i16(i32 0, <vscale x 8 x i16> %2, <vscale x 8 x i16> %5, <vscale x 8 x i16> poison, i32 0)
   tail call void @llvm.aarch64.sme.uvdot.lane.za32.vg1x2.nxv8i16(i32 0, <vscale x 8 x i16> %3, <vscale x 8 x i16> %6, <vscale x 8 x i16> poison, i32 0)
+  store <vscale x 8 x i16> %scalable_arg, ptr %ptr
   ret void
 }
 
@@ -407,7 +416,7 @@ entry:
   ret void
 }
 
-define void @uvdot_form_4x_tuple_svecc(ptr %ptr, i64 %stride, <vscale x 16 x i8> %scalable_arg) #0 {
+define void @uvdot_form_4x_tuple_svecc(ptr %ptr, i64 %stride, <vscale x 8 x i16> %scalable_arg) #0 {
 ; CHECK-LABEL: uvdot_form_4x_tuple_svecc:
 ; CHECK:       // %bb.0: // %entry
 ; CHECK-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill
@@ -415,33 +424,35 @@ define void @uvdot_form_4x_tuple_svecc(ptr %ptr, i64 %stride, <vscale x 16 x i8>
 ; CHECK-NEXT:    lsl x9, x1, #1
 ; CHECK-NEXT:    str p8, [sp, #7, mul vl] // 2-byte Folded Spill
 ; CHECK-NEXT:    ptrue pn8.b
-; CHECK-NEXT:    str z15, [sp, #1, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z23, [sp, #1, mul vl] // 16-byte Folded Spill
 ; CHECK-NEXT:    mov w8, wzr
-; CHECK-NEXT:    str z14, [sp, #2, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    ptrue p0.h
+; CHECK-NEXT:    str z22, [sp, #2, mul vl] // 16-byte Folded Spill
 ; CHECK-NEXT:    add x10, x9, x1
-; CHECK-NEXT:    str z13, [sp, #3, mul vl] // 16-byte Folded Spill
-; CHECK-NEXT:    str z12, [sp, #4, mul vl] // 16-byte Folded Spill
-; CHECK-NEXT:    str z11, [sp, #5, mul vl] // 16-byte Folded Spill
-; CHECK-NEXT:    str z10, [sp, #6, mul vl] // 16-byte Folded Spill
-; CHECK-NEXT:    str z9, [sp, #7, mul vl] // 16-byte Folded Spill
-; CHECK-NEXT:    str z8, [sp, #8, mul vl] // 16-byte Folded Spill
-; CHECK-NEXT:    ld1b { z0.b, z4.b, z8.b, z12.b }, pn8/z, [x0]
-; CHECK-NEXT:    ld1b { z1.b, z5.b, z9.b, z13.b }, pn8/z, [x0, x1]
-; CHECK-NEXT:    ld1b { z2.b, z6.b, z10.b, z14.b }, pn8/z, [x0, x9]
-; CHECK-NEXT:    ld1b { z3.b, z7.b, z11.b, z15.b }, pn8/z, [x0, x10]
-; CHECK-NEXT:    uvdot za.s[w8, 0, vgx4], { z0.b - z3.b }, z0.b[0]
-; CHECK-NEXT:    uvdot za.s[w8, 0, vgx4], { z4.b - z7.b }, z0.b[0]
-; CHECK-NEXT:    uvdot za.s[w8, 0, vgx4], { z8.b - z11.b }, z0.b[0]
-; CHECK-NEXT:    uvdot za.s[w8, 0, vgx4], { z12.b - z15.b }, z0.b[0]
-; CHECK-NEXT:    ldr z15, [sp, #1, mul vl] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr z14, [sp, #2, mul vl] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr z13, [sp, #3, mul vl] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr z12, [sp, #4, mul vl] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr z11, [sp, #5, mul vl] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr z10, [sp, #6, mul vl] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr z9, [sp, #7, mul vl] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr z8, [sp, #8, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    str z21, [sp, #3, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z20, [sp, #4, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z19, [sp, #5, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z18, [sp, #6, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z17, [sp, #7, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z16, [sp, #8, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    ld1b { z16.b, z20.b, z24.b, z28.b }, pn8/z, [x0]
+; CHECK-NEXT:    ld1b { z17.b, z21.b, z25.b, z29.b }, pn8/z, [x0, x1]
+; CHECK-NEXT:    ld1b { z18.b, z22.b, z26.b, z30.b }, pn8/z, [x0, x9]
+; CHECK-NEXT:    ld1b { z19.b, z23.b, z27.b, z31.b }, pn8/z, [x0, x10]
+; CHECK-NEXT:    uvdot za.s[w8, 0, vgx4], { z16.b - z19.b }, z0.b[0]
+; CHECK-NEXT:    uvdot za.s[w8, 0, vgx4], { z20.b - z23.b }, z0.b[0]
+; CHECK-NEXT:    uvdot za.s[w8, 0, vgx4], { z24.b - z27.b }, z0.b[0]
+; CHECK-NEXT:    uvdot za.s[w8, 0, vgx4], { z28.b - z31.b }, z0.b[0]
+; CHECK-NEXT:    ldr z23, [sp, #1, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z22, [sp, #2, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z21, [sp, #3, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z20, [sp, #4, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z19, [sp, #5, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z18, [sp, #6, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z17, [sp, #7, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z16, [sp, #8, mul vl] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldr p8, [sp, #7, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    st1h { z0.h }, p0, [x0]
 ; CHECK-NEXT:    addvl sp, sp, #9
 ; CHECK-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
 ; CHECK-NEXT:    ret
@@ -476,6 +487,7 @@ entry:
   tail call void @llvm.aarch64.sme.uvdot.lane.za32.vg1x4.nxv16i8(i32 0, <vscale x 16 x i8> %3, <vscale x 16 x i8> %8, <vscale x 16 x i8> %13, <vscale x 16 x i8> %18, <vscale x 16 x i8> poison, i32 0)
   tail call void @llvm.aarch64.sme.uvdot.lane.za32.vg1x4.nxv16i8(i32 0, <vscale x 16 x i8> %4, <vscale x 16 x i8> %9, <vscale x 16 x i8> %14, <vscale x 16 x i8> %19, <vscale x 16 x i8> poison, i32 0)
   tail call void @llvm.aarch64.sme.uvdot.lane.za32.vg1x4.nxv16i8(i32 0, <vscale x 16 x i8> %5, <vscale x 16 x i8> %10, <vscale x 16 x i8> %15, <vscale x 16 x i8> %20, <vscale x 16 x i8> poison, i32 0)
+  store <vscale x 8 x i16> %scalable_arg, ptr %ptr
   ret void
 }
 
@@ -544,7 +556,7 @@ entry:
   ret void
 }
 
-define void @suvdot_form_4x_tuple_svecc(ptr %ptr, i64 %stride, <vscale x 16 x i8> %scalable_arg) #0 {
+define void @suvdot_form_4x_tuple_svecc(ptr %ptr, i64 %stride, <vscale x 8 x i16> %scalable_arg) #0 {
 ; CHECK-LABEL: suvdot_form_4x_tuple_svecc:
 ; CHECK:       // %bb.0: // %entry
 ; CHECK-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill
@@ -552,33 +564,35 @@ define void @suvdot_form_4x_tuple_svecc(ptr %ptr, i64 %stride, <vscale x 16 x i8
 ; CHECK-NEXT:    lsl x9, x1, #1
 ; CHECK-NEXT:    str p8, [sp, #7, mul vl] // 2-byte Folded Spill
 ; CHECK-NEXT:    ptrue pn8.b
-; CHECK-NEXT:    str z15, [sp, #1, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z23, [sp, #1, mul vl] // 16-byte Folded Spill
 ; CHECK-NEXT:    mov w8, wzr
-; CHECK-NEXT:    str z14, [sp, #2, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    ptrue p0.h
+; CHECK-NEXT:    str z22, [sp, #2, mul vl] // 16-byte Folded Spill
 ; CHECK-NEXT:    add x10, x9, x1
-; CHECK-NEXT:    str z13, [sp, #3, mul vl] // 16-byte Folded Spill
-; CHECK-NEXT:    str z12, [sp, #4, mul vl] // 16-byte Folded Spill
-; CHECK-NEXT:    str z11, [sp, #5, mul vl] // 16-byte Folded Spill
-; CHECK-NEXT:    str z10, [sp, #6, mul vl] // 16-byte Folded Spill
-; CHECK-NEXT:    str z9, [sp, #7, mul vl] // 16-byte Folded Spill
-; CHECK-NEXT:    str z8, [sp, #8, mul vl] // 16-byte Folded Spill
-; CHECK-NEXT:    ld1b { z0.b, z4.b, z8.b, z12.b }, pn8/z, [x0]
-; CHECK-NEXT:    ld1b { z1.b, z5.b, z9.b, z13.b }, pn8/z, [x0, x1]
-; CHECK-NEXT:    ld1b { z2.b, z6.b, z10.b, z14.b }, pn8/z, [x0, x9]
-; CHECK-NEXT:    ld1b { z3.b, z7.b, z11.b, z15.b }, pn8/z, [x0, x10]
-; CHECK-NEXT:    suvdot za.s[w8, 0, vgx4], { z0.b - z3.b }, z0.b[0]
-; CHECK-NEXT:    suvdot za.s[w8, 0, vgx4], { z4.b - z7.b }, z0.b[0]
-; CHECK-NEXT:    suvdot za.s[w8, 0, vgx4], { z8.b - z11.b }, z0.b[0]
-; CHECK-NEXT:    suvdot za.s[w8, 0, vgx4], { z12.b - z15.b }, z0.b[0]
-; CHECK-NEXT:    ldr z15, [sp, #1, mul vl] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr z14, [sp, #2, mul vl] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr z13, [sp, #3, mul vl] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr z12, [sp, #4, mul vl] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr z11, [sp, #5, mul vl] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr z10, [sp, #6, mul vl] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr z9, [sp, #7, mul vl] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr z8, [sp, #8, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    str z21, [sp, #3, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z20, [sp, #4, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z19, [sp, #5, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z18, [sp, #6, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z17, [sp, #7, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z16, [sp, #8, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    ld1b { z16.b, z20.b, z24.b, z28.b }, pn8/z, [x0]
+; CHECK-NEXT:    ld1b { z17.b, z21.b, z25.b, z29.b }, pn8/z, [x0, x1]
+; CHECK-NEXT:    ld1b { z18.b, z22.b, z26.b, z30.b }, pn8/z, [x0, x9]
+; CHECK-NEXT:    ld1b { z19.b, z23.b, z27.b, z31.b }, pn8/z, [x0, x10]
+; CHECK-NEXT:    suvdot za.s[w8, 0, vgx4], { z16.b - z19.b }, z0.b[0]
+; CHECK-NEXT:    suvdot za.s[w8, 0, vgx4], { z20.b - z23.b }, z0.b[0]
+; CHECK-NEXT:    suvdot za.s[w8, 0, vgx4], { z24.b - z27.b }, z0.b[0]
+; CHECK-NEXT:    suvdot za.s[w8, 0, vgx4], { z28.b - z31.b }, z0.b[0]
+; CHECK-NEXT:    ldr z23, [sp, #1, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z22, [sp, #2, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z21, [sp, #3, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z20, [sp, #4, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z19, [sp, #5, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z18, [sp, #6, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z17, [sp, #7, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z16, [sp, #8, mul vl] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldr p8, [sp, #7, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    st1h { z0.h }, p0, [x0]
 ; CHECK-NEXT:    addvl sp, sp, #9
 ; CHECK-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
 ; CHECK-NEXT:    ret
@@ -613,6 +627,7 @@ entry:
   tail call void @llvm.aarch64.sme.suvdot.lane.za32.vg1x4.nxv16i8(i32 0, <vscale x 16 x i8> %3, <vscale x 16 x i8> %8, <vscale x 16 x i8> %13, <vscale x 16 x i8> %18, <vscale x 16 x i8> poison, i32 0)
   tail call void @llvm.aarch64.sme.suvdot.lane.za32.vg1x4.nxv16i8(i32 0, <vscale x 16 x i8> %4, <vscale x 16 x i8> %9, <vscale x 16 x i8> %14, <vscale x 16 x i8> %19, <vscale x 16 x i8> poison, i32 0)
   tail call void @llvm.aarch64.sme.suvdot.lane.za32.vg1x4.nxv16i8(i32 0, <vscale x 16 x i8> %5, <vscale x 16 x i8> %10, <vscale x 16 x i8> %15, <vscale x 16 x i8> %20, <vscale x 16 x i8> poison, i32 0)
+  store <vscale x 8 x i16> %scalable_arg, ptr %ptr
   ret void
 }
 
@@ -681,7 +696,7 @@ entry:
   ret void
 }
 
-define void @usvdot_form_4x_tuple_svecc(ptr %ptr, i64 %stride, <vscale x 16 x i8> %scalable_arg) #0 {
+define void @usvdot_form_4x_tuple_svecc(ptr %ptr, i64 %stride, <vscale x 8 x i16> %scalable_arg) #0 {
 ; CHECK-LABEL: usvdot_form_4x_tuple_svecc:
 ; CHECK:       // %bb.0: // %entry
 ; CHECK-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill
@@ -689,33 +704,35 @@ define void @usvdot_form_4x_tuple_svecc(ptr %ptr, i64 %stride, <vscale x 16 x i8
 ; CHECK-NEXT:    lsl x9, x1, #1
 ; CHECK-NEXT:    str p8, [sp, #7, mul vl] // 2-byte Folded Spill
 ; CHECK-NEXT:    ptrue pn8.b
-; CHECK-NEXT:    str z15, [sp, #1, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z23, [sp, #1, mul vl] // 16-byte Folded Spill
 ; CHECK-NEXT:    mov w8, wzr
-; CHECK-NEXT:    str z14, [sp, #2, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    ptrue p0.h
+; CHECK-NEXT:    str z22, [sp, #2, mul vl] // 16-byte Folded Spill
 ; CHECK-NEXT:    add x10, x9, x1
-; CHECK-NEXT:    str z13, [sp, #3, mul vl] // 16-byte Folded Spill
-; CHECK-NEXT:    str z12, [sp, #4, mul vl] // 16-byte Folded Spill
-; CHECK-NEXT:    str z11, [sp, #5, mul vl] // 16-byte Folded Spill
-; CHECK-NEXT:    str z10, [sp, #6, mul vl] // 16-byte Folded Spill
-; CHECK-NEXT:    str z9, [sp, #7, mul vl] // 16-byte Folded Spill
-; CHECK-NEXT:    str z8, [sp, #8, mul vl] // 16-byte Folded Spill
-; CHECK-NEXT:    ld1b { z0.b, z4.b, z8.b, z12.b }, pn8/z, [x0]
-; CHECK-NEXT:    ld1b { z1.b, z5.b, z9.b, z13.b }, pn8/z, [x0, x1]
-; CHECK-NEXT:    ld1b { z2.b, z6.b, z10.b, z14.b }, pn8/z, [x0, x9]
-; CHECK-NEXT:    ld1b { z3.b, z7.b, z11.b, z15.b }, pn8/z, [x0, x10]
-; CHECK-NEXT:    usvdot za.s[w8, 0, vgx4], { z0.b - z3.b }, z0.b[0]
-; CHECK-NEXT:    usvdot za.s[w8, 0, vgx4], { z4.b - z7.b }, z0.b[0]
-; CHECK-NEXT:    usvdot za.s[w8, 0, vgx4], { z8.b - z11.b }, z0.b[0]
-; CHECK-NEXT:    usvdot za.s[w8, 0, vgx4], { z12.b - z15.b }, z0.b[0]
-; CHECK-NEXT:    ldr z15, [sp, #1, mul vl] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr z14, [sp, #2, mul vl] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr z13, [sp, #3, mul vl] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr z12, [sp, #4, mul vl] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr z11, [sp, #5, mul vl] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr z10, [sp, #6, mul vl] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr z9, [sp, #7, mul vl] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr z8, [sp, #8, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    str z21, [sp, #3, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z20, [sp, #4, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z19, [sp, #5, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z18, [sp, #6, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z17, [sp, #7, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z16, [sp, #8, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    ld1b { z16.b, z20.b, z24.b, z28.b }, pn8/z, [x0]
+; CHECK-NEXT:    ld1b { z17.b, z21.b, z25.b, z29.b }, pn8/z, [x0, x1]
+; CHECK-NEXT:    ld1b { z18.b, z22.b, z26.b, z30.b }, pn8/z, [x0, x9]
+; CHECK-NEXT:    ld1b { z19.b, z23.b, z27.b, z31.b }, pn8/z, [x0, x10]
+; CHECK-NEXT:    usvdot za.s[w8, 0, vgx4], { z16.b - z19.b }, z0.b[0]
+; CHECK-NEXT:    usvdot za.s[w8, 0, vgx4], { z20.b - z23.b }, z0.b[0]
+; CHECK-NEXT:    usvdot za.s[w8, 0, vgx4], { z24.b - z27.b }, z0.b[0]
+; CHECK-NEXT:    usvdot za.s[w8, 0, vgx4], { z28.b - z31.b }, z0.b[0]
+; CHECK-NEXT:    ldr z23, [sp, #1, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z22, [sp, #2, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z21, [sp, #3, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z20, [sp, #4, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z19, [sp, #5, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z18, [sp, #6, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z17, [sp, #7, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z16, [sp, #8, mul vl] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldr p8, [sp, #7, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    st1h { z0.h }, p0, [x0]
 ; CHECK-NEXT:    addvl sp, sp, #9
 ; CHECK-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
 ; CHECK-NEXT:    ret
@@ -750,6 +767,7 @@ entry:
   tail call void @llvm.aarch64.sme.usvdot.lane.za32.vg1x4.nxv16i8(i32 0, <vscale x 16 x i8> %3, <vscale x 16 x i8> %8, <vscale x 16 x i8> %13, <vscale x 16 x i8> %18, <vscale x 16 x i8> poison, i32 0)
   tail call void @llvm.aarch64.sme.usvdot.lane.za32.vg1x4.nxv16i8(i32 0, <vscale x 16 x i8> %4, <vscale x 16 x i8> %9, <vscale x 16 x i8> %14, <vscale x 16 x i8> %19, <vscale x 16 x i8> poison, i32 0)
   tail call void @llvm.aarch64.sme.usvdot.lane.za32.vg1x4.nxv16i8(i32 0, <vscale x 16 x i8> %5, <vscale x 16 x i8> %10, <vscale x 16 x i8> %15, <vscale x 16 x i8> %20, <vscale x 16 x i8> poison, i32 0)
+  store <vscale x 8 x i16> %scalable_arg, ptr %ptr
   ret void
 }
 
diff --git a/llvm/test/CodeGen/AArch64/sme2-multivec-regalloc.mir b/llvm/test/CodeGen/AArch64/sme2-multivec-regalloc.mir
new file mode 100644
index 0000000000000..1d04cc6d7ca2a
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/sme2-multivec-regalloc.mir
@@ -0,0 +1,184 @@
+# RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sme2 -force-streaming -verify-machineinstrs -enable-subreg-liveness -start-before=greedy %s -o - | FileCheck %s
+
+# No available group of four strided x4 registers, fall back on default allocation order
+---
+name:            form_4x_tuple_many_live
+tracksRegLiveness: true
+stack:
+  - { id: 0, name: '', type: default, offset: 0, size: 32, alignment: 16,
+      stack-id: scalable-vector, callee-saved-register: '', callee-saved-restored: true,
+      debug-info-variable: '', debug-info-expression: '', debug-info-location: '' }
+body:             |
+  bb.0.entry:
+    liveins: $x0, $x1, $z0, $z17
+
+    ; CHECK-LABEL: form_4x_tuple_many_live
+    ; CHECK: stp d11, d10, [sp, #-48]!
+    ; CHECK-NEXT: stp d9, d8, [sp, #16]
+    ; CHECK-NEXT: str x29, [sp, #32]
+    ; CHECK-NEXT: addvl sp, sp, #-2
+    ; CHECK-NEXT: .cfi_escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x30, 0x22, 0x11, 0x10, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 48 + 16 * VG
+    ; CHECK-NEXT: .cfi_offset w29, -16
+    ; CHECK-NEXT: .cfi_offset b8, -24
+    ; CHECK-NEXT: .cfi_offset b9, -32
+    ; CHECK-NEXT: .cfi_offset b10, -40
+    ; CHECK-NEXT: .cfi_offset b11, -48
+    ; CHECK-NEXT: lsl x9, x1, #1
+    ; CHECK-NEXT: ptrue pn8.b
+    ; CHECK-NEXT: mov w8, wzr
+    ; CHECK-NEXT: ld1b { z16.b, z20.b, z24.b, z28.b }, pn8/z, [x0]
+    ; CHECK-NEXT: ld1b { z18.b, z22.b, z26.b, z30.b }, pn8/z, [x0, x1]
+    ; CHECK-NEXT: ptrue p0.b
+    ; CHECK-NEXT: add x10, x9, x1
+    ; CHECK-NEXT: ld1b { z19.b, z23.b, z27.b, z31.b }, pn8/z, [x0, x9]
+    ; CHECK-NEXT: ld1b { z4.b - z7.b }, pn8/z, [x0, x10]
+    ; CHECK-NEXT: mov z8.d, z16.d
+    ; CHECK-NEXT: mov z9.d, z18.d
+    ; CHECK-NEXT: mov z21.d, z22.d
+    ; CHECK-NEXT: mov z10.d, z19.d
+    ; CHECK-NEXT: mov z22.d, z23.d
+    ; CHECK-NEXT: mov z25.d, z26.d
+    ; CHECK-NEXT: mov z11.d, z4.d
+    ; CHECK-NEXT: mov z23.d, z5.d
+    ; CHECK-NEXT: mov z26.d, z27.d
+    ; CHECK-NEXT: mov z27.d, z6.d
+    ; CHECK-NEXT: mov z29.d, z30.d
+    ; CHECK-NEXT: mov z30.d, z31.d
+    ; CHECK-NEXT: mov z31.d, z7.d
+    ; CHECK-NEXT: udot za.s[w8, 0, vgx4], { z8.b - z11.b }, z0.b[0]
+    ; CHECK-NEXT: udot za.s[w8, 0, vgx4], { z20.b - z23.b }, z0.b[0]
+    ; CHECK-NEXT: udot za.s[w8, 0, vgx4], { z24.b - z27.b }, z0.b[0]
+    ; CHECK-NEXT: udot za.s[w8, 0, vgx4], { z28.b - z31.b }, z0.b[0]
+    ; CHECK-NEXT: st1b { z0.b }, p0, [x0]
+    ; CHECK-NEXT: st1b { z17.b }, p0, [x0]
+    ; CHECK-NEXT: addvl sp, sp, #2
+    ; CHECK-NEXT: ldp d9, d8, [sp, #16]
+    ; CHECK-NEXT: ldr x29, [sp, #32]
+    ; CHECK-NEXT: ldp d11, d10, [sp], #48
+    ; CHECK-NEXT: ret
+
+    %0:gpr64common = COPY $x0
+    %1:gpr64 = COPY $x1
+    %2:zpr = COPY $z0
+    %3:zpr = COPY $z17
+    %5:matrixindexgpr32_8_11 = COPY $wzr
+    %6:gpr64 = UBFMXri %1, 63, 62
+    %pred:pnr_p8to15 = PTRUE_C_B implicit $vg
+    %7:ppr_3b = PTRUE_B 31, implicit $vg
+    %8:gpr64 = ADDXrr %6, %1
+    %9:zpr4stridedorcontiguous  = LD1B_4Z_IMM_PSEUDO %pred, %0, 0
+    %10:zpr4stridedorcontiguous = LD1B_4Z_PSEUDO %pred, %0, %1
+    %11:zpr4stridedorcontiguous = LD1B_4Z_PSEUDO %pred, %0, %6
+    %12:zpr4stridedorcontiguous = LD1B_4Z_PSEUDO %pred, %0, %8
+    %13:zpr4mul4 = FORM_TRANSPOSED_REG_TUPLE_X4_PSEUDO %9.zsub0, %10.zsub0, %11.zsub0, %12.zsub0
+    %14:zpr4mul4 = FORM_TRANSPOSED_REG_TUPLE_X4_PSEUDO %9.zsub1, %10.zsub1, %11.zsub1, %12.zsub1
+    %15:zpr4mul4 = FORM_TRANSPOSED_REG_TUPLE_X4_PSEUDO %9.zsub2, %10.zsub2, %11.zsub2, %12.zsub2
+    %16:zpr4mul4 = FORM_TRANSPOSED_REG_TUPLE_X4_PSEUDO %9.zsub3, %10.zsub3, %11.zsub3, %12.zsub3
+    $za = UDOT_VG4_M4ZZI_BtoS $za, %5, 0, %13, undef %28:zpr_4b, 0
+    $za = UDOT_VG4_M4ZZI_BtoS $za, %5, 0, %14, undef %30:zpr_4b, 0
+    $za = UDOT_VG4_M4ZZI_BtoS $za, %5, 0, %15, undef %32:zpr_4b, 0
+    $za = UDOT_VG4_M4ZZI_BtoS $za, %5, 0, %16, undef %34:zpr_4b, 0
+    ST1B_IMM %2, %7, %0, 0 :: (store (<vscale x 1 x s128>) into %stack.0)
+    ST1B_IMM %3, %7, %0, 0 :: (store (<vscale x 1 x s128>) into %stack.0)
+    RET_ReallyLR
+...
+
+# First multi-vector load to be allocated is not the first operand of the FORM_TRANSPOSED pseudo
+---
+name:            form_4x_tuple_allocation_order
+tracksRegLiveness: true
+stack:
+  - { id: 0, name: '', type: default, offset: 0, size: 32, alignment: 16,
+      stack-id: scalable-vector, callee-saved-register: '', callee-saved-restored: true,
+      debug-info-variable: '', debug-info-expression: '', debug-info-location: '' }
+body:             |
+  bb.0.entry:
+    liveins: $x0, $x1, $z0
+
+    ; CHECK: str x29, [sp, #-16]!
+    ; CHECK-NEXT: addvl	sp, sp, #-2
+    ; CHECK-NEXT: .cfi_escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x10, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 16 * VG
+    ; CHECK-NEXT: .cfi_offset w29, -16
+    ; CHECK-NEXT: lsl x9, x1, #1
+    ; CHECK-NEXT: ptrue	pn8.b
+    ; CHECK-NEXT: mov w8, wzr
+    ; CHECK-NEXT: ptrue	p0.b
+    ; CHECK-NEXT: add x10, x9, x1
+    ; CHECK-NEXT: ld1b { z19.b, z23.b, z27.b, z31.b }, pn8/z, [x0, x10]
+    ; CHECK-NEXT: ld1b { z18.b, z22.b, z26.b, z30.b }, pn8/z, [x0, x9]
+    ; CHECK-NEXT: ld1b { z17.b, z21.b, z25.b, z29.b }, pn8/z, [x0, x1]
+    ; CHECK-NEXT: ld1b { z16.b, z20.b, z24.b, z28.b }, pn8/z, [x0]
+    ; CHECK-NEXT: udot za.s[w8, 0, vgx4], { z16.b - z19.b }, z0.b[0]
+    ; CHECK-NEXT: udot za.s[w8, 0, vgx4], { z20.b - z23.b }, z0.b[0]
+    ; CHECK-NEXT: udot za.s[w8, 0, vgx4], { z24.b - z27.b }, z0.b[0]
+    ; CHECK-NEXT: udot za.s[w8, 0, vgx4], { z28.b - z31.b }, z0.b[0]
+    ; CHECK-NEXT: st1b { z0.b }, p0, [x0]
+    ; CHECK-NEXT: addvl	sp, sp, #2
+    ; CHECK-NEXT: ldr x29, [sp], #16
+    ; CHECK-NEXT: ret
+
+    %0:gpr64common = COPY $x0
+    %1:gpr64 = COPY $x1
+    %2:zpr = COPY $z0
+    %5:matrixindexgpr32_8_11 = COPY $wzr
+    %6:gpr64 = UBFMXri %1, 63, 62
+    %pred:pnr_p8to15 = PTRUE_C_B implicit $vg
+    %7:ppr_3b = PTRUE_B 31, implicit $vg
+    %8:gpr64 = ADDXrr %6, %1
+    %9:zpr4stridedorcontiguous = LD1B_4Z_PSEUDO %pred, %0, %8
+    %10:zpr4stridedorcontiguous = LD1B_4Z_PSEUDO %pred, %0, %6
+    %11:zpr4stridedorcontiguous = LD1B_4Z_PSEUDO %pred, %0, %1
+    %12:zpr4stridedorcontiguous  = LD1B_4Z_IMM_PSEUDO %pred, %0, 0
+    %13:zpr4mul4 = FORM_TRANSPOSED_REG_TUPLE_X4_PSEUDO %12.zsub0, %11.zsub0, %10.zsub0, %9.zsub0
+    %14:zpr4mul4 = FORM_TRANSPOSED_REG_TUPLE_X4_PSEUDO %12.zsub1, %11.zsub1, %10.zsub1, %9.zsub1
+    %15:zpr4mul4 = FORM_TRANSPOSED_REG_TUPLE_X4_PSEUDO %12.zsub2, %11.zsub2, %10.zsub2, %9.zsub2
+    %16:zpr4mul4 = FORM_TRANSPOSED_REG_TUPLE_X4_PSEUDO %12.zsub3, %11.zsub3, %10.zsub3, %9.zsub3
+    $za = UDOT_VG4_M4ZZI_BtoS $za, %5, 0, %13, undef %28:zpr_4b, 0
+    $za = UDOT_VG4_M4ZZI_BtoS $za, %5, 0, %14, undef %30:zpr_4b, 0
+    $za = UDOT_VG4_M4ZZI_BtoS $za, %5, 0, %15, undef %32:zpr_4b, 0
+    $za = UDOT_VG4_M4ZZI_BtoS $za, %5, 0, %16, undef %34:zpr_4b, 0
+    ST1B_IMM %2, %7, %0, 0 :: (store (<vscale x 1 x s128>) into %stack.0)
+    RET_ReallyLR
+...
+
+# Strided order is [ $z16_z24 $z17_z25 $z18_z26 $z19_z27 $z20_z28 $z21_z29 $z22_z30 $z23_z31 $z0_z8 $z1_z9 $z2_z10 $z3_z11 $z4_z12 $z5_z13 $z6_z14 $z7_z15 ]
+# Ensure we don't allocate $z23_z31 & $z0_z8 although they are consecutive
+---
+ name:            udot_form_2x_tuple_live_reg_order
+ tracksRegLiveness: true
+ body:             |
+   bb.0.entry:
+     liveins: $x0, $x1, $z16, $z17, $z18, $z19, $z20, $z21, $z22
+
+    ; CHECK: stp d9, d8, [sp, #-16]!
+    ; CHECK-NEXT: .cfi_def_cfa_offset 16
+    ; CHECK-NEXT: .cfi_offset b8, -8
+    ; CHECK-NEXT: .cfi_offset b9, -16
+    ; CHECK-NEXT: ptrue	pn8.b
+    ; CHECK-NEXT: mov w8, wzr
+    ; CHECK-NEXT: ld1b { z0.b, z8.b }, pn8/z, [x0]
+    ; CHECK-NEXT: ld1b { z1.b, z9.b }, pn8/z, [x0, x1]
+    ; CHECK-NEXT: udot za.s[w8, 0, vgx2], { z0.b, z1.b }, z0.b
+    ; CHECK-NEXT: udot za.s[w8, 0, vgx2], { z8.b, z9.b }, z0.b
+    ; CHECK-NEXT: ldp d9, d8, [sp], #16
+    ; CHECK-NEXT: ret
+
+     %0:gpr64 = COPY $x1
+     %1:gpr64common = COPY $x0
+     %2:zpr = COPY $z16
+     %3:zpr = COPY $z17
+     %4:zpr = COPY $z18
+     %5:zpr = COPY $z19
+     %6:zpr = COPY $z20
+     %7:zpr = COPY $z21
+     %8:zpr = COPY $z22
+     %9:matrixindexgpr32_8_11 = COPY $wzr
+     %10:pnr_p8to15 = PTRUE_C_B implicit $vg
+     %11:zpr2stridedorcontiguous = LD1B_2Z_IMM_PSEUDO %10, %1, 0
+     %12:zpr2stridedorcontiguous = LD1B_2Z_PSEUDO %10, %1, %0
+     %13:zpr2 = FORM_TRANSPOSED_REG_TUPLE_X2_PSEUDO %11.zsub0, %12.zsub0
+     %14:zpr2 = FORM_TRANSPOSED_REG_TUPLE_X2_PSEUDO %11.zsub1, %12.zsub1
+     $za = UDOT_VG2_M2ZZ_BtoS $za, %9, 0, %13, undef %15:zpr_4b
+     $za = UDOT_VG2_M2ZZ_BtoS $za, %9, 0, %14, undef %16:zpr_4b
+     RET_ReallyLR
+...
diff --git a/llvm/test/CodeGen/AArch64/smul_fix.ll b/llvm/test/CodeGen/AArch64/smul_fix.ll
index 50a189df2be53..dacce720a7319 100644
--- a/llvm/test/CodeGen/AArch64/smul_fix.ll
+++ b/llvm/test/CodeGen/AArch64/smul_fix.ll
@@ -144,8 +144,7 @@ define <4 x i16> @widemul(<4 x i16> %x, <4 x i16> %y) nounwind {
 ; CHECK-NEXT:    smull v0.4s, v0.4h, v1.4h
 ; CHECK-NEXT:    shrn v1.4h, v0.4s, #16
 ; CHECK-NEXT:    xtn v2.4h, v0.4s
-; CHECK-NEXT:    add v1.4h, v1.4h, v1.4h
-; CHECK-NEXT:    shl v0.4h, v1.4h, #13
+; CHECK-NEXT:    shl v0.4h, v1.4h, #14
 ; CHECK-NEXT:    usra v0.4h, v2.4h, #2
 ; CHECK-NEXT:    ret
   %tmp = call <4 x i16> @llvm.smul.fix.v4i16(<4 x i16> %x, <4 x i16> %y, i32 2)
diff --git a/llvm/test/CodeGen/AArch64/spill-fill-zpr-predicates.mir b/llvm/test/CodeGen/AArch64/spill-fill-zpr-predicates.mir
new file mode 100644
index 0000000000000..bff0cacfd5190
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/spill-fill-zpr-predicates.mir
@@ -0,0 +1,1013 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 5
+# RUN: llc -mtriple=aarch64-linux-gnu -aarch64-enable-zpr-predicate-spills -run-pass=greedy %s -o - | FileCheck %s
+# RUN: llc -mtriple=aarch64-linux-gnu -aarch64-enable-zpr-predicate-spills -start-before=greedy -stop-after=aarch64-expand-pseudo -verify-machineinstrs %s -o - | FileCheck %s --check-prefix=EXPAND
+--- |
+  source_filename = "<stdin>"
+  target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
+  target triple = "aarch64--linux-gnu"
+
+  define aarch64_sve_vector_pcs void @zpr_predicate_spill() #0 { entry: unreachable }
+
+  define aarch64_sve_vector_pcs void @zpr_predicate_spill__save_restore_nzcv() #0 { entry: unreachable }
+
+  define aarch64_sve_vector_pcs void @zpr_predicate_spill__save_restore_nzcv__scavenge_csr_gpr() #0 { entry: unreachable }
+
+  define aarch64_sve_vector_pcs void @zpr_predicate_spill__spill_zpr() #0 { entry: unreachable }
+
+  define aarch64_sve_vector_pcs void @zpr_predicate_spill_above_p7() #0 { entry: unreachable }
+
+  define aarch64_sve_vector_pcs void @zpr_predicate_spill_p4_saved() #0 { entry: unreachable }
+
+  attributes #0 = {nounwind "target-features"="+sme,+sve" "aarch64_pstate_sm_compatible"}
+...
+---
+name: zpr_predicate_spill
+tracksRegLiveness: true
+stack:
+liveins:
+  - { reg: '$p0' }
+body:             |
+  bb.0.entry:
+    liveins: $p0
+
+    ; CHECK-LABEL: name: zpr_predicate_spill
+    ; CHECK: stack:
+    ; CHECK:      - { id: 0, name: '', type: spill-slot, offset: 0, size: 16, alignment: 16,
+    ; CHECK-NEXT:     stack-id: scalable-vector, callee-saved-register:
+    ; CHECK: liveins: $p0
+    ; CHECK-NEXT: {{  $}}
+    ;
+    ; CHECK-NEXT: SPILL_PPR_TO_ZPR_SLOT_PSEUDO $p0, %stack.0, 0 :: (store (s128) into %stack.0)
+    ;
+    ; CHECK-NEXT: $p0 = IMPLICIT_DEF
+    ; CHECK-NEXT: $p1 = IMPLICIT_DEF
+    ; CHECK-NEXT: $p2 = IMPLICIT_DEF
+    ; CHECK-NEXT: $p3 = IMPLICIT_DEF
+    ; CHECK-NEXT: $p4 = IMPLICIT_DEF
+    ; CHECK-NEXT: $p5 = IMPLICIT_DEF
+    ; CHECK-NEXT: $p6 = IMPLICIT_DEF
+    ; CHECK-NEXT: $p7 = IMPLICIT_DEF
+    ; CHECK-NEXT: $p8 = IMPLICIT_DEF
+    ; CHECK-NEXT: $p9 = IMPLICIT_DEF
+    ; CHECK-NEXT: $p10 = IMPLICIT_DEF
+    ; CHECK-NEXT: $p11 = IMPLICIT_DEF
+    ; CHECK-NEXT: $p12 = IMPLICIT_DEF
+    ; CHECK-NEXT: $p13 = IMPLICIT_DEF
+    ; CHECK-NEXT: $p14 = IMPLICIT_DEF
+    ; CHECK-NEXT: $p15 = IMPLICIT_DEF
+    ;
+    ; CHECK-NEXT: $p0 = FILL_PPR_FROM_ZPR_SLOT_PSEUDO %stack.0, 0 :: (load (s128) from %stack.0)
+    ;
+    ; CHECK-NEXT: RET_ReallyLR implicit $p0
+
+    ; EXPAND-LABEL: name: zpr_predicate_spill
+    ; EXPAND: liveins: $p0, $fp, $p15, $p14, $p13, $p12, $p11, $p10, $p9, $p8, $p7, $p6, $p5, $p4
+    ; EXPAND-NEXT: {{  $}}
+    ;
+    ; EXPAND-NEXT: $sp = frame-setup SUBXri $sp, 1040, 0
+    ; EXPAND-NEXT: frame-setup STRXui killed $fp, $sp, 128 :: (store (s64) into %stack.14)
+    ; EXPAND-NEXT: $sp = frame-setup ADDVL_XXI $sp, -12, implicit $vg
+    ; EXPAND-NEXT: $z0 = frame-setup CPY_ZPzI_B killed $p15, 1, 0
+    ; EXPAND-NEXT: frame-setup STR_ZXI $z0, $sp, 0 :: (store (s128) into %stack.13)
+    ; EXPAND-NEXT: $z0 = frame-setup CPY_ZPzI_B killed $p14, 1, 0
+    ; EXPAND-NEXT: frame-setup STR_ZXI $z0, $sp, 1 :: (store (s128) into %stack.12)
+    ; EXPAND-NEXT: $z0 = frame-setup CPY_ZPzI_B killed $p13, 1, 0
+    ; EXPAND-NEXT: frame-setup STR_ZXI $z0, $sp, 2 :: (store (s128) into %stack.11)
+    ; EXPAND-NEXT: $z0 = frame-setup CPY_ZPzI_B killed $p12, 1, 0
+    ; EXPAND-NEXT: frame-setup STR_ZXI $z0, $sp, 3 :: (store (s128) into %stack.10)
+    ; EXPAND-NEXT: $z0 = frame-setup CPY_ZPzI_B killed $p11, 1, 0
+    ; EXPAND-NEXT: frame-setup STR_ZXI $z0, $sp, 4 :: (store (s128) into %stack.9)
+    ; EXPAND-NEXT: $z0 = frame-setup CPY_ZPzI_B killed $p10, 1, 0
+    ; EXPAND-NEXT: frame-setup STR_ZXI $z0, $sp, 5 :: (store (s128) into %stack.8)
+    ; EXPAND-NEXT: $z0 = frame-setup CPY_ZPzI_B killed $p9, 1, 0
+    ; EXPAND-NEXT: frame-setup STR_ZXI $z0, $sp, 6 :: (store (s128) into %stack.7)
+    ; EXPAND-NEXT: $z0 = frame-setup CPY_ZPzI_B killed $p8, 1, 0
+    ; EXPAND-NEXT: frame-setup STR_ZXI $z0, $sp, 7 :: (store (s128) into %stack.6)
+    ; EXPAND-NEXT: $z0 = frame-setup CPY_ZPzI_B killed $p7, 1, 0
+    ; EXPAND-NEXT: frame-setup STR_ZXI $z0, $sp, 8 :: (store (s128) into %stack.5)
+    ; EXPAND-NEXT: $z0 = frame-setup CPY_ZPzI_B killed $p6, 1, 0
+    ; EXPAND-NEXT: frame-setup STR_ZXI $z0, $sp, 9 :: (store (s128) into %stack.4)
+    ; EXPAND-NEXT: $z0 = frame-setup CPY_ZPzI_B killed $p5, 1, 0
+    ; EXPAND-NEXT: frame-setup STR_ZXI $z0, $sp, 10 :: (store (s128) into %stack.3)
+    ; EXPAND-NEXT: $z0 = frame-setup CPY_ZPzI_B killed $p4, 1, 0
+    ; EXPAND-NEXT: frame-setup STR_ZXI $z0, $sp, 11 :: (store (s128) into %stack.2)
+    ; EXPAND-NEXT: $sp = frame-setup SUBXri $sp, 1024, 0
+    ; EXPAND-NEXT: $sp = frame-setup ADDVL_XXI $sp, -1, implicit $vg
+    ;
+    ; EXPAND-NEXT: $z0 = CPY_ZPzI_B $p0, 1, 0
+    ; EXPAND-NEXT: $x8 = ADDXri $sp, 1024, 0
+    ; EXPAND-NEXT: STR_ZXI $z0, $x8, 0 :: (store (s128) into %stack.0)
+    ;
+    ; EXPAND-NEXT: $p0 = IMPLICIT_DEF
+    ; EXPAND-NEXT: $p1 = IMPLICIT_DEF
+    ; EXPAND-NEXT: $p2 = IMPLICIT_DEF
+    ; EXPAND-NEXT: $p3 = IMPLICIT_DEF
+    ; EXPAND-NEXT: $p4 = IMPLICIT_DEF
+    ; EXPAND-NEXT: $p5 = IMPLICIT_DEF
+    ; EXPAND-NEXT: $p6 = IMPLICIT_DEF
+    ; EXPAND-NEXT: $p7 = IMPLICIT_DEF
+    ; EXPAND-NEXT: $p8 = IMPLICIT_DEF
+    ; EXPAND-NEXT: $p9 = IMPLICIT_DEF
+    ; EXPAND-NEXT: $p10 = IMPLICIT_DEF
+    ; EXPAND-NEXT: $p11 = IMPLICIT_DEF
+    ; EXPAND-NEXT: $p12 = IMPLICIT_DEF
+    ; EXPAND-NEXT: $p13 = IMPLICIT_DEF
+    ; EXPAND-NEXT: $p14 = IMPLICIT_DEF
+    ; EXPAND-NEXT: $p15 = IMPLICIT_DEF
+    ;
+    ; EXPAND-NEXT: $z0 = LDR_ZXI killed $x8, 0 :: (load (s128) from %stack.0)
+    ; EXPAND-NEXT: $p1 = frame-destroy PTRUE_B 31, implicit $vg
+    ; EXPAND-NEXT: $p0 = CMPNE_PPzZI_B $p1, $z0, 0, implicit-def $nzcv, implicit-def $nzcv
+    ;
+    ; EXPAND-NEXT: $sp = frame-destroy ADDXri $sp, 1024, 0
+    ; EXPAND-NEXT: $sp = frame-destroy ADDVL_XXI $sp, 1, implicit $vg
+    ; EXPAND-NEXT: $z0 = frame-destroy LDR_ZXI $sp, 0 :: (load (s128) from %stack.13)
+    ; EXPAND-NEXT: $p15 = frame-destroy CMPNE_PPzZI_B $p1, $z0, 0, implicit-def $nzcv, implicit-def $nzcv
+    ; EXPAND-NEXT: $z0 = frame-destroy LDR_ZXI $sp, 1 :: (load (s128) from %stack.12)
+    ; EXPAND-NEXT: $p14 = frame-destroy CMPNE_PPzZI_B $p1, $z0, 0, implicit-def $nzcv, implicit-def $nzcv
+    ; EXPAND-NEXT: $z0 = frame-destroy LDR_ZXI $sp, 2 :: (load (s128) from %stack.11)
+    ; EXPAND-NEXT: $p13 = frame-destroy CMPNE_PPzZI_B $p1, $z0, 0, implicit-def $nzcv, implicit-def $nzcv
+    ; EXPAND-NEXT: $z0 = frame-destroy LDR_ZXI $sp, 3 :: (load (s128) from %stack.10)
+    ; EXPAND-NEXT: $p12 = frame-destroy CMPNE_PPzZI_B $p1, $z0, 0, implicit-def $nzcv, implicit-def $nzcv
+    ; EXPAND-NEXT: $z0 = frame-destroy LDR_ZXI $sp, 4 :: (load (s128) from %stack.9)
+    ; EXPAND-NEXT: $p11 = frame-destroy CMPNE_PPzZI_B $p1, $z0, 0, implicit-def $nzcv, implicit-def $nzcv
+    ; EXPAND-NEXT: $z0 = frame-destroy LDR_ZXI $sp, 5 :: (load (s128) from %stack.8)
+    ; EXPAND-NEXT: $p10 = frame-destroy CMPNE_PPzZI_B $p1, $z0, 0, implicit-def $nzcv, implicit-def $nzcv
+    ; EXPAND-NEXT: $z0 = frame-destroy LDR_ZXI $sp, 6 :: (load (s128) from %stack.7)
+    ; EXPAND-NEXT: $p9 = frame-destroy CMPNE_PPzZI_B $p1, $z0, 0, implicit-def $nzcv, implicit-def $nzcv
+    ; EXPAND-NEXT: $z0 = frame-destroy LDR_ZXI $sp, 7 :: (load (s128) from %stack.6)
+    ; EXPAND-NEXT: $p8 = frame-destroy CMPNE_PPzZI_B $p1, $z0, 0, implicit-def $nzcv, implicit-def $nzcv
+    ; EXPAND-NEXT: $z0 = frame-destroy LDR_ZXI $sp, 8 :: (load (s128) from %stack.5)
+    ; EXPAND-NEXT: $p7 = frame-destroy CMPNE_PPzZI_B $p1, $z0, 0, implicit-def $nzcv, implicit-def $nzcv
+    ; EXPAND-NEXT: $z0 = frame-destroy LDR_ZXI $sp, 9 :: (load (s128) from %stack.4)
+    ; EXPAND-NEXT: $p6 = frame-destroy CMPNE_PPzZI_B $p1, $z0, 0, implicit-def $nzcv, implicit-def $nzcv
+    ; EXPAND-NEXT: $z0 = frame-destroy LDR_ZXI $sp, 10 :: (load (s128) from %stack.3)
+    ; EXPAND-NEXT: $p5 = frame-destroy CMPNE_PPzZI_B $p1, $z0, 0, implicit-def $nzcv, implicit-def $nzcv
+    ; EXPAND-NEXT: $z0 = frame-destroy LDR_ZXI $sp, 11 :: (load (s128) from %stack.2)
+    ; EXPAND-NEXT: $p4 = frame-destroy CMPNE_PPzZI_B $p1, $z0, 0, implicit-def $nzcv, implicit-def $nzcv
+    ; EXPAND-NEXT: $sp = frame-destroy ADDVL_XXI $sp, 12, implicit $vg
+    ; EXPAND-NEXT: $fp = frame-destroy LDRXui $sp, 128 :: (load (s64) from %stack.14)
+    ; EXPAND-NEXT: $sp = frame-destroy ADDXri $sp, 1040, 0
+    ; EXPAND-NEXT: RET undef $lr, implicit $p0
+    %1:ppr = COPY $p0
+
+    $p0 = IMPLICIT_DEF
+    $p1 = IMPLICIT_DEF
+    $p2 = IMPLICIT_DEF
+    $p3 = IMPLICIT_DEF
+    $p4 = IMPLICIT_DEF
+    $p5 = IMPLICIT_DEF
+    $p6 = IMPLICIT_DEF
+    $p7 = IMPLICIT_DEF
+    $p8 = IMPLICIT_DEF
+    $p9 = IMPLICIT_DEF
+    $p10 = IMPLICIT_DEF
+    $p11 = IMPLICIT_DEF
+    $p12 = IMPLICIT_DEF
+    $p13 = IMPLICIT_DEF
+    $p14 = IMPLICIT_DEF
+    $p15 = IMPLICIT_DEF
+
+    $p0 = COPY %1
+
+    RET_ReallyLR implicit $p0
+...
+---
+name: zpr_predicate_spill__save_restore_nzcv
+tracksRegLiveness: true
+stack:
+liveins:
+  - { reg: '$p0' }
+body:             |
+  bb.0.entry:
+    liveins: $p0
+
+    ; CHECK-LABEL: name: zpr_predicate_spill__save_restore_nzcv
+    ; CHECK: stack:
+    ; CHECK:      - { id: 0, name: '', type: spill-slot, offset: 0, size: 16, alignment: 16,
+    ; CHECK-NEXT:     stack-id: scalable-vector, callee-saved-register:
+    ; CHECK: liveins: $p0
+    ; CHECK-NEXT: {{  $}}
+    ;
+    ; CHECK-NEXT: $nzcv = IMPLICIT_DEF
+    ;
+    ; CHECK-NEXT: SPILL_PPR_TO_ZPR_SLOT_PSEUDO $p0, %stack.0, 0 :: (store (s128) into %stack.0)
+    ;
+    ; CHECK-NEXT: $p0 = IMPLICIT_DEF
+    ; CHECK-NEXT: $p1 = IMPLICIT_DEF
+    ; CHECK-NEXT: $p2 = IMPLICIT_DEF
+    ; CHECK-NEXT: $p3 = IMPLICIT_DEF
+    ; CHECK-NEXT: $p4 = IMPLICIT_DEF
+    ; CHECK-NEXT: $p5 = IMPLICIT_DEF
+    ; CHECK-NEXT: $p6 = IMPLICIT_DEF
+    ; CHECK-NEXT: $p7 = IMPLICIT_DEF
+    ; CHECK-NEXT: $p8 = IMPLICIT_DEF
+    ; CHECK-NEXT: $p9 = IMPLICIT_DEF
+    ; CHECK-NEXT: $p10 = IMPLICIT_DEF
+    ; CHECK-NEXT: $p11 = IMPLICIT_DEF
+    ; CHECK-NEXT: $p12 = IMPLICIT_DEF
+    ; CHECK-NEXT: $p13 = IMPLICIT_DEF
+    ; CHECK-NEXT: $p14 = IMPLICIT_DEF
+    ; CHECK-NEXT: $p15 = IMPLICIT_DEF
+    ;
+    ; CHECK-NEXT: $p0 = FILL_PPR_FROM_ZPR_SLOT_PSEUDO %stack.0, 0 :: (load (s128) from %stack.0)
+    ;
+    ; CHECK-NEXT: FAKE_USE implicit $nzcv
+    ;
+    ; CHECK-NEXT: RET_ReallyLR implicit $p0
+
+    ; EXPAND-LABEL: name: zpr_predicate_spill__save_restore_nzcv
+    ; EXPAND: liveins: $p0, $fp, $p15, $p14, $p13, $p12, $p11, $p10, $p9, $p8, $p7, $p6, $p5, $p4
+    ; EXPAND-NEXT: {{  $}}
+    ;
+    ; EXPAND-NEXT: $sp = frame-setup SUBXri $sp, 1040, 0
+    ; EXPAND-NEXT: frame-setup STRXui killed $fp, $sp, 128 :: (store (s64) into %stack.14)
+    ; EXPAND-NEXT: $sp = frame-setup ADDVL_XXI $sp, -12, implicit $vg
+    ; EXPAND-NEXT: $z0 = frame-setup CPY_ZPzI_B killed $p15, 1, 0
+    ; EXPAND-NEXT: frame-setup STR_ZXI $z0, $sp, 0 :: (store (s128) into %stack.13)
+    ; EXPAND-NEXT: $z0 = frame-setup CPY_ZPzI_B killed $p14, 1, 0
+    ; EXPAND-NEXT: frame-setup STR_ZXI $z0, $sp, 1 :: (store (s128) into %stack.12)
+    ; EXPAND-NEXT: $z0 = frame-setup CPY_ZPzI_B killed $p13, 1, 0
+    ; EXPAND-NEXT: frame-setup STR_ZXI $z0, $sp, 2 :: (store (s128) into %stack.11)
+    ; EXPAND-NEXT: $z0 = frame-setup CPY_ZPzI_B killed $p12, 1, 0
+    ; EXPAND-NEXT: frame-setup STR_ZXI $z0, $sp, 3 :: (store (s128) into %stack.10)
+    ; EXPAND-NEXT: $z0 = frame-setup CPY_ZPzI_B killed $p11, 1, 0
+    ; EXPAND-NEXT: frame-setup STR_ZXI $z0, $sp, 4 :: (store (s128) into %stack.9)
+    ; EXPAND-NEXT: $z0 = frame-setup CPY_ZPzI_B killed $p10, 1, 0
+    ; EXPAND-NEXT: frame-setup STR_ZXI $z0, $sp, 5 :: (store (s128) into %stack.8)
+    ; EXPAND-NEXT: $z0 = frame-setup CPY_ZPzI_B killed $p9, 1, 0
+    ; EXPAND-NEXT: frame-setup STR_ZXI $z0, $sp, 6 :: (store (s128) into %stack.7)
+    ; EXPAND-NEXT: $z0 = frame-setup CPY_ZPzI_B killed $p8, 1, 0
+    ; EXPAND-NEXT: frame-setup STR_ZXI $z0, $sp, 7 :: (store (s128) into %stack.6)
+    ; EXPAND-NEXT: $z0 = frame-setup CPY_ZPzI_B killed $p7, 1, 0
+    ; EXPAND-NEXT: frame-setup STR_ZXI $z0, $sp, 8 :: (store (s128) into %stack.5)
+    ; EXPAND-NEXT: $z0 = frame-setup CPY_ZPzI_B killed $p6, 1, 0
+    ; EXPAND-NEXT: frame-setup STR_ZXI $z0, $sp, 9 :: (store (s128) into %stack.4)
+    ; EXPAND-NEXT: $z0 = frame-setup CPY_ZPzI_B killed $p5, 1, 0
+    ; EXPAND-NEXT: frame-setup STR_ZXI $z0, $sp, 10 :: (store (s128) into %stack.3)
+    ; EXPAND-NEXT: $z0 = frame-setup CPY_ZPzI_B killed $p4, 1, 0
+    ; EXPAND-NEXT: frame-setup STR_ZXI $z0, $sp, 11 :: (store (s128) into %stack.2)
+    ; EXPAND-NEXT: $sp = frame-setup SUBXri $sp, 1024, 0
+    ; EXPAND-NEXT: $sp = frame-setup ADDVL_XXI $sp, -1, implicit $vg
+    ;
+    ; EXPAND-NEXT: $nzcv = IMPLICIT_DEF
+    ;
+    ; EXPAND-NEXT: $z0 = CPY_ZPzI_B $p0, 1, 0
+    ; EXPAND-NEXT: $x8 = ADDXri $sp, 1024, 0
+    ; EXPAND-NEXT: STR_ZXI $z0, $x8, 0 :: (store (s128) into %stack.0)
+    ;
+    ; EXPAND-NEXT: $p0 = IMPLICIT_DEF
+    ; EXPAND-NEXT: $p1 = IMPLICIT_DEF
+    ; EXPAND-NEXT: $p2 = IMPLICIT_DEF
+    ; EXPAND-NEXT: $p3 = IMPLICIT_DEF
+    ; EXPAND-NEXT: $p4 = IMPLICIT_DEF
+    ; EXPAND-NEXT: $p5 = IMPLICIT_DEF
+    ; EXPAND-NEXT: $p6 = IMPLICIT_DEF
+    ; EXPAND-NEXT: $p7 = IMPLICIT_DEF
+    ; EXPAND-NEXT: $p8 = IMPLICIT_DEF
+    ; EXPAND-NEXT: $p9 = IMPLICIT_DEF
+    ; EXPAND-NEXT: $p10 = IMPLICIT_DEF
+    ; EXPAND-NEXT: $p11 = IMPLICIT_DEF
+    ; EXPAND-NEXT: $p12 = IMPLICIT_DEF
+    ; EXPAND-NEXT: $p13 = IMPLICIT_DEF
+    ; EXPAND-NEXT: $p14 = IMPLICIT_DEF
+    ; EXPAND-NEXT: $p15 = IMPLICIT_DEF
+    ;
+    ; EXPAND-NEXT: $z0 = LDR_ZXI killed $x8, 0 :: (load (s128) from %stack.0)
+    ; EXPAND-NEXT: $fp = MRS 55824, implicit-def $nzcv, implicit $nzcv
+    ; EXPAND-NEXT: $p0 = PTRUE_B 31, implicit $vg
+    ; EXPAND-NEXT: $p0 = CMPNE_PPzZI_B $p0, $z0, 0, implicit-def $nzcv, implicit-def $nzcv
+    ; EXPAND-NEXT: MSR 55824, $fp, implicit-def $nzcv
+    ;
+    ; EXPAND-NEXT: FAKE_USE implicit $nzcv
+    ;
+    ; EXPAND-NEXT: $sp = frame-destroy ADDXri $sp, 1024, 0
+    ; EXPAND-NEXT: $sp = frame-destroy ADDVL_XXI $sp, 1, implicit $vg
+    ; EXPAND-NEXT: $z0 = frame-destroy LDR_ZXI $sp, 0 :: (load (s128) from %stack.13)
+    ; EXPAND-NEXT: $p1 = frame-destroy PTRUE_B 31, implicit $vg
+    ; EXPAND-NEXT: $p15 = frame-destroy CMPNE_PPzZI_B $p1, $z0, 0, implicit-def $nzcv, implicit-def $nzcv
+    ; EXPAND-NEXT: $z0 = frame-destroy LDR_ZXI $sp, 1 :: (load (s128) from %stack.12)
+    ; EXPAND-NEXT: $p14 = frame-destroy CMPNE_PPzZI_B $p1, $z0, 0, implicit-def $nzcv, implicit-def $nzcv
+    ; EXPAND-NEXT: $z0 = frame-destroy LDR_ZXI $sp, 2 :: (load (s128) from %stack.11)
+    ; EXPAND-NEXT: $p13 = frame-destroy CMPNE_PPzZI_B $p1, $z0, 0, implicit-def $nzcv, implicit-def $nzcv
+    ; EXPAND-NEXT: $z0 = frame-destroy LDR_ZXI $sp, 3 :: (load (s128) from %stack.10)
+    ; EXPAND-NEXT: $p12 = frame-destroy CMPNE_PPzZI_B $p1, $z0, 0, implicit-def $nzcv, implicit-def $nzcv
+    ; EXPAND-NEXT: $z0 = frame-destroy LDR_ZXI $sp, 4 :: (load (s128) from %stack.9)
+    ; EXPAND-NEXT: $p11 = frame-destroy CMPNE_PPzZI_B $p1, $z0, 0, implicit-def $nzcv, implicit-def $nzcv
+    ; EXPAND-NEXT: $z0 = frame-destroy LDR_ZXI $sp, 5 :: (load (s128) from %stack.8)
+    ; EXPAND-NEXT: $p10 = frame-destroy CMPNE_PPzZI_B $p1, $z0, 0, implicit-def $nzcv, implicit-def $nzcv
+    ; EXPAND-NEXT: $z0 = frame-destroy LDR_ZXI $sp, 6 :: (load (s128) from %stack.7)
+    ; EXPAND-NEXT: $p9 = frame-destroy CMPNE_PPzZI_B $p1, $z0, 0, implicit-def $nzcv, implicit-def $nzcv
+    ; EXPAND-NEXT: $z0 = frame-destroy LDR_ZXI $sp, 7 :: (load (s128) from %stack.6)
+    ; EXPAND-NEXT: $p8 = frame-destroy CMPNE_PPzZI_B $p1, $z0, 0, implicit-def $nzcv, implicit-def $nzcv
+    ; EXPAND-NEXT: $z0 = frame-destroy LDR_ZXI $sp, 8 :: (load (s128) from %stack.5)
+    ; EXPAND-NEXT: $p7 = frame-destroy CMPNE_PPzZI_B $p1, $z0, 0, implicit-def $nzcv, implicit-def $nzcv
+    ; EXPAND-NEXT: $z0 = frame-destroy LDR_ZXI $sp, 9 :: (load (s128) from %stack.4)
+    ; EXPAND-NEXT: $p6 = frame-destroy CMPNE_PPzZI_B $p1, $z0, 0, implicit-def $nzcv, implicit-def $nzcv
+    ; EXPAND-NEXT: $z0 = frame-destroy LDR_ZXI $sp, 10 :: (load (s128) from %stack.3)
+    ; EXPAND-NEXT: $p5 = frame-destroy CMPNE_PPzZI_B $p1, $z0, 0, implicit-def $nzcv, implicit-def $nzcv
+    ; EXPAND-NEXT: $z0 = frame-destroy LDR_ZXI $sp, 11 :: (load (s128) from %stack.2)
+    ; EXPAND-NEXT: $p4 = frame-destroy CMPNE_PPzZI_B $p1, $z0, 0, implicit-def $nzcv, implicit-def $nzcv
+    ; EXPAND-NEXT: $sp = frame-destroy ADDVL_XXI $sp, 12, implicit $vg
+    ; EXPAND-NEXT: $fp = frame-destroy LDRXui $sp, 128 :: (load (s64) from %stack.14)
+    ; EXPAND-NEXT: $sp = frame-destroy ADDXri $sp, 1040, 0
+    ; EXPAND-NEXT: RET undef $lr, implicit $p0
+    $nzcv = IMPLICIT_DEF
+
+    %1:ppr = COPY $p0
+
+    $p0 = IMPLICIT_DEF
+    $p1 = IMPLICIT_DEF
+    $p2 = IMPLICIT_DEF
+    $p3 = IMPLICIT_DEF
+    $p4 = IMPLICIT_DEF
+    $p5 = IMPLICIT_DEF
+    $p6 = IMPLICIT_DEF
+    $p7 = IMPLICIT_DEF
+    $p8 = IMPLICIT_DEF
+    $p9 = IMPLICIT_DEF
+    $p10 = IMPLICIT_DEF
+    $p11 = IMPLICIT_DEF
+    $p12 = IMPLICIT_DEF
+    $p13 = IMPLICIT_DEF
+    $p14 = IMPLICIT_DEF
+    $p15 = IMPLICIT_DEF
+
+    $p0 = COPY %1
+
+    FAKE_USE implicit $nzcv
+
+    RET_ReallyLR implicit $p0
+...
+---
+name: zpr_predicate_spill__save_restore_nzcv__scavenge_csr_gpr
+tracksRegLiveness: true
+stack:
+liveins:
+  - { reg: '$p0' }
+  - { reg: '$x0' }
+  - { reg: '$x1' }
+  - { reg: '$x2' }
+  - { reg: '$x3' }
+  - { reg: '$x4' }
+  - { reg: '$x5' }
+  - { reg: '$x6' }
+  - { reg: '$x7' }
+body:             |
+  bb.0.entry:
+    liveins: $p0, $x0, $x1, $x2, $x3, $x4, $x5, $x6, $x7
+
+    ; CHECK-LABEL: name: zpr_predicate_spill__save_restore_nzcv__scavenge_csr_gpr
+    ; CHECK: stack:
+    ; CHECK:      - { id: 0, name: '', type: spill-slot, offset: 0, size: 16, alignment: 16,
+    ; CHECK-NEXT:     stack-id: scalable-vector, callee-saved-register:
+    ; CHECK: liveins: $p0, $x0, $x1, $x2, $x3, $x4, $x5, $x6, $x7
+    ; CHECK-NEXT: {{  $}}
+    ;
+    ; CHECK-NEXT: $nzcv = IMPLICIT_DEF
+    ;
+    ; CHECK-NEXT: $x8 = IMPLICIT_DEF
+    ; CHECK-NEXT: $x9 = IMPLICIT_DEF
+    ; CHECK-NEXT: $x10 = IMPLICIT_DEF
+    ; CHECK-NEXT: $x11 = IMPLICIT_DEF
+    ; CHECK-NEXT: $x12 = IMPLICIT_DEF
+    ; CHECK-NEXT: $x13 = IMPLICIT_DEF
+    ; CHECK-NEXT: $x14 = IMPLICIT_DEF
+    ; CHECK-NEXT: $x15 = IMPLICIT_DEF
+    ; CHECK-NEXT: $x16 = IMPLICIT_DEF
+    ; CHECK-NEXT: $x17 = IMPLICIT_DEF
+    ; CHECK-NEXT: $x18 = IMPLICIT_DEF
+    ;
+    ; CHECK-NEXT: SPILL_PPR_TO_ZPR_SLOT_PSEUDO $p0, %stack.0, 0 :: (store (s128) into %stack.0)
+    ;
+    ; CHECK-NEXT: $p0 = IMPLICIT_DEF
+    ; CHECK-NEXT: $p1 = IMPLICIT_DEF
+    ; CHECK-NEXT: $p2 = IMPLICIT_DEF
+    ; CHECK-NEXT: $p3 = IMPLICIT_DEF
+    ; CHECK-NEXT: $p4 = IMPLICIT_DEF
+    ; CHECK-NEXT: $p5 = IMPLICIT_DEF
+    ; CHECK-NEXT: $p6 = IMPLICIT_DEF
+    ; CHECK-NEXT: $p7 = IMPLICIT_DEF
+    ; CHECK-NEXT: $p8 = IMPLICIT_DEF
+    ; CHECK-NEXT: $p9 = IMPLICIT_DEF
+    ; CHECK-NEXT: $p10 = IMPLICIT_DEF
+    ; CHECK-NEXT: $p11 = IMPLICIT_DEF
+    ; CHECK-NEXT: $p12 = IMPLICIT_DEF
+    ; CHECK-NEXT: $p13 = IMPLICIT_DEF
+    ; CHECK-NEXT: $p14 = IMPLICIT_DEF
+    ; CHECK-NEXT: $p15 = IMPLICIT_DEF
+    ;
+    ; CHECK-NEXT: $p0 = FILL_PPR_FROM_ZPR_SLOT_PSEUDO %stack.0, 0 :: (load (s128) from %stack.0)
+    ;
+    ; CHECK-NEXT: FAKE_USE implicit $nzcv, implicit $x8, implicit $x9, implicit $x10, implicit $x11, implicit $x12, implicit $x13, implicit $x14, implicit $x15, implicit $x16, implicit $x17, implicit $x18
+    ;
+    ; CHECK-NEXT: RET_ReallyLR implicit $p0, implicit $x0, implicit $x1, implicit $x2, implicit $x3, implicit $x4, implicit $x5, implicit $x6, implicit $x7, implicit $x8, implicit $x9, implicit $x10, implicit $x11, implicit $x12, implicit $x13, implicit $x14, implicit $x15, implicit $x16, implicit $x17, implicit $x18
+
+    ; EXPAND-LABEL: name: zpr_predicate_spill__save_restore_nzcv__scavenge_csr_gpr
+    ; EXPAND: liveins: $p0, $x0, $x1, $x2, $x3, $x4, $x5, $x6, $x7, $fp, $p15, $p14, $p13, $p12, $p11, $p10, $p9, $p8, $p7, $p6, $p5, $p4
+    ; EXPAND-NEXT: {{  $}}
+    ;
+    ; EXPAND-NEXT: $sp = frame-setup SUBXri $sp, 1040, 0
+    ; EXPAND-NEXT: frame-setup STRXui killed $fp, $sp, 128 :: (store (s64) into %stack.14)
+    ; EXPAND-NEXT: $sp = frame-setup ADDVL_XXI $sp, -12, implicit $vg
+    ; EXPAND-NEXT: $z0 = frame-setup CPY_ZPzI_B killed $p15, 1, 0
+    ; EXPAND-NEXT: frame-setup STR_ZXI $z0, $sp, 0 :: (store (s128) into %stack.13)
+    ; EXPAND-NEXT: $z0 = frame-setup CPY_ZPzI_B killed $p14, 1, 0
+    ; EXPAND-NEXT: frame-setup STR_ZXI $z0, $sp, 1 :: (store (s128) into %stack.12)
+    ; EXPAND-NEXT: $z0 = frame-setup CPY_ZPzI_B killed $p13, 1, 0
+    ; EXPAND-NEXT: frame-setup STR_ZXI $z0, $sp, 2 :: (store (s128) into %stack.11)
+    ; EXPAND-NEXT: $z0 = frame-setup CPY_ZPzI_B killed $p12, 1, 0
+    ; EXPAND-NEXT: frame-setup STR_ZXI $z0, $sp, 3 :: (store (s128) into %stack.10)
+    ; EXPAND-NEXT: $z0 = frame-setup CPY_ZPzI_B killed $p11, 1, 0
+    ; EXPAND-NEXT: frame-setup STR_ZXI $z0, $sp, 4 :: (store (s128) into %stack.9)
+    ; EXPAND-NEXT: $z0 = frame-setup CPY_ZPzI_B killed $p10, 1, 0
+    ; EXPAND-NEXT: frame-setup STR_ZXI $z0, $sp, 5 :: (store (s128) into %stack.8)
+    ; EXPAND-NEXT: $z0 = frame-setup CPY_ZPzI_B killed $p9, 1, 0
+    ; EXPAND-NEXT: frame-setup STR_ZXI $z0, $sp, 6 :: (store (s128) into %stack.7)
+    ; EXPAND-NEXT: $z0 = frame-setup CPY_ZPzI_B killed $p8, 1, 0
+    ; EXPAND-NEXT: frame-setup STR_ZXI $z0, $sp, 7 :: (store (s128) into %stack.6)
+    ; EXPAND-NEXT: $z0 = frame-setup CPY_ZPzI_B killed $p7, 1, 0
+    ; EXPAND-NEXT: frame-setup STR_ZXI $z0, $sp, 8 :: (store (s128) into %stack.5)
+    ; EXPAND-NEXT: $z0 = frame-setup CPY_ZPzI_B killed $p6, 1, 0
+    ; EXPAND-NEXT: frame-setup STR_ZXI $z0, $sp, 9 :: (store (s128) into %stack.4)
+    ; EXPAND-NEXT: $z0 = frame-setup CPY_ZPzI_B killed $p5, 1, 0
+    ; EXPAND-NEXT: frame-setup STR_ZXI $z0, $sp, 10 :: (store (s128) into %stack.3)
+    ; EXPAND-NEXT: $z0 = frame-setup CPY_ZPzI_B killed $p4, 1, 0
+    ; EXPAND-NEXT: frame-setup STR_ZXI $z0, $sp, 11 :: (store (s128) into %stack.2)
+    ; EXPAND-NEXT: $sp = frame-setup SUBXri $sp, 1024, 0
+    ; EXPAND-NEXT: $sp = frame-setup ADDVL_XXI $sp, -1, implicit $vg
+    ;
+    ; EXPAND-NEXT: $nzcv = IMPLICIT_DEF
+    ;
+    ; EXPAND-NEXT: $x8 = IMPLICIT_DEF
+    ; EXPAND-NEXT: $x9 = IMPLICIT_DEF
+    ; EXPAND-NEXT: $x10 = IMPLICIT_DEF
+    ; EXPAND-NEXT: $x11 = IMPLICIT_DEF
+    ; EXPAND-NEXT: $x12 = IMPLICIT_DEF
+    ; EXPAND-NEXT: $x13 = IMPLICIT_DEF
+    ; EXPAND-NEXT: $x14 = IMPLICIT_DEF
+    ; EXPAND-NEXT: $x15 = IMPLICIT_DEF
+    ; EXPAND-NEXT: $x16 = IMPLICIT_DEF
+    ; EXPAND-NEXT: $x17 = IMPLICIT_DEF
+    ; EXPAND-NEXT: $x18 = IMPLICIT_DEF
+    ;
+    ; EXPAND-NEXT: $z0 = CPY_ZPzI_B $p0, 1, 0
+    ; EXPAND-NEXT: $fp = ADDXri $sp, 1024, 0
+    ; EXPAND-NEXT: STR_ZXI $z0, $fp, 0 :: (store (s128) into %stack.0)
+    ;
+    ; EXPAND-NEXT: $p0 = IMPLICIT_DEF
+    ; EXPAND-NEXT: $p1 = IMPLICIT_DEF
+    ; EXPAND-NEXT: $p2 = IMPLICIT_DEF
+    ; EXPAND-NEXT: $p3 = IMPLICIT_DEF
+    ; EXPAND-NEXT: $p4 = IMPLICIT_DEF
+    ; EXPAND-NEXT: $p5 = IMPLICIT_DEF
+    ; EXPAND-NEXT: $p6 = IMPLICIT_DEF
+    ; EXPAND-NEXT: $p7 = IMPLICIT_DEF
+    ; EXPAND-NEXT: $p8 = IMPLICIT_DEF
+    ; EXPAND-NEXT: $p9 = IMPLICIT_DEF
+    ; EXPAND-NEXT: $p10 = IMPLICIT_DEF
+    ; EXPAND-NEXT: $p11 = IMPLICIT_DEF
+    ; EXPAND-NEXT: $p12 = IMPLICIT_DEF
+    ; EXPAND-NEXT: $p13 = IMPLICIT_DEF
+    ; EXPAND-NEXT: $p14 = IMPLICIT_DEF
+    ; EXPAND-NEXT: $p15 = IMPLICIT_DEF
+    ;
+    ; EXPAND-NEXT: $z0 = LDR_ZXI killed $fp, 0 :: (load (s128) from %stack.0)
+    ; EXPAND-NEXT: $fp = MRS 55824, implicit-def $nzcv, implicit $nzcv
+    ; EXPAND-NEXT: $p0 = PTRUE_B 31, implicit $vg
+    ; EXPAND-NEXT: $p0 = CMPNE_PPzZI_B $p0, $z0, 0, implicit-def $nzcv, implicit-def $nzcv
+    ; EXPAND-NEXT: MSR 55824, $fp, implicit-def $nzcv
+    ;
+    ; EXPAND-NEXT: FAKE_USE implicit $nzcv, implicit $x8, implicit $x9, implicit $x10, implicit $x11, implicit $x12, implicit $x13, implicit $x14, implicit $x15, implicit $x16, implicit $x17, implicit $x18
+    ;
+    ; EXPAND-NEXT: $sp = frame-destroy ADDXri $sp, 1024, 0
+    ; EXPAND-NEXT: $sp = frame-destroy ADDVL_XXI $sp, 1, implicit $vg
+    ; EXPAND-NEXT: $z0 = frame-destroy LDR_ZXI $sp, 0 :: (load (s128) from %stack.13)
+    ; EXPAND-NEXT: $p1 = frame-destroy PTRUE_B 31, implicit $vg
+    ; EXPAND-NEXT: $p15 = frame-destroy CMPNE_PPzZI_B $p1, $z0, 0, implicit-def $nzcv, implicit-def $nzcv
+    ; EXPAND-NEXT: $z0 = frame-destroy LDR_ZXI $sp, 1 :: (load (s128) from %stack.12)
+    ; EXPAND-NEXT: $p14 = frame-destroy CMPNE_PPzZI_B $p1, $z0, 0, implicit-def $nzcv, implicit-def $nzcv
+    ; EXPAND-NEXT: $z0 = frame-destroy LDR_ZXI $sp, 2 :: (load (s128) from %stack.11)
+    ; EXPAND-NEXT: $p13 = frame-destroy CMPNE_PPzZI_B $p1, $z0, 0, implicit-def $nzcv, implicit-def $nzcv
+    ; EXPAND-NEXT: $z0 = frame-destroy LDR_ZXI $sp, 3 :: (load (s128) from %stack.10)
+    ; EXPAND-NEXT: $p12 = frame-destroy CMPNE_PPzZI_B $p1, $z0, 0, implicit-def $nzcv, implicit-def $nzcv
+    ; EXPAND-NEXT: $z0 = frame-destroy LDR_ZXI $sp, 4 :: (load (s128) from %stack.9)
+    ; EXPAND-NEXT: $p11 = frame-destroy CMPNE_PPzZI_B $p1, $z0, 0, implicit-def $nzcv, implicit-def $nzcv
+    ; EXPAND-NEXT: $z0 = frame-destroy LDR_ZXI $sp, 5 :: (load (s128) from %stack.8)
+    ; EXPAND-NEXT: $p10 = frame-destroy CMPNE_PPzZI_B $p1, $z0, 0, implicit-def $nzcv, implicit-def $nzcv
+    ; EXPAND-NEXT: $z0 = frame-destroy LDR_ZXI $sp, 6 :: (load (s128) from %stack.7)
+    ; EXPAND-NEXT: $p9 = frame-destroy CMPNE_PPzZI_B $p1, $z0, 0, implicit-def $nzcv, implicit-def $nzcv
+    ; EXPAND-NEXT: $z0 = frame-destroy LDR_ZXI $sp, 7 :: (load (s128) from %stack.6)
+    ; EXPAND-NEXT: $p8 = frame-destroy CMPNE_PPzZI_B $p1, $z0, 0, implicit-def $nzcv, implicit-def $nzcv
+    ; EXPAND-NEXT: $z0 = frame-destroy LDR_ZXI $sp, 8 :: (load (s128) from %stack.5)
+    ; EXPAND-NEXT: $p7 = frame-destroy CMPNE_PPzZI_B $p1, $z0, 0, implicit-def $nzcv, implicit-def $nzcv
+    ; EXPAND-NEXT: $z0 = frame-destroy LDR_ZXI $sp, 9 :: (load (s128) from %stack.4)
+    ; EXPAND-NEXT: $p6 = frame-destroy CMPNE_PPzZI_B $p1, $z0, 0, implicit-def $nzcv, implicit-def $nzcv
+    ; EXPAND-NEXT: $z0 = frame-destroy LDR_ZXI $sp, 10 :: (load (s128) from %stack.3)
+    ; EXPAND-NEXT: $p5 = frame-destroy CMPNE_PPzZI_B $p1, $z0, 0, implicit-def $nzcv, implicit-def $nzcv
+    ; EXPAND-NEXT: $z0 = frame-destroy LDR_ZXI $sp, 11 :: (load (s128) from %stack.2)
+    ; EXPAND-NEXT: $p4 = frame-destroy CMPNE_PPzZI_B $p1, $z0, 0, implicit-def $nzcv, implicit-def $nzcv
+    ; EXPAND-NEXT: $sp = frame-destroy ADDVL_XXI $sp, 12, implicit $vg
+    ; EXPAND-NEXT: $fp = frame-destroy LDRXui $sp, 128 :: (load (s64) from %stack.14)
+    ; EXPAND-NEXT: $sp = frame-destroy ADDXri $sp, 1040, 0
+    ; EXPAND-NEXT: RET undef $lr, implicit $p0, implicit $x0, implicit $x1, implicit $x2, implicit $x3, implicit $x4, implicit $x5, implicit $x6, implicit $x7, implicit $x8, implicit $x9, implicit $x10, implicit $x11, implicit $x12, implicit $x13, implicit $x14, implicit $x15, implicit $x16, implicit $x17, implicit $x18
+    $nzcv = IMPLICIT_DEF
+    $x8 = IMPLICIT_DEF
+    $x9 = IMPLICIT_DEF
+    $x10 = IMPLICIT_DEF
+    $x11 = IMPLICIT_DEF
+    $x12 = IMPLICIT_DEF
+    $x13 = IMPLICIT_DEF
+    $x14 = IMPLICIT_DEF
+    $x15 = IMPLICIT_DEF
+    $x16 = IMPLICIT_DEF
+    $x17 = IMPLICIT_DEF
+    $x18 = IMPLICIT_DEF
+
+    %1:ppr = COPY $p0
+
+    $p0 = IMPLICIT_DEF
+    $p1 = IMPLICIT_DEF
+    $p2 = IMPLICIT_DEF
+    $p3 = IMPLICIT_DEF
+    $p4 = IMPLICIT_DEF
+    $p5 = IMPLICIT_DEF
+    $p6 = IMPLICIT_DEF
+    $p7 = IMPLICIT_DEF
+    $p8 = IMPLICIT_DEF
+    $p9 = IMPLICIT_DEF
+    $p10 = IMPLICIT_DEF
+    $p11 = IMPLICIT_DEF
+    $p12 = IMPLICIT_DEF
+    $p13 = IMPLICIT_DEF
+    $p14 = IMPLICIT_DEF
+    $p15 = IMPLICIT_DEF
+
+    $p0 = COPY %1
+
+    FAKE_USE implicit $nzcv, implicit $x8, implicit $x9, implicit $x10, implicit $x11, implicit $x12, implicit $x13, implicit $x14, implicit $x15, implicit $x16, implicit $x17, implicit $x18
+
+    RET_ReallyLR implicit $p0, implicit $x0, implicit $x1, implicit $x2, implicit $x3, implicit $x4, implicit $x5, implicit $x6, implicit $x7, implicit $x8, implicit $x9, implicit $x10, implicit $x11, implicit $x12, implicit $x13, implicit $x14, implicit $x15, implicit $x16, implicit $x17, implicit $x18
+...
+---
+name: zpr_predicate_spill__spill_zpr
+tracksRegLiveness: true
+stack:
+liveins:
+  - { reg: '$p0' }
+  - { reg: '$z0' }
+  - { reg: '$z1' }
+  - { reg: '$z2' }
+  - { reg: '$z3' }
+  - { reg: '$z4' }
+  - { reg: '$z5' }
+  - { reg: '$z6' }
+  - { reg: '$z7' }
+body:             |
+  bb.0.entry:
+    liveins: $p0, $z0, $z1, $z2, $z3, $z4, $z5, $z6, $z7
+
+    ; CHECK-LABEL: name: zpr_predicate_spill__spill_zpr
+    ; CHECK: stack:
+    ; CHECK:      - { id: 0, name: '', type: spill-slot, offset: 0, size: 16, alignment: 16,
+    ; CHECK-NEXT:     stack-id: scalable-vector, callee-saved-register:
+    ; CHECK: liveins: $p0, $z0, $z1, $z2, $z3, $z4, $z5, $z6, $z7
+    ; CHECK-NEXT: {{  $}}
+    ;
+    ; CHECK-NEXT: $z16 = IMPLICIT_DEF
+    ; CHECK-NEXT: $z17 = IMPLICIT_DEF
+    ; CHECK-NEXT: $z18 = IMPLICIT_DEF
+    ; CHECK-NEXT: $z19 = IMPLICIT_DEF
+    ; CHECK-NEXT: $z20 = IMPLICIT_DEF
+    ; CHECK-NEXT: $z21 = IMPLICIT_DEF
+    ; CHECK-NEXT: $z22 = IMPLICIT_DEF
+    ; CHECK-NEXT: $z23 = IMPLICIT_DEF
+    ; CHECK-NEXT: $z24 = IMPLICIT_DEF
+    ; CHECK-NEXT: $z25 = IMPLICIT_DEF
+    ; CHECK-NEXT: $z26 = IMPLICIT_DEF
+    ; CHECK-NEXT: $z27 = IMPLICIT_DEF
+    ; CHECK-NEXT: $z28 = IMPLICIT_DEF
+    ; CHECK-NEXT: $z29 = IMPLICIT_DEF
+    ; CHECK-NEXT: $z30 = IMPLICIT_DEF
+    ; CHECK-NEXT: $z31 = IMPLICIT_DEF
+    ;
+    ; CHECK-NEXT: SPILL_PPR_TO_ZPR_SLOT_PSEUDO $p0, %stack.0, 0 :: (store (s128) into %stack.0)
+    ;
+    ; CHECK-NEXT: $p0 = IMPLICIT_DEF
+    ; CHECK-NEXT: $p1 = IMPLICIT_DEF
+    ; CHECK-NEXT: $p2 = IMPLICIT_DEF
+    ; CHECK-NEXT: $p3 = IMPLICIT_DEF
+    ; CHECK-NEXT: $p4 = IMPLICIT_DEF
+    ; CHECK-NEXT: $p5 = IMPLICIT_DEF
+    ; CHECK-NEXT: $p6 = IMPLICIT_DEF
+    ; CHECK-NEXT: $p7 = IMPLICIT_DEF
+    ; CHECK-NEXT: $p8 = IMPLICIT_DEF
+    ; CHECK-NEXT: $p9 = IMPLICIT_DEF
+    ; CHECK-NEXT: $p10 = IMPLICIT_DEF
+    ; CHECK-NEXT: $p11 = IMPLICIT_DEF
+    ; CHECK-NEXT: $p12 = IMPLICIT_DEF
+    ; CHECK-NEXT: $p13 = IMPLICIT_DEF
+    ; CHECK-NEXT: $p14 = IMPLICIT_DEF
+    ; CHECK-NEXT: $p15 = IMPLICIT_DEF
+    ;
+    ; CHECK-NEXT: $p0 = FILL_PPR_FROM_ZPR_SLOT_PSEUDO %stack.0, 0 :: (load (s128) from %stack.0)
+    ;
+    ; CHECK-NEXT: FAKE_USE implicit $z16, implicit $z17, implicit $z18, implicit $z19, implicit $z20, implicit $z21, implicit $z22, implicit $z23, implicit $z24, implicit $z25, implicit $z26, implicit $z27, implicit $z28, implicit $z29, implicit $z30, implicit $z31
+    ;
+    ; CHECK-NEXT: RET_ReallyLR implicit $p0, implicit $z0, implicit $z1, implicit $z2, implicit $z3, implicit $z4, implicit $z5, implicit $z6, implicit $z7
+
+    ; EXPAND-LABEL: name: zpr_predicate_spill__spill_zpr
+    ; EXPAND: liveins: $p0, $z0, $z1, $z2, $z3, $z4, $z5, $z6, $z7, $fp, $p15, $p14, $p13, $p12, $p11, $p10, $p9, $p8, $p7, $p6, $p5, $p4, $z23, $z22, $z21, $z20, $z19, $z18, $z17, $z16
+    ; EXPAND-NEXT: {{  $}}
+    ;
+    ; EXPAND-NEXT: $sp = frame-setup SUBXri $sp, 1040, 0
+    ; EXPAND-NEXT: frame-setup STRXui killed $fp, $sp, 128 :: (store (s64) into %stack.22)
+    ; EXPAND-NEXT: $sp = frame-setup ADDVL_XXI $sp, -20, implicit $vg
+    ; EXPAND-NEXT: $z24 = frame-setup CPY_ZPzI_B killed $p15, 1, 0
+    ; EXPAND-NEXT: frame-setup STR_ZXI $z24, $sp, 0 :: (store (s128) into %stack.21)
+    ; EXPAND-NEXT: $z24 = frame-setup CPY_ZPzI_B killed $p14, 1, 0
+    ; EXPAND-NEXT: frame-setup STR_ZXI $z24, $sp, 1 :: (store (s128) into %stack.20)
+    ; EXPAND-NEXT: $z24 = frame-setup CPY_ZPzI_B killed $p13, 1, 0
+    ; EXPAND-NEXT: frame-setup STR_ZXI $z24, $sp, 2 :: (store (s128) into %stack.19)
+    ; EXPAND-NEXT: $z24 = frame-setup CPY_ZPzI_B killed $p12, 1, 0
+    ; EXPAND-NEXT: frame-setup STR_ZXI $z24, $sp, 3 :: (store (s128) into %stack.18)
+    ; EXPAND-NEXT: $z24 = frame-setup CPY_ZPzI_B killed $p11, 1, 0
+    ; EXPAND-NEXT: frame-setup STR_ZXI $z24, $sp, 4 :: (store (s128) into %stack.17)
+    ; EXPAND-NEXT: $z24 = frame-setup CPY_ZPzI_B killed $p10, 1, 0
+    ; EXPAND-NEXT: frame-setup STR_ZXI $z24, $sp, 5 :: (store (s128) into %stack.16)
+    ; EXPAND-NEXT: $z24 = frame-setup CPY_ZPzI_B killed $p9, 1, 0
+    ; EXPAND-NEXT: frame-setup STR_ZXI $z24, $sp, 6 :: (store (s128) into %stack.15)
+    ; EXPAND-NEXT: $z24 = frame-setup CPY_ZPzI_B killed $p8, 1, 0
+    ; EXPAND-NEXT: frame-setup STR_ZXI $z24, $sp, 7 :: (store (s128) into %stack.14)
+    ; EXPAND-NEXT: $z24 = frame-setup CPY_ZPzI_B killed $p7, 1, 0
+    ; EXPAND-NEXT: frame-setup STR_ZXI $z24, $sp, 8 :: (store (s128) into %stack.13)
+    ; EXPAND-NEXT: $z24 = frame-setup CPY_ZPzI_B killed $p6, 1, 0
+    ; EXPAND-NEXT: frame-setup STR_ZXI $z24, $sp, 9 :: (store (s128) into %stack.12)
+    ; EXPAND-NEXT: $z24 = frame-setup CPY_ZPzI_B killed $p5, 1, 0
+    ; EXPAND-NEXT: frame-setup STR_ZXI $z24, $sp, 10 :: (store (s128) into %stack.11)
+    ; EXPAND-NEXT: $z24 = frame-setup CPY_ZPzI_B killed $p4, 1, 0
+    ; EXPAND-NEXT: frame-setup STR_ZXI $z24, $sp, 11 :: (store (s128) into %stack.10)
+    ; EXPAND-NEXT: frame-setup STR_ZXI killed $z23, $sp, 12 :: (store (s128) into %stack.9)
+    ; EXPAND-NEXT: frame-setup STR_ZXI killed $z22, $sp, 13 :: (store (s128) into %stack.8)
+    ; EXPAND-NEXT: frame-setup STR_ZXI killed $z21, $sp, 14 :: (store (s128) into %stack.7)
+    ; EXPAND-NEXT: frame-setup STR_ZXI killed $z20, $sp, 15 :: (store (s128) into %stack.6)
+    ; EXPAND-NEXT: frame-setup STR_ZXI killed $z19, $sp, 16 :: (store (s128) into %stack.5)
+    ; EXPAND-NEXT: frame-setup STR_ZXI killed $z18, $sp, 17 :: (store (s128) into %stack.4)
+    ; EXPAND-NEXT: frame-setup STR_ZXI killed $z17, $sp, 18 :: (store (s128) into %stack.3)
+    ; EXPAND-NEXT: frame-setup STR_ZXI killed $z16, $sp, 19 :: (store (s128) into %stack.2)
+    ; EXPAND-NEXT: $sp = frame-setup SUBXri $sp, 1024, 0
+    ; EXPAND-NEXT: $sp = frame-setup ADDVL_XXI $sp, -2, implicit $vg
+    ;
+    ; EXPAND-NEXT: $z16 = IMPLICIT_DEF
+    ; EXPAND-NEXT: $z17 = IMPLICIT_DEF
+    ; EXPAND-NEXT: $z18 = IMPLICIT_DEF
+    ; EXPAND-NEXT: $z19 = IMPLICIT_DEF
+    ; EXPAND-NEXT: $z20 = IMPLICIT_DEF
+    ; EXPAND-NEXT: $z21 = IMPLICIT_DEF
+    ; EXPAND-NEXT: $z22 = IMPLICIT_DEF
+    ; EXPAND-NEXT: $z23 = IMPLICIT_DEF
+    ; EXPAND-NEXT: $z24 = IMPLICIT_DEF
+    ; EXPAND-NEXT: $z25 = IMPLICIT_DEF
+    ; EXPAND-NEXT: $z26 = IMPLICIT_DEF
+    ; EXPAND-NEXT: $z27 = IMPLICIT_DEF
+    ; EXPAND-NEXT: $z28 = IMPLICIT_DEF
+    ; EXPAND-NEXT: $z29 = IMPLICIT_DEF
+    ; EXPAND-NEXT: $z30 = IMPLICIT_DEF
+    ; EXPAND-NEXT: $z31 = IMPLICIT_DEF
+    ;
+    ; EXPAND-NEXT: $x8 = ADDXri $sp, 1024, 0
+    ; EXPAND-NEXT: STR_ZXI $z0, $x8, 0 :: (store (s128) into %stack.24)
+    ; EXPAND-NEXT: $z0 = CPY_ZPzI_B $p0, 1, 0
+    ; EXPAND-NEXT: STR_ZXI $z0, $x8, 1 :: (store (s128) into %stack.0)
+    ; EXPAND-NEXT: $z0 = LDR_ZXI $x8, 0 :: (load (s128) from %stack.24)
+    ;
+    ; EXPAND-NEXT: $p0 = IMPLICIT_DEF
+    ; EXPAND-NEXT: $p1 = IMPLICIT_DEF
+    ; EXPAND-NEXT: $p2 = IMPLICIT_DEF
+    ; EXPAND-NEXT: $p3 = IMPLICIT_DEF
+    ; EXPAND-NEXT: $p4 = IMPLICIT_DEF
+    ; EXPAND-NEXT: $p5 = IMPLICIT_DEF
+    ; EXPAND-NEXT: $p6 = IMPLICIT_DEF
+    ; EXPAND-NEXT: $p7 = IMPLICIT_DEF
+    ; EXPAND-NEXT: $p8 = IMPLICIT_DEF
+    ; EXPAND-NEXT: $p9 = IMPLICIT_DEF
+    ; EXPAND-NEXT: $p10 = IMPLICIT_DEF
+    ; EXPAND-NEXT: $p11 = IMPLICIT_DEF
+    ; EXPAND-NEXT: $p12 = IMPLICIT_DEF
+    ; EXPAND-NEXT: $p13 = IMPLICIT_DEF
+    ; EXPAND-NEXT: $p14 = IMPLICIT_DEF
+    ; EXPAND-NEXT: $p15 = IMPLICIT_DEF
+    ;
+    ; EXPAND-NEXT: STR_ZXI $z0, $x8, 0 :: (store (s128) into %stack.24)
+    ; EXPAND-NEXT: $z0 = LDR_ZXI $x8, 1 :: (load (s128) from %stack.0)
+    ; EXPAND-NEXT: $p0 = PTRUE_B 31, implicit $vg
+    ; EXPAND-NEXT: $p0 = CMPNE_PPzZI_B $p0, $z0, 0, implicit-def $nzcv, implicit-def $nzcv
+    ; EXPAND-NEXT: $z0 = LDR_ZXI killed $x8, 0 :: (load (s128) from %stack.24)
+    ;
+    ; EXPAND-NEXT: FAKE_USE implicit $z16, implicit $z17, implicit $z18, implicit $z19, implicit $z20, implicit $z21, implicit $z22, implicit $z23, implicit $z24, implicit $z25, implicit $z26, implicit $z27, implicit $z28, implicit $z29, implicit $z30, implicit $z31
+    ;
+    ; EXPAND-NEXT: $sp = frame-destroy ADDXri $sp, 1024, 0
+    ; EXPAND-NEXT: $sp = frame-destroy ADDVL_XXI $sp, 2, implicit $vg
+    ; EXPAND-NEXT: $z23 = frame-destroy LDR_ZXI $sp, 12 :: (load (s128) from %stack.9)
+    ; EXPAND-NEXT: $z22 = frame-destroy LDR_ZXI $sp, 13 :: (load (s128) from %stack.8)
+    ; EXPAND-NEXT: $z21 = frame-destroy LDR_ZXI $sp, 14 :: (load (s128) from %stack.7)
+    ; EXPAND-NEXT: $z20 = frame-destroy LDR_ZXI $sp, 15 :: (load (s128) from %stack.6)
+    ; EXPAND-NEXT: $z19 = frame-destroy LDR_ZXI $sp, 16 :: (load (s128) from %stack.5)
+    ; EXPAND-NEXT: $z18 = frame-destroy LDR_ZXI $sp, 17 :: (load (s128) from %stack.4)
+    ; EXPAND-NEXT: $z17 = frame-destroy LDR_ZXI $sp, 18 :: (load (s128) from %stack.3)
+    ; EXPAND-NEXT: $z16 = frame-destroy LDR_ZXI $sp, 19 :: (load (s128) from %stack.2)
+    ; EXPAND-NEXT: $z24 = frame-destroy LDR_ZXI $sp, 0 :: (load (s128) from %stack.21)
+    ; EXPAND-NEXT: $p1 = frame-destroy PTRUE_B 31, implicit $vg
+    ; EXPAND-NEXT: $p15 = frame-destroy CMPNE_PPzZI_B $p1, $z24, 0, implicit-def $nzcv, implicit-def $nzcv
+    ; EXPAND-NEXT: $z24 = frame-destroy LDR_ZXI $sp, 1 :: (load (s128) from %stack.20)
+    ; EXPAND-NEXT: $p14 = frame-destroy CMPNE_PPzZI_B $p1, $z24, 0, implicit-def $nzcv, implicit-def $nzcv
+    ; EXPAND-NEXT: $z24 = frame-destroy LDR_ZXI $sp, 2 :: (load (s128) from %stack.19)
+    ; EXPAND-NEXT: $p13 = frame-destroy CMPNE_PPzZI_B $p1, $z24, 0, implicit-def $nzcv, implicit-def $nzcv
+    ; EXPAND-NEXT: $z24 = frame-destroy LDR_ZXI $sp, 3 :: (load (s128) from %stack.18)
+    ; EXPAND-NEXT: $p12 = frame-destroy CMPNE_PPzZI_B $p1, $z24, 0, implicit-def $nzcv, implicit-def $nzcv
+    ; EXPAND-NEXT: $z24 = frame-destroy LDR_ZXI $sp, 4 :: (load (s128) from %stack.17)
+    ; EXPAND-NEXT: $p11 = frame-destroy CMPNE_PPzZI_B $p1, $z24, 0, implicit-def $nzcv, implicit-def $nzcv
+    ; EXPAND-NEXT: $z24 = frame-destroy LDR_ZXI $sp, 5 :: (load (s128) from %stack.16)
+    ; EXPAND-NEXT: $p10 = frame-destroy CMPNE_PPzZI_B $p1, $z24, 0, implicit-def $nzcv, implicit-def $nzcv
+    ; EXPAND-NEXT: $z24 = frame-destroy LDR_ZXI $sp, 6 :: (load (s128) from %stack.15)
+    ; EXPAND-NEXT: $p9 = frame-destroy CMPNE_PPzZI_B $p1, $z24, 0, implicit-def $nzcv, implicit-def $nzcv
+    ; EXPAND-NEXT: $z24 = frame-destroy LDR_ZXI $sp, 7 :: (load (s128) from %stack.14)
+    ; EXPAND-NEXT: $p8 = frame-destroy CMPNE_PPzZI_B $p1, $z24, 0, implicit-def $nzcv, implicit-def $nzcv
+    ; EXPAND-NEXT: $z24 = frame-destroy LDR_ZXI $sp, 8 :: (load (s128) from %stack.13)
+    ; EXPAND-NEXT: $p7 = frame-destroy CMPNE_PPzZI_B $p1, $z24, 0, implicit-def $nzcv, implicit-def $nzcv
+    ; EXPAND-NEXT: $z24 = frame-destroy LDR_ZXI $sp, 9 :: (load (s128) from %stack.12)
+    ; EXPAND-NEXT: $p6 = frame-destroy CMPNE_PPzZI_B $p1, $z24, 0, implicit-def $nzcv, implicit-def $nzcv
+    ; EXPAND-NEXT: $z24 = frame-destroy LDR_ZXI $sp, 10 :: (load (s128) from %stack.11)
+    ; EXPAND-NEXT: $p5 = frame-destroy CMPNE_PPzZI_B $p1, $z24, 0, implicit-def $nzcv, implicit-def $nzcv
+    ; EXPAND-NEXT: $z24 = frame-destroy LDR_ZXI $sp, 11 :: (load (s128) from %stack.10)
+    ; EXPAND-NEXT: $p4 = frame-destroy CMPNE_PPzZI_B $p1, $z24, 0, implicit-def $nzcv, implicit-def $nzcv
+    ; EXPAND-NEXT: $sp = frame-destroy ADDVL_XXI $sp, 20, implicit $vg
+    ; EXPAND-NEXT: $fp = frame-destroy LDRXui $sp, 128 :: (load (s64) from %stack.22)
+    ; EXPAND-NEXT: $sp = frame-destroy ADDXri $sp, 1040, 0
+    ; EXPAND-NEXT: RET undef $lr, implicit $p0, implicit $z0, implicit $z1, implicit $z2, implicit $z3, implicit $z4, implicit $z5, implicit $z6, implicit $z7
+    $z16 = IMPLICIT_DEF
+    $z17 = IMPLICIT_DEF
+    $z18 = IMPLICIT_DEF
+    $z19 = IMPLICIT_DEF
+    $z20 = IMPLICIT_DEF
+    $z21 = IMPLICIT_DEF
+    $z22 = IMPLICIT_DEF
+    $z23 = IMPLICIT_DEF
+    $z24 = IMPLICIT_DEF
+    $z25 = IMPLICIT_DEF
+    $z26 = IMPLICIT_DEF
+    $z27 = IMPLICIT_DEF
+    $z28 = IMPLICIT_DEF
+    $z29 = IMPLICIT_DEF
+    $z30 = IMPLICIT_DEF
+    $z31 = IMPLICIT_DEF
+
+    %1:ppr = COPY $p0
+
+    $p0 = IMPLICIT_DEF
+    $p1 = IMPLICIT_DEF
+    $p2 = IMPLICIT_DEF
+    $p3 = IMPLICIT_DEF
+    $p4 = IMPLICIT_DEF
+    $p5 = IMPLICIT_DEF
+    $p6 = IMPLICIT_DEF
+    $p7 = IMPLICIT_DEF
+    $p8 = IMPLICIT_DEF
+    $p9 = IMPLICIT_DEF
+    $p10 = IMPLICIT_DEF
+    $p11 = IMPLICIT_DEF
+    $p12 = IMPLICIT_DEF
+    $p13 = IMPLICIT_DEF
+    $p14 = IMPLICIT_DEF
+    $p15 = IMPLICIT_DEF
+
+    $p0 = COPY %1
+
+    FAKE_USE implicit $z16, implicit $z17, implicit $z18, implicit $z19, implicit $z20, implicit $z21, implicit $z22, implicit $z23, implicit $z24, implicit $z25, implicit $z26, implicit $z27, implicit $z28, implicit $z29, implicit $z30, implicit $z31
+
+    RET_ReallyLR implicit $p0, implicit $z0, implicit $z1, implicit $z2, implicit $z3, implicit $z4, implicit $z5, implicit $z6, implicit $z7
+...
+---
+name: zpr_predicate_spill_above_p7
+tracksRegLiveness: true
+stack:
+liveins:
+  - { reg: '$p0' }
+  - { reg: '$p1' }
+  - { reg: '$p2' }
+  - { reg: '$p3' }
+body:             |
+  bb.0.entry:
+    liveins: $p0, $p1, $p2, $p3
+
+    ; CHECK-LABEL: name: zpr_predicate_spill_above_p7
+    ; CHECK: stack:
+    ; CHECK:      - { id: 0, name: '', type: spill-slot, offset: 0, size: 16, alignment: 16,
+    ; CHECK-NEXT:     stack-id: scalable-vector, callee-saved-register:
+    ; CHECK: liveins: $p0, $p1, $p2, $p3
+    ; CHECK-NEXT: {{  $}}
+    ;
+    ; CHECK-NEXT: $p15 = IMPLICIT_DEF
+    ;
+    ; CHECK-NEXT: SPILL_PPR_TO_ZPR_SLOT_PSEUDO $p15, %stack.0, 0 :: (store (s128) into %stack.0)
+    ;
+    ; CHECK-NEXT: $p0 = IMPLICIT_DEF
+    ; CHECK-NEXT: $p1 = IMPLICIT_DEF
+    ; CHECK-NEXT: $p2 = IMPLICIT_DEF
+    ; CHECK-NEXT: $p3 = IMPLICIT_DEF
+    ; CHECK-NEXT: $p4 = IMPLICIT_DEF
+    ; CHECK-NEXT: $p5 = IMPLICIT_DEF
+    ; CHECK-NEXT: $p6 = IMPLICIT_DEF
+    ; CHECK-NEXT: $p7 = IMPLICIT_DEF
+    ; CHECK-NEXT: $p8 = IMPLICIT_DEF
+    ; CHECK-NEXT: $p9 = IMPLICIT_DEF
+    ; CHECK-NEXT: $p10 = IMPLICIT_DEF
+    ; CHECK-NEXT: $p11 = IMPLICIT_DEF
+    ; CHECK-NEXT: $p12 = IMPLICIT_DEF
+    ; CHECK-NEXT: $p13 = IMPLICIT_DEF
+    ; CHECK-NEXT: $p14 = IMPLICIT_DEF
+    ; CHECK-NEXT: $p15 = IMPLICIT_DEF
+    ;
+    ; CHECK-NEXT: $p15 = FILL_PPR_FROM_ZPR_SLOT_PSEUDO %stack.0, 0 :: (load (s128) from %stack.0)
+    ;
+    ; CHECK-NEXT: FAKE_USE implicit $p4, implicit $p5, implicit $p6, implicit $p7
+    ;
+    ; CHECK-NEXT: RET_ReallyLR implicit $p0, implicit $p1, implicit $p2, implicit $p3
+
+    ; EXPAND-LABEL: name: zpr_predicate_spill_above_p7
+    ; EXPAND: liveins: $p0, $p1, $p2, $p3, $fp, $p15, $p14, $p13, $p12, $p11, $p10, $p9, $p8, $p7, $p6, $p5, $p4
+    ; EXPAND-NEXT: {{  $}}
+    ;
+    ; EXPAND-NEXT: $sp = frame-setup SUBXri $sp, 1040, 0
+    ; EXPAND-NEXT: frame-setup STRXui killed $fp, $sp, 128 :: (store (s64) into %stack.14)
+    ; EXPAND-NEXT: $sp = frame-setup ADDVL_XXI $sp, -12, implicit $vg
+    ; EXPAND-NEXT: $z0 = frame-setup CPY_ZPzI_B killed $p15, 1, 0
+    ; EXPAND-NEXT: frame-setup STR_ZXI $z0, $sp, 0 :: (store (s128) into %stack.13)
+    ; EXPAND-NEXT: $z0 = frame-setup CPY_ZPzI_B killed $p14, 1, 0
+    ; EXPAND-NEXT: frame-setup STR_ZXI $z0, $sp, 1 :: (store (s128) into %stack.12)
+    ; EXPAND-NEXT: $z0 = frame-setup CPY_ZPzI_B killed $p13, 1, 0
+    ; EXPAND-NEXT: frame-setup STR_ZXI $z0, $sp, 2 :: (store (s128) into %stack.11)
+    ; EXPAND-NEXT: $z0 = frame-setup CPY_ZPzI_B killed $p12, 1, 0
+    ; EXPAND-NEXT: frame-setup STR_ZXI $z0, $sp, 3 :: (store (s128) into %stack.10)
+    ; EXPAND-NEXT: $z0 = frame-setup CPY_ZPzI_B killed $p11, 1, 0
+    ; EXPAND-NEXT: frame-setup STR_ZXI $z0, $sp, 4 :: (store (s128) into %stack.9)
+    ; EXPAND-NEXT: $z0 = frame-setup CPY_ZPzI_B killed $p10, 1, 0
+    ; EXPAND-NEXT: frame-setup STR_ZXI $z0, $sp, 5 :: (store (s128) into %stack.8)
+    ; EXPAND-NEXT: $z0 = frame-setup CPY_ZPzI_B killed $p9, 1, 0
+    ; EXPAND-NEXT: frame-setup STR_ZXI $z0, $sp, 6 :: (store (s128) into %stack.7)
+    ; EXPAND-NEXT: $z0 = frame-setup CPY_ZPzI_B killed $p8, 1, 0
+    ; EXPAND-NEXT: frame-setup STR_ZXI $z0, $sp, 7 :: (store (s128) into %stack.6)
+    ; EXPAND-NEXT: $z0 = frame-setup CPY_ZPzI_B killed $p7, 1, 0
+    ; EXPAND-NEXT: frame-setup STR_ZXI $z0, $sp, 8 :: (store (s128) into %stack.5)
+    ; EXPAND-NEXT: $z0 = frame-setup CPY_ZPzI_B killed $p6, 1, 0
+    ; EXPAND-NEXT: frame-setup STR_ZXI $z0, $sp, 9 :: (store (s128) into %stack.4)
+    ; EXPAND-NEXT: $z0 = frame-setup CPY_ZPzI_B killed $p5, 1, 0
+    ; EXPAND-NEXT: frame-setup STR_ZXI $z0, $sp, 10 :: (store (s128) into %stack.3)
+    ; EXPAND-NEXT: $z0 = frame-setup CPY_ZPzI_B killed $p4, 1, 0
+    ; EXPAND-NEXT: frame-setup STR_ZXI $z0, $sp, 11 :: (store (s128) into %stack.2)
+    ; EXPAND-NEXT: $sp = frame-setup SUBXri $sp, 1024, 0
+    ; EXPAND-NEXT: $sp = frame-setup ADDVL_XXI $sp, -2, implicit $vg
+    ;
+    ; EXPAND-NEXT: $p15 = IMPLICIT_DEF
+    ;
+    ; EXPAND-NEXT: $z0 = CPY_ZPzI_B $p15, 1, 0
+    ; EXPAND-NEXT: $x8 = ADDXri $sp, 1024, 0
+    ; EXPAND-NEXT: STR_ZXI $z0, $x8, 1 :: (store (s128) into %stack.0)
+    ;
+    ; EXPAND-NEXT: $p0 = IMPLICIT_DEF
+    ; EXPAND-NEXT: $p1 = IMPLICIT_DEF
+    ; EXPAND-NEXT: $p2 = IMPLICIT_DEF
+    ; EXPAND-NEXT: $p3 = IMPLICIT_DEF
+    ; EXPAND-NEXT: $p4 = IMPLICIT_DEF
+    ; EXPAND-NEXT: $p5 = IMPLICIT_DEF
+    ; EXPAND-NEXT: $p6 = IMPLICIT_DEF
+    ; EXPAND-NEXT: $p7 = IMPLICIT_DEF
+    ; EXPAND-NEXT: $p8 = IMPLICIT_DEF
+    ; EXPAND-NEXT: $p9 = IMPLICIT_DEF
+    ; EXPAND-NEXT: $p10 = IMPLICIT_DEF
+    ; EXPAND-NEXT: $p11 = IMPLICIT_DEF
+    ; EXPAND-NEXT: $p12 = IMPLICIT_DEF
+    ; EXPAND-NEXT: $p13 = IMPLICIT_DEF
+    ; EXPAND-NEXT: $p14 = IMPLICIT_DEF
+    ; EXPAND-NEXT: $p15 = IMPLICIT_DEF
+    ;
+    ; EXPAND-NEXT: $z0 = CPY_ZPzI_B $p0, 1, 0
+    ; EXPAND-NEXT: STR_ZXI $z0, $x8, 0 :: (store (s128) into %stack.16)
+    ; EXPAND-NEXT: $z0 = LDR_ZXI $x8, 1 :: (load (s128) from %stack.0)
+    ; EXPAND-NEXT: $p0 = PTRUE_B 31, implicit $vg
+    ; EXPAND-NEXT: $p15 = CMPNE_PPzZI_B $p0, $z0, 0, implicit-def $nzcv, implicit-def $nzcv
+    ; EXPAND-NEXT: $z0 = LDR_ZXI killed $x8, 0 :: (load (s128) from %stack.16)
+    ; EXPAND-NEXT: $p0 = PTRUE_B 31, implicit $vg
+    ; EXPAND-NEXT: $p0 = CMPNE_PPzZI_B $p0, $z0, 0, implicit-def $nzcv, implicit-def $nzcv
+    ;
+    ; EXPAND-NEXT: FAKE_USE implicit $p4, implicit $p5, implicit $p6, implicit $p7
+    ;
+    ; EXPAND-NEXT: $sp = frame-destroy ADDXri $sp, 1024, 0
+    ; EXPAND-NEXT: $sp = frame-destroy ADDVL_XXI $sp, 2, implicit $vg
+    ; EXPAND-NEXT: $z0 = frame-destroy LDR_ZXI $sp, 0 :: (load (s128) from %stack.13)
+    ; EXPAND-NEXT: $p4 = frame-destroy PTRUE_B 31, implicit $vg
+    ; EXPAND-NEXT: $p15 = frame-destroy CMPNE_PPzZI_B $p4, $z0, 0, implicit-def $nzcv, implicit-def $nzcv
+    ; EXPAND-NEXT: $z0 = frame-destroy LDR_ZXI $sp, 1 :: (load (s128) from %stack.12)
+    ; EXPAND-NEXT: $p14 = frame-destroy CMPNE_PPzZI_B $p4, $z0, 0, implicit-def $nzcv, implicit-def $nzcv
+    ; EXPAND-NEXT: $z0 = frame-destroy LDR_ZXI $sp, 2 :: (load (s128) from %stack.11)
+    ; EXPAND-NEXT: $p13 = frame-destroy CMPNE_PPzZI_B $p4, $z0, 0, implicit-def $nzcv, implicit-def $nzcv
+    ; EXPAND-NEXT: $z0 = frame-destroy LDR_ZXI $sp, 3 :: (load (s128) from %stack.10)
+    ; EXPAND-NEXT: $p12 = frame-destroy CMPNE_PPzZI_B $p4, $z0, 0, implicit-def $nzcv, implicit-def $nzcv
+    ; EXPAND-NEXT: $z0 = frame-destroy LDR_ZXI $sp, 4 :: (load (s128) from %stack.9)
+    ; EXPAND-NEXT: $p11 = frame-destroy CMPNE_PPzZI_B $p4, $z0, 0, implicit-def $nzcv, implicit-def $nzcv
+    ; EXPAND-NEXT: $z0 = frame-destroy LDR_ZXI $sp, 5 :: (load (s128) from %stack.8)
+    ; EXPAND-NEXT: $p10 = frame-destroy CMPNE_PPzZI_B $p4, $z0, 0, implicit-def $nzcv, implicit-def $nzcv
+    ; EXPAND-NEXT: $z0 = frame-destroy LDR_ZXI $sp, 6 :: (load (s128) from %stack.7)
+    ; EXPAND-NEXT: $p9 = frame-destroy CMPNE_PPzZI_B $p4, $z0, 0, implicit-def $nzcv, implicit-def $nzcv
+    ; EXPAND-NEXT: $z0 = frame-destroy LDR_ZXI $sp, 7 :: (load (s128) from %stack.6)
+    ; EXPAND-NEXT: $p8 = frame-destroy CMPNE_PPzZI_B $p4, $z0, 0, implicit-def $nzcv, implicit-def $nzcv
+    ; EXPAND-NEXT: $z0 = frame-destroy LDR_ZXI $sp, 8 :: (load (s128) from %stack.5)
+    ; EXPAND-NEXT: $p7 = frame-destroy CMPNE_PPzZI_B $p4, $z0, 0, implicit-def $nzcv, implicit-def $nzcv
+    ; EXPAND-NEXT: $z0 = frame-destroy LDR_ZXI $sp, 9 :: (load (s128) from %stack.4)
+    ; EXPAND-NEXT: $p6 = frame-destroy CMPNE_PPzZI_B $p4, $z0, 0, implicit-def $nzcv, implicit-def $nzcv
+    ; EXPAND-NEXT: $z0 = frame-destroy LDR_ZXI $sp, 10 :: (load (s128) from %stack.3)
+    ; EXPAND-NEXT: $p5 = frame-destroy CMPNE_PPzZI_B $p4, $z0, 0, implicit-def $nzcv, implicit-def $nzcv
+    ; EXPAND-NEXT: $z0 = frame-destroy LDR_ZXI $sp, 11 :: (load (s128) from %stack.2)
+    ; EXPAND-NEXT: $p4 = frame-destroy CMPNE_PPzZI_B $p4, $z0, 0, implicit-def $nzcv, implicit-def $nzcv
+    ; EXPAND-NEXT: $sp = frame-destroy ADDVL_XXI $sp, 12, implicit $vg
+    ; EXPAND-NEXT: $fp = frame-destroy LDRXui $sp, 128 :: (load (s64) from %stack.14)
+    ; EXPAND-NEXT: $sp = frame-destroy ADDXri $sp, 1040, 0
+    ; EXPAND-NEXT: RET undef $lr, implicit $p0, implicit $p1, implicit $p2, implicit $p3
+    $p15 = IMPLICIT_DEF
+    %1:ppr = COPY $p15
+
+    $p0 = IMPLICIT_DEF
+    $p1 = IMPLICIT_DEF
+    $p2 = IMPLICIT_DEF
+    $p3 = IMPLICIT_DEF
+    $p4 = IMPLICIT_DEF
+    $p5 = IMPLICIT_DEF
+    $p6 = IMPLICIT_DEF
+    $p7 = IMPLICIT_DEF
+    $p8 = IMPLICIT_DEF
+    $p9 = IMPLICIT_DEF
+    $p10 = IMPLICIT_DEF
+    $p11 = IMPLICIT_DEF
+    $p12 = IMPLICIT_DEF
+    $p13 = IMPLICIT_DEF
+    $p14 = IMPLICIT_DEF
+    $p15 = IMPLICIT_DEF
+
+    $p15 = COPY %1
+
+    FAKE_USE implicit $p4, implicit $p5, implicit $p6, implicit $p7
+
+    RET_ReallyLR implicit $p0, implicit $p1, implicit $p2, implicit $p3
+...
+---
+name: zpr_predicate_spill_p4_saved
+tracksRegLiveness: true
+stack:
+liveins:
+  - { reg: '$p0' }
+  - { reg: '$p1' }
+  - { reg: '$p2' }
+  - { reg: '$p3' }
+body:             |
+  bb.0.entry:
+    liveins: $p0, $p1, $p2, $p3
+
+    ; CHECK-LABEL: name: zpr_predicate_spill_p4_saved
+    ; CHECK: liveins: $p0, $p1, $p2, $p3
+    ; CHECK-NEXT: {{  $}}
+    ;
+    ; CHECK-NEXT: $p8 = IMPLICIT_DEF
+    ;
+    ; CHECK-NEXT: RET_ReallyLR implicit $p0, implicit $p1, implicit $p2, implicit $p3
+
+    ; EXPAND-LABEL: name: zpr_predicate_spill_p4_saved
+    ; EXPAND: liveins: $p0, $p1, $p2, $p3, $fp, $p8, $p4
+    ; EXPAND-NEXT: {{  $}}
+    ; EXPAND-NEXT: $sp = frame-setup SUBXri $sp, 1040, 0
+    ; EXPAND-NEXT: frame-setup STRXui killed $fp, $sp, 128 :: (store (s64) into %stack.3)
+    ; EXPAND-NEXT: $sp = frame-setup ADDVL_XXI $sp, -2, implicit $vg
+    ; EXPAND-NEXT: $z0 = frame-setup CPY_ZPzI_B killed $p8, 1, 0
+    ; EXPAND-NEXT: frame-setup STR_ZXI $z0, $sp, 0 :: (store (s128) into %stack.2)
+    ; EXPAND-NEXT: $z0 = frame-setup CPY_ZPzI_B killed $p4, 1, 0
+    ; EXPAND-NEXT: frame-setup STR_ZXI $z0, $sp, 1 :: (store (s128) into %stack.1)
+    ; EXPAND-NEXT: $sp = frame-setup SUBXri $sp, 1024, 0
+    ;
+    ; EXPAND-NEXT: $p8 = IMPLICIT_DEF
+    ;
+    ; EXPAND-NEXT: $sp = frame-destroy ADDXri $sp, 1024, 0
+    ; EXPAND-NEXT: $z0 = frame-destroy LDR_ZXI $sp, 0 :: (load (s128) from %stack.2)
+    ; EXPAND-NEXT: $p4 = frame-destroy PTRUE_B 31, implicit $vg
+    ; EXPAND-NEXT: $p8 = frame-destroy CMPNE_PPzZI_B $p4, $z0, 0, implicit-def $nzcv, implicit-def $nzcv
+    ; EXPAND-NEXT: $z0 = frame-destroy LDR_ZXI $sp, 1 :: (load (s128) from %stack.1)
+    ; EXPAND-NEXT: $p4 = frame-destroy CMPNE_PPzZI_B $p4, $z0, 0, implicit-def $nzcv, implicit-def $nzcv
+    ; EXPAND-NEXT: $sp = frame-destroy ADDVL_XXI $sp, 2, implicit $vg
+    ; EXPAND-NEXT: $fp = frame-destroy LDRXui $sp, 128 :: (load (s64) from %stack.3)
+    ; EXPAND-NEXT: $sp = frame-destroy ADDXri $sp, 1040, 0
+    ; EXPAND-NEXT: RET undef $lr, implicit $p0, implicit $p1, implicit $p2, implicit $p3
+
+    ; If we spill a register above p8, p4 must also be saved, so we can guarantee
+    ; they will be a register (in the range p0-p7 to for the cmpne reload).
+    $p8 = IMPLICIT_DEF
+
+    RET_ReallyLR implicit $p0, implicit $p1, implicit $p2, implicit $p3
+...
diff --git a/llvm/test/CodeGen/AArch64/ssve-stack-hazard-remarks.ll b/llvm/test/CodeGen/AArch64/ssve-stack-hazard-remarks.ll
index 0b6bf3892a0c2..c67d91952c618 100644
--- a/llvm/test/CodeGen/AArch64/ssve-stack-hazard-remarks.ll
+++ b/llvm/test/CodeGen/AArch64/ssve-stack-hazard-remarks.ll
@@ -1,5 +1,7 @@
 ; RUN: llc < %s -mtriple=aarch64 -mattr=+sve2 -pass-remarks-analysis=sme -aarch64-stack-hazard-remark-size=64 -o /dev/null < %s 2>&1 | FileCheck %s --check-prefixes=CHECK
 ; RUN: llc < %s -mtriple=aarch64 -mattr=+sve2 -pass-remarks-analysis=sme -aarch64-stack-hazard-size=1024 -o /dev/null < %s 2>&1 | FileCheck %s --check-prefixes=CHECK-PADDING
+; RUN: llc < %s -mtriple=aarch64 -mattr=+sve2 -pass-remarks-analysis=sme -aarch64-enable-zpr-predicate-spills -aarch64-stack-hazard-remark-size=64 -o /dev/null < %s 2>&1 | FileCheck %s --check-prefixes=CHECK-ZPR-PRED-SPILLS
+; RUN: llc < %s -mtriple=aarch64 -mattr=+sve2 -pass-remarks-analysis=sme -aarch64-enable-zpr-predicate-spills -aarch64-stack-hazard-size=1024 -o /dev/null < %s 2>&1 | FileCheck %s --check-prefixes=CHECK-ZPR-PRED-SPILLS-WITH-PADDING
 
 ; Don't emit remarks for non-streaming functions.
 define float @csr_x20_stackargs_notsc(float %a, float %b, float %c, float %d, float %e, float %f, float %g, float %h, float %i) {
@@ -66,13 +68,18 @@ entry:
 }
 
 ; SVE calling conventions
-; Predicate register spills end up in FP region, currently.
+; Predicate register spills end up in FP region, currently. This can be
+; mitigated with the -aarch64-enable-zpr-predicate-spills option.
 
 define i32 @svecc_call(<4 x i16> %P0, ptr %P1, i32 %P2, <vscale x 16 x i8> %P3, i16 %P4) #2 {
 ; CHECK: remark: <unknown>:0:0: stack hazard in 'svecc_call': PPR stack object at [SP-48-258 * vscale] is too close to FPR stack object at [SP-48-256 * vscale]
 ; CHECK: remark: <unknown>:0:0: stack hazard in 'svecc_call': FPR stack object at [SP-48-16 * vscale] is too close to GPR stack object at [SP-48]
 ; CHECK-PADDING: remark: <unknown>:0:0: stack hazard in 'svecc_call': PPR stack object at [SP-1072-258 * vscale] is too close to FPR stack object at [SP-1072-256 * vscale]
 ; CHECK-PADDING-NOT: remark: <unknown>:0:0: stack hazard in 'svecc_call':
+; CHECK-ZPR-PRED-SPILLS-NOT: <unknown>:0:0: stack hazard in 'svecc_call': PPR stack object at {{.*}} is too close to FPR stack object
+; CHECK-ZPR-PRED-SPILLS: <unknown>:0:0: stack hazard in 'svecc_call': FPR stack object at [SP-48-16 * vscale] is too close to GPR stack object at [SP-48]
+; CHECK-ZPR-PRED-SPILLS-WITH-PADDING-NOT: <unknown>:0:0: stack hazard in 'svecc_call': PPR stack object at {{.*}} is too close to FPR stack object
+; CHECK-ZPR-PRED-SPILLS-WITH-PADDING-NOT: <unknown>:0:0: stack hazard in 'svecc_call': FPR stack object at {{.*}} is too close to GPR stack object
 entry:
   tail call void asm sideeffect "", "~{x0},~{x28},~{x27},~{x3}"() #2
   %call = call ptr @memset(ptr noundef nonnull %P1, i32 noundef 45, i32 noundef 37)
@@ -84,6 +91,10 @@ define i32 @svecc_alloca_call(<4 x i16> %P0, ptr %P1, i32 %P2, <vscale x 16 x i8
 ; CHECK: remark: <unknown>:0:0: stack hazard in 'svecc_alloca_call': FPR stack object at [SP-48-16 * vscale] is too close to GPR stack object at [SP-48]
 ; CHECK-PADDING: remark: <unknown>:0:0: stack hazard in 'svecc_alloca_call': PPR stack object at [SP-1072-258 * vscale] is too close to FPR stack object at [SP-1072-256 * vscale]
 ; CHECK-PADDING-NOT: remark: <unknown>:0:0: stack hazard in 'svecc_alloca_call':
+; CHECK-ZPR-PRED-SPILLS-NOT: <unknown>:0:0: stack hazard in 'svecc_call': PPR stack object at {{.*}} is too close to FPR stack object
+; CHECK-ZPR-PRED-SPILLS: <unknown>:0:0: stack hazard in 'svecc_alloca_call': FPR stack object at [SP-48-16 * vscale] is too close to GPR stack object at [SP-48]
+; CHECK-ZPR-PRED-SPILLS-WITH-PADDING-NOT: <unknown>:0:0: stack hazard in 'svecc_alloca_call': PPR stack object at {{.*}} is too close to FPR stack object
+; CHECK-ZPR-PRED-SPILLS-WITH-PADDING-NOT: <unknown>:0:0: stack hazard in 'svecc_alloca_call': FPR stack object at {{.*}} is too close to GPR stack object
 entry:
   tail call void asm sideeffect "", "~{x0},~{x28},~{x27},~{x3}"() #2
   %0 = alloca [37 x i8], align 16
diff --git a/llvm/test/CodeGen/AArch64/stack-hazard-windows.ll b/llvm/test/CodeGen/AArch64/stack-hazard-windows.ll
index 2a034fe5e5290..927d8b68c46be 100644
--- a/llvm/test/CodeGen/AArch64/stack-hazard-windows.ll
+++ b/llvm/test/CodeGen/AArch64/stack-hazard-windows.ll
@@ -76,7 +76,9 @@ define i32 @fpr_csr_stackobj(double %x) "aarch64_pstate_sm_compatible" "frame-po
 ; CHECK1024:       .seh_proc fpr_csr_stackobj
 ; CHECK1024-NEXT:  // %bb.0: // %entry
 ; CHECK1024-NEXT:    sub sp, sp, #1072
+; CHECK1024-NEXT:    .seh_stackalloc 1072
 ; CHECK1024-NEXT:    str x23, [sp] // 8-byte Folded Spill
+; CHECK1024-NEXT:    .seh_save_reg x23, 0
 ; CHECK1024-NEXT:    str x29, [sp, #8] // 8-byte Folded Spill
 ; CHECK1024-NEXT:    .seh_save_reg x29, 8
 ; CHECK1024-NEXT:    str x30, [sp, #16] // 8-byte Folded Spill
@@ -105,7 +107,9 @@ define i32 @fpr_csr_stackobj(double %x) "aarch64_pstate_sm_compatible" "frame-po
 ; CHECK1024-NEXT:    ldr x29, [sp, #8] // 8-byte Folded Reload
 ; CHECK1024-NEXT:    .seh_save_reg x29, 8
 ; CHECK1024-NEXT:    ldr x23, [sp] // 8-byte Folded Reload
+; CHECK1024-NEXT:    .seh_save_reg x23, 0
 ; CHECK1024-NEXT:    add sp, sp, #1072
+; CHECK1024-NEXT:    .seh_stackalloc 1072
 ; CHECK1024-NEXT:    .seh_endepilogue
 ; CHECK1024-NEXT:    ret
 ; CHECK1024-NEXT:    .seh_endfunclet
diff --git a/llvm/test/CodeGen/AArch64/sve-fixed-length-subvector.ll b/llvm/test/CodeGen/AArch64/sve-fixed-length-subvector.ll
index aef19d23109b4..3e6a7ce34a9ae 100644
--- a/llvm/test/CodeGen/AArch64/sve-fixed-length-subvector.ll
+++ b/llvm/test/CodeGen/AArch64/sve-fixed-length-subvector.ll
@@ -48,10 +48,10 @@ define void @subvector_v32i16(ptr %in, ptr %out) #0 {
 ; VBITS_GE_256:       // %bb.0:
 ; VBITS_GE_256-NEXT:    ptrue p0.h, vl16
 ; VBITS_GE_256-NEXT:    mov x8, #16 // =0x10
-; VBITS_GE_256-NEXT:    ld1h { z0.h }, p0/z, [x0, x8, lsl #1]
-; VBITS_GE_256-NEXT:    ld1h { z1.h }, p0/z, [x0]
-; VBITS_GE_256-NEXT:    st1h { z0.h }, p0, [x1, x8, lsl #1]
-; VBITS_GE_256-NEXT:    st1h { z1.h }, p0, [x1]
+; VBITS_GE_256-NEXT:    ld1h { z1.h }, p0/z, [x0, x8, lsl #1]
+; VBITS_GE_256-NEXT:    ld1h { z0.h }, p0/z, [x0]
+; VBITS_GE_256-NEXT:    st1h { z1.h }, p0, [x1, x8, lsl #1]
+; VBITS_GE_256-NEXT:    st1h { z0.h }, p0, [x1]
 ; VBITS_GE_256-NEXT:    ret
 ;
 ; VBITS_GE_512-LABEL: subvector_v32i16:
@@ -103,10 +103,10 @@ define void @subvector_v16i32(ptr %in, ptr %out) #0 {
 ; VBITS_GE_256:       // %bb.0:
 ; VBITS_GE_256-NEXT:    ptrue p0.s, vl8
 ; VBITS_GE_256-NEXT:    mov x8, #8 // =0x8
-; VBITS_GE_256-NEXT:    ld1w { z0.s }, p0/z, [x0, x8, lsl #2]
-; VBITS_GE_256-NEXT:    ld1w { z1.s }, p0/z, [x0]
-; VBITS_GE_256-NEXT:    st1w { z0.s }, p0, [x1, x8, lsl #2]
-; VBITS_GE_256-NEXT:    st1w { z1.s }, p0, [x1]
+; VBITS_GE_256-NEXT:    ld1w { z1.s }, p0/z, [x0, x8, lsl #2]
+; VBITS_GE_256-NEXT:    ld1w { z0.s }, p0/z, [x0]
+; VBITS_GE_256-NEXT:    st1w { z1.s }, p0, [x1, x8, lsl #2]
+; VBITS_GE_256-NEXT:    st1w { z0.s }, p0, [x1]
 ; VBITS_GE_256-NEXT:    ret
 ;
 ; VBITS_GE_512-LABEL: subvector_v16i32:
@@ -159,10 +159,10 @@ define void @subvector_v8i64(ptr %in, ptr %out) vscale_range(2,0) #0 {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ptrue p0.d, vl4
 ; CHECK-NEXT:    mov x8, #4 // =0x4
-; CHECK-NEXT:    ld1d { z0.d }, p0/z, [x0, x8, lsl #3]
-; CHECK-NEXT:    ld1d { z1.d }, p0/z, [x0]
-; CHECK-NEXT:    st1d { z0.d }, p0, [x1, x8, lsl #3]
-; CHECK-NEXT:    st1d { z1.d }, p0, [x1]
+; CHECK-NEXT:    ld1d { z1.d }, p0/z, [x0, x8, lsl #3]
+; CHECK-NEXT:    ld1d { z0.d }, p0/z, [x0]
+; CHECK-NEXT:    st1d { z1.d }, p0, [x1, x8, lsl #3]
+; CHECK-NEXT:    st1d { z0.d }, p0, [x1]
 ; CHECK-NEXT:    ret
   %a = load <8 x i64>, ptr %in
   br label %bb1
@@ -236,10 +236,10 @@ define void @subvector_v32f16(ptr %in, ptr %out) #0 {
 ; VBITS_GE_256:       // %bb.0:
 ; VBITS_GE_256-NEXT:    ptrue p0.h, vl16
 ; VBITS_GE_256-NEXT:    mov x8, #16 // =0x10
-; VBITS_GE_256-NEXT:    ld1h { z0.h }, p0/z, [x0, x8, lsl #1]
-; VBITS_GE_256-NEXT:    ld1h { z1.h }, p0/z, [x0]
-; VBITS_GE_256-NEXT:    st1h { z0.h }, p0, [x1, x8, lsl #1]
-; VBITS_GE_256-NEXT:    st1h { z1.h }, p0, [x1]
+; VBITS_GE_256-NEXT:    ld1h { z1.h }, p0/z, [x0, x8, lsl #1]
+; VBITS_GE_256-NEXT:    ld1h { z0.h }, p0/z, [x0]
+; VBITS_GE_256-NEXT:    st1h { z1.h }, p0, [x1, x8, lsl #1]
+; VBITS_GE_256-NEXT:    st1h { z0.h }, p0, [x1]
 ; VBITS_GE_256-NEXT:    ret
 ;
 ; VBITS_GE_512-LABEL: subvector_v32f16:
@@ -291,10 +291,10 @@ define void @subvector_v16f32(ptr %in, ptr %out) #0 {
 ; VBITS_GE_256:       // %bb.0:
 ; VBITS_GE_256-NEXT:    ptrue p0.s, vl8
 ; VBITS_GE_256-NEXT:    mov x8, #8 // =0x8
-; VBITS_GE_256-NEXT:    ld1w { z0.s }, p0/z, [x0, x8, lsl #2]
-; VBITS_GE_256-NEXT:    ld1w { z1.s }, p0/z, [x0]
-; VBITS_GE_256-NEXT:    st1w { z0.s }, p0, [x1, x8, lsl #2]
-; VBITS_GE_256-NEXT:    st1w { z1.s }, p0, [x1]
+; VBITS_GE_256-NEXT:    ld1w { z1.s }, p0/z, [x0, x8, lsl #2]
+; VBITS_GE_256-NEXT:    ld1w { z0.s }, p0/z, [x0]
+; VBITS_GE_256-NEXT:    st1w { z1.s }, p0, [x1, x8, lsl #2]
+; VBITS_GE_256-NEXT:    st1w { z0.s }, p0, [x1]
 ; VBITS_GE_256-NEXT:    ret
 ;
 ; VBITS_GE_512-LABEL: subvector_v16f32:
@@ -345,10 +345,10 @@ define void @subvector_v8f64(ptr %in, ptr %out) #0 {
 ; VBITS_GE_256:       // %bb.0:
 ; VBITS_GE_256-NEXT:    ptrue p0.d, vl4
 ; VBITS_GE_256-NEXT:    mov x8, #4 // =0x4
-; VBITS_GE_256-NEXT:    ld1d { z0.d }, p0/z, [x0, x8, lsl #3]
-; VBITS_GE_256-NEXT:    ld1d { z1.d }, p0/z, [x0]
-; VBITS_GE_256-NEXT:    st1d { z0.d }, p0, [x1, x8, lsl #3]
-; VBITS_GE_256-NEXT:    st1d { z1.d }, p0, [x1]
+; VBITS_GE_256-NEXT:    ld1d { z1.d }, p0/z, [x0, x8, lsl #3]
+; VBITS_GE_256-NEXT:    ld1d { z0.d }, p0/z, [x0]
+; VBITS_GE_256-NEXT:    st1d { z1.d }, p0, [x1, x8, lsl #3]
+; VBITS_GE_256-NEXT:    st1d { z0.d }, p0, [x1]
 ; VBITS_GE_256-NEXT:    ret
 ;
 ; VBITS_GE_512-LABEL: subvector_v8f64:
diff --git a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-fp-to-int.ll b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-fp-to-int.ll
index b61c30af37994..94d756a36ab92 100644
--- a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-fp-to-int.ll
+++ b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-fp-to-int.ll
@@ -1335,50 +1335,50 @@ define <8 x i16> @fcvtzu_v8f64_v8i16(ptr %a) {
 define void @fcvtzu_v16f64_v16i16(ptr %a, ptr %b) {
 ; CHECK-LABEL: fcvtzu_v16f64_v16i16:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    ldp q5, q6, [x0, #96]
+; CHECK-NEXT:    ldp q0, q1, [x0, #96]
 ; CHECK-NEXT:    ptrue p0.d, vl2
-; CHECK-NEXT:    ldp q0, q4, [x0, #32]
-; CHECK-NEXT:    ldp q2, q7, [x0, #64]
-; CHECK-NEXT:    ldp q1, q3, [x0]
-; CHECK-NEXT:    fcvtzs z6.d, p0/m, z6.d
-; CHECK-NEXT:    fcvtzs z4.d, p0/m, z4.d
-; CHECK-NEXT:    fcvtzs z5.d, p0/m, z5.d
+; CHECK-NEXT:    ldp q2, q3, [x0, #32]
+; CHECK-NEXT:    ldp q4, q5, [x0, #64]
+; CHECK-NEXT:    ldp q6, q7, [x0]
+; CHECK-NEXT:    fcvtzs z1.d, p0/m, z1.d
+; CHECK-NEXT:    fcvtzs z3.d, p0/m, z3.d
 ; CHECK-NEXT:    fcvtzs z0.d, p0/m, z0.d
-; CHECK-NEXT:    fcvtzs z7.d, p0/m, z7.d
 ; CHECK-NEXT:    fcvtzs z2.d, p0/m, z2.d
-; CHECK-NEXT:    fcvtzs z3.d, p0/m, z3.d
-; CHECK-NEXT:    fcvtzs z1.d, p0/m, z1.d
-; CHECK-NEXT:    uzp1 z6.s, z6.s, z6.s
-; CHECK-NEXT:    uzp1 z4.s, z4.s, z4.s
-; CHECK-NEXT:    uzp1 z5.s, z5.s, z5.s
+; CHECK-NEXT:    fcvtzs z5.d, p0/m, z5.d
+; CHECK-NEXT:    fcvtzs z4.d, p0/m, z4.d
+; CHECK-NEXT:    fcvtzs z7.d, p0/m, z7.d
+; CHECK-NEXT:    fcvtzs z6.d, p0/m, z6.d
+; CHECK-NEXT:    uzp1 z1.s, z1.s, z1.s
+; CHECK-NEXT:    uzp1 z3.s, z3.s, z3.s
 ; CHECK-NEXT:    uzp1 z0.s, z0.s, z0.s
-; CHECK-NEXT:    uzp1 z7.s, z7.s, z7.s
 ; CHECK-NEXT:    uzp1 z2.s, z2.s, z2.s
-; CHECK-NEXT:    uzp1 z3.s, z3.s, z3.s
-; CHECK-NEXT:    uzp1 z1.s, z1.s, z1.s
-; CHECK-NEXT:    mov z17.s, z6.s[1]
-; CHECK-NEXT:    mov z16.s, z4.s[1]
-; CHECK-NEXT:    mov z18.s, z5.s[1]
-; CHECK-NEXT:    mov z21.s, z0.s[1]
-; CHECK-NEXT:    mov z19.s, z7.s[1]
-; CHECK-NEXT:    mov z20.s, z2.s[1]
-; CHECK-NEXT:    mov z22.s, z3.s[1]
-; CHECK-NEXT:    mov z23.s, z1.s[1]
-; CHECK-NEXT:    zip1 z6.h, z6.h, z17.h
-; CHECK-NEXT:    zip1 z4.h, z4.h, z16.h
-; CHECK-NEXT:    zip1 z5.h, z5.h, z18.h
-; CHECK-NEXT:    zip1 z0.h, z0.h, z21.h
-; CHECK-NEXT:    zip1 z7.h, z7.h, z19.h
-; CHECK-NEXT:    zip1 z2.h, z2.h, z20.h
-; CHECK-NEXT:    zip1 z3.h, z3.h, z22.h
-; CHECK-NEXT:    zip1 z1.h, z1.h, z23.h
-; CHECK-NEXT:    zip1 z5.s, z5.s, z6.s
-; CHECK-NEXT:    zip1 z0.s, z0.s, z4.s
-; CHECK-NEXT:    zip1 z2.s, z2.s, z7.s
-; CHECK-NEXT:    zip1 z1.s, z1.s, z3.s
-; CHECK-NEXT:    zip1 z2.d, z2.d, z5.d
+; CHECK-NEXT:    uzp1 z5.s, z5.s, z5.s
+; CHECK-NEXT:    uzp1 z4.s, z4.s, z4.s
+; CHECK-NEXT:    uzp1 z7.s, z7.s, z7.s
+; CHECK-NEXT:    uzp1 z6.s, z6.s, z6.s
+; CHECK-NEXT:    mov z17.s, z1.s[1]
+; CHECK-NEXT:    mov z16.s, z3.s[1]
+; CHECK-NEXT:    mov z18.s, z0.s[1]
+; CHECK-NEXT:    mov z21.s, z2.s[1]
+; CHECK-NEXT:    mov z19.s, z5.s[1]
+; CHECK-NEXT:    mov z20.s, z4.s[1]
+; CHECK-NEXT:    mov z22.s, z7.s[1]
+; CHECK-NEXT:    mov z23.s, z6.s[1]
+; CHECK-NEXT:    zip1 z1.h, z1.h, z17.h
+; CHECK-NEXT:    zip1 z3.h, z3.h, z16.h
+; CHECK-NEXT:    zip1 z0.h, z0.h, z18.h
+; CHECK-NEXT:    zip1 z2.h, z2.h, z21.h
+; CHECK-NEXT:    zip1 z5.h, z5.h, z19.h
+; CHECK-NEXT:    zip1 z4.h, z4.h, z20.h
+; CHECK-NEXT:    zip1 z7.h, z7.h, z22.h
+; CHECK-NEXT:    zip1 z6.h, z6.h, z23.h
+; CHECK-NEXT:    zip1 z0.s, z0.s, z1.s
+; CHECK-NEXT:    zip1 z2.s, z2.s, z3.s
+; CHECK-NEXT:    zip1 z1.s, z4.s, z5.s
+; CHECK-NEXT:    zip1 z3.s, z6.s, z7.s
 ; CHECK-NEXT:    zip1 z0.d, z1.d, z0.d
-; CHECK-NEXT:    stp q0, q2, [x1]
+; CHECK-NEXT:    zip1 z1.d, z3.d, z2.d
+; CHECK-NEXT:    stp q1, q0, [x1]
 ; CHECK-NEXT:    ret
 ;
 ; NONEON-NOSVE-LABEL: fcvtzu_v16f64_v16i16:
@@ -3033,50 +3033,50 @@ define <8 x i16> @fcvtzs_v8f64_v8i16(ptr %a) {
 define void @fcvtzs_v16f64_v16i16(ptr %a, ptr %b) {
 ; CHECK-LABEL: fcvtzs_v16f64_v16i16:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    ldp q5, q6, [x0, #96]
+; CHECK-NEXT:    ldp q0, q1, [x0, #96]
 ; CHECK-NEXT:    ptrue p0.d, vl2
-; CHECK-NEXT:    ldp q0, q4, [x0, #32]
-; CHECK-NEXT:    ldp q2, q7, [x0, #64]
-; CHECK-NEXT:    ldp q1, q3, [x0]
-; CHECK-NEXT:    fcvtzs z6.d, p0/m, z6.d
-; CHECK-NEXT:    fcvtzs z4.d, p0/m, z4.d
-; CHECK-NEXT:    fcvtzs z5.d, p0/m, z5.d
+; CHECK-NEXT:    ldp q2, q3, [x0, #32]
+; CHECK-NEXT:    ldp q4, q5, [x0, #64]
+; CHECK-NEXT:    ldp q6, q7, [x0]
+; CHECK-NEXT:    fcvtzs z1.d, p0/m, z1.d
+; CHECK-NEXT:    fcvtzs z3.d, p0/m, z3.d
 ; CHECK-NEXT:    fcvtzs z0.d, p0/m, z0.d
-; CHECK-NEXT:    fcvtzs z7.d, p0/m, z7.d
 ; CHECK-NEXT:    fcvtzs z2.d, p0/m, z2.d
-; CHECK-NEXT:    fcvtzs z3.d, p0/m, z3.d
-; CHECK-NEXT:    fcvtzs z1.d, p0/m, z1.d
-; CHECK-NEXT:    uzp1 z6.s, z6.s, z6.s
-; CHECK-NEXT:    uzp1 z4.s, z4.s, z4.s
-; CHECK-NEXT:    uzp1 z5.s, z5.s, z5.s
+; CHECK-NEXT:    fcvtzs z5.d, p0/m, z5.d
+; CHECK-NEXT:    fcvtzs z4.d, p0/m, z4.d
+; CHECK-NEXT:    fcvtzs z7.d, p0/m, z7.d
+; CHECK-NEXT:    fcvtzs z6.d, p0/m, z6.d
+; CHECK-NEXT:    uzp1 z1.s, z1.s, z1.s
+; CHECK-NEXT:    uzp1 z3.s, z3.s, z3.s
 ; CHECK-NEXT:    uzp1 z0.s, z0.s, z0.s
-; CHECK-NEXT:    uzp1 z7.s, z7.s, z7.s
 ; CHECK-NEXT:    uzp1 z2.s, z2.s, z2.s
-; CHECK-NEXT:    uzp1 z3.s, z3.s, z3.s
-; CHECK-NEXT:    uzp1 z1.s, z1.s, z1.s
-; CHECK-NEXT:    mov z17.s, z6.s[1]
-; CHECK-NEXT:    mov z16.s, z4.s[1]
-; CHECK-NEXT:    mov z18.s, z5.s[1]
-; CHECK-NEXT:    mov z21.s, z0.s[1]
-; CHECK-NEXT:    mov z19.s, z7.s[1]
-; CHECK-NEXT:    mov z20.s, z2.s[1]
-; CHECK-NEXT:    mov z22.s, z3.s[1]
-; CHECK-NEXT:    mov z23.s, z1.s[1]
-; CHECK-NEXT:    zip1 z6.h, z6.h, z17.h
-; CHECK-NEXT:    zip1 z4.h, z4.h, z16.h
-; CHECK-NEXT:    zip1 z5.h, z5.h, z18.h
-; CHECK-NEXT:    zip1 z0.h, z0.h, z21.h
-; CHECK-NEXT:    zip1 z7.h, z7.h, z19.h
-; CHECK-NEXT:    zip1 z2.h, z2.h, z20.h
-; CHECK-NEXT:    zip1 z3.h, z3.h, z22.h
-; CHECK-NEXT:    zip1 z1.h, z1.h, z23.h
-; CHECK-NEXT:    zip1 z5.s, z5.s, z6.s
-; CHECK-NEXT:    zip1 z0.s, z0.s, z4.s
-; CHECK-NEXT:    zip1 z2.s, z2.s, z7.s
-; CHECK-NEXT:    zip1 z1.s, z1.s, z3.s
-; CHECK-NEXT:    zip1 z2.d, z2.d, z5.d
+; CHECK-NEXT:    uzp1 z5.s, z5.s, z5.s
+; CHECK-NEXT:    uzp1 z4.s, z4.s, z4.s
+; CHECK-NEXT:    uzp1 z7.s, z7.s, z7.s
+; CHECK-NEXT:    uzp1 z6.s, z6.s, z6.s
+; CHECK-NEXT:    mov z17.s, z1.s[1]
+; CHECK-NEXT:    mov z16.s, z3.s[1]
+; CHECK-NEXT:    mov z18.s, z0.s[1]
+; CHECK-NEXT:    mov z21.s, z2.s[1]
+; CHECK-NEXT:    mov z19.s, z5.s[1]
+; CHECK-NEXT:    mov z20.s, z4.s[1]
+; CHECK-NEXT:    mov z22.s, z7.s[1]
+; CHECK-NEXT:    mov z23.s, z6.s[1]
+; CHECK-NEXT:    zip1 z1.h, z1.h, z17.h
+; CHECK-NEXT:    zip1 z3.h, z3.h, z16.h
+; CHECK-NEXT:    zip1 z0.h, z0.h, z18.h
+; CHECK-NEXT:    zip1 z2.h, z2.h, z21.h
+; CHECK-NEXT:    zip1 z5.h, z5.h, z19.h
+; CHECK-NEXT:    zip1 z4.h, z4.h, z20.h
+; CHECK-NEXT:    zip1 z7.h, z7.h, z22.h
+; CHECK-NEXT:    zip1 z6.h, z6.h, z23.h
+; CHECK-NEXT:    zip1 z0.s, z0.s, z1.s
+; CHECK-NEXT:    zip1 z2.s, z2.s, z3.s
+; CHECK-NEXT:    zip1 z1.s, z4.s, z5.s
+; CHECK-NEXT:    zip1 z3.s, z6.s, z7.s
 ; CHECK-NEXT:    zip1 z0.d, z1.d, z0.d
-; CHECK-NEXT:    stp q0, q2, [x1]
+; CHECK-NEXT:    zip1 z1.d, z3.d, z2.d
+; CHECK-NEXT:    stp q1, q0, [x1]
 ; CHECK-NEXT:    ret
 ;
 ; NONEON-NOSVE-LABEL: fcvtzs_v16f64_v16i16:
diff --git a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-permute-zip-uzp-trn.ll b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-permute-zip-uzp-trn.ll
index 8b296d9fbc215..168ad6a77badb 100644
--- a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-permute-zip-uzp-trn.ll
+++ b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-permute-zip-uzp-trn.ll
@@ -151,20 +151,20 @@ define void @zip_v32i16(ptr %a, ptr %b) {
 ; CHECK-NEXT:    .cfi_offset b13, -48
 ; CHECK-NEXT:    .cfi_offset b14, -56
 ; CHECK-NEXT:    .cfi_offset b15, -64
-; CHECK-NEXT:    ldp q0, q1, [x0]
+; CHECK-NEXT:    ldp q1, q0, [x0]
 ; CHECK-NEXT:    ldp q2, q3, [x1]
-; CHECK-NEXT:    mov z5.h, z1.h[7]
-; CHECK-NEXT:    mov z7.h, z1.h[6]
-; CHECK-NEXT:    mov z17.h, z1.h[5]
+; CHECK-NEXT:    mov z5.h, z0.h[7]
+; CHECK-NEXT:    mov z7.h, z0.h[6]
+; CHECK-NEXT:    mov z17.h, z0.h[5]
 ; CHECK-NEXT:    mov z4.h, z3.h[7]
 ; CHECK-NEXT:    mov z6.h, z3.h[6]
 ; CHECK-NEXT:    mov z16.h, z3.h[5]
 ; CHECK-NEXT:    mov z20.h, z2.h[7]
-; CHECK-NEXT:    mov z21.h, z0.h[7]
+; CHECK-NEXT:    mov z21.h, z1.h[7]
 ; CHECK-NEXT:    mov z18.h, z3.h[4]
-; CHECK-NEXT:    mov z19.h, z1.h[4]
+; CHECK-NEXT:    mov z19.h, z0.h[4]
 ; CHECK-NEXT:    mov z22.h, z2.h[6]
-; CHECK-NEXT:    mov z23.h, z0.h[6]
+; CHECK-NEXT:    mov z23.h, z1.h[6]
 ; CHECK-NEXT:    zip1 z24.h, z5.h, z4.h
 ; CHECK-NEXT:    zip1 z25.h, z7.h, z6.h
 ; CHECK-NEXT:    zip1 z17.h, z17.h, z16.h
@@ -176,10 +176,10 @@ define void @zip_v32i16(ptr %a, ptr %b) {
 ; CHECK-NEXT:    zip1 z22.h, z23.h, z22.h
 ; CHECK-NEXT:    mov z23.h, z2.h[5]
 ; CHECK-NEXT:    mov z21.h, z6.h[7]
-; CHECK-NEXT:    mov z24.h, z0.h[5]
+; CHECK-NEXT:    mov z24.h, z1.h[5]
 ; CHECK-NEXT:    mov z25.h, z2.h[4]
 ; CHECK-NEXT:    mov z20.h, z7.h[7]
-; CHECK-NEXT:    mov z26.h, z0.h[4]
+; CHECK-NEXT:    mov z26.h, z1.h[4]
 ; CHECK-NEXT:    mov z27.h, z6.h[6]
 ; CHECK-NEXT:    mov z28.h, z7.h[5]
 ; CHECK-NEXT:    mov z29.h, z6.h[5]
@@ -212,22 +212,22 @@ define void @zip_v32i16(ptr %a, ptr %b) {
 ; CHECK-NEXT:    zip1 z20.s, z28.s, z27.s
 ; CHECK-NEXT:    zip1 z16.s, z22.s, z16.s
 ; CHECK-NEXT:    zip1 z21.s, z24.s, z23.s
-; CHECK-NEXT:    zip1 z1.h, z1.h, z3.h
+; CHECK-NEXT:    zip1 z0.h, z0.h, z3.h
 ; CHECK-NEXT:    zip1 z3.s, z26.s, z25.s
 ; CHECK-NEXT:    zip1 z22.s, z30.s, z29.s
 ; CHECK-NEXT:    zip1 z6.h, z6.h, z7.h
 ; CHECK-NEXT:    zip1 z7.d, z17.d, z19.d
 ; CHECK-NEXT:    zip1 z17.d, z20.d, z18.d
-; CHECK-NEXT:    zip1 z0.h, z0.h, z2.h
+; CHECK-NEXT:    zip1 z1.h, z1.h, z2.h
 ; CHECK-NEXT:    zip1 z2.h, z4.h, z5.h
 ; CHECK-NEXT:    zip1 z4.d, z21.d, z16.d
 ; CHECK-NEXT:    zip1 z3.d, z22.d, z3.d
-; CHECK-NEXT:    add z1.h, z1.h, z6.h
+; CHECK-NEXT:    add z0.h, z0.h, z6.h
 ; CHECK-NEXT:    add z5.h, z7.h, z17.h
-; CHECK-NEXT:    add z0.h, z0.h, z2.h
+; CHECK-NEXT:    add z1.h, z1.h, z2.h
 ; CHECK-NEXT:    add z2.h, z4.h, z3.h
-; CHECK-NEXT:    stp q1, q5, [x0, #32]
-; CHECK-NEXT:    stp q0, q2, [x0]
+; CHECK-NEXT:    stp q0, q5, [x0, #32]
+; CHECK-NEXT:    stp q1, q2, [x0]
 ; CHECK-NEXT:    ldp d15, d14, [sp], #64 // 16-byte Folded Reload
 ; CHECK-NEXT:    ret
 ;
diff --git a/llvm/test/CodeGen/AArch64/sve2-vscale-sinking.ll b/llvm/test/CodeGen/AArch64/sve2-vscale-sinking.ll
index 27cffc1f4c6f4..790da46cd831f 100644
--- a/llvm/test/CodeGen/AArch64/sve2-vscale-sinking.ll
+++ b/llvm/test/CodeGen/AArch64/sve2-vscale-sinking.ll
@@ -109,7 +109,7 @@ for.cond.cleanup:
 
 define void @gep(i32 noundef %first, i32 noundef %N, ptr nocapture noundef writeonly %ptr, <vscale x 16 x i1> %pg, <vscale x 16 x i8> %val) #0 {
 ; CHECK-LABEL: define void @gep
-; CHECK-SAME: (i32 noundef [[FIRST:%.*]], i32 noundef [[N:%.*]], ptr nocapture noundef writeonly [[PTR:%.*]], <vscale x 16 x i1> [[PG:%.*]], <vscale x 16 x i8> [[VAL:%.*]]) #[[ATTR0]] {
+; CHECK-SAME: (i32 noundef [[FIRST:%.*]], i32 noundef [[N:%.*]], ptr noundef writeonly captures(none) [[PTR:%.*]], <vscale x 16 x i1> [[PG:%.*]], <vscale x 16 x i8> [[VAL:%.*]]) #[[ATTR0]] {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK:       for.body:
@@ -272,7 +272,7 @@ for.cond.cleanup:
 
 define void @gep_i32(i32 noundef %first, i32 noundef %N, ptr nocapture noundef writeonly %ptr, <vscale x 16 x i1> %pg, <vscale x 16 x i8> %val) #0 {
 ; CHECK-LABEL: define void @gep_i32
-; CHECK-SAME: (i32 noundef [[FIRST:%.*]], i32 noundef [[N:%.*]], ptr nocapture noundef writeonly [[PTR:%.*]], <vscale x 16 x i1> [[PG:%.*]], <vscale x 16 x i8> [[VAL:%.*]]) #[[ATTR0]] {
+; CHECK-SAME: (i32 noundef [[FIRST:%.*]], i32 noundef [[N:%.*]], ptr noundef writeonly captures(none) [[PTR:%.*]], <vscale x 16 x i1> [[PG:%.*]], <vscale x 16 x i8> [[VAL:%.*]]) #[[ATTR0]] {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK:       for.body:
diff --git a/llvm/test/CodeGen/AArch64/umul_fix.ll b/llvm/test/CodeGen/AArch64/umul_fix.ll
index 9f4da82dd74b6..823f4bdecbad6 100644
--- a/llvm/test/CodeGen/AArch64/umul_fix.ll
+++ b/llvm/test/CodeGen/AArch64/umul_fix.ll
@@ -152,8 +152,7 @@ define <4 x i16> @widemul(<4 x i16> %x, <4 x i16> %y) nounwind {
 ; CHECK-NEXT:    umull v0.4s, v0.4h, v1.4h
 ; CHECK-NEXT:    shrn v1.4h, v0.4s, #16
 ; CHECK-NEXT:    xtn v2.4h, v0.4s
-; CHECK-NEXT:    add v1.4h, v1.4h, v1.4h
-; CHECK-NEXT:    shl v0.4h, v1.4h, #11
+; CHECK-NEXT:    shl v0.4h, v1.4h, #12
 ; CHECK-NEXT:    usra v0.4h, v2.4h, #4
 ; CHECK-NEXT:    ret
   %tmp = call <4 x i16> @llvm.umul.fix.v4i16(<4 x i16> %x, <4 x i16> %y, i32 4)
diff --git a/llvm/test/CodeGen/AArch64/v8.4-atomic.ll b/llvm/test/CodeGen/AArch64/v8.4-atomic.ll
new file mode 100644
index 0000000000000..1394b89159c41
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/v8.4-atomic.ll
@@ -0,0 +1,113 @@
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+lse2 %s -o - | FileCheck %s
+
+define dso_local half @load_atomic_f16_aligned_unordered(ptr %ptr) {
+; CHECK-LABEL: load_atomic_f16_aligned_unordered:
+; CHECK:    ldrh w8, [x0]
+    %r = load atomic half, ptr %ptr unordered, align 2
+    ret half %r
+}
+
+define dso_local half @load_atomic_f16_aligned_unordered_const(ptr readonly %ptr) {
+; CHECK-LABEL: load_atomic_f16_aligned_unordered_const:
+; CHECK:    ldrh w8, [x0]
+    %r = load atomic half, ptr %ptr unordered, align 2
+    ret half %r
+}
+
+define dso_local half @load_atomic_f16_aligned_monotonic(ptr %ptr) {
+; CHECK-LABEL: load_atomic_f16_aligned_monotonic:
+; CHECK:    ldrh w8, [x0]
+    %r = load atomic half, ptr %ptr monotonic, align 2
+    ret half %r
+}
+
+define dso_local half @load_atomic_f16_aligned_monotonic_const(ptr readonly %ptr) {
+; CHECK-LABEL: load_atomic_f16_aligned_monotonic_const:
+; CHECK:    ldrh w8, [x0]
+    %r = load atomic half, ptr %ptr monotonic, align 2
+    ret half %r
+}
+
+define dso_local half @load_atomic_f16_aligned_acquire(ptr %ptr) {
+; CHECK-LABEL: load_atomic_f16_aligned_acquire:
+; CHECK:    ldarh w8, [x0]
+    %r = load atomic half, ptr %ptr acquire, align 2
+    ret half %r
+}
+
+define dso_local half @load_atomic_f16_aligned_acquire_const(ptr readonly %ptr) {
+; CHECK-LABEL: load_atomic_f16_aligned_acquire_const:
+; CHECK:    ldarh w8, [x0]
+    %r = load atomic half, ptr %ptr acquire, align 2
+    ret half %r
+}
+
+define dso_local half @load_atomic_f16_aligned_seq_cst(ptr %ptr) {
+; CHECK-LABEL: load_atomic_f16_aligned_seq_cst:
+; CHECK:    ldarh w8, [x0]
+    %r = load atomic half, ptr %ptr seq_cst, align 2
+    ret half %r
+}
+
+define dso_local half @load_atomic_f16_aligned_seq_cst_const(ptr readonly %ptr) {
+; CHECK-LABEL: load_atomic_f16_aligned_seq_cst_const:
+; CHECK:    ldarh w8, [x0]
+    %r = load atomic half, ptr %ptr seq_cst, align 2
+    ret half %r
+}
+
+define dso_local bfloat @load_atomic_bf16_aligned_unordered(ptr %ptr) {
+; CHECK-LABEL: load_atomic_bf16_aligned_unordered:
+; CHECK:    ldrh w8, [x0]
+    %r = load atomic bfloat, ptr %ptr unordered, align 2
+    ret bfloat %r
+}
+
+define dso_local bfloat @load_atomic_bf16_aligned_unordered_const(ptr readonly %ptr) {
+; CHECK-LABEL: load_atomic_bf16_aligned_unordered_const:
+; CHECK:    ldrh w8, [x0]
+    %r = load atomic bfloat, ptr %ptr unordered, align 2
+    ret bfloat %r
+}
+
+define dso_local bfloat @load_atomic_bf16_aligned_monotonic(ptr %ptr) {
+; CHECK-LABEL: load_atomic_bf16_aligned_monotonic:
+; CHECK:    ldrh w8, [x0]
+    %r = load atomic bfloat, ptr %ptr monotonic, align 2
+    ret bfloat %r
+}
+
+define dso_local bfloat @load_atomic_bf16_aligned_monotonic_const(ptr readonly %ptr) {
+; CHECK-LABEL: load_atomic_bf16_aligned_monotonic_const:
+; CHECK:    ldrh w8, [x0]
+    %r = load atomic bfloat, ptr %ptr monotonic, align 2
+    ret bfloat %r
+}
+
+define dso_local bfloat @load_atomic_bf16_aligned_acquire(ptr %ptr) {
+; CHECK-LABEL: load_atomic_bf16_aligned_acquire:
+; CHECK:    ldarh w8, [x0]
+    %r = load atomic bfloat, ptr %ptr acquire, align 2
+    ret bfloat %r
+}
+
+define dso_local bfloat @load_atomic_bf16_aligned_acquire_const(ptr readonly %ptr) {
+; CHECK-LABEL: load_atomic_bf16_aligned_acquire_const:
+; CHECK:    ldarh w8, [x0]
+    %r = load atomic bfloat, ptr %ptr acquire, align 2
+    ret bfloat %r
+}
+
+define dso_local bfloat @load_atomic_bf16_aligned_seq_cst(ptr %ptr) {
+; CHECK-LABEL: load_atomic_bf16_aligned_seq_cst:
+; CHECK:    ldarh w8, [x0]
+    %r = load atomic bfloat, ptr %ptr seq_cst, align 2
+    ret bfloat %r
+}
+
+define dso_local bfloat @load_atomic_bf16_aligned_seq_cst_const(ptr readonly %ptr) {
+; CHECK-LABEL: load_atomic_bf16_aligned_seq_cst_const:
+; CHECK:    ldarh w8, [x0]
+    %r = load atomic bfloat, ptr %ptr seq_cst, align 2
+    ret bfloat %r
+}
diff --git a/llvm/test/CodeGen/AArch64/vec-combine-compare-to-bitmask.ll b/llvm/test/CodeGen/AArch64/vec-combine-compare-to-bitmask.ll
index c9fe258f11556..b0a30b7150637 100644
--- a/llvm/test/CodeGen/AArch64/vec-combine-compare-to-bitmask.ll
+++ b/llvm/test/CodeGen/AArch64/vec-combine-compare-to-bitmask.ll
@@ -378,36 +378,36 @@ define i4 @convert_to_bitmask_with_unknown_type_in_long_chain(<4 x i32> %vec1, <
 ; CHECK-GI-NEXT:    .cfi_def_cfa_offset 16
 ; CHECK-GI-NEXT:    mov w8, #1 ; =0x1
 ; CHECK-GI-NEXT:    mov w9, #0 ; =0x0
-; CHECK-GI-NEXT:    cmeq.4s v5, v0, #0
+; CHECK-GI-NEXT:    cmeq.4s v0, v0, #0
 ; CHECK-GI-NEXT:    fmov s2, w8
 ; CHECK-GI-NEXT:    fmov s4, w9
 ; CHECK-GI-NEXT:    cmeq.4s v1, v1, #0
 ; CHECK-GI-NEXT:    mov.16b v3, v2
-; CHECK-GI-NEXT:    mov.16b v0, v4
+; CHECK-GI-NEXT:    mov.16b v5, v4
 ; CHECK-GI-NEXT:    mov.h v4[1], w8
-; CHECK-GI-NEXT:    bic.16b v1, v1, v5
-; CHECK-GI-NEXT:    mov.16b v5, v2
+; CHECK-GI-NEXT:    bic.16b v0, v1, v0
+; CHECK-GI-NEXT:    mov.16b v1, v2
 ; CHECK-GI-NEXT:    mov.h v2[1], w8
 ; CHECK-GI-NEXT:    mov.h v3[1], w8
-; CHECK-GI-NEXT:    mov.h v0[1], w8
 ; CHECK-GI-NEXT:    mov.h v5[1], w8
+; CHECK-GI-NEXT:    mov.h v1[1], w8
 ; CHECK-GI-NEXT:    mov.h v4[2], w8
-; CHECK-GI-NEXT:    xtn.4h v1, v1
+; CHECK-GI-NEXT:    xtn.4h v0, v0
 ; CHECK-GI-NEXT:    mov.h v2[2], w8
 ; CHECK-GI-NEXT:    mov.h v3[2], w9
-; CHECK-GI-NEXT:    mov.h v0[2], w9
 ; CHECK-GI-NEXT:    mov.h v5[2], w9
+; CHECK-GI-NEXT:    mov.h v1[2], w9
 ; CHECK-GI-NEXT:    mov.h v4[3], w9
 ; CHECK-GI-NEXT:    mov.h v2[3], w9
 ; CHECK-GI-NEXT:    mov.h v3[3], w9
-; CHECK-GI-NEXT:    mov.h v0[3], w8
 ; CHECK-GI-NEXT:    mov.h v5[3], w8
-; CHECK-GI-NEXT:    orr.8b v1, v1, v3
-; CHECK-GI-NEXT:    eor.8b v0, v1, v0
-; CHECK-GI-NEXT:    eor.8b v1, v4, v1
-; CHECK-GI-NEXT:    and.8b v0, v0, v5
-; CHECK-GI-NEXT:    orr.8b v1, v2, v1
-; CHECK-GI-NEXT:    orr.8b v0, v0, v1
+; CHECK-GI-NEXT:    mov.h v1[3], w8
+; CHECK-GI-NEXT:    orr.8b v0, v0, v3
+; CHECK-GI-NEXT:    eor.8b v3, v0, v5
+; CHECK-GI-NEXT:    eor.8b v0, v4, v0
+; CHECK-GI-NEXT:    and.8b v1, v3, v1
+; CHECK-GI-NEXT:    orr.8b v0, v2, v0
+; CHECK-GI-NEXT:    orr.8b v0, v1, v0
 ; CHECK-GI-NEXT:    ushll.4s v0, v0, #0
 ; CHECK-GI-NEXT:    mov.s w8, v0[1]
 ; CHECK-GI-NEXT:    mov.s w9, v0[2]
diff --git a/llvm/test/CodeGen/AArch64/veclib-llvm.modf.ll b/llvm/test/CodeGen/AArch64/veclib-llvm.modf.ll
new file mode 100644
index 0000000000000..78dbc09a57afd
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/veclib-llvm.modf.ll
@@ -0,0 +1,144 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc -mtriple=aarch64-gnu-linux -mattr=+neon,+sve -vector-library=sleefgnuabi < %s | FileCheck %s -check-prefix=SLEEF
+; RUN: llc -mtriple=aarch64-gnu-linux -mattr=+neon,+sve -vector-library=ArmPL < %s | FileCheck %s -check-prefix=ARMPL
+
+define <4 x float> @test_modf_v4f32(<4 x float> %x, ptr %out_integral) {
+; SLEEF-LABEL: test_modf_v4f32:
+; SLEEF:       // %bb.0:
+; SLEEF-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; SLEEF-NEXT:    .cfi_def_cfa_offset 16
+; SLEEF-NEXT:    .cfi_offset w30, -16
+; SLEEF-NEXT:    bl _ZGVnN4vl4_modff
+; SLEEF-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; SLEEF-NEXT:    ret
+;
+; ARMPL-LABEL: test_modf_v4f32:
+; ARMPL:       // %bb.0:
+; ARMPL-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; ARMPL-NEXT:    .cfi_def_cfa_offset 16
+; ARMPL-NEXT:    .cfi_offset w30, -16
+; ARMPL-NEXT:    bl armpl_vmodfq_f32
+; ARMPL-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; ARMPL-NEXT:    ret
+  %result = call { <4 x float>, <4 x float> } @llvm.modf.v4f32(<4 x float> %x)
+  %result.0 = extractvalue { <4 x float>, <4 x float> } %result, 0
+  %result.1 = extractvalue { <4 x float>, <4 x float> } %result, 1
+  store <4 x float> %result.1, ptr %out_integral, align 4
+  ret <4 x float> %result.0
+}
+
+define <2 x double> @test_modf_v2f64(<2 x double> %x, ptr %out_integral) {
+; SLEEF-LABEL: test_modf_v2f64:
+; SLEEF:       // %bb.0:
+; SLEEF-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; SLEEF-NEXT:    .cfi_def_cfa_offset 16
+; SLEEF-NEXT:    .cfi_offset w30, -16
+; SLEEF-NEXT:    bl _ZGVnN2vl8_modf
+; SLEEF-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; SLEEF-NEXT:    ret
+;
+; ARMPL-LABEL: test_modf_v2f64:
+; ARMPL:       // %bb.0:
+; ARMPL-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; ARMPL-NEXT:    .cfi_def_cfa_offset 16
+; ARMPL-NEXT:    .cfi_offset w30, -16
+; ARMPL-NEXT:    bl armpl_vmodfq_f64
+; ARMPL-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; ARMPL-NEXT:    ret
+  %result = call { <2 x double>, <2 x double> } @llvm.modf.v2f64(<2 x double> %x)
+  %result.0 = extractvalue { <2 x double>, <2 x double> } %result, 0
+  %result.1 = extractvalue { <2 x double>, <2 x double> } %result, 1
+  store <2 x double> %result.1, ptr %out_integral, align 8
+  ret <2 x double> %result.0
+}
+
+define <vscale x 4 x float> @test_modf_nxv4f32(<vscale x 4 x float> %x, ptr %out_integral) {
+; SLEEF-LABEL: test_modf_nxv4f32:
+; SLEEF:       // %bb.0:
+; SLEEF-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; SLEEF-NEXT:    .cfi_def_cfa_offset 16
+; SLEEF-NEXT:    .cfi_offset w30, -16
+; SLEEF-NEXT:    bl _ZGVsNxvl4_modff
+; SLEEF-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; SLEEF-NEXT:    ret
+;
+; ARMPL-LABEL: test_modf_nxv4f32:
+; ARMPL:       // %bb.0:
+; ARMPL-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; ARMPL-NEXT:    .cfi_def_cfa_offset 16
+; ARMPL-NEXT:    .cfi_offset w30, -16
+; ARMPL-NEXT:    ptrue p0.s
+; ARMPL-NEXT:    bl armpl_svmodf_f32_x
+; ARMPL-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; ARMPL-NEXT:    ret
+  %result = call { <vscale x 4 x float>, <vscale x 4 x float> } @llvm.modf.nxv4f32(<vscale x 4 x float> %x)
+  %result.0 = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } %result, 0
+  %result.1 = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } %result, 1
+  store <vscale x 4 x float> %result.1, ptr %out_integral, align 4
+  ret <vscale x 4 x float> %result.0
+}
+
+define <vscale x 2 x double> @test_modf_nxv2f64(<vscale x 2 x double> %x, ptr %out_integral) {
+; SLEEF-LABEL: test_modf_nxv2f64:
+; SLEEF:       // %bb.0:
+; SLEEF-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; SLEEF-NEXT:    .cfi_def_cfa_offset 16
+; SLEEF-NEXT:    .cfi_offset w30, -16
+; SLEEF-NEXT:    bl _ZGVsNxvl8_modf
+; SLEEF-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; SLEEF-NEXT:    ret
+;
+; ARMPL-LABEL: test_modf_nxv2f64:
+; ARMPL:       // %bb.0:
+; ARMPL-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; ARMPL-NEXT:    .cfi_def_cfa_offset 16
+; ARMPL-NEXT:    .cfi_offset w30, -16
+; ARMPL-NEXT:    ptrue p0.d
+; ARMPL-NEXT:    bl armpl_svmodf_f64_x
+; ARMPL-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; ARMPL-NEXT:    ret
+  %result = call { <vscale x 2 x double>, <vscale x 2 x double> } @llvm.modf.nxv2f64(<vscale x 2 x double> %x)
+  %result.0 = extractvalue { <vscale x 2 x double>, <vscale x 2 x double> } %result, 0
+  %result.1 = extractvalue { <vscale x 2 x double>, <vscale x 2 x double> } %result, 1
+  store <vscale x 2 x double> %result.1, ptr %out_integral, align 8
+  ret <vscale x 2 x double> %result.0
+}
+
+define <4 x float> @modf_store_merging_load_before_store(<4 x float> %x, ptr %out_integral) {
+; SLEEF-LABEL: modf_store_merging_load_before_store:
+; SLEEF:       // %bb.0:
+; SLEEF-NEXT:    sub sp, sp, #32
+; SLEEF-NEXT:    str x30, [sp, #16] // 8-byte Folded Spill
+; SLEEF-NEXT:    .cfi_def_cfa_offset 32
+; SLEEF-NEXT:    .cfi_offset w30, -16
+; SLEEF-NEXT:    ldr q1, [x0]
+; SLEEF-NEXT:    str q1, [sp] // 16-byte Folded Spill
+; SLEEF-NEXT:    bl _ZGVnN4vl4_modff
+; SLEEF-NEXT:    ldr q1, [sp] // 16-byte Folded Reload
+; SLEEF-NEXT:    ldr x30, [sp, #16] // 8-byte Folded Reload
+; SLEEF-NEXT:    fadd v0.4s, v1.4s, v0.4s
+; SLEEF-NEXT:    add sp, sp, #32
+; SLEEF-NEXT:    ret
+;
+; ARMPL-LABEL: modf_store_merging_load_before_store:
+; ARMPL:       // %bb.0:
+; ARMPL-NEXT:    sub sp, sp, #32
+; ARMPL-NEXT:    str x30, [sp, #16] // 8-byte Folded Spill
+; ARMPL-NEXT:    .cfi_def_cfa_offset 32
+; ARMPL-NEXT:    .cfi_offset w30, -16
+; ARMPL-NEXT:    ldr q1, [x0]
+; ARMPL-NEXT:    str q1, [sp] // 16-byte Folded Spill
+; ARMPL-NEXT:    bl armpl_vmodfq_f32
+; ARMPL-NEXT:    ldr q1, [sp] // 16-byte Folded Reload
+; ARMPL-NEXT:    ldr x30, [sp, #16] // 8-byte Folded Reload
+; ARMPL-NEXT:    fadd v0.4s, v1.4s, v0.4s
+; ARMPL-NEXT:    add sp, sp, #32
+; ARMPL-NEXT:    ret
+  %result = call { <4 x float>, <4 x float> } @llvm.modf.v4f32(<4 x float> %x)
+  %result.0 = extractvalue { <4 x float>, <4 x float> } %result, 0
+  %result.1 = extractvalue { <4 x float>, <4 x float> } %result, 1
+  %original_intergral = load <4 x float>, ptr %out_integral, align 4
+  store <4 x float> %result.1, ptr %out_integral, align 4
+  %return = fadd <4 x float> %original_intergral, %result.0
+  ret <4 x float> %return
+}
diff --git a/llvm/test/CodeGen/AArch64/veclib-llvm.sincospi.ll b/llvm/test/CodeGen/AArch64/veclib-llvm.sincospi.ll
new file mode 100644
index 0000000000000..fad865d20f7df
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/veclib-llvm.sincospi.ll
@@ -0,0 +1,109 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc -mtriple=aarch64-gnu-linux -mattr=+neon,+sve -vector-library=sleefgnuabi < %s | FileCheck %s -check-prefix=SLEEF
+; RUN: llc -mtriple=aarch64-gnu-linux -mattr=+neon,+sve -vector-library=ArmPL < %s | FileCheck %s -check-prefix=ARMPL
+
+define void @test_sincospi_v4f32(<4 x float> %x, ptr noalias %out_sin, ptr noalias %out_cos) {
+; SLEEF-LABEL: test_sincospi_v4f32:
+; SLEEF:       // %bb.0:
+; SLEEF-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; SLEEF-NEXT:    .cfi_def_cfa_offset 16
+; SLEEF-NEXT:    .cfi_offset w30, -16
+; SLEEF-NEXT:    bl _ZGVnN4vl4l4_sincospif
+; SLEEF-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; SLEEF-NEXT:    ret
+;
+; ARMPL-LABEL: test_sincospi_v4f32:
+; ARMPL:       // %bb.0:
+; ARMPL-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; ARMPL-NEXT:    .cfi_def_cfa_offset 16
+; ARMPL-NEXT:    .cfi_offset w30, -16
+; ARMPL-NEXT:    bl armpl_vsincospiq_f32
+; ARMPL-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; ARMPL-NEXT:    ret
+  %result = call { <4 x float>, <4 x float> } @llvm.sincospi.v4f32(<4 x float> %x)
+  %result.0 = extractvalue { <4 x float>, <4 x float> } %result, 0
+  %result.1 = extractvalue { <4 x float>, <4 x float> } %result, 1
+  store <4 x float> %result.0, ptr %out_sin, align 4
+  store <4 x float> %result.1, ptr %out_cos, align 4
+  ret void
+}
+
+define void @test_sincospi_v2f64(<2 x double> %x, ptr noalias %out_sin, ptr noalias %out_cos) {
+; SLEEF-LABEL: test_sincospi_v2f64:
+; SLEEF:       // %bb.0:
+; SLEEF-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; SLEEF-NEXT:    .cfi_def_cfa_offset 16
+; SLEEF-NEXT:    .cfi_offset w30, -16
+; SLEEF-NEXT:    bl _ZGVnN2vl8l8_sincospi
+; SLEEF-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; SLEEF-NEXT:    ret
+;
+; ARMPL-LABEL: test_sincospi_v2f64:
+; ARMPL:       // %bb.0:
+; ARMPL-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; ARMPL-NEXT:    .cfi_def_cfa_offset 16
+; ARMPL-NEXT:    .cfi_offset w30, -16
+; ARMPL-NEXT:    bl armpl_vsincospiq_f64
+; ARMPL-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; ARMPL-NEXT:    ret
+  %result = call { <2 x double>, <2 x double> } @llvm.sincospi.v2f64(<2 x double> %x)
+  %result.0 = extractvalue { <2 x double>, <2 x double> } %result, 0
+  %result.1 = extractvalue { <2 x double>, <2 x double> } %result, 1
+  store <2 x double> %result.0, ptr %out_sin, align 8
+  store <2 x double> %result.1, ptr %out_cos, align 8
+  ret void
+}
+
+define void @test_sincospi_nxv4f32(<vscale x 4 x float> %x, ptr noalias %out_sin, ptr noalias %out_cos) {
+; SLEEF-LABEL: test_sincospi_nxv4f32:
+; SLEEF:       // %bb.0:
+; SLEEF-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; SLEEF-NEXT:    .cfi_def_cfa_offset 16
+; SLEEF-NEXT:    .cfi_offset w30, -16
+; SLEEF-NEXT:    bl _ZGVsNxvl4l4_sincospif
+; SLEEF-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; SLEEF-NEXT:    ret
+;
+; ARMPL-LABEL: test_sincospi_nxv4f32:
+; ARMPL:       // %bb.0:
+; ARMPL-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; ARMPL-NEXT:    .cfi_def_cfa_offset 16
+; ARMPL-NEXT:    .cfi_offset w30, -16
+; ARMPL-NEXT:    ptrue p0.s
+; ARMPL-NEXT:    bl armpl_svsincospi_f32_x
+; ARMPL-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; ARMPL-NEXT:    ret
+  %result = call { <vscale x 4 x float>, <vscale x 4 x float> } @llvm.sincospi.nxv4f32(<vscale x 4 x float> %x)
+  %result.0 = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } %result, 0
+  %result.1 = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } %result, 1
+  store <vscale x 4 x float> %result.0, ptr %out_sin, align 4
+  store <vscale x 4 x float> %result.1, ptr %out_cos, align 4
+  ret void
+}
+
+define void @test_sincospi_nxv2f64(<vscale x 2 x double> %x, ptr noalias %out_sin, ptr noalias %out_cos) {
+; SLEEF-LABEL: test_sincospi_nxv2f64:
+; SLEEF:       // %bb.0:
+; SLEEF-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; SLEEF-NEXT:    .cfi_def_cfa_offset 16
+; SLEEF-NEXT:    .cfi_offset w30, -16
+; SLEEF-NEXT:    bl _ZGVsNxvl8l8_sincospi
+; SLEEF-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; SLEEF-NEXT:    ret
+;
+; ARMPL-LABEL: test_sincospi_nxv2f64:
+; ARMPL:       // %bb.0:
+; ARMPL-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; ARMPL-NEXT:    .cfi_def_cfa_offset 16
+; ARMPL-NEXT:    .cfi_offset w30, -16
+; ARMPL-NEXT:    ptrue p0.d
+; ARMPL-NEXT:    bl armpl_svsincospi_f64_x
+; ARMPL-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; ARMPL-NEXT:    ret
+  %result = call { <vscale x 2 x double>, <vscale x 2 x double> } @llvm.sincospi.nxv2f64(<vscale x 2 x double> %x)
+  %result.0 = extractvalue { <vscale x 2 x double>, <vscale x 2 x double> } %result, 0
+  %result.1 = extractvalue { <vscale x 2 x double>, <vscale x 2 x double> } %result, 1
+  store <vscale x 2 x double> %result.0, ptr %out_sin, align 8
+  store <vscale x 2 x double> %result.1, ptr %out_cos, align 8
+  ret void
+}
diff --git a/llvm/test/CodeGen/AArch64/vecreduce-add.ll b/llvm/test/CodeGen/AArch64/vecreduce-add.ll
index 5d6b523f1549a..c72d00e65fcab 100644
--- a/llvm/test/CodeGen/AArch64/vecreduce-add.ll
+++ b/llvm/test/CodeGen/AArch64/vecreduce-add.ll
@@ -2069,39 +2069,32 @@ define i32 @test_udot_v24i8(ptr %p1, ptr %p2) {
 ;
 ; CHECK-GI-BASE-LABEL: test_udot_v24i8:
 ; CHECK-GI-BASE:       // %bb.0: // %entry
-; CHECK-GI-BASE-NEXT:    ldr q0, [x0]
-; CHECK-GI-BASE-NEXT:    ldr q1, [x1]
+; CHECK-GI-BASE-NEXT:    mov v0.s[0], wzr
+; CHECK-GI-BASE-NEXT:    ldr q1, [x0]
 ; CHECK-GI-BASE-NEXT:    ldr d2, [x0, #16]
-; CHECK-GI-BASE-NEXT:    ldr d3, [x1, #16]
-; CHECK-GI-BASE-NEXT:    ushll v4.8h, v0.8b, #0
-; CHECK-GI-BASE-NEXT:    ushll2 v0.8h, v0.16b, #0
+; CHECK-GI-BASE-NEXT:    ldr q3, [x1]
+; CHECK-GI-BASE-NEXT:    ldr d4, [x1, #16]
 ; CHECK-GI-BASE-NEXT:    ushll v5.8h, v1.8b, #0
-; CHECK-GI-BASE-NEXT:    ushll v2.8h, v2.8b, #0
 ; CHECK-GI-BASE-NEXT:    ushll2 v1.8h, v1.16b, #0
-; CHECK-GI-BASE-NEXT:    ushll v3.8h, v3.8b, #0
-; CHECK-GI-BASE-NEXT:    umull v6.4s, v5.4h, v4.4h
-; CHECK-GI-BASE-NEXT:    umull2 v4.4s, v5.8h, v4.8h
-; CHECK-GI-BASE-NEXT:    umull2 v5.4s, v1.8h, v0.8h
-; CHECK-GI-BASE-NEXT:    umull v7.4s, v3.4h, v2.4h
-; CHECK-GI-BASE-NEXT:    umull v0.4s, v1.4h, v0.4h
-; CHECK-GI-BASE-NEXT:    umull2 v1.4s, v3.8h, v2.8h
-; CHECK-GI-BASE-NEXT:    addv s2, v6.4s
-; CHECK-GI-BASE-NEXT:    addv s3, v4.4s
-; CHECK-GI-BASE-NEXT:    addv s4, v5.4s
-; CHECK-GI-BASE-NEXT:    addv s5, v7.4s
+; CHECK-GI-BASE-NEXT:    ushll v2.8h, v2.8b, #0
+; CHECK-GI-BASE-NEXT:    ushll v6.8h, v3.8b, #0
+; CHECK-GI-BASE-NEXT:    ushll2 v3.8h, v3.16b, #0
+; CHECK-GI-BASE-NEXT:    ushll v4.8h, v4.8b, #0
+; CHECK-GI-BASE-NEXT:    mov v0.s[1], wzr
+; CHECK-GI-BASE-NEXT:    umull v7.4s, v6.4h, v5.4h
+; CHECK-GI-BASE-NEXT:    umull v16.4s, v3.4h, v1.4h
+; CHECK-GI-BASE-NEXT:    umull v17.4s, v4.4h, v2.4h
+; CHECK-GI-BASE-NEXT:    mov v0.s[2], wzr
+; CHECK-GI-BASE-NEXT:    umlal2 v7.4s, v6.8h, v5.8h
+; CHECK-GI-BASE-NEXT:    umlal2 v16.4s, v3.8h, v1.8h
+; CHECK-GI-BASE-NEXT:    umlal2 v17.4s, v4.8h, v2.8h
+; CHECK-GI-BASE-NEXT:    mov v0.s[3], wzr
+; CHECK-GI-BASE-NEXT:    add v1.4s, v7.4s, v16.4s
+; CHECK-GI-BASE-NEXT:    add v0.4s, v0.4s, v0.4s
+; CHECK-GI-BASE-NEXT:    add v0.4s, v17.4s, v0.4s
+; CHECK-GI-BASE-NEXT:    add v0.4s, v1.4s, v0.4s
 ; CHECK-GI-BASE-NEXT:    addv s0, v0.4s
-; CHECK-GI-BASE-NEXT:    addv s1, v1.4s
-; CHECK-GI-BASE-NEXT:    fmov w8, s2
-; CHECK-GI-BASE-NEXT:    fmov w9, s3
-; CHECK-GI-BASE-NEXT:    fmov w10, s4
-; CHECK-GI-BASE-NEXT:    fmov w11, s5
-; CHECK-GI-BASE-NEXT:    add w8, w8, w9
-; CHECK-GI-BASE-NEXT:    fmov w9, s0
-; CHECK-GI-BASE-NEXT:    add w10, w10, w11
-; CHECK-GI-BASE-NEXT:    fmov w11, s1
-; CHECK-GI-BASE-NEXT:    add w8, w8, w9
-; CHECK-GI-BASE-NEXT:    add w9, w10, w11
-; CHECK-GI-BASE-NEXT:    add w0, w8, w9
+; CHECK-GI-BASE-NEXT:    fmov w0, s0
 ; CHECK-GI-BASE-NEXT:    ret
 ;
 ; CHECK-GI-DOT-LABEL: test_udot_v24i8:
@@ -2172,91 +2165,71 @@ define i32 @test_udot_v48i8(ptr %p1, ptr %p2) {
 ;
 ; CHECK-GI-BASE-LABEL: test_udot_v48i8:
 ; CHECK-GI-BASE:       // %bb.0: // %entry
-; CHECK-GI-BASE-NEXT:    ldp q0, q3, [x1]
-; CHECK-GI-BASE-NEXT:    ldr q6, [x0, #32]
-; CHECK-GI-BASE-NEXT:    ldp q1, q2, [x0]
-; CHECK-GI-BASE-NEXT:    ldr q7, [x1, #32]
-; CHECK-GI-BASE-NEXT:    ushll v20.8h, v6.8b, #0
-; CHECK-GI-BASE-NEXT:    ushll2 v6.8h, v6.16b, #0
-; CHECK-GI-BASE-NEXT:    ushll v4.8h, v0.8b, #0
-; CHECK-GI-BASE-NEXT:    ushll2 v0.8h, v0.16b, #0
-; CHECK-GI-BASE-NEXT:    ushll v16.8h, v3.8b, #0
-; CHECK-GI-BASE-NEXT:    ushll v5.8h, v1.8b, #0
+; CHECK-GI-BASE-NEXT:    mov v0.s[0], wzr
+; CHECK-GI-BASE-NEXT:    ldp q1, q5, [x1]
+; CHECK-GI-BASE-NEXT:    ldp q2, q3, [x0]
+; CHECK-GI-BASE-NEXT:    ldr q4, [x0, #32]
+; CHECK-GI-BASE-NEXT:    ldr q6, [x1, #32]
+; CHECK-GI-BASE-NEXT:    ushll v7.8h, v1.8b, #0
 ; CHECK-GI-BASE-NEXT:    ushll2 v1.8h, v1.16b, #0
-; CHECK-GI-BASE-NEXT:    ushll v17.8h, v2.8b, #0
-; CHECK-GI-BASE-NEXT:    ushll2 v3.8h, v3.16b, #0
+; CHECK-GI-BASE-NEXT:    ushll v16.8h, v5.8b, #0
+; CHECK-GI-BASE-NEXT:    mov v0.s[1], wzr
+; CHECK-GI-BASE-NEXT:    ushll2 v5.8h, v5.16b, #0
+; CHECK-GI-BASE-NEXT:    ushll v17.8h, v6.8b, #0
+; CHECK-GI-BASE-NEXT:    ushll2 v6.8h, v6.16b, #0
+; CHECK-GI-BASE-NEXT:    ushll v18.8h, v2.8b, #0
 ; CHECK-GI-BASE-NEXT:    ushll2 v2.8h, v2.16b, #0
-; CHECK-GI-BASE-NEXT:    umull v18.4s, v4.4h, v5.4h
-; CHECK-GI-BASE-NEXT:    umull2 v4.4s, v4.8h, v5.8h
-; CHECK-GI-BASE-NEXT:    umull v5.4s, v0.4h, v1.4h
-; CHECK-GI-BASE-NEXT:    umull2 v0.4s, v0.8h, v1.8h
-; CHECK-GI-BASE-NEXT:    umull v19.4s, v16.4h, v17.4h
-; CHECK-GI-BASE-NEXT:    ushll v1.8h, v7.8b, #0
-; CHECK-GI-BASE-NEXT:    umull2 v16.4s, v16.8h, v17.8h
-; CHECK-GI-BASE-NEXT:    umull v17.4s, v3.4h, v2.4h
-; CHECK-GI-BASE-NEXT:    umull2 v2.4s, v3.8h, v2.8h
-; CHECK-GI-BASE-NEXT:    ushll2 v7.8h, v7.16b, #0
-; CHECK-GI-BASE-NEXT:    addv s18, v18.4s
-; CHECK-GI-BASE-NEXT:    addv s4, v4.4s
-; CHECK-GI-BASE-NEXT:    addv s5, v5.4s
-; CHECK-GI-BASE-NEXT:    addv s0, v0.4s
-; CHECK-GI-BASE-NEXT:    addv s19, v19.4s
-; CHECK-GI-BASE-NEXT:    umull v3.4s, v1.4h, v20.4h
-; CHECK-GI-BASE-NEXT:    addv s2, v2.4s
-; CHECK-GI-BASE-NEXT:    umull2 v1.4s, v1.8h, v20.8h
-; CHECK-GI-BASE-NEXT:    umull v20.4s, v7.4h, v6.4h
-; CHECK-GI-BASE-NEXT:    fmov w8, s18
-; CHECK-GI-BASE-NEXT:    fmov w9, s4
-; CHECK-GI-BASE-NEXT:    fmov w10, s5
-; CHECK-GI-BASE-NEXT:    fmov w11, s0
-; CHECK-GI-BASE-NEXT:    fmov w12, s19
-; CHECK-GI-BASE-NEXT:    addv s4, v16.4s
-; CHECK-GI-BASE-NEXT:    addv s5, v17.4s
-; CHECK-GI-BASE-NEXT:    addv s3, v3.4s
-; CHECK-GI-BASE-NEXT:    umull2 v0.4s, v7.8h, v6.8h
-; CHECK-GI-BASE-NEXT:    add w8, w8, w9
-; CHECK-GI-BASE-NEXT:    addv s1, v1.4s
-; CHECK-GI-BASE-NEXT:    add w9, w11, w12
-; CHECK-GI-BASE-NEXT:    add w8, w8, w10
-; CHECK-GI-BASE-NEXT:    fmov w10, s4
-; CHECK-GI-BASE-NEXT:    fmov w11, s5
-; CHECK-GI-BASE-NEXT:    fmov w12, s2
-; CHECK-GI-BASE-NEXT:    addv s4, v20.4s
+; CHECK-GI-BASE-NEXT:    ushll v19.8h, v3.8b, #0
+; CHECK-GI-BASE-NEXT:    ushll2 v3.8h, v3.16b, #0
+; CHECK-GI-BASE-NEXT:    ushll v20.8h, v4.8b, #0
+; CHECK-GI-BASE-NEXT:    ushll2 v4.8h, v4.16b, #0
+; CHECK-GI-BASE-NEXT:    mov v0.s[2], wzr
+; CHECK-GI-BASE-NEXT:    umull v21.4s, v7.4h, v18.4h
+; CHECK-GI-BASE-NEXT:    umull v22.4s, v1.4h, v2.4h
+; CHECK-GI-BASE-NEXT:    umull v23.4s, v16.4h, v19.4h
+; CHECK-GI-BASE-NEXT:    umull v24.4s, v5.4h, v3.4h
+; CHECK-GI-BASE-NEXT:    umull v25.4s, v17.4h, v20.4h
+; CHECK-GI-BASE-NEXT:    umull v26.4s, v6.4h, v4.4h
+; CHECK-GI-BASE-NEXT:    mov v0.s[3], wzr
+; CHECK-GI-BASE-NEXT:    umlal2 v21.4s, v7.8h, v18.8h
+; CHECK-GI-BASE-NEXT:    umlal2 v22.4s, v1.8h, v2.8h
+; CHECK-GI-BASE-NEXT:    umlal2 v23.4s, v16.8h, v19.8h
+; CHECK-GI-BASE-NEXT:    umlal2 v24.4s, v5.8h, v3.8h
+; CHECK-GI-BASE-NEXT:    umlal2 v25.4s, v17.8h, v20.8h
+; CHECK-GI-BASE-NEXT:    umlal2 v26.4s, v6.8h, v4.8h
+; CHECK-GI-BASE-NEXT:    add v0.4s, v0.4s, v0.4s
+; CHECK-GI-BASE-NEXT:    add v1.4s, v21.4s, v22.4s
+; CHECK-GI-BASE-NEXT:    add v2.4s, v23.4s, v24.4s
+; CHECK-GI-BASE-NEXT:    add v3.4s, v25.4s, v26.4s
+; CHECK-GI-BASE-NEXT:    add v0.4s, v0.4s, v0.4s
+; CHECK-GI-BASE-NEXT:    add v1.4s, v1.4s, v2.4s
+; CHECK-GI-BASE-NEXT:    add v0.4s, v3.4s, v0.4s
+; CHECK-GI-BASE-NEXT:    add v0.4s, v1.4s, v0.4s
 ; CHECK-GI-BASE-NEXT:    addv s0, v0.4s
-; CHECK-GI-BASE-NEXT:    add w9, w9, w10
-; CHECK-GI-BASE-NEXT:    add w10, w11, w12
-; CHECK-GI-BASE-NEXT:    fmov w11, s3
-; CHECK-GI-BASE-NEXT:    add w8, w8, w9
-; CHECK-GI-BASE-NEXT:    add w9, w10, w11
-; CHECK-GI-BASE-NEXT:    fmov w10, s1
-; CHECK-GI-BASE-NEXT:    fmov w11, s0
-; CHECK-GI-BASE-NEXT:    add w9, w9, w10
-; CHECK-GI-BASE-NEXT:    fmov w10, s4
-; CHECK-GI-BASE-NEXT:    add w8, w8, w9
-; CHECK-GI-BASE-NEXT:    add w9, w10, w11
-; CHECK-GI-BASE-NEXT:    add w0, w8, w9
+; CHECK-GI-BASE-NEXT:    fmov w0, s0
 ; CHECK-GI-BASE-NEXT:    ret
 ;
 ; CHECK-GI-DOT-LABEL: test_udot_v48i8:
 ; CHECK-GI-DOT:       // %bb.0: // %entry
-; CHECK-GI-DOT-NEXT:    movi v0.2d, #0000000000000000
+; CHECK-GI-DOT-NEXT:    mov v0.s[0], wzr
 ; CHECK-GI-DOT-NEXT:    movi v1.2d, #0000000000000000
 ; CHECK-GI-DOT-NEXT:    ldr q7, [x0, #32]
-; CHECK-GI-DOT-NEXT:    ldp q3, q4, [x0]
 ; CHECK-GI-DOT-NEXT:    movi v2.2d, #0000000000000000
-; CHECK-GI-DOT-NEXT:    ldp q5, q6, [x1]
-; CHECK-GI-DOT-NEXT:    ldr q16, [x1, #32]
-; CHECK-GI-DOT-NEXT:    udot v0.4s, v5.16b, v3.16b
+; CHECK-GI-DOT-NEXT:    movi v3.2d, #0000000000000000
+; CHECK-GI-DOT-NEXT:    ldr q17, [x1, #32]
+; CHECK-GI-DOT-NEXT:    ldp q4, q5, [x0]
+; CHECK-GI-DOT-NEXT:    ldp q6, q16, [x1]
+; CHECK-GI-DOT-NEXT:    mov v0.s[1], wzr
+; CHECK-GI-DOT-NEXT:    udot v2.4s, v17.16b, v7.16b
 ; CHECK-GI-DOT-NEXT:    udot v1.4s, v6.16b, v4.16b
-; CHECK-GI-DOT-NEXT:    udot v2.4s, v16.16b, v7.16b
+; CHECK-GI-DOT-NEXT:    udot v3.4s, v16.16b, v5.16b
+; CHECK-GI-DOT-NEXT:    mov v0.s[2], wzr
+; CHECK-GI-DOT-NEXT:    add v1.4s, v1.4s, v3.4s
+; CHECK-GI-DOT-NEXT:    mov v0.s[3], wzr
+; CHECK-GI-DOT-NEXT:    add v0.4s, v2.4s, v0.4s
+; CHECK-GI-DOT-NEXT:    add v0.4s, v1.4s, v0.4s
 ; CHECK-GI-DOT-NEXT:    addv s0, v0.4s
-; CHECK-GI-DOT-NEXT:    addv s1, v1.4s
-; CHECK-GI-DOT-NEXT:    addv s2, v2.4s
-; CHECK-GI-DOT-NEXT:    fmov w8, s0
-; CHECK-GI-DOT-NEXT:    fmov w9, s1
-; CHECK-GI-DOT-NEXT:    add w8, w8, w9
-; CHECK-GI-DOT-NEXT:    fmov w9, s2
-; CHECK-GI-DOT-NEXT:    add w0, w8, w9
+; CHECK-GI-DOT-NEXT:    fmov w0, s0
 ; CHECK-GI-DOT-NEXT:    ret
 entry:
   %a = load <48 x i8>, ptr %p1
@@ -2397,39 +2370,32 @@ define i32 @test_sdot_v24i8(ptr %p1, ptr %p2) {
 ;
 ; CHECK-GI-BASE-LABEL: test_sdot_v24i8:
 ; CHECK-GI-BASE:       // %bb.0: // %entry
-; CHECK-GI-BASE-NEXT:    ldr q0, [x0]
-; CHECK-GI-BASE-NEXT:    ldr q1, [x1]
+; CHECK-GI-BASE-NEXT:    mov v0.s[0], wzr
+; CHECK-GI-BASE-NEXT:    ldr q1, [x0]
 ; CHECK-GI-BASE-NEXT:    ldr d2, [x0, #16]
-; CHECK-GI-BASE-NEXT:    ldr d3, [x1, #16]
-; CHECK-GI-BASE-NEXT:    sshll v4.8h, v0.8b, #0
-; CHECK-GI-BASE-NEXT:    sshll2 v0.8h, v0.16b, #0
+; CHECK-GI-BASE-NEXT:    ldr q3, [x1]
+; CHECK-GI-BASE-NEXT:    ldr d4, [x1, #16]
 ; CHECK-GI-BASE-NEXT:    sshll v5.8h, v1.8b, #0
-; CHECK-GI-BASE-NEXT:    sshll v2.8h, v2.8b, #0
 ; CHECK-GI-BASE-NEXT:    sshll2 v1.8h, v1.16b, #0
-; CHECK-GI-BASE-NEXT:    sshll v3.8h, v3.8b, #0
-; CHECK-GI-BASE-NEXT:    smull v6.4s, v5.4h, v4.4h
-; CHECK-GI-BASE-NEXT:    smull2 v4.4s, v5.8h, v4.8h
-; CHECK-GI-BASE-NEXT:    smull2 v5.4s, v1.8h, v0.8h
-; CHECK-GI-BASE-NEXT:    smull v7.4s, v3.4h, v2.4h
-; CHECK-GI-BASE-NEXT:    smull v0.4s, v1.4h, v0.4h
-; CHECK-GI-BASE-NEXT:    smull2 v1.4s, v3.8h, v2.8h
-; CHECK-GI-BASE-NEXT:    addv s2, v6.4s
-; CHECK-GI-BASE-NEXT:    addv s3, v4.4s
-; CHECK-GI-BASE-NEXT:    addv s4, v5.4s
-; CHECK-GI-BASE-NEXT:    addv s5, v7.4s
+; CHECK-GI-BASE-NEXT:    sshll v2.8h, v2.8b, #0
+; CHECK-GI-BASE-NEXT:    sshll v6.8h, v3.8b, #0
+; CHECK-GI-BASE-NEXT:    sshll2 v3.8h, v3.16b, #0
+; CHECK-GI-BASE-NEXT:    sshll v4.8h, v4.8b, #0
+; CHECK-GI-BASE-NEXT:    mov v0.s[1], wzr
+; CHECK-GI-BASE-NEXT:    smull v7.4s, v6.4h, v5.4h
+; CHECK-GI-BASE-NEXT:    smull v16.4s, v3.4h, v1.4h
+; CHECK-GI-BASE-NEXT:    smull v17.4s, v4.4h, v2.4h
+; CHECK-GI-BASE-NEXT:    mov v0.s[2], wzr
+; CHECK-GI-BASE-NEXT:    smlal2 v7.4s, v6.8h, v5.8h
+; CHECK-GI-BASE-NEXT:    smlal2 v16.4s, v3.8h, v1.8h
+; CHECK-GI-BASE-NEXT:    smlal2 v17.4s, v4.8h, v2.8h
+; CHECK-GI-BASE-NEXT:    mov v0.s[3], wzr
+; CHECK-GI-BASE-NEXT:    add v1.4s, v7.4s, v16.4s
+; CHECK-GI-BASE-NEXT:    add v0.4s, v0.4s, v0.4s
+; CHECK-GI-BASE-NEXT:    add v0.4s, v17.4s, v0.4s
+; CHECK-GI-BASE-NEXT:    add v0.4s, v1.4s, v0.4s
 ; CHECK-GI-BASE-NEXT:    addv s0, v0.4s
-; CHECK-GI-BASE-NEXT:    addv s1, v1.4s
-; CHECK-GI-BASE-NEXT:    fmov w8, s2
-; CHECK-GI-BASE-NEXT:    fmov w9, s3
-; CHECK-GI-BASE-NEXT:    fmov w10, s4
-; CHECK-GI-BASE-NEXT:    fmov w11, s5
-; CHECK-GI-BASE-NEXT:    add w8, w8, w9
-; CHECK-GI-BASE-NEXT:    fmov w9, s0
-; CHECK-GI-BASE-NEXT:    add w10, w10, w11
-; CHECK-GI-BASE-NEXT:    fmov w11, s1
-; CHECK-GI-BASE-NEXT:    add w8, w8, w9
-; CHECK-GI-BASE-NEXT:    add w9, w10, w11
-; CHECK-GI-BASE-NEXT:    add w0, w8, w9
+; CHECK-GI-BASE-NEXT:    fmov w0, s0
 ; CHECK-GI-BASE-NEXT:    ret
 ;
 ; CHECK-GI-DOT-LABEL: test_sdot_v24i8:
@@ -2500,91 +2466,71 @@ define i32 @test_sdot_v48i8(ptr %p1, ptr %p2) {
 ;
 ; CHECK-GI-BASE-LABEL: test_sdot_v48i8:
 ; CHECK-GI-BASE:       // %bb.0: // %entry
-; CHECK-GI-BASE-NEXT:    ldp q0, q3, [x1]
-; CHECK-GI-BASE-NEXT:    ldr q6, [x0, #32]
-; CHECK-GI-BASE-NEXT:    ldp q1, q2, [x0]
-; CHECK-GI-BASE-NEXT:    ldr q7, [x1, #32]
-; CHECK-GI-BASE-NEXT:    sshll v20.8h, v6.8b, #0
-; CHECK-GI-BASE-NEXT:    sshll2 v6.8h, v6.16b, #0
-; CHECK-GI-BASE-NEXT:    sshll v4.8h, v0.8b, #0
-; CHECK-GI-BASE-NEXT:    sshll2 v0.8h, v0.16b, #0
-; CHECK-GI-BASE-NEXT:    sshll v16.8h, v3.8b, #0
-; CHECK-GI-BASE-NEXT:    sshll v5.8h, v1.8b, #0
+; CHECK-GI-BASE-NEXT:    mov v0.s[0], wzr
+; CHECK-GI-BASE-NEXT:    ldp q1, q5, [x1]
+; CHECK-GI-BASE-NEXT:    ldp q2, q3, [x0]
+; CHECK-GI-BASE-NEXT:    ldr q4, [x0, #32]
+; CHECK-GI-BASE-NEXT:    ldr q6, [x1, #32]
+; CHECK-GI-BASE-NEXT:    sshll v7.8h, v1.8b, #0
 ; CHECK-GI-BASE-NEXT:    sshll2 v1.8h, v1.16b, #0
-; CHECK-GI-BASE-NEXT:    sshll v17.8h, v2.8b, #0
-; CHECK-GI-BASE-NEXT:    sshll2 v3.8h, v3.16b, #0
+; CHECK-GI-BASE-NEXT:    sshll v16.8h, v5.8b, #0
+; CHECK-GI-BASE-NEXT:    mov v0.s[1], wzr
+; CHECK-GI-BASE-NEXT:    sshll2 v5.8h, v5.16b, #0
+; CHECK-GI-BASE-NEXT:    sshll v17.8h, v6.8b, #0
+; CHECK-GI-BASE-NEXT:    sshll2 v6.8h, v6.16b, #0
+; CHECK-GI-BASE-NEXT:    sshll v18.8h, v2.8b, #0
 ; CHECK-GI-BASE-NEXT:    sshll2 v2.8h, v2.16b, #0
-; CHECK-GI-BASE-NEXT:    smull v18.4s, v4.4h, v5.4h
-; CHECK-GI-BASE-NEXT:    smull2 v4.4s, v4.8h, v5.8h
-; CHECK-GI-BASE-NEXT:    smull v5.4s, v0.4h, v1.4h
-; CHECK-GI-BASE-NEXT:    smull2 v0.4s, v0.8h, v1.8h
-; CHECK-GI-BASE-NEXT:    smull v19.4s, v16.4h, v17.4h
-; CHECK-GI-BASE-NEXT:    sshll v1.8h, v7.8b, #0
-; CHECK-GI-BASE-NEXT:    smull2 v16.4s, v16.8h, v17.8h
-; CHECK-GI-BASE-NEXT:    smull v17.4s, v3.4h, v2.4h
-; CHECK-GI-BASE-NEXT:    smull2 v2.4s, v3.8h, v2.8h
-; CHECK-GI-BASE-NEXT:    sshll2 v7.8h, v7.16b, #0
-; CHECK-GI-BASE-NEXT:    addv s18, v18.4s
-; CHECK-GI-BASE-NEXT:    addv s4, v4.4s
-; CHECK-GI-BASE-NEXT:    addv s5, v5.4s
-; CHECK-GI-BASE-NEXT:    addv s0, v0.4s
-; CHECK-GI-BASE-NEXT:    addv s19, v19.4s
-; CHECK-GI-BASE-NEXT:    smull v3.4s, v1.4h, v20.4h
-; CHECK-GI-BASE-NEXT:    addv s2, v2.4s
-; CHECK-GI-BASE-NEXT:    smull2 v1.4s, v1.8h, v20.8h
-; CHECK-GI-BASE-NEXT:    smull v20.4s, v7.4h, v6.4h
-; CHECK-GI-BASE-NEXT:    fmov w8, s18
-; CHECK-GI-BASE-NEXT:    fmov w9, s4
-; CHECK-GI-BASE-NEXT:    fmov w10, s5
-; CHECK-GI-BASE-NEXT:    fmov w11, s0
-; CHECK-GI-BASE-NEXT:    fmov w12, s19
-; CHECK-GI-BASE-NEXT:    addv s4, v16.4s
-; CHECK-GI-BASE-NEXT:    addv s5, v17.4s
-; CHECK-GI-BASE-NEXT:    addv s3, v3.4s
-; CHECK-GI-BASE-NEXT:    smull2 v0.4s, v7.8h, v6.8h
-; CHECK-GI-BASE-NEXT:    add w8, w8, w9
-; CHECK-GI-BASE-NEXT:    addv s1, v1.4s
-; CHECK-GI-BASE-NEXT:    add w9, w11, w12
-; CHECK-GI-BASE-NEXT:    add w8, w8, w10
-; CHECK-GI-BASE-NEXT:    fmov w10, s4
-; CHECK-GI-BASE-NEXT:    fmov w11, s5
-; CHECK-GI-BASE-NEXT:    fmov w12, s2
-; CHECK-GI-BASE-NEXT:    addv s4, v20.4s
+; CHECK-GI-BASE-NEXT:    sshll v19.8h, v3.8b, #0
+; CHECK-GI-BASE-NEXT:    sshll2 v3.8h, v3.16b, #0
+; CHECK-GI-BASE-NEXT:    sshll v20.8h, v4.8b, #0
+; CHECK-GI-BASE-NEXT:    sshll2 v4.8h, v4.16b, #0
+; CHECK-GI-BASE-NEXT:    mov v0.s[2], wzr
+; CHECK-GI-BASE-NEXT:    smull v21.4s, v7.4h, v18.4h
+; CHECK-GI-BASE-NEXT:    smull v22.4s, v1.4h, v2.4h
+; CHECK-GI-BASE-NEXT:    smull v23.4s, v16.4h, v19.4h
+; CHECK-GI-BASE-NEXT:    smull v24.4s, v5.4h, v3.4h
+; CHECK-GI-BASE-NEXT:    smull v25.4s, v17.4h, v20.4h
+; CHECK-GI-BASE-NEXT:    smull v26.4s, v6.4h, v4.4h
+; CHECK-GI-BASE-NEXT:    mov v0.s[3], wzr
+; CHECK-GI-BASE-NEXT:    smlal2 v21.4s, v7.8h, v18.8h
+; CHECK-GI-BASE-NEXT:    smlal2 v22.4s, v1.8h, v2.8h
+; CHECK-GI-BASE-NEXT:    smlal2 v23.4s, v16.8h, v19.8h
+; CHECK-GI-BASE-NEXT:    smlal2 v24.4s, v5.8h, v3.8h
+; CHECK-GI-BASE-NEXT:    smlal2 v25.4s, v17.8h, v20.8h
+; CHECK-GI-BASE-NEXT:    smlal2 v26.4s, v6.8h, v4.8h
+; CHECK-GI-BASE-NEXT:    add v0.4s, v0.4s, v0.4s
+; CHECK-GI-BASE-NEXT:    add v1.4s, v21.4s, v22.4s
+; CHECK-GI-BASE-NEXT:    add v2.4s, v23.4s, v24.4s
+; CHECK-GI-BASE-NEXT:    add v3.4s, v25.4s, v26.4s
+; CHECK-GI-BASE-NEXT:    add v0.4s, v0.4s, v0.4s
+; CHECK-GI-BASE-NEXT:    add v1.4s, v1.4s, v2.4s
+; CHECK-GI-BASE-NEXT:    add v0.4s, v3.4s, v0.4s
+; CHECK-GI-BASE-NEXT:    add v0.4s, v1.4s, v0.4s
 ; CHECK-GI-BASE-NEXT:    addv s0, v0.4s
-; CHECK-GI-BASE-NEXT:    add w9, w9, w10
-; CHECK-GI-BASE-NEXT:    add w10, w11, w12
-; CHECK-GI-BASE-NEXT:    fmov w11, s3
-; CHECK-GI-BASE-NEXT:    add w8, w8, w9
-; CHECK-GI-BASE-NEXT:    add w9, w10, w11
-; CHECK-GI-BASE-NEXT:    fmov w10, s1
-; CHECK-GI-BASE-NEXT:    fmov w11, s0
-; CHECK-GI-BASE-NEXT:    add w9, w9, w10
-; CHECK-GI-BASE-NEXT:    fmov w10, s4
-; CHECK-GI-BASE-NEXT:    add w8, w8, w9
-; CHECK-GI-BASE-NEXT:    add w9, w10, w11
-; CHECK-GI-BASE-NEXT:    add w0, w8, w9
+; CHECK-GI-BASE-NEXT:    fmov w0, s0
 ; CHECK-GI-BASE-NEXT:    ret
 ;
 ; CHECK-GI-DOT-LABEL: test_sdot_v48i8:
 ; CHECK-GI-DOT:       // %bb.0: // %entry
-; CHECK-GI-DOT-NEXT:    movi v0.2d, #0000000000000000
+; CHECK-GI-DOT-NEXT:    mov v0.s[0], wzr
 ; CHECK-GI-DOT-NEXT:    movi v1.2d, #0000000000000000
 ; CHECK-GI-DOT-NEXT:    ldr q7, [x0, #32]
-; CHECK-GI-DOT-NEXT:    ldp q3, q4, [x0]
 ; CHECK-GI-DOT-NEXT:    movi v2.2d, #0000000000000000
-; CHECK-GI-DOT-NEXT:    ldp q5, q6, [x1]
-; CHECK-GI-DOT-NEXT:    ldr q16, [x1, #32]
-; CHECK-GI-DOT-NEXT:    sdot v0.4s, v5.16b, v3.16b
+; CHECK-GI-DOT-NEXT:    movi v3.2d, #0000000000000000
+; CHECK-GI-DOT-NEXT:    ldr q17, [x1, #32]
+; CHECK-GI-DOT-NEXT:    ldp q4, q5, [x0]
+; CHECK-GI-DOT-NEXT:    ldp q6, q16, [x1]
+; CHECK-GI-DOT-NEXT:    mov v0.s[1], wzr
+; CHECK-GI-DOT-NEXT:    sdot v2.4s, v17.16b, v7.16b
 ; CHECK-GI-DOT-NEXT:    sdot v1.4s, v6.16b, v4.16b
-; CHECK-GI-DOT-NEXT:    sdot v2.4s, v16.16b, v7.16b
+; CHECK-GI-DOT-NEXT:    sdot v3.4s, v16.16b, v5.16b
+; CHECK-GI-DOT-NEXT:    mov v0.s[2], wzr
+; CHECK-GI-DOT-NEXT:    add v1.4s, v1.4s, v3.4s
+; CHECK-GI-DOT-NEXT:    mov v0.s[3], wzr
+; CHECK-GI-DOT-NEXT:    add v0.4s, v2.4s, v0.4s
+; CHECK-GI-DOT-NEXT:    add v0.4s, v1.4s, v0.4s
 ; CHECK-GI-DOT-NEXT:    addv s0, v0.4s
-; CHECK-GI-DOT-NEXT:    addv s1, v1.4s
-; CHECK-GI-DOT-NEXT:    addv s2, v2.4s
-; CHECK-GI-DOT-NEXT:    fmov w8, s0
-; CHECK-GI-DOT-NEXT:    fmov w9, s1
-; CHECK-GI-DOT-NEXT:    add w8, w8, w9
-; CHECK-GI-DOT-NEXT:    fmov w9, s2
-; CHECK-GI-DOT-NEXT:    add w0, w8, w9
+; CHECK-GI-DOT-NEXT:    fmov w0, s0
 ; CHECK-GI-DOT-NEXT:    ret
 entry:
   %a = load <48 x i8>, ptr %p1
diff --git a/llvm/test/CodeGen/AArch64/vector-extract-last-active.ll b/llvm/test/CodeGen/AArch64/vector-extract-last-active.ll
index 3b11e67d072e7..bb9a09ca3cc80 100644
--- a/llvm/test/CodeGen/AArch64/vector-extract-last-active.ll
+++ b/llvm/test/CodeGen/AArch64/vector-extract-last-active.ll
@@ -1,6 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 3
-; RUN: llc -mtriple=aarch64 -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,NEON-FIXED
-; RUN: llc -mtriple=aarch64 -mattr=+sve -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,SVE-FIXED
+; RUN: llc -mtriple=aarch64 -mattr=+bf16 -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,NEON-FIXED
+; RUN: llc -mtriple=aarch64 -mattr=+sve,+bf16 -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,SVE-FIXED
 
 define i8 @extract_last_i8(<16 x i8> %data, <16 x i8> %mask, i8 %passthru) {
 ; NEON-FIXED-LABEL: extract_last_i8:
@@ -194,15 +194,115 @@ define i64 @extract_last_i64(<2 x i64> %data, <2 x i64> %mask, i64 %passthru) {
   ret i64 %res
 }
 
+define half @extract_last_half(<8 x half> %data, <8 x i16> %mask, half %passthru) {
+; NEON-FIXED-LABEL: extract_last_half:
+; NEON-FIXED:       // %bb.0:
+; NEON-FIXED-NEXT:    sub sp, sp, #16
+; NEON-FIXED-NEXT:    .cfi_def_cfa_offset 16
+; NEON-FIXED-NEXT:    cmtst v1.8h, v1.8h, v1.8h
+; NEON-FIXED-NEXT:    adrp x8, .LCPI4_0
+; NEON-FIXED-NEXT:    mov x9, sp
+; NEON-FIXED-NEXT:    ldr d4, [x8, :lo12:.LCPI4_0]
+; NEON-FIXED-NEXT:    str q0, [sp]
+; NEON-FIXED-NEXT:    // kill: def $h2 killed $h2 def $s2
+; NEON-FIXED-NEXT:    xtn v3.8b, v1.8h
+; NEON-FIXED-NEXT:    umaxv h1, v1.8h
+; NEON-FIXED-NEXT:    and v3.8b, v3.8b, v4.8b
+; NEON-FIXED-NEXT:    umaxv b3, v3.8b
+; NEON-FIXED-NEXT:    fmov w8, s3
+; NEON-FIXED-NEXT:    bfi x9, x8, #1, #3
+; NEON-FIXED-NEXT:    fmov w8, s1
+; NEON-FIXED-NEXT:    ldr h0, [x9]
+; NEON-FIXED-NEXT:    tst w8, #0x1
+; NEON-FIXED-NEXT:    fcsel s0, s0, s2, ne
+; NEON-FIXED-NEXT:    // kill: def $h0 killed $h0 killed $s0
+; NEON-FIXED-NEXT:    add sp, sp, #16
+; NEON-FIXED-NEXT:    ret
+;
+; SVE-FIXED-LABEL: extract_last_half:
+; SVE-FIXED:       // %bb.0:
+; SVE-FIXED-NEXT:    sub sp, sp, #16
+; SVE-FIXED-NEXT:    .cfi_def_cfa_offset 16
+; SVE-FIXED-NEXT:    cmtst v1.8h, v1.8h, v1.8h
+; SVE-FIXED-NEXT:    index z4.b, #0, #1
+; SVE-FIXED-NEXT:    mov x9, sp
+; SVE-FIXED-NEXT:    str q0, [sp]
+; SVE-FIXED-NEXT:    xtn v3.8b, v1.8h
+; SVE-FIXED-NEXT:    umaxv h1, v1.8h
+; SVE-FIXED-NEXT:    and v3.8b, v3.8b, v4.8b
+; SVE-FIXED-NEXT:    umaxv b3, v3.8b
+; SVE-FIXED-NEXT:    fmov w8, s3
+; SVE-FIXED-NEXT:    bfi x9, x8, #1, #3
+; SVE-FIXED-NEXT:    fmov w8, s1
+; SVE-FIXED-NEXT:    ldr h0, [x9]
+; SVE-FIXED-NEXT:    tst w8, #0x1
+; SVE-FIXED-NEXT:    fcsel h0, h0, h2, ne
+; SVE-FIXED-NEXT:    add sp, sp, #16
+; SVE-FIXED-NEXT:    ret
+  %notzero = icmp ne <8 x i16> %mask, zeroinitializer
+  %res = call half @llvm.experimental.vector.extract.last.active.v8f16(<8 x half> %data, <8 x i1> %notzero, half %passthru)
+  ret half %res
+}
+
+define bfloat @extract_last_bfloat(<8 x bfloat> %data, <8 x i16> %mask, bfloat %passthru) {
+; NEON-FIXED-LABEL: extract_last_bfloat:
+; NEON-FIXED:       // %bb.0:
+; NEON-FIXED-NEXT:    sub sp, sp, #16
+; NEON-FIXED-NEXT:    .cfi_def_cfa_offset 16
+; NEON-FIXED-NEXT:    cmtst v1.8h, v1.8h, v1.8h
+; NEON-FIXED-NEXT:    adrp x8, .LCPI5_0
+; NEON-FIXED-NEXT:    mov x9, sp
+; NEON-FIXED-NEXT:    ldr d4, [x8, :lo12:.LCPI5_0]
+; NEON-FIXED-NEXT:    str q0, [sp]
+; NEON-FIXED-NEXT:    // kill: def $h2 killed $h2 def $s2
+; NEON-FIXED-NEXT:    xtn v3.8b, v1.8h
+; NEON-FIXED-NEXT:    umaxv h1, v1.8h
+; NEON-FIXED-NEXT:    and v3.8b, v3.8b, v4.8b
+; NEON-FIXED-NEXT:    umaxv b3, v3.8b
+; NEON-FIXED-NEXT:    fmov w8, s3
+; NEON-FIXED-NEXT:    bfi x9, x8, #1, #3
+; NEON-FIXED-NEXT:    fmov w8, s1
+; NEON-FIXED-NEXT:    ldr h0, [x9]
+; NEON-FIXED-NEXT:    tst w8, #0x1
+; NEON-FIXED-NEXT:    fcsel s0, s0, s2, ne
+; NEON-FIXED-NEXT:    // kill: def $h0 killed $h0 killed $s0
+; NEON-FIXED-NEXT:    add sp, sp, #16
+; NEON-FIXED-NEXT:    ret
+;
+; SVE-FIXED-LABEL: extract_last_bfloat:
+; SVE-FIXED:       // %bb.0:
+; SVE-FIXED-NEXT:    sub sp, sp, #16
+; SVE-FIXED-NEXT:    .cfi_def_cfa_offset 16
+; SVE-FIXED-NEXT:    cmtst v1.8h, v1.8h, v1.8h
+; SVE-FIXED-NEXT:    index z4.b, #0, #1
+; SVE-FIXED-NEXT:    mov x9, sp
+; SVE-FIXED-NEXT:    str q0, [sp]
+; SVE-FIXED-NEXT:    xtn v3.8b, v1.8h
+; SVE-FIXED-NEXT:    umaxv h1, v1.8h
+; SVE-FIXED-NEXT:    and v3.8b, v3.8b, v4.8b
+; SVE-FIXED-NEXT:    umaxv b3, v3.8b
+; SVE-FIXED-NEXT:    fmov w8, s3
+; SVE-FIXED-NEXT:    bfi x9, x8, #1, #3
+; SVE-FIXED-NEXT:    fmov w8, s1
+; SVE-FIXED-NEXT:    ldr h0, [x9]
+; SVE-FIXED-NEXT:    tst w8, #0x1
+; SVE-FIXED-NEXT:    fcsel h0, h0, h2, ne
+; SVE-FIXED-NEXT:    add sp, sp, #16
+; SVE-FIXED-NEXT:    ret
+  %notzero = icmp ne <8 x i16> %mask, zeroinitializer
+  %res = call bfloat @llvm.experimental.vector.extract.last.active.v8bf16(<8 x bfloat> %data, <8 x i1> %notzero, bfloat %passthru)
+  ret bfloat %res
+}
+
 define float @extract_last_float(<4 x float> %data, <4 x i32> %mask, float %passthru) {
 ; NEON-FIXED-LABEL: extract_last_float:
 ; NEON-FIXED:       // %bb.0:
 ; NEON-FIXED-NEXT:    sub sp, sp, #16
 ; NEON-FIXED-NEXT:    .cfi_def_cfa_offset 16
 ; NEON-FIXED-NEXT:    cmtst v1.4s, v1.4s, v1.4s
-; NEON-FIXED-NEXT:    adrp x8, .LCPI4_0
+; NEON-FIXED-NEXT:    adrp x8, .LCPI6_0
 ; NEON-FIXED-NEXT:    mov x9, sp
-; NEON-FIXED-NEXT:    ldr d4, [x8, :lo12:.LCPI4_0]
+; NEON-FIXED-NEXT:    ldr d4, [x8, :lo12:.LCPI6_0]
 ; NEON-FIXED-NEXT:    str q0, [sp]
 ; NEON-FIXED-NEXT:    xtn v3.4h, v1.4s
 ; NEON-FIXED-NEXT:    umaxv s1, v1.4s
@@ -248,9 +348,9 @@ define double @extract_last_double(<2 x double> %data, <2 x i64> %mask, double %
 ; NEON-FIXED-NEXT:    sub sp, sp, #16
 ; NEON-FIXED-NEXT:    .cfi_def_cfa_offset 16
 ; NEON-FIXED-NEXT:    cmtst v1.2d, v1.2d, v1.2d
-; NEON-FIXED-NEXT:    adrp x8, .LCPI5_0
+; NEON-FIXED-NEXT:    adrp x8, .LCPI7_0
 ; NEON-FIXED-NEXT:    mov x9, sp
-; NEON-FIXED-NEXT:    ldr d4, [x8, :lo12:.LCPI5_0]
+; NEON-FIXED-NEXT:    ldr d4, [x8, :lo12:.LCPI7_0]
 ; NEON-FIXED-NEXT:    str q0, [sp]
 ; NEON-FIXED-NEXT:    xtn v3.2s, v1.2d
 ; NEON-FIXED-NEXT:    umaxv s1, v1.4s
@@ -293,17 +393,7 @@ define double @extract_last_double(<2 x double> %data, <2 x i64> %mask, double %
 define i8 @extract_last_i8_scalable(<vscale x 16 x i8> %data, <vscale x 16 x i1> %mask, i8 %passthru) #0 {
 ; CHECK-LABEL: extract_last_i8_scalable:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    index z1.b, #0, #1
-; CHECK-NEXT:    mov z2.b, #0 // =0x0
-; CHECK-NEXT:    ptrue p1.b
-; CHECK-NEXT:    sel z1.b, p0, z1.b, z2.b
-; CHECK-NEXT:    umaxv b1, p1, z1.b
-; CHECK-NEXT:    fmov w8, s1
-; CHECK-NEXT:    and x8, x8, #0xff
-; CHECK-NEXT:    whilels p1.b, xzr, x8
-; CHECK-NEXT:    ptest p0, p0.b
-; CHECK-NEXT:    lastb w8, p1, z0.b
-; CHECK-NEXT:    csel w0, w8, w0, ne
+; CHECK-NEXT:    clastb w0, p0, w0, z0.b
 ; CHECK-NEXT:    ret
   %res = call i8 @llvm.experimental.vector.extract.last.active.nxv16i8(<vscale x 16 x i8> %data, <vscale x 16 x i1> %mask, i8 %passthru)
   ret i8 %res
@@ -312,17 +402,7 @@ define i8 @extract_last_i8_scalable(<vscale x 16 x i8> %data, <vscale x 16 x i1>
 define i16 @extract_last_i16_scalable(<vscale x 8 x i16> %data, <vscale x 8 x i1> %mask, i16 %passthru) #0 {
 ; CHECK-LABEL: extract_last_i16_scalable:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    index z1.h, #0, #1
-; CHECK-NEXT:    mov z2.h, #0 // =0x0
-; CHECK-NEXT:    ptrue p1.h
-; CHECK-NEXT:    sel z1.h, p0, z1.h, z2.h
-; CHECK-NEXT:    umaxv h1, p1, z1.h
-; CHECK-NEXT:    fmov w8, s1
-; CHECK-NEXT:    and x8, x8, #0xffff
-; CHECK-NEXT:    whilels p2.h, xzr, x8
-; CHECK-NEXT:    ptest p1, p0.b
-; CHECK-NEXT:    lastb w8, p2, z0.h
-; CHECK-NEXT:    csel w0, w8, w0, ne
+; CHECK-NEXT:    clastb w0, p0, w0, z0.h
 ; CHECK-NEXT:    ret
   %res = call i16 @llvm.experimental.vector.extract.last.active.nxv8i16(<vscale x 8 x i16> %data, <vscale x 8 x i1> %mask, i16 %passthru)
   ret i16 %res
@@ -331,17 +411,7 @@ define i16 @extract_last_i16_scalable(<vscale x 8 x i16> %data, <vscale x 8 x i1
 define i32 @extract_last_i32_scalable(<vscale x 4 x i32> %data, <vscale x 4 x i1> %mask, i32 %passthru) #0 {
 ; CHECK-LABEL: extract_last_i32_scalable:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    index z1.s, #0, #1
-; CHECK-NEXT:    mov z2.s, #0 // =0x0
-; CHECK-NEXT:    ptrue p1.s
-; CHECK-NEXT:    sel z1.s, p0, z1.s, z2.s
-; CHECK-NEXT:    umaxv s1, p1, z1.s
-; CHECK-NEXT:    fmov w8, s1
-; CHECK-NEXT:    mov w8, w8
-; CHECK-NEXT:    whilels p2.s, xzr, x8
-; CHECK-NEXT:    ptest p1, p0.b
-; CHECK-NEXT:    lastb w8, p2, z0.s
-; CHECK-NEXT:    csel w0, w8, w0, ne
+; CHECK-NEXT:    clastb w0, p0, w0, z0.s
 ; CHECK-NEXT:    ret
   %res = call i32 @llvm.experimental.vector.extract.last.active.nxv4i32(<vscale x 4 x i32> %data, <vscale x 4 x i1> %mask, i32 %passthru)
   ret i32 %res
@@ -350,35 +420,37 @@ define i32 @extract_last_i32_scalable(<vscale x 4 x i32> %data, <vscale x 4 x i1
 define i64 @extract_last_i64_scalable(<vscale x 2 x i64> %data, <vscale x 2 x i1> %mask, i64 %passthru) #0 {
 ; CHECK-LABEL: extract_last_i64_scalable:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    index z1.d, #0, #1
-; CHECK-NEXT:    mov z2.d, #0 // =0x0
-; CHECK-NEXT:    ptrue p1.d
-; CHECK-NEXT:    sel z1.d, p0, z1.d, z2.d
-; CHECK-NEXT:    umaxv d1, p1, z1.d
-; CHECK-NEXT:    fmov x8, d1
-; CHECK-NEXT:    whilels p2.d, xzr, x8
-; CHECK-NEXT:    ptest p1, p0.b
-; CHECK-NEXT:    lastb x8, p2, z0.d
-; CHECK-NEXT:    csel x0, x8, x0, ne
+; CHECK-NEXT:    clastb x0, p0, x0, z0.d
 ; CHECK-NEXT:    ret
   %res = call i64 @llvm.experimental.vector.extract.last.active.nxv2i64(<vscale x 2 x i64> %data, <vscale x 2 x i1> %mask, i64 %passthru)
   ret i64 %res
 }
 
+define half @extract_last_half_scalable(<vscale x 8 x half> %data, <vscale x 8 x i1> %mask, half %passthru) #0 {
+; CHECK-LABEL: extract_last_half_scalable:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    clastb h1, p0, h1, z0.h
+; CHECK-NEXT:    fmov s0, s1
+; CHECK-NEXT:    ret
+  %res = call half @llvm.experimental.vector.extract.last.active.nxv8f16(<vscale x 8 x half> %data, <vscale x 8 x i1> %mask, half %passthru)
+  ret half %res
+}
+
+define bfloat @extract_last_bfloat_scalable(<vscale x 8 x bfloat> %data, <vscale x 8 x i1> %mask, bfloat %passthru) #0 {
+; CHECK-LABEL: extract_last_bfloat_scalable:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    clastb h1, p0, h1, z0.h
+; CHECK-NEXT:    fmov s0, s1
+; CHECK-NEXT:    ret
+  %res = call bfloat @llvm.experimental.vector.extract.last.active.nxv8bf16(<vscale x 8 x bfloat> %data, <vscale x 8 x i1> %mask, bfloat %passthru)
+  ret bfloat %res
+}
+
 define float @extract_last_float_scalable(<vscale x 4 x float> %data, <vscale x 4 x i1> %mask, float %passthru) #0 {
 ; CHECK-LABEL: extract_last_float_scalable:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    index z2.s, #0, #1
-; CHECK-NEXT:    mov z3.s, #0 // =0x0
-; CHECK-NEXT:    ptrue p1.s
-; CHECK-NEXT:    sel z2.s, p0, z2.s, z3.s
-; CHECK-NEXT:    umaxv s2, p1, z2.s
-; CHECK-NEXT:    fmov w8, s2
-; CHECK-NEXT:    mov w8, w8
-; CHECK-NEXT:    whilels p2.s, xzr, x8
-; CHECK-NEXT:    ptest p1, p0.b
-; CHECK-NEXT:    lastb s0, p2, z0.s
-; CHECK-NEXT:    fcsel s0, s0, s1, ne
+; CHECK-NEXT:    clastb s1, p0, s1, z0.s
+; CHECK-NEXT:    fmov s0, s1
 ; CHECK-NEXT:    ret
   %res = call float @llvm.experimental.vector.extract.last.active.nxv4f32(<vscale x 4 x float> %data, <vscale x 4 x i1> %mask, float %passthru)
   ret float %res
@@ -387,16 +459,8 @@ define float @extract_last_float_scalable(<vscale x 4 x float> %data, <vscale x
 define double @extract_last_double_scalable(<vscale x 2 x double> %data, <vscale x 2 x i1> %mask, double %passthru) #0 {
 ; CHECK-LABEL: extract_last_double_scalable:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    index z2.d, #0, #1
-; CHECK-NEXT:    mov z3.d, #0 // =0x0
-; CHECK-NEXT:    ptrue p1.d
-; CHECK-NEXT:    sel z2.d, p0, z2.d, z3.d
-; CHECK-NEXT:    umaxv d2, p1, z2.d
-; CHECK-NEXT:    fmov x8, d2
-; CHECK-NEXT:    whilels p2.d, xzr, x8
-; CHECK-NEXT:    ptest p1, p0.b
-; CHECK-NEXT:    lastb d0, p2, z0.d
-; CHECK-NEXT:    fcsel d0, d0, d1, ne
+; CHECK-NEXT:    clastb d1, p0, d1, z0.d
+; CHECK-NEXT:    fmov d0, d1
 ; CHECK-NEXT:    ret
   %res = call double @llvm.experimental.vector.extract.last.active.nxv2f64(<vscale x 2 x double> %data, <vscale x 2 x i1> %mask, double %passthru)
   ret double %res
@@ -406,31 +470,42 @@ define double @extract_last_double_scalable(<vscale x 2 x double> %data, <vscale
 define i8 @extract_last_i8_scalable_poison_passthru(<vscale x 16 x i8> %data, <vscale x 16 x i1> %mask) #0 {
 ; CHECK-LABEL: extract_last_i8_scalable_poison_passthru:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    index z1.b, #0, #1
-; CHECK-NEXT:    mov z2.b, #0 // =0x0
-; CHECK-NEXT:    sel z1.b, p0, z1.b, z2.b
-; CHECK-NEXT:    ptrue p0.b
-; CHECK-NEXT:    umaxv b1, p0, z1.b
-; CHECK-NEXT:    fmov w8, s1
-; CHECK-NEXT:    and x8, x8, #0xff
-; CHECK-NEXT:    whilels p0.b, xzr, x8
 ; CHECK-NEXT:    lastb w0, p0, z0.b
 ; CHECK-NEXT:    ret
   %res = call i8 @llvm.experimental.vector.extract.last.active.nxv16i8(<vscale x 16 x i8> %data, <vscale x 16 x i1> %mask, i8 poison)
   ret i8 %res
 }
 
+;; (c)lastb doesn't exist for predicate types; check we get functional codegen
+define i1 @extract_last_i1_scalable(<vscale x 16 x i1> %data, <vscale x 16 x i1> %mask) #0 {
+; CHECK-LABEL: extract_last_i1_scalable:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mov z0.b, p0/z, #1 // =0x1
+; CHECK-NEXT:    ptest p1, p1.b
+; CHECK-NEXT:    cset w9, ne
+; CHECK-NEXT:    lastb w8, p1, z0.b
+; CHECK-NEXT:    and w0, w9, w8
+; CHECK-NEXT:    ret
+  %res = call i1 @llvm.experimental.vector.extract.last.active.nxv16i1(<vscale x 16 x i1> %data, <vscale x 16 x i1> %mask, i1 false)
+  ret i1 %res
+}
+
 declare i8 @llvm.experimental.vector.extract.last.active.v16i8(<16 x i8>, <16 x i1>, i8)
 declare i16 @llvm.experimental.vector.extract.last.active.v8i16(<8 x i16>, <8 x i1>, i16)
 declare i32 @llvm.experimental.vector.extract.last.active.v4i32(<4 x i32>, <4 x i1>, i32)
 declare i64 @llvm.experimental.vector.extract.last.active.v2i64(<2 x i64>, <2 x i1>, i64)
+declare half @llvm.experimental.vector.extract.last.active.v8f16(<8 x half>, <8 x i1>, half)
+declare bfloat @llvm.experimental.vector.extract.last.active.v8bf16(<8 x bfloat>, <8 x i1>, bfloat)
 declare float @llvm.experimental.vector.extract.last.active.v4f32(<4 x float>, <4 x i1>, float)
 declare double @llvm.experimental.vector.extract.last.active.v2f64(<2 x double>, <2 x i1>, double)
 declare i8 @llvm.experimental.vector.extract.last.active.nxv16i8(<vscale x 16 x i8>, <vscale x 16 x i1>, i8)
 declare i16 @llvm.experimental.vector.extract.last.active.nxv8i16(<vscale x 8 x i16>, <vscale x 8 x i1>, i16)
 declare i32 @llvm.experimental.vector.extract.last.active.nxv4i32(<vscale x 4 x i32>, <vscale x 4 x i1>, i32)
 declare i64 @llvm.experimental.vector.extract.last.active.nxv2i64(<vscale x 2 x i64>, <vscale x 2 x i1>, i64)
+declare half @llvm.experimental.vector.extract.last.active.nxv8f16(<vscale x 8 x half>, <vscale x 8 x i1>, half)
+declare bfloat @llvm.experimental.vector.extract.last.active.nxv8bf16(<vscale x 8 x bfloat>, <vscale x 8 x i1>, bfloat)
 declare float @llvm.experimental.vector.extract.last.active.nxv4f32(<vscale x 4 x float>, <vscale x 4 x i1>, float)
 declare double @llvm.experimental.vector.extract.last.active.nxv2f64(<vscale x 2 x double>, <vscale x 2 x i1>, double)
+declare i1 @llvm.experimental.vector.extract.last.active.nxv16i1(<vscale x 16 x i1>, <vscale x 16 x i1>, i1)
 
 attributes #0 = { "target-features"="+sve" vscale_range(1, 16) }
diff --git a/llvm/test/CodeGen/AArch64/win-import-call-optimization-nocalls.ll b/llvm/test/CodeGen/AArch64/win-import-call-optimization-nocalls.ll
index 81d6d6369dcbf..b17f2d113c0d2 100644
--- a/llvm/test/CodeGen/AArch64/win-import-call-optimization-nocalls.ll
+++ b/llvm/test/CodeGen/AArch64/win-import-call-optimization-nocalls.ll
@@ -1,4 +1,4 @@
-; RUN: llc -mtriple=aarch64-pc-windows-msvc -aarch64-win-import-call-optimization < %s | FileCheck %s
+; RUN: llc -mtriple=aarch64-pc-windows-msvc < %s | FileCheck %s
 
 define dso_local void @normal_call() local_unnamed_addr {
 entry:
@@ -16,3 +16,6 @@ declare void @a() local_unnamed_addr
 ; CHECK-LABEL  .section   .impcall,"yi"
 ; CHECK-NEXT   .asciz  "Imp_Call_V1"
 ; CHECK-NOT    .secnum
+
+!llvm.module.flags = !{!0}
+!0 = !{i32 1, !"import-call-optimization", i32 1}
diff --git a/llvm/test/CodeGen/AArch64/win-import-call-optimization.ll b/llvm/test/CodeGen/AArch64/win-import-call-optimization.ll
index 6bb118ba1e159..2cf0f8bb6baa5 100644
--- a/llvm/test/CodeGen/AArch64/win-import-call-optimization.ll
+++ b/llvm/test/CodeGen/AArch64/win-import-call-optimization.ll
@@ -1,7 +1,4 @@
-; RUN: llc -mtriple=aarch64-pc-windows-msvc -aarch64-win-import-call-optimization < %s | FileCheck %s --check-prefix=CHECK-ENABLED
-; RUN: llc -mtriple=aarch64-pc-windows-msvc < %s | FileCheck %s --check-prefix=CHECK-DISABLED
-
-; CHECK-DISABLED-NOT: .section        .impcall
+; RUN: llc -mtriple=aarch64-pc-windows-msvc < %s | FileCheck %s --check-prefix=CHECK
 
 define dso_local void @normal_call() local_unnamed_addr section "nc_sect" {
 entry:
@@ -9,40 +6,43 @@ entry:
   call void @a()
   ret void
 }
-; CHECK-ENABLED-LABEL:  normal_call:
-; CHECK-ENABLED:        adrp    [[ADRPREG:x[0-9]+]], __imp_a
-; CHECK-ENABLED-NEXT:   ldr     [[LDRREG:x[0-9]+]], [[[ADRPREG]], :lo12:__imp_a]
-; CHECK-ENABLED-NEXT:   .Limpcall0:
-; CHECK-ENABLED-NEXT:   blr     [[LDRREG]]
-; CHECK-ENABLED-NEXT:   .Limpcall1:
-; CHECK-ENABLED-NEXT:   blr     [[LDRREG]]
+; CHECK-LABEL:  normal_call:
+; CHECK:        adrp    [[ADRPREG:x[0-9]+]], __imp_a
+; CHECK-NEXT:   ldr     [[LDRREG:x[0-9]+]], [[[ADRPREG]], :lo12:__imp_a]
+; CHECK-NEXT:   .Limpcall0:
+; CHECK-NEXT:   blr     [[LDRREG]]
+; CHECK-NEXT:   .Limpcall1:
+; CHECK-NEXT:   blr     [[LDRREG]]
 
 define dso_local void @tail_call() local_unnamed_addr section "tc_sect" {
 entry:
   tail call void @b()
   ret void
 }
-; CHECK-ENABLED-LABEL:  tail_call:
-; CHECK-ENABLED:        adrp    [[ADRPREG:x[0-9]+]], __imp_b
-; CHECK-ENABLED-NEXT:   ldr     [[LDRREG:x[0-9]+]], [[[ADRPREG]], :lo12:__imp_b]
-; CHECK-ENABLED-NEXT:   .Limpcall2:
-; CHECK-ENABLED-NEXT:   br      [[LDRREG]]
+; CHECK-LABEL:  tail_call:
+; CHECK:        adrp    [[ADRPREG:x[0-9]+]], __imp_b
+; CHECK-NEXT:   ldr     [[LDRREG:x[0-9]+]], [[[ADRPREG]], :lo12:__imp_b]
+; CHECK-NEXT:   .Limpcall2:
+; CHECK-NEXT:   br      [[LDRREG]]
 
 declare dllimport void @a() local_unnamed_addr
 declare dllimport void @b() local_unnamed_addr
 
-; CHECK-ENABLED-LABEL  .section   .impcall,"yi"
-; CHECK-ENABLED-NEXT   .asciz  "Imp_Call_V1"
-; CHECK-ENABLED-NEXT   .word   32
-; CHECK-ENABLED-NEXT   .secnum nc_sect
-; CHECK-ENABLED-NEXT   .word   19
-; CHECK-ENABLED-NEXT   .secoffset      .Limpcall0
-; CHECK-ENABLED-NEXT   .symidx __imp_a
-; CHECK-ENABLED-NEXT   .word   19
-; CHECK-ENABLED-NEXT   .secoffset      .Limpcall1
-; CHECK-ENABLED-NEXT   .symidx __imp_a
-; CHECK-ENABLED-NEXT   .word   20
-; CHECK-ENABLED-NEXT   .secnum tc_sect
-; CHECK-ENABLED-NEXT   .word   19
-; CHECK-ENABLED-NEXT   .secoffset      .Limpcall2
-; CHECK-ENABLED-NEXT   .symidx __imp_b
+; CHECK-LABEL  .section   .impcall,"yi"
+; CHECK-NEXT   .asciz  "Imp_Call_V1"
+; CHECK-NEXT   .word   32
+; CHECK-NEXT   .secnum nc_sect
+; CHECK-NEXT   .word   19
+; CHECK-NEXT   .secoffset      .Limpcall0
+; CHECK-NEXT   .symidx __imp_a
+; CHECK-NEXT   .word   19
+; CHECK-NEXT   .secoffset      .Limpcall1
+; CHECK-NEXT   .symidx __imp_a
+; CHECK-NEXT   .word   20
+; CHECK-NEXT   .secnum tc_sect
+; CHECK-NEXT   .word   19
+; CHECK-NEXT   .secoffset      .Limpcall2
+; CHECK-NEXT   .symidx __imp_b
+
+!llvm.module.flags = !{!0}
+!0 = !{i32 1, !"import-call-optimization", i32 1}
diff --git a/llvm/test/CodeGen/AArch64/wincfi-missing-seh-directives.ll b/llvm/test/CodeGen/AArch64/wincfi-missing-seh-directives.ll
new file mode 100644
index 0000000000000..2002c37cb2528
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/wincfi-missing-seh-directives.ll
@@ -0,0 +1,86 @@
+; RUN: llc -mtriple=aarch64-windows %s --filetype obj -o /dev/null
+; RUN: llc -mtriple=aarch64-windows %s --filetype asm -o - | FileCheck %s
+
+; Check that it doesn't crash and that each instruction in the
+; prologue has a corresponding seh directive.
+;
+; CHECK-NOT: error: Incorrect size for
+; CHECK: foo:
+; CHECK: .seh_proc foo
+; CHECK: sub     sp, sp, #288
+; CHECK: .seh_stackalloc 288
+; CHECK: str     x19, [sp]                       // 8-byte Folded Spill
+; CHECK: .seh_save_reg   x19, 0
+; CHECK: str     x21, [sp, #8]                   // 8-byte Folded Spill
+; CHECK: .seh_save_reg   x21, 8
+; CHECK: stp     x23, x24, [sp, #16]             // 16-byte Folded Spill
+; CHECK: .seh_save_regp  x23, 16
+; CHECK: stp     x25, x26, [sp, #32]             // 16-byte Folded Spill
+; CHECK: .seh_save_regp  x25, 32
+; CHECK: stp     x27, x28, [sp, #48]             // 16-byte Folded Spill
+; CHECK: .seh_save_regp  x27, 48
+; CHECK: stp     x29, x30, [sp, #64]             // 16-byte Folded Spill
+; CHECK: .seh_save_fplr  64
+; CHECK: sub     sp, sp, #224
+; CHECK: .seh_stackalloc 224
+; CHECK: .seh_endprologue
+
+target datalayout = "e-m:w-p:64:64-i32:32-i64:64-i128:128-n32:64-S128-Fn32"
+target triple = "aarch64-unknown-windows-msvc19.42.34436"
+
+%swift.refcounted = type { ptr, i64 }
+%TScA_pSg = type <{ [16 x i8] }>
+%T5repro4TestVSg = type <{ [32 x i8] }>
+%T5repro4TestV = type <{ %TSS, %TSS }>
+%TSS = type <{ %Ts11_StringGutsV }>
+%Ts11_StringGutsV = type <{ %Ts13_StringObjectV }>
+%Ts13_StringObjectV = type <{ %Ts6UInt64V, ptr }>
+%Ts6UInt64V = type <{ i64 }>
+
+declare swiftcc ptr @swift_task_alloc()
+
+declare swifttailcc void @bar(ptr, ptr, i64, i64, i64, ptr, i64, i64, i64, i64, i64, ptr, i64, ptr, i64, ptr, i64, ptr, i64, ptr, i64, ptr, i64, ptr, i64, ptr, i64, ptr, i64, ptr, i64, ptr, i64, ptr, i64, ptr)
+
+define swifttailcc void @foo(ptr %0, ptr swiftasync %1, ptr swiftself %2, ptr %3, ptr %._guts2._object._object, ptr %.rid4._guts._object._object, ptr %4, ptr %.idx8, ptr %.idx8._guts._object._object, ptr %5, ptr %.rid9._guts._object._object, ptr %6) {
+entry:
+  %7 = load i64, ptr null, align 8
+  %8 = load i64, ptr %3, align 8
+  %9 = getelementptr <{ %swift.refcounted, %TScA_pSg, %TSS, %T5repro4TestVSg, %T5repro4TestV, %TSS, %TSS, %TSS, %T5repro4TestV, %TSS, %T5repro4TestV, %T5repro4TestV, %TSS }>, ptr %2, i32 0, i32 2
+  %10 = load i64, ptr %9, align 8
+  %11 = load ptr, ptr %1, align 8
+  %12 = getelementptr <{ %swift.refcounted, %TScA_pSg, %TSS, %T5repro4TestVSg, %T5repro4TestV, %TSS, %TSS, %TSS, %T5repro4TestV, %TSS, %T5repro4TestV, %T5repro4TestV, %TSS }>, ptr %2, i32 0, i32 3
+  %13 = load i64, ptr %.rid9._guts._object._object, align 8
+  %14 = load i64, ptr %.idx8._guts._object._object, align 8
+  %15 = load i64, ptr %5, align 8
+  %16 = getelementptr { i64, i64, i64, i64 }, ptr %12, i32 0, i32 3
+  %17 = load i64, ptr %16, align 8
+  %18 = getelementptr <{ %swift.refcounted, %TScA_pSg, %TSS, %T5repro4TestVSg, %T5repro4TestV, %TSS, %TSS, %TSS, %T5repro4TestV, %TSS, %T5repro4TestV, %T5repro4TestV, %TSS }>, ptr %2, i32 0, i32 4
+  %19 = load i64, ptr %18, align 8
+  %.rid._guts._object._object = getelementptr %Ts13_StringObjectV, ptr %18, i32 0, i32 1
+  %20 = load ptr, ptr %.rid._guts._object._object, align 8
+  %21 = load i64, ptr %.rid4._guts._object._object, align 8
+  %22 = load i64, ptr %0, align 8
+  %23 = load ptr, ptr %6, align 8
+  %24 = load i64, ptr %2, align 8
+  %25 = load ptr, ptr %._guts2._object._object, align 8
+  %26 = getelementptr <{ %swift.refcounted, %TScA_pSg, %TSS, %T5repro4TestVSg, %T5repro4TestV, %TSS, %TSS, %TSS, %T5repro4TestV, %TSS, %T5repro4TestV, %T5repro4TestV, %TSS }>, ptr %2, i32 0, i32 7
+  %27 = load i64, ptr %26, align 8
+  %._guts3._object._object = getelementptr %Ts13_StringObjectV, ptr %26, i32 0, i32 1
+  %28 = load ptr, ptr %._guts3._object._object, align 8
+  %29 = getelementptr <{ %swift.refcounted, %TScA_pSg, %TSS, %T5repro4TestVSg, %T5repro4TestV, %TSS, %TSS, %TSS, %T5repro4TestV, %TSS, %T5repro4TestV, %T5repro4TestV, %TSS }>, ptr %2, i32 0, i32 8
+  %30 = load i64, ptr %29, align 8
+  %.idx5 = getelementptr %T5repro4TestV, ptr %29, i32 0, i32 1
+  %31 = load i64, ptr %.idx5, align 8
+  %.idx5._guts._object._object = getelementptr %Ts13_StringObjectV, ptr %.idx5, i32 0, i32 1
+  %32 = load ptr, ptr %.idx5._guts._object._object, align 8
+  %33 = getelementptr <{ %swift.refcounted, %TScA_pSg, %TSS, %T5repro4TestVSg, %T5repro4TestV, %TSS, %TSS, %TSS, %T5repro4TestV, %TSS, %T5repro4TestV, %T5repro4TestV, %TSS }>, ptr %2, i32 0, i32 9
+  %34 = load i64, ptr %33, align 8
+  %35 = load i64, ptr %4, align 8
+  %36 = load i64, ptr %.idx8, align 8
+  %37 = load i64, ptr %1, align 8
+  %38 = call swiftcc ptr @swift_task_alloc()
+  store ptr null, ptr %3, align 8
+  store ptr null, ptr %4, align 8
+  musttail call swifttailcc void @bar(ptr null, ptr swiftasync %.rid4._guts._object._object, i64 %7, i64 %8, i64 %10, ptr %5, i64 %13, i64 %14, i64 %15, i64 %17, i64 %19, ptr %20, i64 %21, ptr %.idx8, i64 %22, ptr %23, i64 %24, ptr %25, i64 %27, ptr %28, i64 %30, ptr %.idx8._guts._object._object, i64 %31, ptr %32, i64 %34, ptr %._guts2._object._object, i64 %35, ptr %2, i64 %36, ptr %1, i64 %37, ptr %0, i64 0, ptr null, i64 0, ptr null)
+  ret void
+}
diff --git a/llvm/test/CodeGen/AArch64/xbfiz.ll b/llvm/test/CodeGen/AArch64/xbfiz.ll
index b777ddcb7efcc..d3bebf7c6637c 100644
--- a/llvm/test/CodeGen/AArch64/xbfiz.ll
+++ b/llvm/test/CodeGen/AArch64/xbfiz.ll
@@ -69,3 +69,37 @@ define i64 @lsl32_not_ubfiz64(i64 %v) {
   %and = and i64 %shl, 4294967295
   ret i64 %and
 }
+
+define i64 @lsl_zext_i8_i64(i8 %b) {
+; CHECK-LABEL: lsl_zext_i8_i64:
+; CHECK:    ubfiz x0, x0, #1, #8
+  %1 = zext i8 %b to i64
+  %2 = shl i64 %1, 1
+  ret i64 %2
+}
+
+define i64 @lsl_zext_i16_i64(i16 %b) {
+; CHECK-LABEL: lsl_zext_i16_i64:
+; CHECK:    ubfiz x0, x0, #1, #16
+  %1 = zext i16 %b to i64
+  %2 = shl i64 %1, 1
+  ret i64 %2
+}
+
+; Regression test for:
+; https://github.com/llvm/llvm-project/pull/118974#issuecomment-2598521878
+; that exposed infinite loop in DAGCombiner.
+define void @_f(ptr %0, ptr %1, i64 %2) {
+; CHECK-LABEL: @_f
+; CHECK-NOT: ubfiz
+  store i64 -2401053089408754003, ptr %1, align 8
+  %4 = and i64 %2, -2401053089408754003
+  %5 = shl i64 %4, 1
+  store i64 %5, ptr %0, align 1
+  %6 = lshr i64 %4, 54
+  %7 = shl i64 %2, 10
+  %8 = and i64 %7, 131072
+  %9 = or i64 %8, %6
+  store i64 %9, ptr %1, align 1
+  ret void
+}
diff --git a/llvm/test/CodeGen/AArch64/zeroing-forms-ext.ll b/llvm/test/CodeGen/AArch64/zeroing-forms-ext.ll
new file mode 100644
index 0000000000000..b29805c2b8f05
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/zeroing-forms-ext.ll
@@ -0,0 +1,1028 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc -mattr=+sve    < %s | FileCheck %s
+; RUN: llc -mattr=+sve2p2 < %s | FileCheck %s -check-prefix CHECK-2p2
+
+; RUN: llc -mattr=+sme    -force-streaming < %s | FileCheck %s
+; RUN: llc -mattr=+sme2p2 -force-streaming < %s | FileCheck %s -check-prefix CHECK-2p2
+
+target triple = "aarch64-linux"
+
+define <vscale x 8 x i16> @test_svextb_s16_x_1(<vscale x 8 x i1> %pg, <vscale x 8 x i16> %x) {
+; CHECK-LABEL: test_svextb_s16_x_1:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    sxtb z0.h, p0/m, z0.h
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svextb_s16_x_1:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    sxtb z0.h, p0/z, z0.h
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 8 x i16> @llvm.aarch64.sve.sxtb.nxv8i16(<vscale x 8 x i16> poison, <vscale x 8 x i1> %pg, <vscale x 8 x i16> %x)
+  ret <vscale x 8 x i16> %0
+}
+
+define <vscale x 8 x i16> @test_svextb_s16_x_2(<vscale x 8 x i1> %pg, double %z0, <vscale x 8 x i16> %x) {
+; CHECK-LABEL: test_svextb_s16_x_2:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    movprfx z0, z1
+; CHECK-NEXT:    sxtb z0.h, p0/m, z1.h
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svextb_s16_x_2:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    sxtb z0.h, p0/z, z1.h
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 8 x i16> @llvm.aarch64.sve.sxtb.nxv8i16(<vscale x 8 x i16> poison, <vscale x 8 x i1> %pg, <vscale x 8 x i16> %x)
+  ret <vscale x 8 x i16> %0
+}
+
+define <vscale x 8 x i16> @test_svextb_s16_z(<vscale x 8 x i1> %pg, double %z0, <vscale x 8 x i16> %x) {
+; CHECK-LABEL: test_svextb_s16_z:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    mov z0.h, #0 // =0x0
+; CHECK-NEXT:    sxtb z0.h, p0/m, z1.h
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svextb_s16_z:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    sxtb z0.h, p0/z, z1.h
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 8 x i16> @llvm.aarch64.sve.sxtb.nxv8i16(<vscale x 8 x i16> zeroinitializer, <vscale x 8 x i1> %pg, <vscale x 8 x i16> %x)
+  ret <vscale x 8 x i16> %0
+}
+
+define <vscale x 4 x i32> @test_svextb_s32_x_1(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %x) {
+; CHECK-LABEL: test_svextb_s32_x_1:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    sxtb z0.s, p0/m, z0.s
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svextb_s32_x_1:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    sxtb z0.s, p0/z, z0.s
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 4 x i32> @llvm.aarch64.sve.sxtb.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i1> %pg, <vscale x 4 x i32> %x)
+  ret <vscale x 4 x i32> %0
+}
+
+define <vscale x 4 x i32> @test_svextb_s32_x_2(<vscale x 4 x i1> %pg, double %z0, <vscale x 4 x i32> %x) {
+; CHECK-LABEL: test_svextb_s32_x_2:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    movprfx z0, z1
+; CHECK-NEXT:    sxtb z0.s, p0/m, z1.s
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svextb_s32_x_2:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    sxtb z0.s, p0/z, z1.s
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 4 x i32> @llvm.aarch64.sve.sxtb.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i1> %pg, <vscale x 4 x i32> %x)
+  ret <vscale x 4 x i32> %0
+}
+
+define <vscale x 4 x i32> @test_svextb_s32_z(<vscale x 4 x i1> %pg, double %z0, <vscale x 4 x i32> %x) {
+; CHECK-LABEL: test_svextb_s32_z:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    mov z0.s, #0 // =0x0
+; CHECK-NEXT:    sxtb z0.s, p0/m, z1.s
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svextb_s32_z:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    sxtb z0.s, p0/z, z1.s
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 4 x i32> @llvm.aarch64.sve.sxtb.nxv4i32(<vscale x 4 x i32> zeroinitializer, <vscale x 4 x i1> %pg, <vscale x 4 x i32> %x)
+  ret <vscale x 4 x i32> %0
+}
+
+define <vscale x 2 x i64> @test_svextb_s64_x_1(<vscale x 2 x i1> %pg, <vscale x 2 x i64> %x) {
+; CHECK-LABEL: test_svextb_s64_x_1:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    sxtb z0.d, p0/m, z0.d
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svextb_s64_x_1:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    sxtb z0.d, p0/z, z0.d
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 2 x i64> @llvm.aarch64.sve.sxtb.nxv2i64(<vscale x 2 x i64> poison, <vscale x 2 x i1> %pg, <vscale x 2 x i64> %x)
+  ret <vscale x 2 x i64> %0
+}
+
+define <vscale x 2 x i64> @test_svextb_s64_x_2(<vscale x 2 x i1> %pg, double %z0, <vscale x 2 x i64> %x) {
+; CHECK-LABEL: test_svextb_s64_x_2:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    movprfx z0, z1
+; CHECK-NEXT:    sxtb z0.d, p0/m, z1.d
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svextb_s64_x_2:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    sxtb z0.d, p0/z, z1.d
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 2 x i64> @llvm.aarch64.sve.sxtb.nxv2i64(<vscale x 2 x i64> poison, <vscale x 2 x i1> %pg, <vscale x 2 x i64> %x)
+  ret <vscale x 2 x i64> %0
+}
+
+define <vscale x 2 x i64> @test_svextb_s64_z(<vscale x 2 x i1> %pg, double %z0, <vscale x 2 x i64> %x) {
+; CHECK-LABEL: test_svextb_s64_z:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    mov z0.d, #0 // =0x0
+; CHECK-NEXT:    sxtb z0.d, p0/m, z1.d
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svextb_s64_z:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    sxtb z0.d, p0/z, z1.d
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 2 x i64> @llvm.aarch64.sve.sxtb.nxv2i64(<vscale x 2 x i64> zeroinitializer, <vscale x 2 x i1> %pg, <vscale x 2 x i64> %x)
+  ret <vscale x 2 x i64> %0
+}
+
+define <vscale x 8 x i16> @test_svextb_u16_x_1(<vscale x 8 x i1> %pg, <vscale x 8 x i16> %x) {
+; CHECK-LABEL: test_svextb_u16_x_1:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    uxtb z0.h, p0/m, z0.h
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svextb_u16_x_1:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    uxtb z0.h, p0/z, z0.h
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 8 x i16> @llvm.aarch64.sve.uxtb.nxv8i16(<vscale x 8 x i16> poison, <vscale x 8 x i1> %pg, <vscale x 8 x i16> %x)
+  ret <vscale x 8 x i16> %0
+}
+
+define <vscale x 8 x i16> @test_svextb_u16_x_2(<vscale x 8 x i1> %pg, double %z0, <vscale x 8 x i16> %x) {
+; CHECK-LABEL: test_svextb_u16_x_2:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    movprfx z0, z1
+; CHECK-NEXT:    uxtb z0.h, p0/m, z1.h
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svextb_u16_x_2:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    uxtb z0.h, p0/z, z1.h
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 8 x i16> @llvm.aarch64.sve.uxtb.nxv8i16(<vscale x 8 x i16> poison, <vscale x 8 x i1> %pg, <vscale x 8 x i16> %x)
+  ret <vscale x 8 x i16> %0
+}
+
+define <vscale x 8 x i16> @test_svextb_u16_z(<vscale x 8 x i1> %pg, double %z0, <vscale x 8 x i16> %x) {
+; CHECK-LABEL: test_svextb_u16_z:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    mov z0.h, #0 // =0x0
+; CHECK-NEXT:    uxtb z0.h, p0/m, z1.h
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svextb_u16_z:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    uxtb z0.h, p0/z, z1.h
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 8 x i16> @llvm.aarch64.sve.uxtb.nxv8i16(<vscale x 8 x i16> zeroinitializer, <vscale x 8 x i1> %pg, <vscale x 8 x i16> %x)
+  ret <vscale x 8 x i16> %0
+}
+
+define <vscale x 4 x i32> @test_svextb_u32_x_1(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %x) {
+; CHECK-LABEL: test_svextb_u32_x_1:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    uxtb z0.s, p0/m, z0.s
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svextb_u32_x_1:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    uxtb z0.s, p0/z, z0.s
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 4 x i32> @llvm.aarch64.sve.uxtb.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i1> %pg, <vscale x 4 x i32> %x)
+  ret <vscale x 4 x i32> %0
+}
+
+define <vscale x 4 x i32> @test_svextb_u32_x_2(<vscale x 4 x i1> %pg, double %z0, <vscale x 4 x i32> %x) {
+; CHECK-LABEL: test_svextb_u32_x_2:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    movprfx z0, z1
+; CHECK-NEXT:    uxtb z0.s, p0/m, z1.s
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svextb_u32_x_2:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    uxtb z0.s, p0/z, z1.s
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 4 x i32> @llvm.aarch64.sve.uxtb.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i1> %pg, <vscale x 4 x i32> %x)
+  ret <vscale x 4 x i32> %0
+}
+
+define <vscale x 4 x i32> @test_svextb_u32_z(<vscale x 4 x i1> %pg, double %z0, <vscale x 4 x i32> %x) {
+; CHECK-LABEL: test_svextb_u32_z:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    mov z0.s, #0 // =0x0
+; CHECK-NEXT:    uxtb z0.s, p0/m, z1.s
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svextb_u32_z:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    uxtb z0.s, p0/z, z1.s
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 4 x i32> @llvm.aarch64.sve.uxtb.nxv4i32(<vscale x 4 x i32> zeroinitializer, <vscale x 4 x i1> %pg, <vscale x 4 x i32> %x)
+  ret <vscale x 4 x i32> %0
+}
+
+define <vscale x 2 x i64> @test_svextb_u64_x_1(<vscale x 2 x i1> %pg, <vscale x 2 x i64> %x) {
+; CHECK-LABEL: test_svextb_u64_x_1:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    uxtb z0.d, p0/m, z0.d
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svextb_u64_x_1:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    uxtb z0.d, p0/z, z0.d
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 2 x i64> @llvm.aarch64.sve.uxtb.nxv2i64(<vscale x 2 x i64> poison, <vscale x 2 x i1> %pg, <vscale x 2 x i64> %x)
+  ret <vscale x 2 x i64> %0
+}
+
+define <vscale x 2 x i64> @test_svextb_u64_x_2(<vscale x 2 x i1> %pg, double %z0, <vscale x 2 x i64> %x) {
+; CHECK-LABEL: test_svextb_u64_x_2:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    movprfx z0, z1
+; CHECK-NEXT:    uxtb z0.d, p0/m, z1.d
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svextb_u64_x_2:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    uxtb z0.d, p0/z, z1.d
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 2 x i64> @llvm.aarch64.sve.uxtb.nxv2i64(<vscale x 2 x i64> poison, <vscale x 2 x i1> %pg, <vscale x 2 x i64> %x)
+  ret <vscale x 2 x i64> %0
+}
+
+define <vscale x 2 x i64> @test_svextb_u64_z(<vscale x 2 x i1> %pg, double %z0, <vscale x 2 x i64> %x) {
+; CHECK-LABEL: test_svextb_u64_z:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    mov z0.d, #0 // =0x0
+; CHECK-NEXT:    uxtb z0.d, p0/m, z1.d
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svextb_u64_z:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    uxtb z0.d, p0/z, z1.d
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 2 x i64> @llvm.aarch64.sve.uxtb.nxv2i64(<vscale x 2 x i64> zeroinitializer, <vscale x 2 x i1> %pg, <vscale x 2 x i64> %x)
+  ret <vscale x 2 x i64> %0
+}
+
+define <vscale x 4 x i32> @test_svexth_s32_x_1(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %x) {
+; CHECK-LABEL: test_svexth_s32_x_1:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    sxth z0.s, p0/m, z0.s
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svexth_s32_x_1:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    sxth z0.s, p0/z, z0.s
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 4 x i32> @llvm.aarch64.sve.sxth.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i1> %pg, <vscale x 4 x i32> %x)
+  ret <vscale x 4 x i32> %0
+}
+
+define <vscale x 4 x i32> @test_svexth_s32_x_2(<vscale x 4 x i1> %pg, double %z0, <vscale x 4 x i32> %x) {
+; CHECK-LABEL: test_svexth_s32_x_2:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    movprfx z0, z1
+; CHECK-NEXT:    sxth z0.s, p0/m, z1.s
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svexth_s32_x_2:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    sxth z0.s, p0/z, z1.s
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 4 x i32> @llvm.aarch64.sve.sxth.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i1> %pg, <vscale x 4 x i32> %x)
+  ret <vscale x 4 x i32> %0
+}
+
+define <vscale x 4 x i32> @test_svexth_s32_z(<vscale x 4 x i1> %pg, double %z0, <vscale x 4 x i32> %x) {
+; CHECK-LABEL: test_svexth_s32_z:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    mov z0.s, #0 // =0x0
+; CHECK-NEXT:    sxth z0.s, p0/m, z1.s
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svexth_s32_z:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    sxth z0.s, p0/z, z1.s
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 4 x i32> @llvm.aarch64.sve.sxth.nxv4i32(<vscale x 4 x i32> zeroinitializer, <vscale x 4 x i1> %pg, <vscale x 4 x i32> %x)
+  ret <vscale x 4 x i32> %0
+}
+
+define <vscale x 2 x i64> @test_svexth_s64_x_1(<vscale x 2 x i1> %pg, <vscale x 2 x i64> %x) {
+; CHECK-LABEL: test_svexth_s64_x_1:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    sxth z0.d, p0/m, z0.d
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svexth_s64_x_1:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    sxth z0.d, p0/z, z0.d
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 2 x i64> @llvm.aarch64.sve.sxth.nxv2i64(<vscale x 2 x i64> poison, <vscale x 2 x i1> %pg, <vscale x 2 x i64> %x)
+  ret <vscale x 2 x i64> %0
+}
+
+define <vscale x 2 x i64> @test_svexth_s64_x_2(<vscale x 2 x i1> %pg, double %z0, <vscale x 2 x i64> %x) {
+; CHECK-LABEL: test_svexth_s64_x_2:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    movprfx z0, z1
+; CHECK-NEXT:    sxth z0.d, p0/m, z1.d
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svexth_s64_x_2:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    sxth z0.d, p0/z, z1.d
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 2 x i64> @llvm.aarch64.sve.sxth.nxv2i64(<vscale x 2 x i64> poison, <vscale x 2 x i1> %pg, <vscale x 2 x i64> %x)
+  ret <vscale x 2 x i64> %0
+}
+
+define <vscale x 2 x i64> @test_svexth_s64_z(<vscale x 2 x i1> %pg, double %z0, <vscale x 2 x i64> %x) {
+; CHECK-LABEL: test_svexth_s64_z:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    mov z0.d, #0 // =0x0
+; CHECK-NEXT:    sxth z0.d, p0/m, z1.d
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svexth_s64_z:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    sxth z0.d, p0/z, z1.d
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 2 x i64> @llvm.aarch64.sve.sxth.nxv2i64(<vscale x 2 x i64> zeroinitializer, <vscale x 2 x i1> %pg, <vscale x 2 x i64> %x)
+  ret <vscale x 2 x i64> %0
+}
+
+define <vscale x 4 x i32> @test_svexth_u32_x_1(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %x) {
+; CHECK-LABEL: test_svexth_u32_x_1:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    uxth z0.s, p0/m, z0.s
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svexth_u32_x_1:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    uxth z0.s, p0/z, z0.s
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 4 x i32> @llvm.aarch64.sve.uxth.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i1> %pg, <vscale x 4 x i32> %x)
+  ret <vscale x 4 x i32> %0
+}
+
+define <vscale x 4 x i32> @test_svexth_u32_x_2(<vscale x 4 x i1> %pg, double %z0, <vscale x 4 x i32> %x) {
+; CHECK-LABEL: test_svexth_u32_x_2:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    movprfx z0, z1
+; CHECK-NEXT:    uxth z0.s, p0/m, z1.s
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svexth_u32_x_2:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    uxth z0.s, p0/z, z1.s
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 4 x i32> @llvm.aarch64.sve.uxth.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i1> %pg, <vscale x 4 x i32> %x)
+  ret <vscale x 4 x i32> %0
+}
+
+define <vscale x 4 x i32> @test_svexth_u32_z(<vscale x 4 x i1> %pg, double %z0, <vscale x 4 x i32> %x) {
+; CHECK-LABEL: test_svexth_u32_z:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    mov z0.s, #0 // =0x0
+; CHECK-NEXT:    uxth z0.s, p0/m, z1.s
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svexth_u32_z:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    uxth z0.s, p0/z, z1.s
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 4 x i32> @llvm.aarch64.sve.uxth.nxv4i32(<vscale x 4 x i32> zeroinitializer, <vscale x 4 x i1> %pg, <vscale x 4 x i32> %x)
+  ret <vscale x 4 x i32> %0
+}
+
+define <vscale x 2 x i64> @test_svexth_u64_x_1(<vscale x 2 x i1> %pg, <vscale x 2 x i64> %x) {
+; CHECK-LABEL: test_svexth_u64_x_1:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    uxth z0.d, p0/m, z0.d
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svexth_u64_x_1:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    uxth z0.d, p0/z, z0.d
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 2 x i64> @llvm.aarch64.sve.uxth.nxv2i64(<vscale x 2 x i64> poison, <vscale x 2 x i1> %pg, <vscale x 2 x i64> %x)
+  ret <vscale x 2 x i64> %0
+}
+
+define <vscale x 2 x i64> @test_svexth_u64_x_2(<vscale x 2 x i1> %pg, double %z0, <vscale x 2 x i64> %x) {
+; CHECK-LABEL: test_svexth_u64_x_2:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    movprfx z0, z1
+; CHECK-NEXT:    uxth z0.d, p0/m, z1.d
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svexth_u64_x_2:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    uxth z0.d, p0/z, z1.d
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 2 x i64> @llvm.aarch64.sve.uxth.nxv2i64(<vscale x 2 x i64> poison, <vscale x 2 x i1> %pg, <vscale x 2 x i64> %x)
+  ret <vscale x 2 x i64> %0
+}
+
+define <vscale x 2 x i64> @test_svexth_u64_z(<vscale x 2 x i1> %pg, double %z0, <vscale x 2 x i64> %x) {
+; CHECK-LABEL: test_svexth_u64_z:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    mov z0.d, #0 // =0x0
+; CHECK-NEXT:    uxth z0.d, p0/m, z1.d
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svexth_u64_z:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    uxth z0.d, p0/z, z1.d
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 2 x i64> @llvm.aarch64.sve.uxth.nxv2i64(<vscale x 2 x i64> zeroinitializer, <vscale x 2 x i1> %pg, <vscale x 2 x i64> %x)
+  ret <vscale x 2 x i64> %0
+}
+
+define <vscale x 2 x i64> @test_svextw_s64_x_1(<vscale x 2 x i1> %pg, <vscale x 2 x i64> %x) {
+; CHECK-LABEL: test_svextw_s64_x_1:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    sxtw z0.d, p0/m, z0.d
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svextw_s64_x_1:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    sxtw z0.d, p0/z, z0.d
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 2 x i64> @llvm.aarch64.sve.sxtw.nxv2i64(<vscale x 2 x i64> poison, <vscale x 2 x i1> %pg, <vscale x 2 x i64> %x)
+  ret <vscale x 2 x i64> %0
+}
+
+define <vscale x 2 x i64> @test_svextw_s64_x_2(<vscale x 2 x i1> %pg, double %z0, <vscale x 2 x i64> %x) {
+; CHECK-LABEL: test_svextw_s64_x_2:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    movprfx z0, z1
+; CHECK-NEXT:    sxtw z0.d, p0/m, z1.d
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svextw_s64_x_2:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    sxtw z0.d, p0/z, z1.d
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 2 x i64> @llvm.aarch64.sve.sxtw.nxv2i64(<vscale x 2 x i64> poison, <vscale x 2 x i1> %pg, <vscale x 2 x i64> %x)
+  ret <vscale x 2 x i64> %0
+}
+
+define <vscale x 2 x i64> @test_svextw_s64_z(<vscale x 2 x i1> %pg, double %z0, <vscale x 2 x i64> %x) {
+; CHECK-LABEL: test_svextw_s64_z:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    mov z0.d, #0 // =0x0
+; CHECK-NEXT:    sxtw z0.d, p0/m, z1.d
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svextw_s64_z:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    sxtw z0.d, p0/z, z1.d
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 2 x i64> @llvm.aarch64.sve.sxtw.nxv2i64(<vscale x 2 x i64> zeroinitializer, <vscale x 2 x i1> %pg, <vscale x 2 x i64> %x)
+  ret <vscale x 2 x i64> %0
+}
+
+define <vscale x 2 x i64> @test_svextw_u64_x_1(<vscale x 2 x i1> %pg, <vscale x 2 x i64> %x) {
+; CHECK-LABEL: test_svextw_u64_x_1:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    uxtw z0.d, p0/m, z0.d
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svextw_u64_x_1:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    uxtw z0.d, p0/z, z0.d
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 2 x i64> @llvm.aarch64.sve.uxtw.nxv2i64(<vscale x 2 x i64> poison, <vscale x 2 x i1> %pg, <vscale x 2 x i64> %x)
+  ret <vscale x 2 x i64> %0
+}
+
+define <vscale x 2 x i64> @test_svextw_u64_x_2(<vscale x 2 x i1> %pg, double %z0, <vscale x 2 x i64> %x) {
+; CHECK-LABEL: test_svextw_u64_x_2:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    movprfx z0, z1
+; CHECK-NEXT:    uxtw z0.d, p0/m, z1.d
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svextw_u64_x_2:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    uxtw z0.d, p0/z, z1.d
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 2 x i64> @llvm.aarch64.sve.uxtw.nxv2i64(<vscale x 2 x i64> poison, <vscale x 2 x i1> %pg, <vscale x 2 x i64> %x)
+  ret <vscale x 2 x i64> %0
+}
+
+define <vscale x 2 x i64> @test_svextw_u64_z(<vscale x 2 x i1> %pg, double %z0, <vscale x 2 x i64> %x) {
+; CHECK-LABEL: test_svextw_u64_z:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    mov z0.d, #0 // =0x0
+; CHECK-NEXT:    uxtw z0.d, p0/m, z1.d
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svextw_u64_z:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    uxtw z0.d, p0/z, z1.d
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 2 x i64> @llvm.aarch64.sve.uxtw.nxv2i64(<vscale x 2 x i64> zeroinitializer, <vscale x 2 x i1> %pg, <vscale x 2 x i64> %x)
+  ret <vscale x 2 x i64> %0
+}
+
+define <vscale x 8 x i16> @test_svsxtb_nxv8i16_ptrue_u(double %z0, <vscale x 8 x i16> %x) {
+; CHECK-LABEL: test_svsxtb_nxv8i16_ptrue_u:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ptrue p0.h
+; CHECK-NEXT:    movprfx z0, z1
+; CHECK-NEXT:    sxtb z0.h, p0/m, z1.h
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svsxtb_nxv8i16_ptrue_u:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    ptrue p0.h
+; CHECK-2p2-NEXT:    sxtb z0.h, p0/z, z1.h
+; CHECK-2p2-NEXT:    ret
+entry:
+  %pg = call <vscale x 8 x i1> @llvm.aarch64.sve.ptrue.nxv8i1(i32 31)
+  %0 = tail call <vscale x 8 x i16> @llvm.aarch64.sve.sxtb.nxv8i16(<vscale x 8 x i16> poison, <vscale x 8 x i1> %pg, <vscale x 8 x i16> %x)
+  ret <vscale x 8 x i16> %0
+}
+
+define <vscale x 8 x i16> @test_svsxtb_nxv8i16_ptrue(double %z0, <vscale x 8 x i16> %x, <vscale x 8 x i16> %y) {
+; CHECK-LABEL: test_svsxtb_nxv8i16_ptrue:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ptrue p0.h
+; CHECK-NEXT:    movprfx z0, z2
+; CHECK-NEXT:    sxtb z0.h, p0/m, z2.h
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svsxtb_nxv8i16_ptrue:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    ptrue p0.h
+; CHECK-2p2-NEXT:    sxtb z0.h, p0/z, z2.h
+; CHECK-2p2-NEXT:    ret
+entry:
+  %pg = call <vscale x 8 x i1> @llvm.aarch64.sve.ptrue.nxv8i1(i32 31)
+  %0 = tail call <vscale x 8 x i16> @llvm.aarch64.sve.sxtb.nxv8i16(<vscale x 8 x i16> %x, <vscale x 8 x i1> %pg, <vscale x 8 x i16> %y)
+  ret <vscale x 8 x i16> %0
+}
+
+define <vscale x 4 x i32> @test_svsxtb_nxv4i32_ptrue_u(double %z0, <vscale x 4 x i32> %x) {
+; CHECK-LABEL: test_svsxtb_nxv4i32_ptrue_u:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ptrue p0.s
+; CHECK-NEXT:    movprfx z0, z1
+; CHECK-NEXT:    sxtb z0.s, p0/m, z1.s
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svsxtb_nxv4i32_ptrue_u:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    ptrue p0.s
+; CHECK-2p2-NEXT:    sxtb z0.s, p0/z, z1.s
+; CHECK-2p2-NEXT:    ret
+entry:
+  %pg = call <vscale x 4 x i1> @llvm.aarch64.sve.ptrue.nxv4i1(i32 31)
+  %0 = tail call <vscale x 4 x i32> @llvm.aarch64.sve.sxtb.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i1> %pg, <vscale x 4 x i32> %x)
+  ret <vscale x 4 x i32> %0
+}
+
+define <vscale x 4 x i32> @test_svsxtb_nxv4i32_ptrue(double %z0, <vscale x 4 x i32> %x, <vscale x 4 x i32> %y) {
+; CHECK-LABEL: test_svsxtb_nxv4i32_ptrue:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ptrue p0.s
+; CHECK-NEXT:    movprfx z0, z2
+; CHECK-NEXT:    sxtb z0.s, p0/m, z2.s
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svsxtb_nxv4i32_ptrue:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    ptrue p0.s
+; CHECK-2p2-NEXT:    sxtb z0.s, p0/z, z2.s
+; CHECK-2p2-NEXT:    ret
+entry:
+  %pg = call <vscale x 4 x i1> @llvm.aarch64.sve.ptrue.nxv4i1(i32 31)
+  %0 = tail call <vscale x 4 x i32> @llvm.aarch64.sve.sxtb.nxv4i32(<vscale x 4 x i32> %x, <vscale x 4 x i1> %pg, <vscale x 4 x i32> %y)
+  ret <vscale x 4 x i32> %0
+}
+
+define <vscale x 2 x i64> @test_svsxtb_nxv2i64_ptrue_u(double %z0, <vscale x 2 x i64> %x) {
+; CHECK-LABEL: test_svsxtb_nxv2i64_ptrue_u:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ptrue p0.d
+; CHECK-NEXT:    movprfx z0, z1
+; CHECK-NEXT:    sxtb z0.d, p0/m, z1.d
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svsxtb_nxv2i64_ptrue_u:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    ptrue p0.d
+; CHECK-2p2-NEXT:    sxtb z0.d, p0/z, z1.d
+; CHECK-2p2-NEXT:    ret
+entry:
+  %pg = call <vscale x 2 x i1> @llvm.aarch64.sve.ptrue.nxv2i1(i32 31)
+  %0 = tail call <vscale x 2 x i64> @llvm.aarch64.sve.sxtb.nxv2i64(<vscale x 2 x i64> poison, <vscale x 2 x i1> %pg, <vscale x 2 x i64> %x)
+  ret <vscale x 2 x i64> %0
+}
+
+define <vscale x 2 x i64> @test_svsxtb_nxv2i64_ptrue(double %z0, <vscale x 2 x i64> %x, <vscale x 2 x i64> %y) {
+; CHECK-LABEL: test_svsxtb_nxv2i64_ptrue:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ptrue p0.d
+; CHECK-NEXT:    movprfx z0, z2
+; CHECK-NEXT:    sxtb z0.d, p0/m, z2.d
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svsxtb_nxv2i64_ptrue:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    ptrue p0.d
+; CHECK-2p2-NEXT:    sxtb z0.d, p0/z, z2.d
+; CHECK-2p2-NEXT:    ret
+entry:
+  %pg = call <vscale x 2 x i1> @llvm.aarch64.sve.ptrue.nxv2i1(i32 31)
+  %0 = tail call <vscale x 2 x i64> @llvm.aarch64.sve.sxtb.nxv2i64(<vscale x 2 x i64> %x, <vscale x 2 x i1> %pg, <vscale x 2 x i64> %y)
+  ret <vscale x 2 x i64> %0
+}
+
+define <vscale x 8 x i16> @test_svuxtb_nxv8i16_ptrue_u(double %z0, <vscale x 8 x i16> %x) {
+; CHECK-LABEL: test_svuxtb_nxv8i16_ptrue_u:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ptrue p0.h
+; CHECK-NEXT:    movprfx z0, z1
+; CHECK-NEXT:    uxtb z0.h, p0/m, z1.h
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svuxtb_nxv8i16_ptrue_u:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    ptrue p0.h
+; CHECK-2p2-NEXT:    uxtb z0.h, p0/z, z1.h
+; CHECK-2p2-NEXT:    ret
+entry:
+  %pg = call <vscale x 8 x i1> @llvm.aarch64.sve.ptrue.nxv8i1(i32 31)
+  %0 = tail call <vscale x 8 x i16> @llvm.aarch64.sve.uxtb.nxv8i16(<vscale x 8 x i16> poison, <vscale x 8 x i1> %pg, <vscale x 8 x i16> %x)
+  ret <vscale x 8 x i16> %0
+}
+
+define <vscale x 8 x i16> @test_svuxtb_nxv8i16_ptrue(double %z0, <vscale x 8 x i16> %x, <vscale x 8 x i16> %y) {
+; CHECK-LABEL: test_svuxtb_nxv8i16_ptrue:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ptrue p0.h
+; CHECK-NEXT:    movprfx z0, z2
+; CHECK-NEXT:    uxtb z0.h, p0/m, z2.h
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svuxtb_nxv8i16_ptrue:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    ptrue p0.h
+; CHECK-2p2-NEXT:    uxtb z0.h, p0/z, z2.h
+; CHECK-2p2-NEXT:    ret
+entry:
+  %pg = call <vscale x 8 x i1> @llvm.aarch64.sve.ptrue.nxv8i1(i32 31)
+  %0 = tail call <vscale x 8 x i16> @llvm.aarch64.sve.uxtb.nxv8i16(<vscale x 8 x i16> %x, <vscale x 8 x i1> %pg, <vscale x 8 x i16> %y)
+  ret <vscale x 8 x i16> %0
+}
+
+define <vscale x 4 x i32> @test_svuxtb_nxv4i32_ptrue_u(double %z0, <vscale x 4 x i32> %x) {
+; CHECK-LABEL: test_svuxtb_nxv4i32_ptrue_u:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ptrue p0.s
+; CHECK-NEXT:    movprfx z0, z1
+; CHECK-NEXT:    uxtb z0.s, p0/m, z1.s
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svuxtb_nxv4i32_ptrue_u:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    ptrue p0.s
+; CHECK-2p2-NEXT:    uxtb z0.s, p0/z, z1.s
+; CHECK-2p2-NEXT:    ret
+entry:
+  %pg = call <vscale x 4 x i1> @llvm.aarch64.sve.ptrue.nxv4i1(i32 31)
+  %0 = tail call <vscale x 4 x i32> @llvm.aarch64.sve.uxtb.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i1> %pg, <vscale x 4 x i32> %x)
+  ret <vscale x 4 x i32> %0
+}
+
+define <vscale x 4 x i32> @test_svuxtb_nxv4i32_ptrue(double %z0, <vscale x 4 x i32> %x, <vscale x 4 x i32> %y) {
+; CHECK-LABEL: test_svuxtb_nxv4i32_ptrue:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ptrue p0.s
+; CHECK-NEXT:    movprfx z0, z2
+; CHECK-NEXT:    uxtb z0.s, p0/m, z2.s
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svuxtb_nxv4i32_ptrue:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    ptrue p0.s
+; CHECK-2p2-NEXT:    uxtb z0.s, p0/z, z2.s
+; CHECK-2p2-NEXT:    ret
+entry:
+  %pg = call <vscale x 4 x i1> @llvm.aarch64.sve.ptrue.nxv4i1(i32 31)
+  %0 = tail call <vscale x 4 x i32> @llvm.aarch64.sve.uxtb.nxv4i32(<vscale x 4 x i32> %x, <vscale x 4 x i1> %pg, <vscale x 4 x i32> %y)
+  ret <vscale x 4 x i32> %0
+}
+
+define <vscale x 2 x i64> @test_svuxtb_nxv2i64_ptrue_u(double %z0, <vscale x 2 x i64> %x) {
+; CHECK-LABEL: test_svuxtb_nxv2i64_ptrue_u:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ptrue p0.d
+; CHECK-NEXT:    movprfx z0, z1
+; CHECK-NEXT:    uxtb z0.d, p0/m, z1.d
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svuxtb_nxv2i64_ptrue_u:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    ptrue p0.d
+; CHECK-2p2-NEXT:    uxtb z0.d, p0/z, z1.d
+; CHECK-2p2-NEXT:    ret
+entry:
+  %pg = call <vscale x 2 x i1> @llvm.aarch64.sve.ptrue.nxv2i1(i32 31)
+  %0 = tail call <vscale x 2 x i64> @llvm.aarch64.sve.uxtb.nxv2i64(<vscale x 2 x i64> poison, <vscale x 2 x i1> %pg, <vscale x 2 x i64> %x)
+  ret <vscale x 2 x i64> %0
+}
+
+define <vscale x 2 x i64> @test_svuxtb_nxv2i64_ptrue(double %z0, <vscale x 2 x i64> %x, <vscale x 2 x i64> %y) {
+; CHECK-LABEL: test_svuxtb_nxv2i64_ptrue:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ptrue p0.d
+; CHECK-NEXT:    movprfx z0, z2
+; CHECK-NEXT:    uxtb z0.d, p0/m, z2.d
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svuxtb_nxv2i64_ptrue:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    ptrue p0.d
+; CHECK-2p2-NEXT:    uxtb z0.d, p0/z, z2.d
+; CHECK-2p2-NEXT:    ret
+entry:
+  %pg = call <vscale x 2 x i1> @llvm.aarch64.sve.ptrue.nxv2i1(i32 31)
+  %0 = tail call <vscale x 2 x i64> @llvm.aarch64.sve.uxtb.nxv2i64(<vscale x 2 x i64> %x, <vscale x 2 x i1> %pg, <vscale x 2 x i64> %y)
+  ret <vscale x 2 x i64> %0
+}
+
+define <vscale x 4 x i32> @test_svsxth_nxv4i32_ptrue_u(double %z0, <vscale x 4 x i32> %x) {
+; CHECK-LABEL: test_svsxth_nxv4i32_ptrue_u:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ptrue p0.s
+; CHECK-NEXT:    movprfx z0, z1
+; CHECK-NEXT:    sxth z0.s, p0/m, z1.s
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svsxth_nxv4i32_ptrue_u:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    ptrue p0.s
+; CHECK-2p2-NEXT:    sxth z0.s, p0/z, z1.s
+; CHECK-2p2-NEXT:    ret
+entry:
+  %pg = call <vscale x 4 x i1> @llvm.aarch64.sve.ptrue.nxv4i1(i32 31)
+  %0 = tail call <vscale x 4 x i32> @llvm.aarch64.sve.sxth.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i1> %pg, <vscale x 4 x i32> %x)
+  ret <vscale x 4 x i32> %0
+}
+
+define <vscale x 4 x i32> @test_svsxth_nxv4i32_ptrue(double %z0, <vscale x 4 x i32> %x, <vscale x 4 x i32> %y) {
+; CHECK-LABEL: test_svsxth_nxv4i32_ptrue:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ptrue p0.s
+; CHECK-NEXT:    movprfx z0, z2
+; CHECK-NEXT:    sxth z0.s, p0/m, z2.s
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svsxth_nxv4i32_ptrue:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    ptrue p0.s
+; CHECK-2p2-NEXT:    sxth z0.s, p0/z, z2.s
+; CHECK-2p2-NEXT:    ret
+entry:
+  %pg = call <vscale x 4 x i1> @llvm.aarch64.sve.ptrue.nxv4i1(i32 31)
+  %0 = tail call <vscale x 4 x i32> @llvm.aarch64.sve.sxth.nxv4i32(<vscale x 4 x i32> %x, <vscale x 4 x i1> %pg, <vscale x 4 x i32> %y)
+  ret <vscale x 4 x i32> %0
+}
+
+define <vscale x 2 x i64> @test_svsxth_nxv2i64_ptrue_u(double %z0, <vscale x 2 x i64> %x) {
+; CHECK-LABEL: test_svsxth_nxv2i64_ptrue_u:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ptrue p0.d
+; CHECK-NEXT:    movprfx z0, z1
+; CHECK-NEXT:    sxth z0.d, p0/m, z1.d
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svsxth_nxv2i64_ptrue_u:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    ptrue p0.d
+; CHECK-2p2-NEXT:    sxth z0.d, p0/z, z1.d
+; CHECK-2p2-NEXT:    ret
+entry:
+  %pg = call <vscale x 2 x i1> @llvm.aarch64.sve.ptrue.nxv2i1(i32 31)
+  %0 = tail call <vscale x 2 x i64> @llvm.aarch64.sve.sxth.nxv2i64(<vscale x 2 x i64> poison, <vscale x 2 x i1> %pg, <vscale x 2 x i64> %x)
+  ret <vscale x 2 x i64> %0
+}
+
+define <vscale x 2 x i64> @test_svsxth_nxv2i64_ptrue(double %z0, <vscale x 2 x i64> %x, <vscale x 2 x i64> %y) {
+; CHECK-LABEL: test_svsxth_nxv2i64_ptrue:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ptrue p0.d
+; CHECK-NEXT:    movprfx z0, z2
+; CHECK-NEXT:    sxth z0.d, p0/m, z2.d
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svsxth_nxv2i64_ptrue:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    ptrue p0.d
+; CHECK-2p2-NEXT:    sxth z0.d, p0/z, z2.d
+; CHECK-2p2-NEXT:    ret
+entry:
+  %pg = call <vscale x 2 x i1> @llvm.aarch64.sve.ptrue.nxv2i1(i32 31)
+  %0 = tail call <vscale x 2 x i64> @llvm.aarch64.sve.sxth.nxv2i64(<vscale x 2 x i64> %x, <vscale x 2 x i1> %pg, <vscale x 2 x i64> %y)
+  ret <vscale x 2 x i64> %0
+}
+
+define <vscale x 4 x i32> @test_svuxth_nxv4i32_ptrue_u(double %z0, <vscale x 4 x i32> %x) {
+; CHECK-LABEL: test_svuxth_nxv4i32_ptrue_u:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ptrue p0.s
+; CHECK-NEXT:    movprfx z0, z1
+; CHECK-NEXT:    uxth z0.s, p0/m, z1.s
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svuxth_nxv4i32_ptrue_u:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    ptrue p0.s
+; CHECK-2p2-NEXT:    uxth z0.s, p0/z, z1.s
+; CHECK-2p2-NEXT:    ret
+entry:
+  %pg = call <vscale x 4 x i1> @llvm.aarch64.sve.ptrue.nxv4i1(i32 31)
+  %0 = tail call <vscale x 4 x i32> @llvm.aarch64.sve.uxth.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i1> %pg, <vscale x 4 x i32> %x)
+  ret <vscale x 4 x i32> %0
+}
+
+define <vscale x 4 x i32> @test_svuxth_nxv4i32_ptrue(double %z0, <vscale x 4 x i32> %x, <vscale x 4 x i32> %y) {
+; CHECK-LABEL: test_svuxth_nxv4i32_ptrue:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ptrue p0.s
+; CHECK-NEXT:    movprfx z0, z2
+; CHECK-NEXT:    uxth z0.s, p0/m, z2.s
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svuxth_nxv4i32_ptrue:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    ptrue p0.s
+; CHECK-2p2-NEXT:    uxth z0.s, p0/z, z2.s
+; CHECK-2p2-NEXT:    ret
+entry:
+  %pg = call <vscale x 4 x i1> @llvm.aarch64.sve.ptrue.nxv4i1(i32 31)
+  %0 = tail call <vscale x 4 x i32> @llvm.aarch64.sve.uxth.nxv4i32(<vscale x 4 x i32> %x, <vscale x 4 x i1> %pg, <vscale x 4 x i32> %y)
+  ret <vscale x 4 x i32> %0
+}
+
+define <vscale x 2 x i64> @test_svuxth_nxv2i64_ptrue_u(double %z0, <vscale x 2 x i64> %x) {
+; CHECK-LABEL: test_svuxth_nxv2i64_ptrue_u:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ptrue p0.d
+; CHECK-NEXT:    movprfx z0, z1
+; CHECK-NEXT:    uxth z0.d, p0/m, z1.d
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svuxth_nxv2i64_ptrue_u:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    ptrue p0.d
+; CHECK-2p2-NEXT:    uxth z0.d, p0/z, z1.d
+; CHECK-2p2-NEXT:    ret
+entry:
+  %pg = call <vscale x 2 x i1> @llvm.aarch64.sve.ptrue.nxv2i1(i32 31)
+  %0 = tail call <vscale x 2 x i64> @llvm.aarch64.sve.uxth.nxv2i64(<vscale x 2 x i64> poison, <vscale x 2 x i1> %pg, <vscale x 2 x i64> %x)
+  ret <vscale x 2 x i64> %0
+}
+
+define <vscale x 2 x i64> @test_svuxth_nxv2i64_ptrue(double %z0, <vscale x 2 x i64> %x, <vscale x 2 x i64> %y) {
+; CHECK-LABEL: test_svuxth_nxv2i64_ptrue:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ptrue p0.d
+; CHECK-NEXT:    movprfx z0, z2
+; CHECK-NEXT:    uxth z0.d, p0/m, z2.d
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svuxth_nxv2i64_ptrue:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    ptrue p0.d
+; CHECK-2p2-NEXT:    uxth z0.d, p0/z, z2.d
+; CHECK-2p2-NEXT:    ret
+entry:
+  %pg = call <vscale x 2 x i1> @llvm.aarch64.sve.ptrue.nxv2i1(i32 31)
+  %0 = tail call <vscale x 2 x i64> @llvm.aarch64.sve.uxth.nxv2i64(<vscale x 2 x i64> %x, <vscale x 2 x i1> %pg, <vscale x 2 x i64> %y)
+  ret <vscale x 2 x i64> %0
+}
+
+define <vscale x 2 x i64> @test_svsxtw_nxv2i64_ptrue_u(double %z0, <vscale x 2 x i64> %x) {
+; CHECK-LABEL: test_svsxtw_nxv2i64_ptrue_u:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ptrue p0.d
+; CHECK-NEXT:    movprfx z0, z1
+; CHECK-NEXT:    sxtw z0.d, p0/m, z1.d
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svsxtw_nxv2i64_ptrue_u:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    ptrue p0.d
+; CHECK-2p2-NEXT:    sxtw z0.d, p0/z, z1.d
+; CHECK-2p2-NEXT:    ret
+entry:
+  %pg = call <vscale x 2 x i1> @llvm.aarch64.sve.ptrue.nxv2i1(i32 31)
+  %0 = tail call <vscale x 2 x i64> @llvm.aarch64.sve.sxtw.nxv2i64(<vscale x 2 x i64> poison, <vscale x 2 x i1> %pg, <vscale x 2 x i64> %x)
+  ret <vscale x 2 x i64> %0
+}
+
+define <vscale x 2 x i64> @test_svsxtw_nxv2i64_ptrue(double %z0, <vscale x 2 x i64> %x, <vscale x 2 x i64> %y) {
+; CHECK-LABEL: test_svsxtw_nxv2i64_ptrue:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ptrue p0.d
+; CHECK-NEXT:    movprfx z0, z2
+; CHECK-NEXT:    sxtw z0.d, p0/m, z2.d
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svsxtw_nxv2i64_ptrue:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    ptrue p0.d
+; CHECK-2p2-NEXT:    sxtw z0.d, p0/z, z2.d
+; CHECK-2p2-NEXT:    ret
+entry:
+  %pg = call <vscale x 2 x i1> @llvm.aarch64.sve.ptrue.nxv2i1(i32 31)
+  %0 = tail call <vscale x 2 x i64> @llvm.aarch64.sve.sxtw.nxv2i64(<vscale x 2 x i64> %x, <vscale x 2 x i1> %pg, <vscale x 2 x i64> %y)
+  ret <vscale x 2 x i64> %0
+}
+
+define <vscale x 2 x i64> @test_svuxtw_nxv2i64_ptrue_u(double %z0, <vscale x 2 x i64> %x) {
+; CHECK-LABEL: test_svuxtw_nxv2i64_ptrue_u:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ptrue p0.d
+; CHECK-NEXT:    movprfx z0, z1
+; CHECK-NEXT:    uxtw z0.d, p0/m, z1.d
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svuxtw_nxv2i64_ptrue_u:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    ptrue p0.d
+; CHECK-2p2-NEXT:    uxtw z0.d, p0/z, z1.d
+; CHECK-2p2-NEXT:    ret
+entry:
+  %pg = call <vscale x 2 x i1> @llvm.aarch64.sve.ptrue.nxv2i1(i32 31)
+  %0 = tail call <vscale x 2 x i64> @llvm.aarch64.sve.uxtw.nxv2i64(<vscale x 2 x i64> poison, <vscale x 2 x i1> %pg, <vscale x 2 x i64> %x)
+  ret <vscale x 2 x i64> %0
+}
+
+define <vscale x 2 x i64> @test_svuxtw_nxv2i64_ptrue(double %z0, <vscale x 2 x i64> %x, <vscale x 2 x i64> %y) {
+; CHECK-LABEL: test_svuxtw_nxv2i64_ptrue:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ptrue p0.d
+; CHECK-NEXT:    movprfx z0, z2
+; CHECK-NEXT:    uxtw z0.d, p0/m, z2.d
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svuxtw_nxv2i64_ptrue:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    ptrue p0.d
+; CHECK-2p2-NEXT:    uxtw z0.d, p0/z, z2.d
+; CHECK-2p2-NEXT:    ret
+entry:
+  %pg = call <vscale x 2 x i1> @llvm.aarch64.sve.ptrue.nxv2i1(i32 31)
+  %0 = tail call <vscale x 2 x i64> @llvm.aarch64.sve.uxtw.nxv2i64(<vscale x 2 x i64> %x, <vscale x 2 x i1> %pg, <vscale x 2 x i64> %y)
+  ret <vscale x 2 x i64> %0
+}
diff --git a/llvm/test/CodeGen/AArch64/zeroing-forms-flogb.ll b/llvm/test/CodeGen/AArch64/zeroing-forms-flogb.ll
new file mode 100644
index 0000000000000..23620a3419b99
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/zeroing-forms-flogb.ll
@@ -0,0 +1,258 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc -mattr=+sve2   < %s | FileCheck %s
+; RUN: llc -mattr=+sve2p2 < %s | FileCheck %s -check-prefix CHECK-2p2
+
+; RUN: llc -mattr=+sme2   -force-streaming < %s | FileCheck %s
+; RUN: llc -mattr=+sme2p2 -force-streaming < %s | FileCheck %s -check-prefix CHECK-2p2
+
+target triple = "aarch64-linux"
+
+define <vscale x 8 x i16> @test_svlogb_f16_x_1(<vscale x 8 x i1> %pg, <vscale x 8 x half> %x) {
+; CHECK-LABEL: test_svlogb_f16_x_1:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    flogb z0.h, p0/m, z0.h
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svlogb_f16_x_1:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    flogb z0.h, p0/z, z0.h
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 8 x i16> @llvm.aarch64.sve.flogb.nxv8f16(<vscale x 8 x i16> poison, <vscale x 8 x i1> %pg, <vscale x 8 x half> %x)
+  ret <vscale x 8 x i16> %0
+}
+
+define <vscale x 8 x i16> @test_svlogb_f16_x_2(<vscale x 8 x i1> %pg, double %z0, <vscale x 8 x half> %x) {
+; CHECK-LABEL: test_svlogb_f16_x_2:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    flogb z0.h, p0/m, z1.h
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svlogb_f16_x_2:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    flogb z0.h, p0/z, z1.h
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 8 x i16> @llvm.aarch64.sve.flogb.nxv8f16(<vscale x 8 x i16> poison, <vscale x 8 x i1> %pg, <vscale x 8 x half> %x)
+  ret <vscale x 8 x i16> %0
+}
+
+define <vscale x 8 x i16> @test_svlogb_f16_z(<vscale x 8 x i1> %pg, double %z0, <vscale x 8 x half> %x) {
+; CHECK-LABEL: test_svlogb_f16_z:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    mov z0.h, #0 // =0x0
+; CHECK-NEXT:    flogb z0.h, p0/m, z1.h
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svlogb_f16_z:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    flogb z0.h, p0/z, z1.h
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 8 x i16> @llvm.aarch64.sve.flogb.nxv8f16(<vscale x 8 x i16> zeroinitializer, <vscale x 8 x i1> %pg, <vscale x 8 x half> %x)
+  ret <vscale x 8 x i16> %0
+}
+
+define <vscale x 4 x i32> @test_svlogb_f32_x_1(<vscale x 4 x i1> %pg, <vscale x 4 x float> %x) {
+; CHECK-LABEL: test_svlogb_f32_x_1:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    flogb z0.s, p0/m, z0.s
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svlogb_f32_x_1:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    flogb z0.s, p0/z, z0.s
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 4 x i32> @llvm.aarch64.sve.flogb.nxv4f32(<vscale x 4 x i32> poison, <vscale x 4 x i1> %pg, <vscale x 4 x float> %x)
+  ret <vscale x 4 x i32> %0
+}
+
+define <vscale x 4 x i32> @test_svlogb_f32_x_2(<vscale x 4 x i1> %pg, double %z0, <vscale x 4 x float> %x) {
+; CHECK-LABEL: test_svlogb_f32_x_2:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    flogb z0.s, p0/m, z1.s
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svlogb_f32_x_2:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    flogb z0.s, p0/z, z1.s
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 4 x i32> @llvm.aarch64.sve.flogb.nxv4f32(<vscale x 4 x i32> poison, <vscale x 4 x i1> %pg, <vscale x 4 x float> %x)
+  ret <vscale x 4 x i32> %0
+}
+
+define <vscale x 4 x i32> @test_svlogb_f32_z(<vscale x 4 x i1> %pg, double %z0, <vscale x 4 x float> %x) {
+; CHECK-LABEL: test_svlogb_f32_z:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    mov z0.s, #0 // =0x0
+; CHECK-NEXT:    flogb z0.s, p0/m, z1.s
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svlogb_f32_z:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    flogb z0.s, p0/z, z1.s
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 4 x i32> @llvm.aarch64.sve.flogb.nxv4f32(<vscale x 4 x i32> zeroinitializer, <vscale x 4 x i1> %pg, <vscale x 4 x float> %x)
+  ret <vscale x 4 x i32> %0
+}
+
+define <vscale x 2 x i64> @test_svlogb_f64_x_1(<vscale x 2 x i1> %pg, <vscale x 2 x double> %x) {
+; CHECK-LABEL: test_svlogb_f64_x_1:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    flogb z0.d, p0/m, z0.d
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svlogb_f64_x_1:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    flogb z0.d, p0/z, z0.d
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 2 x i64> @llvm.aarch64.sve.flogb.nxv2f64(<vscale x 2 x i64> poison, <vscale x 2 x i1> %pg, <vscale x 2 x double> %x)
+  ret <vscale x 2 x i64> %0
+}
+
+define <vscale x 2 x i64> @test_svlogb_f64_x_2(<vscale x 2 x i1> %pg, double %z0, <vscale x 2 x double> %x) {
+; CHECK-LABEL: test_svlogb_f64_x_2:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    flogb z0.d, p0/m, z1.d
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svlogb_f64_x_2:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    flogb z0.d, p0/z, z1.d
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 2 x i64> @llvm.aarch64.sve.flogb.nxv2f64(<vscale x 2 x i64> poison, <vscale x 2 x i1> %pg, <vscale x 2 x double> %x)
+  ret <vscale x 2 x i64> %0
+}
+
+define <vscale x 2 x i64> @test_svlogb_f64_z(<vscale x 2 x i1> %pg, double %z0, <vscale x 2 x double> %x) {
+; CHECK-LABEL: test_svlogb_f64_z:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    mov z0.d, #0 // =0x0
+; CHECK-NEXT:    flogb z0.d, p0/m, z1.d
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svlogb_f64_z:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    flogb z0.d, p0/z, z1.d
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 2 x i64> @llvm.aarch64.sve.flogb.nxv2f64(<vscale x 2 x i64> zeroinitializer, <vscale x 2 x i1> %pg, <vscale x 2 x double> %x)
+  ret <vscale x 2 x i64> %0
+}
+
+define <vscale x 8 x i16> @test_svlogb_nxv8f16_ptrue_u(double %z0, <vscale x 8 x half> %x) {
+; CHECK-LABEL: test_svlogb_nxv8f16_ptrue_u:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ptrue p0.h
+; CHECK-NEXT:    flogb z0.h, p0/m, z1.h
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svlogb_nxv8f16_ptrue_u:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    ptrue p0.h
+; CHECK-2p2-NEXT:    flogb z0.h, p0/z, z1.h
+; CHECK-2p2-NEXT:    ret
+entry:
+  %pg = call <vscale x 8 x i1> @llvm.aarch64.sve.ptrue.nxv8i1(i32 31)
+  %0 = tail call <vscale x 8 x i16> @llvm.aarch64.sve.flogb.nxv8f16(<vscale x 8 x i16> poison, <vscale x 8 x i1> %pg, <vscale x 8 x half> %x)
+  ret <vscale x 8 x i16> %0
+}
+
+define <vscale x 8 x i16> @test_svlogb_nxv8f16_ptrue(double %z0, <vscale x 8 x i16> %x, <vscale x 8 x half> %y) {
+; CHECK-LABEL: test_svlogb_nxv8f16_ptrue:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    mov z0.d, z1.d
+; CHECK-NEXT:    ptrue p0.h
+; CHECK-NEXT:    flogb z0.h, p0/m, z2.h
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svlogb_nxv8f16_ptrue:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    ptrue p0.h
+; CHECK-2p2-NEXT:    flogb z0.h, p0/z, z2.h
+; CHECK-2p2-NEXT:    ret
+entry:
+  %pg = call <vscale x 8 x i1> @llvm.aarch64.sve.ptrue.nxv8i1(i32 31)
+  %0 = tail call <vscale x 8 x i16> @llvm.aarch64.sve.flogb.nxv8f16(<vscale x 8 x i16> %x, <vscale x 8 x i1> %pg, <vscale x 8 x half> %y)
+  ret <vscale x 8 x i16> %0
+}
+
+define <vscale x 4 x i32> @test_svlogb_nxv4f32_ptrue_u(double %z0, <vscale x 4 x float> %x) {
+; CHECK-LABEL: test_svlogb_nxv4f32_ptrue_u:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ptrue p0.s
+; CHECK-NEXT:    flogb z0.s, p0/m, z1.s
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svlogb_nxv4f32_ptrue_u:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    ptrue p0.s
+; CHECK-2p2-NEXT:    flogb z0.s, p0/z, z1.s
+; CHECK-2p2-NEXT:    ret
+entry:
+  %pg = call <vscale x 4 x i1> @llvm.aarch64.sve.ptrue.nxv4i1(i32 31)
+  %0 = tail call <vscale x 4 x i32> @llvm.aarch64.sve.flogb.nxv4f32(<vscale x 4 x i32> poison, <vscale x 4 x i1> %pg, <vscale x 4 x float> %x)
+  ret <vscale x 4 x i32> %0
+}
+
+define <vscale x 4 x i32> @test_svlogb_nxv4f32_ptrue(double %z0, <vscale x 4 x i32> %x, <vscale x 4 x float> %y) {
+; CHECK-LABEL: test_svlogb_nxv4f32_ptrue:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    mov z0.d, z1.d
+; CHECK-NEXT:    ptrue p0.s
+; CHECK-NEXT:    flogb z0.s, p0/m, z2.s
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svlogb_nxv4f32_ptrue:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    ptrue p0.s
+; CHECK-2p2-NEXT:    flogb z0.s, p0/z, z2.s
+; CHECK-2p2-NEXT:    ret
+entry:
+  %pg = call <vscale x 4 x i1> @llvm.aarch64.sve.ptrue.nxv4i1(i32 31)
+  %0 = tail call <vscale x 4 x i32> @llvm.aarch64.sve.flogb.nxv4f32(<vscale x 4 x i32> %x, <vscale x 4 x i1> %pg, <vscale x 4 x float> %y)
+  ret <vscale x 4 x i32> %0
+}
+
+define <vscale x 2 x i64> @test_svlogb_nxv2f64_ptrue_u(double %z0, <vscale x 2 x double> %x) {
+; CHECK-LABEL: test_svlogb_nxv2f64_ptrue_u:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ptrue p0.d
+; CHECK-NEXT:    flogb z0.d, p0/m, z1.d
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svlogb_nxv2f64_ptrue_u:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    ptrue p0.d
+; CHECK-2p2-NEXT:    flogb z0.d, p0/z, z1.d
+; CHECK-2p2-NEXT:    ret
+entry:
+  %pg = call <vscale x 2 x i1> @llvm.aarch64.sve.ptrue.nxv2i1(i32 31)
+  %0 = tail call <vscale x 2 x i64> @llvm.aarch64.sve.flogb.nxv2f64(<vscale x 2 x i64> poison, <vscale x 2 x i1> %pg, <vscale x 2 x double> %x)
+  ret <vscale x 2 x i64> %0
+}
+
+define <vscale x 2 x i64> @test_svlogb_nxv2f64_ptrue(double %z0, <vscale x 2 x i64> %x, <vscale x 2 x double> %y) {
+; CHECK-LABEL: test_svlogb_nxv2f64_ptrue:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    mov z0.d, z1.d
+; CHECK-NEXT:    ptrue p0.d
+; CHECK-NEXT:    flogb z0.d, p0/m, z2.d
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svlogb_nxv2f64_ptrue:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    ptrue p0.d
+; CHECK-2p2-NEXT:    flogb z0.d, p0/z, z2.d
+; CHECK-2p2-NEXT:    ret
+entry:
+  %pg = call <vscale x 2 x i1> @llvm.aarch64.sve.ptrue.nxv2i1(i32 31)
+  %0 = tail call <vscale x 2 x i64> @llvm.aarch64.sve.flogb.nxv2f64(<vscale x 2 x i64> %x, <vscale x 2 x i1> %pg, <vscale x 2 x double> %y)
+  ret <vscale x 2 x i64> %0
+}
+
diff --git a/llvm/test/CodeGen/AArch64/zeroing-forms-frint-frecpx-fsqrt.ll b/llvm/test/CodeGen/AArch64/zeroing-forms-frint-frecpx-fsqrt.ll
new file mode 100644
index 0000000000000..c493ec2dcc95d
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/zeroing-forms-frint-frecpx-fsqrt.ll
@@ -0,0 +1,4656 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc -mattr=+sve    < %s | FileCheck %s
+; RUN: llc -mattr=+sve2p2 < %s | FileCheck %s -check-prefix CHECK-2p2
+
+; RUN: llc -mattr=+sme    -force-streaming < %s | FileCheck %s
+; RUN: llc -mattr=+sme2p2 -force-streaming < %s | FileCheck %s -check-prefix CHECK-2p2
+
+target triple = "aarch64-linux"
+
+define <vscale x 8 x half> @test_svrinta_f16_x_1(<vscale x 8 x i1> %pg, <vscale x 8 x half> %x) {
+; CHECK-LABEL: test_svrinta_f16_x_1:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    frinta z0.h, p0/m, z0.h
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svrinta_f16_x_1:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    frinta z0.h, p0/z, z0.h
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 8 x half> @llvm.aarch64.sve.frinta.nxv8f16(<vscale x 8 x half> poison, <vscale x 8 x i1> %pg, <vscale x 8 x half> %x)
+  ret <vscale x 8 x half> %0
+}
+
+define <vscale x 8 x half> @test_svrinta_f16_x_2(<vscale x 8 x i1> %pg, double %z0, <vscale x 8 x half> %x) {
+; CHECK-LABEL: test_svrinta_f16_x_2:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    movprfx z0, z1
+; CHECK-NEXT:    frinta z0.h, p0/m, z1.h
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svrinta_f16_x_2:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    frinta z0.h, p0/z, z1.h
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 8 x half> @llvm.aarch64.sve.frinta.nxv8f16(<vscale x 8 x half> poison, <vscale x 8 x i1> %pg, <vscale x 8 x half> %x)
+  ret <vscale x 8 x half> %0
+}
+
+define <vscale x 8 x half> @test_svrinta_f16_z(<vscale x 8 x i1> %pg, double %z0, <vscale x 8 x half> %x) {
+; CHECK-LABEL: test_svrinta_f16_z:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    mov z0.h, #0 // =0x0
+; CHECK-NEXT:    frinta z0.h, p0/m, z1.h
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svrinta_f16_z:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    frinta z0.h, p0/z, z1.h
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 8 x half> @llvm.aarch64.sve.frinta.nxv8f16(<vscale x 8 x half> zeroinitializer, <vscale x 8 x i1> %pg, <vscale x 8 x half> %x)
+  ret <vscale x 8 x half> %0
+}
+
+define <vscale x 4 x half> @test_svrinta_4f16_x_1(<vscale x 4 x i1> %pg, <vscale x 4 x half> %x) {
+; CHECK-LABEL: test_svrinta_4f16_x_1:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    frinta z0.h, p0/m, z0.h
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svrinta_4f16_x_1:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    frinta z0.h, p0/z, z0.h
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 4 x half> @llvm.aarch64.sve.frinta.nxv4f16(<vscale x 4 x half> poison, <vscale x 4 x i1> %pg, <vscale x 4 x half> %x)
+  ret <vscale x 4 x half> %0
+}
+
+define <vscale x 4 x half> @test_svrinta_4f16_x_2(<vscale x 4 x i1> %pg, double %z0, <vscale x 4 x half> %x) {
+; CHECK-LABEL: test_svrinta_4f16_x_2:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    movprfx z0, z1
+; CHECK-NEXT:    frinta z0.h, p0/m, z1.h
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svrinta_4f16_x_2:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    frinta z0.h, p0/z, z1.h
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 4 x half> @llvm.aarch64.sve.frinta.nxv4f16(<vscale x 4 x half> poison, <vscale x 4 x i1> %pg, <vscale x 4 x half> %x)
+  ret <vscale x 4 x half> %0
+}
+
+define <vscale x 4 x half> @test_svrinta_4f16_z(<vscale x 4 x i1> %pg, double %z0, <vscale x 4 x half> %x) {
+; CHECK-LABEL: test_svrinta_4f16_z:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    mov z0.h, #0 // =0x0
+; CHECK-NEXT:    frinta z0.h, p0/m, z1.h
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svrinta_4f16_z:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    frinta z0.h, p0/z, z1.h
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 4 x half> @llvm.aarch64.sve.frinta.nxv4f16(<vscale x 4 x half> zeroinitializer, <vscale x 4 x i1> %pg, <vscale x 4 x half> %x)
+  ret <vscale x 4 x half> %0
+}
+
+define <vscale x 2 x half> @test_svrinta_2f16_x_1(<vscale x 2 x i1> %pg, <vscale x 2 x half> %x) {
+; CHECK-LABEL: test_svrinta_2f16_x_1:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    frinta z0.h, p0/m, z0.h
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svrinta_2f16_x_1:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    frinta z0.h, p0/z, z0.h
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 2 x half> @llvm.aarch64.sve.frinta.nxv2f16(<vscale x 2 x half> poison, <vscale x 2 x i1> %pg, <vscale x 2 x half> %x)
+  ret <vscale x 2 x half> %0
+}
+
+define <vscale x 2 x half> @test_svrinta_2f16_x_2(<vscale x 2 x i1> %pg, double %z0, <vscale x 2 x half> %x) {
+; CHECK-LABEL: test_svrinta_2f16_x_2:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    movprfx z0, z1
+; CHECK-NEXT:    frinta z0.h, p0/m, z1.h
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svrinta_2f16_x_2:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    frinta z0.h, p0/z, z1.h
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 2 x half> @llvm.aarch64.sve.frinta.nxv2f16(<vscale x 2 x half> poison, <vscale x 2 x i1> %pg, <vscale x 2 x half> %x)
+  ret <vscale x 2 x half> %0
+}
+
+define <vscale x 2 x half> @test_svrinta_2f16_z(<vscale x 2 x i1> %pg, double %z0, <vscale x 2 x half> %x) {
+; CHECK-LABEL: test_svrinta_2f16_z:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    mov z0.h, #0 // =0x0
+; CHECK-NEXT:    frinta z0.h, p0/m, z1.h
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svrinta_2f16_z:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    frinta z0.h, p0/z, z1.h
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 2 x half> @llvm.aarch64.sve.frinta.nxv2f16(<vscale x 2 x half> zeroinitializer, <vscale x 2 x i1> %pg, <vscale x 2 x half> %x)
+  ret <vscale x 2 x half> %0
+}
+
+define <vscale x 2 x float> @test_svrinta_2f32_x_1(<vscale x 2 x i1> %pg, <vscale x 2 x float> %x) {
+; CHECK-LABEL: test_svrinta_2f32_x_1:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    frinta z0.s, p0/m, z0.s
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svrinta_2f32_x_1:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    frinta z0.s, p0/z, z0.s
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 2 x float> @llvm.aarch64.sve.frinta.nxv2f32(<vscale x 2 x float> poison, <vscale x 2 x i1> %pg, <vscale x 2 x float> %x)
+  ret <vscale x 2 x float> %0
+}
+
+define <vscale x 2 x float> @test_svrinta_2f32_x_2(<vscale x 2 x i1> %pg, double %z0, <vscale x 2 x float> %x) {
+; CHECK-LABEL: test_svrinta_2f32_x_2:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    movprfx z0, z1
+; CHECK-NEXT:    frinta z0.s, p0/m, z1.s
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svrinta_2f32_x_2:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    frinta z0.s, p0/z, z1.s
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 2 x float> @llvm.aarch64.sve.frinta.nxv2f32(<vscale x 2 x float> poison, <vscale x 2 x i1> %pg, <vscale x 2 x float> %x)
+  ret <vscale x 2 x float> %0
+}
+
+define <vscale x 2 x float> @test_svrinta_2f32_z(<vscale x 2 x i1> %pg, double %z0, <vscale x 2 x float> %x) {
+; CHECK-LABEL: test_svrinta_2f32_z:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    mov z0.s, #0 // =0x0
+; CHECK-NEXT:    frinta z0.s, p0/m, z1.s
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svrinta_2f32_z:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    frinta z0.s, p0/z, z1.s
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 2 x float> @llvm.aarch64.sve.frinta.nxv2f32(<vscale x 2 x float> zeroinitializer, <vscale x 2 x i1> %pg, <vscale x 2 x float> %x)
+  ret <vscale x 2 x float> %0
+}
+
+define <vscale x 4 x float> @test_svrinta_f32_x_1(<vscale x 4 x i1> %pg, <vscale x 4 x float> %x) {
+; CHECK-LABEL: test_svrinta_f32_x_1:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    frinta z0.s, p0/m, z0.s
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svrinta_f32_x_1:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    frinta z0.s, p0/z, z0.s
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 4 x float> @llvm.aarch64.sve.frinta.nxv4f32(<vscale x 4 x float> poison, <vscale x 4 x i1> %pg, <vscale x 4 x float> %x)
+  ret <vscale x 4 x float> %0
+}
+
+define <vscale x 4 x float> @test_svrinta_f32_x_2(<vscale x 4 x i1> %pg, double %z0, <vscale x 4 x float> %x) {
+; CHECK-LABEL: test_svrinta_f32_x_2:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    movprfx z0, z1
+; CHECK-NEXT:    frinta z0.s, p0/m, z1.s
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svrinta_f32_x_2:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    frinta z0.s, p0/z, z1.s
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 4 x float> @llvm.aarch64.sve.frinta.nxv4f32(<vscale x 4 x float> poison, <vscale x 4 x i1> %pg, <vscale x 4 x float> %x)
+  ret <vscale x 4 x float> %0
+}
+
+define <vscale x 4 x float> @test_svrinta_f32_z(<vscale x 4 x i1> %pg, double %z0, <vscale x 4 x float> %x) {
+; CHECK-LABEL: test_svrinta_f32_z:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    mov z0.s, #0 // =0x0
+; CHECK-NEXT:    frinta z0.s, p0/m, z1.s
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svrinta_f32_z:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    frinta z0.s, p0/z, z1.s
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 4 x float> @llvm.aarch64.sve.frinta.nxv4f32(<vscale x 4 x float> zeroinitializer, <vscale x 4 x i1> %pg, <vscale x 4 x float> %x)
+  ret <vscale x 4 x float> %0
+}
+
+define <vscale x 2 x double> @test_svrinta_f64_x_1(<vscale x 2 x i1> %pg, <vscale x 2 x double> %x) {
+; CHECK-LABEL: test_svrinta_f64_x_1:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    frinta z0.d, p0/m, z0.d
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svrinta_f64_x_1:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    frinta z0.d, p0/z, z0.d
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 2 x double> @llvm.aarch64.sve.frinta.nxv2f64(<vscale x 2 x double> poison, <vscale x 2 x i1> %pg, <vscale x 2 x double> %x)
+  ret <vscale x 2 x double> %0
+}
+
+define <vscale x 2 x double> @test_svrinta_f64_x_2(<vscale x 2 x i1> %pg, double %z0, <vscale x 2 x double> %x) {
+; CHECK-LABEL: test_svrinta_f64_x_2:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    movprfx z0, z1
+; CHECK-NEXT:    frinta z0.d, p0/m, z1.d
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svrinta_f64_x_2:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    frinta z0.d, p0/z, z1.d
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 2 x double> @llvm.aarch64.sve.frinta.nxv2f64(<vscale x 2 x double> poison, <vscale x 2 x i1> %pg, <vscale x 2 x double> %x)
+  ret <vscale x 2 x double> %0
+}
+
+define <vscale x 2 x double> @test_svrinta_f64_z(<vscale x 2 x i1> %pg, double %z0, <vscale x 2 x double> %x) {
+; CHECK-LABEL: test_svrinta_f64_z:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    mov z0.d, #0 // =0x0
+; CHECK-NEXT:    frinta z0.d, p0/m, z1.d
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svrinta_f64_z:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    frinta z0.d, p0/z, z1.d
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 2 x double> @llvm.aarch64.sve.frinta.nxv2f64(<vscale x 2 x double> zeroinitializer, <vscale x 2 x i1> %pg, <vscale x 2 x double> %x)
+  ret <vscale x 2 x double> %0
+}
+
+define <vscale x 8 x half> @test_svrinti_f16_x_1(<vscale x 8 x i1> %pg, <vscale x 8 x half> %x) {
+; CHECK-LABEL: test_svrinti_f16_x_1:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    frinti z0.h, p0/m, z0.h
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svrinti_f16_x_1:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    frinti z0.h, p0/z, z0.h
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 8 x half> @llvm.aarch64.sve.frinti.nxv8f16(<vscale x 8 x half> poison, <vscale x 8 x i1> %pg, <vscale x 8 x half> %x)
+  ret <vscale x 8 x half> %0
+}
+
+define <vscale x 8 x half> @test_svrinti_f16_x_2(<vscale x 8 x i1> %pg, double %z0, <vscale x 8 x half> %x) {
+; CHECK-LABEL: test_svrinti_f16_x_2:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    movprfx z0, z1
+; CHECK-NEXT:    frinti z0.h, p0/m, z1.h
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svrinti_f16_x_2:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    frinti z0.h, p0/z, z1.h
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 8 x half> @llvm.aarch64.sve.frinti.nxv8f16(<vscale x 8 x half> poison, <vscale x 8 x i1> %pg, <vscale x 8 x half> %x)
+  ret <vscale x 8 x half> %0
+}
+
+define <vscale x 8 x half> @test_svrinti_f16_z(<vscale x 8 x i1> %pg, double %z0, <vscale x 8 x half> %x) {
+; CHECK-LABEL: test_svrinti_f16_z:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    mov z0.h, #0 // =0x0
+; CHECK-NEXT:    frinti z0.h, p0/m, z1.h
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svrinti_f16_z:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    frinti z0.h, p0/z, z1.h
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 8 x half> @llvm.aarch64.sve.frinti.nxv8f16(<vscale x 8 x half> zeroinitializer, <vscale x 8 x i1> %pg, <vscale x 8 x half> %x)
+  ret <vscale x 8 x half> %0
+}
+
+define <vscale x 4 x half> @test_svrinti_4f16_x_1(<vscale x 4 x i1> %pg, <vscale x 4 x half> %x) {
+; CHECK-LABEL: test_svrinti_4f16_x_1:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    frinti z0.h, p0/m, z0.h
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svrinti_4f16_x_1:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    frinti z0.h, p0/z, z0.h
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 4 x half> @llvm.aarch64.sve.frinti.nxv4f16(<vscale x 4 x half> poison, <vscale x 4 x i1> %pg, <vscale x 4 x half> %x)
+  ret <vscale x 4 x half> %0
+}
+
+define <vscale x 4 x half> @test_svrinti_4f16_x_2(<vscale x 4 x i1> %pg, double %z0, <vscale x 4 x half> %x) {
+; CHECK-LABEL: test_svrinti_4f16_x_2:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    movprfx z0, z1
+; CHECK-NEXT:    frinti z0.h, p0/m, z1.h
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svrinti_4f16_x_2:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    frinti z0.h, p0/z, z1.h
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 4 x half> @llvm.aarch64.sve.frinti.nxv4f16(<vscale x 4 x half> poison, <vscale x 4 x i1> %pg, <vscale x 4 x half> %x)
+  ret <vscale x 4 x half> %0
+}
+
+define <vscale x 4 x half> @test_svrinti_4f16_z(<vscale x 4 x i1> %pg, double %z0, <vscale x 4 x half> %x) {
+; CHECK-LABEL: test_svrinti_4f16_z:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    mov z0.h, #0 // =0x0
+; CHECK-NEXT:    frinti z0.h, p0/m, z1.h
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svrinti_4f16_z:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    frinti z0.h, p0/z, z1.h
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 4 x half> @llvm.aarch64.sve.frinti.nxv4f16(<vscale x 4 x half> zeroinitializer, <vscale x 4 x i1> %pg, <vscale x 4 x half> %x)
+  ret <vscale x 4 x half> %0
+}
+
+define <vscale x 2 x half> @test_svrinti_2f16_x_1(<vscale x 2 x i1> %pg, <vscale x 2 x half> %x) {
+; CHECK-LABEL: test_svrinti_2f16_x_1:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    frinti z0.h, p0/m, z0.h
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svrinti_2f16_x_1:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    frinti z0.h, p0/z, z0.h
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 2 x half> @llvm.aarch64.sve.frinti.nxv2f16(<vscale x 2 x half> poison, <vscale x 2 x i1> %pg, <vscale x 2 x half> %x)
+  ret <vscale x 2 x half> %0
+}
+
+define <vscale x 2 x half> @test_svrinti_2f16_x_2(<vscale x 2 x i1> %pg, double %z0, <vscale x 2 x half> %x) {
+; CHECK-LABEL: test_svrinti_2f16_x_2:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    movprfx z0, z1
+; CHECK-NEXT:    frinti z0.h, p0/m, z1.h
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svrinti_2f16_x_2:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    frinti z0.h, p0/z, z1.h
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 2 x half> @llvm.aarch64.sve.frinti.nxv2f16(<vscale x 2 x half> poison, <vscale x 2 x i1> %pg, <vscale x 2 x half> %x)
+  ret <vscale x 2 x half> %0
+}
+
+define <vscale x 2 x half> @test_svrinti_2f16_z(<vscale x 2 x i1> %pg, double %z0, <vscale x 2 x half> %x) {
+; CHECK-LABEL: test_svrinti_2f16_z:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    mov z0.h, #0 // =0x0
+; CHECK-NEXT:    frinti z0.h, p0/m, z1.h
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svrinti_2f16_z:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    frinti z0.h, p0/z, z1.h
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 2 x half> @llvm.aarch64.sve.frinti.nxv2f16(<vscale x 2 x half> zeroinitializer, <vscale x 2 x i1> %pg, <vscale x 2 x half> %x)
+  ret <vscale x 2 x half> %0
+}
+
+define <vscale x 2 x float> @test_svrinti_2f32_x_1(<vscale x 2 x i1> %pg, <vscale x 2 x float> %x) {
+; CHECK-LABEL: test_svrinti_2f32_x_1:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    frinti z0.s, p0/m, z0.s
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svrinti_2f32_x_1:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    frinti z0.s, p0/z, z0.s
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 2 x float> @llvm.aarch64.sve.frinti.nxv2f32(<vscale x 2 x float> poison, <vscale x 2 x i1> %pg, <vscale x 2 x float> %x)
+  ret <vscale x 2 x float> %0
+}
+
+define <vscale x 2 x float> @test_svrinti_2f32_x_2(<vscale x 2 x i1> %pg, double %z0, <vscale x 2 x float> %x) {
+; CHECK-LABEL: test_svrinti_2f32_x_2:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    movprfx z0, z1
+; CHECK-NEXT:    frinti z0.s, p0/m, z1.s
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svrinti_2f32_x_2:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    frinti z0.s, p0/z, z1.s
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 2 x float> @llvm.aarch64.sve.frinti.nxv2f32(<vscale x 2 x float> poison, <vscale x 2 x i1> %pg, <vscale x 2 x float> %x)
+  ret <vscale x 2 x float> %0
+}
+
+define <vscale x 2 x float> @test_svrinti_2f32_z(<vscale x 2 x i1> %pg, double %z0, <vscale x 2 x float> %x) {
+; CHECK-LABEL: test_svrinti_2f32_z:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    mov z0.s, #0 // =0x0
+; CHECK-NEXT:    frinti z0.s, p0/m, z1.s
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svrinti_2f32_z:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    frinti z0.s, p0/z, z1.s
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 2 x float> @llvm.aarch64.sve.frinti.nxv2f32(<vscale x 2 x float> zeroinitializer, <vscale x 2 x i1> %pg, <vscale x 2 x float> %x)
+  ret <vscale x 2 x float> %0
+}
+
+define <vscale x 4 x float> @test_svrinti_f32_x_1(<vscale x 4 x i1> %pg, <vscale x 4 x float> %x) {
+; CHECK-LABEL: test_svrinti_f32_x_1:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    frinti z0.s, p0/m, z0.s
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svrinti_f32_x_1:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    frinti z0.s, p0/z, z0.s
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 4 x float> @llvm.aarch64.sve.frinti.nxv4f32(<vscale x 4 x float> poison, <vscale x 4 x i1> %pg, <vscale x 4 x float> %x)
+  ret <vscale x 4 x float> %0
+}
+
+define <vscale x 4 x float> @test_svrinti_f32_x_2(<vscale x 4 x i1> %pg, double %z0, <vscale x 4 x float> %x) {
+; CHECK-LABEL: test_svrinti_f32_x_2:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    movprfx z0, z1
+; CHECK-NEXT:    frinti z0.s, p0/m, z1.s
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svrinti_f32_x_2:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    frinti z0.s, p0/z, z1.s
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 4 x float> @llvm.aarch64.sve.frinti.nxv4f32(<vscale x 4 x float> poison, <vscale x 4 x i1> %pg, <vscale x 4 x float> %x)
+  ret <vscale x 4 x float> %0
+}
+
+define <vscale x 4 x float> @test_svrinti_f32_z(<vscale x 4 x i1> %pg, double %z0, <vscale x 4 x float> %x) {
+; CHECK-LABEL: test_svrinti_f32_z:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    mov z0.s, #0 // =0x0
+; CHECK-NEXT:    frinti z0.s, p0/m, z1.s
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svrinti_f32_z:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    frinti z0.s, p0/z, z1.s
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 4 x float> @llvm.aarch64.sve.frinti.nxv4f32(<vscale x 4 x float> zeroinitializer, <vscale x 4 x i1> %pg, <vscale x 4 x float> %x)
+  ret <vscale x 4 x float> %0
+}
+
+define <vscale x 2 x double> @test_svrinti_f64_x_1(<vscale x 2 x i1> %pg, <vscale x 2 x double> %x) {
+; CHECK-LABEL: test_svrinti_f64_x_1:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    frinti z0.d, p0/m, z0.d
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svrinti_f64_x_1:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    frinti z0.d, p0/z, z0.d
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 2 x double> @llvm.aarch64.sve.frinti.nxv2f64(<vscale x 2 x double> poison, <vscale x 2 x i1> %pg, <vscale x 2 x double> %x)
+  ret <vscale x 2 x double> %0
+}
+
+define <vscale x 2 x double> @test_svrinti_f64_x_2(<vscale x 2 x i1> %pg, double %z0, <vscale x 2 x double> %x) {
+; CHECK-LABEL: test_svrinti_f64_x_2:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    movprfx z0, z1
+; CHECK-NEXT:    frinti z0.d, p0/m, z1.d
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svrinti_f64_x_2:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    frinti z0.d, p0/z, z1.d
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 2 x double> @llvm.aarch64.sve.frinti.nxv2f64(<vscale x 2 x double> poison, <vscale x 2 x i1> %pg, <vscale x 2 x double> %x)
+  ret <vscale x 2 x double> %0
+}
+
+
+define <vscale x 2 x double> @test_svrinti_f64_z(<vscale x 2 x i1> %pg, double %z0, <vscale x 2 x double> %x) {
+; CHECK-LABEL: test_svrinti_f64_z:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    mov z0.d, #0 // =0x0
+; CHECK-NEXT:    frinti z0.d, p0/m, z1.d
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svrinti_f64_z:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    frinti z0.d, p0/z, z1.d
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 2 x double> @llvm.aarch64.sve.frinti.nxv2f64(<vscale x 2 x double> zeroinitializer, <vscale x 2 x i1> %pg, <vscale x 2 x double> %x)
+  ret <vscale x 2 x double> %0
+}
+
+
+define <vscale x 8 x half> @test_svrintm_f16_x_1(<vscale x 8 x i1> %pg, <vscale x 8 x half> %x) {
+; CHECK-LABEL: test_svrintm_f16_x_1:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    frintm z0.h, p0/m, z0.h
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svrintm_f16_x_1:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    frintm z0.h, p0/z, z0.h
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 8 x half> @llvm.aarch64.sve.frintm.nxv8f16(<vscale x 8 x half> poison, <vscale x 8 x i1> %pg, <vscale x 8 x half> %x)
+  ret <vscale x 8 x half> %0
+}
+
+define <vscale x 8 x half> @test_svrintm_f16_x_2(<vscale x 8 x i1> %pg, double %z0, <vscale x 8 x half> %x) {
+; CHECK-LABEL: test_svrintm_f16_x_2:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    movprfx z0, z1
+; CHECK-NEXT:    frintm z0.h, p0/m, z1.h
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svrintm_f16_x_2:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    frintm z0.h, p0/z, z1.h
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 8 x half> @llvm.aarch64.sve.frintm.nxv8f16(<vscale x 8 x half> poison, <vscale x 8 x i1> %pg, <vscale x 8 x half> %x)
+  ret <vscale x 8 x half> %0
+}
+
+
+define <vscale x 8 x half> @test_svrintm_f16_z(<vscale x 8 x i1> %pg, double %z0, <vscale x 8 x half> %x) {
+; CHECK-LABEL: test_svrintm_f16_z:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    mov z0.h, #0 // =0x0
+; CHECK-NEXT:    frintm z0.h, p0/m, z1.h
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svrintm_f16_z:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    frintm z0.h, p0/z, z1.h
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 8 x half> @llvm.aarch64.sve.frintm.nxv8f16(<vscale x 8 x half> zeroinitializer, <vscale x 8 x i1> %pg, <vscale x 8 x half> %x)
+  ret <vscale x 8 x half> %0
+}
+
+define <vscale x 4 x half> @test_svrintm_4f16_x_1(<vscale x 4 x i1> %pg, <vscale x 4 x half> %x) {
+; CHECK-LABEL: test_svrintm_4f16_x_1:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    frintm z0.h, p0/m, z0.h
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svrintm_4f16_x_1:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    frintm z0.h, p0/z, z0.h
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 4 x half> @llvm.aarch64.sve.frintm.nxv4f16(<vscale x 4 x half> poison, <vscale x 4 x i1> %pg, <vscale x 4 x half> %x)
+  ret <vscale x 4 x half> %0
+}
+
+define <vscale x 4 x half> @test_svrintm_4f16_x_2(<vscale x 4 x i1> %pg, double %z0, <vscale x 4 x half> %x) {
+; CHECK-LABEL: test_svrintm_4f16_x_2:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    movprfx z0, z1
+; CHECK-NEXT:    frintm z0.h, p0/m, z1.h
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svrintm_4f16_x_2:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    frintm z0.h, p0/z, z1.h
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 4 x half> @llvm.aarch64.sve.frintm.nxv4f16(<vscale x 4 x half> poison, <vscale x 4 x i1> %pg, <vscale x 4 x half> %x)
+  ret <vscale x 4 x half> %0
+}
+
+define <vscale x 4 x half> @test_svrintm_4f16_z(<vscale x 4 x i1> %pg, double %z0, <vscale x 4 x half> %x) {
+; CHECK-LABEL: test_svrintm_4f16_z:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    mov z0.h, #0 // =0x0
+; CHECK-NEXT:    frintm z0.h, p0/m, z1.h
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svrintm_4f16_z:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    frintm z0.h, p0/z, z1.h
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 4 x half> @llvm.aarch64.sve.frintm.nxv4f16(<vscale x 4 x half> zeroinitializer, <vscale x 4 x i1> %pg, <vscale x 4 x half> %x)
+  ret <vscale x 4 x half> %0
+}
+
+define <vscale x 2 x half> @test_svrintm_2f16_x_1(<vscale x 2 x i1> %pg, <vscale x 2 x half> %x) {
+; CHECK-LABEL: test_svrintm_2f16_x_1:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    frintm z0.h, p0/m, z0.h
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svrintm_2f16_x_1:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    frintm z0.h, p0/z, z0.h
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 2 x half> @llvm.aarch64.sve.frintm.nxv2f16(<vscale x 2 x half> poison, <vscale x 2 x i1> %pg, <vscale x 2 x half> %x)
+  ret <vscale x 2 x half> %0
+}
+
+define <vscale x 2 x half> @test_svrintm_2f16_x_2(<vscale x 2 x i1> %pg, double %z0, <vscale x 2 x half> %x) {
+; CHECK-LABEL: test_svrintm_2f16_x_2:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    movprfx z0, z1
+; CHECK-NEXT:    frintm z0.h, p0/m, z1.h
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svrintm_2f16_x_2:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    frintm z0.h, p0/z, z1.h
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 2 x half> @llvm.aarch64.sve.frintm.nxv2f16(<vscale x 2 x half> poison, <vscale x 2 x i1> %pg, <vscale x 2 x half> %x)
+  ret <vscale x 2 x half> %0
+}
+
+define <vscale x 2 x half> @test_svrintm_2f16_z(<vscale x 2 x i1> %pg, double %z0, <vscale x 2 x half> %x) {
+; CHECK-LABEL: test_svrintm_2f16_z:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    mov z0.h, #0 // =0x0
+; CHECK-NEXT:    frintm z0.h, p0/m, z1.h
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svrintm_2f16_z:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    frintm z0.h, p0/z, z1.h
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 2 x half> @llvm.aarch64.sve.frintm.nxv2f16(<vscale x 2 x half> zeroinitializer, <vscale x 2 x i1> %pg, <vscale x 2 x half> %x)
+  ret <vscale x 2 x half> %0
+}
+
+define <vscale x 2 x float> @test_svrintm_2f32_x_1(<vscale x 2 x i1> %pg, <vscale x 2 x float> %x) {
+; CHECK-LABEL: test_svrintm_2f32_x_1:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    frintm z0.s, p0/m, z0.s
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svrintm_2f32_x_1:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    frintm z0.s, p0/z, z0.s
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 2 x float> @llvm.aarch64.sve.frintm.nxv2f32(<vscale x 2 x float> poison, <vscale x 2 x i1> %pg, <vscale x 2 x float> %x)
+  ret <vscale x 2 x float> %0
+}
+
+define <vscale x 2 x float> @test_svrintm_2f32_x_2(<vscale x 2 x i1> %pg, double %z0, <vscale x 2 x float> %x) {
+; CHECK-LABEL: test_svrintm_2f32_x_2:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    movprfx z0, z1
+; CHECK-NEXT:    frintm z0.s, p0/m, z1.s
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svrintm_2f32_x_2:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    frintm z0.s, p0/z, z1.s
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 2 x float> @llvm.aarch64.sve.frintm.nxv2f32(<vscale x 2 x float> poison, <vscale x 2 x i1> %pg, <vscale x 2 x float> %x)
+  ret <vscale x 2 x float> %0
+}
+
+define <vscale x 2 x float> @test_svrintm_2f32_z(<vscale x 2 x i1> %pg, double %z0, <vscale x 2 x float> %x) {
+; CHECK-LABEL: test_svrintm_2f32_z:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    mov z0.s, #0 // =0x0
+; CHECK-NEXT:    frintm z0.s, p0/m, z1.s
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svrintm_2f32_z:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    frintm z0.s, p0/z, z1.s
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 2 x float> @llvm.aarch64.sve.frintm.nxv2f32(<vscale x 2 x float> zeroinitializer, <vscale x 2 x i1> %pg, <vscale x 2 x float> %x)
+  ret <vscale x 2 x float> %0
+}
+
+define <vscale x 4 x float> @test_svrintm_f32_x_1(<vscale x 4 x i1> %pg, <vscale x 4 x float> %x) {
+; CHECK-LABEL: test_svrintm_f32_x_1:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    frintm z0.s, p0/m, z0.s
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svrintm_f32_x_1:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    frintm z0.s, p0/z, z0.s
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 4 x float> @llvm.aarch64.sve.frintm.nxv4f32(<vscale x 4 x float> poison, <vscale x 4 x i1> %pg, <vscale x 4 x float> %x)
+  ret <vscale x 4 x float> %0
+}
+
+define <vscale x 4 x float> @test_svrintm_f32_x_2(<vscale x 4 x i1> %pg, double %z0, <vscale x 4 x float> %x) {
+; CHECK-LABEL: test_svrintm_f32_x_2:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    movprfx z0, z1
+; CHECK-NEXT:    frintm z0.s, p0/m, z1.s
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svrintm_f32_x_2:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    frintm z0.s, p0/z, z1.s
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 4 x float> @llvm.aarch64.sve.frintm.nxv4f32(<vscale x 4 x float> poison, <vscale x 4 x i1> %pg, <vscale x 4 x float> %x)
+  ret <vscale x 4 x float> %0
+}
+
+define <vscale x 4 x float> @test_svrintm_f32_z(<vscale x 4 x i1> %pg, double %z0, <vscale x 4 x float> %x) {
+; CHECK-LABEL: test_svrintm_f32_z:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    mov z0.s, #0 // =0x0
+; CHECK-NEXT:    frintm z0.s, p0/m, z1.s
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svrintm_f32_z:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    frintm z0.s, p0/z, z1.s
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 4 x float> @llvm.aarch64.sve.frintm.nxv4f32(<vscale x 4 x float> zeroinitializer, <vscale x 4 x i1> %pg, <vscale x 4 x float> %x)
+  ret <vscale x 4 x float> %0
+}
+
+define <vscale x 2 x double> @test_svrintm_f64_x_1(<vscale x 2 x i1> %pg, <vscale x 2 x double> %x) {
+; CHECK-LABEL: test_svrintm_f64_x_1:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    frintm z0.d, p0/m, z0.d
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svrintm_f64_x_1:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    frintm z0.d, p0/z, z0.d
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 2 x double> @llvm.aarch64.sve.frintm.nxv2f64(<vscale x 2 x double> poison, <vscale x 2 x i1> %pg, <vscale x 2 x double> %x)
+  ret <vscale x 2 x double> %0
+}
+
+define <vscale x 2 x double> @test_svrintm_f64_x_2(<vscale x 2 x i1> %pg, double %z0, <vscale x 2 x double> %x) {
+; CHECK-LABEL: test_svrintm_f64_x_2:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    movprfx z0, z1
+; CHECK-NEXT:    frintm z0.d, p0/m, z1.d
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svrintm_f64_x_2:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    frintm z0.d, p0/z, z1.d
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 2 x double> @llvm.aarch64.sve.frintm.nxv2f64(<vscale x 2 x double> poison, <vscale x 2 x i1> %pg, <vscale x 2 x double> %x)
+  ret <vscale x 2 x double> %0
+}
+
+define <vscale x 2 x double> @test_svrintm_f64_z(<vscale x 2 x i1> %pg, double %z0, <vscale x 2 x double> %x) {
+; CHECK-LABEL: test_svrintm_f64_z:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    mov z0.d, #0 // =0x0
+; CHECK-NEXT:    frintm z0.d, p0/m, z1.d
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svrintm_f64_z:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    frintm z0.d, p0/z, z1.d
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 2 x double> @llvm.aarch64.sve.frintm.nxv2f64(<vscale x 2 x double> zeroinitializer, <vscale x 2 x i1> %pg, <vscale x 2 x double> %x)
+  ret <vscale x 2 x double> %0
+}
+
+define <vscale x 8 x half> @test_svrintn_f16_x_1(<vscale x 8 x i1> %pg, <vscale x 8 x half> %x) {
+; CHECK-LABEL: test_svrintn_f16_x_1:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    frintn z0.h, p0/m, z0.h
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svrintn_f16_x_1:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    frintn z0.h, p0/z, z0.h
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 8 x half> @llvm.aarch64.sve.frintn.nxv8f16(<vscale x 8 x half> poison, <vscale x 8 x i1> %pg, <vscale x 8 x half> %x)
+  ret <vscale x 8 x half> %0
+}
+
+define <vscale x 8 x half> @test_svrintn_f16_x_2(<vscale x 8 x i1> %pg, double %z0, <vscale x 8 x half> %x) {
+; CHECK-LABEL: test_svrintn_f16_x_2:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    movprfx z0, z1
+; CHECK-NEXT:    frintn z0.h, p0/m, z1.h
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svrintn_f16_x_2:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    frintn z0.h, p0/z, z1.h
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 8 x half> @llvm.aarch64.sve.frintn.nxv8f16(<vscale x 8 x half> poison, <vscale x 8 x i1> %pg, <vscale x 8 x half> %x)
+  ret <vscale x 8 x half> %0
+}
+
+define <vscale x 8 x half> @test_svrintn_f16_z(<vscale x 8 x i1> %pg, double %z0, <vscale x 8 x half> %x) {
+; CHECK-LABEL: test_svrintn_f16_z:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    mov z0.h, #0 // =0x0
+; CHECK-NEXT:    frintn z0.h, p0/m, z1.h
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svrintn_f16_z:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    frintn z0.h, p0/z, z1.h
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 8 x half> @llvm.aarch64.sve.frintn.nxv8f16(<vscale x 8 x half> zeroinitializer, <vscale x 8 x i1> %pg, <vscale x 8 x half> %x)
+  ret <vscale x 8 x half> %0
+}
+
+define <vscale x 4 x half> @test_svrintn_4f16_x_1(<vscale x 4 x i1> %pg, <vscale x 4 x half> %x) {
+; CHECK-LABEL: test_svrintn_4f16_x_1:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    frintn z0.h, p0/m, z0.h
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svrintn_4f16_x_1:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    frintn z0.h, p0/z, z0.h
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 4 x half> @llvm.aarch64.sve.frintn.nxv4f16(<vscale x 4 x half> poison, <vscale x 4 x i1> %pg, <vscale x 4 x half> %x)
+  ret <vscale x 4 x half> %0
+}
+
+define <vscale x 4 x half> @test_svrintn_4f16_x_2(<vscale x 4 x i1> %pg, double %z0, <vscale x 4 x half> %x) {
+; CHECK-LABEL: test_svrintn_4f16_x_2:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    movprfx z0, z1
+; CHECK-NEXT:    frintn z0.h, p0/m, z1.h
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svrintn_4f16_x_2:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    frintn z0.h, p0/z, z1.h
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 4 x half> @llvm.aarch64.sve.frintn.nxv4f16(<vscale x 4 x half> poison, <vscale x 4 x i1> %pg, <vscale x 4 x half> %x)
+  ret <vscale x 4 x half> %0
+}
+
+define <vscale x 4 x half> @test_svrintn_4f16_z(<vscale x 4 x i1> %pg, double %z0, <vscale x 4 x half> %x) {
+; CHECK-LABEL: test_svrintn_4f16_z:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    mov z0.h, #0 // =0x0
+; CHECK-NEXT:    frintn z0.h, p0/m, z1.h
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svrintn_4f16_z:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    frintn z0.h, p0/z, z1.h
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 4 x half> @llvm.aarch64.sve.frintn.nxv4f16(<vscale x 4 x half> zeroinitializer, <vscale x 4 x i1> %pg, <vscale x 4 x half> %x)
+  ret <vscale x 4 x half> %0
+}
+
+define <vscale x 2 x half> @test_svrintn_2f16_x_1(<vscale x 2 x i1> %pg, <vscale x 2 x half> %x) {
+; CHECK-LABEL: test_svrintn_2f16_x_1:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    frintn z0.h, p0/m, z0.h
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svrintn_2f16_x_1:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    frintn z0.h, p0/z, z0.h
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 2 x half> @llvm.aarch64.sve.frintn.nxv2f16(<vscale x 2 x half> poison, <vscale x 2 x i1> %pg, <vscale x 2 x half> %x)
+  ret <vscale x 2 x half> %0
+}
+
+define <vscale x 2 x half> @test_svrintn_2f16_x_2(<vscale x 2 x i1> %pg, double %z0, <vscale x 2 x half> %x) {
+; CHECK-LABEL: test_svrintn_2f16_x_2:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    movprfx z0, z1
+; CHECK-NEXT:    frintn z0.h, p0/m, z1.h
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svrintn_2f16_x_2:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    frintn z0.h, p0/z, z1.h
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 2 x half> @llvm.aarch64.sve.frintn.nxv2f16(<vscale x 2 x half> poison, <vscale x 2 x i1> %pg, <vscale x 2 x half> %x)
+  ret <vscale x 2 x half> %0
+}
+
+define <vscale x 2 x half> @test_svrintn_2f16_z(<vscale x 2 x i1> %pg, double %z0, <vscale x 2 x half> %x) {
+; CHECK-LABEL: test_svrintn_2f16_z:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    mov z0.h, #0 // =0x0
+; CHECK-NEXT:    frintn z0.h, p0/m, z1.h
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svrintn_2f16_z:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    frintn z0.h, p0/z, z1.h
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 2 x half> @llvm.aarch64.sve.frintn.nxv2f16(<vscale x 2 x half> zeroinitializer, <vscale x 2 x i1> %pg, <vscale x 2 x half> %x)
+  ret <vscale x 2 x half> %0
+}
+
+define <vscale x 2 x float> @test_svrintn_2f32_x_1(<vscale x 2 x i1> %pg, <vscale x 2 x float> %x) {
+; CHECK-LABEL: test_svrintn_2f32_x_1:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    frintn z0.s, p0/m, z0.s
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svrintn_2f32_x_1:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    frintn z0.s, p0/z, z0.s
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 2 x float> @llvm.aarch64.sve.frintn.nxv2f32(<vscale x 2 x float> poison, <vscale x 2 x i1> %pg, <vscale x 2 x float> %x)
+  ret <vscale x 2 x float> %0
+}
+
+define <vscale x 2 x float> @test_svrintn_2f32_x_2(<vscale x 2 x i1> %pg, double %z0, <vscale x 2 x float> %x) {
+; CHECK-LABEL: test_svrintn_2f32_x_2:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    movprfx z0, z1
+; CHECK-NEXT:    frintn z0.s, p0/m, z1.s
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svrintn_2f32_x_2:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    frintn z0.s, p0/z, z1.s
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 2 x float> @llvm.aarch64.sve.frintn.nxv2f32(<vscale x 2 x float> poison, <vscale x 2 x i1> %pg, <vscale x 2 x float> %x)
+  ret <vscale x 2 x float> %0
+}
+
+define <vscale x 2 x float> @test_svrintn_2f32_z(<vscale x 2 x i1> %pg, double %z0, <vscale x 2 x float> %x) {
+; CHECK-LABEL: test_svrintn_2f32_z:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    mov z0.s, #0 // =0x0
+; CHECK-NEXT:    frintn z0.s, p0/m, z1.s
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svrintn_2f32_z:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    frintn z0.s, p0/z, z1.s
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 2 x float> @llvm.aarch64.sve.frintn.nxv2f32(<vscale x 2 x float> zeroinitializer, <vscale x 2 x i1> %pg, <vscale x 2 x float> %x)
+  ret <vscale x 2 x float> %0
+}
+
+define <vscale x 4 x float> @test_svrintn_f32_x_1(<vscale x 4 x i1> %pg, <vscale x 4 x float> %x) {
+; CHECK-LABEL: test_svrintn_f32_x_1:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    frintn z0.s, p0/m, z0.s
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svrintn_f32_x_1:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    frintn z0.s, p0/z, z0.s
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 4 x float> @llvm.aarch64.sve.frintn.nxv4f32(<vscale x 4 x float> poison, <vscale x 4 x i1> %pg, <vscale x 4 x float> %x)
+  ret <vscale x 4 x float> %0
+}
+
+define <vscale x 4 x float> @test_svrintn_f32_x_2(<vscale x 4 x i1> %pg, double %z0, <vscale x 4 x float> %x) {
+; CHECK-LABEL: test_svrintn_f32_x_2:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    movprfx z0, z1
+; CHECK-NEXT:    frintn z0.s, p0/m, z1.s
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svrintn_f32_x_2:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    frintn z0.s, p0/z, z1.s
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 4 x float> @llvm.aarch64.sve.frintn.nxv4f32(<vscale x 4 x float> poison, <vscale x 4 x i1> %pg, <vscale x 4 x float> %x)
+  ret <vscale x 4 x float> %0
+}
+
+define <vscale x 4 x float> @test_svrintn_f32_z(<vscale x 4 x i1> %pg, double %z0, <vscale x 4 x float> %x) {
+; CHECK-LABEL: test_svrintn_f32_z:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    mov z0.s, #0 // =0x0
+; CHECK-NEXT:    frintn z0.s, p0/m, z1.s
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svrintn_f32_z:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    frintn z0.s, p0/z, z1.s
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 4 x float> @llvm.aarch64.sve.frintn.nxv4f32(<vscale x 4 x float> zeroinitializer, <vscale x 4 x i1> %pg, <vscale x 4 x float> %x)
+  ret <vscale x 4 x float> %0
+}
+
+define <vscale x 2 x double> @test_svrintn_f64_x_1(<vscale x 2 x i1> %pg, <vscale x 2 x double> %x) {
+; CHECK-LABEL: test_svrintn_f64_x_1:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    frintn z0.d, p0/m, z0.d
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svrintn_f64_x_1:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    frintn z0.d, p0/z, z0.d
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 2 x double> @llvm.aarch64.sve.frintn.nxv2f64(<vscale x 2 x double> poison, <vscale x 2 x i1> %pg, <vscale x 2 x double> %x)
+  ret <vscale x 2 x double> %0
+}
+
+define <vscale x 2 x double> @test_svrintn_f64_x_2(<vscale x 2 x i1> %pg, double %z0, <vscale x 2 x double> %x) {
+; CHECK-LABEL: test_svrintn_f64_x_2:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    movprfx z0, z1
+; CHECK-NEXT:    frintn z0.d, p0/m, z1.d
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svrintn_f64_x_2:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    frintn z0.d, p0/z, z1.d
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 2 x double> @llvm.aarch64.sve.frintn.nxv2f64(<vscale x 2 x double> poison, <vscale x 2 x i1> %pg, <vscale x 2 x double> %x)
+  ret <vscale x 2 x double> %0
+}
+
+define <vscale x 2 x double> @test_svrintn_f64_z(<vscale x 2 x i1> %pg, double %z0, <vscale x 2 x double> %x) {
+; CHECK-LABEL: test_svrintn_f64_z:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    mov z0.d, #0 // =0x0
+; CHECK-NEXT:    frintn z0.d, p0/m, z1.d
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svrintn_f64_z:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    frintn z0.d, p0/z, z1.d
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 2 x double> @llvm.aarch64.sve.frintn.nxv2f64(<vscale x 2 x double> zeroinitializer, <vscale x 2 x i1> %pg, <vscale x 2 x double> %x)
+  ret <vscale x 2 x double> %0
+}
+
+define <vscale x 8 x half> @test_svrintp_f16_x_1(<vscale x 8 x i1> %pg, <vscale x 8 x half> %x) {
+; CHECK-LABEL: test_svrintp_f16_x_1:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    frintp z0.h, p0/m, z0.h
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svrintp_f16_x_1:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    frintp z0.h, p0/z, z0.h
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 8 x half> @llvm.aarch64.sve.frintp.nxv8f16(<vscale x 8 x half> poison, <vscale x 8 x i1> %pg, <vscale x 8 x half> %x)
+  ret <vscale x 8 x half> %0
+}
+
+define <vscale x 8 x half> @test_svrintp_f16_x_2(<vscale x 8 x i1> %pg, double %z0, <vscale x 8 x half> %x) {
+; CHECK-LABEL: test_svrintp_f16_x_2:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    movprfx z0, z1
+; CHECK-NEXT:    frintp z0.h, p0/m, z1.h
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svrintp_f16_x_2:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    frintp z0.h, p0/z, z1.h
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 8 x half> @llvm.aarch64.sve.frintp.nxv8f16(<vscale x 8 x half> poison, <vscale x 8 x i1> %pg, <vscale x 8 x half> %x)
+  ret <vscale x 8 x half> %0
+}
+
+define <vscale x 8 x half> @test_svrintp_f16_z(<vscale x 8 x i1> %pg, double %z0, <vscale x 8 x half> %x) {
+; CHECK-LABEL: test_svrintp_f16_z:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    mov z0.h, #0 // =0x0
+; CHECK-NEXT:    frintp z0.h, p0/m, z1.h
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svrintp_f16_z:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    frintp z0.h, p0/z, z1.h
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 8 x half> @llvm.aarch64.sve.frintp.nxv8f16(<vscale x 8 x half> zeroinitializer, <vscale x 8 x i1> %pg, <vscale x 8 x half> %x)
+  ret <vscale x 8 x half> %0
+}
+
+define <vscale x 4 x half> @test_svrintp_4f16_x_1(<vscale x 4 x i1> %pg, <vscale x 4 x half> %x) {
+; CHECK-LABEL: test_svrintp_4f16_x_1:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    frintp z0.h, p0/m, z0.h
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svrintp_4f16_x_1:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    frintp z0.h, p0/z, z0.h
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 4 x half> @llvm.aarch64.sve.frintp.nxv4f16(<vscale x 4 x half> poison, <vscale x 4 x i1> %pg, <vscale x 4 x half> %x)
+  ret <vscale x 4 x half> %0
+}
+
+define <vscale x 4 x half> @test_svrintp_4f16_x_2(<vscale x 4 x i1> %pg, double %z0, <vscale x 4 x half> %x) {
+; CHECK-LABEL: test_svrintp_4f16_x_2:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    movprfx z0, z1
+; CHECK-NEXT:    frintp z0.h, p0/m, z1.h
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svrintp_4f16_x_2:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    frintp z0.h, p0/z, z1.h
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 4 x half> @llvm.aarch64.sve.frintp.nxv4f16(<vscale x 4 x half> poison, <vscale x 4 x i1> %pg, <vscale x 4 x half> %x)
+  ret <vscale x 4 x half> %0
+}
+
+define <vscale x 4 x half> @test_svrintp_4f16_z(<vscale x 4 x i1> %pg, double %z0, <vscale x 4 x half> %x) {
+; CHECK-LABEL: test_svrintp_4f16_z:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    mov z0.h, #0 // =0x0
+; CHECK-NEXT:    frintp z0.h, p0/m, z1.h
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svrintp_4f16_z:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    frintp z0.h, p0/z, z1.h
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 4 x half> @llvm.aarch64.sve.frintp.nxv4f16(<vscale x 4 x half> zeroinitializer, <vscale x 4 x i1> %pg, <vscale x 4 x half> %x)
+  ret <vscale x 4 x half> %0
+}
+
+define <vscale x 2 x half> @test_svrintp_2f16_x_1(<vscale x 2 x i1> %pg, <vscale x 2 x half> %x) {
+; CHECK-LABEL: test_svrintp_2f16_x_1:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    frintp z0.h, p0/m, z0.h
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svrintp_2f16_x_1:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    frintp z0.h, p0/z, z0.h
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 2 x half> @llvm.aarch64.sve.frintp.nxv2f16(<vscale x 2 x half> poison, <vscale x 2 x i1> %pg, <vscale x 2 x half> %x)
+  ret <vscale x 2 x half> %0
+}
+
+define <vscale x 2 x half> @test_svrintp_2f16_x_2(<vscale x 2 x i1> %pg, double %z0, <vscale x 2 x half> %x) {
+; CHECK-LABEL: test_svrintp_2f16_x_2:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    movprfx z0, z1
+; CHECK-NEXT:    frintp z0.h, p0/m, z1.h
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svrintp_2f16_x_2:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    frintp z0.h, p0/z, z1.h
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 2 x half> @llvm.aarch64.sve.frintp.nxv2f16(<vscale x 2 x half> poison, <vscale x 2 x i1> %pg, <vscale x 2 x half> %x)
+  ret <vscale x 2 x half> %0
+}
+
+define <vscale x 2 x half> @test_svrintp_2f16_z(<vscale x 2 x i1> %pg, double %z0, <vscale x 2 x half> %x) {
+; CHECK-LABEL: test_svrintp_2f16_z:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    mov z0.h, #0 // =0x0
+; CHECK-NEXT:    frintp z0.h, p0/m, z1.h
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svrintp_2f16_z:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    frintp z0.h, p0/z, z1.h
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 2 x half> @llvm.aarch64.sve.frintp.nxv2f16(<vscale x 2 x half> zeroinitializer, <vscale x 2 x i1> %pg, <vscale x 2 x half> %x)
+  ret <vscale x 2 x half> %0
+}
+
+define <vscale x 2 x float> @test_svrintp_2f32_x_1(<vscale x 2 x i1> %pg, <vscale x 2 x float> %x) {
+; CHECK-LABEL: test_svrintp_2f32_x_1:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    frintp z0.s, p0/m, z0.s
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svrintp_2f32_x_1:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    frintp z0.s, p0/z, z0.s
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 2 x float> @llvm.aarch64.sve.frintp.nxv2f32(<vscale x 2 x float> poison, <vscale x 2 x i1> %pg, <vscale x 2 x float> %x)
+  ret <vscale x 2 x float> %0
+}
+
+define <vscale x 2 x float> @test_svrintp_2f32_x_2(<vscale x 2 x i1> %pg, double %z0, <vscale x 2 x float> %x) {
+; CHECK-LABEL: test_svrintp_2f32_x_2:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    movprfx z0, z1
+; CHECK-NEXT:    frintp z0.s, p0/m, z1.s
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svrintp_2f32_x_2:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    frintp z0.s, p0/z, z1.s
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 2 x float> @llvm.aarch64.sve.frintp.nxv2f32(<vscale x 2 x float> poison, <vscale x 2 x i1> %pg, <vscale x 2 x float> %x)
+  ret <vscale x 2 x float> %0
+}
+
+define <vscale x 2 x float> @test_svrintp_2f32_z(<vscale x 2 x i1> %pg, double %z0, <vscale x 2 x float> %x) {
+; CHECK-LABEL: test_svrintp_2f32_z:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    mov z0.s, #0 // =0x0
+; CHECK-NEXT:    frintp z0.s, p0/m, z1.s
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svrintp_2f32_z:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    frintp z0.s, p0/z, z1.s
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 2 x float> @llvm.aarch64.sve.frintp.nxv2f32(<vscale x 2 x float> zeroinitializer, <vscale x 2 x i1> %pg, <vscale x 2 x float> %x)
+  ret <vscale x 2 x float> %0
+}
+
+define <vscale x 4 x float> @test_svrintp_f32_x_1(<vscale x 4 x i1> %pg, <vscale x 4 x float> %x) {
+; CHECK-LABEL: test_svrintp_f32_x_1:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    frintp z0.s, p0/m, z0.s
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svrintp_f32_x_1:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    frintp z0.s, p0/z, z0.s
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 4 x float> @llvm.aarch64.sve.frintp.nxv4f32(<vscale x 4 x float> poison, <vscale x 4 x i1> %pg, <vscale x 4 x float> %x)
+  ret <vscale x 4 x float> %0
+}
+
+define <vscale x 4 x float> @test_svrintp_f32_x_2(<vscale x 4 x i1> %pg, double %z0, <vscale x 4 x float> %x) {
+; CHECK-LABEL: test_svrintp_f32_x_2:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    movprfx z0, z1
+; CHECK-NEXT:    frintp z0.s, p0/m, z1.s
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svrintp_f32_x_2:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    frintp z0.s, p0/z, z1.s
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 4 x float> @llvm.aarch64.sve.frintp.nxv4f32(<vscale x 4 x float> poison, <vscale x 4 x i1> %pg, <vscale x 4 x float> %x)
+  ret <vscale x 4 x float> %0
+}
+
+define <vscale x 4 x float> @test_svrintp_f32_z(<vscale x 4 x i1> %pg, double %z0, <vscale x 4 x float> %x) {
+; CHECK-LABEL: test_svrintp_f32_z:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    mov z0.s, #0 // =0x0
+; CHECK-NEXT:    frintp z0.s, p0/m, z1.s
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svrintp_f32_z:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    frintp z0.s, p0/z, z1.s
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 4 x float> @llvm.aarch64.sve.frintp.nxv4f32(<vscale x 4 x float> zeroinitializer, <vscale x 4 x i1> %pg, <vscale x 4 x float> %x)
+  ret <vscale x 4 x float> %0
+}
+
+define <vscale x 2 x double> @test_svrintp_f64_x_1(<vscale x 2 x i1> %pg, <vscale x 2 x double> %x) {
+; CHECK-LABEL: test_svrintp_f64_x_1:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    frintp z0.d, p0/m, z0.d
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svrintp_f64_x_1:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    frintp z0.d, p0/z, z0.d
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 2 x double> @llvm.aarch64.sve.frintp.nxv2f64(<vscale x 2 x double> poison, <vscale x 2 x i1> %pg, <vscale x 2 x double> %x)
+  ret <vscale x 2 x double> %0
+}
+
+define <vscale x 2 x double> @test_svrintp_f64_x_2(<vscale x 2 x i1> %pg, double %z0, <vscale x 2 x double> %x) {
+; CHECK-LABEL: test_svrintp_f64_x_2:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    movprfx z0, z1
+; CHECK-NEXT:    frintp z0.d, p0/m, z1.d
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svrintp_f64_x_2:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    frintp z0.d, p0/z, z1.d
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 2 x double> @llvm.aarch64.sve.frintp.nxv2f64(<vscale x 2 x double> poison, <vscale x 2 x i1> %pg, <vscale x 2 x double> %x)
+  ret <vscale x 2 x double> %0
+}
+
+
+define <vscale x 2 x double> @test_svrintp_f64_z(<vscale x 2 x i1> %pg, double %z0, <vscale x 2 x double> %x) {
+; CHECK-LABEL: test_svrintp_f64_z:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    mov z0.d, #0 // =0x0
+; CHECK-NEXT:    frintp z0.d, p0/m, z1.d
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svrintp_f64_z:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    frintp z0.d, p0/z, z1.d
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 2 x double> @llvm.aarch64.sve.frintp.nxv2f64(<vscale x 2 x double> zeroinitializer, <vscale x 2 x i1> %pg, <vscale x 2 x double> %x)
+  ret <vscale x 2 x double> %0
+}
+
+define <vscale x 8 x half> @test_svrintx_f16_x_1(<vscale x 8 x i1> %pg, <vscale x 8 x half> %x) {
+; CHECK-LABEL: test_svrintx_f16_x_1:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    frintx z0.h, p0/m, z0.h
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svrintx_f16_x_1:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    frintx z0.h, p0/z, z0.h
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 8 x half> @llvm.aarch64.sve.frintx.nxv8f16(<vscale x 8 x half> poison, <vscale x 8 x i1> %pg, <vscale x 8 x half> %x)
+  ret <vscale x 8 x half> %0
+}
+
+define <vscale x 8 x half> @test_svrintx_f16_x_2(<vscale x 8 x i1> %pg, double %z0, <vscale x 8 x half> %x) {
+; CHECK-LABEL: test_svrintx_f16_x_2:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    movprfx z0, z1
+; CHECK-NEXT:    frintx z0.h, p0/m, z1.h
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svrintx_f16_x_2:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    frintx z0.h, p0/z, z1.h
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 8 x half> @llvm.aarch64.sve.frintx.nxv8f16(<vscale x 8 x half> poison, <vscale x 8 x i1> %pg, <vscale x 8 x half> %x)
+  ret <vscale x 8 x half> %0
+}
+
+define <vscale x 8 x half> @test_svrintx_f16_z(<vscale x 8 x i1> %pg, double %z0, <vscale x 8 x half> %x) {
+; CHECK-LABEL: test_svrintx_f16_z:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    mov z0.h, #0 // =0x0
+; CHECK-NEXT:    frintx z0.h, p0/m, z1.h
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svrintx_f16_z:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    frintx z0.h, p0/z, z1.h
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 8 x half> @llvm.aarch64.sve.frintx.nxv8f16(<vscale x 8 x half> zeroinitializer, <vscale x 8 x i1> %pg, <vscale x 8 x half> %x)
+  ret <vscale x 8 x half> %0
+}
+
+define <vscale x 4 x half> @test_svrintx_4f16_x_1(<vscale x 4 x i1> %pg, <vscale x 4 x half> %x) {
+; CHECK-LABEL: test_svrintx_4f16_x_1:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    frintx z0.h, p0/m, z0.h
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svrintx_4f16_x_1:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    frintx z0.h, p0/z, z0.h
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 4 x half> @llvm.aarch64.sve.frintx.nxv4f16(<vscale x 4 x half> poison, <vscale x 4 x i1> %pg, <vscale x 4 x half> %x)
+  ret <vscale x 4 x half> %0
+}
+
+define <vscale x 4 x half> @test_svrintx_4f16_x_2(<vscale x 4 x i1> %pg, double %z0, <vscale x 4 x half> %x) {
+; CHECK-LABEL: test_svrintx_4f16_x_2:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    movprfx z0, z1
+; CHECK-NEXT:    frintx z0.h, p0/m, z1.h
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svrintx_4f16_x_2:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    frintx z0.h, p0/z, z1.h
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 4 x half> @llvm.aarch64.sve.frintx.nxv4f16(<vscale x 4 x half> poison, <vscale x 4 x i1> %pg, <vscale x 4 x half> %x)
+  ret <vscale x 4 x half> %0
+}
+
+define <vscale x 4 x half> @test_svrintx_4f16_z(<vscale x 4 x i1> %pg, double %z0, <vscale x 4 x half> %x) {
+; CHECK-LABEL: test_svrintx_4f16_z:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    mov z0.h, #0 // =0x0
+; CHECK-NEXT:    frintx z0.h, p0/m, z1.h
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svrintx_4f16_z:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    frintx z0.h, p0/z, z1.h
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 4 x half> @llvm.aarch64.sve.frintx.nxv4f16(<vscale x 4 x half> zeroinitializer, <vscale x 4 x i1> %pg, <vscale x 4 x half> %x)
+  ret <vscale x 4 x half> %0
+}
+
+define <vscale x 2 x half> @test_svrintx_2f16_x_1(<vscale x 2 x i1> %pg, <vscale x 2 x half> %x) {
+; CHECK-LABEL: test_svrintx_2f16_x_1:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    frintx z0.h, p0/m, z0.h
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svrintx_2f16_x_1:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    frintx z0.h, p0/z, z0.h
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 2 x half> @llvm.aarch64.sve.frintx.nxv2f16(<vscale x 2 x half> poison, <vscale x 2 x i1> %pg, <vscale x 2 x half> %x)
+  ret <vscale x 2 x half> %0
+}
+
+define <vscale x 2 x half> @test_svrintx_2f16_x_2(<vscale x 2 x i1> %pg, double %z0, <vscale x 2 x half> %x) {
+; CHECK-LABEL: test_svrintx_2f16_x_2:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    movprfx z0, z1
+; CHECK-NEXT:    frintx z0.h, p0/m, z1.h
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svrintx_2f16_x_2:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    frintx z0.h, p0/z, z1.h
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 2 x half> @llvm.aarch64.sve.frintx.nxv2f16(<vscale x 2 x half> poison, <vscale x 2 x i1> %pg, <vscale x 2 x half> %x)
+  ret <vscale x 2 x half> %0
+}
+
+define <vscale x 2 x half> @test_svrintx_2f16_z(<vscale x 2 x i1> %pg, double %z0, <vscale x 2 x half> %x) {
+; CHECK-LABEL: test_svrintx_2f16_z:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    mov z0.h, #0 // =0x0
+; CHECK-NEXT:    frintx z0.h, p0/m, z1.h
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svrintx_2f16_z:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    frintx z0.h, p0/z, z1.h
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 2 x half> @llvm.aarch64.sve.frintx.nxv2f16(<vscale x 2 x half> zeroinitializer, <vscale x 2 x i1> %pg, <vscale x 2 x half> %x)
+  ret <vscale x 2 x half> %0
+}
+
+define <vscale x 2 x float> @test_svrintx_2f32_x_1(<vscale x 2 x i1> %pg, <vscale x 2 x float> %x) {
+; CHECK-LABEL: test_svrintx_2f32_x_1:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    frintx z0.s, p0/m, z0.s
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svrintx_2f32_x_1:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    frintx z0.s, p0/z, z0.s
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 2 x float> @llvm.aarch64.sve.frintx.nxv2f32(<vscale x 2 x float> poison, <vscale x 2 x i1> %pg, <vscale x 2 x float> %x)
+  ret <vscale x 2 x float> %0
+}
+
+define <vscale x 2 x float> @test_svrintx_2f32_x_2(<vscale x 2 x i1> %pg, double %z0, <vscale x 2 x float> %x) {
+; CHECK-LABEL: test_svrintx_2f32_x_2:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    movprfx z0, z1
+; CHECK-NEXT:    frintx z0.s, p0/m, z1.s
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svrintx_2f32_x_2:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    frintx z0.s, p0/z, z1.s
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 2 x float> @llvm.aarch64.sve.frintx.nxv2f32(<vscale x 2 x float> poison, <vscale x 2 x i1> %pg, <vscale x 2 x float> %x)
+  ret <vscale x 2 x float> %0
+}
+
+define <vscale x 2 x float> @test_svrintx_2f32_z(<vscale x 2 x i1> %pg, double %z0, <vscale x 2 x float> %x) {
+; CHECK-LABEL: test_svrintx_2f32_z:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    mov z0.s, #0 // =0x0
+; CHECK-NEXT:    frintx z0.s, p0/m, z1.s
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svrintx_2f32_z:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    frintx z0.s, p0/z, z1.s
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 2 x float> @llvm.aarch64.sve.frintx.nxv2f32(<vscale x 2 x float> zeroinitializer, <vscale x 2 x i1> %pg, <vscale x 2 x float> %x)
+  ret <vscale x 2 x float> %0
+}
+
+define <vscale x 4 x float> @test_svrintx_f32_x_1(<vscale x 4 x i1> %pg, <vscale x 4 x float> %x) {
+; CHECK-LABEL: test_svrintx_f32_x_1:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    frintx z0.s, p0/m, z0.s
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svrintx_f32_x_1:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    frintx z0.s, p0/z, z0.s
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 4 x float> @llvm.aarch64.sve.frintx.nxv4f32(<vscale x 4 x float> poison, <vscale x 4 x i1> %pg, <vscale x 4 x float> %x)
+  ret <vscale x 4 x float> %0
+}
+
+define <vscale x 4 x float> @test_svrintx_f32_x_2(<vscale x 4 x i1> %pg, double %z0, <vscale x 4 x float> %x) {
+; CHECK-LABEL: test_svrintx_f32_x_2:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    movprfx z0, z1
+; CHECK-NEXT:    frintx z0.s, p0/m, z1.s
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svrintx_f32_x_2:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    frintx z0.s, p0/z, z1.s
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 4 x float> @llvm.aarch64.sve.frintx.nxv4f32(<vscale x 4 x float> poison, <vscale x 4 x i1> %pg, <vscale x 4 x float> %x)
+  ret <vscale x 4 x float> %0
+}
+
+define <vscale x 4 x float> @test_svrintx_f32_z(<vscale x 4 x i1> %pg, double %z0, <vscale x 4 x float> %x) {
+; CHECK-LABEL: test_svrintx_f32_z:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    mov z0.s, #0 // =0x0
+; CHECK-NEXT:    frintx z0.s, p0/m, z1.s
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svrintx_f32_z:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    frintx z0.s, p0/z, z1.s
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 4 x float> @llvm.aarch64.sve.frintx.nxv4f32(<vscale x 4 x float> zeroinitializer, <vscale x 4 x i1> %pg, <vscale x 4 x float> %x)
+  ret <vscale x 4 x float> %0
+}
+
+define <vscale x 2 x double> @test_svrintx_f64_x_1(<vscale x 2 x i1> %pg, <vscale x 2 x double> %x) {
+; CHECK-LABEL: test_svrintx_f64_x_1:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    frintx z0.d, p0/m, z0.d
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svrintx_f64_x_1:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    frintx z0.d, p0/z, z0.d
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 2 x double> @llvm.aarch64.sve.frintx.nxv2f64(<vscale x 2 x double> poison, <vscale x 2 x i1> %pg, <vscale x 2 x double> %x)
+  ret <vscale x 2 x double> %0
+}
+
+define <vscale x 2 x double> @test_svrintx_f64_x_2(<vscale x 2 x i1> %pg, double %z0, <vscale x 2 x double> %x) {
+; CHECK-LABEL: test_svrintx_f64_x_2:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    movprfx z0, z1
+; CHECK-NEXT:    frintx z0.d, p0/m, z1.d
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svrintx_f64_x_2:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    frintx z0.d, p0/z, z1.d
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 2 x double> @llvm.aarch64.sve.frintx.nxv2f64(<vscale x 2 x double> poison, <vscale x 2 x i1> %pg, <vscale x 2 x double> %x)
+  ret <vscale x 2 x double> %0
+}
+
+define <vscale x 2 x double> @test_svrintx_f64_z(<vscale x 2 x i1> %pg, double %z0, <vscale x 2 x double> %x) {
+; CHECK-LABEL: test_svrintx_f64_z:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    mov z0.d, #0 // =0x0
+; CHECK-NEXT:    frintx z0.d, p0/m, z1.d
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svrintx_f64_z:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    frintx z0.d, p0/z, z1.d
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 2 x double> @llvm.aarch64.sve.frintx.nxv2f64(<vscale x 2 x double> zeroinitializer, <vscale x 2 x i1> %pg, <vscale x 2 x double> %x)
+  ret <vscale x 2 x double> %0
+}
+
+define <vscale x 8 x half> @test_svrintz_f16_x_1(<vscale x 8 x i1> %pg, <vscale x 8 x half> %x) {
+; CHECK-LABEL: test_svrintz_f16_x_1:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    frintz z0.h, p0/m, z0.h
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svrintz_f16_x_1:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    frintz z0.h, p0/z, z0.h
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 8 x half> @llvm.aarch64.sve.frintz.nxv8f16(<vscale x 8 x half> poison, <vscale x 8 x i1> %pg, <vscale x 8 x half> %x)
+  ret <vscale x 8 x half> %0
+}
+
+define <vscale x 8 x half> @test_svrintz_f16_x_2(<vscale x 8 x i1> %pg, double %z0, <vscale x 8 x half> %x) {
+; CHECK-LABEL: test_svrintz_f16_x_2:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    movprfx z0, z1
+; CHECK-NEXT:    frintz z0.h, p0/m, z1.h
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svrintz_f16_x_2:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    frintz z0.h, p0/z, z1.h
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 8 x half> @llvm.aarch64.sve.frintz.nxv8f16(<vscale x 8 x half> poison, <vscale x 8 x i1> %pg, <vscale x 8 x half> %x)
+  ret <vscale x 8 x half> %0
+}
+
+define <vscale x 8 x half> @test_svrintz_f16_z(<vscale x 8 x i1> %pg, double %z0, <vscale x 8 x half> %x) {
+; CHECK-LABEL: test_svrintz_f16_z:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    mov z0.h, #0 // =0x0
+; CHECK-NEXT:    frintz z0.h, p0/m, z1.h
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svrintz_f16_z:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    frintz z0.h, p0/z, z1.h
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 8 x half> @llvm.aarch64.sve.frintz.nxv8f16(<vscale x 8 x half> zeroinitializer, <vscale x 8 x i1> %pg, <vscale x 8 x half> %x)
+  ret <vscale x 8 x half> %0
+}
+
+define <vscale x 4 x half> @test_svrintz_4f16_x_1(<vscale x 4 x i1> %pg, <vscale x 4 x half> %x) {
+; CHECK-LABEL: test_svrintz_4f16_x_1:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    frintz z0.h, p0/m, z0.h
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svrintz_4f16_x_1:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    frintz z0.h, p0/z, z0.h
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 4 x half> @llvm.aarch64.sve.frintz.nxv4f16(<vscale x 4 x half> poison, <vscale x 4 x i1> %pg, <vscale x 4 x half> %x)
+  ret <vscale x 4 x half> %0
+}
+
+define <vscale x 4 x half> @test_svrintz_4f16_x_2(<vscale x 4 x i1> %pg, double %z0, <vscale x 4 x half> %x) {
+; CHECK-LABEL: test_svrintz_4f16_x_2:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    movprfx z0, z1
+; CHECK-NEXT:    frintz z0.h, p0/m, z1.h
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svrintz_4f16_x_2:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    frintz z0.h, p0/z, z1.h
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 4 x half> @llvm.aarch64.sve.frintz.nxv4f16(<vscale x 4 x half> poison, <vscale x 4 x i1> %pg, <vscale x 4 x half> %x)
+  ret <vscale x 4 x half> %0
+}
+
+define <vscale x 4 x half> @test_svrintz_4f16_z(<vscale x 4 x i1> %pg, double %z0, <vscale x 4 x half> %x) {
+; CHECK-LABEL: test_svrintz_4f16_z:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    mov z0.h, #0 // =0x0
+; CHECK-NEXT:    frintz z0.h, p0/m, z1.h
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svrintz_4f16_z:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    frintz z0.h, p0/z, z1.h
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 4 x half> @llvm.aarch64.sve.frintz.nxv4f16(<vscale x 4 x half> zeroinitializer, <vscale x 4 x i1> %pg, <vscale x 4 x half> %x)
+  ret <vscale x 4 x half> %0
+}
+
+define <vscale x 2 x half> @test_svrintz_2f16_x_1(<vscale x 2 x i1> %pg, <vscale x 2 x half> %x) {
+; CHECK-LABEL: test_svrintz_2f16_x_1:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    frintz z0.h, p0/m, z0.h
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svrintz_2f16_x_1:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    frintz z0.h, p0/z, z0.h
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 2 x half> @llvm.aarch64.sve.frintz.nxv2f16(<vscale x 2 x half> poison, <vscale x 2 x i1> %pg, <vscale x 2 x half> %x)
+  ret <vscale x 2 x half> %0
+}
+
+define <vscale x 2 x half> @test_svrintz_2f16_x_2(<vscale x 2 x i1> %pg, double %z0, <vscale x 2 x half> %x) {
+; CHECK-LABEL: test_svrintz_2f16_x_2:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    movprfx z0, z1
+; CHECK-NEXT:    frintz z0.h, p0/m, z1.h
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svrintz_2f16_x_2:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    frintz z0.h, p0/z, z1.h
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 2 x half> @llvm.aarch64.sve.frintz.nxv2f16(<vscale x 2 x half> poison, <vscale x 2 x i1> %pg, <vscale x 2 x half> %x)
+  ret <vscale x 2 x half> %0
+}
+
+define <vscale x 2 x half> @test_svrintz_2f16_z(<vscale x 2 x i1> %pg, double %z0, <vscale x 2 x half> %x) {
+; CHECK-LABEL: test_svrintz_2f16_z:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    mov z0.h, #0 // =0x0
+; CHECK-NEXT:    frintz z0.h, p0/m, z1.h
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svrintz_2f16_z:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    frintz z0.h, p0/z, z1.h
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 2 x half> @llvm.aarch64.sve.frintz.nxv2f16(<vscale x 2 x half> zeroinitializer, <vscale x 2 x i1> %pg, <vscale x 2 x half> %x)
+  ret <vscale x 2 x half> %0
+}
+
+define <vscale x 2 x float> @test_svrintz_2f32_x_1(<vscale x 2 x i1> %pg, <vscale x 2 x float> %x) {
+; CHECK-LABEL: test_svrintz_2f32_x_1:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    frintz z0.s, p0/m, z0.s
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svrintz_2f32_x_1:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    frintz z0.s, p0/z, z0.s
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 2 x float> @llvm.aarch64.sve.frintz.nxv2f32(<vscale x 2 x float> poison, <vscale x 2 x i1> %pg, <vscale x 2 x float> %x)
+  ret <vscale x 2 x float> %0
+}
+
+define <vscale x 2 x float> @test_svrintz_2f32_x_2(<vscale x 2 x i1> %pg, double %z0, <vscale x 2 x float> %x) {
+; CHECK-LABEL: test_svrintz_2f32_x_2:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    movprfx z0, z1
+; CHECK-NEXT:    frintz z0.s, p0/m, z1.s
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svrintz_2f32_x_2:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    frintz z0.s, p0/z, z1.s
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 2 x float> @llvm.aarch64.sve.frintz.nxv2f32(<vscale x 2 x float> poison, <vscale x 2 x i1> %pg, <vscale x 2 x float> %x)
+  ret <vscale x 2 x float> %0
+}
+
+define <vscale x 2 x float> @test_svrintz_2f32_z(<vscale x 2 x i1> %pg, double %z0, <vscale x 2 x float> %x) {
+; CHECK-LABEL: test_svrintz_2f32_z:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    mov z0.s, #0 // =0x0
+; CHECK-NEXT:    frintz z0.s, p0/m, z1.s
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svrintz_2f32_z:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    frintz z0.s, p0/z, z1.s
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 2 x float> @llvm.aarch64.sve.frintz.nxv2f32(<vscale x 2 x float> zeroinitializer, <vscale x 2 x i1> %pg, <vscale x 2 x float> %x)
+  ret <vscale x 2 x float> %0
+}
+
+define <vscale x 4 x float> @test_svrintz_f32_x_1(<vscale x 4 x i1> %pg, <vscale x 4 x float> %x) {
+; CHECK-LABEL: test_svrintz_f32_x_1:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    frintz z0.s, p0/m, z0.s
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svrintz_f32_x_1:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    frintz z0.s, p0/z, z0.s
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 4 x float> @llvm.aarch64.sve.frintz.nxv4f32(<vscale x 4 x float> poison, <vscale x 4 x i1> %pg, <vscale x 4 x float> %x)
+  ret <vscale x 4 x float> %0
+}
+
+define <vscale x 4 x float> @test_svrintz_f32_x_2(<vscale x 4 x i1> %pg, double %z0, <vscale x 4 x float> %x) {
+; CHECK-LABEL: test_svrintz_f32_x_2:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    movprfx z0, z1
+; CHECK-NEXT:    frintz z0.s, p0/m, z1.s
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svrintz_f32_x_2:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    frintz z0.s, p0/z, z1.s
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 4 x float> @llvm.aarch64.sve.frintz.nxv4f32(<vscale x 4 x float> poison, <vscale x 4 x i1> %pg, <vscale x 4 x float> %x)
+  ret <vscale x 4 x float> %0
+}
+
+define <vscale x 4 x float> @test_svrintz_f32_z(<vscale x 4 x i1> %pg, double %z0, <vscale x 4 x float> %x) {
+; CHECK-LABEL: test_svrintz_f32_z:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    mov z0.s, #0 // =0x0
+; CHECK-NEXT:    frintz z0.s, p0/m, z1.s
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svrintz_f32_z:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    frintz z0.s, p0/z, z1.s
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 4 x float> @llvm.aarch64.sve.frintz.nxv4f32(<vscale x 4 x float> zeroinitializer, <vscale x 4 x i1> %pg, <vscale x 4 x float> %x)
+  ret <vscale x 4 x float> %0
+}
+
+define <vscale x 2 x double> @test_svrintz_f64_x_1(<vscale x 2 x i1> %pg, <vscale x 2 x double> %x) {
+; CHECK-LABEL: test_svrintz_f64_x_1:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    frintz z0.d, p0/m, z0.d
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svrintz_f64_x_1:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    frintz z0.d, p0/z, z0.d
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 2 x double> @llvm.aarch64.sve.frintz.nxv2f64(<vscale x 2 x double> poison, <vscale x 2 x i1> %pg, <vscale x 2 x double> %x)
+  ret <vscale x 2 x double> %0
+}
+
+define <vscale x 2 x double> @test_svrintz_f64_x_2(<vscale x 2 x i1> %pg, double %z0, <vscale x 2 x double> %x) {
+; CHECK-LABEL: test_svrintz_f64_x_2:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    movprfx z0, z1
+; CHECK-NEXT:    frintz z0.d, p0/m, z1.d
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svrintz_f64_x_2:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    frintz z0.d, p0/z, z1.d
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 2 x double> @llvm.aarch64.sve.frintz.nxv2f64(<vscale x 2 x double> poison, <vscale x 2 x i1> %pg, <vscale x 2 x double> %x)
+  ret <vscale x 2 x double> %0
+}
+
+define <vscale x 2 x double> @test_svrintz_f64_z(<vscale x 2 x i1> %pg, double %z0, <vscale x 2 x double> %x) {
+; CHECK-LABEL: test_svrintz_f64_z:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    mov z0.d, #0 // =0x0
+; CHECK-NEXT:    frintz z0.d, p0/m, z1.d
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svrintz_f64_z:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    frintz z0.d, p0/z, z1.d
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 2 x double> @llvm.aarch64.sve.frintz.nxv2f64(<vscale x 2 x double> zeroinitializer, <vscale x 2 x i1> %pg, <vscale x 2 x double> %x)
+  ret <vscale x 2 x double> %0
+}
+
+define <vscale x 8 x half> @test_svrecpx_f16_x_1(<vscale x 8 x i1> %pg, <vscale x 8 x half> %x) {
+; CHECK-LABEL: test_svrecpx_f16_x_1:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    frecpx z0.h, p0/m, z0.h
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svrecpx_f16_x_1:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    frecpx z0.h, p0/z, z0.h
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 8 x half> @llvm.aarch64.sve.frecpx.nxv8f16(<vscale x 8 x half> poison, <vscale x 8 x i1> %pg, <vscale x 8 x half> %x)
+  ret <vscale x 8 x half> %0
+}
+
+define <vscale x 8 x half> @test_svrecpx_f16_x_2(<vscale x 8 x i1> %pg, double %z0, <vscale x 8 x half> %x) {
+; CHECK-LABEL: test_svrecpx_f16_x_2:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    movprfx z0, z1
+; CHECK-NEXT:    frecpx z0.h, p0/m, z1.h
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svrecpx_f16_x_2:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    frecpx z0.h, p0/z, z1.h
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 8 x half> @llvm.aarch64.sve.frecpx.nxv8f16(<vscale x 8 x half> poison, <vscale x 8 x i1> %pg, <vscale x 8 x half> %x)
+  ret <vscale x 8 x half> %0
+}
+
+define <vscale x 8 x half> @test_svrecpx_f16_z(<vscale x 8 x i1> %pg, double %z0, <vscale x 8 x half> %x) {
+; CHECK-LABEL: test_svrecpx_f16_z:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    mov z0.h, #0 // =0x0
+; CHECK-NEXT:    frecpx z0.h, p0/m, z1.h
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svrecpx_f16_z:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    frecpx z0.h, p0/z, z1.h
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 8 x half> @llvm.aarch64.sve.frecpx.nxv8f16(<vscale x 8 x half> zeroinitializer, <vscale x 8 x i1> %pg, <vscale x 8 x half> %x)
+  ret <vscale x 8 x half> %0
+}
+
+define <vscale x 4 x half> @test_svrecpx_4f16_x_1(<vscale x 4 x i1> %pg, <vscale x 4 x half> %x) {
+; CHECK-LABEL: test_svrecpx_4f16_x_1:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    frecpx z0.h, p0/m, z0.h
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svrecpx_4f16_x_1:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    frecpx z0.h, p0/z, z0.h
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 4 x half> @llvm.aarch64.sve.frecpx.nxv4f16(<vscale x 4 x half> poison, <vscale x 4 x i1> %pg, <vscale x 4 x half> %x)
+  ret <vscale x 4 x half> %0
+}
+
+define <vscale x 4 x half> @test_svrecpx_4f16_x_2(<vscale x 4 x i1> %pg, double %z0, <vscale x 4 x half> %x) {
+; CHECK-LABEL: test_svrecpx_4f16_x_2:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    movprfx z0, z1
+; CHECK-NEXT:    frecpx z0.h, p0/m, z1.h
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svrecpx_4f16_x_2:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    frecpx z0.h, p0/z, z1.h
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 4 x half> @llvm.aarch64.sve.frecpx.nxv4f16(<vscale x 4 x half> poison, <vscale x 4 x i1> %pg, <vscale x 4 x half> %x)
+  ret <vscale x 4 x half> %0
+}
+
+define <vscale x 4 x half> @test_svrecpx_4f16_z(<vscale x 4 x i1> %pg, double %z0, <vscale x 4 x half> %x) {
+; CHECK-LABEL: test_svrecpx_4f16_z:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    mov z0.h, #0 // =0x0
+; CHECK-NEXT:    frecpx z0.h, p0/m, z1.h
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svrecpx_4f16_z:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    frecpx z0.h, p0/z, z1.h
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 4 x half> @llvm.aarch64.sve.frecpx.nxv4f16(<vscale x 4 x half> zeroinitializer, <vscale x 4 x i1> %pg, <vscale x 4 x half> %x)
+  ret <vscale x 4 x half> %0
+}
+
+define <vscale x 2 x half> @test_svrecpx_2f16_x_1(<vscale x 2 x i1> %pg, <vscale x 2 x half> %x) {
+; CHECK-LABEL: test_svrecpx_2f16_x_1:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    frecpx z0.h, p0/m, z0.h
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svrecpx_2f16_x_1:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    frecpx z0.h, p0/z, z0.h
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 2 x half> @llvm.aarch64.sve.frecpx.nxv2f16(<vscale x 2 x half> poison, <vscale x 2 x i1> %pg, <vscale x 2 x half> %x)
+  ret <vscale x 2 x half> %0
+}
+
+define <vscale x 2 x half> @test_svrecpx_2f16_x_2(<vscale x 2 x i1> %pg, double %z0, <vscale x 2 x half> %x) {
+; CHECK-LABEL: test_svrecpx_2f16_x_2:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    movprfx z0, z1
+; CHECK-NEXT:    frecpx z0.h, p0/m, z1.h
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svrecpx_2f16_x_2:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    frecpx z0.h, p0/z, z1.h
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 2 x half> @llvm.aarch64.sve.frecpx.nxv2f16(<vscale x 2 x half> poison, <vscale x 2 x i1> %pg, <vscale x 2 x half> %x)
+  ret <vscale x 2 x half> %0
+}
+
+define <vscale x 2 x half> @test_svrecpx_2f16_z(<vscale x 2 x i1> %pg, double %z0, <vscale x 2 x half> %x) {
+; CHECK-LABEL: test_svrecpx_2f16_z:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    mov z0.h, #0 // =0x0
+; CHECK-NEXT:    frecpx z0.h, p0/m, z1.h
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svrecpx_2f16_z:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    frecpx z0.h, p0/z, z1.h
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 2 x half> @llvm.aarch64.sve.frecpx.nxv2f16(<vscale x 2 x half> zeroinitializer, <vscale x 2 x i1> %pg, <vscale x 2 x half> %x)
+  ret <vscale x 2 x half> %0
+}
+
+define <vscale x 2 x float> @test_svrecpx_2f32_x_1(<vscale x 2 x i1> %pg, <vscale x 2 x float> %x) {
+; CHECK-LABEL: test_svrecpx_2f32_x_1:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    frecpx z0.s, p0/m, z0.s
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svrecpx_2f32_x_1:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    frecpx z0.s, p0/z, z0.s
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 2 x float> @llvm.aarch64.sve.frecpx.nxv2f32(<vscale x 2 x float> poison, <vscale x 2 x i1> %pg, <vscale x 2 x float> %x)
+  ret <vscale x 2 x float> %0
+}
+
+define <vscale x 2 x float> @test_svrecpx_2f32_x_2(<vscale x 2 x i1> %pg, double %z0, <vscale x 2 x float> %x) {
+; CHECK-LABEL: test_svrecpx_2f32_x_2:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    movprfx z0, z1
+; CHECK-NEXT:    frecpx z0.s, p0/m, z1.s
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svrecpx_2f32_x_2:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    frecpx z0.s, p0/z, z1.s
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 2 x float> @llvm.aarch64.sve.frecpx.nxv2f32(<vscale x 2 x float> poison, <vscale x 2 x i1> %pg, <vscale x 2 x float> %x)
+  ret <vscale x 2 x float> %0
+}
+
+define <vscale x 2 x float> @test_svrecpx_2f32_z(<vscale x 2 x i1> %pg, double %z0, <vscale x 2 x float> %x) {
+; CHECK-LABEL: test_svrecpx_2f32_z:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    mov z0.s, #0 // =0x0
+; CHECK-NEXT:    frecpx z0.s, p0/m, z1.s
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svrecpx_2f32_z:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    frecpx z0.s, p0/z, z1.s
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 2 x float> @llvm.aarch64.sve.frecpx.nxv2f32(<vscale x 2 x float> zeroinitializer, <vscale x 2 x i1> %pg, <vscale x 2 x float> %x)
+  ret <vscale x 2 x float> %0
+}
+
+define <vscale x 4 x float> @test_svrecpx_f32_x_1(<vscale x 4 x i1> %pg, <vscale x 4 x float> %x) {
+; CHECK-LABEL: test_svrecpx_f32_x_1:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    frecpx z0.s, p0/m, z0.s
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svrecpx_f32_x_1:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    frecpx z0.s, p0/z, z0.s
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 4 x float> @llvm.aarch64.sve.frecpx.nxv4f32(<vscale x 4 x float> poison, <vscale x 4 x i1> %pg, <vscale x 4 x float> %x)
+  ret <vscale x 4 x float> %0
+}
+
+define <vscale x 4 x float> @test_svrecpx_f32_x_2(<vscale x 4 x i1> %pg, double %z0, <vscale x 4 x float> %x) {
+; CHECK-LABEL: test_svrecpx_f32_x_2:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    movprfx z0, z1
+; CHECK-NEXT:    frecpx z0.s, p0/m, z1.s
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svrecpx_f32_x_2:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    frecpx z0.s, p0/z, z1.s
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 4 x float> @llvm.aarch64.sve.frecpx.nxv4f32(<vscale x 4 x float> poison, <vscale x 4 x i1> %pg, <vscale x 4 x float> %x)
+  ret <vscale x 4 x float> %0
+}
+
+define <vscale x 4 x float> @test_svrecpx_f32_z(<vscale x 4 x i1> %pg, double %z0, <vscale x 4 x float> %x) {
+; CHECK-LABEL: test_svrecpx_f32_z:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    mov z0.s, #0 // =0x0
+; CHECK-NEXT:    frecpx z0.s, p0/m, z1.s
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svrecpx_f32_z:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    frecpx z0.s, p0/z, z1.s
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 4 x float> @llvm.aarch64.sve.frecpx.nxv4f32(<vscale x 4 x float> zeroinitializer, <vscale x 4 x i1> %pg, <vscale x 4 x float> %x)
+  ret <vscale x 4 x float> %0
+}
+
+define <vscale x 2 x double> @test_svrecpx_f64_x_1(<vscale x 2 x i1> %pg, <vscale x 2 x double> %x) {
+; CHECK-LABEL: test_svrecpx_f64_x_1:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    frecpx z0.d, p0/m, z0.d
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svrecpx_f64_x_1:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    frecpx z0.d, p0/z, z0.d
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 2 x double> @llvm.aarch64.sve.frecpx.nxv2f64(<vscale x 2 x double> poison, <vscale x 2 x i1> %pg, <vscale x 2 x double> %x)
+  ret <vscale x 2 x double> %0
+}
+
+define <vscale x 2 x double> @test_svrecpx_f64_x_2(<vscale x 2 x i1> %pg, double %z0, <vscale x 2 x double> %x) {
+; CHECK-LABEL: test_svrecpx_f64_x_2:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    movprfx z0, z1
+; CHECK-NEXT:    frecpx z0.d, p0/m, z1.d
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svrecpx_f64_x_2:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    frecpx z0.d, p0/z, z1.d
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 2 x double> @llvm.aarch64.sve.frecpx.nxv2f64(<vscale x 2 x double> poison, <vscale x 2 x i1> %pg, <vscale x 2 x double> %x)
+  ret <vscale x 2 x double> %0
+}
+
+define <vscale x 2 x double> @test_svrecpx_f64_z(<vscale x 2 x i1> %pg, double %z0, <vscale x 2 x double> %x) {
+; CHECK-LABEL: test_svrecpx_f64_z:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    mov z0.d, #0 // =0x0
+; CHECK-NEXT:    frecpx z0.d, p0/m, z1.d
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svrecpx_f64_z:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    frecpx z0.d, p0/z, z1.d
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 2 x double> @llvm.aarch64.sve.frecpx.nxv2f64(<vscale x 2 x double> zeroinitializer, <vscale x 2 x i1> %pg, <vscale x 2 x double> %x)
+  ret <vscale x 2 x double> %0
+}
+
+define <vscale x 8 x half> @test_svsqrt_f16_x_1(<vscale x 8 x i1> %pg, <vscale x 8 x half> %x) {
+; CHECK-LABEL: test_svsqrt_f16_x_1:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fsqrt z0.h, p0/m, z0.h
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svsqrt_f16_x_1:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    fsqrt z0.h, p0/z, z0.h
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 8 x half> @llvm.aarch64.sve.fsqrt.nxv8f16(<vscale x 8 x half> poison, <vscale x 8 x i1> %pg, <vscale x 8 x half> %x)
+  ret <vscale x 8 x half> %0
+}
+
+define <vscale x 8 x half> @test_svsqrt_f16_x_2(<vscale x 8 x i1> %pg, double %z0, <vscale x 8 x half> %x) {
+; CHECK-LABEL: test_svsqrt_f16_x_2:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    movprfx z0, z1
+; CHECK-NEXT:    fsqrt z0.h, p0/m, z1.h
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svsqrt_f16_x_2:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    fsqrt z0.h, p0/z, z1.h
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 8 x half> @llvm.aarch64.sve.fsqrt.nxv8f16(<vscale x 8 x half> poison, <vscale x 8 x i1> %pg, <vscale x 8 x half> %x)
+  ret <vscale x 8 x half> %0
+}
+
+define <vscale x 8 x half> @test_svsqrt_f16_z(<vscale x 8 x i1> %pg, double %z0, <vscale x 8 x half> %x) {
+; CHECK-LABEL: test_svsqrt_f16_z:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    mov z0.h, #0 // =0x0
+; CHECK-NEXT:    fsqrt z0.h, p0/m, z1.h
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svsqrt_f16_z:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    fsqrt z0.h, p0/z, z1.h
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 8 x half> @llvm.aarch64.sve.fsqrt.nxv8f16(<vscale x 8 x half> zeroinitializer, <vscale x 8 x i1> %pg, <vscale x 8 x half> %x)
+  ret <vscale x 8 x half> %0
+}
+
+define <vscale x 4 x half> @test_svsqrt_4f16_x_1(<vscale x 4 x i1> %pg, <vscale x 4 x half> %x) {
+; CHECK-LABEL: test_svsqrt_4f16_x_1:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fsqrt z0.h, p0/m, z0.h
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svsqrt_4f16_x_1:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    fsqrt z0.h, p0/z, z0.h
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 4 x half> @llvm.aarch64.sve.fsqrt.nxv4f16(<vscale x 4 x half> poison, <vscale x 4 x i1> %pg, <vscale x 4 x half> %x)
+  ret <vscale x 4 x half> %0
+}
+
+define <vscale x 4 x half> @test_svsqrt_4f16_x_2(<vscale x 4 x i1> %pg, double %z0, <vscale x 4 x half> %x) {
+; CHECK-LABEL: test_svsqrt_4f16_x_2:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    movprfx z0, z1
+; CHECK-NEXT:    fsqrt z0.h, p0/m, z1.h
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svsqrt_4f16_x_2:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    fsqrt z0.h, p0/z, z1.h
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 4 x half> @llvm.aarch64.sve.fsqrt.nxv4f16(<vscale x 4 x half> poison, <vscale x 4 x i1> %pg, <vscale x 4 x half> %x)
+  ret <vscale x 4 x half> %0
+}
+
+define <vscale x 4 x half> @test_svsqrt_4f16_z(<vscale x 4 x i1> %pg, double %z0, <vscale x 4 x half> %x) {
+; CHECK-LABEL: test_svsqrt_4f16_z:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    mov z0.h, #0 // =0x0
+; CHECK-NEXT:    fsqrt z0.h, p0/m, z1.h
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svsqrt_4f16_z:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    fsqrt z0.h, p0/z, z1.h
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 4 x half> @llvm.aarch64.sve.fsqrt.nxv4f16(<vscale x 4 x half> zeroinitializer, <vscale x 4 x i1> %pg, <vscale x 4 x half> %x)
+  ret <vscale x 4 x half> %0
+}
+
+define <vscale x 2 x half> @test_svsqrt_2f16_x_1(<vscale x 2 x i1> %pg, <vscale x 2 x half> %x) {
+; CHECK-LABEL: test_svsqrt_2f16_x_1:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fsqrt z0.h, p0/m, z0.h
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svsqrt_2f16_x_1:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    fsqrt z0.h, p0/z, z0.h
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 2 x half> @llvm.aarch64.sve.fsqrt.nxv2f16(<vscale x 2 x half> poison, <vscale x 2 x i1> %pg, <vscale x 2 x half> %x)
+  ret <vscale x 2 x half> %0
+}
+
+define <vscale x 2 x half> @test_svsqrt_2f16_x_2(<vscale x 2 x i1> %pg, double %z0, <vscale x 2 x half> %x) {
+; CHECK-LABEL: test_svsqrt_2f16_x_2:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    movprfx z0, z1
+; CHECK-NEXT:    fsqrt z0.h, p0/m, z1.h
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svsqrt_2f16_x_2:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    fsqrt z0.h, p0/z, z1.h
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 2 x half> @llvm.aarch64.sve.fsqrt.nxv2f16(<vscale x 2 x half> poison, <vscale x 2 x i1> %pg, <vscale x 2 x half> %x)
+  ret <vscale x 2 x half> %0
+}
+
+define <vscale x 2 x half> @test_svsqrt_2f16_z(<vscale x 2 x i1> %pg, double %z0, <vscale x 2 x half> %x) {
+; CHECK-LABEL: test_svsqrt_2f16_z:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    mov z0.h, #0 // =0x0
+; CHECK-NEXT:    fsqrt z0.h, p0/m, z1.h
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svsqrt_2f16_z:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    fsqrt z0.h, p0/z, z1.h
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 2 x half> @llvm.aarch64.sve.fsqrt.nxv2f16(<vscale x 2 x half> zeroinitializer, <vscale x 2 x i1> %pg, <vscale x 2 x half> %x)
+  ret <vscale x 2 x half> %0
+}
+
+define <vscale x 2 x float> @test_svsqrt_2f32_x_1(<vscale x 2 x i1> %pg, <vscale x 2 x float> %x) {
+; CHECK-LABEL: test_svsqrt_2f32_x_1:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fsqrt z0.s, p0/m, z0.s
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svsqrt_2f32_x_1:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    fsqrt z0.s, p0/z, z0.s
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 2 x float> @llvm.aarch64.sve.fsqrt.nxv2f32(<vscale x 2 x float> poison, <vscale x 2 x i1> %pg, <vscale x 2 x float> %x)
+  ret <vscale x 2 x float> %0
+}
+
+define <vscale x 2 x float> @test_svsqrt_2f32_x_2(<vscale x 2 x i1> %pg, double %z0, <vscale x 2 x float> %x) {
+; CHECK-LABEL: test_svsqrt_2f32_x_2:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    movprfx z0, z1
+; CHECK-NEXT:    fsqrt z0.s, p0/m, z1.s
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svsqrt_2f32_x_2:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    fsqrt z0.s, p0/z, z1.s
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 2 x float> @llvm.aarch64.sve.fsqrt.nxv2f32(<vscale x 2 x float> poison, <vscale x 2 x i1> %pg, <vscale x 2 x float> %x)
+  ret <vscale x 2 x float> %0
+}
+
+define <vscale x 2 x float> @test_svsqrt_2f32_z(<vscale x 2 x i1> %pg, double %z0, <vscale x 2 x float> %x) {
+; CHECK-LABEL: test_svsqrt_2f32_z:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    mov z0.s, #0 // =0x0
+; CHECK-NEXT:    fsqrt z0.s, p0/m, z1.s
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svsqrt_2f32_z:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    fsqrt z0.s, p0/z, z1.s
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 2 x float> @llvm.aarch64.sve.fsqrt.nxv2f32(<vscale x 2 x float> zeroinitializer, <vscale x 2 x i1> %pg, <vscale x 2 x float> %x)
+  ret <vscale x 2 x float> %0
+}
+
+define <vscale x 4 x float> @test_svsqrt_f32_x_1(<vscale x 4 x i1> %pg, <vscale x 4 x float> %x) {
+; CHECK-LABEL: test_svsqrt_f32_x_1:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fsqrt z0.s, p0/m, z0.s
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svsqrt_f32_x_1:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    fsqrt z0.s, p0/z, z0.s
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 4 x float> @llvm.aarch64.sve.fsqrt.nxv4f32(<vscale x 4 x float> poison, <vscale x 4 x i1> %pg, <vscale x 4 x float> %x)
+  ret <vscale x 4 x float> %0
+}
+
+define <vscale x 4 x float> @test_svsqrt_f32_x_2(<vscale x 4 x i1> %pg, double %z0, <vscale x 4 x float> %x) {
+; CHECK-LABEL: test_svsqrt_f32_x_2:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    movprfx z0, z1
+; CHECK-NEXT:    fsqrt z0.s, p0/m, z1.s
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svsqrt_f32_x_2:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    fsqrt z0.s, p0/z, z1.s
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 4 x float> @llvm.aarch64.sve.fsqrt.nxv4f32(<vscale x 4 x float> poison, <vscale x 4 x i1> %pg, <vscale x 4 x float> %x)
+  ret <vscale x 4 x float> %0
+}
+
+define <vscale x 4 x float> @test_svsqrt_f32_z(<vscale x 4 x i1> %pg, double %z0, <vscale x 4 x float> %x) {
+; CHECK-LABEL: test_svsqrt_f32_z:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    mov z0.s, #0 // =0x0
+; CHECK-NEXT:    fsqrt z0.s, p0/m, z1.s
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svsqrt_f32_z:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    fsqrt z0.s, p0/z, z1.s
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 4 x float> @llvm.aarch64.sve.fsqrt.nxv4f32(<vscale x 4 x float> zeroinitializer, <vscale x 4 x i1> %pg, <vscale x 4 x float> %x)
+  ret <vscale x 4 x float> %0
+}
+
+define <vscale x 2 x double> @test_svsqrt_f64_x_1(<vscale x 2 x i1> %pg, <vscale x 2 x double> %x) {
+; CHECK-LABEL: test_svsqrt_f64_x_1:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fsqrt z0.d, p0/m, z0.d
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svsqrt_f64_x_1:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    fsqrt z0.d, p0/z, z0.d
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 2 x double> @llvm.aarch64.sve.fsqrt.nxv2f64(<vscale x 2 x double> poison, <vscale x 2 x i1> %pg, <vscale x 2 x double> %x)
+  ret <vscale x 2 x double> %0
+}
+
+define <vscale x 2 x double> @test_svsqrt_f64_x_2(<vscale x 2 x i1> %pg, double %z0, <vscale x 2 x double> %x) {
+; CHECK-LABEL: test_svsqrt_f64_x_2:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    movprfx z0, z1
+; CHECK-NEXT:    fsqrt z0.d, p0/m, z1.d
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svsqrt_f64_x_2:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    fsqrt z0.d, p0/z, z1.d
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 2 x double> @llvm.aarch64.sve.fsqrt.nxv2f64(<vscale x 2 x double> poison, <vscale x 2 x i1> %pg, <vscale x 2 x double> %x)
+  ret <vscale x 2 x double> %0
+}
+
+define <vscale x 2 x double> @test_svsqrt_f64_z(<vscale x 2 x i1> %pg, double %z0, <vscale x 2 x double> %x) {
+; CHECK-LABEL: test_svsqrt_f64_z:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    mov z0.d, #0 // =0x0
+; CHECK-NEXT:    fsqrt z0.d, p0/m, z1.d
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svsqrt_f64_z:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    fsqrt z0.d, p0/z, z1.d
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 2 x double> @llvm.aarch64.sve.fsqrt.nxv2f64(<vscale x 2 x double> zeroinitializer, <vscale x 2 x i1> %pg, <vscale x 2 x double> %x)
+  ret <vscale x 2 x double> %0
+}
+
+define <vscale x 8 x half> @test_svfrinta_nxv8f16_ptrue_u(double %z0, <vscale x 8 x half> %x) {
+; CHECK-LABEL: test_svfrinta_nxv8f16_ptrue_u:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ptrue p0.h
+; CHECK-NEXT:    movprfx z0, z1
+; CHECK-NEXT:    frinta z0.h, p0/m, z1.h
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svfrinta_nxv8f16_ptrue_u:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    ptrue p0.h
+; CHECK-2p2-NEXT:    frinta z0.h, p0/z, z1.h
+; CHECK-2p2-NEXT:    ret
+entry:
+  %pg = call  <vscale x 8 x i1> @llvm.aarch64.sve.ptrue.nxv8i1(i32 31)
+  %0 = tail call <vscale x 8 x half> @llvm.aarch64.sve.frinta.nxv8f16(<vscale x 8 x half> poison,  <vscale x 8 x i1> %pg, <vscale x 8 x half> %x)
+  ret <vscale x 8 x half> %0
+}
+
+define <vscale x 8 x half> @test_svfrinta_nxv8f16_ptrue(double %z0, <vscale x 8 x half> %x, <vscale x 8 x half> %y) {
+; CHECK-LABEL: test_svfrinta_nxv8f16_ptrue:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ptrue p0.h
+; CHECK-NEXT:    movprfx z0, z2
+; CHECK-NEXT:    frinta z0.h, p0/m, z2.h
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svfrinta_nxv8f16_ptrue:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    ptrue p0.h
+; CHECK-2p2-NEXT:    frinta z0.h, p0/z, z2.h
+; CHECK-2p2-NEXT:    ret
+entry:
+  %pg = call  <vscale x 8 x i1> @llvm.aarch64.sve.ptrue.nxv8i1(i32 31)
+  %0 = tail call <vscale x 8 x half> @llvm.aarch64.sve.frinta.nxv8f16(<vscale x 8 x half> %x,  <vscale x 8 x i1> %pg, <vscale x 8 x half> %y)
+  ret <vscale x 8 x half> %0
+}
+
+
+define <vscale x 4 x half> @test_svfrinta_nxv4f16_ptrue_u(double %z0, <vscale x 4 x half> %x) {
+; CHECK-LABEL: test_svfrinta_nxv4f16_ptrue_u:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ptrue p0.s
+; CHECK-NEXT:    movprfx z0, z1
+; CHECK-NEXT:    frinta z0.h, p0/m, z1.h
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svfrinta_nxv4f16_ptrue_u:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    ptrue p0.s
+; CHECK-2p2-NEXT:    frinta z0.h, p0/z, z1.h
+; CHECK-2p2-NEXT:    ret
+entry:
+  %pg = call  <vscale x 4 x i1> @llvm.aarch64.sve.ptrue.nxv4i1(i32 31)
+  %0 = tail call <vscale x 4 x half> @llvm.aarch64.sve.frinta.nxv4f16(<vscale x 4 x half> poison,  <vscale x 4 x i1> %pg, <vscale x 4 x half> %x)
+  ret <vscale x 4 x half> %0
+}
+
+define <vscale x 4 x half> @test_svfrinta_nxv4f16_ptrue(double %z0, <vscale x 4 x half> %x, <vscale x 4 x half> %y) {
+; CHECK-LABEL: test_svfrinta_nxv4f16_ptrue:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ptrue p0.s
+; CHECK-NEXT:    movprfx z0, z2
+; CHECK-NEXT:    frinta z0.h, p0/m, z2.h
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svfrinta_nxv4f16_ptrue:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    ptrue p0.s
+; CHECK-2p2-NEXT:    frinta z0.h, p0/z, z2.h
+; CHECK-2p2-NEXT:    ret
+entry:
+  %pg = call  <vscale x 4 x i1> @llvm.aarch64.sve.ptrue.nxv4i1(i32 31)
+  %0 = tail call <vscale x 4 x half> @llvm.aarch64.sve.frinta.nxv4f16(<vscale x 4 x half> %x,  <vscale x 4 x i1> %pg, <vscale x 4 x half> %y)
+  ret <vscale x 4 x half> %0
+}
+
+
+define <vscale x 2 x half> @test_svfrinta_nxv2f16_ptrue_u(double %z0, <vscale x 2 x half> %x) {
+; CHECK-LABEL: test_svfrinta_nxv2f16_ptrue_u:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ptrue p0.d
+; CHECK-NEXT:    movprfx z0, z1
+; CHECK-NEXT:    frinta z0.h, p0/m, z1.h
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svfrinta_nxv2f16_ptrue_u:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    ptrue p0.d
+; CHECK-2p2-NEXT:    frinta z0.h, p0/z, z1.h
+; CHECK-2p2-NEXT:    ret
+entry:
+  %pg = call  <vscale x 2 x i1> @llvm.aarch64.sve.ptrue.nxv2i1(i32 31)
+  %0 = tail call <vscale x 2 x half> @llvm.aarch64.sve.frinta.nxv2f16(<vscale x 2 x half> poison,  <vscale x 2 x i1> %pg, <vscale x 2 x half> %x)
+  ret <vscale x 2 x half> %0
+}
+
+define <vscale x 2 x half> @test_svfrinta_nxv2f16_ptrue(double %z0, <vscale x 2 x half> %x, <vscale x 2 x half> %y) {
+; CHECK-LABEL: test_svfrinta_nxv2f16_ptrue:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ptrue p0.d
+; CHECK-NEXT:    movprfx z0, z2
+; CHECK-NEXT:    frinta z0.h, p0/m, z2.h
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svfrinta_nxv2f16_ptrue:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    ptrue p0.d
+; CHECK-2p2-NEXT:    frinta z0.h, p0/z, z2.h
+; CHECK-2p2-NEXT:    ret
+entry:
+  %pg = call  <vscale x 2 x i1> @llvm.aarch64.sve.ptrue.nxv2i1(i32 31)
+  %0 = tail call <vscale x 2 x half> @llvm.aarch64.sve.frinta.nxv2f16(<vscale x 2 x half> %x,  <vscale x 2 x i1> %pg, <vscale x 2 x half> %y)
+  ret <vscale x 2 x half> %0
+}
+
+
+define <vscale x 2 x float> @test_svfrinta_nxv2f32_ptrue_u(double %z0, <vscale x 2 x float> %x) {
+; CHECK-LABEL: test_svfrinta_nxv2f32_ptrue_u:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ptrue p0.d
+; CHECK-NEXT:    movprfx z0, z1
+; CHECK-NEXT:    frinta z0.s, p0/m, z1.s
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svfrinta_nxv2f32_ptrue_u:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    ptrue p0.d
+; CHECK-2p2-NEXT:    frinta z0.s, p0/z, z1.s
+; CHECK-2p2-NEXT:    ret
+entry:
+  %pg = call  <vscale x 2 x i1> @llvm.aarch64.sve.ptrue.nxv2i1(i32 31)
+  %0 = tail call <vscale x 2 x float> @llvm.aarch64.sve.frinta.nxv2f32(<vscale x 2 x float> poison,  <vscale x 2 x i1> %pg, <vscale x 2 x float> %x)
+  ret <vscale x 2 x float> %0
+}
+
+define <vscale x 2 x float> @test_svfrinta_nxv2f32_ptrue(double %z0, <vscale x 2 x float> %x, <vscale x 2 x float> %y) {
+; CHECK-LABEL: test_svfrinta_nxv2f32_ptrue:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ptrue p0.d
+; CHECK-NEXT:    movprfx z0, z2
+; CHECK-NEXT:    frinta z0.s, p0/m, z2.s
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svfrinta_nxv2f32_ptrue:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    ptrue p0.d
+; CHECK-2p2-NEXT:    frinta z0.s, p0/z, z2.s
+; CHECK-2p2-NEXT:    ret
+entry:
+  %pg = call  <vscale x 2 x i1> @llvm.aarch64.sve.ptrue.nxv2i1(i32 31)
+  %0 = tail call <vscale x 2 x float> @llvm.aarch64.sve.frinta.nxv2f32(<vscale x 2 x float> %x,  <vscale x 2 x i1> %pg, <vscale x 2 x float> %y)
+  ret <vscale x 2 x float> %0
+}
+
+
+define <vscale x 4 x float> @test_svfrinta_nxv4f32_ptrue_u(double %z0, <vscale x 4 x float> %x) {
+; CHECK-LABEL: test_svfrinta_nxv4f32_ptrue_u:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ptrue p0.s
+; CHECK-NEXT:    movprfx z0, z1
+; CHECK-NEXT:    frinta z0.s, p0/m, z1.s
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svfrinta_nxv4f32_ptrue_u:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    ptrue p0.s
+; CHECK-2p2-NEXT:    frinta z0.s, p0/z, z1.s
+; CHECK-2p2-NEXT:    ret
+entry:
+  %pg = call  <vscale x 4 x i1> @llvm.aarch64.sve.ptrue.nxv4i1(i32 31)
+  %0 = tail call <vscale x 4 x float> @llvm.aarch64.sve.frinta.nxv4f32(<vscale x 4 x float> poison,  <vscale x 4 x i1> %pg, <vscale x 4 x float> %x)
+  ret <vscale x 4 x float> %0
+}
+
+define <vscale x 4 x float> @test_svfrinta_nxv4f32_ptrue(double %z0, <vscale x 4 x float> %x, <vscale x 4 x float> %y) {
+; CHECK-LABEL: test_svfrinta_nxv4f32_ptrue:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ptrue p0.s
+; CHECK-NEXT:    movprfx z0, z2
+; CHECK-NEXT:    frinta z0.s, p0/m, z2.s
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svfrinta_nxv4f32_ptrue:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    ptrue p0.s
+; CHECK-2p2-NEXT:    frinta z0.s, p0/z, z2.s
+; CHECK-2p2-NEXT:    ret
+entry:
+  %pg = call  <vscale x 4 x i1> @llvm.aarch64.sve.ptrue.nxv4i1(i32 31)
+  %0 = tail call <vscale x 4 x float> @llvm.aarch64.sve.frinta.nxv4f32(<vscale x 4 x float> %x,  <vscale x 4 x i1> %pg, <vscale x 4 x float> %y)
+  ret <vscale x 4 x float> %0
+}
+
+
+define <vscale x 2 x double> @test_svfrinta_nxv2f64_ptrue_u(double %z0, <vscale x 2 x double> %x) {
+; CHECK-LABEL: test_svfrinta_nxv2f64_ptrue_u:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ptrue p0.d
+; CHECK-NEXT:    movprfx z0, z1
+; CHECK-NEXT:    frinta z0.d, p0/m, z1.d
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svfrinta_nxv2f64_ptrue_u:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    ptrue p0.d
+; CHECK-2p2-NEXT:    frinta z0.d, p0/z, z1.d
+; CHECK-2p2-NEXT:    ret
+entry:
+  %pg = call  <vscale x 2 x i1> @llvm.aarch64.sve.ptrue.nxv2i1(i32 31)
+  %0 = tail call <vscale x 2 x double> @llvm.aarch64.sve.frinta.nxv2f64(<vscale x 2 x double> poison,  <vscale x 2 x i1> %pg, <vscale x 2 x double> %x)
+  ret <vscale x 2 x double> %0
+}
+
+define <vscale x 2 x double> @test_svfrinta_nxv2f64_ptrue(double %z0, <vscale x 2 x double> %x, <vscale x 2 x double> %y) {
+; CHECK-LABEL: test_svfrinta_nxv2f64_ptrue:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ptrue p0.d
+; CHECK-NEXT:    movprfx z0, z2
+; CHECK-NEXT:    frinta z0.d, p0/m, z2.d
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svfrinta_nxv2f64_ptrue:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    ptrue p0.d
+; CHECK-2p2-NEXT:    frinta z0.d, p0/z, z2.d
+; CHECK-2p2-NEXT:    ret
+entry:
+  %pg = call  <vscale x 2 x i1> @llvm.aarch64.sve.ptrue.nxv2i1(i32 31)
+  %0 = tail call <vscale x 2 x double> @llvm.aarch64.sve.frinta.nxv2f64(<vscale x 2 x double> %x,  <vscale x 2 x i1> %pg, <vscale x 2 x double> %y)
+  ret <vscale x 2 x double> %0
+}
+
+
+define <vscale x 8 x half> @test_svfrinti_nxv8f16_ptrue_u(double %z0, <vscale x 8 x half> %x) {
+; CHECK-LABEL: test_svfrinti_nxv8f16_ptrue_u:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ptrue p0.h
+; CHECK-NEXT:    movprfx z0, z1
+; CHECK-NEXT:    frinti z0.h, p0/m, z1.h
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svfrinti_nxv8f16_ptrue_u:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    ptrue p0.h
+; CHECK-2p2-NEXT:    frinti z0.h, p0/z, z1.h
+; CHECK-2p2-NEXT:    ret
+entry:
+  %pg = call  <vscale x 8 x i1> @llvm.aarch64.sve.ptrue.nxv8i1(i32 31)
+  %0 = tail call <vscale x 8 x half> @llvm.aarch64.sve.frinti.nxv8f16(<vscale x 8 x half> poison,  <vscale x 8 x i1> %pg, <vscale x 8 x half> %x)
+  ret <vscale x 8 x half> %0
+}
+
+define <vscale x 8 x half> @test_svfrinti_nxv8f16_ptrue(double %z0, <vscale x 8 x half> %x, <vscale x 8 x half> %y) {
+; CHECK-LABEL: test_svfrinti_nxv8f16_ptrue:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ptrue p0.h
+; CHECK-NEXT:    movprfx z0, z2
+; CHECK-NEXT:    frinti z0.h, p0/m, z2.h
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svfrinti_nxv8f16_ptrue:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    ptrue p0.h
+; CHECK-2p2-NEXT:    frinti z0.h, p0/z, z2.h
+; CHECK-2p2-NEXT:    ret
+entry:
+  %pg = call  <vscale x 8 x i1> @llvm.aarch64.sve.ptrue.nxv8i1(i32 31)
+  %0 = tail call <vscale x 8 x half> @llvm.aarch64.sve.frinti.nxv8f16(<vscale x 8 x half> %x,  <vscale x 8 x i1> %pg, <vscale x 8 x half> %y)
+  ret <vscale x 8 x half> %0
+}
+
+
+define <vscale x 4 x half> @test_svfrinti_nxv4f16_ptrue_u(double %z0, <vscale x 4 x half> %x) {
+; CHECK-LABEL: test_svfrinti_nxv4f16_ptrue_u:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ptrue p0.s
+; CHECK-NEXT:    movprfx z0, z1
+; CHECK-NEXT:    frinti z0.h, p0/m, z1.h
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svfrinti_nxv4f16_ptrue_u:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    ptrue p0.s
+; CHECK-2p2-NEXT:    frinti z0.h, p0/z, z1.h
+; CHECK-2p2-NEXT:    ret
+entry:
+  %pg = call  <vscale x 4 x i1> @llvm.aarch64.sve.ptrue.nxv4i1(i32 31)
+  %0 = tail call <vscale x 4 x half> @llvm.aarch64.sve.frinti.nxv4f16(<vscale x 4 x half> poison,  <vscale x 4 x i1> %pg, <vscale x 4 x half> %x)
+  ret <vscale x 4 x half> %0
+}
+
+define <vscale x 4 x half> @test_svfrinti_nxv4f16_ptrue(double %z0, <vscale x 4 x half> %x, <vscale x 4 x half> %y) {
+; CHECK-LABEL: test_svfrinti_nxv4f16_ptrue:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ptrue p0.s
+; CHECK-NEXT:    movprfx z0, z2
+; CHECK-NEXT:    frinti z0.h, p0/m, z2.h
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svfrinti_nxv4f16_ptrue:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    ptrue p0.s
+; CHECK-2p2-NEXT:    frinti z0.h, p0/z, z2.h
+; CHECK-2p2-NEXT:    ret
+entry:
+  %pg = call  <vscale x 4 x i1> @llvm.aarch64.sve.ptrue.nxv4i1(i32 31)
+  %0 = tail call <vscale x 4 x half> @llvm.aarch64.sve.frinti.nxv4f16(<vscale x 4 x half> %x,  <vscale x 4 x i1> %pg, <vscale x 4 x half> %y)
+  ret <vscale x 4 x half> %0
+}
+
+
+define <vscale x 2 x half> @test_svfrinti_nxv2f16_ptrue_u(double %z0, <vscale x 2 x half> %x) {
+; CHECK-LABEL: test_svfrinti_nxv2f16_ptrue_u:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ptrue p0.d
+; CHECK-NEXT:    movprfx z0, z1
+; CHECK-NEXT:    frinti z0.h, p0/m, z1.h
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svfrinti_nxv2f16_ptrue_u:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    ptrue p0.d
+; CHECK-2p2-NEXT:    frinti z0.h, p0/z, z1.h
+; CHECK-2p2-NEXT:    ret
+entry:
+  %pg = call  <vscale x 2 x i1> @llvm.aarch64.sve.ptrue.nxv2i1(i32 31)
+  %0 = tail call <vscale x 2 x half> @llvm.aarch64.sve.frinti.nxv2f16(<vscale x 2 x half> poison,  <vscale x 2 x i1> %pg, <vscale x 2 x half> %x)
+  ret <vscale x 2 x half> %0
+}
+
+define <vscale x 2 x half> @test_svfrinti_nxv2f16_ptrue(double %z0, <vscale x 2 x half> %x, <vscale x 2 x half> %y) {
+; CHECK-LABEL: test_svfrinti_nxv2f16_ptrue:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ptrue p0.d
+; CHECK-NEXT:    movprfx z0, z2
+; CHECK-NEXT:    frinti z0.h, p0/m, z2.h
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svfrinti_nxv2f16_ptrue:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    ptrue p0.d
+; CHECK-2p2-NEXT:    frinti z0.h, p0/z, z2.h
+; CHECK-2p2-NEXT:    ret
+entry:
+  %pg = call  <vscale x 2 x i1> @llvm.aarch64.sve.ptrue.nxv2i1(i32 31)
+  %0 = tail call <vscale x 2 x half> @llvm.aarch64.sve.frinti.nxv2f16(<vscale x 2 x half> %x,  <vscale x 2 x i1> %pg, <vscale x 2 x half> %y)
+  ret <vscale x 2 x half> %0
+}
+
+
+define <vscale x 2 x float> @test_svfrinti_nxv2f32_ptrue_u(double %z0, <vscale x 2 x float> %x) {
+; CHECK-LABEL: test_svfrinti_nxv2f32_ptrue_u:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ptrue p0.d
+; CHECK-NEXT:    movprfx z0, z1
+; CHECK-NEXT:    frinti z0.s, p0/m, z1.s
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svfrinti_nxv2f32_ptrue_u:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    ptrue p0.d
+; CHECK-2p2-NEXT:    frinti z0.s, p0/z, z1.s
+; CHECK-2p2-NEXT:    ret
+entry:
+  %pg = call  <vscale x 2 x i1> @llvm.aarch64.sve.ptrue.nxv2i1(i32 31)
+  %0 = tail call <vscale x 2 x float> @llvm.aarch64.sve.frinti.nxv2f32(<vscale x 2 x float> poison,  <vscale x 2 x i1> %pg, <vscale x 2 x float> %x)
+  ret <vscale x 2 x float> %0
+}
+
+define <vscale x 2 x float> @test_svfrinti_nxv2f32_ptrue(double %z0, <vscale x 2 x float> %x, <vscale x 2 x float> %y) {
+; CHECK-LABEL: test_svfrinti_nxv2f32_ptrue:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ptrue p0.d
+; CHECK-NEXT:    movprfx z0, z2
+; CHECK-NEXT:    frinti z0.s, p0/m, z2.s
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svfrinti_nxv2f32_ptrue:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    ptrue p0.d
+; CHECK-2p2-NEXT:    frinti z0.s, p0/z, z2.s
+; CHECK-2p2-NEXT:    ret
+entry:
+  %pg = call  <vscale x 2 x i1> @llvm.aarch64.sve.ptrue.nxv2i1(i32 31)
+  %0 = tail call <vscale x 2 x float> @llvm.aarch64.sve.frinti.nxv2f32(<vscale x 2 x float> %x,  <vscale x 2 x i1> %pg, <vscale x 2 x float> %y)
+  ret <vscale x 2 x float> %0
+}
+
+
+define <vscale x 4 x float> @test_svfrinti_nxv4f32_ptrue_u(double %z0, <vscale x 4 x float> %x) {
+; CHECK-LABEL: test_svfrinti_nxv4f32_ptrue_u:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ptrue p0.s
+; CHECK-NEXT:    movprfx z0, z1
+; CHECK-NEXT:    frinti z0.s, p0/m, z1.s
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svfrinti_nxv4f32_ptrue_u:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    ptrue p0.s
+; CHECK-2p2-NEXT:    frinti z0.s, p0/z, z1.s
+; CHECK-2p2-NEXT:    ret
+entry:
+  %pg = call  <vscale x 4 x i1> @llvm.aarch64.sve.ptrue.nxv4i1(i32 31)
+  %0 = tail call <vscale x 4 x float> @llvm.aarch64.sve.frinti.nxv4f32(<vscale x 4 x float> poison,  <vscale x 4 x i1> %pg, <vscale x 4 x float> %x)
+  ret <vscale x 4 x float> %0
+}
+
+define <vscale x 4 x float> @test_svfrinti_nxv4f32_ptrue(double %z0, <vscale x 4 x float> %x, <vscale x 4 x float> %y) {
+; CHECK-LABEL: test_svfrinti_nxv4f32_ptrue:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ptrue p0.s
+; CHECK-NEXT:    movprfx z0, z2
+; CHECK-NEXT:    frinti z0.s, p0/m, z2.s
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svfrinti_nxv4f32_ptrue:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    ptrue p0.s
+; CHECK-2p2-NEXT:    frinti z0.s, p0/z, z2.s
+; CHECK-2p2-NEXT:    ret
+entry:
+  %pg = call  <vscale x 4 x i1> @llvm.aarch64.sve.ptrue.nxv4i1(i32 31)
+  %0 = tail call <vscale x 4 x float> @llvm.aarch64.sve.frinti.nxv4f32(<vscale x 4 x float> %x,  <vscale x 4 x i1> %pg, <vscale x 4 x float> %y)
+  ret <vscale x 4 x float> %0
+}
+
+
+define <vscale x 2 x double> @test_svfrinti_nxv2f64_ptrue_u(double %z0, <vscale x 2 x double> %x) {
+; CHECK-LABEL: test_svfrinti_nxv2f64_ptrue_u:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ptrue p0.d
+; CHECK-NEXT:    movprfx z0, z1
+; CHECK-NEXT:    frinti z0.d, p0/m, z1.d
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svfrinti_nxv2f64_ptrue_u:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    ptrue p0.d
+; CHECK-2p2-NEXT:    frinti z0.d, p0/z, z1.d
+; CHECK-2p2-NEXT:    ret
+entry:
+  %pg = call  <vscale x 2 x i1> @llvm.aarch64.sve.ptrue.nxv2i1(i32 31)
+  %0 = tail call <vscale x 2 x double> @llvm.aarch64.sve.frinti.nxv2f64(<vscale x 2 x double> poison,  <vscale x 2 x i1> %pg, <vscale x 2 x double> %x)
+  ret <vscale x 2 x double> %0
+}
+
+define <vscale x 2 x double> @test_svfrinti_nxv2f64_ptrue(double %z0, <vscale x 2 x double> %x, <vscale x 2 x double> %y) {
+; CHECK-LABEL: test_svfrinti_nxv2f64_ptrue:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ptrue p0.d
+; CHECK-NEXT:    movprfx z0, z2
+; CHECK-NEXT:    frinti z0.d, p0/m, z2.d
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svfrinti_nxv2f64_ptrue:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    ptrue p0.d
+; CHECK-2p2-NEXT:    frinti z0.d, p0/z, z2.d
+; CHECK-2p2-NEXT:    ret
+entry:
+  %pg = call  <vscale x 2 x i1> @llvm.aarch64.sve.ptrue.nxv2i1(i32 31)
+  %0 = tail call <vscale x 2 x double> @llvm.aarch64.sve.frinti.nxv2f64(<vscale x 2 x double> %x,  <vscale x 2 x i1> %pg, <vscale x 2 x double> %y)
+  ret <vscale x 2 x double> %0
+}
+
+
+define <vscale x 8 x half> @test_svfrintm_nxv8f16_ptrue_u(double %z0, <vscale x 8 x half> %x) {
+; CHECK-LABEL: test_svfrintm_nxv8f16_ptrue_u:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ptrue p0.h
+; CHECK-NEXT:    movprfx z0, z1
+; CHECK-NEXT:    frintm z0.h, p0/m, z1.h
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svfrintm_nxv8f16_ptrue_u:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    ptrue p0.h
+; CHECK-2p2-NEXT:    frintm z0.h, p0/z, z1.h
+; CHECK-2p2-NEXT:    ret
+entry:
+  %pg = call  <vscale x 8 x i1> @llvm.aarch64.sve.ptrue.nxv8i1(i32 31)
+  %0 = tail call <vscale x 8 x half> @llvm.aarch64.sve.frintm.nxv8f16(<vscale x 8 x half> poison,  <vscale x 8 x i1> %pg, <vscale x 8 x half> %x)
+  ret <vscale x 8 x half> %0
+}
+
+define <vscale x 8 x half> @test_svfrintm_nxv8f16_ptrue(double %z0, <vscale x 8 x half> %x, <vscale x 8 x half> %y) {
+; CHECK-LABEL: test_svfrintm_nxv8f16_ptrue:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ptrue p0.h
+; CHECK-NEXT:    movprfx z0, z2
+; CHECK-NEXT:    frintm z0.h, p0/m, z2.h
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svfrintm_nxv8f16_ptrue:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    ptrue p0.h
+; CHECK-2p2-NEXT:    frintm z0.h, p0/z, z2.h
+; CHECK-2p2-NEXT:    ret
+entry:
+  %pg = call  <vscale x 8 x i1> @llvm.aarch64.sve.ptrue.nxv8i1(i32 31)
+  %0 = tail call <vscale x 8 x half> @llvm.aarch64.sve.frintm.nxv8f16(<vscale x 8 x half> %x,  <vscale x 8 x i1> %pg, <vscale x 8 x half> %y)
+  ret <vscale x 8 x half> %0
+}
+
+
+define <vscale x 4 x half> @test_svfrintm_nxv4f16_ptrue_u(double %z0, <vscale x 4 x half> %x) {
+; CHECK-LABEL: test_svfrintm_nxv4f16_ptrue_u:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ptrue p0.s
+; CHECK-NEXT:    movprfx z0, z1
+; CHECK-NEXT:    frintm z0.h, p0/m, z1.h
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svfrintm_nxv4f16_ptrue_u:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    ptrue p0.s
+; CHECK-2p2-NEXT:    frintm z0.h, p0/z, z1.h
+; CHECK-2p2-NEXT:    ret
+entry:
+  %pg = call  <vscale x 4 x i1> @llvm.aarch64.sve.ptrue.nxv4i1(i32 31)
+  %0 = tail call <vscale x 4 x half> @llvm.aarch64.sve.frintm.nxv4f16(<vscale x 4 x half> poison,  <vscale x 4 x i1> %pg, <vscale x 4 x half> %x)
+  ret <vscale x 4 x half> %0
+}
+
+define <vscale x 4 x half> @test_svfrintm_nxv4f16_ptrue(double %z0, <vscale x 4 x half> %x, <vscale x 4 x half> %y) {
+; CHECK-LABEL: test_svfrintm_nxv4f16_ptrue:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ptrue p0.s
+; CHECK-NEXT:    movprfx z0, z2
+; CHECK-NEXT:    frintm z0.h, p0/m, z2.h
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svfrintm_nxv4f16_ptrue:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    ptrue p0.s
+; CHECK-2p2-NEXT:    frintm z0.h, p0/z, z2.h
+; CHECK-2p2-NEXT:    ret
+entry:
+  %pg = call  <vscale x 4 x i1> @llvm.aarch64.sve.ptrue.nxv4i1(i32 31)
+  %0 = tail call <vscale x 4 x half> @llvm.aarch64.sve.frintm.nxv4f16(<vscale x 4 x half> %x,  <vscale x 4 x i1> %pg, <vscale x 4 x half> %y)
+  ret <vscale x 4 x half> %0
+}
+
+
+define <vscale x 2 x half> @test_svfrintm_nxv2f16_ptrue_u(double %z0, <vscale x 2 x half> %x) {
+; CHECK-LABEL: test_svfrintm_nxv2f16_ptrue_u:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ptrue p0.d
+; CHECK-NEXT:    movprfx z0, z1
+; CHECK-NEXT:    frintm z0.h, p0/m, z1.h
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svfrintm_nxv2f16_ptrue_u:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    ptrue p0.d
+; CHECK-2p2-NEXT:    frintm z0.h, p0/z, z1.h
+; CHECK-2p2-NEXT:    ret
+entry:
+  %pg = call  <vscale x 2 x i1> @llvm.aarch64.sve.ptrue.nxv2i1(i32 31)
+  %0 = tail call <vscale x 2 x half> @llvm.aarch64.sve.frintm.nxv2f16(<vscale x 2 x half> poison,  <vscale x 2 x i1> %pg, <vscale x 2 x half> %x)
+  ret <vscale x 2 x half> %0
+}
+
+define <vscale x 2 x half> @test_svfrintm_nxv2f16_ptrue(double %z0, <vscale x 2 x half> %x, <vscale x 2 x half> %y) {
+; CHECK-LABEL: test_svfrintm_nxv2f16_ptrue:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ptrue p0.d
+; CHECK-NEXT:    movprfx z0, z2
+; CHECK-NEXT:    frintm z0.h, p0/m, z2.h
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svfrintm_nxv2f16_ptrue:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    ptrue p0.d
+; CHECK-2p2-NEXT:    frintm z0.h, p0/z, z2.h
+; CHECK-2p2-NEXT:    ret
+entry:
+  %pg = call  <vscale x 2 x i1> @llvm.aarch64.sve.ptrue.nxv2i1(i32 31)
+  %0 = tail call <vscale x 2 x half> @llvm.aarch64.sve.frintm.nxv2f16(<vscale x 2 x half> %x,  <vscale x 2 x i1> %pg, <vscale x 2 x half> %y)
+  ret <vscale x 2 x half> %0
+}
+
+
+define <vscale x 2 x float> @test_svfrintm_nxv2f32_ptrue_u(double %z0, <vscale x 2 x float> %x) {
+; CHECK-LABEL: test_svfrintm_nxv2f32_ptrue_u:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ptrue p0.d
+; CHECK-NEXT:    movprfx z0, z1
+; CHECK-NEXT:    frintm z0.s, p0/m, z1.s
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svfrintm_nxv2f32_ptrue_u:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    ptrue p0.d
+; CHECK-2p2-NEXT:    frintm z0.s, p0/z, z1.s
+; CHECK-2p2-NEXT:    ret
+entry:
+  %pg = call  <vscale x 2 x i1> @llvm.aarch64.sve.ptrue.nxv2i1(i32 31)
+  %0 = tail call <vscale x 2 x float> @llvm.aarch64.sve.frintm.nxv2f32(<vscale x 2 x float> poison,  <vscale x 2 x i1> %pg, <vscale x 2 x float> %x)
+  ret <vscale x 2 x float> %0
+}
+
+define <vscale x 2 x float> @test_svfrintm_nxv2f32_ptrue(double %z0, <vscale x 2 x float> %x, <vscale x 2 x float> %y) {
+; CHECK-LABEL: test_svfrintm_nxv2f32_ptrue:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ptrue p0.d
+; CHECK-NEXT:    movprfx z0, z2
+; CHECK-NEXT:    frintm z0.s, p0/m, z2.s
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svfrintm_nxv2f32_ptrue:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    ptrue p0.d
+; CHECK-2p2-NEXT:    frintm z0.s, p0/z, z2.s
+; CHECK-2p2-NEXT:    ret
+entry:
+  %pg = call  <vscale x 2 x i1> @llvm.aarch64.sve.ptrue.nxv2i1(i32 31)
+  %0 = tail call <vscale x 2 x float> @llvm.aarch64.sve.frintm.nxv2f32(<vscale x 2 x float> %x,  <vscale x 2 x i1> %pg, <vscale x 2 x float> %y)
+  ret <vscale x 2 x float> %0
+}
+
+
+define <vscale x 4 x float> @test_svfrintm_nxv4f32_ptrue_u(double %z0, <vscale x 4 x float> %x) {
+; CHECK-LABEL: test_svfrintm_nxv4f32_ptrue_u:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ptrue p0.s
+; CHECK-NEXT:    movprfx z0, z1
+; CHECK-NEXT:    frintm z0.s, p0/m, z1.s
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svfrintm_nxv4f32_ptrue_u:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    ptrue p0.s
+; CHECK-2p2-NEXT:    frintm z0.s, p0/z, z1.s
+; CHECK-2p2-NEXT:    ret
+entry:
+  %pg = call  <vscale x 4 x i1> @llvm.aarch64.sve.ptrue.nxv4i1(i32 31)
+  %0 = tail call <vscale x 4 x float> @llvm.aarch64.sve.frintm.nxv4f32(<vscale x 4 x float> poison,  <vscale x 4 x i1> %pg, <vscale x 4 x float> %x)
+  ret <vscale x 4 x float> %0
+}
+
+define <vscale x 4 x float> @test_svfrintm_nxv4f32_ptrue(double %z0, <vscale x 4 x float> %x, <vscale x 4 x float> %y) {
+; CHECK-LABEL: test_svfrintm_nxv4f32_ptrue:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ptrue p0.s
+; CHECK-NEXT:    movprfx z0, z2
+; CHECK-NEXT:    frintm z0.s, p0/m, z2.s
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svfrintm_nxv4f32_ptrue:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    ptrue p0.s
+; CHECK-2p2-NEXT:    frintm z0.s, p0/z, z2.s
+; CHECK-2p2-NEXT:    ret
+entry:
+  %pg = call  <vscale x 4 x i1> @llvm.aarch64.sve.ptrue.nxv4i1(i32 31)
+  %0 = tail call <vscale x 4 x float> @llvm.aarch64.sve.frintm.nxv4f32(<vscale x 4 x float> %x,  <vscale x 4 x i1> %pg, <vscale x 4 x float> %y)
+  ret <vscale x 4 x float> %0
+}
+
+
+define <vscale x 2 x double> @test_svfrintm_nxv2f64_ptrue_u(double %z0, <vscale x 2 x double> %x) {
+; CHECK-LABEL: test_svfrintm_nxv2f64_ptrue_u:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ptrue p0.d
+; CHECK-NEXT:    movprfx z0, z1
+; CHECK-NEXT:    frintm z0.d, p0/m, z1.d
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svfrintm_nxv2f64_ptrue_u:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    ptrue p0.d
+; CHECK-2p2-NEXT:    frintm z0.d, p0/z, z1.d
+; CHECK-2p2-NEXT:    ret
+entry:
+  %pg = call  <vscale x 2 x i1> @llvm.aarch64.sve.ptrue.nxv2i1(i32 31)
+  %0 = tail call <vscale x 2 x double> @llvm.aarch64.sve.frintm.nxv2f64(<vscale x 2 x double> poison,  <vscale x 2 x i1> %pg, <vscale x 2 x double> %x)
+  ret <vscale x 2 x double> %0
+}
+
+define <vscale x 2 x double> @test_svfrintm_nxv2f64_ptrue(double %z0, <vscale x 2 x double> %x, <vscale x 2 x double> %y) {
+; CHECK-LABEL: test_svfrintm_nxv2f64_ptrue:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ptrue p0.d
+; CHECK-NEXT:    movprfx z0, z2
+; CHECK-NEXT:    frintm z0.d, p0/m, z2.d
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svfrintm_nxv2f64_ptrue:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    ptrue p0.d
+; CHECK-2p2-NEXT:    frintm z0.d, p0/z, z2.d
+; CHECK-2p2-NEXT:    ret
+entry:
+  %pg = call  <vscale x 2 x i1> @llvm.aarch64.sve.ptrue.nxv2i1(i32 31)
+  %0 = tail call <vscale x 2 x double> @llvm.aarch64.sve.frintm.nxv2f64(<vscale x 2 x double> %x,  <vscale x 2 x i1> %pg, <vscale x 2 x double> %y)
+  ret <vscale x 2 x double> %0
+}
+
+
+define <vscale x 8 x half> @test_svfrintn_nxv8f16_ptrue_u(double %z0, <vscale x 8 x half> %x) {
+; CHECK-LABEL: test_svfrintn_nxv8f16_ptrue_u:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ptrue p0.h
+; CHECK-NEXT:    movprfx z0, z1
+; CHECK-NEXT:    frintn z0.h, p0/m, z1.h
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svfrintn_nxv8f16_ptrue_u:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    ptrue p0.h
+; CHECK-2p2-NEXT:    frintn z0.h, p0/z, z1.h
+; CHECK-2p2-NEXT:    ret
+entry:
+  %pg = call  <vscale x 8 x i1> @llvm.aarch64.sve.ptrue.nxv8i1(i32 31)
+  %0 = tail call <vscale x 8 x half> @llvm.aarch64.sve.frintn.nxv8f16(<vscale x 8 x half> poison,  <vscale x 8 x i1> %pg, <vscale x 8 x half> %x)
+  ret <vscale x 8 x half> %0
+}
+
+define <vscale x 8 x half> @test_svfrintn_nxv8f16_ptrue(double %z0, <vscale x 8 x half> %x, <vscale x 8 x half> %y) {
+; CHECK-LABEL: test_svfrintn_nxv8f16_ptrue:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ptrue p0.h
+; CHECK-NEXT:    movprfx z0, z2
+; CHECK-NEXT:    frintn z0.h, p0/m, z2.h
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svfrintn_nxv8f16_ptrue:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    ptrue p0.h
+; CHECK-2p2-NEXT:    frintn z0.h, p0/z, z2.h
+; CHECK-2p2-NEXT:    ret
+entry:
+  %pg = call  <vscale x 8 x i1> @llvm.aarch64.sve.ptrue.nxv8i1(i32 31)
+  %0 = tail call <vscale x 8 x half> @llvm.aarch64.sve.frintn.nxv8f16(<vscale x 8 x half> %x,  <vscale x 8 x i1> %pg, <vscale x 8 x half> %y)
+  ret <vscale x 8 x half> %0
+}
+
+
+define <vscale x 4 x half> @test_svfrintn_nxv4f16_ptrue_u(double %z0, <vscale x 4 x half> %x) {
+; CHECK-LABEL: test_svfrintn_nxv4f16_ptrue_u:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ptrue p0.s
+; CHECK-NEXT:    movprfx z0, z1
+; CHECK-NEXT:    frintn z0.h, p0/m, z1.h
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svfrintn_nxv4f16_ptrue_u:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    ptrue p0.s
+; CHECK-2p2-NEXT:    frintn z0.h, p0/z, z1.h
+; CHECK-2p2-NEXT:    ret
+entry:
+  %pg = call  <vscale x 4 x i1> @llvm.aarch64.sve.ptrue.nxv4i1(i32 31)
+  %0 = tail call <vscale x 4 x half> @llvm.aarch64.sve.frintn.nxv4f16(<vscale x 4 x half> poison,  <vscale x 4 x i1> %pg, <vscale x 4 x half> %x)
+  ret <vscale x 4 x half> %0
+}
+
+define <vscale x 4 x half> @test_svfrintn_nxv4f16_ptrue(double %z0, <vscale x 4 x half> %x, <vscale x 4 x half> %y) {
+; CHECK-LABEL: test_svfrintn_nxv4f16_ptrue:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ptrue p0.s
+; CHECK-NEXT:    movprfx z0, z2
+; CHECK-NEXT:    frintn z0.h, p0/m, z2.h
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svfrintn_nxv4f16_ptrue:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    ptrue p0.s
+; CHECK-2p2-NEXT:    frintn z0.h, p0/z, z2.h
+; CHECK-2p2-NEXT:    ret
+entry:
+  %pg = call  <vscale x 4 x i1> @llvm.aarch64.sve.ptrue.nxv4i1(i32 31)
+  %0 = tail call <vscale x 4 x half> @llvm.aarch64.sve.frintn.nxv4f16(<vscale x 4 x half> %x,  <vscale x 4 x i1> %pg, <vscale x 4 x half> %y)
+  ret <vscale x 4 x half> %0
+}
+
+
+define <vscale x 2 x half> @test_svfrintn_nxv2f16_ptrue_u(double %z0, <vscale x 2 x half> %x) {
+; CHECK-LABEL: test_svfrintn_nxv2f16_ptrue_u:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ptrue p0.d
+; CHECK-NEXT:    movprfx z0, z1
+; CHECK-NEXT:    frintn z0.h, p0/m, z1.h
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svfrintn_nxv2f16_ptrue_u:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    ptrue p0.d
+; CHECK-2p2-NEXT:    frintn z0.h, p0/z, z1.h
+; CHECK-2p2-NEXT:    ret
+entry:
+  %pg = call  <vscale x 2 x i1> @llvm.aarch64.sve.ptrue.nxv2i1(i32 31)
+  %0 = tail call <vscale x 2 x half> @llvm.aarch64.sve.frintn.nxv2f16(<vscale x 2 x half> poison,  <vscale x 2 x i1> %pg, <vscale x 2 x half> %x)
+  ret <vscale x 2 x half> %0
+}
+
+define <vscale x 2 x half> @test_svfrintn_nxv2f16_ptrue(double %z0, <vscale x 2 x half> %x, <vscale x 2 x half> %y) {
+; CHECK-LABEL: test_svfrintn_nxv2f16_ptrue:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ptrue p0.d
+; CHECK-NEXT:    movprfx z0, z2
+; CHECK-NEXT:    frintn z0.h, p0/m, z2.h
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svfrintn_nxv2f16_ptrue:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    ptrue p0.d
+; CHECK-2p2-NEXT:    frintn z0.h, p0/z, z2.h
+; CHECK-2p2-NEXT:    ret
+entry:
+  %pg = call  <vscale x 2 x i1> @llvm.aarch64.sve.ptrue.nxv2i1(i32 31)
+  %0 = tail call <vscale x 2 x half> @llvm.aarch64.sve.frintn.nxv2f16(<vscale x 2 x half> %x,  <vscale x 2 x i1> %pg, <vscale x 2 x half> %y)
+  ret <vscale x 2 x half> %0
+}
+
+
+define <vscale x 2 x float> @test_svfrintn_nxv2f32_ptrue_u(double %z0, <vscale x 2 x float> %x) {
+; CHECK-LABEL: test_svfrintn_nxv2f32_ptrue_u:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ptrue p0.d
+; CHECK-NEXT:    movprfx z0, z1
+; CHECK-NEXT:    frintn z0.s, p0/m, z1.s
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svfrintn_nxv2f32_ptrue_u:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    ptrue p0.d
+; CHECK-2p2-NEXT:    frintn z0.s, p0/z, z1.s
+; CHECK-2p2-NEXT:    ret
+entry:
+  %pg = call  <vscale x 2 x i1> @llvm.aarch64.sve.ptrue.nxv2i1(i32 31)
+  %0 = tail call <vscale x 2 x float> @llvm.aarch64.sve.frintn.nxv2f32(<vscale x 2 x float> poison,  <vscale x 2 x i1> %pg, <vscale x 2 x float> %x)
+  ret <vscale x 2 x float> %0
+}
+
+define <vscale x 2 x float> @test_svfrintn_nxv2f32_ptrue(double %z0, <vscale x 2 x float> %x, <vscale x 2 x float> %y) {
+; CHECK-LABEL: test_svfrintn_nxv2f32_ptrue:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ptrue p0.d
+; CHECK-NEXT:    movprfx z0, z2
+; CHECK-NEXT:    frintn z0.s, p0/m, z2.s
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svfrintn_nxv2f32_ptrue:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    ptrue p0.d
+; CHECK-2p2-NEXT:    frintn z0.s, p0/z, z2.s
+; CHECK-2p2-NEXT:    ret
+entry:
+  %pg = call  <vscale x 2 x i1> @llvm.aarch64.sve.ptrue.nxv2i1(i32 31)
+  %0 = tail call <vscale x 2 x float> @llvm.aarch64.sve.frintn.nxv2f32(<vscale x 2 x float> %x,  <vscale x 2 x i1> %pg, <vscale x 2 x float> %y)
+  ret <vscale x 2 x float> %0
+}
+
+
+define <vscale x 4 x float> @test_svfrintn_nxv4f32_ptrue_u(double %z0, <vscale x 4 x float> %x) {
+; CHECK-LABEL: test_svfrintn_nxv4f32_ptrue_u:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ptrue p0.s
+; CHECK-NEXT:    movprfx z0, z1
+; CHECK-NEXT:    frintn z0.s, p0/m, z1.s
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svfrintn_nxv4f32_ptrue_u:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    ptrue p0.s
+; CHECK-2p2-NEXT:    frintn z0.s, p0/z, z1.s
+; CHECK-2p2-NEXT:    ret
+entry:
+  %pg = call  <vscale x 4 x i1> @llvm.aarch64.sve.ptrue.nxv4i1(i32 31)
+  %0 = tail call <vscale x 4 x float> @llvm.aarch64.sve.frintn.nxv4f32(<vscale x 4 x float> poison,  <vscale x 4 x i1> %pg, <vscale x 4 x float> %x)
+  ret <vscale x 4 x float> %0
+}
+
+define <vscale x 4 x float> @test_svfrintn_nxv4f32_ptrue(double %z0, <vscale x 4 x float> %x, <vscale x 4 x float> %y) {
+; CHECK-LABEL: test_svfrintn_nxv4f32_ptrue:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ptrue p0.s
+; CHECK-NEXT:    movprfx z0, z2
+; CHECK-NEXT:    frintn z0.s, p0/m, z2.s
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svfrintn_nxv4f32_ptrue:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    ptrue p0.s
+; CHECK-2p2-NEXT:    frintn z0.s, p0/z, z2.s
+; CHECK-2p2-NEXT:    ret
+entry:
+  %pg = call  <vscale x 4 x i1> @llvm.aarch64.sve.ptrue.nxv4i1(i32 31)
+  %0 = tail call <vscale x 4 x float> @llvm.aarch64.sve.frintn.nxv4f32(<vscale x 4 x float> %x,  <vscale x 4 x i1> %pg, <vscale x 4 x float> %y)
+  ret <vscale x 4 x float> %0
+}
+
+
+define <vscale x 2 x double> @test_svfrintn_nxv2f64_ptrue_u(double %z0, <vscale x 2 x double> %x) {
+; CHECK-LABEL: test_svfrintn_nxv2f64_ptrue_u:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ptrue p0.d
+; CHECK-NEXT:    movprfx z0, z1
+; CHECK-NEXT:    frintn z0.d, p0/m, z1.d
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svfrintn_nxv2f64_ptrue_u:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    ptrue p0.d
+; CHECK-2p2-NEXT:    frintn z0.d, p0/z, z1.d
+; CHECK-2p2-NEXT:    ret
+entry:
+  %pg = call  <vscale x 2 x i1> @llvm.aarch64.sve.ptrue.nxv2i1(i32 31)
+  %0 = tail call <vscale x 2 x double> @llvm.aarch64.sve.frintn.nxv2f64(<vscale x 2 x double> poison,  <vscale x 2 x i1> %pg, <vscale x 2 x double> %x)
+  ret <vscale x 2 x double> %0
+}
+
+define <vscale x 2 x double> @test_svfrintn_nxv2f64_ptrue(double %z0, <vscale x 2 x double> %x, <vscale x 2 x double> %y) {
+; CHECK-LABEL: test_svfrintn_nxv2f64_ptrue:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ptrue p0.d
+; CHECK-NEXT:    movprfx z0, z2
+; CHECK-NEXT:    frintn z0.d, p0/m, z2.d
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svfrintn_nxv2f64_ptrue:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    ptrue p0.d
+; CHECK-2p2-NEXT:    frintn z0.d, p0/z, z2.d
+; CHECK-2p2-NEXT:    ret
+entry:
+  %pg = call  <vscale x 2 x i1> @llvm.aarch64.sve.ptrue.nxv2i1(i32 31)
+  %0 = tail call <vscale x 2 x double> @llvm.aarch64.sve.frintn.nxv2f64(<vscale x 2 x double> %x,  <vscale x 2 x i1> %pg, <vscale x 2 x double> %y)
+  ret <vscale x 2 x double> %0
+}
+
+
+define <vscale x 8 x half> @test_svfrintp_nxv8f16_ptrue_u(double %z0, <vscale x 8 x half> %x) {
+; CHECK-LABEL: test_svfrintp_nxv8f16_ptrue_u:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ptrue p0.h
+; CHECK-NEXT:    movprfx z0, z1
+; CHECK-NEXT:    frintp z0.h, p0/m, z1.h
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svfrintp_nxv8f16_ptrue_u:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    ptrue p0.h
+; CHECK-2p2-NEXT:    frintp z0.h, p0/z, z1.h
+; CHECK-2p2-NEXT:    ret
+entry:
+  %pg = call  <vscale x 8 x i1> @llvm.aarch64.sve.ptrue.nxv8i1(i32 31)
+  %0 = tail call <vscale x 8 x half> @llvm.aarch64.sve.frintp.nxv8f16(<vscale x 8 x half> poison,  <vscale x 8 x i1> %pg, <vscale x 8 x half> %x)
+  ret <vscale x 8 x half> %0
+}
+
+define <vscale x 8 x half> @test_svfrintp_nxv8f16_ptrue(double %z0, <vscale x 8 x half> %x, <vscale x 8 x half> %y) {
+; CHECK-LABEL: test_svfrintp_nxv8f16_ptrue:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ptrue p0.h
+; CHECK-NEXT:    movprfx z0, z2
+; CHECK-NEXT:    frintp z0.h, p0/m, z2.h
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svfrintp_nxv8f16_ptrue:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    ptrue p0.h
+; CHECK-2p2-NEXT:    frintp z0.h, p0/z, z2.h
+; CHECK-2p2-NEXT:    ret
+entry:
+  %pg = call  <vscale x 8 x i1> @llvm.aarch64.sve.ptrue.nxv8i1(i32 31)
+  %0 = tail call <vscale x 8 x half> @llvm.aarch64.sve.frintp.nxv8f16(<vscale x 8 x half> %x,  <vscale x 8 x i1> %pg, <vscale x 8 x half> %y)
+  ret <vscale x 8 x half> %0
+}
+
+
+define <vscale x 4 x half> @test_svfrintp_nxv4f16_ptrue_u(double %z0, <vscale x 4 x half> %x) {
+; CHECK-LABEL: test_svfrintp_nxv4f16_ptrue_u:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ptrue p0.s
+; CHECK-NEXT:    movprfx z0, z1
+; CHECK-NEXT:    frintp z0.h, p0/m, z1.h
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svfrintp_nxv4f16_ptrue_u:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    ptrue p0.s
+; CHECK-2p2-NEXT:    frintp z0.h, p0/z, z1.h
+; CHECK-2p2-NEXT:    ret
+entry:
+  %pg = call  <vscale x 4 x i1> @llvm.aarch64.sve.ptrue.nxv4i1(i32 31)
+  %0 = tail call <vscale x 4 x half> @llvm.aarch64.sve.frintp.nxv4f16(<vscale x 4 x half> poison,  <vscale x 4 x i1> %pg, <vscale x 4 x half> %x)
+  ret <vscale x 4 x half> %0
+}
+
+define <vscale x 4 x half> @test_svfrintp_nxv4f16_ptrue(double %z0, <vscale x 4 x half> %x, <vscale x 4 x half> %y) {
+; CHECK-LABEL: test_svfrintp_nxv4f16_ptrue:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ptrue p0.s
+; CHECK-NEXT:    movprfx z0, z2
+; CHECK-NEXT:    frintp z0.h, p0/m, z2.h
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svfrintp_nxv4f16_ptrue:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    ptrue p0.s
+; CHECK-2p2-NEXT:    frintp z0.h, p0/z, z2.h
+; CHECK-2p2-NEXT:    ret
+entry:
+  %pg = call  <vscale x 4 x i1> @llvm.aarch64.sve.ptrue.nxv4i1(i32 31)
+  %0 = tail call <vscale x 4 x half> @llvm.aarch64.sve.frintp.nxv4f16(<vscale x 4 x half> %x,  <vscale x 4 x i1> %pg, <vscale x 4 x half> %y)
+  ret <vscale x 4 x half> %0
+}
+
+
+define <vscale x 2 x half> @test_svfrintp_nxv2f16_ptrue_u(double %z0, <vscale x 2 x half> %x) {
+; CHECK-LABEL: test_svfrintp_nxv2f16_ptrue_u:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ptrue p0.d
+; CHECK-NEXT:    movprfx z0, z1
+; CHECK-NEXT:    frintp z0.h, p0/m, z1.h
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svfrintp_nxv2f16_ptrue_u:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    ptrue p0.d
+; CHECK-2p2-NEXT:    frintp z0.h, p0/z, z1.h
+; CHECK-2p2-NEXT:    ret
+entry:
+  %pg = call  <vscale x 2 x i1> @llvm.aarch64.sve.ptrue.nxv2i1(i32 31)
+  %0 = tail call <vscale x 2 x half> @llvm.aarch64.sve.frintp.nxv2f16(<vscale x 2 x half> poison,  <vscale x 2 x i1> %pg, <vscale x 2 x half> %x)
+  ret <vscale x 2 x half> %0
+}
+
+define <vscale x 2 x half> @test_svfrintp_nxv2f16_ptrue(double %z0, <vscale x 2 x half> %x, <vscale x 2 x half> %y) {
+; CHECK-LABEL: test_svfrintp_nxv2f16_ptrue:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ptrue p0.d
+; CHECK-NEXT:    movprfx z0, z2
+; CHECK-NEXT:    frintp z0.h, p0/m, z2.h
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svfrintp_nxv2f16_ptrue:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    ptrue p0.d
+; CHECK-2p2-NEXT:    frintp z0.h, p0/z, z2.h
+; CHECK-2p2-NEXT:    ret
+entry:
+  %pg = call  <vscale x 2 x i1> @llvm.aarch64.sve.ptrue.nxv2i1(i32 31)
+  %0 = tail call <vscale x 2 x half> @llvm.aarch64.sve.frintp.nxv2f16(<vscale x 2 x half> %x,  <vscale x 2 x i1> %pg, <vscale x 2 x half> %y)
+  ret <vscale x 2 x half> %0
+}
+
+
+define <vscale x 2 x float> @test_svfrintp_nxv2f32_ptrue_u(double %z0, <vscale x 2 x float> %x) {
+; CHECK-LABEL: test_svfrintp_nxv2f32_ptrue_u:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ptrue p0.d
+; CHECK-NEXT:    movprfx z0, z1
+; CHECK-NEXT:    frintp z0.s, p0/m, z1.s
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svfrintp_nxv2f32_ptrue_u:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    ptrue p0.d
+; CHECK-2p2-NEXT:    frintp z0.s, p0/z, z1.s
+; CHECK-2p2-NEXT:    ret
+entry:
+  %pg = call  <vscale x 2 x i1> @llvm.aarch64.sve.ptrue.nxv2i1(i32 31)
+  %0 = tail call <vscale x 2 x float> @llvm.aarch64.sve.frintp.nxv2f32(<vscale x 2 x float> poison,  <vscale x 2 x i1> %pg, <vscale x 2 x float> %x)
+  ret <vscale x 2 x float> %0
+}
+
+define <vscale x 2 x float> @test_svfrintp_nxv2f32_ptrue(double %z0, <vscale x 2 x float> %x, <vscale x 2 x float> %y) {
+; CHECK-LABEL: test_svfrintp_nxv2f32_ptrue:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ptrue p0.d
+; CHECK-NEXT:    movprfx z0, z2
+; CHECK-NEXT:    frintp z0.s, p0/m, z2.s
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svfrintp_nxv2f32_ptrue:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    ptrue p0.d
+; CHECK-2p2-NEXT:    frintp z0.s, p0/z, z2.s
+; CHECK-2p2-NEXT:    ret
+entry:
+  %pg = call  <vscale x 2 x i1> @llvm.aarch64.sve.ptrue.nxv2i1(i32 31)
+  %0 = tail call <vscale x 2 x float> @llvm.aarch64.sve.frintp.nxv2f32(<vscale x 2 x float> %x,  <vscale x 2 x i1> %pg, <vscale x 2 x float> %y)
+  ret <vscale x 2 x float> %0
+}
+
+
+define <vscale x 4 x float> @test_svfrintp_nxv4f32_ptrue_u(double %z0, <vscale x 4 x float> %x) {
+; CHECK-LABEL: test_svfrintp_nxv4f32_ptrue_u:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ptrue p0.s
+; CHECK-NEXT:    movprfx z0, z1
+; CHECK-NEXT:    frintp z0.s, p0/m, z1.s
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svfrintp_nxv4f32_ptrue_u:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    ptrue p0.s
+; CHECK-2p2-NEXT:    frintp z0.s, p0/z, z1.s
+; CHECK-2p2-NEXT:    ret
+entry:
+  %pg = call  <vscale x 4 x i1> @llvm.aarch64.sve.ptrue.nxv4i1(i32 31)
+  %0 = tail call <vscale x 4 x float> @llvm.aarch64.sve.frintp.nxv4f32(<vscale x 4 x float> poison,  <vscale x 4 x i1> %pg, <vscale x 4 x float> %x)
+  ret <vscale x 4 x float> %0
+}
+
+define <vscale x 4 x float> @test_svfrintp_nxv4f32_ptrue(double %z0, <vscale x 4 x float> %x, <vscale x 4 x float> %y) {
+; CHECK-LABEL: test_svfrintp_nxv4f32_ptrue:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ptrue p0.s
+; CHECK-NEXT:    movprfx z0, z2
+; CHECK-NEXT:    frintp z0.s, p0/m, z2.s
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svfrintp_nxv4f32_ptrue:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    ptrue p0.s
+; CHECK-2p2-NEXT:    frintp z0.s, p0/z, z2.s
+; CHECK-2p2-NEXT:    ret
+entry:
+  %pg = call  <vscale x 4 x i1> @llvm.aarch64.sve.ptrue.nxv4i1(i32 31)
+  %0 = tail call <vscale x 4 x float> @llvm.aarch64.sve.frintp.nxv4f32(<vscale x 4 x float> %x,  <vscale x 4 x i1> %pg, <vscale x 4 x float> %y)
+  ret <vscale x 4 x float> %0
+}
+
+
+define <vscale x 2 x double> @test_svfrintp_nxv2f64_ptrue_u(double %z0, <vscale x 2 x double> %x) {
+; CHECK-LABEL: test_svfrintp_nxv2f64_ptrue_u:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ptrue p0.d
+; CHECK-NEXT:    movprfx z0, z1
+; CHECK-NEXT:    frintp z0.d, p0/m, z1.d
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svfrintp_nxv2f64_ptrue_u:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    ptrue p0.d
+; CHECK-2p2-NEXT:    frintp z0.d, p0/z, z1.d
+; CHECK-2p2-NEXT:    ret
+entry:
+  %pg = call  <vscale x 2 x i1> @llvm.aarch64.sve.ptrue.nxv2i1(i32 31)
+  %0 = tail call <vscale x 2 x double> @llvm.aarch64.sve.frintp.nxv2f64(<vscale x 2 x double> poison,  <vscale x 2 x i1> %pg, <vscale x 2 x double> %x)
+  ret <vscale x 2 x double> %0
+}
+
+define <vscale x 2 x double> @test_svfrintp_nxv2f64_ptrue(double %z0, <vscale x 2 x double> %x, <vscale x 2 x double> %y) {
+; CHECK-LABEL: test_svfrintp_nxv2f64_ptrue:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ptrue p0.d
+; CHECK-NEXT:    movprfx z0, z2
+; CHECK-NEXT:    frintp z0.d, p0/m, z2.d
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svfrintp_nxv2f64_ptrue:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    ptrue p0.d
+; CHECK-2p2-NEXT:    frintp z0.d, p0/z, z2.d
+; CHECK-2p2-NEXT:    ret
+entry:
+  %pg = call  <vscale x 2 x i1> @llvm.aarch64.sve.ptrue.nxv2i1(i32 31)
+  %0 = tail call <vscale x 2 x double> @llvm.aarch64.sve.frintp.nxv2f64(<vscale x 2 x double> %x,  <vscale x 2 x i1> %pg, <vscale x 2 x double> %y)
+  ret <vscale x 2 x double> %0
+}
+
+
+define <vscale x 8 x half> @test_svfrintx_nxv8f16_ptrue_u(double %z0, <vscale x 8 x half> %x) {
+; CHECK-LABEL: test_svfrintx_nxv8f16_ptrue_u:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ptrue p0.h
+; CHECK-NEXT:    movprfx z0, z1
+; CHECK-NEXT:    frintx z0.h, p0/m, z1.h
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svfrintx_nxv8f16_ptrue_u:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    ptrue p0.h
+; CHECK-2p2-NEXT:    frintx z0.h, p0/z, z1.h
+; CHECK-2p2-NEXT:    ret
+entry:
+  %pg = call  <vscale x 8 x i1> @llvm.aarch64.sve.ptrue.nxv8i1(i32 31)
+  %0 = tail call <vscale x 8 x half> @llvm.aarch64.sve.frintx.nxv8f16(<vscale x 8 x half> poison,  <vscale x 8 x i1> %pg, <vscale x 8 x half> %x)
+  ret <vscale x 8 x half> %0
+}
+
+define <vscale x 8 x half> @test_svfrintx_nxv8f16_ptrue(double %z0, <vscale x 8 x half> %x, <vscale x 8 x half> %y) {
+; CHECK-LABEL: test_svfrintx_nxv8f16_ptrue:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ptrue p0.h
+; CHECK-NEXT:    movprfx z0, z2
+; CHECK-NEXT:    frintx z0.h, p0/m, z2.h
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svfrintx_nxv8f16_ptrue:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    ptrue p0.h
+; CHECK-2p2-NEXT:    frintx z0.h, p0/z, z2.h
+; CHECK-2p2-NEXT:    ret
+entry:
+  %pg = call  <vscale x 8 x i1> @llvm.aarch64.sve.ptrue.nxv8i1(i32 31)
+  %0 = tail call <vscale x 8 x half> @llvm.aarch64.sve.frintx.nxv8f16(<vscale x 8 x half> %x,  <vscale x 8 x i1> %pg, <vscale x 8 x half> %y)
+  ret <vscale x 8 x half> %0
+}
+
+
+define <vscale x 4 x half> @test_svfrintx_nxv4f16_ptrue_u(double %z0, <vscale x 4 x half> %x) {
+; CHECK-LABEL: test_svfrintx_nxv4f16_ptrue_u:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ptrue p0.s
+; CHECK-NEXT:    movprfx z0, z1
+; CHECK-NEXT:    frintx z0.h, p0/m, z1.h
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svfrintx_nxv4f16_ptrue_u:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    ptrue p0.s
+; CHECK-2p2-NEXT:    frintx z0.h, p0/z, z1.h
+; CHECK-2p2-NEXT:    ret
+entry:
+  %pg = call  <vscale x 4 x i1> @llvm.aarch64.sve.ptrue.nxv4i1(i32 31)
+  %0 = tail call <vscale x 4 x half> @llvm.aarch64.sve.frintx.nxv4f16(<vscale x 4 x half> poison,  <vscale x 4 x i1> %pg, <vscale x 4 x half> %x)
+  ret <vscale x 4 x half> %0
+}
+
+define <vscale x 4 x half> @test_svfrintx_nxv4f16_ptrue(double %z0, <vscale x 4 x half> %x, <vscale x 4 x half> %y) {
+; CHECK-LABEL: test_svfrintx_nxv4f16_ptrue:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ptrue p0.s
+; CHECK-NEXT:    movprfx z0, z2
+; CHECK-NEXT:    frintx z0.h, p0/m, z2.h
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svfrintx_nxv4f16_ptrue:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    ptrue p0.s
+; CHECK-2p2-NEXT:    frintx z0.h, p0/z, z2.h
+; CHECK-2p2-NEXT:    ret
+entry:
+  %pg = call  <vscale x 4 x i1> @llvm.aarch64.sve.ptrue.nxv4i1(i32 31)
+  %0 = tail call <vscale x 4 x half> @llvm.aarch64.sve.frintx.nxv4f16(<vscale x 4 x half> %x,  <vscale x 4 x i1> %pg, <vscale x 4 x half> %y)
+  ret <vscale x 4 x half> %0
+}
+
+
+define <vscale x 2 x half> @test_svfrintx_nxv2f16_ptrue_u(double %z0, <vscale x 2 x half> %x) {
+; CHECK-LABEL: test_svfrintx_nxv2f16_ptrue_u:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ptrue p0.d
+; CHECK-NEXT:    movprfx z0, z1
+; CHECK-NEXT:    frintx z0.h, p0/m, z1.h
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svfrintx_nxv2f16_ptrue_u:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    ptrue p0.d
+; CHECK-2p2-NEXT:    frintx z0.h, p0/z, z1.h
+; CHECK-2p2-NEXT:    ret
+entry:
+  %pg = call  <vscale x 2 x i1> @llvm.aarch64.sve.ptrue.nxv2i1(i32 31)
+  %0 = tail call <vscale x 2 x half> @llvm.aarch64.sve.frintx.nxv2f16(<vscale x 2 x half> poison,  <vscale x 2 x i1> %pg, <vscale x 2 x half> %x)
+  ret <vscale x 2 x half> %0
+}
+
+define <vscale x 2 x half> @test_svfrintx_nxv2f16_ptrue(double %z0, <vscale x 2 x half> %x, <vscale x 2 x half> %y) {
+; CHECK-LABEL: test_svfrintx_nxv2f16_ptrue:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ptrue p0.d
+; CHECK-NEXT:    movprfx z0, z2
+; CHECK-NEXT:    frintx z0.h, p0/m, z2.h
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svfrintx_nxv2f16_ptrue:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    ptrue p0.d
+; CHECK-2p2-NEXT:    frintx z0.h, p0/z, z2.h
+; CHECK-2p2-NEXT:    ret
+entry:
+  %pg = call  <vscale x 2 x i1> @llvm.aarch64.sve.ptrue.nxv2i1(i32 31)
+  %0 = tail call <vscale x 2 x half> @llvm.aarch64.sve.frintx.nxv2f16(<vscale x 2 x half> %x,  <vscale x 2 x i1> %pg, <vscale x 2 x half> %y)
+  ret <vscale x 2 x half> %0
+}
+
+
+define <vscale x 2 x float> @test_svfrintx_nxv2f32_ptrue_u(double %z0, <vscale x 2 x float> %x) {
+; CHECK-LABEL: test_svfrintx_nxv2f32_ptrue_u:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ptrue p0.d
+; CHECK-NEXT:    movprfx z0, z1
+; CHECK-NEXT:    frintx z0.s, p0/m, z1.s
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svfrintx_nxv2f32_ptrue_u:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    ptrue p0.d
+; CHECK-2p2-NEXT:    frintx z0.s, p0/z, z1.s
+; CHECK-2p2-NEXT:    ret
+entry:
+  %pg = call  <vscale x 2 x i1> @llvm.aarch64.sve.ptrue.nxv2i1(i32 31)
+  %0 = tail call <vscale x 2 x float> @llvm.aarch64.sve.frintx.nxv2f32(<vscale x 2 x float> poison,  <vscale x 2 x i1> %pg, <vscale x 2 x float> %x)
+  ret <vscale x 2 x float> %0
+}
+
+define <vscale x 2 x float> @test_svfrintx_nxv2f32_ptrue(double %z0, <vscale x 2 x float> %x, <vscale x 2 x float> %y) {
+; CHECK-LABEL: test_svfrintx_nxv2f32_ptrue:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ptrue p0.d
+; CHECK-NEXT:    movprfx z0, z2
+; CHECK-NEXT:    frintx z0.s, p0/m, z2.s
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svfrintx_nxv2f32_ptrue:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    ptrue p0.d
+; CHECK-2p2-NEXT:    frintx z0.s, p0/z, z2.s
+; CHECK-2p2-NEXT:    ret
+entry:
+  %pg = call  <vscale x 2 x i1> @llvm.aarch64.sve.ptrue.nxv2i1(i32 31)
+  %0 = tail call <vscale x 2 x float> @llvm.aarch64.sve.frintx.nxv2f32(<vscale x 2 x float> %x,  <vscale x 2 x i1> %pg, <vscale x 2 x float> %y)
+  ret <vscale x 2 x float> %0
+}
+
+
+define <vscale x 4 x float> @test_svfrintx_nxv4f32_ptrue_u(double %z0, <vscale x 4 x float> %x) {
+; CHECK-LABEL: test_svfrintx_nxv4f32_ptrue_u:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ptrue p0.s
+; CHECK-NEXT:    movprfx z0, z1
+; CHECK-NEXT:    frintx z0.s, p0/m, z1.s
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svfrintx_nxv4f32_ptrue_u:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    ptrue p0.s
+; CHECK-2p2-NEXT:    frintx z0.s, p0/z, z1.s
+; CHECK-2p2-NEXT:    ret
+entry:
+  %pg = call  <vscale x 4 x i1> @llvm.aarch64.sve.ptrue.nxv4i1(i32 31)
+  %0 = tail call <vscale x 4 x float> @llvm.aarch64.sve.frintx.nxv4f32(<vscale x 4 x float> poison,  <vscale x 4 x i1> %pg, <vscale x 4 x float> %x)
+  ret <vscale x 4 x float> %0
+}
+
+define <vscale x 4 x float> @test_svfrintx_nxv4f32_ptrue(double %z0, <vscale x 4 x float> %x, <vscale x 4 x float> %y) {
+; CHECK-LABEL: test_svfrintx_nxv4f32_ptrue:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ptrue p0.s
+; CHECK-NEXT:    movprfx z0, z2
+; CHECK-NEXT:    frintx z0.s, p0/m, z2.s
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svfrintx_nxv4f32_ptrue:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    ptrue p0.s
+; CHECK-2p2-NEXT:    frintx z0.s, p0/z, z2.s
+; CHECK-2p2-NEXT:    ret
+entry:
+  %pg = call  <vscale x 4 x i1> @llvm.aarch64.sve.ptrue.nxv4i1(i32 31)
+  %0 = tail call <vscale x 4 x float> @llvm.aarch64.sve.frintx.nxv4f32(<vscale x 4 x float> %x,  <vscale x 4 x i1> %pg, <vscale x 4 x float> %y)
+  ret <vscale x 4 x float> %0
+}
+
+
+define <vscale x 2 x double> @test_svfrintx_nxv2f64_ptrue_u(double %z0, <vscale x 2 x double> %x) {
+; CHECK-LABEL: test_svfrintx_nxv2f64_ptrue_u:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ptrue p0.d
+; CHECK-NEXT:    movprfx z0, z1
+; CHECK-NEXT:    frintx z0.d, p0/m, z1.d
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svfrintx_nxv2f64_ptrue_u:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    ptrue p0.d
+; CHECK-2p2-NEXT:    frintx z0.d, p0/z, z1.d
+; CHECK-2p2-NEXT:    ret
+entry:
+  %pg = call  <vscale x 2 x i1> @llvm.aarch64.sve.ptrue.nxv2i1(i32 31)
+  %0 = tail call <vscale x 2 x double> @llvm.aarch64.sve.frintx.nxv2f64(<vscale x 2 x double> poison,  <vscale x 2 x i1> %pg, <vscale x 2 x double> %x)
+  ret <vscale x 2 x double> %0
+}
+
+define <vscale x 2 x double> @test_svfrintx_nxv2f64_ptrue(double %z0, <vscale x 2 x double> %x, <vscale x 2 x double> %y) {
+; CHECK-LABEL: test_svfrintx_nxv2f64_ptrue:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ptrue p0.d
+; CHECK-NEXT:    movprfx z0, z2
+; CHECK-NEXT:    frintx z0.d, p0/m, z2.d
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svfrintx_nxv2f64_ptrue:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    ptrue p0.d
+; CHECK-2p2-NEXT:    frintx z0.d, p0/z, z2.d
+; CHECK-2p2-NEXT:    ret
+entry:
+  %pg = call  <vscale x 2 x i1> @llvm.aarch64.sve.ptrue.nxv2i1(i32 31)
+  %0 = tail call <vscale x 2 x double> @llvm.aarch64.sve.frintx.nxv2f64(<vscale x 2 x double> %x,  <vscale x 2 x i1> %pg, <vscale x 2 x double> %y)
+  ret <vscale x 2 x double> %0
+}
+
+
+define <vscale x 8 x half> @test_svfrintz_nxv8f16_ptrue_u(double %z0, <vscale x 8 x half> %x) {
+; CHECK-LABEL: test_svfrintz_nxv8f16_ptrue_u:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ptrue p0.h
+; CHECK-NEXT:    movprfx z0, z1
+; CHECK-NEXT:    frintz z0.h, p0/m, z1.h
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svfrintz_nxv8f16_ptrue_u:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    ptrue p0.h
+; CHECK-2p2-NEXT:    frintz z0.h, p0/z, z1.h
+; CHECK-2p2-NEXT:    ret
+entry:
+  %pg = call  <vscale x 8 x i1> @llvm.aarch64.sve.ptrue.nxv8i1(i32 31)
+  %0 = tail call <vscale x 8 x half> @llvm.aarch64.sve.frintz.nxv8f16(<vscale x 8 x half> poison,  <vscale x 8 x i1> %pg, <vscale x 8 x half> %x)
+  ret <vscale x 8 x half> %0
+}
+
+define <vscale x 8 x half> @test_svfrintz_nxv8f16_ptrue(double %z0, <vscale x 8 x half> %x, <vscale x 8 x half> %y) {
+; CHECK-LABEL: test_svfrintz_nxv8f16_ptrue:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ptrue p0.h
+; CHECK-NEXT:    movprfx z0, z2
+; CHECK-NEXT:    frintz z0.h, p0/m, z2.h
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svfrintz_nxv8f16_ptrue:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    ptrue p0.h
+; CHECK-2p2-NEXT:    frintz z0.h, p0/z, z2.h
+; CHECK-2p2-NEXT:    ret
+entry:
+  %pg = call  <vscale x 8 x i1> @llvm.aarch64.sve.ptrue.nxv8i1(i32 31)
+  %0 = tail call <vscale x 8 x half> @llvm.aarch64.sve.frintz.nxv8f16(<vscale x 8 x half> %x,  <vscale x 8 x i1> %pg, <vscale x 8 x half> %y)
+  ret <vscale x 8 x half> %0
+}
+
+
+define <vscale x 4 x half> @test_svfrintz_nxv4f16_ptrue_u(double %z0, <vscale x 4 x half> %x) {
+; CHECK-LABEL: test_svfrintz_nxv4f16_ptrue_u:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ptrue p0.s
+; CHECK-NEXT:    movprfx z0, z1
+; CHECK-NEXT:    frintz z0.h, p0/m, z1.h
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svfrintz_nxv4f16_ptrue_u:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    ptrue p0.s
+; CHECK-2p2-NEXT:    frintz z0.h, p0/z, z1.h
+; CHECK-2p2-NEXT:    ret
+entry:
+  %pg = call  <vscale x 4 x i1> @llvm.aarch64.sve.ptrue.nxv4i1(i32 31)
+  %0 = tail call <vscale x 4 x half> @llvm.aarch64.sve.frintz.nxv4f16(<vscale x 4 x half> poison,  <vscale x 4 x i1> %pg, <vscale x 4 x half> %x)
+  ret <vscale x 4 x half> %0
+}
+
+define <vscale x 4 x half> @test_svfrintz_nxv4f16_ptrue(double %z0, <vscale x 4 x half> %x, <vscale x 4 x half> %y) {
+; CHECK-LABEL: test_svfrintz_nxv4f16_ptrue:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ptrue p0.s
+; CHECK-NEXT:    movprfx z0, z2
+; CHECK-NEXT:    frintz z0.h, p0/m, z2.h
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svfrintz_nxv4f16_ptrue:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    ptrue p0.s
+; CHECK-2p2-NEXT:    frintz z0.h, p0/z, z2.h
+; CHECK-2p2-NEXT:    ret
+entry:
+  %pg = call  <vscale x 4 x i1> @llvm.aarch64.sve.ptrue.nxv4i1(i32 31)
+  %0 = tail call <vscale x 4 x half> @llvm.aarch64.sve.frintz.nxv4f16(<vscale x 4 x half> %x,  <vscale x 4 x i1> %pg, <vscale x 4 x half> %y)
+  ret <vscale x 4 x half> %0
+}
+
+
+define <vscale x 2 x half> @test_svfrintz_nxv2f16_ptrue_u(double %z0, <vscale x 2 x half> %x) {
+; CHECK-LABEL: test_svfrintz_nxv2f16_ptrue_u:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ptrue p0.d
+; CHECK-NEXT:    movprfx z0, z1
+; CHECK-NEXT:    frintz z0.h, p0/m, z1.h
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svfrintz_nxv2f16_ptrue_u:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    ptrue p0.d
+; CHECK-2p2-NEXT:    frintz z0.h, p0/z, z1.h
+; CHECK-2p2-NEXT:    ret
+entry:
+  %pg = call  <vscale x 2 x i1> @llvm.aarch64.sve.ptrue.nxv2i1(i32 31)
+  %0 = tail call <vscale x 2 x half> @llvm.aarch64.sve.frintz.nxv2f16(<vscale x 2 x half> poison,  <vscale x 2 x i1> %pg, <vscale x 2 x half> %x)
+  ret <vscale x 2 x half> %0
+}
+
+define <vscale x 2 x half> @test_svfrintz_nxv2f16_ptrue(double %z0, <vscale x 2 x half> %x, <vscale x 2 x half> %y) {
+; CHECK-LABEL: test_svfrintz_nxv2f16_ptrue:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ptrue p0.d
+; CHECK-NEXT:    movprfx z0, z2
+; CHECK-NEXT:    frintz z0.h, p0/m, z2.h
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svfrintz_nxv2f16_ptrue:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    ptrue p0.d
+; CHECK-2p2-NEXT:    frintz z0.h, p0/z, z2.h
+; CHECK-2p2-NEXT:    ret
+entry:
+  %pg = call  <vscale x 2 x i1> @llvm.aarch64.sve.ptrue.nxv2i1(i32 31)
+  %0 = tail call <vscale x 2 x half> @llvm.aarch64.sve.frintz.nxv2f16(<vscale x 2 x half> %x,  <vscale x 2 x i1> %pg, <vscale x 2 x half> %y)
+  ret <vscale x 2 x half> %0
+}
+
+
+define <vscale x 2 x float> @test_svfrintz_nxv2f32_ptrue_u(double %z0, <vscale x 2 x float> %x) {
+; CHECK-LABEL: test_svfrintz_nxv2f32_ptrue_u:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ptrue p0.d
+; CHECK-NEXT:    movprfx z0, z1
+; CHECK-NEXT:    frintz z0.s, p0/m, z1.s
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svfrintz_nxv2f32_ptrue_u:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    ptrue p0.d
+; CHECK-2p2-NEXT:    frintz z0.s, p0/z, z1.s
+; CHECK-2p2-NEXT:    ret
+entry:
+  %pg = call  <vscale x 2 x i1> @llvm.aarch64.sve.ptrue.nxv2i1(i32 31)
+  %0 = tail call <vscale x 2 x float> @llvm.aarch64.sve.frintz.nxv2f32(<vscale x 2 x float> poison,  <vscale x 2 x i1> %pg, <vscale x 2 x float> %x)
+  ret <vscale x 2 x float> %0
+}
+
+define <vscale x 2 x float> @test_svfrintz_nxv2f32_ptrue(double %z0, <vscale x 2 x float> %x, <vscale x 2 x float> %y) {
+; CHECK-LABEL: test_svfrintz_nxv2f32_ptrue:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ptrue p0.d
+; CHECK-NEXT:    movprfx z0, z2
+; CHECK-NEXT:    frintz z0.s, p0/m, z2.s
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svfrintz_nxv2f32_ptrue:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    ptrue p0.d
+; CHECK-2p2-NEXT:    frintz z0.s, p0/z, z2.s
+; CHECK-2p2-NEXT:    ret
+entry:
+  %pg = call  <vscale x 2 x i1> @llvm.aarch64.sve.ptrue.nxv2i1(i32 31)
+  %0 = tail call <vscale x 2 x float> @llvm.aarch64.sve.frintz.nxv2f32(<vscale x 2 x float> %x,  <vscale x 2 x i1> %pg, <vscale x 2 x float> %y)
+  ret <vscale x 2 x float> %0
+}
+
+
+define <vscale x 4 x float> @test_svfrintz_nxv4f32_ptrue_u(double %z0, <vscale x 4 x float> %x) {
+; CHECK-LABEL: test_svfrintz_nxv4f32_ptrue_u:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ptrue p0.s
+; CHECK-NEXT:    movprfx z0, z1
+; CHECK-NEXT:    frintz z0.s, p0/m, z1.s
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svfrintz_nxv4f32_ptrue_u:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    ptrue p0.s
+; CHECK-2p2-NEXT:    frintz z0.s, p0/z, z1.s
+; CHECK-2p2-NEXT:    ret
+entry:
+  %pg = call  <vscale x 4 x i1> @llvm.aarch64.sve.ptrue.nxv4i1(i32 31)
+  %0 = tail call <vscale x 4 x float> @llvm.aarch64.sve.frintz.nxv4f32(<vscale x 4 x float> poison,  <vscale x 4 x i1> %pg, <vscale x 4 x float> %x)
+  ret <vscale x 4 x float> %0
+}
+
+define <vscale x 4 x float> @test_svfrintz_nxv4f32_ptrue(double %z0, <vscale x 4 x float> %x, <vscale x 4 x float> %y) {
+; CHECK-LABEL: test_svfrintz_nxv4f32_ptrue:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ptrue p0.s
+; CHECK-NEXT:    movprfx z0, z2
+; CHECK-NEXT:    frintz z0.s, p0/m, z2.s
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svfrintz_nxv4f32_ptrue:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    ptrue p0.s
+; CHECK-2p2-NEXT:    frintz z0.s, p0/z, z2.s
+; CHECK-2p2-NEXT:    ret
+entry:
+  %pg = call  <vscale x 4 x i1> @llvm.aarch64.sve.ptrue.nxv4i1(i32 31)
+  %0 = tail call <vscale x 4 x float> @llvm.aarch64.sve.frintz.nxv4f32(<vscale x 4 x float> %x,  <vscale x 4 x i1> %pg, <vscale x 4 x float> %y)
+  ret <vscale x 4 x float> %0
+}
+
+
+define <vscale x 2 x double> @test_svfrintz_nxv2f64_ptrue_u(double %z0, <vscale x 2 x double> %x) {
+; CHECK-LABEL: test_svfrintz_nxv2f64_ptrue_u:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ptrue p0.d
+; CHECK-NEXT:    movprfx z0, z1
+; CHECK-NEXT:    frintz z0.d, p0/m, z1.d
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svfrintz_nxv2f64_ptrue_u:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    ptrue p0.d
+; CHECK-2p2-NEXT:    frintz z0.d, p0/z, z1.d
+; CHECK-2p2-NEXT:    ret
+entry:
+  %pg = call  <vscale x 2 x i1> @llvm.aarch64.sve.ptrue.nxv2i1(i32 31)
+  %0 = tail call <vscale x 2 x double> @llvm.aarch64.sve.frintz.nxv2f64(<vscale x 2 x double> poison,  <vscale x 2 x i1> %pg, <vscale x 2 x double> %x)
+  ret <vscale x 2 x double> %0
+}
+
+define <vscale x 2 x double> @test_svfrintz_nxv2f64_ptrue(double %z0, <vscale x 2 x double> %x, <vscale x 2 x double> %y) {
+; CHECK-LABEL: test_svfrintz_nxv2f64_ptrue:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ptrue p0.d
+; CHECK-NEXT:    movprfx z0, z2
+; CHECK-NEXT:    frintz z0.d, p0/m, z2.d
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svfrintz_nxv2f64_ptrue:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    ptrue p0.d
+; CHECK-2p2-NEXT:    frintz z0.d, p0/z, z2.d
+; CHECK-2p2-NEXT:    ret
+entry:
+  %pg = call  <vscale x 2 x i1> @llvm.aarch64.sve.ptrue.nxv2i1(i32 31)
+  %0 = tail call <vscale x 2 x double> @llvm.aarch64.sve.frintz.nxv2f64(<vscale x 2 x double> %x,  <vscale x 2 x i1> %pg, <vscale x 2 x double> %y)
+  ret <vscale x 2 x double> %0
+}
+
+
+define <vscale x 8 x half> @test_svfrecpx_nxv8f16_ptrue_u(double %z0, <vscale x 8 x half> %x) {
+; CHECK-LABEL: test_svfrecpx_nxv8f16_ptrue_u:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ptrue p0.h
+; CHECK-NEXT:    movprfx z0, z1
+; CHECK-NEXT:    frecpx z0.h, p0/m, z1.h
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svfrecpx_nxv8f16_ptrue_u:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    ptrue p0.h
+; CHECK-2p2-NEXT:    frecpx z0.h, p0/z, z1.h
+; CHECK-2p2-NEXT:    ret
+entry:
+  %pg = call  <vscale x 8 x i1> @llvm.aarch64.sve.ptrue.nxv8i1(i32 31)
+  %0 = tail call <vscale x 8 x half> @llvm.aarch64.sve.frecpx.nxv8f16(<vscale x 8 x half> poison,  <vscale x 8 x i1> %pg, <vscale x 8 x half> %x)
+  ret <vscale x 8 x half> %0
+}
+
+define <vscale x 8 x half> @test_svfrecpx_nxv8f16_ptrue(double %z0, <vscale x 8 x half> %x, <vscale x 8 x half> %y) {
+; CHECK-LABEL: test_svfrecpx_nxv8f16_ptrue:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ptrue p0.h
+; CHECK-NEXT:    movprfx z0, z2
+; CHECK-NEXT:    frecpx z0.h, p0/m, z2.h
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svfrecpx_nxv8f16_ptrue:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    ptrue p0.h
+; CHECK-2p2-NEXT:    frecpx z0.h, p0/z, z2.h
+; CHECK-2p2-NEXT:    ret
+entry:
+  %pg = call  <vscale x 8 x i1> @llvm.aarch64.sve.ptrue.nxv8i1(i32 31)
+  %0 = tail call <vscale x 8 x half> @llvm.aarch64.sve.frecpx.nxv8f16(<vscale x 8 x half> %x,  <vscale x 8 x i1> %pg, <vscale x 8 x half> %y)
+  ret <vscale x 8 x half> %0
+}
+
+
+define <vscale x 4 x half> @test_svfrecpx_nxv4f16_ptrue_u(double %z0, <vscale x 4 x half> %x) {
+; CHECK-LABEL: test_svfrecpx_nxv4f16_ptrue_u:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ptrue p0.s
+; CHECK-NEXT:    movprfx z0, z1
+; CHECK-NEXT:    frecpx z0.h, p0/m, z1.h
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svfrecpx_nxv4f16_ptrue_u:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    ptrue p0.s
+; CHECK-2p2-NEXT:    frecpx z0.h, p0/z, z1.h
+; CHECK-2p2-NEXT:    ret
+entry:
+  %pg = call  <vscale x 4 x i1> @llvm.aarch64.sve.ptrue.nxv4i1(i32 31)
+  %0 = tail call <vscale x 4 x half> @llvm.aarch64.sve.frecpx.nxv4f16(<vscale x 4 x half> poison,  <vscale x 4 x i1> %pg, <vscale x 4 x half> %x)
+  ret <vscale x 4 x half> %0
+}
+
+define <vscale x 4 x half> @test_svfrecpx_nxv4f16_ptrue(double %z0, <vscale x 4 x half> %x, <vscale x 4 x half> %y) {
+; CHECK-LABEL: test_svfrecpx_nxv4f16_ptrue:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ptrue p0.s
+; CHECK-NEXT:    movprfx z0, z2
+; CHECK-NEXT:    frecpx z0.h, p0/m, z2.h
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svfrecpx_nxv4f16_ptrue:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    ptrue p0.s
+; CHECK-2p2-NEXT:    frecpx z0.h, p0/z, z2.h
+; CHECK-2p2-NEXT:    ret
+entry:
+  %pg = call  <vscale x 4 x i1> @llvm.aarch64.sve.ptrue.nxv4i1(i32 31)
+  %0 = tail call <vscale x 4 x half> @llvm.aarch64.sve.frecpx.nxv4f16(<vscale x 4 x half> %x,  <vscale x 4 x i1> %pg, <vscale x 4 x half> %y)
+  ret <vscale x 4 x half> %0
+}
+
+
+define <vscale x 2 x half> @test_svfrecpx_nxv2f16_ptrue_u(double %z0, <vscale x 2 x half> %x) {
+; CHECK-LABEL: test_svfrecpx_nxv2f16_ptrue_u:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ptrue p0.d
+; CHECK-NEXT:    movprfx z0, z1
+; CHECK-NEXT:    frecpx z0.h, p0/m, z1.h
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svfrecpx_nxv2f16_ptrue_u:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    ptrue p0.d
+; CHECK-2p2-NEXT:    frecpx z0.h, p0/z, z1.h
+; CHECK-2p2-NEXT:    ret
+entry:
+  %pg = call  <vscale x 2 x i1> @llvm.aarch64.sve.ptrue.nxv2i1(i32 31)
+  %0 = tail call <vscale x 2 x half> @llvm.aarch64.sve.frecpx.nxv2f16(<vscale x 2 x half> poison,  <vscale x 2 x i1> %pg, <vscale x 2 x half> %x)
+  ret <vscale x 2 x half> %0
+}
+
+define <vscale x 2 x half> @test_svfrecpx_nxv2f16_ptrue(double %z0, <vscale x 2 x half> %x, <vscale x 2 x half> %y) {
+; CHECK-LABEL: test_svfrecpx_nxv2f16_ptrue:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ptrue p0.d
+; CHECK-NEXT:    movprfx z0, z2
+; CHECK-NEXT:    frecpx z0.h, p0/m, z2.h
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svfrecpx_nxv2f16_ptrue:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    ptrue p0.d
+; CHECK-2p2-NEXT:    frecpx z0.h, p0/z, z2.h
+; CHECK-2p2-NEXT:    ret
+entry:
+  %pg = call  <vscale x 2 x i1> @llvm.aarch64.sve.ptrue.nxv2i1(i32 31)
+  %0 = tail call <vscale x 2 x half> @llvm.aarch64.sve.frecpx.nxv2f16(<vscale x 2 x half> %x,  <vscale x 2 x i1> %pg, <vscale x 2 x half> %y)
+  ret <vscale x 2 x half> %0
+}
+
+
+define <vscale x 2 x float> @test_svfrecpx_nxv2f32_ptrue_u(double %z0, <vscale x 2 x float> %x) {
+; CHECK-LABEL: test_svfrecpx_nxv2f32_ptrue_u:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ptrue p0.d
+; CHECK-NEXT:    movprfx z0, z1
+; CHECK-NEXT:    frecpx z0.s, p0/m, z1.s
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svfrecpx_nxv2f32_ptrue_u:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    ptrue p0.d
+; CHECK-2p2-NEXT:    frecpx z0.s, p0/z, z1.s
+; CHECK-2p2-NEXT:    ret
+entry:
+  %pg = call  <vscale x 2 x i1> @llvm.aarch64.sve.ptrue.nxv2i1(i32 31)
+  %0 = tail call <vscale x 2 x float> @llvm.aarch64.sve.frecpx.nxv2f32(<vscale x 2 x float> poison,  <vscale x 2 x i1> %pg, <vscale x 2 x float> %x)
+  ret <vscale x 2 x float> %0
+}
+
+define <vscale x 2 x float> @test_svfrecpx_nxv2f32_ptrue(double %z0, <vscale x 2 x float> %x, <vscale x 2 x float> %y) {
+; CHECK-LABEL: test_svfrecpx_nxv2f32_ptrue:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ptrue p0.d
+; CHECK-NEXT:    movprfx z0, z2
+; CHECK-NEXT:    frecpx z0.s, p0/m, z2.s
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svfrecpx_nxv2f32_ptrue:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    ptrue p0.d
+; CHECK-2p2-NEXT:    frecpx z0.s, p0/z, z2.s
+; CHECK-2p2-NEXT:    ret
+entry:
+  %pg = call  <vscale x 2 x i1> @llvm.aarch64.sve.ptrue.nxv2i1(i32 31)
+  %0 = tail call <vscale x 2 x float> @llvm.aarch64.sve.frecpx.nxv2f32(<vscale x 2 x float> %x,  <vscale x 2 x i1> %pg, <vscale x 2 x float> %y)
+  ret <vscale x 2 x float> %0
+}
+
+
+define <vscale x 4 x float> @test_svfrecpx_nxv4f32_ptrue_u(double %z0, <vscale x 4 x float> %x) {
+; CHECK-LABEL: test_svfrecpx_nxv4f32_ptrue_u:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ptrue p0.s
+; CHECK-NEXT:    movprfx z0, z1
+; CHECK-NEXT:    frecpx z0.s, p0/m, z1.s
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svfrecpx_nxv4f32_ptrue_u:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    ptrue p0.s
+; CHECK-2p2-NEXT:    frecpx z0.s, p0/z, z1.s
+; CHECK-2p2-NEXT:    ret
+entry:
+  %pg = call  <vscale x 4 x i1> @llvm.aarch64.sve.ptrue.nxv4i1(i32 31)
+  %0 = tail call <vscale x 4 x float> @llvm.aarch64.sve.frecpx.nxv4f32(<vscale x 4 x float> poison,  <vscale x 4 x i1> %pg, <vscale x 4 x float> %x)
+  ret <vscale x 4 x float> %0
+}
+
+define <vscale x 4 x float> @test_svfrecpx_nxv4f32_ptrue(double %z0, <vscale x 4 x float> %x, <vscale x 4 x float> %y) {
+; CHECK-LABEL: test_svfrecpx_nxv4f32_ptrue:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ptrue p0.s
+; CHECK-NEXT:    movprfx z0, z2
+; CHECK-NEXT:    frecpx z0.s, p0/m, z2.s
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svfrecpx_nxv4f32_ptrue:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    ptrue p0.s
+; CHECK-2p2-NEXT:    frecpx z0.s, p0/z, z2.s
+; CHECK-2p2-NEXT:    ret
+entry:
+  %pg = call  <vscale x 4 x i1> @llvm.aarch64.sve.ptrue.nxv4i1(i32 31)
+  %0 = tail call <vscale x 4 x float> @llvm.aarch64.sve.frecpx.nxv4f32(<vscale x 4 x float> %x,  <vscale x 4 x i1> %pg, <vscale x 4 x float> %y)
+  ret <vscale x 4 x float> %0
+}
+
+
+define <vscale x 2 x double> @test_svfrecpx_nxv2f64_ptrue_u(double %z0, <vscale x 2 x double> %x) {
+; CHECK-LABEL: test_svfrecpx_nxv2f64_ptrue_u:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ptrue p0.d
+; CHECK-NEXT:    movprfx z0, z1
+; CHECK-NEXT:    frecpx z0.d, p0/m, z1.d
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svfrecpx_nxv2f64_ptrue_u:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    ptrue p0.d
+; CHECK-2p2-NEXT:    frecpx z0.d, p0/z, z1.d
+; CHECK-2p2-NEXT:    ret
+entry:
+  %pg = call  <vscale x 2 x i1> @llvm.aarch64.sve.ptrue.nxv2i1(i32 31)
+  %0 = tail call <vscale x 2 x double> @llvm.aarch64.sve.frecpx.nxv2f64(<vscale x 2 x double> poison,  <vscale x 2 x i1> %pg, <vscale x 2 x double> %x)
+  ret <vscale x 2 x double> %0
+}
+
+define <vscale x 2 x double> @test_svfrecpx_nxv2f64_ptrue(double %z0, <vscale x 2 x double> %x, <vscale x 2 x double> %y) {
+; CHECK-LABEL: test_svfrecpx_nxv2f64_ptrue:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ptrue p0.d
+; CHECK-NEXT:    movprfx z0, z2
+; CHECK-NEXT:    frecpx z0.d, p0/m, z2.d
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svfrecpx_nxv2f64_ptrue:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    ptrue p0.d
+; CHECK-2p2-NEXT:    frecpx z0.d, p0/z, z2.d
+; CHECK-2p2-NEXT:    ret
+entry:
+  %pg = call  <vscale x 2 x i1> @llvm.aarch64.sve.ptrue.nxv2i1(i32 31)
+  %0 = tail call <vscale x 2 x double> @llvm.aarch64.sve.frecpx.nxv2f64(<vscale x 2 x double> %x,  <vscale x 2 x i1> %pg, <vscale x 2 x double> %y)
+  ret <vscale x 2 x double> %0
+}
+
+
+define <vscale x 8 x half> @test_svfsqrt_nxv8f16_ptrue_u(double %z0, <vscale x 8 x half> %x) {
+; CHECK-LABEL: test_svfsqrt_nxv8f16_ptrue_u:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ptrue p0.h
+; CHECK-NEXT:    movprfx z0, z1
+; CHECK-NEXT:    fsqrt z0.h, p0/m, z1.h
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svfsqrt_nxv8f16_ptrue_u:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    ptrue p0.h
+; CHECK-2p2-NEXT:    fsqrt z0.h, p0/z, z1.h
+; CHECK-2p2-NEXT:    ret
+entry:
+  %pg = call  <vscale x 8 x i1> @llvm.aarch64.sve.ptrue.nxv8i1(i32 31)
+  %0 = tail call <vscale x 8 x half> @llvm.aarch64.sve.fsqrt.nxv8f16(<vscale x 8 x half> poison,  <vscale x 8 x i1> %pg, <vscale x 8 x half> %x)
+  ret <vscale x 8 x half> %0
+}
+
+define <vscale x 8 x half> @test_svfsqrt_nxv8f16_ptrue(double %z0, <vscale x 8 x half> %x, <vscale x 8 x half> %y) {
+; CHECK-LABEL: test_svfsqrt_nxv8f16_ptrue:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ptrue p0.h
+; CHECK-NEXT:    movprfx z0, z2
+; CHECK-NEXT:    fsqrt z0.h, p0/m, z2.h
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svfsqrt_nxv8f16_ptrue:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    ptrue p0.h
+; CHECK-2p2-NEXT:    fsqrt z0.h, p0/z, z2.h
+; CHECK-2p2-NEXT:    ret
+entry:
+  %pg = call  <vscale x 8 x i1> @llvm.aarch64.sve.ptrue.nxv8i1(i32 31)
+  %0 = tail call <vscale x 8 x half> @llvm.aarch64.sve.fsqrt.nxv8f16(<vscale x 8 x half> %x,  <vscale x 8 x i1> %pg, <vscale x 8 x half> %y)
+  ret <vscale x 8 x half> %0
+}
+
+
+define <vscale x 4 x half> @test_svfsqrt_nxv4f16_ptrue_u(double %z0, <vscale x 4 x half> %x) {
+; CHECK-LABEL: test_svfsqrt_nxv4f16_ptrue_u:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ptrue p0.s
+; CHECK-NEXT:    movprfx z0, z1
+; CHECK-NEXT:    fsqrt z0.h, p0/m, z1.h
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svfsqrt_nxv4f16_ptrue_u:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    ptrue p0.s
+; CHECK-2p2-NEXT:    fsqrt z0.h, p0/z, z1.h
+; CHECK-2p2-NEXT:    ret
+entry:
+  %pg = call  <vscale x 4 x i1> @llvm.aarch64.sve.ptrue.nxv4i1(i32 31)
+  %0 = tail call <vscale x 4 x half> @llvm.aarch64.sve.fsqrt.nxv4f16(<vscale x 4 x half> poison,  <vscale x 4 x i1> %pg, <vscale x 4 x half> %x)
+  ret <vscale x 4 x half> %0
+}
+
+define <vscale x 4 x half> @test_svfsqrt_nxv4f16_ptrue(double %z0, <vscale x 4 x half> %x, <vscale x 4 x half> %y) {
+; CHECK-LABEL: test_svfsqrt_nxv4f16_ptrue:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ptrue p0.s
+; CHECK-NEXT:    movprfx z0, z2
+; CHECK-NEXT:    fsqrt z0.h, p0/m, z2.h
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svfsqrt_nxv4f16_ptrue:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    ptrue p0.s
+; CHECK-2p2-NEXT:    fsqrt z0.h, p0/z, z2.h
+; CHECK-2p2-NEXT:    ret
+entry:
+  %pg = call  <vscale x 4 x i1> @llvm.aarch64.sve.ptrue.nxv4i1(i32 31)
+  %0 = tail call <vscale x 4 x half> @llvm.aarch64.sve.fsqrt.nxv4f16(<vscale x 4 x half> %x,  <vscale x 4 x i1> %pg, <vscale x 4 x half> %y)
+  ret <vscale x 4 x half> %0
+}
+
+
+define <vscale x 2 x half> @test_svfsqrt_nxv2f16_ptrue_u(double %z0, <vscale x 2 x half> %x) {
+; CHECK-LABEL: test_svfsqrt_nxv2f16_ptrue_u:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ptrue p0.d
+; CHECK-NEXT:    movprfx z0, z1
+; CHECK-NEXT:    fsqrt z0.h, p0/m, z1.h
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svfsqrt_nxv2f16_ptrue_u:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    ptrue p0.d
+; CHECK-2p2-NEXT:    fsqrt z0.h, p0/z, z1.h
+; CHECK-2p2-NEXT:    ret
+entry:
+  %pg = call  <vscale x 2 x i1> @llvm.aarch64.sve.ptrue.nxv2i1(i32 31)
+  %0 = tail call <vscale x 2 x half> @llvm.aarch64.sve.fsqrt.nxv2f16(<vscale x 2 x half> poison,  <vscale x 2 x i1> %pg, <vscale x 2 x half> %x)
+  ret <vscale x 2 x half> %0
+}
+
+define <vscale x 2 x half> @test_svfsqrt_nxv2f16_ptrue(double %z0, <vscale x 2 x half> %x, <vscale x 2 x half> %y) {
+; CHECK-LABEL: test_svfsqrt_nxv2f16_ptrue:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ptrue p0.d
+; CHECK-NEXT:    movprfx z0, z2
+; CHECK-NEXT:    fsqrt z0.h, p0/m, z2.h
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svfsqrt_nxv2f16_ptrue:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    ptrue p0.d
+; CHECK-2p2-NEXT:    fsqrt z0.h, p0/z, z2.h
+; CHECK-2p2-NEXT:    ret
+entry:
+  %pg = call  <vscale x 2 x i1> @llvm.aarch64.sve.ptrue.nxv2i1(i32 31)
+  %0 = tail call <vscale x 2 x half> @llvm.aarch64.sve.fsqrt.nxv2f16(<vscale x 2 x half> %x,  <vscale x 2 x i1> %pg, <vscale x 2 x half> %y)
+  ret <vscale x 2 x half> %0
+}
+
+
+define <vscale x 2 x float> @test_svfsqrt_nxv2f32_ptrue_u(double %z0, <vscale x 2 x float> %x) {
+; CHECK-LABEL: test_svfsqrt_nxv2f32_ptrue_u:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ptrue p0.d
+; CHECK-NEXT:    movprfx z0, z1
+; CHECK-NEXT:    fsqrt z0.s, p0/m, z1.s
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svfsqrt_nxv2f32_ptrue_u:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    ptrue p0.d
+; CHECK-2p2-NEXT:    fsqrt z0.s, p0/z, z1.s
+; CHECK-2p2-NEXT:    ret
+entry:
+  %pg = call  <vscale x 2 x i1> @llvm.aarch64.sve.ptrue.nxv2i1(i32 31)
+  %0 = tail call <vscale x 2 x float> @llvm.aarch64.sve.fsqrt.nxv2f32(<vscale x 2 x float> poison,  <vscale x 2 x i1> %pg, <vscale x 2 x float> %x)
+  ret <vscale x 2 x float> %0
+}
+
+define <vscale x 2 x float> @test_svfsqrt_nxv2f32_ptrue(double %z0, <vscale x 2 x float> %x, <vscale x 2 x float> %y) {
+; CHECK-LABEL: test_svfsqrt_nxv2f32_ptrue:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ptrue p0.d
+; CHECK-NEXT:    movprfx z0, z2
+; CHECK-NEXT:    fsqrt z0.s, p0/m, z2.s
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svfsqrt_nxv2f32_ptrue:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    ptrue p0.d
+; CHECK-2p2-NEXT:    fsqrt z0.s, p0/z, z2.s
+; CHECK-2p2-NEXT:    ret
+entry:
+  %pg = call  <vscale x 2 x i1> @llvm.aarch64.sve.ptrue.nxv2i1(i32 31)
+  %0 = tail call <vscale x 2 x float> @llvm.aarch64.sve.fsqrt.nxv2f32(<vscale x 2 x float> %x,  <vscale x 2 x i1> %pg, <vscale x 2 x float> %y)
+  ret <vscale x 2 x float> %0
+}
+
+
+define <vscale x 4 x float> @test_svfsqrt_nxv4f32_ptrue_u(double %z0, <vscale x 4 x float> %x) {
+; CHECK-LABEL: test_svfsqrt_nxv4f32_ptrue_u:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ptrue p0.s
+; CHECK-NEXT:    movprfx z0, z1
+; CHECK-NEXT:    fsqrt z0.s, p0/m, z1.s
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svfsqrt_nxv4f32_ptrue_u:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    ptrue p0.s
+; CHECK-2p2-NEXT:    fsqrt z0.s, p0/z, z1.s
+; CHECK-2p2-NEXT:    ret
+entry:
+  %pg = call  <vscale x 4 x i1> @llvm.aarch64.sve.ptrue.nxv4i1(i32 31)
+  %0 = tail call <vscale x 4 x float> @llvm.aarch64.sve.fsqrt.nxv4f32(<vscale x 4 x float> poison,  <vscale x 4 x i1> %pg, <vscale x 4 x float> %x)
+  ret <vscale x 4 x float> %0
+}
+
+define <vscale x 4 x float> @test_svfsqrt_nxv4f32_ptrue(double %z0, <vscale x 4 x float> %x, <vscale x 4 x float> %y) {
+; CHECK-LABEL: test_svfsqrt_nxv4f32_ptrue:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ptrue p0.s
+; CHECK-NEXT:    movprfx z0, z2
+; CHECK-NEXT:    fsqrt z0.s, p0/m, z2.s
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svfsqrt_nxv4f32_ptrue:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    ptrue p0.s
+; CHECK-2p2-NEXT:    fsqrt z0.s, p0/z, z2.s
+; CHECK-2p2-NEXT:    ret
+entry:
+  %pg = call  <vscale x 4 x i1> @llvm.aarch64.sve.ptrue.nxv4i1(i32 31)
+  %0 = tail call <vscale x 4 x float> @llvm.aarch64.sve.fsqrt.nxv4f32(<vscale x 4 x float> %x,  <vscale x 4 x i1> %pg, <vscale x 4 x float> %y)
+  ret <vscale x 4 x float> %0
+}
+
+
+define <vscale x 2 x double> @test_svfsqrt_nxv2f64_ptrue_u(double %z0, <vscale x 2 x double> %x) {
+; CHECK-LABEL: test_svfsqrt_nxv2f64_ptrue_u:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ptrue p0.d
+; CHECK-NEXT:    movprfx z0, z1
+; CHECK-NEXT:    fsqrt z0.d, p0/m, z1.d
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svfsqrt_nxv2f64_ptrue_u:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    ptrue p0.d
+; CHECK-2p2-NEXT:    fsqrt z0.d, p0/z, z1.d
+; CHECK-2p2-NEXT:    ret
+entry:
+  %pg = call  <vscale x 2 x i1> @llvm.aarch64.sve.ptrue.nxv2i1(i32 31)
+  %0 = tail call <vscale x 2 x double> @llvm.aarch64.sve.fsqrt.nxv2f64(<vscale x 2 x double> poison,  <vscale x 2 x i1> %pg, <vscale x 2 x double> %x)
+  ret <vscale x 2 x double> %0
+}
+
+define <vscale x 2 x double> @test_svfsqrt_nxv2f64_ptrue(double %z0, <vscale x 2 x double> %x, <vscale x 2 x double> %y) {
+; CHECK-LABEL: test_svfsqrt_nxv2f64_ptrue:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ptrue p0.d
+; CHECK-NEXT:    movprfx z0, z2
+; CHECK-NEXT:    fsqrt z0.d, p0/m, z2.d
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svfsqrt_nxv2f64_ptrue:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    ptrue p0.d
+; CHECK-2p2-NEXT:    fsqrt z0.d, p0/z, z2.d
+; CHECK-2p2-NEXT:    ret
+entry:
+  %pg = call  <vscale x 2 x i1> @llvm.aarch64.sve.ptrue.nxv2i1(i32 31)
+  %0 = tail call <vscale x 2 x double> @llvm.aarch64.sve.fsqrt.nxv2f64(<vscale x 2 x double> %x,  <vscale x 2 x i1> %pg, <vscale x 2 x double> %y)
+  ret <vscale x 2 x double> %0
+}
+
diff --git a/llvm/test/CodeGen/AArch64/zeroing-forms-rev.ll b/llvm/test/CodeGen/AArch64/zeroing-forms-rev.ll
new file mode 100644
index 0000000000000..d7a51c8cf8062
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/zeroing-forms-rev.ll
@@ -0,0 +1,1502 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc -mattr=+sve2p1 < %s | FileCheck %s
+; RUN: llc -mattr=+sve2p2 < %s | FileCheck %s -check-prefix CHECK-2p2
+
+; RUN: llc -mattr=+sme    -force-streaming < %s | FileCheck %s
+; RUN: llc -mattr=+sme2p2 -force-streaming < %s | FileCheck %s -check-prefix CHECK-2p2
+
+target triple = "aarch64-linux"
+
+define <vscale x 16 x i8> @test_svrbit_s8_x_1(<vscale x 16 x i1> %pg, <vscale x 16 x i8> %x) {
+; CHECK-LABEL: test_svrbit_s8_x_1:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    rbit z0.b, p0/m, z0.b
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svrbit_s8_x_1:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    rbit z0.b, p0/z, z0.b
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 16 x i8> @llvm.aarch64.sve.rbit.nxv16i8(<vscale x 16 x i8> poison, <vscale x 16 x i1> %pg, <vscale x 16 x i8> %x)
+  ret <vscale x 16 x i8> %0
+}
+
+define <vscale x 16 x i8> @test_svrbit_s8_x_2(<vscale x 16 x i1> %pg, double %z0, <vscale x 16 x i8> %x) {
+; CHECK-LABEL: test_svrbit_s8_x_2:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    rbit z0.b, p0/m, z1.b
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svrbit_s8_x_2:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    rbit z0.b, p0/z, z1.b
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 16 x i8> @llvm.aarch64.sve.rbit.nxv16i8(<vscale x 16 x i8> poison, <vscale x 16 x i1> %pg, <vscale x 16 x i8> %x)
+  ret <vscale x 16 x i8> %0
+}
+
+define <vscale x 16 x i8> @test_svrbit_s8_z(<vscale x 16 x i1> %pg, double %z0, <vscale x 16 x i8> %x) {
+; CHECK-LABEL: test_svrbit_s8_z:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    mov z0.b, #0 // =0x0
+; CHECK-NEXT:    rbit z0.b, p0/m, z1.b
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svrbit_s8_z:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    rbit z0.b, p0/z, z1.b
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 16 x i8> @llvm.aarch64.sve.rbit.nxv16i8(<vscale x 16 x i8> zeroinitializer, <vscale x 16 x i1> %pg, <vscale x 16 x i8> %x)
+  ret <vscale x 16 x i8> %0
+}
+
+define <vscale x 8 x i16> @test_svrbit_s16_x_1(<vscale x 8 x i1> %pg, <vscale x 8 x i16> %x) {
+; CHECK-LABEL: test_svrbit_s16_x_1:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    rbit z0.h, p0/m, z0.h
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svrbit_s16_x_1:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    rbit z0.h, p0/z, z0.h
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 8 x i16> @llvm.aarch64.sve.rbit.nxv8i16(<vscale x 8 x i16> poison, <vscale x 8 x i1> %pg, <vscale x 8 x i16> %x)
+  ret <vscale x 8 x i16> %0
+}
+
+define <vscale x 8 x i16> @test_svrbit_s16_x_2(<vscale x 8 x i1> %pg, double %z0, <vscale x 8 x i16> %x) {
+; CHECK-LABEL: test_svrbit_s16_x_2:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    rbit z0.h, p0/m, z1.h
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svrbit_s16_x_2:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    rbit z0.h, p0/z, z1.h
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 8 x i16> @llvm.aarch64.sve.rbit.nxv8i16(<vscale x 8 x i16> poison, <vscale x 8 x i1> %pg, <vscale x 8 x i16> %x)
+  ret <vscale x 8 x i16> %0
+}
+
+define <vscale x 8 x i16> @test_svrbit_s16_z(<vscale x 8 x i1> %pg, double %z0, <vscale x 8 x i16> %x) {
+; CHECK-LABEL: test_svrbit_s16_z:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    mov z0.h, #0 // =0x0
+; CHECK-NEXT:    rbit z0.h, p0/m, z1.h
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svrbit_s16_z:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    rbit z0.h, p0/z, z1.h
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 8 x i16> @llvm.aarch64.sve.rbit.nxv8i16(<vscale x 8 x i16> zeroinitializer, <vscale x 8 x i1> %pg, <vscale x 8 x i16> %x)
+  ret <vscale x 8 x i16> %0
+}
+
+define <vscale x 4 x i32> @test_svrbit_s32_x_1(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %x) {
+; CHECK-LABEL: test_svrbit_s32_x_1:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    rbit z0.s, p0/m, z0.s
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svrbit_s32_x_1:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    rbit z0.s, p0/z, z0.s
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 4 x i32> @llvm.aarch64.sve.rbit.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i1> %pg, <vscale x 4 x i32> %x)
+  ret <vscale x 4 x i32> %0
+}
+
+define <vscale x 4 x i32> @test_svrbit_s32_x_2(<vscale x 4 x i1> %pg, double %z0, <vscale x 4 x i32> %x) {
+; CHECK-LABEL: test_svrbit_s32_x_2:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    rbit z0.s, p0/m, z1.s
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svrbit_s32_x_2:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    rbit z0.s, p0/z, z1.s
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 4 x i32> @llvm.aarch64.sve.rbit.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i1> %pg, <vscale x 4 x i32> %x)
+  ret <vscale x 4 x i32> %0
+}
+
+define <vscale x 4 x i32> @test_svrbit_s32_z(<vscale x 4 x i1> %pg, double %z0, <vscale x 4 x i32> %x) {
+; CHECK-LABEL: test_svrbit_s32_z:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    mov z0.s, #0 // =0x0
+; CHECK-NEXT:    rbit z0.s, p0/m, z1.s
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svrbit_s32_z:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    rbit z0.s, p0/z, z1.s
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 4 x i32> @llvm.aarch64.sve.rbit.nxv4i32(<vscale x 4 x i32> zeroinitializer, <vscale x 4 x i1> %pg, <vscale x 4 x i32> %x)
+  ret <vscale x 4 x i32> %0
+}
+
+define <vscale x 2 x i64> @test_svrbit_s64_x_1(<vscale x 2 x i1> %pg, <vscale x 2 x i64> %x) {
+; CHECK-LABEL: test_svrbit_s64_x_1:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    rbit z0.d, p0/m, z0.d
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svrbit_s64_x_1:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    rbit z0.d, p0/z, z0.d
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 2 x i64> @llvm.aarch64.sve.rbit.nxv2i64(<vscale x 2 x i64> poison, <vscale x 2 x i1> %pg, <vscale x 2 x i64> %x)
+  ret <vscale x 2 x i64> %0
+}
+
+define <vscale x 2 x i64> @test_svrbit_s64_x_2(<vscale x 2 x i1> %pg, double %z0, <vscale x 2 x i64> %x) {
+; CHECK-LABEL: test_svrbit_s64_x_2:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    rbit z0.d, p0/m, z1.d
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svrbit_s64_x_2:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    rbit z0.d, p0/z, z1.d
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 2 x i64> @llvm.aarch64.sve.rbit.nxv2i64(<vscale x 2 x i64> poison, <vscale x 2 x i1> %pg, <vscale x 2 x i64> %x)
+  ret <vscale x 2 x i64> %0
+}
+
+define <vscale x 2 x i64> @test_svrbit_s64_z(<vscale x 2 x i1> %pg, double %z0, <vscale x 2 x i64> %x) {
+; CHECK-LABEL: test_svrbit_s64_z:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    mov z0.d, #0 // =0x0
+; CHECK-NEXT:    rbit z0.d, p0/m, z1.d
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svrbit_s64_z:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    rbit z0.d, p0/z, z1.d
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 2 x i64> @llvm.aarch64.sve.rbit.nxv2i64(<vscale x 2 x i64> zeroinitializer, <vscale x 2 x i1> %pg, <vscale x 2 x i64> %x)
+  ret <vscale x 2 x i64> %0
+}
+
+define <vscale x 8 x i16> @test_svrevb_s16_x_1(<vscale x 8 x i1> %pg, <vscale x 8 x i16> %x) {
+; CHECK-LABEL: test_svrevb_s16_x_1:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    revb z0.h, p0/m, z0.h
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svrevb_s16_x_1:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    revb z0.h, p0/z, z0.h
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 8 x i16> @llvm.aarch64.sve.revb.nxv8i16(<vscale x 8 x i16> poison, <vscale x 8 x i1> %pg, <vscale x 8 x i16> %x)
+  ret <vscale x 8 x i16> %0
+}
+
+define <vscale x 8 x i16> @test_svrevb_s16_x_2(<vscale x 8 x i1> %pg, double %z0, <vscale x 8 x i16> %x) {
+; CHECK-LABEL: test_svrevb_s16_x_2:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    revb z0.h, p0/m, z1.h
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svrevb_s16_x_2:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    revb z0.h, p0/z, z1.h
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 8 x i16> @llvm.aarch64.sve.revb.nxv8i16(<vscale x 8 x i16> poison, <vscale x 8 x i1> %pg, <vscale x 8 x i16> %x)
+  ret <vscale x 8 x i16> %0
+}
+
+define <vscale x 8 x i16> @test_svrevb_s16_z(<vscale x 8 x i1> %pg, double %z0, <vscale x 8 x i16> %x) {
+; CHECK-LABEL: test_svrevb_s16_z:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    mov z0.h, #0 // =0x0
+; CHECK-NEXT:    revb z0.h, p0/m, z1.h
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svrevb_s16_z:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    revb z0.h, p0/z, z1.h
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 8 x i16> @llvm.aarch64.sve.revb.nxv8i16(<vscale x 8 x i16> zeroinitializer, <vscale x 8 x i1> %pg, <vscale x 8 x i16> %x)
+  ret <vscale x 8 x i16> %0
+}
+
+define <vscale x 4 x i32> @test_svrevb_s32_x_1(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %x) {
+; CHECK-LABEL: test_svrevb_s32_x_1:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    revb z0.s, p0/m, z0.s
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svrevb_s32_x_1:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    revb z0.s, p0/z, z0.s
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 4 x i32> @llvm.aarch64.sve.revb.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i1> %pg, <vscale x 4 x i32> %x)
+  ret <vscale x 4 x i32> %0
+}
+
+define <vscale x 4 x i32> @test_svrevb_s32_x_2(<vscale x 4 x i1> %pg, double %z0, <vscale x 4 x i32> %x) {
+; CHECK-LABEL: test_svrevb_s32_x_2:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    revb z0.s, p0/m, z1.s
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svrevb_s32_x_2:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    revb z0.s, p0/z, z1.s
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 4 x i32> @llvm.aarch64.sve.revb.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i1> %pg, <vscale x 4 x i32> %x)
+  ret <vscale x 4 x i32> %0
+}
+
+define <vscale x 4 x i32> @test_svrevb_s32_z(<vscale x 4 x i1> %pg, double %z0, <vscale x 4 x i32> %x) {
+; CHECK-LABEL: test_svrevb_s32_z:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    mov z0.s, #0 // =0x0
+; CHECK-NEXT:    revb z0.s, p0/m, z1.s
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svrevb_s32_z:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    revb z0.s, p0/z, z1.s
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 4 x i32> @llvm.aarch64.sve.revb.nxv4i32(<vscale x 4 x i32> zeroinitializer, <vscale x 4 x i1> %pg, <vscale x 4 x i32> %x)
+  ret <vscale x 4 x i32> %0
+}
+
+define <vscale x 2 x i64> @test_svrevb_s64_x_1(<vscale x 2 x i1> %pg, <vscale x 2 x i64> %x) {
+; CHECK-LABEL: test_svrevb_s64_x_1:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    revb z0.d, p0/m, z0.d
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svrevb_s64_x_1:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    revb z0.d, p0/z, z0.d
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 2 x i64> @llvm.aarch64.sve.revb.nxv2i64(<vscale x 2 x i64> poison, <vscale x 2 x i1> %pg, <vscale x 2 x i64> %x)
+  ret <vscale x 2 x i64> %0
+}
+
+define <vscale x 2 x i64> @test_svrevb_s64_x_2(<vscale x 2 x i1> %pg, double %z0, <vscale x 2 x i64> %x) {
+; CHECK-LABEL: test_svrevb_s64_x_2:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    revb z0.d, p0/m, z1.d
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svrevb_s64_x_2:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    revb z0.d, p0/z, z1.d
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 2 x i64> @llvm.aarch64.sve.revb.nxv2i64(<vscale x 2 x i64> poison, <vscale x 2 x i1> %pg, <vscale x 2 x i64> %x)
+  ret <vscale x 2 x i64> %0
+}
+
+define <vscale x 2 x i64> @test_svrevb_s64_z(<vscale x 2 x i1> %pg, double %z0, <vscale x 2 x i64> %x) {
+; CHECK-LABEL: test_svrevb_s64_z:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    mov z0.d, #0 // =0x0
+; CHECK-NEXT:    revb z0.d, p0/m, z1.d
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svrevb_s64_z:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    revb z0.d, p0/z, z1.d
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 2 x i64> @llvm.aarch64.sve.revb.nxv2i64(<vscale x 2 x i64> zeroinitializer, <vscale x 2 x i1> %pg, <vscale x 2 x i64> %x)
+  ret <vscale x 2 x i64> %0
+}
+
+define <vscale x 4 x i32> @test_svrevh_s32_x_1(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %x) {
+; CHECK-LABEL: test_svrevh_s32_x_1:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    revh z0.s, p0/m, z0.s
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svrevh_s32_x_1:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    revh z0.s, p0/z, z0.s
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 4 x i32> @llvm.aarch64.sve.revh.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i1> %pg, <vscale x 4 x i32> %x)
+  ret <vscale x 4 x i32> %0
+}
+
+define <vscale x 4 x i32> @test_svrevh_s32_x_2(<vscale x 4 x i1> %pg, double %z0, <vscale x 4 x i32> %x) {
+; CHECK-LABEL: test_svrevh_s32_x_2:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    revh z0.s, p0/m, z1.s
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svrevh_s32_x_2:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    revh z0.s, p0/z, z1.s
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 4 x i32> @llvm.aarch64.sve.revh.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i1> %pg, <vscale x 4 x i32> %x)
+  ret <vscale x 4 x i32> %0
+}
+
+define <vscale x 4 x i32> @test_svrevh_s32_z(<vscale x 4 x i1> %pg, double %z0, <vscale x 4 x i32> %x) {
+; CHECK-LABEL: test_svrevh_s32_z:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    mov z0.s, #0 // =0x0
+; CHECK-NEXT:    revh z0.s, p0/m, z1.s
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svrevh_s32_z:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    revh z0.s, p0/z, z1.s
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 4 x i32> @llvm.aarch64.sve.revh.nxv4i32(<vscale x 4 x i32> zeroinitializer, <vscale x 4 x i1> %pg, <vscale x 4 x i32> %x)
+  ret <vscale x 4 x i32> %0
+}
+
+define <vscale x 2 x i64> @test_svrevh_s64_x_1(<vscale x 2 x i1> %pg, <vscale x 2 x i64> %x) {
+; CHECK-LABEL: test_svrevh_s64_x_1:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    revh z0.d, p0/m, z0.d
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svrevh_s64_x_1:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    revh z0.d, p0/z, z0.d
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 2 x i64> @llvm.aarch64.sve.revh.nxv2i64(<vscale x 2 x i64> poison, <vscale x 2 x i1> %pg, <vscale x 2 x i64> %x)
+  ret <vscale x 2 x i64> %0
+}
+
+define <vscale x 2 x i64> @test_svrevh_s64_x_2(<vscale x 2 x i1> %pg, double %z0, <vscale x 2 x i64> %x) {
+; CHECK-LABEL: test_svrevh_s64_x_2:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    revh z0.d, p0/m, z1.d
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svrevh_s64_x_2:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    revh z0.d, p0/z, z1.d
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 2 x i64> @llvm.aarch64.sve.revh.nxv2i64(<vscale x 2 x i64> poison, <vscale x 2 x i1> %pg, <vscale x 2 x i64> %x)
+  ret <vscale x 2 x i64> %0
+}
+
+define <vscale x 2 x i64> @test_svrevh_s64_z(<vscale x 2 x i1> %pg, double %z0, <vscale x 2 x i64> %x) {
+; CHECK-LABEL: test_svrevh_s64_z:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    mov z0.d, #0 // =0x0
+; CHECK-NEXT:    revh z0.d, p0/m, z1.d
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svrevh_s64_z:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    revh z0.d, p0/z, z1.d
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 2 x i64> @llvm.aarch64.sve.revh.nxv2i64(<vscale x 2 x i64> zeroinitializer, <vscale x 2 x i1> %pg, <vscale x 2 x i64> %x)
+  ret <vscale x 2 x i64> %0
+}
+
+define <vscale x 2 x i64> @test_svrevw_s64_x_1(<vscale x 2 x i1> %pg, <vscale x 2 x i64> %x) {
+; CHECK-LABEL: test_svrevw_s64_x_1:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    revw z0.d, p0/m, z0.d
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svrevw_s64_x_1:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    revw z0.d, p0/z, z0.d
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 2 x i64> @llvm.aarch64.sve.revw.nxv2i64(<vscale x 2 x i64> poison, <vscale x 2 x i1> %pg, <vscale x 2 x i64> %x)
+  ret <vscale x 2 x i64> %0
+}
+
+define <vscale x 2 x i64> @test_svrevw_s64_x_2(<vscale x 2 x i1> %pg, double %z0, <vscale x 2 x i64> %x) {
+; CHECK-LABEL: test_svrevw_s64_x_2:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    revw z0.d, p0/m, z1.d
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svrevw_s64_x_2:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    revw z0.d, p0/z, z1.d
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 2 x i64> @llvm.aarch64.sve.revw.nxv2i64(<vscale x 2 x i64> poison, <vscale x 2 x i1> %pg, <vscale x 2 x i64> %x)
+  ret <vscale x 2 x i64> %0
+}
+
+define <vscale x 2 x i64> @test_svrevw_s64_z(<vscale x 2 x i1> %pg, double %z0, <vscale x 2 x i64> %x) {
+; CHECK-LABEL: test_svrevw_s64_z:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    mov z0.d, #0 // =0x0
+; CHECK-NEXT:    revw z0.d, p0/m, z1.d
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svrevw_s64_z:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    revw z0.d, p0/z, z1.d
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 2 x i64> @llvm.aarch64.sve.revw.nxv2i64(<vscale x 2 x i64> zeroinitializer, <vscale x 2 x i1> %pg, <vscale x 2 x i64> %x)
+  ret <vscale x 2 x i64> %0
+}
+
+define <vscale x 16 x i8> @test_svrevd_s8_x_1(<vscale x 16 x i1> %pg, <vscale x 16 x i8> %x) {
+; CHECK-LABEL: test_svrevd_s8_x_1:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    revd z0.q, p0/m, z0.q
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svrevd_s8_x_1:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    revd z0.q, p0/z, z0.q
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 16 x i8> @llvm.aarch64.sve.revd.nxv16i8(<vscale x 16 x i8> poison, <vscale x 16 x i1> %pg, <vscale x 16 x i8> %x)
+  ret <vscale x 16 x i8> %0
+}
+
+define <vscale x 16 x i8> @test_svrevd_s8_x_2(<vscale x 16 x i1> %pg, double %z0, <vscale x 16 x i8> %x) {
+; CHECK-LABEL: test_svrevd_s8_x_2:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    revd z0.q, p0/m, z1.q
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svrevd_s8_x_2:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    revd z0.q, p0/z, z1.q
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 16 x i8> @llvm.aarch64.sve.revd.nxv16i8(<vscale x 16 x i8> poison, <vscale x 16 x i1> %pg, <vscale x 16 x i8> %x)
+  ret <vscale x 16 x i8> %0
+}
+
+define <vscale x 16 x i8> @test_svrevd_s8_z(<vscale x 16 x i1> %pg, double %z0, <vscale x 16 x i8> %x) {
+; CHECK-LABEL: test_svrevd_s8_z:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    mov z0.b, #0 // =0x0
+; CHECK-NEXT:    revd z0.q, p0/m, z1.q
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svrevd_s8_z:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    revd z0.q, p0/z, z1.q
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 16 x i8> @llvm.aarch64.sve.revd.nxv16i8(<vscale x 16 x i8> zeroinitializer, <vscale x 16 x i1> %pg, <vscale x 16 x i8> %x)
+  ret <vscale x 16 x i8> %0
+}
+
+define <vscale x 8 x i16> @test_svrevd_s16_x_1(<vscale x 8 x i1> %pg, <vscale x 8 x i16> %x) {
+; CHECK-LABEL: test_svrevd_s16_x_1:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    revd z0.q, p0/m, z0.q
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svrevd_s16_x_1:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    revd z0.q, p0/z, z0.q
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 8 x i16> @llvm.aarch64.sve.revd.nxv8i16(<vscale x 8 x i16> poison, <vscale x 8 x i1> %pg, <vscale x 8 x i16> %x)
+  ret <vscale x 8 x i16> %0
+}
+
+define <vscale x 8 x i16> @test_svrevd_s16_x_2(<vscale x 8 x i1> %pg, double %z0, <vscale x 8 x i16> %x) {
+; CHECK-LABEL: test_svrevd_s16_x_2:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    revd z0.q, p0/m, z1.q
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svrevd_s16_x_2:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    revd z0.q, p0/z, z1.q
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 8 x i16> @llvm.aarch64.sve.revd.nxv8i16(<vscale x 8 x i16> poison, <vscale x 8 x i1> %pg, <vscale x 8 x i16> %x)
+  ret <vscale x 8 x i16> %0
+}
+
+define <vscale x 8 x i16> @test_svrevd_s16_z(<vscale x 8 x i1> %pg, double %z0, <vscale x 8 x i16> %x) {
+; CHECK-LABEL: test_svrevd_s16_z:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    mov z0.h, #0 // =0x0
+; CHECK-NEXT:    revd z0.q, p0/m, z1.q
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svrevd_s16_z:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    revd z0.q, p0/z, z1.q
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 8 x i16> @llvm.aarch64.sve.revd.nxv8i16(<vscale x 8 x i16> zeroinitializer, <vscale x 8 x i1> %pg, <vscale x 8 x i16> %x)
+  ret <vscale x 8 x i16> %0
+}
+
+define <vscale x 4 x i32> @test_svrevd_s32_x_1(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %x) {
+; CHECK-LABEL: test_svrevd_s32_x_1:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    revd z0.q, p0/m, z0.q
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svrevd_s32_x_1:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    revd z0.q, p0/z, z0.q
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 4 x i32> @llvm.aarch64.sve.revd.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i1> %pg, <vscale x 4 x i32> %x)
+  ret <vscale x 4 x i32> %0
+}
+
+define <vscale x 4 x i32> @test_svrevd_s32_x_2(<vscale x 4 x i1> %pg, double %z0, <vscale x 4 x i32> %x) {
+; CHECK-LABEL: test_svrevd_s32_x_2:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    revd z0.q, p0/m, z1.q
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svrevd_s32_x_2:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    revd z0.q, p0/z, z1.q
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 4 x i32> @llvm.aarch64.sve.revd.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i1> %pg, <vscale x 4 x i32> %x)
+  ret <vscale x 4 x i32> %0
+}
+
+define <vscale x 4 x i32> @test_svrevd_s32_z(<vscale x 4 x i1> %pg, double %z0, <vscale x 4 x i32> %x) {
+; CHECK-LABEL: test_svrevd_s32_z:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    mov z0.s, #0 // =0x0
+; CHECK-NEXT:    revd z0.q, p0/m, z1.q
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svrevd_s32_z:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    revd z0.q, p0/z, z1.q
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 4 x i32> @llvm.aarch64.sve.revd.nxv4i32(<vscale x 4 x i32> zeroinitializer, <vscale x 4 x i1> %pg, <vscale x 4 x i32> %x)
+  ret <vscale x 4 x i32> %0
+}
+
+define <vscale x 2 x i64> @test_svrevd_s64_x_1(<vscale x 2 x i1> %pg, <vscale x 2 x i64> %x) {
+; CHECK-LABEL: test_svrevd_s64_x_1:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    revd z0.q, p0/m, z0.q
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svrevd_s64_x_1:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    revd z0.q, p0/z, z0.q
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 2 x i64> @llvm.aarch64.sve.revd.nxv2i64(<vscale x 2 x i64> poison, <vscale x 2 x i1> %pg, <vscale x 2 x i64> %x)
+  ret <vscale x 2 x i64> %0
+}
+
+define <vscale x 2 x i64> @test_svrevd_s64_x_2(<vscale x 2 x i1> %pg, double %z0, <vscale x 2 x i64> %x) {
+; CHECK-LABEL: test_svrevd_s64_x_2:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    revd z0.q, p0/m, z1.q
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svrevd_s64_x_2:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    revd z0.q, p0/z, z1.q
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 2 x i64> @llvm.aarch64.sve.revd.nxv2i64(<vscale x 2 x i64> poison, <vscale x 2 x i1> %pg, <vscale x 2 x i64> %x)
+  ret <vscale x 2 x i64> %0
+}
+
+define <vscale x 2 x i64> @test_svrevd_s64_z(<vscale x 2 x i1> %pg, double %z0, <vscale x 2 x i64> %x) {
+; CHECK-LABEL: test_svrevd_s64_z:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    mov z0.d, #0 // =0x0
+; CHECK-NEXT:    revd z0.q, p0/m, z1.q
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svrevd_s64_z:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    revd z0.q, p0/z, z1.q
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 2 x i64> @llvm.aarch64.sve.revd.nxv2i64(<vscale x 2 x i64> zeroinitializer, <vscale x 2 x i1> %pg, <vscale x 2 x i64> %x)
+  ret <vscale x 2 x i64> %0
+}
+
+define <vscale x 8 x half> @test_svrevd_f16_x_1(<vscale x 8 x i1> %pg, <vscale x 8 x half> %x) {
+; CHECK-LABEL: test_svrevd_f16_x_1:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    revd z0.q, p0/m, z0.q
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svrevd_f16_x_1:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    revd z0.q, p0/z, z0.q
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 8 x half> @llvm.aarch64.sve.revd.nxv8f16(<vscale x 8 x half> poison, <vscale x 8 x i1> %pg, <vscale x 8 x half> %x)
+  ret <vscale x 8 x half> %0
+}
+
+define <vscale x 8 x half> @test_svrevd_f16_x_2(<vscale x 8 x i1> %pg, double %z0, <vscale x 8 x half> %x) {
+; CHECK-LABEL: test_svrevd_f16_x_2:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    revd z0.q, p0/m, z1.q
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svrevd_f16_x_2:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    revd z0.q, p0/z, z1.q
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 8 x half> @llvm.aarch64.sve.revd.nxv8f16(<vscale x 8 x half> poison, <vscale x 8 x i1> %pg, <vscale x 8 x half> %x)
+  ret <vscale x 8 x half> %0
+}
+
+define <vscale x 8 x half> @test_svrevd_f16_z(<vscale x 8 x i1> %pg, double %z0, <vscale x 8 x half> %x) {
+; CHECK-LABEL: test_svrevd_f16_z:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    mov z0.h, #0 // =0x0
+; CHECK-NEXT:    revd z0.q, p0/m, z1.q
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svrevd_f16_z:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    revd z0.q, p0/z, z1.q
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 8 x half> @llvm.aarch64.sve.revd.nxv8f16(<vscale x 8 x half> zeroinitializer, <vscale x 8 x i1> %pg, <vscale x 8 x half> %x)
+  ret <vscale x 8 x half> %0
+}
+
+define <vscale x 8 x bfloat> @test_svrevd_bf16_x_1(<vscale x 8 x i1> %pg, <vscale x 8 x bfloat> %x) {
+; CHECK-LABEL: test_svrevd_bf16_x_1:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    revd z0.q, p0/m, z0.q
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svrevd_bf16_x_1:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    revd z0.q, p0/z, z0.q
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 8 x bfloat> @llvm.aarch64.sve.revd.nxv8bf16(<vscale x 8 x bfloat> poison, <vscale x 8 x i1> %pg, <vscale x 8 x bfloat> %x)
+  ret <vscale x 8 x bfloat> %0
+}
+
+define <vscale x 8 x bfloat> @test_svrevd_bf16_x_2(<vscale x 8 x i1> %pg, double %z0, <vscale x 8 x bfloat> %x) {
+; CHECK-LABEL: test_svrevd_bf16_x_2:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    revd z0.q, p0/m, z1.q
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svrevd_bf16_x_2:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    revd z0.q, p0/z, z1.q
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 8 x bfloat> @llvm.aarch64.sve.revd.nxv8bf16(<vscale x 8 x bfloat> poison, <vscale x 8 x i1> %pg, <vscale x 8 x bfloat> %x)
+  ret <vscale x 8 x bfloat> %0
+}
+
+define <vscale x 8 x bfloat> @test_svrevd_bf16_z(<vscale x 8 x i1> %pg, double %z0, <vscale x 8 x bfloat> %x) {
+; CHECK-LABEL: test_svrevd_bf16_z:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    mov z0.h, #0 // =0x0
+; CHECK-NEXT:    revd z0.q, p0/m, z1.q
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svrevd_bf16_z:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    revd z0.q, p0/z, z1.q
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 8 x bfloat> @llvm.aarch64.sve.revd.nxv8bf16(<vscale x 8 x bfloat> zeroinitializer, <vscale x 8 x i1> %pg, <vscale x 8 x bfloat> %x)
+  ret <vscale x 8 x bfloat> %0
+}
+
+define <vscale x 4 x float> @test_svrevd_f32_x_1(<vscale x 4 x i1> %pg, <vscale x 4 x float> %x) {
+; CHECK-LABEL: test_svrevd_f32_x_1:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    revd z0.q, p0/m, z0.q
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svrevd_f32_x_1:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    revd z0.q, p0/z, z0.q
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 4 x float> @llvm.aarch64.sve.revd.nxv4f32(<vscale x 4 x float> poison, <vscale x 4 x i1> %pg, <vscale x 4 x float> %x)
+  ret <vscale x 4 x float> %0
+}
+
+define <vscale x 4 x float> @test_svrevd_f32_x_2(<vscale x 4 x i1> %pg, double %z0, <vscale x 4 x float> %x) {
+; CHECK-LABEL: test_svrevd_f32_x_2:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    revd z0.q, p0/m, z1.q
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svrevd_f32_x_2:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    revd z0.q, p0/z, z1.q
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 4 x float> @llvm.aarch64.sve.revd.nxv4f32(<vscale x 4 x float> poison, <vscale x 4 x i1> %pg, <vscale x 4 x float> %x)
+  ret <vscale x 4 x float> %0
+}
+
+define <vscale x 4 x float> @test_svrevd_f32_z(<vscale x 4 x i1> %pg, double %z0, <vscale x 4 x float> %x) {
+; CHECK-LABEL: test_svrevd_f32_z:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    mov z0.s, #0 // =0x0
+; CHECK-NEXT:    revd z0.q, p0/m, z1.q
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svrevd_f32_z:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    revd z0.q, p0/z, z1.q
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 4 x float> @llvm.aarch64.sve.revd.nxv4f32(<vscale x 4 x float> zeroinitializer, <vscale x 4 x i1> %pg, <vscale x 4 x float> %x)
+  ret <vscale x 4 x float> %0
+}
+
+define <vscale x 2 x double> @test_svrevd_f64_x_1(<vscale x 2 x i1> %pg, <vscale x 2 x double> %x) {
+; CHECK-LABEL: test_svrevd_f64_x_1:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    revd z0.q, p0/m, z0.q
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svrevd_f64_x_1:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    revd z0.q, p0/z, z0.q
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 2 x double> @llvm.aarch64.sve.revd.nxv2f64(<vscale x 2 x double> poison, <vscale x 2 x i1> %pg, <vscale x 2 x double> %x)
+  ret <vscale x 2 x double> %0
+}
+
+define <vscale x 2 x double> @test_svrevd_f64_x_2(<vscale x 2 x i1> %pg, double %z0, <vscale x 2 x double> %x) {
+; CHECK-LABEL: test_svrevd_f64_x_2:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    revd z0.q, p0/m, z1.q
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svrevd_f64_x_2:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    revd z0.q, p0/z, z1.q
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 2 x double> @llvm.aarch64.sve.revd.nxv2f64(<vscale x 2 x double> poison, <vscale x 2 x i1> %pg, <vscale x 2 x double> %x)
+  ret <vscale x 2 x double> %0
+}
+
+define <vscale x 2 x double> @test_svrevd_f64_z(<vscale x 2 x i1> %pg, double %z0, <vscale x 2 x double> %x) {
+; CHECK-LABEL: test_svrevd_f64_z:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    mov z0.d, #0 // =0x0
+; CHECK-NEXT:    revd z0.q, p0/m, z1.q
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svrevd_f64_z:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    revd z0.q, p0/z, z1.q
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 2 x double> @llvm.aarch64.sve.revd.nxv2f64(<vscale x 2 x double> zeroinitializer, <vscale x 2 x i1> %pg, <vscale x 2 x double> %x)
+  ret <vscale x 2 x double> %0
+}
+
+define <vscale x 16 x i8> @test_svrbit_nxv16i8_ptrue_u(double %z0, <vscale x 16 x i8> %x) {
+; CHECK-LABEL: test_svrbit_nxv16i8_ptrue_u:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ptrue p0.b
+; CHECK-NEXT:    rbit z0.b, p0/m, z1.b
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svrbit_nxv16i8_ptrue_u:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    ptrue p0.b
+; CHECK-2p2-NEXT:    rbit z0.b, p0/z, z1.b
+; CHECK-2p2-NEXT:    ret
+entry:
+  %pg = call <vscale x 16 x i1> @llvm.aarch64.sve.ptrue.nxv16i1(i32 31)
+  %0 = tail call <vscale x 16 x i8> @llvm.aarch64.sve.rbit.nxv16i8(<vscale x 16 x i8> poison, <vscale x 16 x i1> %pg, <vscale x 16 x i8> %x)
+  ret <vscale x 16 x i8> %0
+}
+
+define <vscale x 16 x i8> @test_svrbit_nxv16i8_ptrue(double %z0, <vscale x 16 x i8> %x, <vscale x 16 x i8> %y) {
+; CHECK-LABEL: test_svrbit_nxv16i8_ptrue:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    mov z0.d, z1.d
+; CHECK-NEXT:    ptrue p0.b
+; CHECK-NEXT:    rbit z0.b, p0/m, z2.b
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svrbit_nxv16i8_ptrue:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    ptrue p0.b
+; CHECK-2p2-NEXT:    rbit z0.b, p0/z, z2.b
+; CHECK-2p2-NEXT:    ret
+entry:
+  %pg = call <vscale x 16 x i1> @llvm.aarch64.sve.ptrue.nxv16i1(i32 31)
+  %0 = tail call <vscale x 16 x i8> @llvm.aarch64.sve.rbit.nxv16i8(<vscale x 16 x i8> %x, <vscale x 16 x i1> %pg, <vscale x 16 x i8> %y)
+  ret <vscale x 16 x i8> %0
+}
+
+define <vscale x 8 x i16> @test_svrbit_nxv8i16_ptrue_u(double %z0, <vscale x 8 x i16> %x) {
+; CHECK-LABEL: test_svrbit_nxv8i16_ptrue_u:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ptrue p0.h
+; CHECK-NEXT:    rbit z0.h, p0/m, z1.h
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svrbit_nxv8i16_ptrue_u:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    ptrue p0.h
+; CHECK-2p2-NEXT:    rbit z0.h, p0/z, z1.h
+; CHECK-2p2-NEXT:    ret
+entry:
+  %pg = call <vscale x 8 x i1> @llvm.aarch64.sve.ptrue.nxv8i1(i32 31)
+  %0 = tail call <vscale x 8 x i16> @llvm.aarch64.sve.rbit.nxv8i16(<vscale x 8 x i16> poison, <vscale x 8 x i1> %pg, <vscale x 8 x i16> %x)
+  ret <vscale x 8 x i16> %0
+}
+
+define <vscale x 8 x i16> @test_svrbit_nxv8i16_ptrue(double %z0, <vscale x 8 x i16> %x, <vscale x 8 x i16> %y) {
+; CHECK-LABEL: test_svrbit_nxv8i16_ptrue:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    mov z0.d, z1.d
+; CHECK-NEXT:    ptrue p0.h
+; CHECK-NEXT:    rbit z0.h, p0/m, z2.h
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svrbit_nxv8i16_ptrue:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    ptrue p0.h
+; CHECK-2p2-NEXT:    rbit z0.h, p0/z, z2.h
+; CHECK-2p2-NEXT:    ret
+entry:
+  %pg = call <vscale x 8 x i1> @llvm.aarch64.sve.ptrue.nxv8i1(i32 31)
+  %0 = tail call <vscale x 8 x i16> @llvm.aarch64.sve.rbit.nxv8i16(<vscale x 8 x i16> %x, <vscale x 8 x i1> %pg, <vscale x 8 x i16> %y)
+  ret <vscale x 8 x i16> %0
+}
+
+define <vscale x 4 x i32> @test_svrbit_nxv4i32_ptrue_u(double %z0, <vscale x 4 x i32> %x) {
+; CHECK-LABEL: test_svrbit_nxv4i32_ptrue_u:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ptrue p0.s
+; CHECK-NEXT:    rbit z0.s, p0/m, z1.s
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svrbit_nxv4i32_ptrue_u:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    ptrue p0.s
+; CHECK-2p2-NEXT:    rbit z0.s, p0/z, z1.s
+; CHECK-2p2-NEXT:    ret
+entry:
+  %pg = call <vscale x 4 x i1> @llvm.aarch64.sve.ptrue.nxv4i1(i32 31)
+  %0 = tail call <vscale x 4 x i32> @llvm.aarch64.sve.rbit.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i1> %pg, <vscale x 4 x i32> %x)
+  ret <vscale x 4 x i32> %0
+}
+
+define <vscale x 4 x i32> @test_svrbit_nxv4i32_ptrue(double %z0, <vscale x 4 x i32> %x, <vscale x 4 x i32> %y) {
+; CHECK-LABEL: test_svrbit_nxv4i32_ptrue:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    mov z0.d, z1.d
+; CHECK-NEXT:    ptrue p0.s
+; CHECK-NEXT:    rbit z0.s, p0/m, z2.s
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svrbit_nxv4i32_ptrue:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    ptrue p0.s
+; CHECK-2p2-NEXT:    rbit z0.s, p0/z, z2.s
+; CHECK-2p2-NEXT:    ret
+entry:
+  %pg = call <vscale x 4 x i1> @llvm.aarch64.sve.ptrue.nxv4i1(i32 31)
+  %0 = tail call <vscale x 4 x i32> @llvm.aarch64.sve.rbit.nxv4i32(<vscale x 4 x i32> %x, <vscale x 4 x i1> %pg, <vscale x 4 x i32> %y)
+  ret <vscale x 4 x i32> %0
+}
+
+define <vscale x 2 x i64> @test_svrbit_nxv2i64_ptrue_u(double %z0, <vscale x 2 x i64> %x) {
+; CHECK-LABEL: test_svrbit_nxv2i64_ptrue_u:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ptrue p0.d
+; CHECK-NEXT:    rbit z0.d, p0/m, z1.d
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svrbit_nxv2i64_ptrue_u:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    ptrue p0.d
+; CHECK-2p2-NEXT:    rbit z0.d, p0/z, z1.d
+; CHECK-2p2-NEXT:    ret
+entry:
+  %pg = call <vscale x 2 x i1> @llvm.aarch64.sve.ptrue.nxv2i1(i32 31)
+  %0 = tail call <vscale x 2 x i64> @llvm.aarch64.sve.rbit.nxv2i64(<vscale x 2 x i64> poison, <vscale x 2 x i1> %pg, <vscale x 2 x i64> %x)
+  ret <vscale x 2 x i64> %0
+}
+
+define <vscale x 2 x i64> @test_svrbit_nxv2i64_ptrue(double %z0, <vscale x 2 x i64> %x, <vscale x 2 x i64> %y) {
+; CHECK-LABEL: test_svrbit_nxv2i64_ptrue:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    mov z0.d, z1.d
+; CHECK-NEXT:    ptrue p0.d
+; CHECK-NEXT:    rbit z0.d, p0/m, z2.d
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svrbit_nxv2i64_ptrue:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    ptrue p0.d
+; CHECK-2p2-NEXT:    rbit z0.d, p0/z, z2.d
+; CHECK-2p2-NEXT:    ret
+entry:
+  %pg = call <vscale x 2 x i1> @llvm.aarch64.sve.ptrue.nxv2i1(i32 31)
+  %0 = tail call <vscale x 2 x i64> @llvm.aarch64.sve.rbit.nxv2i64(<vscale x 2 x i64> %x, <vscale x 2 x i1> %pg, <vscale x 2 x i64> %y)
+  ret <vscale x 2 x i64> %0
+}
+
+define <vscale x 8 x i16> @test_svrevb_nxv8i16_ptrue_u(double %z0, <vscale x 8 x i16> %x) {
+; CHECK-LABEL: test_svrevb_nxv8i16_ptrue_u:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ptrue p0.h
+; CHECK-NEXT:    revb z0.h, p0/m, z1.h
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svrevb_nxv8i16_ptrue_u:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    ptrue p0.h
+; CHECK-2p2-NEXT:    revb z0.h, p0/z, z1.h
+; CHECK-2p2-NEXT:    ret
+entry:
+  %pg = call <vscale x 8 x i1> @llvm.aarch64.sve.ptrue.nxv8i1(i32 31)
+  %0 = tail call <vscale x 8 x i16> @llvm.aarch64.sve.revb.nxv8i16(<vscale x 8 x i16> poison, <vscale x 8 x i1> %pg, <vscale x 8 x i16> %x)
+  ret <vscale x 8 x i16> %0
+}
+
+define <vscale x 8 x i16> @test_svrevb_nxv8i16_ptrue(double %z0, <vscale x 8 x i16> %x, <vscale x 8 x i16> %y) {
+; CHECK-LABEL: test_svrevb_nxv8i16_ptrue:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    mov z0.d, z1.d
+; CHECK-NEXT:    ptrue p0.h
+; CHECK-NEXT:    revb z0.h, p0/m, z2.h
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svrevb_nxv8i16_ptrue:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    ptrue p0.h
+; CHECK-2p2-NEXT:    revb z0.h, p0/z, z2.h
+; CHECK-2p2-NEXT:    ret
+entry:
+  %pg = call <vscale x 8 x i1> @llvm.aarch64.sve.ptrue.nxv8i1(i32 31)
+  %0 = tail call <vscale x 8 x i16> @llvm.aarch64.sve.revb.nxv8i16(<vscale x 8 x i16> %x, <vscale x 8 x i1> %pg, <vscale x 8 x i16> %y)
+  ret <vscale x 8 x i16> %0
+}
+
+define <vscale x 4 x i32> @test_svrevb_nxv4i32_ptrue_u(double %z0, <vscale x 4 x i32> %x) {
+; CHECK-LABEL: test_svrevb_nxv4i32_ptrue_u:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ptrue p0.s
+; CHECK-NEXT:    revb z0.s, p0/m, z1.s
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svrevb_nxv4i32_ptrue_u:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    ptrue p0.s
+; CHECK-2p2-NEXT:    revb z0.s, p0/z, z1.s
+; CHECK-2p2-NEXT:    ret
+entry:
+  %pg = call <vscale x 4 x i1> @llvm.aarch64.sve.ptrue.nxv4i1(i32 31)
+  %0 = tail call <vscale x 4 x i32> @llvm.aarch64.sve.revb.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i1> %pg, <vscale x 4 x i32> %x)
+  ret <vscale x 4 x i32> %0
+}
+
+define <vscale x 4 x i32> @test_svrevb_nxv4i32_ptrue(double %z0, <vscale x 4 x i32> %x, <vscale x 4 x i32> %y) {
+; CHECK-LABEL: test_svrevb_nxv4i32_ptrue:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    mov z0.d, z1.d
+; CHECK-NEXT:    ptrue p0.s
+; CHECK-NEXT:    revb z0.s, p0/m, z2.s
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svrevb_nxv4i32_ptrue:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    ptrue p0.s
+; CHECK-2p2-NEXT:    revb z0.s, p0/z, z2.s
+; CHECK-2p2-NEXT:    ret
+entry:
+  %pg = call <vscale x 4 x i1> @llvm.aarch64.sve.ptrue.nxv4i1(i32 31)
+  %0 = tail call <vscale x 4 x i32> @llvm.aarch64.sve.revb.nxv4i32(<vscale x 4 x i32> %x, <vscale x 4 x i1> %pg, <vscale x 4 x i32> %y)
+  ret <vscale x 4 x i32> %0
+}
+
+define <vscale x 2 x i64> @test_svrevb_nxv2i64_ptrue_u(double %z0, <vscale x 2 x i64> %x) {
+; CHECK-LABEL: test_svrevb_nxv2i64_ptrue_u:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ptrue p0.d
+; CHECK-NEXT:    revb z0.d, p0/m, z1.d
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svrevb_nxv2i64_ptrue_u:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    ptrue p0.d
+; CHECK-2p2-NEXT:    revb z0.d, p0/z, z1.d
+; CHECK-2p2-NEXT:    ret
+entry:
+  %pg = call <vscale x 2 x i1> @llvm.aarch64.sve.ptrue.nxv2i1(i32 31)
+  %0 = tail call <vscale x 2 x i64> @llvm.aarch64.sve.revb.nxv2i64(<vscale x 2 x i64> poison, <vscale x 2 x i1> %pg, <vscale x 2 x i64> %x)
+  ret <vscale x 2 x i64> %0
+}
+
+define <vscale x 2 x i64> @test_svrevb_nxv2i64_ptrue(double %z0, <vscale x 2 x i64> %x, <vscale x 2 x i64> %y) {
+; CHECK-LABEL: test_svrevb_nxv2i64_ptrue:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    mov z0.d, z1.d
+; CHECK-NEXT:    ptrue p0.d
+; CHECK-NEXT:    revb z0.d, p0/m, z2.d
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svrevb_nxv2i64_ptrue:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    ptrue p0.d
+; CHECK-2p2-NEXT:    revb z0.d, p0/z, z2.d
+; CHECK-2p2-NEXT:    ret
+entry:
+  %pg = call <vscale x 2 x i1> @llvm.aarch64.sve.ptrue.nxv2i1(i32 31)
+  %0 = tail call <vscale x 2 x i64> @llvm.aarch64.sve.revb.nxv2i64(<vscale x 2 x i64> %x, <vscale x 2 x i1> %pg, <vscale x 2 x i64> %y)
+  ret <vscale x 2 x i64> %0
+}
+
+define <vscale x 4 x i32> @test_svrevh_nxv4i32_ptrue_u(double %z0, <vscale x 4 x i32> %x) {
+; CHECK-LABEL: test_svrevh_nxv4i32_ptrue_u:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ptrue p0.s
+; CHECK-NEXT:    revh z0.s, p0/m, z1.s
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svrevh_nxv4i32_ptrue_u:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    ptrue p0.s
+; CHECK-2p2-NEXT:    revh z0.s, p0/z, z1.s
+; CHECK-2p2-NEXT:    ret
+entry:
+  %pg = call <vscale x 4 x i1> @llvm.aarch64.sve.ptrue.nxv4i1(i32 31)
+  %0 = tail call <vscale x 4 x i32> @llvm.aarch64.sve.revh.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i1> %pg, <vscale x 4 x i32> %x)
+  ret <vscale x 4 x i32> %0
+}
+
+define <vscale x 4 x i32> @test_svrevh_nxv4i32_ptrue(double %z0, <vscale x 4 x i32> %x, <vscale x 4 x i32> %y) {
+; CHECK-LABEL: test_svrevh_nxv4i32_ptrue:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    mov z0.d, z1.d
+; CHECK-NEXT:    ptrue p0.s
+; CHECK-NEXT:    revh z0.s, p0/m, z2.s
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svrevh_nxv4i32_ptrue:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    ptrue p0.s
+; CHECK-2p2-NEXT:    revh z0.s, p0/z, z2.s
+; CHECK-2p2-NEXT:    ret
+entry:
+  %pg = call <vscale x 4 x i1> @llvm.aarch64.sve.ptrue.nxv4i1(i32 31)
+  %0 = tail call <vscale x 4 x i32> @llvm.aarch64.sve.revh.nxv4i32(<vscale x 4 x i32> %x, <vscale x 4 x i1> %pg, <vscale x 4 x i32> %y)
+  ret <vscale x 4 x i32> %0
+}
+
+define <vscale x 2 x i64> @test_svrevh_nxv2i64_ptrue_u(double %z0, <vscale x 2 x i64> %x) {
+; CHECK-LABEL: test_svrevh_nxv2i64_ptrue_u:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ptrue p0.d
+; CHECK-NEXT:    revh z0.d, p0/m, z1.d
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svrevh_nxv2i64_ptrue_u:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    ptrue p0.d
+; CHECK-2p2-NEXT:    revh z0.d, p0/z, z1.d
+; CHECK-2p2-NEXT:    ret
+entry:
+  %pg = call <vscale x 2 x i1> @llvm.aarch64.sve.ptrue.nxv2i1(i32 31)
+  %0 = tail call <vscale x 2 x i64> @llvm.aarch64.sve.revh.nxv2i64(<vscale x 2 x i64> poison, <vscale x 2 x i1> %pg, <vscale x 2 x i64> %x)
+  ret <vscale x 2 x i64> %0
+}
+
+define <vscale x 2 x i64> @test_svrevh_nxv2i64_ptrue(double %z0, <vscale x 2 x i64> %x, <vscale x 2 x i64> %y) {
+; CHECK-LABEL: test_svrevh_nxv2i64_ptrue:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    mov z0.d, z1.d
+; CHECK-NEXT:    ptrue p0.d
+; CHECK-NEXT:    revh z0.d, p0/m, z2.d
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svrevh_nxv2i64_ptrue:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    ptrue p0.d
+; CHECK-2p2-NEXT:    revh z0.d, p0/z, z2.d
+; CHECK-2p2-NEXT:    ret
+entry:
+  %pg = call <vscale x 2 x i1> @llvm.aarch64.sve.ptrue.nxv2i1(i32 31)
+  %0 = tail call <vscale x 2 x i64> @llvm.aarch64.sve.revh.nxv2i64(<vscale x 2 x i64> %x, <vscale x 2 x i1> %pg, <vscale x 2 x i64> %y)
+  ret <vscale x 2 x i64> %0
+}
+
+define <vscale x 2 x i64> @test_svrevw_nxv2i64_ptrue_u(double %z0, <vscale x 2 x i64> %x) {
+; CHECK-LABEL: test_svrevw_nxv2i64_ptrue_u:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ptrue p0.d
+; CHECK-NEXT:    revw z0.d, p0/m, z1.d
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svrevw_nxv2i64_ptrue_u:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    ptrue p0.d
+; CHECK-2p2-NEXT:    revw z0.d, p0/z, z1.d
+; CHECK-2p2-NEXT:    ret
+entry:
+  %pg = call <vscale x 2 x i1> @llvm.aarch64.sve.ptrue.nxv2i1(i32 31)
+  %0 = tail call <vscale x 2 x i64> @llvm.aarch64.sve.revw.nxv2i64(<vscale x 2 x i64> poison, <vscale x 2 x i1> %pg, <vscale x 2 x i64> %x)
+  ret <vscale x 2 x i64> %0
+}
+
+define <vscale x 2 x i64> @test_svrevw_nxv2i64_ptrue(double %z0, <vscale x 2 x i64> %x, <vscale x 2 x i64> %y) {
+; CHECK-LABEL: test_svrevw_nxv2i64_ptrue:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    mov z0.d, z1.d
+; CHECK-NEXT:    ptrue p0.d
+; CHECK-NEXT:    revw z0.d, p0/m, z2.d
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svrevw_nxv2i64_ptrue:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    ptrue p0.d
+; CHECK-2p2-NEXT:    revw z0.d, p0/z, z2.d
+; CHECK-2p2-NEXT:    ret
+entry:
+  %pg = call <vscale x 2 x i1> @llvm.aarch64.sve.ptrue.nxv2i1(i32 31)
+  %0 = tail call <vscale x 2 x i64> @llvm.aarch64.sve.revw.nxv2i64(<vscale x 2 x i64> %x, <vscale x 2 x i1> %pg, <vscale x 2 x i64> %y)
+  ret <vscale x 2 x i64> %0
+}
+
+define <vscale x 16 x i8> @test_svrevd_nxv16i8_ptrue_u(double %z0, <vscale x 16 x i8> %x) {
+; CHECK-LABEL: test_svrevd_nxv16i8_ptrue_u:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ptrue p0.b
+; CHECK-NEXT:    revd z0.q, p0/m, z1.q
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svrevd_nxv16i8_ptrue_u:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    ptrue p0.b
+; CHECK-2p2-NEXT:    revd z0.q, p0/z, z1.q
+; CHECK-2p2-NEXT:    ret
+entry:
+  %pg = call <vscale x 16 x i1> @llvm.aarch64.sve.ptrue.nxv16i1(i32 31)
+  %0 = tail call <vscale x 16 x i8> @llvm.aarch64.sve.revd.nxv16i8(<vscale x 16 x i8> poison, <vscale x 16 x i1> %pg, <vscale x 16 x i8> %x)
+  ret <vscale x 16 x i8> %0
+}
+
+define <vscale x 16 x i8> @test_svrevd_nxv16i8_ptrue(double %z0, <vscale x 16 x i8> %x, <vscale x 16 x i8> %y) {
+; CHECK-LABEL: test_svrevd_nxv16i8_ptrue:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    mov z0.d, z1.d
+; CHECK-NEXT:    ptrue p0.b
+; CHECK-NEXT:    revd z0.q, p0/m, z2.q
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svrevd_nxv16i8_ptrue:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    ptrue p0.b
+; CHECK-2p2-NEXT:    revd z0.q, p0/z, z2.q
+; CHECK-2p2-NEXT:    ret
+entry:
+  %pg = call <vscale x 16 x i1> @llvm.aarch64.sve.ptrue.nxv16i1(i32 31)
+  %0 = tail call <vscale x 16 x i8> @llvm.aarch64.sve.revd.nxv16i8(<vscale x 16 x i8> %x, <vscale x 16 x i1> %pg, <vscale x 16 x i8> %y)
+  ret <vscale x 16 x i8> %0
+}
+
+define <vscale x 8 x i16> @test_svrevd_nxv8i16_ptrue_u(double %z0, <vscale x 8 x i16> %x) {
+; CHECK-LABEL: test_svrevd_nxv8i16_ptrue_u:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ptrue p0.h
+; CHECK-NEXT:    revd z0.q, p0/m, z1.q
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svrevd_nxv8i16_ptrue_u:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    ptrue p0.h
+; CHECK-2p2-NEXT:    revd z0.q, p0/z, z1.q
+; CHECK-2p2-NEXT:    ret
+entry:
+  %pg = call <vscale x 8 x i1> @llvm.aarch64.sve.ptrue.nxv8i1(i32 31)
+  %0 = tail call <vscale x 8 x i16> @llvm.aarch64.sve.revd.nxv8i16(<vscale x 8 x i16> poison, <vscale x 8 x i1> %pg, <vscale x 8 x i16> %x)
+  ret <vscale x 8 x i16> %0
+}
+
+define <vscale x 8 x i16> @test_svrevd_nxv8i16_ptrue(double %z0, <vscale x 8 x i16> %x, <vscale x 8 x i16> %y) {
+; CHECK-LABEL: test_svrevd_nxv8i16_ptrue:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    mov z0.d, z1.d
+; CHECK-NEXT:    ptrue p0.h
+; CHECK-NEXT:    revd z0.q, p0/m, z2.q
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svrevd_nxv8i16_ptrue:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    ptrue p0.h
+; CHECK-2p2-NEXT:    revd z0.q, p0/z, z2.q
+; CHECK-2p2-NEXT:    ret
+entry:
+  %pg = call <vscale x 8 x i1> @llvm.aarch64.sve.ptrue.nxv8i1(i32 31)
+  %0 = tail call <vscale x 8 x i16> @llvm.aarch64.sve.revd.nxv8i16(<vscale x 8 x i16> %x, <vscale x 8 x i1> %pg, <vscale x 8 x i16> %y)
+  ret <vscale x 8 x i16> %0
+}
+
+define <vscale x 4 x i32> @test_svrevd_nxv4i32_ptrue_u(double %z0, <vscale x 4 x i32> %x) {
+; CHECK-LABEL: test_svrevd_nxv4i32_ptrue_u:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ptrue p0.s
+; CHECK-NEXT:    revd z0.q, p0/m, z1.q
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svrevd_nxv4i32_ptrue_u:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    ptrue p0.s
+; CHECK-2p2-NEXT:    revd z0.q, p0/z, z1.q
+; CHECK-2p2-NEXT:    ret
+entry:
+  %pg = call <vscale x 4 x i1> @llvm.aarch64.sve.ptrue.nxv4i1(i32 31)
+  %0 = tail call <vscale x 4 x i32> @llvm.aarch64.sve.revd.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i1> %pg, <vscale x 4 x i32> %x)
+  ret <vscale x 4 x i32> %0
+}
+
+define <vscale x 4 x i32> @test_svrevd_nxv4i32_ptrue(double %z0, <vscale x 4 x i32> %x, <vscale x 4 x i32> %y) {
+; CHECK-LABEL: test_svrevd_nxv4i32_ptrue:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    mov z0.d, z1.d
+; CHECK-NEXT:    ptrue p0.s
+; CHECK-NEXT:    revd z0.q, p0/m, z2.q
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svrevd_nxv4i32_ptrue:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    ptrue p0.s
+; CHECK-2p2-NEXT:    revd z0.q, p0/z, z2.q
+; CHECK-2p2-NEXT:    ret
+entry:
+  %pg = call <vscale x 4 x i1> @llvm.aarch64.sve.ptrue.nxv4i1(i32 31)
+  %0 = tail call <vscale x 4 x i32> @llvm.aarch64.sve.revd.nxv4i32(<vscale x 4 x i32> %x, <vscale x 4 x i1> %pg, <vscale x 4 x i32> %y)
+  ret <vscale x 4 x i32> %0
+}
+
+define <vscale x 2 x i64> @test_svrevd_nxv2i64_ptrue_u(double %z0, <vscale x 2 x i64> %x) {
+; CHECK-LABEL: test_svrevd_nxv2i64_ptrue_u:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ptrue p0.d
+; CHECK-NEXT:    revd z0.q, p0/m, z1.q
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svrevd_nxv2i64_ptrue_u:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    ptrue p0.d
+; CHECK-2p2-NEXT:    revd z0.q, p0/z, z1.q
+; CHECK-2p2-NEXT:    ret
+entry:
+  %pg = call <vscale x 2 x i1> @llvm.aarch64.sve.ptrue.nxv2i1(i32 31)
+  %0 = tail call <vscale x 2 x i64> @llvm.aarch64.sve.revd.nxv2i64(<vscale x 2 x i64> poison, <vscale x 2 x i1> %pg, <vscale x 2 x i64> %x)
+  ret <vscale x 2 x i64> %0
+}
+
+define <vscale x 2 x i64> @test_svrevd_nxv2i64_ptrue(double %z0, <vscale x 2 x i64> %x, <vscale x 2 x i64> %y) {
+; CHECK-LABEL: test_svrevd_nxv2i64_ptrue:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    mov z0.d, z1.d
+; CHECK-NEXT:    ptrue p0.d
+; CHECK-NEXT:    revd z0.q, p0/m, z2.q
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svrevd_nxv2i64_ptrue:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    ptrue p0.d
+; CHECK-2p2-NEXT:    revd z0.q, p0/z, z2.q
+; CHECK-2p2-NEXT:    ret
+entry:
+  %pg = call <vscale x 2 x i1> @llvm.aarch64.sve.ptrue.nxv2i1(i32 31)
+  %0 = tail call <vscale x 2 x i64> @llvm.aarch64.sve.revd.nxv2i64(<vscale x 2 x i64> %x, <vscale x 2 x i1> %pg, <vscale x 2 x i64> %y)
+  ret <vscale x 2 x i64> %0
+}
+
+define <vscale x 8 x half> @test_svrevd_nxv8f16_ptrue_u(double %z0, <vscale x 8 x half> %x) {
+; CHECK-LABEL: test_svrevd_nxv8f16_ptrue_u:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ptrue p0.h
+; CHECK-NEXT:    revd z0.q, p0/m, z1.q
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svrevd_nxv8f16_ptrue_u:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    ptrue p0.h
+; CHECK-2p2-NEXT:    revd z0.q, p0/z, z1.q
+; CHECK-2p2-NEXT:    ret
+entry:
+  %pg = call <vscale x 8 x i1> @llvm.aarch64.sve.ptrue.nxv8i1(i32 31)
+  %0 = tail call <vscale x 8 x half> @llvm.aarch64.sve.revd.nxv8f16(<vscale x 8 x half> poison, <vscale x 8 x i1> %pg, <vscale x 8 x half> %x)
+  ret <vscale x 8 x half> %0
+}
+
+define <vscale x 8 x half> @test_svrevd_nxv8f16_ptrue(double %z0, <vscale x 8 x half> %x, <vscale x 8 x half> %y) {
+; CHECK-LABEL: test_svrevd_nxv8f16_ptrue:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    mov z0.d, z1.d
+; CHECK-NEXT:    ptrue p0.h
+; CHECK-NEXT:    revd z0.q, p0/m, z2.q
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svrevd_nxv8f16_ptrue:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    ptrue p0.h
+; CHECK-2p2-NEXT:    revd z0.q, p0/z, z2.q
+; CHECK-2p2-NEXT:    ret
+entry:
+  %pg = call <vscale x 8 x i1> @llvm.aarch64.sve.ptrue.nxv8i1(i32 31)
+  %0 = tail call <vscale x 8 x half> @llvm.aarch64.sve.revd.nxv8f16(<vscale x 8 x half> %x, <vscale x 8 x i1> %pg, <vscale x 8 x half> %y)
+  ret <vscale x 8 x half> %0
+}
+
+define <vscale x 8 x bfloat> @test_svrevd_nxv8bf16_ptrue_u(double %z0, <vscale x 8 x bfloat> %x) {
+; CHECK-LABEL: test_svrevd_nxv8bf16_ptrue_u:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ptrue p0.h
+; CHECK-NEXT:    revd z0.q, p0/m, z1.q
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svrevd_nxv8bf16_ptrue_u:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    ptrue p0.h
+; CHECK-2p2-NEXT:    revd z0.q, p0/z, z1.q
+; CHECK-2p2-NEXT:    ret
+entry:
+  %pg = call <vscale x 8 x i1> @llvm.aarch64.sve.ptrue.nxv8i1(i32 31)
+  %0 = tail call <vscale x 8 x bfloat> @llvm.aarch64.sve.revd.nxv8bf16(<vscale x 8 x bfloat> poison, <vscale x 8 x i1> %pg, <vscale x 8 x bfloat> %x)
+  ret <vscale x 8 x bfloat> %0
+}
+
+define <vscale x 8 x bfloat> @test_svrevd_nxv8bf16_ptrue(double %z0, <vscale x 8 x bfloat> %x, <vscale x 8 x bfloat> %y) {
+; CHECK-LABEL: test_svrevd_nxv8bf16_ptrue:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    mov z0.d, z1.d
+; CHECK-NEXT:    ptrue p0.h
+; CHECK-NEXT:    revd z0.q, p0/m, z2.q
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svrevd_nxv8bf16_ptrue:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    ptrue p0.h
+; CHECK-2p2-NEXT:    revd z0.q, p0/z, z2.q
+; CHECK-2p2-NEXT:    ret
+entry:
+  %pg = call <vscale x 8 x i1> @llvm.aarch64.sve.ptrue.nxv8i1(i32 31)
+  %0 = tail call <vscale x 8 x bfloat> @llvm.aarch64.sve.revd.nxv8bf16(<vscale x 8 x bfloat> %x, <vscale x 8 x i1> %pg, <vscale x 8 x bfloat> %y)
+  ret <vscale x 8 x bfloat> %0
+}
+
+define <vscale x 4 x float> @test_svrevd_nxv4f32_ptrue_u(double %z0, <vscale x 4 x float> %x) {
+; CHECK-LABEL: test_svrevd_nxv4f32_ptrue_u:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ptrue p0.s
+; CHECK-NEXT:    revd z0.q, p0/m, z1.q
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svrevd_nxv4f32_ptrue_u:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    ptrue p0.s
+; CHECK-2p2-NEXT:    revd z0.q, p0/z, z1.q
+; CHECK-2p2-NEXT:    ret
+entry:
+  %pg = call <vscale x 4 x i1> @llvm.aarch64.sve.ptrue.nxv4i1(i32 31)
+  %0 = tail call <vscale x 4 x float> @llvm.aarch64.sve.revd.nxv4f32(<vscale x 4 x float> poison, <vscale x 4 x i1> %pg, <vscale x 4 x float> %x)
+  ret <vscale x 4 x float> %0
+}
+
+define <vscale x 4 x float> @test_svrevd_nxv4f32_ptrue(double %z0, <vscale x 4 x float> %x, <vscale x 4 x float> %y) {
+; CHECK-LABEL: test_svrevd_nxv4f32_ptrue:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    mov z0.d, z1.d
+; CHECK-NEXT:    ptrue p0.s
+; CHECK-NEXT:    revd z0.q, p0/m, z2.q
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svrevd_nxv4f32_ptrue:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    ptrue p0.s
+; CHECK-2p2-NEXT:    revd z0.q, p0/z, z2.q
+; CHECK-2p2-NEXT:    ret
+entry:
+  %pg = call <vscale x 4 x i1> @llvm.aarch64.sve.ptrue.nxv4i1(i32 31)
+  %0 = tail call <vscale x 4 x float> @llvm.aarch64.sve.revd.nxv4f32(<vscale x 4 x float> %x, <vscale x 4 x i1> %pg, <vscale x 4 x float> %y)
+  ret <vscale x 4 x float> %0
+}
+
+define <vscale x 2 x double> @test_svrevd_nxv2f64_ptrue_u(double %z0, <vscale x 2 x double> %x) {
+; CHECK-LABEL: test_svrevd_nxv2f64_ptrue_u:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ptrue p0.d
+; CHECK-NEXT:    revd z0.q, p0/m, z1.q
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svrevd_nxv2f64_ptrue_u:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    ptrue p0.d
+; CHECK-2p2-NEXT:    revd z0.q, p0/z, z1.q
+; CHECK-2p2-NEXT:    ret
+entry:
+  %pg = call <vscale x 2 x i1> @llvm.aarch64.sve.ptrue.nxv2i1(i32 31)
+  %0 = tail call <vscale x 2 x double> @llvm.aarch64.sve.revd.nxv2f64(<vscale x 2 x double> poison, <vscale x 2 x i1> %pg, <vscale x 2 x double> %x)
+  ret <vscale x 2 x double> %0
+}
+
+define <vscale x 2 x double> @test_svrevd_nxv2f64_ptrue(double %z0, <vscale x 2 x double> %x, <vscale x 2 x double> %y) {
+; CHECK-LABEL: test_svrevd_nxv2f64_ptrue:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    mov z0.d, z1.d
+; CHECK-NEXT:    ptrue p0.d
+; CHECK-NEXT:    revd z0.q, p0/m, z2.q
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svrevd_nxv2f64_ptrue:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    ptrue p0.d
+; CHECK-2p2-NEXT:    revd z0.q, p0/z, z2.q
+; CHECK-2p2-NEXT:    ret
+entry:
+  %pg = call <vscale x 2 x i1> @llvm.aarch64.sve.ptrue.nxv2i1(i32 31)
+  %0 = tail call <vscale x 2 x double> @llvm.aarch64.sve.revd.nxv2f64(<vscale x 2 x double> %x, <vscale x 2 x i1> %pg, <vscale x 2 x double> %y)
+  ret <vscale x 2 x double> %0
+}
diff --git a/llvm/test/CodeGen/AArch64/zeroing-forms-urecpe-ursqrte-sqabs-sqneg.ll b/llvm/test/CodeGen/AArch64/zeroing-forms-urecpe-ursqrte-sqabs-sqneg.ll
new file mode 100644
index 0000000000000..787ac4458079c
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/zeroing-forms-urecpe-ursqrte-sqabs-sqneg.ll
@@ -0,0 +1,858 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc -mattr=+sve2   < %s | FileCheck %s
+; RUN: llc -mattr=+sve2p2 < %s | FileCheck %s -check-prefix CHECK-2p2
+
+; RUN: llc -mattr=+sme    --force-streaming < %s | FileCheck %s
+; RUN: llc -mattr=+sme2p2 --force-streaming < %s | FileCheck %s -check-prefix CHECK-2p2
+
+target triple = "aarch64-linux"
+
+define <vscale x 4 x i32> @test_svrecpe_x_1(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %x) {
+; CHECK-LABEL: test_svrecpe_x_1:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    urecpe z0.s, p0/m, z0.s
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svrecpe_x_1:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    urecpe z0.s, p0/z, z0.s
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 4 x i32> @llvm.aarch64.sve.urecpe.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i1> %pg, <vscale x 4 x i32> %x)
+  ret <vscale x 4 x i32> %0
+}
+
+define <vscale x 4 x i32> @test_svrecpe_x_2(<vscale x 4 x i1> %pg, double %z0, <vscale x 4 x i32> %x) {
+; CHECK-LABEL: test_svrecpe_x_2:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    movprfx z0, z1
+; CHECK-NEXT:    urecpe z0.s, p0/m, z1.s
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svrecpe_x_2:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    urecpe z0.s, p0/z, z1.s
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 4 x i32> @llvm.aarch64.sve.urecpe.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i1> %pg, <vscale x 4 x i32> %x)
+  ret <vscale x 4 x i32> %0
+}
+
+define <vscale x 4 x i32> @test_svrecpe_z(<vscale x 4 x i1> %pg, double %z0, <vscale x 4 x i32> %x) {
+; CHECK-LABEL: test_svrecpe_z:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    mov z0.s, #0 // =0x0
+; CHECK-NEXT:    urecpe z0.s, p0/m, z1.s
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svrecpe_z:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    urecpe z0.s, p0/z, z1.s
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 4 x i32> @llvm.aarch64.sve.urecpe.nxv4i32(<vscale x 4 x i32> zeroinitializer, <vscale x 4 x i1> %pg, <vscale x 4 x i32> %x)
+  ret <vscale x 4 x i32> %0
+}
+
+define <vscale x 4 x i32> @test_svrsqrte_x_1(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %x) {
+; CHECK-LABEL: test_svrsqrte_x_1:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ursqrte z0.s, p0/m, z0.s
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svrsqrte_x_1:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    ursqrte z0.s, p0/z, z0.s
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 4 x i32> @llvm.aarch64.sve.ursqrte.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i1> %pg, <vscale x 4 x i32> %x)
+  ret <vscale x 4 x i32> %0
+}
+
+define <vscale x 4 x i32> @test_svrsqrte_x_2(<vscale x 4 x i1> %pg, double %z0, <vscale x 4 x i32> %x) {
+; CHECK-LABEL: test_svrsqrte_x_2:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    movprfx z0, z1
+; CHECK-NEXT:    ursqrte z0.s, p0/m, z1.s
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svrsqrte_x_2:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    ursqrte z0.s, p0/z, z1.s
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 4 x i32> @llvm.aarch64.sve.ursqrte.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i1> %pg, <vscale x 4 x i32> %x)
+  ret <vscale x 4 x i32> %0
+}
+
+define <vscale x 4 x i32> @test_svrsqrte_z(<vscale x 4 x i1> %pg, double %z0, <vscale x 4 x i32> %x) {
+; CHECK-LABEL: test_svrsqrte_z:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    mov z0.s, #0 // =0x0
+; CHECK-NEXT:    ursqrte z0.s, p0/m, z1.s
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svrsqrte_z:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    ursqrte z0.s, p0/z, z1.s
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 4 x i32> @llvm.aarch64.sve.ursqrte.nxv4i32(<vscale x 4 x i32> zeroinitializer, <vscale x 4 x i1> %pg, <vscale x 4 x i32> %x)
+  ret <vscale x 4 x i32> %0
+}
+
+define <vscale x 16 x i8> @test_svqabs_s8_x_1(<vscale x 16 x i1> %pg, <vscale x 16 x i8> %x) {
+; CHECK-LABEL: test_svqabs_s8_x_1:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    sqabs z0.b, p0/m, z0.b
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svqabs_s8_x_1:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    sqabs z0.b, p0/z, z0.b
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 16 x i8> @llvm.aarch64.sve.sqabs.nxv16i8(<vscale x 16 x i8> poison, <vscale x 16 x i1> %pg, <vscale x 16 x i8> %x)
+  ret <vscale x 16 x i8> %0
+}
+
+define <vscale x 16 x i8> @test_svqabs_s8_x_2(<vscale x 16 x i1> %pg, double %z0, <vscale x 16 x i8> %x) {
+; CHECK-LABEL: test_svqabs_s8_x_2:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    movprfx z0, z1
+; CHECK-NEXT:    sqabs z0.b, p0/m, z1.b
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svqabs_s8_x_2:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    sqabs z0.b, p0/z, z1.b
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 16 x i8> @llvm.aarch64.sve.sqabs.nxv16i8(<vscale x 16 x i8> poison, <vscale x 16 x i1> %pg, <vscale x 16 x i8> %x)
+  ret <vscale x 16 x i8> %0
+}
+
+define <vscale x 16 x i8> @test_svqabs_s8_z(<vscale x 16 x i1> %pg, double %z0, <vscale x 16 x i8> %x) {
+; CHECK-LABEL: test_svqabs_s8_z:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    mov z0.b, #0 // =0x0
+; CHECK-NEXT:    sqabs z0.b, p0/m, z1.b
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svqabs_s8_z:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    sqabs z0.b, p0/z, z1.b
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 16 x i8> @llvm.aarch64.sve.sqabs.nxv16i8(<vscale x 16 x i8> zeroinitializer, <vscale x 16 x i1> %pg, <vscale x 16 x i8> %x)
+  ret <vscale x 16 x i8> %0
+}
+
+define <vscale x 8 x i16> @test_svqabs_s16_x_1(<vscale x 8 x i1> %pg, <vscale x 8 x i16> %x) {
+; CHECK-LABEL: test_svqabs_s16_x_1:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    sqabs z0.h, p0/m, z0.h
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svqabs_s16_x_1:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    sqabs z0.h, p0/z, z0.h
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 8 x i16> @llvm.aarch64.sve.sqabs.nxv8i16(<vscale x 8 x i16> poison, <vscale x 8 x i1> %pg, <vscale x 8 x i16> %x)
+  ret <vscale x 8 x i16> %0
+}
+
+define <vscale x 8 x i16> @test_svqabs_s16_x_2(<vscale x 8 x i1> %pg, double %z0, <vscale x 8 x i16> %x) {
+; CHECK-LABEL: test_svqabs_s16_x_2:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    movprfx z0, z1
+; CHECK-NEXT:    sqabs z0.h, p0/m, z1.h
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svqabs_s16_x_2:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    sqabs z0.h, p0/z, z1.h
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 8 x i16> @llvm.aarch64.sve.sqabs.nxv8i16(<vscale x 8 x i16> poison, <vscale x 8 x i1> %pg, <vscale x 8 x i16> %x)
+  ret <vscale x 8 x i16> %0
+}
+
+define <vscale x 8 x i16> @test_svqabs_s16_z(<vscale x 8 x i1> %pg, double %z0, <vscale x 8 x i16> %x) {
+; CHECK-LABEL: test_svqabs_s16_z:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    mov z0.h, #0 // =0x0
+; CHECK-NEXT:    sqabs z0.h, p0/m, z1.h
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svqabs_s16_z:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    sqabs z0.h, p0/z, z1.h
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 8 x i16> @llvm.aarch64.sve.sqabs.nxv8i16(<vscale x 8 x i16> zeroinitializer, <vscale x 8 x i1> %pg, <vscale x 8 x i16> %x)
+  ret <vscale x 8 x i16> %0
+}
+
+define <vscale x 4 x i32> @test_svqabs_s32_x_1(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %x) {
+; CHECK-LABEL: test_svqabs_s32_x_1:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    sqabs z0.s, p0/m, z0.s
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svqabs_s32_x_1:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    sqabs z0.s, p0/z, z0.s
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 4 x i32> @llvm.aarch64.sve.sqabs.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i1> %pg, <vscale x 4 x i32> %x)
+  ret <vscale x 4 x i32> %0
+}
+
+define <vscale x 4 x i32> @test_svqabs_s32_x_2(<vscale x 4 x i1> %pg, double %z0, <vscale x 4 x i32> %x) {
+; CHECK-LABEL: test_svqabs_s32_x_2:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    movprfx z0, z1
+; CHECK-NEXT:    sqabs z0.s, p0/m, z1.s
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svqabs_s32_x_2:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    sqabs z0.s, p0/z, z1.s
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 4 x i32> @llvm.aarch64.sve.sqabs.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i1> %pg, <vscale x 4 x i32> %x)
+  ret <vscale x 4 x i32> %0
+}
+
+define <vscale x 4 x i32> @test_svqabs_s32_z(<vscale x 4 x i1> %pg, double %z0, <vscale x 4 x i32> %x) {
+; CHECK-LABEL: test_svqabs_s32_z:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    mov z0.s, #0 // =0x0
+; CHECK-NEXT:    sqabs z0.s, p0/m, z1.s
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svqabs_s32_z:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    sqabs z0.s, p0/z, z1.s
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 4 x i32> @llvm.aarch64.sve.sqabs.nxv4i32(<vscale x 4 x i32> zeroinitializer, <vscale x 4 x i1> %pg, <vscale x 4 x i32> %x)
+  ret <vscale x 4 x i32> %0
+}
+
+define <vscale x 2 x i64> @test_svqabs_s64_x_1(<vscale x 2 x i1> %pg, <vscale x 2 x i64> %x) {
+; CHECK-LABEL: test_svqabs_s64_x_1:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    sqabs z0.d, p0/m, z0.d
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svqabs_s64_x_1:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    sqabs z0.d, p0/z, z0.d
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 2 x i64> @llvm.aarch64.sve.sqabs.nxv2i64(<vscale x 2 x i64> poison, <vscale x 2 x i1> %pg, <vscale x 2 x i64> %x)
+  ret <vscale x 2 x i64> %0
+}
+
+define <vscale x 2 x i64> @test_svqabs_s64_x_2(<vscale x 2 x i1> %pg, double %z0, <vscale x 2 x i64> %x) {
+; CHECK-LABEL: test_svqabs_s64_x_2:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    movprfx z0, z1
+; CHECK-NEXT:    sqabs z0.d, p0/m, z1.d
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svqabs_s64_x_2:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    sqabs z0.d, p0/z, z1.d
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 2 x i64> @llvm.aarch64.sve.sqabs.nxv2i64(<vscale x 2 x i64> poison, <vscale x 2 x i1> %pg, <vscale x 2 x i64> %x)
+  ret <vscale x 2 x i64> %0
+}
+
+define <vscale x 2 x i64> @test_svqabs_s64_z(<vscale x 2 x i1> %pg, double %z0, <vscale x 2 x i64> %x) {
+; CHECK-LABEL: test_svqabs_s64_z:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    mov z0.d, #0 // =0x0
+; CHECK-NEXT:    sqabs z0.d, p0/m, z1.d
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svqabs_s64_z:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    sqabs z0.d, p0/z, z1.d
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 2 x i64> @llvm.aarch64.sve.sqabs.nxv2i64(<vscale x 2 x i64> zeroinitializer, <vscale x 2 x i1> %pg, <vscale x 2 x i64> %x)
+  ret <vscale x 2 x i64> %0
+}
+
+define <vscale x 16 x i8> @test_svqneg_s8_x_1(<vscale x 16 x i1> %pg, <vscale x 16 x i8> %x) {
+; CHECK-LABEL: test_svqneg_s8_x_1:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    sqneg z0.b, p0/m, z0.b
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svqneg_s8_x_1:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    sqneg z0.b, p0/z, z0.b
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 16 x i8> @llvm.aarch64.sve.sqneg.nxv16i8(<vscale x 16 x i8> poison, <vscale x 16 x i1> %pg, <vscale x 16 x i8> %x)
+  ret <vscale x 16 x i8> %0
+}
+
+define <vscale x 16 x i8> @test_svqneg_s8_x_2(<vscale x 16 x i1> %pg, double %z0, <vscale x 16 x i8> %x) {
+; CHECK-LABEL: test_svqneg_s8_x_2:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    movprfx z0, z1
+; CHECK-NEXT:    sqneg z0.b, p0/m, z1.b
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svqneg_s8_x_2:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    sqneg z0.b, p0/z, z1.b
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 16 x i8> @llvm.aarch64.sve.sqneg.nxv16i8(<vscale x 16 x i8> poison, <vscale x 16 x i1> %pg, <vscale x 16 x i8> %x)
+  ret <vscale x 16 x i8> %0
+}
+
+define <vscale x 16 x i8> @test_svqneg_s8_z(<vscale x 16 x i1> %pg, double %z0, <vscale x 16 x i8> %x) {
+; CHECK-LABEL: test_svqneg_s8_z:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    mov z0.b, #0 // =0x0
+; CHECK-NEXT:    sqneg z0.b, p0/m, z1.b
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svqneg_s8_z:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    sqneg z0.b, p0/z, z1.b
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 16 x i8> @llvm.aarch64.sve.sqneg.nxv16i8(<vscale x 16 x i8> zeroinitializer, <vscale x 16 x i1> %pg, <vscale x 16 x i8> %x)
+  ret <vscale x 16 x i8> %0
+}
+
+define <vscale x 8 x i16> @test_svqneg_s16_x_1(<vscale x 8 x i1> %pg, <vscale x 8 x i16> %x) {
+; CHECK-LABEL: test_svqneg_s16_x_1:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    sqneg z0.h, p0/m, z0.h
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svqneg_s16_x_1:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    sqneg z0.h, p0/z, z0.h
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 8 x i16> @llvm.aarch64.sve.sqneg.nxv8i16(<vscale x 8 x i16> poison, <vscale x 8 x i1> %pg, <vscale x 8 x i16> %x)
+  ret <vscale x 8 x i16> %0
+}
+
+define <vscale x 8 x i16> @test_svqneg_s16_x_2(<vscale x 8 x i1> %pg, double %z0, <vscale x 8 x i16> %x) {
+; CHECK-LABEL: test_svqneg_s16_x_2:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    movprfx z0, z1
+; CHECK-NEXT:    sqneg z0.h, p0/m, z1.h
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svqneg_s16_x_2:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    sqneg z0.h, p0/z, z1.h
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 8 x i16> @llvm.aarch64.sve.sqneg.nxv8i16(<vscale x 8 x i16> poison, <vscale x 8 x i1> %pg, <vscale x 8 x i16> %x)
+  ret <vscale x 8 x i16> %0
+}
+
+define <vscale x 8 x i16> @test_svqneg_s16_z(<vscale x 8 x i1> %pg, double %z0, <vscale x 8 x i16> %x) {
+; CHECK-LABEL: test_svqneg_s16_z:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    mov z0.h, #0 // =0x0
+; CHECK-NEXT:    sqneg z0.h, p0/m, z1.h
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svqneg_s16_z:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    sqneg z0.h, p0/z, z1.h
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 8 x i16> @llvm.aarch64.sve.sqneg.nxv8i16(<vscale x 8 x i16> zeroinitializer, <vscale x 8 x i1> %pg, <vscale x 8 x i16> %x)
+  ret <vscale x 8 x i16> %0
+}
+
+define <vscale x 4 x i32> @test_svqneg_s32_x_1(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %x) {
+; CHECK-LABEL: test_svqneg_s32_x_1:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    sqneg z0.s, p0/m, z0.s
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svqneg_s32_x_1:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    sqneg z0.s, p0/z, z0.s
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 4 x i32> @llvm.aarch64.sve.sqneg.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i1> %pg, <vscale x 4 x i32> %x)
+  ret <vscale x 4 x i32> %0
+}
+
+define <vscale x 4 x i32> @test_svqneg_s32_x_2(<vscale x 4 x i1> %pg, double %z0, <vscale x 4 x i32> %x) {
+; CHECK-LABEL: test_svqneg_s32_x_2:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    movprfx z0, z1
+; CHECK-NEXT:    sqneg z0.s, p0/m, z1.s
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svqneg_s32_x_2:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    sqneg z0.s, p0/z, z1.s
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 4 x i32> @llvm.aarch64.sve.sqneg.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i1> %pg, <vscale x 4 x i32> %x)
+  ret <vscale x 4 x i32> %0
+}
+
+define <vscale x 4 x i32> @test_svqneg_s32_z(<vscale x 4 x i1> %pg, double %z0, <vscale x 4 x i32> %x) {
+; CHECK-LABEL: test_svqneg_s32_z:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    mov z0.s, #0 // =0x0
+; CHECK-NEXT:    sqneg z0.s, p0/m, z1.s
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svqneg_s32_z:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    sqneg z0.s, p0/z, z1.s
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 4 x i32> @llvm.aarch64.sve.sqneg.nxv4i32(<vscale x 4 x i32> zeroinitializer, <vscale x 4 x i1> %pg, <vscale x 4 x i32> %x)
+  ret <vscale x 4 x i32> %0
+}
+
+define <vscale x 2 x i64> @test_svqneg_s64_x_1(<vscale x 2 x i1> %pg, <vscale x 2 x i64> %x) {
+; CHECK-LABEL: test_svqneg_s64_x_1:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    sqneg z0.d, p0/m, z0.d
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svqneg_s64_x_1:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    sqneg z0.d, p0/z, z0.d
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 2 x i64> @llvm.aarch64.sve.sqneg.nxv2i64(<vscale x 2 x i64> poison, <vscale x 2 x i1> %pg, <vscale x 2 x i64> %x)
+  ret <vscale x 2 x i64> %0
+}
+
+define <vscale x 2 x i64> @test_svqneg_s64_x_2(<vscale x 2 x i1> %pg, double %z0, <vscale x 2 x i64> %x) {
+; CHECK-LABEL: test_svqneg_s64_x_2:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    movprfx z0, z1
+; CHECK-NEXT:    sqneg z0.d, p0/m, z1.d
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svqneg_s64_x_2:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    sqneg z0.d, p0/z, z1.d
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 2 x i64> @llvm.aarch64.sve.sqneg.nxv2i64(<vscale x 2 x i64> poison, <vscale x 2 x i1> %pg, <vscale x 2 x i64> %x)
+  ret <vscale x 2 x i64> %0
+}
+
+define <vscale x 2 x i64> @test_svqneg_s64_z(<vscale x 2 x i1> %pg, double %z0, <vscale x 2 x i64> %x) {
+; CHECK-LABEL: test_svqneg_s64_z:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    mov z0.d, #0 // =0x0
+; CHECK-NEXT:    sqneg z0.d, p0/m, z1.d
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svqneg_s64_z:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    sqneg z0.d, p0/z, z1.d
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 2 x i64> @llvm.aarch64.sve.sqneg.nxv2i64(<vscale x 2 x i64> zeroinitializer, <vscale x 2 x i1> %pg, <vscale x 2 x i64> %x)
+  ret <vscale x 2 x i64> %0
+}
+
+define <vscale x 4 x i32> @test_svurecpe_nxv4i32_ptrue_u(double %z0, <vscale x 4 x i32> %x) {
+; CHECK-LABEL: test_svurecpe_nxv4i32_ptrue_u:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ptrue p0.s
+; CHECK-NEXT:    movprfx z0, z1
+; CHECK-NEXT:    urecpe z0.s, p0/m, z1.s
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svurecpe_nxv4i32_ptrue_u:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    ptrue p0.s
+; CHECK-2p2-NEXT:    urecpe z0.s, p0/z, z1.s
+; CHECK-2p2-NEXT:    ret
+entry:
+  %pg = call <vscale x 4 x i1> @llvm.aarch64.sve.ptrue.nxv4i1(i32 31)
+  %0 = tail call <vscale x 4 x i32> @llvm.aarch64.sve.urecpe.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i1> %pg, <vscale x 4 x i32> %x)
+  ret <vscale x 4 x i32> %0
+}
+
+define <vscale x 4 x i32> @test_svurecpe_nxv4i32_ptrue(double %z0, <vscale x 4 x i32> %x, <vscale x 4 x i32> %y) {
+; CHECK-LABEL: test_svurecpe_nxv4i32_ptrue:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ptrue p0.s
+; CHECK-NEXT:    movprfx z0, z2
+; CHECK-NEXT:    urecpe z0.s, p0/m, z2.s
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svurecpe_nxv4i32_ptrue:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    ptrue p0.s
+; CHECK-2p2-NEXT:    urecpe z0.s, p0/z, z2.s
+; CHECK-2p2-NEXT:    ret
+entry:
+  %pg = call <vscale x 4 x i1> @llvm.aarch64.sve.ptrue.nxv4i1(i32 31)
+  %0 = tail call <vscale x 4 x i32> @llvm.aarch64.sve.urecpe.nxv4i32(<vscale x 4 x i32> %x, <vscale x 4 x i1> %pg, <vscale x 4 x i32> %y)
+  ret <vscale x 4 x i32> %0
+}
+
+define <vscale x 4 x i32> @test_svursqrte_nxv4i32_ptrue_u(double %z0, <vscale x 4 x i32> %x) {
+; CHECK-LABEL: test_svursqrte_nxv4i32_ptrue_u:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ptrue p0.s
+; CHECK-NEXT:    movprfx z0, z1
+; CHECK-NEXT:    ursqrte z0.s, p0/m, z1.s
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svursqrte_nxv4i32_ptrue_u:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    ptrue p0.s
+; CHECK-2p2-NEXT:    ursqrte z0.s, p0/z, z1.s
+; CHECK-2p2-NEXT:    ret
+entry:
+  %pg = call <vscale x 4 x i1> @llvm.aarch64.sve.ptrue.nxv4i1(i32 31)
+  %0 = tail call <vscale x 4 x i32> @llvm.aarch64.sve.ursqrte.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i1> %pg, <vscale x 4 x i32> %x)
+  ret <vscale x 4 x i32> %0
+}
+
+define <vscale x 4 x i32> @test_svursqrte_nxv4i32_ptrue(double %z0, <vscale x 4 x i32> %x, <vscale x 4 x i32> %y) {
+; CHECK-LABEL: test_svursqrte_nxv4i32_ptrue:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ptrue p0.s
+; CHECK-NEXT:    movprfx z0, z2
+; CHECK-NEXT:    ursqrte z0.s, p0/m, z2.s
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svursqrte_nxv4i32_ptrue:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    ptrue p0.s
+; CHECK-2p2-NEXT:    ursqrte z0.s, p0/z, z2.s
+; CHECK-2p2-NEXT:    ret
+entry:
+  %pg = call <vscale x 4 x i1> @llvm.aarch64.sve.ptrue.nxv4i1(i32 31)
+  %0 = tail call <vscale x 4 x i32> @llvm.aarch64.sve.ursqrte.nxv4i32(<vscale x 4 x i32> %x, <vscale x 4 x i1> %pg, <vscale x 4 x i32> %y)
+  ret <vscale x 4 x i32> %0
+}
+
+define <vscale x 16 x i8> @test_svsqabs_nxv16i8_ptrue_u(double %z0, <vscale x 16 x i8> %x) {
+; CHECK-LABEL: test_svsqabs_nxv16i8_ptrue_u:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ptrue p0.b
+; CHECK-NEXT:    movprfx z0, z1
+; CHECK-NEXT:    sqabs z0.b, p0/m, z1.b
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svsqabs_nxv16i8_ptrue_u:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    ptrue p0.b
+; CHECK-2p2-NEXT:    sqabs z0.b, p0/z, z1.b
+; CHECK-2p2-NEXT:    ret
+entry:
+  %pg = call <vscale x 16 x i1> @llvm.aarch64.sve.ptrue.nxv16i1(i32 31)
+  %0 = tail call <vscale x 16 x i8> @llvm.aarch64.sve.sqabs.nxv16i8(<vscale x 16 x i8> poison, <vscale x 16 x i1> %pg, <vscale x 16 x i8> %x)
+  ret <vscale x 16 x i8> %0
+}
+
+define <vscale x 16 x i8> @test_svsqabs_nxv16i8_ptrue(double %z0, <vscale x 16 x i8> %x, <vscale x 16 x i8> %y) {
+; CHECK-LABEL: test_svsqabs_nxv16i8_ptrue:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ptrue p0.b
+; CHECK-NEXT:    movprfx z0, z2
+; CHECK-NEXT:    sqabs z0.b, p0/m, z2.b
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svsqabs_nxv16i8_ptrue:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    ptrue p0.b
+; CHECK-2p2-NEXT:    sqabs z0.b, p0/z, z2.b
+; CHECK-2p2-NEXT:    ret
+entry:
+  %pg = call <vscale x 16 x i1> @llvm.aarch64.sve.ptrue.nxv16i1(i32 31)
+  %0 = tail call <vscale x 16 x i8> @llvm.aarch64.sve.sqabs.nxv16i8(<vscale x 16 x i8> %x, <vscale x 16 x i1> %pg, <vscale x 16 x i8> %y)
+  ret <vscale x 16 x i8> %0
+}
+
+define <vscale x 8 x i16> @test_svsqabs_nxv8i16_ptrue_u(double %z0, <vscale x 8 x i16> %x) {
+; CHECK-LABEL: test_svsqabs_nxv8i16_ptrue_u:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ptrue p0.h
+; CHECK-NEXT:    movprfx z0, z1
+; CHECK-NEXT:    sqabs z0.h, p0/m, z1.h
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svsqabs_nxv8i16_ptrue_u:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    ptrue p0.h
+; CHECK-2p2-NEXT:    sqabs z0.h, p0/z, z1.h
+; CHECK-2p2-NEXT:    ret
+entry:
+  %pg = call <vscale x 8 x i1> @llvm.aarch64.sve.ptrue.nxv8i1(i32 31)
+  %0 = tail call <vscale x 8 x i16> @llvm.aarch64.sve.sqabs.nxv8i16(<vscale x 8 x i16> poison, <vscale x 8 x i1> %pg, <vscale x 8 x i16> %x)
+  ret <vscale x 8 x i16> %0
+}
+
+define <vscale x 8 x i16> @test_svsqabs_nxv8i16_ptrue(double %z0, <vscale x 8 x i16> %x, <vscale x 8 x i16> %y) {
+; CHECK-LABEL: test_svsqabs_nxv8i16_ptrue:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ptrue p0.h
+; CHECK-NEXT:    movprfx z0, z2
+; CHECK-NEXT:    sqabs z0.h, p0/m, z2.h
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svsqabs_nxv8i16_ptrue:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    ptrue p0.h
+; CHECK-2p2-NEXT:    sqabs z0.h, p0/z, z2.h
+; CHECK-2p2-NEXT:    ret
+entry:
+  %pg = call <vscale x 8 x i1> @llvm.aarch64.sve.ptrue.nxv8i1(i32 31)
+  %0 = tail call <vscale x 8 x i16> @llvm.aarch64.sve.sqabs.nxv8i16(<vscale x 8 x i16> %x, <vscale x 8 x i1> %pg, <vscale x 8 x i16> %y)
+  ret <vscale x 8 x i16> %0
+}
+
+define <vscale x 4 x i32> @test_svsqabs_nxv4i32_ptrue_u(double %z0, <vscale x 4 x i32> %x) {
+; CHECK-LABEL: test_svsqabs_nxv4i32_ptrue_u:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ptrue p0.s
+; CHECK-NEXT:    movprfx z0, z1
+; CHECK-NEXT:    sqabs z0.s, p0/m, z1.s
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svsqabs_nxv4i32_ptrue_u:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    ptrue p0.s
+; CHECK-2p2-NEXT:    sqabs z0.s, p0/z, z1.s
+; CHECK-2p2-NEXT:    ret
+entry:
+  %pg = call <vscale x 4 x i1> @llvm.aarch64.sve.ptrue.nxv4i1(i32 31)
+  %0 = tail call <vscale x 4 x i32> @llvm.aarch64.sve.sqabs.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i1> %pg, <vscale x 4 x i32> %x)
+  ret <vscale x 4 x i32> %0
+}
+
+define <vscale x 4 x i32> @test_svsqabs_nxv4i32_ptrue(double %z0, <vscale x 4 x i32> %x, <vscale x 4 x i32> %y) {
+; CHECK-LABEL: test_svsqabs_nxv4i32_ptrue:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ptrue p0.s
+; CHECK-NEXT:    movprfx z0, z2
+; CHECK-NEXT:    sqabs z0.s, p0/m, z2.s
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svsqabs_nxv4i32_ptrue:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    ptrue p0.s
+; CHECK-2p2-NEXT:    sqabs z0.s, p0/z, z2.s
+; CHECK-2p2-NEXT:    ret
+entry:
+  %pg = call <vscale x 4 x i1> @llvm.aarch64.sve.ptrue.nxv4i1(i32 31)
+  %0 = tail call <vscale x 4 x i32> @llvm.aarch64.sve.sqabs.nxv4i32(<vscale x 4 x i32> %x, <vscale x 4 x i1> %pg, <vscale x 4 x i32> %y)
+  ret <vscale x 4 x i32> %0
+}
+
+define <vscale x 2 x i64> @test_svsqabs_nxv2i64_ptrue_u(double %z0, <vscale x 2 x i64> %x) {
+; CHECK-LABEL: test_svsqabs_nxv2i64_ptrue_u:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ptrue p0.d
+; CHECK-NEXT:    movprfx z0, z1
+; CHECK-NEXT:    sqabs z0.d, p0/m, z1.d
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svsqabs_nxv2i64_ptrue_u:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    ptrue p0.d
+; CHECK-2p2-NEXT:    sqabs z0.d, p0/z, z1.d
+; CHECK-2p2-NEXT:    ret
+entry:
+  %pg = call <vscale x 2 x i1> @llvm.aarch64.sve.ptrue.nxv2i1(i32 31)
+  %0 = tail call <vscale x 2 x i64> @llvm.aarch64.sve.sqabs.nxv2i64(<vscale x 2 x i64> poison, <vscale x 2 x i1> %pg, <vscale x 2 x i64> %x)
+  ret <vscale x 2 x i64> %0
+}
+
+define <vscale x 2 x i64> @test_svsqabs_nxv2i64_ptrue(double %z0, <vscale x 2 x i64> %x, <vscale x 2 x i64> %y) {
+; CHECK-LABEL: test_svsqabs_nxv2i64_ptrue:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ptrue p0.d
+; CHECK-NEXT:    movprfx z0, z2
+; CHECK-NEXT:    sqabs z0.d, p0/m, z2.d
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svsqabs_nxv2i64_ptrue:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    ptrue p0.d
+; CHECK-2p2-NEXT:    sqabs z0.d, p0/z, z2.d
+; CHECK-2p2-NEXT:    ret
+entry:
+  %pg = call <vscale x 2 x i1> @llvm.aarch64.sve.ptrue.nxv2i1(i32 31)
+  %0 = tail call <vscale x 2 x i64> @llvm.aarch64.sve.sqabs.nxv2i64(<vscale x 2 x i64> %x, <vscale x 2 x i1> %pg, <vscale x 2 x i64> %y)
+  ret <vscale x 2 x i64> %0
+}
+
+define <vscale x 16 x i8> @test_svsqneg_nxv16i8_ptrue_u(double %z0, <vscale x 16 x i8> %x) {
+; CHECK-LABEL: test_svsqneg_nxv16i8_ptrue_u:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ptrue p0.b
+; CHECK-NEXT:    movprfx z0, z1
+; CHECK-NEXT:    sqneg z0.b, p0/m, z1.b
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svsqneg_nxv16i8_ptrue_u:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    ptrue p0.b
+; CHECK-2p2-NEXT:    sqneg z0.b, p0/z, z1.b
+; CHECK-2p2-NEXT:    ret
+entry:
+  %pg = call <vscale x 16 x i1> @llvm.aarch64.sve.ptrue.nxv16i1(i32 31)
+  %0 = tail call <vscale x 16 x i8> @llvm.aarch64.sve.sqneg.nxv16i8(<vscale x 16 x i8> poison, <vscale x 16 x i1> %pg, <vscale x 16 x i8> %x)
+  ret <vscale x 16 x i8> %0
+}
+
+define <vscale x 16 x i8> @test_svsqneg_nxv16i8_ptrue(double %z0, <vscale x 16 x i8> %x, <vscale x 16 x i8> %y) {
+; CHECK-LABEL: test_svsqneg_nxv16i8_ptrue:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ptrue p0.b
+; CHECK-NEXT:    movprfx z0, z2
+; CHECK-NEXT:    sqneg z0.b, p0/m, z2.b
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svsqneg_nxv16i8_ptrue:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    ptrue p0.b
+; CHECK-2p2-NEXT:    sqneg z0.b, p0/z, z2.b
+; CHECK-2p2-NEXT:    ret
+entry:
+  %pg = call <vscale x 16 x i1> @llvm.aarch64.sve.ptrue.nxv16i1(i32 31)
+  %0 = tail call <vscale x 16 x i8> @llvm.aarch64.sve.sqneg.nxv16i8(<vscale x 16 x i8> %x, <vscale x 16 x i1> %pg, <vscale x 16 x i8> %y)
+  ret <vscale x 16 x i8> %0
+}
+
+define <vscale x 8 x i16> @test_svsqneg_nxv8i16_ptrue_u(double %z0, <vscale x 8 x i16> %x) {
+; CHECK-LABEL: test_svsqneg_nxv8i16_ptrue_u:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ptrue p0.h
+; CHECK-NEXT:    movprfx z0, z1
+; CHECK-NEXT:    sqneg z0.h, p0/m, z1.h
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svsqneg_nxv8i16_ptrue_u:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    ptrue p0.h
+; CHECK-2p2-NEXT:    sqneg z0.h, p0/z, z1.h
+; CHECK-2p2-NEXT:    ret
+entry:
+  %pg = call <vscale x 8 x i1> @llvm.aarch64.sve.ptrue.nxv8i1(i32 31)
+  %0 = tail call <vscale x 8 x i16> @llvm.aarch64.sve.sqneg.nxv8i16(<vscale x 8 x i16> poison, <vscale x 8 x i1> %pg, <vscale x 8 x i16> %x)
+  ret <vscale x 8 x i16> %0
+}
+
+define <vscale x 8 x i16> @test_svsqneg_nxv8i16_ptrue(double %z0, <vscale x 8 x i16> %x, <vscale x 8 x i16> %y) {
+; CHECK-LABEL: test_svsqneg_nxv8i16_ptrue:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ptrue p0.h
+; CHECK-NEXT:    movprfx z0, z2
+; CHECK-NEXT:    sqneg z0.h, p0/m, z2.h
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svsqneg_nxv8i16_ptrue:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    ptrue p0.h
+; CHECK-2p2-NEXT:    sqneg z0.h, p0/z, z2.h
+; CHECK-2p2-NEXT:    ret
+entry:
+  %pg = call <vscale x 8 x i1> @llvm.aarch64.sve.ptrue.nxv8i1(i32 31)
+  %0 = tail call <vscale x 8 x i16> @llvm.aarch64.sve.sqneg.nxv8i16(<vscale x 8 x i16> %x, <vscale x 8 x i1> %pg, <vscale x 8 x i16> %y)
+  ret <vscale x 8 x i16> %0
+}
+
+define <vscale x 4 x i32> @test_svsqneg_nxv4i32_ptrue_u(double %z0, <vscale x 4 x i32> %x) {
+; CHECK-LABEL: test_svsqneg_nxv4i32_ptrue_u:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ptrue p0.s
+; CHECK-NEXT:    movprfx z0, z1
+; CHECK-NEXT:    sqneg z0.s, p0/m, z1.s
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svsqneg_nxv4i32_ptrue_u:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    ptrue p0.s
+; CHECK-2p2-NEXT:    sqneg z0.s, p0/z, z1.s
+; CHECK-2p2-NEXT:    ret
+entry:
+  %pg = call <vscale x 4 x i1> @llvm.aarch64.sve.ptrue.nxv4i1(i32 31)
+  %0 = tail call <vscale x 4 x i32> @llvm.aarch64.sve.sqneg.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i1> %pg, <vscale x 4 x i32> %x)
+  ret <vscale x 4 x i32> %0
+}
+
+define <vscale x 4 x i32> @test_svsqneg_nxv4i32_ptrue(double %z0, <vscale x 4 x i32> %x, <vscale x 4 x i32> %y) {
+; CHECK-LABEL: test_svsqneg_nxv4i32_ptrue:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ptrue p0.s
+; CHECK-NEXT:    movprfx z0, z2
+; CHECK-NEXT:    sqneg z0.s, p0/m, z2.s
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svsqneg_nxv4i32_ptrue:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    ptrue p0.s
+; CHECK-2p2-NEXT:    sqneg z0.s, p0/z, z2.s
+; CHECK-2p2-NEXT:    ret
+entry:
+  %pg = call <vscale x 4 x i1> @llvm.aarch64.sve.ptrue.nxv4i1(i32 31)
+  %0 = tail call <vscale x 4 x i32> @llvm.aarch64.sve.sqneg.nxv4i32(<vscale x 4 x i32> %x, <vscale x 4 x i1> %pg, <vscale x 4 x i32> %y)
+  ret <vscale x 4 x i32> %0
+}
+
+define <vscale x 2 x i64> @test_svsqneg_nxv2i64_ptrue_u(double %z0, <vscale x 2 x i64> %x) {
+; CHECK-LABEL: test_svsqneg_nxv2i64_ptrue_u:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ptrue p0.d
+; CHECK-NEXT:    movprfx z0, z1
+; CHECK-NEXT:    sqneg z0.d, p0/m, z1.d
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svsqneg_nxv2i64_ptrue_u:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    ptrue p0.d
+; CHECK-2p2-NEXT:    sqneg z0.d, p0/z, z1.d
+; CHECK-2p2-NEXT:    ret
+entry:
+  %pg = call <vscale x 2 x i1> @llvm.aarch64.sve.ptrue.nxv2i1(i32 31)
+  %0 = tail call <vscale x 2 x i64> @llvm.aarch64.sve.sqneg.nxv2i64(<vscale x 2 x i64> poison, <vscale x 2 x i1> %pg, <vscale x 2 x i64> %x)
+  ret <vscale x 2 x i64> %0
+}
+
+define <vscale x 2 x i64> @test_svsqneg_nxv2i64_ptrue(double %z0, <vscale x 2 x i64> %x, <vscale x 2 x i64> %y) {
+; CHECK-LABEL: test_svsqneg_nxv2i64_ptrue:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ptrue p0.d
+; CHECK-NEXT:    movprfx z0, z2
+; CHECK-NEXT:    sqneg z0.d, p0/m, z2.d
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svsqneg_nxv2i64_ptrue:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    ptrue p0.d
+; CHECK-2p2-NEXT:    sqneg z0.d, p0/z, z2.d
+; CHECK-2p2-NEXT:    ret
+entry:
+  %pg = call <vscale x 2 x i1> @llvm.aarch64.sve.ptrue.nxv2i1(i32 31)
+  %0 = tail call <vscale x 2 x i64> @llvm.aarch64.sve.sqneg.nxv2i64(<vscale x 2 x i64> %x, <vscale x 2 x i1> %pg, <vscale x 2 x i64> %y)
+  ret <vscale x 2 x i64> %0
+}
diff --git a/llvm/test/CodeGen/AArch64/zext-shuffle.ll b/llvm/test/CodeGen/AArch64/zext-shuffle.ll
index 2965996ddcb02..20d2071d7fe54 100644
--- a/llvm/test/CodeGen/AArch64/zext-shuffle.ll
+++ b/llvm/test/CodeGen/AArch64/zext-shuffle.ll
@@ -543,3 +543,146 @@ define <8 x double> @uitofp_load_fadd(ptr %p) {
     ret <8 x double> %c
 }
 
+define <4 x i32> @isUndefDeInterleave_b0(<8 x i16> %a, <8 x i16> %b) {
+; CHECK-LABEL: isUndefDeInterleave_b0:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    movi v1.2d, #0x00ffff0000ffff
+; CHECK-NEXT:    uzp1 v0.4s, v0.4s, v0.4s
+; CHECK-NEXT:    and v0.16b, v0.16b, v1.16b
+; CHECK-NEXT:    ret
+  %2 = shufflevector <8 x i16> %a, <8 x i16> poison, <8 x i32> <i32 poison, i32 poison, i32 0, i32 4, i32 poison, i32 poison, i32 poison, i32 poison>
+  %s2 = shufflevector <8 x i16> %2, <8 x i16> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %3 = zext <4 x i16> %s2 to <4 x i32>
+  ret <4 x i32> %3
+}
+
+define <4 x i32> @isUndefDeInterleave_b1(<8 x i16> %a) {
+; CHECK-LABEL: isUndefDeInterleave_b1:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    uzp1 v0.4s, v0.4s, v0.4s
+; CHECK-NEXT:    ushr v0.4s, v0.4s, #16
+; CHECK-NEXT:    ret
+  %2 = shufflevector <8 x i16> %a, <8 x i16> poison, <8 x i32> <i32 poison, i32 poison, i32 1, i32 5, i32 poison, i32 poison, i32 poison, i32 poison>
+  %s2 = shufflevector <8 x i16> %2, <8 x i16> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %3 = zext <4 x i16> %s2 to <4 x i32>
+  ret <4 x i32> %3
+}
+
+define <4 x i32> @isUndefDeInterleave_b2(<8 x i16> %a) {
+; CHECK-LABEL: isUndefDeInterleave_b2:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    movi v1.2d, #0x00ffff0000ffff
+; CHECK-NEXT:    uzp2 v0.4s, v0.4s, v0.4s
+; CHECK-NEXT:    and v0.16b, v0.16b, v1.16b
+; CHECK-NEXT:    ret
+  %2 = shufflevector <8 x i16> %a, <8 x i16> poison, <8 x i32> <i32 poison, i32 poison, i32 2, i32 6, i32 poison, i32 poison, i32 poison, i32 poison>
+  %s2 = shufflevector <8 x i16> %2, <8 x i16> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %3 = zext <4 x i16> %s2 to <4 x i32>
+  ret <4 x i32> %3
+}
+
+define <4 x i32> @isUndefDeInterleave_b3(<8 x i16> %a) {
+; CHECK-LABEL: isUndefDeInterleave_b3:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    uzp2 v0.4s, v0.4s, v0.4s
+; CHECK-NEXT:    ushr v0.4s, v0.4s, #16
+; CHECK-NEXT:    ret
+  %2 = shufflevector <8 x i16> %a, <8 x i16> poison, <8 x i32> <i32 poison, i32 poison, i32 3, i32 7, i32 poison, i32 poison, i32 poison, i32 poison>
+  %s2 = shufflevector <8 x i16> %2, <8 x i16> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %3 = zext <4 x i16> %s2 to <4 x i32>
+  ret <4 x i32> %3
+}
+
+define <4 x i32> @isUndefDeInterleave_t0(<8 x i16> %a, <8 x i16> %b) {
+; CHECK-LABEL: isUndefDeInterleave_t0:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    movi v1.2d, #0x00ffff0000ffff
+; CHECK-NEXT:    uzp1 v0.4s, v0.4s, v0.4s
+; CHECK-NEXT:    and v0.16b, v0.16b, v1.16b
+; CHECK-NEXT:    ret
+  %2 = shufflevector <8 x i16> %a, <8 x i16> poison, <8 x i32> <i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 0, i32 4>
+  %s2 = shufflevector <8 x i16> %2, <8 x i16> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+  %3 = zext <4 x i16> %s2 to <4 x i32>
+  ret <4 x i32> %3
+}
+
+define <4 x i32> @isUndefDeInterleave_t1(<8 x i16> %a) {
+; CHECK-LABEL: isUndefDeInterleave_t1:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    uzp1 v0.4s, v0.4s, v0.4s
+; CHECK-NEXT:    ushr v0.4s, v0.4s, #16
+; CHECK-NEXT:    ret
+  %2 = shufflevector <8 x i16> %a, <8 x i16> poison, <8 x i32> <i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 1, i32 5>
+  %s2 = shufflevector <8 x i16> %2, <8 x i16> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+  %3 = zext <4 x i16> %s2 to <4 x i32>
+  ret <4 x i32> %3
+}
+
+define <4 x i32> @isUndefDeInterleave_t2(<8 x i16> %a) {
+; CHECK-LABEL: isUndefDeInterleave_t2:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    movi v1.2d, #0x00ffff0000ffff
+; CHECK-NEXT:    uzp2 v0.4s, v0.4s, v0.4s
+; CHECK-NEXT:    and v0.16b, v0.16b, v1.16b
+; CHECK-NEXT:    ret
+  %2 = shufflevector <8 x i16> %a, <8 x i16> poison, <8 x i32> <i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 2, i32 6>
+  %s2 = shufflevector <8 x i16> %2, <8 x i16> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+  %3 = zext <4 x i16> %s2 to <4 x i32>
+  ret <4 x i32> %3
+}
+
+define <4 x i32> @isUndefDeInterleave_t3(<8 x i16> %a) {
+; CHECK-LABEL: isUndefDeInterleave_t3:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    uzp2 v0.4s, v0.4s, v0.4s
+; CHECK-NEXT:    ushr v0.4s, v0.4s, #16
+; CHECK-NEXT:    ret
+  %2 = shufflevector <8 x i16> %a, <8 x i16> poison, <8 x i32> <i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 3, i32 7>
+  %s2 = shufflevector <8 x i16> %2, <8 x i16> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+  %3 = zext <4 x i16> %s2 to <4 x i32>
+  ret <4 x i32> %3
+}
+
+define <4 x i32> @isUndefDeInterleave_b0_bad(<8 x i16> %a, <8 x i16> %b) {
+; CHECK-LABEL: isUndefDeInterleave_b0_bad:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    adrp x8, .LCPI40_0
+; CHECK-NEXT:    ldr q1, [x8, :lo12:.LCPI40_0]
+; CHECK-NEXT:    tbl v0.16b, { v0.16b }, v1.16b
+; CHECK-NEXT:    ushll v0.4s, v0.4h, #0
+; CHECK-NEXT:    ret
+  %2 = shufflevector <8 x i16> %a, <8 x i16> poison, <8 x i32> <i32 4, i32 4, i32 0, i32 4, i32 poison, i32 poison, i32 poison, i32 poison>
+  %s2 = shufflevector <8 x i16> %2, <8 x i16> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %3 = zext <4 x i16> %s2 to <4 x i32>
+  ret <4 x i32> %3
+}
+
+define <4 x i32> @isUndefDeInterleave_t1_bad(<8 x i16> %a) {
+; CHECK-LABEL: isUndefDeInterleave_t1_bad:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    adrp x8, .LCPI41_0
+; CHECK-NEXT:    ldr q1, [x8, :lo12:.LCPI41_0]
+; CHECK-NEXT:    tbl v0.16b, { v0.16b }, v1.16b
+; CHECK-NEXT:    ushll2 v0.4s, v0.8h, #0
+; CHECK-NEXT:    ret
+  %2 = shufflevector <8 x i16> %a, <8 x i16> poison, <8 x i32> <i32 poison, i32 poison, i32 poison, i32 poison, i32 4, i32 4, i32 1, i32 5>
+  %s2 = shufflevector <8 x i16> %2, <8 x i16> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+  %3 = zext <4 x i16> %s2 to <4 x i32>
+  ret <4 x i32> %3
+}
+
+define i16 @undeftop(<8 x i16> %0) {
+; CHECK-LABEL: undeftop:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    dup v0.8h, v0.h[4]
+; CHECK-NEXT:    uaddl v0.4s, v0.4h, v0.4h
+; CHECK-NEXT:    xtn v0.4h, v0.4s
+; CHECK-NEXT:    umov w0, v0.h[0]
+; CHECK-NEXT:    ret
+  %2 = shufflevector <8 x i16> %0, <8 x i16> zeroinitializer, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 9, i32 7, i32 5, i32 3>
+  %3 = zext <8 x i16> %2 to <8 x i64>
+  %new0 = add <8 x i64> %3, %3
+  %last = trunc <8 x i64> %new0 to <8 x i16>
+  %4 = extractelement <8 x i16> %last, i32 0
+  ret i16 %4
+}
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/assert-align.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/assert-align.ll
index c477732e5cd59..a91e41e78da8e 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/assert-align.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/assert-align.ll
@@ -15,12 +15,12 @@ define ptr addrspace(1) @call_assert_align() {
 ; CHECK-NEXT:    v_writelane_b32 v40, s16, 2
 ; CHECK-NEXT:    s_addk_i32 s32, 0x400
 ; CHECK-NEXT:    v_writelane_b32 v40, s30, 0
-; CHECK-NEXT:    v_mov_b32_e32 v0, 0
-; CHECK-NEXT:    v_mov_b32_e32 v1, 0
-; CHECK-NEXT:    v_writelane_b32 v40, s31, 1
 ; CHECK-NEXT:    s_getpc_b64 s[16:17]
 ; CHECK-NEXT:    s_add_u32 s16, s16, ext@rel32@lo+4
 ; CHECK-NEXT:    s_addc_u32 s17, s17, ext@rel32@hi+12
+; CHECK-NEXT:    v_mov_b32_e32 v0, 0
+; CHECK-NEXT:    v_mov_b32_e32 v1, 0
+; CHECK-NEXT:    v_writelane_b32 v40, s31, 1
 ; CHECK-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; CHECK-NEXT:    v_mov_b32_e32 v2, 0
 ; CHECK-NEXT:    global_store_dword v[0:1], v2, off
@@ -45,11 +45,11 @@ define ptr addrspace(1) @tail_call_assert_align() {
 ; CHECK-LABEL: tail_call_assert_align:
 ; CHECK:       ; %bb.0: ; %entry
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    v_mov_b32_e32 v0, 0
-; CHECK-NEXT:    v_mov_b32_e32 v1, 0
 ; CHECK-NEXT:    s_getpc_b64 s[16:17]
 ; CHECK-NEXT:    s_add_u32 s16, s16, ext@rel32@lo+4
 ; CHECK-NEXT:    s_addc_u32 s17, s17, ext@rel32@hi+12
+; CHECK-NEXT:    v_mov_b32_e32 v0, 0
+; CHECK-NEXT:    v_mov_b32_e32 v1, 0
 ; CHECK-NEXT:    s_setpc_b64 s[16:17]
 entry:
   %call = tail call align 4 ptr addrspace(1) @ext(ptr addrspace(1) null)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/atomicrmw_fmax.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/atomicrmw_fmax.ll
index 23f24a9dc9982..424388a30e99b 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/atomicrmw_fmax.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/atomicrmw_fmax.ll
@@ -624,7 +624,6 @@ define double @global_agent_atomic_fmax_ret_f64__amdgpu_no_fine_grained_memory(p
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX12-NEXT:    v_dual_mov_b32 v0, v4 :: v_dual_mov_b32 v1, v5
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: global_agent_atomic_fmax_ret_f64__amdgpu_no_fine_grained_memory:
@@ -780,7 +779,6 @@ define void @global_agent_atomic_fmax_noret_f64__amdgpu_no_fine_grained_memory(p
 ; GFX12-NEXT:    s_cbranch_execnz .LBB7_1
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: global_agent_atomic_fmax_noret_f64__amdgpu_no_fine_grained_memory:
@@ -1212,7 +1210,6 @@ define double @flat_agent_atomic_fmax_ret_f64__amdgpu_no_fine_grained_memory(ptr
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX12-NEXT:    v_dual_mov_b32 v0, v4 :: v_dual_mov_b32 v1, v5
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: flat_agent_atomic_fmax_ret_f64__amdgpu_no_fine_grained_memory:
@@ -1366,7 +1363,6 @@ define void @flat_agent_atomic_fmax_noret_f64__amdgpu_no_fine_grained_memory(ptr
 ; GFX12-NEXT:    s_cbranch_execnz .LBB11_1
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: flat_agent_atomic_fmax_noret_f64__amdgpu_no_fine_grained_memory:
@@ -1822,14 +1818,12 @@ define double @buffer_fat_ptr_agent_atomic_fmax_ret_f64__amdgpu_no_fine_grained_
 ; GFX12-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-NEXT:    global_inv scope:SCOPE_DEV
 ; GFX12-NEXT:    v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[9:10]
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_or_b32 s4, vcc_lo, s4
 ; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s4
 ; GFX12-NEXT:    s_cbranch_execnz .LBB14_1
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s4
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_f64__amdgpu_no_fine_grained_memory:
@@ -1994,14 +1988,12 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_f64__amdgpu_no_fine_grained_
 ; GFX12-NEXT:    global_inv scope:SCOPE_DEV
 ; GFX12-NEXT:    v_cmp_eq_u64_e32 vcc_lo, v[7:8], v[2:3]
 ; GFX12-NEXT:    v_dual_mov_b32 v2, v7 :: v_dual_mov_b32 v3, v8
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_or_b32 s4, vcc_lo, s4
 ; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s4
 ; GFX12-NEXT:    s_cbranch_execnz .LBB15_1
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s4
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: buffer_fat_ptr_agent_atomic_fmax_noret_f64__amdgpu_no_fine_grained_memory:
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/atomicrmw_fmin.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/atomicrmw_fmin.ll
index 11024b0a88d6b..b52a39f1a55c8 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/atomicrmw_fmin.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/atomicrmw_fmin.ll
@@ -624,7 +624,6 @@ define double @global_agent_atomic_fmin_ret_f64__amdgpu_no_fine_grained_memory(p
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX12-NEXT:    v_dual_mov_b32 v0, v4 :: v_dual_mov_b32 v1, v5
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: global_agent_atomic_fmin_ret_f64__amdgpu_no_fine_grained_memory:
@@ -780,7 +779,6 @@ define void @global_agent_atomic_fmin_noret_f64__amdgpu_no_fine_grained_memory(p
 ; GFX12-NEXT:    s_cbranch_execnz .LBB7_1
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: global_agent_atomic_fmin_noret_f64__amdgpu_no_fine_grained_memory:
@@ -1212,7 +1210,6 @@ define double @flat_agent_atomic_fmin_ret_f64__amdgpu_no_fine_grained_memory(ptr
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX12-NEXT:    v_dual_mov_b32 v0, v4 :: v_dual_mov_b32 v1, v5
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: flat_agent_atomic_fmin_ret_f64__amdgpu_no_fine_grained_memory:
@@ -1366,7 +1363,6 @@ define void @flat_agent_atomic_fmin_noret_f64__amdgpu_no_fine_grained_memory(ptr
 ; GFX12-NEXT:    s_cbranch_execnz .LBB11_1
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: flat_agent_atomic_fmin_noret_f64__amdgpu_no_fine_grained_memory:
@@ -1822,14 +1818,12 @@ define double @buffer_fat_ptr_agent_atomic_fmin_ret_f64__amdgpu_no_fine_grained_
 ; GFX12-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-NEXT:    global_inv scope:SCOPE_DEV
 ; GFX12-NEXT:    v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[9:10]
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_or_b32 s4, vcc_lo, s4
 ; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s4
 ; GFX12-NEXT:    s_cbranch_execnz .LBB14_1
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s4
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f64__amdgpu_no_fine_grained_memory:
@@ -1994,14 +1988,12 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_f64__amdgpu_no_fine_grained_
 ; GFX12-NEXT:    global_inv scope:SCOPE_DEV
 ; GFX12-NEXT:    v_cmp_eq_u64_e32 vcc_lo, v[7:8], v[2:3]
 ; GFX12-NEXT:    v_dual_mov_b32 v2, v7 :: v_dual_mov_b32 v3, v8
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_or_b32 s4, vcc_lo, s4
 ; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s4
 ; GFX12-NEXT:    s_cbranch_execnz .LBB15_1
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s4
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_f64__amdgpu_no_fine_grained_memory:
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/atomicrmw_uinc_wrap.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/atomicrmw_uinc_wrap.ll
index d62da6921b347..e1397e7331d3c 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/atomicrmw_uinc_wrap.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/atomicrmw_uinc_wrap.ll
@@ -2853,6 +2853,7 @@ define amdgpu_kernel void @flat_atomic_inc_ret_i32_offset_addr64(ptr %out, ptr %
 ; GFX12-NEXT:    global_inv scope:SCOPE_DEV
 ; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
 ; GFX12-NEXT:    v_add_co_u32 v0, vcc_lo, v0, v2
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2)
 ; GFX12-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
 ; GFX12-NEXT:    flat_store_b32 v[0:1], v3
@@ -3840,6 +3841,7 @@ define amdgpu_kernel void @flat_atomic_inc_ret_i64_offset_addr64(ptr %out, ptr %
 ; GFX12-NEXT:    global_inv scope:SCOPE_DEV
 ; GFX12-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
 ; GFX12-NEXT:    v_add_co_u32 v2, vcc_lo, v2, v4
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2)
 ; GFX12-NEXT:    v_add_co_ci_u32_e32 v3, vcc_lo, 0, v3, vcc_lo
 ; GFX12-NEXT:    flat_store_b64 v[2:3], v[0:1]
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/call-outgoing-stack-args.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/call-outgoing-stack-args.ll
index 410d3b1bb7062..c99424fe2f7d9 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/call-outgoing-stack-args.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/call-outgoing-stack-args.ll
@@ -44,8 +44,8 @@ define amdgpu_kernel void @kernel_caller_stack() {
 ; FLATSCR-NEXT:    scratch_store_dword off, v0, s0
 ; FLATSCR-NEXT:    s_add_u32 s0, s32, 12
 ; FLATSCR-NEXT:    v_mov_b32_e32 v0, 11
-; FLATSCR-NEXT:    scratch_store_dword off, v0, s0
 ; FLATSCR-NEXT:    s_add_u32 s2, s32, 16
+; FLATSCR-NEXT:    scratch_store_dword off, v0, s0
 ; FLATSCR-NEXT:    v_mov_b32_e32 v0, 12
 ; FLATSCR-NEXT:    s_getpc_b64 s[0:1]
 ; FLATSCR-NEXT:    s_add_u32 s0, s0, external_void_func_v16i32_v16i32_v4i32@rel32@lo+4
@@ -239,11 +239,11 @@ define void @func_caller_stack() {
 ; MUBUF-NEXT:    v_writelane_b32 v40, s30, 0
 ; MUBUF-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:12
 ; MUBUF-NEXT:    v_mov_b32_e32 v0, 12
-; MUBUF-NEXT:    v_writelane_b32 v40, s31, 1
-; MUBUF-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:16
 ; MUBUF-NEXT:    s_getpc_b64 s[4:5]
 ; MUBUF-NEXT:    s_add_u32 s4, s4, external_void_func_v16i32_v16i32_v4i32@rel32@lo+4
 ; MUBUF-NEXT:    s_addc_u32 s5, s5, external_void_func_v16i32_v16i32_v4i32@rel32@hi+12
+; MUBUF-NEXT:    v_writelane_b32 v40, s31, 1
+; MUBUF-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:16
 ; MUBUF-NEXT:    s_swappc_b64 s[30:31], s[4:5]
 ; MUBUF-NEXT:    v_readlane_b32 s31, v40, 1
 ; MUBUF-NEXT:    v_readlane_b32 s30, v40, 0
@@ -274,15 +274,15 @@ define void @func_caller_stack() {
 ; FLATSCR-NEXT:    scratch_store_dword off, v0, s0
 ; FLATSCR-NEXT:    s_add_u32 s0, s32, 12
 ; FLATSCR-NEXT:    v_mov_b32_e32 v0, 11
-; FLATSCR-NEXT:    v_writelane_b32 v40, s30, 0
 ; FLATSCR-NEXT:    scratch_store_dword off, v0, s0
 ; FLATSCR-NEXT:    s_add_u32 s0, s32, 16
 ; FLATSCR-NEXT:    v_mov_b32_e32 v0, 12
-; FLATSCR-NEXT:    v_writelane_b32 v40, s31, 1
+; FLATSCR-NEXT:    v_writelane_b32 v40, s30, 0
 ; FLATSCR-NEXT:    scratch_store_dword off, v0, s0
 ; FLATSCR-NEXT:    s_getpc_b64 s[0:1]
 ; FLATSCR-NEXT:    s_add_u32 s0, s0, external_void_func_v16i32_v16i32_v4i32@rel32@lo+4
 ; FLATSCR-NEXT:    s_addc_u32 s1, s1, external_void_func_v16i32_v16i32_v4i32@rel32@hi+12
+; FLATSCR-NEXT:    v_writelane_b32 v40, s31, 1
 ; FLATSCR-NEXT:    s_swappc_b64 s[30:31], s[0:1]
 ; FLATSCR-NEXT:    v_readlane_b32 s31, v40, 1
 ; FLATSCR-NEXT:    v_readlane_b32 s30, v40, 0
@@ -312,10 +312,10 @@ define void @func_caller_byval(ptr addrspace(5) %argptr) {
 ; MUBUF-NEXT:    s_addk_i32 s32, 0x400
 ; MUBUF-NEXT:    v_writelane_b32 v40, s4, 2
 ; MUBUF-NEXT:    v_writelane_b32 v40, s30, 0
-; MUBUF-NEXT:    v_writelane_b32 v40, s31, 1
 ; MUBUF-NEXT:    s_getpc_b64 s[4:5]
 ; MUBUF-NEXT:    s_add_u32 s4, s4, external_void_func_byval@rel32@lo+4
 ; MUBUF-NEXT:    s_addc_u32 s5, s5, external_void_func_byval@rel32@hi+12
+; MUBUF-NEXT:    v_writelane_b32 v40, s31, 1
 ; MUBUF-NEXT:    s_waitcnt vmcnt(1)
 ; MUBUF-NEXT:    buffer_store_dword v1, off, s[0:3], s32
 ; MUBUF-NEXT:    s_waitcnt vmcnt(1)
@@ -394,8 +394,8 @@ define void @func_caller_byval(ptr addrspace(5) %argptr) {
 ; FLATSCR-NEXT:    v_add_u32_e32 v3, 8, v0
 ; FLATSCR-NEXT:    v_writelane_b32 v40, s0, 2
 ; FLATSCR-NEXT:    s_add_u32 s0, s32, 8
-; FLATSCR-NEXT:    v_writelane_b32 v40, s30, 0
 ; FLATSCR-NEXT:    s_add_u32 s2, s32, 56
+; FLATSCR-NEXT:    v_writelane_b32 v40, s30, 0
 ; FLATSCR-NEXT:    v_writelane_b32 v40, s31, 1
 ; FLATSCR-NEXT:    s_waitcnt vmcnt(0)
 ; FLATSCR-NEXT:    scratch_store_dwordx2 off, v[1:2], s32
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/divergence-divergent-i1-used-outside-loop.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/divergence-divergent-i1-used-outside-loop.ll
index 935200d595307..91e16d91ddd15 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/divergence-divergent-i1-used-outside-loop.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/divergence-divergent-i1-used-outside-loop.ll
@@ -191,9 +191,9 @@ define void @divergent_i1_xor_used_outside_loop_larger_loop_body(i32 %num.elts,
 ; GFX10-LABEL: divergent_i1_xor_used_outside_loop_larger_loop_body:
 ; GFX10:       ; %bb.0: ; %entry
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
 ; GFX10-NEXT:    s_mov_b32 s5, 0
 ; GFX10-NEXT:    s_mov_b32 s6, -1
-; GFX10-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
 ; GFX10-NEXT:    s_and_saveexec_b32 s4, vcc_lo
 ; GFX10-NEXT:    s_cbranch_execz .LBB3_6
 ; GFX10-NEXT:  ; %bb.1: ; %loop.start.preheader
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/extractelement-stack-lower.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/extractelement-stack-lower.ll
index 43f3dcc86f426..a948446aceff1 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/extractelement-stack-lower.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/extractelement-stack-lower.ll
@@ -28,9 +28,11 @@ define i32 @v_extract_v64i32_varidx(ptr addrspace(1) %ptr, i32 %idx) {
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_lshlrev_b32_e32 v2, 2, v2
 ; GFX12-NEXT:    v_add_co_u32 v0, vcc_lo, v0, v2
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
 ; GFX12-NEXT:    global_load_b32 v0, v[0:1], off
 ; GFX12-NEXT:    s_wait_loadcnt 0x0
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
   %vec = load <64 x i32>, ptr addrspace(1) %ptr
   %elt = extractelement <64 x i32> %vec, i32 %idx
@@ -60,9 +62,11 @@ define i16 @v_extract_v128i16_varidx(ptr addrspace(1) %ptr, i32 %idx) {
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_lshlrev_b32_e32 v2, 1, v2
 ; GFX12-NEXT:    v_add_co_u32 v0, vcc_lo, v0, v2
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
 ; GFX12-NEXT:    global_load_u16 v0, v[0:1], off
 ; GFX12-NEXT:    s_wait_loadcnt 0x0
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
   %vec = load <128 x i16>, ptr addrspace(1) %ptr
   %elt = extractelement <128 x i16> %vec, i32 %idx
@@ -92,9 +96,11 @@ define i64 @v_extract_v32i64_varidx(ptr addrspace(1) %ptr, i32 %idx) {
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_lshlrev_b32_e32 v2, 3, v2
 ; GFX12-NEXT:    v_add_co_u32 v0, vcc_lo, v0, v2
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
 ; GFX12-NEXT:    global_load_b64 v[0:1], v[0:1], off
 ; GFX12-NEXT:    s_wait_loadcnt 0x0
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
   %vec = load <32 x i64>, ptr addrspace(1) %ptr
   %elt = extractelement <32 x i64> %vec, i32 %idx
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-amdgcn.fcmp.constants.w32.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-amdgcn.fcmp.constants.w32.mir
index cdb67caea12cf..49383135ab0c5 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-amdgcn.fcmp.constants.w32.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-amdgcn.fcmp.constants.w32.mir
@@ -17,11 +17,9 @@ body: |
     ; GFX11-TRUE16-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; GFX11-TRUE16-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
     ; GFX11-TRUE16-NEXT: [[V_CVT_F16_F32_t16_e64_:%[0-9]+]]:vgpr_16 = nofpexcept V_CVT_F16_F32_t16_e64 0, [[COPY]], 0, 0, 0, implicit $mode, implicit $exec
-    ; GFX11-TRUE16-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY [[V_CVT_F16_F32_t16_e64_]]
     ; GFX11-TRUE16-NEXT: [[V_CVT_F16_F32_t16_e64_1:%[0-9]+]]:vgpr_16 = nofpexcept V_CVT_F16_F32_t16_e64 0, [[COPY1]], 0, 0, 0, implicit $mode, implicit $exec
-    ; GFX11-TRUE16-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY [[V_CVT_F16_F32_t16_e64_1]]
-    ; GFX11-TRUE16-NEXT: [[V_CMP_F_F16_fake16_e64_:%[0-9]+]]:sreg_32 = V_CMP_F_F16_fake16_e64 0, [[COPY2]], 0, [[COPY3]], 0, implicit $mode, implicit $exec
-    ; GFX11-TRUE16-NEXT: S_ENDPGM 0, implicit [[V_CMP_F_F16_fake16_e64_]]
+    ; GFX11-TRUE16-NEXT: [[V_CMP_F_F16_t16_e64_:%[0-9]+]]:sreg_32 = V_CMP_F_F16_t16_e64 0, [[V_CVT_F16_F32_t16_e64_]], 0, [[V_CVT_F16_F32_t16_e64_1]], 0, 0, implicit $mode, implicit $exec
+    ; GFX11-TRUE16-NEXT: S_ENDPGM 0, implicit [[V_CMP_F_F16_t16_e64_]]
     ;
     ; GFX11-FAKE16-LABEL: name: fcmp_false_f16
     ; GFX11-FAKE16: liveins: $vgpr0, $vgpr1
@@ -55,11 +53,9 @@ body: |
     ; GFX11-TRUE16-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; GFX11-TRUE16-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
     ; GFX11-TRUE16-NEXT: [[V_CVT_F16_F32_t16_e64_:%[0-9]+]]:vgpr_16 = nofpexcept V_CVT_F16_F32_t16_e64 0, [[COPY]], 0, 0, 0, implicit $mode, implicit $exec
-    ; GFX11-TRUE16-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY [[V_CVT_F16_F32_t16_e64_]]
     ; GFX11-TRUE16-NEXT: [[V_CVT_F16_F32_t16_e64_1:%[0-9]+]]:vgpr_16 = nofpexcept V_CVT_F16_F32_t16_e64 0, [[COPY1]], 0, 0, 0, implicit $mode, implicit $exec
-    ; GFX11-TRUE16-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY [[V_CVT_F16_F32_t16_e64_1]]
-    ; GFX11-TRUE16-NEXT: [[V_CMP_TRU_F16_fake16_e64_:%[0-9]+]]:sreg_32 = V_CMP_TRU_F16_fake16_e64 0, [[COPY2]], 0, [[COPY3]], 0, implicit $mode, implicit $exec
-    ; GFX11-TRUE16-NEXT: S_ENDPGM 0, implicit [[V_CMP_TRU_F16_fake16_e64_]]
+    ; GFX11-TRUE16-NEXT: [[V_CMP_TRU_F16_t16_e64_:%[0-9]+]]:sreg_32 = V_CMP_TRU_F16_t16_e64 0, [[V_CVT_F16_F32_t16_e64_]], 0, [[V_CVT_F16_F32_t16_e64_1]], 0, 0, implicit $mode, implicit $exec
+    ; GFX11-TRUE16-NEXT: S_ENDPGM 0, implicit [[V_CMP_TRU_F16_t16_e64_]]
     ;
     ; GFX11-FAKE16-LABEL: name: fcmp_true_f16
     ; GFX11-FAKE16: liveins: $vgpr0, $vgpr1
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-amdgcn.fcmp.constants.w64.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-amdgcn.fcmp.constants.w64.mir
index ed811d37c3d0f..828eb5d3fb40a 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-amdgcn.fcmp.constants.w64.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-amdgcn.fcmp.constants.w64.mir
@@ -17,11 +17,9 @@ body: |
     ; GFX11-TRUE16-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; GFX11-TRUE16-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
     ; GFX11-TRUE16-NEXT: [[V_CVT_F16_F32_t16_e64_:%[0-9]+]]:vgpr_16 = nofpexcept V_CVT_F16_F32_t16_e64 0, [[COPY]], 0, 0, 0, implicit $mode, implicit $exec
-    ; GFX11-TRUE16-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY [[V_CVT_F16_F32_t16_e64_]]
     ; GFX11-TRUE16-NEXT: [[V_CVT_F16_F32_t16_e64_1:%[0-9]+]]:vgpr_16 = nofpexcept V_CVT_F16_F32_t16_e64 0, [[COPY1]], 0, 0, 0, implicit $mode, implicit $exec
-    ; GFX11-TRUE16-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY [[V_CVT_F16_F32_t16_e64_1]]
-    ; GFX11-TRUE16-NEXT: [[V_CMP_F_F16_fake16_e64_:%[0-9]+]]:sreg_64 = V_CMP_F_F16_fake16_e64 0, [[COPY2]], 0, [[COPY3]], 0, implicit $mode, implicit $exec
-    ; GFX11-TRUE16-NEXT: S_ENDPGM 0, implicit [[V_CMP_F_F16_fake16_e64_]]
+    ; GFX11-TRUE16-NEXT: [[V_CMP_F_F16_t16_e64_:%[0-9]+]]:sreg_64 = V_CMP_F_F16_t16_e64 0, [[V_CVT_F16_F32_t16_e64_]], 0, [[V_CVT_F16_F32_t16_e64_1]], 0, 0, implicit $mode, implicit $exec
+    ; GFX11-TRUE16-NEXT: S_ENDPGM 0, implicit [[V_CMP_F_F16_t16_e64_]]
     ;
     ; GFX11-FAKE16-LABEL: name: fcmp_false_f16
     ; GFX11-FAKE16: liveins: $vgpr0, $vgpr1
@@ -55,11 +53,9 @@ body: |
     ; GFX11-TRUE16-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; GFX11-TRUE16-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
     ; GFX11-TRUE16-NEXT: [[V_CVT_F16_F32_t16_e64_:%[0-9]+]]:vgpr_16 = nofpexcept V_CVT_F16_F32_t16_e64 0, [[COPY]], 0, 0, 0, implicit $mode, implicit $exec
-    ; GFX11-TRUE16-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY [[V_CVT_F16_F32_t16_e64_]]
     ; GFX11-TRUE16-NEXT: [[V_CVT_F16_F32_t16_e64_1:%[0-9]+]]:vgpr_16 = nofpexcept V_CVT_F16_F32_t16_e64 0, [[COPY1]], 0, 0, 0, implicit $mode, implicit $exec
-    ; GFX11-TRUE16-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY [[V_CVT_F16_F32_t16_e64_1]]
-    ; GFX11-TRUE16-NEXT: [[V_CMP_TRU_F16_fake16_e64_:%[0-9]+]]:sreg_64 = V_CMP_TRU_F16_fake16_e64 0, [[COPY2]], 0, [[COPY3]], 0, implicit $mode, implicit $exec
-    ; GFX11-TRUE16-NEXT: S_ENDPGM 0, implicit [[V_CMP_TRU_F16_fake16_e64_]]
+    ; GFX11-TRUE16-NEXT: [[V_CMP_TRU_F16_t16_e64_:%[0-9]+]]:sreg_64 = V_CMP_TRU_F16_t16_e64 0, [[V_CVT_F16_F32_t16_e64_]], 0, [[V_CVT_F16_F32_t16_e64_1]], 0, 0, implicit $mode, implicit $exec
+    ; GFX11-TRUE16-NEXT: S_ENDPGM 0, implicit [[V_CMP_TRU_F16_t16_e64_]]
     ;
     ; GFX11-FAKE16-LABEL: name: fcmp_true_f16
     ; GFX11-FAKE16: liveins: $vgpr0, $vgpr1
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-fcmp.s16.gfx11plus-fake16.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-fcmp.s16.gfx11plus-fake16.mir
new file mode 100644
index 0000000000000..23da26d96b629
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-fcmp.s16.gfx11plus-fake16.mir
@@ -0,0 +1,374 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -mtriple=amdgcn -mcpu=gfx1100 -run-pass=instruction-select -mattr=-real-true16 -verify-machineinstrs -global-isel-abort=0 -o - %s  | FileCheck -check-prefix=GFX11 %s
+
+---
+name: fcmp_false_s16_vv
+legalized: true
+regBankSelected: true
+
+body: |
+  bb.0:
+    liveins: $vgpr0, $vgpr1
+    ; GFX11-LABEL: name: fcmp_false_s16_vv
+    ; GFX11: liveins: $vgpr0, $vgpr1
+    ; GFX11-NEXT: {{  $}}
+    ; GFX11-NEXT: [[COPY:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
+    ; GFX11-NEXT: [[COPY1:%[0-9]+]]:vgpr(s32) = COPY $vgpr1
+    ; GFX11-NEXT: [[TRUNC:%[0-9]+]]:vgpr(s16) = G_TRUNC [[COPY]](s32)
+    ; GFX11-NEXT: [[TRUNC1:%[0-9]+]]:vgpr(s16) = G_TRUNC [[COPY1]](s32)
+    ; GFX11-NEXT: [[FCMP:%[0-9]+]]:vcc(s1) = G_FCMP floatpred(false), [[TRUNC]](s16), [[TRUNC1]]
+    ; GFX11-NEXT: S_ENDPGM 0, implicit [[FCMP]](s1)
+    %0:vgpr(s32) = COPY $vgpr0
+    %1:vgpr(s32) = COPY $vgpr1
+    %2:vgpr(s16) = G_TRUNC %0
+    %3:vgpr(s16) = G_TRUNC %1
+    %4:vcc(s1) = G_FCMP floatpred(false), %2, %3
+    S_ENDPGM 0, implicit %4
+...
+
+---
+name: fcmp_oeq_s16_vv
+legalized: true
+regBankSelected: true
+
+body: |
+  bb.0:
+    liveins: $vgpr0, $vgpr1
+    ; GFX11-LABEL: name: fcmp_oeq_s16_vv
+    ; GFX11: liveins: $vgpr0, $vgpr1
+    ; GFX11-NEXT: {{  $}}
+    ; GFX11-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+    ; GFX11-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+    ; GFX11-NEXT: [[V_CMP_EQ_F16_fake16_e64_:%[0-9]+]]:sreg_32_xm0_xexec = nofpexcept V_CMP_EQ_F16_fake16_e64 0, [[COPY]], 0, [[COPY1]], 0, implicit $mode, implicit $exec
+    ; GFX11-NEXT: S_ENDPGM 0, implicit [[V_CMP_EQ_F16_fake16_e64_]]
+    %0:vgpr(s32) = COPY $vgpr0
+    %1:vgpr(s32) = COPY $vgpr1
+    %2:vgpr(s16) = G_TRUNC %0
+    %3:vgpr(s16) = G_TRUNC %1
+    %4:vcc(s1) = G_FCMP floatpred(oeq), %2, %3
+    S_ENDPGM 0, implicit %4
+...
+
+---
+name: fcmp_ogt_s16_vv
+legalized: true
+regBankSelected: true
+
+body: |
+  bb.0:
+    liveins: $vgpr0, $vgpr1
+    ; GFX11-LABEL: name: fcmp_ogt_s16_vv
+    ; GFX11: liveins: $vgpr0, $vgpr1
+    ; GFX11-NEXT: {{  $}}
+    ; GFX11-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+    ; GFX11-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+    ; GFX11-NEXT: [[V_CMP_GT_F16_fake16_e64_:%[0-9]+]]:sreg_32_xm0_xexec = nofpexcept V_CMP_GT_F16_fake16_e64 0, [[COPY]], 0, [[COPY1]], 0, implicit $mode, implicit $exec
+    ; GFX11-NEXT: S_ENDPGM 0, implicit [[V_CMP_GT_F16_fake16_e64_]]
+    %0:vgpr(s32) = COPY $vgpr0
+    %1:vgpr(s32) = COPY $vgpr1
+    %2:vgpr(s16) = G_TRUNC %0
+    %3:vgpr(s16) = G_TRUNC %1
+    %4:vcc(s1) = G_FCMP floatpred(ogt), %2, %3
+    S_ENDPGM 0, implicit %4
+...
+
+---
+name: fcmp_oge_s16_vv
+legalized: true
+regBankSelected: true
+
+body: |
+  bb.0:
+    liveins: $vgpr0, $vgpr1
+    ; GFX11-LABEL: name: fcmp_oge_s16_vv
+    ; GFX11: liveins: $vgpr0, $vgpr1
+    ; GFX11-NEXT: {{  $}}
+    ; GFX11-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+    ; GFX11-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+    ; GFX11-NEXT: [[V_CMP_GE_F16_fake16_e64_:%[0-9]+]]:sreg_32_xm0_xexec = nofpexcept V_CMP_GE_F16_fake16_e64 0, [[COPY]], 0, [[COPY1]], 0, implicit $mode, implicit $exec
+    ; GFX11-NEXT: S_ENDPGM 0, implicit [[V_CMP_GE_F16_fake16_e64_]]
+    %0:vgpr(s32) = COPY $vgpr0
+    %1:vgpr(s32) = COPY $vgpr1
+    %2:vgpr(s16) = G_TRUNC %0
+    %3:vgpr(s16) = G_TRUNC %1
+    %4:vcc(s1) = G_FCMP floatpred(oge), %2, %3
+    S_ENDPGM 0, implicit %4
+...
+
+---
+name: fcmp_olt_s16_vv
+legalized: true
+regBankSelected: true
+
+body: |
+  bb.0:
+    liveins: $vgpr0, $vgpr1
+    ; GFX11-LABEL: name: fcmp_olt_s16_vv
+    ; GFX11: liveins: $vgpr0, $vgpr1
+    ; GFX11-NEXT: {{  $}}
+    ; GFX11-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+    ; GFX11-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+    ; GFX11-NEXT: [[V_CMP_LT_F16_fake16_e64_:%[0-9]+]]:sreg_32_xm0_xexec = nofpexcept V_CMP_LT_F16_fake16_e64 0, [[COPY]], 0, [[COPY1]], 0, implicit $mode, implicit $exec
+    ; GFX11-NEXT: S_ENDPGM 0, implicit [[V_CMP_LT_F16_fake16_e64_]]
+    %0:vgpr(s32) = COPY $vgpr0
+    %1:vgpr(s32) = COPY $vgpr1
+    %2:vgpr(s16) = G_TRUNC %0
+    %3:vgpr(s16) = G_TRUNC %1
+    %4:vcc(s1) = G_FCMP floatpred(olt), %2, %3
+    S_ENDPGM 0, implicit %4
+...
+
+---
+name: fcmp_ole_s16_vv
+legalized: true
+regBankSelected: true
+
+body: |
+  bb.0:
+    liveins: $vgpr0, $vgpr1
+    ; GFX11-LABEL: name: fcmp_ole_s16_vv
+    ; GFX11: liveins: $vgpr0, $vgpr1
+    ; GFX11-NEXT: {{  $}}
+    ; GFX11-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+    ; GFX11-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+    ; GFX11-NEXT: [[V_CMP_LE_F16_fake16_e64_:%[0-9]+]]:sreg_32_xm0_xexec = nofpexcept V_CMP_LE_F16_fake16_e64 0, [[COPY]], 0, [[COPY1]], 0, implicit $mode, implicit $exec
+    ; GFX11-NEXT: S_ENDPGM 0, implicit [[V_CMP_LE_F16_fake16_e64_]]
+    %0:vgpr(s32) = COPY $vgpr0
+    %1:vgpr(s32) = COPY $vgpr1
+    %2:vgpr(s16) = G_TRUNC %0
+    %3:vgpr(s16) = G_TRUNC %1
+    %4:vcc(s1) = G_FCMP floatpred(ole), %2, %3
+    S_ENDPGM 0, implicit %4
+...
+---
+name: fcmp_one_s16_vv
+legalized: true
+regBankSelected: true
+
+body: |
+  bb.0:
+    liveins: $vgpr0, $vgpr1
+    ; GFX11-LABEL: name: fcmp_one_s16_vv
+    ; GFX11: liveins: $vgpr0, $vgpr1
+    ; GFX11-NEXT: {{  $}}
+    ; GFX11-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+    ; GFX11-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+    ; GFX11-NEXT: [[V_CMP_LG_F16_fake16_e64_:%[0-9]+]]:sreg_32_xm0_xexec = nofpexcept V_CMP_LG_F16_fake16_e64 0, [[COPY]], 0, [[COPY1]], 0, implicit $mode, implicit $exec
+    ; GFX11-NEXT: S_ENDPGM 0, implicit [[V_CMP_LG_F16_fake16_e64_]]
+    %0:vgpr(s32) = COPY $vgpr0
+    %1:vgpr(s32) = COPY $vgpr1
+    %2:vgpr(s16) = G_TRUNC %0
+    %3:vgpr(s16) = G_TRUNC %1
+    %4:vcc(s1) = G_FCMP floatpred(one), %2, %3
+    S_ENDPGM 0, implicit %4
+...
+
+---
+name: fcmp_ord_s16_vv
+legalized: true
+regBankSelected: true
+
+body: |
+  bb.0:
+    liveins: $vgpr0, $vgpr1
+    ; GFX11-LABEL: name: fcmp_ord_s16_vv
+    ; GFX11: liveins: $vgpr0, $vgpr1
+    ; GFX11-NEXT: {{  $}}
+    ; GFX11-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+    ; GFX11-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+    ; GFX11-NEXT: [[V_CMP_LG_F16_fake16_e64_:%[0-9]+]]:sreg_32_xm0_xexec = nofpexcept V_CMP_LG_F16_fake16_e64 0, [[COPY]], 0, [[COPY1]], 0, implicit $mode, implicit $exec
+    ; GFX11-NEXT: S_ENDPGM 0, implicit [[V_CMP_LG_F16_fake16_e64_]]
+    %0:vgpr(s32) = COPY $vgpr0
+    %1:vgpr(s32) = COPY $vgpr1
+    %2:vgpr(s16) = G_TRUNC %0
+    %3:vgpr(s16) = G_TRUNC %1
+    %4:vcc(s1) = G_FCMP floatpred(one), %2, %3
+    S_ENDPGM 0, implicit %4
+...
+
+---
+name: fcmp_uno_s16_vv
+legalized: true
+regBankSelected: true
+
+body: |
+  bb.0:
+    liveins: $vgpr0, $vgpr1
+    ; GFX11-LABEL: name: fcmp_uno_s16_vv
+    ; GFX11: liveins: $vgpr0, $vgpr1
+    ; GFX11-NEXT: {{  $}}
+    ; GFX11-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+    ; GFX11-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+    ; GFX11-NEXT: [[V_CMP_U_F16_fake16_e64_:%[0-9]+]]:sreg_32_xm0_xexec = nofpexcept V_CMP_U_F16_fake16_e64 0, [[COPY]], 0, [[COPY1]], 0, implicit $mode, implicit $exec
+    ; GFX11-NEXT: S_ENDPGM 0, implicit [[V_CMP_U_F16_fake16_e64_]]
+    %0:vgpr(s32) = COPY $vgpr0
+    %1:vgpr(s32) = COPY $vgpr1
+    %2:vgpr(s16) = G_TRUNC %0
+    %3:vgpr(s16) = G_TRUNC %1
+    %4:vcc(s1) = G_FCMP floatpred(uno), %2, %3
+    S_ENDPGM 0, implicit %4
+...
+
+---
+name: fcmp_ueq_s16_vv
+legalized: true
+regBankSelected: true
+
+body: |
+  bb.0:
+    liveins: $vgpr0, $vgpr1
+    ; GFX11-LABEL: name: fcmp_ueq_s16_vv
+    ; GFX11: liveins: $vgpr0, $vgpr1
+    ; GFX11-NEXT: {{  $}}
+    ; GFX11-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+    ; GFX11-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+    ; GFX11-NEXT: [[V_CMP_NLG_F16_fake16_e64_:%[0-9]+]]:sreg_32_xm0_xexec = nofpexcept V_CMP_NLG_F16_fake16_e64 0, [[COPY]], 0, [[COPY1]], 0, implicit $mode, implicit $exec
+    ; GFX11-NEXT: S_ENDPGM 0, implicit [[V_CMP_NLG_F16_fake16_e64_]]
+    %0:vgpr(s32) = COPY $vgpr0
+    %1:vgpr(s32) = COPY $vgpr1
+    %2:vgpr(s16) = G_TRUNC %0
+    %3:vgpr(s16) = G_TRUNC %1
+    %4:vcc(s1) = G_FCMP floatpred(ueq), %2, %3
+    S_ENDPGM 0, implicit %4
+...
+
+---
+name: fcmp_ugt_s16_vv
+legalized: true
+regBankSelected: true
+
+body: |
+  bb.0:
+    liveins: $vgpr0, $vgpr1
+    ; GFX11-LABEL: name: fcmp_ugt_s16_vv
+    ; GFX11: liveins: $vgpr0, $vgpr1
+    ; GFX11-NEXT: {{  $}}
+    ; GFX11-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+    ; GFX11-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+    ; GFX11-NEXT: [[V_CMP_NLE_F16_fake16_e64_:%[0-9]+]]:sreg_32_xm0_xexec = nofpexcept V_CMP_NLE_F16_fake16_e64 0, [[COPY]], 0, [[COPY1]], 0, implicit $mode, implicit $exec
+    ; GFX11-NEXT: S_ENDPGM 0, implicit [[V_CMP_NLE_F16_fake16_e64_]]
+    %0:vgpr(s32) = COPY $vgpr0
+    %1:vgpr(s32) = COPY $vgpr1
+    %2:vgpr(s16) = G_TRUNC %0
+    %3:vgpr(s16) = G_TRUNC %1
+    %4:vcc(s1) = G_FCMP floatpred(ugt), %2, %3
+    S_ENDPGM 0, implicit %4
+...
+
+---
+name: fcmp_uge_s16_vv
+legalized: true
+regBankSelected: true
+
+body: |
+  bb.0:
+    liveins: $vgpr0, $vgpr1
+    ; GFX11-LABEL: name: fcmp_uge_s16_vv
+    ; GFX11: liveins: $vgpr0, $vgpr1
+    ; GFX11-NEXT: {{  $}}
+    ; GFX11-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+    ; GFX11-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+    ; GFX11-NEXT: [[V_CMP_NLT_F16_fake16_e64_:%[0-9]+]]:sreg_32_xm0_xexec = nofpexcept V_CMP_NLT_F16_fake16_e64 0, [[COPY]], 0, [[COPY1]], 0, implicit $mode, implicit $exec
+    ; GFX11-NEXT: S_ENDPGM 0, implicit [[V_CMP_NLT_F16_fake16_e64_]]
+    %0:vgpr(s32) = COPY $vgpr0
+    %1:vgpr(s32) = COPY $vgpr1
+    %2:vgpr(s16) = G_TRUNC %0
+    %3:vgpr(s16) = G_TRUNC %1
+    %4:vcc(s1) = G_FCMP floatpred(uge), %2, %3
+    S_ENDPGM 0, implicit %4
+...
+
+---
+name: fcmp_ult_s16_vv
+legalized: true
+regBankSelected: true
+
+body: |
+  bb.0:
+    liveins: $vgpr0, $vgpr1
+    ; GFX11-LABEL: name: fcmp_ult_s16_vv
+    ; GFX11: liveins: $vgpr0, $vgpr1
+    ; GFX11-NEXT: {{  $}}
+    ; GFX11-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+    ; GFX11-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+    ; GFX11-NEXT: [[V_CMP_NGE_F16_fake16_e64_:%[0-9]+]]:sreg_32_xm0_xexec = nofpexcept V_CMP_NGE_F16_fake16_e64 0, [[COPY]], 0, [[COPY1]], 0, implicit $mode, implicit $exec
+    ; GFX11-NEXT: S_ENDPGM 0, implicit [[V_CMP_NGE_F16_fake16_e64_]]
+    %0:vgpr(s32) = COPY $vgpr0
+    %1:vgpr(s32) = COPY $vgpr1
+    %2:vgpr(s16) = G_TRUNC %0
+    %3:vgpr(s16) = G_TRUNC %1
+    %4:vcc(s1) = G_FCMP floatpred(ult), %2, %3
+    S_ENDPGM 0, implicit %4
+...
+
+---
+name: fcmp_ule_s16_vv
+legalized: true
+regBankSelected: true
+
+body: |
+  bb.0:
+    liveins: $vgpr0, $vgpr1
+    ; GFX11-LABEL: name: fcmp_ule_s16_vv
+    ; GFX11: liveins: $vgpr0, $vgpr1
+    ; GFX11-NEXT: {{  $}}
+    ; GFX11-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+    ; GFX11-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+    ; GFX11-NEXT: [[V_CMP_NGT_F16_fake16_e64_:%[0-9]+]]:sreg_32_xm0_xexec = nofpexcept V_CMP_NGT_F16_fake16_e64 0, [[COPY]], 0, [[COPY1]], 0, implicit $mode, implicit $exec
+    ; GFX11-NEXT: S_ENDPGM 0, implicit [[V_CMP_NGT_F16_fake16_e64_]]
+    %0:vgpr(s32) = COPY $vgpr0
+    %1:vgpr(s32) = COPY $vgpr1
+    %2:vgpr(s16) = G_TRUNC %0
+    %3:vgpr(s16) = G_TRUNC %1
+    %4:vcc(s1) = G_FCMP floatpred(ule), %2, %3
+    S_ENDPGM 0, implicit %4
+...
+
+---
+name: fcmp_une_s16_vv
+legalized: true
+regBankSelected: true
+
+body: |
+  bb.0:
+    liveins: $vgpr0, $vgpr1
+    ; GFX11-LABEL: name: fcmp_une_s16_vv
+    ; GFX11: liveins: $vgpr0, $vgpr1
+    ; GFX11-NEXT: {{  $}}
+    ; GFX11-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+    ; GFX11-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+    ; GFX11-NEXT: [[V_CMP_NEQ_F16_fake16_e64_:%[0-9]+]]:sreg_32_xm0_xexec = nofpexcept V_CMP_NEQ_F16_fake16_e64 0, [[COPY]], 0, [[COPY1]], 0, implicit $mode, implicit $exec
+    ; GFX11-NEXT: S_ENDPGM 0, implicit [[V_CMP_NEQ_F16_fake16_e64_]]
+    %0:vgpr(s32) = COPY $vgpr0
+    %1:vgpr(s32) = COPY $vgpr1
+    %2:vgpr(s16) = G_TRUNC %0
+    %3:vgpr(s16) = G_TRUNC %1
+    %4:vcc(s1) = G_FCMP floatpred(une), %2, %3
+    S_ENDPGM 0, implicit %4
+...
+
+---
+name: fcmp_true_s16_vv
+legalized: true
+regBankSelected: true
+
+body: |
+  bb.0:
+    liveins: $vgpr0, $vgpr1
+    ; GFX11-LABEL: name: fcmp_true_s16_vv
+    ; GFX11: liveins: $vgpr0, $vgpr1
+    ; GFX11-NEXT: {{  $}}
+    ; GFX11-NEXT: [[COPY:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
+    ; GFX11-NEXT: [[COPY1:%[0-9]+]]:vgpr(s32) = COPY $vgpr1
+    ; GFX11-NEXT: [[TRUNC:%[0-9]+]]:vgpr(s16) = G_TRUNC [[COPY]](s32)
+    ; GFX11-NEXT: [[TRUNC1:%[0-9]+]]:vgpr(s16) = G_TRUNC [[COPY1]](s32)
+    ; GFX11-NEXT: [[FCMP:%[0-9]+]]:vcc(s1) = G_FCMP floatpred(true), [[TRUNC]](s16), [[TRUNC1]]
+    ; GFX11-NEXT: S_ENDPGM 0, implicit [[FCMP]](s1)
+    %0:vgpr(s32) = COPY $vgpr0
+    %1:vgpr(s32) = COPY $vgpr1
+    %2:vgpr(s16) = G_TRUNC %0
+    %3:vgpr(s16) = G_TRUNC %1
+    %4:vcc(s1) = G_FCMP floatpred(true), %2, %3
+    S_ENDPGM 0, implicit %4
+...
+
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-fcmp.s16.gfx11plus.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-fcmp.s16.gfx11plus.mir
new file mode 100644
index 0000000000000..a7140e6a74fd4
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-fcmp.s16.gfx11plus.mir
@@ -0,0 +1,402 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -mtriple=amdgcn -mcpu=gfx1100 -run-pass=instruction-select -mattr=+real-true16 -verify-machineinstrs -global-isel-abort=0 -o - %s  | FileCheck -check-prefix=GFX11 %s
+
+---
+name: fcmp_false_s16_vv
+legalized: true
+regBankSelected: true
+
+body: |
+  bb.0:
+    liveins: $vgpr0, $vgpr1
+    ; GFX11-LABEL: name: fcmp_false_s16_vv
+    ; GFX11: liveins: $vgpr0, $vgpr1
+    ; GFX11-NEXT: {{  $}}
+    ; GFX11-NEXT: [[COPY:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
+    ; GFX11-NEXT: [[COPY1:%[0-9]+]]:vgpr(s32) = COPY $vgpr1
+    ; GFX11-NEXT: [[TRUNC:%[0-9]+]]:vgpr(s16) = G_TRUNC [[COPY]](s32)
+    ; GFX11-NEXT: [[TRUNC1:%[0-9]+]]:vgpr(s16) = G_TRUNC [[COPY1]](s32)
+    ; GFX11-NEXT: [[FCMP:%[0-9]+]]:vcc(s1) = G_FCMP floatpred(false), [[TRUNC]](s16), [[TRUNC1]]
+    ; GFX11-NEXT: S_ENDPGM 0, implicit [[FCMP]](s1)
+    %0:vgpr(s32) = COPY $vgpr0
+    %1:vgpr(s32) = COPY $vgpr1
+    %2:vgpr(s16) = G_TRUNC %0
+    %3:vgpr(s16) = G_TRUNC %1
+    %4:vcc(s1) = G_FCMP floatpred(false), %2, %3
+    S_ENDPGM 0, implicit %4
+...
+
+---
+name: fcmp_oeq_s16_vv
+legalized: true
+regBankSelected: true
+
+body: |
+  bb.0:
+    liveins: $vgpr0, $vgpr1
+    ; GFX11-LABEL: name: fcmp_oeq_s16_vv
+    ; GFX11: liveins: $vgpr0, $vgpr1
+    ; GFX11-NEXT: {{  $}}
+    ; GFX11-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+    ; GFX11-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+    ; GFX11-NEXT: [[COPY2:%[0-9]+]]:vgpr_16 = COPY [[COPY]].lo16
+    ; GFX11-NEXT: [[COPY3:%[0-9]+]]:vgpr_16 = COPY [[COPY1]].lo16
+    ; GFX11-NEXT: [[V_CMP_EQ_F16_t16_e64_:%[0-9]+]]:sreg_32_xm0_xexec = nofpexcept V_CMP_EQ_F16_t16_e64 0, [[COPY2]], 0, [[COPY3]], 0, 0, implicit $mode, implicit $exec
+    ; GFX11-NEXT: S_ENDPGM 0, implicit [[V_CMP_EQ_F16_t16_e64_]]
+    %0:vgpr(s32) = COPY $vgpr0
+    %1:vgpr(s32) = COPY $vgpr1
+    %2:vgpr(s16) = G_TRUNC %0
+    %3:vgpr(s16) = G_TRUNC %1
+    %4:vcc(s1) = G_FCMP floatpred(oeq), %2, %3
+    S_ENDPGM 0, implicit %4
+...
+
+---
+name: fcmp_ogt_s16_vv
+legalized: true
+regBankSelected: true
+
+body: |
+  bb.0:
+    liveins: $vgpr0, $vgpr1
+    ; GFX11-LABEL: name: fcmp_ogt_s16_vv
+    ; GFX11: liveins: $vgpr0, $vgpr1
+    ; GFX11-NEXT: {{  $}}
+    ; GFX11-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+    ; GFX11-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+    ; GFX11-NEXT: [[COPY2:%[0-9]+]]:vgpr_16 = COPY [[COPY]].lo16
+    ; GFX11-NEXT: [[COPY3:%[0-9]+]]:vgpr_16 = COPY [[COPY1]].lo16
+    ; GFX11-NEXT: [[V_CMP_GT_F16_t16_e64_:%[0-9]+]]:sreg_32_xm0_xexec = nofpexcept V_CMP_GT_F16_t16_e64 0, [[COPY2]], 0, [[COPY3]], 0, 0, implicit $mode, implicit $exec
+    ; GFX11-NEXT: S_ENDPGM 0, implicit [[V_CMP_GT_F16_t16_e64_]]
+    %0:vgpr(s32) = COPY $vgpr0
+    %1:vgpr(s32) = COPY $vgpr1
+    %2:vgpr(s16) = G_TRUNC %0
+    %3:vgpr(s16) = G_TRUNC %1
+    %4:vcc(s1) = G_FCMP floatpred(ogt), %2, %3
+    S_ENDPGM 0, implicit %4
+...
+
+---
+name: fcmp_oge_s16_vv
+legalized: true
+regBankSelected: true
+
+body: |
+  bb.0:
+    liveins: $vgpr0, $vgpr1
+    ; GFX11-LABEL: name: fcmp_oge_s16_vv
+    ; GFX11: liveins: $vgpr0, $vgpr1
+    ; GFX11-NEXT: {{  $}}
+    ; GFX11-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+    ; GFX11-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+    ; GFX11-NEXT: [[COPY2:%[0-9]+]]:vgpr_16 = COPY [[COPY]].lo16
+    ; GFX11-NEXT: [[COPY3:%[0-9]+]]:vgpr_16 = COPY [[COPY1]].lo16
+    ; GFX11-NEXT: [[V_CMP_GE_F16_t16_e64_:%[0-9]+]]:sreg_32_xm0_xexec = nofpexcept V_CMP_GE_F16_t16_e64 0, [[COPY2]], 0, [[COPY3]], 0, 0, implicit $mode, implicit $exec
+    ; GFX11-NEXT: S_ENDPGM 0, implicit [[V_CMP_GE_F16_t16_e64_]]
+    %0:vgpr(s32) = COPY $vgpr0
+    %1:vgpr(s32) = COPY $vgpr1
+    %2:vgpr(s16) = G_TRUNC %0
+    %3:vgpr(s16) = G_TRUNC %1
+    %4:vcc(s1) = G_FCMP floatpred(oge), %2, %3
+    S_ENDPGM 0, implicit %4
+...
+
+---
+name: fcmp_olt_s16_vv
+legalized: true
+regBankSelected: true
+
+body: |
+  bb.0:
+    liveins: $vgpr0, $vgpr1
+    ; GFX11-LABEL: name: fcmp_olt_s16_vv
+    ; GFX11: liveins: $vgpr0, $vgpr1
+    ; GFX11-NEXT: {{  $}}
+    ; GFX11-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+    ; GFX11-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+    ; GFX11-NEXT: [[COPY2:%[0-9]+]]:vgpr_16 = COPY [[COPY]].lo16
+    ; GFX11-NEXT: [[COPY3:%[0-9]+]]:vgpr_16 = COPY [[COPY1]].lo16
+    ; GFX11-NEXT: [[V_CMP_LT_F16_t16_e64_:%[0-9]+]]:sreg_32_xm0_xexec = nofpexcept V_CMP_LT_F16_t16_e64 0, [[COPY2]], 0, [[COPY3]], 0, 0, implicit $mode, implicit $exec
+    ; GFX11-NEXT: S_ENDPGM 0, implicit [[V_CMP_LT_F16_t16_e64_]]
+    %0:vgpr(s32) = COPY $vgpr0
+    %1:vgpr(s32) = COPY $vgpr1
+    %2:vgpr(s16) = G_TRUNC %0
+    %3:vgpr(s16) = G_TRUNC %1
+    %4:vcc(s1) = G_FCMP floatpred(olt), %2, %3
+    S_ENDPGM 0, implicit %4
+...
+
+---
+name: fcmp_ole_s16_vv
+legalized: true
+regBankSelected: true
+
+body: |
+  bb.0:
+    liveins: $vgpr0, $vgpr1
+    ; GFX11-LABEL: name: fcmp_ole_s16_vv
+    ; GFX11: liveins: $vgpr0, $vgpr1
+    ; GFX11-NEXT: {{  $}}
+    ; GFX11-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+    ; GFX11-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+    ; GFX11-NEXT: [[COPY2:%[0-9]+]]:vgpr_16 = COPY [[COPY]].lo16
+    ; GFX11-NEXT: [[COPY3:%[0-9]+]]:vgpr_16 = COPY [[COPY1]].lo16
+    ; GFX11-NEXT: [[V_CMP_LE_F16_t16_e64_:%[0-9]+]]:sreg_32_xm0_xexec = nofpexcept V_CMP_LE_F16_t16_e64 0, [[COPY2]], 0, [[COPY3]], 0, 0, implicit $mode, implicit $exec
+    ; GFX11-NEXT: S_ENDPGM 0, implicit [[V_CMP_LE_F16_t16_e64_]]
+    %0:vgpr(s32) = COPY $vgpr0
+    %1:vgpr(s32) = COPY $vgpr1
+    %2:vgpr(s16) = G_TRUNC %0
+    %3:vgpr(s16) = G_TRUNC %1
+    %4:vcc(s1) = G_FCMP floatpred(ole), %2, %3
+    S_ENDPGM 0, implicit %4
+...
+---
+name: fcmp_one_s16_vv
+legalized: true
+regBankSelected: true
+
+body: |
+  bb.0:
+    liveins: $vgpr0, $vgpr1
+    ; GFX11-LABEL: name: fcmp_one_s16_vv
+    ; GFX11: liveins: $vgpr0, $vgpr1
+    ; GFX11-NEXT: {{  $}}
+    ; GFX11-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+    ; GFX11-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+    ; GFX11-NEXT: [[COPY2:%[0-9]+]]:vgpr_16 = COPY [[COPY]].lo16
+    ; GFX11-NEXT: [[COPY3:%[0-9]+]]:vgpr_16 = COPY [[COPY1]].lo16
+    ; GFX11-NEXT: [[V_CMP_LG_F16_t16_e64_:%[0-9]+]]:sreg_32_xm0_xexec = nofpexcept V_CMP_LG_F16_t16_e64 0, [[COPY2]], 0, [[COPY3]], 0, 0, implicit $mode, implicit $exec
+    ; GFX11-NEXT: S_ENDPGM 0, implicit [[V_CMP_LG_F16_t16_e64_]]
+    %0:vgpr(s32) = COPY $vgpr0
+    %1:vgpr(s32) = COPY $vgpr1
+    %2:vgpr(s16) = G_TRUNC %0
+    %3:vgpr(s16) = G_TRUNC %1
+    %4:vcc(s1) = G_FCMP floatpred(one), %2, %3
+    S_ENDPGM 0, implicit %4
+...
+
+---
+name: fcmp_ord_s16_vv
+legalized: true
+regBankSelected: true
+
+body: |
+  bb.0:
+    liveins: $vgpr0, $vgpr1
+    ; GFX11-LABEL: name: fcmp_ord_s16_vv
+    ; GFX11: liveins: $vgpr0, $vgpr1
+    ; GFX11-NEXT: {{  $}}
+    ; GFX11-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+    ; GFX11-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+    ; GFX11-NEXT: [[COPY2:%[0-9]+]]:vgpr_16 = COPY [[COPY]].lo16
+    ; GFX11-NEXT: [[COPY3:%[0-9]+]]:vgpr_16 = COPY [[COPY1]].lo16
+    ; GFX11-NEXT: [[V_CMP_LG_F16_t16_e64_:%[0-9]+]]:sreg_32_xm0_xexec = nofpexcept V_CMP_LG_F16_t16_e64 0, [[COPY2]], 0, [[COPY3]], 0, 0, implicit $mode, implicit $exec
+    ; GFX11-NEXT: S_ENDPGM 0, implicit [[V_CMP_LG_F16_t16_e64_]]
+    %0:vgpr(s32) = COPY $vgpr0
+    %1:vgpr(s32) = COPY $vgpr1
+    %2:vgpr(s16) = G_TRUNC %0
+    %3:vgpr(s16) = G_TRUNC %1
+    %4:vcc(s1) = G_FCMP floatpred(one), %2, %3
+    S_ENDPGM 0, implicit %4
+...
+
+---
+name: fcmp_uno_s16_vv
+legalized: true
+regBankSelected: true
+
+body: |
+  bb.0:
+    liveins: $vgpr0, $vgpr1
+    ; GFX11-LABEL: name: fcmp_uno_s16_vv
+    ; GFX11: liveins: $vgpr0, $vgpr1
+    ; GFX11-NEXT: {{  $}}
+    ; GFX11-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+    ; GFX11-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+    ; GFX11-NEXT: [[COPY2:%[0-9]+]]:vgpr_16 = COPY [[COPY]].lo16
+    ; GFX11-NEXT: [[COPY3:%[0-9]+]]:vgpr_16 = COPY [[COPY1]].lo16
+    ; GFX11-NEXT: [[V_CMP_U_F16_t16_e64_:%[0-9]+]]:sreg_32_xm0_xexec = nofpexcept V_CMP_U_F16_t16_e64 0, [[COPY2]], 0, [[COPY3]], 0, 0, implicit $mode, implicit $exec
+    ; GFX11-NEXT: S_ENDPGM 0, implicit [[V_CMP_U_F16_t16_e64_]]
+    %0:vgpr(s32) = COPY $vgpr0
+    %1:vgpr(s32) = COPY $vgpr1
+    %2:vgpr(s16) = G_TRUNC %0
+    %3:vgpr(s16) = G_TRUNC %1
+    %4:vcc(s1) = G_FCMP floatpred(uno), %2, %3
+    S_ENDPGM 0, implicit %4
+...
+
+---
+name: fcmp_ueq_s16_vv
+legalized: true
+regBankSelected: true
+
+body: |
+  bb.0:
+    liveins: $vgpr0, $vgpr1
+    ; GFX11-LABEL: name: fcmp_ueq_s16_vv
+    ; GFX11: liveins: $vgpr0, $vgpr1
+    ; GFX11-NEXT: {{  $}}
+    ; GFX11-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+    ; GFX11-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+    ; GFX11-NEXT: [[COPY2:%[0-9]+]]:vgpr_16 = COPY [[COPY]].lo16
+    ; GFX11-NEXT: [[COPY3:%[0-9]+]]:vgpr_16 = COPY [[COPY1]].lo16
+    ; GFX11-NEXT: [[V_CMP_NLG_F16_t16_e64_:%[0-9]+]]:sreg_32_xm0_xexec = nofpexcept V_CMP_NLG_F16_t16_e64 0, [[COPY2]], 0, [[COPY3]], 0, 0, implicit $mode, implicit $exec
+    ; GFX11-NEXT: S_ENDPGM 0, implicit [[V_CMP_NLG_F16_t16_e64_]]
+    %0:vgpr(s32) = COPY $vgpr0
+    %1:vgpr(s32) = COPY $vgpr1
+    %2:vgpr(s16) = G_TRUNC %0
+    %3:vgpr(s16) = G_TRUNC %1
+    %4:vcc(s1) = G_FCMP floatpred(ueq), %2, %3
+    S_ENDPGM 0, implicit %4
+...
+
+---
+name: fcmp_ugt_s16_vv
+legalized: true
+regBankSelected: true
+
+body: |
+  bb.0:
+    liveins: $vgpr0, $vgpr1
+    ; GFX11-LABEL: name: fcmp_ugt_s16_vv
+    ; GFX11: liveins: $vgpr0, $vgpr1
+    ; GFX11-NEXT: {{  $}}
+    ; GFX11-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+    ; GFX11-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+    ; GFX11-NEXT: [[COPY2:%[0-9]+]]:vgpr_16 = COPY [[COPY]].lo16
+    ; GFX11-NEXT: [[COPY3:%[0-9]+]]:vgpr_16 = COPY [[COPY1]].lo16
+    ; GFX11-NEXT: [[V_CMP_NLE_F16_t16_e64_:%[0-9]+]]:sreg_32_xm0_xexec = nofpexcept V_CMP_NLE_F16_t16_e64 0, [[COPY2]], 0, [[COPY3]], 0, 0, implicit $mode, implicit $exec
+    ; GFX11-NEXT: S_ENDPGM 0, implicit [[V_CMP_NLE_F16_t16_e64_]]
+    %0:vgpr(s32) = COPY $vgpr0
+    %1:vgpr(s32) = COPY $vgpr1
+    %2:vgpr(s16) = G_TRUNC %0
+    %3:vgpr(s16) = G_TRUNC %1
+    %4:vcc(s1) = G_FCMP floatpred(ugt), %2, %3
+    S_ENDPGM 0, implicit %4
+...
+
+---
+name: fcmp_uge_s16_vv
+legalized: true
+regBankSelected: true
+
+body: |
+  bb.0:
+    liveins: $vgpr0, $vgpr1
+    ; GFX11-LABEL: name: fcmp_uge_s16_vv
+    ; GFX11: liveins: $vgpr0, $vgpr1
+    ; GFX11-NEXT: {{  $}}
+    ; GFX11-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+    ; GFX11-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+    ; GFX11-NEXT: [[COPY2:%[0-9]+]]:vgpr_16 = COPY [[COPY]].lo16
+    ; GFX11-NEXT: [[COPY3:%[0-9]+]]:vgpr_16 = COPY [[COPY1]].lo16
+    ; GFX11-NEXT: [[V_CMP_NLT_F16_t16_e64_:%[0-9]+]]:sreg_32_xm0_xexec = nofpexcept V_CMP_NLT_F16_t16_e64 0, [[COPY2]], 0, [[COPY3]], 0, 0, implicit $mode, implicit $exec
+    ; GFX11-NEXT: S_ENDPGM 0, implicit [[V_CMP_NLT_F16_t16_e64_]]
+    %0:vgpr(s32) = COPY $vgpr0
+    %1:vgpr(s32) = COPY $vgpr1
+    %2:vgpr(s16) = G_TRUNC %0
+    %3:vgpr(s16) = G_TRUNC %1
+    %4:vcc(s1) = G_FCMP floatpred(uge), %2, %3
+    S_ENDPGM 0, implicit %4
+...
+
+---
+name: fcmp_ult_s16_vv
+legalized: true
+regBankSelected: true
+
+body: |
+  bb.0:
+    liveins: $vgpr0, $vgpr1
+    ; GFX11-LABEL: name: fcmp_ult_s16_vv
+    ; GFX11: liveins: $vgpr0, $vgpr1
+    ; GFX11-NEXT: {{  $}}
+    ; GFX11-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+    ; GFX11-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+    ; GFX11-NEXT: [[COPY2:%[0-9]+]]:vgpr_16 = COPY [[COPY]].lo16
+    ; GFX11-NEXT: [[COPY3:%[0-9]+]]:vgpr_16 = COPY [[COPY1]].lo16
+    ; GFX11-NEXT: [[V_CMP_NGE_F16_t16_e64_:%[0-9]+]]:sreg_32_xm0_xexec = nofpexcept V_CMP_NGE_F16_t16_e64 0, [[COPY2]], 0, [[COPY3]], 0, 0, implicit $mode, implicit $exec
+    ; GFX11-NEXT: S_ENDPGM 0, implicit [[V_CMP_NGE_F16_t16_e64_]]
+    %0:vgpr(s32) = COPY $vgpr0
+    %1:vgpr(s32) = COPY $vgpr1
+    %2:vgpr(s16) = G_TRUNC %0
+    %3:vgpr(s16) = G_TRUNC %1
+    %4:vcc(s1) = G_FCMP floatpred(ult), %2, %3
+    S_ENDPGM 0, implicit %4
+...
+
+---
+name: fcmp_ule_s16_vv
+legalized: true
+regBankSelected: true
+
+body: |
+  bb.0:
+    liveins: $vgpr0, $vgpr1
+    ; GFX11-LABEL: name: fcmp_ule_s16_vv
+    ; GFX11: liveins: $vgpr0, $vgpr1
+    ; GFX11-NEXT: {{  $}}
+    ; GFX11-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+    ; GFX11-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+    ; GFX11-NEXT: [[COPY2:%[0-9]+]]:vgpr_16 = COPY [[COPY]].lo16
+    ; GFX11-NEXT: [[COPY3:%[0-9]+]]:vgpr_16 = COPY [[COPY1]].lo16
+    ; GFX11-NEXT: [[V_CMP_NGT_F16_t16_e64_:%[0-9]+]]:sreg_32_xm0_xexec = nofpexcept V_CMP_NGT_F16_t16_e64 0, [[COPY2]], 0, [[COPY3]], 0, 0, implicit $mode, implicit $exec
+    ; GFX11-NEXT: S_ENDPGM 0, implicit [[V_CMP_NGT_F16_t16_e64_]]
+    %0:vgpr(s32) = COPY $vgpr0
+    %1:vgpr(s32) = COPY $vgpr1
+    %2:vgpr(s16) = G_TRUNC %0
+    %3:vgpr(s16) = G_TRUNC %1
+    %4:vcc(s1) = G_FCMP floatpred(ule), %2, %3
+    S_ENDPGM 0, implicit %4
+...
+
+---
+name: fcmp_une_s16_vv
+legalized: true
+regBankSelected: true
+
+body: |
+  bb.0:
+    liveins: $vgpr0, $vgpr1
+    ; GFX11-LABEL: name: fcmp_une_s16_vv
+    ; GFX11: liveins: $vgpr0, $vgpr1
+    ; GFX11-NEXT: {{  $}}
+    ; GFX11-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+    ; GFX11-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+    ; GFX11-NEXT: [[COPY2:%[0-9]+]]:vgpr_16 = COPY [[COPY]].lo16
+    ; GFX11-NEXT: [[COPY3:%[0-9]+]]:vgpr_16 = COPY [[COPY1]].lo16
+    ; GFX11-NEXT: [[V_CMP_NEQ_F16_t16_e64_:%[0-9]+]]:sreg_32_xm0_xexec = nofpexcept V_CMP_NEQ_F16_t16_e64 0, [[COPY2]], 0, [[COPY3]], 0, 0, implicit $mode, implicit $exec
+    ; GFX11-NEXT: S_ENDPGM 0, implicit [[V_CMP_NEQ_F16_t16_e64_]]
+    %0:vgpr(s32) = COPY $vgpr0
+    %1:vgpr(s32) = COPY $vgpr1
+    %2:vgpr(s16) = G_TRUNC %0
+    %3:vgpr(s16) = G_TRUNC %1
+    %4:vcc(s1) = G_FCMP floatpred(une), %2, %3
+    S_ENDPGM 0, implicit %4
+...
+
+---
+name: fcmp_true_s16_vv
+legalized: true
+regBankSelected: true
+
+body: |
+  bb.0:
+    liveins: $vgpr0, $vgpr1
+    ; GFX11-LABEL: name: fcmp_true_s16_vv
+    ; GFX11: liveins: $vgpr0, $vgpr1
+    ; GFX11-NEXT: {{  $}}
+    ; GFX11-NEXT: [[COPY:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
+    ; GFX11-NEXT: [[COPY1:%[0-9]+]]:vgpr(s32) = COPY $vgpr1
+    ; GFX11-NEXT: [[TRUNC:%[0-9]+]]:vgpr(s16) = G_TRUNC [[COPY]](s32)
+    ; GFX11-NEXT: [[TRUNC1:%[0-9]+]]:vgpr(s16) = G_TRUNC [[COPY1]](s32)
+    ; GFX11-NEXT: [[FCMP:%[0-9]+]]:vcc(s1) = G_FCMP floatpred(true), [[TRUNC]](s16), [[TRUNC1]]
+    ; GFX11-NEXT: S_ENDPGM 0, implicit [[FCMP]](s1)
+    %0:vgpr(s32) = COPY $vgpr0
+    %1:vgpr(s32) = COPY $vgpr1
+    %2:vgpr(s16) = G_TRUNC %0
+    %3:vgpr(s16) = G_TRUNC %1
+    %4:vcc(s1) = G_FCMP floatpred(true), %2, %3
+    S_ENDPGM 0, implicit %4
+...
+
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-fcmp.s16.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-fcmp.s16.mir
index 5c387baf46752..85b1d402146ce 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-fcmp.s16.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-fcmp.s16.mir
@@ -1,7 +1,6 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
 # RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=instruction-select -verify-machineinstrs -global-isel-abort=0 -o - %s  | FileCheck -check-prefix=WAVE64 %s
 # RUN: llc -mtriple=amdgcn -mcpu=gfx1010 -run-pass=instruction-select -verify-machineinstrs -global-isel-abort=0 -o - %s  | FileCheck -check-prefix=WAVE32 %s
-# RUN: llc -mtriple=amdgcn -mcpu=gfx1100 -run-pass=instruction-select -verify-machineinstrs -global-isel-abort=0 -o - %s  | FileCheck -check-prefix=GFX11 %s
 
 ---
 name: fcmp_false_s16_vv
@@ -31,15 +30,6 @@ body: |
     ; WAVE32-NEXT: [[FCMP:%[0-9]+]]:vcc(s1) = G_FCMP floatpred(false), [[TRUNC]](s16), [[TRUNC1]]
     ; WAVE32-NEXT: S_ENDPGM 0, implicit [[FCMP]](s1)
     ;
-    ; GFX11-LABEL: name: fcmp_false_s16_vv
-    ; GFX11: liveins: $vgpr0, $vgpr1
-    ; GFX11-NEXT: {{  $}}
-    ; GFX11-NEXT: [[COPY:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
-    ; GFX11-NEXT: [[COPY1:%[0-9]+]]:vgpr(s32) = COPY $vgpr1
-    ; GFX11-NEXT: [[TRUNC:%[0-9]+]]:vgpr(s16) = G_TRUNC [[COPY]](s32)
-    ; GFX11-NEXT: [[TRUNC1:%[0-9]+]]:vgpr(s16) = G_TRUNC [[COPY1]](s32)
-    ; GFX11-NEXT: [[FCMP:%[0-9]+]]:vcc(s1) = G_FCMP floatpred(false), [[TRUNC]](s16), [[TRUNC1]]
-    ; GFX11-NEXT: S_ENDPGM 0, implicit [[FCMP]](s1)
     %0:vgpr(s32) = COPY $vgpr0
     %1:vgpr(s32) = COPY $vgpr1
     %2:vgpr(s16) = G_TRUNC %0
@@ -72,13 +62,6 @@ body: |
     ; WAVE32-NEXT: [[V_CMP_EQ_F16_e64_:%[0-9]+]]:sreg_32_xm0_xexec = nofpexcept V_CMP_EQ_F16_e64 0, [[COPY]], 0, [[COPY1]], 0, implicit $mode, implicit $exec
     ; WAVE32-NEXT: S_ENDPGM 0, implicit [[V_CMP_EQ_F16_e64_]]
     ;
-    ; GFX11-LABEL: name: fcmp_oeq_s16_vv
-    ; GFX11: liveins: $vgpr0, $vgpr1
-    ; GFX11-NEXT: {{  $}}
-    ; GFX11-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
-    ; GFX11-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
-    ; GFX11-NEXT: [[V_CMP_EQ_F16_fake16_e64_:%[0-9]+]]:sreg_32_xm0_xexec = nofpexcept V_CMP_EQ_F16_fake16_e64 0, [[COPY]], 0, [[COPY1]], 0, implicit $mode, implicit $exec
-    ; GFX11-NEXT: S_ENDPGM 0, implicit [[V_CMP_EQ_F16_fake16_e64_]]
     %0:vgpr(s32) = COPY $vgpr0
     %1:vgpr(s32) = COPY $vgpr1
     %2:vgpr(s16) = G_TRUNC %0
@@ -111,13 +94,6 @@ body: |
     ; WAVE32-NEXT: [[V_CMP_GT_F16_e64_:%[0-9]+]]:sreg_32_xm0_xexec = nofpexcept V_CMP_GT_F16_e64 0, [[COPY]], 0, [[COPY1]], 0, implicit $mode, implicit $exec
     ; WAVE32-NEXT: S_ENDPGM 0, implicit [[V_CMP_GT_F16_e64_]]
     ;
-    ; GFX11-LABEL: name: fcmp_ogt_s16_vv
-    ; GFX11: liveins: $vgpr0, $vgpr1
-    ; GFX11-NEXT: {{  $}}
-    ; GFX11-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
-    ; GFX11-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
-    ; GFX11-NEXT: [[V_CMP_GT_F16_fake16_e64_:%[0-9]+]]:sreg_32_xm0_xexec = nofpexcept V_CMP_GT_F16_fake16_e64 0, [[COPY]], 0, [[COPY1]], 0, implicit $mode, implicit $exec
-    ; GFX11-NEXT: S_ENDPGM 0, implicit [[V_CMP_GT_F16_fake16_e64_]]
     %0:vgpr(s32) = COPY $vgpr0
     %1:vgpr(s32) = COPY $vgpr1
     %2:vgpr(s16) = G_TRUNC %0
@@ -150,13 +126,6 @@ body: |
     ; WAVE32-NEXT: [[V_CMP_GE_F16_e64_:%[0-9]+]]:sreg_32_xm0_xexec = nofpexcept V_CMP_GE_F16_e64 0, [[COPY]], 0, [[COPY1]], 0, implicit $mode, implicit $exec
     ; WAVE32-NEXT: S_ENDPGM 0, implicit [[V_CMP_GE_F16_e64_]]
     ;
-    ; GFX11-LABEL: name: fcmp_oge_s16_vv
-    ; GFX11: liveins: $vgpr0, $vgpr1
-    ; GFX11-NEXT: {{  $}}
-    ; GFX11-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
-    ; GFX11-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
-    ; GFX11-NEXT: [[V_CMP_GE_F16_fake16_e64_:%[0-9]+]]:sreg_32_xm0_xexec = nofpexcept V_CMP_GE_F16_fake16_e64 0, [[COPY]], 0, [[COPY1]], 0, implicit $mode, implicit $exec
-    ; GFX11-NEXT: S_ENDPGM 0, implicit [[V_CMP_GE_F16_fake16_e64_]]
     %0:vgpr(s32) = COPY $vgpr0
     %1:vgpr(s32) = COPY $vgpr1
     %2:vgpr(s16) = G_TRUNC %0
@@ -189,13 +158,6 @@ body: |
     ; WAVE32-NEXT: [[V_CMP_LT_F16_e64_:%[0-9]+]]:sreg_32_xm0_xexec = nofpexcept V_CMP_LT_F16_e64 0, [[COPY]], 0, [[COPY1]], 0, implicit $mode, implicit $exec
     ; WAVE32-NEXT: S_ENDPGM 0, implicit [[V_CMP_LT_F16_e64_]]
     ;
-    ; GFX11-LABEL: name: fcmp_olt_s16_vv
-    ; GFX11: liveins: $vgpr0, $vgpr1
-    ; GFX11-NEXT: {{  $}}
-    ; GFX11-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
-    ; GFX11-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
-    ; GFX11-NEXT: [[V_CMP_LT_F16_fake16_e64_:%[0-9]+]]:sreg_32_xm0_xexec = nofpexcept V_CMP_LT_F16_fake16_e64 0, [[COPY]], 0, [[COPY1]], 0, implicit $mode, implicit $exec
-    ; GFX11-NEXT: S_ENDPGM 0, implicit [[V_CMP_LT_F16_fake16_e64_]]
     %0:vgpr(s32) = COPY $vgpr0
     %1:vgpr(s32) = COPY $vgpr1
     %2:vgpr(s16) = G_TRUNC %0
@@ -228,13 +190,6 @@ body: |
     ; WAVE32-NEXT: [[V_CMP_LE_F16_e64_:%[0-9]+]]:sreg_32_xm0_xexec = nofpexcept V_CMP_LE_F16_e64 0, [[COPY]], 0, [[COPY1]], 0, implicit $mode, implicit $exec
     ; WAVE32-NEXT: S_ENDPGM 0, implicit [[V_CMP_LE_F16_e64_]]
     ;
-    ; GFX11-LABEL: name: fcmp_ole_s16_vv
-    ; GFX11: liveins: $vgpr0, $vgpr1
-    ; GFX11-NEXT: {{  $}}
-    ; GFX11-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
-    ; GFX11-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
-    ; GFX11-NEXT: [[V_CMP_LE_F16_fake16_e64_:%[0-9]+]]:sreg_32_xm0_xexec = nofpexcept V_CMP_LE_F16_fake16_e64 0, [[COPY]], 0, [[COPY1]], 0, implicit $mode, implicit $exec
-    ; GFX11-NEXT: S_ENDPGM 0, implicit [[V_CMP_LE_F16_fake16_e64_]]
     %0:vgpr(s32) = COPY $vgpr0
     %1:vgpr(s32) = COPY $vgpr1
     %2:vgpr(s16) = G_TRUNC %0
@@ -266,13 +221,6 @@ body: |
     ; WAVE32-NEXT: [[V_CMP_LG_F16_e64_:%[0-9]+]]:sreg_32_xm0_xexec = nofpexcept V_CMP_LG_F16_e64 0, [[COPY]], 0, [[COPY1]], 0, implicit $mode, implicit $exec
     ; WAVE32-NEXT: S_ENDPGM 0, implicit [[V_CMP_LG_F16_e64_]]
     ;
-    ; GFX11-LABEL: name: fcmp_one_s16_vv
-    ; GFX11: liveins: $vgpr0, $vgpr1
-    ; GFX11-NEXT: {{  $}}
-    ; GFX11-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
-    ; GFX11-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
-    ; GFX11-NEXT: [[V_CMP_LG_F16_fake16_e64_:%[0-9]+]]:sreg_32_xm0_xexec = nofpexcept V_CMP_LG_F16_fake16_e64 0, [[COPY]], 0, [[COPY1]], 0, implicit $mode, implicit $exec
-    ; GFX11-NEXT: S_ENDPGM 0, implicit [[V_CMP_LG_F16_fake16_e64_]]
     %0:vgpr(s32) = COPY $vgpr0
     %1:vgpr(s32) = COPY $vgpr1
     %2:vgpr(s16) = G_TRUNC %0
@@ -305,13 +253,6 @@ body: |
     ; WAVE32-NEXT: [[V_CMP_LG_F16_e64_:%[0-9]+]]:sreg_32_xm0_xexec = nofpexcept V_CMP_LG_F16_e64 0, [[COPY]], 0, [[COPY1]], 0, implicit $mode, implicit $exec
     ; WAVE32-NEXT: S_ENDPGM 0, implicit [[V_CMP_LG_F16_e64_]]
     ;
-    ; GFX11-LABEL: name: fcmp_ord_s16_vv
-    ; GFX11: liveins: $vgpr0, $vgpr1
-    ; GFX11-NEXT: {{  $}}
-    ; GFX11-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
-    ; GFX11-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
-    ; GFX11-NEXT: [[V_CMP_LG_F16_fake16_e64_:%[0-9]+]]:sreg_32_xm0_xexec = nofpexcept V_CMP_LG_F16_fake16_e64 0, [[COPY]], 0, [[COPY1]], 0, implicit $mode, implicit $exec
-    ; GFX11-NEXT: S_ENDPGM 0, implicit [[V_CMP_LG_F16_fake16_e64_]]
     %0:vgpr(s32) = COPY $vgpr0
     %1:vgpr(s32) = COPY $vgpr1
     %2:vgpr(s16) = G_TRUNC %0
@@ -344,13 +285,6 @@ body: |
     ; WAVE32-NEXT: [[V_CMP_U_F16_e64_:%[0-9]+]]:sreg_32_xm0_xexec = nofpexcept V_CMP_U_F16_e64 0, [[COPY]], 0, [[COPY1]], 0, implicit $mode, implicit $exec
     ; WAVE32-NEXT: S_ENDPGM 0, implicit [[V_CMP_U_F16_e64_]]
     ;
-    ; GFX11-LABEL: name: fcmp_uno_s16_vv
-    ; GFX11: liveins: $vgpr0, $vgpr1
-    ; GFX11-NEXT: {{  $}}
-    ; GFX11-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
-    ; GFX11-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
-    ; GFX11-NEXT: [[V_CMP_U_F16_fake16_e64_:%[0-9]+]]:sreg_32_xm0_xexec = nofpexcept V_CMP_U_F16_fake16_e64 0, [[COPY]], 0, [[COPY1]], 0, implicit $mode, implicit $exec
-    ; GFX11-NEXT: S_ENDPGM 0, implicit [[V_CMP_U_F16_fake16_e64_]]
     %0:vgpr(s32) = COPY $vgpr0
     %1:vgpr(s32) = COPY $vgpr1
     %2:vgpr(s16) = G_TRUNC %0
@@ -383,13 +317,6 @@ body: |
     ; WAVE32-NEXT: [[V_CMP_NLG_F16_e64_:%[0-9]+]]:sreg_32_xm0_xexec = nofpexcept V_CMP_NLG_F16_e64 0, [[COPY]], 0, [[COPY1]], 0, implicit $mode, implicit $exec
     ; WAVE32-NEXT: S_ENDPGM 0, implicit [[V_CMP_NLG_F16_e64_]]
     ;
-    ; GFX11-LABEL: name: fcmp_ueq_s16_vv
-    ; GFX11: liveins: $vgpr0, $vgpr1
-    ; GFX11-NEXT: {{  $}}
-    ; GFX11-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
-    ; GFX11-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
-    ; GFX11-NEXT: [[V_CMP_NLG_F16_fake16_e64_:%[0-9]+]]:sreg_32_xm0_xexec = nofpexcept V_CMP_NLG_F16_fake16_e64 0, [[COPY]], 0, [[COPY1]], 0, implicit $mode, implicit $exec
-    ; GFX11-NEXT: S_ENDPGM 0, implicit [[V_CMP_NLG_F16_fake16_e64_]]
     %0:vgpr(s32) = COPY $vgpr0
     %1:vgpr(s32) = COPY $vgpr1
     %2:vgpr(s16) = G_TRUNC %0
@@ -422,13 +349,6 @@ body: |
     ; WAVE32-NEXT: [[V_CMP_NLE_F16_e64_:%[0-9]+]]:sreg_32_xm0_xexec = nofpexcept V_CMP_NLE_F16_e64 0, [[COPY]], 0, [[COPY1]], 0, implicit $mode, implicit $exec
     ; WAVE32-NEXT: S_ENDPGM 0, implicit [[V_CMP_NLE_F16_e64_]]
     ;
-    ; GFX11-LABEL: name: fcmp_ugt_s16_vv
-    ; GFX11: liveins: $vgpr0, $vgpr1
-    ; GFX11-NEXT: {{  $}}
-    ; GFX11-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
-    ; GFX11-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
-    ; GFX11-NEXT: [[V_CMP_NLE_F16_fake16_e64_:%[0-9]+]]:sreg_32_xm0_xexec = nofpexcept V_CMP_NLE_F16_fake16_e64 0, [[COPY]], 0, [[COPY1]], 0, implicit $mode, implicit $exec
-    ; GFX11-NEXT: S_ENDPGM 0, implicit [[V_CMP_NLE_F16_fake16_e64_]]
     %0:vgpr(s32) = COPY $vgpr0
     %1:vgpr(s32) = COPY $vgpr1
     %2:vgpr(s16) = G_TRUNC %0
@@ -461,13 +381,6 @@ body: |
     ; WAVE32-NEXT: [[V_CMP_NLT_F16_e64_:%[0-9]+]]:sreg_32_xm0_xexec = nofpexcept V_CMP_NLT_F16_e64 0, [[COPY]], 0, [[COPY1]], 0, implicit $mode, implicit $exec
     ; WAVE32-NEXT: S_ENDPGM 0, implicit [[V_CMP_NLT_F16_e64_]]
     ;
-    ; GFX11-LABEL: name: fcmp_uge_s16_vv
-    ; GFX11: liveins: $vgpr0, $vgpr1
-    ; GFX11-NEXT: {{  $}}
-    ; GFX11-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
-    ; GFX11-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
-    ; GFX11-NEXT: [[V_CMP_NLT_F16_fake16_e64_:%[0-9]+]]:sreg_32_xm0_xexec = nofpexcept V_CMP_NLT_F16_fake16_e64 0, [[COPY]], 0, [[COPY1]], 0, implicit $mode, implicit $exec
-    ; GFX11-NEXT: S_ENDPGM 0, implicit [[V_CMP_NLT_F16_fake16_e64_]]
     %0:vgpr(s32) = COPY $vgpr0
     %1:vgpr(s32) = COPY $vgpr1
     %2:vgpr(s16) = G_TRUNC %0
@@ -500,13 +413,6 @@ body: |
     ; WAVE32-NEXT: [[V_CMP_NGE_F16_e64_:%[0-9]+]]:sreg_32_xm0_xexec = nofpexcept V_CMP_NGE_F16_e64 0, [[COPY]], 0, [[COPY1]], 0, implicit $mode, implicit $exec
     ; WAVE32-NEXT: S_ENDPGM 0, implicit [[V_CMP_NGE_F16_e64_]]
     ;
-    ; GFX11-LABEL: name: fcmp_ult_s16_vv
-    ; GFX11: liveins: $vgpr0, $vgpr1
-    ; GFX11-NEXT: {{  $}}
-    ; GFX11-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
-    ; GFX11-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
-    ; GFX11-NEXT: [[V_CMP_NGE_F16_fake16_e64_:%[0-9]+]]:sreg_32_xm0_xexec = nofpexcept V_CMP_NGE_F16_fake16_e64 0, [[COPY]], 0, [[COPY1]], 0, implicit $mode, implicit $exec
-    ; GFX11-NEXT: S_ENDPGM 0, implicit [[V_CMP_NGE_F16_fake16_e64_]]
     %0:vgpr(s32) = COPY $vgpr0
     %1:vgpr(s32) = COPY $vgpr1
     %2:vgpr(s16) = G_TRUNC %0
@@ -539,13 +445,6 @@ body: |
     ; WAVE32-NEXT: [[V_CMP_NGT_F16_e64_:%[0-9]+]]:sreg_32_xm0_xexec = nofpexcept V_CMP_NGT_F16_e64 0, [[COPY]], 0, [[COPY1]], 0, implicit $mode, implicit $exec
     ; WAVE32-NEXT: S_ENDPGM 0, implicit [[V_CMP_NGT_F16_e64_]]
     ;
-    ; GFX11-LABEL: name: fcmp_ule_s16_vv
-    ; GFX11: liveins: $vgpr0, $vgpr1
-    ; GFX11-NEXT: {{  $}}
-    ; GFX11-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
-    ; GFX11-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
-    ; GFX11-NEXT: [[V_CMP_NGT_F16_fake16_e64_:%[0-9]+]]:sreg_32_xm0_xexec = nofpexcept V_CMP_NGT_F16_fake16_e64 0, [[COPY]], 0, [[COPY1]], 0, implicit $mode, implicit $exec
-    ; GFX11-NEXT: S_ENDPGM 0, implicit [[V_CMP_NGT_F16_fake16_e64_]]
     %0:vgpr(s32) = COPY $vgpr0
     %1:vgpr(s32) = COPY $vgpr1
     %2:vgpr(s16) = G_TRUNC %0
@@ -578,13 +477,6 @@ body: |
     ; WAVE32-NEXT: [[V_CMP_NEQ_F16_e64_:%[0-9]+]]:sreg_32_xm0_xexec = nofpexcept V_CMP_NEQ_F16_e64 0, [[COPY]], 0, [[COPY1]], 0, implicit $mode, implicit $exec
     ; WAVE32-NEXT: S_ENDPGM 0, implicit [[V_CMP_NEQ_F16_e64_]]
     ;
-    ; GFX11-LABEL: name: fcmp_une_s16_vv
-    ; GFX11: liveins: $vgpr0, $vgpr1
-    ; GFX11-NEXT: {{  $}}
-    ; GFX11-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
-    ; GFX11-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
-    ; GFX11-NEXT: [[V_CMP_NEQ_F16_fake16_e64_:%[0-9]+]]:sreg_32_xm0_xexec = nofpexcept V_CMP_NEQ_F16_fake16_e64 0, [[COPY]], 0, [[COPY1]], 0, implicit $mode, implicit $exec
-    ; GFX11-NEXT: S_ENDPGM 0, implicit [[V_CMP_NEQ_F16_fake16_e64_]]
     %0:vgpr(s32) = COPY $vgpr0
     %1:vgpr(s32) = COPY $vgpr1
     %2:vgpr(s16) = G_TRUNC %0
@@ -621,15 +513,6 @@ body: |
     ; WAVE32-NEXT: [[FCMP:%[0-9]+]]:vcc(s1) = G_FCMP floatpred(true), [[TRUNC]](s16), [[TRUNC1]]
     ; WAVE32-NEXT: S_ENDPGM 0, implicit [[FCMP]](s1)
     ;
-    ; GFX11-LABEL: name: fcmp_true_s16_vv
-    ; GFX11: liveins: $vgpr0, $vgpr1
-    ; GFX11-NEXT: {{  $}}
-    ; GFX11-NEXT: [[COPY:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
-    ; GFX11-NEXT: [[COPY1:%[0-9]+]]:vgpr(s32) = COPY $vgpr1
-    ; GFX11-NEXT: [[TRUNC:%[0-9]+]]:vgpr(s16) = G_TRUNC [[COPY]](s32)
-    ; GFX11-NEXT: [[TRUNC1:%[0-9]+]]:vgpr(s16) = G_TRUNC [[COPY1]](s32)
-    ; GFX11-NEXT: [[FCMP:%[0-9]+]]:vcc(s1) = G_FCMP floatpred(true), [[TRUNC]](s16), [[TRUNC1]]
-    ; GFX11-NEXT: S_ENDPGM 0, implicit [[FCMP]](s1)
     %0:vgpr(s32) = COPY $vgpr0
     %1:vgpr(s32) = COPY $vgpr1
     %2:vgpr(s16) = G_TRUNC %0
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-call.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-call.ll
index 7691f4c30de04..78f33a174980d 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-call.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-call.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-; RUN: llc -global-isel -stop-after=irtranslator -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx900 -verify-machineinstrs -o - %s | FileCheck -enable-var-scope %s
+; RUN: llc -global-isel -global-isel-abort=2 -stop-after=irtranslator -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx900 -verify-machineinstrs -o - %s | FileCheck -enable-var-scope %s
 
 declare hidden void @external_void_func_void() #0
 
@@ -5594,48 +5594,14 @@ define void @test_call_external_void_func_f16_inreg(half inreg %arg) #0 {
 
 define void @test_call_external_void_func_bf16_inreg(bfloat inreg %arg) #0 {
   ; CHECK-LABEL: name: test_call_external_void_func_bf16_inreg
-  ; CHECK: bb.1 (%ir-block.0):
-  ; CHECK-NEXT:   liveins: $sgpr12, $sgpr13, $sgpr14, $sgpr15, $sgpr16, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9, $sgpr10_sgpr11
+  ; CHECK: bb.0:
+  ; CHECK-NEXT:   successors: %bb.1(0x80000000)
+  ; CHECK-NEXT:   liveins: $sgpr16
   ; CHECK-NEXT: {{  $}}
-  ; CHECK-NEXT:   [[COPY:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr31
-  ; CHECK-NEXT:   [[COPY1:%[0-9]+]]:sgpr_32 = COPY $sgpr15
-  ; CHECK-NEXT:   [[COPY2:%[0-9]+]]:sgpr_32 = COPY $sgpr14
-  ; CHECK-NEXT:   [[COPY3:%[0-9]+]]:sgpr_32 = COPY $sgpr13
-  ; CHECK-NEXT:   [[COPY4:%[0-9]+]]:sgpr_32 = COPY $sgpr12
-  ; CHECK-NEXT:   [[COPY5:%[0-9]+]]:sgpr_64 = COPY $sgpr10_sgpr11
-  ; CHECK-NEXT:   [[COPY6:%[0-9]+]]:sgpr_64 = COPY $sgpr8_sgpr9
-  ; CHECK-NEXT:   [[COPY7:%[0-9]+]]:sgpr_64 = COPY $sgpr6_sgpr7
-  ; CHECK-NEXT:   [[COPY8:%[0-9]+]]:sgpr_64 = COPY $sgpr4_sgpr5
-  ; CHECK-NEXT:   [[COPY9:%[0-9]+]]:_(s32) = COPY $sgpr16
-  ; CHECK-NEXT:   [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY9]](s32)
-  ; CHECK-NEXT:   ADJCALLSTACKUP 0, 0, implicit-def $scc
-  ; CHECK-NEXT:   [[GV:%[0-9]+]]:_(p0) = G_GLOBAL_VALUE @external_void_func_bf16_inreg
-  ; CHECK-NEXT:   [[COPY10:%[0-9]+]]:_(p4) = COPY [[COPY8]]
-  ; CHECK-NEXT:   [[COPY11:%[0-9]+]]:_(p4) = COPY [[COPY7]]
-  ; CHECK-NEXT:   [[COPY12:%[0-9]+]]:_(p4) = COPY [[COPY6]]
-  ; CHECK-NEXT:   [[COPY13:%[0-9]+]]:_(s64) = COPY [[COPY5]]
-  ; CHECK-NEXT:   [[COPY14:%[0-9]+]]:_(s32) = COPY [[COPY4]]
-  ; CHECK-NEXT:   [[COPY15:%[0-9]+]]:_(s32) = COPY [[COPY3]]
-  ; CHECK-NEXT:   [[COPY16:%[0-9]+]]:_(s32) = COPY [[COPY2]]
-  ; CHECK-NEXT:   [[COPY17:%[0-9]+]]:_(s32) = COPY [[COPY1]]
-  ; CHECK-NEXT:   [[COPY18:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
-  ; CHECK-NEXT:   [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[TRUNC]](s16)
-  ; CHECK-NEXT:   [[INTRINSIC_CONVERGENT:%[0-9]+]]:_(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[ANYEXT]](s32)
-  ; CHECK-NEXT:   $sgpr0 = COPY [[INTRINSIC_CONVERGENT]](s32)
-  ; CHECK-NEXT:   [[COPY19:%[0-9]+]]:_(<4 x s32>) = COPY $sgpr0_sgpr1_sgpr2_sgpr3
-  ; CHECK-NEXT:   $sgpr0_sgpr1_sgpr2_sgpr3 = COPY [[COPY19]](<4 x s32>)
-  ; CHECK-NEXT:   $sgpr4_sgpr5 = COPY [[COPY10]](p4)
-  ; CHECK-NEXT:   $sgpr6_sgpr7 = COPY [[COPY11]](p4)
-  ; CHECK-NEXT:   $sgpr8_sgpr9 = COPY [[COPY12]](p4)
-  ; CHECK-NEXT:   $sgpr10_sgpr11 = COPY [[COPY13]](s64)
-  ; CHECK-NEXT:   $sgpr12 = COPY [[COPY14]](s32)
-  ; CHECK-NEXT:   $sgpr13 = COPY [[COPY15]](s32)
-  ; CHECK-NEXT:   $sgpr14 = COPY [[COPY16]](s32)
-  ; CHECK-NEXT:   $sgpr15 = COPY [[COPY17]](s32)
-  ; CHECK-NEXT:   $vgpr31 = COPY [[COPY18]](s32)
-  ; CHECK-NEXT:   $sgpr30_sgpr31 = noconvergent G_SI_CALL [[GV]](p0), @external_void_func_bf16_inreg, csr_amdgpu, implicit $sgpr0, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $sgpr4_sgpr5, implicit $sgpr6_sgpr7, implicit $sgpr8_sgpr9, implicit $sgpr10_sgpr11, implicit $sgpr12, implicit $sgpr13, implicit $sgpr14, implicit $sgpr15, implicit $vgpr31
-  ; CHECK-NEXT:   ADJCALLSTACKDOWN 0, 0, implicit-def $scc
-  ; CHECK-NEXT:   SI_RETURN
+  ; CHECK-NEXT:   [[COPY:%[0-9]+]]:_(s32) = COPY $sgpr16
+  ; CHECK-NEXT:   [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY]](s32)
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.1 (%ir-block.0):
   call void @external_void_func_bf16_inreg(bfloat inreg %arg)
   ret void
 }
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fshl.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fshl.mir
index 70ed6ffcd2aa9..240036207bd0d 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fshl.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fshl.mir
@@ -164,7 +164,7 @@ body: |
     ; SI-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[LSHR]], [[C3]]
     ; SI-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[AND3]], [[ZEXT1]](s32)
     ; SI-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](s32)
-    ; SI-NEXT: [[OR:%[0-9]+]]:_(s16) = G_OR [[TRUNC1]], [[TRUNC2]]
+    ; SI-NEXT: [[OR:%[0-9]+]]:_(s16) = disjoint G_OR [[TRUNC1]], [[TRUNC2]]
     ; SI-NEXT: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[OR]](s16)
     ; SI-NEXT: $vgpr0 = COPY [[ANYEXT]](s32)
     ;
@@ -186,7 +186,7 @@ body: |
     ; VI-NEXT: [[SHL:%[0-9]+]]:_(s16) = G_SHL [[TRUNC]], [[AND]](s16)
     ; VI-NEXT: [[LSHR:%[0-9]+]]:_(s16) = G_LSHR [[TRUNC1]], [[C2]](s16)
     ; VI-NEXT: [[LSHR1:%[0-9]+]]:_(s16) = G_LSHR [[LSHR]], [[AND1]](s16)
-    ; VI-NEXT: [[OR:%[0-9]+]]:_(s16) = G_OR [[SHL]], [[LSHR1]]
+    ; VI-NEXT: [[OR:%[0-9]+]]:_(s16) = disjoint G_OR [[SHL]], [[LSHR1]]
     ; VI-NEXT: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[OR]](s16)
     ; VI-NEXT: $vgpr0 = COPY [[ANYEXT]](s32)
     ;
@@ -208,7 +208,7 @@ body: |
     ; GFX9-NEXT: [[SHL:%[0-9]+]]:_(s16) = G_SHL [[TRUNC]], [[AND]](s16)
     ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(s16) = G_LSHR [[TRUNC1]], [[C2]](s16)
     ; GFX9-NEXT: [[LSHR1:%[0-9]+]]:_(s16) = G_LSHR [[LSHR]], [[AND1]](s16)
-    ; GFX9-NEXT: [[OR:%[0-9]+]]:_(s16) = G_OR [[SHL]], [[LSHR1]]
+    ; GFX9-NEXT: [[OR:%[0-9]+]]:_(s16) = disjoint G_OR [[SHL]], [[LSHR1]]
     ; GFX9-NEXT: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[OR]](s16)
     ; GFX9-NEXT: $vgpr0 = COPY [[ANYEXT]](s32)
     %0:_(s32) = COPY $vgpr0
@@ -258,7 +258,7 @@ body: |
     ; SI-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[AND1]](s16)
     ; SI-NEXT: [[LSHR4:%[0-9]+]]:_(s32) = G_LSHR [[LSHR3]], [[ZEXT1]](s32)
     ; SI-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR4]](s32)
-    ; SI-NEXT: [[OR:%[0-9]+]]:_(s16) = G_OR [[TRUNC2]], [[TRUNC3]]
+    ; SI-NEXT: [[OR:%[0-9]+]]:_(s16) = disjoint G_OR [[TRUNC2]], [[TRUNC3]]
     ; SI-NEXT: [[AND3:%[0-9]+]]:_(s16) = G_AND [[TRUNC1]], [[C1]]
     ; SI-NEXT: [[XOR1:%[0-9]+]]:_(s16) = G_XOR [[TRUNC1]], [[C2]]
     ; SI-NEXT: [[AND4:%[0-9]+]]:_(s16) = G_AND [[XOR1]], [[C1]]
@@ -270,7 +270,7 @@ body: |
     ; SI-NEXT: [[ZEXT3:%[0-9]+]]:_(s32) = G_ZEXT [[AND4]](s16)
     ; SI-NEXT: [[LSHR6:%[0-9]+]]:_(s32) = G_LSHR [[LSHR5]], [[ZEXT3]](s32)
     ; SI-NEXT: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR6]](s32)
-    ; SI-NEXT: [[OR1:%[0-9]+]]:_(s16) = G_OR [[TRUNC4]], [[TRUNC5]]
+    ; SI-NEXT: [[OR1:%[0-9]+]]:_(s16) = disjoint G_OR [[TRUNC4]], [[TRUNC5]]
     ; SI-NEXT: [[ZEXT4:%[0-9]+]]:_(s32) = G_ZEXT [[OR]](s16)
     ; SI-NEXT: [[ZEXT5:%[0-9]+]]:_(s32) = G_ZEXT [[OR1]](s16)
     ; SI-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[ZEXT5]], [[C]](s32)
@@ -306,14 +306,14 @@ body: |
     ; VI-NEXT: [[SHL:%[0-9]+]]:_(s16) = G_SHL [[TRUNC]], [[AND]](s16)
     ; VI-NEXT: [[LSHR3:%[0-9]+]]:_(s16) = G_LSHR [[TRUNC2]], [[C3]](s16)
     ; VI-NEXT: [[LSHR4:%[0-9]+]]:_(s16) = G_LSHR [[LSHR3]], [[AND1]](s16)
-    ; VI-NEXT: [[OR:%[0-9]+]]:_(s16) = G_OR [[SHL]], [[LSHR4]]
+    ; VI-NEXT: [[OR:%[0-9]+]]:_(s16) = disjoint G_OR [[SHL]], [[LSHR4]]
     ; VI-NEXT: [[AND2:%[0-9]+]]:_(s16) = G_AND [[TRUNC5]], [[C1]]
     ; VI-NEXT: [[XOR1:%[0-9]+]]:_(s16) = G_XOR [[TRUNC5]], [[C2]]
     ; VI-NEXT: [[AND3:%[0-9]+]]:_(s16) = G_AND [[XOR1]], [[C1]]
     ; VI-NEXT: [[SHL1:%[0-9]+]]:_(s16) = G_SHL [[TRUNC1]], [[AND2]](s16)
     ; VI-NEXT: [[LSHR5:%[0-9]+]]:_(s16) = G_LSHR [[TRUNC3]], [[C3]](s16)
     ; VI-NEXT: [[LSHR6:%[0-9]+]]:_(s16) = G_LSHR [[LSHR5]], [[AND3]](s16)
-    ; VI-NEXT: [[OR1:%[0-9]+]]:_(s16) = G_OR [[SHL1]], [[LSHR6]]
+    ; VI-NEXT: [[OR1:%[0-9]+]]:_(s16) = disjoint G_OR [[SHL1]], [[LSHR6]]
     ; VI-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[OR]](s16)
     ; VI-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[OR1]](s16)
     ; VI-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C]](s32)
@@ -339,7 +339,7 @@ body: |
     ; GFX9-NEXT: [[SHL:%[0-9]+]]:_(<2 x s16>) = G_SHL [[COPY]], [[AND]](<2 x s16>)
     ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(<2 x s16>) = G_LSHR [[COPY1]], [[BUILD_VECTOR2]](<2 x s16>)
     ; GFX9-NEXT: [[LSHR1:%[0-9]+]]:_(<2 x s16>) = G_LSHR [[LSHR]], [[AND1]](<2 x s16>)
-    ; GFX9-NEXT: [[OR:%[0-9]+]]:_(<2 x s16>) = G_OR [[SHL]], [[LSHR1]]
+    ; GFX9-NEXT: [[OR:%[0-9]+]]:_(<2 x s16>) = disjoint G_OR [[SHL]], [[LSHR1]]
     ; GFX9-NEXT: $vgpr0 = COPY [[OR]](<2 x s16>)
     %0:_(<2 x s16>) = COPY $vgpr0
     %1:_(<2 x s16>) = COPY $vgpr1
@@ -371,7 +371,7 @@ body: |
     ; SI-NEXT: [[LSHR:%[0-9]+]]:_(s64) = G_LSHR [[COPY1]], [[C2]](s32)
     ; SI-NEXT: [[TRUNC1:%[0-9]+]]:_(s32) = G_TRUNC [[AND1]](s64)
     ; SI-NEXT: [[LSHR1:%[0-9]+]]:_(s64) = G_LSHR [[LSHR]], [[TRUNC1]](s32)
-    ; SI-NEXT: [[OR:%[0-9]+]]:_(s64) = G_OR [[SHL]], [[LSHR1]]
+    ; SI-NEXT: [[OR:%[0-9]+]]:_(s64) = disjoint G_OR [[SHL]], [[LSHR1]]
     ; SI-NEXT: $vgpr0_vgpr1 = COPY [[OR]](s64)
     ;
     ; VI-LABEL: name: test_fshl_s64_s64
@@ -391,7 +391,7 @@ body: |
     ; VI-NEXT: [[LSHR:%[0-9]+]]:_(s64) = G_LSHR [[COPY1]], [[C2]](s32)
     ; VI-NEXT: [[TRUNC1:%[0-9]+]]:_(s32) = G_TRUNC [[AND1]](s64)
     ; VI-NEXT: [[LSHR1:%[0-9]+]]:_(s64) = G_LSHR [[LSHR]], [[TRUNC1]](s32)
-    ; VI-NEXT: [[OR:%[0-9]+]]:_(s64) = G_OR [[SHL]], [[LSHR1]]
+    ; VI-NEXT: [[OR:%[0-9]+]]:_(s64) = disjoint G_OR [[SHL]], [[LSHR1]]
     ; VI-NEXT: $vgpr0_vgpr1 = COPY [[OR]](s64)
     ;
     ; GFX9-LABEL: name: test_fshl_s64_s64
@@ -411,7 +411,7 @@ body: |
     ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(s64) = G_LSHR [[COPY1]], [[C2]](s32)
     ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(s32) = G_TRUNC [[AND1]](s64)
     ; GFX9-NEXT: [[LSHR1:%[0-9]+]]:_(s64) = G_LSHR [[LSHR]], [[TRUNC1]](s32)
-    ; GFX9-NEXT: [[OR:%[0-9]+]]:_(s64) = G_OR [[SHL]], [[LSHR1]]
+    ; GFX9-NEXT: [[OR:%[0-9]+]]:_(s64) = disjoint G_OR [[SHL]], [[LSHR1]]
     ; GFX9-NEXT: $vgpr0_vgpr1 = COPY [[OR]](s64)
     %0:_(s64) = COPY $vgpr0_vgpr1
     %1:_(s64) = COPY $vgpr2_vgpr3
@@ -444,7 +444,7 @@ body: |
     ; SI-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[AND2]], [[C2]](s32)
     ; SI-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[LSHR]], [[C3]]
     ; SI-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[AND3]], [[AND1]](s32)
-    ; SI-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL]], [[LSHR1]]
+    ; SI-NEXT: [[OR:%[0-9]+]]:_(s32) = disjoint G_OR [[SHL]], [[LSHR1]]
     ; SI-NEXT: $vgpr0 = COPY [[OR]](s32)
     ;
     ; VI-LABEL: name: test_fshl_s8_s8
@@ -473,7 +473,7 @@ body: |
     ; VI-NEXT: [[LSHR1:%[0-9]+]]:_(s16) = G_LSHR [[AND5]], [[AND4]](s16)
     ; VI-NEXT: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[SHL]](s16)
     ; VI-NEXT: [[ANYEXT1:%[0-9]+]]:_(s32) = G_ANYEXT [[LSHR1]](s16)
-    ; VI-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[ANYEXT]], [[ANYEXT1]]
+    ; VI-NEXT: [[OR:%[0-9]+]]:_(s32) = disjoint G_OR [[ANYEXT]], [[ANYEXT1]]
     ; VI-NEXT: $vgpr0 = COPY [[OR]](s32)
     ;
     ; GFX9-LABEL: name: test_fshl_s8_s8
@@ -502,7 +502,7 @@ body: |
     ; GFX9-NEXT: [[LSHR1:%[0-9]+]]:_(s16) = G_LSHR [[AND5]], [[AND4]](s16)
     ; GFX9-NEXT: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[SHL]](s16)
     ; GFX9-NEXT: [[ANYEXT1:%[0-9]+]]:_(s32) = G_ANYEXT [[LSHR1]](s16)
-    ; GFX9-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[ANYEXT]], [[ANYEXT1]]
+    ; GFX9-NEXT: [[OR:%[0-9]+]]:_(s32) = disjoint G_OR [[ANYEXT]], [[ANYEXT1]]
     ; GFX9-NEXT: $vgpr0 = COPY [[OR]](s32)
     %0:_(s32) = COPY $vgpr0
     %1:_(s32) = COPY $vgpr1
@@ -559,7 +559,7 @@ body: |
     ; SI-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[SUB4]], [[C2]]
     ; SI-NEXT: [[AND4:%[0-9]+]]:_(s32) = G_AND [[LSHR]], [[C2]]
     ; SI-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[AND4]], [[AND3]](s32)
-    ; SI-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL]], [[LSHR1]]
+    ; SI-NEXT: [[OR:%[0-9]+]]:_(s32) = disjoint G_OR [[SHL]], [[LSHR1]]
     ; SI-NEXT: $vgpr0 = COPY [[OR]](s32)
     ;
     ; VI-LABEL: name: test_fshl_s24_s24
@@ -600,7 +600,7 @@ body: |
     ; VI-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[SUB4]], [[C2]]
     ; VI-NEXT: [[AND4:%[0-9]+]]:_(s32) = G_AND [[LSHR]], [[C2]]
     ; VI-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[AND4]], [[AND3]](s32)
-    ; VI-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL]], [[LSHR1]]
+    ; VI-NEXT: [[OR:%[0-9]+]]:_(s32) = disjoint G_OR [[SHL]], [[LSHR1]]
     ; VI-NEXT: $vgpr0 = COPY [[OR]](s32)
     ;
     ; GFX9-LABEL: name: test_fshl_s24_s24
@@ -641,7 +641,7 @@ body: |
     ; GFX9-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[SUB4]], [[C2]]
     ; GFX9-NEXT: [[AND4:%[0-9]+]]:_(s32) = G_AND [[LSHR]], [[C2]]
     ; GFX9-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[AND4]], [[AND3]](s32)
-    ; GFX9-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL]], [[LSHR1]]
+    ; GFX9-NEXT: [[OR:%[0-9]+]]:_(s32) = disjoint G_OR [[SHL]], [[LSHR1]]
     ; GFX9-NEXT: $vgpr0 = COPY [[OR]](s32)
     %0:_(s32) = COPY $vgpr0
     %1:_(s32) = COPY $vgpr1
@@ -697,7 +697,7 @@ body: |
     ; SI-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[AND1]](s16)
     ; SI-NEXT: [[LSHR4:%[0-9]+]]:_(s32) = G_LSHR [[LSHR3]], [[ZEXT1]](s32)
     ; SI-NEXT: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR4]](s32)
-    ; SI-NEXT: [[OR:%[0-9]+]]:_(s16) = G_OR [[TRUNC3]], [[TRUNC4]]
+    ; SI-NEXT: [[OR:%[0-9]+]]:_(s16) = disjoint G_OR [[TRUNC3]], [[TRUNC4]]
     ; SI-NEXT: [[AND3:%[0-9]+]]:_(s16) = G_AND [[TRUNC1]], [[C1]]
     ; SI-NEXT: [[XOR1:%[0-9]+]]:_(s16) = G_XOR [[TRUNC1]], [[C2]]
     ; SI-NEXT: [[AND4:%[0-9]+]]:_(s16) = G_AND [[XOR1]], [[C1]]
@@ -709,7 +709,7 @@ body: |
     ; SI-NEXT: [[ZEXT3:%[0-9]+]]:_(s32) = G_ZEXT [[AND4]](s16)
     ; SI-NEXT: [[LSHR6:%[0-9]+]]:_(s32) = G_LSHR [[LSHR5]], [[ZEXT3]](s32)
     ; SI-NEXT: [[TRUNC6:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR6]](s32)
-    ; SI-NEXT: [[OR1:%[0-9]+]]:_(s16) = G_OR [[TRUNC5]], [[TRUNC6]]
+    ; SI-NEXT: [[OR1:%[0-9]+]]:_(s16) = disjoint G_OR [[TRUNC5]], [[TRUNC6]]
     ; SI-NEXT: [[AND5:%[0-9]+]]:_(s16) = G_AND [[TRUNC2]], [[C1]]
     ; SI-NEXT: [[XOR2:%[0-9]+]]:_(s16) = G_XOR [[TRUNC2]], [[C2]]
     ; SI-NEXT: [[AND6:%[0-9]+]]:_(s16) = G_AND [[XOR2]], [[C1]]
@@ -722,7 +722,7 @@ body: |
     ; SI-NEXT: [[ZEXT5:%[0-9]+]]:_(s32) = G_ZEXT [[AND6]](s16)
     ; SI-NEXT: [[LSHR8:%[0-9]+]]:_(s32) = G_LSHR [[LSHR7]], [[ZEXT5]](s32)
     ; SI-NEXT: [[TRUNC8:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR8]](s32)
-    ; SI-NEXT: [[OR2:%[0-9]+]]:_(s16) = G_OR [[TRUNC7]], [[TRUNC8]]
+    ; SI-NEXT: [[OR2:%[0-9]+]]:_(s16) = disjoint G_OR [[TRUNC7]], [[TRUNC8]]
     ; SI-NEXT: [[DEF:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
     ; SI-NEXT: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[DEF]](<4 x s16>)
     ; SI-NEXT: [[BITCAST6:%[0-9]+]]:_(s32) = G_BITCAST [[UV]](<2 x s16>)
@@ -783,21 +783,21 @@ body: |
     ; VI-NEXT: [[SHL:%[0-9]+]]:_(s16) = G_SHL [[TRUNC]], [[AND]](s16)
     ; VI-NEXT: [[LSHR3:%[0-9]+]]:_(s16) = G_LSHR [[TRUNC3]], [[C3]](s16)
     ; VI-NEXT: [[LSHR4:%[0-9]+]]:_(s16) = G_LSHR [[LSHR3]], [[AND1]](s16)
-    ; VI-NEXT: [[OR:%[0-9]+]]:_(s16) = G_OR [[SHL]], [[LSHR4]]
+    ; VI-NEXT: [[OR:%[0-9]+]]:_(s16) = disjoint G_OR [[SHL]], [[LSHR4]]
     ; VI-NEXT: [[AND2:%[0-9]+]]:_(s16) = G_AND [[TRUNC7]], [[C1]]
     ; VI-NEXT: [[XOR1:%[0-9]+]]:_(s16) = G_XOR [[TRUNC7]], [[C2]]
     ; VI-NEXT: [[AND3:%[0-9]+]]:_(s16) = G_AND [[XOR1]], [[C1]]
     ; VI-NEXT: [[SHL1:%[0-9]+]]:_(s16) = G_SHL [[TRUNC1]], [[AND2]](s16)
     ; VI-NEXT: [[LSHR5:%[0-9]+]]:_(s16) = G_LSHR [[TRUNC4]], [[C3]](s16)
     ; VI-NEXT: [[LSHR6:%[0-9]+]]:_(s16) = G_LSHR [[LSHR5]], [[AND3]](s16)
-    ; VI-NEXT: [[OR1:%[0-9]+]]:_(s16) = G_OR [[SHL1]], [[LSHR6]]
+    ; VI-NEXT: [[OR1:%[0-9]+]]:_(s16) = disjoint G_OR [[SHL1]], [[LSHR6]]
     ; VI-NEXT: [[AND4:%[0-9]+]]:_(s16) = G_AND [[TRUNC8]], [[C1]]
     ; VI-NEXT: [[XOR2:%[0-9]+]]:_(s16) = G_XOR [[TRUNC8]], [[C2]]
     ; VI-NEXT: [[AND5:%[0-9]+]]:_(s16) = G_AND [[XOR2]], [[C1]]
     ; VI-NEXT: [[SHL2:%[0-9]+]]:_(s16) = G_SHL [[TRUNC2]], [[AND4]](s16)
     ; VI-NEXT: [[LSHR7:%[0-9]+]]:_(s16) = G_LSHR [[TRUNC5]], [[C3]](s16)
     ; VI-NEXT: [[LSHR8:%[0-9]+]]:_(s16) = G_LSHR [[LSHR7]], [[AND5]](s16)
-    ; VI-NEXT: [[OR2:%[0-9]+]]:_(s16) = G_OR [[SHL2]], [[LSHR8]]
+    ; VI-NEXT: [[OR2:%[0-9]+]]:_(s16) = disjoint G_OR [[SHL2]], [[LSHR8]]
     ; VI-NEXT: [[DEF:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
     ; VI-NEXT: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[DEF]](<4 x s16>)
     ; VI-NEXT: [[BITCAST6:%[0-9]+]]:_(s32) = G_BITCAST [[UV]](<2 x s16>)
@@ -843,7 +843,7 @@ body: |
     ; GFX9-NEXT: [[SHL:%[0-9]+]]:_(<2 x s16>) = G_SHL [[COPY]], [[AND]](<2 x s16>)
     ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(<2 x s16>) = G_LSHR [[COPY2]], [[BUILD_VECTOR2]](<2 x s16>)
     ; GFX9-NEXT: [[LSHR1:%[0-9]+]]:_(<2 x s16>) = G_LSHR [[LSHR]], [[AND1]](<2 x s16>)
-    ; GFX9-NEXT: [[OR:%[0-9]+]]:_(<2 x s16>) = G_OR [[SHL]], [[LSHR1]]
+    ; GFX9-NEXT: [[OR:%[0-9]+]]:_(<2 x s16>) = disjoint G_OR [[SHL]], [[LSHR1]]
     ; GFX9-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[C]](s16), [[C]](s16)
     ; GFX9-NEXT: [[AND2:%[0-9]+]]:_(<2 x s16>) = G_AND [[COPY5]], [[BUILD_VECTOR3]]
     ; GFX9-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[C1]](s16), [[C1]](s16)
@@ -853,7 +853,7 @@ body: |
     ; GFX9-NEXT: [[SHL1:%[0-9]+]]:_(<2 x s16>) = G_SHL [[COPY1]], [[AND2]](<2 x s16>)
     ; GFX9-NEXT: [[LSHR2:%[0-9]+]]:_(<2 x s16>) = G_LSHR [[COPY3]], [[BUILD_VECTOR5]](<2 x s16>)
     ; GFX9-NEXT: [[LSHR3:%[0-9]+]]:_(<2 x s16>) = G_LSHR [[LSHR2]], [[AND3]](<2 x s16>)
-    ; GFX9-NEXT: [[OR1:%[0-9]+]]:_(<2 x s16>) = G_OR [[SHL1]], [[LSHR3]]
+    ; GFX9-NEXT: [[OR1:%[0-9]+]]:_(<2 x s16>) = disjoint G_OR [[SHL1]], [[LSHR3]]
     ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[OR1]](<2 x s16>)
     ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
     ; GFX9-NEXT: [[DEF:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
@@ -939,7 +939,7 @@ body: |
     ; SI-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[AND1]](s16)
     ; SI-NEXT: [[LSHR7:%[0-9]+]]:_(s32) = G_LSHR [[LSHR6]], [[ZEXT1]](s32)
     ; SI-NEXT: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR7]](s32)
-    ; SI-NEXT: [[OR:%[0-9]+]]:_(s16) = G_OR [[TRUNC4]], [[TRUNC5]]
+    ; SI-NEXT: [[OR:%[0-9]+]]:_(s16) = disjoint G_OR [[TRUNC4]], [[TRUNC5]]
     ; SI-NEXT: [[AND3:%[0-9]+]]:_(s16) = G_AND [[TRUNC1]], [[C1]]
     ; SI-NEXT: [[XOR1:%[0-9]+]]:_(s16) = G_XOR [[TRUNC1]], [[C2]]
     ; SI-NEXT: [[AND4:%[0-9]+]]:_(s16) = G_AND [[XOR1]], [[C1]]
@@ -951,7 +951,7 @@ body: |
     ; SI-NEXT: [[ZEXT3:%[0-9]+]]:_(s32) = G_ZEXT [[AND4]](s16)
     ; SI-NEXT: [[LSHR9:%[0-9]+]]:_(s32) = G_LSHR [[LSHR8]], [[ZEXT3]](s32)
     ; SI-NEXT: [[TRUNC7:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR9]](s32)
-    ; SI-NEXT: [[OR1:%[0-9]+]]:_(s16) = G_OR [[TRUNC6]], [[TRUNC7]]
+    ; SI-NEXT: [[OR1:%[0-9]+]]:_(s16) = disjoint G_OR [[TRUNC6]], [[TRUNC7]]
     ; SI-NEXT: [[AND5:%[0-9]+]]:_(s16) = G_AND [[TRUNC2]], [[C1]]
     ; SI-NEXT: [[XOR2:%[0-9]+]]:_(s16) = G_XOR [[TRUNC2]], [[C2]]
     ; SI-NEXT: [[AND6:%[0-9]+]]:_(s16) = G_AND [[XOR2]], [[C1]]
@@ -964,7 +964,7 @@ body: |
     ; SI-NEXT: [[ZEXT5:%[0-9]+]]:_(s32) = G_ZEXT [[AND6]](s16)
     ; SI-NEXT: [[LSHR11:%[0-9]+]]:_(s32) = G_LSHR [[LSHR10]], [[ZEXT5]](s32)
     ; SI-NEXT: [[TRUNC9:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR11]](s32)
-    ; SI-NEXT: [[OR2:%[0-9]+]]:_(s16) = G_OR [[TRUNC8]], [[TRUNC9]]
+    ; SI-NEXT: [[OR2:%[0-9]+]]:_(s16) = disjoint G_OR [[TRUNC8]], [[TRUNC9]]
     ; SI-NEXT: [[AND8:%[0-9]+]]:_(s16) = G_AND [[TRUNC3]], [[C1]]
     ; SI-NEXT: [[XOR3:%[0-9]+]]:_(s16) = G_XOR [[TRUNC3]], [[C2]]
     ; SI-NEXT: [[AND9:%[0-9]+]]:_(s16) = G_AND [[XOR3]], [[C1]]
@@ -976,7 +976,7 @@ body: |
     ; SI-NEXT: [[ZEXT7:%[0-9]+]]:_(s32) = G_ZEXT [[AND9]](s16)
     ; SI-NEXT: [[LSHR13:%[0-9]+]]:_(s32) = G_LSHR [[LSHR12]], [[ZEXT7]](s32)
     ; SI-NEXT: [[TRUNC11:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR13]](s32)
-    ; SI-NEXT: [[OR3:%[0-9]+]]:_(s16) = G_OR [[TRUNC10]], [[TRUNC11]]
+    ; SI-NEXT: [[OR3:%[0-9]+]]:_(s16) = disjoint G_OR [[TRUNC10]], [[TRUNC11]]
     ; SI-NEXT: [[ZEXT8:%[0-9]+]]:_(s32) = G_ZEXT [[OR]](s16)
     ; SI-NEXT: [[ZEXT9:%[0-9]+]]:_(s32) = G_ZEXT [[OR1]](s16)
     ; SI-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[ZEXT9]], [[C]](s32)
@@ -1033,28 +1033,28 @@ body: |
     ; VI-NEXT: [[SHL:%[0-9]+]]:_(s16) = G_SHL [[TRUNC]], [[AND]](s16)
     ; VI-NEXT: [[LSHR6:%[0-9]+]]:_(s16) = G_LSHR [[TRUNC4]], [[C3]](s16)
     ; VI-NEXT: [[LSHR7:%[0-9]+]]:_(s16) = G_LSHR [[LSHR6]], [[AND1]](s16)
-    ; VI-NEXT: [[OR:%[0-9]+]]:_(s16) = G_OR [[SHL]], [[LSHR7]]
+    ; VI-NEXT: [[OR:%[0-9]+]]:_(s16) = disjoint G_OR [[SHL]], [[LSHR7]]
     ; VI-NEXT: [[AND2:%[0-9]+]]:_(s16) = G_AND [[TRUNC9]], [[C1]]
     ; VI-NEXT: [[XOR1:%[0-9]+]]:_(s16) = G_XOR [[TRUNC9]], [[C2]]
     ; VI-NEXT: [[AND3:%[0-9]+]]:_(s16) = G_AND [[XOR1]], [[C1]]
     ; VI-NEXT: [[SHL1:%[0-9]+]]:_(s16) = G_SHL [[TRUNC1]], [[AND2]](s16)
     ; VI-NEXT: [[LSHR8:%[0-9]+]]:_(s16) = G_LSHR [[TRUNC5]], [[C3]](s16)
     ; VI-NEXT: [[LSHR9:%[0-9]+]]:_(s16) = G_LSHR [[LSHR8]], [[AND3]](s16)
-    ; VI-NEXT: [[OR1:%[0-9]+]]:_(s16) = G_OR [[SHL1]], [[LSHR9]]
+    ; VI-NEXT: [[OR1:%[0-9]+]]:_(s16) = disjoint G_OR [[SHL1]], [[LSHR9]]
     ; VI-NEXT: [[AND4:%[0-9]+]]:_(s16) = G_AND [[TRUNC10]], [[C1]]
     ; VI-NEXT: [[XOR2:%[0-9]+]]:_(s16) = G_XOR [[TRUNC10]], [[C2]]
     ; VI-NEXT: [[AND5:%[0-9]+]]:_(s16) = G_AND [[XOR2]], [[C1]]
     ; VI-NEXT: [[SHL2:%[0-9]+]]:_(s16) = G_SHL [[TRUNC2]], [[AND4]](s16)
     ; VI-NEXT: [[LSHR10:%[0-9]+]]:_(s16) = G_LSHR [[TRUNC6]], [[C3]](s16)
     ; VI-NEXT: [[LSHR11:%[0-9]+]]:_(s16) = G_LSHR [[LSHR10]], [[AND5]](s16)
-    ; VI-NEXT: [[OR2:%[0-9]+]]:_(s16) = G_OR [[SHL2]], [[LSHR11]]
+    ; VI-NEXT: [[OR2:%[0-9]+]]:_(s16) = disjoint G_OR [[SHL2]], [[LSHR11]]
     ; VI-NEXT: [[AND6:%[0-9]+]]:_(s16) = G_AND [[TRUNC11]], [[C1]]
     ; VI-NEXT: [[XOR3:%[0-9]+]]:_(s16) = G_XOR [[TRUNC11]], [[C2]]
     ; VI-NEXT: [[AND7:%[0-9]+]]:_(s16) = G_AND [[XOR3]], [[C1]]
     ; VI-NEXT: [[SHL3:%[0-9]+]]:_(s16) = G_SHL [[TRUNC3]], [[AND6]](s16)
     ; VI-NEXT: [[LSHR12:%[0-9]+]]:_(s16) = G_LSHR [[TRUNC7]], [[C3]](s16)
     ; VI-NEXT: [[LSHR13:%[0-9]+]]:_(s16) = G_LSHR [[LSHR12]], [[AND7]](s16)
-    ; VI-NEXT: [[OR3:%[0-9]+]]:_(s16) = G_OR [[SHL3]], [[LSHR13]]
+    ; VI-NEXT: [[OR3:%[0-9]+]]:_(s16) = disjoint G_OR [[SHL3]], [[LSHR13]]
     ; VI-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[OR]](s16)
     ; VI-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[OR1]](s16)
     ; VI-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C]](s32)
@@ -1089,7 +1089,7 @@ body: |
     ; GFX9-NEXT: [[SHL:%[0-9]+]]:_(<2 x s16>) = G_SHL [[UV]], [[AND]](<2 x s16>)
     ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(<2 x s16>) = G_LSHR [[UV2]], [[BUILD_VECTOR2]](<2 x s16>)
     ; GFX9-NEXT: [[LSHR1:%[0-9]+]]:_(<2 x s16>) = G_LSHR [[LSHR]], [[AND1]](<2 x s16>)
-    ; GFX9-NEXT: [[OR:%[0-9]+]]:_(<2 x s16>) = G_OR [[SHL]], [[LSHR1]]
+    ; GFX9-NEXT: [[OR:%[0-9]+]]:_(<2 x s16>) = disjoint G_OR [[SHL]], [[LSHR1]]
     ; GFX9-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[C]](s16), [[C]](s16)
     ; GFX9-NEXT: [[AND2:%[0-9]+]]:_(<2 x s16>) = G_AND [[UV5]], [[BUILD_VECTOR3]]
     ; GFX9-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[C1]](s16), [[C1]](s16)
@@ -1099,7 +1099,7 @@ body: |
     ; GFX9-NEXT: [[SHL1:%[0-9]+]]:_(<2 x s16>) = G_SHL [[UV1]], [[AND2]](<2 x s16>)
     ; GFX9-NEXT: [[LSHR2:%[0-9]+]]:_(<2 x s16>) = G_LSHR [[UV3]], [[BUILD_VECTOR5]](<2 x s16>)
     ; GFX9-NEXT: [[LSHR3:%[0-9]+]]:_(<2 x s16>) = G_LSHR [[LSHR2]], [[AND3]](<2 x s16>)
-    ; GFX9-NEXT: [[OR1:%[0-9]+]]:_(<2 x s16>) = G_OR [[SHL1]], [[LSHR3]]
+    ; GFX9-NEXT: [[OR1:%[0-9]+]]:_(<2 x s16>) = disjoint G_OR [[SHL1]], [[LSHR3]]
     ; GFX9-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[OR]](<2 x s16>), [[OR1]](<2 x s16>)
     ; GFX9-NEXT: $vgpr0_vgpr1 = COPY [[CONCAT_VECTORS]](<4 x s16>)
     %0:_(<4 x s16>) = COPY $vgpr0_vgpr1
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fshr.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fshr.mir
index 4502629afd421..fa18012eb43fa 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fshr.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fshr.mir
@@ -126,7 +126,7 @@ body: |
     ; SI-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C3]]
     ; SI-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[AND2]], [[ZEXT1]](s32)
     ; SI-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
-    ; SI-NEXT: [[OR:%[0-9]+]]:_(s16) = G_OR [[TRUNC1]], [[TRUNC2]]
+    ; SI-NEXT: [[OR:%[0-9]+]]:_(s16) = disjoint G_OR [[TRUNC1]], [[TRUNC2]]
     ; SI-NEXT: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[OR]](s16)
     ; SI-NEXT: $vgpr0 = COPY [[ANYEXT]](s32)
     ;
@@ -148,7 +148,7 @@ body: |
     ; VI-NEXT: [[SHL:%[0-9]+]]:_(s16) = G_SHL [[TRUNC]], [[C2]](s16)
     ; VI-NEXT: [[SHL1:%[0-9]+]]:_(s16) = G_SHL [[SHL]], [[AND1]](s16)
     ; VI-NEXT: [[LSHR:%[0-9]+]]:_(s16) = G_LSHR [[TRUNC1]], [[AND]](s16)
-    ; VI-NEXT: [[OR:%[0-9]+]]:_(s16) = G_OR [[SHL1]], [[LSHR]]
+    ; VI-NEXT: [[OR:%[0-9]+]]:_(s16) = disjoint G_OR [[SHL1]], [[LSHR]]
     ; VI-NEXT: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[OR]](s16)
     ; VI-NEXT: $vgpr0 = COPY [[ANYEXT]](s32)
     ;
@@ -170,7 +170,7 @@ body: |
     ; GFX9-NEXT: [[SHL:%[0-9]+]]:_(s16) = G_SHL [[TRUNC]], [[C2]](s16)
     ; GFX9-NEXT: [[SHL1:%[0-9]+]]:_(s16) = G_SHL [[SHL]], [[AND1]](s16)
     ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(s16) = G_LSHR [[TRUNC1]], [[AND]](s16)
-    ; GFX9-NEXT: [[OR:%[0-9]+]]:_(s16) = G_OR [[SHL1]], [[LSHR]]
+    ; GFX9-NEXT: [[OR:%[0-9]+]]:_(s16) = disjoint G_OR [[SHL1]], [[LSHR]]
     ; GFX9-NEXT: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[OR]](s16)
     ; GFX9-NEXT: $vgpr0 = COPY [[ANYEXT]](s32)
     %0:_(s32) = COPY $vgpr0
@@ -217,7 +217,7 @@ body: |
     ; SI-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[AND1]](s16)
     ; SI-NEXT: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[LSHR2]], [[ZEXT1]](s32)
     ; SI-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR3]](s32)
-    ; SI-NEXT: [[OR:%[0-9]+]]:_(s16) = G_OR [[TRUNC]], [[TRUNC1]]
+    ; SI-NEXT: [[OR:%[0-9]+]]:_(s16) = disjoint G_OR [[TRUNC]], [[TRUNC1]]
     ; SI-NEXT: [[AND3:%[0-9]+]]:_(s16) = G_AND [[C]], [[C2]]
     ; SI-NEXT: [[XOR1:%[0-9]+]]:_(s16) = G_XOR [[C]], [[C3]]
     ; SI-NEXT: [[AND4:%[0-9]+]]:_(s16) = G_AND [[XOR1]], [[C2]]
@@ -229,7 +229,7 @@ body: |
     ; SI-NEXT: [[ZEXT3:%[0-9]+]]:_(s32) = G_ZEXT [[AND4]](s16)
     ; SI-NEXT: [[LSHR5:%[0-9]+]]:_(s32) = G_LSHR [[LSHR4]], [[ZEXT3]](s32)
     ; SI-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR5]](s32)
-    ; SI-NEXT: [[OR1:%[0-9]+]]:_(s16) = G_OR [[TRUNC2]], [[TRUNC3]]
+    ; SI-NEXT: [[OR1:%[0-9]+]]:_(s16) = disjoint G_OR [[TRUNC2]], [[TRUNC3]]
     ; SI-NEXT: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[COPY1]](<2 x s16>)
     ; SI-NEXT: [[LSHR6:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C1]](s32)
     ; SI-NEXT: [[COPY4:%[0-9]+]]:_(s32) = COPY [[C4]](s32)
@@ -259,7 +259,7 @@ body: |
     ; SI-NEXT: [[ZEXT5:%[0-9]+]]:_(s32) = G_ZEXT [[AND6]](s16)
     ; SI-NEXT: [[LSHR9:%[0-9]+]]:_(s32) = G_LSHR [[LSHR8]], [[ZEXT5]](s32)
     ; SI-NEXT: [[TRUNC7:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR9]](s32)
-    ; SI-NEXT: [[OR3:%[0-9]+]]:_(s16) = G_OR [[TRUNC6]], [[TRUNC7]]
+    ; SI-NEXT: [[OR3:%[0-9]+]]:_(s16) = disjoint G_OR [[TRUNC6]], [[TRUNC7]]
     ; SI-NEXT: [[AND8:%[0-9]+]]:_(s16) = G_AND [[TRUNC5]], [[C2]]
     ; SI-NEXT: [[XOR4:%[0-9]+]]:_(s16) = G_XOR [[TRUNC5]], [[C3]]
     ; SI-NEXT: [[AND9:%[0-9]+]]:_(s16) = G_AND [[XOR4]], [[C2]]
@@ -273,7 +273,7 @@ body: |
     ; SI-NEXT: [[ZEXT7:%[0-9]+]]:_(s32) = G_ZEXT [[AND9]](s16)
     ; SI-NEXT: [[LSHR11:%[0-9]+]]:_(s32) = G_LSHR [[LSHR10]], [[ZEXT7]](s32)
     ; SI-NEXT: [[TRUNC9:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR11]](s32)
-    ; SI-NEXT: [[OR4:%[0-9]+]]:_(s16) = G_OR [[TRUNC8]], [[TRUNC9]]
+    ; SI-NEXT: [[OR4:%[0-9]+]]:_(s16) = disjoint G_OR [[TRUNC8]], [[TRUNC9]]
     ; SI-NEXT: [[ZEXT8:%[0-9]+]]:_(s32) = G_ZEXT [[OR3]](s16)
     ; SI-NEXT: [[ZEXT9:%[0-9]+]]:_(s32) = G_ZEXT [[OR4]](s16)
     ; SI-NEXT: [[SHL7:%[0-9]+]]:_(s32) = G_SHL [[ZEXT9]], [[C1]](s32)
@@ -305,14 +305,14 @@ body: |
     ; VI-NEXT: [[SHL:%[0-9]+]]:_(s16) = G_SHL [[TRUNC]], [[AND]](s16)
     ; VI-NEXT: [[LSHR2:%[0-9]+]]:_(s16) = G_LSHR [[TRUNC2]], [[C]](s16)
     ; VI-NEXT: [[LSHR3:%[0-9]+]]:_(s16) = G_LSHR [[LSHR2]], [[AND1]](s16)
-    ; VI-NEXT: [[OR:%[0-9]+]]:_(s16) = G_OR [[SHL]], [[LSHR3]]
+    ; VI-NEXT: [[OR:%[0-9]+]]:_(s16) = disjoint G_OR [[SHL]], [[LSHR3]]
     ; VI-NEXT: [[AND2:%[0-9]+]]:_(s16) = G_AND [[C]], [[C2]]
     ; VI-NEXT: [[XOR1:%[0-9]+]]:_(s16) = G_XOR [[C]], [[C3]]
     ; VI-NEXT: [[AND3:%[0-9]+]]:_(s16) = G_AND [[XOR1]], [[C2]]
     ; VI-NEXT: [[SHL1:%[0-9]+]]:_(s16) = G_SHL [[TRUNC1]], [[AND2]](s16)
     ; VI-NEXT: [[LSHR4:%[0-9]+]]:_(s16) = G_LSHR [[TRUNC3]], [[C]](s16)
     ; VI-NEXT: [[LSHR5:%[0-9]+]]:_(s16) = G_LSHR [[LSHR4]], [[AND3]](s16)
-    ; VI-NEXT: [[OR1:%[0-9]+]]:_(s16) = G_OR [[SHL1]], [[LSHR5]]
+    ; VI-NEXT: [[OR1:%[0-9]+]]:_(s16) = disjoint G_OR [[SHL1]], [[LSHR5]]
     ; VI-NEXT: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[COPY1]](<2 x s16>)
     ; VI-NEXT: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST2]](s32)
     ; VI-NEXT: [[LSHR6:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C1]](s32)
@@ -335,14 +335,14 @@ body: |
     ; VI-NEXT: [[SHL5:%[0-9]+]]:_(s16) = G_SHL [[OR]], [[AND4]](s16)
     ; VI-NEXT: [[LSHR8:%[0-9]+]]:_(s16) = G_LSHR [[SHL2]], [[C]](s16)
     ; VI-NEXT: [[LSHR9:%[0-9]+]]:_(s16) = G_LSHR [[LSHR8]], [[AND5]](s16)
-    ; VI-NEXT: [[OR3:%[0-9]+]]:_(s16) = G_OR [[SHL5]], [[LSHR9]]
+    ; VI-NEXT: [[OR3:%[0-9]+]]:_(s16) = disjoint G_OR [[SHL5]], [[LSHR9]]
     ; VI-NEXT: [[AND6:%[0-9]+]]:_(s16) = G_AND [[TRUNC7]], [[C2]]
     ; VI-NEXT: [[XOR4:%[0-9]+]]:_(s16) = G_XOR [[TRUNC7]], [[C3]]
     ; VI-NEXT: [[AND7:%[0-9]+]]:_(s16) = G_AND [[XOR4]], [[C2]]
     ; VI-NEXT: [[SHL6:%[0-9]+]]:_(s16) = G_SHL [[OR1]], [[AND6]](s16)
     ; VI-NEXT: [[LSHR10:%[0-9]+]]:_(s16) = G_LSHR [[SHL3]], [[C]](s16)
     ; VI-NEXT: [[LSHR11:%[0-9]+]]:_(s16) = G_LSHR [[LSHR10]], [[AND7]](s16)
-    ; VI-NEXT: [[OR4:%[0-9]+]]:_(s16) = G_OR [[SHL6]], [[LSHR11]]
+    ; VI-NEXT: [[OR4:%[0-9]+]]:_(s16) = disjoint G_OR [[SHL6]], [[LSHR11]]
     ; VI-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[OR3]](s16)
     ; VI-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[OR4]](s16)
     ; VI-NEXT: [[SHL7:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C1]](s32)
@@ -368,7 +368,7 @@ body: |
     ; GFX9-NEXT: [[SHL:%[0-9]+]]:_(<2 x s16>) = G_SHL [[COPY]], [[BUILD_VECTOR2]](<2 x s16>)
     ; GFX9-NEXT: [[SHL1:%[0-9]+]]:_(<2 x s16>) = G_SHL [[SHL]], [[AND1]](<2 x s16>)
     ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(<2 x s16>) = G_LSHR [[COPY1]], [[AND]](<2 x s16>)
-    ; GFX9-NEXT: [[OR:%[0-9]+]]:_(<2 x s16>) = G_OR [[SHL1]], [[LSHR]]
+    ; GFX9-NEXT: [[OR:%[0-9]+]]:_(<2 x s16>) = disjoint G_OR [[SHL1]], [[LSHR]]
     ; GFX9-NEXT: $vgpr0 = COPY [[OR]](<2 x s16>)
     %0:_(<2 x s16>) = COPY $vgpr0
     %1:_(<2 x s16>) = COPY $vgpr1
@@ -400,7 +400,7 @@ body: |
     ; SI-NEXT: [[SHL1:%[0-9]+]]:_(s64) = G_SHL [[SHL]], [[TRUNC]](s32)
     ; SI-NEXT: [[TRUNC1:%[0-9]+]]:_(s32) = G_TRUNC [[AND]](s64)
     ; SI-NEXT: [[LSHR:%[0-9]+]]:_(s64) = G_LSHR [[COPY1]], [[TRUNC1]](s32)
-    ; SI-NEXT: [[OR:%[0-9]+]]:_(s64) = G_OR [[SHL1]], [[LSHR]]
+    ; SI-NEXT: [[OR:%[0-9]+]]:_(s64) = disjoint G_OR [[SHL1]], [[LSHR]]
     ; SI-NEXT: $vgpr0_vgpr1 = COPY [[OR]](s64)
     ;
     ; VI-LABEL: name: test_fshr_s64_s64
@@ -420,7 +420,7 @@ body: |
     ; VI-NEXT: [[SHL1:%[0-9]+]]:_(s64) = G_SHL [[SHL]], [[TRUNC]](s32)
     ; VI-NEXT: [[TRUNC1:%[0-9]+]]:_(s32) = G_TRUNC [[AND]](s64)
     ; VI-NEXT: [[LSHR:%[0-9]+]]:_(s64) = G_LSHR [[COPY1]], [[TRUNC1]](s32)
-    ; VI-NEXT: [[OR:%[0-9]+]]:_(s64) = G_OR [[SHL1]], [[LSHR]]
+    ; VI-NEXT: [[OR:%[0-9]+]]:_(s64) = disjoint G_OR [[SHL1]], [[LSHR]]
     ; VI-NEXT: $vgpr0_vgpr1 = COPY [[OR]](s64)
     ;
     ; GFX9-LABEL: name: test_fshr_s64_s64
@@ -440,7 +440,7 @@ body: |
     ; GFX9-NEXT: [[SHL1:%[0-9]+]]:_(s64) = G_SHL [[SHL]], [[TRUNC]](s32)
     ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(s32) = G_TRUNC [[AND]](s64)
     ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(s64) = G_LSHR [[COPY1]], [[TRUNC1]](s32)
-    ; GFX9-NEXT: [[OR:%[0-9]+]]:_(s64) = G_OR [[SHL1]], [[LSHR]]
+    ; GFX9-NEXT: [[OR:%[0-9]+]]:_(s64) = disjoint G_OR [[SHL1]], [[LSHR]]
     ; GFX9-NEXT: $vgpr0_vgpr1 = COPY [[OR]](s64)
     %0:_(s64) = COPY $vgpr0_vgpr1
     %1:_(s64) = COPY $vgpr2_vgpr3
@@ -472,7 +472,7 @@ body: |
     ; SI-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
     ; SI-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C3]]
     ; SI-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[AND2]], [[AND]](s32)
-    ; SI-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL1]], [[LSHR]]
+    ; SI-NEXT: [[OR:%[0-9]+]]:_(s32) = disjoint G_OR [[SHL1]], [[LSHR]]
     ; SI-NEXT: $vgpr0 = COPY [[OR]](s32)
     ;
     ; VI-LABEL: name: test_fshr_s8_s8
@@ -500,7 +500,7 @@ body: |
     ; VI-NEXT: [[LSHR:%[0-9]+]]:_(s16) = G_LSHR [[AND4]], [[AND3]](s16)
     ; VI-NEXT: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[SHL1]](s16)
     ; VI-NEXT: [[ANYEXT1:%[0-9]+]]:_(s32) = G_ANYEXT [[LSHR]](s16)
-    ; VI-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[ANYEXT]], [[ANYEXT1]]
+    ; VI-NEXT: [[OR:%[0-9]+]]:_(s32) = disjoint G_OR [[ANYEXT]], [[ANYEXT1]]
     ; VI-NEXT: $vgpr0 = COPY [[OR]](s32)
     ;
     ; GFX9-LABEL: name: test_fshr_s8_s8
@@ -528,7 +528,7 @@ body: |
     ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(s16) = G_LSHR [[AND4]], [[AND3]](s16)
     ; GFX9-NEXT: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[SHL1]](s16)
     ; GFX9-NEXT: [[ANYEXT1:%[0-9]+]]:_(s32) = G_ANYEXT [[LSHR]](s16)
-    ; GFX9-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[ANYEXT]], [[ANYEXT1]]
+    ; GFX9-NEXT: [[OR:%[0-9]+]]:_(s32) = disjoint G_OR [[ANYEXT]], [[ANYEXT1]]
     ; GFX9-NEXT: $vgpr0 = COPY [[OR]](s32)
     %0:_(s32) = COPY $vgpr0
     %1:_(s32) = COPY $vgpr1
@@ -584,7 +584,7 @@ body: |
     ; SI-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[SELECT1]], [[C2]]
     ; SI-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C2]]
     ; SI-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[AND3]], [[AND2]](s32)
-    ; SI-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL1]], [[LSHR]]
+    ; SI-NEXT: [[OR:%[0-9]+]]:_(s32) = disjoint G_OR [[SHL1]], [[LSHR]]
     ; SI-NEXT: $vgpr0 = COPY [[OR]](s32)
     ;
     ; VI-LABEL: name: test_fshr_s24_s24
@@ -624,7 +624,7 @@ body: |
     ; VI-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[SELECT1]], [[C2]]
     ; VI-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C2]]
     ; VI-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[AND3]], [[AND2]](s32)
-    ; VI-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL1]], [[LSHR]]
+    ; VI-NEXT: [[OR:%[0-9]+]]:_(s32) = disjoint G_OR [[SHL1]], [[LSHR]]
     ; VI-NEXT: $vgpr0 = COPY [[OR]](s32)
     ;
     ; GFX9-LABEL: name: test_fshr_s24_s24
@@ -664,7 +664,7 @@ body: |
     ; GFX9-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[SELECT1]], [[C2]]
     ; GFX9-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C2]]
     ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[AND3]], [[AND2]](s32)
-    ; GFX9-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL1]], [[LSHR]]
+    ; GFX9-NEXT: [[OR:%[0-9]+]]:_(s32) = disjoint G_OR [[SHL1]], [[LSHR]]
     ; GFX9-NEXT: $vgpr0 = COPY [[OR]](s32)
     %0:_(s32) = COPY $vgpr0
     %1:_(s32) = COPY $vgpr1
@@ -715,7 +715,7 @@ body: |
     ; SI-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[AND1]](s16)
     ; SI-NEXT: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[LSHR2]], [[ZEXT1]](s32)
     ; SI-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR3]](s32)
-    ; SI-NEXT: [[OR:%[0-9]+]]:_(s16) = G_OR [[TRUNC]], [[TRUNC1]]
+    ; SI-NEXT: [[OR:%[0-9]+]]:_(s16) = disjoint G_OR [[TRUNC]], [[TRUNC1]]
     ; SI-NEXT: [[AND3:%[0-9]+]]:_(s16) = G_AND [[C1]], [[C2]]
     ; SI-NEXT: [[XOR1:%[0-9]+]]:_(s16) = G_XOR [[C1]], [[C3]]
     ; SI-NEXT: [[AND4:%[0-9]+]]:_(s16) = G_AND [[XOR1]], [[C2]]
@@ -727,7 +727,7 @@ body: |
     ; SI-NEXT: [[ZEXT3:%[0-9]+]]:_(s32) = G_ZEXT [[AND4]](s16)
     ; SI-NEXT: [[LSHR5:%[0-9]+]]:_(s32) = G_LSHR [[LSHR4]], [[ZEXT3]](s32)
     ; SI-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR5]](s32)
-    ; SI-NEXT: [[OR1:%[0-9]+]]:_(s16) = G_OR [[TRUNC2]], [[TRUNC3]]
+    ; SI-NEXT: [[OR1:%[0-9]+]]:_(s16) = disjoint G_OR [[TRUNC2]], [[TRUNC3]]
     ; SI-NEXT: [[COPY7:%[0-9]+]]:_(s32) = COPY [[C4]](s32)
     ; SI-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[BITCAST2]], [[COPY7]](s32)
     ; SI-NEXT: [[COPY8:%[0-9]+]]:_(s32) = COPY [[C4]](s32)
@@ -755,7 +755,7 @@ body: |
     ; SI-NEXT: [[ZEXT5:%[0-9]+]]:_(s32) = G_ZEXT [[AND6]](s16)
     ; SI-NEXT: [[LSHR8:%[0-9]+]]:_(s32) = G_LSHR [[LSHR7]], [[ZEXT5]](s32)
     ; SI-NEXT: [[TRUNC7:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR8]](s32)
-    ; SI-NEXT: [[OR3:%[0-9]+]]:_(s16) = G_OR [[TRUNC6]], [[TRUNC7]]
+    ; SI-NEXT: [[OR3:%[0-9]+]]:_(s16) = disjoint G_OR [[TRUNC6]], [[TRUNC7]]
     ; SI-NEXT: [[AND8:%[0-9]+]]:_(s16) = G_AND [[TRUNC5]], [[C2]]
     ; SI-NEXT: [[XOR4:%[0-9]+]]:_(s16) = G_XOR [[TRUNC5]], [[C3]]
     ; SI-NEXT: [[AND9:%[0-9]+]]:_(s16) = G_AND [[XOR4]], [[C2]]
@@ -769,7 +769,7 @@ body: |
     ; SI-NEXT: [[ZEXT7:%[0-9]+]]:_(s32) = G_ZEXT [[AND9]](s16)
     ; SI-NEXT: [[LSHR10:%[0-9]+]]:_(s32) = G_LSHR [[LSHR9]], [[ZEXT7]](s32)
     ; SI-NEXT: [[TRUNC9:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR10]](s32)
-    ; SI-NEXT: [[OR4:%[0-9]+]]:_(s16) = G_OR [[TRUNC8]], [[TRUNC9]]
+    ; SI-NEXT: [[OR4:%[0-9]+]]:_(s16) = disjoint G_OR [[TRUNC8]], [[TRUNC9]]
     ; SI-NEXT: [[AND11:%[0-9]+]]:_(s16) = G_AND [[C1]], [[C2]]
     ; SI-NEXT: [[XOR5:%[0-9]+]]:_(s16) = G_XOR [[C1]], [[C3]]
     ; SI-NEXT: [[AND12:%[0-9]+]]:_(s16) = G_AND [[XOR5]], [[C2]]
@@ -782,7 +782,7 @@ body: |
     ; SI-NEXT: [[ZEXT9:%[0-9]+]]:_(s32) = G_ZEXT [[AND12]](s16)
     ; SI-NEXT: [[LSHR12:%[0-9]+]]:_(s32) = G_LSHR [[LSHR11]], [[ZEXT9]](s32)
     ; SI-NEXT: [[TRUNC11:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR12]](s32)
-    ; SI-NEXT: [[OR5:%[0-9]+]]:_(s16) = G_OR [[TRUNC10]], [[TRUNC11]]
+    ; SI-NEXT: [[OR5:%[0-9]+]]:_(s16) = disjoint G_OR [[TRUNC10]], [[TRUNC11]]
     ; SI-NEXT: [[COPY14:%[0-9]+]]:_(s32) = COPY [[C4]](s32)
     ; SI-NEXT: [[SHL8:%[0-9]+]]:_(s32) = G_SHL [[BITCAST3]], [[COPY14]](s32)
     ; SI-NEXT: [[COPY15:%[0-9]+]]:_(s32) = COPY [[C5]](s32)
@@ -806,7 +806,7 @@ body: |
     ; SI-NEXT: [[ZEXT11:%[0-9]+]]:_(s32) = G_ZEXT [[AND15]](s16)
     ; SI-NEXT: [[LSHR14:%[0-9]+]]:_(s32) = G_LSHR [[LSHR13]], [[ZEXT11]](s32)
     ; SI-NEXT: [[TRUNC14:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR14]](s32)
-    ; SI-NEXT: [[OR7:%[0-9]+]]:_(s16) = G_OR [[TRUNC13]], [[TRUNC14]]
+    ; SI-NEXT: [[OR7:%[0-9]+]]:_(s16) = disjoint G_OR [[TRUNC13]], [[TRUNC14]]
     ; SI-NEXT: [[DEF:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
     ; SI-NEXT: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[DEF]](<4 x s16>)
     ; SI-NEXT: [[BITCAST8:%[0-9]+]]:_(s32) = G_BITCAST [[UV]](<2 x s16>)
@@ -861,14 +861,14 @@ body: |
     ; VI-NEXT: [[SHL:%[0-9]+]]:_(s16) = G_SHL [[TRUNC]], [[AND]](s16)
     ; VI-NEXT: [[LSHR2:%[0-9]+]]:_(s16) = G_LSHR [[TRUNC3]], [[C1]](s16)
     ; VI-NEXT: [[LSHR3:%[0-9]+]]:_(s16) = G_LSHR [[LSHR2]], [[AND1]](s16)
-    ; VI-NEXT: [[OR:%[0-9]+]]:_(s16) = G_OR [[SHL]], [[LSHR3]]
+    ; VI-NEXT: [[OR:%[0-9]+]]:_(s16) = disjoint G_OR [[SHL]], [[LSHR3]]
     ; VI-NEXT: [[AND2:%[0-9]+]]:_(s16) = G_AND [[C1]], [[C2]]
     ; VI-NEXT: [[XOR1:%[0-9]+]]:_(s16) = G_XOR [[C1]], [[C3]]
     ; VI-NEXT: [[AND3:%[0-9]+]]:_(s16) = G_AND [[XOR1]], [[C2]]
     ; VI-NEXT: [[SHL1:%[0-9]+]]:_(s16) = G_SHL [[TRUNC1]], [[AND2]](s16)
     ; VI-NEXT: [[LSHR4:%[0-9]+]]:_(s16) = G_LSHR [[TRUNC4]], [[C1]](s16)
     ; VI-NEXT: [[LSHR5:%[0-9]+]]:_(s16) = G_LSHR [[LSHR4]], [[AND3]](s16)
-    ; VI-NEXT: [[OR1:%[0-9]+]]:_(s16) = G_OR [[SHL1]], [[LSHR5]]
+    ; VI-NEXT: [[OR1:%[0-9]+]]:_(s16) = disjoint G_OR [[SHL1]], [[LSHR5]]
     ; VI-NEXT: [[SHL2:%[0-9]+]]:_(s16) = G_SHL [[TRUNC3]], [[C1]](s16)
     ; VI-NEXT: [[SHL3:%[0-9]+]]:_(s16) = G_SHL [[TRUNC4]], [[C1]](s16)
     ; VI-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
@@ -887,21 +887,21 @@ body: |
     ; VI-NEXT: [[SHL5:%[0-9]+]]:_(s16) = G_SHL [[OR]], [[AND4]](s16)
     ; VI-NEXT: [[LSHR7:%[0-9]+]]:_(s16) = G_LSHR [[SHL2]], [[C1]](s16)
     ; VI-NEXT: [[LSHR8:%[0-9]+]]:_(s16) = G_LSHR [[LSHR7]], [[AND5]](s16)
-    ; VI-NEXT: [[OR3:%[0-9]+]]:_(s16) = G_OR [[SHL5]], [[LSHR8]]
+    ; VI-NEXT: [[OR3:%[0-9]+]]:_(s16) = disjoint G_OR [[SHL5]], [[LSHR8]]
     ; VI-NEXT: [[AND6:%[0-9]+]]:_(s16) = G_AND [[TRUNC7]], [[C2]]
     ; VI-NEXT: [[XOR4:%[0-9]+]]:_(s16) = G_XOR [[TRUNC7]], [[C3]]
     ; VI-NEXT: [[AND7:%[0-9]+]]:_(s16) = G_AND [[XOR4]], [[C2]]
     ; VI-NEXT: [[SHL6:%[0-9]+]]:_(s16) = G_SHL [[OR1]], [[AND6]](s16)
     ; VI-NEXT: [[LSHR9:%[0-9]+]]:_(s16) = G_LSHR [[SHL3]], [[C1]](s16)
     ; VI-NEXT: [[LSHR10:%[0-9]+]]:_(s16) = G_LSHR [[LSHR9]], [[AND7]](s16)
-    ; VI-NEXT: [[OR4:%[0-9]+]]:_(s16) = G_OR [[SHL6]], [[LSHR10]]
+    ; VI-NEXT: [[OR4:%[0-9]+]]:_(s16) = disjoint G_OR [[SHL6]], [[LSHR10]]
     ; VI-NEXT: [[AND8:%[0-9]+]]:_(s16) = G_AND [[C1]], [[C2]]
     ; VI-NEXT: [[XOR5:%[0-9]+]]:_(s16) = G_XOR [[C1]], [[C3]]
     ; VI-NEXT: [[AND9:%[0-9]+]]:_(s16) = G_AND [[XOR5]], [[C2]]
     ; VI-NEXT: [[SHL7:%[0-9]+]]:_(s16) = G_SHL [[TRUNC2]], [[AND8]](s16)
     ; VI-NEXT: [[LSHR11:%[0-9]+]]:_(s16) = G_LSHR [[TRUNC5]], [[C1]](s16)
     ; VI-NEXT: [[LSHR12:%[0-9]+]]:_(s16) = G_LSHR [[LSHR11]], [[AND9]](s16)
-    ; VI-NEXT: [[OR5:%[0-9]+]]:_(s16) = G_OR [[SHL7]], [[LSHR12]]
+    ; VI-NEXT: [[OR5:%[0-9]+]]:_(s16) = disjoint G_OR [[SHL7]], [[LSHR12]]
     ; VI-NEXT: [[SHL8:%[0-9]+]]:_(s16) = G_SHL [[TRUNC5]], [[C1]](s16)
     ; VI-NEXT: [[COPY7:%[0-9]+]]:_(s32) = COPY [[C4]](s32)
     ; VI-NEXT: [[COPY8:%[0-9]+]]:_(s32) = COPY [[C4]](s32)
@@ -917,7 +917,7 @@ body: |
     ; VI-NEXT: [[SHL10:%[0-9]+]]:_(s16) = G_SHL [[OR5]], [[AND10]](s16)
     ; VI-NEXT: [[LSHR13:%[0-9]+]]:_(s16) = G_LSHR [[SHL8]], [[C1]](s16)
     ; VI-NEXT: [[LSHR14:%[0-9]+]]:_(s16) = G_LSHR [[LSHR13]], [[AND11]](s16)
-    ; VI-NEXT: [[OR7:%[0-9]+]]:_(s16) = G_OR [[SHL10]], [[LSHR14]]
+    ; VI-NEXT: [[OR7:%[0-9]+]]:_(s16) = disjoint G_OR [[SHL10]], [[LSHR14]]
     ; VI-NEXT: [[DEF:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
     ; VI-NEXT: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[DEF]](<4 x s16>)
     ; VI-NEXT: [[BITCAST8:%[0-9]+]]:_(s32) = G_BITCAST [[UV]](<2 x s16>)
@@ -962,7 +962,7 @@ body: |
     ; GFX9-NEXT: [[SHL:%[0-9]+]]:_(<2 x s16>) = G_SHL [[COPY]], [[BUILD_VECTOR2]](<2 x s16>)
     ; GFX9-NEXT: [[SHL1:%[0-9]+]]:_(<2 x s16>) = G_SHL [[SHL]], [[AND1]](<2 x s16>)
     ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(<2 x s16>) = G_LSHR [[COPY2]], [[AND]](<2 x s16>)
-    ; GFX9-NEXT: [[OR:%[0-9]+]]:_(<2 x s16>) = G_OR [[SHL1]], [[LSHR]]
+    ; GFX9-NEXT: [[OR:%[0-9]+]]:_(<2 x s16>) = disjoint G_OR [[SHL1]], [[LSHR]]
     ; GFX9-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[C]](s16), [[C]](s16)
     ; GFX9-NEXT: [[AND2:%[0-9]+]]:_(<2 x s16>) = G_AND [[COPY5]], [[BUILD_VECTOR3]]
     ; GFX9-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[C1]](s16), [[C1]](s16)
@@ -972,7 +972,7 @@ body: |
     ; GFX9-NEXT: [[SHL2:%[0-9]+]]:_(<2 x s16>) = G_SHL [[COPY1]], [[BUILD_VECTOR5]](<2 x s16>)
     ; GFX9-NEXT: [[SHL3:%[0-9]+]]:_(<2 x s16>) = G_SHL [[SHL2]], [[AND3]](<2 x s16>)
     ; GFX9-NEXT: [[LSHR1:%[0-9]+]]:_(<2 x s16>) = G_LSHR [[COPY3]], [[AND2]](<2 x s16>)
-    ; GFX9-NEXT: [[OR1:%[0-9]+]]:_(<2 x s16>) = G_OR [[SHL3]], [[LSHR1]]
+    ; GFX9-NEXT: [[OR1:%[0-9]+]]:_(<2 x s16>) = disjoint G_OR [[SHL3]], [[LSHR1]]
     ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[OR1]](<2 x s16>)
     ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
     ; GFX9-NEXT: [[DEF:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
@@ -1047,7 +1047,7 @@ body: |
     ; SI-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[AND1]](s16)
     ; SI-NEXT: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[LSHR2]], [[ZEXT1]](s32)
     ; SI-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR3]](s32)
-    ; SI-NEXT: [[OR:%[0-9]+]]:_(s16) = G_OR [[TRUNC]], [[TRUNC1]]
+    ; SI-NEXT: [[OR:%[0-9]+]]:_(s16) = disjoint G_OR [[TRUNC]], [[TRUNC1]]
     ; SI-NEXT: [[AND3:%[0-9]+]]:_(s16) = G_AND [[C]], [[C2]]
     ; SI-NEXT: [[XOR1:%[0-9]+]]:_(s16) = G_XOR [[C]], [[C3]]
     ; SI-NEXT: [[AND4:%[0-9]+]]:_(s16) = G_AND [[XOR1]], [[C2]]
@@ -1059,7 +1059,7 @@ body: |
     ; SI-NEXT: [[ZEXT3:%[0-9]+]]:_(s32) = G_ZEXT [[AND4]](s16)
     ; SI-NEXT: [[LSHR5:%[0-9]+]]:_(s32) = G_LSHR [[LSHR4]], [[ZEXT3]](s32)
     ; SI-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR5]](s32)
-    ; SI-NEXT: [[OR1:%[0-9]+]]:_(s16) = G_OR [[TRUNC2]], [[TRUNC3]]
+    ; SI-NEXT: [[OR1:%[0-9]+]]:_(s16) = disjoint G_OR [[TRUNC2]], [[TRUNC3]]
     ; SI-NEXT: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[UV2]](<2 x s16>)
     ; SI-NEXT: [[LSHR6:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C1]](s32)
     ; SI-NEXT: [[COPY4:%[0-9]+]]:_(s32) = COPY [[C4]](s32)
@@ -1089,7 +1089,7 @@ body: |
     ; SI-NEXT: [[ZEXT5:%[0-9]+]]:_(s32) = G_ZEXT [[AND6]](s16)
     ; SI-NEXT: [[LSHR9:%[0-9]+]]:_(s32) = G_LSHR [[LSHR8]], [[ZEXT5]](s32)
     ; SI-NEXT: [[TRUNC7:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR9]](s32)
-    ; SI-NEXT: [[OR3:%[0-9]+]]:_(s16) = G_OR [[TRUNC6]], [[TRUNC7]]
+    ; SI-NEXT: [[OR3:%[0-9]+]]:_(s16) = disjoint G_OR [[TRUNC6]], [[TRUNC7]]
     ; SI-NEXT: [[AND8:%[0-9]+]]:_(s16) = G_AND [[TRUNC5]], [[C2]]
     ; SI-NEXT: [[XOR4:%[0-9]+]]:_(s16) = G_XOR [[TRUNC5]], [[C3]]
     ; SI-NEXT: [[AND9:%[0-9]+]]:_(s16) = G_AND [[XOR4]], [[C2]]
@@ -1103,7 +1103,7 @@ body: |
     ; SI-NEXT: [[ZEXT7:%[0-9]+]]:_(s32) = G_ZEXT [[AND9]](s16)
     ; SI-NEXT: [[LSHR11:%[0-9]+]]:_(s32) = G_LSHR [[LSHR10]], [[ZEXT7]](s32)
     ; SI-NEXT: [[TRUNC9:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR11]](s32)
-    ; SI-NEXT: [[OR4:%[0-9]+]]:_(s16) = G_OR [[TRUNC8]], [[TRUNC9]]
+    ; SI-NEXT: [[OR4:%[0-9]+]]:_(s16) = disjoint G_OR [[TRUNC8]], [[TRUNC9]]
     ; SI-NEXT: [[ZEXT8:%[0-9]+]]:_(s32) = G_ZEXT [[OR3]](s16)
     ; SI-NEXT: [[ZEXT9:%[0-9]+]]:_(s32) = G_ZEXT [[OR4]](s16)
     ; SI-NEXT: [[SHL7:%[0-9]+]]:_(s32) = G_SHL [[ZEXT9]], [[C1]](s32)
@@ -1125,7 +1125,7 @@ body: |
     ; SI-NEXT: [[ZEXT11:%[0-9]+]]:_(s32) = G_ZEXT [[AND12]](s16)
     ; SI-NEXT: [[LSHR15:%[0-9]+]]:_(s32) = G_LSHR [[LSHR14]], [[ZEXT11]](s32)
     ; SI-NEXT: [[TRUNC11:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR15]](s32)
-    ; SI-NEXT: [[OR6:%[0-9]+]]:_(s16) = G_OR [[TRUNC10]], [[TRUNC11]]
+    ; SI-NEXT: [[OR6:%[0-9]+]]:_(s16) = disjoint G_OR [[TRUNC10]], [[TRUNC11]]
     ; SI-NEXT: [[AND14:%[0-9]+]]:_(s16) = G_AND [[C]], [[C2]]
     ; SI-NEXT: [[XOR6:%[0-9]+]]:_(s16) = G_XOR [[C]], [[C3]]
     ; SI-NEXT: [[AND15:%[0-9]+]]:_(s16) = G_AND [[XOR6]], [[C2]]
@@ -1137,7 +1137,7 @@ body: |
     ; SI-NEXT: [[ZEXT13:%[0-9]+]]:_(s32) = G_ZEXT [[AND15]](s16)
     ; SI-NEXT: [[LSHR17:%[0-9]+]]:_(s32) = G_LSHR [[LSHR16]], [[ZEXT13]](s32)
     ; SI-NEXT: [[TRUNC13:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR17]](s32)
-    ; SI-NEXT: [[OR7:%[0-9]+]]:_(s16) = G_OR [[TRUNC12]], [[TRUNC13]]
+    ; SI-NEXT: [[OR7:%[0-9]+]]:_(s16) = disjoint G_OR [[TRUNC12]], [[TRUNC13]]
     ; SI-NEXT: [[BITCAST8:%[0-9]+]]:_(s32) = G_BITCAST [[UV3]](<2 x s16>)
     ; SI-NEXT: [[LSHR18:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST8]], [[C1]](s32)
     ; SI-NEXT: [[COPY12:%[0-9]+]]:_(s32) = COPY [[C4]](s32)
@@ -1167,7 +1167,7 @@ body: |
     ; SI-NEXT: [[ZEXT15:%[0-9]+]]:_(s32) = G_ZEXT [[AND17]](s16)
     ; SI-NEXT: [[LSHR21:%[0-9]+]]:_(s32) = G_LSHR [[LSHR20]], [[ZEXT15]](s32)
     ; SI-NEXT: [[TRUNC17:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR21]](s32)
-    ; SI-NEXT: [[OR9:%[0-9]+]]:_(s16) = G_OR [[TRUNC16]], [[TRUNC17]]
+    ; SI-NEXT: [[OR9:%[0-9]+]]:_(s16) = disjoint G_OR [[TRUNC16]], [[TRUNC17]]
     ; SI-NEXT: [[AND19:%[0-9]+]]:_(s16) = G_AND [[TRUNC15]], [[C2]]
     ; SI-NEXT: [[XOR9:%[0-9]+]]:_(s16) = G_XOR [[TRUNC15]], [[C3]]
     ; SI-NEXT: [[AND20:%[0-9]+]]:_(s16) = G_AND [[XOR9]], [[C2]]
@@ -1181,7 +1181,7 @@ body: |
     ; SI-NEXT: [[ZEXT17:%[0-9]+]]:_(s32) = G_ZEXT [[AND20]](s16)
     ; SI-NEXT: [[LSHR23:%[0-9]+]]:_(s32) = G_LSHR [[LSHR22]], [[ZEXT17]](s32)
     ; SI-NEXT: [[TRUNC19:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR23]](s32)
-    ; SI-NEXT: [[OR10:%[0-9]+]]:_(s16) = G_OR [[TRUNC18]], [[TRUNC19]]
+    ; SI-NEXT: [[OR10:%[0-9]+]]:_(s16) = disjoint G_OR [[TRUNC18]], [[TRUNC19]]
     ; SI-NEXT: [[ZEXT18:%[0-9]+]]:_(s32) = G_ZEXT [[OR9]](s16)
     ; SI-NEXT: [[ZEXT19:%[0-9]+]]:_(s32) = G_ZEXT [[OR10]](s16)
     ; SI-NEXT: [[SHL15:%[0-9]+]]:_(s32) = G_SHL [[ZEXT19]], [[C1]](s32)
@@ -1217,14 +1217,14 @@ body: |
     ; VI-NEXT: [[SHL:%[0-9]+]]:_(s16) = G_SHL [[TRUNC]], [[AND]](s16)
     ; VI-NEXT: [[LSHR2:%[0-9]+]]:_(s16) = G_LSHR [[TRUNC2]], [[C]](s16)
     ; VI-NEXT: [[LSHR3:%[0-9]+]]:_(s16) = G_LSHR [[LSHR2]], [[AND1]](s16)
-    ; VI-NEXT: [[OR:%[0-9]+]]:_(s16) = G_OR [[SHL]], [[LSHR3]]
+    ; VI-NEXT: [[OR:%[0-9]+]]:_(s16) = disjoint G_OR [[SHL]], [[LSHR3]]
     ; VI-NEXT: [[AND2:%[0-9]+]]:_(s16) = G_AND [[C]], [[C2]]
     ; VI-NEXT: [[XOR1:%[0-9]+]]:_(s16) = G_XOR [[C]], [[C3]]
     ; VI-NEXT: [[AND3:%[0-9]+]]:_(s16) = G_AND [[XOR1]], [[C2]]
     ; VI-NEXT: [[SHL1:%[0-9]+]]:_(s16) = G_SHL [[TRUNC1]], [[AND2]](s16)
     ; VI-NEXT: [[LSHR4:%[0-9]+]]:_(s16) = G_LSHR [[TRUNC3]], [[C]](s16)
     ; VI-NEXT: [[LSHR5:%[0-9]+]]:_(s16) = G_LSHR [[LSHR4]], [[AND3]](s16)
-    ; VI-NEXT: [[OR1:%[0-9]+]]:_(s16) = G_OR [[SHL1]], [[LSHR5]]
+    ; VI-NEXT: [[OR1:%[0-9]+]]:_(s16) = disjoint G_OR [[SHL1]], [[LSHR5]]
     ; VI-NEXT: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[UV2]](<2 x s16>)
     ; VI-NEXT: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST2]](s32)
     ; VI-NEXT: [[LSHR6:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C1]](s32)
@@ -1247,14 +1247,14 @@ body: |
     ; VI-NEXT: [[SHL5:%[0-9]+]]:_(s16) = G_SHL [[OR]], [[AND4]](s16)
     ; VI-NEXT: [[LSHR8:%[0-9]+]]:_(s16) = G_LSHR [[SHL2]], [[C]](s16)
     ; VI-NEXT: [[LSHR9:%[0-9]+]]:_(s16) = G_LSHR [[LSHR8]], [[AND5]](s16)
-    ; VI-NEXT: [[OR3:%[0-9]+]]:_(s16) = G_OR [[SHL5]], [[LSHR9]]
+    ; VI-NEXT: [[OR3:%[0-9]+]]:_(s16) = disjoint G_OR [[SHL5]], [[LSHR9]]
     ; VI-NEXT: [[AND6:%[0-9]+]]:_(s16) = G_AND [[TRUNC7]], [[C2]]
     ; VI-NEXT: [[XOR4:%[0-9]+]]:_(s16) = G_XOR [[TRUNC7]], [[C3]]
     ; VI-NEXT: [[AND7:%[0-9]+]]:_(s16) = G_AND [[XOR4]], [[C2]]
     ; VI-NEXT: [[SHL6:%[0-9]+]]:_(s16) = G_SHL [[OR1]], [[AND6]](s16)
     ; VI-NEXT: [[LSHR10:%[0-9]+]]:_(s16) = G_LSHR [[SHL3]], [[C]](s16)
     ; VI-NEXT: [[LSHR11:%[0-9]+]]:_(s16) = G_LSHR [[LSHR10]], [[AND7]](s16)
-    ; VI-NEXT: [[OR4:%[0-9]+]]:_(s16) = G_OR [[SHL6]], [[LSHR11]]
+    ; VI-NEXT: [[OR4:%[0-9]+]]:_(s16) = disjoint G_OR [[SHL6]], [[LSHR11]]
     ; VI-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[OR3]](s16)
     ; VI-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[OR4]](s16)
     ; VI-NEXT: [[SHL7:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C1]](s32)
@@ -1274,14 +1274,14 @@ body: |
     ; VI-NEXT: [[SHL8:%[0-9]+]]:_(s16) = G_SHL [[TRUNC8]], [[AND8]](s16)
     ; VI-NEXT: [[LSHR14:%[0-9]+]]:_(s16) = G_LSHR [[TRUNC10]], [[C]](s16)
     ; VI-NEXT: [[LSHR15:%[0-9]+]]:_(s16) = G_LSHR [[LSHR14]], [[AND9]](s16)
-    ; VI-NEXT: [[OR6:%[0-9]+]]:_(s16) = G_OR [[SHL8]], [[LSHR15]]
+    ; VI-NEXT: [[OR6:%[0-9]+]]:_(s16) = disjoint G_OR [[SHL8]], [[LSHR15]]
     ; VI-NEXT: [[AND10:%[0-9]+]]:_(s16) = G_AND [[C]], [[C2]]
     ; VI-NEXT: [[XOR6:%[0-9]+]]:_(s16) = G_XOR [[C]], [[C3]]
     ; VI-NEXT: [[AND11:%[0-9]+]]:_(s16) = G_AND [[XOR6]], [[C2]]
     ; VI-NEXT: [[SHL9:%[0-9]+]]:_(s16) = G_SHL [[TRUNC9]], [[AND10]](s16)
     ; VI-NEXT: [[LSHR16:%[0-9]+]]:_(s16) = G_LSHR [[TRUNC11]], [[C]](s16)
     ; VI-NEXT: [[LSHR17:%[0-9]+]]:_(s16) = G_LSHR [[LSHR16]], [[AND11]](s16)
-    ; VI-NEXT: [[OR7:%[0-9]+]]:_(s16) = G_OR [[SHL9]], [[LSHR17]]
+    ; VI-NEXT: [[OR7:%[0-9]+]]:_(s16) = disjoint G_OR [[SHL9]], [[LSHR17]]
     ; VI-NEXT: [[BITCAST8:%[0-9]+]]:_(s32) = G_BITCAST [[UV3]](<2 x s16>)
     ; VI-NEXT: [[TRUNC12:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST8]](s32)
     ; VI-NEXT: [[LSHR18:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST8]], [[C1]](s32)
@@ -1304,14 +1304,14 @@ body: |
     ; VI-NEXT: [[SHL13:%[0-9]+]]:_(s16) = G_SHL [[OR6]], [[AND12]](s16)
     ; VI-NEXT: [[LSHR20:%[0-9]+]]:_(s16) = G_LSHR [[SHL10]], [[C]](s16)
     ; VI-NEXT: [[LSHR21:%[0-9]+]]:_(s16) = G_LSHR [[LSHR20]], [[AND13]](s16)
-    ; VI-NEXT: [[OR9:%[0-9]+]]:_(s16) = G_OR [[SHL13]], [[LSHR21]]
+    ; VI-NEXT: [[OR9:%[0-9]+]]:_(s16) = disjoint G_OR [[SHL13]], [[LSHR21]]
     ; VI-NEXT: [[AND14:%[0-9]+]]:_(s16) = G_AND [[TRUNC15]], [[C2]]
     ; VI-NEXT: [[XOR9:%[0-9]+]]:_(s16) = G_XOR [[TRUNC15]], [[C3]]
     ; VI-NEXT: [[AND15:%[0-9]+]]:_(s16) = G_AND [[XOR9]], [[C2]]
     ; VI-NEXT: [[SHL14:%[0-9]+]]:_(s16) = G_SHL [[OR7]], [[AND14]](s16)
     ; VI-NEXT: [[LSHR22:%[0-9]+]]:_(s16) = G_LSHR [[SHL11]], [[C]](s16)
     ; VI-NEXT: [[LSHR23:%[0-9]+]]:_(s16) = G_LSHR [[LSHR22]], [[AND15]](s16)
-    ; VI-NEXT: [[OR10:%[0-9]+]]:_(s16) = G_OR [[SHL14]], [[LSHR23]]
+    ; VI-NEXT: [[OR10:%[0-9]+]]:_(s16) = disjoint G_OR [[SHL14]], [[LSHR23]]
     ; VI-NEXT: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[OR9]](s16)
     ; VI-NEXT: [[ZEXT3:%[0-9]+]]:_(s32) = G_ZEXT [[OR10]](s16)
     ; VI-NEXT: [[SHL15:%[0-9]+]]:_(s32) = G_SHL [[ZEXT3]], [[C1]](s32)
@@ -1341,7 +1341,7 @@ body: |
     ; GFX9-NEXT: [[SHL:%[0-9]+]]:_(<2 x s16>) = G_SHL [[UV]], [[BUILD_VECTOR2]](<2 x s16>)
     ; GFX9-NEXT: [[SHL1:%[0-9]+]]:_(<2 x s16>) = G_SHL [[SHL]], [[AND1]](<2 x s16>)
     ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(<2 x s16>) = G_LSHR [[UV2]], [[AND]](<2 x s16>)
-    ; GFX9-NEXT: [[OR:%[0-9]+]]:_(<2 x s16>) = G_OR [[SHL1]], [[LSHR]]
+    ; GFX9-NEXT: [[OR:%[0-9]+]]:_(<2 x s16>) = disjoint G_OR [[SHL1]], [[LSHR]]
     ; GFX9-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[C]](s16), [[C]](s16)
     ; GFX9-NEXT: [[AND2:%[0-9]+]]:_(<2 x s16>) = G_AND [[UV5]], [[BUILD_VECTOR3]]
     ; GFX9-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[C1]](s16), [[C1]](s16)
@@ -1351,7 +1351,7 @@ body: |
     ; GFX9-NEXT: [[SHL2:%[0-9]+]]:_(<2 x s16>) = G_SHL [[UV1]], [[BUILD_VECTOR5]](<2 x s16>)
     ; GFX9-NEXT: [[SHL3:%[0-9]+]]:_(<2 x s16>) = G_SHL [[SHL2]], [[AND3]](<2 x s16>)
     ; GFX9-NEXT: [[LSHR1:%[0-9]+]]:_(<2 x s16>) = G_LSHR [[UV3]], [[AND2]](<2 x s16>)
-    ; GFX9-NEXT: [[OR1:%[0-9]+]]:_(<2 x s16>) = G_OR [[SHL3]], [[LSHR1]]
+    ; GFX9-NEXT: [[OR1:%[0-9]+]]:_(<2 x s16>) = disjoint G_OR [[SHL3]], [[LSHR1]]
     ; GFX9-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[OR]](<2 x s16>), [[OR1]](<2 x s16>)
     ; GFX9-NEXT: $vgpr0_vgpr1 = COPY [[CONCAT_VECTORS]](<4 x s16>)
     %0:_(<4 x s16>) = COPY $vgpr0_vgpr1
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-load-memory-metadata.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-load-memory-metadata.mir
index a862d4a9032e8..da88d5f13c1eb 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-load-memory-metadata.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-load-memory-metadata.mir
@@ -30,6 +30,7 @@
   !0 = !{i24 0, i24 1048575}
   !1 = !{!"omnipotent char", !2}
   !2 = !{!"Simple C/C++ TBAA"}
+  !3 = !{i24 0, i24 1048575}
 ...
 
 # Make sure range metadata is not preserved when widening loads, but
@@ -67,7 +68,7 @@ body: |
     ; SI-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p1) :: (load (s32), !tbaa !1, addrspace 1)
     ; SI-NEXT: $vgpr0 = COPY [[LOAD]](s32)
     %0:_(p1) = COPY $vgpr0_vgpr1
-    %1:_(s32) = G_LOAD %0 :: (load (s24), align 4, addrspace 1, !range !0, !tbaa !1)
+    %1:_(s32) = G_LOAD %0 :: (load (s24), align 4, addrspace 1, !range !3, !tbaa !1)
     $vgpr0 = COPY %1
 
 ...
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.rsq.clamp.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.rsq.clamp.ll
index 9ef54ed724ec0..bf8e10143003a 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.rsq.clamp.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.rsq.clamp.ll
@@ -171,6 +171,7 @@ define float @v_rsq_clamp_undef_f32() #0 {
 ; GFX12-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-NEXT:    v_s_rsq_f32 s0, s0
 ; GFX12-NEXT:    v_mov_b32_e32 v0, 0xff7fffff
+; GFX12-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-NEXT:    s_delay_alu instid0(TRANS32_DEP_1) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_minmax_num_f32 v0, s0, 0x7f7fffff, v0
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.s.buffer.load.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.s.buffer.load.ll
index 91cde52cd2d67..79b333c08cb2d 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.s.buffer.load.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.s.buffer.load.ll
@@ -6846,6 +6846,75 @@ define amdgpu_ps float @s_buffer_load_f32_offset_add_imm_vgpr_sgpr(<4 x i32> inr
   ret float %val
 }
 
+define amdgpu_ps float @s_buffer_load_f32_offset_or_vgpr_imm(<4 x i32> inreg %rsrc, i32 inreg %offset.s) {
+  ; GFX6-LABEL: name: s_buffer_load_f32_offset_or_vgpr_imm
+  ; GFX6: bb.1 (%ir-block.0):
+  ; GFX6-NEXT:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6
+  ; GFX6-NEXT: {{  $}}
+  ; GFX6-NEXT:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; GFX6-NEXT:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; GFX6-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
+  ; GFX6-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
+  ; GFX6-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; GFX6-NEXT:   [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; GFX6-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 -2147483648
+  ; GFX6-NEXT:   [[S_OR_B32_:%[0-9]+]]:sreg_32 = S_OR_B32 [[COPY4]], [[S_MOV_B32_]], implicit-def dead $scc
+  ; GFX6-NEXT:   [[S_BUFFER_LOAD_DWORD_SGPR:%[0-9]+]]:sreg_32_xm0_xexec = S_BUFFER_LOAD_DWORD_SGPR [[REG_SEQUENCE]], [[S_OR_B32_]], 0 :: (dereferenceable invariant load (s32))
+  ; GFX6-NEXT:   $vgpr0 = COPY [[S_BUFFER_LOAD_DWORD_SGPR]]
+  ; GFX6-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
+  ;
+  ; GFX7-LABEL: name: s_buffer_load_f32_offset_or_vgpr_imm
+  ; GFX7: bb.1 (%ir-block.0):
+  ; GFX7-NEXT:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6
+  ; GFX7-NEXT: {{  $}}
+  ; GFX7-NEXT:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; GFX7-NEXT:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; GFX7-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
+  ; GFX7-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
+  ; GFX7-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; GFX7-NEXT:   [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; GFX7-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 -2147483648
+  ; GFX7-NEXT:   [[S_OR_B32_:%[0-9]+]]:sreg_32 = S_OR_B32 [[COPY4]], [[S_MOV_B32_]], implicit-def dead $scc
+  ; GFX7-NEXT:   [[S_BUFFER_LOAD_DWORD_SGPR:%[0-9]+]]:sreg_32_xm0_xexec = S_BUFFER_LOAD_DWORD_SGPR [[REG_SEQUENCE]], [[S_OR_B32_]], 0 :: (dereferenceable invariant load (s32))
+  ; GFX7-NEXT:   $vgpr0 = COPY [[S_BUFFER_LOAD_DWORD_SGPR]]
+  ; GFX7-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
+  ;
+  ; GFX8-LABEL: name: s_buffer_load_f32_offset_or_vgpr_imm
+  ; GFX8: bb.1 (%ir-block.0):
+  ; GFX8-NEXT:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6
+  ; GFX8-NEXT: {{  $}}
+  ; GFX8-NEXT:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; GFX8-NEXT:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; GFX8-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
+  ; GFX8-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
+  ; GFX8-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; GFX8-NEXT:   [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; GFX8-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 -2147483648
+  ; GFX8-NEXT:   [[S_OR_B32_:%[0-9]+]]:sreg_32 = S_OR_B32 [[COPY4]], [[S_MOV_B32_]], implicit-def dead $scc
+  ; GFX8-NEXT:   [[S_BUFFER_LOAD_DWORD_SGPR:%[0-9]+]]:sreg_32_xm0_xexec = S_BUFFER_LOAD_DWORD_SGPR [[REG_SEQUENCE]], [[S_OR_B32_]], 0 :: (dereferenceable invariant load (s32))
+  ; GFX8-NEXT:   $vgpr0 = COPY [[S_BUFFER_LOAD_DWORD_SGPR]]
+  ; GFX8-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
+  ;
+  ; GFX12-LABEL: name: s_buffer_load_f32_offset_or_vgpr_imm
+  ; GFX12: bb.1 (%ir-block.0):
+  ; GFX12-NEXT:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6
+  ; GFX12-NEXT: {{  $}}
+  ; GFX12-NEXT:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; GFX12-NEXT:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; GFX12-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
+  ; GFX12-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
+  ; GFX12-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; GFX12-NEXT:   [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; GFX12-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 -2147483648
+  ; GFX12-NEXT:   [[S_OR_B32_:%[0-9]+]]:sreg_32 = S_OR_B32 [[COPY4]], [[S_MOV_B32_]], implicit-def dead $scc
+  ; GFX12-NEXT:   [[S_BUFFER_LOAD_DWORD_SGPR_IMM:%[0-9]+]]:sreg_32_xm0_xexec = S_BUFFER_LOAD_DWORD_SGPR_IMM [[REG_SEQUENCE]], [[S_OR_B32_]], 0, 0 :: (dereferenceable invariant load (s32))
+  ; GFX12-NEXT:   $vgpr0 = COPY [[S_BUFFER_LOAD_DWORD_SGPR_IMM]]
+  ; GFX12-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
+  %offset = or i32 %offset.s, -2147483648
+  %val = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %rsrc, i32 %offset, i32 0)
+  ret float %val
+}
+
 declare i32 @llvm.amdgcn.s.buffer.load.i32(<4 x i32>, i32, i32 immarg)
 declare <2 x i32> @llvm.amdgcn.s.buffer.load.v2i32(<4 x i32>, i32, i32 immarg)
 declare <3 x i32> @llvm.amdgcn.s.buffer.load.v3i32(<4 x i32>, i32, i32 immarg)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.set.inactive.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.set.inactive.ll
index 73b891e43de99..ee89b28a0d2bb 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.set.inactive.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.set.inactive.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -global-isel -mtriple=amdgcn -mcpu=tonga -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -check-prefix=GCN %s
+; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgcn -mcpu=tonga -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -check-prefix=GCN %s
 
 define amdgpu_kernel void @set_inactive(ptr addrspace(1) %out, i32 %in) {
 ; GCN-LABEL: set_inactive:
@@ -284,17 +284,15 @@ define amdgpu_kernel void @set_inactive_v2bf16(ptr addrspace(1) %out, <2 x bfloa
 ; GCN:       ; %bb.0:
 ; GCN-NEXT:    s_load_dword s6, s[4:5], 0x2c
 ; GCN-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GCN-NEXT:    s_or_saveexec_b64 s[2:3], -1
-; GCN-NEXT:    v_mov_b32_e32 v0, 0x3f803f80
-; GCN-NEXT:    s_mov_b64 exec, s[2:3]
+; GCN-NEXT:    s_mov_b32 s3, 0xf000
 ; GCN-NEXT:    s_mov_b32 s2, -1
 ; GCN-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN-NEXT:    v_mov_b32_e32 v1, s6
 ; GCN-NEXT:    s_or_saveexec_b64 s[4:5], -1
+; GCN-NEXT:    v_mov_b32_e32 v0, 0x3f803f80
 ; GCN-NEXT:    v_cndmask_b32_e64 v0, v0, v1, s[4:5]
 ; GCN-NEXT:    s_mov_b64 exec, s[4:5]
 ; GCN-NEXT:    v_mov_b32_e32 v1, v0
-; GCN-NEXT:    s_mov_b32 s3, 0xf000
 ; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], 0
 ; GCN-NEXT:    s_endpgm
   %tmp.0 = call <2 x bfloat> @llvm.amdgcn.set.inactive.v2bf16(<2 x bfloat> %in, <2 x bfloat> <bfloat 1.0, bfloat 1.0>) #0
@@ -359,21 +357,23 @@ define amdgpu_kernel void @set_inactive_v4bf16(ptr addrspace(1) %out, <4 x bfloa
 ; GCN-LABEL: set_inactive_v4bf16:
 ; GCN:       ; %bb.0:
 ; GCN-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GCN-NEXT:    s_or_saveexec_b64 s[4:5], -1
-; GCN-NEXT:    v_mov_b32_e32 v0, 0x3f803f80
-; GCN-NEXT:    s_mov_b64 exec, s[4:5]
+; GCN-NEXT:    s_mov_b32 s7, 0xf000
+; GCN-NEXT:    s_mov_b32 s6, -1
 ; GCN-NEXT:    s_waitcnt lgkmcnt(0)
-; GCN-NEXT:    v_mov_b32_e32 v3, s2
-; GCN-NEXT:    v_mov_b32_e32 v4, s3
-; GCN-NEXT:    s_or_saveexec_b64 s[4:5], -1
-; GCN-NEXT:    s_mov_b32 s2, -1
-; GCN-NEXT:    v_cndmask_b32_e64 v1, v0, v3, s[4:5]
-; GCN-NEXT:    v_cndmask_b32_e64 v2, v0, v4, s[4:5]
-; GCN-NEXT:    s_mov_b64 exec, s[4:5]
+; GCN-NEXT:    s_mov_b32 s4, s0
+; GCN-NEXT:    s_mov_b32 s5, s1
+; GCN-NEXT:    v_mov_b32_e32 v2, s3
+; GCN-NEXT:    s_or_saveexec_b64 s[0:1], -1
+; GCN-NEXT:    v_mov_b32_e32 v0, 0x3f803f80
+; GCN-NEXT:    v_cndmask_b32_e64 v1, v0, v2, s[0:1]
+; GCN-NEXT:    s_mov_b64 exec, s[0:1]
+; GCN-NEXT:    v_mov_b32_e32 v2, s2
+; GCN-NEXT:    s_or_saveexec_b64 s[0:1], -1
+; GCN-NEXT:    v_cndmask_b32_e64 v0, v0, v2, s[0:1]
+; GCN-NEXT:    s_mov_b64 exec, s[0:1]
+; GCN-NEXT:    v_mov_b32_e32 v2, v0
 ; GCN-NEXT:    v_mov_b32_e32 v3, v1
-; GCN-NEXT:    v_mov_b32_e32 v4, v2
-; GCN-NEXT:    s_mov_b32 s3, 0xf000
-; GCN-NEXT:    buffer_store_dwordx2 v[3:4], off, s[0:3], 0
+; GCN-NEXT:    buffer_store_dwordx2 v[2:3], off, s[4:7], 0
 ; GCN-NEXT:    s_endpgm
   %tmp.0 = call <4 x bfloat> @llvm.amdgcn.set.inactive.v4bf16(<4 x bfloat> %in, <4 x bfloat> <bfloat 1.0, bfloat 1.0, bfloat 1.0, bfloat 1.0>) #0
   %tmp = call <4 x bfloat> @llvm.amdgcn.strict.wwm.v4bf16(<4 x bfloat> %tmp.0)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/mubuf-global.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/mubuf-global.ll
index 2c71366772fc9..31526bcfead4e 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/mubuf-global.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/mubuf-global.ll
@@ -832,6 +832,7 @@ define amdgpu_ps float @mubuf_load_vgpr_ptr_offset4294967296(ptr addrspace(1) %p
 ; GFX12-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, 4, v1, vcc_lo
 ; GFX12-NEXT:    global_load_b32 v0, v[0:1], off scope:SCOPE_SYS
 ; GFX12-NEXT:    s_wait_loadcnt 0x0
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    ; return to shader part epilog
   %gep = getelementptr float, ptr addrspace(1) %ptr, i64 4294967296
   %val = load volatile float, ptr addrspace(1) %gep
@@ -865,6 +866,7 @@ define amdgpu_ps float @mubuf_load_vgpr_ptr_offset4294967297(ptr addrspace(1) %p
 ; GFX12-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, 4, v1, vcc_lo
 ; GFX12-NEXT:    global_load_b32 v0, v[0:1], off scope:SCOPE_SYS
 ; GFX12-NEXT:    s_wait_loadcnt 0x0
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    ; return to shader part epilog
   %gep = getelementptr float, ptr addrspace(1) %ptr, i64 4294967297
   %val = load volatile float, ptr addrspace(1) %gep
@@ -979,6 +981,7 @@ define amdgpu_ps float @mubuf_load_vgpr_ptr_sgpr_offset(ptr addrspace(1) %ptr, i
 ; GFX12-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, v1, v3, vcc_lo
 ; GFX12-NEXT:    global_load_b32 v0, v[0:1], off scope:SCOPE_SYS
 ; GFX12-NEXT:    s_wait_loadcnt 0x0
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    ; return to shader part epilog
   %gep = getelementptr float, ptr addrspace(1) %ptr, i32 %soffset
   %val = load volatile float, ptr addrspace(1) %gep
@@ -1017,6 +1020,7 @@ define amdgpu_ps float @mubuf_load_vgpr_ptr_sgpr_offset_offset256(ptr addrspace(
 ; GFX12-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, v1, v3, vcc_lo
 ; GFX12-NEXT:    global_load_b32 v0, v[0:1], off offset:1024 scope:SCOPE_SYS
 ; GFX12-NEXT:    s_wait_loadcnt 0x0
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    ; return to shader part epilog
   %gep0 = getelementptr float, ptr addrspace(1) %ptr, i32 %soffset
   %gep1 = getelementptr float, ptr addrspace(1) %gep0, i32 256
@@ -1056,6 +1060,7 @@ define amdgpu_ps float @mubuf_load_vgpr_ptr_sgpr_offset256_offset(ptr addrspace(
 ; GFX12-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, v1, v3, vcc_lo
 ; GFX12-NEXT:    global_load_b32 v0, v[0:1], off offset:1024 scope:SCOPE_SYS
 ; GFX12-NEXT:    s_wait_loadcnt 0x0
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    ; return to shader part epilog
   %gep0 = getelementptr float, ptr addrspace(1) %ptr, i64 256
   %gep1 = getelementptr float, ptr addrspace(1) %gep0, i32 %soffset
@@ -1099,6 +1104,7 @@ define amdgpu_ps float @mubuf_load_sgpr_ptr_vgpr_offset(ptr addrspace(1) inreg %
 ; GFX12-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, v3, v1, vcc_lo
 ; GFX12-NEXT:    global_load_b32 v0, v[0:1], off scope:SCOPE_SYS
 ; GFX12-NEXT:    s_wait_loadcnt 0x0
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    ; return to shader part epilog
   %gep = getelementptr float, ptr addrspace(1) %ptr, i32 %voffset
   %val = load volatile float, ptr addrspace(1) %gep
@@ -1143,6 +1149,7 @@ define amdgpu_ps float @mubuf_load_sgpr_ptr_vgpr_offset_offset4095(ptr addrspace
 ; GFX12-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, v3, v1, vcc_lo
 ; GFX12-NEXT:    global_load_b32 v0, v[0:1], off offset:16380 scope:SCOPE_SYS
 ; GFX12-NEXT:    s_wait_loadcnt 0x0
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    ; return to shader part epilog
   %gep0 = getelementptr float, ptr addrspace(1) %ptr, i32 %voffset
   %gep1 = getelementptr float, ptr addrspace(1) %gep0, i64 4095
@@ -1187,6 +1194,7 @@ define amdgpu_ps float @mubuf_load_sgpr_ptr_offset4095_vgpr_offset(ptr addrspace
 ; GFX12-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, v3, v1, vcc_lo
 ; GFX12-NEXT:    global_load_b32 v0, v[0:1], off offset:16380 scope:SCOPE_SYS
 ; GFX12-NEXT:    s_wait_loadcnt 0x0
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    ; return to shader part epilog
   %gep0 = getelementptr float, ptr addrspace(1) %ptr, i64 4095
   %gep1 = getelementptr float, ptr addrspace(1) %gep0, i32 %voffset
@@ -1359,6 +1367,7 @@ define amdgpu_ps float @mubuf_atomicrmw_vgpr_ptr_offset4294967296(ptr addrspace(
 ; GFX12-NEXT:    global_atomic_add_u32 v0, v[0:1], v2, off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
 ; GFX12-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-NEXT:    global_inv scope:SCOPE_DEV
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    ; return to shader part epilog
   %gep = getelementptr i32, ptr addrspace(1) %ptr, i64 4294967296
   %result = atomicrmw add ptr addrspace(1) %gep, i32 2 syncscope("agent") seq_cst
@@ -1411,6 +1420,7 @@ define amdgpu_ps float @mubuf_atomicrmw_sgpr_ptr_vgpr_offset(ptr addrspace(1) in
 ; GFX12-NEXT:    global_atomic_add_u32 v0, v[0:1], v4, off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
 ; GFX12-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-NEXT:    global_inv scope:SCOPE_DEV
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    ; return to shader part epilog
   %gep = getelementptr i32, ptr addrspace(1) %ptr, i32 %voffset
   %result = atomicrmw add ptr addrspace(1) %gep, i32 2 syncscope("agent") seq_cst
@@ -1590,6 +1600,7 @@ define amdgpu_ps float @mubuf_cmpxchg_vgpr_ptr_offset4294967296(ptr addrspace(1)
 ; GFX12-NEXT:    global_atomic_cmpswap_b32 v0, v[0:1], v[3:4], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
 ; GFX12-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-NEXT:    global_inv scope:SCOPE_DEV
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    ; return to shader part epilog
   %gep = getelementptr i32, ptr addrspace(1) %ptr, i64 4294967296
   %result.struct = cmpxchg ptr addrspace(1) %gep, i32 %old, i32 %in syncscope("agent") seq_cst seq_cst
@@ -1643,6 +1654,7 @@ define amdgpu_ps float @mubuf_cmpxchg_sgpr_ptr_vgpr_offset(ptr addrspace(1) inre
 ; GFX12-NEXT:    global_atomic_cmpswap_b32 v0, v[0:1], v[2:3], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
 ; GFX12-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-NEXT:    global_inv scope:SCOPE_DEV
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    ; return to shader part epilog
   %gep = getelementptr i32, ptr addrspace(1) %ptr, i32 %voffset
   %result.struct = cmpxchg ptr addrspace(1) %gep, i32 %old, i32 %in syncscope("agent") seq_cst seq_cst
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/mul.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/mul.ll
index 7c6daf769aec2..ba2af13338be6 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/mul.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/mul.ll
@@ -1072,14 +1072,17 @@ define i128 @v_mul_i128(i128 %num, i128 %den) {
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_mov_b32_e32 v2, v11
 ; GFX12-NEXT:    v_mad_co_u64_u32 v[1:2], vcc_lo, v8, v5, v[1:2]
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_4)
 ; GFX12-NEXT:    v_mad_co_u64_u32 v[1:2], s0, v9, v4, v[1:2]
+; GFX12-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-NEXT:    v_add_co_ci_u32_e64 v7, s0, v12, v7, s0
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_add_co_ci_u32_e32 v6, vcc_lo, v7, v6, vcc_lo
 ; GFX12-NEXT:    v_mad_co_u64_u32 v[5:6], null, v10, v5, v[6:7]
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-NEXT:    v_mad_co_u64_u32 v[3:4], null, v3, v4, v[5:6]
+; GFX12-NEXT:    s_wait_alu 0xf1fd
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
   %result = mul i128 %num, %den
   ret i128 %result
@@ -2433,71 +2436,83 @@ define i256 @v_mul_i256(i256 %num, i256 %den) {
 ; GFX12-NEXT:    v_mad_co_u64_u32 v[18:19], null, v16, v12, 0
 ; GFX12-NEXT:    v_mul_lo_u32 v30, v17, v14
 ; GFX12-NEXT:    v_mad_co_u64_u32 v[0:1], null, v17, v13, v[0:1]
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_3)
 ; GFX12-NEXT:    v_mad_co_u64_u32 v[18:19], s0, v17, v11, v[18:19]
+; GFX12-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-NEXT:    v_cndmask_b32_e64 v20, 0, 1, s0
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
 ; GFX12-NEXT:    v_mad_co_u64_u32 v[0:1], null, v2, v12, v[0:1]
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
 ; GFX12-NEXT:    v_mad_co_u64_u32 v[18:19], vcc_lo, v2, v10, v[18:19]
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_add_co_ci_u32_e32 v22, vcc_lo, 0, v20, vcc_lo
 ; GFX12-NEXT:    v_mad_co_u64_u32 v[20:21], null, v16, v10, 0
-; GFX12-NEXT:    v_mad_co_u64_u32 v[0:1], null, v3, v11, v[0:1]
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX12-NEXT:    v_mad_co_u64_u32 v[0:1], null, v3, v11, v[0:1]
 ; GFX12-NEXT:    v_mad_co_u64_u32 v[18:19], vcc_lo, v3, v9, v[18:19]
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_add_co_ci_u32_e32 v24, vcc_lo, 0, v22, vcc_lo
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
 ; GFX12-NEXT:    v_mad_co_u64_u32 v[0:1], null, v4, v10, v[0:1]
 ; GFX12-NEXT:    v_mad_co_u64_u32 v[18:19], vcc_lo, v4, v8, v[18:19]
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
 ; GFX12-NEXT:    v_add_co_ci_u32_e32 v26, vcc_lo, 0, v24, vcc_lo
 ; GFX12-NEXT:    v_mad_co_u64_u32 v[0:1], null, v5, v9, v[0:1]
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_3)
 ; GFX12-NEXT:    v_mad_co_u64_u32 v[22:23], null, v6, v8, v[0:1]
 ; GFX12-NEXT:    v_mad_co_u64_u32 v[0:1], s0, v17, v9, v[20:21]
+; GFX12-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-NEXT:    v_cndmask_b32_e64 v25, 0, 1, s0
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
 ; GFX12-NEXT:    v_mov_b32_e32 v20, v22
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
 ; GFX12-NEXT:    v_mad_co_u64_u32 v[21:22], vcc_lo, v2, v8, v[0:1]
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_add_co_ci_u32_e32 v29, vcc_lo, 0, v25, vcc_lo
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
 ; GFX12-NEXT:    v_mad_co_u64_u32 v[0:1], s0, v16, v13, v[19:20]
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
 ; GFX12-NEXT:    v_mov_b32_e32 v19, v22
 ; GFX12-NEXT:    v_mul_lo_u32 v22, v16, v15
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_mad_co_u64_u32 v[24:25], vcc_lo, v17, v12, v[0:1]
 ; GFX12-NEXT:    v_mad_co_u64_u32 v[0:1], null, v16, v8, 0
 ; GFX12-NEXT:    v_mov_b32_e32 v20, v18
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_4)
 ; GFX12-NEXT:    v_mad_co_u64_u32 v[14:15], s2, v16, v11, v[19:20]
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_4)
 ; GFX12-NEXT:    v_mad_co_u64_u32 v[18:19], s1, v2, v11, v[24:25]
 ; GFX12-NEXT:    v_mul_lo_u32 v20, v4, v11
 ; GFX12-NEXT:    v_mul_lo_u32 v25, v3, v12
+; GFX12-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-NEXT:    v_cndmask_b32_e64 v6, 0, 1, s2
 ; GFX12-NEXT:    v_mul_lo_u32 v24, v2, v13
 ; GFX12-NEXT:    v_mov_b32_e32 v13, v1
 ; GFX12-NEXT:    v_mad_co_u64_u32 v[11:12], s2, v17, v10, v[14:15]
 ; GFX12-NEXT:    v_mad_co_u64_u32 v[18:19], s3, v3, v10, v[18:19]
+; GFX12-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-NEXT:    v_add_co_ci_u32_e64 v6, s2, 0, v6, s2
 ; GFX12-NEXT:    v_mov_b32_e32 v14, v21
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3)
 ; GFX12-NEXT:    v_mad_co_u64_u32 v[1:2], s2, v2, v9, v[11:12]
+; GFX12-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-NEXT:    v_add_co_ci_u32_e64 v6, s2, 0, v6, s2
 ; GFX12-NEXT:    v_mad_co_u64_u32 v[10:11], s2, v4, v9, v[18:19]
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_mad_co_u64_u32 v[12:13], s4, v16, v9, v[13:14]
+; GFX12-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-NEXT:    v_cndmask_b32_e64 v9, 0, 1, s4
 ; GFX12-NEXT:    v_mad_co_u64_u32 v[3:4], s4, v3, v8, v[1:2]
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX12-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-NEXT:    v_add_co_ci_u32_e64 v14, s4, 0, v6, s4
 ; GFX12-NEXT:    v_mad_co_u64_u32 v[5:6], s4, v5, v8, v[10:11]
 ; GFX12-NEXT:    v_mad_co_u64_u32 v[1:2], s5, v17, v8, v[12:13]
+; GFX12-NEXT:    s_wait_alu 0xf1ff
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_4)
 ; GFX12-NEXT:    v_add_co_ci_u32_e64 v3, s5, v9, v3, s5
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-NEXT:    v_add_co_ci_u32_e64 v4, s5, v29, v4, s5
+; GFX12-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-NEXT:    v_add_co_ci_u32_e64 v5, s5, v14, v5, s5
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-NEXT:    v_add_co_ci_u32_e64 v6, s5, v26, v6, s5
+; GFX12-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-NEXT:    v_add_co_ci_u32_e64 v9, s5, v23, v22, s5
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_add_co_ci_u32_e64 v9, s4, v9, v30, s4
@@ -2505,11 +2520,13 @@ define i256 @v_mul_i256(i256 %num, i256 %den) {
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_add_co_ci_u32_e64 v9, s2, v9, v25, s3
 ; GFX12-NEXT:    v_add_co_ci_u32_e64 v9, s1, v9, v20, s1
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_add_co_ci_u32_e32 v9, vcc_lo, v9, v28, vcc_lo
 ; GFX12-NEXT:    v_add_co_ci_u32_e64 v9, vcc_lo, v9, v27, s0
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-NEXT:    v_mad_co_u64_u32 v[7:8], null, v7, v8, v[9:10]
+; GFX12-NEXT:    s_wait_alu 0xf1fd
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
   %result = mul i256 %num, %den
   ret i256 %result
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/no-ctlz-from-umul-to-lshr-in-postlegalizer.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/no-ctlz-from-umul-to-lshr-in-postlegalizer.ll
new file mode 100644
index 0000000000000..c237911319329
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/no-ctlz-from-umul-to-lshr-in-postlegalizer.ll
@@ -0,0 +1,98 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx90a -O0 -global-isel=true %s -o - | FileCheck %s
+
+define void @test(ptr %p) {
+; CHECK-LABEL: test:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    v_mov_b32_e32 v2, v1
+; CHECK-NEXT:    ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec
+; CHECK-NEXT:    v_mov_b32_e32 v1, v2
+; CHECK-NEXT:    s_mov_b32 s5, 16
+; CHECK-NEXT:    s_mov_b32 s6, 0
+; CHECK-NEXT:    v_mov_b32_e32 v2, s6
+; CHECK-NEXT:    v_cvt_f32_ubyte0_e64 v2, v2
+; CHECK-NEXT:    v_rcp_iflag_f32_e64 v2, v2
+; CHECK-NEXT:    s_mov_b32 s4, 0x4f7ffffe
+; CHECK-NEXT:    v_mov_b32_e32 v3, s4
+; CHECK-NEXT:    v_mul_f32_e64 v2, v2, v3
+; CHECK-NEXT:    v_cvt_u32_f32_e64 v2, v2
+; CHECK-NEXT:    s_mov_b32 s7, 0
+; CHECK-NEXT:    v_mov_b32_e32 v3, s7
+; CHECK-NEXT:    v_mul_hi_u32 v3, v2, v3
+; CHECK-NEXT:    v_add_u32_e64 v2, v2, v3
+; CHECK-NEXT:    v_mov_b32_e32 v3, s5
+; CHECK-NEXT:    v_mul_hi_u32 v2, v2, v3
+; CHECK-NEXT:    s_mov_b32 s7, 2
+; CHECK-NEXT:    v_mov_b32_e32 v3, s7
+; CHECK-NEXT:    v_add_u32_e64 v2, v2, v3
+; CHECK-NEXT:    v_mov_b32_e32 v3, s6
+; CHECK-NEXT:    v_cvt_f32_ubyte0_e64 v3, v3
+; CHECK-NEXT:    v_rcp_iflag_f32_e64 v3, v3
+; CHECK-NEXT:    v_mov_b32_e32 v4, s4
+; CHECK-NEXT:    v_mul_f32_e64 v3, v3, v4
+; CHECK-NEXT:    v_cvt_u32_f32_e64 v3, v3
+; CHECK-NEXT:    s_mov_b32 s7, 0
+; CHECK-NEXT:    v_mov_b32_e32 v4, s7
+; CHECK-NEXT:    v_mul_hi_u32 v4, v3, v4
+; CHECK-NEXT:    v_add_u32_e64 v3, v3, v4
+; CHECK-NEXT:    v_mov_b32_e32 v4, s5
+; CHECK-NEXT:    v_mul_hi_u32 v3, v3, v4
+; CHECK-NEXT:    s_mov_b32 s7, 2
+; CHECK-NEXT:    v_mov_b32_e32 v4, s7
+; CHECK-NEXT:    v_add_u32_e64 v6, v3, v4
+; CHECK-NEXT:    v_mov_b32_e32 v3, s6
+; CHECK-NEXT:    v_cvt_f32_ubyte0_e64 v3, v3
+; CHECK-NEXT:    v_rcp_iflag_f32_e64 v3, v3
+; CHECK-NEXT:    v_mov_b32_e32 v4, s4
+; CHECK-NEXT:    v_mul_f32_e64 v3, v3, v4
+; CHECK-NEXT:    v_cvt_u32_f32_e64 v3, v3
+; CHECK-NEXT:    s_mov_b32 s7, 0
+; CHECK-NEXT:    v_mov_b32_e32 v4, s7
+; CHECK-NEXT:    v_mul_hi_u32 v4, v3, v4
+; CHECK-NEXT:    v_add_u32_e64 v3, v3, v4
+; CHECK-NEXT:    v_mov_b32_e32 v4, s5
+; CHECK-NEXT:    v_mul_hi_u32 v3, v3, v4
+; CHECK-NEXT:    s_mov_b32 s7, 2
+; CHECK-NEXT:    v_mov_b32_e32 v4, s7
+; CHECK-NEXT:    v_add_u32_e64 v3, v3, v4
+; CHECK-NEXT:    v_mov_b32_e32 v4, s6
+; CHECK-NEXT:    v_cvt_f32_ubyte0_e64 v4, v4
+; CHECK-NEXT:    v_rcp_iflag_f32_e64 v4, v4
+; CHECK-NEXT:    v_mov_b32_e32 v5, s4
+; CHECK-NEXT:    v_mul_f32_e64 v4, v4, v5
+; CHECK-NEXT:    v_cvt_u32_f32_e64 v4, v4
+; CHECK-NEXT:    s_mov_b32 s4, 0
+; CHECK-NEXT:    v_mov_b32_e32 v5, s4
+; CHECK-NEXT:    v_mul_hi_u32 v5, v4, v5
+; CHECK-NEXT:    v_add_u32_e64 v4, v4, v5
+; CHECK-NEXT:    v_mov_b32_e32 v5, s5
+; CHECK-NEXT:    v_mul_hi_u32 v4, v4, v5
+; CHECK-NEXT:    s_mov_b32 s4, 2
+; CHECK-NEXT:    v_mov_b32_e32 v5, s4
+; CHECK-NEXT:    v_add_u32_e64 v4, v4, v5
+; CHECK-NEXT:    s_mov_b32 s4, 0xff
+; CHECK-NEXT:    v_mov_b32_e32 v5, s4
+; CHECK-NEXT:    v_mov_b32_e32 v7, s4
+; CHECK-NEXT:    v_and_b32_e64 v7, v6, v7
+; CHECK-NEXT:    s_mov_b32 s6, 8
+; CHECK-NEXT:    v_mov_b32_e32 v6, s6
+; CHECK-NEXT:    v_lshlrev_b32_e64 v6, v6, v7
+; CHECK-NEXT:    v_and_or_b32 v2, v2, v5, v6
+; CHECK-NEXT:    v_mov_b32_e32 v5, s4
+; CHECK-NEXT:    v_and_b32_e64 v5, v3, v5
+; CHECK-NEXT:    v_mov_b32_e32 v3, s5
+; CHECK-NEXT:    v_lshlrev_b32_e64 v3, v3, v5
+; CHECK-NEXT:    v_mov_b32_e32 v5, s4
+; CHECK-NEXT:    v_and_b32_e64 v5, v4, v5
+; CHECK-NEXT:    s_mov_b32 s4, 24
+; CHECK-NEXT:    v_mov_b32_e32 v4, s4
+; CHECK-NEXT:    v_lshlrev_b32_e64 v4, v4, v5
+; CHECK-NEXT:    v_or3_b32 v2, v2, v3, v4
+; CHECK-NEXT:    flat_store_dword v[0:1], v2
+; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    s_setpc_b64 s[30:31]
+  %B = udiv <4 x i8> splat (i8 16), zeroinitializer
+  store <4 x i8> %B, ptr %p, align 4
+  ret void
+}
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/no-ctlz-from-umul-to-lshr-in-postlegalizer.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/no-ctlz-from-umul-to-lshr-in-postlegalizer.mir
new file mode 100644
index 0000000000000..00ead74cb37bb
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/no-ctlz-from-umul-to-lshr-in-postlegalizer.mir
@@ -0,0 +1,23 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 5
+# RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx90a -run-pass=amdgpu-postlegalizer-combiner %s -o - | FileCheck %s
+
+---
+name: test
+tracksRegLiveness: true
+legalized: true
+body: |
+  bb.0:
+    liveins: $vgpr0, $vgpr1
+    ; CHECK-LABEL: name: test
+    ; CHECK: liveins: $vgpr0, $vgpr1
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
+    ; CHECK-NEXT: [[UMULH:%[0-9]+]]:_(s32) = G_UMULH [[COPY]], [[C]]
+    ; CHECK-NEXT: $vgpr0 = COPY [[UMULH]](s32)
+    ; CHECK-NEXT: SI_RETURN implicit $vgpr0
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s32) = G_CONSTANT i32 4
+    %2:_(s32) = G_UMULH %0:_, %1:_
+    $vgpr0 = COPY %2:_(s32)
+    SI_RETURN implicit $vgpr0
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/promote-dependency-on-invariant-result.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/promote-dependency-on-invariant-result.ll
new file mode 100644
index 0000000000000..090aa067a5260
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/promote-dependency-on-invariant-result.ll
@@ -0,0 +1,23 @@
+; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 5
+; RUN: llc -mtriple=amdgcn -mcpu=gfx90a -O0 -global-isel=true --stop-after=irtranslator -o - %s | FileCheck %s
+
+declare ptr @llvm.invariant.start.p5(i64 immarg, ptr addrspace(5) nocapture)
+declare void @llvm.invariant.end.p5(ptr, i64 immarg, ptr addrspace(5) nocapture)
+
+define void @use_invariant_promotable_lds(ptr addrspace(5) %arg, i32 %i) {
+  ; CHECK-LABEL: name: use_invariant_promotable_lds
+  ; CHECK: bb.1.bb:
+  ; CHECK-NEXT:   liveins: $vgpr0, $vgpr1
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
+  ; CHECK-NEXT:   [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+  ; CHECK-NEXT:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+  ; CHECK-NEXT:   [[DEF:%[0-9]+]]:_(p0) = G_IMPLICIT_DEF
+  ; CHECK-NEXT:   G_STORE [[C]](s32), [[DEF]](p0) :: (store (s32) into %ir.tmp)
+  ; CHECK-NEXT:   SI_RETURN
+bb:
+  %tmp = call ptr @llvm.invariant.start.p5(i64 4, ptr addrspace(5) %arg)
+  call void @llvm.invariant.end.p5(ptr %tmp, i64 4, ptr addrspace(5) %arg)
+  store i32 0, ptr %tmp, align 4
+  ret void
+}
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-mui.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-mui.ll
index 191739b37672e..951182f703a4e 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-mui.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-mui.ll
@@ -387,8 +387,8 @@ define amdgpu_ps void @and_i1_scc(i32 inreg %a, i32 inreg %b, ptr addrspace(1) %
 define amdgpu_ps void @divergent_phi_with_uniform_inputs(i32 %a, ptr addrspace(1) %out) {
 ; OLD_RBS-LABEL: divergent_phi_with_uniform_inputs:
 ; OLD_RBS:       ; %bb.0: ; %A
-; OLD_RBS-NEXT:    s_mov_b32 s0, 0
 ; OLD_RBS-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
+; OLD_RBS-NEXT:    s_mov_b32 s0, 0
 ; OLD_RBS-NEXT:    s_and_saveexec_b32 s1, vcc_lo
 ; OLD_RBS-NEXT:  ; %bb.1: ; %B
 ; OLD_RBS-NEXT:    s_mov_b32 s0, 1
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-split-scalar-load-metadata.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-split-scalar-load-metadata.mir
index c2dbeafce3df7..d15919fb12a73 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-split-scalar-load-metadata.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-split-scalar-load-metadata.mir
@@ -17,6 +17,7 @@
   !0 = !{i96 0, i96 9223372036854775808}
   !1 = !{!"omnipotent char", !2}
   !2 = !{!"Simple C/C++ TBAA"}
+  !3 = !{i32 0, i32 2147483646}
 ...
 
 # Make sure range metadata is not preserved when widening loads, but
@@ -44,10 +45,10 @@ body: |
     ; GFX12: liveins: $sgpr0_sgpr1
     ; GFX12-NEXT: {{  $}}
     ; GFX12-NEXT: [[COPY:%[0-9]+]]:sgpr(p4) = COPY $sgpr0_sgpr1
-    ; GFX12-NEXT: [[LOAD:%[0-9]+]]:sgpr(<3 x s32>) = G_LOAD [[COPY]](p4) :: (load (<3 x s32>), align 8, !range !0, addrspace 4)
+    ; GFX12-NEXT: [[LOAD:%[0-9]+]]:sgpr(<3 x s32>) = G_LOAD [[COPY]](p4) :: (load (<3 x s32>), align 8
     ; GFX12-NEXT: $sgpr0_sgpr1_sgpr2 = COPY [[LOAD]](<3 x s32>)
     %0:_(p4) = COPY $sgpr0_sgpr1
-    %1:_(<3 x s32>) = G_LOAD %0 :: (load (<3 x s32>), align 8, addrspace 4, !range !0)
+    %1:_(<3 x s32>) = G_LOAD %0 :: (load (<3 x s32>), align 8, addrspace 4, !range !3)
     $sgpr0_sgpr1_sgpr2 = COPY %1
 
 ...
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/sdivrem.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/sdivrem.ll
index 40f29c56c8f12..d41601cc0d76e 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/sdivrem.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/sdivrem.ll
@@ -1635,6 +1635,7 @@ define amdgpu_kernel void @sdivrem_v2i64(ptr addrspace(1) %out0, ptr addrspace(1
 ; GFX9-NEXT:    v_add_co_u32_e32 v3, vcc, v3, v0
 ; GFX9-NEXT:    v_addc_co_u32_e32 v4, vcc, v4, v1, vcc
 ; GFX9-NEXT:    v_mad_u64_u32 v[0:1], s[0:1], s16, v3, 0
+; GFX9-NEXT:    v_mov_b32_e32 v7, s11
 ; GFX9-NEXT:    v_mad_u64_u32 v[1:2], s[0:1], s16, v4, v[1:2]
 ; GFX9-NEXT:    v_mul_hi_u32 v6, v3, v0
 ; GFX9-NEXT:    v_mad_u64_u32 v[1:2], s[0:1], s17, v3, v[1:2]
@@ -1682,33 +1683,32 @@ define amdgpu_kernel void @sdivrem_v2i64(ptr addrspace(1) %out0, ptr addrspace(1
 ; GFX9-NEXT:    v_mad_u64_u32 v[0:1], s[0:1], s8, v5, 0
 ; GFX9-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc
 ; GFX9-NEXT:    v_add_u32_e32 v3, v4, v3
-; GFX9-NEXT:    v_add3_u32 v3, v3, v2, v6
-; GFX9-NEXT:    v_mad_u64_u32 v[1:2], s[0:1], s8, v3, v[1:2]
-; GFX9-NEXT:    v_mov_b32_e32 v6, s11
-; GFX9-NEXT:    v_sub_co_u32_e32 v7, vcc, s10, v0
-; GFX9-NEXT:    v_mad_u64_u32 v[1:2], s[0:1], s9, v5, v[1:2]
+; GFX9-NEXT:    v_add3_u32 v6, v3, v2, v6
+; GFX9-NEXT:    v_mad_u64_u32 v[1:2], s[0:1], s8, v6, v[1:2]
+; GFX9-NEXT:    v_sub_co_u32_e32 v0, vcc, s10, v0
+; GFX9-NEXT:    v_mad_u64_u32 v[2:3], s[0:1], s9, v5, v[1:2]
 ; GFX9-NEXT:    v_mov_b32_e32 v4, s9
 ; GFX9-NEXT:    s_ashr_i32 s10, s3, 31
-; GFX9-NEXT:    v_subb_co_u32_e64 v6, s[0:1], v6, v1, vcc
-; GFX9-NEXT:    v_sub_u32_e32 v0, s11, v1
-; GFX9-NEXT:    v_cmp_le_u32_e64 s[0:1], s9, v6
-; GFX9-NEXT:    v_cndmask_b32_e64 v1, 0, -1, s[0:1]
-; GFX9-NEXT:    v_cmp_le_u32_e64 s[0:1], s8, v7
-; GFX9-NEXT:    v_subb_co_u32_e32 v0, vcc, v0, v4, vcc
-; GFX9-NEXT:    v_cndmask_b32_e64 v2, 0, -1, s[0:1]
-; GFX9-NEXT:    v_cmp_eq_u32_e64 s[0:1], s9, v6
-; GFX9-NEXT:    v_subrev_co_u32_e32 v9, vcc, s8, v7
-; GFX9-NEXT:    v_cndmask_b32_e64 v8, v1, v2, s[0:1]
-; GFX9-NEXT:    v_subbrev_co_u32_e64 v10, s[0:1], 0, v0, vcc
-; GFX9-NEXT:    v_add_co_u32_e64 v2, s[0:1], 1, v5
-; GFX9-NEXT:    v_addc_co_u32_e64 v11, s[0:1], 0, v3, s[0:1]
-; GFX9-NEXT:    v_cmp_le_u32_e64 s[0:1], s9, v10
-; GFX9-NEXT:    v_cndmask_b32_e64 v1, 0, -1, s[0:1]
-; GFX9-NEXT:    v_cmp_le_u32_e64 s[0:1], s8, v9
+; GFX9-NEXT:    v_subb_co_u32_e64 v1, s[0:1], v7, v2, vcc
+; GFX9-NEXT:    v_sub_u32_e32 v2, s11, v2
+; GFX9-NEXT:    v_cmp_le_u32_e64 s[0:1], s9, v1
+; GFX9-NEXT:    v_cndmask_b32_e64 v3, 0, -1, s[0:1]
+; GFX9-NEXT:    v_cmp_le_u32_e64 s[0:1], s8, v0
+; GFX9-NEXT:    v_subb_co_u32_e32 v2, vcc, v2, v4, vcc
+; GFX9-NEXT:    v_cndmask_b32_e64 v7, 0, -1, s[0:1]
+; GFX9-NEXT:    v_cmp_eq_u32_e64 s[0:1], s9, v1
+; GFX9-NEXT:    v_subrev_co_u32_e32 v8, vcc, s8, v0
+; GFX9-NEXT:    v_cndmask_b32_e64 v7, v3, v7, s[0:1]
+; GFX9-NEXT:    v_subbrev_co_u32_e64 v9, s[0:1], 0, v2, vcc
+; GFX9-NEXT:    v_add_co_u32_e64 v10, s[0:1], 1, v5
+; GFX9-NEXT:    v_addc_co_u32_e64 v11, s[0:1], 0, v6, s[0:1]
+; GFX9-NEXT:    v_cmp_le_u32_e64 s[0:1], s9, v9
+; GFX9-NEXT:    v_cndmask_b32_e64 v3, 0, -1, s[0:1]
+; GFX9-NEXT:    v_cmp_le_u32_e64 s[0:1], s8, v8
 ; GFX9-NEXT:    v_cndmask_b32_e64 v12, 0, -1, s[0:1]
-; GFX9-NEXT:    v_cmp_eq_u32_e64 s[0:1], s9, v10
-; GFX9-NEXT:    v_cndmask_b32_e64 v12, v1, v12, s[0:1]
-; GFX9-NEXT:    v_add_co_u32_e64 v13, s[0:1], 1, v2
+; GFX9-NEXT:    v_cmp_eq_u32_e64 s[0:1], s9, v9
+; GFX9-NEXT:    v_cndmask_b32_e64 v12, v3, v12, s[0:1]
+; GFX9-NEXT:    v_add_co_u32_e64 v13, s[0:1], 1, v10
 ; GFX9-NEXT:    v_addc_co_u32_e64 v14, s[0:1], 0, v11, s[0:1]
 ; GFX9-NEXT:    s_add_u32 s0, s18, s6
 ; GFX9-NEXT:    s_addc_u32 s1, s19, s6
@@ -1716,116 +1716,116 @@ define amdgpu_kernel void @sdivrem_v2i64(ptr addrspace(1) %out0, ptr addrspace(1
 ; GFX9-NEXT:    s_mov_b32 s11, s10
 ; GFX9-NEXT:    s_addc_u32 s3, s3, s10
 ; GFX9-NEXT:    s_xor_b64 s[2:3], s[2:3], s[10:11]
-; GFX9-NEXT:    v_cvt_f32_u32_e32 v1, s3
+; GFX9-NEXT:    v_cvt_f32_u32_e32 v3, s3
 ; GFX9-NEXT:    v_cvt_f32_u32_e32 v15, s2
-; GFX9-NEXT:    v_subb_co_u32_e32 v0, vcc, v0, v4, vcc
-; GFX9-NEXT:    v_mul_f32_e32 v1, 0x4f800000, v1
-; GFX9-NEXT:    v_add_f32_e32 v1, v1, v15
-; GFX9-NEXT:    v_rcp_iflag_f32_e32 v1, v1
-; GFX9-NEXT:    v_subrev_co_u32_e32 v4, vcc, s8, v9
-; GFX9-NEXT:    v_subbrev_co_u32_e32 v15, vcc, 0, v0, vcc
-; GFX9-NEXT:    v_mul_f32_e32 v0, 0x5f7ffffc, v1
-; GFX9-NEXT:    v_mul_f32_e32 v1, 0x2f800000, v0
-; GFX9-NEXT:    v_trunc_f32_e32 v16, v1
-; GFX9-NEXT:    v_mul_f32_e32 v1, 0xcf800000, v16
-; GFX9-NEXT:    v_add_f32_e32 v0, v1, v0
-; GFX9-NEXT:    v_cvt_u32_f32_e32 v17, v0
+; GFX9-NEXT:    v_subb_co_u32_e32 v2, vcc, v2, v4, vcc
+; GFX9-NEXT:    v_mul_f32_e32 v3, 0x4f800000, v3
+; GFX9-NEXT:    v_add_f32_e32 v3, v3, v15
+; GFX9-NEXT:    v_rcp_iflag_f32_e32 v3, v3
+; GFX9-NEXT:    v_subrev_co_u32_e32 v15, vcc, s8, v8
+; GFX9-NEXT:    v_subbrev_co_u32_e32 v16, vcc, 0, v2, vcc
+; GFX9-NEXT:    v_mul_f32_e32 v2, 0x5f7ffffc, v3
+; GFX9-NEXT:    v_mul_f32_e32 v3, 0x2f800000, v2
+; GFX9-NEXT:    v_trunc_f32_e32 v4, v3
+; GFX9-NEXT:    v_mul_f32_e32 v3, 0xcf800000, v4
+; GFX9-NEXT:    v_add_f32_e32 v2, v3, v2
+; GFX9-NEXT:    v_cvt_u32_f32_e32 v17, v2
 ; GFX9-NEXT:    s_xor_b64 s[8:9], s[0:1], s[6:7]
 ; GFX9-NEXT:    s_sub_u32 s5, 0, s2
 ; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v12
-; GFX9-NEXT:    v_mad_u64_u32 v[0:1], s[0:1], s5, v17, 0
-; GFX9-NEXT:    v_cndmask_b32_e32 v12, v2, v13, vcc
-; GFX9-NEXT:    v_cvt_u32_f32_e32 v13, v16
+; GFX9-NEXT:    v_mad_u64_u32 v[2:3], s[0:1], s5, v17, 0
+; GFX9-NEXT:    v_cvt_u32_f32_e32 v12, v4
 ; GFX9-NEXT:    s_subb_u32 s20, 0, s3
 ; GFX9-NEXT:    v_cndmask_b32_e32 v11, v11, v14, vcc
-; GFX9-NEXT:    v_cndmask_b32_e32 v4, v9, v4, vcc
-; GFX9-NEXT:    v_mad_u64_u32 v[1:2], s[0:1], s5, v13, v[1:2]
-; GFX9-NEXT:    v_cmp_ne_u32_e64 s[0:1], 0, v8
-; GFX9-NEXT:    v_cndmask_b32_e64 v8, v3, v11, s[0:1]
-; GFX9-NEXT:    v_mad_u64_u32 v[1:2], s[18:19], s20, v17, v[1:2]
-; GFX9-NEXT:    v_mul_lo_u32 v2, v13, v0
-; GFX9-NEXT:    v_cndmask_b32_e32 v9, v10, v15, vcc
-; GFX9-NEXT:    v_mul_lo_u32 v3, v17, v1
-; GFX9-NEXT:    v_mul_hi_u32 v10, v17, v0
-; GFX9-NEXT:    v_mul_hi_u32 v0, v13, v0
-; GFX9-NEXT:    v_cndmask_b32_e64 v5, v5, v12, s[0:1]
-; GFX9-NEXT:    v_add_co_u32_e32 v2, vcc, v2, v3
-; GFX9-NEXT:    v_cndmask_b32_e64 v3, 0, 1, vcc
-; GFX9-NEXT:    v_add_co_u32_e32 v2, vcc, v2, v10
-; GFX9-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc
-; GFX9-NEXT:    v_mul_lo_u32 v10, v13, v1
-; GFX9-NEXT:    v_add_u32_e32 v2, v3, v2
-; GFX9-NEXT:    v_mul_hi_u32 v3, v17, v1
-; GFX9-NEXT:    v_mul_hi_u32 v1, v13, v1
-; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, v10, v0
+; GFX9-NEXT:    v_cndmask_b32_e32 v10, v10, v13, vcc
+; GFX9-NEXT:    v_mad_u64_u32 v[3:4], s[0:1], s5, v12, v[3:4]
+; GFX9-NEXT:    v_cmp_ne_u32_e64 s[0:1], 0, v7
+; GFX9-NEXT:    v_mul_lo_u32 v7, v12, v2
+; GFX9-NEXT:    v_mad_u64_u32 v[3:4], s[18:19], s20, v17, v[3:4]
+; GFX9-NEXT:    v_cndmask_b32_e64 v4, v6, v11, s[0:1]
+; GFX9-NEXT:    v_cndmask_b32_e32 v6, v8, v15, vcc
+; GFX9-NEXT:    v_mul_lo_u32 v8, v17, v3
+; GFX9-NEXT:    v_cndmask_b32_e64 v5, v5, v10, s[0:1]
+; GFX9-NEXT:    v_mul_hi_u32 v10, v17, v2
+; GFX9-NEXT:    v_cndmask_b32_e32 v9, v9, v16, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v7, vcc, v7, v8
+; GFX9-NEXT:    v_cndmask_b32_e64 v8, 0, 1, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v7, vcc, v7, v10
+; GFX9-NEXT:    v_cndmask_b32_e64 v7, 0, 1, vcc
+; GFX9-NEXT:    v_mul_lo_u32 v10, v12, v3
+; GFX9-NEXT:    v_mul_hi_u32 v2, v12, v2
+; GFX9-NEXT:    v_add_u32_e32 v7, v8, v7
+; GFX9-NEXT:    v_mul_hi_u32 v8, v17, v3
+; GFX9-NEXT:    v_mul_hi_u32 v3, v12, v3
+; GFX9-NEXT:    v_add_co_u32_e32 v2, vcc, v10, v2
 ; GFX9-NEXT:    v_cndmask_b32_e64 v10, 0, 1, vcc
-; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, v0, v3
-; GFX9-NEXT:    v_cndmask_b32_e64 v3, 0, 1, vcc
-; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, v0, v2
-; GFX9-NEXT:    v_add_u32_e32 v3, v10, v3
-; GFX9-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc
-; GFX9-NEXT:    v_add_co_u32_e32 v10, vcc, v17, v0
-; GFX9-NEXT:    v_add3_u32 v1, v3, v2, v1
-; GFX9-NEXT:    v_mad_u64_u32 v[2:3], s[18:19], s5, v10, 0
-; GFX9-NEXT:    v_addc_co_u32_e32 v11, vcc, v13, v1, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v2, vcc, v2, v8
+; GFX9-NEXT:    v_cndmask_b32_e64 v8, 0, 1, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v2, vcc, v2, v7
+; GFX9-NEXT:    v_add_u32_e32 v8, v10, v8
+; GFX9-NEXT:    v_cndmask_b32_e64 v7, 0, 1, vcc
+; GFX9-NEXT:    v_add3_u32 v3, v8, v7, v3
+; GFX9-NEXT:    v_add_co_u32_e32 v7, vcc, v17, v2
+; GFX9-NEXT:    v_addc_co_u32_e32 v8, vcc, v12, v3, vcc
+; GFX9-NEXT:    v_mad_u64_u32 v[2:3], s[18:19], s5, v7, 0
+; GFX9-NEXT:    v_cndmask_b32_e64 v6, v0, v6, s[0:1]
+; GFX9-NEXT:    v_cndmask_b32_e64 v9, v1, v9, s[0:1]
 ; GFX9-NEXT:    v_mov_b32_e32 v0, v3
-; GFX9-NEXT:    v_cndmask_b32_e64 v7, v7, v4, s[0:1]
-; GFX9-NEXT:    v_cndmask_b32_e64 v6, v6, v9, s[0:1]
-; GFX9-NEXT:    v_mad_u64_u32 v[0:1], s[0:1], s5, v11, v[0:1]
+; GFX9-NEXT:    v_mad_u64_u32 v[0:1], s[0:1], s5, v8, v[0:1]
+; GFX9-NEXT:    v_xor_b32_e32 v10, s17, v4
 ; GFX9-NEXT:    v_xor_b32_e32 v5, s16, v5
-; GFX9-NEXT:    v_xor_b32_e32 v8, s17, v8
-; GFX9-NEXT:    v_mad_u64_u32 v[3:4], s[0:1], s20, v10, v[0:1]
-; GFX9-NEXT:    v_mov_b32_e32 v9, s17
+; GFX9-NEXT:    v_mad_u64_u32 v[3:4], s[0:1], s20, v7, v[0:1]
+; GFX9-NEXT:    v_mov_b32_e32 v11, s17
 ; GFX9-NEXT:    v_subrev_co_u32_e32 v0, vcc, s16, v5
-; GFX9-NEXT:    v_xor_b32_e32 v4, s4, v7
-; GFX9-NEXT:    v_mul_lo_u32 v5, v11, v2
-; GFX9-NEXT:    v_mul_lo_u32 v7, v10, v3
-; GFX9-NEXT:    v_subb_co_u32_e32 v1, vcc, v8, v9, vcc
-; GFX9-NEXT:    v_mul_hi_u32 v8, v10, v2
-; GFX9-NEXT:    v_add_co_u32_e32 v5, vcc, v5, v7
-; GFX9-NEXT:    v_cndmask_b32_e64 v7, 0, 1, vcc
-; GFX9-NEXT:    v_add_co_u32_e32 v5, vcc, v5, v8
+; GFX9-NEXT:    v_xor_b32_e32 v4, s4, v6
+; GFX9-NEXT:    v_mul_lo_u32 v5, v8, v2
+; GFX9-NEXT:    v_mul_lo_u32 v6, v7, v3
+; GFX9-NEXT:    v_subb_co_u32_e32 v1, vcc, v10, v11, vcc
+; GFX9-NEXT:    v_mul_hi_u32 v10, v7, v2
+; GFX9-NEXT:    v_add_co_u32_e32 v5, vcc, v5, v6
+; GFX9-NEXT:    v_cndmask_b32_e64 v6, 0, 1, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v5, vcc, v5, v10
 ; GFX9-NEXT:    v_cndmask_b32_e64 v5, 0, 1, vcc
-; GFX9-NEXT:    v_mul_lo_u32 v8, v11, v3
-; GFX9-NEXT:    v_mul_hi_u32 v2, v11, v2
-; GFX9-NEXT:    v_add_u32_e32 v5, v7, v5
-; GFX9-NEXT:    v_mul_hi_u32 v7, v10, v3
-; GFX9-NEXT:    v_mul_hi_u32 v3, v11, v3
-; GFX9-NEXT:    v_add_co_u32_e32 v2, vcc, v8, v2
-; GFX9-NEXT:    v_cndmask_b32_e64 v8, 0, 1, vcc
-; GFX9-NEXT:    v_add_co_u32_e32 v2, vcc, v2, v7
-; GFX9-NEXT:    v_cndmask_b32_e64 v7, 0, 1, vcc
+; GFX9-NEXT:    v_mul_lo_u32 v10, v8, v3
+; GFX9-NEXT:    v_mul_hi_u32 v2, v8, v2
+; GFX9-NEXT:    v_add_u32_e32 v5, v6, v5
+; GFX9-NEXT:    v_mul_hi_u32 v6, v7, v3
+; GFX9-NEXT:    v_mul_hi_u32 v3, v8, v3
+; GFX9-NEXT:    v_add_co_u32_e32 v2, vcc, v10, v2
+; GFX9-NEXT:    v_cndmask_b32_e64 v10, 0, 1, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v2, vcc, v2, v6
+; GFX9-NEXT:    v_cndmask_b32_e64 v6, 0, 1, vcc
 ; GFX9-NEXT:    v_add_co_u32_e32 v2, vcc, v2, v5
-; GFX9-NEXT:    v_add_u32_e32 v7, v8, v7
+; GFX9-NEXT:    v_add_u32_e32 v6, v10, v6
 ; GFX9-NEXT:    v_cndmask_b32_e64 v5, 0, 1, vcc
-; GFX9-NEXT:    v_add3_u32 v3, v7, v5, v3
-; GFX9-NEXT:    v_add_co_u32_e32 v2, vcc, v10, v2
-; GFX9-NEXT:    v_addc_co_u32_e32 v3, vcc, v11, v3, vcc
+; GFX9-NEXT:    v_add3_u32 v3, v6, v5, v3
+; GFX9-NEXT:    v_add_co_u32_e32 v2, vcc, v7, v2
+; GFX9-NEXT:    v_addc_co_u32_e32 v3, vcc, v8, v3, vcc
 ; GFX9-NEXT:    v_mul_lo_u32 v5, s9, v2
-; GFX9-NEXT:    v_mul_lo_u32 v7, s8, v3
-; GFX9-NEXT:    v_mul_hi_u32 v9, s8, v2
+; GFX9-NEXT:    v_mul_lo_u32 v6, s8, v3
+; GFX9-NEXT:    v_mul_hi_u32 v8, s8, v2
 ; GFX9-NEXT:    v_mul_hi_u32 v2, s9, v2
 ; GFX9-NEXT:    v_mul_hi_u32 v12, s9, v3
-; GFX9-NEXT:    v_add_co_u32_e32 v5, vcc, v5, v7
-; GFX9-NEXT:    v_cndmask_b32_e64 v7, 0, 1, vcc
-; GFX9-NEXT:    v_add_co_u32_e32 v5, vcc, v5, v9
+; GFX9-NEXT:    v_add_co_u32_e32 v5, vcc, v5, v6
+; GFX9-NEXT:    v_cndmask_b32_e64 v6, 0, 1, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v5, vcc, v5, v8
 ; GFX9-NEXT:    v_cndmask_b32_e64 v5, 0, 1, vcc
-; GFX9-NEXT:    v_mul_lo_u32 v9, s9, v3
-; GFX9-NEXT:    v_add_u32_e32 v5, v7, v5
-; GFX9-NEXT:    v_mul_hi_u32 v7, s8, v3
-; GFX9-NEXT:    v_xor_b32_e32 v6, s4, v6
-; GFX9-NEXT:    v_add_co_u32_e32 v2, vcc, v9, v2
-; GFX9-NEXT:    v_cndmask_b32_e64 v9, 0, 1, vcc
-; GFX9-NEXT:    v_add_co_u32_e32 v2, vcc, v2, v7
-; GFX9-NEXT:    v_cndmask_b32_e64 v7, 0, 1, vcc
+; GFX9-NEXT:    v_mul_lo_u32 v8, s9, v3
+; GFX9-NEXT:    v_add_u32_e32 v5, v6, v5
+; GFX9-NEXT:    v_mul_hi_u32 v6, s8, v3
+; GFX9-NEXT:    v_xor_b32_e32 v9, s4, v9
+; GFX9-NEXT:    v_add_co_u32_e32 v2, vcc, v8, v2
+; GFX9-NEXT:    v_cndmask_b32_e64 v8, 0, 1, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v2, vcc, v2, v6
+; GFX9-NEXT:    v_cndmask_b32_e64 v6, 0, 1, vcc
 ; GFX9-NEXT:    v_add_co_u32_e32 v10, vcc, v2, v5
 ; GFX9-NEXT:    v_mad_u64_u32 v[2:3], s[0:1], s2, v10, 0
-; GFX9-NEXT:    v_mov_b32_e32 v8, s4
 ; GFX9-NEXT:    v_cndmask_b32_e64 v11, 0, 1, vcc
+; GFX9-NEXT:    v_add_u32_e32 v6, v8, v6
+; GFX9-NEXT:    v_mov_b32_e32 v7, s4
 ; GFX9-NEXT:    v_subrev_co_u32_e32 v4, vcc, s4, v4
-; GFX9-NEXT:    v_subb_co_u32_e32 v5, vcc, v6, v8, vcc
-; GFX9-NEXT:    v_add_u32_e32 v6, v9, v7
 ; GFX9-NEXT:    v_add3_u32 v8, v6, v11, v12
+; GFX9-NEXT:    v_subb_co_u32_e32 v5, vcc, v9, v7, vcc
 ; GFX9-NEXT:    v_mad_u64_u32 v[6:7], s[0:1], s2, v8, v[3:4]
 ; GFX9-NEXT:    v_mov_b32_e32 v9, s9
 ; GFX9-NEXT:    v_sub_co_u32_e32 v2, vcc, s8, v2
diff --git a/llvm/test/CodeGen/AMDGPU/abi-attribute-hints-undefined-behavior.ll b/llvm/test/CodeGen/AMDGPU/abi-attribute-hints-undefined-behavior.ll
index 194a23fa0d4a9..b7fbb8b2236c3 100644
--- a/llvm/test/CodeGen/AMDGPU/abi-attribute-hints-undefined-behavior.ll
+++ b/llvm/test/CodeGen/AMDGPU/abi-attribute-hints-undefined-behavior.ll
@@ -25,10 +25,10 @@ define void @parent_func_missing_inputs() #0 {
 ; FIXEDABI-NEXT:    v_writelane_b32 v40, s16, 2
 ; FIXEDABI-NEXT:    s_addk_i32 s32, 0x400
 ; FIXEDABI-NEXT:    v_writelane_b32 v40, s30, 0
-; FIXEDABI-NEXT:    v_writelane_b32 v40, s31, 1
 ; FIXEDABI-NEXT:    s_getpc_b64 s[16:17]
 ; FIXEDABI-NEXT:    s_add_u32 s16, s16, requires_all_inputs@rel32@lo+4
 ; FIXEDABI-NEXT:    s_addc_u32 s17, s17, requires_all_inputs@rel32@hi+12
+; FIXEDABI-NEXT:    v_writelane_b32 v40, s31, 1
 ; FIXEDABI-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; FIXEDABI-NEXT:    v_readlane_b32 s31, v40, 1
 ; FIXEDABI-NEXT:    v_readlane_b32 s30, v40, 0
@@ -49,21 +49,21 @@ define amdgpu_kernel void @parent_kernel_missing_inputs() #0 {
 ; FIXEDABI-SDAG:       ; %bb.0:
 ; FIXEDABI-SDAG-NEXT:    s_add_i32 s4, s4, s9
 ; FIXEDABI-SDAG-NEXT:    s_lshr_b32 flat_scratch_hi, s4, 8
-; FIXEDABI-SDAG-NEXT:    v_lshlrev_b32_e32 v1, 10, v1
 ; FIXEDABI-SDAG-NEXT:    s_add_u32 s0, s0, s9
+; FIXEDABI-SDAG-NEXT:    v_lshlrev_b32_e32 v1, 10, v1
+; FIXEDABI-SDAG-NEXT:    s_addc_u32 s1, s1, 0
 ; FIXEDABI-SDAG-NEXT:    v_lshlrev_b32_e32 v2, 20, v2
 ; FIXEDABI-SDAG-NEXT:    v_or_b32_e32 v0, v0, v1
-; FIXEDABI-SDAG-NEXT:    s_addc_u32 s1, s1, 0
+; FIXEDABI-SDAG-NEXT:    s_mov_b32 flat_scratch_lo, s5
 ; FIXEDABI-SDAG-NEXT:    s_mov_b32 s14, s8
+; FIXEDABI-SDAG-NEXT:    s_getpc_b64 s[4:5]
+; FIXEDABI-SDAG-NEXT:    s_add_u32 s4, s4, requires_all_inputs@rel32@lo+4
+; FIXEDABI-SDAG-NEXT:    s_addc_u32 s5, s5, requires_all_inputs@rel32@hi+12
 ; FIXEDABI-SDAG-NEXT:    v_or_b32_e32 v31, v0, v2
 ; FIXEDABI-SDAG-NEXT:    s_mov_b64 s[8:9], 0
 ; FIXEDABI-SDAG-NEXT:    s_mov_b32 s12, s6
 ; FIXEDABI-SDAG-NEXT:    s_mov_b32 s13, s7
 ; FIXEDABI-SDAG-NEXT:    s_mov_b32 s32, 0
-; FIXEDABI-SDAG-NEXT:    s_mov_b32 flat_scratch_lo, s5
-; FIXEDABI-SDAG-NEXT:    s_getpc_b64 s[4:5]
-; FIXEDABI-SDAG-NEXT:    s_add_u32 s4, s4, requires_all_inputs@rel32@lo+4
-; FIXEDABI-SDAG-NEXT:    s_addc_u32 s5, s5, requires_all_inputs@rel32@hi+12
 ; FIXEDABI-SDAG-NEXT:    s_swappc_b64 s[30:31], s[4:5]
 ; FIXEDABI-SDAG-NEXT:    s_endpgm
 ;
@@ -71,21 +71,21 @@ define amdgpu_kernel void @parent_kernel_missing_inputs() #0 {
 ; FIXEDABI-GISEL:       ; %bb.0:
 ; FIXEDABI-GISEL-NEXT:    s_add_i32 s4, s4, s9
 ; FIXEDABI-GISEL-NEXT:    s_lshr_b32 flat_scratch_hi, s4, 8
-; FIXEDABI-GISEL-NEXT:    v_lshlrev_b32_e32 v1, 10, v1
 ; FIXEDABI-GISEL-NEXT:    s_add_u32 s0, s0, s9
+; FIXEDABI-GISEL-NEXT:    v_lshlrev_b32_e32 v1, 10, v1
+; FIXEDABI-GISEL-NEXT:    s_addc_u32 s1, s1, 0
 ; FIXEDABI-GISEL-NEXT:    v_or_b32_e32 v0, v0, v1
 ; FIXEDABI-GISEL-NEXT:    v_lshlrev_b32_e32 v1, 20, v2
-; FIXEDABI-GISEL-NEXT:    s_addc_u32 s1, s1, 0
+; FIXEDABI-GISEL-NEXT:    s_mov_b32 flat_scratch_lo, s5
 ; FIXEDABI-GISEL-NEXT:    s_mov_b32 s14, s8
 ; FIXEDABI-GISEL-NEXT:    v_or_b32_e32 v31, v0, v1
+; FIXEDABI-GISEL-NEXT:    s_getpc_b64 s[4:5]
+; FIXEDABI-GISEL-NEXT:    s_add_u32 s4, s4, requires_all_inputs@rel32@lo+4
+; FIXEDABI-GISEL-NEXT:    s_addc_u32 s5, s5, requires_all_inputs@rel32@hi+12
 ; FIXEDABI-GISEL-NEXT:    s_mov_b64 s[8:9], 0
 ; FIXEDABI-GISEL-NEXT:    s_mov_b32 s12, s6
 ; FIXEDABI-GISEL-NEXT:    s_mov_b32 s13, s7
 ; FIXEDABI-GISEL-NEXT:    s_mov_b32 s32, 0
-; FIXEDABI-GISEL-NEXT:    s_mov_b32 flat_scratch_lo, s5
-; FIXEDABI-GISEL-NEXT:    s_getpc_b64 s[4:5]
-; FIXEDABI-GISEL-NEXT:    s_add_u32 s4, s4, requires_all_inputs@rel32@lo+4
-; FIXEDABI-GISEL-NEXT:    s_addc_u32 s5, s5, requires_all_inputs@rel32@hi+12
 ; FIXEDABI-GISEL-NEXT:    s_swappc_b64 s[30:31], s[4:5]
 ; FIXEDABI-GISEL-NEXT:    s_endpgm
   call void @requires_all_inputs()
diff --git a/llvm/test/CodeGen/AMDGPU/add_i1.ll b/llvm/test/CodeGen/AMDGPU/add_i1.ll
index e9e3fa765b52f..ff1a3ee38be1d 100644
--- a/llvm/test/CodeGen/AMDGPU/add_i1.ll
+++ b/llvm/test/CodeGen/AMDGPU/add_i1.ll
@@ -3,8 +3,8 @@
 ; RUN: llc -mtriple=amdgcn -mcpu=gfx1100 -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,GFX10 %s
 
 ; GCN-LABEL: {{^}}add_var_var_i1:
-; GFX9:  s_xor_b64
-; GFX10: s_xor_b32
+; GFX9:  v_xor_b32_e32
+; GFX10: v_xor_b32_e32
 define amdgpu_kernel void @add_var_var_i1(ptr addrspace(1) %out, ptr addrspace(1) %in0, ptr addrspace(1) %in1) {
   %a = load volatile i1, ptr addrspace(1) %in0
   %b = load volatile i1, ptr addrspace(1) %in1
@@ -14,8 +14,8 @@ define amdgpu_kernel void @add_var_var_i1(ptr addrspace(1) %out, ptr addrspace(1
 }
 
 ; GCN-LABEL: {{^}}add_var_imm_i1:
-; GFX9:  s_not_b64
-; GFX10: s_not_b32
+; GFX9:  s_xor_b64
+; GFX10: s_xor_b32
 define amdgpu_kernel void @add_var_imm_i1(ptr addrspace(1) %out, ptr addrspace(1) %in) {
   %a = load volatile i1, ptr addrspace(1) %in
   %add = add i1 %a, 1
@@ -25,8 +25,8 @@ define amdgpu_kernel void @add_var_imm_i1(ptr addrspace(1) %out, ptr addrspace(1
 
 ; GCN-LABEL: {{^}}add_i1_cf:
 ; GCN: ; %endif
-; GFX9: s_not_b64
-; GFX10: s_not_b32
+; GFX9: s_xor_b64
+; GFX10: s_xor_b32
 define amdgpu_kernel void @add_i1_cf(ptr addrspace(1) %out, ptr addrspace(1) %a, ptr addrspace(1) %b) {
 entry:
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
diff --git a/llvm/test/CodeGen/AMDGPU/agpr-to-agpr-copy.mir b/llvm/test/CodeGen/AMDGPU/agpr-to-agpr-copy.mir
index 86a1a26cb7abf..6eb5b7bad166d 100644
--- a/llvm/test/CodeGen/AMDGPU/agpr-to-agpr-copy.mir
+++ b/llvm/test/CodeGen/AMDGPU/agpr-to-agpr-copy.mir
@@ -1,5 +1,6 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgcn -mcpu=gfx908 -run-pass=liveintervals,amdgpu-pre-ra-optimizations -verify-machineinstrs %s -o - | FileCheck -check-prefix=GFX908 %s
+# RUN: llc -mtriple=amdgcn -mcpu=gfx908 -run-pass=amdgpu-pre-ra-optimizations -verify-machineinstrs %s -o - | FileCheck -check-prefix=GFX908 %s
+# RUN: llc -mtriple=amdgcn -mcpu=gfx908 -passes="amdgpu-pre-ra-optimizations" %s -o - | FileCheck -check-prefix=GFX908 %s
 
 ---
 name: test_mfma_f32_4x4x1f32_propagate_vgpr
diff --git a/llvm/test/CodeGen/AMDGPU/amdgpu-branch-weight-metadata.ll b/llvm/test/CodeGen/AMDGPU/amdgpu-branch-weight-metadata.ll
new file mode 100644
index 0000000000000..1da8cd6646d53
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/amdgpu-branch-weight-metadata.ll
@@ -0,0 +1,361 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc -mtriple=amdgcn -mcpu=gfx90a < %s | FileCheck -check-prefixes=GFX9 %s
+; RUN: llc -mtriple=amdgcn -mcpu=gfx1010 < %s | FileCheck -check-prefixes=GFX10,GFX1010 %s
+; RUN: llc -mtriple=amdgcn -mcpu=gfx1030 < %s | FileCheck -check-prefixes=GFX10,GFX1030 %s
+
+define void @uniform_br_no_metadata(i32 noundef inreg %value, ptr addrspace(8) nocapture writeonly inreg %res, i32 noundef inreg %v_offset, i32 noundef inreg %0, i32 noundef inreg %flag) {
+; GFX9-LABEL: uniform_br_no_metadata:
+; GFX9:       ; %bb.0: ; %entry
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    s_cmp_lt_i32 s23, 1
+; GFX9-NEXT:    s_cbranch_scc1 .LBB0_2
+; GFX9-NEXT:  ; %bb.1: ; %if.then
+; GFX9-NEXT:    s_mov_b32 s7, s20
+; GFX9-NEXT:    s_mov_b32 s6, s19
+; GFX9-NEXT:    s_mov_b32 s5, s18
+; GFX9-NEXT:    s_mov_b32 s4, s17
+; GFX9-NEXT:    v_mov_b32_e32 v0, s16
+; GFX9-NEXT:    v_mov_b32_e32 v1, s21
+; GFX9-NEXT:    buffer_store_dword v0, v1, s[4:7], 0 offen
+; GFX9-NEXT:  .LBB0_2: ; %if.end
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: uniform_br_no_metadata:
+; GFX10:       ; %bb.0: ; %entry
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_cmp_lt_i32 s23, 1
+; GFX10-NEXT:    s_cbranch_scc1 .LBB0_2
+; GFX10-NEXT:  ; %bb.1: ; %if.then
+; GFX10-NEXT:    v_mov_b32_e32 v0, s16
+; GFX10-NEXT:    v_mov_b32_e32 v1, s21
+; GFX10-NEXT:    s_mov_b32 s7, s20
+; GFX10-NEXT:    s_mov_b32 s6, s19
+; GFX10-NEXT:    s_mov_b32 s5, s18
+; GFX10-NEXT:    s_mov_b32 s4, s17
+; GFX10-NEXT:    buffer_store_dword v0, v1, s[4:7], 0 offen
+; GFX10-NEXT:  .LBB0_2: ; %if.end
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+entry:
+  %cmp = icmp sgt i32 %flag, 0
+  br i1 %cmp, label %if.then, label %if.end
+
+if.then:
+  tail call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 %value, ptr addrspace(8) %res, i32 %v_offset, i32 0, i32 0)
+  br label %if.end
+
+if.end:
+  call void @llvm.amdgcn.s.waitcnt(i32 0)
+  ret void
+}
+
+define void @uniform_br_same_weight(i32 noundef inreg %value, ptr addrspace(8) nocapture writeonly inreg %res, i32 noundef inreg %v_offset, i32 noundef inreg %0, i32 noundef inreg %flag) {
+; GFX9-LABEL: uniform_br_same_weight:
+; GFX9:       ; %bb.0: ; %entry
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    s_cmp_lt_i32 s23, 1
+; GFX9-NEXT:    s_cbranch_scc1 .LBB1_2
+; GFX9-NEXT:  ; %bb.1: ; %if.then
+; GFX9-NEXT:    s_mov_b32 s7, s20
+; GFX9-NEXT:    s_mov_b32 s6, s19
+; GFX9-NEXT:    s_mov_b32 s5, s18
+; GFX9-NEXT:    s_mov_b32 s4, s17
+; GFX9-NEXT:    v_mov_b32_e32 v0, s16
+; GFX9-NEXT:    v_mov_b32_e32 v1, s21
+; GFX9-NEXT:    buffer_store_dword v0, v1, s[4:7], 0 offen
+; GFX9-NEXT:  .LBB1_2: ; %if.end
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: uniform_br_same_weight:
+; GFX10:       ; %bb.0: ; %entry
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_cmp_lt_i32 s23, 1
+; GFX10-NEXT:    s_cbranch_scc1 .LBB1_2
+; GFX10-NEXT:  ; %bb.1: ; %if.then
+; GFX10-NEXT:    v_mov_b32_e32 v0, s16
+; GFX10-NEXT:    v_mov_b32_e32 v1, s21
+; GFX10-NEXT:    s_mov_b32 s7, s20
+; GFX10-NEXT:    s_mov_b32 s6, s19
+; GFX10-NEXT:    s_mov_b32 s5, s18
+; GFX10-NEXT:    s_mov_b32 s4, s17
+; GFX10-NEXT:    buffer_store_dword v0, v1, s[4:7], 0 offen
+; GFX10-NEXT:  .LBB1_2: ; %if.end
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+entry:
+  %cmp = icmp sgt i32 %flag, 0
+  br i1 %cmp, label %if.then, label %if.end, !prof !0
+
+if.then:
+  tail call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 %value, ptr addrspace(8) %res, i32 %v_offset, i32 0, i32 0)
+  br label %if.end
+
+if.end:
+  call void @llvm.amdgcn.s.waitcnt(i32 0)
+  ret void
+}
+
+define void @uniform_br_then_likely(i32 noundef inreg %value, ptr addrspace(8) nocapture writeonly inreg %res, i32 noundef inreg %v_offset, i32 noundef inreg %0, i32 noundef inreg %flag) {
+; GFX9-LABEL: uniform_br_then_likely:
+; GFX9:       ; %bb.0: ; %entry
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    s_cmp_lt_i32 s23, 1
+; GFX9-NEXT:    s_cbranch_scc1 .LBB2_2
+; GFX9-NEXT:  ; %bb.1: ; %if.then
+; GFX9-NEXT:    s_mov_b32 s7, s20
+; GFX9-NEXT:    s_mov_b32 s6, s19
+; GFX9-NEXT:    s_mov_b32 s5, s18
+; GFX9-NEXT:    s_mov_b32 s4, s17
+; GFX9-NEXT:    v_mov_b32_e32 v0, s16
+; GFX9-NEXT:    v_mov_b32_e32 v1, s21
+; GFX9-NEXT:    buffer_store_dword v0, v1, s[4:7], 0 offen
+; GFX9-NEXT:  .LBB2_2: ; %if.end
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: uniform_br_then_likely:
+; GFX10:       ; %bb.0: ; %entry
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_cmp_lt_i32 s23, 1
+; GFX10-NEXT:    s_cbranch_scc1 .LBB2_2
+; GFX10-NEXT:  ; %bb.1: ; %if.then
+; GFX10-NEXT:    v_mov_b32_e32 v0, s16
+; GFX10-NEXT:    v_mov_b32_e32 v1, s21
+; GFX10-NEXT:    s_mov_b32 s7, s20
+; GFX10-NEXT:    s_mov_b32 s6, s19
+; GFX10-NEXT:    s_mov_b32 s5, s18
+; GFX10-NEXT:    s_mov_b32 s4, s17
+; GFX10-NEXT:    buffer_store_dword v0, v1, s[4:7], 0 offen
+; GFX10-NEXT:  .LBB2_2: ; %if.end
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+entry:
+  %cmp = icmp sgt i32 %flag, 0
+  br i1 %cmp, label %if.then, label %if.end, !prof !1
+
+if.then:
+  tail call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 %value, ptr addrspace(8) %res, i32 %v_offset, i32 0, i32 0)
+  br label %if.end
+
+if.end:
+  call void @llvm.amdgcn.s.waitcnt(i32 0)
+  ret void
+}
+
+define void @divergent_br_no_metadata(i32 noundef inreg %value, ptr addrspace(8) nocapture writeonly inreg %res, i32 noundef inreg %v_offset, i32 noundef inreg %0, i32 noundef %flag) {
+; GFX9-LABEL: divergent_br_no_metadata:
+; GFX9:       ; %bb.0: ; %entry
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_cmp_lt_i32_e32 vcc, 0, v0
+; GFX9-NEXT:    s_and_saveexec_b64 s[8:9], vcc
+; GFX9-NEXT:    s_cbranch_execz .LBB3_2
+; GFX9-NEXT:  ; %bb.1: ; %if.then
+; GFX9-NEXT:    s_mov_b32 s7, s20
+; GFX9-NEXT:    s_mov_b32 s6, s19
+; GFX9-NEXT:    s_mov_b32 s5, s18
+; GFX9-NEXT:    s_mov_b32 s4, s17
+; GFX9-NEXT:    v_mov_b32_e32 v0, s16
+; GFX9-NEXT:    v_mov_b32_e32 v1, s21
+; GFX9-NEXT:    buffer_store_dword v0, v1, s[4:7], 0 offen
+; GFX9-NEXT:  .LBB3_2: ; %if.end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[8:9]
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX1010-LABEL: divergent_br_no_metadata:
+; GFX1010:       ; %bb.0: ; %entry
+; GFX1010-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1010-NEXT:    v_cmp_lt_i32_e32 vcc_lo, 0, v0
+; GFX1010-NEXT:    s_and_saveexec_b32 s8, vcc_lo
+; GFX1010-NEXT:    s_cbranch_execz .LBB3_2
+; GFX1010-NEXT:  ; %bb.1: ; %if.then
+; GFX1010-NEXT:    v_mov_b32_e32 v0, s16
+; GFX1010-NEXT:    v_mov_b32_e32 v1, s21
+; GFX1010-NEXT:    s_mov_b32 s7, s20
+; GFX1010-NEXT:    s_mov_b32 s6, s19
+; GFX1010-NEXT:    s_mov_b32 s5, s18
+; GFX1010-NEXT:    s_mov_b32 s4, s17
+; GFX1010-NEXT:    buffer_store_dword v0, v1, s[4:7], 0 offen
+; GFX1010-NEXT:  .LBB3_2: ; %if.end
+; GFX1010-NEXT:    s_waitcnt_depctr 0xffe3
+; GFX1010-NEXT:    s_or_b32 exec_lo, exec_lo, s8
+; GFX1010-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1010-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX1030-LABEL: divergent_br_no_metadata:
+; GFX1030:       ; %bb.0: ; %entry
+; GFX1030-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1030-NEXT:    s_mov_b32 s8, exec_lo
+; GFX1030-NEXT:    v_cmpx_lt_i32_e32 0, v0
+; GFX1030-NEXT:    s_cbranch_execz .LBB3_2
+; GFX1030-NEXT:  ; %bb.1: ; %if.then
+; GFX1030-NEXT:    v_mov_b32_e32 v0, s16
+; GFX1030-NEXT:    v_mov_b32_e32 v1, s21
+; GFX1030-NEXT:    s_mov_b32 s7, s20
+; GFX1030-NEXT:    s_mov_b32 s6, s19
+; GFX1030-NEXT:    s_mov_b32 s5, s18
+; GFX1030-NEXT:    s_mov_b32 s4, s17
+; GFX1030-NEXT:    buffer_store_dword v0, v1, s[4:7], 0 offen
+; GFX1030-NEXT:  .LBB3_2: ; %if.end
+; GFX1030-NEXT:    s_or_b32 exec_lo, exec_lo, s8
+; GFX1030-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1030-NEXT:    s_setpc_b64 s[30:31]
+entry:
+  %cmp = icmp sgt i32 %flag, 0
+  br i1 %cmp, label %if.then, label %if.end
+
+if.then:
+  tail call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 %value, ptr addrspace(8) %res, i32 %v_offset, i32 0, i32 0)
+  br label %if.end
+
+if.end:
+  call void @llvm.amdgcn.s.waitcnt(i32 0)
+  ret void
+}
+
+define void @divergent_br_same_weight(i32 noundef inreg %value, ptr addrspace(8) nocapture writeonly inreg %res, i32 noundef inreg %v_offset, i32 noundef inreg %0, i32 noundef %flag) {
+; GFX9-LABEL: divergent_br_same_weight:
+; GFX9:       ; %bb.0: ; %entry
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_cmp_lt_i32_e32 vcc, 0, v0
+; GFX9-NEXT:    s_and_saveexec_b64 s[8:9], vcc
+; GFX9-NEXT:    s_cbranch_execz .LBB4_2
+; GFX9-NEXT:  ; %bb.1: ; %if.then
+; GFX9-NEXT:    s_mov_b32 s7, s20
+; GFX9-NEXT:    s_mov_b32 s6, s19
+; GFX9-NEXT:    s_mov_b32 s5, s18
+; GFX9-NEXT:    s_mov_b32 s4, s17
+; GFX9-NEXT:    v_mov_b32_e32 v0, s16
+; GFX9-NEXT:    v_mov_b32_e32 v1, s21
+; GFX9-NEXT:    buffer_store_dword v0, v1, s[4:7], 0 offen
+; GFX9-NEXT:  .LBB4_2: ; %if.end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[8:9]
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX1010-LABEL: divergent_br_same_weight:
+; GFX1010:       ; %bb.0: ; %entry
+; GFX1010-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1010-NEXT:    v_cmp_lt_i32_e32 vcc_lo, 0, v0
+; GFX1010-NEXT:    s_and_saveexec_b32 s8, vcc_lo
+; GFX1010-NEXT:    s_cbranch_execz .LBB4_2
+; GFX1010-NEXT:  ; %bb.1: ; %if.then
+; GFX1010-NEXT:    v_mov_b32_e32 v0, s16
+; GFX1010-NEXT:    v_mov_b32_e32 v1, s21
+; GFX1010-NEXT:    s_mov_b32 s7, s20
+; GFX1010-NEXT:    s_mov_b32 s6, s19
+; GFX1010-NEXT:    s_mov_b32 s5, s18
+; GFX1010-NEXT:    s_mov_b32 s4, s17
+; GFX1010-NEXT:    buffer_store_dword v0, v1, s[4:7], 0 offen
+; GFX1010-NEXT:  .LBB4_2: ; %if.end
+; GFX1010-NEXT:    s_waitcnt_depctr 0xffe3
+; GFX1010-NEXT:    s_or_b32 exec_lo, exec_lo, s8
+; GFX1010-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1010-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX1030-LABEL: divergent_br_same_weight:
+; GFX1030:       ; %bb.0: ; %entry
+; GFX1030-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1030-NEXT:    s_mov_b32 s8, exec_lo
+; GFX1030-NEXT:    v_cmpx_lt_i32_e32 0, v0
+; GFX1030-NEXT:    s_cbranch_execz .LBB4_2
+; GFX1030-NEXT:  ; %bb.1: ; %if.then
+; GFX1030-NEXT:    v_mov_b32_e32 v0, s16
+; GFX1030-NEXT:    v_mov_b32_e32 v1, s21
+; GFX1030-NEXT:    s_mov_b32 s7, s20
+; GFX1030-NEXT:    s_mov_b32 s6, s19
+; GFX1030-NEXT:    s_mov_b32 s5, s18
+; GFX1030-NEXT:    s_mov_b32 s4, s17
+; GFX1030-NEXT:    buffer_store_dword v0, v1, s[4:7], 0 offen
+; GFX1030-NEXT:  .LBB4_2: ; %if.end
+; GFX1030-NEXT:    s_or_b32 exec_lo, exec_lo, s8
+; GFX1030-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1030-NEXT:    s_setpc_b64 s[30:31]
+entry:
+  %cmp = icmp sgt i32 %flag, 0
+  br i1 %cmp, label %if.then, label %if.end, !prof !0
+
+if.then:
+  tail call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 %value, ptr addrspace(8) %res, i32 %v_offset, i32 0, i32 0)
+  br label %if.end
+
+if.end:
+  call void @llvm.amdgcn.s.waitcnt(i32 0)
+  ret void
+}
+
+define void @divergent_br_then_likely(i32 noundef inreg %value, ptr addrspace(8) nocapture writeonly inreg %res, i32 noundef inreg %v_offset, i32 noundef inreg %0, i32 noundef %flag) {
+; GFX9-LABEL: divergent_br_then_likely:
+; GFX9:       ; %bb.0: ; %entry
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_cmp_lt_i32_e32 vcc, 0, v0
+; GFX9-NEXT:    s_and_saveexec_b64 s[8:9], vcc
+; GFX9-NEXT:  ; %bb.1: ; %if.then
+; GFX9-NEXT:    s_mov_b32 s7, s20
+; GFX9-NEXT:    s_mov_b32 s6, s19
+; GFX9-NEXT:    s_mov_b32 s5, s18
+; GFX9-NEXT:    s_mov_b32 s4, s17
+; GFX9-NEXT:    v_mov_b32_e32 v0, s16
+; GFX9-NEXT:    v_mov_b32_e32 v1, s21
+; GFX9-NEXT:    buffer_store_dword v0, v1, s[4:7], 0 offen
+; GFX9-NEXT:  ; %bb.2: ; %if.end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[8:9]
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX1010-LABEL: divergent_br_then_likely:
+; GFX1010:       ; %bb.0: ; %entry
+; GFX1010-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1010-NEXT:    v_cmp_lt_i32_e32 vcc_lo, 0, v0
+; GFX1010-NEXT:    s_and_saveexec_b32 s8, vcc_lo
+; GFX1010-NEXT:  ; %bb.1: ; %if.then
+; GFX1010-NEXT:    v_mov_b32_e32 v0, s16
+; GFX1010-NEXT:    v_mov_b32_e32 v1, s21
+; GFX1010-NEXT:    s_mov_b32 s7, s20
+; GFX1010-NEXT:    s_mov_b32 s6, s19
+; GFX1010-NEXT:    s_mov_b32 s5, s18
+; GFX1010-NEXT:    s_mov_b32 s4, s17
+; GFX1010-NEXT:    buffer_store_dword v0, v1, s[4:7], 0 offen
+; GFX1010-NEXT:  ; %bb.2: ; %if.end
+; GFX1010-NEXT:    s_waitcnt_depctr 0xffe3
+; GFX1010-NEXT:    s_or_b32 exec_lo, exec_lo, s8
+; GFX1010-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1010-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX1030-LABEL: divergent_br_then_likely:
+; GFX1030:       ; %bb.0: ; %entry
+; GFX1030-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1030-NEXT:    s_mov_b32 s8, exec_lo
+; GFX1030-NEXT:    v_cmpx_lt_i32_e32 0, v0
+; GFX1030-NEXT:  ; %bb.1: ; %if.then
+; GFX1030-NEXT:    v_mov_b32_e32 v0, s16
+; GFX1030-NEXT:    v_mov_b32_e32 v1, s21
+; GFX1030-NEXT:    s_mov_b32 s7, s20
+; GFX1030-NEXT:    s_mov_b32 s6, s19
+; GFX1030-NEXT:    s_mov_b32 s5, s18
+; GFX1030-NEXT:    s_mov_b32 s4, s17
+; GFX1030-NEXT:    buffer_store_dword v0, v1, s[4:7], 0 offen
+; GFX1030-NEXT:  ; %bb.2: ; %if.end
+; GFX1030-NEXT:    s_or_b32 exec_lo, exec_lo, s8
+; GFX1030-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1030-NEXT:    s_setpc_b64 s[30:31]
+entry:
+  %cmp = icmp sgt i32 %flag, 0
+  br i1 %cmp, label %if.then, label %if.end, !prof !1
+
+if.then:
+  tail call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 %value, ptr addrspace(8) %res, i32 %v_offset, i32 0, i32 0)
+  br label %if.end
+
+if.end:
+  call void @llvm.amdgcn.s.waitcnt(i32 0)
+  ret void
+}
+
+declare void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32, ptr addrspace(8) nocapture writeonly, i32, i32, i32 immarg)
+declare void @llvm.amdgcn.s.waitcnt(i32)
+
+!0 = !{!"branch_weights", i32 1000, i32 1000}
+!1 = !{!"branch_weights", i32 2000, i32 1}
diff --git a/llvm/test/CodeGen/AMDGPU/amdgpu-codegenprepare-idiv.ll b/llvm/test/CodeGen/AMDGPU/amdgpu-codegenprepare-idiv.ll
index 4f04c15b3d44a..8e16889c72e65 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgpu-codegenprepare-idiv.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgpu-codegenprepare-idiv.ll
@@ -10041,11 +10041,9 @@ define i64 @udiv_i64_gt_smax(i8 %size) {
 ; GFX9-NEXT:    v_mov_b32_e32 v1, 0
 ; GFX9-NEXT:    s_mov_b32 s6, 0xcccccccc
 ; GFX9-NEXT:    v_mad_u64_u32 v[2:3], s[4:5], v5, s4, v[0:1]
-; GFX9-NEXT:    v_mov_b32_e32 v6, v3
-; GFX9-NEXT:    v_mov_b32_e32 v3, v1
-; GFX9-NEXT:    v_mad_u64_u32 v[0:1], s[4:5], v4, s6, v[2:3]
-; GFX9-NEXT:    v_mov_b32_e32 v0, v1
-; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, v6, v0
+; GFX9-NEXT:    v_mov_b32_e32 v0, v2
+; GFX9-NEXT:    v_mad_u64_u32 v[0:1], s[4:5], v4, s6, v[0:1]
+; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, v3, v1
 ; GFX9-NEXT:    v_addc_co_u32_e64 v1, s[4:5], 0, 0, vcc
 ; GFX9-NEXT:    v_mad_u64_u32 v[0:1], s[4:5], v5, s6, v[0:1]
 ; GFX9-NEXT:    v_alignbit_b32 v0, v1, v0, 3
diff --git a/llvm/test/CodeGen/AMDGPU/amdgpu-demote-scc-branches.ll b/llvm/test/CodeGen/AMDGPU/amdgpu-demote-scc-branches.ll
deleted file mode 100644
index ba5f9b7aa0d06..0000000000000
--- a/llvm/test/CodeGen/AMDGPU/amdgpu-demote-scc-branches.ll
+++ /dev/null
@@ -1,361 +0,0 @@
-; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
-; RUN: llc -mtriple=amdgcn -mcpu=gfx90a < %s | FileCheck -check-prefixes=GFX9 %s
-; RUN: llc -mtriple=amdgcn -mcpu=gfx1010 < %s | FileCheck -check-prefixes=GFX10,GFX1010 %s
-; RUN: llc -mtriple=amdgcn -mcpu=gfx1030 < %s | FileCheck -check-prefixes=GFX10,GFX1030 %s
-
-define void @uniform_br_no_metadata(i32 noundef inreg %value, ptr addrspace(8) nocapture writeonly inreg %res, i32 noundef inreg %v_offset, i32 noundef inreg %0, i32 noundef inreg %flag) {
-; GFX9-LABEL: uniform_br_no_metadata:
-; GFX9:       ; %bb.0: ; %entry
-; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    s_cmp_lt_i32 s23, 1
-; GFX9-NEXT:    s_cbranch_scc1 .LBB0_2
-; GFX9-NEXT:  ; %bb.1: ; %if.then
-; GFX9-NEXT:    s_mov_b32 s7, s20
-; GFX9-NEXT:    s_mov_b32 s6, s19
-; GFX9-NEXT:    s_mov_b32 s5, s18
-; GFX9-NEXT:    s_mov_b32 s4, s17
-; GFX9-NEXT:    v_mov_b32_e32 v0, s16
-; GFX9-NEXT:    v_mov_b32_e32 v1, s21
-; GFX9-NEXT:    buffer_store_dword v0, v1, s[4:7], 0 offen
-; GFX9-NEXT:  .LBB0_2: ; %if.end
-; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    s_setpc_b64 s[30:31]
-;
-; GFX10-LABEL: uniform_br_no_metadata:
-; GFX10:       ; %bb.0: ; %entry
-; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT:    s_cmp_lt_i32 s23, 1
-; GFX10-NEXT:    s_cbranch_scc1 .LBB0_2
-; GFX10-NEXT:  ; %bb.1: ; %if.then
-; GFX10-NEXT:    v_mov_b32_e32 v0, s16
-; GFX10-NEXT:    v_mov_b32_e32 v1, s21
-; GFX10-NEXT:    s_mov_b32 s7, s20
-; GFX10-NEXT:    s_mov_b32 s6, s19
-; GFX10-NEXT:    s_mov_b32 s5, s18
-; GFX10-NEXT:    s_mov_b32 s4, s17
-; GFX10-NEXT:    buffer_store_dword v0, v1, s[4:7], 0 offen
-; GFX10-NEXT:  .LBB0_2: ; %if.end
-; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT:    s_setpc_b64 s[30:31]
-entry:
-  %cmp = icmp sgt i32 %flag, 0
-  br i1 %cmp, label %if.then, label %if.end
-
-if.then:
-  tail call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 %value, ptr addrspace(8) %res, i32 %v_offset, i32 0, i32 0)
-  br label %if.end
-
-if.end:
-  call void @llvm.amdgcn.s.waitcnt(i32 0)
-  ret void
-}
-
-define void @uniform_br_unprofitable(i32 noundef inreg %value, ptr addrspace(8) nocapture writeonly inreg %res, i32 noundef inreg %v_offset, i32 noundef inreg %0, i32 noundef inreg %flag) {
-; GFX9-LABEL: uniform_br_unprofitable:
-; GFX9:       ; %bb.0: ; %entry
-; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    s_cmp_lt_i32 s23, 1
-; GFX9-NEXT:    s_cbranch_scc1 .LBB1_2
-; GFX9-NEXT:  ; %bb.1: ; %if.then
-; GFX9-NEXT:    s_mov_b32 s7, s20
-; GFX9-NEXT:    s_mov_b32 s6, s19
-; GFX9-NEXT:    s_mov_b32 s5, s18
-; GFX9-NEXT:    s_mov_b32 s4, s17
-; GFX9-NEXT:    v_mov_b32_e32 v0, s16
-; GFX9-NEXT:    v_mov_b32_e32 v1, s21
-; GFX9-NEXT:    buffer_store_dword v0, v1, s[4:7], 0 offen
-; GFX9-NEXT:  .LBB1_2: ; %if.end
-; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    s_setpc_b64 s[30:31]
-;
-; GFX10-LABEL: uniform_br_unprofitable:
-; GFX10:       ; %bb.0: ; %entry
-; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT:    s_cmp_lt_i32 s23, 1
-; GFX10-NEXT:    s_cbranch_scc1 .LBB1_2
-; GFX10-NEXT:  ; %bb.1: ; %if.then
-; GFX10-NEXT:    v_mov_b32_e32 v0, s16
-; GFX10-NEXT:    v_mov_b32_e32 v1, s21
-; GFX10-NEXT:    s_mov_b32 s7, s20
-; GFX10-NEXT:    s_mov_b32 s6, s19
-; GFX10-NEXT:    s_mov_b32 s5, s18
-; GFX10-NEXT:    s_mov_b32 s4, s17
-; GFX10-NEXT:    buffer_store_dword v0, v1, s[4:7], 0 offen
-; GFX10-NEXT:  .LBB1_2: ; %if.end
-; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT:    s_setpc_b64 s[30:31]
-entry:
-  %cmp = icmp sgt i32 %flag, 0
-  br i1 %cmp, label %if.then, label %if.end, !prof !0
-
-if.then:
-  tail call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 %value, ptr addrspace(8) %res, i32 %v_offset, i32 0, i32 0)
-  br label %if.end
-
-if.end:
-  call void @llvm.amdgcn.s.waitcnt(i32 0)
-  ret void
-}
-
-define void @uniform_br_profitable(i32 noundef inreg %value, ptr addrspace(8) nocapture writeonly inreg %res, i32 noundef inreg %v_offset, i32 noundef inreg %0, i32 noundef inreg %flag) {
-; GFX9-LABEL: uniform_br_profitable:
-; GFX9:       ; %bb.0: ; %entry
-; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    s_cmp_lt_i32 s23, 1
-; GFX9-NEXT:    s_cbranch_scc1 .LBB2_2
-; GFX9-NEXT:  ; %bb.1: ; %if.then
-; GFX9-NEXT:    s_mov_b32 s7, s20
-; GFX9-NEXT:    s_mov_b32 s6, s19
-; GFX9-NEXT:    s_mov_b32 s5, s18
-; GFX9-NEXT:    s_mov_b32 s4, s17
-; GFX9-NEXT:    v_mov_b32_e32 v0, s16
-; GFX9-NEXT:    v_mov_b32_e32 v1, s21
-; GFX9-NEXT:    buffer_store_dword v0, v1, s[4:7], 0 offen
-; GFX9-NEXT:  .LBB2_2: ; %if.end
-; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    s_setpc_b64 s[30:31]
-;
-; GFX10-LABEL: uniform_br_profitable:
-; GFX10:       ; %bb.0: ; %entry
-; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT:    s_cmp_lt_i32 s23, 1
-; GFX10-NEXT:    s_cbranch_scc1 .LBB2_2
-; GFX10-NEXT:  ; %bb.1: ; %if.then
-; GFX10-NEXT:    v_mov_b32_e32 v0, s16
-; GFX10-NEXT:    v_mov_b32_e32 v1, s21
-; GFX10-NEXT:    s_mov_b32 s7, s20
-; GFX10-NEXT:    s_mov_b32 s6, s19
-; GFX10-NEXT:    s_mov_b32 s5, s18
-; GFX10-NEXT:    s_mov_b32 s4, s17
-; GFX10-NEXT:    buffer_store_dword v0, v1, s[4:7], 0 offen
-; GFX10-NEXT:  .LBB2_2: ; %if.end
-; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT:    s_setpc_b64 s[30:31]
-entry:
-  %cmp = icmp sgt i32 %flag, 0
-  br i1 %cmp, label %if.then, label %if.end, !prof !1
-
-if.then:
-  tail call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 %value, ptr addrspace(8) %res, i32 %v_offset, i32 0, i32 0)
-  br label %if.end
-
-if.end:
-  call void @llvm.amdgcn.s.waitcnt(i32 0)
-  ret void
-}
-
-define void @divergent_br_no_metadata(i32 noundef inreg %value, ptr addrspace(8) nocapture writeonly inreg %res, i32 noundef inreg %v_offset, i32 noundef inreg %0, i32 noundef %flag) {
-; GFX9-LABEL: divergent_br_no_metadata:
-; GFX9:       ; %bb.0: ; %entry
-; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_cmp_lt_i32_e32 vcc, 0, v0
-; GFX9-NEXT:    s_and_saveexec_b64 s[8:9], vcc
-; GFX9-NEXT:    s_cbranch_execz .LBB3_2
-; GFX9-NEXT:  ; %bb.1: ; %if.then
-; GFX9-NEXT:    s_mov_b32 s7, s20
-; GFX9-NEXT:    s_mov_b32 s6, s19
-; GFX9-NEXT:    s_mov_b32 s5, s18
-; GFX9-NEXT:    s_mov_b32 s4, s17
-; GFX9-NEXT:    v_mov_b32_e32 v0, s16
-; GFX9-NEXT:    v_mov_b32_e32 v1, s21
-; GFX9-NEXT:    buffer_store_dword v0, v1, s[4:7], 0 offen
-; GFX9-NEXT:  .LBB3_2: ; %if.end
-; GFX9-NEXT:    s_or_b64 exec, exec, s[8:9]
-; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    s_setpc_b64 s[30:31]
-;
-; GFX1010-LABEL: divergent_br_no_metadata:
-; GFX1010:       ; %bb.0: ; %entry
-; GFX1010-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX1010-NEXT:    v_cmp_lt_i32_e32 vcc_lo, 0, v0
-; GFX1010-NEXT:    s_and_saveexec_b32 s8, vcc_lo
-; GFX1010-NEXT:    s_cbranch_execz .LBB3_2
-; GFX1010-NEXT:  ; %bb.1: ; %if.then
-; GFX1010-NEXT:    v_mov_b32_e32 v0, s16
-; GFX1010-NEXT:    v_mov_b32_e32 v1, s21
-; GFX1010-NEXT:    s_mov_b32 s7, s20
-; GFX1010-NEXT:    s_mov_b32 s6, s19
-; GFX1010-NEXT:    s_mov_b32 s5, s18
-; GFX1010-NEXT:    s_mov_b32 s4, s17
-; GFX1010-NEXT:    buffer_store_dword v0, v1, s[4:7], 0 offen
-; GFX1010-NEXT:  .LBB3_2: ; %if.end
-; GFX1010-NEXT:    s_waitcnt_depctr 0xffe3
-; GFX1010-NEXT:    s_or_b32 exec_lo, exec_lo, s8
-; GFX1010-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX1010-NEXT:    s_setpc_b64 s[30:31]
-;
-; GFX1030-LABEL: divergent_br_no_metadata:
-; GFX1030:       ; %bb.0: ; %entry
-; GFX1030-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX1030-NEXT:    s_mov_b32 s8, exec_lo
-; GFX1030-NEXT:    v_cmpx_lt_i32_e32 0, v0
-; GFX1030-NEXT:    s_cbranch_execz .LBB3_2
-; GFX1030-NEXT:  ; %bb.1: ; %if.then
-; GFX1030-NEXT:    v_mov_b32_e32 v0, s16
-; GFX1030-NEXT:    v_mov_b32_e32 v1, s21
-; GFX1030-NEXT:    s_mov_b32 s7, s20
-; GFX1030-NEXT:    s_mov_b32 s6, s19
-; GFX1030-NEXT:    s_mov_b32 s5, s18
-; GFX1030-NEXT:    s_mov_b32 s4, s17
-; GFX1030-NEXT:    buffer_store_dword v0, v1, s[4:7], 0 offen
-; GFX1030-NEXT:  .LBB3_2: ; %if.end
-; GFX1030-NEXT:    s_or_b32 exec_lo, exec_lo, s8
-; GFX1030-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX1030-NEXT:    s_setpc_b64 s[30:31]
-entry:
-  %cmp = icmp sgt i32 %flag, 0
-  br i1 %cmp, label %if.then, label %if.end
-
-if.then:
-  tail call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 %value, ptr addrspace(8) %res, i32 %v_offset, i32 0, i32 0)
-  br label %if.end
-
-if.end:
-  call void @llvm.amdgcn.s.waitcnt(i32 0)
-  ret void
-}
-
-define void @divergent_br_unprofitable(i32 noundef inreg %value, ptr addrspace(8) nocapture writeonly inreg %res, i32 noundef inreg %v_offset, i32 noundef inreg %0, i32 noundef %flag) {
-; GFX9-LABEL: divergent_br_unprofitable:
-; GFX9:       ; %bb.0: ; %entry
-; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_cmp_lt_i32_e32 vcc, 0, v0
-; GFX9-NEXT:    s_and_saveexec_b64 s[8:9], vcc
-; GFX9-NEXT:    s_cbranch_execz .LBB4_2
-; GFX9-NEXT:  ; %bb.1: ; %if.then
-; GFX9-NEXT:    s_mov_b32 s7, s20
-; GFX9-NEXT:    s_mov_b32 s6, s19
-; GFX9-NEXT:    s_mov_b32 s5, s18
-; GFX9-NEXT:    s_mov_b32 s4, s17
-; GFX9-NEXT:    v_mov_b32_e32 v0, s16
-; GFX9-NEXT:    v_mov_b32_e32 v1, s21
-; GFX9-NEXT:    buffer_store_dword v0, v1, s[4:7], 0 offen
-; GFX9-NEXT:  .LBB4_2: ; %if.end
-; GFX9-NEXT:    s_or_b64 exec, exec, s[8:9]
-; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    s_setpc_b64 s[30:31]
-;
-; GFX1010-LABEL: divergent_br_unprofitable:
-; GFX1010:       ; %bb.0: ; %entry
-; GFX1010-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX1010-NEXT:    v_cmp_lt_i32_e32 vcc_lo, 0, v0
-; GFX1010-NEXT:    s_and_saveexec_b32 s8, vcc_lo
-; GFX1010-NEXT:    s_cbranch_execz .LBB4_2
-; GFX1010-NEXT:  ; %bb.1: ; %if.then
-; GFX1010-NEXT:    v_mov_b32_e32 v0, s16
-; GFX1010-NEXT:    v_mov_b32_e32 v1, s21
-; GFX1010-NEXT:    s_mov_b32 s7, s20
-; GFX1010-NEXT:    s_mov_b32 s6, s19
-; GFX1010-NEXT:    s_mov_b32 s5, s18
-; GFX1010-NEXT:    s_mov_b32 s4, s17
-; GFX1010-NEXT:    buffer_store_dword v0, v1, s[4:7], 0 offen
-; GFX1010-NEXT:  .LBB4_2: ; %if.end
-; GFX1010-NEXT:    s_waitcnt_depctr 0xffe3
-; GFX1010-NEXT:    s_or_b32 exec_lo, exec_lo, s8
-; GFX1010-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX1010-NEXT:    s_setpc_b64 s[30:31]
-;
-; GFX1030-LABEL: divergent_br_unprofitable:
-; GFX1030:       ; %bb.0: ; %entry
-; GFX1030-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX1030-NEXT:    s_mov_b32 s8, exec_lo
-; GFX1030-NEXT:    v_cmpx_lt_i32_e32 0, v0
-; GFX1030-NEXT:    s_cbranch_execz .LBB4_2
-; GFX1030-NEXT:  ; %bb.1: ; %if.then
-; GFX1030-NEXT:    v_mov_b32_e32 v0, s16
-; GFX1030-NEXT:    v_mov_b32_e32 v1, s21
-; GFX1030-NEXT:    s_mov_b32 s7, s20
-; GFX1030-NEXT:    s_mov_b32 s6, s19
-; GFX1030-NEXT:    s_mov_b32 s5, s18
-; GFX1030-NEXT:    s_mov_b32 s4, s17
-; GFX1030-NEXT:    buffer_store_dword v0, v1, s[4:7], 0 offen
-; GFX1030-NEXT:  .LBB4_2: ; %if.end
-; GFX1030-NEXT:    s_or_b32 exec_lo, exec_lo, s8
-; GFX1030-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX1030-NEXT:    s_setpc_b64 s[30:31]
-entry:
-  %cmp = icmp sgt i32 %flag, 0
-  br i1 %cmp, label %if.then, label %if.end, !prof !0
-
-if.then:
-  tail call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 %value, ptr addrspace(8) %res, i32 %v_offset, i32 0, i32 0)
-  br label %if.end
-
-if.end:
-  call void @llvm.amdgcn.s.waitcnt(i32 0)
-  ret void
-}
-
-define void @divergent_br_profitable(i32 noundef inreg %value, ptr addrspace(8) nocapture writeonly inreg %res, i32 noundef inreg %v_offset, i32 noundef inreg %0, i32 noundef %flag) {
-; GFX9-LABEL: divergent_br_profitable:
-; GFX9:       ; %bb.0: ; %entry
-; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_cmp_lt_i32_e32 vcc, 0, v0
-; GFX9-NEXT:    s_and_saveexec_b64 s[8:9], vcc
-; GFX9-NEXT:  ; %bb.1: ; %if.then
-; GFX9-NEXT:    s_mov_b32 s7, s20
-; GFX9-NEXT:    s_mov_b32 s6, s19
-; GFX9-NEXT:    s_mov_b32 s5, s18
-; GFX9-NEXT:    s_mov_b32 s4, s17
-; GFX9-NEXT:    v_mov_b32_e32 v0, s16
-; GFX9-NEXT:    v_mov_b32_e32 v1, s21
-; GFX9-NEXT:    buffer_store_dword v0, v1, s[4:7], 0 offen
-; GFX9-NEXT:  ; %bb.2: ; %if.end
-; GFX9-NEXT:    s_or_b64 exec, exec, s[8:9]
-; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    s_setpc_b64 s[30:31]
-;
-; GFX1010-LABEL: divergent_br_profitable:
-; GFX1010:       ; %bb.0: ; %entry
-; GFX1010-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX1010-NEXT:    v_cmp_lt_i32_e32 vcc_lo, 0, v0
-; GFX1010-NEXT:    s_and_saveexec_b32 s8, vcc_lo
-; GFX1010-NEXT:  ; %bb.1: ; %if.then
-; GFX1010-NEXT:    v_mov_b32_e32 v0, s16
-; GFX1010-NEXT:    v_mov_b32_e32 v1, s21
-; GFX1010-NEXT:    s_mov_b32 s7, s20
-; GFX1010-NEXT:    s_mov_b32 s6, s19
-; GFX1010-NEXT:    s_mov_b32 s5, s18
-; GFX1010-NEXT:    s_mov_b32 s4, s17
-; GFX1010-NEXT:    buffer_store_dword v0, v1, s[4:7], 0 offen
-; GFX1010-NEXT:  ; %bb.2: ; %if.end
-; GFX1010-NEXT:    s_waitcnt_depctr 0xffe3
-; GFX1010-NEXT:    s_or_b32 exec_lo, exec_lo, s8
-; GFX1010-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX1010-NEXT:    s_setpc_b64 s[30:31]
-;
-; GFX1030-LABEL: divergent_br_profitable:
-; GFX1030:       ; %bb.0: ; %entry
-; GFX1030-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX1030-NEXT:    s_mov_b32 s8, exec_lo
-; GFX1030-NEXT:    v_cmpx_lt_i32_e32 0, v0
-; GFX1030-NEXT:  ; %bb.1: ; %if.then
-; GFX1030-NEXT:    v_mov_b32_e32 v0, s16
-; GFX1030-NEXT:    v_mov_b32_e32 v1, s21
-; GFX1030-NEXT:    s_mov_b32 s7, s20
-; GFX1030-NEXT:    s_mov_b32 s6, s19
-; GFX1030-NEXT:    s_mov_b32 s5, s18
-; GFX1030-NEXT:    s_mov_b32 s4, s17
-; GFX1030-NEXT:    buffer_store_dword v0, v1, s[4:7], 0 offen
-; GFX1030-NEXT:  ; %bb.2: ; %if.end
-; GFX1030-NEXT:    s_or_b32 exec_lo, exec_lo, s8
-; GFX1030-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX1030-NEXT:    s_setpc_b64 s[30:31]
-entry:
-  %cmp = icmp sgt i32 %flag, 0
-  br i1 %cmp, label %if.then, label %if.end, !prof !1
-
-if.then:
-  tail call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 %value, ptr addrspace(8) %res, i32 %v_offset, i32 0, i32 0)
-  br label %if.end
-
-if.end:
-  call void @llvm.amdgcn.s.waitcnt(i32 0)
-  ret void
-}
-
-declare void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32, ptr addrspace(8) nocapture writeonly, i32, i32, i32 immarg)
-declare void @llvm.amdgcn.s.waitcnt(i32)
-
-!0 = !{!"branch_weights", i32 1000, i32 1000}
-!1 = !{!"branch_weights", i32 2000, i32 1}
diff --git a/llvm/test/CodeGen/AMDGPU/amdgpu-libcall-sincos-pass-ordering.ll b/llvm/test/CodeGen/AMDGPU/amdgpu-libcall-sincos-pass-ordering.ll
index 317a069eed26e..f8959f1751025 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgpu-libcall-sincos-pass-ordering.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgpu-libcall-sincos-pass-ordering.ll
@@ -10,10 +10,10 @@
 ; Should have call to sincos declarations, not calls to the asm pseudo-libcalls
 define protected amdgpu_kernel void @swdev456865(ptr addrspace(1) %out0, ptr addrspace(1) %out1, ptr addrspace(1) %out2, float noundef %x) #0 {
 ; CHECK-LABEL: define protected amdgpu_kernel void @swdev456865(
-; CHECK-SAME: ptr addrspace(1) nocapture writeonly initializes((0, 8)) [[OUT0:%.*]], ptr addrspace(1) nocapture writeonly initializes((0, 8)) [[OUT1:%.*]], ptr addrspace(1) nocapture writeonly initializes((0, 8)) [[OUT2:%.*]], float noundef [[X:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
+; CHECK-SAME: ptr addrspace(1) writeonly captures(none) initializes((0, 8)) [[OUT0:%.*]], ptr addrspace(1) writeonly captures(none) initializes((0, 8)) [[OUT1:%.*]], ptr addrspace(1) writeonly captures(none) initializes((0, 8)) [[OUT2:%.*]], float noundef [[X:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[__SINCOS_:%.*]] = alloca float, align 4, addrspace(5)
-; CHECK-NEXT:    [[I_I:%.*]] = call float @_Z6sincosfPU3AS5f(float [[X]], ptr addrspace(5) [[__SINCOS_]]) #[[ATTR1:[0-9]+]]
+; CHECK-NEXT:    [[I_I:%.*]] = call float @_Z6sincosfPU3AS5f(float [[X]], ptr addrspace(5) [[__SINCOS_]]) #[[ATTR2:[0-9]+]]
 ; CHECK-NEXT:    [[I_I2:%.*]] = load float, ptr addrspace(5) [[__SINCOS_]], align 4
 ; CHECK-NEXT:    [[ADD:%.*]] = fadd float [[I_I]], [[I_I2]]
 ; CHECK-NEXT:    [[CONV:%.*]] = fpext float [[X]] to double
diff --git a/llvm/test/CodeGen/AMDGPU/amdgpu-simplify-libcall-sincos.defined.ll b/llvm/test/CodeGen/AMDGPU/amdgpu-simplify-libcall-sincos.defined.ll
index a7a68d6780605..ad7e913e6d22c 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgpu-simplify-libcall-sincos.defined.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgpu-simplify-libcall-sincos.defined.ll
@@ -103,7 +103,7 @@ define <2 x float> @_Z6sincosDv2_fPU3AS0S_(<2 x float> %x, ptr %ptr) {
 
 define void @sincos_f32(float %x, ptr addrspace(1) nocapture writeonly %sin_out, ptr addrspace(1) nocapture writeonly %cos_out) {
 ; CHECK-LABEL: define void @sincos_f32
-; CHECK-SAME: (float [[X:%.*]], ptr addrspace(1) nocapture writeonly [[SIN_OUT:%.*]], ptr addrspace(1) nocapture writeonly [[COS_OUT:%.*]]) {
+; CHECK-SAME: (float [[X:%.*]], ptr addrspace(1) writeonly captures(none) [[SIN_OUT:%.*]], ptr addrspace(1) writeonly captures(none) [[COS_OUT:%.*]]) {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[__SINCOS_:%.*]] = alloca float, align 4, addrspace(5)
 ; CHECK-NEXT:    [[TMP0:%.*]] = call contract float @_Z6sincosfPU3AS5f(float [[X]], ptr addrspace(5) [[__SINCOS_]])
@@ -123,7 +123,7 @@ entry:
 
 define void @sincos_f32_value_is_same_constantfp(ptr addrspace(1) nocapture writeonly %sin_out, ptr addrspace(1) nocapture writeonly %cos_out) {
 ; CHECK-LABEL: define void @sincos_f32_value_is_same_constantfp
-; CHECK-SAME: (ptr addrspace(1) nocapture writeonly [[SIN_OUT:%.*]], ptr addrspace(1) nocapture writeonly [[COS_OUT:%.*]]) {
+; CHECK-SAME: (ptr addrspace(1) writeonly captures(none) [[SIN_OUT:%.*]], ptr addrspace(1) writeonly captures(none) [[COS_OUT:%.*]]) {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[__SINCOS_:%.*]] = alloca float, align 4, addrspace(5)
 ; CHECK-NEXT:    [[TMP0:%.*]] = call contract float @_Z6sincosfPU3AS5f(float 4.200000e+01, ptr addrspace(5) [[__SINCOS_]])
@@ -152,7 +152,7 @@ define void @sincos_v2f32(<2 x float> %x, ptr addrspace(1) nocapture writeonly %
 ; GCN-NEXT:    ret void
 ;
 ; CHECK-LABEL: define void @sincos_v2f32
-; CHECK-SAME: (<2 x float> [[X:%.*]], ptr addrspace(1) nocapture writeonly [[SIN_OUT:%.*]], ptr addrspace(1) nocapture writeonly [[COS_OUT:%.*]]) {
+; CHECK-SAME: (<2 x float> [[X:%.*]], ptr addrspace(1) writeonly captures(none) [[SIN_OUT:%.*]], ptr addrspace(1) writeonly captures(none) [[COS_OUT:%.*]]) {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[__SINCOS_:%.*]] = alloca <2 x float>, align 8, addrspace(5)
 ; CHECK-NEXT:    [[TMP0:%.*]] = call contract <2 x float> @_Z6sincosDv2_fPU3AS5S_(<2 x float> [[X]], ptr addrspace(5) [[__SINCOS_]])
@@ -172,7 +172,7 @@ entry:
 
 define void @sincos_f32_nobuiltin_callsite(float %x, ptr addrspace(1) nocapture writeonly %sin_out, ptr addrspace(1) nocapture writeonly %cos_out) {
 ; CHECK-LABEL: define void @sincos_f32_nobuiltin_callsite
-; CHECK-SAME: (float [[X:%.*]], ptr addrspace(1) nocapture writeonly [[SIN_OUT:%.*]], ptr addrspace(1) nocapture writeonly [[COS_OUT:%.*]]) {
+; CHECK-SAME: (float [[X:%.*]], ptr addrspace(1) writeonly captures(none) [[SIN_OUT:%.*]], ptr addrspace(1) writeonly captures(none) [[COS_OUT:%.*]]) {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[CALL:%.*]] = tail call contract float @_Z3sinf(float [[X]]) #[[ATTR0:[0-9]+]]
 ; CHECK-NEXT:    store float [[CALL]], ptr addrspace(1) [[SIN_OUT]], align 4
diff --git a/llvm/test/CodeGen/AMDGPU/amdgpu-simplify-libcall-sincos.defined.nobuiltin.ll b/llvm/test/CodeGen/AMDGPU/amdgpu-simplify-libcall-sincos.defined.nobuiltin.ll
index d85f4b8bb6cee..efd7013182b4f 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgpu-simplify-libcall-sincos.defined.nobuiltin.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgpu-simplify-libcall-sincos.defined.nobuiltin.ll
@@ -54,7 +54,7 @@ define float @_Z6sincosfPU3AS0f(float %x, ptr %ptr) {
 
 define void @sincos_f32(float %x, ptr addrspace(1) nocapture writeonly %sin_out, ptr addrspace(1) nocapture writeonly %cos_out) {
 ; CHECK-LABEL: define void @sincos_f32
-; CHECK-SAME: (float [[X:%.*]], ptr addrspace(1) nocapture writeonly [[SIN_OUT:%.*]], ptr addrspace(1) nocapture writeonly [[COS_OUT:%.*]]) {
+; CHECK-SAME: (float [[X:%.*]], ptr addrspace(1) writeonly captures(none) [[SIN_OUT:%.*]], ptr addrspace(1) writeonly captures(none) [[COS_OUT:%.*]]) {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[__SINCOS_:%.*]] = alloca float, align 4, addrspace(5)
 ; CHECK-NEXT:    [[TMP0:%.*]] = addrspacecast ptr addrspace(5) [[__SINCOS_]] to ptr
diff --git a/llvm/test/CodeGen/AMDGPU/amdgpu-simplify-libcall-sincos.defined.sin.cos.ll b/llvm/test/CodeGen/AMDGPU/amdgpu-simplify-libcall-sincos.defined.sin.cos.ll
index 960019c277a73..1b5b8c45ee86e 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgpu-simplify-libcall-sincos.defined.sin.cos.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgpu-simplify-libcall-sincos.defined.sin.cos.ll
@@ -47,7 +47,7 @@ define <2 x float> @_Z3cosDv2_f(<2 x float> noundef %x) {
 
 define void @sincos_f32(float noundef %x, ptr addrspace(1) nocapture noundef writeonly %sin_out, ptr addrspace(1) nocapture noundef writeonly %cos_out) {
 ; CHECK-LABEL: define void @sincos_f32
-; CHECK-SAME: (float noundef [[X:%.*]], ptr addrspace(1) nocapture noundef writeonly [[SIN_OUT:%.*]], ptr addrspace(1) nocapture noundef writeonly [[COS_OUT:%.*]]) {
+; CHECK-SAME: (float noundef [[X:%.*]], ptr addrspace(1) noundef writeonly captures(none) [[SIN_OUT:%.*]], ptr addrspace(1) noundef writeonly captures(none) [[COS_OUT:%.*]]) {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[CALL:%.*]] = tail call contract float @_Z3sinf(float noundef [[X]])
 ; CHECK-NEXT:    store float [[CALL]], ptr addrspace(1) [[SIN_OUT]], align 4
@@ -65,7 +65,7 @@ entry:
 
 define void @sincos_f32_value_is_same_constantfp(ptr addrspace(1) nocapture noundef writeonly %sin_out, ptr addrspace(1) nocapture noundef writeonly %cos_out) {
 ; CHECK-LABEL: define void @sincos_f32_value_is_same_constantfp
-; CHECK-SAME: (ptr addrspace(1) nocapture noundef writeonly [[SIN_OUT:%.*]], ptr addrspace(1) nocapture noundef writeonly [[COS_OUT:%.*]]) {
+; CHECK-SAME: (ptr addrspace(1) noundef writeonly captures(none) [[SIN_OUT:%.*]], ptr addrspace(1) noundef writeonly captures(none) [[COS_OUT:%.*]]) {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[CALL:%.*]] = tail call contract float @_Z3sinf(float 4.200000e+01)
 ; CHECK-NEXT:    store float [[CALL]], ptr addrspace(1) [[SIN_OUT]], align 4
@@ -92,7 +92,7 @@ define void @sincos_v2f32(<2 x float> noundef %x, ptr addrspace(1) nocapture nou
 ; GCN-NEXT:    ret void
 ;
 ; CHECK-LABEL: define void @sincos_v2f32
-; CHECK-SAME: (<2 x float> noundef [[X:%.*]], ptr addrspace(1) nocapture noundef writeonly [[SIN_OUT:%.*]], ptr addrspace(1) nocapture noundef writeonly [[COS_OUT:%.*]]) {
+; CHECK-SAME: (<2 x float> noundef [[X:%.*]], ptr addrspace(1) noundef writeonly captures(none) [[SIN_OUT:%.*]], ptr addrspace(1) noundef writeonly captures(none) [[COS_OUT:%.*]]) {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[CALL:%.*]] = tail call contract <2 x float> @_Z3sinDv2_f(<2 x float> noundef [[X]])
 ; CHECK-NEXT:    store <2 x float> [[CALL]], ptr addrspace(1) [[SIN_OUT]], align 8
diff --git a/llvm/test/CodeGen/AMDGPU/amdgpu-simplify-libcall-sincos.ll b/llvm/test/CodeGen/AMDGPU/amdgpu-simplify-libcall-sincos.ll
index 07587eaacd703..da5686972a86b 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgpu-simplify-libcall-sincos.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgpu-simplify-libcall-sincos.ll
@@ -49,7 +49,7 @@ declare float @_Z6sincosfPU3AS0f(float %x, ptr writeonly %ptr) #1
 
 define void @sincos_f16_nocontract(half %x, ptr addrspace(1) nocapture writeonly %sin_out, ptr addrspace(1) nocapture writeonly %cos_out) {
 ; CHECK-LABEL: define void @sincos_f16_nocontract
-; CHECK-SAME: (half [[X:%.*]], ptr addrspace(1) nocapture writeonly initializes((0, 2)) [[SIN_OUT:%.*]], ptr addrspace(1) nocapture writeonly initializes((0, 2)) [[COS_OUT:%.*]]) local_unnamed_addr #[[ATTR2:[0-9]+]] {
+; CHECK-SAME: (half [[X:%.*]], ptr addrspace(1) writeonly captures(none) initializes((0, 2)) [[SIN_OUT:%.*]], ptr addrspace(1) writeonly captures(none) initializes((0, 2)) [[COS_OUT:%.*]]) local_unnamed_addr #[[ATTR2:[0-9]+]] {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[CALL:%.*]] = tail call half @_Z3sinDh(half [[X]])
 ; CHECK-NEXT:    store half [[CALL]], ptr addrspace(1) [[SIN_OUT]], align 2
@@ -68,7 +68,7 @@ entry:
 
 define void @sincos_v2f16_nocontract(<2 x half> %x, ptr addrspace(1) nocapture writeonly %sin_out, ptr addrspace(1) nocapture writeonly %cos_out) {
 ; CHECK-LABEL: define void @sincos_v2f16_nocontract
-; CHECK-SAME: (<2 x half> [[X:%.*]], ptr addrspace(1) nocapture writeonly initializes((0, 4)) [[SIN_OUT:%.*]], ptr addrspace(1) nocapture writeonly initializes((0, 4)) [[COS_OUT:%.*]]) local_unnamed_addr #[[ATTR2]] {
+; CHECK-SAME: (<2 x half> [[X:%.*]], ptr addrspace(1) writeonly captures(none) initializes((0, 4)) [[SIN_OUT:%.*]], ptr addrspace(1) writeonly captures(none) initializes((0, 4)) [[COS_OUT:%.*]]) local_unnamed_addr #[[ATTR2]] {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[CALL:%.*]] = tail call <2 x half> @_Z3sinDv2_Dh(<2 x half> [[X]])
 ; CHECK-NEXT:    store <2 x half> [[CALL]], ptr addrspace(1) [[SIN_OUT]], align 4
@@ -87,7 +87,7 @@ entry:
 
 define void @sincos_f16(half %x, ptr addrspace(1) nocapture writeonly %sin_out, ptr addrspace(1) nocapture writeonly %cos_out) {
 ; CHECK-LABEL: define void @sincos_f16
-; CHECK-SAME: (half [[X:%.*]], ptr addrspace(1) nocapture writeonly initializes((0, 2)) [[SIN_OUT:%.*]], ptr addrspace(1) nocapture writeonly initializes((0, 2)) [[COS_OUT:%.*]]) local_unnamed_addr #[[ATTR2]] {
+; CHECK-SAME: (half [[X:%.*]], ptr addrspace(1) writeonly captures(none) initializes((0, 2)) [[SIN_OUT:%.*]], ptr addrspace(1) writeonly captures(none) initializes((0, 2)) [[COS_OUT:%.*]]) local_unnamed_addr #[[ATTR2]] {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[CALL:%.*]] = tail call contract half @_Z3sinDh(half [[X]])
 ; CHECK-NEXT:    store half [[CALL]], ptr addrspace(1) [[SIN_OUT]], align 2
@@ -105,7 +105,7 @@ entry:
 
 define void @sincos_f16_order1(half %x, ptr addrspace(1) nocapture writeonly %sin_out, ptr addrspace(1) nocapture writeonly %cos_out) {
 ; CHECK-LABEL: define void @sincos_f16_order1
-; CHECK-SAME: (half [[X:%.*]], ptr addrspace(1) nocapture writeonly initializes((0, 2)) [[SIN_OUT:%.*]], ptr addrspace(1) nocapture writeonly initializes((0, 2)) [[COS_OUT:%.*]]) local_unnamed_addr #[[ATTR2]] {
+; CHECK-SAME: (half [[X:%.*]], ptr addrspace(1) writeonly captures(none) initializes((0, 2)) [[SIN_OUT:%.*]], ptr addrspace(1) writeonly captures(none) initializes((0, 2)) [[COS_OUT:%.*]]) local_unnamed_addr #[[ATTR2]] {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[CALL1:%.*]] = tail call contract half @_Z3cosDh(half [[X]])
 ; CHECK-NEXT:    store half [[CALL1]], ptr addrspace(1) [[COS_OUT]], align 2
@@ -123,7 +123,7 @@ entry:
 
 define void @sincos_v2f16(<2 x half> %x, ptr addrspace(1) nocapture writeonly %sin_out, ptr addrspace(1) nocapture writeonly %cos_out) {
 ; CHECK-LABEL: define void @sincos_v2f16
-; CHECK-SAME: (<2 x half> [[X:%.*]], ptr addrspace(1) nocapture writeonly initializes((0, 4)) [[SIN_OUT:%.*]], ptr addrspace(1) nocapture writeonly initializes((0, 4)) [[COS_OUT:%.*]]) local_unnamed_addr #[[ATTR2]] {
+; CHECK-SAME: (<2 x half> [[X:%.*]], ptr addrspace(1) writeonly captures(none) initializes((0, 4)) [[SIN_OUT:%.*]], ptr addrspace(1) writeonly captures(none) initializes((0, 4)) [[COS_OUT:%.*]]) local_unnamed_addr #[[ATTR2]] {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[CALL:%.*]] = tail call contract <2 x half> @_Z3sinDv2_Dh(<2 x half> [[X]])
 ; CHECK-NEXT:    store <2 x half> [[CALL]], ptr addrspace(1) [[SIN_OUT]], align 4
@@ -141,7 +141,7 @@ entry:
 
 define void @sincos_v3f16(<3 x half> %x, ptr addrspace(1) nocapture writeonly %sin_out, ptr addrspace(1) nocapture writeonly %cos_out) {
 ; CHECK-LABEL: define void @sincos_v3f16
-; CHECK-SAME: (<3 x half> [[X:%.*]], ptr addrspace(1) nocapture writeonly initializes((0, 8)) [[SIN_OUT:%.*]], ptr addrspace(1) nocapture writeonly initializes((0, 8)) [[COS_OUT:%.*]]) local_unnamed_addr #[[ATTR2]] {
+; CHECK-SAME: (<3 x half> [[X:%.*]], ptr addrspace(1) writeonly captures(none) initializes((0, 8)) [[SIN_OUT:%.*]], ptr addrspace(1) writeonly captures(none) initializes((0, 8)) [[COS_OUT:%.*]]) local_unnamed_addr #[[ATTR2]] {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[CALL:%.*]] = tail call contract <3 x half> @_Z3sinDv3_Dh(<3 x half> [[X]])
 ; CHECK-NEXT:    [[EXTRACTVEC2:%.*]] = shufflevector <3 x half> [[CALL]], <3 x half> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 poison>
@@ -164,7 +164,7 @@ entry:
 
 define void @sincos_v4f16(<4 x half> %x, ptr addrspace(1) nocapture writeonly %sin_out, ptr addrspace(1) nocapture writeonly %cos_out) {
 ; CHECK-LABEL: define void @sincos_v4f16
-; CHECK-SAME: (<4 x half> [[X:%.*]], ptr addrspace(1) nocapture writeonly initializes((0, 8)) [[SIN_OUT:%.*]], ptr addrspace(1) nocapture writeonly initializes((0, 8)) [[COS_OUT:%.*]]) local_unnamed_addr #[[ATTR2]] {
+; CHECK-SAME: (<4 x half> [[X:%.*]], ptr addrspace(1) writeonly captures(none) initializes((0, 8)) [[SIN_OUT:%.*]], ptr addrspace(1) writeonly captures(none) initializes((0, 8)) [[COS_OUT:%.*]]) local_unnamed_addr #[[ATTR2]] {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[CALL:%.*]] = tail call contract <4 x half> @_Z3sinDv4_Dh(<4 x half> [[X]])
 ; CHECK-NEXT:    store <4 x half> [[CALL]], ptr addrspace(1) [[SIN_OUT]], align 8
@@ -182,7 +182,7 @@ entry:
 
 define void @sincos_v8f16(<8 x half> %x, ptr addrspace(1) nocapture writeonly %sin_out, ptr addrspace(1) nocapture writeonly %cos_out) {
 ; CHECK-LABEL: define void @sincos_v8f16
-; CHECK-SAME: (<8 x half> [[X:%.*]], ptr addrspace(1) nocapture writeonly initializes((0, 16)) [[SIN_OUT:%.*]], ptr addrspace(1) nocapture writeonly initializes((0, 16)) [[COS_OUT:%.*]]) local_unnamed_addr #[[ATTR2]] {
+; CHECK-SAME: (<8 x half> [[X:%.*]], ptr addrspace(1) writeonly captures(none) initializes((0, 16)) [[SIN_OUT:%.*]], ptr addrspace(1) writeonly captures(none) initializes((0, 16)) [[COS_OUT:%.*]]) local_unnamed_addr #[[ATTR2]] {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[CALL:%.*]] = tail call contract <8 x half> @_Z3sinDv8_Dh(<8 x half> [[X]])
 ; CHECK-NEXT:    store <8 x half> [[CALL]], ptr addrspace(1) [[SIN_OUT]], align 16
@@ -201,7 +201,7 @@ entry:
 
 define void @sincos_v16f16(<16 x half> %x, ptr addrspace(1) nocapture writeonly %sin_out, ptr addrspace(1) nocapture writeonly %cos_out) {
 ; CHECK-LABEL: define void @sincos_v16f16
-; CHECK-SAME: (<16 x half> [[X:%.*]], ptr addrspace(1) nocapture writeonly initializes((0, 32)) [[SIN_OUT:%.*]], ptr addrspace(1) nocapture writeonly initializes((0, 32)) [[COS_OUT:%.*]]) local_unnamed_addr #[[ATTR2]] {
+; CHECK-SAME: (<16 x half> [[X:%.*]], ptr addrspace(1) writeonly captures(none) initializes((0, 32)) [[SIN_OUT:%.*]], ptr addrspace(1) writeonly captures(none) initializes((0, 32)) [[COS_OUT:%.*]]) local_unnamed_addr #[[ATTR2]] {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[CALL:%.*]] = tail call contract <16 x half> @_Z3sinDv16_Dh(<16 x half> [[X]])
 ; CHECK-NEXT:    store <16 x half> [[CALL]], ptr addrspace(1) [[SIN_OUT]], align 32
@@ -220,7 +220,7 @@ entry:
 
 define void @sincos_f32_nocontract(float %x, ptr addrspace(1) nocapture writeonly %sin_out, ptr addrspace(1) nocapture writeonly %cos_out) {
 ; CHECK-LABEL: define void @sincos_f32_nocontract
-; CHECK-SAME: (float [[X:%.*]], ptr addrspace(1) nocapture writeonly initializes((0, 4)) [[SIN_OUT:%.*]], ptr addrspace(1) nocapture writeonly initializes((0, 4)) [[COS_OUT:%.*]]) local_unnamed_addr #[[ATTR3:[0-9]+]] {
+; CHECK-SAME: (float [[X:%.*]], ptr addrspace(1) writeonly captures(none) initializes((0, 4)) [[SIN_OUT:%.*]], ptr addrspace(1) writeonly captures(none) initializes((0, 4)) [[COS_OUT:%.*]]) local_unnamed_addr #[[ATTR3:[0-9]+]] {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[__SINCOS_:%.*]] = alloca float, align 4, addrspace(5)
 ; CHECK-NEXT:    [[TMP0:%.*]] = call float @_Z6sincosfPU3AS5f(float [[X]], ptr addrspace(5) [[__SINCOS_]])
@@ -240,7 +240,7 @@ entry:
 
 define void @sincos_v2f32_nocontract(<2 x float> %x, ptr addrspace(1) nocapture writeonly %sin_out, ptr addrspace(1) nocapture writeonly %cos_out) {
 ; CHECK-LABEL: define void @sincos_v2f32_nocontract
-; CHECK-SAME: (<2 x float> [[X:%.*]], ptr addrspace(1) nocapture writeonly initializes((0, 8)) [[SIN_OUT:%.*]], ptr addrspace(1) nocapture writeonly initializes((0, 8)) [[COS_OUT:%.*]]) local_unnamed_addr #[[ATTR4:[0-9]+]] {
+; CHECK-SAME: (<2 x float> [[X:%.*]], ptr addrspace(1) writeonly captures(none) initializes((0, 8)) [[SIN_OUT:%.*]], ptr addrspace(1) writeonly captures(none) initializes((0, 8)) [[COS_OUT:%.*]]) local_unnamed_addr #[[ATTR4:[0-9]+]] {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[__SINCOS_:%.*]] = alloca <2 x float>, align 8, addrspace(5)
 ; CHECK-NEXT:    [[TMP0:%.*]] = call <2 x float> @_Z6sincosDv2_fPU3AS5S_(<2 x float> [[X]], ptr addrspace(5) [[__SINCOS_]])
@@ -260,7 +260,7 @@ entry:
 
 define void @sincos_f32(float %x, ptr addrspace(1) nocapture writeonly %sin_out, ptr addrspace(1) nocapture writeonly %cos_out) {
 ; CHECK-LABEL: define void @sincos_f32
-; CHECK-SAME: (float [[X:%.*]], ptr addrspace(1) nocapture writeonly initializes((0, 4)) [[SIN_OUT:%.*]], ptr addrspace(1) nocapture writeonly initializes((0, 4)) [[COS_OUT:%.*]]) local_unnamed_addr #[[ATTR3]] {
+; CHECK-SAME: (float [[X:%.*]], ptr addrspace(1) writeonly captures(none) initializes((0, 4)) [[SIN_OUT:%.*]], ptr addrspace(1) writeonly captures(none) initializes((0, 4)) [[COS_OUT:%.*]]) local_unnamed_addr #[[ATTR3]] {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[__SINCOS_:%.*]] = alloca float, align 4, addrspace(5)
 ; CHECK-NEXT:    [[TMP0:%.*]] = call contract float @_Z6sincosfPU3AS5f(float [[X]], ptr addrspace(5) [[__SINCOS_]])
@@ -279,7 +279,7 @@ entry:
 
 define void @sincos_f32_order1(float %x, ptr addrspace(1) nocapture writeonly %sin_out, ptr addrspace(1) nocapture writeonly %cos_out) {
 ; CHECK-LABEL: define void @sincos_f32_order1
-; CHECK-SAME: (float [[X:%.*]], ptr addrspace(1) nocapture writeonly initializes((0, 4)) [[SIN_OUT:%.*]], ptr addrspace(1) nocapture writeonly initializes((0, 4)) [[COS_OUT:%.*]]) local_unnamed_addr #[[ATTR3]] {
+; CHECK-SAME: (float [[X:%.*]], ptr addrspace(1) writeonly captures(none) initializes((0, 4)) [[SIN_OUT:%.*]], ptr addrspace(1) writeonly captures(none) initializes((0, 4)) [[COS_OUT:%.*]]) local_unnamed_addr #[[ATTR3]] {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[__SINCOS_:%.*]] = alloca float, align 4, addrspace(5)
 ; CHECK-NEXT:    [[TMP0:%.*]] = call contract float @_Z6sincosfPU3AS5f(float [[X]], ptr addrspace(5) [[__SINCOS_]])
@@ -298,7 +298,7 @@ entry:
 
 define void @sincos_v2f32(<2 x float> %x, ptr addrspace(1) nocapture writeonly %sin_out, ptr addrspace(1) nocapture writeonly %cos_out) {
 ; CHECK-LABEL: define void @sincos_v2f32
-; CHECK-SAME: (<2 x float> [[X:%.*]], ptr addrspace(1) nocapture writeonly initializes((0, 8)) [[SIN_OUT:%.*]], ptr addrspace(1) nocapture writeonly initializes((0, 8)) [[COS_OUT:%.*]]) local_unnamed_addr #[[ATTR4]] {
+; CHECK-SAME: (<2 x float> [[X:%.*]], ptr addrspace(1) writeonly captures(none) initializes((0, 8)) [[SIN_OUT:%.*]], ptr addrspace(1) writeonly captures(none) initializes((0, 8)) [[COS_OUT:%.*]]) local_unnamed_addr #[[ATTR4]] {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[__SINCOS_:%.*]] = alloca <2 x float>, align 8, addrspace(5)
 ; CHECK-NEXT:    [[TMP0:%.*]] = call contract <2 x float> @_Z6sincosDv2_fPU3AS5S_(<2 x float> [[X]], ptr addrspace(5) [[__SINCOS_]])
@@ -317,7 +317,7 @@ entry:
 
 define void @sincos_v3f32(<3 x float> %x, ptr addrspace(1) nocapture writeonly %sin_out, ptr addrspace(1) nocapture writeonly %cos_out) {
 ; CHECK-LABEL: define void @sincos_v3f32
-; CHECK-SAME: (<3 x float> [[X:%.*]], ptr addrspace(1) nocapture writeonly initializes((0, 16)) [[SIN_OUT:%.*]], ptr addrspace(1) nocapture writeonly initializes((0, 16)) [[COS_OUT:%.*]]) local_unnamed_addr #[[ATTR4]] {
+; CHECK-SAME: (<3 x float> [[X:%.*]], ptr addrspace(1) writeonly captures(none) initializes((0, 16)) [[SIN_OUT:%.*]], ptr addrspace(1) writeonly captures(none) initializes((0, 16)) [[COS_OUT:%.*]]) local_unnamed_addr #[[ATTR4]] {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[__SINCOS_:%.*]] = alloca <3 x float>, align 16, addrspace(5)
 ; CHECK-NEXT:    [[TMP0:%.*]] = call contract <3 x float> @_Z6sincosDv3_fPU3AS5S_(<3 x float> [[X]], ptr addrspace(5) [[__SINCOS_]])
@@ -340,7 +340,7 @@ entry:
 
 define void @sincos_v4f32(<4 x float> %x, ptr addrspace(1) nocapture writeonly %sin_out, ptr addrspace(1) nocapture writeonly %cos_out) {
 ; CHECK-LABEL: define void @sincos_v4f32
-; CHECK-SAME: (<4 x float> [[X:%.*]], ptr addrspace(1) nocapture writeonly initializes((0, 16)) [[SIN_OUT:%.*]], ptr addrspace(1) nocapture writeonly initializes((0, 16)) [[COS_OUT:%.*]]) local_unnamed_addr #[[ATTR4]] {
+; CHECK-SAME: (<4 x float> [[X:%.*]], ptr addrspace(1) writeonly captures(none) initializes((0, 16)) [[SIN_OUT:%.*]], ptr addrspace(1) writeonly captures(none) initializes((0, 16)) [[COS_OUT:%.*]]) local_unnamed_addr #[[ATTR4]] {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[__SINCOS_:%.*]] = alloca <4 x float>, align 16, addrspace(5)
 ; CHECK-NEXT:    [[TMP0:%.*]] = call contract <4 x float> @_Z6sincosDv4_fPU3AS5S_(<4 x float> [[X]], ptr addrspace(5) [[__SINCOS_]])
@@ -359,7 +359,7 @@ entry:
 
 define void @sincos_v8f32(<8 x float> %x, ptr addrspace(1) nocapture writeonly %sin_out, ptr addrspace(1) nocapture writeonly %cos_out) {
 ; CHECK-LABEL: define void @sincos_v8f32
-; CHECK-SAME: (<8 x float> [[X:%.*]], ptr addrspace(1) nocapture writeonly initializes((0, 32)) [[SIN_OUT:%.*]], ptr addrspace(1) nocapture writeonly initializes((0, 32)) [[COS_OUT:%.*]]) local_unnamed_addr #[[ATTR4]] {
+; CHECK-SAME: (<8 x float> [[X:%.*]], ptr addrspace(1) writeonly captures(none) initializes((0, 32)) [[SIN_OUT:%.*]], ptr addrspace(1) writeonly captures(none) initializes((0, 32)) [[COS_OUT:%.*]]) local_unnamed_addr #[[ATTR4]] {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[__SINCOS_:%.*]] = alloca <8 x float>, align 32, addrspace(5)
 ; CHECK-NEXT:    [[TMP0:%.*]] = call contract <8 x float> @_Z6sincosDv8_fPU3AS5S_(<8 x float> [[X]], ptr addrspace(5) [[__SINCOS_]])
@@ -378,7 +378,7 @@ entry:
 
 define void @sincos_v16f32(<16 x float> %x, ptr addrspace(1) nocapture writeonly %sin_out, ptr addrspace(1) nocapture writeonly %cos_out) {
 ; CHECK-LABEL: define void @sincos_v16f32
-; CHECK-SAME: (<16 x float> [[X:%.*]], ptr addrspace(1) nocapture writeonly initializes((0, 64)) [[SIN_OUT:%.*]], ptr addrspace(1) nocapture writeonly initializes((0, 64)) [[COS_OUT:%.*]]) local_unnamed_addr #[[ATTR4]] {
+; CHECK-SAME: (<16 x float> [[X:%.*]], ptr addrspace(1) writeonly captures(none) initializes((0, 64)) [[SIN_OUT:%.*]], ptr addrspace(1) writeonly captures(none) initializes((0, 64)) [[COS_OUT:%.*]]) local_unnamed_addr #[[ATTR4]] {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[__SINCOS_:%.*]] = alloca <16 x float>, align 64, addrspace(5)
 ; CHECK-NEXT:    [[TMP0:%.*]] = call contract <16 x float> @_Z6sincosDv16_fPU3AS5S_(<16 x float> [[X]], ptr addrspace(5) [[__SINCOS_]])
@@ -397,7 +397,7 @@ entry:
 
 define void @sincos_f64_nocontract(double %x, ptr addrspace(1) nocapture writeonly %sin_out, ptr addrspace(1) nocapture writeonly %cos_out) {
 ; CHECK-LABEL: define void @sincos_f64_nocontract
-; CHECK-SAME: (double [[X:%.*]], ptr addrspace(1) nocapture writeonly initializes((0, 8)) [[SIN_OUT:%.*]], ptr addrspace(1) nocapture writeonly initializes((0, 8)) [[COS_OUT:%.*]]) local_unnamed_addr #[[ATTR4]] {
+; CHECK-SAME: (double [[X:%.*]], ptr addrspace(1) writeonly captures(none) initializes((0, 8)) [[SIN_OUT:%.*]], ptr addrspace(1) writeonly captures(none) initializes((0, 8)) [[COS_OUT:%.*]]) local_unnamed_addr #[[ATTR4]] {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[__SINCOS_:%.*]] = alloca double, align 8, addrspace(5)
 ; CHECK-NEXT:    [[TMP0:%.*]] = call double @_Z6sincosdPU3AS5d(double [[X]], ptr addrspace(5) [[__SINCOS_]])
@@ -417,7 +417,7 @@ entry:
 
 define void @sincos_v2f64_nocontract(<2 x double> %x, ptr addrspace(1) nocapture writeonly %sin_out, ptr addrspace(1) nocapture writeonly %cos_out) {
 ; CHECK-LABEL: define void @sincos_v2f64_nocontract
-; CHECK-SAME: (<2 x double> [[X:%.*]], ptr addrspace(1) nocapture writeonly initializes((0, 16)) [[SIN_OUT:%.*]], ptr addrspace(1) nocapture writeonly initializes((0, 16)) [[COS_OUT:%.*]]) local_unnamed_addr #[[ATTR4]] {
+; CHECK-SAME: (<2 x double> [[X:%.*]], ptr addrspace(1) writeonly captures(none) initializes((0, 16)) [[SIN_OUT:%.*]], ptr addrspace(1) writeonly captures(none) initializes((0, 16)) [[COS_OUT:%.*]]) local_unnamed_addr #[[ATTR4]] {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[__SINCOS_:%.*]] = alloca <2 x double>, align 16, addrspace(5)
 ; CHECK-NEXT:    [[TMP0:%.*]] = call <2 x double> @_Z6sincosDv2_dPU3AS5S_(<2 x double> [[X]], ptr addrspace(5) [[__SINCOS_]])
@@ -436,7 +436,7 @@ entry:
 
 define void @sincos_f64(double %x, ptr addrspace(1) nocapture writeonly %sin_out, ptr addrspace(1) nocapture writeonly %cos_out) {
 ; CHECK-LABEL: define void @sincos_f64
-; CHECK-SAME: (double [[X:%.*]], ptr addrspace(1) nocapture writeonly initializes((0, 8)) [[SIN_OUT:%.*]], ptr addrspace(1) nocapture writeonly initializes((0, 8)) [[COS_OUT:%.*]]) local_unnamed_addr #[[ATTR4]] {
+; CHECK-SAME: (double [[X:%.*]], ptr addrspace(1) writeonly captures(none) initializes((0, 8)) [[SIN_OUT:%.*]], ptr addrspace(1) writeonly captures(none) initializes((0, 8)) [[COS_OUT:%.*]]) local_unnamed_addr #[[ATTR4]] {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[__SINCOS_:%.*]] = alloca double, align 8, addrspace(5)
 ; CHECK-NEXT:    [[TMP0:%.*]] = call contract double @_Z6sincosdPU3AS5d(double [[X]], ptr addrspace(5) [[__SINCOS_]])
@@ -455,7 +455,7 @@ entry:
 
 define void @sincos_f64_order1(double %x, ptr addrspace(1) nocapture writeonly %sin_out, ptr addrspace(1) nocapture writeonly %cos_out) {
 ; CHECK-LABEL: define void @sincos_f64_order1
-; CHECK-SAME: (double [[X:%.*]], ptr addrspace(1) nocapture writeonly initializes((0, 8)) [[SIN_OUT:%.*]], ptr addrspace(1) nocapture writeonly initializes((0, 8)) [[COS_OUT:%.*]]) local_unnamed_addr #[[ATTR4]] {
+; CHECK-SAME: (double [[X:%.*]], ptr addrspace(1) writeonly captures(none) initializes((0, 8)) [[SIN_OUT:%.*]], ptr addrspace(1) writeonly captures(none) initializes((0, 8)) [[COS_OUT:%.*]]) local_unnamed_addr #[[ATTR4]] {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[__SINCOS_:%.*]] = alloca double, align 8, addrspace(5)
 ; CHECK-NEXT:    [[TMP0:%.*]] = call contract double @_Z6sincosdPU3AS5d(double [[X]], ptr addrspace(5) [[__SINCOS_]])
@@ -474,7 +474,7 @@ entry:
 
 define void @sincos_v2f64(<2 x double> %x, ptr addrspace(1) nocapture writeonly %sin_out, ptr addrspace(1) nocapture writeonly %cos_out) {
 ; CHECK-LABEL: define void @sincos_v2f64
-; CHECK-SAME: (<2 x double> [[X:%.*]], ptr addrspace(1) nocapture writeonly initializes((0, 16)) [[SIN_OUT:%.*]], ptr addrspace(1) nocapture writeonly initializes((0, 16)) [[COS_OUT:%.*]]) local_unnamed_addr #[[ATTR4]] {
+; CHECK-SAME: (<2 x double> [[X:%.*]], ptr addrspace(1) writeonly captures(none) initializes((0, 16)) [[SIN_OUT:%.*]], ptr addrspace(1) writeonly captures(none) initializes((0, 16)) [[COS_OUT:%.*]]) local_unnamed_addr #[[ATTR4]] {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[__SINCOS_:%.*]] = alloca <2 x double>, align 16, addrspace(5)
 ; CHECK-NEXT:    [[TMP0:%.*]] = call contract <2 x double> @_Z6sincosDv2_dPU3AS5S_(<2 x double> [[X]], ptr addrspace(5) [[__SINCOS_]])
@@ -493,7 +493,7 @@ entry:
 
 define void @sincos_v3f64(<3 x double> %x, ptr addrspace(1) nocapture writeonly %sin_out, ptr addrspace(1) nocapture writeonly %cos_out) {
 ; CHECK-LABEL: define void @sincos_v3f64
-; CHECK-SAME: (<3 x double> [[X:%.*]], ptr addrspace(1) nocapture writeonly initializes((0, 32)) [[SIN_OUT:%.*]], ptr addrspace(1) nocapture writeonly initializes((0, 32)) [[COS_OUT:%.*]]) local_unnamed_addr #[[ATTR4]] {
+; CHECK-SAME: (<3 x double> [[X:%.*]], ptr addrspace(1) writeonly captures(none) initializes((0, 32)) [[SIN_OUT:%.*]], ptr addrspace(1) writeonly captures(none) initializes((0, 32)) [[COS_OUT:%.*]]) local_unnamed_addr #[[ATTR4]] {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[__SINCOS_:%.*]] = alloca <3 x double>, align 32, addrspace(5)
 ; CHECK-NEXT:    [[TMP0:%.*]] = call contract <3 x double> @_Z6sincosDv3_dPU3AS5S_(<3 x double> [[X]], ptr addrspace(5) [[__SINCOS_]])
@@ -516,7 +516,7 @@ entry:
 
 define void @sincos_v4f64(<4 x double> %x, ptr addrspace(1) nocapture writeonly %sin_out, ptr addrspace(1) nocapture writeonly %cos_out) {
 ; CHECK-LABEL: define void @sincos_v4f64
-; CHECK-SAME: (<4 x double> [[X:%.*]], ptr addrspace(1) nocapture writeonly initializes((0, 32)) [[SIN_OUT:%.*]], ptr addrspace(1) nocapture writeonly initializes((0, 32)) [[COS_OUT:%.*]]) local_unnamed_addr #[[ATTR4]] {
+; CHECK-SAME: (<4 x double> [[X:%.*]], ptr addrspace(1) writeonly captures(none) initializes((0, 32)) [[SIN_OUT:%.*]], ptr addrspace(1) writeonly captures(none) initializes((0, 32)) [[COS_OUT:%.*]]) local_unnamed_addr #[[ATTR4]] {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[__SINCOS_:%.*]] = alloca <4 x double>, align 32, addrspace(5)
 ; CHECK-NEXT:    [[TMP0:%.*]] = call contract <4 x double> @_Z6sincosDv4_dPU3AS5S_(<4 x double> [[X]], ptr addrspace(5) [[__SINCOS_]])
@@ -535,7 +535,7 @@ entry:
 
 define void @sincos_v8f64(<8 x double> %x, ptr addrspace(1) nocapture writeonly %sin_out, ptr addrspace(1) nocapture writeonly %cos_out) {
 ; CHECK-LABEL: define void @sincos_v8f64
-; CHECK-SAME: (<8 x double> [[X:%.*]], ptr addrspace(1) nocapture writeonly initializes((0, 64)) [[SIN_OUT:%.*]], ptr addrspace(1) nocapture writeonly initializes((0, 64)) [[COS_OUT:%.*]]) local_unnamed_addr #[[ATTR4]] {
+; CHECK-SAME: (<8 x double> [[X:%.*]], ptr addrspace(1) writeonly captures(none) initializes((0, 64)) [[SIN_OUT:%.*]], ptr addrspace(1) writeonly captures(none) initializes((0, 64)) [[COS_OUT:%.*]]) local_unnamed_addr #[[ATTR4]] {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[__SINCOS_:%.*]] = alloca <8 x double>, align 64, addrspace(5)
 ; CHECK-NEXT:    [[TMP0:%.*]] = call contract <8 x double> @_Z6sincosDv8_dPU3AS5S_(<8 x double> [[X]], ptr addrspace(5) [[__SINCOS_]])
@@ -554,7 +554,7 @@ entry:
 
 define void @sincos_v16f64(<16 x double> %x, ptr addrspace(1) nocapture writeonly %sin_out, ptr addrspace(1) nocapture writeonly %cos_out) {
 ; CHECK-LABEL: define void @sincos_v16f64
-; CHECK-SAME: (<16 x double> [[X:%.*]], ptr addrspace(1) nocapture writeonly initializes((0, 128)) [[SIN_OUT:%.*]], ptr addrspace(1) nocapture writeonly initializes((0, 128)) [[COS_OUT:%.*]]) local_unnamed_addr #[[ATTR4]] {
+; CHECK-SAME: (<16 x double> [[X:%.*]], ptr addrspace(1) writeonly captures(none) initializes((0, 128)) [[SIN_OUT:%.*]], ptr addrspace(1) writeonly captures(none) initializes((0, 128)) [[COS_OUT:%.*]]) local_unnamed_addr #[[ATTR4]] {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[__SINCOS_:%.*]] = alloca <16 x double>, align 128, addrspace(5)
 ; CHECK-NEXT:    [[TMP0:%.*]] = call contract <16 x double> @_Z6sincosDv16_dPU3AS5S_(<16 x double> [[X]], ptr addrspace(5) [[__SINCOS_]])
@@ -575,7 +575,7 @@ entry:
 
 define void @sincos_f32_different_blocks(i1 %cond, float %x, ptr addrspace(1) nocapture writeonly %sin_out, ptr addrspace(1) nocapture writeonly %cos_out) {
 ; CHECK-LABEL: define void @sincos_f32_different_blocks
-; CHECK-SAME: (i1 [[COND:%.*]], float [[X:%.*]], ptr addrspace(1) nocapture writeonly [[SIN_OUT:%.*]], ptr addrspace(1) nocapture writeonly [[COS_OUT:%.*]]) local_unnamed_addr #[[ATTR3]] {
+; CHECK-SAME: (i1 [[COND:%.*]], float [[X:%.*]], ptr addrspace(1) writeonly captures(none) [[SIN_OUT:%.*]], ptr addrspace(1) writeonly captures(none) [[COS_OUT:%.*]]) local_unnamed_addr #[[ATTR3]] {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[__SINCOS_:%.*]] = alloca float, align 4, addrspace(5)
 ; CHECK-NEXT:    [[TMP0:%.*]] = call contract float @_Z6sincosfPU3AS5f(float [[X]], ptr addrspace(5) [[__SINCOS_]])
@@ -607,7 +607,7 @@ bb1:
 ; The sin and cos are in different blocks but always execute
 define void @sincos_f32_different_blocks_dominating_always_execute(i1 %cond, float %x, ptr addrspace(1) nocapture writeonly %sin_out, ptr addrspace(1) nocapture writeonly %cos_out, ptr addrspace(1) %other) {
 ; CHECK-LABEL: define void @sincos_f32_different_blocks_dominating_always_execute
-; CHECK-SAME: (i1 [[COND:%.*]], float [[X:%.*]], ptr addrspace(1) nocapture writeonly initializes((0, 4)) [[SIN_OUT:%.*]], ptr addrspace(1) nocapture writeonly initializes((0, 4)) [[COS_OUT:%.*]], ptr addrspace(1) nocapture writeonly [[OTHER:%.*]]) local_unnamed_addr #[[ATTR3]] {
+; CHECK-SAME: (i1 [[COND:%.*]], float [[X:%.*]], ptr addrspace(1) writeonly captures(none) initializes((0, 4)) [[SIN_OUT:%.*]], ptr addrspace(1) writeonly captures(none) initializes((0, 4)) [[COS_OUT:%.*]], ptr addrspace(1) writeonly captures(none) [[OTHER:%.*]]) local_unnamed_addr #[[ATTR3]] {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[__SINCOS_:%.*]] = alloca float, align 4, addrspace(5)
 ; CHECK-NEXT:    [[TMP0:%.*]] = call contract float @_Z6sincosfPU3AS5f(float [[X]], ptr addrspace(5) [[__SINCOS_]])
@@ -639,7 +639,7 @@ bb1:
 ; sin dominates cos but cos doesn't always execute.
 define void @sincos_f32_different_blocks_dominating_conditional_execute(i1 %cond, float %x, ptr addrspace(1) nocapture writeonly %sin_out, ptr addrspace(1) nocapture writeonly %cos_out, ptr addrspace(1) %other) {
 ; CHECK-LABEL: define void @sincos_f32_different_blocks_dominating_conditional_execute
-; CHECK-SAME: (i1 [[COND:%.*]], float [[X:%.*]], ptr addrspace(1) nocapture writeonly initializes((0, 4)) [[SIN_OUT:%.*]], ptr addrspace(1) nocapture writeonly initializes((0, 4)) [[COS_OUT:%.*]], ptr addrspace(1) nocapture readnone [[OTHER:%.*]]) local_unnamed_addr #[[ATTR3]] {
+; CHECK-SAME: (i1 [[COND:%.*]], float [[X:%.*]], ptr addrspace(1) writeonly captures(none) initializes((0, 4)) [[SIN_OUT:%.*]], ptr addrspace(1) writeonly captures(none) initializes((0, 4)) [[COS_OUT:%.*]], ptr addrspace(1) readnone captures(none) [[OTHER:%.*]]) local_unnamed_addr #[[ATTR3]] {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[__SINCOS_:%.*]] = alloca float, align 4, addrspace(5)
 ; CHECK-NEXT:    [[TMP0:%.*]] = call contract float @_Z6sincosfPU3AS5f(float [[X]], ptr addrspace(5) [[__SINCOS_]])
@@ -685,7 +685,7 @@ declare void @func(ptr addrspace(1))
 
 define void @sincos_f32_value_is_instr(ptr addrspace(1) %value.ptr, ptr addrspace(1) nocapture writeonly %sin_out, ptr addrspace(1) nocapture writeonly %cos_out) {
 ; CHECK-LABEL: define void @sincos_f32_value_is_instr
-; CHECK-SAME: (ptr addrspace(1) [[VALUE_PTR:%.*]], ptr addrspace(1) nocapture writeonly initializes((0, 4)) [[SIN_OUT:%.*]], ptr addrspace(1) nocapture writeonly initializes((0, 4)) [[COS_OUT:%.*]]) local_unnamed_addr #[[ATTR4]] {
+; CHECK-SAME: (ptr addrspace(1) [[VALUE_PTR:%.*]], ptr addrspace(1) writeonly captures(none) initializes((0, 4)) [[SIN_OUT:%.*]], ptr addrspace(1) writeonly captures(none) initializes((0, 4)) [[COS_OUT:%.*]]) local_unnamed_addr #[[ATTR4]] {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[__SINCOS_:%.*]] = alloca float, align 4, addrspace(5)
 ; CHECK-NEXT:    tail call void @func(ptr addrspace(1) [[VALUE_PTR]])
@@ -708,7 +708,7 @@ entry:
 
 define void @sincos_f32_value_is_same_constexpr(ptr addrspace(1) nocapture writeonly %sin_out, ptr addrspace(1) nocapture writeonly %cos_out) {
 ; CHECK-LABEL: define void @sincos_f32_value_is_same_constexpr
-; CHECK-SAME: (ptr addrspace(1) nocapture writeonly initializes((0, 4)) [[SIN_OUT:%.*]], ptr addrspace(1) nocapture writeonly initializes((0, 4)) [[COS_OUT:%.*]]) local_unnamed_addr #[[ATTR3]] {
+; CHECK-SAME: (ptr addrspace(1) writeonly captures(none) initializes((0, 4)) [[SIN_OUT:%.*]], ptr addrspace(1) writeonly captures(none) initializes((0, 4)) [[COS_OUT:%.*]]) local_unnamed_addr #[[ATTR3]] {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[__SINCOS_:%.*]] = alloca float, align 4, addrspace(5)
 ; CHECK-NEXT:    [[TMP0:%.*]] = call contract float @_Z6sincosfPU3AS5f(float bitcast (i32 ptrtoint (ptr @func to i32) to float), ptr addrspace(5) [[__SINCOS_]])
@@ -727,7 +727,7 @@ entry:
 
 define void @sincos_f32_value_is_different_constexpr(ptr addrspace(1) nocapture writeonly %sin_out, ptr addrspace(1) nocapture writeonly %cos_out) {
 ; CHECK-LABEL: define void @sincos_f32_value_is_different_constexpr
-; CHECK-SAME: (ptr addrspace(1) nocapture writeonly initializes((0, 4)) [[SIN_OUT:%.*]], ptr addrspace(1) nocapture writeonly initializes((0, 4)) [[COS_OUT:%.*]]) #[[ATTR2]] {
+; CHECK-SAME: (ptr addrspace(1) writeonly captures(none) initializes((0, 4)) [[SIN_OUT:%.*]], ptr addrspace(1) writeonly captures(none) initializes((0, 4)) [[COS_OUT:%.*]]) #[[ATTR2]] {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[CALL:%.*]] = tail call contract float @_Z3sinf(float bitcast (i32 ptrtoint (ptr @func to i32) to float))
 ; CHECK-NEXT:    store float [[CALL]], ptr addrspace(1) [[SIN_OUT]], align 4
@@ -745,7 +745,7 @@ entry:
 
 define void @sincos_f32_value_is_same_constantfp(ptr addrspace(1) nocapture writeonly %sin_out, ptr addrspace(1) nocapture writeonly %cos_out) {
 ; CHECK-LABEL: define void @sincos_f32_value_is_same_constantfp
-; CHECK-SAME: (ptr addrspace(1) nocapture writeonly initializes((0, 4)) [[SIN_OUT:%.*]], ptr addrspace(1) nocapture writeonly initializes((0, 4)) [[COS_OUT:%.*]]) local_unnamed_addr #[[ATTR3]] {
+; CHECK-SAME: (ptr addrspace(1) writeonly captures(none) initializes((0, 4)) [[SIN_OUT:%.*]], ptr addrspace(1) writeonly captures(none) initializes((0, 4)) [[COS_OUT:%.*]]) local_unnamed_addr #[[ATTR3]] {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[__SINCOS_:%.*]] = alloca float, align 4, addrspace(5)
 ; CHECK-NEXT:    [[TMP0:%.*]] = call contract float @_Z6sincosfPU3AS5f(float 4.200000e+01, ptr addrspace(5) [[__SINCOS_]])
@@ -764,7 +764,7 @@ entry:
 
 define void @sincos_f32_value_is_different_constantfp(ptr addrspace(1) nocapture writeonly %sin_out, ptr addrspace(1) nocapture writeonly %cos_out) {
 ; CHECK-LABEL: define void @sincos_f32_value_is_different_constantfp
-; CHECK-SAME: (ptr addrspace(1) nocapture writeonly initializes((0, 4)) [[SIN_OUT:%.*]], ptr addrspace(1) nocapture writeonly initializes((0, 4)) [[COS_OUT:%.*]]) local_unnamed_addr #[[ATTR2]] {
+; CHECK-SAME: (ptr addrspace(1) writeonly captures(none) initializes((0, 4)) [[SIN_OUT:%.*]], ptr addrspace(1) writeonly captures(none) initializes((0, 4)) [[COS_OUT:%.*]]) local_unnamed_addr #[[ATTR2]] {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[CALL:%.*]] = tail call contract float @_Z3sinf(float 4.200000e+01)
 ; CHECK-NEXT:    store float [[CALL]], ptr addrspace(1) [[SIN_OUT]], align 4
@@ -782,7 +782,7 @@ entry:
 
 define void @sincos_f32_different_args(float %x, float %y, ptr addrspace(1) nocapture writeonly %sin_out, ptr addrspace(1) nocapture writeonly %cos_out) {
 ; CHECK-LABEL: define void @sincos_f32_different_args
-; CHECK-SAME: (float [[X:%.*]], float [[Y:%.*]], ptr addrspace(1) nocapture writeonly initializes((0, 4)) [[SIN_OUT:%.*]], ptr addrspace(1) nocapture writeonly initializes((0, 4)) [[COS_OUT:%.*]]) local_unnamed_addr #[[ATTR2]] {
+; CHECK-SAME: (float [[X:%.*]], float [[Y:%.*]], ptr addrspace(1) writeonly captures(none) initializes((0, 4)) [[SIN_OUT:%.*]], ptr addrspace(1) writeonly captures(none) initializes((0, 4)) [[COS_OUT:%.*]]) local_unnamed_addr #[[ATTR2]] {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[CALL:%.*]] = tail call contract float @_Z3sinf(float [[X]])
 ; CHECK-NEXT:    store float [[CALL]], ptr addrspace(1) [[SIN_OUT]], align 4
@@ -800,7 +800,7 @@ entry:
 
 define void @sincos_f32_flag_intersect0(float %x, ptr addrspace(1) nocapture writeonly %sin_out, ptr addrspace(1) nocapture writeonly %cos_out) {
 ; CHECK-LABEL: define void @sincos_f32_flag_intersect0
-; CHECK-SAME: (float [[X:%.*]], ptr addrspace(1) nocapture writeonly initializes((0, 4)) [[SIN_OUT:%.*]], ptr addrspace(1) nocapture writeonly initializes((0, 4)) [[COS_OUT:%.*]]) local_unnamed_addr #[[ATTR3]] {
+; CHECK-SAME: (float [[X:%.*]], ptr addrspace(1) writeonly captures(none) initializes((0, 4)) [[SIN_OUT:%.*]], ptr addrspace(1) writeonly captures(none) initializes((0, 4)) [[COS_OUT:%.*]]) local_unnamed_addr #[[ATTR3]] {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[__SINCOS_:%.*]] = alloca float, align 4, addrspace(5)
 ; CHECK-NEXT:    [[TMP0:%.*]] = call contract float @_Z6sincosfPU3AS5f(float [[X]], ptr addrspace(5) [[__SINCOS_]])
@@ -819,7 +819,7 @@ entry:
 
 define void @sincos_f32_flag_intersect1(float %x, ptr addrspace(1) nocapture writeonly %sin_out, ptr addrspace(1) nocapture writeonly %cos_out) {
 ; CHECK-LABEL: define void @sincos_f32_flag_intersect1
-; CHECK-SAME: (float [[X:%.*]], ptr addrspace(1) nocapture writeonly initializes((0, 4)) [[SIN_OUT:%.*]], ptr addrspace(1) nocapture writeonly initializes((0, 4)) [[COS_OUT:%.*]]) local_unnamed_addr #[[ATTR3]] {
+; CHECK-SAME: (float [[X:%.*]], ptr addrspace(1) writeonly captures(none) initializes((0, 4)) [[SIN_OUT:%.*]], ptr addrspace(1) writeonly captures(none) initializes((0, 4)) [[COS_OUT:%.*]]) local_unnamed_addr #[[ATTR3]] {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[__SINCOS_:%.*]] = alloca float, align 4, addrspace(5)
 ; CHECK-NEXT:    [[TMP0:%.*]] = call nnan contract float @_Z6sincosfPU3AS5f(float [[X]], ptr addrspace(5) [[__SINCOS_]])
@@ -838,7 +838,7 @@ entry:
 
 define void @sincos_v2f32_flag_intersect1(<2 x float> %x, ptr addrspace(1) nocapture writeonly %sin_out, ptr addrspace(1) nocapture writeonly %cos_out) {
 ; CHECK-LABEL: define void @sincos_v2f32_flag_intersect1
-; CHECK-SAME: (<2 x float> [[X:%.*]], ptr addrspace(1) nocapture writeonly initializes((0, 8)) [[SIN_OUT:%.*]], ptr addrspace(1) nocapture writeonly initializes((0, 8)) [[COS_OUT:%.*]]) local_unnamed_addr #[[ATTR4]] {
+; CHECK-SAME: (<2 x float> [[X:%.*]], ptr addrspace(1) writeonly captures(none) initializes((0, 8)) [[SIN_OUT:%.*]], ptr addrspace(1) writeonly captures(none) initializes((0, 8)) [[COS_OUT:%.*]]) local_unnamed_addr #[[ATTR4]] {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[__SINCOS_:%.*]] = alloca <2 x float>, align 8, addrspace(5)
 ; CHECK-NEXT:    [[TMP0:%.*]] = call nnan contract <2 x float> @_Z6sincosDv2_fPU3AS5S_(<2 x float> [[X]], ptr addrspace(5) [[__SINCOS_]])
@@ -947,7 +947,7 @@ entry:
 
 define void @sin_f32_indirect_call_user(float %x, ptr addrspace(1) nocapture writeonly %sin_out, ptr addrspace(1) nocapture writeonly %cos_out, ptr %func.ptr) {
 ; CHECK-LABEL: define void @sin_f32_indirect_call_user
-; CHECK-SAME: (float [[X:%.*]], ptr addrspace(1) nocapture writeonly initializes((0, 4)) [[SIN_OUT:%.*]], ptr addrspace(1) nocapture writeonly initializes((0, 4)) [[COS_OUT:%.*]], ptr nocapture readonly [[FUNC_PTR:%.*]]) local_unnamed_addr #[[ATTR4]] {
+; CHECK-SAME: (float [[X:%.*]], ptr addrspace(1) writeonly captures(none) initializes((0, 4)) [[SIN_OUT:%.*]], ptr addrspace(1) writeonly captures(none) initializes((0, 4)) [[COS_OUT:%.*]], ptr readonly captures(none) [[FUNC_PTR:%.*]]) local_unnamed_addr #[[ATTR4]] {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[CALL:%.*]] = tail call contract float @_Z3sinf(float [[X]])
 ; CHECK-NEXT:    store float [[CALL]], ptr addrspace(1) [[SIN_OUT]], align 4
@@ -965,7 +965,7 @@ entry:
 
 define void @cos_f32_indirect_call_user(float %x, ptr addrspace(1) nocapture writeonly %sin_out, ptr addrspace(1) nocapture writeonly %cos_out, ptr %func.ptr) {
 ; CHECK-LABEL: define void @cos_f32_indirect_call_user
-; CHECK-SAME: (float [[X:%.*]], ptr addrspace(1) nocapture writeonly initializes((0, 4)) [[SIN_OUT:%.*]], ptr addrspace(1) nocapture writeonly initializes((0, 4)) [[COS_OUT:%.*]], ptr nocapture readonly [[FUNC_PTR:%.*]]) local_unnamed_addr #[[ATTR4]] {
+; CHECK-SAME: (float [[X:%.*]], ptr addrspace(1) writeonly captures(none) initializes((0, 4)) [[SIN_OUT:%.*]], ptr addrspace(1) writeonly captures(none) initializes((0, 4)) [[COS_OUT:%.*]], ptr readonly captures(none) [[FUNC_PTR:%.*]]) local_unnamed_addr #[[ATTR4]] {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[CALL:%.*]] = tail call contract float @_Z3cosf(float [[X]])
 ; CHECK-NEXT:    store float [[CALL]], ptr addrspace(1) [[COS_OUT]], align 4
@@ -983,7 +983,7 @@ entry:
 
 define void @sincos_f32_preserve_fpmath_0(float %x, ptr addrspace(1) nocapture writeonly %sin_out, ptr addrspace(1) nocapture writeonly %cos_out) {
 ; CHECK-LABEL: define void @sincos_f32_preserve_fpmath_0
-; CHECK-SAME: (float [[X:%.*]], ptr addrspace(1) nocapture writeonly initializes((0, 4)) [[SIN_OUT:%.*]], ptr addrspace(1) nocapture writeonly initializes((0, 4)) [[COS_OUT:%.*]]) local_unnamed_addr #[[ATTR3]] {
+; CHECK-SAME: (float [[X:%.*]], ptr addrspace(1) writeonly captures(none) initializes((0, 4)) [[SIN_OUT:%.*]], ptr addrspace(1) writeonly captures(none) initializes((0, 4)) [[COS_OUT:%.*]]) local_unnamed_addr #[[ATTR3]] {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[__SINCOS_:%.*]] = alloca float, align 4, addrspace(5)
 ; CHECK-NEXT:    [[TMP0:%.*]] = call contract float @_Z6sincosfPU3AS5f(float [[X]], ptr addrspace(5) [[__SINCOS_]]), !fpmath [[META5:![0-9]+]]
@@ -1002,7 +1002,7 @@ entry:
 
 define void @sincos_f32_preserve_fpmath_1(float %x, ptr addrspace(1) nocapture writeonly %sin_out, ptr addrspace(1) nocapture writeonly %cos_out) {
 ; CHECK-LABEL: define void @sincos_f32_preserve_fpmath_1
-; CHECK-SAME: (float [[X:%.*]], ptr addrspace(1) nocapture writeonly initializes((0, 4)) [[SIN_OUT:%.*]], ptr addrspace(1) nocapture writeonly initializes((0, 4)) [[COS_OUT:%.*]]) local_unnamed_addr #[[ATTR3]] {
+; CHECK-SAME: (float [[X:%.*]], ptr addrspace(1) writeonly captures(none) initializes((0, 4)) [[SIN_OUT:%.*]], ptr addrspace(1) writeonly captures(none) initializes((0, 4)) [[COS_OUT:%.*]]) local_unnamed_addr #[[ATTR3]] {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[__SINCOS_:%.*]] = alloca float, align 4, addrspace(5)
 ; CHECK-NEXT:    [[TMP0:%.*]] = call contract float @_Z6sincosfPU3AS5f(float [[X]], ptr addrspace(5) [[__SINCOS_]]), !fpmath [[META6:![0-9]+]]
@@ -1022,7 +1022,7 @@ entry:
 ; Should drop the metadata
 define void @sincos_f32_drop_fpmath(float %x, ptr addrspace(1) nocapture writeonly %sin_out, ptr addrspace(1) nocapture writeonly %cos_out) {
 ; CHECK-LABEL: define void @sincos_f32_drop_fpmath
-; CHECK-SAME: (float [[X:%.*]], ptr addrspace(1) nocapture writeonly initializes((0, 4)) [[SIN_OUT:%.*]], ptr addrspace(1) nocapture writeonly initializes((0, 4)) [[COS_OUT:%.*]]) local_unnamed_addr #[[ATTR3]] {
+; CHECK-SAME: (float [[X:%.*]], ptr addrspace(1) writeonly captures(none) initializes((0, 4)) [[SIN_OUT:%.*]], ptr addrspace(1) writeonly captures(none) initializes((0, 4)) [[COS_OUT:%.*]]) local_unnamed_addr #[[ATTR3]] {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[__SINCOS_:%.*]] = alloca float, align 4, addrspace(5)
 ; CHECK-NEXT:    [[TMP0:%.*]] = call contract float @_Z6sincosfPU3AS5f(float [[X]], ptr addrspace(5) [[__SINCOS_]])
@@ -1041,7 +1041,7 @@ entry:
 
 define void @sincos_f32_debuginfo(float %x, ptr addrspace(1) nocapture writeonly %sin_out, ptr addrspace(1) nocapture writeonly %cos_out) !dbg !15 {
 ; CHECK-LABEL: define void @sincos_f32_debuginfo
-; CHECK-SAME: (float [[X:%.*]], ptr addrspace(1) nocapture writeonly initializes((0, 4)) [[SIN_OUT:%.*]], ptr addrspace(1) nocapture writeonly initializes((0, 4)) [[COS_OUT:%.*]]) local_unnamed_addr #[[ATTR3]] !dbg [[DBG7:![0-9]+]] {
+; CHECK-SAME: (float [[X:%.*]], ptr addrspace(1) writeonly captures(none) initializes((0, 4)) [[SIN_OUT:%.*]], ptr addrspace(1) writeonly captures(none) initializes((0, 4)) [[COS_OUT:%.*]]) local_unnamed_addr #[[ATTR3]] !dbg [[DBG7:![0-9]+]] {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[__SINCOS_:%.*]] = alloca float, align 4, addrspace(5), !dbg [[DBG14:![0-9]+]]
 ; CHECK-NEXT:    [[TMP0:%.*]] = call contract float @_Z6sincosfPU3AS5f(float [[X]], ptr addrspace(5) [[__SINCOS_]]), !dbg [[DBG14]]
@@ -1064,7 +1064,7 @@ entry:
 
 define float @sin_sincos_private_f32(float %x, ptr addrspace(1) %sin_out, ptr addrspace(1) %cos_out) {
 ; CHECK-LABEL: define float @sin_sincos_private_f32
-; CHECK-SAME: (float [[X:%.*]], ptr addrspace(1) nocapture writeonly initializes((0, 4)) [[SIN_OUT:%.*]], ptr addrspace(1) nocapture writeonly initializes((0, 4)) [[COS_OUT:%.*]]) local_unnamed_addr #[[ATTR3]] {
+; CHECK-SAME: (float [[X:%.*]], ptr addrspace(1) writeonly captures(none) initializes((0, 4)) [[SIN_OUT:%.*]], ptr addrspace(1) writeonly captures(none) initializes((0, 4)) [[COS_OUT:%.*]]) local_unnamed_addr #[[ATTR3]] {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[COS_TMP:%.*]] = alloca float, align 4, addrspace(5)
 ; CHECK-NEXT:    [[SIN0:%.*]] = tail call nnan ninf nsz contract float @_Z3sinf(float [[X]]), !fpmath [[META5]]
@@ -1086,7 +1086,7 @@ entry:
 
 define float @sin_sincos_generic_f32(float %x, ptr addrspace(1) %sin_out, ptr addrspace(1) %cos_out) {
 ; CHECK-LABEL: define float @sin_sincos_generic_f32
-; CHECK-SAME: (float [[X:%.*]], ptr addrspace(1) nocapture writeonly initializes((0, 4)) [[SIN_OUT:%.*]], ptr addrspace(1) nocapture writeonly initializes((0, 4)) [[COS_OUT:%.*]]) local_unnamed_addr #[[ATTR3]] {
+; CHECK-SAME: (float [[X:%.*]], ptr addrspace(1) writeonly captures(none) initializes((0, 4)) [[SIN_OUT:%.*]], ptr addrspace(1) writeonly captures(none) initializes((0, 4)) [[COS_OUT:%.*]]) local_unnamed_addr #[[ATTR3]] {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[COS_TMP:%.*]] = alloca float, align 4, addrspace(5)
 ; CHECK-NEXT:    [[SIN0:%.*]] = tail call nsz contract float @_Z3sinf(float [[X]]), !fpmath [[META5]]
@@ -1110,7 +1110,7 @@ entry:
 
 define float @cos_sincos_private_f32(float %x, ptr addrspace(1) %sin_out, ptr addrspace(1) %cos_out) {
 ; CHECK-LABEL: define float @cos_sincos_private_f32
-; CHECK-SAME: (float [[X:%.*]], ptr addrspace(1) nocapture readnone [[SIN_OUT:%.*]], ptr addrspace(1) nocapture writeonly initializes((0, 4)) [[COS_OUT:%.*]]) local_unnamed_addr #[[ATTR3]] {
+; CHECK-SAME: (float [[X:%.*]], ptr addrspace(1) readnone captures(none) [[SIN_OUT:%.*]], ptr addrspace(1) writeonly captures(none) initializes((0, 4)) [[COS_OUT:%.*]]) local_unnamed_addr #[[ATTR3]] {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[COS_TMP:%.*]] = alloca float, align 4, addrspace(5)
 ; CHECK-NEXT:    [[COS0:%.*]] = tail call contract float @_Z3cosf(float [[X]])
@@ -1132,7 +1132,7 @@ entry:
 
 define float @cos_sincos_generic_f32(float %x, ptr addrspace(1) %sin_out, ptr addrspace(1) %cos_out) {
 ; CHECK-LABEL: define float @cos_sincos_generic_f32
-; CHECK-SAME: (float [[X:%.*]], ptr addrspace(1) nocapture readnone [[SIN_OUT:%.*]], ptr addrspace(1) nocapture writeonly initializes((0, 4)) [[COS_OUT:%.*]]) local_unnamed_addr #[[ATTR3]] {
+; CHECK-SAME: (float [[X:%.*]], ptr addrspace(1) readnone captures(none) [[SIN_OUT:%.*]], ptr addrspace(1) writeonly captures(none) initializes((0, 4)) [[COS_OUT:%.*]]) local_unnamed_addr #[[ATTR3]] {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[COS_TMP:%.*]] = alloca float, align 4, addrspace(5)
 ; CHECK-NEXT:    [[COS0:%.*]] = tail call contract float @_Z3cosf(float [[X]])
@@ -1156,7 +1156,7 @@ entry:
 
 define float @sincos_private_f32_x2(float %x, ptr addrspace(1) %sin_out, ptr addrspace(1) %cos_out) {
 ; CHECK-LABEL: define float @sincos_private_f32_x2
-; CHECK-SAME: (float [[X:%.*]], ptr addrspace(1) nocapture readnone [[SIN_OUT:%.*]], ptr addrspace(1) nocapture writeonly initializes((0, 4)) [[COS_OUT:%.*]]) local_unnamed_addr #[[ATTR3]] {
+; CHECK-SAME: (float [[X:%.*]], ptr addrspace(1) readnone captures(none) [[SIN_OUT:%.*]], ptr addrspace(1) writeonly captures(none) initializes((0, 4)) [[COS_OUT:%.*]]) local_unnamed_addr #[[ATTR3]] {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[COS_TMP0:%.*]] = alloca float, align 4, addrspace(5)
 ; CHECK-NEXT:    [[COS_TMP1:%.*]] = alloca float, align 4, addrspace(5)
@@ -1184,7 +1184,7 @@ entry:
 
 define float @sincos_generic_f32_x2(float %x, ptr addrspace(1) %sin_out, ptr addrspace(1) %cos_out) {
 ; CHECK-LABEL: define float @sincos_generic_f32_x2
-; CHECK-SAME: (float [[X:%.*]], ptr addrspace(1) nocapture readnone [[SIN_OUT:%.*]], ptr addrspace(1) nocapture writeonly initializes((0, 4)) [[COS_OUT:%.*]]) local_unnamed_addr #[[ATTR3]] {
+; CHECK-SAME: (float [[X:%.*]], ptr addrspace(1) readnone captures(none) [[SIN_OUT:%.*]], ptr addrspace(1) writeonly captures(none) initializes((0, 4)) [[COS_OUT:%.*]]) local_unnamed_addr #[[ATTR3]] {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[COS_TMP0:%.*]] = alloca float, align 4, addrspace(5)
 ; CHECK-NEXT:    [[COS_TMP1:%.*]] = alloca float, align 4, addrspace(5)
@@ -1213,7 +1213,7 @@ entry:
 
 define float @sincos_generic_private_f32(float %x, ptr addrspace(1) %sin_out, ptr addrspace(1) %cos_out) {
 ; CHECK-LABEL: define float @sincos_generic_private_f32
-; CHECK-SAME: (float [[X:%.*]], ptr addrspace(1) nocapture readnone [[SIN_OUT:%.*]], ptr addrspace(1) nocapture writeonly initializes((0, 4)) [[COS_OUT:%.*]]) local_unnamed_addr #[[ATTR3]] {
+; CHECK-SAME: (float [[X:%.*]], ptr addrspace(1) readnone captures(none) [[SIN_OUT:%.*]], ptr addrspace(1) writeonly captures(none) initializes((0, 4)) [[COS_OUT:%.*]]) local_unnamed_addr #[[ATTR3]] {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[COS_TMP0:%.*]] = alloca float, align 4, addrspace(5)
 ; CHECK-NEXT:    [[COS_TMP1:%.*]] = alloca float, align 4, addrspace(5)
@@ -1240,7 +1240,7 @@ entry:
 
 define float @sincos_mixed_sin_cos_generic_private_f32(float %x, ptr addrspace(1) %sin_out, ptr addrspace(1) %cos_out) {
 ; CHECK-LABEL: define float @sincos_mixed_sin_cos_generic_private_f32
-; CHECK-SAME: (float [[X:%.*]], ptr addrspace(1) nocapture writeonly initializes((0, 4)) [[SIN_OUT:%.*]], ptr addrspace(1) nocapture writeonly initializes((0, 4)) [[COS_OUT:%.*]]) local_unnamed_addr #[[ATTR3]] {
+; CHECK-SAME: (float [[X:%.*]], ptr addrspace(1) writeonly captures(none) initializes((0, 4)) [[SIN_OUT:%.*]], ptr addrspace(1) writeonly captures(none) initializes((0, 4)) [[COS_OUT:%.*]]) local_unnamed_addr #[[ATTR3]] {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[COS_TMP0:%.*]] = alloca float, align 4, addrspace(5)
 ; CHECK-NEXT:    [[__SINCOS_:%.*]] = alloca float, align 4, addrspace(5)
diff --git a/llvm/test/CodeGen/AMDGPU/amdgpu-simplify-libcall-sincos.nobuiltins.ll b/llvm/test/CodeGen/AMDGPU/amdgpu-simplify-libcall-sincos.nobuiltins.ll
index 5d765b614db36..9920e7ad191c1 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgpu-simplify-libcall-sincos.nobuiltins.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgpu-simplify-libcall-sincos.nobuiltins.ll
@@ -11,7 +11,7 @@ declare <2 x float> @_Z3cosDv2_f(<2 x float> noundef)
 define void @sincos_f32_nobuiltin(float noundef %x, ptr addrspace(1) nocapture noundef writeonly %sin_out, ptr addrspace(1) nocapture noundef writeonly %cos_out) #0 {
 ; CHECK: Function Attrs: nobuiltin
 ; CHECK-LABEL: define void @sincos_f32_nobuiltin
-; CHECK-SAME: (float noundef [[X:%.*]], ptr addrspace(1) nocapture noundef writeonly [[SIN_OUT:%.*]], ptr addrspace(1) nocapture noundef writeonly [[COS_OUT:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-SAME: (float noundef [[X:%.*]], ptr addrspace(1) noundef writeonly captures(none) [[SIN_OUT:%.*]], ptr addrspace(1) noundef writeonly captures(none) [[COS_OUT:%.*]]) #[[ATTR0:[0-9]+]] {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[__SINCOS_:%.*]] = alloca float, align 4, addrspace(5)
 ; CHECK-NEXT:    [[TMP0:%.*]] = call contract float @_Z6sincosfPU3AS5f(float [[X]], ptr addrspace(5) [[__SINCOS_]])
@@ -32,7 +32,7 @@ entry:
 define void @sincos_v2f32_nobuiltin(<2 x float> noundef %x, ptr addrspace(1) nocapture noundef writeonly %sin_out, ptr addrspace(1) nocapture noundef writeonly %cos_out) #0 {
 ; CHECK: Function Attrs: nobuiltin
 ; CHECK-LABEL: define void @sincos_v2f32_nobuiltin
-; CHECK-SAME: (<2 x float> noundef [[X:%.*]], ptr addrspace(1) nocapture noundef writeonly [[SIN_OUT:%.*]], ptr addrspace(1) nocapture noundef writeonly [[COS_OUT:%.*]]) #[[ATTR0]] {
+; CHECK-SAME: (<2 x float> noundef [[X:%.*]], ptr addrspace(1) noundef writeonly captures(none) [[SIN_OUT:%.*]], ptr addrspace(1) noundef writeonly captures(none) [[COS_OUT:%.*]]) #[[ATTR0]] {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[__SINCOS_:%.*]] = alloca <2 x float>, align 8, addrspace(5)
 ; CHECK-NEXT:    [[TMP0:%.*]] = call contract <2 x float> @_Z6sincosDv2_fPU3AS5S_(<2 x float> [[X]], ptr addrspace(5) [[__SINCOS_]])
@@ -52,7 +52,7 @@ entry:
 
 define void @sincos_f32_no_builtins(float noundef %x, ptr addrspace(1) nocapture noundef writeonly %sin_out, ptr addrspace(1) nocapture noundef writeonly %cos_out) #1 {
 ; CHECK-LABEL: define void @sincos_f32_no_builtins
-; CHECK-SAME: (float noundef [[X:%.*]], ptr addrspace(1) nocapture noundef writeonly [[SIN_OUT:%.*]], ptr addrspace(1) nocapture noundef writeonly [[COS_OUT:%.*]]) #[[ATTR1:[0-9]+]] {
+; CHECK-SAME: (float noundef [[X:%.*]], ptr addrspace(1) noundef writeonly captures(none) [[SIN_OUT:%.*]], ptr addrspace(1) noundef writeonly captures(none) [[COS_OUT:%.*]]) #[[ATTR1:[0-9]+]] {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[__SINCOS_:%.*]] = alloca float, align 4, addrspace(5)
 ; CHECK-NEXT:    [[TMP0:%.*]] = call contract float @_Z6sincosfPU3AS5f(float [[X]], ptr addrspace(5) [[__SINCOS_]])
@@ -72,7 +72,7 @@ entry:
 
 define void @sincos_v2f32_no_builtins(<2 x float> noundef %x, ptr addrspace(1) nocapture noundef writeonly %sin_out, ptr addrspace(1) nocapture noundef writeonly %cos_out) #1 {
 ; CHECK-LABEL: define void @sincos_v2f32_no_builtins
-; CHECK-SAME: (<2 x float> noundef [[X:%.*]], ptr addrspace(1) nocapture noundef writeonly [[SIN_OUT:%.*]], ptr addrspace(1) nocapture noundef writeonly [[COS_OUT:%.*]]) #[[ATTR1]] {
+; CHECK-SAME: (<2 x float> noundef [[X:%.*]], ptr addrspace(1) noundef writeonly captures(none) [[SIN_OUT:%.*]], ptr addrspace(1) noundef writeonly captures(none) [[COS_OUT:%.*]]) #[[ATTR1]] {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[__SINCOS_:%.*]] = alloca <2 x float>, align 8, addrspace(5)
 ; CHECK-NEXT:    [[TMP0:%.*]] = call contract <2 x float> @_Z6sincosDv2_fPU3AS5S_(<2 x float> [[X]], ptr addrspace(5) [[__SINCOS_]])
@@ -92,7 +92,7 @@ entry:
 
 define void @sincos_f32_nobuiltin_callsite(float noundef %x, ptr addrspace(1) nocapture noundef writeonly %sin_out, ptr addrspace(1) nocapture noundef writeonly %cos_out) {
 ; CHECK-LABEL: define void @sincos_f32_nobuiltin_callsite
-; CHECK-SAME: (float noundef [[X:%.*]], ptr addrspace(1) nocapture noundef writeonly [[SIN_OUT:%.*]], ptr addrspace(1) nocapture noundef writeonly [[COS_OUT:%.*]]) {
+; CHECK-SAME: (float noundef [[X:%.*]], ptr addrspace(1) noundef writeonly captures(none) [[SIN_OUT:%.*]], ptr addrspace(1) noundef writeonly captures(none) [[COS_OUT:%.*]]) {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[CALL:%.*]] = tail call contract float @_Z3sinf(float noundef [[X]]) #[[ATTR0]]
 ; CHECK-NEXT:    store float [[CALL]], ptr addrspace(1) [[SIN_OUT]], align 4
@@ -110,7 +110,7 @@ entry:
 
 define void @sincos_f32_nobuiltin_callsite0(float noundef %x, ptr addrspace(1) nocapture noundef writeonly %sin_out, ptr addrspace(1) nocapture noundef writeonly %cos_out) {
 ; CHECK-LABEL: define void @sincos_f32_nobuiltin_callsite0
-; CHECK-SAME: (float noundef [[X:%.*]], ptr addrspace(1) nocapture noundef writeonly [[SIN_OUT:%.*]], ptr addrspace(1) nocapture noundef writeonly [[COS_OUT:%.*]]) {
+; CHECK-SAME: (float noundef [[X:%.*]], ptr addrspace(1) noundef writeonly captures(none) [[SIN_OUT:%.*]], ptr addrspace(1) noundef writeonly captures(none) [[COS_OUT:%.*]]) {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[CALL:%.*]] = tail call contract float @_Z3sinf(float noundef [[X]]) #[[ATTR0]]
 ; CHECK-NEXT:    store float [[CALL]], ptr addrspace(1) [[SIN_OUT]], align 4
@@ -128,7 +128,7 @@ entry:
 
 define void @sincos_f32_nobuiltin_callsite1(float noundef %x, ptr addrspace(1) nocapture noundef writeonly %sin_out, ptr addrspace(1) nocapture noundef writeonly %cos_out) {
 ; CHECK-LABEL: define void @sincos_f32_nobuiltin_callsite1
-; CHECK-SAME: (float noundef [[X:%.*]], ptr addrspace(1) nocapture noundef writeonly [[SIN_OUT:%.*]], ptr addrspace(1) nocapture noundef writeonly [[COS_OUT:%.*]]) {
+; CHECK-SAME: (float noundef [[X:%.*]], ptr addrspace(1) noundef writeonly captures(none) [[SIN_OUT:%.*]], ptr addrspace(1) noundef writeonly captures(none) [[COS_OUT:%.*]]) {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[CALL:%.*]] = tail call contract float @_Z3sinf(float noundef [[X]])
 ; CHECK-NEXT:    store float [[CALL]], ptr addrspace(1) [[SIN_OUT]], align 4
@@ -146,7 +146,7 @@ entry:
 
 define void @sincos_v2f32_nobuiltin_callsite(<2 x float> noundef %x, ptr addrspace(1) nocapture noundef writeonly %sin_out, ptr addrspace(1) nocapture noundef writeonly %cos_out) {
 ; CHECK-LABEL: define void @sincos_v2f32_nobuiltin_callsite
-; CHECK-SAME: (<2 x float> noundef [[X:%.*]], ptr addrspace(1) nocapture noundef writeonly [[SIN_OUT:%.*]], ptr addrspace(1) nocapture noundef writeonly [[COS_OUT:%.*]]) {
+; CHECK-SAME: (<2 x float> noundef [[X:%.*]], ptr addrspace(1) noundef writeonly captures(none) [[SIN_OUT:%.*]], ptr addrspace(1) noundef writeonly captures(none) [[COS_OUT:%.*]]) {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[CALL:%.*]] = tail call contract <2 x float> @_Z3sinDv2_f(<2 x float> noundef [[X]]) #[[ATTR0]]
 ; CHECK-NEXT:    store <2 x float> [[CALL]], ptr addrspace(1) [[SIN_OUT]], align 8
diff --git a/llvm/test/CodeGen/AMDGPU/amdgpu-simplify-libcall-sincos.weak.ll b/llvm/test/CodeGen/AMDGPU/amdgpu-simplify-libcall-sincos.weak.ll
index ef76ca4979237..3bf297d278e0c 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgpu-simplify-libcall-sincos.weak.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgpu-simplify-libcall-sincos.weak.ll
@@ -8,7 +8,7 @@ declare extern_weak float @_Z3cosf(float noundef)
 
 define void @sincos_f32(float noundef %x, ptr addrspace(1) nocapture noundef writeonly %sin_out, ptr addrspace(1) nocapture noundef writeonly %cos_out) {
 ; CHECK-LABEL: define void @sincos_f32
-; CHECK-SAME: (float noundef [[X:%.*]], ptr addrspace(1) nocapture noundef writeonly [[SIN_OUT:%.*]], ptr addrspace(1) nocapture noundef writeonly [[COS_OUT:%.*]]) {
+; CHECK-SAME: (float noundef [[X:%.*]], ptr addrspace(1) noundef writeonly captures(none) [[SIN_OUT:%.*]], ptr addrspace(1) noundef writeonly captures(none) [[COS_OUT:%.*]]) {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[CALL:%.*]] = tail call contract float @_Z3sinf(float noundef [[X]])
 ; CHECK-NEXT:    store float [[CALL]], ptr addrspace(1) [[SIN_OUT]], align 4
@@ -26,7 +26,7 @@ entry:
 
 define void @sincos_f32_value_is_same_constantfp(ptr addrspace(1) nocapture noundef writeonly %sin_out, ptr addrspace(1) nocapture noundef writeonly %cos_out) {
 ; CHECK-LABEL: define void @sincos_f32_value_is_same_constantfp
-; CHECK-SAME: (ptr addrspace(1) nocapture noundef writeonly [[SIN_OUT:%.*]], ptr addrspace(1) nocapture noundef writeonly [[COS_OUT:%.*]]) {
+; CHECK-SAME: (ptr addrspace(1) noundef writeonly captures(none) [[SIN_OUT:%.*]], ptr addrspace(1) noundef writeonly captures(none) [[COS_OUT:%.*]]) {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[CALL:%.*]] = tail call contract float @_Z3sinf(float 4.200000e+01)
 ; CHECK-NEXT:    store float [[CALL]], ptr addrspace(1) [[SIN_OUT]], align 4
diff --git a/llvm/test/CodeGen/AMDGPU/amdpal-cs.ll b/llvm/test/CodeGen/AMDGPU/amdpal-cs.ll
index 0818f607da0a5..96775f4763e34 100644
--- a/llvm/test/CodeGen/AMDGPU/amdpal-cs.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdpal-cs.ll
@@ -8,6 +8,7 @@
 ; GCN-NEXT: amdpal.pipelines:
 ; GCN-NEXT:   - .hardware_stages:
 ; GCN-NEXT:       .cs:
+; GCN-NEXT:         .entry_point:    _amdgpu_cs
 ; GCN-NEXT:         .entry_point_symbol:    cs_amdpal
 ; GCN-NEXT:         .scratch_memory_size: 0
 ; GCN:     .registers:
diff --git a/llvm/test/CodeGen/AMDGPU/amdpal-es.ll b/llvm/test/CodeGen/AMDGPU/amdpal-es.ll
index e37d22c7df372..1379246c32573 100644
--- a/llvm/test/CodeGen/AMDGPU/amdpal-es.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdpal-es.ll
@@ -7,6 +7,7 @@
 ; GCN-NEXT: amdpal.pipelines:
 ; GCN-NEXT:   - .hardware_stages:
 ; GCN-NEXT:       .es:
+; GCN-NEXT:         .entry_point:    _amdgpu_es
 ; GCN-NEXT:         .entry_point_symbol:    es_amdpal
 ; GCN-NEXT:         .scratch_memory_size: 0
 ; GCN:     .registers:
diff --git a/llvm/test/CodeGen/AMDGPU/amdpal-gs.ll b/llvm/test/CodeGen/AMDGPU/amdpal-gs.ll
index d847f75a5c093..1fba34a50094f 100644
--- a/llvm/test/CodeGen/AMDGPU/amdpal-gs.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdpal-gs.ll
@@ -8,6 +8,7 @@
 ; GCN-NEXT: amdpal.pipelines:
 ; GCN-NEXT:   - .hardware_stages:
 ; GCN-NEXT:       .gs:
+; GCN-NEXT:         .entry_point:    _amdgpu_gs
 ; GCN-NEXT:         .entry_point_symbol:    gs_amdpal
 ; GCN-NEXT:         .scratch_memory_size: 0
 ; GCN:     .registers:
diff --git a/llvm/test/CodeGen/AMDGPU/amdpal-hs.ll b/llvm/test/CodeGen/AMDGPU/amdpal-hs.ll
index 74f5f440c99d7..53c6b95f07351 100644
--- a/llvm/test/CodeGen/AMDGPU/amdpal-hs.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdpal-hs.ll
@@ -8,6 +8,7 @@
 ; GCN-NEXT: amdpal.pipelines:
 ; GCN-NEXT:   - .hardware_stages:
 ; GCN-NEXT:       .hs:
+; GCN-NEXT:         .entry_point:    _amdgpu_hs
 ; GCN-NEXT:         .entry_point_symbol:    hs_amdpal
 ; GCN-NEXT:         .scratch_memory_size: 0
 ; GCN:     .registers:
diff --git a/llvm/test/CodeGen/AMDGPU/amdpal-ls.ll b/llvm/test/CodeGen/AMDGPU/amdpal-ls.ll
index 287cc1201a3c5..ebe753134a42d 100644
--- a/llvm/test/CodeGen/AMDGPU/amdpal-ls.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdpal-ls.ll
@@ -7,6 +7,7 @@
 ; GCN-NEXT: amdpal.pipelines:
 ; GCN-NEXT:   - .hardware_stages:
 ; GCN-NEXT:       .ls:
+; GCN-NEXT:         .entry_point:    _amdgpu_ls
 ; GCN-NEXT:         .entry_point_symbol:    ls_amdpal
 ; GCN-NEXT:         .scratch_memory_size: 0
 ; GCN:     .registers:
diff --git a/llvm/test/CodeGen/AMDGPU/amdpal-psenable.ll b/llvm/test/CodeGen/AMDGPU/amdpal-psenable.ll
index e1767182c359d..32f19e2af32ec 100644
--- a/llvm/test/CodeGen/AMDGPU/amdpal-psenable.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdpal-psenable.ll
@@ -11,6 +11,7 @@
 ; GCN-NEXT: amdpal.pipelines:
 ; GCN-NEXT:   - .hardware_stages:
 ; GCN-NEXT:       .ps:
+; GCN-NEXT:         .entry_point:    _amdgpu_ps
 ; GCN-NEXT:         .entry_point_symbol:    amdpal_psenable
 ; GCN-NEXT:         .scratch_memory_size: 0
 ; GCN:     .registers:
diff --git a/llvm/test/CodeGen/AMDGPU/amdpal-vs.ll b/llvm/test/CodeGen/AMDGPU/amdpal-vs.ll
index b225d978601ab..853d221ee3aab 100644
--- a/llvm/test/CodeGen/AMDGPU/amdpal-vs.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdpal-vs.ll
@@ -8,6 +8,7 @@
 ; GCN-NEXT: amdpal.pipelines:
 ; GCN-NEXT:   - .hardware_stages:
 ; GCN-NEXT:       .vs:
+; GCN-NEXT:         .entry_point:    _amdgpu_vs
 ; GCN-NEXT:         .entry_point_symbol:    vs_amdpal
 ; GCN-NEXT:         .scratch_memory_size: 0
 ; GCN:     .registers:
diff --git a/llvm/test/CodeGen/AMDGPU/amdpal.ll b/llvm/test/CodeGen/AMDGPU/amdpal.ll
index 97fcf0606b5b7..171df029615ed 100644
--- a/llvm/test/CodeGen/AMDGPU/amdpal.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdpal.ll
@@ -86,6 +86,7 @@ declare void @llvm.amdgcn.raw.ptr.buffer.store.f32(float, ptr addrspace(8), i32,
 ; PAL-NEXT: amdpal.pipelines:
 ; PAL-NEXT:   - .hardware_stages:
 ; PAL-NEXT:       .cs:
+; PAL-NEXT:         .entry_point:    _amdgpu_cs
 ; PAL-NEXT:         .entry_point_symbol:    scratch2_cs
 ; PAL-NEXT:         .scratch_memory_size: 0x10
 ; PAL-NEXT:         .sgpr_count:     0x
diff --git a/llvm/test/CodeGen/AMDGPU/atomic_optimizations_buffer.ll b/llvm/test/CodeGen/AMDGPU/atomic_optimizations_buffer.ll
index a9e092fa39fbe..cd405fabf002d 100644
--- a/llvm/test/CodeGen/AMDGPU/atomic_optimizations_buffer.ll
+++ b/llvm/test/CodeGen/AMDGPU/atomic_optimizations_buffer.ll
@@ -220,7 +220,6 @@ define amdgpu_kernel void @add_i32_constant(ptr addrspace(1) %out, ptr addrspace
 ; GFX12W64-NEXT:    s_mov_b64 s[0:1], exec
 ; GFX12W64-NEXT:    v_mbcnt_lo_u32_b32 v0, s2, 0
 ; GFX12W64-NEXT:    ; implicit-def: $vgpr1
-; GFX12W64-NEXT:    s_wait_alu 0xfffe
 ; GFX12W64-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12W64-NEXT:    v_mbcnt_hi_u32_b32 v0, s3, v0
 ; GFX12W64-NEXT:    v_cmpx_eq_u32_e32 0, v0
@@ -240,6 +239,7 @@ define amdgpu_kernel void @add_i32_constant(ptr addrspace(1) %out, ptr addrspace
 ; GFX12W64-NEXT:    s_wait_loadcnt 0x0
 ; GFX12W64-NEXT:    v_readfirstlane_b32 s2, v1
 ; GFX12W64-NEXT:    v_mov_b32_e32 v1, 0
+; GFX12W64-NEXT:    s_wait_alu 0xf1ff
 ; GFX12W64-NEXT:    s_delay_alu instid0(VALU_DEP_2)
 ; GFX12W64-NEXT:    v_mad_u32_u24 v0, v0, 5, s2
 ; GFX12W64-NEXT:    s_wait_kmcnt 0x0
@@ -257,7 +257,6 @@ define amdgpu_kernel void @add_i32_constant(ptr addrspace(1) %out, ptr addrspace
 ; GFX12W32-NEXT:    s_cbranch_execz .LBB0_2
 ; GFX12W32-NEXT:  ; %bb.1:
 ; GFX12W32-NEXT:    s_load_b128 s[8:11], s[4:5], 0x34
-; GFX12W32-NEXT:    s_wait_alu 0xfffe
 ; GFX12W32-NEXT:    s_bcnt1_i32_b32 s1, s1
 ; GFX12W32-NEXT:    s_wait_alu 0xfffe
 ; GFX12W32-NEXT:    s_mul_i32 s1, s1, 5
@@ -266,7 +265,6 @@ define amdgpu_kernel void @add_i32_constant(ptr addrspace(1) %out, ptr addrspace
 ; GFX12W32-NEXT:    s_wait_kmcnt 0x0
 ; GFX12W32-NEXT:    buffer_atomic_add_u32 v1, off, s[8:11], null th:TH_ATOMIC_RETURN
 ; GFX12W32-NEXT:  .LBB0_2:
-; GFX12W32-NEXT:    s_wait_alu 0xfffe
 ; GFX12W32-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX12W32-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
 ; GFX12W32-NEXT:    s_wait_loadcnt 0x0
@@ -499,7 +497,6 @@ define amdgpu_kernel void @add_i32_uniform(ptr addrspace(1) %out, ptr addrspace(
 ; GFX12W64-NEXT:    s_mov_b64 s[0:1], exec
 ; GFX12W64-NEXT:    v_mbcnt_lo_u32_b32 v0, s2, 0
 ; GFX12W64-NEXT:    ; implicit-def: $vgpr1
-; GFX12W64-NEXT:    s_wait_alu 0xfffe
 ; GFX12W64-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12W64-NEXT:    v_mbcnt_hi_u32_b32 v0, s3, v0
 ; GFX12W64-NEXT:    v_cmpx_eq_u32_e32 0, v0
@@ -519,6 +516,7 @@ define amdgpu_kernel void @add_i32_uniform(ptr addrspace(1) %out, ptr addrspace(
 ; GFX12W64-NEXT:    s_wait_loadcnt 0x0
 ; GFX12W64-NEXT:    v_readfirstlane_b32 s2, v1
 ; GFX12W64-NEXT:    s_wait_kmcnt 0x0
+; GFX12W64-NEXT:    s_wait_alu 0xf1ff
 ; GFX12W64-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12W64-NEXT:    v_mad_co_u64_u32 v[0:1], null, s6, v0, s[2:3]
 ; GFX12W64-NEXT:    v_mov_b32_e32 v1, 0
@@ -537,7 +535,6 @@ define amdgpu_kernel void @add_i32_uniform(ptr addrspace(1) %out, ptr addrspace(
 ; GFX12W32-NEXT:    s_cbranch_execz .LBB1_2
 ; GFX12W32-NEXT:  ; %bb.1:
 ; GFX12W32-NEXT:    s_load_b128 s[8:11], s[4:5], 0x34
-; GFX12W32-NEXT:    s_wait_alu 0xfffe
 ; GFX12W32-NEXT:    s_bcnt1_i32_b32 s2, s2
 ; GFX12W32-NEXT:    s_wait_kmcnt 0x0
 ; GFX12W32-NEXT:    s_wait_alu 0xfffe
@@ -883,7 +880,6 @@ define amdgpu_kernel void @add_i32_varying_vdata(ptr addrspace(1) %out, ptr addr
 ; GFX12W64-NEXT:    s_cbranch_execz .LBB2_4
 ; GFX12W64-NEXT:  ; %bb.3:
 ; GFX12W64-NEXT:    s_load_b128 s[8:11], s[4:5], 0x34
-; GFX12W64-NEXT:    s_wait_alu 0xfffe
 ; GFX12W64-NEXT:    v_mov_b32_e32 v1, s2
 ; GFX12W64-NEXT:    s_wait_kmcnt 0x0
 ; GFX12W64-NEXT:    buffer_atomic_add_u32 v1, off, s[8:11], null th:TH_ATOMIC_RETURN
@@ -893,7 +889,7 @@ define amdgpu_kernel void @add_i32_varying_vdata(ptr addrspace(1) %out, ptr addr
 ; GFX12W64-NEXT:    s_wait_loadcnt 0x0
 ; GFX12W64-NEXT:    v_readfirstlane_b32 s2, v1
 ; GFX12W64-NEXT:    v_mov_b32_e32 v1, 0
-; GFX12W64-NEXT:    s_wait_alu 0xfffe
+; GFX12W64-NEXT:    s_wait_alu 0xf1ff
 ; GFX12W64-NEXT:    s_delay_alu instid0(VALU_DEP_2)
 ; GFX12W64-NEXT:    v_add_nc_u32_e32 v0, s2, v0
 ; GFX12W64-NEXT:    s_wait_kmcnt 0x0
@@ -908,7 +904,6 @@ define amdgpu_kernel void @add_i32_varying_vdata(ptr addrspace(1) %out, ptr addr
 ; GFX12W32-NEXT:    ; implicit-def: $vgpr0
 ; GFX12W32-NEXT:  .LBB2_1: ; %ComputeLoop
 ; GFX12W32-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX12W32-NEXT:    s_wait_alu 0xfffe
 ; GFX12W32-NEXT:    s_ctz_i32_b32 s2, s1
 ; GFX12W32-NEXT:    s_wait_alu 0xfffe
 ; GFX12W32-NEXT:    v_readlane_b32 s3, v1, s2
@@ -940,6 +935,7 @@ define amdgpu_kernel void @add_i32_varying_vdata(ptr addrspace(1) %out, ptr addr
 ; GFX12W32-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
 ; GFX12W32-NEXT:    s_wait_loadcnt 0x0
 ; GFX12W32-NEXT:    v_readfirstlane_b32 s2, v1
+; GFX12W32-NEXT:    s_wait_alu 0xf1ff
 ; GFX12W32-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12W32-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_add_nc_u32 v0, s2, v0
 ; GFX12W32-NEXT:    s_wait_kmcnt 0x0
@@ -1292,7 +1288,6 @@ define amdgpu_kernel void @struct_add_i32_varying_vdata(ptr addrspace(1) %out, p
 ; GFX12W64-NEXT:    s_clause 0x1
 ; GFX12W64-NEXT:    s_load_b32 s3, s[4:5], 0x44
 ; GFX12W64-NEXT:    s_load_b128 s[8:11], s[4:5], 0x34
-; GFX12W64-NEXT:    s_wait_alu 0xfffe
 ; GFX12W64-NEXT:    v_mov_b32_e32 v1, s2
 ; GFX12W64-NEXT:    s_wait_kmcnt 0x0
 ; GFX12W64-NEXT:    v_mov_b32_e32 v2, s3
@@ -1303,7 +1298,7 @@ define amdgpu_kernel void @struct_add_i32_varying_vdata(ptr addrspace(1) %out, p
 ; GFX12W64-NEXT:    s_wait_loadcnt 0x0
 ; GFX12W64-NEXT:    v_readfirstlane_b32 s2, v1
 ; GFX12W64-NEXT:    v_mov_b32_e32 v1, 0
-; GFX12W64-NEXT:    s_wait_alu 0xfffe
+; GFX12W64-NEXT:    s_wait_alu 0xf1ff
 ; GFX12W64-NEXT:    s_delay_alu instid0(VALU_DEP_2)
 ; GFX12W64-NEXT:    v_add_nc_u32_e32 v0, s2, v0
 ; GFX12W64-NEXT:    s_wait_kmcnt 0x0
@@ -1318,7 +1313,6 @@ define amdgpu_kernel void @struct_add_i32_varying_vdata(ptr addrspace(1) %out, p
 ; GFX12W32-NEXT:    ; implicit-def: $vgpr0
 ; GFX12W32-NEXT:  .LBB3_1: ; %ComputeLoop
 ; GFX12W32-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX12W32-NEXT:    s_wait_alu 0xfffe
 ; GFX12W32-NEXT:    s_ctz_i32_b32 s2, s1
 ; GFX12W32-NEXT:    s_wait_alu 0xfffe
 ; GFX12W32-NEXT:    v_readlane_b32 s3, v1, s2
@@ -1352,6 +1346,7 @@ define amdgpu_kernel void @struct_add_i32_varying_vdata(ptr addrspace(1) %out, p
 ; GFX12W32-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
 ; GFX12W32-NEXT:    s_wait_loadcnt 0x0
 ; GFX12W32-NEXT:    v_readfirstlane_b32 s2, v1
+; GFX12W32-NEXT:    s_wait_alu 0xf1ff
 ; GFX12W32-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12W32-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_add_nc_u32 v0, s2, v0
 ; GFX12W32-NEXT:    s_wait_kmcnt 0x0
@@ -1686,7 +1681,6 @@ define amdgpu_kernel void @sub_i32_constant(ptr addrspace(1) %out, ptr addrspace
 ; GFX12W64-NEXT:    s_mov_b64 s[0:1], exec
 ; GFX12W64-NEXT:    v_mbcnt_lo_u32_b32 v0, s2, 0
 ; GFX12W64-NEXT:    ; implicit-def: $vgpr1
-; GFX12W64-NEXT:    s_wait_alu 0xfffe
 ; GFX12W64-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12W64-NEXT:    v_mbcnt_hi_u32_b32 v0, s3, v0
 ; GFX12W64-NEXT:    v_cmpx_eq_u32_e32 0, v0
@@ -1707,6 +1701,7 @@ define amdgpu_kernel void @sub_i32_constant(ptr addrspace(1) %out, ptr addrspace
 ; GFX12W64-NEXT:    v_readfirstlane_b32 s2, v1
 ; GFX12W64-NEXT:    v_mul_u32_u24_e32 v0, 5, v0
 ; GFX12W64-NEXT:    v_mov_b32_e32 v1, 0
+; GFX12W64-NEXT:    s_wait_alu 0xf1ff
 ; GFX12W64-NEXT:    s_delay_alu instid0(VALU_DEP_2)
 ; GFX12W64-NEXT:    v_sub_nc_u32_e32 v0, s2, v0
 ; GFX12W64-NEXT:    s_wait_kmcnt 0x0
@@ -1724,7 +1719,6 @@ define amdgpu_kernel void @sub_i32_constant(ptr addrspace(1) %out, ptr addrspace
 ; GFX12W32-NEXT:    s_cbranch_execz .LBB5_2
 ; GFX12W32-NEXT:  ; %bb.1:
 ; GFX12W32-NEXT:    s_load_b128 s[8:11], s[4:5], 0x34
-; GFX12W32-NEXT:    s_wait_alu 0xfffe
 ; GFX12W32-NEXT:    s_bcnt1_i32_b32 s1, s1
 ; GFX12W32-NEXT:    s_wait_alu 0xfffe
 ; GFX12W32-NEXT:    s_mul_i32 s1, s1, 5
@@ -1733,7 +1727,6 @@ define amdgpu_kernel void @sub_i32_constant(ptr addrspace(1) %out, ptr addrspace
 ; GFX12W32-NEXT:    s_wait_kmcnt 0x0
 ; GFX12W32-NEXT:    buffer_atomic_sub_u32 v1, off, s[8:11], null th:TH_ATOMIC_RETURN
 ; GFX12W32-NEXT:  .LBB5_2:
-; GFX12W32-NEXT:    s_wait_alu 0xfffe
 ; GFX12W32-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX12W32-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
 ; GFX12W32-NEXT:    s_wait_loadcnt 0x0
@@ -1970,7 +1963,6 @@ define amdgpu_kernel void @sub_i32_uniform(ptr addrspace(1) %out, ptr addrspace(
 ; GFX12W64-NEXT:    s_mov_b64 s[0:1], exec
 ; GFX12W64-NEXT:    v_mbcnt_lo_u32_b32 v0, s2, 0
 ; GFX12W64-NEXT:    ; implicit-def: $vgpr1
-; GFX12W64-NEXT:    s_wait_alu 0xfffe
 ; GFX12W64-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12W64-NEXT:    v_mbcnt_hi_u32_b32 v0, s3, v0
 ; GFX12W64-NEXT:    v_cmpx_eq_u32_e32 0, v0
@@ -1992,6 +1984,7 @@ define amdgpu_kernel void @sub_i32_uniform(ptr addrspace(1) %out, ptr addrspace(
 ; GFX12W64-NEXT:    s_wait_loadcnt 0x0
 ; GFX12W64-NEXT:    v_readfirstlane_b32 s2, v1
 ; GFX12W64-NEXT:    v_mov_b32_e32 v1, 0
+; GFX12W64-NEXT:    s_wait_alu 0xf1ff
 ; GFX12W64-NEXT:    s_delay_alu instid0(VALU_DEP_2)
 ; GFX12W64-NEXT:    v_sub_nc_u32_e32 v0, s2, v0
 ; GFX12W64-NEXT:    global_store_b32 v1, v0, s[0:1]
@@ -2009,7 +2002,6 @@ define amdgpu_kernel void @sub_i32_uniform(ptr addrspace(1) %out, ptr addrspace(
 ; GFX12W32-NEXT:    s_cbranch_execz .LBB6_2
 ; GFX12W32-NEXT:  ; %bb.1:
 ; GFX12W32-NEXT:    s_load_b128 s[8:11], s[4:5], 0x34
-; GFX12W32-NEXT:    s_wait_alu 0xfffe
 ; GFX12W32-NEXT:    s_bcnt1_i32_b32 s2, s2
 ; GFX12W32-NEXT:    s_wait_kmcnt 0x0
 ; GFX12W32-NEXT:    s_wait_alu 0xfffe
@@ -2025,6 +2017,7 @@ define amdgpu_kernel void @sub_i32_uniform(ptr addrspace(1) %out, ptr addrspace(
 ; GFX12W32-NEXT:    s_wait_loadcnt 0x0
 ; GFX12W32-NEXT:    v_readfirstlane_b32 s0, v1
 ; GFX12W32-NEXT:    v_mov_b32_e32 v1, 0
+; GFX12W32-NEXT:    s_wait_alu 0xf1ff
 ; GFX12W32-NEXT:    s_delay_alu instid0(VALU_DEP_2)
 ; GFX12W32-NEXT:    v_sub_nc_u32_e32 v0, s0, v0
 ; GFX12W32-NEXT:    global_store_b32 v1, v0, s[2:3]
@@ -2357,7 +2350,6 @@ define amdgpu_kernel void @sub_i32_varying_vdata(ptr addrspace(1) %out, ptr addr
 ; GFX12W64-NEXT:    s_cbranch_execz .LBB7_4
 ; GFX12W64-NEXT:  ; %bb.3:
 ; GFX12W64-NEXT:    s_load_b128 s[8:11], s[4:5], 0x34
-; GFX12W64-NEXT:    s_wait_alu 0xfffe
 ; GFX12W64-NEXT:    v_mov_b32_e32 v1, s2
 ; GFX12W64-NEXT:    s_wait_kmcnt 0x0
 ; GFX12W64-NEXT:    buffer_atomic_sub_u32 v1, off, s[8:11], null th:TH_ATOMIC_RETURN
@@ -2367,7 +2359,7 @@ define amdgpu_kernel void @sub_i32_varying_vdata(ptr addrspace(1) %out, ptr addr
 ; GFX12W64-NEXT:    s_wait_loadcnt 0x0
 ; GFX12W64-NEXT:    v_readfirstlane_b32 s2, v1
 ; GFX12W64-NEXT:    v_mov_b32_e32 v1, 0
-; GFX12W64-NEXT:    s_wait_alu 0xfffe
+; GFX12W64-NEXT:    s_wait_alu 0xf1ff
 ; GFX12W64-NEXT:    s_delay_alu instid0(VALU_DEP_2)
 ; GFX12W64-NEXT:    v_sub_nc_u32_e32 v0, s2, v0
 ; GFX12W64-NEXT:    s_wait_kmcnt 0x0
@@ -2382,7 +2374,6 @@ define amdgpu_kernel void @sub_i32_varying_vdata(ptr addrspace(1) %out, ptr addr
 ; GFX12W32-NEXT:    ; implicit-def: $vgpr0
 ; GFX12W32-NEXT:  .LBB7_1: ; %ComputeLoop
 ; GFX12W32-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX12W32-NEXT:    s_wait_alu 0xfffe
 ; GFX12W32-NEXT:    s_ctz_i32_b32 s2, s1
 ; GFX12W32-NEXT:    s_wait_alu 0xfffe
 ; GFX12W32-NEXT:    v_readlane_b32 s3, v1, s2
@@ -2415,6 +2406,7 @@ define amdgpu_kernel void @sub_i32_varying_vdata(ptr addrspace(1) %out, ptr addr
 ; GFX12W32-NEXT:    s_wait_loadcnt 0x0
 ; GFX12W32-NEXT:    v_readfirstlane_b32 s2, v1
 ; GFX12W32-NEXT:    v_mov_b32_e32 v1, 0
+; GFX12W32-NEXT:    s_wait_alu 0xf1ff
 ; GFX12W32-NEXT:    s_delay_alu instid0(VALU_DEP_2)
 ; GFX12W32-NEXT:    v_sub_nc_u32_e32 v0, s2, v0
 ; GFX12W32-NEXT:    s_wait_kmcnt 0x0
diff --git a/llvm/test/CodeGen/AMDGPU/atomic_optimizations_global_pointer.ll b/llvm/test/CodeGen/AMDGPU/atomic_optimizations_global_pointer.ll
index 9577230c6c52e..8bb8ecb079a34 100644
--- a/llvm/test/CodeGen/AMDGPU/atomic_optimizations_global_pointer.ll
+++ b/llvm/test/CodeGen/AMDGPU/atomic_optimizations_global_pointer.ll
@@ -262,7 +262,6 @@ define amdgpu_kernel void @add_i32_constant(ptr addrspace(1) %out, ptr addrspace
 ; GFX1264-NEXT:    s_mov_b64 s[4:5], exec
 ; GFX1264-NEXT:    v_mbcnt_lo_u32_b32 v0, s6, 0
 ; GFX1264-NEXT:    ; implicit-def: $vgpr1
-; GFX1264-NEXT:    s_wait_alu 0xfffe
 ; GFX1264-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX1264-NEXT:    v_mbcnt_hi_u32_b32 v0, s7, v0
 ; GFX1264-NEXT:    v_cmpx_eq_u32_e32 0, v0
@@ -586,7 +585,6 @@ define amdgpu_kernel void @add_i32_uniform(ptr addrspace(1) %out, ptr addrspace(
 ; GFX1264-NEXT:    s_mov_b64 s[4:5], exec
 ; GFX1264-NEXT:    v_mbcnt_lo_u32_b32 v0, s6, 0
 ; GFX1264-NEXT:    ; implicit-def: $vgpr1
-; GFX1264-NEXT:    s_wait_alu 0xfffe
 ; GFX1264-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX1264-NEXT:    v_mbcnt_hi_u32_b32 v0, s7, v0
 ; GFX1264-NEXT:    v_cmpx_eq_u32_e32 0, v0
@@ -629,7 +627,6 @@ define amdgpu_kernel void @add_i32_uniform(ptr addrspace(1) %out, ptr addrspace(
 ; GFX1232-NEXT:    v_cmpx_eq_u32_e32 0, v0
 ; GFX1232-NEXT:    s_cbranch_execz .LBB1_2
 ; GFX1232-NEXT:  ; %bb.1:
-; GFX1232-NEXT:    s_wait_alu 0xfffe
 ; GFX1232-NEXT:    s_bcnt1_i32_b32 s6, s6
 ; GFX1232-NEXT:    s_mov_b32 s11, 0x31016000
 ; GFX1232-NEXT:    s_wait_kmcnt 0x0
@@ -1018,7 +1015,6 @@ define amdgpu_kernel void @add_i32_varying(ptr addrspace(1) %out, ptr addrspace(
 ; GFX1264_ITERATIVE-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
 ; GFX1264_ITERATIVE-NEXT:    s_cbranch_execz .LBB2_4
 ; GFX1264_ITERATIVE-NEXT:  ; %bb.3:
-; GFX1264_ITERATIVE-NEXT:    s_wait_alu 0xfffe
 ; GFX1264_ITERATIVE-NEXT:    v_mov_b32_e32 v1, s6
 ; GFX1264_ITERATIVE-NEXT:    s_mov_b32 s11, 0x31016000
 ; GFX1264_ITERATIVE-NEXT:    s_mov_b32 s10, -1
@@ -1047,7 +1043,6 @@ define amdgpu_kernel void @add_i32_varying(ptr addrspace(1) %out, ptr addrspace(
 ; GFX1232_ITERATIVE-NEXT:    ; implicit-def: $vgpr0
 ; GFX1232_ITERATIVE-NEXT:  .LBB2_1: ; %ComputeLoop
 ; GFX1232_ITERATIVE-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX1232_ITERATIVE-NEXT:    s_wait_alu 0xfffe
 ; GFX1232_ITERATIVE-NEXT:    s_ctz_i32_b32 s1, s0
 ; GFX1232_ITERATIVE-NEXT:    s_wait_alu 0xfffe
 ; GFX1232_ITERATIVE-NEXT:    v_readlane_b32 s2, v1, s1
@@ -1291,9 +1286,9 @@ define amdgpu_kernel void @add_i32_varying(ptr addrspace(1) %out, ptr addrspace(
 ; GFX1032_DPP-NEXT:    s_or_saveexec_b32 s4, -1
 ; GFX1032_DPP-NEXT:    v_writelane_b32 v3, s5, 16
 ; GFX1032_DPP-NEXT:    s_mov_b32 exec_lo, s4
+; GFX1032_DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
 ; GFX1032_DPP-NEXT:    s_mov_b32 s4, s6
 ; GFX1032_DPP-NEXT:    s_mov_b32 s6, -1
-; GFX1032_DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
 ; GFX1032_DPP-NEXT:    ; implicit-def: $vgpr0
 ; GFX1032_DPP-NEXT:    s_and_saveexec_b32 s8, vcc_lo
 ; GFX1032_DPP-NEXT:    s_cbranch_execz .LBB2_2
@@ -1417,9 +1412,10 @@ define amdgpu_kernel void @add_i32_varying(ptr addrspace(1) %out, ptr addrspace(
 ; GFX1132_DPP-NEXT:    s_or_saveexec_b32 s4, -1
 ; GFX1132_DPP-NEXT:    v_writelane_b32 v3, s5, 16
 ; GFX1132_DPP-NEXT:    s_mov_b32 exec_lo, s4
+; GFX1132_DPP-NEXT:    s_delay_alu instid0(VALU_DEP_2)
+; GFX1132_DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
 ; GFX1132_DPP-NEXT:    s_mov_b32 s4, s6
 ; GFX1132_DPP-NEXT:    s_mov_b32 s6, -1
-; GFX1132_DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
 ; GFX1132_DPP-NEXT:    ; implicit-def: $vgpr0
 ; GFX1132_DPP-NEXT:    s_and_saveexec_b32 s8, vcc_lo
 ; GFX1132_DPP-NEXT:    s_cbranch_execz .LBB2_2
@@ -1509,6 +1505,7 @@ define amdgpu_kernel void @add_i32_varying(ptr addrspace(1) %out, ptr addrspace(
 ; GFX1264_DPP-NEXT:    v_readfirstlane_b32 s2, v0
 ; GFX1264_DPP-NEXT:    v_mov_b32_e32 v0, v3
 ; GFX1264_DPP-NEXT:    s_mov_b32 s3, 0x31016000
+; GFX1264_DPP-NEXT:    s_wait_alu 0xf1ff
 ; GFX1264_DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX1264_DPP-NEXT:    v_add_nc_u32_e32 v0, s2, v0
 ; GFX1264_DPP-NEXT:    s_mov_b32 s2, s6
@@ -1543,11 +1540,11 @@ define amdgpu_kernel void @add_i32_varying(ptr addrspace(1) %out, ptr addrspace(
 ; GFX1232_DPP-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0
 ; GFX1232_DPP-NEXT:    s_or_saveexec_b32 s4, -1
 ; GFX1232_DPP-NEXT:    v_writelane_b32 v3, s5, 16
-; GFX1232_DPP-NEXT:    s_wait_alu 0xfffe
 ; GFX1232_DPP-NEXT:    s_mov_b32 exec_lo, s4
+; GFX1232_DPP-NEXT:    s_delay_alu instid0(VALU_DEP_2)
+; GFX1232_DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
 ; GFX1232_DPP-NEXT:    s_mov_b32 s4, s6
 ; GFX1232_DPP-NEXT:    s_mov_b32 s6, -1
-; GFX1232_DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
 ; GFX1232_DPP-NEXT:    ; implicit-def: $vgpr0
 ; GFX1232_DPP-NEXT:    s_and_saveexec_b32 s8, vcc_lo
 ; GFX1232_DPP-NEXT:    s_cbranch_execz .LBB2_2
@@ -1843,7 +1840,6 @@ define amdgpu_kernel void @add_i64_constant(ptr addrspace(1) %out, ptr addrspace
 ; GFX1264-NEXT:    s_mov_b32 s9, 0
 ; GFX1264-NEXT:    v_mbcnt_lo_u32_b32 v0, s6, 0
 ; GFX1264-NEXT:    s_mov_b64 s[4:5], exec
-; GFX1264-NEXT:    s_wait_alu 0xfffe
 ; GFX1264-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX1264-NEXT:    v_mbcnt_hi_u32_b32 v2, s7, v0
 ; GFX1264-NEXT:    ; implicit-def: $vgpr0_vgpr1
@@ -1887,7 +1883,6 @@ define amdgpu_kernel void @add_i64_constant(ptr addrspace(1) %out, ptr addrspace
 ; GFX1232-NEXT:    v_cmpx_eq_u32_e32 0, v2
 ; GFX1232-NEXT:    s_cbranch_execz .LBB3_2
 ; GFX1232-NEXT:  ; %bb.1:
-; GFX1232-NEXT:    s_wait_alu 0xfffe
 ; GFX1232-NEXT:    s_bcnt1_i32_b32 s4, s7
 ; GFX1232-NEXT:    s_mov_b32 s11, 0x31016000
 ; GFX1232-NEXT:    s_mul_u64 s[4:5], s[4:5], 5
@@ -2223,32 +2218,31 @@ define amdgpu_kernel void @add_i64_uniform(ptr addrspace(1) %out, ptr addrspace(
 ; GFX1264-NEXT:    s_clause 0x1
 ; GFX1264-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
 ; GFX1264-NEXT:    s_load_b64 s[4:5], s[4:5], 0x34
-; GFX1264-NEXT:    s_mov_b64 s[8:9], exec
-; GFX1264-NEXT:    s_mov_b32 s11, 0
-; GFX1264-NEXT:    v_mbcnt_lo_u32_b32 v0, s8, 0
 ; GFX1264-NEXT:    s_mov_b64 s[6:7], exec
-; GFX1264-NEXT:    s_wait_alu 0xfffe
+; GFX1264-NEXT:    s_mov_b32 s11, 0
+; GFX1264-NEXT:    v_mbcnt_lo_u32_b32 v0, s6, 0
+; GFX1264-NEXT:    s_mov_b64 s[8:9], exec
 ; GFX1264-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1264-NEXT:    v_mbcnt_hi_u32_b32 v2, s9, v0
+; GFX1264-NEXT:    v_mbcnt_hi_u32_b32 v2, s7, v0
 ; GFX1264-NEXT:    ; implicit-def: $vgpr0_vgpr1
 ; GFX1264-NEXT:    v_cmpx_eq_u32_e32 0, v2
 ; GFX1264-NEXT:    s_cbranch_execz .LBB4_2
 ; GFX1264-NEXT:  ; %bb.1:
-; GFX1264-NEXT:    s_bcnt1_i32_b64 s10, s[8:9]
+; GFX1264-NEXT:    s_bcnt1_i32_b64 s10, s[6:7]
+; GFX1264-NEXT:    s_mov_b32 s15, 0x31016000
 ; GFX1264-NEXT:    s_wait_kmcnt 0x0
-; GFX1264-NEXT:    s_mul_u64 s[8:9], s[4:5], s[10:11]
-; GFX1264-NEXT:    s_mov_b32 s11, 0x31016000
+; GFX1264-NEXT:    s_mul_u64 s[6:7], s[4:5], s[10:11]
+; GFX1264-NEXT:    s_mov_b32 s14, -1
 ; GFX1264-NEXT:    s_wait_alu 0xfffe
-; GFX1264-NEXT:    v_mov_b32_e32 v0, s8
-; GFX1264-NEXT:    v_mov_b32_e32 v1, s9
-; GFX1264-NEXT:    s_mov_b32 s10, -1
-; GFX1264-NEXT:    s_mov_b32 s8, s2
-; GFX1264-NEXT:    s_mov_b32 s9, s3
-; GFX1264-NEXT:    buffer_atomic_add_u64 v[0:1], off, s[8:11], null th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX1264-NEXT:    v_mov_b32_e32 v0, s6
+; GFX1264-NEXT:    v_mov_b32_e32 v1, s7
+; GFX1264-NEXT:    s_mov_b32 s12, s2
+; GFX1264-NEXT:    s_mov_b32 s13, s3
+; GFX1264-NEXT:    buffer_atomic_add_u64 v[0:1], off, s[12:15], null th:TH_ATOMIC_RETURN scope:SCOPE_DEV
 ; GFX1264-NEXT:    s_wait_loadcnt 0x0
 ; GFX1264-NEXT:    global_inv scope:SCOPE_DEV
 ; GFX1264-NEXT:  .LBB4_2:
-; GFX1264-NEXT:    s_or_b64 exec, exec, s[6:7]
+; GFX1264-NEXT:    s_or_b64 exec, exec, s[8:9]
 ; GFX1264-NEXT:    s_wait_kmcnt 0x0
 ; GFX1264-NEXT:    v_readfirstlane_b32 s3, v1
 ; GFX1264-NEXT:    v_readfirstlane_b32 s2, v0
@@ -2274,7 +2268,6 @@ define amdgpu_kernel void @add_i64_uniform(ptr addrspace(1) %out, ptr addrspace(
 ; GFX1232-NEXT:    v_cmpx_eq_u32_e32 0, v2
 ; GFX1232-NEXT:    s_cbranch_execz .LBB4_2
 ; GFX1232-NEXT:  ; %bb.1:
-; GFX1232-NEXT:    s_wait_alu 0xfffe
 ; GFX1232-NEXT:    s_bcnt1_i32_b32 s6, s6
 ; GFX1232-NEXT:    s_mov_b32 s15, 0x31016000
 ; GFX1232-NEXT:    s_wait_kmcnt 0x0
@@ -2719,7 +2712,6 @@ define amdgpu_kernel void @add_i64_varying(ptr addrspace(1) %out, ptr addrspace(
 ; GFX1264_ITERATIVE-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
 ; GFX1264_ITERATIVE-NEXT:    s_cbranch_execz .LBB5_4
 ; GFX1264_ITERATIVE-NEXT:  ; %bb.3:
-; GFX1264_ITERATIVE-NEXT:    s_wait_alu 0xfffe
 ; GFX1264_ITERATIVE-NEXT:    v_mov_b32_e32 v2, s6
 ; GFX1264_ITERATIVE-NEXT:    v_mov_b32_e32 v3, s7
 ; GFX1264_ITERATIVE-NEXT:    s_mov_b32 s11, 0x31016000
@@ -2751,7 +2743,6 @@ define amdgpu_kernel void @add_i64_varying(ptr addrspace(1) %out, ptr addrspace(
 ; GFX1232_ITERATIVE-NEXT:    ; implicit-def: $vgpr0_vgpr1
 ; GFX1232_ITERATIVE-NEXT:  .LBB5_1: ; %ComputeLoop
 ; GFX1232_ITERATIVE-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX1232_ITERATIVE-NEXT:    s_wait_alu 0xfffe
 ; GFX1232_ITERATIVE-NEXT:    s_ctz_i32_b32 s1, s0
 ; GFX1232_ITERATIVE-NEXT:    s_wait_alu 0xfffe
 ; GFX1232_ITERATIVE-NEXT:    v_readlane_b32 s3, v3, s1
@@ -3140,8 +3131,8 @@ define amdgpu_kernel void @add_i64_varying(ptr addrspace(1) %out, ptr addrspace(
 ; GFX1032_DPP-NEXT:    v_writelane_b32 v8, s8, 16
 ; GFX1032_DPP-NEXT:    v_writelane_b32 v7, s7, 16
 ; GFX1032_DPP-NEXT:    s_mov_b32 exec_lo, s6
-; GFX1032_DPP-NEXT:    s_mov_b32 s6, -1
 ; GFX1032_DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
+; GFX1032_DPP-NEXT:    s_mov_b32 s6, -1
 ; GFX1032_DPP-NEXT:    ; implicit-def: $vgpr9_vgpr10
 ; GFX1032_DPP-NEXT:    s_and_saveexec_b32 s8, vcc_lo
 ; GFX1032_DPP-NEXT:    s_cbranch_execz .LBB5_2
@@ -3378,20 +3369,22 @@ define amdgpu_kernel void @add_i64_varying(ptr addrspace(1) %out, ptr addrspace(
 ; GFX1264_DPP-NEXT:    v_add_co_ci_u32_e32 v1, vcc, v1, v2, vcc
 ; GFX1264_DPP-NEXT:    v_add_co_u32_e64_dpp v3, vcc, v3, v3 row_shr:2 row_mask:0xf bank_mask:0xf bound_ctrl:1
 ; GFX1264_DPP-NEXT:    v_mov_b32_e32 v2, 0
-; GFX1264_DPP-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1264_DPP-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
 ; GFX1264_DPP-NEXT:    v_mov_b32_dpp v4, v1 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1264_DPP-NEXT:    s_wait_alu 0xfffd
 ; GFX1264_DPP-NEXT:    v_add_co_ci_u32_e32 v1, vcc, v1, v4, vcc
-; GFX1264_DPP-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3)
 ; GFX1264_DPP-NEXT:    v_add_co_u32_e64_dpp v3, vcc, v3, v3 row_shr:4 row_mask:0xf bank_mask:0xf bound_ctrl:1
 ; GFX1264_DPP-NEXT:    v_mov_b32_e32 v4, 0
+; GFX1264_DPP-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
 ; GFX1264_DPP-NEXT:    v_mov_b32_dpp v2, v1 row_shr:4 row_mask:0xf bank_mask:0xf
-; GFX1264_DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX1264_DPP-NEXT:    s_wait_alu 0xfffd
 ; GFX1264_DPP-NEXT:    v_add_co_ci_u32_e32 v1, vcc, v1, v2, vcc
 ; GFX1264_DPP-NEXT:    v_add_co_u32_e64_dpp v2, vcc, v3, v3 row_shr:8 row_mask:0xf bank_mask:0xf bound_ctrl:1
 ; GFX1264_DPP-NEXT:    v_mov_b32_e32 v3, 0
 ; GFX1264_DPP-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
 ; GFX1264_DPP-NEXT:    v_mov_b32_dpp v4, v1 row_shr:8 row_mask:0xf bank_mask:0xf
 ; GFX1264_DPP-NEXT:    v_permlanex16_b32 v5, v2, -1, -1
+; GFX1264_DPP-NEXT:    s_wait_alu 0xfffd
 ; GFX1264_DPP-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
 ; GFX1264_DPP-NEXT:    v_add_co_ci_u32_e32 v1, vcc, v1, v4, vcc
 ; GFX1264_DPP-NEXT:    v_add_co_u32_e64_dpp v2, vcc, v5, v2 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf
@@ -3399,6 +3392,7 @@ define amdgpu_kernel void @add_i64_varying(ptr addrspace(1) %out, ptr addrspace(
 ; GFX1264_DPP-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX1264_DPP-NEXT:    v_permlanex16_b32 v4, v1, -1, -1
 ; GFX1264_DPP-NEXT:    v_mov_b32_dpp v3, v4 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf
+; GFX1264_DPP-NEXT:    s_wait_alu 0xfffd
 ; GFX1264_DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX1264_DPP-NEXT:    v_add_co_ci_u32_e32 v1, vcc, v1, v3, vcc
 ; GFX1264_DPP-NEXT:    v_mov_b32_e32 v3, 0
@@ -3407,9 +3401,11 @@ define amdgpu_kernel void @add_i64_varying(ptr addrspace(1) %out, ptr addrspace(
 ; GFX1264_DPP-NEXT:    v_mov_b32_e32 v4, s2
 ; GFX1264_DPP-NEXT:    v_readlane_b32 s2, v2, 31
 ; GFX1264_DPP-NEXT:    v_mov_b32_dpp v3, v4 quad_perm:[0,1,2,3] row_mask:0xc bank_mask:0xf
-; GFX1264_DPP-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX1264_DPP-NEXT:    s_wait_alu 0xf1ff
+; GFX1264_DPP-NEXT:    s_delay_alu instid0(VALU_DEP_2)
 ; GFX1264_DPP-NEXT:    v_add_co_u32_e64_dpp v2, vcc, v2, s2 quad_perm:[0,1,2,3] row_mask:0xc bank_mask:0xf
 ; GFX1264_DPP-NEXT:    v_mov_b32_e32 v4, 0
+; GFX1264_DPP-NEXT:    s_wait_alu 0xfffd
 ; GFX1264_DPP-NEXT:    v_add_co_ci_u32_e32 v1, vcc, v1, v3, vcc
 ; GFX1264_DPP-NEXT:    s_mov_b64 exec, s[0:1]
 ; GFX1264_DPP-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
@@ -3461,9 +3457,11 @@ define amdgpu_kernel void @add_i64_varying(ptr addrspace(1) %out, ptr addrspace(
 ; GFX1264_DPP-NEXT:    v_mov_b32_e32 v8, v4
 ; GFX1264_DPP-NEXT:    v_mov_b32_e32 v9, v5
 ; GFX1264_DPP-NEXT:    v_readfirstlane_b32 s3, v7
-; GFX1264_DPP-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1264_DPP-NEXT:    s_wait_alu 0xf1ff
+; GFX1264_DPP-NEXT:    s_delay_alu instid0(VALU_DEP_3)
 ; GFX1264_DPP-NEXT:    v_add_co_u32 v6, vcc, s2, v8
 ; GFX1264_DPP-NEXT:    s_mov_b32 s2, s6
+; GFX1264_DPP-NEXT:    s_wait_alu 0xfffd
 ; GFX1264_DPP-NEXT:    v_add_co_ci_u32_e32 v7, vcc, s3, v9, vcc
 ; GFX1264_DPP-NEXT:    s_mov_b32 s3, 0x31016000
 ; GFX1264_DPP-NEXT:    buffer_store_b64 v[6:7], off, s[0:3], null
@@ -3486,20 +3484,22 @@ define amdgpu_kernel void @add_i64_varying(ptr addrspace(1) %out, ptr addrspace(
 ; GFX1232_DPP-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, v1, v2, vcc_lo
 ; GFX1232_DPP-NEXT:    v_add_co_u32_e64_dpp v3, vcc_lo, v3, v3 row_shr:2 row_mask:0xf bank_mask:0xf bound_ctrl:1
 ; GFX1232_DPP-NEXT:    v_mov_b32_e32 v2, 0
-; GFX1232_DPP-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1232_DPP-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
 ; GFX1232_DPP-NEXT:    v_mov_b32_dpp v4, v1 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1232_DPP-NEXT:    s_wait_alu 0xfffd
 ; GFX1232_DPP-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, v1, v4, vcc_lo
-; GFX1232_DPP-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3)
 ; GFX1232_DPP-NEXT:    v_add_co_u32_e64_dpp v3, vcc_lo, v3, v3 row_shr:4 row_mask:0xf bank_mask:0xf bound_ctrl:1
 ; GFX1232_DPP-NEXT:    v_mov_b32_e32 v4, 0
+; GFX1232_DPP-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
 ; GFX1232_DPP-NEXT:    v_mov_b32_dpp v2, v1 row_shr:4 row_mask:0xf bank_mask:0xf
-; GFX1232_DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX1232_DPP-NEXT:    s_wait_alu 0xfffd
 ; GFX1232_DPP-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, v1, v2, vcc_lo
 ; GFX1232_DPP-NEXT:    v_add_co_u32_e64_dpp v2, vcc_lo, v3, v3 row_shr:8 row_mask:0xf bank_mask:0xf bound_ctrl:1
 ; GFX1232_DPP-NEXT:    v_mov_b32_e32 v3, 0
 ; GFX1232_DPP-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
 ; GFX1232_DPP-NEXT:    v_mov_b32_dpp v4, v1 row_shr:8 row_mask:0xf bank_mask:0xf
 ; GFX1232_DPP-NEXT:    v_permlanex16_b32 v5, v2, -1, -1
+; GFX1232_DPP-NEXT:    s_wait_alu 0xfffd
 ; GFX1232_DPP-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
 ; GFX1232_DPP-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, v1, v4, vcc_lo
 ; GFX1232_DPP-NEXT:    v_add_co_u32_e64_dpp v2, vcc_lo, v5, v2 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf
@@ -3523,7 +3523,6 @@ define amdgpu_kernel void @add_i64_varying(ptr addrspace(1) %out, ptr addrspace(
 ; GFX1232_DPP-NEXT:    s_or_saveexec_b32 s6, -1
 ; GFX1232_DPP-NEXT:    v_writelane_b32 v6, s7, 16
 ; GFX1232_DPP-NEXT:    v_writelane_b32 v7, s8, 16
-; GFX1232_DPP-NEXT:    s_wait_alu 0xfffe
 ; GFX1232_DPP-NEXT:    s_mov_b32 exec_lo, s6
 ; GFX1232_DPP-NEXT:    s_mov_b32 s6, -1
 ; GFX1232_DPP-NEXT:    s_mov_b32 s8, exec_lo
@@ -3547,9 +3546,10 @@ define amdgpu_kernel void @add_i64_varying(ptr addrspace(1) %out, ptr addrspace(
 ; GFX1232_DPP-NEXT:    v_mov_b32_e32 v10, v6
 ; GFX1232_DPP-NEXT:    v_mov_b32_e32 v11, v7
 ; GFX1232_DPP-NEXT:    v_readfirstlane_b32 s3, v9
-; GFX1232_DPP-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1232_DPP-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_2)
 ; GFX1232_DPP-NEXT:    v_add_co_u32 v8, vcc_lo, s2, v10
 ; GFX1232_DPP-NEXT:    s_mov_b32 s2, s6
+; GFX1232_DPP-NEXT:    s_wait_alu 0xfffd
 ; GFX1232_DPP-NEXT:    v_add_co_ci_u32_e32 v9, vcc_lo, s3, v11, vcc_lo
 ; GFX1232_DPP-NEXT:    s_mov_b32 s3, 0x31016000
 ; GFX1232_DPP-NEXT:    buffer_store_b64 v[8:9], off, s[0:3], null
@@ -3809,7 +3809,6 @@ define amdgpu_kernel void @sub_i32_constant(ptr addrspace(1) %out, ptr addrspace
 ; GFX1264-NEXT:    s_mov_b64 s[4:5], exec
 ; GFX1264-NEXT:    v_mbcnt_lo_u32_b32 v0, s6, 0
 ; GFX1264-NEXT:    ; implicit-def: $vgpr1
-; GFX1264-NEXT:    s_wait_alu 0xfffe
 ; GFX1264-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX1264-NEXT:    v_mbcnt_hi_u32_b32 v0, s7, v0
 ; GFX1264-NEXT:    v_cmpx_eq_u32_e32 0, v0
@@ -4139,7 +4138,6 @@ define amdgpu_kernel void @sub_i32_uniform(ptr addrspace(1) %out, ptr addrspace(
 ; GFX1264-NEXT:    s_mov_b64 s[4:5], exec
 ; GFX1264-NEXT:    v_mbcnt_lo_u32_b32 v0, s6, 0
 ; GFX1264-NEXT:    ; implicit-def: $vgpr1
-; GFX1264-NEXT:    s_wait_alu 0xfffe
 ; GFX1264-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX1264-NEXT:    v_mbcnt_hi_u32_b32 v0, s7, v0
 ; GFX1264-NEXT:    v_cmpx_eq_u32_e32 0, v0
@@ -4183,7 +4181,6 @@ define amdgpu_kernel void @sub_i32_uniform(ptr addrspace(1) %out, ptr addrspace(
 ; GFX1232-NEXT:    v_cmpx_eq_u32_e32 0, v0
 ; GFX1232-NEXT:    s_cbranch_execz .LBB7_2
 ; GFX1232-NEXT:  ; %bb.1:
-; GFX1232-NEXT:    s_wait_alu 0xfffe
 ; GFX1232-NEXT:    s_bcnt1_i32_b32 s6, s6
 ; GFX1232-NEXT:    s_mov_b32 s11, 0x31016000
 ; GFX1232-NEXT:    s_wait_kmcnt 0x0
@@ -4573,7 +4570,6 @@ define amdgpu_kernel void @sub_i32_varying(ptr addrspace(1) %out, ptr addrspace(
 ; GFX1264_ITERATIVE-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
 ; GFX1264_ITERATIVE-NEXT:    s_cbranch_execz .LBB8_4
 ; GFX1264_ITERATIVE-NEXT:  ; %bb.3:
-; GFX1264_ITERATIVE-NEXT:    s_wait_alu 0xfffe
 ; GFX1264_ITERATIVE-NEXT:    v_mov_b32_e32 v1, s6
 ; GFX1264_ITERATIVE-NEXT:    s_mov_b32 s11, 0x31016000
 ; GFX1264_ITERATIVE-NEXT:    s_mov_b32 s10, -1
@@ -4602,7 +4598,6 @@ define amdgpu_kernel void @sub_i32_varying(ptr addrspace(1) %out, ptr addrspace(
 ; GFX1232_ITERATIVE-NEXT:    ; implicit-def: $vgpr0
 ; GFX1232_ITERATIVE-NEXT:  .LBB8_1: ; %ComputeLoop
 ; GFX1232_ITERATIVE-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX1232_ITERATIVE-NEXT:    s_wait_alu 0xfffe
 ; GFX1232_ITERATIVE-NEXT:    s_ctz_i32_b32 s1, s0
 ; GFX1232_ITERATIVE-NEXT:    s_wait_alu 0xfffe
 ; GFX1232_ITERATIVE-NEXT:    v_readlane_b32 s2, v1, s1
@@ -4846,9 +4841,9 @@ define amdgpu_kernel void @sub_i32_varying(ptr addrspace(1) %out, ptr addrspace(
 ; GFX1032_DPP-NEXT:    s_or_saveexec_b32 s4, -1
 ; GFX1032_DPP-NEXT:    v_writelane_b32 v3, s5, 16
 ; GFX1032_DPP-NEXT:    s_mov_b32 exec_lo, s4
+; GFX1032_DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
 ; GFX1032_DPP-NEXT:    s_mov_b32 s4, s6
 ; GFX1032_DPP-NEXT:    s_mov_b32 s6, -1
-; GFX1032_DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
 ; GFX1032_DPP-NEXT:    ; implicit-def: $vgpr0
 ; GFX1032_DPP-NEXT:    s_and_saveexec_b32 s8, vcc_lo
 ; GFX1032_DPP-NEXT:    s_cbranch_execz .LBB8_2
@@ -4972,9 +4967,10 @@ define amdgpu_kernel void @sub_i32_varying(ptr addrspace(1) %out, ptr addrspace(
 ; GFX1132_DPP-NEXT:    s_or_saveexec_b32 s4, -1
 ; GFX1132_DPP-NEXT:    v_writelane_b32 v3, s5, 16
 ; GFX1132_DPP-NEXT:    s_mov_b32 exec_lo, s4
+; GFX1132_DPP-NEXT:    s_delay_alu instid0(VALU_DEP_2)
+; GFX1132_DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
 ; GFX1132_DPP-NEXT:    s_mov_b32 s4, s6
 ; GFX1132_DPP-NEXT:    s_mov_b32 s6, -1
-; GFX1132_DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
 ; GFX1132_DPP-NEXT:    ; implicit-def: $vgpr0
 ; GFX1132_DPP-NEXT:    s_and_saveexec_b32 s8, vcc_lo
 ; GFX1132_DPP-NEXT:    s_cbranch_execz .LBB8_2
@@ -5064,6 +5060,7 @@ define amdgpu_kernel void @sub_i32_varying(ptr addrspace(1) %out, ptr addrspace(
 ; GFX1264_DPP-NEXT:    v_readfirstlane_b32 s2, v0
 ; GFX1264_DPP-NEXT:    v_mov_b32_e32 v0, v3
 ; GFX1264_DPP-NEXT:    s_mov_b32 s3, 0x31016000
+; GFX1264_DPP-NEXT:    s_wait_alu 0xf1ff
 ; GFX1264_DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX1264_DPP-NEXT:    v_sub_nc_u32_e32 v0, s2, v0
 ; GFX1264_DPP-NEXT:    s_mov_b32 s2, s6
@@ -5098,11 +5095,11 @@ define amdgpu_kernel void @sub_i32_varying(ptr addrspace(1) %out, ptr addrspace(
 ; GFX1232_DPP-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0
 ; GFX1232_DPP-NEXT:    s_or_saveexec_b32 s4, -1
 ; GFX1232_DPP-NEXT:    v_writelane_b32 v3, s5, 16
-; GFX1232_DPP-NEXT:    s_wait_alu 0xfffe
 ; GFX1232_DPP-NEXT:    s_mov_b32 exec_lo, s4
+; GFX1232_DPP-NEXT:    s_delay_alu instid0(VALU_DEP_2)
+; GFX1232_DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
 ; GFX1232_DPP-NEXT:    s_mov_b32 s4, s6
 ; GFX1232_DPP-NEXT:    s_mov_b32 s6, -1
-; GFX1232_DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
 ; GFX1232_DPP-NEXT:    ; implicit-def: $vgpr0
 ; GFX1232_DPP-NEXT:    s_and_saveexec_b32 s8, vcc_lo
 ; GFX1232_DPP-NEXT:    s_cbranch_execz .LBB8_2
@@ -5412,7 +5409,6 @@ define amdgpu_kernel void @sub_i64_constant(ptr addrspace(1) %out, ptr addrspace
 ; GFX1264-NEXT:    s_mov_b32 s9, 0
 ; GFX1264-NEXT:    v_mbcnt_lo_u32_b32 v0, s6, 0
 ; GFX1264-NEXT:    s_mov_b64 s[4:5], exec
-; GFX1264-NEXT:    s_wait_alu 0xfffe
 ; GFX1264-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX1264-NEXT:    v_mbcnt_hi_u32_b32 v2, s7, v0
 ; GFX1264-NEXT:    ; implicit-def: $vgpr0_vgpr1
@@ -5459,7 +5455,6 @@ define amdgpu_kernel void @sub_i64_constant(ptr addrspace(1) %out, ptr addrspace
 ; GFX1232-NEXT:    v_cmpx_eq_u32_e32 0, v2
 ; GFX1232-NEXT:    s_cbranch_execz .LBB9_2
 ; GFX1232-NEXT:  ; %bb.1:
-; GFX1232-NEXT:    s_wait_alu 0xfffe
 ; GFX1232-NEXT:    s_bcnt1_i32_b32 s4, s7
 ; GFX1232-NEXT:    s_mov_b32 s11, 0x31016000
 ; GFX1232-NEXT:    s_mul_u64 s[4:5], s[4:5], 5
@@ -5609,17 +5604,17 @@ define amdgpu_kernel void @sub_i64_uniform(ptr addrspace(1) %out, ptr addrspace(
 ; GFX9-NEXT:  .LBB10_2:
 ; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NEXT:    v_mad_u64_u32 v[3:4], s[2:3], s6, v2, 0
+; GFX9-NEXT:    v_mad_u64_u32 v[3:4], s[4:5], s6, v2, 0
+; GFX9-NEXT:    v_readfirstlane_b32 s8, v0
+; GFX9-NEXT:    v_readfirstlane_b32 s6, v1
+; GFX9-NEXT:    v_mov_b32_e32 v0, v4
+; GFX9-NEXT:    v_mad_u64_u32 v[0:1], s[4:5], s7, v2, v[0:1]
+; GFX9-NEXT:    v_mov_b32_e32 v2, s6
+; GFX9-NEXT:    v_sub_co_u32_e32 v1, vcc, s8, v3
 ; GFX9-NEXT:    s_mov_b32 s3, 0xf000
 ; GFX9-NEXT:    s_mov_b32 s2, -1
-; GFX9-NEXT:    v_mad_u64_u32 v[4:5], s[4:5], s7, v2, v[4:5]
-; GFX9-NEXT:    v_readfirstlane_b32 s4, v1
-; GFX9-NEXT:    v_readfirstlane_b32 s5, v0
-; GFX9-NEXT:    v_mov_b32_e32 v1, v4
-; GFX9-NEXT:    v_mov_b32_e32 v2, s4
-; GFX9-NEXT:    v_sub_co_u32_e32 v0, vcc, s5, v3
-; GFX9-NEXT:    v_subb_co_u32_e32 v1, vcc, v2, v1, vcc
-; GFX9-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
+; GFX9-NEXT:    v_subb_co_u32_e32 v2, vcc, v2, v0, vcc
+; GFX9-NEXT:    buffer_store_dwordx2 v[1:2], off, s[0:3], 0
 ; GFX9-NEXT:    s_endpgm
 ;
 ; GFX1064-LABEL: sub_i64_uniform:
@@ -5660,10 +5655,9 @@ define amdgpu_kernel void @sub_i64_uniform(ptr addrspace(1) %out, ptr addrspace(
 ; GFX1064-NEXT:    v_readfirstlane_b32 s2, v0
 ; GFX1064-NEXT:    v_readfirstlane_b32 s3, v1
 ; GFX1064-NEXT:    v_sub_co_u32 v0, vcc, s2, v3
-; GFX1064-NEXT:    v_mov_b32_e32 v1, v4
-; GFX1064-NEXT:    s_mov_b32 s2, -1
-; GFX1064-NEXT:    v_sub_co_ci_u32_e32 v1, vcc, s3, v1, vcc
+; GFX1064-NEXT:    v_sub_co_ci_u32_e32 v1, vcc, s3, v4, vcc
 ; GFX1064-NEXT:    s_mov_b32 s3, 0x31016000
+; GFX1064-NEXT:    s_mov_b32 s2, -1
 ; GFX1064-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
 ; GFX1064-NEXT:    s_endpgm
 ;
@@ -5704,10 +5698,9 @@ define amdgpu_kernel void @sub_i64_uniform(ptr addrspace(1) %out, ptr addrspace(
 ; GFX1032-NEXT:    v_mad_u64_u32 v[4:5], s2, s7, v2, v[4:5]
 ; GFX1032-NEXT:    v_readfirstlane_b32 s2, v0
 ; GFX1032-NEXT:    v_sub_co_u32 v0, vcc_lo, s2, v3
-; GFX1032-NEXT:    v_mov_b32_e32 v1, v4
-; GFX1032-NEXT:    s_mov_b32 s2, -1
-; GFX1032-NEXT:    v_sub_co_ci_u32_e32 v1, vcc_lo, s3, v1, vcc_lo
+; GFX1032-NEXT:    v_sub_co_ci_u32_e32 v1, vcc_lo, s3, v4, vcc_lo
 ; GFX1032-NEXT:    s_mov_b32 s3, 0x31016000
+; GFX1032-NEXT:    s_mov_b32 s2, -1
 ; GFX1032-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
 ; GFX1032-NEXT:    s_endpgm
 ;
@@ -5751,9 +5744,8 @@ define amdgpu_kernel void @sub_i64_uniform(ptr addrspace(1) %out, ptr addrspace(
 ; GFX1164-NEXT:    v_mad_u64_u32 v[5:6], null, s5, v2, v[4:5]
 ; GFX1164-NEXT:    v_sub_co_u32 v0, vcc, s2, v3
 ; GFX1164-NEXT:    s_mov_b32 s2, -1
-; GFX1164-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-NEXT:    v_mov_b32_e32 v1, v5
-; GFX1164-NEXT:    v_sub_co_ci_u32_e32 v1, vcc, s3, v1, vcc
+; GFX1164-NEXT:    s_delay_alu instid0(VALU_DEP_2)
+; GFX1164-NEXT:    v_sub_co_ci_u32_e32 v1, vcc, s3, v5, vcc
 ; GFX1164-NEXT:    s_mov_b32 s3, 0x31016000
 ; GFX1164-NEXT:    buffer_store_b64 v[0:1], off, s[0:3], 0
 ; GFX1164-NEXT:    s_endpgm
@@ -5797,9 +5789,8 @@ define amdgpu_kernel void @sub_i64_uniform(ptr addrspace(1) %out, ptr addrspace(
 ; GFX1132-NEXT:    v_mad_u64_u32 v[5:6], null, s5, v2, v[4:5]
 ; GFX1132-NEXT:    v_sub_co_u32 v0, vcc_lo, s2, v3
 ; GFX1132-NEXT:    s_mov_b32 s2, -1
-; GFX1132-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-NEXT:    v_mov_b32_e32 v1, v5
-; GFX1132-NEXT:    v_sub_co_ci_u32_e32 v1, vcc_lo, s3, v1, vcc_lo
+; GFX1132-NEXT:    s_delay_alu instid0(VALU_DEP_2)
+; GFX1132-NEXT:    v_sub_co_ci_u32_e32 v1, vcc_lo, s3, v5, vcc_lo
 ; GFX1132-NEXT:    s_mov_b32 s3, 0x31016000
 ; GFX1132-NEXT:    buffer_store_b64 v[0:1], off, s[0:3], 0
 ; GFX1132-NEXT:    s_endpgm
@@ -5809,32 +5800,31 @@ define amdgpu_kernel void @sub_i64_uniform(ptr addrspace(1) %out, ptr addrspace(
 ; GFX1264-NEXT:    s_clause 0x1
 ; GFX1264-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
 ; GFX1264-NEXT:    s_load_b64 s[4:5], s[4:5], 0x34
-; GFX1264-NEXT:    s_mov_b64 s[8:9], exec
-; GFX1264-NEXT:    s_mov_b32 s11, 0
-; GFX1264-NEXT:    v_mbcnt_lo_u32_b32 v0, s8, 0
 ; GFX1264-NEXT:    s_mov_b64 s[6:7], exec
-; GFX1264-NEXT:    s_wait_alu 0xfffe
+; GFX1264-NEXT:    s_mov_b32 s11, 0
+; GFX1264-NEXT:    v_mbcnt_lo_u32_b32 v0, s6, 0
+; GFX1264-NEXT:    s_mov_b64 s[8:9], exec
 ; GFX1264-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1264-NEXT:    v_mbcnt_hi_u32_b32 v2, s9, v0
+; GFX1264-NEXT:    v_mbcnt_hi_u32_b32 v2, s7, v0
 ; GFX1264-NEXT:    ; implicit-def: $vgpr0_vgpr1
 ; GFX1264-NEXT:    v_cmpx_eq_u32_e32 0, v2
 ; GFX1264-NEXT:    s_cbranch_execz .LBB10_2
 ; GFX1264-NEXT:  ; %bb.1:
-; GFX1264-NEXT:    s_bcnt1_i32_b64 s10, s[8:9]
+; GFX1264-NEXT:    s_bcnt1_i32_b64 s10, s[6:7]
+; GFX1264-NEXT:    s_mov_b32 s15, 0x31016000
 ; GFX1264-NEXT:    s_wait_kmcnt 0x0
-; GFX1264-NEXT:    s_mul_u64 s[8:9], s[4:5], s[10:11]
-; GFX1264-NEXT:    s_mov_b32 s11, 0x31016000
+; GFX1264-NEXT:    s_mul_u64 s[6:7], s[4:5], s[10:11]
+; GFX1264-NEXT:    s_mov_b32 s14, -1
 ; GFX1264-NEXT:    s_wait_alu 0xfffe
-; GFX1264-NEXT:    v_mov_b32_e32 v0, s8
-; GFX1264-NEXT:    v_mov_b32_e32 v1, s9
-; GFX1264-NEXT:    s_mov_b32 s10, -1
-; GFX1264-NEXT:    s_mov_b32 s8, s2
-; GFX1264-NEXT:    s_mov_b32 s9, s3
-; GFX1264-NEXT:    buffer_atomic_sub_u64 v[0:1], off, s[8:11], null th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX1264-NEXT:    v_mov_b32_e32 v0, s6
+; GFX1264-NEXT:    v_mov_b32_e32 v1, s7
+; GFX1264-NEXT:    s_mov_b32 s12, s2
+; GFX1264-NEXT:    s_mov_b32 s13, s3
+; GFX1264-NEXT:    buffer_atomic_sub_u64 v[0:1], off, s[12:15], null th:TH_ATOMIC_RETURN scope:SCOPE_DEV
 ; GFX1264-NEXT:    s_wait_loadcnt 0x0
 ; GFX1264-NEXT:    global_inv scope:SCOPE_DEV
 ; GFX1264-NEXT:  .LBB10_2:
-; GFX1264-NEXT:    s_or_b64 exec, exec, s[6:7]
+; GFX1264-NEXT:    s_or_b64 exec, exec, s[8:9]
 ; GFX1264-NEXT:    s_wait_kmcnt 0x0
 ; GFX1264-NEXT:    v_mad_co_u64_u32 v[3:4], null, s4, v2, 0
 ; GFX1264-NEXT:    v_readfirstlane_b32 s2, v0
@@ -5843,9 +5833,8 @@ define amdgpu_kernel void @sub_i64_uniform(ptr addrspace(1) %out, ptr addrspace(
 ; GFX1264-NEXT:    v_mad_co_u64_u32 v[4:5], null, s5, v2, v[4:5]
 ; GFX1264-NEXT:    v_sub_co_u32 v0, vcc, s2, v3
 ; GFX1264-NEXT:    s_mov_b32 s2, -1
-; GFX1264-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1264-NEXT:    v_mov_b32_e32 v1, v4
-; GFX1264-NEXT:    v_sub_co_ci_u32_e32 v1, vcc, s3, v1, vcc
+; GFX1264-NEXT:    s_delay_alu instid0(VALU_DEP_2)
+; GFX1264-NEXT:    v_sub_co_ci_u32_e32 v1, vcc, s3, v4, vcc
 ; GFX1264-NEXT:    s_mov_b32 s3, 0x31016000
 ; GFX1264-NEXT:    buffer_store_b64 v[0:1], off, s[0:3], null
 ; GFX1264-NEXT:    s_endpgm
@@ -5864,7 +5853,6 @@ define amdgpu_kernel void @sub_i64_uniform(ptr addrspace(1) %out, ptr addrspace(
 ; GFX1232-NEXT:    v_cmpx_eq_u32_e32 0, v2
 ; GFX1232-NEXT:    s_cbranch_execz .LBB10_2
 ; GFX1232-NEXT:  ; %bb.1:
-; GFX1232-NEXT:    s_wait_alu 0xfffe
 ; GFX1232-NEXT:    s_bcnt1_i32_b32 s6, s6
 ; GFX1232-NEXT:    s_mov_b32 s15, 0x31016000
 ; GFX1232-NEXT:    s_wait_kmcnt 0x0
@@ -5888,9 +5876,8 @@ define amdgpu_kernel void @sub_i64_uniform(ptr addrspace(1) %out, ptr addrspace(
 ; GFX1232-NEXT:    v_mad_co_u64_u32 v[4:5], null, s5, v2, v[4:5]
 ; GFX1232-NEXT:    v_sub_co_u32 v0, vcc_lo, s2, v3
 ; GFX1232-NEXT:    s_mov_b32 s2, -1
-; GFX1232-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1232-NEXT:    v_mov_b32_e32 v1, v4
-; GFX1232-NEXT:    v_sub_co_ci_u32_e32 v1, vcc_lo, s3, v1, vcc_lo
+; GFX1232-NEXT:    s_delay_alu instid0(VALU_DEP_2)
+; GFX1232-NEXT:    v_sub_co_ci_u32_e32 v1, vcc_lo, s3, v4, vcc_lo
 ; GFX1232-NEXT:    s_mov_b32 s3, 0x31016000
 ; GFX1232-NEXT:    buffer_store_b64 v[0:1], off, s[0:3], null
 ; GFX1232-NEXT:    s_endpgm
@@ -6313,7 +6300,6 @@ define amdgpu_kernel void @sub_i64_varying(ptr addrspace(1) %out, ptr addrspace(
 ; GFX1264_ITERATIVE-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
 ; GFX1264_ITERATIVE-NEXT:    s_cbranch_execz .LBB11_4
 ; GFX1264_ITERATIVE-NEXT:  ; %bb.3:
-; GFX1264_ITERATIVE-NEXT:    s_wait_alu 0xfffe
 ; GFX1264_ITERATIVE-NEXT:    v_mov_b32_e32 v2, s6
 ; GFX1264_ITERATIVE-NEXT:    v_mov_b32_e32 v3, s7
 ; GFX1264_ITERATIVE-NEXT:    s_mov_b32 s11, 0x31016000
@@ -6345,7 +6331,6 @@ define amdgpu_kernel void @sub_i64_varying(ptr addrspace(1) %out, ptr addrspace(
 ; GFX1232_ITERATIVE-NEXT:    ; implicit-def: $vgpr0_vgpr1
 ; GFX1232_ITERATIVE-NEXT:  .LBB11_1: ; %ComputeLoop
 ; GFX1232_ITERATIVE-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX1232_ITERATIVE-NEXT:    s_wait_alu 0xfffe
 ; GFX1232_ITERATIVE-NEXT:    s_ctz_i32_b32 s1, s0
 ; GFX1232_ITERATIVE-NEXT:    s_wait_alu 0xfffe
 ; GFX1232_ITERATIVE-NEXT:    v_readlane_b32 s3, v3, s1
@@ -6734,8 +6719,8 @@ define amdgpu_kernel void @sub_i64_varying(ptr addrspace(1) %out, ptr addrspace(
 ; GFX1032_DPP-NEXT:    v_writelane_b32 v8, s8, 16
 ; GFX1032_DPP-NEXT:    v_writelane_b32 v7, s7, 16
 ; GFX1032_DPP-NEXT:    s_mov_b32 exec_lo, s6
-; GFX1032_DPP-NEXT:    s_mov_b32 s6, -1
 ; GFX1032_DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
+; GFX1032_DPP-NEXT:    s_mov_b32 s6, -1
 ; GFX1032_DPP-NEXT:    ; implicit-def: $vgpr9_vgpr10
 ; GFX1032_DPP-NEXT:    s_and_saveexec_b32 s8, vcc_lo
 ; GFX1032_DPP-NEXT:    s_cbranch_execz .LBB11_2
@@ -6972,20 +6957,22 @@ define amdgpu_kernel void @sub_i64_varying(ptr addrspace(1) %out, ptr addrspace(
 ; GFX1264_DPP-NEXT:    v_add_co_ci_u32_e32 v1, vcc, v1, v2, vcc
 ; GFX1264_DPP-NEXT:    v_add_co_u32_e64_dpp v3, vcc, v3, v3 row_shr:2 row_mask:0xf bank_mask:0xf bound_ctrl:1
 ; GFX1264_DPP-NEXT:    v_mov_b32_e32 v2, 0
-; GFX1264_DPP-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1264_DPP-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
 ; GFX1264_DPP-NEXT:    v_mov_b32_dpp v4, v1 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1264_DPP-NEXT:    s_wait_alu 0xfffd
 ; GFX1264_DPP-NEXT:    v_add_co_ci_u32_e32 v1, vcc, v1, v4, vcc
-; GFX1264_DPP-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3)
 ; GFX1264_DPP-NEXT:    v_add_co_u32_e64_dpp v3, vcc, v3, v3 row_shr:4 row_mask:0xf bank_mask:0xf bound_ctrl:1
 ; GFX1264_DPP-NEXT:    v_mov_b32_e32 v4, 0
+; GFX1264_DPP-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
 ; GFX1264_DPP-NEXT:    v_mov_b32_dpp v2, v1 row_shr:4 row_mask:0xf bank_mask:0xf
-; GFX1264_DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX1264_DPP-NEXT:    s_wait_alu 0xfffd
 ; GFX1264_DPP-NEXT:    v_add_co_ci_u32_e32 v1, vcc, v1, v2, vcc
 ; GFX1264_DPP-NEXT:    v_add_co_u32_e64_dpp v2, vcc, v3, v3 row_shr:8 row_mask:0xf bank_mask:0xf bound_ctrl:1
 ; GFX1264_DPP-NEXT:    v_mov_b32_e32 v3, 0
 ; GFX1264_DPP-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
 ; GFX1264_DPP-NEXT:    v_mov_b32_dpp v4, v1 row_shr:8 row_mask:0xf bank_mask:0xf
 ; GFX1264_DPP-NEXT:    v_permlanex16_b32 v5, v2, -1, -1
+; GFX1264_DPP-NEXT:    s_wait_alu 0xfffd
 ; GFX1264_DPP-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
 ; GFX1264_DPP-NEXT:    v_add_co_ci_u32_e32 v1, vcc, v1, v4, vcc
 ; GFX1264_DPP-NEXT:    v_add_co_u32_e64_dpp v2, vcc, v5, v2 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf
@@ -6993,6 +6980,7 @@ define amdgpu_kernel void @sub_i64_varying(ptr addrspace(1) %out, ptr addrspace(
 ; GFX1264_DPP-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX1264_DPP-NEXT:    v_permlanex16_b32 v4, v1, -1, -1
 ; GFX1264_DPP-NEXT:    v_mov_b32_dpp v3, v4 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf
+; GFX1264_DPP-NEXT:    s_wait_alu 0xfffd
 ; GFX1264_DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX1264_DPP-NEXT:    v_add_co_ci_u32_e32 v1, vcc, v1, v3, vcc
 ; GFX1264_DPP-NEXT:    v_mov_b32_e32 v3, 0
@@ -7001,9 +6989,11 @@ define amdgpu_kernel void @sub_i64_varying(ptr addrspace(1) %out, ptr addrspace(
 ; GFX1264_DPP-NEXT:    v_mov_b32_e32 v4, s2
 ; GFX1264_DPP-NEXT:    v_readlane_b32 s2, v2, 31
 ; GFX1264_DPP-NEXT:    v_mov_b32_dpp v3, v4 quad_perm:[0,1,2,3] row_mask:0xc bank_mask:0xf
-; GFX1264_DPP-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX1264_DPP-NEXT:    s_wait_alu 0xf1ff
+; GFX1264_DPP-NEXT:    s_delay_alu instid0(VALU_DEP_2)
 ; GFX1264_DPP-NEXT:    v_add_co_u32_e64_dpp v2, vcc, v2, s2 quad_perm:[0,1,2,3] row_mask:0xc bank_mask:0xf
 ; GFX1264_DPP-NEXT:    v_mov_b32_e32 v4, 0
+; GFX1264_DPP-NEXT:    s_wait_alu 0xfffd
 ; GFX1264_DPP-NEXT:    v_add_co_ci_u32_e32 v1, vcc, v1, v3, vcc
 ; GFX1264_DPP-NEXT:    s_mov_b64 exec, s[0:1]
 ; GFX1264_DPP-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
@@ -7055,9 +7045,11 @@ define amdgpu_kernel void @sub_i64_varying(ptr addrspace(1) %out, ptr addrspace(
 ; GFX1264_DPP-NEXT:    v_mov_b32_e32 v8, v4
 ; GFX1264_DPP-NEXT:    v_mov_b32_e32 v9, v5
 ; GFX1264_DPP-NEXT:    v_readfirstlane_b32 s3, v7
-; GFX1264_DPP-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1264_DPP-NEXT:    s_wait_alu 0xf1ff
+; GFX1264_DPP-NEXT:    s_delay_alu instid0(VALU_DEP_3)
 ; GFX1264_DPP-NEXT:    v_sub_co_u32 v6, vcc, s2, v8
 ; GFX1264_DPP-NEXT:    s_mov_b32 s2, s6
+; GFX1264_DPP-NEXT:    s_wait_alu 0xfffd
 ; GFX1264_DPP-NEXT:    v_sub_co_ci_u32_e32 v7, vcc, s3, v9, vcc
 ; GFX1264_DPP-NEXT:    s_mov_b32 s3, 0x31016000
 ; GFX1264_DPP-NEXT:    buffer_store_b64 v[6:7], off, s[0:3], null
@@ -7080,20 +7072,22 @@ define amdgpu_kernel void @sub_i64_varying(ptr addrspace(1) %out, ptr addrspace(
 ; GFX1232_DPP-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, v1, v2, vcc_lo
 ; GFX1232_DPP-NEXT:    v_add_co_u32_e64_dpp v3, vcc_lo, v3, v3 row_shr:2 row_mask:0xf bank_mask:0xf bound_ctrl:1
 ; GFX1232_DPP-NEXT:    v_mov_b32_e32 v2, 0
-; GFX1232_DPP-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1232_DPP-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
 ; GFX1232_DPP-NEXT:    v_mov_b32_dpp v4, v1 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1232_DPP-NEXT:    s_wait_alu 0xfffd
 ; GFX1232_DPP-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, v1, v4, vcc_lo
-; GFX1232_DPP-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3)
 ; GFX1232_DPP-NEXT:    v_add_co_u32_e64_dpp v3, vcc_lo, v3, v3 row_shr:4 row_mask:0xf bank_mask:0xf bound_ctrl:1
 ; GFX1232_DPP-NEXT:    v_mov_b32_e32 v4, 0
+; GFX1232_DPP-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
 ; GFX1232_DPP-NEXT:    v_mov_b32_dpp v2, v1 row_shr:4 row_mask:0xf bank_mask:0xf
-; GFX1232_DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX1232_DPP-NEXT:    s_wait_alu 0xfffd
 ; GFX1232_DPP-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, v1, v2, vcc_lo
 ; GFX1232_DPP-NEXT:    v_add_co_u32_e64_dpp v2, vcc_lo, v3, v3 row_shr:8 row_mask:0xf bank_mask:0xf bound_ctrl:1
 ; GFX1232_DPP-NEXT:    v_mov_b32_e32 v3, 0
 ; GFX1232_DPP-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
 ; GFX1232_DPP-NEXT:    v_mov_b32_dpp v4, v1 row_shr:8 row_mask:0xf bank_mask:0xf
 ; GFX1232_DPP-NEXT:    v_permlanex16_b32 v5, v2, -1, -1
+; GFX1232_DPP-NEXT:    s_wait_alu 0xfffd
 ; GFX1232_DPP-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
 ; GFX1232_DPP-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, v1, v4, vcc_lo
 ; GFX1232_DPP-NEXT:    v_add_co_u32_e64_dpp v2, vcc_lo, v5, v2 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf
@@ -7117,7 +7111,6 @@ define amdgpu_kernel void @sub_i64_varying(ptr addrspace(1) %out, ptr addrspace(
 ; GFX1232_DPP-NEXT:    s_or_saveexec_b32 s6, -1
 ; GFX1232_DPP-NEXT:    v_writelane_b32 v6, s7, 16
 ; GFX1232_DPP-NEXT:    v_writelane_b32 v7, s8, 16
-; GFX1232_DPP-NEXT:    s_wait_alu 0xfffe
 ; GFX1232_DPP-NEXT:    s_mov_b32 exec_lo, s6
 ; GFX1232_DPP-NEXT:    s_mov_b32 s6, -1
 ; GFX1232_DPP-NEXT:    s_mov_b32 s8, exec_lo
@@ -7141,9 +7134,10 @@ define amdgpu_kernel void @sub_i64_varying(ptr addrspace(1) %out, ptr addrspace(
 ; GFX1232_DPP-NEXT:    v_mov_b32_e32 v10, v6
 ; GFX1232_DPP-NEXT:    v_mov_b32_e32 v11, v7
 ; GFX1232_DPP-NEXT:    v_readfirstlane_b32 s3, v9
-; GFX1232_DPP-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1232_DPP-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_2)
 ; GFX1232_DPP-NEXT:    v_sub_co_u32 v8, vcc_lo, s2, v10
 ; GFX1232_DPP-NEXT:    s_mov_b32 s2, s6
+; GFX1232_DPP-NEXT:    s_wait_alu 0xfffd
 ; GFX1232_DPP-NEXT:    v_sub_co_ci_u32_e32 v9, vcc_lo, s3, v11, vcc_lo
 ; GFX1232_DPP-NEXT:    s_mov_b32 s3, 0x31016000
 ; GFX1232_DPP-NEXT:    buffer_store_b64 v[8:9], off, s[0:3], null
diff --git a/llvm/test/CodeGen/AMDGPU/atomic_optimizations_local_pointer.ll b/llvm/test/CodeGen/AMDGPU/atomic_optimizations_local_pointer.ll
index 45b161d7959f4..3c0646c46efd0 100644
--- a/llvm/test/CodeGen/AMDGPU/atomic_optimizations_local_pointer.ll
+++ b/llvm/test/CodeGen/AMDGPU/atomic_optimizations_local_pointer.ll
@@ -919,9 +919,9 @@ define amdgpu_kernel void @add_i32_varying(ptr addrspace(1) %out) {
 ; GFX1032_DPP-NEXT:    s_or_saveexec_b32 s0, -1
 ; GFX1032_DPP-NEXT:    v_writelane_b32 v3, s1, 16
 ; GFX1032_DPP-NEXT:    s_mov_b32 exec_lo, s0
+; GFX1032_DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
 ; GFX1032_DPP-NEXT:    s_mov_b32 s0, s2
 ; GFX1032_DPP-NEXT:    s_mov_b32 s2, -1
-; GFX1032_DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
 ; GFX1032_DPP-NEXT:    ; implicit-def: $vgpr0
 ; GFX1032_DPP-NEXT:    s_and_saveexec_b32 s1, vcc_lo
 ; GFX1032_DPP-NEXT:    s_cbranch_execz .LBB2_2
@@ -1030,9 +1030,9 @@ define amdgpu_kernel void @add_i32_varying(ptr addrspace(1) %out) {
 ; GFX1132_DPP-NEXT:    s_or_saveexec_b32 s0, -1
 ; GFX1132_DPP-NEXT:    v_writelane_b32 v3, s1, 16
 ; GFX1132_DPP-NEXT:    s_mov_b32 exec_lo, s0
+; GFX1132_DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
 ; GFX1132_DPP-NEXT:    s_mov_b32 s0, s2
 ; GFX1132_DPP-NEXT:    s_mov_b32 s2, -1
-; GFX1132_DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
 ; GFX1132_DPP-NEXT:    ; implicit-def: $vgpr0
 ; GFX1132_DPP-NEXT:    s_and_saveexec_b32 s1, vcc_lo
 ; GFX1132_DPP-NEXT:    s_cbranch_execz .LBB2_2
@@ -2630,8 +2630,8 @@ define amdgpu_kernel void @add_i64_varying(ptr addrspace(1) %out) {
 ; GFX1032_DPP-NEXT:    v_writelane_b32 v8, s6, 16
 ; GFX1032_DPP-NEXT:    v_writelane_b32 v7, s3, 16
 ; GFX1032_DPP-NEXT:    s_mov_b32 exec_lo, s2
-; GFX1032_DPP-NEXT:    s_mov_b32 s2, -1
 ; GFX1032_DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v9
+; GFX1032_DPP-NEXT:    s_mov_b32 s2, -1
 ; GFX1032_DPP-NEXT:    ; implicit-def: $vgpr9_vgpr10
 ; GFX1032_DPP-NEXT:    s_and_saveexec_b32 s3, vcc_lo
 ; GFX1032_DPP-NEXT:    s_cbranch_execz .LBB6_2
@@ -2812,8 +2812,8 @@ define amdgpu_kernel void @add_i64_varying(ptr addrspace(1) %out) {
 ; GFX1132_DPP-NEXT:    v_writelane_b32 v6, s3, 16
 ; GFX1132_DPP-NEXT:    v_writelane_b32 v7, s6, 16
 ; GFX1132_DPP-NEXT:    s_mov_b32 exec_lo, s2
-; GFX1132_DPP-NEXT:    s_mov_b32 s2, -1
 ; GFX1132_DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v8
+; GFX1132_DPP-NEXT:    s_mov_b32 s2, -1
 ; GFX1132_DPP-NEXT:    ; implicit-def: $vgpr8_vgpr9
 ; GFX1132_DPP-NEXT:    s_and_saveexec_b32 s3, vcc_lo
 ; GFX1132_DPP-NEXT:    s_cbranch_execz .LBB6_2
@@ -3301,8 +3301,8 @@ define amdgpu_kernel void @add_i64_varying_nouse() {
 ; GFX1032_DPP-NEXT:    v_add_co_u32 v1, vcc_lo, v1, v3
 ; GFX1032_DPP-NEXT:    v_add_co_ci_u32_e32 v2, vcc_lo, v2, v4, vcc_lo
 ; GFX1032_DPP-NEXT:    s_mov_b32 exec_lo, s0
-; GFX1032_DPP-NEXT:    v_mov_b32_e32 v7, v1
 ; GFX1032_DPP-NEXT:    v_mbcnt_lo_u32_b32 v9, exec_lo, 0
+; GFX1032_DPP-NEXT:    v_mov_b32_e32 v7, v1
 ; GFX1032_DPP-NEXT:    v_mov_b32_e32 v0, 0
 ; GFX1032_DPP-NEXT:    v_mov_b32_e32 v8, v2
 ; GFX1032_DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v9
@@ -4341,9 +4341,9 @@ define amdgpu_kernel void @sub_i32_varying(ptr addrspace(1) %out) {
 ; GFX1032_DPP-NEXT:    s_or_saveexec_b32 s0, -1
 ; GFX1032_DPP-NEXT:    v_writelane_b32 v3, s1, 16
 ; GFX1032_DPP-NEXT:    s_mov_b32 exec_lo, s0
+; GFX1032_DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
 ; GFX1032_DPP-NEXT:    s_mov_b32 s0, s2
 ; GFX1032_DPP-NEXT:    s_mov_b32 s2, -1
-; GFX1032_DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
 ; GFX1032_DPP-NEXT:    ; implicit-def: $vgpr0
 ; GFX1032_DPP-NEXT:    s_and_saveexec_b32 s1, vcc_lo
 ; GFX1032_DPP-NEXT:    s_cbranch_execz .LBB10_2
@@ -4452,9 +4452,9 @@ define amdgpu_kernel void @sub_i32_varying(ptr addrspace(1) %out) {
 ; GFX1132_DPP-NEXT:    s_or_saveexec_b32 s0, -1
 ; GFX1132_DPP-NEXT:    v_writelane_b32 v3, s1, 16
 ; GFX1132_DPP-NEXT:    s_mov_b32 exec_lo, s0
+; GFX1132_DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
 ; GFX1132_DPP-NEXT:    s_mov_b32 s0, s2
 ; GFX1132_DPP-NEXT:    s_mov_b32 s2, -1
-; GFX1132_DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
 ; GFX1132_DPP-NEXT:    ; implicit-def: $vgpr0
 ; GFX1132_DPP-NEXT:    s_and_saveexec_b32 s1, vcc_lo
 ; GFX1132_DPP-NEXT:    s_cbranch_execz .LBB10_2
@@ -5235,19 +5235,19 @@ define amdgpu_kernel void @sub_i64_uniform(ptr addrspace(1) %out, i64 %subitive)
 ; GFX9-NEXT:  .LBB13_2:
 ; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NEXT:    v_mad_u64_u32 v[3:4], s[4:5], s2, v2, 0
+; GFX9-NEXT:    v_mad_u64_u32 v[3:4], s[8:9], s2, v2, 0
+; GFX9-NEXT:    v_readfirstlane_b32 s8, v0
 ; GFX9-NEXT:    s_mov_b32 s4, s0
+; GFX9-NEXT:    v_mov_b32_e32 v0, v4
 ; GFX9-NEXT:    s_mov_b32 s5, s1
-; GFX9-NEXT:    v_mad_u64_u32 v[4:5], s[2:3], s3, v2, v[4:5]
-; GFX9-NEXT:    v_readfirstlane_b32 s0, v1
-; GFX9-NEXT:    v_readfirstlane_b32 s1, v0
-; GFX9-NEXT:    v_mov_b32_e32 v1, v4
-; GFX9-NEXT:    v_mov_b32_e32 v2, s0
-; GFX9-NEXT:    v_sub_co_u32_e32 v0, vcc, s1, v3
+; GFX9-NEXT:    v_readfirstlane_b32 s2, v1
+; GFX9-NEXT:    v_mad_u64_u32 v[0:1], s[0:1], s3, v2, v[0:1]
+; GFX9-NEXT:    v_mov_b32_e32 v2, s2
+; GFX9-NEXT:    v_sub_co_u32_e32 v1, vcc, s8, v3
 ; GFX9-NEXT:    s_mov_b32 s7, 0xf000
 ; GFX9-NEXT:    s_mov_b32 s6, -1
-; GFX9-NEXT:    v_subb_co_u32_e32 v1, vcc, v2, v1, vcc
-; GFX9-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
+; GFX9-NEXT:    v_subb_co_u32_e32 v2, vcc, v2, v0, vcc
+; GFX9-NEXT:    buffer_store_dwordx2 v[1:2], off, s[4:7], 0
 ; GFX9-NEXT:    s_endpgm
 ;
 ; GFX1064-LABEL: sub_i64_uniform:
@@ -5283,9 +5283,8 @@ define amdgpu_kernel void @sub_i64_uniform(ptr addrspace(1) %out, i64 %subitive)
 ; GFX1064-NEXT:    v_readfirstlane_b32 s2, v0
 ; GFX1064-NEXT:    s_mov_b32 s3, 0x31016000
 ; GFX1064-NEXT:    v_sub_co_u32 v0, vcc, s2, v3
-; GFX1064-NEXT:    v_mov_b32_e32 v1, v4
+; GFX1064-NEXT:    v_sub_co_ci_u32_e32 v1, vcc, s4, v4, vcc
 ; GFX1064-NEXT:    s_mov_b32 s2, -1
-; GFX1064-NEXT:    v_sub_co_ci_u32_e32 v1, vcc, s4, v1, vcc
 ; GFX1064-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
 ; GFX1064-NEXT:    s_endpgm
 ;
@@ -5321,9 +5320,8 @@ define amdgpu_kernel void @sub_i64_uniform(ptr addrspace(1) %out, i64 %subitive)
 ; GFX1032-NEXT:    v_readfirstlane_b32 s2, v0
 ; GFX1032-NEXT:    s_mov_b32 s3, 0x31016000
 ; GFX1032-NEXT:    v_sub_co_u32 v0, vcc_lo, s2, v3
-; GFX1032-NEXT:    v_mov_b32_e32 v1, v4
+; GFX1032-NEXT:    v_sub_co_ci_u32_e32 v1, vcc_lo, s4, v4, vcc_lo
 ; GFX1032-NEXT:    s_mov_b32 s2, -1
-; GFX1032-NEXT:    v_sub_co_ci_u32_e32 v1, vcc_lo, s4, v1, vcc_lo
 ; GFX1032-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
 ; GFX1032-NEXT:    s_endpgm
 ;
@@ -5362,9 +5360,8 @@ define amdgpu_kernel void @sub_i64_uniform(ptr addrspace(1) %out, i64 %subitive)
 ; GFX1164-NEXT:    v_sub_co_u32 v0, vcc, s2, v3
 ; GFX1164-NEXT:    s_mov_b32 s3, 0x31016000
 ; GFX1164-NEXT:    s_mov_b32 s2, -1
-; GFX1164-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-NEXT:    v_mov_b32_e32 v1, v5
-; GFX1164-NEXT:    v_sub_co_ci_u32_e32 v1, vcc, s4, v1, vcc
+; GFX1164-NEXT:    s_delay_alu instid0(VALU_DEP_2)
+; GFX1164-NEXT:    v_sub_co_ci_u32_e32 v1, vcc, s4, v5, vcc
 ; GFX1164-NEXT:    buffer_store_b64 v[0:1], off, s[0:3], 0
 ; GFX1164-NEXT:    s_endpgm
 ;
@@ -5402,9 +5399,8 @@ define amdgpu_kernel void @sub_i64_uniform(ptr addrspace(1) %out, i64 %subitive)
 ; GFX1132-NEXT:    v_sub_co_u32 v0, vcc_lo, s2, v3
 ; GFX1132-NEXT:    s_mov_b32 s3, 0x31016000
 ; GFX1132-NEXT:    s_mov_b32 s2, -1
-; GFX1132-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-NEXT:    v_mov_b32_e32 v1, v5
-; GFX1132-NEXT:    v_sub_co_ci_u32_e32 v1, vcc_lo, s4, v1, vcc_lo
+; GFX1132-NEXT:    s_delay_alu instid0(VALU_DEP_2)
+; GFX1132-NEXT:    v_sub_co_ci_u32_e32 v1, vcc_lo, s4, v5, vcc_lo
 ; GFX1132-NEXT:    buffer_store_b64 v[0:1], off, s[0:3], 0
 ; GFX1132-NEXT:    s_endpgm
 entry:
@@ -6079,8 +6075,8 @@ define amdgpu_kernel void @sub_i64_varying(ptr addrspace(1) %out) {
 ; GFX1032_DPP-NEXT:    v_writelane_b32 v8, s6, 16
 ; GFX1032_DPP-NEXT:    v_writelane_b32 v7, s3, 16
 ; GFX1032_DPP-NEXT:    s_mov_b32 exec_lo, s2
-; GFX1032_DPP-NEXT:    s_mov_b32 s2, -1
 ; GFX1032_DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v9
+; GFX1032_DPP-NEXT:    s_mov_b32 s2, -1
 ; GFX1032_DPP-NEXT:    ; implicit-def: $vgpr9_vgpr10
 ; GFX1032_DPP-NEXT:    s_and_saveexec_b32 s3, vcc_lo
 ; GFX1032_DPP-NEXT:    s_cbranch_execz .LBB14_2
@@ -6261,8 +6257,8 @@ define amdgpu_kernel void @sub_i64_varying(ptr addrspace(1) %out) {
 ; GFX1132_DPP-NEXT:    v_writelane_b32 v6, s3, 16
 ; GFX1132_DPP-NEXT:    v_writelane_b32 v7, s6, 16
 ; GFX1132_DPP-NEXT:    s_mov_b32 exec_lo, s2
-; GFX1132_DPP-NEXT:    s_mov_b32 s2, -1
 ; GFX1132_DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v8
+; GFX1132_DPP-NEXT:    s_mov_b32 s2, -1
 ; GFX1132_DPP-NEXT:    ; implicit-def: $vgpr8_vgpr9
 ; GFX1132_DPP-NEXT:    s_and_saveexec_b32 s3, vcc_lo
 ; GFX1132_DPP-NEXT:    s_cbranch_execz .LBB14_2
@@ -6761,9 +6757,9 @@ define amdgpu_kernel void @and_i32_varying(ptr addrspace(1) %out) {
 ; GFX1032_DPP-NEXT:    s_or_saveexec_b32 s0, -1
 ; GFX1032_DPP-NEXT:    v_writelane_b32 v3, s1, 16
 ; GFX1032_DPP-NEXT:    s_mov_b32 exec_lo, s0
+; GFX1032_DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
 ; GFX1032_DPP-NEXT:    s_mov_b32 s0, s2
 ; GFX1032_DPP-NEXT:    s_mov_b32 s2, -1
-; GFX1032_DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
 ; GFX1032_DPP-NEXT:    ; implicit-def: $vgpr0
 ; GFX1032_DPP-NEXT:    s_and_saveexec_b32 s1, vcc_lo
 ; GFX1032_DPP-NEXT:    s_cbranch_execz .LBB15_2
@@ -6872,9 +6868,10 @@ define amdgpu_kernel void @and_i32_varying(ptr addrspace(1) %out) {
 ; GFX1132_DPP-NEXT:    s_or_saveexec_b32 s0, -1
 ; GFX1132_DPP-NEXT:    v_writelane_b32 v3, s1, 16
 ; GFX1132_DPP-NEXT:    s_mov_b32 exec_lo, s0
+; GFX1132_DPP-NEXT:    s_delay_alu instid0(VALU_DEP_2)
+; GFX1132_DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
 ; GFX1132_DPP-NEXT:    s_mov_b32 s0, s2
 ; GFX1132_DPP-NEXT:    s_mov_b32 s2, -1
-; GFX1132_DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
 ; GFX1132_DPP-NEXT:    ; implicit-def: $vgpr0
 ; GFX1132_DPP-NEXT:    s_and_saveexec_b32 s1, vcc_lo
 ; GFX1132_DPP-NEXT:    s_cbranch_execz .LBB15_2
@@ -7468,8 +7465,8 @@ define amdgpu_kernel void @and_i64_varying(ptr addrspace(1) %out) {
 ; GFX1032_DPP-NEXT:    v_writelane_b32 v6, s3, 16
 ; GFX1032_DPP-NEXT:    v_writelane_b32 v5, s6, 16
 ; GFX1032_DPP-NEXT:    s_mov_b32 exec_lo, s2
-; GFX1032_DPP-NEXT:    s_mov_b32 s2, -1
 ; GFX1032_DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v7
+; GFX1032_DPP-NEXT:    s_mov_b32 s2, -1
 ; GFX1032_DPP-NEXT:    ; implicit-def: $vgpr7_vgpr8
 ; GFX1032_DPP-NEXT:    s_and_saveexec_b32 s3, vcc_lo
 ; GFX1032_DPP-NEXT:    s_cbranch_execz .LBB16_2
@@ -7625,8 +7622,8 @@ define amdgpu_kernel void @and_i64_varying(ptr addrspace(1) %out) {
 ; GFX1132_DPP-NEXT:    v_writelane_b32 v6, s3, 16
 ; GFX1132_DPP-NEXT:    v_writelane_b32 v5, s6, 16
 ; GFX1132_DPP-NEXT:    s_mov_b32 exec_lo, s2
-; GFX1132_DPP-NEXT:    s_mov_b32 s2, -1
 ; GFX1132_DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v7
+; GFX1132_DPP-NEXT:    s_mov_b32 s2, -1
 ; GFX1132_DPP-NEXT:    ; implicit-def: $vgpr7_vgpr8
 ; GFX1132_DPP-NEXT:    s_and_saveexec_b32 s3, vcc_lo
 ; GFX1132_DPP-NEXT:    s_cbranch_execz .LBB16_2
@@ -8126,9 +8123,9 @@ define amdgpu_kernel void @or_i32_varying(ptr addrspace(1) %out) {
 ; GFX1032_DPP-NEXT:    s_or_saveexec_b32 s0, -1
 ; GFX1032_DPP-NEXT:    v_writelane_b32 v3, s1, 16
 ; GFX1032_DPP-NEXT:    s_mov_b32 exec_lo, s0
+; GFX1032_DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
 ; GFX1032_DPP-NEXT:    s_mov_b32 s0, s2
 ; GFX1032_DPP-NEXT:    s_mov_b32 s2, -1
-; GFX1032_DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
 ; GFX1032_DPP-NEXT:    ; implicit-def: $vgpr0
 ; GFX1032_DPP-NEXT:    s_and_saveexec_b32 s1, vcc_lo
 ; GFX1032_DPP-NEXT:    s_cbranch_execz .LBB17_2
@@ -8237,9 +8234,9 @@ define amdgpu_kernel void @or_i32_varying(ptr addrspace(1) %out) {
 ; GFX1132_DPP-NEXT:    s_or_saveexec_b32 s0, -1
 ; GFX1132_DPP-NEXT:    v_writelane_b32 v3, s1, 16
 ; GFX1132_DPP-NEXT:    s_mov_b32 exec_lo, s0
+; GFX1132_DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
 ; GFX1132_DPP-NEXT:    s_mov_b32 s0, s2
 ; GFX1132_DPP-NEXT:    s_mov_b32 s2, -1
-; GFX1132_DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
 ; GFX1132_DPP-NEXT:    ; implicit-def: $vgpr0
 ; GFX1132_DPP-NEXT:    s_and_saveexec_b32 s1, vcc_lo
 ; GFX1132_DPP-NEXT:    s_cbranch_execz .LBB17_2
@@ -8832,8 +8829,8 @@ define amdgpu_kernel void @or_i64_varying(ptr addrspace(1) %out) {
 ; GFX1032_DPP-NEXT:    v_writelane_b32 v6, s3, 16
 ; GFX1032_DPP-NEXT:    v_writelane_b32 v5, s6, 16
 ; GFX1032_DPP-NEXT:    s_mov_b32 exec_lo, s2
-; GFX1032_DPP-NEXT:    s_mov_b32 s2, -1
 ; GFX1032_DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v7
+; GFX1032_DPP-NEXT:    s_mov_b32 s2, -1
 ; GFX1032_DPP-NEXT:    ; implicit-def: $vgpr7_vgpr8
 ; GFX1032_DPP-NEXT:    s_and_saveexec_b32 s3, vcc_lo
 ; GFX1032_DPP-NEXT:    s_cbranch_execz .LBB18_2
@@ -8989,8 +8986,8 @@ define amdgpu_kernel void @or_i64_varying(ptr addrspace(1) %out) {
 ; GFX1132_DPP-NEXT:    v_writelane_b32 v6, s3, 16
 ; GFX1132_DPP-NEXT:    v_writelane_b32 v5, s6, 16
 ; GFX1132_DPP-NEXT:    s_mov_b32 exec_lo, s2
-; GFX1132_DPP-NEXT:    s_mov_b32 s2, -1
 ; GFX1132_DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v7
+; GFX1132_DPP-NEXT:    s_mov_b32 s2, -1
 ; GFX1132_DPP-NEXT:    ; implicit-def: $vgpr7_vgpr8
 ; GFX1132_DPP-NEXT:    s_and_saveexec_b32 s3, vcc_lo
 ; GFX1132_DPP-NEXT:    s_cbranch_execz .LBB18_2
@@ -9490,9 +9487,9 @@ define amdgpu_kernel void @xor_i32_varying(ptr addrspace(1) %out) {
 ; GFX1032_DPP-NEXT:    s_or_saveexec_b32 s0, -1
 ; GFX1032_DPP-NEXT:    v_writelane_b32 v3, s1, 16
 ; GFX1032_DPP-NEXT:    s_mov_b32 exec_lo, s0
+; GFX1032_DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
 ; GFX1032_DPP-NEXT:    s_mov_b32 s0, s2
 ; GFX1032_DPP-NEXT:    s_mov_b32 s2, -1
-; GFX1032_DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
 ; GFX1032_DPP-NEXT:    ; implicit-def: $vgpr0
 ; GFX1032_DPP-NEXT:    s_and_saveexec_b32 s1, vcc_lo
 ; GFX1032_DPP-NEXT:    s_cbranch_execz .LBB19_2
@@ -9601,9 +9598,9 @@ define amdgpu_kernel void @xor_i32_varying(ptr addrspace(1) %out) {
 ; GFX1132_DPP-NEXT:    s_or_saveexec_b32 s0, -1
 ; GFX1132_DPP-NEXT:    v_writelane_b32 v3, s1, 16
 ; GFX1132_DPP-NEXT:    s_mov_b32 exec_lo, s0
+; GFX1132_DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
 ; GFX1132_DPP-NEXT:    s_mov_b32 s0, s2
 ; GFX1132_DPP-NEXT:    s_mov_b32 s2, -1
-; GFX1132_DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
 ; GFX1132_DPP-NEXT:    ; implicit-def: $vgpr0
 ; GFX1132_DPP-NEXT:    s_and_saveexec_b32 s1, vcc_lo
 ; GFX1132_DPP-NEXT:    s_cbranch_execz .LBB19_2
@@ -10196,8 +10193,8 @@ define amdgpu_kernel void @xor_i64_varying(ptr addrspace(1) %out) {
 ; GFX1032_DPP-NEXT:    v_writelane_b32 v6, s3, 16
 ; GFX1032_DPP-NEXT:    v_writelane_b32 v5, s6, 16
 ; GFX1032_DPP-NEXT:    s_mov_b32 exec_lo, s2
-; GFX1032_DPP-NEXT:    s_mov_b32 s2, -1
 ; GFX1032_DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v7
+; GFX1032_DPP-NEXT:    s_mov_b32 s2, -1
 ; GFX1032_DPP-NEXT:    ; implicit-def: $vgpr7_vgpr8
 ; GFX1032_DPP-NEXT:    s_and_saveexec_b32 s3, vcc_lo
 ; GFX1032_DPP-NEXT:    s_cbranch_execz .LBB20_2
@@ -10353,8 +10350,8 @@ define amdgpu_kernel void @xor_i64_varying(ptr addrspace(1) %out) {
 ; GFX1132_DPP-NEXT:    v_writelane_b32 v6, s3, 16
 ; GFX1132_DPP-NEXT:    v_writelane_b32 v5, s6, 16
 ; GFX1132_DPP-NEXT:    s_mov_b32 exec_lo, s2
-; GFX1132_DPP-NEXT:    s_mov_b32 s2, -1
 ; GFX1132_DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v7
+; GFX1132_DPP-NEXT:    s_mov_b32 s2, -1
 ; GFX1132_DPP-NEXT:    ; implicit-def: $vgpr7_vgpr8
 ; GFX1132_DPP-NEXT:    s_and_saveexec_b32 s3, vcc_lo
 ; GFX1132_DPP-NEXT:    s_cbranch_execz .LBB20_2
@@ -10853,9 +10850,9 @@ define amdgpu_kernel void @max_i32_varying(ptr addrspace(1) %out) {
 ; GFX1032_DPP-NEXT:    s_or_saveexec_b32 s0, -1
 ; GFX1032_DPP-NEXT:    v_writelane_b32 v3, s1, 16
 ; GFX1032_DPP-NEXT:    s_mov_b32 exec_lo, s0
+; GFX1032_DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
 ; GFX1032_DPP-NEXT:    s_mov_b32 s0, s2
 ; GFX1032_DPP-NEXT:    s_mov_b32 s2, -1
-; GFX1032_DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
 ; GFX1032_DPP-NEXT:    ; implicit-def: $vgpr0
 ; GFX1032_DPP-NEXT:    s_and_saveexec_b32 s1, vcc_lo
 ; GFX1032_DPP-NEXT:    s_cbranch_execz .LBB21_2
@@ -10964,9 +10961,10 @@ define amdgpu_kernel void @max_i32_varying(ptr addrspace(1) %out) {
 ; GFX1132_DPP-NEXT:    s_or_saveexec_b32 s0, -1
 ; GFX1132_DPP-NEXT:    v_writelane_b32 v3, s1, 16
 ; GFX1132_DPP-NEXT:    s_mov_b32 exec_lo, s0
+; GFX1132_DPP-NEXT:    s_delay_alu instid0(VALU_DEP_2)
+; GFX1132_DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
 ; GFX1132_DPP-NEXT:    s_mov_b32 s0, s2
 ; GFX1132_DPP-NEXT:    s_mov_b32 s2, -1
-; GFX1132_DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
 ; GFX1132_DPP-NEXT:    ; implicit-def: $vgpr0
 ; GFX1132_DPP-NEXT:    s_and_saveexec_b32 s1, vcc_lo
 ; GFX1132_DPP-NEXT:    s_cbranch_execz .LBB21_2
@@ -11971,8 +11969,8 @@ define amdgpu_kernel void @max_i64_varying(ptr addrspace(1) %out) {
 ; GFX1032_DPP-NEXT:    v_writelane_b32 v5, s3, 16
 ; GFX1032_DPP-NEXT:    v_writelane_b32 v4, s6, 16
 ; GFX1032_DPP-NEXT:    s_mov_b32 exec_lo, s2
-; GFX1032_DPP-NEXT:    s_mov_b32 s2, -1
 ; GFX1032_DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v7
+; GFX1032_DPP-NEXT:    s_mov_b32 s2, -1
 ; GFX1032_DPP-NEXT:    ; implicit-def: $vgpr7_vgpr8
 ; GFX1032_DPP-NEXT:    s_and_saveexec_b32 s3, vcc_lo
 ; GFX1032_DPP-NEXT:    s_cbranch_execz .LBB23_2
@@ -12185,8 +12183,8 @@ define amdgpu_kernel void @max_i64_varying(ptr addrspace(1) %out) {
 ; GFX1132_DPP-NEXT:    v_writelane_b32 v5, s3, 16
 ; GFX1132_DPP-NEXT:    v_writelane_b32 v4, s6, 16
 ; GFX1132_DPP-NEXT:    s_mov_b32 exec_lo, s2
-; GFX1132_DPP-NEXT:    s_mov_b32 s2, -1
 ; GFX1132_DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v7
+; GFX1132_DPP-NEXT:    s_mov_b32 s2, -1
 ; GFX1132_DPP-NEXT:    ; implicit-def: $vgpr7_vgpr8
 ; GFX1132_DPP-NEXT:    s_and_saveexec_b32 s3, vcc_lo
 ; GFX1132_DPP-NEXT:    s_cbranch_execz .LBB23_2
@@ -12686,9 +12684,9 @@ define amdgpu_kernel void @min_i32_varying(ptr addrspace(1) %out) {
 ; GFX1032_DPP-NEXT:    s_or_saveexec_b32 s0, -1
 ; GFX1032_DPP-NEXT:    v_writelane_b32 v3, s1, 16
 ; GFX1032_DPP-NEXT:    s_mov_b32 exec_lo, s0
+; GFX1032_DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
 ; GFX1032_DPP-NEXT:    s_mov_b32 s0, s2
 ; GFX1032_DPP-NEXT:    s_mov_b32 s2, -1
-; GFX1032_DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
 ; GFX1032_DPP-NEXT:    ; implicit-def: $vgpr0
 ; GFX1032_DPP-NEXT:    s_and_saveexec_b32 s1, vcc_lo
 ; GFX1032_DPP-NEXT:    s_cbranch_execz .LBB24_2
@@ -12797,9 +12795,10 @@ define amdgpu_kernel void @min_i32_varying(ptr addrspace(1) %out) {
 ; GFX1132_DPP-NEXT:    s_or_saveexec_b32 s0, -1
 ; GFX1132_DPP-NEXT:    v_writelane_b32 v3, s1, 16
 ; GFX1132_DPP-NEXT:    s_mov_b32 exec_lo, s0
+; GFX1132_DPP-NEXT:    s_delay_alu instid0(VALU_DEP_2)
+; GFX1132_DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
 ; GFX1132_DPP-NEXT:    s_mov_b32 s0, s2
 ; GFX1132_DPP-NEXT:    s_mov_b32 s2, -1
-; GFX1132_DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
 ; GFX1132_DPP-NEXT:    ; implicit-def: $vgpr0
 ; GFX1132_DPP-NEXT:    s_and_saveexec_b32 s1, vcc_lo
 ; GFX1132_DPP-NEXT:    s_cbranch_execz .LBB24_2
@@ -13804,8 +13803,8 @@ define amdgpu_kernel void @min_i64_varying(ptr addrspace(1) %out) {
 ; GFX1032_DPP-NEXT:    v_writelane_b32 v5, s3, 16
 ; GFX1032_DPP-NEXT:    v_writelane_b32 v4, s6, 16
 ; GFX1032_DPP-NEXT:    s_mov_b32 exec_lo, s2
-; GFX1032_DPP-NEXT:    s_mov_b32 s2, -1
 ; GFX1032_DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v7
+; GFX1032_DPP-NEXT:    s_mov_b32 s2, -1
 ; GFX1032_DPP-NEXT:    ; implicit-def: $vgpr7_vgpr8
 ; GFX1032_DPP-NEXT:    s_and_saveexec_b32 s3, vcc_lo
 ; GFX1032_DPP-NEXT:    s_cbranch_execz .LBB26_2
@@ -14018,8 +14017,8 @@ define amdgpu_kernel void @min_i64_varying(ptr addrspace(1) %out) {
 ; GFX1132_DPP-NEXT:    v_writelane_b32 v5, s3, 16
 ; GFX1132_DPP-NEXT:    v_writelane_b32 v4, s6, 16
 ; GFX1132_DPP-NEXT:    s_mov_b32 exec_lo, s2
-; GFX1132_DPP-NEXT:    s_mov_b32 s2, -1
 ; GFX1132_DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v7
+; GFX1132_DPP-NEXT:    s_mov_b32 s2, -1
 ; GFX1132_DPP-NEXT:    ; implicit-def: $vgpr7_vgpr8
 ; GFX1132_DPP-NEXT:    s_and_saveexec_b32 s3, vcc_lo
 ; GFX1132_DPP-NEXT:    s_cbranch_execz .LBB26_2
@@ -14520,9 +14519,9 @@ define amdgpu_kernel void @umax_i32_varying(ptr addrspace(1) %out) {
 ; GFX1032_DPP-NEXT:    s_or_saveexec_b32 s0, -1
 ; GFX1032_DPP-NEXT:    v_writelane_b32 v3, s1, 16
 ; GFX1032_DPP-NEXT:    s_mov_b32 exec_lo, s0
+; GFX1032_DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
 ; GFX1032_DPP-NEXT:    s_mov_b32 s0, s2
 ; GFX1032_DPP-NEXT:    s_mov_b32 s2, -1
-; GFX1032_DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
 ; GFX1032_DPP-NEXT:    ; implicit-def: $vgpr0
 ; GFX1032_DPP-NEXT:    s_and_saveexec_b32 s1, vcc_lo
 ; GFX1032_DPP-NEXT:    s_cbranch_execz .LBB27_2
@@ -14631,9 +14630,9 @@ define amdgpu_kernel void @umax_i32_varying(ptr addrspace(1) %out) {
 ; GFX1132_DPP-NEXT:    s_or_saveexec_b32 s0, -1
 ; GFX1132_DPP-NEXT:    v_writelane_b32 v3, s1, 16
 ; GFX1132_DPP-NEXT:    s_mov_b32 exec_lo, s0
+; GFX1132_DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
 ; GFX1132_DPP-NEXT:    s_mov_b32 s0, s2
 ; GFX1132_DPP-NEXT:    s_mov_b32 s2, -1
-; GFX1132_DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
 ; GFX1132_DPP-NEXT:    ; implicit-def: $vgpr0
 ; GFX1132_DPP-NEXT:    s_and_saveexec_b32 s1, vcc_lo
 ; GFX1132_DPP-NEXT:    s_cbranch_execz .LBB27_2
@@ -15629,8 +15628,8 @@ define amdgpu_kernel void @umax_i64_varying(ptr addrspace(1) %out) {
 ; GFX1032_DPP-NEXT:    v_writelane_b32 v5, s3, 16
 ; GFX1032_DPP-NEXT:    v_writelane_b32 v4, s6, 16
 ; GFX1032_DPP-NEXT:    s_mov_b32 exec_lo, s2
-; GFX1032_DPP-NEXT:    s_mov_b32 s2, -1
 ; GFX1032_DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v7
+; GFX1032_DPP-NEXT:    s_mov_b32 s2, -1
 ; GFX1032_DPP-NEXT:    ; implicit-def: $vgpr7_vgpr8
 ; GFX1032_DPP-NEXT:    s_and_saveexec_b32 s3, vcc_lo
 ; GFX1032_DPP-NEXT:    s_cbranch_execz .LBB29_2
@@ -15837,8 +15836,8 @@ define amdgpu_kernel void @umax_i64_varying(ptr addrspace(1) %out) {
 ; GFX1132_DPP-NEXT:    v_writelane_b32 v5, s3, 16
 ; GFX1132_DPP-NEXT:    v_writelane_b32 v4, s6, 16
 ; GFX1132_DPP-NEXT:    s_mov_b32 exec_lo, s2
-; GFX1132_DPP-NEXT:    s_mov_b32 s2, -1
 ; GFX1132_DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v7
+; GFX1132_DPP-NEXT:    s_mov_b32 s2, -1
 ; GFX1132_DPP-NEXT:    ; implicit-def: $vgpr7_vgpr8
 ; GFX1132_DPP-NEXT:    s_and_saveexec_b32 s3, vcc_lo
 ; GFX1132_DPP-NEXT:    s_cbranch_execz .LBB29_2
@@ -16338,9 +16337,9 @@ define amdgpu_kernel void @umin_i32_varying(ptr addrspace(1) %out) {
 ; GFX1032_DPP-NEXT:    s_or_saveexec_b32 s0, -1
 ; GFX1032_DPP-NEXT:    v_writelane_b32 v3, s1, 16
 ; GFX1032_DPP-NEXT:    s_mov_b32 exec_lo, s0
+; GFX1032_DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
 ; GFX1032_DPP-NEXT:    s_mov_b32 s0, s2
 ; GFX1032_DPP-NEXT:    s_mov_b32 s2, -1
-; GFX1032_DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
 ; GFX1032_DPP-NEXT:    ; implicit-def: $vgpr0
 ; GFX1032_DPP-NEXT:    s_and_saveexec_b32 s1, vcc_lo
 ; GFX1032_DPP-NEXT:    s_cbranch_execz .LBB30_2
@@ -16449,9 +16448,10 @@ define amdgpu_kernel void @umin_i32_varying(ptr addrspace(1) %out) {
 ; GFX1132_DPP-NEXT:    s_or_saveexec_b32 s0, -1
 ; GFX1132_DPP-NEXT:    v_writelane_b32 v3, s1, 16
 ; GFX1132_DPP-NEXT:    s_mov_b32 exec_lo, s0
+; GFX1132_DPP-NEXT:    s_delay_alu instid0(VALU_DEP_2)
+; GFX1132_DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
 ; GFX1132_DPP-NEXT:    s_mov_b32 s0, s2
 ; GFX1132_DPP-NEXT:    s_mov_b32 s2, -1
-; GFX1132_DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
 ; GFX1132_DPP-NEXT:    ; implicit-def: $vgpr0
 ; GFX1132_DPP-NEXT:    s_and_saveexec_b32 s1, vcc_lo
 ; GFX1132_DPP-NEXT:    s_cbranch_execz .LBB30_2
@@ -17446,8 +17446,8 @@ define amdgpu_kernel void @umin_i64_varying(ptr addrspace(1) %out) {
 ; GFX1032_DPP-NEXT:    v_writelane_b32 v5, s3, 16
 ; GFX1032_DPP-NEXT:    v_writelane_b32 v4, s6, 16
 ; GFX1032_DPP-NEXT:    s_mov_b32 exec_lo, s2
-; GFX1032_DPP-NEXT:    s_mov_b32 s2, -1
 ; GFX1032_DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v7
+; GFX1032_DPP-NEXT:    s_mov_b32 s2, -1
 ; GFX1032_DPP-NEXT:    ; implicit-def: $vgpr7_vgpr8
 ; GFX1032_DPP-NEXT:    s_and_saveexec_b32 s3, vcc_lo
 ; GFX1032_DPP-NEXT:    s_cbranch_execz .LBB32_2
@@ -17654,8 +17654,8 @@ define amdgpu_kernel void @umin_i64_varying(ptr addrspace(1) %out) {
 ; GFX1132_DPP-NEXT:    v_writelane_b32 v5, s3, 16
 ; GFX1132_DPP-NEXT:    v_writelane_b32 v4, s6, 16
 ; GFX1132_DPP-NEXT:    s_mov_b32 exec_lo, s2
-; GFX1132_DPP-NEXT:    s_mov_b32 s2, -1
 ; GFX1132_DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v7
+; GFX1132_DPP-NEXT:    s_mov_b32 s2, -1
 ; GFX1132_DPP-NEXT:    ; implicit-def: $vgpr7_vgpr8
 ; GFX1132_DPP-NEXT:    s_and_saveexec_b32 s3, vcc_lo
 ; GFX1132_DPP-NEXT:    s_cbranch_execz .LBB32_2
diff --git a/llvm/test/CodeGen/AMDGPU/atomic_optimizations_raw_buffer.ll b/llvm/test/CodeGen/AMDGPU/atomic_optimizations_raw_buffer.ll
index 905a515d7c125..8c6224cc86284 100644
--- a/llvm/test/CodeGen/AMDGPU/atomic_optimizations_raw_buffer.ll
+++ b/llvm/test/CodeGen/AMDGPU/atomic_optimizations_raw_buffer.ll
@@ -219,7 +219,6 @@ define amdgpu_kernel void @add_i32_constant(ptr addrspace(1) %out, ptr addrspace
 ; GFX12W64-NEXT:    s_mov_b64 s[0:1], exec
 ; GFX12W64-NEXT:    v_mbcnt_lo_u32_b32 v0, s2, 0
 ; GFX12W64-NEXT:    ; implicit-def: $vgpr1
-; GFX12W64-NEXT:    s_wait_alu 0xfffe
 ; GFX12W64-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12W64-NEXT:    v_mbcnt_hi_u32_b32 v0, s3, v0
 ; GFX12W64-NEXT:    v_cmpx_eq_u32_e32 0, v0
@@ -239,6 +238,7 @@ define amdgpu_kernel void @add_i32_constant(ptr addrspace(1) %out, ptr addrspace
 ; GFX12W64-NEXT:    s_wait_loadcnt 0x0
 ; GFX12W64-NEXT:    v_readfirstlane_b32 s2, v1
 ; GFX12W64-NEXT:    v_mov_b32_e32 v1, 0
+; GFX12W64-NEXT:    s_wait_alu 0xf1ff
 ; GFX12W64-NEXT:    s_delay_alu instid0(VALU_DEP_2)
 ; GFX12W64-NEXT:    v_mad_u32_u24 v0, v0, 5, s2
 ; GFX12W64-NEXT:    s_wait_kmcnt 0x0
@@ -256,7 +256,6 @@ define amdgpu_kernel void @add_i32_constant(ptr addrspace(1) %out, ptr addrspace
 ; GFX12W32-NEXT:    s_cbranch_execz .LBB0_2
 ; GFX12W32-NEXT:  ; %bb.1:
 ; GFX12W32-NEXT:    s_load_b128 s[8:11], s[4:5], 0x34
-; GFX12W32-NEXT:    s_wait_alu 0xfffe
 ; GFX12W32-NEXT:    s_bcnt1_i32_b32 s1, s1
 ; GFX12W32-NEXT:    s_wait_alu 0xfffe
 ; GFX12W32-NEXT:    s_mul_i32 s1, s1, 5
@@ -265,7 +264,6 @@ define amdgpu_kernel void @add_i32_constant(ptr addrspace(1) %out, ptr addrspace
 ; GFX12W32-NEXT:    s_wait_kmcnt 0x0
 ; GFX12W32-NEXT:    buffer_atomic_add_u32 v1, off, s[8:11], null th:TH_ATOMIC_RETURN
 ; GFX12W32-NEXT:  .LBB0_2:
-; GFX12W32-NEXT:    s_wait_alu 0xfffe
 ; GFX12W32-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX12W32-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
 ; GFX12W32-NEXT:    s_wait_loadcnt 0x0
@@ -498,7 +496,6 @@ define amdgpu_kernel void @add_i32_uniform(ptr addrspace(1) %out, ptr addrspace(
 ; GFX12W64-NEXT:    s_mov_b64 s[0:1], exec
 ; GFX12W64-NEXT:    v_mbcnt_lo_u32_b32 v0, s2, 0
 ; GFX12W64-NEXT:    ; implicit-def: $vgpr1
-; GFX12W64-NEXT:    s_wait_alu 0xfffe
 ; GFX12W64-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12W64-NEXT:    v_mbcnt_hi_u32_b32 v0, s3, v0
 ; GFX12W64-NEXT:    v_cmpx_eq_u32_e32 0, v0
@@ -518,6 +515,7 @@ define amdgpu_kernel void @add_i32_uniform(ptr addrspace(1) %out, ptr addrspace(
 ; GFX12W64-NEXT:    s_wait_loadcnt 0x0
 ; GFX12W64-NEXT:    v_readfirstlane_b32 s2, v1
 ; GFX12W64-NEXT:    s_wait_kmcnt 0x0
+; GFX12W64-NEXT:    s_wait_alu 0xf1ff
 ; GFX12W64-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12W64-NEXT:    v_mad_co_u64_u32 v[0:1], null, s6, v0, s[2:3]
 ; GFX12W64-NEXT:    v_mov_b32_e32 v1, 0
@@ -536,7 +534,6 @@ define amdgpu_kernel void @add_i32_uniform(ptr addrspace(1) %out, ptr addrspace(
 ; GFX12W32-NEXT:    s_cbranch_execz .LBB1_2
 ; GFX12W32-NEXT:  ; %bb.1:
 ; GFX12W32-NEXT:    s_load_b128 s[8:11], s[4:5], 0x34
-; GFX12W32-NEXT:    s_wait_alu 0xfffe
 ; GFX12W32-NEXT:    s_bcnt1_i32_b32 s2, s2
 ; GFX12W32-NEXT:    s_wait_kmcnt 0x0
 ; GFX12W32-NEXT:    s_wait_alu 0xfffe
@@ -882,7 +879,6 @@ define amdgpu_kernel void @add_i32_varying_vdata(ptr addrspace(1) %out, ptr addr
 ; GFX12W64-NEXT:    s_cbranch_execz .LBB2_4
 ; GFX12W64-NEXT:  ; %bb.3:
 ; GFX12W64-NEXT:    s_load_b128 s[8:11], s[4:5], 0x34
-; GFX12W64-NEXT:    s_wait_alu 0xfffe
 ; GFX12W64-NEXT:    v_mov_b32_e32 v1, s2
 ; GFX12W64-NEXT:    s_wait_kmcnt 0x0
 ; GFX12W64-NEXT:    buffer_atomic_add_u32 v1, off, s[8:11], null th:TH_ATOMIC_RETURN
@@ -892,7 +888,7 @@ define amdgpu_kernel void @add_i32_varying_vdata(ptr addrspace(1) %out, ptr addr
 ; GFX12W64-NEXT:    s_wait_loadcnt 0x0
 ; GFX12W64-NEXT:    v_readfirstlane_b32 s2, v1
 ; GFX12W64-NEXT:    v_mov_b32_e32 v1, 0
-; GFX12W64-NEXT:    s_wait_alu 0xfffe
+; GFX12W64-NEXT:    s_wait_alu 0xf1ff
 ; GFX12W64-NEXT:    s_delay_alu instid0(VALU_DEP_2)
 ; GFX12W64-NEXT:    v_add_nc_u32_e32 v0, s2, v0
 ; GFX12W64-NEXT:    s_wait_kmcnt 0x0
@@ -907,7 +903,6 @@ define amdgpu_kernel void @add_i32_varying_vdata(ptr addrspace(1) %out, ptr addr
 ; GFX12W32-NEXT:    ; implicit-def: $vgpr0
 ; GFX12W32-NEXT:  .LBB2_1: ; %ComputeLoop
 ; GFX12W32-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX12W32-NEXT:    s_wait_alu 0xfffe
 ; GFX12W32-NEXT:    s_ctz_i32_b32 s2, s1
 ; GFX12W32-NEXT:    s_wait_alu 0xfffe
 ; GFX12W32-NEXT:    v_readlane_b32 s3, v1, s2
@@ -939,6 +934,7 @@ define amdgpu_kernel void @add_i32_varying_vdata(ptr addrspace(1) %out, ptr addr
 ; GFX12W32-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
 ; GFX12W32-NEXT:    s_wait_loadcnt 0x0
 ; GFX12W32-NEXT:    v_readfirstlane_b32 s2, v1
+; GFX12W32-NEXT:    s_wait_alu 0xf1ff
 ; GFX12W32-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12W32-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_add_nc_u32 v0, s2, v0
 ; GFX12W32-NEXT:    s_wait_kmcnt 0x0
@@ -1273,7 +1269,6 @@ define amdgpu_kernel void @sub_i32_constant(ptr addrspace(1) %out, ptr addrspace
 ; GFX12W64-NEXT:    s_mov_b64 s[0:1], exec
 ; GFX12W64-NEXT:    v_mbcnt_lo_u32_b32 v0, s2, 0
 ; GFX12W64-NEXT:    ; implicit-def: $vgpr1
-; GFX12W64-NEXT:    s_wait_alu 0xfffe
 ; GFX12W64-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12W64-NEXT:    v_mbcnt_hi_u32_b32 v0, s3, v0
 ; GFX12W64-NEXT:    v_cmpx_eq_u32_e32 0, v0
@@ -1294,6 +1289,7 @@ define amdgpu_kernel void @sub_i32_constant(ptr addrspace(1) %out, ptr addrspace
 ; GFX12W64-NEXT:    v_readfirstlane_b32 s2, v1
 ; GFX12W64-NEXT:    v_mul_u32_u24_e32 v0, 5, v0
 ; GFX12W64-NEXT:    v_mov_b32_e32 v1, 0
+; GFX12W64-NEXT:    s_wait_alu 0xf1ff
 ; GFX12W64-NEXT:    s_delay_alu instid0(VALU_DEP_2)
 ; GFX12W64-NEXT:    v_sub_nc_u32_e32 v0, s2, v0
 ; GFX12W64-NEXT:    s_wait_kmcnt 0x0
@@ -1311,7 +1307,6 @@ define amdgpu_kernel void @sub_i32_constant(ptr addrspace(1) %out, ptr addrspace
 ; GFX12W32-NEXT:    s_cbranch_execz .LBB4_2
 ; GFX12W32-NEXT:  ; %bb.1:
 ; GFX12W32-NEXT:    s_load_b128 s[8:11], s[4:5], 0x34
-; GFX12W32-NEXT:    s_wait_alu 0xfffe
 ; GFX12W32-NEXT:    s_bcnt1_i32_b32 s1, s1
 ; GFX12W32-NEXT:    s_wait_alu 0xfffe
 ; GFX12W32-NEXT:    s_mul_i32 s1, s1, 5
@@ -1320,7 +1315,6 @@ define amdgpu_kernel void @sub_i32_constant(ptr addrspace(1) %out, ptr addrspace
 ; GFX12W32-NEXT:    s_wait_kmcnt 0x0
 ; GFX12W32-NEXT:    buffer_atomic_sub_u32 v1, off, s[8:11], null th:TH_ATOMIC_RETURN
 ; GFX12W32-NEXT:  .LBB4_2:
-; GFX12W32-NEXT:    s_wait_alu 0xfffe
 ; GFX12W32-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX12W32-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
 ; GFX12W32-NEXT:    s_wait_loadcnt 0x0
@@ -1557,7 +1551,6 @@ define amdgpu_kernel void @sub_i32_uniform(ptr addrspace(1) %out, ptr addrspace(
 ; GFX12W64-NEXT:    s_mov_b64 s[0:1], exec
 ; GFX12W64-NEXT:    v_mbcnt_lo_u32_b32 v0, s2, 0
 ; GFX12W64-NEXT:    ; implicit-def: $vgpr1
-; GFX12W64-NEXT:    s_wait_alu 0xfffe
 ; GFX12W64-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12W64-NEXT:    v_mbcnt_hi_u32_b32 v0, s3, v0
 ; GFX12W64-NEXT:    v_cmpx_eq_u32_e32 0, v0
@@ -1579,6 +1572,7 @@ define amdgpu_kernel void @sub_i32_uniform(ptr addrspace(1) %out, ptr addrspace(
 ; GFX12W64-NEXT:    s_wait_loadcnt 0x0
 ; GFX12W64-NEXT:    v_readfirstlane_b32 s2, v1
 ; GFX12W64-NEXT:    v_mov_b32_e32 v1, 0
+; GFX12W64-NEXT:    s_wait_alu 0xf1ff
 ; GFX12W64-NEXT:    s_delay_alu instid0(VALU_DEP_2)
 ; GFX12W64-NEXT:    v_sub_nc_u32_e32 v0, s2, v0
 ; GFX12W64-NEXT:    global_store_b32 v1, v0, s[0:1]
@@ -1596,7 +1590,6 @@ define amdgpu_kernel void @sub_i32_uniform(ptr addrspace(1) %out, ptr addrspace(
 ; GFX12W32-NEXT:    s_cbranch_execz .LBB5_2
 ; GFX12W32-NEXT:  ; %bb.1:
 ; GFX12W32-NEXT:    s_load_b128 s[8:11], s[4:5], 0x34
-; GFX12W32-NEXT:    s_wait_alu 0xfffe
 ; GFX12W32-NEXT:    s_bcnt1_i32_b32 s2, s2
 ; GFX12W32-NEXT:    s_wait_kmcnt 0x0
 ; GFX12W32-NEXT:    s_wait_alu 0xfffe
@@ -1612,6 +1605,7 @@ define amdgpu_kernel void @sub_i32_uniform(ptr addrspace(1) %out, ptr addrspace(
 ; GFX12W32-NEXT:    s_wait_loadcnt 0x0
 ; GFX12W32-NEXT:    v_readfirstlane_b32 s0, v1
 ; GFX12W32-NEXT:    v_mov_b32_e32 v1, 0
+; GFX12W32-NEXT:    s_wait_alu 0xf1ff
 ; GFX12W32-NEXT:    s_delay_alu instid0(VALU_DEP_2)
 ; GFX12W32-NEXT:    v_sub_nc_u32_e32 v0, s0, v0
 ; GFX12W32-NEXT:    global_store_b32 v1, v0, s[2:3]
@@ -1944,7 +1938,6 @@ define amdgpu_kernel void @sub_i32_varying_vdata(ptr addrspace(1) %out, ptr addr
 ; GFX12W64-NEXT:    s_cbranch_execz .LBB6_4
 ; GFX12W64-NEXT:  ; %bb.3:
 ; GFX12W64-NEXT:    s_load_b128 s[8:11], s[4:5], 0x34
-; GFX12W64-NEXT:    s_wait_alu 0xfffe
 ; GFX12W64-NEXT:    v_mov_b32_e32 v1, s2
 ; GFX12W64-NEXT:    s_wait_kmcnt 0x0
 ; GFX12W64-NEXT:    buffer_atomic_sub_u32 v1, off, s[8:11], null th:TH_ATOMIC_RETURN
@@ -1954,7 +1947,7 @@ define amdgpu_kernel void @sub_i32_varying_vdata(ptr addrspace(1) %out, ptr addr
 ; GFX12W64-NEXT:    s_wait_loadcnt 0x0
 ; GFX12W64-NEXT:    v_readfirstlane_b32 s2, v1
 ; GFX12W64-NEXT:    v_mov_b32_e32 v1, 0
-; GFX12W64-NEXT:    s_wait_alu 0xfffe
+; GFX12W64-NEXT:    s_wait_alu 0xf1ff
 ; GFX12W64-NEXT:    s_delay_alu instid0(VALU_DEP_2)
 ; GFX12W64-NEXT:    v_sub_nc_u32_e32 v0, s2, v0
 ; GFX12W64-NEXT:    s_wait_kmcnt 0x0
@@ -1969,7 +1962,6 @@ define amdgpu_kernel void @sub_i32_varying_vdata(ptr addrspace(1) %out, ptr addr
 ; GFX12W32-NEXT:    ; implicit-def: $vgpr0
 ; GFX12W32-NEXT:  .LBB6_1: ; %ComputeLoop
 ; GFX12W32-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX12W32-NEXT:    s_wait_alu 0xfffe
 ; GFX12W32-NEXT:    s_ctz_i32_b32 s2, s1
 ; GFX12W32-NEXT:    s_wait_alu 0xfffe
 ; GFX12W32-NEXT:    v_readlane_b32 s3, v1, s2
@@ -2002,6 +1994,7 @@ define amdgpu_kernel void @sub_i32_varying_vdata(ptr addrspace(1) %out, ptr addr
 ; GFX12W32-NEXT:    s_wait_loadcnt 0x0
 ; GFX12W32-NEXT:    v_readfirstlane_b32 s2, v1
 ; GFX12W32-NEXT:    v_mov_b32_e32 v1, 0
+; GFX12W32-NEXT:    s_wait_alu 0xf1ff
 ; GFX12W32-NEXT:    s_delay_alu instid0(VALU_DEP_2)
 ; GFX12W32-NEXT:    v_sub_nc_u32_e32 v0, s2, v0
 ; GFX12W32-NEXT:    s_wait_kmcnt 0x0
diff --git a/llvm/test/CodeGen/AMDGPU/atomic_optimizations_struct_buffer.ll b/llvm/test/CodeGen/AMDGPU/atomic_optimizations_struct_buffer.ll
index 9801e6ede5eeb..63b46eba41225 100644
--- a/llvm/test/CodeGen/AMDGPU/atomic_optimizations_struct_buffer.ll
+++ b/llvm/test/CodeGen/AMDGPU/atomic_optimizations_struct_buffer.ll
@@ -226,7 +226,6 @@ define amdgpu_kernel void @add_i32_constant(ptr addrspace(1) %out, ptr addrspace
 ; GFX12W64-NEXT:    s_mov_b64 s[0:1], exec
 ; GFX12W64-NEXT:    v_mbcnt_lo_u32_b32 v0, s2, 0
 ; GFX12W64-NEXT:    ; implicit-def: $vgpr1
-; GFX12W64-NEXT:    s_wait_alu 0xfffe
 ; GFX12W64-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12W64-NEXT:    v_mbcnt_hi_u32_b32 v0, s3, v0
 ; GFX12W64-NEXT:    v_cmpx_eq_u32_e32 0, v0
@@ -247,6 +246,7 @@ define amdgpu_kernel void @add_i32_constant(ptr addrspace(1) %out, ptr addrspace
 ; GFX12W64-NEXT:    s_wait_loadcnt 0x0
 ; GFX12W64-NEXT:    v_readfirstlane_b32 s2, v1
 ; GFX12W64-NEXT:    v_mov_b32_e32 v1, 0
+; GFX12W64-NEXT:    s_wait_alu 0xf1ff
 ; GFX12W64-NEXT:    s_delay_alu instid0(VALU_DEP_2)
 ; GFX12W64-NEXT:    v_mad_u32_u24 v0, v0, 5, s2
 ; GFX12W64-NEXT:    s_wait_kmcnt 0x0
@@ -264,7 +264,6 @@ define amdgpu_kernel void @add_i32_constant(ptr addrspace(1) %out, ptr addrspace
 ; GFX12W32-NEXT:    s_cbranch_execz .LBB0_2
 ; GFX12W32-NEXT:  ; %bb.1:
 ; GFX12W32-NEXT:    s_load_b128 s[8:11], s[4:5], 0x34
-; GFX12W32-NEXT:    s_wait_alu 0xfffe
 ; GFX12W32-NEXT:    s_bcnt1_i32_b32 s1, s1
 ; GFX12W32-NEXT:    s_wait_alu 0xfffe
 ; GFX12W32-NEXT:    s_mul_i32 s1, s1, 5
@@ -273,7 +272,6 @@ define amdgpu_kernel void @add_i32_constant(ptr addrspace(1) %out, ptr addrspace
 ; GFX12W32-NEXT:    s_wait_kmcnt 0x0
 ; GFX12W32-NEXT:    buffer_atomic_add_u32 v1, v2, s[8:11], null idxen th:TH_ATOMIC_RETURN
 ; GFX12W32-NEXT:  .LBB0_2:
-; GFX12W32-NEXT:    s_wait_alu 0xfffe
 ; GFX12W32-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX12W32-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
 ; GFX12W32-NEXT:    s_wait_loadcnt 0x0
@@ -513,7 +511,6 @@ define amdgpu_kernel void @add_i32_uniform(ptr addrspace(1) %out, ptr addrspace(
 ; GFX12W64-NEXT:    s_mov_b64 s[0:1], exec
 ; GFX12W64-NEXT:    v_mbcnt_lo_u32_b32 v0, s2, 0
 ; GFX12W64-NEXT:    ; implicit-def: $vgpr1
-; GFX12W64-NEXT:    s_wait_alu 0xfffe
 ; GFX12W64-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12W64-NEXT:    v_mbcnt_hi_u32_b32 v0, s3, v0
 ; GFX12W64-NEXT:    v_cmpx_eq_u32_e32 0, v0
@@ -534,6 +531,7 @@ define amdgpu_kernel void @add_i32_uniform(ptr addrspace(1) %out, ptr addrspace(
 ; GFX12W64-NEXT:    s_wait_loadcnt 0x0
 ; GFX12W64-NEXT:    v_readfirstlane_b32 s2, v1
 ; GFX12W64-NEXT:    s_wait_kmcnt 0x0
+; GFX12W64-NEXT:    s_wait_alu 0xf1ff
 ; GFX12W64-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12W64-NEXT:    v_mad_co_u64_u32 v[0:1], null, s6, v0, s[2:3]
 ; GFX12W64-NEXT:    v_mov_b32_e32 v1, 0
@@ -552,7 +550,6 @@ define amdgpu_kernel void @add_i32_uniform(ptr addrspace(1) %out, ptr addrspace(
 ; GFX12W32-NEXT:    s_cbranch_execz .LBB1_2
 ; GFX12W32-NEXT:  ; %bb.1:
 ; GFX12W32-NEXT:    s_load_b128 s[8:11], s[4:5], 0x34
-; GFX12W32-NEXT:    s_wait_alu 0xfffe
 ; GFX12W32-NEXT:    s_bcnt1_i32_b32 s2, s2
 ; GFX12W32-NEXT:    s_wait_kmcnt 0x0
 ; GFX12W32-NEXT:    s_wait_alu 0xfffe
@@ -905,7 +902,6 @@ define amdgpu_kernel void @add_i32_varying_vdata(ptr addrspace(1) %out, ptr addr
 ; GFX12W64-NEXT:  ; %bb.3:
 ; GFX12W64-NEXT:    s_load_b128 s[8:11], s[4:5], 0x34
 ; GFX12W64-NEXT:    v_mov_b32_e32 v2, 0
-; GFX12W64-NEXT:    s_wait_alu 0xfffe
 ; GFX12W64-NEXT:    v_mov_b32_e32 v1, s2
 ; GFX12W64-NEXT:    s_wait_kmcnt 0x0
 ; GFX12W64-NEXT:    buffer_atomic_add_u32 v1, v2, s[8:11], null idxen th:TH_ATOMIC_RETURN
@@ -915,7 +911,7 @@ define amdgpu_kernel void @add_i32_varying_vdata(ptr addrspace(1) %out, ptr addr
 ; GFX12W64-NEXT:    s_wait_loadcnt 0x0
 ; GFX12W64-NEXT:    v_readfirstlane_b32 s2, v1
 ; GFX12W64-NEXT:    v_mov_b32_e32 v1, 0
-; GFX12W64-NEXT:    s_wait_alu 0xfffe
+; GFX12W64-NEXT:    s_wait_alu 0xf1ff
 ; GFX12W64-NEXT:    s_delay_alu instid0(VALU_DEP_2)
 ; GFX12W64-NEXT:    v_add_nc_u32_e32 v0, s2, v0
 ; GFX12W64-NEXT:    s_wait_kmcnt 0x0
@@ -930,7 +926,6 @@ define amdgpu_kernel void @add_i32_varying_vdata(ptr addrspace(1) %out, ptr addr
 ; GFX12W32-NEXT:    ; implicit-def: $vgpr0
 ; GFX12W32-NEXT:  .LBB2_1: ; %ComputeLoop
 ; GFX12W32-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX12W32-NEXT:    s_wait_alu 0xfffe
 ; GFX12W32-NEXT:    s_ctz_i32_b32 s2, s1
 ; GFX12W32-NEXT:    s_wait_alu 0xfffe
 ; GFX12W32-NEXT:    v_readlane_b32 s3, v1, s2
@@ -962,6 +957,7 @@ define amdgpu_kernel void @add_i32_varying_vdata(ptr addrspace(1) %out, ptr addr
 ; GFX12W32-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
 ; GFX12W32-NEXT:    s_wait_loadcnt 0x0
 ; GFX12W32-NEXT:    v_readfirstlane_b32 s2, v1
+; GFX12W32-NEXT:    s_wait_alu 0xf1ff
 ; GFX12W32-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12W32-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_add_nc_u32 v0, s2, v0
 ; GFX12W32-NEXT:    s_wait_kmcnt 0x0
@@ -1431,7 +1427,6 @@ define amdgpu_kernel void @sub_i32_constant(ptr addrspace(1) %out, ptr addrspace
 ; GFX12W64-NEXT:    s_mov_b64 s[0:1], exec
 ; GFX12W64-NEXT:    v_mbcnt_lo_u32_b32 v0, s2, 0
 ; GFX12W64-NEXT:    ; implicit-def: $vgpr1
-; GFX12W64-NEXT:    s_wait_alu 0xfffe
 ; GFX12W64-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12W64-NEXT:    v_mbcnt_hi_u32_b32 v0, s3, v0
 ; GFX12W64-NEXT:    v_cmpx_eq_u32_e32 0, v0
@@ -1453,6 +1448,7 @@ define amdgpu_kernel void @sub_i32_constant(ptr addrspace(1) %out, ptr addrspace
 ; GFX12W64-NEXT:    v_readfirstlane_b32 s2, v1
 ; GFX12W64-NEXT:    v_mul_u32_u24_e32 v0, 5, v0
 ; GFX12W64-NEXT:    v_mov_b32_e32 v1, 0
+; GFX12W64-NEXT:    s_wait_alu 0xf1ff
 ; GFX12W64-NEXT:    s_delay_alu instid0(VALU_DEP_2)
 ; GFX12W64-NEXT:    v_sub_nc_u32_e32 v0, s2, v0
 ; GFX12W64-NEXT:    s_wait_kmcnt 0x0
@@ -1470,7 +1466,6 @@ define amdgpu_kernel void @sub_i32_constant(ptr addrspace(1) %out, ptr addrspace
 ; GFX12W32-NEXT:    s_cbranch_execz .LBB5_2
 ; GFX12W32-NEXT:  ; %bb.1:
 ; GFX12W32-NEXT:    s_load_b128 s[8:11], s[4:5], 0x34
-; GFX12W32-NEXT:    s_wait_alu 0xfffe
 ; GFX12W32-NEXT:    s_bcnt1_i32_b32 s1, s1
 ; GFX12W32-NEXT:    s_wait_alu 0xfffe
 ; GFX12W32-NEXT:    s_mul_i32 s1, s1, 5
@@ -1479,7 +1474,6 @@ define amdgpu_kernel void @sub_i32_constant(ptr addrspace(1) %out, ptr addrspace
 ; GFX12W32-NEXT:    s_wait_kmcnt 0x0
 ; GFX12W32-NEXT:    buffer_atomic_sub_u32 v1, v2, s[8:11], null idxen th:TH_ATOMIC_RETURN
 ; GFX12W32-NEXT:  .LBB5_2:
-; GFX12W32-NEXT:    s_wait_alu 0xfffe
 ; GFX12W32-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX12W32-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
 ; GFX12W32-NEXT:    s_wait_loadcnt 0x0
@@ -1723,7 +1717,6 @@ define amdgpu_kernel void @sub_i32_uniform(ptr addrspace(1) %out, ptr addrspace(
 ; GFX12W64-NEXT:    s_mov_b64 s[0:1], exec
 ; GFX12W64-NEXT:    v_mbcnt_lo_u32_b32 v0, s2, 0
 ; GFX12W64-NEXT:    ; implicit-def: $vgpr1
-; GFX12W64-NEXT:    s_wait_alu 0xfffe
 ; GFX12W64-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12W64-NEXT:    v_mbcnt_hi_u32_b32 v0, s3, v0
 ; GFX12W64-NEXT:    v_cmpx_eq_u32_e32 0, v0
@@ -1746,6 +1739,7 @@ define amdgpu_kernel void @sub_i32_uniform(ptr addrspace(1) %out, ptr addrspace(
 ; GFX12W64-NEXT:    s_wait_loadcnt 0x0
 ; GFX12W64-NEXT:    v_readfirstlane_b32 s2, v1
 ; GFX12W64-NEXT:    v_mov_b32_e32 v1, 0
+; GFX12W64-NEXT:    s_wait_alu 0xf1ff
 ; GFX12W64-NEXT:    s_delay_alu instid0(VALU_DEP_2)
 ; GFX12W64-NEXT:    v_sub_nc_u32_e32 v0, s2, v0
 ; GFX12W64-NEXT:    global_store_b32 v1, v0, s[0:1]
@@ -1763,7 +1757,6 @@ define amdgpu_kernel void @sub_i32_uniform(ptr addrspace(1) %out, ptr addrspace(
 ; GFX12W32-NEXT:    s_cbranch_execz .LBB6_2
 ; GFX12W32-NEXT:  ; %bb.1:
 ; GFX12W32-NEXT:    s_load_b128 s[8:11], s[4:5], 0x34
-; GFX12W32-NEXT:    s_wait_alu 0xfffe
 ; GFX12W32-NEXT:    s_bcnt1_i32_b32 s2, s2
 ; GFX12W32-NEXT:    s_wait_kmcnt 0x0
 ; GFX12W32-NEXT:    s_wait_alu 0xfffe
@@ -1779,6 +1772,7 @@ define amdgpu_kernel void @sub_i32_uniform(ptr addrspace(1) %out, ptr addrspace(
 ; GFX12W32-NEXT:    s_wait_loadcnt 0x0
 ; GFX12W32-NEXT:    v_readfirstlane_b32 s0, v1
 ; GFX12W32-NEXT:    v_mov_b32_e32 v1, 0
+; GFX12W32-NEXT:    s_wait_alu 0xf1ff
 ; GFX12W32-NEXT:    s_delay_alu instid0(VALU_DEP_2)
 ; GFX12W32-NEXT:    v_sub_nc_u32_e32 v0, s0, v0
 ; GFX12W32-NEXT:    global_store_b32 v1, v0, s[2:3]
@@ -2118,7 +2112,6 @@ define amdgpu_kernel void @sub_i32_varying_vdata(ptr addrspace(1) %out, ptr addr
 ; GFX12W64-NEXT:  ; %bb.3:
 ; GFX12W64-NEXT:    s_load_b128 s[8:11], s[4:5], 0x34
 ; GFX12W64-NEXT:    v_mov_b32_e32 v2, 0
-; GFX12W64-NEXT:    s_wait_alu 0xfffe
 ; GFX12W64-NEXT:    v_mov_b32_e32 v1, s2
 ; GFX12W64-NEXT:    s_wait_kmcnt 0x0
 ; GFX12W64-NEXT:    buffer_atomic_sub_u32 v1, v2, s[8:11], null idxen th:TH_ATOMIC_RETURN
@@ -2128,7 +2121,7 @@ define amdgpu_kernel void @sub_i32_varying_vdata(ptr addrspace(1) %out, ptr addr
 ; GFX12W64-NEXT:    s_wait_loadcnt 0x0
 ; GFX12W64-NEXT:    v_readfirstlane_b32 s2, v1
 ; GFX12W64-NEXT:    v_mov_b32_e32 v1, 0
-; GFX12W64-NEXT:    s_wait_alu 0xfffe
+; GFX12W64-NEXT:    s_wait_alu 0xf1ff
 ; GFX12W64-NEXT:    s_delay_alu instid0(VALU_DEP_2)
 ; GFX12W64-NEXT:    v_sub_nc_u32_e32 v0, s2, v0
 ; GFX12W64-NEXT:    s_wait_kmcnt 0x0
@@ -2143,7 +2136,6 @@ define amdgpu_kernel void @sub_i32_varying_vdata(ptr addrspace(1) %out, ptr addr
 ; GFX12W32-NEXT:    ; implicit-def: $vgpr0
 ; GFX12W32-NEXT:  .LBB7_1: ; %ComputeLoop
 ; GFX12W32-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX12W32-NEXT:    s_wait_alu 0xfffe
 ; GFX12W32-NEXT:    s_ctz_i32_b32 s2, s1
 ; GFX12W32-NEXT:    s_wait_alu 0xfffe
 ; GFX12W32-NEXT:    v_readlane_b32 s3, v1, s2
@@ -2176,6 +2168,7 @@ define amdgpu_kernel void @sub_i32_varying_vdata(ptr addrspace(1) %out, ptr addr
 ; GFX12W32-NEXT:    s_wait_loadcnt 0x0
 ; GFX12W32-NEXT:    v_readfirstlane_b32 s2, v1
 ; GFX12W32-NEXT:    v_mov_b32_e32 v1, 0
+; GFX12W32-NEXT:    s_wait_alu 0xf1ff
 ; GFX12W32-NEXT:    s_delay_alu instid0(VALU_DEP_2)
 ; GFX12W32-NEXT:    v_sub_nc_u32_e32 v0, s2, v0
 ; GFX12W32-NEXT:    s_wait_kmcnt 0x0
diff --git a/llvm/test/CodeGen/AMDGPU/attributor-flatscratchinit.ll b/llvm/test/CodeGen/AMDGPU/attributor-flatscratchinit.ll
index 25da00e6bde35..349679b9a84b4 100644
--- a/llvm/test/CodeGen/AMDGPU/attributor-flatscratchinit.ll
+++ b/llvm/test/CodeGen/AMDGPU/attributor-flatscratchinit.ll
@@ -614,12 +614,12 @@ define amdgpu_kernel void @with_cast_call_with_private_to_flat_addrspacecast_cc_
 
 define amdgpu_kernel void @private_constant_expression_use(ptr addrspace(1) nocapture %out) {
 ; GFX9-LABEL: define amdgpu_kernel void @private_constant_expression_use(
-; GFX9-SAME: ptr addrspace(1) nocapture [[OUT:%.*]]) #[[ATTR1]] {
+; GFX9-SAME: ptr addrspace(1) captures(none) [[OUT:%.*]]) #[[ATTR1]] {
 ; GFX9-NEXT:    store volatile ptr addrspacecast (ptr addrspace(5) inttoptr (i32 123 to ptr addrspace(5)) to ptr), ptr addrspace(1) [[OUT]], align 8
 ; GFX9-NEXT:    ret void
 ;
 ; GFX10-LABEL: define amdgpu_kernel void @private_constant_expression_use(
-; GFX10-SAME: ptr addrspace(1) nocapture [[OUT:%.*]]) #[[ATTR1]] {
+; GFX10-SAME: ptr addrspace(1) captures(none) [[OUT:%.*]]) #[[ATTR1]] {
 ; GFX10-NEXT:    store volatile ptr addrspacecast (ptr addrspace(5) inttoptr (i32 123 to ptr addrspace(5)) to ptr), ptr addrspace(1) [[OUT]], align 8
 ; GFX10-NEXT:    ret void
 ;
diff --git a/llvm/test/CodeGen/AMDGPU/bitcast_vector_bigint.ll b/llvm/test/CodeGen/AMDGPU/bitcast_vector_bigint.ll
new file mode 100644
index 0000000000000..2c6aabec76330
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/bitcast_vector_bigint.ll
@@ -0,0 +1,351 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 < %s | FileCheck -check-prefix=GFX9 %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1200 < %s | FileCheck -check-prefix=GFX12 %s
+
+; Make sure stack use isn't introduced for these bitcasts.
+
+define i160 @bitcast_v5i32_to_i160(<5 x i32> %vec) {
+; GFX9-LABEL: bitcast_v5i32_to_i160:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX12-LABEL: bitcast_v5i32_to_i160:
+; GFX12:       ; %bb.0:
+; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT:    s_wait_expcnt 0x0
+; GFX12-NEXT:    s_wait_samplecnt 0x0
+; GFX12-NEXT:    s_wait_bvhcnt 0x0
+; GFX12-NEXT:    s_wait_kmcnt 0x0
+; GFX12-NEXT:    s_setpc_b64 s[30:31]
+  %bitcast = bitcast <5 x i32> %vec to i160
+  ret i160 %bitcast
+}
+
+define i192 @bitcast_v6i32_to_i192(<6 x i32> %vec) {
+; GFX9-LABEL: bitcast_v6i32_to_i192:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX12-LABEL: bitcast_v6i32_to_i192:
+; GFX12:       ; %bb.0:
+; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT:    s_wait_expcnt 0x0
+; GFX12-NEXT:    s_wait_samplecnt 0x0
+; GFX12-NEXT:    s_wait_bvhcnt 0x0
+; GFX12-NEXT:    s_wait_kmcnt 0x0
+; GFX12-NEXT:    s_setpc_b64 s[30:31]
+  %bitcast = bitcast <6 x i32> %vec to i192
+  ret i192 %bitcast
+}
+
+define i224 @bitcast_v7i32_to_i224(<7 x i32> %vec) {
+; GFX9-LABEL: bitcast_v7i32_to_i224:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX12-LABEL: bitcast_v7i32_to_i224:
+; GFX12:       ; %bb.0:
+; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT:    s_wait_expcnt 0x0
+; GFX12-NEXT:    s_wait_samplecnt 0x0
+; GFX12-NEXT:    s_wait_bvhcnt 0x0
+; GFX12-NEXT:    s_wait_kmcnt 0x0
+; GFX12-NEXT:    s_setpc_b64 s[30:31]
+  %bitcast = bitcast <7 x i32> %vec to i224
+  ret i224 %bitcast
+}
+
+define i256 @bitcast_v8i32_to_i256(<8 x i32> %vec) {
+; GFX9-LABEL: bitcast_v8i32_to_i256:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX12-LABEL: bitcast_v8i32_to_i256:
+; GFX12:       ; %bb.0:
+; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT:    s_wait_expcnt 0x0
+; GFX12-NEXT:    s_wait_samplecnt 0x0
+; GFX12-NEXT:    s_wait_bvhcnt 0x0
+; GFX12-NEXT:    s_wait_kmcnt 0x0
+; GFX12-NEXT:    s_setpc_b64 s[30:31]
+  %bitcast = bitcast <8 x i32> %vec to i256
+  ret i256 %bitcast
+}
+
+define <5 x i32> @bitcast_i160_to_v5i32(i160 %int) {
+; GFX9-LABEL: bitcast_i160_to_v5i32:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX12-LABEL: bitcast_i160_to_v5i32:
+; GFX12:       ; %bb.0:
+; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT:    s_wait_expcnt 0x0
+; GFX12-NEXT:    s_wait_samplecnt 0x0
+; GFX12-NEXT:    s_wait_bvhcnt 0x0
+; GFX12-NEXT:    s_wait_kmcnt 0x0
+; GFX12-NEXT:    s_setpc_b64 s[30:31]
+  %bitcast = bitcast i160 %int to <5 x i32>
+  ret <5 x i32> %bitcast
+}
+
+define <6 x i32> @bitcast_i192_to_v6i32(i192 %int) {
+; GFX9-LABEL: bitcast_i192_to_v6i32:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX12-LABEL: bitcast_i192_to_v6i32:
+; GFX12:       ; %bb.0:
+; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT:    s_wait_expcnt 0x0
+; GFX12-NEXT:    s_wait_samplecnt 0x0
+; GFX12-NEXT:    s_wait_bvhcnt 0x0
+; GFX12-NEXT:    s_wait_kmcnt 0x0
+; GFX12-NEXT:    s_setpc_b64 s[30:31]
+  %bitcast = bitcast i192 %int to <6 x i32>
+  ret <6 x i32> %bitcast
+}
+
+define <7 x i32> @bitcast_i224_to_v7i32(i224 %int) {
+; GFX9-LABEL: bitcast_i224_to_v7i32:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX12-LABEL: bitcast_i224_to_v7i32:
+; GFX12:       ; %bb.0:
+; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT:    s_wait_expcnt 0x0
+; GFX12-NEXT:    s_wait_samplecnt 0x0
+; GFX12-NEXT:    s_wait_bvhcnt 0x0
+; GFX12-NEXT:    s_wait_kmcnt 0x0
+; GFX12-NEXT:    s_setpc_b64 s[30:31]
+  %bitcast = bitcast i224 %int to <7 x i32>
+  ret <7 x i32> %bitcast
+}
+
+define <8 x i32> @bitcast_i256_to_v8i32(i256 %int) {
+; GFX9-LABEL: bitcast_i256_to_v8i32:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX12-LABEL: bitcast_i256_to_v8i32:
+; GFX12:       ; %bb.0:
+; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT:    s_wait_expcnt 0x0
+; GFX12-NEXT:    s_wait_samplecnt 0x0
+; GFX12-NEXT:    s_wait_bvhcnt 0x0
+; GFX12-NEXT:    s_wait_kmcnt 0x0
+; GFX12-NEXT:    s_setpc_b64 s[30:31]
+  %bitcast = bitcast i256 %int to <8 x i32>
+  ret <8 x i32> %bitcast
+}
+
+define i192 @bitcast_v3i64_to_i192(<3 x i64> %vec) {
+; GFX9-LABEL: bitcast_v3i64_to_i192:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX12-LABEL: bitcast_v3i64_to_i192:
+; GFX12:       ; %bb.0:
+; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT:    s_wait_expcnt 0x0
+; GFX12-NEXT:    s_wait_samplecnt 0x0
+; GFX12-NEXT:    s_wait_bvhcnt 0x0
+; GFX12-NEXT:    s_wait_kmcnt 0x0
+; GFX12-NEXT:    s_setpc_b64 s[30:31]
+  %bitcast = bitcast <3 x i64> %vec to i192
+  ret i192 %bitcast
+}
+
+define <3 x i64> @bitcast_i192_to_v3i64(i192 %int) {
+; GFX9-LABEL: bitcast_i192_to_v3i64:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX12-LABEL: bitcast_i192_to_v3i64:
+; GFX12:       ; %bb.0:
+; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT:    s_wait_expcnt 0x0
+; GFX12-NEXT:    s_wait_samplecnt 0x0
+; GFX12-NEXT:    s_wait_bvhcnt 0x0
+; GFX12-NEXT:    s_wait_kmcnt 0x0
+; GFX12-NEXT:    s_setpc_b64 s[30:31]
+  %bitcast = bitcast i192 %int to <3 x i64>
+  ret <3 x i64> %bitcast
+}
+
+define <10 x i16> @bitcast_i160_to_v10i16(i160 %int) {
+; GFX9-LABEL: bitcast_i160_to_v10i16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    s_mov_b32 s4, 0xffff
+; GFX9-NEXT:    v_bfi_b32 v0, s4, v0, v0
+; GFX9-NEXT:    v_bfi_b32 v2, s4, v2, v2
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX12-LABEL: bitcast_i160_to_v10i16:
+; GFX12:       ; %bb.0:
+; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT:    s_wait_expcnt 0x0
+; GFX12-NEXT:    s_wait_samplecnt 0x0
+; GFX12-NEXT:    s_wait_bvhcnt 0x0
+; GFX12-NEXT:    s_wait_kmcnt 0x0
+; GFX12-NEXT:    v_bfi_b32 v0, 0xffff, v0, v0
+; GFX12-NEXT:    v_bfi_b32 v2, 0xffff, v2, v2
+; GFX12-NEXT:    s_setpc_b64 s[30:31]
+  %bitcast = bitcast i160 %int to <10 x i16>
+  ret <10 x i16> %bitcast
+}
+
+define i160 @bitcast_v10i16_to_i160(<10 x i16> %vec) {
+; GFX9-LABEL: bitcast_v10i16_to_i160:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX12-LABEL: bitcast_v10i16_to_i160:
+; GFX12:       ; %bb.0:
+; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT:    s_wait_expcnt 0x0
+; GFX12-NEXT:    s_wait_samplecnt 0x0
+; GFX12-NEXT:    s_wait_bvhcnt 0x0
+; GFX12-NEXT:    s_wait_kmcnt 0x0
+; GFX12-NEXT:    s_setpc_b64 s[30:31]
+  %bitcast = bitcast <10 x i16> %vec to i160
+  ret i160 %bitcast
+}
+
+define i12 @bitcast_v2i6_to_i12(<2 x i6> %vec) {
+; GFX9-LABEL: bitcast_v2i6_to_i12:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_lshlrev_b16_e32 v1, 6, v1
+; GFX9-NEXT:    v_and_b32_e32 v0, 63, v0
+; GFX9-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX9-NEXT:    v_and_b32_e32 v0, 0xfff, v0
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX12-LABEL: bitcast_v2i6_to_i12:
+; GFX12:       ; %bb.0:
+; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT:    s_wait_expcnt 0x0
+; GFX12-NEXT:    s_wait_samplecnt 0x0
+; GFX12-NEXT:    s_wait_bvhcnt 0x0
+; GFX12-NEXT:    s_wait_kmcnt 0x0
+; GFX12-NEXT:    v_lshlrev_b16 v1, 6, v1
+; GFX12-NEXT:    v_and_b32_e32 v0, 63, v0
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX12-NEXT:    v_and_b32_e32 v0, 0xfff, v0
+; GFX12-NEXT:    s_setpc_b64 s[30:31]
+  %bitcast = bitcast <2 x i6> %vec to i12
+  ret i12 %bitcast
+}
+
+define <2 x i6> @bitcast_i12_to_v2i6(i12 %int) {
+; GFX9-LABEL: bitcast_i12_to_v2i6:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_and_b32_e32 v2, 63, v0
+; GFX9-NEXT:    v_lshrrev_b16_e32 v0, 6, v0
+; GFX9-NEXT:    v_and_b32_e32 v1, 63, v0
+; GFX9-NEXT:    v_mov_b32_e32 v0, v2
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX12-LABEL: bitcast_i12_to_v2i6:
+; GFX12:       ; %bb.0:
+; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT:    s_wait_expcnt 0x0
+; GFX12-NEXT:    s_wait_samplecnt 0x0
+; GFX12-NEXT:    s_wait_bvhcnt 0x0
+; GFX12-NEXT:    s_wait_kmcnt 0x0
+; GFX12-NEXT:    v_lshrrev_b16 v1, 6, v0
+; GFX12-NEXT:    v_and_b32_e32 v0, 63, v0
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2)
+; GFX12-NEXT:    v_and_b32_e32 v1, 63, v1
+; GFX12-NEXT:    s_setpc_b64 s[30:31]
+  %bitcast = bitcast i12 %int to <2 x i6>
+  ret <2 x i6> %bitcast
+}
+
+define i160 @bitcast_v5f32_to_i160(<5 x float> %vec) {
+; GFX9-LABEL: bitcast_v5f32_to_i160:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX12-LABEL: bitcast_v5f32_to_i160:
+; GFX12:       ; %bb.0:
+; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT:    s_wait_expcnt 0x0
+; GFX12-NEXT:    s_wait_samplecnt 0x0
+; GFX12-NEXT:    s_wait_bvhcnt 0x0
+; GFX12-NEXT:    s_wait_kmcnt 0x0
+; GFX12-NEXT:    s_setpc_b64 s[30:31]
+  %bitcast = bitcast <5 x float> %vec to i160
+  ret i160 %bitcast
+}
+
+define <5 x float> @bitcast_i160_to_v5f32(i160 %int) {
+; GFX9-LABEL: bitcast_i160_to_v5f32:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX12-LABEL: bitcast_i160_to_v5f32:
+; GFX12:       ; %bb.0:
+; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT:    s_wait_expcnt 0x0
+; GFX12-NEXT:    s_wait_samplecnt 0x0
+; GFX12-NEXT:    s_wait_bvhcnt 0x0
+; GFX12-NEXT:    s_wait_kmcnt 0x0
+; GFX12-NEXT:    s_setpc_b64 s[30:31]
+  %bitcast = bitcast i160 %int to <5 x float>
+  ret <5 x float> %bitcast
+}
+
+define <6 x float> @bitcast_i192_to_v6f32(i192 %int) {
+; GFX9-LABEL: bitcast_i192_to_v6f32:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX12-LABEL: bitcast_i192_to_v6f32:
+; GFX12:       ; %bb.0:
+; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT:    s_wait_expcnt 0x0
+; GFX12-NEXT:    s_wait_samplecnt 0x0
+; GFX12-NEXT:    s_wait_bvhcnt 0x0
+; GFX12-NEXT:    s_wait_kmcnt 0x0
+; GFX12-NEXT:    s_setpc_b64 s[30:31]
+  %bitcast = bitcast i192 %int to <6 x float>
+  ret <6 x float> %bitcast
+}
+
+define i192 @bitcast_v6f32_to_i192(<6 x float> %vec) {
+; GFX9-LABEL: bitcast_v6f32_to_i192:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX12-LABEL: bitcast_v6f32_to_i192:
+; GFX12:       ; %bb.0:
+; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT:    s_wait_expcnt 0x0
+; GFX12-NEXT:    s_wait_samplecnt 0x0
+; GFX12-NEXT:    s_wait_bvhcnt 0x0
+; GFX12-NEXT:    s_wait_kmcnt 0x0
+; GFX12-NEXT:    s_setpc_b64 s[30:31]
+  %bitcast = bitcast <6 x float> %vec to i192
+  ret i192 %bitcast
+}
diff --git a/llvm/test/CodeGen/AMDGPU/blender-coalescer-verifier-error-empty-subrange.mir b/llvm/test/CodeGen/AMDGPU/blender-coalescer-verifier-error-empty-subrange.mir
index 007c4c094029c..c9b47a4575200 100644
--- a/llvm/test/CodeGen/AMDGPU/blender-coalescer-verifier-error-empty-subrange.mir
+++ b/llvm/test/CodeGen/AMDGPU/blender-coalescer-verifier-error-empty-subrange.mir
@@ -1,5 +1,6 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 2
 # RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1031 -run-pass=register-coalescer -verify-coalescing -o - %s | FileCheck %s
+# RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1031 -passes=register-coalescer -verify-coalescing -o - %s | FileCheck %s
 
 # Testcase variants from
 # liveout-implicit-def-subreg-redef-blender-verifier-error.mir which
diff --git a/llvm/test/CodeGen/AMDGPU/branch-relaxation.ll b/llvm/test/CodeGen/AMDGPU/branch-relaxation.ll
index ff47c865c67e6..a03ad4daab014 100644
--- a/llvm/test/CodeGen/AMDGPU/branch-relaxation.ll
+++ b/llvm/test/CodeGen/AMDGPU/branch-relaxation.ll
@@ -417,12 +417,13 @@ define amdgpu_kernel void @min_long_forward_vbranch(ptr addrspace(1) %arg) #0 {
 ; GFX12:       ; %bb.0: ; %bb
 ; GFX12-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
 ; GFX12-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
 ; GFX12-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-NEXT:    global_load_b32 v2, v0, s[0:1] scope:SCOPE_SYS
 ; GFX12-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-NEXT:    v_add_co_u32 v0, s0, s0, v0
+; GFX12-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-NEXT:    v_add_co_ci_u32_e64 v1, null, s1, 0, s0
 ; GFX12-NEXT:    s_mov_b32 s0, exec_lo
 ; GFX12-NEXT:    v_cmpx_ne_u32_e32 0, v2
@@ -1495,7 +1496,6 @@ define amdgpu_kernel void @long_branch_hang(ptr addrspace(1) nocapture %arg, i32
 ; GFX12-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24
 ; GFX12-NEXT:    v_mov_b32_e32 v1, 0
 ; GFX12-NEXT:    s_wait_kmcnt 0x0
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_lshl_b64 s[0:1], s[0:1], 2
 ; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[2:3], s[0:1]
diff --git a/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fadd.ll b/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fadd.ll
index a969e3d4f4f79..0ea73ad4c5019 100644
--- a/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fadd.ll
+++ b/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fadd.ll
@@ -382,10 +382,10 @@ define float @buffer_fat_ptr_agent_atomic_fadd_ret_f32__offset__waterfall__amdgp
 ; GFX12-NEXT:    v_readfirstlane_b32 s5, v1
 ; GFX12-NEXT:    v_readfirstlane_b32 s6, v2
 ; GFX12-NEXT:    v_readfirstlane_b32 s7, v3
+; GFX12-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
 ; GFX12-NEXT:    v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
 ; GFX12-NEXT:    v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-NEXT:    s_and_b32 s0, vcc_lo, s0
 ; GFX12-NEXT:    s_wait_alu 0xfffe
@@ -394,7 +394,6 @@ define float @buffer_fat_ptr_agent_atomic_fadd_ret_f32__offset__waterfall__amdgp
 ; GFX12-NEXT:    buffer_atomic_add_f32 v5, v4, s[4:7], null offen offset:1024 th:TH_ATOMIC_RETURN
 ; GFX12-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
 ; GFX12-NEXT:    ; implicit-def: $vgpr4
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_xor_b32 exec_lo, exec_lo, s0
 ; GFX12-NEXT:    s_cbranch_execnz .LBB2_1
 ; GFX12-NEXT:  ; %bb.2:
@@ -402,7 +401,6 @@ define float @buffer_fat_ptr_agent_atomic_fadd_ret_f32__offset__waterfall__amdgp
 ; GFX12-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-NEXT:    v_mov_b32_e32 v0, v5
 ; GFX12-NEXT:    global_inv scope:SCOPE_DEV
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_f32__offset__waterfall__amdgpu_no_fine_grained_memory__amdgpu_ignore_denormal_mode:
@@ -1894,14 +1892,12 @@ define double @buffer_fat_ptr_agent_atomic_fadd_ret_f64__offset__amdgpu_no_fine_
 ; GFX12-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-NEXT:    global_inv scope:SCOPE_DEV
 ; GFX12-NEXT:    v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[9:10]
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_or_b32 s4, vcc_lo, s4
 ; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s4
 ; GFX12-NEXT:    s_cbranch_execnz .LBB8_1
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s4
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_f64__offset__amdgpu_no_fine_grained_memory:
@@ -2145,14 +2141,12 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_f64__offset__amdgpu_no_fine_
 ; GFX12-NEXT:    global_inv scope:SCOPE_DEV
 ; GFX12-NEXT:    v_cmp_eq_u64_e32 vcc_lo, v[7:8], v[4:5]
 ; GFX12-NEXT:    v_dual_mov_b32 v4, v7 :: v_dual_mov_b32 v5, v8
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_or_b32 s4, vcc_lo, s4
 ; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s4
 ; GFX12-NEXT:    s_cbranch_execnz .LBB9_1
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s4
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: buffer_fat_ptr_agent_atomic_fadd_noret_f64__offset__amdgpu_no_fine_grained_memory:
@@ -2376,10 +2370,10 @@ define double @buffer_fat_ptr_agent_atomic_fadd_ret_f64__offset__waterfall__amdg
 ; GFX12-NEXT:    v_readfirstlane_b32 s5, v10
 ; GFX12-NEXT:    v_readfirstlane_b32 s6, v7
 ; GFX12-NEXT:    v_readfirstlane_b32 s7, v8
+; GFX12-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
 ; GFX12-NEXT:    v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[9:10]
 ; GFX12-NEXT:    v_cmp_eq_u64_e64 s0, s[6:7], v[7:8]
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-NEXT:    s_and_b32 s0, vcc_lo, s0
 ; GFX12-NEXT:    s_wait_alu 0xfffe
@@ -2387,7 +2381,6 @@ define double @buffer_fat_ptr_agent_atomic_fadd_ret_f64__offset__waterfall__amdg
 ; GFX12-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-NEXT:    buffer_load_b64 v[13:14], v4, s[4:7], null offen offset:2048
 ; GFX12-NEXT:    ; implicit-def: $vgpr4
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_xor_b32 exec_lo, exec_lo, s0
 ; GFX12-NEXT:    s_cbranch_execnz .LBB10_1
 ; GFX12-NEXT:  ; %bb.2:
@@ -2409,17 +2402,16 @@ define double @buffer_fat_ptr_agent_atomic_fadd_ret_f64__offset__waterfall__amdg
 ; GFX12-NEXT:    v_readfirstlane_b32 s5, v10
 ; GFX12-NEXT:    v_readfirstlane_b32 s6, v7
 ; GFX12-NEXT:    v_readfirstlane_b32 s7, v8
+; GFX12-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
 ; GFX12-NEXT:    v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[9:10]
 ; GFX12-NEXT:    v_cmp_eq_u64_e64 s0, s[6:7], v[7:8]
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-NEXT:    s_and_b32 s0, vcc_lo, s0
 ; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_and_saveexec_b32 s0, s0
 ; GFX12-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-NEXT:    buffer_atomic_cmpswap_b64 v[0:3], v15, s[4:7], null offen th:TH_ATOMIC_RETURN
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_xor_b32 exec_lo, exec_lo, s0
 ; GFX12-NEXT:    s_cbranch_execnz .LBB10_4
 ; GFX12-NEXT:  ; %bb.5: ; in Loop: Header=BB10_3 Depth=1
@@ -2434,7 +2426,6 @@ define double @buffer_fat_ptr_agent_atomic_fadd_ret_f64__offset__waterfall__amdg
 ; GFX12-NEXT:    s_cbranch_execnz .LBB10_3
 ; GFX12-NEXT:  ; %bb.6: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s1
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_f64__offset__waterfall__amdgpu_no_fine_grained_memory:
@@ -2921,14 +2912,12 @@ define double @buffer_fat_ptr_agent_atomic_fadd_ret_f64__offset__amdgpu_no_remot
 ; GFX12-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-NEXT:    global_inv scope:SCOPE_DEV
 ; GFX12-NEXT:    v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[9:10]
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_or_b32 s4, vcc_lo, s4
 ; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s4
 ; GFX12-NEXT:    s_cbranch_execnz .LBB11_1
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s4
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_f64__offset__amdgpu_no_remote_memory:
@@ -3192,14 +3181,12 @@ define double @buffer_fat_ptr_agent_atomic_fadd_ret_f64__offset__amdgpu_no_fine_
 ; GFX12-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-NEXT:    global_inv scope:SCOPE_DEV
 ; GFX12-NEXT:    v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[9:10]
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_or_b32 s4, vcc_lo, s4
 ; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s4
 ; GFX12-NEXT:    s_cbranch_execnz .LBB12_1
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s4
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_f64__offset__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
@@ -3439,7 +3426,6 @@ define half @buffer_fat_ptr_agent_atomic_fadd_ret_f16__offset__amdgpu_no_fine_gr
 ; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_lshl_b32 s5, 0xffff, s4
 ; GFX12-NEXT:    buffer_load_b32 v2, v5, s[0:3], null offen
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_not_b32 s6, s5
 ; GFX12-NEXT:    s_mov_b32 s5, 0
 ; GFX12-NEXT:  .LBB13_1: ; %atomicrmw.start
@@ -3468,7 +3454,6 @@ define half @buffer_fat_ptr_agent_atomic_fadd_ret_f16__offset__amdgpu_no_fine_gr
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s5
 ; GFX12-NEXT:    v_lshrrev_b32_e32 v0, s4, v3
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_f16__offset__amdgpu_no_fine_grained_memory:
@@ -3790,7 +3775,6 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_f16__offset__amdgpu_no_fine_
 ; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_lshl_b32 s5, 0xffff, s4
 ; GFX12-NEXT:    buffer_load_b32 v2, v3, s[0:3], null offen
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_not_b32 s6, s5
 ; GFX12-NEXT:    s_mov_b32 s5, 0
 ; GFX12-NEXT:  .LBB14_1: ; %atomicrmw.start
@@ -3818,7 +3802,6 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_f16__offset__amdgpu_no_fine_
 ; GFX12-NEXT:    s_cbranch_execnz .LBB14_1
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s5
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: buffer_fat_ptr_agent_atomic_fadd_noret_f16__offset__amdgpu_no_fine_grained_memory:
@@ -4133,17 +4116,16 @@ define half @buffer_fat_ptr_agent_atomic_fadd_ret_f16__offset__waterfall__amdgpu
 ; GFX12-NEXT:    v_readfirstlane_b32 s5, v1
 ; GFX12-NEXT:    v_readfirstlane_b32 s6, v2
 ; GFX12-NEXT:    v_readfirstlane_b32 s7, v3
+; GFX12-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
 ; GFX12-NEXT:    v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
 ; GFX12-NEXT:    v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-NEXT:    s_and_b32 s0, vcc_lo, s0
 ; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_and_saveexec_b32 s0, s0
 ; GFX12-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-NEXT:    buffer_load_b32 v7, v10, s[4:7], null offen
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_xor_b32 exec_lo, exec_lo, s0
 ; GFX12-NEXT:    s_cbranch_execnz .LBB15_1
 ; GFX12-NEXT:  ; %bb.2:
@@ -4170,17 +4152,16 @@ define half @buffer_fat_ptr_agent_atomic_fadd_ret_f16__offset__waterfall__amdgpu
 ; GFX12-NEXT:    v_readfirstlane_b32 s5, v1
 ; GFX12-NEXT:    v_readfirstlane_b32 s6, v2
 ; GFX12-NEXT:    v_readfirstlane_b32 s7, v3
+; GFX12-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
 ; GFX12-NEXT:    v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
 ; GFX12-NEXT:    v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-NEXT:    s_and_b32 s0, vcc_lo, s0
 ; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_and_saveexec_b32 s0, s0
 ; GFX12-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-NEXT:    buffer_atomic_cmpswap_b32 v[8:9], v10, s[4:7], null offen th:TH_ATOMIC_RETURN
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_xor_b32 exec_lo, exec_lo, s0
 ; GFX12-NEXT:    s_cbranch_execnz .LBB15_4
 ; GFX12-NEXT:  ; %bb.5: ; in Loop: Header=BB15_3 Depth=1
@@ -4196,7 +4177,6 @@ define half @buffer_fat_ptr_agent_atomic_fadd_ret_f16__offset__waterfall__amdgpu
 ; GFX12-NEXT:  ; %bb.6: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s1
 ; GFX12-NEXT:    v_lshrrev_b32_e32 v0, v4, v8
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_f16__offset__waterfall__amdgpu_no_fine_grained_memory:
@@ -4779,7 +4759,6 @@ define bfloat @buffer_fat_ptr_agent_atomic_fadd_ret_bf16__offset__amdgpu_no_fine
 ; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_lshl_b32 s5, 0xffff, s4
 ; GFX12-NEXT:    buffer_load_b32 v1, v4, s[0:3], null offen
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_not_b32 s6, s5
 ; GFX12-NEXT:    s_mov_b32 s5, 0
 ; GFX12-NEXT:  .LBB16_1: ; %atomicrmw.start
@@ -4795,6 +4774,7 @@ define bfloat @buffer_fat_ptr_agent_atomic_fadd_ret_bf16__offset__amdgpu_no_fine
 ; GFX12-NEXT:    v_or_b32_e32 v3, 0x400000, v0
 ; GFX12-NEXT:    v_cmp_u_f32_e32 vcc_lo, v0, v0
 ; GFX12-NEXT:    v_add3_u32 v2, v2, v0, 0x7fff
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_cndmask_b32_e32 v0, v2, v3, vcc_lo
 ; GFX12-NEXT:    v_lshrrev_b32_e32 v0, 16, v0
@@ -4816,7 +4796,6 @@ define bfloat @buffer_fat_ptr_agent_atomic_fadd_ret_bf16__offset__amdgpu_no_fine
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s5
 ; GFX12-NEXT:    v_lshrrev_b32_e32 v0, s4, v2
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_bf16__offset__amdgpu_no_fine_grained_memory:
@@ -5185,7 +5164,6 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_bf16__offset__amdgpu_no_fine
 ; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_lshl_b32 s5, 0xffff, s4
 ; GFX12-NEXT:    buffer_load_b32 v1, v2, s[0:3], null offen
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_not_b32 s6, s5
 ; GFX12-NEXT:    s_mov_b32 s5, 0
 ; GFX12-NEXT:  .LBB17_1: ; %atomicrmw.start
@@ -5201,6 +5179,7 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_bf16__offset__amdgpu_no_fine
 ; GFX12-NEXT:    v_or_b32_e32 v5, 0x400000, v0
 ; GFX12-NEXT:    v_cmp_u_f32_e32 vcc_lo, v0, v0
 ; GFX12-NEXT:    v_add3_u32 v4, v4, v0, 0x7fff
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_cndmask_b32_e32 v0, v4, v5, vcc_lo
 ; GFX12-NEXT:    v_lshrrev_b32_e32 v0, 16, v0
@@ -5221,7 +5200,6 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_bf16__offset__amdgpu_no_fine
 ; GFX12-NEXT:    s_cbranch_execnz .LBB17_1
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s5
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: buffer_fat_ptr_agent_atomic_fadd_noret_bf16__offset__amdgpu_no_fine_grained_memory:
@@ -5582,17 +5560,16 @@ define bfloat @buffer_fat_ptr_agent_atomic_fadd_ret_bf16__offset__waterfall__amd
 ; GFX12-NEXT:    v_readfirstlane_b32 s5, v1
 ; GFX12-NEXT:    v_readfirstlane_b32 s6, v2
 ; GFX12-NEXT:    v_readfirstlane_b32 s7, v3
+; GFX12-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
 ; GFX12-NEXT:    v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
 ; GFX12-NEXT:    v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-NEXT:    s_and_b32 s0, vcc_lo, s0
 ; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_and_saveexec_b32 s0, s0
 ; GFX12-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-NEXT:    buffer_load_b32 v6, v8, s[4:7], null offen
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_xor_b32 exec_lo, exec_lo, s0
 ; GFX12-NEXT:    s_cbranch_execnz .LBB18_1
 ; GFX12-NEXT:  ; %bb.2:
@@ -5614,6 +5591,7 @@ define bfloat @buffer_fat_ptr_agent_atomic_fadd_ret_bf16__offset__waterfall__amd
 ; GFX12-NEXT:    v_or_b32_e32 v11, 0x400000, v4
 ; GFX12-NEXT:    v_cmp_u_f32_e32 vcc_lo, v4, v4
 ; GFX12-NEXT:    v_add3_u32 v5, v5, v4, 0x7fff
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_cndmask_b32_e32 v4, v5, v11, vcc_lo
 ; GFX12-NEXT:    v_lshrrev_b32_e32 v4, 16, v4
@@ -5629,17 +5607,16 @@ define bfloat @buffer_fat_ptr_agent_atomic_fadd_ret_bf16__offset__waterfall__amd
 ; GFX12-NEXT:    v_readfirstlane_b32 s5, v1
 ; GFX12-NEXT:    v_readfirstlane_b32 s6, v2
 ; GFX12-NEXT:    v_readfirstlane_b32 s7, v3
+; GFX12-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
 ; GFX12-NEXT:    v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
 ; GFX12-NEXT:    v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-NEXT:    s_and_b32 s0, vcc_lo, s0
 ; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_and_saveexec_b32 s0, s0
 ; GFX12-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-NEXT:    buffer_atomic_cmpswap_b32 v[4:5], v8, s[4:7], null offen th:TH_ATOMIC_RETURN
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_xor_b32 exec_lo, exec_lo, s0
 ; GFX12-NEXT:    s_cbranch_execnz .LBB18_4
 ; GFX12-NEXT:  ; %bb.5: ; in Loop: Header=BB18_3 Depth=1
@@ -5655,7 +5632,6 @@ define bfloat @buffer_fat_ptr_agent_atomic_fadd_ret_bf16__offset__waterfall__amd
 ; GFX12-NEXT:  ; %bb.6: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s1
 ; GFX12-NEXT:    v_lshrrev_b32_e32 v0, v7, v4
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_bf16__offset__waterfall__amdgpu_no_fine_grained_memory:
@@ -6759,10 +6735,10 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fadd_ret_v2f16__offset__waterfall
 ; GFX12-NEXT:    v_readfirstlane_b32 s5, v1
 ; GFX12-NEXT:    v_readfirstlane_b32 s6, v2
 ; GFX12-NEXT:    v_readfirstlane_b32 s7, v3
+; GFX12-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
 ; GFX12-NEXT:    v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
 ; GFX12-NEXT:    v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-NEXT:    s_and_b32 s0, vcc_lo, s0
 ; GFX12-NEXT:    s_wait_alu 0xfffe
@@ -6771,7 +6747,6 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fadd_ret_v2f16__offset__waterfall
 ; GFX12-NEXT:    buffer_atomic_pk_add_f16 v5, v4, s[4:7], null offen offset:1024 th:TH_ATOMIC_RETURN
 ; GFX12-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
 ; GFX12-NEXT:    ; implicit-def: $vgpr4
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_xor_b32 exec_lo, exec_lo, s0
 ; GFX12-NEXT:    s_cbranch_execnz .LBB21_1
 ; GFX12-NEXT:  ; %bb.2:
@@ -6779,7 +6754,6 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fadd_ret_v2f16__offset__waterfall
 ; GFX12-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-NEXT:    v_mov_b32_e32 v0, v5
 ; GFX12-NEXT:    global_inv scope:SCOPE_DEV
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_v2f16__offset__waterfall__amdgpu_no_fine_grained_memory:
@@ -9080,10 +9054,10 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__waterf
 ; GFX12-NEXT:    v_readfirstlane_b32 s5, v1
 ; GFX12-NEXT:    v_readfirstlane_b32 s6, v2
 ; GFX12-NEXT:    v_readfirstlane_b32 s7, v3
+; GFX12-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
 ; GFX12-NEXT:    v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
 ; GFX12-NEXT:    v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-NEXT:    s_and_b32 s0, vcc_lo, s0
 ; GFX12-NEXT:    s_wait_alu 0xfffe
@@ -9092,7 +9066,6 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__waterf
 ; GFX12-NEXT:    buffer_atomic_pk_add_bf16 v5, v4, s[4:7], null offen offset:1024 th:TH_ATOMIC_RETURN
 ; GFX12-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
 ; GFX12-NEXT:    ; implicit-def: $vgpr4
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_xor_b32 exec_lo, exec_lo, s0
 ; GFX12-NEXT:    s_cbranch_execnz .LBB28_1
 ; GFX12-NEXT:  ; %bb.2:
@@ -9100,7 +9073,6 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__waterf
 ; GFX12-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-NEXT:    v_mov_b32_e32 v0, v5
 ; GFX12-NEXT:    global_inv scope:SCOPE_DEV
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__waterfall__amdgpu_no_fine_grained_memory:
diff --git a/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fmax.ll b/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fmax.ll
index c7511a2df9fe1..7f06d169a6b13 100644
--- a/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fmax.ll
+++ b/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fmax.ll
@@ -374,10 +374,10 @@ define float @buffer_fat_ptr_agent_atomic_fmax_ret_f32__offset__waterfall__amdgp
 ; GFX12-NEXT:    v_readfirstlane_b32 s5, v1
 ; GFX12-NEXT:    v_readfirstlane_b32 s6, v2
 ; GFX12-NEXT:    v_readfirstlane_b32 s7, v3
+; GFX12-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
 ; GFX12-NEXT:    v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
 ; GFX12-NEXT:    v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-NEXT:    s_and_b32 s0, vcc_lo, s0
 ; GFX12-NEXT:    s_wait_alu 0xfffe
@@ -386,7 +386,6 @@ define float @buffer_fat_ptr_agent_atomic_fmax_ret_f32__offset__waterfall__amdgp
 ; GFX12-NEXT:    buffer_atomic_max_num_f32 v5, v4, s[4:7], null offen offset:1024 th:TH_ATOMIC_RETURN
 ; GFX12-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
 ; GFX12-NEXT:    ; implicit-def: $vgpr4
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_xor_b32 exec_lo, exec_lo, s0
 ; GFX12-NEXT:    s_cbranch_execnz .LBB2_1
 ; GFX12-NEXT:  ; %bb.2:
@@ -394,7 +393,6 @@ define float @buffer_fat_ptr_agent_atomic_fmax_ret_f32__offset__waterfall__amdgp
 ; GFX12-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-NEXT:    v_mov_b32_e32 v0, v5
 ; GFX12-NEXT:    global_inv scope:SCOPE_DEV
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_f32__offset__waterfall__amdgpu_no_fine_grained_memory:
@@ -1220,14 +1218,12 @@ define double @buffer_fat_ptr_agent_atomic_fmax_ret_f64__offset__amdgpu_no_fine_
 ; GFX12-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-NEXT:    global_inv scope:SCOPE_DEV
 ; GFX12-NEXT:    v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[9:10]
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_or_b32 s4, vcc_lo, s4
 ; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s4
 ; GFX12-NEXT:    s_cbranch_execnz .LBB5_1
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s4
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_f64__offset__amdgpu_no_fine_grained_memory:
@@ -1413,14 +1409,12 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_f64__offset__amdgpu_no_fine_
 ; GFX12-NEXT:    global_inv scope:SCOPE_DEV
 ; GFX12-NEXT:    v_cmp_eq_u64_e32 vcc_lo, v[7:8], v[2:3]
 ; GFX12-NEXT:    v_dual_mov_b32 v2, v7 :: v_dual_mov_b32 v3, v8
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_or_b32 s4, vcc_lo, s4
 ; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s4
 ; GFX12-NEXT:    s_cbranch_execnz .LBB6_1
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s4
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: buffer_fat_ptr_agent_atomic_fmax_noret_f64__offset__amdgpu_no_fine_grained_memory:
@@ -1589,10 +1583,10 @@ define double @buffer_fat_ptr_agent_atomic_fmax_ret_f64__offset__waterfall__amdg
 ; GFX12-NEXT:    v_readfirstlane_b32 s5, v10
 ; GFX12-NEXT:    v_readfirstlane_b32 s6, v7
 ; GFX12-NEXT:    v_readfirstlane_b32 s7, v8
+; GFX12-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
 ; GFX12-NEXT:    v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[9:10]
 ; GFX12-NEXT:    v_cmp_eq_u64_e64 s0, s[6:7], v[7:8]
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-NEXT:    s_and_b32 s0, vcc_lo, s0
 ; GFX12-NEXT:    s_wait_alu 0xfffe
@@ -1600,7 +1594,6 @@ define double @buffer_fat_ptr_agent_atomic_fmax_ret_f64__offset__waterfall__amdg
 ; GFX12-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-NEXT:    buffer_load_b64 v[13:14], v4, s[4:7], null offen offset:2048
 ; GFX12-NEXT:    ; implicit-def: $vgpr4
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_xor_b32 exec_lo, exec_lo, s0
 ; GFX12-NEXT:    s_cbranch_execnz .LBB7_1
 ; GFX12-NEXT:  ; %bb.2:
@@ -1624,17 +1617,16 @@ define double @buffer_fat_ptr_agent_atomic_fmax_ret_f64__offset__waterfall__amdg
 ; GFX12-NEXT:    v_readfirstlane_b32 s5, v10
 ; GFX12-NEXT:    v_readfirstlane_b32 s6, v7
 ; GFX12-NEXT:    v_readfirstlane_b32 s7, v8
+; GFX12-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
 ; GFX12-NEXT:    v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[9:10]
 ; GFX12-NEXT:    v_cmp_eq_u64_e64 s0, s[6:7], v[7:8]
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-NEXT:    s_and_b32 s0, vcc_lo, s0
 ; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_and_saveexec_b32 s0, s0
 ; GFX12-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-NEXT:    buffer_atomic_cmpswap_b64 v[0:3], v15, s[4:7], null offen th:TH_ATOMIC_RETURN
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_xor_b32 exec_lo, exec_lo, s0
 ; GFX12-NEXT:    s_cbranch_execnz .LBB7_4
 ; GFX12-NEXT:  ; %bb.5: ; in Loop: Header=BB7_3 Depth=1
@@ -1649,7 +1641,6 @@ define double @buffer_fat_ptr_agent_atomic_fmax_ret_f64__offset__waterfall__amdg
 ; GFX12-NEXT:    s_cbranch_execnz .LBB7_3
 ; GFX12-NEXT:  ; %bb.6: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s1
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_f64__offset__waterfall__amdgpu_no_fine_grained_memory:
@@ -2034,14 +2025,12 @@ define double @buffer_fat_ptr_agent_atomic_fmax_ret_f64__offset__amdgpu_no_remot
 ; GFX12-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-NEXT:    global_inv scope:SCOPE_DEV
 ; GFX12-NEXT:    v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[9:10]
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_or_b32 s4, vcc_lo, s4
 ; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s4
 ; GFX12-NEXT:    s_cbranch_execnz .LBB8_1
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s4
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_f64__offset__amdgpu_no_remote_memory:
@@ -2323,14 +2312,12 @@ define double @buffer_fat_ptr_agent_atomic_fmax_ret_f64__offset__amdgpu_no_fine_
 ; GFX12-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-NEXT:    global_inv scope:SCOPE_DEV
 ; GFX12-NEXT:    v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[9:10]
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_or_b32 s4, vcc_lo, s4
 ; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s4
 ; GFX12-NEXT:    s_cbranch_execnz .LBB9_1
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s4
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_f64__offset__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
@@ -2511,7 +2498,6 @@ define half @buffer_fat_ptr_agent_atomic_fmax_ret_f16__offset__amdgpu_no_fine_gr
 ; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_lshl_b32 s5, 0xffff, s4
 ; GFX12-NEXT:    buffer_load_b32 v1, v4, s[0:3], null offen
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_not_b32 s6, s5
 ; GFX12-NEXT:    s_mov_b32 s5, 0
 ; GFX12-NEXT:  .LBB10_1: ; %atomicrmw.start
@@ -2541,7 +2527,6 @@ define half @buffer_fat_ptr_agent_atomic_fmax_ret_f16__offset__amdgpu_no_fine_gr
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s5
 ; GFX12-NEXT:    v_lshrrev_b32_e32 v0, s4, v2
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_f16__offset__amdgpu_no_fine_grained_memory:
@@ -2877,7 +2862,6 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_f16__offset__amdgpu_no_fine_
 ; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_lshl_b32 s5, 0xffff, s4
 ; GFX12-NEXT:    buffer_load_b32 v1, v2, s[0:3], null offen
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_not_b32 s6, s5
 ; GFX12-NEXT:    s_mov_b32 s5, 0
 ; GFX12-NEXT:  .LBB11_1: ; %atomicrmw.start
@@ -2906,7 +2890,6 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_f16__offset__amdgpu_no_fine_
 ; GFX12-NEXT:    s_cbranch_execnz .LBB11_1
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s5
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: buffer_fat_ptr_agent_atomic_fmax_noret_f16__offset__amdgpu_no_fine_grained_memory:
@@ -3234,17 +3217,16 @@ define half @buffer_fat_ptr_agent_atomic_fmax_ret_f16__offset__waterfall__amdgpu
 ; GFX12-NEXT:    v_readfirstlane_b32 s5, v1
 ; GFX12-NEXT:    v_readfirstlane_b32 s6, v2
 ; GFX12-NEXT:    v_readfirstlane_b32 s7, v3
+; GFX12-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
 ; GFX12-NEXT:    v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
 ; GFX12-NEXT:    v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-NEXT:    s_and_b32 s0, vcc_lo, s0
 ; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_and_saveexec_b32 s0, s0
 ; GFX12-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-NEXT:    buffer_load_b32 v6, v8, s[4:7], null offen
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_xor_b32 exec_lo, exec_lo, s0
 ; GFX12-NEXT:    s_cbranch_execnz .LBB12_1
 ; GFX12-NEXT:  ; %bb.2:
@@ -3274,17 +3256,16 @@ define half @buffer_fat_ptr_agent_atomic_fmax_ret_f16__offset__waterfall__amdgpu
 ; GFX12-NEXT:    v_readfirstlane_b32 s5, v1
 ; GFX12-NEXT:    v_readfirstlane_b32 s6, v2
 ; GFX12-NEXT:    v_readfirstlane_b32 s7, v3
+; GFX12-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
 ; GFX12-NEXT:    v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
 ; GFX12-NEXT:    v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-NEXT:    s_and_b32 s0, vcc_lo, s0
 ; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_and_saveexec_b32 s0, s0
 ; GFX12-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-NEXT:    buffer_atomic_cmpswap_b32 v[4:5], v8, s[4:7], null offen th:TH_ATOMIC_RETURN
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_xor_b32 exec_lo, exec_lo, s0
 ; GFX12-NEXT:    s_cbranch_execnz .LBB12_4
 ; GFX12-NEXT:  ; %bb.5: ; in Loop: Header=BB12_3 Depth=1
@@ -3300,7 +3281,6 @@ define half @buffer_fat_ptr_agent_atomic_fmax_ret_f16__offset__waterfall__amdgpu
 ; GFX12-NEXT:  ; %bb.6: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s1
 ; GFX12-NEXT:    v_lshrrev_b32_e32 v0, v7, v4
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_f16__offset__waterfall__amdgpu_no_fine_grained_memory:
@@ -3896,7 +3876,6 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__amdgpu_no_fine
 ; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_lshl_b32 s5, 0xffff, s4
 ; GFX12-NEXT:    buffer_load_b32 v1, v4, s[0:3], null offen
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_not_b32 s6, s5
 ; GFX12-NEXT:    s_mov_b32 s5, 0
 ; GFX12-NEXT:  .LBB13_1: ; %atomicrmw.start
@@ -3912,6 +3891,7 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__amdgpu_no_fine
 ; GFX12-NEXT:    v_or_b32_e32 v3, 0x400000, v0
 ; GFX12-NEXT:    v_cmp_u_f32_e32 vcc_lo, v0, v0
 ; GFX12-NEXT:    v_add3_u32 v2, v2, v0, 0x7fff
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_cndmask_b32_e32 v0, v2, v3, vcc_lo
 ; GFX12-NEXT:    v_lshrrev_b32_e32 v0, 16, v0
@@ -3933,7 +3913,6 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__amdgpu_no_fine
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s5
 ; GFX12-NEXT:    v_lshrrev_b32_e32 v0, s4, v2
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__amdgpu_no_fine_grained_memory:
@@ -4304,7 +4283,6 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_bf16__offset__amdgpu_no_fine
 ; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_lshl_b32 s5, 0xffff, s4
 ; GFX12-NEXT:    buffer_load_b32 v1, v2, s[0:3], null offen
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_not_b32 s6, s5
 ; GFX12-NEXT:    s_mov_b32 s5, 0
 ; GFX12-NEXT:  .LBB14_1: ; %atomicrmw.start
@@ -4320,6 +4298,7 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_bf16__offset__amdgpu_no_fine
 ; GFX12-NEXT:    v_or_b32_e32 v5, 0x400000, v0
 ; GFX12-NEXT:    v_cmp_u_f32_e32 vcc_lo, v0, v0
 ; GFX12-NEXT:    v_add3_u32 v4, v4, v0, 0x7fff
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_cndmask_b32_e32 v0, v4, v5, vcc_lo
 ; GFX12-NEXT:    v_lshrrev_b32_e32 v0, 16, v0
@@ -4340,7 +4319,6 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_bf16__offset__amdgpu_no_fine
 ; GFX12-NEXT:    s_cbranch_execnz .LBB14_1
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s5
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: buffer_fat_ptr_agent_atomic_fmax_noret_bf16__offset__amdgpu_no_fine_grained_memory:
@@ -4703,17 +4681,16 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__waterfall__amd
 ; GFX12-NEXT:    v_readfirstlane_b32 s5, v1
 ; GFX12-NEXT:    v_readfirstlane_b32 s6, v2
 ; GFX12-NEXT:    v_readfirstlane_b32 s7, v3
+; GFX12-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
 ; GFX12-NEXT:    v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
 ; GFX12-NEXT:    v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-NEXT:    s_and_b32 s0, vcc_lo, s0
 ; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_and_saveexec_b32 s0, s0
 ; GFX12-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-NEXT:    buffer_load_b32 v6, v8, s[4:7], null offen
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_xor_b32 exec_lo, exec_lo, s0
 ; GFX12-NEXT:    s_cbranch_execnz .LBB15_1
 ; GFX12-NEXT:  ; %bb.2:
@@ -4735,6 +4712,7 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__waterfall__amd
 ; GFX12-NEXT:    v_or_b32_e32 v11, 0x400000, v4
 ; GFX12-NEXT:    v_cmp_u_f32_e32 vcc_lo, v4, v4
 ; GFX12-NEXT:    v_add3_u32 v5, v5, v4, 0x7fff
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_cndmask_b32_e32 v4, v5, v11, vcc_lo
 ; GFX12-NEXT:    v_lshrrev_b32_e32 v4, 16, v4
@@ -4750,17 +4728,16 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__waterfall__amd
 ; GFX12-NEXT:    v_readfirstlane_b32 s5, v1
 ; GFX12-NEXT:    v_readfirstlane_b32 s6, v2
 ; GFX12-NEXT:    v_readfirstlane_b32 s7, v3
+; GFX12-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
 ; GFX12-NEXT:    v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
 ; GFX12-NEXT:    v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-NEXT:    s_and_b32 s0, vcc_lo, s0
 ; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_and_saveexec_b32 s0, s0
 ; GFX12-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-NEXT:    buffer_atomic_cmpswap_b32 v[4:5], v8, s[4:7], null offen th:TH_ATOMIC_RETURN
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_xor_b32 exec_lo, exec_lo, s0
 ; GFX12-NEXT:    s_cbranch_execnz .LBB15_4
 ; GFX12-NEXT:  ; %bb.5: ; in Loop: Header=BB15_3 Depth=1
@@ -4776,7 +4753,6 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__waterfall__amd
 ; GFX12-NEXT:  ; %bb.6: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s1
 ; GFX12-NEXT:    v_lshrrev_b32_e32 v0, v7, v4
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__waterfall__amdgpu_no_fine_grained_memory:
@@ -5418,14 +5394,12 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset__amdgpu_no
 ; GFX12-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-NEXT:    global_inv scope:SCOPE_DEV
 ; GFX12-NEXT:    v_cmp_eq_u32_e32 vcc_lo, v0, v5
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_or_b32 s4, vcc_lo, s4
 ; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s4
 ; GFX12-NEXT:    s_cbranch_execnz .LBB16_1
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s4
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset__amdgpu_no_fine_grained_memory:
@@ -5738,14 +5712,12 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_v2f16__offset__amdgpu_no_fin
 ; GFX12-NEXT:    global_inv scope:SCOPE_DEV
 ; GFX12-NEXT:    v_cmp_eq_u32_e32 vcc_lo, v4, v1
 ; GFX12-NEXT:    v_mov_b32_e32 v1, v4
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_or_b32 s4, vcc_lo, s4
 ; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s4
 ; GFX12-NEXT:    s_cbranch_execnz .LBB17_1
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s4
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: buffer_fat_ptr_agent_atomic_fmax_noret_v2f16__offset__amdgpu_no_fine_grained_memory:
@@ -6038,10 +6010,10 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset__waterfall
 ; GFX12-NEXT:    v_readfirstlane_b32 s5, v1
 ; GFX12-NEXT:    v_readfirstlane_b32 s6, v2
 ; GFX12-NEXT:    v_readfirstlane_b32 s7, v3
+; GFX12-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
 ; GFX12-NEXT:    v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
 ; GFX12-NEXT:    v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-NEXT:    s_and_b32 s0, vcc_lo, s0
 ; GFX12-NEXT:    s_wait_alu 0xfffe
@@ -6049,7 +6021,6 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset__waterfall
 ; GFX12-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-NEXT:    buffer_load_b32 v6, v4, s[4:7], null offen offset:1024
 ; GFX12-NEXT:    ; implicit-def: $vgpr4
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_xor_b32 exec_lo, exec_lo, s0
 ; GFX12-NEXT:    s_cbranch_execnz .LBB18_1
 ; GFX12-NEXT:  ; %bb.2:
@@ -6073,17 +6044,16 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset__waterfall
 ; GFX12-NEXT:    v_readfirstlane_b32 s5, v1
 ; GFX12-NEXT:    v_readfirstlane_b32 s6, v2
 ; GFX12-NEXT:    v_readfirstlane_b32 s7, v3
+; GFX12-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
 ; GFX12-NEXT:    v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
 ; GFX12-NEXT:    v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-NEXT:    s_and_b32 s0, vcc_lo, s0
 ; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_and_saveexec_b32 s0, s0
 ; GFX12-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-NEXT:    buffer_atomic_cmpswap_b32 v[4:5], v7, s[4:7], null offen th:TH_ATOMIC_RETURN
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_xor_b32 exec_lo, exec_lo, s0
 ; GFX12-NEXT:    s_cbranch_execnz .LBB18_4
 ; GFX12-NEXT:  ; %bb.5: ; in Loop: Header=BB18_3 Depth=1
@@ -6099,7 +6069,6 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset__waterfall
 ; GFX12-NEXT:  ; %bb.6: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s1
 ; GFX12-NEXT:    v_mov_b32_e32 v0, v4
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset__waterfall__amdgpu_no_fine_grained_memory:
@@ -6673,6 +6642,7 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__amdgpu
 ; GFX12-NEXT:    v_or_b32_e32 v9, 0x400000, v1
 ; GFX12-NEXT:    v_cmp_u_f32_e32 vcc_lo, v1, v1
 ; GFX12-NEXT:    v_add3_u32 v7, v7, v1, 0x7fff
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_dual_cndmask_b32 v1, v7, v9 :: v_dual_lshlrev_b32 v0, 16, v6
 ; GFX12-NEXT:    v_max_num_f32_e32 v0, v0, v2
@@ -6681,6 +6651,7 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__amdgpu
 ; GFX12-NEXT:    v_or_b32_e32 v8, 0x400000, v0
 ; GFX12-NEXT:    v_cmp_u_f32_e64 s4, v0, v0
 ; GFX12-NEXT:    v_add3_u32 v5, v5, v0, 0x7fff
+; GFX12-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_cndmask_b32_e64 v0, v5, v8, s4
 ; GFX12-NEXT:    v_perm_b32 v5, v1, v0, 0x7060302
@@ -6690,14 +6661,12 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__amdgpu
 ; GFX12-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-NEXT:    global_inv scope:SCOPE_DEV
 ; GFX12-NEXT:    v_cmp_eq_u32_e32 vcc_lo, v0, v6
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_or_b32 s5, vcc_lo, s5
 ; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s5
 ; GFX12-NEXT:    s_cbranch_execnz .LBB19_1
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s5
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__amdgpu_no_fine_grained_memory:
@@ -7099,8 +7068,10 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_v2bf16__offset__amdgpu_no_fi
 ; GFX12-NEXT:    v_add3_u32 v7, v7, v5, 0x7fff
 ; GFX12-NEXT:    v_add3_u32 v6, v6, v0, 0x7fff
 ; GFX12-NEXT:    v_cmp_u_f32_e64 s4, v0, v0
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-NEXT:    s_wait_alu 0xfffd
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX12-NEXT:    v_cndmask_b32_e32 v5, v7, v9, vcc_lo
+; GFX12-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-NEXT:    v_cndmask_b32_e64 v0, v6, v8, s4
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_perm_b32 v0, v5, v0, 0x7060302
@@ -7110,14 +7081,12 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_v2bf16__offset__amdgpu_no_fi
 ; GFX12-NEXT:    global_inv scope:SCOPE_DEV
 ; GFX12-NEXT:    v_cmp_eq_u32_e32 vcc_lo, v5, v1
 ; GFX12-NEXT:    v_mov_b32_e32 v1, v5
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_or_b32 s5, vcc_lo, s5
 ; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s5
 ; GFX12-NEXT:    s_cbranch_execnz .LBB20_1
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s5
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: buffer_fat_ptr_agent_atomic_fmax_noret_v2bf16__offset__amdgpu_no_fine_grained_memory:
@@ -7494,10 +7463,10 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__waterf
 ; GFX12-NEXT:    v_readfirstlane_b32 s5, v1
 ; GFX12-NEXT:    v_readfirstlane_b32 s6, v2
 ; GFX12-NEXT:    v_readfirstlane_b32 s7, v3
+; GFX12-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
 ; GFX12-NEXT:    v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
 ; GFX12-NEXT:    v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-NEXT:    s_and_b32 s0, vcc_lo, s0
 ; GFX12-NEXT:    s_wait_alu 0xfffe
@@ -7505,7 +7474,6 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__waterf
 ; GFX12-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-NEXT:    buffer_load_b32 v6, v4, s[4:7], null offen offset:1024
 ; GFX12-NEXT:    ; implicit-def: $vgpr4
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_xor_b32 exec_lo, exec_lo, s0
 ; GFX12-NEXT:    s_cbranch_execnz .LBB21_1
 ; GFX12-NEXT:  ; %bb.2:
@@ -7531,12 +7499,14 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__waterf
 ; GFX12-NEXT:    v_or_b32_e32 v13, 0x400000, v5
 ; GFX12-NEXT:    v_add3_u32 v11, v11, v5, 0x7fff
 ; GFX12-NEXT:    v_add3_u32 v10, v10, v4, 0x7fff
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX12-NEXT:    s_wait_alu 0xfffd
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_cndmask_b32_e32 v4, v10, v12, vcc_lo
 ; GFX12-NEXT:    v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_cndmask_b32_e32 v5, v11, v13, vcc_lo
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_perm_b32 v5, v5, v4, 0x7060302
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-NEXT:    v_mov_b32_e32 v4, v5
 ; GFX12-NEXT:    v_mov_b32_e32 v5, v6
 ; GFX12-NEXT:  .LBB21_4: ; Parent Loop BB21_3 Depth=1
@@ -7545,17 +7515,16 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__waterf
 ; GFX12-NEXT:    v_readfirstlane_b32 s5, v1
 ; GFX12-NEXT:    v_readfirstlane_b32 s6, v2
 ; GFX12-NEXT:    v_readfirstlane_b32 s7, v3
+; GFX12-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
 ; GFX12-NEXT:    v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
 ; GFX12-NEXT:    v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-NEXT:    s_and_b32 s0, vcc_lo, s0
 ; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_and_saveexec_b32 s0, s0
 ; GFX12-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-NEXT:    buffer_atomic_cmpswap_b32 v[4:5], v7, s[4:7], null offen th:TH_ATOMIC_RETURN
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_xor_b32 exec_lo, exec_lo, s0
 ; GFX12-NEXT:    s_cbranch_execnz .LBB21_4
 ; GFX12-NEXT:  ; %bb.5: ; in Loop: Header=BB21_3 Depth=1
@@ -7571,7 +7540,6 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__waterf
 ; GFX12-NEXT:  ; %bb.6: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s1
 ; GFX12-NEXT:    v_mov_b32_e32 v0, v4
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__waterfall__amdgpu_no_fine_grained_memory:
diff --git a/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fmin.ll b/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fmin.ll
index 0bcaacc6b08e8..a6eb81fcbf515 100644
--- a/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fmin.ll
+++ b/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fmin.ll
@@ -374,10 +374,10 @@ define float @buffer_fat_ptr_agent_atomic_fmin_ret_f32__offset__waterfall__amdgp
 ; GFX12-NEXT:    v_readfirstlane_b32 s5, v1
 ; GFX12-NEXT:    v_readfirstlane_b32 s6, v2
 ; GFX12-NEXT:    v_readfirstlane_b32 s7, v3
+; GFX12-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
 ; GFX12-NEXT:    v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
 ; GFX12-NEXT:    v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-NEXT:    s_and_b32 s0, vcc_lo, s0
 ; GFX12-NEXT:    s_wait_alu 0xfffe
@@ -386,7 +386,6 @@ define float @buffer_fat_ptr_agent_atomic_fmin_ret_f32__offset__waterfall__amdgp
 ; GFX12-NEXT:    buffer_atomic_min_num_f32 v5, v4, s[4:7], null offen offset:1024 th:TH_ATOMIC_RETURN
 ; GFX12-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
 ; GFX12-NEXT:    ; implicit-def: $vgpr4
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_xor_b32 exec_lo, exec_lo, s0
 ; GFX12-NEXT:    s_cbranch_execnz .LBB2_1
 ; GFX12-NEXT:  ; %bb.2:
@@ -394,7 +393,6 @@ define float @buffer_fat_ptr_agent_atomic_fmin_ret_f32__offset__waterfall__amdgp
 ; GFX12-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-NEXT:    v_mov_b32_e32 v0, v5
 ; GFX12-NEXT:    global_inv scope:SCOPE_DEV
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f32__offset__waterfall__amdgpu_no_fine_grained_memory:
@@ -1220,14 +1218,12 @@ define double @buffer_fat_ptr_agent_atomic_fmin_ret_f64__offset__amdgpu_no_fine_
 ; GFX12-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-NEXT:    global_inv scope:SCOPE_DEV
 ; GFX12-NEXT:    v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[9:10]
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_or_b32 s4, vcc_lo, s4
 ; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s4
 ; GFX12-NEXT:    s_cbranch_execnz .LBB5_1
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s4
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f64__offset__amdgpu_no_fine_grained_memory:
@@ -1413,14 +1409,12 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_f64__offset__amdgpu_no_fine_
 ; GFX12-NEXT:    global_inv scope:SCOPE_DEV
 ; GFX12-NEXT:    v_cmp_eq_u64_e32 vcc_lo, v[7:8], v[2:3]
 ; GFX12-NEXT:    v_dual_mov_b32 v2, v7 :: v_dual_mov_b32 v3, v8
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_or_b32 s4, vcc_lo, s4
 ; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s4
 ; GFX12-NEXT:    s_cbranch_execnz .LBB6_1
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s4
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_f64__offset__amdgpu_no_fine_grained_memory:
@@ -1589,10 +1583,10 @@ define double @buffer_fat_ptr_agent_atomic_fmin_ret_f64__offset__waterfall__amdg
 ; GFX12-NEXT:    v_readfirstlane_b32 s5, v10
 ; GFX12-NEXT:    v_readfirstlane_b32 s6, v7
 ; GFX12-NEXT:    v_readfirstlane_b32 s7, v8
+; GFX12-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
 ; GFX12-NEXT:    v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[9:10]
 ; GFX12-NEXT:    v_cmp_eq_u64_e64 s0, s[6:7], v[7:8]
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-NEXT:    s_and_b32 s0, vcc_lo, s0
 ; GFX12-NEXT:    s_wait_alu 0xfffe
@@ -1600,7 +1594,6 @@ define double @buffer_fat_ptr_agent_atomic_fmin_ret_f64__offset__waterfall__amdg
 ; GFX12-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-NEXT:    buffer_load_b64 v[13:14], v4, s[4:7], null offen offset:2048
 ; GFX12-NEXT:    ; implicit-def: $vgpr4
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_xor_b32 exec_lo, exec_lo, s0
 ; GFX12-NEXT:    s_cbranch_execnz .LBB7_1
 ; GFX12-NEXT:  ; %bb.2:
@@ -1624,17 +1617,16 @@ define double @buffer_fat_ptr_agent_atomic_fmin_ret_f64__offset__waterfall__amdg
 ; GFX12-NEXT:    v_readfirstlane_b32 s5, v10
 ; GFX12-NEXT:    v_readfirstlane_b32 s6, v7
 ; GFX12-NEXT:    v_readfirstlane_b32 s7, v8
+; GFX12-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
 ; GFX12-NEXT:    v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[9:10]
 ; GFX12-NEXT:    v_cmp_eq_u64_e64 s0, s[6:7], v[7:8]
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-NEXT:    s_and_b32 s0, vcc_lo, s0
 ; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_and_saveexec_b32 s0, s0
 ; GFX12-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-NEXT:    buffer_atomic_cmpswap_b64 v[0:3], v15, s[4:7], null offen th:TH_ATOMIC_RETURN
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_xor_b32 exec_lo, exec_lo, s0
 ; GFX12-NEXT:    s_cbranch_execnz .LBB7_4
 ; GFX12-NEXT:  ; %bb.5: ; in Loop: Header=BB7_3 Depth=1
@@ -1649,7 +1641,6 @@ define double @buffer_fat_ptr_agent_atomic_fmin_ret_f64__offset__waterfall__amdg
 ; GFX12-NEXT:    s_cbranch_execnz .LBB7_3
 ; GFX12-NEXT:  ; %bb.6: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s1
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f64__offset__waterfall__amdgpu_no_fine_grained_memory:
@@ -2034,14 +2025,12 @@ define double @buffer_fat_ptr_agent_atomic_fmin_ret_f64__offset__amdgpu_no_remot
 ; GFX12-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-NEXT:    global_inv scope:SCOPE_DEV
 ; GFX12-NEXT:    v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[9:10]
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_or_b32 s4, vcc_lo, s4
 ; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s4
 ; GFX12-NEXT:    s_cbranch_execnz .LBB8_1
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s4
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f64__offset__amdgpu_no_remote_memory:
@@ -2323,14 +2312,12 @@ define double @buffer_fat_ptr_agent_atomic_fmin_ret_f64__offset__amdgpu_no_fine_
 ; GFX12-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-NEXT:    global_inv scope:SCOPE_DEV
 ; GFX12-NEXT:    v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[9:10]
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_or_b32 s4, vcc_lo, s4
 ; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s4
 ; GFX12-NEXT:    s_cbranch_execnz .LBB9_1
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s4
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f64__offset__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
@@ -2511,7 +2498,6 @@ define half @buffer_fat_ptr_agent_atomic_fmin_ret_f16__offset__amdgpu_no_fine_gr
 ; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_lshl_b32 s5, 0xffff, s4
 ; GFX12-NEXT:    buffer_load_b32 v1, v4, s[0:3], null offen
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_not_b32 s6, s5
 ; GFX12-NEXT:    s_mov_b32 s5, 0
 ; GFX12-NEXT:  .LBB10_1: ; %atomicrmw.start
@@ -2541,7 +2527,6 @@ define half @buffer_fat_ptr_agent_atomic_fmin_ret_f16__offset__amdgpu_no_fine_gr
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s5
 ; GFX12-NEXT:    v_lshrrev_b32_e32 v0, s4, v2
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f16__offset__amdgpu_no_fine_grained_memory:
@@ -2877,7 +2862,6 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_f16__offset__amdgpu_no_fine_
 ; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_lshl_b32 s5, 0xffff, s4
 ; GFX12-NEXT:    buffer_load_b32 v1, v2, s[0:3], null offen
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_not_b32 s6, s5
 ; GFX12-NEXT:    s_mov_b32 s5, 0
 ; GFX12-NEXT:  .LBB11_1: ; %atomicrmw.start
@@ -2906,7 +2890,6 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_f16__offset__amdgpu_no_fine_
 ; GFX12-NEXT:    s_cbranch_execnz .LBB11_1
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s5
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_f16__offset__amdgpu_no_fine_grained_memory:
@@ -3234,17 +3217,16 @@ define half @buffer_fat_ptr_agent_atomic_fmin_ret_f16__offset__waterfall__amdgpu
 ; GFX12-NEXT:    v_readfirstlane_b32 s5, v1
 ; GFX12-NEXT:    v_readfirstlane_b32 s6, v2
 ; GFX12-NEXT:    v_readfirstlane_b32 s7, v3
+; GFX12-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
 ; GFX12-NEXT:    v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
 ; GFX12-NEXT:    v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-NEXT:    s_and_b32 s0, vcc_lo, s0
 ; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_and_saveexec_b32 s0, s0
 ; GFX12-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-NEXT:    buffer_load_b32 v6, v8, s[4:7], null offen
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_xor_b32 exec_lo, exec_lo, s0
 ; GFX12-NEXT:    s_cbranch_execnz .LBB12_1
 ; GFX12-NEXT:  ; %bb.2:
@@ -3274,17 +3256,16 @@ define half @buffer_fat_ptr_agent_atomic_fmin_ret_f16__offset__waterfall__amdgpu
 ; GFX12-NEXT:    v_readfirstlane_b32 s5, v1
 ; GFX12-NEXT:    v_readfirstlane_b32 s6, v2
 ; GFX12-NEXT:    v_readfirstlane_b32 s7, v3
+; GFX12-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
 ; GFX12-NEXT:    v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
 ; GFX12-NEXT:    v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-NEXT:    s_and_b32 s0, vcc_lo, s0
 ; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_and_saveexec_b32 s0, s0
 ; GFX12-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-NEXT:    buffer_atomic_cmpswap_b32 v[4:5], v8, s[4:7], null offen th:TH_ATOMIC_RETURN
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_xor_b32 exec_lo, exec_lo, s0
 ; GFX12-NEXT:    s_cbranch_execnz .LBB12_4
 ; GFX12-NEXT:  ; %bb.5: ; in Loop: Header=BB12_3 Depth=1
@@ -3300,7 +3281,6 @@ define half @buffer_fat_ptr_agent_atomic_fmin_ret_f16__offset__waterfall__amdgpu
 ; GFX12-NEXT:  ; %bb.6: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s1
 ; GFX12-NEXT:    v_lshrrev_b32_e32 v0, v7, v4
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f16__offset__waterfall__amdgpu_no_fine_grained_memory:
@@ -3896,7 +3876,6 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__amdgpu_no_fine
 ; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_lshl_b32 s5, 0xffff, s4
 ; GFX12-NEXT:    buffer_load_b32 v1, v4, s[0:3], null offen
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_not_b32 s6, s5
 ; GFX12-NEXT:    s_mov_b32 s5, 0
 ; GFX12-NEXT:  .LBB13_1: ; %atomicrmw.start
@@ -3912,6 +3891,7 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__amdgpu_no_fine
 ; GFX12-NEXT:    v_or_b32_e32 v3, 0x400000, v0
 ; GFX12-NEXT:    v_cmp_u_f32_e32 vcc_lo, v0, v0
 ; GFX12-NEXT:    v_add3_u32 v2, v2, v0, 0x7fff
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_cndmask_b32_e32 v0, v2, v3, vcc_lo
 ; GFX12-NEXT:    v_lshrrev_b32_e32 v0, 16, v0
@@ -3933,7 +3913,6 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__amdgpu_no_fine
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s5
 ; GFX12-NEXT:    v_lshrrev_b32_e32 v0, s4, v2
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__amdgpu_no_fine_grained_memory:
@@ -4304,7 +4283,6 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_bf16__offset__amdgpu_no_fine
 ; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_lshl_b32 s5, 0xffff, s4
 ; GFX12-NEXT:    buffer_load_b32 v1, v2, s[0:3], null offen
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_not_b32 s6, s5
 ; GFX12-NEXT:    s_mov_b32 s5, 0
 ; GFX12-NEXT:  .LBB14_1: ; %atomicrmw.start
@@ -4320,6 +4298,7 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_bf16__offset__amdgpu_no_fine
 ; GFX12-NEXT:    v_or_b32_e32 v5, 0x400000, v0
 ; GFX12-NEXT:    v_cmp_u_f32_e32 vcc_lo, v0, v0
 ; GFX12-NEXT:    v_add3_u32 v4, v4, v0, 0x7fff
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_cndmask_b32_e32 v0, v4, v5, vcc_lo
 ; GFX12-NEXT:    v_lshrrev_b32_e32 v0, 16, v0
@@ -4340,7 +4319,6 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_bf16__offset__amdgpu_no_fine
 ; GFX12-NEXT:    s_cbranch_execnz .LBB14_1
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s5
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_bf16__offset__amdgpu_no_fine_grained_memory:
@@ -4703,17 +4681,16 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__waterfall__amd
 ; GFX12-NEXT:    v_readfirstlane_b32 s5, v1
 ; GFX12-NEXT:    v_readfirstlane_b32 s6, v2
 ; GFX12-NEXT:    v_readfirstlane_b32 s7, v3
+; GFX12-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
 ; GFX12-NEXT:    v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
 ; GFX12-NEXT:    v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-NEXT:    s_and_b32 s0, vcc_lo, s0
 ; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_and_saveexec_b32 s0, s0
 ; GFX12-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-NEXT:    buffer_load_b32 v6, v8, s[4:7], null offen
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_xor_b32 exec_lo, exec_lo, s0
 ; GFX12-NEXT:    s_cbranch_execnz .LBB15_1
 ; GFX12-NEXT:  ; %bb.2:
@@ -4735,6 +4712,7 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__waterfall__amd
 ; GFX12-NEXT:    v_or_b32_e32 v11, 0x400000, v4
 ; GFX12-NEXT:    v_cmp_u_f32_e32 vcc_lo, v4, v4
 ; GFX12-NEXT:    v_add3_u32 v5, v5, v4, 0x7fff
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_cndmask_b32_e32 v4, v5, v11, vcc_lo
 ; GFX12-NEXT:    v_lshrrev_b32_e32 v4, 16, v4
@@ -4750,17 +4728,16 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__waterfall__amd
 ; GFX12-NEXT:    v_readfirstlane_b32 s5, v1
 ; GFX12-NEXT:    v_readfirstlane_b32 s6, v2
 ; GFX12-NEXT:    v_readfirstlane_b32 s7, v3
+; GFX12-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
 ; GFX12-NEXT:    v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
 ; GFX12-NEXT:    v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-NEXT:    s_and_b32 s0, vcc_lo, s0
 ; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_and_saveexec_b32 s0, s0
 ; GFX12-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-NEXT:    buffer_atomic_cmpswap_b32 v[4:5], v8, s[4:7], null offen th:TH_ATOMIC_RETURN
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_xor_b32 exec_lo, exec_lo, s0
 ; GFX12-NEXT:    s_cbranch_execnz .LBB15_4
 ; GFX12-NEXT:  ; %bb.5: ; in Loop: Header=BB15_3 Depth=1
@@ -4776,7 +4753,6 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__waterfall__amd
 ; GFX12-NEXT:  ; %bb.6: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s1
 ; GFX12-NEXT:    v_lshrrev_b32_e32 v0, v7, v4
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__waterfall__amdgpu_no_fine_grained_memory:
@@ -5418,14 +5394,12 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset__amdgpu_no
 ; GFX12-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-NEXT:    global_inv scope:SCOPE_DEV
 ; GFX12-NEXT:    v_cmp_eq_u32_e32 vcc_lo, v0, v5
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_or_b32 s4, vcc_lo, s4
 ; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s4
 ; GFX12-NEXT:    s_cbranch_execnz .LBB16_1
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s4
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset__amdgpu_no_fine_grained_memory:
@@ -5738,14 +5712,12 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_v2f16__offset__amdgpu_no_fin
 ; GFX12-NEXT:    global_inv scope:SCOPE_DEV
 ; GFX12-NEXT:    v_cmp_eq_u32_e32 vcc_lo, v4, v1
 ; GFX12-NEXT:    v_mov_b32_e32 v1, v4
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_or_b32 s4, vcc_lo, s4
 ; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s4
 ; GFX12-NEXT:    s_cbranch_execnz .LBB17_1
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s4
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_v2f16__offset__amdgpu_no_fine_grained_memory:
@@ -6038,10 +6010,10 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset__waterfall
 ; GFX12-NEXT:    v_readfirstlane_b32 s5, v1
 ; GFX12-NEXT:    v_readfirstlane_b32 s6, v2
 ; GFX12-NEXT:    v_readfirstlane_b32 s7, v3
+; GFX12-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
 ; GFX12-NEXT:    v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
 ; GFX12-NEXT:    v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-NEXT:    s_and_b32 s0, vcc_lo, s0
 ; GFX12-NEXT:    s_wait_alu 0xfffe
@@ -6049,7 +6021,6 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset__waterfall
 ; GFX12-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-NEXT:    buffer_load_b32 v6, v4, s[4:7], null offen offset:1024
 ; GFX12-NEXT:    ; implicit-def: $vgpr4
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_xor_b32 exec_lo, exec_lo, s0
 ; GFX12-NEXT:    s_cbranch_execnz .LBB18_1
 ; GFX12-NEXT:  ; %bb.2:
@@ -6073,17 +6044,16 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset__waterfall
 ; GFX12-NEXT:    v_readfirstlane_b32 s5, v1
 ; GFX12-NEXT:    v_readfirstlane_b32 s6, v2
 ; GFX12-NEXT:    v_readfirstlane_b32 s7, v3
+; GFX12-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
 ; GFX12-NEXT:    v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
 ; GFX12-NEXT:    v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-NEXT:    s_and_b32 s0, vcc_lo, s0
 ; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_and_saveexec_b32 s0, s0
 ; GFX12-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-NEXT:    buffer_atomic_cmpswap_b32 v[4:5], v7, s[4:7], null offen th:TH_ATOMIC_RETURN
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_xor_b32 exec_lo, exec_lo, s0
 ; GFX12-NEXT:    s_cbranch_execnz .LBB18_4
 ; GFX12-NEXT:  ; %bb.5: ; in Loop: Header=BB18_3 Depth=1
@@ -6099,7 +6069,6 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset__waterfall
 ; GFX12-NEXT:  ; %bb.6: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s1
 ; GFX12-NEXT:    v_mov_b32_e32 v0, v4
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset__waterfall__amdgpu_no_fine_grained_memory:
@@ -6673,6 +6642,7 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__amdgpu
 ; GFX12-NEXT:    v_or_b32_e32 v9, 0x400000, v1
 ; GFX12-NEXT:    v_cmp_u_f32_e32 vcc_lo, v1, v1
 ; GFX12-NEXT:    v_add3_u32 v7, v7, v1, 0x7fff
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_dual_cndmask_b32 v1, v7, v9 :: v_dual_lshlrev_b32 v0, 16, v6
 ; GFX12-NEXT:    v_min_num_f32_e32 v0, v0, v2
@@ -6681,6 +6651,7 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__amdgpu
 ; GFX12-NEXT:    v_or_b32_e32 v8, 0x400000, v0
 ; GFX12-NEXT:    v_cmp_u_f32_e64 s4, v0, v0
 ; GFX12-NEXT:    v_add3_u32 v5, v5, v0, 0x7fff
+; GFX12-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_cndmask_b32_e64 v0, v5, v8, s4
 ; GFX12-NEXT:    v_perm_b32 v5, v1, v0, 0x7060302
@@ -6690,14 +6661,12 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__amdgpu
 ; GFX12-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-NEXT:    global_inv scope:SCOPE_DEV
 ; GFX12-NEXT:    v_cmp_eq_u32_e32 vcc_lo, v0, v6
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_or_b32 s5, vcc_lo, s5
 ; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s5
 ; GFX12-NEXT:    s_cbranch_execnz .LBB19_1
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s5
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__amdgpu_no_fine_grained_memory:
@@ -7099,8 +7068,10 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_v2bf16__offset__amdgpu_no_fi
 ; GFX12-NEXT:    v_add3_u32 v7, v7, v5, 0x7fff
 ; GFX12-NEXT:    v_add3_u32 v6, v6, v0, 0x7fff
 ; GFX12-NEXT:    v_cmp_u_f32_e64 s4, v0, v0
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-NEXT:    s_wait_alu 0xfffd
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX12-NEXT:    v_cndmask_b32_e32 v5, v7, v9, vcc_lo
+; GFX12-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-NEXT:    v_cndmask_b32_e64 v0, v6, v8, s4
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_perm_b32 v0, v5, v0, 0x7060302
@@ -7110,14 +7081,12 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_v2bf16__offset__amdgpu_no_fi
 ; GFX12-NEXT:    global_inv scope:SCOPE_DEV
 ; GFX12-NEXT:    v_cmp_eq_u32_e32 vcc_lo, v5, v1
 ; GFX12-NEXT:    v_mov_b32_e32 v1, v5
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_or_b32 s5, vcc_lo, s5
 ; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s5
 ; GFX12-NEXT:    s_cbranch_execnz .LBB20_1
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s5
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_v2bf16__offset__amdgpu_no_fine_grained_memory:
@@ -7494,10 +7463,10 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__waterf
 ; GFX12-NEXT:    v_readfirstlane_b32 s5, v1
 ; GFX12-NEXT:    v_readfirstlane_b32 s6, v2
 ; GFX12-NEXT:    v_readfirstlane_b32 s7, v3
+; GFX12-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
 ; GFX12-NEXT:    v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
 ; GFX12-NEXT:    v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-NEXT:    s_and_b32 s0, vcc_lo, s0
 ; GFX12-NEXT:    s_wait_alu 0xfffe
@@ -7505,7 +7474,6 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__waterf
 ; GFX12-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-NEXT:    buffer_load_b32 v6, v4, s[4:7], null offen offset:1024
 ; GFX12-NEXT:    ; implicit-def: $vgpr4
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_xor_b32 exec_lo, exec_lo, s0
 ; GFX12-NEXT:    s_cbranch_execnz .LBB21_1
 ; GFX12-NEXT:  ; %bb.2:
@@ -7531,12 +7499,14 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__waterf
 ; GFX12-NEXT:    v_or_b32_e32 v13, 0x400000, v5
 ; GFX12-NEXT:    v_add3_u32 v11, v11, v5, 0x7fff
 ; GFX12-NEXT:    v_add3_u32 v10, v10, v4, 0x7fff
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX12-NEXT:    s_wait_alu 0xfffd
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_cndmask_b32_e32 v4, v10, v12, vcc_lo
 ; GFX12-NEXT:    v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_cndmask_b32_e32 v5, v11, v13, vcc_lo
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_perm_b32 v5, v5, v4, 0x7060302
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-NEXT:    v_mov_b32_e32 v4, v5
 ; GFX12-NEXT:    v_mov_b32_e32 v5, v6
 ; GFX12-NEXT:  .LBB21_4: ; Parent Loop BB21_3 Depth=1
@@ -7545,17 +7515,16 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__waterf
 ; GFX12-NEXT:    v_readfirstlane_b32 s5, v1
 ; GFX12-NEXT:    v_readfirstlane_b32 s6, v2
 ; GFX12-NEXT:    v_readfirstlane_b32 s7, v3
+; GFX12-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
 ; GFX12-NEXT:    v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
 ; GFX12-NEXT:    v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-NEXT:    s_and_b32 s0, vcc_lo, s0
 ; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_and_saveexec_b32 s0, s0
 ; GFX12-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-NEXT:    buffer_atomic_cmpswap_b32 v[4:5], v7, s[4:7], null offen th:TH_ATOMIC_RETURN
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_xor_b32 exec_lo, exec_lo, s0
 ; GFX12-NEXT:    s_cbranch_execnz .LBB21_4
 ; GFX12-NEXT:  ; %bb.5: ; in Loop: Header=BB21_3 Depth=1
@@ -7571,7 +7540,6 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__waterf
 ; GFX12-NEXT:  ; %bb.6: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s1
 ; GFX12-NEXT:    v_mov_b32_e32 v0, v4
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__waterfall__amdgpu_no_fine_grained_memory:
diff --git a/llvm/test/CodeGen/AMDGPU/buffer-fat-pointers-contents-legalization.ll b/llvm/test/CodeGen/AMDGPU/buffer-fat-pointers-contents-legalization.ll
index 9912ce3604a49..405058b24dcc2 100644
--- a/llvm/test/CodeGen/AMDGPU/buffer-fat-pointers-contents-legalization.ll
+++ b/llvm/test/CodeGen/AMDGPU/buffer-fat-pointers-contents-legalization.ll
@@ -1,6 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
 ; RUN: llc -mtriple=amdgcn -mcpu=gfx900 < %s | FileCheck -check-prefix=SDAG %s
-; RUN: llc -mtriple=amdgcn -mcpu=gfx900 -global-isel=1 < %s | FileCheck -check-prefix=GISEL %s
+; RUN: llc -mtriple=amdgcn -mcpu=gfx900 -global-isel=1 -global-isel-abort=2 < %s | FileCheck -check-prefix=GISEL %s
 
 ; Note: if you're adding tests here, also add them to
 ; lower-buffer-fat-pointers-contents-legalization.ll to verify the IR produced by
@@ -629,7 +629,6 @@ define <4 x bfloat> @load_v4bf16(ptr addrspace(8) inreg %buf) {
 ; GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GISEL-NEXT:    buffer_load_dwordx2 v[0:1], off, s[16:19], 0
 ; GISEL-NEXT:    s_waitcnt vmcnt(0)
-; GISEL-NEXT:    v_lshrrev_b32_e32 v1, 16, v0
 ; GISEL-NEXT:    s_setpc_b64 s[30:31]
   %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
   %ret = load <4 x bfloat>, ptr addrspace(7) %p
@@ -647,10 +646,6 @@ define void @store_v4bf16(<4 x bfloat> %data, ptr addrspace(8) inreg %buf) {
 ; GISEL-LABEL: store_v4bf16:
 ; GISEL:       ; %bb.0:
 ; GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GISEL-NEXT:    v_lshrrev_b32_e32 v2, 16, v0
-; GISEL-NEXT:    v_lshrrev_b32_e32 v3, 16, v1
-; GISEL-NEXT:    v_mov_b32_sdwa v0, v2 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0
-; GISEL-NEXT:    v_mov_b32_sdwa v1, v3 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0
 ; GISEL-NEXT:    buffer_store_dwordx2 v[0:1], off, s[16:19], 0
 ; GISEL-NEXT:    s_waitcnt vmcnt(0)
 ; GISEL-NEXT:    s_setpc_b64 s[30:31]
@@ -3096,15 +3091,6 @@ define i160 @load_i160(ptr addrspace(8) inreg %buf) {
 ; SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; SDAG-NEXT:    buffer_load_dwordx4 v[0:3], off, s[16:19], 0
 ; SDAG-NEXT:    buffer_load_dword v4, off, s[16:19], 0 offset:16
-; SDAG-NEXT:    s_mov_b32 s4, s33
-; SDAG-NEXT:    s_add_i32 s33, s32, 0x7c0
-; SDAG-NEXT:    s_and_b32 s33, s33, 0xfffff800
-; SDAG-NEXT:    s_mov_b32 s5, s34
-; SDAG-NEXT:    s_mov_b32 s34, s32
-; SDAG-NEXT:    s_addk_i32 s32, 0x1800
-; SDAG-NEXT:    s_mov_b32 s32, s34
-; SDAG-NEXT:    s_mov_b32 s34, s5
-; SDAG-NEXT:    s_mov_b32 s33, s4
 ; SDAG-NEXT:    s_waitcnt vmcnt(0)
 ; SDAG-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -3124,17 +3110,8 @@ define void @store_i160(i160 %data, ptr addrspace(8) inreg %buf) {
 ; SDAG-LABEL: store_i160:
 ; SDAG:       ; %bb.0:
 ; SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SDAG-NEXT:    s_mov_b32 s4, s33
-; SDAG-NEXT:    s_add_i32 s33, s32, 0x7c0
-; SDAG-NEXT:    s_and_b32 s33, s33, 0xfffff800
-; SDAG-NEXT:    s_mov_b32 s5, s34
-; SDAG-NEXT:    s_mov_b32 s34, s32
-; SDAG-NEXT:    s_addk_i32 s32, 0x1000
 ; SDAG-NEXT:    buffer_store_dwordx4 v[0:3], off, s[16:19], 0
 ; SDAG-NEXT:    buffer_store_dword v4, off, s[16:19], 0 offset:16
-; SDAG-NEXT:    s_mov_b32 s32, s34
-; SDAG-NEXT:    s_mov_b32 s34, s5
-; SDAG-NEXT:    s_mov_b32 s33, s4
 ; SDAG-NEXT:    s_waitcnt vmcnt(0)
 ; SDAG-NEXT:    s_setpc_b64 s[30:31]
 ;
diff --git a/llvm/test/CodeGen/AMDGPU/bundle-latency.mir b/llvm/test/CodeGen/AMDGPU/bundle-latency.mir
index d2846fd161030..9f2c6d1b53bb8 100644
--- a/llvm/test/CodeGen/AMDGPU/bundle-latency.mir
+++ b/llvm/test/CodeGen/AMDGPU/bundle-latency.mir
@@ -1,5 +1,6 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
 # RUN: llc -mtriple=amdgcn -mcpu=gfx900 -verify-machineinstrs -run-pass=post-RA-sched %s -o - | FileCheck -check-prefix=GCN %s
+# RUN: llc -mtriple=amdgcn -mcpu=gfx900 -passes=post-RA-sched %s -o - | FileCheck -check-prefix=GCN %s
 
 # Check that we move consumer further from producer, even if one of them is in a bundle.
 
diff --git a/llvm/test/CodeGen/AMDGPU/call-args-inreg.ll b/llvm/test/CodeGen/AMDGPU/call-args-inreg.ll
index 704b68aa296a9..0b8ad359ccb94 100644
--- a/llvm/test/CodeGen/AMDGPU/call-args-inreg.ll
+++ b/llvm/test/CodeGen/AMDGPU/call-args-inreg.ll
@@ -43,11 +43,11 @@ define void @test_call_external_void_func_i8_inreg(i8 inreg %arg) #0 {
 ; GFX9-NEXT:    v_writelane_b32 v40, s17, 2
 ; GFX9-NEXT:    s_addk_i32 s32, 0x400
 ; GFX9-NEXT:    v_writelane_b32 v40, s30, 0
-; GFX9-NEXT:    s_mov_b32 s0, s16
-; GFX9-NEXT:    v_writelane_b32 v40, s31, 1
 ; GFX9-NEXT:    s_getpc_b64 s[18:19]
 ; GFX9-NEXT:    s_add_u32 s18, s18, external_void_func_i8_inreg@rel32@lo+4
 ; GFX9-NEXT:    s_addc_u32 s19, s19, external_void_func_i8_inreg@rel32@hi+12
+; GFX9-NEXT:    s_mov_b32 s0, s16
+; GFX9-NEXT:    v_writelane_b32 v40, s31, 1
 ; GFX9-NEXT:    s_swappc_b64 s[30:31], s[18:19]
 ; GFX9-NEXT:    v_readlane_b32 s31, v40, 1
 ; GFX9-NEXT:    v_readlane_b32 s30, v40, 0
@@ -103,11 +103,11 @@ define void @test_call_external_void_func_i16_inreg(i16 inreg %arg) #0 {
 ; GFX9-NEXT:    v_writelane_b32 v40, s17, 2
 ; GFX9-NEXT:    s_addk_i32 s32, 0x400
 ; GFX9-NEXT:    v_writelane_b32 v40, s30, 0
-; GFX9-NEXT:    s_mov_b32 s0, s16
-; GFX9-NEXT:    v_writelane_b32 v40, s31, 1
 ; GFX9-NEXT:    s_getpc_b64 s[18:19]
 ; GFX9-NEXT:    s_add_u32 s18, s18, external_void_func_i16_inreg@rel32@lo+4
 ; GFX9-NEXT:    s_addc_u32 s19, s19, external_void_func_i16_inreg@rel32@hi+12
+; GFX9-NEXT:    s_mov_b32 s0, s16
+; GFX9-NEXT:    v_writelane_b32 v40, s31, 1
 ; GFX9-NEXT:    s_swappc_b64 s[30:31], s[18:19]
 ; GFX9-NEXT:    v_readlane_b32 s31, v40, 1
 ; GFX9-NEXT:    v_readlane_b32 s30, v40, 0
@@ -163,11 +163,11 @@ define void @test_call_external_void_func_i32_inreg(i32 inreg %arg) #0 {
 ; GFX9-NEXT:    v_writelane_b32 v40, s17, 2
 ; GFX9-NEXT:    s_addk_i32 s32, 0x400
 ; GFX9-NEXT:    v_writelane_b32 v40, s30, 0
-; GFX9-NEXT:    s_mov_b32 s0, s16
-; GFX9-NEXT:    v_writelane_b32 v40, s31, 1
 ; GFX9-NEXT:    s_getpc_b64 s[18:19]
 ; GFX9-NEXT:    s_add_u32 s18, s18, external_void_func_i32_inreg@rel32@lo+4
 ; GFX9-NEXT:    s_addc_u32 s19, s19, external_void_func_i32_inreg@rel32@hi+12
+; GFX9-NEXT:    s_mov_b32 s0, s16
+; GFX9-NEXT:    v_writelane_b32 v40, s31, 1
 ; GFX9-NEXT:    s_swappc_b64 s[30:31], s[18:19]
 ; GFX9-NEXT:    v_readlane_b32 s31, v40, 1
 ; GFX9-NEXT:    v_readlane_b32 s30, v40, 0
@@ -223,12 +223,12 @@ define void @test_call_external_void_func_i64_inreg(i64 inreg %arg) #0 {
 ; GFX9-NEXT:    v_writelane_b32 v40, s18, 2
 ; GFX9-NEXT:    s_addk_i32 s32, 0x400
 ; GFX9-NEXT:    v_writelane_b32 v40, s30, 0
-; GFX9-NEXT:    s_mov_b32 s1, s17
-; GFX9-NEXT:    s_mov_b32 s0, s16
-; GFX9-NEXT:    v_writelane_b32 v40, s31, 1
 ; GFX9-NEXT:    s_getpc_b64 s[18:19]
 ; GFX9-NEXT:    s_add_u32 s18, s18, external_void_func_i64_inreg@rel32@lo+4
 ; GFX9-NEXT:    s_addc_u32 s19, s19, external_void_func_i64_inreg@rel32@hi+12
+; GFX9-NEXT:    s_mov_b32 s1, s17
+; GFX9-NEXT:    s_mov_b32 s0, s16
+; GFX9-NEXT:    v_writelane_b32 v40, s31, 1
 ; GFX9-NEXT:    s_swappc_b64 s[30:31], s[18:19]
 ; GFX9-NEXT:    v_readlane_b32 s31, v40, 1
 ; GFX9-NEXT:    v_readlane_b32 s30, v40, 0
@@ -284,12 +284,12 @@ define void @test_call_external_void_func_v2i32_inreg(<2 x i32> inreg %arg) #0 {
 ; GFX9-NEXT:    v_writelane_b32 v40, s18, 2
 ; GFX9-NEXT:    s_addk_i32 s32, 0x400
 ; GFX9-NEXT:    v_writelane_b32 v40, s30, 0
-; GFX9-NEXT:    s_mov_b32 s1, s17
-; GFX9-NEXT:    s_mov_b32 s0, s16
-; GFX9-NEXT:    v_writelane_b32 v40, s31, 1
 ; GFX9-NEXT:    s_getpc_b64 s[18:19]
 ; GFX9-NEXT:    s_add_u32 s18, s18, external_void_func_v2i32_inreg@rel32@lo+4
 ; GFX9-NEXT:    s_addc_u32 s19, s19, external_void_func_v2i32_inreg@rel32@hi+12
+; GFX9-NEXT:    s_mov_b32 s1, s17
+; GFX9-NEXT:    s_mov_b32 s0, s16
+; GFX9-NEXT:    v_writelane_b32 v40, s31, 1
 ; GFX9-NEXT:    s_swappc_b64 s[30:31], s[18:19]
 ; GFX9-NEXT:    v_readlane_b32 s31, v40, 1
 ; GFX9-NEXT:    v_readlane_b32 s30, v40, 0
@@ -345,13 +345,13 @@ define void @test_call_external_void_func_v3i32_inreg(<3 x i32> inreg %arg) #0 {
 ; GFX9-NEXT:    v_writelane_b32 v40, s19, 2
 ; GFX9-NEXT:    s_addk_i32 s32, 0x400
 ; GFX9-NEXT:    v_writelane_b32 v40, s30, 0
+; GFX9-NEXT:    s_getpc_b64 s[20:21]
+; GFX9-NEXT:    s_add_u32 s20, s20, external_void_func_v3i32_inreg@rel32@lo+4
+; GFX9-NEXT:    s_addc_u32 s21, s21, external_void_func_v3i32_inreg@rel32@hi+12
 ; GFX9-NEXT:    s_mov_b32 s2, s18
 ; GFX9-NEXT:    s_mov_b32 s1, s17
 ; GFX9-NEXT:    s_mov_b32 s0, s16
 ; GFX9-NEXT:    v_writelane_b32 v40, s31, 1
-; GFX9-NEXT:    s_getpc_b64 s[20:21]
-; GFX9-NEXT:    s_add_u32 s20, s20, external_void_func_v3i32_inreg@rel32@lo+4
-; GFX9-NEXT:    s_addc_u32 s21, s21, external_void_func_v3i32_inreg@rel32@hi+12
 ; GFX9-NEXT:    s_swappc_b64 s[30:31], s[20:21]
 ; GFX9-NEXT:    v_readlane_b32 s31, v40, 1
 ; GFX9-NEXT:    v_readlane_b32 s30, v40, 0
@@ -407,14 +407,14 @@ define void @test_call_external_void_func_v4i32_inreg(<4 x i32> inreg %arg) #0 {
 ; GFX9-NEXT:    v_writelane_b32 v40, s20, 2
 ; GFX9-NEXT:    s_addk_i32 s32, 0x400
 ; GFX9-NEXT:    v_writelane_b32 v40, s30, 0
+; GFX9-NEXT:    s_getpc_b64 s[20:21]
+; GFX9-NEXT:    s_add_u32 s20, s20, external_void_func_v4i32_inreg@rel32@lo+4
+; GFX9-NEXT:    s_addc_u32 s21, s21, external_void_func_v4i32_inreg@rel32@hi+12
 ; GFX9-NEXT:    s_mov_b32 s3, s19
 ; GFX9-NEXT:    s_mov_b32 s2, s18
 ; GFX9-NEXT:    s_mov_b32 s1, s17
 ; GFX9-NEXT:    s_mov_b32 s0, s16
 ; GFX9-NEXT:    v_writelane_b32 v40, s31, 1
-; GFX9-NEXT:    s_getpc_b64 s[20:21]
-; GFX9-NEXT:    s_add_u32 s20, s20, external_void_func_v4i32_inreg@rel32@lo+4
-; GFX9-NEXT:    s_addc_u32 s21, s21, external_void_func_v4i32_inreg@rel32@hi+12
 ; GFX9-NEXT:    s_swappc_b64 s[30:31], s[20:21]
 ; GFX9-NEXT:    v_readlane_b32 s31, v40, 1
 ; GFX9-NEXT:    v_readlane_b32 s30, v40, 0
@@ -470,6 +470,9 @@ define void @test_call_external_void_func_v8i32_inreg(<8 x i32> inreg %arg) #0 {
 ; GFX9-NEXT:    v_writelane_b32 v40, s24, 2
 ; GFX9-NEXT:    s_addk_i32 s32, 0x400
 ; GFX9-NEXT:    v_writelane_b32 v40, s30, 0
+; GFX9-NEXT:    s_getpc_b64 s[24:25]
+; GFX9-NEXT:    s_add_u32 s24, s24, external_void_func_v8i32_inreg@rel32@lo+4
+; GFX9-NEXT:    s_addc_u32 s25, s25, external_void_func_v8i32_inreg@rel32@hi+12
 ; GFX9-NEXT:    s_mov_b32 s3, s19
 ; GFX9-NEXT:    s_mov_b32 s2, s18
 ; GFX9-NEXT:    s_mov_b32 s1, s17
@@ -479,9 +482,6 @@ define void @test_call_external_void_func_v8i32_inreg(<8 x i32> inreg %arg) #0 {
 ; GFX9-NEXT:    s_mov_b32 s18, s22
 ; GFX9-NEXT:    s_mov_b32 s19, s23
 ; GFX9-NEXT:    v_writelane_b32 v40, s31, 1
-; GFX9-NEXT:    s_getpc_b64 s[24:25]
-; GFX9-NEXT:    s_add_u32 s24, s24, external_void_func_v8i32_inreg@rel32@lo+4
-; GFX9-NEXT:    s_addc_u32 s25, s25, external_void_func_v8i32_inreg@rel32@hi+12
 ; GFX9-NEXT:    s_swappc_b64 s[30:31], s[24:25]
 ; GFX9-NEXT:    v_readlane_b32 s31, v40, 1
 ; GFX9-NEXT:    v_readlane_b32 s30, v40, 0
@@ -537,11 +537,11 @@ define void @test_call_external_void_func_f16_inreg(half inreg %arg) #0 {
 ; GFX9-NEXT:    v_writelane_b32 v40, s17, 2
 ; GFX9-NEXT:    s_addk_i32 s32, 0x400
 ; GFX9-NEXT:    v_writelane_b32 v40, s30, 0
-; GFX9-NEXT:    s_mov_b32 s0, s16
-; GFX9-NEXT:    v_writelane_b32 v40, s31, 1
 ; GFX9-NEXT:    s_getpc_b64 s[18:19]
 ; GFX9-NEXT:    s_add_u32 s18, s18, external_void_func_f16_inreg@rel32@lo+4
 ; GFX9-NEXT:    s_addc_u32 s19, s19, external_void_func_f16_inreg@rel32@hi+12
+; GFX9-NEXT:    s_mov_b32 s0, s16
+; GFX9-NEXT:    v_writelane_b32 v40, s31, 1
 ; GFX9-NEXT:    s_swappc_b64 s[30:31], s[18:19]
 ; GFX9-NEXT:    v_readlane_b32 s31, v40, 1
 ; GFX9-NEXT:    v_readlane_b32 s30, v40, 0
@@ -597,11 +597,11 @@ define void @test_call_external_void_func_bf16_inreg(bfloat inreg %arg) #0 {
 ; GFX9-NEXT:    v_writelane_b32 v40, s17, 2
 ; GFX9-NEXT:    s_addk_i32 s32, 0x400
 ; GFX9-NEXT:    v_writelane_b32 v40, s30, 0
-; GFX9-NEXT:    s_mov_b32 s0, s16
-; GFX9-NEXT:    v_writelane_b32 v40, s31, 1
 ; GFX9-NEXT:    s_getpc_b64 s[18:19]
 ; GFX9-NEXT:    s_add_u32 s18, s18, external_void_func_bf16_inreg@rel32@lo+4
 ; GFX9-NEXT:    s_addc_u32 s19, s19, external_void_func_bf16_inreg@rel32@hi+12
+; GFX9-NEXT:    s_mov_b32 s0, s16
+; GFX9-NEXT:    v_writelane_b32 v40, s31, 1
 ; GFX9-NEXT:    s_swappc_b64 s[30:31], s[18:19]
 ; GFX9-NEXT:    v_readlane_b32 s31, v40, 1
 ; GFX9-NEXT:    v_readlane_b32 s30, v40, 0
@@ -657,11 +657,11 @@ define void @test_call_external_void_func_f32_inreg(float inreg %arg) #0 {
 ; GFX9-NEXT:    v_writelane_b32 v40, s17, 2
 ; GFX9-NEXT:    s_addk_i32 s32, 0x400
 ; GFX9-NEXT:    v_writelane_b32 v40, s30, 0
-; GFX9-NEXT:    s_mov_b32 s0, s16
-; GFX9-NEXT:    v_writelane_b32 v40, s31, 1
 ; GFX9-NEXT:    s_getpc_b64 s[18:19]
 ; GFX9-NEXT:    s_add_u32 s18, s18, external_void_func_f32_inreg@rel32@lo+4
 ; GFX9-NEXT:    s_addc_u32 s19, s19, external_void_func_f32_inreg@rel32@hi+12
+; GFX9-NEXT:    s_mov_b32 s0, s16
+; GFX9-NEXT:    v_writelane_b32 v40, s31, 1
 ; GFX9-NEXT:    s_swappc_b64 s[30:31], s[18:19]
 ; GFX9-NEXT:    v_readlane_b32 s31, v40, 1
 ; GFX9-NEXT:    v_readlane_b32 s30, v40, 0
@@ -717,12 +717,12 @@ define void @test_call_external_void_func_f64_inreg(double inreg %arg) #0 {
 ; GFX9-NEXT:    v_writelane_b32 v40, s18, 2
 ; GFX9-NEXT:    s_addk_i32 s32, 0x400
 ; GFX9-NEXT:    v_writelane_b32 v40, s30, 0
-; GFX9-NEXT:    s_mov_b32 s1, s17
-; GFX9-NEXT:    s_mov_b32 s0, s16
-; GFX9-NEXT:    v_writelane_b32 v40, s31, 1
 ; GFX9-NEXT:    s_getpc_b64 s[18:19]
 ; GFX9-NEXT:    s_add_u32 s18, s18, external_void_func_f64_inreg@rel32@lo+4
 ; GFX9-NEXT:    s_addc_u32 s19, s19, external_void_func_f64_inreg@rel32@hi+12
+; GFX9-NEXT:    s_mov_b32 s1, s17
+; GFX9-NEXT:    s_mov_b32 s0, s16
+; GFX9-NEXT:    v_writelane_b32 v40, s31, 1
 ; GFX9-NEXT:    s_swappc_b64 s[30:31], s[18:19]
 ; GFX9-NEXT:    v_readlane_b32 s31, v40, 1
 ; GFX9-NEXT:    v_readlane_b32 s30, v40, 0
@@ -778,11 +778,11 @@ define void @test_call_external_void_func_v2f16_inreg(<2 x half> inreg %arg) #0
 ; GFX9-NEXT:    v_writelane_b32 v40, s17, 2
 ; GFX9-NEXT:    s_addk_i32 s32, 0x400
 ; GFX9-NEXT:    v_writelane_b32 v40, s30, 0
-; GFX9-NEXT:    s_mov_b32 s0, s16
-; GFX9-NEXT:    v_writelane_b32 v40, s31, 1
 ; GFX9-NEXT:    s_getpc_b64 s[18:19]
 ; GFX9-NEXT:    s_add_u32 s18, s18, external_void_func_v2f16_inreg@rel32@lo+4
 ; GFX9-NEXT:    s_addc_u32 s19, s19, external_void_func_v2f16_inreg@rel32@hi+12
+; GFX9-NEXT:    s_mov_b32 s0, s16
+; GFX9-NEXT:    v_writelane_b32 v40, s31, 1
 ; GFX9-NEXT:    s_swappc_b64 s[30:31], s[18:19]
 ; GFX9-NEXT:    v_readlane_b32 s31, v40, 1
 ; GFX9-NEXT:    v_readlane_b32 s30, v40, 0
@@ -839,11 +839,11 @@ define void @test_call_external_void_func_v2bf16_inreg(<2 x bfloat> inreg %arg)
 ; GFX9-NEXT:    v_writelane_b32 v40, s17, 2
 ; GFX9-NEXT:    s_addk_i32 s32, 0x400
 ; GFX9-NEXT:    v_writelane_b32 v40, s30, 0
-; GFX9-NEXT:    s_mov_b32 s0, s16
-; GFX9-NEXT:    v_writelane_b32 v40, s31, 1
 ; GFX9-NEXT:    s_getpc_b64 s[18:19]
 ; GFX9-NEXT:    s_add_u32 s18, s18, external_void_func_v2bf16_inreg@rel32@lo+4
 ; GFX9-NEXT:    s_addc_u32 s19, s19, external_void_func_v2bf16_inreg@rel32@hi+12
+; GFX9-NEXT:    s_mov_b32 s0, s16
+; GFX9-NEXT:    v_writelane_b32 v40, s31, 1
 ; GFX9-NEXT:    s_swappc_b64 s[30:31], s[18:19]
 ; GFX9-NEXT:    v_readlane_b32 s31, v40, 1
 ; GFX9-NEXT:    v_readlane_b32 s30, v40, 0
@@ -899,12 +899,12 @@ define void @test_call_external_void_func_v3f16_inreg(<3 x half> inreg %arg) #0
 ; GFX9-NEXT:    v_writelane_b32 v40, s18, 2
 ; GFX9-NEXT:    s_addk_i32 s32, 0x400
 ; GFX9-NEXT:    v_writelane_b32 v40, s30, 0
-; GFX9-NEXT:    s_mov_b32 s1, s17
-; GFX9-NEXT:    s_mov_b32 s0, s16
-; GFX9-NEXT:    v_writelane_b32 v40, s31, 1
 ; GFX9-NEXT:    s_getpc_b64 s[18:19]
 ; GFX9-NEXT:    s_add_u32 s18, s18, external_void_func_v3f16_inreg@rel32@lo+4
 ; GFX9-NEXT:    s_addc_u32 s19, s19, external_void_func_v3f16_inreg@rel32@hi+12
+; GFX9-NEXT:    s_mov_b32 s1, s17
+; GFX9-NEXT:    s_mov_b32 s0, s16
+; GFX9-NEXT:    v_writelane_b32 v40, s31, 1
 ; GFX9-NEXT:    s_swappc_b64 s[30:31], s[18:19]
 ; GFX9-NEXT:    v_readlane_b32 s31, v40, 1
 ; GFX9-NEXT:    v_readlane_b32 s30, v40, 0
@@ -960,12 +960,12 @@ define void @test_call_external_void_func_v4f16_inreg(<4 x half> inreg %arg) #0
 ; GFX9-NEXT:    v_writelane_b32 v40, s18, 2
 ; GFX9-NEXT:    s_addk_i32 s32, 0x400
 ; GFX9-NEXT:    v_writelane_b32 v40, s30, 0
-; GFX9-NEXT:    s_mov_b32 s1, s17
-; GFX9-NEXT:    s_mov_b32 s0, s16
-; GFX9-NEXT:    v_writelane_b32 v40, s31, 1
 ; GFX9-NEXT:    s_getpc_b64 s[18:19]
 ; GFX9-NEXT:    s_add_u32 s18, s18, external_void_func_v4f16_inreg@rel32@lo+4
 ; GFX9-NEXT:    s_addc_u32 s19, s19, external_void_func_v4f16_inreg@rel32@hi+12
+; GFX9-NEXT:    s_mov_b32 s1, s17
+; GFX9-NEXT:    s_mov_b32 s0, s16
+; GFX9-NEXT:    v_writelane_b32 v40, s31, 1
 ; GFX9-NEXT:    s_swappc_b64 s[30:31], s[18:19]
 ; GFX9-NEXT:    v_readlane_b32 s31, v40, 1
 ; GFX9-NEXT:    v_readlane_b32 s30, v40, 0
@@ -1021,12 +1021,12 @@ define void @test_call_external_void_func_p0_inreg(ptr inreg %arg) #0 {
 ; GFX9-NEXT:    v_writelane_b32 v40, s18, 2
 ; GFX9-NEXT:    s_addk_i32 s32, 0x400
 ; GFX9-NEXT:    v_writelane_b32 v40, s30, 0
-; GFX9-NEXT:    s_mov_b32 s1, s17
-; GFX9-NEXT:    s_mov_b32 s0, s16
-; GFX9-NEXT:    v_writelane_b32 v40, s31, 1
 ; GFX9-NEXT:    s_getpc_b64 s[18:19]
 ; GFX9-NEXT:    s_add_u32 s18, s18, external_void_func_p0_inreg@rel32@lo+4
 ; GFX9-NEXT:    s_addc_u32 s19, s19, external_void_func_p0_inreg@rel32@hi+12
+; GFX9-NEXT:    s_mov_b32 s1, s17
+; GFX9-NEXT:    s_mov_b32 s0, s16
+; GFX9-NEXT:    v_writelane_b32 v40, s31, 1
 ; GFX9-NEXT:    s_swappc_b64 s[30:31], s[18:19]
 ; GFX9-NEXT:    v_readlane_b32 s31, v40, 1
 ; GFX9-NEXT:    v_readlane_b32 s30, v40, 0
@@ -1082,12 +1082,12 @@ define void @test_call_external_void_func_p1_inreg(ptr addrspace(1) inreg %arg)
 ; GFX9-NEXT:    v_writelane_b32 v40, s18, 2
 ; GFX9-NEXT:    s_addk_i32 s32, 0x400
 ; GFX9-NEXT:    v_writelane_b32 v40, s30, 0
-; GFX9-NEXT:    s_mov_b32 s1, s17
-; GFX9-NEXT:    s_mov_b32 s0, s16
-; GFX9-NEXT:    v_writelane_b32 v40, s31, 1
 ; GFX9-NEXT:    s_getpc_b64 s[18:19]
 ; GFX9-NEXT:    s_add_u32 s18, s18, external_void_func_p1_inreg@rel32@lo+4
 ; GFX9-NEXT:    s_addc_u32 s19, s19, external_void_func_p1_inreg@rel32@hi+12
+; GFX9-NEXT:    s_mov_b32 s1, s17
+; GFX9-NEXT:    s_mov_b32 s0, s16
+; GFX9-NEXT:    v_writelane_b32 v40, s31, 1
 ; GFX9-NEXT:    s_swappc_b64 s[30:31], s[18:19]
 ; GFX9-NEXT:    v_readlane_b32 s31, v40, 1
 ; GFX9-NEXT:    v_readlane_b32 s30, v40, 0
@@ -1143,11 +1143,11 @@ define void @test_call_external_void_func_p3_inreg(ptr addrspace(3) inreg %arg)
 ; GFX9-NEXT:    v_writelane_b32 v40, s17, 2
 ; GFX9-NEXT:    s_addk_i32 s32, 0x400
 ; GFX9-NEXT:    v_writelane_b32 v40, s30, 0
-; GFX9-NEXT:    s_mov_b32 s0, s16
-; GFX9-NEXT:    v_writelane_b32 v40, s31, 1
 ; GFX9-NEXT:    s_getpc_b64 s[18:19]
 ; GFX9-NEXT:    s_add_u32 s18, s18, external_void_func_p3_inreg@rel32@lo+4
 ; GFX9-NEXT:    s_addc_u32 s19, s19, external_void_func_p3_inreg@rel32@hi+12
+; GFX9-NEXT:    s_mov_b32 s0, s16
+; GFX9-NEXT:    v_writelane_b32 v40, s31, 1
 ; GFX9-NEXT:    s_swappc_b64 s[30:31], s[18:19]
 ; GFX9-NEXT:    v_readlane_b32 s31, v40, 1
 ; GFX9-NEXT:    v_readlane_b32 s30, v40, 0
@@ -1203,14 +1203,14 @@ define void @test_call_external_void_func_v2p1_inreg(<2 x ptr addrspace(1)> inre
 ; GFX9-NEXT:    v_writelane_b32 v40, s20, 2
 ; GFX9-NEXT:    s_addk_i32 s32, 0x400
 ; GFX9-NEXT:    v_writelane_b32 v40, s30, 0
+; GFX9-NEXT:    s_getpc_b64 s[20:21]
+; GFX9-NEXT:    s_add_u32 s20, s20, external_void_func_v2p1_inreg@rel32@lo+4
+; GFX9-NEXT:    s_addc_u32 s21, s21, external_void_func_v2p1_inreg@rel32@hi+12
 ; GFX9-NEXT:    s_mov_b32 s3, s19
 ; GFX9-NEXT:    s_mov_b32 s2, s18
 ; GFX9-NEXT:    s_mov_b32 s1, s17
 ; GFX9-NEXT:    s_mov_b32 s0, s16
 ; GFX9-NEXT:    v_writelane_b32 v40, s31, 1
-; GFX9-NEXT:    s_getpc_b64 s[20:21]
-; GFX9-NEXT:    s_add_u32 s20, s20, external_void_func_v2p1_inreg@rel32@lo+4
-; GFX9-NEXT:    s_addc_u32 s21, s21, external_void_func_v2p1_inreg@rel32@hi+12
 ; GFX9-NEXT:    s_swappc_b64 s[30:31], s[20:21]
 ; GFX9-NEXT:    v_readlane_b32 s31, v40, 1
 ; GFX9-NEXT:    v_readlane_b32 s30, v40, 0
@@ -1266,12 +1266,12 @@ define void @test_call_external_void_func_v2p5_inreg(<2 x ptr addrspace(5)> inre
 ; GFX9-NEXT:    v_writelane_b32 v40, s18, 2
 ; GFX9-NEXT:    s_addk_i32 s32, 0x400
 ; GFX9-NEXT:    v_writelane_b32 v40, s30, 0
-; GFX9-NEXT:    s_mov_b32 s1, s17
-; GFX9-NEXT:    s_mov_b32 s0, s16
-; GFX9-NEXT:    v_writelane_b32 v40, s31, 1
 ; GFX9-NEXT:    s_getpc_b64 s[18:19]
 ; GFX9-NEXT:    s_add_u32 s18, s18, external_void_func_v2p5_inreg@rel32@lo+4
 ; GFX9-NEXT:    s_addc_u32 s19, s19, external_void_func_v2p5_inreg@rel32@hi+12
+; GFX9-NEXT:    s_mov_b32 s1, s17
+; GFX9-NEXT:    s_mov_b32 s0, s16
+; GFX9-NEXT:    v_writelane_b32 v40, s31, 1
 ; GFX9-NEXT:    s_swappc_b64 s[30:31], s[18:19]
 ; GFX9-NEXT:    v_readlane_b32 s31, v40, 1
 ; GFX9-NEXT:    v_readlane_b32 s30, v40, 0
@@ -1327,15 +1327,15 @@ define void @test_call_external_void_func_i64_inreg_i32_inreg_i64_inreg(i64 inre
 ; GFX9-NEXT:    v_writelane_b32 v40, s21, 2
 ; GFX9-NEXT:    s_addk_i32 s32, 0x400
 ; GFX9-NEXT:    v_writelane_b32 v40, s30, 0
+; GFX9-NEXT:    s_getpc_b64 s[22:23]
+; GFX9-NEXT:    s_add_u32 s22, s22, external_void_func_i64_inreg_i32_inreg_i64_inreg@rel32@lo+4
+; GFX9-NEXT:    s_addc_u32 s23, s23, external_void_func_i64_inreg_i32_inreg_i64_inreg@rel32@hi+12
 ; GFX9-NEXT:    s_mov_b32 s3, s19
 ; GFX9-NEXT:    s_mov_b32 s2, s18
 ; GFX9-NEXT:    s_mov_b32 s1, s17
 ; GFX9-NEXT:    s_mov_b32 s0, s16
 ; GFX9-NEXT:    s_mov_b32 s16, s20
 ; GFX9-NEXT:    v_writelane_b32 v40, s31, 1
-; GFX9-NEXT:    s_getpc_b64 s[22:23]
-; GFX9-NEXT:    s_add_u32 s22, s22, external_void_func_i64_inreg_i32_inreg_i64_inreg@rel32@lo+4
-; GFX9-NEXT:    s_addc_u32 s23, s23, external_void_func_i64_inreg_i32_inreg_i64_inreg@rel32@hi+12
 ; GFX9-NEXT:    s_swappc_b64 s[30:31], s[22:23]
 ; GFX9-NEXT:    v_readlane_b32 s31, v40, 1
 ; GFX9-NEXT:    v_readlane_b32 s30, v40, 0
@@ -1391,6 +1391,9 @@ define void @test_call_external_void_func_a15i32_inreg([13 x i32] inreg %arg0) #
 ; GFX9-NEXT:    v_writelane_b32 v40, s29, 2
 ; GFX9-NEXT:    s_addk_i32 s32, 0x400
 ; GFX9-NEXT:    v_writelane_b32 v40, s30, 0
+; GFX9-NEXT:    s_getpc_b64 vcc
+; GFX9-NEXT:    s_add_u32 vcc_lo, vcc_lo, external_void_func_a15i32_inreg@rel32@lo+4
+; GFX9-NEXT:    s_addc_u32 vcc_hi, vcc_hi, external_void_func_a15i32_inreg@rel32@hi+12
 ; GFX9-NEXT:    s_mov_b32 s3, s19
 ; GFX9-NEXT:    s_mov_b32 s2, s18
 ; GFX9-NEXT:    s_mov_b32 s1, s17
@@ -1405,9 +1408,6 @@ define void @test_call_external_void_func_a15i32_inreg([13 x i32] inreg %arg0) #
 ; GFX9-NEXT:    s_mov_b32 s23, s27
 ; GFX9-NEXT:    s_mov_b32 s24, s28
 ; GFX9-NEXT:    v_writelane_b32 v40, s31, 1
-; GFX9-NEXT:    s_getpc_b64 vcc
-; GFX9-NEXT:    s_add_u32 vcc_lo, vcc_lo, external_void_func_a15i32_inreg@rel32@lo+4
-; GFX9-NEXT:    s_addc_u32 vcc_hi, vcc_hi, external_void_func_a15i32_inreg@rel32@hi+12
 ; GFX9-NEXT:    s_swappc_b64 s[30:31], vcc
 ; GFX9-NEXT:    v_readlane_b32 s31, v40, 1
 ; GFX9-NEXT:    v_readlane_b32 s30, v40, 0
@@ -1465,6 +1465,9 @@ define void @test_call_external_void_func_a15i32_inreg_i32_inreg([13 x i32] inre
 ; GFX9-NEXT:    v_writelane_b32 v40, s21, 2
 ; GFX9-NEXT:    s_addk_i32 s32, 0x400
 ; GFX9-NEXT:    v_writelane_b32 v40, s30, 0
+; GFX9-NEXT:    s_getpc_b64 s[22:23]
+; GFX9-NEXT:    s_add_u32 s22, s22, external_void_func_a15i32_inreg_i32_inreg__noimplicit@rel32@lo+4
+; GFX9-NEXT:    s_addc_u32 s23, s23, external_void_func_a15i32_inreg_i32_inreg__noimplicit@rel32@hi+12
 ; GFX9-NEXT:    s_mov_b32 s3, s7
 ; GFX9-NEXT:    s_mov_b32 s2, s6
 ; GFX9-NEXT:    s_mov_b32 s1, s5
@@ -1480,9 +1483,6 @@ define void @test_call_external_void_func_a15i32_inreg_i32_inreg([13 x i32] inre
 ; GFX9-NEXT:    s_mov_b32 s15, s19
 ; GFX9-NEXT:    s_mov_b32 s16, s20
 ; GFX9-NEXT:    v_writelane_b32 v40, s31, 1
-; GFX9-NEXT:    s_getpc_b64 s[22:23]
-; GFX9-NEXT:    s_add_u32 s22, s22, external_void_func_a15i32_inreg_i32_inreg__noimplicit@rel32@lo+4
-; GFX9-NEXT:    s_addc_u32 s23, s23, external_void_func_a15i32_inreg_i32_inreg__noimplicit@rel32@hi+12
 ; GFX9-NEXT:    s_swappc_b64 s[30:31], s[22:23]
 ; GFX9-NEXT:    v_readlane_b32 s31, v40, 1
 ; GFX9-NEXT:    v_readlane_b32 s30, v40, 0
diff --git a/llvm/test/CodeGen/AMDGPU/call-argument-types.ll b/llvm/test/CodeGen/AMDGPU/call-argument-types.ll
index 35d00390067d8..16fe85bf138b2 100644
--- a/llvm/test/CodeGen/AMDGPU/call-argument-types.ll
+++ b/llvm/test/CodeGen/AMDGPU/call-argument-types.ll
@@ -71,12 +71,12 @@ define amdgpu_kernel void @test_call_external_void_func_i1_imm() #0 {
 ; VI-NEXT:    s_addc_u32 s37, s37, 0
 ; VI-NEXT:    s_mov_b64 s[6:7], s[0:1]
 ; VI-NEXT:    s_mov_b64 s[0:1], s[36:37]
-; VI-NEXT:    s_mov_b64 s[2:3], s[38:39]
-; VI-NEXT:    v_mov_b32_e32 v0, 1
-; VI-NEXT:    s_mov_b32 s32, 0
 ; VI-NEXT:    s_getpc_b64 s[4:5]
 ; VI-NEXT:    s_add_u32 s4, s4, external_void_func_i1@rel32@lo+4
 ; VI-NEXT:    s_addc_u32 s5, s5, external_void_func_i1@rel32@hi+12
+; VI-NEXT:    s_mov_b64 s[2:3], s[38:39]
+; VI-NEXT:    v_mov_b32_e32 v0, 1
+; VI-NEXT:    s_mov_b32 s32, 0
 ; VI-NEXT:    s_swappc_b64 s[30:31], s[4:5]
 ; VI-NEXT:    s_endpgm
 ;
@@ -90,12 +90,12 @@ define amdgpu_kernel void @test_call_external_void_func_i1_imm() #0 {
 ; CI-NEXT:    s_addc_u32 s37, s37, 0
 ; CI-NEXT:    s_mov_b64 s[6:7], s[0:1]
 ; CI-NEXT:    s_mov_b64 s[0:1], s[36:37]
-; CI-NEXT:    s_mov_b64 s[2:3], s[38:39]
-; CI-NEXT:    v_mov_b32_e32 v0, 1
-; CI-NEXT:    s_mov_b32 s32, 0
 ; CI-NEXT:    s_getpc_b64 s[4:5]
 ; CI-NEXT:    s_add_u32 s4, s4, external_void_func_i1@rel32@lo+4
 ; CI-NEXT:    s_addc_u32 s5, s5, external_void_func_i1@rel32@hi+12
+; CI-NEXT:    s_mov_b64 s[2:3], s[38:39]
+; CI-NEXT:    v_mov_b32_e32 v0, 1
+; CI-NEXT:    s_mov_b32 s32, 0
 ; CI-NEXT:    s_swappc_b64 s[30:31], s[4:5]
 ; CI-NEXT:    s_endpgm
 ;
@@ -109,24 +109,23 @@ define amdgpu_kernel void @test_call_external_void_func_i1_imm() #0 {
 ; GFX9-NEXT:    s_addc_u32 s37, s37, 0
 ; GFX9-NEXT:    s_mov_b64 s[6:7], s[0:1]
 ; GFX9-NEXT:    s_mov_b64 s[0:1], s[36:37]
-; GFX9-NEXT:    s_mov_b64 s[2:3], s[38:39]
-; GFX9-NEXT:    v_mov_b32_e32 v0, 1
-; GFX9-NEXT:    s_mov_b32 s32, 0
 ; GFX9-NEXT:    s_getpc_b64 s[4:5]
 ; GFX9-NEXT:    s_add_u32 s4, s4, external_void_func_i1@rel32@lo+4
 ; GFX9-NEXT:    s_addc_u32 s5, s5, external_void_func_i1@rel32@hi+12
+; GFX9-NEXT:    s_mov_b64 s[2:3], s[38:39]
+; GFX9-NEXT:    v_mov_b32_e32 v0, 1
+; GFX9-NEXT:    s_mov_b32 s32, 0
 ; GFX9-NEXT:    s_swappc_b64 s[30:31], s[4:5]
 ; GFX9-NEXT:    s_endpgm
 ;
 ; GFX11-LABEL: test_call_external_void_func_i1_imm:
 ; GFX11:       ; %bb.0:
 ; GFX11-NEXT:    v_mov_b32_e32 v0, 1
-; GFX11-NEXT:    s_mov_b64 s[6:7], s[0:1]
-; GFX11-NEXT:    s_mov_b32 s32, 0
 ; GFX11-NEXT:    s_getpc_b64 s[2:3]
 ; GFX11-NEXT:    s_add_u32 s2, s2, external_void_func_i1@rel32@lo+4
 ; GFX11-NEXT:    s_addc_u32 s3, s3, external_void_func_i1@rel32@hi+12
-; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_mov_b64 s[6:7], s[0:1]
+; GFX11-NEXT:    s_mov_b32 s32, 0
 ; GFX11-NEXT:    s_swappc_b64 s[30:31], s[2:3]
 ; GFX11-NEXT:    s_endpgm
 ;
@@ -135,14 +134,14 @@ define amdgpu_kernel void @test_call_external_void_func_i1_imm() #0 {
 ; HSA-NEXT:    s_add_i32 s6, s6, s9
 ; HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s6, 8
 ; HSA-NEXT:    s_add_u32 s0, s0, s9
-; HSA-NEXT:    s_mov_b32 flat_scratch_lo, s7
 ; HSA-NEXT:    s_addc_u32 s1, s1, 0
-; HSA-NEXT:    s_mov_b64 s[6:7], s[4:5]
-; HSA-NEXT:    v_mov_b32_e32 v0, 1
-; HSA-NEXT:    s_mov_b32 s32, 0
+; HSA-NEXT:    s_mov_b32 flat_scratch_lo, s7
 ; HSA-NEXT:    s_getpc_b64 s[8:9]
 ; HSA-NEXT:    s_add_u32 s8, s8, external_void_func_i1@rel32@lo+4
 ; HSA-NEXT:    s_addc_u32 s9, s9, external_void_func_i1@rel32@hi+12
+; HSA-NEXT:    s_mov_b64 s[6:7], s[4:5]
+; HSA-NEXT:    v_mov_b32_e32 v0, 1
+; HSA-NEXT:    s_mov_b32 s32, 0
 ; HSA-NEXT:    s_swappc_b64 s[30:31], s[8:9]
 ; HSA-NEXT:    s_endpgm
   call void @external_void_func_i1(i1 true)
@@ -164,11 +163,11 @@ define amdgpu_kernel void @test_call_external_void_func_i1_signext(i32) #0 {
 ; VI-NEXT:    s_addc_u32 s37, s37, 0
 ; VI-NEXT:    s_mov_b64 s[6:7], s[0:1]
 ; VI-NEXT:    s_mov_b64 s[0:1], s[36:37]
-; VI-NEXT:    s_mov_b64 s[2:3], s[38:39]
-; VI-NEXT:    s_mov_b32 s32, 0
 ; VI-NEXT:    s_getpc_b64 s[4:5]
 ; VI-NEXT:    s_add_u32 s4, s4, external_void_func_i1_signext@rel32@lo+4
 ; VI-NEXT:    s_addc_u32 s5, s5, external_void_func_i1_signext@rel32@hi+12
+; VI-NEXT:    s_mov_b64 s[2:3], s[38:39]
+; VI-NEXT:    s_mov_b32 s32, 0
 ; VI-NEXT:    v_bfe_i32 v0, v0, 0, 1
 ; VI-NEXT:    s_swappc_b64 s[30:31], s[4:5]
 ; VI-NEXT:    s_endpgm
@@ -187,11 +186,11 @@ define amdgpu_kernel void @test_call_external_void_func_i1_signext(i32) #0 {
 ; CI-NEXT:    s_addc_u32 s37, s37, 0
 ; CI-NEXT:    s_mov_b64 s[6:7], s[0:1]
 ; CI-NEXT:    s_mov_b64 s[0:1], s[36:37]
-; CI-NEXT:    s_mov_b64 s[2:3], s[38:39]
-; CI-NEXT:    s_mov_b32 s32, 0
 ; CI-NEXT:    s_getpc_b64 s[4:5]
 ; CI-NEXT:    s_add_u32 s4, s4, external_void_func_i1_signext@rel32@lo+4
 ; CI-NEXT:    s_addc_u32 s5, s5, external_void_func_i1_signext@rel32@hi+12
+; CI-NEXT:    s_mov_b64 s[2:3], s[38:39]
+; CI-NEXT:    s_mov_b32 s32, 0
 ; CI-NEXT:    v_bfe_i32 v0, v0, 0, 1
 ; CI-NEXT:    s_swappc_b64 s[30:31], s[4:5]
 ; CI-NEXT:    s_endpgm
@@ -210,11 +209,11 @@ define amdgpu_kernel void @test_call_external_void_func_i1_signext(i32) #0 {
 ; GFX9-NEXT:    s_addc_u32 s37, s37, 0
 ; GFX9-NEXT:    s_mov_b64 s[6:7], s[0:1]
 ; GFX9-NEXT:    s_mov_b64 s[0:1], s[36:37]
-; GFX9-NEXT:    s_mov_b64 s[2:3], s[38:39]
-; GFX9-NEXT:    s_mov_b32 s32, 0
 ; GFX9-NEXT:    s_getpc_b64 s[4:5]
 ; GFX9-NEXT:    s_add_u32 s4, s4, external_void_func_i1_signext@rel32@lo+4
 ; GFX9-NEXT:    s_addc_u32 s5, s5, external_void_func_i1_signext@rel32@hi+12
+; GFX9-NEXT:    s_mov_b64 s[2:3], s[38:39]
+; GFX9-NEXT:    s_mov_b32 s32, 0
 ; GFX9-NEXT:    v_bfe_i32 v0, v0, 0, 1
 ; GFX9-NEXT:    s_swappc_b64 s[30:31], s[4:5]
 ; GFX9-NEXT:    s_endpgm
@@ -226,10 +225,10 @@ define amdgpu_kernel void @test_call_external_void_func_i1_signext(i32) #0 {
 ; GFX11-NEXT:    s_mov_b64 s[6:7], s[0:1]
 ; GFX11-NEXT:    buffer_load_u8 v0, off, s[0:3], 0 glc dlc
 ; GFX11-NEXT:    s_waitcnt vmcnt(0)
-; GFX11-NEXT:    s_mov_b32 s32, 0
 ; GFX11-NEXT:    s_getpc_b64 s[2:3]
 ; GFX11-NEXT:    s_add_u32 s2, s2, external_void_func_i1_signext@rel32@lo+4
 ; GFX11-NEXT:    s_addc_u32 s3, s3, external_void_func_i1_signext@rel32@hi+12
+; GFX11-NEXT:    s_mov_b32 s32, 0
 ; GFX11-NEXT:    v_bfe_i32 v0, v0, 0, 1
 ; GFX11-NEXT:    s_swappc_b64 s[30:31], s[2:3]
 ; GFX11-NEXT:    s_endpgm
@@ -244,12 +243,12 @@ define amdgpu_kernel void @test_call_external_void_func_i1_signext(i32) #0 {
 ; HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s8, 8
 ; HSA-NEXT:    s_add_u32 s0, s0, s11
 ; HSA-NEXT:    s_addc_u32 s1, s1, 0
-; HSA-NEXT:    s_mov_b64 s[6:7], s[4:5]
-; HSA-NEXT:    s_mov_b32 s32, 0
 ; HSA-NEXT:    s_mov_b32 flat_scratch_lo, s9
 ; HSA-NEXT:    s_getpc_b64 s[8:9]
 ; HSA-NEXT:    s_add_u32 s8, s8, external_void_func_i1_signext@rel32@lo+4
 ; HSA-NEXT:    s_addc_u32 s9, s9, external_void_func_i1_signext@rel32@hi+12
+; HSA-NEXT:    s_mov_b64 s[6:7], s[4:5]
+; HSA-NEXT:    s_mov_b32 s32, 0
 ; HSA-NEXT:    v_bfe_i32 v0, v0, 0, 1
 ; HSA-NEXT:    s_swappc_b64 s[30:31], s[8:9]
 ; HSA-NEXT:    s_endpgm
@@ -274,11 +273,11 @@ define amdgpu_kernel void @test_call_external_void_func_i1_zeroext(i32) #0 {
 ; VI-NEXT:    s_addc_u32 s37, s37, 0
 ; VI-NEXT:    s_mov_b64 s[6:7], s[0:1]
 ; VI-NEXT:    s_mov_b64 s[0:1], s[36:37]
-; VI-NEXT:    s_mov_b64 s[2:3], s[38:39]
-; VI-NEXT:    s_mov_b32 s32, 0
 ; VI-NEXT:    s_getpc_b64 s[4:5]
 ; VI-NEXT:    s_add_u32 s4, s4, external_void_func_i1_zeroext@rel32@lo+4
 ; VI-NEXT:    s_addc_u32 s5, s5, external_void_func_i1_zeroext@rel32@hi+12
+; VI-NEXT:    s_mov_b64 s[2:3], s[38:39]
+; VI-NEXT:    s_mov_b32 s32, 0
 ; VI-NEXT:    v_and_b32_e32 v0, 1, v0
 ; VI-NEXT:    s_swappc_b64 s[30:31], s[4:5]
 ; VI-NEXT:    s_endpgm
@@ -297,11 +296,11 @@ define amdgpu_kernel void @test_call_external_void_func_i1_zeroext(i32) #0 {
 ; CI-NEXT:    s_addc_u32 s37, s37, 0
 ; CI-NEXT:    s_mov_b64 s[6:7], s[0:1]
 ; CI-NEXT:    s_mov_b64 s[0:1], s[36:37]
-; CI-NEXT:    s_mov_b64 s[2:3], s[38:39]
-; CI-NEXT:    s_mov_b32 s32, 0
 ; CI-NEXT:    s_getpc_b64 s[4:5]
 ; CI-NEXT:    s_add_u32 s4, s4, external_void_func_i1_zeroext@rel32@lo+4
 ; CI-NEXT:    s_addc_u32 s5, s5, external_void_func_i1_zeroext@rel32@hi+12
+; CI-NEXT:    s_mov_b64 s[2:3], s[38:39]
+; CI-NEXT:    s_mov_b32 s32, 0
 ; CI-NEXT:    v_and_b32_e32 v0, 1, v0
 ; CI-NEXT:    s_swappc_b64 s[30:31], s[4:5]
 ; CI-NEXT:    s_endpgm
@@ -320,11 +319,11 @@ define amdgpu_kernel void @test_call_external_void_func_i1_zeroext(i32) #0 {
 ; GFX9-NEXT:    s_addc_u32 s37, s37, 0
 ; GFX9-NEXT:    s_mov_b64 s[6:7], s[0:1]
 ; GFX9-NEXT:    s_mov_b64 s[0:1], s[36:37]
-; GFX9-NEXT:    s_mov_b64 s[2:3], s[38:39]
-; GFX9-NEXT:    s_mov_b32 s32, 0
 ; GFX9-NEXT:    s_getpc_b64 s[4:5]
 ; GFX9-NEXT:    s_add_u32 s4, s4, external_void_func_i1_zeroext@rel32@lo+4
 ; GFX9-NEXT:    s_addc_u32 s5, s5, external_void_func_i1_zeroext@rel32@hi+12
+; GFX9-NEXT:    s_mov_b64 s[2:3], s[38:39]
+; GFX9-NEXT:    s_mov_b32 s32, 0
 ; GFX9-NEXT:    v_and_b32_e32 v0, 1, v0
 ; GFX9-NEXT:    s_swappc_b64 s[30:31], s[4:5]
 ; GFX9-NEXT:    s_endpgm
@@ -336,10 +335,10 @@ define amdgpu_kernel void @test_call_external_void_func_i1_zeroext(i32) #0 {
 ; GFX11-NEXT:    s_mov_b64 s[6:7], s[0:1]
 ; GFX11-NEXT:    buffer_load_u8 v0, off, s[0:3], 0 glc dlc
 ; GFX11-NEXT:    s_waitcnt vmcnt(0)
-; GFX11-NEXT:    s_mov_b32 s32, 0
 ; GFX11-NEXT:    s_getpc_b64 s[2:3]
 ; GFX11-NEXT:    s_add_u32 s2, s2, external_void_func_i1_zeroext@rel32@lo+4
 ; GFX11-NEXT:    s_addc_u32 s3, s3, external_void_func_i1_zeroext@rel32@hi+12
+; GFX11-NEXT:    s_mov_b32 s32, 0
 ; GFX11-NEXT:    v_and_b32_e32 v0, 1, v0
 ; GFX11-NEXT:    s_swappc_b64 s[30:31], s[2:3]
 ; GFX11-NEXT:    s_endpgm
@@ -354,12 +353,12 @@ define amdgpu_kernel void @test_call_external_void_func_i1_zeroext(i32) #0 {
 ; HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s8, 8
 ; HSA-NEXT:    s_add_u32 s0, s0, s11
 ; HSA-NEXT:    s_addc_u32 s1, s1, 0
-; HSA-NEXT:    s_mov_b64 s[6:7], s[4:5]
-; HSA-NEXT:    s_mov_b32 s32, 0
 ; HSA-NEXT:    s_mov_b32 flat_scratch_lo, s9
 ; HSA-NEXT:    s_getpc_b64 s[8:9]
 ; HSA-NEXT:    s_add_u32 s8, s8, external_void_func_i1_zeroext@rel32@lo+4
 ; HSA-NEXT:    s_addc_u32 s9, s9, external_void_func_i1_zeroext@rel32@hi+12
+; HSA-NEXT:    s_mov_b64 s[6:7], s[4:5]
+; HSA-NEXT:    s_mov_b32 s32, 0
 ; HSA-NEXT:    v_and_b32_e32 v0, 1, v0
 ; HSA-NEXT:    s_swappc_b64 s[30:31], s[8:9]
 ; HSA-NEXT:    s_endpgm
@@ -379,12 +378,12 @@ define amdgpu_kernel void @test_call_external_void_func_i8_imm(i32) #0 {
 ; VI-NEXT:    s_addc_u32 s37, s37, 0
 ; VI-NEXT:    s_mov_b64 s[6:7], s[0:1]
 ; VI-NEXT:    s_mov_b64 s[0:1], s[36:37]
-; VI-NEXT:    s_mov_b64 s[2:3], s[38:39]
-; VI-NEXT:    v_mov_b32_e32 v0, 0x7b
-; VI-NEXT:    s_mov_b32 s32, 0
 ; VI-NEXT:    s_getpc_b64 s[4:5]
 ; VI-NEXT:    s_add_u32 s4, s4, external_void_func_i8@rel32@lo+4
 ; VI-NEXT:    s_addc_u32 s5, s5, external_void_func_i8@rel32@hi+12
+; VI-NEXT:    s_mov_b64 s[2:3], s[38:39]
+; VI-NEXT:    v_mov_b32_e32 v0, 0x7b
+; VI-NEXT:    s_mov_b32 s32, 0
 ; VI-NEXT:    s_swappc_b64 s[30:31], s[4:5]
 ; VI-NEXT:    s_endpgm
 ;
@@ -398,12 +397,12 @@ define amdgpu_kernel void @test_call_external_void_func_i8_imm(i32) #0 {
 ; CI-NEXT:    s_addc_u32 s37, s37, 0
 ; CI-NEXT:    s_mov_b64 s[6:7], s[0:1]
 ; CI-NEXT:    s_mov_b64 s[0:1], s[36:37]
-; CI-NEXT:    s_mov_b64 s[2:3], s[38:39]
-; CI-NEXT:    v_mov_b32_e32 v0, 0x7b
-; CI-NEXT:    s_mov_b32 s32, 0
 ; CI-NEXT:    s_getpc_b64 s[4:5]
 ; CI-NEXT:    s_add_u32 s4, s4, external_void_func_i8@rel32@lo+4
 ; CI-NEXT:    s_addc_u32 s5, s5, external_void_func_i8@rel32@hi+12
+; CI-NEXT:    s_mov_b64 s[2:3], s[38:39]
+; CI-NEXT:    v_mov_b32_e32 v0, 0x7b
+; CI-NEXT:    s_mov_b32 s32, 0
 ; CI-NEXT:    s_swappc_b64 s[30:31], s[4:5]
 ; CI-NEXT:    s_endpgm
 ;
@@ -417,24 +416,23 @@ define amdgpu_kernel void @test_call_external_void_func_i8_imm(i32) #0 {
 ; GFX9-NEXT:    s_addc_u32 s37, s37, 0
 ; GFX9-NEXT:    s_mov_b64 s[6:7], s[0:1]
 ; GFX9-NEXT:    s_mov_b64 s[0:1], s[36:37]
-; GFX9-NEXT:    s_mov_b64 s[2:3], s[38:39]
-; GFX9-NEXT:    v_mov_b32_e32 v0, 0x7b
-; GFX9-NEXT:    s_mov_b32 s32, 0
 ; GFX9-NEXT:    s_getpc_b64 s[4:5]
 ; GFX9-NEXT:    s_add_u32 s4, s4, external_void_func_i8@rel32@lo+4
 ; GFX9-NEXT:    s_addc_u32 s5, s5, external_void_func_i8@rel32@hi+12
+; GFX9-NEXT:    s_mov_b64 s[2:3], s[38:39]
+; GFX9-NEXT:    v_mov_b32_e32 v0, 0x7b
+; GFX9-NEXT:    s_mov_b32 s32, 0
 ; GFX9-NEXT:    s_swappc_b64 s[30:31], s[4:5]
 ; GFX9-NEXT:    s_endpgm
 ;
 ; GFX11-LABEL: test_call_external_void_func_i8_imm:
 ; GFX11:       ; %bb.0:
 ; GFX11-NEXT:    v_mov_b32_e32 v0, 0x7b
-; GFX11-NEXT:    s_mov_b64 s[6:7], s[0:1]
-; GFX11-NEXT:    s_mov_b32 s32, 0
 ; GFX11-NEXT:    s_getpc_b64 s[2:3]
 ; GFX11-NEXT:    s_add_u32 s2, s2, external_void_func_i8@rel32@lo+4
 ; GFX11-NEXT:    s_addc_u32 s3, s3, external_void_func_i8@rel32@hi+12
-; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_mov_b64 s[6:7], s[0:1]
+; GFX11-NEXT:    s_mov_b32 s32, 0
 ; GFX11-NEXT:    s_swappc_b64 s[30:31], s[2:3]
 ; GFX11-NEXT:    s_endpgm
 ;
@@ -444,13 +442,13 @@ define amdgpu_kernel void @test_call_external_void_func_i8_imm(i32) #0 {
 ; HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s8, 8
 ; HSA-NEXT:    s_add_u32 s0, s0, s11
 ; HSA-NEXT:    s_addc_u32 s1, s1, 0
-; HSA-NEXT:    s_mov_b64 s[6:7], s[4:5]
-; HSA-NEXT:    v_mov_b32_e32 v0, 0x7b
-; HSA-NEXT:    s_mov_b32 s32, 0
 ; HSA-NEXT:    s_mov_b32 flat_scratch_lo, s9
 ; HSA-NEXT:    s_getpc_b64 s[8:9]
 ; HSA-NEXT:    s_add_u32 s8, s8, external_void_func_i8@rel32@lo+4
 ; HSA-NEXT:    s_addc_u32 s9, s9, external_void_func_i8@rel32@hi+12
+; HSA-NEXT:    s_mov_b64 s[6:7], s[4:5]
+; HSA-NEXT:    v_mov_b32_e32 v0, 0x7b
+; HSA-NEXT:    s_mov_b32 s32, 0
 ; HSA-NEXT:    s_swappc_b64 s[30:31], s[8:9]
 ; HSA-NEXT:    s_endpgm
   call void @external_void_func_i8(i8 123)
@@ -473,11 +471,11 @@ define amdgpu_kernel void @test_call_external_void_func_i8_signext(i32) #0 {
 ; VI-NEXT:    s_addc_u32 s37, s37, 0
 ; VI-NEXT:    s_mov_b64 s[6:7], s[0:1]
 ; VI-NEXT:    s_mov_b64 s[0:1], s[36:37]
-; VI-NEXT:    s_mov_b64 s[2:3], s[38:39]
-; VI-NEXT:    s_mov_b32 s32, 0
 ; VI-NEXT:    s_getpc_b64 s[4:5]
 ; VI-NEXT:    s_add_u32 s4, s4, external_void_func_i8_signext@rel32@lo+4
 ; VI-NEXT:    s_addc_u32 s5, s5, external_void_func_i8_signext@rel32@hi+12
+; VI-NEXT:    s_mov_b64 s[2:3], s[38:39]
+; VI-NEXT:    s_mov_b32 s32, 0
 ; VI-NEXT:    s_swappc_b64 s[30:31], s[4:5]
 ; VI-NEXT:    s_endpgm
 ;
@@ -495,11 +493,11 @@ define amdgpu_kernel void @test_call_external_void_func_i8_signext(i32) #0 {
 ; CI-NEXT:    s_addc_u32 s37, s37, 0
 ; CI-NEXT:    s_mov_b64 s[6:7], s[0:1]
 ; CI-NEXT:    s_mov_b64 s[0:1], s[36:37]
-; CI-NEXT:    s_mov_b64 s[2:3], s[38:39]
-; CI-NEXT:    s_mov_b32 s32, 0
 ; CI-NEXT:    s_getpc_b64 s[4:5]
 ; CI-NEXT:    s_add_u32 s4, s4, external_void_func_i8_signext@rel32@lo+4
 ; CI-NEXT:    s_addc_u32 s5, s5, external_void_func_i8_signext@rel32@hi+12
+; CI-NEXT:    s_mov_b64 s[2:3], s[38:39]
+; CI-NEXT:    s_mov_b32 s32, 0
 ; CI-NEXT:    s_swappc_b64 s[30:31], s[4:5]
 ; CI-NEXT:    s_endpgm
 ;
@@ -517,11 +515,11 @@ define amdgpu_kernel void @test_call_external_void_func_i8_signext(i32) #0 {
 ; GFX9-NEXT:    s_addc_u32 s37, s37, 0
 ; GFX9-NEXT:    s_mov_b64 s[6:7], s[0:1]
 ; GFX9-NEXT:    s_mov_b64 s[0:1], s[36:37]
-; GFX9-NEXT:    s_mov_b64 s[2:3], s[38:39]
-; GFX9-NEXT:    s_mov_b32 s32, 0
 ; GFX9-NEXT:    s_getpc_b64 s[4:5]
 ; GFX9-NEXT:    s_add_u32 s4, s4, external_void_func_i8_signext@rel32@lo+4
 ; GFX9-NEXT:    s_addc_u32 s5, s5, external_void_func_i8_signext@rel32@hi+12
+; GFX9-NEXT:    s_mov_b64 s[2:3], s[38:39]
+; GFX9-NEXT:    s_mov_b32 s32, 0
 ; GFX9-NEXT:    s_swappc_b64 s[30:31], s[4:5]
 ; GFX9-NEXT:    s_endpgm
 ;
@@ -532,11 +530,10 @@ define amdgpu_kernel void @test_call_external_void_func_i8_signext(i32) #0 {
 ; GFX11-NEXT:    s_mov_b64 s[6:7], s[0:1]
 ; GFX11-NEXT:    buffer_load_i8 v0, off, s[0:3], 0 glc dlc
 ; GFX11-NEXT:    s_waitcnt vmcnt(0)
-; GFX11-NEXT:    s_mov_b32 s32, 0
 ; GFX11-NEXT:    s_getpc_b64 s[2:3]
 ; GFX11-NEXT:    s_add_u32 s2, s2, external_void_func_i8_signext@rel32@lo+4
 ; GFX11-NEXT:    s_addc_u32 s3, s3, external_void_func_i8_signext@rel32@hi+12
-; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_mov_b32 s32, 0
 ; GFX11-NEXT:    s_swappc_b64 s[30:31], s[2:3]
 ; GFX11-NEXT:    s_endpgm
 ;
@@ -550,12 +547,12 @@ define amdgpu_kernel void @test_call_external_void_func_i8_signext(i32) #0 {
 ; HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s8, 8
 ; HSA-NEXT:    s_add_u32 s0, s0, s11
 ; HSA-NEXT:    s_addc_u32 s1, s1, 0
-; HSA-NEXT:    s_mov_b64 s[6:7], s[4:5]
-; HSA-NEXT:    s_mov_b32 s32, 0
 ; HSA-NEXT:    s_mov_b32 flat_scratch_lo, s9
 ; HSA-NEXT:    s_getpc_b64 s[8:9]
 ; HSA-NEXT:    s_add_u32 s8, s8, external_void_func_i8_signext@rel32@lo+4
 ; HSA-NEXT:    s_addc_u32 s9, s9, external_void_func_i8_signext@rel32@hi+12
+; HSA-NEXT:    s_mov_b64 s[6:7], s[4:5]
+; HSA-NEXT:    s_mov_b32 s32, 0
 ; HSA-NEXT:    s_swappc_b64 s[30:31], s[8:9]
 ; HSA-NEXT:    s_endpgm
   %var = load volatile i8, ptr addrspace(1) undef
@@ -578,11 +575,11 @@ define amdgpu_kernel void @test_call_external_void_func_i8_zeroext(i32) #0 {
 ; VI-NEXT:    s_addc_u32 s37, s37, 0
 ; VI-NEXT:    s_mov_b64 s[6:7], s[0:1]
 ; VI-NEXT:    s_mov_b64 s[0:1], s[36:37]
-; VI-NEXT:    s_mov_b64 s[2:3], s[38:39]
-; VI-NEXT:    s_mov_b32 s32, 0
 ; VI-NEXT:    s_getpc_b64 s[4:5]
 ; VI-NEXT:    s_add_u32 s4, s4, external_void_func_i8_zeroext@rel32@lo+4
 ; VI-NEXT:    s_addc_u32 s5, s5, external_void_func_i8_zeroext@rel32@hi+12
+; VI-NEXT:    s_mov_b64 s[2:3], s[38:39]
+; VI-NEXT:    s_mov_b32 s32, 0
 ; VI-NEXT:    s_swappc_b64 s[30:31], s[4:5]
 ; VI-NEXT:    s_endpgm
 ;
@@ -600,11 +597,11 @@ define amdgpu_kernel void @test_call_external_void_func_i8_zeroext(i32) #0 {
 ; CI-NEXT:    s_addc_u32 s37, s37, 0
 ; CI-NEXT:    s_mov_b64 s[6:7], s[0:1]
 ; CI-NEXT:    s_mov_b64 s[0:1], s[36:37]
-; CI-NEXT:    s_mov_b64 s[2:3], s[38:39]
-; CI-NEXT:    s_mov_b32 s32, 0
 ; CI-NEXT:    s_getpc_b64 s[4:5]
 ; CI-NEXT:    s_add_u32 s4, s4, external_void_func_i8_zeroext@rel32@lo+4
 ; CI-NEXT:    s_addc_u32 s5, s5, external_void_func_i8_zeroext@rel32@hi+12
+; CI-NEXT:    s_mov_b64 s[2:3], s[38:39]
+; CI-NEXT:    s_mov_b32 s32, 0
 ; CI-NEXT:    s_swappc_b64 s[30:31], s[4:5]
 ; CI-NEXT:    s_endpgm
 ;
@@ -622,11 +619,11 @@ define amdgpu_kernel void @test_call_external_void_func_i8_zeroext(i32) #0 {
 ; GFX9-NEXT:    s_addc_u32 s37, s37, 0
 ; GFX9-NEXT:    s_mov_b64 s[6:7], s[0:1]
 ; GFX9-NEXT:    s_mov_b64 s[0:1], s[36:37]
-; GFX9-NEXT:    s_mov_b64 s[2:3], s[38:39]
-; GFX9-NEXT:    s_mov_b32 s32, 0
 ; GFX9-NEXT:    s_getpc_b64 s[4:5]
 ; GFX9-NEXT:    s_add_u32 s4, s4, external_void_func_i8_zeroext@rel32@lo+4
 ; GFX9-NEXT:    s_addc_u32 s5, s5, external_void_func_i8_zeroext@rel32@hi+12
+; GFX9-NEXT:    s_mov_b64 s[2:3], s[38:39]
+; GFX9-NEXT:    s_mov_b32 s32, 0
 ; GFX9-NEXT:    s_swappc_b64 s[30:31], s[4:5]
 ; GFX9-NEXT:    s_endpgm
 ;
@@ -637,11 +634,10 @@ define amdgpu_kernel void @test_call_external_void_func_i8_zeroext(i32) #0 {
 ; GFX11-NEXT:    s_mov_b64 s[6:7], s[0:1]
 ; GFX11-NEXT:    buffer_load_u8 v0, off, s[0:3], 0 glc dlc
 ; GFX11-NEXT:    s_waitcnt vmcnt(0)
-; GFX11-NEXT:    s_mov_b32 s32, 0
 ; GFX11-NEXT:    s_getpc_b64 s[2:3]
 ; GFX11-NEXT:    s_add_u32 s2, s2, external_void_func_i8_zeroext@rel32@lo+4
 ; GFX11-NEXT:    s_addc_u32 s3, s3, external_void_func_i8_zeroext@rel32@hi+12
-; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_mov_b32 s32, 0
 ; GFX11-NEXT:    s_swappc_b64 s[30:31], s[2:3]
 ; GFX11-NEXT:    s_endpgm
 ;
@@ -655,12 +651,12 @@ define amdgpu_kernel void @test_call_external_void_func_i8_zeroext(i32) #0 {
 ; HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s8, 8
 ; HSA-NEXT:    s_add_u32 s0, s0, s11
 ; HSA-NEXT:    s_addc_u32 s1, s1, 0
-; HSA-NEXT:    s_mov_b64 s[6:7], s[4:5]
-; HSA-NEXT:    s_mov_b32 s32, 0
 ; HSA-NEXT:    s_mov_b32 flat_scratch_lo, s9
 ; HSA-NEXT:    s_getpc_b64 s[8:9]
 ; HSA-NEXT:    s_add_u32 s8, s8, external_void_func_i8_zeroext@rel32@lo+4
 ; HSA-NEXT:    s_addc_u32 s9, s9, external_void_func_i8_zeroext@rel32@hi+12
+; HSA-NEXT:    s_mov_b64 s[6:7], s[4:5]
+; HSA-NEXT:    s_mov_b32 s32, 0
 ; HSA-NEXT:    s_swappc_b64 s[30:31], s[8:9]
 ; HSA-NEXT:    s_endpgm
   %var = load volatile i8, ptr addrspace(1) undef
@@ -679,12 +675,12 @@ define amdgpu_kernel void @test_call_external_void_func_i16_imm() #0 {
 ; VI-NEXT:    s_addc_u32 s37, s37, 0
 ; VI-NEXT:    s_mov_b64 s[6:7], s[0:1]
 ; VI-NEXT:    s_mov_b64 s[0:1], s[36:37]
-; VI-NEXT:    s_mov_b64 s[2:3], s[38:39]
-; VI-NEXT:    v_mov_b32_e32 v0, 0x7b
-; VI-NEXT:    s_mov_b32 s32, 0
 ; VI-NEXT:    s_getpc_b64 s[4:5]
 ; VI-NEXT:    s_add_u32 s4, s4, external_void_func_i16@rel32@lo+4
 ; VI-NEXT:    s_addc_u32 s5, s5, external_void_func_i16@rel32@hi+12
+; VI-NEXT:    s_mov_b64 s[2:3], s[38:39]
+; VI-NEXT:    v_mov_b32_e32 v0, 0x7b
+; VI-NEXT:    s_mov_b32 s32, 0
 ; VI-NEXT:    s_swappc_b64 s[30:31], s[4:5]
 ; VI-NEXT:    s_endpgm
 ;
@@ -698,12 +694,12 @@ define amdgpu_kernel void @test_call_external_void_func_i16_imm() #0 {
 ; CI-NEXT:    s_addc_u32 s37, s37, 0
 ; CI-NEXT:    s_mov_b64 s[6:7], s[0:1]
 ; CI-NEXT:    s_mov_b64 s[0:1], s[36:37]
-; CI-NEXT:    s_mov_b64 s[2:3], s[38:39]
-; CI-NEXT:    v_mov_b32_e32 v0, 0x7b
-; CI-NEXT:    s_mov_b32 s32, 0
 ; CI-NEXT:    s_getpc_b64 s[4:5]
 ; CI-NEXT:    s_add_u32 s4, s4, external_void_func_i16@rel32@lo+4
 ; CI-NEXT:    s_addc_u32 s5, s5, external_void_func_i16@rel32@hi+12
+; CI-NEXT:    s_mov_b64 s[2:3], s[38:39]
+; CI-NEXT:    v_mov_b32_e32 v0, 0x7b
+; CI-NEXT:    s_mov_b32 s32, 0
 ; CI-NEXT:    s_swappc_b64 s[30:31], s[4:5]
 ; CI-NEXT:    s_endpgm
 ;
@@ -717,24 +713,23 @@ define amdgpu_kernel void @test_call_external_void_func_i16_imm() #0 {
 ; GFX9-NEXT:    s_addc_u32 s37, s37, 0
 ; GFX9-NEXT:    s_mov_b64 s[6:7], s[0:1]
 ; GFX9-NEXT:    s_mov_b64 s[0:1], s[36:37]
-; GFX9-NEXT:    s_mov_b64 s[2:3], s[38:39]
-; GFX9-NEXT:    v_mov_b32_e32 v0, 0x7b
-; GFX9-NEXT:    s_mov_b32 s32, 0
 ; GFX9-NEXT:    s_getpc_b64 s[4:5]
 ; GFX9-NEXT:    s_add_u32 s4, s4, external_void_func_i16@rel32@lo+4
 ; GFX9-NEXT:    s_addc_u32 s5, s5, external_void_func_i16@rel32@hi+12
+; GFX9-NEXT:    s_mov_b64 s[2:3], s[38:39]
+; GFX9-NEXT:    v_mov_b32_e32 v0, 0x7b
+; GFX9-NEXT:    s_mov_b32 s32, 0
 ; GFX9-NEXT:    s_swappc_b64 s[30:31], s[4:5]
 ; GFX9-NEXT:    s_endpgm
 ;
 ; GFX11-LABEL: test_call_external_void_func_i16_imm:
 ; GFX11:       ; %bb.0:
 ; GFX11-NEXT:    v_mov_b32_e32 v0, 0x7b
-; GFX11-NEXT:    s_mov_b64 s[6:7], s[0:1]
-; GFX11-NEXT:    s_mov_b32 s32, 0
 ; GFX11-NEXT:    s_getpc_b64 s[2:3]
 ; GFX11-NEXT:    s_add_u32 s2, s2, external_void_func_i16@rel32@lo+4
 ; GFX11-NEXT:    s_addc_u32 s3, s3, external_void_func_i16@rel32@hi+12
-; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_mov_b64 s[6:7], s[0:1]
+; GFX11-NEXT:    s_mov_b32 s32, 0
 ; GFX11-NEXT:    s_swappc_b64 s[30:31], s[2:3]
 ; GFX11-NEXT:    s_endpgm
 ;
@@ -743,14 +738,14 @@ define amdgpu_kernel void @test_call_external_void_func_i16_imm() #0 {
 ; HSA-NEXT:    s_add_i32 s6, s6, s9
 ; HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s6, 8
 ; HSA-NEXT:    s_add_u32 s0, s0, s9
-; HSA-NEXT:    s_mov_b32 flat_scratch_lo, s7
 ; HSA-NEXT:    s_addc_u32 s1, s1, 0
-; HSA-NEXT:    s_mov_b64 s[6:7], s[4:5]
-; HSA-NEXT:    v_mov_b32_e32 v0, 0x7b
-; HSA-NEXT:    s_mov_b32 s32, 0
+; HSA-NEXT:    s_mov_b32 flat_scratch_lo, s7
 ; HSA-NEXT:    s_getpc_b64 s[8:9]
 ; HSA-NEXT:    s_add_u32 s8, s8, external_void_func_i16@rel32@lo+4
 ; HSA-NEXT:    s_addc_u32 s9, s9, external_void_func_i16@rel32@hi+12
+; HSA-NEXT:    s_mov_b64 s[6:7], s[4:5]
+; HSA-NEXT:    v_mov_b32_e32 v0, 0x7b
+; HSA-NEXT:    s_mov_b32 s32, 0
 ; HSA-NEXT:    s_swappc_b64 s[30:31], s[8:9]
 ; HSA-NEXT:    s_endpgm
   call void @external_void_func_i16(i16 123)
@@ -772,11 +767,11 @@ define amdgpu_kernel void @test_call_external_void_func_i16_signext(i32) #0 {
 ; VI-NEXT:    s_addc_u32 s37, s37, 0
 ; VI-NEXT:    s_mov_b64 s[6:7], s[0:1]
 ; VI-NEXT:    s_mov_b64 s[0:1], s[36:37]
-; VI-NEXT:    s_mov_b64 s[2:3], s[38:39]
-; VI-NEXT:    s_mov_b32 s32, 0
 ; VI-NEXT:    s_getpc_b64 s[4:5]
 ; VI-NEXT:    s_add_u32 s4, s4, external_void_func_i16_signext@rel32@lo+4
 ; VI-NEXT:    s_addc_u32 s5, s5, external_void_func_i16_signext@rel32@hi+12
+; VI-NEXT:    s_mov_b64 s[2:3], s[38:39]
+; VI-NEXT:    s_mov_b32 s32, 0
 ; VI-NEXT:    s_swappc_b64 s[30:31], s[4:5]
 ; VI-NEXT:    s_endpgm
 ;
@@ -794,11 +789,11 @@ define amdgpu_kernel void @test_call_external_void_func_i16_signext(i32) #0 {
 ; CI-NEXT:    s_addc_u32 s37, s37, 0
 ; CI-NEXT:    s_mov_b64 s[6:7], s[0:1]
 ; CI-NEXT:    s_mov_b64 s[0:1], s[36:37]
-; CI-NEXT:    s_mov_b64 s[2:3], s[38:39]
-; CI-NEXT:    s_mov_b32 s32, 0
 ; CI-NEXT:    s_getpc_b64 s[4:5]
 ; CI-NEXT:    s_add_u32 s4, s4, external_void_func_i16_signext@rel32@lo+4
 ; CI-NEXT:    s_addc_u32 s5, s5, external_void_func_i16_signext@rel32@hi+12
+; CI-NEXT:    s_mov_b64 s[2:3], s[38:39]
+; CI-NEXT:    s_mov_b32 s32, 0
 ; CI-NEXT:    s_swappc_b64 s[30:31], s[4:5]
 ; CI-NEXT:    s_endpgm
 ;
@@ -816,11 +811,11 @@ define amdgpu_kernel void @test_call_external_void_func_i16_signext(i32) #0 {
 ; GFX9-NEXT:    s_addc_u32 s37, s37, 0
 ; GFX9-NEXT:    s_mov_b64 s[6:7], s[0:1]
 ; GFX9-NEXT:    s_mov_b64 s[0:1], s[36:37]
-; GFX9-NEXT:    s_mov_b64 s[2:3], s[38:39]
-; GFX9-NEXT:    s_mov_b32 s32, 0
 ; GFX9-NEXT:    s_getpc_b64 s[4:5]
 ; GFX9-NEXT:    s_add_u32 s4, s4, external_void_func_i16_signext@rel32@lo+4
 ; GFX9-NEXT:    s_addc_u32 s5, s5, external_void_func_i16_signext@rel32@hi+12
+; GFX9-NEXT:    s_mov_b64 s[2:3], s[38:39]
+; GFX9-NEXT:    s_mov_b32 s32, 0
 ; GFX9-NEXT:    s_swappc_b64 s[30:31], s[4:5]
 ; GFX9-NEXT:    s_endpgm
 ;
@@ -831,11 +826,10 @@ define amdgpu_kernel void @test_call_external_void_func_i16_signext(i32) #0 {
 ; GFX11-NEXT:    s_mov_b64 s[6:7], s[0:1]
 ; GFX11-NEXT:    buffer_load_i16 v0, off, s[0:3], 0 glc dlc
 ; GFX11-NEXT:    s_waitcnt vmcnt(0)
-; GFX11-NEXT:    s_mov_b32 s32, 0
 ; GFX11-NEXT:    s_getpc_b64 s[2:3]
 ; GFX11-NEXT:    s_add_u32 s2, s2, external_void_func_i16_signext@rel32@lo+4
 ; GFX11-NEXT:    s_addc_u32 s3, s3, external_void_func_i16_signext@rel32@hi+12
-; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_mov_b32 s32, 0
 ; GFX11-NEXT:    s_swappc_b64 s[30:31], s[2:3]
 ; GFX11-NEXT:    s_endpgm
 ;
@@ -849,12 +843,12 @@ define amdgpu_kernel void @test_call_external_void_func_i16_signext(i32) #0 {
 ; HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s8, 8
 ; HSA-NEXT:    s_add_u32 s0, s0, s11
 ; HSA-NEXT:    s_addc_u32 s1, s1, 0
-; HSA-NEXT:    s_mov_b64 s[6:7], s[4:5]
-; HSA-NEXT:    s_mov_b32 s32, 0
 ; HSA-NEXT:    s_mov_b32 flat_scratch_lo, s9
 ; HSA-NEXT:    s_getpc_b64 s[8:9]
 ; HSA-NEXT:    s_add_u32 s8, s8, external_void_func_i16_signext@rel32@lo+4
 ; HSA-NEXT:    s_addc_u32 s9, s9, external_void_func_i16_signext@rel32@hi+12
+; HSA-NEXT:    s_mov_b64 s[6:7], s[4:5]
+; HSA-NEXT:    s_mov_b32 s32, 0
 ; HSA-NEXT:    s_swappc_b64 s[30:31], s[8:9]
 ; HSA-NEXT:    s_endpgm
   %var = load volatile i16, ptr addrspace(1) undef
@@ -877,11 +871,11 @@ define amdgpu_kernel void @test_call_external_void_func_i16_zeroext(i32) #0 {
 ; VI-NEXT:    s_addc_u32 s37, s37, 0
 ; VI-NEXT:    s_mov_b64 s[6:7], s[0:1]
 ; VI-NEXT:    s_mov_b64 s[0:1], s[36:37]
-; VI-NEXT:    s_mov_b64 s[2:3], s[38:39]
-; VI-NEXT:    s_mov_b32 s32, 0
 ; VI-NEXT:    s_getpc_b64 s[4:5]
 ; VI-NEXT:    s_add_u32 s4, s4, external_void_func_i16_zeroext@rel32@lo+4
 ; VI-NEXT:    s_addc_u32 s5, s5, external_void_func_i16_zeroext@rel32@hi+12
+; VI-NEXT:    s_mov_b64 s[2:3], s[38:39]
+; VI-NEXT:    s_mov_b32 s32, 0
 ; VI-NEXT:    s_swappc_b64 s[30:31], s[4:5]
 ; VI-NEXT:    s_endpgm
 ;
@@ -899,11 +893,11 @@ define amdgpu_kernel void @test_call_external_void_func_i16_zeroext(i32) #0 {
 ; CI-NEXT:    s_addc_u32 s37, s37, 0
 ; CI-NEXT:    s_mov_b64 s[6:7], s[0:1]
 ; CI-NEXT:    s_mov_b64 s[0:1], s[36:37]
-; CI-NEXT:    s_mov_b64 s[2:3], s[38:39]
-; CI-NEXT:    s_mov_b32 s32, 0
 ; CI-NEXT:    s_getpc_b64 s[4:5]
 ; CI-NEXT:    s_add_u32 s4, s4, external_void_func_i16_zeroext@rel32@lo+4
 ; CI-NEXT:    s_addc_u32 s5, s5, external_void_func_i16_zeroext@rel32@hi+12
+; CI-NEXT:    s_mov_b64 s[2:3], s[38:39]
+; CI-NEXT:    s_mov_b32 s32, 0
 ; CI-NEXT:    s_swappc_b64 s[30:31], s[4:5]
 ; CI-NEXT:    s_endpgm
 ;
@@ -921,11 +915,11 @@ define amdgpu_kernel void @test_call_external_void_func_i16_zeroext(i32) #0 {
 ; GFX9-NEXT:    s_addc_u32 s37, s37, 0
 ; GFX9-NEXT:    s_mov_b64 s[6:7], s[0:1]
 ; GFX9-NEXT:    s_mov_b64 s[0:1], s[36:37]
-; GFX9-NEXT:    s_mov_b64 s[2:3], s[38:39]
-; GFX9-NEXT:    s_mov_b32 s32, 0
 ; GFX9-NEXT:    s_getpc_b64 s[4:5]
 ; GFX9-NEXT:    s_add_u32 s4, s4, external_void_func_i16_zeroext@rel32@lo+4
 ; GFX9-NEXT:    s_addc_u32 s5, s5, external_void_func_i16_zeroext@rel32@hi+12
+; GFX9-NEXT:    s_mov_b64 s[2:3], s[38:39]
+; GFX9-NEXT:    s_mov_b32 s32, 0
 ; GFX9-NEXT:    s_swappc_b64 s[30:31], s[4:5]
 ; GFX9-NEXT:    s_endpgm
 ;
@@ -936,11 +930,10 @@ define amdgpu_kernel void @test_call_external_void_func_i16_zeroext(i32) #0 {
 ; GFX11-NEXT:    s_mov_b64 s[6:7], s[0:1]
 ; GFX11-NEXT:    buffer_load_u16 v0, off, s[0:3], 0 glc dlc
 ; GFX11-NEXT:    s_waitcnt vmcnt(0)
-; GFX11-NEXT:    s_mov_b32 s32, 0
 ; GFX11-NEXT:    s_getpc_b64 s[2:3]
 ; GFX11-NEXT:    s_add_u32 s2, s2, external_void_func_i16_zeroext@rel32@lo+4
 ; GFX11-NEXT:    s_addc_u32 s3, s3, external_void_func_i16_zeroext@rel32@hi+12
-; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_mov_b32 s32, 0
 ; GFX11-NEXT:    s_swappc_b64 s[30:31], s[2:3]
 ; GFX11-NEXT:    s_endpgm
 ;
@@ -954,12 +947,12 @@ define amdgpu_kernel void @test_call_external_void_func_i16_zeroext(i32) #0 {
 ; HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s8, 8
 ; HSA-NEXT:    s_add_u32 s0, s0, s11
 ; HSA-NEXT:    s_addc_u32 s1, s1, 0
-; HSA-NEXT:    s_mov_b64 s[6:7], s[4:5]
-; HSA-NEXT:    s_mov_b32 s32, 0
 ; HSA-NEXT:    s_mov_b32 flat_scratch_lo, s9
 ; HSA-NEXT:    s_getpc_b64 s[8:9]
 ; HSA-NEXT:    s_add_u32 s8, s8, external_void_func_i16_zeroext@rel32@lo+4
 ; HSA-NEXT:    s_addc_u32 s9, s9, external_void_func_i16_zeroext@rel32@hi+12
+; HSA-NEXT:    s_mov_b64 s[6:7], s[4:5]
+; HSA-NEXT:    s_mov_b32 s32, 0
 ; HSA-NEXT:    s_swappc_b64 s[30:31], s[8:9]
 ; HSA-NEXT:    s_endpgm
   %var = load volatile i16, ptr addrspace(1) undef
@@ -978,12 +971,12 @@ define amdgpu_kernel void @test_call_external_void_func_i32_imm(i32) #0 {
 ; VI-NEXT:    s_addc_u32 s37, s37, 0
 ; VI-NEXT:    s_mov_b64 s[6:7], s[0:1]
 ; VI-NEXT:    s_mov_b64 s[0:1], s[36:37]
-; VI-NEXT:    s_mov_b64 s[2:3], s[38:39]
-; VI-NEXT:    v_mov_b32_e32 v0, 42
-; VI-NEXT:    s_mov_b32 s32, 0
 ; VI-NEXT:    s_getpc_b64 s[4:5]
 ; VI-NEXT:    s_add_u32 s4, s4, external_void_func_i32@rel32@lo+4
 ; VI-NEXT:    s_addc_u32 s5, s5, external_void_func_i32@rel32@hi+12
+; VI-NEXT:    s_mov_b64 s[2:3], s[38:39]
+; VI-NEXT:    v_mov_b32_e32 v0, 42
+; VI-NEXT:    s_mov_b32 s32, 0
 ; VI-NEXT:    s_swappc_b64 s[30:31], s[4:5]
 ; VI-NEXT:    s_endpgm
 ;
@@ -997,12 +990,12 @@ define amdgpu_kernel void @test_call_external_void_func_i32_imm(i32) #0 {
 ; CI-NEXT:    s_addc_u32 s37, s37, 0
 ; CI-NEXT:    s_mov_b64 s[6:7], s[0:1]
 ; CI-NEXT:    s_mov_b64 s[0:1], s[36:37]
-; CI-NEXT:    s_mov_b64 s[2:3], s[38:39]
-; CI-NEXT:    v_mov_b32_e32 v0, 42
-; CI-NEXT:    s_mov_b32 s32, 0
 ; CI-NEXT:    s_getpc_b64 s[4:5]
 ; CI-NEXT:    s_add_u32 s4, s4, external_void_func_i32@rel32@lo+4
 ; CI-NEXT:    s_addc_u32 s5, s5, external_void_func_i32@rel32@hi+12
+; CI-NEXT:    s_mov_b64 s[2:3], s[38:39]
+; CI-NEXT:    v_mov_b32_e32 v0, 42
+; CI-NEXT:    s_mov_b32 s32, 0
 ; CI-NEXT:    s_swappc_b64 s[30:31], s[4:5]
 ; CI-NEXT:    s_endpgm
 ;
@@ -1016,24 +1009,23 @@ define amdgpu_kernel void @test_call_external_void_func_i32_imm(i32) #0 {
 ; GFX9-NEXT:    s_addc_u32 s37, s37, 0
 ; GFX9-NEXT:    s_mov_b64 s[6:7], s[0:1]
 ; GFX9-NEXT:    s_mov_b64 s[0:1], s[36:37]
-; GFX9-NEXT:    s_mov_b64 s[2:3], s[38:39]
-; GFX9-NEXT:    v_mov_b32_e32 v0, 42
-; GFX9-NEXT:    s_mov_b32 s32, 0
 ; GFX9-NEXT:    s_getpc_b64 s[4:5]
 ; GFX9-NEXT:    s_add_u32 s4, s4, external_void_func_i32@rel32@lo+4
 ; GFX9-NEXT:    s_addc_u32 s5, s5, external_void_func_i32@rel32@hi+12
+; GFX9-NEXT:    s_mov_b64 s[2:3], s[38:39]
+; GFX9-NEXT:    v_mov_b32_e32 v0, 42
+; GFX9-NEXT:    s_mov_b32 s32, 0
 ; GFX9-NEXT:    s_swappc_b64 s[30:31], s[4:5]
 ; GFX9-NEXT:    s_endpgm
 ;
 ; GFX11-LABEL: test_call_external_void_func_i32_imm:
 ; GFX11:       ; %bb.0:
 ; GFX11-NEXT:    v_mov_b32_e32 v0, 42
-; GFX11-NEXT:    s_mov_b64 s[6:7], s[0:1]
-; GFX11-NEXT:    s_mov_b32 s32, 0
 ; GFX11-NEXT:    s_getpc_b64 s[2:3]
 ; GFX11-NEXT:    s_add_u32 s2, s2, external_void_func_i32@rel32@lo+4
 ; GFX11-NEXT:    s_addc_u32 s3, s3, external_void_func_i32@rel32@hi+12
-; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_mov_b64 s[6:7], s[0:1]
+; GFX11-NEXT:    s_mov_b32 s32, 0
 ; GFX11-NEXT:    s_swappc_b64 s[30:31], s[2:3]
 ; GFX11-NEXT:    s_endpgm
 ;
@@ -1043,13 +1035,13 @@ define amdgpu_kernel void @test_call_external_void_func_i32_imm(i32) #0 {
 ; HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s8, 8
 ; HSA-NEXT:    s_add_u32 s0, s0, s11
 ; HSA-NEXT:    s_addc_u32 s1, s1, 0
-; HSA-NEXT:    s_mov_b64 s[6:7], s[4:5]
-; HSA-NEXT:    v_mov_b32_e32 v0, 42
-; HSA-NEXT:    s_mov_b32 s32, 0
 ; HSA-NEXT:    s_mov_b32 flat_scratch_lo, s9
 ; HSA-NEXT:    s_getpc_b64 s[8:9]
 ; HSA-NEXT:    s_add_u32 s8, s8, external_void_func_i32@rel32@lo+4
 ; HSA-NEXT:    s_addc_u32 s9, s9, external_void_func_i32@rel32@hi+12
+; HSA-NEXT:    s_mov_b64 s[6:7], s[4:5]
+; HSA-NEXT:    v_mov_b32_e32 v0, 42
+; HSA-NEXT:    s_mov_b32 s32, 0
 ; HSA-NEXT:    s_swappc_b64 s[30:31], s[8:9]
 ; HSA-NEXT:    s_endpgm
   call void @external_void_func_i32(i32 42)
@@ -1067,13 +1059,13 @@ define amdgpu_kernel void @test_call_external_void_func_i64_imm() #0 {
 ; VI-NEXT:    s_addc_u32 s37, s37, 0
 ; VI-NEXT:    s_mov_b64 s[6:7], s[0:1]
 ; VI-NEXT:    s_mov_b64 s[0:1], s[36:37]
+; VI-NEXT:    s_getpc_b64 s[4:5]
+; VI-NEXT:    s_add_u32 s4, s4, external_void_func_i64@rel32@lo+4
+; VI-NEXT:    s_addc_u32 s5, s5, external_void_func_i64@rel32@hi+12
 ; VI-NEXT:    s_mov_b64 s[2:3], s[38:39]
 ; VI-NEXT:    v_mov_b32_e32 v0, 0x7b
 ; VI-NEXT:    v_mov_b32_e32 v1, 0
 ; VI-NEXT:    s_mov_b32 s32, 0
-; VI-NEXT:    s_getpc_b64 s[4:5]
-; VI-NEXT:    s_add_u32 s4, s4, external_void_func_i64@rel32@lo+4
-; VI-NEXT:    s_addc_u32 s5, s5, external_void_func_i64@rel32@hi+12
 ; VI-NEXT:    s_swappc_b64 s[30:31], s[4:5]
 ; VI-NEXT:    s_endpgm
 ;
@@ -1087,13 +1079,13 @@ define amdgpu_kernel void @test_call_external_void_func_i64_imm() #0 {
 ; CI-NEXT:    s_addc_u32 s37, s37, 0
 ; CI-NEXT:    s_mov_b64 s[6:7], s[0:1]
 ; CI-NEXT:    s_mov_b64 s[0:1], s[36:37]
+; CI-NEXT:    s_getpc_b64 s[4:5]
+; CI-NEXT:    s_add_u32 s4, s4, external_void_func_i64@rel32@lo+4
+; CI-NEXT:    s_addc_u32 s5, s5, external_void_func_i64@rel32@hi+12
 ; CI-NEXT:    s_mov_b64 s[2:3], s[38:39]
 ; CI-NEXT:    v_mov_b32_e32 v0, 0x7b
 ; CI-NEXT:    v_mov_b32_e32 v1, 0
 ; CI-NEXT:    s_mov_b32 s32, 0
-; CI-NEXT:    s_getpc_b64 s[4:5]
-; CI-NEXT:    s_add_u32 s4, s4, external_void_func_i64@rel32@lo+4
-; CI-NEXT:    s_addc_u32 s5, s5, external_void_func_i64@rel32@hi+12
 ; CI-NEXT:    s_swappc_b64 s[30:31], s[4:5]
 ; CI-NEXT:    s_endpgm
 ;
@@ -1107,25 +1099,24 @@ define amdgpu_kernel void @test_call_external_void_func_i64_imm() #0 {
 ; GFX9-NEXT:    s_addc_u32 s37, s37, 0
 ; GFX9-NEXT:    s_mov_b64 s[6:7], s[0:1]
 ; GFX9-NEXT:    s_mov_b64 s[0:1], s[36:37]
+; GFX9-NEXT:    s_getpc_b64 s[4:5]
+; GFX9-NEXT:    s_add_u32 s4, s4, external_void_func_i64@rel32@lo+4
+; GFX9-NEXT:    s_addc_u32 s5, s5, external_void_func_i64@rel32@hi+12
 ; GFX9-NEXT:    s_mov_b64 s[2:3], s[38:39]
 ; GFX9-NEXT:    v_mov_b32_e32 v0, 0x7b
 ; GFX9-NEXT:    v_mov_b32_e32 v1, 0
 ; GFX9-NEXT:    s_mov_b32 s32, 0
-; GFX9-NEXT:    s_getpc_b64 s[4:5]
-; GFX9-NEXT:    s_add_u32 s4, s4, external_void_func_i64@rel32@lo+4
-; GFX9-NEXT:    s_addc_u32 s5, s5, external_void_func_i64@rel32@hi+12
 ; GFX9-NEXT:    s_swappc_b64 s[30:31], s[4:5]
 ; GFX9-NEXT:    s_endpgm
 ;
 ; GFX11-LABEL: test_call_external_void_func_i64_imm:
 ; GFX11:       ; %bb.0:
 ; GFX11-NEXT:    v_dual_mov_b32 v0, 0x7b :: v_dual_mov_b32 v1, 0
-; GFX11-NEXT:    s_mov_b64 s[6:7], s[0:1]
-; GFX11-NEXT:    s_mov_b32 s32, 0
 ; GFX11-NEXT:    s_getpc_b64 s[2:3]
 ; GFX11-NEXT:    s_add_u32 s2, s2, external_void_func_i64@rel32@lo+4
 ; GFX11-NEXT:    s_addc_u32 s3, s3, external_void_func_i64@rel32@hi+12
-; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_mov_b64 s[6:7], s[0:1]
+; GFX11-NEXT:    s_mov_b32 s32, 0
 ; GFX11-NEXT:    s_swappc_b64 s[30:31], s[2:3]
 ; GFX11-NEXT:    s_endpgm
 ;
@@ -1134,15 +1125,15 @@ define amdgpu_kernel void @test_call_external_void_func_i64_imm() #0 {
 ; HSA-NEXT:    s_add_i32 s6, s6, s9
 ; HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s6, 8
 ; HSA-NEXT:    s_add_u32 s0, s0, s9
-; HSA-NEXT:    s_mov_b32 flat_scratch_lo, s7
 ; HSA-NEXT:    s_addc_u32 s1, s1, 0
+; HSA-NEXT:    s_mov_b32 flat_scratch_lo, s7
+; HSA-NEXT:    s_getpc_b64 s[8:9]
+; HSA-NEXT:    s_add_u32 s8, s8, external_void_func_i64@rel32@lo+4
+; HSA-NEXT:    s_addc_u32 s9, s9, external_void_func_i64@rel32@hi+12
 ; HSA-NEXT:    s_mov_b64 s[6:7], s[4:5]
 ; HSA-NEXT:    v_mov_b32_e32 v0, 0x7b
 ; HSA-NEXT:    v_mov_b32_e32 v1, 0
 ; HSA-NEXT:    s_mov_b32 s32, 0
-; HSA-NEXT:    s_getpc_b64 s[8:9]
-; HSA-NEXT:    s_add_u32 s8, s8, external_void_func_i64@rel32@lo+4
-; HSA-NEXT:    s_addc_u32 s9, s9, external_void_func_i64@rel32@hi+12
 ; HSA-NEXT:    s_swappc_b64 s[30:31], s[8:9]
 ; HSA-NEXT:    s_endpgm
   call void @external_void_func_i64(i64 123)
@@ -1165,11 +1156,11 @@ define amdgpu_kernel void @test_call_external_void_func_v2i64() #0 {
 ; VI-NEXT:    buffer_load_dwordx4 v[0:3], off, s[0:3], 0
 ; VI-NEXT:    s_addc_u32 s37, s37, 0
 ; VI-NEXT:    s_mov_b64 s[0:1], s[36:37]
-; VI-NEXT:    s_mov_b64 s[2:3], s[38:39]
-; VI-NEXT:    s_mov_b32 s32, 0
 ; VI-NEXT:    s_getpc_b64 s[4:5]
 ; VI-NEXT:    s_add_u32 s4, s4, external_void_func_v2i64@rel32@lo+4
 ; VI-NEXT:    s_addc_u32 s5, s5, external_void_func_v2i64@rel32@hi+12
+; VI-NEXT:    s_mov_b64 s[2:3], s[38:39]
+; VI-NEXT:    s_mov_b32 s32, 0
 ; VI-NEXT:    s_swappc_b64 s[30:31], s[4:5]
 ; VI-NEXT:    s_endpgm
 ;
@@ -1188,11 +1179,11 @@ define amdgpu_kernel void @test_call_external_void_func_v2i64() #0 {
 ; CI-NEXT:    buffer_load_dwordx4 v[0:3], off, s[0:3], 0
 ; CI-NEXT:    s_addc_u32 s37, s37, 0
 ; CI-NEXT:    s_mov_b64 s[0:1], s[36:37]
-; CI-NEXT:    s_mov_b64 s[2:3], s[38:39]
-; CI-NEXT:    s_mov_b32 s32, 0
 ; CI-NEXT:    s_getpc_b64 s[4:5]
 ; CI-NEXT:    s_add_u32 s4, s4, external_void_func_v2i64@rel32@lo+4
 ; CI-NEXT:    s_addc_u32 s5, s5, external_void_func_v2i64@rel32@hi+12
+; CI-NEXT:    s_mov_b64 s[2:3], s[38:39]
+; CI-NEXT:    s_mov_b32 s32, 0
 ; CI-NEXT:    s_swappc_b64 s[30:31], s[4:5]
 ; CI-NEXT:    s_endpgm
 ;
@@ -1211,11 +1202,11 @@ define amdgpu_kernel void @test_call_external_void_func_v2i64() #0 {
 ; GFX9-NEXT:    buffer_load_dwordx4 v[0:3], off, s[0:3], 0
 ; GFX9-NEXT:    s_addc_u32 s37, s37, 0
 ; GFX9-NEXT:    s_mov_b64 s[0:1], s[36:37]
-; GFX9-NEXT:    s_mov_b64 s[2:3], s[38:39]
-; GFX9-NEXT:    s_mov_b32 s32, 0
 ; GFX9-NEXT:    s_getpc_b64 s[4:5]
 ; GFX9-NEXT:    s_add_u32 s4, s4, external_void_func_v2i64@rel32@lo+4
 ; GFX9-NEXT:    s_addc_u32 s5, s5, external_void_func_v2i64@rel32@hi+12
+; GFX9-NEXT:    s_mov_b64 s[2:3], s[38:39]
+; GFX9-NEXT:    s_mov_b32 s32, 0
 ; GFX9-NEXT:    s_swappc_b64 s[30:31], s[4:5]
 ; GFX9-NEXT:    s_endpgm
 ;
@@ -1225,13 +1216,12 @@ define amdgpu_kernel void @test_call_external_void_func_v2i64() #0 {
 ; GFX11-NEXT:    s_mov_b32 s7, 0x31016000
 ; GFX11-NEXT:    s_mov_b32 s6, -1
 ; GFX11-NEXT:    s_mov_b32 s5, s4
-; GFX11-NEXT:    s_mov_b32 s32, 0
-; GFX11-NEXT:    buffer_load_b128 v[0:3], off, s[4:7], 0
-; GFX11-NEXT:    s_mov_b64 s[6:7], s[0:1]
 ; GFX11-NEXT:    s_getpc_b64 s[2:3]
 ; GFX11-NEXT:    s_add_u32 s2, s2, external_void_func_v2i64@rel32@lo+4
 ; GFX11-NEXT:    s_addc_u32 s3, s3, external_void_func_v2i64@rel32@hi+12
-; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    buffer_load_b128 v[0:3], off, s[4:7], 0
+; GFX11-NEXT:    s_mov_b64 s[6:7], s[0:1]
+; GFX11-NEXT:    s_mov_b32 s32, 0
 ; GFX11-NEXT:    s_swappc_b64 s[30:31], s[2:3]
 ; GFX11-NEXT:    s_endpgm
 ;
@@ -1245,13 +1235,13 @@ define amdgpu_kernel void @test_call_external_void_func_v2i64() #0 {
 ; HSA-NEXT:    s_mov_b32 s10, -1
 ; HSA-NEXT:    s_mov_b32 s9, s8
 ; HSA-NEXT:    buffer_load_dwordx4 v[0:3], off, s[8:11], 0
-; HSA-NEXT:    s_mov_b32 flat_scratch_lo, s7
 ; HSA-NEXT:    s_addc_u32 s1, s1, 0
-; HSA-NEXT:    s_mov_b64 s[6:7], s[4:5]
-; HSA-NEXT:    s_mov_b32 s32, 0
+; HSA-NEXT:    s_mov_b32 flat_scratch_lo, s7
 ; HSA-NEXT:    s_getpc_b64 s[8:9]
 ; HSA-NEXT:    s_add_u32 s8, s8, external_void_func_v2i64@rel32@lo+4
 ; HSA-NEXT:    s_addc_u32 s9, s9, external_void_func_v2i64@rel32@hi+12
+; HSA-NEXT:    s_mov_b64 s[6:7], s[4:5]
+; HSA-NEXT:    s_mov_b32 s32, 0
 ; HSA-NEXT:    s_swappc_b64 s[30:31], s[8:9]
 ; HSA-NEXT:    s_endpgm
   %val = load <2 x i64>, ptr addrspace(1) null
@@ -1270,15 +1260,15 @@ define amdgpu_kernel void @test_call_external_void_func_v2i64_imm() #0 {
 ; VI-NEXT:    s_addc_u32 s37, s37, 0
 ; VI-NEXT:    s_mov_b64 s[6:7], s[0:1]
 ; VI-NEXT:    s_mov_b64 s[0:1], s[36:37]
+; VI-NEXT:    s_getpc_b64 s[4:5]
+; VI-NEXT:    s_add_u32 s4, s4, external_void_func_v2i64@rel32@lo+4
+; VI-NEXT:    s_addc_u32 s5, s5, external_void_func_v2i64@rel32@hi+12
 ; VI-NEXT:    s_mov_b64 s[2:3], s[38:39]
 ; VI-NEXT:    v_mov_b32_e32 v0, 1
 ; VI-NEXT:    v_mov_b32_e32 v1, 2
 ; VI-NEXT:    v_mov_b32_e32 v2, 3
 ; VI-NEXT:    v_mov_b32_e32 v3, 4
 ; VI-NEXT:    s_mov_b32 s32, 0
-; VI-NEXT:    s_getpc_b64 s[4:5]
-; VI-NEXT:    s_add_u32 s4, s4, external_void_func_v2i64@rel32@lo+4
-; VI-NEXT:    s_addc_u32 s5, s5, external_void_func_v2i64@rel32@hi+12
 ; VI-NEXT:    s_swappc_b64 s[30:31], s[4:5]
 ; VI-NEXT:    s_endpgm
 ;
@@ -1292,15 +1282,15 @@ define amdgpu_kernel void @test_call_external_void_func_v2i64_imm() #0 {
 ; CI-NEXT:    s_addc_u32 s37, s37, 0
 ; CI-NEXT:    s_mov_b64 s[6:7], s[0:1]
 ; CI-NEXT:    s_mov_b64 s[0:1], s[36:37]
+; CI-NEXT:    s_getpc_b64 s[4:5]
+; CI-NEXT:    s_add_u32 s4, s4, external_void_func_v2i64@rel32@lo+4
+; CI-NEXT:    s_addc_u32 s5, s5, external_void_func_v2i64@rel32@hi+12
 ; CI-NEXT:    s_mov_b64 s[2:3], s[38:39]
 ; CI-NEXT:    v_mov_b32_e32 v0, 1
 ; CI-NEXT:    v_mov_b32_e32 v1, 2
 ; CI-NEXT:    v_mov_b32_e32 v2, 3
 ; CI-NEXT:    v_mov_b32_e32 v3, 4
 ; CI-NEXT:    s_mov_b32 s32, 0
-; CI-NEXT:    s_getpc_b64 s[4:5]
-; CI-NEXT:    s_add_u32 s4, s4, external_void_func_v2i64@rel32@lo+4
-; CI-NEXT:    s_addc_u32 s5, s5, external_void_func_v2i64@rel32@hi+12
 ; CI-NEXT:    s_swappc_b64 s[30:31], s[4:5]
 ; CI-NEXT:    s_endpgm
 ;
@@ -1314,15 +1304,15 @@ define amdgpu_kernel void @test_call_external_void_func_v2i64_imm() #0 {
 ; GFX9-NEXT:    s_addc_u32 s37, s37, 0
 ; GFX9-NEXT:    s_mov_b64 s[6:7], s[0:1]
 ; GFX9-NEXT:    s_mov_b64 s[0:1], s[36:37]
+; GFX9-NEXT:    s_getpc_b64 s[4:5]
+; GFX9-NEXT:    s_add_u32 s4, s4, external_void_func_v2i64@rel32@lo+4
+; GFX9-NEXT:    s_addc_u32 s5, s5, external_void_func_v2i64@rel32@hi+12
 ; GFX9-NEXT:    s_mov_b64 s[2:3], s[38:39]
 ; GFX9-NEXT:    v_mov_b32_e32 v0, 1
 ; GFX9-NEXT:    v_mov_b32_e32 v1, 2
 ; GFX9-NEXT:    v_mov_b32_e32 v2, 3
 ; GFX9-NEXT:    v_mov_b32_e32 v3, 4
 ; GFX9-NEXT:    s_mov_b32 s32, 0
-; GFX9-NEXT:    s_getpc_b64 s[4:5]
-; GFX9-NEXT:    s_add_u32 s4, s4, external_void_func_v2i64@rel32@lo+4
-; GFX9-NEXT:    s_addc_u32 s5, s5, external_void_func_v2i64@rel32@hi+12
 ; GFX9-NEXT:    s_swappc_b64 s[30:31], s[4:5]
 ; GFX9-NEXT:    s_endpgm
 ;
@@ -1330,12 +1320,11 @@ define amdgpu_kernel void @test_call_external_void_func_v2i64_imm() #0 {
 ; GFX11:       ; %bb.0:
 ; GFX11-NEXT:    v_dual_mov_b32 v0, 1 :: v_dual_mov_b32 v1, 2
 ; GFX11-NEXT:    v_dual_mov_b32 v2, 3 :: v_dual_mov_b32 v3, 4
-; GFX11-NEXT:    s_mov_b64 s[6:7], s[0:1]
-; GFX11-NEXT:    s_mov_b32 s32, 0
 ; GFX11-NEXT:    s_getpc_b64 s[2:3]
 ; GFX11-NEXT:    s_add_u32 s2, s2, external_void_func_v2i64@rel32@lo+4
 ; GFX11-NEXT:    s_addc_u32 s3, s3, external_void_func_v2i64@rel32@hi+12
-; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_mov_b64 s[6:7], s[0:1]
+; GFX11-NEXT:    s_mov_b32 s32, 0
 ; GFX11-NEXT:    s_swappc_b64 s[30:31], s[2:3]
 ; GFX11-NEXT:    s_endpgm
 ;
@@ -1344,17 +1333,17 @@ define amdgpu_kernel void @test_call_external_void_func_v2i64_imm() #0 {
 ; HSA-NEXT:    s_add_i32 s6, s6, s9
 ; HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s6, 8
 ; HSA-NEXT:    s_add_u32 s0, s0, s9
-; HSA-NEXT:    s_mov_b32 flat_scratch_lo, s7
 ; HSA-NEXT:    s_addc_u32 s1, s1, 0
+; HSA-NEXT:    s_mov_b32 flat_scratch_lo, s7
+; HSA-NEXT:    s_getpc_b64 s[8:9]
+; HSA-NEXT:    s_add_u32 s8, s8, external_void_func_v2i64@rel32@lo+4
+; HSA-NEXT:    s_addc_u32 s9, s9, external_void_func_v2i64@rel32@hi+12
 ; HSA-NEXT:    s_mov_b64 s[6:7], s[4:5]
 ; HSA-NEXT:    v_mov_b32_e32 v0, 1
 ; HSA-NEXT:    v_mov_b32_e32 v1, 2
 ; HSA-NEXT:    v_mov_b32_e32 v2, 3
 ; HSA-NEXT:    v_mov_b32_e32 v3, 4
 ; HSA-NEXT:    s_mov_b32 s32, 0
-; HSA-NEXT:    s_getpc_b64 s[8:9]
-; HSA-NEXT:    s_add_u32 s8, s8, external_void_func_v2i64@rel32@lo+4
-; HSA-NEXT:    s_addc_u32 s9, s9, external_void_func_v2i64@rel32@hi+12
 ; HSA-NEXT:    s_swappc_b64 s[30:31], s[8:9]
 ; HSA-NEXT:    s_endpgm
   call void @external_void_func_v2i64(<2 x i64> <i64 8589934593, i64 17179869187>)
@@ -1377,13 +1366,13 @@ define amdgpu_kernel void @test_call_external_void_func_v3i64() #0 {
 ; VI-NEXT:    buffer_load_dwordx4 v[0:3], off, s[0:3], 0
 ; VI-NEXT:    s_addc_u32 s37, s37, 0
 ; VI-NEXT:    s_mov_b64 s[0:1], s[36:37]
+; VI-NEXT:    s_getpc_b64 s[4:5]
+; VI-NEXT:    s_add_u32 s4, s4, external_void_func_v3i64@rel32@lo+4
+; VI-NEXT:    s_addc_u32 s5, s5, external_void_func_v3i64@rel32@hi+12
 ; VI-NEXT:    s_mov_b64 s[2:3], s[38:39]
 ; VI-NEXT:    v_mov_b32_e32 v4, 1
 ; VI-NEXT:    v_mov_b32_e32 v5, 2
 ; VI-NEXT:    s_mov_b32 s32, 0
-; VI-NEXT:    s_getpc_b64 s[4:5]
-; VI-NEXT:    s_add_u32 s4, s4, external_void_func_v3i64@rel32@lo+4
-; VI-NEXT:    s_addc_u32 s5, s5, external_void_func_v3i64@rel32@hi+12
 ; VI-NEXT:    s_swappc_b64 s[30:31], s[4:5]
 ; VI-NEXT:    s_endpgm
 ;
@@ -1402,13 +1391,13 @@ define amdgpu_kernel void @test_call_external_void_func_v3i64() #0 {
 ; CI-NEXT:    buffer_load_dwordx4 v[0:3], off, s[0:3], 0
 ; CI-NEXT:    s_addc_u32 s37, s37, 0
 ; CI-NEXT:    s_mov_b64 s[0:1], s[36:37]
+; CI-NEXT:    s_getpc_b64 s[4:5]
+; CI-NEXT:    s_add_u32 s4, s4, external_void_func_v3i64@rel32@lo+4
+; CI-NEXT:    s_addc_u32 s5, s5, external_void_func_v3i64@rel32@hi+12
 ; CI-NEXT:    s_mov_b64 s[2:3], s[38:39]
 ; CI-NEXT:    v_mov_b32_e32 v4, 1
 ; CI-NEXT:    v_mov_b32_e32 v5, 2
 ; CI-NEXT:    s_mov_b32 s32, 0
-; CI-NEXT:    s_getpc_b64 s[4:5]
-; CI-NEXT:    s_add_u32 s4, s4, external_void_func_v3i64@rel32@lo+4
-; CI-NEXT:    s_addc_u32 s5, s5, external_void_func_v3i64@rel32@hi+12
 ; CI-NEXT:    s_swappc_b64 s[30:31], s[4:5]
 ; CI-NEXT:    s_endpgm
 ;
@@ -1427,13 +1416,13 @@ define amdgpu_kernel void @test_call_external_void_func_v3i64() #0 {
 ; GFX9-NEXT:    buffer_load_dwordx4 v[0:3], off, s[0:3], 0
 ; GFX9-NEXT:    s_addc_u32 s37, s37, 0
 ; GFX9-NEXT:    s_mov_b64 s[0:1], s[36:37]
+; GFX9-NEXT:    s_getpc_b64 s[4:5]
+; GFX9-NEXT:    s_add_u32 s4, s4, external_void_func_v3i64@rel32@lo+4
+; GFX9-NEXT:    s_addc_u32 s5, s5, external_void_func_v3i64@rel32@hi+12
 ; GFX9-NEXT:    s_mov_b64 s[2:3], s[38:39]
 ; GFX9-NEXT:    v_mov_b32_e32 v4, 1
 ; GFX9-NEXT:    v_mov_b32_e32 v5, 2
 ; GFX9-NEXT:    s_mov_b32 s32, 0
-; GFX9-NEXT:    s_getpc_b64 s[4:5]
-; GFX9-NEXT:    s_add_u32 s4, s4, external_void_func_v3i64@rel32@lo+4
-; GFX9-NEXT:    s_addc_u32 s5, s5, external_void_func_v3i64@rel32@hi+12
 ; GFX9-NEXT:    s_swappc_b64 s[30:31], s[4:5]
 ; GFX9-NEXT:    s_endpgm
 ;
@@ -1445,12 +1434,11 @@ define amdgpu_kernel void @test_call_external_void_func_v3i64() #0 {
 ; GFX11-NEXT:    s_mov_b32 s5, s4
 ; GFX11-NEXT:    v_dual_mov_b32 v4, 1 :: v_dual_mov_b32 v5, 2
 ; GFX11-NEXT:    buffer_load_b128 v[0:3], off, s[4:7], 0
-; GFX11-NEXT:    s_mov_b64 s[6:7], s[0:1]
-; GFX11-NEXT:    s_mov_b32 s32, 0
 ; GFX11-NEXT:    s_getpc_b64 s[2:3]
 ; GFX11-NEXT:    s_add_u32 s2, s2, external_void_func_v3i64@rel32@lo+4
 ; GFX11-NEXT:    s_addc_u32 s3, s3, external_void_func_v3i64@rel32@hi+12
-; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_mov_b64 s[6:7], s[0:1]
+; GFX11-NEXT:    s_mov_b32 s32, 0
 ; GFX11-NEXT:    s_swappc_b64 s[30:31], s[2:3]
 ; GFX11-NEXT:    s_endpgm
 ;
@@ -1464,15 +1452,15 @@ define amdgpu_kernel void @test_call_external_void_func_v3i64() #0 {
 ; HSA-NEXT:    s_mov_b32 s10, -1
 ; HSA-NEXT:    s_mov_b32 s9, s8
 ; HSA-NEXT:    buffer_load_dwordx4 v[0:3], off, s[8:11], 0
-; HSA-NEXT:    s_mov_b32 flat_scratch_lo, s7
 ; HSA-NEXT:    s_addc_u32 s1, s1, 0
+; HSA-NEXT:    s_mov_b32 flat_scratch_lo, s7
+; HSA-NEXT:    s_getpc_b64 s[8:9]
+; HSA-NEXT:    s_add_u32 s8, s8, external_void_func_v3i64@rel32@lo+4
+; HSA-NEXT:    s_addc_u32 s9, s9, external_void_func_v3i64@rel32@hi+12
 ; HSA-NEXT:    v_mov_b32_e32 v4, 1
 ; HSA-NEXT:    s_mov_b64 s[6:7], s[4:5]
 ; HSA-NEXT:    v_mov_b32_e32 v5, 2
 ; HSA-NEXT:    s_mov_b32 s32, 0
-; HSA-NEXT:    s_getpc_b64 s[8:9]
-; HSA-NEXT:    s_add_u32 s8, s8, external_void_func_v3i64@rel32@lo+4
-; HSA-NEXT:    s_addc_u32 s9, s9, external_void_func_v3i64@rel32@hi+12
 ; HSA-NEXT:    s_swappc_b64 s[30:31], s[8:9]
 ; HSA-NEXT:    s_endpgm
   %load = load <2 x i64>, ptr addrspace(1) null
@@ -1498,15 +1486,15 @@ define amdgpu_kernel void @test_call_external_void_func_v4i64() #0 {
 ; VI-NEXT:    buffer_load_dwordx4 v[0:3], off, s[0:3], 0
 ; VI-NEXT:    s_addc_u32 s37, s37, 0
 ; VI-NEXT:    s_mov_b64 s[0:1], s[36:37]
+; VI-NEXT:    s_getpc_b64 s[4:5]
+; VI-NEXT:    s_add_u32 s4, s4, external_void_func_v4i64@rel32@lo+4
+; VI-NEXT:    s_addc_u32 s5, s5, external_void_func_v4i64@rel32@hi+12
 ; VI-NEXT:    s_mov_b64 s[2:3], s[38:39]
 ; VI-NEXT:    v_mov_b32_e32 v4, 1
 ; VI-NEXT:    v_mov_b32_e32 v5, 2
 ; VI-NEXT:    v_mov_b32_e32 v6, 3
 ; VI-NEXT:    v_mov_b32_e32 v7, 4
 ; VI-NEXT:    s_mov_b32 s32, 0
-; VI-NEXT:    s_getpc_b64 s[4:5]
-; VI-NEXT:    s_add_u32 s4, s4, external_void_func_v4i64@rel32@lo+4
-; VI-NEXT:    s_addc_u32 s5, s5, external_void_func_v4i64@rel32@hi+12
 ; VI-NEXT:    s_swappc_b64 s[30:31], s[4:5]
 ; VI-NEXT:    s_endpgm
 ;
@@ -1525,15 +1513,15 @@ define amdgpu_kernel void @test_call_external_void_func_v4i64() #0 {
 ; CI-NEXT:    buffer_load_dwordx4 v[0:3], off, s[0:3], 0
 ; CI-NEXT:    s_addc_u32 s37, s37, 0
 ; CI-NEXT:    s_mov_b64 s[0:1], s[36:37]
+; CI-NEXT:    s_getpc_b64 s[4:5]
+; CI-NEXT:    s_add_u32 s4, s4, external_void_func_v4i64@rel32@lo+4
+; CI-NEXT:    s_addc_u32 s5, s5, external_void_func_v4i64@rel32@hi+12
 ; CI-NEXT:    s_mov_b64 s[2:3], s[38:39]
 ; CI-NEXT:    v_mov_b32_e32 v4, 1
 ; CI-NEXT:    v_mov_b32_e32 v5, 2
 ; CI-NEXT:    v_mov_b32_e32 v6, 3
 ; CI-NEXT:    v_mov_b32_e32 v7, 4
 ; CI-NEXT:    s_mov_b32 s32, 0
-; CI-NEXT:    s_getpc_b64 s[4:5]
-; CI-NEXT:    s_add_u32 s4, s4, external_void_func_v4i64@rel32@lo+4
-; CI-NEXT:    s_addc_u32 s5, s5, external_void_func_v4i64@rel32@hi+12
 ; CI-NEXT:    s_swappc_b64 s[30:31], s[4:5]
 ; CI-NEXT:    s_endpgm
 ;
@@ -1552,15 +1540,15 @@ define amdgpu_kernel void @test_call_external_void_func_v4i64() #0 {
 ; GFX9-NEXT:    buffer_load_dwordx4 v[0:3], off, s[0:3], 0
 ; GFX9-NEXT:    s_addc_u32 s37, s37, 0
 ; GFX9-NEXT:    s_mov_b64 s[0:1], s[36:37]
+; GFX9-NEXT:    s_getpc_b64 s[4:5]
+; GFX9-NEXT:    s_add_u32 s4, s4, external_void_func_v4i64@rel32@lo+4
+; GFX9-NEXT:    s_addc_u32 s5, s5, external_void_func_v4i64@rel32@hi+12
 ; GFX9-NEXT:    s_mov_b64 s[2:3], s[38:39]
 ; GFX9-NEXT:    v_mov_b32_e32 v4, 1
 ; GFX9-NEXT:    v_mov_b32_e32 v5, 2
 ; GFX9-NEXT:    v_mov_b32_e32 v6, 3
 ; GFX9-NEXT:    v_mov_b32_e32 v7, 4
 ; GFX9-NEXT:    s_mov_b32 s32, 0
-; GFX9-NEXT:    s_getpc_b64 s[4:5]
-; GFX9-NEXT:    s_add_u32 s4, s4, external_void_func_v4i64@rel32@lo+4
-; GFX9-NEXT:    s_addc_u32 s5, s5, external_void_func_v4i64@rel32@hi+12
 ; GFX9-NEXT:    s_swappc_b64 s[30:31], s[4:5]
 ; GFX9-NEXT:    s_endpgm
 ;
@@ -1573,12 +1561,11 @@ define amdgpu_kernel void @test_call_external_void_func_v4i64() #0 {
 ; GFX11-NEXT:    v_dual_mov_b32 v4, 1 :: v_dual_mov_b32 v5, 2
 ; GFX11-NEXT:    buffer_load_b128 v[0:3], off, s[4:7], 0
 ; GFX11-NEXT:    v_dual_mov_b32 v6, 3 :: v_dual_mov_b32 v7, 4
-; GFX11-NEXT:    s_mov_b64 s[6:7], s[0:1]
-; GFX11-NEXT:    s_mov_b32 s32, 0
 ; GFX11-NEXT:    s_getpc_b64 s[2:3]
 ; GFX11-NEXT:    s_add_u32 s2, s2, external_void_func_v4i64@rel32@lo+4
 ; GFX11-NEXT:    s_addc_u32 s3, s3, external_void_func_v4i64@rel32@hi+12
-; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_mov_b64 s[6:7], s[0:1]
+; GFX11-NEXT:    s_mov_b32 s32, 0
 ; GFX11-NEXT:    s_swappc_b64 s[30:31], s[2:3]
 ; GFX11-NEXT:    s_endpgm
 ;
@@ -1592,17 +1579,17 @@ define amdgpu_kernel void @test_call_external_void_func_v4i64() #0 {
 ; HSA-NEXT:    s_mov_b32 s10, -1
 ; HSA-NEXT:    s_mov_b32 s9, s8
 ; HSA-NEXT:    buffer_load_dwordx4 v[0:3], off, s[8:11], 0
-; HSA-NEXT:    s_mov_b32 flat_scratch_lo, s7
 ; HSA-NEXT:    s_addc_u32 s1, s1, 0
+; HSA-NEXT:    s_mov_b32 flat_scratch_lo, s7
+; HSA-NEXT:    s_getpc_b64 s[8:9]
+; HSA-NEXT:    s_add_u32 s8, s8, external_void_func_v4i64@rel32@lo+4
+; HSA-NEXT:    s_addc_u32 s9, s9, external_void_func_v4i64@rel32@hi+12
 ; HSA-NEXT:    v_mov_b32_e32 v4, 1
 ; HSA-NEXT:    v_mov_b32_e32 v5, 2
 ; HSA-NEXT:    v_mov_b32_e32 v6, 3
 ; HSA-NEXT:    s_mov_b64 s[6:7], s[4:5]
 ; HSA-NEXT:    v_mov_b32_e32 v7, 4
 ; HSA-NEXT:    s_mov_b32 s32, 0
-; HSA-NEXT:    s_getpc_b64 s[8:9]
-; HSA-NEXT:    s_add_u32 s8, s8, external_void_func_v4i64@rel32@lo+4
-; HSA-NEXT:    s_addc_u32 s9, s9, external_void_func_v4i64@rel32@hi+12
 ; HSA-NEXT:    s_swappc_b64 s[30:31], s[8:9]
 ; HSA-NEXT:    s_endpgm
   %load = load <2 x i64>, ptr addrspace(1) null
@@ -1622,12 +1609,12 @@ define amdgpu_kernel void @test_call_external_void_func_f16_imm() #0 {
 ; VI-NEXT:    s_addc_u32 s37, s37, 0
 ; VI-NEXT:    s_mov_b64 s[6:7], s[0:1]
 ; VI-NEXT:    s_mov_b64 s[0:1], s[36:37]
-; VI-NEXT:    s_mov_b64 s[2:3], s[38:39]
-; VI-NEXT:    v_mov_b32_e32 v0, 0x4400
-; VI-NEXT:    s_mov_b32 s32, 0
 ; VI-NEXT:    s_getpc_b64 s[4:5]
 ; VI-NEXT:    s_add_u32 s4, s4, external_void_func_f16@rel32@lo+4
 ; VI-NEXT:    s_addc_u32 s5, s5, external_void_func_f16@rel32@hi+12
+; VI-NEXT:    s_mov_b64 s[2:3], s[38:39]
+; VI-NEXT:    v_mov_b32_e32 v0, 0x4400
+; VI-NEXT:    s_mov_b32 s32, 0
 ; VI-NEXT:    s_swappc_b64 s[30:31], s[4:5]
 ; VI-NEXT:    s_endpgm
 ;
@@ -1641,12 +1628,12 @@ define amdgpu_kernel void @test_call_external_void_func_f16_imm() #0 {
 ; CI-NEXT:    s_addc_u32 s37, s37, 0
 ; CI-NEXT:    s_mov_b64 s[6:7], s[0:1]
 ; CI-NEXT:    s_mov_b64 s[0:1], s[36:37]
-; CI-NEXT:    s_mov_b64 s[2:3], s[38:39]
-; CI-NEXT:    v_mov_b32_e32 v0, 4.0
-; CI-NEXT:    s_mov_b32 s32, 0
 ; CI-NEXT:    s_getpc_b64 s[4:5]
 ; CI-NEXT:    s_add_u32 s4, s4, external_void_func_f16@rel32@lo+4
 ; CI-NEXT:    s_addc_u32 s5, s5, external_void_func_f16@rel32@hi+12
+; CI-NEXT:    s_mov_b64 s[2:3], s[38:39]
+; CI-NEXT:    v_mov_b32_e32 v0, 4.0
+; CI-NEXT:    s_mov_b32 s32, 0
 ; CI-NEXT:    s_swappc_b64 s[30:31], s[4:5]
 ; CI-NEXT:    s_endpgm
 ;
@@ -1660,24 +1647,23 @@ define amdgpu_kernel void @test_call_external_void_func_f16_imm() #0 {
 ; GFX9-NEXT:    s_addc_u32 s37, s37, 0
 ; GFX9-NEXT:    s_mov_b64 s[6:7], s[0:1]
 ; GFX9-NEXT:    s_mov_b64 s[0:1], s[36:37]
-; GFX9-NEXT:    s_mov_b64 s[2:3], s[38:39]
-; GFX9-NEXT:    v_mov_b32_e32 v0, 0x4400
-; GFX9-NEXT:    s_mov_b32 s32, 0
 ; GFX9-NEXT:    s_getpc_b64 s[4:5]
 ; GFX9-NEXT:    s_add_u32 s4, s4, external_void_func_f16@rel32@lo+4
 ; GFX9-NEXT:    s_addc_u32 s5, s5, external_void_func_f16@rel32@hi+12
+; GFX9-NEXT:    s_mov_b64 s[2:3], s[38:39]
+; GFX9-NEXT:    v_mov_b32_e32 v0, 0x4400
+; GFX9-NEXT:    s_mov_b32 s32, 0
 ; GFX9-NEXT:    s_swappc_b64 s[30:31], s[4:5]
 ; GFX9-NEXT:    s_endpgm
 ;
 ; GFX11-LABEL: test_call_external_void_func_f16_imm:
 ; GFX11:       ; %bb.0:
 ; GFX11-NEXT:    v_mov_b32_e32 v0, 0x4400
-; GFX11-NEXT:    s_mov_b64 s[6:7], s[0:1]
-; GFX11-NEXT:    s_mov_b32 s32, 0
 ; GFX11-NEXT:    s_getpc_b64 s[2:3]
 ; GFX11-NEXT:    s_add_u32 s2, s2, external_void_func_f16@rel32@lo+4
 ; GFX11-NEXT:    s_addc_u32 s3, s3, external_void_func_f16@rel32@hi+12
-; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_mov_b64 s[6:7], s[0:1]
+; GFX11-NEXT:    s_mov_b32 s32, 0
 ; GFX11-NEXT:    s_swappc_b64 s[30:31], s[2:3]
 ; GFX11-NEXT:    s_endpgm
 ;
@@ -1686,14 +1672,14 @@ define amdgpu_kernel void @test_call_external_void_func_f16_imm() #0 {
 ; HSA-NEXT:    s_add_i32 s6, s6, s9
 ; HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s6, 8
 ; HSA-NEXT:    s_add_u32 s0, s0, s9
-; HSA-NEXT:    s_mov_b32 flat_scratch_lo, s7
 ; HSA-NEXT:    s_addc_u32 s1, s1, 0
-; HSA-NEXT:    s_mov_b64 s[6:7], s[4:5]
-; HSA-NEXT:    v_mov_b32_e32 v0, 0x4400
-; HSA-NEXT:    s_mov_b32 s32, 0
+; HSA-NEXT:    s_mov_b32 flat_scratch_lo, s7
 ; HSA-NEXT:    s_getpc_b64 s[8:9]
 ; HSA-NEXT:    s_add_u32 s8, s8, external_void_func_f16@rel32@lo+4
 ; HSA-NEXT:    s_addc_u32 s9, s9, external_void_func_f16@rel32@hi+12
+; HSA-NEXT:    s_mov_b64 s[6:7], s[4:5]
+; HSA-NEXT:    v_mov_b32_e32 v0, 0x4400
+; HSA-NEXT:    s_mov_b32 s32, 0
 ; HSA-NEXT:    s_swappc_b64 s[30:31], s[8:9]
 ; HSA-NEXT:    s_endpgm
   call void @external_void_func_f16(half 4.0)
@@ -1711,12 +1697,12 @@ define amdgpu_kernel void @test_call_external_void_func_f32_imm() #0 {
 ; VI-NEXT:    s_addc_u32 s37, s37, 0
 ; VI-NEXT:    s_mov_b64 s[6:7], s[0:1]
 ; VI-NEXT:    s_mov_b64 s[0:1], s[36:37]
-; VI-NEXT:    s_mov_b64 s[2:3], s[38:39]
-; VI-NEXT:    v_mov_b32_e32 v0, 4.0
-; VI-NEXT:    s_mov_b32 s32, 0
 ; VI-NEXT:    s_getpc_b64 s[4:5]
 ; VI-NEXT:    s_add_u32 s4, s4, external_void_func_f32@rel32@lo+4
 ; VI-NEXT:    s_addc_u32 s5, s5, external_void_func_f32@rel32@hi+12
+; VI-NEXT:    s_mov_b64 s[2:3], s[38:39]
+; VI-NEXT:    v_mov_b32_e32 v0, 4.0
+; VI-NEXT:    s_mov_b32 s32, 0
 ; VI-NEXT:    s_swappc_b64 s[30:31], s[4:5]
 ; VI-NEXT:    s_endpgm
 ;
@@ -1730,12 +1716,12 @@ define amdgpu_kernel void @test_call_external_void_func_f32_imm() #0 {
 ; CI-NEXT:    s_addc_u32 s37, s37, 0
 ; CI-NEXT:    s_mov_b64 s[6:7], s[0:1]
 ; CI-NEXT:    s_mov_b64 s[0:1], s[36:37]
-; CI-NEXT:    s_mov_b64 s[2:3], s[38:39]
-; CI-NEXT:    v_mov_b32_e32 v0, 4.0
-; CI-NEXT:    s_mov_b32 s32, 0
 ; CI-NEXT:    s_getpc_b64 s[4:5]
 ; CI-NEXT:    s_add_u32 s4, s4, external_void_func_f32@rel32@lo+4
 ; CI-NEXT:    s_addc_u32 s5, s5, external_void_func_f32@rel32@hi+12
+; CI-NEXT:    s_mov_b64 s[2:3], s[38:39]
+; CI-NEXT:    v_mov_b32_e32 v0, 4.0
+; CI-NEXT:    s_mov_b32 s32, 0
 ; CI-NEXT:    s_swappc_b64 s[30:31], s[4:5]
 ; CI-NEXT:    s_endpgm
 ;
@@ -1749,24 +1735,23 @@ define amdgpu_kernel void @test_call_external_void_func_f32_imm() #0 {
 ; GFX9-NEXT:    s_addc_u32 s37, s37, 0
 ; GFX9-NEXT:    s_mov_b64 s[6:7], s[0:1]
 ; GFX9-NEXT:    s_mov_b64 s[0:1], s[36:37]
-; GFX9-NEXT:    s_mov_b64 s[2:3], s[38:39]
-; GFX9-NEXT:    v_mov_b32_e32 v0, 4.0
-; GFX9-NEXT:    s_mov_b32 s32, 0
 ; GFX9-NEXT:    s_getpc_b64 s[4:5]
 ; GFX9-NEXT:    s_add_u32 s4, s4, external_void_func_f32@rel32@lo+4
 ; GFX9-NEXT:    s_addc_u32 s5, s5, external_void_func_f32@rel32@hi+12
+; GFX9-NEXT:    s_mov_b64 s[2:3], s[38:39]
+; GFX9-NEXT:    v_mov_b32_e32 v0, 4.0
+; GFX9-NEXT:    s_mov_b32 s32, 0
 ; GFX9-NEXT:    s_swappc_b64 s[30:31], s[4:5]
 ; GFX9-NEXT:    s_endpgm
 ;
 ; GFX11-LABEL: test_call_external_void_func_f32_imm:
 ; GFX11:       ; %bb.0:
 ; GFX11-NEXT:    v_mov_b32_e32 v0, 4.0
-; GFX11-NEXT:    s_mov_b64 s[6:7], s[0:1]
-; GFX11-NEXT:    s_mov_b32 s32, 0
 ; GFX11-NEXT:    s_getpc_b64 s[2:3]
 ; GFX11-NEXT:    s_add_u32 s2, s2, external_void_func_f32@rel32@lo+4
 ; GFX11-NEXT:    s_addc_u32 s3, s3, external_void_func_f32@rel32@hi+12
-; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_mov_b64 s[6:7], s[0:1]
+; GFX11-NEXT:    s_mov_b32 s32, 0
 ; GFX11-NEXT:    s_swappc_b64 s[30:31], s[2:3]
 ; GFX11-NEXT:    s_endpgm
 ;
@@ -1775,14 +1760,14 @@ define amdgpu_kernel void @test_call_external_void_func_f32_imm() #0 {
 ; HSA-NEXT:    s_add_i32 s6, s6, s9
 ; HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s6, 8
 ; HSA-NEXT:    s_add_u32 s0, s0, s9
-; HSA-NEXT:    s_mov_b32 flat_scratch_lo, s7
 ; HSA-NEXT:    s_addc_u32 s1, s1, 0
-; HSA-NEXT:    s_mov_b64 s[6:7], s[4:5]
-; HSA-NEXT:    v_mov_b32_e32 v0, 4.0
-; HSA-NEXT:    s_mov_b32 s32, 0
+; HSA-NEXT:    s_mov_b32 flat_scratch_lo, s7
 ; HSA-NEXT:    s_getpc_b64 s[8:9]
 ; HSA-NEXT:    s_add_u32 s8, s8, external_void_func_f32@rel32@lo+4
 ; HSA-NEXT:    s_addc_u32 s9, s9, external_void_func_f32@rel32@hi+12
+; HSA-NEXT:    s_mov_b64 s[6:7], s[4:5]
+; HSA-NEXT:    v_mov_b32_e32 v0, 4.0
+; HSA-NEXT:    s_mov_b32 s32, 0
 ; HSA-NEXT:    s_swappc_b64 s[30:31], s[8:9]
 ; HSA-NEXT:    s_endpgm
   call void @external_void_func_f32(float 4.0)
@@ -1800,13 +1785,13 @@ define amdgpu_kernel void @test_call_external_void_func_v2f32_imm() #0 {
 ; VI-NEXT:    s_addc_u32 s37, s37, 0
 ; VI-NEXT:    s_mov_b64 s[6:7], s[0:1]
 ; VI-NEXT:    s_mov_b64 s[0:1], s[36:37]
+; VI-NEXT:    s_getpc_b64 s[4:5]
+; VI-NEXT:    s_add_u32 s4, s4, external_void_func_v2f32@rel32@lo+4
+; VI-NEXT:    s_addc_u32 s5, s5, external_void_func_v2f32@rel32@hi+12
 ; VI-NEXT:    s_mov_b64 s[2:3], s[38:39]
 ; VI-NEXT:    v_mov_b32_e32 v0, 1.0
 ; VI-NEXT:    v_mov_b32_e32 v1, 2.0
 ; VI-NEXT:    s_mov_b32 s32, 0
-; VI-NEXT:    s_getpc_b64 s[4:5]
-; VI-NEXT:    s_add_u32 s4, s4, external_void_func_v2f32@rel32@lo+4
-; VI-NEXT:    s_addc_u32 s5, s5, external_void_func_v2f32@rel32@hi+12
 ; VI-NEXT:    s_swappc_b64 s[30:31], s[4:5]
 ; VI-NEXT:    s_endpgm
 ;
@@ -1820,13 +1805,13 @@ define amdgpu_kernel void @test_call_external_void_func_v2f32_imm() #0 {
 ; CI-NEXT:    s_addc_u32 s37, s37, 0
 ; CI-NEXT:    s_mov_b64 s[6:7], s[0:1]
 ; CI-NEXT:    s_mov_b64 s[0:1], s[36:37]
+; CI-NEXT:    s_getpc_b64 s[4:5]
+; CI-NEXT:    s_add_u32 s4, s4, external_void_func_v2f32@rel32@lo+4
+; CI-NEXT:    s_addc_u32 s5, s5, external_void_func_v2f32@rel32@hi+12
 ; CI-NEXT:    s_mov_b64 s[2:3], s[38:39]
 ; CI-NEXT:    v_mov_b32_e32 v0, 1.0
 ; CI-NEXT:    v_mov_b32_e32 v1, 2.0
 ; CI-NEXT:    s_mov_b32 s32, 0
-; CI-NEXT:    s_getpc_b64 s[4:5]
-; CI-NEXT:    s_add_u32 s4, s4, external_void_func_v2f32@rel32@lo+4
-; CI-NEXT:    s_addc_u32 s5, s5, external_void_func_v2f32@rel32@hi+12
 ; CI-NEXT:    s_swappc_b64 s[30:31], s[4:5]
 ; CI-NEXT:    s_endpgm
 ;
@@ -1840,25 +1825,24 @@ define amdgpu_kernel void @test_call_external_void_func_v2f32_imm() #0 {
 ; GFX9-NEXT:    s_addc_u32 s37, s37, 0
 ; GFX9-NEXT:    s_mov_b64 s[6:7], s[0:1]
 ; GFX9-NEXT:    s_mov_b64 s[0:1], s[36:37]
+; GFX9-NEXT:    s_getpc_b64 s[4:5]
+; GFX9-NEXT:    s_add_u32 s4, s4, external_void_func_v2f32@rel32@lo+4
+; GFX9-NEXT:    s_addc_u32 s5, s5, external_void_func_v2f32@rel32@hi+12
 ; GFX9-NEXT:    s_mov_b64 s[2:3], s[38:39]
 ; GFX9-NEXT:    v_mov_b32_e32 v0, 1.0
 ; GFX9-NEXT:    v_mov_b32_e32 v1, 2.0
 ; GFX9-NEXT:    s_mov_b32 s32, 0
-; GFX9-NEXT:    s_getpc_b64 s[4:5]
-; GFX9-NEXT:    s_add_u32 s4, s4, external_void_func_v2f32@rel32@lo+4
-; GFX9-NEXT:    s_addc_u32 s5, s5, external_void_func_v2f32@rel32@hi+12
 ; GFX9-NEXT:    s_swappc_b64 s[30:31], s[4:5]
 ; GFX9-NEXT:    s_endpgm
 ;
 ; GFX11-LABEL: test_call_external_void_func_v2f32_imm:
 ; GFX11:       ; %bb.0:
 ; GFX11-NEXT:    v_dual_mov_b32 v0, 1.0 :: v_dual_mov_b32 v1, 2.0
-; GFX11-NEXT:    s_mov_b64 s[6:7], s[0:1]
-; GFX11-NEXT:    s_mov_b32 s32, 0
 ; GFX11-NEXT:    s_getpc_b64 s[2:3]
 ; GFX11-NEXT:    s_add_u32 s2, s2, external_void_func_v2f32@rel32@lo+4
 ; GFX11-NEXT:    s_addc_u32 s3, s3, external_void_func_v2f32@rel32@hi+12
-; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_mov_b64 s[6:7], s[0:1]
+; GFX11-NEXT:    s_mov_b32 s32, 0
 ; GFX11-NEXT:    s_swappc_b64 s[30:31], s[2:3]
 ; GFX11-NEXT:    s_endpgm
 ;
@@ -1867,15 +1851,15 @@ define amdgpu_kernel void @test_call_external_void_func_v2f32_imm() #0 {
 ; HSA-NEXT:    s_add_i32 s6, s6, s9
 ; HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s6, 8
 ; HSA-NEXT:    s_add_u32 s0, s0, s9
-; HSA-NEXT:    s_mov_b32 flat_scratch_lo, s7
 ; HSA-NEXT:    s_addc_u32 s1, s1, 0
+; HSA-NEXT:    s_mov_b32 flat_scratch_lo, s7
+; HSA-NEXT:    s_getpc_b64 s[8:9]
+; HSA-NEXT:    s_add_u32 s8, s8, external_void_func_v2f32@rel32@lo+4
+; HSA-NEXT:    s_addc_u32 s9, s9, external_void_func_v2f32@rel32@hi+12
 ; HSA-NEXT:    s_mov_b64 s[6:7], s[4:5]
 ; HSA-NEXT:    v_mov_b32_e32 v0, 1.0
 ; HSA-NEXT:    v_mov_b32_e32 v1, 2.0
 ; HSA-NEXT:    s_mov_b32 s32, 0
-; HSA-NEXT:    s_getpc_b64 s[8:9]
-; HSA-NEXT:    s_add_u32 s8, s8, external_void_func_v2f32@rel32@lo+4
-; HSA-NEXT:    s_addc_u32 s9, s9, external_void_func_v2f32@rel32@hi+12
 ; HSA-NEXT:    s_swappc_b64 s[30:31], s[8:9]
 ; HSA-NEXT:    s_endpgm
   call void @external_void_func_v2f32(<2 x float> <float 1.0, float 2.0>)
@@ -1893,14 +1877,14 @@ define amdgpu_kernel void @test_call_external_void_func_v3f32_imm() #0 {
 ; VI-NEXT:    s_addc_u32 s37, s37, 0
 ; VI-NEXT:    s_mov_b64 s[6:7], s[0:1]
 ; VI-NEXT:    s_mov_b64 s[0:1], s[36:37]
+; VI-NEXT:    s_getpc_b64 s[4:5]
+; VI-NEXT:    s_add_u32 s4, s4, external_void_func_v3f32@rel32@lo+4
+; VI-NEXT:    s_addc_u32 s5, s5, external_void_func_v3f32@rel32@hi+12
 ; VI-NEXT:    s_mov_b64 s[2:3], s[38:39]
 ; VI-NEXT:    v_mov_b32_e32 v0, 1.0
 ; VI-NEXT:    v_mov_b32_e32 v1, 2.0
 ; VI-NEXT:    v_mov_b32_e32 v2, 4.0
 ; VI-NEXT:    s_mov_b32 s32, 0
-; VI-NEXT:    s_getpc_b64 s[4:5]
-; VI-NEXT:    s_add_u32 s4, s4, external_void_func_v3f32@rel32@lo+4
-; VI-NEXT:    s_addc_u32 s5, s5, external_void_func_v3f32@rel32@hi+12
 ; VI-NEXT:    s_swappc_b64 s[30:31], s[4:5]
 ; VI-NEXT:    s_endpgm
 ;
@@ -1914,14 +1898,14 @@ define amdgpu_kernel void @test_call_external_void_func_v3f32_imm() #0 {
 ; CI-NEXT:    s_addc_u32 s37, s37, 0
 ; CI-NEXT:    s_mov_b64 s[6:7], s[0:1]
 ; CI-NEXT:    s_mov_b64 s[0:1], s[36:37]
+; CI-NEXT:    s_getpc_b64 s[4:5]
+; CI-NEXT:    s_add_u32 s4, s4, external_void_func_v3f32@rel32@lo+4
+; CI-NEXT:    s_addc_u32 s5, s5, external_void_func_v3f32@rel32@hi+12
 ; CI-NEXT:    s_mov_b64 s[2:3], s[38:39]
 ; CI-NEXT:    v_mov_b32_e32 v0, 1.0
 ; CI-NEXT:    v_mov_b32_e32 v1, 2.0
 ; CI-NEXT:    v_mov_b32_e32 v2, 4.0
 ; CI-NEXT:    s_mov_b32 s32, 0
-; CI-NEXT:    s_getpc_b64 s[4:5]
-; CI-NEXT:    s_add_u32 s4, s4, external_void_func_v3f32@rel32@lo+4
-; CI-NEXT:    s_addc_u32 s5, s5, external_void_func_v3f32@rel32@hi+12
 ; CI-NEXT:    s_swappc_b64 s[30:31], s[4:5]
 ; CI-NEXT:    s_endpgm
 ;
@@ -1935,14 +1919,14 @@ define amdgpu_kernel void @test_call_external_void_func_v3f32_imm() #0 {
 ; GFX9-NEXT:    s_addc_u32 s37, s37, 0
 ; GFX9-NEXT:    s_mov_b64 s[6:7], s[0:1]
 ; GFX9-NEXT:    s_mov_b64 s[0:1], s[36:37]
+; GFX9-NEXT:    s_getpc_b64 s[4:5]
+; GFX9-NEXT:    s_add_u32 s4, s4, external_void_func_v3f32@rel32@lo+4
+; GFX9-NEXT:    s_addc_u32 s5, s5, external_void_func_v3f32@rel32@hi+12
 ; GFX9-NEXT:    s_mov_b64 s[2:3], s[38:39]
 ; GFX9-NEXT:    v_mov_b32_e32 v0, 1.0
 ; GFX9-NEXT:    v_mov_b32_e32 v1, 2.0
 ; GFX9-NEXT:    v_mov_b32_e32 v2, 4.0
 ; GFX9-NEXT:    s_mov_b32 s32, 0
-; GFX9-NEXT:    s_getpc_b64 s[4:5]
-; GFX9-NEXT:    s_add_u32 s4, s4, external_void_func_v3f32@rel32@lo+4
-; GFX9-NEXT:    s_addc_u32 s5, s5, external_void_func_v3f32@rel32@hi+12
 ; GFX9-NEXT:    s_swappc_b64 s[30:31], s[4:5]
 ; GFX9-NEXT:    s_endpgm
 ;
@@ -1950,12 +1934,11 @@ define amdgpu_kernel void @test_call_external_void_func_v3f32_imm() #0 {
 ; GFX11:       ; %bb.0:
 ; GFX11-NEXT:    v_dual_mov_b32 v0, 1.0 :: v_dual_mov_b32 v1, 2.0
 ; GFX11-NEXT:    v_mov_b32_e32 v2, 4.0
-; GFX11-NEXT:    s_mov_b64 s[6:7], s[0:1]
-; GFX11-NEXT:    s_mov_b32 s32, 0
 ; GFX11-NEXT:    s_getpc_b64 s[2:3]
 ; GFX11-NEXT:    s_add_u32 s2, s2, external_void_func_v3f32@rel32@lo+4
 ; GFX11-NEXT:    s_addc_u32 s3, s3, external_void_func_v3f32@rel32@hi+12
-; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_mov_b64 s[6:7], s[0:1]
+; GFX11-NEXT:    s_mov_b32 s32, 0
 ; GFX11-NEXT:    s_swappc_b64 s[30:31], s[2:3]
 ; GFX11-NEXT:    s_endpgm
 ;
@@ -1964,16 +1947,16 @@ define amdgpu_kernel void @test_call_external_void_func_v3f32_imm() #0 {
 ; HSA-NEXT:    s_add_i32 s6, s6, s9
 ; HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s6, 8
 ; HSA-NEXT:    s_add_u32 s0, s0, s9
-; HSA-NEXT:    s_mov_b32 flat_scratch_lo, s7
 ; HSA-NEXT:    s_addc_u32 s1, s1, 0
+; HSA-NEXT:    s_mov_b32 flat_scratch_lo, s7
+; HSA-NEXT:    s_getpc_b64 s[8:9]
+; HSA-NEXT:    s_add_u32 s8, s8, external_void_func_v3f32@rel32@lo+4
+; HSA-NEXT:    s_addc_u32 s9, s9, external_void_func_v3f32@rel32@hi+12
 ; HSA-NEXT:    s_mov_b64 s[6:7], s[4:5]
 ; HSA-NEXT:    v_mov_b32_e32 v0, 1.0
 ; HSA-NEXT:    v_mov_b32_e32 v1, 2.0
 ; HSA-NEXT:    v_mov_b32_e32 v2, 4.0
 ; HSA-NEXT:    s_mov_b32 s32, 0
-; HSA-NEXT:    s_getpc_b64 s[8:9]
-; HSA-NEXT:    s_add_u32 s8, s8, external_void_func_v3f32@rel32@lo+4
-; HSA-NEXT:    s_addc_u32 s9, s9, external_void_func_v3f32@rel32@hi+12
 ; HSA-NEXT:    s_swappc_b64 s[30:31], s[8:9]
 ; HSA-NEXT:    s_endpgm
   call void @external_void_func_v3f32(<3 x float> <float 1.0, float 2.0, float 4.0>)
@@ -1991,6 +1974,9 @@ define amdgpu_kernel void @test_call_external_void_func_v5f32_imm() #0 {
 ; VI-NEXT:    s_addc_u32 s37, s37, 0
 ; VI-NEXT:    s_mov_b64 s[6:7], s[0:1]
 ; VI-NEXT:    s_mov_b64 s[0:1], s[36:37]
+; VI-NEXT:    s_getpc_b64 s[4:5]
+; VI-NEXT:    s_add_u32 s4, s4, external_void_func_v5f32@rel32@lo+4
+; VI-NEXT:    s_addc_u32 s5, s5, external_void_func_v5f32@rel32@hi+12
 ; VI-NEXT:    s_mov_b64 s[2:3], s[38:39]
 ; VI-NEXT:    v_mov_b32_e32 v0, 1.0
 ; VI-NEXT:    v_mov_b32_e32 v1, 2.0
@@ -1998,9 +1984,6 @@ define amdgpu_kernel void @test_call_external_void_func_v5f32_imm() #0 {
 ; VI-NEXT:    v_mov_b32_e32 v3, -1.0
 ; VI-NEXT:    v_mov_b32_e32 v4, 0.5
 ; VI-NEXT:    s_mov_b32 s32, 0
-; VI-NEXT:    s_getpc_b64 s[4:5]
-; VI-NEXT:    s_add_u32 s4, s4, external_void_func_v5f32@rel32@lo+4
-; VI-NEXT:    s_addc_u32 s5, s5, external_void_func_v5f32@rel32@hi+12
 ; VI-NEXT:    s_swappc_b64 s[30:31], s[4:5]
 ; VI-NEXT:    s_endpgm
 ;
@@ -2014,6 +1997,9 @@ define amdgpu_kernel void @test_call_external_void_func_v5f32_imm() #0 {
 ; CI-NEXT:    s_addc_u32 s37, s37, 0
 ; CI-NEXT:    s_mov_b64 s[6:7], s[0:1]
 ; CI-NEXT:    s_mov_b64 s[0:1], s[36:37]
+; CI-NEXT:    s_getpc_b64 s[4:5]
+; CI-NEXT:    s_add_u32 s4, s4, external_void_func_v5f32@rel32@lo+4
+; CI-NEXT:    s_addc_u32 s5, s5, external_void_func_v5f32@rel32@hi+12
 ; CI-NEXT:    s_mov_b64 s[2:3], s[38:39]
 ; CI-NEXT:    v_mov_b32_e32 v0, 1.0
 ; CI-NEXT:    v_mov_b32_e32 v1, 2.0
@@ -2021,9 +2007,6 @@ define amdgpu_kernel void @test_call_external_void_func_v5f32_imm() #0 {
 ; CI-NEXT:    v_mov_b32_e32 v3, -1.0
 ; CI-NEXT:    v_mov_b32_e32 v4, 0.5
 ; CI-NEXT:    s_mov_b32 s32, 0
-; CI-NEXT:    s_getpc_b64 s[4:5]
-; CI-NEXT:    s_add_u32 s4, s4, external_void_func_v5f32@rel32@lo+4
-; CI-NEXT:    s_addc_u32 s5, s5, external_void_func_v5f32@rel32@hi+12
 ; CI-NEXT:    s_swappc_b64 s[30:31], s[4:5]
 ; CI-NEXT:    s_endpgm
 ;
@@ -2037,6 +2020,9 @@ define amdgpu_kernel void @test_call_external_void_func_v5f32_imm() #0 {
 ; GFX9-NEXT:    s_addc_u32 s37, s37, 0
 ; GFX9-NEXT:    s_mov_b64 s[6:7], s[0:1]
 ; GFX9-NEXT:    s_mov_b64 s[0:1], s[36:37]
+; GFX9-NEXT:    s_getpc_b64 s[4:5]
+; GFX9-NEXT:    s_add_u32 s4, s4, external_void_func_v5f32@rel32@lo+4
+; GFX9-NEXT:    s_addc_u32 s5, s5, external_void_func_v5f32@rel32@hi+12
 ; GFX9-NEXT:    s_mov_b64 s[2:3], s[38:39]
 ; GFX9-NEXT:    v_mov_b32_e32 v0, 1.0
 ; GFX9-NEXT:    v_mov_b32_e32 v1, 2.0
@@ -2044,9 +2030,6 @@ define amdgpu_kernel void @test_call_external_void_func_v5f32_imm() #0 {
 ; GFX9-NEXT:    v_mov_b32_e32 v3, -1.0
 ; GFX9-NEXT:    v_mov_b32_e32 v4, 0.5
 ; GFX9-NEXT:    s_mov_b32 s32, 0
-; GFX9-NEXT:    s_getpc_b64 s[4:5]
-; GFX9-NEXT:    s_add_u32 s4, s4, external_void_func_v5f32@rel32@lo+4
-; GFX9-NEXT:    s_addc_u32 s5, s5, external_void_func_v5f32@rel32@hi+12
 ; GFX9-NEXT:    s_swappc_b64 s[30:31], s[4:5]
 ; GFX9-NEXT:    s_endpgm
 ;
@@ -2055,12 +2038,11 @@ define amdgpu_kernel void @test_call_external_void_func_v5f32_imm() #0 {
 ; GFX11-NEXT:    v_dual_mov_b32 v0, 1.0 :: v_dual_mov_b32 v1, 2.0
 ; GFX11-NEXT:    v_dual_mov_b32 v2, 4.0 :: v_dual_mov_b32 v3, -1.0
 ; GFX11-NEXT:    v_mov_b32_e32 v4, 0.5
-; GFX11-NEXT:    s_mov_b64 s[6:7], s[0:1]
-; GFX11-NEXT:    s_mov_b32 s32, 0
 ; GFX11-NEXT:    s_getpc_b64 s[2:3]
 ; GFX11-NEXT:    s_add_u32 s2, s2, external_void_func_v5f32@rel32@lo+4
 ; GFX11-NEXT:    s_addc_u32 s3, s3, external_void_func_v5f32@rel32@hi+12
-; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_mov_b64 s[6:7], s[0:1]
+; GFX11-NEXT:    s_mov_b32 s32, 0
 ; GFX11-NEXT:    s_swappc_b64 s[30:31], s[2:3]
 ; GFX11-NEXT:    s_endpgm
 ;
@@ -2069,8 +2051,11 @@ define amdgpu_kernel void @test_call_external_void_func_v5f32_imm() #0 {
 ; HSA-NEXT:    s_add_i32 s6, s6, s9
 ; HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s6, 8
 ; HSA-NEXT:    s_add_u32 s0, s0, s9
-; HSA-NEXT:    s_mov_b32 flat_scratch_lo, s7
 ; HSA-NEXT:    s_addc_u32 s1, s1, 0
+; HSA-NEXT:    s_mov_b32 flat_scratch_lo, s7
+; HSA-NEXT:    s_getpc_b64 s[8:9]
+; HSA-NEXT:    s_add_u32 s8, s8, external_void_func_v5f32@rel32@lo+4
+; HSA-NEXT:    s_addc_u32 s9, s9, external_void_func_v5f32@rel32@hi+12
 ; HSA-NEXT:    s_mov_b64 s[6:7], s[4:5]
 ; HSA-NEXT:    v_mov_b32_e32 v0, 1.0
 ; HSA-NEXT:    v_mov_b32_e32 v1, 2.0
@@ -2078,9 +2063,6 @@ define amdgpu_kernel void @test_call_external_void_func_v5f32_imm() #0 {
 ; HSA-NEXT:    v_mov_b32_e32 v3, -1.0
 ; HSA-NEXT:    v_mov_b32_e32 v4, 0.5
 ; HSA-NEXT:    s_mov_b32 s32, 0
-; HSA-NEXT:    s_getpc_b64 s[8:9]
-; HSA-NEXT:    s_add_u32 s8, s8, external_void_func_v5f32@rel32@lo+4
-; HSA-NEXT:    s_addc_u32 s9, s9, external_void_func_v5f32@rel32@hi+12
 ; HSA-NEXT:    s_swappc_b64 s[30:31], s[8:9]
 ; HSA-NEXT:    s_endpgm
   call void @external_void_func_v5f32(<5 x float> <float 1.0, float 2.0, float 4.0, float -1.0, float 0.5>)
@@ -2098,13 +2080,13 @@ define amdgpu_kernel void @test_call_external_void_func_f64_imm() #0 {
 ; VI-NEXT:    s_addc_u32 s37, s37, 0
 ; VI-NEXT:    s_mov_b64 s[6:7], s[0:1]
 ; VI-NEXT:    s_mov_b64 s[0:1], s[36:37]
+; VI-NEXT:    s_getpc_b64 s[4:5]
+; VI-NEXT:    s_add_u32 s4, s4, external_void_func_f64@rel32@lo+4
+; VI-NEXT:    s_addc_u32 s5, s5, external_void_func_f64@rel32@hi+12
 ; VI-NEXT:    s_mov_b64 s[2:3], s[38:39]
 ; VI-NEXT:    v_mov_b32_e32 v0, 0
 ; VI-NEXT:    v_mov_b32_e32 v1, 0x40100000
 ; VI-NEXT:    s_mov_b32 s32, 0
-; VI-NEXT:    s_getpc_b64 s[4:5]
-; VI-NEXT:    s_add_u32 s4, s4, external_void_func_f64@rel32@lo+4
-; VI-NEXT:    s_addc_u32 s5, s5, external_void_func_f64@rel32@hi+12
 ; VI-NEXT:    s_swappc_b64 s[30:31], s[4:5]
 ; VI-NEXT:    s_endpgm
 ;
@@ -2118,13 +2100,13 @@ define amdgpu_kernel void @test_call_external_void_func_f64_imm() #0 {
 ; CI-NEXT:    s_addc_u32 s37, s37, 0
 ; CI-NEXT:    s_mov_b64 s[6:7], s[0:1]
 ; CI-NEXT:    s_mov_b64 s[0:1], s[36:37]
+; CI-NEXT:    s_getpc_b64 s[4:5]
+; CI-NEXT:    s_add_u32 s4, s4, external_void_func_f64@rel32@lo+4
+; CI-NEXT:    s_addc_u32 s5, s5, external_void_func_f64@rel32@hi+12
 ; CI-NEXT:    s_mov_b64 s[2:3], s[38:39]
 ; CI-NEXT:    v_mov_b32_e32 v0, 0
 ; CI-NEXT:    v_mov_b32_e32 v1, 0x40100000
 ; CI-NEXT:    s_mov_b32 s32, 0
-; CI-NEXT:    s_getpc_b64 s[4:5]
-; CI-NEXT:    s_add_u32 s4, s4, external_void_func_f64@rel32@lo+4
-; CI-NEXT:    s_addc_u32 s5, s5, external_void_func_f64@rel32@hi+12
 ; CI-NEXT:    s_swappc_b64 s[30:31], s[4:5]
 ; CI-NEXT:    s_endpgm
 ;
@@ -2138,25 +2120,24 @@ define amdgpu_kernel void @test_call_external_void_func_f64_imm() #0 {
 ; GFX9-NEXT:    s_addc_u32 s37, s37, 0
 ; GFX9-NEXT:    s_mov_b64 s[6:7], s[0:1]
 ; GFX9-NEXT:    s_mov_b64 s[0:1], s[36:37]
+; GFX9-NEXT:    s_getpc_b64 s[4:5]
+; GFX9-NEXT:    s_add_u32 s4, s4, external_void_func_f64@rel32@lo+4
+; GFX9-NEXT:    s_addc_u32 s5, s5, external_void_func_f64@rel32@hi+12
 ; GFX9-NEXT:    s_mov_b64 s[2:3], s[38:39]
 ; GFX9-NEXT:    v_mov_b32_e32 v0, 0
 ; GFX9-NEXT:    v_mov_b32_e32 v1, 0x40100000
 ; GFX9-NEXT:    s_mov_b32 s32, 0
-; GFX9-NEXT:    s_getpc_b64 s[4:5]
-; GFX9-NEXT:    s_add_u32 s4, s4, external_void_func_f64@rel32@lo+4
-; GFX9-NEXT:    s_addc_u32 s5, s5, external_void_func_f64@rel32@hi+12
 ; GFX9-NEXT:    s_swappc_b64 s[30:31], s[4:5]
 ; GFX9-NEXT:    s_endpgm
 ;
 ; GFX11-LABEL: test_call_external_void_func_f64_imm:
 ; GFX11:       ; %bb.0:
 ; GFX11-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, 0x40100000
-; GFX11-NEXT:    s_mov_b64 s[6:7], s[0:1]
-; GFX11-NEXT:    s_mov_b32 s32, 0
 ; GFX11-NEXT:    s_getpc_b64 s[2:3]
 ; GFX11-NEXT:    s_add_u32 s2, s2, external_void_func_f64@rel32@lo+4
 ; GFX11-NEXT:    s_addc_u32 s3, s3, external_void_func_f64@rel32@hi+12
-; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_mov_b64 s[6:7], s[0:1]
+; GFX11-NEXT:    s_mov_b32 s32, 0
 ; GFX11-NEXT:    s_swappc_b64 s[30:31], s[2:3]
 ; GFX11-NEXT:    s_endpgm
 ;
@@ -2165,15 +2146,15 @@ define amdgpu_kernel void @test_call_external_void_func_f64_imm() #0 {
 ; HSA-NEXT:    s_add_i32 s6, s6, s9
 ; HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s6, 8
 ; HSA-NEXT:    s_add_u32 s0, s0, s9
-; HSA-NEXT:    s_mov_b32 flat_scratch_lo, s7
 ; HSA-NEXT:    s_addc_u32 s1, s1, 0
+; HSA-NEXT:    s_mov_b32 flat_scratch_lo, s7
+; HSA-NEXT:    s_getpc_b64 s[8:9]
+; HSA-NEXT:    s_add_u32 s8, s8, external_void_func_f64@rel32@lo+4
+; HSA-NEXT:    s_addc_u32 s9, s9, external_void_func_f64@rel32@hi+12
 ; HSA-NEXT:    s_mov_b64 s[6:7], s[4:5]
 ; HSA-NEXT:    v_mov_b32_e32 v0, 0
 ; HSA-NEXT:    v_mov_b32_e32 v1, 0x40100000
 ; HSA-NEXT:    s_mov_b32 s32, 0
-; HSA-NEXT:    s_getpc_b64 s[8:9]
-; HSA-NEXT:    s_add_u32 s8, s8, external_void_func_f64@rel32@lo+4
-; HSA-NEXT:    s_addc_u32 s9, s9, external_void_func_f64@rel32@hi+12
 ; HSA-NEXT:    s_swappc_b64 s[30:31], s[8:9]
 ; HSA-NEXT:    s_endpgm
   call void @external_void_func_f64(double 4.0)
@@ -2191,15 +2172,15 @@ define amdgpu_kernel void @test_call_external_void_func_v2f64_imm() #0 {
 ; VI-NEXT:    s_addc_u32 s37, s37, 0
 ; VI-NEXT:    s_mov_b64 s[6:7], s[0:1]
 ; VI-NEXT:    s_mov_b64 s[0:1], s[36:37]
+; VI-NEXT:    s_getpc_b64 s[4:5]
+; VI-NEXT:    s_add_u32 s4, s4, external_void_func_v2f64@rel32@lo+4
+; VI-NEXT:    s_addc_u32 s5, s5, external_void_func_v2f64@rel32@hi+12
 ; VI-NEXT:    s_mov_b64 s[2:3], s[38:39]
 ; VI-NEXT:    v_mov_b32_e32 v0, 0
 ; VI-NEXT:    v_mov_b32_e32 v1, 2.0
 ; VI-NEXT:    v_mov_b32_e32 v2, 0
-; VI-NEXT:    v_mov_b32_e32 v3, 0x40100000
-; VI-NEXT:    s_mov_b32 s32, 0
-; VI-NEXT:    s_getpc_b64 s[4:5]
-; VI-NEXT:    s_add_u32 s4, s4, external_void_func_v2f64@rel32@lo+4
-; VI-NEXT:    s_addc_u32 s5, s5, external_void_func_v2f64@rel32@hi+12
+; VI-NEXT:    v_mov_b32_e32 v3, 0x40100000
+; VI-NEXT:    s_mov_b32 s32, 0
 ; VI-NEXT:    s_swappc_b64 s[30:31], s[4:5]
 ; VI-NEXT:    s_endpgm
 ;
@@ -2213,15 +2194,15 @@ define amdgpu_kernel void @test_call_external_void_func_v2f64_imm() #0 {
 ; CI-NEXT:    s_addc_u32 s37, s37, 0
 ; CI-NEXT:    s_mov_b64 s[6:7], s[0:1]
 ; CI-NEXT:    s_mov_b64 s[0:1], s[36:37]
+; CI-NEXT:    s_getpc_b64 s[4:5]
+; CI-NEXT:    s_add_u32 s4, s4, external_void_func_v2f64@rel32@lo+4
+; CI-NEXT:    s_addc_u32 s5, s5, external_void_func_v2f64@rel32@hi+12
 ; CI-NEXT:    s_mov_b64 s[2:3], s[38:39]
 ; CI-NEXT:    v_mov_b32_e32 v0, 0
 ; CI-NEXT:    v_mov_b32_e32 v1, 2.0
 ; CI-NEXT:    v_mov_b32_e32 v2, 0
 ; CI-NEXT:    v_mov_b32_e32 v3, 0x40100000
 ; CI-NEXT:    s_mov_b32 s32, 0
-; CI-NEXT:    s_getpc_b64 s[4:5]
-; CI-NEXT:    s_add_u32 s4, s4, external_void_func_v2f64@rel32@lo+4
-; CI-NEXT:    s_addc_u32 s5, s5, external_void_func_v2f64@rel32@hi+12
 ; CI-NEXT:    s_swappc_b64 s[30:31], s[4:5]
 ; CI-NEXT:    s_endpgm
 ;
@@ -2235,15 +2216,15 @@ define amdgpu_kernel void @test_call_external_void_func_v2f64_imm() #0 {
 ; GFX9-NEXT:    s_addc_u32 s37, s37, 0
 ; GFX9-NEXT:    s_mov_b64 s[6:7], s[0:1]
 ; GFX9-NEXT:    s_mov_b64 s[0:1], s[36:37]
+; GFX9-NEXT:    s_getpc_b64 s[4:5]
+; GFX9-NEXT:    s_add_u32 s4, s4, external_void_func_v2f64@rel32@lo+4
+; GFX9-NEXT:    s_addc_u32 s5, s5, external_void_func_v2f64@rel32@hi+12
 ; GFX9-NEXT:    s_mov_b64 s[2:3], s[38:39]
 ; GFX9-NEXT:    v_mov_b32_e32 v0, 0
 ; GFX9-NEXT:    v_mov_b32_e32 v1, 2.0
 ; GFX9-NEXT:    v_mov_b32_e32 v2, 0
 ; GFX9-NEXT:    v_mov_b32_e32 v3, 0x40100000
 ; GFX9-NEXT:    s_mov_b32 s32, 0
-; GFX9-NEXT:    s_getpc_b64 s[4:5]
-; GFX9-NEXT:    s_add_u32 s4, s4, external_void_func_v2f64@rel32@lo+4
-; GFX9-NEXT:    s_addc_u32 s5, s5, external_void_func_v2f64@rel32@hi+12
 ; GFX9-NEXT:    s_swappc_b64 s[30:31], s[4:5]
 ; GFX9-NEXT:    s_endpgm
 ;
@@ -2251,12 +2232,11 @@ define amdgpu_kernel void @test_call_external_void_func_v2f64_imm() #0 {
 ; GFX11:       ; %bb.0:
 ; GFX11-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, 2.0
 ; GFX11-NEXT:    v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v3, 0x40100000
-; GFX11-NEXT:    s_mov_b64 s[6:7], s[0:1]
-; GFX11-NEXT:    s_mov_b32 s32, 0
 ; GFX11-NEXT:    s_getpc_b64 s[2:3]
 ; GFX11-NEXT:    s_add_u32 s2, s2, external_void_func_v2f64@rel32@lo+4
 ; GFX11-NEXT:    s_addc_u32 s3, s3, external_void_func_v2f64@rel32@hi+12
-; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_mov_b64 s[6:7], s[0:1]
+; GFX11-NEXT:    s_mov_b32 s32, 0
 ; GFX11-NEXT:    s_swappc_b64 s[30:31], s[2:3]
 ; GFX11-NEXT:    s_endpgm
 ;
@@ -2265,17 +2245,17 @@ define amdgpu_kernel void @test_call_external_void_func_v2f64_imm() #0 {
 ; HSA-NEXT:    s_add_i32 s6, s6, s9
 ; HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s6, 8
 ; HSA-NEXT:    s_add_u32 s0, s0, s9
-; HSA-NEXT:    s_mov_b32 flat_scratch_lo, s7
 ; HSA-NEXT:    s_addc_u32 s1, s1, 0
+; HSA-NEXT:    s_mov_b32 flat_scratch_lo, s7
+; HSA-NEXT:    s_getpc_b64 s[8:9]
+; HSA-NEXT:    s_add_u32 s8, s8, external_void_func_v2f64@rel32@lo+4
+; HSA-NEXT:    s_addc_u32 s9, s9, external_void_func_v2f64@rel32@hi+12
 ; HSA-NEXT:    s_mov_b64 s[6:7], s[4:5]
 ; HSA-NEXT:    v_mov_b32_e32 v0, 0
 ; HSA-NEXT:    v_mov_b32_e32 v1, 2.0
 ; HSA-NEXT:    v_mov_b32_e32 v2, 0
 ; HSA-NEXT:    v_mov_b32_e32 v3, 0x40100000
 ; HSA-NEXT:    s_mov_b32 s32, 0
-; HSA-NEXT:    s_getpc_b64 s[8:9]
-; HSA-NEXT:    s_add_u32 s8, s8, external_void_func_v2f64@rel32@lo+4
-; HSA-NEXT:    s_addc_u32 s9, s9, external_void_func_v2f64@rel32@hi+12
 ; HSA-NEXT:    s_swappc_b64 s[30:31], s[8:9]
 ; HSA-NEXT:    s_endpgm
   call void @external_void_func_v2f64(<2 x double> <double 2.0, double 4.0>)
@@ -2293,6 +2273,9 @@ define amdgpu_kernel void @test_call_external_void_func_v3f64_imm() #0 {
 ; VI-NEXT:    s_addc_u32 s37, s37, 0
 ; VI-NEXT:    s_mov_b64 s[6:7], s[0:1]
 ; VI-NEXT:    s_mov_b64 s[0:1], s[36:37]
+; VI-NEXT:    s_getpc_b64 s[4:5]
+; VI-NEXT:    s_add_u32 s4, s4, external_void_func_v3f64@rel32@lo+4
+; VI-NEXT:    s_addc_u32 s5, s5, external_void_func_v3f64@rel32@hi+12
 ; VI-NEXT:    s_mov_b64 s[2:3], s[38:39]
 ; VI-NEXT:    v_mov_b32_e32 v0, 0
 ; VI-NEXT:    v_mov_b32_e32 v1, 2.0
@@ -2301,9 +2284,6 @@ define amdgpu_kernel void @test_call_external_void_func_v3f64_imm() #0 {
 ; VI-NEXT:    v_mov_b32_e32 v4, 0
 ; VI-NEXT:    v_mov_b32_e32 v5, 0x40200000
 ; VI-NEXT:    s_mov_b32 s32, 0
-; VI-NEXT:    s_getpc_b64 s[4:5]
-; VI-NEXT:    s_add_u32 s4, s4, external_void_func_v3f64@rel32@lo+4
-; VI-NEXT:    s_addc_u32 s5, s5, external_void_func_v3f64@rel32@hi+12
 ; VI-NEXT:    s_swappc_b64 s[30:31], s[4:5]
 ; VI-NEXT:    s_endpgm
 ;
@@ -2317,6 +2297,9 @@ define amdgpu_kernel void @test_call_external_void_func_v3f64_imm() #0 {
 ; CI-NEXT:    s_addc_u32 s37, s37, 0
 ; CI-NEXT:    s_mov_b64 s[6:7], s[0:1]
 ; CI-NEXT:    s_mov_b64 s[0:1], s[36:37]
+; CI-NEXT:    s_getpc_b64 s[4:5]
+; CI-NEXT:    s_add_u32 s4, s4, external_void_func_v3f64@rel32@lo+4
+; CI-NEXT:    s_addc_u32 s5, s5, external_void_func_v3f64@rel32@hi+12
 ; CI-NEXT:    s_mov_b64 s[2:3], s[38:39]
 ; CI-NEXT:    v_mov_b32_e32 v0, 0
 ; CI-NEXT:    v_mov_b32_e32 v1, 2.0
@@ -2325,9 +2308,6 @@ define amdgpu_kernel void @test_call_external_void_func_v3f64_imm() #0 {
 ; CI-NEXT:    v_mov_b32_e32 v4, 0
 ; CI-NEXT:    v_mov_b32_e32 v5, 0x40200000
 ; CI-NEXT:    s_mov_b32 s32, 0
-; CI-NEXT:    s_getpc_b64 s[4:5]
-; CI-NEXT:    s_add_u32 s4, s4, external_void_func_v3f64@rel32@lo+4
-; CI-NEXT:    s_addc_u32 s5, s5, external_void_func_v3f64@rel32@hi+12
 ; CI-NEXT:    s_swappc_b64 s[30:31], s[4:5]
 ; CI-NEXT:    s_endpgm
 ;
@@ -2341,6 +2321,9 @@ define amdgpu_kernel void @test_call_external_void_func_v3f64_imm() #0 {
 ; GFX9-NEXT:    s_addc_u32 s37, s37, 0
 ; GFX9-NEXT:    s_mov_b64 s[6:7], s[0:1]
 ; GFX9-NEXT:    s_mov_b64 s[0:1], s[36:37]
+; GFX9-NEXT:    s_getpc_b64 s[4:5]
+; GFX9-NEXT:    s_add_u32 s4, s4, external_void_func_v3f64@rel32@lo+4
+; GFX9-NEXT:    s_addc_u32 s5, s5, external_void_func_v3f64@rel32@hi+12
 ; GFX9-NEXT:    s_mov_b64 s[2:3], s[38:39]
 ; GFX9-NEXT:    v_mov_b32_e32 v0, 0
 ; GFX9-NEXT:    v_mov_b32_e32 v1, 2.0
@@ -2349,9 +2332,6 @@ define amdgpu_kernel void @test_call_external_void_func_v3f64_imm() #0 {
 ; GFX9-NEXT:    v_mov_b32_e32 v4, 0
 ; GFX9-NEXT:    v_mov_b32_e32 v5, 0x40200000
 ; GFX9-NEXT:    s_mov_b32 s32, 0
-; GFX9-NEXT:    s_getpc_b64 s[4:5]
-; GFX9-NEXT:    s_add_u32 s4, s4, external_void_func_v3f64@rel32@lo+4
-; GFX9-NEXT:    s_addc_u32 s5, s5, external_void_func_v3f64@rel32@hi+12
 ; GFX9-NEXT:    s_swappc_b64 s[30:31], s[4:5]
 ; GFX9-NEXT:    s_endpgm
 ;
@@ -2360,12 +2340,11 @@ define amdgpu_kernel void @test_call_external_void_func_v3f64_imm() #0 {
 ; GFX11-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, 2.0
 ; GFX11-NEXT:    v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v3, 0x40100000
 ; GFX11-NEXT:    v_dual_mov_b32 v4, 0 :: v_dual_mov_b32 v5, 0x40200000
-; GFX11-NEXT:    s_mov_b64 s[6:7], s[0:1]
-; GFX11-NEXT:    s_mov_b32 s32, 0
 ; GFX11-NEXT:    s_getpc_b64 s[2:3]
 ; GFX11-NEXT:    s_add_u32 s2, s2, external_void_func_v3f64@rel32@lo+4
 ; GFX11-NEXT:    s_addc_u32 s3, s3, external_void_func_v3f64@rel32@hi+12
-; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_mov_b64 s[6:7], s[0:1]
+; GFX11-NEXT:    s_mov_b32 s32, 0
 ; GFX11-NEXT:    s_swappc_b64 s[30:31], s[2:3]
 ; GFX11-NEXT:    s_endpgm
 ;
@@ -2374,8 +2353,11 @@ define amdgpu_kernel void @test_call_external_void_func_v3f64_imm() #0 {
 ; HSA-NEXT:    s_add_i32 s6, s6, s9
 ; HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s6, 8
 ; HSA-NEXT:    s_add_u32 s0, s0, s9
-; HSA-NEXT:    s_mov_b32 flat_scratch_lo, s7
 ; HSA-NEXT:    s_addc_u32 s1, s1, 0
+; HSA-NEXT:    s_mov_b32 flat_scratch_lo, s7
+; HSA-NEXT:    s_getpc_b64 s[8:9]
+; HSA-NEXT:    s_add_u32 s8, s8, external_void_func_v3f64@rel32@lo+4
+; HSA-NEXT:    s_addc_u32 s9, s9, external_void_func_v3f64@rel32@hi+12
 ; HSA-NEXT:    s_mov_b64 s[6:7], s[4:5]
 ; HSA-NEXT:    v_mov_b32_e32 v0, 0
 ; HSA-NEXT:    v_mov_b32_e32 v1, 2.0
@@ -2384,9 +2366,6 @@ define amdgpu_kernel void @test_call_external_void_func_v3f64_imm() #0 {
 ; HSA-NEXT:    v_mov_b32_e32 v4, 0
 ; HSA-NEXT:    v_mov_b32_e32 v5, 0x40200000
 ; HSA-NEXT:    s_mov_b32 s32, 0
-; HSA-NEXT:    s_getpc_b64 s[8:9]
-; HSA-NEXT:    s_add_u32 s8, s8, external_void_func_v3f64@rel32@lo+4
-; HSA-NEXT:    s_addc_u32 s9, s9, external_void_func_v3f64@rel32@hi+12
 ; HSA-NEXT:    s_swappc_b64 s[30:31], s[8:9]
 ; HSA-NEXT:    s_endpgm
   call void @external_void_func_v3f64(<3 x double> <double 2.0, double 4.0, double 8.0>)
@@ -2407,11 +2386,11 @@ define amdgpu_kernel void @test_call_external_void_func_v2i16() #0 {
 ; VI-NEXT:    s_addc_u32 s37, s37, 0
 ; VI-NEXT:    s_mov_b64 s[6:7], s[0:1]
 ; VI-NEXT:    s_mov_b64 s[0:1], s[36:37]
-; VI-NEXT:    s_mov_b64 s[2:3], s[38:39]
-; VI-NEXT:    s_mov_b32 s32, 0
 ; VI-NEXT:    s_getpc_b64 s[4:5]
 ; VI-NEXT:    s_add_u32 s4, s4, external_void_func_v2i16@rel32@lo+4
 ; VI-NEXT:    s_addc_u32 s5, s5, external_void_func_v2i16@rel32@hi+12
+; VI-NEXT:    s_mov_b64 s[2:3], s[38:39]
+; VI-NEXT:    s_mov_b32 s32, 0
 ; VI-NEXT:    s_swappc_b64 s[30:31], s[4:5]
 ; VI-NEXT:    s_endpgm
 ;
@@ -2428,11 +2407,11 @@ define amdgpu_kernel void @test_call_external_void_func_v2i16() #0 {
 ; CI-NEXT:    s_addc_u32 s37, s37, 0
 ; CI-NEXT:    s_mov_b64 s[6:7], s[0:1]
 ; CI-NEXT:    s_mov_b64 s[0:1], s[36:37]
-; CI-NEXT:    s_mov_b64 s[2:3], s[38:39]
-; CI-NEXT:    s_mov_b32 s32, 0
 ; CI-NEXT:    s_getpc_b64 s[4:5]
 ; CI-NEXT:    s_add_u32 s4, s4, external_void_func_v2i16@rel32@lo+4
 ; CI-NEXT:    s_addc_u32 s5, s5, external_void_func_v2i16@rel32@hi+12
+; CI-NEXT:    s_mov_b64 s[2:3], s[38:39]
+; CI-NEXT:    s_mov_b32 s32, 0
 ; CI-NEXT:    s_waitcnt vmcnt(0)
 ; CI-NEXT:    v_lshrrev_b32_e32 v1, 16, v0
 ; CI-NEXT:    s_swappc_b64 s[30:31], s[4:5]
@@ -2451,11 +2430,11 @@ define amdgpu_kernel void @test_call_external_void_func_v2i16() #0 {
 ; GFX9-NEXT:    s_addc_u32 s37, s37, 0
 ; GFX9-NEXT:    s_mov_b64 s[6:7], s[0:1]
 ; GFX9-NEXT:    s_mov_b64 s[0:1], s[36:37]
-; GFX9-NEXT:    s_mov_b64 s[2:3], s[38:39]
-; GFX9-NEXT:    s_mov_b32 s32, 0
 ; GFX9-NEXT:    s_getpc_b64 s[4:5]
 ; GFX9-NEXT:    s_add_u32 s4, s4, external_void_func_v2i16@rel32@lo+4
 ; GFX9-NEXT:    s_addc_u32 s5, s5, external_void_func_v2i16@rel32@hi+12
+; GFX9-NEXT:    s_mov_b64 s[2:3], s[38:39]
+; GFX9-NEXT:    s_mov_b32 s32, 0
 ; GFX9-NEXT:    s_swappc_b64 s[30:31], s[4:5]
 ; GFX9-NEXT:    s_endpgm
 ;
@@ -2465,11 +2444,10 @@ define amdgpu_kernel void @test_call_external_void_func_v2i16() #0 {
 ; GFX11-NEXT:    s_mov_b32 s2, -1
 ; GFX11-NEXT:    s_mov_b64 s[6:7], s[0:1]
 ; GFX11-NEXT:    buffer_load_b32 v0, off, s[0:3], 0
-; GFX11-NEXT:    s_mov_b32 s32, 0
 ; GFX11-NEXT:    s_getpc_b64 s[2:3]
 ; GFX11-NEXT:    s_add_u32 s2, s2, external_void_func_v2i16@rel32@lo+4
 ; GFX11-NEXT:    s_addc_u32 s3, s3, external_void_func_v2i16@rel32@hi+12
-; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_mov_b32 s32, 0
 ; GFX11-NEXT:    s_swappc_b64 s[30:31], s[2:3]
 ; GFX11-NEXT:    s_endpgm
 ;
@@ -2483,11 +2461,11 @@ define amdgpu_kernel void @test_call_external_void_func_v2i16() #0 {
 ; HSA-NEXT:    buffer_load_dword v0, off, s[4:7], 0
 ; HSA-NEXT:    s_add_u32 s0, s0, s9
 ; HSA-NEXT:    s_addc_u32 s1, s1, 0
-; HSA-NEXT:    s_mov_b64 s[6:7], s[4:5]
-; HSA-NEXT:    s_mov_b32 s32, 0
 ; HSA-NEXT:    s_getpc_b64 s[8:9]
 ; HSA-NEXT:    s_add_u32 s8, s8, external_void_func_v2i16@rel32@lo+4
 ; HSA-NEXT:    s_addc_u32 s9, s9, external_void_func_v2i16@rel32@hi+12
+; HSA-NEXT:    s_mov_b64 s[6:7], s[4:5]
+; HSA-NEXT:    s_mov_b32 s32, 0
 ; HSA-NEXT:    s_swappc_b64 s[30:31], s[8:9]
 ; HSA-NEXT:    s_endpgm
   %val = load <2 x i16>, ptr addrspace(1) undef
@@ -2509,11 +2487,11 @@ define amdgpu_kernel void @test_call_external_void_func_v3i16() #0 {
 ; VI-NEXT:    s_addc_u32 s37, s37, 0
 ; VI-NEXT:    s_mov_b64 s[6:7], s[0:1]
 ; VI-NEXT:    s_mov_b64 s[0:1], s[36:37]
-; VI-NEXT:    s_mov_b64 s[2:3], s[38:39]
-; VI-NEXT:    s_mov_b32 s32, 0
 ; VI-NEXT:    s_getpc_b64 s[4:5]
 ; VI-NEXT:    s_add_u32 s4, s4, external_void_func_v3i16@rel32@lo+4
 ; VI-NEXT:    s_addc_u32 s5, s5, external_void_func_v3i16@rel32@hi+12
+; VI-NEXT:    s_mov_b64 s[2:3], s[38:39]
+; VI-NEXT:    s_mov_b32 s32, 0
 ; VI-NEXT:    s_swappc_b64 s[30:31], s[4:5]
 ; VI-NEXT:    s_endpgm
 ;
@@ -2530,11 +2508,11 @@ define amdgpu_kernel void @test_call_external_void_func_v3i16() #0 {
 ; CI-NEXT:    s_addc_u32 s37, s37, 0
 ; CI-NEXT:    s_mov_b64 s[6:7], s[0:1]
 ; CI-NEXT:    s_mov_b64 s[0:1], s[36:37]
-; CI-NEXT:    s_mov_b64 s[2:3], s[38:39]
-; CI-NEXT:    s_mov_b32 s32, 0
 ; CI-NEXT:    s_getpc_b64 s[4:5]
 ; CI-NEXT:    s_add_u32 s4, s4, external_void_func_v3i16@rel32@lo+4
 ; CI-NEXT:    s_addc_u32 s5, s5, external_void_func_v3i16@rel32@hi+12
+; CI-NEXT:    s_mov_b64 s[2:3], s[38:39]
+; CI-NEXT:    s_mov_b32 s32, 0
 ; CI-NEXT:    s_waitcnt vmcnt(0)
 ; CI-NEXT:    v_alignbit_b32 v1, v3, v2, 16
 ; CI-NEXT:    v_mov_b32_e32 v0, v2
@@ -2555,11 +2533,11 @@ define amdgpu_kernel void @test_call_external_void_func_v3i16() #0 {
 ; GFX9-NEXT:    s_addc_u32 s37, s37, 0
 ; GFX9-NEXT:    s_mov_b64 s[6:7], s[0:1]
 ; GFX9-NEXT:    s_mov_b64 s[0:1], s[36:37]
-; GFX9-NEXT:    s_mov_b64 s[2:3], s[38:39]
-; GFX9-NEXT:    s_mov_b32 s32, 0
 ; GFX9-NEXT:    s_getpc_b64 s[4:5]
 ; GFX9-NEXT:    s_add_u32 s4, s4, external_void_func_v3i16@rel32@lo+4
 ; GFX9-NEXT:    s_addc_u32 s5, s5, external_void_func_v3i16@rel32@hi+12
+; GFX9-NEXT:    s_mov_b64 s[2:3], s[38:39]
+; GFX9-NEXT:    s_mov_b32 s32, 0
 ; GFX9-NEXT:    s_swappc_b64 s[30:31], s[4:5]
 ; GFX9-NEXT:    s_endpgm
 ;
@@ -2569,11 +2547,10 @@ define amdgpu_kernel void @test_call_external_void_func_v3i16() #0 {
 ; GFX11-NEXT:    s_mov_b32 s2, -1
 ; GFX11-NEXT:    s_mov_b64 s[6:7], s[0:1]
 ; GFX11-NEXT:    buffer_load_b64 v[0:1], off, s[0:3], 0
-; GFX11-NEXT:    s_mov_b32 s32, 0
 ; GFX11-NEXT:    s_getpc_b64 s[2:3]
 ; GFX11-NEXT:    s_add_u32 s2, s2, external_void_func_v3i16@rel32@lo+4
 ; GFX11-NEXT:    s_addc_u32 s3, s3, external_void_func_v3i16@rel32@hi+12
-; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_mov_b32 s32, 0
 ; GFX11-NEXT:    s_swappc_b64 s[30:31], s[2:3]
 ; GFX11-NEXT:    s_endpgm
 ;
@@ -2587,11 +2564,11 @@ define amdgpu_kernel void @test_call_external_void_func_v3i16() #0 {
 ; HSA-NEXT:    buffer_load_dwordx2 v[0:1], off, s[4:7], 0
 ; HSA-NEXT:    s_add_u32 s0, s0, s9
 ; HSA-NEXT:    s_addc_u32 s1, s1, 0
-; HSA-NEXT:    s_mov_b64 s[6:7], s[4:5]
-; HSA-NEXT:    s_mov_b32 s32, 0
 ; HSA-NEXT:    s_getpc_b64 s[8:9]
 ; HSA-NEXT:    s_add_u32 s8, s8, external_void_func_v3i16@rel32@lo+4
 ; HSA-NEXT:    s_addc_u32 s9, s9, external_void_func_v3i16@rel32@hi+12
+; HSA-NEXT:    s_mov_b64 s[6:7], s[4:5]
+; HSA-NEXT:    s_mov_b32 s32, 0
 ; HSA-NEXT:    s_swappc_b64 s[30:31], s[8:9]
 ; HSA-NEXT:    s_endpgm
   %val = load <3 x i16>, ptr addrspace(1) undef
@@ -2613,11 +2590,11 @@ define amdgpu_kernel void @test_call_external_void_func_v3f16() #0 {
 ; VI-NEXT:    s_addc_u32 s37, s37, 0
 ; VI-NEXT:    s_mov_b64 s[6:7], s[0:1]
 ; VI-NEXT:    s_mov_b64 s[0:1], s[36:37]
-; VI-NEXT:    s_mov_b64 s[2:3], s[38:39]
-; VI-NEXT:    s_mov_b32 s32, 0
 ; VI-NEXT:    s_getpc_b64 s[4:5]
 ; VI-NEXT:    s_add_u32 s4, s4, external_void_func_v3f16@rel32@lo+4
 ; VI-NEXT:    s_addc_u32 s5, s5, external_void_func_v3f16@rel32@hi+12
+; VI-NEXT:    s_mov_b64 s[2:3], s[38:39]
+; VI-NEXT:    s_mov_b32 s32, 0
 ; VI-NEXT:    s_swappc_b64 s[30:31], s[4:5]
 ; VI-NEXT:    s_endpgm
 ;
@@ -2634,11 +2611,11 @@ define amdgpu_kernel void @test_call_external_void_func_v3f16() #0 {
 ; CI-NEXT:    s_addc_u32 s37, s37, 0
 ; CI-NEXT:    s_mov_b64 s[6:7], s[0:1]
 ; CI-NEXT:    s_mov_b64 s[0:1], s[36:37]
-; CI-NEXT:    s_mov_b64 s[2:3], s[38:39]
-; CI-NEXT:    s_mov_b32 s32, 0
 ; CI-NEXT:    s_getpc_b64 s[4:5]
 ; CI-NEXT:    s_add_u32 s4, s4, external_void_func_v3f16@rel32@lo+4
 ; CI-NEXT:    s_addc_u32 s5, s5, external_void_func_v3f16@rel32@hi+12
+; CI-NEXT:    s_mov_b64 s[2:3], s[38:39]
+; CI-NEXT:    s_mov_b32 s32, 0
 ; CI-NEXT:    s_waitcnt vmcnt(0)
 ; CI-NEXT:    v_cvt_f32_f16_e32 v0, v1
 ; CI-NEXT:    v_lshrrev_b32_e32 v1, 16, v1
@@ -2660,11 +2637,11 @@ define amdgpu_kernel void @test_call_external_void_func_v3f16() #0 {
 ; GFX9-NEXT:    s_addc_u32 s37, s37, 0
 ; GFX9-NEXT:    s_mov_b64 s[6:7], s[0:1]
 ; GFX9-NEXT:    s_mov_b64 s[0:1], s[36:37]
-; GFX9-NEXT:    s_mov_b64 s[2:3], s[38:39]
-; GFX9-NEXT:    s_mov_b32 s32, 0
 ; GFX9-NEXT:    s_getpc_b64 s[4:5]
 ; GFX9-NEXT:    s_add_u32 s4, s4, external_void_func_v3f16@rel32@lo+4
 ; GFX9-NEXT:    s_addc_u32 s5, s5, external_void_func_v3f16@rel32@hi+12
+; GFX9-NEXT:    s_mov_b64 s[2:3], s[38:39]
+; GFX9-NEXT:    s_mov_b32 s32, 0
 ; GFX9-NEXT:    s_swappc_b64 s[30:31], s[4:5]
 ; GFX9-NEXT:    s_endpgm
 ;
@@ -2674,11 +2651,10 @@ define amdgpu_kernel void @test_call_external_void_func_v3f16() #0 {
 ; GFX11-NEXT:    s_mov_b32 s2, -1
 ; GFX11-NEXT:    s_mov_b64 s[6:7], s[0:1]
 ; GFX11-NEXT:    buffer_load_b64 v[0:1], off, s[0:3], 0
-; GFX11-NEXT:    s_mov_b32 s32, 0
 ; GFX11-NEXT:    s_getpc_b64 s[2:3]
 ; GFX11-NEXT:    s_add_u32 s2, s2, external_void_func_v3f16@rel32@lo+4
 ; GFX11-NEXT:    s_addc_u32 s3, s3, external_void_func_v3f16@rel32@hi+12
-; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_mov_b32 s32, 0
 ; GFX11-NEXT:    s_swappc_b64 s[30:31], s[2:3]
 ; GFX11-NEXT:    s_endpgm
 ;
@@ -2692,11 +2668,11 @@ define amdgpu_kernel void @test_call_external_void_func_v3f16() #0 {
 ; HSA-NEXT:    buffer_load_dwordx2 v[0:1], off, s[4:7], 0
 ; HSA-NEXT:    s_add_u32 s0, s0, s9
 ; HSA-NEXT:    s_addc_u32 s1, s1, 0
-; HSA-NEXT:    s_mov_b64 s[6:7], s[4:5]
-; HSA-NEXT:    s_mov_b32 s32, 0
 ; HSA-NEXT:    s_getpc_b64 s[8:9]
 ; HSA-NEXT:    s_add_u32 s8, s8, external_void_func_v3f16@rel32@lo+4
 ; HSA-NEXT:    s_addc_u32 s9, s9, external_void_func_v3f16@rel32@hi+12
+; HSA-NEXT:    s_mov_b64 s[6:7], s[4:5]
+; HSA-NEXT:    s_mov_b32 s32, 0
 ; HSA-NEXT:    s_swappc_b64 s[30:31], s[8:9]
 ; HSA-NEXT:    s_endpgm
   %val = load <3 x half>, ptr addrspace(1) undef
@@ -2715,13 +2691,13 @@ define amdgpu_kernel void @test_call_external_void_func_v3i16_imm() #0 {
 ; VI-NEXT:    s_addc_u32 s37, s37, 0
 ; VI-NEXT:    s_mov_b64 s[6:7], s[0:1]
 ; VI-NEXT:    s_mov_b64 s[0:1], s[36:37]
+; VI-NEXT:    s_getpc_b64 s[4:5]
+; VI-NEXT:    s_add_u32 s4, s4, external_void_func_v3i16@rel32@lo+4
+; VI-NEXT:    s_addc_u32 s5, s5, external_void_func_v3i16@rel32@hi+12
 ; VI-NEXT:    s_mov_b64 s[2:3], s[38:39]
 ; VI-NEXT:    v_mov_b32_e32 v0, 0x20001
 ; VI-NEXT:    v_mov_b32_e32 v1, 3
 ; VI-NEXT:    s_mov_b32 s32, 0
-; VI-NEXT:    s_getpc_b64 s[4:5]
-; VI-NEXT:    s_add_u32 s4, s4, external_void_func_v3i16@rel32@lo+4
-; VI-NEXT:    s_addc_u32 s5, s5, external_void_func_v3i16@rel32@hi+12
 ; VI-NEXT:    s_swappc_b64 s[30:31], s[4:5]
 ; VI-NEXT:    s_endpgm
 ;
@@ -2735,14 +2711,14 @@ define amdgpu_kernel void @test_call_external_void_func_v3i16_imm() #0 {
 ; CI-NEXT:    s_addc_u32 s37, s37, 0
 ; CI-NEXT:    s_mov_b64 s[6:7], s[0:1]
 ; CI-NEXT:    s_mov_b64 s[0:1], s[36:37]
+; CI-NEXT:    s_getpc_b64 s[4:5]
+; CI-NEXT:    s_add_u32 s4, s4, external_void_func_v3i16@rel32@lo+4
+; CI-NEXT:    s_addc_u32 s5, s5, external_void_func_v3i16@rel32@hi+12
 ; CI-NEXT:    s_mov_b64 s[2:3], s[38:39]
 ; CI-NEXT:    v_mov_b32_e32 v0, 1
 ; CI-NEXT:    v_mov_b32_e32 v1, 2
 ; CI-NEXT:    v_mov_b32_e32 v2, 3
 ; CI-NEXT:    s_mov_b32 s32, 0
-; CI-NEXT:    s_getpc_b64 s[4:5]
-; CI-NEXT:    s_add_u32 s4, s4, external_void_func_v3i16@rel32@lo+4
-; CI-NEXT:    s_addc_u32 s5, s5, external_void_func_v3i16@rel32@hi+12
 ; CI-NEXT:    s_swappc_b64 s[30:31], s[4:5]
 ; CI-NEXT:    s_endpgm
 ;
@@ -2756,25 +2732,24 @@ define amdgpu_kernel void @test_call_external_void_func_v3i16_imm() #0 {
 ; GFX9-NEXT:    s_addc_u32 s37, s37, 0
 ; GFX9-NEXT:    s_mov_b64 s[6:7], s[0:1]
 ; GFX9-NEXT:    s_mov_b64 s[0:1], s[36:37]
+; GFX9-NEXT:    s_getpc_b64 s[4:5]
+; GFX9-NEXT:    s_add_u32 s4, s4, external_void_func_v3i16@rel32@lo+4
+; GFX9-NEXT:    s_addc_u32 s5, s5, external_void_func_v3i16@rel32@hi+12
 ; GFX9-NEXT:    s_mov_b64 s[2:3], s[38:39]
 ; GFX9-NEXT:    v_mov_b32_e32 v0, 0x20001
 ; GFX9-NEXT:    v_mov_b32_e32 v1, 3
 ; GFX9-NEXT:    s_mov_b32 s32, 0
-; GFX9-NEXT:    s_getpc_b64 s[4:5]
-; GFX9-NEXT:    s_add_u32 s4, s4, external_void_func_v3i16@rel32@lo+4
-; GFX9-NEXT:    s_addc_u32 s5, s5, external_void_func_v3i16@rel32@hi+12
 ; GFX9-NEXT:    s_swappc_b64 s[30:31], s[4:5]
 ; GFX9-NEXT:    s_endpgm
 ;
 ; GFX11-LABEL: test_call_external_void_func_v3i16_imm:
 ; GFX11:       ; %bb.0:
 ; GFX11-NEXT:    v_dual_mov_b32 v0, 0x20001 :: v_dual_mov_b32 v1, 3
-; GFX11-NEXT:    s_mov_b64 s[6:7], s[0:1]
-; GFX11-NEXT:    s_mov_b32 s32, 0
 ; GFX11-NEXT:    s_getpc_b64 s[2:3]
 ; GFX11-NEXT:    s_add_u32 s2, s2, external_void_func_v3i16@rel32@lo+4
 ; GFX11-NEXT:    s_addc_u32 s3, s3, external_void_func_v3i16@rel32@hi+12
-; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_mov_b64 s[6:7], s[0:1]
+; GFX11-NEXT:    s_mov_b32 s32, 0
 ; GFX11-NEXT:    s_swappc_b64 s[30:31], s[2:3]
 ; GFX11-NEXT:    s_endpgm
 ;
@@ -2783,15 +2758,15 @@ define amdgpu_kernel void @test_call_external_void_func_v3i16_imm() #0 {
 ; HSA-NEXT:    s_add_i32 s6, s6, s9
 ; HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s6, 8
 ; HSA-NEXT:    s_add_u32 s0, s0, s9
-; HSA-NEXT:    s_mov_b32 flat_scratch_lo, s7
 ; HSA-NEXT:    s_addc_u32 s1, s1, 0
+; HSA-NEXT:    s_mov_b32 flat_scratch_lo, s7
+; HSA-NEXT:    s_getpc_b64 s[8:9]
+; HSA-NEXT:    s_add_u32 s8, s8, external_void_func_v3i16@rel32@lo+4
+; HSA-NEXT:    s_addc_u32 s9, s9, external_void_func_v3i16@rel32@hi+12
 ; HSA-NEXT:    s_mov_b64 s[6:7], s[4:5]
 ; HSA-NEXT:    v_mov_b32_e32 v0, 0x20001
 ; HSA-NEXT:    v_mov_b32_e32 v1, 3
 ; HSA-NEXT:    s_mov_b32 s32, 0
-; HSA-NEXT:    s_getpc_b64 s[8:9]
-; HSA-NEXT:    s_add_u32 s8, s8, external_void_func_v3i16@rel32@lo+4
-; HSA-NEXT:    s_addc_u32 s9, s9, external_void_func_v3i16@rel32@hi+12
 ; HSA-NEXT:    s_swappc_b64 s[30:31], s[8:9]
 ; HSA-NEXT:    s_endpgm
   call void @external_void_func_v3i16(<3 x i16> <i16 1, i16 2, i16 3>)
@@ -2809,13 +2784,13 @@ define amdgpu_kernel void @test_call_external_void_func_v3f16_imm() #0 {
 ; VI-NEXT:    s_addc_u32 s37, s37, 0
 ; VI-NEXT:    s_mov_b64 s[6:7], s[0:1]
 ; VI-NEXT:    s_mov_b64 s[0:1], s[36:37]
+; VI-NEXT:    s_getpc_b64 s[4:5]
+; VI-NEXT:    s_add_u32 s4, s4, external_void_func_v3f16@rel32@lo+4
+; VI-NEXT:    s_addc_u32 s5, s5, external_void_func_v3f16@rel32@hi+12
 ; VI-NEXT:    s_mov_b64 s[2:3], s[38:39]
 ; VI-NEXT:    v_mov_b32_e32 v0, 0x40003c00
 ; VI-NEXT:    v_mov_b32_e32 v1, 0x4400
 ; VI-NEXT:    s_mov_b32 s32, 0
-; VI-NEXT:    s_getpc_b64 s[4:5]
-; VI-NEXT:    s_add_u32 s4, s4, external_void_func_v3f16@rel32@lo+4
-; VI-NEXT:    s_addc_u32 s5, s5, external_void_func_v3f16@rel32@hi+12
 ; VI-NEXT:    s_swappc_b64 s[30:31], s[4:5]
 ; VI-NEXT:    s_endpgm
 ;
@@ -2829,14 +2804,14 @@ define amdgpu_kernel void @test_call_external_void_func_v3f16_imm() #0 {
 ; CI-NEXT:    s_addc_u32 s37, s37, 0
 ; CI-NEXT:    s_mov_b64 s[6:7], s[0:1]
 ; CI-NEXT:    s_mov_b64 s[0:1], s[36:37]
+; CI-NEXT:    s_getpc_b64 s[4:5]
+; CI-NEXT:    s_add_u32 s4, s4, external_void_func_v3f16@rel32@lo+4
+; CI-NEXT:    s_addc_u32 s5, s5, external_void_func_v3f16@rel32@hi+12
 ; CI-NEXT:    s_mov_b64 s[2:3], s[38:39]
 ; CI-NEXT:    v_mov_b32_e32 v0, 1.0
 ; CI-NEXT:    v_mov_b32_e32 v1, 2.0
 ; CI-NEXT:    v_mov_b32_e32 v2, 4.0
 ; CI-NEXT:    s_mov_b32 s32, 0
-; CI-NEXT:    s_getpc_b64 s[4:5]
-; CI-NEXT:    s_add_u32 s4, s4, external_void_func_v3f16@rel32@lo+4
-; CI-NEXT:    s_addc_u32 s5, s5, external_void_func_v3f16@rel32@hi+12
 ; CI-NEXT:    s_swappc_b64 s[30:31], s[4:5]
 ; CI-NEXT:    s_endpgm
 ;
@@ -2850,13 +2825,13 @@ define amdgpu_kernel void @test_call_external_void_func_v3f16_imm() #0 {
 ; GFX9-NEXT:    s_addc_u32 s37, s37, 0
 ; GFX9-NEXT:    s_mov_b64 s[6:7], s[0:1]
 ; GFX9-NEXT:    s_mov_b64 s[0:1], s[36:37]
+; GFX9-NEXT:    s_getpc_b64 s[4:5]
+; GFX9-NEXT:    s_add_u32 s4, s4, external_void_func_v3f16@rel32@lo+4
+; GFX9-NEXT:    s_addc_u32 s5, s5, external_void_func_v3f16@rel32@hi+12
 ; GFX9-NEXT:    s_mov_b64 s[2:3], s[38:39]
 ; GFX9-NEXT:    v_mov_b32_e32 v0, 0x40003c00
 ; GFX9-NEXT:    v_mov_b32_e32 v1, 0x4400
 ; GFX9-NEXT:    s_mov_b32 s32, 0
-; GFX9-NEXT:    s_getpc_b64 s[4:5]
-; GFX9-NEXT:    s_add_u32 s4, s4, external_void_func_v3f16@rel32@lo+4
-; GFX9-NEXT:    s_addc_u32 s5, s5, external_void_func_v3f16@rel32@hi+12
 ; GFX9-NEXT:    s_swappc_b64 s[30:31], s[4:5]
 ; GFX9-NEXT:    s_endpgm
 ;
@@ -2864,12 +2839,11 @@ define amdgpu_kernel void @test_call_external_void_func_v3f16_imm() #0 {
 ; GFX11:       ; %bb.0:
 ; GFX11-NEXT:    v_mov_b32_e32 v0, 0x40003c00
 ; GFX11-NEXT:    v_mov_b32_e32 v1, 0x4400
-; GFX11-NEXT:    s_mov_b64 s[6:7], s[0:1]
-; GFX11-NEXT:    s_mov_b32 s32, 0
 ; GFX11-NEXT:    s_getpc_b64 s[2:3]
 ; GFX11-NEXT:    s_add_u32 s2, s2, external_void_func_v3f16@rel32@lo+4
 ; GFX11-NEXT:    s_addc_u32 s3, s3, external_void_func_v3f16@rel32@hi+12
-; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_mov_b64 s[6:7], s[0:1]
+; GFX11-NEXT:    s_mov_b32 s32, 0
 ; GFX11-NEXT:    s_swappc_b64 s[30:31], s[2:3]
 ; GFX11-NEXT:    s_endpgm
 ;
@@ -2878,15 +2852,15 @@ define amdgpu_kernel void @test_call_external_void_func_v3f16_imm() #0 {
 ; HSA-NEXT:    s_add_i32 s6, s6, s9
 ; HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s6, 8
 ; HSA-NEXT:    s_add_u32 s0, s0, s9
-; HSA-NEXT:    s_mov_b32 flat_scratch_lo, s7
 ; HSA-NEXT:    s_addc_u32 s1, s1, 0
+; HSA-NEXT:    s_mov_b32 flat_scratch_lo, s7
+; HSA-NEXT:    s_getpc_b64 s[8:9]
+; HSA-NEXT:    s_add_u32 s8, s8, external_void_func_v3f16@rel32@lo+4
+; HSA-NEXT:    s_addc_u32 s9, s9, external_void_func_v3f16@rel32@hi+12
 ; HSA-NEXT:    s_mov_b64 s[6:7], s[4:5]
 ; HSA-NEXT:    v_mov_b32_e32 v0, 0x40003c00
 ; HSA-NEXT:    v_mov_b32_e32 v1, 0x4400
 ; HSA-NEXT:    s_mov_b32 s32, 0
-; HSA-NEXT:    s_getpc_b64 s[8:9]
-; HSA-NEXT:    s_add_u32 s8, s8, external_void_func_v3f16@rel32@lo+4
-; HSA-NEXT:    s_addc_u32 s9, s9, external_void_func_v3f16@rel32@hi+12
 ; HSA-NEXT:    s_swappc_b64 s[30:31], s[8:9]
 ; HSA-NEXT:    s_endpgm
   call void @external_void_func_v3f16(<3 x half> <half 1.0, half 2.0, half 4.0>)
@@ -2907,11 +2881,11 @@ define amdgpu_kernel void @test_call_external_void_func_v4i16() #0 {
 ; VI-NEXT:    s_addc_u32 s37, s37, 0
 ; VI-NEXT:    s_mov_b64 s[6:7], s[0:1]
 ; VI-NEXT:    s_mov_b64 s[0:1], s[36:37]
-; VI-NEXT:    s_mov_b64 s[2:3], s[38:39]
-; VI-NEXT:    s_mov_b32 s32, 0
 ; VI-NEXT:    s_getpc_b64 s[4:5]
 ; VI-NEXT:    s_add_u32 s4, s4, external_void_func_v4i16@rel32@lo+4
 ; VI-NEXT:    s_addc_u32 s5, s5, external_void_func_v4i16@rel32@hi+12
+; VI-NEXT:    s_mov_b64 s[2:3], s[38:39]
+; VI-NEXT:    s_mov_b32 s32, 0
 ; VI-NEXT:    s_swappc_b64 s[30:31], s[4:5]
 ; VI-NEXT:    s_endpgm
 ;
@@ -2928,11 +2902,11 @@ define amdgpu_kernel void @test_call_external_void_func_v4i16() #0 {
 ; CI-NEXT:    s_addc_u32 s37, s37, 0
 ; CI-NEXT:    s_mov_b64 s[6:7], s[0:1]
 ; CI-NEXT:    s_mov_b64 s[0:1], s[36:37]
-; CI-NEXT:    s_mov_b64 s[2:3], s[38:39]
-; CI-NEXT:    s_mov_b32 s32, 0
 ; CI-NEXT:    s_getpc_b64 s[4:5]
 ; CI-NEXT:    s_add_u32 s4, s4, external_void_func_v4i16@rel32@lo+4
 ; CI-NEXT:    s_addc_u32 s5, s5, external_void_func_v4i16@rel32@hi+12
+; CI-NEXT:    s_mov_b64 s[2:3], s[38:39]
+; CI-NEXT:    s_mov_b32 s32, 0
 ; CI-NEXT:    s_waitcnt vmcnt(0)
 ; CI-NEXT:    v_lshrrev_b32_e32 v4, 16, v0
 ; CI-NEXT:    v_lshrrev_b32_e32 v3, 16, v1
@@ -2954,11 +2928,11 @@ define amdgpu_kernel void @test_call_external_void_func_v4i16() #0 {
 ; GFX9-NEXT:    s_addc_u32 s37, s37, 0
 ; GFX9-NEXT:    s_mov_b64 s[6:7], s[0:1]
 ; GFX9-NEXT:    s_mov_b64 s[0:1], s[36:37]
-; GFX9-NEXT:    s_mov_b64 s[2:3], s[38:39]
-; GFX9-NEXT:    s_mov_b32 s32, 0
 ; GFX9-NEXT:    s_getpc_b64 s[4:5]
 ; GFX9-NEXT:    s_add_u32 s4, s4, external_void_func_v4i16@rel32@lo+4
 ; GFX9-NEXT:    s_addc_u32 s5, s5, external_void_func_v4i16@rel32@hi+12
+; GFX9-NEXT:    s_mov_b64 s[2:3], s[38:39]
+; GFX9-NEXT:    s_mov_b32 s32, 0
 ; GFX9-NEXT:    s_swappc_b64 s[30:31], s[4:5]
 ; GFX9-NEXT:    s_endpgm
 ;
@@ -2968,11 +2942,10 @@ define amdgpu_kernel void @test_call_external_void_func_v4i16() #0 {
 ; GFX11-NEXT:    s_mov_b32 s2, -1
 ; GFX11-NEXT:    s_mov_b64 s[6:7], s[0:1]
 ; GFX11-NEXT:    buffer_load_b64 v[0:1], off, s[0:3], 0
-; GFX11-NEXT:    s_mov_b32 s32, 0
 ; GFX11-NEXT:    s_getpc_b64 s[2:3]
 ; GFX11-NEXT:    s_add_u32 s2, s2, external_void_func_v4i16@rel32@lo+4
 ; GFX11-NEXT:    s_addc_u32 s3, s3, external_void_func_v4i16@rel32@hi+12
-; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_mov_b32 s32, 0
 ; GFX11-NEXT:    s_swappc_b64 s[30:31], s[2:3]
 ; GFX11-NEXT:    s_endpgm
 ;
@@ -2986,11 +2959,11 @@ define amdgpu_kernel void @test_call_external_void_func_v4i16() #0 {
 ; HSA-NEXT:    buffer_load_dwordx2 v[0:1], off, s[4:7], 0
 ; HSA-NEXT:    s_add_u32 s0, s0, s9
 ; HSA-NEXT:    s_addc_u32 s1, s1, 0
-; HSA-NEXT:    s_mov_b64 s[6:7], s[4:5]
-; HSA-NEXT:    s_mov_b32 s32, 0
 ; HSA-NEXT:    s_getpc_b64 s[8:9]
 ; HSA-NEXT:    s_add_u32 s8, s8, external_void_func_v4i16@rel32@lo+4
 ; HSA-NEXT:    s_addc_u32 s9, s9, external_void_func_v4i16@rel32@hi+12
+; HSA-NEXT:    s_mov_b64 s[6:7], s[4:5]
+; HSA-NEXT:    s_mov_b32 s32, 0
 ; HSA-NEXT:    s_swappc_b64 s[30:31], s[8:9]
 ; HSA-NEXT:    s_endpgm
   %val = load <4 x i16>, ptr addrspace(1) undef
@@ -3009,13 +2982,13 @@ define amdgpu_kernel void @test_call_external_void_func_v4i16_imm() #0 {
 ; VI-NEXT:    s_addc_u32 s37, s37, 0
 ; VI-NEXT:    s_mov_b64 s[6:7], s[0:1]
 ; VI-NEXT:    s_mov_b64 s[0:1], s[36:37]
+; VI-NEXT:    s_getpc_b64 s[4:5]
+; VI-NEXT:    s_add_u32 s4, s4, external_void_func_v4i16@rel32@lo+4
+; VI-NEXT:    s_addc_u32 s5, s5, external_void_func_v4i16@rel32@hi+12
 ; VI-NEXT:    s_mov_b64 s[2:3], s[38:39]
 ; VI-NEXT:    v_mov_b32_e32 v0, 0x20001
 ; VI-NEXT:    v_mov_b32_e32 v1, 0x40003
 ; VI-NEXT:    s_mov_b32 s32, 0
-; VI-NEXT:    s_getpc_b64 s[4:5]
-; VI-NEXT:    s_add_u32 s4, s4, external_void_func_v4i16@rel32@lo+4
-; VI-NEXT:    s_addc_u32 s5, s5, external_void_func_v4i16@rel32@hi+12
 ; VI-NEXT:    s_swappc_b64 s[30:31], s[4:5]
 ; VI-NEXT:    s_endpgm
 ;
@@ -3029,15 +3002,15 @@ define amdgpu_kernel void @test_call_external_void_func_v4i16_imm() #0 {
 ; CI-NEXT:    s_addc_u32 s37, s37, 0
 ; CI-NEXT:    s_mov_b64 s[6:7], s[0:1]
 ; CI-NEXT:    s_mov_b64 s[0:1], s[36:37]
+; CI-NEXT:    s_getpc_b64 s[4:5]
+; CI-NEXT:    s_add_u32 s4, s4, external_void_func_v4i16@rel32@lo+4
+; CI-NEXT:    s_addc_u32 s5, s5, external_void_func_v4i16@rel32@hi+12
 ; CI-NEXT:    s_mov_b64 s[2:3], s[38:39]
 ; CI-NEXT:    v_mov_b32_e32 v0, 1
 ; CI-NEXT:    v_mov_b32_e32 v1, 2
 ; CI-NEXT:    v_mov_b32_e32 v2, 3
 ; CI-NEXT:    v_mov_b32_e32 v3, 4
 ; CI-NEXT:    s_mov_b32 s32, 0
-; CI-NEXT:    s_getpc_b64 s[4:5]
-; CI-NEXT:    s_add_u32 s4, s4, external_void_func_v4i16@rel32@lo+4
-; CI-NEXT:    s_addc_u32 s5, s5, external_void_func_v4i16@rel32@hi+12
 ; CI-NEXT:    s_swappc_b64 s[30:31], s[4:5]
 ; CI-NEXT:    s_endpgm
 ;
@@ -3051,13 +3024,13 @@ define amdgpu_kernel void @test_call_external_void_func_v4i16_imm() #0 {
 ; GFX9-NEXT:    s_addc_u32 s37, s37, 0
 ; GFX9-NEXT:    s_mov_b64 s[6:7], s[0:1]
 ; GFX9-NEXT:    s_mov_b64 s[0:1], s[36:37]
+; GFX9-NEXT:    s_getpc_b64 s[4:5]
+; GFX9-NEXT:    s_add_u32 s4, s4, external_void_func_v4i16@rel32@lo+4
+; GFX9-NEXT:    s_addc_u32 s5, s5, external_void_func_v4i16@rel32@hi+12
 ; GFX9-NEXT:    s_mov_b64 s[2:3], s[38:39]
 ; GFX9-NEXT:    v_mov_b32_e32 v0, 0x20001
 ; GFX9-NEXT:    v_mov_b32_e32 v1, 0x40003
 ; GFX9-NEXT:    s_mov_b32 s32, 0
-; GFX9-NEXT:    s_getpc_b64 s[4:5]
-; GFX9-NEXT:    s_add_u32 s4, s4, external_void_func_v4i16@rel32@lo+4
-; GFX9-NEXT:    s_addc_u32 s5, s5, external_void_func_v4i16@rel32@hi+12
 ; GFX9-NEXT:    s_swappc_b64 s[30:31], s[4:5]
 ; GFX9-NEXT:    s_endpgm
 ;
@@ -3065,12 +3038,11 @@ define amdgpu_kernel void @test_call_external_void_func_v4i16_imm() #0 {
 ; GFX11:       ; %bb.0:
 ; GFX11-NEXT:    v_mov_b32_e32 v0, 0x20001
 ; GFX11-NEXT:    v_mov_b32_e32 v1, 0x40003
-; GFX11-NEXT:    s_mov_b64 s[6:7], s[0:1]
-; GFX11-NEXT:    s_mov_b32 s32, 0
 ; GFX11-NEXT:    s_getpc_b64 s[2:3]
 ; GFX11-NEXT:    s_add_u32 s2, s2, external_void_func_v4i16@rel32@lo+4
 ; GFX11-NEXT:    s_addc_u32 s3, s3, external_void_func_v4i16@rel32@hi+12
-; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_mov_b64 s[6:7], s[0:1]
+; GFX11-NEXT:    s_mov_b32 s32, 0
 ; GFX11-NEXT:    s_swappc_b64 s[30:31], s[2:3]
 ; GFX11-NEXT:    s_endpgm
 ;
@@ -3079,15 +3051,15 @@ define amdgpu_kernel void @test_call_external_void_func_v4i16_imm() #0 {
 ; HSA-NEXT:    s_add_i32 s6, s6, s9
 ; HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s6, 8
 ; HSA-NEXT:    s_add_u32 s0, s0, s9
-; HSA-NEXT:    s_mov_b32 flat_scratch_lo, s7
 ; HSA-NEXT:    s_addc_u32 s1, s1, 0
+; HSA-NEXT:    s_mov_b32 flat_scratch_lo, s7
+; HSA-NEXT:    s_getpc_b64 s[8:9]
+; HSA-NEXT:    s_add_u32 s8, s8, external_void_func_v4i16@rel32@lo+4
+; HSA-NEXT:    s_addc_u32 s9, s9, external_void_func_v4i16@rel32@hi+12
 ; HSA-NEXT:    s_mov_b64 s[6:7], s[4:5]
 ; HSA-NEXT:    v_mov_b32_e32 v0, 0x20001
 ; HSA-NEXT:    v_mov_b32_e32 v1, 0x40003
 ; HSA-NEXT:    s_mov_b32 s32, 0
-; HSA-NEXT:    s_getpc_b64 s[8:9]
-; HSA-NEXT:    s_add_u32 s8, s8, external_void_func_v4i16@rel32@lo+4
-; HSA-NEXT:    s_addc_u32 s9, s9, external_void_func_v4i16@rel32@hi+12
 ; HSA-NEXT:    s_swappc_b64 s[30:31], s[8:9]
 ; HSA-NEXT:    s_endpgm
   call void @external_void_func_v4i16(<4 x i16> <i16 1, i16 2, i16 3, i16 4>)
@@ -3108,11 +3080,11 @@ define amdgpu_kernel void @test_call_external_void_func_v2f16() #0 {
 ; VI-NEXT:    s_addc_u32 s37, s37, 0
 ; VI-NEXT:    s_mov_b64 s[6:7], s[0:1]
 ; VI-NEXT:    s_mov_b64 s[0:1], s[36:37]
-; VI-NEXT:    s_mov_b64 s[2:3], s[38:39]
-; VI-NEXT:    s_mov_b32 s32, 0
 ; VI-NEXT:    s_getpc_b64 s[4:5]
 ; VI-NEXT:    s_add_u32 s4, s4, external_void_func_v2f16@rel32@lo+4
 ; VI-NEXT:    s_addc_u32 s5, s5, external_void_func_v2f16@rel32@hi+12
+; VI-NEXT:    s_mov_b64 s[2:3], s[38:39]
+; VI-NEXT:    s_mov_b32 s32, 0
 ; VI-NEXT:    s_swappc_b64 s[30:31], s[4:5]
 ; VI-NEXT:    s_endpgm
 ;
@@ -3129,11 +3101,11 @@ define amdgpu_kernel void @test_call_external_void_func_v2f16() #0 {
 ; CI-NEXT:    s_addc_u32 s37, s37, 0
 ; CI-NEXT:    s_mov_b64 s[6:7], s[0:1]
 ; CI-NEXT:    s_mov_b64 s[0:1], s[36:37]
-; CI-NEXT:    s_mov_b64 s[2:3], s[38:39]
-; CI-NEXT:    s_mov_b32 s32, 0
 ; CI-NEXT:    s_getpc_b64 s[4:5]
 ; CI-NEXT:    s_add_u32 s4, s4, external_void_func_v2f16@rel32@lo+4
 ; CI-NEXT:    s_addc_u32 s5, s5, external_void_func_v2f16@rel32@hi+12
+; CI-NEXT:    s_mov_b64 s[2:3], s[38:39]
+; CI-NEXT:    s_mov_b32 s32, 0
 ; CI-NEXT:    s_waitcnt vmcnt(0)
 ; CI-NEXT:    v_cvt_f32_f16_e32 v0, v1
 ; CI-NEXT:    v_lshrrev_b32_e32 v1, 16, v1
@@ -3154,11 +3126,11 @@ define amdgpu_kernel void @test_call_external_void_func_v2f16() #0 {
 ; GFX9-NEXT:    s_addc_u32 s37, s37, 0
 ; GFX9-NEXT:    s_mov_b64 s[6:7], s[0:1]
 ; GFX9-NEXT:    s_mov_b64 s[0:1], s[36:37]
-; GFX9-NEXT:    s_mov_b64 s[2:3], s[38:39]
-; GFX9-NEXT:    s_mov_b32 s32, 0
 ; GFX9-NEXT:    s_getpc_b64 s[4:5]
 ; GFX9-NEXT:    s_add_u32 s4, s4, external_void_func_v2f16@rel32@lo+4
 ; GFX9-NEXT:    s_addc_u32 s5, s5, external_void_func_v2f16@rel32@hi+12
+; GFX9-NEXT:    s_mov_b64 s[2:3], s[38:39]
+; GFX9-NEXT:    s_mov_b32 s32, 0
 ; GFX9-NEXT:    s_swappc_b64 s[30:31], s[4:5]
 ; GFX9-NEXT:    s_endpgm
 ;
@@ -3168,11 +3140,10 @@ define amdgpu_kernel void @test_call_external_void_func_v2f16() #0 {
 ; GFX11-NEXT:    s_mov_b32 s2, -1
 ; GFX11-NEXT:    s_mov_b64 s[6:7], s[0:1]
 ; GFX11-NEXT:    buffer_load_b32 v0, off, s[0:3], 0
-; GFX11-NEXT:    s_mov_b32 s32, 0
 ; GFX11-NEXT:    s_getpc_b64 s[2:3]
 ; GFX11-NEXT:    s_add_u32 s2, s2, external_void_func_v2f16@rel32@lo+4
 ; GFX11-NEXT:    s_addc_u32 s3, s3, external_void_func_v2f16@rel32@hi+12
-; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_mov_b32 s32, 0
 ; GFX11-NEXT:    s_swappc_b64 s[30:31], s[2:3]
 ; GFX11-NEXT:    s_endpgm
 ;
@@ -3186,11 +3157,11 @@ define amdgpu_kernel void @test_call_external_void_func_v2f16() #0 {
 ; HSA-NEXT:    buffer_load_dword v0, off, s[4:7], 0
 ; HSA-NEXT:    s_add_u32 s0, s0, s9
 ; HSA-NEXT:    s_addc_u32 s1, s1, 0
-; HSA-NEXT:    s_mov_b64 s[6:7], s[4:5]
-; HSA-NEXT:    s_mov_b32 s32, 0
 ; HSA-NEXT:    s_getpc_b64 s[8:9]
 ; HSA-NEXT:    s_add_u32 s8, s8, external_void_func_v2f16@rel32@lo+4
 ; HSA-NEXT:    s_addc_u32 s9, s9, external_void_func_v2f16@rel32@hi+12
+; HSA-NEXT:    s_mov_b64 s[6:7], s[4:5]
+; HSA-NEXT:    s_mov_b32 s32, 0
 ; HSA-NEXT:    s_swappc_b64 s[30:31], s[8:9]
 ; HSA-NEXT:    s_endpgm
   %val = load <2 x half>, ptr addrspace(1) undef
@@ -3212,11 +3183,11 @@ define amdgpu_kernel void @test_call_external_void_func_v2i32() #0 {
 ; VI-NEXT:    s_addc_u32 s37, s37, 0
 ; VI-NEXT:    s_mov_b64 s[6:7], s[0:1]
 ; VI-NEXT:    s_mov_b64 s[0:1], s[36:37]
-; VI-NEXT:    s_mov_b64 s[2:3], s[38:39]
-; VI-NEXT:    s_mov_b32 s32, 0
 ; VI-NEXT:    s_getpc_b64 s[4:5]
 ; VI-NEXT:    s_add_u32 s4, s4, external_void_func_v2i32@rel32@lo+4
 ; VI-NEXT:    s_addc_u32 s5, s5, external_void_func_v2i32@rel32@hi+12
+; VI-NEXT:    s_mov_b64 s[2:3], s[38:39]
+; VI-NEXT:    s_mov_b32 s32, 0
 ; VI-NEXT:    s_swappc_b64 s[30:31], s[4:5]
 ; VI-NEXT:    s_endpgm
 ;
@@ -3233,11 +3204,11 @@ define amdgpu_kernel void @test_call_external_void_func_v2i32() #0 {
 ; CI-NEXT:    s_addc_u32 s37, s37, 0
 ; CI-NEXT:    s_mov_b64 s[6:7], s[0:1]
 ; CI-NEXT:    s_mov_b64 s[0:1], s[36:37]
-; CI-NEXT:    s_mov_b64 s[2:3], s[38:39]
-; CI-NEXT:    s_mov_b32 s32, 0
 ; CI-NEXT:    s_getpc_b64 s[4:5]
 ; CI-NEXT:    s_add_u32 s4, s4, external_void_func_v2i32@rel32@lo+4
 ; CI-NEXT:    s_addc_u32 s5, s5, external_void_func_v2i32@rel32@hi+12
+; CI-NEXT:    s_mov_b64 s[2:3], s[38:39]
+; CI-NEXT:    s_mov_b32 s32, 0
 ; CI-NEXT:    s_swappc_b64 s[30:31], s[4:5]
 ; CI-NEXT:    s_endpgm
 ;
@@ -3254,11 +3225,11 @@ define amdgpu_kernel void @test_call_external_void_func_v2i32() #0 {
 ; GFX9-NEXT:    s_addc_u32 s37, s37, 0
 ; GFX9-NEXT:    s_mov_b64 s[6:7], s[0:1]
 ; GFX9-NEXT:    s_mov_b64 s[0:1], s[36:37]
-; GFX9-NEXT:    s_mov_b64 s[2:3], s[38:39]
-; GFX9-NEXT:    s_mov_b32 s32, 0
 ; GFX9-NEXT:    s_getpc_b64 s[4:5]
 ; GFX9-NEXT:    s_add_u32 s4, s4, external_void_func_v2i32@rel32@lo+4
 ; GFX9-NEXT:    s_addc_u32 s5, s5, external_void_func_v2i32@rel32@hi+12
+; GFX9-NEXT:    s_mov_b64 s[2:3], s[38:39]
+; GFX9-NEXT:    s_mov_b32 s32, 0
 ; GFX9-NEXT:    s_swappc_b64 s[30:31], s[4:5]
 ; GFX9-NEXT:    s_endpgm
 ;
@@ -3268,11 +3239,10 @@ define amdgpu_kernel void @test_call_external_void_func_v2i32() #0 {
 ; GFX11-NEXT:    s_mov_b32 s2, -1
 ; GFX11-NEXT:    s_mov_b64 s[6:7], s[0:1]
 ; GFX11-NEXT:    buffer_load_b64 v[0:1], off, s[0:3], 0
-; GFX11-NEXT:    s_mov_b32 s32, 0
 ; GFX11-NEXT:    s_getpc_b64 s[2:3]
 ; GFX11-NEXT:    s_add_u32 s2, s2, external_void_func_v2i32@rel32@lo+4
 ; GFX11-NEXT:    s_addc_u32 s3, s3, external_void_func_v2i32@rel32@hi+12
-; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_mov_b32 s32, 0
 ; GFX11-NEXT:    s_swappc_b64 s[30:31], s[2:3]
 ; GFX11-NEXT:    s_endpgm
 ;
@@ -3286,11 +3256,11 @@ define amdgpu_kernel void @test_call_external_void_func_v2i32() #0 {
 ; HSA-NEXT:    buffer_load_dwordx2 v[0:1], off, s[4:7], 0
 ; HSA-NEXT:    s_add_u32 s0, s0, s9
 ; HSA-NEXT:    s_addc_u32 s1, s1, 0
-; HSA-NEXT:    s_mov_b64 s[6:7], s[4:5]
-; HSA-NEXT:    s_mov_b32 s32, 0
 ; HSA-NEXT:    s_getpc_b64 s[8:9]
 ; HSA-NEXT:    s_add_u32 s8, s8, external_void_func_v2i32@rel32@lo+4
 ; HSA-NEXT:    s_addc_u32 s9, s9, external_void_func_v2i32@rel32@hi+12
+; HSA-NEXT:    s_mov_b64 s[6:7], s[4:5]
+; HSA-NEXT:    s_mov_b32 s32, 0
 ; HSA-NEXT:    s_swappc_b64 s[30:31], s[8:9]
 ; HSA-NEXT:    s_endpgm
   %val = load <2 x i32>, ptr addrspace(1) undef
@@ -3309,13 +3279,13 @@ define amdgpu_kernel void @test_call_external_void_func_v2i32_imm() #0 {
 ; VI-NEXT:    s_addc_u32 s37, s37, 0
 ; VI-NEXT:    s_mov_b64 s[6:7], s[0:1]
 ; VI-NEXT:    s_mov_b64 s[0:1], s[36:37]
+; VI-NEXT:    s_getpc_b64 s[4:5]
+; VI-NEXT:    s_add_u32 s4, s4, external_void_func_v2i32@rel32@lo+4
+; VI-NEXT:    s_addc_u32 s5, s5, external_void_func_v2i32@rel32@hi+12
 ; VI-NEXT:    s_mov_b64 s[2:3], s[38:39]
 ; VI-NEXT:    v_mov_b32_e32 v0, 1
 ; VI-NEXT:    v_mov_b32_e32 v1, 2
 ; VI-NEXT:    s_mov_b32 s32, 0
-; VI-NEXT:    s_getpc_b64 s[4:5]
-; VI-NEXT:    s_add_u32 s4, s4, external_void_func_v2i32@rel32@lo+4
-; VI-NEXT:    s_addc_u32 s5, s5, external_void_func_v2i32@rel32@hi+12
 ; VI-NEXT:    s_swappc_b64 s[30:31], s[4:5]
 ; VI-NEXT:    s_endpgm
 ;
@@ -3329,13 +3299,13 @@ define amdgpu_kernel void @test_call_external_void_func_v2i32_imm() #0 {
 ; CI-NEXT:    s_addc_u32 s37, s37, 0
 ; CI-NEXT:    s_mov_b64 s[6:7], s[0:1]
 ; CI-NEXT:    s_mov_b64 s[0:1], s[36:37]
+; CI-NEXT:    s_getpc_b64 s[4:5]
+; CI-NEXT:    s_add_u32 s4, s4, external_void_func_v2i32@rel32@lo+4
+; CI-NEXT:    s_addc_u32 s5, s5, external_void_func_v2i32@rel32@hi+12
 ; CI-NEXT:    s_mov_b64 s[2:3], s[38:39]
 ; CI-NEXT:    v_mov_b32_e32 v0, 1
 ; CI-NEXT:    v_mov_b32_e32 v1, 2
 ; CI-NEXT:    s_mov_b32 s32, 0
-; CI-NEXT:    s_getpc_b64 s[4:5]
-; CI-NEXT:    s_add_u32 s4, s4, external_void_func_v2i32@rel32@lo+4
-; CI-NEXT:    s_addc_u32 s5, s5, external_void_func_v2i32@rel32@hi+12
 ; CI-NEXT:    s_swappc_b64 s[30:31], s[4:5]
 ; CI-NEXT:    s_endpgm
 ;
@@ -3349,25 +3319,24 @@ define amdgpu_kernel void @test_call_external_void_func_v2i32_imm() #0 {
 ; GFX9-NEXT:    s_addc_u32 s37, s37, 0
 ; GFX9-NEXT:    s_mov_b64 s[6:7], s[0:1]
 ; GFX9-NEXT:    s_mov_b64 s[0:1], s[36:37]
+; GFX9-NEXT:    s_getpc_b64 s[4:5]
+; GFX9-NEXT:    s_add_u32 s4, s4, external_void_func_v2i32@rel32@lo+4
+; GFX9-NEXT:    s_addc_u32 s5, s5, external_void_func_v2i32@rel32@hi+12
 ; GFX9-NEXT:    s_mov_b64 s[2:3], s[38:39]
 ; GFX9-NEXT:    v_mov_b32_e32 v0, 1
 ; GFX9-NEXT:    v_mov_b32_e32 v1, 2
 ; GFX9-NEXT:    s_mov_b32 s32, 0
-; GFX9-NEXT:    s_getpc_b64 s[4:5]
-; GFX9-NEXT:    s_add_u32 s4, s4, external_void_func_v2i32@rel32@lo+4
-; GFX9-NEXT:    s_addc_u32 s5, s5, external_void_func_v2i32@rel32@hi+12
 ; GFX9-NEXT:    s_swappc_b64 s[30:31], s[4:5]
 ; GFX9-NEXT:    s_endpgm
 ;
 ; GFX11-LABEL: test_call_external_void_func_v2i32_imm:
 ; GFX11:       ; %bb.0:
 ; GFX11-NEXT:    v_dual_mov_b32 v0, 1 :: v_dual_mov_b32 v1, 2
-; GFX11-NEXT:    s_mov_b64 s[6:7], s[0:1]
-; GFX11-NEXT:    s_mov_b32 s32, 0
 ; GFX11-NEXT:    s_getpc_b64 s[2:3]
 ; GFX11-NEXT:    s_add_u32 s2, s2, external_void_func_v2i32@rel32@lo+4
 ; GFX11-NEXT:    s_addc_u32 s3, s3, external_void_func_v2i32@rel32@hi+12
-; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_mov_b64 s[6:7], s[0:1]
+; GFX11-NEXT:    s_mov_b32 s32, 0
 ; GFX11-NEXT:    s_swappc_b64 s[30:31], s[2:3]
 ; GFX11-NEXT:    s_endpgm
 ;
@@ -3376,15 +3345,15 @@ define amdgpu_kernel void @test_call_external_void_func_v2i32_imm() #0 {
 ; HSA-NEXT:    s_add_i32 s6, s6, s9
 ; HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s6, 8
 ; HSA-NEXT:    s_add_u32 s0, s0, s9
-; HSA-NEXT:    s_mov_b32 flat_scratch_lo, s7
 ; HSA-NEXT:    s_addc_u32 s1, s1, 0
+; HSA-NEXT:    s_mov_b32 flat_scratch_lo, s7
+; HSA-NEXT:    s_getpc_b64 s[8:9]
+; HSA-NEXT:    s_add_u32 s8, s8, external_void_func_v2i32@rel32@lo+4
+; HSA-NEXT:    s_addc_u32 s9, s9, external_void_func_v2i32@rel32@hi+12
 ; HSA-NEXT:    s_mov_b64 s[6:7], s[4:5]
 ; HSA-NEXT:    v_mov_b32_e32 v0, 1
 ; HSA-NEXT:    v_mov_b32_e32 v1, 2
 ; HSA-NEXT:    s_mov_b32 s32, 0
-; HSA-NEXT:    s_getpc_b64 s[8:9]
-; HSA-NEXT:    s_add_u32 s8, s8, external_void_func_v2i32@rel32@lo+4
-; HSA-NEXT:    s_addc_u32 s9, s9, external_void_func_v2i32@rel32@hi+12
 ; HSA-NEXT:    s_swappc_b64 s[30:31], s[8:9]
 ; HSA-NEXT:    s_endpgm
   call void @external_void_func_v2i32(<2 x i32> <i32 1, i32 2>)
@@ -3402,14 +3371,14 @@ define amdgpu_kernel void @test_call_external_void_func_v3i32_imm(i32) #0 {
 ; VI-NEXT:    s_addc_u32 s37, s37, 0
 ; VI-NEXT:    s_mov_b64 s[6:7], s[0:1]
 ; VI-NEXT:    s_mov_b64 s[0:1], s[36:37]
+; VI-NEXT:    s_getpc_b64 s[4:5]
+; VI-NEXT:    s_add_u32 s4, s4, external_void_func_v3i32@rel32@lo+4
+; VI-NEXT:    s_addc_u32 s5, s5, external_void_func_v3i32@rel32@hi+12
 ; VI-NEXT:    s_mov_b64 s[2:3], s[38:39]
 ; VI-NEXT:    v_mov_b32_e32 v0, 3
 ; VI-NEXT:    v_mov_b32_e32 v1, 4
 ; VI-NEXT:    v_mov_b32_e32 v2, 5
 ; VI-NEXT:    s_mov_b32 s32, 0
-; VI-NEXT:    s_getpc_b64 s[4:5]
-; VI-NEXT:    s_add_u32 s4, s4, external_void_func_v3i32@rel32@lo+4
-; VI-NEXT:    s_addc_u32 s5, s5, external_void_func_v3i32@rel32@hi+12
 ; VI-NEXT:    s_swappc_b64 s[30:31], s[4:5]
 ; VI-NEXT:    s_endpgm
 ;
@@ -3423,14 +3392,14 @@ define amdgpu_kernel void @test_call_external_void_func_v3i32_imm(i32) #0 {
 ; CI-NEXT:    s_addc_u32 s37, s37, 0
 ; CI-NEXT:    s_mov_b64 s[6:7], s[0:1]
 ; CI-NEXT:    s_mov_b64 s[0:1], s[36:37]
+; CI-NEXT:    s_getpc_b64 s[4:5]
+; CI-NEXT:    s_add_u32 s4, s4, external_void_func_v3i32@rel32@lo+4
+; CI-NEXT:    s_addc_u32 s5, s5, external_void_func_v3i32@rel32@hi+12
 ; CI-NEXT:    s_mov_b64 s[2:3], s[38:39]
 ; CI-NEXT:    v_mov_b32_e32 v0, 3
 ; CI-NEXT:    v_mov_b32_e32 v1, 4
 ; CI-NEXT:    v_mov_b32_e32 v2, 5
 ; CI-NEXT:    s_mov_b32 s32, 0
-; CI-NEXT:    s_getpc_b64 s[4:5]
-; CI-NEXT:    s_add_u32 s4, s4, external_void_func_v3i32@rel32@lo+4
-; CI-NEXT:    s_addc_u32 s5, s5, external_void_func_v3i32@rel32@hi+12
 ; CI-NEXT:    s_swappc_b64 s[30:31], s[4:5]
 ; CI-NEXT:    s_endpgm
 ;
@@ -3444,14 +3413,14 @@ define amdgpu_kernel void @test_call_external_void_func_v3i32_imm(i32) #0 {
 ; GFX9-NEXT:    s_addc_u32 s37, s37, 0
 ; GFX9-NEXT:    s_mov_b64 s[6:7], s[0:1]
 ; GFX9-NEXT:    s_mov_b64 s[0:1], s[36:37]
+; GFX9-NEXT:    s_getpc_b64 s[4:5]
+; GFX9-NEXT:    s_add_u32 s4, s4, external_void_func_v3i32@rel32@lo+4
+; GFX9-NEXT:    s_addc_u32 s5, s5, external_void_func_v3i32@rel32@hi+12
 ; GFX9-NEXT:    s_mov_b64 s[2:3], s[38:39]
 ; GFX9-NEXT:    v_mov_b32_e32 v0, 3
 ; GFX9-NEXT:    v_mov_b32_e32 v1, 4
 ; GFX9-NEXT:    v_mov_b32_e32 v2, 5
 ; GFX9-NEXT:    s_mov_b32 s32, 0
-; GFX9-NEXT:    s_getpc_b64 s[4:5]
-; GFX9-NEXT:    s_add_u32 s4, s4, external_void_func_v3i32@rel32@lo+4
-; GFX9-NEXT:    s_addc_u32 s5, s5, external_void_func_v3i32@rel32@hi+12
 ; GFX9-NEXT:    s_swappc_b64 s[30:31], s[4:5]
 ; GFX9-NEXT:    s_endpgm
 ;
@@ -3459,12 +3428,11 @@ define amdgpu_kernel void @test_call_external_void_func_v3i32_imm(i32) #0 {
 ; GFX11:       ; %bb.0:
 ; GFX11-NEXT:    v_dual_mov_b32 v0, 3 :: v_dual_mov_b32 v1, 4
 ; GFX11-NEXT:    v_mov_b32_e32 v2, 5
-; GFX11-NEXT:    s_mov_b64 s[6:7], s[0:1]
-; GFX11-NEXT:    s_mov_b32 s32, 0
 ; GFX11-NEXT:    s_getpc_b64 s[2:3]
 ; GFX11-NEXT:    s_add_u32 s2, s2, external_void_func_v3i32@rel32@lo+4
 ; GFX11-NEXT:    s_addc_u32 s3, s3, external_void_func_v3i32@rel32@hi+12
-; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_mov_b64 s[6:7], s[0:1]
+; GFX11-NEXT:    s_mov_b32 s32, 0
 ; GFX11-NEXT:    s_swappc_b64 s[30:31], s[2:3]
 ; GFX11-NEXT:    s_endpgm
 ;
@@ -3474,15 +3442,15 @@ define amdgpu_kernel void @test_call_external_void_func_v3i32_imm(i32) #0 {
 ; HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s8, 8
 ; HSA-NEXT:    s_add_u32 s0, s0, s11
 ; HSA-NEXT:    s_addc_u32 s1, s1, 0
+; HSA-NEXT:    s_mov_b32 flat_scratch_lo, s9
+; HSA-NEXT:    s_getpc_b64 s[8:9]
+; HSA-NEXT:    s_add_u32 s8, s8, external_void_func_v3i32@rel32@lo+4
+; HSA-NEXT:    s_addc_u32 s9, s9, external_void_func_v3i32@rel32@hi+12
 ; HSA-NEXT:    s_mov_b64 s[6:7], s[4:5]
 ; HSA-NEXT:    v_mov_b32_e32 v0, 3
 ; HSA-NEXT:    v_mov_b32_e32 v1, 4
 ; HSA-NEXT:    v_mov_b32_e32 v2, 5
 ; HSA-NEXT:    s_mov_b32 s32, 0
-; HSA-NEXT:    s_mov_b32 flat_scratch_lo, s9
-; HSA-NEXT:    s_getpc_b64 s[8:9]
-; HSA-NEXT:    s_add_u32 s8, s8, external_void_func_v3i32@rel32@lo+4
-; HSA-NEXT:    s_addc_u32 s9, s9, external_void_func_v3i32@rel32@hi+12
 ; HSA-NEXT:    s_swappc_b64 s[30:31], s[8:9]
 ; HSA-NEXT:    s_endpgm
   call void @external_void_func_v3i32(<3 x i32> <i32 3, i32 4, i32 5>)
@@ -3500,15 +3468,15 @@ define amdgpu_kernel void @test_call_external_void_func_v3i32_i32(i32) #0 {
 ; VI-NEXT:    s_addc_u32 s37, s37, 0
 ; VI-NEXT:    s_mov_b64 s[6:7], s[0:1]
 ; VI-NEXT:    s_mov_b64 s[0:1], s[36:37]
+; VI-NEXT:    s_getpc_b64 s[4:5]
+; VI-NEXT:    s_add_u32 s4, s4, external_void_func_v3i32_i32@rel32@lo+4
+; VI-NEXT:    s_addc_u32 s5, s5, external_void_func_v3i32_i32@rel32@hi+12
 ; VI-NEXT:    s_mov_b64 s[2:3], s[38:39]
 ; VI-NEXT:    v_mov_b32_e32 v0, 3
 ; VI-NEXT:    v_mov_b32_e32 v1, 4
 ; VI-NEXT:    v_mov_b32_e32 v2, 5
 ; VI-NEXT:    v_mov_b32_e32 v3, 6
 ; VI-NEXT:    s_mov_b32 s32, 0
-; VI-NEXT:    s_getpc_b64 s[4:5]
-; VI-NEXT:    s_add_u32 s4, s4, external_void_func_v3i32_i32@rel32@lo+4
-; VI-NEXT:    s_addc_u32 s5, s5, external_void_func_v3i32_i32@rel32@hi+12
 ; VI-NEXT:    s_swappc_b64 s[30:31], s[4:5]
 ; VI-NEXT:    s_endpgm
 ;
@@ -3522,15 +3490,15 @@ define amdgpu_kernel void @test_call_external_void_func_v3i32_i32(i32) #0 {
 ; CI-NEXT:    s_addc_u32 s37, s37, 0
 ; CI-NEXT:    s_mov_b64 s[6:7], s[0:1]
 ; CI-NEXT:    s_mov_b64 s[0:1], s[36:37]
+; CI-NEXT:    s_getpc_b64 s[4:5]
+; CI-NEXT:    s_add_u32 s4, s4, external_void_func_v3i32_i32@rel32@lo+4
+; CI-NEXT:    s_addc_u32 s5, s5, external_void_func_v3i32_i32@rel32@hi+12
 ; CI-NEXT:    s_mov_b64 s[2:3], s[38:39]
 ; CI-NEXT:    v_mov_b32_e32 v0, 3
 ; CI-NEXT:    v_mov_b32_e32 v1, 4
 ; CI-NEXT:    v_mov_b32_e32 v2, 5
 ; CI-NEXT:    v_mov_b32_e32 v3, 6
 ; CI-NEXT:    s_mov_b32 s32, 0
-; CI-NEXT:    s_getpc_b64 s[4:5]
-; CI-NEXT:    s_add_u32 s4, s4, external_void_func_v3i32_i32@rel32@lo+4
-; CI-NEXT:    s_addc_u32 s5, s5, external_void_func_v3i32_i32@rel32@hi+12
 ; CI-NEXT:    s_swappc_b64 s[30:31], s[4:5]
 ; CI-NEXT:    s_endpgm
 ;
@@ -3544,15 +3512,15 @@ define amdgpu_kernel void @test_call_external_void_func_v3i32_i32(i32) #0 {
 ; GFX9-NEXT:    s_addc_u32 s37, s37, 0
 ; GFX9-NEXT:    s_mov_b64 s[6:7], s[0:1]
 ; GFX9-NEXT:    s_mov_b64 s[0:1], s[36:37]
+; GFX9-NEXT:    s_getpc_b64 s[4:5]
+; GFX9-NEXT:    s_add_u32 s4, s4, external_void_func_v3i32_i32@rel32@lo+4
+; GFX9-NEXT:    s_addc_u32 s5, s5, external_void_func_v3i32_i32@rel32@hi+12
 ; GFX9-NEXT:    s_mov_b64 s[2:3], s[38:39]
 ; GFX9-NEXT:    v_mov_b32_e32 v0, 3
 ; GFX9-NEXT:    v_mov_b32_e32 v1, 4
 ; GFX9-NEXT:    v_mov_b32_e32 v2, 5
 ; GFX9-NEXT:    v_mov_b32_e32 v3, 6
 ; GFX9-NEXT:    s_mov_b32 s32, 0
-; GFX9-NEXT:    s_getpc_b64 s[4:5]
-; GFX9-NEXT:    s_add_u32 s4, s4, external_void_func_v3i32_i32@rel32@lo+4
-; GFX9-NEXT:    s_addc_u32 s5, s5, external_void_func_v3i32_i32@rel32@hi+12
 ; GFX9-NEXT:    s_swappc_b64 s[30:31], s[4:5]
 ; GFX9-NEXT:    s_endpgm
 ;
@@ -3560,12 +3528,11 @@ define amdgpu_kernel void @test_call_external_void_func_v3i32_i32(i32) #0 {
 ; GFX11:       ; %bb.0:
 ; GFX11-NEXT:    v_dual_mov_b32 v0, 3 :: v_dual_mov_b32 v1, 4
 ; GFX11-NEXT:    v_dual_mov_b32 v2, 5 :: v_dual_mov_b32 v3, 6
-; GFX11-NEXT:    s_mov_b64 s[6:7], s[0:1]
-; GFX11-NEXT:    s_mov_b32 s32, 0
 ; GFX11-NEXT:    s_getpc_b64 s[2:3]
 ; GFX11-NEXT:    s_add_u32 s2, s2, external_void_func_v3i32_i32@rel32@lo+4
 ; GFX11-NEXT:    s_addc_u32 s3, s3, external_void_func_v3i32_i32@rel32@hi+12
-; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_mov_b64 s[6:7], s[0:1]
+; GFX11-NEXT:    s_mov_b32 s32, 0
 ; GFX11-NEXT:    s_swappc_b64 s[30:31], s[2:3]
 ; GFX11-NEXT:    s_endpgm
 ;
@@ -3575,16 +3542,16 @@ define amdgpu_kernel void @test_call_external_void_func_v3i32_i32(i32) #0 {
 ; HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s8, 8
 ; HSA-NEXT:    s_add_u32 s0, s0, s11
 ; HSA-NEXT:    s_addc_u32 s1, s1, 0
+; HSA-NEXT:    s_mov_b32 flat_scratch_lo, s9
+; HSA-NEXT:    s_getpc_b64 s[8:9]
+; HSA-NEXT:    s_add_u32 s8, s8, external_void_func_v3i32_i32@rel32@lo+4
+; HSA-NEXT:    s_addc_u32 s9, s9, external_void_func_v3i32_i32@rel32@hi+12
 ; HSA-NEXT:    s_mov_b64 s[6:7], s[4:5]
 ; HSA-NEXT:    v_mov_b32_e32 v0, 3
 ; HSA-NEXT:    v_mov_b32_e32 v1, 4
 ; HSA-NEXT:    v_mov_b32_e32 v2, 5
 ; HSA-NEXT:    v_mov_b32_e32 v3, 6
 ; HSA-NEXT:    s_mov_b32 s32, 0
-; HSA-NEXT:    s_mov_b32 flat_scratch_lo, s9
-; HSA-NEXT:    s_getpc_b64 s[8:9]
-; HSA-NEXT:    s_add_u32 s8, s8, external_void_func_v3i32_i32@rel32@lo+4
-; HSA-NEXT:    s_addc_u32 s9, s9, external_void_func_v3i32_i32@rel32@hi+12
 ; HSA-NEXT:    s_swappc_b64 s[30:31], s[8:9]
 ; HSA-NEXT:    s_endpgm
   call void @external_void_func_v3i32_i32(<3 x i32> <i32 3, i32 4, i32 5>, i32 6)
@@ -3605,11 +3572,11 @@ define amdgpu_kernel void @test_call_external_void_func_v4i32() #0 {
 ; VI-NEXT:    s_addc_u32 s37, s37, 0
 ; VI-NEXT:    s_mov_b64 s[6:7], s[0:1]
 ; VI-NEXT:    s_mov_b64 s[0:1], s[36:37]
-; VI-NEXT:    s_mov_b64 s[2:3], s[38:39]
-; VI-NEXT:    s_mov_b32 s32, 0
 ; VI-NEXT:    s_getpc_b64 s[4:5]
 ; VI-NEXT:    s_add_u32 s4, s4, external_void_func_v4i32@rel32@lo+4
 ; VI-NEXT:    s_addc_u32 s5, s5, external_void_func_v4i32@rel32@hi+12
+; VI-NEXT:    s_mov_b64 s[2:3], s[38:39]
+; VI-NEXT:    s_mov_b32 s32, 0
 ; VI-NEXT:    s_swappc_b64 s[30:31], s[4:5]
 ; VI-NEXT:    s_endpgm
 ;
@@ -3626,11 +3593,11 @@ define amdgpu_kernel void @test_call_external_void_func_v4i32() #0 {
 ; CI-NEXT:    s_addc_u32 s37, s37, 0
 ; CI-NEXT:    s_mov_b64 s[6:7], s[0:1]
 ; CI-NEXT:    s_mov_b64 s[0:1], s[36:37]
-; CI-NEXT:    s_mov_b64 s[2:3], s[38:39]
-; CI-NEXT:    s_mov_b32 s32, 0
 ; CI-NEXT:    s_getpc_b64 s[4:5]
 ; CI-NEXT:    s_add_u32 s4, s4, external_void_func_v4i32@rel32@lo+4
 ; CI-NEXT:    s_addc_u32 s5, s5, external_void_func_v4i32@rel32@hi+12
+; CI-NEXT:    s_mov_b64 s[2:3], s[38:39]
+; CI-NEXT:    s_mov_b32 s32, 0
 ; CI-NEXT:    s_swappc_b64 s[30:31], s[4:5]
 ; CI-NEXT:    s_endpgm
 ;
@@ -3647,11 +3614,11 @@ define amdgpu_kernel void @test_call_external_void_func_v4i32() #0 {
 ; GFX9-NEXT:    s_addc_u32 s37, s37, 0
 ; GFX9-NEXT:    s_mov_b64 s[6:7], s[0:1]
 ; GFX9-NEXT:    s_mov_b64 s[0:1], s[36:37]
-; GFX9-NEXT:    s_mov_b64 s[2:3], s[38:39]
-; GFX9-NEXT:    s_mov_b32 s32, 0
 ; GFX9-NEXT:    s_getpc_b64 s[4:5]
 ; GFX9-NEXT:    s_add_u32 s4, s4, external_void_func_v4i32@rel32@lo+4
 ; GFX9-NEXT:    s_addc_u32 s5, s5, external_void_func_v4i32@rel32@hi+12
+; GFX9-NEXT:    s_mov_b64 s[2:3], s[38:39]
+; GFX9-NEXT:    s_mov_b32 s32, 0
 ; GFX9-NEXT:    s_swappc_b64 s[30:31], s[4:5]
 ; GFX9-NEXT:    s_endpgm
 ;
@@ -3661,11 +3628,10 @@ define amdgpu_kernel void @test_call_external_void_func_v4i32() #0 {
 ; GFX11-NEXT:    s_mov_b32 s2, -1
 ; GFX11-NEXT:    s_mov_b64 s[6:7], s[0:1]
 ; GFX11-NEXT:    buffer_load_b128 v[0:3], off, s[0:3], 0
-; GFX11-NEXT:    s_mov_b32 s32, 0
 ; GFX11-NEXT:    s_getpc_b64 s[2:3]
 ; GFX11-NEXT:    s_add_u32 s2, s2, external_void_func_v4i32@rel32@lo+4
 ; GFX11-NEXT:    s_addc_u32 s3, s3, external_void_func_v4i32@rel32@hi+12
-; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_mov_b32 s32, 0
 ; GFX11-NEXT:    s_swappc_b64 s[30:31], s[2:3]
 ; GFX11-NEXT:    s_endpgm
 ;
@@ -3679,11 +3645,11 @@ define amdgpu_kernel void @test_call_external_void_func_v4i32() #0 {
 ; HSA-NEXT:    buffer_load_dwordx4 v[0:3], off, s[4:7], 0
 ; HSA-NEXT:    s_add_u32 s0, s0, s9
 ; HSA-NEXT:    s_addc_u32 s1, s1, 0
-; HSA-NEXT:    s_mov_b64 s[6:7], s[4:5]
-; HSA-NEXT:    s_mov_b32 s32, 0
 ; HSA-NEXT:    s_getpc_b64 s[8:9]
 ; HSA-NEXT:    s_add_u32 s8, s8, external_void_func_v4i32@rel32@lo+4
 ; HSA-NEXT:    s_addc_u32 s9, s9, external_void_func_v4i32@rel32@hi+12
+; HSA-NEXT:    s_mov_b64 s[6:7], s[4:5]
+; HSA-NEXT:    s_mov_b32 s32, 0
 ; HSA-NEXT:    s_swappc_b64 s[30:31], s[8:9]
 ; HSA-NEXT:    s_endpgm
   %val = load <4 x i32>, ptr addrspace(1) undef
@@ -3702,15 +3668,15 @@ define amdgpu_kernel void @test_call_external_void_func_v4i32_imm() #0 {
 ; VI-NEXT:    s_addc_u32 s37, s37, 0
 ; VI-NEXT:    s_mov_b64 s[6:7], s[0:1]
 ; VI-NEXT:    s_mov_b64 s[0:1], s[36:37]
+; VI-NEXT:    s_getpc_b64 s[4:5]
+; VI-NEXT:    s_add_u32 s4, s4, external_void_func_v4i32@rel32@lo+4
+; VI-NEXT:    s_addc_u32 s5, s5, external_void_func_v4i32@rel32@hi+12
 ; VI-NEXT:    s_mov_b64 s[2:3], s[38:39]
 ; VI-NEXT:    v_mov_b32_e32 v0, 1
 ; VI-NEXT:    v_mov_b32_e32 v1, 2
 ; VI-NEXT:    v_mov_b32_e32 v2, 3
 ; VI-NEXT:    v_mov_b32_e32 v3, 4
 ; VI-NEXT:    s_mov_b32 s32, 0
-; VI-NEXT:    s_getpc_b64 s[4:5]
-; VI-NEXT:    s_add_u32 s4, s4, external_void_func_v4i32@rel32@lo+4
-; VI-NEXT:    s_addc_u32 s5, s5, external_void_func_v4i32@rel32@hi+12
 ; VI-NEXT:    s_swappc_b64 s[30:31], s[4:5]
 ; VI-NEXT:    s_endpgm
 ;
@@ -3724,15 +3690,15 @@ define amdgpu_kernel void @test_call_external_void_func_v4i32_imm() #0 {
 ; CI-NEXT:    s_addc_u32 s37, s37, 0
 ; CI-NEXT:    s_mov_b64 s[6:7], s[0:1]
 ; CI-NEXT:    s_mov_b64 s[0:1], s[36:37]
+; CI-NEXT:    s_getpc_b64 s[4:5]
+; CI-NEXT:    s_add_u32 s4, s4, external_void_func_v4i32@rel32@lo+4
+; CI-NEXT:    s_addc_u32 s5, s5, external_void_func_v4i32@rel32@hi+12
 ; CI-NEXT:    s_mov_b64 s[2:3], s[38:39]
 ; CI-NEXT:    v_mov_b32_e32 v0, 1
 ; CI-NEXT:    v_mov_b32_e32 v1, 2
 ; CI-NEXT:    v_mov_b32_e32 v2, 3
 ; CI-NEXT:    v_mov_b32_e32 v3, 4
 ; CI-NEXT:    s_mov_b32 s32, 0
-; CI-NEXT:    s_getpc_b64 s[4:5]
-; CI-NEXT:    s_add_u32 s4, s4, external_void_func_v4i32@rel32@lo+4
-; CI-NEXT:    s_addc_u32 s5, s5, external_void_func_v4i32@rel32@hi+12
 ; CI-NEXT:    s_swappc_b64 s[30:31], s[4:5]
 ; CI-NEXT:    s_endpgm
 ;
@@ -3746,15 +3712,15 @@ define amdgpu_kernel void @test_call_external_void_func_v4i32_imm() #0 {
 ; GFX9-NEXT:    s_addc_u32 s37, s37, 0
 ; GFX9-NEXT:    s_mov_b64 s[6:7], s[0:1]
 ; GFX9-NEXT:    s_mov_b64 s[0:1], s[36:37]
+; GFX9-NEXT:    s_getpc_b64 s[4:5]
+; GFX9-NEXT:    s_add_u32 s4, s4, external_void_func_v4i32@rel32@lo+4
+; GFX9-NEXT:    s_addc_u32 s5, s5, external_void_func_v4i32@rel32@hi+12
 ; GFX9-NEXT:    s_mov_b64 s[2:3], s[38:39]
 ; GFX9-NEXT:    v_mov_b32_e32 v0, 1
 ; GFX9-NEXT:    v_mov_b32_e32 v1, 2
 ; GFX9-NEXT:    v_mov_b32_e32 v2, 3
 ; GFX9-NEXT:    v_mov_b32_e32 v3, 4
 ; GFX9-NEXT:    s_mov_b32 s32, 0
-; GFX9-NEXT:    s_getpc_b64 s[4:5]
-; GFX9-NEXT:    s_add_u32 s4, s4, external_void_func_v4i32@rel32@lo+4
-; GFX9-NEXT:    s_addc_u32 s5, s5, external_void_func_v4i32@rel32@hi+12
 ; GFX9-NEXT:    s_swappc_b64 s[30:31], s[4:5]
 ; GFX9-NEXT:    s_endpgm
 ;
@@ -3762,12 +3728,11 @@ define amdgpu_kernel void @test_call_external_void_func_v4i32_imm() #0 {
 ; GFX11:       ; %bb.0:
 ; GFX11-NEXT:    v_dual_mov_b32 v0, 1 :: v_dual_mov_b32 v1, 2
 ; GFX11-NEXT:    v_dual_mov_b32 v2, 3 :: v_dual_mov_b32 v3, 4
-; GFX11-NEXT:    s_mov_b64 s[6:7], s[0:1]
-; GFX11-NEXT:    s_mov_b32 s32, 0
 ; GFX11-NEXT:    s_getpc_b64 s[2:3]
 ; GFX11-NEXT:    s_add_u32 s2, s2, external_void_func_v4i32@rel32@lo+4
 ; GFX11-NEXT:    s_addc_u32 s3, s3, external_void_func_v4i32@rel32@hi+12
-; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_mov_b64 s[6:7], s[0:1]
+; GFX11-NEXT:    s_mov_b32 s32, 0
 ; GFX11-NEXT:    s_swappc_b64 s[30:31], s[2:3]
 ; GFX11-NEXT:    s_endpgm
 ;
@@ -3776,17 +3741,17 @@ define amdgpu_kernel void @test_call_external_void_func_v4i32_imm() #0 {
 ; HSA-NEXT:    s_add_i32 s6, s6, s9
 ; HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s6, 8
 ; HSA-NEXT:    s_add_u32 s0, s0, s9
-; HSA-NEXT:    s_mov_b32 flat_scratch_lo, s7
 ; HSA-NEXT:    s_addc_u32 s1, s1, 0
+; HSA-NEXT:    s_mov_b32 flat_scratch_lo, s7
+; HSA-NEXT:    s_getpc_b64 s[8:9]
+; HSA-NEXT:    s_add_u32 s8, s8, external_void_func_v4i32@rel32@lo+4
+; HSA-NEXT:    s_addc_u32 s9, s9, external_void_func_v4i32@rel32@hi+12
 ; HSA-NEXT:    s_mov_b64 s[6:7], s[4:5]
 ; HSA-NEXT:    v_mov_b32_e32 v0, 1
 ; HSA-NEXT:    v_mov_b32_e32 v1, 2
 ; HSA-NEXT:    v_mov_b32_e32 v2, 3
 ; HSA-NEXT:    v_mov_b32_e32 v3, 4
 ; HSA-NEXT:    s_mov_b32 s32, 0
-; HSA-NEXT:    s_getpc_b64 s[8:9]
-; HSA-NEXT:    s_add_u32 s8, s8, external_void_func_v4i32@rel32@lo+4
-; HSA-NEXT:    s_addc_u32 s9, s9, external_void_func_v4i32@rel32@hi+12
 ; HSA-NEXT:    s_swappc_b64 s[30:31], s[8:9]
 ; HSA-NEXT:    s_endpgm
   call void @external_void_func_v4i32(<4 x i32> <i32 1, i32 2, i32 3, i32 4>)
@@ -3804,6 +3769,9 @@ define amdgpu_kernel void @test_call_external_void_func_v5i32_imm() #0 {
 ; VI-NEXT:    s_addc_u32 s37, s37, 0
 ; VI-NEXT:    s_mov_b64 s[6:7], s[0:1]
 ; VI-NEXT:    s_mov_b64 s[0:1], s[36:37]
+; VI-NEXT:    s_getpc_b64 s[4:5]
+; VI-NEXT:    s_add_u32 s4, s4, external_void_func_v5i32@rel32@lo+4
+; VI-NEXT:    s_addc_u32 s5, s5, external_void_func_v5i32@rel32@hi+12
 ; VI-NEXT:    s_mov_b64 s[2:3], s[38:39]
 ; VI-NEXT:    v_mov_b32_e32 v0, 1
 ; VI-NEXT:    v_mov_b32_e32 v1, 2
@@ -3811,9 +3779,6 @@ define amdgpu_kernel void @test_call_external_void_func_v5i32_imm() #0 {
 ; VI-NEXT:    v_mov_b32_e32 v3, 4
 ; VI-NEXT:    v_mov_b32_e32 v4, 5
 ; VI-NEXT:    s_mov_b32 s32, 0
-; VI-NEXT:    s_getpc_b64 s[4:5]
-; VI-NEXT:    s_add_u32 s4, s4, external_void_func_v5i32@rel32@lo+4
-; VI-NEXT:    s_addc_u32 s5, s5, external_void_func_v5i32@rel32@hi+12
 ; VI-NEXT:    s_swappc_b64 s[30:31], s[4:5]
 ; VI-NEXT:    s_endpgm
 ;
@@ -3827,6 +3792,9 @@ define amdgpu_kernel void @test_call_external_void_func_v5i32_imm() #0 {
 ; CI-NEXT:    s_addc_u32 s37, s37, 0
 ; CI-NEXT:    s_mov_b64 s[6:7], s[0:1]
 ; CI-NEXT:    s_mov_b64 s[0:1], s[36:37]
+; CI-NEXT:    s_getpc_b64 s[4:5]
+; CI-NEXT:    s_add_u32 s4, s4, external_void_func_v5i32@rel32@lo+4
+; CI-NEXT:    s_addc_u32 s5, s5, external_void_func_v5i32@rel32@hi+12
 ; CI-NEXT:    s_mov_b64 s[2:3], s[38:39]
 ; CI-NEXT:    v_mov_b32_e32 v0, 1
 ; CI-NEXT:    v_mov_b32_e32 v1, 2
@@ -3834,9 +3802,6 @@ define amdgpu_kernel void @test_call_external_void_func_v5i32_imm() #0 {
 ; CI-NEXT:    v_mov_b32_e32 v3, 4
 ; CI-NEXT:    v_mov_b32_e32 v4, 5
 ; CI-NEXT:    s_mov_b32 s32, 0
-; CI-NEXT:    s_getpc_b64 s[4:5]
-; CI-NEXT:    s_add_u32 s4, s4, external_void_func_v5i32@rel32@lo+4
-; CI-NEXT:    s_addc_u32 s5, s5, external_void_func_v5i32@rel32@hi+12
 ; CI-NEXT:    s_swappc_b64 s[30:31], s[4:5]
 ; CI-NEXT:    s_endpgm
 ;
@@ -3850,6 +3815,9 @@ define amdgpu_kernel void @test_call_external_void_func_v5i32_imm() #0 {
 ; GFX9-NEXT:    s_addc_u32 s37, s37, 0
 ; GFX9-NEXT:    s_mov_b64 s[6:7], s[0:1]
 ; GFX9-NEXT:    s_mov_b64 s[0:1], s[36:37]
+; GFX9-NEXT:    s_getpc_b64 s[4:5]
+; GFX9-NEXT:    s_add_u32 s4, s4, external_void_func_v5i32@rel32@lo+4
+; GFX9-NEXT:    s_addc_u32 s5, s5, external_void_func_v5i32@rel32@hi+12
 ; GFX9-NEXT:    s_mov_b64 s[2:3], s[38:39]
 ; GFX9-NEXT:    v_mov_b32_e32 v0, 1
 ; GFX9-NEXT:    v_mov_b32_e32 v1, 2
@@ -3857,9 +3825,6 @@ define amdgpu_kernel void @test_call_external_void_func_v5i32_imm() #0 {
 ; GFX9-NEXT:    v_mov_b32_e32 v3, 4
 ; GFX9-NEXT:    v_mov_b32_e32 v4, 5
 ; GFX9-NEXT:    s_mov_b32 s32, 0
-; GFX9-NEXT:    s_getpc_b64 s[4:5]
-; GFX9-NEXT:    s_add_u32 s4, s4, external_void_func_v5i32@rel32@lo+4
-; GFX9-NEXT:    s_addc_u32 s5, s5, external_void_func_v5i32@rel32@hi+12
 ; GFX9-NEXT:    s_swappc_b64 s[30:31], s[4:5]
 ; GFX9-NEXT:    s_endpgm
 ;
@@ -3868,12 +3833,11 @@ define amdgpu_kernel void @test_call_external_void_func_v5i32_imm() #0 {
 ; GFX11-NEXT:    v_dual_mov_b32 v0, 1 :: v_dual_mov_b32 v1, 2
 ; GFX11-NEXT:    v_dual_mov_b32 v2, 3 :: v_dual_mov_b32 v3, 4
 ; GFX11-NEXT:    v_mov_b32_e32 v4, 5
-; GFX11-NEXT:    s_mov_b64 s[6:7], s[0:1]
-; GFX11-NEXT:    s_mov_b32 s32, 0
 ; GFX11-NEXT:    s_getpc_b64 s[2:3]
 ; GFX11-NEXT:    s_add_u32 s2, s2, external_void_func_v5i32@rel32@lo+4
 ; GFX11-NEXT:    s_addc_u32 s3, s3, external_void_func_v5i32@rel32@hi+12
-; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_mov_b64 s[6:7], s[0:1]
+; GFX11-NEXT:    s_mov_b32 s32, 0
 ; GFX11-NEXT:    s_swappc_b64 s[30:31], s[2:3]
 ; GFX11-NEXT:    s_endpgm
 ;
@@ -3882,8 +3846,11 @@ define amdgpu_kernel void @test_call_external_void_func_v5i32_imm() #0 {
 ; HSA-NEXT:    s_add_i32 s6, s6, s9
 ; HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s6, 8
 ; HSA-NEXT:    s_add_u32 s0, s0, s9
-; HSA-NEXT:    s_mov_b32 flat_scratch_lo, s7
 ; HSA-NEXT:    s_addc_u32 s1, s1, 0
+; HSA-NEXT:    s_mov_b32 flat_scratch_lo, s7
+; HSA-NEXT:    s_getpc_b64 s[8:9]
+; HSA-NEXT:    s_add_u32 s8, s8, external_void_func_v5i32@rel32@lo+4
+; HSA-NEXT:    s_addc_u32 s9, s9, external_void_func_v5i32@rel32@hi+12
 ; HSA-NEXT:    s_mov_b64 s[6:7], s[4:5]
 ; HSA-NEXT:    v_mov_b32_e32 v0, 1
 ; HSA-NEXT:    v_mov_b32_e32 v1, 2
@@ -3891,9 +3858,6 @@ define amdgpu_kernel void @test_call_external_void_func_v5i32_imm() #0 {
 ; HSA-NEXT:    v_mov_b32_e32 v3, 4
 ; HSA-NEXT:    v_mov_b32_e32 v4, 5
 ; HSA-NEXT:    s_mov_b32 s32, 0
-; HSA-NEXT:    s_getpc_b64 s[8:9]
-; HSA-NEXT:    s_add_u32 s8, s8, external_void_func_v5i32@rel32@lo+4
-; HSA-NEXT:    s_addc_u32 s9, s9, external_void_func_v5i32@rel32@hi+12
 ; HSA-NEXT:    s_swappc_b64 s[30:31], s[8:9]
 ; HSA-NEXT:    s_endpgm
   call void @external_void_func_v5i32(<5 x i32> <i32 1, i32 2, i32 3, i32 4, i32 5>)
@@ -3917,11 +3881,11 @@ define amdgpu_kernel void @test_call_external_void_func_v8i32() #0 {
 ; VI-NEXT:    buffer_load_dwordx4 v[4:7], off, s[0:3], 0 offset:16
 ; VI-NEXT:    s_addc_u32 s37, s37, 0
 ; VI-NEXT:    s_mov_b64 s[0:1], s[36:37]
-; VI-NEXT:    s_mov_b64 s[2:3], s[38:39]
-; VI-NEXT:    s_mov_b32 s32, 0
 ; VI-NEXT:    s_getpc_b64 s[4:5]
 ; VI-NEXT:    s_add_u32 s4, s4, external_void_func_v8i32@rel32@lo+4
 ; VI-NEXT:    s_addc_u32 s5, s5, external_void_func_v8i32@rel32@hi+12
+; VI-NEXT:    s_mov_b64 s[2:3], s[38:39]
+; VI-NEXT:    s_mov_b32 s32, 0
 ; VI-NEXT:    s_swappc_b64 s[30:31], s[4:5]
 ; VI-NEXT:    s_endpgm
 ;
@@ -3941,11 +3905,11 @@ define amdgpu_kernel void @test_call_external_void_func_v8i32() #0 {
 ; CI-NEXT:    buffer_load_dwordx4 v[4:7], off, s[0:3], 0 offset:16
 ; CI-NEXT:    s_addc_u32 s37, s37, 0
 ; CI-NEXT:    s_mov_b64 s[0:1], s[36:37]
-; CI-NEXT:    s_mov_b64 s[2:3], s[38:39]
-; CI-NEXT:    s_mov_b32 s32, 0
 ; CI-NEXT:    s_getpc_b64 s[4:5]
 ; CI-NEXT:    s_add_u32 s4, s4, external_void_func_v8i32@rel32@lo+4
 ; CI-NEXT:    s_addc_u32 s5, s5, external_void_func_v8i32@rel32@hi+12
+; CI-NEXT:    s_mov_b64 s[2:3], s[38:39]
+; CI-NEXT:    s_mov_b32 s32, 0
 ; CI-NEXT:    s_swappc_b64 s[30:31], s[4:5]
 ; CI-NEXT:    s_endpgm
 ;
@@ -3965,11 +3929,11 @@ define amdgpu_kernel void @test_call_external_void_func_v8i32() #0 {
 ; GFX9-NEXT:    buffer_load_dwordx4 v[4:7], off, s[0:3], 0 offset:16
 ; GFX9-NEXT:    s_addc_u32 s37, s37, 0
 ; GFX9-NEXT:    s_mov_b64 s[0:1], s[36:37]
-; GFX9-NEXT:    s_mov_b64 s[2:3], s[38:39]
-; GFX9-NEXT:    s_mov_b32 s32, 0
 ; GFX9-NEXT:    s_getpc_b64 s[4:5]
 ; GFX9-NEXT:    s_add_u32 s4, s4, external_void_func_v8i32@rel32@lo+4
 ; GFX9-NEXT:    s_addc_u32 s5, s5, external_void_func_v8i32@rel32@hi+12
+; GFX9-NEXT:    s_mov_b64 s[2:3], s[38:39]
+; GFX9-NEXT:    s_mov_b32 s32, 0
 ; GFX9-NEXT:    s_swappc_b64 s[30:31], s[4:5]
 ; GFX9-NEXT:    s_endpgm
 ;
@@ -3978,10 +3942,10 @@ define amdgpu_kernel void @test_call_external_void_func_v8i32() #0 {
 ; GFX11-NEXT:    s_load_b64 s[4:5], s[0:1], 0x0
 ; GFX11-NEXT:    s_mov_b32 s7, 0x31016000
 ; GFX11-NEXT:    s_mov_b32 s6, -1
-; GFX11-NEXT:    s_mov_b32 s32, 0
 ; GFX11-NEXT:    s_getpc_b64 s[2:3]
 ; GFX11-NEXT:    s_add_u32 s2, s2, external_void_func_v8i32@rel32@lo+4
 ; GFX11-NEXT:    s_addc_u32 s3, s3, external_void_func_v8i32@rel32@hi+12
+; GFX11-NEXT:    s_mov_b32 s32, 0
 ; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX11-NEXT:    s_clause 0x1
 ; GFX11-NEXT:    buffer_load_b128 v[0:3], off, s[4:7], 0
@@ -3998,16 +3962,16 @@ define amdgpu_kernel void @test_call_external_void_func_v8i32() #0 {
 ; HSA-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; HSA-NEXT:    s_mov_b32 s11, 0x1100f000
 ; HSA-NEXT:    s_mov_b32 s10, -1
-; HSA-NEXT:    s_mov_b32 flat_scratch_lo, s7
+; HSA-NEXT:    s_addc_u32 s1, s1, 0
 ; HSA-NEXT:    s_waitcnt lgkmcnt(0)
 ; HSA-NEXT:    buffer_load_dwordx4 v[0:3], off, s[8:11], 0
 ; HSA-NEXT:    buffer_load_dwordx4 v[4:7], off, s[8:11], 0 offset:16
-; HSA-NEXT:    s_addc_u32 s1, s1, 0
-; HSA-NEXT:    s_mov_b64 s[6:7], s[4:5]
-; HSA-NEXT:    s_mov_b32 s32, 0
+; HSA-NEXT:    s_mov_b32 flat_scratch_lo, s7
 ; HSA-NEXT:    s_getpc_b64 s[8:9]
 ; HSA-NEXT:    s_add_u32 s8, s8, external_void_func_v8i32@rel32@lo+4
 ; HSA-NEXT:    s_addc_u32 s9, s9, external_void_func_v8i32@rel32@hi+12
+; HSA-NEXT:    s_mov_b64 s[6:7], s[4:5]
+; HSA-NEXT:    s_mov_b32 s32, 0
 ; HSA-NEXT:    s_swappc_b64 s[30:31], s[8:9]
 ; HSA-NEXT:    s_endpgm
   %ptr = load ptr addrspace(1), ptr addrspace(4) undef
@@ -4027,6 +3991,9 @@ define amdgpu_kernel void @test_call_external_void_func_v8i32_imm() #0 {
 ; VI-NEXT:    s_addc_u32 s37, s37, 0
 ; VI-NEXT:    s_mov_b64 s[6:7], s[0:1]
 ; VI-NEXT:    s_mov_b64 s[0:1], s[36:37]
+; VI-NEXT:    s_getpc_b64 s[4:5]
+; VI-NEXT:    s_add_u32 s4, s4, external_void_func_v8i32@rel32@lo+4
+; VI-NEXT:    s_addc_u32 s5, s5, external_void_func_v8i32@rel32@hi+12
 ; VI-NEXT:    s_mov_b64 s[2:3], s[38:39]
 ; VI-NEXT:    v_mov_b32_e32 v0, 1
 ; VI-NEXT:    v_mov_b32_e32 v1, 2
@@ -4037,9 +4004,6 @@ define amdgpu_kernel void @test_call_external_void_func_v8i32_imm() #0 {
 ; VI-NEXT:    v_mov_b32_e32 v6, 7
 ; VI-NEXT:    v_mov_b32_e32 v7, 8
 ; VI-NEXT:    s_mov_b32 s32, 0
-; VI-NEXT:    s_getpc_b64 s[4:5]
-; VI-NEXT:    s_add_u32 s4, s4, external_void_func_v8i32@rel32@lo+4
-; VI-NEXT:    s_addc_u32 s5, s5, external_void_func_v8i32@rel32@hi+12
 ; VI-NEXT:    s_swappc_b64 s[30:31], s[4:5]
 ; VI-NEXT:    s_endpgm
 ;
@@ -4053,6 +4017,9 @@ define amdgpu_kernel void @test_call_external_void_func_v8i32_imm() #0 {
 ; CI-NEXT:    s_addc_u32 s37, s37, 0
 ; CI-NEXT:    s_mov_b64 s[6:7], s[0:1]
 ; CI-NEXT:    s_mov_b64 s[0:1], s[36:37]
+; CI-NEXT:    s_getpc_b64 s[4:5]
+; CI-NEXT:    s_add_u32 s4, s4, external_void_func_v8i32@rel32@lo+4
+; CI-NEXT:    s_addc_u32 s5, s5, external_void_func_v8i32@rel32@hi+12
 ; CI-NEXT:    s_mov_b64 s[2:3], s[38:39]
 ; CI-NEXT:    v_mov_b32_e32 v0, 1
 ; CI-NEXT:    v_mov_b32_e32 v1, 2
@@ -4063,9 +4030,6 @@ define amdgpu_kernel void @test_call_external_void_func_v8i32_imm() #0 {
 ; CI-NEXT:    v_mov_b32_e32 v6, 7
 ; CI-NEXT:    v_mov_b32_e32 v7, 8
 ; CI-NEXT:    s_mov_b32 s32, 0
-; CI-NEXT:    s_getpc_b64 s[4:5]
-; CI-NEXT:    s_add_u32 s4, s4, external_void_func_v8i32@rel32@lo+4
-; CI-NEXT:    s_addc_u32 s5, s5, external_void_func_v8i32@rel32@hi+12
 ; CI-NEXT:    s_swappc_b64 s[30:31], s[4:5]
 ; CI-NEXT:    s_endpgm
 ;
@@ -4079,6 +4043,9 @@ define amdgpu_kernel void @test_call_external_void_func_v8i32_imm() #0 {
 ; GFX9-NEXT:    s_addc_u32 s37, s37, 0
 ; GFX9-NEXT:    s_mov_b64 s[6:7], s[0:1]
 ; GFX9-NEXT:    s_mov_b64 s[0:1], s[36:37]
+; GFX9-NEXT:    s_getpc_b64 s[4:5]
+; GFX9-NEXT:    s_add_u32 s4, s4, external_void_func_v8i32@rel32@lo+4
+; GFX9-NEXT:    s_addc_u32 s5, s5, external_void_func_v8i32@rel32@hi+12
 ; GFX9-NEXT:    s_mov_b64 s[2:3], s[38:39]
 ; GFX9-NEXT:    v_mov_b32_e32 v0, 1
 ; GFX9-NEXT:    v_mov_b32_e32 v1, 2
@@ -4089,9 +4056,6 @@ define amdgpu_kernel void @test_call_external_void_func_v8i32_imm() #0 {
 ; GFX9-NEXT:    v_mov_b32_e32 v6, 7
 ; GFX9-NEXT:    v_mov_b32_e32 v7, 8
 ; GFX9-NEXT:    s_mov_b32 s32, 0
-; GFX9-NEXT:    s_getpc_b64 s[4:5]
-; GFX9-NEXT:    s_add_u32 s4, s4, external_void_func_v8i32@rel32@lo+4
-; GFX9-NEXT:    s_addc_u32 s5, s5, external_void_func_v8i32@rel32@hi+12
 ; GFX9-NEXT:    s_swappc_b64 s[30:31], s[4:5]
 ; GFX9-NEXT:    s_endpgm
 ;
@@ -4101,12 +4065,11 @@ define amdgpu_kernel void @test_call_external_void_func_v8i32_imm() #0 {
 ; GFX11-NEXT:    v_dual_mov_b32 v2, 3 :: v_dual_mov_b32 v3, 4
 ; GFX11-NEXT:    v_dual_mov_b32 v4, 5 :: v_dual_mov_b32 v5, 6
 ; GFX11-NEXT:    v_dual_mov_b32 v6, 7 :: v_dual_mov_b32 v7, 8
-; GFX11-NEXT:    s_mov_b64 s[6:7], s[0:1]
-; GFX11-NEXT:    s_mov_b32 s32, 0
 ; GFX11-NEXT:    s_getpc_b64 s[2:3]
 ; GFX11-NEXT:    s_add_u32 s2, s2, external_void_func_v8i32@rel32@lo+4
 ; GFX11-NEXT:    s_addc_u32 s3, s3, external_void_func_v8i32@rel32@hi+12
-; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_mov_b64 s[6:7], s[0:1]
+; GFX11-NEXT:    s_mov_b32 s32, 0
 ; GFX11-NEXT:    s_swappc_b64 s[30:31], s[2:3]
 ; GFX11-NEXT:    s_endpgm
 ;
@@ -4115,8 +4078,11 @@ define amdgpu_kernel void @test_call_external_void_func_v8i32_imm() #0 {
 ; HSA-NEXT:    s_add_i32 s6, s6, s9
 ; HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s6, 8
 ; HSA-NEXT:    s_add_u32 s0, s0, s9
-; HSA-NEXT:    s_mov_b32 flat_scratch_lo, s7
 ; HSA-NEXT:    s_addc_u32 s1, s1, 0
+; HSA-NEXT:    s_mov_b32 flat_scratch_lo, s7
+; HSA-NEXT:    s_getpc_b64 s[8:9]
+; HSA-NEXT:    s_add_u32 s8, s8, external_void_func_v8i32@rel32@lo+4
+; HSA-NEXT:    s_addc_u32 s9, s9, external_void_func_v8i32@rel32@hi+12
 ; HSA-NEXT:    s_mov_b64 s[6:7], s[4:5]
 ; HSA-NEXT:    v_mov_b32_e32 v0, 1
 ; HSA-NEXT:    v_mov_b32_e32 v1, 2
@@ -4127,9 +4093,6 @@ define amdgpu_kernel void @test_call_external_void_func_v8i32_imm() #0 {
 ; HSA-NEXT:    v_mov_b32_e32 v6, 7
 ; HSA-NEXT:    v_mov_b32_e32 v7, 8
 ; HSA-NEXT:    s_mov_b32 s32, 0
-; HSA-NEXT:    s_getpc_b64 s[8:9]
-; HSA-NEXT:    s_add_u32 s8, s8, external_void_func_v8i32@rel32@lo+4
-; HSA-NEXT:    s_addc_u32 s9, s9, external_void_func_v8i32@rel32@hi+12
 ; HSA-NEXT:    s_swappc_b64 s[30:31], s[8:9]
 ; HSA-NEXT:    s_endpgm
   call void @external_void_func_v8i32(<8 x i32> <i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8>)
@@ -4155,11 +4118,11 @@ define amdgpu_kernel void @test_call_external_void_func_v16i32() #0 {
 ; VI-NEXT:    buffer_load_dwordx4 v[12:15], off, s[0:3], 0 offset:48
 ; VI-NEXT:    s_addc_u32 s37, s37, 0
 ; VI-NEXT:    s_mov_b64 s[0:1], s[36:37]
-; VI-NEXT:    s_mov_b64 s[2:3], s[38:39]
-; VI-NEXT:    s_mov_b32 s32, 0
 ; VI-NEXT:    s_getpc_b64 s[4:5]
 ; VI-NEXT:    s_add_u32 s4, s4, external_void_func_v16i32@rel32@lo+4
 ; VI-NEXT:    s_addc_u32 s5, s5, external_void_func_v16i32@rel32@hi+12
+; VI-NEXT:    s_mov_b64 s[2:3], s[38:39]
+; VI-NEXT:    s_mov_b32 s32, 0
 ; VI-NEXT:    s_swappc_b64 s[30:31], s[4:5]
 ; VI-NEXT:    s_endpgm
 ;
@@ -4181,11 +4144,11 @@ define amdgpu_kernel void @test_call_external_void_func_v16i32() #0 {
 ; CI-NEXT:    buffer_load_dwordx4 v[12:15], off, s[0:3], 0 offset:48
 ; CI-NEXT:    s_addc_u32 s37, s37, 0
 ; CI-NEXT:    s_mov_b64 s[0:1], s[36:37]
-; CI-NEXT:    s_mov_b64 s[2:3], s[38:39]
-; CI-NEXT:    s_mov_b32 s32, 0
 ; CI-NEXT:    s_getpc_b64 s[4:5]
 ; CI-NEXT:    s_add_u32 s4, s4, external_void_func_v16i32@rel32@lo+4
 ; CI-NEXT:    s_addc_u32 s5, s5, external_void_func_v16i32@rel32@hi+12
+; CI-NEXT:    s_mov_b64 s[2:3], s[38:39]
+; CI-NEXT:    s_mov_b32 s32, 0
 ; CI-NEXT:    s_swappc_b64 s[30:31], s[4:5]
 ; CI-NEXT:    s_endpgm
 ;
@@ -4207,11 +4170,11 @@ define amdgpu_kernel void @test_call_external_void_func_v16i32() #0 {
 ; GFX9-NEXT:    buffer_load_dwordx4 v[12:15], off, s[0:3], 0 offset:48
 ; GFX9-NEXT:    s_addc_u32 s37, s37, 0
 ; GFX9-NEXT:    s_mov_b64 s[0:1], s[36:37]
-; GFX9-NEXT:    s_mov_b64 s[2:3], s[38:39]
-; GFX9-NEXT:    s_mov_b32 s32, 0
 ; GFX9-NEXT:    s_getpc_b64 s[4:5]
 ; GFX9-NEXT:    s_add_u32 s4, s4, external_void_func_v16i32@rel32@lo+4
 ; GFX9-NEXT:    s_addc_u32 s5, s5, external_void_func_v16i32@rel32@hi+12
+; GFX9-NEXT:    s_mov_b64 s[2:3], s[38:39]
+; GFX9-NEXT:    s_mov_b32 s32, 0
 ; GFX9-NEXT:    s_swappc_b64 s[30:31], s[4:5]
 ; GFX9-NEXT:    s_endpgm
 ;
@@ -4220,10 +4183,10 @@ define amdgpu_kernel void @test_call_external_void_func_v16i32() #0 {
 ; GFX11-NEXT:    s_load_b64 s[4:5], s[0:1], 0x0
 ; GFX11-NEXT:    s_mov_b32 s7, 0x31016000
 ; GFX11-NEXT:    s_mov_b32 s6, -1
-; GFX11-NEXT:    s_mov_b32 s32, 0
 ; GFX11-NEXT:    s_getpc_b64 s[2:3]
 ; GFX11-NEXT:    s_add_u32 s2, s2, external_void_func_v16i32@rel32@lo+4
 ; GFX11-NEXT:    s_addc_u32 s3, s3, external_void_func_v16i32@rel32@hi+12
+; GFX11-NEXT:    s_mov_b32 s32, 0
 ; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX11-NEXT:    s_clause 0x3
 ; GFX11-NEXT:    buffer_load_b128 v[0:3], off, s[4:7], 0
@@ -4242,18 +4205,18 @@ define amdgpu_kernel void @test_call_external_void_func_v16i32() #0 {
 ; HSA-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; HSA-NEXT:    s_mov_b32 s11, 0x1100f000
 ; HSA-NEXT:    s_mov_b32 s10, -1
-; HSA-NEXT:    s_mov_b32 flat_scratch_lo, s7
+; HSA-NEXT:    s_addc_u32 s1, s1, 0
 ; HSA-NEXT:    s_waitcnt lgkmcnt(0)
 ; HSA-NEXT:    buffer_load_dwordx4 v[0:3], off, s[8:11], 0
 ; HSA-NEXT:    buffer_load_dwordx4 v[4:7], off, s[8:11], 0 offset:16
 ; HSA-NEXT:    buffer_load_dwordx4 v[8:11], off, s[8:11], 0 offset:32
 ; HSA-NEXT:    buffer_load_dwordx4 v[12:15], off, s[8:11], 0 offset:48
-; HSA-NEXT:    s_addc_u32 s1, s1, 0
-; HSA-NEXT:    s_mov_b64 s[6:7], s[4:5]
-; HSA-NEXT:    s_mov_b32 s32, 0
+; HSA-NEXT:    s_mov_b32 flat_scratch_lo, s7
 ; HSA-NEXT:    s_getpc_b64 s[8:9]
 ; HSA-NEXT:    s_add_u32 s8, s8, external_void_func_v16i32@rel32@lo+4
 ; HSA-NEXT:    s_addc_u32 s9, s9, external_void_func_v16i32@rel32@hi+12
+; HSA-NEXT:    s_mov_b64 s[6:7], s[4:5]
+; HSA-NEXT:    s_mov_b32 s32, 0
 ; HSA-NEXT:    s_swappc_b64 s[30:31], s[8:9]
 ; HSA-NEXT:    s_endpgm
   %ptr = load ptr addrspace(1), ptr addrspace(4) undef
@@ -4286,10 +4249,10 @@ define amdgpu_kernel void @test_call_external_void_func_v32i32() #0 {
 ; VI-NEXT:    s_mov_b64 s[6:7], s[0:1]
 ; VI-NEXT:    s_mov_b64 s[0:1], s[36:37]
 ; VI-NEXT:    s_mov_b32 s32, 0
-; VI-NEXT:    s_mov_b64 s[2:3], s[38:39]
 ; VI-NEXT:    s_getpc_b64 s[8:9]
 ; VI-NEXT:    s_add_u32 s8, s8, external_void_func_v32i32@rel32@lo+4
 ; VI-NEXT:    s_addc_u32 s9, s9, external_void_func_v32i32@rel32@hi+12
+; VI-NEXT:    s_mov_b64 s[2:3], s[38:39]
 ; VI-NEXT:    s_waitcnt vmcnt(7)
 ; VI-NEXT:    buffer_store_dword v31, off, s[36:39], s32
 ; VI-NEXT:    s_swappc_b64 s[30:31], s[8:9]
@@ -4318,10 +4281,10 @@ define amdgpu_kernel void @test_call_external_void_func_v32i32() #0 {
 ; CI-NEXT:    s_mov_b64 s[6:7], s[0:1]
 ; CI-NEXT:    s_mov_b64 s[0:1], s[36:37]
 ; CI-NEXT:    s_mov_b32 s32, 0
-; CI-NEXT:    s_mov_b64 s[2:3], s[38:39]
 ; CI-NEXT:    s_getpc_b64 s[8:9]
 ; CI-NEXT:    s_add_u32 s8, s8, external_void_func_v32i32@rel32@lo+4
 ; CI-NEXT:    s_addc_u32 s9, s9, external_void_func_v32i32@rel32@hi+12
+; CI-NEXT:    s_mov_b64 s[2:3], s[38:39]
 ; CI-NEXT:    s_waitcnt vmcnt(7)
 ; CI-NEXT:    buffer_store_dword v31, off, s[36:39], s32
 ; CI-NEXT:    s_swappc_b64 s[30:31], s[8:9]
@@ -4350,10 +4313,10 @@ define amdgpu_kernel void @test_call_external_void_func_v32i32() #0 {
 ; GFX9-NEXT:    s_mov_b64 s[6:7], s[0:1]
 ; GFX9-NEXT:    s_mov_b64 s[0:1], s[36:37]
 ; GFX9-NEXT:    s_mov_b32 s32, 0
-; GFX9-NEXT:    s_mov_b64 s[2:3], s[38:39]
 ; GFX9-NEXT:    s_getpc_b64 s[8:9]
 ; GFX9-NEXT:    s_add_u32 s8, s8, external_void_func_v32i32@rel32@lo+4
 ; GFX9-NEXT:    s_addc_u32 s9, s9, external_void_func_v32i32@rel32@hi+12
+; GFX9-NEXT:    s_mov_b64 s[2:3], s[38:39]
 ; GFX9-NEXT:    s_waitcnt vmcnt(7)
 ; GFX9-NEXT:    buffer_store_dword v31, off, s[36:39], s32
 ; GFX9-NEXT:    s_swappc_b64 s[30:31], s[8:9]
@@ -4392,7 +4355,7 @@ define amdgpu_kernel void @test_call_external_void_func_v32i32() #0 {
 ; HSA-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; HSA-NEXT:    s_mov_b32 s11, 0x1100f000
 ; HSA-NEXT:    s_mov_b32 s10, -1
-; HSA-NEXT:    s_mov_b32 s32, 0
+; HSA-NEXT:    s_addc_u32 s1, s1, 0
 ; HSA-NEXT:    s_waitcnt lgkmcnt(0)
 ; HSA-NEXT:    buffer_load_dwordx4 v[28:31], off, s[8:11], 0 offset:112
 ; HSA-NEXT:    buffer_load_dwordx4 v[0:3], off, s[8:11], 0
@@ -4402,12 +4365,12 @@ define amdgpu_kernel void @test_call_external_void_func_v32i32() #0 {
 ; HSA-NEXT:    buffer_load_dwordx4 v[16:19], off, s[8:11], 0 offset:64
 ; HSA-NEXT:    buffer_load_dwordx4 v[20:23], off, s[8:11], 0 offset:80
 ; HSA-NEXT:    buffer_load_dwordx4 v[24:27], off, s[8:11], 0 offset:96
+; HSA-NEXT:    s_mov_b32 s32, 0
 ; HSA-NEXT:    s_mov_b32 flat_scratch_lo, s7
-; HSA-NEXT:    s_addc_u32 s1, s1, 0
-; HSA-NEXT:    s_mov_b64 s[6:7], s[4:5]
 ; HSA-NEXT:    s_getpc_b64 s[12:13]
 ; HSA-NEXT:    s_add_u32 s12, s12, external_void_func_v32i32@rel32@lo+4
 ; HSA-NEXT:    s_addc_u32 s13, s13, external_void_func_v32i32@rel32@hi+12
+; HSA-NEXT:    s_mov_b64 s[6:7], s[4:5]
 ; HSA-NEXT:    s_waitcnt vmcnt(7)
 ; HSA-NEXT:    buffer_store_dword v31, off, s[0:3], s32
 ; HSA-NEXT:    s_swappc_b64 s[30:31], s[12:13]
@@ -4443,10 +4406,10 @@ define amdgpu_kernel void @test_call_external_void_func_v32i32_i32(i32) #0 {
 ; VI-NEXT:    s_mov_b64 s[6:7], s[0:1]
 ; VI-NEXT:    s_mov_b64 s[0:1], s[36:37]
 ; VI-NEXT:    s_mov_b32 s32, 0
-; VI-NEXT:    s_mov_b64 s[2:3], s[38:39]
 ; VI-NEXT:    s_getpc_b64 s[4:5]
 ; VI-NEXT:    s_add_u32 s4, s4, external_void_func_v32i32_i32@rel32@lo+4
 ; VI-NEXT:    s_addc_u32 s5, s5, external_void_func_v32i32_i32@rel32@hi+12
+; VI-NEXT:    s_mov_b64 s[2:3], s[38:39]
 ; VI-NEXT:    s_waitcnt vmcnt(8)
 ; VI-NEXT:    buffer_store_dword v32, off, s[36:39], s32 offset:4
 ; VI-NEXT:    s_waitcnt vmcnt(8)
@@ -4478,10 +4441,10 @@ define amdgpu_kernel void @test_call_external_void_func_v32i32_i32(i32) #0 {
 ; CI-NEXT:    s_mov_b64 s[6:7], s[0:1]
 ; CI-NEXT:    s_mov_b64 s[0:1], s[36:37]
 ; CI-NEXT:    s_mov_b32 s32, 0
-; CI-NEXT:    s_mov_b64 s[2:3], s[38:39]
 ; CI-NEXT:    s_getpc_b64 s[4:5]
 ; CI-NEXT:    s_add_u32 s4, s4, external_void_func_v32i32_i32@rel32@lo+4
 ; CI-NEXT:    s_addc_u32 s5, s5, external_void_func_v32i32_i32@rel32@hi+12
+; CI-NEXT:    s_mov_b64 s[2:3], s[38:39]
 ; CI-NEXT:    s_waitcnt vmcnt(8)
 ; CI-NEXT:    buffer_store_dword v32, off, s[36:39], s32 offset:4
 ; CI-NEXT:    s_waitcnt vmcnt(8)
@@ -4513,10 +4476,10 @@ define amdgpu_kernel void @test_call_external_void_func_v32i32_i32(i32) #0 {
 ; GFX9-NEXT:    s_mov_b64 s[6:7], s[0:1]
 ; GFX9-NEXT:    s_mov_b64 s[0:1], s[36:37]
 ; GFX9-NEXT:    s_mov_b32 s32, 0
-; GFX9-NEXT:    s_mov_b64 s[2:3], s[38:39]
 ; GFX9-NEXT:    s_getpc_b64 s[4:5]
 ; GFX9-NEXT:    s_add_u32 s4, s4, external_void_func_v32i32_i32@rel32@lo+4
 ; GFX9-NEXT:    s_addc_u32 s5, s5, external_void_func_v32i32_i32@rel32@hi+12
+; GFX9-NEXT:    s_mov_b64 s[2:3], s[38:39]
 ; GFX9-NEXT:    s_waitcnt vmcnt(8)
 ; GFX9-NEXT:    buffer_store_dword v32, off, s[36:39], s32 offset:4
 ; GFX9-NEXT:    s_waitcnt vmcnt(8)
@@ -4572,12 +4535,12 @@ define amdgpu_kernel void @test_call_external_void_func_v32i32_i32(i32) #0 {
 ; HSA-NEXT:    buffer_load_dwordx4 v[16:19], off, s[8:11], 0 offset:64
 ; HSA-NEXT:    buffer_load_dwordx4 v[20:23], off, s[8:11], 0 offset:80
 ; HSA-NEXT:    buffer_load_dwordx4 v[24:27], off, s[8:11], 0 offset:96
-; HSA-NEXT:    s_mov_b32 s32, 0
 ; HSA-NEXT:    s_addc_u32 s1, s1, 0
-; HSA-NEXT:    s_mov_b64 s[6:7], s[4:5]
+; HSA-NEXT:    s_mov_b32 s32, 0
 ; HSA-NEXT:    s_getpc_b64 s[8:9]
 ; HSA-NEXT:    s_add_u32 s8, s8, external_void_func_v32i32_i32@rel32@lo+4
 ; HSA-NEXT:    s_addc_u32 s9, s9, external_void_func_v32i32_i32@rel32@hi+12
+; HSA-NEXT:    s_mov_b64 s[6:7], s[4:5]
 ; HSA-NEXT:    s_waitcnt vmcnt(8)
 ; HSA-NEXT:    buffer_store_dword v32, off, s[0:3], s32 offset:4
 ; HSA-NEXT:    s_waitcnt vmcnt(8)
@@ -4603,14 +4566,14 @@ define amdgpu_kernel void @test_call_external_i32_func_i32_imm(ptr addrspace(1)
 ; VI-NEXT:    s_addc_u32 s41, s41, 0
 ; VI-NEXT:    s_mov_b64 s[6:7], s[0:1]
 ; VI-NEXT:    s_mov_b64 s[0:1], s[40:41]
+; VI-NEXT:    s_getpc_b64 s[4:5]
+; VI-NEXT:    s_add_u32 s4, s4, external_i32_func_i32@rel32@lo+4
+; VI-NEXT:    s_addc_u32 s5, s5, external_i32_func_i32@rel32@hi+12
 ; VI-NEXT:    s_mov_b64 s[2:3], s[42:43]
 ; VI-NEXT:    v_mov_b32_e32 v0, 42
 ; VI-NEXT:    s_mov_b32 s32, 0
 ; VI-NEXT:    s_mov_b32 s39, 0xf000
 ; VI-NEXT:    s_mov_b32 s38, -1
-; VI-NEXT:    s_getpc_b64 s[4:5]
-; VI-NEXT:    s_add_u32 s4, s4, external_i32_func_i32@rel32@lo+4
-; VI-NEXT:    s_addc_u32 s5, s5, external_i32_func_i32@rel32@hi+12
 ; VI-NEXT:    s_swappc_b64 s[30:31], s[4:5]
 ; VI-NEXT:    buffer_store_dword v0, off, s[36:39], 0
 ; VI-NEXT:    s_waitcnt vmcnt(0)
@@ -4627,14 +4590,14 @@ define amdgpu_kernel void @test_call_external_i32_func_i32_imm(ptr addrspace(1)
 ; CI-NEXT:    s_addc_u32 s41, s41, 0
 ; CI-NEXT:    s_mov_b64 s[6:7], s[0:1]
 ; CI-NEXT:    s_mov_b64 s[0:1], s[40:41]
+; CI-NEXT:    s_getpc_b64 s[4:5]
+; CI-NEXT:    s_add_u32 s4, s4, external_i32_func_i32@rel32@lo+4
+; CI-NEXT:    s_addc_u32 s5, s5, external_i32_func_i32@rel32@hi+12
 ; CI-NEXT:    s_mov_b64 s[2:3], s[42:43]
 ; CI-NEXT:    v_mov_b32_e32 v0, 42
 ; CI-NEXT:    s_mov_b32 s32, 0
 ; CI-NEXT:    s_mov_b32 s39, 0xf000
 ; CI-NEXT:    s_mov_b32 s38, -1
-; CI-NEXT:    s_getpc_b64 s[4:5]
-; CI-NEXT:    s_add_u32 s4, s4, external_i32_func_i32@rel32@lo+4
-; CI-NEXT:    s_addc_u32 s5, s5, external_i32_func_i32@rel32@hi+12
 ; CI-NEXT:    s_swappc_b64 s[30:31], s[4:5]
 ; CI-NEXT:    buffer_store_dword v0, off, s[36:39], 0
 ; CI-NEXT:    s_waitcnt vmcnt(0)
@@ -4651,14 +4614,14 @@ define amdgpu_kernel void @test_call_external_i32_func_i32_imm(ptr addrspace(1)
 ; GFX9-NEXT:    s_addc_u32 s41, s41, 0
 ; GFX9-NEXT:    s_mov_b64 s[6:7], s[0:1]
 ; GFX9-NEXT:    s_mov_b64 s[0:1], s[40:41]
+; GFX9-NEXT:    s_getpc_b64 s[4:5]
+; GFX9-NEXT:    s_add_u32 s4, s4, external_i32_func_i32@rel32@lo+4
+; GFX9-NEXT:    s_addc_u32 s5, s5, external_i32_func_i32@rel32@hi+12
 ; GFX9-NEXT:    s_mov_b64 s[2:3], s[42:43]
 ; GFX9-NEXT:    v_mov_b32_e32 v0, 42
 ; GFX9-NEXT:    s_mov_b32 s32, 0
 ; GFX9-NEXT:    s_mov_b32 s39, 0xf000
 ; GFX9-NEXT:    s_mov_b32 s38, -1
-; GFX9-NEXT:    s_getpc_b64 s[4:5]
-; GFX9-NEXT:    s_add_u32 s4, s4, external_i32_func_i32@rel32@lo+4
-; GFX9-NEXT:    s_addc_u32 s5, s5, external_i32_func_i32@rel32@hi+12
 ; GFX9-NEXT:    s_swappc_b64 s[30:31], s[4:5]
 ; GFX9-NEXT:    buffer_store_dword v0, off, s[36:39], 0
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
@@ -4668,14 +4631,13 @@ define amdgpu_kernel void @test_call_external_i32_func_i32_imm(ptr addrspace(1)
 ; GFX11:       ; %bb.0:
 ; GFX11-NEXT:    s_load_b64 s[36:37], s[2:3], 0x24
 ; GFX11-NEXT:    v_mov_b32_e32 v0, 42
+; GFX11-NEXT:    s_getpc_b64 s[2:3]
+; GFX11-NEXT:    s_add_u32 s2, s2, external_i32_func_i32@rel32@lo+4
+; GFX11-NEXT:    s_addc_u32 s3, s3, external_i32_func_i32@rel32@hi+12
 ; GFX11-NEXT:    s_mov_b64 s[6:7], s[0:1]
 ; GFX11-NEXT:    s_mov_b32 s32, 0
 ; GFX11-NEXT:    s_mov_b32 s39, 0x31016000
 ; GFX11-NEXT:    s_mov_b32 s38, -1
-; GFX11-NEXT:    s_getpc_b64 s[2:3]
-; GFX11-NEXT:    s_add_u32 s2, s2, external_i32_func_i32@rel32@lo+4
-; GFX11-NEXT:    s_addc_u32 s3, s3, external_i32_func_i32@rel32@hi+12
-; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
 ; GFX11-NEXT:    s_swappc_b64 s[30:31], s[2:3]
 ; GFX11-NEXT:    buffer_store_b32 v0, off, s[36:39], 0 dlc
 ; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
@@ -4684,19 +4646,19 @@ define amdgpu_kernel void @test_call_external_i32_func_i32_imm(ptr addrspace(1)
 ; HSA-LABEL: test_call_external_i32_func_i32_imm:
 ; HSA:       ; %bb.0:
 ; HSA-NEXT:    s_add_i32 s8, s8, s11
-; HSA-NEXT:    s_load_dwordx2 s[36:37], s[6:7], 0x0
 ; HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s8, 8
+; HSA-NEXT:    s_load_dwordx2 s[36:37], s[6:7], 0x0
 ; HSA-NEXT:    s_add_u32 s0, s0, s11
 ; HSA-NEXT:    s_addc_u32 s1, s1, 0
+; HSA-NEXT:    s_mov_b32 flat_scratch_lo, s9
+; HSA-NEXT:    s_getpc_b64 s[8:9]
+; HSA-NEXT:    s_add_u32 s8, s8, external_i32_func_i32@rel32@lo+4
+; HSA-NEXT:    s_addc_u32 s9, s9, external_i32_func_i32@rel32@hi+12
 ; HSA-NEXT:    s_mov_b64 s[6:7], s[4:5]
 ; HSA-NEXT:    v_mov_b32_e32 v0, 42
 ; HSA-NEXT:    s_mov_b32 s32, 0
-; HSA-NEXT:    s_mov_b32 flat_scratch_lo, s9
 ; HSA-NEXT:    s_mov_b32 s39, 0x1100f000
 ; HSA-NEXT:    s_mov_b32 s38, -1
-; HSA-NEXT:    s_getpc_b64 s[8:9]
-; HSA-NEXT:    s_add_u32 s8, s8, external_i32_func_i32@rel32@lo+4
-; HSA-NEXT:    s_addc_u32 s9, s9, external_i32_func_i32@rel32@hi+12
 ; HSA-NEXT:    s_swappc_b64 s[30:31], s[8:9]
 ; HSA-NEXT:    buffer_store_dword v0, off, s[36:39], 0
 ; HSA-NEXT:    s_waitcnt vmcnt(0)
@@ -4723,11 +4685,11 @@ define amdgpu_kernel void @test_call_external_void_func_struct_i8_i32() #0 {
 ; VI-NEXT:    buffer_load_dword v1, off, s[0:3], 0 offset:4
 ; VI-NEXT:    s_addc_u32 s37, s37, 0
 ; VI-NEXT:    s_mov_b64 s[0:1], s[36:37]
-; VI-NEXT:    s_mov_b64 s[2:3], s[38:39]
-; VI-NEXT:    s_mov_b32 s32, 0
 ; VI-NEXT:    s_getpc_b64 s[4:5]
 ; VI-NEXT:    s_add_u32 s4, s4, external_void_func_struct_i8_i32@rel32@lo+4
 ; VI-NEXT:    s_addc_u32 s5, s5, external_void_func_struct_i8_i32@rel32@hi+12
+; VI-NEXT:    s_mov_b64 s[2:3], s[38:39]
+; VI-NEXT:    s_mov_b32 s32, 0
 ; VI-NEXT:    s_swappc_b64 s[30:31], s[4:5]
 ; VI-NEXT:    s_endpgm
 ;
@@ -4747,11 +4709,11 @@ define amdgpu_kernel void @test_call_external_void_func_struct_i8_i32() #0 {
 ; CI-NEXT:    buffer_load_dword v1, off, s[0:3], 0 offset:4
 ; CI-NEXT:    s_addc_u32 s37, s37, 0
 ; CI-NEXT:    s_mov_b64 s[0:1], s[36:37]
-; CI-NEXT:    s_mov_b64 s[2:3], s[38:39]
-; CI-NEXT:    s_mov_b32 s32, 0
 ; CI-NEXT:    s_getpc_b64 s[4:5]
 ; CI-NEXT:    s_add_u32 s4, s4, external_void_func_struct_i8_i32@rel32@lo+4
 ; CI-NEXT:    s_addc_u32 s5, s5, external_void_func_struct_i8_i32@rel32@hi+12
+; CI-NEXT:    s_mov_b64 s[2:3], s[38:39]
+; CI-NEXT:    s_mov_b32 s32, 0
 ; CI-NEXT:    s_swappc_b64 s[30:31], s[4:5]
 ; CI-NEXT:    s_endpgm
 ;
@@ -4771,11 +4733,11 @@ define amdgpu_kernel void @test_call_external_void_func_struct_i8_i32() #0 {
 ; GFX9-NEXT:    buffer_load_dword v1, off, s[0:3], 0 offset:4
 ; GFX9-NEXT:    s_addc_u32 s37, s37, 0
 ; GFX9-NEXT:    s_mov_b64 s[0:1], s[36:37]
-; GFX9-NEXT:    s_mov_b64 s[2:3], s[38:39]
-; GFX9-NEXT:    s_mov_b32 s32, 0
 ; GFX9-NEXT:    s_getpc_b64 s[4:5]
 ; GFX9-NEXT:    s_add_u32 s4, s4, external_void_func_struct_i8_i32@rel32@lo+4
 ; GFX9-NEXT:    s_addc_u32 s5, s5, external_void_func_struct_i8_i32@rel32@hi+12
+; GFX9-NEXT:    s_mov_b64 s[2:3], s[38:39]
+; GFX9-NEXT:    s_mov_b32 s32, 0
 ; GFX9-NEXT:    s_swappc_b64 s[30:31], s[4:5]
 ; GFX9-NEXT:    s_endpgm
 ;
@@ -4784,10 +4746,10 @@ define amdgpu_kernel void @test_call_external_void_func_struct_i8_i32() #0 {
 ; GFX11-NEXT:    s_load_b64 s[4:5], s[0:1], 0x0
 ; GFX11-NEXT:    s_mov_b32 s7, 0x31016000
 ; GFX11-NEXT:    s_mov_b32 s6, -1
-; GFX11-NEXT:    s_mov_b32 s32, 0
 ; GFX11-NEXT:    s_getpc_b64 s[2:3]
 ; GFX11-NEXT:    s_add_u32 s2, s2, external_void_func_struct_i8_i32@rel32@lo+4
 ; GFX11-NEXT:    s_addc_u32 s3, s3, external_void_func_struct_i8_i32@rel32@hi+12
+; GFX11-NEXT:    s_mov_b32 s32, 0
 ; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX11-NEXT:    s_clause 0x1
 ; GFX11-NEXT:    buffer_load_u8 v0, off, s[4:7], 0
@@ -4804,16 +4766,16 @@ define amdgpu_kernel void @test_call_external_void_func_struct_i8_i32() #0 {
 ; HSA-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; HSA-NEXT:    s_mov_b32 s11, 0x1100f000
 ; HSA-NEXT:    s_mov_b32 s10, -1
-; HSA-NEXT:    s_mov_b32 flat_scratch_lo, s7
+; HSA-NEXT:    s_addc_u32 s1, s1, 0
 ; HSA-NEXT:    s_waitcnt lgkmcnt(0)
 ; HSA-NEXT:    buffer_load_ubyte v0, off, s[8:11], 0
 ; HSA-NEXT:    buffer_load_dword v1, off, s[8:11], 0 offset:4
-; HSA-NEXT:    s_addc_u32 s1, s1, 0
-; HSA-NEXT:    s_mov_b64 s[6:7], s[4:5]
-; HSA-NEXT:    s_mov_b32 s32, 0
+; HSA-NEXT:    s_mov_b32 flat_scratch_lo, s7
 ; HSA-NEXT:    s_getpc_b64 s[8:9]
 ; HSA-NEXT:    s_add_u32 s8, s8, external_void_func_struct_i8_i32@rel32@lo+4
 ; HSA-NEXT:    s_addc_u32 s9, s9, external_void_func_struct_i8_i32@rel32@hi+12
+; HSA-NEXT:    s_mov_b64 s[6:7], s[4:5]
+; HSA-NEXT:    s_mov_b32 s32, 0
 ; HSA-NEXT:    s_swappc_b64 s[30:31], s[8:9]
 ; HSA-NEXT:    s_endpgm
   %ptr0 = load ptr addrspace(1), ptr addrspace(4) undef
@@ -4840,10 +4802,10 @@ define amdgpu_kernel void @test_call_external_void_func_byval_struct_i8_i32() #0
 ; VI-NEXT:    s_mov_b64 s[6:7], s[0:1]
 ; VI-NEXT:    s_mov_b64 s[0:1], s[36:37]
 ; VI-NEXT:    s_movk_i32 s32, 0x400
-; VI-NEXT:    s_mov_b64 s[2:3], s[38:39]
 ; VI-NEXT:    s_getpc_b64 s[4:5]
 ; VI-NEXT:    s_add_u32 s4, s4, external_void_func_byval_struct_i8_i32@rel32@lo+4
 ; VI-NEXT:    s_addc_u32 s5, s5, external_void_func_byval_struct_i8_i32@rel32@hi+12
+; VI-NEXT:    s_mov_b64 s[2:3], s[38:39]
 ; VI-NEXT:    s_waitcnt vmcnt(1)
 ; VI-NEXT:    buffer_store_dword v0, off, s[36:39], s32 offset:4
 ; VI-NEXT:    s_waitcnt vmcnt(1)
@@ -4868,10 +4830,10 @@ define amdgpu_kernel void @test_call_external_void_func_byval_struct_i8_i32() #0
 ; CI-NEXT:    s_mov_b64 s[6:7], s[0:1]
 ; CI-NEXT:    s_mov_b64 s[0:1], s[36:37]
 ; CI-NEXT:    s_movk_i32 s32, 0x400
-; CI-NEXT:    s_mov_b64 s[2:3], s[38:39]
 ; CI-NEXT:    s_getpc_b64 s[4:5]
 ; CI-NEXT:    s_add_u32 s4, s4, external_void_func_byval_struct_i8_i32@rel32@lo+4
 ; CI-NEXT:    s_addc_u32 s5, s5, external_void_func_byval_struct_i8_i32@rel32@hi+12
+; CI-NEXT:    s_mov_b64 s[2:3], s[38:39]
 ; CI-NEXT:    s_waitcnt vmcnt(1)
 ; CI-NEXT:    buffer_store_dword v0, off, s[36:39], s32 offset:4
 ; CI-NEXT:    s_waitcnt vmcnt(1)
@@ -4897,10 +4859,10 @@ define amdgpu_kernel void @test_call_external_void_func_byval_struct_i8_i32() #0
 ; GFX9-NEXT:    s_mov_b64 s[6:7], s[0:1]
 ; GFX9-NEXT:    s_mov_b64 s[0:1], s[36:37]
 ; GFX9-NEXT:    s_movk_i32 s32, 0x400
-; GFX9-NEXT:    s_mov_b64 s[2:3], s[38:39]
 ; GFX9-NEXT:    s_getpc_b64 s[4:5]
 ; GFX9-NEXT:    s_add_u32 s4, s4, external_void_func_byval_struct_i8_i32@rel32@lo+4
 ; GFX9-NEXT:    s_addc_u32 s5, s5, external_void_func_byval_struct_i8_i32@rel32@hi+12
+; GFX9-NEXT:    s_mov_b64 s[2:3], s[38:39]
 ; GFX9-NEXT:    s_waitcnt vmcnt(1)
 ; GFX9-NEXT:    buffer_store_dword v0, off, s[36:39], s32 offset:4
 ; GFX9-NEXT:    s_waitcnt vmcnt(1)
@@ -4912,10 +4874,10 @@ define amdgpu_kernel void @test_call_external_void_func_byval_struct_i8_i32() #0
 ; GFX11:       ; %bb.0:
 ; GFX11-NEXT:    v_dual_mov_b32 v0, 3 :: v_dual_mov_b32 v1, 8
 ; GFX11-NEXT:    s_mov_b32 s32, 16
-; GFX11-NEXT:    s_mov_b64 s[6:7], s[0:1]
 ; GFX11-NEXT:    s_getpc_b64 s[2:3]
 ; GFX11-NEXT:    s_add_u32 s2, s2, external_void_func_byval_struct_i8_i32@rel32@lo+4
 ; GFX11-NEXT:    s_addc_u32 s3, s3, external_void_func_byval_struct_i8_i32@rel32@hi+12
+; GFX11-NEXT:    s_mov_b64 s[6:7], s[0:1]
 ; GFX11-NEXT:    s_clause 0x1
 ; GFX11-NEXT:    scratch_store_b8 off, v0, off
 ; GFX11-NEXT:    scratch_store_b32 off, v1, off offset:4
@@ -4939,10 +4901,10 @@ define amdgpu_kernel void @test_call_external_void_func_byval_struct_i8_i32() #0
 ; HSA-NEXT:    buffer_load_dword v1, off, s[0:3], 0
 ; HSA-NEXT:    s_movk_i32 s32, 0x400
 ; HSA-NEXT:    s_mov_b32 flat_scratch_lo, s7
-; HSA-NEXT:    s_mov_b64 s[6:7], s[4:5]
 ; HSA-NEXT:    s_getpc_b64 s[8:9]
 ; HSA-NEXT:    s_add_u32 s8, s8, external_void_func_byval_struct_i8_i32@rel32@lo+4
 ; HSA-NEXT:    s_addc_u32 s9, s9, external_void_func_byval_struct_i8_i32@rel32@hi+12
+; HSA-NEXT:    s_mov_b64 s[6:7], s[4:5]
 ; HSA-NEXT:    s_waitcnt vmcnt(1)
 ; HSA-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:4
 ; HSA-NEXT:    s_waitcnt vmcnt(1)
@@ -4976,10 +4938,10 @@ define amdgpu_kernel void @test_call_external_void_func_sret_struct_i8_i32_byval
 ; VI-NEXT:    s_movk_i32 s32, 0x800
 ; VI-NEXT:    s_mov_b64 s[6:7], s[0:1]
 ; VI-NEXT:    s_mov_b64 s[0:1], s[36:37]
-; VI-NEXT:    s_mov_b64 s[2:3], s[38:39]
 ; VI-NEXT:    s_getpc_b64 s[4:5]
 ; VI-NEXT:    s_add_u32 s4, s4, external_void_func_sret_struct_i8_i32_byval_struct_i8_i32@rel32@lo+4
 ; VI-NEXT:    s_addc_u32 s5, s5, external_void_func_sret_struct_i8_i32_byval_struct_i8_i32@rel32@hi+12
+; VI-NEXT:    s_mov_b64 s[2:3], s[38:39]
 ; VI-NEXT:    s_waitcnt vmcnt(1)
 ; VI-NEXT:    buffer_store_dword v0, off, s[36:39], s32 offset:4
 ; VI-NEXT:    s_waitcnt vmcnt(1)
@@ -5014,10 +4976,10 @@ define amdgpu_kernel void @test_call_external_void_func_sret_struct_i8_i32_byval
 ; CI-NEXT:    s_movk_i32 s32, 0x800
 ; CI-NEXT:    s_mov_b64 s[6:7], s[0:1]
 ; CI-NEXT:    s_mov_b64 s[0:1], s[36:37]
-; CI-NEXT:    s_mov_b64 s[2:3], s[38:39]
 ; CI-NEXT:    s_getpc_b64 s[4:5]
 ; CI-NEXT:    s_add_u32 s4, s4, external_void_func_sret_struct_i8_i32_byval_struct_i8_i32@rel32@lo+4
 ; CI-NEXT:    s_addc_u32 s5, s5, external_void_func_sret_struct_i8_i32_byval_struct_i8_i32@rel32@hi+12
+; CI-NEXT:    s_mov_b64 s[2:3], s[38:39]
 ; CI-NEXT:    s_waitcnt vmcnt(1)
 ; CI-NEXT:    buffer_store_dword v0, off, s[36:39], s32 offset:4
 ; CI-NEXT:    s_waitcnt vmcnt(1)
@@ -5053,10 +5015,10 @@ define amdgpu_kernel void @test_call_external_void_func_sret_struct_i8_i32_byval
 ; GFX9-NEXT:    s_movk_i32 s32, 0x800
 ; GFX9-NEXT:    s_mov_b64 s[6:7], s[0:1]
 ; GFX9-NEXT:    s_mov_b64 s[0:1], s[36:37]
-; GFX9-NEXT:    s_mov_b64 s[2:3], s[38:39]
 ; GFX9-NEXT:    s_getpc_b64 s[4:5]
 ; GFX9-NEXT:    s_add_u32 s4, s4, external_void_func_sret_struct_i8_i32_byval_struct_i8_i32@rel32@lo+4
 ; GFX9-NEXT:    s_addc_u32 s5, s5, external_void_func_sret_struct_i8_i32_byval_struct_i8_i32@rel32@hi+12
+; GFX9-NEXT:    s_mov_b64 s[2:3], s[38:39]
 ; GFX9-NEXT:    s_waitcnt vmcnt(1)
 ; GFX9-NEXT:    buffer_store_dword v0, off, s[36:39], s32 offset:4
 ; GFX9-NEXT:    s_waitcnt vmcnt(1)
@@ -5078,10 +5040,10 @@ define amdgpu_kernel void @test_call_external_void_func_sret_struct_i8_i32_byval
 ; GFX11:       ; %bb.0:
 ; GFX11-NEXT:    v_dual_mov_b32 v0, 3 :: v_dual_mov_b32 v1, 8
 ; GFX11-NEXT:    s_mov_b32 s32, 32
-; GFX11-NEXT:    s_mov_b64 s[6:7], s[0:1]
 ; GFX11-NEXT:    s_getpc_b64 s[2:3]
 ; GFX11-NEXT:    s_add_u32 s2, s2, external_void_func_sret_struct_i8_i32_byval_struct_i8_i32@rel32@lo+4
 ; GFX11-NEXT:    s_addc_u32 s3, s3, external_void_func_sret_struct_i8_i32_byval_struct_i8_i32@rel32@hi+12
+; GFX11-NEXT:    s_mov_b64 s[6:7], s[0:1]
 ; GFX11-NEXT:    s_clause 0x1
 ; GFX11-NEXT:    scratch_store_b8 off, v0, off
 ; GFX11-NEXT:    scratch_store_b32 off, v1, off offset:4
@@ -5118,11 +5080,11 @@ define amdgpu_kernel void @test_call_external_void_func_sret_struct_i8_i32_byval
 ; HSA-NEXT:    buffer_load_dword v0, off, s[0:3], 0 offset:4
 ; HSA-NEXT:    buffer_load_dword v1, off, s[0:3], 0
 ; HSA-NEXT:    s_movk_i32 s32, 0x800
-; HSA-NEXT:    s_mov_b64 s[6:7], s[4:5]
 ; HSA-NEXT:    s_mov_b32 flat_scratch_lo, s9
 ; HSA-NEXT:    s_getpc_b64 s[8:9]
 ; HSA-NEXT:    s_add_u32 s8, s8, external_void_func_sret_struct_i8_i32_byval_struct_i8_i32@rel32@lo+4
 ; HSA-NEXT:    s_addc_u32 s9, s9, external_void_func_sret_struct_i8_i32_byval_struct_i8_i32@rel32@hi+12
+; HSA-NEXT:    s_mov_b64 s[6:7], s[4:5]
 ; HSA-NEXT:    s_waitcnt vmcnt(1)
 ; HSA-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:4
 ; HSA-NEXT:    s_waitcnt vmcnt(1)
@@ -5172,11 +5134,11 @@ define amdgpu_kernel void @test_call_external_void_func_v16i8() #0 {
 ; VI-NEXT:    buffer_load_dwordx4 v[0:3], off, s[0:3], 0
 ; VI-NEXT:    s_addc_u32 s37, s37, 0
 ; VI-NEXT:    s_mov_b64 s[0:1], s[36:37]
-; VI-NEXT:    s_mov_b64 s[2:3], s[38:39]
-; VI-NEXT:    s_mov_b32 s32, 0
 ; VI-NEXT:    s_getpc_b64 s[4:5]
 ; VI-NEXT:    s_add_u32 s4, s4, external_void_func_v16i8@rel32@lo+4
 ; VI-NEXT:    s_addc_u32 s5, s5, external_void_func_v16i8@rel32@hi+12
+; VI-NEXT:    s_mov_b64 s[2:3], s[38:39]
+; VI-NEXT:    s_mov_b32 s32, 0
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_lshrrev_b32_e32 v16, 8, v0
 ; VI-NEXT:    v_lshrrev_b32_e32 v17, 16, v0
@@ -5214,11 +5176,11 @@ define amdgpu_kernel void @test_call_external_void_func_v16i8() #0 {
 ; CI-NEXT:    buffer_load_dwordx4 v[0:3], off, s[0:3], 0
 ; CI-NEXT:    s_addc_u32 s37, s37, 0
 ; CI-NEXT:    s_mov_b64 s[0:1], s[36:37]
-; CI-NEXT:    s_mov_b64 s[2:3], s[38:39]
-; CI-NEXT:    s_mov_b32 s32, 0
 ; CI-NEXT:    s_getpc_b64 s[4:5]
 ; CI-NEXT:    s_add_u32 s4, s4, external_void_func_v16i8@rel32@lo+4
 ; CI-NEXT:    s_addc_u32 s5, s5, external_void_func_v16i8@rel32@hi+12
+; CI-NEXT:    s_mov_b64 s[2:3], s[38:39]
+; CI-NEXT:    s_mov_b32 s32, 0
 ; CI-NEXT:    s_waitcnt vmcnt(0)
 ; CI-NEXT:    v_lshrrev_b32_e32 v16, 8, v0
 ; CI-NEXT:    v_lshrrev_b32_e32 v17, 16, v0
@@ -5256,11 +5218,11 @@ define amdgpu_kernel void @test_call_external_void_func_v16i8() #0 {
 ; GFX9-NEXT:    buffer_load_dwordx4 v[0:3], off, s[0:3], 0
 ; GFX9-NEXT:    s_addc_u32 s37, s37, 0
 ; GFX9-NEXT:    s_mov_b64 s[0:1], s[36:37]
-; GFX9-NEXT:    s_mov_b64 s[2:3], s[38:39]
-; GFX9-NEXT:    s_mov_b32 s32, 0
 ; GFX9-NEXT:    s_getpc_b64 s[4:5]
 ; GFX9-NEXT:    s_add_u32 s4, s4, external_void_func_v16i8@rel32@lo+4
 ; GFX9-NEXT:    s_addc_u32 s5, s5, external_void_func_v16i8@rel32@hi+12
+; GFX9-NEXT:    s_mov_b64 s[2:3], s[38:39]
+; GFX9-NEXT:    s_mov_b32 s32, 0
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_lshrrev_b32_e32 v16, 8, v0
 ; GFX9-NEXT:    v_lshrrev_b32_e32 v17, 16, v0
@@ -5288,10 +5250,10 @@ define amdgpu_kernel void @test_call_external_void_func_v16i8() #0 {
 ; GFX11-NEXT:    s_load_b64 s[4:5], s[0:1], 0x0
 ; GFX11-NEXT:    s_mov_b32 s7, 0x31016000
 ; GFX11-NEXT:    s_mov_b32 s6, -1
-; GFX11-NEXT:    s_mov_b32 s32, 0
 ; GFX11-NEXT:    s_getpc_b64 s[2:3]
 ; GFX11-NEXT:    s_add_u32 s2, s2, external_void_func_v16i8@rel32@lo+4
 ; GFX11-NEXT:    s_addc_u32 s3, s3, external_void_func_v16i8@rel32@hi+12
+; GFX11-NEXT:    s_mov_b32 s32, 0
 ; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX11-NEXT:    buffer_load_b128 v[0:3], off, s[4:7], 0
 ; GFX11-NEXT:    s_mov_b64 s[6:7], s[0:1]
@@ -5323,15 +5285,15 @@ define amdgpu_kernel void @test_call_external_void_func_v16i8() #0 {
 ; HSA-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; HSA-NEXT:    s_mov_b32 s11, 0x1100f000
 ; HSA-NEXT:    s_mov_b32 s10, -1
-; HSA-NEXT:    s_mov_b32 flat_scratch_lo, s7
 ; HSA-NEXT:    s_addc_u32 s1, s1, 0
+; HSA-NEXT:    s_mov_b32 flat_scratch_lo, s7
 ; HSA-NEXT:    s_waitcnt lgkmcnt(0)
 ; HSA-NEXT:    buffer_load_dwordx4 v[0:3], off, s[8:11], 0
-; HSA-NEXT:    s_mov_b64 s[6:7], s[4:5]
-; HSA-NEXT:    s_mov_b32 s32, 0
 ; HSA-NEXT:    s_getpc_b64 s[8:9]
 ; HSA-NEXT:    s_add_u32 s8, s8, external_void_func_v16i8@rel32@lo+4
 ; HSA-NEXT:    s_addc_u32 s9, s9, external_void_func_v16i8@rel32@hi+12
+; HSA-NEXT:    s_mov_b64 s[6:7], s[4:5]
+; HSA-NEXT:    s_mov_b32 s32, 0
 ; HSA-NEXT:    s_waitcnt vmcnt(0)
 ; HSA-NEXT:    v_lshrrev_b32_e32 v16, 8, v0
 ; HSA-NEXT:    v_lshrrev_b32_e32 v17, 16, v0
@@ -5381,6 +5343,9 @@ define amdgpu_kernel void @stack_passed_arg_alignment_v32i32_f64(<32 x i32> %val
 ; VI-NEXT:    s_mov_b64 s[6:7], s[0:1]
 ; VI-NEXT:    s_mov_b64 s[0:1], s[52:53]
 ; VI-NEXT:    buffer_store_dword v0, off, s[52:55], s32 offset:8
+; VI-NEXT:    s_getpc_b64 s[4:5]
+; VI-NEXT:    s_add_u32 s4, s4, stack_passed_f64_arg@rel32@lo+4
+; VI-NEXT:    s_addc_u32 s5, s5, stack_passed_f64_arg@rel32@hi+12
 ; VI-NEXT:    s_mov_b64 s[2:3], s[54:55]
 ; VI-NEXT:    v_mov_b32_e32 v0, s36
 ; VI-NEXT:    v_mov_b32_e32 v1, s37
@@ -5413,9 +5378,6 @@ define amdgpu_kernel void @stack_passed_arg_alignment_v32i32_f64(<32 x i32> %val
 ; VI-NEXT:    v_mov_b32_e32 v28, s20
 ; VI-NEXT:    v_mov_b32_e32 v29, s21
 ; VI-NEXT:    v_mov_b32_e32 v30, s22
-; VI-NEXT:    s_getpc_b64 s[4:5]
-; VI-NEXT:    s_add_u32 s4, s4, stack_passed_f64_arg@rel32@lo+4
-; VI-NEXT:    s_addc_u32 s5, s5, stack_passed_f64_arg@rel32@hi+12
 ; VI-NEXT:    s_swappc_b64 s[30:31], s[4:5]
 ; VI-NEXT:    s_endpgm
 ;
@@ -5440,6 +5402,9 @@ define amdgpu_kernel void @stack_passed_arg_alignment_v32i32_f64(<32 x i32> %val
 ; CI-NEXT:    s_mov_b64 s[6:7], s[0:1]
 ; CI-NEXT:    s_mov_b64 s[0:1], s[52:53]
 ; CI-NEXT:    buffer_store_dword v0, off, s[52:55], s32 offset:8
+; CI-NEXT:    s_getpc_b64 s[4:5]
+; CI-NEXT:    s_add_u32 s4, s4, stack_passed_f64_arg@rel32@lo+4
+; CI-NEXT:    s_addc_u32 s5, s5, stack_passed_f64_arg@rel32@hi+12
 ; CI-NEXT:    s_mov_b64 s[2:3], s[54:55]
 ; CI-NEXT:    v_mov_b32_e32 v0, s36
 ; CI-NEXT:    v_mov_b32_e32 v1, s37
@@ -5472,9 +5437,6 @@ define amdgpu_kernel void @stack_passed_arg_alignment_v32i32_f64(<32 x i32> %val
 ; CI-NEXT:    v_mov_b32_e32 v28, s20
 ; CI-NEXT:    v_mov_b32_e32 v29, s21
 ; CI-NEXT:    v_mov_b32_e32 v30, s22
-; CI-NEXT:    s_getpc_b64 s[4:5]
-; CI-NEXT:    s_add_u32 s4, s4, stack_passed_f64_arg@rel32@lo+4
-; CI-NEXT:    s_addc_u32 s5, s5, stack_passed_f64_arg@rel32@hi+12
 ; CI-NEXT:    s_swappc_b64 s[30:31], s[4:5]
 ; CI-NEXT:    s_endpgm
 ;
@@ -5499,6 +5461,9 @@ define amdgpu_kernel void @stack_passed_arg_alignment_v32i32_f64(<32 x i32> %val
 ; GFX9-NEXT:    s_mov_b64 s[6:7], s[0:1]
 ; GFX9-NEXT:    s_mov_b64 s[0:1], s[52:53]
 ; GFX9-NEXT:    buffer_store_dword v0, off, s[52:55], s32 offset:8
+; GFX9-NEXT:    s_getpc_b64 s[4:5]
+; GFX9-NEXT:    s_add_u32 s4, s4, stack_passed_f64_arg@rel32@lo+4
+; GFX9-NEXT:    s_addc_u32 s5, s5, stack_passed_f64_arg@rel32@hi+12
 ; GFX9-NEXT:    s_mov_b64 s[2:3], s[54:55]
 ; GFX9-NEXT:    v_mov_b32_e32 v0, s36
 ; GFX9-NEXT:    v_mov_b32_e32 v1, s37
@@ -5531,9 +5496,6 @@ define amdgpu_kernel void @stack_passed_arg_alignment_v32i32_f64(<32 x i32> %val
 ; GFX9-NEXT:    v_mov_b32_e32 v28, s20
 ; GFX9-NEXT:    v_mov_b32_e32 v29, s21
 ; GFX9-NEXT:    v_mov_b32_e32 v30, s22
-; GFX9-NEXT:    s_getpc_b64 s[4:5]
-; GFX9-NEXT:    s_add_u32 s4, s4, stack_passed_f64_arg@rel32@lo+4
-; GFX9-NEXT:    s_addc_u32 s5, s5, stack_passed_f64_arg@rel32@hi+12
 ; GFX9-NEXT:    s_swappc_b64 s[30:31], s[4:5]
 ; GFX9-NEXT:    s_endpgm
 ;
@@ -5569,11 +5531,10 @@ define amdgpu_kernel void @stack_passed_arg_alignment_v32i32_f64(<32 x i32> %val
 ; GFX11-NEXT:    v_dual_mov_b32 v27, s15 :: v_dual_mov_b32 v26, s14
 ; GFX11-NEXT:    v_dual_mov_b32 v29, s17 :: v_dual_mov_b32 v28, s16
 ; GFX11-NEXT:    v_mov_b32_e32 v30, s18
-; GFX11-NEXT:    s_mov_b64 s[6:7], s[0:1]
 ; GFX11-NEXT:    s_getpc_b64 s[2:3]
 ; GFX11-NEXT:    s_add_u32 s2, s2, stack_passed_f64_arg@rel32@lo+4
 ; GFX11-NEXT:    s_addc_u32 s3, s3, stack_passed_f64_arg@rel32@hi+12
-; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_mov_b64 s[6:7], s[0:1]
 ; GFX11-NEXT:    s_swappc_b64 s[30:31], s[2:3]
 ; GFX11-NEXT:    s_endpgm
 ;
@@ -5595,6 +5556,9 @@ define amdgpu_kernel void @stack_passed_arg_alignment_v32i32_f64(<32 x i32> %val
 ; HSA-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:4
 ; HSA-NEXT:    v_mov_b32_e32 v0, s25
 ; HSA-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:8
+; HSA-NEXT:    s_getpc_b64 s[24:25]
+; HSA-NEXT:    s_add_u32 s24, s24, stack_passed_f64_arg@rel32@lo+4
+; HSA-NEXT:    s_addc_u32 s25, s25, stack_passed_f64_arg@rel32@hi+12
 ; HSA-NEXT:    s_mov_b64 s[6:7], s[4:5]
 ; HSA-NEXT:    v_mov_b32_e32 v0, s36
 ; HSA-NEXT:    v_mov_b32_e32 v1, s37
@@ -5627,9 +5591,6 @@ define amdgpu_kernel void @stack_passed_arg_alignment_v32i32_f64(<32 x i32> %val
 ; HSA-NEXT:    v_mov_b32_e32 v28, s20
 ; HSA-NEXT:    v_mov_b32_e32 v29, s21
 ; HSA-NEXT:    v_mov_b32_e32 v30, s22
-; HSA-NEXT:    s_getpc_b64 s[24:25]
-; HSA-NEXT:    s_add_u32 s24, s24, stack_passed_f64_arg@rel32@lo+4
-; HSA-NEXT:    s_addc_u32 s25, s25, stack_passed_f64_arg@rel32@hi+12
 ; HSA-NEXT:    s_swappc_b64 s[30:31], s[24:25]
 ; HSA-NEXT:    s_endpgm
 entry:
@@ -5835,6 +5796,9 @@ define void @stack_12xv3i32() #0 {
 ; VI-NEXT:    v_mov_b32_e32 v0, 15
 ; VI-NEXT:    v_writelane_b32 v40, s30, 0
 ; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:16
+; VI-NEXT:    s_getpc_b64 s[4:5]
+; VI-NEXT:    s_add_u32 s4, s4, external_void_func_12xv3i32@rel32@lo+4
+; VI-NEXT:    s_addc_u32 s5, s5, external_void_func_12xv3i32@rel32@hi+12
 ; VI-NEXT:    v_mov_b32_e32 v0, 0
 ; VI-NEXT:    v_mov_b32_e32 v1, 0
 ; VI-NEXT:    v_mov_b32_e32 v2, 0
@@ -5867,9 +5831,6 @@ define void @stack_12xv3i32() #0 {
 ; VI-NEXT:    v_mov_b32_e32 v29, 9
 ; VI-NEXT:    v_mov_b32_e32 v30, 10
 ; VI-NEXT:    v_writelane_b32 v40, s31, 1
-; VI-NEXT:    s_getpc_b64 s[4:5]
-; VI-NEXT:    s_add_u32 s4, s4, external_void_func_12xv3i32@rel32@lo+4
-; VI-NEXT:    s_addc_u32 s5, s5, external_void_func_12xv3i32@rel32@hi+12
 ; VI-NEXT:    s_swappc_b64 s[30:31], s[4:5]
 ; VI-NEXT:    v_readlane_b32 s31, v40, 1
 ; VI-NEXT:    v_readlane_b32 s30, v40, 0
@@ -5903,6 +5864,9 @@ define void @stack_12xv3i32() #0 {
 ; CI-NEXT:    v_mov_b32_e32 v0, 15
 ; CI-NEXT:    v_writelane_b32 v40, s30, 0
 ; CI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:16
+; CI-NEXT:    s_getpc_b64 s[4:5]
+; CI-NEXT:    s_add_u32 s4, s4, external_void_func_12xv3i32@rel32@lo+4
+; CI-NEXT:    s_addc_u32 s5, s5, external_void_func_12xv3i32@rel32@hi+12
 ; CI-NEXT:    v_mov_b32_e32 v0, 0
 ; CI-NEXT:    v_mov_b32_e32 v1, 0
 ; CI-NEXT:    v_mov_b32_e32 v2, 0
@@ -5935,9 +5899,6 @@ define void @stack_12xv3i32() #0 {
 ; CI-NEXT:    v_mov_b32_e32 v29, 9
 ; CI-NEXT:    v_mov_b32_e32 v30, 10
 ; CI-NEXT:    v_writelane_b32 v40, s31, 1
-; CI-NEXT:    s_getpc_b64 s[4:5]
-; CI-NEXT:    s_add_u32 s4, s4, external_void_func_12xv3i32@rel32@lo+4
-; CI-NEXT:    s_addc_u32 s5, s5, external_void_func_12xv3i32@rel32@hi+12
 ; CI-NEXT:    s_swappc_b64 s[30:31], s[4:5]
 ; CI-NEXT:    v_readlane_b32 s31, v40, 1
 ; CI-NEXT:    v_readlane_b32 s30, v40, 0
@@ -5971,6 +5932,9 @@ define void @stack_12xv3i32() #0 {
 ; GFX9-NEXT:    v_mov_b32_e32 v0, 15
 ; GFX9-NEXT:    v_writelane_b32 v40, s30, 0
 ; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:16
+; GFX9-NEXT:    s_getpc_b64 s[4:5]
+; GFX9-NEXT:    s_add_u32 s4, s4, external_void_func_12xv3i32@rel32@lo+4
+; GFX9-NEXT:    s_addc_u32 s5, s5, external_void_func_12xv3i32@rel32@hi+12
 ; GFX9-NEXT:    v_mov_b32_e32 v0, 0
 ; GFX9-NEXT:    v_mov_b32_e32 v1, 0
 ; GFX9-NEXT:    v_mov_b32_e32 v2, 0
@@ -6003,9 +5967,6 @@ define void @stack_12xv3i32() #0 {
 ; GFX9-NEXT:    v_mov_b32_e32 v29, 9
 ; GFX9-NEXT:    v_mov_b32_e32 v30, 10
 ; GFX9-NEXT:    v_writelane_b32 v40, s31, 1
-; GFX9-NEXT:    s_getpc_b64 s[4:5]
-; GFX9-NEXT:    s_add_u32 s4, s4, external_void_func_12xv3i32@rel32@lo+4
-; GFX9-NEXT:    s_addc_u32 s5, s5, external_void_func_12xv3i32@rel32@hi+12
 ; GFX9-NEXT:    s_swappc_b64 s[30:31], s[4:5]
 ; GFX9-NEXT:    v_readlane_b32 s31, v40, 1
 ; GFX9-NEXT:    v_readlane_b32 s30, v40, 0
@@ -6051,12 +6012,12 @@ define void @stack_12xv3i32() #0 {
 ; GFX11-NEXT:    v_dual_mov_b32 v27, 9 :: v_dual_mov_b32 v26, 8
 ; GFX11-NEXT:    v_dual_mov_b32 v29, 9 :: v_dual_mov_b32 v28, 9
 ; GFX11-NEXT:    v_mov_b32_e32 v30, 10
-; GFX11-NEXT:    v_writelane_b32 v40, s31, 1
 ; GFX11-NEXT:    s_getpc_b64 s[0:1]
 ; GFX11-NEXT:    s_add_u32 s0, s0, external_void_func_12xv3i32@rel32@lo+4
 ; GFX11-NEXT:    s_addc_u32 s1, s1, external_void_func_12xv3i32@rel32@hi+12
-; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    v_writelane_b32 v40, s31, 1
 ; GFX11-NEXT:    s_swappc_b64 s[30:31], s[0:1]
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX11-NEXT:    v_readlane_b32 s31, v40, 1
 ; GFX11-NEXT:    v_readlane_b32 s30, v40, 0
 ; GFX11-NEXT:    s_mov_b32 s32, s33
@@ -6089,6 +6050,9 @@ define void @stack_12xv3i32() #0 {
 ; HSA-NEXT:    v_mov_b32_e32 v0, 15
 ; HSA-NEXT:    v_writelane_b32 v40, s30, 0
 ; HSA-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:16
+; HSA-NEXT:    s_getpc_b64 s[4:5]
+; HSA-NEXT:    s_add_u32 s4, s4, external_void_func_12xv3i32@rel32@lo+4
+; HSA-NEXT:    s_addc_u32 s5, s5, external_void_func_12xv3i32@rel32@hi+12
 ; HSA-NEXT:    v_mov_b32_e32 v0, 0
 ; HSA-NEXT:    v_mov_b32_e32 v1, 0
 ; HSA-NEXT:    v_mov_b32_e32 v2, 0
@@ -6121,9 +6085,6 @@ define void @stack_12xv3i32() #0 {
 ; HSA-NEXT:    v_mov_b32_e32 v29, 9
 ; HSA-NEXT:    v_mov_b32_e32 v30, 10
 ; HSA-NEXT:    v_writelane_b32 v40, s31, 1
-; HSA-NEXT:    s_getpc_b64 s[4:5]
-; HSA-NEXT:    s_add_u32 s4, s4, external_void_func_12xv3i32@rel32@lo+4
-; HSA-NEXT:    s_addc_u32 s5, s5, external_void_func_12xv3i32@rel32@hi+12
 ; HSA-NEXT:    s_swappc_b64 s[30:31], s[4:5]
 ; HSA-NEXT:    v_readlane_b32 s31, v40, 1
 ; HSA-NEXT:    v_readlane_b32 s30, v40, 0
@@ -6174,6 +6135,9 @@ define void @stack_12xv3f32() #0 {
 ; VI-NEXT:    v_mov_b32_e32 v0, 0x41700000
 ; VI-NEXT:    v_writelane_b32 v40, s30, 0
 ; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:16
+; VI-NEXT:    s_getpc_b64 s[4:5]
+; VI-NEXT:    s_add_u32 s4, s4, external_void_func_12xv3f32@rel32@lo+4
+; VI-NEXT:    s_addc_u32 s5, s5, external_void_func_12xv3f32@rel32@hi+12
 ; VI-NEXT:    v_mov_b32_e32 v0, 0
 ; VI-NEXT:    v_mov_b32_e32 v1, 0
 ; VI-NEXT:    v_mov_b32_e32 v2, 0
@@ -6206,9 +6170,6 @@ define void @stack_12xv3f32() #0 {
 ; VI-NEXT:    v_mov_b32_e32 v29, 0x41100000
 ; VI-NEXT:    v_mov_b32_e32 v30, 0x41200000
 ; VI-NEXT:    v_writelane_b32 v40, s31, 1
-; VI-NEXT:    s_getpc_b64 s[4:5]
-; VI-NEXT:    s_add_u32 s4, s4, external_void_func_12xv3f32@rel32@lo+4
-; VI-NEXT:    s_addc_u32 s5, s5, external_void_func_12xv3f32@rel32@hi+12
 ; VI-NEXT:    s_swappc_b64 s[30:31], s[4:5]
 ; VI-NEXT:    v_readlane_b32 s31, v40, 1
 ; VI-NEXT:    v_readlane_b32 s30, v40, 0
@@ -6242,6 +6203,9 @@ define void @stack_12xv3f32() #0 {
 ; CI-NEXT:    v_mov_b32_e32 v0, 0x41700000
 ; CI-NEXT:    v_writelane_b32 v40, s30, 0
 ; CI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:16
+; CI-NEXT:    s_getpc_b64 s[4:5]
+; CI-NEXT:    s_add_u32 s4, s4, external_void_func_12xv3f32@rel32@lo+4
+; CI-NEXT:    s_addc_u32 s5, s5, external_void_func_12xv3f32@rel32@hi+12
 ; CI-NEXT:    v_mov_b32_e32 v0, 0
 ; CI-NEXT:    v_mov_b32_e32 v1, 0
 ; CI-NEXT:    v_mov_b32_e32 v2, 0
@@ -6274,9 +6238,6 @@ define void @stack_12xv3f32() #0 {
 ; CI-NEXT:    v_mov_b32_e32 v29, 0x41100000
 ; CI-NEXT:    v_mov_b32_e32 v30, 0x41200000
 ; CI-NEXT:    v_writelane_b32 v40, s31, 1
-; CI-NEXT:    s_getpc_b64 s[4:5]
-; CI-NEXT:    s_add_u32 s4, s4, external_void_func_12xv3f32@rel32@lo+4
-; CI-NEXT:    s_addc_u32 s5, s5, external_void_func_12xv3f32@rel32@hi+12
 ; CI-NEXT:    s_swappc_b64 s[30:31], s[4:5]
 ; CI-NEXT:    v_readlane_b32 s31, v40, 1
 ; CI-NEXT:    v_readlane_b32 s30, v40, 0
@@ -6310,6 +6271,9 @@ define void @stack_12xv3f32() #0 {
 ; GFX9-NEXT:    v_mov_b32_e32 v0, 0x41700000
 ; GFX9-NEXT:    v_writelane_b32 v40, s30, 0
 ; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:16
+; GFX9-NEXT:    s_getpc_b64 s[4:5]
+; GFX9-NEXT:    s_add_u32 s4, s4, external_void_func_12xv3f32@rel32@lo+4
+; GFX9-NEXT:    s_addc_u32 s5, s5, external_void_func_12xv3f32@rel32@hi+12
 ; GFX9-NEXT:    v_mov_b32_e32 v0, 0
 ; GFX9-NEXT:    v_mov_b32_e32 v1, 0
 ; GFX9-NEXT:    v_mov_b32_e32 v2, 0
@@ -6342,9 +6306,6 @@ define void @stack_12xv3f32() #0 {
 ; GFX9-NEXT:    v_mov_b32_e32 v29, 0x41100000
 ; GFX9-NEXT:    v_mov_b32_e32 v30, 0x41200000
 ; GFX9-NEXT:    v_writelane_b32 v40, s31, 1
-; GFX9-NEXT:    s_getpc_b64 s[4:5]
-; GFX9-NEXT:    s_add_u32 s4, s4, external_void_func_12xv3f32@rel32@lo+4
-; GFX9-NEXT:    s_addc_u32 s5, s5, external_void_func_12xv3f32@rel32@hi+12
 ; GFX9-NEXT:    s_swappc_b64 s[30:31], s[4:5]
 ; GFX9-NEXT:    v_readlane_b32 s31, v40, 1
 ; GFX9-NEXT:    v_readlane_b32 s30, v40, 0
@@ -6394,12 +6355,12 @@ define void @stack_12xv3f32() #0 {
 ; GFX11-NEXT:    v_dual_mov_b32 v27, 0x41100000 :: v_dual_mov_b32 v28, 0x41100000
 ; GFX11-NEXT:    v_mov_b32_e32 v29, 0x41100000
 ; GFX11-NEXT:    v_mov_b32_e32 v30, 0x41200000
-; GFX11-NEXT:    v_writelane_b32 v40, s31, 1
 ; GFX11-NEXT:    s_getpc_b64 s[0:1]
 ; GFX11-NEXT:    s_add_u32 s0, s0, external_void_func_12xv3f32@rel32@lo+4
 ; GFX11-NEXT:    s_addc_u32 s1, s1, external_void_func_12xv3f32@rel32@hi+12
-; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    v_writelane_b32 v40, s31, 1
 ; GFX11-NEXT:    s_swappc_b64 s[30:31], s[0:1]
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX11-NEXT:    v_readlane_b32 s31, v40, 1
 ; GFX11-NEXT:    v_readlane_b32 s30, v40, 0
 ; GFX11-NEXT:    s_mov_b32 s32, s33
@@ -6432,6 +6393,9 @@ define void @stack_12xv3f32() #0 {
 ; HSA-NEXT:    v_mov_b32_e32 v0, 0x41700000
 ; HSA-NEXT:    v_writelane_b32 v40, s30, 0
 ; HSA-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:16
+; HSA-NEXT:    s_getpc_b64 s[4:5]
+; HSA-NEXT:    s_add_u32 s4, s4, external_void_func_12xv3f32@rel32@lo+4
+; HSA-NEXT:    s_addc_u32 s5, s5, external_void_func_12xv3f32@rel32@hi+12
 ; HSA-NEXT:    v_mov_b32_e32 v0, 0
 ; HSA-NEXT:    v_mov_b32_e32 v1, 0
 ; HSA-NEXT:    v_mov_b32_e32 v2, 0
@@ -6464,9 +6428,6 @@ define void @stack_12xv3f32() #0 {
 ; HSA-NEXT:    v_mov_b32_e32 v29, 0x41100000
 ; HSA-NEXT:    v_mov_b32_e32 v30, 0x41200000
 ; HSA-NEXT:    v_writelane_b32 v40, s31, 1
-; HSA-NEXT:    s_getpc_b64 s[4:5]
-; HSA-NEXT:    s_add_u32 s4, s4, external_void_func_12xv3f32@rel32@lo+4
-; HSA-NEXT:    s_addc_u32 s5, s5, external_void_func_12xv3f32@rel32@hi+12
 ; HSA-NEXT:    s_swappc_b64 s[30:31], s[4:5]
 ; HSA-NEXT:    v_readlane_b32 s31, v40, 1
 ; HSA-NEXT:    v_readlane_b32 s30, v40, 0
@@ -6525,6 +6486,9 @@ define void @stack_8xv5i32() #0 {
 ; VI-NEXT:    v_mov_b32_e32 v0, 15
 ; VI-NEXT:    v_writelane_b32 v40, s30, 0
 ; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:32
+; VI-NEXT:    s_getpc_b64 s[4:5]
+; VI-NEXT:    s_add_u32 s4, s4, external_void_func_8xv5i32@rel32@lo+4
+; VI-NEXT:    s_addc_u32 s5, s5, external_void_func_8xv5i32@rel32@hi+12
 ; VI-NEXT:    v_mov_b32_e32 v0, 0
 ; VI-NEXT:    v_mov_b32_e32 v1, 0
 ; VI-NEXT:    v_mov_b32_e32 v2, 0
@@ -6557,9 +6521,6 @@ define void @stack_8xv5i32() #0 {
 ; VI-NEXT:    v_mov_b32_e32 v29, 5
 ; VI-NEXT:    v_mov_b32_e32 v30, 6
 ; VI-NEXT:    v_writelane_b32 v40, s31, 1
-; VI-NEXT:    s_getpc_b64 s[4:5]
-; VI-NEXT:    s_add_u32 s4, s4, external_void_func_8xv5i32@rel32@lo+4
-; VI-NEXT:    s_addc_u32 s5, s5, external_void_func_8xv5i32@rel32@hi+12
 ; VI-NEXT:    s_swappc_b64 s[30:31], s[4:5]
 ; VI-NEXT:    v_readlane_b32 s31, v40, 1
 ; VI-NEXT:    v_readlane_b32 s30, v40, 0
@@ -6601,6 +6562,9 @@ define void @stack_8xv5i32() #0 {
 ; CI-NEXT:    v_mov_b32_e32 v0, 15
 ; CI-NEXT:    v_writelane_b32 v40, s30, 0
 ; CI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:32
+; CI-NEXT:    s_getpc_b64 s[4:5]
+; CI-NEXT:    s_add_u32 s4, s4, external_void_func_8xv5i32@rel32@lo+4
+; CI-NEXT:    s_addc_u32 s5, s5, external_void_func_8xv5i32@rel32@hi+12
 ; CI-NEXT:    v_mov_b32_e32 v0, 0
 ; CI-NEXT:    v_mov_b32_e32 v1, 0
 ; CI-NEXT:    v_mov_b32_e32 v2, 0
@@ -6633,9 +6597,6 @@ define void @stack_8xv5i32() #0 {
 ; CI-NEXT:    v_mov_b32_e32 v29, 5
 ; CI-NEXT:    v_mov_b32_e32 v30, 6
 ; CI-NEXT:    v_writelane_b32 v40, s31, 1
-; CI-NEXT:    s_getpc_b64 s[4:5]
-; CI-NEXT:    s_add_u32 s4, s4, external_void_func_8xv5i32@rel32@lo+4
-; CI-NEXT:    s_addc_u32 s5, s5, external_void_func_8xv5i32@rel32@hi+12
 ; CI-NEXT:    s_swappc_b64 s[30:31], s[4:5]
 ; CI-NEXT:    v_readlane_b32 s31, v40, 1
 ; CI-NEXT:    v_readlane_b32 s30, v40, 0
@@ -6677,6 +6638,9 @@ define void @stack_8xv5i32() #0 {
 ; GFX9-NEXT:    v_mov_b32_e32 v0, 15
 ; GFX9-NEXT:    v_writelane_b32 v40, s30, 0
 ; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:32
+; GFX9-NEXT:    s_getpc_b64 s[4:5]
+; GFX9-NEXT:    s_add_u32 s4, s4, external_void_func_8xv5i32@rel32@lo+4
+; GFX9-NEXT:    s_addc_u32 s5, s5, external_void_func_8xv5i32@rel32@hi+12
 ; GFX9-NEXT:    v_mov_b32_e32 v0, 0
 ; GFX9-NEXT:    v_mov_b32_e32 v1, 0
 ; GFX9-NEXT:    v_mov_b32_e32 v2, 0
@@ -6709,9 +6673,6 @@ define void @stack_8xv5i32() #0 {
 ; GFX9-NEXT:    v_mov_b32_e32 v29, 5
 ; GFX9-NEXT:    v_mov_b32_e32 v30, 6
 ; GFX9-NEXT:    v_writelane_b32 v40, s31, 1
-; GFX9-NEXT:    s_getpc_b64 s[4:5]
-; GFX9-NEXT:    s_add_u32 s4, s4, external_void_func_8xv5i32@rel32@lo+4
-; GFX9-NEXT:    s_addc_u32 s5, s5, external_void_func_8xv5i32@rel32@hi+12
 ; GFX9-NEXT:    s_swappc_b64 s[30:31], s[4:5]
 ; GFX9-NEXT:    v_readlane_b32 s31, v40, 1
 ; GFX9-NEXT:    v_readlane_b32 s30, v40, 0
@@ -6762,12 +6723,12 @@ define void @stack_8xv5i32() #0 {
 ; GFX11-NEXT:    v_dual_mov_b32 v26, 5 :: v_dual_mov_b32 v29, 5
 ; GFX11-NEXT:    v_mov_b32_e32 v28, 5
 ; GFX11-NEXT:    v_mov_b32_e32 v30, 6
-; GFX11-NEXT:    v_writelane_b32 v40, s31, 1
 ; GFX11-NEXT:    s_getpc_b64 s[0:1]
 ; GFX11-NEXT:    s_add_u32 s0, s0, external_void_func_8xv5i32@rel32@lo+4
 ; GFX11-NEXT:    s_addc_u32 s1, s1, external_void_func_8xv5i32@rel32@hi+12
-; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    v_writelane_b32 v40, s31, 1
 ; GFX11-NEXT:    s_swappc_b64 s[30:31], s[0:1]
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX11-NEXT:    v_readlane_b32 s31, v40, 1
 ; GFX11-NEXT:    v_readlane_b32 s30, v40, 0
 ; GFX11-NEXT:    s_mov_b32 s32, s33
@@ -6808,6 +6769,9 @@ define void @stack_8xv5i32() #0 {
 ; HSA-NEXT:    v_mov_b32_e32 v0, 15
 ; HSA-NEXT:    v_writelane_b32 v40, s30, 0
 ; HSA-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:32
+; HSA-NEXT:    s_getpc_b64 s[4:5]
+; HSA-NEXT:    s_add_u32 s4, s4, external_void_func_8xv5i32@rel32@lo+4
+; HSA-NEXT:    s_addc_u32 s5, s5, external_void_func_8xv5i32@rel32@hi+12
 ; HSA-NEXT:    v_mov_b32_e32 v0, 0
 ; HSA-NEXT:    v_mov_b32_e32 v1, 0
 ; HSA-NEXT:    v_mov_b32_e32 v2, 0
@@ -6840,9 +6804,6 @@ define void @stack_8xv5i32() #0 {
 ; HSA-NEXT:    v_mov_b32_e32 v29, 5
 ; HSA-NEXT:    v_mov_b32_e32 v30, 6
 ; HSA-NEXT:    v_writelane_b32 v40, s31, 1
-; HSA-NEXT:    s_getpc_b64 s[4:5]
-; HSA-NEXT:    s_add_u32 s4, s4, external_void_func_8xv5i32@rel32@lo+4
-; HSA-NEXT:    s_addc_u32 s5, s5, external_void_func_8xv5i32@rel32@hi+12
 ; HSA-NEXT:    s_swappc_b64 s[30:31], s[4:5]
 ; HSA-NEXT:    v_readlane_b32 s31, v40, 1
 ; HSA-NEXT:    v_readlane_b32 s30, v40, 0
@@ -6897,6 +6858,9 @@ define void @stack_8xv5f32() #0 {
 ; VI-NEXT:    v_mov_b32_e32 v0, 0x41700000
 ; VI-NEXT:    v_writelane_b32 v40, s30, 0
 ; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:32
+; VI-NEXT:    s_getpc_b64 s[4:5]
+; VI-NEXT:    s_add_u32 s4, s4, external_void_func_8xv5f32@rel32@lo+4
+; VI-NEXT:    s_addc_u32 s5, s5, external_void_func_8xv5f32@rel32@hi+12
 ; VI-NEXT:    v_mov_b32_e32 v0, 0
 ; VI-NEXT:    v_mov_b32_e32 v1, 0
 ; VI-NEXT:    v_mov_b32_e32 v2, 0
@@ -6929,9 +6893,6 @@ define void @stack_8xv5f32() #0 {
 ; VI-NEXT:    v_mov_b32_e32 v29, 0x40a00000
 ; VI-NEXT:    v_mov_b32_e32 v30, 0x40c00000
 ; VI-NEXT:    v_writelane_b32 v40, s31, 1
-; VI-NEXT:    s_getpc_b64 s[4:5]
-; VI-NEXT:    s_add_u32 s4, s4, external_void_func_8xv5f32@rel32@lo+4
-; VI-NEXT:    s_addc_u32 s5, s5, external_void_func_8xv5f32@rel32@hi+12
 ; VI-NEXT:    s_swappc_b64 s[30:31], s[4:5]
 ; VI-NEXT:    v_readlane_b32 s31, v40, 1
 ; VI-NEXT:    v_readlane_b32 s30, v40, 0
@@ -6973,6 +6934,9 @@ define void @stack_8xv5f32() #0 {
 ; CI-NEXT:    v_mov_b32_e32 v0, 0x41700000
 ; CI-NEXT:    v_writelane_b32 v40, s30, 0
 ; CI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:32
+; CI-NEXT:    s_getpc_b64 s[4:5]
+; CI-NEXT:    s_add_u32 s4, s4, external_void_func_8xv5f32@rel32@lo+4
+; CI-NEXT:    s_addc_u32 s5, s5, external_void_func_8xv5f32@rel32@hi+12
 ; CI-NEXT:    v_mov_b32_e32 v0, 0
 ; CI-NEXT:    v_mov_b32_e32 v1, 0
 ; CI-NEXT:    v_mov_b32_e32 v2, 0
@@ -7005,9 +6969,6 @@ define void @stack_8xv5f32() #0 {
 ; CI-NEXT:    v_mov_b32_e32 v29, 0x40a00000
 ; CI-NEXT:    v_mov_b32_e32 v30, 0x40c00000
 ; CI-NEXT:    v_writelane_b32 v40, s31, 1
-; CI-NEXT:    s_getpc_b64 s[4:5]
-; CI-NEXT:    s_add_u32 s4, s4, external_void_func_8xv5f32@rel32@lo+4
-; CI-NEXT:    s_addc_u32 s5, s5, external_void_func_8xv5f32@rel32@hi+12
 ; CI-NEXT:    s_swappc_b64 s[30:31], s[4:5]
 ; CI-NEXT:    v_readlane_b32 s31, v40, 1
 ; CI-NEXT:    v_readlane_b32 s30, v40, 0
@@ -7049,6 +7010,9 @@ define void @stack_8xv5f32() #0 {
 ; GFX9-NEXT:    v_mov_b32_e32 v0, 0x41700000
 ; GFX9-NEXT:    v_writelane_b32 v40, s30, 0
 ; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:32
+; GFX9-NEXT:    s_getpc_b64 s[4:5]
+; GFX9-NEXT:    s_add_u32 s4, s4, external_void_func_8xv5f32@rel32@lo+4
+; GFX9-NEXT:    s_addc_u32 s5, s5, external_void_func_8xv5f32@rel32@hi+12
 ; GFX9-NEXT:    v_mov_b32_e32 v0, 0
 ; GFX9-NEXT:    v_mov_b32_e32 v1, 0
 ; GFX9-NEXT:    v_mov_b32_e32 v2, 0
@@ -7081,9 +7045,6 @@ define void @stack_8xv5f32() #0 {
 ; GFX9-NEXT:    v_mov_b32_e32 v29, 0x40a00000
 ; GFX9-NEXT:    v_mov_b32_e32 v30, 0x40c00000
 ; GFX9-NEXT:    v_writelane_b32 v40, s31, 1
-; GFX9-NEXT:    s_getpc_b64 s[4:5]
-; GFX9-NEXT:    s_add_u32 s4, s4, external_void_func_8xv5f32@rel32@lo+4
-; GFX9-NEXT:    s_addc_u32 s5, s5, external_void_func_8xv5f32@rel32@hi+12
 ; GFX9-NEXT:    s_swappc_b64 s[30:31], s[4:5]
 ; GFX9-NEXT:    v_readlane_b32 s31, v40, 1
 ; GFX9-NEXT:    v_readlane_b32 s30, v40, 0
@@ -7137,12 +7098,12 @@ define void @stack_8xv5f32() #0 {
 ; GFX11-NEXT:    v_dual_mov_b32 v27, 0x40a00000 :: v_dual_mov_b32 v26, 0x40a00000
 ; GFX11-NEXT:    v_dual_mov_b32 v29, 0x40a00000 :: v_dual_mov_b32 v28, 0x40a00000
 ; GFX11-NEXT:    v_mov_b32_e32 v30, 0x40c00000
-; GFX11-NEXT:    v_writelane_b32 v40, s31, 1
 ; GFX11-NEXT:    s_getpc_b64 s[0:1]
 ; GFX11-NEXT:    s_add_u32 s0, s0, external_void_func_8xv5f32@rel32@lo+4
 ; GFX11-NEXT:    s_addc_u32 s1, s1, external_void_func_8xv5f32@rel32@hi+12
-; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    v_writelane_b32 v40, s31, 1
 ; GFX11-NEXT:    s_swappc_b64 s[30:31], s[0:1]
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX11-NEXT:    v_readlane_b32 s31, v40, 1
 ; GFX11-NEXT:    v_readlane_b32 s30, v40, 0
 ; GFX11-NEXT:    s_mov_b32 s32, s33
@@ -7183,6 +7144,9 @@ define void @stack_8xv5f32() #0 {
 ; HSA-NEXT:    v_mov_b32_e32 v0, 0x41700000
 ; HSA-NEXT:    v_writelane_b32 v40, s30, 0
 ; HSA-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:32
+; HSA-NEXT:    s_getpc_b64 s[4:5]
+; HSA-NEXT:    s_add_u32 s4, s4, external_void_func_8xv5f32@rel32@lo+4
+; HSA-NEXT:    s_addc_u32 s5, s5, external_void_func_8xv5f32@rel32@hi+12
 ; HSA-NEXT:    v_mov_b32_e32 v0, 0
 ; HSA-NEXT:    v_mov_b32_e32 v1, 0
 ; HSA-NEXT:    v_mov_b32_e32 v2, 0
@@ -7215,9 +7179,6 @@ define void @stack_8xv5f32() #0 {
 ; HSA-NEXT:    v_mov_b32_e32 v29, 0x40a00000
 ; HSA-NEXT:    v_mov_b32_e32 v30, 0x40c00000
 ; HSA-NEXT:    v_writelane_b32 v40, s31, 1
-; HSA-NEXT:    s_getpc_b64 s[4:5]
-; HSA-NEXT:    s_add_u32 s4, s4, external_void_func_8xv5f32@rel32@lo+4
-; HSA-NEXT:    s_addc_u32 s5, s5, external_void_func_8xv5f32@rel32@hi+12
 ; HSA-NEXT:    s_swappc_b64 s[30:31], s[4:5]
 ; HSA-NEXT:    v_readlane_b32 s31, v40, 1
 ; HSA-NEXT:    v_readlane_b32 s30, v40, 0
diff --git a/llvm/test/CodeGen/AMDGPU/call-preserved-registers.ll b/llvm/test/CodeGen/AMDGPU/call-preserved-registers.ll
index c7f9ff85806fc..ff80e05197b0d 100644
--- a/llvm/test/CodeGen/AMDGPU/call-preserved-registers.ll
+++ b/llvm/test/CodeGen/AMDGPU/call-preserved-registers.ll
@@ -102,10 +102,10 @@ define hidden void @void_func_void_clobber_vcc() #2 {
 }
 
 ; GCN-LABEL: {{^}}test_call_void_func_void_clobber_vcc:
-; GCN: s_mov_b64 s[34:35], vcc
-; GCN-NEXT: s_getpc_b64
+; GCN: s_getpc_b64
 ; GCN-NEXT: s_add_u32
 ; GCN-NEXT: s_addc_u32
+; GCN: s_mov_b64 s[34:35], vcc
 ; GCN-NEXT: s_swappc_b64
 ; GCN: s_mov_b64 vcc, s[34:35]
 define amdgpu_kernel void @test_call_void_func_void_clobber_vcc(ptr addrspace(1) %out) #0 {
@@ -142,21 +142,27 @@ define amdgpu_kernel void @test_call_void_func_void_mayclobber_v31(ptr addrspace
 ; FIXME: What is the expected behavior for reserved registers here?
 
 ; GCN-LABEL: {{^}}test_call_void_func_void_preserves_s33:
-; GCN: #ASMSTART
-; GCN-NEXT: ; def s33
-; GCN-NEXT: #ASMEND
 ; FLATSCR:      s_getpc_b64 s[0:1]
 ; FLATSCR-NEXT: s_add_u32 s0, s0, external_void_func_void@rel32@lo+4
 ; FLATSCR-NEXT: s_addc_u32 s1, s1, external_void_func_void@rel32@hi+12
-; FLATSCR-NEXT: s_swappc_b64 s[30:31], s[0:1]
 ; MUBUF:        s_getpc_b64 s[4:5]
 ; MUBUF-NEXT:   s_add_u32 s4, s4, external_void_func_void@rel32@lo+4
 ; MUBUF-NEXT:   s_addc_u32 s5, s5, external_void_func_void@rel32@hi+12
-; MUBUF-NEXT:   s_swappc_b64 s[30:31], s[4:5]
+
+; GCN: #ASMSTART
+; GCN-NEXT: ; def s33
+; GCN-NEXT: #ASMEND
+
+; GCN-NOT: s33
+
+; FLATSCR: s_swappc_b64 s[30:31], s[0:1]
+; MUBUF:   s_swappc_b64 s[30:31], s[4:5]
+
+; GCN-NOT: s33
+
 ; GCN: ;;#ASMSTART
 ; GCN-NEXT: ; use s33
 ; GCN-NEXT: ;;#ASMEND
-; GCN-NOT: s33
 ; GCN-NEXT: s_endpgm
 define amdgpu_kernel void @test_call_void_func_void_preserves_s33(ptr addrspace(1) %out) #0 {
   %s33 = call i32 asm sideeffect "; def $0", "={s33}"()
@@ -168,20 +174,20 @@ define amdgpu_kernel void @test_call_void_func_void_preserves_s33(ptr addrspace(
 ; GCN-LABEL: {{^}}test_call_void_func_void_preserves_s34: {{.*}}
 ; GCN-NOT: s34
 
-; GCN: s_mov_b32 s32, 0
-
-; GCN-NOT: s34
-; GCN: ;;#ASMSTART
-; GCN-NEXT: ; def s34
-; GCN-NEXT: ;;#ASMEND
 ; FLATSCR:      s_getpc_b64 s[0:1]
 ; FLATSCR-NEXT: s_add_u32 s0, s0, external_void_func_void@rel32@lo+4
 ; FLATSCR-NEXT: s_addc_u32 s1, s1, external_void_func_void@rel32@hi+12
 ; MUBUF:        s_getpc_b64 s[4:5]
 ; MUBUF-NEXT:   s_add_u32 s4, s4, external_void_func_void@rel32@lo+4
 ; MUBUF-NEXT:   s_addc_u32 s5, s5, external_void_func_void@rel32@hi+12
+; GCN: s_mov_b32 s32, 0
+
+; GCN: ;;#ASMSTART
+; GCN-NEXT: ; def s34
+; GCN-NEXT: ;;#ASMEND
 
 ; GCN-NOT: s34
+
 ; MUBUF:   s_swappc_b64 s[30:31], s[4:5]
 ; FLATSCR: s_swappc_b64 s[30:31], s[0:1]
 
@@ -200,19 +206,19 @@ define amdgpu_kernel void @test_call_void_func_void_preserves_s34(ptr addrspace(
 
 ; GCN-LABEL: {{^}}test_call_void_func_void_preserves_v40: {{.*}}
 
-; GCN-NOT: v32
-; GCN: s_mov_b32 s32, 0
-; GCN-NOT: v40
-
-; GCN: ;;#ASMSTART
-; GCN-NEXT: ; def v40
-; GCN-NEXT: ;;#ASMEND
 ; MUBUF: s_getpc_b64 s[4:5]
 ; MUBUF-NEXT:   s_add_u32 s4, s4, external_void_func_void@rel32@lo+4
 ; MUBUF-NEXT:   s_addc_u32 s5, s5, external_void_func_void@rel32@hi+12
 ; FLATSCR:      s_getpc_b64 s[0:1]
 ; FLATSCR-NEXT: s_add_u32 s0, s0, external_void_func_void@rel32@lo+4
 ; FLATSCR-NEXT: s_addc_u32 s1, s1, external_void_func_void@rel32@hi+12
+; GCN: s_mov_b32 s32, 0
+
+; GCN: ;;#ASMSTART
+; GCN-NEXT: ; def v40
+; GCN-NEXT: ;;#ASMEND
+
+; GCN-NOT: v40
 
 ; MUBUF:   s_swappc_b64 s[30:31], s[4:5]
 ; FLATSCR: s_swappc_b64 s[30:31], s[0:1]
@@ -255,10 +261,10 @@ define hidden void @void_func_void_clobber_s34() #2 {
 }
 
 ; GCN-LABEL: {{^}}test_call_void_func_void_clobber_s33:
-; GCN: s_mov_b32 s32, 0
 ; GCN: s_getpc_b64
 ; GCN-NEXT: s_add_u32
 ; GCN-NEXT: s_addc_u32
+; GCN: s_mov_b32 s32, 0
 ; GCN: s_swappc_b64
 ; GCN-NEXT: s_endpgm
 define amdgpu_kernel void @test_call_void_func_void_clobber_s33() #0 {
@@ -267,10 +273,10 @@ define amdgpu_kernel void @test_call_void_func_void_clobber_s33() #0 {
 }
 
 ; GCN-LABEL: {{^}}test_call_void_func_void_clobber_s34:
-; GCN: s_mov_b32 s32, 0
 ; GCN: s_getpc_b64
 ; GCN-NEXT: s_add_u32
 ; GCN-NEXT: s_addc_u32
+; GCN: s_mov_b32 s32, 0
 ; GCN: s_swappc_b64
 ; GCN-NEXT: s_endpgm
 define amdgpu_kernel void @test_call_void_func_void_clobber_s34() #0 {
diff --git a/llvm/test/CodeGen/AMDGPU/call-reqd-group-size.ll b/llvm/test/CodeGen/AMDGPU/call-reqd-group-size.ll
index 3626b2b316fba..093ca55698fe3 100644
--- a/llvm/test/CodeGen/AMDGPU/call-reqd-group-size.ll
+++ b/llvm/test/CodeGen/AMDGPU/call-reqd-group-size.ll
@@ -1,6 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -o - %s | FileCheck %s
-; RUN: llc -global-isel -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -o - %s | FileCheck %s
+; RUN: llc -global-isel -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -o - %s | FileCheck %s -check-prefix=GISEL
 
 ; Check for optimizing the passed implicit workitem ID based on the
 ; required group size. This should avoid a few bit packing operations.
@@ -13,15 +13,30 @@ define amdgpu_kernel void @known_x_0(ptr addrspace(1) %out) !reqd_work_group_siz
 ; CHECK-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
 ; CHECK-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; CHECK-NEXT:    s_add_u32 s0, s0, s17
-; CHECK-NEXT:    v_lshlrev_b32_e32 v0, 20, v2
 ; CHECK-NEXT:    s_addc_u32 s1, s1, 0
-; CHECK-NEXT:    v_lshl_or_b32 v31, v1, 10, v0
-; CHECK-NEXT:    s_mov_b32 s32, 0
+; CHECK-NEXT:    v_lshlrev_b32_e32 v0, 20, v2
 ; CHECK-NEXT:    s_getpc_b64 s[4:5]
 ; CHECK-NEXT:    s_add_u32 s4, s4, callee@rel32@lo+4
 ; CHECK-NEXT:    s_addc_u32 s5, s5, callee@rel32@hi+12
+; CHECK-NEXT:    v_lshl_or_b32 v31, v1, 10, v0
+; CHECK-NEXT:    s_mov_b32 s32, 0
 ; CHECK-NEXT:    s_swappc_b64 s[30:31], s[4:5]
 ; CHECK-NEXT:    s_endpgm
+;
+; GISEL-LABEL: known_x_0:
+; GISEL:       ; %bb.0:
+; GISEL-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GISEL-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
+; GISEL-NEXT:    s_add_u32 s0, s0, s17
+; GISEL-NEXT:    s_addc_u32 s1, s1, 0
+; GISEL-NEXT:    v_lshlrev_b32_e32 v0, 20, v2
+; GISEL-NEXT:    v_lshl_or_b32 v31, v1, 10, v0
+; GISEL-NEXT:    s_getpc_b64 s[4:5]
+; GISEL-NEXT:    s_add_u32 s4, s4, callee@rel32@lo+4
+; GISEL-NEXT:    s_addc_u32 s5, s5, callee@rel32@hi+12
+; GISEL-NEXT:    s_mov_b32 s32, 0
+; GISEL-NEXT:    s_swappc_b64 s[30:31], s[4:5]
+; GISEL-NEXT:    s_endpgm
   call void @callee()
   ret void
 }
@@ -34,13 +49,27 @@ define amdgpu_kernel void @known_y_0(ptr addrspace(1) %out) !reqd_work_group_siz
 ; CHECK-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; CHECK-NEXT:    s_add_u32 s0, s0, s17
 ; CHECK-NEXT:    s_addc_u32 s1, s1, 0
-; CHECK-NEXT:    v_lshl_or_b32 v31, v2, 20, v0
-; CHECK-NEXT:    s_mov_b32 s32, 0
 ; CHECK-NEXT:    s_getpc_b64 s[4:5]
 ; CHECK-NEXT:    s_add_u32 s4, s4, callee@rel32@lo+4
 ; CHECK-NEXT:    s_addc_u32 s5, s5, callee@rel32@hi+12
+; CHECK-NEXT:    v_lshl_or_b32 v31, v2, 20, v0
+; CHECK-NEXT:    s_mov_b32 s32, 0
 ; CHECK-NEXT:    s_swappc_b64 s[30:31], s[4:5]
 ; CHECK-NEXT:    s_endpgm
+;
+; GISEL-LABEL: known_y_0:
+; GISEL:       ; %bb.0:
+; GISEL-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GISEL-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
+; GISEL-NEXT:    s_add_u32 s0, s0, s17
+; GISEL-NEXT:    s_addc_u32 s1, s1, 0
+; GISEL-NEXT:    v_lshl_or_b32 v31, v2, 20, v0
+; GISEL-NEXT:    s_getpc_b64 s[4:5]
+; GISEL-NEXT:    s_add_u32 s4, s4, callee@rel32@lo+4
+; GISEL-NEXT:    s_addc_u32 s5, s5, callee@rel32@hi+12
+; GISEL-NEXT:    s_mov_b32 s32, 0
+; GISEL-NEXT:    s_swappc_b64 s[30:31], s[4:5]
+; GISEL-NEXT:    s_endpgm
   call void @callee()
   ret void
 }
@@ -53,13 +82,27 @@ define amdgpu_kernel void @known_z_0(ptr addrspace(1) %out) !reqd_work_group_siz
 ; CHECK-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; CHECK-NEXT:    s_add_u32 s0, s0, s17
 ; CHECK-NEXT:    s_addc_u32 s1, s1, 0
-; CHECK-NEXT:    v_lshl_or_b32 v31, v1, 10, v0
-; CHECK-NEXT:    s_mov_b32 s32, 0
 ; CHECK-NEXT:    s_getpc_b64 s[4:5]
 ; CHECK-NEXT:    s_add_u32 s4, s4, callee@rel32@lo+4
 ; CHECK-NEXT:    s_addc_u32 s5, s5, callee@rel32@hi+12
+; CHECK-NEXT:    v_lshl_or_b32 v31, v1, 10, v0
+; CHECK-NEXT:    s_mov_b32 s32, 0
 ; CHECK-NEXT:    s_swappc_b64 s[30:31], s[4:5]
 ; CHECK-NEXT:    s_endpgm
+;
+; GISEL-LABEL: known_z_0:
+; GISEL:       ; %bb.0:
+; GISEL-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GISEL-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
+; GISEL-NEXT:    s_add_u32 s0, s0, s17
+; GISEL-NEXT:    s_addc_u32 s1, s1, 0
+; GISEL-NEXT:    v_lshl_or_b32 v31, v1, 10, v0
+; GISEL-NEXT:    s_getpc_b64 s[4:5]
+; GISEL-NEXT:    s_add_u32 s4, s4, callee@rel32@lo+4
+; GISEL-NEXT:    s_addc_u32 s5, s5, callee@rel32@hi+12
+; GISEL-NEXT:    s_mov_b32 s32, 0
+; GISEL-NEXT:    s_swappc_b64 s[30:31], s[4:5]
+; GISEL-NEXT:    s_endpgm
   call void @callee()
   ret void
 }
@@ -72,13 +115,27 @@ define amdgpu_kernel void @known_yz_0(ptr addrspace(1) %out) !reqd_work_group_si
 ; CHECK-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; CHECK-NEXT:    s_add_u32 s0, s0, s17
 ; CHECK-NEXT:    s_addc_u32 s1, s1, 0
-; CHECK-NEXT:    v_mov_b32_e32 v31, v0
-; CHECK-NEXT:    s_mov_b32 s32, 0
 ; CHECK-NEXT:    s_getpc_b64 s[4:5]
 ; CHECK-NEXT:    s_add_u32 s4, s4, callee@rel32@lo+4
 ; CHECK-NEXT:    s_addc_u32 s5, s5, callee@rel32@hi+12
+; CHECK-NEXT:    v_mov_b32_e32 v31, v0
+; CHECK-NEXT:    s_mov_b32 s32, 0
 ; CHECK-NEXT:    s_swappc_b64 s[30:31], s[4:5]
 ; CHECK-NEXT:    s_endpgm
+;
+; GISEL-LABEL: known_yz_0:
+; GISEL:       ; %bb.0:
+; GISEL-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GISEL-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
+; GISEL-NEXT:    s_add_u32 s0, s0, s17
+; GISEL-NEXT:    s_addc_u32 s1, s1, 0
+; GISEL-NEXT:    s_getpc_b64 s[4:5]
+; GISEL-NEXT:    s_add_u32 s4, s4, callee@rel32@lo+4
+; GISEL-NEXT:    s_addc_u32 s5, s5, callee@rel32@hi+12
+; GISEL-NEXT:    v_mov_b32_e32 v31, v0
+; GISEL-NEXT:    s_mov_b32 s32, 0
+; GISEL-NEXT:    s_swappc_b64 s[30:31], s[4:5]
+; GISEL-NEXT:    s_endpgm
   call void @callee()
   ret void
 }
@@ -91,13 +148,27 @@ define amdgpu_kernel void @known_xz_0(ptr addrspace(1) %out) !reqd_work_group_si
 ; CHECK-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; CHECK-NEXT:    s_add_u32 s0, s0, s17
 ; CHECK-NEXT:    s_addc_u32 s1, s1, 0
-; CHECK-NEXT:    v_lshlrev_b32_e32 v31, 10, v1
-; CHECK-NEXT:    s_mov_b32 s32, 0
 ; CHECK-NEXT:    s_getpc_b64 s[4:5]
 ; CHECK-NEXT:    s_add_u32 s4, s4, callee@rel32@lo+4
 ; CHECK-NEXT:    s_addc_u32 s5, s5, callee@rel32@hi+12
+; CHECK-NEXT:    v_lshlrev_b32_e32 v31, 10, v1
+; CHECK-NEXT:    s_mov_b32 s32, 0
 ; CHECK-NEXT:    s_swappc_b64 s[30:31], s[4:5]
 ; CHECK-NEXT:    s_endpgm
+;
+; GISEL-LABEL: known_xz_0:
+; GISEL:       ; %bb.0:
+; GISEL-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GISEL-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
+; GISEL-NEXT:    s_add_u32 s0, s0, s17
+; GISEL-NEXT:    s_addc_u32 s1, s1, 0
+; GISEL-NEXT:    v_lshlrev_b32_e32 v31, 10, v1
+; GISEL-NEXT:    s_getpc_b64 s[4:5]
+; GISEL-NEXT:    s_add_u32 s4, s4, callee@rel32@lo+4
+; GISEL-NEXT:    s_addc_u32 s5, s5, callee@rel32@hi+12
+; GISEL-NEXT:    s_mov_b32 s32, 0
+; GISEL-NEXT:    s_swappc_b64 s[30:31], s[4:5]
+; GISEL-NEXT:    s_endpgm
   call void @callee()
   ret void
 }
@@ -111,13 +182,27 @@ define amdgpu_kernel void @known_xyz_0(ptr addrspace(1) %out) !reqd_work_group_s
 ; CHECK-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; CHECK-NEXT:    s_add_u32 s0, s0, s17
 ; CHECK-NEXT:    s_addc_u32 s1, s1, 0
-; CHECK-NEXT:    v_mov_b32_e32 v31, 0
-; CHECK-NEXT:    s_mov_b32 s32, 0
 ; CHECK-NEXT:    s_getpc_b64 s[4:5]
 ; CHECK-NEXT:    s_add_u32 s4, s4, callee@rel32@lo+4
 ; CHECK-NEXT:    s_addc_u32 s5, s5, callee@rel32@hi+12
+; CHECK-NEXT:    v_mov_b32_e32 v31, 0
+; CHECK-NEXT:    s_mov_b32 s32, 0
 ; CHECK-NEXT:    s_swappc_b64 s[30:31], s[4:5]
 ; CHECK-NEXT:    s_endpgm
+;
+; GISEL-LABEL: known_xyz_0:
+; GISEL:       ; %bb.0:
+; GISEL-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GISEL-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
+; GISEL-NEXT:    s_add_u32 s0, s0, s17
+; GISEL-NEXT:    s_addc_u32 s1, s1, 0
+; GISEL-NEXT:    s_getpc_b64 s[4:5]
+; GISEL-NEXT:    s_add_u32 s4, s4, callee@rel32@lo+4
+; GISEL-NEXT:    s_addc_u32 s5, s5, callee@rel32@hi+12
+; GISEL-NEXT:    v_mov_b32_e32 v31, 0
+; GISEL-NEXT:    s_mov_b32 s32, 0
+; GISEL-NEXT:    s_swappc_b64 s[30:31], s[4:5]
+; GISEL-NEXT:    s_endpgm
   call void @callee()
   ret void
 }
diff --git a/llvm/test/CodeGen/AMDGPU/call-waitcnt.ll b/llvm/test/CodeGen/AMDGPU/call-waitcnt.ll
index 10f0efea59b60..15152409eacf9 100644
--- a/llvm/test/CodeGen/AMDGPU/call-waitcnt.ll
+++ b/llvm/test/CodeGen/AMDGPU/call-waitcnt.ll
@@ -13,11 +13,11 @@ define amdgpu_kernel void @call_memory_arg_load(ptr addrspace(3) %ptr, i32) #0 {
 ; GCN-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN-NEXT:    v_mov_b32_e32 v0, s6
 ; GCN-NEXT:    ds_read_b32 v0, v0
-; GCN-NEXT:    s_mov_b64 s[6:7], s[4:5]
-; GCN-NEXT:    s_mov_b32 s32, 0
 ; GCN-NEXT:    s_getpc_b64 s[8:9]
 ; GCN-NEXT:    s_add_u32 s8, s8, func@rel32@lo+4
 ; GCN-NEXT:    s_addc_u32 s9, s9, func@rel32@hi+12
+; GCN-NEXT:    s_mov_b64 s[6:7], s[4:5]
+; GCN-NEXT:    s_mov_b32 s32, 0
 ; GCN-NEXT:    s_swappc_b64 s[30:31], s[8:9]
 ; GCN-NEXT:    s_endpgm
   %vgpr = load volatile i32, ptr addrspace(3) %ptr
@@ -33,16 +33,16 @@ define amdgpu_kernel void @call_memory_no_dep(ptr addrspace(1) %ptr, i32) #0 {
 ; GCN-NEXT:    s_add_u32 flat_scratch_lo, s8, s11
 ; GCN-NEXT:    s_addc_u32 flat_scratch_hi, s9, 0
 ; GCN-NEXT:    s_add_u32 s0, s0, s11
-; GCN-NEXT:    v_mov_b32_e32 v0, 0
 ; GCN-NEXT:    s_addc_u32 s1, s1, 0
+; GCN-NEXT:    v_mov_b32_e32 v0, 0
+; GCN-NEXT:    s_getpc_b64 s[8:9]
+; GCN-NEXT:    s_add_u32 s8, s8, func@rel32@lo+4
+; GCN-NEXT:    s_addc_u32 s9, s9, func@rel32@hi+12
 ; GCN-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN-NEXT:    global_store_dword v0, v0, s[6:7]
 ; GCN-NEXT:    s_mov_b64 s[6:7], s[4:5]
 ; GCN-NEXT:    v_mov_b32_e32 v0, 0
 ; GCN-NEXT:    s_mov_b32 s32, 0
-; GCN-NEXT:    s_getpc_b64 s[8:9]
-; GCN-NEXT:    s_add_u32 s8, s8, func@rel32@lo+4
-; GCN-NEXT:    s_addc_u32 s9, s9, func@rel32@hi+12
 ; GCN-NEXT:    s_swappc_b64 s[30:31], s[8:9]
 ; GCN-NEXT:    s_endpgm
   store i32 0, ptr addrspace(1) %ptr
@@ -55,16 +55,16 @@ define amdgpu_kernel void @call_no_wait_after_call(ptr addrspace(1) %ptr, i32) #
 ; GCN-LABEL: call_no_wait_after_call:
 ; GCN:       ; %bb.0:
 ; GCN-NEXT:    s_add_u32 flat_scratch_lo, s8, s11
-; GCN-NEXT:    s_load_dwordx2 s[34:35], s[6:7], 0x0
 ; GCN-NEXT:    s_addc_u32 flat_scratch_hi, s9, 0
+; GCN-NEXT:    s_load_dwordx2 s[34:35], s[6:7], 0x0
 ; GCN-NEXT:    s_add_u32 s0, s0, s11
 ; GCN-NEXT:    s_addc_u32 s1, s1, 0
-; GCN-NEXT:    s_mov_b64 s[6:7], s[4:5]
-; GCN-NEXT:    v_mov_b32_e32 v0, 0
-; GCN-NEXT:    s_mov_b32 s32, 0
 ; GCN-NEXT:    s_getpc_b64 s[8:9]
 ; GCN-NEXT:    s_add_u32 s8, s8, func@rel32@lo+4
 ; GCN-NEXT:    s_addc_u32 s9, s9, func@rel32@hi+12
+; GCN-NEXT:    s_mov_b64 s[6:7], s[4:5]
+; GCN-NEXT:    v_mov_b32_e32 v0, 0
+; GCN-NEXT:    s_mov_b32 s32, 0
 ; GCN-NEXT:    v_mov_b32_e32 v40, 0
 ; GCN-NEXT:    s_swappc_b64 s[30:31], s[8:9]
 ; GCN-NEXT:    global_store_dword v40, v40, s[34:35]
@@ -78,16 +78,16 @@ define amdgpu_kernel void @call_no_wait_after_call_return_val(ptr addrspace(1) %
 ; GCN-LABEL: call_no_wait_after_call_return_val:
 ; GCN:       ; %bb.0:
 ; GCN-NEXT:    s_add_u32 flat_scratch_lo, s8, s11
-; GCN-NEXT:    s_load_dwordx2 s[34:35], s[6:7], 0x0
 ; GCN-NEXT:    s_addc_u32 flat_scratch_hi, s9, 0
+; GCN-NEXT:    s_load_dwordx2 s[34:35], s[6:7], 0x0
 ; GCN-NEXT:    s_add_u32 s0, s0, s11
 ; GCN-NEXT:    s_addc_u32 s1, s1, 0
-; GCN-NEXT:    s_mov_b64 s[6:7], s[4:5]
-; GCN-NEXT:    v_mov_b32_e32 v0, 0
-; GCN-NEXT:    s_mov_b32 s32, 0
 ; GCN-NEXT:    s_getpc_b64 s[8:9]
 ; GCN-NEXT:    s_add_u32 s8, s8, func.return@rel32@lo+4
 ; GCN-NEXT:    s_addc_u32 s9, s9, func.return@rel32@hi+12
+; GCN-NEXT:    s_mov_b64 s[6:7], s[4:5]
+; GCN-NEXT:    v_mov_b32_e32 v0, 0
+; GCN-NEXT:    s_mov_b32 s32, 0
 ; GCN-NEXT:    v_mov_b32_e32 v40, 0
 ; GCN-NEXT:    s_swappc_b64 s[30:31], s[8:9]
 ; GCN-NEXT:    global_store_dword v40, v0, s[34:35]
diff --git a/llvm/test/CodeGen/AMDGPU/callee-frame-setup.ll b/llvm/test/CodeGen/AMDGPU/callee-frame-setup.ll
index 6fb071dd42d2f..4c6f2d22080e0 100644
--- a/llvm/test/CodeGen/AMDGPU/callee-frame-setup.ll
+++ b/llvm/test/CodeGen/AMDGPU/callee-frame-setup.ll
@@ -1,123 +1,177 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
 ; RUN: llc -mtriple=amdgcn -mcpu=hawaii -verify-machineinstrs < %s | FileCheck  -enable-var-scope -check-prefixes=GCN,MUBUF %s
 ; RUN: llc -mtriple=amdgcn -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck  -enable-var-scope -check-prefixes=GCN,MUBUF %s
 ; RUN: llc -mtriple=amdgcn -mcpu=gfx900 -verify-machineinstrs -mattr=+enable-flat-scratch < %s | FileCheck  -enable-var-scope -check-prefixes=GCN,FLATSCR %s
 
-; GCN-LABEL: {{^}}callee_no_stack:
-; GCN: ; %bb.0:
-; GCN-NEXT: s_waitcnt
-; GCN-NEXT: s_setpc_b64
 define void @callee_no_stack() #0 {
+; GCN-LABEL: callee_no_stack:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    s_setpc_b64 s[30:31]
   ret void
 }
 
-; GCN-LABEL: {{^}}callee_no_stack_no_fp_elim_all:
-; GCN: ; %bb.0:
-; GCN-NEXT: s_waitcnt
-; MUBUF-NEXT:   s_mov_b32 [[FP_COPY:s4]], s33
-; FLATSCR-NEXT: s_mov_b32 [[FP_COPY:s0]], s33
-; GCN-NEXT: s_mov_b32 s33, s32
-; GCN-NEXT: s_mov_b32 s33, [[FP_COPY]]
-; GCN-NEXT: s_setpc_b64
 define void @callee_no_stack_no_fp_elim_all() #1 {
+; MUBUF-LABEL: callee_no_stack_no_fp_elim_all:
+; MUBUF:       ; %bb.0:
+; MUBUF-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; MUBUF-NEXT:    s_mov_b32 s4, s33
+; MUBUF-NEXT:    s_mov_b32 s33, s32
+; MUBUF-NEXT:    s_mov_b32 s33, s4
+; MUBUF-NEXT:    s_setpc_b64 s[30:31]
+;
+; FLATSCR-LABEL: callee_no_stack_no_fp_elim_all:
+; FLATSCR:       ; %bb.0:
+; FLATSCR-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; FLATSCR-NEXT:    s_mov_b32 s0, s33
+; FLATSCR-NEXT:    s_mov_b32 s33, s32
+; FLATSCR-NEXT:    s_mov_b32 s33, s0
+; FLATSCR-NEXT:    s_setpc_b64 s[30:31]
   ret void
 }
 
-; GCN-LABEL: {{^}}callee_no_stack_no_fp_elim_nonleaf:
-; GCN: ; %bb.0:
-; GCN-NEXT: s_waitcnt
-; GCN-NEXT: s_setpc_b64
 define void @callee_no_stack_no_fp_elim_nonleaf() #2 {
+; GCN-LABEL: callee_no_stack_no_fp_elim_nonleaf:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    s_setpc_b64 s[30:31]
   ret void
 }
 
-; GCN-LABEL: {{^}}callee_with_stack:
-; GCN: ; %bb.0:
-; GCN-NEXT: s_waitcnt
-; GCN-NEXT: v_mov_b32_e32 v0, 0{{$}}
-; MUBUF-NEXT:   buffer_store_dword v0, off, s[0:3], s32{{$}}
-; FLATSCR-NEXT: scratch_store_dword off, v0, s32
-; GCN-NEXT: s_waitcnt
-; GCN-NEXT: s_setpc_b64
 define void @callee_with_stack() #0 {
+; MUBUF-LABEL: callee_with_stack:
+; MUBUF:       ; %bb.0:
+; MUBUF-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; MUBUF-NEXT:    v_mov_b32_e32 v0, 0
+; MUBUF-NEXT:    buffer_store_dword v0, off, s[0:3], s32
+; MUBUF-NEXT:    s_waitcnt vmcnt(0)
+; MUBUF-NEXT:    s_setpc_b64 s[30:31]
+;
+; FLATSCR-LABEL: callee_with_stack:
+; FLATSCR:       ; %bb.0:
+; FLATSCR-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; FLATSCR-NEXT:    v_mov_b32_e32 v0, 0
+; FLATSCR-NEXT:    scratch_store_dword off, v0, s32
+; FLATSCR-NEXT:    s_waitcnt vmcnt(0)
+; FLATSCR-NEXT:    s_setpc_b64 s[30:31]
   %alloca = alloca i32, addrspace(5)
   store volatile i32 0, ptr addrspace(5) %alloca
   ret void
 }
 
 ; Can use free call clobbered register to preserve original FP value.
-
-; GCN-LABEL: {{^}}callee_with_stack_no_fp_elim_all:
-; GCN: ; %bb.0:
-; GCN-NEXT: s_waitcnt
-; MUBUF-NEXT:   s_mov_b32 [[FP_COPY:s4]], s33
-; FLATSCR-NEXT: s_mov_b32 [[FP_COPY:s0]], s33
-; GCN-NEXT: s_mov_b32 s33, s32
-; MUBUF-NEXT:   s_addk_i32 s32, 0x200
-; FLATSCR-NEXT: s_add_i32 s32, s32, 8
-; GCN-NEXT: v_mov_b32_e32 v0, 0{{$}}
-; MUBUF-NEXT:   buffer_store_dword v0, off, s[0:3], s33{{$}}
-; FLATSCR-NEXT: scratch_store_dword off, v0, s33{{$}}
-; GCN-NEXT: s_waitcnt vmcnt(0)
-; MUBUF-NEXT:   s_mov_b32 s32, s33
-; FLATSCR-NEXT: s_mov_b32 s32, s33
-; GCN-NEXT: s_mov_b32 s33, [[FP_COPY]]
-; GCN-NEXT: s_setpc_b64
 define void @callee_with_stack_no_fp_elim_all() #1 {
+; MUBUF-LABEL: callee_with_stack_no_fp_elim_all:
+; MUBUF:       ; %bb.0:
+; MUBUF-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; MUBUF-NEXT:    s_mov_b32 s4, s33
+; MUBUF-NEXT:    s_mov_b32 s33, s32
+; MUBUF-NEXT:    s_addk_i32 s32, 0x200
+; MUBUF-NEXT:    v_mov_b32_e32 v0, 0
+; MUBUF-NEXT:    buffer_store_dword v0, off, s[0:3], s33
+; MUBUF-NEXT:    s_waitcnt vmcnt(0)
+; MUBUF-NEXT:    s_mov_b32 s32, s33
+; MUBUF-NEXT:    s_mov_b32 s33, s4
+; MUBUF-NEXT:    s_setpc_b64 s[30:31]
+;
+; FLATSCR-LABEL: callee_with_stack_no_fp_elim_all:
+; FLATSCR:       ; %bb.0:
+; FLATSCR-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; FLATSCR-NEXT:    s_mov_b32 s0, s33
+; FLATSCR-NEXT:    s_mov_b32 s33, s32
+; FLATSCR-NEXT:    s_add_i32 s32, s32, 8
+; FLATSCR-NEXT:    v_mov_b32_e32 v0, 0
+; FLATSCR-NEXT:    scratch_store_dword off, v0, s33
+; FLATSCR-NEXT:    s_waitcnt vmcnt(0)
+; FLATSCR-NEXT:    s_mov_b32 s32, s33
+; FLATSCR-NEXT:    s_mov_b32 s33, s0
+; FLATSCR-NEXT:    s_setpc_b64 s[30:31]
   %alloca = alloca i32, addrspace(5)
   store volatile i32 0, ptr addrspace(5) %alloca
   ret void
 }
 
-; GCN-LABEL: {{^}}callee_with_stack_no_fp_elim_non_leaf:
-; GCN: ; %bb.0:
-; GCN-NEXT: s_waitcnt
-; GCN-NEXT: v_mov_b32_e32 v0, 0{{$}}
-; MUBUF-NEXT:   buffer_store_dword v0, off, s[0:3], s32{{$}}
-; FLATSCR-NEXT: scratch_store_dword off, v0, s32{{$}}
-; GCN-NEXT: s_waitcnt
-; GCN-NEXT: s_setpc_b64
 define void @callee_with_stack_no_fp_elim_non_leaf() #2 {
+; MUBUF-LABEL: callee_with_stack_no_fp_elim_non_leaf:
+; MUBUF:       ; %bb.0:
+; MUBUF-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; MUBUF-NEXT:    v_mov_b32_e32 v0, 0
+; MUBUF-NEXT:    buffer_store_dword v0, off, s[0:3], s32
+; MUBUF-NEXT:    s_waitcnt vmcnt(0)
+; MUBUF-NEXT:    s_setpc_b64 s[30:31]
+;
+; FLATSCR-LABEL: callee_with_stack_no_fp_elim_non_leaf:
+; FLATSCR:       ; %bb.0:
+; FLATSCR-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; FLATSCR-NEXT:    v_mov_b32_e32 v0, 0
+; FLATSCR-NEXT:    scratch_store_dword off, v0, s32
+; FLATSCR-NEXT:    s_waitcnt vmcnt(0)
+; FLATSCR-NEXT:    s_setpc_b64 s[30:31]
   %alloca = alloca i32, addrspace(5)
   store volatile i32 0, ptr addrspace(5) %alloca
   ret void
 }
 
-; GCN-LABEL: {{^}}callee_with_stack_and_call:
-; GCN: ; %bb.0:
-; GCN-NEXT: s_waitcnt
-; GCN-NEXT: s_mov_b32 [[FP_SCRATCH_COPY:s[0-9]+]], s33
-; GCN-NEXT: s_mov_b32 s33, s32
-; GCN-NEXT: s_or_saveexec_b64 [[COPY_EXEC0:s\[[0-9]+:[0-9]+\]]], -1{{$}}
-; MUBUF-NEXT:   buffer_store_dword [[CSR_VGPR:v[0-9]+]], off, s[0:3], s33 offset:4 ; 4-byte Folded Spill
-; FLATSCR-NEXT: scratch_store_dword off, [[CSR_VGPR:v[0-9]+]], s33 offset:4 ; 4-byte Folded Spill
-; GCN-NEXT: s_mov_b64 exec, [[COPY_EXEC0]]
-; GCN: v_writelane_b32 [[CSR_VGPR]], [[FP_SCRATCH_COPY]], 2
-; MUBUF-DAG:   s_addk_i32 s32, 0x400{{$}}
-; FLATSCR-DAG: s_add_i32 s32, s32, 16{{$}}
-; GCN-DAG: v_writelane_b32 [[CSR_VGPR]], s30,
-; GCN-DAG: v_mov_b32_e32 [[ZERO:v[0-9]+]], 0{{$}}
-; GCN-DAG: v_writelane_b32 [[CSR_VGPR]], s31,
-
-; MUBUF-DAG:   buffer_store_dword [[ZERO]], off, s[0:3], s33{{$}}
-; FLATSCR-DAG: scratch_store_dword off, [[ZERO]], s33{{$}}
-
-; GCN: s_swappc_b64
-
-; GCN-DAG: v_readlane_b32 s30, [[CSR_VGPR]]
-; GCN-DAG: v_readlane_b32 s31, [[CSR_VGPR]]
-
-; MUBUF:    s_mov_b32 s32, s33{{$}}
-; FLATSCR:  s_mov_b32 s32, s33{{$}}
-; GCN-NEXT: v_readlane_b32 [[FP_SCRATCH_COPY:s[0-9]+]], [[CSR_VGPR]], 2
-; GCN-NEXT: s_or_saveexec_b64 [[COPY_EXEC1:s\[[0-9]+:[0-9]+\]]], -1{{$}}
-; MUBUF-NEXT:   buffer_load_dword [[CSR_VGPR]], off, s[0:3], s33 offset:4 ; 4-byte Folded Reload
-; FLATSCR-NEXT: scratch_load_dword [[CSR_VGPR]], off, s33 offset:4 ; 4-byte Folded Reload
-; GCN-NEXT: s_mov_b64 exec, [[COPY_EXEC1]]
-; GCN-NEXT: s_mov_b32 s33, [[FP_SCRATCH_COPY]]
-; GCN-NEXT: s_waitcnt vmcnt(0)
-
-; GCN-NEXT: s_setpc_b64 s[30:31]
 define void @callee_with_stack_and_call() #0 {
+; MUBUF-LABEL: callee_with_stack_and_call:
+; MUBUF:       ; %bb.0:
+; MUBUF-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; MUBUF-NEXT:    s_mov_b32 s16, s33
+; MUBUF-NEXT:    s_mov_b32 s33, s32
+; MUBUF-NEXT:    s_or_saveexec_b64 s[18:19], -1
+; MUBUF-NEXT:    buffer_store_dword v40, off, s[0:3], s33 offset:4 ; 4-byte Folded Spill
+; MUBUF-NEXT:    s_mov_b64 exec, s[18:19]
+; MUBUF-NEXT:    v_writelane_b32 v40, s16, 2
+; MUBUF-NEXT:    s_addk_i32 s32, 0x400
+; MUBUF-NEXT:    v_writelane_b32 v40, s30, 0
+; MUBUF-NEXT:    v_mov_b32_e32 v0, 0
+; MUBUF-NEXT:    s_getpc_b64 s[16:17]
+; MUBUF-NEXT:    s_add_u32 s16, s16, external_void_func_void@rel32@lo+4
+; MUBUF-NEXT:    s_addc_u32 s17, s17, external_void_func_void@rel32@hi+12
+; MUBUF-NEXT:    v_writelane_b32 v40, s31, 1
+; MUBUF-NEXT:    buffer_store_dword v0, off, s[0:3], s33
+; MUBUF-NEXT:    s_waitcnt vmcnt(0)
+; MUBUF-NEXT:    s_swappc_b64 s[30:31], s[16:17]
+; MUBUF-NEXT:    v_readlane_b32 s31, v40, 1
+; MUBUF-NEXT:    v_readlane_b32 s30, v40, 0
+; MUBUF-NEXT:    s_mov_b32 s32, s33
+; MUBUF-NEXT:    v_readlane_b32 s4, v40, 2
+; MUBUF-NEXT:    s_or_saveexec_b64 s[6:7], -1
+; MUBUF-NEXT:    buffer_load_dword v40, off, s[0:3], s33 offset:4 ; 4-byte Folded Reload
+; MUBUF-NEXT:    s_mov_b64 exec, s[6:7]
+; MUBUF-NEXT:    s_mov_b32 s33, s4
+; MUBUF-NEXT:    s_waitcnt vmcnt(0)
+; MUBUF-NEXT:    s_setpc_b64 s[30:31]
+;
+; FLATSCR-LABEL: callee_with_stack_and_call:
+; FLATSCR:       ; %bb.0:
+; FLATSCR-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; FLATSCR-NEXT:    s_mov_b32 s0, s33
+; FLATSCR-NEXT:    s_mov_b32 s33, s32
+; FLATSCR-NEXT:    s_or_saveexec_b64 s[2:3], -1
+; FLATSCR-NEXT:    scratch_store_dword off, v40, s33 offset:4 ; 4-byte Folded Spill
+; FLATSCR-NEXT:    s_mov_b64 exec, s[2:3]
+; FLATSCR-NEXT:    v_writelane_b32 v40, s0, 2
+; FLATSCR-NEXT:    s_add_i32 s32, s32, 16
+; FLATSCR-NEXT:    v_writelane_b32 v40, s30, 0
+; FLATSCR-NEXT:    v_mov_b32_e32 v0, 0
+; FLATSCR-NEXT:    s_getpc_b64 s[0:1]
+; FLATSCR-NEXT:    s_add_u32 s0, s0, external_void_func_void@rel32@lo+4
+; FLATSCR-NEXT:    s_addc_u32 s1, s1, external_void_func_void@rel32@hi+12
+; FLATSCR-NEXT:    v_writelane_b32 v40, s31, 1
+; FLATSCR-NEXT:    scratch_store_dword off, v0, s33
+; FLATSCR-NEXT:    s_waitcnt vmcnt(0)
+; FLATSCR-NEXT:    s_swappc_b64 s[30:31], s[0:1]
+; FLATSCR-NEXT:    v_readlane_b32 s31, v40, 1
+; FLATSCR-NEXT:    v_readlane_b32 s30, v40, 0
+; FLATSCR-NEXT:    s_mov_b32 s32, s33
+; FLATSCR-NEXT:    v_readlane_b32 s0, v40, 2
+; FLATSCR-NEXT:    s_or_saveexec_b64 s[2:3], -1
+; FLATSCR-NEXT:    scratch_load_dword v40, off, s33 offset:4 ; 4-byte Folded Reload
+; FLATSCR-NEXT:    s_mov_b64 exec, s[2:3]
+; FLATSCR-NEXT:    s_mov_b32 s33, s0
+; FLATSCR-NEXT:    s_waitcnt vmcnt(0)
+; FLATSCR-NEXT:    s_setpc_b64 s[30:31]
   %alloca = alloca i32, addrspace(5)
   store volatile i32 0, ptr addrspace(5) %alloca
   call void @external_void_func_void()
@@ -130,36 +184,60 @@ define void @callee_with_stack_and_call() #0 {
 ; There is stack usage only because of the need to evict a VGPR for
 ; spilling CSR SGPRs.
 
-; GCN-LABEL: {{^}}callee_no_stack_with_call:
-; GCN: s_waitcnt
-; GCN: s_mov_b32 [[FP_SCRATCH_COPY:s[0-9]+]], s33
-; GCN-NEXT: s_mov_b32 s33, s32
-; GCN-NEXT: s_or_saveexec_b64 [[COPY_EXEC0:s\[[0-9]+:[0-9]+\]]], -1{{$}}
-; MUBUF-NEXT:   buffer_store_dword [[CSR_VGPR:v[0-9]+]], off, s[0:3], s33 ; 4-byte Folded Spill
-; FLATSCR-NEXT: scratch_store_dword off, [[CSR_VGPR:v[0-9]+]], s33 ; 4-byte Folded Spill
-; GCN-NEXT: s_mov_b64 exec, [[COPY_EXEC0]]
-; MUBUF-DAG:   s_addk_i32 s32, 0x400
-; FLATSCR-DAG: s_add_i32 s32, s32, 16
-; GCN-DAG: v_writelane_b32 [[CSR_VGPR]], [[FP_SCRATCH_COPY]], [[FP_SPILL_LANE:[0-9]+]]
-
-; GCN-DAG: v_writelane_b32 [[CSR_VGPR]], s30, 0
-; GCN-DAG: v_writelane_b32 [[CSR_VGPR]], s31, 1
-; GCN: s_swappc_b64
-
-; GCN-DAG: v_readlane_b32 s30, [[CSR_VGPR]], 0
-; GCN-DAG: v_readlane_b32 s31, [[CSR_VGPR]], 1
-
-; MUBUF:   s_mov_b32 s32, s33
-; FLATSCR: s_mov_b32 s32, s33
-; GCN-NEXT: v_readlane_b32 [[FP_SCRATCH_COPY:s[0-9]+]], [[CSR_VGPR]], [[FP_SPILL_LANE]]
-; GCN-NEXT: s_or_saveexec_b64 [[COPY_EXEC1:s\[[0-9]+:[0-9]+\]]], -1{{$}}
-; MUBUF-NEXT:   buffer_load_dword [[CSR_VGPR]], off, s[0:3], s33 ; 4-byte Folded Reload
-; FLATSCR-NEXT: scratch_load_dword [[CSR_VGPR]], off, s33 ; 4-byte Folded Reload
-; GCN-NEXT: s_mov_b64 exec, [[COPY_EXEC1]]
-; GCN-NEXT: s_mov_b32 s33, [[FP_SCRATCH_COPY]]
-; GCN-NEXT: s_waitcnt vmcnt(0)
-; GCN-NEXT: s_setpc_b64 s[30:31]
 define void @callee_no_stack_with_call() #0 {
+; MUBUF-LABEL: callee_no_stack_with_call:
+; MUBUF:       ; %bb.0:
+; MUBUF-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; MUBUF-NEXT:    s_mov_b32 s16, s33
+; MUBUF-NEXT:    s_mov_b32 s33, s32
+; MUBUF-NEXT:    s_or_saveexec_b64 s[18:19], -1
+; MUBUF-NEXT:    buffer_store_dword v40, off, s[0:3], s33 ; 4-byte Folded Spill
+; MUBUF-NEXT:    s_mov_b64 exec, s[18:19]
+; MUBUF-NEXT:    v_writelane_b32 v40, s16, 2
+; MUBUF-NEXT:    s_addk_i32 s32, 0x400
+; MUBUF-NEXT:    v_writelane_b32 v40, s30, 0
+; MUBUF-NEXT:    s_getpc_b64 s[16:17]
+; MUBUF-NEXT:    s_add_u32 s16, s16, external_void_func_void@rel32@lo+4
+; MUBUF-NEXT:    s_addc_u32 s17, s17, external_void_func_void@rel32@hi+12
+; MUBUF-NEXT:    v_writelane_b32 v40, s31, 1
+; MUBUF-NEXT:    s_swappc_b64 s[30:31], s[16:17]
+; MUBUF-NEXT:    v_readlane_b32 s31, v40, 1
+; MUBUF-NEXT:    v_readlane_b32 s30, v40, 0
+; MUBUF-NEXT:    s_mov_b32 s32, s33
+; MUBUF-NEXT:    v_readlane_b32 s4, v40, 2
+; MUBUF-NEXT:    s_or_saveexec_b64 s[6:7], -1
+; MUBUF-NEXT:    buffer_load_dword v40, off, s[0:3], s33 ; 4-byte Folded Reload
+; MUBUF-NEXT:    s_mov_b64 exec, s[6:7]
+; MUBUF-NEXT:    s_mov_b32 s33, s4
+; MUBUF-NEXT:    s_waitcnt vmcnt(0)
+; MUBUF-NEXT:    s_setpc_b64 s[30:31]
+;
+; FLATSCR-LABEL: callee_no_stack_with_call:
+; FLATSCR:       ; %bb.0:
+; FLATSCR-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; FLATSCR-NEXT:    s_mov_b32 s0, s33
+; FLATSCR-NEXT:    s_mov_b32 s33, s32
+; FLATSCR-NEXT:    s_or_saveexec_b64 s[2:3], -1
+; FLATSCR-NEXT:    scratch_store_dword off, v40, s33 ; 4-byte Folded Spill
+; FLATSCR-NEXT:    s_mov_b64 exec, s[2:3]
+; FLATSCR-NEXT:    v_writelane_b32 v40, s0, 2
+; FLATSCR-NEXT:    s_add_i32 s32, s32, 16
+; FLATSCR-NEXT:    v_writelane_b32 v40, s30, 0
+; FLATSCR-NEXT:    s_getpc_b64 s[0:1]
+; FLATSCR-NEXT:    s_add_u32 s0, s0, external_void_func_void@rel32@lo+4
+; FLATSCR-NEXT:    s_addc_u32 s1, s1, external_void_func_void@rel32@hi+12
+; FLATSCR-NEXT:    v_writelane_b32 v40, s31, 1
+; FLATSCR-NEXT:    s_swappc_b64 s[30:31], s[0:1]
+; FLATSCR-NEXT:    v_readlane_b32 s31, v40, 1
+; FLATSCR-NEXT:    v_readlane_b32 s30, v40, 0
+; FLATSCR-NEXT:    s_mov_b32 s32, s33
+; FLATSCR-NEXT:    v_readlane_b32 s0, v40, 2
+; FLATSCR-NEXT:    s_or_saveexec_b64 s[2:3], -1
+; FLATSCR-NEXT:    scratch_load_dword v40, off, s33 ; 4-byte Folded Reload
+; FLATSCR-NEXT:    s_mov_b64 exec, s[2:3]
+; FLATSCR-NEXT:    s_mov_b32 s33, s0
+; FLATSCR-NEXT:    s_waitcnt vmcnt(0)
+; FLATSCR-NEXT:    s_setpc_b64 s[30:31]
   call void @external_void_func_void()
   ret void
 }
@@ -168,26 +246,306 @@ declare hidden void @external_void_func_void() #0
 
 ; Make sure if a CSR vgpr is used for SGPR spilling, it is saved and
 ; restored. No FP is required.
-;
-; GCN-LABEL: {{^}}callee_func_sgpr_spill_no_calls:
-; GCN: s_or_saveexec_b64 [[COPY_EXEC0:s\[[0-9]+:[0-9]+\]]], -1{{$}}
-; MUBUF-NEXT:   buffer_store_dword [[CSR_VGPR:v[0-9]+]], off, s[0:3], s32 ; 4-byte Folded Spill
-; FLATSCR-NEXT: scratch_store_dword off, [[CSR_VGPR:v[0-9]+]], s32 ; 4-byte Folded Spill
-; GCN-NEXT: s_mov_b64 exec, [[COPY_EXEC0]]
-; GCN: v_writelane_b32 [[CSR_VGPR]], s
-; GCN: v_writelane_b32 [[CSR_VGPR]], s
-
-; GCN: ;;#ASMSTART
-; GCN: v_readlane_b32 s{{[0-9]+}}, [[CSR_VGPR]]
-; GCN: v_readlane_b32 s{{[0-9]+}}, [[CSR_VGPR]]
-
-; GCN: s_or_saveexec_b64 [[COPY_EXEC1:s\[[0-9]+:[0-9]+\]]], -1{{$}}
-; MUBUF-NEXT:   buffer_load_dword [[CSR_VGPR]], off, s[0:3], s32 ; 4-byte Folded Reload
-; FLATSCR-NEXT: scratch_load_dword [[CSR_VGPR]], off, s32 ; 4-byte Folded Reload
-; GCN-NEXT: s_mov_b64 exec, [[COPY_EXEC1]]
-; GCN-NEXT: s_waitcnt
-; GCN-NEXT: s_setpc_b64
 define void @callee_func_sgpr_spill_no_calls(i32 %in) #0 {
+; MUBUF-LABEL: callee_func_sgpr_spill_no_calls:
+; MUBUF:       ; %bb.0:
+; MUBUF-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; MUBUF-NEXT:    s_or_saveexec_b64 s[4:5], -1
+; MUBUF-NEXT:    buffer_store_dword v40, off, s[0:3], s32 ; 4-byte Folded Spill
+; MUBUF-NEXT:    s_mov_b64 exec, s[4:5]
+; MUBUF-NEXT:    v_writelane_b32 v40, s36, 0
+; MUBUF-NEXT:    v_writelane_b32 v40, s37, 1
+; MUBUF-NEXT:    v_writelane_b32 v40, s38, 2
+; MUBUF-NEXT:    v_writelane_b32 v40, s39, 3
+; MUBUF-NEXT:    v_writelane_b32 v40, s40, 4
+; MUBUF-NEXT:    v_writelane_b32 v40, s41, 5
+; MUBUF-NEXT:    v_writelane_b32 v40, s42, 6
+; MUBUF-NEXT:    v_writelane_b32 v40, s43, 7
+; MUBUF-NEXT:    v_writelane_b32 v40, s44, 8
+; MUBUF-NEXT:    v_writelane_b32 v40, s45, 9
+; MUBUF-NEXT:    v_writelane_b32 v40, s46, 10
+; MUBUF-NEXT:    v_writelane_b32 v40, s47, 11
+; MUBUF-NEXT:    v_writelane_b32 v40, s48, 12
+; MUBUF-NEXT:    v_writelane_b32 v40, s49, 13
+; MUBUF-NEXT:    v_writelane_b32 v40, s50, 14
+; MUBUF-NEXT:    v_writelane_b32 v40, s51, 15
+; MUBUF-NEXT:    v_writelane_b32 v40, s52, 16
+; MUBUF-NEXT:    v_writelane_b32 v40, s53, 17
+; MUBUF-NEXT:    v_writelane_b32 v40, s54, 18
+; MUBUF-NEXT:    v_writelane_b32 v40, s55, 19
+; MUBUF-NEXT:    v_writelane_b32 v40, s56, 20
+; MUBUF-NEXT:    v_writelane_b32 v40, s57, 21
+; MUBUF-NEXT:    v_writelane_b32 v40, s58, 22
+; MUBUF-NEXT:    v_writelane_b32 v40, s59, 23
+; MUBUF-NEXT:    v_writelane_b32 v40, s60, 24
+; MUBUF-NEXT:    v_writelane_b32 v40, s61, 25
+; MUBUF-NEXT:    v_writelane_b32 v40, s62, 26
+; MUBUF-NEXT:    v_writelane_b32 v40, s63, 27
+; MUBUF-NEXT:    v_writelane_b32 v40, s64, 28
+; MUBUF-NEXT:    v_writelane_b32 v40, s65, 29
+; MUBUF-NEXT:    v_writelane_b32 v40, s66, 30
+; MUBUF-NEXT:    v_writelane_b32 v40, s67, 31
+; MUBUF-NEXT:    v_writelane_b32 v40, s68, 32
+; MUBUF-NEXT:    v_writelane_b32 v40, s69, 33
+; MUBUF-NEXT:    v_writelane_b32 v40, s70, 34
+; MUBUF-NEXT:    v_writelane_b32 v40, s71, 35
+; MUBUF-NEXT:    v_writelane_b32 v40, s72, 36
+; MUBUF-NEXT:    v_writelane_b32 v40, s73, 37
+; MUBUF-NEXT:    v_writelane_b32 v40, s74, 38
+; MUBUF-NEXT:    v_writelane_b32 v40, s75, 39
+; MUBUF-NEXT:    v_writelane_b32 v40, s76, 40
+; MUBUF-NEXT:    v_writelane_b32 v40, s77, 41
+; MUBUF-NEXT:    v_writelane_b32 v40, s78, 42
+; MUBUF-NEXT:    v_writelane_b32 v40, s79, 43
+; MUBUF-NEXT:    v_writelane_b32 v40, s80, 44
+; MUBUF-NEXT:    v_writelane_b32 v40, s81, 45
+; MUBUF-NEXT:    v_writelane_b32 v40, s82, 46
+; MUBUF-NEXT:    v_writelane_b32 v40, s83, 47
+; MUBUF-NEXT:    ;;#ASMSTART
+; MUBUF-NEXT:    ;;#ASMEND
+; MUBUF-NEXT:    ;;#ASMSTART
+; MUBUF-NEXT:    ;;#ASMEND
+; MUBUF-NEXT:    ;;#ASMSTART
+; MUBUF-NEXT:    ;;#ASMEND
+; MUBUF-NEXT:    ;;#ASMSTART
+; MUBUF-NEXT:    ;;#ASMEND
+; MUBUF-NEXT:    ;;#ASMSTART
+; MUBUF-NEXT:    ;;#ASMEND
+; MUBUF-NEXT:    ;;#ASMSTART
+; MUBUF-NEXT:    ; def s[68:83]
+; MUBUF-NEXT:    ;;#ASMEND
+; MUBUF-NEXT:    ;;#ASMSTART
+; MUBUF-NEXT:    ; def s[52:67]
+; MUBUF-NEXT:    ;;#ASMEND
+; MUBUF-NEXT:    ;;#ASMSTART
+; MUBUF-NEXT:    ; def s[36:51]
+; MUBUF-NEXT:    ;;#ASMEND
+; MUBUF-NEXT:    ;;#ASMSTART
+; MUBUF-NEXT:    ; def s[4:19]
+; MUBUF-NEXT:    ;;#ASMEND
+; MUBUF-NEXT:    ;;#ASMSTART
+; MUBUF-NEXT:    ; def s[20:27]
+; MUBUF-NEXT:    ;;#ASMEND
+; MUBUF-NEXT:    ;;#ASMSTART
+; MUBUF-NEXT:    ; def s[28:29]
+; MUBUF-NEXT:    ;;#ASMEND
+; MUBUF-NEXT:    ;;#ASMSTART
+; MUBUF-NEXT:    ; use s[68:83]
+; MUBUF-NEXT:    ;;#ASMEND
+; MUBUF-NEXT:    ;;#ASMSTART
+; MUBUF-NEXT:    ; use s[52:67]
+; MUBUF-NEXT:    ;;#ASMEND
+; MUBUF-NEXT:    ;;#ASMSTART
+; MUBUF-NEXT:    ; use s[36:51]
+; MUBUF-NEXT:    ;;#ASMEND
+; MUBUF-NEXT:    ;;#ASMSTART
+; MUBUF-NEXT:    ; use s[20:27]
+; MUBUF-NEXT:    ;;#ASMEND
+; MUBUF-NEXT:    ;;#ASMSTART
+; MUBUF-NEXT:    ; use s[28:29]
+; MUBUF-NEXT:    ;;#ASMEND
+; MUBUF-NEXT:    ;;#ASMSTART
+; MUBUF-NEXT:    ; use s[4:19]
+; MUBUF-NEXT:    ;;#ASMEND
+; MUBUF-NEXT:    v_readlane_b32 s83, v40, 47
+; MUBUF-NEXT:    v_readlane_b32 s82, v40, 46
+; MUBUF-NEXT:    v_readlane_b32 s81, v40, 45
+; MUBUF-NEXT:    v_readlane_b32 s80, v40, 44
+; MUBUF-NEXT:    v_readlane_b32 s79, v40, 43
+; MUBUF-NEXT:    v_readlane_b32 s78, v40, 42
+; MUBUF-NEXT:    v_readlane_b32 s77, v40, 41
+; MUBUF-NEXT:    v_readlane_b32 s76, v40, 40
+; MUBUF-NEXT:    v_readlane_b32 s75, v40, 39
+; MUBUF-NEXT:    v_readlane_b32 s74, v40, 38
+; MUBUF-NEXT:    v_readlane_b32 s73, v40, 37
+; MUBUF-NEXT:    v_readlane_b32 s72, v40, 36
+; MUBUF-NEXT:    v_readlane_b32 s71, v40, 35
+; MUBUF-NEXT:    v_readlane_b32 s70, v40, 34
+; MUBUF-NEXT:    v_readlane_b32 s69, v40, 33
+; MUBUF-NEXT:    v_readlane_b32 s68, v40, 32
+; MUBUF-NEXT:    v_readlane_b32 s67, v40, 31
+; MUBUF-NEXT:    v_readlane_b32 s66, v40, 30
+; MUBUF-NEXT:    v_readlane_b32 s65, v40, 29
+; MUBUF-NEXT:    v_readlane_b32 s64, v40, 28
+; MUBUF-NEXT:    v_readlane_b32 s63, v40, 27
+; MUBUF-NEXT:    v_readlane_b32 s62, v40, 26
+; MUBUF-NEXT:    v_readlane_b32 s61, v40, 25
+; MUBUF-NEXT:    v_readlane_b32 s60, v40, 24
+; MUBUF-NEXT:    v_readlane_b32 s59, v40, 23
+; MUBUF-NEXT:    v_readlane_b32 s58, v40, 22
+; MUBUF-NEXT:    v_readlane_b32 s57, v40, 21
+; MUBUF-NEXT:    v_readlane_b32 s56, v40, 20
+; MUBUF-NEXT:    v_readlane_b32 s55, v40, 19
+; MUBUF-NEXT:    v_readlane_b32 s54, v40, 18
+; MUBUF-NEXT:    v_readlane_b32 s53, v40, 17
+; MUBUF-NEXT:    v_readlane_b32 s52, v40, 16
+; MUBUF-NEXT:    v_readlane_b32 s51, v40, 15
+; MUBUF-NEXT:    v_readlane_b32 s50, v40, 14
+; MUBUF-NEXT:    v_readlane_b32 s49, v40, 13
+; MUBUF-NEXT:    v_readlane_b32 s48, v40, 12
+; MUBUF-NEXT:    v_readlane_b32 s47, v40, 11
+; MUBUF-NEXT:    v_readlane_b32 s46, v40, 10
+; MUBUF-NEXT:    v_readlane_b32 s45, v40, 9
+; MUBUF-NEXT:    v_readlane_b32 s44, v40, 8
+; MUBUF-NEXT:    v_readlane_b32 s43, v40, 7
+; MUBUF-NEXT:    v_readlane_b32 s42, v40, 6
+; MUBUF-NEXT:    v_readlane_b32 s41, v40, 5
+; MUBUF-NEXT:    v_readlane_b32 s40, v40, 4
+; MUBUF-NEXT:    v_readlane_b32 s39, v40, 3
+; MUBUF-NEXT:    v_readlane_b32 s38, v40, 2
+; MUBUF-NEXT:    v_readlane_b32 s37, v40, 1
+; MUBUF-NEXT:    v_readlane_b32 s36, v40, 0
+; MUBUF-NEXT:    s_or_saveexec_b64 s[4:5], -1
+; MUBUF-NEXT:    buffer_load_dword v40, off, s[0:3], s32 ; 4-byte Folded Reload
+; MUBUF-NEXT:    s_mov_b64 exec, s[4:5]
+; MUBUF-NEXT:    s_waitcnt vmcnt(0)
+; MUBUF-NEXT:    s_setpc_b64 s[30:31]
+;
+; FLATSCR-LABEL: callee_func_sgpr_spill_no_calls:
+; FLATSCR:       ; %bb.0:
+; FLATSCR-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; FLATSCR-NEXT:    s_or_saveexec_b64 s[0:1], -1
+; FLATSCR-NEXT:    scratch_store_dword off, v40, s32 ; 4-byte Folded Spill
+; FLATSCR-NEXT:    s_mov_b64 exec, s[0:1]
+; FLATSCR-NEXT:    v_writelane_b32 v40, s30, 0
+; FLATSCR-NEXT:    v_writelane_b32 v40, s31, 1
+; FLATSCR-NEXT:    v_writelane_b32 v40, s34, 2
+; FLATSCR-NEXT:    v_writelane_b32 v40, s35, 3
+; FLATSCR-NEXT:    v_writelane_b32 v40, s36, 4
+; FLATSCR-NEXT:    v_writelane_b32 v40, s37, 5
+; FLATSCR-NEXT:    v_writelane_b32 v40, s38, 6
+; FLATSCR-NEXT:    v_writelane_b32 v40, s39, 7
+; FLATSCR-NEXT:    v_writelane_b32 v40, s40, 8
+; FLATSCR-NEXT:    v_writelane_b32 v40, s41, 9
+; FLATSCR-NEXT:    v_writelane_b32 v40, s42, 10
+; FLATSCR-NEXT:    v_writelane_b32 v40, s43, 11
+; FLATSCR-NEXT:    v_writelane_b32 v40, s44, 12
+; FLATSCR-NEXT:    v_writelane_b32 v40, s45, 13
+; FLATSCR-NEXT:    v_writelane_b32 v40, s46, 14
+; FLATSCR-NEXT:    v_writelane_b32 v40, s47, 15
+; FLATSCR-NEXT:    v_writelane_b32 v40, s48, 16
+; FLATSCR-NEXT:    v_writelane_b32 v40, s49, 17
+; FLATSCR-NEXT:    v_writelane_b32 v40, s50, 18
+; FLATSCR-NEXT:    v_writelane_b32 v40, s51, 19
+; FLATSCR-NEXT:    v_writelane_b32 v40, s52, 20
+; FLATSCR-NEXT:    v_writelane_b32 v40, s53, 21
+; FLATSCR-NEXT:    v_writelane_b32 v40, s54, 22
+; FLATSCR-NEXT:    v_writelane_b32 v40, s55, 23
+; FLATSCR-NEXT:    v_writelane_b32 v40, s56, 24
+; FLATSCR-NEXT:    v_writelane_b32 v40, s57, 25
+; FLATSCR-NEXT:    v_writelane_b32 v40, s58, 26
+; FLATSCR-NEXT:    v_writelane_b32 v40, s59, 27
+; FLATSCR-NEXT:    v_writelane_b32 v40, s60, 28
+; FLATSCR-NEXT:    v_writelane_b32 v40, s61, 29
+; FLATSCR-NEXT:    v_writelane_b32 v40, s62, 30
+; FLATSCR-NEXT:    v_writelane_b32 v40, s63, 31
+; FLATSCR-NEXT:    v_writelane_b32 v40, s64, 32
+; FLATSCR-NEXT:    v_writelane_b32 v40, s65, 33
+; FLATSCR-NEXT:    v_writelane_b32 v40, s66, 34
+; FLATSCR-NEXT:    v_writelane_b32 v40, s67, 35
+; FLATSCR-NEXT:    v_writelane_b32 v40, s68, 36
+; FLATSCR-NEXT:    v_writelane_b32 v40, s69, 37
+; FLATSCR-NEXT:    v_writelane_b32 v40, s70, 38
+; FLATSCR-NEXT:    v_writelane_b32 v40, s71, 39
+; FLATSCR-NEXT:    v_writelane_b32 v40, s72, 40
+; FLATSCR-NEXT:    v_writelane_b32 v40, s73, 41
+; FLATSCR-NEXT:    v_writelane_b32 v40, s74, 42
+; FLATSCR-NEXT:    v_writelane_b32 v40, s75, 43
+; FLATSCR-NEXT:    ;;#ASMSTART
+; FLATSCR-NEXT:    ;;#ASMEND
+; FLATSCR-NEXT:    ;;#ASMSTART
+; FLATSCR-NEXT:    ;;#ASMEND
+; FLATSCR-NEXT:    ;;#ASMSTART
+; FLATSCR-NEXT:    ;;#ASMEND
+; FLATSCR-NEXT:    ;;#ASMSTART
+; FLATSCR-NEXT:    ;;#ASMEND
+; FLATSCR-NEXT:    ;;#ASMSTART
+; FLATSCR-NEXT:    ;;#ASMEND
+; FLATSCR-NEXT:    ;;#ASMSTART
+; FLATSCR-NEXT:    ; def s[52:67]
+; FLATSCR-NEXT:    ;;#ASMEND
+; FLATSCR-NEXT:    ;;#ASMSTART
+; FLATSCR-NEXT:    ; def s[36:51]
+; FLATSCR-NEXT:    ;;#ASMEND
+; FLATSCR-NEXT:    ;;#ASMSTART
+; FLATSCR-NEXT:    ; def s[16:31]
+; FLATSCR-NEXT:    ;;#ASMEND
+; FLATSCR-NEXT:    ;;#ASMSTART
+; FLATSCR-NEXT:    ; def s[0:15]
+; FLATSCR-NEXT:    ;;#ASMEND
+; FLATSCR-NEXT:    ;;#ASMSTART
+; FLATSCR-NEXT:    ; def s[68:75]
+; FLATSCR-NEXT:    ;;#ASMEND
+; FLATSCR-NEXT:    ;;#ASMSTART
+; FLATSCR-NEXT:    ; def s[34:35]
+; FLATSCR-NEXT:    ;;#ASMEND
+; FLATSCR-NEXT:    ;;#ASMSTART
+; FLATSCR-NEXT:    ; use s[52:67]
+; FLATSCR-NEXT:    ;;#ASMEND
+; FLATSCR-NEXT:    ;;#ASMSTART
+; FLATSCR-NEXT:    ; use s[36:51]
+; FLATSCR-NEXT:    ;;#ASMEND
+; FLATSCR-NEXT:    ;;#ASMSTART
+; FLATSCR-NEXT:    ; use s[16:31]
+; FLATSCR-NEXT:    ;;#ASMEND
+; FLATSCR-NEXT:    ;;#ASMSTART
+; FLATSCR-NEXT:    ; use s[68:75]
+; FLATSCR-NEXT:    ;;#ASMEND
+; FLATSCR-NEXT:    ;;#ASMSTART
+; FLATSCR-NEXT:    ; use s[34:35]
+; FLATSCR-NEXT:    ;;#ASMEND
+; FLATSCR-NEXT:    ;;#ASMSTART
+; FLATSCR-NEXT:    ; use s[0:15]
+; FLATSCR-NEXT:    ;;#ASMEND
+; FLATSCR-NEXT:    v_readlane_b32 s75, v40, 43
+; FLATSCR-NEXT:    v_readlane_b32 s74, v40, 42
+; FLATSCR-NEXT:    v_readlane_b32 s73, v40, 41
+; FLATSCR-NEXT:    v_readlane_b32 s72, v40, 40
+; FLATSCR-NEXT:    v_readlane_b32 s71, v40, 39
+; FLATSCR-NEXT:    v_readlane_b32 s70, v40, 38
+; FLATSCR-NEXT:    v_readlane_b32 s69, v40, 37
+; FLATSCR-NEXT:    v_readlane_b32 s68, v40, 36
+; FLATSCR-NEXT:    v_readlane_b32 s67, v40, 35
+; FLATSCR-NEXT:    v_readlane_b32 s66, v40, 34
+; FLATSCR-NEXT:    v_readlane_b32 s65, v40, 33
+; FLATSCR-NEXT:    v_readlane_b32 s64, v40, 32
+; FLATSCR-NEXT:    v_readlane_b32 s63, v40, 31
+; FLATSCR-NEXT:    v_readlane_b32 s62, v40, 30
+; FLATSCR-NEXT:    v_readlane_b32 s61, v40, 29
+; FLATSCR-NEXT:    v_readlane_b32 s60, v40, 28
+; FLATSCR-NEXT:    v_readlane_b32 s59, v40, 27
+; FLATSCR-NEXT:    v_readlane_b32 s58, v40, 26
+; FLATSCR-NEXT:    v_readlane_b32 s57, v40, 25
+; FLATSCR-NEXT:    v_readlane_b32 s56, v40, 24
+; FLATSCR-NEXT:    v_readlane_b32 s55, v40, 23
+; FLATSCR-NEXT:    v_readlane_b32 s54, v40, 22
+; FLATSCR-NEXT:    v_readlane_b32 s53, v40, 21
+; FLATSCR-NEXT:    v_readlane_b32 s52, v40, 20
+; FLATSCR-NEXT:    v_readlane_b32 s51, v40, 19
+; FLATSCR-NEXT:    v_readlane_b32 s50, v40, 18
+; FLATSCR-NEXT:    v_readlane_b32 s49, v40, 17
+; FLATSCR-NEXT:    v_readlane_b32 s48, v40, 16
+; FLATSCR-NEXT:    v_readlane_b32 s47, v40, 15
+; FLATSCR-NEXT:    v_readlane_b32 s46, v40, 14
+; FLATSCR-NEXT:    v_readlane_b32 s45, v40, 13
+; FLATSCR-NEXT:    v_readlane_b32 s44, v40, 12
+; FLATSCR-NEXT:    v_readlane_b32 s43, v40, 11
+; FLATSCR-NEXT:    v_readlane_b32 s42, v40, 10
+; FLATSCR-NEXT:    v_readlane_b32 s41, v40, 9
+; FLATSCR-NEXT:    v_readlane_b32 s40, v40, 8
+; FLATSCR-NEXT:    v_readlane_b32 s39, v40, 7
+; FLATSCR-NEXT:    v_readlane_b32 s38, v40, 6
+; FLATSCR-NEXT:    v_readlane_b32 s37, v40, 5
+; FLATSCR-NEXT:    v_readlane_b32 s36, v40, 4
+; FLATSCR-NEXT:    v_readlane_b32 s35, v40, 3
+; FLATSCR-NEXT:    v_readlane_b32 s34, v40, 2
+; FLATSCR-NEXT:    v_readlane_b32 s31, v40, 1
+; FLATSCR-NEXT:    v_readlane_b32 s30, v40, 0
+; FLATSCR-NEXT:    s_or_saveexec_b64 s[0:1], -1
+; FLATSCR-NEXT:    scratch_load_dword v40, off, s32 ; 4-byte Folded Reload
+; FLATSCR-NEXT:    s_mov_b64 exec, s[0:1]
+; FLATSCR-NEXT:    s_waitcnt vmcnt(0)
+; FLATSCR-NEXT:    s_setpc_b64 s[30:31]
   call void asm sideeffect "", "~{v0},~{v1},~{v2},~{v3},~{v4},~{v5},~{v6},~{v7}"() #0
   call void asm sideeffect "", "~{v8},~{v9},~{v10},~{v11},~{v12},~{v13},~{v14},~{v15}"() #0
   call void asm sideeffect "", "~{v16},~{v17},~{v18},~{v19},~{v20},~{v21},~{v22},~{v23}"() #0
@@ -212,55 +570,83 @@ define void @callee_func_sgpr_spill_no_calls(i32 %in) #0 {
 
 ; Has no spilled CSR VGPRs used for SGPR spilling, so no need to
 ; enable all lanes and restore.
-
-; GCN-LABEL: {{^}}spill_only_csr_sgpr:
-; GCN: s_waitcnt
-; GCN-NEXT: s_xor_saveexec_b64
-; MUBUF-NEXT: buffer_store_dword v0, off, s[0:3], s32 ; 4-byte Folded Spill
-; FLATSCR-NEXT: scratch_store_dword off, v0, s32 ; 4-byte Folded Spill
-; GCN-NEXT: s_mov_b64 exec,
-; GCN-NEXT: v_writelane_b32 v0, s42, 0
-; GCN-NEXT: ;;#ASMSTART
-; GCN-NEXT: ; clobber s42
-; GCN-NEXT: ;;#ASMEND
-; GCN-NEXT: v_readlane_b32 s42, v0, 0
-; GCN-NEXT: s_xor_saveexec_b64
-; MUBUF-NEXT: buffer_load_dword v0, off, s[0:3], s32 ; 4-byte Folded Reload
-; FLATSCR-NEXT: scratch_load_dword v0, off, s32 ; 4-byte Folded Reload
-; GCN-NEXT: s_mov_b64 exec,
-; GCN-NEXT: s_waitcnt vmcnt(0)
-; GCN-NEXT: s_setpc_b64
 define void @spill_only_csr_sgpr() {
+; MUBUF-LABEL: spill_only_csr_sgpr:
+; MUBUF:       ; %bb.0:
+; MUBUF-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; MUBUF-NEXT:    s_xor_saveexec_b64 s[4:5], -1
+; MUBUF-NEXT:    buffer_store_dword v0, off, s[0:3], s32 ; 4-byte Folded Spill
+; MUBUF-NEXT:    s_mov_b64 exec, s[4:5]
+; MUBUF-NEXT:    v_writelane_b32 v0, s42, 0
+; MUBUF-NEXT:    ;;#ASMSTART
+; MUBUF-NEXT:    ; clobber s42
+; MUBUF-NEXT:    ;;#ASMEND
+; MUBUF-NEXT:    v_readlane_b32 s42, v0, 0
+; MUBUF-NEXT:    s_xor_saveexec_b64 s[4:5], -1
+; MUBUF-NEXT:    buffer_load_dword v0, off, s[0:3], s32 ; 4-byte Folded Reload
+; MUBUF-NEXT:    s_mov_b64 exec, s[4:5]
+; MUBUF-NEXT:    s_waitcnt vmcnt(0)
+; MUBUF-NEXT:    s_setpc_b64 s[30:31]
+;
+; FLATSCR-LABEL: spill_only_csr_sgpr:
+; FLATSCR:       ; %bb.0:
+; FLATSCR-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; FLATSCR-NEXT:    s_xor_saveexec_b64 s[0:1], -1
+; FLATSCR-NEXT:    scratch_store_dword off, v0, s32 ; 4-byte Folded Spill
+; FLATSCR-NEXT:    s_mov_b64 exec, s[0:1]
+; FLATSCR-NEXT:    v_writelane_b32 v0, s42, 0
+; FLATSCR-NEXT:    ;;#ASMSTART
+; FLATSCR-NEXT:    ; clobber s42
+; FLATSCR-NEXT:    ;;#ASMEND
+; FLATSCR-NEXT:    v_readlane_b32 s42, v0, 0
+; FLATSCR-NEXT:    s_xor_saveexec_b64 s[0:1], -1
+; FLATSCR-NEXT:    scratch_load_dword v0, off, s32 ; 4-byte Folded Reload
+; FLATSCR-NEXT:    s_mov_b64 exec, s[0:1]
+; FLATSCR-NEXT:    s_waitcnt vmcnt(0)
+; FLATSCR-NEXT:    s_setpc_b64 s[30:31]
   call void asm sideeffect "; clobber s42", "~{s42}"()
   ret void
 }
 
 ; TODO: Can the SP inc/deec be remvoed?
-; GCN-LABEL: {{^}}callee_with_stack_no_fp_elim_csr_vgpr:
-; GCN: s_waitcnt
-; GCN-NEXT:s_mov_b32 [[FP_COPY:s[0-9]+]], s33
-; GCN-NEXT: s_mov_b32 s33, s32
-; GCN: v_mov_b32_e32 [[ZERO:v[0-9]+]], 0
-; MUBUF-DAG:   buffer_store_dword v41, off, s[0:3], s33 ; 4-byte Folded Spill
-; FLATSCR-DAG: scratch_store_dword off, v41, s33 ; 4-byte Folded Spill
-; MUBUF-DAG:   buffer_store_dword [[ZERO]], off, s[0:3], s33 offset:4
-; FLATSCR-DAG: scratch_store_dword off, [[ZERO]], s33 offset:4
-
-; GCN:	;;#ASMSTART
-; GCN-NEXT: ; clobber v41
-; GCN-NEXT: ;;#ASMEND
-
-; MUBUF:   buffer_load_dword v41, off, s[0:3], s33 ; 4-byte Folded Reload
-; FLATSCR: scratch_load_dword v41, off, s33 ; 4-byte Folded Reload
-; MUBUF:        s_addk_i32 s32, 0x300
-; MUBUF-NEXT:   s_mov_b32 s32, s33
-; MUBUF-NEXT:   s_mov_b32 s33, s4
-; FLATSCR:      s_add_i32 s32, s32, 12
-; FLATSCR-NEXT:   s_mov_b32 s32, s33
-; FLATSCR-NEXT: s_mov_b32 s33, s0
-; GCN-NEXT: s_waitcnt vmcnt(0)
-; GCN-NEXT: s_setpc_b64
 define void @callee_with_stack_no_fp_elim_csr_vgpr() #1 {
+; MUBUF-LABEL: callee_with_stack_no_fp_elim_csr_vgpr:
+; MUBUF:       ; %bb.0:
+; MUBUF-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; MUBUF-NEXT:    s_mov_b32 s4, s33
+; MUBUF-NEXT:    s_mov_b32 s33, s32
+; MUBUF-NEXT:    v_mov_b32_e32 v0, 0
+; MUBUF-NEXT:    buffer_store_dword v41, off, s[0:3], s33 ; 4-byte Folded Spill
+; MUBUF-NEXT:    buffer_store_dword v0, off, s[0:3], s33 offset:4
+; MUBUF-NEXT:    s_waitcnt vmcnt(0)
+; MUBUF-NEXT:    ;;#ASMSTART
+; MUBUF-NEXT:    ; clobber v41
+; MUBUF-NEXT:    ;;#ASMEND
+; MUBUF-NEXT:    buffer_load_dword v41, off, s[0:3], s33 ; 4-byte Folded Reload
+; MUBUF-NEXT:    s_addk_i32 s32, 0x300
+; MUBUF-NEXT:    s_mov_b32 s32, s33
+; MUBUF-NEXT:    s_mov_b32 s33, s4
+; MUBUF-NEXT:    s_waitcnt vmcnt(0)
+; MUBUF-NEXT:    s_setpc_b64 s[30:31]
+;
+; FLATSCR-LABEL: callee_with_stack_no_fp_elim_csr_vgpr:
+; FLATSCR:       ; %bb.0:
+; FLATSCR-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; FLATSCR-NEXT:    s_mov_b32 s0, s33
+; FLATSCR-NEXT:    s_mov_b32 s33, s32
+; FLATSCR-NEXT:    v_mov_b32_e32 v0, 0
+; FLATSCR-NEXT:    scratch_store_dword off, v41, s33 ; 4-byte Folded Spill
+; FLATSCR-NEXT:    scratch_store_dword off, v0, s33 offset:4
+; FLATSCR-NEXT:    s_waitcnt vmcnt(0)
+; FLATSCR-NEXT:    ;;#ASMSTART
+; FLATSCR-NEXT:    ; clobber v41
+; FLATSCR-NEXT:    ;;#ASMEND
+; FLATSCR-NEXT:    scratch_load_dword v41, off, s33 ; 4-byte Folded Reload
+; FLATSCR-NEXT:    s_add_i32 s32, s32, 12
+; FLATSCR-NEXT:    s_mov_b32 s32, s33
+; FLATSCR-NEXT:    s_mov_b32 s33, s0
+; FLATSCR-NEXT:    s_waitcnt vmcnt(0)
+; FLATSCR-NEXT:    s_setpc_b64 s[30:31]
   %alloca = alloca i32, addrspace(5)
   store volatile i32 0, ptr addrspace(5) %alloca
   call void asm sideeffect "; clobber v41", "~{v41}"()
@@ -268,32 +654,312 @@ define void @callee_with_stack_no_fp_elim_csr_vgpr() #1 {
 }
 
 ; Use a copy to a free SGPR instead of introducing a second CSR VGPR.
-; GCN-LABEL: {{^}}last_lane_vgpr_for_fp_csr:
-; GCN: s_waitcnt
-; GCN-NEXT: s_mov_b32 [[TMP_SGPR:s[0-9]+]], s33
-; GCN: s_mov_b32 s33, s32
-; GCN-NEXT: s_xor_saveexec_b64 [[COPY_EXEC0:s\[[0-9]+:[0-9]+\]]], -1{{$}}
-; MUBUF-NEXT:   buffer_store_dword [[CSR_VGPR:v[0-9]+]], off, s[0:3], s33 offset:8 ; 4-byte Folded Spill
-; FLATSCR-NEXT: scratch_store_dword off, [[CSR_VGPR:v[0-9]+]], s33 offset:8 ; 4-byte Folded Spill
-; GCN-NEXT: s_mov_b64 exec, [[COPY_EXEC0]]
-; MUBUF:   buffer_store_dword v41, off, s[0:3], s33 ; 4-byte Folded Spill
-; FLATSCR: scratch_store_dword off, v41, s33 ; 4-byte Folded Spill
-; GCN: v_writelane_b32 v1
-; MUBUF:   buffer_store_dword v{{[0-9]+}}, off, s[0:3], s33 offset:4
-; FLATSCR: scratch_store_dword off, v{{[0-9]+}}, s33 offset:4
-; GCN: ;;#ASMSTART
-
-; MUBUF:        s_mov_b32 s32, s33
-; FLATSCR:      s_mov_b32 s32, s33
-
-; GCN: s_xor_saveexec_b64 [[COPY_EXEC1:s\[[0-9]+:[0-9]+\]]], -1{{$}}
-; MUBUF-NEXT:   buffer_load_dword [[CSR_VGPR]], off, s[0:3], s33 offset:8 ; 4-byte Folded Reload
-; FLATSCR-NEXT: scratch_load_dword [[CSR_VGPR]], off, s33 offset:8 ; 4-byte Folded Reload
-; GCN-NEXT: s_mov_b64 exec, [[COPY_EXEC1]]
-; GCN-NEXT: s_mov_b32 s33, [[TMP_SGPR]]
-; GCN-NEXT: s_waitcnt vmcnt(0)
-; GCN-NEXT: s_setpc_b64
 define void @last_lane_vgpr_for_fp_csr() #1 {
+; MUBUF-LABEL: last_lane_vgpr_for_fp_csr:
+; MUBUF:       ; %bb.0:
+; MUBUF-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; MUBUF-NEXT:    s_mov_b32 s4, s33
+; MUBUF-NEXT:    s_mov_b32 s33, s32
+; MUBUF-NEXT:    s_xor_saveexec_b64 s[6:7], -1
+; MUBUF-NEXT:    buffer_store_dword v1, off, s[0:3], s33 offset:8 ; 4-byte Folded Spill
+; MUBUF-NEXT:    s_mov_b64 exec, s[6:7]
+; MUBUF-NEXT:    v_writelane_b32 v1, s40, 0
+; MUBUF-NEXT:    v_writelane_b32 v1, s41, 1
+; MUBUF-NEXT:    v_writelane_b32 v1, s42, 2
+; MUBUF-NEXT:    v_writelane_b32 v1, s43, 3
+; MUBUF-NEXT:    v_writelane_b32 v1, s44, 4
+; MUBUF-NEXT:    v_writelane_b32 v1, s45, 5
+; MUBUF-NEXT:    v_writelane_b32 v1, s46, 6
+; MUBUF-NEXT:    v_writelane_b32 v1, s47, 7
+; MUBUF-NEXT:    v_writelane_b32 v1, s48, 8
+; MUBUF-NEXT:    v_writelane_b32 v1, s49, 9
+; MUBUF-NEXT:    v_writelane_b32 v1, s50, 10
+; MUBUF-NEXT:    v_writelane_b32 v1, s51, 11
+; MUBUF-NEXT:    v_writelane_b32 v1, s52, 12
+; MUBUF-NEXT:    v_writelane_b32 v1, s53, 13
+; MUBUF-NEXT:    v_writelane_b32 v1, s54, 14
+; MUBUF-NEXT:    v_writelane_b32 v1, s55, 15
+; MUBUF-NEXT:    v_writelane_b32 v1, s56, 16
+; MUBUF-NEXT:    v_writelane_b32 v1, s57, 17
+; MUBUF-NEXT:    v_writelane_b32 v1, s58, 18
+; MUBUF-NEXT:    v_writelane_b32 v1, s59, 19
+; MUBUF-NEXT:    v_writelane_b32 v1, s60, 20
+; MUBUF-NEXT:    v_writelane_b32 v1, s61, 21
+; MUBUF-NEXT:    v_writelane_b32 v1, s62, 22
+; MUBUF-NEXT:    v_writelane_b32 v1, s63, 23
+; MUBUF-NEXT:    v_writelane_b32 v1, s64, 24
+; MUBUF-NEXT:    v_writelane_b32 v1, s65, 25
+; MUBUF-NEXT:    v_writelane_b32 v1, s66, 26
+; MUBUF-NEXT:    v_writelane_b32 v1, s67, 27
+; MUBUF-NEXT:    v_writelane_b32 v1, s68, 28
+; MUBUF-NEXT:    v_writelane_b32 v1, s69, 29
+; MUBUF-NEXT:    v_writelane_b32 v1, s70, 30
+; MUBUF-NEXT:    v_writelane_b32 v1, s71, 31
+; MUBUF-NEXT:    v_writelane_b32 v1, s72, 32
+; MUBUF-NEXT:    v_writelane_b32 v1, s73, 33
+; MUBUF-NEXT:    v_writelane_b32 v1, s74, 34
+; MUBUF-NEXT:    v_writelane_b32 v1, s75, 35
+; MUBUF-NEXT:    v_writelane_b32 v1, s76, 36
+; MUBUF-NEXT:    v_writelane_b32 v1, s77, 37
+; MUBUF-NEXT:    v_writelane_b32 v1, s78, 38
+; MUBUF-NEXT:    v_writelane_b32 v1, s79, 39
+; MUBUF-NEXT:    v_writelane_b32 v1, s80, 40
+; MUBUF-NEXT:    v_writelane_b32 v1, s81, 41
+; MUBUF-NEXT:    v_writelane_b32 v1, s82, 42
+; MUBUF-NEXT:    v_writelane_b32 v1, s83, 43
+; MUBUF-NEXT:    v_writelane_b32 v1, s84, 44
+; MUBUF-NEXT:    v_writelane_b32 v1, s85, 45
+; MUBUF-NEXT:    v_writelane_b32 v1, s86, 46
+; MUBUF-NEXT:    v_writelane_b32 v1, s87, 47
+; MUBUF-NEXT:    v_writelane_b32 v1, s88, 48
+; MUBUF-NEXT:    v_writelane_b32 v1, s89, 49
+; MUBUF-NEXT:    v_writelane_b32 v1, s90, 50
+; MUBUF-NEXT:    v_writelane_b32 v1, s91, 51
+; MUBUF-NEXT:    v_writelane_b32 v1, s92, 52
+; MUBUF-NEXT:    v_writelane_b32 v1, s93, 53
+; MUBUF-NEXT:    v_writelane_b32 v1, s94, 54
+; MUBUF-NEXT:    v_writelane_b32 v1, s95, 55
+; MUBUF-NEXT:    v_writelane_b32 v1, s96, 56
+; MUBUF-NEXT:    v_writelane_b32 v1, s97, 57
+; MUBUF-NEXT:    v_writelane_b32 v1, s98, 58
+; MUBUF-NEXT:    v_writelane_b32 v1, s99, 59
+; MUBUF-NEXT:    v_writelane_b32 v1, s100, 60
+; MUBUF-NEXT:    v_mov_b32_e32 v0, 0
+; MUBUF-NEXT:    buffer_store_dword v41, off, s[0:3], s33 ; 4-byte Folded Spill
+; MUBUF-NEXT:    v_writelane_b32 v1, s101, 61
+; MUBUF-NEXT:    buffer_store_dword v0, off, s[0:3], s33 offset:4
+; MUBUF-NEXT:    s_waitcnt vmcnt(0)
+; MUBUF-NEXT:    ;;#ASMSTART
+; MUBUF-NEXT:    ; clobber v41
+; MUBUF-NEXT:    ;;#ASMEND
+; MUBUF-NEXT:    v_writelane_b32 v1, s102, 62
+; MUBUF-NEXT:    ;;#ASMSTART
+; MUBUF-NEXT:    ;;#ASMEND
+; MUBUF-NEXT:    buffer_load_dword v41, off, s[0:3], s33 ; 4-byte Folded Reload
+; MUBUF-NEXT:    s_addk_i32 s32, 0x400
+; MUBUF-NEXT:    v_readlane_b32 s102, v1, 62
+; MUBUF-NEXT:    v_readlane_b32 s101, v1, 61
+; MUBUF-NEXT:    v_readlane_b32 s100, v1, 60
+; MUBUF-NEXT:    v_readlane_b32 s99, v1, 59
+; MUBUF-NEXT:    v_readlane_b32 s98, v1, 58
+; MUBUF-NEXT:    v_readlane_b32 s97, v1, 57
+; MUBUF-NEXT:    v_readlane_b32 s96, v1, 56
+; MUBUF-NEXT:    v_readlane_b32 s95, v1, 55
+; MUBUF-NEXT:    v_readlane_b32 s94, v1, 54
+; MUBUF-NEXT:    v_readlane_b32 s93, v1, 53
+; MUBUF-NEXT:    v_readlane_b32 s92, v1, 52
+; MUBUF-NEXT:    v_readlane_b32 s91, v1, 51
+; MUBUF-NEXT:    v_readlane_b32 s90, v1, 50
+; MUBUF-NEXT:    v_readlane_b32 s89, v1, 49
+; MUBUF-NEXT:    v_readlane_b32 s88, v1, 48
+; MUBUF-NEXT:    v_readlane_b32 s87, v1, 47
+; MUBUF-NEXT:    v_readlane_b32 s86, v1, 46
+; MUBUF-NEXT:    v_readlane_b32 s85, v1, 45
+; MUBUF-NEXT:    v_readlane_b32 s84, v1, 44
+; MUBUF-NEXT:    v_readlane_b32 s83, v1, 43
+; MUBUF-NEXT:    v_readlane_b32 s82, v1, 42
+; MUBUF-NEXT:    v_readlane_b32 s81, v1, 41
+; MUBUF-NEXT:    v_readlane_b32 s80, v1, 40
+; MUBUF-NEXT:    v_readlane_b32 s79, v1, 39
+; MUBUF-NEXT:    v_readlane_b32 s78, v1, 38
+; MUBUF-NEXT:    v_readlane_b32 s77, v1, 37
+; MUBUF-NEXT:    v_readlane_b32 s76, v1, 36
+; MUBUF-NEXT:    v_readlane_b32 s75, v1, 35
+; MUBUF-NEXT:    v_readlane_b32 s74, v1, 34
+; MUBUF-NEXT:    v_readlane_b32 s73, v1, 33
+; MUBUF-NEXT:    v_readlane_b32 s72, v1, 32
+; MUBUF-NEXT:    v_readlane_b32 s71, v1, 31
+; MUBUF-NEXT:    v_readlane_b32 s70, v1, 30
+; MUBUF-NEXT:    v_readlane_b32 s69, v1, 29
+; MUBUF-NEXT:    v_readlane_b32 s68, v1, 28
+; MUBUF-NEXT:    v_readlane_b32 s67, v1, 27
+; MUBUF-NEXT:    v_readlane_b32 s66, v1, 26
+; MUBUF-NEXT:    v_readlane_b32 s65, v1, 25
+; MUBUF-NEXT:    v_readlane_b32 s64, v1, 24
+; MUBUF-NEXT:    v_readlane_b32 s63, v1, 23
+; MUBUF-NEXT:    v_readlane_b32 s62, v1, 22
+; MUBUF-NEXT:    v_readlane_b32 s61, v1, 21
+; MUBUF-NEXT:    v_readlane_b32 s60, v1, 20
+; MUBUF-NEXT:    v_readlane_b32 s59, v1, 19
+; MUBUF-NEXT:    v_readlane_b32 s58, v1, 18
+; MUBUF-NEXT:    v_readlane_b32 s57, v1, 17
+; MUBUF-NEXT:    v_readlane_b32 s56, v1, 16
+; MUBUF-NEXT:    v_readlane_b32 s55, v1, 15
+; MUBUF-NEXT:    v_readlane_b32 s54, v1, 14
+; MUBUF-NEXT:    v_readlane_b32 s53, v1, 13
+; MUBUF-NEXT:    v_readlane_b32 s52, v1, 12
+; MUBUF-NEXT:    v_readlane_b32 s51, v1, 11
+; MUBUF-NEXT:    v_readlane_b32 s50, v1, 10
+; MUBUF-NEXT:    v_readlane_b32 s49, v1, 9
+; MUBUF-NEXT:    v_readlane_b32 s48, v1, 8
+; MUBUF-NEXT:    v_readlane_b32 s47, v1, 7
+; MUBUF-NEXT:    v_readlane_b32 s46, v1, 6
+; MUBUF-NEXT:    v_readlane_b32 s45, v1, 5
+; MUBUF-NEXT:    v_readlane_b32 s44, v1, 4
+; MUBUF-NEXT:    v_readlane_b32 s43, v1, 3
+; MUBUF-NEXT:    v_readlane_b32 s42, v1, 2
+; MUBUF-NEXT:    v_readlane_b32 s41, v1, 1
+; MUBUF-NEXT:    v_readlane_b32 s40, v1, 0
+; MUBUF-NEXT:    s_mov_b32 s32, s33
+; MUBUF-NEXT:    s_xor_saveexec_b64 s[6:7], -1
+; MUBUF-NEXT:    buffer_load_dword v1, off, s[0:3], s33 offset:8 ; 4-byte Folded Reload
+; MUBUF-NEXT:    s_mov_b64 exec, s[6:7]
+; MUBUF-NEXT:    s_mov_b32 s33, s4
+; MUBUF-NEXT:    s_waitcnt vmcnt(0)
+; MUBUF-NEXT:    s_setpc_b64 s[30:31]
+;
+; FLATSCR-LABEL: last_lane_vgpr_for_fp_csr:
+; FLATSCR:       ; %bb.0:
+; FLATSCR-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; FLATSCR-NEXT:    s_mov_b32 s0, s33
+; FLATSCR-NEXT:    s_mov_b32 s33, s32
+; FLATSCR-NEXT:    s_xor_saveexec_b64 s[2:3], -1
+; FLATSCR-NEXT:    scratch_store_dword off, v1, s33 offset:8 ; 4-byte Folded Spill
+; FLATSCR-NEXT:    s_mov_b64 exec, s[2:3]
+; FLATSCR-NEXT:    v_writelane_b32 v1, s40, 0
+; FLATSCR-NEXT:    v_writelane_b32 v1, s41, 1
+; FLATSCR-NEXT:    v_writelane_b32 v1, s42, 2
+; FLATSCR-NEXT:    v_writelane_b32 v1, s43, 3
+; FLATSCR-NEXT:    v_writelane_b32 v1, s44, 4
+; FLATSCR-NEXT:    v_writelane_b32 v1, s45, 5
+; FLATSCR-NEXT:    v_writelane_b32 v1, s46, 6
+; FLATSCR-NEXT:    v_writelane_b32 v1, s47, 7
+; FLATSCR-NEXT:    v_writelane_b32 v1, s48, 8
+; FLATSCR-NEXT:    v_writelane_b32 v1, s49, 9
+; FLATSCR-NEXT:    v_writelane_b32 v1, s50, 10
+; FLATSCR-NEXT:    v_writelane_b32 v1, s51, 11
+; FLATSCR-NEXT:    v_writelane_b32 v1, s52, 12
+; FLATSCR-NEXT:    v_writelane_b32 v1, s53, 13
+; FLATSCR-NEXT:    v_writelane_b32 v1, s54, 14
+; FLATSCR-NEXT:    v_writelane_b32 v1, s55, 15
+; FLATSCR-NEXT:    v_writelane_b32 v1, s56, 16
+; FLATSCR-NEXT:    v_writelane_b32 v1, s57, 17
+; FLATSCR-NEXT:    v_writelane_b32 v1, s58, 18
+; FLATSCR-NEXT:    v_writelane_b32 v1, s59, 19
+; FLATSCR-NEXT:    v_writelane_b32 v1, s60, 20
+; FLATSCR-NEXT:    v_writelane_b32 v1, s61, 21
+; FLATSCR-NEXT:    v_writelane_b32 v1, s62, 22
+; FLATSCR-NEXT:    v_writelane_b32 v1, s63, 23
+; FLATSCR-NEXT:    v_writelane_b32 v1, s64, 24
+; FLATSCR-NEXT:    v_writelane_b32 v1, s65, 25
+; FLATSCR-NEXT:    v_writelane_b32 v1, s66, 26
+; FLATSCR-NEXT:    v_writelane_b32 v1, s67, 27
+; FLATSCR-NEXT:    v_writelane_b32 v1, s68, 28
+; FLATSCR-NEXT:    v_writelane_b32 v1, s69, 29
+; FLATSCR-NEXT:    v_writelane_b32 v1, s70, 30
+; FLATSCR-NEXT:    v_writelane_b32 v1, s71, 31
+; FLATSCR-NEXT:    v_writelane_b32 v1, s72, 32
+; FLATSCR-NEXT:    v_writelane_b32 v1, s73, 33
+; FLATSCR-NEXT:    v_writelane_b32 v1, s74, 34
+; FLATSCR-NEXT:    v_writelane_b32 v1, s75, 35
+; FLATSCR-NEXT:    v_writelane_b32 v1, s76, 36
+; FLATSCR-NEXT:    v_writelane_b32 v1, s77, 37
+; FLATSCR-NEXT:    v_writelane_b32 v1, s78, 38
+; FLATSCR-NEXT:    v_writelane_b32 v1, s79, 39
+; FLATSCR-NEXT:    v_writelane_b32 v1, s80, 40
+; FLATSCR-NEXT:    v_writelane_b32 v1, s81, 41
+; FLATSCR-NEXT:    v_writelane_b32 v1, s82, 42
+; FLATSCR-NEXT:    v_writelane_b32 v1, s83, 43
+; FLATSCR-NEXT:    v_writelane_b32 v1, s84, 44
+; FLATSCR-NEXT:    v_writelane_b32 v1, s85, 45
+; FLATSCR-NEXT:    v_writelane_b32 v1, s86, 46
+; FLATSCR-NEXT:    v_writelane_b32 v1, s87, 47
+; FLATSCR-NEXT:    v_writelane_b32 v1, s88, 48
+; FLATSCR-NEXT:    v_writelane_b32 v1, s89, 49
+; FLATSCR-NEXT:    v_writelane_b32 v1, s90, 50
+; FLATSCR-NEXT:    v_writelane_b32 v1, s91, 51
+; FLATSCR-NEXT:    v_writelane_b32 v1, s92, 52
+; FLATSCR-NEXT:    v_writelane_b32 v1, s93, 53
+; FLATSCR-NEXT:    v_writelane_b32 v1, s94, 54
+; FLATSCR-NEXT:    v_writelane_b32 v1, s95, 55
+; FLATSCR-NEXT:    v_writelane_b32 v1, s96, 56
+; FLATSCR-NEXT:    v_writelane_b32 v1, s97, 57
+; FLATSCR-NEXT:    v_writelane_b32 v1, s98, 58
+; FLATSCR-NEXT:    v_writelane_b32 v1, s99, 59
+; FLATSCR-NEXT:    v_writelane_b32 v1, s100, 60
+; FLATSCR-NEXT:    v_mov_b32_e32 v0, 0
+; FLATSCR-NEXT:    scratch_store_dword off, v41, s33 ; 4-byte Folded Spill
+; FLATSCR-NEXT:    v_writelane_b32 v1, s101, 61
+; FLATSCR-NEXT:    scratch_store_dword off, v0, s33 offset:4
+; FLATSCR-NEXT:    s_waitcnt vmcnt(0)
+; FLATSCR-NEXT:    ;;#ASMSTART
+; FLATSCR-NEXT:    ; clobber v41
+; FLATSCR-NEXT:    ;;#ASMEND
+; FLATSCR-NEXT:    v_writelane_b32 v1, s102, 62
+; FLATSCR-NEXT:    ;;#ASMSTART
+; FLATSCR-NEXT:    ;;#ASMEND
+; FLATSCR-NEXT:    scratch_load_dword v41, off, s33 ; 4-byte Folded Reload
+; FLATSCR-NEXT:    s_add_i32 s32, s32, 16
+; FLATSCR-NEXT:    v_readlane_b32 s102, v1, 62
+; FLATSCR-NEXT:    v_readlane_b32 s101, v1, 61
+; FLATSCR-NEXT:    v_readlane_b32 s100, v1, 60
+; FLATSCR-NEXT:    v_readlane_b32 s99, v1, 59
+; FLATSCR-NEXT:    v_readlane_b32 s98, v1, 58
+; FLATSCR-NEXT:    v_readlane_b32 s97, v1, 57
+; FLATSCR-NEXT:    v_readlane_b32 s96, v1, 56
+; FLATSCR-NEXT:    v_readlane_b32 s95, v1, 55
+; FLATSCR-NEXT:    v_readlane_b32 s94, v1, 54
+; FLATSCR-NEXT:    v_readlane_b32 s93, v1, 53
+; FLATSCR-NEXT:    v_readlane_b32 s92, v1, 52
+; FLATSCR-NEXT:    v_readlane_b32 s91, v1, 51
+; FLATSCR-NEXT:    v_readlane_b32 s90, v1, 50
+; FLATSCR-NEXT:    v_readlane_b32 s89, v1, 49
+; FLATSCR-NEXT:    v_readlane_b32 s88, v1, 48
+; FLATSCR-NEXT:    v_readlane_b32 s87, v1, 47
+; FLATSCR-NEXT:    v_readlane_b32 s86, v1, 46
+; FLATSCR-NEXT:    v_readlane_b32 s85, v1, 45
+; FLATSCR-NEXT:    v_readlane_b32 s84, v1, 44
+; FLATSCR-NEXT:    v_readlane_b32 s83, v1, 43
+; FLATSCR-NEXT:    v_readlane_b32 s82, v1, 42
+; FLATSCR-NEXT:    v_readlane_b32 s81, v1, 41
+; FLATSCR-NEXT:    v_readlane_b32 s80, v1, 40
+; FLATSCR-NEXT:    v_readlane_b32 s79, v1, 39
+; FLATSCR-NEXT:    v_readlane_b32 s78, v1, 38
+; FLATSCR-NEXT:    v_readlane_b32 s77, v1, 37
+; FLATSCR-NEXT:    v_readlane_b32 s76, v1, 36
+; FLATSCR-NEXT:    v_readlane_b32 s75, v1, 35
+; FLATSCR-NEXT:    v_readlane_b32 s74, v1, 34
+; FLATSCR-NEXT:    v_readlane_b32 s73, v1, 33
+; FLATSCR-NEXT:    v_readlane_b32 s72, v1, 32
+; FLATSCR-NEXT:    v_readlane_b32 s71, v1, 31
+; FLATSCR-NEXT:    v_readlane_b32 s70, v1, 30
+; FLATSCR-NEXT:    v_readlane_b32 s69, v1, 29
+; FLATSCR-NEXT:    v_readlane_b32 s68, v1, 28
+; FLATSCR-NEXT:    v_readlane_b32 s67, v1, 27
+; FLATSCR-NEXT:    v_readlane_b32 s66, v1, 26
+; FLATSCR-NEXT:    v_readlane_b32 s65, v1, 25
+; FLATSCR-NEXT:    v_readlane_b32 s64, v1, 24
+; FLATSCR-NEXT:    v_readlane_b32 s63, v1, 23
+; FLATSCR-NEXT:    v_readlane_b32 s62, v1, 22
+; FLATSCR-NEXT:    v_readlane_b32 s61, v1, 21
+; FLATSCR-NEXT:    v_readlane_b32 s60, v1, 20
+; FLATSCR-NEXT:    v_readlane_b32 s59, v1, 19
+; FLATSCR-NEXT:    v_readlane_b32 s58, v1, 18
+; FLATSCR-NEXT:    v_readlane_b32 s57, v1, 17
+; FLATSCR-NEXT:    v_readlane_b32 s56, v1, 16
+; FLATSCR-NEXT:    v_readlane_b32 s55, v1, 15
+; FLATSCR-NEXT:    v_readlane_b32 s54, v1, 14
+; FLATSCR-NEXT:    v_readlane_b32 s53, v1, 13
+; FLATSCR-NEXT:    v_readlane_b32 s52, v1, 12
+; FLATSCR-NEXT:    v_readlane_b32 s51, v1, 11
+; FLATSCR-NEXT:    v_readlane_b32 s50, v1, 10
+; FLATSCR-NEXT:    v_readlane_b32 s49, v1, 9
+; FLATSCR-NEXT:    v_readlane_b32 s48, v1, 8
+; FLATSCR-NEXT:    v_readlane_b32 s47, v1, 7
+; FLATSCR-NEXT:    v_readlane_b32 s46, v1, 6
+; FLATSCR-NEXT:    v_readlane_b32 s45, v1, 5
+; FLATSCR-NEXT:    v_readlane_b32 s44, v1, 4
+; FLATSCR-NEXT:    v_readlane_b32 s43, v1, 3
+; FLATSCR-NEXT:    v_readlane_b32 s42, v1, 2
+; FLATSCR-NEXT:    v_readlane_b32 s41, v1, 1
+; FLATSCR-NEXT:    v_readlane_b32 s40, v1, 0
+; FLATSCR-NEXT:    s_mov_b32 s32, s33
+; FLATSCR-NEXT:    s_xor_saveexec_b64 s[2:3], -1
+; FLATSCR-NEXT:    scratch_load_dword v1, off, s33 offset:8 ; 4-byte Folded Reload
+; FLATSCR-NEXT:    s_mov_b64 exec, s[2:3]
+; FLATSCR-NEXT:    s_mov_b32 s33, s0
+; FLATSCR-NEXT:    s_waitcnt vmcnt(0)
+; FLATSCR-NEXT:    s_setpc_b64 s[30:31]
   %alloca = alloca i32, addrspace(5)
   store volatile i32 0, ptr addrspace(5) %alloca
   call void asm sideeffect "; clobber v41", "~{v41}"()
@@ -310,37 +976,316 @@ define void @last_lane_vgpr_for_fp_csr() #1 {
 }
 
 ; Use a copy to a free SGPR instead of introducing a second CSR VGPR.
-; GCN-LABEL: {{^}}no_new_vgpr_for_fp_csr:
-; GCN: s_waitcnt
-; GCN-NEXT: s_mov_b32 [[FP_COPY:s[0-9]+]], s33
-; GCN-NEXT: s_mov_b32 s33, s32
-; GCN-NEXT: s_xor_saveexec_b64 [[COPY_EXEC0:s\[[0-9]+:[0-9]+\]]], -1{{$}}
-; MUBUF-NEXT:   buffer_store_dword [[CSR_VGPR:v[0-9]+]], off, s[0:3], s33 offset:8 ; 4-byte Folded Spill
-; FLATSCR-NEXT: scratch_store_dword off, [[CSR_VGPR:v[0-9]+]], s33 offset:8 ; 4-byte Folded Spill
-; GCN-NEXT: s_mov_b64 exec, [[COPY_EXEC0]]
-; GCN-COUNT-61: v_writelane_b32 v1,
-; MUBUF:   buffer_store_dword v41, off, s[0:3], s33 ; 4-byte Folded Spill
-; FLATSCR: scratch_store_dword off, v41, s33 ; 4-byte Folded Spill
-; GCN: v_writelane_b32 v1,
-; MUBUF:   buffer_store_dword
-; FLATSCR: scratch_store_dword
-; GCN: ;;#ASMSTART
-; GCN: v_writelane_b32 v1,
-; MUBUF:   buffer_load_dword v41, off, s[0:3], s33 ; 4-byte Folded Reload
-; FLATSCR: scratch_load_dword v41, off, s33 ; 4-byte Folded Reload
-; MUBUF:        s_addk_i32 s32, 0x400
-; FLATSCR:      s_add_i32 s32, s32, 16
-; GCN-COUNT-64: v_readlane_b32 s{{[0-9]+}}, v1
-; MUBUF-NEXT:   s_mov_b32 s32, s33
-; FLATSCR-NEXT: s_mov_b32 s32, s33
-; GCN-NEXT: s_xor_saveexec_b64 [[COPY_EXEC1:s\[[0-9]+:[0-9]+\]]], -1{{$}}
-; MUBUF-NEXT:   buffer_load_dword [[CSR_VGPR]], off, s[0:3], s33 offset:8 ; 4-byte Folded Reload
-; FLATSCR-NEXT: scratch_load_dword [[CSR_VGPR]], off, s33 offset:8 ; 4-byte Folded Reload
-; GCN-NEXT: s_mov_b64 exec, [[COPY_EXEC1]]
-; GCN-NEXT: s_mov_b32 s33, [[FP_COPY]]
-; GCN-NEXT: s_waitcnt vmcnt(0)
-; GCN-NEXT: s_setpc_b64
 define void @no_new_vgpr_for_fp_csr() #1 {
+; MUBUF-LABEL: no_new_vgpr_for_fp_csr:
+; MUBUF:       ; %bb.0:
+; MUBUF-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; MUBUF-NEXT:    s_mov_b32 s4, s33
+; MUBUF-NEXT:    s_mov_b32 s33, s32
+; MUBUF-NEXT:    s_xor_saveexec_b64 s[6:7], -1
+; MUBUF-NEXT:    buffer_store_dword v1, off, s[0:3], s33 offset:8 ; 4-byte Folded Spill
+; MUBUF-NEXT:    s_mov_b64 exec, s[6:7]
+; MUBUF-NEXT:    v_writelane_b32 v1, s39, 0
+; MUBUF-NEXT:    v_writelane_b32 v1, s40, 1
+; MUBUF-NEXT:    v_writelane_b32 v1, s41, 2
+; MUBUF-NEXT:    v_writelane_b32 v1, s42, 3
+; MUBUF-NEXT:    v_writelane_b32 v1, s43, 4
+; MUBUF-NEXT:    v_writelane_b32 v1, s44, 5
+; MUBUF-NEXT:    v_writelane_b32 v1, s45, 6
+; MUBUF-NEXT:    v_writelane_b32 v1, s46, 7
+; MUBUF-NEXT:    v_writelane_b32 v1, s47, 8
+; MUBUF-NEXT:    v_writelane_b32 v1, s48, 9
+; MUBUF-NEXT:    v_writelane_b32 v1, s49, 10
+; MUBUF-NEXT:    v_writelane_b32 v1, s50, 11
+; MUBUF-NEXT:    v_writelane_b32 v1, s51, 12
+; MUBUF-NEXT:    v_writelane_b32 v1, s52, 13
+; MUBUF-NEXT:    v_writelane_b32 v1, s53, 14
+; MUBUF-NEXT:    v_writelane_b32 v1, s54, 15
+; MUBUF-NEXT:    v_writelane_b32 v1, s55, 16
+; MUBUF-NEXT:    v_writelane_b32 v1, s56, 17
+; MUBUF-NEXT:    v_writelane_b32 v1, s57, 18
+; MUBUF-NEXT:    v_writelane_b32 v1, s58, 19
+; MUBUF-NEXT:    v_writelane_b32 v1, s59, 20
+; MUBUF-NEXT:    v_writelane_b32 v1, s60, 21
+; MUBUF-NEXT:    v_writelane_b32 v1, s61, 22
+; MUBUF-NEXT:    v_writelane_b32 v1, s62, 23
+; MUBUF-NEXT:    v_writelane_b32 v1, s63, 24
+; MUBUF-NEXT:    v_writelane_b32 v1, s64, 25
+; MUBUF-NEXT:    v_writelane_b32 v1, s65, 26
+; MUBUF-NEXT:    v_writelane_b32 v1, s66, 27
+; MUBUF-NEXT:    v_writelane_b32 v1, s67, 28
+; MUBUF-NEXT:    v_writelane_b32 v1, s68, 29
+; MUBUF-NEXT:    v_writelane_b32 v1, s69, 30
+; MUBUF-NEXT:    v_writelane_b32 v1, s70, 31
+; MUBUF-NEXT:    v_writelane_b32 v1, s71, 32
+; MUBUF-NEXT:    v_writelane_b32 v1, s72, 33
+; MUBUF-NEXT:    v_writelane_b32 v1, s73, 34
+; MUBUF-NEXT:    v_writelane_b32 v1, s74, 35
+; MUBUF-NEXT:    v_writelane_b32 v1, s75, 36
+; MUBUF-NEXT:    v_writelane_b32 v1, s76, 37
+; MUBUF-NEXT:    v_writelane_b32 v1, s77, 38
+; MUBUF-NEXT:    v_writelane_b32 v1, s78, 39
+; MUBUF-NEXT:    v_writelane_b32 v1, s79, 40
+; MUBUF-NEXT:    v_writelane_b32 v1, s80, 41
+; MUBUF-NEXT:    v_writelane_b32 v1, s81, 42
+; MUBUF-NEXT:    v_writelane_b32 v1, s82, 43
+; MUBUF-NEXT:    v_writelane_b32 v1, s83, 44
+; MUBUF-NEXT:    v_writelane_b32 v1, s84, 45
+; MUBUF-NEXT:    v_writelane_b32 v1, s85, 46
+; MUBUF-NEXT:    v_writelane_b32 v1, s86, 47
+; MUBUF-NEXT:    v_writelane_b32 v1, s87, 48
+; MUBUF-NEXT:    v_writelane_b32 v1, s88, 49
+; MUBUF-NEXT:    v_writelane_b32 v1, s89, 50
+; MUBUF-NEXT:    v_writelane_b32 v1, s90, 51
+; MUBUF-NEXT:    v_writelane_b32 v1, s91, 52
+; MUBUF-NEXT:    v_writelane_b32 v1, s92, 53
+; MUBUF-NEXT:    v_writelane_b32 v1, s93, 54
+; MUBUF-NEXT:    v_writelane_b32 v1, s94, 55
+; MUBUF-NEXT:    v_writelane_b32 v1, s95, 56
+; MUBUF-NEXT:    v_writelane_b32 v1, s96, 57
+; MUBUF-NEXT:    v_writelane_b32 v1, s97, 58
+; MUBUF-NEXT:    v_writelane_b32 v1, s98, 59
+; MUBUF-NEXT:    v_writelane_b32 v1, s99, 60
+; MUBUF-NEXT:    v_writelane_b32 v1, s100, 61
+; MUBUF-NEXT:    v_mov_b32_e32 v0, 0
+; MUBUF-NEXT:    buffer_store_dword v41, off, s[0:3], s33 ; 4-byte Folded Spill
+; MUBUF-NEXT:    v_writelane_b32 v1, s101, 62
+; MUBUF-NEXT:    buffer_store_dword v0, off, s[0:3], s33 offset:4
+; MUBUF-NEXT:    s_waitcnt vmcnt(0)
+; MUBUF-NEXT:    ;;#ASMSTART
+; MUBUF-NEXT:    ; clobber v41
+; MUBUF-NEXT:    ;;#ASMEND
+; MUBUF-NEXT:    v_writelane_b32 v1, s102, 63
+; MUBUF-NEXT:    ;;#ASMSTART
+; MUBUF-NEXT:    ;;#ASMEND
+; MUBUF-NEXT:    buffer_load_dword v41, off, s[0:3], s33 ; 4-byte Folded Reload
+; MUBUF-NEXT:    s_addk_i32 s32, 0x400
+; MUBUF-NEXT:    v_readlane_b32 s102, v1, 63
+; MUBUF-NEXT:    v_readlane_b32 s101, v1, 62
+; MUBUF-NEXT:    v_readlane_b32 s100, v1, 61
+; MUBUF-NEXT:    v_readlane_b32 s99, v1, 60
+; MUBUF-NEXT:    v_readlane_b32 s98, v1, 59
+; MUBUF-NEXT:    v_readlane_b32 s97, v1, 58
+; MUBUF-NEXT:    v_readlane_b32 s96, v1, 57
+; MUBUF-NEXT:    v_readlane_b32 s95, v1, 56
+; MUBUF-NEXT:    v_readlane_b32 s94, v1, 55
+; MUBUF-NEXT:    v_readlane_b32 s93, v1, 54
+; MUBUF-NEXT:    v_readlane_b32 s92, v1, 53
+; MUBUF-NEXT:    v_readlane_b32 s91, v1, 52
+; MUBUF-NEXT:    v_readlane_b32 s90, v1, 51
+; MUBUF-NEXT:    v_readlane_b32 s89, v1, 50
+; MUBUF-NEXT:    v_readlane_b32 s88, v1, 49
+; MUBUF-NEXT:    v_readlane_b32 s87, v1, 48
+; MUBUF-NEXT:    v_readlane_b32 s86, v1, 47
+; MUBUF-NEXT:    v_readlane_b32 s85, v1, 46
+; MUBUF-NEXT:    v_readlane_b32 s84, v1, 45
+; MUBUF-NEXT:    v_readlane_b32 s83, v1, 44
+; MUBUF-NEXT:    v_readlane_b32 s82, v1, 43
+; MUBUF-NEXT:    v_readlane_b32 s81, v1, 42
+; MUBUF-NEXT:    v_readlane_b32 s80, v1, 41
+; MUBUF-NEXT:    v_readlane_b32 s79, v1, 40
+; MUBUF-NEXT:    v_readlane_b32 s78, v1, 39
+; MUBUF-NEXT:    v_readlane_b32 s77, v1, 38
+; MUBUF-NEXT:    v_readlane_b32 s76, v1, 37
+; MUBUF-NEXT:    v_readlane_b32 s75, v1, 36
+; MUBUF-NEXT:    v_readlane_b32 s74, v1, 35
+; MUBUF-NEXT:    v_readlane_b32 s73, v1, 34
+; MUBUF-NEXT:    v_readlane_b32 s72, v1, 33
+; MUBUF-NEXT:    v_readlane_b32 s71, v1, 32
+; MUBUF-NEXT:    v_readlane_b32 s70, v1, 31
+; MUBUF-NEXT:    v_readlane_b32 s69, v1, 30
+; MUBUF-NEXT:    v_readlane_b32 s68, v1, 29
+; MUBUF-NEXT:    v_readlane_b32 s67, v1, 28
+; MUBUF-NEXT:    v_readlane_b32 s66, v1, 27
+; MUBUF-NEXT:    v_readlane_b32 s65, v1, 26
+; MUBUF-NEXT:    v_readlane_b32 s64, v1, 25
+; MUBUF-NEXT:    v_readlane_b32 s63, v1, 24
+; MUBUF-NEXT:    v_readlane_b32 s62, v1, 23
+; MUBUF-NEXT:    v_readlane_b32 s61, v1, 22
+; MUBUF-NEXT:    v_readlane_b32 s60, v1, 21
+; MUBUF-NEXT:    v_readlane_b32 s59, v1, 20
+; MUBUF-NEXT:    v_readlane_b32 s58, v1, 19
+; MUBUF-NEXT:    v_readlane_b32 s57, v1, 18
+; MUBUF-NEXT:    v_readlane_b32 s56, v1, 17
+; MUBUF-NEXT:    v_readlane_b32 s55, v1, 16
+; MUBUF-NEXT:    v_readlane_b32 s54, v1, 15
+; MUBUF-NEXT:    v_readlane_b32 s53, v1, 14
+; MUBUF-NEXT:    v_readlane_b32 s52, v1, 13
+; MUBUF-NEXT:    v_readlane_b32 s51, v1, 12
+; MUBUF-NEXT:    v_readlane_b32 s50, v1, 11
+; MUBUF-NEXT:    v_readlane_b32 s49, v1, 10
+; MUBUF-NEXT:    v_readlane_b32 s48, v1, 9
+; MUBUF-NEXT:    v_readlane_b32 s47, v1, 8
+; MUBUF-NEXT:    v_readlane_b32 s46, v1, 7
+; MUBUF-NEXT:    v_readlane_b32 s45, v1, 6
+; MUBUF-NEXT:    v_readlane_b32 s44, v1, 5
+; MUBUF-NEXT:    v_readlane_b32 s43, v1, 4
+; MUBUF-NEXT:    v_readlane_b32 s42, v1, 3
+; MUBUF-NEXT:    v_readlane_b32 s41, v1, 2
+; MUBUF-NEXT:    v_readlane_b32 s40, v1, 1
+; MUBUF-NEXT:    v_readlane_b32 s39, v1, 0
+; MUBUF-NEXT:    s_mov_b32 s32, s33
+; MUBUF-NEXT:    s_xor_saveexec_b64 s[6:7], -1
+; MUBUF-NEXT:    buffer_load_dword v1, off, s[0:3], s33 offset:8 ; 4-byte Folded Reload
+; MUBUF-NEXT:    s_mov_b64 exec, s[6:7]
+; MUBUF-NEXT:    s_mov_b32 s33, s4
+; MUBUF-NEXT:    s_waitcnt vmcnt(0)
+; MUBUF-NEXT:    s_setpc_b64 s[30:31]
+;
+; FLATSCR-LABEL: no_new_vgpr_for_fp_csr:
+; FLATSCR:       ; %bb.0:
+; FLATSCR-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; FLATSCR-NEXT:    s_mov_b32 s0, s33
+; FLATSCR-NEXT:    s_mov_b32 s33, s32
+; FLATSCR-NEXT:    s_xor_saveexec_b64 s[2:3], -1
+; FLATSCR-NEXT:    scratch_store_dword off, v1, s33 offset:8 ; 4-byte Folded Spill
+; FLATSCR-NEXT:    s_mov_b64 exec, s[2:3]
+; FLATSCR-NEXT:    v_writelane_b32 v1, s39, 0
+; FLATSCR-NEXT:    v_writelane_b32 v1, s40, 1
+; FLATSCR-NEXT:    v_writelane_b32 v1, s41, 2
+; FLATSCR-NEXT:    v_writelane_b32 v1, s42, 3
+; FLATSCR-NEXT:    v_writelane_b32 v1, s43, 4
+; FLATSCR-NEXT:    v_writelane_b32 v1, s44, 5
+; FLATSCR-NEXT:    v_writelane_b32 v1, s45, 6
+; FLATSCR-NEXT:    v_writelane_b32 v1, s46, 7
+; FLATSCR-NEXT:    v_writelane_b32 v1, s47, 8
+; FLATSCR-NEXT:    v_writelane_b32 v1, s48, 9
+; FLATSCR-NEXT:    v_writelane_b32 v1, s49, 10
+; FLATSCR-NEXT:    v_writelane_b32 v1, s50, 11
+; FLATSCR-NEXT:    v_writelane_b32 v1, s51, 12
+; FLATSCR-NEXT:    v_writelane_b32 v1, s52, 13
+; FLATSCR-NEXT:    v_writelane_b32 v1, s53, 14
+; FLATSCR-NEXT:    v_writelane_b32 v1, s54, 15
+; FLATSCR-NEXT:    v_writelane_b32 v1, s55, 16
+; FLATSCR-NEXT:    v_writelane_b32 v1, s56, 17
+; FLATSCR-NEXT:    v_writelane_b32 v1, s57, 18
+; FLATSCR-NEXT:    v_writelane_b32 v1, s58, 19
+; FLATSCR-NEXT:    v_writelane_b32 v1, s59, 20
+; FLATSCR-NEXT:    v_writelane_b32 v1, s60, 21
+; FLATSCR-NEXT:    v_writelane_b32 v1, s61, 22
+; FLATSCR-NEXT:    v_writelane_b32 v1, s62, 23
+; FLATSCR-NEXT:    v_writelane_b32 v1, s63, 24
+; FLATSCR-NEXT:    v_writelane_b32 v1, s64, 25
+; FLATSCR-NEXT:    v_writelane_b32 v1, s65, 26
+; FLATSCR-NEXT:    v_writelane_b32 v1, s66, 27
+; FLATSCR-NEXT:    v_writelane_b32 v1, s67, 28
+; FLATSCR-NEXT:    v_writelane_b32 v1, s68, 29
+; FLATSCR-NEXT:    v_writelane_b32 v1, s69, 30
+; FLATSCR-NEXT:    v_writelane_b32 v1, s70, 31
+; FLATSCR-NEXT:    v_writelane_b32 v1, s71, 32
+; FLATSCR-NEXT:    v_writelane_b32 v1, s72, 33
+; FLATSCR-NEXT:    v_writelane_b32 v1, s73, 34
+; FLATSCR-NEXT:    v_writelane_b32 v1, s74, 35
+; FLATSCR-NEXT:    v_writelane_b32 v1, s75, 36
+; FLATSCR-NEXT:    v_writelane_b32 v1, s76, 37
+; FLATSCR-NEXT:    v_writelane_b32 v1, s77, 38
+; FLATSCR-NEXT:    v_writelane_b32 v1, s78, 39
+; FLATSCR-NEXT:    v_writelane_b32 v1, s79, 40
+; FLATSCR-NEXT:    v_writelane_b32 v1, s80, 41
+; FLATSCR-NEXT:    v_writelane_b32 v1, s81, 42
+; FLATSCR-NEXT:    v_writelane_b32 v1, s82, 43
+; FLATSCR-NEXT:    v_writelane_b32 v1, s83, 44
+; FLATSCR-NEXT:    v_writelane_b32 v1, s84, 45
+; FLATSCR-NEXT:    v_writelane_b32 v1, s85, 46
+; FLATSCR-NEXT:    v_writelane_b32 v1, s86, 47
+; FLATSCR-NEXT:    v_writelane_b32 v1, s87, 48
+; FLATSCR-NEXT:    v_writelane_b32 v1, s88, 49
+; FLATSCR-NEXT:    v_writelane_b32 v1, s89, 50
+; FLATSCR-NEXT:    v_writelane_b32 v1, s90, 51
+; FLATSCR-NEXT:    v_writelane_b32 v1, s91, 52
+; FLATSCR-NEXT:    v_writelane_b32 v1, s92, 53
+; FLATSCR-NEXT:    v_writelane_b32 v1, s93, 54
+; FLATSCR-NEXT:    v_writelane_b32 v1, s94, 55
+; FLATSCR-NEXT:    v_writelane_b32 v1, s95, 56
+; FLATSCR-NEXT:    v_writelane_b32 v1, s96, 57
+; FLATSCR-NEXT:    v_writelane_b32 v1, s97, 58
+; FLATSCR-NEXT:    v_writelane_b32 v1, s98, 59
+; FLATSCR-NEXT:    v_writelane_b32 v1, s99, 60
+; FLATSCR-NEXT:    v_writelane_b32 v1, s100, 61
+; FLATSCR-NEXT:    v_mov_b32_e32 v0, 0
+; FLATSCR-NEXT:    scratch_store_dword off, v41, s33 ; 4-byte Folded Spill
+; FLATSCR-NEXT:    v_writelane_b32 v1, s101, 62
+; FLATSCR-NEXT:    scratch_store_dword off, v0, s33 offset:4
+; FLATSCR-NEXT:    s_waitcnt vmcnt(0)
+; FLATSCR-NEXT:    ;;#ASMSTART
+; FLATSCR-NEXT:    ; clobber v41
+; FLATSCR-NEXT:    ;;#ASMEND
+; FLATSCR-NEXT:    v_writelane_b32 v1, s102, 63
+; FLATSCR-NEXT:    ;;#ASMSTART
+; FLATSCR-NEXT:    ;;#ASMEND
+; FLATSCR-NEXT:    scratch_load_dword v41, off, s33 ; 4-byte Folded Reload
+; FLATSCR-NEXT:    s_add_i32 s32, s32, 16
+; FLATSCR-NEXT:    v_readlane_b32 s102, v1, 63
+; FLATSCR-NEXT:    v_readlane_b32 s101, v1, 62
+; FLATSCR-NEXT:    v_readlane_b32 s100, v1, 61
+; FLATSCR-NEXT:    v_readlane_b32 s99, v1, 60
+; FLATSCR-NEXT:    v_readlane_b32 s98, v1, 59
+; FLATSCR-NEXT:    v_readlane_b32 s97, v1, 58
+; FLATSCR-NEXT:    v_readlane_b32 s96, v1, 57
+; FLATSCR-NEXT:    v_readlane_b32 s95, v1, 56
+; FLATSCR-NEXT:    v_readlane_b32 s94, v1, 55
+; FLATSCR-NEXT:    v_readlane_b32 s93, v1, 54
+; FLATSCR-NEXT:    v_readlane_b32 s92, v1, 53
+; FLATSCR-NEXT:    v_readlane_b32 s91, v1, 52
+; FLATSCR-NEXT:    v_readlane_b32 s90, v1, 51
+; FLATSCR-NEXT:    v_readlane_b32 s89, v1, 50
+; FLATSCR-NEXT:    v_readlane_b32 s88, v1, 49
+; FLATSCR-NEXT:    v_readlane_b32 s87, v1, 48
+; FLATSCR-NEXT:    v_readlane_b32 s86, v1, 47
+; FLATSCR-NEXT:    v_readlane_b32 s85, v1, 46
+; FLATSCR-NEXT:    v_readlane_b32 s84, v1, 45
+; FLATSCR-NEXT:    v_readlane_b32 s83, v1, 44
+; FLATSCR-NEXT:    v_readlane_b32 s82, v1, 43
+; FLATSCR-NEXT:    v_readlane_b32 s81, v1, 42
+; FLATSCR-NEXT:    v_readlane_b32 s80, v1, 41
+; FLATSCR-NEXT:    v_readlane_b32 s79, v1, 40
+; FLATSCR-NEXT:    v_readlane_b32 s78, v1, 39
+; FLATSCR-NEXT:    v_readlane_b32 s77, v1, 38
+; FLATSCR-NEXT:    v_readlane_b32 s76, v1, 37
+; FLATSCR-NEXT:    v_readlane_b32 s75, v1, 36
+; FLATSCR-NEXT:    v_readlane_b32 s74, v1, 35
+; FLATSCR-NEXT:    v_readlane_b32 s73, v1, 34
+; FLATSCR-NEXT:    v_readlane_b32 s72, v1, 33
+; FLATSCR-NEXT:    v_readlane_b32 s71, v1, 32
+; FLATSCR-NEXT:    v_readlane_b32 s70, v1, 31
+; FLATSCR-NEXT:    v_readlane_b32 s69, v1, 30
+; FLATSCR-NEXT:    v_readlane_b32 s68, v1, 29
+; FLATSCR-NEXT:    v_readlane_b32 s67, v1, 28
+; FLATSCR-NEXT:    v_readlane_b32 s66, v1, 27
+; FLATSCR-NEXT:    v_readlane_b32 s65, v1, 26
+; FLATSCR-NEXT:    v_readlane_b32 s64, v1, 25
+; FLATSCR-NEXT:    v_readlane_b32 s63, v1, 24
+; FLATSCR-NEXT:    v_readlane_b32 s62, v1, 23
+; FLATSCR-NEXT:    v_readlane_b32 s61, v1, 22
+; FLATSCR-NEXT:    v_readlane_b32 s60, v1, 21
+; FLATSCR-NEXT:    v_readlane_b32 s59, v1, 20
+; FLATSCR-NEXT:    v_readlane_b32 s58, v1, 19
+; FLATSCR-NEXT:    v_readlane_b32 s57, v1, 18
+; FLATSCR-NEXT:    v_readlane_b32 s56, v1, 17
+; FLATSCR-NEXT:    v_readlane_b32 s55, v1, 16
+; FLATSCR-NEXT:    v_readlane_b32 s54, v1, 15
+; FLATSCR-NEXT:    v_readlane_b32 s53, v1, 14
+; FLATSCR-NEXT:    v_readlane_b32 s52, v1, 13
+; FLATSCR-NEXT:    v_readlane_b32 s51, v1, 12
+; FLATSCR-NEXT:    v_readlane_b32 s50, v1, 11
+; FLATSCR-NEXT:    v_readlane_b32 s49, v1, 10
+; FLATSCR-NEXT:    v_readlane_b32 s48, v1, 9
+; FLATSCR-NEXT:    v_readlane_b32 s47, v1, 8
+; FLATSCR-NEXT:    v_readlane_b32 s46, v1, 7
+; FLATSCR-NEXT:    v_readlane_b32 s45, v1, 6
+; FLATSCR-NEXT:    v_readlane_b32 s44, v1, 5
+; FLATSCR-NEXT:    v_readlane_b32 s43, v1, 4
+; FLATSCR-NEXT:    v_readlane_b32 s42, v1, 3
+; FLATSCR-NEXT:    v_readlane_b32 s41, v1, 2
+; FLATSCR-NEXT:    v_readlane_b32 s40, v1, 1
+; FLATSCR-NEXT:    v_readlane_b32 s39, v1, 0
+; FLATSCR-NEXT:    s_mov_b32 s32, s33
+; FLATSCR-NEXT:    s_xor_saveexec_b64 s[2:3], -1
+; FLATSCR-NEXT:    scratch_load_dword v1, off, s33 offset:8 ; 4-byte Folded Reload
+; FLATSCR-NEXT:    s_mov_b64 exec, s[2:3]
+; FLATSCR-NEXT:    s_mov_b32 s33, s0
+; FLATSCR-NEXT:    s_waitcnt vmcnt(0)
+; FLATSCR-NEXT:    s_setpc_b64 s[30:31]
   %alloca = alloca i32, addrspace(5)
   store volatile i32 0, ptr addrspace(5) %alloca
   call void asm sideeffect "; clobber v41", "~{v41}"()
@@ -356,64 +1301,99 @@ define void @no_new_vgpr_for_fp_csr() #1 {
   ret void
 }
 
-; GCN-LABEL: {{^}}realign_stack_no_fp_elim:
-; GCN: s_waitcnt
-; MUBUF-NEXT:   s_mov_b32 [[FP_COPY:s4]], s33
-; FLATSCR-NEXT: s_mov_b32 [[FP_COPY:s0]], s33
-; MUBUF-NEXT:   s_add_i32 s33, s32, 0x7ffc0
-; FLATSCR-NEXT: s_add_i32 s33, s32, 0x1fff
-; MUBUF-NEXT:   s_and_b32 s33, s33, 0xfff80000
-; FLATSCR-NEXT: s_and_b32 s33, s33, 0xffffe000
-; MUBUF-NEXT:   s_mov_b32 s5, s34
-; FLATSCR-NEXT: s_mov_b32 s1, s34
-; MUBUF-NEXT:   s_mov_b32 s34, s32 
-; FLATSCR-NEXT: s_mov_b32 s34, s32
-; MUBUF-NEXT:   s_add_i32 s32, s32, 0x180000
-; FLATSCR-NEXT: s_addk_i32 s32, 0x6000
-; GCN-NEXT:     v_mov_b32_e32 [[ZERO:v[0-9]+]], 0
-; MUBUF-NEXT:   v_mov_b32_e32 [[OFFSET:v[0-9]+]], 0x2000{{$}}
-; MUBUF-NEXT:   buffer_store_dword [[ZERO]], [[OFFSET]], s[0:3], s33 offen{{$}}
-; FLATSCR-NEXT: s_add_i32 s2, s33, 0x2000
-; FLATSCR-NEXT: scratch_store_dword off, [[ZERO]], s2
-; GCN-NEXT: s_waitcnt vmcnt(0)
-; MUBUF-NEXT:   s_mov_b32 s32, s34 
-; MUBUF-NEXT:   s_mov_b32 s34, s5 
-; FLATSCR-NEXT:   s_mov_b32 s32, s34 
-; FLATSCR-NEXT:   s_mov_b32 s34, s1 
-; GCN-NEXT: s_mov_b32 s33, [[FP_COPY]]
-; GCN-NEXT: s_setpc_b64
 define void @realign_stack_no_fp_elim() #1 {
+; MUBUF-LABEL: realign_stack_no_fp_elim:
+; MUBUF:       ; %bb.0:
+; MUBUF-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; MUBUF-NEXT:    s_mov_b32 s4, s33
+; MUBUF-NEXT:    s_add_i32 s33, s32, 0x7ffc0
+; MUBUF-NEXT:    s_and_b32 s33, s33, 0xfff80000
+; MUBUF-NEXT:    s_mov_b32 s5, s34
+; MUBUF-NEXT:    s_mov_b32 s34, s32
+; MUBUF-NEXT:    s_add_i32 s32, s32, 0x180000
+; MUBUF-NEXT:    v_mov_b32_e32 v0, 0
+; MUBUF-NEXT:    v_mov_b32_e32 v1, 0x2000
+; MUBUF-NEXT:    buffer_store_dword v0, v1, s[0:3], s33 offen
+; MUBUF-NEXT:    s_waitcnt vmcnt(0)
+; MUBUF-NEXT:    s_mov_b32 s32, s34
+; MUBUF-NEXT:    s_mov_b32 s34, s5
+; MUBUF-NEXT:    s_mov_b32 s33, s4
+; MUBUF-NEXT:    s_setpc_b64 s[30:31]
+;
+; FLATSCR-LABEL: realign_stack_no_fp_elim:
+; FLATSCR:       ; %bb.0:
+; FLATSCR-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; FLATSCR-NEXT:    s_mov_b32 s0, s33
+; FLATSCR-NEXT:    s_add_i32 s33, s32, 0x1fff
+; FLATSCR-NEXT:    s_and_b32 s33, s33, 0xffffe000
+; FLATSCR-NEXT:    s_mov_b32 s1, s34
+; FLATSCR-NEXT:    s_mov_b32 s34, s32
+; FLATSCR-NEXT:    s_addk_i32 s32, 0x6000
+; FLATSCR-NEXT:    v_mov_b32_e32 v0, 0
+; FLATSCR-NEXT:    s_add_i32 s2, s33, 0x2000
+; FLATSCR-NEXT:    scratch_store_dword off, v0, s2
+; FLATSCR-NEXT:    s_waitcnt vmcnt(0)
+; FLATSCR-NEXT:    s_mov_b32 s32, s34
+; FLATSCR-NEXT:    s_mov_b32 s34, s1
+; FLATSCR-NEXT:    s_mov_b32 s33, s0
+; FLATSCR-NEXT:    s_setpc_b64 s[30:31]
   %alloca = alloca i32, align 8192, addrspace(5)
   store volatile i32 0, ptr addrspace(5) %alloca
   ret void
 }
 
-; GCN-LABEL: {{^}}no_unused_non_csr_sgpr_for_fp:
-; GCN: s_waitcnt
-; GCN-NEXT: s_mov_b32 vcc_lo, s33
-; GCN-NEXT: s_mov_b32 s33, s32
-; GCN-NEXT: s_xor_saveexec_b64 [[COPY_EXEC0:s\[[0-9]+:[0-9]+\]]], -1{{$}}
-; MUBUF-NEXT:   buffer_store_dword [[CSR_VGPR:v[0-9]+]], off, s[0:3], s33 offset:4 ; 4-byte Folded Spill
-; FLATSCR-NEXT: scratch_store_dword off, [[CSR_VGPR:v[0-9]+]], s33 offset:4 ; 4-byte Folded Spill
-; GCN-NEXT: s_mov_b64 exec, [[COPY_EXEC0]]
-; GCN: v_writelane_b32 [[CSR_VGPR]], s30, 0
-; GCN: v_mov_b32_e32 [[ZERO:v[0-9]+]], 0
-; GCN: v_writelane_b32 [[CSR_VGPR]], s31, 1
-; MUBUF:   buffer_store_dword [[ZERO]], off, s[0:3], s33{{$}}
-; FLATSCR: scratch_store_dword off, [[ZERO]], s33{{$}}
-; GCN-NEXT:     s_waitcnt vmcnt(0)
-; GCN: ;;#ASMSTART
-; GCN:     v_readlane_b32 s31, [[CSR_VGPR]], 1
-; GCN:     v_readlane_b32 s30, [[CSR_VGPR]], 0
-;GCN-NEXT: s_mov_b32 s32, s33
-; GCN-NEXT: s_xor_saveexec_b64 [[COPY_EXEC1:s\[[0-9]+:[0-9]+\]]], -1{{$}}
-; MUBUF-NEXT:   buffer_load_dword [[CSR_VGPR]], off, s[0:3], s33 offset:4 ; 4-byte Folded Reload
-; FLATSCR-NEXT: scratch_load_dword [[CSR_VGPR]], off, s33 offset:4 ; 4-byte Folded Reload
-; GCN-NEXT: s_mov_b64 exec, [[COPY_EXEC1]]
-; GCN-NEXT: s_mov_b32 s33, vcc_lo
-; GCN-NEXT: s_waitcnt vmcnt(0)
-; GCN-NEXT: s_setpc_b64 s[30:31]
 define void @no_unused_non_csr_sgpr_for_fp() #1 {
+; MUBUF-LABEL: no_unused_non_csr_sgpr_for_fp:
+; MUBUF:       ; %bb.0:
+; MUBUF-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; MUBUF-NEXT:    s_mov_b32 vcc_lo, s33
+; MUBUF-NEXT:    s_mov_b32 s33, s32
+; MUBUF-NEXT:    s_xor_saveexec_b64 s[4:5], -1
+; MUBUF-NEXT:    buffer_store_dword v1, off, s[0:3], s33 offset:4 ; 4-byte Folded Spill
+; MUBUF-NEXT:    s_mov_b64 exec, s[4:5]
+; MUBUF-NEXT:    v_writelane_b32 v1, s30, 0
+; MUBUF-NEXT:    v_mov_b32_e32 v0, 0
+; MUBUF-NEXT:    s_addk_i32 s32, 0x300
+; MUBUF-NEXT:    v_writelane_b32 v1, s31, 1
+; MUBUF-NEXT:    buffer_store_dword v0, off, s[0:3], s33
+; MUBUF-NEXT:    s_waitcnt vmcnt(0)
+; MUBUF-NEXT:    ;;#ASMSTART
+; MUBUF-NEXT:    ;;#ASMEND
+; MUBUF-NEXT:    v_readlane_b32 s31, v1, 1
+; MUBUF-NEXT:    v_readlane_b32 s30, v1, 0
+; MUBUF-NEXT:    s_mov_b32 s32, s33
+; MUBUF-NEXT:    s_xor_saveexec_b64 s[4:5], -1
+; MUBUF-NEXT:    buffer_load_dword v1, off, s[0:3], s33 offset:4 ; 4-byte Folded Reload
+; MUBUF-NEXT:    s_mov_b64 exec, s[4:5]
+; MUBUF-NEXT:    s_mov_b32 s33, vcc_lo
+; MUBUF-NEXT:    s_waitcnt vmcnt(0)
+; MUBUF-NEXT:    s_setpc_b64 s[30:31]
+;
+; FLATSCR-LABEL: no_unused_non_csr_sgpr_for_fp:
+; FLATSCR:       ; %bb.0:
+; FLATSCR-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; FLATSCR-NEXT:    s_mov_b32 vcc_lo, s33
+; FLATSCR-NEXT:    s_mov_b32 s33, s32
+; FLATSCR-NEXT:    s_xor_saveexec_b64 s[0:1], -1
+; FLATSCR-NEXT:    scratch_store_dword off, v1, s33 offset:4 ; 4-byte Folded Spill
+; FLATSCR-NEXT:    s_mov_b64 exec, s[0:1]
+; FLATSCR-NEXT:    v_writelane_b32 v1, s30, 0
+; FLATSCR-NEXT:    v_mov_b32_e32 v0, 0
+; FLATSCR-NEXT:    s_add_i32 s32, s32, 12
+; FLATSCR-NEXT:    v_writelane_b32 v1, s31, 1
+; FLATSCR-NEXT:    scratch_store_dword off, v0, s33
+; FLATSCR-NEXT:    s_waitcnt vmcnt(0)
+; FLATSCR-NEXT:    ;;#ASMSTART
+; FLATSCR-NEXT:    ;;#ASMEND
+; FLATSCR-NEXT:    v_readlane_b32 s31, v1, 1
+; FLATSCR-NEXT:    v_readlane_b32 s30, v1, 0
+; FLATSCR-NEXT:    s_mov_b32 s32, s33
+; FLATSCR-NEXT:    s_xor_saveexec_b64 s[0:1], -1
+; FLATSCR-NEXT:    scratch_load_dword v1, off, s33 offset:4 ; 4-byte Folded Reload
+; FLATSCR-NEXT:    s_mov_b64 exec, s[0:1]
+; FLATSCR-NEXT:    s_mov_b32 s33, vcc_lo
+; FLATSCR-NEXT:    s_waitcnt vmcnt(0)
+; FLATSCR-NEXT:    s_setpc_b64 s[30:31]
   %alloca = alloca i32, addrspace(5)
   store volatile i32 0, ptr addrspace(5) %alloca
 
@@ -428,31 +1408,64 @@ define void @no_unused_non_csr_sgpr_for_fp() #1 {
 }
 
 ; Need a new CSR VGPR to satisfy the FP spill.
-; GCN-LABEL: {{^}}no_unused_non_csr_sgpr_for_fp_no_scratch_vgpr:
-; GCN: s_waitcnt
-; GCN-NEXT: s_mov_b32 vcc_lo, s33
-; GCN-NEXT: s_mov_b32 s33, s32
-; GCN-NEXT: s_or_saveexec_b64 [[COPY_EXEC0:s\[[0-9]+:[0-9]+\]]], -1{{$}}
-; MUBUF-NEXT:   buffer_store_dword [[CSR_VGPR:v[0-9]+]], off, s[0:3], s33 offset:4 ; 4-byte Folded Spill
-; FLATSCR-NEXT: scratch_store_dword off, [[CSR_VGPR:v[0-9]+]], s33 offset:4 ; 4-byte Folded Spill
-; GCN-NEXT: s_mov_b64 exec, [[COPY_EXEC0]]
-; GCN: v_mov_b32_e32
-
-; MUBUF:       s_addk_i32 s32, 0x300{{$}}
-; FLATSCR:     s_add_i32 s32, s32, 12{{$}}
-; MUBUF-DAG:   buffer_store_dword
-; FLATSCR-DAG: scratch_store_dword
-
-; GCN: ;;#ASMSTART
-; GCN: s_mov_b32 s32, s33
-; GCN: s_or_saveexec_b64 [[COPY_EXEC1:s\[[0-9]+:[0-9]+\]]], -1{{$}}
-; MUBUF-NEXT:   buffer_load_dword [[CSR_VGPR]], off, s[0:3], s33 offset:4 ; 4-byte Folded Reload
-; FLATSCR-NEXT: scratch_load_dword [[CSR_VGPR]], off, s33 offset:4 ; 4-byte Folded Reload
-; GCN-NEXT: s_mov_b64 exec, [[COPY_EXEC1]]
-; GCN-NEXT: s_mov_b32 s33, vcc_lo
-; GCN-NEXT: s_waitcnt vmcnt(0)
-; GCN-NEXT: s_setpc_b64 s[30:31]
 define void @no_unused_non_csr_sgpr_for_fp_no_scratch_vgpr() #1 {
+; MUBUF-LABEL: no_unused_non_csr_sgpr_for_fp_no_scratch_vgpr:
+; MUBUF:       ; %bb.0:
+; MUBUF-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; MUBUF-NEXT:    s_mov_b32 vcc_lo, s33
+; MUBUF-NEXT:    s_mov_b32 s33, s32
+; MUBUF-NEXT:    s_or_saveexec_b64 s[4:5], -1
+; MUBUF-NEXT:    buffer_store_dword v40, off, s[0:3], s33 offset:4 ; 4-byte Folded Spill
+; MUBUF-NEXT:    s_mov_b64 exec, s[4:5]
+; MUBUF-NEXT:    v_writelane_b32 v40, s30, 0
+; MUBUF-NEXT:    v_mov_b32_e32 v0, 0
+; MUBUF-NEXT:    s_addk_i32 s32, 0x300
+; MUBUF-NEXT:    v_writelane_b32 v40, s31, 1
+; MUBUF-NEXT:    buffer_store_dword v0, off, s[0:3], s33
+; MUBUF-NEXT:    s_waitcnt vmcnt(0)
+; MUBUF-NEXT:    ;;#ASMSTART
+; MUBUF-NEXT:    ;;#ASMEND
+; MUBUF-NEXT:    ;;#ASMSTART
+; MUBUF-NEXT:    ; clobber nonpreserved initial VGPRs
+; MUBUF-NEXT:    ;;#ASMEND
+; MUBUF-NEXT:    v_readlane_b32 s31, v40, 1
+; MUBUF-NEXT:    v_readlane_b32 s30, v40, 0
+; MUBUF-NEXT:    s_mov_b32 s32, s33
+; MUBUF-NEXT:    s_or_saveexec_b64 s[4:5], -1
+; MUBUF-NEXT:    buffer_load_dword v40, off, s[0:3], s33 offset:4 ; 4-byte Folded Reload
+; MUBUF-NEXT:    s_mov_b64 exec, s[4:5]
+; MUBUF-NEXT:    s_mov_b32 s33, vcc_lo
+; MUBUF-NEXT:    s_waitcnt vmcnt(0)
+; MUBUF-NEXT:    s_setpc_b64 s[30:31]
+;
+; FLATSCR-LABEL: no_unused_non_csr_sgpr_for_fp_no_scratch_vgpr:
+; FLATSCR:       ; %bb.0:
+; FLATSCR-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; FLATSCR-NEXT:    s_mov_b32 vcc_lo, s33
+; FLATSCR-NEXT:    s_mov_b32 s33, s32
+; FLATSCR-NEXT:    s_or_saveexec_b64 s[0:1], -1
+; FLATSCR-NEXT:    scratch_store_dword off, v40, s33 offset:4 ; 4-byte Folded Spill
+; FLATSCR-NEXT:    s_mov_b64 exec, s[0:1]
+; FLATSCR-NEXT:    v_writelane_b32 v40, s30, 0
+; FLATSCR-NEXT:    v_mov_b32_e32 v0, 0
+; FLATSCR-NEXT:    s_add_i32 s32, s32, 12
+; FLATSCR-NEXT:    v_writelane_b32 v40, s31, 1
+; FLATSCR-NEXT:    scratch_store_dword off, v0, s33
+; FLATSCR-NEXT:    s_waitcnt vmcnt(0)
+; FLATSCR-NEXT:    ;;#ASMSTART
+; FLATSCR-NEXT:    ;;#ASMEND
+; FLATSCR-NEXT:    ;;#ASMSTART
+; FLATSCR-NEXT:    ; clobber nonpreserved initial VGPRs
+; FLATSCR-NEXT:    ;;#ASMEND
+; FLATSCR-NEXT:    v_readlane_b32 s31, v40, 1
+; FLATSCR-NEXT:    v_readlane_b32 s30, v40, 0
+; FLATSCR-NEXT:    s_mov_b32 s32, s33
+; FLATSCR-NEXT:    s_or_saveexec_b64 s[0:1], -1
+; FLATSCR-NEXT:    scratch_load_dword v40, off, s33 offset:4 ; 4-byte Folded Reload
+; FLATSCR-NEXT:    s_mov_b64 exec, s[0:1]
+; FLATSCR-NEXT:    s_mov_b32 s33, vcc_lo
+; FLATSCR-NEXT:    s_waitcnt vmcnt(0)
+; FLATSCR-NEXT:    s_setpc_b64 s[30:31]
   %alloca = alloca i32, addrspace(5)
   store volatile i32 0, ptr addrspace(5) %alloca
 
@@ -474,32 +1487,72 @@ define void @no_unused_non_csr_sgpr_for_fp_no_scratch_vgpr() #1 {
 
 ; The byval argument exceeds the MUBUF constant offset, so a scratch
 ; register is needed to access the CSR VGPR slot.
-; GCN-LABEL: {{^}}scratch_reg_needed_mubuf_offset:
-; GCN: s_waitcnt
-; GCN-NEXT: s_mov_b32 vcc_lo, s33
-; GCN-DAG:  s_mov_b32 s33, s32
-; GCN-NEXT: s_or_saveexec_b64 [[COPY_EXEC0:s\[[0-9]+:[0-9]+\]]], -1{{$}}
-; MUBUF-NEXT: s_add_i32 [[SCRATCH_SGPR:s[0-9]+]], s33, 0x40100
-; FLATSCR-NEXT: s_add_i32 [[SCRATCH_SGPR:s[0-9]+]], s33, 0x1004
-; MUBUF-NEXT: buffer_store_dword [[CSR_VGPR:v[0-9]+]], off, s[0:3], [[SCRATCH_SGPR]] ; 4-byte Folded Spill
-; FLATSCR-NEXT: scratch_store_dword off, [[CSR_VGPR:v[0-9]+]], [[SCRATCH_SGPR]] ; 4-byte Folded Spill
-; GCN-NEXT: s_mov_b64 exec, [[COPY_EXEC0]]
-; MUBUF-DAG:   s_add_i32 s32, s32, 0x40300{{$}}
-; FLATSCR-DAG: s_addk_i32 s32, 0x100c{{$}}
-; MUBUF-DAG:   buffer_store_dword
-; FLATSCR-DAG: scratch_store_dword
-
-; GCN: ;;#ASMSTART
-; GCN: s_or_saveexec_b64 [[COPY_EXEC1:s\[[0-9]+:[0-9]+\]]], -1{{$}}
-; MUBUF-NEXT: s_add_i32 [[SCRATCH_SGPR:s[0-9]+]], s33, 0x40100
-; MUBUF-NEXT: buffer_load_dword [[CSR_VGPR]], off, s[0:3], [[SCRATCH_SGPR]] ; 4-byte Folded Reload
-; FLATSCR-NEXT: s_add_i32 [[SCRATCH_SGPR:s[0-9]+]], s33, 0x1004
-; FLATSCR-NEXT: scratch_load_dword [[CSR_VGPR]], off, [[SCRATCH_SGPR]] ; 4-byte Folded Reload
-; GCN-NEXT: s_mov_b64 exec, [[COPY_EXEC1]]
-; GCN-NEXT: s_mov_b32 s33, vcc_lo
-; GCN-NEXT: s_waitcnt vmcnt(0)
-; GCN-NEXT: s_setpc_b64 s[30:31]
 define void @scratch_reg_needed_mubuf_offset(ptr addrspace(5) byval([4096 x i8]) align 4 %arg) #1 {
+; MUBUF-LABEL: scratch_reg_needed_mubuf_offset:
+; MUBUF:       ; %bb.0:
+; MUBUF-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; MUBUF-NEXT:    s_mov_b32 vcc_lo, s33
+; MUBUF-NEXT:    s_mov_b32 s33, s32
+; MUBUF-NEXT:    s_or_saveexec_b64 s[4:5], -1
+; MUBUF-NEXT:    s_add_i32 s6, s33, 0x40100
+; MUBUF-NEXT:    buffer_store_dword v40, off, s[0:3], s6 ; 4-byte Folded Spill
+; MUBUF-NEXT:    s_mov_b64 exec, s[4:5]
+; MUBUF-NEXT:    v_writelane_b32 v40, s30, 0
+; MUBUF-NEXT:    v_mov_b32_e32 v0, 0
+; MUBUF-NEXT:    v_mov_b32_e32 v1, 0x1000
+; MUBUF-NEXT:    s_add_i32 s32, s32, 0x40300
+; MUBUF-NEXT:    v_writelane_b32 v40, s31, 1
+; MUBUF-NEXT:    buffer_store_dword v0, v1, s[0:3], s33 offen
+; MUBUF-NEXT:    s_waitcnt vmcnt(0)
+; MUBUF-NEXT:    ;;#ASMSTART
+; MUBUF-NEXT:    ; clobber nonpreserved SGPRs
+; MUBUF-NEXT:    ;;#ASMEND
+; MUBUF-NEXT:    ;;#ASMSTART
+; MUBUF-NEXT:    ; clobber nonpreserved VGPRs
+; MUBUF-NEXT:    ;;#ASMEND
+; MUBUF-NEXT:    v_readlane_b32 s31, v40, 1
+; MUBUF-NEXT:    v_readlane_b32 s30, v40, 0
+; MUBUF-NEXT:    s_mov_b32 s32, s33
+; MUBUF-NEXT:    s_or_saveexec_b64 s[4:5], -1
+; MUBUF-NEXT:    s_add_i32 s6, s33, 0x40100
+; MUBUF-NEXT:    buffer_load_dword v40, off, s[0:3], s6 ; 4-byte Folded Reload
+; MUBUF-NEXT:    s_mov_b64 exec, s[4:5]
+; MUBUF-NEXT:    s_mov_b32 s33, vcc_lo
+; MUBUF-NEXT:    s_waitcnt vmcnt(0)
+; MUBUF-NEXT:    s_setpc_b64 s[30:31]
+;
+; FLATSCR-LABEL: scratch_reg_needed_mubuf_offset:
+; FLATSCR:       ; %bb.0:
+; FLATSCR-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; FLATSCR-NEXT:    s_mov_b32 vcc_lo, s33
+; FLATSCR-NEXT:    s_mov_b32 s33, s32
+; FLATSCR-NEXT:    s_or_saveexec_b64 s[0:1], -1
+; FLATSCR-NEXT:    s_add_i32 s2, s33, 0x1004
+; FLATSCR-NEXT:    scratch_store_dword off, v40, s2 ; 4-byte Folded Spill
+; FLATSCR-NEXT:    s_mov_b64 exec, s[0:1]
+; FLATSCR-NEXT:    s_addk_i32 s32, 0x100c
+; FLATSCR-NEXT:    v_writelane_b32 v40, s30, 0
+; FLATSCR-NEXT:    v_mov_b32_e32 v0, 0
+; FLATSCR-NEXT:    s_add_i32 s0, s33, 0x1000
+; FLATSCR-NEXT:    v_writelane_b32 v40, s31, 1
+; FLATSCR-NEXT:    scratch_store_dword off, v0, s0
+; FLATSCR-NEXT:    s_waitcnt vmcnt(0)
+; FLATSCR-NEXT:    ;;#ASMSTART
+; FLATSCR-NEXT:    ; clobber nonpreserved SGPRs
+; FLATSCR-NEXT:    ;;#ASMEND
+; FLATSCR-NEXT:    ;;#ASMSTART
+; FLATSCR-NEXT:    ; clobber nonpreserved VGPRs
+; FLATSCR-NEXT:    ;;#ASMEND
+; FLATSCR-NEXT:    v_readlane_b32 s31, v40, 1
+; FLATSCR-NEXT:    v_readlane_b32 s30, v40, 0
+; FLATSCR-NEXT:    s_mov_b32 s32, s33
+; FLATSCR-NEXT:    s_or_saveexec_b64 s[0:1], -1
+; FLATSCR-NEXT:    s_add_i32 s2, s33, 0x1004
+; FLATSCR-NEXT:    scratch_load_dword v40, off, s2 ; 4-byte Folded Reload
+; FLATSCR-NEXT:    s_mov_b64 exec, s[0:1]
+; FLATSCR-NEXT:    s_mov_b32 s33, vcc_lo
+; FLATSCR-NEXT:    s_waitcnt vmcnt(0)
+; FLATSCR-NEXT:    s_setpc_b64 s[30:31]
   %alloca = alloca i32, addrspace(5)
   store volatile i32 0, ptr addrspace(5) %alloca
 
@@ -520,25 +1573,72 @@ define void @scratch_reg_needed_mubuf_offset(ptr addrspace(5) byval([4096 x i8])
   ret void
 }
 
-; GCN-LABEL: {{^}}local_empty_func:
-; GCN: s_waitcnt
-; GCN-NEXT: s_setpc_b64
 define internal void @local_empty_func() #0 {
+; GCN-LABEL: local_empty_func:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    s_setpc_b64 s[30:31]
   ret void
 }
 
 ; An FP is needed, despite not needing any spills
 ; TODO: Ccould see callee does not use stack and omit FP.
-; GCN-LABEL: {{^}}ipra_call_with_stack:
-; GCN: s_mov_b32 [[TMP_SGPR:s[0-9]+]], s33
-; GCN: s_mov_b32 s33, s32
-; MUBUF:   s_addk_i32 s32, 0x400
-; FLATSCR: s_add_i32 s32, s32, 16
-; MUBUF:   buffer_store_dword v{{[0-9]+}}, off, s[0:3], s33{{$}}
-; FLATSCR: scratch_store_dword off, v{{[0-9]+}}, s33{{$}}
-; GCN:     s_swappc_b64
-; GCN: s_mov_b32 s33, [[TMP_SGPR]]
 define void @ipra_call_with_stack() #0 {
+; MUBUF-LABEL: ipra_call_with_stack:
+; MUBUF:       ; %bb.0:
+; MUBUF-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; MUBUF-NEXT:    s_mov_b32 s18, s33
+; MUBUF-NEXT:    s_mov_b32 s33, s32
+; MUBUF-NEXT:    s_xor_saveexec_b64 s[16:17], -1
+; MUBUF-NEXT:    buffer_store_dword v1, off, s[0:3], s33 offset:4 ; 4-byte Folded Spill
+; MUBUF-NEXT:    s_mov_b64 exec, s[16:17]
+; MUBUF-NEXT:    s_addk_i32 s32, 0x400
+; MUBUF-NEXT:    v_writelane_b32 v1, s30, 0
+; MUBUF-NEXT:    v_mov_b32_e32 v0, 0
+; MUBUF-NEXT:    s_getpc_b64 s[16:17]
+; MUBUF-NEXT:    s_add_u32 s16, s16, local_empty_func@rel32@lo+4
+; MUBUF-NEXT:    s_addc_u32 s17, s17, local_empty_func@rel32@hi+12
+; MUBUF-NEXT:    v_writelane_b32 v1, s31, 1
+; MUBUF-NEXT:    buffer_store_dword v0, off, s[0:3], s33
+; MUBUF-NEXT:    s_waitcnt vmcnt(0)
+; MUBUF-NEXT:    s_swappc_b64 s[30:31], s[16:17]
+; MUBUF-NEXT:    v_readlane_b32 s31, v1, 1
+; MUBUF-NEXT:    v_readlane_b32 s30, v1, 0
+; MUBUF-NEXT:    s_mov_b32 s32, s33
+; MUBUF-NEXT:    s_xor_saveexec_b64 s[4:5], -1
+; MUBUF-NEXT:    buffer_load_dword v1, off, s[0:3], s33 offset:4 ; 4-byte Folded Reload
+; MUBUF-NEXT:    s_mov_b64 exec, s[4:5]
+; MUBUF-NEXT:    s_mov_b32 s33, s18
+; MUBUF-NEXT:    s_waitcnt vmcnt(0)
+; MUBUF-NEXT:    s_setpc_b64 s[30:31]
+;
+; FLATSCR-LABEL: ipra_call_with_stack:
+; FLATSCR:       ; %bb.0:
+; FLATSCR-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; FLATSCR-NEXT:    s_mov_b32 s2, s33
+; FLATSCR-NEXT:    s_mov_b32 s33, s32
+; FLATSCR-NEXT:    s_xor_saveexec_b64 s[0:1], -1
+; FLATSCR-NEXT:    scratch_store_dword off, v1, s33 offset:4 ; 4-byte Folded Spill
+; FLATSCR-NEXT:    s_mov_b64 exec, s[0:1]
+; FLATSCR-NEXT:    s_add_i32 s32, s32, 16
+; FLATSCR-NEXT:    v_writelane_b32 v1, s30, 0
+; FLATSCR-NEXT:    v_mov_b32_e32 v0, 0
+; FLATSCR-NEXT:    s_getpc_b64 s[0:1]
+; FLATSCR-NEXT:    s_add_u32 s0, s0, local_empty_func@rel32@lo+4
+; FLATSCR-NEXT:    s_addc_u32 s1, s1, local_empty_func@rel32@hi+12
+; FLATSCR-NEXT:    v_writelane_b32 v1, s31, 1
+; FLATSCR-NEXT:    scratch_store_dword off, v0, s33
+; FLATSCR-NEXT:    s_waitcnt vmcnt(0)
+; FLATSCR-NEXT:    s_swappc_b64 s[30:31], s[0:1]
+; FLATSCR-NEXT:    v_readlane_b32 s31, v1, 1
+; FLATSCR-NEXT:    v_readlane_b32 s30, v1, 0
+; FLATSCR-NEXT:    s_mov_b32 s32, s33
+; FLATSCR-NEXT:    s_xor_saveexec_b64 s[0:1], -1
+; FLATSCR-NEXT:    scratch_load_dword v1, off, s33 offset:4 ; 4-byte Folded Reload
+; FLATSCR-NEXT:    s_mov_b64 exec, s[0:1]
+; FLATSCR-NEXT:    s_mov_b32 s33, s2
+; FLATSCR-NEXT:    s_waitcnt vmcnt(0)
+; FLATSCR-NEXT:    s_setpc_b64 s[30:31]
   %alloca = alloca i32, addrspace(5)
   store volatile i32 0, ptr addrspace(5) %alloca
   call void @local_empty_func()
@@ -546,21 +1646,41 @@ define void @ipra_call_with_stack() #0 {
 }
 
 ; With no free registers, we must spill the FP to memory.
-; GCN-LABEL: {{^}}callee_need_to_spill_fp_to_memory:
-; MUBUF:   s_mov_b32 [[FP_SCRATCH_COPY:s[0-9]+]], s33
-; FLATSCR: s_mov_b32 s0, s33
-; GCN:     s_mov_b32 s33, s32
-; MUBUF:   v_mov_b32_e32 [[TMP_VGPR1:v[0-9]+]], [[FP_SCRATCH_COPY]]
-; MUBUF:   buffer_store_dword [[TMP_VGPR1]], off, s[0:3], s33 ; 4-byte Folded Spill
-; MUBUF:   buffer_load_dword [[TMP_VGPR2:v[0-9]+]], off, s[0:3], s33 ; 4-byte Folded Reload
-; MUBUF:   s_waitcnt vmcnt(0)
-; MUBUF:   v_readfirstlane_b32 [[FP_SCRATCH_COPY:s[0-9]+]], [[TMP_VGPR2]]
-; MUBUF:   s_mov_b32 s33, [[FP_SCRATCH_COPY]]
-; FLATSCR: s_mov_b32 s33, s0
-; GCN:     s_setpc_b64
-; MUBUF:   ScratchSize: 8
-; FLATSCR: ScratchSize: 0
 define void @callee_need_to_spill_fp_to_memory() #3 {
+; MUBUF-LABEL: callee_need_to_spill_fp_to_memory:
+; MUBUF:       ; %bb.0:
+; MUBUF-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; MUBUF-NEXT:    s_mov_b32 s4, s33
+; MUBUF-NEXT:    s_mov_b32 s33, s32
+; MUBUF-NEXT:    v_mov_b32_e32 v0, s4
+; MUBUF-NEXT:    buffer_store_dword v0, off, s[0:3], s33 ; 4-byte Folded Spill
+; MUBUF-NEXT:    ;;#ASMSTART
+; MUBUF-NEXT:    ; clobber nonpreserved SGPRs
+; MUBUF-NEXT:    ;;#ASMEND
+; MUBUF-NEXT:    ;;#ASMSTART
+; MUBUF-NEXT:    ; clobber all VGPRs
+; MUBUF-NEXT:    ;;#ASMEND
+; MUBUF-NEXT:    buffer_load_dword v0, off, s[0:3], s33 ; 4-byte Folded Reload
+; MUBUF-NEXT:    s_addk_i32 s32, 0x200
+; MUBUF-NEXT:    s_mov_b32 s32, s33
+; MUBUF-NEXT:    s_waitcnt vmcnt(0)
+; MUBUF-NEXT:    v_readfirstlane_b32 s4, v0
+; MUBUF-NEXT:    s_mov_b32 s33, s4
+; MUBUF-NEXT:    s_setpc_b64 s[30:31]
+;
+; FLATSCR-LABEL: callee_need_to_spill_fp_to_memory:
+; FLATSCR:       ; %bb.0:
+; FLATSCR-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; FLATSCR-NEXT:    s_mov_b32 s0, s33
+; FLATSCR-NEXT:    s_mov_b32 s33, s32
+; FLATSCR-NEXT:    ;;#ASMSTART
+; FLATSCR-NEXT:    ; clobber nonpreserved SGPRs
+; FLATSCR-NEXT:    ;;#ASMEND
+; FLATSCR-NEXT:    ;;#ASMSTART
+; FLATSCR-NEXT:    ; clobber all VGPRs
+; FLATSCR-NEXT:    ;;#ASMEND
+; FLATSCR-NEXT:    s_mov_b32 s33, s0
+; FLATSCR-NEXT:    s_setpc_b64 s[30:31]
   call void asm sideeffect "; clobber nonpreserved SGPRs",
     "~{s4},~{s5},~{s6},~{s7},~{s8},~{s9}
     ,~{s10},~{s11},~{s12},~{s13},~{s14},~{s15},~{s16},~{s17},~{s18},~{s19}
@@ -578,23 +1698,313 @@ define void @callee_need_to_spill_fp_to_memory() #3 {
 ; If we have a reserved VGPR that can be used for SGPR spills, we may still
 ; need to spill the FP to memory if there are no free lanes in the reserved
 ; VGPR.
-; GCN-LABEL: {{^}}callee_need_to_spill_fp_to_memory_full_reserved_vgpr:
-; MUBUF:   s_mov_b32 [[FP_SCRATCH_COPY:s[0-9]+]], s33
-; GCN:     s_mov_b32 s33, s32
-; MUBUF:   s_xor_saveexec_b64 [[COPY_EXEC1:s\[[0-9]+:[0-9]+\]]], -1{{$}}
-; MUBUF:   s_mov_b64 exec, [[COPY_EXEC1]]
-; MUBUF:   v_mov_b32_e32 [[TMP_VGPR1:v[0-9]+]], [[FP_SCRATCH_COPY]]
-; MUBUF:   buffer_store_dword [[TMP_VGPR1]], off, s[0:3], s33 offset:[[OFF:[0-9]+]]
-; GCN-NOT: v_writelane_b32 v40, s33
-; GCN-NOT: v_readlane_b32 s33, v40
-; GCN-NOT: v_readlane_b32 s33, v40
-; MUBUF:   buffer_load_dword [[TMP_VGPR2:v[0-9]+]], off, s[0:3], s33 offset:[[OFF]]
-; MUBUF:   v_readfirstlane_b32 [[FP_SCRATCH_COPY:s[0-9]+]], [[TMP_VGPR2]]
-; MUBUF:   s_xor_saveexec_b64 [[COPY_EXEC2:s\[[0-9]+:[0-9]+\]]], -1{{$}}
-; MUBUF:   s_mov_b64 exec, [[COPY_EXEC2]]
-; MUBUF:   s_mov_b32 s33, [[FP_SCRATCH_COPY]]
-; GCN:     s_setpc_b64
 define void @callee_need_to_spill_fp_to_memory_full_reserved_vgpr() #3 {
+; MUBUF-LABEL: callee_need_to_spill_fp_to_memory_full_reserved_vgpr:
+; MUBUF:       ; %bb.0:
+; MUBUF-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; MUBUF-NEXT:    s_mov_b32 s4, s33
+; MUBUF-NEXT:    s_mov_b32 s33, s32
+; MUBUF-NEXT:    s_xor_saveexec_b64 s[6:7], -1
+; MUBUF-NEXT:    buffer_store_dword v39, off, s[0:3], s33 ; 4-byte Folded Spill
+; MUBUF-NEXT:    s_mov_b64 exec, s[6:7]
+; MUBUF-NEXT:    v_writelane_b32 v39, s39, 0
+; MUBUF-NEXT:    v_writelane_b32 v39, s40, 1
+; MUBUF-NEXT:    v_writelane_b32 v39, s41, 2
+; MUBUF-NEXT:    v_writelane_b32 v39, s42, 3
+; MUBUF-NEXT:    v_writelane_b32 v39, s43, 4
+; MUBUF-NEXT:    v_writelane_b32 v39, s44, 5
+; MUBUF-NEXT:    v_writelane_b32 v39, s45, 6
+; MUBUF-NEXT:    v_writelane_b32 v39, s46, 7
+; MUBUF-NEXT:    v_writelane_b32 v39, s47, 8
+; MUBUF-NEXT:    v_writelane_b32 v39, s48, 9
+; MUBUF-NEXT:    v_writelane_b32 v39, s49, 10
+; MUBUF-NEXT:    v_writelane_b32 v39, s50, 11
+; MUBUF-NEXT:    v_writelane_b32 v39, s51, 12
+; MUBUF-NEXT:    v_writelane_b32 v39, s52, 13
+; MUBUF-NEXT:    v_writelane_b32 v39, s53, 14
+; MUBUF-NEXT:    v_writelane_b32 v39, s54, 15
+; MUBUF-NEXT:    v_writelane_b32 v39, s55, 16
+; MUBUF-NEXT:    v_writelane_b32 v39, s56, 17
+; MUBUF-NEXT:    v_writelane_b32 v39, s57, 18
+; MUBUF-NEXT:    v_writelane_b32 v39, s58, 19
+; MUBUF-NEXT:    v_writelane_b32 v39, s59, 20
+; MUBUF-NEXT:    v_writelane_b32 v39, s60, 21
+; MUBUF-NEXT:    v_writelane_b32 v39, s61, 22
+; MUBUF-NEXT:    v_writelane_b32 v39, s62, 23
+; MUBUF-NEXT:    v_writelane_b32 v39, s63, 24
+; MUBUF-NEXT:    v_writelane_b32 v39, s64, 25
+; MUBUF-NEXT:    v_writelane_b32 v39, s65, 26
+; MUBUF-NEXT:    v_writelane_b32 v39, s66, 27
+; MUBUF-NEXT:    v_writelane_b32 v39, s67, 28
+; MUBUF-NEXT:    v_writelane_b32 v39, s68, 29
+; MUBUF-NEXT:    v_writelane_b32 v39, s69, 30
+; MUBUF-NEXT:    v_writelane_b32 v39, s70, 31
+; MUBUF-NEXT:    v_writelane_b32 v39, s71, 32
+; MUBUF-NEXT:    v_writelane_b32 v39, s72, 33
+; MUBUF-NEXT:    v_writelane_b32 v39, s73, 34
+; MUBUF-NEXT:    v_writelane_b32 v39, s74, 35
+; MUBUF-NEXT:    v_writelane_b32 v39, s75, 36
+; MUBUF-NEXT:    v_writelane_b32 v39, s76, 37
+; MUBUF-NEXT:    v_writelane_b32 v39, s77, 38
+; MUBUF-NEXT:    v_writelane_b32 v39, s78, 39
+; MUBUF-NEXT:    v_writelane_b32 v39, s79, 40
+; MUBUF-NEXT:    v_writelane_b32 v39, s80, 41
+; MUBUF-NEXT:    v_writelane_b32 v39, s81, 42
+; MUBUF-NEXT:    v_writelane_b32 v39, s82, 43
+; MUBUF-NEXT:    v_writelane_b32 v39, s83, 44
+; MUBUF-NEXT:    v_writelane_b32 v39, s84, 45
+; MUBUF-NEXT:    v_writelane_b32 v39, s85, 46
+; MUBUF-NEXT:    v_writelane_b32 v39, s86, 47
+; MUBUF-NEXT:    v_writelane_b32 v39, s87, 48
+; MUBUF-NEXT:    v_writelane_b32 v39, s88, 49
+; MUBUF-NEXT:    v_writelane_b32 v39, s89, 50
+; MUBUF-NEXT:    v_writelane_b32 v39, s90, 51
+; MUBUF-NEXT:    v_writelane_b32 v39, s91, 52
+; MUBUF-NEXT:    v_writelane_b32 v39, s92, 53
+; MUBUF-NEXT:    v_writelane_b32 v39, s93, 54
+; MUBUF-NEXT:    v_writelane_b32 v39, s94, 55
+; MUBUF-NEXT:    v_writelane_b32 v39, s95, 56
+; MUBUF-NEXT:    v_writelane_b32 v39, s96, 57
+; MUBUF-NEXT:    v_writelane_b32 v39, s97, 58
+; MUBUF-NEXT:    v_writelane_b32 v39, s98, 59
+; MUBUF-NEXT:    v_writelane_b32 v39, s99, 60
+; MUBUF-NEXT:    v_writelane_b32 v39, s100, 61
+; MUBUF-NEXT:    v_mov_b32_e32 v0, s4
+; MUBUF-NEXT:    v_writelane_b32 v39, s101, 62
+; MUBUF-NEXT:    buffer_store_dword v0, off, s[0:3], s33 offset:4 ; 4-byte Folded Spill
+; MUBUF-NEXT:    v_writelane_b32 v39, s102, 63
+; MUBUF-NEXT:    ;;#ASMSTART
+; MUBUF-NEXT:    ; clobber nonpreserved SGPRs and 64 CSRs
+; MUBUF-NEXT:    ;;#ASMEND
+; MUBUF-NEXT:    ;;#ASMSTART
+; MUBUF-NEXT:    ; clobber all VGPRs except CSR v40
+; MUBUF-NEXT:    ;;#ASMEND
+; MUBUF-NEXT:    buffer_load_dword v0, off, s[0:3], s33 offset:4 ; 4-byte Folded Reload
+; MUBUF-NEXT:    s_addk_i32 s32, 0x300
+; MUBUF-NEXT:    v_readlane_b32 s102, v39, 63
+; MUBUF-NEXT:    v_readlane_b32 s101, v39, 62
+; MUBUF-NEXT:    v_readlane_b32 s100, v39, 61
+; MUBUF-NEXT:    v_readlane_b32 s99, v39, 60
+; MUBUF-NEXT:    v_readlane_b32 s98, v39, 59
+; MUBUF-NEXT:    v_readlane_b32 s97, v39, 58
+; MUBUF-NEXT:    v_readlane_b32 s96, v39, 57
+; MUBUF-NEXT:    v_readlane_b32 s95, v39, 56
+; MUBUF-NEXT:    v_readlane_b32 s94, v39, 55
+; MUBUF-NEXT:    v_readlane_b32 s93, v39, 54
+; MUBUF-NEXT:    v_readlane_b32 s92, v39, 53
+; MUBUF-NEXT:    v_readlane_b32 s91, v39, 52
+; MUBUF-NEXT:    v_readlane_b32 s90, v39, 51
+; MUBUF-NEXT:    v_readlane_b32 s89, v39, 50
+; MUBUF-NEXT:    v_readlane_b32 s88, v39, 49
+; MUBUF-NEXT:    v_readlane_b32 s87, v39, 48
+; MUBUF-NEXT:    v_readlane_b32 s86, v39, 47
+; MUBUF-NEXT:    v_readlane_b32 s85, v39, 46
+; MUBUF-NEXT:    v_readlane_b32 s84, v39, 45
+; MUBUF-NEXT:    v_readlane_b32 s83, v39, 44
+; MUBUF-NEXT:    v_readlane_b32 s82, v39, 43
+; MUBUF-NEXT:    v_readlane_b32 s81, v39, 42
+; MUBUF-NEXT:    v_readlane_b32 s80, v39, 41
+; MUBUF-NEXT:    v_readlane_b32 s79, v39, 40
+; MUBUF-NEXT:    v_readlane_b32 s78, v39, 39
+; MUBUF-NEXT:    v_readlane_b32 s77, v39, 38
+; MUBUF-NEXT:    v_readlane_b32 s76, v39, 37
+; MUBUF-NEXT:    v_readlane_b32 s75, v39, 36
+; MUBUF-NEXT:    v_readlane_b32 s74, v39, 35
+; MUBUF-NEXT:    v_readlane_b32 s73, v39, 34
+; MUBUF-NEXT:    v_readlane_b32 s72, v39, 33
+; MUBUF-NEXT:    v_readlane_b32 s71, v39, 32
+; MUBUF-NEXT:    v_readlane_b32 s70, v39, 31
+; MUBUF-NEXT:    v_readlane_b32 s69, v39, 30
+; MUBUF-NEXT:    v_readlane_b32 s68, v39, 29
+; MUBUF-NEXT:    v_readlane_b32 s67, v39, 28
+; MUBUF-NEXT:    v_readlane_b32 s66, v39, 27
+; MUBUF-NEXT:    v_readlane_b32 s65, v39, 26
+; MUBUF-NEXT:    v_readlane_b32 s64, v39, 25
+; MUBUF-NEXT:    v_readlane_b32 s63, v39, 24
+; MUBUF-NEXT:    v_readlane_b32 s62, v39, 23
+; MUBUF-NEXT:    v_readlane_b32 s61, v39, 22
+; MUBUF-NEXT:    v_readlane_b32 s60, v39, 21
+; MUBUF-NEXT:    v_readlane_b32 s59, v39, 20
+; MUBUF-NEXT:    v_readlane_b32 s58, v39, 19
+; MUBUF-NEXT:    v_readlane_b32 s57, v39, 18
+; MUBUF-NEXT:    v_readlane_b32 s56, v39, 17
+; MUBUF-NEXT:    v_readlane_b32 s55, v39, 16
+; MUBUF-NEXT:    v_readlane_b32 s54, v39, 15
+; MUBUF-NEXT:    v_readlane_b32 s53, v39, 14
+; MUBUF-NEXT:    v_readlane_b32 s52, v39, 13
+; MUBUF-NEXT:    v_readlane_b32 s51, v39, 12
+; MUBUF-NEXT:    v_readlane_b32 s50, v39, 11
+; MUBUF-NEXT:    v_readlane_b32 s49, v39, 10
+; MUBUF-NEXT:    v_readlane_b32 s48, v39, 9
+; MUBUF-NEXT:    v_readlane_b32 s47, v39, 8
+; MUBUF-NEXT:    v_readlane_b32 s46, v39, 7
+; MUBUF-NEXT:    v_readlane_b32 s45, v39, 6
+; MUBUF-NEXT:    v_readlane_b32 s44, v39, 5
+; MUBUF-NEXT:    v_readlane_b32 s43, v39, 4
+; MUBUF-NEXT:    v_readlane_b32 s42, v39, 3
+; MUBUF-NEXT:    v_readlane_b32 s41, v39, 2
+; MUBUF-NEXT:    v_readlane_b32 s40, v39, 1
+; MUBUF-NEXT:    v_readlane_b32 s39, v39, 0
+; MUBUF-NEXT:    s_mov_b32 s32, s33
+; MUBUF-NEXT:    s_waitcnt vmcnt(0)
+; MUBUF-NEXT:    v_readfirstlane_b32 s4, v0
+; MUBUF-NEXT:    s_xor_saveexec_b64 s[6:7], -1
+; MUBUF-NEXT:    buffer_load_dword v39, off, s[0:3], s33 ; 4-byte Folded Reload
+; MUBUF-NEXT:    s_mov_b64 exec, s[6:7]
+; MUBUF-NEXT:    s_mov_b32 s33, s4
+; MUBUF-NEXT:    s_waitcnt vmcnt(0)
+; MUBUF-NEXT:    s_setpc_b64 s[30:31]
+;
+; FLATSCR-LABEL: callee_need_to_spill_fp_to_memory_full_reserved_vgpr:
+; FLATSCR:       ; %bb.0:
+; FLATSCR-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; FLATSCR-NEXT:    s_mov_b32 s0, s33
+; FLATSCR-NEXT:    s_mov_b32 s33, s32
+; FLATSCR-NEXT:    s_xor_saveexec_b64 s[2:3], -1
+; FLATSCR-NEXT:    scratch_store_dword off, v39, s33 ; 4-byte Folded Spill
+; FLATSCR-NEXT:    s_mov_b64 exec, s[2:3]
+; FLATSCR-NEXT:    v_writelane_b32 v39, s39, 0
+; FLATSCR-NEXT:    v_writelane_b32 v39, s40, 1
+; FLATSCR-NEXT:    v_writelane_b32 v39, s41, 2
+; FLATSCR-NEXT:    v_writelane_b32 v39, s42, 3
+; FLATSCR-NEXT:    v_writelane_b32 v39, s43, 4
+; FLATSCR-NEXT:    v_writelane_b32 v39, s44, 5
+; FLATSCR-NEXT:    v_writelane_b32 v39, s45, 6
+; FLATSCR-NEXT:    v_writelane_b32 v39, s46, 7
+; FLATSCR-NEXT:    v_writelane_b32 v39, s47, 8
+; FLATSCR-NEXT:    v_writelane_b32 v39, s48, 9
+; FLATSCR-NEXT:    v_writelane_b32 v39, s49, 10
+; FLATSCR-NEXT:    v_writelane_b32 v39, s50, 11
+; FLATSCR-NEXT:    v_writelane_b32 v39, s51, 12
+; FLATSCR-NEXT:    v_writelane_b32 v39, s52, 13
+; FLATSCR-NEXT:    v_writelane_b32 v39, s53, 14
+; FLATSCR-NEXT:    v_writelane_b32 v39, s54, 15
+; FLATSCR-NEXT:    v_writelane_b32 v39, s55, 16
+; FLATSCR-NEXT:    v_writelane_b32 v39, s56, 17
+; FLATSCR-NEXT:    v_writelane_b32 v39, s57, 18
+; FLATSCR-NEXT:    v_writelane_b32 v39, s58, 19
+; FLATSCR-NEXT:    v_writelane_b32 v39, s59, 20
+; FLATSCR-NEXT:    v_writelane_b32 v39, s60, 21
+; FLATSCR-NEXT:    v_writelane_b32 v39, s61, 22
+; FLATSCR-NEXT:    v_writelane_b32 v39, s62, 23
+; FLATSCR-NEXT:    v_writelane_b32 v39, s63, 24
+; FLATSCR-NEXT:    v_writelane_b32 v39, s64, 25
+; FLATSCR-NEXT:    v_writelane_b32 v39, s65, 26
+; FLATSCR-NEXT:    v_writelane_b32 v39, s66, 27
+; FLATSCR-NEXT:    v_writelane_b32 v39, s67, 28
+; FLATSCR-NEXT:    v_writelane_b32 v39, s68, 29
+; FLATSCR-NEXT:    v_writelane_b32 v39, s69, 30
+; FLATSCR-NEXT:    v_writelane_b32 v39, s70, 31
+; FLATSCR-NEXT:    v_writelane_b32 v39, s71, 32
+; FLATSCR-NEXT:    v_writelane_b32 v39, s72, 33
+; FLATSCR-NEXT:    v_writelane_b32 v39, s73, 34
+; FLATSCR-NEXT:    v_writelane_b32 v39, s74, 35
+; FLATSCR-NEXT:    v_writelane_b32 v39, s75, 36
+; FLATSCR-NEXT:    v_writelane_b32 v39, s76, 37
+; FLATSCR-NEXT:    v_writelane_b32 v39, s77, 38
+; FLATSCR-NEXT:    v_writelane_b32 v39, s78, 39
+; FLATSCR-NEXT:    v_writelane_b32 v39, s79, 40
+; FLATSCR-NEXT:    v_writelane_b32 v39, s80, 41
+; FLATSCR-NEXT:    v_writelane_b32 v39, s81, 42
+; FLATSCR-NEXT:    v_writelane_b32 v39, s82, 43
+; FLATSCR-NEXT:    v_writelane_b32 v39, s83, 44
+; FLATSCR-NEXT:    v_writelane_b32 v39, s84, 45
+; FLATSCR-NEXT:    v_writelane_b32 v39, s85, 46
+; FLATSCR-NEXT:    v_writelane_b32 v39, s86, 47
+; FLATSCR-NEXT:    v_writelane_b32 v39, s87, 48
+; FLATSCR-NEXT:    v_writelane_b32 v39, s88, 49
+; FLATSCR-NEXT:    v_writelane_b32 v39, s89, 50
+; FLATSCR-NEXT:    v_writelane_b32 v39, s90, 51
+; FLATSCR-NEXT:    v_writelane_b32 v39, s91, 52
+; FLATSCR-NEXT:    v_writelane_b32 v39, s92, 53
+; FLATSCR-NEXT:    v_writelane_b32 v39, s93, 54
+; FLATSCR-NEXT:    v_writelane_b32 v39, s94, 55
+; FLATSCR-NEXT:    v_writelane_b32 v39, s95, 56
+; FLATSCR-NEXT:    v_writelane_b32 v39, s96, 57
+; FLATSCR-NEXT:    v_writelane_b32 v39, s97, 58
+; FLATSCR-NEXT:    v_writelane_b32 v39, s98, 59
+; FLATSCR-NEXT:    v_writelane_b32 v39, s99, 60
+; FLATSCR-NEXT:    v_writelane_b32 v39, s100, 61
+; FLATSCR-NEXT:    v_writelane_b32 v39, s101, 62
+; FLATSCR-NEXT:    s_add_i32 s32, s32, 8
+; FLATSCR-NEXT:    v_writelane_b32 v39, s102, 63
+; FLATSCR-NEXT:    ;;#ASMSTART
+; FLATSCR-NEXT:    ; clobber nonpreserved SGPRs and 64 CSRs
+; FLATSCR-NEXT:    ;;#ASMEND
+; FLATSCR-NEXT:    ;;#ASMSTART
+; FLATSCR-NEXT:    ; clobber all VGPRs except CSR v40
+; FLATSCR-NEXT:    ;;#ASMEND
+; FLATSCR-NEXT:    v_readlane_b32 s102, v39, 63
+; FLATSCR-NEXT:    v_readlane_b32 s101, v39, 62
+; FLATSCR-NEXT:    v_readlane_b32 s100, v39, 61
+; FLATSCR-NEXT:    v_readlane_b32 s99, v39, 60
+; FLATSCR-NEXT:    v_readlane_b32 s98, v39, 59
+; FLATSCR-NEXT:    v_readlane_b32 s97, v39, 58
+; FLATSCR-NEXT:    v_readlane_b32 s96, v39, 57
+; FLATSCR-NEXT:    v_readlane_b32 s95, v39, 56
+; FLATSCR-NEXT:    v_readlane_b32 s94, v39, 55
+; FLATSCR-NEXT:    v_readlane_b32 s93, v39, 54
+; FLATSCR-NEXT:    v_readlane_b32 s92, v39, 53
+; FLATSCR-NEXT:    v_readlane_b32 s91, v39, 52
+; FLATSCR-NEXT:    v_readlane_b32 s90, v39, 51
+; FLATSCR-NEXT:    v_readlane_b32 s89, v39, 50
+; FLATSCR-NEXT:    v_readlane_b32 s88, v39, 49
+; FLATSCR-NEXT:    v_readlane_b32 s87, v39, 48
+; FLATSCR-NEXT:    v_readlane_b32 s86, v39, 47
+; FLATSCR-NEXT:    v_readlane_b32 s85, v39, 46
+; FLATSCR-NEXT:    v_readlane_b32 s84, v39, 45
+; FLATSCR-NEXT:    v_readlane_b32 s83, v39, 44
+; FLATSCR-NEXT:    v_readlane_b32 s82, v39, 43
+; FLATSCR-NEXT:    v_readlane_b32 s81, v39, 42
+; FLATSCR-NEXT:    v_readlane_b32 s80, v39, 41
+; FLATSCR-NEXT:    v_readlane_b32 s79, v39, 40
+; FLATSCR-NEXT:    v_readlane_b32 s78, v39, 39
+; FLATSCR-NEXT:    v_readlane_b32 s77, v39, 38
+; FLATSCR-NEXT:    v_readlane_b32 s76, v39, 37
+; FLATSCR-NEXT:    v_readlane_b32 s75, v39, 36
+; FLATSCR-NEXT:    v_readlane_b32 s74, v39, 35
+; FLATSCR-NEXT:    v_readlane_b32 s73, v39, 34
+; FLATSCR-NEXT:    v_readlane_b32 s72, v39, 33
+; FLATSCR-NEXT:    v_readlane_b32 s71, v39, 32
+; FLATSCR-NEXT:    v_readlane_b32 s70, v39, 31
+; FLATSCR-NEXT:    v_readlane_b32 s69, v39, 30
+; FLATSCR-NEXT:    v_readlane_b32 s68, v39, 29
+; FLATSCR-NEXT:    v_readlane_b32 s67, v39, 28
+; FLATSCR-NEXT:    v_readlane_b32 s66, v39, 27
+; FLATSCR-NEXT:    v_readlane_b32 s65, v39, 26
+; FLATSCR-NEXT:    v_readlane_b32 s64, v39, 25
+; FLATSCR-NEXT:    v_readlane_b32 s63, v39, 24
+; FLATSCR-NEXT:    v_readlane_b32 s62, v39, 23
+; FLATSCR-NEXT:    v_readlane_b32 s61, v39, 22
+; FLATSCR-NEXT:    v_readlane_b32 s60, v39, 21
+; FLATSCR-NEXT:    v_readlane_b32 s59, v39, 20
+; FLATSCR-NEXT:    v_readlane_b32 s58, v39, 19
+; FLATSCR-NEXT:    v_readlane_b32 s57, v39, 18
+; FLATSCR-NEXT:    v_readlane_b32 s56, v39, 17
+; FLATSCR-NEXT:    v_readlane_b32 s55, v39, 16
+; FLATSCR-NEXT:    v_readlane_b32 s54, v39, 15
+; FLATSCR-NEXT:    v_readlane_b32 s53, v39, 14
+; FLATSCR-NEXT:    v_readlane_b32 s52, v39, 13
+; FLATSCR-NEXT:    v_readlane_b32 s51, v39, 12
+; FLATSCR-NEXT:    v_readlane_b32 s50, v39, 11
+; FLATSCR-NEXT:    v_readlane_b32 s49, v39, 10
+; FLATSCR-NEXT:    v_readlane_b32 s48, v39, 9
+; FLATSCR-NEXT:    v_readlane_b32 s47, v39, 8
+; FLATSCR-NEXT:    v_readlane_b32 s46, v39, 7
+; FLATSCR-NEXT:    v_readlane_b32 s45, v39, 6
+; FLATSCR-NEXT:    v_readlane_b32 s44, v39, 5
+; FLATSCR-NEXT:    v_readlane_b32 s43, v39, 4
+; FLATSCR-NEXT:    v_readlane_b32 s42, v39, 3
+; FLATSCR-NEXT:    v_readlane_b32 s41, v39, 2
+; FLATSCR-NEXT:    v_readlane_b32 s40, v39, 1
+; FLATSCR-NEXT:    v_readlane_b32 s39, v39, 0
+; FLATSCR-NEXT:    s_mov_b32 s32, s33
+; FLATSCR-NEXT:    s_xor_saveexec_b64 s[2:3], -1
+; FLATSCR-NEXT:    scratch_load_dword v39, off, s33 ; 4-byte Folded Reload
+; FLATSCR-NEXT:    s_mov_b64 exec, s[2:3]
+; FLATSCR-NEXT:    s_mov_b32 s33, s0
+; FLATSCR-NEXT:    s_waitcnt vmcnt(0)
+; FLATSCR-NEXT:    s_setpc_b64 s[30:31]
   call void asm sideeffect "; clobber nonpreserved SGPRs and 64 CSRs",
     "~{s4},~{s5},~{s6},~{s7},~{s8},~{s9}
     ,~{s10},~{s11},~{s12},~{s13},~{s14},~{s15},~{s16},~{s17},~{s18},~{s19}
@@ -619,17 +2029,312 @@ define void @callee_need_to_spill_fp_to_memory_full_reserved_vgpr() #3 {
 ; the exec register is saved to s0 when saving CSR in the function prolog.
 ; Make sure that the FP save happens after restoring exec from the same
 ; register.
-; GCN-LABEL: {{^}}callee_need_to_spill_fp_to_reg:
-; FLATSCR: s_mov_b32 [[FP_SCRATCH_COPY:s[0-9]+]], s33
-; FLATSCR: s_mov_b32 s33, s32
-; GCN-NOT: v_writelane_b32 v40, s33
-; FLATSCR: s_or_saveexec_b64 [[COPY_EXEC0:s\[[0-9]+:[0-9]+\]]], -1{{$}}
-; FLATSCR: s_mov_b64 exec, [[COPY_EXEC0]]
-; FLATSCR: s_or_saveexec_b64 [[COPY_EXEC1:s\[[0-9]+:[0-9]+\]]], -1{{$}}
-; GCN-NOT: v_readlane_b32 s33, v40
-; FLATSCR: s_mov_b32 s33, [[FP_SCRATCH_COPY]]
-; GCN:     s_setpc_b64
 define void @callee_need_to_spill_fp_to_reg() #1 {
+; MUBUF-LABEL: callee_need_to_spill_fp_to_reg:
+; MUBUF:       ; %bb.0:
+; MUBUF-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; MUBUF-NEXT:    s_mov_b32 s4, s33
+; MUBUF-NEXT:    s_mov_b32 s33, s32
+; MUBUF-NEXT:    s_or_saveexec_b64 s[6:7], -1
+; MUBUF-NEXT:    buffer_store_dword v40, off, s[0:3], s33 ; 4-byte Folded Spill
+; MUBUF-NEXT:    buffer_store_dword v41, off, s[0:3], s33 offset:4 ; 4-byte Folded Spill
+; MUBUF-NEXT:    s_mov_b64 exec, s[6:7]
+; MUBUF-NEXT:    v_writelane_b32 v40, s39, 0
+; MUBUF-NEXT:    v_writelane_b32 v40, s40, 1
+; MUBUF-NEXT:    v_writelane_b32 v40, s41, 2
+; MUBUF-NEXT:    v_writelane_b32 v40, s42, 3
+; MUBUF-NEXT:    v_writelane_b32 v40, s43, 4
+; MUBUF-NEXT:    v_writelane_b32 v40, s44, 5
+; MUBUF-NEXT:    v_writelane_b32 v40, s45, 6
+; MUBUF-NEXT:    v_writelane_b32 v40, s46, 7
+; MUBUF-NEXT:    v_writelane_b32 v40, s47, 8
+; MUBUF-NEXT:    v_writelane_b32 v40, s48, 9
+; MUBUF-NEXT:    v_writelane_b32 v40, s49, 10
+; MUBUF-NEXT:    v_writelane_b32 v40, s50, 11
+; MUBUF-NEXT:    v_writelane_b32 v40, s51, 12
+; MUBUF-NEXT:    v_writelane_b32 v40, s52, 13
+; MUBUF-NEXT:    v_writelane_b32 v40, s53, 14
+; MUBUF-NEXT:    v_writelane_b32 v40, s54, 15
+; MUBUF-NEXT:    v_writelane_b32 v40, s55, 16
+; MUBUF-NEXT:    v_writelane_b32 v40, s56, 17
+; MUBUF-NEXT:    v_writelane_b32 v40, s57, 18
+; MUBUF-NEXT:    v_writelane_b32 v40, s58, 19
+; MUBUF-NEXT:    v_writelane_b32 v40, s59, 20
+; MUBUF-NEXT:    v_writelane_b32 v40, s60, 21
+; MUBUF-NEXT:    v_writelane_b32 v40, s61, 22
+; MUBUF-NEXT:    v_writelane_b32 v40, s62, 23
+; MUBUF-NEXT:    v_writelane_b32 v40, s63, 24
+; MUBUF-NEXT:    v_writelane_b32 v40, s64, 25
+; MUBUF-NEXT:    v_writelane_b32 v40, s65, 26
+; MUBUF-NEXT:    v_writelane_b32 v40, s66, 27
+; MUBUF-NEXT:    v_writelane_b32 v40, s67, 28
+; MUBUF-NEXT:    v_writelane_b32 v40, s68, 29
+; MUBUF-NEXT:    v_writelane_b32 v40, s69, 30
+; MUBUF-NEXT:    v_writelane_b32 v40, s70, 31
+; MUBUF-NEXT:    v_writelane_b32 v40, s71, 32
+; MUBUF-NEXT:    v_writelane_b32 v40, s72, 33
+; MUBUF-NEXT:    v_writelane_b32 v40, s73, 34
+; MUBUF-NEXT:    v_writelane_b32 v40, s74, 35
+; MUBUF-NEXT:    v_writelane_b32 v40, s75, 36
+; MUBUF-NEXT:    v_writelane_b32 v40, s76, 37
+; MUBUF-NEXT:    v_writelane_b32 v40, s77, 38
+; MUBUF-NEXT:    v_writelane_b32 v40, s78, 39
+; MUBUF-NEXT:    v_writelane_b32 v40, s79, 40
+; MUBUF-NEXT:    v_writelane_b32 v40, s80, 41
+; MUBUF-NEXT:    v_writelane_b32 v40, s81, 42
+; MUBUF-NEXT:    v_writelane_b32 v40, s82, 43
+; MUBUF-NEXT:    v_writelane_b32 v40, s83, 44
+; MUBUF-NEXT:    v_writelane_b32 v40, s84, 45
+; MUBUF-NEXT:    v_writelane_b32 v40, s85, 46
+; MUBUF-NEXT:    v_writelane_b32 v40, s86, 47
+; MUBUF-NEXT:    v_writelane_b32 v40, s87, 48
+; MUBUF-NEXT:    v_writelane_b32 v40, s88, 49
+; MUBUF-NEXT:    v_writelane_b32 v40, s89, 50
+; MUBUF-NEXT:    v_writelane_b32 v40, s90, 51
+; MUBUF-NEXT:    v_writelane_b32 v40, s91, 52
+; MUBUF-NEXT:    v_writelane_b32 v40, s92, 53
+; MUBUF-NEXT:    v_writelane_b32 v40, s93, 54
+; MUBUF-NEXT:    v_writelane_b32 v40, s94, 55
+; MUBUF-NEXT:    v_writelane_b32 v40, s95, 56
+; MUBUF-NEXT:    v_writelane_b32 v40, s96, 57
+; MUBUF-NEXT:    v_writelane_b32 v40, s97, 58
+; MUBUF-NEXT:    v_writelane_b32 v40, s98, 59
+; MUBUF-NEXT:    v_writelane_b32 v40, s99, 60
+; MUBUF-NEXT:    v_writelane_b32 v40, s100, 61
+; MUBUF-NEXT:    v_writelane_b32 v40, s101, 62
+; MUBUF-NEXT:    v_writelane_b32 v41, s4, 0
+; MUBUF-NEXT:    s_addk_i32 s32, 0x300
+; MUBUF-NEXT:    v_writelane_b32 v40, s102, 63
+; MUBUF-NEXT:    ;;#ASMSTART
+; MUBUF-NEXT:    ; clobber nonpreserved SGPRs and 64 CSRs
+; MUBUF-NEXT:    ;;#ASMEND
+; MUBUF-NEXT:    ;;#ASMSTART
+; MUBUF-NEXT:    ; clobber all VGPRs except CSR v40
+; MUBUF-NEXT:    ;;#ASMEND
+; MUBUF-NEXT:    v_readlane_b32 s102, v40, 63
+; MUBUF-NEXT:    v_readlane_b32 s101, v40, 62
+; MUBUF-NEXT:    v_readlane_b32 s100, v40, 61
+; MUBUF-NEXT:    v_readlane_b32 s99, v40, 60
+; MUBUF-NEXT:    v_readlane_b32 s98, v40, 59
+; MUBUF-NEXT:    v_readlane_b32 s97, v40, 58
+; MUBUF-NEXT:    v_readlane_b32 s96, v40, 57
+; MUBUF-NEXT:    v_readlane_b32 s95, v40, 56
+; MUBUF-NEXT:    v_readlane_b32 s94, v40, 55
+; MUBUF-NEXT:    v_readlane_b32 s93, v40, 54
+; MUBUF-NEXT:    v_readlane_b32 s92, v40, 53
+; MUBUF-NEXT:    v_readlane_b32 s91, v40, 52
+; MUBUF-NEXT:    v_readlane_b32 s90, v40, 51
+; MUBUF-NEXT:    v_readlane_b32 s89, v40, 50
+; MUBUF-NEXT:    v_readlane_b32 s88, v40, 49
+; MUBUF-NEXT:    v_readlane_b32 s87, v40, 48
+; MUBUF-NEXT:    v_readlane_b32 s86, v40, 47
+; MUBUF-NEXT:    v_readlane_b32 s85, v40, 46
+; MUBUF-NEXT:    v_readlane_b32 s84, v40, 45
+; MUBUF-NEXT:    v_readlane_b32 s83, v40, 44
+; MUBUF-NEXT:    v_readlane_b32 s82, v40, 43
+; MUBUF-NEXT:    v_readlane_b32 s81, v40, 42
+; MUBUF-NEXT:    v_readlane_b32 s80, v40, 41
+; MUBUF-NEXT:    v_readlane_b32 s79, v40, 40
+; MUBUF-NEXT:    v_readlane_b32 s78, v40, 39
+; MUBUF-NEXT:    v_readlane_b32 s77, v40, 38
+; MUBUF-NEXT:    v_readlane_b32 s76, v40, 37
+; MUBUF-NEXT:    v_readlane_b32 s75, v40, 36
+; MUBUF-NEXT:    v_readlane_b32 s74, v40, 35
+; MUBUF-NEXT:    v_readlane_b32 s73, v40, 34
+; MUBUF-NEXT:    v_readlane_b32 s72, v40, 33
+; MUBUF-NEXT:    v_readlane_b32 s71, v40, 32
+; MUBUF-NEXT:    v_readlane_b32 s70, v40, 31
+; MUBUF-NEXT:    v_readlane_b32 s69, v40, 30
+; MUBUF-NEXT:    v_readlane_b32 s68, v40, 29
+; MUBUF-NEXT:    v_readlane_b32 s67, v40, 28
+; MUBUF-NEXT:    v_readlane_b32 s66, v40, 27
+; MUBUF-NEXT:    v_readlane_b32 s65, v40, 26
+; MUBUF-NEXT:    v_readlane_b32 s64, v40, 25
+; MUBUF-NEXT:    v_readlane_b32 s63, v40, 24
+; MUBUF-NEXT:    v_readlane_b32 s62, v40, 23
+; MUBUF-NEXT:    v_readlane_b32 s61, v40, 22
+; MUBUF-NEXT:    v_readlane_b32 s60, v40, 21
+; MUBUF-NEXT:    v_readlane_b32 s59, v40, 20
+; MUBUF-NEXT:    v_readlane_b32 s58, v40, 19
+; MUBUF-NEXT:    v_readlane_b32 s57, v40, 18
+; MUBUF-NEXT:    v_readlane_b32 s56, v40, 17
+; MUBUF-NEXT:    v_readlane_b32 s55, v40, 16
+; MUBUF-NEXT:    v_readlane_b32 s54, v40, 15
+; MUBUF-NEXT:    v_readlane_b32 s53, v40, 14
+; MUBUF-NEXT:    v_readlane_b32 s52, v40, 13
+; MUBUF-NEXT:    v_readlane_b32 s51, v40, 12
+; MUBUF-NEXT:    v_readlane_b32 s50, v40, 11
+; MUBUF-NEXT:    v_readlane_b32 s49, v40, 10
+; MUBUF-NEXT:    v_readlane_b32 s48, v40, 9
+; MUBUF-NEXT:    v_readlane_b32 s47, v40, 8
+; MUBUF-NEXT:    v_readlane_b32 s46, v40, 7
+; MUBUF-NEXT:    v_readlane_b32 s45, v40, 6
+; MUBUF-NEXT:    v_readlane_b32 s44, v40, 5
+; MUBUF-NEXT:    v_readlane_b32 s43, v40, 4
+; MUBUF-NEXT:    v_readlane_b32 s42, v40, 3
+; MUBUF-NEXT:    v_readlane_b32 s41, v40, 2
+; MUBUF-NEXT:    v_readlane_b32 s40, v40, 1
+; MUBUF-NEXT:    v_readlane_b32 s39, v40, 0
+; MUBUF-NEXT:    s_mov_b32 s32, s33
+; MUBUF-NEXT:    v_readlane_b32 s4, v41, 0
+; MUBUF-NEXT:    s_or_saveexec_b64 s[6:7], -1
+; MUBUF-NEXT:    buffer_load_dword v40, off, s[0:3], s33 ; 4-byte Folded Reload
+; MUBUF-NEXT:    buffer_load_dword v41, off, s[0:3], s33 offset:4 ; 4-byte Folded Reload
+; MUBUF-NEXT:    s_mov_b64 exec, s[6:7]
+; MUBUF-NEXT:    s_mov_b32 s33, s4
+; MUBUF-NEXT:    s_waitcnt vmcnt(0)
+; MUBUF-NEXT:    s_setpc_b64 s[30:31]
+;
+; FLATSCR-LABEL: callee_need_to_spill_fp_to_reg:
+; FLATSCR:       ; %bb.0:
+; FLATSCR-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; FLATSCR-NEXT:    s_mov_b32 s0, s33
+; FLATSCR-NEXT:    s_mov_b32 s33, s32
+; FLATSCR-NEXT:    s_or_saveexec_b64 s[2:3], -1
+; FLATSCR-NEXT:    scratch_store_dword off, v40, s33 ; 4-byte Folded Spill
+; FLATSCR-NEXT:    s_mov_b64 exec, s[2:3]
+; FLATSCR-NEXT:    v_writelane_b32 v40, s39, 0
+; FLATSCR-NEXT:    v_writelane_b32 v40, s40, 1
+; FLATSCR-NEXT:    v_writelane_b32 v40, s41, 2
+; FLATSCR-NEXT:    v_writelane_b32 v40, s42, 3
+; FLATSCR-NEXT:    v_writelane_b32 v40, s43, 4
+; FLATSCR-NEXT:    v_writelane_b32 v40, s44, 5
+; FLATSCR-NEXT:    v_writelane_b32 v40, s45, 6
+; FLATSCR-NEXT:    v_writelane_b32 v40, s46, 7
+; FLATSCR-NEXT:    v_writelane_b32 v40, s47, 8
+; FLATSCR-NEXT:    v_writelane_b32 v40, s48, 9
+; FLATSCR-NEXT:    v_writelane_b32 v40, s49, 10
+; FLATSCR-NEXT:    v_writelane_b32 v40, s50, 11
+; FLATSCR-NEXT:    v_writelane_b32 v40, s51, 12
+; FLATSCR-NEXT:    v_writelane_b32 v40, s52, 13
+; FLATSCR-NEXT:    v_writelane_b32 v40, s53, 14
+; FLATSCR-NEXT:    v_writelane_b32 v40, s54, 15
+; FLATSCR-NEXT:    v_writelane_b32 v40, s55, 16
+; FLATSCR-NEXT:    v_writelane_b32 v40, s56, 17
+; FLATSCR-NEXT:    v_writelane_b32 v40, s57, 18
+; FLATSCR-NEXT:    v_writelane_b32 v40, s58, 19
+; FLATSCR-NEXT:    v_writelane_b32 v40, s59, 20
+; FLATSCR-NEXT:    v_writelane_b32 v40, s60, 21
+; FLATSCR-NEXT:    v_writelane_b32 v40, s61, 22
+; FLATSCR-NEXT:    v_writelane_b32 v40, s62, 23
+; FLATSCR-NEXT:    v_writelane_b32 v40, s63, 24
+; FLATSCR-NEXT:    v_writelane_b32 v40, s64, 25
+; FLATSCR-NEXT:    v_writelane_b32 v40, s65, 26
+; FLATSCR-NEXT:    v_writelane_b32 v40, s66, 27
+; FLATSCR-NEXT:    v_writelane_b32 v40, s67, 28
+; FLATSCR-NEXT:    v_writelane_b32 v40, s68, 29
+; FLATSCR-NEXT:    v_writelane_b32 v40, s69, 30
+; FLATSCR-NEXT:    v_writelane_b32 v40, s70, 31
+; FLATSCR-NEXT:    v_writelane_b32 v40, s71, 32
+; FLATSCR-NEXT:    v_writelane_b32 v40, s72, 33
+; FLATSCR-NEXT:    v_writelane_b32 v40, s73, 34
+; FLATSCR-NEXT:    v_writelane_b32 v40, s74, 35
+; FLATSCR-NEXT:    v_writelane_b32 v40, s75, 36
+; FLATSCR-NEXT:    v_writelane_b32 v40, s76, 37
+; FLATSCR-NEXT:    v_writelane_b32 v40, s77, 38
+; FLATSCR-NEXT:    v_writelane_b32 v40, s78, 39
+; FLATSCR-NEXT:    v_writelane_b32 v40, s79, 40
+; FLATSCR-NEXT:    v_writelane_b32 v40, s80, 41
+; FLATSCR-NEXT:    v_writelane_b32 v40, s81, 42
+; FLATSCR-NEXT:    v_writelane_b32 v40, s82, 43
+; FLATSCR-NEXT:    v_writelane_b32 v40, s83, 44
+; FLATSCR-NEXT:    v_writelane_b32 v40, s84, 45
+; FLATSCR-NEXT:    v_writelane_b32 v40, s85, 46
+; FLATSCR-NEXT:    v_writelane_b32 v40, s86, 47
+; FLATSCR-NEXT:    v_writelane_b32 v40, s87, 48
+; FLATSCR-NEXT:    v_writelane_b32 v40, s88, 49
+; FLATSCR-NEXT:    v_writelane_b32 v40, s89, 50
+; FLATSCR-NEXT:    v_writelane_b32 v40, s90, 51
+; FLATSCR-NEXT:    v_writelane_b32 v40, s91, 52
+; FLATSCR-NEXT:    v_writelane_b32 v40, s92, 53
+; FLATSCR-NEXT:    v_writelane_b32 v40, s93, 54
+; FLATSCR-NEXT:    v_writelane_b32 v40, s94, 55
+; FLATSCR-NEXT:    v_writelane_b32 v40, s95, 56
+; FLATSCR-NEXT:    v_writelane_b32 v40, s96, 57
+; FLATSCR-NEXT:    v_writelane_b32 v40, s97, 58
+; FLATSCR-NEXT:    v_writelane_b32 v40, s98, 59
+; FLATSCR-NEXT:    v_writelane_b32 v40, s99, 60
+; FLATSCR-NEXT:    v_writelane_b32 v40, s100, 61
+; FLATSCR-NEXT:    v_writelane_b32 v40, s101, 62
+; FLATSCR-NEXT:    s_add_i32 s32, s32, 8
+; FLATSCR-NEXT:    v_writelane_b32 v40, s102, 63
+; FLATSCR-NEXT:    ;;#ASMSTART
+; FLATSCR-NEXT:    ; clobber nonpreserved SGPRs and 64 CSRs
+; FLATSCR-NEXT:    ;;#ASMEND
+; FLATSCR-NEXT:    ;;#ASMSTART
+; FLATSCR-NEXT:    ; clobber all VGPRs except CSR v40
+; FLATSCR-NEXT:    ;;#ASMEND
+; FLATSCR-NEXT:    v_readlane_b32 s102, v40, 63
+; FLATSCR-NEXT:    v_readlane_b32 s101, v40, 62
+; FLATSCR-NEXT:    v_readlane_b32 s100, v40, 61
+; FLATSCR-NEXT:    v_readlane_b32 s99, v40, 60
+; FLATSCR-NEXT:    v_readlane_b32 s98, v40, 59
+; FLATSCR-NEXT:    v_readlane_b32 s97, v40, 58
+; FLATSCR-NEXT:    v_readlane_b32 s96, v40, 57
+; FLATSCR-NEXT:    v_readlane_b32 s95, v40, 56
+; FLATSCR-NEXT:    v_readlane_b32 s94, v40, 55
+; FLATSCR-NEXT:    v_readlane_b32 s93, v40, 54
+; FLATSCR-NEXT:    v_readlane_b32 s92, v40, 53
+; FLATSCR-NEXT:    v_readlane_b32 s91, v40, 52
+; FLATSCR-NEXT:    v_readlane_b32 s90, v40, 51
+; FLATSCR-NEXT:    v_readlane_b32 s89, v40, 50
+; FLATSCR-NEXT:    v_readlane_b32 s88, v40, 49
+; FLATSCR-NEXT:    v_readlane_b32 s87, v40, 48
+; FLATSCR-NEXT:    v_readlane_b32 s86, v40, 47
+; FLATSCR-NEXT:    v_readlane_b32 s85, v40, 46
+; FLATSCR-NEXT:    v_readlane_b32 s84, v40, 45
+; FLATSCR-NEXT:    v_readlane_b32 s83, v40, 44
+; FLATSCR-NEXT:    v_readlane_b32 s82, v40, 43
+; FLATSCR-NEXT:    v_readlane_b32 s81, v40, 42
+; FLATSCR-NEXT:    v_readlane_b32 s80, v40, 41
+; FLATSCR-NEXT:    v_readlane_b32 s79, v40, 40
+; FLATSCR-NEXT:    v_readlane_b32 s78, v40, 39
+; FLATSCR-NEXT:    v_readlane_b32 s77, v40, 38
+; FLATSCR-NEXT:    v_readlane_b32 s76, v40, 37
+; FLATSCR-NEXT:    v_readlane_b32 s75, v40, 36
+; FLATSCR-NEXT:    v_readlane_b32 s74, v40, 35
+; FLATSCR-NEXT:    v_readlane_b32 s73, v40, 34
+; FLATSCR-NEXT:    v_readlane_b32 s72, v40, 33
+; FLATSCR-NEXT:    v_readlane_b32 s71, v40, 32
+; FLATSCR-NEXT:    v_readlane_b32 s70, v40, 31
+; FLATSCR-NEXT:    v_readlane_b32 s69, v40, 30
+; FLATSCR-NEXT:    v_readlane_b32 s68, v40, 29
+; FLATSCR-NEXT:    v_readlane_b32 s67, v40, 28
+; FLATSCR-NEXT:    v_readlane_b32 s66, v40, 27
+; FLATSCR-NEXT:    v_readlane_b32 s65, v40, 26
+; FLATSCR-NEXT:    v_readlane_b32 s64, v40, 25
+; FLATSCR-NEXT:    v_readlane_b32 s63, v40, 24
+; FLATSCR-NEXT:    v_readlane_b32 s62, v40, 23
+; FLATSCR-NEXT:    v_readlane_b32 s61, v40, 22
+; FLATSCR-NEXT:    v_readlane_b32 s60, v40, 21
+; FLATSCR-NEXT:    v_readlane_b32 s59, v40, 20
+; FLATSCR-NEXT:    v_readlane_b32 s58, v40, 19
+; FLATSCR-NEXT:    v_readlane_b32 s57, v40, 18
+; FLATSCR-NEXT:    v_readlane_b32 s56, v40, 17
+; FLATSCR-NEXT:    v_readlane_b32 s55, v40, 16
+; FLATSCR-NEXT:    v_readlane_b32 s54, v40, 15
+; FLATSCR-NEXT:    v_readlane_b32 s53, v40, 14
+; FLATSCR-NEXT:    v_readlane_b32 s52, v40, 13
+; FLATSCR-NEXT:    v_readlane_b32 s51, v40, 12
+; FLATSCR-NEXT:    v_readlane_b32 s50, v40, 11
+; FLATSCR-NEXT:    v_readlane_b32 s49, v40, 10
+; FLATSCR-NEXT:    v_readlane_b32 s48, v40, 9
+; FLATSCR-NEXT:    v_readlane_b32 s47, v40, 8
+; FLATSCR-NEXT:    v_readlane_b32 s46, v40, 7
+; FLATSCR-NEXT:    v_readlane_b32 s45, v40, 6
+; FLATSCR-NEXT:    v_readlane_b32 s44, v40, 5
+; FLATSCR-NEXT:    v_readlane_b32 s43, v40, 4
+; FLATSCR-NEXT:    v_readlane_b32 s42, v40, 3
+; FLATSCR-NEXT:    v_readlane_b32 s41, v40, 2
+; FLATSCR-NEXT:    v_readlane_b32 s40, v40, 1
+; FLATSCR-NEXT:    v_readlane_b32 s39, v40, 0
+; FLATSCR-NEXT:    s_mov_b32 s32, s33
+; FLATSCR-NEXT:    s_or_saveexec_b64 s[2:3], -1
+; FLATSCR-NEXT:    scratch_load_dword v40, off, s33 ; 4-byte Folded Reload
+; FLATSCR-NEXT:    s_mov_b64 exec, s[2:3]
+; FLATSCR-NEXT:    s_mov_b32 s33, s0
+; FLATSCR-NEXT:    s_waitcnt vmcnt(0)
+; FLATSCR-NEXT:    s_setpc_b64 s[30:31]
   call void asm sideeffect "; clobber nonpreserved SGPRs and 64 CSRs",
     "~{s4},~{s5},~{s6},~{s7},~{s8},~{s9}
     ,~{s10},~{s11},~{s12},~{s13},~{s14},~{s15},~{s16},~{s17},~{s18},~{s19}
@@ -652,20 +2357,327 @@ define void @callee_need_to_spill_fp_to_reg() #1 {
 
 ; If the size of the offset exceeds the MUBUF offset field we need another
 ; scratch VGPR to hold the offset.
-; GCN-LABEL: {{^}}spill_fp_to_memory_scratch_reg_needed_mubuf_offset
-; MUBUF: s_mov_b32 [[FP_SCRATCH_COPY:s[0-9]+]], s33
-; MUBUF-NEXT: s_mov_b32 s33, s32
-; MUBUF-NEXT: s_xor_saveexec_b64 s[6:7], -1
-; MUBUF-NEXT: s_add_i32 [[SCRATCH_SGPR:s[0-9]+]], s33, 0x40100
-; MUBUF-NEXT: buffer_store_dword v39, off, s[0:3], [[SCRATCH_SGPR]] ; 4-byte Folded Spill
-; MUBUF: v_mov_b32_e32 v0, [[FP_SCRATCH_COPY]]
-; GCN-NOT: v_mov_b32_e32 v0, 0x100c
-; MUBUF-NEXT: s_add_i32 [[SCRATCH_SGPR:s[0-9]+]], s33, 0x40200
-; MUBUF: buffer_store_dword v0, off, s[0:3], [[SCRATCH_SGPR]] ; 4-byte Folded Spill
-; FLATSCR: v_mov_b32_e32 v0, 0
-; FLATSCR: s_add_i32 [[SOFF:s[0-9]+]], s33, 0x1000
-; FLATSCR: scratch_store_dword off, v0, [[SOFF]]
 define void @spill_fp_to_memory_scratch_reg_needed_mubuf_offset(ptr addrspace(5) byval([4096 x i8]) align 4 %arg) #3 {
+; MUBUF-LABEL: spill_fp_to_memory_scratch_reg_needed_mubuf_offset:
+; MUBUF:       ; %bb.0:
+; MUBUF-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; MUBUF-NEXT:    s_mov_b32 s4, s33
+; MUBUF-NEXT:    s_mov_b32 s33, s32
+; MUBUF-NEXT:    s_xor_saveexec_b64 s[6:7], -1
+; MUBUF-NEXT:    s_add_i32 s5, s33, 0x40100
+; MUBUF-NEXT:    buffer_store_dword v39, off, s[0:3], s5 ; 4-byte Folded Spill
+; MUBUF-NEXT:    s_mov_b64 exec, s[6:7]
+; MUBUF-NEXT:    v_writelane_b32 v39, s39, 0
+; MUBUF-NEXT:    v_writelane_b32 v39, s40, 1
+; MUBUF-NEXT:    v_writelane_b32 v39, s41, 2
+; MUBUF-NEXT:    v_writelane_b32 v39, s42, 3
+; MUBUF-NEXT:    v_writelane_b32 v39, s43, 4
+; MUBUF-NEXT:    v_writelane_b32 v39, s44, 5
+; MUBUF-NEXT:    v_writelane_b32 v39, s45, 6
+; MUBUF-NEXT:    v_writelane_b32 v39, s46, 7
+; MUBUF-NEXT:    v_writelane_b32 v39, s47, 8
+; MUBUF-NEXT:    v_writelane_b32 v39, s48, 9
+; MUBUF-NEXT:    v_writelane_b32 v39, s49, 10
+; MUBUF-NEXT:    v_writelane_b32 v39, s50, 11
+; MUBUF-NEXT:    v_writelane_b32 v39, s51, 12
+; MUBUF-NEXT:    v_writelane_b32 v39, s52, 13
+; MUBUF-NEXT:    v_writelane_b32 v39, s53, 14
+; MUBUF-NEXT:    v_writelane_b32 v39, s54, 15
+; MUBUF-NEXT:    v_writelane_b32 v39, s55, 16
+; MUBUF-NEXT:    v_writelane_b32 v39, s56, 17
+; MUBUF-NEXT:    v_writelane_b32 v39, s57, 18
+; MUBUF-NEXT:    v_writelane_b32 v39, s58, 19
+; MUBUF-NEXT:    v_writelane_b32 v39, s59, 20
+; MUBUF-NEXT:    v_writelane_b32 v39, s60, 21
+; MUBUF-NEXT:    v_writelane_b32 v39, s61, 22
+; MUBUF-NEXT:    v_writelane_b32 v39, s62, 23
+; MUBUF-NEXT:    v_writelane_b32 v39, s63, 24
+; MUBUF-NEXT:    v_writelane_b32 v39, s64, 25
+; MUBUF-NEXT:    v_writelane_b32 v39, s65, 26
+; MUBUF-NEXT:    v_writelane_b32 v39, s66, 27
+; MUBUF-NEXT:    v_writelane_b32 v39, s67, 28
+; MUBUF-NEXT:    v_writelane_b32 v39, s68, 29
+; MUBUF-NEXT:    v_writelane_b32 v39, s69, 30
+; MUBUF-NEXT:    v_writelane_b32 v39, s70, 31
+; MUBUF-NEXT:    v_writelane_b32 v39, s71, 32
+; MUBUF-NEXT:    v_writelane_b32 v39, s72, 33
+; MUBUF-NEXT:    v_writelane_b32 v39, s73, 34
+; MUBUF-NEXT:    v_writelane_b32 v39, s74, 35
+; MUBUF-NEXT:    v_writelane_b32 v39, s75, 36
+; MUBUF-NEXT:    v_writelane_b32 v39, s76, 37
+; MUBUF-NEXT:    v_writelane_b32 v39, s77, 38
+; MUBUF-NEXT:    v_writelane_b32 v39, s78, 39
+; MUBUF-NEXT:    v_writelane_b32 v39, s79, 40
+; MUBUF-NEXT:    v_writelane_b32 v39, s80, 41
+; MUBUF-NEXT:    v_writelane_b32 v39, s81, 42
+; MUBUF-NEXT:    v_writelane_b32 v39, s82, 43
+; MUBUF-NEXT:    v_writelane_b32 v39, s83, 44
+; MUBUF-NEXT:    v_writelane_b32 v39, s84, 45
+; MUBUF-NEXT:    v_writelane_b32 v39, s85, 46
+; MUBUF-NEXT:    v_writelane_b32 v39, s86, 47
+; MUBUF-NEXT:    v_writelane_b32 v39, s87, 48
+; MUBUF-NEXT:    v_writelane_b32 v39, s88, 49
+; MUBUF-NEXT:    v_writelane_b32 v39, s89, 50
+; MUBUF-NEXT:    v_writelane_b32 v39, s90, 51
+; MUBUF-NEXT:    v_writelane_b32 v39, s91, 52
+; MUBUF-NEXT:    v_writelane_b32 v39, s92, 53
+; MUBUF-NEXT:    v_writelane_b32 v39, s93, 54
+; MUBUF-NEXT:    v_writelane_b32 v39, s94, 55
+; MUBUF-NEXT:    v_writelane_b32 v39, s95, 56
+; MUBUF-NEXT:    v_writelane_b32 v39, s96, 57
+; MUBUF-NEXT:    v_writelane_b32 v39, s97, 58
+; MUBUF-NEXT:    v_writelane_b32 v39, s98, 59
+; MUBUF-NEXT:    v_writelane_b32 v39, s99, 60
+; MUBUF-NEXT:    v_mov_b32_e32 v0, s4
+; MUBUF-NEXT:    s_add_i32 s5, s33, 0x40200
+; MUBUF-NEXT:    v_writelane_b32 v39, s100, 61
+; MUBUF-NEXT:    buffer_store_dword v0, off, s[0:3], s5 ; 4-byte Folded Spill
+; MUBUF-NEXT:    v_writelane_b32 v39, s101, 62
+; MUBUF-NEXT:    v_mov_b32_e32 v0, 0
+; MUBUF-NEXT:    v_mov_b32_e32 v1, 0x1000
+; MUBUF-NEXT:    v_writelane_b32 v39, s102, 63
+; MUBUF-NEXT:    buffer_store_dword v0, v1, s[0:3], s33 offen
+; MUBUF-NEXT:    s_waitcnt vmcnt(0)
+; MUBUF-NEXT:    ;;#ASMSTART
+; MUBUF-NEXT:    ; clobber nonpreserved SGPRs and 64 CSRs
+; MUBUF-NEXT:    ;;#ASMEND
+; MUBUF-NEXT:    ;;#ASMSTART
+; MUBUF-NEXT:    ; clobber all VGPRs except CSR v40
+; MUBUF-NEXT:    ;;#ASMEND
+; MUBUF-NEXT:    s_add_i32 s5, s33, 0x40200
+; MUBUF-NEXT:    buffer_load_dword v0, off, s[0:3], s5 ; 4-byte Folded Reload
+; MUBUF-NEXT:    s_add_i32 s32, s32, 0x40400
+; MUBUF-NEXT:    v_readlane_b32 s102, v39, 63
+; MUBUF-NEXT:    v_readlane_b32 s101, v39, 62
+; MUBUF-NEXT:    v_readlane_b32 s100, v39, 61
+; MUBUF-NEXT:    v_readlane_b32 s99, v39, 60
+; MUBUF-NEXT:    v_readlane_b32 s98, v39, 59
+; MUBUF-NEXT:    v_readlane_b32 s97, v39, 58
+; MUBUF-NEXT:    v_readlane_b32 s96, v39, 57
+; MUBUF-NEXT:    v_readlane_b32 s95, v39, 56
+; MUBUF-NEXT:    v_readlane_b32 s94, v39, 55
+; MUBUF-NEXT:    v_readlane_b32 s93, v39, 54
+; MUBUF-NEXT:    v_readlane_b32 s92, v39, 53
+; MUBUF-NEXT:    v_readlane_b32 s91, v39, 52
+; MUBUF-NEXT:    v_readlane_b32 s90, v39, 51
+; MUBUF-NEXT:    v_readlane_b32 s89, v39, 50
+; MUBUF-NEXT:    v_readlane_b32 s88, v39, 49
+; MUBUF-NEXT:    v_readlane_b32 s87, v39, 48
+; MUBUF-NEXT:    v_readlane_b32 s86, v39, 47
+; MUBUF-NEXT:    v_readlane_b32 s85, v39, 46
+; MUBUF-NEXT:    v_readlane_b32 s84, v39, 45
+; MUBUF-NEXT:    v_readlane_b32 s83, v39, 44
+; MUBUF-NEXT:    v_readlane_b32 s82, v39, 43
+; MUBUF-NEXT:    v_readlane_b32 s81, v39, 42
+; MUBUF-NEXT:    v_readlane_b32 s80, v39, 41
+; MUBUF-NEXT:    v_readlane_b32 s79, v39, 40
+; MUBUF-NEXT:    v_readlane_b32 s78, v39, 39
+; MUBUF-NEXT:    v_readlane_b32 s77, v39, 38
+; MUBUF-NEXT:    v_readlane_b32 s76, v39, 37
+; MUBUF-NEXT:    v_readlane_b32 s75, v39, 36
+; MUBUF-NEXT:    v_readlane_b32 s74, v39, 35
+; MUBUF-NEXT:    v_readlane_b32 s73, v39, 34
+; MUBUF-NEXT:    v_readlane_b32 s72, v39, 33
+; MUBUF-NEXT:    v_readlane_b32 s71, v39, 32
+; MUBUF-NEXT:    v_readlane_b32 s70, v39, 31
+; MUBUF-NEXT:    v_readlane_b32 s69, v39, 30
+; MUBUF-NEXT:    v_readlane_b32 s68, v39, 29
+; MUBUF-NEXT:    v_readlane_b32 s67, v39, 28
+; MUBUF-NEXT:    v_readlane_b32 s66, v39, 27
+; MUBUF-NEXT:    v_readlane_b32 s65, v39, 26
+; MUBUF-NEXT:    v_readlane_b32 s64, v39, 25
+; MUBUF-NEXT:    v_readlane_b32 s63, v39, 24
+; MUBUF-NEXT:    v_readlane_b32 s62, v39, 23
+; MUBUF-NEXT:    v_readlane_b32 s61, v39, 22
+; MUBUF-NEXT:    v_readlane_b32 s60, v39, 21
+; MUBUF-NEXT:    v_readlane_b32 s59, v39, 20
+; MUBUF-NEXT:    v_readlane_b32 s58, v39, 19
+; MUBUF-NEXT:    v_readlane_b32 s57, v39, 18
+; MUBUF-NEXT:    v_readlane_b32 s56, v39, 17
+; MUBUF-NEXT:    v_readlane_b32 s55, v39, 16
+; MUBUF-NEXT:    v_readlane_b32 s54, v39, 15
+; MUBUF-NEXT:    v_readlane_b32 s53, v39, 14
+; MUBUF-NEXT:    v_readlane_b32 s52, v39, 13
+; MUBUF-NEXT:    v_readlane_b32 s51, v39, 12
+; MUBUF-NEXT:    v_readlane_b32 s50, v39, 11
+; MUBUF-NEXT:    v_readlane_b32 s49, v39, 10
+; MUBUF-NEXT:    v_readlane_b32 s48, v39, 9
+; MUBUF-NEXT:    v_readlane_b32 s47, v39, 8
+; MUBUF-NEXT:    v_readlane_b32 s46, v39, 7
+; MUBUF-NEXT:    v_readlane_b32 s45, v39, 6
+; MUBUF-NEXT:    v_readlane_b32 s44, v39, 5
+; MUBUF-NEXT:    v_readlane_b32 s43, v39, 4
+; MUBUF-NEXT:    v_readlane_b32 s42, v39, 3
+; MUBUF-NEXT:    v_readlane_b32 s41, v39, 2
+; MUBUF-NEXT:    v_readlane_b32 s40, v39, 1
+; MUBUF-NEXT:    v_readlane_b32 s39, v39, 0
+; MUBUF-NEXT:    s_mov_b32 s32, s33
+; MUBUF-NEXT:    s_waitcnt vmcnt(0)
+; MUBUF-NEXT:    v_readfirstlane_b32 s4, v0
+; MUBUF-NEXT:    s_xor_saveexec_b64 s[6:7], -1
+; MUBUF-NEXT:    s_add_i32 s5, s33, 0x40100
+; MUBUF-NEXT:    buffer_load_dword v39, off, s[0:3], s5 ; 4-byte Folded Reload
+; MUBUF-NEXT:    s_mov_b64 exec, s[6:7]
+; MUBUF-NEXT:    s_mov_b32 s33, s4
+; MUBUF-NEXT:    s_waitcnt vmcnt(0)
+; MUBUF-NEXT:    s_setpc_b64 s[30:31]
+;
+; FLATSCR-LABEL: spill_fp_to_memory_scratch_reg_needed_mubuf_offset:
+; FLATSCR:       ; %bb.0:
+; FLATSCR-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; FLATSCR-NEXT:    s_mov_b32 s0, s33
+; FLATSCR-NEXT:    s_mov_b32 s33, s32
+; FLATSCR-NEXT:    s_xor_saveexec_b64 s[2:3], -1
+; FLATSCR-NEXT:    s_add_i32 s1, s33, 0x1004
+; FLATSCR-NEXT:    scratch_store_dword off, v39, s1 ; 4-byte Folded Spill
+; FLATSCR-NEXT:    s_mov_b64 exec, s[2:3]
+; FLATSCR-NEXT:    v_writelane_b32 v39, s39, 0
+; FLATSCR-NEXT:    v_writelane_b32 v39, s40, 1
+; FLATSCR-NEXT:    v_writelane_b32 v39, s41, 2
+; FLATSCR-NEXT:    v_writelane_b32 v39, s42, 3
+; FLATSCR-NEXT:    v_writelane_b32 v39, s43, 4
+; FLATSCR-NEXT:    v_writelane_b32 v39, s44, 5
+; FLATSCR-NEXT:    v_writelane_b32 v39, s45, 6
+; FLATSCR-NEXT:    v_writelane_b32 v39, s46, 7
+; FLATSCR-NEXT:    v_writelane_b32 v39, s47, 8
+; FLATSCR-NEXT:    v_writelane_b32 v39, s48, 9
+; FLATSCR-NEXT:    v_writelane_b32 v39, s49, 10
+; FLATSCR-NEXT:    v_writelane_b32 v39, s50, 11
+; FLATSCR-NEXT:    v_writelane_b32 v39, s51, 12
+; FLATSCR-NEXT:    v_writelane_b32 v39, s52, 13
+; FLATSCR-NEXT:    v_writelane_b32 v39, s53, 14
+; FLATSCR-NEXT:    v_writelane_b32 v39, s54, 15
+; FLATSCR-NEXT:    v_writelane_b32 v39, s55, 16
+; FLATSCR-NEXT:    v_writelane_b32 v39, s56, 17
+; FLATSCR-NEXT:    v_writelane_b32 v39, s57, 18
+; FLATSCR-NEXT:    v_writelane_b32 v39, s58, 19
+; FLATSCR-NEXT:    v_writelane_b32 v39, s59, 20
+; FLATSCR-NEXT:    v_writelane_b32 v39, s60, 21
+; FLATSCR-NEXT:    v_writelane_b32 v39, s61, 22
+; FLATSCR-NEXT:    v_writelane_b32 v39, s62, 23
+; FLATSCR-NEXT:    v_writelane_b32 v39, s63, 24
+; FLATSCR-NEXT:    v_writelane_b32 v39, s64, 25
+; FLATSCR-NEXT:    v_writelane_b32 v39, s65, 26
+; FLATSCR-NEXT:    v_writelane_b32 v39, s66, 27
+; FLATSCR-NEXT:    v_writelane_b32 v39, s67, 28
+; FLATSCR-NEXT:    v_writelane_b32 v39, s68, 29
+; FLATSCR-NEXT:    v_writelane_b32 v39, s69, 30
+; FLATSCR-NEXT:    v_writelane_b32 v39, s70, 31
+; FLATSCR-NEXT:    v_writelane_b32 v39, s71, 32
+; FLATSCR-NEXT:    v_writelane_b32 v39, s72, 33
+; FLATSCR-NEXT:    v_writelane_b32 v39, s73, 34
+; FLATSCR-NEXT:    v_writelane_b32 v39, s74, 35
+; FLATSCR-NEXT:    v_writelane_b32 v39, s75, 36
+; FLATSCR-NEXT:    v_writelane_b32 v39, s76, 37
+; FLATSCR-NEXT:    v_writelane_b32 v39, s77, 38
+; FLATSCR-NEXT:    v_writelane_b32 v39, s78, 39
+; FLATSCR-NEXT:    v_writelane_b32 v39, s79, 40
+; FLATSCR-NEXT:    v_writelane_b32 v39, s80, 41
+; FLATSCR-NEXT:    v_writelane_b32 v39, s81, 42
+; FLATSCR-NEXT:    v_writelane_b32 v39, s82, 43
+; FLATSCR-NEXT:    v_writelane_b32 v39, s83, 44
+; FLATSCR-NEXT:    v_writelane_b32 v39, s84, 45
+; FLATSCR-NEXT:    v_writelane_b32 v39, s85, 46
+; FLATSCR-NEXT:    v_writelane_b32 v39, s86, 47
+; FLATSCR-NEXT:    v_writelane_b32 v39, s87, 48
+; FLATSCR-NEXT:    v_writelane_b32 v39, s88, 49
+; FLATSCR-NEXT:    v_writelane_b32 v39, s89, 50
+; FLATSCR-NEXT:    v_writelane_b32 v39, s90, 51
+; FLATSCR-NEXT:    v_writelane_b32 v39, s91, 52
+; FLATSCR-NEXT:    v_writelane_b32 v39, s92, 53
+; FLATSCR-NEXT:    v_writelane_b32 v39, s93, 54
+; FLATSCR-NEXT:    v_writelane_b32 v39, s94, 55
+; FLATSCR-NEXT:    v_writelane_b32 v39, s95, 56
+; FLATSCR-NEXT:    v_writelane_b32 v39, s96, 57
+; FLATSCR-NEXT:    v_writelane_b32 v39, s97, 58
+; FLATSCR-NEXT:    v_writelane_b32 v39, s98, 59
+; FLATSCR-NEXT:    v_writelane_b32 v39, s99, 60
+; FLATSCR-NEXT:    s_addk_i32 s32, 0x100c
+; FLATSCR-NEXT:    v_writelane_b32 v39, s100, 61
+; FLATSCR-NEXT:    v_writelane_b32 v39, s101, 62
+; FLATSCR-NEXT:    v_mov_b32_e32 v0, 0
+; FLATSCR-NEXT:    s_add_i32 s1, s33, 0x1000
+; FLATSCR-NEXT:    v_writelane_b32 v39, s102, 63
+; FLATSCR-NEXT:    scratch_store_dword off, v0, s1
+; FLATSCR-NEXT:    s_waitcnt vmcnt(0)
+; FLATSCR-NEXT:    ;;#ASMSTART
+; FLATSCR-NEXT:    ; clobber nonpreserved SGPRs and 64 CSRs
+; FLATSCR-NEXT:    ;;#ASMEND
+; FLATSCR-NEXT:    ;;#ASMSTART
+; FLATSCR-NEXT:    ; clobber all VGPRs except CSR v40
+; FLATSCR-NEXT:    ;;#ASMEND
+; FLATSCR-NEXT:    v_readlane_b32 s102, v39, 63
+; FLATSCR-NEXT:    v_readlane_b32 s101, v39, 62
+; FLATSCR-NEXT:    v_readlane_b32 s100, v39, 61
+; FLATSCR-NEXT:    v_readlane_b32 s99, v39, 60
+; FLATSCR-NEXT:    v_readlane_b32 s98, v39, 59
+; FLATSCR-NEXT:    v_readlane_b32 s97, v39, 58
+; FLATSCR-NEXT:    v_readlane_b32 s96, v39, 57
+; FLATSCR-NEXT:    v_readlane_b32 s95, v39, 56
+; FLATSCR-NEXT:    v_readlane_b32 s94, v39, 55
+; FLATSCR-NEXT:    v_readlane_b32 s93, v39, 54
+; FLATSCR-NEXT:    v_readlane_b32 s92, v39, 53
+; FLATSCR-NEXT:    v_readlane_b32 s91, v39, 52
+; FLATSCR-NEXT:    v_readlane_b32 s90, v39, 51
+; FLATSCR-NEXT:    v_readlane_b32 s89, v39, 50
+; FLATSCR-NEXT:    v_readlane_b32 s88, v39, 49
+; FLATSCR-NEXT:    v_readlane_b32 s87, v39, 48
+; FLATSCR-NEXT:    v_readlane_b32 s86, v39, 47
+; FLATSCR-NEXT:    v_readlane_b32 s85, v39, 46
+; FLATSCR-NEXT:    v_readlane_b32 s84, v39, 45
+; FLATSCR-NEXT:    v_readlane_b32 s83, v39, 44
+; FLATSCR-NEXT:    v_readlane_b32 s82, v39, 43
+; FLATSCR-NEXT:    v_readlane_b32 s81, v39, 42
+; FLATSCR-NEXT:    v_readlane_b32 s80, v39, 41
+; FLATSCR-NEXT:    v_readlane_b32 s79, v39, 40
+; FLATSCR-NEXT:    v_readlane_b32 s78, v39, 39
+; FLATSCR-NEXT:    v_readlane_b32 s77, v39, 38
+; FLATSCR-NEXT:    v_readlane_b32 s76, v39, 37
+; FLATSCR-NEXT:    v_readlane_b32 s75, v39, 36
+; FLATSCR-NEXT:    v_readlane_b32 s74, v39, 35
+; FLATSCR-NEXT:    v_readlane_b32 s73, v39, 34
+; FLATSCR-NEXT:    v_readlane_b32 s72, v39, 33
+; FLATSCR-NEXT:    v_readlane_b32 s71, v39, 32
+; FLATSCR-NEXT:    v_readlane_b32 s70, v39, 31
+; FLATSCR-NEXT:    v_readlane_b32 s69, v39, 30
+; FLATSCR-NEXT:    v_readlane_b32 s68, v39, 29
+; FLATSCR-NEXT:    v_readlane_b32 s67, v39, 28
+; FLATSCR-NEXT:    v_readlane_b32 s66, v39, 27
+; FLATSCR-NEXT:    v_readlane_b32 s65, v39, 26
+; FLATSCR-NEXT:    v_readlane_b32 s64, v39, 25
+; FLATSCR-NEXT:    v_readlane_b32 s63, v39, 24
+; FLATSCR-NEXT:    v_readlane_b32 s62, v39, 23
+; FLATSCR-NEXT:    v_readlane_b32 s61, v39, 22
+; FLATSCR-NEXT:    v_readlane_b32 s60, v39, 21
+; FLATSCR-NEXT:    v_readlane_b32 s59, v39, 20
+; FLATSCR-NEXT:    v_readlane_b32 s58, v39, 19
+; FLATSCR-NEXT:    v_readlane_b32 s57, v39, 18
+; FLATSCR-NEXT:    v_readlane_b32 s56, v39, 17
+; FLATSCR-NEXT:    v_readlane_b32 s55, v39, 16
+; FLATSCR-NEXT:    v_readlane_b32 s54, v39, 15
+; FLATSCR-NEXT:    v_readlane_b32 s53, v39, 14
+; FLATSCR-NEXT:    v_readlane_b32 s52, v39, 13
+; FLATSCR-NEXT:    v_readlane_b32 s51, v39, 12
+; FLATSCR-NEXT:    v_readlane_b32 s50, v39, 11
+; FLATSCR-NEXT:    v_readlane_b32 s49, v39, 10
+; FLATSCR-NEXT:    v_readlane_b32 s48, v39, 9
+; FLATSCR-NEXT:    v_readlane_b32 s47, v39, 8
+; FLATSCR-NEXT:    v_readlane_b32 s46, v39, 7
+; FLATSCR-NEXT:    v_readlane_b32 s45, v39, 6
+; FLATSCR-NEXT:    v_readlane_b32 s44, v39, 5
+; FLATSCR-NEXT:    v_readlane_b32 s43, v39, 4
+; FLATSCR-NEXT:    v_readlane_b32 s42, v39, 3
+; FLATSCR-NEXT:    v_readlane_b32 s41, v39, 2
+; FLATSCR-NEXT:    v_readlane_b32 s40, v39, 1
+; FLATSCR-NEXT:    v_readlane_b32 s39, v39, 0
+; FLATSCR-NEXT:    s_mov_b32 s32, s33
+; FLATSCR-NEXT:    s_xor_saveexec_b64 s[2:3], -1
+; FLATSCR-NEXT:    s_add_i32 s1, s33, 0x1004
+; FLATSCR-NEXT:    scratch_load_dword v39, off, s1 ; 4-byte Folded Reload
+; FLATSCR-NEXT:    s_mov_b64 exec, s[2:3]
+; FLATSCR-NEXT:    s_mov_b32 s33, s0
+; FLATSCR-NEXT:    s_waitcnt vmcnt(0)
+; FLATSCR-NEXT:    s_setpc_b64 s[30:31]
   %alloca = alloca i32, addrspace(5)
   store volatile i32 0, ptr addrspace(5) %alloca
 
diff --git a/llvm/test/CodeGen/AMDGPU/callee-special-input-sgprs-fixed-abi.ll b/llvm/test/CodeGen/AMDGPU/callee-special-input-sgprs-fixed-abi.ll
index 609425329e106..5d4db904fe6ea 100644
--- a/llvm/test/CodeGen/AMDGPU/callee-special-input-sgprs-fixed-abi.ll
+++ b/llvm/test/CodeGen/AMDGPU/callee-special-input-sgprs-fixed-abi.ll
@@ -198,11 +198,12 @@ define hidden void @use_workgroup_id_yz() #1 {
 
 ; GCN-LABEL: {{^}}kern_indirect_use_workgroup_id_x:
 ; GCN-NOT: s6
-; GCN: s_mov_b32 s12, s6
-; GCN: s_mov_b32 s32, 0
 ; GCN: s_getpc_b64 s[4:5]
 ; GCN-NEXT: s_add_u32 s4, s4, use_workgroup_id_x@rel32@lo+4
 ; GCN-NEXT: s_addc_u32 s5, s5, use_workgroup_id_x@rel32@hi+12
+; GCN-NOT: s6
+; GCN: s_mov_b32 s12, s6
+; GCN: s_mov_b32 s32, 0
 ; GCN: s_swappc_b64
 ; GCN-NEXT: s_endpgm
 
diff --git a/llvm/test/CodeGen/AMDGPU/cc-update.ll b/llvm/test/CodeGen/AMDGPU/cc-update.ll
index c511f88aeaf86..fc24041fe771a 100644
--- a/llvm/test/CodeGen/AMDGPU/cc-update.ll
+++ b/llvm/test/CodeGen/AMDGPU/cc-update.ll
@@ -69,20 +69,20 @@ define amdgpu_kernel void @test_kern_call() local_unnamed_addr #0 {
 ; GFX803:       ; %bb.0: ; %entry
 ; GFX803-NEXT:    s_add_i32 s12, s12, s17
 ; GFX803-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
-; GFX803-NEXT:    v_lshlrev_b32_e32 v1, 10, v1
 ; GFX803-NEXT:    s_add_u32 s0, s0, s17
+; GFX803-NEXT:    v_lshlrev_b32_e32 v1, 10, v1
+; GFX803-NEXT:    s_addc_u32 s1, s1, 0
 ; GFX803-NEXT:    v_lshlrev_b32_e32 v2, 20, v2
 ; GFX803-NEXT:    v_or_b32_e32 v0, v0, v1
 ; GFX803-NEXT:    s_mov_b32 flat_scratch_lo, s13
-; GFX803-NEXT:    s_addc_u32 s1, s1, 0
 ; GFX803-NEXT:    s_mov_b32 s13, s15
 ; GFX803-NEXT:    s_mov_b32 s12, s14
-; GFX803-NEXT:    v_or_b32_e32 v31, v0, v2
-; GFX803-NEXT:    s_mov_b32 s14, s16
-; GFX803-NEXT:    s_mov_b32 s32, 0
 ; GFX803-NEXT:    s_getpc_b64 s[18:19]
 ; GFX803-NEXT:    s_add_u32 s18, s18, ex@rel32@lo+4
 ; GFX803-NEXT:    s_addc_u32 s19, s19, ex@rel32@hi+12
+; GFX803-NEXT:    v_or_b32_e32 v31, v0, v2
+; GFX803-NEXT:    s_mov_b32 s14, s16
+; GFX803-NEXT:    s_mov_b32 s32, 0
 ; GFX803-NEXT:    s_swappc_b64 s[30:31], s[18:19]
 ; GFX803-NEXT:    s_endpgm
 ;
@@ -91,17 +91,17 @@ define amdgpu_kernel void @test_kern_call() local_unnamed_addr #0 {
 ; GFX900-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
 ; GFX900-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX900-NEXT:    s_add_u32 s0, s0, s17
+; GFX900-NEXT:    s_addc_u32 s1, s1, 0
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v2, 20, v2
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v1, 10, v1
-; GFX900-NEXT:    s_addc_u32 s1, s1, 0
 ; GFX900-NEXT:    s_mov_b32 s13, s15
 ; GFX900-NEXT:    s_mov_b32 s12, s14
-; GFX900-NEXT:    v_or3_b32 v31, v0, v1, v2
-; GFX900-NEXT:    s_mov_b32 s14, s16
-; GFX900-NEXT:    s_mov_b32 s32, 0
 ; GFX900-NEXT:    s_getpc_b64 s[18:19]
 ; GFX900-NEXT:    s_add_u32 s18, s18, ex@rel32@lo+4
 ; GFX900-NEXT:    s_addc_u32 s19, s19, ex@rel32@hi+12
+; GFX900-NEXT:    v_or3_b32 v31, v0, v1, v2
+; GFX900-NEXT:    s_mov_b32 s14, s16
+; GFX900-NEXT:    s_mov_b32 s32, 0
 ; GFX900-NEXT:    s_swappc_b64 s[30:31], s[18:19]
 ; GFX900-NEXT:    s_endpgm
 ;
@@ -119,10 +119,10 @@ define amdgpu_kernel void @test_kern_call() local_unnamed_addr #0 {
 ; GFX1010-NEXT:    s_mov_b32 s13, s15
 ; GFX1010-NEXT:    s_mov_b32 s12, s14
 ; GFX1010-NEXT:    v_or3_b32 v31, v0, v1, v2
-; GFX1010-NEXT:    s_mov_b32 s14, s16
 ; GFX1010-NEXT:    s_getpc_b64 s[18:19]
 ; GFX1010-NEXT:    s_add_u32 s18, s18, ex@rel32@lo+4
 ; GFX1010-NEXT:    s_addc_u32 s19, s19, ex@rel32@hi+12
+; GFX1010-NEXT:    s_mov_b32 s14, s16
 ; GFX1010-NEXT:    s_swappc_b64 s[30:31], s[18:19]
 ; GFX1010-NEXT:    s_endpgm
 ;
@@ -132,14 +132,14 @@ define amdgpu_kernel void @test_kern_call() local_unnamed_addr #0 {
 ; GFX1100-NEXT:    s_mov_b32 s12, s13
 ; GFX1100-NEXT:    s_mov_b64 s[10:11], s[6:7]
 ; GFX1100-NEXT:    s_mov_b64 s[8:9], s[4:5]
+; GFX1100-NEXT:    s_getpc_b64 s[16:17]
+; GFX1100-NEXT:    s_add_u32 s16, s16, ex@rel32@lo+4
+; GFX1100-NEXT:    s_addc_u32 s17, s17, ex@rel32@hi+12
 ; GFX1100-NEXT:    s_mov_b64 s[4:5], s[0:1]
 ; GFX1100-NEXT:    s_mov_b64 s[6:7], s[2:3]
 ; GFX1100-NEXT:    s_mov_b32 s13, s14
 ; GFX1100-NEXT:    s_mov_b32 s14, s15
 ; GFX1100-NEXT:    s_mov_b32 s32, 0
-; GFX1100-NEXT:    s_getpc_b64 s[16:17]
-; GFX1100-NEXT:    s_add_u32 s16, s16, ex@rel32@lo+4
-; GFX1100-NEXT:    s_addc_u32 s17, s17, ex@rel32@hi+12
 ; GFX1100-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GFX1100-NEXT:    s_endpgm
 
@@ -153,23 +153,23 @@ define amdgpu_kernel void @test_kern_stack_and_call() local_unnamed_addr #0 {
 ; GFX803:       ; %bb.0: ; %entry
 ; GFX803-NEXT:    s_add_i32 s12, s12, s17
 ; GFX803-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
-; GFX803-NEXT:    v_lshlrev_b32_e32 v1, 10, v1
 ; GFX803-NEXT:    s_add_u32 s0, s0, s17
+; GFX803-NEXT:    v_lshlrev_b32_e32 v1, 10, v1
+; GFX803-NEXT:    s_addc_u32 s1, s1, 0
 ; GFX803-NEXT:    v_lshlrev_b32_e32 v2, 20, v2
 ; GFX803-NEXT:    v_or_b32_e32 v0, v0, v1
 ; GFX803-NEXT:    s_mov_b32 flat_scratch_lo, s13
-; GFX803-NEXT:    s_addc_u32 s1, s1, 0
 ; GFX803-NEXT:    s_mov_b32 s13, s15
 ; GFX803-NEXT:    s_mov_b32 s12, s14
 ; GFX803-NEXT:    v_mov_b32_e32 v3, 0
+; GFX803-NEXT:    s_getpc_b64 s[18:19]
+; GFX803-NEXT:    s_add_u32 s18, s18, ex@rel32@lo+4
+; GFX803-NEXT:    s_addc_u32 s19, s19, ex@rel32@hi+12
 ; GFX803-NEXT:    v_or_b32_e32 v31, v0, v2
 ; GFX803-NEXT:    s_mov_b32 s14, s16
 ; GFX803-NEXT:    s_movk_i32 s32, 0x400
 ; GFX803-NEXT:    buffer_store_dword v3, off, s[0:3], 0
 ; GFX803-NEXT:    s_waitcnt vmcnt(0)
-; GFX803-NEXT:    s_getpc_b64 s[18:19]
-; GFX803-NEXT:    s_add_u32 s18, s18, ex@rel32@lo+4
-; GFX803-NEXT:    s_addc_u32 s19, s19, ex@rel32@hi+12
 ; GFX803-NEXT:    s_swappc_b64 s[30:31], s[18:19]
 ; GFX803-NEXT:    s_endpgm
 ;
@@ -178,20 +178,20 @@ define amdgpu_kernel void @test_kern_stack_and_call() local_unnamed_addr #0 {
 ; GFX900-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
 ; GFX900-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX900-NEXT:    s_add_u32 s0, s0, s17
+; GFX900-NEXT:    s_addc_u32 s1, s1, 0
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v2, 20, v2
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v1, 10, v1
-; GFX900-NEXT:    s_addc_u32 s1, s1, 0
 ; GFX900-NEXT:    s_mov_b32 s13, s15
 ; GFX900-NEXT:    s_mov_b32 s12, s14
 ; GFX900-NEXT:    v_mov_b32_e32 v3, 0
+; GFX900-NEXT:    s_getpc_b64 s[18:19]
+; GFX900-NEXT:    s_add_u32 s18, s18, ex@rel32@lo+4
+; GFX900-NEXT:    s_addc_u32 s19, s19, ex@rel32@hi+12
 ; GFX900-NEXT:    v_or3_b32 v31, v0, v1, v2
 ; GFX900-NEXT:    s_mov_b32 s14, s16
 ; GFX900-NEXT:    s_movk_i32 s32, 0x400
 ; GFX900-NEXT:    buffer_store_dword v3, off, s[0:3], 0
 ; GFX900-NEXT:    s_waitcnt vmcnt(0)
-; GFX900-NEXT:    s_getpc_b64 s[18:19]
-; GFX900-NEXT:    s_add_u32 s18, s18, ex@rel32@lo+4
-; GFX900-NEXT:    s_addc_u32 s19, s19, ex@rel32@hi+12
 ; GFX900-NEXT:    s_swappc_b64 s[30:31], s[18:19]
 ; GFX900-NEXT:    s_endpgm
 ;
@@ -210,12 +210,12 @@ define amdgpu_kernel void @test_kern_stack_and_call() local_unnamed_addr #0 {
 ; GFX1010-NEXT:    s_mov_b32 s13, s15
 ; GFX1010-NEXT:    v_or3_b32 v31, v0, v1, v2
 ; GFX1010-NEXT:    s_mov_b32 s12, s14
-; GFX1010-NEXT:    s_mov_b32 s14, s16
-; GFX1010-NEXT:    buffer_store_dword v3, off, s[0:3], 0
-; GFX1010-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX1010-NEXT:    s_getpc_b64 s[18:19]
 ; GFX1010-NEXT:    s_add_u32 s18, s18, ex@rel32@lo+4
 ; GFX1010-NEXT:    s_addc_u32 s19, s19, ex@rel32@hi+12
+; GFX1010-NEXT:    s_mov_b32 s14, s16
+; GFX1010-NEXT:    buffer_store_dword v3, off, s[0:3], 0
+; GFX1010-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX1010-NEXT:    s_swappc_b64 s[30:31], s[18:19]
 ; GFX1010-NEXT:    s_endpgm
 ;
@@ -226,6 +226,9 @@ define amdgpu_kernel void @test_kern_stack_and_call() local_unnamed_addr #0 {
 ; GFX1100-NEXT:    s_mov_b32 s12, s13
 ; GFX1100-NEXT:    s_mov_b64 s[10:11], s[6:7]
 ; GFX1100-NEXT:    s_mov_b64 s[8:9], s[4:5]
+; GFX1100-NEXT:    s_getpc_b64 s[16:17]
+; GFX1100-NEXT:    s_add_u32 s16, s16, ex@rel32@lo+4
+; GFX1100-NEXT:    s_addc_u32 s17, s17, ex@rel32@hi+12
 ; GFX1100-NEXT:    s_mov_b64 s[4:5], s[0:1]
 ; GFX1100-NEXT:    s_mov_b64 s[6:7], s[2:3]
 ; GFX1100-NEXT:    s_mov_b32 s13, s14
@@ -233,9 +236,6 @@ define amdgpu_kernel void @test_kern_stack_and_call() local_unnamed_addr #0 {
 ; GFX1100-NEXT:    s_mov_b32 s32, 16
 ; GFX1100-NEXT:    scratch_store_b32 off, v1, off dlc
 ; GFX1100-NEXT:    s_waitcnt_vscnt null, 0x0
-; GFX1100-NEXT:    s_getpc_b64 s[16:17]
-; GFX1100-NEXT:    s_add_u32 s16, s16, ex@rel32@lo+4
-; GFX1100-NEXT:    s_addc_u32 s17, s17, ex@rel32@hi+12
 ; GFX1100-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GFX1100-NEXT:    s_endpgm
 
@@ -320,21 +320,21 @@ define amdgpu_kernel void @test_force_fp_kern_call() local_unnamed_addr #2 {
 ; GFX803:       ; %bb.0: ; %entry
 ; GFX803-NEXT:    s_add_i32 s12, s12, s17
 ; GFX803-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
-; GFX803-NEXT:    v_lshlrev_b32_e32 v1, 10, v1
 ; GFX803-NEXT:    s_add_u32 s0, s0, s17
+; GFX803-NEXT:    v_lshlrev_b32_e32 v1, 10, v1
+; GFX803-NEXT:    s_addc_u32 s1, s1, 0
 ; GFX803-NEXT:    v_lshlrev_b32_e32 v2, 20, v2
 ; GFX803-NEXT:    v_or_b32_e32 v0, v0, v1
 ; GFX803-NEXT:    s_mov_b32 flat_scratch_lo, s13
-; GFX803-NEXT:    s_addc_u32 s1, s1, 0
 ; GFX803-NEXT:    s_mov_b32 s13, s15
 ; GFX803-NEXT:    s_mov_b32 s12, s14
+; GFX803-NEXT:    s_getpc_b64 s[18:19]
+; GFX803-NEXT:    s_add_u32 s18, s18, ex@rel32@lo+4
+; GFX803-NEXT:    s_addc_u32 s19, s19, ex@rel32@hi+12
 ; GFX803-NEXT:    v_or_b32_e32 v31, v0, v2
 ; GFX803-NEXT:    s_mov_b32 s14, s16
 ; GFX803-NEXT:    s_mov_b32 s33, 0
 ; GFX803-NEXT:    s_mov_b32 s32, 0
-; GFX803-NEXT:    s_getpc_b64 s[18:19]
-; GFX803-NEXT:    s_add_u32 s18, s18, ex@rel32@lo+4
-; GFX803-NEXT:    s_addc_u32 s19, s19, ex@rel32@hi+12
 ; GFX803-NEXT:    s_swappc_b64 s[30:31], s[18:19]
 ; GFX803-NEXT:    s_endpgm
 ;
@@ -343,18 +343,18 @@ define amdgpu_kernel void @test_force_fp_kern_call() local_unnamed_addr #2 {
 ; GFX900-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
 ; GFX900-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX900-NEXT:    s_add_u32 s0, s0, s17
+; GFX900-NEXT:    s_addc_u32 s1, s1, 0
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v2, 20, v2
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v1, 10, v1
-; GFX900-NEXT:    s_addc_u32 s1, s1, 0
 ; GFX900-NEXT:    s_mov_b32 s13, s15
 ; GFX900-NEXT:    s_mov_b32 s12, s14
+; GFX900-NEXT:    s_getpc_b64 s[18:19]
+; GFX900-NEXT:    s_add_u32 s18, s18, ex@rel32@lo+4
+; GFX900-NEXT:    s_addc_u32 s19, s19, ex@rel32@hi+12
 ; GFX900-NEXT:    v_or3_b32 v31, v0, v1, v2
 ; GFX900-NEXT:    s_mov_b32 s14, s16
 ; GFX900-NEXT:    s_mov_b32 s33, 0
 ; GFX900-NEXT:    s_mov_b32 s32, 0
-; GFX900-NEXT:    s_getpc_b64 s[18:19]
-; GFX900-NEXT:    s_add_u32 s18, s18, ex@rel32@lo+4
-; GFX900-NEXT:    s_addc_u32 s19, s19, ex@rel32@hi+12
 ; GFX900-NEXT:    s_swappc_b64 s[30:31], s[18:19]
 ; GFX900-NEXT:    s_endpgm
 ;
@@ -373,10 +373,10 @@ define amdgpu_kernel void @test_force_fp_kern_call() local_unnamed_addr #2 {
 ; GFX1010-NEXT:    s_mov_b32 s13, s15
 ; GFX1010-NEXT:    s_mov_b32 s12, s14
 ; GFX1010-NEXT:    v_or3_b32 v31, v0, v1, v2
-; GFX1010-NEXT:    s_mov_b32 s14, s16
 ; GFX1010-NEXT:    s_getpc_b64 s[18:19]
 ; GFX1010-NEXT:    s_add_u32 s18, s18, ex@rel32@lo+4
 ; GFX1010-NEXT:    s_addc_u32 s19, s19, ex@rel32@hi+12
+; GFX1010-NEXT:    s_mov_b32 s14, s16
 ; GFX1010-NEXT:    s_swappc_b64 s[30:31], s[18:19]
 ; GFX1010-NEXT:    s_endpgm
 ;
@@ -386,15 +386,15 @@ define amdgpu_kernel void @test_force_fp_kern_call() local_unnamed_addr #2 {
 ; GFX1100-NEXT:    s_mov_b32 s12, s13
 ; GFX1100-NEXT:    s_mov_b64 s[10:11], s[6:7]
 ; GFX1100-NEXT:    s_mov_b64 s[8:9], s[4:5]
+; GFX1100-NEXT:    s_getpc_b64 s[16:17]
+; GFX1100-NEXT:    s_add_u32 s16, s16, ex@rel32@lo+4
+; GFX1100-NEXT:    s_addc_u32 s17, s17, ex@rel32@hi+12
 ; GFX1100-NEXT:    s_mov_b64 s[4:5], s[0:1]
 ; GFX1100-NEXT:    s_mov_b64 s[6:7], s[2:3]
 ; GFX1100-NEXT:    s_mov_b32 s13, s14
 ; GFX1100-NEXT:    s_mov_b32 s14, s15
 ; GFX1100-NEXT:    s_mov_b32 s33, 0
 ; GFX1100-NEXT:    s_mov_b32 s32, 0
-; GFX1100-NEXT:    s_getpc_b64 s[16:17]
-; GFX1100-NEXT:    s_add_u32 s16, s16, ex@rel32@lo+4
-; GFX1100-NEXT:    s_addc_u32 s17, s17, ex@rel32@hi+12
 ; GFX1100-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GFX1100-NEXT:    s_endpgm
 ; GFX1010-NEXT    s_add_u32 s12, s12, s17
@@ -426,24 +426,24 @@ define amdgpu_kernel void @test_force_fp_kern_stack_and_call() local_unnamed_add
 ; GFX803:       ; %bb.0: ; %entry
 ; GFX803-NEXT:    s_add_i32 s12, s12, s17
 ; GFX803-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
-; GFX803-NEXT:    v_lshlrev_b32_e32 v1, 10, v1
 ; GFX803-NEXT:    s_add_u32 s0, s0, s17
+; GFX803-NEXT:    v_lshlrev_b32_e32 v1, 10, v1
+; GFX803-NEXT:    s_addc_u32 s1, s1, 0
 ; GFX803-NEXT:    v_lshlrev_b32_e32 v2, 20, v2
 ; GFX803-NEXT:    v_or_b32_e32 v0, v0, v1
 ; GFX803-NEXT:    s_mov_b32 s33, 0
 ; GFX803-NEXT:    s_mov_b32 flat_scratch_lo, s13
-; GFX803-NEXT:    s_addc_u32 s1, s1, 0
 ; GFX803-NEXT:    s_mov_b32 s13, s15
 ; GFX803-NEXT:    s_mov_b32 s12, s14
 ; GFX803-NEXT:    v_mov_b32_e32 v3, 0
+; GFX803-NEXT:    s_getpc_b64 s[18:19]
+; GFX803-NEXT:    s_add_u32 s18, s18, ex@rel32@lo+4
+; GFX803-NEXT:    s_addc_u32 s19, s19, ex@rel32@hi+12
 ; GFX803-NEXT:    v_or_b32_e32 v31, v0, v2
 ; GFX803-NEXT:    s_mov_b32 s14, s16
 ; GFX803-NEXT:    s_movk_i32 s32, 0x400
 ; GFX803-NEXT:    buffer_store_dword v3, off, s[0:3], s33
 ; GFX803-NEXT:    s_waitcnt vmcnt(0)
-; GFX803-NEXT:    s_getpc_b64 s[18:19]
-; GFX803-NEXT:    s_add_u32 s18, s18, ex@rel32@lo+4
-; GFX803-NEXT:    s_addc_u32 s19, s19, ex@rel32@hi+12
 ; GFX803-NEXT:    s_swappc_b64 s[30:31], s[18:19]
 ; GFX803-NEXT:    s_endpgm
 ;
@@ -452,21 +452,21 @@ define amdgpu_kernel void @test_force_fp_kern_stack_and_call() local_unnamed_add
 ; GFX900-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
 ; GFX900-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX900-NEXT:    s_add_u32 s0, s0, s17
+; GFX900-NEXT:    s_addc_u32 s1, s1, 0
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v2, 20, v2
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v1, 10, v1
 ; GFX900-NEXT:    s_mov_b32 s33, 0
-; GFX900-NEXT:    s_addc_u32 s1, s1, 0
 ; GFX900-NEXT:    s_mov_b32 s13, s15
 ; GFX900-NEXT:    s_mov_b32 s12, s14
 ; GFX900-NEXT:    v_mov_b32_e32 v3, 0
+; GFX900-NEXT:    s_getpc_b64 s[18:19]
+; GFX900-NEXT:    s_add_u32 s18, s18, ex@rel32@lo+4
+; GFX900-NEXT:    s_addc_u32 s19, s19, ex@rel32@hi+12
 ; GFX900-NEXT:    v_or3_b32 v31, v0, v1, v2
 ; GFX900-NEXT:    s_mov_b32 s14, s16
 ; GFX900-NEXT:    s_movk_i32 s32, 0x400
 ; GFX900-NEXT:    buffer_store_dword v3, off, s[0:3], s33
 ; GFX900-NEXT:    s_waitcnt vmcnt(0)
-; GFX900-NEXT:    s_getpc_b64 s[18:19]
-; GFX900-NEXT:    s_add_u32 s18, s18, ex@rel32@lo+4
-; GFX900-NEXT:    s_addc_u32 s19, s19, ex@rel32@hi+12
 ; GFX900-NEXT:    s_swappc_b64 s[30:31], s[18:19]
 ; GFX900-NEXT:    s_endpgm
 ;
@@ -486,12 +486,12 @@ define amdgpu_kernel void @test_force_fp_kern_stack_and_call() local_unnamed_add
 ; GFX1010-NEXT:    s_mov_b32 s13, s15
 ; GFX1010-NEXT:    v_or3_b32 v31, v0, v1, v2
 ; GFX1010-NEXT:    s_mov_b32 s12, s14
-; GFX1010-NEXT:    s_mov_b32 s14, s16
-; GFX1010-NEXT:    buffer_store_dword v3, off, s[0:3], s33
-; GFX1010-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX1010-NEXT:    s_getpc_b64 s[18:19]
 ; GFX1010-NEXT:    s_add_u32 s18, s18, ex@rel32@lo+4
 ; GFX1010-NEXT:    s_addc_u32 s19, s19, ex@rel32@hi+12
+; GFX1010-NEXT:    s_mov_b32 s14, s16
+; GFX1010-NEXT:    buffer_store_dword v3, off, s[0:3], s33
+; GFX1010-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX1010-NEXT:    s_swappc_b64 s[30:31], s[18:19]
 ; GFX1010-NEXT:    s_endpgm
 ;
@@ -503,6 +503,9 @@ define amdgpu_kernel void @test_force_fp_kern_stack_and_call() local_unnamed_add
 ; GFX1100-NEXT:    s_mov_b32 s12, s13
 ; GFX1100-NEXT:    s_mov_b64 s[10:11], s[6:7]
 ; GFX1100-NEXT:    s_mov_b64 s[8:9], s[4:5]
+; GFX1100-NEXT:    s_getpc_b64 s[16:17]
+; GFX1100-NEXT:    s_add_u32 s16, s16, ex@rel32@lo+4
+; GFX1100-NEXT:    s_addc_u32 s17, s17, ex@rel32@hi+12
 ; GFX1100-NEXT:    s_mov_b64 s[4:5], s[0:1]
 ; GFX1100-NEXT:    s_mov_b64 s[6:7], s[2:3]
 ; GFX1100-NEXT:    s_mov_b32 s13, s14
@@ -510,9 +513,6 @@ define amdgpu_kernel void @test_force_fp_kern_stack_and_call() local_unnamed_add
 ; GFX1100-NEXT:    s_mov_b32 s32, 16
 ; GFX1100-NEXT:    scratch_store_b32 off, v1, s33 dlc
 ; GFX1100-NEXT:    s_waitcnt_vscnt null, 0x0
-; GFX1100-NEXT:    s_getpc_b64 s[16:17]
-; GFX1100-NEXT:    s_add_u32 s16, s16, ex@rel32@lo+4
-; GFX1100-NEXT:    s_addc_u32 s17, s17, ex@rel32@hi+12
 ; GFX1100-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GFX1100-NEXT:    s_endpgm
 entry:
diff --git a/llvm/test/CodeGen/AMDGPU/cluster-flat-loads-postra.mir b/llvm/test/CodeGen/AMDGPU/cluster-flat-loads-postra.mir
index 2b7b03df1cb9e..e42a1d15f0b2b 100644
--- a/llvm/test/CodeGen/AMDGPU/cluster-flat-loads-postra.mir
+++ b/llvm/test/CodeGen/AMDGPU/cluster-flat-loads-postra.mir
@@ -1,4 +1,5 @@
 # RUN: llc -mtriple=amdgcn -mcpu=tonga -mattr=-xnack -run-pass post-RA-sched -verify-machineinstrs -o - %s | FileCheck -check-prefix=GCN %s
+# RUN: llc -mtriple=amdgcn -mcpu=tonga -mattr=-xnack -passes=post-RA-sched -o - %s | FileCheck -check-prefix=GCN %s
 
 # GCN:      FLAT_LOAD_DWORD
 # GCN-NEXT: FLAT_LOAD_DWORD
diff --git a/llvm/test/CodeGen/AMDGPU/coalescing-with-subregs-in-loop-bug.mir b/llvm/test/CodeGen/AMDGPU/coalescing-with-subregs-in-loop-bug.mir
index e066a48d9a3c2..dcc5d9dff33da 100644
--- a/llvm/test/CodeGen/AMDGPU/coalescing-with-subregs-in-loop-bug.mir
+++ b/llvm/test/CodeGen/AMDGPU/coalescing-with-subregs-in-loop-bug.mir
@@ -1,4 +1,5 @@
 # RUN: llc -mtriple=amdgcn--amdpal -mcpu=gfx803 -run-pass=register-coalescer,rename-independent-subregs %s -o - | FileCheck -check-prefix=GCN %s
+# RUN: llc -mtriple=amdgcn--amdpal -mcpu=gfx803 -passes=register-coalescer,rename-independent-subregs %s -o - | FileCheck -check-prefix=GCN %s
 
 # This test is for a bug where the following happens:
 #
diff --git a/llvm/test/CodeGen/AMDGPU/combine-sreg64-inits.mir b/llvm/test/CodeGen/AMDGPU/combine-sreg64-inits.mir
index 57afb456d6037..4f4b5b121ace6 100644
--- a/llvm/test/CodeGen/AMDGPU/combine-sreg64-inits.mir
+++ b/llvm/test/CodeGen/AMDGPU/combine-sreg64-inits.mir
@@ -1,5 +1,6 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgcn -verify-machineinstrs -run-pass=liveintervals,amdgpu-pre-ra-optimizations %s -o - | FileCheck -check-prefix=GCN %s
+# RUN: llc -mtriple=amdgcn -verify-machineinstrs -run-pass=amdgpu-pre-ra-optimizations %s -o - | FileCheck -check-prefix=GCN %s
+# RUN: llc -mtriple=amdgcn -passes="amdgpu-pre-ra-optimizations" %s -o - | FileCheck -check-prefix=GCN %s
 
 ---
 name:            combine_sreg64_inits
diff --git a/llvm/test/CodeGen/AMDGPU/copy-illegal-type.ll b/llvm/test/CodeGen/AMDGPU/copy-illegal-type.ll
index c6c0b9cf8f027..cc2f775ff22bc 100644
--- a/llvm/test/CodeGen/AMDGPU/copy-illegal-type.ll
+++ b/llvm/test/CodeGen/AMDGPU/copy-illegal-type.ll
@@ -163,33 +163,33 @@ define amdgpu_kernel void @test_copy_v4i8_x3(ptr addrspace(1) %out0, ptr addrspa
 define amdgpu_kernel void @test_copy_v4i8_x4(ptr addrspace(1) %out0, ptr addrspace(1) %out1, ptr addrspace(1) %out2, ptr addrspace(1) %out3, ptr addrspace(1) %in) nounwind {
 ; SI-LABEL: test_copy_v4i8_x4:
 ; SI:       ; %bb.0:
-; SI-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x11
-; SI-NEXT:    s_mov_b32 s3, 0xf000
-; SI-NEXT:    s_mov_b32 s10, 0
-; SI-NEXT:    s_mov_b32 s11, s3
+; SI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x11
+; SI-NEXT:    s_mov_b32 s11, 0xf000
+; SI-NEXT:    s_mov_b32 s2, 0
+; SI-NEXT:    s_mov_b32 s3, s11
 ; SI-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
 ; SI-NEXT:    v_mov_b32_e32 v1, 0
 ; SI-NEXT:    s_waitcnt lgkmcnt(0)
-; SI-NEXT:    buffer_load_dword v0, v[0:1], s[8:11], 0 addr64
-; SI-NEXT:    s_load_dwordx8 s[4:11], s[4:5], 0x9
-; SI-NEXT:    s_mov_b32 s2, -1
-; SI-NEXT:    s_mov_b32 s14, s2
-; SI-NEXT:    s_mov_b32 s15, s3
-; SI-NEXT:    s_mov_b32 s18, s2
+; SI-NEXT:    buffer_load_dword v0, v[0:1], s[0:3], 0 addr64
+; SI-NEXT:    s_load_dwordx8 s[0:7], s[4:5], 0x9
+; SI-NEXT:    s_mov_b32 s10, -1
+; SI-NEXT:    s_mov_b32 s14, s10
+; SI-NEXT:    s_mov_b32 s15, s11
+; SI-NEXT:    s_mov_b32 s18, s10
 ; SI-NEXT:    s_waitcnt lgkmcnt(0)
-; SI-NEXT:    s_mov_b32 s0, s4
-; SI-NEXT:    s_mov_b32 s1, s5
-; SI-NEXT:    s_mov_b32 s19, s3
-; SI-NEXT:    s_mov_b32 s22, s2
-; SI-NEXT:    s_mov_b32 s23, s3
-; SI-NEXT:    s_mov_b32 s12, s6
-; SI-NEXT:    s_mov_b32 s13, s7
-; SI-NEXT:    s_mov_b32 s16, s8
-; SI-NEXT:    s_mov_b32 s17, s9
-; SI-NEXT:    s_mov_b32 s20, s10
-; SI-NEXT:    s_mov_b32 s21, s11
+; SI-NEXT:    s_mov_b32 s8, s0
+; SI-NEXT:    s_mov_b32 s9, s1
+; SI-NEXT:    s_mov_b32 s19, s11
+; SI-NEXT:    s_mov_b32 s22, s10
+; SI-NEXT:    s_mov_b32 s23, s11
+; SI-NEXT:    s_mov_b32 s12, s2
+; SI-NEXT:    s_mov_b32 s13, s3
+; SI-NEXT:    s_mov_b32 s16, s4
+; SI-NEXT:    s_mov_b32 s17, s5
+; SI-NEXT:    s_mov_b32 s20, s6
+; SI-NEXT:    s_mov_b32 s21, s7
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; SI-NEXT:    buffer_store_dword v0, off, s[8:11], 0
 ; SI-NEXT:    buffer_store_dword v0, off, s[12:15], 0
 ; SI-NEXT:    buffer_store_dword v0, off, s[16:19], 0
 ; SI-NEXT:    buffer_store_dword v0, off, s[20:23], 0
diff --git a/llvm/test/CodeGen/AMDGPU/copy-to-reg-frameindex.ll b/llvm/test/CodeGen/AMDGPU/copy-to-reg-frameindex.ll
new file mode 100644
index 0000000000000..9558d9f0bc4c9
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/copy-to-reg-frameindex.ll
@@ -0,0 +1,37 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=amdgcn -mcpu=gfx90a | FileCheck %s
+
+define amdgpu_kernel void @copy_to_reg_frameindex(ptr addrspace(1) %out, i32 %a, i32 %b, i32 %c) {
+; CHECK-LABEL: copy_to_reg_frameindex:
+; CHECK:       ; %bb.0: ; %entry
+; CHECK-NEXT:    ; implicit-def: $vgpr0
+; CHECK-NEXT:  .LBB0_1: ; %loop
+; CHECK-NEXT:    ; =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    s_cmp_lt_u32 0, 16
+; CHECK-NEXT:    s_set_gpr_idx_on 0, gpr_idx(DST)
+; CHECK-NEXT:    v_mov_b32_e32 v0, 0
+; CHECK-NEXT:    s_set_gpr_idx_off
+; CHECK-NEXT:    s_cbranch_scc1 .LBB0_1
+; CHECK-NEXT:  ; %bb.2: ; %done
+; CHECK-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
+; CHECK-NEXT:    v_mov_b32_e32 v1, 0
+; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
+; CHECK-NEXT:    global_store_dword v1, v0, s[0:1]
+; CHECK-NEXT:    s_endpgm
+entry:
+  %B = srem i32 %c, -1
+  br label %loop
+
+loop:
+  %promotealloca = phi <16 x i32> [ undef, %entry ], [ %0, %loop ]
+  %inc = phi i32 [ 0, %entry ], [ %inc.i, %loop ]
+  %0 = insertelement <16 x i32> %promotealloca, i32 %inc, i32 %inc
+  %inc.i = add i32 %inc, %B
+  %cnd = icmp uge i32 %inc.i, 16
+  br i1 %cnd, label %done, label %loop
+
+done:
+  %1 = extractelement <16 x i32> %0, i32 0
+  store i32 %1, ptr addrspace(1) %out, align 4
+  ret void
+}
diff --git a/llvm/test/CodeGen/AMDGPU/cross-block-use-is-not-abi-copy.ll b/llvm/test/CodeGen/AMDGPU/cross-block-use-is-not-abi-copy.ll
index 9bef0b7d76ad5..8b8e5197a9c61 100644
--- a/llvm/test/CodeGen/AMDGPU/cross-block-use-is-not-abi-copy.ll
+++ b/llvm/test/CodeGen/AMDGPU/cross-block-use-is-not-abi-copy.ll
@@ -35,10 +35,10 @@ define float @call_split_type_used_outside_block_v2f32() #0 {
 ; GCN-NEXT:    v_writelane_b32 v40, s16, 2
 ; GCN-NEXT:    s_addk_i32 s32, 0x400
 ; GCN-NEXT:    v_writelane_b32 v40, s30, 0
-; GCN-NEXT:    v_writelane_b32 v40, s31, 1
 ; GCN-NEXT:    s_getpc_b64 s[16:17]
 ; GCN-NEXT:    s_add_u32 s16, s16, func_v2f32@rel32@lo+4
 ; GCN-NEXT:    s_addc_u32 s17, s17, func_v2f32@rel32@hi+12
+; GCN-NEXT:    v_writelane_b32 v40, s31, 1
 ; GCN-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GCN-NEXT:    v_readlane_b32 s31, v40, 1
 ; GCN-NEXT:    v_readlane_b32 s30, v40, 0
@@ -71,10 +71,10 @@ define float @call_split_type_used_outside_block_v3f32() #0 {
 ; GCN-NEXT:    v_writelane_b32 v40, s16, 2
 ; GCN-NEXT:    s_addk_i32 s32, 0x400
 ; GCN-NEXT:    v_writelane_b32 v40, s30, 0
-; GCN-NEXT:    v_writelane_b32 v40, s31, 1
 ; GCN-NEXT:    s_getpc_b64 s[16:17]
 ; GCN-NEXT:    s_add_u32 s16, s16, func_v3f32@rel32@lo+4
 ; GCN-NEXT:    s_addc_u32 s17, s17, func_v3f32@rel32@hi+12
+; GCN-NEXT:    v_writelane_b32 v40, s31, 1
 ; GCN-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GCN-NEXT:    v_readlane_b32 s31, v40, 1
 ; GCN-NEXT:    v_readlane_b32 s30, v40, 0
@@ -107,10 +107,10 @@ define half @call_split_type_used_outside_block_v4f16() #0 {
 ; GCN-NEXT:    v_writelane_b32 v40, s16, 2
 ; GCN-NEXT:    s_addk_i32 s32, 0x400
 ; GCN-NEXT:    v_writelane_b32 v40, s30, 0
-; GCN-NEXT:    v_writelane_b32 v40, s31, 1
 ; GCN-NEXT:    s_getpc_b64 s[16:17]
 ; GCN-NEXT:    s_add_u32 s16, s16, func_v4f16@rel32@lo+4
 ; GCN-NEXT:    s_addc_u32 s17, s17, func_v4f16@rel32@hi+12
+; GCN-NEXT:    v_writelane_b32 v40, s31, 1
 ; GCN-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GCN-NEXT:    v_readlane_b32 s31, v40, 1
 ; GCN-NEXT:    v_readlane_b32 s30, v40, 0
@@ -143,10 +143,10 @@ define { i32, half } @call_split_type_used_outside_block_struct() #0 {
 ; GCN-NEXT:    v_writelane_b32 v40, s16, 2
 ; GCN-NEXT:    s_addk_i32 s32, 0x400
 ; GCN-NEXT:    v_writelane_b32 v40, s30, 0
-; GCN-NEXT:    v_writelane_b32 v40, s31, 1
 ; GCN-NEXT:    s_getpc_b64 s[16:17]
 ; GCN-NEXT:    s_add_u32 s16, s16, func_struct@rel32@lo+4
 ; GCN-NEXT:    s_addc_u32 s17, s17, func_struct@rel32@hi+12
+; GCN-NEXT:    v_writelane_b32 v40, s31, 1
 ; GCN-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GCN-NEXT:    v_readlane_b32 s31, v40, 1
 ; GCN-NEXT:    v_readlane_b32 s30, v40, 0
@@ -189,16 +189,16 @@ define amdgpu_kernel void @v3i16_registers(i1 %cond) #0 {
 ; GCN-NEXT:    s_cbranch_vccnz .LBB4_2
 ; GCN-NEXT:  ; %bb.1: ; %if.else
 ; GCN-NEXT:    s_add_u32 s8, s8, 8
+; GCN-NEXT:    s_addc_u32 s9, s9, 0
 ; GCN-NEXT:    v_lshlrev_b32_e32 v2, 20, v2
 ; GCN-NEXT:    v_lshlrev_b32_e32 v1, 10, v1
-; GCN-NEXT:    s_addc_u32 s9, s9, 0
+; GCN-NEXT:    s_getpc_b64 s[18:19]
+; GCN-NEXT:    s_add_u32 s18, s18, func_v3i16@rel32@lo+4
+; GCN-NEXT:    s_addc_u32 s19, s19, func_v3i16@rel32@hi+12
 ; GCN-NEXT:    v_or3_b32 v31, v0, v1, v2
 ; GCN-NEXT:    s_mov_b32 s12, s14
 ; GCN-NEXT:    s_mov_b32 s13, s15
 ; GCN-NEXT:    s_mov_b32 s14, s16
-; GCN-NEXT:    s_getpc_b64 s[18:19]
-; GCN-NEXT:    s_add_u32 s18, s18, func_v3i16@rel32@lo+4
-; GCN-NEXT:    s_addc_u32 s19, s19, func_v3i16@rel32@hi+12
 ; GCN-NEXT:    s_swappc_b64 s[30:31], s[18:19]
 ; GCN-NEXT:    s_branch .LBB4_3
 ; GCN-NEXT:  .LBB4_2:
@@ -240,16 +240,16 @@ define amdgpu_kernel void @v3f16_registers(i1 %cond) #0 {
 ; GCN-NEXT:    s_cbranch_vccnz .LBB5_2
 ; GCN-NEXT:  ; %bb.1: ; %if.else
 ; GCN-NEXT:    s_add_u32 s8, s8, 8
+; GCN-NEXT:    s_addc_u32 s9, s9, 0
 ; GCN-NEXT:    v_lshlrev_b32_e32 v2, 20, v2
 ; GCN-NEXT:    v_lshlrev_b32_e32 v1, 10, v1
-; GCN-NEXT:    s_addc_u32 s9, s9, 0
+; GCN-NEXT:    s_getpc_b64 s[18:19]
+; GCN-NEXT:    s_add_u32 s18, s18, func_v3f16@rel32@lo+4
+; GCN-NEXT:    s_addc_u32 s19, s19, func_v3f16@rel32@hi+12
 ; GCN-NEXT:    v_or3_b32 v31, v0, v1, v2
 ; GCN-NEXT:    s_mov_b32 s12, s14
 ; GCN-NEXT:    s_mov_b32 s13, s15
 ; GCN-NEXT:    s_mov_b32 s14, s16
-; GCN-NEXT:    s_getpc_b64 s[18:19]
-; GCN-NEXT:    s_add_u32 s18, s18, func_v3f16@rel32@lo+4
-; GCN-NEXT:    s_addc_u32 s19, s19, func_v3f16@rel32@hi+12
 ; GCN-NEXT:    s_swappc_b64 s[30:31], s[18:19]
 ; GCN-NEXT:    s_branch .LBB5_3
 ; GCN-NEXT:  .LBB5_2:
diff --git a/llvm/test/CodeGen/AMDGPU/ctpop16.ll b/llvm/test/CodeGen/AMDGPU/ctpop16.ll
index 17ab8fc780fb4..6bf126af5ade2 100644
--- a/llvm/test/CodeGen/AMDGPU/ctpop16.ll
+++ b/llvm/test/CodeGen/AMDGPU/ctpop16.ll
@@ -457,58 +457,27 @@ define amdgpu_kernel void @v_ctpop_v4i16(ptr addrspace(1) noalias %out, ptr addr
 ;
 ; EG-LABEL: v_ctpop_v4i16:
 ; EG:       ; %bb.0:
-; EG-NEXT:    ALU 3, @8, KC0[CB0:0-32], KC1[]
+; EG-NEXT:    ALU 2, @8, KC0[CB0:0-32], KC1[]
 ; EG-NEXT:    TEX 0 @6
-; EG-NEXT:    ALU 37, @12, KC0[CB0:0-32], KC1[]
-; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T8.XY, T0.X, 1
+; EG-NEXT:    ALU 7, @11, KC0[CB0:0-32], KC1[]
+; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T0.XY, T6.X, 1
 ; EG-NEXT:    CF_END
 ; EG-NEXT:    PAD
 ; EG-NEXT:    Fetch clause starting at 6:
-; EG-NEXT:     VTX_READ_64 T8.XY, T0.X, 0, #1
+; EG-NEXT:     VTX_READ_32 T0.X, T0.X, 0, #1
 ; EG-NEXT:    ALU clause starting at 8:
-; EG-NEXT:     MOV T0.Y, T4.X,
-; EG-NEXT:     LSHL * T0.W, T0.X, literal.x, BS:VEC_120/SCL_212
+; EG-NEXT:     LSHL * T0.W, T0.X, literal.x,
 ; EG-NEXT:    3(4.203895e-45), 0(0.000000e+00)
 ; EG-NEXT:     ADD_INT * T0.X, KC0[2].Z, PV.W,
-; EG-NEXT:    ALU clause starting at 12:
-; EG-NEXT:     AND_INT * T0.W, T8.X, literal.x,
-; EG-NEXT:    65535(9.183409e-41), 0(0.000000e+00)
-; EG-NEXT:     BCNT_INT T0.W, PV.W,
-; EG-NEXT:     AND_INT * T1.W, T0.Y, literal.x,
-; EG-NEXT:    -65536(nan), 0(0.000000e+00)
-; EG-NEXT:     OR_INT * T0.W, PS, PV.W,
-; EG-NEXT:     MOV * T4.X, PV.W,
-; EG-NEXT:     MOV T0.X, PV.X,
-; EG-NEXT:     LSHR * T0.W, T8.X, literal.x,
-; EG-NEXT:    16(2.242078e-44), 0(0.000000e+00)
-; EG-NEXT:     BCNT_INT T0.W, PV.W,
-; EG-NEXT:     AND_INT * T1.W, PV.X, literal.x,
-; EG-NEXT:    65535(9.183409e-41), 0(0.000000e+00)
-; EG-NEXT:     LSHL * T0.W, PV.W, literal.x,
-; EG-NEXT:    16(2.242078e-44), 0(0.000000e+00)
-; EG-NEXT:     OR_INT * T0.W, T1.W, PV.W,
-; EG-NEXT:     MOV T4.X, PV.W,
-; EG-NEXT:     MOV * T0.X, T5.X,
-; EG-NEXT:     AND_INT * T0.W, T8.Y, literal.x,
-; EG-NEXT:    65535(9.183409e-41), 0(0.000000e+00)
-; EG-NEXT:     BCNT_INT T0.W, PV.W,
-; EG-NEXT:     AND_INT * T1.W, T0.X, literal.x,
-; EG-NEXT:    -65536(nan), 0(0.000000e+00)
-; EG-NEXT:     OR_INT * T0.W, PS, PV.W,
-; EG-NEXT:     MOV * T5.X, PV.W,
-; EG-NEXT:     MOV T0.X, PV.X,
-; EG-NEXT:     LSHR * T0.W, T8.Y, literal.x,
+; EG-NEXT:    ALU clause starting at 11:
+; EG-NEXT:     LSHR * T0.W, T0.X, literal.x,
 ; EG-NEXT:    16(2.242078e-44), 0(0.000000e+00)
-; EG-NEXT:     BCNT_INT T0.W, PV.W,
-; EG-NEXT:     AND_INT * T1.W, PV.X, literal.x,
+; EG-NEXT:     BCNT_INT T0.Y, PV.W,
+; EG-NEXT:     AND_INT * T0.W, T0.X, literal.x,
 ; EG-NEXT:    65535(9.183409e-41), 0(0.000000e+00)
-; EG-NEXT:     LSHL * T0.W, PV.W, literal.x,
-; EG-NEXT:    16(2.242078e-44), 0(0.000000e+00)
-; EG-NEXT:     LSHR T0.X, KC0[2].Y, literal.x,
-; EG-NEXT:     OR_INT * T8.Y, T1.W, PV.W,
+; EG-NEXT:     BCNT_INT T0.X, PV.W,
+; EG-NEXT:     LSHR * T6.X, KC0[2].Y, literal.x,
 ; EG-NEXT:    2(2.802597e-45), 0(0.000000e+00)
-; EG-NEXT:     MOV T5.X, PV.Y,
-; EG-NEXT:     MOV * T8.X, T4.X,
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
   %in.gep = getelementptr <4 x i16>, ptr addrspace(1) %in, i32 %tid
   %val = load <4 x i16>, ptr addrspace(1) %in.gep, align 16
@@ -601,94 +570,33 @@ define amdgpu_kernel void @v_ctpop_v8i16(ptr addrspace(1) noalias %out, ptr addr
 ;
 ; EG-LABEL: v_ctpop_v8i16:
 ; EG:       ; %bb.0:
-; EG-NEXT:    ALU 3, @8, KC0[CB0:0-32], KC1[]
+; EG-NEXT:    ALU 2, @8, KC0[CB0:0-32], KC1[]
 ; EG-NEXT:    TEX 0 @6
-; EG-NEXT:    ALU 73, @12, KC0[CB0:0-32], KC1[]
-; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T0.XYZW, T12.X, 1
+; EG-NEXT:    ALU 13, @11, KC0[CB0:0-32], KC1[]
+; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T0.XYZW, T8.X, 1
 ; EG-NEXT:    CF_END
 ; EG-NEXT:    PAD
 ; EG-NEXT:    Fetch clause starting at 6:
-; EG-NEXT:     VTX_READ_128 T12.XYZW, T0.X, 0, #1
+; EG-NEXT:     VTX_READ_128 T0.XYZW, T0.X, 0, #1
 ; EG-NEXT:    ALU clause starting at 8:
-; EG-NEXT:     MOV T0.Y, T4.X,
-; EG-NEXT:     LSHL * T0.W, T0.X, literal.x, BS:VEC_120/SCL_212
+; EG-NEXT:     LSHL * T0.W, T0.X, literal.x,
 ; EG-NEXT:    4(5.605194e-45), 0(0.000000e+00)
 ; EG-NEXT:     ADD_INT * T0.X, KC0[2].Z, PV.W,
-; EG-NEXT:    ALU clause starting at 12:
-; EG-NEXT:     LSHR * T0.W, T12.X, literal.x,
-; EG-NEXT:    16(2.242078e-44), 0(0.000000e+00)
-; EG-NEXT:     BCNT_INT * T0.W, PV.W,
-; EG-NEXT:     LSHL T0.W, PV.W, literal.x,
-; EG-NEXT:     AND_INT * T1.W, T0.Y, literal.y,
-; EG-NEXT:    16(2.242078e-44), 65535(9.183409e-41)
-; EG-NEXT:     OR_INT * T0.W, PS, PV.W,
-; EG-NEXT:     MOV * T4.X, PV.W,
-; EG-NEXT:     MOV T0.X, PV.X,
-; EG-NEXT:     AND_INT * T0.W, T12.X, literal.x,
-; EG-NEXT:    65535(9.183409e-41), 0(0.000000e+00)
-; EG-NEXT:     BCNT_INT T0.W, PV.W,
-; EG-NEXT:     AND_INT * T1.W, PV.X, literal.x,
-; EG-NEXT:    -65536(nan), 0(0.000000e+00)
-; EG-NEXT:     OR_INT * T0.W, PS, PV.W,
-; EG-NEXT:     MOV T4.X, PV.W,
-; EG-NEXT:     MOV * T0.X, T5.X,
-; EG-NEXT:     LSHR * T0.W, T12.Y, literal.x,
-; EG-NEXT:    16(2.242078e-44), 0(0.000000e+00)
-; EG-NEXT:     BCNT_INT T0.W, PV.W,
-; EG-NEXT:     AND_INT * T1.W, T0.X, literal.x,
-; EG-NEXT:    65535(9.183409e-41), 0(0.000000e+00)
-; EG-NEXT:     LSHL * T0.W, PV.W, literal.x,
-; EG-NEXT:    16(2.242078e-44), 0(0.000000e+00)
-; EG-NEXT:     OR_INT * T0.W, T1.W, PV.W,
-; EG-NEXT:     MOV * T5.X, PV.W,
-; EG-NEXT:     MOV T0.X, PV.X,
-; EG-NEXT:     AND_INT * T0.W, T12.Y, literal.x,
-; EG-NEXT:    65535(9.183409e-41), 0(0.000000e+00)
-; EG-NEXT:     BCNT_INT T0.W, PV.W,
-; EG-NEXT:     AND_INT * T1.W, PV.X, literal.x,
-; EG-NEXT:    -65536(nan), 0(0.000000e+00)
-; EG-NEXT:     OR_INT * T0.Y, PS, PV.W,
-; EG-NEXT:     MOV T5.X, PV.Y,
-; EG-NEXT:     MOV * T0.X, T8.X,
-; EG-NEXT:     LSHR * T0.W, T12.Z, literal.x,
+; EG-NEXT:    ALU clause starting at 11:
+; EG-NEXT:     LSHR * T0.W, T0.Z, literal.x,
 ; EG-NEXT:    16(2.242078e-44), 0(0.000000e+00)
 ; EG-NEXT:     BCNT_INT T0.W, PV.W,
-; EG-NEXT:     AND_INT * T1.W, T0.X, literal.x,
-; EG-NEXT:    65535(9.183409e-41), 0(0.000000e+00)
-; EG-NEXT:     LSHL * T0.W, PV.W, literal.x,
-; EG-NEXT:    16(2.242078e-44), 0(0.000000e+00)
-; EG-NEXT:     OR_INT * T0.W, T1.W, PV.W,
-; EG-NEXT:     MOV * T8.X, PV.W,
-; EG-NEXT:     MOV T0.X, PV.X,
-; EG-NEXT:     AND_INT * T0.W, T12.Z, literal.x,
+; EG-NEXT:     AND_INT * T1.W, T0.Z, literal.x,
 ; EG-NEXT:    65535(9.183409e-41), 0(0.000000e+00)
-; EG-NEXT:     BCNT_INT T0.W, PV.W,
-; EG-NEXT:     AND_INT * T1.W, PV.X, literal.x,
-; EG-NEXT:    -65536(nan), 0(0.000000e+00)
-; EG-NEXT:     OR_INT * T0.W, PS, PV.W,
-; EG-NEXT:     MOV T8.X, PV.W,
-; EG-NEXT:     MOV * T0.X, T9.X,
-; EG-NEXT:     LSHR * T0.W, T12.W, literal.x,
+; EG-NEXT:     BCNT_INT T0.Z, PS,
+; EG-NEXT:     LSHR * T1.W, T0.X, literal.x,
 ; EG-NEXT:    16(2.242078e-44), 0(0.000000e+00)
-; EG-NEXT:     BCNT_INT T0.W, PV.W,
+; EG-NEXT:     BCNT_INT T0.Y, PV.W,
 ; EG-NEXT:     AND_INT * T1.W, T0.X, literal.x,
 ; EG-NEXT:    65535(9.183409e-41), 0(0.000000e+00)
-; EG-NEXT:     LSHL * T0.W, PV.W, literal.x,
-; EG-NEXT:    16(2.242078e-44), 0(0.000000e+00)
-; EG-NEXT:     OR_INT * T0.W, T1.W, PV.W,
-; EG-NEXT:     MOV * T9.X, PV.W,
-; EG-NEXT:     MOV T0.X, PV.X,
-; EG-NEXT:     AND_INT * T0.W, T12.W, literal.x,
-; EG-NEXT:    65535(9.183409e-41), 0(0.000000e+00)
-; EG-NEXT:     BCNT_INT T0.W, PV.W,
-; EG-NEXT:     AND_INT * T1.W, PV.X, literal.x,
-; EG-NEXT:    -65536(nan), 0(0.000000e+00)
-; EG-NEXT:     LSHR T12.X, KC0[2].Y, literal.x,
-; EG-NEXT:     OR_INT * T0.W, PS, PV.W,
+; EG-NEXT:     BCNT_INT T0.X, PV.W,
+; EG-NEXT:     LSHR * T8.X, KC0[2].Y, literal.x,
 ; EG-NEXT:    2(2.802597e-45), 0(0.000000e+00)
-; EG-NEXT:     MOV T9.X, PV.W,
-; EG-NEXT:     MOV * T0.X, T4.X,
-; EG-NEXT:     MOV * T0.Z, T8.X,
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
   %in.gep = getelementptr <8 x i16>, ptr addrspace(1) %in, i32 %tid
   %val = load <8 x i16>, ptr addrspace(1) %in.gep, align 32
@@ -837,174 +745,46 @@ define amdgpu_kernel void @v_ctpop_v16i16(ptr addrspace(1) noalias %out, ptr add
 ;
 ; EG-LABEL: v_ctpop_v16i16:
 ; EG:       ; %bb.0:
-; EG-NEXT:    ALU 3, @12, KC0[CB0:0-32], KC1[]
-; EG-NEXT:    TEX 1 @8
-; EG-NEXT:    ALU 114, @16, KC0[], KC1[]
-; EG-NEXT:    ALU 34, @131, KC0[CB0:0-32], KC1[]
-; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T0.XYZW, T22.X, 0
-; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T20.XYZW, T21.X, 1
+; EG-NEXT:    ALU 2, @10, KC0[CB0:0-32], KC1[]
+; EG-NEXT:    TEX 1 @6
+; EG-NEXT:    ALU 25, @13, KC0[CB0:0-32], KC1[]
+; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T0.XYZW, T14.X, 0
+; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T12.XYZW, T13.X, 1
 ; EG-NEXT:    CF_END
-; EG-NEXT:    PAD
-; EG-NEXT:    Fetch clause starting at 8:
-; EG-NEXT:     VTX_READ_128 T20.XYZW, T0.X, 16, #1
-; EG-NEXT:     VTX_READ_128 T21.XYZW, T0.X, 0, #1
-; EG-NEXT:    ALU clause starting at 12:
-; EG-NEXT:     MOV T0.Y, T4.X,
-; EG-NEXT:     LSHL * T0.W, T0.X, literal.x, BS:VEC_120/SCL_212
+; EG-NEXT:    Fetch clause starting at 6:
+; EG-NEXT:     VTX_READ_128 T12.XYZW, T0.X, 16, #1
+; EG-NEXT:     VTX_READ_128 T0.XYZW, T0.X, 0, #1
+; EG-NEXT:    ALU clause starting at 10:
+; EG-NEXT:     LSHL * T0.W, T0.X, literal.x,
 ; EG-NEXT:    5(7.006492e-45), 0(0.000000e+00)
 ; EG-NEXT:     ADD_INT * T0.X, KC0[2].Z, PV.W,
-; EG-NEXT:    ALU clause starting at 16:
-; EG-NEXT:     LSHR * T0.W, T20.X, literal.x,
-; EG-NEXT:    16(2.242078e-44), 0(0.000000e+00)
-; EG-NEXT:     BCNT_INT * T0.W, PV.W,
-; EG-NEXT:     LSHL T0.W, PV.W, literal.x,
-; EG-NEXT:     AND_INT * T1.W, T0.Y, literal.y,
-; EG-NEXT:    16(2.242078e-44), 65535(9.183409e-41)
-; EG-NEXT:     OR_INT * T0.W, PS, PV.W,
-; EG-NEXT:     MOV * T4.X, PV.W,
-; EG-NEXT:     MOV T0.X, PV.X,
-; EG-NEXT:     AND_INT * T0.W, T20.X, literal.x,
-; EG-NEXT:    65535(9.183409e-41), 0(0.000000e+00)
-; EG-NEXT:     BCNT_INT T0.W, PV.W,
-; EG-NEXT:     AND_INT * T1.W, PV.X, literal.x,
-; EG-NEXT:    -65536(nan), 0(0.000000e+00)
-; EG-NEXT:     OR_INT * T0.W, PS, PV.W,
-; EG-NEXT:     MOV T4.X, PV.W,
-; EG-NEXT:     MOV * T0.X, T5.X,
-; EG-NEXT:     LSHR * T0.W, T20.Y, literal.x,
-; EG-NEXT:    16(2.242078e-44), 0(0.000000e+00)
-; EG-NEXT:     BCNT_INT T0.W, PV.W,
-; EG-NEXT:     AND_INT * T1.W, T0.X, literal.x,
-; EG-NEXT:    65535(9.183409e-41), 0(0.000000e+00)
-; EG-NEXT:     LSHL * T0.W, PV.W, literal.x,
-; EG-NEXT:    16(2.242078e-44), 0(0.000000e+00)
-; EG-NEXT:     OR_INT * T0.W, T1.W, PV.W,
-; EG-NEXT:     MOV * T5.X, PV.W,
-; EG-NEXT:     MOV T0.X, PV.X,
-; EG-NEXT:     AND_INT * T0.W, T20.Y, literal.x,
-; EG-NEXT:    65535(9.183409e-41), 0(0.000000e+00)
-; EG-NEXT:     BCNT_INT T0.W, PV.W,
-; EG-NEXT:     AND_INT * T1.W, PV.X, literal.x,
-; EG-NEXT:    -65536(nan), 0(0.000000e+00)
-; EG-NEXT:     OR_INT * T0.Y, PS, PV.W,
-; EG-NEXT:     MOV T5.X, PV.Y,
-; EG-NEXT:     MOV * T0.X, T8.X,
-; EG-NEXT:     LSHR * T0.W, T20.Z, literal.x,
-; EG-NEXT:    16(2.242078e-44), 0(0.000000e+00)
-; EG-NEXT:     BCNT_INT T0.W, PV.W,
-; EG-NEXT:     AND_INT * T1.W, T0.X, literal.x,
-; EG-NEXT:    65535(9.183409e-41), 0(0.000000e+00)
-; EG-NEXT:     LSHL * T0.W, PV.W, literal.x,
-; EG-NEXT:    16(2.242078e-44), 0(0.000000e+00)
-; EG-NEXT:     OR_INT * T0.W, T1.W, PV.W,
-; EG-NEXT:     MOV * T8.X, PV.W,
-; EG-NEXT:     MOV T0.X, PV.X,
-; EG-NEXT:     AND_INT * T0.W, T20.Z, literal.x,
-; EG-NEXT:    65535(9.183409e-41), 0(0.000000e+00)
-; EG-NEXT:     BCNT_INT T0.W, PV.W,
-; EG-NEXT:     AND_INT * T1.W, PV.X, literal.x,
-; EG-NEXT:    -65536(nan), 0(0.000000e+00)
-; EG-NEXT:     OR_INT * T0.W, PS, PV.W,
-; EG-NEXT:     MOV T8.X, PV.W,
-; EG-NEXT:     MOV * T0.X, T9.X,
-; EG-NEXT:     LSHR * T0.W, T20.W, literal.x,
+; EG-NEXT:    ALU clause starting at 13:
+; EG-NEXT:     LSHR * T0.W, T12.Z, literal.x,
 ; EG-NEXT:    16(2.242078e-44), 0(0.000000e+00)
-; EG-NEXT:     BCNT_INT T0.W, PV.W,
-; EG-NEXT:     AND_INT * T1.W, T0.X, literal.x,
+; EG-NEXT:     BCNT_INT T12.W, PV.W,
+; EG-NEXT:     AND_INT * T0.W, T12.Z, literal.x,
 ; EG-NEXT:    65535(9.183409e-41), 0(0.000000e+00)
-; EG-NEXT:     LSHL * T0.W, PV.W, literal.x,
+; EG-NEXT:     BCNT_INT T12.Z, PS,
+; EG-NEXT:     LSHR T0.W, T0.Z, literal.x,
+; EG-NEXT:     LSHR * T1.W, T12.X, literal.x,
 ; EG-NEXT:    16(2.242078e-44), 0(0.000000e+00)
-; EG-NEXT:     OR_INT * T0.W, T1.W, PV.W,
-; EG-NEXT:     MOV * T9.X, PV.W,
-; EG-NEXT:     MOV T0.X, PV.X,
-; EG-NEXT:     AND_INT * T0.W, T20.W, literal.x,
-; EG-NEXT:    65535(9.183409e-41), 0(0.000000e+00)
+; EG-NEXT:     BCNT_INT T12.Y, PS,
+; EG-NEXT:     AND_INT T0.Z, T0.Z, literal.x,
 ; EG-NEXT:     BCNT_INT T0.W, PV.W,
-; EG-NEXT:     AND_INT * T1.W, PV.X, literal.x,
-; EG-NEXT:    -65536(nan), 0(0.000000e+00)
-; EG-NEXT:     OR_INT * T0.W, PS, PV.W,
-; EG-NEXT:     MOV T9.X, PV.W,
-; EG-NEXT:     MOV * T0.X, T12.X,
-; EG-NEXT:     LSHR * T1.W, T21.X, literal.x,
-; EG-NEXT:    16(2.242078e-44), 0(0.000000e+00)
-; EG-NEXT:     BCNT_INT T1.W, PV.W,
-; EG-NEXT:     AND_INT * T2.W, T0.X, literal.x,
-; EG-NEXT:    65535(9.183409e-41), 0(0.000000e+00)
-; EG-NEXT:     LSHL * T1.W, PV.W, literal.x,
-; EG-NEXT:    16(2.242078e-44), 0(0.000000e+00)
-; EG-NEXT:     OR_INT * T1.W, T2.W, PV.W,
-; EG-NEXT:     MOV * T12.X, PV.W,
-; EG-NEXT:     MOV T0.X, PV.X,
-; EG-NEXT:     AND_INT * T1.W, T21.X, literal.x,
-; EG-NEXT:    65535(9.183409e-41), 0(0.000000e+00)
-; EG-NEXT:     BCNT_INT T1.W, PV.W,
-; EG-NEXT:     AND_INT * T2.W, PV.X, literal.x,
-; EG-NEXT:    -65536(nan), 0(0.000000e+00)
-; EG-NEXT:     OR_INT * T1.W, PS, PV.W,
-; EG-NEXT:     MOV T12.X, PV.W,
-; EG-NEXT:     MOV * T0.X, T13.X,
-; EG-NEXT:     LSHR * T1.W, T21.Y, literal.x,
-; EG-NEXT:    16(2.242078e-44), 0(0.000000e+00)
-; EG-NEXT:     BCNT_INT T1.W, PV.W,
-; EG-NEXT:     AND_INT * T2.W, T0.X, literal.x,
-; EG-NEXT:    65535(9.183409e-41), 0(0.000000e+00)
-; EG-NEXT:     LSHL * T1.W, PV.W, literal.x,
-; EG-NEXT:    16(2.242078e-44), 0(0.000000e+00)
-; EG-NEXT:     OR_INT * T1.W, T2.W, PV.W,
-; EG-NEXT:     MOV * T13.X, PV.W,
-; EG-NEXT:     MOV T0.X, PV.X,
-; EG-NEXT:     AND_INT * T1.W, T21.Y, literal.x,
-; EG-NEXT:    65535(9.183409e-41), 0(0.000000e+00)
-; EG-NEXT:     BCNT_INT T1.W, PV.W,
-; EG-NEXT:     AND_INT * T2.W, PV.X, literal.x,
-; EG-NEXT:    -65536(nan), 0(0.000000e+00)
-; EG-NEXT:     OR_INT * T20.Y, PS, PV.W,
-; EG-NEXT:     MOV T13.X, PV.Y,
-; EG-NEXT:     MOV * T0.X, T16.X,
-; EG-NEXT:     LSHR * T1.W, T21.Z, literal.x,
-; EG-NEXT:    16(2.242078e-44), 0(0.000000e+00)
-; EG-NEXT:     BCNT_INT T1.W, PV.W,
-; EG-NEXT:     AND_INT * T2.W, T0.X, literal.x,
-; EG-NEXT:    65535(9.183409e-41), 0(0.000000e+00)
-; EG-NEXT:     LSHL * T1.W, PV.W, literal.x,
-; EG-NEXT:    16(2.242078e-44), 0(0.000000e+00)
-; EG-NEXT:     OR_INT * T1.W, T2.W, PV.W,
-; EG-NEXT:    ALU clause starting at 131:
-; EG-NEXT:     MOV * T16.X, T1.W,
-; EG-NEXT:     MOV T0.X, PV.X,
-; EG-NEXT:     AND_INT * T1.W, T21.Z, literal.x,
-; EG-NEXT:    65535(9.183409e-41), 0(0.000000e+00)
-; EG-NEXT:     BCNT_INT T1.W, PV.W,
-; EG-NEXT:     AND_INT * T2.W, PV.X, literal.x,
-; EG-NEXT:    -65536(nan), 0(0.000000e+00)
-; EG-NEXT:     OR_INT * T1.W, PS, PV.W,
-; EG-NEXT:     MOV T16.X, PV.W,
-; EG-NEXT:     MOV * T0.X, T17.X,
-; EG-NEXT:     LSHR * T1.W, T21.W, literal.x,
-; EG-NEXT:    16(2.242078e-44), 0(0.000000e+00)
-; EG-NEXT:     BCNT_INT T1.W, PV.W,
-; EG-NEXT:     AND_INT * T2.W, T0.X, literal.x,
+; EG-NEXT:     AND_INT * T1.W, T12.X, literal.x,
 ; EG-NEXT:    65535(9.183409e-41), 0(0.000000e+00)
-; EG-NEXT:     LSHL * T1.W, PV.W, literal.x,
+; EG-NEXT:     BCNT_INT T12.X, PS,
+; EG-NEXT:     BCNT_INT T0.Z, PV.Z,
+; EG-NEXT:     LSHR T1.W, T0.X, literal.x,
+; EG-NEXT:     ADD_INT * T2.W, KC0[2].Y, literal.x,
 ; EG-NEXT:    16(2.242078e-44), 0(0.000000e+00)
-; EG-NEXT:     OR_INT * T1.W, T2.W, PV.W,
-; EG-NEXT:     MOV * T17.X, PV.W,
-; EG-NEXT:     MOV T0.X, PV.X,
-; EG-NEXT:     AND_INT T1.W, T21.W, literal.x,
-; EG-NEXT:     LSHR * T21.X, KC0[2].Y, literal.y,
-; EG-NEXT:    65535(9.183409e-41), 2(2.802597e-45)
-; EG-NEXT:     AND_INT T0.Z, PV.X, literal.x,
-; EG-NEXT:     BCNT_INT T1.W, PV.W,
-; EG-NEXT:     ADD_INT * T2.W, KC0[2].Y, literal.y,
-; EG-NEXT:    -65536(nan), 16(2.242078e-44)
-; EG-NEXT:     LSHR T22.X, PS, literal.x,
-; EG-NEXT:     OR_INT * T20.W, PV.Z, PV.W,
+; EG-NEXT:     LSHR T13.X, PS, literal.x,
+; EG-NEXT:     BCNT_INT T0.Y, PV.W,
+; EG-NEXT:     AND_INT * T1.W, T0.X, literal.y,
+; EG-NEXT:    2(2.802597e-45), 65535(9.183409e-41)
+; EG-NEXT:     BCNT_INT T0.X, PV.W,
+; EG-NEXT:     LSHR * T14.X, KC0[2].Y, literal.x,
 ; EG-NEXT:    2(2.802597e-45), 0(0.000000e+00)
-; EG-NEXT:     MOV T17.X, PV.W,
-; EG-NEXT:     MOV * T0.X, T4.X,
-; EG-NEXT:     MOV * T0.Z, T8.X,
-; EG-NEXT:     MOV T20.X, T12.X,
-; EG-NEXT:     MOV * T20.Z, T16.X, BS:VEC_120/SCL_212
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
   %in.gep = getelementptr <16 x i16>, ptr addrspace(1) %in, i32 %tid
   %val = load <16 x i16>, ptr addrspace(1) %in.gep, align 32
diff --git a/llvm/test/CodeGen/AMDGPU/ctpop64.ll b/llvm/test/CodeGen/AMDGPU/ctpop64.ll
index 3504546801c93..2258f6a7b5483 100644
--- a/llvm/test/CodeGen/AMDGPU/ctpop64.ll
+++ b/llvm/test/CodeGen/AMDGPU/ctpop64.ll
@@ -334,58 +334,58 @@ define amdgpu_kernel void @v_ctpop_v4i64(ptr addrspace(1) noalias %out, ptr addr
 define amdgpu_kernel void @ctpop_i64_in_br(ptr addrspace(1) %out, ptr addrspace(1) %in, i64 %ctpop_arg, i32 %cond) {
 ; SI-LABEL: ctpop_i64_in_br:
 ; SI:       ; %bb.0: ; %entry
-; SI-NEXT:    s_load_dword s8, s[4:5], 0xf
+; SI-NEXT:    s_load_dword s6, s[4:5], 0xf
 ; SI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
-; SI-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0xd
+; SI-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0xd
 ; SI-NEXT:    s_waitcnt lgkmcnt(0)
-; SI-NEXT:    s_cmp_lg_u32 s8, 0
+; SI-NEXT:    s_cmp_lg_u32 s6, 0
 ; SI-NEXT:    s_cbranch_scc0 .LBB7_4
 ; SI-NEXT:  ; %bb.1: ; %else
-; SI-NEXT:    s_load_dwordx2 s[4:5], s[2:3], 0x2
+; SI-NEXT:    s_load_dwordx2 s[6:7], s[2:3], 0x2
 ; SI-NEXT:    s_mov_b64 s[2:3], 0
 ; SI-NEXT:    s_andn2_b64 vcc, exec, s[2:3]
 ; SI-NEXT:    s_waitcnt lgkmcnt(0)
 ; SI-NEXT:    s_mov_b64 vcc, vcc
 ; SI-NEXT:    s_cbranch_vccnz .LBB7_3
 ; SI-NEXT:  .LBB7_2: ; %if
-; SI-NEXT:    s_bcnt1_i32_b64 s4, s[6:7]
-; SI-NEXT:    s_mov_b32 s5, 0
+; SI-NEXT:    s_bcnt1_i32_b64 s6, s[4:5]
+; SI-NEXT:    s_mov_b32 s7, 0
 ; SI-NEXT:  .LBB7_3: ; %endif
-; SI-NEXT:    v_mov_b32_e32 v0, s4
+; SI-NEXT:    v_mov_b32_e32 v0, s6
 ; SI-NEXT:    s_mov_b32 s3, 0xf000
 ; SI-NEXT:    s_mov_b32 s2, -1
-; SI-NEXT:    v_mov_b32_e32 v1, s5
+; SI-NEXT:    v_mov_b32_e32 v1, s7
 ; SI-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
 ; SI-NEXT:    s_endpgm
 ; SI-NEXT:  .LBB7_4:
-; SI-NEXT:    ; implicit-def: $sgpr4_sgpr5
+; SI-NEXT:    ; implicit-def: $sgpr6_sgpr7
 ; SI-NEXT:    s_branch .LBB7_2
 ;
 ; VI-LABEL: ctpop_i64_in_br:
 ; VI:       ; %bb.0: ; %entry
-; VI-NEXT:    s_load_dword s8, s[4:5], 0x3c
+; VI-NEXT:    s_load_dword s6, s[4:5], 0x3c
 ; VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
-; VI-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x34
+; VI-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x34
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
-; VI-NEXT:    s_cmp_lg_u32 s8, 0
+; VI-NEXT:    s_cmp_lg_u32 s6, 0
 ; VI-NEXT:    s_cbranch_scc0 .LBB7_4
 ; VI-NEXT:  ; %bb.1: ; %else
-; VI-NEXT:    s_load_dwordx2 s[4:5], s[2:3], 0x8
+; VI-NEXT:    s_load_dwordx2 s[6:7], s[2:3], 0x8
 ; VI-NEXT:    s_cbranch_execnz .LBB7_3
 ; VI-NEXT:  .LBB7_2: ; %if
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
-; VI-NEXT:    s_bcnt1_i32_b64 s4, s[6:7]
-; VI-NEXT:    s_mov_b32 s5, 0
+; VI-NEXT:    s_bcnt1_i32_b64 s6, s[4:5]
+; VI-NEXT:    s_mov_b32 s7, 0
 ; VI-NEXT:  .LBB7_3: ; %endif
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
-; VI-NEXT:    v_mov_b32_e32 v0, s4
+; VI-NEXT:    v_mov_b32_e32 v0, s6
 ; VI-NEXT:    s_mov_b32 s3, 0xf000
 ; VI-NEXT:    s_mov_b32 s2, -1
-; VI-NEXT:    v_mov_b32_e32 v1, s5
+; VI-NEXT:    v_mov_b32_e32 v1, s7
 ; VI-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
 ; VI-NEXT:    s_endpgm
 ; VI-NEXT:  .LBB7_4:
-; VI-NEXT:    ; implicit-def: $sgpr4_sgpr5
+; VI-NEXT:    ; implicit-def: $sgpr6_sgpr7
 ; VI-NEXT:    s_branch .LBB7_2
 entry:
   %tmp0 = icmp eq i32 %cond, 0
diff --git a/llvm/test/CodeGen/AMDGPU/dagcombine-fmul-sel.ll b/llvm/test/CodeGen/AMDGPU/dagcombine-fmul-sel.ll
index b128be2186df2..935ae48654b64 100644
--- a/llvm/test/CodeGen/AMDGPU/dagcombine-fmul-sel.ll
+++ b/llvm/test/CodeGen/AMDGPU/dagcombine-fmul-sel.ll
@@ -1,12 +1,12 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
 ; RUN: llc -global-isel=0 -mtriple=amdgcn-amd-amdhsa -mcpu=gfx700 < %s | FileCheck -check-prefixes=GFX7,GFX7-SDAG %s
-; RUN: llc -global-isel=1 -mtriple=amdgcn-amd-amdhsa -mcpu=gfx700 < %s | FileCheck -check-prefixes=GFX7,GFX7-GISEL %s
+; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgcn-amd-amdhsa -mcpu=gfx700 < %s | FileCheck -check-prefixes=GFX7,GFX7-GISEL %s
 ; RUN: llc -global-isel=0 -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 < %s | FileCheck -check-prefixes=GFX9,GFX9-SDAG %s
-; RUN: llc -global-isel=1 -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 < %s | FileCheck -check-prefixes=GFX9,GFX9-GISEL %s
+; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 < %s | FileCheck -check-prefixes=GFX9,GFX9-GISEL %s
 ; RUN: llc -global-isel=0 -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1030 < %s | FileCheck -check-prefixes=GFX10,GFX10-SDAG %s
-; RUN: llc -global-isel=1 -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1030 < %s | FileCheck -check-prefixes=GFX10,GFX10-GISEL %s
+; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1030 < %s | FileCheck -check-prefixes=GFX10,GFX10-GISEL %s
 ; RUN: llc -global-isel=0 -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1100 < %s | FileCheck -check-prefixes=GFX11,GFX11-SDAG %s
-; RUN: llc -global-isel=1 -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1100 < %s | FileCheck -check-prefixes=GFX11,GFX11-GISEL %s
+; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1100 < %s | FileCheck -check-prefixes=GFX11,GFX11-GISEL %s
 
 define float @fmul_select_f32_test1(float %x, i32 %bool.arg1, i32 %bool.arg2) {
 ; GFX7-LABEL: fmul_select_f32_test1:
@@ -2541,114 +2541,72 @@ define half @fmul_select_f16_test11_sel_log2val_pos7_neg14(half %x, i32 %bool.ar
 }
 
 define bfloat @fmul_select_bf16_test1(bfloat %x, i32 %bool.arg1, i32 %bool.arg2) {
-; GFX7-SDAG-LABEL: fmul_select_bf16_test1:
-; GFX7-SDAG:       ; %bb.0:
-; GFX7-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-SDAG-NEXT:    v_mul_f32_e32 v0, 1.0, v0
-; GFX7-SDAG-NEXT:    v_cmp_eq_u32_e32 vcc, v1, v2
-; GFX7-SDAG-NEXT:    v_cndmask_b32_e64 v1, 1.0, 2.0, vcc
-; GFX7-SDAG-NEXT:    v_and_b32_e32 v0, 0xffff0000, v0
-; GFX7-SDAG-NEXT:    v_mul_f32_e32 v0, v0, v1
-; GFX7-SDAG-NEXT:    v_and_b32_e32 v0, 0xffff0000, v0
-; GFX7-SDAG-NEXT:    s_setpc_b64 s[30:31]
-;
-; GFX7-GISEL-LABEL: fmul_select_bf16_test1:
-; GFX7-GISEL:       ; %bb.0:
-; GFX7-GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-GISEL-NEXT:    v_cvt_f32_f16_e32 v0, v0
-; GFX7-GISEL-NEXT:    v_cmp_eq_u32_e32 vcc, v1, v2
-; GFX7-GISEL-NEXT:    v_cndmask_b32_e64 v1, 0, 1, vcc
-; GFX7-GISEL-NEXT:    v_ldexp_f32_e32 v0, v0, v1
-; GFX7-GISEL-NEXT:    v_cvt_f16_f32_e32 v0, v0
-; GFX7-GISEL-NEXT:    s_setpc_b64 s[30:31]
-;
-; GFX9-SDAG-LABEL: fmul_select_bf16_test1:
-; GFX9-SDAG:       ; %bb.0:
-; GFX9-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-SDAG-NEXT:    v_mov_b32_e32 v3, 0x3f80
-; GFX9-SDAG-NEXT:    v_mov_b32_e32 v4, 0x4000
-; GFX9-SDAG-NEXT:    v_cmp_eq_u32_e32 vcc, v1, v2
-; GFX9-SDAG-NEXT:    v_cndmask_b32_e32 v1, v3, v4, vcc
-; GFX9-SDAG-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
-; GFX9-SDAG-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
-; GFX9-SDAG-NEXT:    v_mul_f32_e32 v0, v0, v1
-; GFX9-SDAG-NEXT:    v_bfe_u32 v1, v0, 16, 1
-; GFX9-SDAG-NEXT:    s_movk_i32 s4, 0x7fff
-; GFX9-SDAG-NEXT:    v_add3_u32 v1, v1, v0, s4
-; GFX9-SDAG-NEXT:    v_or_b32_e32 v2, 0x400000, v0
-; GFX9-SDAG-NEXT:    v_cmp_u_f32_e32 vcc, v0, v0
-; GFX9-SDAG-NEXT:    v_cndmask_b32_e32 v0, v1, v2, vcc
-; GFX9-SDAG-NEXT:    v_lshrrev_b32_e32 v0, 16, v0
-; GFX9-SDAG-NEXT:    s_setpc_b64 s[30:31]
-;
-; GFX9-GISEL-LABEL: fmul_select_bf16_test1:
-; GFX9-GISEL:       ; %bb.0:
-; GFX9-GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-GISEL-NEXT:    v_cmp_eq_u32_e32 vcc, v1, v2
-; GFX9-GISEL-NEXT:    v_cndmask_b32_e64 v1, 0, 1, vcc
-; GFX9-GISEL-NEXT:    v_mov_b32_e32 v2, 0xffff8000
-; GFX9-GISEL-NEXT:    v_mov_b32_e32 v3, 0x7fff
-; GFX9-GISEL-NEXT:    v_med3_i32 v1, v1, v2, v3
-; GFX9-GISEL-NEXT:    v_ldexp_f16_e32 v0, v0, v1
-; GFX9-GISEL-NEXT:    s_setpc_b64 s[30:31]
-;
-; GFX10-SDAG-LABEL: fmul_select_bf16_test1:
-; GFX10-SDAG:       ; %bb.0:
-; GFX10-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-SDAG-NEXT:    v_mov_b32_e32 v3, 0x4000
-; GFX10-SDAG-NEXT:    v_cmp_eq_u32_e32 vcc_lo, v1, v2
-; GFX10-SDAG-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
-; GFX10-SDAG-NEXT:    v_cndmask_b32_e32 v1, 0x3f80, v3, vcc_lo
-; GFX10-SDAG-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
-; GFX10-SDAG-NEXT:    v_mul_f32_e32 v0, v0, v1
-; GFX10-SDAG-NEXT:    v_bfe_u32 v1, v0, 16, 1
-; GFX10-SDAG-NEXT:    v_or_b32_e32 v2, 0x400000, v0
-; GFX10-SDAG-NEXT:    v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX10-SDAG-NEXT:    v_add3_u32 v1, v1, v0, 0x7fff
-; GFX10-SDAG-NEXT:    v_cndmask_b32_e32 v0, v1, v2, vcc_lo
-; GFX10-SDAG-NEXT:    v_lshrrev_b32_e32 v0, 16, v0
-; GFX10-SDAG-NEXT:    s_setpc_b64 s[30:31]
+; GFX7-LABEL: fmul_select_bf16_test1:
+; GFX7:       ; %bb.0:
+; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    v_mul_f32_e32 v0, 1.0, v0
+; GFX7-NEXT:    v_cmp_eq_u32_e32 vcc, v1, v2
+; GFX7-NEXT:    v_cndmask_b32_e64 v1, 1.0, 2.0, vcc
+; GFX7-NEXT:    v_and_b32_e32 v0, 0xffff0000, v0
+; GFX7-NEXT:    v_mul_f32_e32 v0, v0, v1
+; GFX7-NEXT:    v_and_b32_e32 v0, 0xffff0000, v0
+; GFX7-NEXT:    s_setpc_b64 s[30:31]
 ;
-; GFX10-GISEL-LABEL: fmul_select_bf16_test1:
-; GFX10-GISEL:       ; %bb.0:
-; GFX10-GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-GISEL-NEXT:    v_cmp_eq_u32_e32 vcc_lo, v1, v2
-; GFX10-GISEL-NEXT:    v_mov_b32_e32 v2, 0x7fff
-; GFX10-GISEL-NEXT:    v_cndmask_b32_e64 v1, 0, 1, vcc_lo
-; GFX10-GISEL-NEXT:    v_med3_i32 v1, 0xffff8000, v1, v2
-; GFX10-GISEL-NEXT:    v_ldexp_f16_e32 v0, v0, v1
-; GFX10-GISEL-NEXT:    s_setpc_b64 s[30:31]
+; GFX9-LABEL: fmul_select_bf16_test1:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v3, 0x3f80
+; GFX9-NEXT:    v_mov_b32_e32 v4, 0x4000
+; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, v1, v2
+; GFX9-NEXT:    v_cndmask_b32_e32 v1, v3, v4, vcc
+; GFX9-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
+; GFX9-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
+; GFX9-NEXT:    v_mul_f32_e32 v0, v0, v1
+; GFX9-NEXT:    v_bfe_u32 v1, v0, 16, 1
+; GFX9-NEXT:    s_movk_i32 s4, 0x7fff
+; GFX9-NEXT:    v_add3_u32 v1, v1, v0, s4
+; GFX9-NEXT:    v_or_b32_e32 v2, 0x400000, v0
+; GFX9-NEXT:    v_cmp_u_f32_e32 vcc, v0, v0
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v1, v2, vcc
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v0
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
-; GFX11-SDAG-LABEL: fmul_select_bf16_test1:
-; GFX11-SDAG:       ; %bb.0:
-; GFX11-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-SDAG-NEXT:    v_dual_mov_b32 v3, 0x4000 :: v_dual_lshlrev_b32 v0, 16, v0
-; GFX11-SDAG-NEXT:    v_cmp_eq_u32_e32 vcc_lo, v1, v2
-; GFX11-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SDAG-NEXT:    v_cndmask_b32_e32 v1, 0x3f80, v3, vcc_lo
-; GFX11-SDAG-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
-; GFX11-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SDAG-NEXT:    v_mul_f32_e32 v0, v0, v1
-; GFX11-SDAG-NEXT:    v_bfe_u32 v1, v0, 16, 1
-; GFX11-SDAG-NEXT:    v_or_b32_e32 v2, 0x400000, v0
-; GFX11-SDAG-NEXT:    v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SDAG-NEXT:    v_add3_u32 v1, v1, v0, 0x7fff
-; GFX11-SDAG-NEXT:    v_cndmask_b32_e32 v0, v1, v2, vcc_lo
-; GFX11-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX11-SDAG-NEXT:    v_lshrrev_b32_e32 v0, 16, v0
-; GFX11-SDAG-NEXT:    s_setpc_b64 s[30:31]
+; GFX10-LABEL: fmul_select_bf16_test1:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    v_mov_b32_e32 v3, 0x4000
+; GFX10-NEXT:    v_cmp_eq_u32_e32 vcc_lo, v1, v2
+; GFX10-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
+; GFX10-NEXT:    v_cndmask_b32_e32 v1, 0x3f80, v3, vcc_lo
+; GFX10-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
+; GFX10-NEXT:    v_mul_f32_e32 v0, v0, v1
+; GFX10-NEXT:    v_bfe_u32 v1, v0, 16, 1
+; GFX10-NEXT:    v_or_b32_e32 v2, 0x400000, v0
+; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX10-NEXT:    v_add3_u32 v1, v1, v0, 0x7fff
+; GFX10-NEXT:    v_cndmask_b32_e32 v0, v1, v2, vcc_lo
+; GFX10-NEXT:    v_lshrrev_b32_e32 v0, 16, v0
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
 ;
-; GFX11-GISEL-LABEL: fmul_select_bf16_test1:
-; GFX11-GISEL:       ; %bb.0:
-; GFX11-GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-GISEL-NEXT:    v_cmp_eq_u32_e32 vcc_lo, v1, v2
-; GFX11-GISEL-NEXT:    v_mov_b32_e32 v2, 0x7fff
-; GFX11-GISEL-NEXT:    v_cndmask_b32_e64 v1, 0, 1, vcc_lo
-; GFX11-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-GISEL-NEXT:    v_med3_i32 v1, 0xffff8000, v1, v2
-; GFX11-GISEL-NEXT:    v_ldexp_f16_e32 v0, v0, v1
-; GFX11-GISEL-NEXT:    s_setpc_b64 s[30:31]
+; GFX11-LABEL: fmul_select_bf16_test1:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    v_dual_mov_b32 v3, 0x4000 :: v_dual_lshlrev_b32 v0, 16, v0
+; GFX11-NEXT:    v_cmp_eq_u32_e32 vcc_lo, v1, v2
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_cndmask_b32_e32 v1, 0x3f80, v3, vcc_lo
+; GFX11-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_mul_f32_e32 v0, v0, v1
+; GFX11-NEXT:    v_bfe_u32 v1, v0, 16, 1
+; GFX11-NEXT:    v_or_b32_e32 v2, 0x400000, v0
+; GFX11-NEXT:    v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_add3_u32 v1, v1, v0, 0x7fff
+; GFX11-NEXT:    v_cndmask_b32_e32 v0, v1, v2, vcc_lo
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT:    v_lshrrev_b32_e32 v0, 16, v0
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
   %bool = icmp eq i32 %bool.arg1, %bool.arg2
   %y = select i1 %bool, bfloat 2.000000e+00, bfloat 1.000000e+00
   %ldexp = fmul bfloat %x, %y
@@ -2656,114 +2614,72 @@ define bfloat @fmul_select_bf16_test1(bfloat %x, i32 %bool.arg1, i32 %bool.arg2)
 }
 
 define bfloat @fmul_select_bf16_test2(bfloat %x, i32 %bool.arg1, i32 %bool.arg2) {
-; GFX7-SDAG-LABEL: fmul_select_bf16_test2:
-; GFX7-SDAG:       ; %bb.0:
-; GFX7-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-SDAG-NEXT:    v_mul_f32_e32 v0, 1.0, v0
-; GFX7-SDAG-NEXT:    v_cmp_eq_u32_e32 vcc, v1, v2
-; GFX7-SDAG-NEXT:    v_cndmask_b32_e64 v1, 1.0, 0.5, vcc
-; GFX7-SDAG-NEXT:    v_and_b32_e32 v0, 0xffff0000, v0
-; GFX7-SDAG-NEXT:    v_mul_f32_e32 v0, v0, v1
-; GFX7-SDAG-NEXT:    v_and_b32_e32 v0, 0xffff0000, v0
-; GFX7-SDAG-NEXT:    s_setpc_b64 s[30:31]
-;
-; GFX7-GISEL-LABEL: fmul_select_bf16_test2:
-; GFX7-GISEL:       ; %bb.0:
-; GFX7-GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-GISEL-NEXT:    v_cvt_f32_f16_e32 v0, v0
-; GFX7-GISEL-NEXT:    v_cmp_eq_u32_e32 vcc, v1, v2
-; GFX7-GISEL-NEXT:    v_cndmask_b32_e64 v1, 0, -1, vcc
-; GFX7-GISEL-NEXT:    v_ldexp_f32_e32 v0, v0, v1
-; GFX7-GISEL-NEXT:    v_cvt_f16_f32_e32 v0, v0
-; GFX7-GISEL-NEXT:    s_setpc_b64 s[30:31]
-;
-; GFX9-SDAG-LABEL: fmul_select_bf16_test2:
-; GFX9-SDAG:       ; %bb.0:
-; GFX9-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-SDAG-NEXT:    v_mov_b32_e32 v3, 0x3f80
-; GFX9-SDAG-NEXT:    v_mov_b32_e32 v4, 0x3f00
-; GFX9-SDAG-NEXT:    v_cmp_eq_u32_e32 vcc, v1, v2
-; GFX9-SDAG-NEXT:    v_cndmask_b32_e32 v1, v3, v4, vcc
-; GFX9-SDAG-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
-; GFX9-SDAG-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
-; GFX9-SDAG-NEXT:    v_mul_f32_e32 v0, v0, v1
-; GFX9-SDAG-NEXT:    v_bfe_u32 v1, v0, 16, 1
-; GFX9-SDAG-NEXT:    s_movk_i32 s4, 0x7fff
-; GFX9-SDAG-NEXT:    v_add3_u32 v1, v1, v0, s4
-; GFX9-SDAG-NEXT:    v_or_b32_e32 v2, 0x400000, v0
-; GFX9-SDAG-NEXT:    v_cmp_u_f32_e32 vcc, v0, v0
-; GFX9-SDAG-NEXT:    v_cndmask_b32_e32 v0, v1, v2, vcc
-; GFX9-SDAG-NEXT:    v_lshrrev_b32_e32 v0, 16, v0
-; GFX9-SDAG-NEXT:    s_setpc_b64 s[30:31]
-;
-; GFX9-GISEL-LABEL: fmul_select_bf16_test2:
-; GFX9-GISEL:       ; %bb.0:
-; GFX9-GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-GISEL-NEXT:    v_cmp_eq_u32_e32 vcc, v1, v2
-; GFX9-GISEL-NEXT:    v_cndmask_b32_e64 v1, 0, -1, vcc
-; GFX9-GISEL-NEXT:    v_mov_b32_e32 v2, 0xffff8000
-; GFX9-GISEL-NEXT:    v_mov_b32_e32 v3, 0x7fff
-; GFX9-GISEL-NEXT:    v_med3_i32 v1, v1, v2, v3
-; GFX9-GISEL-NEXT:    v_ldexp_f16_e32 v0, v0, v1
-; GFX9-GISEL-NEXT:    s_setpc_b64 s[30:31]
-;
-; GFX10-SDAG-LABEL: fmul_select_bf16_test2:
-; GFX10-SDAG:       ; %bb.0:
-; GFX10-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-SDAG-NEXT:    v_mov_b32_e32 v3, 0x3f00
-; GFX10-SDAG-NEXT:    v_cmp_eq_u32_e32 vcc_lo, v1, v2
-; GFX10-SDAG-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
-; GFX10-SDAG-NEXT:    v_cndmask_b32_e32 v1, 0x3f80, v3, vcc_lo
-; GFX10-SDAG-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
-; GFX10-SDAG-NEXT:    v_mul_f32_e32 v0, v0, v1
-; GFX10-SDAG-NEXT:    v_bfe_u32 v1, v0, 16, 1
-; GFX10-SDAG-NEXT:    v_or_b32_e32 v2, 0x400000, v0
-; GFX10-SDAG-NEXT:    v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX10-SDAG-NEXT:    v_add3_u32 v1, v1, v0, 0x7fff
-; GFX10-SDAG-NEXT:    v_cndmask_b32_e32 v0, v1, v2, vcc_lo
-; GFX10-SDAG-NEXT:    v_lshrrev_b32_e32 v0, 16, v0
-; GFX10-SDAG-NEXT:    s_setpc_b64 s[30:31]
+; GFX7-LABEL: fmul_select_bf16_test2:
+; GFX7:       ; %bb.0:
+; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    v_mul_f32_e32 v0, 1.0, v0
+; GFX7-NEXT:    v_cmp_eq_u32_e32 vcc, v1, v2
+; GFX7-NEXT:    v_cndmask_b32_e64 v1, 1.0, 0.5, vcc
+; GFX7-NEXT:    v_and_b32_e32 v0, 0xffff0000, v0
+; GFX7-NEXT:    v_mul_f32_e32 v0, v0, v1
+; GFX7-NEXT:    v_and_b32_e32 v0, 0xffff0000, v0
+; GFX7-NEXT:    s_setpc_b64 s[30:31]
 ;
-; GFX10-GISEL-LABEL: fmul_select_bf16_test2:
-; GFX10-GISEL:       ; %bb.0:
-; GFX10-GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-GISEL-NEXT:    v_cmp_eq_u32_e32 vcc_lo, v1, v2
-; GFX10-GISEL-NEXT:    v_mov_b32_e32 v2, 0x7fff
-; GFX10-GISEL-NEXT:    v_cndmask_b32_e64 v1, 0, -1, vcc_lo
-; GFX10-GISEL-NEXT:    v_med3_i32 v1, 0xffff8000, v1, v2
-; GFX10-GISEL-NEXT:    v_ldexp_f16_e32 v0, v0, v1
-; GFX10-GISEL-NEXT:    s_setpc_b64 s[30:31]
+; GFX9-LABEL: fmul_select_bf16_test2:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v3, 0x3f80
+; GFX9-NEXT:    v_mov_b32_e32 v4, 0x3f00
+; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, v1, v2
+; GFX9-NEXT:    v_cndmask_b32_e32 v1, v3, v4, vcc
+; GFX9-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
+; GFX9-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
+; GFX9-NEXT:    v_mul_f32_e32 v0, v0, v1
+; GFX9-NEXT:    v_bfe_u32 v1, v0, 16, 1
+; GFX9-NEXT:    s_movk_i32 s4, 0x7fff
+; GFX9-NEXT:    v_add3_u32 v1, v1, v0, s4
+; GFX9-NEXT:    v_or_b32_e32 v2, 0x400000, v0
+; GFX9-NEXT:    v_cmp_u_f32_e32 vcc, v0, v0
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v1, v2, vcc
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v0
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
-; GFX11-SDAG-LABEL: fmul_select_bf16_test2:
-; GFX11-SDAG:       ; %bb.0:
-; GFX11-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-SDAG-NEXT:    v_dual_mov_b32 v3, 0x3f00 :: v_dual_lshlrev_b32 v0, 16, v0
-; GFX11-SDAG-NEXT:    v_cmp_eq_u32_e32 vcc_lo, v1, v2
-; GFX11-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SDAG-NEXT:    v_cndmask_b32_e32 v1, 0x3f80, v3, vcc_lo
-; GFX11-SDAG-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
-; GFX11-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SDAG-NEXT:    v_mul_f32_e32 v0, v0, v1
-; GFX11-SDAG-NEXT:    v_bfe_u32 v1, v0, 16, 1
-; GFX11-SDAG-NEXT:    v_or_b32_e32 v2, 0x400000, v0
-; GFX11-SDAG-NEXT:    v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SDAG-NEXT:    v_add3_u32 v1, v1, v0, 0x7fff
-; GFX11-SDAG-NEXT:    v_cndmask_b32_e32 v0, v1, v2, vcc_lo
-; GFX11-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX11-SDAG-NEXT:    v_lshrrev_b32_e32 v0, 16, v0
-; GFX11-SDAG-NEXT:    s_setpc_b64 s[30:31]
+; GFX10-LABEL: fmul_select_bf16_test2:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    v_mov_b32_e32 v3, 0x3f00
+; GFX10-NEXT:    v_cmp_eq_u32_e32 vcc_lo, v1, v2
+; GFX10-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
+; GFX10-NEXT:    v_cndmask_b32_e32 v1, 0x3f80, v3, vcc_lo
+; GFX10-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
+; GFX10-NEXT:    v_mul_f32_e32 v0, v0, v1
+; GFX10-NEXT:    v_bfe_u32 v1, v0, 16, 1
+; GFX10-NEXT:    v_or_b32_e32 v2, 0x400000, v0
+; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX10-NEXT:    v_add3_u32 v1, v1, v0, 0x7fff
+; GFX10-NEXT:    v_cndmask_b32_e32 v0, v1, v2, vcc_lo
+; GFX10-NEXT:    v_lshrrev_b32_e32 v0, 16, v0
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
 ;
-; GFX11-GISEL-LABEL: fmul_select_bf16_test2:
-; GFX11-GISEL:       ; %bb.0:
-; GFX11-GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-GISEL-NEXT:    v_cmp_eq_u32_e32 vcc_lo, v1, v2
-; GFX11-GISEL-NEXT:    v_mov_b32_e32 v2, 0x7fff
-; GFX11-GISEL-NEXT:    v_cndmask_b32_e64 v1, 0, -1, vcc_lo
-; GFX11-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-GISEL-NEXT:    v_med3_i32 v1, 0xffff8000, v1, v2
-; GFX11-GISEL-NEXT:    v_ldexp_f16_e32 v0, v0, v1
-; GFX11-GISEL-NEXT:    s_setpc_b64 s[30:31]
+; GFX11-LABEL: fmul_select_bf16_test2:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    v_dual_mov_b32 v3, 0x3f00 :: v_dual_lshlrev_b32 v0, 16, v0
+; GFX11-NEXT:    v_cmp_eq_u32_e32 vcc_lo, v1, v2
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_cndmask_b32_e32 v1, 0x3f80, v3, vcc_lo
+; GFX11-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_mul_f32_e32 v0, v0, v1
+; GFX11-NEXT:    v_bfe_u32 v1, v0, 16, 1
+; GFX11-NEXT:    v_or_b32_e32 v2, 0x400000, v0
+; GFX11-NEXT:    v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_add3_u32 v1, v1, v0, 0x7fff
+; GFX11-NEXT:    v_cndmask_b32_e32 v0, v1, v2, vcc_lo
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT:    v_lshrrev_b32_e32 v0, 16, v0
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
   %bool = icmp eq i32 %bool.arg1, %bool.arg2
   %y = select i1 %bool, bfloat 5.000000e-01, bfloat 1.000000e+00
   %ldexp = fmul bfloat %x, %y
@@ -2771,158 +2687,111 @@ define bfloat @fmul_select_bf16_test2(bfloat %x, i32 %bool.arg1, i32 %bool.arg2)
 }
 
 define <2 x bfloat> @fmul_select_v2bf16_test3(<2 x bfloat> %x, <2 x i32> %bool.arg1, <2 x i32> %bool.arg2) {
-; GFX7-SDAG-LABEL: fmul_select_v2bf16_test3:
-; GFX7-SDAG:       ; %bb.0:
-; GFX7-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-SDAG-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v4
-; GFX7-SDAG-NEXT:    v_mul_f32_e32 v0, 1.0, v0
-; GFX7-SDAG-NEXT:    v_mul_f32_e32 v1, 1.0, v1
-; GFX7-SDAG-NEXT:    v_cndmask_b32_e64 v2, 1.0, 2.0, vcc
-; GFX7-SDAG-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v5
-; GFX7-SDAG-NEXT:    v_cndmask_b32_e64 v3, 1.0, 2.0, vcc
-; GFX7-SDAG-NEXT:    v_and_b32_e32 v1, 0xffff0000, v1
-; GFX7-SDAG-NEXT:    v_and_b32_e32 v0, 0xffff0000, v0
-; GFX7-SDAG-NEXT:    v_mul_f32_e32 v1, v1, v3
-; GFX7-SDAG-NEXT:    v_mul_f32_e32 v0, v0, v2
-; GFX7-SDAG-NEXT:    v_and_b32_e32 v0, 0xffff0000, v0
-; GFX7-SDAG-NEXT:    v_and_b32_e32 v1, 0xffff0000, v1
-; GFX7-SDAG-NEXT:    s_setpc_b64 s[30:31]
-;
-; GFX7-GISEL-LABEL: fmul_select_v2bf16_test3:
-; GFX7-GISEL:       ; %bb.0:
-; GFX7-GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-GISEL-NEXT:    v_cvt_f32_f16_e32 v0, v0
-; GFX7-GISEL-NEXT:    v_cvt_f32_f16_e32 v1, v1
-; GFX7-GISEL-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v4
-; GFX7-GISEL-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc
-; GFX7-GISEL-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v5
-; GFX7-GISEL-NEXT:    v_cndmask_b32_e64 v3, 0, 1, vcc
-; GFX7-GISEL-NEXT:    v_ldexp_f32_e32 v0, v0, v2
-; GFX7-GISEL-NEXT:    v_ldexp_f32_e32 v1, v1, v3
-; GFX7-GISEL-NEXT:    v_cvt_f16_f32_e32 v0, v0
-; GFX7-GISEL-NEXT:    v_cvt_f16_f32_e32 v1, v1
-; GFX7-GISEL-NEXT:    s_setpc_b64 s[30:31]
-;
-; GFX9-SDAG-LABEL: fmul_select_v2bf16_test3:
-; GFX9-SDAG:       ; %bb.0:
-; GFX9-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-SDAG-NEXT:    v_mov_b32_e32 v5, 0x3f80
-; GFX9-SDAG-NEXT:    v_mov_b32_e32 v6, 0x4000
-; GFX9-SDAG-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v4
-; GFX9-SDAG-NEXT:    v_cndmask_b32_e32 v2, v5, v6, vcc
-; GFX9-SDAG-NEXT:    v_cmp_eq_u32_e32 vcc, v1, v3
-; GFX9-SDAG-NEXT:    v_cndmask_b32_e32 v1, v5, v6, vcc
-; GFX9-SDAG-NEXT:    v_lshlrev_b32_e32 v3, 16, v0
-; GFX9-SDAG-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
-; GFX9-SDAG-NEXT:    v_mul_f32_e32 v1, v3, v1
-; GFX9-SDAG-NEXT:    v_and_b32_e32 v0, 0xffff0000, v0
-; GFX9-SDAG-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
-; GFX9-SDAG-NEXT:    v_bfe_u32 v3, v1, 16, 1
-; GFX9-SDAG-NEXT:    s_movk_i32 s4, 0x7fff
-; GFX9-SDAG-NEXT:    v_mul_f32_e32 v0, v0, v2
-; GFX9-SDAG-NEXT:    v_add3_u32 v3, v3, v1, s4
-; GFX9-SDAG-NEXT:    v_or_b32_e32 v4, 0x400000, v1
-; GFX9-SDAG-NEXT:    v_cmp_u_f32_e32 vcc, v1, v1
-; GFX9-SDAG-NEXT:    v_bfe_u32 v2, v0, 16, 1
-; GFX9-SDAG-NEXT:    v_cndmask_b32_e32 v1, v3, v4, vcc
-; GFX9-SDAG-NEXT:    v_add3_u32 v2, v2, v0, s4
-; GFX9-SDAG-NEXT:    v_or_b32_e32 v3, 0x400000, v0
-; GFX9-SDAG-NEXT:    v_cmp_u_f32_e32 vcc, v0, v0
-; GFX9-SDAG-NEXT:    v_cndmask_b32_e32 v0, v2, v3, vcc
-; GFX9-SDAG-NEXT:    s_mov_b32 s4, 0x7060302
-; GFX9-SDAG-NEXT:    v_perm_b32 v0, v0, v1, s4
-; GFX9-SDAG-NEXT:    s_setpc_b64 s[30:31]
-;
-; GFX9-GISEL-LABEL: fmul_select_v2bf16_test3:
-; GFX9-GISEL:       ; %bb.0:
-; GFX9-GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-GISEL-NEXT:    v_cmp_eq_u32_e32 vcc, v1, v3
-; GFX9-GISEL-NEXT:    v_cndmask_b32_e64 v1, 0, 1, vcc
-; GFX9-GISEL-NEXT:    v_mov_b32_e32 v2, 0xffff8000
-; GFX9-GISEL-NEXT:    v_mov_b32_e32 v3, 0x7fff
-; GFX9-GISEL-NEXT:    v_med3_i32 v1, v1, v2, v3
-; GFX9-GISEL-NEXT:    v_ldexp_f16_e32 v0, v0, v1
-; GFX9-GISEL-NEXT:    s_setpc_b64 s[30:31]
-;
-; GFX10-SDAG-LABEL: fmul_select_v2bf16_test3:
-; GFX10-SDAG:       ; %bb.0:
-; GFX10-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-SDAG-NEXT:    v_mov_b32_e32 v5, 0x4000
-; GFX10-SDAG-NEXT:    v_cmp_eq_u32_e32 vcc_lo, v1, v3
-; GFX10-SDAG-NEXT:    v_lshlrev_b32_e32 v3, 16, v0
-; GFX10-SDAG-NEXT:    v_and_b32_e32 v0, 0xffff0000, v0
-; GFX10-SDAG-NEXT:    v_cndmask_b32_e32 v1, 0x3f80, v5, vcc_lo
-; GFX10-SDAG-NEXT:    v_cmp_eq_u32_e32 vcc_lo, v2, v4
-; GFX10-SDAG-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
-; GFX10-SDAG-NEXT:    v_cndmask_b32_e32 v2, 0x3f80, v5, vcc_lo
-; GFX10-SDAG-NEXT:    v_mul_f32_e32 v1, v3, v1
-; GFX10-SDAG-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
-; GFX10-SDAG-NEXT:    v_or_b32_e32 v4, 0x400000, v1
-; GFX10-SDAG-NEXT:    v_mul_f32_e32 v0, v0, v2
-; GFX10-SDAG-NEXT:    v_bfe_u32 v2, v1, 16, 1
-; GFX10-SDAG-NEXT:    v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX10-SDAG-NEXT:    v_bfe_u32 v3, v0, 16, 1
-; GFX10-SDAG-NEXT:    v_add3_u32 v2, v2, v1, 0x7fff
-; GFX10-SDAG-NEXT:    v_or_b32_e32 v5, 0x400000, v0
-; GFX10-SDAG-NEXT:    v_add3_u32 v3, v3, v0, 0x7fff
-; GFX10-SDAG-NEXT:    v_cndmask_b32_e32 v1, v2, v4, vcc_lo
-; GFX10-SDAG-NEXT:    v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX10-SDAG-NEXT:    v_cndmask_b32_e32 v0, v3, v5, vcc_lo
-; GFX10-SDAG-NEXT:    v_perm_b32 v0, v0, v1, 0x7060302
-; GFX10-SDAG-NEXT:    s_setpc_b64 s[30:31]
+; GFX7-LABEL: fmul_select_v2bf16_test3:
+; GFX7:       ; %bb.0:
+; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v4
+; GFX7-NEXT:    v_mul_f32_e32 v0, 1.0, v0
+; GFX7-NEXT:    v_mul_f32_e32 v1, 1.0, v1
+; GFX7-NEXT:    v_cndmask_b32_e64 v2, 1.0, 2.0, vcc
+; GFX7-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX7-NEXT:    v_cndmask_b32_e64 v3, 1.0, 2.0, vcc
+; GFX7-NEXT:    v_and_b32_e32 v1, 0xffff0000, v1
+; GFX7-NEXT:    v_and_b32_e32 v0, 0xffff0000, v0
+; GFX7-NEXT:    v_mul_f32_e32 v1, v1, v3
+; GFX7-NEXT:    v_mul_f32_e32 v0, v0, v2
+; GFX7-NEXT:    v_and_b32_e32 v0, 0xffff0000, v0
+; GFX7-NEXT:    v_and_b32_e32 v1, 0xffff0000, v1
+; GFX7-NEXT:    s_setpc_b64 s[30:31]
 ;
-; GFX10-GISEL-LABEL: fmul_select_v2bf16_test3:
-; GFX10-GISEL:       ; %bb.0:
-; GFX10-GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-GISEL-NEXT:    v_cmp_eq_u32_e32 vcc_lo, v1, v3
-; GFX10-GISEL-NEXT:    v_mov_b32_e32 v2, 0x7fff
-; GFX10-GISEL-NEXT:    v_cndmask_b32_e64 v1, 0, 1, vcc_lo
-; GFX10-GISEL-NEXT:    v_med3_i32 v1, 0xffff8000, v1, v2
-; GFX10-GISEL-NEXT:    v_ldexp_f16_e32 v0, v0, v1
-; GFX10-GISEL-NEXT:    s_setpc_b64 s[30:31]
+; GFX9-LABEL: fmul_select_v2bf16_test3:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v5, 0x3f80
+; GFX9-NEXT:    v_mov_b32_e32 v6, 0x4000
+; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v4
+; GFX9-NEXT:    v_cndmask_b32_e32 v2, v5, v6, vcc
+; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, v1, v3
+; GFX9-NEXT:    v_cndmask_b32_e32 v1, v5, v6, vcc
+; GFX9-NEXT:    v_lshlrev_b32_e32 v3, 16, v0
+; GFX9-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
+; GFX9-NEXT:    v_mul_f32_e32 v1, v3, v1
+; GFX9-NEXT:    v_and_b32_e32 v0, 0xffff0000, v0
+; GFX9-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GFX9-NEXT:    v_bfe_u32 v3, v1, 16, 1
+; GFX9-NEXT:    s_movk_i32 s4, 0x7fff
+; GFX9-NEXT:    v_mul_f32_e32 v0, v0, v2
+; GFX9-NEXT:    v_add3_u32 v3, v3, v1, s4
+; GFX9-NEXT:    v_or_b32_e32 v4, 0x400000, v1
+; GFX9-NEXT:    v_cmp_u_f32_e32 vcc, v1, v1
+; GFX9-NEXT:    v_bfe_u32 v2, v0, 16, 1
+; GFX9-NEXT:    v_cndmask_b32_e32 v1, v3, v4, vcc
+; GFX9-NEXT:    v_add3_u32 v2, v2, v0, s4
+; GFX9-NEXT:    v_or_b32_e32 v3, 0x400000, v0
+; GFX9-NEXT:    v_cmp_u_f32_e32 vcc, v0, v0
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v2, v3, vcc
+; GFX9-NEXT:    s_mov_b32 s4, 0x7060302
+; GFX9-NEXT:    v_perm_b32 v0, v0, v1, s4
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
-; GFX11-SDAG-LABEL: fmul_select_v2bf16_test3:
-; GFX11-SDAG:       ; %bb.0:
-; GFX11-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-SDAG-NEXT:    v_mov_b32_e32 v5, 0x4000
-; GFX11-SDAG-NEXT:    v_cmp_eq_u32_e32 vcc_lo, v1, v3
-; GFX11-SDAG-NEXT:    v_lshlrev_b32_e32 v3, 16, v0
-; GFX11-SDAG-NEXT:    v_and_b32_e32 v0, 0xffff0000, v0
-; GFX11-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX11-SDAG-NEXT:    v_cndmask_b32_e32 v1, 0x3f80, v5, vcc_lo
-; GFX11-SDAG-NEXT:    v_cmp_eq_u32_e32 vcc_lo, v2, v4
-; GFX11-SDAG-NEXT:    v_cndmask_b32_e32 v2, 0x3f80, v5, vcc_lo
-; GFX11-SDAG-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
-; GFX11-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SDAG-NEXT:    v_dual_mul_f32 v0, v0, v2 :: v_dual_lshlrev_b32 v1, 16, v1
-; GFX11-SDAG-NEXT:    v_or_b32_e32 v5, 0x400000, v0
-; GFX11-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-SDAG-NEXT:    v_mul_f32_e32 v1, v3, v1
-; GFX11-SDAG-NEXT:    v_bfe_u32 v3, v0, 16, 1
-; GFX11-SDAG-NEXT:    v_bfe_u32 v2, v1, 16, 1
-; GFX11-SDAG-NEXT:    v_or_b32_e32 v4, 0x400000, v1
-; GFX11-SDAG-NEXT:    v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX11-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-SDAG-NEXT:    v_add3_u32 v3, v3, v0, 0x7fff
-; GFX11-SDAG-NEXT:    v_add3_u32 v2, v2, v1, 0x7fff
-; GFX11-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-SDAG-NEXT:    v_cndmask_b32_e32 v1, v2, v4, vcc_lo
-; GFX11-SDAG-NEXT:    v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-SDAG-NEXT:    v_cndmask_b32_e32 v0, v3, v5, vcc_lo
-; GFX11-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX11-SDAG-NEXT:    v_perm_b32 v0, v0, v1, 0x7060302
-; GFX11-SDAG-NEXT:    s_setpc_b64 s[30:31]
+; GFX10-LABEL: fmul_select_v2bf16_test3:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    v_mov_b32_e32 v5, 0x4000
+; GFX10-NEXT:    v_cmp_eq_u32_e32 vcc_lo, v1, v3
+; GFX10-NEXT:    v_lshlrev_b32_e32 v3, 16, v0
+; GFX10-NEXT:    v_and_b32_e32 v0, 0xffff0000, v0
+; GFX10-NEXT:    v_cndmask_b32_e32 v1, 0x3f80, v5, vcc_lo
+; GFX10-NEXT:    v_cmp_eq_u32_e32 vcc_lo, v2, v4
+; GFX10-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
+; GFX10-NEXT:    v_cndmask_b32_e32 v2, 0x3f80, v5, vcc_lo
+; GFX10-NEXT:    v_mul_f32_e32 v1, v3, v1
+; GFX10-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GFX10-NEXT:    v_or_b32_e32 v4, 0x400000, v1
+; GFX10-NEXT:    v_mul_f32_e32 v0, v0, v2
+; GFX10-NEXT:    v_bfe_u32 v2, v1, 16, 1
+; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX10-NEXT:    v_bfe_u32 v3, v0, 16, 1
+; GFX10-NEXT:    v_add3_u32 v2, v2, v1, 0x7fff
+; GFX10-NEXT:    v_or_b32_e32 v5, 0x400000, v0
+; GFX10-NEXT:    v_add3_u32 v3, v3, v0, 0x7fff
+; GFX10-NEXT:    v_cndmask_b32_e32 v1, v2, v4, vcc_lo
+; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX10-NEXT:    v_cndmask_b32_e32 v0, v3, v5, vcc_lo
+; GFX10-NEXT:    v_perm_b32 v0, v0, v1, 0x7060302
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
 ;
-; GFX11-GISEL-LABEL: fmul_select_v2bf16_test3:
-; GFX11-GISEL:       ; %bb.0:
-; GFX11-GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-GISEL-NEXT:    v_cmp_eq_u32_e32 vcc_lo, v1, v3
-; GFX11-GISEL-NEXT:    v_mov_b32_e32 v2, 0x7fff
-; GFX11-GISEL-NEXT:    v_cndmask_b32_e64 v1, 0, 1, vcc_lo
-; GFX11-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-GISEL-NEXT:    v_med3_i32 v1, 0xffff8000, v1, v2
-; GFX11-GISEL-NEXT:    v_ldexp_f16_e32 v0, v0, v1
-; GFX11-GISEL-NEXT:    s_setpc_b64 s[30:31]
+; GFX11-LABEL: fmul_select_v2bf16_test3:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    v_mov_b32_e32 v5, 0x4000
+; GFX11-NEXT:    v_cmp_eq_u32_e32 vcc_lo, v1, v3
+; GFX11-NEXT:    v_lshlrev_b32_e32 v3, 16, v0
+; GFX11-NEXT:    v_and_b32_e32 v0, 0xffff0000, v0
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_cndmask_b32_e32 v1, 0x3f80, v5, vcc_lo
+; GFX11-NEXT:    v_cmp_eq_u32_e32 vcc_lo, v2, v4
+; GFX11-NEXT:    v_cndmask_b32_e32 v2, 0x3f80, v5, vcc_lo
+; GFX11-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_dual_mul_f32 v0, v0, v2 :: v_dual_lshlrev_b32 v1, 16, v1
+; GFX11-NEXT:    v_or_b32_e32 v5, 0x400000, v0
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-NEXT:    v_mul_f32_e32 v1, v3, v1
+; GFX11-NEXT:    v_bfe_u32 v3, v0, 16, 1
+; GFX11-NEXT:    v_bfe_u32 v2, v1, 16, 1
+; GFX11-NEXT:    v_or_b32_e32 v4, 0x400000, v1
+; GFX11-NEXT:    v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-NEXT:    v_add3_u32 v3, v3, v0, 0x7fff
+; GFX11-NEXT:    v_add3_u32 v2, v2, v1, 0x7fff
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-NEXT:    v_cndmask_b32_e32 v1, v2, v4, vcc_lo
+; GFX11-NEXT:    v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-NEXT:    v_cndmask_b32_e32 v0, v3, v5, vcc_lo
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT:    v_perm_b32 v0, v0, v1, 0x7060302
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
   %bool = icmp eq <2 x i32> %bool.arg1, %bool.arg2
   %y = select <2 x i1> %bool, <2 x bfloat> <bfloat 2.000000e+00, bfloat 2.000000e+00>, <2 x bfloat> <bfloat 1.000000e+00, bfloat 1.000000e+00>
   %ldexp = fmul <2 x bfloat> %x, %y
@@ -2930,267 +2799,185 @@ define <2 x bfloat> @fmul_select_v2bf16_test3(<2 x bfloat> %x, <2 x i32> %bool.a
 }
 
 define <2 x bfloat> @fmul_select_v2bf16_test4(<2 x bfloat> %x, <2 x i32> %bool.arg1, <2 x i32> %bool.arg2) {
-; GFX7-SDAG-LABEL: fmul_select_v2bf16_test4:
-; GFX7-SDAG:       ; %bb.0:
-; GFX7-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-SDAG-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v4
-; GFX7-SDAG-NEXT:    v_mul_f32_e32 v0, 1.0, v0
-; GFX7-SDAG-NEXT:    v_mul_f32_e32 v1, 1.0, v1
-; GFX7-SDAG-NEXT:    v_cndmask_b32_e64 v2, 1.0, 0.5, vcc
-; GFX7-SDAG-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v5
-; GFX7-SDAG-NEXT:    v_cndmask_b32_e64 v3, 1.0, 0.5, vcc
-; GFX7-SDAG-NEXT:    v_and_b32_e32 v1, 0xffff0000, v1
-; GFX7-SDAG-NEXT:    v_and_b32_e32 v0, 0xffff0000, v0
-; GFX7-SDAG-NEXT:    v_mul_f32_e32 v1, v1, v3
-; GFX7-SDAG-NEXT:    v_mul_f32_e32 v0, v0, v2
-; GFX7-SDAG-NEXT:    v_and_b32_e32 v0, 0xffff0000, v0
-; GFX7-SDAG-NEXT:    v_and_b32_e32 v1, 0xffff0000, v1
-; GFX7-SDAG-NEXT:    s_setpc_b64 s[30:31]
-;
-; GFX7-GISEL-LABEL: fmul_select_v2bf16_test4:
-; GFX7-GISEL:       ; %bb.0:
-; GFX7-GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-GISEL-NEXT:    v_cvt_f32_f16_e32 v0, v0
-; GFX7-GISEL-NEXT:    v_cvt_f32_f16_e32 v1, v1
-; GFX7-GISEL-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v4
-; GFX7-GISEL-NEXT:    v_cndmask_b32_e64 v2, 0, -1, vcc
-; GFX7-GISEL-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v5
-; GFX7-GISEL-NEXT:    v_cndmask_b32_e64 v3, 0, -1, vcc
-; GFX7-GISEL-NEXT:    v_ldexp_f32_e32 v0, v0, v2
-; GFX7-GISEL-NEXT:    v_ldexp_f32_e32 v1, v1, v3
-; GFX7-GISEL-NEXT:    v_cvt_f16_f32_e32 v0, v0
-; GFX7-GISEL-NEXT:    v_cvt_f16_f32_e32 v1, v1
-; GFX7-GISEL-NEXT:    s_setpc_b64 s[30:31]
-;
-; GFX9-SDAG-LABEL: fmul_select_v2bf16_test4:
-; GFX9-SDAG:       ; %bb.0:
-; GFX9-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-SDAG-NEXT:    v_mov_b32_e32 v5, 0x3f80
-; GFX9-SDAG-NEXT:    v_mov_b32_e32 v6, 0x3f00
-; GFX9-SDAG-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v4
-; GFX9-SDAG-NEXT:    v_cndmask_b32_e32 v2, v5, v6, vcc
-; GFX9-SDAG-NEXT:    v_cmp_eq_u32_e32 vcc, v1, v3
-; GFX9-SDAG-NEXT:    v_cndmask_b32_e32 v1, v5, v6, vcc
-; GFX9-SDAG-NEXT:    v_lshlrev_b32_e32 v3, 16, v0
-; GFX9-SDAG-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
-; GFX9-SDAG-NEXT:    v_mul_f32_e32 v1, v3, v1
-; GFX9-SDAG-NEXT:    v_and_b32_e32 v0, 0xffff0000, v0
-; GFX9-SDAG-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
-; GFX9-SDAG-NEXT:    v_bfe_u32 v3, v1, 16, 1
-; GFX9-SDAG-NEXT:    s_movk_i32 s4, 0x7fff
-; GFX9-SDAG-NEXT:    v_mul_f32_e32 v0, v0, v2
-; GFX9-SDAG-NEXT:    v_add3_u32 v3, v3, v1, s4
-; GFX9-SDAG-NEXT:    v_or_b32_e32 v4, 0x400000, v1
-; GFX9-SDAG-NEXT:    v_cmp_u_f32_e32 vcc, v1, v1
-; GFX9-SDAG-NEXT:    v_bfe_u32 v2, v0, 16, 1
-; GFX9-SDAG-NEXT:    v_cndmask_b32_e32 v1, v3, v4, vcc
-; GFX9-SDAG-NEXT:    v_add3_u32 v2, v2, v0, s4
-; GFX9-SDAG-NEXT:    v_or_b32_e32 v3, 0x400000, v0
-; GFX9-SDAG-NEXT:    v_cmp_u_f32_e32 vcc, v0, v0
-; GFX9-SDAG-NEXT:    v_cndmask_b32_e32 v0, v2, v3, vcc
-; GFX9-SDAG-NEXT:    s_mov_b32 s4, 0x7060302
-; GFX9-SDAG-NEXT:    v_perm_b32 v0, v0, v1, s4
-; GFX9-SDAG-NEXT:    s_setpc_b64 s[30:31]
-;
-; GFX9-GISEL-LABEL: fmul_select_v2bf16_test4:
-; GFX9-GISEL:       ; %bb.0:
-; GFX9-GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-GISEL-NEXT:    v_cmp_eq_u32_e32 vcc, v1, v3
-; GFX9-GISEL-NEXT:    v_cndmask_b32_e64 v1, 0, -1, vcc
-; GFX9-GISEL-NEXT:    v_mov_b32_e32 v2, 0xffff8000
-; GFX9-GISEL-NEXT:    v_mov_b32_e32 v3, 0x7fff
-; GFX9-GISEL-NEXT:    v_med3_i32 v1, v1, v2, v3
-; GFX9-GISEL-NEXT:    v_ldexp_f16_e32 v0, v0, v1
-; GFX9-GISEL-NEXT:    s_setpc_b64 s[30:31]
-;
-; GFX10-SDAG-LABEL: fmul_select_v2bf16_test4:
-; GFX10-SDAG:       ; %bb.0:
-; GFX10-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-SDAG-NEXT:    v_mov_b32_e32 v5, 0x3f00
-; GFX10-SDAG-NEXT:    v_cmp_eq_u32_e32 vcc_lo, v1, v3
-; GFX10-SDAG-NEXT:    v_lshlrev_b32_e32 v3, 16, v0
-; GFX10-SDAG-NEXT:    v_and_b32_e32 v0, 0xffff0000, v0
-; GFX10-SDAG-NEXT:    v_cndmask_b32_e32 v1, 0x3f80, v5, vcc_lo
-; GFX10-SDAG-NEXT:    v_cmp_eq_u32_e32 vcc_lo, v2, v4
-; GFX10-SDAG-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
-; GFX10-SDAG-NEXT:    v_cndmask_b32_e32 v2, 0x3f80, v5, vcc_lo
-; GFX10-SDAG-NEXT:    v_mul_f32_e32 v1, v3, v1
-; GFX10-SDAG-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
-; GFX10-SDAG-NEXT:    v_or_b32_e32 v4, 0x400000, v1
-; GFX10-SDAG-NEXT:    v_mul_f32_e32 v0, v0, v2
-; GFX10-SDAG-NEXT:    v_bfe_u32 v2, v1, 16, 1
-; GFX10-SDAG-NEXT:    v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX10-SDAG-NEXT:    v_bfe_u32 v3, v0, 16, 1
-; GFX10-SDAG-NEXT:    v_add3_u32 v2, v2, v1, 0x7fff
-; GFX10-SDAG-NEXT:    v_or_b32_e32 v5, 0x400000, v0
-; GFX10-SDAG-NEXT:    v_add3_u32 v3, v3, v0, 0x7fff
-; GFX10-SDAG-NEXT:    v_cndmask_b32_e32 v1, v2, v4, vcc_lo
-; GFX10-SDAG-NEXT:    v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX10-SDAG-NEXT:    v_cndmask_b32_e32 v0, v3, v5, vcc_lo
-; GFX10-SDAG-NEXT:    v_perm_b32 v0, v0, v1, 0x7060302
-; GFX10-SDAG-NEXT:    s_setpc_b64 s[30:31]
+; GFX7-LABEL: fmul_select_v2bf16_test4:
+; GFX7:       ; %bb.0:
+; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v4
+; GFX7-NEXT:    v_mul_f32_e32 v0, 1.0, v0
+; GFX7-NEXT:    v_mul_f32_e32 v1, 1.0, v1
+; GFX7-NEXT:    v_cndmask_b32_e64 v2, 1.0, 0.5, vcc
+; GFX7-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX7-NEXT:    v_cndmask_b32_e64 v3, 1.0, 0.5, vcc
+; GFX7-NEXT:    v_and_b32_e32 v1, 0xffff0000, v1
+; GFX7-NEXT:    v_and_b32_e32 v0, 0xffff0000, v0
+; GFX7-NEXT:    v_mul_f32_e32 v1, v1, v3
+; GFX7-NEXT:    v_mul_f32_e32 v0, v0, v2
+; GFX7-NEXT:    v_and_b32_e32 v0, 0xffff0000, v0
+; GFX7-NEXT:    v_and_b32_e32 v1, 0xffff0000, v1
+; GFX7-NEXT:    s_setpc_b64 s[30:31]
 ;
-; GFX10-GISEL-LABEL: fmul_select_v2bf16_test4:
-; GFX10-GISEL:       ; %bb.0:
-; GFX10-GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-GISEL-NEXT:    v_cmp_eq_u32_e32 vcc_lo, v1, v3
-; GFX10-GISEL-NEXT:    v_mov_b32_e32 v2, 0x7fff
-; GFX10-GISEL-NEXT:    v_cndmask_b32_e64 v1, 0, -1, vcc_lo
-; GFX10-GISEL-NEXT:    v_med3_i32 v1, 0xffff8000, v1, v2
-; GFX10-GISEL-NEXT:    v_ldexp_f16_e32 v0, v0, v1
-; GFX10-GISEL-NEXT:    s_setpc_b64 s[30:31]
+; GFX9-LABEL: fmul_select_v2bf16_test4:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v5, 0x3f80
+; GFX9-NEXT:    v_mov_b32_e32 v6, 0x3f00
+; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v4
+; GFX9-NEXT:    v_cndmask_b32_e32 v2, v5, v6, vcc
+; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, v1, v3
+; GFX9-NEXT:    v_cndmask_b32_e32 v1, v5, v6, vcc
+; GFX9-NEXT:    v_lshlrev_b32_e32 v3, 16, v0
+; GFX9-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
+; GFX9-NEXT:    v_mul_f32_e32 v1, v3, v1
+; GFX9-NEXT:    v_and_b32_e32 v0, 0xffff0000, v0
+; GFX9-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GFX9-NEXT:    v_bfe_u32 v3, v1, 16, 1
+; GFX9-NEXT:    s_movk_i32 s4, 0x7fff
+; GFX9-NEXT:    v_mul_f32_e32 v0, v0, v2
+; GFX9-NEXT:    v_add3_u32 v3, v3, v1, s4
+; GFX9-NEXT:    v_or_b32_e32 v4, 0x400000, v1
+; GFX9-NEXT:    v_cmp_u_f32_e32 vcc, v1, v1
+; GFX9-NEXT:    v_bfe_u32 v2, v0, 16, 1
+; GFX9-NEXT:    v_cndmask_b32_e32 v1, v3, v4, vcc
+; GFX9-NEXT:    v_add3_u32 v2, v2, v0, s4
+; GFX9-NEXT:    v_or_b32_e32 v3, 0x400000, v0
+; GFX9-NEXT:    v_cmp_u_f32_e32 vcc, v0, v0
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v2, v3, vcc
+; GFX9-NEXT:    s_mov_b32 s4, 0x7060302
+; GFX9-NEXT:    v_perm_b32 v0, v0, v1, s4
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
-; GFX11-SDAG-LABEL: fmul_select_v2bf16_test4:
-; GFX11-SDAG:       ; %bb.0:
-; GFX11-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-SDAG-NEXT:    v_mov_b32_e32 v5, 0x3f00
-; GFX11-SDAG-NEXT:    v_cmp_eq_u32_e32 vcc_lo, v1, v3
-; GFX11-SDAG-NEXT:    v_lshlrev_b32_e32 v3, 16, v0
-; GFX11-SDAG-NEXT:    v_and_b32_e32 v0, 0xffff0000, v0
-; GFX11-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX11-SDAG-NEXT:    v_cndmask_b32_e32 v1, 0x3f80, v5, vcc_lo
-; GFX11-SDAG-NEXT:    v_cmp_eq_u32_e32 vcc_lo, v2, v4
-; GFX11-SDAG-NEXT:    v_cndmask_b32_e32 v2, 0x3f80, v5, vcc_lo
-; GFX11-SDAG-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
-; GFX11-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SDAG-NEXT:    v_dual_mul_f32 v0, v0, v2 :: v_dual_lshlrev_b32 v1, 16, v1
-; GFX11-SDAG-NEXT:    v_or_b32_e32 v5, 0x400000, v0
-; GFX11-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-SDAG-NEXT:    v_mul_f32_e32 v1, v3, v1
-; GFX11-SDAG-NEXT:    v_bfe_u32 v3, v0, 16, 1
-; GFX11-SDAG-NEXT:    v_bfe_u32 v2, v1, 16, 1
-; GFX11-SDAG-NEXT:    v_or_b32_e32 v4, 0x400000, v1
-; GFX11-SDAG-NEXT:    v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX11-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-SDAG-NEXT:    v_add3_u32 v3, v3, v0, 0x7fff
-; GFX11-SDAG-NEXT:    v_add3_u32 v2, v2, v1, 0x7fff
-; GFX11-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-SDAG-NEXT:    v_cndmask_b32_e32 v1, v2, v4, vcc_lo
-; GFX11-SDAG-NEXT:    v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-SDAG-NEXT:    v_cndmask_b32_e32 v0, v3, v5, vcc_lo
-; GFX11-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX11-SDAG-NEXT:    v_perm_b32 v0, v0, v1, 0x7060302
-; GFX11-SDAG-NEXT:    s_setpc_b64 s[30:31]
+; GFX10-LABEL: fmul_select_v2bf16_test4:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    v_mov_b32_e32 v5, 0x3f00
+; GFX10-NEXT:    v_cmp_eq_u32_e32 vcc_lo, v1, v3
+; GFX10-NEXT:    v_lshlrev_b32_e32 v3, 16, v0
+; GFX10-NEXT:    v_and_b32_e32 v0, 0xffff0000, v0
+; GFX10-NEXT:    v_cndmask_b32_e32 v1, 0x3f80, v5, vcc_lo
+; GFX10-NEXT:    v_cmp_eq_u32_e32 vcc_lo, v2, v4
+; GFX10-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
+; GFX10-NEXT:    v_cndmask_b32_e32 v2, 0x3f80, v5, vcc_lo
+; GFX10-NEXT:    v_mul_f32_e32 v1, v3, v1
+; GFX10-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GFX10-NEXT:    v_or_b32_e32 v4, 0x400000, v1
+; GFX10-NEXT:    v_mul_f32_e32 v0, v0, v2
+; GFX10-NEXT:    v_bfe_u32 v2, v1, 16, 1
+; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX10-NEXT:    v_bfe_u32 v3, v0, 16, 1
+; GFX10-NEXT:    v_add3_u32 v2, v2, v1, 0x7fff
+; GFX10-NEXT:    v_or_b32_e32 v5, 0x400000, v0
+; GFX10-NEXT:    v_add3_u32 v3, v3, v0, 0x7fff
+; GFX10-NEXT:    v_cndmask_b32_e32 v1, v2, v4, vcc_lo
+; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX10-NEXT:    v_cndmask_b32_e32 v0, v3, v5, vcc_lo
+; GFX10-NEXT:    v_perm_b32 v0, v0, v1, 0x7060302
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
 ;
-; GFX11-GISEL-LABEL: fmul_select_v2bf16_test4:
-; GFX11-GISEL:       ; %bb.0:
-; GFX11-GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-GISEL-NEXT:    v_cmp_eq_u32_e32 vcc_lo, v1, v3
-; GFX11-GISEL-NEXT:    v_mov_b32_e32 v2, 0x7fff
-; GFX11-GISEL-NEXT:    v_cndmask_b32_e64 v1, 0, -1, vcc_lo
-; GFX11-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-GISEL-NEXT:    v_med3_i32 v1, 0xffff8000, v1, v2
-; GFX11-GISEL-NEXT:    v_ldexp_f16_e32 v0, v0, v1
-; GFX11-GISEL-NEXT:    s_setpc_b64 s[30:31]
+; GFX11-LABEL: fmul_select_v2bf16_test4:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    v_mov_b32_e32 v5, 0x3f00
+; GFX11-NEXT:    v_cmp_eq_u32_e32 vcc_lo, v1, v3
+; GFX11-NEXT:    v_lshlrev_b32_e32 v3, 16, v0
+; GFX11-NEXT:    v_and_b32_e32 v0, 0xffff0000, v0
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_cndmask_b32_e32 v1, 0x3f80, v5, vcc_lo
+; GFX11-NEXT:    v_cmp_eq_u32_e32 vcc_lo, v2, v4
+; GFX11-NEXT:    v_cndmask_b32_e32 v2, 0x3f80, v5, vcc_lo
+; GFX11-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_dual_mul_f32 v0, v0, v2 :: v_dual_lshlrev_b32 v1, 16, v1
+; GFX11-NEXT:    v_or_b32_e32 v5, 0x400000, v0
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-NEXT:    v_mul_f32_e32 v1, v3, v1
+; GFX11-NEXT:    v_bfe_u32 v3, v0, 16, 1
+; GFX11-NEXT:    v_bfe_u32 v2, v1, 16, 1
+; GFX11-NEXT:    v_or_b32_e32 v4, 0x400000, v1
+; GFX11-NEXT:    v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-NEXT:    v_add3_u32 v3, v3, v0, 0x7fff
+; GFX11-NEXT:    v_add3_u32 v2, v2, v1, 0x7fff
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-NEXT:    v_cndmask_b32_e32 v1, v2, v4, vcc_lo
+; GFX11-NEXT:    v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-NEXT:    v_cndmask_b32_e32 v0, v3, v5, vcc_lo
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT:    v_perm_b32 v0, v0, v1, 0x7060302
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
   %bool = icmp eq <2 x i32> %bool.arg1, %bool.arg2
   %y = select <2 x i1> %bool, <2 x bfloat> <bfloat 5.000000e-01, bfloat 5.000000e-01>, <2 x bfloat> <bfloat 1.000000e+00, bfloat 1.000000e+00>
   %ldexp = fmul <2 x bfloat> %x, %y
   ret <2 x bfloat> %ldexp
 }
-
-define bfloat @fmul_select_bf16_test5(bfloat %x, i32 %bool.arg1, i32 %bool.arg2) {
-; GFX7-SDAG-LABEL: fmul_select_bf16_test5:
-; GFX7-SDAG:       ; %bb.0:
-; GFX7-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-SDAG-NEXT:    v_mul_f32_e32 v0, 1.0, v0
-; GFX7-SDAG-NEXT:    v_mov_b32_e32 v3, 0x41000000
-; GFX7-SDAG-NEXT:    v_cmp_eq_u32_e32 vcc, v1, v2
-; GFX7-SDAG-NEXT:    v_cndmask_b32_e64 v1, v3, 2.0, vcc
-; GFX7-SDAG-NEXT:    v_and_b32_e32 v0, 0xffff0000, v0
-; GFX7-SDAG-NEXT:    v_mul_f32_e32 v0, v0, v1
-; GFX7-SDAG-NEXT:    v_and_b32_e32 v0, 0xffff0000, v0
-; GFX7-SDAG-NEXT:    s_setpc_b64 s[30:31]
-;
-; GFX7-GISEL-LABEL: fmul_select_bf16_test5:
-; GFX7-GISEL:       ; %bb.0:
-; GFX7-GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-GISEL-NEXT:    v_cvt_f32_f16_e32 v0, v0
-; GFX7-GISEL-NEXT:    v_cmp_eq_u32_e32 vcc, v1, v2
-; GFX7-GISEL-NEXT:    v_cndmask_b32_e64 v1, 3, 1, vcc
-; GFX7-GISEL-NEXT:    v_ldexp_f32_e32 v0, v0, v1
-; GFX7-GISEL-NEXT:    v_cvt_f16_f32_e32 v0, v0
-; GFX7-GISEL-NEXT:    s_setpc_b64 s[30:31]
-;
-; GFX9-SDAG-LABEL: fmul_select_bf16_test5:
-; GFX9-SDAG:       ; %bb.0:
-; GFX9-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-SDAG-NEXT:    v_mov_b32_e32 v3, 0x4100
-; GFX9-SDAG-NEXT:    v_mov_b32_e32 v4, 0x4000
-; GFX9-SDAG-NEXT:    v_cmp_eq_u32_e32 vcc, v1, v2
-; GFX9-SDAG-NEXT:    v_cndmask_b32_e32 v1, v3, v4, vcc
-; GFX9-SDAG-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
-; GFX9-SDAG-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
-; GFX9-SDAG-NEXT:    v_mul_f32_e32 v0, v0, v1
-; GFX9-SDAG-NEXT:    v_bfe_u32 v1, v0, 16, 1
-; GFX9-SDAG-NEXT:    s_movk_i32 s4, 0x7fff
-; GFX9-SDAG-NEXT:    v_add3_u32 v1, v1, v0, s4
-; GFX9-SDAG-NEXT:    v_or_b32_e32 v2, 0x400000, v0
-; GFX9-SDAG-NEXT:    v_cmp_u_f32_e32 vcc, v0, v0
-; GFX9-SDAG-NEXT:    v_cndmask_b32_e32 v0, v1, v2, vcc
-; GFX9-SDAG-NEXT:    v_lshrrev_b32_e32 v0, 16, v0
-; GFX9-SDAG-NEXT:    s_setpc_b64 s[30:31]
-;
-; GFX9-GISEL-LABEL: fmul_select_bf16_test5:
-; GFX9-GISEL:       ; %bb.0:
-; GFX9-GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-GISEL-NEXT:    v_cmp_eq_u32_e32 vcc, v1, v2
-; GFX9-GISEL-NEXT:    v_cndmask_b32_e64 v1, 3, 1, vcc
-; GFX9-GISEL-NEXT:    v_ldexp_f16_e32 v0, v0, v1
-; GFX9-GISEL-NEXT:    s_setpc_b64 s[30:31]
-;
-; GFX10-SDAG-LABEL: fmul_select_bf16_test5:
-; GFX10-SDAG:       ; %bb.0:
-; GFX10-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-SDAG-NEXT:    v_mov_b32_e32 v3, 0x4000
-; GFX10-SDAG-NEXT:    v_cmp_eq_u32_e32 vcc_lo, v1, v2
-; GFX10-SDAG-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
-; GFX10-SDAG-NEXT:    v_cndmask_b32_e32 v1, 0x4100, v3, vcc_lo
-; GFX10-SDAG-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
-; GFX10-SDAG-NEXT:    v_mul_f32_e32 v0, v0, v1
-; GFX10-SDAG-NEXT:    v_bfe_u32 v1, v0, 16, 1
-; GFX10-SDAG-NEXT:    v_or_b32_e32 v2, 0x400000, v0
-; GFX10-SDAG-NEXT:    v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX10-SDAG-NEXT:    v_add3_u32 v1, v1, v0, 0x7fff
-; GFX10-SDAG-NEXT:    v_cndmask_b32_e32 v0, v1, v2, vcc_lo
-; GFX10-SDAG-NEXT:    v_lshrrev_b32_e32 v0, 16, v0
-; GFX10-SDAG-NEXT:    s_setpc_b64 s[30:31]
+
+define bfloat @fmul_select_bf16_test5(bfloat %x, i32 %bool.arg1, i32 %bool.arg2) {
+; GFX7-LABEL: fmul_select_bf16_test5:
+; GFX7:       ; %bb.0:
+; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    v_mul_f32_e32 v0, 1.0, v0
+; GFX7-NEXT:    v_mov_b32_e32 v3, 0x41000000
+; GFX7-NEXT:    v_cmp_eq_u32_e32 vcc, v1, v2
+; GFX7-NEXT:    v_cndmask_b32_e64 v1, v3, 2.0, vcc
+; GFX7-NEXT:    v_and_b32_e32 v0, 0xffff0000, v0
+; GFX7-NEXT:    v_mul_f32_e32 v0, v0, v1
+; GFX7-NEXT:    v_and_b32_e32 v0, 0xffff0000, v0
+; GFX7-NEXT:    s_setpc_b64 s[30:31]
 ;
-; GFX10-GISEL-LABEL: fmul_select_bf16_test5:
-; GFX10-GISEL:       ; %bb.0:
-; GFX10-GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-GISEL-NEXT:    v_cmp_eq_u32_e32 vcc_lo, v1, v2
-; GFX10-GISEL-NEXT:    v_cndmask_b32_e64 v1, 3, 1, vcc_lo
-; GFX10-GISEL-NEXT:    v_ldexp_f16_e32 v0, v0, v1
-; GFX10-GISEL-NEXT:    s_setpc_b64 s[30:31]
+; GFX9-LABEL: fmul_select_bf16_test5:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v3, 0x4100
+; GFX9-NEXT:    v_mov_b32_e32 v4, 0x4000
+; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, v1, v2
+; GFX9-NEXT:    v_cndmask_b32_e32 v1, v3, v4, vcc
+; GFX9-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
+; GFX9-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
+; GFX9-NEXT:    v_mul_f32_e32 v0, v0, v1
+; GFX9-NEXT:    v_bfe_u32 v1, v0, 16, 1
+; GFX9-NEXT:    s_movk_i32 s4, 0x7fff
+; GFX9-NEXT:    v_add3_u32 v1, v1, v0, s4
+; GFX9-NEXT:    v_or_b32_e32 v2, 0x400000, v0
+; GFX9-NEXT:    v_cmp_u_f32_e32 vcc, v0, v0
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v1, v2, vcc
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v0
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
-; GFX11-SDAG-LABEL: fmul_select_bf16_test5:
-; GFX11-SDAG:       ; %bb.0:
-; GFX11-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-SDAG-NEXT:    v_dual_mov_b32 v3, 0x4000 :: v_dual_lshlrev_b32 v0, 16, v0
-; GFX11-SDAG-NEXT:    v_cmp_eq_u32_e32 vcc_lo, v1, v2
-; GFX11-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SDAG-NEXT:    v_cndmask_b32_e32 v1, 0x4100, v3, vcc_lo
-; GFX11-SDAG-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
-; GFX11-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SDAG-NEXT:    v_mul_f32_e32 v0, v0, v1
-; GFX11-SDAG-NEXT:    v_bfe_u32 v1, v0, 16, 1
-; GFX11-SDAG-NEXT:    v_or_b32_e32 v2, 0x400000, v0
-; GFX11-SDAG-NEXT:    v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SDAG-NEXT:    v_add3_u32 v1, v1, v0, 0x7fff
-; GFX11-SDAG-NEXT:    v_cndmask_b32_e32 v0, v1, v2, vcc_lo
-; GFX11-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX11-SDAG-NEXT:    v_lshrrev_b32_e32 v0, 16, v0
-; GFX11-SDAG-NEXT:    s_setpc_b64 s[30:31]
+; GFX10-LABEL: fmul_select_bf16_test5:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    v_mov_b32_e32 v3, 0x4000
+; GFX10-NEXT:    v_cmp_eq_u32_e32 vcc_lo, v1, v2
+; GFX10-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
+; GFX10-NEXT:    v_cndmask_b32_e32 v1, 0x4100, v3, vcc_lo
+; GFX10-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
+; GFX10-NEXT:    v_mul_f32_e32 v0, v0, v1
+; GFX10-NEXT:    v_bfe_u32 v1, v0, 16, 1
+; GFX10-NEXT:    v_or_b32_e32 v2, 0x400000, v0
+; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX10-NEXT:    v_add3_u32 v1, v1, v0, 0x7fff
+; GFX10-NEXT:    v_cndmask_b32_e32 v0, v1, v2, vcc_lo
+; GFX10-NEXT:    v_lshrrev_b32_e32 v0, 16, v0
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
 ;
-; GFX11-GISEL-LABEL: fmul_select_bf16_test5:
-; GFX11-GISEL:       ; %bb.0:
-; GFX11-GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-GISEL-NEXT:    v_cmp_eq_u32_e32 vcc_lo, v1, v2
-; GFX11-GISEL-NEXT:    v_cndmask_b32_e64 v1, 3, 1, vcc_lo
-; GFX11-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX11-GISEL-NEXT:    v_ldexp_f16_e32 v0, v0, v1
-; GFX11-GISEL-NEXT:    s_setpc_b64 s[30:31]
+; GFX11-LABEL: fmul_select_bf16_test5:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    v_dual_mov_b32 v3, 0x4000 :: v_dual_lshlrev_b32 v0, 16, v0
+; GFX11-NEXT:    v_cmp_eq_u32_e32 vcc_lo, v1, v2
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_cndmask_b32_e32 v1, 0x4100, v3, vcc_lo
+; GFX11-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_mul_f32_e32 v0, v0, v1
+; GFX11-NEXT:    v_bfe_u32 v1, v0, 16, 1
+; GFX11-NEXT:    v_or_b32_e32 v2, 0x400000, v0
+; GFX11-NEXT:    v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_add3_u32 v1, v1, v0, 0x7fff
+; GFX11-NEXT:    v_cndmask_b32_e32 v0, v1, v2, vcc_lo
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT:    v_lshrrev_b32_e32 v0, 16, v0
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
   %bool = icmp eq i32 %bool.arg1, %bool.arg2
   %y = select i1 %bool, bfloat 2.000000e+00, bfloat 8.000000e+00
   %ldexp = fmul bfloat %x, %y
@@ -3198,116 +2985,74 @@ define bfloat @fmul_select_bf16_test5(bfloat %x, i32 %bool.arg1, i32 %bool.arg2)
 }
 
 define bfloat @fmul_select_bf16_test6(bfloat %x, i32 %bool.arg1, i32 %bool.arg2) {
-; GFX7-SDAG-LABEL: fmul_select_bf16_test6:
-; GFX7-SDAG:       ; %bb.0:
-; GFX7-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-SDAG-NEXT:    v_mul_f32_e32 v0, 1.0, v0
-; GFX7-SDAG-NEXT:    v_mov_b32_e32 v3, 0x40400000
-; GFX7-SDAG-NEXT:    v_mov_b32_e32 v4, 0xc1000000
-; GFX7-SDAG-NEXT:    v_cmp_eq_u32_e32 vcc, v1, v2
-; GFX7-SDAG-NEXT:    v_cndmask_b32_e32 v1, v3, v4, vcc
-; GFX7-SDAG-NEXT:    v_and_b32_e32 v0, 0xffff0000, v0
-; GFX7-SDAG-NEXT:    v_mul_f32_e32 v0, v0, v1
-; GFX7-SDAG-NEXT:    v_and_b32_e32 v0, 0xffff0000, v0
-; GFX7-SDAG-NEXT:    s_setpc_b64 s[30:31]
-;
-; GFX7-GISEL-LABEL: fmul_select_bf16_test6:
-; GFX7-GISEL:       ; %bb.0:
-; GFX7-GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-GISEL-NEXT:    v_mov_b32_e32 v3, 0xc100
-; GFX7-GISEL-NEXT:    v_mov_b32_e32 v4, 0x4040
-; GFX7-GISEL-NEXT:    v_cmp_eq_u32_e32 vcc, v1, v2
-; GFX7-GISEL-NEXT:    v_cndmask_b32_e32 v1, v4, v3, vcc
-; GFX7-GISEL-NEXT:    v_cvt_f32_f16_e32 v0, v0
-; GFX7-GISEL-NEXT:    v_cvt_f32_f16_e32 v1, v1
-; GFX7-GISEL-NEXT:    v_mul_f32_e32 v0, v0, v1
-; GFX7-GISEL-NEXT:    v_cvt_f16_f32_e32 v0, v0
-; GFX7-GISEL-NEXT:    s_setpc_b64 s[30:31]
-;
-; GFX9-SDAG-LABEL: fmul_select_bf16_test6:
-; GFX9-SDAG:       ; %bb.0:
-; GFX9-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-SDAG-NEXT:    v_mov_b32_e32 v3, 0x4040
-; GFX9-SDAG-NEXT:    v_mov_b32_e32 v4, 0xffffc100
-; GFX9-SDAG-NEXT:    v_cmp_eq_u32_e32 vcc, v1, v2
-; GFX9-SDAG-NEXT:    v_cndmask_b32_e32 v1, v3, v4, vcc
-; GFX9-SDAG-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
-; GFX9-SDAG-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
-; GFX9-SDAG-NEXT:    v_mul_f32_e32 v0, v0, v1
-; GFX9-SDAG-NEXT:    v_bfe_u32 v1, v0, 16, 1
-; GFX9-SDAG-NEXT:    s_movk_i32 s4, 0x7fff
-; GFX9-SDAG-NEXT:    v_add3_u32 v1, v1, v0, s4
-; GFX9-SDAG-NEXT:    v_or_b32_e32 v2, 0x400000, v0
-; GFX9-SDAG-NEXT:    v_cmp_u_f32_e32 vcc, v0, v0
-; GFX9-SDAG-NEXT:    v_cndmask_b32_e32 v0, v1, v2, vcc
-; GFX9-SDAG-NEXT:    v_lshrrev_b32_e32 v0, 16, v0
-; GFX9-SDAG-NEXT:    s_setpc_b64 s[30:31]
-;
-; GFX9-GISEL-LABEL: fmul_select_bf16_test6:
-; GFX9-GISEL:       ; %bb.0:
-; GFX9-GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-GISEL-NEXT:    v_mov_b32_e32 v3, 0xc100
-; GFX9-GISEL-NEXT:    v_mov_b32_e32 v4, 0x4040
-; GFX9-GISEL-NEXT:    v_cmp_eq_u32_e32 vcc, v1, v2
-; GFX9-GISEL-NEXT:    v_cndmask_b32_e32 v1, v4, v3, vcc
-; GFX9-GISEL-NEXT:    v_mul_f16_e32 v0, v0, v1
-; GFX9-GISEL-NEXT:    s_setpc_b64 s[30:31]
-;
-; GFX10-SDAG-LABEL: fmul_select_bf16_test6:
-; GFX10-SDAG:       ; %bb.0:
-; GFX10-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-SDAG-NEXT:    v_mov_b32_e32 v3, 0xffffc100
-; GFX10-SDAG-NEXT:    v_cmp_eq_u32_e32 vcc_lo, v1, v2
-; GFX10-SDAG-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
-; GFX10-SDAG-NEXT:    v_cndmask_b32_e32 v1, 0x4040, v3, vcc_lo
-; GFX10-SDAG-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
-; GFX10-SDAG-NEXT:    v_mul_f32_e32 v0, v0, v1
-; GFX10-SDAG-NEXT:    v_bfe_u32 v1, v0, 16, 1
-; GFX10-SDAG-NEXT:    v_or_b32_e32 v2, 0x400000, v0
-; GFX10-SDAG-NEXT:    v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX10-SDAG-NEXT:    v_add3_u32 v1, v1, v0, 0x7fff
-; GFX10-SDAG-NEXT:    v_cndmask_b32_e32 v0, v1, v2, vcc_lo
-; GFX10-SDAG-NEXT:    v_lshrrev_b32_e32 v0, 16, v0
-; GFX10-SDAG-NEXT:    s_setpc_b64 s[30:31]
+; GFX7-LABEL: fmul_select_bf16_test6:
+; GFX7:       ; %bb.0:
+; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    v_mul_f32_e32 v0, 1.0, v0
+; GFX7-NEXT:    v_mov_b32_e32 v3, 0x40400000
+; GFX7-NEXT:    v_mov_b32_e32 v4, 0xc1000000
+; GFX7-NEXT:    v_cmp_eq_u32_e32 vcc, v1, v2
+; GFX7-NEXT:    v_cndmask_b32_e32 v1, v3, v4, vcc
+; GFX7-NEXT:    v_and_b32_e32 v0, 0xffff0000, v0
+; GFX7-NEXT:    v_mul_f32_e32 v0, v0, v1
+; GFX7-NEXT:    v_and_b32_e32 v0, 0xffff0000, v0
+; GFX7-NEXT:    s_setpc_b64 s[30:31]
 ;
-; GFX10-GISEL-LABEL: fmul_select_bf16_test6:
-; GFX10-GISEL:       ; %bb.0:
-; GFX10-GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-GISEL-NEXT:    v_mov_b32_e32 v3, 0x4040
-; GFX10-GISEL-NEXT:    v_cmp_eq_u32_e32 vcc_lo, v1, v2
-; GFX10-GISEL-NEXT:    v_cndmask_b32_e64 v1, v3, 0xc100, vcc_lo
-; GFX10-GISEL-NEXT:    v_mul_f16_e32 v0, v0, v1
-; GFX10-GISEL-NEXT:    s_setpc_b64 s[30:31]
+; GFX9-LABEL: fmul_select_bf16_test6:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v3, 0x4040
+; GFX9-NEXT:    v_mov_b32_e32 v4, 0xffffc100
+; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, v1, v2
+; GFX9-NEXT:    v_cndmask_b32_e32 v1, v3, v4, vcc
+; GFX9-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
+; GFX9-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
+; GFX9-NEXT:    v_mul_f32_e32 v0, v0, v1
+; GFX9-NEXT:    v_bfe_u32 v1, v0, 16, 1
+; GFX9-NEXT:    s_movk_i32 s4, 0x7fff
+; GFX9-NEXT:    v_add3_u32 v1, v1, v0, s4
+; GFX9-NEXT:    v_or_b32_e32 v2, 0x400000, v0
+; GFX9-NEXT:    v_cmp_u_f32_e32 vcc, v0, v0
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v1, v2, vcc
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v0
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
-; GFX11-SDAG-LABEL: fmul_select_bf16_test6:
-; GFX11-SDAG:       ; %bb.0:
-; GFX11-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-SDAG-NEXT:    v_dual_mov_b32 v3, 0xffffc100 :: v_dual_lshlrev_b32 v0, 16, v0
-; GFX11-SDAG-NEXT:    v_cmp_eq_u32_e32 vcc_lo, v1, v2
-; GFX11-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SDAG-NEXT:    v_cndmask_b32_e32 v1, 0x4040, v3, vcc_lo
-; GFX11-SDAG-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
-; GFX11-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SDAG-NEXT:    v_mul_f32_e32 v0, v0, v1
-; GFX11-SDAG-NEXT:    v_bfe_u32 v1, v0, 16, 1
-; GFX11-SDAG-NEXT:    v_or_b32_e32 v2, 0x400000, v0
-; GFX11-SDAG-NEXT:    v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SDAG-NEXT:    v_add3_u32 v1, v1, v0, 0x7fff
-; GFX11-SDAG-NEXT:    v_cndmask_b32_e32 v0, v1, v2, vcc_lo
-; GFX11-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX11-SDAG-NEXT:    v_lshrrev_b32_e32 v0, 16, v0
-; GFX11-SDAG-NEXT:    s_setpc_b64 s[30:31]
+; GFX10-LABEL: fmul_select_bf16_test6:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    v_mov_b32_e32 v3, 0xffffc100
+; GFX10-NEXT:    v_cmp_eq_u32_e32 vcc_lo, v1, v2
+; GFX10-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
+; GFX10-NEXT:    v_cndmask_b32_e32 v1, 0x4040, v3, vcc_lo
+; GFX10-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
+; GFX10-NEXT:    v_mul_f32_e32 v0, v0, v1
+; GFX10-NEXT:    v_bfe_u32 v1, v0, 16, 1
+; GFX10-NEXT:    v_or_b32_e32 v2, 0x400000, v0
+; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX10-NEXT:    v_add3_u32 v1, v1, v0, 0x7fff
+; GFX10-NEXT:    v_cndmask_b32_e32 v0, v1, v2, vcc_lo
+; GFX10-NEXT:    v_lshrrev_b32_e32 v0, 16, v0
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
 ;
-; GFX11-GISEL-LABEL: fmul_select_bf16_test6:
-; GFX11-GISEL:       ; %bb.0:
-; GFX11-GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-GISEL-NEXT:    v_mov_b32_e32 v3, 0x4040
-; GFX11-GISEL-NEXT:    v_cmp_eq_u32_e32 vcc_lo, v1, v2
-; GFX11-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-GISEL-NEXT:    v_cndmask_b32_e64 v1, v3, 0xc100, vcc_lo
-; GFX11-GISEL-NEXT:    v_mul_f16_e32 v0, v0, v1
-; GFX11-GISEL-NEXT:    s_setpc_b64 s[30:31]
+; GFX11-LABEL: fmul_select_bf16_test6:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    v_dual_mov_b32 v3, 0xffffc100 :: v_dual_lshlrev_b32 v0, 16, v0
+; GFX11-NEXT:    v_cmp_eq_u32_e32 vcc_lo, v1, v2
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_cndmask_b32_e32 v1, 0x4040, v3, vcc_lo
+; GFX11-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_mul_f32_e32 v0, v0, v1
+; GFX11-NEXT:    v_bfe_u32 v1, v0, 16, 1
+; GFX11-NEXT:    v_or_b32_e32 v2, 0x400000, v0
+; GFX11-NEXT:    v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_add3_u32 v1, v1, v0, 0x7fff
+; GFX11-NEXT:    v_cndmask_b32_e32 v0, v1, v2, vcc_lo
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT:    v_lshrrev_b32_e32 v0, 16, v0
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
   %bool = icmp eq i32 %bool.arg1, %bool.arg2
   %y = select i1 %bool, bfloat -8.000000e+00, bfloat 3.000000e+00
   %ldexp = fmul bfloat %x, %y
@@ -3315,115 +3060,73 @@ define bfloat @fmul_select_bf16_test6(bfloat %x, i32 %bool.arg1, i32 %bool.arg2)
 }
 
 define bfloat @fmul_select_bf16_test7(bfloat %x, i32 %bool.arg1, i32 %bool.arg2) {
-; GFX7-SDAG-LABEL: fmul_select_bf16_test7:
-; GFX7-SDAG:       ; %bb.0:
-; GFX7-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-SDAG-NEXT:    v_mul_f32_e32 v0, 1.0, v0
-; GFX7-SDAG-NEXT:    v_mov_b32_e32 v3, 0x41000000
-; GFX7-SDAG-NEXT:    v_cmp_eq_u32_e32 vcc, v1, v2
-; GFX7-SDAG-NEXT:    v_cndmask_b32_e32 v1, -4.0, v3, vcc
-; GFX7-SDAG-NEXT:    v_and_b32_e32 v0, 0xffff0000, v0
-; GFX7-SDAG-NEXT:    v_mul_f32_e32 v0, v0, v1
-; GFX7-SDAG-NEXT:    v_and_b32_e32 v0, 0xffff0000, v0
-; GFX7-SDAG-NEXT:    s_setpc_b64 s[30:31]
-;
-; GFX7-GISEL-LABEL: fmul_select_bf16_test7:
-; GFX7-GISEL:       ; %bb.0:
-; GFX7-GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-GISEL-NEXT:    v_mov_b32_e32 v3, 0x4100
-; GFX7-GISEL-NEXT:    v_mov_b32_e32 v4, 0xc080
-; GFX7-GISEL-NEXT:    v_cmp_eq_u32_e32 vcc, v1, v2
-; GFX7-GISEL-NEXT:    v_cndmask_b32_e32 v1, v4, v3, vcc
-; GFX7-GISEL-NEXT:    v_cvt_f32_f16_e32 v0, v0
-; GFX7-GISEL-NEXT:    v_cvt_f32_f16_e32 v1, v1
-; GFX7-GISEL-NEXT:    v_mul_f32_e32 v0, v0, v1
-; GFX7-GISEL-NEXT:    v_cvt_f16_f32_e32 v0, v0
-; GFX7-GISEL-NEXT:    s_setpc_b64 s[30:31]
-;
-; GFX9-SDAG-LABEL: fmul_select_bf16_test7:
-; GFX9-SDAG:       ; %bb.0:
-; GFX9-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-SDAG-NEXT:    v_mov_b32_e32 v3, 0xffffc080
-; GFX9-SDAG-NEXT:    v_mov_b32_e32 v4, 0x4100
-; GFX9-SDAG-NEXT:    v_cmp_eq_u32_e32 vcc, v1, v2
-; GFX9-SDAG-NEXT:    v_cndmask_b32_e32 v1, v3, v4, vcc
-; GFX9-SDAG-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
-; GFX9-SDAG-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
-; GFX9-SDAG-NEXT:    v_mul_f32_e32 v0, v0, v1
-; GFX9-SDAG-NEXT:    v_bfe_u32 v1, v0, 16, 1
-; GFX9-SDAG-NEXT:    s_movk_i32 s4, 0x7fff
-; GFX9-SDAG-NEXT:    v_add3_u32 v1, v1, v0, s4
-; GFX9-SDAG-NEXT:    v_or_b32_e32 v2, 0x400000, v0
-; GFX9-SDAG-NEXT:    v_cmp_u_f32_e32 vcc, v0, v0
-; GFX9-SDAG-NEXT:    v_cndmask_b32_e32 v0, v1, v2, vcc
-; GFX9-SDAG-NEXT:    v_lshrrev_b32_e32 v0, 16, v0
-; GFX9-SDAG-NEXT:    s_setpc_b64 s[30:31]
-;
-; GFX9-GISEL-LABEL: fmul_select_bf16_test7:
-; GFX9-GISEL:       ; %bb.0:
-; GFX9-GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-GISEL-NEXT:    v_mov_b32_e32 v3, 0x4100
-; GFX9-GISEL-NEXT:    v_mov_b32_e32 v4, 0xc080
-; GFX9-GISEL-NEXT:    v_cmp_eq_u32_e32 vcc, v1, v2
-; GFX9-GISEL-NEXT:    v_cndmask_b32_e32 v1, v4, v3, vcc
-; GFX9-GISEL-NEXT:    v_mul_f16_e32 v0, v0, v1
-; GFX9-GISEL-NEXT:    s_setpc_b64 s[30:31]
-;
-; GFX10-SDAG-LABEL: fmul_select_bf16_test7:
-; GFX10-SDAG:       ; %bb.0:
-; GFX10-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-SDAG-NEXT:    v_mov_b32_e32 v3, 0x4100
-; GFX10-SDAG-NEXT:    v_cmp_eq_u32_e32 vcc_lo, v1, v2
-; GFX10-SDAG-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
-; GFX10-SDAG-NEXT:    v_cndmask_b32_e32 v1, 0xffffc080, v3, vcc_lo
-; GFX10-SDAG-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
-; GFX10-SDAG-NEXT:    v_mul_f32_e32 v0, v0, v1
-; GFX10-SDAG-NEXT:    v_bfe_u32 v1, v0, 16, 1
-; GFX10-SDAG-NEXT:    v_or_b32_e32 v2, 0x400000, v0
-; GFX10-SDAG-NEXT:    v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX10-SDAG-NEXT:    v_add3_u32 v1, v1, v0, 0x7fff
-; GFX10-SDAG-NEXT:    v_cndmask_b32_e32 v0, v1, v2, vcc_lo
-; GFX10-SDAG-NEXT:    v_lshrrev_b32_e32 v0, 16, v0
-; GFX10-SDAG-NEXT:    s_setpc_b64 s[30:31]
+; GFX7-LABEL: fmul_select_bf16_test7:
+; GFX7:       ; %bb.0:
+; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    v_mul_f32_e32 v0, 1.0, v0
+; GFX7-NEXT:    v_mov_b32_e32 v3, 0x41000000
+; GFX7-NEXT:    v_cmp_eq_u32_e32 vcc, v1, v2
+; GFX7-NEXT:    v_cndmask_b32_e32 v1, -4.0, v3, vcc
+; GFX7-NEXT:    v_and_b32_e32 v0, 0xffff0000, v0
+; GFX7-NEXT:    v_mul_f32_e32 v0, v0, v1
+; GFX7-NEXT:    v_and_b32_e32 v0, 0xffff0000, v0
+; GFX7-NEXT:    s_setpc_b64 s[30:31]
 ;
-; GFX10-GISEL-LABEL: fmul_select_bf16_test7:
-; GFX10-GISEL:       ; %bb.0:
-; GFX10-GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-GISEL-NEXT:    v_mov_b32_e32 v3, 0xc080
-; GFX10-GISEL-NEXT:    v_cmp_eq_u32_e32 vcc_lo, v1, v2
-; GFX10-GISEL-NEXT:    v_cndmask_b32_e64 v1, v3, 0x4100, vcc_lo
-; GFX10-GISEL-NEXT:    v_mul_f16_e32 v0, v0, v1
-; GFX10-GISEL-NEXT:    s_setpc_b64 s[30:31]
+; GFX9-LABEL: fmul_select_bf16_test7:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v3, 0xffffc080
+; GFX9-NEXT:    v_mov_b32_e32 v4, 0x4100
+; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, v1, v2
+; GFX9-NEXT:    v_cndmask_b32_e32 v1, v3, v4, vcc
+; GFX9-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
+; GFX9-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
+; GFX9-NEXT:    v_mul_f32_e32 v0, v0, v1
+; GFX9-NEXT:    v_bfe_u32 v1, v0, 16, 1
+; GFX9-NEXT:    s_movk_i32 s4, 0x7fff
+; GFX9-NEXT:    v_add3_u32 v1, v1, v0, s4
+; GFX9-NEXT:    v_or_b32_e32 v2, 0x400000, v0
+; GFX9-NEXT:    v_cmp_u_f32_e32 vcc, v0, v0
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v1, v2, vcc
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v0
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
-; GFX11-SDAG-LABEL: fmul_select_bf16_test7:
-; GFX11-SDAG:       ; %bb.0:
-; GFX11-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-SDAG-NEXT:    v_dual_mov_b32 v3, 0x4100 :: v_dual_lshlrev_b32 v0, 16, v0
-; GFX11-SDAG-NEXT:    v_cmp_eq_u32_e32 vcc_lo, v1, v2
-; GFX11-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SDAG-NEXT:    v_cndmask_b32_e32 v1, 0xffffc080, v3, vcc_lo
-; GFX11-SDAG-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
-; GFX11-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SDAG-NEXT:    v_mul_f32_e32 v0, v0, v1
-; GFX11-SDAG-NEXT:    v_bfe_u32 v1, v0, 16, 1
-; GFX11-SDAG-NEXT:    v_or_b32_e32 v2, 0x400000, v0
-; GFX11-SDAG-NEXT:    v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SDAG-NEXT:    v_add3_u32 v1, v1, v0, 0x7fff
-; GFX11-SDAG-NEXT:    v_cndmask_b32_e32 v0, v1, v2, vcc_lo
-; GFX11-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX11-SDAG-NEXT:    v_lshrrev_b32_e32 v0, 16, v0
-; GFX11-SDAG-NEXT:    s_setpc_b64 s[30:31]
+; GFX10-LABEL: fmul_select_bf16_test7:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    v_mov_b32_e32 v3, 0x4100
+; GFX10-NEXT:    v_cmp_eq_u32_e32 vcc_lo, v1, v2
+; GFX10-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
+; GFX10-NEXT:    v_cndmask_b32_e32 v1, 0xffffc080, v3, vcc_lo
+; GFX10-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
+; GFX10-NEXT:    v_mul_f32_e32 v0, v0, v1
+; GFX10-NEXT:    v_bfe_u32 v1, v0, 16, 1
+; GFX10-NEXT:    v_or_b32_e32 v2, 0x400000, v0
+; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX10-NEXT:    v_add3_u32 v1, v1, v0, 0x7fff
+; GFX10-NEXT:    v_cndmask_b32_e32 v0, v1, v2, vcc_lo
+; GFX10-NEXT:    v_lshrrev_b32_e32 v0, 16, v0
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
 ;
-; GFX11-GISEL-LABEL: fmul_select_bf16_test7:
-; GFX11-GISEL:       ; %bb.0:
-; GFX11-GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-GISEL-NEXT:    v_mov_b32_e32 v3, 0xc080
-; GFX11-GISEL-NEXT:    v_cmp_eq_u32_e32 vcc_lo, v1, v2
-; GFX11-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-GISEL-NEXT:    v_cndmask_b32_e64 v1, v3, 0x4100, vcc_lo
-; GFX11-GISEL-NEXT:    v_mul_f16_e32 v0, v0, v1
-; GFX11-GISEL-NEXT:    s_setpc_b64 s[30:31]
+; GFX11-LABEL: fmul_select_bf16_test7:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    v_dual_mov_b32 v3, 0x4100 :: v_dual_lshlrev_b32 v0, 16, v0
+; GFX11-NEXT:    v_cmp_eq_u32_e32 vcc_lo, v1, v2
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_cndmask_b32_e32 v1, 0xffffc080, v3, vcc_lo
+; GFX11-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_mul_f32_e32 v0, v0, v1
+; GFX11-NEXT:    v_bfe_u32 v1, v0, 16, 1
+; GFX11-NEXT:    v_or_b32_e32 v2, 0x400000, v0
+; GFX11-NEXT:    v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_add3_u32 v1, v1, v0, 0x7fff
+; GFX11-NEXT:    v_cndmask_b32_e32 v0, v1, v2, vcc_lo
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT:    v_lshrrev_b32_e32 v0, 16, v0
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
   %bool = icmp eq i32 %bool.arg1, %bool.arg2
   %y = select i1 %bool, bfloat 8.000000e+00, bfloat -4.000000e+00
   %ldexp = fmul bfloat %x, %y
@@ -3431,111 +3134,73 @@ define bfloat @fmul_select_bf16_test7(bfloat %x, i32 %bool.arg1, i32 %bool.arg2)
 }
 
 define bfloat @fmul_select_bf16_test8(bfloat %x, i32 %bool.arg1, i32 %bool.arg2) {
-; GFX7-SDAG-LABEL: fmul_select_bf16_test8:
-; GFX7-SDAG:       ; %bb.0:
-; GFX7-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-SDAG-NEXT:    v_cmp_eq_u32_e32 vcc, v1, v2
-; GFX7-SDAG-NEXT:    v_mul_f32_e32 v0, 1.0, v0
-; GFX7-SDAG-NEXT:    v_cndmask_b32_e64 v1, 0, 1, vcc
-; GFX7-SDAG-NEXT:    v_and_b32_e32 v0, 0xffff0000, v0
-; GFX7-SDAG-NEXT:    v_lshlrev_b32_e32 v1, 31, v1
-; GFX7-SDAG-NEXT:    v_mul_f32_e32 v0, v0, v1
-; GFX7-SDAG-NEXT:    v_and_b32_e32 v0, 0xffff0000, v0
-; GFX7-SDAG-NEXT:    s_setpc_b64 s[30:31]
-;
-; GFX7-GISEL-LABEL: fmul_select_bf16_test8:
-; GFX7-GISEL:       ; %bb.0:
-; GFX7-GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-GISEL-NEXT:    v_mov_b32_e32 v3, 0x8000
-; GFX7-GISEL-NEXT:    v_cmp_eq_u32_e32 vcc, v1, v2
-; GFX7-GISEL-NEXT:    v_cndmask_b32_e32 v1, 0, v3, vcc
-; GFX7-GISEL-NEXT:    v_cvt_f32_f16_e32 v0, v0
-; GFX7-GISEL-NEXT:    v_cvt_f32_f16_e32 v1, v1
-; GFX7-GISEL-NEXT:    v_mul_f32_e32 v0, v0, v1
-; GFX7-GISEL-NEXT:    v_cvt_f16_f32_e32 v0, v0
-; GFX7-GISEL-NEXT:    s_setpc_b64 s[30:31]
-;
-; GFX9-SDAG-LABEL: fmul_select_bf16_test8:
-; GFX9-SDAG:       ; %bb.0:
-; GFX9-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-SDAG-NEXT:    v_cmp_eq_u32_e32 vcc, v1, v2
-; GFX9-SDAG-NEXT:    v_cndmask_b32_e64 v1, 0, 1, vcc
-; GFX9-SDAG-NEXT:    v_mov_b32_e32 v2, 15
-; GFX9-SDAG-NEXT:    v_lshlrev_b16_sdwa v1, v2, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX9-SDAG-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
-; GFX9-SDAG-NEXT:    v_mul_f32_e32 v0, v0, v1
-; GFX9-SDAG-NEXT:    v_bfe_u32 v1, v0, 16, 1
-; GFX9-SDAG-NEXT:    s_movk_i32 s4, 0x7fff
-; GFX9-SDAG-NEXT:    v_add3_u32 v1, v1, v0, s4
-; GFX9-SDAG-NEXT:    v_or_b32_e32 v2, 0x400000, v0
-; GFX9-SDAG-NEXT:    v_cmp_u_f32_e32 vcc, v0, v0
-; GFX9-SDAG-NEXT:    v_cndmask_b32_e32 v0, v1, v2, vcc
-; GFX9-SDAG-NEXT:    v_lshrrev_b32_e32 v0, 16, v0
-; GFX9-SDAG-NEXT:    s_setpc_b64 s[30:31]
-;
-; GFX9-GISEL-LABEL: fmul_select_bf16_test8:
-; GFX9-GISEL:       ; %bb.0:
-; GFX9-GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-GISEL-NEXT:    v_mov_b32_e32 v3, 0x8000
-; GFX9-GISEL-NEXT:    v_cmp_eq_u32_e32 vcc, v1, v2
-; GFX9-GISEL-NEXT:    v_cndmask_b32_e32 v1, 0, v3, vcc
-; GFX9-GISEL-NEXT:    v_mul_f16_e32 v0, v0, v1
-; GFX9-GISEL-NEXT:    s_setpc_b64 s[30:31]
-;
-; GFX10-SDAG-LABEL: fmul_select_bf16_test8:
-; GFX10-SDAG:       ; %bb.0:
-; GFX10-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-SDAG-NEXT:    v_cmp_eq_u32_e32 vcc_lo, v1, v2
-; GFX10-SDAG-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
-; GFX10-SDAG-NEXT:    v_cndmask_b32_e64 v1, 0, 1, vcc_lo
-; GFX10-SDAG-NEXT:    v_lshlrev_b16 v1, 15, v1
-; GFX10-SDAG-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
-; GFX10-SDAG-NEXT:    v_mul_f32_e32 v0, v0, v1
-; GFX10-SDAG-NEXT:    v_bfe_u32 v1, v0, 16, 1
-; GFX10-SDAG-NEXT:    v_or_b32_e32 v2, 0x400000, v0
-; GFX10-SDAG-NEXT:    v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX10-SDAG-NEXT:    v_add3_u32 v1, v1, v0, 0x7fff
-; GFX10-SDAG-NEXT:    v_cndmask_b32_e32 v0, v1, v2, vcc_lo
-; GFX10-SDAG-NEXT:    v_lshrrev_b32_e32 v0, 16, v0
-; GFX10-SDAG-NEXT:    s_setpc_b64 s[30:31]
-;
-; GFX10-GISEL-LABEL: fmul_select_bf16_test8:
-; GFX10-GISEL:       ; %bb.0:
-; GFX10-GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-GISEL-NEXT:    v_cmp_eq_u32_e32 vcc_lo, v1, v2
-; GFX10-GISEL-NEXT:    v_cndmask_b32_e64 v1, 0, 0x8000, vcc_lo
-; GFX10-GISEL-NEXT:    v_mul_f16_e32 v0, v0, v1
-; GFX10-GISEL-NEXT:    s_setpc_b64 s[30:31]
+; GFX7-LABEL: fmul_select_bf16_test8:
+; GFX7:       ; %bb.0:
+; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    v_cmp_eq_u32_e32 vcc, v1, v2
+; GFX7-NEXT:    v_mul_f32_e32 v0, 1.0, v0
+; GFX7-NEXT:    v_cndmask_b32_e64 v1, 0, 1, vcc
+; GFX7-NEXT:    v_and_b32_e32 v0, 0xffff0000, v0
+; GFX7-NEXT:    v_lshlrev_b32_e32 v1, 31, v1
+; GFX7-NEXT:    v_mul_f32_e32 v0, v0, v1
+; GFX7-NEXT:    v_and_b32_e32 v0, 0xffff0000, v0
+; GFX7-NEXT:    s_setpc_b64 s[30:31]
 ;
-; GFX11-SDAG-LABEL: fmul_select_bf16_test8:
-; GFX11-SDAG:       ; %bb.0:
-; GFX11-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-SDAG-NEXT:    v_cmp_eq_u32_e32 vcc_lo, v1, v2
-; GFX11-SDAG-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
-; GFX11-SDAG-NEXT:    v_cndmask_b32_e64 v1, 0, 1, vcc_lo
-; GFX11-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SDAG-NEXT:    v_lshlrev_b16 v1, 15, v1
-; GFX11-SDAG-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
-; GFX11-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SDAG-NEXT:    v_mul_f32_e32 v0, v0, v1
-; GFX11-SDAG-NEXT:    v_bfe_u32 v1, v0, 16, 1
-; GFX11-SDAG-NEXT:    v_or_b32_e32 v2, 0x400000, v0
-; GFX11-SDAG-NEXT:    v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SDAG-NEXT:    v_add3_u32 v1, v1, v0, 0x7fff
-; GFX11-SDAG-NEXT:    v_cndmask_b32_e32 v0, v1, v2, vcc_lo
-; GFX11-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX11-SDAG-NEXT:    v_lshrrev_b32_e32 v0, 16, v0
-; GFX11-SDAG-NEXT:    s_setpc_b64 s[30:31]
+; GFX9-LABEL: fmul_select_bf16_test8:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, v1, v2
+; GFX9-NEXT:    v_cndmask_b32_e64 v1, 0, 1, vcc
+; GFX9-NEXT:    v_mov_b32_e32 v2, 15
+; GFX9-NEXT:    v_lshlrev_b16_sdwa v1, v2, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
+; GFX9-NEXT:    v_mul_f32_e32 v0, v0, v1
+; GFX9-NEXT:    v_bfe_u32 v1, v0, 16, 1
+; GFX9-NEXT:    s_movk_i32 s4, 0x7fff
+; GFX9-NEXT:    v_add3_u32 v1, v1, v0, s4
+; GFX9-NEXT:    v_or_b32_e32 v2, 0x400000, v0
+; GFX9-NEXT:    v_cmp_u_f32_e32 vcc, v0, v0
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v1, v2, vcc
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v0
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
-; GFX11-GISEL-LABEL: fmul_select_bf16_test8:
-; GFX11-GISEL:       ; %bb.0:
-; GFX11-GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-GISEL-NEXT:    v_cmp_eq_u32_e32 vcc_lo, v1, v2
-; GFX11-GISEL-NEXT:    v_cndmask_b32_e64 v1, 0, 0x8000, vcc_lo
-; GFX11-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX11-GISEL-NEXT:    v_mul_f16_e32 v0, v0, v1
-; GFX11-GISEL-NEXT:    s_setpc_b64 s[30:31]
+; GFX10-LABEL: fmul_select_bf16_test8:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    v_cmp_eq_u32_e32 vcc_lo, v1, v2
+; GFX10-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
+; GFX10-NEXT:    v_cndmask_b32_e64 v1, 0, 1, vcc_lo
+; GFX10-NEXT:    v_lshlrev_b16 v1, 15, v1
+; GFX10-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
+; GFX10-NEXT:    v_mul_f32_e32 v0, v0, v1
+; GFX10-NEXT:    v_bfe_u32 v1, v0, 16, 1
+; GFX10-NEXT:    v_or_b32_e32 v2, 0x400000, v0
+; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX10-NEXT:    v_add3_u32 v1, v1, v0, 0x7fff
+; GFX10-NEXT:    v_cndmask_b32_e32 v0, v1, v2, vcc_lo
+; GFX10-NEXT:    v_lshrrev_b32_e32 v0, 16, v0
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: fmul_select_bf16_test8:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    v_cmp_eq_u32_e32 vcc_lo, v1, v2
+; GFX11-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
+; GFX11-NEXT:    v_cndmask_b32_e64 v1, 0, 1, vcc_lo
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_lshlrev_b16 v1, 15, v1
+; GFX11-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_mul_f32_e32 v0, v0, v1
+; GFX11-NEXT:    v_bfe_u32 v1, v0, 16, 1
+; GFX11-NEXT:    v_or_b32_e32 v2, 0x400000, v0
+; GFX11-NEXT:    v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_add3_u32 v1, v1, v0, 0x7fff
+; GFX11-NEXT:    v_cndmask_b32_e32 v0, v1, v2, vcc_lo
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT:    v_lshrrev_b32_e32 v0, 16, v0
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
   %bool = icmp eq i32 %bool.arg1, %bool.arg2
   %y = select i1 %bool, bfloat -0.000000e+00, bfloat 0.000000e+00
   %ldexp = fmul bfloat %x, %y
@@ -3543,121 +3208,74 @@ define bfloat @fmul_select_bf16_test8(bfloat %x, i32 %bool.arg1, i32 %bool.arg2)
 }
 
 define bfloat @fmul_select_bf16_test9(bfloat %x, i32 %bool.arg1, i32 %bool.arg2) {
-; GFX7-SDAG-LABEL: fmul_select_bf16_test9:
-; GFX7-SDAG:       ; %bb.0:
-; GFX7-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-SDAG-NEXT:    v_mul_f32_e32 v0, 1.0, v0
-; GFX7-SDAG-NEXT:    v_mov_b32_e32 v3, 0xc2000000
-; GFX7-SDAG-NEXT:    v_mov_b32_e32 v4, 0xc1800000
-; GFX7-SDAG-NEXT:    v_cmp_eq_u32_e32 vcc, v1, v2
-; GFX7-SDAG-NEXT:    v_cndmask_b32_e32 v1, v3, v4, vcc
-; GFX7-SDAG-NEXT:    v_and_b32_e32 v0, 0xffff0000, v0
-; GFX7-SDAG-NEXT:    v_mul_f32_e32 v0, v0, v1
-; GFX7-SDAG-NEXT:    v_and_b32_e32 v0, 0xffff0000, v0
-; GFX7-SDAG-NEXT:    s_setpc_b64 s[30:31]
-;
-; GFX7-GISEL-LABEL: fmul_select_bf16_test9:
-; GFX7-GISEL:       ; %bb.0:
-; GFX7-GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-GISEL-NEXT:    v_cvt_f32_f16_e64 v0, -v0
-; GFX7-GISEL-NEXT:    v_cmp_eq_u32_e32 vcc, v1, v2
-; GFX7-GISEL-NEXT:    v_cndmask_b32_e64 v1, 0, -1, vcc
-; GFX7-GISEL-NEXT:    v_add_i32_e32 v1, vcc, 5, v1
-; GFX7-GISEL-NEXT:    v_ldexp_f32_e32 v0, v0, v1
-; GFX7-GISEL-NEXT:    v_cvt_f16_f32_e32 v0, v0
-; GFX7-GISEL-NEXT:    s_setpc_b64 s[30:31]
-;
-; GFX9-SDAG-LABEL: fmul_select_bf16_test9:
-; GFX9-SDAG:       ; %bb.0:
-; GFX9-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-SDAG-NEXT:    v_mov_b32_e32 v3, 0xffffc200
-; GFX9-SDAG-NEXT:    v_mov_b32_e32 v4, 0xffffc180
-; GFX9-SDAG-NEXT:    v_cmp_eq_u32_e32 vcc, v1, v2
-; GFX9-SDAG-NEXT:    v_cndmask_b32_e32 v1, v3, v4, vcc
-; GFX9-SDAG-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
-; GFX9-SDAG-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
-; GFX9-SDAG-NEXT:    v_mul_f32_e32 v0, v0, v1
-; GFX9-SDAG-NEXT:    v_bfe_u32 v1, v0, 16, 1
-; GFX9-SDAG-NEXT:    s_movk_i32 s4, 0x7fff
-; GFX9-SDAG-NEXT:    v_add3_u32 v1, v1, v0, s4
-; GFX9-SDAG-NEXT:    v_or_b32_e32 v2, 0x400000, v0
-; GFX9-SDAG-NEXT:    v_cmp_u_f32_e32 vcc, v0, v0
-; GFX9-SDAG-NEXT:    v_cndmask_b32_e32 v0, v1, v2, vcc
-; GFX9-SDAG-NEXT:    v_lshrrev_b32_e32 v0, 16, v0
-; GFX9-SDAG-NEXT:    s_setpc_b64 s[30:31]
-;
-; GFX9-GISEL-LABEL: fmul_select_bf16_test9:
-; GFX9-GISEL:       ; %bb.0:
-; GFX9-GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-GISEL-NEXT:    v_cmp_eq_u32_e32 vcc, v1, v2
-; GFX9-GISEL-NEXT:    v_cndmask_b32_e64 v1, 0, -1, vcc
-; GFX9-GISEL-NEXT:    v_add_u32_e32 v1, 5, v1
-; GFX9-GISEL-NEXT:    v_mov_b32_e32 v2, 0xffff8000
-; GFX9-GISEL-NEXT:    v_mov_b32_e32 v3, 0x7fff
-; GFX9-GISEL-NEXT:    v_med3_i32 v1, v1, v2, v3
-; GFX9-GISEL-NEXT:    v_ldexp_f16_e64 v0, -v0, v1
-; GFX9-GISEL-NEXT:    s_setpc_b64 s[30:31]
-;
-; GFX10-SDAG-LABEL: fmul_select_bf16_test9:
-; GFX10-SDAG:       ; %bb.0:
-; GFX10-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-SDAG-NEXT:    v_mov_b32_e32 v3, 0xffffc180
-; GFX10-SDAG-NEXT:    v_cmp_eq_u32_e32 vcc_lo, v1, v2
-; GFX10-SDAG-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
-; GFX10-SDAG-NEXT:    v_cndmask_b32_e32 v1, 0xffffc200, v3, vcc_lo
-; GFX10-SDAG-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
-; GFX10-SDAG-NEXT:    v_mul_f32_e32 v0, v0, v1
-; GFX10-SDAG-NEXT:    v_bfe_u32 v1, v0, 16, 1
-; GFX10-SDAG-NEXT:    v_or_b32_e32 v2, 0x400000, v0
-; GFX10-SDAG-NEXT:    v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX10-SDAG-NEXT:    v_add3_u32 v1, v1, v0, 0x7fff
-; GFX10-SDAG-NEXT:    v_cndmask_b32_e32 v0, v1, v2, vcc_lo
-; GFX10-SDAG-NEXT:    v_lshrrev_b32_e32 v0, 16, v0
-; GFX10-SDAG-NEXT:    s_setpc_b64 s[30:31]
+; GFX7-LABEL: fmul_select_bf16_test9:
+; GFX7:       ; %bb.0:
+; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    v_mul_f32_e32 v0, 1.0, v0
+; GFX7-NEXT:    v_mov_b32_e32 v3, 0xc2000000
+; GFX7-NEXT:    v_mov_b32_e32 v4, 0xc1800000
+; GFX7-NEXT:    v_cmp_eq_u32_e32 vcc, v1, v2
+; GFX7-NEXT:    v_cndmask_b32_e32 v1, v3, v4, vcc
+; GFX7-NEXT:    v_and_b32_e32 v0, 0xffff0000, v0
+; GFX7-NEXT:    v_mul_f32_e32 v0, v0, v1
+; GFX7-NEXT:    v_and_b32_e32 v0, 0xffff0000, v0
+; GFX7-NEXT:    s_setpc_b64 s[30:31]
 ;
-; GFX10-GISEL-LABEL: fmul_select_bf16_test9:
-; GFX10-GISEL:       ; %bb.0:
-; GFX10-GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-GISEL-NEXT:    v_cmp_eq_u32_e32 vcc_lo, v1, v2
-; GFX10-GISEL-NEXT:    v_mov_b32_e32 v2, 0x7fff
-; GFX10-GISEL-NEXT:    v_cndmask_b32_e64 v1, 0, -1, vcc_lo
-; GFX10-GISEL-NEXT:    v_add_nc_u32_e32 v1, 5, v1
-; GFX10-GISEL-NEXT:    v_med3_i32 v1, 0xffff8000, v1, v2
-; GFX10-GISEL-NEXT:    v_ldexp_f16_e64 v0, -v0, v1
-; GFX10-GISEL-NEXT:    s_setpc_b64 s[30:31]
+; GFX9-LABEL: fmul_select_bf16_test9:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v3, 0xffffc200
+; GFX9-NEXT:    v_mov_b32_e32 v4, 0xffffc180
+; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, v1, v2
+; GFX9-NEXT:    v_cndmask_b32_e32 v1, v3, v4, vcc
+; GFX9-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
+; GFX9-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
+; GFX9-NEXT:    v_mul_f32_e32 v0, v0, v1
+; GFX9-NEXT:    v_bfe_u32 v1, v0, 16, 1
+; GFX9-NEXT:    s_movk_i32 s4, 0x7fff
+; GFX9-NEXT:    v_add3_u32 v1, v1, v0, s4
+; GFX9-NEXT:    v_or_b32_e32 v2, 0x400000, v0
+; GFX9-NEXT:    v_cmp_u_f32_e32 vcc, v0, v0
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v1, v2, vcc
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v0
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
-; GFX11-SDAG-LABEL: fmul_select_bf16_test9:
-; GFX11-SDAG:       ; %bb.0:
-; GFX11-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-SDAG-NEXT:    v_dual_mov_b32 v3, 0xffffc180 :: v_dual_lshlrev_b32 v0, 16, v0
-; GFX11-SDAG-NEXT:    v_cmp_eq_u32_e32 vcc_lo, v1, v2
-; GFX11-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SDAG-NEXT:    v_cndmask_b32_e32 v1, 0xffffc200, v3, vcc_lo
-; GFX11-SDAG-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
-; GFX11-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SDAG-NEXT:    v_mul_f32_e32 v0, v0, v1
-; GFX11-SDAG-NEXT:    v_bfe_u32 v1, v0, 16, 1
-; GFX11-SDAG-NEXT:    v_or_b32_e32 v2, 0x400000, v0
-; GFX11-SDAG-NEXT:    v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SDAG-NEXT:    v_add3_u32 v1, v1, v0, 0x7fff
-; GFX11-SDAG-NEXT:    v_cndmask_b32_e32 v0, v1, v2, vcc_lo
-; GFX11-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX11-SDAG-NEXT:    v_lshrrev_b32_e32 v0, 16, v0
-; GFX11-SDAG-NEXT:    s_setpc_b64 s[30:31]
+; GFX10-LABEL: fmul_select_bf16_test9:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    v_mov_b32_e32 v3, 0xffffc180
+; GFX10-NEXT:    v_cmp_eq_u32_e32 vcc_lo, v1, v2
+; GFX10-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
+; GFX10-NEXT:    v_cndmask_b32_e32 v1, 0xffffc200, v3, vcc_lo
+; GFX10-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
+; GFX10-NEXT:    v_mul_f32_e32 v0, v0, v1
+; GFX10-NEXT:    v_bfe_u32 v1, v0, 16, 1
+; GFX10-NEXT:    v_or_b32_e32 v2, 0x400000, v0
+; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX10-NEXT:    v_add3_u32 v1, v1, v0, 0x7fff
+; GFX10-NEXT:    v_cndmask_b32_e32 v0, v1, v2, vcc_lo
+; GFX10-NEXT:    v_lshrrev_b32_e32 v0, 16, v0
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
 ;
-; GFX11-GISEL-LABEL: fmul_select_bf16_test9:
-; GFX11-GISEL:       ; %bb.0:
-; GFX11-GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-GISEL-NEXT:    v_cmp_eq_u32_e32 vcc_lo, v1, v2
-; GFX11-GISEL-NEXT:    v_mov_b32_e32 v2, 0x7fff
-; GFX11-GISEL-NEXT:    v_cndmask_b32_e64 v1, 0, -1, vcc_lo
-; GFX11-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-GISEL-NEXT:    v_add_nc_u32_e32 v1, 5, v1
-; GFX11-GISEL-NEXT:    v_med3_i32 v1, 0xffff8000, v1, v2
-; GFX11-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX11-GISEL-NEXT:    v_ldexp_f16_e64 v0, -v0, v1
-; GFX11-GISEL-NEXT:    s_setpc_b64 s[30:31]
+; GFX11-LABEL: fmul_select_bf16_test9:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    v_dual_mov_b32 v3, 0xffffc180 :: v_dual_lshlrev_b32 v0, 16, v0
+; GFX11-NEXT:    v_cmp_eq_u32_e32 vcc_lo, v1, v2
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_cndmask_b32_e32 v1, 0xffffc200, v3, vcc_lo
+; GFX11-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_mul_f32_e32 v0, v0, v1
+; GFX11-NEXT:    v_bfe_u32 v1, v0, 16, 1
+; GFX11-NEXT:    v_or_b32_e32 v2, 0x400000, v0
+; GFX11-NEXT:    v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_add3_u32 v1, v1, v0, 0x7fff
+; GFX11-NEXT:    v_cndmask_b32_e32 v0, v1, v2, vcc_lo
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT:    v_lshrrev_b32_e32 v0, 16, v0
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
   %bool = icmp eq i32 %bool.arg1, %bool.arg2
   %y = select i1 %bool, bfloat -1.600000e+01, bfloat -3.200000e+01
   %ldexp = fmul bfloat %x, %y
@@ -3665,111 +3283,74 @@ define bfloat @fmul_select_bf16_test9(bfloat %x, i32 %bool.arg1, i32 %bool.arg2)
 }
 
 define bfloat @fmul_select_bf16_test10_sel_log2val_pos65_pos56(bfloat %x, i32 %bool.arg1, i32 %bool.arg2) {
-; GFX7-SDAG-LABEL: fmul_select_bf16_test10_sel_log2val_pos65_pos56:
-; GFX7-SDAG:       ; %bb.0:
-; GFX7-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-SDAG-NEXT:    v_mul_f32_e32 v0, 1.0, v0
-; GFX7-SDAG-NEXT:    v_mov_b32_e32 v3, 0xdb800000
-; GFX7-SDAG-NEXT:    v_bfrev_b32_e32 v4, 7
-; GFX7-SDAG-NEXT:    v_cmp_eq_u32_e32 vcc, v1, v2
-; GFX7-SDAG-NEXT:    v_cndmask_b32_e32 v1, v3, v4, vcc
-; GFX7-SDAG-NEXT:    v_and_b32_e32 v0, 0xffff0000, v0
-; GFX7-SDAG-NEXT:    v_mul_f32_e32 v0, v0, v1
-; GFX7-SDAG-NEXT:    v_and_b32_e32 v0, 0xffff0000, v0
-; GFX7-SDAG-NEXT:    s_setpc_b64 s[30:31]
-;
-; GFX7-GISEL-LABEL: fmul_select_bf16_test10_sel_log2val_pos65_pos56:
-; GFX7-GISEL:       ; %bb.0:
-; GFX7-GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-GISEL-NEXT:    v_cvt_f32_f16_e64 v0, -v0
-; GFX7-GISEL-NEXT:    v_mov_b32_e32 v3, 0x41
-; GFX7-GISEL-NEXT:    v_cmp_eq_u32_e32 vcc, v1, v2
-; GFX7-GISEL-NEXT:    v_cndmask_b32_e32 v1, 56, v3, vcc
-; GFX7-GISEL-NEXT:    v_ldexp_f32_e32 v0, v0, v1
-; GFX7-GISEL-NEXT:    v_cvt_f16_f32_e32 v0, v0
-; GFX7-GISEL-NEXT:    s_setpc_b64 s[30:31]
-;
-; GFX9-SDAG-LABEL: fmul_select_bf16_test10_sel_log2val_pos65_pos56:
-; GFX9-SDAG:       ; %bb.0:
-; GFX9-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-SDAG-NEXT:    v_mov_b32_e32 v3, 0xffffdb80
-; GFX9-SDAG-NEXT:    v_mov_b32_e32 v4, 0xffffe000
-; GFX9-SDAG-NEXT:    v_cmp_eq_u32_e32 vcc, v1, v2
-; GFX9-SDAG-NEXT:    v_cndmask_b32_e32 v1, v3, v4, vcc
-; GFX9-SDAG-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
-; GFX9-SDAG-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
-; GFX9-SDAG-NEXT:    v_mul_f32_e32 v0, v0, v1
-; GFX9-SDAG-NEXT:    v_bfe_u32 v1, v0, 16, 1
-; GFX9-SDAG-NEXT:    s_movk_i32 s4, 0x7fff
-; GFX9-SDAG-NEXT:    v_add3_u32 v1, v1, v0, s4
-; GFX9-SDAG-NEXT:    v_or_b32_e32 v2, 0x400000, v0
-; GFX9-SDAG-NEXT:    v_cmp_u_f32_e32 vcc, v0, v0
-; GFX9-SDAG-NEXT:    v_cndmask_b32_e32 v0, v1, v2, vcc
-; GFX9-SDAG-NEXT:    v_lshrrev_b32_e32 v0, 16, v0
-; GFX9-SDAG-NEXT:    s_setpc_b64 s[30:31]
-;
-; GFX9-GISEL-LABEL: fmul_select_bf16_test10_sel_log2val_pos65_pos56:
-; GFX9-GISEL:       ; %bb.0:
-; GFX9-GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-GISEL-NEXT:    v_mov_b32_e32 v3, 0x41
-; GFX9-GISEL-NEXT:    v_cmp_eq_u32_e32 vcc, v1, v2
-; GFX9-GISEL-NEXT:    v_cndmask_b32_e32 v1, 56, v3, vcc
-; GFX9-GISEL-NEXT:    v_ldexp_f16_e64 v0, -v0, v1
-; GFX9-GISEL-NEXT:    s_setpc_b64 s[30:31]
-;
-; GFX10-SDAG-LABEL: fmul_select_bf16_test10_sel_log2val_pos65_pos56:
-; GFX10-SDAG:       ; %bb.0:
-; GFX10-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-SDAG-NEXT:    v_mov_b32_e32 v3, 0xffffe000
-; GFX10-SDAG-NEXT:    v_cmp_eq_u32_e32 vcc_lo, v1, v2
-; GFX10-SDAG-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
-; GFX10-SDAG-NEXT:    v_cndmask_b32_e32 v1, 0xffffdb80, v3, vcc_lo
-; GFX10-SDAG-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
-; GFX10-SDAG-NEXT:    v_mul_f32_e32 v0, v0, v1
-; GFX10-SDAG-NEXT:    v_bfe_u32 v1, v0, 16, 1
-; GFX10-SDAG-NEXT:    v_or_b32_e32 v2, 0x400000, v0
-; GFX10-SDAG-NEXT:    v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX10-SDAG-NEXT:    v_add3_u32 v1, v1, v0, 0x7fff
-; GFX10-SDAG-NEXT:    v_cndmask_b32_e32 v0, v1, v2, vcc_lo
-; GFX10-SDAG-NEXT:    v_lshrrev_b32_e32 v0, 16, v0
-; GFX10-SDAG-NEXT:    s_setpc_b64 s[30:31]
+; GFX7-LABEL: fmul_select_bf16_test10_sel_log2val_pos65_pos56:
+; GFX7:       ; %bb.0:
+; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    v_mul_f32_e32 v0, 1.0, v0
+; GFX7-NEXT:    v_mov_b32_e32 v3, 0xdb800000
+; GFX7-NEXT:    v_bfrev_b32_e32 v4, 7
+; GFX7-NEXT:    v_cmp_eq_u32_e32 vcc, v1, v2
+; GFX7-NEXT:    v_cndmask_b32_e32 v1, v3, v4, vcc
+; GFX7-NEXT:    v_and_b32_e32 v0, 0xffff0000, v0
+; GFX7-NEXT:    v_mul_f32_e32 v0, v0, v1
+; GFX7-NEXT:    v_and_b32_e32 v0, 0xffff0000, v0
+; GFX7-NEXT:    s_setpc_b64 s[30:31]
 ;
-; GFX10-GISEL-LABEL: fmul_select_bf16_test10_sel_log2val_pos65_pos56:
-; GFX10-GISEL:       ; %bb.0:
-; GFX10-GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-GISEL-NEXT:    v_cmp_eq_u32_e32 vcc_lo, v1, v2
-; GFX10-GISEL-NEXT:    v_cndmask_b32_e64 v1, 56, 0x41, vcc_lo
-; GFX10-GISEL-NEXT:    v_ldexp_f16_e64 v0, -v0, v1
-; GFX10-GISEL-NEXT:    s_setpc_b64 s[30:31]
+; GFX9-LABEL: fmul_select_bf16_test10_sel_log2val_pos65_pos56:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v3, 0xffffdb80
+; GFX9-NEXT:    v_mov_b32_e32 v4, 0xffffe000
+; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, v1, v2
+; GFX9-NEXT:    v_cndmask_b32_e32 v1, v3, v4, vcc
+; GFX9-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
+; GFX9-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
+; GFX9-NEXT:    v_mul_f32_e32 v0, v0, v1
+; GFX9-NEXT:    v_bfe_u32 v1, v0, 16, 1
+; GFX9-NEXT:    s_movk_i32 s4, 0x7fff
+; GFX9-NEXT:    v_add3_u32 v1, v1, v0, s4
+; GFX9-NEXT:    v_or_b32_e32 v2, 0x400000, v0
+; GFX9-NEXT:    v_cmp_u_f32_e32 vcc, v0, v0
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v1, v2, vcc
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v0
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
-; GFX11-SDAG-LABEL: fmul_select_bf16_test10_sel_log2val_pos65_pos56:
-; GFX11-SDAG:       ; %bb.0:
-; GFX11-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-SDAG-NEXT:    v_dual_mov_b32 v3, 0xffffe000 :: v_dual_lshlrev_b32 v0, 16, v0
-; GFX11-SDAG-NEXT:    v_cmp_eq_u32_e32 vcc_lo, v1, v2
-; GFX11-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SDAG-NEXT:    v_cndmask_b32_e32 v1, 0xffffdb80, v3, vcc_lo
-; GFX11-SDAG-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
-; GFX11-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SDAG-NEXT:    v_mul_f32_e32 v0, v0, v1
-; GFX11-SDAG-NEXT:    v_bfe_u32 v1, v0, 16, 1
-; GFX11-SDAG-NEXT:    v_or_b32_e32 v2, 0x400000, v0
-; GFX11-SDAG-NEXT:    v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SDAG-NEXT:    v_add3_u32 v1, v1, v0, 0x7fff
-; GFX11-SDAG-NEXT:    v_cndmask_b32_e32 v0, v1, v2, vcc_lo
-; GFX11-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX11-SDAG-NEXT:    v_lshrrev_b32_e32 v0, 16, v0
-; GFX11-SDAG-NEXT:    s_setpc_b64 s[30:31]
+; GFX10-LABEL: fmul_select_bf16_test10_sel_log2val_pos65_pos56:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    v_mov_b32_e32 v3, 0xffffe000
+; GFX10-NEXT:    v_cmp_eq_u32_e32 vcc_lo, v1, v2
+; GFX10-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
+; GFX10-NEXT:    v_cndmask_b32_e32 v1, 0xffffdb80, v3, vcc_lo
+; GFX10-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
+; GFX10-NEXT:    v_mul_f32_e32 v0, v0, v1
+; GFX10-NEXT:    v_bfe_u32 v1, v0, 16, 1
+; GFX10-NEXT:    v_or_b32_e32 v2, 0x400000, v0
+; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX10-NEXT:    v_add3_u32 v1, v1, v0, 0x7fff
+; GFX10-NEXT:    v_cndmask_b32_e32 v0, v1, v2, vcc_lo
+; GFX10-NEXT:    v_lshrrev_b32_e32 v0, 16, v0
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
 ;
-; GFX11-GISEL-LABEL: fmul_select_bf16_test10_sel_log2val_pos65_pos56:
-; GFX11-GISEL:       ; %bb.0:
-; GFX11-GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-GISEL-NEXT:    v_cmp_eq_u32_e32 vcc_lo, v1, v2
-; GFX11-GISEL-NEXT:    v_cndmask_b32_e64 v1, 56, 0x41, vcc_lo
-; GFX11-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX11-GISEL-NEXT:    v_ldexp_f16_e64 v0, -v0, v1
-; GFX11-GISEL-NEXT:    s_setpc_b64 s[30:31]
+; GFX11-LABEL: fmul_select_bf16_test10_sel_log2val_pos65_pos56:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    v_dual_mov_b32 v3, 0xffffe000 :: v_dual_lshlrev_b32 v0, 16, v0
+; GFX11-NEXT:    v_cmp_eq_u32_e32 vcc_lo, v1, v2
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_cndmask_b32_e32 v1, 0xffffdb80, v3, vcc_lo
+; GFX11-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_mul_f32_e32 v0, v0, v1
+; GFX11-NEXT:    v_bfe_u32 v1, v0, 16, 1
+; GFX11-NEXT:    v_or_b32_e32 v2, 0x400000, v0
+; GFX11-NEXT:    v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_add3_u32 v1, v1, v0, 0x7fff
+; GFX11-NEXT:    v_cndmask_b32_e32 v0, v1, v2, vcc_lo
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT:    v_lshrrev_b32_e32 v0, 16, v0
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
   %bool = icmp eq i32 %bool.arg1, %bool.arg2
   %y = select i1 %bool, bfloat 0xRE000, bfloat 0xRDB80
   %ldexp = fmul bfloat %x, %y
@@ -3777,111 +3358,74 @@ define bfloat @fmul_select_bf16_test10_sel_log2val_pos65_pos56(bfloat %x, i32 %b
 }
 
 define bfloat @fmul_select_bf16_test11_sel_log2val_neg22_pos25(bfloat %x, i32 %bool.arg1, i32 %bool.arg2) {
-; GFX7-SDAG-LABEL: fmul_select_bf16_test11_sel_log2val_neg22_pos25:
-; GFX7-SDAG:       ; %bb.0:
-; GFX7-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-SDAG-NEXT:    v_mul_f32_e32 v0, 1.0, v0
-; GFX7-SDAG-NEXT:    v_bfrev_b32_e32 v3, 50
-; GFX7-SDAG-NEXT:    v_mov_b32_e32 v4, 0x34800000
-; GFX7-SDAG-NEXT:    v_cmp_eq_u32_e32 vcc, v1, v2
-; GFX7-SDAG-NEXT:    v_cndmask_b32_e32 v1, v3, v4, vcc
-; GFX7-SDAG-NEXT:    v_and_b32_e32 v0, 0xffff0000, v0
-; GFX7-SDAG-NEXT:    v_mul_f32_e32 v0, v0, v1
-; GFX7-SDAG-NEXT:    v_and_b32_e32 v0, 0xffff0000, v0
-; GFX7-SDAG-NEXT:    s_setpc_b64 s[30:31]
-;
-; GFX7-GISEL-LABEL: fmul_select_bf16_test11_sel_log2val_neg22_pos25:
-; GFX7-GISEL:       ; %bb.0:
-; GFX7-GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-GISEL-NEXT:    v_cvt_f32_f16_e32 v0, v0
-; GFX7-GISEL-NEXT:    v_not_b32_e32 v3, 21
-; GFX7-GISEL-NEXT:    v_cmp_eq_u32_e32 vcc, v1, v2
-; GFX7-GISEL-NEXT:    v_cndmask_b32_e32 v1, 25, v3, vcc
-; GFX7-GISEL-NEXT:    v_ldexp_f32_e32 v0, v0, v1
-; GFX7-GISEL-NEXT:    v_cvt_f16_f32_e32 v0, v0
-; GFX7-GISEL-NEXT:    s_setpc_b64 s[30:31]
-;
-; GFX9-SDAG-LABEL: fmul_select_bf16_test11_sel_log2val_neg22_pos25:
-; GFX9-SDAG:       ; %bb.0:
-; GFX9-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-SDAG-NEXT:    v_mov_b32_e32 v3, 0x4c00
-; GFX9-SDAG-NEXT:    v_mov_b32_e32 v4, 0x3480
-; GFX9-SDAG-NEXT:    v_cmp_eq_u32_e32 vcc, v1, v2
-; GFX9-SDAG-NEXT:    v_cndmask_b32_e32 v1, v3, v4, vcc
-; GFX9-SDAG-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
-; GFX9-SDAG-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
-; GFX9-SDAG-NEXT:    v_mul_f32_e32 v0, v0, v1
-; GFX9-SDAG-NEXT:    v_bfe_u32 v1, v0, 16, 1
-; GFX9-SDAG-NEXT:    s_movk_i32 s4, 0x7fff
-; GFX9-SDAG-NEXT:    v_add3_u32 v1, v1, v0, s4
-; GFX9-SDAG-NEXT:    v_or_b32_e32 v2, 0x400000, v0
-; GFX9-SDAG-NEXT:    v_cmp_u_f32_e32 vcc, v0, v0
-; GFX9-SDAG-NEXT:    v_cndmask_b32_e32 v0, v1, v2, vcc
-; GFX9-SDAG-NEXT:    v_lshrrev_b32_e32 v0, 16, v0
-; GFX9-SDAG-NEXT:    s_setpc_b64 s[30:31]
-;
-; GFX9-GISEL-LABEL: fmul_select_bf16_test11_sel_log2val_neg22_pos25:
-; GFX9-GISEL:       ; %bb.0:
-; GFX9-GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-GISEL-NEXT:    v_not_b32_e32 v3, 21
-; GFX9-GISEL-NEXT:    v_cmp_eq_u32_e32 vcc, v1, v2
-; GFX9-GISEL-NEXT:    v_cndmask_b32_e32 v1, 25, v3, vcc
-; GFX9-GISEL-NEXT:    v_ldexp_f16_e32 v0, v0, v1
-; GFX9-GISEL-NEXT:    s_setpc_b64 s[30:31]
-;
-; GFX10-SDAG-LABEL: fmul_select_bf16_test11_sel_log2val_neg22_pos25:
-; GFX10-SDAG:       ; %bb.0:
-; GFX10-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-SDAG-NEXT:    v_mov_b32_e32 v3, 0x3480
-; GFX10-SDAG-NEXT:    v_cmp_eq_u32_e32 vcc_lo, v1, v2
-; GFX10-SDAG-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
-; GFX10-SDAG-NEXT:    v_cndmask_b32_e32 v1, 0x4c00, v3, vcc_lo
-; GFX10-SDAG-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
-; GFX10-SDAG-NEXT:    v_mul_f32_e32 v0, v0, v1
-; GFX10-SDAG-NEXT:    v_bfe_u32 v1, v0, 16, 1
-; GFX10-SDAG-NEXT:    v_or_b32_e32 v2, 0x400000, v0
-; GFX10-SDAG-NEXT:    v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX10-SDAG-NEXT:    v_add3_u32 v1, v1, v0, 0x7fff
-; GFX10-SDAG-NEXT:    v_cndmask_b32_e32 v0, v1, v2, vcc_lo
-; GFX10-SDAG-NEXT:    v_lshrrev_b32_e32 v0, 16, v0
-; GFX10-SDAG-NEXT:    s_setpc_b64 s[30:31]
+; GFX7-LABEL: fmul_select_bf16_test11_sel_log2val_neg22_pos25:
+; GFX7:       ; %bb.0:
+; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    v_mul_f32_e32 v0, 1.0, v0
+; GFX7-NEXT:    v_bfrev_b32_e32 v3, 50
+; GFX7-NEXT:    v_mov_b32_e32 v4, 0x34800000
+; GFX7-NEXT:    v_cmp_eq_u32_e32 vcc, v1, v2
+; GFX7-NEXT:    v_cndmask_b32_e32 v1, v3, v4, vcc
+; GFX7-NEXT:    v_and_b32_e32 v0, 0xffff0000, v0
+; GFX7-NEXT:    v_mul_f32_e32 v0, v0, v1
+; GFX7-NEXT:    v_and_b32_e32 v0, 0xffff0000, v0
+; GFX7-NEXT:    s_setpc_b64 s[30:31]
 ;
-; GFX10-GISEL-LABEL: fmul_select_bf16_test11_sel_log2val_neg22_pos25:
-; GFX10-GISEL:       ; %bb.0:
-; GFX10-GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-GISEL-NEXT:    v_cmp_eq_u32_e32 vcc_lo, v1, v2
-; GFX10-GISEL-NEXT:    v_cndmask_b32_e64 v1, 25, 0xffffffea, vcc_lo
-; GFX10-GISEL-NEXT:    v_ldexp_f16_e32 v0, v0, v1
-; GFX10-GISEL-NEXT:    s_setpc_b64 s[30:31]
+; GFX9-LABEL: fmul_select_bf16_test11_sel_log2val_neg22_pos25:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v3, 0x4c00
+; GFX9-NEXT:    v_mov_b32_e32 v4, 0x3480
+; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, v1, v2
+; GFX9-NEXT:    v_cndmask_b32_e32 v1, v3, v4, vcc
+; GFX9-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
+; GFX9-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
+; GFX9-NEXT:    v_mul_f32_e32 v0, v0, v1
+; GFX9-NEXT:    v_bfe_u32 v1, v0, 16, 1
+; GFX9-NEXT:    s_movk_i32 s4, 0x7fff
+; GFX9-NEXT:    v_add3_u32 v1, v1, v0, s4
+; GFX9-NEXT:    v_or_b32_e32 v2, 0x400000, v0
+; GFX9-NEXT:    v_cmp_u_f32_e32 vcc, v0, v0
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v1, v2, vcc
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v0
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
-; GFX11-SDAG-LABEL: fmul_select_bf16_test11_sel_log2val_neg22_pos25:
-; GFX11-SDAG:       ; %bb.0:
-; GFX11-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-SDAG-NEXT:    v_dual_mov_b32 v3, 0x3480 :: v_dual_lshlrev_b32 v0, 16, v0
-; GFX11-SDAG-NEXT:    v_cmp_eq_u32_e32 vcc_lo, v1, v2
-; GFX11-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SDAG-NEXT:    v_cndmask_b32_e32 v1, 0x4c00, v3, vcc_lo
-; GFX11-SDAG-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
-; GFX11-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SDAG-NEXT:    v_mul_f32_e32 v0, v0, v1
-; GFX11-SDAG-NEXT:    v_bfe_u32 v1, v0, 16, 1
-; GFX11-SDAG-NEXT:    v_or_b32_e32 v2, 0x400000, v0
-; GFX11-SDAG-NEXT:    v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SDAG-NEXT:    v_add3_u32 v1, v1, v0, 0x7fff
-; GFX11-SDAG-NEXT:    v_cndmask_b32_e32 v0, v1, v2, vcc_lo
-; GFX11-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX11-SDAG-NEXT:    v_lshrrev_b32_e32 v0, 16, v0
-; GFX11-SDAG-NEXT:    s_setpc_b64 s[30:31]
+; GFX10-LABEL: fmul_select_bf16_test11_sel_log2val_neg22_pos25:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    v_mov_b32_e32 v3, 0x3480
+; GFX10-NEXT:    v_cmp_eq_u32_e32 vcc_lo, v1, v2
+; GFX10-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
+; GFX10-NEXT:    v_cndmask_b32_e32 v1, 0x4c00, v3, vcc_lo
+; GFX10-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
+; GFX10-NEXT:    v_mul_f32_e32 v0, v0, v1
+; GFX10-NEXT:    v_bfe_u32 v1, v0, 16, 1
+; GFX10-NEXT:    v_or_b32_e32 v2, 0x400000, v0
+; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX10-NEXT:    v_add3_u32 v1, v1, v0, 0x7fff
+; GFX10-NEXT:    v_cndmask_b32_e32 v0, v1, v2, vcc_lo
+; GFX10-NEXT:    v_lshrrev_b32_e32 v0, 16, v0
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
 ;
-; GFX11-GISEL-LABEL: fmul_select_bf16_test11_sel_log2val_neg22_pos25:
-; GFX11-GISEL:       ; %bb.0:
-; GFX11-GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-GISEL-NEXT:    v_cmp_eq_u32_e32 vcc_lo, v1, v2
-; GFX11-GISEL-NEXT:    v_cndmask_b32_e64 v1, 25, 0xffffffea, vcc_lo
-; GFX11-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX11-GISEL-NEXT:    v_ldexp_f16_e32 v0, v0, v1
-; GFX11-GISEL-NEXT:    s_setpc_b64 s[30:31]
+; GFX11-LABEL: fmul_select_bf16_test11_sel_log2val_neg22_pos25:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    v_dual_mov_b32 v3, 0x3480 :: v_dual_lshlrev_b32 v0, 16, v0
+; GFX11-NEXT:    v_cmp_eq_u32_e32 vcc_lo, v1, v2
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_cndmask_b32_e32 v1, 0x4c00, v3, vcc_lo
+; GFX11-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_mul_f32_e32 v0, v0, v1
+; GFX11-NEXT:    v_bfe_u32 v1, v0, 16, 1
+; GFX11-NEXT:    v_or_b32_e32 v2, 0x400000, v0
+; GFX11-NEXT:    v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_add3_u32 v1, v1, v0, 0x7fff
+; GFX11-NEXT:    v_cndmask_b32_e32 v0, v1, v2, vcc_lo
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT:    v_lshrrev_b32_e32 v0, 16, v0
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
   %bool = icmp eq i32 %bool.arg1, %bool.arg2
   %y = select i1 %bool, bfloat 0xR3480, bfloat 0xR4C00
   %ldexp = fmul bfloat %x, %y
diff --git a/llvm/test/CodeGen/AMDGPU/dead_copy.mir b/llvm/test/CodeGen/AMDGPU/dead_copy.mir
index 2b54c61056a93..5bc42e9c47195 100644
--- a/llvm/test/CodeGen/AMDGPU/dead_copy.mir
+++ b/llvm/test/CodeGen/AMDGPU/dead_copy.mir
@@ -1,4 +1,5 @@
 # RUN: llc -o - %s -mtriple=amdgcn -mcpu=fiji  -run-pass=machine-cp -verify-machineinstrs | FileCheck -check-prefix=GCN %s
+# RUN: llc -o - %s -mtriple=amdgcn -mcpu=fiji  -passes=machine-cp | FileCheck -check-prefix=GCN %s
 
 # GCN-LABEL: dead_copy
 # GCN:       bb.0
diff --git a/llvm/test/CodeGen/AMDGPU/div-rem-by-constant-64.ll b/llvm/test/CodeGen/AMDGPU/div-rem-by-constant-64.ll
index 1c6808b613427..0c5b67580c352 100644
--- a/llvm/test/CodeGen/AMDGPU/div-rem-by-constant-64.ll
+++ b/llvm/test/CodeGen/AMDGPU/div-rem-by-constant-64.ll
@@ -15,16 +15,14 @@ define noundef i64 @srem64_3(i64 noundef %i)  {
 ; GFX9-NEXT:    v_mov_b32_e32 v3, 0
 ; GFX9-NEXT:    s_mov_b32 s7, 0x55555555
 ; GFX9-NEXT:    v_mad_u64_u32 v[4:5], s[4:5], v1, s6, v[2:3]
-; GFX9-NEXT:    v_mov_b32_e32 v6, v5
-; GFX9-NEXT:    v_mov_b32_e32 v5, v3
-; GFX9-NEXT:    v_mad_u64_u32 v[2:3], s[4:5], v0, s7, v[4:5]
+; GFX9-NEXT:    v_mov_b32_e32 v2, v4
+; GFX9-NEXT:    v_mad_u64_u32 v[2:3], s[4:5], v0, s7, v[2:3]
 ; GFX9-NEXT:    v_ashrrev_i32_e32 v4, 31, v1
-; GFX9-NEXT:    v_mul_lo_u32 v5, v4, s7
-; GFX9-NEXT:    v_mov_b32_e32 v2, v3
-; GFX9-NEXT:    v_add_co_u32_e32 v2, vcc, v6, v2
+; GFX9-NEXT:    v_mul_lo_u32 v6, v4, s6
+; GFX9-NEXT:    v_add_co_u32_e32 v2, vcc, v5, v3
 ; GFX9-NEXT:    v_addc_co_u32_e64 v3, s[4:5], 0, 0, vcc
 ; GFX9-NEXT:    v_mad_u64_u32 v[2:3], s[4:5], v1, s7, v[2:3]
-; GFX9-NEXT:    v_mul_lo_u32 v6, v4, s6
+; GFX9-NEXT:    v_mul_lo_u32 v5, v4, s7
 ; GFX9-NEXT:    v_mad_u64_u32 v[2:3], s[4:5], v4, s6, v[2:3]
 ; GFX9-NEXT:    v_add3_u32 v3, v6, v3, v5
 ; GFX9-NEXT:    v_lshrrev_b32_e32 v4, 31, v3
@@ -61,10 +59,9 @@ define noundef i64 @srem64_3(i64 noundef %i)  {
 ; GFX942-NEXT:    v_mad_u64_u32 v[4:5], s[0:1], v2, 3, 0
 ; GFX942-NEXT:    v_mov_b32_e32 v2, v5
 ; GFX942-NEXT:    v_mad_u64_u32 v[2:3], s[0:1], v3, 3, v[2:3]
-; GFX942-NEXT:    v_mov_b32_e32 v3, v2
 ; GFX942-NEXT:    v_sub_co_u32_e32 v0, vcc, v0, v4
 ; GFX942-NEXT:    s_nop 1
-; GFX942-NEXT:    v_subb_co_u32_e32 v1, vcc, v1, v3, vcc
+; GFX942-NEXT:    v_subb_co_u32_e32 v1, vcc, v1, v2, vcc
 ; GFX942-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX1030-LABEL: srem64_3:
@@ -73,14 +70,12 @@ define noundef i64 @srem64_3(i64 noundef %i)  {
 ; GFX1030-NEXT:    v_mul_hi_u32 v2, 0x55555556, v0
 ; GFX1030-NEXT:    v_mov_b32_e32 v3, 0
 ; GFX1030-NEXT:    v_mad_u64_u32 v[4:5], null, 0x55555556, v1, v[2:3]
-; GFX1030-NEXT:    v_mov_b32_e32 v6, v5
-; GFX1030-NEXT:    v_mov_b32_e32 v5, v3
-; GFX1030-NEXT:    v_mad_u64_u32 v[2:3], null, 0x55555555, v0, v[4:5]
+; GFX1030-NEXT:    v_mov_b32_e32 v2, v4
 ; GFX1030-NEXT:    v_ashrrev_i32_e32 v4, 31, v1
-; GFX1030-NEXT:    v_mul_lo_u32 v5, 0x55555555, v4
-; GFX1030-NEXT:    v_mov_b32_e32 v2, v3
-; GFX1030-NEXT:    v_add_co_u32 v2, s4, v6, v2
+; GFX1030-NEXT:    v_mad_u64_u32 v[2:3], null, 0x55555555, v0, v[2:3]
+; GFX1030-NEXT:    v_add_co_u32 v2, s4, v5, v3
 ; GFX1030-NEXT:    v_add_co_ci_u32_e64 v3, null, 0, 0, s4
+; GFX1030-NEXT:    v_mul_lo_u32 v5, 0x55555555, v4
 ; GFX1030-NEXT:    v_mad_u64_u32 v[2:3], null, 0x55555555, v1, v[2:3]
 ; GFX1030-NEXT:    v_mad_u64_u32 v[2:3], null, 0x55555556, v4, v[2:3]
 ; GFX1030-NEXT:    v_mul_lo_u32 v4, 0x55555556, v4
@@ -107,16 +102,14 @@ define noundef i64 @srem64_6(i64 noundef %i)  {
 ; GFX9-NEXT:    v_mov_b32_e32 v3, 0
 ; GFX9-NEXT:    s_mov_b32 s7, 0x55555555
 ; GFX9-NEXT:    v_mad_u64_u32 v[4:5], s[4:5], v1, s6, v[2:3]
-; GFX9-NEXT:    v_mov_b32_e32 v6, v5
-; GFX9-NEXT:    v_mov_b32_e32 v5, v3
-; GFX9-NEXT:    v_mad_u64_u32 v[2:3], s[4:5], v0, s7, v[4:5]
+; GFX9-NEXT:    v_mov_b32_e32 v2, v4
+; GFX9-NEXT:    v_mad_u64_u32 v[2:3], s[4:5], v0, s7, v[2:3]
 ; GFX9-NEXT:    v_ashrrev_i32_e32 v4, 31, v1
-; GFX9-NEXT:    v_mul_lo_u32 v5, v4, s7
-; GFX9-NEXT:    v_mov_b32_e32 v2, v3
-; GFX9-NEXT:    v_add_co_u32_e32 v2, vcc, v6, v2
+; GFX9-NEXT:    v_mul_lo_u32 v6, v4, s6
+; GFX9-NEXT:    v_add_co_u32_e32 v2, vcc, v5, v3
 ; GFX9-NEXT:    v_addc_co_u32_e64 v3, s[4:5], 0, 0, vcc
 ; GFX9-NEXT:    v_mad_u64_u32 v[2:3], s[4:5], v1, s7, v[2:3]
-; GFX9-NEXT:    v_mul_lo_u32 v6, v4, s6
+; GFX9-NEXT:    v_mul_lo_u32 v5, v4, s7
 ; GFX9-NEXT:    v_mad_u64_u32 v[2:3], s[4:5], v4, s6, v[2:3]
 ; GFX9-NEXT:    v_add3_u32 v3, v6, v3, v5
 ; GFX9-NEXT:    v_lshrrev_b32_e32 v4, 31, v3
@@ -153,10 +146,9 @@ define noundef i64 @srem64_6(i64 noundef %i)  {
 ; GFX942-NEXT:    v_mad_u64_u32 v[4:5], s[0:1], v2, 3, 0
 ; GFX942-NEXT:    v_mov_b32_e32 v2, v5
 ; GFX942-NEXT:    v_mad_u64_u32 v[2:3], s[0:1], v3, 3, v[2:3]
-; GFX942-NEXT:    v_mov_b32_e32 v3, v2
 ; GFX942-NEXT:    v_sub_co_u32_e32 v0, vcc, v0, v4
 ; GFX942-NEXT:    s_nop 1
-; GFX942-NEXT:    v_subb_co_u32_e32 v1, vcc, v1, v3, vcc
+; GFX942-NEXT:    v_subb_co_u32_e32 v1, vcc, v1, v2, vcc
 ; GFX942-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX1030-LABEL: srem64_6:
@@ -165,14 +157,12 @@ define noundef i64 @srem64_6(i64 noundef %i)  {
 ; GFX1030-NEXT:    v_mul_hi_u32 v2, 0x55555556, v0
 ; GFX1030-NEXT:    v_mov_b32_e32 v3, 0
 ; GFX1030-NEXT:    v_mad_u64_u32 v[4:5], null, 0x55555556, v1, v[2:3]
-; GFX1030-NEXT:    v_mov_b32_e32 v6, v5
-; GFX1030-NEXT:    v_mov_b32_e32 v5, v3
-; GFX1030-NEXT:    v_mad_u64_u32 v[2:3], null, 0x55555555, v0, v[4:5]
+; GFX1030-NEXT:    v_mov_b32_e32 v2, v4
 ; GFX1030-NEXT:    v_ashrrev_i32_e32 v4, 31, v1
-; GFX1030-NEXT:    v_mul_lo_u32 v5, 0x55555555, v4
-; GFX1030-NEXT:    v_mov_b32_e32 v2, v3
-; GFX1030-NEXT:    v_add_co_u32 v2, s4, v6, v2
+; GFX1030-NEXT:    v_mad_u64_u32 v[2:3], null, 0x55555555, v0, v[2:3]
+; GFX1030-NEXT:    v_add_co_u32 v2, s4, v5, v3
 ; GFX1030-NEXT:    v_add_co_ci_u32_e64 v3, null, 0, 0, s4
+; GFX1030-NEXT:    v_mul_lo_u32 v5, 0x55555555, v4
 ; GFX1030-NEXT:    v_mad_u64_u32 v[2:3], null, 0x55555555, v1, v[2:3]
 ; GFX1030-NEXT:    v_mad_u64_u32 v[2:3], null, 0x55555556, v4, v[2:3]
 ; GFX1030-NEXT:    v_mul_lo_u32 v4, 0x55555556, v4
@@ -199,11 +189,9 @@ define noundef i64 @urem64_3(i64 noundef %i)  {
 ; GFX9-NEXT:    v_mov_b32_e32 v3, 0
 ; GFX9-NEXT:    s_mov_b32 s6, 0xaaaaaaaa
 ; GFX9-NEXT:    v_mad_u64_u32 v[4:5], s[4:5], v1, s4, v[2:3]
-; GFX9-NEXT:    v_mov_b32_e32 v6, v5
-; GFX9-NEXT:    v_mov_b32_e32 v5, v3
-; GFX9-NEXT:    v_mad_u64_u32 v[2:3], s[4:5], v0, s6, v[4:5]
-; GFX9-NEXT:    v_mov_b32_e32 v2, v3
-; GFX9-NEXT:    v_add_co_u32_e32 v2, vcc, v6, v2
+; GFX9-NEXT:    v_mov_b32_e32 v2, v4
+; GFX9-NEXT:    v_mad_u64_u32 v[2:3], s[4:5], v0, s6, v[2:3]
+; GFX9-NEXT:    v_add_co_u32_e32 v2, vcc, v5, v3
 ; GFX9-NEXT:    v_addc_co_u32_e64 v3, s[4:5], 0, 0, vcc
 ; GFX9-NEXT:    v_mad_u64_u32 v[2:3], s[4:5], v1, s6, v[2:3]
 ; GFX9-NEXT:    v_alignbit_b32 v2, v3, v2, 1
@@ -235,10 +223,9 @@ define noundef i64 @urem64_3(i64 noundef %i)  {
 ; GFX942-NEXT:    v_mov_b32_e32 v2, v5
 ; GFX942-NEXT:    v_lshrrev_b32_e32 v3, 1, v3
 ; GFX942-NEXT:    v_mad_u64_u32 v[2:3], s[0:1], v3, 3, v[2:3]
-; GFX942-NEXT:    v_mov_b32_e32 v3, v2
 ; GFX942-NEXT:    v_sub_co_u32_e32 v0, vcc, v0, v4
 ; GFX942-NEXT:    s_nop 1
-; GFX942-NEXT:    v_subb_co_u32_e32 v1, vcc, v1, v3, vcc
+; GFX942-NEXT:    v_subb_co_u32_e32 v1, vcc, v1, v2, vcc
 ; GFX942-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX1030-LABEL: urem64_3:
@@ -247,11 +234,9 @@ define noundef i64 @urem64_3(i64 noundef %i)  {
 ; GFX1030-NEXT:    v_mul_hi_u32 v2, 0xaaaaaaab, v0
 ; GFX1030-NEXT:    v_mov_b32_e32 v3, 0
 ; GFX1030-NEXT:    v_mad_u64_u32 v[4:5], null, 0xaaaaaaab, v1, v[2:3]
-; GFX1030-NEXT:    v_mov_b32_e32 v6, v5
-; GFX1030-NEXT:    v_mov_b32_e32 v5, v3
-; GFX1030-NEXT:    v_mad_u64_u32 v[2:3], null, 0xaaaaaaaa, v0, v[4:5]
-; GFX1030-NEXT:    v_mov_b32_e32 v2, v3
-; GFX1030-NEXT:    v_add_co_u32 v2, s4, v6, v2
+; GFX1030-NEXT:    v_mov_b32_e32 v2, v4
+; GFX1030-NEXT:    v_mad_u64_u32 v[2:3], null, 0xaaaaaaaa, v0, v[2:3]
+; GFX1030-NEXT:    v_add_co_u32 v2, s4, v5, v3
 ; GFX1030-NEXT:    v_add_co_ci_u32_e64 v3, null, 0, 0, s4
 ; GFX1030-NEXT:    v_mad_u64_u32 v[2:3], null, 0xaaaaaaaa, v1, v[2:3]
 ; GFX1030-NEXT:    v_alignbit_b32 v2, v3, v2, 1
@@ -276,11 +261,9 @@ define noundef i64 @urem64_6(i64 noundef %i)  {
 ; GFX9-NEXT:    v_mov_b32_e32 v3, 0
 ; GFX9-NEXT:    s_mov_b32 s6, 0xaaaaaaaa
 ; GFX9-NEXT:    v_mad_u64_u32 v[4:5], s[4:5], v1, s4, v[2:3]
-; GFX9-NEXT:    v_mov_b32_e32 v6, v5
-; GFX9-NEXT:    v_mov_b32_e32 v5, v3
-; GFX9-NEXT:    v_mad_u64_u32 v[2:3], s[4:5], v0, s6, v[4:5]
-; GFX9-NEXT:    v_mov_b32_e32 v2, v3
-; GFX9-NEXT:    v_add_co_u32_e32 v2, vcc, v6, v2
+; GFX9-NEXT:    v_mov_b32_e32 v2, v4
+; GFX9-NEXT:    v_mad_u64_u32 v[2:3], s[4:5], v0, s6, v[2:3]
+; GFX9-NEXT:    v_add_co_u32_e32 v2, vcc, v5, v3
 ; GFX9-NEXT:    v_addc_co_u32_e64 v3, s[4:5], 0, 0, vcc
 ; GFX9-NEXT:    v_mad_u64_u32 v[2:3], s[4:5], v1, s6, v[2:3]
 ; GFX9-NEXT:    v_alignbit_b32 v2, v3, v2, 2
@@ -312,10 +295,9 @@ define noundef i64 @urem64_6(i64 noundef %i)  {
 ; GFX942-NEXT:    v_mov_b32_e32 v2, v5
 ; GFX942-NEXT:    v_lshrrev_b32_e32 v3, 2, v3
 ; GFX942-NEXT:    v_mad_u64_u32 v[2:3], s[0:1], v3, 6, v[2:3]
-; GFX942-NEXT:    v_mov_b32_e32 v3, v2
 ; GFX942-NEXT:    v_sub_co_u32_e32 v0, vcc, v0, v4
 ; GFX942-NEXT:    s_nop 1
-; GFX942-NEXT:    v_subb_co_u32_e32 v1, vcc, v1, v3, vcc
+; GFX942-NEXT:    v_subb_co_u32_e32 v1, vcc, v1, v2, vcc
 ; GFX942-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX1030-LABEL: urem64_6:
@@ -324,11 +306,9 @@ define noundef i64 @urem64_6(i64 noundef %i)  {
 ; GFX1030-NEXT:    v_mul_hi_u32 v2, 0xaaaaaaab, v0
 ; GFX1030-NEXT:    v_mov_b32_e32 v3, 0
 ; GFX1030-NEXT:    v_mad_u64_u32 v[4:5], null, 0xaaaaaaab, v1, v[2:3]
-; GFX1030-NEXT:    v_mov_b32_e32 v6, v5
-; GFX1030-NEXT:    v_mov_b32_e32 v5, v3
-; GFX1030-NEXT:    v_mad_u64_u32 v[2:3], null, 0xaaaaaaaa, v0, v[4:5]
-; GFX1030-NEXT:    v_mov_b32_e32 v2, v3
-; GFX1030-NEXT:    v_add_co_u32 v2, s4, v6, v2
+; GFX1030-NEXT:    v_mov_b32_e32 v2, v4
+; GFX1030-NEXT:    v_mad_u64_u32 v[2:3], null, 0xaaaaaaaa, v0, v[2:3]
+; GFX1030-NEXT:    v_add_co_u32 v2, s4, v5, v3
 ; GFX1030-NEXT:    v_add_co_ci_u32_e64 v3, null, 0, 0, s4
 ; GFX1030-NEXT:    v_mad_u64_u32 v[2:3], null, 0xaaaaaaaa, v1, v[2:3]
 ; GFX1030-NEXT:    v_alignbit_b32 v2, v3, v2, 2
@@ -353,15 +333,13 @@ define noundef i64 @sdiv64_3(i64 noundef %i)  {
 ; GFX9-NEXT:    v_mov_b32_e32 v3, 0
 ; GFX9-NEXT:    s_mov_b32 s7, 0x55555555
 ; GFX9-NEXT:    v_mad_u64_u32 v[4:5], s[4:5], v1, s6, v[2:3]
-; GFX9-NEXT:    v_mov_b32_e32 v6, v5
-; GFX9-NEXT:    v_mov_b32_e32 v5, v3
-; GFX9-NEXT:    v_mad_u64_u32 v[2:3], s[4:5], v0, s7, v[4:5]
-; GFX9-NEXT:    v_mov_b32_e32 v0, v3
-; GFX9-NEXT:    v_add_co_u32_e32 v2, vcc, v6, v0
-; GFX9-NEXT:    v_addc_co_u32_e64 v3, s[4:5], 0, 0, vcc
-; GFX9-NEXT:    v_mad_u64_u32 v[2:3], s[4:5], v1, s7, v[2:3]
+; GFX9-NEXT:    v_mov_b32_e32 v2, v4
+; GFX9-NEXT:    v_mad_u64_u32 v[2:3], s[4:5], v0, s7, v[2:3]
 ; GFX9-NEXT:    v_ashrrev_i32_e32 v0, 31, v1
 ; GFX9-NEXT:    v_mul_lo_u32 v4, v0, s7
+; GFX9-NEXT:    v_add_co_u32_e32 v2, vcc, v5, v3
+; GFX9-NEXT:    v_addc_co_u32_e64 v3, s[4:5], 0, 0, vcc
+; GFX9-NEXT:    v_mad_u64_u32 v[2:3], s[4:5], v1, s7, v[2:3]
 ; GFX9-NEXT:    v_mul_lo_u32 v5, v0, s6
 ; GFX9-NEXT:    v_mad_u64_u32 v[0:1], s[4:5], v0, s6, v[2:3]
 ; GFX9-NEXT:    v_add3_u32 v1, v5, v1, v4
@@ -400,16 +378,14 @@ define noundef i64 @sdiv64_3(i64 noundef %i)  {
 ; GFX1030-NEXT:    v_mul_hi_u32 v2, 0x55555556, v0
 ; GFX1030-NEXT:    v_mov_b32_e32 v3, 0
 ; GFX1030-NEXT:    v_mad_u64_u32 v[4:5], null, 0x55555556, v1, v[2:3]
-; GFX1030-NEXT:    v_mov_b32_e32 v6, v5
-; GFX1030-NEXT:    v_mov_b32_e32 v5, v3
-; GFX1030-NEXT:    v_mad_u64_u32 v[2:3], null, 0x55555555, v0, v[4:5]
-; GFX1030-NEXT:    v_mov_b32_e32 v0, v3
-; GFX1030-NEXT:    v_add_co_u32 v2, s4, v6, v0
-; GFX1030-NEXT:    v_add_co_ci_u32_e64 v3, null, 0, 0, s4
+; GFX1030-NEXT:    v_mov_b32_e32 v2, v4
+; GFX1030-NEXT:    v_mad_u64_u32 v[2:3], null, 0x55555555, v0, v[2:3]
 ; GFX1030-NEXT:    v_ashrrev_i32_e32 v0, 31, v1
-; GFX1030-NEXT:    v_mad_u64_u32 v[2:3], null, 0x55555555, v1, v[2:3]
 ; GFX1030-NEXT:    v_mul_lo_u32 v4, 0x55555555, v0
+; GFX1030-NEXT:    v_add_co_u32 v2, s4, v5, v3
+; GFX1030-NEXT:    v_add_co_ci_u32_e64 v3, null, 0, 0, s4
 ; GFX1030-NEXT:    v_mul_lo_u32 v5, 0x55555556, v0
+; GFX1030-NEXT:    v_mad_u64_u32 v[2:3], null, 0x55555555, v1, v[2:3]
 ; GFX1030-NEXT:    v_mad_u64_u32 v[0:1], null, 0x55555556, v0, v[2:3]
 ; GFX1030-NEXT:    v_add3_u32 v1, v5, v1, v4
 ; GFX1030-NEXT:    v_lshrrev_b32_e32 v2, 31, v1
@@ -430,15 +406,13 @@ define noundef i64 @sdiv64_6(i64 noundef %i)  {
 ; GFX9-NEXT:    v_mov_b32_e32 v3, 0
 ; GFX9-NEXT:    s_mov_b32 s7, 0x55555555
 ; GFX9-NEXT:    v_mad_u64_u32 v[4:5], s[4:5], v1, s6, v[2:3]
-; GFX9-NEXT:    v_mov_b32_e32 v6, v5
-; GFX9-NEXT:    v_mov_b32_e32 v5, v3
-; GFX9-NEXT:    v_mad_u64_u32 v[2:3], s[4:5], v0, s7, v[4:5]
-; GFX9-NEXT:    v_mov_b32_e32 v0, v3
-; GFX9-NEXT:    v_add_co_u32_e32 v2, vcc, v6, v0
-; GFX9-NEXT:    v_addc_co_u32_e64 v3, s[4:5], 0, 0, vcc
-; GFX9-NEXT:    v_mad_u64_u32 v[2:3], s[4:5], v1, s7, v[2:3]
+; GFX9-NEXT:    v_mov_b32_e32 v2, v4
+; GFX9-NEXT:    v_mad_u64_u32 v[2:3], s[4:5], v0, s7, v[2:3]
 ; GFX9-NEXT:    v_ashrrev_i32_e32 v0, 31, v1
 ; GFX9-NEXT:    v_mul_lo_u32 v4, v0, s7
+; GFX9-NEXT:    v_add_co_u32_e32 v2, vcc, v5, v3
+; GFX9-NEXT:    v_addc_co_u32_e64 v3, s[4:5], 0, 0, vcc
+; GFX9-NEXT:    v_mad_u64_u32 v[2:3], s[4:5], v1, s7, v[2:3]
 ; GFX9-NEXT:    v_mul_lo_u32 v5, v0, s6
 ; GFX9-NEXT:    v_mad_u64_u32 v[0:1], s[4:5], v0, s6, v[2:3]
 ; GFX9-NEXT:    v_add3_u32 v1, v5, v1, v4
@@ -477,16 +451,14 @@ define noundef i64 @sdiv64_6(i64 noundef %i)  {
 ; GFX1030-NEXT:    v_mul_hi_u32 v2, 0x55555556, v0
 ; GFX1030-NEXT:    v_mov_b32_e32 v3, 0
 ; GFX1030-NEXT:    v_mad_u64_u32 v[4:5], null, 0x55555556, v1, v[2:3]
-; GFX1030-NEXT:    v_mov_b32_e32 v6, v5
-; GFX1030-NEXT:    v_mov_b32_e32 v5, v3
-; GFX1030-NEXT:    v_mad_u64_u32 v[2:3], null, 0x55555555, v0, v[4:5]
-; GFX1030-NEXT:    v_mov_b32_e32 v0, v3
-; GFX1030-NEXT:    v_add_co_u32 v2, s4, v6, v0
-; GFX1030-NEXT:    v_add_co_ci_u32_e64 v3, null, 0, 0, s4
+; GFX1030-NEXT:    v_mov_b32_e32 v2, v4
+; GFX1030-NEXT:    v_mad_u64_u32 v[2:3], null, 0x55555555, v0, v[2:3]
 ; GFX1030-NEXT:    v_ashrrev_i32_e32 v0, 31, v1
-; GFX1030-NEXT:    v_mad_u64_u32 v[2:3], null, 0x55555555, v1, v[2:3]
 ; GFX1030-NEXT:    v_mul_lo_u32 v4, 0x55555555, v0
+; GFX1030-NEXT:    v_add_co_u32 v2, s4, v5, v3
+; GFX1030-NEXT:    v_add_co_ci_u32_e64 v3, null, 0, 0, s4
 ; GFX1030-NEXT:    v_mul_lo_u32 v5, 0x55555556, v0
+; GFX1030-NEXT:    v_mad_u64_u32 v[2:3], null, 0x55555555, v1, v[2:3]
 ; GFX1030-NEXT:    v_mad_u64_u32 v[0:1], null, 0x55555556, v0, v[2:3]
 ; GFX1030-NEXT:    v_add3_u32 v1, v5, v1, v4
 ; GFX1030-NEXT:    v_lshrrev_b32_e32 v2, 31, v1
@@ -507,11 +479,9 @@ define noundef i64 @udiv64_3(i64 noundef %i)  {
 ; GFX9-NEXT:    v_mov_b32_e32 v3, 0
 ; GFX9-NEXT:    s_mov_b32 s6, 0xaaaaaaaa
 ; GFX9-NEXT:    v_mad_u64_u32 v[4:5], s[4:5], v1, s4, v[2:3]
-; GFX9-NEXT:    v_mov_b32_e32 v6, v5
-; GFX9-NEXT:    v_mov_b32_e32 v5, v3
-; GFX9-NEXT:    v_mad_u64_u32 v[2:3], s[4:5], v0, s6, v[4:5]
-; GFX9-NEXT:    v_mov_b32_e32 v0, v3
-; GFX9-NEXT:    v_add_co_u32_e32 v2, vcc, v6, v0
+; GFX9-NEXT:    v_mov_b32_e32 v2, v4
+; GFX9-NEXT:    v_mad_u64_u32 v[2:3], s[4:5], v0, s6, v[2:3]
+; GFX9-NEXT:    v_add_co_u32_e32 v2, vcc, v5, v3
 ; GFX9-NEXT:    v_addc_co_u32_e64 v3, s[4:5], 0, 0, vcc
 ; GFX9-NEXT:    v_mad_u64_u32 v[0:1], s[4:5], v1, s6, v[2:3]
 ; GFX9-NEXT:    v_alignbit_b32 v0, v1, v0, 1
@@ -543,11 +513,9 @@ define noundef i64 @udiv64_3(i64 noundef %i)  {
 ; GFX1030-NEXT:    v_mul_hi_u32 v2, 0xaaaaaaab, v0
 ; GFX1030-NEXT:    v_mov_b32_e32 v3, 0
 ; GFX1030-NEXT:    v_mad_u64_u32 v[4:5], null, 0xaaaaaaab, v1, v[2:3]
-; GFX1030-NEXT:    v_mov_b32_e32 v6, v5
-; GFX1030-NEXT:    v_mov_b32_e32 v5, v3
-; GFX1030-NEXT:    v_mad_u64_u32 v[2:3], null, 0xaaaaaaaa, v0, v[4:5]
-; GFX1030-NEXT:    v_mov_b32_e32 v0, v3
-; GFX1030-NEXT:    v_add_co_u32 v2, s4, v6, v0
+; GFX1030-NEXT:    v_mov_b32_e32 v2, v4
+; GFX1030-NEXT:    v_mad_u64_u32 v[2:3], null, 0xaaaaaaaa, v0, v[2:3]
+; GFX1030-NEXT:    v_add_co_u32 v2, s4, v5, v3
 ; GFX1030-NEXT:    v_add_co_ci_u32_e64 v3, null, 0, 0, s4
 ; GFX1030-NEXT:    v_mad_u64_u32 v[0:1], null, 0xaaaaaaaa, v1, v[2:3]
 ; GFX1030-NEXT:    v_alignbit_b32 v0, v1, v0, 1
@@ -567,11 +535,9 @@ define noundef i64 @udiv64_6(i64 noundef %i)  {
 ; GFX9-NEXT:    v_mov_b32_e32 v3, 0
 ; GFX9-NEXT:    s_mov_b32 s6, 0xaaaaaaaa
 ; GFX9-NEXT:    v_mad_u64_u32 v[4:5], s[4:5], v1, s4, v[2:3]
-; GFX9-NEXT:    v_mov_b32_e32 v6, v5
-; GFX9-NEXT:    v_mov_b32_e32 v5, v3
-; GFX9-NEXT:    v_mad_u64_u32 v[2:3], s[4:5], v0, s6, v[4:5]
-; GFX9-NEXT:    v_mov_b32_e32 v0, v3
-; GFX9-NEXT:    v_add_co_u32_e32 v2, vcc, v6, v0
+; GFX9-NEXT:    v_mov_b32_e32 v2, v4
+; GFX9-NEXT:    v_mad_u64_u32 v[2:3], s[4:5], v0, s6, v[2:3]
+; GFX9-NEXT:    v_add_co_u32_e32 v2, vcc, v5, v3
 ; GFX9-NEXT:    v_addc_co_u32_e64 v3, s[4:5], 0, 0, vcc
 ; GFX9-NEXT:    v_mad_u64_u32 v[0:1], s[4:5], v1, s6, v[2:3]
 ; GFX9-NEXT:    v_alignbit_b32 v0, v1, v0, 2
@@ -603,11 +569,9 @@ define noundef i64 @udiv64_6(i64 noundef %i)  {
 ; GFX1030-NEXT:    v_mul_hi_u32 v2, 0xaaaaaaab, v0
 ; GFX1030-NEXT:    v_mov_b32_e32 v3, 0
 ; GFX1030-NEXT:    v_mad_u64_u32 v[4:5], null, 0xaaaaaaab, v1, v[2:3]
-; GFX1030-NEXT:    v_mov_b32_e32 v6, v5
-; GFX1030-NEXT:    v_mov_b32_e32 v5, v3
-; GFX1030-NEXT:    v_mad_u64_u32 v[2:3], null, 0xaaaaaaaa, v0, v[4:5]
-; GFX1030-NEXT:    v_mov_b32_e32 v0, v3
-; GFX1030-NEXT:    v_add_co_u32 v2, s4, v6, v0
+; GFX1030-NEXT:    v_mov_b32_e32 v2, v4
+; GFX1030-NEXT:    v_mad_u64_u32 v[2:3], null, 0xaaaaaaaa, v0, v[2:3]
+; GFX1030-NEXT:    v_add_co_u32 v2, s4, v5, v3
 ; GFX1030-NEXT:    v_add_co_ci_u32_e64 v3, null, 0, 0, s4
 ; GFX1030-NEXT:    v_mad_u64_u32 v[0:1], null, 0xaaaaaaaa, v1, v[2:3]
 ; GFX1030-NEXT:    v_alignbit_b32 v0, v1, v0, 2
@@ -1005,8 +969,7 @@ define noundef i64 @udiv64_i32min(i64 noundef %i)  {
 ; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 31, v1
 ; GFX9-NEXT:    v_mov_b32_e32 v1, 0
 ; GFX9-NEXT:    v_mad_u64_u32 v[1:2], s[4:5], v2, 1, v[0:1]
-; GFX9-NEXT:    v_mov_b32_e32 v1, v2
-; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, v0, v1
+; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, v0, v2
 ; GFX9-NEXT:    v_addc_co_u32_e64 v1, s[4:5], 0, 0, vcc
 ; GFX9-NEXT:    v_alignbit_b32 v0, v1, v0, 1
 ; GFX9-NEXT:    v_mov_b32_e32 v1, 0
@@ -1033,8 +996,7 @@ define noundef i64 @udiv64_i32min(i64 noundef %i)  {
 ; GFX1030-NEXT:    v_alignbit_b32 v0, v1, v0, 31
 ; GFX1030-NEXT:    v_lshrrev_b32_e32 v1, 31, v1
 ; GFX1030-NEXT:    v_mad_u64_u32 v[2:3], null, v0, 1, v[1:2]
-; GFX1030-NEXT:    v_mov_b32_e32 v0, v3
-; GFX1030-NEXT:    v_add_co_u32 v0, s4, v1, v0
+; GFX1030-NEXT:    v_add_co_u32 v0, s4, v1, v3
 ; GFX1030-NEXT:    v_add_co_ci_u32_e64 v1, null, 0, 0, s4
 ; GFX1030-NEXT:    v_alignbit_b32 v0, v1, v0, 1
 ; GFX1030-NEXT:    v_mov_b32_e32 v1, 0
@@ -1049,19 +1011,17 @@ define noundef i64 @srem64_i32max(i64 noundef %i)  {
 ; GFX9:       ; %bb.0: ; %entry
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX9-NEXT:    v_mul_hi_u32 v2, v0, 3
+; GFX9-NEXT:    v_ashrrev_i32_e32 v8, 31, v1
 ; GFX9-NEXT:    v_mov_b32_e32 v3, 0
+; GFX9-NEXT:    v_mad_u64_u32 v[4:5], s[4:5], v8, 3, 0
+; GFX9-NEXT:    v_mad_u64_u32 v[6:7], s[4:5], v1, 3, v[2:3]
+; GFX9-NEXT:    v_lshl_add_u32 v2, v8, 31, v8
+; GFX9-NEXT:    v_add3_u32 v5, v5, v2, v4
+; GFX9-NEXT:    v_mov_b32_e32 v2, v6
 ; GFX9-NEXT:    s_mov_b32 s6, 0x80000001
-; GFX9-NEXT:    v_ashrrev_i32_e32 v6, 31, v1
-; GFX9-NEXT:    v_mad_u64_u32 v[4:5], s[4:5], v1, 3, v[2:3]
-; GFX9-NEXT:    v_lshl_add_u32 v8, v6, 31, v6
-; GFX9-NEXT:    v_mad_u64_u32 v[6:7], s[4:5], v6, 3, 0
-; GFX9-NEXT:    v_mov_b32_e32 v9, v5
-; GFX9-NEXT:    v_mov_b32_e32 v5, v3
-; GFX9-NEXT:    v_mad_u64_u32 v[2:3], s[4:5], v0, s6, v[4:5]
-; GFX9-NEXT:    v_add3_u32 v7, v7, v8, v6
-; GFX9-NEXT:    v_mad_u64_u32 v[4:5], s[4:5], v0, -1, v[6:7]
-; GFX9-NEXT:    v_mov_b32_e32 v2, v3
-; GFX9-NEXT:    v_add_co_u32_e32 v2, vcc, v9, v2
+; GFX9-NEXT:    v_mad_u64_u32 v[2:3], s[4:5], v0, s6, v[2:3]
+; GFX9-NEXT:    v_mad_u64_u32 v[4:5], s[4:5], v0, -1, v[4:5]
+; GFX9-NEXT:    v_add_co_u32_e32 v2, vcc, v7, v3
 ; GFX9-NEXT:    v_addc_co_u32_e64 v3, s[4:5], 0, 0, vcc
 ; GFX9-NEXT:    v_mad_u64_u32 v[2:3], s[4:5], v1, s6, v[2:3]
 ; GFX9-NEXT:    v_sub_u32_e32 v5, v5, v1
@@ -1112,10 +1072,9 @@ define noundef i64 @srem64_i32max(i64 noundef %i)  {
 ; GFX942-NEXT:    v_mad_u64_u32 v[4:5], s[0:1], v2, s2, 0
 ; GFX942-NEXT:    v_mov_b32_e32 v2, v5
 ; GFX942-NEXT:    v_mad_u64_u32 v[2:3], s[0:1], v3, s2, v[2:3]
-; GFX942-NEXT:    v_mov_b32_e32 v3, v2
 ; GFX942-NEXT:    v_sub_co_u32_e32 v0, vcc, v0, v4
 ; GFX942-NEXT:    s_nop 1
-; GFX942-NEXT:    v_subb_co_u32_e32 v1, vcc, v1, v3, vcc
+; GFX942-NEXT:    v_subb_co_u32_e32 v1, vcc, v1, v2, vcc
 ; GFX942-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX1030-LABEL: srem64_i32max:
@@ -1123,23 +1082,21 @@ define noundef i64 @srem64_i32max(i64 noundef %i)  {
 ; GFX1030-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX1030-NEXT:    v_mul_hi_u32 v2, v0, 3
 ; GFX1030-NEXT:    v_mov_b32_e32 v3, 0
-; GFX1030-NEXT:    v_ashrrev_i32_e32 v6, 31, v1
+; GFX1030-NEXT:    v_ashrrev_i32_e32 v8, 31, v1
+; GFX1030-NEXT:    v_mad_u64_u32 v[6:7], null, v8, 3, 0
 ; GFX1030-NEXT:    v_mad_u64_u32 v[4:5], null, v1, 3, v[2:3]
-; GFX1030-NEXT:    v_mov_b32_e32 v7, v5
-; GFX1030-NEXT:    v_mov_b32_e32 v5, v3
-; GFX1030-NEXT:    v_mad_u64_u32 v[2:3], null, v6, 3, 0
-; GFX1030-NEXT:    v_lshl_add_u32 v6, v6, 31, v6
-; GFX1030-NEXT:    v_mad_u64_u32 v[4:5], null, 0x80000001, v0, v[4:5]
-; GFX1030-NEXT:    v_add3_u32 v3, v3, v6, v2
-; GFX1030-NEXT:    v_mov_b32_e32 v4, v5
-; GFX1030-NEXT:    v_mad_u64_u32 v[2:3], null, v0, -1, v[2:3]
-; GFX1030-NEXT:    v_add_co_u32 v4, s4, v7, v4
-; GFX1030-NEXT:    v_add_co_ci_u32_e64 v5, null, 0, 0, s4
-; GFX1030-NEXT:    v_sub_nc_u32_e32 v6, v3, v1
-; GFX1030-NEXT:    v_mad_u64_u32 v[3:4], null, 0x80000001, v1, v[4:5]
-; GFX1030-NEXT:    v_sub_nc_u32_e32 v5, v6, v0
-; GFX1030-NEXT:    v_add_co_u32 v2, vcc_lo, v3, v2
-; GFX1030-NEXT:    v_add_co_ci_u32_e32 v3, vcc_lo, v4, v5, vcc_lo
+; GFX1030-NEXT:    v_lshl_add_u32 v8, v8, 31, v8
+; GFX1030-NEXT:    v_add3_u32 v7, v7, v8, v6
+; GFX1030-NEXT:    v_mov_b32_e32 v2, v4
+; GFX1030-NEXT:    v_mad_u64_u32 v[6:7], null, v0, -1, v[6:7]
+; GFX1030-NEXT:    v_mad_u64_u32 v[2:3], null, 0x80000001, v0, v[2:3]
+; GFX1030-NEXT:    v_sub_nc_u32_e32 v4, v7, v1
+; GFX1030-NEXT:    v_add_co_u32 v2, s4, v5, v3
+; GFX1030-NEXT:    v_add_co_ci_u32_e64 v3, null, 0, 0, s4
+; GFX1030-NEXT:    v_sub_nc_u32_e32 v4, v4, v0
+; GFX1030-NEXT:    v_mad_u64_u32 v[2:3], null, 0x80000001, v1, v[2:3]
+; GFX1030-NEXT:    v_add_co_u32 v2, vcc_lo, v2, v6
+; GFX1030-NEXT:    v_add_co_ci_u32_e32 v3, vcc_lo, v3, v4, vcc_lo
 ; GFX1030-NEXT:    v_mad_u64_u32 v[2:3], null, v0, 1, v[2:3]
 ; GFX1030-NEXT:    v_add_nc_u32_e32 v3, v1, v3
 ; GFX1030-NEXT:    v_ashrrev_i64 v[4:5], 30, v[2:3]
@@ -1161,19 +1118,17 @@ define noundef i64 @sdiv64_i32max(i64 noundef %i)  {
 ; GFX9:       ; %bb.0: ; %entry
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX9-NEXT:    v_mul_hi_u32 v2, v0, 3
+; GFX9-NEXT:    v_ashrrev_i32_e32 v8, 31, v1
 ; GFX9-NEXT:    v_mov_b32_e32 v3, 0
+; GFX9-NEXT:    v_mad_u64_u32 v[4:5], s[4:5], v8, 3, 0
+; GFX9-NEXT:    v_mad_u64_u32 v[6:7], s[4:5], v1, 3, v[2:3]
+; GFX9-NEXT:    v_lshl_add_u32 v2, v8, 31, v8
+; GFX9-NEXT:    v_add3_u32 v5, v5, v2, v4
+; GFX9-NEXT:    v_mov_b32_e32 v2, v6
 ; GFX9-NEXT:    s_mov_b32 s6, 0x80000001
-; GFX9-NEXT:    v_ashrrev_i32_e32 v6, 31, v1
-; GFX9-NEXT:    v_mad_u64_u32 v[4:5], s[4:5], v1, 3, v[2:3]
-; GFX9-NEXT:    v_lshl_add_u32 v8, v6, 31, v6
-; GFX9-NEXT:    v_mad_u64_u32 v[6:7], s[4:5], v6, 3, 0
-; GFX9-NEXT:    v_mov_b32_e32 v9, v5
-; GFX9-NEXT:    v_mov_b32_e32 v5, v3
-; GFX9-NEXT:    v_mad_u64_u32 v[2:3], s[4:5], v0, s6, v[4:5]
-; GFX9-NEXT:    v_add3_u32 v7, v7, v8, v6
-; GFX9-NEXT:    v_mad_u64_u32 v[4:5], s[4:5], v0, -1, v[6:7]
-; GFX9-NEXT:    v_mov_b32_e32 v2, v3
-; GFX9-NEXT:    v_add_co_u32_e32 v2, vcc, v9, v2
+; GFX9-NEXT:    v_mad_u64_u32 v[2:3], s[4:5], v0, s6, v[2:3]
+; GFX9-NEXT:    v_mad_u64_u32 v[4:5], s[4:5], v0, -1, v[4:5]
+; GFX9-NEXT:    v_add_co_u32_e32 v2, vcc, v7, v3
 ; GFX9-NEXT:    v_addc_co_u32_e64 v3, s[4:5], 0, 0, vcc
 ; GFX9-NEXT:    v_mad_u64_u32 v[2:3], s[4:5], v1, s6, v[2:3]
 ; GFX9-NEXT:    v_sub_u32_e32 v5, v5, v1
@@ -1222,23 +1177,21 @@ define noundef i64 @sdiv64_i32max(i64 noundef %i)  {
 ; GFX1030-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX1030-NEXT:    v_mul_hi_u32 v2, v0, 3
 ; GFX1030-NEXT:    v_mov_b32_e32 v3, 0
-; GFX1030-NEXT:    v_ashrrev_i32_e32 v6, 31, v1
+; GFX1030-NEXT:    v_ashrrev_i32_e32 v8, 31, v1
+; GFX1030-NEXT:    v_mad_u64_u32 v[6:7], null, v8, 3, 0
 ; GFX1030-NEXT:    v_mad_u64_u32 v[4:5], null, v1, 3, v[2:3]
-; GFX1030-NEXT:    v_mov_b32_e32 v7, v5
-; GFX1030-NEXT:    v_mov_b32_e32 v5, v3
-; GFX1030-NEXT:    v_mad_u64_u32 v[2:3], null, v6, 3, 0
-; GFX1030-NEXT:    v_lshl_add_u32 v6, v6, 31, v6
-; GFX1030-NEXT:    v_mad_u64_u32 v[4:5], null, 0x80000001, v0, v[4:5]
-; GFX1030-NEXT:    v_add3_u32 v3, v3, v6, v2
-; GFX1030-NEXT:    v_mov_b32_e32 v4, v5
-; GFX1030-NEXT:    v_mad_u64_u32 v[2:3], null, v0, -1, v[2:3]
-; GFX1030-NEXT:    v_add_co_u32 v4, s4, v7, v4
-; GFX1030-NEXT:    v_add_co_ci_u32_e64 v5, null, 0, 0, s4
-; GFX1030-NEXT:    v_sub_nc_u32_e32 v6, v3, v1
-; GFX1030-NEXT:    v_mad_u64_u32 v[3:4], null, 0x80000001, v1, v[4:5]
-; GFX1030-NEXT:    v_sub_nc_u32_e32 v5, v6, v0
-; GFX1030-NEXT:    v_add_co_u32 v2, vcc_lo, v3, v2
-; GFX1030-NEXT:    v_add_co_ci_u32_e32 v3, vcc_lo, v4, v5, vcc_lo
+; GFX1030-NEXT:    v_lshl_add_u32 v8, v8, 31, v8
+; GFX1030-NEXT:    v_add3_u32 v7, v7, v8, v6
+; GFX1030-NEXT:    v_mov_b32_e32 v2, v4
+; GFX1030-NEXT:    v_mad_u64_u32 v[6:7], null, v0, -1, v[6:7]
+; GFX1030-NEXT:    v_mad_u64_u32 v[2:3], null, 0x80000001, v0, v[2:3]
+; GFX1030-NEXT:    v_sub_nc_u32_e32 v4, v7, v1
+; GFX1030-NEXT:    v_add_co_u32 v2, s4, v5, v3
+; GFX1030-NEXT:    v_add_co_ci_u32_e64 v3, null, 0, 0, s4
+; GFX1030-NEXT:    v_sub_nc_u32_e32 v4, v4, v0
+; GFX1030-NEXT:    v_mad_u64_u32 v[2:3], null, 0x80000001, v1, v[2:3]
+; GFX1030-NEXT:    v_add_co_u32 v2, vcc_lo, v2, v6
+; GFX1030-NEXT:    v_add_co_ci_u32_e32 v3, vcc_lo, v3, v4, vcc_lo
 ; GFX1030-NEXT:    v_mad_u64_u32 v[2:3], null, v0, 1, v[2:3]
 ; GFX1030-NEXT:    v_add_nc_u32_e32 v3, v1, v3
 ; GFX1030-NEXT:    v_ashrrev_i64 v[0:1], 30, v[2:3]
@@ -1259,11 +1212,9 @@ define noundef i64 @urem64_i32max(i64 noundef %i)  {
 ; GFX9-NEXT:    v_mov_b32_e32 v3, 0
 ; GFX9-NEXT:    s_brev_b32 s6, -2
 ; GFX9-NEXT:    v_mad_u64_u32 v[4:5], s[4:5], v1, 5, v[2:3]
-; GFX9-NEXT:    v_mov_b32_e32 v6, v5
-; GFX9-NEXT:    v_mov_b32_e32 v5, v3
-; GFX9-NEXT:    v_mad_u64_u32 v[2:3], s[4:5], v0, 2, v[4:5]
-; GFX9-NEXT:    v_mov_b32_e32 v2, v3
-; GFX9-NEXT:    v_add_co_u32_e32 v2, vcc, v6, v2
+; GFX9-NEXT:    v_mov_b32_e32 v2, v4
+; GFX9-NEXT:    v_mad_u64_u32 v[2:3], s[4:5], v0, 2, v[2:3]
+; GFX9-NEXT:    v_add_co_u32_e32 v2, vcc, v5, v3
 ; GFX9-NEXT:    v_addc_co_u32_e64 v3, s[4:5], 0, 0, vcc
 ; GFX9-NEXT:    v_mad_u64_u32 v[2:3], s[4:5], v1, 2, v[2:3]
 ; GFX9-NEXT:    v_sub_co_u32_e32 v4, vcc, v0, v2
@@ -1303,10 +1254,9 @@ define noundef i64 @urem64_i32max(i64 noundef %i)  {
 ; GFX942-NEXT:    v_mov_b32_e32 v2, v5
 ; GFX942-NEXT:    v_lshrrev_b32_e32 v3, 30, v3
 ; GFX942-NEXT:    v_mad_u64_u32 v[2:3], s[0:1], v3, s2, v[2:3]
-; GFX942-NEXT:    v_mov_b32_e32 v3, v2
 ; GFX942-NEXT:    v_sub_co_u32_e32 v0, vcc, v0, v4
 ; GFX942-NEXT:    s_nop 1
-; GFX942-NEXT:    v_subb_co_u32_e32 v1, vcc, v1, v3, vcc
+; GFX942-NEXT:    v_subb_co_u32_e32 v1, vcc, v1, v2, vcc
 ; GFX942-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX1030-LABEL: urem64_i32max:
@@ -1315,11 +1265,9 @@ define noundef i64 @urem64_i32max(i64 noundef %i)  {
 ; GFX1030-NEXT:    v_mul_hi_u32 v2, v0, 5
 ; GFX1030-NEXT:    v_mov_b32_e32 v3, 0
 ; GFX1030-NEXT:    v_mad_u64_u32 v[4:5], null, v1, 5, v[2:3]
-; GFX1030-NEXT:    v_mov_b32_e32 v6, v5
-; GFX1030-NEXT:    v_mov_b32_e32 v5, v3
-; GFX1030-NEXT:    v_mad_u64_u32 v[2:3], null, v0, 2, v[4:5]
-; GFX1030-NEXT:    v_mov_b32_e32 v2, v3
-; GFX1030-NEXT:    v_add_co_u32 v2, s4, v6, v2
+; GFX1030-NEXT:    v_mov_b32_e32 v2, v4
+; GFX1030-NEXT:    v_mad_u64_u32 v[2:3], null, v0, 2, v[2:3]
+; GFX1030-NEXT:    v_add_co_u32 v2, s4, v5, v3
 ; GFX1030-NEXT:    v_add_co_ci_u32_e64 v3, null, 0, 0, s4
 ; GFX1030-NEXT:    v_mad_u64_u32 v[2:3], null, v1, 2, v[2:3]
 ; GFX1030-NEXT:    v_sub_co_u32 v4, vcc_lo, v0, v2
@@ -1346,11 +1294,9 @@ define noundef i64 @udiv64_i32max(i64 noundef %i)  {
 ; GFX9-NEXT:    v_mul_hi_u32 v2, v0, 5
 ; GFX9-NEXT:    v_mov_b32_e32 v3, 0
 ; GFX9-NEXT:    v_mad_u64_u32 v[4:5], s[4:5], v1, 5, v[2:3]
-; GFX9-NEXT:    v_mov_b32_e32 v6, v5
-; GFX9-NEXT:    v_mov_b32_e32 v5, v3
-; GFX9-NEXT:    v_mad_u64_u32 v[2:3], s[4:5], v0, 2, v[4:5]
-; GFX9-NEXT:    v_mov_b32_e32 v2, v3
-; GFX9-NEXT:    v_add_co_u32_e32 v2, vcc, v6, v2
+; GFX9-NEXT:    v_mov_b32_e32 v2, v4
+; GFX9-NEXT:    v_mad_u64_u32 v[2:3], s[4:5], v0, 2, v[2:3]
+; GFX9-NEXT:    v_add_co_u32_e32 v2, vcc, v5, v3
 ; GFX9-NEXT:    v_addc_co_u32_e64 v3, s[4:5], 0, 0, vcc
 ; GFX9-NEXT:    v_mad_u64_u32 v[2:3], s[4:5], v1, 2, v[2:3]
 ; GFX9-NEXT:    v_sub_co_u32_e32 v0, vcc, v0, v2
@@ -1390,11 +1336,9 @@ define noundef i64 @udiv64_i32max(i64 noundef %i)  {
 ; GFX1030-NEXT:    v_mul_hi_u32 v2, v0, 5
 ; GFX1030-NEXT:    v_mov_b32_e32 v3, 0
 ; GFX1030-NEXT:    v_mad_u64_u32 v[4:5], null, v1, 5, v[2:3]
-; GFX1030-NEXT:    v_mov_b32_e32 v6, v5
-; GFX1030-NEXT:    v_mov_b32_e32 v5, v3
-; GFX1030-NEXT:    v_mad_u64_u32 v[2:3], null, v0, 2, v[4:5]
-; GFX1030-NEXT:    v_mov_b32_e32 v2, v3
-; GFX1030-NEXT:    v_add_co_u32 v2, s4, v6, v2
+; GFX1030-NEXT:    v_mov_b32_e32 v2, v4
+; GFX1030-NEXT:    v_mad_u64_u32 v[2:3], null, v0, 2, v[2:3]
+; GFX1030-NEXT:    v_add_co_u32 v2, s4, v5, v3
 ; GFX1030-NEXT:    v_add_co_ci_u32_e64 v3, null, 0, 0, s4
 ; GFX1030-NEXT:    v_mad_u64_u32 v[2:3], null, v1, 2, v[2:3]
 ; GFX1030-NEXT:    v_sub_co_u32 v0, vcc_lo, v0, v2
diff --git a/llvm/test/CodeGen/AMDGPU/div_v2i128.ll b/llvm/test/CodeGen/AMDGPU/div_v2i128.ll
index 8d65fa053eaa4..41999b249a0e8 100644
--- a/llvm/test/CodeGen/AMDGPU/div_v2i128.ll
+++ b/llvm/test/CodeGen/AMDGPU/div_v2i128.ll
@@ -1954,65 +1954,62 @@ define <2 x i128> @v_srem_v2i128_vv(<2 x i128> %lhs, <2 x i128> %rhs) {
 ; SDAG-NEXT:    v_mul_lo_u32 v12, v33, v3
 ; SDAG-NEXT:    v_mad_u64_u32 v[10:11], s[4:5], v33, v2, 0
 ; SDAG-NEXT:    v_mul_lo_u32 v24, v27, v2
-; SDAG-NEXT:    v_mul_lo_u32 v35, v35, v31
-; SDAG-NEXT:    v_mul_lo_u32 v38, v32, v30
+; SDAG-NEXT:    v_mul_lo_u32 v25, v35, v31
+; SDAG-NEXT:    v_mul_lo_u32 v35, v32, v30
 ; SDAG-NEXT:    v_mad_u64_u32 v[2:3], s[4:5], v31, v33, 0
 ; SDAG-NEXT:    v_mov_b32_e32 v13, 0
-; SDAG-NEXT:    v_mul_lo_u32 v25, v14, v7
+; SDAG-NEXT:    v_mul_lo_u32 v38, v14, v7
 ; SDAG-NEXT:    v_mad_u64_u32 v[20:21], s[4:5], v14, v6, 0
 ; SDAG-NEXT:    v_mul_lo_u32 v39, v15, v6
-; SDAG-NEXT:    v_mul_lo_u32 v19, v19, v37
-; SDAG-NEXT:    v_mul_lo_u32 v48, v18, v36
+; SDAG-NEXT:    v_mul_lo_u32 v48, v19, v37
+; SDAG-NEXT:    v_mul_lo_u32 v49, v18, v36
 ; SDAG-NEXT:    v_mad_u64_u32 v[6:7], s[4:5], v37, v14, 0
 ; SDAG-NEXT:    v_add_i32_e32 v11, vcc, v11, v12
 ; SDAG-NEXT:    v_mov_b32_e32 v12, v3
 ; SDAG-NEXT:    v_mad_u64_u32 v[22:23], s[4:5], v30, v33, v[12:13]
-; SDAG-NEXT:    v_sub_i32_e32 v12, vcc, v16, v2
-; SDAG-NEXT:    v_add_i32_e64 v16, s[4:5], v21, v25
+; SDAG-NEXT:    v_sub_i32_e32 v16, vcc, v16, v2
+; SDAG-NEXT:    v_add_i32_e64 v19, s[4:5], v21, v38
 ; SDAG-NEXT:    v_add_i32_e64 v11, s[4:5], v11, v24
-; SDAG-NEXT:    v_mov_b32_e32 v24, v23
-; SDAG-NEXT:    v_mov_b32_e32 v23, v13
-; SDAG-NEXT:    v_mad_u64_u32 v[2:3], s[4:5], v31, v27, v[22:23]
-; SDAG-NEXT:    v_xor_b32_e32 v33, v12, v28
-; SDAG-NEXT:    v_add_i32_e64 v21, s[4:5], v16, v39
-; SDAG-NEXT:    v_mov_b32_e32 v12, v7
-; SDAG-NEXT:    v_mad_u64_u32 v[22:23], s[4:5], v36, v14, v[12:13]
+; SDAG-NEXT:    v_mov_b32_e32 v12, v22
+; SDAG-NEXT:    v_mad_u64_u32 v[2:3], s[4:5], v31, v27, v[12:13]
+; SDAG-NEXT:    v_xor_b32_e32 v24, v16, v28
+; SDAG-NEXT:    v_add_i32_e64 v21, s[4:5], v19, v39
 ; SDAG-NEXT:    v_mad_u64_u32 v[10:11], s[4:5], v32, v31, v[10:11]
-; SDAG-NEXT:    v_add_i32_e64 v24, s[4:5], v24, v3
-; SDAG-NEXT:    v_addc_u32_e64 v25, s[4:5], 0, 0, s[4:5]
-; SDAG-NEXT:    v_subb_u32_e32 v7, vcc, v17, v2, vcc
+; SDAG-NEXT:    v_add_i32_e64 v22, s[4:5], v23, v3
+; SDAG-NEXT:    v_addc_u32_e64 v23, s[4:5], 0, 0, s[4:5]
+; SDAG-NEXT:    v_subb_u32_e32 v31, vcc, v17, v2, vcc
 ; SDAG-NEXT:    v_mad_u64_u32 v[2:3], s[4:5], v18, v37, v[20:21]
-; SDAG-NEXT:    v_mov_b32_e32 v14, v23
-; SDAG-NEXT:    v_mov_b32_e32 v23, v13
-; SDAG-NEXT:    v_mad_u64_u32 v[12:13], s[4:5], v37, v15, v[22:23]
-; SDAG-NEXT:    v_add_i32_e64 v11, s[4:5], v35, v11
-; SDAG-NEXT:    v_mad_u64_u32 v[16:17], s[4:5], v30, v27, v[24:25]
-; SDAG-NEXT:    v_xor_b32_e32 v7, v7, v29
-; SDAG-NEXT:    v_add_i32_e64 v3, s[4:5], v19, v3
-; SDAG-NEXT:    v_add_i32_e64 v13, s[4:5], v14, v13
-; SDAG-NEXT:    v_addc_u32_e64 v14, s[4:5], 0, 0, s[4:5]
-; SDAG-NEXT:    v_mov_b32_e32 v18, v12
-; SDAG-NEXT:    v_add_i32_e64 v19, s[4:5], v38, v11
+; SDAG-NEXT:    v_mov_b32_e32 v12, v7
+; SDAG-NEXT:    v_mad_u64_u32 v[16:17], s[4:5], v36, v14, v[12:13]
+; SDAG-NEXT:    v_add_i32_e64 v7, s[4:5], v25, v11
+; SDAG-NEXT:    v_mad_u64_u32 v[18:19], s[4:5], v30, v27, v[22:23]
+; SDAG-NEXT:    v_xor_b32_e32 v14, v31, v29
 ; SDAG-NEXT:    v_add_i32_e64 v3, s[4:5], v48, v3
-; SDAG-NEXT:    v_mad_u64_u32 v[11:12], s[4:5], v36, v15, v[13:14]
-; SDAG-NEXT:    v_add_i32_e64 v10, s[4:5], v16, v10
-; SDAG-NEXT:    v_addc_u32_e64 v13, s[4:5], v17, v19, s[4:5]
+; SDAG-NEXT:    v_mov_b32_e32 v12, v16
+; SDAG-NEXT:    v_mad_u64_u32 v[11:12], s[4:5], v37, v15, v[12:13]
+; SDAG-NEXT:    v_add_i32_e64 v7, s[4:5], v35, v7
+; SDAG-NEXT:    v_add_i32_e64 v3, s[4:5], v49, v3
+; SDAG-NEXT:    v_add_i32_e64 v12, s[4:5], v17, v12
+; SDAG-NEXT:    v_addc_u32_e64 v13, s[4:5], 0, 0, s[4:5]
+; SDAG-NEXT:    v_add_i32_e64 v10, s[4:5], v18, v10
+; SDAG-NEXT:    v_addc_u32_e64 v7, s[4:5], v19, v7, s[4:5]
 ; SDAG-NEXT:    v_subb_u32_e32 v0, vcc, v0, v10, vcc
-; SDAG-NEXT:    v_add_i32_e64 v10, s[4:5], v11, v2
-; SDAG-NEXT:    v_addc_u32_e64 v11, s[4:5], v12, v3, s[4:5]
-; SDAG-NEXT:    v_subb_u32_e32 v1, vcc, v1, v13, vcc
-; SDAG-NEXT:    v_xor_b32_e32 v2, v0, v28
+; SDAG-NEXT:    v_mad_u64_u32 v[12:13], s[4:5], v36, v15, v[12:13]
+; SDAG-NEXT:    v_subb_u32_e32 v1, vcc, v1, v7, vcc
+; SDAG-NEXT:    v_xor_b32_e32 v7, v0, v28
+; SDAG-NEXT:    v_add_i32_e32 v10, vcc, v12, v2
+; SDAG-NEXT:    v_addc_u32_e32 v12, vcc, v13, v3, vcc
 ; SDAG-NEXT:    v_xor_b32_e32 v3, v1, v29
-; SDAG-NEXT:    v_sub_i32_e32 v0, vcc, v33, v28
-; SDAG-NEXT:    v_subb_u32_e32 v1, vcc, v7, v29, vcc
-; SDAG-NEXT:    v_subb_u32_e32 v2, vcc, v2, v28, vcc
+; SDAG-NEXT:    v_sub_i32_e32 v0, vcc, v24, v28
+; SDAG-NEXT:    v_subb_u32_e32 v1, vcc, v14, v29, vcc
+; SDAG-NEXT:    v_subb_u32_e32 v2, vcc, v7, v28, vcc
 ; SDAG-NEXT:    v_subb_u32_e32 v3, vcc, v3, v29, vcc
 ; SDAG-NEXT:    v_sub_i32_e32 v6, vcc, v8, v6
-; SDAG-NEXT:    v_subb_u32_e32 v7, vcc, v9, v18, vcc
+; SDAG-NEXT:    v_subb_u32_e32 v7, vcc, v9, v11, vcc
 ; SDAG-NEXT:    v_xor_b32_e32 v6, v6, v26
 ; SDAG-NEXT:    v_subb_u32_e32 v4, vcc, v4, v10, vcc
 ; SDAG-NEXT:    v_xor_b32_e32 v7, v7, v34
-; SDAG-NEXT:    v_subb_u32_e32 v5, vcc, v5, v11, vcc
+; SDAG-NEXT:    v_subb_u32_e32 v5, vcc, v5, v12, vcc
 ; SDAG-NEXT:    v_xor_b32_e32 v8, v4, v26
 ; SDAG-NEXT:    v_xor_b32_e32 v9, v5, v34
 ; SDAG-NEXT:    v_sub_i32_e32 v4, vcc, v6, v26
@@ -2827,49 +2824,43 @@ define <2 x i128> @v_urem_v2i128_vv(<2 x i128> %lhs, <2 x i128> %rhs) {
 ; SDAG-NEXT:    v_mul_lo_u32 v34, v18, v15
 ; SDAG-NEXT:    v_mad_u64_u32 v[24:25], s[4:5], v18, v14, 0
 ; SDAG-NEXT:    v_mul_lo_u32 v35, v19, v14
-; SDAG-NEXT:    v_mul_lo_u32 v23, v23, v12
-; SDAG-NEXT:    v_mul_lo_u32 v36, v22, v13
+; SDAG-NEXT:    v_mul_lo_u32 v36, v23, v12
+; SDAG-NEXT:    v_mul_lo_u32 v37, v22, v13
 ; SDAG-NEXT:    v_mad_u64_u32 v[14:15], s[4:5], v12, v18, 0
 ; SDAG-NEXT:    v_add_i32_e32 v17, vcc, v17, v20
 ; SDAG-NEXT:    v_mov_b32_e32 v20, v11
 ; SDAG-NEXT:    v_mad_u64_u32 v[26:27], s[4:5], v9, v32, v[20:21]
 ; SDAG-NEXT:    v_sub_i32_e32 v0, vcc, v0, v10
-; SDAG-NEXT:    v_add_i32_e64 v20, s[4:5], v25, v34
+; SDAG-NEXT:    v_add_i32_e64 v23, s[4:5], v25, v34
 ; SDAG-NEXT:    v_add_i32_e64 v17, s[4:5], v17, v28
-; SDAG-NEXT:    v_mov_b32_e32 v28, v27
-; SDAG-NEXT:    v_mov_b32_e32 v27, v21
-; SDAG-NEXT:    v_mad_u64_u32 v[10:11], s[4:5], v8, v30, v[26:27]
-; SDAG-NEXT:    v_add_i32_e64 v25, s[4:5], v20, v35
+; SDAG-NEXT:    v_mov_b32_e32 v20, v26
+; SDAG-NEXT:    v_mad_u64_u32 v[10:11], s[4:5], v8, v30, v[20:21]
+; SDAG-NEXT:    v_add_i32_e64 v25, s[4:5], v23, v35
+; SDAG-NEXT:    v_mad_u64_u32 v[16:17], s[4:5], v31, v8, v[16:17]
+; SDAG-NEXT:    v_add_i32_e64 v26, s[4:5], v27, v11
+; SDAG-NEXT:    v_addc_u32_e64 v27, s[4:5], 0, 0, s[4:5]
+; SDAG-NEXT:    v_subb_u32_e32 v1, vcc, v1, v10, vcc
+; SDAG-NEXT:    v_mad_u64_u32 v[10:11], s[4:5], v22, v12, v[24:25]
 ; SDAG-NEXT:    v_mov_b32_e32 v20, v15
-; SDAG-NEXT:    v_mad_u64_u32 v[26:27], s[4:5], v13, v18, v[20:21]
-; SDAG-NEXT:    v_mad_u64_u32 v[15:16], s[4:5], v31, v8, v[16:17]
-; SDAG-NEXT:    v_mov_b32_e32 v8, v11
-; SDAG-NEXT:    v_add_i32_e64 v17, s[4:5], v28, v8
+; SDAG-NEXT:    v_mad_u64_u32 v[22:23], s[4:5], v13, v18, v[20:21]
+; SDAG-NEXT:    v_add_i32_e64 v15, s[4:5], v29, v17
+; SDAG-NEXT:    v_mad_u64_u32 v[8:9], s[4:5], v9, v30, v[26:27]
+; SDAG-NEXT:    v_add_i32_e64 v17, s[4:5], v36, v11
+; SDAG-NEXT:    v_mov_b32_e32 v20, v22
+; SDAG-NEXT:    v_mad_u64_u32 v[11:12], s[4:5], v12, v19, v[20:21]
+; SDAG-NEXT:    v_add_i32_e64 v15, s[4:5], v33, v15
+; SDAG-NEXT:    v_add_i32_e64 v20, s[4:5], v37, v17
+; SDAG-NEXT:    v_add_i32_e64 v17, s[4:5], v23, v12
 ; SDAG-NEXT:    v_addc_u32_e64 v18, s[4:5], 0, 0, s[4:5]
-; SDAG-NEXT:    v_mov_b32_e32 v8, v10
-; SDAG-NEXT:    v_subb_u32_e32 v1, vcc, v1, v8, vcc
-; SDAG-NEXT:    v_mad_u64_u32 v[10:11], s[4:5], v22, v12, v[24:25]
-; SDAG-NEXT:    v_mov_b32_e32 v22, v27
-; SDAG-NEXT:    v_mov_b32_e32 v27, v21
-; SDAG-NEXT:    v_mad_u64_u32 v[20:21], s[4:5], v12, v19, v[26:27]
-; SDAG-NEXT:    v_add_i32_e64 v16, s[4:5], v29, v16
-; SDAG-NEXT:    v_mad_u64_u32 v[8:9], s[4:5], v9, v30, v[17:18]
-; SDAG-NEXT:    v_add_i32_e64 v17, s[4:5], v23, v11
-; SDAG-NEXT:    v_mov_b32_e32 v11, v21
-; SDAG-NEXT:    v_add_i32_e64 v11, s[4:5], v22, v11
-; SDAG-NEXT:    v_addc_u32_e64 v12, s[4:5], 0, 0, s[4:5]
-; SDAG-NEXT:    v_add_i32_e64 v16, s[4:5], v33, v16
-; SDAG-NEXT:    v_add_i32_e64 v17, s[4:5], v36, v17
-; SDAG-NEXT:    v_mad_u64_u32 v[11:12], s[4:5], v13, v19, v[11:12]
-; SDAG-NEXT:    v_add_i32_e64 v8, s[4:5], v8, v15
-; SDAG-NEXT:    v_addc_u32_e64 v9, s[4:5], v9, v16, s[4:5]
+; SDAG-NEXT:    v_add_i32_e64 v8, s[4:5], v8, v16
+; SDAG-NEXT:    v_addc_u32_e64 v12, s[4:5], v9, v15, s[4:5]
 ; SDAG-NEXT:    v_subb_u32_e32 v2, vcc, v2, v8, vcc
-; SDAG-NEXT:    v_subb_u32_e32 v3, vcc, v3, v9, vcc
-; SDAG-NEXT:    v_add_i32_e32 v8, vcc, v11, v10
-; SDAG-NEXT:    v_addc_u32_e32 v9, vcc, v12, v17, vcc
-; SDAG-NEXT:    v_mov_b32_e32 v10, v20
+; SDAG-NEXT:    v_mad_u64_u32 v[8:9], s[4:5], v13, v19, v[17:18]
+; SDAG-NEXT:    v_subb_u32_e32 v3, vcc, v3, v12, vcc
+; SDAG-NEXT:    v_add_i32_e32 v8, vcc, v8, v10
+; SDAG-NEXT:    v_addc_u32_e32 v9, vcc, v9, v20, vcc
 ; SDAG-NEXT:    v_sub_i32_e32 v4, vcc, v4, v14
-; SDAG-NEXT:    v_subb_u32_e32 v5, vcc, v5, v10, vcc
+; SDAG-NEXT:    v_subb_u32_e32 v5, vcc, v5, v11, vcc
 ; SDAG-NEXT:    v_subb_u32_e32 v6, vcc, v6, v8, vcc
 ; SDAG-NEXT:    v_subb_u32_e32 v7, vcc, v7, v9, vcc
 ; SDAG-NEXT:    s_setpc_b64 s[30:31]
diff --git a/llvm/test/CodeGen/AMDGPU/do-not-unify-divergent-exit-nodes-with-musttail.ll b/llvm/test/CodeGen/AMDGPU/do-not-unify-divergent-exit-nodes-with-musttail.ll
new file mode 100644
index 0000000000000..007e3f0a6bdbc
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/do-not-unify-divergent-exit-nodes-with-musttail.ll
@@ -0,0 +1,53 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -S -passes=amdgpu-unify-divergent-exit-nodes -mtriple=amdgcn-amd-amdhsa -mcpu=gfx90a %s -o - | FileCheck %s
+
+declare void @foo(ptr)
+declare i1 @bar(ptr)
+
+define void @musttail_call_without_return_value(ptr %p) {
+; CHECK-LABEL: define void @musttail_call_without_return_value(
+; CHECK-SAME: ptr [[P:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[LOAD:%.*]] = load i1, ptr [[P]], align 1
+; CHECK-NEXT:    br i1 [[LOAD]], label %[[BB_0:.*]], label %[[BB_1:.*]]
+; CHECK:       [[BB_0]]:
+; CHECK-NEXT:    musttail call void @foo(ptr [[P]])
+; CHECK-NEXT:    ret void
+; CHECK:       [[BB_1]]:
+; CHECK-NEXT:    ret void
+;
+entry:
+  %load = load i1, ptr %p, align 1
+  br i1 %load, label %bb.0, label %bb.1
+
+bb.0:
+  musttail call void @foo(ptr %p)
+  ret void
+
+bb.1:
+  ret void
+}
+
+define i1 @musttail_call_with_return_value(ptr %p) {
+; CHECK-LABEL: define i1 @musttail_call_with_return_value(
+; CHECK-SAME: ptr [[P:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[LOAD:%.*]] = load i1, ptr [[P]], align 1
+; CHECK-NEXT:    br i1 [[LOAD]], label %[[BB_0:.*]], label %[[BB_1:.*]]
+; CHECK:       [[BB_0]]:
+; CHECK-NEXT:    [[RET:%.*]] = musttail call i1 @bar(ptr [[P]])
+; CHECK-NEXT:    ret i1 [[RET]]
+; CHECK:       [[BB_1]]:
+; CHECK-NEXT:    ret i1 [[LOAD]]
+;
+entry:
+  %load = load i1, ptr %p, align 1
+  br i1 %load, label %bb.0, label %bb.1
+
+bb.0:
+  %ret = musttail call i1 @bar(ptr %p)
+  ret i1 %ret
+
+bb.1:
+  ret i1 %load
+}
diff --git a/llvm/test/CodeGen/AMDGPU/elf-notes.ll b/llvm/test/CodeGen/AMDGPU/elf-notes.ll
index e91bed464136f..b205678bd9085 100644
--- a/llvm/test/CodeGen/AMDGPU/elf-notes.ll
+++ b/llvm/test/CodeGen/AMDGPU/elf-notes.ll
@@ -66,6 +66,7 @@
 ; OSABI-PAL-ELF: amdpal.pipelines:
 ; OSABI-PAL-ELF:   - .hardware_stages:
 ; OSABI-PAL-ELF:       .cs:
+; OSABI-PAL-ELF:         .entry_point:    _amdgpu_cs
 ; OSABI-PAL-ELF:         .entry_point_symbol:    elf_notes
 ; OSABI-PAL-ELF:         .scratch_memory_size: 0
 ; OSABI-PAL-ELF:         .sgpr_count:     96
diff --git a/llvm/test/CodeGen/AMDGPU/fcmp.f16.ll b/llvm/test/CodeGen/AMDGPU/fcmp.f16.ll
index 23b54c6741e51..9b98bd6c9ab03 100644
--- a/llvm/test/CodeGen/AMDGPU/fcmp.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/fcmp.f16.ll
@@ -1,7 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 2
 ; RUN:  llc -amdgpu-scalarize-global-loads=false -mtriple=amdgcn -verify-machineinstrs < %s | FileCheck -allow-deprecated-dag-overlap -enable-var-scope -check-prefixes=SI %s
 ; RUN:  llc -amdgpu-scalarize-global-loads=false -mtriple=amdgcn -mcpu=fiji -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -allow-deprecated-dag-overlap -enable-var-scope -check-prefixes=VI %s
-; RUN:  llc -amdgpu-scalarize-global-loads=false -mtriple=amdgcn -mcpu=gfx1100 -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -allow-deprecated-dag-overlap -enable-var-scope -check-prefixes=GFX11 %s
+; RUN:  llc -amdgpu-scalarize-global-loads=false -mtriple=amdgcn -mcpu=gfx1100 -mattr=-flat-for-global,+real-true16 -verify-machineinstrs < %s | FileCheck -allow-deprecated-dag-overlap -enable-var-scope -check-prefixes=GFX11-TRUE16 %s
+; RUN:  llc -amdgpu-scalarize-global-loads=false -mtriple=amdgcn -mcpu=gfx1100 -mattr=-flat-for-global,-real-true16 -verify-machineinstrs < %s | FileCheck -allow-deprecated-dag-overlap -enable-var-scope -check-prefixes=GFX11-FAKE16 %s
 ; RUN:  llc -amdgpu-scalarize-global-loads=false -mtriple=amdgcn -mcpu=gfx1200 -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -allow-deprecated-dag-overlap -enable-var-scope -check-prefixes=GFX12 %s
 
 define amdgpu_kernel void @fcmp_f16_lt(
@@ -55,30 +56,57 @@ define amdgpu_kernel void @fcmp_f16_lt(
 ; VI-NEXT:    buffer_store_dword v0, off, s[4:7], 0
 ; VI-NEXT:    s_endpgm
 ;
-; GFX11-LABEL: fcmp_f16_lt:
-; GFX11:       ; %bb.0: ; %entry
-; GFX11-NEXT:    s_clause 0x1
-; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
-; GFX11-NEXT:    s_load_b64 s[4:5], s[4:5], 0x34
-; GFX11-NEXT:    s_mov_b32 s10, -1
-; GFX11-NEXT:    s_mov_b32 s11, 0x31016000
-; GFX11-NEXT:    s_mov_b32 s14, s10
-; GFX11-NEXT:    s_mov_b32 s15, s11
-; GFX11-NEXT:    s_mov_b32 s6, s10
-; GFX11-NEXT:    s_mov_b32 s7, s11
-; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX11-NEXT:    s_mov_b32 s12, s2
-; GFX11-NEXT:    s_mov_b32 s13, s3
-; GFX11-NEXT:    buffer_load_u16 v0, off, s[12:15], 0 glc dlc
-; GFX11-NEXT:    s_waitcnt vmcnt(0)
-; GFX11-NEXT:    buffer_load_u16 v1, off, s[4:7], 0 glc dlc
-; GFX11-NEXT:    s_waitcnt vmcnt(0)
-; GFX11-NEXT:    s_mov_b32 s8, s0
-; GFX11-NEXT:    s_mov_b32 s9, s1
-; GFX11-NEXT:    v_cmp_lt_f16_e32 vcc_lo, v0, v1
-; GFX11-NEXT:    v_cndmask_b32_e64 v0, 0, -1, vcc_lo
-; GFX11-NEXT:    buffer_store_b32 v0, off, s[8:11], 0
-; GFX11-NEXT:    s_endpgm
+; GFX11-TRUE16-LABEL: fcmp_f16_lt:
+; GFX11-TRUE16:       ; %bb.0: ; %entry
+; GFX11-TRUE16-NEXT:    s_clause 0x1
+; GFX11-TRUE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-TRUE16-NEXT:    s_load_b64 s[4:5], s[4:5], 0x34
+; GFX11-TRUE16-NEXT:    s_mov_b32 s10, -1
+; GFX11-TRUE16-NEXT:    s_mov_b32 s11, 0x31016000
+; GFX11-TRUE16-NEXT:    s_mov_b32 s14, s10
+; GFX11-TRUE16-NEXT:    s_mov_b32 s15, s11
+; GFX11-TRUE16-NEXT:    s_mov_b32 s6, s10
+; GFX11-TRUE16-NEXT:    s_mov_b32 s7, s11
+; GFX11-TRUE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-TRUE16-NEXT:    s_mov_b32 s12, s2
+; GFX11-TRUE16-NEXT:    s_mov_b32 s13, s3
+; GFX11-TRUE16-NEXT:    buffer_load_u16 v0, off, s[12:15], 0 glc dlc
+; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT:    buffer_load_u16 v1, off, s[4:7], 0 glc dlc
+; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT:    s_mov_b32 s8, s0
+; GFX11-TRUE16-NEXT:    s_mov_b32 s9, s1
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v0.h, v1.l
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_cmp_lt_f16_e32 vcc_lo, v0.l, v0.h
+; GFX11-TRUE16-NEXT:    v_cndmask_b32_e64 v0, 0, -1, vcc_lo
+; GFX11-TRUE16-NEXT:    buffer_store_b32 v0, off, s[8:11], 0
+; GFX11-TRUE16-NEXT:    s_endpgm
+;
+; GFX11-FAKE16-LABEL: fcmp_f16_lt:
+; GFX11-FAKE16:       ; %bb.0: ; %entry
+; GFX11-FAKE16-NEXT:    s_clause 0x1
+; GFX11-FAKE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-FAKE16-NEXT:    s_load_b64 s[4:5], s[4:5], 0x34
+; GFX11-FAKE16-NEXT:    s_mov_b32 s10, -1
+; GFX11-FAKE16-NEXT:    s_mov_b32 s11, 0x31016000
+; GFX11-FAKE16-NEXT:    s_mov_b32 s14, s10
+; GFX11-FAKE16-NEXT:    s_mov_b32 s15, s11
+; GFX11-FAKE16-NEXT:    s_mov_b32 s6, s10
+; GFX11-FAKE16-NEXT:    s_mov_b32 s7, s11
+; GFX11-FAKE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-FAKE16-NEXT:    s_mov_b32 s12, s2
+; GFX11-FAKE16-NEXT:    s_mov_b32 s13, s3
+; GFX11-FAKE16-NEXT:    buffer_load_u16 v0, off, s[12:15], 0 glc dlc
+; GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT:    buffer_load_u16 v1, off, s[4:7], 0 glc dlc
+; GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT:    s_mov_b32 s8, s0
+; GFX11-FAKE16-NEXT:    s_mov_b32 s9, s1
+; GFX11-FAKE16-NEXT:    v_cmp_lt_f16_e32 vcc_lo, v0, v1
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e64 v0, 0, -1, vcc_lo
+; GFX11-FAKE16-NEXT:    buffer_store_b32 v0, off, s[8:11], 0
+; GFX11-FAKE16-NEXT:    s_endpgm
 ;
 ; GFX12-LABEL: fcmp_f16_lt:
 ; GFX12:       ; %bb.0: ; %entry
@@ -167,31 +195,58 @@ define amdgpu_kernel void @fcmp_f16_lt_abs(
 ; VI-NEXT:    buffer_store_dword v0, off, s[4:7], 0
 ; VI-NEXT:    s_endpgm
 ;
-; GFX11-LABEL: fcmp_f16_lt_abs:
-; GFX11:       ; %bb.0: ; %entry
-; GFX11-NEXT:    s_clause 0x1
-; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
-; GFX11-NEXT:    s_load_b64 s[4:5], s[4:5], 0x34
-; GFX11-NEXT:    s_mov_b32 s10, -1
-; GFX11-NEXT:    s_mov_b32 s11, 0x31016000
-; GFX11-NEXT:    s_mov_b32 s14, s10
-; GFX11-NEXT:    s_mov_b32 s15, s11
-; GFX11-NEXT:    s_mov_b32 s6, s10
-; GFX11-NEXT:    s_mov_b32 s7, s11
-; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX11-NEXT:    s_mov_b32 s12, s2
-; GFX11-NEXT:    s_mov_b32 s13, s3
-; GFX11-NEXT:    buffer_load_u16 v0, off, s[12:15], 0 glc dlc
-; GFX11-NEXT:    s_waitcnt vmcnt(0)
-; GFX11-NEXT:    buffer_load_u16 v1, off, s[4:7], 0 glc dlc
-; GFX11-NEXT:    s_waitcnt vmcnt(0)
-; GFX11-NEXT:    s_mov_b32 s8, s0
-; GFX11-NEXT:    s_mov_b32 s9, s1
-; GFX11-NEXT:    v_cmp_lt_f16_e64 s2, |v0|, |v1|
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT:    v_cndmask_b32_e64 v0, 0, -1, s2
-; GFX11-NEXT:    buffer_store_b32 v0, off, s[8:11], 0
-; GFX11-NEXT:    s_endpgm
+; GFX11-TRUE16-LABEL: fcmp_f16_lt_abs:
+; GFX11-TRUE16:       ; %bb.0: ; %entry
+; GFX11-TRUE16-NEXT:    s_clause 0x1
+; GFX11-TRUE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-TRUE16-NEXT:    s_load_b64 s[4:5], s[4:5], 0x34
+; GFX11-TRUE16-NEXT:    s_mov_b32 s10, -1
+; GFX11-TRUE16-NEXT:    s_mov_b32 s11, 0x31016000
+; GFX11-TRUE16-NEXT:    s_mov_b32 s14, s10
+; GFX11-TRUE16-NEXT:    s_mov_b32 s15, s11
+; GFX11-TRUE16-NEXT:    s_mov_b32 s6, s10
+; GFX11-TRUE16-NEXT:    s_mov_b32 s7, s11
+; GFX11-TRUE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-TRUE16-NEXT:    s_mov_b32 s12, s2
+; GFX11-TRUE16-NEXT:    s_mov_b32 s13, s3
+; GFX11-TRUE16-NEXT:    buffer_load_u16 v0, off, s[12:15], 0 glc dlc
+; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT:    buffer_load_u16 v1, off, s[4:7], 0 glc dlc
+; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT:    s_mov_b32 s8, s0
+; GFX11-TRUE16-NEXT:    s_mov_b32 s9, s1
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v0.h, v1.l
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_cmp_lt_f16_e64 s2, |v0.l|, |v0.h|
+; GFX11-TRUE16-NEXT:    v_cndmask_b32_e64 v0, 0, -1, s2
+; GFX11-TRUE16-NEXT:    buffer_store_b32 v0, off, s[8:11], 0
+; GFX11-TRUE16-NEXT:    s_endpgm
+;
+; GFX11-FAKE16-LABEL: fcmp_f16_lt_abs:
+; GFX11-FAKE16:       ; %bb.0: ; %entry
+; GFX11-FAKE16-NEXT:    s_clause 0x1
+; GFX11-FAKE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-FAKE16-NEXT:    s_load_b64 s[4:5], s[4:5], 0x34
+; GFX11-FAKE16-NEXT:    s_mov_b32 s10, -1
+; GFX11-FAKE16-NEXT:    s_mov_b32 s11, 0x31016000
+; GFX11-FAKE16-NEXT:    s_mov_b32 s14, s10
+; GFX11-FAKE16-NEXT:    s_mov_b32 s15, s11
+; GFX11-FAKE16-NEXT:    s_mov_b32 s6, s10
+; GFX11-FAKE16-NEXT:    s_mov_b32 s7, s11
+; GFX11-FAKE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-FAKE16-NEXT:    s_mov_b32 s12, s2
+; GFX11-FAKE16-NEXT:    s_mov_b32 s13, s3
+; GFX11-FAKE16-NEXT:    buffer_load_u16 v0, off, s[12:15], 0 glc dlc
+; GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT:    buffer_load_u16 v1, off, s[4:7], 0 glc dlc
+; GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT:    s_mov_b32 s8, s0
+; GFX11-FAKE16-NEXT:    s_mov_b32 s9, s1
+; GFX11-FAKE16-NEXT:    v_cmp_lt_f16_e64 s2, |v0|, |v1|
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e64 v0, 0, -1, s2
+; GFX11-FAKE16-NEXT:    buffer_store_b32 v0, off, s[8:11], 0
+; GFX11-FAKE16-NEXT:    s_endpgm
 ;
 ; GFX12-LABEL: fcmp_f16_lt_abs:
 ; GFX12:       ; %bb.0: ; %entry
@@ -285,30 +340,57 @@ define amdgpu_kernel void @fcmp_f16_eq(
 ; VI-NEXT:    buffer_store_dword v0, off, s[4:7], 0
 ; VI-NEXT:    s_endpgm
 ;
-; GFX11-LABEL: fcmp_f16_eq:
-; GFX11:       ; %bb.0: ; %entry
-; GFX11-NEXT:    s_clause 0x1
-; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
-; GFX11-NEXT:    s_load_b64 s[4:5], s[4:5], 0x34
-; GFX11-NEXT:    s_mov_b32 s10, -1
-; GFX11-NEXT:    s_mov_b32 s11, 0x31016000
-; GFX11-NEXT:    s_mov_b32 s14, s10
-; GFX11-NEXT:    s_mov_b32 s15, s11
-; GFX11-NEXT:    s_mov_b32 s6, s10
-; GFX11-NEXT:    s_mov_b32 s7, s11
-; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX11-NEXT:    s_mov_b32 s12, s2
-; GFX11-NEXT:    s_mov_b32 s13, s3
-; GFX11-NEXT:    buffer_load_u16 v0, off, s[12:15], 0 glc dlc
-; GFX11-NEXT:    s_waitcnt vmcnt(0)
-; GFX11-NEXT:    buffer_load_u16 v1, off, s[4:7], 0 glc dlc
-; GFX11-NEXT:    s_waitcnt vmcnt(0)
-; GFX11-NEXT:    s_mov_b32 s8, s0
-; GFX11-NEXT:    s_mov_b32 s9, s1
-; GFX11-NEXT:    v_cmp_eq_f16_e32 vcc_lo, v0, v1
-; GFX11-NEXT:    v_cndmask_b32_e64 v0, 0, -1, vcc_lo
-; GFX11-NEXT:    buffer_store_b32 v0, off, s[8:11], 0
-; GFX11-NEXT:    s_endpgm
+; GFX11-TRUE16-LABEL: fcmp_f16_eq:
+; GFX11-TRUE16:       ; %bb.0: ; %entry
+; GFX11-TRUE16-NEXT:    s_clause 0x1
+; GFX11-TRUE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-TRUE16-NEXT:    s_load_b64 s[4:5], s[4:5], 0x34
+; GFX11-TRUE16-NEXT:    s_mov_b32 s10, -1
+; GFX11-TRUE16-NEXT:    s_mov_b32 s11, 0x31016000
+; GFX11-TRUE16-NEXT:    s_mov_b32 s14, s10
+; GFX11-TRUE16-NEXT:    s_mov_b32 s15, s11
+; GFX11-TRUE16-NEXT:    s_mov_b32 s6, s10
+; GFX11-TRUE16-NEXT:    s_mov_b32 s7, s11
+; GFX11-TRUE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-TRUE16-NEXT:    s_mov_b32 s12, s2
+; GFX11-TRUE16-NEXT:    s_mov_b32 s13, s3
+; GFX11-TRUE16-NEXT:    buffer_load_u16 v0, off, s[12:15], 0 glc dlc
+; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT:    buffer_load_u16 v1, off, s[4:7], 0 glc dlc
+; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT:    s_mov_b32 s8, s0
+; GFX11-TRUE16-NEXT:    s_mov_b32 s9, s1
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v0.h, v1.l
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_cmp_eq_f16_e32 vcc_lo, v0.l, v0.h
+; GFX11-TRUE16-NEXT:    v_cndmask_b32_e64 v0, 0, -1, vcc_lo
+; GFX11-TRUE16-NEXT:    buffer_store_b32 v0, off, s[8:11], 0
+; GFX11-TRUE16-NEXT:    s_endpgm
+;
+; GFX11-FAKE16-LABEL: fcmp_f16_eq:
+; GFX11-FAKE16:       ; %bb.0: ; %entry
+; GFX11-FAKE16-NEXT:    s_clause 0x1
+; GFX11-FAKE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-FAKE16-NEXT:    s_load_b64 s[4:5], s[4:5], 0x34
+; GFX11-FAKE16-NEXT:    s_mov_b32 s10, -1
+; GFX11-FAKE16-NEXT:    s_mov_b32 s11, 0x31016000
+; GFX11-FAKE16-NEXT:    s_mov_b32 s14, s10
+; GFX11-FAKE16-NEXT:    s_mov_b32 s15, s11
+; GFX11-FAKE16-NEXT:    s_mov_b32 s6, s10
+; GFX11-FAKE16-NEXT:    s_mov_b32 s7, s11
+; GFX11-FAKE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-FAKE16-NEXT:    s_mov_b32 s12, s2
+; GFX11-FAKE16-NEXT:    s_mov_b32 s13, s3
+; GFX11-FAKE16-NEXT:    buffer_load_u16 v0, off, s[12:15], 0 glc dlc
+; GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT:    buffer_load_u16 v1, off, s[4:7], 0 glc dlc
+; GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT:    s_mov_b32 s8, s0
+; GFX11-FAKE16-NEXT:    s_mov_b32 s9, s1
+; GFX11-FAKE16-NEXT:    v_cmp_eq_f16_e32 vcc_lo, v0, v1
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e64 v0, 0, -1, vcc_lo
+; GFX11-FAKE16-NEXT:    buffer_store_b32 v0, off, s[8:11], 0
+; GFX11-FAKE16-NEXT:    s_endpgm
 ;
 ; GFX12-LABEL: fcmp_f16_eq:
 ; GFX12:       ; %bb.0: ; %entry
@@ -397,30 +479,57 @@ define amdgpu_kernel void @fcmp_f16_le(
 ; VI-NEXT:    buffer_store_dword v0, off, s[4:7], 0
 ; VI-NEXT:    s_endpgm
 ;
-; GFX11-LABEL: fcmp_f16_le:
-; GFX11:       ; %bb.0: ; %entry
-; GFX11-NEXT:    s_clause 0x1
-; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
-; GFX11-NEXT:    s_load_b64 s[4:5], s[4:5], 0x34
-; GFX11-NEXT:    s_mov_b32 s10, -1
-; GFX11-NEXT:    s_mov_b32 s11, 0x31016000
-; GFX11-NEXT:    s_mov_b32 s14, s10
-; GFX11-NEXT:    s_mov_b32 s15, s11
-; GFX11-NEXT:    s_mov_b32 s6, s10
-; GFX11-NEXT:    s_mov_b32 s7, s11
-; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX11-NEXT:    s_mov_b32 s12, s2
-; GFX11-NEXT:    s_mov_b32 s13, s3
-; GFX11-NEXT:    buffer_load_u16 v0, off, s[12:15], 0 glc dlc
-; GFX11-NEXT:    s_waitcnt vmcnt(0)
-; GFX11-NEXT:    buffer_load_u16 v1, off, s[4:7], 0 glc dlc
-; GFX11-NEXT:    s_waitcnt vmcnt(0)
-; GFX11-NEXT:    s_mov_b32 s8, s0
-; GFX11-NEXT:    s_mov_b32 s9, s1
-; GFX11-NEXT:    v_cmp_le_f16_e32 vcc_lo, v0, v1
-; GFX11-NEXT:    v_cndmask_b32_e64 v0, 0, -1, vcc_lo
-; GFX11-NEXT:    buffer_store_b32 v0, off, s[8:11], 0
-; GFX11-NEXT:    s_endpgm
+; GFX11-TRUE16-LABEL: fcmp_f16_le:
+; GFX11-TRUE16:       ; %bb.0: ; %entry
+; GFX11-TRUE16-NEXT:    s_clause 0x1
+; GFX11-TRUE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-TRUE16-NEXT:    s_load_b64 s[4:5], s[4:5], 0x34
+; GFX11-TRUE16-NEXT:    s_mov_b32 s10, -1
+; GFX11-TRUE16-NEXT:    s_mov_b32 s11, 0x31016000
+; GFX11-TRUE16-NEXT:    s_mov_b32 s14, s10
+; GFX11-TRUE16-NEXT:    s_mov_b32 s15, s11
+; GFX11-TRUE16-NEXT:    s_mov_b32 s6, s10
+; GFX11-TRUE16-NEXT:    s_mov_b32 s7, s11
+; GFX11-TRUE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-TRUE16-NEXT:    s_mov_b32 s12, s2
+; GFX11-TRUE16-NEXT:    s_mov_b32 s13, s3
+; GFX11-TRUE16-NEXT:    buffer_load_u16 v0, off, s[12:15], 0 glc dlc
+; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT:    buffer_load_u16 v1, off, s[4:7], 0 glc dlc
+; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT:    s_mov_b32 s8, s0
+; GFX11-TRUE16-NEXT:    s_mov_b32 s9, s1
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v0.h, v1.l
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_cmp_le_f16_e32 vcc_lo, v0.l, v0.h
+; GFX11-TRUE16-NEXT:    v_cndmask_b32_e64 v0, 0, -1, vcc_lo
+; GFX11-TRUE16-NEXT:    buffer_store_b32 v0, off, s[8:11], 0
+; GFX11-TRUE16-NEXT:    s_endpgm
+;
+; GFX11-FAKE16-LABEL: fcmp_f16_le:
+; GFX11-FAKE16:       ; %bb.0: ; %entry
+; GFX11-FAKE16-NEXT:    s_clause 0x1
+; GFX11-FAKE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-FAKE16-NEXT:    s_load_b64 s[4:5], s[4:5], 0x34
+; GFX11-FAKE16-NEXT:    s_mov_b32 s10, -1
+; GFX11-FAKE16-NEXT:    s_mov_b32 s11, 0x31016000
+; GFX11-FAKE16-NEXT:    s_mov_b32 s14, s10
+; GFX11-FAKE16-NEXT:    s_mov_b32 s15, s11
+; GFX11-FAKE16-NEXT:    s_mov_b32 s6, s10
+; GFX11-FAKE16-NEXT:    s_mov_b32 s7, s11
+; GFX11-FAKE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-FAKE16-NEXT:    s_mov_b32 s12, s2
+; GFX11-FAKE16-NEXT:    s_mov_b32 s13, s3
+; GFX11-FAKE16-NEXT:    buffer_load_u16 v0, off, s[12:15], 0 glc dlc
+; GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT:    buffer_load_u16 v1, off, s[4:7], 0 glc dlc
+; GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT:    s_mov_b32 s8, s0
+; GFX11-FAKE16-NEXT:    s_mov_b32 s9, s1
+; GFX11-FAKE16-NEXT:    v_cmp_le_f16_e32 vcc_lo, v0, v1
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e64 v0, 0, -1, vcc_lo
+; GFX11-FAKE16-NEXT:    buffer_store_b32 v0, off, s[8:11], 0
+; GFX11-FAKE16-NEXT:    s_endpgm
 ;
 ; GFX12-LABEL: fcmp_f16_le:
 ; GFX12:       ; %bb.0: ; %entry
@@ -509,30 +618,57 @@ define amdgpu_kernel void @fcmp_f16_gt(
 ; VI-NEXT:    buffer_store_dword v0, off, s[4:7], 0
 ; VI-NEXT:    s_endpgm
 ;
-; GFX11-LABEL: fcmp_f16_gt:
-; GFX11:       ; %bb.0: ; %entry
-; GFX11-NEXT:    s_clause 0x1
-; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
-; GFX11-NEXT:    s_load_b64 s[4:5], s[4:5], 0x34
-; GFX11-NEXT:    s_mov_b32 s10, -1
-; GFX11-NEXT:    s_mov_b32 s11, 0x31016000
-; GFX11-NEXT:    s_mov_b32 s14, s10
-; GFX11-NEXT:    s_mov_b32 s15, s11
-; GFX11-NEXT:    s_mov_b32 s6, s10
-; GFX11-NEXT:    s_mov_b32 s7, s11
-; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX11-NEXT:    s_mov_b32 s12, s2
-; GFX11-NEXT:    s_mov_b32 s13, s3
-; GFX11-NEXT:    buffer_load_u16 v0, off, s[12:15], 0 glc dlc
-; GFX11-NEXT:    s_waitcnt vmcnt(0)
-; GFX11-NEXT:    buffer_load_u16 v1, off, s[4:7], 0 glc dlc
-; GFX11-NEXT:    s_waitcnt vmcnt(0)
-; GFX11-NEXT:    s_mov_b32 s8, s0
-; GFX11-NEXT:    s_mov_b32 s9, s1
-; GFX11-NEXT:    v_cmp_gt_f16_e32 vcc_lo, v0, v1
-; GFX11-NEXT:    v_cndmask_b32_e64 v0, 0, -1, vcc_lo
-; GFX11-NEXT:    buffer_store_b32 v0, off, s[8:11], 0
-; GFX11-NEXT:    s_endpgm
+; GFX11-TRUE16-LABEL: fcmp_f16_gt:
+; GFX11-TRUE16:       ; %bb.0: ; %entry
+; GFX11-TRUE16-NEXT:    s_clause 0x1
+; GFX11-TRUE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-TRUE16-NEXT:    s_load_b64 s[4:5], s[4:5], 0x34
+; GFX11-TRUE16-NEXT:    s_mov_b32 s10, -1
+; GFX11-TRUE16-NEXT:    s_mov_b32 s11, 0x31016000
+; GFX11-TRUE16-NEXT:    s_mov_b32 s14, s10
+; GFX11-TRUE16-NEXT:    s_mov_b32 s15, s11
+; GFX11-TRUE16-NEXT:    s_mov_b32 s6, s10
+; GFX11-TRUE16-NEXT:    s_mov_b32 s7, s11
+; GFX11-TRUE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-TRUE16-NEXT:    s_mov_b32 s12, s2
+; GFX11-TRUE16-NEXT:    s_mov_b32 s13, s3
+; GFX11-TRUE16-NEXT:    buffer_load_u16 v0, off, s[12:15], 0 glc dlc
+; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT:    buffer_load_u16 v1, off, s[4:7], 0 glc dlc
+; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT:    s_mov_b32 s8, s0
+; GFX11-TRUE16-NEXT:    s_mov_b32 s9, s1
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v0.h, v1.l
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_cmp_gt_f16_e32 vcc_lo, v0.l, v0.h
+; GFX11-TRUE16-NEXT:    v_cndmask_b32_e64 v0, 0, -1, vcc_lo
+; GFX11-TRUE16-NEXT:    buffer_store_b32 v0, off, s[8:11], 0
+; GFX11-TRUE16-NEXT:    s_endpgm
+;
+; GFX11-FAKE16-LABEL: fcmp_f16_gt:
+; GFX11-FAKE16:       ; %bb.0: ; %entry
+; GFX11-FAKE16-NEXT:    s_clause 0x1
+; GFX11-FAKE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-FAKE16-NEXT:    s_load_b64 s[4:5], s[4:5], 0x34
+; GFX11-FAKE16-NEXT:    s_mov_b32 s10, -1
+; GFX11-FAKE16-NEXT:    s_mov_b32 s11, 0x31016000
+; GFX11-FAKE16-NEXT:    s_mov_b32 s14, s10
+; GFX11-FAKE16-NEXT:    s_mov_b32 s15, s11
+; GFX11-FAKE16-NEXT:    s_mov_b32 s6, s10
+; GFX11-FAKE16-NEXT:    s_mov_b32 s7, s11
+; GFX11-FAKE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-FAKE16-NEXT:    s_mov_b32 s12, s2
+; GFX11-FAKE16-NEXT:    s_mov_b32 s13, s3
+; GFX11-FAKE16-NEXT:    buffer_load_u16 v0, off, s[12:15], 0 glc dlc
+; GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT:    buffer_load_u16 v1, off, s[4:7], 0 glc dlc
+; GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT:    s_mov_b32 s8, s0
+; GFX11-FAKE16-NEXT:    s_mov_b32 s9, s1
+; GFX11-FAKE16-NEXT:    v_cmp_gt_f16_e32 vcc_lo, v0, v1
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e64 v0, 0, -1, vcc_lo
+; GFX11-FAKE16-NEXT:    buffer_store_b32 v0, off, s[8:11], 0
+; GFX11-FAKE16-NEXT:    s_endpgm
 ;
 ; GFX12-LABEL: fcmp_f16_gt:
 ; GFX12:       ; %bb.0: ; %entry
@@ -621,30 +757,57 @@ define amdgpu_kernel void @fcmp_f16_lg(
 ; VI-NEXT:    buffer_store_dword v0, off, s[4:7], 0
 ; VI-NEXT:    s_endpgm
 ;
-; GFX11-LABEL: fcmp_f16_lg:
-; GFX11:       ; %bb.0: ; %entry
-; GFX11-NEXT:    s_clause 0x1
-; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
-; GFX11-NEXT:    s_load_b64 s[4:5], s[4:5], 0x34
-; GFX11-NEXT:    s_mov_b32 s10, -1
-; GFX11-NEXT:    s_mov_b32 s11, 0x31016000
-; GFX11-NEXT:    s_mov_b32 s14, s10
-; GFX11-NEXT:    s_mov_b32 s15, s11
-; GFX11-NEXT:    s_mov_b32 s6, s10
-; GFX11-NEXT:    s_mov_b32 s7, s11
-; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX11-NEXT:    s_mov_b32 s12, s2
-; GFX11-NEXT:    s_mov_b32 s13, s3
-; GFX11-NEXT:    buffer_load_u16 v0, off, s[12:15], 0 glc dlc
-; GFX11-NEXT:    s_waitcnt vmcnt(0)
-; GFX11-NEXT:    buffer_load_u16 v1, off, s[4:7], 0 glc dlc
-; GFX11-NEXT:    s_waitcnt vmcnt(0)
-; GFX11-NEXT:    s_mov_b32 s8, s0
-; GFX11-NEXT:    s_mov_b32 s9, s1
-; GFX11-NEXT:    v_cmp_lg_f16_e32 vcc_lo, v0, v1
-; GFX11-NEXT:    v_cndmask_b32_e64 v0, 0, -1, vcc_lo
-; GFX11-NEXT:    buffer_store_b32 v0, off, s[8:11], 0
-; GFX11-NEXT:    s_endpgm
+; GFX11-TRUE16-LABEL: fcmp_f16_lg:
+; GFX11-TRUE16:       ; %bb.0: ; %entry
+; GFX11-TRUE16-NEXT:    s_clause 0x1
+; GFX11-TRUE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-TRUE16-NEXT:    s_load_b64 s[4:5], s[4:5], 0x34
+; GFX11-TRUE16-NEXT:    s_mov_b32 s10, -1
+; GFX11-TRUE16-NEXT:    s_mov_b32 s11, 0x31016000
+; GFX11-TRUE16-NEXT:    s_mov_b32 s14, s10
+; GFX11-TRUE16-NEXT:    s_mov_b32 s15, s11
+; GFX11-TRUE16-NEXT:    s_mov_b32 s6, s10
+; GFX11-TRUE16-NEXT:    s_mov_b32 s7, s11
+; GFX11-TRUE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-TRUE16-NEXT:    s_mov_b32 s12, s2
+; GFX11-TRUE16-NEXT:    s_mov_b32 s13, s3
+; GFX11-TRUE16-NEXT:    buffer_load_u16 v0, off, s[12:15], 0 glc dlc
+; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT:    buffer_load_u16 v1, off, s[4:7], 0 glc dlc
+; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT:    s_mov_b32 s8, s0
+; GFX11-TRUE16-NEXT:    s_mov_b32 s9, s1
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v0.h, v1.l
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_cmp_lg_f16_e32 vcc_lo, v0.l, v0.h
+; GFX11-TRUE16-NEXT:    v_cndmask_b32_e64 v0, 0, -1, vcc_lo
+; GFX11-TRUE16-NEXT:    buffer_store_b32 v0, off, s[8:11], 0
+; GFX11-TRUE16-NEXT:    s_endpgm
+;
+; GFX11-FAKE16-LABEL: fcmp_f16_lg:
+; GFX11-FAKE16:       ; %bb.0: ; %entry
+; GFX11-FAKE16-NEXT:    s_clause 0x1
+; GFX11-FAKE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-FAKE16-NEXT:    s_load_b64 s[4:5], s[4:5], 0x34
+; GFX11-FAKE16-NEXT:    s_mov_b32 s10, -1
+; GFX11-FAKE16-NEXT:    s_mov_b32 s11, 0x31016000
+; GFX11-FAKE16-NEXT:    s_mov_b32 s14, s10
+; GFX11-FAKE16-NEXT:    s_mov_b32 s15, s11
+; GFX11-FAKE16-NEXT:    s_mov_b32 s6, s10
+; GFX11-FAKE16-NEXT:    s_mov_b32 s7, s11
+; GFX11-FAKE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-FAKE16-NEXT:    s_mov_b32 s12, s2
+; GFX11-FAKE16-NEXT:    s_mov_b32 s13, s3
+; GFX11-FAKE16-NEXT:    buffer_load_u16 v0, off, s[12:15], 0 glc dlc
+; GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT:    buffer_load_u16 v1, off, s[4:7], 0 glc dlc
+; GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT:    s_mov_b32 s8, s0
+; GFX11-FAKE16-NEXT:    s_mov_b32 s9, s1
+; GFX11-FAKE16-NEXT:    v_cmp_lg_f16_e32 vcc_lo, v0, v1
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e64 v0, 0, -1, vcc_lo
+; GFX11-FAKE16-NEXT:    buffer_store_b32 v0, off, s[8:11], 0
+; GFX11-FAKE16-NEXT:    s_endpgm
 ;
 ; GFX12-LABEL: fcmp_f16_lg:
 ; GFX12:       ; %bb.0: ; %entry
@@ -733,30 +896,57 @@ define amdgpu_kernel void @fcmp_f16_ge(
 ; VI-NEXT:    buffer_store_dword v0, off, s[4:7], 0
 ; VI-NEXT:    s_endpgm
 ;
-; GFX11-LABEL: fcmp_f16_ge:
-; GFX11:       ; %bb.0: ; %entry
-; GFX11-NEXT:    s_clause 0x1
-; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
-; GFX11-NEXT:    s_load_b64 s[4:5], s[4:5], 0x34
-; GFX11-NEXT:    s_mov_b32 s10, -1
-; GFX11-NEXT:    s_mov_b32 s11, 0x31016000
-; GFX11-NEXT:    s_mov_b32 s14, s10
-; GFX11-NEXT:    s_mov_b32 s15, s11
-; GFX11-NEXT:    s_mov_b32 s6, s10
-; GFX11-NEXT:    s_mov_b32 s7, s11
-; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX11-NEXT:    s_mov_b32 s12, s2
-; GFX11-NEXT:    s_mov_b32 s13, s3
-; GFX11-NEXT:    buffer_load_u16 v0, off, s[12:15], 0 glc dlc
-; GFX11-NEXT:    s_waitcnt vmcnt(0)
-; GFX11-NEXT:    buffer_load_u16 v1, off, s[4:7], 0 glc dlc
-; GFX11-NEXT:    s_waitcnt vmcnt(0)
-; GFX11-NEXT:    s_mov_b32 s8, s0
-; GFX11-NEXT:    s_mov_b32 s9, s1
-; GFX11-NEXT:    v_cmp_ge_f16_e32 vcc_lo, v0, v1
-; GFX11-NEXT:    v_cndmask_b32_e64 v0, 0, -1, vcc_lo
-; GFX11-NEXT:    buffer_store_b32 v0, off, s[8:11], 0
-; GFX11-NEXT:    s_endpgm
+; GFX11-TRUE16-LABEL: fcmp_f16_ge:
+; GFX11-TRUE16:       ; %bb.0: ; %entry
+; GFX11-TRUE16-NEXT:    s_clause 0x1
+; GFX11-TRUE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-TRUE16-NEXT:    s_load_b64 s[4:5], s[4:5], 0x34
+; GFX11-TRUE16-NEXT:    s_mov_b32 s10, -1
+; GFX11-TRUE16-NEXT:    s_mov_b32 s11, 0x31016000
+; GFX11-TRUE16-NEXT:    s_mov_b32 s14, s10
+; GFX11-TRUE16-NEXT:    s_mov_b32 s15, s11
+; GFX11-TRUE16-NEXT:    s_mov_b32 s6, s10
+; GFX11-TRUE16-NEXT:    s_mov_b32 s7, s11
+; GFX11-TRUE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-TRUE16-NEXT:    s_mov_b32 s12, s2
+; GFX11-TRUE16-NEXT:    s_mov_b32 s13, s3
+; GFX11-TRUE16-NEXT:    buffer_load_u16 v0, off, s[12:15], 0 glc dlc
+; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT:    buffer_load_u16 v1, off, s[4:7], 0 glc dlc
+; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT:    s_mov_b32 s8, s0
+; GFX11-TRUE16-NEXT:    s_mov_b32 s9, s1
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v0.h, v1.l
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_cmp_ge_f16_e32 vcc_lo, v0.l, v0.h
+; GFX11-TRUE16-NEXT:    v_cndmask_b32_e64 v0, 0, -1, vcc_lo
+; GFX11-TRUE16-NEXT:    buffer_store_b32 v0, off, s[8:11], 0
+; GFX11-TRUE16-NEXT:    s_endpgm
+;
+; GFX11-FAKE16-LABEL: fcmp_f16_ge:
+; GFX11-FAKE16:       ; %bb.0: ; %entry
+; GFX11-FAKE16-NEXT:    s_clause 0x1
+; GFX11-FAKE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-FAKE16-NEXT:    s_load_b64 s[4:5], s[4:5], 0x34
+; GFX11-FAKE16-NEXT:    s_mov_b32 s10, -1
+; GFX11-FAKE16-NEXT:    s_mov_b32 s11, 0x31016000
+; GFX11-FAKE16-NEXT:    s_mov_b32 s14, s10
+; GFX11-FAKE16-NEXT:    s_mov_b32 s15, s11
+; GFX11-FAKE16-NEXT:    s_mov_b32 s6, s10
+; GFX11-FAKE16-NEXT:    s_mov_b32 s7, s11
+; GFX11-FAKE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-FAKE16-NEXT:    s_mov_b32 s12, s2
+; GFX11-FAKE16-NEXT:    s_mov_b32 s13, s3
+; GFX11-FAKE16-NEXT:    buffer_load_u16 v0, off, s[12:15], 0 glc dlc
+; GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT:    buffer_load_u16 v1, off, s[4:7], 0 glc dlc
+; GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT:    s_mov_b32 s8, s0
+; GFX11-FAKE16-NEXT:    s_mov_b32 s9, s1
+; GFX11-FAKE16-NEXT:    v_cmp_ge_f16_e32 vcc_lo, v0, v1
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e64 v0, 0, -1, vcc_lo
+; GFX11-FAKE16-NEXT:    buffer_store_b32 v0, off, s[8:11], 0
+; GFX11-FAKE16-NEXT:    s_endpgm
 ;
 ; GFX12-LABEL: fcmp_f16_ge:
 ; GFX12:       ; %bb.0: ; %entry
@@ -845,30 +1035,57 @@ define amdgpu_kernel void @fcmp_f16_o(
 ; VI-NEXT:    buffer_store_dword v0, off, s[4:7], 0
 ; VI-NEXT:    s_endpgm
 ;
-; GFX11-LABEL: fcmp_f16_o:
-; GFX11:       ; %bb.0: ; %entry
-; GFX11-NEXT:    s_clause 0x1
-; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
-; GFX11-NEXT:    s_load_b64 s[4:5], s[4:5], 0x34
-; GFX11-NEXT:    s_mov_b32 s10, -1
-; GFX11-NEXT:    s_mov_b32 s11, 0x31016000
-; GFX11-NEXT:    s_mov_b32 s14, s10
-; GFX11-NEXT:    s_mov_b32 s15, s11
-; GFX11-NEXT:    s_mov_b32 s6, s10
-; GFX11-NEXT:    s_mov_b32 s7, s11
-; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX11-NEXT:    s_mov_b32 s12, s2
-; GFX11-NEXT:    s_mov_b32 s13, s3
-; GFX11-NEXT:    buffer_load_u16 v0, off, s[12:15], 0 glc dlc
-; GFX11-NEXT:    s_waitcnt vmcnt(0)
-; GFX11-NEXT:    buffer_load_u16 v1, off, s[4:7], 0 glc dlc
-; GFX11-NEXT:    s_waitcnt vmcnt(0)
-; GFX11-NEXT:    s_mov_b32 s8, s0
-; GFX11-NEXT:    s_mov_b32 s9, s1
-; GFX11-NEXT:    v_cmp_o_f16_e32 vcc_lo, v0, v1
-; GFX11-NEXT:    v_cndmask_b32_e64 v0, 0, -1, vcc_lo
-; GFX11-NEXT:    buffer_store_b32 v0, off, s[8:11], 0
-; GFX11-NEXT:    s_endpgm
+; GFX11-TRUE16-LABEL: fcmp_f16_o:
+; GFX11-TRUE16:       ; %bb.0: ; %entry
+; GFX11-TRUE16-NEXT:    s_clause 0x1
+; GFX11-TRUE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-TRUE16-NEXT:    s_load_b64 s[4:5], s[4:5], 0x34
+; GFX11-TRUE16-NEXT:    s_mov_b32 s10, -1
+; GFX11-TRUE16-NEXT:    s_mov_b32 s11, 0x31016000
+; GFX11-TRUE16-NEXT:    s_mov_b32 s14, s10
+; GFX11-TRUE16-NEXT:    s_mov_b32 s15, s11
+; GFX11-TRUE16-NEXT:    s_mov_b32 s6, s10
+; GFX11-TRUE16-NEXT:    s_mov_b32 s7, s11
+; GFX11-TRUE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-TRUE16-NEXT:    s_mov_b32 s12, s2
+; GFX11-TRUE16-NEXT:    s_mov_b32 s13, s3
+; GFX11-TRUE16-NEXT:    buffer_load_u16 v0, off, s[12:15], 0 glc dlc
+; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT:    buffer_load_u16 v1, off, s[4:7], 0 glc dlc
+; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT:    s_mov_b32 s8, s0
+; GFX11-TRUE16-NEXT:    s_mov_b32 s9, s1
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v0.h, v1.l
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_cmp_o_f16_e32 vcc_lo, v0.l, v0.h
+; GFX11-TRUE16-NEXT:    v_cndmask_b32_e64 v0, 0, -1, vcc_lo
+; GFX11-TRUE16-NEXT:    buffer_store_b32 v0, off, s[8:11], 0
+; GFX11-TRUE16-NEXT:    s_endpgm
+;
+; GFX11-FAKE16-LABEL: fcmp_f16_o:
+; GFX11-FAKE16:       ; %bb.0: ; %entry
+; GFX11-FAKE16-NEXT:    s_clause 0x1
+; GFX11-FAKE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-FAKE16-NEXT:    s_load_b64 s[4:5], s[4:5], 0x34
+; GFX11-FAKE16-NEXT:    s_mov_b32 s10, -1
+; GFX11-FAKE16-NEXT:    s_mov_b32 s11, 0x31016000
+; GFX11-FAKE16-NEXT:    s_mov_b32 s14, s10
+; GFX11-FAKE16-NEXT:    s_mov_b32 s15, s11
+; GFX11-FAKE16-NEXT:    s_mov_b32 s6, s10
+; GFX11-FAKE16-NEXT:    s_mov_b32 s7, s11
+; GFX11-FAKE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-FAKE16-NEXT:    s_mov_b32 s12, s2
+; GFX11-FAKE16-NEXT:    s_mov_b32 s13, s3
+; GFX11-FAKE16-NEXT:    buffer_load_u16 v0, off, s[12:15], 0 glc dlc
+; GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT:    buffer_load_u16 v1, off, s[4:7], 0 glc dlc
+; GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT:    s_mov_b32 s8, s0
+; GFX11-FAKE16-NEXT:    s_mov_b32 s9, s1
+; GFX11-FAKE16-NEXT:    v_cmp_o_f16_e32 vcc_lo, v0, v1
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e64 v0, 0, -1, vcc_lo
+; GFX11-FAKE16-NEXT:    buffer_store_b32 v0, off, s[8:11], 0
+; GFX11-FAKE16-NEXT:    s_endpgm
 ;
 ; GFX12-LABEL: fcmp_f16_o:
 ; GFX12:       ; %bb.0: ; %entry
@@ -957,30 +1174,57 @@ define amdgpu_kernel void @fcmp_f16_u(
 ; VI-NEXT:    buffer_store_dword v0, off, s[4:7], 0
 ; VI-NEXT:    s_endpgm
 ;
-; GFX11-LABEL: fcmp_f16_u:
-; GFX11:       ; %bb.0: ; %entry
-; GFX11-NEXT:    s_clause 0x1
-; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
-; GFX11-NEXT:    s_load_b64 s[4:5], s[4:5], 0x34
-; GFX11-NEXT:    s_mov_b32 s10, -1
-; GFX11-NEXT:    s_mov_b32 s11, 0x31016000
-; GFX11-NEXT:    s_mov_b32 s14, s10
-; GFX11-NEXT:    s_mov_b32 s15, s11
-; GFX11-NEXT:    s_mov_b32 s6, s10
-; GFX11-NEXT:    s_mov_b32 s7, s11
-; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX11-NEXT:    s_mov_b32 s12, s2
-; GFX11-NEXT:    s_mov_b32 s13, s3
-; GFX11-NEXT:    buffer_load_u16 v0, off, s[12:15], 0 glc dlc
-; GFX11-NEXT:    s_waitcnt vmcnt(0)
-; GFX11-NEXT:    buffer_load_u16 v1, off, s[4:7], 0 glc dlc
-; GFX11-NEXT:    s_waitcnt vmcnt(0)
-; GFX11-NEXT:    s_mov_b32 s8, s0
-; GFX11-NEXT:    s_mov_b32 s9, s1
-; GFX11-NEXT:    v_cmp_u_f16_e32 vcc_lo, v0, v1
-; GFX11-NEXT:    v_cndmask_b32_e64 v0, 0, -1, vcc_lo
-; GFX11-NEXT:    buffer_store_b32 v0, off, s[8:11], 0
-; GFX11-NEXT:    s_endpgm
+; GFX11-TRUE16-LABEL: fcmp_f16_u:
+; GFX11-TRUE16:       ; %bb.0: ; %entry
+; GFX11-TRUE16-NEXT:    s_clause 0x1
+; GFX11-TRUE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-TRUE16-NEXT:    s_load_b64 s[4:5], s[4:5], 0x34
+; GFX11-TRUE16-NEXT:    s_mov_b32 s10, -1
+; GFX11-TRUE16-NEXT:    s_mov_b32 s11, 0x31016000
+; GFX11-TRUE16-NEXT:    s_mov_b32 s14, s10
+; GFX11-TRUE16-NEXT:    s_mov_b32 s15, s11
+; GFX11-TRUE16-NEXT:    s_mov_b32 s6, s10
+; GFX11-TRUE16-NEXT:    s_mov_b32 s7, s11
+; GFX11-TRUE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-TRUE16-NEXT:    s_mov_b32 s12, s2
+; GFX11-TRUE16-NEXT:    s_mov_b32 s13, s3
+; GFX11-TRUE16-NEXT:    buffer_load_u16 v0, off, s[12:15], 0 glc dlc
+; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT:    buffer_load_u16 v1, off, s[4:7], 0 glc dlc
+; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT:    s_mov_b32 s8, s0
+; GFX11-TRUE16-NEXT:    s_mov_b32 s9, s1
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v0.h, v1.l
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_cmp_u_f16_e32 vcc_lo, v0.l, v0.h
+; GFX11-TRUE16-NEXT:    v_cndmask_b32_e64 v0, 0, -1, vcc_lo
+; GFX11-TRUE16-NEXT:    buffer_store_b32 v0, off, s[8:11], 0
+; GFX11-TRUE16-NEXT:    s_endpgm
+;
+; GFX11-FAKE16-LABEL: fcmp_f16_u:
+; GFX11-FAKE16:       ; %bb.0: ; %entry
+; GFX11-FAKE16-NEXT:    s_clause 0x1
+; GFX11-FAKE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-FAKE16-NEXT:    s_load_b64 s[4:5], s[4:5], 0x34
+; GFX11-FAKE16-NEXT:    s_mov_b32 s10, -1
+; GFX11-FAKE16-NEXT:    s_mov_b32 s11, 0x31016000
+; GFX11-FAKE16-NEXT:    s_mov_b32 s14, s10
+; GFX11-FAKE16-NEXT:    s_mov_b32 s15, s11
+; GFX11-FAKE16-NEXT:    s_mov_b32 s6, s10
+; GFX11-FAKE16-NEXT:    s_mov_b32 s7, s11
+; GFX11-FAKE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-FAKE16-NEXT:    s_mov_b32 s12, s2
+; GFX11-FAKE16-NEXT:    s_mov_b32 s13, s3
+; GFX11-FAKE16-NEXT:    buffer_load_u16 v0, off, s[12:15], 0 glc dlc
+; GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT:    buffer_load_u16 v1, off, s[4:7], 0 glc dlc
+; GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT:    s_mov_b32 s8, s0
+; GFX11-FAKE16-NEXT:    s_mov_b32 s9, s1
+; GFX11-FAKE16-NEXT:    v_cmp_u_f16_e32 vcc_lo, v0, v1
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e64 v0, 0, -1, vcc_lo
+; GFX11-FAKE16-NEXT:    buffer_store_b32 v0, off, s[8:11], 0
+; GFX11-FAKE16-NEXT:    s_endpgm
 ;
 ; GFX12-LABEL: fcmp_f16_u:
 ; GFX12:       ; %bb.0: ; %entry
@@ -1069,30 +1313,57 @@ define amdgpu_kernel void @fcmp_f16_nge(
 ; VI-NEXT:    buffer_store_dword v0, off, s[4:7], 0
 ; VI-NEXT:    s_endpgm
 ;
-; GFX11-LABEL: fcmp_f16_nge:
-; GFX11:       ; %bb.0: ; %entry
-; GFX11-NEXT:    s_clause 0x1
-; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
-; GFX11-NEXT:    s_load_b64 s[4:5], s[4:5], 0x34
-; GFX11-NEXT:    s_mov_b32 s10, -1
-; GFX11-NEXT:    s_mov_b32 s11, 0x31016000
-; GFX11-NEXT:    s_mov_b32 s14, s10
-; GFX11-NEXT:    s_mov_b32 s15, s11
-; GFX11-NEXT:    s_mov_b32 s6, s10
-; GFX11-NEXT:    s_mov_b32 s7, s11
-; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX11-NEXT:    s_mov_b32 s12, s2
-; GFX11-NEXT:    s_mov_b32 s13, s3
-; GFX11-NEXT:    buffer_load_u16 v0, off, s[12:15], 0 glc dlc
-; GFX11-NEXT:    s_waitcnt vmcnt(0)
-; GFX11-NEXT:    buffer_load_u16 v1, off, s[4:7], 0 glc dlc
-; GFX11-NEXT:    s_waitcnt vmcnt(0)
-; GFX11-NEXT:    s_mov_b32 s8, s0
-; GFX11-NEXT:    s_mov_b32 s9, s1
-; GFX11-NEXT:    v_cmp_nge_f16_e32 vcc_lo, v0, v1
-; GFX11-NEXT:    v_cndmask_b32_e64 v0, 0, -1, vcc_lo
-; GFX11-NEXT:    buffer_store_b32 v0, off, s[8:11], 0
-; GFX11-NEXT:    s_endpgm
+; GFX11-TRUE16-LABEL: fcmp_f16_nge:
+; GFX11-TRUE16:       ; %bb.0: ; %entry
+; GFX11-TRUE16-NEXT:    s_clause 0x1
+; GFX11-TRUE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-TRUE16-NEXT:    s_load_b64 s[4:5], s[4:5], 0x34
+; GFX11-TRUE16-NEXT:    s_mov_b32 s10, -1
+; GFX11-TRUE16-NEXT:    s_mov_b32 s11, 0x31016000
+; GFX11-TRUE16-NEXT:    s_mov_b32 s14, s10
+; GFX11-TRUE16-NEXT:    s_mov_b32 s15, s11
+; GFX11-TRUE16-NEXT:    s_mov_b32 s6, s10
+; GFX11-TRUE16-NEXT:    s_mov_b32 s7, s11
+; GFX11-TRUE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-TRUE16-NEXT:    s_mov_b32 s12, s2
+; GFX11-TRUE16-NEXT:    s_mov_b32 s13, s3
+; GFX11-TRUE16-NEXT:    buffer_load_u16 v0, off, s[12:15], 0 glc dlc
+; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT:    buffer_load_u16 v1, off, s[4:7], 0 glc dlc
+; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT:    s_mov_b32 s8, s0
+; GFX11-TRUE16-NEXT:    s_mov_b32 s9, s1
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v0.h, v1.l
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_cmp_nge_f16_e32 vcc_lo, v0.l, v0.h
+; GFX11-TRUE16-NEXT:    v_cndmask_b32_e64 v0, 0, -1, vcc_lo
+; GFX11-TRUE16-NEXT:    buffer_store_b32 v0, off, s[8:11], 0
+; GFX11-TRUE16-NEXT:    s_endpgm
+;
+; GFX11-FAKE16-LABEL: fcmp_f16_nge:
+; GFX11-FAKE16:       ; %bb.0: ; %entry
+; GFX11-FAKE16-NEXT:    s_clause 0x1
+; GFX11-FAKE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-FAKE16-NEXT:    s_load_b64 s[4:5], s[4:5], 0x34
+; GFX11-FAKE16-NEXT:    s_mov_b32 s10, -1
+; GFX11-FAKE16-NEXT:    s_mov_b32 s11, 0x31016000
+; GFX11-FAKE16-NEXT:    s_mov_b32 s14, s10
+; GFX11-FAKE16-NEXT:    s_mov_b32 s15, s11
+; GFX11-FAKE16-NEXT:    s_mov_b32 s6, s10
+; GFX11-FAKE16-NEXT:    s_mov_b32 s7, s11
+; GFX11-FAKE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-FAKE16-NEXT:    s_mov_b32 s12, s2
+; GFX11-FAKE16-NEXT:    s_mov_b32 s13, s3
+; GFX11-FAKE16-NEXT:    buffer_load_u16 v0, off, s[12:15], 0 glc dlc
+; GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT:    buffer_load_u16 v1, off, s[4:7], 0 glc dlc
+; GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT:    s_mov_b32 s8, s0
+; GFX11-FAKE16-NEXT:    s_mov_b32 s9, s1
+; GFX11-FAKE16-NEXT:    v_cmp_nge_f16_e32 vcc_lo, v0, v1
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e64 v0, 0, -1, vcc_lo
+; GFX11-FAKE16-NEXT:    buffer_store_b32 v0, off, s[8:11], 0
+; GFX11-FAKE16-NEXT:    s_endpgm
 ;
 ; GFX12-LABEL: fcmp_f16_nge:
 ; GFX12:       ; %bb.0: ; %entry
@@ -1181,30 +1452,57 @@ define amdgpu_kernel void @fcmp_f16_nlg(
 ; VI-NEXT:    buffer_store_dword v0, off, s[4:7], 0
 ; VI-NEXT:    s_endpgm
 ;
-; GFX11-LABEL: fcmp_f16_nlg:
-; GFX11:       ; %bb.0: ; %entry
-; GFX11-NEXT:    s_clause 0x1
-; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
-; GFX11-NEXT:    s_load_b64 s[4:5], s[4:5], 0x34
-; GFX11-NEXT:    s_mov_b32 s10, -1
-; GFX11-NEXT:    s_mov_b32 s11, 0x31016000
-; GFX11-NEXT:    s_mov_b32 s14, s10
-; GFX11-NEXT:    s_mov_b32 s15, s11
-; GFX11-NEXT:    s_mov_b32 s6, s10
-; GFX11-NEXT:    s_mov_b32 s7, s11
-; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX11-NEXT:    s_mov_b32 s12, s2
-; GFX11-NEXT:    s_mov_b32 s13, s3
-; GFX11-NEXT:    buffer_load_u16 v0, off, s[12:15], 0 glc dlc
-; GFX11-NEXT:    s_waitcnt vmcnt(0)
-; GFX11-NEXT:    buffer_load_u16 v1, off, s[4:7], 0 glc dlc
-; GFX11-NEXT:    s_waitcnt vmcnt(0)
-; GFX11-NEXT:    s_mov_b32 s8, s0
-; GFX11-NEXT:    s_mov_b32 s9, s1
-; GFX11-NEXT:    v_cmp_nlg_f16_e32 vcc_lo, v0, v1
-; GFX11-NEXT:    v_cndmask_b32_e64 v0, 0, -1, vcc_lo
-; GFX11-NEXT:    buffer_store_b32 v0, off, s[8:11], 0
-; GFX11-NEXT:    s_endpgm
+; GFX11-TRUE16-LABEL: fcmp_f16_nlg:
+; GFX11-TRUE16:       ; %bb.0: ; %entry
+; GFX11-TRUE16-NEXT:    s_clause 0x1
+; GFX11-TRUE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-TRUE16-NEXT:    s_load_b64 s[4:5], s[4:5], 0x34
+; GFX11-TRUE16-NEXT:    s_mov_b32 s10, -1
+; GFX11-TRUE16-NEXT:    s_mov_b32 s11, 0x31016000
+; GFX11-TRUE16-NEXT:    s_mov_b32 s14, s10
+; GFX11-TRUE16-NEXT:    s_mov_b32 s15, s11
+; GFX11-TRUE16-NEXT:    s_mov_b32 s6, s10
+; GFX11-TRUE16-NEXT:    s_mov_b32 s7, s11
+; GFX11-TRUE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-TRUE16-NEXT:    s_mov_b32 s12, s2
+; GFX11-TRUE16-NEXT:    s_mov_b32 s13, s3
+; GFX11-TRUE16-NEXT:    buffer_load_u16 v0, off, s[12:15], 0 glc dlc
+; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT:    buffer_load_u16 v1, off, s[4:7], 0 glc dlc
+; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT:    s_mov_b32 s8, s0
+; GFX11-TRUE16-NEXT:    s_mov_b32 s9, s1
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v0.h, v1.l
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_cmp_nlg_f16_e32 vcc_lo, v0.l, v0.h
+; GFX11-TRUE16-NEXT:    v_cndmask_b32_e64 v0, 0, -1, vcc_lo
+; GFX11-TRUE16-NEXT:    buffer_store_b32 v0, off, s[8:11], 0
+; GFX11-TRUE16-NEXT:    s_endpgm
+;
+; GFX11-FAKE16-LABEL: fcmp_f16_nlg:
+; GFX11-FAKE16:       ; %bb.0: ; %entry
+; GFX11-FAKE16-NEXT:    s_clause 0x1
+; GFX11-FAKE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-FAKE16-NEXT:    s_load_b64 s[4:5], s[4:5], 0x34
+; GFX11-FAKE16-NEXT:    s_mov_b32 s10, -1
+; GFX11-FAKE16-NEXT:    s_mov_b32 s11, 0x31016000
+; GFX11-FAKE16-NEXT:    s_mov_b32 s14, s10
+; GFX11-FAKE16-NEXT:    s_mov_b32 s15, s11
+; GFX11-FAKE16-NEXT:    s_mov_b32 s6, s10
+; GFX11-FAKE16-NEXT:    s_mov_b32 s7, s11
+; GFX11-FAKE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-FAKE16-NEXT:    s_mov_b32 s12, s2
+; GFX11-FAKE16-NEXT:    s_mov_b32 s13, s3
+; GFX11-FAKE16-NEXT:    buffer_load_u16 v0, off, s[12:15], 0 glc dlc
+; GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT:    buffer_load_u16 v1, off, s[4:7], 0 glc dlc
+; GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT:    s_mov_b32 s8, s0
+; GFX11-FAKE16-NEXT:    s_mov_b32 s9, s1
+; GFX11-FAKE16-NEXT:    v_cmp_nlg_f16_e32 vcc_lo, v0, v1
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e64 v0, 0, -1, vcc_lo
+; GFX11-FAKE16-NEXT:    buffer_store_b32 v0, off, s[8:11], 0
+; GFX11-FAKE16-NEXT:    s_endpgm
 ;
 ; GFX12-LABEL: fcmp_f16_nlg:
 ; GFX12:       ; %bb.0: ; %entry
@@ -1293,30 +1591,57 @@ define amdgpu_kernel void @fcmp_f16_ngt(
 ; VI-NEXT:    buffer_store_dword v0, off, s[4:7], 0
 ; VI-NEXT:    s_endpgm
 ;
-; GFX11-LABEL: fcmp_f16_ngt:
-; GFX11:       ; %bb.0: ; %entry
-; GFX11-NEXT:    s_clause 0x1
-; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
-; GFX11-NEXT:    s_load_b64 s[4:5], s[4:5], 0x34
-; GFX11-NEXT:    s_mov_b32 s10, -1
-; GFX11-NEXT:    s_mov_b32 s11, 0x31016000
-; GFX11-NEXT:    s_mov_b32 s14, s10
-; GFX11-NEXT:    s_mov_b32 s15, s11
-; GFX11-NEXT:    s_mov_b32 s6, s10
-; GFX11-NEXT:    s_mov_b32 s7, s11
-; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX11-NEXT:    s_mov_b32 s12, s2
-; GFX11-NEXT:    s_mov_b32 s13, s3
-; GFX11-NEXT:    buffer_load_u16 v0, off, s[12:15], 0 glc dlc
-; GFX11-NEXT:    s_waitcnt vmcnt(0)
-; GFX11-NEXT:    buffer_load_u16 v1, off, s[4:7], 0 glc dlc
-; GFX11-NEXT:    s_waitcnt vmcnt(0)
-; GFX11-NEXT:    s_mov_b32 s8, s0
-; GFX11-NEXT:    s_mov_b32 s9, s1
-; GFX11-NEXT:    v_cmp_ngt_f16_e32 vcc_lo, v0, v1
-; GFX11-NEXT:    v_cndmask_b32_e64 v0, 0, -1, vcc_lo
-; GFX11-NEXT:    buffer_store_b32 v0, off, s[8:11], 0
-; GFX11-NEXT:    s_endpgm
+; GFX11-TRUE16-LABEL: fcmp_f16_ngt:
+; GFX11-TRUE16:       ; %bb.0: ; %entry
+; GFX11-TRUE16-NEXT:    s_clause 0x1
+; GFX11-TRUE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-TRUE16-NEXT:    s_load_b64 s[4:5], s[4:5], 0x34
+; GFX11-TRUE16-NEXT:    s_mov_b32 s10, -1
+; GFX11-TRUE16-NEXT:    s_mov_b32 s11, 0x31016000
+; GFX11-TRUE16-NEXT:    s_mov_b32 s14, s10
+; GFX11-TRUE16-NEXT:    s_mov_b32 s15, s11
+; GFX11-TRUE16-NEXT:    s_mov_b32 s6, s10
+; GFX11-TRUE16-NEXT:    s_mov_b32 s7, s11
+; GFX11-TRUE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-TRUE16-NEXT:    s_mov_b32 s12, s2
+; GFX11-TRUE16-NEXT:    s_mov_b32 s13, s3
+; GFX11-TRUE16-NEXT:    buffer_load_u16 v0, off, s[12:15], 0 glc dlc
+; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT:    buffer_load_u16 v1, off, s[4:7], 0 glc dlc
+; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT:    s_mov_b32 s8, s0
+; GFX11-TRUE16-NEXT:    s_mov_b32 s9, s1
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v0.h, v1.l
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_cmp_ngt_f16_e32 vcc_lo, v0.l, v0.h
+; GFX11-TRUE16-NEXT:    v_cndmask_b32_e64 v0, 0, -1, vcc_lo
+; GFX11-TRUE16-NEXT:    buffer_store_b32 v0, off, s[8:11], 0
+; GFX11-TRUE16-NEXT:    s_endpgm
+;
+; GFX11-FAKE16-LABEL: fcmp_f16_ngt:
+; GFX11-FAKE16:       ; %bb.0: ; %entry
+; GFX11-FAKE16-NEXT:    s_clause 0x1
+; GFX11-FAKE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-FAKE16-NEXT:    s_load_b64 s[4:5], s[4:5], 0x34
+; GFX11-FAKE16-NEXT:    s_mov_b32 s10, -1
+; GFX11-FAKE16-NEXT:    s_mov_b32 s11, 0x31016000
+; GFX11-FAKE16-NEXT:    s_mov_b32 s14, s10
+; GFX11-FAKE16-NEXT:    s_mov_b32 s15, s11
+; GFX11-FAKE16-NEXT:    s_mov_b32 s6, s10
+; GFX11-FAKE16-NEXT:    s_mov_b32 s7, s11
+; GFX11-FAKE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-FAKE16-NEXT:    s_mov_b32 s12, s2
+; GFX11-FAKE16-NEXT:    s_mov_b32 s13, s3
+; GFX11-FAKE16-NEXT:    buffer_load_u16 v0, off, s[12:15], 0 glc dlc
+; GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT:    buffer_load_u16 v1, off, s[4:7], 0 glc dlc
+; GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT:    s_mov_b32 s8, s0
+; GFX11-FAKE16-NEXT:    s_mov_b32 s9, s1
+; GFX11-FAKE16-NEXT:    v_cmp_ngt_f16_e32 vcc_lo, v0, v1
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e64 v0, 0, -1, vcc_lo
+; GFX11-FAKE16-NEXT:    buffer_store_b32 v0, off, s[8:11], 0
+; GFX11-FAKE16-NEXT:    s_endpgm
 ;
 ; GFX12-LABEL: fcmp_f16_ngt:
 ; GFX12:       ; %bb.0: ; %entry
@@ -1405,30 +1730,57 @@ define amdgpu_kernel void @fcmp_f16_nle(
 ; VI-NEXT:    buffer_store_dword v0, off, s[4:7], 0
 ; VI-NEXT:    s_endpgm
 ;
-; GFX11-LABEL: fcmp_f16_nle:
-; GFX11:       ; %bb.0: ; %entry
-; GFX11-NEXT:    s_clause 0x1
-; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
-; GFX11-NEXT:    s_load_b64 s[4:5], s[4:5], 0x34
-; GFX11-NEXT:    s_mov_b32 s10, -1
-; GFX11-NEXT:    s_mov_b32 s11, 0x31016000
-; GFX11-NEXT:    s_mov_b32 s14, s10
-; GFX11-NEXT:    s_mov_b32 s15, s11
-; GFX11-NEXT:    s_mov_b32 s6, s10
-; GFX11-NEXT:    s_mov_b32 s7, s11
-; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX11-NEXT:    s_mov_b32 s12, s2
-; GFX11-NEXT:    s_mov_b32 s13, s3
-; GFX11-NEXT:    buffer_load_u16 v0, off, s[12:15], 0 glc dlc
-; GFX11-NEXT:    s_waitcnt vmcnt(0)
-; GFX11-NEXT:    buffer_load_u16 v1, off, s[4:7], 0 glc dlc
-; GFX11-NEXT:    s_waitcnt vmcnt(0)
-; GFX11-NEXT:    s_mov_b32 s8, s0
-; GFX11-NEXT:    s_mov_b32 s9, s1
-; GFX11-NEXT:    v_cmp_nle_f16_e32 vcc_lo, v0, v1
-; GFX11-NEXT:    v_cndmask_b32_e64 v0, 0, -1, vcc_lo
-; GFX11-NEXT:    buffer_store_b32 v0, off, s[8:11], 0
-; GFX11-NEXT:    s_endpgm
+; GFX11-TRUE16-LABEL: fcmp_f16_nle:
+; GFX11-TRUE16:       ; %bb.0: ; %entry
+; GFX11-TRUE16-NEXT:    s_clause 0x1
+; GFX11-TRUE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-TRUE16-NEXT:    s_load_b64 s[4:5], s[4:5], 0x34
+; GFX11-TRUE16-NEXT:    s_mov_b32 s10, -1
+; GFX11-TRUE16-NEXT:    s_mov_b32 s11, 0x31016000
+; GFX11-TRUE16-NEXT:    s_mov_b32 s14, s10
+; GFX11-TRUE16-NEXT:    s_mov_b32 s15, s11
+; GFX11-TRUE16-NEXT:    s_mov_b32 s6, s10
+; GFX11-TRUE16-NEXT:    s_mov_b32 s7, s11
+; GFX11-TRUE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-TRUE16-NEXT:    s_mov_b32 s12, s2
+; GFX11-TRUE16-NEXT:    s_mov_b32 s13, s3
+; GFX11-TRUE16-NEXT:    buffer_load_u16 v0, off, s[12:15], 0 glc dlc
+; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT:    buffer_load_u16 v1, off, s[4:7], 0 glc dlc
+; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT:    s_mov_b32 s8, s0
+; GFX11-TRUE16-NEXT:    s_mov_b32 s9, s1
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v0.h, v1.l
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_cmp_nle_f16_e32 vcc_lo, v0.l, v0.h
+; GFX11-TRUE16-NEXT:    v_cndmask_b32_e64 v0, 0, -1, vcc_lo
+; GFX11-TRUE16-NEXT:    buffer_store_b32 v0, off, s[8:11], 0
+; GFX11-TRUE16-NEXT:    s_endpgm
+;
+; GFX11-FAKE16-LABEL: fcmp_f16_nle:
+; GFX11-FAKE16:       ; %bb.0: ; %entry
+; GFX11-FAKE16-NEXT:    s_clause 0x1
+; GFX11-FAKE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-FAKE16-NEXT:    s_load_b64 s[4:5], s[4:5], 0x34
+; GFX11-FAKE16-NEXT:    s_mov_b32 s10, -1
+; GFX11-FAKE16-NEXT:    s_mov_b32 s11, 0x31016000
+; GFX11-FAKE16-NEXT:    s_mov_b32 s14, s10
+; GFX11-FAKE16-NEXT:    s_mov_b32 s15, s11
+; GFX11-FAKE16-NEXT:    s_mov_b32 s6, s10
+; GFX11-FAKE16-NEXT:    s_mov_b32 s7, s11
+; GFX11-FAKE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-FAKE16-NEXT:    s_mov_b32 s12, s2
+; GFX11-FAKE16-NEXT:    s_mov_b32 s13, s3
+; GFX11-FAKE16-NEXT:    buffer_load_u16 v0, off, s[12:15], 0 glc dlc
+; GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT:    buffer_load_u16 v1, off, s[4:7], 0 glc dlc
+; GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT:    s_mov_b32 s8, s0
+; GFX11-FAKE16-NEXT:    s_mov_b32 s9, s1
+; GFX11-FAKE16-NEXT:    v_cmp_nle_f16_e32 vcc_lo, v0, v1
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e64 v0, 0, -1, vcc_lo
+; GFX11-FAKE16-NEXT:    buffer_store_b32 v0, off, s[8:11], 0
+; GFX11-FAKE16-NEXT:    s_endpgm
 ;
 ; GFX12-LABEL: fcmp_f16_nle:
 ; GFX12:       ; %bb.0: ; %entry
@@ -1517,30 +1869,57 @@ define amdgpu_kernel void @fcmp_f16_neq(
 ; VI-NEXT:    buffer_store_dword v0, off, s[4:7], 0
 ; VI-NEXT:    s_endpgm
 ;
-; GFX11-LABEL: fcmp_f16_neq:
-; GFX11:       ; %bb.0: ; %entry
-; GFX11-NEXT:    s_clause 0x1
-; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
-; GFX11-NEXT:    s_load_b64 s[4:5], s[4:5], 0x34
-; GFX11-NEXT:    s_mov_b32 s10, -1
-; GFX11-NEXT:    s_mov_b32 s11, 0x31016000
-; GFX11-NEXT:    s_mov_b32 s14, s10
-; GFX11-NEXT:    s_mov_b32 s15, s11
-; GFX11-NEXT:    s_mov_b32 s6, s10
-; GFX11-NEXT:    s_mov_b32 s7, s11
-; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX11-NEXT:    s_mov_b32 s12, s2
-; GFX11-NEXT:    s_mov_b32 s13, s3
-; GFX11-NEXT:    buffer_load_u16 v0, off, s[12:15], 0 glc dlc
-; GFX11-NEXT:    s_waitcnt vmcnt(0)
-; GFX11-NEXT:    buffer_load_u16 v1, off, s[4:7], 0 glc dlc
-; GFX11-NEXT:    s_waitcnt vmcnt(0)
-; GFX11-NEXT:    s_mov_b32 s8, s0
-; GFX11-NEXT:    s_mov_b32 s9, s1
-; GFX11-NEXT:    v_cmp_neq_f16_e32 vcc_lo, v0, v1
-; GFX11-NEXT:    v_cndmask_b32_e64 v0, 0, -1, vcc_lo
-; GFX11-NEXT:    buffer_store_b32 v0, off, s[8:11], 0
-; GFX11-NEXT:    s_endpgm
+; GFX11-TRUE16-LABEL: fcmp_f16_neq:
+; GFX11-TRUE16:       ; %bb.0: ; %entry
+; GFX11-TRUE16-NEXT:    s_clause 0x1
+; GFX11-TRUE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-TRUE16-NEXT:    s_load_b64 s[4:5], s[4:5], 0x34
+; GFX11-TRUE16-NEXT:    s_mov_b32 s10, -1
+; GFX11-TRUE16-NEXT:    s_mov_b32 s11, 0x31016000
+; GFX11-TRUE16-NEXT:    s_mov_b32 s14, s10
+; GFX11-TRUE16-NEXT:    s_mov_b32 s15, s11
+; GFX11-TRUE16-NEXT:    s_mov_b32 s6, s10
+; GFX11-TRUE16-NEXT:    s_mov_b32 s7, s11
+; GFX11-TRUE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-TRUE16-NEXT:    s_mov_b32 s12, s2
+; GFX11-TRUE16-NEXT:    s_mov_b32 s13, s3
+; GFX11-TRUE16-NEXT:    buffer_load_u16 v0, off, s[12:15], 0 glc dlc
+; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT:    buffer_load_u16 v1, off, s[4:7], 0 glc dlc
+; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT:    s_mov_b32 s8, s0
+; GFX11-TRUE16-NEXT:    s_mov_b32 s9, s1
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v0.h, v1.l
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_cmp_neq_f16_e32 vcc_lo, v0.l, v0.h
+; GFX11-TRUE16-NEXT:    v_cndmask_b32_e64 v0, 0, -1, vcc_lo
+; GFX11-TRUE16-NEXT:    buffer_store_b32 v0, off, s[8:11], 0
+; GFX11-TRUE16-NEXT:    s_endpgm
+;
+; GFX11-FAKE16-LABEL: fcmp_f16_neq:
+; GFX11-FAKE16:       ; %bb.0: ; %entry
+; GFX11-FAKE16-NEXT:    s_clause 0x1
+; GFX11-FAKE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-FAKE16-NEXT:    s_load_b64 s[4:5], s[4:5], 0x34
+; GFX11-FAKE16-NEXT:    s_mov_b32 s10, -1
+; GFX11-FAKE16-NEXT:    s_mov_b32 s11, 0x31016000
+; GFX11-FAKE16-NEXT:    s_mov_b32 s14, s10
+; GFX11-FAKE16-NEXT:    s_mov_b32 s15, s11
+; GFX11-FAKE16-NEXT:    s_mov_b32 s6, s10
+; GFX11-FAKE16-NEXT:    s_mov_b32 s7, s11
+; GFX11-FAKE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-FAKE16-NEXT:    s_mov_b32 s12, s2
+; GFX11-FAKE16-NEXT:    s_mov_b32 s13, s3
+; GFX11-FAKE16-NEXT:    buffer_load_u16 v0, off, s[12:15], 0 glc dlc
+; GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT:    buffer_load_u16 v1, off, s[4:7], 0 glc dlc
+; GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT:    s_mov_b32 s8, s0
+; GFX11-FAKE16-NEXT:    s_mov_b32 s9, s1
+; GFX11-FAKE16-NEXT:    v_cmp_neq_f16_e32 vcc_lo, v0, v1
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e64 v0, 0, -1, vcc_lo
+; GFX11-FAKE16-NEXT:    buffer_store_b32 v0, off, s[8:11], 0
+; GFX11-FAKE16-NEXT:    s_endpgm
 ;
 ; GFX12-LABEL: fcmp_f16_neq:
 ; GFX12:       ; %bb.0: ; %entry
@@ -1629,30 +2008,57 @@ define amdgpu_kernel void @fcmp_f16_nlt(
 ; VI-NEXT:    buffer_store_dword v0, off, s[4:7], 0
 ; VI-NEXT:    s_endpgm
 ;
-; GFX11-LABEL: fcmp_f16_nlt:
-; GFX11:       ; %bb.0: ; %entry
-; GFX11-NEXT:    s_clause 0x1
-; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
-; GFX11-NEXT:    s_load_b64 s[4:5], s[4:5], 0x34
-; GFX11-NEXT:    s_mov_b32 s10, -1
-; GFX11-NEXT:    s_mov_b32 s11, 0x31016000
-; GFX11-NEXT:    s_mov_b32 s14, s10
-; GFX11-NEXT:    s_mov_b32 s15, s11
-; GFX11-NEXT:    s_mov_b32 s6, s10
-; GFX11-NEXT:    s_mov_b32 s7, s11
-; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX11-NEXT:    s_mov_b32 s12, s2
-; GFX11-NEXT:    s_mov_b32 s13, s3
-; GFX11-NEXT:    buffer_load_u16 v0, off, s[12:15], 0 glc dlc
-; GFX11-NEXT:    s_waitcnt vmcnt(0)
-; GFX11-NEXT:    buffer_load_u16 v1, off, s[4:7], 0 glc dlc
-; GFX11-NEXT:    s_waitcnt vmcnt(0)
-; GFX11-NEXT:    s_mov_b32 s8, s0
-; GFX11-NEXT:    s_mov_b32 s9, s1
-; GFX11-NEXT:    v_cmp_nlt_f16_e32 vcc_lo, v0, v1
-; GFX11-NEXT:    v_cndmask_b32_e64 v0, 0, -1, vcc_lo
-; GFX11-NEXT:    buffer_store_b32 v0, off, s[8:11], 0
-; GFX11-NEXT:    s_endpgm
+; GFX11-TRUE16-LABEL: fcmp_f16_nlt:
+; GFX11-TRUE16:       ; %bb.0: ; %entry
+; GFX11-TRUE16-NEXT:    s_clause 0x1
+; GFX11-TRUE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-TRUE16-NEXT:    s_load_b64 s[4:5], s[4:5], 0x34
+; GFX11-TRUE16-NEXT:    s_mov_b32 s10, -1
+; GFX11-TRUE16-NEXT:    s_mov_b32 s11, 0x31016000
+; GFX11-TRUE16-NEXT:    s_mov_b32 s14, s10
+; GFX11-TRUE16-NEXT:    s_mov_b32 s15, s11
+; GFX11-TRUE16-NEXT:    s_mov_b32 s6, s10
+; GFX11-TRUE16-NEXT:    s_mov_b32 s7, s11
+; GFX11-TRUE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-TRUE16-NEXT:    s_mov_b32 s12, s2
+; GFX11-TRUE16-NEXT:    s_mov_b32 s13, s3
+; GFX11-TRUE16-NEXT:    buffer_load_u16 v0, off, s[12:15], 0 glc dlc
+; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT:    buffer_load_u16 v1, off, s[4:7], 0 glc dlc
+; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT:    s_mov_b32 s8, s0
+; GFX11-TRUE16-NEXT:    s_mov_b32 s9, s1
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v0.h, v1.l
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_cmp_nlt_f16_e32 vcc_lo, v0.l, v0.h
+; GFX11-TRUE16-NEXT:    v_cndmask_b32_e64 v0, 0, -1, vcc_lo
+; GFX11-TRUE16-NEXT:    buffer_store_b32 v0, off, s[8:11], 0
+; GFX11-TRUE16-NEXT:    s_endpgm
+;
+; GFX11-FAKE16-LABEL: fcmp_f16_nlt:
+; GFX11-FAKE16:       ; %bb.0: ; %entry
+; GFX11-FAKE16-NEXT:    s_clause 0x1
+; GFX11-FAKE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-FAKE16-NEXT:    s_load_b64 s[4:5], s[4:5], 0x34
+; GFX11-FAKE16-NEXT:    s_mov_b32 s10, -1
+; GFX11-FAKE16-NEXT:    s_mov_b32 s11, 0x31016000
+; GFX11-FAKE16-NEXT:    s_mov_b32 s14, s10
+; GFX11-FAKE16-NEXT:    s_mov_b32 s15, s11
+; GFX11-FAKE16-NEXT:    s_mov_b32 s6, s10
+; GFX11-FAKE16-NEXT:    s_mov_b32 s7, s11
+; GFX11-FAKE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-FAKE16-NEXT:    s_mov_b32 s12, s2
+; GFX11-FAKE16-NEXT:    s_mov_b32 s13, s3
+; GFX11-FAKE16-NEXT:    buffer_load_u16 v0, off, s[12:15], 0 glc dlc
+; GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT:    buffer_load_u16 v1, off, s[4:7], 0 glc dlc
+; GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT:    s_mov_b32 s8, s0
+; GFX11-FAKE16-NEXT:    s_mov_b32 s9, s1
+; GFX11-FAKE16-NEXT:    v_cmp_nlt_f16_e32 vcc_lo, v0, v1
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e64 v0, 0, -1, vcc_lo
+; GFX11-FAKE16-NEXT:    buffer_store_b32 v0, off, s[8:11], 0
+; GFX11-FAKE16-NEXT:    s_endpgm
 ;
 ; GFX12-LABEL: fcmp_f16_nlt:
 ; GFX12:       ; %bb.0: ; %entry
@@ -1751,35 +2157,65 @@ define amdgpu_kernel void @fcmp_v2f16_lt(
 ; VI-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
 ; VI-NEXT:    s_endpgm
 ;
-; GFX11-LABEL: fcmp_v2f16_lt:
-; GFX11:       ; %bb.0: ; %entry
-; GFX11-NEXT:    s_clause 0x1
-; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
-; GFX11-NEXT:    s_load_b64 s[4:5], s[4:5], 0x34
-; GFX11-NEXT:    s_mov_b32 s10, -1
-; GFX11-NEXT:    s_mov_b32 s11, 0x31016000
-; GFX11-NEXT:    s_mov_b32 s6, s10
-; GFX11-NEXT:    s_mov_b32 s7, s11
-; GFX11-NEXT:    s_mov_b32 s14, s10
-; GFX11-NEXT:    s_mov_b32 s15, s11
-; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX11-NEXT:    s_mov_b32 s12, s2
-; GFX11-NEXT:    s_mov_b32 s13, s3
-; GFX11-NEXT:    buffer_load_b32 v0, off, s[4:7], 0
-; GFX11-NEXT:    buffer_load_b32 v1, off, s[12:15], 0
-; GFX11-NEXT:    s_mov_b32 s8, s0
-; GFX11-NEXT:    s_mov_b32 s9, s1
-; GFX11-NEXT:    s_waitcnt vmcnt(1)
-; GFX11-NEXT:    v_lshrrev_b32_e32 v2, 16, v0
-; GFX11-NEXT:    s_waitcnt vmcnt(0)
-; GFX11-NEXT:    v_lshrrev_b32_e32 v3, 16, v1
-; GFX11-NEXT:    v_cmp_lt_f16_e32 vcc_lo, v1, v0
-; GFX11-NEXT:    v_cndmask_b32_e64 v0, 0, -1, vcc_lo
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_3)
-; GFX11-NEXT:    v_cmp_lt_f16_e32 vcc_lo, v3, v2
-; GFX11-NEXT:    v_cndmask_b32_e64 v1, 0, -1, vcc_lo
-; GFX11-NEXT:    buffer_store_b64 v[0:1], off, s[8:11], 0
-; GFX11-NEXT:    s_endpgm
+; GFX11-TRUE16-LABEL: fcmp_v2f16_lt:
+; GFX11-TRUE16:       ; %bb.0: ; %entry
+; GFX11-TRUE16-NEXT:    s_clause 0x1
+; GFX11-TRUE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-TRUE16-NEXT:    s_load_b64 s[4:5], s[4:5], 0x34
+; GFX11-TRUE16-NEXT:    s_mov_b32 s10, -1
+; GFX11-TRUE16-NEXT:    s_mov_b32 s11, 0x31016000
+; GFX11-TRUE16-NEXT:    s_mov_b32 s6, s10
+; GFX11-TRUE16-NEXT:    s_mov_b32 s7, s11
+; GFX11-TRUE16-NEXT:    s_mov_b32 s14, s10
+; GFX11-TRUE16-NEXT:    s_mov_b32 s15, s11
+; GFX11-TRUE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-TRUE16-NEXT:    s_mov_b32 s12, s2
+; GFX11-TRUE16-NEXT:    s_mov_b32 s13, s3
+; GFX11-TRUE16-NEXT:    buffer_load_b32 v0, off, s[4:7], 0
+; GFX11-TRUE16-NEXT:    buffer_load_b32 v1, off, s[12:15], 0
+; GFX11-TRUE16-NEXT:    s_mov_b32 s8, s0
+; GFX11-TRUE16-NEXT:    s_mov_b32 s9, s1
+; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(1)
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v2, 16, v0
+; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v3, 16, v1
+; GFX11-TRUE16-NEXT:    v_cmp_lt_f16_e32 vcc_lo, v1.l, v0.l
+; GFX11-TRUE16-NEXT:    v_cndmask_b32_e64 v0, 0, -1, vcc_lo
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT:    v_cmp_lt_f16_e32 vcc_lo, v3.l, v2.l
+; GFX11-TRUE16-NEXT:    v_cndmask_b32_e64 v1, 0, -1, vcc_lo
+; GFX11-TRUE16-NEXT:    buffer_store_b64 v[0:1], off, s[8:11], 0
+; GFX11-TRUE16-NEXT:    s_endpgm
+;
+; GFX11-FAKE16-LABEL: fcmp_v2f16_lt:
+; GFX11-FAKE16:       ; %bb.0: ; %entry
+; GFX11-FAKE16-NEXT:    s_clause 0x1
+; GFX11-FAKE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-FAKE16-NEXT:    s_load_b64 s[4:5], s[4:5], 0x34
+; GFX11-FAKE16-NEXT:    s_mov_b32 s10, -1
+; GFX11-FAKE16-NEXT:    s_mov_b32 s11, 0x31016000
+; GFX11-FAKE16-NEXT:    s_mov_b32 s6, s10
+; GFX11-FAKE16-NEXT:    s_mov_b32 s7, s11
+; GFX11-FAKE16-NEXT:    s_mov_b32 s14, s10
+; GFX11-FAKE16-NEXT:    s_mov_b32 s15, s11
+; GFX11-FAKE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-FAKE16-NEXT:    s_mov_b32 s12, s2
+; GFX11-FAKE16-NEXT:    s_mov_b32 s13, s3
+; GFX11-FAKE16-NEXT:    buffer_load_b32 v0, off, s[4:7], 0
+; GFX11-FAKE16-NEXT:    buffer_load_b32 v1, off, s[12:15], 0
+; GFX11-FAKE16-NEXT:    s_mov_b32 s8, s0
+; GFX11-FAKE16-NEXT:    s_mov_b32 s9, s1
+; GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(1)
+; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v2, 16, v0
+; GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v3, 16, v1
+; GFX11-FAKE16-NEXT:    v_cmp_lt_f16_e32 vcc_lo, v1, v0
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e64 v0, 0, -1, vcc_lo
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3)
+; GFX11-FAKE16-NEXT:    v_cmp_lt_f16_e32 vcc_lo, v3, v2
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e64 v1, 0, -1, vcc_lo
+; GFX11-FAKE16-NEXT:    buffer_store_b64 v[0:1], off, s[8:11], 0
+; GFX11-FAKE16-NEXT:    s_endpgm
 ;
 ; GFX12-LABEL: fcmp_v2f16_lt:
 ; GFX12:       ; %bb.0: ; %entry
@@ -1807,6 +2243,7 @@ define amdgpu_kernel void @fcmp_v2f16_lt(
 ; GFX12-NEXT:    v_cndmask_b32_e64 v0, 0, -1, vcc_lo
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3)
 ; GFX12-NEXT:    v_cmp_lt_f16_e32 vcc_lo, v3, v2
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_cndmask_b32_e64 v1, 0, -1, vcc_lo
 ; GFX12-NEXT:    buffer_store_b64 v[0:1], off, s[8:11], null
 ; GFX12-NEXT:    s_endpgm
@@ -1884,35 +2321,65 @@ define amdgpu_kernel void @fcmp_v2f16_eq(
 ; VI-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
 ; VI-NEXT:    s_endpgm
 ;
-; GFX11-LABEL: fcmp_v2f16_eq:
-; GFX11:       ; %bb.0: ; %entry
-; GFX11-NEXT:    s_clause 0x1
-; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
-; GFX11-NEXT:    s_load_b64 s[4:5], s[4:5], 0x34
-; GFX11-NEXT:    s_mov_b32 s10, -1
-; GFX11-NEXT:    s_mov_b32 s11, 0x31016000
-; GFX11-NEXT:    s_mov_b32 s6, s10
-; GFX11-NEXT:    s_mov_b32 s7, s11
-; GFX11-NEXT:    s_mov_b32 s14, s10
-; GFX11-NEXT:    s_mov_b32 s15, s11
-; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX11-NEXT:    s_mov_b32 s12, s2
-; GFX11-NEXT:    s_mov_b32 s13, s3
-; GFX11-NEXT:    buffer_load_b32 v0, off, s[4:7], 0
-; GFX11-NEXT:    buffer_load_b32 v1, off, s[12:15], 0
-; GFX11-NEXT:    s_mov_b32 s8, s0
-; GFX11-NEXT:    s_mov_b32 s9, s1
-; GFX11-NEXT:    s_waitcnt vmcnt(1)
-; GFX11-NEXT:    v_lshrrev_b32_e32 v2, 16, v0
-; GFX11-NEXT:    s_waitcnt vmcnt(0)
-; GFX11-NEXT:    v_lshrrev_b32_e32 v3, 16, v1
-; GFX11-NEXT:    v_cmp_eq_f16_e32 vcc_lo, v1, v0
-; GFX11-NEXT:    v_cndmask_b32_e64 v0, 0, -1, vcc_lo
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_3)
-; GFX11-NEXT:    v_cmp_eq_f16_e32 vcc_lo, v3, v2
-; GFX11-NEXT:    v_cndmask_b32_e64 v1, 0, -1, vcc_lo
-; GFX11-NEXT:    buffer_store_b64 v[0:1], off, s[8:11], 0
-; GFX11-NEXT:    s_endpgm
+; GFX11-TRUE16-LABEL: fcmp_v2f16_eq:
+; GFX11-TRUE16:       ; %bb.0: ; %entry
+; GFX11-TRUE16-NEXT:    s_clause 0x1
+; GFX11-TRUE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-TRUE16-NEXT:    s_load_b64 s[4:5], s[4:5], 0x34
+; GFX11-TRUE16-NEXT:    s_mov_b32 s10, -1
+; GFX11-TRUE16-NEXT:    s_mov_b32 s11, 0x31016000
+; GFX11-TRUE16-NEXT:    s_mov_b32 s6, s10
+; GFX11-TRUE16-NEXT:    s_mov_b32 s7, s11
+; GFX11-TRUE16-NEXT:    s_mov_b32 s14, s10
+; GFX11-TRUE16-NEXT:    s_mov_b32 s15, s11
+; GFX11-TRUE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-TRUE16-NEXT:    s_mov_b32 s12, s2
+; GFX11-TRUE16-NEXT:    s_mov_b32 s13, s3
+; GFX11-TRUE16-NEXT:    buffer_load_b32 v0, off, s[4:7], 0
+; GFX11-TRUE16-NEXT:    buffer_load_b32 v1, off, s[12:15], 0
+; GFX11-TRUE16-NEXT:    s_mov_b32 s8, s0
+; GFX11-TRUE16-NEXT:    s_mov_b32 s9, s1
+; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(1)
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v2, 16, v0
+; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v3, 16, v1
+; GFX11-TRUE16-NEXT:    v_cmp_eq_f16_e32 vcc_lo, v1.l, v0.l
+; GFX11-TRUE16-NEXT:    v_cndmask_b32_e64 v0, 0, -1, vcc_lo
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT:    v_cmp_eq_f16_e32 vcc_lo, v3.l, v2.l
+; GFX11-TRUE16-NEXT:    v_cndmask_b32_e64 v1, 0, -1, vcc_lo
+; GFX11-TRUE16-NEXT:    buffer_store_b64 v[0:1], off, s[8:11], 0
+; GFX11-TRUE16-NEXT:    s_endpgm
+;
+; GFX11-FAKE16-LABEL: fcmp_v2f16_eq:
+; GFX11-FAKE16:       ; %bb.0: ; %entry
+; GFX11-FAKE16-NEXT:    s_clause 0x1
+; GFX11-FAKE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-FAKE16-NEXT:    s_load_b64 s[4:5], s[4:5], 0x34
+; GFX11-FAKE16-NEXT:    s_mov_b32 s10, -1
+; GFX11-FAKE16-NEXT:    s_mov_b32 s11, 0x31016000
+; GFX11-FAKE16-NEXT:    s_mov_b32 s6, s10
+; GFX11-FAKE16-NEXT:    s_mov_b32 s7, s11
+; GFX11-FAKE16-NEXT:    s_mov_b32 s14, s10
+; GFX11-FAKE16-NEXT:    s_mov_b32 s15, s11
+; GFX11-FAKE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-FAKE16-NEXT:    s_mov_b32 s12, s2
+; GFX11-FAKE16-NEXT:    s_mov_b32 s13, s3
+; GFX11-FAKE16-NEXT:    buffer_load_b32 v0, off, s[4:7], 0
+; GFX11-FAKE16-NEXT:    buffer_load_b32 v1, off, s[12:15], 0
+; GFX11-FAKE16-NEXT:    s_mov_b32 s8, s0
+; GFX11-FAKE16-NEXT:    s_mov_b32 s9, s1
+; GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(1)
+; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v2, 16, v0
+; GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v3, 16, v1
+; GFX11-FAKE16-NEXT:    v_cmp_eq_f16_e32 vcc_lo, v1, v0
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e64 v0, 0, -1, vcc_lo
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3)
+; GFX11-FAKE16-NEXT:    v_cmp_eq_f16_e32 vcc_lo, v3, v2
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e64 v1, 0, -1, vcc_lo
+; GFX11-FAKE16-NEXT:    buffer_store_b64 v[0:1], off, s[8:11], 0
+; GFX11-FAKE16-NEXT:    s_endpgm
 ;
 ; GFX12-LABEL: fcmp_v2f16_eq:
 ; GFX12:       ; %bb.0: ; %entry
@@ -1940,6 +2407,7 @@ define amdgpu_kernel void @fcmp_v2f16_eq(
 ; GFX12-NEXT:    v_cndmask_b32_e64 v0, 0, -1, vcc_lo
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3)
 ; GFX12-NEXT:    v_cmp_eq_f16_e32 vcc_lo, v3, v2
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_cndmask_b32_e64 v1, 0, -1, vcc_lo
 ; GFX12-NEXT:    buffer_store_b64 v[0:1], off, s[8:11], null
 ; GFX12-NEXT:    s_endpgm
@@ -2016,35 +2484,65 @@ define amdgpu_kernel void @fcmp_v2f16_le(
 ; VI-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
 ; VI-NEXT:    s_endpgm
 ;
-; GFX11-LABEL: fcmp_v2f16_le:
-; GFX11:       ; %bb.0: ; %entry
-; GFX11-NEXT:    s_clause 0x1
-; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
-; GFX11-NEXT:    s_load_b64 s[4:5], s[4:5], 0x34
-; GFX11-NEXT:    s_mov_b32 s10, -1
-; GFX11-NEXT:    s_mov_b32 s11, 0x31016000
-; GFX11-NEXT:    s_mov_b32 s6, s10
-; GFX11-NEXT:    s_mov_b32 s7, s11
-; GFX11-NEXT:    s_mov_b32 s14, s10
-; GFX11-NEXT:    s_mov_b32 s15, s11
-; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX11-NEXT:    s_mov_b32 s12, s2
-; GFX11-NEXT:    s_mov_b32 s13, s3
-; GFX11-NEXT:    buffer_load_b32 v0, off, s[4:7], 0
-; GFX11-NEXT:    buffer_load_b32 v1, off, s[12:15], 0
-; GFX11-NEXT:    s_mov_b32 s8, s0
-; GFX11-NEXT:    s_mov_b32 s9, s1
-; GFX11-NEXT:    s_waitcnt vmcnt(1)
-; GFX11-NEXT:    v_lshrrev_b32_e32 v2, 16, v0
-; GFX11-NEXT:    s_waitcnt vmcnt(0)
-; GFX11-NEXT:    v_lshrrev_b32_e32 v3, 16, v1
-; GFX11-NEXT:    v_cmp_le_f16_e32 vcc_lo, v1, v0
-; GFX11-NEXT:    v_cndmask_b32_e64 v0, 0, -1, vcc_lo
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_3)
-; GFX11-NEXT:    v_cmp_le_f16_e32 vcc_lo, v3, v2
-; GFX11-NEXT:    v_cndmask_b32_e64 v1, 0, -1, vcc_lo
-; GFX11-NEXT:    buffer_store_b64 v[0:1], off, s[8:11], 0
-; GFX11-NEXT:    s_endpgm
+; GFX11-TRUE16-LABEL: fcmp_v2f16_le:
+; GFX11-TRUE16:       ; %bb.0: ; %entry
+; GFX11-TRUE16-NEXT:    s_clause 0x1
+; GFX11-TRUE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-TRUE16-NEXT:    s_load_b64 s[4:5], s[4:5], 0x34
+; GFX11-TRUE16-NEXT:    s_mov_b32 s10, -1
+; GFX11-TRUE16-NEXT:    s_mov_b32 s11, 0x31016000
+; GFX11-TRUE16-NEXT:    s_mov_b32 s6, s10
+; GFX11-TRUE16-NEXT:    s_mov_b32 s7, s11
+; GFX11-TRUE16-NEXT:    s_mov_b32 s14, s10
+; GFX11-TRUE16-NEXT:    s_mov_b32 s15, s11
+; GFX11-TRUE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-TRUE16-NEXT:    s_mov_b32 s12, s2
+; GFX11-TRUE16-NEXT:    s_mov_b32 s13, s3
+; GFX11-TRUE16-NEXT:    buffer_load_b32 v0, off, s[4:7], 0
+; GFX11-TRUE16-NEXT:    buffer_load_b32 v1, off, s[12:15], 0
+; GFX11-TRUE16-NEXT:    s_mov_b32 s8, s0
+; GFX11-TRUE16-NEXT:    s_mov_b32 s9, s1
+; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(1)
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v2, 16, v0
+; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v3, 16, v1
+; GFX11-TRUE16-NEXT:    v_cmp_le_f16_e32 vcc_lo, v1.l, v0.l
+; GFX11-TRUE16-NEXT:    v_cndmask_b32_e64 v0, 0, -1, vcc_lo
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT:    v_cmp_le_f16_e32 vcc_lo, v3.l, v2.l
+; GFX11-TRUE16-NEXT:    v_cndmask_b32_e64 v1, 0, -1, vcc_lo
+; GFX11-TRUE16-NEXT:    buffer_store_b64 v[0:1], off, s[8:11], 0
+; GFX11-TRUE16-NEXT:    s_endpgm
+;
+; GFX11-FAKE16-LABEL: fcmp_v2f16_le:
+; GFX11-FAKE16:       ; %bb.0: ; %entry
+; GFX11-FAKE16-NEXT:    s_clause 0x1
+; GFX11-FAKE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-FAKE16-NEXT:    s_load_b64 s[4:5], s[4:5], 0x34
+; GFX11-FAKE16-NEXT:    s_mov_b32 s10, -1
+; GFX11-FAKE16-NEXT:    s_mov_b32 s11, 0x31016000
+; GFX11-FAKE16-NEXT:    s_mov_b32 s6, s10
+; GFX11-FAKE16-NEXT:    s_mov_b32 s7, s11
+; GFX11-FAKE16-NEXT:    s_mov_b32 s14, s10
+; GFX11-FAKE16-NEXT:    s_mov_b32 s15, s11
+; GFX11-FAKE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-FAKE16-NEXT:    s_mov_b32 s12, s2
+; GFX11-FAKE16-NEXT:    s_mov_b32 s13, s3
+; GFX11-FAKE16-NEXT:    buffer_load_b32 v0, off, s[4:7], 0
+; GFX11-FAKE16-NEXT:    buffer_load_b32 v1, off, s[12:15], 0
+; GFX11-FAKE16-NEXT:    s_mov_b32 s8, s0
+; GFX11-FAKE16-NEXT:    s_mov_b32 s9, s1
+; GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(1)
+; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v2, 16, v0
+; GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v3, 16, v1
+; GFX11-FAKE16-NEXT:    v_cmp_le_f16_e32 vcc_lo, v1, v0
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e64 v0, 0, -1, vcc_lo
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3)
+; GFX11-FAKE16-NEXT:    v_cmp_le_f16_e32 vcc_lo, v3, v2
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e64 v1, 0, -1, vcc_lo
+; GFX11-FAKE16-NEXT:    buffer_store_b64 v[0:1], off, s[8:11], 0
+; GFX11-FAKE16-NEXT:    s_endpgm
 ;
 ; GFX12-LABEL: fcmp_v2f16_le:
 ; GFX12:       ; %bb.0: ; %entry
@@ -2072,6 +2570,7 @@ define amdgpu_kernel void @fcmp_v2f16_le(
 ; GFX12-NEXT:    v_cndmask_b32_e64 v0, 0, -1, vcc_lo
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3)
 ; GFX12-NEXT:    v_cmp_le_f16_e32 vcc_lo, v3, v2
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_cndmask_b32_e64 v1, 0, -1, vcc_lo
 ; GFX12-NEXT:    buffer_store_b64 v[0:1], off, s[8:11], null
 ; GFX12-NEXT:    s_endpgm
@@ -2148,35 +2647,65 @@ define amdgpu_kernel void @fcmp_v2f16_gt(
 ; VI-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
 ; VI-NEXT:    s_endpgm
 ;
-; GFX11-LABEL: fcmp_v2f16_gt:
-; GFX11:       ; %bb.0: ; %entry
-; GFX11-NEXT:    s_clause 0x1
-; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
-; GFX11-NEXT:    s_load_b64 s[4:5], s[4:5], 0x34
-; GFX11-NEXT:    s_mov_b32 s10, -1
-; GFX11-NEXT:    s_mov_b32 s11, 0x31016000
-; GFX11-NEXT:    s_mov_b32 s6, s10
-; GFX11-NEXT:    s_mov_b32 s7, s11
-; GFX11-NEXT:    s_mov_b32 s14, s10
-; GFX11-NEXT:    s_mov_b32 s15, s11
-; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX11-NEXT:    s_mov_b32 s12, s2
-; GFX11-NEXT:    s_mov_b32 s13, s3
-; GFX11-NEXT:    buffer_load_b32 v0, off, s[4:7], 0
-; GFX11-NEXT:    buffer_load_b32 v1, off, s[12:15], 0
-; GFX11-NEXT:    s_mov_b32 s8, s0
-; GFX11-NEXT:    s_mov_b32 s9, s1
-; GFX11-NEXT:    s_waitcnt vmcnt(1)
-; GFX11-NEXT:    v_lshrrev_b32_e32 v2, 16, v0
-; GFX11-NEXT:    s_waitcnt vmcnt(0)
-; GFX11-NEXT:    v_lshrrev_b32_e32 v3, 16, v1
-; GFX11-NEXT:    v_cmp_gt_f16_e32 vcc_lo, v1, v0
-; GFX11-NEXT:    v_cndmask_b32_e64 v0, 0, -1, vcc_lo
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_3)
-; GFX11-NEXT:    v_cmp_gt_f16_e32 vcc_lo, v3, v2
-; GFX11-NEXT:    v_cndmask_b32_e64 v1, 0, -1, vcc_lo
-; GFX11-NEXT:    buffer_store_b64 v[0:1], off, s[8:11], 0
-; GFX11-NEXT:    s_endpgm
+; GFX11-TRUE16-LABEL: fcmp_v2f16_gt:
+; GFX11-TRUE16:       ; %bb.0: ; %entry
+; GFX11-TRUE16-NEXT:    s_clause 0x1
+; GFX11-TRUE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-TRUE16-NEXT:    s_load_b64 s[4:5], s[4:5], 0x34
+; GFX11-TRUE16-NEXT:    s_mov_b32 s10, -1
+; GFX11-TRUE16-NEXT:    s_mov_b32 s11, 0x31016000
+; GFX11-TRUE16-NEXT:    s_mov_b32 s6, s10
+; GFX11-TRUE16-NEXT:    s_mov_b32 s7, s11
+; GFX11-TRUE16-NEXT:    s_mov_b32 s14, s10
+; GFX11-TRUE16-NEXT:    s_mov_b32 s15, s11
+; GFX11-TRUE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-TRUE16-NEXT:    s_mov_b32 s12, s2
+; GFX11-TRUE16-NEXT:    s_mov_b32 s13, s3
+; GFX11-TRUE16-NEXT:    buffer_load_b32 v0, off, s[4:7], 0
+; GFX11-TRUE16-NEXT:    buffer_load_b32 v1, off, s[12:15], 0
+; GFX11-TRUE16-NEXT:    s_mov_b32 s8, s0
+; GFX11-TRUE16-NEXT:    s_mov_b32 s9, s1
+; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(1)
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v2, 16, v0
+; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v3, 16, v1
+; GFX11-TRUE16-NEXT:    v_cmp_gt_f16_e32 vcc_lo, v1.l, v0.l
+; GFX11-TRUE16-NEXT:    v_cndmask_b32_e64 v0, 0, -1, vcc_lo
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT:    v_cmp_gt_f16_e32 vcc_lo, v3.l, v2.l
+; GFX11-TRUE16-NEXT:    v_cndmask_b32_e64 v1, 0, -1, vcc_lo
+; GFX11-TRUE16-NEXT:    buffer_store_b64 v[0:1], off, s[8:11], 0
+; GFX11-TRUE16-NEXT:    s_endpgm
+;
+; GFX11-FAKE16-LABEL: fcmp_v2f16_gt:
+; GFX11-FAKE16:       ; %bb.0: ; %entry
+; GFX11-FAKE16-NEXT:    s_clause 0x1
+; GFX11-FAKE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-FAKE16-NEXT:    s_load_b64 s[4:5], s[4:5], 0x34
+; GFX11-FAKE16-NEXT:    s_mov_b32 s10, -1
+; GFX11-FAKE16-NEXT:    s_mov_b32 s11, 0x31016000
+; GFX11-FAKE16-NEXT:    s_mov_b32 s6, s10
+; GFX11-FAKE16-NEXT:    s_mov_b32 s7, s11
+; GFX11-FAKE16-NEXT:    s_mov_b32 s14, s10
+; GFX11-FAKE16-NEXT:    s_mov_b32 s15, s11
+; GFX11-FAKE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-FAKE16-NEXT:    s_mov_b32 s12, s2
+; GFX11-FAKE16-NEXT:    s_mov_b32 s13, s3
+; GFX11-FAKE16-NEXT:    buffer_load_b32 v0, off, s[4:7], 0
+; GFX11-FAKE16-NEXT:    buffer_load_b32 v1, off, s[12:15], 0
+; GFX11-FAKE16-NEXT:    s_mov_b32 s8, s0
+; GFX11-FAKE16-NEXT:    s_mov_b32 s9, s1
+; GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(1)
+; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v2, 16, v0
+; GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v3, 16, v1
+; GFX11-FAKE16-NEXT:    v_cmp_gt_f16_e32 vcc_lo, v1, v0
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e64 v0, 0, -1, vcc_lo
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3)
+; GFX11-FAKE16-NEXT:    v_cmp_gt_f16_e32 vcc_lo, v3, v2
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e64 v1, 0, -1, vcc_lo
+; GFX11-FAKE16-NEXT:    buffer_store_b64 v[0:1], off, s[8:11], 0
+; GFX11-FAKE16-NEXT:    s_endpgm
 ;
 ; GFX12-LABEL: fcmp_v2f16_gt:
 ; GFX12:       ; %bb.0: ; %entry
@@ -2204,6 +2733,7 @@ define amdgpu_kernel void @fcmp_v2f16_gt(
 ; GFX12-NEXT:    v_cndmask_b32_e64 v0, 0, -1, vcc_lo
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3)
 ; GFX12-NEXT:    v_cmp_gt_f16_e32 vcc_lo, v3, v2
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_cndmask_b32_e64 v1, 0, -1, vcc_lo
 ; GFX12-NEXT:    buffer_store_b64 v[0:1], off, s[8:11], null
 ; GFX12-NEXT:    s_endpgm
@@ -2281,35 +2811,65 @@ define amdgpu_kernel void @fcmp_v2f16_lg(
 ; VI-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
 ; VI-NEXT:    s_endpgm
 ;
-; GFX11-LABEL: fcmp_v2f16_lg:
-; GFX11:       ; %bb.0: ; %entry
-; GFX11-NEXT:    s_clause 0x1
-; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
-; GFX11-NEXT:    s_load_b64 s[4:5], s[4:5], 0x34
-; GFX11-NEXT:    s_mov_b32 s10, -1
-; GFX11-NEXT:    s_mov_b32 s11, 0x31016000
-; GFX11-NEXT:    s_mov_b32 s6, s10
-; GFX11-NEXT:    s_mov_b32 s7, s11
-; GFX11-NEXT:    s_mov_b32 s14, s10
-; GFX11-NEXT:    s_mov_b32 s15, s11
-; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX11-NEXT:    s_mov_b32 s12, s2
-; GFX11-NEXT:    s_mov_b32 s13, s3
-; GFX11-NEXT:    buffer_load_b32 v0, off, s[4:7], 0
-; GFX11-NEXT:    buffer_load_b32 v1, off, s[12:15], 0
-; GFX11-NEXT:    s_mov_b32 s8, s0
-; GFX11-NEXT:    s_mov_b32 s9, s1
-; GFX11-NEXT:    s_waitcnt vmcnt(1)
-; GFX11-NEXT:    v_lshrrev_b32_e32 v2, 16, v0
-; GFX11-NEXT:    s_waitcnt vmcnt(0)
-; GFX11-NEXT:    v_lshrrev_b32_e32 v3, 16, v1
-; GFX11-NEXT:    v_cmp_lg_f16_e32 vcc_lo, v1, v0
-; GFX11-NEXT:    v_cndmask_b32_e64 v0, 0, -1, vcc_lo
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_3)
-; GFX11-NEXT:    v_cmp_lg_f16_e32 vcc_lo, v3, v2
-; GFX11-NEXT:    v_cndmask_b32_e64 v1, 0, -1, vcc_lo
-; GFX11-NEXT:    buffer_store_b64 v[0:1], off, s[8:11], 0
-; GFX11-NEXT:    s_endpgm
+; GFX11-TRUE16-LABEL: fcmp_v2f16_lg:
+; GFX11-TRUE16:       ; %bb.0: ; %entry
+; GFX11-TRUE16-NEXT:    s_clause 0x1
+; GFX11-TRUE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-TRUE16-NEXT:    s_load_b64 s[4:5], s[4:5], 0x34
+; GFX11-TRUE16-NEXT:    s_mov_b32 s10, -1
+; GFX11-TRUE16-NEXT:    s_mov_b32 s11, 0x31016000
+; GFX11-TRUE16-NEXT:    s_mov_b32 s6, s10
+; GFX11-TRUE16-NEXT:    s_mov_b32 s7, s11
+; GFX11-TRUE16-NEXT:    s_mov_b32 s14, s10
+; GFX11-TRUE16-NEXT:    s_mov_b32 s15, s11
+; GFX11-TRUE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-TRUE16-NEXT:    s_mov_b32 s12, s2
+; GFX11-TRUE16-NEXT:    s_mov_b32 s13, s3
+; GFX11-TRUE16-NEXT:    buffer_load_b32 v0, off, s[4:7], 0
+; GFX11-TRUE16-NEXT:    buffer_load_b32 v1, off, s[12:15], 0
+; GFX11-TRUE16-NEXT:    s_mov_b32 s8, s0
+; GFX11-TRUE16-NEXT:    s_mov_b32 s9, s1
+; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(1)
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v2, 16, v0
+; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v3, 16, v1
+; GFX11-TRUE16-NEXT:    v_cmp_lg_f16_e32 vcc_lo, v1.l, v0.l
+; GFX11-TRUE16-NEXT:    v_cndmask_b32_e64 v0, 0, -1, vcc_lo
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT:    v_cmp_lg_f16_e32 vcc_lo, v3.l, v2.l
+; GFX11-TRUE16-NEXT:    v_cndmask_b32_e64 v1, 0, -1, vcc_lo
+; GFX11-TRUE16-NEXT:    buffer_store_b64 v[0:1], off, s[8:11], 0
+; GFX11-TRUE16-NEXT:    s_endpgm
+;
+; GFX11-FAKE16-LABEL: fcmp_v2f16_lg:
+; GFX11-FAKE16:       ; %bb.0: ; %entry
+; GFX11-FAKE16-NEXT:    s_clause 0x1
+; GFX11-FAKE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-FAKE16-NEXT:    s_load_b64 s[4:5], s[4:5], 0x34
+; GFX11-FAKE16-NEXT:    s_mov_b32 s10, -1
+; GFX11-FAKE16-NEXT:    s_mov_b32 s11, 0x31016000
+; GFX11-FAKE16-NEXT:    s_mov_b32 s6, s10
+; GFX11-FAKE16-NEXT:    s_mov_b32 s7, s11
+; GFX11-FAKE16-NEXT:    s_mov_b32 s14, s10
+; GFX11-FAKE16-NEXT:    s_mov_b32 s15, s11
+; GFX11-FAKE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-FAKE16-NEXT:    s_mov_b32 s12, s2
+; GFX11-FAKE16-NEXT:    s_mov_b32 s13, s3
+; GFX11-FAKE16-NEXT:    buffer_load_b32 v0, off, s[4:7], 0
+; GFX11-FAKE16-NEXT:    buffer_load_b32 v1, off, s[12:15], 0
+; GFX11-FAKE16-NEXT:    s_mov_b32 s8, s0
+; GFX11-FAKE16-NEXT:    s_mov_b32 s9, s1
+; GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(1)
+; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v2, 16, v0
+; GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v3, 16, v1
+; GFX11-FAKE16-NEXT:    v_cmp_lg_f16_e32 vcc_lo, v1, v0
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e64 v0, 0, -1, vcc_lo
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3)
+; GFX11-FAKE16-NEXT:    v_cmp_lg_f16_e32 vcc_lo, v3, v2
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e64 v1, 0, -1, vcc_lo
+; GFX11-FAKE16-NEXT:    buffer_store_b64 v[0:1], off, s[8:11], 0
+; GFX11-FAKE16-NEXT:    s_endpgm
 ;
 ; GFX12-LABEL: fcmp_v2f16_lg:
 ; GFX12:       ; %bb.0: ; %entry
@@ -2337,6 +2897,7 @@ define amdgpu_kernel void @fcmp_v2f16_lg(
 ; GFX12-NEXT:    v_cndmask_b32_e64 v0, 0, -1, vcc_lo
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3)
 ; GFX12-NEXT:    v_cmp_lg_f16_e32 vcc_lo, v3, v2
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_cndmask_b32_e64 v1, 0, -1, vcc_lo
 ; GFX12-NEXT:    buffer_store_b64 v[0:1], off, s[8:11], null
 ; GFX12-NEXT:    s_endpgm
@@ -2414,35 +2975,65 @@ define amdgpu_kernel void @fcmp_v2f16_ge(
 ; VI-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
 ; VI-NEXT:    s_endpgm
 ;
-; GFX11-LABEL: fcmp_v2f16_ge:
-; GFX11:       ; %bb.0: ; %entry
-; GFX11-NEXT:    s_clause 0x1
-; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
-; GFX11-NEXT:    s_load_b64 s[4:5], s[4:5], 0x34
-; GFX11-NEXT:    s_mov_b32 s10, -1
-; GFX11-NEXT:    s_mov_b32 s11, 0x31016000
-; GFX11-NEXT:    s_mov_b32 s6, s10
-; GFX11-NEXT:    s_mov_b32 s7, s11
-; GFX11-NEXT:    s_mov_b32 s14, s10
-; GFX11-NEXT:    s_mov_b32 s15, s11
-; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX11-NEXT:    s_mov_b32 s12, s2
-; GFX11-NEXT:    s_mov_b32 s13, s3
-; GFX11-NEXT:    buffer_load_b32 v0, off, s[4:7], 0
-; GFX11-NEXT:    buffer_load_b32 v1, off, s[12:15], 0
-; GFX11-NEXT:    s_mov_b32 s8, s0
-; GFX11-NEXT:    s_mov_b32 s9, s1
-; GFX11-NEXT:    s_waitcnt vmcnt(1)
-; GFX11-NEXT:    v_lshrrev_b32_e32 v2, 16, v0
-; GFX11-NEXT:    s_waitcnt vmcnt(0)
-; GFX11-NEXT:    v_lshrrev_b32_e32 v3, 16, v1
-; GFX11-NEXT:    v_cmp_ge_f16_e32 vcc_lo, v1, v0
-; GFX11-NEXT:    v_cndmask_b32_e64 v0, 0, -1, vcc_lo
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_3)
-; GFX11-NEXT:    v_cmp_ge_f16_e32 vcc_lo, v3, v2
-; GFX11-NEXT:    v_cndmask_b32_e64 v1, 0, -1, vcc_lo
-; GFX11-NEXT:    buffer_store_b64 v[0:1], off, s[8:11], 0
-; GFX11-NEXT:    s_endpgm
+; GFX11-TRUE16-LABEL: fcmp_v2f16_ge:
+; GFX11-TRUE16:       ; %bb.0: ; %entry
+; GFX11-TRUE16-NEXT:    s_clause 0x1
+; GFX11-TRUE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-TRUE16-NEXT:    s_load_b64 s[4:5], s[4:5], 0x34
+; GFX11-TRUE16-NEXT:    s_mov_b32 s10, -1
+; GFX11-TRUE16-NEXT:    s_mov_b32 s11, 0x31016000
+; GFX11-TRUE16-NEXT:    s_mov_b32 s6, s10
+; GFX11-TRUE16-NEXT:    s_mov_b32 s7, s11
+; GFX11-TRUE16-NEXT:    s_mov_b32 s14, s10
+; GFX11-TRUE16-NEXT:    s_mov_b32 s15, s11
+; GFX11-TRUE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-TRUE16-NEXT:    s_mov_b32 s12, s2
+; GFX11-TRUE16-NEXT:    s_mov_b32 s13, s3
+; GFX11-TRUE16-NEXT:    buffer_load_b32 v0, off, s[4:7], 0
+; GFX11-TRUE16-NEXT:    buffer_load_b32 v1, off, s[12:15], 0
+; GFX11-TRUE16-NEXT:    s_mov_b32 s8, s0
+; GFX11-TRUE16-NEXT:    s_mov_b32 s9, s1
+; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(1)
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v2, 16, v0
+; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v3, 16, v1
+; GFX11-TRUE16-NEXT:    v_cmp_ge_f16_e32 vcc_lo, v1.l, v0.l
+; GFX11-TRUE16-NEXT:    v_cndmask_b32_e64 v0, 0, -1, vcc_lo
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT:    v_cmp_ge_f16_e32 vcc_lo, v3.l, v2.l
+; GFX11-TRUE16-NEXT:    v_cndmask_b32_e64 v1, 0, -1, vcc_lo
+; GFX11-TRUE16-NEXT:    buffer_store_b64 v[0:1], off, s[8:11], 0
+; GFX11-TRUE16-NEXT:    s_endpgm
+;
+; GFX11-FAKE16-LABEL: fcmp_v2f16_ge:
+; GFX11-FAKE16:       ; %bb.0: ; %entry
+; GFX11-FAKE16-NEXT:    s_clause 0x1
+; GFX11-FAKE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-FAKE16-NEXT:    s_load_b64 s[4:5], s[4:5], 0x34
+; GFX11-FAKE16-NEXT:    s_mov_b32 s10, -1
+; GFX11-FAKE16-NEXT:    s_mov_b32 s11, 0x31016000
+; GFX11-FAKE16-NEXT:    s_mov_b32 s6, s10
+; GFX11-FAKE16-NEXT:    s_mov_b32 s7, s11
+; GFX11-FAKE16-NEXT:    s_mov_b32 s14, s10
+; GFX11-FAKE16-NEXT:    s_mov_b32 s15, s11
+; GFX11-FAKE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-FAKE16-NEXT:    s_mov_b32 s12, s2
+; GFX11-FAKE16-NEXT:    s_mov_b32 s13, s3
+; GFX11-FAKE16-NEXT:    buffer_load_b32 v0, off, s[4:7], 0
+; GFX11-FAKE16-NEXT:    buffer_load_b32 v1, off, s[12:15], 0
+; GFX11-FAKE16-NEXT:    s_mov_b32 s8, s0
+; GFX11-FAKE16-NEXT:    s_mov_b32 s9, s1
+; GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(1)
+; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v2, 16, v0
+; GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v3, 16, v1
+; GFX11-FAKE16-NEXT:    v_cmp_ge_f16_e32 vcc_lo, v1, v0
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e64 v0, 0, -1, vcc_lo
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3)
+; GFX11-FAKE16-NEXT:    v_cmp_ge_f16_e32 vcc_lo, v3, v2
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e64 v1, 0, -1, vcc_lo
+; GFX11-FAKE16-NEXT:    buffer_store_b64 v[0:1], off, s[8:11], 0
+; GFX11-FAKE16-NEXT:    s_endpgm
 ;
 ; GFX12-LABEL: fcmp_v2f16_ge:
 ; GFX12:       ; %bb.0: ; %entry
@@ -2470,6 +3061,7 @@ define amdgpu_kernel void @fcmp_v2f16_ge(
 ; GFX12-NEXT:    v_cndmask_b32_e64 v0, 0, -1, vcc_lo
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3)
 ; GFX12-NEXT:    v_cmp_ge_f16_e32 vcc_lo, v3, v2
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_cndmask_b32_e64 v1, 0, -1, vcc_lo
 ; GFX12-NEXT:    buffer_store_b64 v[0:1], off, s[8:11], null
 ; GFX12-NEXT:    s_endpgm
@@ -2547,35 +3139,65 @@ define amdgpu_kernel void @fcmp_v2f16_o(
 ; VI-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
 ; VI-NEXT:    s_endpgm
 ;
-; GFX11-LABEL: fcmp_v2f16_o:
-; GFX11:       ; %bb.0: ; %entry
-; GFX11-NEXT:    s_clause 0x1
-; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
-; GFX11-NEXT:    s_load_b64 s[4:5], s[4:5], 0x34
-; GFX11-NEXT:    s_mov_b32 s10, -1
-; GFX11-NEXT:    s_mov_b32 s11, 0x31016000
-; GFX11-NEXT:    s_mov_b32 s6, s10
-; GFX11-NEXT:    s_mov_b32 s7, s11
-; GFX11-NEXT:    s_mov_b32 s14, s10
-; GFX11-NEXT:    s_mov_b32 s15, s11
-; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX11-NEXT:    s_mov_b32 s12, s2
-; GFX11-NEXT:    s_mov_b32 s13, s3
-; GFX11-NEXT:    buffer_load_b32 v0, off, s[4:7], 0
-; GFX11-NEXT:    buffer_load_b32 v1, off, s[12:15], 0
-; GFX11-NEXT:    s_mov_b32 s8, s0
-; GFX11-NEXT:    s_mov_b32 s9, s1
-; GFX11-NEXT:    s_waitcnt vmcnt(1)
-; GFX11-NEXT:    v_lshrrev_b32_e32 v2, 16, v0
-; GFX11-NEXT:    s_waitcnt vmcnt(0)
-; GFX11-NEXT:    v_lshrrev_b32_e32 v3, 16, v1
-; GFX11-NEXT:    v_cmp_o_f16_e32 vcc_lo, v1, v0
-; GFX11-NEXT:    v_cndmask_b32_e64 v0, 0, -1, vcc_lo
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_3)
-; GFX11-NEXT:    v_cmp_o_f16_e32 vcc_lo, v3, v2
-; GFX11-NEXT:    v_cndmask_b32_e64 v1, 0, -1, vcc_lo
-; GFX11-NEXT:    buffer_store_b64 v[0:1], off, s[8:11], 0
-; GFX11-NEXT:    s_endpgm
+; GFX11-TRUE16-LABEL: fcmp_v2f16_o:
+; GFX11-TRUE16:       ; %bb.0: ; %entry
+; GFX11-TRUE16-NEXT:    s_clause 0x1
+; GFX11-TRUE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-TRUE16-NEXT:    s_load_b64 s[4:5], s[4:5], 0x34
+; GFX11-TRUE16-NEXT:    s_mov_b32 s10, -1
+; GFX11-TRUE16-NEXT:    s_mov_b32 s11, 0x31016000
+; GFX11-TRUE16-NEXT:    s_mov_b32 s6, s10
+; GFX11-TRUE16-NEXT:    s_mov_b32 s7, s11
+; GFX11-TRUE16-NEXT:    s_mov_b32 s14, s10
+; GFX11-TRUE16-NEXT:    s_mov_b32 s15, s11
+; GFX11-TRUE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-TRUE16-NEXT:    s_mov_b32 s12, s2
+; GFX11-TRUE16-NEXT:    s_mov_b32 s13, s3
+; GFX11-TRUE16-NEXT:    buffer_load_b32 v0, off, s[4:7], 0
+; GFX11-TRUE16-NEXT:    buffer_load_b32 v1, off, s[12:15], 0
+; GFX11-TRUE16-NEXT:    s_mov_b32 s8, s0
+; GFX11-TRUE16-NEXT:    s_mov_b32 s9, s1
+; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(1)
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v2, 16, v0
+; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v3, 16, v1
+; GFX11-TRUE16-NEXT:    v_cmp_o_f16_e32 vcc_lo, v1.l, v0.l
+; GFX11-TRUE16-NEXT:    v_cndmask_b32_e64 v0, 0, -1, vcc_lo
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT:    v_cmp_o_f16_e32 vcc_lo, v3.l, v2.l
+; GFX11-TRUE16-NEXT:    v_cndmask_b32_e64 v1, 0, -1, vcc_lo
+; GFX11-TRUE16-NEXT:    buffer_store_b64 v[0:1], off, s[8:11], 0
+; GFX11-TRUE16-NEXT:    s_endpgm
+;
+; GFX11-FAKE16-LABEL: fcmp_v2f16_o:
+; GFX11-FAKE16:       ; %bb.0: ; %entry
+; GFX11-FAKE16-NEXT:    s_clause 0x1
+; GFX11-FAKE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-FAKE16-NEXT:    s_load_b64 s[4:5], s[4:5], 0x34
+; GFX11-FAKE16-NEXT:    s_mov_b32 s10, -1
+; GFX11-FAKE16-NEXT:    s_mov_b32 s11, 0x31016000
+; GFX11-FAKE16-NEXT:    s_mov_b32 s6, s10
+; GFX11-FAKE16-NEXT:    s_mov_b32 s7, s11
+; GFX11-FAKE16-NEXT:    s_mov_b32 s14, s10
+; GFX11-FAKE16-NEXT:    s_mov_b32 s15, s11
+; GFX11-FAKE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-FAKE16-NEXT:    s_mov_b32 s12, s2
+; GFX11-FAKE16-NEXT:    s_mov_b32 s13, s3
+; GFX11-FAKE16-NEXT:    buffer_load_b32 v0, off, s[4:7], 0
+; GFX11-FAKE16-NEXT:    buffer_load_b32 v1, off, s[12:15], 0
+; GFX11-FAKE16-NEXT:    s_mov_b32 s8, s0
+; GFX11-FAKE16-NEXT:    s_mov_b32 s9, s1
+; GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(1)
+; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v2, 16, v0
+; GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v3, 16, v1
+; GFX11-FAKE16-NEXT:    v_cmp_o_f16_e32 vcc_lo, v1, v0
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e64 v0, 0, -1, vcc_lo
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3)
+; GFX11-FAKE16-NEXT:    v_cmp_o_f16_e32 vcc_lo, v3, v2
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e64 v1, 0, -1, vcc_lo
+; GFX11-FAKE16-NEXT:    buffer_store_b64 v[0:1], off, s[8:11], 0
+; GFX11-FAKE16-NEXT:    s_endpgm
 ;
 ; GFX12-LABEL: fcmp_v2f16_o:
 ; GFX12:       ; %bb.0: ; %entry
@@ -2603,6 +3225,7 @@ define amdgpu_kernel void @fcmp_v2f16_o(
 ; GFX12-NEXT:    v_cndmask_b32_e64 v0, 0, -1, vcc_lo
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3)
 ; GFX12-NEXT:    v_cmp_o_f16_e32 vcc_lo, v3, v2
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_cndmask_b32_e64 v1, 0, -1, vcc_lo
 ; GFX12-NEXT:    buffer_store_b64 v[0:1], off, s[8:11], null
 ; GFX12-NEXT:    s_endpgm
@@ -2680,35 +3303,65 @@ define amdgpu_kernel void @fcmp_v2f16_u(
 ; VI-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
 ; VI-NEXT:    s_endpgm
 ;
-; GFX11-LABEL: fcmp_v2f16_u:
-; GFX11:       ; %bb.0: ; %entry
-; GFX11-NEXT:    s_clause 0x1
-; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
-; GFX11-NEXT:    s_load_b64 s[4:5], s[4:5], 0x34
-; GFX11-NEXT:    s_mov_b32 s10, -1
-; GFX11-NEXT:    s_mov_b32 s11, 0x31016000
-; GFX11-NEXT:    s_mov_b32 s6, s10
-; GFX11-NEXT:    s_mov_b32 s7, s11
-; GFX11-NEXT:    s_mov_b32 s14, s10
-; GFX11-NEXT:    s_mov_b32 s15, s11
-; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX11-NEXT:    s_mov_b32 s12, s2
-; GFX11-NEXT:    s_mov_b32 s13, s3
-; GFX11-NEXT:    buffer_load_b32 v0, off, s[4:7], 0
-; GFX11-NEXT:    buffer_load_b32 v1, off, s[12:15], 0
-; GFX11-NEXT:    s_mov_b32 s8, s0
-; GFX11-NEXT:    s_mov_b32 s9, s1
-; GFX11-NEXT:    s_waitcnt vmcnt(1)
-; GFX11-NEXT:    v_lshrrev_b32_e32 v2, 16, v0
-; GFX11-NEXT:    s_waitcnt vmcnt(0)
-; GFX11-NEXT:    v_lshrrev_b32_e32 v3, 16, v1
-; GFX11-NEXT:    v_cmp_u_f16_e32 vcc_lo, v1, v0
-; GFX11-NEXT:    v_cndmask_b32_e64 v0, 0, -1, vcc_lo
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_3)
-; GFX11-NEXT:    v_cmp_u_f16_e32 vcc_lo, v3, v2
-; GFX11-NEXT:    v_cndmask_b32_e64 v1, 0, -1, vcc_lo
-; GFX11-NEXT:    buffer_store_b64 v[0:1], off, s[8:11], 0
-; GFX11-NEXT:    s_endpgm
+; GFX11-TRUE16-LABEL: fcmp_v2f16_u:
+; GFX11-TRUE16:       ; %bb.0: ; %entry
+; GFX11-TRUE16-NEXT:    s_clause 0x1
+; GFX11-TRUE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-TRUE16-NEXT:    s_load_b64 s[4:5], s[4:5], 0x34
+; GFX11-TRUE16-NEXT:    s_mov_b32 s10, -1
+; GFX11-TRUE16-NEXT:    s_mov_b32 s11, 0x31016000
+; GFX11-TRUE16-NEXT:    s_mov_b32 s6, s10
+; GFX11-TRUE16-NEXT:    s_mov_b32 s7, s11
+; GFX11-TRUE16-NEXT:    s_mov_b32 s14, s10
+; GFX11-TRUE16-NEXT:    s_mov_b32 s15, s11
+; GFX11-TRUE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-TRUE16-NEXT:    s_mov_b32 s12, s2
+; GFX11-TRUE16-NEXT:    s_mov_b32 s13, s3
+; GFX11-TRUE16-NEXT:    buffer_load_b32 v0, off, s[4:7], 0
+; GFX11-TRUE16-NEXT:    buffer_load_b32 v1, off, s[12:15], 0
+; GFX11-TRUE16-NEXT:    s_mov_b32 s8, s0
+; GFX11-TRUE16-NEXT:    s_mov_b32 s9, s1
+; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(1)
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v2, 16, v0
+; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v3, 16, v1
+; GFX11-TRUE16-NEXT:    v_cmp_u_f16_e32 vcc_lo, v1.l, v0.l
+; GFX11-TRUE16-NEXT:    v_cndmask_b32_e64 v0, 0, -1, vcc_lo
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT:    v_cmp_u_f16_e32 vcc_lo, v3.l, v2.l
+; GFX11-TRUE16-NEXT:    v_cndmask_b32_e64 v1, 0, -1, vcc_lo
+; GFX11-TRUE16-NEXT:    buffer_store_b64 v[0:1], off, s[8:11], 0
+; GFX11-TRUE16-NEXT:    s_endpgm
+;
+; GFX11-FAKE16-LABEL: fcmp_v2f16_u:
+; GFX11-FAKE16:       ; %bb.0: ; %entry
+; GFX11-FAKE16-NEXT:    s_clause 0x1
+; GFX11-FAKE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-FAKE16-NEXT:    s_load_b64 s[4:5], s[4:5], 0x34
+; GFX11-FAKE16-NEXT:    s_mov_b32 s10, -1
+; GFX11-FAKE16-NEXT:    s_mov_b32 s11, 0x31016000
+; GFX11-FAKE16-NEXT:    s_mov_b32 s6, s10
+; GFX11-FAKE16-NEXT:    s_mov_b32 s7, s11
+; GFX11-FAKE16-NEXT:    s_mov_b32 s14, s10
+; GFX11-FAKE16-NEXT:    s_mov_b32 s15, s11
+; GFX11-FAKE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-FAKE16-NEXT:    s_mov_b32 s12, s2
+; GFX11-FAKE16-NEXT:    s_mov_b32 s13, s3
+; GFX11-FAKE16-NEXT:    buffer_load_b32 v0, off, s[4:7], 0
+; GFX11-FAKE16-NEXT:    buffer_load_b32 v1, off, s[12:15], 0
+; GFX11-FAKE16-NEXT:    s_mov_b32 s8, s0
+; GFX11-FAKE16-NEXT:    s_mov_b32 s9, s1
+; GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(1)
+; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v2, 16, v0
+; GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v3, 16, v1
+; GFX11-FAKE16-NEXT:    v_cmp_u_f16_e32 vcc_lo, v1, v0
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e64 v0, 0, -1, vcc_lo
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3)
+; GFX11-FAKE16-NEXT:    v_cmp_u_f16_e32 vcc_lo, v3, v2
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e64 v1, 0, -1, vcc_lo
+; GFX11-FAKE16-NEXT:    buffer_store_b64 v[0:1], off, s[8:11], 0
+; GFX11-FAKE16-NEXT:    s_endpgm
 ;
 ; GFX12-LABEL: fcmp_v2f16_u:
 ; GFX12:       ; %bb.0: ; %entry
@@ -2736,6 +3389,7 @@ define amdgpu_kernel void @fcmp_v2f16_u(
 ; GFX12-NEXT:    v_cndmask_b32_e64 v0, 0, -1, vcc_lo
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3)
 ; GFX12-NEXT:    v_cmp_u_f16_e32 vcc_lo, v3, v2
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_cndmask_b32_e64 v1, 0, -1, vcc_lo
 ; GFX12-NEXT:    buffer_store_b64 v[0:1], off, s[8:11], null
 ; GFX12-NEXT:    s_endpgm
@@ -2812,35 +3466,65 @@ define amdgpu_kernel void @fcmp_v2f16_nge(
 ; VI-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
 ; VI-NEXT:    s_endpgm
 ;
-; GFX11-LABEL: fcmp_v2f16_nge:
-; GFX11:       ; %bb.0: ; %entry
-; GFX11-NEXT:    s_clause 0x1
-; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
-; GFX11-NEXT:    s_load_b64 s[4:5], s[4:5], 0x34
-; GFX11-NEXT:    s_mov_b32 s10, -1
-; GFX11-NEXT:    s_mov_b32 s11, 0x31016000
-; GFX11-NEXT:    s_mov_b32 s6, s10
-; GFX11-NEXT:    s_mov_b32 s7, s11
-; GFX11-NEXT:    s_mov_b32 s14, s10
-; GFX11-NEXT:    s_mov_b32 s15, s11
-; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX11-NEXT:    s_mov_b32 s12, s2
-; GFX11-NEXT:    s_mov_b32 s13, s3
-; GFX11-NEXT:    buffer_load_b32 v0, off, s[4:7], 0
-; GFX11-NEXT:    buffer_load_b32 v1, off, s[12:15], 0
-; GFX11-NEXT:    s_mov_b32 s8, s0
-; GFX11-NEXT:    s_mov_b32 s9, s1
-; GFX11-NEXT:    s_waitcnt vmcnt(1)
-; GFX11-NEXT:    v_lshrrev_b32_e32 v2, 16, v0
-; GFX11-NEXT:    s_waitcnt vmcnt(0)
-; GFX11-NEXT:    v_lshrrev_b32_e32 v3, 16, v1
-; GFX11-NEXT:    v_cmp_nge_f16_e32 vcc_lo, v1, v0
-; GFX11-NEXT:    v_cndmask_b32_e64 v0, 0, -1, vcc_lo
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_3)
-; GFX11-NEXT:    v_cmp_nge_f16_e32 vcc_lo, v3, v2
-; GFX11-NEXT:    v_cndmask_b32_e64 v1, 0, -1, vcc_lo
-; GFX11-NEXT:    buffer_store_b64 v[0:1], off, s[8:11], 0
-; GFX11-NEXT:    s_endpgm
+; GFX11-TRUE16-LABEL: fcmp_v2f16_nge:
+; GFX11-TRUE16:       ; %bb.0: ; %entry
+; GFX11-TRUE16-NEXT:    s_clause 0x1
+; GFX11-TRUE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-TRUE16-NEXT:    s_load_b64 s[4:5], s[4:5], 0x34
+; GFX11-TRUE16-NEXT:    s_mov_b32 s10, -1
+; GFX11-TRUE16-NEXT:    s_mov_b32 s11, 0x31016000
+; GFX11-TRUE16-NEXT:    s_mov_b32 s6, s10
+; GFX11-TRUE16-NEXT:    s_mov_b32 s7, s11
+; GFX11-TRUE16-NEXT:    s_mov_b32 s14, s10
+; GFX11-TRUE16-NEXT:    s_mov_b32 s15, s11
+; GFX11-TRUE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-TRUE16-NEXT:    s_mov_b32 s12, s2
+; GFX11-TRUE16-NEXT:    s_mov_b32 s13, s3
+; GFX11-TRUE16-NEXT:    buffer_load_b32 v0, off, s[4:7], 0
+; GFX11-TRUE16-NEXT:    buffer_load_b32 v1, off, s[12:15], 0
+; GFX11-TRUE16-NEXT:    s_mov_b32 s8, s0
+; GFX11-TRUE16-NEXT:    s_mov_b32 s9, s1
+; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(1)
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v2, 16, v0
+; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v3, 16, v1
+; GFX11-TRUE16-NEXT:    v_cmp_nge_f16_e32 vcc_lo, v1.l, v0.l
+; GFX11-TRUE16-NEXT:    v_cndmask_b32_e64 v0, 0, -1, vcc_lo
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT:    v_cmp_nge_f16_e32 vcc_lo, v3.l, v2.l
+; GFX11-TRUE16-NEXT:    v_cndmask_b32_e64 v1, 0, -1, vcc_lo
+; GFX11-TRUE16-NEXT:    buffer_store_b64 v[0:1], off, s[8:11], 0
+; GFX11-TRUE16-NEXT:    s_endpgm
+;
+; GFX11-FAKE16-LABEL: fcmp_v2f16_nge:
+; GFX11-FAKE16:       ; %bb.0: ; %entry
+; GFX11-FAKE16-NEXT:    s_clause 0x1
+; GFX11-FAKE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-FAKE16-NEXT:    s_load_b64 s[4:5], s[4:5], 0x34
+; GFX11-FAKE16-NEXT:    s_mov_b32 s10, -1
+; GFX11-FAKE16-NEXT:    s_mov_b32 s11, 0x31016000
+; GFX11-FAKE16-NEXT:    s_mov_b32 s6, s10
+; GFX11-FAKE16-NEXT:    s_mov_b32 s7, s11
+; GFX11-FAKE16-NEXT:    s_mov_b32 s14, s10
+; GFX11-FAKE16-NEXT:    s_mov_b32 s15, s11
+; GFX11-FAKE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-FAKE16-NEXT:    s_mov_b32 s12, s2
+; GFX11-FAKE16-NEXT:    s_mov_b32 s13, s3
+; GFX11-FAKE16-NEXT:    buffer_load_b32 v0, off, s[4:7], 0
+; GFX11-FAKE16-NEXT:    buffer_load_b32 v1, off, s[12:15], 0
+; GFX11-FAKE16-NEXT:    s_mov_b32 s8, s0
+; GFX11-FAKE16-NEXT:    s_mov_b32 s9, s1
+; GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(1)
+; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v2, 16, v0
+; GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v3, 16, v1
+; GFX11-FAKE16-NEXT:    v_cmp_nge_f16_e32 vcc_lo, v1, v0
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e64 v0, 0, -1, vcc_lo
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3)
+; GFX11-FAKE16-NEXT:    v_cmp_nge_f16_e32 vcc_lo, v3, v2
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e64 v1, 0, -1, vcc_lo
+; GFX11-FAKE16-NEXT:    buffer_store_b64 v[0:1], off, s[8:11], 0
+; GFX11-FAKE16-NEXT:    s_endpgm
 ;
 ; GFX12-LABEL: fcmp_v2f16_nge:
 ; GFX12:       ; %bb.0: ; %entry
@@ -2868,6 +3552,7 @@ define amdgpu_kernel void @fcmp_v2f16_nge(
 ; GFX12-NEXT:    v_cndmask_b32_e64 v0, 0, -1, vcc_lo
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3)
 ; GFX12-NEXT:    v_cmp_nge_f16_e32 vcc_lo, v3, v2
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_cndmask_b32_e64 v1, 0, -1, vcc_lo
 ; GFX12-NEXT:    buffer_store_b64 v[0:1], off, s[8:11], null
 ; GFX12-NEXT:    s_endpgm
@@ -2944,35 +3629,65 @@ define amdgpu_kernel void @fcmp_v2f16_nlg(
 ; VI-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
 ; VI-NEXT:    s_endpgm
 ;
-; GFX11-LABEL: fcmp_v2f16_nlg:
-; GFX11:       ; %bb.0: ; %entry
-; GFX11-NEXT:    s_clause 0x1
-; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
-; GFX11-NEXT:    s_load_b64 s[4:5], s[4:5], 0x34
-; GFX11-NEXT:    s_mov_b32 s10, -1
-; GFX11-NEXT:    s_mov_b32 s11, 0x31016000
-; GFX11-NEXT:    s_mov_b32 s6, s10
-; GFX11-NEXT:    s_mov_b32 s7, s11
-; GFX11-NEXT:    s_mov_b32 s14, s10
-; GFX11-NEXT:    s_mov_b32 s15, s11
-; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX11-NEXT:    s_mov_b32 s12, s2
-; GFX11-NEXT:    s_mov_b32 s13, s3
-; GFX11-NEXT:    buffer_load_b32 v0, off, s[4:7], 0
-; GFX11-NEXT:    buffer_load_b32 v1, off, s[12:15], 0
-; GFX11-NEXT:    s_mov_b32 s8, s0
-; GFX11-NEXT:    s_mov_b32 s9, s1
-; GFX11-NEXT:    s_waitcnt vmcnt(1)
-; GFX11-NEXT:    v_lshrrev_b32_e32 v2, 16, v0
-; GFX11-NEXT:    s_waitcnt vmcnt(0)
-; GFX11-NEXT:    v_lshrrev_b32_e32 v3, 16, v1
-; GFX11-NEXT:    v_cmp_nlg_f16_e32 vcc_lo, v1, v0
-; GFX11-NEXT:    v_cndmask_b32_e64 v0, 0, -1, vcc_lo
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_3)
-; GFX11-NEXT:    v_cmp_nlg_f16_e32 vcc_lo, v3, v2
-; GFX11-NEXT:    v_cndmask_b32_e64 v1, 0, -1, vcc_lo
-; GFX11-NEXT:    buffer_store_b64 v[0:1], off, s[8:11], 0
-; GFX11-NEXT:    s_endpgm
+; GFX11-TRUE16-LABEL: fcmp_v2f16_nlg:
+; GFX11-TRUE16:       ; %bb.0: ; %entry
+; GFX11-TRUE16-NEXT:    s_clause 0x1
+; GFX11-TRUE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-TRUE16-NEXT:    s_load_b64 s[4:5], s[4:5], 0x34
+; GFX11-TRUE16-NEXT:    s_mov_b32 s10, -1
+; GFX11-TRUE16-NEXT:    s_mov_b32 s11, 0x31016000
+; GFX11-TRUE16-NEXT:    s_mov_b32 s6, s10
+; GFX11-TRUE16-NEXT:    s_mov_b32 s7, s11
+; GFX11-TRUE16-NEXT:    s_mov_b32 s14, s10
+; GFX11-TRUE16-NEXT:    s_mov_b32 s15, s11
+; GFX11-TRUE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-TRUE16-NEXT:    s_mov_b32 s12, s2
+; GFX11-TRUE16-NEXT:    s_mov_b32 s13, s3
+; GFX11-TRUE16-NEXT:    buffer_load_b32 v0, off, s[4:7], 0
+; GFX11-TRUE16-NEXT:    buffer_load_b32 v1, off, s[12:15], 0
+; GFX11-TRUE16-NEXT:    s_mov_b32 s8, s0
+; GFX11-TRUE16-NEXT:    s_mov_b32 s9, s1
+; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(1)
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v2, 16, v0
+; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v3, 16, v1
+; GFX11-TRUE16-NEXT:    v_cmp_nlg_f16_e32 vcc_lo, v1.l, v0.l
+; GFX11-TRUE16-NEXT:    v_cndmask_b32_e64 v0, 0, -1, vcc_lo
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT:    v_cmp_nlg_f16_e32 vcc_lo, v3.l, v2.l
+; GFX11-TRUE16-NEXT:    v_cndmask_b32_e64 v1, 0, -1, vcc_lo
+; GFX11-TRUE16-NEXT:    buffer_store_b64 v[0:1], off, s[8:11], 0
+; GFX11-TRUE16-NEXT:    s_endpgm
+;
+; GFX11-FAKE16-LABEL: fcmp_v2f16_nlg:
+; GFX11-FAKE16:       ; %bb.0: ; %entry
+; GFX11-FAKE16-NEXT:    s_clause 0x1
+; GFX11-FAKE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-FAKE16-NEXT:    s_load_b64 s[4:5], s[4:5], 0x34
+; GFX11-FAKE16-NEXT:    s_mov_b32 s10, -1
+; GFX11-FAKE16-NEXT:    s_mov_b32 s11, 0x31016000
+; GFX11-FAKE16-NEXT:    s_mov_b32 s6, s10
+; GFX11-FAKE16-NEXT:    s_mov_b32 s7, s11
+; GFX11-FAKE16-NEXT:    s_mov_b32 s14, s10
+; GFX11-FAKE16-NEXT:    s_mov_b32 s15, s11
+; GFX11-FAKE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-FAKE16-NEXT:    s_mov_b32 s12, s2
+; GFX11-FAKE16-NEXT:    s_mov_b32 s13, s3
+; GFX11-FAKE16-NEXT:    buffer_load_b32 v0, off, s[4:7], 0
+; GFX11-FAKE16-NEXT:    buffer_load_b32 v1, off, s[12:15], 0
+; GFX11-FAKE16-NEXT:    s_mov_b32 s8, s0
+; GFX11-FAKE16-NEXT:    s_mov_b32 s9, s1
+; GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(1)
+; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v2, 16, v0
+; GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v3, 16, v1
+; GFX11-FAKE16-NEXT:    v_cmp_nlg_f16_e32 vcc_lo, v1, v0
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e64 v0, 0, -1, vcc_lo
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3)
+; GFX11-FAKE16-NEXT:    v_cmp_nlg_f16_e32 vcc_lo, v3, v2
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e64 v1, 0, -1, vcc_lo
+; GFX11-FAKE16-NEXT:    buffer_store_b64 v[0:1], off, s[8:11], 0
+; GFX11-FAKE16-NEXT:    s_endpgm
 ;
 ; GFX12-LABEL: fcmp_v2f16_nlg:
 ; GFX12:       ; %bb.0: ; %entry
@@ -3000,6 +3715,7 @@ define amdgpu_kernel void @fcmp_v2f16_nlg(
 ; GFX12-NEXT:    v_cndmask_b32_e64 v0, 0, -1, vcc_lo
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3)
 ; GFX12-NEXT:    v_cmp_nlg_f16_e32 vcc_lo, v3, v2
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_cndmask_b32_e64 v1, 0, -1, vcc_lo
 ; GFX12-NEXT:    buffer_store_b64 v[0:1], off, s[8:11], null
 ; GFX12-NEXT:    s_endpgm
@@ -3077,35 +3793,65 @@ define amdgpu_kernel void @fcmp_v2f16_ngt(
 ; VI-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
 ; VI-NEXT:    s_endpgm
 ;
-; GFX11-LABEL: fcmp_v2f16_ngt:
-; GFX11:       ; %bb.0: ; %entry
-; GFX11-NEXT:    s_clause 0x1
-; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
-; GFX11-NEXT:    s_load_b64 s[4:5], s[4:5], 0x34
-; GFX11-NEXT:    s_mov_b32 s10, -1
-; GFX11-NEXT:    s_mov_b32 s11, 0x31016000
-; GFX11-NEXT:    s_mov_b32 s6, s10
-; GFX11-NEXT:    s_mov_b32 s7, s11
-; GFX11-NEXT:    s_mov_b32 s14, s10
-; GFX11-NEXT:    s_mov_b32 s15, s11
-; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX11-NEXT:    s_mov_b32 s12, s2
-; GFX11-NEXT:    s_mov_b32 s13, s3
-; GFX11-NEXT:    buffer_load_b32 v0, off, s[4:7], 0
-; GFX11-NEXT:    buffer_load_b32 v1, off, s[12:15], 0
-; GFX11-NEXT:    s_mov_b32 s8, s0
-; GFX11-NEXT:    s_mov_b32 s9, s1
-; GFX11-NEXT:    s_waitcnt vmcnt(1)
-; GFX11-NEXT:    v_lshrrev_b32_e32 v2, 16, v0
-; GFX11-NEXT:    s_waitcnt vmcnt(0)
-; GFX11-NEXT:    v_lshrrev_b32_e32 v3, 16, v1
-; GFX11-NEXT:    v_cmp_ngt_f16_e32 vcc_lo, v1, v0
-; GFX11-NEXT:    v_cndmask_b32_e64 v0, 0, -1, vcc_lo
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_3)
-; GFX11-NEXT:    v_cmp_ngt_f16_e32 vcc_lo, v3, v2
-; GFX11-NEXT:    v_cndmask_b32_e64 v1, 0, -1, vcc_lo
-; GFX11-NEXT:    buffer_store_b64 v[0:1], off, s[8:11], 0
-; GFX11-NEXT:    s_endpgm
+; GFX11-TRUE16-LABEL: fcmp_v2f16_ngt:
+; GFX11-TRUE16:       ; %bb.0: ; %entry
+; GFX11-TRUE16-NEXT:    s_clause 0x1
+; GFX11-TRUE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-TRUE16-NEXT:    s_load_b64 s[4:5], s[4:5], 0x34
+; GFX11-TRUE16-NEXT:    s_mov_b32 s10, -1
+; GFX11-TRUE16-NEXT:    s_mov_b32 s11, 0x31016000
+; GFX11-TRUE16-NEXT:    s_mov_b32 s6, s10
+; GFX11-TRUE16-NEXT:    s_mov_b32 s7, s11
+; GFX11-TRUE16-NEXT:    s_mov_b32 s14, s10
+; GFX11-TRUE16-NEXT:    s_mov_b32 s15, s11
+; GFX11-TRUE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-TRUE16-NEXT:    s_mov_b32 s12, s2
+; GFX11-TRUE16-NEXT:    s_mov_b32 s13, s3
+; GFX11-TRUE16-NEXT:    buffer_load_b32 v0, off, s[4:7], 0
+; GFX11-TRUE16-NEXT:    buffer_load_b32 v1, off, s[12:15], 0
+; GFX11-TRUE16-NEXT:    s_mov_b32 s8, s0
+; GFX11-TRUE16-NEXT:    s_mov_b32 s9, s1
+; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(1)
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v2, 16, v0
+; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v3, 16, v1
+; GFX11-TRUE16-NEXT:    v_cmp_ngt_f16_e32 vcc_lo, v1.l, v0.l
+; GFX11-TRUE16-NEXT:    v_cndmask_b32_e64 v0, 0, -1, vcc_lo
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT:    v_cmp_ngt_f16_e32 vcc_lo, v3.l, v2.l
+; GFX11-TRUE16-NEXT:    v_cndmask_b32_e64 v1, 0, -1, vcc_lo
+; GFX11-TRUE16-NEXT:    buffer_store_b64 v[0:1], off, s[8:11], 0
+; GFX11-TRUE16-NEXT:    s_endpgm
+;
+; GFX11-FAKE16-LABEL: fcmp_v2f16_ngt:
+; GFX11-FAKE16:       ; %bb.0: ; %entry
+; GFX11-FAKE16-NEXT:    s_clause 0x1
+; GFX11-FAKE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-FAKE16-NEXT:    s_load_b64 s[4:5], s[4:5], 0x34
+; GFX11-FAKE16-NEXT:    s_mov_b32 s10, -1
+; GFX11-FAKE16-NEXT:    s_mov_b32 s11, 0x31016000
+; GFX11-FAKE16-NEXT:    s_mov_b32 s6, s10
+; GFX11-FAKE16-NEXT:    s_mov_b32 s7, s11
+; GFX11-FAKE16-NEXT:    s_mov_b32 s14, s10
+; GFX11-FAKE16-NEXT:    s_mov_b32 s15, s11
+; GFX11-FAKE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-FAKE16-NEXT:    s_mov_b32 s12, s2
+; GFX11-FAKE16-NEXT:    s_mov_b32 s13, s3
+; GFX11-FAKE16-NEXT:    buffer_load_b32 v0, off, s[4:7], 0
+; GFX11-FAKE16-NEXT:    buffer_load_b32 v1, off, s[12:15], 0
+; GFX11-FAKE16-NEXT:    s_mov_b32 s8, s0
+; GFX11-FAKE16-NEXT:    s_mov_b32 s9, s1
+; GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(1)
+; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v2, 16, v0
+; GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v3, 16, v1
+; GFX11-FAKE16-NEXT:    v_cmp_ngt_f16_e32 vcc_lo, v1, v0
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e64 v0, 0, -1, vcc_lo
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3)
+; GFX11-FAKE16-NEXT:    v_cmp_ngt_f16_e32 vcc_lo, v3, v2
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e64 v1, 0, -1, vcc_lo
+; GFX11-FAKE16-NEXT:    buffer_store_b64 v[0:1], off, s[8:11], 0
+; GFX11-FAKE16-NEXT:    s_endpgm
 ;
 ; GFX12-LABEL: fcmp_v2f16_ngt:
 ; GFX12:       ; %bb.0: ; %entry
@@ -3133,6 +3879,7 @@ define amdgpu_kernel void @fcmp_v2f16_ngt(
 ; GFX12-NEXT:    v_cndmask_b32_e64 v0, 0, -1, vcc_lo
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3)
 ; GFX12-NEXT:    v_cmp_ngt_f16_e32 vcc_lo, v3, v2
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_cndmask_b32_e64 v1, 0, -1, vcc_lo
 ; GFX12-NEXT:    buffer_store_b64 v[0:1], off, s[8:11], null
 ; GFX12-NEXT:    s_endpgm
@@ -3209,35 +3956,65 @@ define amdgpu_kernel void @fcmp_v2f16_nle(
 ; VI-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
 ; VI-NEXT:    s_endpgm
 ;
-; GFX11-LABEL: fcmp_v2f16_nle:
-; GFX11:       ; %bb.0: ; %entry
-; GFX11-NEXT:    s_clause 0x1
-; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
-; GFX11-NEXT:    s_load_b64 s[4:5], s[4:5], 0x34
-; GFX11-NEXT:    s_mov_b32 s10, -1
-; GFX11-NEXT:    s_mov_b32 s11, 0x31016000
-; GFX11-NEXT:    s_mov_b32 s6, s10
-; GFX11-NEXT:    s_mov_b32 s7, s11
-; GFX11-NEXT:    s_mov_b32 s14, s10
-; GFX11-NEXT:    s_mov_b32 s15, s11
-; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX11-NEXT:    s_mov_b32 s12, s2
-; GFX11-NEXT:    s_mov_b32 s13, s3
-; GFX11-NEXT:    buffer_load_b32 v0, off, s[4:7], 0
-; GFX11-NEXT:    buffer_load_b32 v1, off, s[12:15], 0
-; GFX11-NEXT:    s_mov_b32 s8, s0
-; GFX11-NEXT:    s_mov_b32 s9, s1
-; GFX11-NEXT:    s_waitcnt vmcnt(1)
-; GFX11-NEXT:    v_lshrrev_b32_e32 v2, 16, v0
-; GFX11-NEXT:    s_waitcnt vmcnt(0)
-; GFX11-NEXT:    v_lshrrev_b32_e32 v3, 16, v1
-; GFX11-NEXT:    v_cmp_nle_f16_e32 vcc_lo, v1, v0
-; GFX11-NEXT:    v_cndmask_b32_e64 v0, 0, -1, vcc_lo
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_3)
-; GFX11-NEXT:    v_cmp_nle_f16_e32 vcc_lo, v3, v2
-; GFX11-NEXT:    v_cndmask_b32_e64 v1, 0, -1, vcc_lo
-; GFX11-NEXT:    buffer_store_b64 v[0:1], off, s[8:11], 0
-; GFX11-NEXT:    s_endpgm
+; GFX11-TRUE16-LABEL: fcmp_v2f16_nle:
+; GFX11-TRUE16:       ; %bb.0: ; %entry
+; GFX11-TRUE16-NEXT:    s_clause 0x1
+; GFX11-TRUE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-TRUE16-NEXT:    s_load_b64 s[4:5], s[4:5], 0x34
+; GFX11-TRUE16-NEXT:    s_mov_b32 s10, -1
+; GFX11-TRUE16-NEXT:    s_mov_b32 s11, 0x31016000
+; GFX11-TRUE16-NEXT:    s_mov_b32 s6, s10
+; GFX11-TRUE16-NEXT:    s_mov_b32 s7, s11
+; GFX11-TRUE16-NEXT:    s_mov_b32 s14, s10
+; GFX11-TRUE16-NEXT:    s_mov_b32 s15, s11
+; GFX11-TRUE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-TRUE16-NEXT:    s_mov_b32 s12, s2
+; GFX11-TRUE16-NEXT:    s_mov_b32 s13, s3
+; GFX11-TRUE16-NEXT:    buffer_load_b32 v0, off, s[4:7], 0
+; GFX11-TRUE16-NEXT:    buffer_load_b32 v1, off, s[12:15], 0
+; GFX11-TRUE16-NEXT:    s_mov_b32 s8, s0
+; GFX11-TRUE16-NEXT:    s_mov_b32 s9, s1
+; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(1)
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v2, 16, v0
+; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v3, 16, v1
+; GFX11-TRUE16-NEXT:    v_cmp_nle_f16_e32 vcc_lo, v1.l, v0.l
+; GFX11-TRUE16-NEXT:    v_cndmask_b32_e64 v0, 0, -1, vcc_lo
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT:    v_cmp_nle_f16_e32 vcc_lo, v3.l, v2.l
+; GFX11-TRUE16-NEXT:    v_cndmask_b32_e64 v1, 0, -1, vcc_lo
+; GFX11-TRUE16-NEXT:    buffer_store_b64 v[0:1], off, s[8:11], 0
+; GFX11-TRUE16-NEXT:    s_endpgm
+;
+; GFX11-FAKE16-LABEL: fcmp_v2f16_nle:
+; GFX11-FAKE16:       ; %bb.0: ; %entry
+; GFX11-FAKE16-NEXT:    s_clause 0x1
+; GFX11-FAKE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-FAKE16-NEXT:    s_load_b64 s[4:5], s[4:5], 0x34
+; GFX11-FAKE16-NEXT:    s_mov_b32 s10, -1
+; GFX11-FAKE16-NEXT:    s_mov_b32 s11, 0x31016000
+; GFX11-FAKE16-NEXT:    s_mov_b32 s6, s10
+; GFX11-FAKE16-NEXT:    s_mov_b32 s7, s11
+; GFX11-FAKE16-NEXT:    s_mov_b32 s14, s10
+; GFX11-FAKE16-NEXT:    s_mov_b32 s15, s11
+; GFX11-FAKE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-FAKE16-NEXT:    s_mov_b32 s12, s2
+; GFX11-FAKE16-NEXT:    s_mov_b32 s13, s3
+; GFX11-FAKE16-NEXT:    buffer_load_b32 v0, off, s[4:7], 0
+; GFX11-FAKE16-NEXT:    buffer_load_b32 v1, off, s[12:15], 0
+; GFX11-FAKE16-NEXT:    s_mov_b32 s8, s0
+; GFX11-FAKE16-NEXT:    s_mov_b32 s9, s1
+; GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(1)
+; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v2, 16, v0
+; GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v3, 16, v1
+; GFX11-FAKE16-NEXT:    v_cmp_nle_f16_e32 vcc_lo, v1, v0
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e64 v0, 0, -1, vcc_lo
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3)
+; GFX11-FAKE16-NEXT:    v_cmp_nle_f16_e32 vcc_lo, v3, v2
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e64 v1, 0, -1, vcc_lo
+; GFX11-FAKE16-NEXT:    buffer_store_b64 v[0:1], off, s[8:11], 0
+; GFX11-FAKE16-NEXT:    s_endpgm
 ;
 ; GFX12-LABEL: fcmp_v2f16_nle:
 ; GFX12:       ; %bb.0: ; %entry
@@ -3265,6 +4042,7 @@ define amdgpu_kernel void @fcmp_v2f16_nle(
 ; GFX12-NEXT:    v_cndmask_b32_e64 v0, 0, -1, vcc_lo
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3)
 ; GFX12-NEXT:    v_cmp_nle_f16_e32 vcc_lo, v3, v2
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_cndmask_b32_e64 v1, 0, -1, vcc_lo
 ; GFX12-NEXT:    buffer_store_b64 v[0:1], off, s[8:11], null
 ; GFX12-NEXT:    s_endpgm
@@ -3341,35 +4119,65 @@ define amdgpu_kernel void @fcmp_v2f16_neq(
 ; VI-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
 ; VI-NEXT:    s_endpgm
 ;
-; GFX11-LABEL: fcmp_v2f16_neq:
-; GFX11:       ; %bb.0: ; %entry
-; GFX11-NEXT:    s_clause 0x1
-; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
-; GFX11-NEXT:    s_load_b64 s[4:5], s[4:5], 0x34
-; GFX11-NEXT:    s_mov_b32 s10, -1
-; GFX11-NEXT:    s_mov_b32 s11, 0x31016000
-; GFX11-NEXT:    s_mov_b32 s6, s10
-; GFX11-NEXT:    s_mov_b32 s7, s11
-; GFX11-NEXT:    s_mov_b32 s14, s10
-; GFX11-NEXT:    s_mov_b32 s15, s11
-; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX11-NEXT:    s_mov_b32 s12, s2
-; GFX11-NEXT:    s_mov_b32 s13, s3
-; GFX11-NEXT:    buffer_load_b32 v0, off, s[4:7], 0
-; GFX11-NEXT:    buffer_load_b32 v1, off, s[12:15], 0
-; GFX11-NEXT:    s_mov_b32 s8, s0
-; GFX11-NEXT:    s_mov_b32 s9, s1
-; GFX11-NEXT:    s_waitcnt vmcnt(1)
-; GFX11-NEXT:    v_lshrrev_b32_e32 v2, 16, v0
-; GFX11-NEXT:    s_waitcnt vmcnt(0)
-; GFX11-NEXT:    v_lshrrev_b32_e32 v3, 16, v1
-; GFX11-NEXT:    v_cmp_neq_f16_e32 vcc_lo, v1, v0
-; GFX11-NEXT:    v_cndmask_b32_e64 v0, 0, -1, vcc_lo
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_3)
-; GFX11-NEXT:    v_cmp_neq_f16_e32 vcc_lo, v3, v2
-; GFX11-NEXT:    v_cndmask_b32_e64 v1, 0, -1, vcc_lo
-; GFX11-NEXT:    buffer_store_b64 v[0:1], off, s[8:11], 0
-; GFX11-NEXT:    s_endpgm
+; GFX11-TRUE16-LABEL: fcmp_v2f16_neq:
+; GFX11-TRUE16:       ; %bb.0: ; %entry
+; GFX11-TRUE16-NEXT:    s_clause 0x1
+; GFX11-TRUE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-TRUE16-NEXT:    s_load_b64 s[4:5], s[4:5], 0x34
+; GFX11-TRUE16-NEXT:    s_mov_b32 s10, -1
+; GFX11-TRUE16-NEXT:    s_mov_b32 s11, 0x31016000
+; GFX11-TRUE16-NEXT:    s_mov_b32 s6, s10
+; GFX11-TRUE16-NEXT:    s_mov_b32 s7, s11
+; GFX11-TRUE16-NEXT:    s_mov_b32 s14, s10
+; GFX11-TRUE16-NEXT:    s_mov_b32 s15, s11
+; GFX11-TRUE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-TRUE16-NEXT:    s_mov_b32 s12, s2
+; GFX11-TRUE16-NEXT:    s_mov_b32 s13, s3
+; GFX11-TRUE16-NEXT:    buffer_load_b32 v0, off, s[4:7], 0
+; GFX11-TRUE16-NEXT:    buffer_load_b32 v1, off, s[12:15], 0
+; GFX11-TRUE16-NEXT:    s_mov_b32 s8, s0
+; GFX11-TRUE16-NEXT:    s_mov_b32 s9, s1
+; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(1)
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v2, 16, v0
+; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v3, 16, v1
+; GFX11-TRUE16-NEXT:    v_cmp_neq_f16_e32 vcc_lo, v1.l, v0.l
+; GFX11-TRUE16-NEXT:    v_cndmask_b32_e64 v0, 0, -1, vcc_lo
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT:    v_cmp_neq_f16_e32 vcc_lo, v3.l, v2.l
+; GFX11-TRUE16-NEXT:    v_cndmask_b32_e64 v1, 0, -1, vcc_lo
+; GFX11-TRUE16-NEXT:    buffer_store_b64 v[0:1], off, s[8:11], 0
+; GFX11-TRUE16-NEXT:    s_endpgm
+;
+; GFX11-FAKE16-LABEL: fcmp_v2f16_neq:
+; GFX11-FAKE16:       ; %bb.0: ; %entry
+; GFX11-FAKE16-NEXT:    s_clause 0x1
+; GFX11-FAKE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-FAKE16-NEXT:    s_load_b64 s[4:5], s[4:5], 0x34
+; GFX11-FAKE16-NEXT:    s_mov_b32 s10, -1
+; GFX11-FAKE16-NEXT:    s_mov_b32 s11, 0x31016000
+; GFX11-FAKE16-NEXT:    s_mov_b32 s6, s10
+; GFX11-FAKE16-NEXT:    s_mov_b32 s7, s11
+; GFX11-FAKE16-NEXT:    s_mov_b32 s14, s10
+; GFX11-FAKE16-NEXT:    s_mov_b32 s15, s11
+; GFX11-FAKE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-FAKE16-NEXT:    s_mov_b32 s12, s2
+; GFX11-FAKE16-NEXT:    s_mov_b32 s13, s3
+; GFX11-FAKE16-NEXT:    buffer_load_b32 v0, off, s[4:7], 0
+; GFX11-FAKE16-NEXT:    buffer_load_b32 v1, off, s[12:15], 0
+; GFX11-FAKE16-NEXT:    s_mov_b32 s8, s0
+; GFX11-FAKE16-NEXT:    s_mov_b32 s9, s1
+; GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(1)
+; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v2, 16, v0
+; GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v3, 16, v1
+; GFX11-FAKE16-NEXT:    v_cmp_neq_f16_e32 vcc_lo, v1, v0
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e64 v0, 0, -1, vcc_lo
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3)
+; GFX11-FAKE16-NEXT:    v_cmp_neq_f16_e32 vcc_lo, v3, v2
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e64 v1, 0, -1, vcc_lo
+; GFX11-FAKE16-NEXT:    buffer_store_b64 v[0:1], off, s[8:11], 0
+; GFX11-FAKE16-NEXT:    s_endpgm
 ;
 ; GFX12-LABEL: fcmp_v2f16_neq:
 ; GFX12:       ; %bb.0: ; %entry
@@ -3397,6 +4205,7 @@ define amdgpu_kernel void @fcmp_v2f16_neq(
 ; GFX12-NEXT:    v_cndmask_b32_e64 v0, 0, -1, vcc_lo
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3)
 ; GFX12-NEXT:    v_cmp_neq_f16_e32 vcc_lo, v3, v2
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_cndmask_b32_e64 v1, 0, -1, vcc_lo
 ; GFX12-NEXT:    buffer_store_b64 v[0:1], off, s[8:11], null
 ; GFX12-NEXT:    s_endpgm
@@ -3473,35 +4282,65 @@ define amdgpu_kernel void @fcmp_v2f16_nlt(
 ; VI-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
 ; VI-NEXT:    s_endpgm
 ;
-; GFX11-LABEL: fcmp_v2f16_nlt:
-; GFX11:       ; %bb.0: ; %entry
-; GFX11-NEXT:    s_clause 0x1
-; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
-; GFX11-NEXT:    s_load_b64 s[4:5], s[4:5], 0x34
-; GFX11-NEXT:    s_mov_b32 s10, -1
-; GFX11-NEXT:    s_mov_b32 s11, 0x31016000
-; GFX11-NEXT:    s_mov_b32 s6, s10
-; GFX11-NEXT:    s_mov_b32 s7, s11
-; GFX11-NEXT:    s_mov_b32 s14, s10
-; GFX11-NEXT:    s_mov_b32 s15, s11
-; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX11-NEXT:    s_mov_b32 s12, s2
-; GFX11-NEXT:    s_mov_b32 s13, s3
-; GFX11-NEXT:    buffer_load_b32 v0, off, s[4:7], 0
-; GFX11-NEXT:    buffer_load_b32 v1, off, s[12:15], 0
-; GFX11-NEXT:    s_mov_b32 s8, s0
-; GFX11-NEXT:    s_mov_b32 s9, s1
-; GFX11-NEXT:    s_waitcnt vmcnt(1)
-; GFX11-NEXT:    v_lshrrev_b32_e32 v2, 16, v0
-; GFX11-NEXT:    s_waitcnt vmcnt(0)
-; GFX11-NEXT:    v_lshrrev_b32_e32 v3, 16, v1
-; GFX11-NEXT:    v_cmp_nlt_f16_e32 vcc_lo, v1, v0
-; GFX11-NEXT:    v_cndmask_b32_e64 v0, 0, -1, vcc_lo
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_3)
-; GFX11-NEXT:    v_cmp_nlt_f16_e32 vcc_lo, v3, v2
-; GFX11-NEXT:    v_cndmask_b32_e64 v1, 0, -1, vcc_lo
-; GFX11-NEXT:    buffer_store_b64 v[0:1], off, s[8:11], 0
-; GFX11-NEXT:    s_endpgm
+; GFX11-TRUE16-LABEL: fcmp_v2f16_nlt:
+; GFX11-TRUE16:       ; %bb.0: ; %entry
+; GFX11-TRUE16-NEXT:    s_clause 0x1
+; GFX11-TRUE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-TRUE16-NEXT:    s_load_b64 s[4:5], s[4:5], 0x34
+; GFX11-TRUE16-NEXT:    s_mov_b32 s10, -1
+; GFX11-TRUE16-NEXT:    s_mov_b32 s11, 0x31016000
+; GFX11-TRUE16-NEXT:    s_mov_b32 s6, s10
+; GFX11-TRUE16-NEXT:    s_mov_b32 s7, s11
+; GFX11-TRUE16-NEXT:    s_mov_b32 s14, s10
+; GFX11-TRUE16-NEXT:    s_mov_b32 s15, s11
+; GFX11-TRUE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-TRUE16-NEXT:    s_mov_b32 s12, s2
+; GFX11-TRUE16-NEXT:    s_mov_b32 s13, s3
+; GFX11-TRUE16-NEXT:    buffer_load_b32 v0, off, s[4:7], 0
+; GFX11-TRUE16-NEXT:    buffer_load_b32 v1, off, s[12:15], 0
+; GFX11-TRUE16-NEXT:    s_mov_b32 s8, s0
+; GFX11-TRUE16-NEXT:    s_mov_b32 s9, s1
+; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(1)
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v2, 16, v0
+; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v3, 16, v1
+; GFX11-TRUE16-NEXT:    v_cmp_nlt_f16_e32 vcc_lo, v1.l, v0.l
+; GFX11-TRUE16-NEXT:    v_cndmask_b32_e64 v0, 0, -1, vcc_lo
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT:    v_cmp_nlt_f16_e32 vcc_lo, v3.l, v2.l
+; GFX11-TRUE16-NEXT:    v_cndmask_b32_e64 v1, 0, -1, vcc_lo
+; GFX11-TRUE16-NEXT:    buffer_store_b64 v[0:1], off, s[8:11], 0
+; GFX11-TRUE16-NEXT:    s_endpgm
+;
+; GFX11-FAKE16-LABEL: fcmp_v2f16_nlt:
+; GFX11-FAKE16:       ; %bb.0: ; %entry
+; GFX11-FAKE16-NEXT:    s_clause 0x1
+; GFX11-FAKE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-FAKE16-NEXT:    s_load_b64 s[4:5], s[4:5], 0x34
+; GFX11-FAKE16-NEXT:    s_mov_b32 s10, -1
+; GFX11-FAKE16-NEXT:    s_mov_b32 s11, 0x31016000
+; GFX11-FAKE16-NEXT:    s_mov_b32 s6, s10
+; GFX11-FAKE16-NEXT:    s_mov_b32 s7, s11
+; GFX11-FAKE16-NEXT:    s_mov_b32 s14, s10
+; GFX11-FAKE16-NEXT:    s_mov_b32 s15, s11
+; GFX11-FAKE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-FAKE16-NEXT:    s_mov_b32 s12, s2
+; GFX11-FAKE16-NEXT:    s_mov_b32 s13, s3
+; GFX11-FAKE16-NEXT:    buffer_load_b32 v0, off, s[4:7], 0
+; GFX11-FAKE16-NEXT:    buffer_load_b32 v1, off, s[12:15], 0
+; GFX11-FAKE16-NEXT:    s_mov_b32 s8, s0
+; GFX11-FAKE16-NEXT:    s_mov_b32 s9, s1
+; GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(1)
+; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v2, 16, v0
+; GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v3, 16, v1
+; GFX11-FAKE16-NEXT:    v_cmp_nlt_f16_e32 vcc_lo, v1, v0
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e64 v0, 0, -1, vcc_lo
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3)
+; GFX11-FAKE16-NEXT:    v_cmp_nlt_f16_e32 vcc_lo, v3, v2
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e64 v1, 0, -1, vcc_lo
+; GFX11-FAKE16-NEXT:    buffer_store_b64 v[0:1], off, s[8:11], 0
+; GFX11-FAKE16-NEXT:    s_endpgm
 ;
 ; GFX12-LABEL: fcmp_v2f16_nlt:
 ; GFX12:       ; %bb.0: ; %entry
@@ -3529,6 +4368,7 @@ define amdgpu_kernel void @fcmp_v2f16_nlt(
 ; GFX12-NEXT:    v_cndmask_b32_e64 v0, 0, -1, vcc_lo
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3)
 ; GFX12-NEXT:    v_cmp_nlt_f16_e32 vcc_lo, v3, v2
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_cndmask_b32_e64 v1, 0, -1, vcc_lo
 ; GFX12-NEXT:    buffer_store_b64 v[0:1], off, s[8:11], null
 ; GFX12-NEXT:    s_endpgm
diff --git a/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fadd.ll b/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fadd.ll
index 72f883928cffb..707cae9534830 100644
--- a/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fadd.ll
+++ b/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fadd.ll
@@ -5705,6 +5705,7 @@ define double @flat_agent_atomic_fadd_ret_f64__amdgpu_no_fine_grained_memory(ptr
 ; GFX12-NEXT:    s_cbranch_execz .LBB30_6
 ; GFX12-NEXT:  ; %bb.5: ; %atomicrmw.private
 ; GFX12-NEXT:    v_cmp_ne_u64_e32 vcc_lo, 0, v[0:1]
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_cndmask_b32_e32 v6, -1, v0, vcc_lo
 ; GFX12-NEXT:    scratch_load_b64 v[4:5], v6, off
 ; GFX12-NEXT:    s_wait_loadcnt 0x0
@@ -5714,7 +5715,6 @@ define double @flat_agent_atomic_fadd_ret_f64__amdgpu_no_fine_grained_memory(ptr
 ; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX12-NEXT:    v_dual_mov_b32 v0, v4 :: v_dual_mov_b32 v1, v5
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: flat_agent_atomic_fadd_ret_f64__amdgpu_no_fine_grained_memory:
@@ -6059,6 +6059,7 @@ define double @flat_agent_atomic_fadd_ret_f64__offset12b_pos__amdgpu_no_fine_gra
 ; GFX12-NEXT:    s_wait_bvhcnt 0x0
 ; GFX12-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-NEXT:    v_add_co_u32 v4, vcc_lo, 0x7f8, v0
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_add_co_ci_u32_e32 v5, vcc_lo, 0, v1, vcc_lo
 ; GFX12-NEXT:    s_mov_b64 s[0:1], src_private_base
 ; GFX12-NEXT:    s_mov_b32 s0, exec_lo
@@ -6075,7 +6076,7 @@ define double @flat_agent_atomic_fadd_ret_f64__offset12b_pos__amdgpu_no_fine_gra
 ; GFX12-NEXT:  .LBB31_2: ; %atomicrmw.phi
 ; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT:    s_wait_alu 0xfffe
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ; GFX12-NEXT:  .LBB31_3: ; %atomicrmw.global
 ; GFX12-NEXT:    flat_load_b64 v[0:1], v[4:5]
@@ -6104,6 +6105,7 @@ define double @flat_agent_atomic_fadd_ret_f64__offset12b_pos__amdgpu_no_fine_gra
 ; GFX12-NEXT:    s_cbranch_execz .LBB31_2
 ; GFX12-NEXT:  .LBB31_6: ; %atomicrmw.private
 ; GFX12-NEXT:    v_cmp_ne_u64_e32 vcc_lo, 0, v[4:5]
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_cndmask_b32_e32 v4, -1, v4, vcc_lo
 ; GFX12-NEXT:    scratch_load_b64 v[0:1], v4, off
 ; GFX12-NEXT:    s_wait_loadcnt 0x0
@@ -6111,7 +6113,6 @@ define double @flat_agent_atomic_fadd_ret_f64__offset12b_pos__amdgpu_no_fine_gra
 ; GFX12-NEXT:    scratch_store_b64 v4, v[2:3], off
 ; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: flat_agent_atomic_fadd_ret_f64__offset12b_pos__amdgpu_no_fine_grained_memory:
@@ -6479,6 +6480,7 @@ define double @flat_agent_atomic_fadd_ret_f64__offset12b_neg__amdgpu_no_fine_gra
 ; GFX12-NEXT:    s_wait_bvhcnt 0x0
 ; GFX12-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-NEXT:    v_add_co_u32 v4, vcc_lo, 0xfffff800, v0
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_add_co_ci_u32_e32 v5, vcc_lo, -1, v1, vcc_lo
 ; GFX12-NEXT:    s_mov_b64 s[0:1], src_private_base
 ; GFX12-NEXT:    s_mov_b32 s0, exec_lo
@@ -6495,7 +6497,7 @@ define double @flat_agent_atomic_fadd_ret_f64__offset12b_neg__amdgpu_no_fine_gra
 ; GFX12-NEXT:  .LBB32_2: ; %atomicrmw.phi
 ; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT:    s_wait_alu 0xfffe
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ; GFX12-NEXT:  .LBB32_3: ; %atomicrmw.global
 ; GFX12-NEXT:    flat_load_b64 v[0:1], v[4:5]
@@ -6524,6 +6526,7 @@ define double @flat_agent_atomic_fadd_ret_f64__offset12b_neg__amdgpu_no_fine_gra
 ; GFX12-NEXT:    s_cbranch_execz .LBB32_2
 ; GFX12-NEXT:  .LBB32_6: ; %atomicrmw.private
 ; GFX12-NEXT:    v_cmp_ne_u64_e32 vcc_lo, 0, v[4:5]
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_cndmask_b32_e32 v4, -1, v4, vcc_lo
 ; GFX12-NEXT:    scratch_load_b64 v[0:1], v4, off
 ; GFX12-NEXT:    s_wait_loadcnt 0x0
@@ -6531,7 +6534,6 @@ define double @flat_agent_atomic_fadd_ret_f64__offset12b_neg__amdgpu_no_fine_gra
 ; GFX12-NEXT:    scratch_store_b64 v4, v[2:3], off
 ; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: flat_agent_atomic_fadd_ret_f64__offset12b_neg__amdgpu_no_fine_grained_memory:
@@ -6912,7 +6914,6 @@ define void @flat_agent_atomic_fadd_noret_f64__amdgpu_no_fine_grained_memory(ptr
 ; GFX12-NEXT:  .LBB33_2: ; %atomicrmw.phi
 ; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ; GFX12-NEXT:  .LBB33_3: ; %atomicrmw.global
 ; GFX12-NEXT:    flat_load_b64 v[6:7], v[0:1]
@@ -6940,6 +6941,7 @@ define void @flat_agent_atomic_fadd_noret_f64__amdgpu_no_fine_grained_memory(ptr
 ; GFX12-NEXT:    s_cbranch_execz .LBB33_2
 ; GFX12-NEXT:  .LBB33_6: ; %atomicrmw.private
 ; GFX12-NEXT:    v_cmp_ne_u64_e32 vcc_lo, 0, v[0:1]
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_cndmask_b32_e32 v4, -1, v0, vcc_lo
 ; GFX12-NEXT:    scratch_load_b64 v[0:1], v4, off
 ; GFX12-NEXT:    s_wait_loadcnt 0x0
@@ -6947,7 +6949,6 @@ define void @flat_agent_atomic_fadd_noret_f64__amdgpu_no_fine_grained_memory(ptr
 ; GFX12-NEXT:    scratch_store_b64 v4, v[0:1], off
 ; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: flat_agent_atomic_fadd_noret_f64__amdgpu_no_fine_grained_memory:
@@ -7291,6 +7292,7 @@ define void @flat_agent_atomic_fadd_noret_f64__offset12b_pos__amdgpu_no_fine_gra
 ; GFX12-NEXT:    s_wait_bvhcnt 0x0
 ; GFX12-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-NEXT:    v_add_co_u32 v0, vcc_lo, 0x7f8, v0
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
 ; GFX12-NEXT:    s_mov_b64 s[0:1], src_private_base
 ; GFX12-NEXT:    s_mov_b32 s0, exec_lo
@@ -7306,7 +7308,7 @@ define void @flat_agent_atomic_fadd_noret_f64__offset12b_pos__amdgpu_no_fine_gra
 ; GFX12-NEXT:  .LBB34_2: ; %atomicrmw.phi
 ; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT:    s_wait_alu 0xfffe
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ; GFX12-NEXT:  .LBB34_3: ; %atomicrmw.global
 ; GFX12-NEXT:    flat_load_b64 v[6:7], v[0:1]
@@ -7334,6 +7336,7 @@ define void @flat_agent_atomic_fadd_noret_f64__offset12b_pos__amdgpu_no_fine_gra
 ; GFX12-NEXT:    s_cbranch_execz .LBB34_2
 ; GFX12-NEXT:  .LBB34_6: ; %atomicrmw.private
 ; GFX12-NEXT:    v_cmp_ne_u64_e32 vcc_lo, 0, v[0:1]
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_cndmask_b32_e32 v4, -1, v0, vcc_lo
 ; GFX12-NEXT:    scratch_load_b64 v[0:1], v4, off
 ; GFX12-NEXT:    s_wait_loadcnt 0x0
@@ -7341,7 +7344,6 @@ define void @flat_agent_atomic_fadd_noret_f64__offset12b_pos__amdgpu_no_fine_gra
 ; GFX12-NEXT:    scratch_store_b64 v4, v[0:1], off
 ; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: flat_agent_atomic_fadd_noret_f64__offset12b_pos__amdgpu_no_fine_grained_memory:
@@ -7701,6 +7703,7 @@ define void @flat_agent_atomic_fadd_noret_f64__offset12b_neg__amdgpu_no_fine_gra
 ; GFX12-NEXT:    s_wait_bvhcnt 0x0
 ; GFX12-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-NEXT:    v_add_co_u32 v0, vcc_lo, 0xfffff800, v0
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, -1, v1, vcc_lo
 ; GFX12-NEXT:    s_mov_b64 s[0:1], src_private_base
 ; GFX12-NEXT:    s_mov_b32 s0, exec_lo
@@ -7716,7 +7719,7 @@ define void @flat_agent_atomic_fadd_noret_f64__offset12b_neg__amdgpu_no_fine_gra
 ; GFX12-NEXT:  .LBB35_2: ; %atomicrmw.phi
 ; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT:    s_wait_alu 0xfffe
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ; GFX12-NEXT:  .LBB35_3: ; %atomicrmw.global
 ; GFX12-NEXT:    flat_load_b64 v[6:7], v[0:1]
@@ -7744,6 +7747,7 @@ define void @flat_agent_atomic_fadd_noret_f64__offset12b_neg__amdgpu_no_fine_gra
 ; GFX12-NEXT:    s_cbranch_execz .LBB35_2
 ; GFX12-NEXT:  .LBB35_6: ; %atomicrmw.private
 ; GFX12-NEXT:    v_cmp_ne_u64_e32 vcc_lo, 0, v[0:1]
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_cndmask_b32_e32 v4, -1, v0, vcc_lo
 ; GFX12-NEXT:    scratch_load_b64 v[0:1], v4, off
 ; GFX12-NEXT:    s_wait_loadcnt 0x0
@@ -7751,7 +7755,6 @@ define void @flat_agent_atomic_fadd_noret_f64__offset12b_neg__amdgpu_no_fine_gra
 ; GFX12-NEXT:    scratch_store_b64 v4, v[0:1], off
 ; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: flat_agent_atomic_fadd_noret_f64__offset12b_neg__amdgpu_no_fine_grained_memory:
@@ -8150,7 +8153,6 @@ define half @flat_agent_atomic_fadd_ret_f16__amdgpu_no_fine_grained_memory(ptr %
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX12-NEXT:    v_lshrrev_b32_e32 v0, v3, v5
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: flat_agent_atomic_fadd_ret_f16__amdgpu_no_fine_grained_memory:
@@ -8405,6 +8407,7 @@ define half @flat_agent_atomic_fadd_ret_f16__offset12b_pos__amdgpu_no_fine_grain
 ; GFX12-NEXT:    s_wait_bvhcnt 0x0
 ; GFX12-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-NEXT:    v_add_co_u32 v3, vcc_lo, 0x7fe, v0
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
 ; GFX12-NEXT:    s_mov_b32 s0, 0
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_1)
@@ -8440,7 +8443,6 @@ define half @flat_agent_atomic_fadd_ret_f16__offset12b_pos__amdgpu_no_fine_grain
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX12-NEXT:    v_lshrrev_b32_e32 v0, v3, v5
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: flat_agent_atomic_fadd_ret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
@@ -8704,6 +8706,7 @@ define half @flat_agent_atomic_fadd_ret_f16__offset12b_neg__amdgpu_no_fine_grain
 ; GFX12-NEXT:    s_wait_bvhcnt 0x0
 ; GFX12-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-NEXT:    v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, -1, v1, vcc_lo
 ; GFX12-NEXT:    s_mov_b32 s0, 0
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_1)
@@ -8739,7 +8742,6 @@ define half @flat_agent_atomic_fadd_ret_f16__offset12b_neg__amdgpu_no_fine_grain
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX12-NEXT:    v_lshrrev_b32_e32 v0, v3, v5
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: flat_agent_atomic_fadd_ret_f16__offset12b_neg__amdgpu_no_fine_grained_memory:
@@ -9036,7 +9038,6 @@ define void @flat_agent_atomic_fadd_noret_f16__amdgpu_no_fine_grained_memory(ptr
 ; GFX12-NEXT:    s_cbranch_execnz .LBB39_1
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: flat_agent_atomic_fadd_noret_f16__amdgpu_no_fine_grained_memory:
@@ -9282,6 +9283,7 @@ define void @flat_agent_atomic_fadd_noret_f16__offset12b_pos__amdgpu_no_fine_gra
 ; GFX12-NEXT:    s_wait_bvhcnt 0x0
 ; GFX12-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-NEXT:    v_add_co_u32 v3, vcc_lo, 0x7fe, v0
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
 ; GFX12-NEXT:    s_mov_b32 s0, 0
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_1)
@@ -9315,7 +9317,6 @@ define void @flat_agent_atomic_fadd_noret_f16__offset12b_pos__amdgpu_no_fine_gra
 ; GFX12-NEXT:    s_cbranch_execnz .LBB40_1
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: flat_agent_atomic_fadd_noret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
@@ -9570,6 +9571,7 @@ define void @flat_agent_atomic_fadd_noret_f16__offset12b_neg__amdgpu_no_fine_gra
 ; GFX12-NEXT:    s_wait_bvhcnt 0x0
 ; GFX12-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-NEXT:    v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, -1, v1, vcc_lo
 ; GFX12-NEXT:    s_mov_b32 s0, 0
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_1)
@@ -9603,7 +9605,6 @@ define void @flat_agent_atomic_fadd_noret_f16__offset12b_neg__amdgpu_no_fine_gra
 ; GFX12-NEXT:    s_cbranch_execnz .LBB41_1
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: flat_agent_atomic_fadd_noret_f16__offset12b_neg__amdgpu_no_fine_grained_memory:
@@ -9880,7 +9881,6 @@ define void @flat_agent_atomic_fadd_noret_f16__offset12b__align4_pos__amdgpu_no_
 ; GFX12-NEXT:    s_cbranch_execnz .LBB42_1
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: flat_agent_atomic_fadd_noret_f16__offset12b__align4_pos__amdgpu_no_fine_grained_memory:
@@ -10097,7 +10097,6 @@ define half @flat_agent_atomic_fadd_ret_f16__offset12b_pos__align4__amdgpu_no_fi
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX12-NEXT:    v_mov_b32_e32 v0, v3
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: flat_agent_atomic_fadd_ret_f16__offset12b_pos__align4__amdgpu_no_fine_grained_memory:
@@ -10297,6 +10296,7 @@ define half @flat_system_atomic_fadd_ret_f16__offset12b_pos__amdgpu_no_fine_grai
 ; GFX12-NEXT:    s_wait_bvhcnt 0x0
 ; GFX12-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-NEXT:    v_add_co_u32 v3, vcc_lo, 0x7fe, v0
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
 ; GFX12-NEXT:    s_mov_b32 s0, 0
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_1)
@@ -10333,7 +10333,6 @@ define half @flat_system_atomic_fadd_ret_f16__offset12b_pos__amdgpu_no_fine_grai
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX12-NEXT:    v_lshrrev_b32_e32 v0, v3, v5
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: flat_system_atomic_fadd_ret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
@@ -10599,6 +10598,7 @@ define void @flat_system_atomic_fadd_noret_f16__offset12b_pos__amdgpu_no_fine_gr
 ; GFX12-NEXT:    s_wait_bvhcnt 0x0
 ; GFX12-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-NEXT:    v_add_co_u32 v3, vcc_lo, 0x7fe, v0
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
 ; GFX12-NEXT:    s_mov_b32 s0, 0
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_1)
@@ -10633,7 +10633,6 @@ define void @flat_system_atomic_fadd_noret_f16__offset12b_pos__amdgpu_no_fine_gr
 ; GFX12-NEXT:    s_cbranch_execnz .LBB45_1
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: flat_system_atomic_fadd_noret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
@@ -10915,8 +10914,9 @@ define bfloat @flat_agent_atomic_fadd_ret_bf16__amdgpu_no_fine_grained_memory(pt
 ; GFX12-NEXT:    v_bfe_u32 v7, v5, 16, 1
 ; GFX12-NEXT:    v_or_b32_e32 v8, 0x400000, v5
 ; GFX12-NEXT:    v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_cndmask_b32_e32 v5, v7, v8, vcc_lo
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_lshrrev_b32_e32 v5, 16, v5
@@ -10936,7 +10936,6 @@ define bfloat @flat_agent_atomic_fadd_ret_bf16__amdgpu_no_fine_grained_memory(pt
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX12-NEXT:    v_lshrrev_b32_e32 v0, v3, v5
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: flat_agent_atomic_fadd_ret_bf16__amdgpu_no_fine_grained_memory:
@@ -11236,6 +11235,7 @@ define bfloat @flat_agent_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_gr
 ; GFX12-NEXT:    s_wait_bvhcnt 0x0
 ; GFX12-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-NEXT:    v_add_co_u32 v3, vcc_lo, 0x7fe, v0
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
 ; GFX12-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
@@ -11259,8 +11259,9 @@ define bfloat @flat_agent_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_gr
 ; GFX12-NEXT:    v_bfe_u32 v7, v5, 16, 1
 ; GFX12-NEXT:    v_or_b32_e32 v8, 0x400000, v5
 ; GFX12-NEXT:    v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_cndmask_b32_e32 v5, v7, v8, vcc_lo
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_lshrrev_b32_e32 v5, 16, v5
@@ -11280,7 +11281,6 @@ define bfloat @flat_agent_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_gr
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX12-NEXT:    v_lshrrev_b32_e32 v0, v3, v5
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: flat_agent_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
@@ -11590,6 +11590,7 @@ define bfloat @flat_agent_atomic_fadd_ret_bf16__offset12b_neg__amdgpu_no_fine_gr
 ; GFX12-NEXT:    s_wait_bvhcnt 0x0
 ; GFX12-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-NEXT:    v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, -1, v1, vcc_lo
 ; GFX12-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
@@ -11613,8 +11614,9 @@ define bfloat @flat_agent_atomic_fadd_ret_bf16__offset12b_neg__amdgpu_no_fine_gr
 ; GFX12-NEXT:    v_bfe_u32 v7, v5, 16, 1
 ; GFX12-NEXT:    v_or_b32_e32 v8, 0x400000, v5
 ; GFX12-NEXT:    v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_cndmask_b32_e32 v5, v7, v8, vcc_lo
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_lshrrev_b32_e32 v5, 16, v5
@@ -11634,7 +11636,6 @@ define bfloat @flat_agent_atomic_fadd_ret_bf16__offset12b_neg__amdgpu_no_fine_gr
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX12-NEXT:    v_lshrrev_b32_e32 v0, v3, v5
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: flat_agent_atomic_fadd_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
@@ -11945,6 +11946,7 @@ define void @flat_agent_atomic_fadd_noret_bf16__offset12b_pos__amdgpu_no_fine_gr
 ; GFX12-NEXT:    s_wait_bvhcnt 0x0
 ; GFX12-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-NEXT:    v_add_co_u32 v4, vcc_lo, 0x7fe, v0
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
 ; GFX12-NEXT:    v_lshlrev_b32_e32 v6, 16, v2
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
@@ -11968,6 +11970,7 @@ define void @flat_agent_atomic_fadd_noret_bf16__offset12b_pos__amdgpu_no_fine_gr
 ; GFX12-NEXT:    v_or_b32_e32 v8, 0x400000, v2
 ; GFX12-NEXT:    v_cmp_u_f32_e32 vcc_lo, v2, v2
 ; GFX12-NEXT:    v_add3_u32 v7, v7, v2, 0x7fff
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_cndmask_b32_e32 v2, v7, v8, vcc_lo
 ; GFX12-NEXT:    v_lshrrev_b32_e32 v2, 16, v2
@@ -11987,7 +11990,6 @@ define void @flat_agent_atomic_fadd_noret_bf16__offset12b_pos__amdgpu_no_fine_gr
 ; GFX12-NEXT:    s_cbranch_execnz .LBB49_1
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: flat_agent_atomic_fadd_noret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
@@ -12288,6 +12290,7 @@ define void @flat_agent_atomic_fadd_noret_bf16__offset12b_neg__amdgpu_no_fine_gr
 ; GFX12-NEXT:    s_wait_bvhcnt 0x0
 ; GFX12-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-NEXT:    v_add_co_u32 v4, vcc_lo, 0xfffff800, v0
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, -1, v1, vcc_lo
 ; GFX12-NEXT:    v_lshlrev_b32_e32 v6, 16, v2
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
@@ -12311,6 +12314,7 @@ define void @flat_agent_atomic_fadd_noret_bf16__offset12b_neg__amdgpu_no_fine_gr
 ; GFX12-NEXT:    v_or_b32_e32 v8, 0x400000, v2
 ; GFX12-NEXT:    v_cmp_u_f32_e32 vcc_lo, v2, v2
 ; GFX12-NEXT:    v_add3_u32 v7, v7, v2, 0x7fff
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_cndmask_b32_e32 v2, v7, v8, vcc_lo
 ; GFX12-NEXT:    v_lshrrev_b32_e32 v2, 16, v2
@@ -12330,7 +12334,6 @@ define void @flat_agent_atomic_fadd_noret_bf16__offset12b_neg__amdgpu_no_fine_gr
 ; GFX12-NEXT:    s_cbranch_execnz .LBB50_1
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: flat_agent_atomic_fadd_noret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
@@ -12646,6 +12649,7 @@ define bfloat @flat_agent_atomic_fadd_ret_bf16__offset12b_pos__align4__amdgpu_no
 ; GFX12-NEXT:    v_or_b32_e32 v6, 0x400000, v3
 ; GFX12-NEXT:    v_cmp_u_f32_e32 vcc_lo, v3, v3
 ; GFX12-NEXT:    v_add3_u32 v5, v5, v3, 0x7fff
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_cndmask_b32_e32 v3, v5, v6, vcc_lo
 ; GFX12-NEXT:    v_lshrrev_b32_e32 v3, 16, v3
@@ -12664,7 +12668,6 @@ define bfloat @flat_agent_atomic_fadd_ret_bf16__offset12b_pos__align4__amdgpu_no
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX12-NEXT:    v_mov_b32_e32 v0, v3
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: flat_agent_atomic_fadd_ret_bf16__offset12b_pos__align4__amdgpu_no_fine_grained_memory:
@@ -12928,8 +12931,9 @@ define void @flat_agent_atomic_fadd_noret_bf16__offset12b__align4_pos__amdgpu_no
 ; GFX12-NEXT:    v_bfe_u32 v5, v2, 16, 1
 ; GFX12-NEXT:    v_or_b32_e32 v6, 0x400000, v2
 ; GFX12-NEXT:    v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_add3_u32 v5, v5, v2, 0x7fff
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_cndmask_b32_e32 v2, v5, v6, vcc_lo
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_lshrrev_b32_e32 v2, 16, v2
@@ -12947,7 +12951,6 @@ define void @flat_agent_atomic_fadd_noret_bf16__offset12b__align4_pos__amdgpu_no
 ; GFX12-NEXT:    s_cbranch_execnz .LBB52_1
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: flat_agent_atomic_fadd_noret_bf16__offset12b__align4_pos__amdgpu_no_fine_grained_memory:
@@ -13215,6 +13218,7 @@ define void @flat_agent_atomic_fadd_noret_bf16__amdgpu_no_fine_grained_memory(pt
 ; GFX12-NEXT:    v_or_b32_e32 v8, 0x400000, v3
 ; GFX12-NEXT:    v_cmp_u_f32_e32 vcc_lo, v3, v3
 ; GFX12-NEXT:    v_add3_u32 v7, v7, v3, 0x7fff
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_cndmask_b32_e32 v3, v7, v8, vcc_lo
 ; GFX12-NEXT:    v_lshrrev_b32_e32 v3, 16, v3
@@ -13234,7 +13238,6 @@ define void @flat_agent_atomic_fadd_noret_bf16__amdgpu_no_fine_grained_memory(pt
 ; GFX12-NEXT:    s_cbranch_execnz .LBB53_1
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: flat_agent_atomic_fadd_noret_bf16__amdgpu_no_fine_grained_memory:
@@ -13525,6 +13528,7 @@ define bfloat @flat_system_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_g
 ; GFX12-NEXT:    s_wait_bvhcnt 0x0
 ; GFX12-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-NEXT:    v_add_co_u32 v3, vcc_lo, 0x7fe, v0
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
 ; GFX12-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
@@ -13548,8 +13552,9 @@ define bfloat @flat_system_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_g
 ; GFX12-NEXT:    v_bfe_u32 v7, v5, 16, 1
 ; GFX12-NEXT:    v_or_b32_e32 v8, 0x400000, v5
 ; GFX12-NEXT:    v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_cndmask_b32_e32 v5, v7, v8, vcc_lo
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_lshrrev_b32_e32 v5, 16, v5
@@ -13570,7 +13575,6 @@ define bfloat @flat_system_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_g
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX12-NEXT:    v_lshrrev_b32_e32 v0, v3, v5
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: flat_system_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
@@ -13882,6 +13886,7 @@ define void @flat_system_atomic_fadd_noret_bf16__offset12b_pos__amdgpu_no_fine_g
 ; GFX12-NEXT:    s_wait_bvhcnt 0x0
 ; GFX12-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-NEXT:    v_add_co_u32 v4, vcc_lo, 0x7fe, v0
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
 ; GFX12-NEXT:    v_lshlrev_b32_e32 v6, 16, v2
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
@@ -13905,6 +13910,7 @@ define void @flat_system_atomic_fadd_noret_bf16__offset12b_pos__amdgpu_no_fine_g
 ; GFX12-NEXT:    v_or_b32_e32 v8, 0x400000, v2
 ; GFX12-NEXT:    v_cmp_u_f32_e32 vcc_lo, v2, v2
 ; GFX12-NEXT:    v_add3_u32 v7, v7, v2, 0x7fff
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_cndmask_b32_e32 v2, v7, v8, vcc_lo
 ; GFX12-NEXT:    v_lshrrev_b32_e32 v2, 16, v2
@@ -13925,7 +13931,6 @@ define void @flat_system_atomic_fadd_noret_bf16__offset12b_pos__amdgpu_no_fine_g
 ; GFX12-NEXT:    s_cbranch_execnz .LBB55_1
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: flat_system_atomic_fadd_noret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
diff --git a/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fmax.ll b/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fmax.ll
index 36aa73fbf8e92..5aa9be627594d 100644
--- a/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fmax.ll
+++ b/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fmax.ll
@@ -2808,6 +2808,7 @@ define double @flat_agent_atomic_fmax_ret_f64__amdgpu_no_fine_grained_memory(ptr
 ; GFX12-NEXT:    s_cbranch_execz .LBB18_6
 ; GFX12-NEXT:  ; %bb.5: ; %atomicrmw.private
 ; GFX12-NEXT:    v_cmp_ne_u64_e32 vcc_lo, 0, v[0:1]
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_cndmask_b32_e32 v6, -1, v0, vcc_lo
 ; GFX12-NEXT:    scratch_load_b64 v[2:3], v6, off
 ; GFX12-NEXT:    s_wait_loadcnt 0x0
@@ -2819,7 +2820,6 @@ define double @flat_agent_atomic_fmax_ret_f64__amdgpu_no_fine_grained_memory(ptr
 ; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX12-NEXT:    v_dual_mov_b32 v0, v2 :: v_dual_mov_b32 v1, v3
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: flat_agent_atomic_fmax_ret_f64__amdgpu_no_fine_grained_memory:
@@ -2917,8 +2917,8 @@ define double @flat_agent_atomic_fmax_ret_f64__amdgpu_no_fine_grained_memory(ptr
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    v_mov_b32_e32 v5, v1
-; GFX10-NEXT:    v_mov_b32_e32 v4, v0
 ; GFX10-NEXT:    s_mov_b64 s[4:5], src_private_base
+; GFX10-NEXT:    v_mov_b32_e32 v4, v0
 ; GFX10-NEXT:    ; implicit-def: $vgpr0_vgpr1
 ; GFX10-NEXT:    v_cmp_ne_u32_e32 vcc_lo, s5, v5
 ; GFX10-NEXT:    s_and_saveexec_b32 s4, vcc_lo
@@ -3161,6 +3161,7 @@ define double @flat_agent_atomic_fmax_ret_f64__offset12b_pos__amdgpu_no_fine_gra
 ; GFX12-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-NEXT:    v_max_num_f64_e32 v[2:3], v[2:3], v[2:3]
 ; GFX12-NEXT:    v_add_co_u32 v4, vcc_lo, 0x7f8, v0
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_add_co_ci_u32_e32 v5, vcc_lo, 0, v1, vcc_lo
 ; GFX12-NEXT:    s_mov_b64 s[0:1], src_private_base
 ; GFX12-NEXT:    s_mov_b32 s0, exec_lo
@@ -3177,7 +3178,7 @@ define double @flat_agent_atomic_fmax_ret_f64__offset12b_pos__amdgpu_no_fine_gra
 ; GFX12-NEXT:  .LBB19_2: ; %atomicrmw.phi
 ; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT:    s_wait_alu 0xfffe
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ; GFX12-NEXT:  .LBB19_3: ; %atomicrmw.global
 ; GFX12-NEXT:    flat_load_b64 v[0:1], v[4:5]
@@ -3207,6 +3208,7 @@ define double @flat_agent_atomic_fmax_ret_f64__offset12b_pos__amdgpu_no_fine_gra
 ; GFX12-NEXT:    s_cbranch_execz .LBB19_2
 ; GFX12-NEXT:  .LBB19_6: ; %atomicrmw.private
 ; GFX12-NEXT:    v_cmp_ne_u64_e32 vcc_lo, 0, v[4:5]
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_cndmask_b32_e32 v6, -1, v4, vcc_lo
 ; GFX12-NEXT:    scratch_load_b64 v[0:1], v6, off
 ; GFX12-NEXT:    s_wait_loadcnt 0x0
@@ -3216,7 +3218,6 @@ define double @flat_agent_atomic_fmax_ret_f64__offset12b_pos__amdgpu_no_fine_gra
 ; GFX12-NEXT:    scratch_store_b64 v6, v[2:3], off
 ; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: flat_agent_atomic_fmax_ret_f64__offset12b_pos__amdgpu_no_fine_grained_memory:
@@ -3573,6 +3574,7 @@ define double @flat_agent_atomic_fmax_ret_f64__offset12b_neg__amdgpu_no_fine_gra
 ; GFX12-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-NEXT:    v_max_num_f64_e32 v[2:3], v[2:3], v[2:3]
 ; GFX12-NEXT:    v_add_co_u32 v4, vcc_lo, 0xfffff800, v0
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_add_co_ci_u32_e32 v5, vcc_lo, -1, v1, vcc_lo
 ; GFX12-NEXT:    s_mov_b64 s[0:1], src_private_base
 ; GFX12-NEXT:    s_mov_b32 s0, exec_lo
@@ -3589,7 +3591,7 @@ define double @flat_agent_atomic_fmax_ret_f64__offset12b_neg__amdgpu_no_fine_gra
 ; GFX12-NEXT:  .LBB20_2: ; %atomicrmw.phi
 ; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT:    s_wait_alu 0xfffe
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ; GFX12-NEXT:  .LBB20_3: ; %atomicrmw.global
 ; GFX12-NEXT:    flat_load_b64 v[0:1], v[4:5]
@@ -3619,6 +3621,7 @@ define double @flat_agent_atomic_fmax_ret_f64__offset12b_neg__amdgpu_no_fine_gra
 ; GFX12-NEXT:    s_cbranch_execz .LBB20_2
 ; GFX12-NEXT:  .LBB20_6: ; %atomicrmw.private
 ; GFX12-NEXT:    v_cmp_ne_u64_e32 vcc_lo, 0, v[4:5]
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_cndmask_b32_e32 v6, -1, v4, vcc_lo
 ; GFX12-NEXT:    scratch_load_b64 v[0:1], v6, off
 ; GFX12-NEXT:    s_wait_loadcnt 0x0
@@ -3628,7 +3631,6 @@ define double @flat_agent_atomic_fmax_ret_f64__offset12b_neg__amdgpu_no_fine_gra
 ; GFX12-NEXT:    scratch_store_b64 v6, v[2:3], off
 ; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: flat_agent_atomic_fmax_ret_f64__offset12b_neg__amdgpu_no_fine_grained_memory:
@@ -3998,7 +4000,6 @@ define void @flat_agent_atomic_fmax_noret_f64__amdgpu_no_fine_grained_memory(ptr
 ; GFX12-NEXT:  .LBB21_2: ; %atomicrmw.phi
 ; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ; GFX12-NEXT:  .LBB21_3: ; %atomicrmw.global
 ; GFX12-NEXT:    flat_load_b64 v[4:5], v[0:1]
@@ -4028,6 +4029,7 @@ define void @flat_agent_atomic_fmax_noret_f64__amdgpu_no_fine_grained_memory(ptr
 ; GFX12-NEXT:    s_cbranch_execz .LBB21_2
 ; GFX12-NEXT:  .LBB21_6: ; %atomicrmw.private
 ; GFX12-NEXT:    v_cmp_ne_u64_e32 vcc_lo, 0, v[0:1]
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_cndmask_b32_e32 v2, -1, v0, vcc_lo
 ; GFX12-NEXT:    scratch_load_b64 v[0:1], v2, off
 ; GFX12-NEXT:    s_wait_loadcnt 0x0
@@ -4037,7 +4039,6 @@ define void @flat_agent_atomic_fmax_noret_f64__amdgpu_no_fine_grained_memory(ptr
 ; GFX12-NEXT:    scratch_store_b64 v2, v[0:1], off
 ; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: flat_agent_atomic_fmax_noret_f64__amdgpu_no_fine_grained_memory:
@@ -4372,6 +4373,7 @@ define void @flat_agent_atomic_fmax_noret_f64__offset12b_pos__amdgpu_no_fine_gra
 ; GFX12-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-NEXT:    v_max_num_f64_e32 v[4:5], v[2:3], v[2:3]
 ; GFX12-NEXT:    v_add_co_u32 v6, vcc_lo, 0x7f8, v0
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_add_co_ci_u32_e32 v7, vcc_lo, 0, v1, vcc_lo
 ; GFX12-NEXT:    s_mov_b64 s[0:1], src_private_base
 ; GFX12-NEXT:    s_mov_b32 s0, exec_lo
@@ -4387,7 +4389,7 @@ define void @flat_agent_atomic_fmax_noret_f64__offset12b_pos__amdgpu_no_fine_gra
 ; GFX12-NEXT:  .LBB22_2: ; %atomicrmw.phi
 ; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT:    s_wait_alu 0xfffe
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ; GFX12-NEXT:  .LBB22_3: ; %atomicrmw.global
 ; GFX12-NEXT:    flat_load_b64 v[2:3], v[6:7]
@@ -4417,6 +4419,7 @@ define void @flat_agent_atomic_fmax_noret_f64__offset12b_pos__amdgpu_no_fine_gra
 ; GFX12-NEXT:    s_cbranch_execz .LBB22_2
 ; GFX12-NEXT:  .LBB22_6: ; %atomicrmw.private
 ; GFX12-NEXT:    v_cmp_ne_u64_e32 vcc_lo, 0, v[6:7]
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_cndmask_b32_e32 v2, -1, v6, vcc_lo
 ; GFX12-NEXT:    scratch_load_b64 v[0:1], v2, off
 ; GFX12-NEXT:    s_wait_loadcnt 0x0
@@ -4426,7 +4429,6 @@ define void @flat_agent_atomic_fmax_noret_f64__offset12b_pos__amdgpu_no_fine_gra
 ; GFX12-NEXT:    scratch_store_b64 v2, v[0:1], off
 ; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: flat_agent_atomic_fmax_noret_f64__offset12b_pos__amdgpu_no_fine_grained_memory:
@@ -4777,6 +4779,7 @@ define void @flat_agent_atomic_fmax_noret_f64__offset12b_neg__amdgpu_no_fine_gra
 ; GFX12-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-NEXT:    v_max_num_f64_e32 v[4:5], v[2:3], v[2:3]
 ; GFX12-NEXT:    v_add_co_u32 v6, vcc_lo, 0xfffff800, v0
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_add_co_ci_u32_e32 v7, vcc_lo, -1, v1, vcc_lo
 ; GFX12-NEXT:    s_mov_b64 s[0:1], src_private_base
 ; GFX12-NEXT:    s_mov_b32 s0, exec_lo
@@ -4792,7 +4795,7 @@ define void @flat_agent_atomic_fmax_noret_f64__offset12b_neg__amdgpu_no_fine_gra
 ; GFX12-NEXT:  .LBB23_2: ; %atomicrmw.phi
 ; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT:    s_wait_alu 0xfffe
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ; GFX12-NEXT:  .LBB23_3: ; %atomicrmw.global
 ; GFX12-NEXT:    flat_load_b64 v[2:3], v[6:7]
@@ -4822,6 +4825,7 @@ define void @flat_agent_atomic_fmax_noret_f64__offset12b_neg__amdgpu_no_fine_gra
 ; GFX12-NEXT:    s_cbranch_execz .LBB23_2
 ; GFX12-NEXT:  .LBB23_6: ; %atomicrmw.private
 ; GFX12-NEXT:    v_cmp_ne_u64_e32 vcc_lo, 0, v[6:7]
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_cndmask_b32_e32 v2, -1, v6, vcc_lo
 ; GFX12-NEXT:    scratch_load_b64 v[0:1], v2, off
 ; GFX12-NEXT:    s_wait_loadcnt 0x0
@@ -4831,7 +4835,6 @@ define void @flat_agent_atomic_fmax_noret_f64__offset12b_neg__amdgpu_no_fine_gra
 ; GFX12-NEXT:    scratch_store_b64 v2, v[0:1], off
 ; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: flat_agent_atomic_fmax_noret_f64__offset12b_neg__amdgpu_no_fine_grained_memory:
@@ -5219,6 +5222,7 @@ define double @flat_agent_atomic_fmax_ret_f64__amdgpu_no_remote_memory(ptr %ptr,
 ; GFX12-NEXT:    s_cbranch_execz .LBB24_6
 ; GFX12-NEXT:  ; %bb.5: ; %atomicrmw.private
 ; GFX12-NEXT:    v_cmp_ne_u64_e32 vcc_lo, 0, v[0:1]
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_cndmask_b32_e32 v6, -1, v0, vcc_lo
 ; GFX12-NEXT:    scratch_load_b64 v[2:3], v6, off
 ; GFX12-NEXT:    s_wait_loadcnt 0x0
@@ -5230,7 +5234,6 @@ define double @flat_agent_atomic_fmax_ret_f64__amdgpu_no_remote_memory(ptr %ptr,
 ; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX12-NEXT:    v_dual_mov_b32 v0, v2 :: v_dual_mov_b32 v1, v3
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: flat_agent_atomic_fmax_ret_f64__amdgpu_no_remote_memory:
@@ -5642,6 +5645,7 @@ define double @flat_agent_atomic_fmax_ret_f64__amdgpu_no_fine_grained_memory__am
 ; GFX12-NEXT:    s_cbranch_execz .LBB25_6
 ; GFX12-NEXT:  ; %bb.5: ; %atomicrmw.private
 ; GFX12-NEXT:    v_cmp_ne_u64_e32 vcc_lo, 0, v[0:1]
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_cndmask_b32_e32 v6, -1, v0, vcc_lo
 ; GFX12-NEXT:    scratch_load_b64 v[2:3], v6, off
 ; GFX12-NEXT:    s_wait_loadcnt 0x0
@@ -5653,7 +5657,6 @@ define double @flat_agent_atomic_fmax_ret_f64__amdgpu_no_fine_grained_memory__am
 ; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX12-NEXT:    v_dual_mov_b32 v0, v2 :: v_dual_mov_b32 v1, v3
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: flat_agent_atomic_fmax_ret_f64__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
@@ -5751,8 +5754,8 @@ define double @flat_agent_atomic_fmax_ret_f64__amdgpu_no_fine_grained_memory__am
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    v_mov_b32_e32 v5, v1
-; GFX10-NEXT:    v_mov_b32_e32 v4, v0
 ; GFX10-NEXT:    s_mov_b64 s[4:5], src_private_base
+; GFX10-NEXT:    v_mov_b32_e32 v4, v0
 ; GFX10-NEXT:    ; implicit-def: $vgpr0_vgpr1
 ; GFX10-NEXT:    v_cmp_ne_u32_e32 vcc_lo, s5, v5
 ; GFX10-NEXT:    s_and_saveexec_b32 s4, vcc_lo
@@ -6034,7 +6037,6 @@ define half @flat_agent_atomic_fmax_ret_f16__amdgpu_no_fine_grained_memory(ptr %
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX12-NEXT:    v_lshrrev_b32_e32 v0, v3, v5
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: flat_agent_atomic_fmax_ret_f16__amdgpu_no_fine_grained_memory:
@@ -6301,6 +6303,7 @@ define half @flat_agent_atomic_fmax_ret_f16__offset12b_pos__amdgpu_no_fine_grain
 ; GFX12-NEXT:    s_wait_bvhcnt 0x0
 ; GFX12-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-NEXT:    v_add_co_u32 v3, vcc_lo, 0x7fe, v0
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
 ; GFX12-NEXT:    v_max_num_f16_e32 v2, v2, v2
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
@@ -6338,7 +6341,6 @@ define half @flat_agent_atomic_fmax_ret_f16__offset12b_pos__amdgpu_no_fine_grain
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX12-NEXT:    v_lshrrev_b32_e32 v0, v3, v5
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: flat_agent_atomic_fmax_ret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
@@ -6614,6 +6616,7 @@ define half @flat_agent_atomic_fmax_ret_f16__offset12b_neg__amdgpu_no_fine_grain
 ; GFX12-NEXT:    s_wait_bvhcnt 0x0
 ; GFX12-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-NEXT:    v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, -1, v1, vcc_lo
 ; GFX12-NEXT:    v_max_num_f16_e32 v2, v2, v2
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
@@ -6651,7 +6654,6 @@ define half @flat_agent_atomic_fmax_ret_f16__offset12b_neg__amdgpu_no_fine_grain
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX12-NEXT:    v_lshrrev_b32_e32 v0, v3, v5
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: flat_agent_atomic_fmax_ret_f16__offset12b_neg__amdgpu_no_fine_grained_memory:
@@ -6963,7 +6965,6 @@ define void @flat_agent_atomic_fmax_noret_f16__amdgpu_no_fine_grained_memory(ptr
 ; GFX12-NEXT:    s_cbranch_execnz .LBB29_1
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: flat_agent_atomic_fmax_noret_f16__amdgpu_no_fine_grained_memory:
@@ -7222,6 +7223,7 @@ define void @flat_agent_atomic_fmax_noret_f16__offset12b_pos__amdgpu_no_fine_gra
 ; GFX12-NEXT:    s_wait_bvhcnt 0x0
 ; GFX12-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-NEXT:    v_add_co_u32 v4, vcc_lo, 0x7fe, v0
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
 ; GFX12-NEXT:    v_max_num_f16_e32 v6, v2, v2
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
@@ -7258,7 +7260,6 @@ define void @flat_agent_atomic_fmax_noret_f16__offset12b_pos__amdgpu_no_fine_gra
 ; GFX12-NEXT:    s_cbranch_execnz .LBB30_1
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: flat_agent_atomic_fmax_noret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
@@ -7526,6 +7527,7 @@ define void @flat_agent_atomic_fmax_noret_f16__offset12b_neg__amdgpu_no_fine_gra
 ; GFX12-NEXT:    s_wait_bvhcnt 0x0
 ; GFX12-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-NEXT:    v_add_co_u32 v4, vcc_lo, 0xfffff800, v0
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, -1, v1, vcc_lo
 ; GFX12-NEXT:    v_max_num_f16_e32 v6, v2, v2
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
@@ -7562,7 +7564,6 @@ define void @flat_agent_atomic_fmax_noret_f16__offset12b_neg__amdgpu_no_fine_gra
 ; GFX12-NEXT:    s_cbranch_execnz .LBB31_1
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: flat_agent_atomic_fmax_noret_f16__offset12b_neg__amdgpu_no_fine_grained_memory:
@@ -7856,7 +7857,6 @@ define half @flat_agent_atomic_fmax_ret_f16__offset12b_pos__align4__amdgpu_no_fi
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX12-NEXT:    v_mov_b32_e32 v0, v3
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: flat_agent_atomic_fmax_ret_f16__offset12b_pos__align4__amdgpu_no_fine_grained_memory:
@@ -8092,7 +8092,6 @@ define void @flat_agent_atomic_fmax_noret_f16__offset12b__align4_pos__amdgpu_no_
 ; GFX12-NEXT:    s_cbranch_execnz .LBB33_1
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: flat_agent_atomic_fmax_noret_f16__offset12b__align4_pos__amdgpu_no_fine_grained_memory:
@@ -8299,6 +8298,7 @@ define half @flat_system_atomic_fmax_ret_f16__offset12b_pos__amdgpu_no_fine_grai
 ; GFX12-NEXT:    s_wait_bvhcnt 0x0
 ; GFX12-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-NEXT:    v_add_co_u32 v3, vcc_lo, 0x7fe, v0
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
 ; GFX12-NEXT:    v_max_num_f16_e32 v2, v2, v2
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
@@ -8337,7 +8337,6 @@ define half @flat_system_atomic_fmax_ret_f16__offset12b_pos__amdgpu_no_fine_grai
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX12-NEXT:    v_lshrrev_b32_e32 v0, v3, v5
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: flat_system_atomic_fmax_ret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
@@ -8615,6 +8614,7 @@ define void @flat_system_atomic_fmax_noret_f16__offset12b_pos__amdgpu_no_fine_gr
 ; GFX12-NEXT:    s_wait_bvhcnt 0x0
 ; GFX12-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-NEXT:    v_add_co_u32 v4, vcc_lo, 0x7fe, v0
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
 ; GFX12-NEXT:    v_max_num_f16_e32 v6, v2, v2
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
@@ -8652,7 +8652,6 @@ define void @flat_system_atomic_fmax_noret_f16__offset12b_pos__amdgpu_no_fine_gr
 ; GFX12-NEXT:    s_cbranch_execnz .LBB35_1
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: flat_system_atomic_fmax_noret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
@@ -8947,8 +8946,9 @@ define bfloat @flat_agent_atomic_fmax_ret_bf16__amdgpu_no_fine_grained_memory(pt
 ; GFX12-NEXT:    v_bfe_u32 v7, v5, 16, 1
 ; GFX12-NEXT:    v_or_b32_e32 v8, 0x400000, v5
 ; GFX12-NEXT:    v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_cndmask_b32_e32 v5, v7, v8, vcc_lo
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_lshrrev_b32_e32 v5, 16, v5
@@ -8968,7 +8968,6 @@ define bfloat @flat_agent_atomic_fmax_ret_bf16__amdgpu_no_fine_grained_memory(pt
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX12-NEXT:    v_lshrrev_b32_e32 v0, v3, v5
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: flat_agent_atomic_fmax_ret_bf16__amdgpu_no_fine_grained_memory:
@@ -9269,6 +9268,7 @@ define bfloat @flat_agent_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_gr
 ; GFX12-NEXT:    s_wait_bvhcnt 0x0
 ; GFX12-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-NEXT:    v_add_co_u32 v3, vcc_lo, 0x7fe, v0
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
 ; GFX12-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
@@ -9292,8 +9292,9 @@ define bfloat @flat_agent_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_gr
 ; GFX12-NEXT:    v_bfe_u32 v7, v5, 16, 1
 ; GFX12-NEXT:    v_or_b32_e32 v8, 0x400000, v5
 ; GFX12-NEXT:    v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_cndmask_b32_e32 v5, v7, v8, vcc_lo
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_lshrrev_b32_e32 v5, 16, v5
@@ -9313,7 +9314,6 @@ define bfloat @flat_agent_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_gr
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX12-NEXT:    v_lshrrev_b32_e32 v0, v3, v5
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: flat_agent_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
@@ -9624,6 +9624,7 @@ define bfloat @flat_agent_atomic_fmax_ret_bf16__offset12b_neg__amdgpu_no_fine_gr
 ; GFX12-NEXT:    s_wait_bvhcnt 0x0
 ; GFX12-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-NEXT:    v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, -1, v1, vcc_lo
 ; GFX12-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
@@ -9647,8 +9648,9 @@ define bfloat @flat_agent_atomic_fmax_ret_bf16__offset12b_neg__amdgpu_no_fine_gr
 ; GFX12-NEXT:    v_bfe_u32 v7, v5, 16, 1
 ; GFX12-NEXT:    v_or_b32_e32 v8, 0x400000, v5
 ; GFX12-NEXT:    v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_cndmask_b32_e32 v5, v7, v8, vcc_lo
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_lshrrev_b32_e32 v5, 16, v5
@@ -9668,7 +9670,6 @@ define bfloat @flat_agent_atomic_fmax_ret_bf16__offset12b_neg__amdgpu_no_fine_gr
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX12-NEXT:    v_lshrrev_b32_e32 v0, v3, v5
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: flat_agent_atomic_fmax_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
@@ -10001,6 +10002,7 @@ define void @flat_agent_atomic_fmax_noret_bf16__amdgpu_no_fine_grained_memory(pt
 ; GFX12-NEXT:    v_or_b32_e32 v8, 0x400000, v3
 ; GFX12-NEXT:    v_cmp_u_f32_e32 vcc_lo, v3, v3
 ; GFX12-NEXT:    v_add3_u32 v7, v7, v3, 0x7fff
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_cndmask_b32_e32 v3, v7, v8, vcc_lo
 ; GFX12-NEXT:    v_lshrrev_b32_e32 v3, 16, v3
@@ -10020,7 +10022,6 @@ define void @flat_agent_atomic_fmax_noret_bf16__amdgpu_no_fine_grained_memory(pt
 ; GFX12-NEXT:    s_cbranch_execnz .LBB39_1
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: flat_agent_atomic_fmax_noret_bf16__amdgpu_no_fine_grained_memory:
@@ -10312,6 +10313,7 @@ define void @flat_agent_atomic_fmax_noret_bf16__offset12b_pos__amdgpu_no_fine_gr
 ; GFX12-NEXT:    s_wait_bvhcnt 0x0
 ; GFX12-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-NEXT:    v_add_co_u32 v4, vcc_lo, 0x7fe, v0
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
 ; GFX12-NEXT:    v_lshlrev_b32_e32 v6, 16, v2
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
@@ -10335,6 +10337,7 @@ define void @flat_agent_atomic_fmax_noret_bf16__offset12b_pos__amdgpu_no_fine_gr
 ; GFX12-NEXT:    v_or_b32_e32 v8, 0x400000, v2
 ; GFX12-NEXT:    v_cmp_u_f32_e32 vcc_lo, v2, v2
 ; GFX12-NEXT:    v_add3_u32 v7, v7, v2, 0x7fff
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_cndmask_b32_e32 v2, v7, v8, vcc_lo
 ; GFX12-NEXT:    v_lshrrev_b32_e32 v2, 16, v2
@@ -10354,7 +10357,6 @@ define void @flat_agent_atomic_fmax_noret_bf16__offset12b_pos__amdgpu_no_fine_gr
 ; GFX12-NEXT:    s_cbranch_execnz .LBB40_1
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: flat_agent_atomic_fmax_noret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
@@ -10656,6 +10658,7 @@ define void @flat_agent_atomic_fmax_noret_bf16__offset12b_neg__amdgpu_no_fine_gr
 ; GFX12-NEXT:    s_wait_bvhcnt 0x0
 ; GFX12-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-NEXT:    v_add_co_u32 v4, vcc_lo, 0xfffff800, v0
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, -1, v1, vcc_lo
 ; GFX12-NEXT:    v_lshlrev_b32_e32 v6, 16, v2
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
@@ -10679,6 +10682,7 @@ define void @flat_agent_atomic_fmax_noret_bf16__offset12b_neg__amdgpu_no_fine_gr
 ; GFX12-NEXT:    v_or_b32_e32 v8, 0x400000, v2
 ; GFX12-NEXT:    v_cmp_u_f32_e32 vcc_lo, v2, v2
 ; GFX12-NEXT:    v_add3_u32 v7, v7, v2, 0x7fff
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_cndmask_b32_e32 v2, v7, v8, vcc_lo
 ; GFX12-NEXT:    v_lshrrev_b32_e32 v2, 16, v2
@@ -10698,7 +10702,6 @@ define void @flat_agent_atomic_fmax_noret_bf16__offset12b_neg__amdgpu_no_fine_gr
 ; GFX12-NEXT:    s_cbranch_execnz .LBB41_1
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: flat_agent_atomic_fmax_noret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
@@ -11015,6 +11018,7 @@ define bfloat @flat_agent_atomic_fmax_ret_bf16__offset12b_pos__align4__amdgpu_no
 ; GFX12-NEXT:    v_or_b32_e32 v6, 0x400000, v3
 ; GFX12-NEXT:    v_cmp_u_f32_e32 vcc_lo, v3, v3
 ; GFX12-NEXT:    v_add3_u32 v5, v5, v3, 0x7fff
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_cndmask_b32_e32 v3, v5, v6, vcc_lo
 ; GFX12-NEXT:    v_lshrrev_b32_e32 v3, 16, v3
@@ -11033,7 +11037,6 @@ define bfloat @flat_agent_atomic_fmax_ret_bf16__offset12b_pos__align4__amdgpu_no
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX12-NEXT:    v_mov_b32_e32 v0, v3
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: flat_agent_atomic_fmax_ret_bf16__offset12b_pos__align4__amdgpu_no_fine_grained_memory:
@@ -11298,8 +11301,9 @@ define void @flat_agent_atomic_fmax_noret_bf16__offset12b__align4_pos__amdgpu_no
 ; GFX12-NEXT:    v_bfe_u32 v5, v2, 16, 1
 ; GFX12-NEXT:    v_or_b32_e32 v6, 0x400000, v2
 ; GFX12-NEXT:    v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_add3_u32 v5, v5, v2, 0x7fff
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_cndmask_b32_e32 v2, v5, v6, vcc_lo
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_lshrrev_b32_e32 v2, 16, v2
@@ -11317,7 +11321,6 @@ define void @flat_agent_atomic_fmax_noret_bf16__offset12b__align4_pos__amdgpu_no
 ; GFX12-NEXT:    s_cbranch_execnz .LBB43_1
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: flat_agent_atomic_fmax_noret_bf16__offset12b__align4_pos__amdgpu_no_fine_grained_memory:
@@ -11565,6 +11568,7 @@ define bfloat @flat_system_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_g
 ; GFX12-NEXT:    s_wait_bvhcnt 0x0
 ; GFX12-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-NEXT:    v_add_co_u32 v3, vcc_lo, 0x7fe, v0
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
 ; GFX12-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
@@ -11588,8 +11592,9 @@ define bfloat @flat_system_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_g
 ; GFX12-NEXT:    v_bfe_u32 v7, v5, 16, 1
 ; GFX12-NEXT:    v_or_b32_e32 v8, 0x400000, v5
 ; GFX12-NEXT:    v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_cndmask_b32_e32 v5, v7, v8, vcc_lo
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_lshrrev_b32_e32 v5, 16, v5
@@ -11610,7 +11615,6 @@ define bfloat @flat_system_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_g
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX12-NEXT:    v_lshrrev_b32_e32 v0, v3, v5
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: flat_system_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
@@ -11923,6 +11927,7 @@ define void @flat_system_atomic_fmax_noret_bf16__offset12b_pos__amdgpu_no_fine_g
 ; GFX12-NEXT:    s_wait_bvhcnt 0x0
 ; GFX12-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-NEXT:    v_add_co_u32 v4, vcc_lo, 0x7fe, v0
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
 ; GFX12-NEXT:    v_lshlrev_b32_e32 v6, 16, v2
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
@@ -11946,6 +11951,7 @@ define void @flat_system_atomic_fmax_noret_bf16__offset12b_pos__amdgpu_no_fine_g
 ; GFX12-NEXT:    v_or_b32_e32 v8, 0x400000, v2
 ; GFX12-NEXT:    v_cmp_u_f32_e32 vcc_lo, v2, v2
 ; GFX12-NEXT:    v_add3_u32 v7, v7, v2, 0x7fff
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_cndmask_b32_e32 v2, v7, v8, vcc_lo
 ; GFX12-NEXT:    v_lshrrev_b32_e32 v2, 16, v2
@@ -11966,7 +11972,6 @@ define void @flat_system_atomic_fmax_noret_bf16__offset12b_pos__amdgpu_no_fine_g
 ; GFX12-NEXT:    s_cbranch_execnz .LBB45_1
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: flat_system_atomic_fmax_noret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
@@ -12296,7 +12301,6 @@ define <2 x half> @flat_agent_atomic_fmax_ret_v2f16__amdgpu_no_fine_grained_memo
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX12-NEXT:    v_mov_b32_e32 v0, v3
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: flat_agent_atomic_fmax_ret_v2f16__amdgpu_no_fine_grained_memory:
@@ -12532,7 +12536,6 @@ define <2 x half> @flat_agent_atomic_fmax_ret_v2f16__offset12b_pos__amdgpu_no_fi
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX12-NEXT:    v_mov_b32_e32 v0, v3
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: flat_agent_atomic_fmax_ret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory:
@@ -12771,7 +12774,6 @@ define <2 x half> @flat_agent_atomic_fmax_ret_v2f16__offset12b_neg__amdgpu_no_fi
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX12-NEXT:    v_mov_b32_e32 v0, v3
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: flat_agent_atomic_fmax_ret_v2f16__offset12b_neg__amdgpu_no_fine_grained_memory:
@@ -13027,7 +13029,6 @@ define void @flat_agent_atomic_fmax_noret_v2f16__amdgpu_no_fine_grained_memory(p
 ; GFX12-NEXT:    s_cbranch_execnz .LBB49_1
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: flat_agent_atomic_fmax_noret_v2f16__amdgpu_no_fine_grained_memory:
@@ -13254,7 +13255,6 @@ define void @flat_agent_atomic_fmax_noret_v2f16__offset12b_pos__amdgpu_no_fine_g
 ; GFX12-NEXT:    s_cbranch_execnz .LBB50_1
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: flat_agent_atomic_fmax_noret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory:
@@ -13488,7 +13488,6 @@ define void @flat_agent_atomic_fmax_noret_v2f16__offset12b_neg__amdgpu_no_fine_g
 ; GFX12-NEXT:    s_cbranch_execnz .LBB51_1
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: flat_agent_atomic_fmax_noret_v2f16__offset12b_neg__amdgpu_no_fine_grained_memory:
@@ -13742,7 +13741,6 @@ define <2 x half> @flat_system_atomic_fmax_ret_v2f16__offset12b_pos__amdgpu_no_f
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX12-NEXT:    v_mov_b32_e32 v0, v3
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: flat_system_atomic_fmax_ret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory:
@@ -13983,7 +13981,6 @@ define void @flat_system_atomic_fmax_noret_v2f16__offset12b_pos__amdgpu_no_fine_
 ; GFX12-NEXT:    s_cbranch_execnz .LBB53_1
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: flat_system_atomic_fmax_noret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory:
@@ -14223,9 +14220,10 @@ define <2 x bfloat> @flat_agent_atomic_fmax_ret_v2bf16__amdgpu_no_fine_grained_m
 ; GFX12-NEXT:    v_bfe_u32 v7, v3, 16, 1
 ; GFX12-NEXT:    v_or_b32_e32 v9, 0x400000, v3
 ; GFX12-NEXT:    v_cmp_u_f32_e64 s0, v3, v3
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_cndmask_b32_e32 v5, v8, v10, vcc_lo
 ; GFX12-NEXT:    v_add3_u32 v7, v7, v3, 0x7fff
+; GFX12-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_cndmask_b32_e64 v3, v7, v9, s0
 ; GFX12-NEXT:    v_perm_b32 v5, v5, v3, 0x7060302
@@ -14242,7 +14240,6 @@ define <2 x bfloat> @flat_agent_atomic_fmax_ret_v2bf16__amdgpu_no_fine_grained_m
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s1
 ; GFX12-NEXT:    v_mov_b32_e32 v0, v3
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: flat_agent_atomic_fmax_ret_v2bf16__amdgpu_no_fine_grained_memory:
@@ -14569,9 +14566,10 @@ define <2 x bfloat> @flat_agent_atomic_fmax_ret_v2bf16__offset12b_pos__amdgpu_no
 ; GFX12-NEXT:    v_bfe_u32 v7, v3, 16, 1
 ; GFX12-NEXT:    v_or_b32_e32 v9, 0x400000, v3
 ; GFX12-NEXT:    v_cmp_u_f32_e64 s0, v3, v3
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_cndmask_b32_e32 v5, v8, v10, vcc_lo
 ; GFX12-NEXT:    v_add3_u32 v7, v7, v3, 0x7fff
+; GFX12-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_cndmask_b32_e64 v3, v7, v9, s0
 ; GFX12-NEXT:    v_perm_b32 v5, v5, v3, 0x7060302
@@ -14588,7 +14586,6 @@ define <2 x bfloat> @flat_agent_atomic_fmax_ret_v2bf16__offset12b_pos__amdgpu_no
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s1
 ; GFX12-NEXT:    v_mov_b32_e32 v0, v3
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: flat_agent_atomic_fmax_ret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
@@ -14918,9 +14915,10 @@ define <2 x bfloat> @flat_agent_atomic_fmax_ret_v2bf16__offset12b_neg__amdgpu_no
 ; GFX12-NEXT:    v_bfe_u32 v7, v3, 16, 1
 ; GFX12-NEXT:    v_or_b32_e32 v9, 0x400000, v3
 ; GFX12-NEXT:    v_cmp_u_f32_e64 s0, v3, v3
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_cndmask_b32_e32 v5, v8, v10, vcc_lo
 ; GFX12-NEXT:    v_add3_u32 v7, v7, v3, 0x7fff
+; GFX12-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_cndmask_b32_e64 v3, v7, v9, s0
 ; GFX12-NEXT:    v_perm_b32 v5, v5, v3, 0x7060302
@@ -14937,7 +14935,6 @@ define <2 x bfloat> @flat_agent_atomic_fmax_ret_v2bf16__offset12b_neg__amdgpu_no
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s1
 ; GFX12-NEXT:    v_mov_b32_e32 v0, v3
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: flat_agent_atomic_fmax_ret_v2bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
@@ -15284,8 +15281,10 @@ define void @flat_agent_atomic_fmax_noret_v2bf16__amdgpu_no_fine_grained_memory(
 ; GFX12-NEXT:    v_add3_u32 v7, v7, v2, 0x7fff
 ; GFX12-NEXT:    v_add3_u32 v8, v8, v6, 0x7fff
 ; GFX12-NEXT:    v_cmp_u_f32_e64 s0, v2, v2
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-NEXT:    s_wait_alu 0xfffd
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX12-NEXT:    v_cndmask_b32_e32 v6, v8, v10, vcc_lo
+; GFX12-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-NEXT:    v_cndmask_b32_e64 v2, v7, v9, s0
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-NEXT:    v_perm_b32 v2, v6, v2, 0x7060302
@@ -15302,7 +15301,6 @@ define void @flat_agent_atomic_fmax_noret_v2bf16__amdgpu_no_fine_grained_memory(
 ; GFX12-NEXT:    s_cbranch_execnz .LBB57_1
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s1
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: flat_agent_atomic_fmax_noret_v2bf16__amdgpu_no_fine_grained_memory:
@@ -15619,8 +15617,10 @@ define void @flat_agent_atomic_fmax_noret_v2bf16__offset12b_pos__amdgpu_no_fine_
 ; GFX12-NEXT:    v_add3_u32 v7, v7, v2, 0x7fff
 ; GFX12-NEXT:    v_add3_u32 v8, v8, v6, 0x7fff
 ; GFX12-NEXT:    v_cmp_u_f32_e64 s0, v2, v2
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-NEXT:    s_wait_alu 0xfffd
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX12-NEXT:    v_cndmask_b32_e32 v6, v8, v10, vcc_lo
+; GFX12-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-NEXT:    v_cndmask_b32_e64 v2, v7, v9, s0
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-NEXT:    v_perm_b32 v2, v6, v2, 0x7060302
@@ -15637,7 +15637,6 @@ define void @flat_agent_atomic_fmax_noret_v2bf16__offset12b_pos__amdgpu_no_fine_
 ; GFX12-NEXT:    s_cbranch_execnz .LBB58_1
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s1
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: flat_agent_atomic_fmax_noret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
@@ -15961,8 +15960,10 @@ define void @flat_agent_atomic_fmax_noret_v2bf16__offset12b_neg__amdgpu_no_fine_
 ; GFX12-NEXT:    v_add3_u32 v7, v7, v2, 0x7fff
 ; GFX12-NEXT:    v_add3_u32 v8, v8, v6, 0x7fff
 ; GFX12-NEXT:    v_cmp_u_f32_e64 s0, v2, v2
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-NEXT:    s_wait_alu 0xfffd
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX12-NEXT:    v_cndmask_b32_e32 v6, v8, v10, vcc_lo
+; GFX12-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-NEXT:    v_cndmask_b32_e64 v2, v7, v9, s0
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-NEXT:    v_perm_b32 v2, v6, v2, 0x7060302
@@ -15979,7 +15980,6 @@ define void @flat_agent_atomic_fmax_noret_v2bf16__offset12b_neg__amdgpu_no_fine_
 ; GFX12-NEXT:    s_cbranch_execnz .LBB59_1
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s1
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: flat_agent_atomic_fmax_noret_v2bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
@@ -16322,9 +16322,10 @@ define <2 x bfloat> @flat_system_atomic_fmax_ret_v2bf16__offset12b_pos__amdgpu_n
 ; GFX12-NEXT:    v_bfe_u32 v7, v3, 16, 1
 ; GFX12-NEXT:    v_or_b32_e32 v9, 0x400000, v3
 ; GFX12-NEXT:    v_cmp_u_f32_e64 s0, v3, v3
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_cndmask_b32_e32 v5, v8, v10, vcc_lo
 ; GFX12-NEXT:    v_add3_u32 v7, v7, v3, 0x7fff
+; GFX12-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_cndmask_b32_e64 v3, v7, v9, s0
 ; GFX12-NEXT:    v_perm_b32 v5, v5, v3, 0x7060302
@@ -16342,7 +16343,6 @@ define <2 x bfloat> @flat_system_atomic_fmax_ret_v2bf16__offset12b_pos__amdgpu_n
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s1
 ; GFX12-NEXT:    v_mov_b32_e32 v0, v3
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: flat_system_atomic_fmax_ret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
@@ -16673,8 +16673,10 @@ define void @flat_system_atomic_fmax_noret_v2bf16__offset12b_pos__amdgpu_no_fine
 ; GFX12-NEXT:    v_add3_u32 v7, v7, v2, 0x7fff
 ; GFX12-NEXT:    v_add3_u32 v8, v8, v6, 0x7fff
 ; GFX12-NEXT:    v_cmp_u_f32_e64 s0, v2, v2
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-NEXT:    s_wait_alu 0xfffd
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX12-NEXT:    v_cndmask_b32_e32 v6, v8, v10, vcc_lo
+; GFX12-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-NEXT:    v_cndmask_b32_e64 v2, v7, v9, s0
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-NEXT:    v_perm_b32 v2, v6, v2, 0x7060302
@@ -16692,7 +16694,6 @@ define void @flat_system_atomic_fmax_noret_v2bf16__offset12b_pos__amdgpu_no_fine
 ; GFX12-NEXT:    s_cbranch_execnz .LBB61_1
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s1
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: flat_system_atomic_fmax_noret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
diff --git a/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fmin.ll b/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fmin.ll
index d96d3db9f005d..065596776cf73 100644
--- a/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fmin.ll
+++ b/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fmin.ll
@@ -2808,6 +2808,7 @@ define double @flat_agent_atomic_fmin_ret_f64__amdgpu_no_fine_grained_memory(ptr
 ; GFX12-NEXT:    s_cbranch_execz .LBB18_6
 ; GFX12-NEXT:  ; %bb.5: ; %atomicrmw.private
 ; GFX12-NEXT:    v_cmp_ne_u64_e32 vcc_lo, 0, v[0:1]
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_cndmask_b32_e32 v6, -1, v0, vcc_lo
 ; GFX12-NEXT:    scratch_load_b64 v[2:3], v6, off
 ; GFX12-NEXT:    s_wait_loadcnt 0x0
@@ -2819,7 +2820,6 @@ define double @flat_agent_atomic_fmin_ret_f64__amdgpu_no_fine_grained_memory(ptr
 ; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX12-NEXT:    v_dual_mov_b32 v0, v2 :: v_dual_mov_b32 v1, v3
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: flat_agent_atomic_fmin_ret_f64__amdgpu_no_fine_grained_memory:
@@ -2917,8 +2917,8 @@ define double @flat_agent_atomic_fmin_ret_f64__amdgpu_no_fine_grained_memory(ptr
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    v_mov_b32_e32 v5, v1
-; GFX10-NEXT:    v_mov_b32_e32 v4, v0
 ; GFX10-NEXT:    s_mov_b64 s[4:5], src_private_base
+; GFX10-NEXT:    v_mov_b32_e32 v4, v0
 ; GFX10-NEXT:    ; implicit-def: $vgpr0_vgpr1
 ; GFX10-NEXT:    v_cmp_ne_u32_e32 vcc_lo, s5, v5
 ; GFX10-NEXT:    s_and_saveexec_b32 s4, vcc_lo
@@ -3161,6 +3161,7 @@ define double @flat_agent_atomic_fmin_ret_f64__offset12b_pos__amdgpu_no_fine_gra
 ; GFX12-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-NEXT:    v_max_num_f64_e32 v[2:3], v[2:3], v[2:3]
 ; GFX12-NEXT:    v_add_co_u32 v4, vcc_lo, 0x7f8, v0
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_add_co_ci_u32_e32 v5, vcc_lo, 0, v1, vcc_lo
 ; GFX12-NEXT:    s_mov_b64 s[0:1], src_private_base
 ; GFX12-NEXT:    s_mov_b32 s0, exec_lo
@@ -3177,7 +3178,7 @@ define double @flat_agent_atomic_fmin_ret_f64__offset12b_pos__amdgpu_no_fine_gra
 ; GFX12-NEXT:  .LBB19_2: ; %atomicrmw.phi
 ; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT:    s_wait_alu 0xfffe
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ; GFX12-NEXT:  .LBB19_3: ; %atomicrmw.global
 ; GFX12-NEXT:    flat_load_b64 v[0:1], v[4:5]
@@ -3207,6 +3208,7 @@ define double @flat_agent_atomic_fmin_ret_f64__offset12b_pos__amdgpu_no_fine_gra
 ; GFX12-NEXT:    s_cbranch_execz .LBB19_2
 ; GFX12-NEXT:  .LBB19_6: ; %atomicrmw.private
 ; GFX12-NEXT:    v_cmp_ne_u64_e32 vcc_lo, 0, v[4:5]
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_cndmask_b32_e32 v6, -1, v4, vcc_lo
 ; GFX12-NEXT:    scratch_load_b64 v[0:1], v6, off
 ; GFX12-NEXT:    s_wait_loadcnt 0x0
@@ -3216,7 +3218,6 @@ define double @flat_agent_atomic_fmin_ret_f64__offset12b_pos__amdgpu_no_fine_gra
 ; GFX12-NEXT:    scratch_store_b64 v6, v[2:3], off
 ; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: flat_agent_atomic_fmin_ret_f64__offset12b_pos__amdgpu_no_fine_grained_memory:
@@ -3573,6 +3574,7 @@ define double @flat_agent_atomic_fmin_ret_f64__offset12b_neg__amdgpu_no_fine_gra
 ; GFX12-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-NEXT:    v_max_num_f64_e32 v[2:3], v[2:3], v[2:3]
 ; GFX12-NEXT:    v_add_co_u32 v4, vcc_lo, 0xfffff800, v0
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_add_co_ci_u32_e32 v5, vcc_lo, -1, v1, vcc_lo
 ; GFX12-NEXT:    s_mov_b64 s[0:1], src_private_base
 ; GFX12-NEXT:    s_mov_b32 s0, exec_lo
@@ -3589,7 +3591,7 @@ define double @flat_agent_atomic_fmin_ret_f64__offset12b_neg__amdgpu_no_fine_gra
 ; GFX12-NEXT:  .LBB20_2: ; %atomicrmw.phi
 ; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT:    s_wait_alu 0xfffe
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ; GFX12-NEXT:  .LBB20_3: ; %atomicrmw.global
 ; GFX12-NEXT:    flat_load_b64 v[0:1], v[4:5]
@@ -3619,6 +3621,7 @@ define double @flat_agent_atomic_fmin_ret_f64__offset12b_neg__amdgpu_no_fine_gra
 ; GFX12-NEXT:    s_cbranch_execz .LBB20_2
 ; GFX12-NEXT:  .LBB20_6: ; %atomicrmw.private
 ; GFX12-NEXT:    v_cmp_ne_u64_e32 vcc_lo, 0, v[4:5]
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_cndmask_b32_e32 v6, -1, v4, vcc_lo
 ; GFX12-NEXT:    scratch_load_b64 v[0:1], v6, off
 ; GFX12-NEXT:    s_wait_loadcnt 0x0
@@ -3628,7 +3631,6 @@ define double @flat_agent_atomic_fmin_ret_f64__offset12b_neg__amdgpu_no_fine_gra
 ; GFX12-NEXT:    scratch_store_b64 v6, v[2:3], off
 ; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: flat_agent_atomic_fmin_ret_f64__offset12b_neg__amdgpu_no_fine_grained_memory:
@@ -3998,7 +4000,6 @@ define void @flat_agent_atomic_fmin_noret_f64__amdgpu_no_fine_grained_memory(ptr
 ; GFX12-NEXT:  .LBB21_2: ; %atomicrmw.phi
 ; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ; GFX12-NEXT:  .LBB21_3: ; %atomicrmw.global
 ; GFX12-NEXT:    flat_load_b64 v[4:5], v[0:1]
@@ -4028,6 +4029,7 @@ define void @flat_agent_atomic_fmin_noret_f64__amdgpu_no_fine_grained_memory(ptr
 ; GFX12-NEXT:    s_cbranch_execz .LBB21_2
 ; GFX12-NEXT:  .LBB21_6: ; %atomicrmw.private
 ; GFX12-NEXT:    v_cmp_ne_u64_e32 vcc_lo, 0, v[0:1]
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_cndmask_b32_e32 v2, -1, v0, vcc_lo
 ; GFX12-NEXT:    scratch_load_b64 v[0:1], v2, off
 ; GFX12-NEXT:    s_wait_loadcnt 0x0
@@ -4037,7 +4039,6 @@ define void @flat_agent_atomic_fmin_noret_f64__amdgpu_no_fine_grained_memory(ptr
 ; GFX12-NEXT:    scratch_store_b64 v2, v[0:1], off
 ; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: flat_agent_atomic_fmin_noret_f64__amdgpu_no_fine_grained_memory:
@@ -4372,6 +4373,7 @@ define void @flat_agent_atomic_fmin_noret_f64__offset12b_pos__amdgpu_no_fine_gra
 ; GFX12-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-NEXT:    v_max_num_f64_e32 v[4:5], v[2:3], v[2:3]
 ; GFX12-NEXT:    v_add_co_u32 v6, vcc_lo, 0x7f8, v0
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_add_co_ci_u32_e32 v7, vcc_lo, 0, v1, vcc_lo
 ; GFX12-NEXT:    s_mov_b64 s[0:1], src_private_base
 ; GFX12-NEXT:    s_mov_b32 s0, exec_lo
@@ -4387,7 +4389,7 @@ define void @flat_agent_atomic_fmin_noret_f64__offset12b_pos__amdgpu_no_fine_gra
 ; GFX12-NEXT:  .LBB22_2: ; %atomicrmw.phi
 ; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT:    s_wait_alu 0xfffe
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ; GFX12-NEXT:  .LBB22_3: ; %atomicrmw.global
 ; GFX12-NEXT:    flat_load_b64 v[2:3], v[6:7]
@@ -4417,6 +4419,7 @@ define void @flat_agent_atomic_fmin_noret_f64__offset12b_pos__amdgpu_no_fine_gra
 ; GFX12-NEXT:    s_cbranch_execz .LBB22_2
 ; GFX12-NEXT:  .LBB22_6: ; %atomicrmw.private
 ; GFX12-NEXT:    v_cmp_ne_u64_e32 vcc_lo, 0, v[6:7]
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_cndmask_b32_e32 v2, -1, v6, vcc_lo
 ; GFX12-NEXT:    scratch_load_b64 v[0:1], v2, off
 ; GFX12-NEXT:    s_wait_loadcnt 0x0
@@ -4426,7 +4429,6 @@ define void @flat_agent_atomic_fmin_noret_f64__offset12b_pos__amdgpu_no_fine_gra
 ; GFX12-NEXT:    scratch_store_b64 v2, v[0:1], off
 ; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: flat_agent_atomic_fmin_noret_f64__offset12b_pos__amdgpu_no_fine_grained_memory:
@@ -4777,6 +4779,7 @@ define void @flat_agent_atomic_fmin_noret_f64__offset12b_neg__amdgpu_no_fine_gra
 ; GFX12-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-NEXT:    v_max_num_f64_e32 v[4:5], v[2:3], v[2:3]
 ; GFX12-NEXT:    v_add_co_u32 v6, vcc_lo, 0xfffff800, v0
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_add_co_ci_u32_e32 v7, vcc_lo, -1, v1, vcc_lo
 ; GFX12-NEXT:    s_mov_b64 s[0:1], src_private_base
 ; GFX12-NEXT:    s_mov_b32 s0, exec_lo
@@ -4792,7 +4795,7 @@ define void @flat_agent_atomic_fmin_noret_f64__offset12b_neg__amdgpu_no_fine_gra
 ; GFX12-NEXT:  .LBB23_2: ; %atomicrmw.phi
 ; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT:    s_wait_alu 0xfffe
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ; GFX12-NEXT:  .LBB23_3: ; %atomicrmw.global
 ; GFX12-NEXT:    flat_load_b64 v[2:3], v[6:7]
@@ -4822,6 +4825,7 @@ define void @flat_agent_atomic_fmin_noret_f64__offset12b_neg__amdgpu_no_fine_gra
 ; GFX12-NEXT:    s_cbranch_execz .LBB23_2
 ; GFX12-NEXT:  .LBB23_6: ; %atomicrmw.private
 ; GFX12-NEXT:    v_cmp_ne_u64_e32 vcc_lo, 0, v[6:7]
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_cndmask_b32_e32 v2, -1, v6, vcc_lo
 ; GFX12-NEXT:    scratch_load_b64 v[0:1], v2, off
 ; GFX12-NEXT:    s_wait_loadcnt 0x0
@@ -4831,7 +4835,6 @@ define void @flat_agent_atomic_fmin_noret_f64__offset12b_neg__amdgpu_no_fine_gra
 ; GFX12-NEXT:    scratch_store_b64 v2, v[0:1], off
 ; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: flat_agent_atomic_fmin_noret_f64__offset12b_neg__amdgpu_no_fine_grained_memory:
@@ -5219,6 +5222,7 @@ define double @flat_agent_atomic_fmin_ret_f64__amdgpu_no_remote_memory(ptr %ptr,
 ; GFX12-NEXT:    s_cbranch_execz .LBB24_6
 ; GFX12-NEXT:  ; %bb.5: ; %atomicrmw.private
 ; GFX12-NEXT:    v_cmp_ne_u64_e32 vcc_lo, 0, v[0:1]
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_cndmask_b32_e32 v6, -1, v0, vcc_lo
 ; GFX12-NEXT:    scratch_load_b64 v[2:3], v6, off
 ; GFX12-NEXT:    s_wait_loadcnt 0x0
@@ -5230,7 +5234,6 @@ define double @flat_agent_atomic_fmin_ret_f64__amdgpu_no_remote_memory(ptr %ptr,
 ; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX12-NEXT:    v_dual_mov_b32 v0, v2 :: v_dual_mov_b32 v1, v3
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: flat_agent_atomic_fmin_ret_f64__amdgpu_no_remote_memory:
@@ -5642,6 +5645,7 @@ define double @flat_agent_atomic_fmin_ret_f64__amdgpu_no_fine_grained_memory__am
 ; GFX12-NEXT:    s_cbranch_execz .LBB25_6
 ; GFX12-NEXT:  ; %bb.5: ; %atomicrmw.private
 ; GFX12-NEXT:    v_cmp_ne_u64_e32 vcc_lo, 0, v[0:1]
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_cndmask_b32_e32 v6, -1, v0, vcc_lo
 ; GFX12-NEXT:    scratch_load_b64 v[2:3], v6, off
 ; GFX12-NEXT:    s_wait_loadcnt 0x0
@@ -5653,7 +5657,6 @@ define double @flat_agent_atomic_fmin_ret_f64__amdgpu_no_fine_grained_memory__am
 ; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX12-NEXT:    v_dual_mov_b32 v0, v2 :: v_dual_mov_b32 v1, v3
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: flat_agent_atomic_fmin_ret_f64__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
@@ -5751,8 +5754,8 @@ define double @flat_agent_atomic_fmin_ret_f64__amdgpu_no_fine_grained_memory__am
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    v_mov_b32_e32 v5, v1
-; GFX10-NEXT:    v_mov_b32_e32 v4, v0
 ; GFX10-NEXT:    s_mov_b64 s[4:5], src_private_base
+; GFX10-NEXT:    v_mov_b32_e32 v4, v0
 ; GFX10-NEXT:    ; implicit-def: $vgpr0_vgpr1
 ; GFX10-NEXT:    v_cmp_ne_u32_e32 vcc_lo, s5, v5
 ; GFX10-NEXT:    s_and_saveexec_b32 s4, vcc_lo
@@ -6034,7 +6037,6 @@ define half @flat_agent_atomic_fmin_ret_f16__amdgpu_no_fine_grained_memory(ptr %
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX12-NEXT:    v_lshrrev_b32_e32 v0, v3, v5
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: flat_agent_atomic_fmin_ret_f16__amdgpu_no_fine_grained_memory:
@@ -6301,6 +6303,7 @@ define half @flat_agent_atomic_fmin_ret_f16__offset12b_pos__amdgpu_no_fine_grain
 ; GFX12-NEXT:    s_wait_bvhcnt 0x0
 ; GFX12-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-NEXT:    v_add_co_u32 v3, vcc_lo, 0x7fe, v0
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
 ; GFX12-NEXT:    v_max_num_f16_e32 v2, v2, v2
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
@@ -6338,7 +6341,6 @@ define half @flat_agent_atomic_fmin_ret_f16__offset12b_pos__amdgpu_no_fine_grain
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX12-NEXT:    v_lshrrev_b32_e32 v0, v3, v5
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: flat_agent_atomic_fmin_ret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
@@ -6614,6 +6616,7 @@ define half @flat_agent_atomic_fmin_ret_f16__offset12b_neg__amdgpu_no_fine_grain
 ; GFX12-NEXT:    s_wait_bvhcnt 0x0
 ; GFX12-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-NEXT:    v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, -1, v1, vcc_lo
 ; GFX12-NEXT:    v_max_num_f16_e32 v2, v2, v2
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
@@ -6651,7 +6654,6 @@ define half @flat_agent_atomic_fmin_ret_f16__offset12b_neg__amdgpu_no_fine_grain
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX12-NEXT:    v_lshrrev_b32_e32 v0, v3, v5
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: flat_agent_atomic_fmin_ret_f16__offset12b_neg__amdgpu_no_fine_grained_memory:
@@ -6963,7 +6965,6 @@ define void @flat_agent_atomic_fmin_noret_f16__amdgpu_no_fine_grained_memory(ptr
 ; GFX12-NEXT:    s_cbranch_execnz .LBB29_1
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: flat_agent_atomic_fmin_noret_f16__amdgpu_no_fine_grained_memory:
@@ -7222,6 +7223,7 @@ define void @flat_agent_atomic_fmin_noret_f16__offset12b_pos__amdgpu_no_fine_gra
 ; GFX12-NEXT:    s_wait_bvhcnt 0x0
 ; GFX12-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-NEXT:    v_add_co_u32 v4, vcc_lo, 0x7fe, v0
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
 ; GFX12-NEXT:    v_max_num_f16_e32 v6, v2, v2
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
@@ -7258,7 +7260,6 @@ define void @flat_agent_atomic_fmin_noret_f16__offset12b_pos__amdgpu_no_fine_gra
 ; GFX12-NEXT:    s_cbranch_execnz .LBB30_1
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: flat_agent_atomic_fmin_noret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
@@ -7526,6 +7527,7 @@ define void @flat_agent_atomic_fmin_noret_f16__offset12b_neg__amdgpu_no_fine_gra
 ; GFX12-NEXT:    s_wait_bvhcnt 0x0
 ; GFX12-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-NEXT:    v_add_co_u32 v4, vcc_lo, 0xfffff800, v0
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, -1, v1, vcc_lo
 ; GFX12-NEXT:    v_max_num_f16_e32 v6, v2, v2
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
@@ -7562,7 +7564,6 @@ define void @flat_agent_atomic_fmin_noret_f16__offset12b_neg__amdgpu_no_fine_gra
 ; GFX12-NEXT:    s_cbranch_execnz .LBB31_1
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: flat_agent_atomic_fmin_noret_f16__offset12b_neg__amdgpu_no_fine_grained_memory:
@@ -7856,7 +7857,6 @@ define half @flat_agent_atomic_fmin_ret_f16__offset12b_pos__align4__amdgpu_no_fi
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX12-NEXT:    v_mov_b32_e32 v0, v3
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: flat_agent_atomic_fmin_ret_f16__offset12b_pos__align4__amdgpu_no_fine_grained_memory:
@@ -8092,7 +8092,6 @@ define void @flat_agent_atomic_fmin_noret_f16__offset12b__align4_pos__amdgpu_no_
 ; GFX12-NEXT:    s_cbranch_execnz .LBB33_1
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: flat_agent_atomic_fmin_noret_f16__offset12b__align4_pos__amdgpu_no_fine_grained_memory:
@@ -8299,6 +8298,7 @@ define half @flat_system_atomic_fmin_ret_f16__offset12b_pos__amdgpu_no_fine_grai
 ; GFX12-NEXT:    s_wait_bvhcnt 0x0
 ; GFX12-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-NEXT:    v_add_co_u32 v3, vcc_lo, 0x7fe, v0
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
 ; GFX12-NEXT:    v_max_num_f16_e32 v2, v2, v2
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
@@ -8337,7 +8337,6 @@ define half @flat_system_atomic_fmin_ret_f16__offset12b_pos__amdgpu_no_fine_grai
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX12-NEXT:    v_lshrrev_b32_e32 v0, v3, v5
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: flat_system_atomic_fmin_ret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
@@ -8615,6 +8614,7 @@ define void @flat_system_atomic_fmin_noret_f16__offset12b_pos__amdgpu_no_fine_gr
 ; GFX12-NEXT:    s_wait_bvhcnt 0x0
 ; GFX12-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-NEXT:    v_add_co_u32 v4, vcc_lo, 0x7fe, v0
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
 ; GFX12-NEXT:    v_max_num_f16_e32 v6, v2, v2
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
@@ -8652,7 +8652,6 @@ define void @flat_system_atomic_fmin_noret_f16__offset12b_pos__amdgpu_no_fine_gr
 ; GFX12-NEXT:    s_cbranch_execnz .LBB35_1
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: flat_system_atomic_fmin_noret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
@@ -8947,8 +8946,9 @@ define bfloat @flat_agent_atomic_fmin_ret_bf16__amdgpu_no_fine_grained_memory(pt
 ; GFX12-NEXT:    v_bfe_u32 v7, v5, 16, 1
 ; GFX12-NEXT:    v_or_b32_e32 v8, 0x400000, v5
 ; GFX12-NEXT:    v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_cndmask_b32_e32 v5, v7, v8, vcc_lo
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_lshrrev_b32_e32 v5, 16, v5
@@ -8968,7 +8968,6 @@ define bfloat @flat_agent_atomic_fmin_ret_bf16__amdgpu_no_fine_grained_memory(pt
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX12-NEXT:    v_lshrrev_b32_e32 v0, v3, v5
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: flat_agent_atomic_fmin_ret_bf16__amdgpu_no_fine_grained_memory:
@@ -9269,6 +9268,7 @@ define bfloat @flat_agent_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_gr
 ; GFX12-NEXT:    s_wait_bvhcnt 0x0
 ; GFX12-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-NEXT:    v_add_co_u32 v3, vcc_lo, 0x7fe, v0
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
 ; GFX12-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
@@ -9292,8 +9292,9 @@ define bfloat @flat_agent_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_gr
 ; GFX12-NEXT:    v_bfe_u32 v7, v5, 16, 1
 ; GFX12-NEXT:    v_or_b32_e32 v8, 0x400000, v5
 ; GFX12-NEXT:    v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_cndmask_b32_e32 v5, v7, v8, vcc_lo
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_lshrrev_b32_e32 v5, 16, v5
@@ -9313,7 +9314,6 @@ define bfloat @flat_agent_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_gr
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX12-NEXT:    v_lshrrev_b32_e32 v0, v3, v5
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: flat_agent_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
@@ -9624,6 +9624,7 @@ define bfloat @flat_agent_atomic_fmin_ret_bf16__offset12b_neg__amdgpu_no_fine_gr
 ; GFX12-NEXT:    s_wait_bvhcnt 0x0
 ; GFX12-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-NEXT:    v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, -1, v1, vcc_lo
 ; GFX12-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
@@ -9647,8 +9648,9 @@ define bfloat @flat_agent_atomic_fmin_ret_bf16__offset12b_neg__amdgpu_no_fine_gr
 ; GFX12-NEXT:    v_bfe_u32 v7, v5, 16, 1
 ; GFX12-NEXT:    v_or_b32_e32 v8, 0x400000, v5
 ; GFX12-NEXT:    v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_cndmask_b32_e32 v5, v7, v8, vcc_lo
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_lshrrev_b32_e32 v5, 16, v5
@@ -9668,7 +9670,6 @@ define bfloat @flat_agent_atomic_fmin_ret_bf16__offset12b_neg__amdgpu_no_fine_gr
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX12-NEXT:    v_lshrrev_b32_e32 v0, v3, v5
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: flat_agent_atomic_fmin_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
@@ -10001,6 +10002,7 @@ define void @flat_agent_atomic_fmin_noret_bf16__amdgpu_no_fine_grained_memory(pt
 ; GFX12-NEXT:    v_or_b32_e32 v8, 0x400000, v3
 ; GFX12-NEXT:    v_cmp_u_f32_e32 vcc_lo, v3, v3
 ; GFX12-NEXT:    v_add3_u32 v7, v7, v3, 0x7fff
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_cndmask_b32_e32 v3, v7, v8, vcc_lo
 ; GFX12-NEXT:    v_lshrrev_b32_e32 v3, 16, v3
@@ -10020,7 +10022,6 @@ define void @flat_agent_atomic_fmin_noret_bf16__amdgpu_no_fine_grained_memory(pt
 ; GFX12-NEXT:    s_cbranch_execnz .LBB39_1
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: flat_agent_atomic_fmin_noret_bf16__amdgpu_no_fine_grained_memory:
@@ -10312,6 +10313,7 @@ define void @flat_agent_atomic_fmin_noret_bf16__offset12b_pos__amdgpu_no_fine_gr
 ; GFX12-NEXT:    s_wait_bvhcnt 0x0
 ; GFX12-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-NEXT:    v_add_co_u32 v4, vcc_lo, 0x7fe, v0
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
 ; GFX12-NEXT:    v_lshlrev_b32_e32 v6, 16, v2
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
@@ -10335,6 +10337,7 @@ define void @flat_agent_atomic_fmin_noret_bf16__offset12b_pos__amdgpu_no_fine_gr
 ; GFX12-NEXT:    v_or_b32_e32 v8, 0x400000, v2
 ; GFX12-NEXT:    v_cmp_u_f32_e32 vcc_lo, v2, v2
 ; GFX12-NEXT:    v_add3_u32 v7, v7, v2, 0x7fff
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_cndmask_b32_e32 v2, v7, v8, vcc_lo
 ; GFX12-NEXT:    v_lshrrev_b32_e32 v2, 16, v2
@@ -10354,7 +10357,6 @@ define void @flat_agent_atomic_fmin_noret_bf16__offset12b_pos__amdgpu_no_fine_gr
 ; GFX12-NEXT:    s_cbranch_execnz .LBB40_1
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: flat_agent_atomic_fmin_noret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
@@ -10656,6 +10658,7 @@ define void @flat_agent_atomic_fmin_noret_bf16__offset12b_neg__amdgpu_no_fine_gr
 ; GFX12-NEXT:    s_wait_bvhcnt 0x0
 ; GFX12-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-NEXT:    v_add_co_u32 v4, vcc_lo, 0xfffff800, v0
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, -1, v1, vcc_lo
 ; GFX12-NEXT:    v_lshlrev_b32_e32 v6, 16, v2
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
@@ -10679,6 +10682,7 @@ define void @flat_agent_atomic_fmin_noret_bf16__offset12b_neg__amdgpu_no_fine_gr
 ; GFX12-NEXT:    v_or_b32_e32 v8, 0x400000, v2
 ; GFX12-NEXT:    v_cmp_u_f32_e32 vcc_lo, v2, v2
 ; GFX12-NEXT:    v_add3_u32 v7, v7, v2, 0x7fff
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_cndmask_b32_e32 v2, v7, v8, vcc_lo
 ; GFX12-NEXT:    v_lshrrev_b32_e32 v2, 16, v2
@@ -10698,7 +10702,6 @@ define void @flat_agent_atomic_fmin_noret_bf16__offset12b_neg__amdgpu_no_fine_gr
 ; GFX12-NEXT:    s_cbranch_execnz .LBB41_1
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: flat_agent_atomic_fmin_noret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
@@ -11015,6 +11018,7 @@ define bfloat @flat_agent_atomic_fmin_ret_bf16__offset12b_pos__align4__amdgpu_no
 ; GFX12-NEXT:    v_or_b32_e32 v6, 0x400000, v3
 ; GFX12-NEXT:    v_cmp_u_f32_e32 vcc_lo, v3, v3
 ; GFX12-NEXT:    v_add3_u32 v5, v5, v3, 0x7fff
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_cndmask_b32_e32 v3, v5, v6, vcc_lo
 ; GFX12-NEXT:    v_lshrrev_b32_e32 v3, 16, v3
@@ -11033,7 +11037,6 @@ define bfloat @flat_agent_atomic_fmin_ret_bf16__offset12b_pos__align4__amdgpu_no
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX12-NEXT:    v_mov_b32_e32 v0, v3
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: flat_agent_atomic_fmin_ret_bf16__offset12b_pos__align4__amdgpu_no_fine_grained_memory:
@@ -11298,8 +11301,9 @@ define void @flat_agent_atomic_fmin_noret_bf16__offset12b__align4_pos__amdgpu_no
 ; GFX12-NEXT:    v_bfe_u32 v5, v2, 16, 1
 ; GFX12-NEXT:    v_or_b32_e32 v6, 0x400000, v2
 ; GFX12-NEXT:    v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_add3_u32 v5, v5, v2, 0x7fff
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_cndmask_b32_e32 v2, v5, v6, vcc_lo
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_lshrrev_b32_e32 v2, 16, v2
@@ -11317,7 +11321,6 @@ define void @flat_agent_atomic_fmin_noret_bf16__offset12b__align4_pos__amdgpu_no
 ; GFX12-NEXT:    s_cbranch_execnz .LBB43_1
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: flat_agent_atomic_fmin_noret_bf16__offset12b__align4_pos__amdgpu_no_fine_grained_memory:
@@ -11565,6 +11568,7 @@ define bfloat @flat_system_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_g
 ; GFX12-NEXT:    s_wait_bvhcnt 0x0
 ; GFX12-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-NEXT:    v_add_co_u32 v3, vcc_lo, 0x7fe, v0
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
 ; GFX12-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
@@ -11588,8 +11592,9 @@ define bfloat @flat_system_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_g
 ; GFX12-NEXT:    v_bfe_u32 v7, v5, 16, 1
 ; GFX12-NEXT:    v_or_b32_e32 v8, 0x400000, v5
 ; GFX12-NEXT:    v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_cndmask_b32_e32 v5, v7, v8, vcc_lo
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_lshrrev_b32_e32 v5, 16, v5
@@ -11610,7 +11615,6 @@ define bfloat @flat_system_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_g
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX12-NEXT:    v_lshrrev_b32_e32 v0, v3, v5
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: flat_system_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
@@ -11923,6 +11927,7 @@ define void @flat_system_atomic_fmin_noret_bf16__offset12b_pos__amdgpu_no_fine_g
 ; GFX12-NEXT:    s_wait_bvhcnt 0x0
 ; GFX12-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-NEXT:    v_add_co_u32 v4, vcc_lo, 0x7fe, v0
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
 ; GFX12-NEXT:    v_lshlrev_b32_e32 v6, 16, v2
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
@@ -11946,6 +11951,7 @@ define void @flat_system_atomic_fmin_noret_bf16__offset12b_pos__amdgpu_no_fine_g
 ; GFX12-NEXT:    v_or_b32_e32 v8, 0x400000, v2
 ; GFX12-NEXT:    v_cmp_u_f32_e32 vcc_lo, v2, v2
 ; GFX12-NEXT:    v_add3_u32 v7, v7, v2, 0x7fff
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_cndmask_b32_e32 v2, v7, v8, vcc_lo
 ; GFX12-NEXT:    v_lshrrev_b32_e32 v2, 16, v2
@@ -11966,7 +11972,6 @@ define void @flat_system_atomic_fmin_noret_bf16__offset12b_pos__amdgpu_no_fine_g
 ; GFX12-NEXT:    s_cbranch_execnz .LBB45_1
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: flat_system_atomic_fmin_noret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
@@ -12296,7 +12301,6 @@ define <2 x half> @flat_agent_atomic_fmin_ret_v2f16__amdgpu_no_fine_grained_memo
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX12-NEXT:    v_mov_b32_e32 v0, v3
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: flat_agent_atomic_fmin_ret_v2f16__amdgpu_no_fine_grained_memory:
@@ -12532,7 +12536,6 @@ define <2 x half> @flat_agent_atomic_fmin_ret_v2f16__offset12b_pos__amdgpu_no_fi
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX12-NEXT:    v_mov_b32_e32 v0, v3
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: flat_agent_atomic_fmin_ret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory:
@@ -12771,7 +12774,6 @@ define <2 x half> @flat_agent_atomic_fmin_ret_v2f16__offset12b_neg__amdgpu_no_fi
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX12-NEXT:    v_mov_b32_e32 v0, v3
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: flat_agent_atomic_fmin_ret_v2f16__offset12b_neg__amdgpu_no_fine_grained_memory:
@@ -13027,7 +13029,6 @@ define void @flat_agent_atomic_fmin_noret_v2f16__amdgpu_no_fine_grained_memory(p
 ; GFX12-NEXT:    s_cbranch_execnz .LBB49_1
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: flat_agent_atomic_fmin_noret_v2f16__amdgpu_no_fine_grained_memory:
@@ -13254,7 +13255,6 @@ define void @flat_agent_atomic_fmin_noret_v2f16__offset12b_pos__amdgpu_no_fine_g
 ; GFX12-NEXT:    s_cbranch_execnz .LBB50_1
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: flat_agent_atomic_fmin_noret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory:
@@ -13488,7 +13488,6 @@ define void @flat_agent_atomic_fmin_noret_v2f16__offset12b_neg__amdgpu_no_fine_g
 ; GFX12-NEXT:    s_cbranch_execnz .LBB51_1
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: flat_agent_atomic_fmin_noret_v2f16__offset12b_neg__amdgpu_no_fine_grained_memory:
@@ -13742,7 +13741,6 @@ define <2 x half> @flat_system_atomic_fmin_ret_v2f16__offset12b_pos__amdgpu_no_f
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX12-NEXT:    v_mov_b32_e32 v0, v3
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: flat_system_atomic_fmin_ret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory:
@@ -13983,7 +13981,6 @@ define void @flat_system_atomic_fmin_noret_v2f16__offset12b_pos__amdgpu_no_fine_
 ; GFX12-NEXT:    s_cbranch_execnz .LBB53_1
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: flat_system_atomic_fmin_noret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory:
@@ -14223,9 +14220,10 @@ define <2 x bfloat> @flat_agent_atomic_fmin_ret_v2bf16__amdgpu_no_fine_grained_m
 ; GFX12-NEXT:    v_bfe_u32 v7, v3, 16, 1
 ; GFX12-NEXT:    v_or_b32_e32 v9, 0x400000, v3
 ; GFX12-NEXT:    v_cmp_u_f32_e64 s0, v3, v3
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_cndmask_b32_e32 v5, v8, v10, vcc_lo
 ; GFX12-NEXT:    v_add3_u32 v7, v7, v3, 0x7fff
+; GFX12-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_cndmask_b32_e64 v3, v7, v9, s0
 ; GFX12-NEXT:    v_perm_b32 v5, v5, v3, 0x7060302
@@ -14242,7 +14240,6 @@ define <2 x bfloat> @flat_agent_atomic_fmin_ret_v2bf16__amdgpu_no_fine_grained_m
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s1
 ; GFX12-NEXT:    v_mov_b32_e32 v0, v3
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: flat_agent_atomic_fmin_ret_v2bf16__amdgpu_no_fine_grained_memory:
@@ -14569,9 +14566,10 @@ define <2 x bfloat> @flat_agent_atomic_fmin_ret_v2bf16__offset12b_pos__amdgpu_no
 ; GFX12-NEXT:    v_bfe_u32 v7, v3, 16, 1
 ; GFX12-NEXT:    v_or_b32_e32 v9, 0x400000, v3
 ; GFX12-NEXT:    v_cmp_u_f32_e64 s0, v3, v3
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_cndmask_b32_e32 v5, v8, v10, vcc_lo
 ; GFX12-NEXT:    v_add3_u32 v7, v7, v3, 0x7fff
+; GFX12-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_cndmask_b32_e64 v3, v7, v9, s0
 ; GFX12-NEXT:    v_perm_b32 v5, v5, v3, 0x7060302
@@ -14588,7 +14586,6 @@ define <2 x bfloat> @flat_agent_atomic_fmin_ret_v2bf16__offset12b_pos__amdgpu_no
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s1
 ; GFX12-NEXT:    v_mov_b32_e32 v0, v3
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: flat_agent_atomic_fmin_ret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
@@ -14918,9 +14915,10 @@ define <2 x bfloat> @flat_agent_atomic_fmin_ret_v2bf16__offset12b_neg__amdgpu_no
 ; GFX12-NEXT:    v_bfe_u32 v7, v3, 16, 1
 ; GFX12-NEXT:    v_or_b32_e32 v9, 0x400000, v3
 ; GFX12-NEXT:    v_cmp_u_f32_e64 s0, v3, v3
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_cndmask_b32_e32 v5, v8, v10, vcc_lo
 ; GFX12-NEXT:    v_add3_u32 v7, v7, v3, 0x7fff
+; GFX12-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_cndmask_b32_e64 v3, v7, v9, s0
 ; GFX12-NEXT:    v_perm_b32 v5, v5, v3, 0x7060302
@@ -14937,7 +14935,6 @@ define <2 x bfloat> @flat_agent_atomic_fmin_ret_v2bf16__offset12b_neg__amdgpu_no
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s1
 ; GFX12-NEXT:    v_mov_b32_e32 v0, v3
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: flat_agent_atomic_fmin_ret_v2bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
@@ -15284,8 +15281,10 @@ define void @flat_agent_atomic_fmin_noret_v2bf16__amdgpu_no_fine_grained_memory(
 ; GFX12-NEXT:    v_add3_u32 v7, v7, v2, 0x7fff
 ; GFX12-NEXT:    v_add3_u32 v8, v8, v6, 0x7fff
 ; GFX12-NEXT:    v_cmp_u_f32_e64 s0, v2, v2
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-NEXT:    s_wait_alu 0xfffd
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX12-NEXT:    v_cndmask_b32_e32 v6, v8, v10, vcc_lo
+; GFX12-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-NEXT:    v_cndmask_b32_e64 v2, v7, v9, s0
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-NEXT:    v_perm_b32 v2, v6, v2, 0x7060302
@@ -15302,7 +15301,6 @@ define void @flat_agent_atomic_fmin_noret_v2bf16__amdgpu_no_fine_grained_memory(
 ; GFX12-NEXT:    s_cbranch_execnz .LBB57_1
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s1
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: flat_agent_atomic_fmin_noret_v2bf16__amdgpu_no_fine_grained_memory:
@@ -15619,8 +15617,10 @@ define void @flat_agent_atomic_fmin_noret_v2bf16__offset12b_pos__amdgpu_no_fine_
 ; GFX12-NEXT:    v_add3_u32 v7, v7, v2, 0x7fff
 ; GFX12-NEXT:    v_add3_u32 v8, v8, v6, 0x7fff
 ; GFX12-NEXT:    v_cmp_u_f32_e64 s0, v2, v2
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-NEXT:    s_wait_alu 0xfffd
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX12-NEXT:    v_cndmask_b32_e32 v6, v8, v10, vcc_lo
+; GFX12-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-NEXT:    v_cndmask_b32_e64 v2, v7, v9, s0
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-NEXT:    v_perm_b32 v2, v6, v2, 0x7060302
@@ -15637,7 +15637,6 @@ define void @flat_agent_atomic_fmin_noret_v2bf16__offset12b_pos__amdgpu_no_fine_
 ; GFX12-NEXT:    s_cbranch_execnz .LBB58_1
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s1
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: flat_agent_atomic_fmin_noret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
@@ -15961,8 +15960,10 @@ define void @flat_agent_atomic_fmin_noret_v2bf16__offset12b_neg__amdgpu_no_fine_
 ; GFX12-NEXT:    v_add3_u32 v7, v7, v2, 0x7fff
 ; GFX12-NEXT:    v_add3_u32 v8, v8, v6, 0x7fff
 ; GFX12-NEXT:    v_cmp_u_f32_e64 s0, v2, v2
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-NEXT:    s_wait_alu 0xfffd
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX12-NEXT:    v_cndmask_b32_e32 v6, v8, v10, vcc_lo
+; GFX12-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-NEXT:    v_cndmask_b32_e64 v2, v7, v9, s0
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-NEXT:    v_perm_b32 v2, v6, v2, 0x7060302
@@ -15979,7 +15980,6 @@ define void @flat_agent_atomic_fmin_noret_v2bf16__offset12b_neg__amdgpu_no_fine_
 ; GFX12-NEXT:    s_cbranch_execnz .LBB59_1
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s1
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: flat_agent_atomic_fmin_noret_v2bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
@@ -16322,9 +16322,10 @@ define <2 x bfloat> @flat_system_atomic_fmin_ret_v2bf16__offset12b_pos__amdgpu_n
 ; GFX12-NEXT:    v_bfe_u32 v7, v3, 16, 1
 ; GFX12-NEXT:    v_or_b32_e32 v9, 0x400000, v3
 ; GFX12-NEXT:    v_cmp_u_f32_e64 s0, v3, v3
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_cndmask_b32_e32 v5, v8, v10, vcc_lo
 ; GFX12-NEXT:    v_add3_u32 v7, v7, v3, 0x7fff
+; GFX12-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_cndmask_b32_e64 v3, v7, v9, s0
 ; GFX12-NEXT:    v_perm_b32 v5, v5, v3, 0x7060302
@@ -16342,7 +16343,6 @@ define <2 x bfloat> @flat_system_atomic_fmin_ret_v2bf16__offset12b_pos__amdgpu_n
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s1
 ; GFX12-NEXT:    v_mov_b32_e32 v0, v3
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: flat_system_atomic_fmin_ret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
@@ -16673,8 +16673,10 @@ define void @flat_system_atomic_fmin_noret_v2bf16__offset12b_pos__amdgpu_no_fine
 ; GFX12-NEXT:    v_add3_u32 v7, v7, v2, 0x7fff
 ; GFX12-NEXT:    v_add3_u32 v8, v8, v6, 0x7fff
 ; GFX12-NEXT:    v_cmp_u_f32_e64 s0, v2, v2
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-NEXT:    s_wait_alu 0xfffd
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX12-NEXT:    v_cndmask_b32_e32 v6, v8, v10, vcc_lo
+; GFX12-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-NEXT:    v_cndmask_b32_e64 v2, v7, v9, s0
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-NEXT:    v_perm_b32 v2, v6, v2, 0x7060302
@@ -16692,7 +16694,6 @@ define void @flat_system_atomic_fmin_noret_v2bf16__offset12b_pos__amdgpu_no_fine
 ; GFX12-NEXT:    s_cbranch_execnz .LBB61_1
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s1
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: flat_system_atomic_fmin_noret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
diff --git a/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fsub.ll b/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fsub.ll
index 9c2a76380d83d..cd1a161346667 100644
--- a/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fsub.ll
+++ b/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fsub.ll
@@ -41,7 +41,6 @@ define float @flat_agent_atomic_fsub_ret_f32(ptr %ptr, float %val) #0 {
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX12-NEXT:    v_mov_b32_e32 v0, v3
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: flat_agent_atomic_fsub_ret_f32:
@@ -237,7 +236,6 @@ define float @flat_agent_atomic_fsub_ret_f32__offset12b_pos(ptr %ptr, float %val
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX12-NEXT:    v_mov_b32_e32 v0, v3
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: flat_agent_atomic_fsub_ret_f32__offset12b_pos:
@@ -437,7 +435,6 @@ define float @flat_agent_atomic_fsub_ret_f32__offset12b_neg(ptr %ptr, float %val
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX12-NEXT:    v_mov_b32_e32 v0, v3
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: flat_agent_atomic_fsub_ret_f32__offset12b_neg:
@@ -653,7 +650,6 @@ define void @flat_agent_atomic_fsub_noret_f32(ptr %ptr, float %val) #0 {
 ; GFX12-NEXT:    s_cbranch_execnz .LBB3_1
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: flat_agent_atomic_fsub_noret_f32:
@@ -839,7 +835,6 @@ define void @flat_agent_atomic_fsub_noret_f32__offset12b_pos(ptr %ptr, float %va
 ; GFX12-NEXT:    s_cbranch_execnz .LBB4_1
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: flat_agent_atomic_fsub_noret_f32__offset12b_pos:
@@ -1032,7 +1027,6 @@ define void @flat_agent_atomic_fsub_noret_f32__offset12b_neg(ptr %ptr, float %va
 ; GFX12-NEXT:    s_cbranch_execnz .LBB5_1
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: flat_agent_atomic_fsub_noret_f32__offset12b_neg:
@@ -1246,7 +1240,6 @@ define float @flat_system_atomic_fsub_ret_f32__offset12b_pos(ptr %ptr, float %va
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX12-NEXT:    v_mov_b32_e32 v0, v3
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: flat_system_atomic_fsub_ret_f32__offset12b_pos:
@@ -1447,7 +1440,6 @@ define void @flat_system_atomic_fsub_noret_f32__offset12b_pos(ptr %ptr, float %v
 ; GFX12-NEXT:    s_cbranch_execnz .LBB7_1
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: flat_system_atomic_fsub_noret_f32__offset12b_pos:
@@ -1648,7 +1640,6 @@ define float @flat_agent_atomic_fsub_ret_f32__ftz(ptr %ptr, float %val) #1 {
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX12-NEXT:    v_mov_b32_e32 v0, v3
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: flat_agent_atomic_fsub_ret_f32__ftz:
@@ -1844,7 +1835,6 @@ define float @flat_agent_atomic_fsub_ret_f32__offset12b_pos__ftz(ptr %ptr, float
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX12-NEXT:    v_mov_b32_e32 v0, v3
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: flat_agent_atomic_fsub_ret_f32__offset12b_pos__ftz:
@@ -2044,7 +2034,6 @@ define float @flat_agent_atomic_fsub_ret_f32__offset12b_neg__ftz(ptr %ptr, float
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX12-NEXT:    v_mov_b32_e32 v0, v3
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: flat_agent_atomic_fsub_ret_f32__offset12b_neg__ftz:
@@ -2260,7 +2249,6 @@ define void @flat_agent_atomic_fsub_noret_f32__ftz(ptr %ptr, float %val) #1 {
 ; GFX12-NEXT:    s_cbranch_execnz .LBB11_1
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: flat_agent_atomic_fsub_noret_f32__ftz:
@@ -2446,7 +2434,6 @@ define void @flat_agent_atomic_fsub_noret_f32__offset12b_pos__ftz(ptr %ptr, floa
 ; GFX12-NEXT:    s_cbranch_execnz .LBB12_1
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: flat_agent_atomic_fsub_noret_f32__offset12b_pos__ftz:
@@ -2639,7 +2626,6 @@ define void @flat_agent_atomic_fsub_noret_f32__offset12b_neg__ftz(ptr %ptr, floa
 ; GFX12-NEXT:    s_cbranch_execnz .LBB13_1
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: flat_agent_atomic_fsub_noret_f32__offset12b_neg__ftz:
@@ -2853,7 +2839,6 @@ define float @flat_system_atomic_fsub_ret_f32__offset12b_pos__ftz(ptr %ptr, floa
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX12-NEXT:    v_mov_b32_e32 v0, v3
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: flat_system_atomic_fsub_ret_f32__offset12b_pos__ftz:
@@ -3054,7 +3039,6 @@ define void @flat_system_atomic_fsub_noret_f32__offset12b_pos__ftz(ptr %ptr, flo
 ; GFX12-NEXT:    s_cbranch_execnz .LBB15_1
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: flat_system_atomic_fsub_noret_f32__offset12b_pos__ftz:
@@ -3270,6 +3254,7 @@ define double @flat_agent_atomic_fsub_ret_f64(ptr %ptr, double %val) #0 {
 ; GFX12-NEXT:    s_cbranch_execz .LBB16_6
 ; GFX12-NEXT:  ; %bb.5: ; %atomicrmw.private
 ; GFX12-NEXT:    v_cmp_ne_u64_e32 vcc_lo, 0, v[0:1]
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_cndmask_b32_e32 v6, -1, v0, vcc_lo
 ; GFX12-NEXT:    scratch_load_b64 v[4:5], v6, off
 ; GFX12-NEXT:    s_wait_loadcnt 0x0
@@ -3279,7 +3264,6 @@ define double @flat_agent_atomic_fsub_ret_f64(ptr %ptr, double %val) #0 {
 ; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX12-NEXT:    v_dual_mov_b32 v0, v4 :: v_dual_mov_b32 v1, v5
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: flat_agent_atomic_fsub_ret_f64:
@@ -3642,6 +3626,7 @@ define double @flat_agent_atomic_fsub_ret_f64__offset12b_pos(ptr %ptr, double %v
 ; GFX12-NEXT:    s_wait_bvhcnt 0x0
 ; GFX12-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-NEXT:    v_add_co_u32 v4, vcc_lo, 0x7f8, v0
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_add_co_ci_u32_e32 v5, vcc_lo, 0, v1, vcc_lo
 ; GFX12-NEXT:    s_mov_b64 s[0:1], src_private_base
 ; GFX12-NEXT:    s_mov_b32 s0, exec_lo
@@ -3658,7 +3643,7 @@ define double @flat_agent_atomic_fsub_ret_f64__offset12b_pos(ptr %ptr, double %v
 ; GFX12-NEXT:  .LBB17_2: ; %atomicrmw.phi
 ; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT:    s_wait_alu 0xfffe
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ; GFX12-NEXT:  .LBB17_3: ; %atomicrmw.global
 ; GFX12-NEXT:    flat_load_b64 v[0:1], v[4:5]
@@ -3687,6 +3672,7 @@ define double @flat_agent_atomic_fsub_ret_f64__offset12b_pos(ptr %ptr, double %v
 ; GFX12-NEXT:    s_cbranch_execz .LBB17_2
 ; GFX12-NEXT:  .LBB17_6: ; %atomicrmw.private
 ; GFX12-NEXT:    v_cmp_ne_u64_e32 vcc_lo, 0, v[4:5]
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_cndmask_b32_e32 v4, -1, v4, vcc_lo
 ; GFX12-NEXT:    scratch_load_b64 v[0:1], v4, off
 ; GFX12-NEXT:    s_wait_loadcnt 0x0
@@ -3694,7 +3680,6 @@ define double @flat_agent_atomic_fsub_ret_f64__offset12b_pos(ptr %ptr, double %v
 ; GFX12-NEXT:    scratch_store_b64 v4, v[2:3], off
 ; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: flat_agent_atomic_fsub_ret_f64__offset12b_pos:
@@ -4088,6 +4073,7 @@ define double @flat_agent_atomic_fsub_ret_f64__offset12b_neg(ptr %ptr, double %v
 ; GFX12-NEXT:    s_wait_bvhcnt 0x0
 ; GFX12-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-NEXT:    v_add_co_u32 v4, vcc_lo, 0xfffff800, v0
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_add_co_ci_u32_e32 v5, vcc_lo, -1, v1, vcc_lo
 ; GFX12-NEXT:    s_mov_b64 s[0:1], src_private_base
 ; GFX12-NEXT:    s_mov_b32 s0, exec_lo
@@ -4104,7 +4090,7 @@ define double @flat_agent_atomic_fsub_ret_f64__offset12b_neg(ptr %ptr, double %v
 ; GFX12-NEXT:  .LBB18_2: ; %atomicrmw.phi
 ; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT:    s_wait_alu 0xfffe
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ; GFX12-NEXT:  .LBB18_3: ; %atomicrmw.global
 ; GFX12-NEXT:    flat_load_b64 v[0:1], v[4:5]
@@ -4133,6 +4119,7 @@ define double @flat_agent_atomic_fsub_ret_f64__offset12b_neg(ptr %ptr, double %v
 ; GFX12-NEXT:    s_cbranch_execz .LBB18_2
 ; GFX12-NEXT:  .LBB18_6: ; %atomicrmw.private
 ; GFX12-NEXT:    v_cmp_ne_u64_e32 vcc_lo, 0, v[4:5]
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_cndmask_b32_e32 v4, -1, v4, vcc_lo
 ; GFX12-NEXT:    scratch_load_b64 v[0:1], v4, off
 ; GFX12-NEXT:    s_wait_loadcnt 0x0
@@ -4140,7 +4127,6 @@ define double @flat_agent_atomic_fsub_ret_f64__offset12b_neg(ptr %ptr, double %v
 ; GFX12-NEXT:    scratch_store_b64 v4, v[2:3], off
 ; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: flat_agent_atomic_fsub_ret_f64__offset12b_neg:
@@ -4547,7 +4533,6 @@ define void @flat_agent_atomic_fsub_noret_f64(ptr %ptr, double %val) #0 {
 ; GFX12-NEXT:  .LBB19_2: ; %atomicrmw.phi
 ; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ; GFX12-NEXT:  .LBB19_3: ; %atomicrmw.global
 ; GFX12-NEXT:    flat_load_b64 v[6:7], v[0:1]
@@ -4575,6 +4560,7 @@ define void @flat_agent_atomic_fsub_noret_f64(ptr %ptr, double %val) #0 {
 ; GFX12-NEXT:    s_cbranch_execz .LBB19_2
 ; GFX12-NEXT:  .LBB19_6: ; %atomicrmw.private
 ; GFX12-NEXT:    v_cmp_ne_u64_e32 vcc_lo, 0, v[0:1]
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_cndmask_b32_e32 v4, -1, v0, vcc_lo
 ; GFX12-NEXT:    scratch_load_b64 v[0:1], v4, off
 ; GFX12-NEXT:    s_wait_loadcnt 0x0
@@ -4582,7 +4568,6 @@ define void @flat_agent_atomic_fsub_noret_f64(ptr %ptr, double %val) #0 {
 ; GFX12-NEXT:    scratch_store_b64 v4, v[0:1], off
 ; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: flat_agent_atomic_fsub_noret_f64:
@@ -4952,6 +4937,7 @@ define void @flat_agent_atomic_fsub_noret_f64__offset12b_pos(ptr %ptr, double %v
 ; GFX12-NEXT:    s_wait_bvhcnt 0x0
 ; GFX12-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-NEXT:    v_add_co_u32 v0, vcc_lo, 0x7f8, v0
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
 ; GFX12-NEXT:    s_mov_b64 s[0:1], src_private_base
 ; GFX12-NEXT:    s_mov_b32 s0, exec_lo
@@ -4967,7 +4953,7 @@ define void @flat_agent_atomic_fsub_noret_f64__offset12b_pos(ptr %ptr, double %v
 ; GFX12-NEXT:  .LBB20_2: ; %atomicrmw.phi
 ; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT:    s_wait_alu 0xfffe
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ; GFX12-NEXT:  .LBB20_3: ; %atomicrmw.global
 ; GFX12-NEXT:    flat_load_b64 v[6:7], v[0:1]
@@ -4995,6 +4981,7 @@ define void @flat_agent_atomic_fsub_noret_f64__offset12b_pos(ptr %ptr, double %v
 ; GFX12-NEXT:    s_cbranch_execz .LBB20_2
 ; GFX12-NEXT:  .LBB20_6: ; %atomicrmw.private
 ; GFX12-NEXT:    v_cmp_ne_u64_e32 vcc_lo, 0, v[0:1]
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_cndmask_b32_e32 v4, -1, v0, vcc_lo
 ; GFX12-NEXT:    scratch_load_b64 v[0:1], v4, off
 ; GFX12-NEXT:    s_wait_loadcnt 0x0
@@ -5002,7 +4989,6 @@ define void @flat_agent_atomic_fsub_noret_f64__offset12b_pos(ptr %ptr, double %v
 ; GFX12-NEXT:    scratch_store_b64 v4, v[0:1], off
 ; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: flat_agent_atomic_fsub_noret_f64__offset12b_pos:
@@ -5388,6 +5374,7 @@ define void @flat_agent_atomic_fsub_noret_f64__offset12b_neg(ptr %ptr, double %v
 ; GFX12-NEXT:    s_wait_bvhcnt 0x0
 ; GFX12-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-NEXT:    v_add_co_u32 v0, vcc_lo, 0xfffff800, v0
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, -1, v1, vcc_lo
 ; GFX12-NEXT:    s_mov_b64 s[0:1], src_private_base
 ; GFX12-NEXT:    s_mov_b32 s0, exec_lo
@@ -5403,7 +5390,7 @@ define void @flat_agent_atomic_fsub_noret_f64__offset12b_neg(ptr %ptr, double %v
 ; GFX12-NEXT:  .LBB21_2: ; %atomicrmw.phi
 ; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT:    s_wait_alu 0xfffe
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ; GFX12-NEXT:  .LBB21_3: ; %atomicrmw.global
 ; GFX12-NEXT:    flat_load_b64 v[6:7], v[0:1]
@@ -5431,6 +5418,7 @@ define void @flat_agent_atomic_fsub_noret_f64__offset12b_neg(ptr %ptr, double %v
 ; GFX12-NEXT:    s_cbranch_execz .LBB21_2
 ; GFX12-NEXT:  .LBB21_6: ; %atomicrmw.private
 ; GFX12-NEXT:    v_cmp_ne_u64_e32 vcc_lo, 0, v[0:1]
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_cndmask_b32_e32 v4, -1, v0, vcc_lo
 ; GFX12-NEXT:    scratch_load_b64 v[0:1], v4, off
 ; GFX12-NEXT:    s_wait_loadcnt 0x0
@@ -5438,7 +5426,6 @@ define void @flat_agent_atomic_fsub_noret_f64__offset12b_neg(ptr %ptr, double %v
 ; GFX12-NEXT:    scratch_store_b64 v4, v[0:1], off
 ; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: flat_agent_atomic_fsub_noret_f64__offset12b_neg:
@@ -5863,7 +5850,6 @@ define half @flat_agent_atomic_fsub_ret_f16(ptr %ptr, half %val) #0 {
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX12-NEXT:    v_lshrrev_b32_e32 v0, v3, v5
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: flat_agent_atomic_fsub_ret_f16:
@@ -6118,6 +6104,7 @@ define half @flat_agent_atomic_fsub_ret_f16__offset12b_pos(ptr %ptr, half %val)
 ; GFX12-NEXT:    s_wait_bvhcnt 0x0
 ; GFX12-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-NEXT:    v_add_co_u32 v3, vcc_lo, 0x7fe, v0
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
 ; GFX12-NEXT:    s_mov_b32 s0, 0
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_1)
@@ -6153,7 +6140,6 @@ define half @flat_agent_atomic_fsub_ret_f16__offset12b_pos(ptr %ptr, half %val)
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX12-NEXT:    v_lshrrev_b32_e32 v0, v3, v5
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: flat_agent_atomic_fsub_ret_f16__offset12b_pos:
@@ -6417,6 +6403,7 @@ define half @flat_agent_atomic_fsub_ret_f16__offset12b_neg(ptr %ptr, half %val)
 ; GFX12-NEXT:    s_wait_bvhcnt 0x0
 ; GFX12-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-NEXT:    v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, -1, v1, vcc_lo
 ; GFX12-NEXT:    s_mov_b32 s0, 0
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_1)
@@ -6452,7 +6439,6 @@ define half @flat_agent_atomic_fsub_ret_f16__offset12b_neg(ptr %ptr, half %val)
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX12-NEXT:    v_lshrrev_b32_e32 v0, v3, v5
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: flat_agent_atomic_fsub_ret_f16__offset12b_neg:
@@ -6749,7 +6735,6 @@ define void @flat_agent_atomic_fsub_noret_f16(ptr %ptr, half %val) #0 {
 ; GFX12-NEXT:    s_cbranch_execnz .LBB25_1
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: flat_agent_atomic_fsub_noret_f16:
@@ -6995,6 +6980,7 @@ define void @flat_agent_atomic_fsub_noret_f16__offset12b_pos(ptr %ptr, half %val
 ; GFX12-NEXT:    s_wait_bvhcnt 0x0
 ; GFX12-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-NEXT:    v_add_co_u32 v3, vcc_lo, 0x7fe, v0
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
 ; GFX12-NEXT:    s_mov_b32 s0, 0
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_1)
@@ -7028,7 +7014,6 @@ define void @flat_agent_atomic_fsub_noret_f16__offset12b_pos(ptr %ptr, half %val
 ; GFX12-NEXT:    s_cbranch_execnz .LBB26_1
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: flat_agent_atomic_fsub_noret_f16__offset12b_pos:
@@ -7283,6 +7268,7 @@ define void @flat_agent_atomic_fsub_noret_f16__offset12b_neg(ptr %ptr, half %val
 ; GFX12-NEXT:    s_wait_bvhcnt 0x0
 ; GFX12-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-NEXT:    v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, -1, v1, vcc_lo
 ; GFX12-NEXT:    s_mov_b32 s0, 0
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_1)
@@ -7316,7 +7302,6 @@ define void @flat_agent_atomic_fsub_noret_f16__offset12b_neg(ptr %ptr, half %val
 ; GFX12-NEXT:    s_cbranch_execnz .LBB27_1
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: flat_agent_atomic_fsub_noret_f16__offset12b_neg:
@@ -7595,7 +7580,6 @@ define half @flat_agent_atomic_fsub_ret_f16__offset12b_pos__align4(ptr %ptr, hal
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX12-NEXT:    v_mov_b32_e32 v0, v3
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: flat_agent_atomic_fsub_ret_f16__offset12b_pos__align4:
@@ -7816,7 +7800,6 @@ define void @flat_agent_atomic_fsub_noret_f16__offset12b__align4_pos(ptr %ptr, h
 ; GFX12-NEXT:    s_cbranch_execnz .LBB29_1
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: flat_agent_atomic_fsub_noret_f16__offset12b__align4_pos:
@@ -8010,6 +7993,7 @@ define half @flat_system_atomic_fsub_ret_f16__offset12b_pos(ptr %ptr, half %val)
 ; GFX12-NEXT:    s_wait_bvhcnt 0x0
 ; GFX12-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-NEXT:    v_add_co_u32 v3, vcc_lo, 0x7fe, v0
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
 ; GFX12-NEXT:    s_mov_b32 s0, 0
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_1)
@@ -8046,7 +8030,6 @@ define half @flat_system_atomic_fsub_ret_f16__offset12b_pos(ptr %ptr, half %val)
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX12-NEXT:    v_lshrrev_b32_e32 v0, v3, v5
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: flat_system_atomic_fsub_ret_f16__offset12b_pos:
@@ -8312,6 +8295,7 @@ define void @flat_system_atomic_fsub_noret_f16__offset12b_pos(ptr %ptr, half %va
 ; GFX12-NEXT:    s_wait_bvhcnt 0x0
 ; GFX12-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-NEXT:    v_add_co_u32 v3, vcc_lo, 0x7fe, v0
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
 ; GFX12-NEXT:    s_mov_b32 s0, 0
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_1)
@@ -8346,7 +8330,6 @@ define void @flat_system_atomic_fsub_noret_f16__offset12b_pos(ptr %ptr, half %va
 ; GFX12-NEXT:    s_cbranch_execnz .LBB31_1
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: flat_system_atomic_fsub_noret_f16__offset12b_pos:
@@ -8628,8 +8611,9 @@ define bfloat @flat_agent_atomic_fsub_ret_bf16(ptr %ptr, bfloat %val) #0 {
 ; GFX12-NEXT:    v_bfe_u32 v7, v5, 16, 1
 ; GFX12-NEXT:    v_or_b32_e32 v8, 0x400000, v5
 ; GFX12-NEXT:    v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_cndmask_b32_e32 v5, v7, v8, vcc_lo
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_lshrrev_b32_e32 v5, 16, v5
@@ -8649,7 +8633,6 @@ define bfloat @flat_agent_atomic_fsub_ret_bf16(ptr %ptr, bfloat %val) #0 {
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX12-NEXT:    v_lshrrev_b32_e32 v0, v3, v5
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: flat_agent_atomic_fsub_ret_bf16:
@@ -8949,6 +8932,7 @@ define bfloat @flat_agent_atomic_fsub_ret_bf16__offset12b_pos(ptr %ptr, bfloat %
 ; GFX12-NEXT:    s_wait_bvhcnt 0x0
 ; GFX12-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-NEXT:    v_add_co_u32 v3, vcc_lo, 0x7fe, v0
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
 ; GFX12-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
@@ -8972,8 +8956,9 @@ define bfloat @flat_agent_atomic_fsub_ret_bf16__offset12b_pos(ptr %ptr, bfloat %
 ; GFX12-NEXT:    v_bfe_u32 v7, v5, 16, 1
 ; GFX12-NEXT:    v_or_b32_e32 v8, 0x400000, v5
 ; GFX12-NEXT:    v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_cndmask_b32_e32 v5, v7, v8, vcc_lo
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_lshrrev_b32_e32 v5, 16, v5
@@ -8993,7 +8978,6 @@ define bfloat @flat_agent_atomic_fsub_ret_bf16__offset12b_pos(ptr %ptr, bfloat %
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX12-NEXT:    v_lshrrev_b32_e32 v0, v3, v5
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: flat_agent_atomic_fsub_ret_bf16__offset12b_pos:
@@ -9303,6 +9287,7 @@ define bfloat @flat_agent_atomic_fsub_ret_bf16__offset12b_neg(ptr %ptr, bfloat %
 ; GFX12-NEXT:    s_wait_bvhcnt 0x0
 ; GFX12-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-NEXT:    v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, -1, v1, vcc_lo
 ; GFX12-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
@@ -9326,8 +9311,9 @@ define bfloat @flat_agent_atomic_fsub_ret_bf16__offset12b_neg(ptr %ptr, bfloat %
 ; GFX12-NEXT:    v_bfe_u32 v7, v5, 16, 1
 ; GFX12-NEXT:    v_or_b32_e32 v8, 0x400000, v5
 ; GFX12-NEXT:    v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_cndmask_b32_e32 v5, v7, v8, vcc_lo
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_lshrrev_b32_e32 v5, 16, v5
@@ -9347,7 +9333,6 @@ define bfloat @flat_agent_atomic_fsub_ret_bf16__offset12b_neg(ptr %ptr, bfloat %
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX12-NEXT:    v_lshrrev_b32_e32 v0, v3, v5
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: flat_agent_atomic_fsub_ret_bf16__offset12b_neg:
@@ -9679,6 +9664,7 @@ define void @flat_agent_atomic_fsub_noret_bf16(ptr %ptr, bfloat %val) #0 {
 ; GFX12-NEXT:    v_or_b32_e32 v8, 0x400000, v3
 ; GFX12-NEXT:    v_cmp_u_f32_e32 vcc_lo, v3, v3
 ; GFX12-NEXT:    v_add3_u32 v7, v7, v3, 0x7fff
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_cndmask_b32_e32 v3, v7, v8, vcc_lo
 ; GFX12-NEXT:    v_lshrrev_b32_e32 v3, 16, v3
@@ -9698,7 +9684,6 @@ define void @flat_agent_atomic_fsub_noret_bf16(ptr %ptr, bfloat %val) #0 {
 ; GFX12-NEXT:    s_cbranch_execnz .LBB35_1
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: flat_agent_atomic_fsub_noret_bf16:
@@ -9989,6 +9974,7 @@ define void @flat_agent_atomic_fsub_noret_bf16__offset12b_pos(ptr %ptr, bfloat %
 ; GFX12-NEXT:    s_wait_bvhcnt 0x0
 ; GFX12-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-NEXT:    v_add_co_u32 v4, vcc_lo, 0x7fe, v0
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
 ; GFX12-NEXT:    v_lshlrev_b32_e32 v6, 16, v2
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
@@ -10012,6 +9998,7 @@ define void @flat_agent_atomic_fsub_noret_bf16__offset12b_pos(ptr %ptr, bfloat %
 ; GFX12-NEXT:    v_or_b32_e32 v8, 0x400000, v2
 ; GFX12-NEXT:    v_cmp_u_f32_e32 vcc_lo, v2, v2
 ; GFX12-NEXT:    v_add3_u32 v7, v7, v2, 0x7fff
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_cndmask_b32_e32 v2, v7, v8, vcc_lo
 ; GFX12-NEXT:    v_lshrrev_b32_e32 v2, 16, v2
@@ -10031,7 +10018,6 @@ define void @flat_agent_atomic_fsub_noret_bf16__offset12b_pos(ptr %ptr, bfloat %
 ; GFX12-NEXT:    s_cbranch_execnz .LBB36_1
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: flat_agent_atomic_fsub_noret_bf16__offset12b_pos:
@@ -10332,6 +10318,7 @@ define void @flat_agent_atomic_fsub_noret_bf16__offset12b_neg(ptr %ptr, bfloat %
 ; GFX12-NEXT:    s_wait_bvhcnt 0x0
 ; GFX12-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-NEXT:    v_add_co_u32 v4, vcc_lo, 0xfffff800, v0
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, -1, v1, vcc_lo
 ; GFX12-NEXT:    v_lshlrev_b32_e32 v6, 16, v2
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
@@ -10355,6 +10342,7 @@ define void @flat_agent_atomic_fsub_noret_bf16__offset12b_neg(ptr %ptr, bfloat %
 ; GFX12-NEXT:    v_or_b32_e32 v8, 0x400000, v2
 ; GFX12-NEXT:    v_cmp_u_f32_e32 vcc_lo, v2, v2
 ; GFX12-NEXT:    v_add3_u32 v7, v7, v2, 0x7fff
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_cndmask_b32_e32 v2, v7, v8, vcc_lo
 ; GFX12-NEXT:    v_lshrrev_b32_e32 v2, 16, v2
@@ -10374,7 +10362,6 @@ define void @flat_agent_atomic_fsub_noret_bf16__offset12b_neg(ptr %ptr, bfloat %
 ; GFX12-NEXT:    s_cbranch_execnz .LBB37_1
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: flat_agent_atomic_fsub_noret_bf16__offset12b_neg:
@@ -10690,6 +10677,7 @@ define bfloat @flat_agent_atomic_fsub_ret_bf16__offset12b_pos__align4(ptr %ptr,
 ; GFX12-NEXT:    v_or_b32_e32 v6, 0x400000, v3
 ; GFX12-NEXT:    v_cmp_u_f32_e32 vcc_lo, v3, v3
 ; GFX12-NEXT:    v_add3_u32 v5, v5, v3, 0x7fff
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_cndmask_b32_e32 v3, v5, v6, vcc_lo
 ; GFX12-NEXT:    v_lshrrev_b32_e32 v3, 16, v3
@@ -10708,7 +10696,6 @@ define bfloat @flat_agent_atomic_fsub_ret_bf16__offset12b_pos__align4(ptr %ptr,
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX12-NEXT:    v_mov_b32_e32 v0, v3
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: flat_agent_atomic_fsub_ret_bf16__offset12b_pos__align4:
@@ -10972,8 +10959,9 @@ define void @flat_agent_atomic_fsub_noret_bf16__offset12b__align4_pos(ptr %ptr,
 ; GFX12-NEXT:    v_bfe_u32 v5, v2, 16, 1
 ; GFX12-NEXT:    v_or_b32_e32 v6, 0x400000, v2
 ; GFX12-NEXT:    v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_add3_u32 v5, v5, v2, 0x7fff
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_cndmask_b32_e32 v2, v5, v6, vcc_lo
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_lshrrev_b32_e32 v2, 16, v2
@@ -10991,7 +10979,6 @@ define void @flat_agent_atomic_fsub_noret_bf16__offset12b__align4_pos(ptr %ptr,
 ; GFX12-NEXT:    s_cbranch_execnz .LBB39_1
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: flat_agent_atomic_fsub_noret_bf16__offset12b__align4_pos:
@@ -11238,6 +11225,7 @@ define bfloat @flat_system_atomic_fsub_ret_bf16__offset12b_pos(ptr %ptr, bfloat
 ; GFX12-NEXT:    s_wait_bvhcnt 0x0
 ; GFX12-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-NEXT:    v_add_co_u32 v3, vcc_lo, 0x7fe, v0
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
 ; GFX12-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
@@ -11261,8 +11249,9 @@ define bfloat @flat_system_atomic_fsub_ret_bf16__offset12b_pos(ptr %ptr, bfloat
 ; GFX12-NEXT:    v_bfe_u32 v7, v5, 16, 1
 ; GFX12-NEXT:    v_or_b32_e32 v8, 0x400000, v5
 ; GFX12-NEXT:    v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_cndmask_b32_e32 v5, v7, v8, vcc_lo
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_lshrrev_b32_e32 v5, 16, v5
@@ -11283,7 +11272,6 @@ define bfloat @flat_system_atomic_fsub_ret_bf16__offset12b_pos(ptr %ptr, bfloat
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX12-NEXT:    v_lshrrev_b32_e32 v0, v3, v5
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: flat_system_atomic_fsub_ret_bf16__offset12b_pos:
@@ -11595,6 +11583,7 @@ define void @flat_system_atomic_fsub_noret_bf16__offset12b_pos(ptr %ptr, bfloat
 ; GFX12-NEXT:    s_wait_bvhcnt 0x0
 ; GFX12-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-NEXT:    v_add_co_u32 v4, vcc_lo, 0x7fe, v0
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
 ; GFX12-NEXT:    v_lshlrev_b32_e32 v6, 16, v2
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
@@ -11618,6 +11607,7 @@ define void @flat_system_atomic_fsub_noret_bf16__offset12b_pos(ptr %ptr, bfloat
 ; GFX12-NEXT:    v_or_b32_e32 v8, 0x400000, v2
 ; GFX12-NEXT:    v_cmp_u_f32_e32 vcc_lo, v2, v2
 ; GFX12-NEXT:    v_add3_u32 v7, v7, v2, 0x7fff
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_cndmask_b32_e32 v2, v7, v8, vcc_lo
 ; GFX12-NEXT:    v_lshrrev_b32_e32 v2, 16, v2
@@ -11638,7 +11628,6 @@ define void @flat_system_atomic_fsub_noret_bf16__offset12b_pos(ptr %ptr, bfloat
 ; GFX12-NEXT:    s_cbranch_execnz .LBB41_1
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: flat_system_atomic_fsub_noret_bf16__offset12b_pos:
@@ -11965,7 +11954,6 @@ define <2 x half> @flat_agent_atomic_fsub_ret_v2f16(ptr %ptr, <2 x half> %val) #
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX12-NEXT:    v_mov_b32_e32 v0, v3
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: flat_agent_atomic_fsub_ret_v2f16:
@@ -12186,7 +12174,6 @@ define <2 x half> @flat_agent_atomic_fsub_ret_v2f16__offset12b_pos(ptr %ptr, <2
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX12-NEXT:    v_mov_b32_e32 v0, v3
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: flat_agent_atomic_fsub_ret_v2f16__offset12b_pos:
@@ -12410,7 +12397,6 @@ define <2 x half> @flat_agent_atomic_fsub_ret_v2f16__offset12b_neg(ptr %ptr, <2
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX12-NEXT:    v_mov_b32_e32 v0, v3
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: flat_agent_atomic_fsub_ret_v2f16__offset12b_neg:
@@ -12650,7 +12636,6 @@ define void @flat_agent_atomic_fsub_noret_v2f16(ptr %ptr, <2 x half> %val) #0 {
 ; GFX12-NEXT:    s_cbranch_execnz .LBB45_1
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: flat_agent_atomic_fsub_noret_v2f16:
@@ -12860,7 +12845,6 @@ define void @flat_agent_atomic_fsub_noret_v2f16__offset12b_pos(ptr %ptr, <2 x ha
 ; GFX12-NEXT:    s_cbranch_execnz .LBB46_1
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: flat_agent_atomic_fsub_noret_v2f16__offset12b_pos:
@@ -13077,7 +13061,6 @@ define void @flat_agent_atomic_fsub_noret_v2f16__offset12b_neg(ptr %ptr, <2 x ha
 ; GFX12-NEXT:    s_cbranch_execnz .LBB47_1
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: flat_agent_atomic_fsub_noret_v2f16__offset12b_neg:
@@ -13315,7 +13298,6 @@ define <2 x half> @flat_system_atomic_fsub_ret_v2f16__offset12b_pos(ptr %ptr, <2
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX12-NEXT:    v_mov_b32_e32 v0, v3
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: flat_system_atomic_fsub_ret_v2f16__offset12b_pos:
@@ -13540,7 +13522,6 @@ define void @flat_system_atomic_fsub_noret_v2f16__offset12b_pos(ptr %ptr, <2 x h
 ; GFX12-NEXT:    s_cbranch_execnz .LBB49_1
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: flat_system_atomic_fsub_noret_v2f16__offset12b_pos:
@@ -13766,9 +13747,10 @@ define <2 x bfloat> @flat_agent_atomic_fsub_ret_v2bf16(ptr %ptr, <2 x bfloat> %v
 ; GFX12-NEXT:    v_bfe_u32 v7, v3, 16, 1
 ; GFX12-NEXT:    v_or_b32_e32 v9, 0x400000, v3
 ; GFX12-NEXT:    v_cmp_u_f32_e64 s0, v3, v3
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_cndmask_b32_e32 v5, v8, v10, vcc_lo
 ; GFX12-NEXT:    v_add3_u32 v7, v7, v3, 0x7fff
+; GFX12-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_cndmask_b32_e64 v3, v7, v9, s0
 ; GFX12-NEXT:    v_perm_b32 v5, v5, v3, 0x7060302
@@ -13785,7 +13767,6 @@ define <2 x bfloat> @flat_agent_atomic_fsub_ret_v2bf16(ptr %ptr, <2 x bfloat> %v
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s1
 ; GFX12-NEXT:    v_mov_b32_e32 v0, v3
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: flat_agent_atomic_fsub_ret_v2bf16:
@@ -14112,9 +14093,10 @@ define <2 x bfloat> @flat_agent_atomic_fsub_ret_v2bf16__offset12b_pos(ptr %ptr,
 ; GFX12-NEXT:    v_bfe_u32 v7, v3, 16, 1
 ; GFX12-NEXT:    v_or_b32_e32 v9, 0x400000, v3
 ; GFX12-NEXT:    v_cmp_u_f32_e64 s0, v3, v3
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_cndmask_b32_e32 v5, v8, v10, vcc_lo
 ; GFX12-NEXT:    v_add3_u32 v7, v7, v3, 0x7fff
+; GFX12-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_cndmask_b32_e64 v3, v7, v9, s0
 ; GFX12-NEXT:    v_perm_b32 v5, v5, v3, 0x7060302
@@ -14131,7 +14113,6 @@ define <2 x bfloat> @flat_agent_atomic_fsub_ret_v2bf16__offset12b_pos(ptr %ptr,
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s1
 ; GFX12-NEXT:    v_mov_b32_e32 v0, v3
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: flat_agent_atomic_fsub_ret_v2bf16__offset12b_pos:
@@ -14461,9 +14442,10 @@ define <2 x bfloat> @flat_agent_atomic_fsub_ret_v2bf16__offset12b_neg(ptr %ptr,
 ; GFX12-NEXT:    v_bfe_u32 v7, v3, 16, 1
 ; GFX12-NEXT:    v_or_b32_e32 v9, 0x400000, v3
 ; GFX12-NEXT:    v_cmp_u_f32_e64 s0, v3, v3
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_cndmask_b32_e32 v5, v8, v10, vcc_lo
 ; GFX12-NEXT:    v_add3_u32 v7, v7, v3, 0x7fff
+; GFX12-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_cndmask_b32_e64 v3, v7, v9, s0
 ; GFX12-NEXT:    v_perm_b32 v5, v5, v3, 0x7060302
@@ -14480,7 +14462,6 @@ define <2 x bfloat> @flat_agent_atomic_fsub_ret_v2bf16__offset12b_neg(ptr %ptr,
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s1
 ; GFX12-NEXT:    v_mov_b32_e32 v0, v3
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: flat_agent_atomic_fsub_ret_v2bf16__offset12b_neg:
@@ -14827,8 +14808,10 @@ define void @flat_agent_atomic_fsub_noret_v2bf16(ptr %ptr, <2 x bfloat> %val) #0
 ; GFX12-NEXT:    v_add3_u32 v7, v7, v2, 0x7fff
 ; GFX12-NEXT:    v_add3_u32 v8, v8, v6, 0x7fff
 ; GFX12-NEXT:    v_cmp_u_f32_e64 s0, v2, v2
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-NEXT:    s_wait_alu 0xfffd
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX12-NEXT:    v_cndmask_b32_e32 v6, v8, v10, vcc_lo
+; GFX12-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-NEXT:    v_cndmask_b32_e64 v2, v7, v9, s0
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-NEXT:    v_perm_b32 v2, v6, v2, 0x7060302
@@ -14845,7 +14828,6 @@ define void @flat_agent_atomic_fsub_noret_v2bf16(ptr %ptr, <2 x bfloat> %val) #0
 ; GFX12-NEXT:    s_cbranch_execnz .LBB53_1
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s1
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: flat_agent_atomic_fsub_noret_v2bf16:
@@ -15162,8 +15144,10 @@ define void @flat_agent_atomic_fsub_noret_v2bf16__offset12b_pos(ptr %ptr, <2 x b
 ; GFX12-NEXT:    v_add3_u32 v7, v7, v2, 0x7fff
 ; GFX12-NEXT:    v_add3_u32 v8, v8, v6, 0x7fff
 ; GFX12-NEXT:    v_cmp_u_f32_e64 s0, v2, v2
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-NEXT:    s_wait_alu 0xfffd
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX12-NEXT:    v_cndmask_b32_e32 v6, v8, v10, vcc_lo
+; GFX12-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-NEXT:    v_cndmask_b32_e64 v2, v7, v9, s0
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-NEXT:    v_perm_b32 v2, v6, v2, 0x7060302
@@ -15180,7 +15164,6 @@ define void @flat_agent_atomic_fsub_noret_v2bf16__offset12b_pos(ptr %ptr, <2 x b
 ; GFX12-NEXT:    s_cbranch_execnz .LBB54_1
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s1
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: flat_agent_atomic_fsub_noret_v2bf16__offset12b_pos:
@@ -15504,8 +15487,10 @@ define void @flat_agent_atomic_fsub_noret_v2bf16__offset12b_neg(ptr %ptr, <2 x b
 ; GFX12-NEXT:    v_add3_u32 v7, v7, v2, 0x7fff
 ; GFX12-NEXT:    v_add3_u32 v8, v8, v6, 0x7fff
 ; GFX12-NEXT:    v_cmp_u_f32_e64 s0, v2, v2
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-NEXT:    s_wait_alu 0xfffd
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX12-NEXT:    v_cndmask_b32_e32 v6, v8, v10, vcc_lo
+; GFX12-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-NEXT:    v_cndmask_b32_e64 v2, v7, v9, s0
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-NEXT:    v_perm_b32 v2, v6, v2, 0x7060302
@@ -15522,7 +15507,6 @@ define void @flat_agent_atomic_fsub_noret_v2bf16__offset12b_neg(ptr %ptr, <2 x b
 ; GFX12-NEXT:    s_cbranch_execnz .LBB55_1
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s1
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: flat_agent_atomic_fsub_noret_v2bf16__offset12b_neg:
@@ -15865,9 +15849,10 @@ define <2 x bfloat> @flat_system_atomic_fsub_ret_v2bf16__offset12b_pos(ptr %ptr,
 ; GFX12-NEXT:    v_bfe_u32 v7, v3, 16, 1
 ; GFX12-NEXT:    v_or_b32_e32 v9, 0x400000, v3
 ; GFX12-NEXT:    v_cmp_u_f32_e64 s0, v3, v3
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_cndmask_b32_e32 v5, v8, v10, vcc_lo
 ; GFX12-NEXT:    v_add3_u32 v7, v7, v3, 0x7fff
+; GFX12-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_cndmask_b32_e64 v3, v7, v9, s0
 ; GFX12-NEXT:    v_perm_b32 v5, v5, v3, 0x7060302
@@ -15885,7 +15870,6 @@ define <2 x bfloat> @flat_system_atomic_fsub_ret_v2bf16__offset12b_pos(ptr %ptr,
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s1
 ; GFX12-NEXT:    v_mov_b32_e32 v0, v3
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: flat_system_atomic_fsub_ret_v2bf16__offset12b_pos:
@@ -16216,8 +16200,10 @@ define void @flat_system_atomic_fsub_noret_v2bf16__offset12b_pos(ptr %ptr, <2 x
 ; GFX12-NEXT:    v_add3_u32 v7, v7, v2, 0x7fff
 ; GFX12-NEXT:    v_add3_u32 v8, v8, v6, 0x7fff
 ; GFX12-NEXT:    v_cmp_u_f32_e64 s0, v2, v2
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-NEXT:    s_wait_alu 0xfffd
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX12-NEXT:    v_cndmask_b32_e32 v6, v8, v10, vcc_lo
+; GFX12-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-NEXT:    v_cndmask_b32_e64 v2, v7, v9, s0
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-NEXT:    v_perm_b32 v2, v6, v2, 0x7060302
@@ -16235,7 +16221,6 @@ define void @flat_system_atomic_fsub_noret_v2bf16__offset12b_pos(ptr %ptr, <2 x
 ; GFX12-NEXT:    s_cbranch_execnz .LBB57_1
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s1
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: flat_system_atomic_fsub_noret_v2bf16__offset12b_pos:
diff --git a/llvm/test/CodeGen/AMDGPU/flat_atomics_i64.ll b/llvm/test/CodeGen/AMDGPU/flat_atomics_i64.ll
index b4d7ff8e7c526..0b6bdedeb48fc 100644
--- a/llvm/test/CodeGen/AMDGPU/flat_atomics_i64.ll
+++ b/llvm/test/CodeGen/AMDGPU/flat_atomics_i64.ll
@@ -128,7 +128,6 @@ define amdgpu_kernel void @atomic_add_i64_offset(ptr %out, i64 %in) {
 ; GFX12-NEXT:    global_inv scope:SCOPE_DEV
 ; GFX12-NEXT:    s_cbranch_execnz .LBB0_2
 ; GFX12-NEXT:  .LBB0_4: ; %atomicrmw.private
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
 ; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
 ; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s0
@@ -271,7 +270,6 @@ define amdgpu_kernel void @atomic_add_i64_ret_offset(ptr %out, ptr %out2, i64 %i
 ; GFX12-NEXT:  .LBB1_2:
 ; GFX12-NEXT:    ; implicit-def: $vgpr0_vgpr1
 ; GFX12-NEXT:  .LBB1_3: ; %atomicrmw.private
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
 ; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
 ; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s0
@@ -428,7 +426,6 @@ define amdgpu_kernel void @atomic_add_i64_addr64_offset(ptr %out, i64 %in, i64 %
 ; GFX12-NEXT:    global_inv scope:SCOPE_DEV
 ; GFX12-NEXT:    s_cbranch_execnz .LBB2_2
 ; GFX12-NEXT:  .LBB2_4: ; %atomicrmw.private
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
 ; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
 ; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s0
@@ -577,7 +574,6 @@ define amdgpu_kernel void @atomic_add_i64_ret_addr64_offset(ptr %out, ptr %out2,
 ; GFX12-NEXT:  .LBB3_2:
 ; GFX12-NEXT:    ; implicit-def: $vgpr0_vgpr1
 ; GFX12-NEXT:  .LBB3_3: ; %atomicrmw.private
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
 ; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
 ; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s0
@@ -1002,7 +998,6 @@ define amdgpu_kernel void @atomic_add_i64_addr64(ptr %out, i64 %in, i64 %index)
 ; GFX12-NEXT:    global_inv scope:SCOPE_DEV
 ; GFX12-NEXT:    s_cbranch_execnz .LBB6_2
 ; GFX12-NEXT:  .LBB6_4: ; %atomicrmw.private
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
 ; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
 ; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s0
@@ -1145,7 +1140,6 @@ define amdgpu_kernel void @atomic_add_i64_ret_addr64(ptr %out, ptr %out2, i64 %i
 ; GFX12-NEXT:  .LBB7_2:
 ; GFX12-NEXT:    ; implicit-def: $vgpr0_vgpr1
 ; GFX12-NEXT:  .LBB7_3: ; %atomicrmw.private
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
 ; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
 ; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s0
@@ -1287,7 +1281,6 @@ define amdgpu_kernel void @atomic_and_i64_offset(ptr %out, i64 %in) {
 ; GFX12-NEXT:    global_inv scope:SCOPE_DEV
 ; GFX12-NEXT:    s_cbranch_execnz .LBB8_2
 ; GFX12-NEXT:  .LBB8_4: ; %atomicrmw.private
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
 ; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
 ; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s0
@@ -1428,7 +1421,6 @@ define amdgpu_kernel void @atomic_and_i64_ret_offset(ptr %out, ptr %out2, i64 %i
 ; GFX12-NEXT:  .LBB9_2:
 ; GFX12-NEXT:    ; implicit-def: $vgpr0_vgpr1
 ; GFX12-NEXT:  .LBB9_3: ; %atomicrmw.private
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
 ; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
 ; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s0
@@ -1583,7 +1575,6 @@ define amdgpu_kernel void @atomic_and_i64_addr64_offset(ptr %out, i64 %in, i64 %
 ; GFX12-NEXT:    global_inv scope:SCOPE_DEV
 ; GFX12-NEXT:    s_cbranch_execnz .LBB10_2
 ; GFX12-NEXT:  .LBB10_4: ; %atomicrmw.private
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
 ; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
 ; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s0
@@ -1730,7 +1721,6 @@ define amdgpu_kernel void @atomic_and_i64_ret_addr64_offset(ptr %out, ptr %out2,
 ; GFX12-NEXT:  .LBB11_2:
 ; GFX12-NEXT:    ; implicit-def: $vgpr0_vgpr1
 ; GFX12-NEXT:  .LBB11_3: ; %atomicrmw.private
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
 ; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
 ; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s0
@@ -2149,7 +2139,6 @@ define amdgpu_kernel void @atomic_and_i64_addr64(ptr %out, i64 %in, i64 %index)
 ; GFX12-NEXT:    global_inv scope:SCOPE_DEV
 ; GFX12-NEXT:    s_cbranch_execnz .LBB14_2
 ; GFX12-NEXT:  .LBB14_4: ; %atomicrmw.private
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
 ; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
 ; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s0
@@ -2290,7 +2279,6 @@ define amdgpu_kernel void @atomic_and_i64_ret_addr64(ptr %out, ptr %out2, i64 %i
 ; GFX12-NEXT:  .LBB15_2:
 ; GFX12-NEXT:    ; implicit-def: $vgpr0_vgpr1
 ; GFX12-NEXT:  .LBB15_3: ; %atomicrmw.private
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
 ; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
 ; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s0
@@ -2434,7 +2422,6 @@ define amdgpu_kernel void @atomic_sub_i64_offset(ptr %out, i64 %in) {
 ; GFX12-NEXT:    global_inv scope:SCOPE_DEV
 ; GFX12-NEXT:    s_cbranch_execnz .LBB16_2
 ; GFX12-NEXT:  .LBB16_4: ; %atomicrmw.private
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
 ; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
 ; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s0
@@ -2577,7 +2564,6 @@ define amdgpu_kernel void @atomic_sub_i64_ret_offset(ptr %out, ptr %out2, i64 %i
 ; GFX12-NEXT:  .LBB17_2:
 ; GFX12-NEXT:    ; implicit-def: $vgpr0_vgpr1
 ; GFX12-NEXT:  .LBB17_3: ; %atomicrmw.private
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
 ; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
 ; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s0
@@ -2734,7 +2720,6 @@ define amdgpu_kernel void @atomic_sub_i64_addr64_offset(ptr %out, i64 %in, i64 %
 ; GFX12-NEXT:    global_inv scope:SCOPE_DEV
 ; GFX12-NEXT:    s_cbranch_execnz .LBB18_2
 ; GFX12-NEXT:  .LBB18_4: ; %atomicrmw.private
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
 ; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
 ; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s0
@@ -2883,7 +2868,6 @@ define amdgpu_kernel void @atomic_sub_i64_ret_addr64_offset(ptr %out, ptr %out2,
 ; GFX12-NEXT:  .LBB19_2:
 ; GFX12-NEXT:    ; implicit-def: $vgpr0_vgpr1
 ; GFX12-NEXT:  .LBB19_3: ; %atomicrmw.private
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
 ; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
 ; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s0
@@ -3308,7 +3292,6 @@ define amdgpu_kernel void @atomic_sub_i64_addr64(ptr %out, i64 %in, i64 %index)
 ; GFX12-NEXT:    global_inv scope:SCOPE_DEV
 ; GFX12-NEXT:    s_cbranch_execnz .LBB22_2
 ; GFX12-NEXT:  .LBB22_4: ; %atomicrmw.private
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
 ; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
 ; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s0
@@ -3451,7 +3434,6 @@ define amdgpu_kernel void @atomic_sub_i64_ret_addr64(ptr %out, ptr %out2, i64 %i
 ; GFX12-NEXT:  .LBB23_2:
 ; GFX12-NEXT:    ; implicit-def: $vgpr0_vgpr1
 ; GFX12-NEXT:  .LBB23_3: ; %atomicrmw.private
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
 ; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
 ; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s0
@@ -3595,7 +3577,6 @@ define amdgpu_kernel void @atomic_max_i64_offset(ptr %out, i64 %in) {
 ; GFX12-NEXT:    global_inv scope:SCOPE_SE
 ; GFX12-NEXT:    s_cbranch_execnz .LBB24_2
 ; GFX12-NEXT:  .LBB24_4: ; %atomicrmw.private
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
 ; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
 ; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s0
@@ -3741,7 +3722,6 @@ define amdgpu_kernel void @atomic_max_i64_ret_offset(ptr %out, ptr %out2, i64 %i
 ; GFX12-NEXT:  .LBB25_2:
 ; GFX12-NEXT:    ; implicit-def: $vgpr0_vgpr1
 ; GFX12-NEXT:  .LBB25_3: ; %atomicrmw.private
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
 ; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
 ; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s0
@@ -3899,7 +3879,6 @@ define amdgpu_kernel void @atomic_max_i64_addr64_offset(ptr %out, i64 %in, i64 %
 ; GFX12-NEXT:    global_inv scope:SCOPE_SE
 ; GFX12-NEXT:    s_cbranch_execnz .LBB26_2
 ; GFX12-NEXT:  .LBB26_4: ; %atomicrmw.private
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
 ; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
 ; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s0
@@ -4051,7 +4030,6 @@ define amdgpu_kernel void @atomic_max_i64_ret_addr64_offset(ptr %out, ptr %out2,
 ; GFX12-NEXT:  .LBB27_2:
 ; GFX12-NEXT:    ; implicit-def: $vgpr0_vgpr1
 ; GFX12-NEXT:  .LBB27_3: ; %atomicrmw.private
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
 ; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
 ; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s0
@@ -4481,7 +4459,6 @@ define amdgpu_kernel void @atomic_max_i64_addr64(ptr %out, i64 %in, i64 %index)
 ; GFX12-NEXT:    global_inv scope:SCOPE_SE
 ; GFX12-NEXT:    s_cbranch_execnz .LBB30_2
 ; GFX12-NEXT:  .LBB30_4: ; %atomicrmw.private
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
 ; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
 ; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s0
@@ -4627,7 +4604,6 @@ define amdgpu_kernel void @atomic_max_i64_ret_addr64(ptr %out, ptr %out2, i64 %i
 ; GFX12-NEXT:  .LBB31_2:
 ; GFX12-NEXT:    ; implicit-def: $vgpr0_vgpr1
 ; GFX12-NEXT:  .LBB31_3: ; %atomicrmw.private
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
 ; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
 ; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s0
@@ -4772,7 +4748,6 @@ define amdgpu_kernel void @atomic_umax_i64_offset(ptr %out, i64 %in) {
 ; GFX12-NEXT:    global_inv scope:SCOPE_SE
 ; GFX12-NEXT:    s_cbranch_execnz .LBB32_2
 ; GFX12-NEXT:  .LBB32_4: ; %atomicrmw.private
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
 ; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
 ; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s0
@@ -4918,7 +4893,6 @@ define amdgpu_kernel void @atomic_umax_i64_ret_offset(ptr %out, ptr %out2, i64 %
 ; GFX12-NEXT:  .LBB33_2:
 ; GFX12-NEXT:    ; implicit-def: $vgpr0_vgpr1
 ; GFX12-NEXT:  .LBB33_3: ; %atomicrmw.private
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
 ; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
 ; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s0
@@ -5076,7 +5050,6 @@ define amdgpu_kernel void @atomic_umax_i64_addr64_offset(ptr %out, i64 %in, i64
 ; GFX12-NEXT:    global_inv scope:SCOPE_SE
 ; GFX12-NEXT:    s_cbranch_execnz .LBB34_2
 ; GFX12-NEXT:  .LBB34_4: ; %atomicrmw.private
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
 ; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
 ; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s0
@@ -5228,7 +5201,6 @@ define amdgpu_kernel void @atomic_umax_i64_ret_addr64_offset(ptr %out, ptr %out2
 ; GFX12-NEXT:  .LBB35_2:
 ; GFX12-NEXT:    ; implicit-def: $vgpr0_vgpr1
 ; GFX12-NEXT:  .LBB35_3: ; %atomicrmw.private
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
 ; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
 ; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s0
@@ -5658,7 +5630,6 @@ define amdgpu_kernel void @atomic_umax_i64_addr64(ptr %out, i64 %in, i64 %index)
 ; GFX12-NEXT:    global_inv scope:SCOPE_SE
 ; GFX12-NEXT:    s_cbranch_execnz .LBB38_2
 ; GFX12-NEXT:  .LBB38_4: ; %atomicrmw.private
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
 ; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
 ; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s0
@@ -5804,7 +5775,6 @@ define amdgpu_kernel void @atomic_umax_i64_ret_addr64(ptr %out, ptr %out2, i64 %
 ; GFX12-NEXT:  .LBB39_2:
 ; GFX12-NEXT:    ; implicit-def: $vgpr0_vgpr1
 ; GFX12-NEXT:  .LBB39_3: ; %atomicrmw.private
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
 ; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
 ; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s0
@@ -5949,7 +5919,6 @@ define amdgpu_kernel void @atomic_min_i64_offset(ptr %out, i64 %in) {
 ; GFX12-NEXT:    global_inv scope:SCOPE_SE
 ; GFX12-NEXT:    s_cbranch_execnz .LBB40_2
 ; GFX12-NEXT:  .LBB40_4: ; %atomicrmw.private
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
 ; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
 ; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s0
@@ -6095,7 +6064,6 @@ define amdgpu_kernel void @atomic_min_i64_ret_offset(ptr %out, ptr %out2, i64 %i
 ; GFX12-NEXT:  .LBB41_2:
 ; GFX12-NEXT:    ; implicit-def: $vgpr0_vgpr1
 ; GFX12-NEXT:  .LBB41_3: ; %atomicrmw.private
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
 ; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
 ; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s0
@@ -6253,7 +6221,6 @@ define amdgpu_kernel void @atomic_min_i64_addr64_offset(ptr %out, i64 %in, i64 %
 ; GFX12-NEXT:    global_inv scope:SCOPE_SE
 ; GFX12-NEXT:    s_cbranch_execnz .LBB42_2
 ; GFX12-NEXT:  .LBB42_4: ; %atomicrmw.private
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
 ; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
 ; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s0
@@ -6405,7 +6372,6 @@ define amdgpu_kernel void @atomic_min_i64_ret_addr64_offset(ptr %out, ptr %out2,
 ; GFX12-NEXT:  .LBB43_2:
 ; GFX12-NEXT:    ; implicit-def: $vgpr0_vgpr1
 ; GFX12-NEXT:  .LBB43_3: ; %atomicrmw.private
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
 ; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
 ; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s0
@@ -6835,7 +6801,6 @@ define amdgpu_kernel void @atomic_min_i64_addr64(ptr %out, i64 %in, i64 %index)
 ; GFX12-NEXT:    global_inv scope:SCOPE_SE
 ; GFX12-NEXT:    s_cbranch_execnz .LBB46_2
 ; GFX12-NEXT:  .LBB46_4: ; %atomicrmw.private
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
 ; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
 ; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s0
@@ -6981,7 +6946,6 @@ define amdgpu_kernel void @atomic_min_i64_ret_addr64(ptr %out, ptr %out2, i64 %i
 ; GFX12-NEXT:  .LBB47_2:
 ; GFX12-NEXT:    ; implicit-def: $vgpr0_vgpr1
 ; GFX12-NEXT:  .LBB47_3: ; %atomicrmw.private
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
 ; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
 ; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s0
@@ -7126,7 +7090,6 @@ define amdgpu_kernel void @atomic_umin_i64_offset(ptr %out, i64 %in) {
 ; GFX12-NEXT:    global_inv scope:SCOPE_SE
 ; GFX12-NEXT:    s_cbranch_execnz .LBB48_2
 ; GFX12-NEXT:  .LBB48_4: ; %atomicrmw.private
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
 ; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
 ; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s0
@@ -7272,7 +7235,6 @@ define amdgpu_kernel void @atomic_umin_i64_ret_offset(ptr %out, ptr %out2, i64 %
 ; GFX12-NEXT:  .LBB49_2:
 ; GFX12-NEXT:    ; implicit-def: $vgpr0_vgpr1
 ; GFX12-NEXT:  .LBB49_3: ; %atomicrmw.private
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
 ; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
 ; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s0
@@ -7430,7 +7392,6 @@ define amdgpu_kernel void @atomic_umin_i64_addr64_offset(ptr %out, i64 %in, i64
 ; GFX12-NEXT:    global_inv scope:SCOPE_SE
 ; GFX12-NEXT:    s_cbranch_execnz .LBB50_2
 ; GFX12-NEXT:  .LBB50_4: ; %atomicrmw.private
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
 ; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
 ; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s0
@@ -7582,7 +7543,6 @@ define amdgpu_kernel void @atomic_umin_i64_ret_addr64_offset(ptr %out, ptr %out2
 ; GFX12-NEXT:  .LBB51_2:
 ; GFX12-NEXT:    ; implicit-def: $vgpr0_vgpr1
 ; GFX12-NEXT:  .LBB51_3: ; %atomicrmw.private
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
 ; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
 ; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s0
@@ -8012,7 +7972,6 @@ define amdgpu_kernel void @atomic_umin_i64_addr64(ptr %out, i64 %in, i64 %index)
 ; GFX12-NEXT:    global_inv scope:SCOPE_SE
 ; GFX12-NEXT:    s_cbranch_execnz .LBB54_2
 ; GFX12-NEXT:  .LBB54_4: ; %atomicrmw.private
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
 ; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
 ; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s0
@@ -8158,7 +8117,6 @@ define amdgpu_kernel void @atomic_umin_i64_ret_addr64(ptr %out, ptr %out2, i64 %
 ; GFX12-NEXT:  .LBB55_2:
 ; GFX12-NEXT:    ; implicit-def: $vgpr0_vgpr1
 ; GFX12-NEXT:  .LBB55_3: ; %atomicrmw.private
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
 ; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
 ; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s0
@@ -8301,7 +8259,6 @@ define amdgpu_kernel void @atomic_or_i64_offset(ptr %out, i64 %in) {
 ; GFX12-NEXT:    global_inv scope:SCOPE_DEV
 ; GFX12-NEXT:    s_cbranch_execnz .LBB56_2
 ; GFX12-NEXT:  .LBB56_4: ; %atomicrmw.private
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
 ; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
 ; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s0
@@ -8442,7 +8399,6 @@ define amdgpu_kernel void @atomic_or_i64_ret_offset(ptr %out, ptr %out2, i64 %in
 ; GFX12-NEXT:  .LBB57_2:
 ; GFX12-NEXT:    ; implicit-def: $vgpr0_vgpr1
 ; GFX12-NEXT:  .LBB57_3: ; %atomicrmw.private
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
 ; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
 ; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s0
@@ -8597,7 +8553,6 @@ define amdgpu_kernel void @atomic_or_i64_addr64_offset(ptr %out, i64 %in, i64 %i
 ; GFX12-NEXT:    global_inv scope:SCOPE_DEV
 ; GFX12-NEXT:    s_cbranch_execnz .LBB58_2
 ; GFX12-NEXT:  .LBB58_4: ; %atomicrmw.private
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
 ; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
 ; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s0
@@ -8744,7 +8699,6 @@ define amdgpu_kernel void @atomic_or_i64_ret_addr64_offset(ptr %out, ptr %out2,
 ; GFX12-NEXT:  .LBB59_2:
 ; GFX12-NEXT:    ; implicit-def: $vgpr0_vgpr1
 ; GFX12-NEXT:  .LBB59_3: ; %atomicrmw.private
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
 ; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
 ; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s0
@@ -9163,7 +9117,6 @@ define amdgpu_kernel void @atomic_or_i64_addr64(ptr %out, i64 %in, i64 %index) {
 ; GFX12-NEXT:    global_inv scope:SCOPE_DEV
 ; GFX12-NEXT:    s_cbranch_execnz .LBB62_2
 ; GFX12-NEXT:  .LBB62_4: ; %atomicrmw.private
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
 ; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
 ; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s0
@@ -9304,7 +9257,6 @@ define amdgpu_kernel void @atomic_or_i64_ret_addr64(ptr %out, ptr %out2, i64 %in
 ; GFX12-NEXT:  .LBB63_2:
 ; GFX12-NEXT:    ; implicit-def: $vgpr0_vgpr1
 ; GFX12-NEXT:  .LBB63_3: ; %atomicrmw.private
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
 ; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
 ; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s0
@@ -9439,7 +9391,6 @@ define amdgpu_kernel void @atomic_xchg_i64_offset(ptr %out, i64 %in) {
 ; GFX12-NEXT:    s_cbranch_execnz .LBB64_2
 ; GFX12-NEXT:  .LBB64_4: ; %atomicrmw.private
 ; GFX12-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
 ; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
 ; GFX12-NEXT:    scratch_store_b64 off, v[0:1], s0
@@ -9566,7 +9517,6 @@ define amdgpu_kernel void @atomic_xchg_f64_offset(ptr %out, double %in) {
 ; GFX12-NEXT:    s_cbranch_execnz .LBB65_2
 ; GFX12-NEXT:  .LBB65_4: ; %atomicrmw.private
 ; GFX12-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
 ; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
 ; GFX12-NEXT:    scratch_store_b64 off, v[0:1], s0
@@ -9693,7 +9643,6 @@ define amdgpu_kernel void @atomic_xchg_pointer_offset(ptr %out, ptr %in) {
 ; GFX12-NEXT:    s_cbranch_execnz .LBB66_2
 ; GFX12-NEXT:  .LBB66_4: ; %atomicrmw.private
 ; GFX12-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
 ; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
 ; GFX12-NEXT:    scratch_store_b64 off, v[0:1], s0
@@ -9828,7 +9777,6 @@ define amdgpu_kernel void @atomic_xchg_i64_ret_offset(ptr %out, ptr %out2, i64 %
 ; GFX12-NEXT:  .LBB67_2:
 ; GFX12-NEXT:    ; implicit-def: $vgpr0_vgpr1
 ; GFX12-NEXT:  .LBB67_3: ; %atomicrmw.private
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
 ; GFX12-NEXT:    v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5
 ; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
@@ -9975,7 +9923,6 @@ define amdgpu_kernel void @atomic_xchg_i64_addr64_offset(ptr %out, i64 %in, i64
 ; GFX12-NEXT:    s_cbranch_execnz .LBB68_2
 ; GFX12-NEXT:  .LBB68_4: ; %atomicrmw.private
 ; GFX12-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
 ; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
 ; GFX12-NEXT:    scratch_store_b64 off, v[0:1], s0
@@ -10116,7 +10063,6 @@ define amdgpu_kernel void @atomic_xchg_i64_ret_addr64_offset(ptr %out, ptr %out2
 ; GFX12-NEXT:  .LBB69_2:
 ; GFX12-NEXT:    ; implicit-def: $vgpr0_vgpr1
 ; GFX12-NEXT:  .LBB69_3: ; %atomicrmw.private
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
 ; GFX12-NEXT:    v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5
 ; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
@@ -10513,7 +10459,6 @@ define amdgpu_kernel void @atomic_xchg_i64_addr64(ptr %out, i64 %in, i64 %index)
 ; GFX12-NEXT:    s_cbranch_execnz .LBB72_2
 ; GFX12-NEXT:  .LBB72_4: ; %atomicrmw.private
 ; GFX12-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
 ; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
 ; GFX12-NEXT:    scratch_store_b64 off, v[0:1], s0
@@ -10648,7 +10593,6 @@ define amdgpu_kernel void @atomic_xchg_i64_ret_addr64(ptr %out, ptr %out2, i64 %
 ; GFX12-NEXT:  .LBB73_2:
 ; GFX12-NEXT:    ; implicit-def: $vgpr0_vgpr1
 ; GFX12-NEXT:  .LBB73_3: ; %atomicrmw.private
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
 ; GFX12-NEXT:    v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5
 ; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
@@ -10789,7 +10733,6 @@ define amdgpu_kernel void @atomic_xor_i64_offset(ptr %out, i64 %in) {
 ; GFX12-NEXT:    global_inv scope:SCOPE_DEV
 ; GFX12-NEXT:    s_cbranch_execnz .LBB74_2
 ; GFX12-NEXT:  .LBB74_4: ; %atomicrmw.private
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
 ; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
 ; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s0
@@ -10930,7 +10873,6 @@ define amdgpu_kernel void @atomic_xor_i64_ret_offset(ptr %out, ptr %out2, i64 %i
 ; GFX12-NEXT:  .LBB75_2:
 ; GFX12-NEXT:    ; implicit-def: $vgpr0_vgpr1
 ; GFX12-NEXT:  .LBB75_3: ; %atomicrmw.private
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
 ; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
 ; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s0
@@ -11085,7 +11027,6 @@ define amdgpu_kernel void @atomic_xor_i64_addr64_offset(ptr %out, i64 %in, i64 %
 ; GFX12-NEXT:    global_inv scope:SCOPE_DEV
 ; GFX12-NEXT:    s_cbranch_execnz .LBB76_2
 ; GFX12-NEXT:  .LBB76_4: ; %atomicrmw.private
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
 ; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
 ; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s0
@@ -11232,7 +11173,6 @@ define amdgpu_kernel void @atomic_xor_i64_ret_addr64_offset(ptr %out, ptr %out2,
 ; GFX12-NEXT:  .LBB77_2:
 ; GFX12-NEXT:    ; implicit-def: $vgpr0_vgpr1
 ; GFX12-NEXT:  .LBB77_3: ; %atomicrmw.private
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
 ; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
 ; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s0
@@ -11651,7 +11591,6 @@ define amdgpu_kernel void @atomic_xor_i64_addr64(ptr %out, i64 %in, i64 %index)
 ; GFX12-NEXT:    global_inv scope:SCOPE_DEV
 ; GFX12-NEXT:    s_cbranch_execnz .LBB80_2
 ; GFX12-NEXT:  .LBB80_4: ; %atomicrmw.private
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
 ; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
 ; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s0
@@ -11792,7 +11731,6 @@ define amdgpu_kernel void @atomic_xor_i64_ret_addr64(ptr %out, ptr %out2, i64 %i
 ; GFX12-NEXT:  .LBB81_2:
 ; GFX12-NEXT:    ; implicit-def: $vgpr0_vgpr1
 ; GFX12-NEXT:  .LBB81_3: ; %atomicrmw.private
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
 ; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
 ; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s0
@@ -12355,7 +12293,6 @@ define amdgpu_kernel void @atomic_cmpxchg_i64_offset(ptr %out, i64 %in, i64 %old
 ; GFX12-NEXT:    global_inv scope:SCOPE_DEV
 ; GFX12-NEXT:    s_cbranch_execnz .LBB90_2
 ; GFX12-NEXT:  .LBB90_4: ; %atomicrmw.private
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
 ; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
 ; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s0
@@ -12507,7 +12444,6 @@ define amdgpu_kernel void @atomic_cmpxchg_i64_soffset(ptr %out, i64 %in, i64 %ol
 ; GFX12-NEXT:    global_inv scope:SCOPE_DEV
 ; GFX12-NEXT:    s_cbranch_execnz .LBB91_2
 ; GFX12-NEXT:  .LBB91_4: ; %atomicrmw.private
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
 ; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
 ; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s0
@@ -12654,7 +12590,6 @@ define amdgpu_kernel void @atomic_cmpxchg_i64_ret_offset(ptr %out, ptr %out2, i6
 ; GFX12-NEXT:  .LBB92_2:
 ; GFX12-NEXT:    ; implicit-def: $vgpr0_vgpr1
 ; GFX12-NEXT:  .LBB92_3: ; %atomicrmw.private
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
 ; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
 ; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s0
@@ -12816,7 +12751,6 @@ define amdgpu_kernel void @atomic_cmpxchg_i64_addr64_offset(ptr %out, i64 %in, i
 ; GFX12-NEXT:    global_inv scope:SCOPE_DEV
 ; GFX12-NEXT:    s_cbranch_execnz .LBB93_2
 ; GFX12-NEXT:  .LBB93_4: ; %atomicrmw.private
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
 ; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
 ; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s0
@@ -12977,7 +12911,6 @@ define amdgpu_kernel void @atomic_cmpxchg_i64_ret_addr64_offset(ptr %out, ptr %o
 ; GFX12-NEXT:  .LBB94_2:
 ; GFX12-NEXT:    ; implicit-def: $vgpr0_vgpr1
 ; GFX12-NEXT:  .LBB94_3: ; %atomicrmw.private
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_cmp_lg_u64 s[2:3], 0
 ; GFX12-NEXT:    s_cselect_b32 s2, s2, -1
 ; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s2
@@ -13425,7 +13358,6 @@ define amdgpu_kernel void @atomic_cmpxchg_i64_addr64(ptr %out, i64 %in, i64 %ind
 ; GFX12-NEXT:    global_inv scope:SCOPE_DEV
 ; GFX12-NEXT:    s_cbranch_execnz .LBB97_2
 ; GFX12-NEXT:  .LBB97_4: ; %atomicrmw.private
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
 ; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
 ; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s0
@@ -13580,7 +13512,6 @@ define amdgpu_kernel void @atomic_cmpxchg_i64_ret_addr64(ptr %out, ptr %out2, i6
 ; GFX12-NEXT:  .LBB98_2:
 ; GFX12-NEXT:    ; implicit-def: $vgpr0_vgpr1
 ; GFX12-NEXT:  .LBB98_3: ; %atomicrmw.private
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_cmp_lg_u64 s[2:3], 0
 ; GFX12-NEXT:    s_cselect_b32 s2, s2, -1
 ; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s2
@@ -14138,7 +14069,6 @@ define amdgpu_kernel void @atomic_inc_i64_offset(ptr %out, i64 %in) {
 ; GFX12-NEXT:    global_inv scope:SCOPE_DEV
 ; GFX12-NEXT:    s_cbranch_execnz .LBB107_2
 ; GFX12-NEXT:  .LBB107_4: ; %atomicrmw.private
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
 ; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
 ; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s0
@@ -14146,6 +14076,7 @@ define amdgpu_kernel void @atomic_inc_i64_offset(ptr %out, i64 %in) {
 ; GFX12-NEXT:    v_add_co_u32 v2, vcc_lo, v0, 1
 ; GFX12-NEXT:    v_add_co_ci_u32_e32 v3, vcc_lo, 0, v1, vcc_lo
 ; GFX12-NEXT:    v_cmp_gt_u64_e32 vcc_lo, s[2:3], v[0:1]
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2)
 ; GFX12-NEXT:    v_dual_cndmask_b32 v1, 0, v3 :: v_dual_cndmask_b32 v0, 0, v2
 ; GFX12-NEXT:    scratch_store_b64 off, v[0:1], s0
@@ -14288,7 +14219,6 @@ define amdgpu_kernel void @atomic_inc_i64_ret_offset(ptr %out, ptr %out2, i64 %i
 ; GFX12-NEXT:  .LBB108_2:
 ; GFX12-NEXT:    ; implicit-def: $vgpr0_vgpr1
 ; GFX12-NEXT:  .LBB108_3: ; %atomicrmw.private
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
 ; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
 ; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s0
@@ -14296,6 +14226,7 @@ define amdgpu_kernel void @atomic_inc_i64_ret_offset(ptr %out, ptr %out2, i64 %i
 ; GFX12-NEXT:    v_add_co_u32 v2, vcc_lo, v0, 1
 ; GFX12-NEXT:    v_add_co_ci_u32_e32 v3, vcc_lo, 0, v1, vcc_lo
 ; GFX12-NEXT:    v_cmp_gt_u64_e32 vcc_lo, s[4:5], v[0:1]
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2)
 ; GFX12-NEXT:    v_dual_cndmask_b32 v3, 0, v3 :: v_dual_cndmask_b32 v2, 0, v2
 ; GFX12-NEXT:    scratch_store_b64 off, v[2:3], s0
@@ -14452,7 +14383,6 @@ define amdgpu_kernel void @atomic_inc_i64_incr64_offset(ptr %out, i64 %in, i64 %
 ; GFX12-NEXT:    global_inv scope:SCOPE_DEV
 ; GFX12-NEXT:    s_cbranch_execnz .LBB109_2
 ; GFX12-NEXT:  .LBB109_4: ; %atomicrmw.private
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
 ; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
 ; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s0
@@ -14460,6 +14390,7 @@ define amdgpu_kernel void @atomic_inc_i64_incr64_offset(ptr %out, i64 %in, i64 %
 ; GFX12-NEXT:    v_add_co_u32 v2, vcc_lo, v0, 1
 ; GFX12-NEXT:    v_add_co_ci_u32_e32 v3, vcc_lo, 0, v1, vcc_lo
 ; GFX12-NEXT:    v_cmp_gt_u64_e32 vcc_lo, s[2:3], v[0:1]
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2)
 ; GFX12-NEXT:    v_dual_cndmask_b32 v1, 0, v3 :: v_dual_cndmask_b32 v0, 0, v2
 ; GFX12-NEXT:    scratch_store_b64 off, v[0:1], s0
@@ -14608,7 +14539,6 @@ define amdgpu_kernel void @atomic_inc_i64_ret_incr64_offset(ptr %out, ptr %out2,
 ; GFX12-NEXT:  .LBB110_2:
 ; GFX12-NEXT:    ; implicit-def: $vgpr0_vgpr1
 ; GFX12-NEXT:  .LBB110_3: ; %atomicrmw.private
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
 ; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
 ; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s0
@@ -14616,6 +14546,7 @@ define amdgpu_kernel void @atomic_inc_i64_ret_incr64_offset(ptr %out, ptr %out2,
 ; GFX12-NEXT:    v_add_co_u32 v2, vcc_lo, v0, 1
 ; GFX12-NEXT:    v_add_co_ci_u32_e32 v3, vcc_lo, 0, v1, vcc_lo
 ; GFX12-NEXT:    v_cmp_gt_u64_e32 vcc_lo, s[4:5], v[0:1]
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2)
 ; GFX12-NEXT:    v_dual_cndmask_b32 v3, 0, v3 :: v_dual_cndmask_b32 v2, 0, v2
 ; GFX12-NEXT:    scratch_store_b64 off, v[2:3], s0
@@ -14762,6 +14693,7 @@ define amdgpu_kernel void @atomic_inc_i64(ptr %out, i64 %in) {
 ; GFX12-NEXT:    v_add_co_u32 v2, vcc_lo, v0, 1
 ; GFX12-NEXT:    v_add_co_ci_u32_e32 v3, vcc_lo, 0, v1, vcc_lo
 ; GFX12-NEXT:    v_cmp_gt_u64_e32 vcc_lo, s[2:3], v[0:1]
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2)
 ; GFX12-NEXT:    v_dual_cndmask_b32 v1, 0, v3 :: v_dual_cndmask_b32 v0, 0, v2
 ; GFX12-NEXT:    scratch_store_b64 off, v[0:1], s0
@@ -14905,6 +14837,7 @@ define amdgpu_kernel void @atomic_inc_i64_ret(ptr %out, ptr %out2, i64 %in) {
 ; GFX12-NEXT:    v_add_co_u32 v2, vcc_lo, v0, 1
 ; GFX12-NEXT:    v_add_co_ci_u32_e32 v3, vcc_lo, 0, v1, vcc_lo
 ; GFX12-NEXT:    v_cmp_gt_u64_e32 vcc_lo, s[4:5], v[0:1]
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2)
 ; GFX12-NEXT:    v_dual_cndmask_b32 v3, 0, v3 :: v_dual_cndmask_b32 v2, 0, v2
 ; GFX12-NEXT:    scratch_store_b64 off, v[2:3], s0
@@ -15054,7 +14987,6 @@ define amdgpu_kernel void @atomic_inc_i64_incr64(ptr %out, i64 %in, i64 %index)
 ; GFX12-NEXT:    global_inv scope:SCOPE_DEV
 ; GFX12-NEXT:    s_cbranch_execnz .LBB113_2
 ; GFX12-NEXT:  .LBB113_4: ; %atomicrmw.private
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
 ; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
 ; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s0
@@ -15062,6 +14994,7 @@ define amdgpu_kernel void @atomic_inc_i64_incr64(ptr %out, i64 %in, i64 %index)
 ; GFX12-NEXT:    v_add_co_u32 v2, vcc_lo, v0, 1
 ; GFX12-NEXT:    v_add_co_ci_u32_e32 v3, vcc_lo, 0, v1, vcc_lo
 ; GFX12-NEXT:    v_cmp_gt_u64_e32 vcc_lo, s[2:3], v[0:1]
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2)
 ; GFX12-NEXT:    v_dual_cndmask_b32 v1, 0, v3 :: v_dual_cndmask_b32 v0, 0, v2
 ; GFX12-NEXT:    scratch_store_b64 off, v[0:1], s0
@@ -15204,7 +15137,6 @@ define amdgpu_kernel void @atomic_inc_i64_ret_incr64(ptr %out, ptr %out2, i64 %i
 ; GFX12-NEXT:  .LBB114_2:
 ; GFX12-NEXT:    ; implicit-def: $vgpr0_vgpr1
 ; GFX12-NEXT:  .LBB114_3: ; %atomicrmw.private
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
 ; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
 ; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s0
@@ -15212,6 +15144,7 @@ define amdgpu_kernel void @atomic_inc_i64_ret_incr64(ptr %out, ptr %out2, i64 %i
 ; GFX12-NEXT:    v_add_co_u32 v2, vcc_lo, v0, 1
 ; GFX12-NEXT:    v_add_co_ci_u32_e32 v3, vcc_lo, 0, v1, vcc_lo
 ; GFX12-NEXT:    v_cmp_gt_u64_e32 vcc_lo, s[4:5], v[0:1]
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2)
 ; GFX12-NEXT:    v_dual_cndmask_b32 v3, 0, v3 :: v_dual_cndmask_b32 v2, 0, v2
 ; GFX12-NEXT:    scratch_store_b64 off, v[2:3], s0
@@ -15361,7 +15294,6 @@ define amdgpu_kernel void @atomic_dec_i64_offset(ptr %out, i64 %in) {
 ; GFX12-NEXT:    global_inv scope:SCOPE_DEV
 ; GFX12-NEXT:    s_cbranch_execnz .LBB115_2
 ; GFX12-NEXT:  .LBB115_4: ; %atomicrmw.private
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
 ; GFX12-NEXT:    s_cselect_b32 s4, s0, -1
 ; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s4
@@ -15369,11 +15301,11 @@ define amdgpu_kernel void @atomic_dec_i64_offset(ptr %out, i64 %in) {
 ; GFX12-NEXT:    v_cmp_eq_u64_e32 vcc_lo, 0, v[0:1]
 ; GFX12-NEXT:    v_cmp_lt_u64_e64 s0, s[2:3], v[0:1]
 ; GFX12-NEXT:    v_add_co_u32 v0, s1, v0, -1
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-NEXT:    v_add_co_ci_u32_e64 v1, s1, -1, v1, s1
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    s_or_b32 s0, vcc_lo, s0
 ; GFX12-NEXT:    s_wait_alu 0xfffe
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-NEXT:    v_cndmask_b32_e64 v1, v1, s3, s0
 ; GFX12-NEXT:    v_cndmask_b32_e64 v0, v0, s2, s0
 ; GFX12-NEXT:    scratch_store_b64 off, v[0:1], s4
@@ -15524,7 +15456,6 @@ define amdgpu_kernel void @atomic_dec_i64_ret_offset(ptr %out, ptr %out2, i64 %i
 ; GFX12-NEXT:  .LBB116_2:
 ; GFX12-NEXT:    ; implicit-def: $vgpr0_vgpr1
 ; GFX12-NEXT:  .LBB116_3: ; %atomicrmw.private
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
 ; GFX12-NEXT:    s_cselect_b32 s6, s0, -1
 ; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s6
@@ -15532,11 +15463,11 @@ define amdgpu_kernel void @atomic_dec_i64_ret_offset(ptr %out, ptr %out2, i64 %i
 ; GFX12-NEXT:    v_cmp_eq_u64_e32 vcc_lo, 0, v[0:1]
 ; GFX12-NEXT:    v_cmp_lt_u64_e64 s0, s[4:5], v[0:1]
 ; GFX12-NEXT:    v_add_co_u32 v2, s1, v0, -1
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-NEXT:    v_add_co_ci_u32_e64 v3, s1, -1, v1, s1
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    s_or_b32 s0, vcc_lo, s0
 ; GFX12-NEXT:    s_wait_alu 0xfffe
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-NEXT:    v_cndmask_b32_e64 v3, v3, s5, s0
 ; GFX12-NEXT:    v_cndmask_b32_e64 v2, v2, s4, s0
 ; GFX12-NEXT:    scratch_store_b64 off, v[2:3], s6
@@ -15699,7 +15630,6 @@ define amdgpu_kernel void @atomic_dec_i64_decr64_offset(ptr %out, i64 %in, i64 %
 ; GFX12-NEXT:    global_inv scope:SCOPE_DEV
 ; GFX12-NEXT:    s_cbranch_execnz .LBB117_2
 ; GFX12-NEXT:  .LBB117_4: ; %atomicrmw.private
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
 ; GFX12-NEXT:    s_cselect_b32 s4, s0, -1
 ; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s4
@@ -15707,11 +15637,11 @@ define amdgpu_kernel void @atomic_dec_i64_decr64_offset(ptr %out, i64 %in, i64 %
 ; GFX12-NEXT:    v_cmp_eq_u64_e32 vcc_lo, 0, v[0:1]
 ; GFX12-NEXT:    v_cmp_lt_u64_e64 s0, s[2:3], v[0:1]
 ; GFX12-NEXT:    v_add_co_u32 v0, s1, v0, -1
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-NEXT:    v_add_co_ci_u32_e64 v1, s1, -1, v1, s1
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    s_or_b32 s0, vcc_lo, s0
 ; GFX12-NEXT:    s_wait_alu 0xfffe
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-NEXT:    v_cndmask_b32_e64 v1, v1, s3, s0
 ; GFX12-NEXT:    v_cndmask_b32_e64 v0, v0, s2, s0
 ; GFX12-NEXT:    scratch_store_b64 off, v[0:1], s4
@@ -15868,7 +15798,6 @@ define amdgpu_kernel void @atomic_dec_i64_ret_decr64_offset(ptr %out, ptr %out2,
 ; GFX12-NEXT:  .LBB118_2:
 ; GFX12-NEXT:    ; implicit-def: $vgpr0_vgpr1
 ; GFX12-NEXT:  .LBB118_3: ; %atomicrmw.private
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
 ; GFX12-NEXT:    s_cselect_b32 s6, s0, -1
 ; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s6
@@ -15876,11 +15805,11 @@ define amdgpu_kernel void @atomic_dec_i64_ret_decr64_offset(ptr %out, ptr %out2,
 ; GFX12-NEXT:    v_cmp_eq_u64_e32 vcc_lo, 0, v[0:1]
 ; GFX12-NEXT:    v_cmp_lt_u64_e64 s0, s[4:5], v[0:1]
 ; GFX12-NEXT:    v_add_co_u32 v2, s1, v0, -1
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-NEXT:    v_add_co_ci_u32_e64 v3, s1, -1, v1, s1
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    s_or_b32 s0, vcc_lo, s0
 ; GFX12-NEXT:    s_wait_alu 0xfffe
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-NEXT:    v_cndmask_b32_e64 v3, v3, s5, s0
 ; GFX12-NEXT:    v_cndmask_b32_e64 v2, v2, s4, s0
 ; GFX12-NEXT:    scratch_store_b64 off, v[2:3], s6
@@ -16033,11 +15962,11 @@ define amdgpu_kernel void @atomic_dec_i64(ptr %out, i64 %in) {
 ; GFX12-NEXT:    v_cmp_eq_u64_e32 vcc_lo, 0, v[0:1]
 ; GFX12-NEXT:    v_cmp_lt_u64_e64 s0, s[2:3], v[0:1]
 ; GFX12-NEXT:    v_add_co_u32 v0, s1, v0, -1
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-NEXT:    v_add_co_ci_u32_e64 v1, s1, -1, v1, s1
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    s_or_b32 s0, vcc_lo, s0
 ; GFX12-NEXT:    s_wait_alu 0xfffe
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-NEXT:    v_cndmask_b32_e64 v1, v1, s3, s0
 ; GFX12-NEXT:    v_cndmask_b32_e64 v0, v0, s2, s0
 ; GFX12-NEXT:    scratch_store_b64 off, v[0:1], s4
@@ -16189,11 +16118,11 @@ define amdgpu_kernel void @atomic_dec_i64_ret(ptr %out, ptr %out2, i64 %in) {
 ; GFX12-NEXT:    v_cmp_eq_u64_e32 vcc_lo, 0, v[0:1]
 ; GFX12-NEXT:    v_cmp_lt_u64_e64 s0, s[4:5], v[0:1]
 ; GFX12-NEXT:    v_add_co_u32 v2, s1, v0, -1
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-NEXT:    v_add_co_ci_u32_e64 v3, s1, -1, v1, s1
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    s_or_b32 s0, vcc_lo, s0
 ; GFX12-NEXT:    s_wait_alu 0xfffe
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-NEXT:    v_cndmask_b32_e64 v3, v3, s5, s0
 ; GFX12-NEXT:    v_cndmask_b32_e64 v2, v2, s4, s0
 ; GFX12-NEXT:    scratch_store_b64 off, v[2:3], s6
@@ -16349,7 +16278,6 @@ define amdgpu_kernel void @atomic_dec_i64_decr64(ptr %out, i64 %in, i64 %index)
 ; GFX12-NEXT:    global_inv scope:SCOPE_DEV
 ; GFX12-NEXT:    s_cbranch_execnz .LBB121_2
 ; GFX12-NEXT:  .LBB121_4: ; %atomicrmw.private
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
 ; GFX12-NEXT:    s_cselect_b32 s4, s0, -1
 ; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s4
@@ -16357,11 +16285,11 @@ define amdgpu_kernel void @atomic_dec_i64_decr64(ptr %out, i64 %in, i64 %index)
 ; GFX12-NEXT:    v_cmp_eq_u64_e32 vcc_lo, 0, v[0:1]
 ; GFX12-NEXT:    v_cmp_lt_u64_e64 s0, s[2:3], v[0:1]
 ; GFX12-NEXT:    v_add_co_u32 v0, s1, v0, -1
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-NEXT:    v_add_co_ci_u32_e64 v1, s1, -1, v1, s1
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    s_or_b32 s0, vcc_lo, s0
 ; GFX12-NEXT:    s_wait_alu 0xfffe
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-NEXT:    v_cndmask_b32_e64 v1, v1, s3, s0
 ; GFX12-NEXT:    v_cndmask_b32_e64 v0, v0, s2, s0
 ; GFX12-NEXT:    scratch_store_b64 off, v[0:1], s4
@@ -16512,7 +16440,6 @@ define amdgpu_kernel void @atomic_dec_i64_ret_decr64(ptr %out, ptr %out2, i64 %i
 ; GFX12-NEXT:  .LBB122_2:
 ; GFX12-NEXT:    ; implicit-def: $vgpr0_vgpr1
 ; GFX12-NEXT:  .LBB122_3: ; %atomicrmw.private
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
 ; GFX12-NEXT:    s_cselect_b32 s6, s0, -1
 ; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s6
@@ -16520,11 +16447,11 @@ define amdgpu_kernel void @atomic_dec_i64_ret_decr64(ptr %out, ptr %out2, i64 %i
 ; GFX12-NEXT:    v_cmp_eq_u64_e32 vcc_lo, 0, v[0:1]
 ; GFX12-NEXT:    v_cmp_lt_u64_e64 s0, s[4:5], v[0:1]
 ; GFX12-NEXT:    v_add_co_u32 v2, s1, v0, -1
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-NEXT:    v_add_co_ci_u32_e64 v3, s1, -1, v1, s1
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    s_or_b32 s0, vcc_lo, s0
 ; GFX12-NEXT:    s_wait_alu 0xfffe
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-NEXT:    v_cndmask_b32_e64 v3, v3, s5, s0
 ; GFX12-NEXT:    v_cndmask_b32_e64 v2, v2, s4, s0
 ; GFX12-NEXT:    scratch_store_b64 off, v[2:3], s6
diff --git a/llvm/test/CodeGen/AMDGPU/fma.f16.ll b/llvm/test/CodeGen/AMDGPU/fma.f16.ll
index 822d40f7349b0..52a23690dcf53 100644
--- a/llvm/test/CodeGen/AMDGPU/fma.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/fma.f16.ll
@@ -250,6 +250,7 @@ define i32 @test_D139469_f16(half %arg) {
 ; GFX12-SDAG-NEXT:    v_min_num_f16_e32 v0, v2, v1
 ; GFX12-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-SDAG-NEXT:    v_cmp_gt_f16_e32 vcc_lo, 0, v0
+; GFX12-SDAG-NEXT:    s_wait_alu 0xfffd
 ; GFX12-SDAG-NEXT:    v_cndmask_b32_e64 v0, 0, 1, vcc_lo
 ; GFX12-SDAG-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -400,9 +401,11 @@ define <2 x i32> @test_D139469_v2f16(<2 x half> %arg) {
 ; GFX12-SDAG-NEXT:    v_pk_min_num_f16 v0, v1, v0
 ; GFX12-SDAG-NEXT:    v_lshrrev_b32_e32 v1, 16, v0
 ; GFX12-SDAG-NEXT:    v_cmp_gt_f16_e32 vcc_lo, 0, v0
+; GFX12-SDAG-NEXT:    s_wait_alu 0xfffd
 ; GFX12-SDAG-NEXT:    v_cndmask_b32_e64 v0, 0, 1, vcc_lo
 ; GFX12-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_3)
 ; GFX12-SDAG-NEXT:    v_cmp_gt_f16_e32 vcc_lo, 0, v1
+; GFX12-SDAG-NEXT:    s_wait_alu 0xfffd
 ; GFX12-SDAG-NEXT:    v_cndmask_b32_e64 v1, 0, 1, vcc_lo
 ; GFX12-SDAG-NEXT:    s_setpc_b64 s[30:31]
 ;
diff --git a/llvm/test/CodeGen/AMDGPU/fmaximum3.ll b/llvm/test/CodeGen/AMDGPU/fmaximum3.ll
index f0fa621e3b4bc..c26e2911ab3ea 100644
--- a/llvm/test/CodeGen/AMDGPU/fmaximum3.ll
+++ b/llvm/test/CodeGen/AMDGPU/fmaximum3.ll
@@ -80,6 +80,7 @@ define amdgpu_ps i32 @s_fmaximum3_f32(float inreg %a, float inreg %b, float inre
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_maximum3_f32 v0, s0, s1, v0
 ; GFX12-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX12-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-NEXT:    ; return to shader part epilog
 ;
 ; GFX940-LABEL: s_fmaximum3_f32:
@@ -1307,6 +1308,7 @@ define amdgpu_ps i32 @s_fmaximum3_f16(half inreg %a, half inreg %b, half inreg %
 ; GFX12-NEXT:    v_and_b32_e32 v0, 0xffff, v0
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX12-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-NEXT:    ; return to shader part epilog
 ;
 ; GFX9-LABEL: s_fmaximum3_f16:
@@ -3005,6 +3007,7 @@ define amdgpu_ps <2 x i32> @s_fmaximum3_f64(double inreg %a, double inreg %b, do
 ; GFX12-NEXT:    v_readfirstlane_b32 s0, v0
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2)
 ; GFX12-NEXT:    v_readfirstlane_b32 s1, v1
+; GFX12-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-NEXT:    ; return to shader part epilog
 ;
 ; GFX9-LABEL: s_fmaximum3_f64:
diff --git a/llvm/test/CodeGen/AMDGPU/fminimum3.ll b/llvm/test/CodeGen/AMDGPU/fminimum3.ll
index 7a8a224c76a83..234a07849a911 100644
--- a/llvm/test/CodeGen/AMDGPU/fminimum3.ll
+++ b/llvm/test/CodeGen/AMDGPU/fminimum3.ll
@@ -80,6 +80,7 @@ define amdgpu_ps i32 @s_fminimum3_f32(float inreg %a, float inreg %b, float inre
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_minimum3_f32 v0, s0, s1, v0
 ; GFX12-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX12-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-NEXT:    ; return to shader part epilog
 ;
 ; GFX940-LABEL: s_fminimum3_f32:
@@ -1307,6 +1308,7 @@ define amdgpu_ps i32 @s_fminimum3_f16(half inreg %a, half inreg %b, half inreg %
 ; GFX12-NEXT:    v_and_b32_e32 v0, 0xffff, v0
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX12-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-NEXT:    ; return to shader part epilog
 ;
 ; GFX9-LABEL: s_fminimum3_f16:
@@ -3005,6 +3007,7 @@ define amdgpu_ps <2 x i32> @s_fminimum3_f64(double inreg %a, double inreg %b, do
 ; GFX12-NEXT:    v_readfirstlane_b32 s0, v0
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2)
 ; GFX12-NEXT:    v_readfirstlane_b32 s1, v1
+; GFX12-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-NEXT:    ; return to shader part epilog
 ;
 ; GFX9-LABEL: s_fminimum3_f64:
diff --git a/llvm/test/CodeGen/AMDGPU/fp-atomics-gfx1200.ll b/llvm/test/CodeGen/AMDGPU/fp-atomics-gfx1200.ll
index a0578756433ff..62f16fe2760ef 100644
--- a/llvm/test/CodeGen/AMDGPU/fp-atomics-gfx1200.ll
+++ b/llvm/test/CodeGen/AMDGPU/fp-atomics-gfx1200.ll
@@ -1,6 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc < %s -mtriple=amdgcn -mcpu=gfx1200 -global-isel=0 | FileCheck %s -check-prefix=GFX12-SDAG
-; RUN: llc < %s -mtriple=amdgcn -mcpu=gfx1200 -global-isel=1 | FileCheck %s -check-prefix=GFX12-GISEL
+; RUN: llc < %s -mtriple=amdgcn -mcpu=gfx1200 -global-isel=1 -global-isel-abort=2 | FileCheck %s -check-prefix=GFX12-GISEL
 
 declare <2 x half> @llvm.amdgcn.struct.buffer.atomic.fadd.v2f16(<2 x half>, <4 x i32>, i32, i32, i32, i32 immarg)
 declare <2 x bfloat> @llvm.amdgcn.struct.buffer.atomic.fadd.v2bf16(<2 x bfloat>, <4 x i32>, i32, i32, i32, i32 immarg)
@@ -15,7 +15,7 @@ define amdgpu_ps void @raw_buffer_atomic_add_v2f16_noret_offset(<2 x half> %val,
 ;
 ; GFX12-GISEL-LABEL: raw_buffer_atomic_add_v2f16_noret_offset:
 ; GFX12-GISEL:       ; %bb.0:
-; GFX12-GISEL-NEXT:    buffer_atomic_pk_add_bf16 v0, off, s[0:3], s4 offset:92
+; GFX12-GISEL-NEXT:    buffer_atomic_pk_add_f16 v0, off, s[0:3], s4 offset:92
 ; GFX12-GISEL-NEXT:    s_endpgm
   %ret = call <2 x half> @llvm.amdgcn.raw.buffer.atomic.fadd.v2f16(<2 x half> %val, <4 x i32> %rsrc, i32 92, i32 %soffset, i32 0)
   ret void
@@ -29,7 +29,7 @@ define amdgpu_ps void @raw_buffer_atomic_add_v2f16_noret(<2 x half> %val, <4 x i
 ;
 ; GFX12-GISEL-LABEL: raw_buffer_atomic_add_v2f16_noret:
 ; GFX12-GISEL:       ; %bb.0:
-; GFX12-GISEL-NEXT:    buffer_atomic_pk_add_bf16 v0, v1, s[0:3], s4 offen
+; GFX12-GISEL-NEXT:    buffer_atomic_pk_add_f16 v0, v1, s[0:3], s4 offen
 ; GFX12-GISEL-NEXT:    s_endpgm
   %ret = call <2 x half> @llvm.amdgcn.raw.buffer.atomic.fadd.v2f16(<2 x half> %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 0)
   ret void
@@ -44,7 +44,7 @@ define amdgpu_ps <2 x half> @raw_buffer_atomic_add_v2f16_ret_offset(<2 x half> %
 ;
 ; GFX12-GISEL-LABEL: raw_buffer_atomic_add_v2f16_ret_offset:
 ; GFX12-GISEL:       ; %bb.0:
-; GFX12-GISEL-NEXT:    buffer_atomic_pk_add_bf16 v0, off, s[0:3], s4 offset:92 th:TH_ATOMIC_RETURN
+; GFX12-GISEL-NEXT:    buffer_atomic_pk_add_f16 v0, off, s[0:3], s4 offset:92 th:TH_ATOMIC_RETURN
 ; GFX12-GISEL-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-GISEL-NEXT:    ; return to shader part epilog
   %ret = call <2 x half> @llvm.amdgcn.raw.buffer.atomic.fadd.v2f16(<2 x half> %val, <4 x i32> %rsrc, i32 92, i32 %soffset, i32 0)
@@ -60,7 +60,7 @@ define amdgpu_ps <2 x half> @raw_buffer_atomic_add_v2f16_ret(<2 x half> %val, <4
 ;
 ; GFX12-GISEL-LABEL: raw_buffer_atomic_add_v2f16_ret:
 ; GFX12-GISEL:       ; %bb.0:
-; GFX12-GISEL-NEXT:    buffer_atomic_pk_add_bf16 v0, v1, s[0:3], s4 offen th:TH_ATOMIC_RETURN
+; GFX12-GISEL-NEXT:    buffer_atomic_pk_add_f16 v0, v1, s[0:3], s4 offen th:TH_ATOMIC_RETURN
 ; GFX12-GISEL-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-GISEL-NEXT:    ; return to shader part epilog
   %ret = call <2 x half> @llvm.amdgcn.raw.buffer.atomic.fadd.v2f16(<2 x half> %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 0)
@@ -76,7 +76,7 @@ define amdgpu_ps float @struct_buffer_atomic_add_v2f16_ret(<2 x half> %val, <4 x
 ;
 ; GFX12-GISEL-LABEL: struct_buffer_atomic_add_v2f16_ret:
 ; GFX12-GISEL:       ; %bb.0:
-; GFX12-GISEL-NEXT:    buffer_atomic_pk_add_bf16 v0, v[1:2], s[0:3], s4 idxen offen th:TH_ATOMIC_RETURN
+; GFX12-GISEL-NEXT:    buffer_atomic_pk_add_f16 v0, v[1:2], s[0:3], s4 idxen offen th:TH_ATOMIC_RETURN
 ; GFX12-GISEL-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-GISEL-NEXT:    ; return to shader part epilog
   %orig = call <2 x half> @llvm.amdgcn.struct.buffer.atomic.fadd.v2f16(<2 x half> %val, <4 x i32> %rsrc, i32 %vindex, i32 %voffset, i32 %soffset, i32 0)
@@ -92,7 +92,7 @@ define amdgpu_ps void @struct_buffer_atomic_add_v2f16_noret(<2 x half> %val, <4
 ;
 ; GFX12-GISEL-LABEL: struct_buffer_atomic_add_v2f16_noret:
 ; GFX12-GISEL:       ; %bb.0:
-; GFX12-GISEL-NEXT:    buffer_atomic_pk_add_bf16 v0, v[1:2], s[0:3], s4 idxen offen
+; GFX12-GISEL-NEXT:    buffer_atomic_pk_add_f16 v0, v[1:2], s[0:3], s4 idxen offen
 ; GFX12-GISEL-NEXT:    s_endpgm
   %orig = call <2 x half> @llvm.amdgcn.struct.buffer.atomic.fadd.v2f16(<2 x half> %val, <4 x i32> %rsrc, i32 %vindex, i32 %voffset, i32 %soffset, i32 0)
   ret void
diff --git a/llvm/test/CodeGen/AMDGPU/fp-atomics-gfx950.ll b/llvm/test/CodeGen/AMDGPU/fp-atomics-gfx950.ll
index d8ea0ddf77b7a..5d9944add13a3 100644
--- a/llvm/test/CodeGen/AMDGPU/fp-atomics-gfx950.ll
+++ b/llvm/test/CodeGen/AMDGPU/fp-atomics-gfx950.ll
@@ -1,6 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
 ; RUN: llc < %s -mtriple=amdgcn -mcpu=gfx950 -global-isel=0 | FileCheck %s -check-prefix=GFX950-SDAG
-; RUN: llc < %s -mtriple=amdgcn -mcpu=gfx950 -global-isel=1 | FileCheck %s -check-prefix=GFX950-GISEL
+; RUN: llc < %s -mtriple=amdgcn -mcpu=gfx950 -global-isel=1 -global-isel-abort=2 | FileCheck %s -check-prefix=GFX950-GISEL
 
 declare <2 x bfloat> @llvm.amdgcn.struct.buffer.atomic.fadd.v2bf16(<2 x bfloat>, <4 x i32>, i32, i32, i32, i32 immarg)
 declare <2 x bfloat> @llvm.amdgcn.raw.buffer.atomic.fadd.v2bf16(<2 x bfloat> %val, <4 x i32> %rsrc, i32, i32, i32)
@@ -20,9 +20,9 @@ define amdgpu_ps float @struct_buffer_atomic_add_v2bf16_ret(<2 x bfloat> %val, <
 ;
 ; GFX950-GISEL-LABEL: struct_buffer_atomic_add_v2bf16_ret:
 ; GFX950-GISEL:       ; %bb.0:
-; GFX950-GISEL-NEXT:    v_mov_b32_e32 v4, v1
-; GFX950-GISEL-NEXT:    v_mov_b32_e32 v5, v2
-; GFX950-GISEL-NEXT:    buffer_atomic_pk_add_bf16 v0, v[4:5], s[0:3], s4 idxen offen sc0
+; GFX950-GISEL-NEXT:    v_mov_b32_e32 v3, v2
+; GFX950-GISEL-NEXT:    v_mov_b32_e32 v2, v1
+; GFX950-GISEL-NEXT:    buffer_atomic_pk_add_bf16 v0, v[2:3], s[0:3], s4 idxen offen sc0
 ; GFX950-GISEL-NEXT:    v_mov_b64_e32 v[2:3], 0
 ; GFX950-GISEL-NEXT:    s_waitcnt vmcnt(0)
 ; GFX950-GISEL-NEXT:    flat_store_dword v[2:3], v0
@@ -44,9 +44,9 @@ define amdgpu_ps void @struct_buffer_atomic_add_v2bf16_noret(<2 x bfloat> %val,
 ;
 ; GFX950-GISEL-LABEL: struct_buffer_atomic_add_v2bf16_noret:
 ; GFX950-GISEL:       ; %bb.0:
-; GFX950-GISEL-NEXT:    v_mov_b32_e32 v4, v1
-; GFX950-GISEL-NEXT:    v_mov_b32_e32 v5, v2
-; GFX950-GISEL-NEXT:    buffer_atomic_pk_add_bf16 v0, v[4:5], s[0:3], s4 idxen offen
+; GFX950-GISEL-NEXT:    v_mov_b32_e32 v3, v2
+; GFX950-GISEL-NEXT:    v_mov_b32_e32 v2, v1
+; GFX950-GISEL-NEXT:    buffer_atomic_pk_add_bf16 v0, v[2:3], s[0:3], s4 idxen offen
 ; GFX950-GISEL-NEXT:    s_endpgm
   %orig = call <2 x bfloat> @llvm.amdgcn.struct.buffer.atomic.fadd.v2bf16(<2 x bfloat> %val, <4 x i32> %rsrc, i32 %vindex, i32 %voffset, i32 %soffset, i32 0)
   ret void
diff --git a/llvm/test/CodeGen/AMDGPU/fp-classify.ll b/llvm/test/CodeGen/AMDGPU/fp-classify.ll
index e7c425a2d2752..cc11e256d5544 100644
--- a/llvm/test/CodeGen/AMDGPU/fp-classify.ll
+++ b/llvm/test/CodeGen/AMDGPU/fp-classify.ll
@@ -1,7 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 2
 ; RUN: llc -mtriple=amdgcn -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=SI %s
 ; RUN: llc -mtriple=amdgcn -mcpu=tonga -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=VI %s
-; RUN: llc -mtriple=amdgcn -mcpu=gfx1100 -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GFX11 %s
+; RUN: llc -mtriple=amdgcn -mcpu=gfx1100 -mattr=+real-true16 -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GFX11,GFX11-TRUE16 %s
+; RUN: llc -mtriple=amdgcn -mcpu=gfx1100 -mattr=-real-true16 -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GFX11,GFX11-FAKE16 %s
 
 declare float @llvm.fabs.f32(float) #1
 declare double @llvm.fabs.f64(double) #1
@@ -619,18 +620,32 @@ define amdgpu_kernel void @test_isinf_pattern_f16(ptr addrspace(1) nocapture %ou
 ; VI-NEXT:    flat_store_dword v[0:1], v2
 ; VI-NEXT:    s_endpgm
 ;
-; GFX11-LABEL: test_isinf_pattern_f16:
-; GFX11:       ; %bb.0:
-; GFX11-NEXT:    s_clause 0x1
-; GFX11-NEXT:    s_load_b32 s2, s[4:5], 0x2c
-; GFX11-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
-; GFX11-NEXT:    v_mov_b32_e32 v0, 0
-; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX11-NEXT:    v_cmp_class_f16_e64 s2, s2, 0x204
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT:    v_cndmask_b32_e64 v1, 0, 1, s2
-; GFX11-NEXT:    global_store_b32 v0, v1, s[0:1]
-; GFX11-NEXT:    s_endpgm
+; GFX11-TRUE16-LABEL: test_isinf_pattern_f16:
+; GFX11-TRUE16:       ; %bb.0:
+; GFX11-TRUE16-NEXT:    s_clause 0x1
+; GFX11-TRUE16-NEXT:    s_load_b32 s2, s[4:5], 0x2c
+; GFX11-TRUE16-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-TRUE16-NEXT:    v_mov_b32_e32 v1, 0
+; GFX11-TRUE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v0.l, s2
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_cmp_class_f16_e64 s2, v0.l, 0x204
+; GFX11-TRUE16-NEXT:    v_cndmask_b32_e64 v0, 0, 1, s2
+; GFX11-TRUE16-NEXT:    global_store_b32 v1, v0, s[0:1]
+; GFX11-TRUE16-NEXT:    s_endpgm
+;
+; GFX11-FAKE16-LABEL: test_isinf_pattern_f16:
+; GFX11-FAKE16:       ; %bb.0:
+; GFX11-FAKE16-NEXT:    s_clause 0x1
+; GFX11-FAKE16-NEXT:    s_load_b32 s2, s[4:5], 0x2c
+; GFX11-FAKE16-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-FAKE16-NEXT:    v_mov_b32_e32 v0, 0
+; GFX11-FAKE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-FAKE16-NEXT:    v_cmp_class_f16_e64 s2, s2, 0x204
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e64 v1, 0, 1, s2
+; GFX11-FAKE16-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX11-FAKE16-NEXT:    s_endpgm
   %fabs = tail call half @llvm.fabs.f16(half %x) #1
   %cmp = fcmp oeq half %fabs, 0xH7C00
   %ext = zext i1 %cmp to i32
@@ -669,18 +684,32 @@ define amdgpu_kernel void @test_isfinite_pattern_0_f16(ptr addrspace(1) nocaptur
 ; VI-NEXT:    flat_store_dword v[0:1], v2
 ; VI-NEXT:    s_endpgm
 ;
-; GFX11-LABEL: test_isfinite_pattern_0_f16:
-; GFX11:       ; %bb.0:
-; GFX11-NEXT:    s_clause 0x1
-; GFX11-NEXT:    s_load_b32 s2, s[4:5], 0x2c
-; GFX11-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
-; GFX11-NEXT:    v_mov_b32_e32 v0, 0
-; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX11-NEXT:    v_cmp_class_f16_e64 s2, s2, 0x1f8
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT:    v_cndmask_b32_e64 v1, 0, 1, s2
-; GFX11-NEXT:    global_store_b32 v0, v1, s[0:1]
-; GFX11-NEXT:    s_endpgm
+; GFX11-TRUE16-LABEL: test_isfinite_pattern_0_f16:
+; GFX11-TRUE16:       ; %bb.0:
+; GFX11-TRUE16-NEXT:    s_clause 0x1
+; GFX11-TRUE16-NEXT:    s_load_b32 s2, s[4:5], 0x2c
+; GFX11-TRUE16-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-TRUE16-NEXT:    v_mov_b32_e32 v1, 0
+; GFX11-TRUE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v0.l, s2
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_cmp_class_f16_e64 s2, v0.l, 0x1f8
+; GFX11-TRUE16-NEXT:    v_cndmask_b32_e64 v0, 0, 1, s2
+; GFX11-TRUE16-NEXT:    global_store_b32 v1, v0, s[0:1]
+; GFX11-TRUE16-NEXT:    s_endpgm
+;
+; GFX11-FAKE16-LABEL: test_isfinite_pattern_0_f16:
+; GFX11-FAKE16:       ; %bb.0:
+; GFX11-FAKE16-NEXT:    s_clause 0x1
+; GFX11-FAKE16-NEXT:    s_load_b32 s2, s[4:5], 0x2c
+; GFX11-FAKE16-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-FAKE16-NEXT:    v_mov_b32_e32 v0, 0
+; GFX11-FAKE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-FAKE16-NEXT:    v_cmp_class_f16_e64 s2, s2, 0x1f8
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e64 v1, 0, 1, s2
+; GFX11-FAKE16-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX11-FAKE16-NEXT:    s_endpgm
   %ord = fcmp ord half %x, 0.0
   %x.fabs = tail call half @llvm.fabs.f16(half %x) #1
   %ninf = fcmp une half %x.fabs, 0xH7C00
@@ -718,18 +747,32 @@ define amdgpu_kernel void @test_isfinite_pattern_4_f16(ptr addrspace(1) nocaptur
 ; VI-NEXT:    flat_store_dword v[0:1], v2
 ; VI-NEXT:    s_endpgm
 ;
-; GFX11-LABEL: test_isfinite_pattern_4_f16:
-; GFX11:       ; %bb.0:
-; GFX11-NEXT:    s_clause 0x1
-; GFX11-NEXT:    s_load_b32 s2, s[4:5], 0x2c
-; GFX11-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
-; GFX11-NEXT:    v_mov_b32_e32 v0, 0
-; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX11-NEXT:    v_cmp_class_f16_e64 s2, s2, 0x1f8
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT:    v_cndmask_b32_e64 v1, 0, 1, s2
-; GFX11-NEXT:    global_store_b32 v0, v1, s[0:1]
-; GFX11-NEXT:    s_endpgm
+; GFX11-TRUE16-LABEL: test_isfinite_pattern_4_f16:
+; GFX11-TRUE16:       ; %bb.0:
+; GFX11-TRUE16-NEXT:    s_clause 0x1
+; GFX11-TRUE16-NEXT:    s_load_b32 s2, s[4:5], 0x2c
+; GFX11-TRUE16-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-TRUE16-NEXT:    v_mov_b32_e32 v1, 0
+; GFX11-TRUE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v0.l, s2
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_cmp_class_f16_e64 s2, v0.l, 0x1f8
+; GFX11-TRUE16-NEXT:    v_cndmask_b32_e64 v0, 0, 1, s2
+; GFX11-TRUE16-NEXT:    global_store_b32 v1, v0, s[0:1]
+; GFX11-TRUE16-NEXT:    s_endpgm
+;
+; GFX11-FAKE16-LABEL: test_isfinite_pattern_4_f16:
+; GFX11-FAKE16:       ; %bb.0:
+; GFX11-FAKE16-NEXT:    s_clause 0x1
+; GFX11-FAKE16-NEXT:    s_load_b32 s2, s[4:5], 0x2c
+; GFX11-FAKE16-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-FAKE16-NEXT:    v_mov_b32_e32 v0, 0
+; GFX11-FAKE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-FAKE16-NEXT:    v_cmp_class_f16_e64 s2, s2, 0x1f8
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e64 v1, 0, 1, s2
+; GFX11-FAKE16-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX11-FAKE16-NEXT:    s_endpgm
   %ord = fcmp ord half %x, 0.0
   %x.fabs = tail call half @llvm.fabs.f16(half %x) #1
   %ninf = fcmp one half %x.fabs, 0xH7C00
diff --git a/llvm/test/CodeGen/AMDGPU/fptoi.i128.ll b/llvm/test/CodeGen/AMDGPU/fptoi.i128.ll
index 3a7f3e41002d2..3e6b812c12d7f 100644
--- a/llvm/test/CodeGen/AMDGPU/fptoi.i128.ll
+++ b/llvm/test/CodeGen/AMDGPU/fptoi.i128.ll
@@ -61,32 +61,31 @@ define i128 @fptosi_f64_to_i128(double %x) {
 ; SDAG-NEXT:    v_cndmask_b32_e64 v2, v2, v0, s[4:5]
 ; SDAG-NEXT:    v_lshlrev_b64 v[0:1], v7, v[4:5]
 ; SDAG-NEXT:    v_cndmask_b32_e64 v2, 0, v2, s[6:7]
-; SDAG-NEXT:    v_cndmask_b32_e64 v11, 0, v0, s[4:5]
-; SDAG-NEXT:    v_cndmask_b32_e64 v7, 0, v1, s[4:5]
-; SDAG-NEXT:    v_mad_u64_u32 v[0:1], s[4:5], v11, v10, 0
+; SDAG-NEXT:    v_cndmask_b32_e64 v12, 0, v0, s[4:5]
+; SDAG-NEXT:    v_cndmask_b32_e64 v11, 0, v1, s[4:5]
+; SDAG-NEXT:    v_mad_u64_u32 v[0:1], s[4:5], v12, v10, 0
 ; SDAG-NEXT:    v_mov_b32_e32 v3, 0
-; SDAG-NEXT:    v_mul_lo_u32 v12, v8, v2
+; SDAG-NEXT:    v_mul_lo_u32 v13, v8, v2
 ; SDAG-NEXT:    v_mad_u64_u32 v[4:5], s[4:5], v10, v2, 0
 ; SDAG-NEXT:    v_mov_b32_e32 v2, v1
-; SDAG-NEXT:    v_mul_lo_u32 v6, v10, v6
-; SDAG-NEXT:    v_mad_u64_u32 v[1:2], s[4:5], v7, v10, v[2:3]
-; SDAG-NEXT:    v_mul_lo_u32 v10, v9, v7
-; SDAG-NEXT:    v_add3_u32 v5, v5, v6, v12
-; SDAG-NEXT:    v_mov_b32_e32 v6, v2
-; SDAG-NEXT:    v_mov_b32_e32 v2, v3
-; SDAG-NEXT:    v_mad_u64_u32 v[1:2], s[4:5], v11, v8, v[1:2]
-; SDAG-NEXT:    v_mad_u64_u32 v[3:4], s[4:5], v9, v11, v[4:5]
-; SDAG-NEXT:    v_add_co_u32_e64 v5, s[4:5], v6, v2
-; SDAG-NEXT:    v_addc_co_u32_e64 v6, s[4:5], 0, 0, s[4:5]
-; SDAG-NEXT:    v_mul_lo_u32 v9, v9, v11
-; SDAG-NEXT:    v_mad_u64_u32 v[5:6], s[4:5], v7, v8, v[5:6]
+; SDAG-NEXT:    v_mul_lo_u32 v14, v10, v6
+; SDAG-NEXT:    v_mad_u64_u32 v[6:7], s[4:5], v11, v10, v[2:3]
+; SDAG-NEXT:    ; implicit-def: $vgpr10
+; SDAG-NEXT:    v_add3_u32 v5, v5, v14, v13
+; SDAG-NEXT:    v_mov_b32_e32 v2, v6
+; SDAG-NEXT:    v_mad_u64_u32 v[1:2], s[4:5], v12, v8, v[2:3]
+; SDAG-NEXT:    v_mad_u64_u32 v[4:5], s[4:5], v9, v12, v[4:5]
+; SDAG-NEXT:    v_add_co_u32_e64 v2, s[4:5], v7, v2
+; SDAG-NEXT:    v_addc_co_u32_e64 v3, s[4:5], 0, 0, s[4:5]
+; SDAG-NEXT:    v_mul_lo_u32 v6, v9, v11
+; SDAG-NEXT:    v_mul_lo_u32 v9, v9, v12
+; SDAG-NEXT:    v_mad_u64_u32 v[2:3], s[4:5], v11, v8, v[2:3]
 ; SDAG-NEXT:    ; implicit-def: $vgpr8
-; SDAG-NEXT:    v_add3_u32 v4, v9, v4, v10
-; SDAG-NEXT:    v_add_co_u32_e64 v2, s[4:5], v5, v3
-; SDAG-NEXT:    v_addc_co_u32_e64 v3, s[4:5], v6, v4, s[4:5]
+; SDAG-NEXT:    v_add3_u32 v5, v9, v5, v6
+; SDAG-NEXT:    v_add_co_u32_e64 v2, s[4:5], v2, v4
+; SDAG-NEXT:    v_addc_co_u32_e64 v3, s[4:5], v3, v5, s[4:5]
 ; SDAG-NEXT:    ; implicit-def: $vgpr6_vgpr7
 ; SDAG-NEXT:    ; implicit-def: $vgpr4_vgpr5
-; SDAG-NEXT:    ; implicit-def: $vgpr10
 ; SDAG-NEXT:    ; implicit-def: $vgpr9
 ; SDAG-NEXT:  .LBB0_4: ; %Flow
 ; SDAG-NEXT:    s_andn2_saveexec_b64 s[12:13], s[12:13]
@@ -103,10 +102,9 @@ define i128 @fptosi_f64_to_i128(double %x) {
 ; SDAG-NEXT:    v_mad_u64_u32 v[0:1], s[4:5], v6, v10, 0
 ; SDAG-NEXT:    v_mov_b32_e32 v2, 0
 ; SDAG-NEXT:    v_mad_u64_u32 v[3:4], s[4:5], v5, v10, v[1:2]
-; SDAG-NEXT:    v_mov_b32_e32 v7, v4
-; SDAG-NEXT:    v_mov_b32_e32 v4, v2
-; SDAG-NEXT:    v_mad_u64_u32 v[1:2], s[4:5], v6, v8, v[3:4]
-; SDAG-NEXT:    v_add_co_u32_e64 v2, s[4:5], v7, v2
+; SDAG-NEXT:    v_mov_b32_e32 v1, v3
+; SDAG-NEXT:    v_mad_u64_u32 v[1:2], s[4:5], v6, v8, v[1:2]
+; SDAG-NEXT:    v_add_co_u32_e64 v2, s[4:5], v4, v2
 ; SDAG-NEXT:    v_addc_co_u32_e64 v3, s[4:5], 0, 0, s[4:5]
 ; SDAG-NEXT:    v_mad_u64_u32 v[2:3], s[4:5], v5, v8, v[2:3]
 ; SDAG-NEXT:    v_mad_u64_u32 v[2:3], s[4:5], v9, v6, v[2:3]
@@ -429,32 +427,31 @@ define i128 @fptoui_f64_to_i128(double %x) {
 ; SDAG-NEXT:    v_cndmask_b32_e64 v2, v2, v0, s[4:5]
 ; SDAG-NEXT:    v_lshlrev_b64 v[0:1], v7, v[4:5]
 ; SDAG-NEXT:    v_cndmask_b32_e64 v2, 0, v2, s[6:7]
-; SDAG-NEXT:    v_cndmask_b32_e64 v11, 0, v0, s[4:5]
-; SDAG-NEXT:    v_cndmask_b32_e64 v7, 0, v1, s[4:5]
-; SDAG-NEXT:    v_mad_u64_u32 v[0:1], s[4:5], v11, v10, 0
+; SDAG-NEXT:    v_cndmask_b32_e64 v12, 0, v0, s[4:5]
+; SDAG-NEXT:    v_cndmask_b32_e64 v11, 0, v1, s[4:5]
+; SDAG-NEXT:    v_mad_u64_u32 v[0:1], s[4:5], v12, v10, 0
 ; SDAG-NEXT:    v_mov_b32_e32 v3, 0
-; SDAG-NEXT:    v_mul_lo_u32 v12, v8, v2
+; SDAG-NEXT:    v_mul_lo_u32 v13, v8, v2
 ; SDAG-NEXT:    v_mad_u64_u32 v[4:5], s[4:5], v10, v2, 0
 ; SDAG-NEXT:    v_mov_b32_e32 v2, v1
-; SDAG-NEXT:    v_mul_lo_u32 v6, v10, v6
-; SDAG-NEXT:    v_mad_u64_u32 v[1:2], s[4:5], v7, v10, v[2:3]
-; SDAG-NEXT:    v_mul_lo_u32 v10, v9, v7
-; SDAG-NEXT:    v_add3_u32 v5, v5, v6, v12
-; SDAG-NEXT:    v_mov_b32_e32 v6, v2
-; SDAG-NEXT:    v_mov_b32_e32 v2, v3
-; SDAG-NEXT:    v_mad_u64_u32 v[1:2], s[4:5], v11, v8, v[1:2]
-; SDAG-NEXT:    v_mad_u64_u32 v[3:4], s[4:5], v9, v11, v[4:5]
-; SDAG-NEXT:    v_add_co_u32_e64 v5, s[4:5], v6, v2
-; SDAG-NEXT:    v_addc_co_u32_e64 v6, s[4:5], 0, 0, s[4:5]
-; SDAG-NEXT:    v_mul_lo_u32 v9, v9, v11
-; SDAG-NEXT:    v_mad_u64_u32 v[5:6], s[4:5], v7, v8, v[5:6]
+; SDAG-NEXT:    v_mul_lo_u32 v14, v10, v6
+; SDAG-NEXT:    v_mad_u64_u32 v[6:7], s[4:5], v11, v10, v[2:3]
+; SDAG-NEXT:    ; implicit-def: $vgpr10
+; SDAG-NEXT:    v_add3_u32 v5, v5, v14, v13
+; SDAG-NEXT:    v_mov_b32_e32 v2, v6
+; SDAG-NEXT:    v_mad_u64_u32 v[1:2], s[4:5], v12, v8, v[2:3]
+; SDAG-NEXT:    v_mad_u64_u32 v[4:5], s[4:5], v9, v12, v[4:5]
+; SDAG-NEXT:    v_add_co_u32_e64 v2, s[4:5], v7, v2
+; SDAG-NEXT:    v_addc_co_u32_e64 v3, s[4:5], 0, 0, s[4:5]
+; SDAG-NEXT:    v_mul_lo_u32 v6, v9, v11
+; SDAG-NEXT:    v_mul_lo_u32 v9, v9, v12
+; SDAG-NEXT:    v_mad_u64_u32 v[2:3], s[4:5], v11, v8, v[2:3]
 ; SDAG-NEXT:    ; implicit-def: $vgpr8
-; SDAG-NEXT:    v_add3_u32 v4, v9, v4, v10
-; SDAG-NEXT:    v_add_co_u32_e64 v2, s[4:5], v5, v3
-; SDAG-NEXT:    v_addc_co_u32_e64 v3, s[4:5], v6, v4, s[4:5]
+; SDAG-NEXT:    v_add3_u32 v5, v9, v5, v6
+; SDAG-NEXT:    v_add_co_u32_e64 v2, s[4:5], v2, v4
+; SDAG-NEXT:    v_addc_co_u32_e64 v3, s[4:5], v3, v5, s[4:5]
 ; SDAG-NEXT:    ; implicit-def: $vgpr6_vgpr7
 ; SDAG-NEXT:    ; implicit-def: $vgpr4_vgpr5
-; SDAG-NEXT:    ; implicit-def: $vgpr10
 ; SDAG-NEXT:    ; implicit-def: $vgpr9
 ; SDAG-NEXT:  .LBB1_4: ; %Flow
 ; SDAG-NEXT:    s_andn2_saveexec_b64 s[12:13], s[12:13]
@@ -471,10 +468,9 @@ define i128 @fptoui_f64_to_i128(double %x) {
 ; SDAG-NEXT:    v_mad_u64_u32 v[0:1], s[4:5], v6, v10, 0
 ; SDAG-NEXT:    v_mov_b32_e32 v2, 0
 ; SDAG-NEXT:    v_mad_u64_u32 v[3:4], s[4:5], v5, v10, v[1:2]
-; SDAG-NEXT:    v_mov_b32_e32 v7, v4
-; SDAG-NEXT:    v_mov_b32_e32 v4, v2
-; SDAG-NEXT:    v_mad_u64_u32 v[1:2], s[4:5], v6, v8, v[3:4]
-; SDAG-NEXT:    v_add_co_u32_e64 v2, s[4:5], v7, v2
+; SDAG-NEXT:    v_mov_b32_e32 v1, v3
+; SDAG-NEXT:    v_mad_u64_u32 v[1:2], s[4:5], v6, v8, v[1:2]
+; SDAG-NEXT:    v_add_co_u32_e64 v2, s[4:5], v4, v2
 ; SDAG-NEXT:    v_addc_co_u32_e64 v3, s[4:5], 0, 0, s[4:5]
 ; SDAG-NEXT:    v_mad_u64_u32 v[2:3], s[4:5], v5, v8, v[2:3]
 ; SDAG-NEXT:    v_mad_u64_u32 v[2:3], s[4:5], v9, v6, v[2:3]
@@ -743,22 +739,22 @@ define i128 @fptosi_f32_to_i128(float %x) {
 ; SDAG:       ; %bb.0: ; %fp-to-i-entry
 ; SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; SDAG-NEXT:    v_mov_b32_e32 v4, v0
-; SDAG-NEXT:    v_bfe_u32 v5, v4, 23, 8
+; SDAG-NEXT:    v_bfe_u32 v6, v4, 23, 8
 ; SDAG-NEXT:    s_movk_i32 s4, 0x7e
 ; SDAG-NEXT:    v_mov_b32_e32 v0, 0
 ; SDAG-NEXT:    v_mov_b32_e32 v2, 0
-; SDAG-NEXT:    v_mov_b32_e32 v6, 0
+; SDAG-NEXT:    v_mov_b32_e32 v7, 0
 ; SDAG-NEXT:    v_mov_b32_e32 v1, 0
 ; SDAG-NEXT:    v_mov_b32_e32 v3, 0
-; SDAG-NEXT:    v_cmp_lt_u32_e32 vcc, s4, v5
+; SDAG-NEXT:    v_cmp_lt_u32_e32 vcc, s4, v6
 ; SDAG-NEXT:    s_and_saveexec_b64 s[8:9], vcc
 ; SDAG-NEXT:    s_cbranch_execz .LBB2_10
 ; SDAG-NEXT:  ; %bb.1: ; %fp-to-i-if-end
-; SDAG-NEXT:    v_add_co_u32_e32 v0, vcc, 0xffffff01, v5
-; SDAG-NEXT:    v_addc_co_u32_e32 v1, vcc, -1, v6, vcc
-; SDAG-NEXT:    v_addc_co_u32_e32 v2, vcc, -1, v6, vcc
+; SDAG-NEXT:    v_add_co_u32_e32 v0, vcc, 0xffffff01, v6
+; SDAG-NEXT:    v_addc_co_u32_e32 v1, vcc, -1, v7, vcc
+; SDAG-NEXT:    v_addc_co_u32_e32 v2, vcc, -1, v7, vcc
 ; SDAG-NEXT:    s_movk_i32 s6, 0xff7f
-; SDAG-NEXT:    v_addc_co_u32_e32 v3, vcc, -1, v6, vcc
+; SDAG-NEXT:    v_addc_co_u32_e32 v3, vcc, -1, v7, vcc
 ; SDAG-NEXT:    s_mov_b32 s7, -1
 ; SDAG-NEXT:    v_cmp_eq_u64_e64 s[4:5], -1, v[2:3]
 ; SDAG-NEXT:    v_cmp_lt_u64_e64 s[6:7], s[6:7], v[0:1]
@@ -774,66 +770,65 @@ define i128 @fptosi_f32_to_i128(float %x) {
 ; SDAG-NEXT:    v_add_co_u32_e64 v9, s[4:5], -1, v0
 ; SDAG-NEXT:    s_mov_b64 s[4:5], 0x95
 ; SDAG-NEXT:    v_and_b32_e32 v0, 0x7fffff, v4
-; SDAG-NEXT:    v_cmp_lt_u64_e64 s[4:5], s[4:5], v[5:6]
-; SDAG-NEXT:    v_mov_b32_e32 v7, 0
+; SDAG-NEXT:    v_cmp_lt_u64_e64 s[4:5], s[4:5], v[6:7]
+; SDAG-NEXT:    v_mov_b32_e32 v5, 0
 ; SDAG-NEXT:    v_cndmask_b32_e64 v8, -1, 0, vcc
 ; SDAG-NEXT:    v_cndmask_b32_e64 v10, -1, 1, vcc
-; SDAG-NEXT:    v_or_b32_e32 v6, 0x800000, v0
+; SDAG-NEXT:    v_or_b32_e32 v4, 0x800000, v0
 ; SDAG-NEXT:    ; implicit-def: $vgpr0_vgpr1
 ; SDAG-NEXT:    ; implicit-def: $vgpr2_vgpr3
 ; SDAG-NEXT:    s_and_saveexec_b64 s[6:7], s[4:5]
 ; SDAG-NEXT:    s_xor_b64 s[12:13], exec, s[6:7]
 ; SDAG-NEXT:    s_cbranch_execz .LBB2_4
 ; SDAG-NEXT:  ; %bb.3: ; %fp-to-i-if-else
-; SDAG-NEXT:    v_sub_u32_e32 v0, 0xd6, v5
-; SDAG-NEXT:    v_add_u32_e32 v2, 0xffffff2a, v5
-; SDAG-NEXT:    v_add_u32_e32 v4, 0xffffff6a, v5
-; SDAG-NEXT:    v_lshrrev_b64 v[0:1], v0, v[6:7]
-; SDAG-NEXT:    v_lshlrev_b64 v[2:3], v2, v[6:7]
-; SDAG-NEXT:    v_cmp_gt_u32_e64 s[4:5], 64, v4
+; SDAG-NEXT:    v_sub_u32_e32 v0, 0xd6, v6
+; SDAG-NEXT:    v_add_u32_e32 v2, 0xffffff2a, v6
+; SDAG-NEXT:    v_add_u32_e32 v7, 0xffffff6a, v6
+; SDAG-NEXT:    v_lshrrev_b64 v[0:1], v0, v[4:5]
+; SDAG-NEXT:    v_lshlrev_b64 v[2:3], v2, v[4:5]
+; SDAG-NEXT:    v_cmp_gt_u32_e64 s[4:5], 64, v7
 ; SDAG-NEXT:    v_cndmask_b32_e64 v1, v3, v1, s[4:5]
-; SDAG-NEXT:    v_cmp_ne_u32_e64 s[6:7], 0, v4
+; SDAG-NEXT:    v_cmp_ne_u32_e64 s[6:7], 0, v7
 ; SDAG-NEXT:    v_cndmask_b32_e64 v3, 0, v1, s[6:7]
 ; SDAG-NEXT:    v_cndmask_b32_e64 v2, v2, v0, s[4:5]
-; SDAG-NEXT:    v_lshlrev_b64 v[0:1], v4, v[6:7]
+; SDAG-NEXT:    v_lshlrev_b64 v[0:1], v7, v[4:5]
 ; SDAG-NEXT:    v_cndmask_b32_e64 v2, 0, v2, s[6:7]
 ; SDAG-NEXT:    v_cndmask_b32_e64 v12, 0, v0, s[4:5]
 ; SDAG-NEXT:    v_cndmask_b32_e64 v11, 0, v1, s[4:5]
 ; SDAG-NEXT:    v_mad_u64_u32 v[0:1], s[4:5], v12, v10, 0
 ; SDAG-NEXT:    v_mul_lo_u32 v13, v8, v2
 ; SDAG-NEXT:    v_mul_lo_u32 v14, v10, v3
-; SDAG-NEXT:    v_mov_b32_e32 v6, v1
-; SDAG-NEXT:    v_mad_u64_u32 v[4:5], s[4:5], v11, v10, v[6:7]
+; SDAG-NEXT:    v_mov_b32_e32 v4, v1
+; SDAG-NEXT:    v_mad_u64_u32 v[6:7], s[4:5], v11, v10, v[4:5]
 ; SDAG-NEXT:    v_mad_u64_u32 v[2:3], s[4:5], v10, v2, 0
-; SDAG-NEXT:    v_mov_b32_e32 v6, v5
-; SDAG-NEXT:    v_mov_b32_e32 v5, v7
+; SDAG-NEXT:    v_mov_b32_e32 v4, v6
 ; SDAG-NEXT:    v_mad_u64_u32 v[4:5], s[4:5], v12, v8, v[4:5]
 ; SDAG-NEXT:    v_add3_u32 v3, v3, v14, v13
 ; SDAG-NEXT:    v_mad_u64_u32 v[1:2], s[4:5], v9, v12, v[2:3]
-; SDAG-NEXT:    v_add_co_u32_e64 v5, s[4:5], v6, v5
+; SDAG-NEXT:    v_add_co_u32_e64 v5, s[4:5], v7, v5
 ; SDAG-NEXT:    v_addc_co_u32_e64 v6, s[4:5], 0, 0, s[4:5]
 ; SDAG-NEXT:    v_mul_lo_u32 v3, v9, v11
-; SDAG-NEXT:    v_mul_lo_u32 v7, v9, v12
+; SDAG-NEXT:    v_mul_lo_u32 v9, v9, v12
 ; SDAG-NEXT:    v_mad_u64_u32 v[5:6], s[4:5], v11, v8, v[5:6]
 ; SDAG-NEXT:    ; implicit-def: $vgpr10
 ; SDAG-NEXT:    ; implicit-def: $vgpr8
-; SDAG-NEXT:    ; implicit-def: $vgpr9
-; SDAG-NEXT:    v_add3_u32 v3, v7, v2, v3
+; SDAG-NEXT:    v_add3_u32 v3, v9, v2, v3
 ; SDAG-NEXT:    v_add_co_u32_e64 v2, s[4:5], v5, v1
 ; SDAG-NEXT:    v_addc_co_u32_e64 v3, s[4:5], v6, v3, s[4:5]
-; SDAG-NEXT:    ; implicit-def: $vgpr5_vgpr6
 ; SDAG-NEXT:    v_mov_b32_e32 v1, v4
 ; SDAG-NEXT:    ; implicit-def: $vgpr6_vgpr7
+; SDAG-NEXT:    ; implicit-def: $vgpr4_vgpr5
+; SDAG-NEXT:    ; implicit-def: $vgpr9
 ; SDAG-NEXT:  .LBB2_4: ; %Flow
 ; SDAG-NEXT:    s_andn2_saveexec_b64 s[6:7], s[12:13]
 ; SDAG-NEXT:    s_cbranch_execz .LBB2_6
 ; SDAG-NEXT:  ; %bb.5: ; %fp-to-i-if-then12
-; SDAG-NEXT:    v_sub_u32_e32 v2, 0x96, v5
-; SDAG-NEXT:    v_lshrrev_b64 v[0:1], v2, v[6:7]
+; SDAG-NEXT:    v_sub_u32_e32 v2, 0x96, v6
+; SDAG-NEXT:    v_lshrrev_b64 v[0:1], v2, v[4:5]
 ; SDAG-NEXT:    v_cmp_gt_u32_e64 s[4:5], 64, v2
 ; SDAG-NEXT:    v_cndmask_b32_e64 v0, 0, v0, s[4:5]
 ; SDAG-NEXT:    v_cmp_eq_u32_e64 s[4:5], 0, v2
-; SDAG-NEXT:    v_cndmask_b32_e64 v3, v0, v6, s[4:5]
+; SDAG-NEXT:    v_cndmask_b32_e64 v3, v0, v4, s[4:5]
 ; SDAG-NEXT:    v_mad_u64_u32 v[0:1], s[4:5], v3, v10, 0
 ; SDAG-NEXT:    v_mov_b32_e32 v2, 0
 ; SDAG-NEXT:    v_mad_u64_u32 v[4:5], s[4:5], v3, v8, v[1:2]
@@ -949,8 +944,8 @@ define i128 @fptosi_f32_to_i128(float %x) {
 ; GISEL-NEXT:    v_and_b32_e32 v0, 0xffff, v0
 ; GISEL-NEXT:    v_and_b32_e32 v1, 0xffff, v1
 ; GISEL-NEXT:    v_lshlrev_b32_e32 v2, 16, v0
-; GISEL-NEXT:    v_lshl_or_b32 v10, v0, 16, v0
-; GISEL-NEXT:    v_or3_b32 v9, v1, v2, 1
+; GISEL-NEXT:    v_lshl_or_b32 v9, v0, 16, v0
+; GISEL-NEXT:    v_or3_b32 v10, v1, v2, 1
 ; GISEL-NEXT:    v_or3_b32 v8, v0, v2, 0
 ; GISEL-NEXT:    v_mov_b32_e32 v0, 0x96
 ; GISEL-NEXT:    v_mov_b32_e32 v1, 0
@@ -968,7 +963,7 @@ define i128 @fptosi_f32_to_i128(float %x) {
 ; GISEL-NEXT:    v_cmp_gt_u32_e32 vcc, 64, v7
 ; GISEL-NEXT:    v_cndmask_b32_e32 v11, 0, v0, vcc
 ; GISEL-NEXT:    v_cndmask_b32_e32 v12, 0, v1, vcc
-; GISEL-NEXT:    v_mad_u64_u32 v[0:1], s[6:7], v11, v10, 0
+; GISEL-NEXT:    v_mad_u64_u32 v[0:1], s[6:7], v11, v9, 0
 ; GISEL-NEXT:    v_add_u32_e32 v6, 0xffffff2a, v6
 ; GISEL-NEXT:    v_sub_u32_e32 v2, 64, v7
 ; GISEL-NEXT:    v_lshrrev_b64 v[2:3], v2, v[4:5]
@@ -977,24 +972,24 @@ define i128 @fptosi_f32_to_i128(float %x) {
 ; GISEL-NEXT:    v_mad_u64_u32 v[6:7], s[8:9], v12, v8, v[0:1]
 ; GISEL-NEXT:    v_cndmask_b32_e32 v2, v4, v2, vcc
 ; GISEL-NEXT:    v_cndmask_b32_e64 v13, v2, 0, s[6:7]
-; GISEL-NEXT:    v_mad_u64_u32 v[6:7], s[8:9], v13, v9, v[6:7]
-; GISEL-NEXT:    v_mad_u64_u32 v[0:1], s[8:9], v11, v9, 0
+; GISEL-NEXT:    v_mad_u64_u32 v[6:7], s[8:9], v13, v10, v[6:7]
+; GISEL-NEXT:    v_mad_u64_u32 v[0:1], s[8:9], v11, v10, 0
 ; GISEL-NEXT:    v_mov_b32_e32 v2, v6
-; GISEL-NEXT:    v_mul_lo_u32 v6, v11, v10
+; GISEL-NEXT:    v_mul_lo_u32 v6, v11, v9
 ; GISEL-NEXT:    v_mad_u64_u32 v[1:2], s[8:9], v11, v8, v[1:2]
-; GISEL-NEXT:    v_mul_lo_u32 v4, v12, v10
+; GISEL-NEXT:    v_mul_lo_u32 v4, v12, v9
 ; GISEL-NEXT:    v_cndmask_b32_e32 v3, v5, v3, vcc
-; GISEL-NEXT:    v_mad_u64_u32 v[1:2], s[10:11], v12, v9, v[1:2]
+; GISEL-NEXT:    v_mad_u64_u32 v[1:2], s[10:11], v12, v10, v[1:2]
 ; GISEL-NEXT:    v_addc_co_u32_e64 v6, s[10:11], v7, v6, s[10:11]
 ; GISEL-NEXT:    v_addc_co_u32_e64 v4, s[8:9], v6, v4, s[8:9]
 ; GISEL-NEXT:    v_mad_u64_u32 v[6:7], s[8:9], v13, v8, v[4:5]
 ; GISEL-NEXT:    v_cndmask_b32_e64 v3, v3, 0, s[6:7]
-; GISEL-NEXT:    ; implicit-def: $vgpr10
+; GISEL-NEXT:    ; implicit-def: $vgpr9
 ; GISEL-NEXT:    ; implicit-def: $vgpr8
-; GISEL-NEXT:    v_mad_u64_u32 v[3:4], s[6:7], v3, v9, v[6:7]
+; GISEL-NEXT:    v_mad_u64_u32 v[3:4], s[6:7], v3, v10, v[6:7]
 ; GISEL-NEXT:    ; implicit-def: $vgpr6
 ; GISEL-NEXT:    ; implicit-def: $vgpr4_vgpr5
-; GISEL-NEXT:    ; implicit-def: $vgpr9
+; GISEL-NEXT:    ; implicit-def: $vgpr10
 ; GISEL-NEXT:  .LBB2_4: ; %Flow
 ; GISEL-NEXT:    s_andn2_saveexec_b64 s[6:7], s[16:17]
 ; GISEL-NEXT:    s_cbranch_execz .LBB2_6
@@ -1005,9 +1000,9 @@ define i128 @fptosi_f32_to_i128(float %x) {
 ; GISEL-NEXT:    v_cndmask_b32_e32 v0, 0, v0, vcc
 ; GISEL-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v2
 ; GISEL-NEXT:    v_cndmask_b32_e32 v4, v0, v4, vcc
-; GISEL-NEXT:    v_mad_u64_u32 v[0:1], s[8:9], v4, v9, 0
-; GISEL-NEXT:    v_mad_u64_u32 v[2:3], s[8:9], v4, v10, 0
-; GISEL-NEXT:    v_mul_lo_u32 v5, v4, v10
+; GISEL-NEXT:    v_mad_u64_u32 v[0:1], s[8:9], v4, v10, 0
+; GISEL-NEXT:    v_mad_u64_u32 v[2:3], s[8:9], v4, v9, 0
+; GISEL-NEXT:    v_mul_lo_u32 v5, v4, v9
 ; GISEL-NEXT:    v_mad_u64_u32 v[1:2], vcc, v4, v8, v[1:2]
 ; GISEL-NEXT:    v_addc_co_u32_e32 v3, vcc, v3, v5, vcc
 ; GISEL-NEXT:  .LBB2_6: ; %Flow1
@@ -1098,22 +1093,22 @@ define i128 @fptoui_f32_to_i128(float %x) {
 ; SDAG:       ; %bb.0: ; %fp-to-i-entry
 ; SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; SDAG-NEXT:    v_mov_b32_e32 v4, v0
-; SDAG-NEXT:    v_bfe_u32 v5, v4, 23, 8
+; SDAG-NEXT:    v_bfe_u32 v6, v4, 23, 8
 ; SDAG-NEXT:    s_movk_i32 s4, 0x7e
 ; SDAG-NEXT:    v_mov_b32_e32 v0, 0
 ; SDAG-NEXT:    v_mov_b32_e32 v2, 0
-; SDAG-NEXT:    v_mov_b32_e32 v6, 0
+; SDAG-NEXT:    v_mov_b32_e32 v7, 0
 ; SDAG-NEXT:    v_mov_b32_e32 v1, 0
 ; SDAG-NEXT:    v_mov_b32_e32 v3, 0
-; SDAG-NEXT:    v_cmp_lt_u32_e32 vcc, s4, v5
+; SDAG-NEXT:    v_cmp_lt_u32_e32 vcc, s4, v6
 ; SDAG-NEXT:    s_and_saveexec_b64 s[8:9], vcc
 ; SDAG-NEXT:    s_cbranch_execz .LBB3_10
 ; SDAG-NEXT:  ; %bb.1: ; %fp-to-i-if-end
-; SDAG-NEXT:    v_add_co_u32_e32 v0, vcc, 0xffffff01, v5
-; SDAG-NEXT:    v_addc_co_u32_e32 v1, vcc, -1, v6, vcc
-; SDAG-NEXT:    v_addc_co_u32_e32 v2, vcc, -1, v6, vcc
+; SDAG-NEXT:    v_add_co_u32_e32 v0, vcc, 0xffffff01, v6
+; SDAG-NEXT:    v_addc_co_u32_e32 v1, vcc, -1, v7, vcc
+; SDAG-NEXT:    v_addc_co_u32_e32 v2, vcc, -1, v7, vcc
 ; SDAG-NEXT:    s_movk_i32 s6, 0xff7f
-; SDAG-NEXT:    v_addc_co_u32_e32 v3, vcc, -1, v6, vcc
+; SDAG-NEXT:    v_addc_co_u32_e32 v3, vcc, -1, v7, vcc
 ; SDAG-NEXT:    s_mov_b32 s7, -1
 ; SDAG-NEXT:    v_cmp_eq_u64_e64 s[4:5], -1, v[2:3]
 ; SDAG-NEXT:    v_cmp_lt_u64_e64 s[6:7], s[6:7], v[0:1]
@@ -1129,66 +1124,65 @@ define i128 @fptoui_f32_to_i128(float %x) {
 ; SDAG-NEXT:    v_add_co_u32_e64 v9, s[4:5], -1, v0
 ; SDAG-NEXT:    s_mov_b64 s[4:5], 0x95
 ; SDAG-NEXT:    v_and_b32_e32 v0, 0x7fffff, v4
-; SDAG-NEXT:    v_cmp_lt_u64_e64 s[4:5], s[4:5], v[5:6]
-; SDAG-NEXT:    v_mov_b32_e32 v7, 0
+; SDAG-NEXT:    v_cmp_lt_u64_e64 s[4:5], s[4:5], v[6:7]
+; SDAG-NEXT:    v_mov_b32_e32 v5, 0
 ; SDAG-NEXT:    v_cndmask_b32_e64 v8, -1, 0, vcc
 ; SDAG-NEXT:    v_cndmask_b32_e64 v10, -1, 1, vcc
-; SDAG-NEXT:    v_or_b32_e32 v6, 0x800000, v0
+; SDAG-NEXT:    v_or_b32_e32 v4, 0x800000, v0
 ; SDAG-NEXT:    ; implicit-def: $vgpr0_vgpr1
 ; SDAG-NEXT:    ; implicit-def: $vgpr2_vgpr3
 ; SDAG-NEXT:    s_and_saveexec_b64 s[6:7], s[4:5]
 ; SDAG-NEXT:    s_xor_b64 s[12:13], exec, s[6:7]
 ; SDAG-NEXT:    s_cbranch_execz .LBB3_4
 ; SDAG-NEXT:  ; %bb.3: ; %fp-to-i-if-else
-; SDAG-NEXT:    v_sub_u32_e32 v0, 0xd6, v5
-; SDAG-NEXT:    v_add_u32_e32 v2, 0xffffff2a, v5
-; SDAG-NEXT:    v_add_u32_e32 v4, 0xffffff6a, v5
-; SDAG-NEXT:    v_lshrrev_b64 v[0:1], v0, v[6:7]
-; SDAG-NEXT:    v_lshlrev_b64 v[2:3], v2, v[6:7]
-; SDAG-NEXT:    v_cmp_gt_u32_e64 s[4:5], 64, v4
+; SDAG-NEXT:    v_sub_u32_e32 v0, 0xd6, v6
+; SDAG-NEXT:    v_add_u32_e32 v2, 0xffffff2a, v6
+; SDAG-NEXT:    v_add_u32_e32 v7, 0xffffff6a, v6
+; SDAG-NEXT:    v_lshrrev_b64 v[0:1], v0, v[4:5]
+; SDAG-NEXT:    v_lshlrev_b64 v[2:3], v2, v[4:5]
+; SDAG-NEXT:    v_cmp_gt_u32_e64 s[4:5], 64, v7
 ; SDAG-NEXT:    v_cndmask_b32_e64 v1, v3, v1, s[4:5]
-; SDAG-NEXT:    v_cmp_ne_u32_e64 s[6:7], 0, v4
+; SDAG-NEXT:    v_cmp_ne_u32_e64 s[6:7], 0, v7
 ; SDAG-NEXT:    v_cndmask_b32_e64 v3, 0, v1, s[6:7]
 ; SDAG-NEXT:    v_cndmask_b32_e64 v2, v2, v0, s[4:5]
-; SDAG-NEXT:    v_lshlrev_b64 v[0:1], v4, v[6:7]
+; SDAG-NEXT:    v_lshlrev_b64 v[0:1], v7, v[4:5]
 ; SDAG-NEXT:    v_cndmask_b32_e64 v2, 0, v2, s[6:7]
 ; SDAG-NEXT:    v_cndmask_b32_e64 v12, 0, v0, s[4:5]
 ; SDAG-NEXT:    v_cndmask_b32_e64 v11, 0, v1, s[4:5]
 ; SDAG-NEXT:    v_mad_u64_u32 v[0:1], s[4:5], v12, v10, 0
 ; SDAG-NEXT:    v_mul_lo_u32 v13, v8, v2
 ; SDAG-NEXT:    v_mul_lo_u32 v14, v10, v3
-; SDAG-NEXT:    v_mov_b32_e32 v6, v1
-; SDAG-NEXT:    v_mad_u64_u32 v[4:5], s[4:5], v11, v10, v[6:7]
+; SDAG-NEXT:    v_mov_b32_e32 v4, v1
+; SDAG-NEXT:    v_mad_u64_u32 v[6:7], s[4:5], v11, v10, v[4:5]
 ; SDAG-NEXT:    v_mad_u64_u32 v[2:3], s[4:5], v10, v2, 0
-; SDAG-NEXT:    v_mov_b32_e32 v6, v5
-; SDAG-NEXT:    v_mov_b32_e32 v5, v7
+; SDAG-NEXT:    v_mov_b32_e32 v4, v6
 ; SDAG-NEXT:    v_mad_u64_u32 v[4:5], s[4:5], v12, v8, v[4:5]
 ; SDAG-NEXT:    v_add3_u32 v3, v3, v14, v13
 ; SDAG-NEXT:    v_mad_u64_u32 v[1:2], s[4:5], v9, v12, v[2:3]
-; SDAG-NEXT:    v_add_co_u32_e64 v5, s[4:5], v6, v5
+; SDAG-NEXT:    v_add_co_u32_e64 v5, s[4:5], v7, v5
 ; SDAG-NEXT:    v_addc_co_u32_e64 v6, s[4:5], 0, 0, s[4:5]
 ; SDAG-NEXT:    v_mul_lo_u32 v3, v9, v11
-; SDAG-NEXT:    v_mul_lo_u32 v7, v9, v12
+; SDAG-NEXT:    v_mul_lo_u32 v9, v9, v12
 ; SDAG-NEXT:    v_mad_u64_u32 v[5:6], s[4:5], v11, v8, v[5:6]
 ; SDAG-NEXT:    ; implicit-def: $vgpr10
 ; SDAG-NEXT:    ; implicit-def: $vgpr8
-; SDAG-NEXT:    ; implicit-def: $vgpr9
-; SDAG-NEXT:    v_add3_u32 v3, v7, v2, v3
+; SDAG-NEXT:    v_add3_u32 v3, v9, v2, v3
 ; SDAG-NEXT:    v_add_co_u32_e64 v2, s[4:5], v5, v1
 ; SDAG-NEXT:    v_addc_co_u32_e64 v3, s[4:5], v6, v3, s[4:5]
-; SDAG-NEXT:    ; implicit-def: $vgpr5_vgpr6
 ; SDAG-NEXT:    v_mov_b32_e32 v1, v4
 ; SDAG-NEXT:    ; implicit-def: $vgpr6_vgpr7
+; SDAG-NEXT:    ; implicit-def: $vgpr4_vgpr5
+; SDAG-NEXT:    ; implicit-def: $vgpr9
 ; SDAG-NEXT:  .LBB3_4: ; %Flow
 ; SDAG-NEXT:    s_andn2_saveexec_b64 s[6:7], s[12:13]
 ; SDAG-NEXT:    s_cbranch_execz .LBB3_6
 ; SDAG-NEXT:  ; %bb.5: ; %fp-to-i-if-then12
-; SDAG-NEXT:    v_sub_u32_e32 v2, 0x96, v5
-; SDAG-NEXT:    v_lshrrev_b64 v[0:1], v2, v[6:7]
+; SDAG-NEXT:    v_sub_u32_e32 v2, 0x96, v6
+; SDAG-NEXT:    v_lshrrev_b64 v[0:1], v2, v[4:5]
 ; SDAG-NEXT:    v_cmp_gt_u32_e64 s[4:5], 64, v2
 ; SDAG-NEXT:    v_cndmask_b32_e64 v0, 0, v0, s[4:5]
 ; SDAG-NEXT:    v_cmp_eq_u32_e64 s[4:5], 0, v2
-; SDAG-NEXT:    v_cndmask_b32_e64 v3, v0, v6, s[4:5]
+; SDAG-NEXT:    v_cndmask_b32_e64 v3, v0, v4, s[4:5]
 ; SDAG-NEXT:    v_mad_u64_u32 v[0:1], s[4:5], v3, v10, 0
 ; SDAG-NEXT:    v_mov_b32_e32 v2, 0
 ; SDAG-NEXT:    v_mad_u64_u32 v[4:5], s[4:5], v3, v8, v[1:2]
@@ -1304,8 +1298,8 @@ define i128 @fptoui_f32_to_i128(float %x) {
 ; GISEL-NEXT:    v_and_b32_e32 v0, 0xffff, v0
 ; GISEL-NEXT:    v_and_b32_e32 v1, 0xffff, v1
 ; GISEL-NEXT:    v_lshlrev_b32_e32 v2, 16, v0
-; GISEL-NEXT:    v_lshl_or_b32 v10, v0, 16, v0
-; GISEL-NEXT:    v_or3_b32 v9, v1, v2, 1
+; GISEL-NEXT:    v_lshl_or_b32 v9, v0, 16, v0
+; GISEL-NEXT:    v_or3_b32 v10, v1, v2, 1
 ; GISEL-NEXT:    v_or3_b32 v8, v0, v2, 0
 ; GISEL-NEXT:    v_mov_b32_e32 v0, 0x96
 ; GISEL-NEXT:    v_mov_b32_e32 v1, 0
@@ -1323,7 +1317,7 @@ define i128 @fptoui_f32_to_i128(float %x) {
 ; GISEL-NEXT:    v_cmp_gt_u32_e32 vcc, 64, v7
 ; GISEL-NEXT:    v_cndmask_b32_e32 v11, 0, v0, vcc
 ; GISEL-NEXT:    v_cndmask_b32_e32 v12, 0, v1, vcc
-; GISEL-NEXT:    v_mad_u64_u32 v[0:1], s[6:7], v11, v10, 0
+; GISEL-NEXT:    v_mad_u64_u32 v[0:1], s[6:7], v11, v9, 0
 ; GISEL-NEXT:    v_add_u32_e32 v6, 0xffffff2a, v6
 ; GISEL-NEXT:    v_sub_u32_e32 v2, 64, v7
 ; GISEL-NEXT:    v_lshrrev_b64 v[2:3], v2, v[4:5]
@@ -1332,24 +1326,24 @@ define i128 @fptoui_f32_to_i128(float %x) {
 ; GISEL-NEXT:    v_mad_u64_u32 v[6:7], s[8:9], v12, v8, v[0:1]
 ; GISEL-NEXT:    v_cndmask_b32_e32 v2, v4, v2, vcc
 ; GISEL-NEXT:    v_cndmask_b32_e64 v13, v2, 0, s[6:7]
-; GISEL-NEXT:    v_mad_u64_u32 v[6:7], s[8:9], v13, v9, v[6:7]
-; GISEL-NEXT:    v_mad_u64_u32 v[0:1], s[8:9], v11, v9, 0
+; GISEL-NEXT:    v_mad_u64_u32 v[6:7], s[8:9], v13, v10, v[6:7]
+; GISEL-NEXT:    v_mad_u64_u32 v[0:1], s[8:9], v11, v10, 0
 ; GISEL-NEXT:    v_mov_b32_e32 v2, v6
-; GISEL-NEXT:    v_mul_lo_u32 v6, v11, v10
+; GISEL-NEXT:    v_mul_lo_u32 v6, v11, v9
 ; GISEL-NEXT:    v_mad_u64_u32 v[1:2], s[8:9], v11, v8, v[1:2]
-; GISEL-NEXT:    v_mul_lo_u32 v4, v12, v10
+; GISEL-NEXT:    v_mul_lo_u32 v4, v12, v9
 ; GISEL-NEXT:    v_cndmask_b32_e32 v3, v5, v3, vcc
-; GISEL-NEXT:    v_mad_u64_u32 v[1:2], s[10:11], v12, v9, v[1:2]
+; GISEL-NEXT:    v_mad_u64_u32 v[1:2], s[10:11], v12, v10, v[1:2]
 ; GISEL-NEXT:    v_addc_co_u32_e64 v6, s[10:11], v7, v6, s[10:11]
 ; GISEL-NEXT:    v_addc_co_u32_e64 v4, s[8:9], v6, v4, s[8:9]
 ; GISEL-NEXT:    v_mad_u64_u32 v[6:7], s[8:9], v13, v8, v[4:5]
 ; GISEL-NEXT:    v_cndmask_b32_e64 v3, v3, 0, s[6:7]
-; GISEL-NEXT:    ; implicit-def: $vgpr10
+; GISEL-NEXT:    ; implicit-def: $vgpr9
 ; GISEL-NEXT:    ; implicit-def: $vgpr8
-; GISEL-NEXT:    v_mad_u64_u32 v[3:4], s[6:7], v3, v9, v[6:7]
+; GISEL-NEXT:    v_mad_u64_u32 v[3:4], s[6:7], v3, v10, v[6:7]
 ; GISEL-NEXT:    ; implicit-def: $vgpr6
 ; GISEL-NEXT:    ; implicit-def: $vgpr4_vgpr5
-; GISEL-NEXT:    ; implicit-def: $vgpr9
+; GISEL-NEXT:    ; implicit-def: $vgpr10
 ; GISEL-NEXT:  .LBB3_4: ; %Flow
 ; GISEL-NEXT:    s_andn2_saveexec_b64 s[6:7], s[16:17]
 ; GISEL-NEXT:    s_cbranch_execz .LBB3_6
@@ -1360,9 +1354,9 @@ define i128 @fptoui_f32_to_i128(float %x) {
 ; GISEL-NEXT:    v_cndmask_b32_e32 v0, 0, v0, vcc
 ; GISEL-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v2
 ; GISEL-NEXT:    v_cndmask_b32_e32 v4, v0, v4, vcc
-; GISEL-NEXT:    v_mad_u64_u32 v[0:1], s[8:9], v4, v9, 0
-; GISEL-NEXT:    v_mad_u64_u32 v[2:3], s[8:9], v4, v10, 0
-; GISEL-NEXT:    v_mul_lo_u32 v5, v4, v10
+; GISEL-NEXT:    v_mad_u64_u32 v[0:1], s[8:9], v4, v10, 0
+; GISEL-NEXT:    v_mad_u64_u32 v[2:3], s[8:9], v4, v9, 0
+; GISEL-NEXT:    v_mul_lo_u32 v5, v4, v9
 ; GISEL-NEXT:    v_mad_u64_u32 v[1:2], vcc, v4, v8, v[1:2]
 ; GISEL-NEXT:    v_addc_co_u32_e32 v3, vcc, v3, v5, vcc
 ; GISEL-NEXT:  .LBB3_6: ; %Flow1
@@ -1481,22 +1475,22 @@ define i128 @fptosi_bf16_to_i128(bfloat %x) {
 ; SDAG:       ; %bb.0: ; %fp-to-i-entry
 ; SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; SDAG-NEXT:    v_mov_b32_e32 v4, v0
-; SDAG-NEXT:    v_bfe_u32 v5, v4, 7, 8
+; SDAG-NEXT:    v_bfe_u32 v6, v4, 7, 8
 ; SDAG-NEXT:    s_movk_i32 s4, 0x7e
 ; SDAG-NEXT:    v_mov_b32_e32 v0, 0
 ; SDAG-NEXT:    v_mov_b32_e32 v2, 0
-; SDAG-NEXT:    v_mov_b32_e32 v6, 0
+; SDAG-NEXT:    v_mov_b32_e32 v7, 0
 ; SDAG-NEXT:    v_mov_b32_e32 v1, 0
 ; SDAG-NEXT:    v_mov_b32_e32 v3, 0
-; SDAG-NEXT:    v_cmp_lt_u32_e32 vcc, s4, v5
+; SDAG-NEXT:    v_cmp_lt_u32_e32 vcc, s4, v6
 ; SDAG-NEXT:    s_and_saveexec_b64 s[8:9], vcc
 ; SDAG-NEXT:    s_cbranch_execz .LBB6_10
 ; SDAG-NEXT:  ; %bb.1: ; %fp-to-i-if-end
-; SDAG-NEXT:    v_add_co_u32_e32 v0, vcc, 0xffffff01, v5
-; SDAG-NEXT:    v_addc_co_u32_e32 v1, vcc, -1, v6, vcc
-; SDAG-NEXT:    v_addc_co_u32_e32 v2, vcc, -1, v6, vcc
+; SDAG-NEXT:    v_add_co_u32_e32 v0, vcc, 0xffffff01, v6
+; SDAG-NEXT:    v_addc_co_u32_e32 v1, vcc, -1, v7, vcc
+; SDAG-NEXT:    v_addc_co_u32_e32 v2, vcc, -1, v7, vcc
 ; SDAG-NEXT:    s_movk_i32 s6, 0xff7f
-; SDAG-NEXT:    v_addc_co_u32_e32 v3, vcc, -1, v6, vcc
+; SDAG-NEXT:    v_addc_co_u32_e32 v3, vcc, -1, v7, vcc
 ; SDAG-NEXT:    s_mov_b32 s7, -1
 ; SDAG-NEXT:    v_cmp_eq_u64_e64 s[4:5], -1, v[2:3]
 ; SDAG-NEXT:    v_cmp_lt_u64_e64 s[6:7], s[6:7], v[0:1]
@@ -1511,66 +1505,65 @@ define i128 @fptosi_bf16_to_i128(bfloat %x) {
 ; SDAG-NEXT:    s_movk_i32 s4, 0x7f
 ; SDAG-NEXT:    v_and_b32_sdwa v0, v4, s4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; SDAG-NEXT:    s_mov_b64 s[4:5], 0x85
-; SDAG-NEXT:    v_cmp_lt_u64_e64 s[4:5], s[4:5], v[5:6]
-; SDAG-NEXT:    v_mov_b32_e32 v7, 0
+; SDAG-NEXT:    v_cmp_lt_u64_e64 s[4:5], s[4:5], v[6:7]
+; SDAG-NEXT:    v_mov_b32_e32 v5, 0
 ; SDAG-NEXT:    v_cndmask_b32_e64 v9, -1, 0, vcc
 ; SDAG-NEXT:    v_cndmask_b32_e64 v8, -1, 1, vcc
-; SDAG-NEXT:    v_or_b32_e32 v6, 0x80, v0
+; SDAG-NEXT:    v_or_b32_e32 v4, 0x80, v0
 ; SDAG-NEXT:    ; implicit-def: $vgpr0_vgpr1
 ; SDAG-NEXT:    ; implicit-def: $vgpr2_vgpr3
 ; SDAG-NEXT:    s_and_saveexec_b64 s[6:7], s[4:5]
 ; SDAG-NEXT:    s_xor_b64 s[12:13], exec, s[6:7]
 ; SDAG-NEXT:    s_cbranch_execz .LBB6_4
 ; SDAG-NEXT:  ; %bb.3: ; %fp-to-i-if-else
-; SDAG-NEXT:    v_sub_u32_e32 v0, 0xc6, v5
-; SDAG-NEXT:    v_add_u32_e32 v2, 0xffffff3a, v5
-; SDAG-NEXT:    v_add_u32_e32 v4, 0xffffff7a, v5
-; SDAG-NEXT:    v_lshrrev_b64 v[0:1], v0, v[6:7]
-; SDAG-NEXT:    v_lshlrev_b64 v[2:3], v2, v[6:7]
-; SDAG-NEXT:    v_cmp_gt_u32_e64 s[4:5], 64, v4
+; SDAG-NEXT:    v_cndmask_b32_e64 v0, 0, 1, vcc
+; SDAG-NEXT:    v_add_co_u32_e64 v10, s[4:5], -1, v0
+; SDAG-NEXT:    v_sub_u32_e32 v0, 0xc6, v6
+; SDAG-NEXT:    v_add_u32_e32 v2, 0xffffff3a, v6
+; SDAG-NEXT:    v_add_u32_e32 v7, 0xffffff7a, v6
+; SDAG-NEXT:    v_lshrrev_b64 v[0:1], v0, v[4:5]
+; SDAG-NEXT:    v_lshlrev_b64 v[2:3], v2, v[4:5]
+; SDAG-NEXT:    v_cmp_gt_u32_e64 s[4:5], 64, v7
 ; SDAG-NEXT:    v_cndmask_b32_e64 v1, v3, v1, s[4:5]
-; SDAG-NEXT:    v_cmp_ne_u32_e64 s[6:7], 0, v4
+; SDAG-NEXT:    v_cmp_ne_u32_e64 s[6:7], 0, v7
 ; SDAG-NEXT:    v_cndmask_b32_e64 v3, 0, v1, s[6:7]
 ; SDAG-NEXT:    v_cndmask_b32_e64 v2, v2, v0, s[4:5]
-; SDAG-NEXT:    v_lshlrev_b64 v[0:1], v4, v[6:7]
+; SDAG-NEXT:    v_lshlrev_b64 v[0:1], v7, v[4:5]
 ; SDAG-NEXT:    v_cndmask_b32_e64 v2, 0, v2, s[6:7]
 ; SDAG-NEXT:    v_cndmask_b32_e64 v12, 0, v0, s[4:5]
 ; SDAG-NEXT:    v_cndmask_b32_e64 v11, 0, v1, s[4:5]
 ; SDAG-NEXT:    v_mad_u64_u32 v[0:1], s[4:5], v12, v8, 0
 ; SDAG-NEXT:    v_mul_lo_u32 v13, v9, v2
 ; SDAG-NEXT:    v_mul_lo_u32 v14, v8, v3
-; SDAG-NEXT:    v_mov_b32_e32 v6, v1
-; SDAG-NEXT:    v_mad_u64_u32 v[4:5], s[4:5], v11, v8, v[6:7]
+; SDAG-NEXT:    v_mov_b32_e32 v4, v1
+; SDAG-NEXT:    v_mad_u64_u32 v[6:7], s[4:5], v11, v8, v[4:5]
 ; SDAG-NEXT:    v_mad_u64_u32 v[2:3], s[4:5], v8, v2, 0
-; SDAG-NEXT:    v_mov_b32_e32 v8, v5
-; SDAG-NEXT:    v_mov_b32_e32 v5, v7
+; SDAG-NEXT:    v_mov_b32_e32 v4, v6
 ; SDAG-NEXT:    v_mad_u64_u32 v[4:5], s[4:5], v12, v9, v[4:5]
-; SDAG-NEXT:    v_cndmask_b32_e64 v10, 0, 1, vcc
-; SDAG-NEXT:    v_add_co_u32_e64 v6, s[4:5], -1, v10
 ; SDAG-NEXT:    v_add3_u32 v3, v3, v14, v13
-; SDAG-NEXT:    v_mad_u64_u32 v[1:2], s[4:5], v6, v12, v[2:3]
-; SDAG-NEXT:    v_add_co_u32_e64 v5, s[4:5], v8, v5
-; SDAG-NEXT:    v_mul_lo_u32 v3, v6, v11
-; SDAG-NEXT:    v_mul_lo_u32 v7, v6, v12
+; SDAG-NEXT:    v_mad_u64_u32 v[1:2], s[4:5], v10, v12, v[2:3]
+; SDAG-NEXT:    v_add_co_u32_e64 v5, s[4:5], v7, v5
 ; SDAG-NEXT:    v_addc_co_u32_e64 v6, s[4:5], 0, 0, s[4:5]
+; SDAG-NEXT:    v_mul_lo_u32 v3, v10, v11
+; SDAG-NEXT:    v_mul_lo_u32 v8, v10, v12
 ; SDAG-NEXT:    v_mad_u64_u32 v[5:6], s[4:5], v11, v9, v[5:6]
-; SDAG-NEXT:    v_add3_u32 v3, v7, v2, v3
-; SDAG-NEXT:    ; implicit-def: $vgpr8
+; SDAG-NEXT:    v_add3_u32 v3, v8, v2, v3
 ; SDAG-NEXT:    v_add_co_u32_e64 v2, s[4:5], v5, v1
 ; SDAG-NEXT:    v_addc_co_u32_e64 v3, s[4:5], v6, v3, s[4:5]
-; SDAG-NEXT:    ; implicit-def: $vgpr5_vgpr6
 ; SDAG-NEXT:    v_mov_b32_e32 v1, v4
 ; SDAG-NEXT:    ; implicit-def: $vgpr6_vgpr7
+; SDAG-NEXT:    ; implicit-def: $vgpr4_vgpr5
+; SDAG-NEXT:    ; implicit-def: $vgpr8
 ; SDAG-NEXT:  .LBB6_4: ; %Flow
 ; SDAG-NEXT:    s_andn2_saveexec_b64 s[6:7], s[12:13]
 ; SDAG-NEXT:    s_cbranch_execz .LBB6_6
 ; SDAG-NEXT:  ; %bb.5: ; %fp-to-i-if-then12
-; SDAG-NEXT:    v_sub_u32_e32 v2, 0x86, v5
-; SDAG-NEXT:    v_lshrrev_b64 v[0:1], v2, v[6:7]
+; SDAG-NEXT:    v_sub_u32_e32 v2, 0x86, v6
+; SDAG-NEXT:    v_lshrrev_b64 v[0:1], v2, v[4:5]
 ; SDAG-NEXT:    v_cmp_gt_u32_e64 s[4:5], 64, v2
 ; SDAG-NEXT:    v_cndmask_b32_e64 v0, 0, v0, s[4:5]
 ; SDAG-NEXT:    v_cmp_eq_u32_e64 s[4:5], 0, v2
-; SDAG-NEXT:    v_cndmask_b32_e64 v0, v0, v6, s[4:5]
+; SDAG-NEXT:    v_cndmask_b32_e64 v0, v0, v4, s[4:5]
 ; SDAG-NEXT:    v_mul_hi_i32_i24_e32 v1, v0, v8
 ; SDAG-NEXT:    v_ashrrev_i32_e32 v2, 31, v1
 ; SDAG-NEXT:    v_mul_i32_i24_e32 v0, v0, v8
@@ -1830,22 +1823,22 @@ define i128 @fptoui_bf16_to_i128(bfloat %x) {
 ; SDAG:       ; %bb.0: ; %fp-to-i-entry
 ; SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; SDAG-NEXT:    v_mov_b32_e32 v4, v0
-; SDAG-NEXT:    v_bfe_u32 v5, v4, 7, 8
+; SDAG-NEXT:    v_bfe_u32 v6, v4, 7, 8
 ; SDAG-NEXT:    s_movk_i32 s4, 0x7e
 ; SDAG-NEXT:    v_mov_b32_e32 v0, 0
 ; SDAG-NEXT:    v_mov_b32_e32 v2, 0
-; SDAG-NEXT:    v_mov_b32_e32 v6, 0
+; SDAG-NEXT:    v_mov_b32_e32 v7, 0
 ; SDAG-NEXT:    v_mov_b32_e32 v1, 0
 ; SDAG-NEXT:    v_mov_b32_e32 v3, 0
-; SDAG-NEXT:    v_cmp_lt_u32_e32 vcc, s4, v5
+; SDAG-NEXT:    v_cmp_lt_u32_e32 vcc, s4, v6
 ; SDAG-NEXT:    s_and_saveexec_b64 s[8:9], vcc
 ; SDAG-NEXT:    s_cbranch_execz .LBB7_10
 ; SDAG-NEXT:  ; %bb.1: ; %fp-to-i-if-end
-; SDAG-NEXT:    v_add_co_u32_e32 v0, vcc, 0xffffff01, v5
-; SDAG-NEXT:    v_addc_co_u32_e32 v1, vcc, -1, v6, vcc
-; SDAG-NEXT:    v_addc_co_u32_e32 v2, vcc, -1, v6, vcc
+; SDAG-NEXT:    v_add_co_u32_e32 v0, vcc, 0xffffff01, v6
+; SDAG-NEXT:    v_addc_co_u32_e32 v1, vcc, -1, v7, vcc
+; SDAG-NEXT:    v_addc_co_u32_e32 v2, vcc, -1, v7, vcc
 ; SDAG-NEXT:    s_movk_i32 s6, 0xff7f
-; SDAG-NEXT:    v_addc_co_u32_e32 v3, vcc, -1, v6, vcc
+; SDAG-NEXT:    v_addc_co_u32_e32 v3, vcc, -1, v7, vcc
 ; SDAG-NEXT:    s_mov_b32 s7, -1
 ; SDAG-NEXT:    v_cmp_eq_u64_e64 s[4:5], -1, v[2:3]
 ; SDAG-NEXT:    v_cmp_lt_u64_e64 s[6:7], s[6:7], v[0:1]
@@ -1860,66 +1853,65 @@ define i128 @fptoui_bf16_to_i128(bfloat %x) {
 ; SDAG-NEXT:    s_movk_i32 s4, 0x7f
 ; SDAG-NEXT:    v_and_b32_sdwa v0, v4, s4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; SDAG-NEXT:    s_mov_b64 s[4:5], 0x85
-; SDAG-NEXT:    v_cmp_lt_u64_e64 s[4:5], s[4:5], v[5:6]
-; SDAG-NEXT:    v_mov_b32_e32 v7, 0
+; SDAG-NEXT:    v_cmp_lt_u64_e64 s[4:5], s[4:5], v[6:7]
+; SDAG-NEXT:    v_mov_b32_e32 v5, 0
 ; SDAG-NEXT:    v_cndmask_b32_e64 v9, -1, 0, vcc
 ; SDAG-NEXT:    v_cndmask_b32_e64 v8, -1, 1, vcc
-; SDAG-NEXT:    v_or_b32_e32 v6, 0x80, v0
+; SDAG-NEXT:    v_or_b32_e32 v4, 0x80, v0
 ; SDAG-NEXT:    ; implicit-def: $vgpr0_vgpr1
 ; SDAG-NEXT:    ; implicit-def: $vgpr2_vgpr3
 ; SDAG-NEXT:    s_and_saveexec_b64 s[6:7], s[4:5]
 ; SDAG-NEXT:    s_xor_b64 s[12:13], exec, s[6:7]
 ; SDAG-NEXT:    s_cbranch_execz .LBB7_4
 ; SDAG-NEXT:  ; %bb.3: ; %fp-to-i-if-else
-; SDAG-NEXT:    v_sub_u32_e32 v0, 0xc6, v5
-; SDAG-NEXT:    v_add_u32_e32 v2, 0xffffff3a, v5
-; SDAG-NEXT:    v_add_u32_e32 v4, 0xffffff7a, v5
-; SDAG-NEXT:    v_lshrrev_b64 v[0:1], v0, v[6:7]
-; SDAG-NEXT:    v_lshlrev_b64 v[2:3], v2, v[6:7]
-; SDAG-NEXT:    v_cmp_gt_u32_e64 s[4:5], 64, v4
+; SDAG-NEXT:    v_cndmask_b32_e64 v0, 0, 1, vcc
+; SDAG-NEXT:    v_add_co_u32_e64 v10, s[4:5], -1, v0
+; SDAG-NEXT:    v_sub_u32_e32 v0, 0xc6, v6
+; SDAG-NEXT:    v_add_u32_e32 v2, 0xffffff3a, v6
+; SDAG-NEXT:    v_add_u32_e32 v7, 0xffffff7a, v6
+; SDAG-NEXT:    v_lshrrev_b64 v[0:1], v0, v[4:5]
+; SDAG-NEXT:    v_lshlrev_b64 v[2:3], v2, v[4:5]
+; SDAG-NEXT:    v_cmp_gt_u32_e64 s[4:5], 64, v7
 ; SDAG-NEXT:    v_cndmask_b32_e64 v1, v3, v1, s[4:5]
-; SDAG-NEXT:    v_cmp_ne_u32_e64 s[6:7], 0, v4
+; SDAG-NEXT:    v_cmp_ne_u32_e64 s[6:7], 0, v7
 ; SDAG-NEXT:    v_cndmask_b32_e64 v3, 0, v1, s[6:7]
 ; SDAG-NEXT:    v_cndmask_b32_e64 v2, v2, v0, s[4:5]
-; SDAG-NEXT:    v_lshlrev_b64 v[0:1], v4, v[6:7]
+; SDAG-NEXT:    v_lshlrev_b64 v[0:1], v7, v[4:5]
 ; SDAG-NEXT:    v_cndmask_b32_e64 v2, 0, v2, s[6:7]
 ; SDAG-NEXT:    v_cndmask_b32_e64 v12, 0, v0, s[4:5]
 ; SDAG-NEXT:    v_cndmask_b32_e64 v11, 0, v1, s[4:5]
 ; SDAG-NEXT:    v_mad_u64_u32 v[0:1], s[4:5], v12, v8, 0
 ; SDAG-NEXT:    v_mul_lo_u32 v13, v9, v2
 ; SDAG-NEXT:    v_mul_lo_u32 v14, v8, v3
-; SDAG-NEXT:    v_mov_b32_e32 v6, v1
-; SDAG-NEXT:    v_mad_u64_u32 v[4:5], s[4:5], v11, v8, v[6:7]
+; SDAG-NEXT:    v_mov_b32_e32 v4, v1
+; SDAG-NEXT:    v_mad_u64_u32 v[6:7], s[4:5], v11, v8, v[4:5]
 ; SDAG-NEXT:    v_mad_u64_u32 v[2:3], s[4:5], v8, v2, 0
-; SDAG-NEXT:    v_mov_b32_e32 v8, v5
-; SDAG-NEXT:    v_mov_b32_e32 v5, v7
+; SDAG-NEXT:    v_mov_b32_e32 v4, v6
 ; SDAG-NEXT:    v_mad_u64_u32 v[4:5], s[4:5], v12, v9, v[4:5]
-; SDAG-NEXT:    v_cndmask_b32_e64 v10, 0, 1, vcc
-; SDAG-NEXT:    v_add_co_u32_e64 v6, s[4:5], -1, v10
 ; SDAG-NEXT:    v_add3_u32 v3, v3, v14, v13
-; SDAG-NEXT:    v_mad_u64_u32 v[1:2], s[4:5], v6, v12, v[2:3]
-; SDAG-NEXT:    v_add_co_u32_e64 v5, s[4:5], v8, v5
-; SDAG-NEXT:    v_mul_lo_u32 v3, v6, v11
-; SDAG-NEXT:    v_mul_lo_u32 v7, v6, v12
+; SDAG-NEXT:    v_mad_u64_u32 v[1:2], s[4:5], v10, v12, v[2:3]
+; SDAG-NEXT:    v_add_co_u32_e64 v5, s[4:5], v7, v5
 ; SDAG-NEXT:    v_addc_co_u32_e64 v6, s[4:5], 0, 0, s[4:5]
+; SDAG-NEXT:    v_mul_lo_u32 v3, v10, v11
+; SDAG-NEXT:    v_mul_lo_u32 v8, v10, v12
 ; SDAG-NEXT:    v_mad_u64_u32 v[5:6], s[4:5], v11, v9, v[5:6]
-; SDAG-NEXT:    v_add3_u32 v3, v7, v2, v3
-; SDAG-NEXT:    ; implicit-def: $vgpr8
+; SDAG-NEXT:    v_add3_u32 v3, v8, v2, v3
 ; SDAG-NEXT:    v_add_co_u32_e64 v2, s[4:5], v5, v1
 ; SDAG-NEXT:    v_addc_co_u32_e64 v3, s[4:5], v6, v3, s[4:5]
-; SDAG-NEXT:    ; implicit-def: $vgpr5_vgpr6
 ; SDAG-NEXT:    v_mov_b32_e32 v1, v4
 ; SDAG-NEXT:    ; implicit-def: $vgpr6_vgpr7
+; SDAG-NEXT:    ; implicit-def: $vgpr4_vgpr5
+; SDAG-NEXT:    ; implicit-def: $vgpr8
 ; SDAG-NEXT:  .LBB7_4: ; %Flow
 ; SDAG-NEXT:    s_andn2_saveexec_b64 s[6:7], s[12:13]
 ; SDAG-NEXT:    s_cbranch_execz .LBB7_6
 ; SDAG-NEXT:  ; %bb.5: ; %fp-to-i-if-then12
-; SDAG-NEXT:    v_sub_u32_e32 v2, 0x86, v5
-; SDAG-NEXT:    v_lshrrev_b64 v[0:1], v2, v[6:7]
+; SDAG-NEXT:    v_sub_u32_e32 v2, 0x86, v6
+; SDAG-NEXT:    v_lshrrev_b64 v[0:1], v2, v[4:5]
 ; SDAG-NEXT:    v_cmp_gt_u32_e64 s[4:5], 64, v2
 ; SDAG-NEXT:    v_cndmask_b32_e64 v0, 0, v0, s[4:5]
 ; SDAG-NEXT:    v_cmp_eq_u32_e64 s[4:5], 0, v2
-; SDAG-NEXT:    v_cndmask_b32_e64 v0, v0, v6, s[4:5]
+; SDAG-NEXT:    v_cndmask_b32_e64 v0, v0, v4, s[4:5]
 ; SDAG-NEXT:    v_mul_hi_i32_i24_e32 v1, v0, v8
 ; SDAG-NEXT:    v_ashrrev_i32_e32 v2, 31, v1
 ; SDAG-NEXT:    v_mul_i32_i24_e32 v0, v0, v8
diff --git a/llvm/test/CodeGen/AMDGPU/fptrunc.ll b/llvm/test/CodeGen/AMDGPU/fptrunc.ll
index 3d3e8bea7e33e..051a0c51b0867 100644
--- a/llvm/test/CodeGen/AMDGPU/fptrunc.ll
+++ b/llvm/test/CodeGen/AMDGPU/fptrunc.ll
@@ -155,61 +155,61 @@ define amdgpu_kernel void @fptrunc_f64_to_f16(ptr addrspace(1) %out, double %in)
 ;
 ; VI-SAFE-SDAG-LABEL: fptrunc_f64_to_f16:
 ; VI-SAFE-SDAG:       ; %bb.0:
-; VI-SAFE-SDAG-NEXT:    s_load_dwordx4 s[4:7], s[4:5], 0x24
-; VI-SAFE-SDAG-NEXT:    s_mov_b32 s3, 0xf000
-; VI-SAFE-SDAG-NEXT:    s_mov_b32 s2, -1
+; VI-SAFE-SDAG-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-SAFE-SDAG-NEXT:    s_mov_b32 s7, 0xf000
+; VI-SAFE-SDAG-NEXT:    s_mov_b32 s6, -1
 ; VI-SAFE-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
-; VI-SAFE-SDAG-NEXT:    s_mov_b32 s0, s4
-; VI-SAFE-SDAG-NEXT:    s_lshr_b32 s4, s7, 8
-; VI-SAFE-SDAG-NEXT:    s_and_b32 s8, s4, 0xffe
-; VI-SAFE-SDAG-NEXT:    s_and_b32 s4, s7, 0x1ff
-; VI-SAFE-SDAG-NEXT:    s_or_b32 s4, s4, s6
-; VI-SAFE-SDAG-NEXT:    s_cmp_lg_u32 s4, 0
-; VI-SAFE-SDAG-NEXT:    s_mov_b32 s1, s5
-; VI-SAFE-SDAG-NEXT:    s_cselect_b64 s[4:5], -1, 0
-; VI-SAFE-SDAG-NEXT:    v_cndmask_b32_e64 v0, 0, 1, s[4:5]
-; VI-SAFE-SDAG-NEXT:    v_readfirstlane_b32 s4, v0
-; VI-SAFE-SDAG-NEXT:    s_bfe_u32 s5, s7, 0xb0014
-; VI-SAFE-SDAG-NEXT:    s_or_b32 s6, s8, s4
-; VI-SAFE-SDAG-NEXT:    s_sub_i32 s8, 0x3f1, s5
+; VI-SAFE-SDAG-NEXT:    s_mov_b32 s4, s0
+; VI-SAFE-SDAG-NEXT:    s_lshr_b32 s0, s3, 8
+; VI-SAFE-SDAG-NEXT:    s_and_b32 s8, s0, 0xffe
+; VI-SAFE-SDAG-NEXT:    s_and_b32 s0, s3, 0x1ff
+; VI-SAFE-SDAG-NEXT:    s_or_b32 s0, s0, s2
+; VI-SAFE-SDAG-NEXT:    s_cmp_lg_u32 s0, 0
+; VI-SAFE-SDAG-NEXT:    s_mov_b32 s5, s1
+; VI-SAFE-SDAG-NEXT:    s_cselect_b64 s[0:1], -1, 0
+; VI-SAFE-SDAG-NEXT:    v_cndmask_b32_e64 v0, 0, 1, s[0:1]
+; VI-SAFE-SDAG-NEXT:    v_readfirstlane_b32 s0, v0
+; VI-SAFE-SDAG-NEXT:    s_bfe_u32 s1, s3, 0xb0014
+; VI-SAFE-SDAG-NEXT:    s_or_b32 s2, s8, s0
+; VI-SAFE-SDAG-NEXT:    s_sub_i32 s8, 0x3f1, s1
 ; VI-SAFE-SDAG-NEXT:    v_med3_i32 v0, s8, 0, 13
-; VI-SAFE-SDAG-NEXT:    s_or_b32 s4, s6, 0x1000
+; VI-SAFE-SDAG-NEXT:    s_or_b32 s0, s2, 0x1000
 ; VI-SAFE-SDAG-NEXT:    v_readfirstlane_b32 s8, v0
-; VI-SAFE-SDAG-NEXT:    s_lshr_b32 s8, s4, s8
+; VI-SAFE-SDAG-NEXT:    s_lshr_b32 s8, s0, s8
 ; VI-SAFE-SDAG-NEXT:    v_lshlrev_b32_e64 v0, v0, s8
-; VI-SAFE-SDAG-NEXT:    v_cmp_ne_u32_e32 vcc, s4, v0
+; VI-SAFE-SDAG-NEXT:    v_cmp_ne_u32_e32 vcc, s0, v0
 ; VI-SAFE-SDAG-NEXT:    v_cndmask_b32_e64 v0, 0, 1, vcc
-; VI-SAFE-SDAG-NEXT:    s_add_i32 s10, s5, 0xfffffc10
-; VI-SAFE-SDAG-NEXT:    v_readfirstlane_b32 s4, v0
-; VI-SAFE-SDAG-NEXT:    s_lshl_b32 s5, s10, 12
-; VI-SAFE-SDAG-NEXT:    s_or_b32 s4, s8, s4
-; VI-SAFE-SDAG-NEXT:    s_or_b32 s5, s6, s5
+; VI-SAFE-SDAG-NEXT:    s_add_i32 s10, s1, 0xfffffc10
+; VI-SAFE-SDAG-NEXT:    v_readfirstlane_b32 s0, v0
+; VI-SAFE-SDAG-NEXT:    s_lshl_b32 s1, s10, 12
+; VI-SAFE-SDAG-NEXT:    s_or_b32 s0, s8, s0
+; VI-SAFE-SDAG-NEXT:    s_or_b32 s1, s2, s1
 ; VI-SAFE-SDAG-NEXT:    s_cmp_lt_i32 s10, 1
-; VI-SAFE-SDAG-NEXT:    s_cselect_b32 s11, s4, s5
+; VI-SAFE-SDAG-NEXT:    s_cselect_b32 s11, s0, s1
 ; VI-SAFE-SDAG-NEXT:    s_and_b32 s8, s11, 7
 ; VI-SAFE-SDAG-NEXT:    s_cmp_gt_i32 s8, 5
-; VI-SAFE-SDAG-NEXT:    s_cselect_b64 s[4:5], -1, 0
+; VI-SAFE-SDAG-NEXT:    s_cselect_b64 s[0:1], -1, 0
 ; VI-SAFE-SDAG-NEXT:    s_cmp_eq_u32 s8, 3
 ; VI-SAFE-SDAG-NEXT:    s_cselect_b64 s[8:9], -1, 0
-; VI-SAFE-SDAG-NEXT:    s_or_b64 s[4:5], s[8:9], s[4:5]
+; VI-SAFE-SDAG-NEXT:    s_or_b64 s[0:1], s[8:9], s[0:1]
 ; VI-SAFE-SDAG-NEXT:    s_lshr_b32 s8, s11, 2
-; VI-SAFE-SDAG-NEXT:    s_cmp_lg_u64 s[4:5], 0
-; VI-SAFE-SDAG-NEXT:    s_addc_u32 s4, s8, 0
+; VI-SAFE-SDAG-NEXT:    s_cmp_lg_u64 s[0:1], 0
+; VI-SAFE-SDAG-NEXT:    s_addc_u32 s0, s8, 0
 ; VI-SAFE-SDAG-NEXT:    s_cmp_lt_i32 s10, 31
-; VI-SAFE-SDAG-NEXT:    s_cselect_b32 s8, s4, 0x7c00
-; VI-SAFE-SDAG-NEXT:    s_cmp_lg_u32 s6, 0
-; VI-SAFE-SDAG-NEXT:    s_cselect_b64 s[4:5], -1, 0
-; VI-SAFE-SDAG-NEXT:    v_cndmask_b32_e64 v0, 0, 1, s[4:5]
+; VI-SAFE-SDAG-NEXT:    s_cselect_b32 s8, s0, 0x7c00
+; VI-SAFE-SDAG-NEXT:    s_cmp_lg_u32 s2, 0
+; VI-SAFE-SDAG-NEXT:    s_cselect_b64 s[0:1], -1, 0
+; VI-SAFE-SDAG-NEXT:    v_cndmask_b32_e64 v0, 0, 1, s[0:1]
 ; VI-SAFE-SDAG-NEXT:    v_lshlrev_b32_e32 v0, 9, v0
 ; VI-SAFE-SDAG-NEXT:    s_cmpk_eq_i32 s10, 0x40f
 ; VI-SAFE-SDAG-NEXT:    v_or_b32_e32 v0, 0x7c00, v0
 ; VI-SAFE-SDAG-NEXT:    v_mov_b32_e32 v1, s8
 ; VI-SAFE-SDAG-NEXT:    s_cselect_b64 vcc, -1, 0
-; VI-SAFE-SDAG-NEXT:    s_lshr_b32 s4, s7, 16
+; VI-SAFE-SDAG-NEXT:    s_lshr_b32 s0, s3, 16
 ; VI-SAFE-SDAG-NEXT:    v_cndmask_b32_e32 v0, v1, v0, vcc
-; VI-SAFE-SDAG-NEXT:    s_and_b32 s4, s4, 0x8000
-; VI-SAFE-SDAG-NEXT:    v_or_b32_e32 v0, s4, v0
-; VI-SAFE-SDAG-NEXT:    buffer_store_short v0, off, s[0:3], 0
+; VI-SAFE-SDAG-NEXT:    s_and_b32 s0, s0, 0x8000
+; VI-SAFE-SDAG-NEXT:    v_or_b32_e32 v0, s0, v0
+; VI-SAFE-SDAG-NEXT:    buffer_store_short v0, off, s[4:7], 0
 ; VI-SAFE-SDAG-NEXT:    s_endpgm
 ;
 ; VI-SAFE-GISEL-LABEL: fptrunc_f64_to_f16:
diff --git a/llvm/test/CodeGen/AMDGPU/fract-match.ll b/llvm/test/CodeGen/AMDGPU/fract-match.ll
index 80b4d64b1236f..fcaf427f6c010 100644
--- a/llvm/test/CodeGen/AMDGPU/fract-match.ll
+++ b/llvm/test/CodeGen/AMDGPU/fract-match.ll
@@ -21,9 +21,9 @@
 ; Expansion as it appears in the library with the extra output for
 ; floor. We can fold in the nan check into the instruction, but the
 ; inf check must remain.
-define float @safe_math_fract_f32(float %x, ptr addrspace(1) nocapture writeonly %ip) {
+define float @safe_math_fract_f32(float %x, ptr addrspace(1) writeonly captures(none) %ip) {
 ; GFX6-IR-LABEL: define float @safe_math_fract_f32
-; GFX6-IR-SAME: (float [[X:%.*]], ptr addrspace(1) nocapture writeonly [[IP:%.*]]) #[[ATTR0:[0-9]+]] {
+; GFX6-IR-SAME: (float [[X:%.*]], ptr addrspace(1) writeonly captures(none) [[IP:%.*]]) #[[ATTR0:[0-9]+]] {
 ; GFX6-IR-NEXT:  entry:
 ; GFX6-IR-NEXT:    [[FLOOR:%.*]] = tail call float @llvm.floor.f32(float [[X]])
 ; GFX6-IR-NEXT:    [[SUB:%.*]] = fsub float [[X]], [[FLOOR]]
@@ -37,7 +37,7 @@ define float @safe_math_fract_f32(float %x, ptr addrspace(1) nocapture writeonly
 ; GFX6-IR-NEXT:    ret float [[COND6]]
 ;
 ; IR-FRACT-LABEL: define float @safe_math_fract_f32
-; IR-FRACT-SAME: (float [[X:%.*]], ptr addrspace(1) nocapture writeonly [[IP:%.*]]) #[[ATTR0:[0-9]+]] {
+; IR-FRACT-SAME: (float [[X:%.*]], ptr addrspace(1) writeonly captures(none) [[IP:%.*]]) #[[ATTR0:[0-9]+]] {
 ; IR-FRACT-NEXT:  entry:
 ; IR-FRACT-NEXT:    [[FLOOR:%.*]] = tail call float @llvm.floor.f32(float [[X]])
 ; IR-FRACT-NEXT:    [[COND:%.*]] = call float @llvm.amdgcn.fract.f32(float [[X]])
@@ -115,6 +115,7 @@ define float @safe_math_fract_f32(float %x, ptr addrspace(1) nocapture writeonly
 ; GFX12-NEXT:    v_fract_f32_e32 v3, v0
 ; GFX12-NEXT:    v_cmp_neq_f32_e64 vcc_lo, 0x7f800000, |v0|
 ; GFX12-NEXT:    v_floor_f32_e32 v4, v0
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3)
 ; GFX12-NEXT:    v_cndmask_b32_e32 v0, 0, v3, vcc_lo
 ; GFX12-NEXT:    global_store_b32 v[1:2], v4, off
@@ -132,9 +133,9 @@ entry:
   ret float %cond6
 }
 
-define float @safe_math_fract_f32_noinf_check(float %x, ptr addrspace(1) nocapture writeonly %ip) {
+define float @safe_math_fract_f32_noinf_check(float %x, ptr addrspace(1) writeonly captures(none) %ip) {
 ; GFX6-IR-LABEL: define float @safe_math_fract_f32_noinf_check
-; GFX6-IR-SAME: (float [[X:%.*]], ptr addrspace(1) nocapture writeonly [[IP:%.*]]) #[[ATTR0]] {
+; GFX6-IR-SAME: (float [[X:%.*]], ptr addrspace(1) writeonly captures(none) [[IP:%.*]]) #[[ATTR0]] {
 ; GFX6-IR-NEXT:  entry:
 ; GFX6-IR-NEXT:    [[FLOOR:%.*]] = tail call float @llvm.floor.f32(float [[X]])
 ; GFX6-IR-NEXT:    [[SUB:%.*]] = fsub float [[X]], [[FLOOR]]
@@ -145,7 +146,7 @@ define float @safe_math_fract_f32_noinf_check(float %x, ptr addrspace(1) nocaptu
 ; GFX6-IR-NEXT:    ret float [[COND]]
 ;
 ; IR-FRACT-LABEL: define float @safe_math_fract_f32_noinf_check
-; IR-FRACT-SAME: (float [[X:%.*]], ptr addrspace(1) nocapture writeonly [[IP:%.*]]) #[[ATTR0]] {
+; IR-FRACT-SAME: (float [[X:%.*]], ptr addrspace(1) writeonly captures(none) [[IP:%.*]]) #[[ATTR0]] {
 ; IR-FRACT-NEXT:  entry:
 ; IR-FRACT-NEXT:    [[FLOOR:%.*]] = tail call float @llvm.floor.f32(float [[X]])
 ; IR-FRACT-NEXT:    [[COND:%.*]] = call float @llvm.amdgcn.fract.f32(float [[X]])
@@ -220,9 +221,9 @@ entry:
 }
 
 ; Cannot match fract without a nan check or no-nans.
-define float @no_nan_check_math_fract_f32(float %x, ptr addrspace(1) nocapture writeonly %ip) {
+define float @no_nan_check_math_fract_f32(float %x, ptr addrspace(1) writeonly captures(none) %ip) {
 ; IR-LABEL: define float @no_nan_check_math_fract_f32
-; IR-SAME: (float [[X:%.*]], ptr addrspace(1) nocapture writeonly [[IP:%.*]]) #[[ATTR0:[0-9]+]] {
+; IR-SAME: (float [[X:%.*]], ptr addrspace(1) writeonly captures(none) [[IP:%.*]]) #[[ATTR0:[0-9]+]] {
 ; IR-NEXT:  entry:
 ; IR-NEXT:    [[FLOOR:%.*]] = tail call float @llvm.floor.f32(float [[X]])
 ; IR-NEXT:    [[SUB:%.*]] = fsub float [[X]], [[FLOOR]]
@@ -301,10 +302,11 @@ define float @no_nan_check_math_fract_f32(float %x, ptr addrspace(1) nocapture w
 ; GFX12-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-NEXT:    v_floor_f32_e32 v3, v0
 ; GFX12-NEXT:    v_cmp_neq_f32_e64 vcc_lo, 0x7f800000, |v0|
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_sub_f32_e32 v4, v0, v3
 ; GFX12-NEXT:    global_store_b32 v[1:2], v3, off
 ; GFX12-NEXT:    v_min_num_f32_e32 v4, 0x3f7fffff, v4
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_cndmask_b32_e32 v0, 0, v4, vcc_lo
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 entry:
@@ -1423,6 +1425,7 @@ define float @wrong_commuted_nan_select_f32(float %x) {
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_sub_f32_e32 v1, v0, v1
 ; GFX12-NEXT:    v_min_num_f32_e32 v1, 0x3f7fffff, v1
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc_lo
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
@@ -1668,9 +1671,9 @@ entry:
   ret double %min
 }
 
-define half @safe_math_fract_f16_noinf_check(half %x, ptr addrspace(1) nocapture writeonly %ip) {
+define half @safe_math_fract_f16_noinf_check(half %x, ptr addrspace(1) writeonly captures(none) %ip) {
 ; GFX6-IR-LABEL: define half @safe_math_fract_f16_noinf_check
-; GFX6-IR-SAME: (half [[X:%.*]], ptr addrspace(1) nocapture writeonly [[IP:%.*]]) #[[ATTR0]] {
+; GFX6-IR-SAME: (half [[X:%.*]], ptr addrspace(1) writeonly captures(none) [[IP:%.*]]) #[[ATTR0]] {
 ; GFX6-IR-NEXT:  entry:
 ; GFX6-IR-NEXT:    [[FLOOR:%.*]] = tail call half @llvm.floor.f16(half [[X]])
 ; GFX6-IR-NEXT:    [[SUB:%.*]] = fsub half [[X]], [[FLOOR]]
@@ -1681,7 +1684,7 @@ define half @safe_math_fract_f16_noinf_check(half %x, ptr addrspace(1) nocapture
 ; GFX6-IR-NEXT:    ret half [[COND]]
 ;
 ; GFX7-IR-LABEL: define half @safe_math_fract_f16_noinf_check
-; GFX7-IR-SAME: (half [[X:%.*]], ptr addrspace(1) nocapture writeonly [[IP:%.*]]) #[[ATTR0]] {
+; GFX7-IR-SAME: (half [[X:%.*]], ptr addrspace(1) writeonly captures(none) [[IP:%.*]]) #[[ATTR0]] {
 ; GFX7-IR-NEXT:  entry:
 ; GFX7-IR-NEXT:    [[FLOOR:%.*]] = tail call half @llvm.floor.f16(half [[X]])
 ; GFX7-IR-NEXT:    [[SUB:%.*]] = fsub half [[X]], [[FLOOR]]
@@ -1692,7 +1695,7 @@ define half @safe_math_fract_f16_noinf_check(half %x, ptr addrspace(1) nocapture
 ; GFX7-IR-NEXT:    ret half [[COND]]
 ;
 ; IR-LEGALF16-LABEL: define half @safe_math_fract_f16_noinf_check
-; IR-LEGALF16-SAME: (half [[X:%.*]], ptr addrspace(1) nocapture writeonly [[IP:%.*]]) #[[ATTR0]] {
+; IR-LEGALF16-SAME: (half [[X:%.*]], ptr addrspace(1) writeonly captures(none) [[IP:%.*]]) #[[ATTR0]] {
 ; IR-LEGALF16-NEXT:  entry:
 ; IR-LEGALF16-NEXT:    [[FLOOR:%.*]] = tail call half @llvm.floor.f16(half [[X]])
 ; IR-LEGALF16-NEXT:    [[COND:%.*]] = call half @llvm.amdgcn.fract.f16(half [[X]])
@@ -1775,9 +1778,9 @@ entry:
   ret half %cond
 }
 
-define double @safe_math_fract_f64_noinf_check(double %x, ptr addrspace(1) nocapture writeonly %ip) {
+define double @safe_math_fract_f64_noinf_check(double %x, ptr addrspace(1) writeonly captures(none) %ip) {
 ; GFX6-IR-LABEL: define double @safe_math_fract_f64_noinf_check
-; GFX6-IR-SAME: (double [[X:%.*]], ptr addrspace(1) nocapture writeonly [[IP:%.*]]) #[[ATTR0]] {
+; GFX6-IR-SAME: (double [[X:%.*]], ptr addrspace(1) writeonly captures(none) [[IP:%.*]]) #[[ATTR0]] {
 ; GFX6-IR-NEXT:  entry:
 ; GFX6-IR-NEXT:    [[FLOOR:%.*]] = tail call double @llvm.floor.f64(double [[X]])
 ; GFX6-IR-NEXT:    [[SUB:%.*]] = fsub double [[X]], [[FLOOR]]
@@ -1788,7 +1791,7 @@ define double @safe_math_fract_f64_noinf_check(double %x, ptr addrspace(1) nocap
 ; GFX6-IR-NEXT:    ret double [[COND]]
 ;
 ; IR-FRACT-LABEL: define double @safe_math_fract_f64_noinf_check
-; IR-FRACT-SAME: (double [[X:%.*]], ptr addrspace(1) nocapture writeonly [[IP:%.*]]) #[[ATTR0]] {
+; IR-FRACT-SAME: (double [[X:%.*]], ptr addrspace(1) writeonly captures(none) [[IP:%.*]]) #[[ATTR0]] {
 ; IR-FRACT-NEXT:  entry:
 ; IR-FRACT-NEXT:    [[FLOOR:%.*]] = tail call double @llvm.floor.f64(double [[X]])
 ; IR-FRACT-NEXT:    [[COND:%.*]] = call double @llvm.amdgcn.fract.f64(double [[X]])
@@ -1998,9 +2001,9 @@ entry:
   ret float %cond
 }
 
-define <2 x float> @safe_math_fract_v2f32(<2 x float> %x, ptr addrspace(1) nocapture writeonly %ip) {
+define <2 x float> @safe_math_fract_v2f32(<2 x float> %x, ptr addrspace(1) writeonly captures(none) %ip) {
 ; GFX6-IR-LABEL: define <2 x float> @safe_math_fract_v2f32
-; GFX6-IR-SAME: (<2 x float> [[X:%.*]], ptr addrspace(1) nocapture writeonly [[IP:%.*]]) #[[ATTR0]] {
+; GFX6-IR-SAME: (<2 x float> [[X:%.*]], ptr addrspace(1) writeonly captures(none) [[IP:%.*]]) #[[ATTR0]] {
 ; GFX6-IR-NEXT:  entry:
 ; GFX6-IR-NEXT:    [[FLOOR:%.*]] = tail call <2 x float> @llvm.floor.v2f32(<2 x float> [[X]])
 ; GFX6-IR-NEXT:    [[SUB:%.*]] = fsub <2 x float> [[X]], [[FLOOR]]
@@ -2014,7 +2017,7 @@ define <2 x float> @safe_math_fract_v2f32(<2 x float> %x, ptr addrspace(1) nocap
 ; GFX6-IR-NEXT:    ret <2 x float> [[COND6]]
 ;
 ; IR-FRACT-LABEL: define <2 x float> @safe_math_fract_v2f32
-; IR-FRACT-SAME: (<2 x float> [[X:%.*]], ptr addrspace(1) nocapture writeonly [[IP:%.*]]) #[[ATTR0]] {
+; IR-FRACT-SAME: (<2 x float> [[X:%.*]], ptr addrspace(1) writeonly captures(none) [[IP:%.*]]) #[[ATTR0]] {
 ; IR-FRACT-NEXT:  entry:
 ; IR-FRACT-NEXT:    [[FLOOR:%.*]] = tail call <2 x float> @llvm.floor.v2f32(<2 x float> [[X]])
 ; IR-FRACT-NEXT:    [[TMP0:%.*]] = extractelement <2 x float> [[X]], i64 0
@@ -2118,10 +2121,11 @@ define <2 x float> @safe_math_fract_v2f32(<2 x float> %x, ptr addrspace(1) nocap
 ; GFX12-NEXT:    v_fract_f32_e32 v7, v1
 ; GFX12-NEXT:    v_floor_f32_e32 v4, v0
 ; GFX12-NEXT:    v_floor_f32_e32 v5, v1
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_4)
+; GFX12-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-NEXT:    v_cndmask_b32_e64 v0, v6, 0, s0
 ; GFX12-NEXT:    v_cmp_class_f32_e64 s0, v1, 0x204
 ; GFX12-NEXT:    global_store_b64 v[2:3], v[4:5], off
+; GFX12-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-NEXT:    v_cndmask_b32_e64 v1, v7, 0, s0
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 entry:
@@ -2137,9 +2141,9 @@ entry:
   ret <2 x float> %cond6
 }
 
-define double @safe_math_fract_f64(double %x, ptr addrspace(1) nocapture writeonly %ip) {
+define double @safe_math_fract_f64(double %x, ptr addrspace(1) writeonly captures(none) %ip) {
 ; GFX6-IR-LABEL: define double @safe_math_fract_f64
-; GFX6-IR-SAME: (double [[X:%.*]], ptr addrspace(1) nocapture writeonly [[IP:%.*]]) #[[ATTR0]] {
+; GFX6-IR-SAME: (double [[X:%.*]], ptr addrspace(1) writeonly captures(none) [[IP:%.*]]) #[[ATTR0]] {
 ; GFX6-IR-NEXT:  entry:
 ; GFX6-IR-NEXT:    [[FLOOR:%.*]] = tail call double @llvm.floor.f64(double [[X]])
 ; GFX6-IR-NEXT:    [[SUB:%.*]] = fsub double [[X]], [[FLOOR]]
@@ -2153,7 +2157,7 @@ define double @safe_math_fract_f64(double %x, ptr addrspace(1) nocapture writeon
 ; GFX6-IR-NEXT:    ret double [[COND6]]
 ;
 ; IR-FRACT-LABEL: define double @safe_math_fract_f64
-; IR-FRACT-SAME: (double [[X:%.*]], ptr addrspace(1) nocapture writeonly [[IP:%.*]]) #[[ATTR0]] {
+; IR-FRACT-SAME: (double [[X:%.*]], ptr addrspace(1) writeonly captures(none) [[IP:%.*]]) #[[ATTR0]] {
 ; IR-FRACT-NEXT:  entry:
 ; IR-FRACT-NEXT:    [[FLOOR:%.*]] = tail call double @llvm.floor.f64(double [[X]])
 ; IR-FRACT-NEXT:    [[COND:%.*]] = call double @llvm.amdgcn.fract.f64(double [[X]])
@@ -2247,6 +2251,7 @@ define double @safe_math_fract_f64(double %x, ptr addrspace(1) nocapture writeon
 ; GFX12-NEXT:    v_fract_f64_e32 v[4:5], v[0:1]
 ; GFX12-NEXT:    v_cmp_neq_f64_e64 vcc_lo, 0x7ff00000, |v[0:1]|
 ; GFX12-NEXT:    v_floor_f64_e32 v[6:7], v[0:1]
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3)
 ; GFX12-NEXT:    v_dual_cndmask_b32 v0, 0, v4 :: v_dual_cndmask_b32 v1, 0, v5
 ; GFX12-NEXT:    global_store_b64 v[2:3], v[6:7], off
@@ -2264,9 +2269,9 @@ entry:
   ret double %cond6
 }
 
-define half @safe_math_fract_f16(half %x, ptr addrspace(1) nocapture writeonly %ip) {
+define half @safe_math_fract_f16(half %x, ptr addrspace(1) writeonly captures(none) %ip) {
 ; GFX6-IR-LABEL: define half @safe_math_fract_f16
-; GFX6-IR-SAME: (half [[X:%.*]], ptr addrspace(1) nocapture writeonly [[IP:%.*]]) #[[ATTR0]] {
+; GFX6-IR-SAME: (half [[X:%.*]], ptr addrspace(1) writeonly captures(none) [[IP:%.*]]) #[[ATTR0]] {
 ; GFX6-IR-NEXT:  entry:
 ; GFX6-IR-NEXT:    [[FLOOR:%.*]] = tail call half @llvm.floor.f16(half [[X]])
 ; GFX6-IR-NEXT:    [[SUB:%.*]] = fsub half [[X]], [[FLOOR]]
@@ -2280,7 +2285,7 @@ define half @safe_math_fract_f16(half %x, ptr addrspace(1) nocapture writeonly %
 ; GFX6-IR-NEXT:    ret half [[COND6]]
 ;
 ; GFX7-IR-LABEL: define half @safe_math_fract_f16
-; GFX7-IR-SAME: (half [[X:%.*]], ptr addrspace(1) nocapture writeonly [[IP:%.*]]) #[[ATTR0]] {
+; GFX7-IR-SAME: (half [[X:%.*]], ptr addrspace(1) writeonly captures(none) [[IP:%.*]]) #[[ATTR0]] {
 ; GFX7-IR-NEXT:  entry:
 ; GFX7-IR-NEXT:    [[FLOOR:%.*]] = tail call half @llvm.floor.f16(half [[X]])
 ; GFX7-IR-NEXT:    [[SUB:%.*]] = fsub half [[X]], [[FLOOR]]
@@ -2294,7 +2299,7 @@ define half @safe_math_fract_f16(half %x, ptr addrspace(1) nocapture writeonly %
 ; GFX7-IR-NEXT:    ret half [[COND6]]
 ;
 ; IR-LEGALF16-LABEL: define half @safe_math_fract_f16
-; IR-LEGALF16-SAME: (half [[X:%.*]], ptr addrspace(1) nocapture writeonly [[IP:%.*]]) #[[ATTR0]] {
+; IR-LEGALF16-SAME: (half [[X:%.*]], ptr addrspace(1) writeonly captures(none) [[IP:%.*]]) #[[ATTR0]] {
 ; IR-LEGALF16-NEXT:  entry:
 ; IR-LEGALF16-NEXT:    [[FLOOR:%.*]] = tail call half @llvm.floor.f16(half [[X]])
 ; IR-LEGALF16-NEXT:    [[COND:%.*]] = call half @llvm.amdgcn.fract.f16(half [[X]])
@@ -2383,6 +2388,7 @@ define half @safe_math_fract_f16(half %x, ptr addrspace(1) nocapture writeonly %
 ; GFX12-NEXT:    v_fract_f16_e32 v3, v0
 ; GFX12-NEXT:    v_cmp_neq_f16_e64 vcc_lo, 0x7c00, |v0|
 ; GFX12-NEXT:    v_floor_f16_e32 v4, v0
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3)
 ; GFX12-NEXT:    v_cndmask_b32_e32 v0, 0, v3, vcc_lo
 ; GFX12-NEXT:    global_store_b16 v[1:2], v4, off
@@ -2400,9 +2406,9 @@ entry:
   ret half %cond6
 }
 
-define <2 x half> @safe_math_fract_v2f16(<2 x half> %x, ptr addrspace(1) nocapture writeonly %ip) {
+define <2 x half> @safe_math_fract_v2f16(<2 x half> %x, ptr addrspace(1) writeonly captures(none) %ip) {
 ; GFX6-IR-LABEL: define <2 x half> @safe_math_fract_v2f16
-; GFX6-IR-SAME: (<2 x half> [[X:%.*]], ptr addrspace(1) nocapture writeonly [[IP:%.*]]) #[[ATTR0]] {
+; GFX6-IR-SAME: (<2 x half> [[X:%.*]], ptr addrspace(1) writeonly captures(none) [[IP:%.*]]) #[[ATTR0]] {
 ; GFX6-IR-NEXT:  entry:
 ; GFX6-IR-NEXT:    [[FLOOR:%.*]] = tail call <2 x half> @llvm.floor.v2f16(<2 x half> [[X]])
 ; GFX6-IR-NEXT:    [[SUB:%.*]] = fsub <2 x half> [[X]], [[FLOOR]]
@@ -2416,7 +2422,7 @@ define <2 x half> @safe_math_fract_v2f16(<2 x half> %x, ptr addrspace(1) nocaptu
 ; GFX6-IR-NEXT:    ret <2 x half> [[COND6]]
 ;
 ; GFX7-IR-LABEL: define <2 x half> @safe_math_fract_v2f16
-; GFX7-IR-SAME: (<2 x half> [[X:%.*]], ptr addrspace(1) nocapture writeonly [[IP:%.*]]) #[[ATTR0]] {
+; GFX7-IR-SAME: (<2 x half> [[X:%.*]], ptr addrspace(1) writeonly captures(none) [[IP:%.*]]) #[[ATTR0]] {
 ; GFX7-IR-NEXT:  entry:
 ; GFX7-IR-NEXT:    [[FLOOR:%.*]] = tail call <2 x half> @llvm.floor.v2f16(<2 x half> [[X]])
 ; GFX7-IR-NEXT:    [[SUB:%.*]] = fsub <2 x half> [[X]], [[FLOOR]]
@@ -2430,7 +2436,7 @@ define <2 x half> @safe_math_fract_v2f16(<2 x half> %x, ptr addrspace(1) nocaptu
 ; GFX7-IR-NEXT:    ret <2 x half> [[COND6]]
 ;
 ; IR-LEGALF16-LABEL: define <2 x half> @safe_math_fract_v2f16
-; IR-LEGALF16-SAME: (<2 x half> [[X:%.*]], ptr addrspace(1) nocapture writeonly [[IP:%.*]]) #[[ATTR0]] {
+; IR-LEGALF16-SAME: (<2 x half> [[X:%.*]], ptr addrspace(1) writeonly captures(none) [[IP:%.*]]) #[[ATTR0]] {
 ; IR-LEGALF16-NEXT:  entry:
 ; IR-LEGALF16-NEXT:    [[FLOOR:%.*]] = tail call <2 x half> @llvm.floor.v2f16(<2 x half> [[X]])
 ; IR-LEGALF16-NEXT:    [[TMP0:%.*]] = extractelement <2 x half> [[X]], i64 0
@@ -2564,14 +2570,16 @@ define <2 x half> @safe_math_fract_v2f16(<2 x half> %x, ptr addrspace(1) nocaptu
 ; GFX12-NEXT:    v_lshrrev_b32_e32 v3, 16, v0
 ; GFX12-NEXT:    v_fract_f16_e32 v6, v0
 ; GFX12-NEXT:    v_floor_f16_e32 v5, v0
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_2)
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_2)
 ; GFX12-NEXT:    v_fract_f16_e32 v4, v3
 ; GFX12-NEXT:    v_cmp_class_f16_e64 s0, v3, 0x204
 ; GFX12-NEXT:    v_floor_f16_e32 v7, v3
+; GFX12-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-NEXT:    v_cndmask_b32_e64 v3, v4, 0, s0
 ; GFX12-NEXT:    v_cmp_class_f16_e64 s0, v0, 0x204
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX12-NEXT:    v_pack_b32_f16 v4, v5, v7
+; GFX12-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-NEXT:    v_cndmask_b32_e64 v0, v6, 0, s0
 ; GFX12-NEXT:    global_store_b32 v[1:2], v4, off
 ; GFX12-NEXT:    v_pack_b32_f16 v0, v0, v3
@@ -2589,9 +2597,9 @@ entry:
   ret <2 x half> %cond6
 }
 
-define <2 x double> @safe_math_fract_v2f64(<2 x double> %x, ptr addrspace(1) nocapture writeonly %ip) {
+define <2 x double> @safe_math_fract_v2f64(<2 x double> %x, ptr addrspace(1) writeonly captures(none) %ip) {
 ; GFX6-IR-LABEL: define <2 x double> @safe_math_fract_v2f64
-; GFX6-IR-SAME: (<2 x double> [[X:%.*]], ptr addrspace(1) nocapture writeonly [[IP:%.*]]) #[[ATTR0]] {
+; GFX6-IR-SAME: (<2 x double> [[X:%.*]], ptr addrspace(1) writeonly captures(none) [[IP:%.*]]) #[[ATTR0]] {
 ; GFX6-IR-NEXT:  entry:
 ; GFX6-IR-NEXT:    [[FLOOR:%.*]] = tail call <2 x double> @llvm.floor.v2f64(<2 x double> [[X]])
 ; GFX6-IR-NEXT:    [[SUB:%.*]] = fsub <2 x double> [[X]], [[FLOOR]]
@@ -2605,7 +2613,7 @@ define <2 x double> @safe_math_fract_v2f64(<2 x double> %x, ptr addrspace(1) noc
 ; GFX6-IR-NEXT:    ret <2 x double> [[COND6]]
 ;
 ; IR-FRACT-LABEL: define <2 x double> @safe_math_fract_v2f64
-; IR-FRACT-SAME: (<2 x double> [[X:%.*]], ptr addrspace(1) nocapture writeonly [[IP:%.*]]) #[[ATTR0]] {
+; IR-FRACT-SAME: (<2 x double> [[X:%.*]], ptr addrspace(1) writeonly captures(none) [[IP:%.*]]) #[[ATTR0]] {
 ; IR-FRACT-NEXT:  entry:
 ; IR-FRACT-NEXT:    [[FLOOR:%.*]] = tail call <2 x double> @llvm.floor.v2f64(<2 x double> [[X]])
 ; IR-FRACT-NEXT:    [[TMP0:%.*]] = extractelement <2 x double> [[X]], i64 0
@@ -2733,6 +2741,7 @@ define <2 x double> @safe_math_fract_v2f64(<2 x double> %x, ptr addrspace(1) noc
 ; GFX12-NEXT:    v_cmp_class_f64_e64 s1, v[2:3], 0x204
 ; GFX12-NEXT:    v_floor_f64_e32 v[8:9], v[2:3]
 ; GFX12-NEXT:    v_floor_f64_e32 v[6:7], v[0:1]
+; GFX12-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-NEXT:    v_cndmask_b32_e64 v0, v10, 0, s0
 ; GFX12-NEXT:    v_cndmask_b32_e64 v1, v11, 0, s0
 ; GFX12-NEXT:    v_cndmask_b32_e64 v2, v12, 0, s1
diff --git a/llvm/test/CodeGen/AMDGPU/frame-setup-without-sgpr-to-vgpr-spills.ll b/llvm/test/CodeGen/AMDGPU/frame-setup-without-sgpr-to-vgpr-spills.ll
index 6684262f5c976..13884eb788d8a 100644
--- a/llvm/test/CodeGen/AMDGPU/frame-setup-without-sgpr-to-vgpr-spills.ll
+++ b/llvm/test/CodeGen/AMDGPU/frame-setup-without-sgpr-to-vgpr-spills.ll
@@ -19,12 +19,12 @@ define void @callee_with_stack_and_call() #0 {
 ; SPILL-TO-VGPR-NEXT:    s_addk_i32 s32, 0x400
 ; SPILL-TO-VGPR-NEXT:    v_writelane_b32 v40, s30, 0
 ; SPILL-TO-VGPR-NEXT:    v_mov_b32_e32 v0, 0
-; SPILL-TO-VGPR-NEXT:    v_writelane_b32 v40, s31, 1
-; SPILL-TO-VGPR-NEXT:    buffer_store_dword v0, off, s[0:3], s33
-; SPILL-TO-VGPR-NEXT:    s_waitcnt vmcnt(0)
 ; SPILL-TO-VGPR-NEXT:    s_getpc_b64 s[4:5]
 ; SPILL-TO-VGPR-NEXT:    s_add_u32 s4, s4, external_void_func_void@rel32@lo+4
 ; SPILL-TO-VGPR-NEXT:    s_addc_u32 s5, s5, external_void_func_void@rel32@hi+12
+; SPILL-TO-VGPR-NEXT:    v_writelane_b32 v40, s31, 1
+; SPILL-TO-VGPR-NEXT:    buffer_store_dword v0, off, s[0:3], s33
+; SPILL-TO-VGPR-NEXT:    s_waitcnt vmcnt(0)
 ; SPILL-TO-VGPR-NEXT:    s_swappc_b64 s[30:31], s[4:5]
 ; SPILL-TO-VGPR-NEXT:    v_readlane_b32 s31, v40, 1
 ; SPILL-TO-VGPR-NEXT:    v_readlane_b32 s30, v40, 0
@@ -62,11 +62,11 @@ define void @callee_with_stack_and_call() #0 {
 ; NO-SPILL-TO-VGPR-NEXT:    s_waitcnt vmcnt(0)
 ; NO-SPILL-TO-VGPR-NEXT:    s_mov_b64 exec, s[4:5]
 ; NO-SPILL-TO-VGPR-NEXT:    v_mov_b32_e32 v0, 0
-; NO-SPILL-TO-VGPR-NEXT:    buffer_store_dword v0, off, s[0:3], s33
-; NO-SPILL-TO-VGPR-NEXT:    s_waitcnt vmcnt(0)
 ; NO-SPILL-TO-VGPR-NEXT:    s_getpc_b64 s[4:5]
 ; NO-SPILL-TO-VGPR-NEXT:    s_add_u32 s4, s4, external_void_func_void@rel32@lo+4
 ; NO-SPILL-TO-VGPR-NEXT:    s_addc_u32 s5, s5, external_void_func_void@rel32@hi+12
+; NO-SPILL-TO-VGPR-NEXT:    buffer_store_dword v0, off, s[0:3], s33
+; NO-SPILL-TO-VGPR-NEXT:    s_waitcnt vmcnt(0)
 ; NO-SPILL-TO-VGPR-NEXT:    s_swappc_b64 s[30:31], s[4:5]
 ; NO-SPILL-TO-VGPR-NEXT:    s_mov_b64 s[4:5], exec
 ; NO-SPILL-TO-VGPR-NEXT:    s_mov_b64 exec, 1
diff --git a/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fadd.ll b/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fadd.ll
index 2be6bf302d35f..73b4428b03c81 100644
--- a/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fadd.ll
+++ b/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fadd.ll
@@ -6936,7 +6936,6 @@ define double @global_agent_atomic_fadd_ret_f64__amdgpu_no_fine_grained_memory(p
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX12-NEXT:    v_dual_mov_b32 v0, v4 :: v_dual_mov_b32 v1, v5
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: global_agent_atomic_fadd_ret_f64__amdgpu_no_fine_grained_memory:
@@ -7158,7 +7157,6 @@ define double @global_agent_atomic_fadd_ret_f64__offset12b_pos__amdgpu_no_fine_g
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX12-NEXT:    v_dual_mov_b32 v0, v4 :: v_dual_mov_b32 v1, v5
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: global_agent_atomic_fadd_ret_f64__offset12b_pos__amdgpu_no_fine_grained_memory:
@@ -7381,7 +7379,6 @@ define double @global_agent_atomic_fadd_ret_f64__offset12b_neg__amdgpu_no_fine_g
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX12-NEXT:    v_dual_mov_b32 v0, v4 :: v_dual_mov_b32 v1, v5
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: global_agent_atomic_fadd_ret_f64__offset12b_neg__amdgpu_no_fine_grained_memory:
@@ -7610,7 +7607,6 @@ define void @global_agent_atomic_fadd_noret_f64__amdgpu_no_fine_grained_memory(p
 ; GFX12-NEXT:    s_cbranch_execnz .LBB41_1
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: global_agent_atomic_fadd_noret_f64__amdgpu_no_fine_grained_memory:
@@ -7814,7 +7810,6 @@ define void @global_agent_atomic_fadd_noret_f64__offset12b_pos__amdgpu_no_fine_g
 ; GFX12-NEXT:    s_cbranch_execnz .LBB42_1
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: global_agent_atomic_fadd_noret_f64__offset12b_pos__amdgpu_no_fine_grained_memory:
@@ -8021,7 +8016,6 @@ define void @global_agent_atomic_fadd_noret_f64__offset12b_neg__amdgpu_no_fine_g
 ; GFX12-NEXT:    s_cbranch_execnz .LBB43_1
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: global_agent_atomic_fadd_noret_f64__offset12b_neg__amdgpu_no_fine_grained_memory:
@@ -8256,7 +8250,6 @@ define half @global_agent_atomic_fadd_ret_f16__amdgpu_no_fine_grained_memory(ptr
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX12-NEXT:    v_lshrrev_b32_e32 v0, v3, v5
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: global_agent_atomic_fadd_ret_f16__amdgpu_no_fine_grained_memory:
@@ -8561,6 +8554,7 @@ define half @global_agent_atomic_fadd_ret_f16__offset12b_pos__amdgpu_no_fine_gra
 ; GFX12-NEXT:    s_wait_bvhcnt 0x0
 ; GFX12-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-NEXT:    v_add_co_u32 v3, vcc_lo, 0x7fe, v0
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
 ; GFX12-NEXT:    s_mov_b32 s0, 0
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_1)
@@ -8596,7 +8590,6 @@ define half @global_agent_atomic_fadd_ret_f16__offset12b_pos__amdgpu_no_fine_gra
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX12-NEXT:    v_lshrrev_b32_e32 v0, v3, v5
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: global_agent_atomic_fadd_ret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
@@ -8912,6 +8905,7 @@ define half @global_agent_atomic_fadd_ret_f16__offset12b_neg__amdgpu_no_fine_gra
 ; GFX12-NEXT:    s_wait_bvhcnt 0x0
 ; GFX12-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-NEXT:    v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, -1, v1, vcc_lo
 ; GFX12-NEXT:    s_mov_b32 s0, 0
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_1)
@@ -8947,7 +8941,6 @@ define half @global_agent_atomic_fadd_ret_f16__offset12b_neg__amdgpu_no_fine_gra
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX12-NEXT:    v_lshrrev_b32_e32 v0, v3, v5
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: global_agent_atomic_fadd_ret_f16__offset12b_neg__amdgpu_no_fine_grained_memory:
@@ -9296,7 +9289,6 @@ define void @global_agent_atomic_fadd_noret_f16__amdgpu_no_fine_grained_memory(p
 ; GFX12-NEXT:    s_cbranch_execnz .LBB47_1
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: global_agent_atomic_fadd_noret_f16__amdgpu_no_fine_grained_memory:
@@ -9591,6 +9583,7 @@ define void @global_agent_atomic_fadd_noret_f16__offset12b_pos__amdgpu_no_fine_g
 ; GFX12-NEXT:    s_wait_bvhcnt 0x0
 ; GFX12-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-NEXT:    v_add_co_u32 v3, vcc_lo, 0x7fe, v0
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
 ; GFX12-NEXT:    s_mov_b32 s0, 0
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_1)
@@ -9624,7 +9617,6 @@ define void @global_agent_atomic_fadd_noret_f16__offset12b_pos__amdgpu_no_fine_g
 ; GFX12-NEXT:    s_cbranch_execnz .LBB48_1
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: global_agent_atomic_fadd_noret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
@@ -9929,6 +9921,7 @@ define void @global_agent_atomic_fadd_noret_f16__offset12b_neg__amdgpu_no_fine_g
 ; GFX12-NEXT:    s_wait_bvhcnt 0x0
 ; GFX12-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-NEXT:    v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, -1, v1, vcc_lo
 ; GFX12-NEXT:    s_mov_b32 s0, 0
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_1)
@@ -9962,7 +9955,6 @@ define void @global_agent_atomic_fadd_noret_f16__offset12b_neg__amdgpu_no_fine_g
 ; GFX12-NEXT:    s_cbranch_execnz .LBB49_1
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: global_agent_atomic_fadd_noret_f16__offset12b_neg__amdgpu_no_fine_grained_memory:
@@ -10291,7 +10283,6 @@ define half @global_agent_atomic_fadd_ret_f16__offset12b_pos__align4__amdgpu_no_
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX12-NEXT:    v_mov_b32_e32 v0, v3
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: global_agent_atomic_fadd_ret_f16__offset12b_pos__align4__amdgpu_no_fine_grained_memory:
@@ -10551,7 +10542,6 @@ define void @global_agent_atomic_fadd_noret_f16__offset12b__align4_pos__amdgpu_n
 ; GFX12-NEXT:    s_cbranch_execnz .LBB51_1
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: global_agent_atomic_fadd_noret_f16__offset12b__align4_pos__amdgpu_no_fine_grained_memory:
@@ -10782,6 +10772,7 @@ define half @global_system_atomic_fadd_ret_f16__offset12b_pos__amdgpu_no_fine_gr
 ; GFX12-NEXT:    s_wait_bvhcnt 0x0
 ; GFX12-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-NEXT:    v_add_co_u32 v3, vcc_lo, 0x7fe, v0
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
 ; GFX12-NEXT:    s_mov_b32 s0, 0
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_1)
@@ -10818,7 +10809,6 @@ define half @global_system_atomic_fadd_ret_f16__offset12b_pos__amdgpu_no_fine_gr
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX12-NEXT:    v_lshrrev_b32_e32 v0, v3, v5
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: global_system_atomic_fadd_ret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
@@ -11136,6 +11126,7 @@ define void @global_system_atomic_fadd_noret_f16__offset12b_pos__amdgpu_no_fine_
 ; GFX12-NEXT:    s_wait_bvhcnt 0x0
 ; GFX12-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-NEXT:    v_add_co_u32 v3, vcc_lo, 0x7fe, v0
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
 ; GFX12-NEXT:    s_mov_b32 s0, 0
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_1)
@@ -11170,7 +11161,6 @@ define void @global_system_atomic_fadd_noret_f16__offset12b_pos__amdgpu_no_fine_
 ; GFX12-NEXT:    s_cbranch_execnz .LBB53_1
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: global_system_atomic_fadd_noret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
@@ -11502,8 +11492,9 @@ define bfloat @global_agent_atomic_fadd_ret_bf16__amdgpu_no_fine_grained_memory(
 ; GFX12-NEXT:    v_bfe_u32 v7, v5, 16, 1
 ; GFX12-NEXT:    v_or_b32_e32 v8, 0x400000, v5
 ; GFX12-NEXT:    v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_cndmask_b32_e32 v5, v7, v8, vcc_lo
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_lshrrev_b32_e32 v5, 16, v5
@@ -11523,7 +11514,6 @@ define bfloat @global_agent_atomic_fadd_ret_bf16__amdgpu_no_fine_grained_memory(
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX12-NEXT:    v_lshrrev_b32_e32 v0, v3, v5
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: global_agent_atomic_fadd_ret_bf16__amdgpu_no_fine_grained_memory:
@@ -11873,6 +11863,7 @@ define bfloat @global_agent_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_
 ; GFX12-NEXT:    s_wait_bvhcnt 0x0
 ; GFX12-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-NEXT:    v_add_co_u32 v3, vcc_lo, 0x7fe, v0
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
 ; GFX12-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
@@ -11896,8 +11887,9 @@ define bfloat @global_agent_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_
 ; GFX12-NEXT:    v_bfe_u32 v7, v5, 16, 1
 ; GFX12-NEXT:    v_or_b32_e32 v8, 0x400000, v5
 ; GFX12-NEXT:    v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_cndmask_b32_e32 v5, v7, v8, vcc_lo
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_lshrrev_b32_e32 v5, 16, v5
@@ -11917,7 +11909,6 @@ define bfloat @global_agent_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX12-NEXT:    v_lshrrev_b32_e32 v0, v3, v5
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: global_agent_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
@@ -12279,6 +12270,7 @@ define bfloat @global_agent_atomic_fadd_ret_bf16__offset12b_neg__amdgpu_no_fine_
 ; GFX12-NEXT:    s_wait_bvhcnt 0x0
 ; GFX12-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-NEXT:    v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, -1, v1, vcc_lo
 ; GFX12-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
@@ -12302,8 +12294,9 @@ define bfloat @global_agent_atomic_fadd_ret_bf16__offset12b_neg__amdgpu_no_fine_
 ; GFX12-NEXT:    v_bfe_u32 v7, v5, 16, 1
 ; GFX12-NEXT:    v_or_b32_e32 v8, 0x400000, v5
 ; GFX12-NEXT:    v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_cndmask_b32_e32 v5, v7, v8, vcc_lo
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_lshrrev_b32_e32 v5, 16, v5
@@ -12323,7 +12316,6 @@ define bfloat @global_agent_atomic_fadd_ret_bf16__offset12b_neg__amdgpu_no_fine_
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX12-NEXT:    v_lshrrev_b32_e32 v0, v3, v5
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: global_agent_atomic_fadd_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
@@ -12707,6 +12699,7 @@ define void @global_agent_atomic_fadd_noret_bf16__amdgpu_no_fine_grained_memory(
 ; GFX12-NEXT:    v_or_b32_e32 v8, 0x400000, v3
 ; GFX12-NEXT:    v_cmp_u_f32_e32 vcc_lo, v3, v3
 ; GFX12-NEXT:    v_add3_u32 v7, v7, v3, 0x7fff
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_cndmask_b32_e32 v3, v7, v8, vcc_lo
 ; GFX12-NEXT:    v_lshrrev_b32_e32 v3, 16, v3
@@ -12726,7 +12719,6 @@ define void @global_agent_atomic_fadd_noret_bf16__amdgpu_no_fine_grained_memory(
 ; GFX12-NEXT:    s_cbranch_execnz .LBB57_1
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: global_agent_atomic_fadd_noret_bf16__amdgpu_no_fine_grained_memory:
@@ -13066,6 +13058,7 @@ define void @global_agent_atomic_fadd_noret_bf16__offset12b_pos__amdgpu_no_fine_
 ; GFX12-NEXT:    s_wait_bvhcnt 0x0
 ; GFX12-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-NEXT:    v_add_co_u32 v4, vcc_lo, 0x7fe, v0
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
 ; GFX12-NEXT:    v_lshlrev_b32_e32 v6, 16, v2
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
@@ -13089,6 +13082,7 @@ define void @global_agent_atomic_fadd_noret_bf16__offset12b_pos__amdgpu_no_fine_
 ; GFX12-NEXT:    v_or_b32_e32 v8, 0x400000, v2
 ; GFX12-NEXT:    v_cmp_u_f32_e32 vcc_lo, v2, v2
 ; GFX12-NEXT:    v_add3_u32 v7, v7, v2, 0x7fff
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_cndmask_b32_e32 v2, v7, v8, vcc_lo
 ; GFX12-NEXT:    v_lshrrev_b32_e32 v2, 16, v2
@@ -13108,7 +13102,6 @@ define void @global_agent_atomic_fadd_noret_bf16__offset12b_pos__amdgpu_no_fine_
 ; GFX12-NEXT:    s_cbranch_execnz .LBB58_1
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: global_agent_atomic_fadd_noret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
@@ -13459,6 +13452,7 @@ define void @global_agent_atomic_fadd_noret_bf16__offset12b_neg__amdgpu_no_fine_
 ; GFX12-NEXT:    s_wait_bvhcnt 0x0
 ; GFX12-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-NEXT:    v_add_co_u32 v4, vcc_lo, 0xfffff800, v0
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, -1, v1, vcc_lo
 ; GFX12-NEXT:    v_lshlrev_b32_e32 v6, 16, v2
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
@@ -13482,6 +13476,7 @@ define void @global_agent_atomic_fadd_noret_bf16__offset12b_neg__amdgpu_no_fine_
 ; GFX12-NEXT:    v_or_b32_e32 v8, 0x400000, v2
 ; GFX12-NEXT:    v_cmp_u_f32_e32 vcc_lo, v2, v2
 ; GFX12-NEXT:    v_add3_u32 v7, v7, v2, 0x7fff
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_cndmask_b32_e32 v2, v7, v8, vcc_lo
 ; GFX12-NEXT:    v_lshrrev_b32_e32 v2, 16, v2
@@ -13501,7 +13496,6 @@ define void @global_agent_atomic_fadd_noret_bf16__offset12b_neg__amdgpu_no_fine_
 ; GFX12-NEXT:    s_cbranch_execnz .LBB59_1
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: global_agent_atomic_fadd_noret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
@@ -13867,6 +13861,7 @@ define bfloat @global_agent_atomic_fadd_ret_bf16__offset12b_pos__align4__amdgpu_
 ; GFX12-NEXT:    v_or_b32_e32 v6, 0x400000, v3
 ; GFX12-NEXT:    v_cmp_u_f32_e32 vcc_lo, v3, v3
 ; GFX12-NEXT:    v_add3_u32 v5, v5, v3, 0x7fff
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_cndmask_b32_e32 v3, v5, v6, vcc_lo
 ; GFX12-NEXT:    v_lshrrev_b32_e32 v3, 16, v3
@@ -13885,7 +13880,6 @@ define bfloat @global_agent_atomic_fadd_ret_bf16__offset12b_pos__align4__amdgpu_
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX12-NEXT:    v_mov_b32_e32 v0, v3
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: global_agent_atomic_fadd_ret_bf16__offset12b_pos__align4__amdgpu_no_fine_grained_memory:
@@ -14188,8 +14182,9 @@ define void @global_agent_atomic_fadd_noret_bf16__offset12b__align4_pos__amdgpu_
 ; GFX12-NEXT:    v_bfe_u32 v5, v2, 16, 1
 ; GFX12-NEXT:    v_or_b32_e32 v6, 0x400000, v2
 ; GFX12-NEXT:    v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_add3_u32 v5, v5, v2, 0x7fff
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_cndmask_b32_e32 v2, v5, v6, vcc_lo
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_lshrrev_b32_e32 v2, 16, v2
@@ -14207,7 +14202,6 @@ define void @global_agent_atomic_fadd_noret_bf16__offset12b__align4_pos__amdgpu_
 ; GFX12-NEXT:    s_cbranch_execnz .LBB61_1
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: global_agent_atomic_fadd_noret_bf16__offset12b__align4_pos__amdgpu_no_fine_grained_memory:
@@ -14491,6 +14485,7 @@ define bfloat @global_system_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine
 ; GFX12-NEXT:    s_wait_bvhcnt 0x0
 ; GFX12-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-NEXT:    v_add_co_u32 v3, vcc_lo, 0x7fe, v0
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
 ; GFX12-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
@@ -14514,8 +14509,9 @@ define bfloat @global_system_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine
 ; GFX12-NEXT:    v_bfe_u32 v7, v5, 16, 1
 ; GFX12-NEXT:    v_or_b32_e32 v8, 0x400000, v5
 ; GFX12-NEXT:    v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_cndmask_b32_e32 v5, v7, v8, vcc_lo
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_lshrrev_b32_e32 v5, 16, v5
@@ -14536,7 +14532,6 @@ define bfloat @global_system_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX12-NEXT:    v_lshrrev_b32_e32 v0, v3, v5
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: global_system_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
@@ -14900,6 +14895,7 @@ define void @global_system_atomic_fadd_noret_bf16__offset12b_pos__amdgpu_no_fine
 ; GFX12-NEXT:    s_wait_bvhcnt 0x0
 ; GFX12-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-NEXT:    v_add_co_u32 v4, vcc_lo, 0x7fe, v0
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
 ; GFX12-NEXT:    v_lshlrev_b32_e32 v6, 16, v2
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
@@ -14923,6 +14919,7 @@ define void @global_system_atomic_fadd_noret_bf16__offset12b_pos__amdgpu_no_fine
 ; GFX12-NEXT:    v_or_b32_e32 v8, 0x400000, v2
 ; GFX12-NEXT:    v_cmp_u_f32_e32 vcc_lo, v2, v2
 ; GFX12-NEXT:    v_add3_u32 v7, v7, v2, 0x7fff
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_cndmask_b32_e32 v2, v7, v8, vcc_lo
 ; GFX12-NEXT:    v_lshrrev_b32_e32 v2, 16, v2
@@ -14943,7 +14940,6 @@ define void @global_system_atomic_fadd_noret_bf16__offset12b_pos__amdgpu_no_fine
 ; GFX12-NEXT:    s_cbranch_execnz .LBB63_1
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: global_system_atomic_fadd_noret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
@@ -23072,7 +23068,6 @@ define amdgpu_kernel void @infer_as_before_atomic(ptr addrspace(4) %arg) #1 {
 ; GFX12-NEXT:    s_cbranch_execz .LBB92_2
 ; GFX12-NEXT:  ; %bb.1:
 ; GFX12-NEXT:    s_load_b64 s[2:3], s[4:5], 0x0
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_bcnt1_i32_b32 s0, s0
 ; GFX12-NEXT:    v_mov_b32_e32 v0, 0
 ; GFX12-NEXT:    s_wait_alu 0xfffe
diff --git a/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fmax.ll b/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fmax.ll
index e2fde562d36b1..cd6ed1e6b98c2 100644
--- a/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fmax.ll
+++ b/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fmax.ll
@@ -3013,7 +3013,6 @@ define double @global_agent_atomic_fmax_ret_f64__amdgpu_no_fine_grained_memory(p
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX12-NEXT:    v_dual_mov_b32 v0, v4 :: v_dual_mov_b32 v1, v5
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: global_agent_atomic_fmax_ret_f64__amdgpu_no_fine_grained_memory:
@@ -3186,7 +3185,6 @@ define double @global_agent_atomic_fmax_ret_f64__offset12b_pos__amdgpu_no_fine_g
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX12-NEXT:    v_dual_mov_b32 v0, v4 :: v_dual_mov_b32 v1, v5
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: global_agent_atomic_fmax_ret_f64__offset12b_pos__amdgpu_no_fine_grained_memory:
@@ -3360,7 +3358,6 @@ define double @global_agent_atomic_fmax_ret_f64__offset12b_neg__amdgpu_no_fine_g
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX12-NEXT:    v_dual_mov_b32 v0, v4 :: v_dual_mov_b32 v1, v5
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: global_agent_atomic_fmax_ret_f64__offset12b_neg__amdgpu_no_fine_grained_memory:
@@ -3533,7 +3530,6 @@ define void @global_agent_atomic_fmax_noret_f64__amdgpu_no_fine_grained_memory(p
 ; GFX12-NEXT:    s_cbranch_execnz .LBB21_1
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: global_agent_atomic_fmax_noret_f64__amdgpu_no_fine_grained_memory:
@@ -3696,7 +3692,6 @@ define void @global_agent_atomic_fmax_noret_f64__offset12b_pos__amdgpu_no_fine_g
 ; GFX12-NEXT:    s_cbranch_execnz .LBB22_1
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: global_agent_atomic_fmax_noret_f64__offset12b_pos__amdgpu_no_fine_grained_memory:
@@ -3862,7 +3857,6 @@ define void @global_agent_atomic_fmax_noret_f64__offset12b_neg__amdgpu_no_fine_g
 ; GFX12-NEXT:    s_cbranch_execnz .LBB23_1
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: global_agent_atomic_fmax_noret_f64__offset12b_neg__amdgpu_no_fine_grained_memory:
@@ -4029,7 +4023,6 @@ define double @global_agent_atomic_fmax_ret_f64__amdgpu_no_remote_memory(ptr add
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX12-NEXT:    v_dual_mov_b32 v0, v4 :: v_dual_mov_b32 v1, v5
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: global_agent_atomic_fmax_ret_f64__amdgpu_no_remote_memory:
@@ -4278,7 +4271,6 @@ define double @global_agent_atomic_fmax_ret_f64__amdgpu_no_fine_grained_memory__
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX12-NEXT:    v_dual_mov_b32 v0, v4 :: v_dual_mov_b32 v1, v5
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: global_agent_atomic_fmax_ret_f64__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
@@ -4469,7 +4461,6 @@ define half @global_agent_atomic_fmax_ret_f16__amdgpu_no_fine_grained_memory(ptr
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX12-NEXT:    v_lshrrev_b32_e32 v0, v3, v5
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: global_agent_atomic_fmax_ret_f16__amdgpu_no_fine_grained_memory:
@@ -4786,6 +4777,7 @@ define half @global_agent_atomic_fmax_ret_f16__offset12b_pos__amdgpu_no_fine_gra
 ; GFX12-NEXT:    s_wait_bvhcnt 0x0
 ; GFX12-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-NEXT:    v_add_co_u32 v3, vcc_lo, 0x7fe, v0
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
 ; GFX12-NEXT:    v_max_num_f16_e32 v2, v2, v2
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
@@ -4823,7 +4815,6 @@ define half @global_agent_atomic_fmax_ret_f16__offset12b_pos__amdgpu_no_fine_gra
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX12-NEXT:    v_lshrrev_b32_e32 v0, v3, v5
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: global_agent_atomic_fmax_ret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
@@ -5151,6 +5142,7 @@ define half @global_agent_atomic_fmax_ret_f16__offset12b_neg__amdgpu_no_fine_gra
 ; GFX12-NEXT:    s_wait_bvhcnt 0x0
 ; GFX12-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-NEXT:    v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, -1, v1, vcc_lo
 ; GFX12-NEXT:    v_max_num_f16_e32 v2, v2, v2
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
@@ -5188,7 +5180,6 @@ define half @global_agent_atomic_fmax_ret_f16__offset12b_neg__amdgpu_no_fine_gra
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX12-NEXT:    v_lshrrev_b32_e32 v0, v3, v5
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: global_agent_atomic_fmax_ret_f16__offset12b_neg__amdgpu_no_fine_grained_memory:
@@ -5552,7 +5543,6 @@ define void @global_agent_atomic_fmax_noret_f16__amdgpu_no_fine_grained_memory(p
 ; GFX12-NEXT:    s_cbranch_execnz .LBB29_1
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: global_agent_atomic_fmax_noret_f16__amdgpu_no_fine_grained_memory:
@@ -5860,6 +5850,7 @@ define void @global_agent_atomic_fmax_noret_f16__offset12b_pos__amdgpu_no_fine_g
 ; GFX12-NEXT:    s_wait_bvhcnt 0x0
 ; GFX12-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-NEXT:    v_add_co_u32 v4, vcc_lo, 0x7fe, v0
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
 ; GFX12-NEXT:    v_max_num_f16_e32 v6, v2, v2
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
@@ -5896,7 +5887,6 @@ define void @global_agent_atomic_fmax_noret_f16__offset12b_pos__amdgpu_no_fine_g
 ; GFX12-NEXT:    s_cbranch_execnz .LBB30_1
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: global_agent_atomic_fmax_noret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
@@ -6214,6 +6204,7 @@ define void @global_agent_atomic_fmax_noret_f16__offset12b_neg__amdgpu_no_fine_g
 ; GFX12-NEXT:    s_wait_bvhcnt 0x0
 ; GFX12-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-NEXT:    v_add_co_u32 v4, vcc_lo, 0xfffff800, v0
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, -1, v1, vcc_lo
 ; GFX12-NEXT:    v_max_num_f16_e32 v6, v2, v2
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
@@ -6250,7 +6241,6 @@ define void @global_agent_atomic_fmax_noret_f16__offset12b_neg__amdgpu_no_fine_g
 ; GFX12-NEXT:    s_cbranch_execnz .LBB31_1
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: global_agent_atomic_fmax_noret_f16__offset12b_neg__amdgpu_no_fine_grained_memory:
@@ -6594,7 +6584,6 @@ define half @global_agent_atomic_fmax_ret_f16__offset12b_pos__align4__amdgpu_no_
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX12-NEXT:    v_mov_b32_e32 v0, v3
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: global_agent_atomic_fmax_ret_f16__offset12b_pos__align4__amdgpu_no_fine_grained_memory:
@@ -6869,7 +6858,6 @@ define void @global_agent_atomic_fmax_noret_f16__offset12b__align4_pos__amdgpu_n
 ; GFX12-NEXT:    s_cbranch_execnz .LBB33_1
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: global_agent_atomic_fmax_noret_f16__offset12b__align4_pos__amdgpu_no_fine_grained_memory:
@@ -7113,6 +7101,7 @@ define half @global_system_atomic_fmax_ret_f16__offset12b_pos__amdgpu_no_fine_gr
 ; GFX12-NEXT:    s_wait_bvhcnt 0x0
 ; GFX12-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-NEXT:    v_add_co_u32 v3, vcc_lo, 0x7fe, v0
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
 ; GFX12-NEXT:    v_max_num_f16_e32 v2, v2, v2
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
@@ -7151,7 +7140,6 @@ define half @global_system_atomic_fmax_ret_f16__offset12b_pos__amdgpu_no_fine_gr
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX12-NEXT:    v_lshrrev_b32_e32 v0, v3, v5
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: global_system_atomic_fmax_ret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
@@ -7481,6 +7469,7 @@ define void @global_system_atomic_fmax_noret_f16__offset12b_pos__amdgpu_no_fine_
 ; GFX12-NEXT:    s_wait_bvhcnt 0x0
 ; GFX12-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-NEXT:    v_add_co_u32 v4, vcc_lo, 0x7fe, v0
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
 ; GFX12-NEXT:    v_max_num_f16_e32 v6, v2, v2
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
@@ -7518,7 +7507,6 @@ define void @global_system_atomic_fmax_noret_f16__offset12b_pos__amdgpu_no_fine_
 ; GFX12-NEXT:    s_cbranch_execnz .LBB35_1
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: global_system_atomic_fmax_noret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
@@ -7863,8 +7851,9 @@ define bfloat @global_agent_atomic_fmax_ret_bf16__amdgpu_no_fine_grained_memory(
 ; GFX12-NEXT:    v_bfe_u32 v7, v5, 16, 1
 ; GFX12-NEXT:    v_or_b32_e32 v8, 0x400000, v5
 ; GFX12-NEXT:    v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_cndmask_b32_e32 v5, v7, v8, vcc_lo
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_lshrrev_b32_e32 v5, 16, v5
@@ -7884,7 +7873,6 @@ define bfloat @global_agent_atomic_fmax_ret_bf16__amdgpu_no_fine_grained_memory(
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX12-NEXT:    v_lshrrev_b32_e32 v0, v3, v5
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: global_agent_atomic_fmax_ret_bf16__amdgpu_no_fine_grained_memory:
@@ -8236,6 +8224,7 @@ define bfloat @global_agent_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_
 ; GFX12-NEXT:    s_wait_bvhcnt 0x0
 ; GFX12-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-NEXT:    v_add_co_u32 v3, vcc_lo, 0x7fe, v0
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
 ; GFX12-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
@@ -8259,8 +8248,9 @@ define bfloat @global_agent_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_
 ; GFX12-NEXT:    v_bfe_u32 v7, v5, 16, 1
 ; GFX12-NEXT:    v_or_b32_e32 v8, 0x400000, v5
 ; GFX12-NEXT:    v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_cndmask_b32_e32 v5, v7, v8, vcc_lo
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_lshrrev_b32_e32 v5, 16, v5
@@ -8280,7 +8270,6 @@ define bfloat @global_agent_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX12-NEXT:    v_lshrrev_b32_e32 v0, v3, v5
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: global_agent_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
@@ -8644,6 +8633,7 @@ define bfloat @global_agent_atomic_fmax_ret_bf16__offset12b_neg__amdgpu_no_fine_
 ; GFX12-NEXT:    s_wait_bvhcnt 0x0
 ; GFX12-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-NEXT:    v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, -1, v1, vcc_lo
 ; GFX12-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
@@ -8667,8 +8657,9 @@ define bfloat @global_agent_atomic_fmax_ret_bf16__offset12b_neg__amdgpu_no_fine_
 ; GFX12-NEXT:    v_bfe_u32 v7, v5, 16, 1
 ; GFX12-NEXT:    v_or_b32_e32 v8, 0x400000, v5
 ; GFX12-NEXT:    v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_cndmask_b32_e32 v5, v7, v8, vcc_lo
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_lshrrev_b32_e32 v5, 16, v5
@@ -8688,7 +8679,6 @@ define bfloat @global_agent_atomic_fmax_ret_bf16__offset12b_neg__amdgpu_no_fine_
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX12-NEXT:    v_lshrrev_b32_e32 v0, v3, v5
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: global_agent_atomic_fmax_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
@@ -9074,6 +9064,7 @@ define void @global_agent_atomic_fmax_noret_bf16__amdgpu_no_fine_grained_memory(
 ; GFX12-NEXT:    v_or_b32_e32 v8, 0x400000, v3
 ; GFX12-NEXT:    v_cmp_u_f32_e32 vcc_lo, v3, v3
 ; GFX12-NEXT:    v_add3_u32 v7, v7, v3, 0x7fff
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_cndmask_b32_e32 v3, v7, v8, vcc_lo
 ; GFX12-NEXT:    v_lshrrev_b32_e32 v3, 16, v3
@@ -9093,7 +9084,6 @@ define void @global_agent_atomic_fmax_noret_bf16__amdgpu_no_fine_grained_memory(
 ; GFX12-NEXT:    s_cbranch_execnz .LBB39_1
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: global_agent_atomic_fmax_noret_bf16__amdgpu_no_fine_grained_memory:
@@ -9435,6 +9425,7 @@ define void @global_agent_atomic_fmax_noret_bf16__offset12b_pos__amdgpu_no_fine_
 ; GFX12-NEXT:    s_wait_bvhcnt 0x0
 ; GFX12-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-NEXT:    v_add_co_u32 v4, vcc_lo, 0x7fe, v0
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
 ; GFX12-NEXT:    v_lshlrev_b32_e32 v6, 16, v2
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
@@ -9458,6 +9449,7 @@ define void @global_agent_atomic_fmax_noret_bf16__offset12b_pos__amdgpu_no_fine_
 ; GFX12-NEXT:    v_or_b32_e32 v8, 0x400000, v2
 ; GFX12-NEXT:    v_cmp_u_f32_e32 vcc_lo, v2, v2
 ; GFX12-NEXT:    v_add3_u32 v7, v7, v2, 0x7fff
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_cndmask_b32_e32 v2, v7, v8, vcc_lo
 ; GFX12-NEXT:    v_lshrrev_b32_e32 v2, 16, v2
@@ -9477,7 +9469,6 @@ define void @global_agent_atomic_fmax_noret_bf16__offset12b_pos__amdgpu_no_fine_
 ; GFX12-NEXT:    s_cbranch_execnz .LBB40_1
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: global_agent_atomic_fmax_noret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
@@ -9830,6 +9821,7 @@ define void @global_agent_atomic_fmax_noret_bf16__offset12b_neg__amdgpu_no_fine_
 ; GFX12-NEXT:    s_wait_bvhcnt 0x0
 ; GFX12-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-NEXT:    v_add_co_u32 v4, vcc_lo, 0xfffff800, v0
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, -1, v1, vcc_lo
 ; GFX12-NEXT:    v_lshlrev_b32_e32 v6, 16, v2
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
@@ -9853,6 +9845,7 @@ define void @global_agent_atomic_fmax_noret_bf16__offset12b_neg__amdgpu_no_fine_
 ; GFX12-NEXT:    v_or_b32_e32 v8, 0x400000, v2
 ; GFX12-NEXT:    v_cmp_u_f32_e32 vcc_lo, v2, v2
 ; GFX12-NEXT:    v_add3_u32 v7, v7, v2, 0x7fff
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_cndmask_b32_e32 v2, v7, v8, vcc_lo
 ; GFX12-NEXT:    v_lshrrev_b32_e32 v2, 16, v2
@@ -9872,7 +9865,6 @@ define void @global_agent_atomic_fmax_noret_bf16__offset12b_neg__amdgpu_no_fine_
 ; GFX12-NEXT:    s_cbranch_execnz .LBB41_1
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: global_agent_atomic_fmax_noret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
@@ -10240,6 +10232,7 @@ define bfloat @global_agent_atomic_fmax_ret_bf16__offset12b_pos__align4__amdgpu_
 ; GFX12-NEXT:    v_or_b32_e32 v6, 0x400000, v3
 ; GFX12-NEXT:    v_cmp_u_f32_e32 vcc_lo, v3, v3
 ; GFX12-NEXT:    v_add3_u32 v5, v5, v3, 0x7fff
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_cndmask_b32_e32 v3, v5, v6, vcc_lo
 ; GFX12-NEXT:    v_lshrrev_b32_e32 v3, 16, v3
@@ -10258,7 +10251,6 @@ define bfloat @global_agent_atomic_fmax_ret_bf16__offset12b_pos__align4__amdgpu_
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX12-NEXT:    v_mov_b32_e32 v0, v3
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: global_agent_atomic_fmax_ret_bf16__offset12b_pos__align4__amdgpu_no_fine_grained_memory:
@@ -10563,8 +10555,9 @@ define void @global_agent_atomic_fmax_noret_bf16__offset12b__align4_pos__amdgpu_
 ; GFX12-NEXT:    v_bfe_u32 v5, v2, 16, 1
 ; GFX12-NEXT:    v_or_b32_e32 v6, 0x400000, v2
 ; GFX12-NEXT:    v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_add3_u32 v5, v5, v2, 0x7fff
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_cndmask_b32_e32 v2, v5, v6, vcc_lo
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_lshrrev_b32_e32 v2, 16, v2
@@ -10582,7 +10575,6 @@ define void @global_agent_atomic_fmax_noret_bf16__offset12b__align4_pos__amdgpu_
 ; GFX12-NEXT:    s_cbranch_execnz .LBB43_1
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: global_agent_atomic_fmax_noret_bf16__offset12b__align4_pos__amdgpu_no_fine_grained_memory:
@@ -10868,6 +10860,7 @@ define bfloat @global_system_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine
 ; GFX12-NEXT:    s_wait_bvhcnt 0x0
 ; GFX12-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-NEXT:    v_add_co_u32 v3, vcc_lo, 0x7fe, v0
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
 ; GFX12-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
@@ -10891,8 +10884,9 @@ define bfloat @global_system_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine
 ; GFX12-NEXT:    v_bfe_u32 v7, v5, 16, 1
 ; GFX12-NEXT:    v_or_b32_e32 v8, 0x400000, v5
 ; GFX12-NEXT:    v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_cndmask_b32_e32 v5, v7, v8, vcc_lo
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_lshrrev_b32_e32 v5, 16, v5
@@ -10913,7 +10907,6 @@ define bfloat @global_system_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX12-NEXT:    v_lshrrev_b32_e32 v0, v3, v5
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: global_system_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
@@ -11279,6 +11272,7 @@ define void @global_system_atomic_fmax_noret_bf16__offset12b_pos__amdgpu_no_fine
 ; GFX12-NEXT:    s_wait_bvhcnt 0x0
 ; GFX12-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-NEXT:    v_add_co_u32 v4, vcc_lo, 0x7fe, v0
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
 ; GFX12-NEXT:    v_lshlrev_b32_e32 v6, 16, v2
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
@@ -11302,6 +11296,7 @@ define void @global_system_atomic_fmax_noret_bf16__offset12b_pos__amdgpu_no_fine
 ; GFX12-NEXT:    v_or_b32_e32 v8, 0x400000, v2
 ; GFX12-NEXT:    v_cmp_u_f32_e32 vcc_lo, v2, v2
 ; GFX12-NEXT:    v_add3_u32 v7, v7, v2, 0x7fff
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_cndmask_b32_e32 v2, v7, v8, vcc_lo
 ; GFX12-NEXT:    v_lshrrev_b32_e32 v2, 16, v2
@@ -11322,7 +11317,6 @@ define void @global_system_atomic_fmax_noret_bf16__offset12b_pos__amdgpu_no_fine
 ; GFX12-NEXT:    s_cbranch_execnz .LBB45_1
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: global_system_atomic_fmax_noret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
@@ -11703,7 +11697,6 @@ define <2 x half> @global_agent_atomic_fmax_ret_v2f16__amdgpu_no_fine_grained_me
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX12-NEXT:    v_mov_b32_e32 v0, v3
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: global_agent_atomic_fmax_ret_v2f16__amdgpu_no_fine_grained_memory:
@@ -11996,7 +11989,6 @@ define <2 x half> @global_agent_atomic_fmax_ret_v2f16__offset12b_pos__amdgpu_no_
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX12-NEXT:    v_mov_b32_e32 v0, v3
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: global_agent_atomic_fmax_ret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory:
@@ -12291,7 +12283,6 @@ define <2 x half> @global_agent_atomic_fmax_ret_v2f16__offset12b_neg__amdgpu_no_
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX12-NEXT:    v_mov_b32_e32 v0, v3
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: global_agent_atomic_fmax_ret_v2f16__offset12b_neg__amdgpu_no_fine_grained_memory:
@@ -12589,7 +12580,6 @@ define void @global_agent_atomic_fmax_noret_v2f16__amdgpu_no_fine_grained_memory
 ; GFX12-NEXT:    s_cbranch_execnz .LBB49_1
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: global_agent_atomic_fmax_noret_v2f16__amdgpu_no_fine_grained_memory:
@@ -12871,7 +12861,6 @@ define void @global_agent_atomic_fmax_noret_v2f16__offset12b_pos__amdgpu_no_fine
 ; GFX12-NEXT:    s_cbranch_execnz .LBB50_1
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: global_agent_atomic_fmax_noret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory:
@@ -13156,7 +13145,6 @@ define void @global_agent_atomic_fmax_noret_v2f16__offset12b_neg__amdgpu_no_fine
 ; GFX12-NEXT:    s_cbranch_execnz .LBB51_1
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: global_agent_atomic_fmax_noret_v2f16__offset12b_neg__amdgpu_no_fine_grained_memory:
@@ -13451,7 +13439,6 @@ define <2 x half> @global_system_atomic_fmax_ret_v2f16__offset12b_pos__amdgpu_no
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX12-NEXT:    v_mov_b32_e32 v0, v3
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: global_system_atomic_fmax_ret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory:
@@ -13748,7 +13735,6 @@ define void @global_system_atomic_fmax_noret_v2f16__offset12b_pos__amdgpu_no_fin
 ; GFX12-NEXT:    s_cbranch_execnz .LBB53_1
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: global_system_atomic_fmax_noret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory:
@@ -14039,9 +14025,10 @@ define <2 x bfloat> @global_agent_atomic_fmax_ret_v2bf16__amdgpu_no_fine_grained
 ; GFX12-NEXT:    v_bfe_u32 v7, v3, 16, 1
 ; GFX12-NEXT:    v_or_b32_e32 v9, 0x400000, v3
 ; GFX12-NEXT:    v_cmp_u_f32_e64 s0, v3, v3
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_cndmask_b32_e32 v5, v8, v10, vcc_lo
 ; GFX12-NEXT:    v_add3_u32 v7, v7, v3, 0x7fff
+; GFX12-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_cndmask_b32_e64 v3, v7, v9, s0
 ; GFX12-NEXT:    v_perm_b32 v5, v5, v3, 0x7060302
@@ -14058,7 +14045,6 @@ define <2 x bfloat> @global_agent_atomic_fmax_ret_v2bf16__amdgpu_no_fine_grained
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s1
 ; GFX12-NEXT:    v_mov_b32_e32 v0, v3
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: global_agent_atomic_fmax_ret_v2bf16__amdgpu_no_fine_grained_memory:
@@ -14438,9 +14424,10 @@ define <2 x bfloat> @global_agent_atomic_fmax_ret_v2bf16__offset12b_pos__amdgpu_
 ; GFX12-NEXT:    v_bfe_u32 v7, v3, 16, 1
 ; GFX12-NEXT:    v_or_b32_e32 v9, 0x400000, v3
 ; GFX12-NEXT:    v_cmp_u_f32_e64 s0, v3, v3
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_cndmask_b32_e32 v5, v8, v10, vcc_lo
 ; GFX12-NEXT:    v_add3_u32 v7, v7, v3, 0x7fff
+; GFX12-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_cndmask_b32_e64 v3, v7, v9, s0
 ; GFX12-NEXT:    v_perm_b32 v5, v5, v3, 0x7060302
@@ -14457,7 +14444,6 @@ define <2 x bfloat> @global_agent_atomic_fmax_ret_v2bf16__offset12b_pos__amdgpu_
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s1
 ; GFX12-NEXT:    v_mov_b32_e32 v0, v3
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: global_agent_atomic_fmax_ret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
@@ -14839,9 +14825,10 @@ define <2 x bfloat> @global_agent_atomic_fmax_ret_v2bf16__offset12b_neg__amdgpu_
 ; GFX12-NEXT:    v_bfe_u32 v7, v3, 16, 1
 ; GFX12-NEXT:    v_or_b32_e32 v9, 0x400000, v3
 ; GFX12-NEXT:    v_cmp_u_f32_e64 s0, v3, v3
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_cndmask_b32_e32 v5, v8, v10, vcc_lo
 ; GFX12-NEXT:    v_add3_u32 v7, v7, v3, 0x7fff
+; GFX12-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_cndmask_b32_e64 v3, v7, v9, s0
 ; GFX12-NEXT:    v_perm_b32 v5, v5, v3, 0x7060302
@@ -14858,7 +14845,6 @@ define <2 x bfloat> @global_agent_atomic_fmax_ret_v2bf16__offset12b_neg__amdgpu_
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s1
 ; GFX12-NEXT:    v_mov_b32_e32 v0, v3
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: global_agent_atomic_fmax_ret_v2bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
@@ -15243,8 +15229,10 @@ define void @global_agent_atomic_fmax_noret_v2bf16__amdgpu_no_fine_grained_memor
 ; GFX12-NEXT:    v_add3_u32 v7, v7, v2, 0x7fff
 ; GFX12-NEXT:    v_add3_u32 v8, v8, v6, 0x7fff
 ; GFX12-NEXT:    v_cmp_u_f32_e64 s0, v2, v2
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-NEXT:    s_wait_alu 0xfffd
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX12-NEXT:    v_cndmask_b32_e32 v6, v8, v10, vcc_lo
+; GFX12-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-NEXT:    v_cndmask_b32_e64 v2, v7, v9, s0
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-NEXT:    v_perm_b32 v2, v6, v2, 0x7060302
@@ -15261,7 +15249,6 @@ define void @global_agent_atomic_fmax_noret_v2bf16__amdgpu_no_fine_grained_memor
 ; GFX12-NEXT:    s_cbranch_execnz .LBB57_1
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s1
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: global_agent_atomic_fmax_noret_v2bf16__amdgpu_no_fine_grained_memory:
@@ -15629,8 +15616,10 @@ define void @global_agent_atomic_fmax_noret_v2bf16__offset12b_pos__amdgpu_no_fin
 ; GFX12-NEXT:    v_add3_u32 v7, v7, v2, 0x7fff
 ; GFX12-NEXT:    v_add3_u32 v8, v8, v6, 0x7fff
 ; GFX12-NEXT:    v_cmp_u_f32_e64 s0, v2, v2
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-NEXT:    s_wait_alu 0xfffd
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX12-NEXT:    v_cndmask_b32_e32 v6, v8, v10, vcc_lo
+; GFX12-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-NEXT:    v_cndmask_b32_e64 v2, v7, v9, s0
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-NEXT:    v_perm_b32 v2, v6, v2, 0x7060302
@@ -15647,7 +15636,6 @@ define void @global_agent_atomic_fmax_noret_v2bf16__offset12b_pos__amdgpu_no_fin
 ; GFX12-NEXT:    s_cbranch_execnz .LBB58_1
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s1
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: global_agent_atomic_fmax_noret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
@@ -16018,8 +16006,10 @@ define void @global_agent_atomic_fmax_noret_v2bf16__offset12b_neg__amdgpu_no_fin
 ; GFX12-NEXT:    v_add3_u32 v7, v7, v2, 0x7fff
 ; GFX12-NEXT:    v_add3_u32 v8, v8, v6, 0x7fff
 ; GFX12-NEXT:    v_cmp_u_f32_e64 s0, v2, v2
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-NEXT:    s_wait_alu 0xfffd
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX12-NEXT:    v_cndmask_b32_e32 v6, v8, v10, vcc_lo
+; GFX12-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-NEXT:    v_cndmask_b32_e64 v2, v7, v9, s0
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-NEXT:    v_perm_b32 v2, v6, v2, 0x7060302
@@ -16036,7 +16026,6 @@ define void @global_agent_atomic_fmax_noret_v2bf16__offset12b_neg__amdgpu_no_fin
 ; GFX12-NEXT:    s_cbranch_execnz .LBB59_1
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s1
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: global_agent_atomic_fmax_noret_v2bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
@@ -16416,9 +16405,10 @@ define <2 x bfloat> @global_system_atomic_fmax_ret_v2bf16__offset12b_pos__amdgpu
 ; GFX12-NEXT:    v_bfe_u32 v7, v3, 16, 1
 ; GFX12-NEXT:    v_or_b32_e32 v9, 0x400000, v3
 ; GFX12-NEXT:    v_cmp_u_f32_e64 s0, v3, v3
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_cndmask_b32_e32 v5, v8, v10, vcc_lo
 ; GFX12-NEXT:    v_add3_u32 v7, v7, v3, 0x7fff
+; GFX12-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_cndmask_b32_e64 v3, v7, v9, s0
 ; GFX12-NEXT:    v_perm_b32 v5, v5, v3, 0x7060302
@@ -16436,7 +16426,6 @@ define <2 x bfloat> @global_system_atomic_fmax_ret_v2bf16__offset12b_pos__amdgpu
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s1
 ; GFX12-NEXT:    v_mov_b32_e32 v0, v3
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: global_system_atomic_fmax_ret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
@@ -16819,8 +16808,10 @@ define void @global_system_atomic_fmax_noret_v2bf16__offset12b_pos__amdgpu_no_fi
 ; GFX12-NEXT:    v_add3_u32 v7, v7, v2, 0x7fff
 ; GFX12-NEXT:    v_add3_u32 v8, v8, v6, 0x7fff
 ; GFX12-NEXT:    v_cmp_u_f32_e64 s0, v2, v2
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-NEXT:    s_wait_alu 0xfffd
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX12-NEXT:    v_cndmask_b32_e32 v6, v8, v10, vcc_lo
+; GFX12-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-NEXT:    v_cndmask_b32_e64 v2, v7, v9, s0
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-NEXT:    v_perm_b32 v2, v6, v2, 0x7060302
@@ -16838,7 +16829,6 @@ define void @global_system_atomic_fmax_noret_v2bf16__offset12b_pos__amdgpu_no_fi
 ; GFX12-NEXT:    s_cbranch_execnz .LBB61_1
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s1
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: global_system_atomic_fmax_noret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
diff --git a/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fmin.ll b/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fmin.ll
index 903e80b15814f..b49047c54d7dd 100644
--- a/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fmin.ll
+++ b/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fmin.ll
@@ -3013,7 +3013,6 @@ define double @global_agent_atomic_fmin_ret_f64__amdgpu_no_fine_grained_memory(p
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX12-NEXT:    v_dual_mov_b32 v0, v4 :: v_dual_mov_b32 v1, v5
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: global_agent_atomic_fmin_ret_f64__amdgpu_no_fine_grained_memory:
@@ -3186,7 +3185,6 @@ define double @global_agent_atomic_fmin_ret_f64__offset12b_pos__amdgpu_no_fine_g
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX12-NEXT:    v_dual_mov_b32 v0, v4 :: v_dual_mov_b32 v1, v5
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: global_agent_atomic_fmin_ret_f64__offset12b_pos__amdgpu_no_fine_grained_memory:
@@ -3360,7 +3358,6 @@ define double @global_agent_atomic_fmin_ret_f64__offset12b_neg__amdgpu_no_fine_g
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX12-NEXT:    v_dual_mov_b32 v0, v4 :: v_dual_mov_b32 v1, v5
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: global_agent_atomic_fmin_ret_f64__offset12b_neg__amdgpu_no_fine_grained_memory:
@@ -3533,7 +3530,6 @@ define void @global_agent_atomic_fmin_noret_f64__amdgpu_no_fine_grained_memory(p
 ; GFX12-NEXT:    s_cbranch_execnz .LBB21_1
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: global_agent_atomic_fmin_noret_f64__amdgpu_no_fine_grained_memory:
@@ -3696,7 +3692,6 @@ define void @global_agent_atomic_fmin_noret_f64__offset12b_pos__amdgpu_no_fine_g
 ; GFX12-NEXT:    s_cbranch_execnz .LBB22_1
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: global_agent_atomic_fmin_noret_f64__offset12b_pos__amdgpu_no_fine_grained_memory:
@@ -3862,7 +3857,6 @@ define void @global_agent_atomic_fmin_noret_f64__offset12b_neg__amdgpu_no_fine_g
 ; GFX12-NEXT:    s_cbranch_execnz .LBB23_1
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: global_agent_atomic_fmin_noret_f64__offset12b_neg__amdgpu_no_fine_grained_memory:
@@ -4029,7 +4023,6 @@ define double @global_agent_atomic_fmin_ret_f64__amdgpu_no_remote_memory(ptr add
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX12-NEXT:    v_dual_mov_b32 v0, v4 :: v_dual_mov_b32 v1, v5
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: global_agent_atomic_fmin_ret_f64__amdgpu_no_remote_memory:
@@ -4278,7 +4271,6 @@ define double @global_agent_atomic_fmin_ret_f64__amdgpu_no_fine_grained_memory__
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX12-NEXT:    v_dual_mov_b32 v0, v4 :: v_dual_mov_b32 v1, v5
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: global_agent_atomic_fmin_ret_f64__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
@@ -4469,7 +4461,6 @@ define half @global_agent_atomic_fmin_ret_f16__amdgpu_no_fine_grained_memory(ptr
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX12-NEXT:    v_lshrrev_b32_e32 v0, v3, v5
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: global_agent_atomic_fmin_ret_f16__amdgpu_no_fine_grained_memory:
@@ -4786,6 +4777,7 @@ define half @global_agent_atomic_fmin_ret_f16__offset12b_pos__amdgpu_no_fine_gra
 ; GFX12-NEXT:    s_wait_bvhcnt 0x0
 ; GFX12-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-NEXT:    v_add_co_u32 v3, vcc_lo, 0x7fe, v0
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
 ; GFX12-NEXT:    v_max_num_f16_e32 v2, v2, v2
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
@@ -4823,7 +4815,6 @@ define half @global_agent_atomic_fmin_ret_f16__offset12b_pos__amdgpu_no_fine_gra
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX12-NEXT:    v_lshrrev_b32_e32 v0, v3, v5
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: global_agent_atomic_fmin_ret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
@@ -5151,6 +5142,7 @@ define half @global_agent_atomic_fmin_ret_f16__offset12b_neg__amdgpu_no_fine_gra
 ; GFX12-NEXT:    s_wait_bvhcnt 0x0
 ; GFX12-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-NEXT:    v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, -1, v1, vcc_lo
 ; GFX12-NEXT:    v_max_num_f16_e32 v2, v2, v2
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
@@ -5188,7 +5180,6 @@ define half @global_agent_atomic_fmin_ret_f16__offset12b_neg__amdgpu_no_fine_gra
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX12-NEXT:    v_lshrrev_b32_e32 v0, v3, v5
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: global_agent_atomic_fmin_ret_f16__offset12b_neg__amdgpu_no_fine_grained_memory:
@@ -5552,7 +5543,6 @@ define void @global_agent_atomic_fmin_noret_f16__amdgpu_no_fine_grained_memory(p
 ; GFX12-NEXT:    s_cbranch_execnz .LBB29_1
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: global_agent_atomic_fmin_noret_f16__amdgpu_no_fine_grained_memory:
@@ -5860,6 +5850,7 @@ define void @global_agent_atomic_fmin_noret_f16__offset12b_pos__amdgpu_no_fine_g
 ; GFX12-NEXT:    s_wait_bvhcnt 0x0
 ; GFX12-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-NEXT:    v_add_co_u32 v4, vcc_lo, 0x7fe, v0
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
 ; GFX12-NEXT:    v_max_num_f16_e32 v6, v2, v2
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
@@ -5896,7 +5887,6 @@ define void @global_agent_atomic_fmin_noret_f16__offset12b_pos__amdgpu_no_fine_g
 ; GFX12-NEXT:    s_cbranch_execnz .LBB30_1
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: global_agent_atomic_fmin_noret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
@@ -6214,6 +6204,7 @@ define void @global_agent_atomic_fmin_noret_f16__offset12b_neg__amdgpu_no_fine_g
 ; GFX12-NEXT:    s_wait_bvhcnt 0x0
 ; GFX12-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-NEXT:    v_add_co_u32 v4, vcc_lo, 0xfffff800, v0
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, -1, v1, vcc_lo
 ; GFX12-NEXT:    v_max_num_f16_e32 v6, v2, v2
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
@@ -6250,7 +6241,6 @@ define void @global_agent_atomic_fmin_noret_f16__offset12b_neg__amdgpu_no_fine_g
 ; GFX12-NEXT:    s_cbranch_execnz .LBB31_1
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: global_agent_atomic_fmin_noret_f16__offset12b_neg__amdgpu_no_fine_grained_memory:
@@ -6594,7 +6584,6 @@ define half @global_agent_atomic_fmin_ret_f16__offset12b_pos__align4__amdgpu_no_
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX12-NEXT:    v_mov_b32_e32 v0, v3
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: global_agent_atomic_fmin_ret_f16__offset12b_pos__align4__amdgpu_no_fine_grained_memory:
@@ -6869,7 +6858,6 @@ define void @global_agent_atomic_fmin_noret_f16__offset12b__align4_pos__amdgpu_n
 ; GFX12-NEXT:    s_cbranch_execnz .LBB33_1
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: global_agent_atomic_fmin_noret_f16__offset12b__align4_pos__amdgpu_no_fine_grained_memory:
@@ -7113,6 +7101,7 @@ define half @global_system_atomic_fmin_ret_f16__offset12b_pos__amdgpu_no_fine_gr
 ; GFX12-NEXT:    s_wait_bvhcnt 0x0
 ; GFX12-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-NEXT:    v_add_co_u32 v3, vcc_lo, 0x7fe, v0
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
 ; GFX12-NEXT:    v_max_num_f16_e32 v2, v2, v2
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
@@ -7151,7 +7140,6 @@ define half @global_system_atomic_fmin_ret_f16__offset12b_pos__amdgpu_no_fine_gr
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX12-NEXT:    v_lshrrev_b32_e32 v0, v3, v5
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: global_system_atomic_fmin_ret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
@@ -7481,6 +7469,7 @@ define void @global_system_atomic_fmin_noret_f16__offset12b_pos__amdgpu_no_fine_
 ; GFX12-NEXT:    s_wait_bvhcnt 0x0
 ; GFX12-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-NEXT:    v_add_co_u32 v4, vcc_lo, 0x7fe, v0
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
 ; GFX12-NEXT:    v_max_num_f16_e32 v6, v2, v2
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
@@ -7518,7 +7507,6 @@ define void @global_system_atomic_fmin_noret_f16__offset12b_pos__amdgpu_no_fine_
 ; GFX12-NEXT:    s_cbranch_execnz .LBB35_1
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: global_system_atomic_fmin_noret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
@@ -7863,8 +7851,9 @@ define bfloat @global_agent_atomic_fmin_ret_bf16__amdgpu_no_fine_grained_memory(
 ; GFX12-NEXT:    v_bfe_u32 v7, v5, 16, 1
 ; GFX12-NEXT:    v_or_b32_e32 v8, 0x400000, v5
 ; GFX12-NEXT:    v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_cndmask_b32_e32 v5, v7, v8, vcc_lo
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_lshrrev_b32_e32 v5, 16, v5
@@ -7884,7 +7873,6 @@ define bfloat @global_agent_atomic_fmin_ret_bf16__amdgpu_no_fine_grained_memory(
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX12-NEXT:    v_lshrrev_b32_e32 v0, v3, v5
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: global_agent_atomic_fmin_ret_bf16__amdgpu_no_fine_grained_memory:
@@ -8236,6 +8224,7 @@ define bfloat @global_agent_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_
 ; GFX12-NEXT:    s_wait_bvhcnt 0x0
 ; GFX12-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-NEXT:    v_add_co_u32 v3, vcc_lo, 0x7fe, v0
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
 ; GFX12-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
@@ -8259,8 +8248,9 @@ define bfloat @global_agent_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_
 ; GFX12-NEXT:    v_bfe_u32 v7, v5, 16, 1
 ; GFX12-NEXT:    v_or_b32_e32 v8, 0x400000, v5
 ; GFX12-NEXT:    v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_cndmask_b32_e32 v5, v7, v8, vcc_lo
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_lshrrev_b32_e32 v5, 16, v5
@@ -8280,7 +8270,6 @@ define bfloat @global_agent_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX12-NEXT:    v_lshrrev_b32_e32 v0, v3, v5
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: global_agent_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
@@ -8644,6 +8633,7 @@ define bfloat @global_agent_atomic_fmin_ret_bf16__offset12b_neg__amdgpu_no_fine_
 ; GFX12-NEXT:    s_wait_bvhcnt 0x0
 ; GFX12-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-NEXT:    v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, -1, v1, vcc_lo
 ; GFX12-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
@@ -8667,8 +8657,9 @@ define bfloat @global_agent_atomic_fmin_ret_bf16__offset12b_neg__amdgpu_no_fine_
 ; GFX12-NEXT:    v_bfe_u32 v7, v5, 16, 1
 ; GFX12-NEXT:    v_or_b32_e32 v8, 0x400000, v5
 ; GFX12-NEXT:    v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_cndmask_b32_e32 v5, v7, v8, vcc_lo
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_lshrrev_b32_e32 v5, 16, v5
@@ -8688,7 +8679,6 @@ define bfloat @global_agent_atomic_fmin_ret_bf16__offset12b_neg__amdgpu_no_fine_
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX12-NEXT:    v_lshrrev_b32_e32 v0, v3, v5
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: global_agent_atomic_fmin_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
@@ -9074,6 +9064,7 @@ define void @global_agent_atomic_fmin_noret_bf16__amdgpu_no_fine_grained_memory(
 ; GFX12-NEXT:    v_or_b32_e32 v8, 0x400000, v3
 ; GFX12-NEXT:    v_cmp_u_f32_e32 vcc_lo, v3, v3
 ; GFX12-NEXT:    v_add3_u32 v7, v7, v3, 0x7fff
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_cndmask_b32_e32 v3, v7, v8, vcc_lo
 ; GFX12-NEXT:    v_lshrrev_b32_e32 v3, 16, v3
@@ -9093,7 +9084,6 @@ define void @global_agent_atomic_fmin_noret_bf16__amdgpu_no_fine_grained_memory(
 ; GFX12-NEXT:    s_cbranch_execnz .LBB39_1
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: global_agent_atomic_fmin_noret_bf16__amdgpu_no_fine_grained_memory:
@@ -9435,6 +9425,7 @@ define void @global_agent_atomic_fmin_noret_bf16__offset12b_pos__amdgpu_no_fine_
 ; GFX12-NEXT:    s_wait_bvhcnt 0x0
 ; GFX12-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-NEXT:    v_add_co_u32 v4, vcc_lo, 0x7fe, v0
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
 ; GFX12-NEXT:    v_lshlrev_b32_e32 v6, 16, v2
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
@@ -9458,6 +9449,7 @@ define void @global_agent_atomic_fmin_noret_bf16__offset12b_pos__amdgpu_no_fine_
 ; GFX12-NEXT:    v_or_b32_e32 v8, 0x400000, v2
 ; GFX12-NEXT:    v_cmp_u_f32_e32 vcc_lo, v2, v2
 ; GFX12-NEXT:    v_add3_u32 v7, v7, v2, 0x7fff
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_cndmask_b32_e32 v2, v7, v8, vcc_lo
 ; GFX12-NEXT:    v_lshrrev_b32_e32 v2, 16, v2
@@ -9477,7 +9469,6 @@ define void @global_agent_atomic_fmin_noret_bf16__offset12b_pos__amdgpu_no_fine_
 ; GFX12-NEXT:    s_cbranch_execnz .LBB40_1
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: global_agent_atomic_fmin_noret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
@@ -9830,6 +9821,7 @@ define void @global_agent_atomic_fmin_noret_bf16__offset12b_neg__amdgpu_no_fine_
 ; GFX12-NEXT:    s_wait_bvhcnt 0x0
 ; GFX12-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-NEXT:    v_add_co_u32 v4, vcc_lo, 0xfffff800, v0
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, -1, v1, vcc_lo
 ; GFX12-NEXT:    v_lshlrev_b32_e32 v6, 16, v2
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
@@ -9853,6 +9845,7 @@ define void @global_agent_atomic_fmin_noret_bf16__offset12b_neg__amdgpu_no_fine_
 ; GFX12-NEXT:    v_or_b32_e32 v8, 0x400000, v2
 ; GFX12-NEXT:    v_cmp_u_f32_e32 vcc_lo, v2, v2
 ; GFX12-NEXT:    v_add3_u32 v7, v7, v2, 0x7fff
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_cndmask_b32_e32 v2, v7, v8, vcc_lo
 ; GFX12-NEXT:    v_lshrrev_b32_e32 v2, 16, v2
@@ -9872,7 +9865,6 @@ define void @global_agent_atomic_fmin_noret_bf16__offset12b_neg__amdgpu_no_fine_
 ; GFX12-NEXT:    s_cbranch_execnz .LBB41_1
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: global_agent_atomic_fmin_noret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
@@ -10240,6 +10232,7 @@ define bfloat @global_agent_atomic_fmin_ret_bf16__offset12b_pos__align4__amdgpu_
 ; GFX12-NEXT:    v_or_b32_e32 v6, 0x400000, v3
 ; GFX12-NEXT:    v_cmp_u_f32_e32 vcc_lo, v3, v3
 ; GFX12-NEXT:    v_add3_u32 v5, v5, v3, 0x7fff
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_cndmask_b32_e32 v3, v5, v6, vcc_lo
 ; GFX12-NEXT:    v_lshrrev_b32_e32 v3, 16, v3
@@ -10258,7 +10251,6 @@ define bfloat @global_agent_atomic_fmin_ret_bf16__offset12b_pos__align4__amdgpu_
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX12-NEXT:    v_mov_b32_e32 v0, v3
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: global_agent_atomic_fmin_ret_bf16__offset12b_pos__align4__amdgpu_no_fine_grained_memory:
@@ -10563,8 +10555,9 @@ define void @global_agent_atomic_fmin_noret_bf16__offset12b__align4_pos__amdgpu_
 ; GFX12-NEXT:    v_bfe_u32 v5, v2, 16, 1
 ; GFX12-NEXT:    v_or_b32_e32 v6, 0x400000, v2
 ; GFX12-NEXT:    v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_add3_u32 v5, v5, v2, 0x7fff
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_cndmask_b32_e32 v2, v5, v6, vcc_lo
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_lshrrev_b32_e32 v2, 16, v2
@@ -10582,7 +10575,6 @@ define void @global_agent_atomic_fmin_noret_bf16__offset12b__align4_pos__amdgpu_
 ; GFX12-NEXT:    s_cbranch_execnz .LBB43_1
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: global_agent_atomic_fmin_noret_bf16__offset12b__align4_pos__amdgpu_no_fine_grained_memory:
@@ -10868,6 +10860,7 @@ define bfloat @global_system_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine
 ; GFX12-NEXT:    s_wait_bvhcnt 0x0
 ; GFX12-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-NEXT:    v_add_co_u32 v3, vcc_lo, 0x7fe, v0
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
 ; GFX12-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
@@ -10891,8 +10884,9 @@ define bfloat @global_system_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine
 ; GFX12-NEXT:    v_bfe_u32 v7, v5, 16, 1
 ; GFX12-NEXT:    v_or_b32_e32 v8, 0x400000, v5
 ; GFX12-NEXT:    v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_cndmask_b32_e32 v5, v7, v8, vcc_lo
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_lshrrev_b32_e32 v5, 16, v5
@@ -10913,7 +10907,6 @@ define bfloat @global_system_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX12-NEXT:    v_lshrrev_b32_e32 v0, v3, v5
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: global_system_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
@@ -11279,6 +11272,7 @@ define void @global_system_atomic_fmin_noret_bf16__offset12b_pos__amdgpu_no_fine
 ; GFX12-NEXT:    s_wait_bvhcnt 0x0
 ; GFX12-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-NEXT:    v_add_co_u32 v4, vcc_lo, 0x7fe, v0
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
 ; GFX12-NEXT:    v_lshlrev_b32_e32 v6, 16, v2
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
@@ -11302,6 +11296,7 @@ define void @global_system_atomic_fmin_noret_bf16__offset12b_pos__amdgpu_no_fine
 ; GFX12-NEXT:    v_or_b32_e32 v8, 0x400000, v2
 ; GFX12-NEXT:    v_cmp_u_f32_e32 vcc_lo, v2, v2
 ; GFX12-NEXT:    v_add3_u32 v7, v7, v2, 0x7fff
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_cndmask_b32_e32 v2, v7, v8, vcc_lo
 ; GFX12-NEXT:    v_lshrrev_b32_e32 v2, 16, v2
@@ -11322,7 +11317,6 @@ define void @global_system_atomic_fmin_noret_bf16__offset12b_pos__amdgpu_no_fine
 ; GFX12-NEXT:    s_cbranch_execnz .LBB45_1
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: global_system_atomic_fmin_noret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
@@ -11703,7 +11697,6 @@ define <2 x half> @global_agent_atomic_fmin_ret_v2f16__amdgpu_no_fine_grained_me
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX12-NEXT:    v_mov_b32_e32 v0, v3
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: global_agent_atomic_fmin_ret_v2f16__amdgpu_no_fine_grained_memory:
@@ -11996,7 +11989,6 @@ define <2 x half> @global_agent_atomic_fmin_ret_v2f16__offset12b_pos__amdgpu_no_
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX12-NEXT:    v_mov_b32_e32 v0, v3
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: global_agent_atomic_fmin_ret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory:
@@ -12291,7 +12283,6 @@ define <2 x half> @global_agent_atomic_fmin_ret_v2f16__offset12b_neg__amdgpu_no_
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX12-NEXT:    v_mov_b32_e32 v0, v3
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: global_agent_atomic_fmin_ret_v2f16__offset12b_neg__amdgpu_no_fine_grained_memory:
@@ -12589,7 +12580,6 @@ define void @global_agent_atomic_fmin_noret_v2f16__amdgpu_no_fine_grained_memory
 ; GFX12-NEXT:    s_cbranch_execnz .LBB49_1
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: global_agent_atomic_fmin_noret_v2f16__amdgpu_no_fine_grained_memory:
@@ -12871,7 +12861,6 @@ define void @global_agent_atomic_fmin_noret_v2f16__offset12b_pos__amdgpu_no_fine
 ; GFX12-NEXT:    s_cbranch_execnz .LBB50_1
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: global_agent_atomic_fmin_noret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory:
@@ -13156,7 +13145,6 @@ define void @global_agent_atomic_fmin_noret_v2f16__offset12b_neg__amdgpu_no_fine
 ; GFX12-NEXT:    s_cbranch_execnz .LBB51_1
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: global_agent_atomic_fmin_noret_v2f16__offset12b_neg__amdgpu_no_fine_grained_memory:
@@ -13451,7 +13439,6 @@ define <2 x half> @global_system_atomic_fmin_ret_v2f16__offset12b_pos__amdgpu_no
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX12-NEXT:    v_mov_b32_e32 v0, v3
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: global_system_atomic_fmin_ret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory:
@@ -13748,7 +13735,6 @@ define void @global_system_atomic_fmin_noret_v2f16__offset12b_pos__amdgpu_no_fin
 ; GFX12-NEXT:    s_cbranch_execnz .LBB53_1
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: global_system_atomic_fmin_noret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory:
@@ -14039,9 +14025,10 @@ define <2 x bfloat> @global_agent_atomic_fmin_ret_v2bf16__amdgpu_no_fine_grained
 ; GFX12-NEXT:    v_bfe_u32 v7, v3, 16, 1
 ; GFX12-NEXT:    v_or_b32_e32 v9, 0x400000, v3
 ; GFX12-NEXT:    v_cmp_u_f32_e64 s0, v3, v3
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_cndmask_b32_e32 v5, v8, v10, vcc_lo
 ; GFX12-NEXT:    v_add3_u32 v7, v7, v3, 0x7fff
+; GFX12-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_cndmask_b32_e64 v3, v7, v9, s0
 ; GFX12-NEXT:    v_perm_b32 v5, v5, v3, 0x7060302
@@ -14058,7 +14045,6 @@ define <2 x bfloat> @global_agent_atomic_fmin_ret_v2bf16__amdgpu_no_fine_grained
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s1
 ; GFX12-NEXT:    v_mov_b32_e32 v0, v3
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: global_agent_atomic_fmin_ret_v2bf16__amdgpu_no_fine_grained_memory:
@@ -14438,9 +14424,10 @@ define <2 x bfloat> @global_agent_atomic_fmin_ret_v2bf16__offset12b_pos__amdgpu_
 ; GFX12-NEXT:    v_bfe_u32 v7, v3, 16, 1
 ; GFX12-NEXT:    v_or_b32_e32 v9, 0x400000, v3
 ; GFX12-NEXT:    v_cmp_u_f32_e64 s0, v3, v3
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_cndmask_b32_e32 v5, v8, v10, vcc_lo
 ; GFX12-NEXT:    v_add3_u32 v7, v7, v3, 0x7fff
+; GFX12-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_cndmask_b32_e64 v3, v7, v9, s0
 ; GFX12-NEXT:    v_perm_b32 v5, v5, v3, 0x7060302
@@ -14457,7 +14444,6 @@ define <2 x bfloat> @global_agent_atomic_fmin_ret_v2bf16__offset12b_pos__amdgpu_
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s1
 ; GFX12-NEXT:    v_mov_b32_e32 v0, v3
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: global_agent_atomic_fmin_ret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
@@ -14839,9 +14825,10 @@ define <2 x bfloat> @global_agent_atomic_fmin_ret_v2bf16__offset12b_neg__amdgpu_
 ; GFX12-NEXT:    v_bfe_u32 v7, v3, 16, 1
 ; GFX12-NEXT:    v_or_b32_e32 v9, 0x400000, v3
 ; GFX12-NEXT:    v_cmp_u_f32_e64 s0, v3, v3
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_cndmask_b32_e32 v5, v8, v10, vcc_lo
 ; GFX12-NEXT:    v_add3_u32 v7, v7, v3, 0x7fff
+; GFX12-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_cndmask_b32_e64 v3, v7, v9, s0
 ; GFX12-NEXT:    v_perm_b32 v5, v5, v3, 0x7060302
@@ -14858,7 +14845,6 @@ define <2 x bfloat> @global_agent_atomic_fmin_ret_v2bf16__offset12b_neg__amdgpu_
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s1
 ; GFX12-NEXT:    v_mov_b32_e32 v0, v3
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: global_agent_atomic_fmin_ret_v2bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
@@ -15243,8 +15229,10 @@ define void @global_agent_atomic_fmin_noret_v2bf16__amdgpu_no_fine_grained_memor
 ; GFX12-NEXT:    v_add3_u32 v7, v7, v2, 0x7fff
 ; GFX12-NEXT:    v_add3_u32 v8, v8, v6, 0x7fff
 ; GFX12-NEXT:    v_cmp_u_f32_e64 s0, v2, v2
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-NEXT:    s_wait_alu 0xfffd
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX12-NEXT:    v_cndmask_b32_e32 v6, v8, v10, vcc_lo
+; GFX12-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-NEXT:    v_cndmask_b32_e64 v2, v7, v9, s0
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-NEXT:    v_perm_b32 v2, v6, v2, 0x7060302
@@ -15261,7 +15249,6 @@ define void @global_agent_atomic_fmin_noret_v2bf16__amdgpu_no_fine_grained_memor
 ; GFX12-NEXT:    s_cbranch_execnz .LBB57_1
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s1
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: global_agent_atomic_fmin_noret_v2bf16__amdgpu_no_fine_grained_memory:
@@ -15629,8 +15616,10 @@ define void @global_agent_atomic_fmin_noret_v2bf16__offset12b_pos__amdgpu_no_fin
 ; GFX12-NEXT:    v_add3_u32 v7, v7, v2, 0x7fff
 ; GFX12-NEXT:    v_add3_u32 v8, v8, v6, 0x7fff
 ; GFX12-NEXT:    v_cmp_u_f32_e64 s0, v2, v2
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-NEXT:    s_wait_alu 0xfffd
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX12-NEXT:    v_cndmask_b32_e32 v6, v8, v10, vcc_lo
+; GFX12-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-NEXT:    v_cndmask_b32_e64 v2, v7, v9, s0
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-NEXT:    v_perm_b32 v2, v6, v2, 0x7060302
@@ -15647,7 +15636,6 @@ define void @global_agent_atomic_fmin_noret_v2bf16__offset12b_pos__amdgpu_no_fin
 ; GFX12-NEXT:    s_cbranch_execnz .LBB58_1
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s1
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: global_agent_atomic_fmin_noret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
@@ -16018,8 +16006,10 @@ define void @global_agent_atomic_fmin_noret_v2bf16__offset12b_neg__amdgpu_no_fin
 ; GFX12-NEXT:    v_add3_u32 v7, v7, v2, 0x7fff
 ; GFX12-NEXT:    v_add3_u32 v8, v8, v6, 0x7fff
 ; GFX12-NEXT:    v_cmp_u_f32_e64 s0, v2, v2
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-NEXT:    s_wait_alu 0xfffd
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX12-NEXT:    v_cndmask_b32_e32 v6, v8, v10, vcc_lo
+; GFX12-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-NEXT:    v_cndmask_b32_e64 v2, v7, v9, s0
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-NEXT:    v_perm_b32 v2, v6, v2, 0x7060302
@@ -16036,7 +16026,6 @@ define void @global_agent_atomic_fmin_noret_v2bf16__offset12b_neg__amdgpu_no_fin
 ; GFX12-NEXT:    s_cbranch_execnz .LBB59_1
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s1
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: global_agent_atomic_fmin_noret_v2bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
@@ -16416,9 +16405,10 @@ define <2 x bfloat> @global_system_atomic_fmin_ret_v2bf16__offset12b_pos__amdgpu
 ; GFX12-NEXT:    v_bfe_u32 v7, v3, 16, 1
 ; GFX12-NEXT:    v_or_b32_e32 v9, 0x400000, v3
 ; GFX12-NEXT:    v_cmp_u_f32_e64 s0, v3, v3
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_cndmask_b32_e32 v5, v8, v10, vcc_lo
 ; GFX12-NEXT:    v_add3_u32 v7, v7, v3, 0x7fff
+; GFX12-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_cndmask_b32_e64 v3, v7, v9, s0
 ; GFX12-NEXT:    v_perm_b32 v5, v5, v3, 0x7060302
@@ -16436,7 +16426,6 @@ define <2 x bfloat> @global_system_atomic_fmin_ret_v2bf16__offset12b_pos__amdgpu
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s1
 ; GFX12-NEXT:    v_mov_b32_e32 v0, v3
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: global_system_atomic_fmin_ret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
@@ -16819,8 +16808,10 @@ define void @global_system_atomic_fmin_noret_v2bf16__offset12b_pos__amdgpu_no_fi
 ; GFX12-NEXT:    v_add3_u32 v7, v7, v2, 0x7fff
 ; GFX12-NEXT:    v_add3_u32 v8, v8, v6, 0x7fff
 ; GFX12-NEXT:    v_cmp_u_f32_e64 s0, v2, v2
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-NEXT:    s_wait_alu 0xfffd
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX12-NEXT:    v_cndmask_b32_e32 v6, v8, v10, vcc_lo
+; GFX12-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-NEXT:    v_cndmask_b32_e64 v2, v7, v9, s0
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-NEXT:    v_perm_b32 v2, v6, v2, 0x7060302
@@ -16838,7 +16829,6 @@ define void @global_system_atomic_fmin_noret_v2bf16__offset12b_pos__amdgpu_no_fi
 ; GFX12-NEXT:    s_cbranch_execnz .LBB61_1
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s1
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: global_system_atomic_fmin_noret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
diff --git a/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fsub.ll b/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fsub.ll
index 24791b60bfc6d..5577029f502d0 100644
--- a/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fsub.ll
+++ b/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fsub.ll
@@ -42,7 +42,6 @@ define float @global_agent_atomic_fsub_ret_f32(ptr addrspace(1) %ptr, float %val
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX12-NEXT:    v_mov_b32_e32 v0, v3
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: global_agent_atomic_fsub_ret_f32:
@@ -274,7 +273,6 @@ define float @global_agent_atomic_fsub_ret_f32__offset12b_pos(ptr addrspace(1) %
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX12-NEXT:    v_mov_b32_e32 v0, v3
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: global_agent_atomic_fsub_ret_f32__offset12b_pos:
@@ -508,7 +506,6 @@ define float @global_agent_atomic_fsub_ret_f32__offset12b_neg(ptr addrspace(1) %
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX12-NEXT:    v_mov_b32_e32 v0, v3
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: global_agent_atomic_fsub_ret_f32__offset12b_neg:
@@ -750,7 +747,6 @@ define void @global_agent_atomic_fsub_noret_f32(ptr addrspace(1) %ptr, float %va
 ; GFX12-NEXT:    s_cbranch_execnz .LBB3_1
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: global_agent_atomic_fsub_noret_f32:
@@ -971,7 +967,6 @@ define void @global_agent_atomic_fsub_noret_f32__offset12b_pos(ptr addrspace(1)
 ; GFX12-NEXT:    s_cbranch_execnz .LBB4_1
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: global_agent_atomic_fsub_noret_f32__offset12b_pos:
@@ -1195,7 +1190,6 @@ define void @global_agent_atomic_fsub_noret_f32__offset12b_neg(ptr addrspace(1)
 ; GFX12-NEXT:    s_cbranch_execnz .LBB5_1
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: global_agent_atomic_fsub_noret_f32__offset12b_neg:
@@ -1430,7 +1424,6 @@ define float @global_system_atomic_fsub_ret_f32__offset12b_pos(ptr addrspace(1)
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX12-NEXT:    v_mov_b32_e32 v0, v3
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: global_system_atomic_fsub_ret_f32__offset12b_pos:
@@ -1665,7 +1658,6 @@ define void @global_system_atomic_fsub_noret_f32__offset12b_pos(ptr addrspace(1)
 ; GFX12-NEXT:    s_cbranch_execnz .LBB7_1
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: global_system_atomic_fsub_noret_f32__offset12b_pos:
@@ -1897,7 +1889,6 @@ define float @global_agent_atomic_fsub_ret_f32__ftz(ptr addrspace(1) %ptr, float
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX12-NEXT:    v_mov_b32_e32 v0, v3
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: global_agent_atomic_fsub_ret_f32__ftz:
@@ -2129,7 +2120,6 @@ define float @global_agent_atomic_fsub_ret_f32__offset12b_pos__ftz(ptr addrspace
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX12-NEXT:    v_mov_b32_e32 v0, v3
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: global_agent_atomic_fsub_ret_f32__offset12b_pos__ftz:
@@ -2363,7 +2353,6 @@ define float @global_agent_atomic_fsub_ret_f32__offset12b_neg__ftz(ptr addrspace
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX12-NEXT:    v_mov_b32_e32 v0, v3
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: global_agent_atomic_fsub_ret_f32__offset12b_neg__ftz:
@@ -2605,7 +2594,6 @@ define void @global_agent_atomic_fsub_noret_f32__ftz(ptr addrspace(1) %ptr, floa
 ; GFX12-NEXT:    s_cbranch_execnz .LBB11_1
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: global_agent_atomic_fsub_noret_f32__ftz:
@@ -2826,7 +2814,6 @@ define void @global_agent_atomic_fsub_noret_f32__offset12b_pos__ftz(ptr addrspac
 ; GFX12-NEXT:    s_cbranch_execnz .LBB12_1
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: global_agent_atomic_fsub_noret_f32__offset12b_pos__ftz:
@@ -3050,7 +3037,6 @@ define void @global_agent_atomic_fsub_noret_f32__offset12b_neg__ftz(ptr addrspac
 ; GFX12-NEXT:    s_cbranch_execnz .LBB13_1
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: global_agent_atomic_fsub_noret_f32__offset12b_neg__ftz:
@@ -3285,7 +3271,6 @@ define float @global_system_atomic_fsub_ret_f32__offset12b_pos__ftz(ptr addrspac
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX12-NEXT:    v_mov_b32_e32 v0, v3
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: global_system_atomic_fsub_ret_f32__offset12b_pos__ftz:
@@ -3520,7 +3505,6 @@ define void @global_system_atomic_fsub_noret_f32__offset12b_pos__ftz(ptr addrspa
 ; GFX12-NEXT:    s_cbranch_execnz .LBB15_1
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: global_system_atomic_fsub_noret_f32__offset12b_pos__ftz:
@@ -3752,7 +3736,6 @@ define double @global_agent_atomic_fsub_ret_f64(ptr addrspace(1) %ptr, double %v
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX12-NEXT:    v_dual_mov_b32 v0, v4 :: v_dual_mov_b32 v1, v5
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: global_agent_atomic_fsub_ret_f64:
@@ -4004,7 +3987,6 @@ define double @global_agent_atomic_fsub_ret_f64__offset12b_pos(ptr addrspace(1)
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX12-NEXT:    v_dual_mov_b32 v0, v4 :: v_dual_mov_b32 v1, v5
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: global_agent_atomic_fsub_ret_f64__offset12b_pos:
@@ -4257,7 +4239,6 @@ define double @global_agent_atomic_fsub_ret_f64__offset12b_neg(ptr addrspace(1)
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX12-NEXT:    v_dual_mov_b32 v0, v4 :: v_dual_mov_b32 v1, v5
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: global_agent_atomic_fsub_ret_f64__offset12b_neg:
@@ -4516,7 +4497,6 @@ define void @global_agent_atomic_fsub_noret_f64(ptr addrspace(1) %ptr, double %v
 ; GFX12-NEXT:    s_cbranch_execnz .LBB19_1
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: global_agent_atomic_fsub_noret_f64:
@@ -4746,7 +4726,6 @@ define void @global_agent_atomic_fsub_noret_f64__offset12b_pos(ptr addrspace(1)
 ; GFX12-NEXT:    s_cbranch_execnz .LBB20_1
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: global_agent_atomic_fsub_noret_f64__offset12b_pos:
@@ -4979,7 +4958,6 @@ define void @global_agent_atomic_fsub_noret_f64__offset12b_neg(ptr addrspace(1)
 ; GFX12-NEXT:    s_cbranch_execnz .LBB21_1
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: global_agent_atomic_fsub_noret_f64__offset12b_neg:
@@ -5240,7 +5218,6 @@ define half @global_agent_atomic_fsub_ret_f16(ptr addrspace(1) %ptr, half %val)
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX12-NEXT:    v_lshrrev_b32_e32 v0, v3, v5
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: global_agent_atomic_fsub_ret_f16:
@@ -5545,6 +5522,7 @@ define half @global_agent_atomic_fsub_ret_f16__offset12b_pos(ptr addrspace(1) %p
 ; GFX12-NEXT:    s_wait_bvhcnt 0x0
 ; GFX12-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-NEXT:    v_add_co_u32 v3, vcc_lo, 0x7fe, v0
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
 ; GFX12-NEXT:    s_mov_b32 s0, 0
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_1)
@@ -5580,7 +5558,6 @@ define half @global_agent_atomic_fsub_ret_f16__offset12b_pos(ptr addrspace(1) %p
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX12-NEXT:    v_lshrrev_b32_e32 v0, v3, v5
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: global_agent_atomic_fsub_ret_f16__offset12b_pos:
@@ -5896,6 +5873,7 @@ define half @global_agent_atomic_fsub_ret_f16__offset12b_neg(ptr addrspace(1) %p
 ; GFX12-NEXT:    s_wait_bvhcnt 0x0
 ; GFX12-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-NEXT:    v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, -1, v1, vcc_lo
 ; GFX12-NEXT:    s_mov_b32 s0, 0
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_1)
@@ -5931,7 +5909,6 @@ define half @global_agent_atomic_fsub_ret_f16__offset12b_neg(ptr addrspace(1) %p
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX12-NEXT:    v_lshrrev_b32_e32 v0, v3, v5
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: global_agent_atomic_fsub_ret_f16__offset12b_neg:
@@ -6280,7 +6257,6 @@ define void @global_agent_atomic_fsub_noret_f16(ptr addrspace(1) %ptr, half %val
 ; GFX12-NEXT:    s_cbranch_execnz .LBB25_1
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: global_agent_atomic_fsub_noret_f16:
@@ -6575,6 +6551,7 @@ define void @global_agent_atomic_fsub_noret_f16__offset12b_pos(ptr addrspace(1)
 ; GFX12-NEXT:    s_wait_bvhcnt 0x0
 ; GFX12-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-NEXT:    v_add_co_u32 v3, vcc_lo, 0x7fe, v0
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
 ; GFX12-NEXT:    s_mov_b32 s0, 0
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_1)
@@ -6608,7 +6585,6 @@ define void @global_agent_atomic_fsub_noret_f16__offset12b_pos(ptr addrspace(1)
 ; GFX12-NEXT:    s_cbranch_execnz .LBB26_1
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: global_agent_atomic_fsub_noret_f16__offset12b_pos:
@@ -6913,6 +6889,7 @@ define void @global_agent_atomic_fsub_noret_f16__offset12b_neg(ptr addrspace(1)
 ; GFX12-NEXT:    s_wait_bvhcnt 0x0
 ; GFX12-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-NEXT:    v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, -1, v1, vcc_lo
 ; GFX12-NEXT:    s_mov_b32 s0, 0
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_1)
@@ -6946,7 +6923,6 @@ define void @global_agent_atomic_fsub_noret_f16__offset12b_neg(ptr addrspace(1)
 ; GFX12-NEXT:    s_cbranch_execnz .LBB27_1
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: global_agent_atomic_fsub_noret_f16__offset12b_neg:
@@ -7275,7 +7251,6 @@ define half @global_agent_atomic_fsub_ret_f16__offset12b_pos__align4(ptr addrspa
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX12-NEXT:    v_mov_b32_e32 v0, v3
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: global_agent_atomic_fsub_ret_f16__offset12b_pos__align4:
@@ -7535,7 +7510,6 @@ define void @global_agent_atomic_fsub_noret_f16__offset12b__align4_pos(ptr addrs
 ; GFX12-NEXT:    s_cbranch_execnz .LBB29_1
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: global_agent_atomic_fsub_noret_f16__offset12b__align4_pos:
@@ -7766,6 +7740,7 @@ define half @global_system_atomic_fsub_ret_f16__offset12b_pos(ptr addrspace(1) %
 ; GFX12-NEXT:    s_wait_bvhcnt 0x0
 ; GFX12-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-NEXT:    v_add_co_u32 v3, vcc_lo, 0x7fe, v0
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
 ; GFX12-NEXT:    s_mov_b32 s0, 0
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_1)
@@ -7802,7 +7777,6 @@ define half @global_system_atomic_fsub_ret_f16__offset12b_pos(ptr addrspace(1) %
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX12-NEXT:    v_lshrrev_b32_e32 v0, v3, v5
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: global_system_atomic_fsub_ret_f16__offset12b_pos:
@@ -8120,6 +8094,7 @@ define void @global_system_atomic_fsub_noret_f16__offset12b_pos(ptr addrspace(1)
 ; GFX12-NEXT:    s_wait_bvhcnt 0x0
 ; GFX12-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-NEXT:    v_add_co_u32 v3, vcc_lo, 0x7fe, v0
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
 ; GFX12-NEXT:    s_mov_b32 s0, 0
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_1)
@@ -8154,7 +8129,6 @@ define void @global_system_atomic_fsub_noret_f16__offset12b_pos(ptr addrspace(1)
 ; GFX12-NEXT:    s_cbranch_execnz .LBB31_1
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: global_system_atomic_fsub_noret_f16__offset12b_pos:
@@ -8486,8 +8460,9 @@ define bfloat @global_agent_atomic_fsub_ret_bf16(ptr addrspace(1) %ptr, bfloat %
 ; GFX12-NEXT:    v_bfe_u32 v7, v5, 16, 1
 ; GFX12-NEXT:    v_or_b32_e32 v8, 0x400000, v5
 ; GFX12-NEXT:    v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_cndmask_b32_e32 v5, v7, v8, vcc_lo
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_lshrrev_b32_e32 v5, 16, v5
@@ -8507,7 +8482,6 @@ define bfloat @global_agent_atomic_fsub_ret_bf16(ptr addrspace(1) %ptr, bfloat %
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX12-NEXT:    v_lshrrev_b32_e32 v0, v3, v5
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: global_agent_atomic_fsub_ret_bf16:
@@ -8857,6 +8831,7 @@ define bfloat @global_agent_atomic_fsub_ret_bf16__offset12b_pos(ptr addrspace(1)
 ; GFX12-NEXT:    s_wait_bvhcnt 0x0
 ; GFX12-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-NEXT:    v_add_co_u32 v3, vcc_lo, 0x7fe, v0
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
 ; GFX12-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
@@ -8880,8 +8855,9 @@ define bfloat @global_agent_atomic_fsub_ret_bf16__offset12b_pos(ptr addrspace(1)
 ; GFX12-NEXT:    v_bfe_u32 v7, v5, 16, 1
 ; GFX12-NEXT:    v_or_b32_e32 v8, 0x400000, v5
 ; GFX12-NEXT:    v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_cndmask_b32_e32 v5, v7, v8, vcc_lo
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_lshrrev_b32_e32 v5, 16, v5
@@ -8901,7 +8877,6 @@ define bfloat @global_agent_atomic_fsub_ret_bf16__offset12b_pos(ptr addrspace(1)
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX12-NEXT:    v_lshrrev_b32_e32 v0, v3, v5
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: global_agent_atomic_fsub_ret_bf16__offset12b_pos:
@@ -9263,6 +9238,7 @@ define bfloat @global_agent_atomic_fsub_ret_bf16__offset12b_neg(ptr addrspace(1)
 ; GFX12-NEXT:    s_wait_bvhcnt 0x0
 ; GFX12-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-NEXT:    v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, -1, v1, vcc_lo
 ; GFX12-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
@@ -9286,8 +9262,9 @@ define bfloat @global_agent_atomic_fsub_ret_bf16__offset12b_neg(ptr addrspace(1)
 ; GFX12-NEXT:    v_bfe_u32 v7, v5, 16, 1
 ; GFX12-NEXT:    v_or_b32_e32 v8, 0x400000, v5
 ; GFX12-NEXT:    v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_cndmask_b32_e32 v5, v7, v8, vcc_lo
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_lshrrev_b32_e32 v5, 16, v5
@@ -9307,7 +9284,6 @@ define bfloat @global_agent_atomic_fsub_ret_bf16__offset12b_neg(ptr addrspace(1)
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX12-NEXT:    v_lshrrev_b32_e32 v0, v3, v5
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: global_agent_atomic_fsub_ret_bf16__offset12b_neg:
@@ -9691,6 +9667,7 @@ define void @global_agent_atomic_fsub_noret_bf16(ptr addrspace(1) %ptr, bfloat %
 ; GFX12-NEXT:    v_or_b32_e32 v8, 0x400000, v3
 ; GFX12-NEXT:    v_cmp_u_f32_e32 vcc_lo, v3, v3
 ; GFX12-NEXT:    v_add3_u32 v7, v7, v3, 0x7fff
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_cndmask_b32_e32 v3, v7, v8, vcc_lo
 ; GFX12-NEXT:    v_lshrrev_b32_e32 v3, 16, v3
@@ -9710,7 +9687,6 @@ define void @global_agent_atomic_fsub_noret_bf16(ptr addrspace(1) %ptr, bfloat %
 ; GFX12-NEXT:    s_cbranch_execnz .LBB35_1
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: global_agent_atomic_fsub_noret_bf16:
@@ -10050,6 +10026,7 @@ define void @global_agent_atomic_fsub_noret_bf16__offset12b_pos(ptr addrspace(1)
 ; GFX12-NEXT:    s_wait_bvhcnt 0x0
 ; GFX12-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-NEXT:    v_add_co_u32 v4, vcc_lo, 0x7fe, v0
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
 ; GFX12-NEXT:    v_lshlrev_b32_e32 v6, 16, v2
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
@@ -10073,6 +10050,7 @@ define void @global_agent_atomic_fsub_noret_bf16__offset12b_pos(ptr addrspace(1)
 ; GFX12-NEXT:    v_or_b32_e32 v8, 0x400000, v2
 ; GFX12-NEXT:    v_cmp_u_f32_e32 vcc_lo, v2, v2
 ; GFX12-NEXT:    v_add3_u32 v7, v7, v2, 0x7fff
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_cndmask_b32_e32 v2, v7, v8, vcc_lo
 ; GFX12-NEXT:    v_lshrrev_b32_e32 v2, 16, v2
@@ -10092,7 +10070,6 @@ define void @global_agent_atomic_fsub_noret_bf16__offset12b_pos(ptr addrspace(1)
 ; GFX12-NEXT:    s_cbranch_execnz .LBB36_1
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: global_agent_atomic_fsub_noret_bf16__offset12b_pos:
@@ -10443,6 +10420,7 @@ define void @global_agent_atomic_fsub_noret_bf16__offset12b_neg(ptr addrspace(1)
 ; GFX12-NEXT:    s_wait_bvhcnt 0x0
 ; GFX12-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-NEXT:    v_add_co_u32 v4, vcc_lo, 0xfffff800, v0
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, -1, v1, vcc_lo
 ; GFX12-NEXT:    v_lshlrev_b32_e32 v6, 16, v2
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
@@ -10466,6 +10444,7 @@ define void @global_agent_atomic_fsub_noret_bf16__offset12b_neg(ptr addrspace(1)
 ; GFX12-NEXT:    v_or_b32_e32 v8, 0x400000, v2
 ; GFX12-NEXT:    v_cmp_u_f32_e32 vcc_lo, v2, v2
 ; GFX12-NEXT:    v_add3_u32 v7, v7, v2, 0x7fff
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_cndmask_b32_e32 v2, v7, v8, vcc_lo
 ; GFX12-NEXT:    v_lshrrev_b32_e32 v2, 16, v2
@@ -10485,7 +10464,6 @@ define void @global_agent_atomic_fsub_noret_bf16__offset12b_neg(ptr addrspace(1)
 ; GFX12-NEXT:    s_cbranch_execnz .LBB37_1
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: global_agent_atomic_fsub_noret_bf16__offset12b_neg:
@@ -10851,6 +10829,7 @@ define bfloat @global_agent_atomic_fsub_ret_bf16__offset12b_pos__align4(ptr addr
 ; GFX12-NEXT:    v_or_b32_e32 v6, 0x400000, v3
 ; GFX12-NEXT:    v_cmp_u_f32_e32 vcc_lo, v3, v3
 ; GFX12-NEXT:    v_add3_u32 v5, v5, v3, 0x7fff
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_cndmask_b32_e32 v3, v5, v6, vcc_lo
 ; GFX12-NEXT:    v_lshrrev_b32_e32 v3, 16, v3
@@ -10869,7 +10848,6 @@ define bfloat @global_agent_atomic_fsub_ret_bf16__offset12b_pos__align4(ptr addr
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX12-NEXT:    v_mov_b32_e32 v0, v3
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: global_agent_atomic_fsub_ret_bf16__offset12b_pos__align4:
@@ -11172,8 +11150,9 @@ define void @global_agent_atomic_fsub_noret_bf16__offset12b__align4_pos(ptr addr
 ; GFX12-NEXT:    v_bfe_u32 v5, v2, 16, 1
 ; GFX12-NEXT:    v_or_b32_e32 v6, 0x400000, v2
 ; GFX12-NEXT:    v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_add3_u32 v5, v5, v2, 0x7fff
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_cndmask_b32_e32 v2, v5, v6, vcc_lo
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_lshrrev_b32_e32 v2, 16, v2
@@ -11191,7 +11170,6 @@ define void @global_agent_atomic_fsub_noret_bf16__offset12b__align4_pos(ptr addr
 ; GFX12-NEXT:    s_cbranch_execnz .LBB39_1
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: global_agent_atomic_fsub_noret_bf16__offset12b__align4_pos:
@@ -11475,6 +11453,7 @@ define bfloat @global_system_atomic_fsub_ret_bf16__offset12b_pos(ptr addrspace(1
 ; GFX12-NEXT:    s_wait_bvhcnt 0x0
 ; GFX12-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-NEXT:    v_add_co_u32 v3, vcc_lo, 0x7fe, v0
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
 ; GFX12-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
@@ -11498,8 +11477,9 @@ define bfloat @global_system_atomic_fsub_ret_bf16__offset12b_pos(ptr addrspace(1
 ; GFX12-NEXT:    v_bfe_u32 v7, v5, 16, 1
 ; GFX12-NEXT:    v_or_b32_e32 v8, 0x400000, v5
 ; GFX12-NEXT:    v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_cndmask_b32_e32 v5, v7, v8, vcc_lo
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_lshrrev_b32_e32 v5, 16, v5
@@ -11520,7 +11500,6 @@ define bfloat @global_system_atomic_fsub_ret_bf16__offset12b_pos(ptr addrspace(1
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX12-NEXT:    v_lshrrev_b32_e32 v0, v3, v5
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: global_system_atomic_fsub_ret_bf16__offset12b_pos:
@@ -11884,6 +11863,7 @@ define void @global_system_atomic_fsub_noret_bf16__offset12b_pos(ptr addrspace(1
 ; GFX12-NEXT:    s_wait_bvhcnt 0x0
 ; GFX12-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-NEXT:    v_add_co_u32 v4, vcc_lo, 0x7fe, v0
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
 ; GFX12-NEXT:    v_lshlrev_b32_e32 v6, 16, v2
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
@@ -11907,6 +11887,7 @@ define void @global_system_atomic_fsub_noret_bf16__offset12b_pos(ptr addrspace(1
 ; GFX12-NEXT:    v_or_b32_e32 v8, 0x400000, v2
 ; GFX12-NEXT:    v_cmp_u_f32_e32 vcc_lo, v2, v2
 ; GFX12-NEXT:    v_add3_u32 v7, v7, v2, 0x7fff
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_cndmask_b32_e32 v2, v7, v8, vcc_lo
 ; GFX12-NEXT:    v_lshrrev_b32_e32 v2, 16, v2
@@ -11927,7 +11908,6 @@ define void @global_system_atomic_fsub_noret_bf16__offset12b_pos(ptr addrspace(1
 ; GFX12-NEXT:    s_cbranch_execnz .LBB41_1
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: global_system_atomic_fsub_noret_bf16__offset12b_pos:
@@ -12304,7 +12284,6 @@ define <2 x half> @global_agent_atomic_fsub_ret_v2f16(ptr addrspace(1) %ptr, <2
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX12-NEXT:    v_mov_b32_e32 v0, v3
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: global_agent_atomic_fsub_ret_v2f16:
@@ -12582,7 +12561,6 @@ define <2 x half> @global_agent_atomic_fsub_ret_v2f16__offset12b_pos(ptr addrspa
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX12-NEXT:    v_mov_b32_e32 v0, v3
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: global_agent_atomic_fsub_ret_v2f16__offset12b_pos:
@@ -12862,7 +12840,6 @@ define <2 x half> @global_agent_atomic_fsub_ret_v2f16__offset12b_neg(ptr addrspa
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX12-NEXT:    v_mov_b32_e32 v0, v3
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: global_agent_atomic_fsub_ret_v2f16__offset12b_neg:
@@ -13144,7 +13121,6 @@ define void @global_agent_atomic_fsub_noret_v2f16(ptr addrspace(1) %ptr, <2 x ha
 ; GFX12-NEXT:    s_cbranch_execnz .LBB45_1
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: global_agent_atomic_fsub_noret_v2f16:
@@ -13409,7 +13385,6 @@ define void @global_agent_atomic_fsub_noret_v2f16__offset12b_pos(ptr addrspace(1
 ; GFX12-NEXT:    s_cbranch_execnz .LBB46_1
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: global_agent_atomic_fsub_noret_v2f16__offset12b_pos:
@@ -13677,7 +13652,6 @@ define void @global_agent_atomic_fsub_noret_v2f16__offset12b_neg(ptr addrspace(1
 ; GFX12-NEXT:    s_cbranch_execnz .LBB47_1
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: global_agent_atomic_fsub_noret_v2f16__offset12b_neg:
@@ -13956,7 +13930,6 @@ define <2 x half> @global_system_atomic_fsub_ret_v2f16__offset12b_pos(ptr addrsp
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX12-NEXT:    v_mov_b32_e32 v0, v3
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: global_system_atomic_fsub_ret_v2f16__offset12b_pos:
@@ -14237,7 +14210,6 @@ define void @global_system_atomic_fsub_noret_v2f16__offset12b_pos(ptr addrspace(
 ; GFX12-NEXT:    s_cbranch_execnz .LBB49_1
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: global_system_atomic_fsub_noret_v2f16__offset12b_pos:
@@ -14514,9 +14486,10 @@ define <2 x bfloat> @global_agent_atomic_fsub_ret_v2bf16(ptr addrspace(1) %ptr,
 ; GFX12-NEXT:    v_bfe_u32 v7, v3, 16, 1
 ; GFX12-NEXT:    v_or_b32_e32 v9, 0x400000, v3
 ; GFX12-NEXT:    v_cmp_u_f32_e64 s0, v3, v3
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_cndmask_b32_e32 v5, v8, v10, vcc_lo
 ; GFX12-NEXT:    v_add3_u32 v7, v7, v3, 0x7fff
+; GFX12-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_cndmask_b32_e64 v3, v7, v9, s0
 ; GFX12-NEXT:    v_perm_b32 v5, v5, v3, 0x7060302
@@ -14533,7 +14506,6 @@ define <2 x bfloat> @global_agent_atomic_fsub_ret_v2bf16(ptr addrspace(1) %ptr,
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s1
 ; GFX12-NEXT:    v_mov_b32_e32 v0, v3
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: global_agent_atomic_fsub_ret_v2bf16:
@@ -14913,9 +14885,10 @@ define <2 x bfloat> @global_agent_atomic_fsub_ret_v2bf16__offset12b_pos(ptr addr
 ; GFX12-NEXT:    v_bfe_u32 v7, v3, 16, 1
 ; GFX12-NEXT:    v_or_b32_e32 v9, 0x400000, v3
 ; GFX12-NEXT:    v_cmp_u_f32_e64 s0, v3, v3
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_cndmask_b32_e32 v5, v8, v10, vcc_lo
 ; GFX12-NEXT:    v_add3_u32 v7, v7, v3, 0x7fff
+; GFX12-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_cndmask_b32_e64 v3, v7, v9, s0
 ; GFX12-NEXT:    v_perm_b32 v5, v5, v3, 0x7060302
@@ -14932,7 +14905,6 @@ define <2 x bfloat> @global_agent_atomic_fsub_ret_v2bf16__offset12b_pos(ptr addr
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s1
 ; GFX12-NEXT:    v_mov_b32_e32 v0, v3
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: global_agent_atomic_fsub_ret_v2bf16__offset12b_pos:
@@ -15314,9 +15286,10 @@ define <2 x bfloat> @global_agent_atomic_fsub_ret_v2bf16__offset12b_neg(ptr addr
 ; GFX12-NEXT:    v_bfe_u32 v7, v3, 16, 1
 ; GFX12-NEXT:    v_or_b32_e32 v9, 0x400000, v3
 ; GFX12-NEXT:    v_cmp_u_f32_e64 s0, v3, v3
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_cndmask_b32_e32 v5, v8, v10, vcc_lo
 ; GFX12-NEXT:    v_add3_u32 v7, v7, v3, 0x7fff
+; GFX12-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_cndmask_b32_e64 v3, v7, v9, s0
 ; GFX12-NEXT:    v_perm_b32 v5, v5, v3, 0x7060302
@@ -15333,7 +15306,6 @@ define <2 x bfloat> @global_agent_atomic_fsub_ret_v2bf16__offset12b_neg(ptr addr
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s1
 ; GFX12-NEXT:    v_mov_b32_e32 v0, v3
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: global_agent_atomic_fsub_ret_v2bf16__offset12b_neg:
@@ -15718,8 +15690,10 @@ define void @global_agent_atomic_fsub_noret_v2bf16(ptr addrspace(1) %ptr, <2 x b
 ; GFX12-NEXT:    v_add3_u32 v7, v7, v2, 0x7fff
 ; GFX12-NEXT:    v_add3_u32 v8, v8, v6, 0x7fff
 ; GFX12-NEXT:    v_cmp_u_f32_e64 s0, v2, v2
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-NEXT:    s_wait_alu 0xfffd
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX12-NEXT:    v_cndmask_b32_e32 v6, v8, v10, vcc_lo
+; GFX12-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-NEXT:    v_cndmask_b32_e64 v2, v7, v9, s0
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-NEXT:    v_perm_b32 v2, v6, v2, 0x7060302
@@ -15736,7 +15710,6 @@ define void @global_agent_atomic_fsub_noret_v2bf16(ptr addrspace(1) %ptr, <2 x b
 ; GFX12-NEXT:    s_cbranch_execnz .LBB53_1
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s1
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: global_agent_atomic_fsub_noret_v2bf16:
@@ -16104,8 +16077,10 @@ define void @global_agent_atomic_fsub_noret_v2bf16__offset12b_pos(ptr addrspace(
 ; GFX12-NEXT:    v_add3_u32 v7, v7, v2, 0x7fff
 ; GFX12-NEXT:    v_add3_u32 v8, v8, v6, 0x7fff
 ; GFX12-NEXT:    v_cmp_u_f32_e64 s0, v2, v2
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-NEXT:    s_wait_alu 0xfffd
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX12-NEXT:    v_cndmask_b32_e32 v6, v8, v10, vcc_lo
+; GFX12-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-NEXT:    v_cndmask_b32_e64 v2, v7, v9, s0
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-NEXT:    v_perm_b32 v2, v6, v2, 0x7060302
@@ -16122,7 +16097,6 @@ define void @global_agent_atomic_fsub_noret_v2bf16__offset12b_pos(ptr addrspace(
 ; GFX12-NEXT:    s_cbranch_execnz .LBB54_1
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s1
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: global_agent_atomic_fsub_noret_v2bf16__offset12b_pos:
@@ -16493,8 +16467,10 @@ define void @global_agent_atomic_fsub_noret_v2bf16__offset12b_neg(ptr addrspace(
 ; GFX12-NEXT:    v_add3_u32 v7, v7, v2, 0x7fff
 ; GFX12-NEXT:    v_add3_u32 v8, v8, v6, 0x7fff
 ; GFX12-NEXT:    v_cmp_u_f32_e64 s0, v2, v2
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-NEXT:    s_wait_alu 0xfffd
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX12-NEXT:    v_cndmask_b32_e32 v6, v8, v10, vcc_lo
+; GFX12-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-NEXT:    v_cndmask_b32_e64 v2, v7, v9, s0
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-NEXT:    v_perm_b32 v2, v6, v2, 0x7060302
@@ -16511,7 +16487,6 @@ define void @global_agent_atomic_fsub_noret_v2bf16__offset12b_neg(ptr addrspace(
 ; GFX12-NEXT:    s_cbranch_execnz .LBB55_1
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s1
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: global_agent_atomic_fsub_noret_v2bf16__offset12b_neg:
@@ -16891,9 +16866,10 @@ define <2 x bfloat> @global_system_atomic_fsub_ret_v2bf16__offset12b_pos(ptr add
 ; GFX12-NEXT:    v_bfe_u32 v7, v3, 16, 1
 ; GFX12-NEXT:    v_or_b32_e32 v9, 0x400000, v3
 ; GFX12-NEXT:    v_cmp_u_f32_e64 s0, v3, v3
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_cndmask_b32_e32 v5, v8, v10, vcc_lo
 ; GFX12-NEXT:    v_add3_u32 v7, v7, v3, 0x7fff
+; GFX12-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_cndmask_b32_e64 v3, v7, v9, s0
 ; GFX12-NEXT:    v_perm_b32 v5, v5, v3, 0x7060302
@@ -16911,7 +16887,6 @@ define <2 x bfloat> @global_system_atomic_fsub_ret_v2bf16__offset12b_pos(ptr add
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s1
 ; GFX12-NEXT:    v_mov_b32_e32 v0, v3
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: global_system_atomic_fsub_ret_v2bf16__offset12b_pos:
@@ -17294,8 +17269,10 @@ define void @global_system_atomic_fsub_noret_v2bf16__offset12b_pos(ptr addrspace
 ; GFX12-NEXT:    v_add3_u32 v7, v7, v2, 0x7fff
 ; GFX12-NEXT:    v_add3_u32 v8, v8, v6, 0x7fff
 ; GFX12-NEXT:    v_cmp_u_f32_e64 s0, v2, v2
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-NEXT:    s_wait_alu 0xfffd
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX12-NEXT:    v_cndmask_b32_e32 v6, v8, v10, vcc_lo
+; GFX12-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-NEXT:    v_cndmask_b32_e64 v2, v7, v9, s0
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-NEXT:    v_perm_b32 v2, v6, v2, 0x7060302
@@ -17313,7 +17290,6 @@ define void @global_system_atomic_fsub_noret_v2bf16__offset12b_pos(ptr addrspace
 ; GFX12-NEXT:    s_cbranch_execnz .LBB57_1
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s1
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: global_system_atomic_fsub_noret_v2bf16__offset12b_pos:
diff --git a/llvm/test/CodeGen/AMDGPU/global-saddr-atomics-min-max-system.ll b/llvm/test/CodeGen/AMDGPU/global-saddr-atomics-min-max-system.ll
index bfd57aebad521..a33d363e11bcf 100644
--- a/llvm/test/CodeGen/AMDGPU/global-saddr-atomics-min-max-system.ll
+++ b/llvm/test/CodeGen/AMDGPU/global-saddr-atomics-min-max-system.ll
@@ -537,6 +537,7 @@ define amdgpu_ps <2 x float> @global_max_saddr_i64_rtn(ptr addrspace(1) inreg %s
 ; GFX12-NEXT:    v_mov_b32_e32 v9, v3
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-NEXT:    v_cmp_gt_i64_e32 vcc, v[9:10], v[1:2]
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_cndmask_b32_e32 v8, v2, v10, vcc
 ; GFX12-NEXT:    v_cndmask_b32_e32 v7, v1, v9, vcc
 ; GFX12-NEXT:    global_wb scope:SCOPE_SYS
@@ -664,6 +665,7 @@ define amdgpu_ps <2 x float> @global_max_saddr_i64_rtn_neg128(ptr addrspace(1) i
 ; GFX12-NEXT:    v_mov_b32_e32 v9, v3
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-NEXT:    v_cmp_gt_i64_e32 vcc, v[9:10], v[1:2]
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_cndmask_b32_e32 v8, v2, v10, vcc
 ; GFX12-NEXT:    v_cndmask_b32_e32 v7, v1, v9, vcc
 ; GFX12-NEXT:    global_wb scope:SCOPE_SYS
@@ -779,6 +781,7 @@ define amdgpu_ps void @global_max_saddr_i64_nortn(ptr addrspace(1) inreg %sbase,
 ; GFX12-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX12-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-NEXT:    v_cmp_gt_i64_e32 vcc, v[5:6], v[1:2]
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_cndmask_b32_e32 v4, v2, v6, vcc
 ; GFX12-NEXT:    v_cndmask_b32_e32 v3, v1, v5, vcc
 ; GFX12-NEXT:    global_wb scope:SCOPE_SYS
@@ -891,6 +894,7 @@ define amdgpu_ps void @global_max_saddr_i64_nortn_neg128(ptr addrspace(1) inreg
 ; GFX12-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX12-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-NEXT:    v_cmp_gt_i64_e32 vcc, v[5:6], v[1:2]
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_cndmask_b32_e32 v4, v2, v6, vcc
 ; GFX12-NEXT:    v_cndmask_b32_e32 v3, v1, v5, vcc
 ; GFX12-NEXT:    global_wb scope:SCOPE_SYS
@@ -1445,6 +1449,7 @@ define amdgpu_ps <2 x float> @global_min_saddr_i64_rtn(ptr addrspace(1) inreg %s
 ; GFX12-NEXT:    v_mov_b32_e32 v9, v3
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-NEXT:    v_cmp_le_i64_e32 vcc, v[9:10], v[1:2]
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_cndmask_b32_e32 v8, v2, v10, vcc
 ; GFX12-NEXT:    v_cndmask_b32_e32 v7, v1, v9, vcc
 ; GFX12-NEXT:    global_wb scope:SCOPE_SYS
@@ -1572,6 +1577,7 @@ define amdgpu_ps <2 x float> @global_min_saddr_i64_rtn_neg128(ptr addrspace(1) i
 ; GFX12-NEXT:    v_mov_b32_e32 v9, v3
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-NEXT:    v_cmp_le_i64_e32 vcc, v[9:10], v[1:2]
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_cndmask_b32_e32 v8, v2, v10, vcc
 ; GFX12-NEXT:    v_cndmask_b32_e32 v7, v1, v9, vcc
 ; GFX12-NEXT:    global_wb scope:SCOPE_SYS
@@ -1687,6 +1693,7 @@ define amdgpu_ps void @global_min_saddr_i64_nortn(ptr addrspace(1) inreg %sbase,
 ; GFX12-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX12-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-NEXT:    v_cmp_le_i64_e32 vcc, v[5:6], v[1:2]
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_cndmask_b32_e32 v4, v2, v6, vcc
 ; GFX12-NEXT:    v_cndmask_b32_e32 v3, v1, v5, vcc
 ; GFX12-NEXT:    global_wb scope:SCOPE_SYS
@@ -1799,6 +1806,7 @@ define amdgpu_ps void @global_min_saddr_i64_nortn_neg128(ptr addrspace(1) inreg
 ; GFX12-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX12-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-NEXT:    v_cmp_le_i64_e32 vcc, v[5:6], v[1:2]
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_cndmask_b32_e32 v4, v2, v6, vcc
 ; GFX12-NEXT:    v_cndmask_b32_e32 v3, v1, v5, vcc
 ; GFX12-NEXT:    global_wb scope:SCOPE_SYS
@@ -2353,6 +2361,7 @@ define amdgpu_ps <2 x float> @global_umax_saddr_i64_rtn(ptr addrspace(1) inreg %
 ; GFX12-NEXT:    v_mov_b32_e32 v9, v3
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-NEXT:    v_cmp_gt_u64_e32 vcc, v[9:10], v[1:2]
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_cndmask_b32_e32 v8, v2, v10, vcc
 ; GFX12-NEXT:    v_cndmask_b32_e32 v7, v1, v9, vcc
 ; GFX12-NEXT:    global_wb scope:SCOPE_SYS
@@ -2480,6 +2489,7 @@ define amdgpu_ps <2 x float> @global_umax_saddr_i64_rtn_neg128(ptr addrspace(1)
 ; GFX12-NEXT:    v_mov_b32_e32 v9, v3
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-NEXT:    v_cmp_gt_u64_e32 vcc, v[9:10], v[1:2]
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_cndmask_b32_e32 v8, v2, v10, vcc
 ; GFX12-NEXT:    v_cndmask_b32_e32 v7, v1, v9, vcc
 ; GFX12-NEXT:    global_wb scope:SCOPE_SYS
@@ -2595,6 +2605,7 @@ define amdgpu_ps void @global_umax_saddr_i64_nortn(ptr addrspace(1) inreg %sbase
 ; GFX12-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX12-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-NEXT:    v_cmp_gt_u64_e32 vcc, v[5:6], v[1:2]
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_cndmask_b32_e32 v4, v2, v6, vcc
 ; GFX12-NEXT:    v_cndmask_b32_e32 v3, v1, v5, vcc
 ; GFX12-NEXT:    global_wb scope:SCOPE_SYS
@@ -2707,6 +2718,7 @@ define amdgpu_ps void @global_umax_saddr_i64_nortn_neg128(ptr addrspace(1) inreg
 ; GFX12-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX12-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-NEXT:    v_cmp_gt_u64_e32 vcc, v[5:6], v[1:2]
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_cndmask_b32_e32 v4, v2, v6, vcc
 ; GFX12-NEXT:    v_cndmask_b32_e32 v3, v1, v5, vcc
 ; GFX12-NEXT:    global_wb scope:SCOPE_SYS
@@ -3261,6 +3273,7 @@ define amdgpu_ps <2 x float> @global_umin_saddr_i64_rtn(ptr addrspace(1) inreg %
 ; GFX12-NEXT:    v_mov_b32_e32 v9, v3
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-NEXT:    v_cmp_le_u64_e32 vcc, v[9:10], v[1:2]
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_cndmask_b32_e32 v8, v2, v10, vcc
 ; GFX12-NEXT:    v_cndmask_b32_e32 v7, v1, v9, vcc
 ; GFX12-NEXT:    global_wb scope:SCOPE_SYS
@@ -3388,6 +3401,7 @@ define amdgpu_ps <2 x float> @global_umin_saddr_i64_rtn_neg128(ptr addrspace(1)
 ; GFX12-NEXT:    v_mov_b32_e32 v9, v3
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-NEXT:    v_cmp_le_u64_e32 vcc, v[9:10], v[1:2]
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_cndmask_b32_e32 v8, v2, v10, vcc
 ; GFX12-NEXT:    v_cndmask_b32_e32 v7, v1, v9, vcc
 ; GFX12-NEXT:    global_wb scope:SCOPE_SYS
@@ -3503,6 +3517,7 @@ define amdgpu_ps void @global_umin_saddr_i64_nortn(ptr addrspace(1) inreg %sbase
 ; GFX12-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX12-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-NEXT:    v_cmp_le_u64_e32 vcc, v[5:6], v[1:2]
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_cndmask_b32_e32 v4, v2, v6, vcc
 ; GFX12-NEXT:    v_cndmask_b32_e32 v3, v1, v5, vcc
 ; GFX12-NEXT:    global_wb scope:SCOPE_SYS
@@ -3615,6 +3630,7 @@ define amdgpu_ps void @global_umin_saddr_i64_nortn_neg128(ptr addrspace(1) inreg
 ; GFX12-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX12-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-NEXT:    v_cmp_le_u64_e32 vcc, v[5:6], v[1:2]
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_cndmask_b32_e32 v4, v2, v6, vcc
 ; GFX12-NEXT:    v_cndmask_b32_e32 v3, v1, v5, vcc
 ; GFX12-NEXT:    global_wb scope:SCOPE_SYS
diff --git a/llvm/test/CodeGen/AMDGPU/global-saddr-load.ll b/llvm/test/CodeGen/AMDGPU/global-saddr-load.ll
index b2f113f08a916..492a30b67089c 100644
--- a/llvm/test/CodeGen/AMDGPU/global-saddr-load.ll
+++ b/llvm/test/CodeGen/AMDGPU/global-saddr-load.ll
@@ -1679,6 +1679,7 @@ define amdgpu_ps float @global_load_saddr_uniform_ptr_in_vgprs(i32 %voffset) {
 ; GFX12-GISEL-NEXT:    v_add_co_ci_u32_e32 v1, vcc, 0, v2, vcc
 ; GFX12-GISEL-NEXT:    global_load_u8 v0, v[0:1], off
 ; GFX12-GISEL-NEXT:    s_wait_loadcnt 0x0
+; GFX12-GISEL-NEXT:    s_wait_alu 0xfffd
 ; GFX12-GISEL-NEXT:    ; return to shader part epilog
   %sbase = load ptr addrspace(1), ptr addrspace(3) @ptr.in.lds
   %zext.offset = zext i32 %voffset to i64
@@ -1745,6 +1746,7 @@ define amdgpu_ps float @global_load_saddr_uniform_ptr_in_vgprs_immoffset(i32 %vo
 ; GFX12-GISEL-NEXT:    v_add_co_ci_u32_e32 v1, vcc, 0, v2, vcc
 ; GFX12-GISEL-NEXT:    global_load_u8 v0, v[0:1], off offset:42
 ; GFX12-GISEL-NEXT:    s_wait_loadcnt 0x0
+; GFX12-GISEL-NEXT:    s_wait_alu 0xfffd
 ; GFX12-GISEL-NEXT:    ; return to shader part epilog
   %sbase = load ptr addrspace(1), ptr addrspace(3) @ptr.in.lds
   %zext.offset = zext i32 %voffset to i64
@@ -1931,6 +1933,7 @@ define amdgpu_ps float @global_load_i8_vgpr64_sgpr32(ptr addrspace(1) %vbase, i3
 ; GFX12-SDAG-NEXT:    v_add_co_ci_u32_e32 v1, vcc, 0, v1, vcc
 ; GFX12-SDAG-NEXT:    global_load_u8 v0, v[0:1], off
 ; GFX12-SDAG-NEXT:    s_wait_loadcnt 0x0
+; GFX12-SDAG-NEXT:    s_wait_alu 0xfffd
 ; GFX12-SDAG-NEXT:    ; return to shader part epilog
 ;
 ; GFX12-GISEL-LABEL: global_load_i8_vgpr64_sgpr32:
@@ -1943,6 +1946,7 @@ define amdgpu_ps float @global_load_i8_vgpr64_sgpr32(ptr addrspace(1) %vbase, i3
 ; GFX12-GISEL-NEXT:    v_add_co_ci_u32_e32 v1, vcc, v1, v3, vcc
 ; GFX12-GISEL-NEXT:    global_load_u8 v0, v[0:1], off
 ; GFX12-GISEL-NEXT:    s_wait_loadcnt 0x0
+; GFX12-GISEL-NEXT:    s_wait_alu 0xfffd
 ; GFX12-GISEL-NEXT:    ; return to shader part epilog
   %zext.offset = zext i32 %soffset to i64
   %gep0 = getelementptr inbounds i8, ptr addrspace(1) %vbase, i64 %zext.offset
@@ -1986,6 +1990,7 @@ define amdgpu_ps float @global_load_i8_vgpr64_sgpr32_offset_4095(ptr addrspace(1
 ; GFX12-SDAG-NEXT:    v_add_co_ci_u32_e32 v1, vcc, 0, v1, vcc
 ; GFX12-SDAG-NEXT:    global_load_u8 v0, v[0:1], off offset:4095
 ; GFX12-SDAG-NEXT:    s_wait_loadcnt 0x0
+; GFX12-SDAG-NEXT:    s_wait_alu 0xfffd
 ; GFX12-SDAG-NEXT:    ; return to shader part epilog
 ;
 ; GFX12-GISEL-LABEL: global_load_i8_vgpr64_sgpr32_offset_4095:
@@ -1998,6 +2003,7 @@ define amdgpu_ps float @global_load_i8_vgpr64_sgpr32_offset_4095(ptr addrspace(1
 ; GFX12-GISEL-NEXT:    v_add_co_ci_u32_e32 v1, vcc, v1, v3, vcc
 ; GFX12-GISEL-NEXT:    global_load_u8 v0, v[0:1], off offset:4095
 ; GFX12-GISEL-NEXT:    s_wait_loadcnt 0x0
+; GFX12-GISEL-NEXT:    s_wait_alu 0xfffd
 ; GFX12-GISEL-NEXT:    ; return to shader part epilog
   %zext.offset = zext i32 %soffset to i64
   %gep0 = getelementptr inbounds i8, ptr addrspace(1) %vbase, i64 %zext.offset
@@ -2065,6 +2071,7 @@ define amdgpu_ps float @global_load_saddr_f32_natural_addressing(ptr addrspace(1
 ; GFX12-SDAG-NEXT:    v_add_co_ci_u32_e32 v1, vcc, s3, v1, vcc
 ; GFX12-SDAG-NEXT:    global_load_b32 v0, v[0:1], off
 ; GFX12-SDAG-NEXT:    s_wait_loadcnt 0x0
+; GFX12-SDAG-NEXT:    s_wait_alu 0xfffd
 ; GFX12-SDAG-NEXT:    ; return to shader part epilog
 ;
 ; GFX12-GISEL-LABEL: global_load_saddr_f32_natural_addressing:
@@ -2081,6 +2088,7 @@ define amdgpu_ps float @global_load_saddr_f32_natural_addressing(ptr addrspace(1
 ; GFX12-GISEL-NEXT:    v_add_co_ci_u32_e32 v1, vcc, v3, v1, vcc
 ; GFX12-GISEL-NEXT:    global_load_b32 v0, v[0:1], off
 ; GFX12-GISEL-NEXT:    s_wait_loadcnt 0x0
+; GFX12-GISEL-NEXT:    s_wait_alu 0xfffd
 ; GFX12-GISEL-NEXT:    ; return to shader part epilog
   %voffset = load i32, ptr addrspace(1) %voffset.ptr
   %zext.offset = zext i32 %voffset to i64
@@ -2246,6 +2254,7 @@ define amdgpu_ps float @global_load_f32_saddr_zext_vgpr_range_too_large(ptr addr
 ; GFX12-SDAG-NEXT:    v_add_co_ci_u32_e32 v1, vcc, s3, v1, vcc
 ; GFX12-SDAG-NEXT:    global_load_b32 v0, v[0:1], off
 ; GFX12-SDAG-NEXT:    s_wait_loadcnt 0x0
+; GFX12-SDAG-NEXT:    s_wait_alu 0xfffd
 ; GFX12-SDAG-NEXT:    ; return to shader part epilog
 ;
 ; GFX12-GISEL-LABEL: global_load_f32_saddr_zext_vgpr_range_too_large:
@@ -2262,6 +2271,7 @@ define amdgpu_ps float @global_load_f32_saddr_zext_vgpr_range_too_large(ptr addr
 ; GFX12-GISEL-NEXT:    v_add_co_ci_u32_e32 v1, vcc, v3, v1, vcc
 ; GFX12-GISEL-NEXT:    global_load_b32 v0, v[0:1], off
 ; GFX12-GISEL-NEXT:    s_wait_loadcnt 0x0
+; GFX12-GISEL-NEXT:    s_wait_alu 0xfffd
 ; GFX12-GISEL-NEXT:    ; return to shader part epilog
   %voffset = load i32, ptr addrspace(1) %voffset.ptr, !range !1, !noundef !{}
   %zext.offset = zext i32 %voffset to i64
@@ -4727,10 +4737,12 @@ define amdgpu_ps void @global_addr_64bit_lsr_iv(ptr addrspace(1) inreg %arg) {
 ; GFX12-GISEL-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX12-GISEL-NEXT:  .LBB132_1: ; %bb3
 ; GFX12-GISEL-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX12-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
 ; GFX12-GISEL-NEXT:    v_add_co_u32 v4, vcc, v0, v2
+; GFX12-GISEL-NEXT:    s_wait_alu 0xfffd
 ; GFX12-GISEL-NEXT:    v_add_co_ci_u32_e32 v5, vcc, v1, v3, vcc
 ; GFX12-GISEL-NEXT:    v_add_co_u32 v2, vcc, v2, 4
+; GFX12-GISEL-NEXT:    s_wait_alu 0xfffd
 ; GFX12-GISEL-NEXT:    v_add_co_ci_u32_e32 v3, vcc, 0, v3, vcc
 ; GFX12-GISEL-NEXT:    global_load_b32 v4, v[4:5], off scope:SCOPE_SYS
 ; GFX12-GISEL-NEXT:    s_wait_loadcnt 0x0
@@ -4845,10 +4857,12 @@ define amdgpu_ps void @global_addr_64bit_lsr_iv_multiload(ptr addrspace(1) inreg
 ; GFX12-GISEL-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX12-GISEL-NEXT:  .LBB133_1: ; %bb3
 ; GFX12-GISEL-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX12-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
 ; GFX12-GISEL-NEXT:    v_add_co_u32 v4, vcc, v0, v2
+; GFX12-GISEL-NEXT:    s_wait_alu 0xfffd
 ; GFX12-GISEL-NEXT:    v_add_co_ci_u32_e32 v5, vcc, v1, v3, vcc
 ; GFX12-GISEL-NEXT:    v_add_co_u32 v2, vcc, v2, 4
+; GFX12-GISEL-NEXT:    s_wait_alu 0xfffd
 ; GFX12-GISEL-NEXT:    v_add_co_ci_u32_e32 v3, vcc, 0, v3, vcc
 ; GFX12-GISEL-NEXT:    global_load_b32 v6, v[4:5], off scope:SCOPE_SYS
 ; GFX12-GISEL-NEXT:    s_wait_loadcnt 0x0
diff --git a/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fadd.ll b/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fadd.ll
index fbe06b3651b06..15be44a335a1d 100644
--- a/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fadd.ll
+++ b/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fadd.ll
@@ -1267,16 +1267,16 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_uni_value_one_as_scope
 ;
 ; GFX1032-LABEL: global_atomic_fadd_uni_address_uni_value_one_as_scope_unsafe_structfp:
 ; GFX1032:       ; %bb.0:
-; GFX1032-NEXT:    s_mov_b32 s12, SCRATCH_RSRC_DWORD0
 ; GFX1032-NEXT:    s_mov_b32 s0, exec_lo
+; GFX1032-NEXT:    s_mov_b32 s12, SCRATCH_RSRC_DWORD0
+; GFX1032-NEXT:    v_mbcnt_lo_u32_b32 v0, s0, 0
 ; GFX1032-NEXT:    s_mov_b32 s13, SCRATCH_RSRC_DWORD1
 ; GFX1032-NEXT:    s_mov_b32 s14, -1
-; GFX1032-NEXT:    v_mbcnt_lo_u32_b32 v0, s0, 0
 ; GFX1032-NEXT:    s_mov_b32 s15, 0x31c16000
 ; GFX1032-NEXT:    s_add_u32 s12, s12, s11
+; GFX1032-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
 ; GFX1032-NEXT:    s_addc_u32 s13, s13, 0
 ; GFX1032-NEXT:    s_mov_b32 s2, 0
-; GFX1032-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
 ; GFX1032-NEXT:    s_and_saveexec_b32 s1, vcc_lo
 ; GFX1032-NEXT:    s_cbranch_execz .LBB2_3
 ; GFX1032-NEXT:  ; %bb.1:
@@ -1487,16 +1487,16 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_uni_value_one_as_scope
 ;
 ; GFX1032-DPP-LABEL: global_atomic_fadd_uni_address_uni_value_one_as_scope_unsafe_structfp:
 ; GFX1032-DPP:       ; %bb.0:
-; GFX1032-DPP-NEXT:    s_mov_b32 s12, SCRATCH_RSRC_DWORD0
 ; GFX1032-DPP-NEXT:    s_mov_b32 s0, exec_lo
+; GFX1032-DPP-NEXT:    s_mov_b32 s12, SCRATCH_RSRC_DWORD0
+; GFX1032-DPP-NEXT:    v_mbcnt_lo_u32_b32 v0, s0, 0
 ; GFX1032-DPP-NEXT:    s_mov_b32 s13, SCRATCH_RSRC_DWORD1
 ; GFX1032-DPP-NEXT:    s_mov_b32 s14, -1
-; GFX1032-DPP-NEXT:    v_mbcnt_lo_u32_b32 v0, s0, 0
 ; GFX1032-DPP-NEXT:    s_mov_b32 s15, 0x31c16000
 ; GFX1032-DPP-NEXT:    s_add_u32 s12, s12, s11
+; GFX1032-DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
 ; GFX1032-DPP-NEXT:    s_addc_u32 s13, s13, 0
 ; GFX1032-DPP-NEXT:    s_mov_b32 s2, 0
-; GFX1032-DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
 ; GFX1032-DPP-NEXT:    s_and_saveexec_b32 s1, vcc_lo
 ; GFX1032-DPP-NEXT:    s_cbranch_execz .LBB2_3
 ; GFX1032-DPP-NEXT:  ; %bb.1:
@@ -2487,16 +2487,16 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_uni_value_agent_scope_
 ;
 ; GFX1032-LABEL: global_atomic_fadd_uni_address_uni_value_agent_scope_strictfp:
 ; GFX1032:       ; %bb.0:
-; GFX1032-NEXT:    s_mov_b32 s12, SCRATCH_RSRC_DWORD0
 ; GFX1032-NEXT:    s_mov_b32 s0, exec_lo
+; GFX1032-NEXT:    s_mov_b32 s12, SCRATCH_RSRC_DWORD0
+; GFX1032-NEXT:    v_mbcnt_lo_u32_b32 v0, s0, 0
 ; GFX1032-NEXT:    s_mov_b32 s13, SCRATCH_RSRC_DWORD1
 ; GFX1032-NEXT:    s_mov_b32 s14, -1
-; GFX1032-NEXT:    v_mbcnt_lo_u32_b32 v0, s0, 0
 ; GFX1032-NEXT:    s_mov_b32 s15, 0x31c16000
 ; GFX1032-NEXT:    s_add_u32 s12, s12, s11
+; GFX1032-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
 ; GFX1032-NEXT:    s_addc_u32 s13, s13, 0
 ; GFX1032-NEXT:    s_mov_b32 s2, 0
-; GFX1032-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
 ; GFX1032-NEXT:    s_and_saveexec_b32 s1, vcc_lo
 ; GFX1032-NEXT:    s_cbranch_execz .LBB4_3
 ; GFX1032-NEXT:  ; %bb.1:
@@ -2737,16 +2737,16 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_uni_value_agent_scope_
 ;
 ; GFX1032-DPP-LABEL: global_atomic_fadd_uni_address_uni_value_agent_scope_strictfp:
 ; GFX1032-DPP:       ; %bb.0:
-; GFX1032-DPP-NEXT:    s_mov_b32 s12, SCRATCH_RSRC_DWORD0
 ; GFX1032-DPP-NEXT:    s_mov_b32 s0, exec_lo
+; GFX1032-DPP-NEXT:    s_mov_b32 s12, SCRATCH_RSRC_DWORD0
+; GFX1032-DPP-NEXT:    v_mbcnt_lo_u32_b32 v0, s0, 0
 ; GFX1032-DPP-NEXT:    s_mov_b32 s13, SCRATCH_RSRC_DWORD1
 ; GFX1032-DPP-NEXT:    s_mov_b32 s14, -1
-; GFX1032-DPP-NEXT:    v_mbcnt_lo_u32_b32 v0, s0, 0
 ; GFX1032-DPP-NEXT:    s_mov_b32 s15, 0x31c16000
 ; GFX1032-DPP-NEXT:    s_add_u32 s12, s12, s11
+; GFX1032-DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
 ; GFX1032-DPP-NEXT:    s_addc_u32 s13, s13, 0
 ; GFX1032-DPP-NEXT:    s_mov_b32 s2, 0
-; GFX1032-DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
 ; GFX1032-DPP-NEXT:    s_and_saveexec_b32 s1, vcc_lo
 ; GFX1032-DPP-NEXT:    s_cbranch_execz .LBB4_3
 ; GFX1032-DPP-NEXT:  ; %bb.1:
@@ -4543,16 +4543,16 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_uni_value_default_scop
 ;
 ; GFX1032-LABEL: global_atomic_fadd_uni_address_uni_value_default_scope_strictfp:
 ; GFX1032:       ; %bb.0:
-; GFX1032-NEXT:    s_mov_b32 s12, SCRATCH_RSRC_DWORD0
 ; GFX1032-NEXT:    s_mov_b32 s0, exec_lo
+; GFX1032-NEXT:    s_mov_b32 s12, SCRATCH_RSRC_DWORD0
+; GFX1032-NEXT:    v_mbcnt_lo_u32_b32 v0, s0, 0
 ; GFX1032-NEXT:    s_mov_b32 s13, SCRATCH_RSRC_DWORD1
 ; GFX1032-NEXT:    s_mov_b32 s14, -1
-; GFX1032-NEXT:    v_mbcnt_lo_u32_b32 v0, s0, 0
 ; GFX1032-NEXT:    s_mov_b32 s15, 0x31c16000
 ; GFX1032-NEXT:    s_add_u32 s12, s12, s11
+; GFX1032-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
 ; GFX1032-NEXT:    s_addc_u32 s13, s13, 0
 ; GFX1032-NEXT:    s_mov_b32 s2, 0
-; GFX1032-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
 ; GFX1032-NEXT:    s_and_saveexec_b32 s1, vcc_lo
 ; GFX1032-NEXT:    s_cbranch_execz .LBB7_3
 ; GFX1032-NEXT:  ; %bb.1:
@@ -4793,16 +4793,16 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_uni_value_default_scop
 ;
 ; GFX1032-DPP-LABEL: global_atomic_fadd_uni_address_uni_value_default_scope_strictfp:
 ; GFX1032-DPP:       ; %bb.0:
-; GFX1032-DPP-NEXT:    s_mov_b32 s12, SCRATCH_RSRC_DWORD0
 ; GFX1032-DPP-NEXT:    s_mov_b32 s0, exec_lo
+; GFX1032-DPP-NEXT:    s_mov_b32 s12, SCRATCH_RSRC_DWORD0
+; GFX1032-DPP-NEXT:    v_mbcnt_lo_u32_b32 v0, s0, 0
 ; GFX1032-DPP-NEXT:    s_mov_b32 s13, SCRATCH_RSRC_DWORD1
 ; GFX1032-DPP-NEXT:    s_mov_b32 s14, -1
-; GFX1032-DPP-NEXT:    v_mbcnt_lo_u32_b32 v0, s0, 0
 ; GFX1032-DPP-NEXT:    s_mov_b32 s15, 0x31c16000
 ; GFX1032-DPP-NEXT:    s_add_u32 s12, s12, s11
+; GFX1032-DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
 ; GFX1032-DPP-NEXT:    s_addc_u32 s13, s13, 0
 ; GFX1032-DPP-NEXT:    s_mov_b32 s2, 0
-; GFX1032-DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
 ; GFX1032-DPP-NEXT:    s_and_saveexec_b32 s1, vcc_lo
 ; GFX1032-DPP-NEXT:    s_cbranch_execz .LBB7_3
 ; GFX1032-DPP-NEXT:  ; %bb.1:
@@ -5987,19 +5987,19 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_uni_value_agent
 ;
 ; GFX1032-LABEL: global_atomic_fadd_double_uni_address_uni_value_agent_scope_unsafe:
 ; GFX1032:       ; %bb.0:
-; GFX1032-NEXT:    s_mov_b32 s48, SCRATCH_RSRC_DWORD0
 ; GFX1032-NEXT:    s_mov_b32 s42, s9
 ; GFX1032-NEXT:    s_mov_b32 s9, exec_lo
+; GFX1032-NEXT:    s_mov_b32 s48, SCRATCH_RSRC_DWORD0
+; GFX1032-NEXT:    v_mbcnt_lo_u32_b32 v3, s9, 0
 ; GFX1032-NEXT:    s_mov_b32 s49, SCRATCH_RSRC_DWORD1
 ; GFX1032-NEXT:    s_mov_b32 s50, -1
-; GFX1032-NEXT:    v_mbcnt_lo_u32_b32 v3, s9, 0
 ; GFX1032-NEXT:    s_mov_b32 s51, 0x31c16000
 ; GFX1032-NEXT:    s_add_u32 s48, s48, s11
+; GFX1032-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v3
 ; GFX1032-NEXT:    s_addc_u32 s49, s49, 0
 ; GFX1032-NEXT:    s_mov_b64 s[40:41], s[0:1]
 ; GFX1032-NEXT:    s_mov_b32 s46, 0
 ; GFX1032-NEXT:    s_movk_i32 s32, 0x400
-; GFX1032-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v3
 ; GFX1032-NEXT:    s_and_saveexec_b32 s0, vcc_lo
 ; GFX1032-NEXT:    s_cbranch_execz .LBB9_3
 ; GFX1032-NEXT:  ; %bb.1:
@@ -6446,19 +6446,19 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_uni_value_agent
 ;
 ; GFX1032-DPP-LABEL: global_atomic_fadd_double_uni_address_uni_value_agent_scope_unsafe:
 ; GFX1032-DPP:       ; %bb.0:
-; GFX1032-DPP-NEXT:    s_mov_b32 s48, SCRATCH_RSRC_DWORD0
 ; GFX1032-DPP-NEXT:    s_mov_b32 s42, s9
 ; GFX1032-DPP-NEXT:    s_mov_b32 s9, exec_lo
+; GFX1032-DPP-NEXT:    s_mov_b32 s48, SCRATCH_RSRC_DWORD0
+; GFX1032-DPP-NEXT:    v_mbcnt_lo_u32_b32 v3, s9, 0
 ; GFX1032-DPP-NEXT:    s_mov_b32 s49, SCRATCH_RSRC_DWORD1
 ; GFX1032-DPP-NEXT:    s_mov_b32 s50, -1
-; GFX1032-DPP-NEXT:    v_mbcnt_lo_u32_b32 v3, s9, 0
 ; GFX1032-DPP-NEXT:    s_mov_b32 s51, 0x31c16000
 ; GFX1032-DPP-NEXT:    s_add_u32 s48, s48, s11
+; GFX1032-DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v3
 ; GFX1032-DPP-NEXT:    s_addc_u32 s49, s49, 0
 ; GFX1032-DPP-NEXT:    s_mov_b64 s[40:41], s[0:1]
 ; GFX1032-DPP-NEXT:    s_mov_b32 s46, 0
 ; GFX1032-DPP-NEXT:    s_movk_i32 s32, 0x400
-; GFX1032-DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v3
 ; GFX1032-DPP-NEXT:    s_and_saveexec_b32 s0, vcc_lo
 ; GFX1032-DPP-NEXT:    s_cbranch_execz .LBB9_3
 ; GFX1032-DPP-NEXT:  ; %bb.1:
@@ -7692,8 +7692,8 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_agent
 ; GFX1032-DPP-NEXT:    v_permlanex16_b32 v10, v8, 0, 0
 ; GFX1032-DPP-NEXT:    v_add_f64 v[8:9], v[8:9], v[10:11]
 ; GFX1032-DPP-NEXT:    s_mov_b32 exec_lo, s0
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v41, v8
 ; GFX1032-DPP-NEXT:    v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v41, v8
 ; GFX1032-DPP-NEXT:    v_mov_b32_e32 v0, 0
 ; GFX1032-DPP-NEXT:    v_mov_b32_e32 v42, v9
 ; GFX1032-DPP-NEXT:    s_mov_b32 s46, 0
@@ -8122,16 +8122,16 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_uni_value_one_a
 ;
 ; GFX1032-LABEL: global_atomic_fadd_double_uni_address_uni_value_one_as_scope_unsafe_structfp:
 ; GFX1032:       ; %bb.0:
-; GFX1032-NEXT:    s_mov_b32 s12, SCRATCH_RSRC_DWORD0
 ; GFX1032-NEXT:    s_mov_b32 s0, exec_lo
+; GFX1032-NEXT:    s_mov_b32 s12, SCRATCH_RSRC_DWORD0
+; GFX1032-NEXT:    v_mbcnt_lo_u32_b32 v0, s0, 0
 ; GFX1032-NEXT:    s_mov_b32 s13, SCRATCH_RSRC_DWORD1
 ; GFX1032-NEXT:    s_mov_b32 s14, -1
-; GFX1032-NEXT:    v_mbcnt_lo_u32_b32 v0, s0, 0
 ; GFX1032-NEXT:    s_mov_b32 s15, 0x31c16000
 ; GFX1032-NEXT:    s_add_u32 s12, s12, s11
+; GFX1032-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
 ; GFX1032-NEXT:    s_addc_u32 s13, s13, 0
 ; GFX1032-NEXT:    s_mov_b32 s2, 0
-; GFX1032-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
 ; GFX1032-NEXT:    s_and_saveexec_b32 s1, vcc_lo
 ; GFX1032-NEXT:    s_cbranch_execz .LBB11_3
 ; GFX1032-NEXT:  ; %bb.1:
@@ -8379,16 +8379,16 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_uni_value_one_a
 ;
 ; GFX1032-DPP-LABEL: global_atomic_fadd_double_uni_address_uni_value_one_as_scope_unsafe_structfp:
 ; GFX1032-DPP:       ; %bb.0:
-; GFX1032-DPP-NEXT:    s_mov_b32 s12, SCRATCH_RSRC_DWORD0
 ; GFX1032-DPP-NEXT:    s_mov_b32 s0, exec_lo
+; GFX1032-DPP-NEXT:    s_mov_b32 s12, SCRATCH_RSRC_DWORD0
+; GFX1032-DPP-NEXT:    v_mbcnt_lo_u32_b32 v0, s0, 0
 ; GFX1032-DPP-NEXT:    s_mov_b32 s13, SCRATCH_RSRC_DWORD1
 ; GFX1032-DPP-NEXT:    s_mov_b32 s14, -1
-; GFX1032-DPP-NEXT:    v_mbcnt_lo_u32_b32 v0, s0, 0
 ; GFX1032-DPP-NEXT:    s_mov_b32 s15, 0x31c16000
 ; GFX1032-DPP-NEXT:    s_add_u32 s12, s12, s11
+; GFX1032-DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
 ; GFX1032-DPP-NEXT:    s_addc_u32 s13, s13, 0
 ; GFX1032-DPP-NEXT:    s_mov_b32 s2, 0
-; GFX1032-DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
 ; GFX1032-DPP-NEXT:    s_and_saveexec_b32 s1, vcc_lo
 ; GFX1032-DPP-NEXT:    s_cbranch_execz .LBB11_3
 ; GFX1032-DPP-NEXT:  ; %bb.1:
@@ -9217,8 +9217,8 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_one_a
 ; GFX1032-DPP-NEXT:    v_permlanex16_b32 v5, v3, 0, 0
 ; GFX1032-DPP-NEXT:    v_add_f64 v[3:4], v[3:4], v[5:6]
 ; GFX1032-DPP-NEXT:    s_mov_b32 exec_lo, s0
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v0, v3
 ; GFX1032-DPP-NEXT:    v_mbcnt_lo_u32_b32 v7, exec_lo, 0
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v0, v3
 ; GFX1032-DPP-NEXT:    v_mov_b32_e32 v2, 0
 ; GFX1032-DPP-NEXT:    v_mov_b32_e32 v1, v4
 ; GFX1032-DPP-NEXT:    s_mov_b32 s2, 0
@@ -9555,16 +9555,16 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_uni_value_agent
 ;
 ; GFX1032-LABEL: global_atomic_fadd_double_uni_address_uni_value_agent_scope_strictfp:
 ; GFX1032:       ; %bb.0:
-; GFX1032-NEXT:    s_mov_b32 s12, SCRATCH_RSRC_DWORD0
 ; GFX1032-NEXT:    s_mov_b32 s0, exec_lo
+; GFX1032-NEXT:    s_mov_b32 s12, SCRATCH_RSRC_DWORD0
+; GFX1032-NEXT:    v_mbcnt_lo_u32_b32 v0, s0, 0
 ; GFX1032-NEXT:    s_mov_b32 s13, SCRATCH_RSRC_DWORD1
 ; GFX1032-NEXT:    s_mov_b32 s14, -1
-; GFX1032-NEXT:    v_mbcnt_lo_u32_b32 v0, s0, 0
 ; GFX1032-NEXT:    s_mov_b32 s15, 0x31c16000
 ; GFX1032-NEXT:    s_add_u32 s12, s12, s11
+; GFX1032-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
 ; GFX1032-NEXT:    s_addc_u32 s13, s13, 0
 ; GFX1032-NEXT:    s_mov_b32 s2, 0
-; GFX1032-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
 ; GFX1032-NEXT:    s_and_saveexec_b32 s1, vcc_lo
 ; GFX1032-NEXT:    s_cbranch_execz .LBB13_3
 ; GFX1032-NEXT:  ; %bb.1:
@@ -9812,16 +9812,16 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_uni_value_agent
 ;
 ; GFX1032-DPP-LABEL: global_atomic_fadd_double_uni_address_uni_value_agent_scope_strictfp:
 ; GFX1032-DPP:       ; %bb.0:
-; GFX1032-DPP-NEXT:    s_mov_b32 s12, SCRATCH_RSRC_DWORD0
 ; GFX1032-DPP-NEXT:    s_mov_b32 s0, exec_lo
+; GFX1032-DPP-NEXT:    s_mov_b32 s12, SCRATCH_RSRC_DWORD0
+; GFX1032-DPP-NEXT:    v_mbcnt_lo_u32_b32 v0, s0, 0
 ; GFX1032-DPP-NEXT:    s_mov_b32 s13, SCRATCH_RSRC_DWORD1
 ; GFX1032-DPP-NEXT:    s_mov_b32 s14, -1
-; GFX1032-DPP-NEXT:    v_mbcnt_lo_u32_b32 v0, s0, 0
 ; GFX1032-DPP-NEXT:    s_mov_b32 s15, 0x31c16000
 ; GFX1032-DPP-NEXT:    s_add_u32 s12, s12, s11
+; GFX1032-DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
 ; GFX1032-DPP-NEXT:    s_addc_u32 s13, s13, 0
 ; GFX1032-DPP-NEXT:    s_mov_b32 s2, 0
-; GFX1032-DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
 ; GFX1032-DPP-NEXT:    s_and_saveexec_b32 s1, vcc_lo
 ; GFX1032-DPP-NEXT:    s_cbranch_execz .LBB13_3
 ; GFX1032-DPP-NEXT:  ; %bb.1:
@@ -10650,8 +10650,8 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_agent
 ; GFX1032-DPP-NEXT:    v_permlanex16_b32 v5, v3, 0, 0
 ; GFX1032-DPP-NEXT:    v_add_f64 v[3:4], v[3:4], v[5:6]
 ; GFX1032-DPP-NEXT:    s_mov_b32 exec_lo, s0
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v0, v3
 ; GFX1032-DPP-NEXT:    v_mbcnt_lo_u32_b32 v7, exec_lo, 0
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v0, v3
 ; GFX1032-DPP-NEXT:    v_mov_b32_e32 v2, 0
 ; GFX1032-DPP-NEXT:    v_mov_b32_e32 v1, v4
 ; GFX1032-DPP-NEXT:    s_mov_b32 s2, 0
@@ -11565,8 +11565,8 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_agent
 ; GFX1032-DPP-NEXT:    v_permlanex16_b32 v5, v3, 0, 0
 ; GFX1032-DPP-NEXT:    v_add_f64 v[3:4], v[3:4], v[5:6]
 ; GFX1032-DPP-NEXT:    s_mov_b32 exec_lo, s0
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v0, v3
 ; GFX1032-DPP-NEXT:    v_mbcnt_lo_u32_b32 v7, exec_lo, 0
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v0, v3
 ; GFX1032-DPP-NEXT:    v_mov_b32_e32 v2, 0
 ; GFX1032-DPP-NEXT:    v_mov_b32_e32 v1, v4
 ; GFX1032-DPP-NEXT:    s_mov_b32 s2, 0
@@ -13748,8 +13748,8 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_defau
 ; GFX1032-DPP-NEXT:    v_permlanex16_b32 v10, v8, 0, 0
 ; GFX1032-DPP-NEXT:    v_add_f64 v[8:9], v[8:9], v[10:11]
 ; GFX1032-DPP-NEXT:    s_mov_b32 exec_lo, s0
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v41, v8
 ; GFX1032-DPP-NEXT:    v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v41, v8
 ; GFX1032-DPP-NEXT:    v_mov_b32_e32 v0, 0
 ; GFX1032-DPP-NEXT:    v_mov_b32_e32 v42, v9
 ; GFX1032-DPP-NEXT:    s_mov_b32 s46, 0
diff --git a/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fmax.ll b/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fmax.ll
index 7792422291998..a4410bb9ed2d0 100644
--- a/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fmax.ll
+++ b/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fmax.ll
@@ -3366,17 +3366,17 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_uni_value_agent
 ;
 ; GFX1032-LABEL: global_atomic_fmax_double_uni_address_uni_value_agent_scope_unsafe:
 ; GFX1032:       ; %bb.0:
+; GFX1032-NEXT:    v_mbcnt_lo_u32_b32 v3, exec_lo, 0
 ; GFX1032-NEXT:    s_mov_b32 s48, SCRATCH_RSRC_DWORD0
 ; GFX1032-NEXT:    s_mov_b32 s49, SCRATCH_RSRC_DWORD1
 ; GFX1032-NEXT:    s_mov_b32 s50, -1
-; GFX1032-NEXT:    v_mbcnt_lo_u32_b32 v3, exec_lo, 0
 ; GFX1032-NEXT:    s_mov_b32 s51, 0x31c16000
+; GFX1032-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v3
 ; GFX1032-NEXT:    s_add_u32 s48, s48, s11
 ; GFX1032-NEXT:    s_addc_u32 s49, s49, 0
 ; GFX1032-NEXT:    s_mov_b64 s[40:41], s[0:1]
 ; GFX1032-NEXT:    s_mov_b32 s46, 0
 ; GFX1032-NEXT:    s_movk_i32 s32, 0x400
-; GFX1032-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v3
 ; GFX1032-NEXT:    s_and_saveexec_b32 s0, vcc_lo
 ; GFX1032-NEXT:    s_cbranch_execz .LBB6_3
 ; GFX1032-NEXT:  ; %bb.1:
@@ -3806,17 +3806,17 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_uni_value_agent
 ;
 ; GFX1032-DPP-LABEL: global_atomic_fmax_double_uni_address_uni_value_agent_scope_unsafe:
 ; GFX1032-DPP:       ; %bb.0:
+; GFX1032-DPP-NEXT:    v_mbcnt_lo_u32_b32 v3, exec_lo, 0
 ; GFX1032-DPP-NEXT:    s_mov_b32 s48, SCRATCH_RSRC_DWORD0
 ; GFX1032-DPP-NEXT:    s_mov_b32 s49, SCRATCH_RSRC_DWORD1
 ; GFX1032-DPP-NEXT:    s_mov_b32 s50, -1
-; GFX1032-DPP-NEXT:    v_mbcnt_lo_u32_b32 v3, exec_lo, 0
 ; GFX1032-DPP-NEXT:    s_mov_b32 s51, 0x31c16000
+; GFX1032-DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v3
 ; GFX1032-DPP-NEXT:    s_add_u32 s48, s48, s11
 ; GFX1032-DPP-NEXT:    s_addc_u32 s49, s49, 0
 ; GFX1032-DPP-NEXT:    s_mov_b64 s[40:41], s[0:1]
 ; GFX1032-DPP-NEXT:    s_mov_b32 s46, 0
 ; GFX1032-DPP-NEXT:    s_movk_i32 s32, 0x400
-; GFX1032-DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v3
 ; GFX1032-DPP-NEXT:    s_and_saveexec_b32 s0, vcc_lo
 ; GFX1032-DPP-NEXT:    s_cbranch_execz .LBB6_3
 ; GFX1032-DPP-NEXT:  ; %bb.1:
@@ -5094,8 +5094,8 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_agent
 ; GFX1032-DPP-NEXT:    v_max_f64 v[10:11], v[10:11], v[10:11]
 ; GFX1032-DPP-NEXT:    v_max_f64 v[8:9], v[8:9], v[10:11]
 ; GFX1032-DPP-NEXT:    s_mov_b32 exec_lo, s0
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v3, v8
 ; GFX1032-DPP-NEXT:    v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v3, v8
 ; GFX1032-DPP-NEXT:    v_mov_b32_e32 v0, 0
 ; GFX1032-DPP-NEXT:    v_mov_b32_e32 v4, v9
 ; GFX1032-DPP-NEXT:    s_mov_b32 s46, 0
@@ -6469,8 +6469,8 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_one_a
 ; GFX1032-DPP-NEXT:    v_max_f64 v[5:6], v[5:6], v[5:6]
 ; GFX1032-DPP-NEXT:    v_max_f64 v[3:4], v[3:4], v[5:6]
 ; GFX1032-DPP-NEXT:    s_mov_b32 exec_lo, s0
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v0, v3
 ; GFX1032-DPP-NEXT:    v_mbcnt_lo_u32_b32 v7, exec_lo, 0
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v0, v3
 ; GFX1032-DPP-NEXT:    v_mov_b32_e32 v2, 0
 ; GFX1032-DPP-NEXT:    v_mov_b32_e32 v1, v4
 ; GFX1032-DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v7
@@ -6914,17 +6914,17 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_uni_value_defau
 ;
 ; GFX1032-LABEL: global_atomic_fmax_double_uni_address_uni_value_default_scope_unsafe:
 ; GFX1032:       ; %bb.0:
+; GFX1032-NEXT:    v_mbcnt_lo_u32_b32 v3, exec_lo, 0
 ; GFX1032-NEXT:    s_mov_b32 s48, SCRATCH_RSRC_DWORD0
 ; GFX1032-NEXT:    s_mov_b32 s49, SCRATCH_RSRC_DWORD1
 ; GFX1032-NEXT:    s_mov_b32 s50, -1
-; GFX1032-NEXT:    v_mbcnt_lo_u32_b32 v3, exec_lo, 0
 ; GFX1032-NEXT:    s_mov_b32 s51, 0x31c16000
+; GFX1032-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v3
 ; GFX1032-NEXT:    s_add_u32 s48, s48, s11
 ; GFX1032-NEXT:    s_addc_u32 s49, s49, 0
 ; GFX1032-NEXT:    s_mov_b64 s[40:41], s[0:1]
 ; GFX1032-NEXT:    s_mov_b32 s46, 0
 ; GFX1032-NEXT:    s_movk_i32 s32, 0x400
-; GFX1032-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v3
 ; GFX1032-NEXT:    s_and_saveexec_b32 s0, vcc_lo
 ; GFX1032-NEXT:    s_cbranch_execz .LBB10_3
 ; GFX1032-NEXT:  ; %bb.1:
@@ -7354,17 +7354,17 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_uni_value_defau
 ;
 ; GFX1032-DPP-LABEL: global_atomic_fmax_double_uni_address_uni_value_default_scope_unsafe:
 ; GFX1032-DPP:       ; %bb.0:
+; GFX1032-DPP-NEXT:    v_mbcnt_lo_u32_b32 v3, exec_lo, 0
 ; GFX1032-DPP-NEXT:    s_mov_b32 s48, SCRATCH_RSRC_DWORD0
 ; GFX1032-DPP-NEXT:    s_mov_b32 s49, SCRATCH_RSRC_DWORD1
 ; GFX1032-DPP-NEXT:    s_mov_b32 s50, -1
-; GFX1032-DPP-NEXT:    v_mbcnt_lo_u32_b32 v3, exec_lo, 0
 ; GFX1032-DPP-NEXT:    s_mov_b32 s51, 0x31c16000
+; GFX1032-DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v3
 ; GFX1032-DPP-NEXT:    s_add_u32 s48, s48, s11
 ; GFX1032-DPP-NEXT:    s_addc_u32 s49, s49, 0
 ; GFX1032-DPP-NEXT:    s_mov_b64 s[40:41], s[0:1]
 ; GFX1032-DPP-NEXT:    s_mov_b32 s46, 0
 ; GFX1032-DPP-NEXT:    s_movk_i32 s32, 0x400
-; GFX1032-DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v3
 ; GFX1032-DPP-NEXT:    s_and_saveexec_b32 s0, vcc_lo
 ; GFX1032-DPP-NEXT:    s_cbranch_execz .LBB10_3
 ; GFX1032-DPP-NEXT:  ; %bb.1:
@@ -8642,8 +8642,8 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_defau
 ; GFX1032-DPP-NEXT:    v_max_f64 v[10:11], v[10:11], v[10:11]
 ; GFX1032-DPP-NEXT:    v_max_f64 v[8:9], v[8:9], v[10:11]
 ; GFX1032-DPP-NEXT:    s_mov_b32 exec_lo, s0
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v3, v8
 ; GFX1032-DPP-NEXT:    v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v3, v8
 ; GFX1032-DPP-NEXT:    v_mov_b32_e32 v0, 0
 ; GFX1032-DPP-NEXT:    v_mov_b32_e32 v4, v9
 ; GFX1032-DPP-NEXT:    s_mov_b32 s46, 0
diff --git a/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fmin.ll b/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fmin.ll
index cb3291df891af..68d7dcc60506c 100644
--- a/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fmin.ll
+++ b/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fmin.ll
@@ -3366,17 +3366,17 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_uni_value_agent
 ;
 ; GFX1032-LABEL: global_atomic_fmin_double_uni_address_uni_value_agent_scope_unsafe:
 ; GFX1032:       ; %bb.0:
+; GFX1032-NEXT:    v_mbcnt_lo_u32_b32 v3, exec_lo, 0
 ; GFX1032-NEXT:    s_mov_b32 s48, SCRATCH_RSRC_DWORD0
 ; GFX1032-NEXT:    s_mov_b32 s49, SCRATCH_RSRC_DWORD1
 ; GFX1032-NEXT:    s_mov_b32 s50, -1
-; GFX1032-NEXT:    v_mbcnt_lo_u32_b32 v3, exec_lo, 0
 ; GFX1032-NEXT:    s_mov_b32 s51, 0x31c16000
+; GFX1032-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v3
 ; GFX1032-NEXT:    s_add_u32 s48, s48, s11
 ; GFX1032-NEXT:    s_addc_u32 s49, s49, 0
 ; GFX1032-NEXT:    s_mov_b64 s[40:41], s[0:1]
 ; GFX1032-NEXT:    s_mov_b32 s46, 0
 ; GFX1032-NEXT:    s_movk_i32 s32, 0x400
-; GFX1032-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v3
 ; GFX1032-NEXT:    s_and_saveexec_b32 s0, vcc_lo
 ; GFX1032-NEXT:    s_cbranch_execz .LBB6_3
 ; GFX1032-NEXT:  ; %bb.1:
@@ -3806,17 +3806,17 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_uni_value_agent
 ;
 ; GFX1032-DPP-LABEL: global_atomic_fmin_double_uni_address_uni_value_agent_scope_unsafe:
 ; GFX1032-DPP:       ; %bb.0:
+; GFX1032-DPP-NEXT:    v_mbcnt_lo_u32_b32 v3, exec_lo, 0
 ; GFX1032-DPP-NEXT:    s_mov_b32 s48, SCRATCH_RSRC_DWORD0
 ; GFX1032-DPP-NEXT:    s_mov_b32 s49, SCRATCH_RSRC_DWORD1
 ; GFX1032-DPP-NEXT:    s_mov_b32 s50, -1
-; GFX1032-DPP-NEXT:    v_mbcnt_lo_u32_b32 v3, exec_lo, 0
 ; GFX1032-DPP-NEXT:    s_mov_b32 s51, 0x31c16000
+; GFX1032-DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v3
 ; GFX1032-DPP-NEXT:    s_add_u32 s48, s48, s11
 ; GFX1032-DPP-NEXT:    s_addc_u32 s49, s49, 0
 ; GFX1032-DPP-NEXT:    s_mov_b64 s[40:41], s[0:1]
 ; GFX1032-DPP-NEXT:    s_mov_b32 s46, 0
 ; GFX1032-DPP-NEXT:    s_movk_i32 s32, 0x400
-; GFX1032-DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v3
 ; GFX1032-DPP-NEXT:    s_and_saveexec_b32 s0, vcc_lo
 ; GFX1032-DPP-NEXT:    s_cbranch_execz .LBB6_3
 ; GFX1032-DPP-NEXT:  ; %bb.1:
@@ -5094,8 +5094,8 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_agent
 ; GFX1032-DPP-NEXT:    v_max_f64 v[10:11], v[10:11], v[10:11]
 ; GFX1032-DPP-NEXT:    v_min_f64 v[8:9], v[8:9], v[10:11]
 ; GFX1032-DPP-NEXT:    s_mov_b32 exec_lo, s0
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v3, v8
 ; GFX1032-DPP-NEXT:    v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v3, v8
 ; GFX1032-DPP-NEXT:    v_mov_b32_e32 v0, 0
 ; GFX1032-DPP-NEXT:    v_mov_b32_e32 v4, v9
 ; GFX1032-DPP-NEXT:    s_mov_b32 s46, 0
@@ -6469,8 +6469,8 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_one_a
 ; GFX1032-DPP-NEXT:    v_max_f64 v[5:6], v[5:6], v[5:6]
 ; GFX1032-DPP-NEXT:    v_min_f64 v[3:4], v[3:4], v[5:6]
 ; GFX1032-DPP-NEXT:    s_mov_b32 exec_lo, s0
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v0, v3
 ; GFX1032-DPP-NEXT:    v_mbcnt_lo_u32_b32 v7, exec_lo, 0
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v0, v3
 ; GFX1032-DPP-NEXT:    v_mov_b32_e32 v2, 0
 ; GFX1032-DPP-NEXT:    v_mov_b32_e32 v1, v4
 ; GFX1032-DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v7
@@ -6914,17 +6914,17 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_uni_value_defau
 ;
 ; GFX1032-LABEL: global_atomic_fmin_double_uni_address_uni_value_default_scope_unsafe:
 ; GFX1032:       ; %bb.0:
+; GFX1032-NEXT:    v_mbcnt_lo_u32_b32 v3, exec_lo, 0
 ; GFX1032-NEXT:    s_mov_b32 s48, SCRATCH_RSRC_DWORD0
 ; GFX1032-NEXT:    s_mov_b32 s49, SCRATCH_RSRC_DWORD1
 ; GFX1032-NEXT:    s_mov_b32 s50, -1
-; GFX1032-NEXT:    v_mbcnt_lo_u32_b32 v3, exec_lo, 0
 ; GFX1032-NEXT:    s_mov_b32 s51, 0x31c16000
+; GFX1032-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v3
 ; GFX1032-NEXT:    s_add_u32 s48, s48, s11
 ; GFX1032-NEXT:    s_addc_u32 s49, s49, 0
 ; GFX1032-NEXT:    s_mov_b64 s[40:41], s[0:1]
 ; GFX1032-NEXT:    s_mov_b32 s46, 0
 ; GFX1032-NEXT:    s_movk_i32 s32, 0x400
-; GFX1032-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v3
 ; GFX1032-NEXT:    s_and_saveexec_b32 s0, vcc_lo
 ; GFX1032-NEXT:    s_cbranch_execz .LBB10_3
 ; GFX1032-NEXT:  ; %bb.1:
@@ -7354,17 +7354,17 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_uni_value_defau
 ;
 ; GFX1032-DPP-LABEL: global_atomic_fmin_double_uni_address_uni_value_default_scope_unsafe:
 ; GFX1032-DPP:       ; %bb.0:
+; GFX1032-DPP-NEXT:    v_mbcnt_lo_u32_b32 v3, exec_lo, 0
 ; GFX1032-DPP-NEXT:    s_mov_b32 s48, SCRATCH_RSRC_DWORD0
 ; GFX1032-DPP-NEXT:    s_mov_b32 s49, SCRATCH_RSRC_DWORD1
 ; GFX1032-DPP-NEXT:    s_mov_b32 s50, -1
-; GFX1032-DPP-NEXT:    v_mbcnt_lo_u32_b32 v3, exec_lo, 0
 ; GFX1032-DPP-NEXT:    s_mov_b32 s51, 0x31c16000
+; GFX1032-DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v3
 ; GFX1032-DPP-NEXT:    s_add_u32 s48, s48, s11
 ; GFX1032-DPP-NEXT:    s_addc_u32 s49, s49, 0
 ; GFX1032-DPP-NEXT:    s_mov_b64 s[40:41], s[0:1]
 ; GFX1032-DPP-NEXT:    s_mov_b32 s46, 0
 ; GFX1032-DPP-NEXT:    s_movk_i32 s32, 0x400
-; GFX1032-DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v3
 ; GFX1032-DPP-NEXT:    s_and_saveexec_b32 s0, vcc_lo
 ; GFX1032-DPP-NEXT:    s_cbranch_execz .LBB10_3
 ; GFX1032-DPP-NEXT:  ; %bb.1:
@@ -8642,8 +8642,8 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_defau
 ; GFX1032-DPP-NEXT:    v_max_f64 v[10:11], v[10:11], v[10:11]
 ; GFX1032-DPP-NEXT:    v_min_f64 v[8:9], v[8:9], v[10:11]
 ; GFX1032-DPP-NEXT:    s_mov_b32 exec_lo, s0
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v3, v8
 ; GFX1032-DPP-NEXT:    v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v3, v8
 ; GFX1032-DPP-NEXT:    v_mov_b32_e32 v0, 0
 ; GFX1032-DPP-NEXT:    v_mov_b32_e32 v4, v9
 ; GFX1032-DPP-NEXT:    s_mov_b32 s46, 0
diff --git a/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fsub.ll b/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fsub.ll
index 6dc3a1971a485..7126680525b87 100644
--- a/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fsub.ll
+++ b/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fsub.ll
@@ -1379,16 +1379,16 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_uni_value_one_as_scope
 ;
 ; GFX1032-LABEL: global_atomic_fsub_uni_address_uni_value_one_as_scope_unsafe_structfp:
 ; GFX1032:       ; %bb.0:
-; GFX1032-NEXT:    s_mov_b32 s12, SCRATCH_RSRC_DWORD0
 ; GFX1032-NEXT:    s_mov_b32 s0, exec_lo
+; GFX1032-NEXT:    s_mov_b32 s12, SCRATCH_RSRC_DWORD0
+; GFX1032-NEXT:    v_mbcnt_lo_u32_b32 v0, s0, 0
 ; GFX1032-NEXT:    s_mov_b32 s13, SCRATCH_RSRC_DWORD1
 ; GFX1032-NEXT:    s_mov_b32 s14, -1
-; GFX1032-NEXT:    v_mbcnt_lo_u32_b32 v0, s0, 0
 ; GFX1032-NEXT:    s_mov_b32 s15, 0x31c16000
 ; GFX1032-NEXT:    s_add_u32 s12, s12, s11
+; GFX1032-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
 ; GFX1032-NEXT:    s_addc_u32 s13, s13, 0
 ; GFX1032-NEXT:    s_mov_b32 s2, 0
-; GFX1032-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
 ; GFX1032-NEXT:    s_and_saveexec_b32 s1, vcc_lo
 ; GFX1032-NEXT:    s_cbranch_execz .LBB2_3
 ; GFX1032-NEXT:  ; %bb.1:
@@ -1629,16 +1629,16 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_uni_value_one_as_scope
 ;
 ; GFX1032-DPP-LABEL: global_atomic_fsub_uni_address_uni_value_one_as_scope_unsafe_structfp:
 ; GFX1032-DPP:       ; %bb.0:
-; GFX1032-DPP-NEXT:    s_mov_b32 s12, SCRATCH_RSRC_DWORD0
 ; GFX1032-DPP-NEXT:    s_mov_b32 s0, exec_lo
+; GFX1032-DPP-NEXT:    s_mov_b32 s12, SCRATCH_RSRC_DWORD0
+; GFX1032-DPP-NEXT:    v_mbcnt_lo_u32_b32 v0, s0, 0
 ; GFX1032-DPP-NEXT:    s_mov_b32 s13, SCRATCH_RSRC_DWORD1
 ; GFX1032-DPP-NEXT:    s_mov_b32 s14, -1
-; GFX1032-DPP-NEXT:    v_mbcnt_lo_u32_b32 v0, s0, 0
 ; GFX1032-DPP-NEXT:    s_mov_b32 s15, 0x31c16000
 ; GFX1032-DPP-NEXT:    s_add_u32 s12, s12, s11
+; GFX1032-DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
 ; GFX1032-DPP-NEXT:    s_addc_u32 s13, s13, 0
 ; GFX1032-DPP-NEXT:    s_mov_b32 s2, 0
-; GFX1032-DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
 ; GFX1032-DPP-NEXT:    s_and_saveexec_b32 s1, vcc_lo
 ; GFX1032-DPP-NEXT:    s_cbranch_execz .LBB2_3
 ; GFX1032-DPP-NEXT:  ; %bb.1:
@@ -2711,16 +2711,16 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_uni_value_agent_scope_
 ;
 ; GFX1032-LABEL: global_atomic_fsub_uni_address_uni_value_agent_scope_strictfp:
 ; GFX1032:       ; %bb.0:
-; GFX1032-NEXT:    s_mov_b32 s12, SCRATCH_RSRC_DWORD0
 ; GFX1032-NEXT:    s_mov_b32 s0, exec_lo
+; GFX1032-NEXT:    s_mov_b32 s12, SCRATCH_RSRC_DWORD0
+; GFX1032-NEXT:    v_mbcnt_lo_u32_b32 v0, s0, 0
 ; GFX1032-NEXT:    s_mov_b32 s13, SCRATCH_RSRC_DWORD1
 ; GFX1032-NEXT:    s_mov_b32 s14, -1
-; GFX1032-NEXT:    v_mbcnt_lo_u32_b32 v0, s0, 0
 ; GFX1032-NEXT:    s_mov_b32 s15, 0x31c16000
 ; GFX1032-NEXT:    s_add_u32 s12, s12, s11
+; GFX1032-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
 ; GFX1032-NEXT:    s_addc_u32 s13, s13, 0
 ; GFX1032-NEXT:    s_mov_b32 s2, 0
-; GFX1032-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
 ; GFX1032-NEXT:    s_and_saveexec_b32 s1, vcc_lo
 ; GFX1032-NEXT:    s_cbranch_execz .LBB4_3
 ; GFX1032-NEXT:  ; %bb.1:
@@ -2961,16 +2961,16 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_uni_value_agent_scope_
 ;
 ; GFX1032-DPP-LABEL: global_atomic_fsub_uni_address_uni_value_agent_scope_strictfp:
 ; GFX1032-DPP:       ; %bb.0:
-; GFX1032-DPP-NEXT:    s_mov_b32 s12, SCRATCH_RSRC_DWORD0
 ; GFX1032-DPP-NEXT:    s_mov_b32 s0, exec_lo
+; GFX1032-DPP-NEXT:    s_mov_b32 s12, SCRATCH_RSRC_DWORD0
+; GFX1032-DPP-NEXT:    v_mbcnt_lo_u32_b32 v0, s0, 0
 ; GFX1032-DPP-NEXT:    s_mov_b32 s13, SCRATCH_RSRC_DWORD1
 ; GFX1032-DPP-NEXT:    s_mov_b32 s14, -1
-; GFX1032-DPP-NEXT:    v_mbcnt_lo_u32_b32 v0, s0, 0
 ; GFX1032-DPP-NEXT:    s_mov_b32 s15, 0x31c16000
 ; GFX1032-DPP-NEXT:    s_add_u32 s12, s12, s11
+; GFX1032-DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
 ; GFX1032-DPP-NEXT:    s_addc_u32 s13, s13, 0
 ; GFX1032-DPP-NEXT:    s_mov_b32 s2, 0
-; GFX1032-DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
 ; GFX1032-DPP-NEXT:    s_and_saveexec_b32 s1, vcc_lo
 ; GFX1032-DPP-NEXT:    s_cbranch_execz .LBB4_3
 ; GFX1032-DPP-NEXT:  ; %bb.1:
@@ -4871,16 +4871,16 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_uni_value_default_scop
 ;
 ; GFX1032-LABEL: global_atomic_fsub_uni_address_uni_value_default_scope_strictfp:
 ; GFX1032:       ; %bb.0:
-; GFX1032-NEXT:    s_mov_b32 s12, SCRATCH_RSRC_DWORD0
 ; GFX1032-NEXT:    s_mov_b32 s0, exec_lo
+; GFX1032-NEXT:    s_mov_b32 s12, SCRATCH_RSRC_DWORD0
+; GFX1032-NEXT:    v_mbcnt_lo_u32_b32 v0, s0, 0
 ; GFX1032-NEXT:    s_mov_b32 s13, SCRATCH_RSRC_DWORD1
 ; GFX1032-NEXT:    s_mov_b32 s14, -1
-; GFX1032-NEXT:    v_mbcnt_lo_u32_b32 v0, s0, 0
 ; GFX1032-NEXT:    s_mov_b32 s15, 0x31c16000
 ; GFX1032-NEXT:    s_add_u32 s12, s12, s11
+; GFX1032-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
 ; GFX1032-NEXT:    s_addc_u32 s13, s13, 0
 ; GFX1032-NEXT:    s_mov_b32 s2, 0
-; GFX1032-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
 ; GFX1032-NEXT:    s_and_saveexec_b32 s1, vcc_lo
 ; GFX1032-NEXT:    s_cbranch_execz .LBB7_3
 ; GFX1032-NEXT:  ; %bb.1:
@@ -5121,16 +5121,16 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_uni_value_default_scop
 ;
 ; GFX1032-DPP-LABEL: global_atomic_fsub_uni_address_uni_value_default_scope_strictfp:
 ; GFX1032-DPP:       ; %bb.0:
-; GFX1032-DPP-NEXT:    s_mov_b32 s12, SCRATCH_RSRC_DWORD0
 ; GFX1032-DPP-NEXT:    s_mov_b32 s0, exec_lo
+; GFX1032-DPP-NEXT:    s_mov_b32 s12, SCRATCH_RSRC_DWORD0
+; GFX1032-DPP-NEXT:    v_mbcnt_lo_u32_b32 v0, s0, 0
 ; GFX1032-DPP-NEXT:    s_mov_b32 s13, SCRATCH_RSRC_DWORD1
 ; GFX1032-DPP-NEXT:    s_mov_b32 s14, -1
-; GFX1032-DPP-NEXT:    v_mbcnt_lo_u32_b32 v0, s0, 0
 ; GFX1032-DPP-NEXT:    s_mov_b32 s15, 0x31c16000
 ; GFX1032-DPP-NEXT:    s_add_u32 s12, s12, s11
+; GFX1032-DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
 ; GFX1032-DPP-NEXT:    s_addc_u32 s13, s13, 0
 ; GFX1032-DPP-NEXT:    s_mov_b32 s2, 0
-; GFX1032-DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
 ; GFX1032-DPP-NEXT:    s_and_saveexec_b32 s1, vcc_lo
 ; GFX1032-DPP-NEXT:    s_cbranch_execz .LBB7_3
 ; GFX1032-DPP-NEXT:  ; %bb.1:
@@ -6315,19 +6315,19 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_uni_value_agent
 ;
 ; GFX1032-LABEL: global_atomic_fsub_double_uni_address_uni_value_agent_scope_unsafe:
 ; GFX1032:       ; %bb.0:
-; GFX1032-NEXT:    s_mov_b32 s48, SCRATCH_RSRC_DWORD0
 ; GFX1032-NEXT:    s_mov_b32 s42, s9
 ; GFX1032-NEXT:    s_mov_b32 s9, exec_lo
+; GFX1032-NEXT:    s_mov_b32 s48, SCRATCH_RSRC_DWORD0
+; GFX1032-NEXT:    v_mbcnt_lo_u32_b32 v3, s9, 0
 ; GFX1032-NEXT:    s_mov_b32 s49, SCRATCH_RSRC_DWORD1
 ; GFX1032-NEXT:    s_mov_b32 s50, -1
-; GFX1032-NEXT:    v_mbcnt_lo_u32_b32 v3, s9, 0
 ; GFX1032-NEXT:    s_mov_b32 s51, 0x31c16000
 ; GFX1032-NEXT:    s_add_u32 s48, s48, s11
+; GFX1032-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v3
 ; GFX1032-NEXT:    s_addc_u32 s49, s49, 0
 ; GFX1032-NEXT:    s_mov_b64 s[40:41], s[0:1]
 ; GFX1032-NEXT:    s_mov_b32 s46, 0
 ; GFX1032-NEXT:    s_movk_i32 s32, 0x400
-; GFX1032-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v3
 ; GFX1032-NEXT:    s_and_saveexec_b32 s0, vcc_lo
 ; GFX1032-NEXT:    s_cbranch_execz .LBB9_3
 ; GFX1032-NEXT:  ; %bb.1:
@@ -6774,19 +6774,19 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_uni_value_agent
 ;
 ; GFX1032-DPP-LABEL: global_atomic_fsub_double_uni_address_uni_value_agent_scope_unsafe:
 ; GFX1032-DPP:       ; %bb.0:
-; GFX1032-DPP-NEXT:    s_mov_b32 s48, SCRATCH_RSRC_DWORD0
 ; GFX1032-DPP-NEXT:    s_mov_b32 s42, s9
 ; GFX1032-DPP-NEXT:    s_mov_b32 s9, exec_lo
+; GFX1032-DPP-NEXT:    s_mov_b32 s48, SCRATCH_RSRC_DWORD0
+; GFX1032-DPP-NEXT:    v_mbcnt_lo_u32_b32 v3, s9, 0
 ; GFX1032-DPP-NEXT:    s_mov_b32 s49, SCRATCH_RSRC_DWORD1
 ; GFX1032-DPP-NEXT:    s_mov_b32 s50, -1
-; GFX1032-DPP-NEXT:    v_mbcnt_lo_u32_b32 v3, s9, 0
 ; GFX1032-DPP-NEXT:    s_mov_b32 s51, 0x31c16000
 ; GFX1032-DPP-NEXT:    s_add_u32 s48, s48, s11
+; GFX1032-DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v3
 ; GFX1032-DPP-NEXT:    s_addc_u32 s49, s49, 0
 ; GFX1032-DPP-NEXT:    s_mov_b64 s[40:41], s[0:1]
 ; GFX1032-DPP-NEXT:    s_mov_b32 s46, 0
 ; GFX1032-DPP-NEXT:    s_movk_i32 s32, 0x400
-; GFX1032-DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v3
 ; GFX1032-DPP-NEXT:    s_and_saveexec_b32 s0, vcc_lo
 ; GFX1032-DPP-NEXT:    s_cbranch_execz .LBB9_3
 ; GFX1032-DPP-NEXT:  ; %bb.1:
@@ -8020,8 +8020,8 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_agent
 ; GFX1032-DPP-NEXT:    v_permlanex16_b32 v10, v8, 0, 0
 ; GFX1032-DPP-NEXT:    v_add_f64 v[8:9], v[8:9], v[10:11]
 ; GFX1032-DPP-NEXT:    s_mov_b32 exec_lo, s0
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v41, v8
 ; GFX1032-DPP-NEXT:    v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v41, v8
 ; GFX1032-DPP-NEXT:    v_mov_b32_e32 v0, 0
 ; GFX1032-DPP-NEXT:    v_mov_b32_e32 v42, v9
 ; GFX1032-DPP-NEXT:    s_mov_b32 s46, 0
@@ -8450,16 +8450,16 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_uni_value_one_a
 ;
 ; GFX1032-LABEL: global_atomic_fsub_double_uni_address_uni_value_one_as_scope_unsafe_structfp:
 ; GFX1032:       ; %bb.0:
-; GFX1032-NEXT:    s_mov_b32 s12, SCRATCH_RSRC_DWORD0
 ; GFX1032-NEXT:    s_mov_b32 s0, exec_lo
+; GFX1032-NEXT:    s_mov_b32 s12, SCRATCH_RSRC_DWORD0
+; GFX1032-NEXT:    v_mbcnt_lo_u32_b32 v0, s0, 0
 ; GFX1032-NEXT:    s_mov_b32 s13, SCRATCH_RSRC_DWORD1
 ; GFX1032-NEXT:    s_mov_b32 s14, -1
-; GFX1032-NEXT:    v_mbcnt_lo_u32_b32 v0, s0, 0
 ; GFX1032-NEXT:    s_mov_b32 s15, 0x31c16000
 ; GFX1032-NEXT:    s_add_u32 s12, s12, s11
+; GFX1032-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
 ; GFX1032-NEXT:    s_addc_u32 s13, s13, 0
 ; GFX1032-NEXT:    s_mov_b32 s2, 0
-; GFX1032-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
 ; GFX1032-NEXT:    s_and_saveexec_b32 s1, vcc_lo
 ; GFX1032-NEXT:    s_cbranch_execz .LBB11_3
 ; GFX1032-NEXT:  ; %bb.1:
@@ -8707,16 +8707,16 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_uni_value_one_a
 ;
 ; GFX1032-DPP-LABEL: global_atomic_fsub_double_uni_address_uni_value_one_as_scope_unsafe_structfp:
 ; GFX1032-DPP:       ; %bb.0:
-; GFX1032-DPP-NEXT:    s_mov_b32 s12, SCRATCH_RSRC_DWORD0
 ; GFX1032-DPP-NEXT:    s_mov_b32 s0, exec_lo
+; GFX1032-DPP-NEXT:    s_mov_b32 s12, SCRATCH_RSRC_DWORD0
+; GFX1032-DPP-NEXT:    v_mbcnt_lo_u32_b32 v0, s0, 0
 ; GFX1032-DPP-NEXT:    s_mov_b32 s13, SCRATCH_RSRC_DWORD1
 ; GFX1032-DPP-NEXT:    s_mov_b32 s14, -1
-; GFX1032-DPP-NEXT:    v_mbcnt_lo_u32_b32 v0, s0, 0
 ; GFX1032-DPP-NEXT:    s_mov_b32 s15, 0x31c16000
 ; GFX1032-DPP-NEXT:    s_add_u32 s12, s12, s11
+; GFX1032-DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
 ; GFX1032-DPP-NEXT:    s_addc_u32 s13, s13, 0
 ; GFX1032-DPP-NEXT:    s_mov_b32 s2, 0
-; GFX1032-DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
 ; GFX1032-DPP-NEXT:    s_and_saveexec_b32 s1, vcc_lo
 ; GFX1032-DPP-NEXT:    s_cbranch_execz .LBB11_3
 ; GFX1032-DPP-NEXT:  ; %bb.1:
@@ -9544,8 +9544,8 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_one_a
 ; GFX1032-DPP-NEXT:    v_permlanex16_b32 v5, v3, 0, 0
 ; GFX1032-DPP-NEXT:    v_add_f64 v[3:4], v[3:4], v[5:6]
 ; GFX1032-DPP-NEXT:    s_mov_b32 exec_lo, s0
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v0, v3
 ; GFX1032-DPP-NEXT:    v_mbcnt_lo_u32_b32 v7, exec_lo, 0
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v0, v3
 ; GFX1032-DPP-NEXT:    v_mov_b32_e32 v2, 0
 ; GFX1032-DPP-NEXT:    v_mov_b32_e32 v1, v4
 ; GFX1032-DPP-NEXT:    s_mov_b32 s2, 0
@@ -9882,16 +9882,16 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_uni_value_agent
 ;
 ; GFX1032-LABEL: global_atomic_fsub_double_uni_address_uni_value_agent_scope_strictfp:
 ; GFX1032:       ; %bb.0:
-; GFX1032-NEXT:    s_mov_b32 s12, SCRATCH_RSRC_DWORD0
 ; GFX1032-NEXT:    s_mov_b32 s0, exec_lo
+; GFX1032-NEXT:    s_mov_b32 s12, SCRATCH_RSRC_DWORD0
+; GFX1032-NEXT:    v_mbcnt_lo_u32_b32 v0, s0, 0
 ; GFX1032-NEXT:    s_mov_b32 s13, SCRATCH_RSRC_DWORD1
 ; GFX1032-NEXT:    s_mov_b32 s14, -1
-; GFX1032-NEXT:    v_mbcnt_lo_u32_b32 v0, s0, 0
 ; GFX1032-NEXT:    s_mov_b32 s15, 0x31c16000
 ; GFX1032-NEXT:    s_add_u32 s12, s12, s11
+; GFX1032-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
 ; GFX1032-NEXT:    s_addc_u32 s13, s13, 0
 ; GFX1032-NEXT:    s_mov_b32 s2, 0
-; GFX1032-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
 ; GFX1032-NEXT:    s_and_saveexec_b32 s1, vcc_lo
 ; GFX1032-NEXT:    s_cbranch_execz .LBB13_3
 ; GFX1032-NEXT:  ; %bb.1:
@@ -10139,16 +10139,16 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_uni_value_agent
 ;
 ; GFX1032-DPP-LABEL: global_atomic_fsub_double_uni_address_uni_value_agent_scope_strictfp:
 ; GFX1032-DPP:       ; %bb.0:
-; GFX1032-DPP-NEXT:    s_mov_b32 s12, SCRATCH_RSRC_DWORD0
 ; GFX1032-DPP-NEXT:    s_mov_b32 s0, exec_lo
+; GFX1032-DPP-NEXT:    s_mov_b32 s12, SCRATCH_RSRC_DWORD0
+; GFX1032-DPP-NEXT:    v_mbcnt_lo_u32_b32 v0, s0, 0
 ; GFX1032-DPP-NEXT:    s_mov_b32 s13, SCRATCH_RSRC_DWORD1
 ; GFX1032-DPP-NEXT:    s_mov_b32 s14, -1
-; GFX1032-DPP-NEXT:    v_mbcnt_lo_u32_b32 v0, s0, 0
 ; GFX1032-DPP-NEXT:    s_mov_b32 s15, 0x31c16000
 ; GFX1032-DPP-NEXT:    s_add_u32 s12, s12, s11
+; GFX1032-DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
 ; GFX1032-DPP-NEXT:    s_addc_u32 s13, s13, 0
 ; GFX1032-DPP-NEXT:    s_mov_b32 s2, 0
-; GFX1032-DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
 ; GFX1032-DPP-NEXT:    s_and_saveexec_b32 s1, vcc_lo
 ; GFX1032-DPP-NEXT:    s_cbranch_execz .LBB13_3
 ; GFX1032-DPP-NEXT:  ; %bb.1:
@@ -10977,8 +10977,8 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_agent
 ; GFX1032-DPP-NEXT:    v_permlanex16_b32 v5, v3, 0, 0
 ; GFX1032-DPP-NEXT:    v_add_f64 v[3:4], v[3:4], v[5:6]
 ; GFX1032-DPP-NEXT:    s_mov_b32 exec_lo, s0
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v0, v3
 ; GFX1032-DPP-NEXT:    v_mbcnt_lo_u32_b32 v7, exec_lo, 0
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v0, v3
 ; GFX1032-DPP-NEXT:    v_mov_b32_e32 v2, 0
 ; GFX1032-DPP-NEXT:    v_mov_b32_e32 v1, v4
 ; GFX1032-DPP-NEXT:    s_mov_b32 s2, 0
@@ -11892,8 +11892,8 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_agent
 ; GFX1032-DPP-NEXT:    v_permlanex16_b32 v5, v3, 0, 0
 ; GFX1032-DPP-NEXT:    v_add_f64 v[3:4], v[3:4], v[5:6]
 ; GFX1032-DPP-NEXT:    s_mov_b32 exec_lo, s0
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v0, v3
 ; GFX1032-DPP-NEXT:    v_mbcnt_lo_u32_b32 v7, exec_lo, 0
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v0, v3
 ; GFX1032-DPP-NEXT:    v_mov_b32_e32 v2, 0
 ; GFX1032-DPP-NEXT:    v_mov_b32_e32 v1, v4
 ; GFX1032-DPP-NEXT:    s_mov_b32 s2, 0
@@ -14074,8 +14074,8 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_defau
 ; GFX1032-DPP-NEXT:    v_permlanex16_b32 v10, v8, 0, 0
 ; GFX1032-DPP-NEXT:    v_add_f64 v[8:9], v[8:9], v[10:11]
 ; GFX1032-DPP-NEXT:    s_mov_b32 exec_lo, s0
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v41, v8
 ; GFX1032-DPP-NEXT:    v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v41, v8
 ; GFX1032-DPP-NEXT:    v_mov_b32_e32 v0, 0
 ; GFX1032-DPP-NEXT:    v_mov_b32_e32 v42, v9
 ; GFX1032-DPP-NEXT:    s_mov_b32 s46, 0
diff --git a/llvm/test/CodeGen/AMDGPU/hazard-kill.mir b/llvm/test/CodeGen/AMDGPU/hazard-kill.mir
index 8327ac2cd26e6..05474879680f8 100644
--- a/llvm/test/CodeGen/AMDGPU/hazard-kill.mir
+++ b/llvm/test/CodeGen/AMDGPU/hazard-kill.mir
@@ -1,4 +1,5 @@
 # RUN: llc -mtriple=amdgcn -mcpu=gfx900 -verify-machineinstrs -run-pass post-RA-sched %s -o - | FileCheck -check-prefix=GFX90 %s
+# RUN: llc -mtriple=amdgcn -mcpu=gfx900 -passes=post-RA-sched %s -o - | FileCheck -check-prefix=GFX90 %s
 
 # This tests that a KILL isn't considered as a valid instruction for a hazard
 # slot (e.g. m0 def followed by V_INTERP for gfx9)
diff --git a/llvm/test/CodeGen/AMDGPU/icmp.i16.ll b/llvm/test/CodeGen/AMDGPU/icmp.i16.ll
index c1a074a81b2aa..790a457c2b337 100644
--- a/llvm/test/CodeGen/AMDGPU/icmp.i16.ll
+++ b/llvm/test/CodeGen/AMDGPU/icmp.i16.ll
@@ -1,5 +1,9 @@
 ; RUN: llc -mtriple=amdgcn -mcpu=tonga -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=VI %s
 ; RUN: llc -mtriple=amdgcn -verify-machineinstrs < %s| FileCheck -check-prefix=GCN -check-prefix=SI %s
+; RUN: llc -mtriple=amdgcn -mcpu=gfx1100 -mattr=-real-true16 -verify-machineinstrs < %s| FileCheck -check-prefixes=GCN,GFX11-FAKE16 %s
+; FIXME-TRUE16. In true16 flow, the codegen introduces addtional s2v copy and mov, and revert the operand order thus picking different cmp instructions
+; This should be corrected after addtional mov/copy is removed
+; RUN: llc -mtriple=amdgcn -mcpu=gfx1100 -mattr=+real-true16 -verify-machineinstrs < %s| FileCheck -check-prefixes=GCN,GFX11-TRUE16 %s
 
 ;;;==========================================================================;;;
 ;; 16-bit integer comparisons
@@ -8,6 +12,8 @@
 ; GCN-LABEL: {{^}}i16_eq:
 ; VI: v_cmp_eq_u16_e32 vcc, v{{[0-9]+}}, v{{[0-9]+}}
 ; SI: v_cmp_eq_u32_e32 vcc, v{{[0-9]+}}, v{{[0-9]+}}
+; GFX11-FAKE16: v_cmp_eq_u16_e32 vcc_lo, v{{[0-9]+}}, v{{[0-9]+}}
+; GFX11-TRUE16: v_cmp_eq_u16_e32 vcc_lo, v{{[0-9]+}}.l, v{{[0-9]+}}.h
 define amdgpu_kernel void @i16_eq(ptr addrspace(1) %out, ptr addrspace(1) %a.ptr, ptr addrspace(1) %b.ptr) #0 {
 entry:
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
@@ -26,6 +32,8 @@ entry:
 ; GCN-LABEL: {{^}}i16_ne:
 ; VI: v_cmp_ne_u16_e32 vcc, v{{[0-9]+}}, v{{[0-9]+}}
 ; SI: v_cmp_ne_u32_e32 vcc, v{{[0-9]+}}, v{{[0-9]+}}
+; GFX11-FAKE16: v_cmp_ne_u16_e32 vcc_lo, v{{[0-9]+}}, v{{[0-9]+}}
+; GFX11-TRUE16: v_cmp_ne_u16_e32 vcc_lo, v{{[0-9]+}}.l, v{{[0-9]+}}.h
 define amdgpu_kernel void @i16_ne(ptr addrspace(1) %out, ptr addrspace(1) %a.ptr, ptr addrspace(1) %b.ptr) #0 {
 entry:
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
@@ -44,6 +52,8 @@ entry:
 ; GCN-LABEL: {{^}}i16_ugt:
 ; VI: v_cmp_gt_u16_e32 vcc, v{{[0-9]+}}, v{{[0-9]+}}
 ; SI: v_cmp_gt_u32_e32 vcc, v{{[0-9]+}}, v{{[0-9]+}}
+; GFX11-FAKE16: v_cmp_gt_u16_e32 vcc_lo, v{{[0-9]+}}, v{{[0-9]+}}
+; GFX11-TRUE16: v_cmp_gt_u16_e32 vcc_lo, v{{[0-9]+}}.l, v{{[0-9]+}}.h
 define amdgpu_kernel void @i16_ugt(ptr addrspace(1) %out, ptr addrspace(1) %a.ptr, ptr addrspace(1) %b.ptr) #0 {
 entry:
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
@@ -62,6 +72,8 @@ entry:
 ; GCN-LABEL: {{^}}i16_uge:
 ; VI: v_cmp_ge_u16_e32 vcc, v{{[0-9]+}}, v{{[0-9]+}}
 ; SI: v_cmp_ge_u32_e32 vcc, v{{[0-9]+}}, v{{[0-9]+}}
+; GFX11-FAKE16: v_cmp_ge_u16_e32 vcc_lo, v{{[0-9]+}}, v{{[0-9]+}}
+; GFX11-TRUE16: v_cmp_ge_u16_e32 vcc_lo, v{{[0-9]+}}.l, v{{[0-9]+}}.h
 define amdgpu_kernel void @i16_uge(ptr addrspace(1) %out, ptr addrspace(1) %a.ptr, ptr addrspace(1) %b.ptr) #0 {
 entry:
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
@@ -80,6 +92,8 @@ entry:
 ; GCN-LABEL: {{^}}i16_ult:
 ; VI: v_cmp_lt_u16_e32 vcc, v{{[0-9]+}}, v{{[0-9]+}}
 ; SI: v_cmp_lt_u32_e32 vcc, v{{[0-9]+}}, v{{[0-9]+}}
+; GFX11-FAKE16: v_cmp_lt_u16_e32 vcc_lo, v{{[0-9]+}}, v{{[0-9]+}}
+; GFX11-TRUE16: v_cmp_lt_u16_e32 vcc_lo, v{{[0-9]+}}.l, v{{[0-9]+}}.h
 define amdgpu_kernel void @i16_ult(ptr addrspace(1) %out, ptr addrspace(1) %a.ptr, ptr addrspace(1) %b.ptr) #0 {
 entry:
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
@@ -98,6 +112,8 @@ entry:
 ; GCN-LABEL: {{^}}i16_ule:
 ; VI: v_cmp_le_u16_e32 vcc, v{{[0-9]+}}, v{{[0-9]+}}
 ; SI: v_cmp_le_u32_e32 vcc, v{{[0-9]+}}, v{{[0-9]+}}
+; GFX11-FAKE16: v_cmp_le_u16_e32 vcc_lo, v{{[0-9]+}}, v{{[0-9]+}}
+; GFX11-TRUE16: v_cmp_le_u16_e32 vcc_lo, v{{[0-9]+}}.l, v{{[0-9]+}}.h
 define amdgpu_kernel void @i16_ule(ptr addrspace(1) %out, ptr addrspace(1) %a.ptr, ptr addrspace(1) %b.ptr) #0 {
 entry:
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
@@ -117,6 +133,8 @@ entry:
 ; GCN-LABEL: {{^}}i16_sgt:
 ; VI: v_cmp_gt_i16_e32 vcc, v{{[0-9]+}}, v{{[0-9]+}}
 ; SI: v_cmp_gt_i32_e32 vcc, v{{[0-9]+}}, v{{[0-9]+}}
+; GFX11-FAKE16: v_cmp_gt_i16_e32 vcc_lo, v{{[0-9]+}}, v{{[0-9]+}}
+; GFX11-TRUE16: v_cmp_gt_i16_e32 vcc_lo, v{{[0-9]+}}.l, v{{[0-9]+}}.h
 define amdgpu_kernel void @i16_sgt(ptr addrspace(1) %out, ptr addrspace(1) %a.ptr, ptr addrspace(1) %b.ptr) #0 {
 entry:
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
@@ -135,6 +153,8 @@ entry:
 ; GCN-LABEL: {{^}}i16_sge:
 ; VI: v_cmp_ge_i16_e32 vcc, v{{[0-9]+}}, v{{[0-9]+}}
 ; SI: v_cmp_ge_i32_e32 vcc, v{{[0-9]+}}, v{{[0-9]+}}
+; GFX11-FAKE16: v_cmp_ge_i16_e32 vcc_lo, v{{[0-9]+}}, v{{[0-9]+}}
+; GFX11-TRUE16: v_cmp_ge_i16_e32 vcc_lo, v{{[0-9]+}}.l, v{{[0-9]+}}.h
 define amdgpu_kernel void @i16_sge(ptr addrspace(1) %out, ptr addrspace(1) %a.ptr, ptr addrspace(1) %b.ptr) #0 {
 entry:
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
@@ -153,6 +173,8 @@ entry:
 ; GCN-LABEL: {{^}}i16_slt:
 ; VI: v_cmp_lt_i16_e32 vcc, v{{[0-9]+}}, v{{[0-9]+}}
 ; SI: v_cmp_lt_i32_e32 vcc, v{{[0-9]+}}, v{{[0-9]+}}
+; GFX11-FAKE16: v_cmp_lt_i16_e32 vcc_lo, v{{[0-9]+}}, v{{[0-9]+}}
+; GFX11-TRUE16: v_cmp_lt_i16_e32 vcc_lo, v{{[0-9]+}}.l, v{{[0-9]+}}.h
 define amdgpu_kernel void @i16_slt(ptr addrspace(1) %out, ptr addrspace(1) %a.ptr, ptr addrspace(1) %b.ptr) #0 {
 entry:
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
@@ -171,6 +193,8 @@ entry:
 ; GCN-LABEL: {{^}}i16_sle:
 ; VI: v_cmp_le_i16_e32 vcc, v{{[0-9]+}}, v{{[0-9]+}}
 ; SI: v_cmp_le_i32_e32 vcc, v{{[0-9]+}}, v{{[0-9]+}}
+; GFX11-FAKE16: v_cmp_le_i16_e32 vcc_lo, v{{[0-9]+}}, v{{[0-9]+}}
+; GFX11-TRUE16: v_cmp_le_i16_e32 vcc_lo, v{{[0-9]+}}.l, v{{[0-9]+}}.h
 define amdgpu_kernel void @i16_sle(ptr addrspace(1) %out, ptr addrspace(1) %a.ptr, ptr addrspace(1) %b.ptr) #0 {
 entry:
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
@@ -190,6 +214,8 @@ entry:
 ; GCN-LABEL: {{^}}i16_eq_v_s:
 ; VI: v_cmp_eq_u16_e32 vcc, s{{[0-9]+}}, v{{[0-9]+}}
 ; SI: v_cmp_eq_u32_e32 vcc, s{{[0-9]+}}, v{{[0-9]+}}
+; GFX11-FAKE16: v_cmp_eq_u16_e32 vcc_lo, s{{[0-9]+}}, v{{[0-9]+}}
+; GFX11-TRUE16: v_cmp_eq_u16_e32 vcc_lo, v{{[0-9]+}}.h, v{{[0-9]+}}.l
 define amdgpu_kernel void @i16_eq_v_s(ptr addrspace(1) %out, ptr addrspace(1) %a.ptr, i16 %b) #0 {
 entry:
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
@@ -206,6 +232,8 @@ entry:
 ; GCN-LABEL: {{^}}i16_ne_v_s:
 ; VI: v_cmp_ne_u16_e32 vcc, s{{[0-9]+}}, v{{[0-9]+}}
 ; SI: v_cmp_ne_u32_e32 vcc, s{{[0-9]+}}, v{{[0-9]+}}
+; GFX11-FAKE16: v_cmp_ne_u16_e32 vcc_lo, s{{[0-9]+}}, v{{[0-9]+}}
+; GFX11-TRUE16: v_cmp_ne_u16_e32 vcc_lo, v{{[0-9]+}}.h, v{{[0-9]+}}.l
 define amdgpu_kernel void @i16_ne_v_s(ptr addrspace(1) %out, ptr addrspace(1) %a.ptr, i16 %b) #0 {
 entry:
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
@@ -222,6 +250,8 @@ entry:
 ; GCN-LABEL: {{^}}i16_ugt_v_s:
 ; VI: v_cmp_lt_u16_e32 vcc, s{{[0-9]+}}, v{{[0-9]+}}
 ; SI: v_cmp_lt_u32_e32 vcc, s{{[0-9]+}}, v{{[0-9]+}}
+; GFX11-FAKE16: v_cmp_lt_u16_e32 vcc_lo, s{{[0-9]+}}, v{{[0-9]+}}
+; GFX11-TRUE16: v_cmp_gt_u16_e32 vcc_lo, v{{[0-9]+}}.h, v{{[0-9]+}}.l
 define amdgpu_kernel void @i16_ugt_v_s(ptr addrspace(1) %out, ptr addrspace(1) %a.ptr, i16 %b) #0 {
 entry:
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
@@ -238,6 +268,8 @@ entry:
 ; GCN-LABEL: {{^}}i16_uge_v_s:
 ; VI: v_cmp_le_u16_e32 vcc, s{{[0-9]+}}, v{{[0-9]+}}
 ; SI: v_cmp_le_u32_e32 vcc, s{{[0-9]+}}, v{{[0-9]+}}
+; GFX11-FAKE16: v_cmp_le_u16_e32 vcc_lo, s{{[0-9]+}}, v{{[0-9]+}}
+; GFX11-TRUE16: v_cmp_ge_u16_e32 vcc_lo, v{{[0-9]+}}.h, v{{[0-9]+}}.l
 define amdgpu_kernel void @i16_uge_v_s(ptr addrspace(1) %out, ptr addrspace(1) %a.ptr, i16 %b) #0 {
 entry:
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
@@ -254,6 +286,8 @@ entry:
 ; GCN-LABEL: {{^}}i16_ult_v_s:
 ; VI: v_cmp_gt_u16_e32 vcc, s{{[0-9]+}}, v{{[0-9]+}}
 ; SI: v_cmp_gt_u32_e32 vcc, s{{[0-9]+}}, v{{[0-9]+}}
+; GFX11-FAKE16: v_cmp_gt_u16_e32 vcc_lo, s{{[0-9]+}}, v{{[0-9]+}}
+; GFX11-TRUE16: v_cmp_lt_u16_e32 vcc_lo, v{{[0-9]+}}.h, v{{[0-9]+}}.l
 define amdgpu_kernel void @i16_ult_v_s(ptr addrspace(1) %out, ptr addrspace(1) %a.ptr, i16 %b) #0 {
 entry:
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
@@ -270,6 +304,8 @@ entry:
 ; GCN-LABEL: {{^}}i16_ule_v_s:
 ; VI: v_cmp_ge_u16_e32 vcc, s{{[0-9]+}}, v{{[0-9]+}}
 ; SI: v_cmp_ge_u32_e32 vcc, s{{[0-9]+}}, v{{[0-9]+}}
+; GFX11-FAKE16: v_cmp_ge_u16_e32 vcc_lo, s{{[0-9]+}}, v{{[0-9]+}}
+; GFX11-TRUE16: v_cmp_le_u16_e32 vcc_lo, v{{[0-9]+}}.h, v{{[0-9]+}}.l
 define amdgpu_kernel void @i16_ule_v_s(ptr addrspace(1) %out, ptr addrspace(1) %a.ptr, i16 %b) #0 {
 entry:
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
@@ -286,6 +322,8 @@ entry:
 ; GCN-LABEL: {{^}}i16_sgt_v_s:
 ; VI: v_cmp_lt_i16_e32 vcc, s{{[0-9]+}}, v{{[0-9]+}}
 ; SI: v_cmp_lt_i32_e32 vcc, s{{[0-9]+}}, v{{[0-9]+}}
+; GFX11-FAKE16: v_cmp_lt_i16_e32 vcc_lo, s{{[0-9]+}}, v{{[0-9]+}}
+; GFX11-TRUE16: v_cmp_gt_i16_e32 vcc_lo, v{{[0-9]+}}.h, v{{[0-9]+}}.l
 define amdgpu_kernel void @i16_sgt_v_s(ptr addrspace(1) %out, ptr addrspace(1) %a.ptr, i16 %b) #0 {
 entry:
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
@@ -302,6 +340,8 @@ entry:
 ; GCN-LABEL: {{^}}i16_sge_v_s:
 ; VI: v_cmp_le_i16_e32 vcc, s{{[0-9]+}}, v{{[0-9]+}}
 ; SI: v_cmp_le_i32_e32 vcc, s{{[0-9]+}}, v{{[0-9]+}}
+; GFX11-FAKE16: v_cmp_le_i16_e32 vcc_lo, s{{[0-9]+}}, v{{[0-9]+}}
+; GFX11-TRUE16: v_cmp_ge_i16_e32 vcc_lo, v{{[0-9]+}}.h, v{{[0-9]+}}.l
 define amdgpu_kernel void @i16_sge_v_s(ptr addrspace(1) %out, ptr addrspace(1) %a.ptr, i16 %b) #0 {
 entry:
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
@@ -318,6 +358,8 @@ entry:
 ; GCN-LABEL: {{^}}i16_slt_v_s:
 ; VI: v_cmp_gt_i16_e32 vcc, s{{[0-9]+}}, v{{[0-9]+}}
 ; SI: v_cmp_gt_i32_e32 vcc, s{{[0-9]+}}, v{{[0-9]+}}
+; GFX11-FAKE16: v_cmp_gt_i16_e32 vcc_lo, s{{[0-9]+}}, v{{[0-9]+}}
+; GFX11-TRUE16: v_cmp_lt_i16_e32 vcc_lo, v{{[0-9]+}}.h, v{{[0-9]+}}.l
 define amdgpu_kernel void @i16_slt_v_s(ptr addrspace(1) %out, ptr addrspace(1) %a.ptr, i16 %b) #0 {
 entry:
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
@@ -334,6 +376,8 @@ entry:
 ; GCN-LABEL: {{^}}i16_sle_v_s:
 ; VI: v_cmp_ge_i16_e32 vcc, s{{[0-9]+}}, v{{[0-9]+}}
 ; SI: v_cmp_ge_i32_e32 vcc, s{{[0-9]+}}, v{{[0-9]+}}
+; GFX11-FAKE16: v_cmp_ge_i16_e32 vcc_lo, s{{[0-9]+}}, v{{[0-9]+}}
+; GFX11-TRUE16: v_cmp_le_i16_e32 vcc_lo, v{{[0-9]+}}.h, v{{[0-9]+}}.l
 define amdgpu_kernel void @i16_sle_v_s(ptr addrspace(1) %out, ptr addrspace(1) %a.ptr, i16 %b) #0 {
 entry:
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
diff --git a/llvm/test/CodeGen/AMDGPU/idot4u.ll b/llvm/test/CodeGen/AMDGPU/idot4u.ll
index 10fac09ef4ec0..3a97724d81fbe 100644
--- a/llvm/test/CodeGen/AMDGPU/idot4u.ll
+++ b/llvm/test/CodeGen/AMDGPU/idot4u.ll
@@ -4906,19 +4906,19 @@ entry:
 define amdgpu_kernel void @idot4_acc32_hilo(ptr addrspace(1) %src1,
 ; GFX7-LABEL: idot4_acc32_hilo:
 ; GFX7:       ; %bb.0: ; %entry
-; GFX7-NEXT:    s_load_dwordx4 s[8:11], s[4:5], 0x9
-; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0xd
-; GFX7-NEXT:    s_mov_b32 s3, 0xf000
-; GFX7-NEXT:    s_mov_b32 s6, 0
-; GFX7-NEXT:    s_mov_b32 s7, s3
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0xd
+; GFX7-NEXT:    s_mov_b32 s7, 0xf000
+; GFX7-NEXT:    s_mov_b32 s10, 0
+; GFX7-NEXT:    s_mov_b32 s11, s7
 ; GFX7-NEXT:    v_lshlrev_b32_e32 v0, 3, v0
 ; GFX7-NEXT:    v_mov_b32_e32 v1, 0
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7-NEXT:    s_mov_b64 s[4:5], s[10:11]
-; GFX7-NEXT:    s_mov_b64 s[10:11], s[6:7]
-; GFX7-NEXT:    buffer_load_dword v2, v[0:1], s[8:11], 0 addr64 offset:4
-; GFX7-NEXT:    buffer_load_dword v0, v[0:1], s[4:7], 0 addr64
-; GFX7-NEXT:    s_mov_b32 s2, -1
+; GFX7-NEXT:    s_mov_b64 s[8:9], s[2:3]
+; GFX7-NEXT:    s_mov_b64 s[2:3], s[10:11]
+; GFX7-NEXT:    buffer_load_dword v2, v[0:1], s[0:3], 0 addr64 offset:4
+; GFX7-NEXT:    buffer_load_dword v0, v[0:1], s[8:11], 0 addr64
+; GFX7-NEXT:    s_mov_b32 s6, -1
 ; GFX7-NEXT:    s_waitcnt vmcnt(1)
 ; GFX7-NEXT:    v_bfe_u32 v4, v2, 8, 8
 ; GFX7-NEXT:    s_waitcnt vmcnt(0)
@@ -4933,7 +4933,7 @@ define amdgpu_kernel void @idot4_acc32_hilo(ptr addrspace(1) %src1,
 ; GFX7-NEXT:    v_lshrrev_b32_e32 v0, 24, v0
 ; GFX7-NEXT:    v_mad_u32_u24 v1, v6, v7, v1
 ; GFX7-NEXT:    v_mad_u32_u24 v0, v2, v0, v1
-; GFX7-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; GFX7-NEXT:    buffer_store_dword v0, off, s[4:7], 0
 ; GFX7-NEXT:    s_endpgm
 ;
 ; GFX8-LABEL: idot4_acc32_hilo:
diff --git a/llvm/test/CodeGen/AMDGPU/indirect-call-known-callees.ll b/llvm/test/CodeGen/AMDGPU/indirect-call-known-callees.ll
index 3e6143866bf88..a15bf7f32dc27 100644
--- a/llvm/test/CodeGen/AMDGPU/indirect-call-known-callees.ll
+++ b/llvm/test/CodeGen/AMDGPU/indirect-call-known-callees.ll
@@ -46,19 +46,15 @@ define amdgpu_kernel void @indirect_call_known_no_special_inputs() {
 ; GFX12:       ; %bb.0: ; %bb
 ; GFX12-NEXT:    s_mov_b64 s[10:11], s[6:7]
 ; GFX12-NEXT:    s_getpc_b64 s[6:7]
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_sext_i32_i16 s7, s7
-; GFX12-NEXT:    s_add_co_u32 s6, s6, snork@gotpcrel32@lo+12
-; GFX12-NEXT:    s_wait_alu 0xfffe
-; GFX12-NEXT:    s_add_co_ci_u32 s7, s7, snork@gotpcrel32@hi+24
+; GFX12-NEXT:    s_add_co_u32 s6, s6, snork@gotpcrel32@lo+8
+; GFX12-NEXT:    s_add_co_ci_u32 s7, s7, snork@gotpcrel32@hi+16
 ; GFX12-NEXT:    s_mov_b64 s[8:9], s[4:5]
 ; GFX12-NEXT:    s_mov_b64 s[4:5], 0
 ; GFX12-NEXT:    s_getpc_b64 s[12:13]
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_sext_i32_i16 s13, s13
-; GFX12-NEXT:    s_add_co_u32 s12, s12, wobble@gotpcrel32@lo+12
-; GFX12-NEXT:    s_wait_alu 0xfffe
-; GFX12-NEXT:    s_add_co_ci_u32 s13, s13, wobble@gotpcrel32@hi+24
+; GFX12-NEXT:    s_add_co_u32 s12, s12, wobble@gotpcrel32@lo+8
+; GFX12-NEXT:    s_add_co_ci_u32 s13, s13, wobble@gotpcrel32@hi+16
 ; GFX12-NEXT:    s_load_u8 s14, s[4:5], 0x0
 ; GFX12-NEXT:    s_load_b64 s[4:5], s[6:7], 0x0
 ; GFX12-NEXT:    s_load_b64 s[6:7], s[12:13], 0x0
@@ -67,13 +63,12 @@ define amdgpu_kernel void @indirect_call_known_no_special_inputs() {
 ; GFX12-NEXT:    s_mov_b32 s32, 0
 ; GFX12-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-NEXT:    s_and_b32 s12, 1, s14
-; GFX12-NEXT:    s_wait_alu 0xfffe
+; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
 ; GFX12-NEXT:    s_cmp_eq_u32 s12, 1
 ; GFX12-NEXT:    s_cselect_b32 s13, s7, s5
 ; GFX12-NEXT:    s_cselect_b32 s12, s6, s4
 ; GFX12-NEXT:    s_mov_b64 s[4:5], s[0:1]
 ; GFX12-NEXT:    s_mov_b64 s[6:7], s[2:3]
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_swappc_b64 s[30:31], s[12:13]
 ; GFX12-NEXT:    s_endpgm
 
diff --git a/llvm/test/CodeGen/AMDGPU/inline-attr.ll b/llvm/test/CodeGen/AMDGPU/inline-attr.ll
index 22e3cc4b047b1..15f24b17219b7 100644
--- a/llvm/test/CodeGen/AMDGPU/inline-attr.ll
+++ b/llvm/test/CodeGen/AMDGPU/inline-attr.ll
@@ -3,7 +3,7 @@
 ; RUN: opt -mtriple=amdgcn--amdhsa -S -O3 -enable-no-infs-fp-math %s | FileCheck -check-prefix=GCN -check-prefix=NOINFS %s
 
 ; GCN: define float @foo(float %x) local_unnamed_addr #0 {
-; GCN: define amdgpu_kernel void @caller(ptr addrspace(1) nocapture %p) local_unnamed_addr #1 {
+; GCN: define amdgpu_kernel void @caller(ptr addrspace(1) captures(none) %p) local_unnamed_addr #1 {
 ; GCN: %mul.i = fmul float %load, 1.500000e+01
 
 ; UNSAFE: attributes #0 = { nounwind "uniform-work-group-size"="false" "unsafe-fp-math"="true" }
diff --git a/llvm/test/CodeGen/AMDGPU/insert_waitcnt_for_precise_memory.ll b/llvm/test/CodeGen/AMDGPU/insert_waitcnt_for_precise_memory.ll
index fbeda72725b2a..5256cbcef123a 100644
--- a/llvm/test/CodeGen/AMDGPU/insert_waitcnt_for_precise_memory.ll
+++ b/llvm/test/CodeGen/AMDGPU/insert_waitcnt_for_precise_memory.ll
@@ -289,7 +289,6 @@ define i32 @atomic_nand_i32_global(ptr addrspace(1) %ptr) nounwind {
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX12-NEXT:    v_mov_b32_e32 v0, v2
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
   %result = atomicrmw nand ptr addrspace(1) %ptr, i32 4 seq_cst
   ret i32 %result
@@ -591,7 +590,6 @@ define amdgpu_kernel void @udiv_i32(ptr addrspace(1) %out, i32 %x, i32 %y) {
 ; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_2) | instskip(NEXT) | instid1(TRANS32_DEP_1)
 ; GFX12-NEXT:    v_rcp_iflag_f32_e32 v0, s4
 ; GFX12-NEXT:    v_readfirstlane_b32 s4, v0
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_2)
 ; GFX12-NEXT:    s_mul_f32 s4, s4, 0x4f7ffffe
 ; GFX12-NEXT:    s_wait_alu 0xfffe
@@ -802,7 +800,6 @@ define amdgpu_kernel void @atomic_add_local(ptr addrspace(3) %local) {
 ; GFX12-NEXT:  ; %bb.1:
 ; GFX12-NEXT:    s_load_b32 s1, s[4:5], 0x24
 ; GFX12-NEXT:    s_wait_kmcnt 0x0
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_bcnt1_i32_b32 s0, s0
 ; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_mul_i32 s0, s0, 5
@@ -1050,7 +1047,6 @@ define amdgpu_kernel void @atomic_add_ret_local(ptr addrspace(1) %out, ptr addrs
 ; GFX12-NEXT:  ; %bb.1:
 ; GFX12-NEXT:    s_load_b32 s2, s[4:5], 0x2c
 ; GFX12-NEXT:    s_wait_kmcnt 0x0
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_bcnt1_i32_b32 s1, s1
 ; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_mul_i32 s1, s1, 5
@@ -1060,12 +1056,12 @@ define amdgpu_kernel void @atomic_add_ret_local(ptr addrspace(1) %out, ptr addrs
 ; GFX12-NEXT:    s_wait_dscnt 0x0
 ; GFX12-NEXT:    global_inv scope:SCOPE_SE
 ; GFX12-NEXT:  .LBB7_2:
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX12-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
 ; GFX12-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-NEXT:    v_readfirstlane_b32 s2, v1
 ; GFX12-NEXT:    v_mov_b32_e32 v1, 0
+; GFX12-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2)
 ; GFX12-NEXT:    v_mad_u32_u24 v0, v0, 5, s2
 ; GFX12-NEXT:    global_store_b32 v1, v0, s[0:1]
@@ -1236,7 +1232,6 @@ define amdgpu_kernel void @add_i32_constant(ptr addrspace(1) %out, ptr addrspace
 ; GFX12-NEXT:  ; %bb.1:
 ; GFX12-NEXT:    s_load_b128 s[8:11], s[4:5], 0x34
 ; GFX12-NEXT:    s_wait_kmcnt 0x0
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_bcnt1_i32_b32 s1, s1
 ; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_mul_i32 s1, s1, 5
@@ -1245,7 +1240,6 @@ define amdgpu_kernel void @add_i32_constant(ptr addrspace(1) %out, ptr addrspace
 ; GFX12-NEXT:    buffer_atomic_add_u32 v1, off, s[8:11], null th:TH_ATOMIC_RETURN
 ; GFX12-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-NEXT:  .LBB8_2:
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX12-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
 ; GFX12-NEXT:    s_wait_kmcnt 0x0
diff --git a/llvm/test/CodeGen/AMDGPU/integer-mad-patterns.ll b/llvm/test/CodeGen/AMDGPU/integer-mad-patterns.ll
index edf900a50cd4b..c0c0d3ded117d 100644
--- a/llvm/test/CodeGen/AMDGPU/integer-mad-patterns.ll
+++ b/llvm/test/CodeGen/AMDGPU/integer-mad-patterns.ll
@@ -5123,6 +5123,7 @@ define i64 @clpeak_imad_pat_i64(i64 %x, i64 %y) {
 ; GFX1200-SDAG-NEXT:    s_wait_bvhcnt 0x0
 ; GFX1200-SDAG-NEXT:    s_wait_kmcnt 0x0
 ; GFX1200-SDAG-NEXT:    v_add_co_u32 v4, vcc_lo, v0, 1
+; GFX1200-SDAG-NEXT:    s_wait_alu 0xfffd
 ; GFX1200-SDAG-NEXT:    v_add_co_ci_u32_e32 v5, vcc_lo, 0, v1, vcc_lo
 ; GFX1200-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
 ; GFX1200-SDAG-NEXT:    v_mul_lo_u32 v7, v4, v3
@@ -5131,9 +5132,10 @@ define i64 @clpeak_imad_pat_i64(i64 %x, i64 %y) {
 ; GFX1200-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
 ; GFX1200-SDAG-NEXT:    v_add_co_u32 v4, vcc_lo, v0, v4
 ; GFX1200-SDAG-NEXT:    v_add3_u32 v1, v1, v7, v6
-; GFX1200-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX1200-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
 ; GFX1200-SDAG-NEXT:    v_mul_lo_u32 v6, v4, v3
 ; GFX1200-SDAG-NEXT:    v_mad_co_u64_u32 v[3:4], null, v4, v2, 0
+; GFX1200-SDAG-NEXT:    s_wait_alu 0xfffd
 ; GFX1200-SDAG-NEXT:    v_add_co_ci_u32_e32 v5, vcc_lo, v1, v5, vcc_lo
 ; GFX1200-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX1200-SDAG-NEXT:    v_mul_lo_u32 v2, v5, v2
@@ -5149,6 +5151,7 @@ define i64 @clpeak_imad_pat_i64(i64 %x, i64 %y) {
 ; GFX1200-SDAG-NEXT:    v_mul_lo_u32 v4, v1, v3
 ; GFX1200-SDAG-NEXT:    v_mad_co_u64_u32 v[0:1], null, v0, v3, v[0:1]
 ; GFX1200-SDAG-NEXT:    v_add3_u32 v1, v4, v1, v2
+; GFX1200-SDAG-NEXT:    s_wait_alu 0xfffd
 ; GFX1200-SDAG-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX1200-GISEL-LABEL: clpeak_imad_pat_i64:
@@ -5159,40 +5162,41 @@ define i64 @clpeak_imad_pat_i64(i64 %x, i64 %y) {
 ; GFX1200-GISEL-NEXT:    s_wait_bvhcnt 0x0
 ; GFX1200-GISEL-NEXT:    s_wait_kmcnt 0x0
 ; GFX1200-GISEL-NEXT:    v_add_co_u32 v4, vcc_lo, v0, 1
+; GFX1200-GISEL-NEXT:    s_wait_alu 0xfffd
 ; GFX1200-GISEL-NEXT:    v_add_co_ci_u32_e32 v5, vcc_lo, 0, v1, vcc_lo
 ; GFX1200-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX1200-GISEL-NEXT:    v_mul_hi_u32 v0, v4, v2
 ; GFX1200-GISEL-NEXT:    v_mul_lo_u32 v6, v4, v2
 ; GFX1200-GISEL-NEXT:    v_mad_co_u64_u32 v[0:1], null, v4, v3, v[0:1]
-; GFX1200-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1200-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX1200-GISEL-NEXT:    v_add_co_u32 v4, vcc_lo, v6, v4
+; GFX1200-GISEL-NEXT:    v_mul_lo_u32 v7, v4, v2
+; GFX1200-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_2)
 ; GFX1200-GISEL-NEXT:    v_mad_co_u64_u32 v[0:1], null, v5, v2, v[0:1]
-; GFX1200-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX1200-GISEL-NEXT:    v_mov_b32_e32 v7, v0
-; GFX1200-GISEL-NEXT:    v_mul_hi_u32 v0, v4, v2
-; GFX1200-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1200-GISEL-NEXT:    v_add_co_ci_u32_e32 v5, vcc_lo, v7, v5, vcc_lo
-; GFX1200-GISEL-NEXT:    v_mad_co_u64_u32 v[0:1], null, v4, v3, v[0:1]
-; GFX1200-GISEL-NEXT:    v_mul_lo_u32 v3, v4, v2
+; GFX1200-GISEL-NEXT:    v_mul_hi_u32 v1, v4, v2
+; GFX1200-GISEL-NEXT:    s_wait_alu 0xfffd
+; GFX1200-GISEL-NEXT:    v_add_co_ci_u32_e32 v5, vcc_lo, v0, v5, vcc_lo
+; GFX1200-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_4) | instid1(VALU_DEP_3)
+; GFX1200-GISEL-NEXT:    v_mad_co_u64_u32 v[3:4], null, v4, v3, v[1:2]
 ; GFX1200-GISEL-NEXT:    v_add_co_u32 v4, vcc_lo, v6, 1
-; GFX1200-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1200-GISEL-NEXT:    v_mad_co_u64_u32 v[0:1], null, v5, v2, v[0:1]
-; GFX1200-GISEL-NEXT:    v_mul_hi_u32 v1, v3, v4
-; GFX1200-GISEL-NEXT:    v_mul_lo_u32 v5, v3, v4
-; GFX1200-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX1200-GISEL-NEXT:    v_mov_b32_e32 v6, v0
-; GFX1200-GISEL-NEXT:    v_add_co_ci_u32_e32 v2, vcc_lo, 0, v7, vcc_lo
-; GFX1200-GISEL-NEXT:    v_add_co_u32 v7, vcc_lo, v3, 1
-; GFX1200-GISEL-NEXT:    v_add_co_ci_u32_e32 v6, vcc_lo, 0, v6, vcc_lo
-; GFX1200-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX1200-GISEL-NEXT:    v_mad_co_u64_u32 v[1:2], null, v3, v2, v[1:2]
-; GFX1200-GISEL-NEXT:    v_mul_hi_u32 v2, v5, v7
-; GFX1200-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX1200-GISEL-NEXT:    v_mad_co_u64_u32 v[3:4], null, v0, v4, v[1:2]
-; GFX1200-GISEL-NEXT:    v_mul_lo_u32 v0, v5, v7
-; GFX1200-GISEL-NEXT:    v_mad_co_u64_u32 v[1:2], null, v5, v6, v[2:3]
-; GFX1200-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX1200-GISEL-NEXT:    v_mad_co_u64_u32 v[1:2], null, v3, v7, v[1:2]
+; GFX1200-GISEL-NEXT:    s_wait_alu 0xfffd
+; GFX1200-GISEL-NEXT:    v_add_co_ci_u32_e32 v6, vcc_lo, 0, v0, vcc_lo
+; GFX1200-GISEL-NEXT:    v_add_co_u32 v8, vcc_lo, v7, 1
+; GFX1200-GISEL-NEXT:    v_mul_hi_u32 v0, v7, v4
+; GFX1200-GISEL-NEXT:    v_mad_co_u64_u32 v[1:2], null, v5, v2, v[3:4]
+; GFX1200-GISEL-NEXT:    v_mul_lo_u32 v5, v7, v4
+; GFX1200-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX1200-GISEL-NEXT:    v_mad_co_u64_u32 v[2:3], null, v7, v6, v[0:1]
+; GFX1200-GISEL-NEXT:    s_wait_alu 0xfffd
+; GFX1200-GISEL-NEXT:    v_add_co_ci_u32_e32 v3, vcc_lo, 0, v1, vcc_lo
+; GFX1200-GISEL-NEXT:    v_mul_hi_u32 v0, v5, v8
+; GFX1200-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1200-GISEL-NEXT:    v_mad_co_u64_u32 v[1:2], null, v1, v4, v[2:3]
+; GFX1200-GISEL-NEXT:    v_mad_co_u64_u32 v[2:3], null, v5, v3, v[0:1]
+; GFX1200-GISEL-NEXT:    v_mul_lo_u32 v0, v5, v8
+; GFX1200-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_2)
+; GFX1200-GISEL-NEXT:    v_mad_co_u64_u32 v[1:2], null, v1, v8, v[2:3]
+; GFX1200-GISEL-NEXT:    s_wait_alu 0xfffd
 ; GFX1200-GISEL-NEXT:    s_setpc_b64 s[30:31]
 entry:
   %y18 = add i64 %x, 1
@@ -5969,8 +5973,10 @@ define <2 x i64> @clpeak_imad_pat_v2i64(<2 x i64> %x, <2 x i64> %y) {
 ; GFX1200-SDAG-NEXT:    s_wait_bvhcnt 0x0
 ; GFX1200-SDAG-NEXT:    s_wait_kmcnt 0x0
 ; GFX1200-SDAG-NEXT:    v_add_co_u32 v8, vcc_lo, v0, 1
+; GFX1200-SDAG-NEXT:    s_wait_alu 0xfffd
 ; GFX1200-SDAG-NEXT:    v_add_co_ci_u32_e32 v9, vcc_lo, 0, v1, vcc_lo
 ; GFX1200-SDAG-NEXT:    v_add_co_u32 v10, vcc_lo, v2, 1
+; GFX1200-SDAG-NEXT:    s_wait_alu 0xfffd
 ; GFX1200-SDAG-NEXT:    v_add_co_ci_u32_e32 v11, vcc_lo, 0, v3, vcc_lo
 ; GFX1200-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_4)
 ; GFX1200-SDAG-NEXT:    v_mul_lo_u32 v12, v9, v4
@@ -5983,11 +5989,12 @@ define <2 x i64> @clpeak_imad_pat_v2i64(<2 x i64> %x, <2 x i64> %y) {
 ; GFX1200-SDAG-NEXT:    v_add3_u32 v12, v1, v13, v12
 ; GFX1200-SDAG-NEXT:    v_add_co_u32 v1, vcc_lo, v0, v8
 ; GFX1200-SDAG-NEXT:    v_add3_u32 v13, v3, v15, v14
-; GFX1200-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX1200-SDAG-NEXT:    s_wait_alu 0xfffd
+; GFX1200-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_3)
 ; GFX1200-SDAG-NEXT:    v_add_co_ci_u32_e32 v3, vcc_lo, v12, v9, vcc_lo
 ; GFX1200-SDAG-NEXT:    v_add_co_u32 v8, vcc_lo, v2, v10
+; GFX1200-SDAG-NEXT:    s_wait_alu 0xfffd
 ; GFX1200-SDAG-NEXT:    v_add_co_ci_u32_e32 v9, vcc_lo, v13, v11, vcc_lo
-; GFX1200-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
 ; GFX1200-SDAG-NEXT:    v_mul_lo_u32 v10, v3, v4
 ; GFX1200-SDAG-NEXT:    v_mul_lo_u32 v11, v1, v5
 ; GFX1200-SDAG-NEXT:    v_mul_lo_u32 v7, v8, v7
@@ -6019,6 +6026,7 @@ define <2 x i64> @clpeak_imad_pat_v2i64(<2 x i64> %x, <2 x i64> %y) {
 ; GFX1200-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
 ; GFX1200-SDAG-NEXT:    v_add3_u32 v1, v9, v1, v4
 ; GFX1200-SDAG-NEXT:    v_add3_u32 v3, v10, v3, v6
+; GFX1200-SDAG-NEXT:    s_wait_alu 0xfffd
 ; GFX1200-SDAG-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX1200-GISEL-LABEL: clpeak_imad_pat_v2i64:
@@ -6029,70 +6037,73 @@ define <2 x i64> @clpeak_imad_pat_v2i64(<2 x i64> %x, <2 x i64> %y) {
 ; GFX1200-GISEL-NEXT:    s_wait_bvhcnt 0x0
 ; GFX1200-GISEL-NEXT:    s_wait_kmcnt 0x0
 ; GFX1200-GISEL-NEXT:    v_add_co_u32 v8, vcc_lo, v0, 1
-; GFX1200-GISEL-NEXT:    v_add_co_ci_u32_e32 v9, vcc_lo, 0, v1, vcc_lo
-; GFX1200-GISEL-NEXT:    v_add_co_u32 v10, vcc_lo, v2, 1
-; GFX1200-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX1200-GISEL-NEXT:    v_add_co_u32 v9, s0, v2, 1
+; GFX1200-GISEL-NEXT:    s_wait_alu 0xfffd
+; GFX1200-GISEL-NEXT:    v_add_co_ci_u32_e32 v10, vcc_lo, 0, v1, vcc_lo
+; GFX1200-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
 ; GFX1200-GISEL-NEXT:    v_mul_hi_u32 v0, v8, v4
-; GFX1200-GISEL-NEXT:    v_add_co_ci_u32_e32 v11, vcc_lo, 0, v3, vcc_lo
-; GFX1200-GISEL-NEXT:    v_mul_hi_u32 v1, v10, v6
+; GFX1200-GISEL-NEXT:    v_mul_hi_u32 v1, v9, v6
+; GFX1200-GISEL-NEXT:    s_wait_alu 0xf1ff
+; GFX1200-GISEL-NEXT:    v_add_co_ci_u32_e64 v11, vcc_lo, 0, v3, s0
 ; GFX1200-GISEL-NEXT:    v_mul_lo_u32 v12, v8, v4
-; GFX1200-GISEL-NEXT:    v_mul_lo_u32 v13, v10, v6
-; GFX1200-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1200-GISEL-NEXT:    v_mul_lo_u32 v13, v9, v6
+; GFX1200-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
 ; GFX1200-GISEL-NEXT:    v_mad_co_u64_u32 v[2:3], null, v8, v5, v[0:1]
-; GFX1200-GISEL-NEXT:    v_mad_co_u64_u32 v[0:1], null, v10, v7, v[1:2]
-; GFX1200-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1200-GISEL-NEXT:    v_mad_co_u64_u32 v[1:2], null, v9, v4, v[2:3]
-; GFX1200-GISEL-NEXT:    v_mov_b32_e32 v14, v1
-; GFX1200-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX1200-GISEL-NEXT:    v_add_co_u32 v14, vcc_lo, v12, v8
+; GFX1200-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX1200-GISEL-NEXT:    v_add_co_u32 v15, s0, v13, v9
+; GFX1200-GISEL-NEXT:    v_mad_co_u64_u32 v[0:1], null, v9, v7, v[1:2]
+; GFX1200-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1200-GISEL-NEXT:    v_mad_co_u64_u32 v[1:2], null, v10, v4, v[2:3]
 ; GFX1200-GISEL-NEXT:    v_mad_co_u64_u32 v[2:3], null, v11, v6, v[0:1]
-; GFX1200-GISEL-NEXT:    v_add_co_u32 v3, vcc_lo, v12, v8
-; GFX1200-GISEL-NEXT:    v_add_co_ci_u32_e32 v9, vcc_lo, v14, v9, vcc_lo
-; GFX1200-GISEL-NEXT:    v_add_co_u32 v10, vcc_lo, v13, v10
-; GFX1200-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_4)
-; GFX1200-GISEL-NEXT:    v_mul_hi_u32 v0, v3, v4
-; GFX1200-GISEL-NEXT:    v_mov_b32_e32 v8, v2
-; GFX1200-GISEL-NEXT:    v_mul_lo_u32 v15, v3, v4
-; GFX1200-GISEL-NEXT:    v_mul_hi_u32 v1, v10, v6
-; GFX1200-GISEL-NEXT:    v_mul_lo_u32 v16, v10, v6
-; GFX1200-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX1200-GISEL-NEXT:    v_add_co_ci_u32_e32 v11, vcc_lo, v8, v11, vcc_lo
-; GFX1200-GISEL-NEXT:    v_add_co_u32 v12, vcc_lo, v12, 1
-; GFX1200-GISEL-NEXT:    v_mad_co_u64_u32 v[2:3], null, v3, v5, v[0:1]
-; GFX1200-GISEL-NEXT:    v_add_co_ci_u32_e32 v5, vcc_lo, 0, v14, vcc_lo
-; GFX1200-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX1200-GISEL-NEXT:    v_mad_co_u64_u32 v[0:1], null, v10, v7, v[1:2]
-; GFX1200-GISEL-NEXT:    v_mul_hi_u32 v1, v15, v12
-; GFX1200-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX1200-GISEL-NEXT:    v_mad_co_u64_u32 v[2:3], null, v9, v4, v[2:3]
+; GFX1200-GISEL-NEXT:    v_mul_hi_u32 v0, v14, v4
+; GFX1200-GISEL-NEXT:    v_mul_hi_u32 v3, v15, v6
+; GFX1200-GISEL-NEXT:    s_wait_alu 0xfffd
+; GFX1200-GISEL-NEXT:    v_add_co_ci_u32_e32 v16, vcc_lo, v1, v10, vcc_lo
+; GFX1200-GISEL-NEXT:    s_wait_alu 0xf1ff
+; GFX1200-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX1200-GISEL-NEXT:    v_add_co_ci_u32_e64 v11, vcc_lo, v2, v11, s0
+; GFX1200-GISEL-NEXT:    v_mad_co_u64_u32 v[8:9], null, v14, v5, v[0:1]
+; GFX1200-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX1200-GISEL-NEXT:    v_mad_co_u64_u32 v[9:10], null, v15, v7, v[3:4]
+; GFX1200-GISEL-NEXT:    v_mul_lo_u32 v10, v15, v6
+; GFX1200-GISEL-NEXT:    v_mul_lo_u32 v7, v14, v4
+; GFX1200-GISEL-NEXT:    v_mad_co_u64_u32 v[3:4], null, v16, v4, v[8:9]
+; GFX1200-GISEL-NEXT:    v_add_co_u32 v8, vcc_lo, v12, 1
+; GFX1200-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_4)
+; GFX1200-GISEL-NEXT:    v_mad_co_u64_u32 v[4:5], null, v11, v6, v[9:10]
+; GFX1200-GISEL-NEXT:    s_wait_alu 0xfffd
+; GFX1200-GISEL-NEXT:    v_add_co_ci_u32_e32 v5, vcc_lo, 0, v1, vcc_lo
 ; GFX1200-GISEL-NEXT:    v_add_co_u32 v9, vcc_lo, v13, 1
-; GFX1200-GISEL-NEXT:    v_mad_co_u64_u32 v[3:4], null, v11, v6, v[0:1]
-; GFX1200-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX1200-GISEL-NEXT:    v_mad_co_u64_u32 v[4:5], null, v15, v5, v[1:2]
-; GFX1200-GISEL-NEXT:    v_mov_b32_e32 v1, v2
-; GFX1200-GISEL-NEXT:    v_mul_hi_u32 v0, v16, v9
-; GFX1200-GISEL-NEXT:    v_mul_lo_u32 v14, v16, v9
-; GFX1200-GISEL-NEXT:    v_mov_b32_e32 v11, v3
-; GFX1200-GISEL-NEXT:    v_add_co_ci_u32_e32 v7, vcc_lo, 0, v8, vcc_lo
-; GFX1200-GISEL-NEXT:    v_mul_lo_u32 v8, v15, v12
-; GFX1200-GISEL-NEXT:    v_add_co_u32 v10, vcc_lo, v15, 1
-; GFX1200-GISEL-NEXT:    v_add_co_ci_u32_e32 v13, vcc_lo, 0, v1, vcc_lo
-; GFX1200-GISEL-NEXT:    v_add_co_u32 v15, vcc_lo, v16, 1
-; GFX1200-GISEL-NEXT:    v_add_co_ci_u32_e32 v11, vcc_lo, 0, v11, vcc_lo
-; GFX1200-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1200-GISEL-NEXT:    v_mul_hi_u32 v1, v8, v10
-; GFX1200-GISEL-NEXT:    v_mad_co_u64_u32 v[5:6], null, v16, v7, v[0:1]
-; GFX1200-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1200-GISEL-NEXT:    v_mul_hi_u32 v0, v14, v15
-; GFX1200-GISEL-NEXT:    v_mad_co_u64_u32 v[6:7], null, v2, v12, v[4:5]
-; GFX1200-GISEL-NEXT:    v_mad_co_u64_u32 v[1:2], null, v8, v13, v[1:2]
-; GFX1200-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1200-GISEL-NEXT:    v_mad_co_u64_u32 v[3:4], null, v3, v9, v[5:6]
-; GFX1200-GISEL-NEXT:    v_mad_co_u64_u32 v[4:5], null, v14, v11, v[0:1]
-; GFX1200-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_4)
-; GFX1200-GISEL-NEXT:    v_mad_co_u64_u32 v[1:2], null, v6, v10, v[1:2]
-; GFX1200-GISEL-NEXT:    v_mul_lo_u32 v0, v8, v10
+; GFX1200-GISEL-NEXT:    s_wait_alu 0xfffd
+; GFX1200-GISEL-NEXT:    v_add_co_ci_u32_e32 v2, vcc_lo, 0, v2, vcc_lo
+; GFX1200-GISEL-NEXT:    v_mul_hi_u32 v0, v7, v8
+; GFX1200-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_3)
+; GFX1200-GISEL-NEXT:    v_mul_hi_u32 v1, v10, v9
+; GFX1200-GISEL-NEXT:    v_add_co_u32 v12, vcc_lo, v7, 1
+; GFX1200-GISEL-NEXT:    v_mul_lo_u32 v14, v10, v9
+; GFX1200-GISEL-NEXT:    s_wait_alu 0xfffd
+; GFX1200-GISEL-NEXT:    v_add_co_ci_u32_e32 v13, vcc_lo, 0, v3, vcc_lo
+; GFX1200-GISEL-NEXT:    v_add_co_u32 v15, vcc_lo, v10, 1
+; GFX1200-GISEL-NEXT:    v_mul_lo_u32 v11, v7, v8
+; GFX1200-GISEL-NEXT:    v_mad_co_u64_u32 v[5:6], null, v7, v5, v[0:1]
+; GFX1200-GISEL-NEXT:    v_mad_co_u64_u32 v[1:2], null, v10, v2, v[1:2]
+; GFX1200-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_4) | instid1(VALU_DEP_4)
+; GFX1200-GISEL-NEXT:    v_mul_hi_u32 v2, v14, v15
+; GFX1200-GISEL-NEXT:    s_wait_alu 0xfffd
+; GFX1200-GISEL-NEXT:    v_add_co_ci_u32_e32 v10, vcc_lo, 0, v4, vcc_lo
+; GFX1200-GISEL-NEXT:    v_mul_hi_u32 v0, v11, v12
+; GFX1200-GISEL-NEXT:    v_mad_co_u64_u32 v[5:6], null, v3, v8, v[5:6]
+; GFX1200-GISEL-NEXT:    v_mad_co_u64_u32 v[3:4], null, v4, v9, v[1:2]
+; GFX1200-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX1200-GISEL-NEXT:    v_mad_co_u64_u32 v[6:7], null, v11, v13, v[0:1]
+; GFX1200-GISEL-NEXT:    v_mul_lo_u32 v0, v11, v12
+; GFX1200-GISEL-NEXT:    v_mad_co_u64_u32 v[7:8], null, v14, v10, v[2:3]
+; GFX1200-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX1200-GISEL-NEXT:    v_mad_co_u64_u32 v[1:2], null, v5, v12, v[6:7]
 ; GFX1200-GISEL-NEXT:    v_mul_lo_u32 v2, v14, v15
-; GFX1200-GISEL-NEXT:    v_mad_co_u64_u32 v[3:4], null, v3, v15, v[4:5]
+; GFX1200-GISEL-NEXT:    v_mad_co_u64_u32 v[3:4], null, v3, v15, v[7:8]
+; GFX1200-GISEL-NEXT:    s_wait_alu 0xfffd
 ; GFX1200-GISEL-NEXT:    s_setpc_b64 s[30:31]
 entry:
   %y18 = add <2 x i64> %x, <i64 1, i64 1>
@@ -9354,7 +9365,9 @@ define i64 @mul_u24_zext_add64(i32 %x, i32 %y, i64 %z) {
 ; GFX1200-NEXT:    v_mul_u32_u24_e32 v0, v0, v1
 ; GFX1200-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX1200-NEXT:    v_add_co_u32 v0, vcc_lo, v0, v2
+; GFX1200-NEXT:    s_wait_alu 0xfffd
 ; GFX1200-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, 0, v3, vcc_lo
+; GFX1200-NEXT:    s_wait_alu 0xfffd
 ; GFX1200-NEXT:    s_setpc_b64 s[30:31]
   %mul = call i32 @llvm.amdgcn.mul.u24(i32 %x, i32 %y)
   %mul.zext = zext i32 %mul to i64
diff --git a/llvm/test/CodeGen/AMDGPU/isel-amdgpu-cs-chain-cc.ll b/llvm/test/CodeGen/AMDGPU/isel-amdgpu-cs-chain-cc.ll
index acd48a64dea1f..befe0d405307b 100644
--- a/llvm/test/CodeGen/AMDGPU/isel-amdgpu-cs-chain-cc.ll
+++ b/llvm/test/CodeGen/AMDGPU/isel-amdgpu-cs-chain-cc.ll
@@ -1,8 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 2
-; RUN: llc -global-isel=1 -mtriple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize32 -stop-after=finalize-isel -verify-machineinstrs < %s | FileCheck -check-prefix=GISEL-GFX11 %s
-; RUN: llc -global-isel=1 -mtriple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize64 -stop-after=finalize-isel -verify-machineinstrs < %s | FileCheck -check-prefix=GISEL-GFX11 %s
-; RUN: llc -global-isel=1 -mtriple=amdgcn -mcpu=gfx1030 -mattr=+wavefrontsize32 -stop-after=finalize-isel -verify-machineinstrs < %s | FileCheck -check-prefix=GISEL-GFX10 %s
-; RUN: llc -global-isel=1 -mtriple=amdgcn -mcpu=gfx1030 -mattr=+wavefrontsize64 -stop-after=finalize-isel -verify-machineinstrs < %s | FileCheck -check-prefix=GISEL-GFX10 %s
+; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize32 -stop-after=finalize-isel -verify-machineinstrs < %s | FileCheck -check-prefix=GISEL-GFX11 %s
+; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize64 -stop-after=finalize-isel -verify-machineinstrs < %s | FileCheck -check-prefix=GISEL-GFX11 %s
+; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgcn -mcpu=gfx1030 -mattr=+wavefrontsize32 -stop-after=finalize-isel -verify-machineinstrs < %s | FileCheck -check-prefix=GISEL-GFX10 %s
+; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgcn -mcpu=gfx1030 -mattr=+wavefrontsize64 -stop-after=finalize-isel -verify-machineinstrs < %s | FileCheck -check-prefix=GISEL-GFX10 %s
 ; RUN: llc -global-isel=0 -mtriple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize32 -stop-after=finalize-isel -verify-machineinstrs < %s | FileCheck -check-prefix=DAGISEL-GFX11 %s
 ; RUN: llc -global-isel=0 -mtriple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize64 -stop-after=finalize-isel -verify-machineinstrs < %s | FileCheck -check-prefix=DAGISEL-GFX11 %s
 ; RUN: llc -global-isel=0 -mtriple=amdgcn -mcpu=gfx1030 -mattr=+wavefrontsize32 -stop-after=finalize-isel -verify-machineinstrs < %s | FileCheck -check-prefix=DAGISEL-GFX10 %s
@@ -594,35 +594,35 @@ define amdgpu_cs_chain void @amdgpu_cs_chain_cc_half(half inreg %a, half %b) {
 
 define amdgpu_cs_chain void @amdgpu_cs_chain_cc_bfloat(bfloat inreg %a, bfloat %b) {
   ; GISEL-GFX11-LABEL: name: amdgpu_cs_chain_cc_bfloat
-  ; GISEL-GFX11: bb.1 (%ir-block.0):
+  ; GISEL-GFX11: bb.0 (%ir-block.0):
   ; GISEL-GFX11-NEXT:   liveins: $sgpr0, $vgpr8
   ; GISEL-GFX11-NEXT: {{  $}}
-  ; GISEL-GFX11-NEXT:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
-  ; GISEL-GFX11-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr8
-  ; GISEL-GFX11-NEXT:   ADJCALLSTACKUP 0, 0, implicit-def $scc, implicit-def $sgpr32, implicit $sgpr32
-  ; GISEL-GFX11-NEXT:   $vgpr0 = COPY [[COPY]]
-  ; GISEL-GFX11-NEXT:   $vgpr1 = COPY [[COPY1]]
-  ; GISEL-GFX11-NEXT:   [[SI_PC_ADD_REL_OFFSET:%[0-9]+]]:sreg_64 = SI_PC_ADD_REL_OFFSET target-flags(amdgpu-gotprel32-lo) @use, target-flags(amdgpu-gotprel32-hi) @use, implicit-def $scc
-  ; GISEL-GFX11-NEXT:   [[S_LOAD_DWORDX2_IMM:%[0-9]+]]:sreg_64_xexec = S_LOAD_DWORDX2_IMM [[SI_PC_ADD_REL_OFFSET]], 0, 0 :: (dereferenceable invariant load (p0) from got, addrspace 4)
-  ; GISEL-GFX11-NEXT:   $sgpr30_sgpr31 = noconvergent SI_CALL [[S_LOAD_DWORDX2_IMM]], @use, csr_amdgpu_si_gfx, implicit $vgpr0, implicit $vgpr1
-  ; GISEL-GFX11-NEXT:   ADJCALLSTACKDOWN 0, 0, implicit-def $scc, implicit-def $sgpr32, implicit $sgpr32
+  ; GISEL-GFX11-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr8
+  ; GISEL-GFX11-NEXT:   [[COPY1:%[0-9]+]]:sgpr_32 = COPY $sgpr0
+  ; GISEL-GFX11-NEXT:   ADJCALLSTACKUP 0, 0, implicit-def dead $scc, implicit-def $sgpr32, implicit $sgpr32
+  ; GISEL-GFX11-NEXT:   [[SI_PC_ADD_REL_OFFSET:%[0-9]+]]:sreg_64 = SI_PC_ADD_REL_OFFSET target-flags(amdgpu-gotprel32-lo) @use, target-flags(amdgpu-gotprel32-hi) @use, implicit-def dead $scc
+  ; GISEL-GFX11-NEXT:   [[S_LOAD_DWORDX2_IMM:%[0-9]+]]:sreg_64_xexec = S_LOAD_DWORDX2_IMM killed [[SI_PC_ADD_REL_OFFSET]], 0, 0 :: (dereferenceable invariant load (s64) from got, addrspace 4)
+  ; GISEL-GFX11-NEXT:   $vgpr0 = COPY [[COPY1]]
+  ; GISEL-GFX11-NEXT:   $vgpr1 = COPY [[COPY]]
+  ; GISEL-GFX11-NEXT:   $sgpr30_sgpr31 = SI_CALL killed [[S_LOAD_DWORDX2_IMM]], @use, csr_amdgpu_si_gfx, implicit $vgpr0, implicit $vgpr1
+  ; GISEL-GFX11-NEXT:   ADJCALLSTACKDOWN 0, 0, implicit-def dead $scc, implicit-def $sgpr32, implicit $sgpr32
   ; GISEL-GFX11-NEXT:   S_ENDPGM 0
   ;
   ; GISEL-GFX10-LABEL: name: amdgpu_cs_chain_cc_bfloat
-  ; GISEL-GFX10: bb.1 (%ir-block.0):
+  ; GISEL-GFX10: bb.0 (%ir-block.0):
   ; GISEL-GFX10-NEXT:   liveins: $sgpr0, $vgpr8
   ; GISEL-GFX10-NEXT: {{  $}}
-  ; GISEL-GFX10-NEXT:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
-  ; GISEL-GFX10-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr8
-  ; GISEL-GFX10-NEXT:   ADJCALLSTACKUP 0, 0, implicit-def $scc, implicit-def $sgpr32, implicit $sgpr32
-  ; GISEL-GFX10-NEXT:   $vgpr0 = COPY [[COPY]]
-  ; GISEL-GFX10-NEXT:   $vgpr1 = COPY [[COPY1]]
+  ; GISEL-GFX10-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr8
+  ; GISEL-GFX10-NEXT:   [[COPY1:%[0-9]+]]:sgpr_32 = COPY $sgpr0
+  ; GISEL-GFX10-NEXT:   ADJCALLSTACKUP 0, 0, implicit-def dead $scc, implicit-def $sgpr32, implicit $sgpr32
+  ; GISEL-GFX10-NEXT:   [[SI_PC_ADD_REL_OFFSET:%[0-9]+]]:sreg_64 = SI_PC_ADD_REL_OFFSET target-flags(amdgpu-gotprel32-lo) @use, target-flags(amdgpu-gotprel32-hi) @use, implicit-def dead $scc
+  ; GISEL-GFX10-NEXT:   [[S_LOAD_DWORDX2_IMM:%[0-9]+]]:sreg_64_xexec = S_LOAD_DWORDX2_IMM killed [[SI_PC_ADD_REL_OFFSET]], 0, 0 :: (dereferenceable invariant load (s64) from got, addrspace 4)
   ; GISEL-GFX10-NEXT:   [[COPY2:%[0-9]+]]:sgpr_128 = COPY $sgpr48_sgpr49_sgpr50_sgpr51
   ; GISEL-GFX10-NEXT:   $sgpr0_sgpr1_sgpr2_sgpr3 = COPY [[COPY2]]
-  ; GISEL-GFX10-NEXT:   [[SI_PC_ADD_REL_OFFSET:%[0-9]+]]:sreg_64 = SI_PC_ADD_REL_OFFSET target-flags(amdgpu-gotprel32-lo) @use, target-flags(amdgpu-gotprel32-hi) @use, implicit-def $scc
-  ; GISEL-GFX10-NEXT:   [[S_LOAD_DWORDX2_IMM:%[0-9]+]]:sreg_64_xexec = S_LOAD_DWORDX2_IMM [[SI_PC_ADD_REL_OFFSET]], 0, 0 :: (dereferenceable invariant load (p0) from got, addrspace 4)
-  ; GISEL-GFX10-NEXT:   $sgpr30_sgpr31 = noconvergent SI_CALL [[S_LOAD_DWORDX2_IMM]], @use, csr_amdgpu_si_gfx, implicit $vgpr0, implicit $vgpr1, implicit $sgpr0_sgpr1_sgpr2_sgpr3
-  ; GISEL-GFX10-NEXT:   ADJCALLSTACKDOWN 0, 0, implicit-def $scc, implicit-def $sgpr32, implicit $sgpr32
+  ; GISEL-GFX10-NEXT:   $vgpr0 = COPY [[COPY1]]
+  ; GISEL-GFX10-NEXT:   $vgpr1 = COPY [[COPY]]
+  ; GISEL-GFX10-NEXT:   $sgpr30_sgpr31 = SI_CALL killed [[S_LOAD_DWORDX2_IMM]], @use, csr_amdgpu_si_gfx, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $vgpr0, implicit $vgpr1
+  ; GISEL-GFX10-NEXT:   ADJCALLSTACKDOWN 0, 0, implicit-def dead $scc, implicit-def $sgpr32, implicit $sgpr32
   ; GISEL-GFX10-NEXT:   S_ENDPGM 0
   ;
   ; DAGISEL-GFX11-LABEL: name: amdgpu_cs_chain_cc_bfloat
diff --git a/llvm/test/CodeGen/AMDGPU/isel-amdgpu-cs-chain-preserve-cc.ll b/llvm/test/CodeGen/AMDGPU/isel-amdgpu-cs-chain-preserve-cc.ll
index 2e2a1094ba99a..ef91f36d60373 100644
--- a/llvm/test/CodeGen/AMDGPU/isel-amdgpu-cs-chain-preserve-cc.ll
+++ b/llvm/test/CodeGen/AMDGPU/isel-amdgpu-cs-chain-preserve-cc.ll
@@ -1,8 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 2
-; RUN: llc -global-isel=1 -mtriple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize32 -stop-after=finalize-isel -verify-machineinstrs < %s | FileCheck -check-prefix=GISEL-GFX11 %s
-; RUN: llc -global-isel=1 -mtriple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize64 -stop-after=finalize-isel -verify-machineinstrs < %s | FileCheck -check-prefix=GISEL-GFX11 %s
-; RUN: llc -global-isel=1 -mtriple=amdgcn -mcpu=gfx1030 -mattr=+wavefrontsize32 -stop-after=finalize-isel -verify-machineinstrs < %s | FileCheck -check-prefix=GISEL-GFX10 %s
-; RUN: llc -global-isel=1 -mtriple=amdgcn -mcpu=gfx1030 -mattr=+wavefrontsize64 -stop-after=finalize-isel -verify-machineinstrs < %s | FileCheck -check-prefix=GISEL-GFX10 %s
+; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize32 -stop-after=finalize-isel -verify-machineinstrs < %s | FileCheck -check-prefix=GISEL-GFX11 %s
+; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize64 -stop-after=finalize-isel -verify-machineinstrs < %s | FileCheck -check-prefix=GISEL-GFX11 %s
+; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgcn -mcpu=gfx1030 -mattr=+wavefrontsize32 -stop-after=finalize-isel -verify-machineinstrs < %s | FileCheck -check-prefix=GISEL-GFX10 %s
+; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgcn -mcpu=gfx1030 -mattr=+wavefrontsize64 -stop-after=finalize-isel -verify-machineinstrs < %s | FileCheck -check-prefix=GISEL-GFX10 %s
 ; RUN: llc -global-isel=0 -mtriple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize32 -stop-after=finalize-isel -verify-machineinstrs < %s | FileCheck -check-prefix=DAGISEL-GFX11-WF32 %s
 ; RUN: llc -global-isel=0 -mtriple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize64 -stop-after=finalize-isel -verify-machineinstrs < %s | FileCheck -check-prefix=DAGISEL-GFX11-WF64 %s
 ; RUN: llc -global-isel=0 -mtriple=amdgcn -mcpu=gfx1030 -mattr=+wavefrontsize32 -stop-after=finalize-isel -verify-machineinstrs < %s | FileCheck -check-prefix=DAGISEL-GFX10-WF32 %s
@@ -873,32 +873,6 @@ define amdgpu_cs_chain_preserve void @amdgpu_cs_chain_preserve_cc_half(half inre
 }
 
 define amdgpu_cs_chain_preserve void @amdgpu_cs_chain_cc_bfloat(bfloat inreg %a, bfloat %b) {
-  ; GISEL-GFX11-LABEL: name: amdgpu_cs_chain_cc_bfloat
-  ; GISEL-GFX11: bb.1 (%ir-block.0):
-  ; GISEL-GFX11-NEXT:   liveins: $sgpr0, $vgpr8
-  ; GISEL-GFX11-NEXT: {{  $}}
-  ; GISEL-GFX11-NEXT:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
-  ; GISEL-GFX11-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr8
-  ; GISEL-GFX11-NEXT:   [[DEF:%[0-9]+]]:sreg_64 = IMPLICIT_DEF
-  ; GISEL-GFX11-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY [[COPY]]
-  ; GISEL-GFX11-NEXT:   [[V_ADD_F16_fake16_e64_:%[0-9]+]]:vgpr_32 = nofpexcept V_ADD_F16_fake16_e64 0, [[COPY2]], 0, [[COPY1]], 0, 0, implicit $mode, implicit $exec
-  ; GISEL-GFX11-NEXT:   [[COPY3:%[0-9]+]]:vreg_64 = COPY [[DEF]]
-  ; GISEL-GFX11-NEXT:   FLAT_STORE_SHORT [[COPY3]], [[V_ADD_F16_fake16_e64_]], 0, 0, implicit $exec, implicit $flat_scr :: (store (s16) into `ptr poison`)
-  ; GISEL-GFX11-NEXT:   S_ENDPGM 0
-  ;
-  ; GISEL-GFX10-LABEL: name: amdgpu_cs_chain_cc_bfloat
-  ; GISEL-GFX10: bb.1 (%ir-block.0):
-  ; GISEL-GFX10-NEXT:   liveins: $sgpr0, $vgpr8
-  ; GISEL-GFX10-NEXT: {{  $}}
-  ; GISEL-GFX10-NEXT:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
-  ; GISEL-GFX10-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr8
-  ; GISEL-GFX10-NEXT:   [[DEF:%[0-9]+]]:sreg_64 = IMPLICIT_DEF
-  ; GISEL-GFX10-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY [[COPY]]
-  ; GISEL-GFX10-NEXT:   [[V_ADD_F16_e64_:%[0-9]+]]:vgpr_32 = nofpexcept V_ADD_F16_e64 0, [[COPY2]], 0, [[COPY1]], 0, 0, implicit $mode, implicit $exec
-  ; GISEL-GFX10-NEXT:   [[COPY3:%[0-9]+]]:vreg_64 = COPY [[DEF]]
-  ; GISEL-GFX10-NEXT:   FLAT_STORE_SHORT [[COPY3]], [[V_ADD_F16_e64_]], 0, 0, implicit $exec, implicit $flat_scr :: (store (s16) into `ptr poison`)
-  ; GISEL-GFX10-NEXT:   S_ENDPGM 0
-  ;
   ; DAGISEL-GFX11-WF32-LABEL: name: amdgpu_cs_chain_cc_bfloat
   ; DAGISEL-GFX11-WF32: bb.0 (%ir-block.0):
   ; DAGISEL-GFX11-WF32-NEXT:   liveins: $sgpr0, $vgpr8
@@ -996,9 +970,9 @@ define amdgpu_cs_chain_preserve void @amdgpu_cs_chain_preserve_cc_i16(i16 inreg
   ; GISEL-GFX11-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr8
   ; GISEL-GFX11-NEXT:   [[DEF:%[0-9]+]]:sreg_64 = IMPLICIT_DEF
   ; GISEL-GFX11-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY [[COPY]]
-  ; GISEL-GFX11-NEXT:   [[V_ADD_NC_U16_e64_:%[0-9]+]]:vgpr_32 = V_ADD_NC_U16_fake16_e64 0, [[COPY2]], 0, [[COPY1]], 0, 0, implicit $exec
+  ; GISEL-GFX11-NEXT:   [[V_ADD_NC_U16_fake16_e64_:%[0-9]+]]:vgpr_32 = V_ADD_NC_U16_fake16_e64 0, [[COPY2]], 0, [[COPY1]], 0, 0, implicit $exec
   ; GISEL-GFX11-NEXT:   [[COPY3:%[0-9]+]]:vreg_64 = COPY [[DEF]]
-  ; GISEL-GFX11-NEXT:   FLAT_STORE_SHORT [[COPY3]], [[V_ADD_NC_U16_e64_]], 0, 0, implicit $exec, implicit $flat_scr :: (store (s16) into `ptr poison`)
+  ; GISEL-GFX11-NEXT:   FLAT_STORE_SHORT [[COPY3]], [[V_ADD_NC_U16_fake16_e64_]], 0, 0, implicit $exec, implicit $flat_scr :: (store (s16) into `ptr poison`)
   ; GISEL-GFX11-NEXT:   S_ENDPGM 0
   ;
   ; GISEL-GFX10-LABEL: name: amdgpu_cs_chain_preserve_cc_i16
@@ -1020,10 +994,10 @@ define amdgpu_cs_chain_preserve void @amdgpu_cs_chain_preserve_cc_i16(i16 inreg
   ; DAGISEL-GFX11-WF32-NEXT: {{  $}}
   ; DAGISEL-GFX11-WF32-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr8
   ; DAGISEL-GFX11-WF32-NEXT:   [[COPY1:%[0-9]+]]:sgpr_32 = COPY $sgpr0
-  ; DAGISEL-GFX11-WF32-NEXT:   [[V_ADD_NC_U16_e64_:%[0-9]+]]:vgpr_32 = V_ADD_NC_U16_fake16_e64 0, [[COPY1]], 0, [[COPY]], 0, 0, implicit $exec
+  ; DAGISEL-GFX11-WF32-NEXT:   [[V_ADD_NC_U16_fake16_e64_:%[0-9]+]]:vgpr_32 = V_ADD_NC_U16_fake16_e64 0, [[COPY1]], 0, [[COPY]], 0, 0, implicit $exec
   ; DAGISEL-GFX11-WF32-NEXT:   [[DEF:%[0-9]+]]:sreg_64 = IMPLICIT_DEF
   ; DAGISEL-GFX11-WF32-NEXT:   [[COPY2:%[0-9]+]]:vreg_64 = COPY [[DEF]]
-  ; DAGISEL-GFX11-WF32-NEXT:   FLAT_STORE_SHORT killed [[COPY2]], killed [[V_ADD_NC_U16_e64_]], 0, 0, implicit $exec, implicit $flat_scr :: (store (s16) into `ptr poison`)
+  ; DAGISEL-GFX11-WF32-NEXT:   FLAT_STORE_SHORT killed [[COPY2]], killed [[V_ADD_NC_U16_fake16_e64_]], 0, 0, implicit $exec, implicit $flat_scr :: (store (s16) into `ptr poison`)
   ; DAGISEL-GFX11-WF32-NEXT:   S_ENDPGM 0
   ;
   ; DAGISEL-GFX11-WF64-LABEL: name: amdgpu_cs_chain_preserve_cc_i16
@@ -1032,10 +1006,10 @@ define amdgpu_cs_chain_preserve void @amdgpu_cs_chain_preserve_cc_i16(i16 inreg
   ; DAGISEL-GFX11-WF64-NEXT: {{  $}}
   ; DAGISEL-GFX11-WF64-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr8
   ; DAGISEL-GFX11-WF64-NEXT:   [[COPY1:%[0-9]+]]:sgpr_32 = COPY $sgpr0
-  ; DAGISEL-GFX11-WF64-NEXT:   [[V_ADD_NC_U16_e64_:%[0-9]+]]:vgpr_32 = V_ADD_NC_U16_fake16_e64 0, [[COPY1]], 0, [[COPY]], 0, 0, implicit $exec
+  ; DAGISEL-GFX11-WF64-NEXT:   [[V_ADD_NC_U16_fake16_e64_:%[0-9]+]]:vgpr_32 = V_ADD_NC_U16_fake16_e64 0, [[COPY1]], 0, [[COPY]], 0, 0, implicit $exec
   ; DAGISEL-GFX11-WF64-NEXT:   [[DEF:%[0-9]+]]:sreg_64 = IMPLICIT_DEF
   ; DAGISEL-GFX11-WF64-NEXT:   [[COPY2:%[0-9]+]]:vreg_64 = COPY [[DEF]]
-  ; DAGISEL-GFX11-WF64-NEXT:   FLAT_STORE_SHORT killed [[COPY2]], killed [[V_ADD_NC_U16_e64_]], 0, 0, implicit $exec, implicit $flat_scr :: (store (s16) into `ptr poison`)
+  ; DAGISEL-GFX11-WF64-NEXT:   FLAT_STORE_SHORT killed [[COPY2]], killed [[V_ADD_NC_U16_fake16_e64_]], 0, 0, implicit $exec, implicit $flat_scr :: (store (s16) into `ptr poison`)
   ; DAGISEL-GFX11-WF64-NEXT:   S_ENDPGM 0
   ;
   ; DAGISEL-GFX10-WF32-LABEL: name: amdgpu_cs_chain_preserve_cc_i16
diff --git a/llvm/test/CodeGen/AMDGPU/kernel-args.ll b/llvm/test/CodeGen/AMDGPU/kernel-args.ll
index 8704f4e780448..121891adef182 100644
--- a/llvm/test/CodeGen/AMDGPU/kernel-args.ll
+++ b/llvm/test/CodeGen/AMDGPU/kernel-args.ll
@@ -1025,74 +1025,67 @@ define amdgpu_kernel void @v3i16_arg(ptr addrspace(1) nocapture %out, <3 x i16>
 ;
 ; EG-LABEL: v3i16_arg:
 ; EG:       ; %bb.0: ; %entry
-; EG-NEXT:    ALU 0, @12, KC0[], KC1[]
-; EG-NEXT:    TEX 2 @6
-; EG-NEXT:    ALU 19, @13, KC0[CB0:0-32], KC1[]
-; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T6.X, T7.X, 0
-; EG-NEXT:    MEM_RAT MSKOR T5.XW, T8.X
+; EG-NEXT:    ALU 0, @10, KC0[], KC1[]
+; EG-NEXT:    TEX 1 @6
+; EG-NEXT:    ALU 14, @11, KC0[CB0:0-32], KC1[]
+; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T1.X, T3.X, 0
+; EG-NEXT:    MEM_RAT MSKOR T2.XW, T0.X
 ; EG-NEXT:    CF_END
 ; EG-NEXT:    Fetch clause starting at 6:
-; EG-NEXT:     VTX_READ_16 T6.X, T5.X, 44, #3
-; EG-NEXT:     VTX_READ_16 T7.X, T5.X, 46, #3
-; EG-NEXT:     VTX_READ_16 T5.X, T5.X, 48, #3
-; EG-NEXT:    ALU clause starting at 12:
-; EG-NEXT:     MOV * T5.X, 0.0,
-; EG-NEXT:    ALU clause starting at 13:
+; EG-NEXT:     VTX_READ_16 T1.X, T0.X, 44, #3
+; EG-NEXT:     VTX_READ_16 T0.X, T0.X, 48, #3
+; EG-NEXT:    ALU clause starting at 10:
+; EG-NEXT:     MOV * T0.X, 0.0,
+; EG-NEXT:    ALU clause starting at 11:
 ; EG-NEXT:     ADD_INT * T0.W, KC0[2].Y, literal.x,
 ; EG-NEXT:    4(5.605194e-45), 0(0.000000e+00)
 ; EG-NEXT:     AND_INT T1.W, PV.W, literal.x,
-; EG-NEXT:     AND_INT * T2.W, T5.X, literal.y,
+; EG-NEXT:     AND_INT * T2.W, T0.X, literal.y,
 ; EG-NEXT:    3(4.203895e-45), 65535(9.183409e-41)
 ; EG-NEXT:     LSHL * T1.W, PV.W, literal.x,
 ; EG-NEXT:    3(4.203895e-45), 0(0.000000e+00)
-; EG-NEXT:     LSHL T5.X, T2.W, PV.W,
-; EG-NEXT:     LSHL * T5.W, literal.x, PV.W,
-; EG-NEXT:    65535(9.183409e-41), 0(0.000000e+00)
-; EG-NEXT:     MOV T5.Y, 0.0,
-; EG-NEXT:     MOV * T5.Z, 0.0,
-; EG-NEXT:     LSHR T8.X, T0.W, literal.x,
-; EG-NEXT:     LSHL T0.W, T7.X, literal.y,
-; EG-NEXT:     AND_INT * T1.W, T6.X, literal.z,
-; EG-NEXT:    2(2.802597e-45), 16(2.242078e-44)
+; EG-NEXT:     LSHL T2.X, T2.W, PV.W,
+; EG-NEXT:     LSHL * T2.W, literal.x, PV.W,
 ; EG-NEXT:    65535(9.183409e-41), 0(0.000000e+00)
-; EG-NEXT:     OR_INT T6.X, PV.W, PS,
-; EG-NEXT:     LSHR * T7.X, KC0[2].Y, literal.x,
+; EG-NEXT:     MOV T2.Y, 0.0,
+; EG-NEXT:     MOV * T2.Z, 0.0,
+; EG-NEXT:     LSHR T0.X, T0.W, literal.x,
+; EG-NEXT:     LSHR * T3.X, KC0[2].Y, literal.x,
 ; EG-NEXT:    2(2.802597e-45), 0(0.000000e+00)
 ;
 ; CM-LABEL: v3i16_arg:
 ; CM:       ; %bb.0: ; %entry
 ; CM-NEXT:    ALU 0, @12, KC0[], KC1[]
-; CM-NEXT:    TEX 2 @6
-; CM-NEXT:    ALU 19, @13, KC0[CB0:0-32], KC1[]
-; CM-NEXT:    MEM_RAT MSKOR T5.XW, T8.X
-; CM-NEXT:    MEM_RAT_CACHELESS STORE_DWORD T6.X, T7.X
+; CM-NEXT:    TEX 0 @8
+; CM-NEXT:    ALU 13, @13, KC0[CB0:0-32], KC1[]
+; CM-NEXT:    MEM_RAT MSKOR T1.XW, T2.X
+; CM-NEXT:    ALU 1, @27, KC0[CB0:0-32], KC1[]
+; CM-NEXT:    TEX 0 @10
+; CM-NEXT:    MEM_RAT_CACHELESS STORE_DWORD T0.X, T1.X
 ; CM-NEXT:    CF_END
-; CM-NEXT:    Fetch clause starting at 6:
-; CM-NEXT:     VTX_READ_16 T6.X, T5.X, 44, #3
-; CM-NEXT:     VTX_READ_16 T7.X, T5.X, 46, #3
-; CM-NEXT:     VTX_READ_16 T5.X, T5.X, 48, #3
+; CM-NEXT:    Fetch clause starting at 8:
+; CM-NEXT:     VTX_READ_16 T1.X, T0.X, 48, #3
+; CM-NEXT:    Fetch clause starting at 10:
+; CM-NEXT:     VTX_READ_16 T0.X, T0.X, 44, #3
 ; CM-NEXT:    ALU clause starting at 12:
-; CM-NEXT:     MOV * T5.X, 0.0,
+; CM-NEXT:     MOV * T0.X, 0.0,
 ; CM-NEXT:    ALU clause starting at 13:
 ; CM-NEXT:     ADD_INT * T0.W, KC0[2].Y, literal.x,
 ; CM-NEXT:    4(5.605194e-45), 0(0.000000e+00)
 ; CM-NEXT:     AND_INT * T1.W, PV.W, literal.x,
 ; CM-NEXT:    3(4.203895e-45), 0(0.000000e+00)
-; CM-NEXT:     AND_INT T0.Z, T5.X, literal.x,
+; CM-NEXT:     AND_INT T0.Z, T1.X, literal.x,
 ; CM-NEXT:     LSHL * T1.W, PV.W, literal.y,
 ; CM-NEXT:    65535(9.183409e-41), 3(4.203895e-45)
-; CM-NEXT:     LSHL T5.X, PV.Z, PV.W,
-; CM-NEXT:     LSHL * T5.W, literal.x, PV.W,
+; CM-NEXT:     LSHL T1.X, PV.Z, PV.W,
+; CM-NEXT:     LSHL * T1.W, literal.x, PV.W,
 ; CM-NEXT:    65535(9.183409e-41), 0(0.000000e+00)
-; CM-NEXT:     MOV T5.Y, 0.0,
-; CM-NEXT:     MOV * T5.Z, 0.0,
-; CM-NEXT:     LSHL T0.Z, T7.X, literal.x,
-; CM-NEXT:     AND_INT * T1.W, T6.X, literal.y, BS:VEC_120/SCL_212
-; CM-NEXT:    16(2.242078e-44), 65535(9.183409e-41)
-; CM-NEXT:     OR_INT * T6.X, PV.Z, PV.W,
-; CM-NEXT:     LSHR * T7.X, KC0[2].Y, literal.x,
+; CM-NEXT:     MOV T1.Y, 0.0,
+; CM-NEXT:     MOV * T1.Z, 0.0,
+; CM-NEXT:     LSHR * T2.X, T0.W, literal.x,
 ; CM-NEXT:    2(2.802597e-45), 0(0.000000e+00)
-; CM-NEXT:     LSHR * T8.X, T0.W, literal.x,
+; CM-NEXT:    ALU clause starting at 27:
+; CM-NEXT:     LSHR * T1.X, KC0[2].Y, literal.x,
 ; CM-NEXT:    2(2.802597e-45), 0(0.000000e+00)
 entry:
   store <3 x i16> %in, ptr addrspace(1) %out, align 4
@@ -2676,205 +2669,47 @@ define amdgpu_kernel void @v8i16_arg(ptr addrspace(1) %out, <8 x i16> %in) {
 ;
 ; EG-LABEL: v8i16_arg:
 ; EG:       ; %bb.0: ; %entry
-; EG-NEXT:    ALU 1, @36, KC0[], KC1[]
-; EG-NEXT:    TEX 0 @20
-; EG-NEXT:    ALU 5, @38, KC0[], KC1[]
-; EG-NEXT:    TEX 0 @22
-; EG-NEXT:    ALU 5, @44, KC0[], KC1[]
-; EG-NEXT:    TEX 0 @24
-; EG-NEXT:    ALU 5, @50, KC0[], KC1[]
-; EG-NEXT:    TEX 0 @26
-; EG-NEXT:    ALU 5, @56, KC0[], KC1[]
-; EG-NEXT:    TEX 0 @28
-; EG-NEXT:    ALU 5, @62, KC0[], KC1[]
-; EG-NEXT:    TEX 0 @30
-; EG-NEXT:    ALU 5, @68, KC0[], KC1[]
-; EG-NEXT:    TEX 0 @32
-; EG-NEXT:    ALU 5, @74, KC0[], KC1[]
-; EG-NEXT:    TEX 0 @34
-; EG-NEXT:    ALU 8, @80, KC0[CB0:0-32], KC1[]
-; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T7.XYZW, T8.X, 1
+; EG-NEXT:    ALU 0, @14, KC0[], KC1[]
+; EG-NEXT:    TEX 3 @6
+; EG-NEXT:    ALU 4, @15, KC0[CB0:0-32], KC1[]
+; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T1.XYZW, T0.X, 1
 ; EG-NEXT:    CF_END
 ; EG-NEXT:    PAD
-; EG-NEXT:    Fetch clause starting at 20:
-; EG-NEXT:     VTX_READ_16 T8.X, T7.X, 66, #3
-; EG-NEXT:    Fetch clause starting at 22:
-; EG-NEXT:     VTX_READ_16 T8.X, T7.X, 58, #3
-; EG-NEXT:    Fetch clause starting at 24:
-; EG-NEXT:     VTX_READ_16 T8.X, T7.X, 64, #3
-; EG-NEXT:    Fetch clause starting at 26:
-; EG-NEXT:     VTX_READ_16 T8.X, T7.X, 56, #3
-; EG-NEXT:    Fetch clause starting at 28:
-; EG-NEXT:     VTX_READ_16 T8.X, T7.X, 62, #3
-; EG-NEXT:    Fetch clause starting at 30:
-; EG-NEXT:     VTX_READ_16 T8.X, T7.X, 54, #3
-; EG-NEXT:    Fetch clause starting at 32:
-; EG-NEXT:     VTX_READ_16 T8.X, T7.X, 60, #3
-; EG-NEXT:    Fetch clause starting at 34:
-; EG-NEXT:     VTX_READ_16 T7.X, T7.X, 52, #3
-; EG-NEXT:    ALU clause starting at 36:
-; EG-NEXT:     MOV * T0.Y, T3.X,
-; EG-NEXT:     MOV * T7.X, 0.0,
-; EG-NEXT:    ALU clause starting at 38:
-; EG-NEXT:     LSHL T0.W, T8.X, literal.x,
-; EG-NEXT:     AND_INT * T1.W, T0.Y, literal.y,
-; EG-NEXT:    16(2.242078e-44), 65535(9.183409e-41)
-; EG-NEXT:     OR_INT * T0.W, PS, PV.W,
-; EG-NEXT:     MOV T3.X, PV.W,
-; EG-NEXT:     MOV * T0.Y, T5.X,
-; EG-NEXT:    ALU clause starting at 44:
-; EG-NEXT:     AND_INT T0.W, T0.Y, literal.x,
-; EG-NEXT:     LSHL * T1.W, T8.X, literal.y,
-; EG-NEXT:    65535(9.183409e-41), 16(2.242078e-44)
-; EG-NEXT:     OR_INT * T0.W, PV.W, PS,
-; EG-NEXT:     MOV T5.X, PV.W,
-; EG-NEXT:     MOV * T0.Y, T3.X,
-; EG-NEXT:    ALU clause starting at 50:
-; EG-NEXT:     AND_INT T0.W, T0.Y, literal.x,
-; EG-NEXT:     AND_INT * T1.W, T8.X, literal.y,
-; EG-NEXT:    -65536(nan), 65535(9.183409e-41)
-; EG-NEXT:     OR_INT * T0.W, PV.W, PS,
-; EG-NEXT:     MOV T3.X, PV.W,
-; EG-NEXT:     MOV * T0.Y, T5.X,
-; EG-NEXT:    ALU clause starting at 56:
-; EG-NEXT:     AND_INT T0.W, T0.Y, literal.x,
-; EG-NEXT:     AND_INT * T1.W, T8.X, literal.y,
-; EG-NEXT:    -65536(nan), 65535(9.183409e-41)
-; EG-NEXT:     OR_INT * T0.W, PV.W, PS,
-; EG-NEXT:     MOV T5.X, PV.W,
-; EG-NEXT:     MOV * T0.Y, T2.X,
-; EG-NEXT:    ALU clause starting at 62:
-; EG-NEXT:     AND_INT T0.W, T0.Y, literal.x,
-; EG-NEXT:     LSHL * T1.W, T8.X, literal.y,
-; EG-NEXT:    65535(9.183409e-41), 16(2.242078e-44)
-; EG-NEXT:     OR_INT * T0.W, PV.W, PS,
-; EG-NEXT:     MOV T2.X, PV.W,
-; EG-NEXT:     MOV * T0.Y, T4.X,
-; EG-NEXT:    ALU clause starting at 68:
-; EG-NEXT:     AND_INT T0.W, T0.Y, literal.x,
-; EG-NEXT:     LSHL * T1.W, T8.X, literal.y,
-; EG-NEXT:    65535(9.183409e-41), 16(2.242078e-44)
-; EG-NEXT:     OR_INT * T0.W, PV.W, PS,
-; EG-NEXT:     MOV T4.X, PV.W,
-; EG-NEXT:     MOV * T0.Y, T2.X,
-; EG-NEXT:    ALU clause starting at 74:
-; EG-NEXT:     AND_INT T0.W, T0.Y, literal.x,
-; EG-NEXT:     AND_INT * T1.W, T8.X, literal.y,
-; EG-NEXT:    -65536(nan), 65535(9.183409e-41)
-; EG-NEXT:     OR_INT * T7.Z, PV.W, PS,
-; EG-NEXT:     MOV T2.X, PV.Z,
-; EG-NEXT:     MOV * T0.Y, T4.X,
-; EG-NEXT:    ALU clause starting at 80:
-; EG-NEXT:     LSHR T8.X, KC0[2].Y, literal.x,
-; EG-NEXT:     AND_INT T0.W, T0.Y, literal.y,
-; EG-NEXT:     AND_INT * T1.W, T7.X, literal.z,
-; EG-NEXT:    2(2.802597e-45), -65536(nan)
-; EG-NEXT:    65535(9.183409e-41), 0(0.000000e+00)
-; EG-NEXT:     OR_INT * T7.X, PV.W, PS,
-; EG-NEXT:     MOV T4.X, PV.X,
-; EG-NEXT:     MOV * T7.W, T3.X,
-; EG-NEXT:     MOV * T7.Y, T5.X,
+; EG-NEXT:    Fetch clause starting at 6:
+; EG-NEXT:     VTX_READ_16 T1.X, T0.X, 52, #3
+; EG-NEXT:     VTX_READ_16 T2.X, T0.X, 54, #3
+; EG-NEXT:     VTX_READ_16 T3.X, T0.X, 62, #3
+; EG-NEXT:     VTX_READ_16 T0.X, T0.X, 60, #3
+; EG-NEXT:    ALU clause starting at 14:
+; EG-NEXT:     MOV * T0.X, 0.0,
+; EG-NEXT:    ALU clause starting at 15:
+; EG-NEXT:     MOV T1.Y, T2.X,
+; EG-NEXT:     MOV * T1.Z, T0.X, BS:VEC_120/SCL_212
+; EG-NEXT:     LSHR T0.X, KC0[2].Y, literal.x,
+; EG-NEXT:     MOV * T1.W, T3.X,
+; EG-NEXT:    2(2.802597e-45), 0(0.000000e+00)
 ;
 ; CM-LABEL: v8i16_arg:
 ; CM:       ; %bb.0: ; %entry
-; CM-NEXT:    ALU 1, @36, KC0[], KC1[]
-; CM-NEXT:    TEX 0 @20
-; CM-NEXT:    ALU 5, @38, KC0[], KC1[]
-; CM-NEXT:    TEX 0 @22
-; CM-NEXT:    ALU 5, @44, KC0[], KC1[]
-; CM-NEXT:    TEX 0 @24
-; CM-NEXT:    ALU 5, @50, KC0[], KC1[]
-; CM-NEXT:    TEX 0 @26
-; CM-NEXT:    ALU 5, @56, KC0[], KC1[]
-; CM-NEXT:    TEX 0 @28
-; CM-NEXT:    ALU 5, @62, KC0[], KC1[]
-; CM-NEXT:    TEX 0 @30
-; CM-NEXT:    ALU 5, @68, KC0[], KC1[]
-; CM-NEXT:    TEX 0 @32
-; CM-NEXT:    ALU 5, @74, KC0[], KC1[]
-; CM-NEXT:    TEX 0 @34
-; CM-NEXT:    ALU 8, @80, KC0[CB0:0-32], KC1[]
-; CM-NEXT:    MEM_RAT_CACHELESS STORE_DWORD T7, T8.X
+; CM-NEXT:    ALU 0, @14, KC0[], KC1[]
+; CM-NEXT:    TEX 3 @6
+; CM-NEXT:    ALU 4, @15, KC0[CB0:0-32], KC1[]
+; CM-NEXT:    MEM_RAT_CACHELESS STORE_DWORD T1, T0.X
 ; CM-NEXT:    CF_END
 ; CM-NEXT:    PAD
-; CM-NEXT:    Fetch clause starting at 20:
-; CM-NEXT:     VTX_READ_16 T8.X, T7.X, 66, #3
-; CM-NEXT:    Fetch clause starting at 22:
-; CM-NEXT:     VTX_READ_16 T8.X, T7.X, 58, #3
-; CM-NEXT:    Fetch clause starting at 24:
-; CM-NEXT:     VTX_READ_16 T8.X, T7.X, 64, #3
-; CM-NEXT:    Fetch clause starting at 26:
-; CM-NEXT:     VTX_READ_16 T8.X, T7.X, 56, #3
-; CM-NEXT:    Fetch clause starting at 28:
-; CM-NEXT:     VTX_READ_16 T8.X, T7.X, 62, #3
-; CM-NEXT:    Fetch clause starting at 30:
-; CM-NEXT:     VTX_READ_16 T8.X, T7.X, 54, #3
-; CM-NEXT:    Fetch clause starting at 32:
-; CM-NEXT:     VTX_READ_16 T8.X, T7.X, 60, #3
-; CM-NEXT:    Fetch clause starting at 34:
-; CM-NEXT:     VTX_READ_16 T7.X, T7.X, 52, #3
-; CM-NEXT:    ALU clause starting at 36:
-; CM-NEXT:     MOV * T0.Y, T3.X,
-; CM-NEXT:     MOV * T7.X, 0.0,
-; CM-NEXT:    ALU clause starting at 38:
-; CM-NEXT:     LSHL T0.Z, T8.X, literal.x,
-; CM-NEXT:     AND_INT * T0.W, T0.Y, literal.y,
-; CM-NEXT:    16(2.242078e-44), 65535(9.183409e-41)
-; CM-NEXT:     OR_INT * T0.W, PV.W, PV.Z,
-; CM-NEXT:     MOV T3.X, PV.W,
-; CM-NEXT:     MOV * T0.Y, T5.X,
-; CM-NEXT:    ALU clause starting at 44:
-; CM-NEXT:     AND_INT T0.Z, T0.Y, literal.x,
-; CM-NEXT:     LSHL * T0.W, T8.X, literal.y,
-; CM-NEXT:    65535(9.183409e-41), 16(2.242078e-44)
-; CM-NEXT:     OR_INT * T0.W, PV.Z, PV.W,
-; CM-NEXT:     MOV T5.X, PV.W,
-; CM-NEXT:     MOV * T0.Y, T3.X,
-; CM-NEXT:    ALU clause starting at 50:
-; CM-NEXT:     AND_INT T0.Z, T0.Y, literal.x,
-; CM-NEXT:     AND_INT * T0.W, T8.X, literal.y,
-; CM-NEXT:    -65536(nan), 65535(9.183409e-41)
-; CM-NEXT:     OR_INT * T0.W, PV.Z, PV.W,
-; CM-NEXT:     MOV T3.X, PV.W,
-; CM-NEXT:     MOV * T0.Y, T5.X,
-; CM-NEXT:    ALU clause starting at 56:
-; CM-NEXT:     AND_INT T0.Z, T0.Y, literal.x,
-; CM-NEXT:     AND_INT * T0.W, T8.X, literal.y,
-; CM-NEXT:    -65536(nan), 65535(9.183409e-41)
-; CM-NEXT:     OR_INT * T0.W, PV.Z, PV.W,
-; CM-NEXT:     MOV T5.X, PV.W,
-; CM-NEXT:     MOV * T0.Y, T2.X,
-; CM-NEXT:    ALU clause starting at 62:
-; CM-NEXT:     AND_INT T0.Z, T0.Y, literal.x,
-; CM-NEXT:     LSHL * T0.W, T8.X, literal.y,
-; CM-NEXT:    65535(9.183409e-41), 16(2.242078e-44)
-; CM-NEXT:     OR_INT * T0.W, PV.Z, PV.W,
-; CM-NEXT:     MOV T2.X, PV.W,
-; CM-NEXT:     MOV * T0.Y, T4.X,
-; CM-NEXT:    ALU clause starting at 68:
-; CM-NEXT:     AND_INT T0.Z, T0.Y, literal.x,
-; CM-NEXT:     LSHL * T0.W, T8.X, literal.y,
-; CM-NEXT:    65535(9.183409e-41), 16(2.242078e-44)
-; CM-NEXT:     OR_INT * T0.W, PV.Z, PV.W,
-; CM-NEXT:     MOV T4.X, PV.W,
-; CM-NEXT:     MOV * T0.Y, T2.X,
-; CM-NEXT:    ALU clause starting at 74:
-; CM-NEXT:     AND_INT T0.Z, T0.Y, literal.x,
-; CM-NEXT:     AND_INT * T0.W, T8.X, literal.y,
-; CM-NEXT:    -65536(nan), 65535(9.183409e-41)
-; CM-NEXT:     OR_INT * T7.Z, PV.Z, PV.W,
-; CM-NEXT:     MOV T2.X, PV.Z,
-; CM-NEXT:     MOV * T0.Y, T4.X,
-; CM-NEXT:    ALU clause starting at 80:
-; CM-NEXT:     LSHR T8.X, KC0[2].Y, literal.x,
-; CM-NEXT:     AND_INT T0.Z, T0.Y, literal.y,
-; CM-NEXT:     AND_INT * T0.W, T7.X, literal.z,
-; CM-NEXT:    2(2.802597e-45), -65536(nan)
-; CM-NEXT:    65535(9.183409e-41), 0(0.000000e+00)
-; CM-NEXT:     OR_INT * T7.X, PV.Z, PV.W,
-; CM-NEXT:     MOV T4.X, PV.X,
-; CM-NEXT:     MOV * T7.W, T3.X,
-; CM-NEXT:     MOV * T7.Y, T5.X,
+; CM-NEXT:    Fetch clause starting at 6:
+; CM-NEXT:     VTX_READ_16 T1.X, T0.X, 52, #3
+; CM-NEXT:     VTX_READ_16 T2.X, T0.X, 54, #3
+; CM-NEXT:     VTX_READ_16 T3.X, T0.X, 62, #3
+; CM-NEXT:     VTX_READ_16 T0.X, T0.X, 60, #3
+; CM-NEXT:    ALU clause starting at 14:
+; CM-NEXT:     MOV * T0.X, 0.0,
+; CM-NEXT:    ALU clause starting at 15:
+; CM-NEXT:     MOV T1.Y, T2.X,
+; CM-NEXT:     MOV * T1.Z, T0.X, BS:VEC_120/SCL_212
+; CM-NEXT:     LSHR T0.X, KC0[2].Y, literal.x,
+; CM-NEXT:     MOV * T1.W, T3.X,
+; CM-NEXT:    2(2.802597e-45), 0(0.000000e+00)
 entry:
   store <8 x i16> %in, ptr addrspace(1) %out
   ret void
@@ -3618,392 +3453,68 @@ define amdgpu_kernel void @v16i16_arg(ptr addrspace(1) %out, <16 x i16> %in) {
 ;
 ; EG-LABEL: v16i16_arg:
 ; EG:       ; %bb.0: ; %entry
-; EG-NEXT:    ALU 1, @68, KC0[], KC1[]
-; EG-NEXT:    TEX 0 @36
-; EG-NEXT:    ALU 5, @70, KC0[], KC1[]
-; EG-NEXT:    TEX 0 @38
-; EG-NEXT:    ALU 5, @76, KC0[], KC1[]
-; EG-NEXT:    TEX 0 @40
-; EG-NEXT:    ALU 5, @82, KC0[], KC1[]
-; EG-NEXT:    TEX 0 @42
-; EG-NEXT:    ALU 5, @88, KC0[], KC1[]
-; EG-NEXT:    TEX 0 @44
-; EG-NEXT:    ALU 5, @94, KC0[], KC1[]
-; EG-NEXT:    TEX 0 @46
-; EG-NEXT:    ALU 5, @100, KC0[], KC1[]
-; EG-NEXT:    TEX 0 @48
-; EG-NEXT:    ALU 5, @106, KC0[], KC1[]
-; EG-NEXT:    TEX 0 @50
-; EG-NEXT:    ALU 5, @112, KC0[], KC1[]
-; EG-NEXT:    TEX 0 @52
-; EG-NEXT:    ALU 5, @118, KC0[], KC1[]
-; EG-NEXT:    TEX 0 @54
-; EG-NEXT:    ALU 5, @124, KC0[], KC1[]
-; EG-NEXT:    TEX 0 @56
-; EG-NEXT:    ALU 5, @130, KC0[], KC1[]
-; EG-NEXT:    TEX 0 @58
-; EG-NEXT:    ALU 5, @136, KC0[], KC1[]
-; EG-NEXT:    TEX 0 @60
-; EG-NEXT:    ALU 5, @142, KC0[], KC1[]
-; EG-NEXT:    TEX 0 @62
-; EG-NEXT:    ALU 5, @148, KC0[], KC1[]
-; EG-NEXT:    TEX 0 @64
-; EG-NEXT:    ALU 5, @154, KC0[], KC1[]
-; EG-NEXT:    TEX 0 @66
-; EG-NEXT:    ALU 13, @160, KC0[CB0:0-32], KC1[]
-; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T12.XYZW, T14.X, 0
-; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T11.XYZW, T13.X, 1
+; EG-NEXT:    ALU 0, @22, KC0[], KC1[]
+; EG-NEXT:    TEX 7 @6
+; EG-NEXT:    ALU 10, @23, KC0[CB0:0-32], KC1[]
+; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T1.XYZW, T2.X, 0
+; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T7.XYZW, T0.X, 1
 ; EG-NEXT:    CF_END
-; EG-NEXT:    Fetch clause starting at 36:
-; EG-NEXT:     VTX_READ_16 T12.X, T11.X, 98, #3
-; EG-NEXT:    Fetch clause starting at 38:
-; EG-NEXT:     VTX_READ_16 T12.X, T11.X, 90, #3
-; EG-NEXT:    Fetch clause starting at 40:
-; EG-NEXT:     VTX_READ_16 T12.X, T11.X, 82, #3
-; EG-NEXT:    Fetch clause starting at 42:
-; EG-NEXT:     VTX_READ_16 T12.X, T11.X, 74, #3
-; EG-NEXT:    Fetch clause starting at 44:
-; EG-NEXT:     VTX_READ_16 T12.X, T11.X, 96, #3
-; EG-NEXT:    Fetch clause starting at 46:
-; EG-NEXT:     VTX_READ_16 T12.X, T11.X, 88, #3
-; EG-NEXT:    Fetch clause starting at 48:
-; EG-NEXT:     VTX_READ_16 T12.X, T11.X, 80, #3
-; EG-NEXT:    Fetch clause starting at 50:
-; EG-NEXT:     VTX_READ_16 T12.X, T11.X, 72, #3
-; EG-NEXT:    Fetch clause starting at 52:
-; EG-NEXT:     VTX_READ_16 T12.X, T11.X, 94, #3
-; EG-NEXT:    Fetch clause starting at 54:
-; EG-NEXT:     VTX_READ_16 T12.X, T11.X, 86, #3
-; EG-NEXT:    Fetch clause starting at 56:
-; EG-NEXT:     VTX_READ_16 T12.X, T11.X, 78, #3
-; EG-NEXT:    Fetch clause starting at 58:
-; EG-NEXT:     VTX_READ_16 T12.X, T11.X, 70, #3
-; EG-NEXT:    Fetch clause starting at 60:
-; EG-NEXT:     VTX_READ_16 T12.X, T11.X, 92, #3
-; EG-NEXT:    Fetch clause starting at 62:
-; EG-NEXT:     VTX_READ_16 T12.X, T11.X, 84, #3
-; EG-NEXT:    Fetch clause starting at 64:
-; EG-NEXT:     VTX_READ_16 T13.X, T11.X, 76, #3
-; EG-NEXT:    Fetch clause starting at 66:
-; EG-NEXT:     VTX_READ_16 T11.X, T11.X, 68, #3
-; EG-NEXT:    ALU clause starting at 68:
-; EG-NEXT:     MOV * T0.Y, T3.X,
-; EG-NEXT:     MOV * T11.X, 0.0,
-; EG-NEXT:    ALU clause starting at 70:
-; EG-NEXT:     LSHL T0.W, T12.X, literal.x,
-; EG-NEXT:     AND_INT * T1.W, T0.Y, literal.y,
-; EG-NEXT:    16(2.242078e-44), 65535(9.183409e-41)
-; EG-NEXT:     OR_INT * T0.W, PS, PV.W,
-; EG-NEXT:     MOV T3.X, PV.W,
-; EG-NEXT:     MOV * T0.Y, T5.X,
-; EG-NEXT:    ALU clause starting at 76:
-; EG-NEXT:     AND_INT T0.W, T0.Y, literal.x,
-; EG-NEXT:     LSHL * T1.W, T12.X, literal.y,
-; EG-NEXT:    65535(9.183409e-41), 16(2.242078e-44)
-; EG-NEXT:     OR_INT * T0.W, PV.W, PS,
-; EG-NEXT:     MOV T5.X, PV.W,
-; EG-NEXT:     MOV * T0.Y, T7.X,
-; EG-NEXT:    ALU clause starting at 82:
-; EG-NEXT:     AND_INT T0.W, T0.Y, literal.x,
-; EG-NEXT:     LSHL * T1.W, T12.X, literal.y,
-; EG-NEXT:    65535(9.183409e-41), 16(2.242078e-44)
-; EG-NEXT:     OR_INT * T0.W, PV.W, PS,
-; EG-NEXT:     MOV T7.X, PV.W,
-; EG-NEXT:     MOV * T0.Y, T9.X,
-; EG-NEXT:    ALU clause starting at 88:
-; EG-NEXT:     AND_INT T0.W, T0.Y, literal.x,
-; EG-NEXT:     LSHL * T1.W, T12.X, literal.y,
-; EG-NEXT:    65535(9.183409e-41), 16(2.242078e-44)
-; EG-NEXT:     OR_INT * T0.W, PV.W, PS,
-; EG-NEXT:     MOV T9.X, PV.W,
-; EG-NEXT:     MOV * T0.Y, T3.X,
-; EG-NEXT:    ALU clause starting at 94:
-; EG-NEXT:     AND_INT T0.W, T0.Y, literal.x,
-; EG-NEXT:     AND_INT * T1.W, T12.X, literal.y,
-; EG-NEXT:    -65536(nan), 65535(9.183409e-41)
-; EG-NEXT:     OR_INT * T0.W, PV.W, PS,
-; EG-NEXT:     MOV T3.X, PV.W,
-; EG-NEXT:     MOV * T0.Y, T5.X,
-; EG-NEXT:    ALU clause starting at 100:
-; EG-NEXT:     AND_INT T0.W, T0.Y, literal.x,
-; EG-NEXT:     AND_INT * T1.W, T12.X, literal.y,
-; EG-NEXT:    -65536(nan), 65535(9.183409e-41)
-; EG-NEXT:     OR_INT * T0.W, PV.W, PS,
-; EG-NEXT:     MOV T5.X, PV.W,
-; EG-NEXT:     MOV * T0.Y, T7.X,
-; EG-NEXT:    ALU clause starting at 106:
-; EG-NEXT:     AND_INT T0.W, T0.Y, literal.x,
-; EG-NEXT:     AND_INT * T1.W, T12.X, literal.y,
-; EG-NEXT:    -65536(nan), 65535(9.183409e-41)
-; EG-NEXT:     OR_INT * T0.W, PV.W, PS,
-; EG-NEXT:     MOV T7.X, PV.W,
-; EG-NEXT:     MOV * T0.Y, T9.X,
-; EG-NEXT:    ALU clause starting at 112:
-; EG-NEXT:     AND_INT T0.W, T0.Y, literal.x,
-; EG-NEXT:     AND_INT * T1.W, T12.X, literal.y,
-; EG-NEXT:    -65536(nan), 65535(9.183409e-41)
-; EG-NEXT:     OR_INT * T0.W, PV.W, PS,
-; EG-NEXT:     MOV T9.X, PV.W,
-; EG-NEXT:     MOV * T0.Y, T2.X,
-; EG-NEXT:    ALU clause starting at 118:
-; EG-NEXT:     AND_INT T0.W, T0.Y, literal.x,
-; EG-NEXT:     LSHL * T1.W, T12.X, literal.y,
-; EG-NEXT:    65535(9.183409e-41), 16(2.242078e-44)
-; EG-NEXT:     OR_INT * T0.W, PV.W, PS,
-; EG-NEXT:     MOV T2.X, PV.W,
-; EG-NEXT:     MOV * T0.Y, T4.X,
-; EG-NEXT:    ALU clause starting at 124:
-; EG-NEXT:     AND_INT T0.W, T0.Y, literal.x,
-; EG-NEXT:     LSHL * T1.W, T12.X, literal.y,
-; EG-NEXT:    65535(9.183409e-41), 16(2.242078e-44)
-; EG-NEXT:     OR_INT * T0.W, PV.W, PS,
-; EG-NEXT:     MOV T4.X, PV.W,
-; EG-NEXT:     MOV * T0.Y, T6.X,
-; EG-NEXT:    ALU clause starting at 130:
-; EG-NEXT:     AND_INT T0.W, T0.Y, literal.x,
-; EG-NEXT:     LSHL * T1.W, T12.X, literal.y,
-; EG-NEXT:    65535(9.183409e-41), 16(2.242078e-44)
-; EG-NEXT:     OR_INT * T0.W, PV.W, PS,
-; EG-NEXT:     MOV T6.X, PV.W,
-; EG-NEXT:     MOV * T0.Y, T8.X,
-; EG-NEXT:    ALU clause starting at 136:
-; EG-NEXT:     AND_INT T0.W, T0.Y, literal.x,
-; EG-NEXT:     LSHL * T1.W, T12.X, literal.y,
-; EG-NEXT:    65535(9.183409e-41), 16(2.242078e-44)
-; EG-NEXT:     OR_INT * T0.W, PV.W, PS,
-; EG-NEXT:     MOV T8.X, PV.W,
-; EG-NEXT:     MOV * T0.Y, T2.X,
-; EG-NEXT:    ALU clause starting at 142:
-; EG-NEXT:     AND_INT T0.W, T0.Y, literal.x,
-; EG-NEXT:     AND_INT * T1.W, T12.X, literal.y,
-; EG-NEXT:    -65536(nan), 65535(9.183409e-41)
-; EG-NEXT:     OR_INT * T12.Z, PV.W, PS,
-; EG-NEXT:     MOV T2.X, PV.Z,
-; EG-NEXT:     MOV * T0.Y, T4.X,
-; EG-NEXT:    ALU clause starting at 148:
-; EG-NEXT:     AND_INT T0.W, T0.Y, literal.x,
-; EG-NEXT:     AND_INT * T1.W, T12.X, literal.y,
-; EG-NEXT:    -65536(nan), 65535(9.183409e-41)
-; EG-NEXT:     OR_INT * T12.X, PV.W, PS,
-; EG-NEXT:     MOV T4.X, PV.X,
-; EG-NEXT:     MOV * T0.Y, T6.X,
-; EG-NEXT:    ALU clause starting at 154:
-; EG-NEXT:     AND_INT T0.W, T0.Y, literal.x,
-; EG-NEXT:     AND_INT * T1.W, T13.X, literal.y,
-; EG-NEXT:    -65536(nan), 65535(9.183409e-41)
-; EG-NEXT:     OR_INT * T11.Z, PV.W, PS,
-; EG-NEXT:     MOV T6.X, PV.Z,
-; EG-NEXT:     MOV * T0.Y, T8.X,
-; EG-NEXT:    ALU clause starting at 160:
-; EG-NEXT:     LSHR T13.X, KC0[2].Y, literal.x,
+; EG-NEXT:    Fetch clause starting at 6:
+; EG-NEXT:     VTX_READ_16 T1.X, T0.X, 84, #3
+; EG-NEXT:     VTX_READ_16 T2.X, T0.X, 86, #3
+; EG-NEXT:     VTX_READ_16 T3.X, T0.X, 94, #3
+; EG-NEXT:     VTX_READ_16 T4.X, T0.X, 78, #3
+; EG-NEXT:     VTX_READ_16 T5.X, T0.X, 76, #3
+; EG-NEXT:     VTX_READ_16 T6.X, T0.X, 92, #3
+; EG-NEXT:     VTX_READ_16 T7.X, T0.X, 68, #3
+; EG-NEXT:     VTX_READ_16 T0.X, T0.X, 70, #3
+; EG-NEXT:    ALU clause starting at 22:
+; EG-NEXT:     MOV * T0.X, 0.0,
+; EG-NEXT:    ALU clause starting at 23:
+; EG-NEXT:     MOV T1.Y, T2.X,
+; EG-NEXT:     MOV * T7.Y, T0.X,
+; EG-NEXT:     MOV * T1.Z, T6.X,
+; EG-NEXT:     LSHR T0.X, KC0[2].Y, literal.x,
+; EG-NEXT:     MOV T7.Z, T5.X,
 ; EG-NEXT:     ADD_INT * T0.W, KC0[2].Y, literal.y,
 ; EG-NEXT:    2(2.802597e-45), 16(2.242078e-44)
-; EG-NEXT:     LSHR T14.X, PV.W, literal.x,
-; EG-NEXT:     AND_INT T0.W, T0.Y, literal.y,
-; EG-NEXT:     AND_INT * T1.W, T11.X, literal.z,
-; EG-NEXT:    2(2.802597e-45), -65536(nan)
-; EG-NEXT:    65535(9.183409e-41), 0(0.000000e+00)
-; EG-NEXT:     OR_INT * T11.X, PV.W, PS,
-; EG-NEXT:     MOV T8.X, PV.X,
-; EG-NEXT:     MOV * T12.W, T3.X,
-; EG-NEXT:     MOV T12.Y, T5.X,
-; EG-NEXT:     MOV T11.W, T7.X, BS:VEC_120/SCL_212
-; EG-NEXT:     MOV * T11.Y, T9.X,
+; EG-NEXT:     LSHR T2.X, PV.W, literal.x,
+; EG-NEXT:     MOV T7.W, T4.X,
+; EG-NEXT:     MOV * T1.W, T3.X,
+; EG-NEXT:    2(2.802597e-45), 0(0.000000e+00)
 ;
 ; CM-LABEL: v16i16_arg:
 ; CM:       ; %bb.0: ; %entry
-; CM-NEXT:    ALU 1, @68, KC0[], KC1[]
-; CM-NEXT:    TEX 0 @36
-; CM-NEXT:    ALU 5, @70, KC0[], KC1[]
-; CM-NEXT:    TEX 0 @38
-; CM-NEXT:    ALU 5, @76, KC0[], KC1[]
-; CM-NEXT:    TEX 0 @40
-; CM-NEXT:    ALU 5, @82, KC0[], KC1[]
-; CM-NEXT:    TEX 0 @42
-; CM-NEXT:    ALU 5, @88, KC0[], KC1[]
-; CM-NEXT:    TEX 0 @44
-; CM-NEXT:    ALU 5, @94, KC0[], KC1[]
-; CM-NEXT:    TEX 0 @46
-; CM-NEXT:    ALU 5, @100, KC0[], KC1[]
-; CM-NEXT:    TEX 0 @48
-; CM-NEXT:    ALU 5, @106, KC0[], KC1[]
-; CM-NEXT:    TEX 0 @50
-; CM-NEXT:    ALU 5, @112, KC0[], KC1[]
-; CM-NEXT:    TEX 0 @52
-; CM-NEXT:    ALU 5, @118, KC0[], KC1[]
-; CM-NEXT:    TEX 0 @54
-; CM-NEXT:    ALU 5, @124, KC0[], KC1[]
-; CM-NEXT:    TEX 0 @56
-; CM-NEXT:    ALU 5, @130, KC0[], KC1[]
-; CM-NEXT:    TEX 0 @58
-; CM-NEXT:    ALU 5, @136, KC0[], KC1[]
-; CM-NEXT:    TEX 0 @60
-; CM-NEXT:    ALU 5, @142, KC0[], KC1[]
-; CM-NEXT:    TEX 0 @62
-; CM-NEXT:    ALU 5, @148, KC0[], KC1[]
-; CM-NEXT:    TEX 0 @64
-; CM-NEXT:    ALU 5, @154, KC0[], KC1[]
-; CM-NEXT:    TEX 0 @66
-; CM-NEXT:    ALU 14, @160, KC0[CB0:0-32], KC1[]
-; CM-NEXT:    MEM_RAT_CACHELESS STORE_DWORD T11, T14.X
-; CM-NEXT:    MEM_RAT_CACHELESS STORE_DWORD T12, T13.X
+; CM-NEXT:    ALU 0, @22, KC0[], KC1[]
+; CM-NEXT:    TEX 7 @6
+; CM-NEXT:    ALU 11, @23, KC0[CB0:0-32], KC1[]
+; CM-NEXT:    MEM_RAT_CACHELESS STORE_DWORD T7, T2.X
+; CM-NEXT:    MEM_RAT_CACHELESS STORE_DWORD T1, T0.X
 ; CM-NEXT:    CF_END
-; CM-NEXT:    Fetch clause starting at 36:
-; CM-NEXT:     VTX_READ_16 T12.X, T11.X, 98, #3
-; CM-NEXT:    Fetch clause starting at 38:
-; CM-NEXT:     VTX_READ_16 T12.X, T11.X, 90, #3
-; CM-NEXT:    Fetch clause starting at 40:
-; CM-NEXT:     VTX_READ_16 T12.X, T11.X, 82, #3
-; CM-NEXT:    Fetch clause starting at 42:
-; CM-NEXT:     VTX_READ_16 T12.X, T11.X, 74, #3
-; CM-NEXT:    Fetch clause starting at 44:
-; CM-NEXT:     VTX_READ_16 T12.X, T11.X, 96, #3
-; CM-NEXT:    Fetch clause starting at 46:
-; CM-NEXT:     VTX_READ_16 T12.X, T11.X, 88, #3
-; CM-NEXT:    Fetch clause starting at 48:
-; CM-NEXT:     VTX_READ_16 T12.X, T11.X, 80, #3
-; CM-NEXT:    Fetch clause starting at 50:
-; CM-NEXT:     VTX_READ_16 T12.X, T11.X, 72, #3
-; CM-NEXT:    Fetch clause starting at 52:
-; CM-NEXT:     VTX_READ_16 T12.X, T11.X, 94, #3
-; CM-NEXT:    Fetch clause starting at 54:
-; CM-NEXT:     VTX_READ_16 T12.X, T11.X, 86, #3
-; CM-NEXT:    Fetch clause starting at 56:
-; CM-NEXT:     VTX_READ_16 T12.X, T11.X, 78, #3
-; CM-NEXT:    Fetch clause starting at 58:
-; CM-NEXT:     VTX_READ_16 T12.X, T11.X, 70, #3
-; CM-NEXT:    Fetch clause starting at 60:
-; CM-NEXT:     VTX_READ_16 T12.X, T11.X, 92, #3
-; CM-NEXT:    Fetch clause starting at 62:
-; CM-NEXT:     VTX_READ_16 T12.X, T11.X, 84, #3
-; CM-NEXT:    Fetch clause starting at 64:
-; CM-NEXT:     VTX_READ_16 T13.X, T11.X, 76, #3
-; CM-NEXT:    Fetch clause starting at 66:
-; CM-NEXT:     VTX_READ_16 T11.X, T11.X, 68, #3
-; CM-NEXT:    ALU clause starting at 68:
-; CM-NEXT:     MOV * T0.Y, T3.X,
-; CM-NEXT:     MOV * T11.X, 0.0,
-; CM-NEXT:    ALU clause starting at 70:
-; CM-NEXT:     LSHL T0.Z, T12.X, literal.x,
-; CM-NEXT:     AND_INT * T0.W, T0.Y, literal.y,
-; CM-NEXT:    16(2.242078e-44), 65535(9.183409e-41)
-; CM-NEXT:     OR_INT * T0.W, PV.W, PV.Z,
-; CM-NEXT:     MOV T3.X, PV.W,
-; CM-NEXT:     MOV * T0.Y, T5.X,
-; CM-NEXT:    ALU clause starting at 76:
-; CM-NEXT:     AND_INT T0.Z, T0.Y, literal.x,
-; CM-NEXT:     LSHL * T0.W, T12.X, literal.y,
-; CM-NEXT:    65535(9.183409e-41), 16(2.242078e-44)
-; CM-NEXT:     OR_INT * T0.W, PV.Z, PV.W,
-; CM-NEXT:     MOV T5.X, PV.W,
-; CM-NEXT:     MOV * T0.Y, T7.X,
-; CM-NEXT:    ALU clause starting at 82:
-; CM-NEXT:     AND_INT T0.Z, T0.Y, literal.x,
-; CM-NEXT:     LSHL * T0.W, T12.X, literal.y,
-; CM-NEXT:    65535(9.183409e-41), 16(2.242078e-44)
-; CM-NEXT:     OR_INT * T0.W, PV.Z, PV.W,
-; CM-NEXT:     MOV T7.X, PV.W,
-; CM-NEXT:     MOV * T0.Y, T9.X,
-; CM-NEXT:    ALU clause starting at 88:
-; CM-NEXT:     AND_INT T0.Z, T0.Y, literal.x,
-; CM-NEXT:     LSHL * T0.W, T12.X, literal.y,
-; CM-NEXT:    65535(9.183409e-41), 16(2.242078e-44)
-; CM-NEXT:     OR_INT * T0.W, PV.Z, PV.W,
-; CM-NEXT:     MOV T9.X, PV.W,
-; CM-NEXT:     MOV * T0.Y, T3.X,
-; CM-NEXT:    ALU clause starting at 94:
-; CM-NEXT:     AND_INT T0.Z, T0.Y, literal.x,
-; CM-NEXT:     AND_INT * T0.W, T12.X, literal.y,
-; CM-NEXT:    -65536(nan), 65535(9.183409e-41)
-; CM-NEXT:     OR_INT * T0.W, PV.Z, PV.W,
-; CM-NEXT:     MOV T3.X, PV.W,
-; CM-NEXT:     MOV * T0.Y, T5.X,
-; CM-NEXT:    ALU clause starting at 100:
-; CM-NEXT:     AND_INT T0.Z, T0.Y, literal.x,
-; CM-NEXT:     AND_INT * T0.W, T12.X, literal.y,
-; CM-NEXT:    -65536(nan), 65535(9.183409e-41)
-; CM-NEXT:     OR_INT * T0.W, PV.Z, PV.W,
-; CM-NEXT:     MOV T5.X, PV.W,
-; CM-NEXT:     MOV * T0.Y, T7.X,
-; CM-NEXT:    ALU clause starting at 106:
-; CM-NEXT:     AND_INT T0.Z, T0.Y, literal.x,
-; CM-NEXT:     AND_INT * T0.W, T12.X, literal.y,
-; CM-NEXT:    -65536(nan), 65535(9.183409e-41)
-; CM-NEXT:     OR_INT * T0.W, PV.Z, PV.W,
-; CM-NEXT:     MOV T7.X, PV.W,
-; CM-NEXT:     MOV * T0.Y, T9.X,
-; CM-NEXT:    ALU clause starting at 112:
-; CM-NEXT:     AND_INT T0.Z, T0.Y, literal.x,
-; CM-NEXT:     AND_INT * T0.W, T12.X, literal.y,
-; CM-NEXT:    -65536(nan), 65535(9.183409e-41)
-; CM-NEXT:     OR_INT * T0.W, PV.Z, PV.W,
-; CM-NEXT:     MOV T9.X, PV.W,
-; CM-NEXT:     MOV * T0.Y, T2.X,
-; CM-NEXT:    ALU clause starting at 118:
-; CM-NEXT:     AND_INT T0.Z, T0.Y, literal.x,
-; CM-NEXT:     LSHL * T0.W, T12.X, literal.y,
-; CM-NEXT:    65535(9.183409e-41), 16(2.242078e-44)
-; CM-NEXT:     OR_INT * T0.W, PV.Z, PV.W,
-; CM-NEXT:     MOV T2.X, PV.W,
-; CM-NEXT:     MOV * T0.Y, T4.X,
-; CM-NEXT:    ALU clause starting at 124:
-; CM-NEXT:     AND_INT T0.Z, T0.Y, literal.x,
-; CM-NEXT:     LSHL * T0.W, T12.X, literal.y,
-; CM-NEXT:    65535(9.183409e-41), 16(2.242078e-44)
-; CM-NEXT:     OR_INT * T0.W, PV.Z, PV.W,
-; CM-NEXT:     MOV T4.X, PV.W,
-; CM-NEXT:     MOV * T0.Y, T6.X,
-; CM-NEXT:    ALU clause starting at 130:
-; CM-NEXT:     AND_INT T0.Z, T0.Y, literal.x,
-; CM-NEXT:     LSHL * T0.W, T12.X, literal.y,
-; CM-NEXT:    65535(9.183409e-41), 16(2.242078e-44)
-; CM-NEXT:     OR_INT * T0.W, PV.Z, PV.W,
-; CM-NEXT:     MOV T6.X, PV.W,
-; CM-NEXT:     MOV * T0.Y, T8.X,
-; CM-NEXT:    ALU clause starting at 136:
-; CM-NEXT:     AND_INT T0.Z, T0.Y, literal.x,
-; CM-NEXT:     LSHL * T0.W, T12.X, literal.y,
-; CM-NEXT:    65535(9.183409e-41), 16(2.242078e-44)
-; CM-NEXT:     OR_INT * T0.W, PV.Z, PV.W,
-; CM-NEXT:     MOV T8.X, PV.W,
-; CM-NEXT:     MOV * T0.Y, T2.X,
-; CM-NEXT:    ALU clause starting at 142:
-; CM-NEXT:     AND_INT T0.Z, T0.Y, literal.x,
-; CM-NEXT:     AND_INT * T0.W, T12.X, literal.y,
-; CM-NEXT:    -65536(nan), 65535(9.183409e-41)
-; CM-NEXT:     OR_INT * T12.Z, PV.Z, PV.W,
-; CM-NEXT:     MOV T2.X, PV.Z,
-; CM-NEXT:     MOV * T0.Y, T4.X,
-; CM-NEXT:    ALU clause starting at 148:
-; CM-NEXT:     AND_INT T0.Z, T0.Y, literal.x,
-; CM-NEXT:     AND_INT * T0.W, T12.X, literal.y,
-; CM-NEXT:    -65536(nan), 65535(9.183409e-41)
-; CM-NEXT:     OR_INT * T12.X, PV.Z, PV.W,
-; CM-NEXT:     MOV T4.X, PV.X,
-; CM-NEXT:     MOV * T0.Y, T6.X,
-; CM-NEXT:    ALU clause starting at 154:
-; CM-NEXT:     AND_INT T0.Z, T0.Y, literal.x,
-; CM-NEXT:     AND_INT * T0.W, T13.X, literal.y,
-; CM-NEXT:    -65536(nan), 65535(9.183409e-41)
-; CM-NEXT:     OR_INT * T11.Z, PV.Z, PV.W,
-; CM-NEXT:     MOV T6.X, PV.Z,
-; CM-NEXT:     MOV * T0.Y, T8.X,
-; CM-NEXT:    ALU clause starting at 160:
+; CM-NEXT:    Fetch clause starting at 6:
+; CM-NEXT:     VTX_READ_16 T1.X, T0.X, 84, #3
+; CM-NEXT:     VTX_READ_16 T2.X, T0.X, 86, #3
+; CM-NEXT:     VTX_READ_16 T3.X, T0.X, 78, #3
+; CM-NEXT:     VTX_READ_16 T4.X, T0.X, 94, #3
+; CM-NEXT:     VTX_READ_16 T5.X, T0.X, 76, #3
+; CM-NEXT:     VTX_READ_16 T6.X, T0.X, 92, #3
+; CM-NEXT:     VTX_READ_16 T7.X, T0.X, 68, #3
+; CM-NEXT:     VTX_READ_16 T0.X, T0.X, 70, #3
+; CM-NEXT:    ALU clause starting at 22:
+; CM-NEXT:     MOV * T0.X, 0.0,
+; CM-NEXT:    ALU clause starting at 23:
+; CM-NEXT:     MOV * T1.Y, T2.X,
+; CM-NEXT:     MOV T7.Y, T0.X,
+; CM-NEXT:     MOV T1.Z, T6.X, BS:VEC_120/SCL_212
 ; CM-NEXT:     ADD_INT * T0.W, KC0[2].Y, literal.x,
 ; CM-NEXT:    16(2.242078e-44), 0(0.000000e+00)
-; CM-NEXT:     LSHR * T13.X, PV.W, literal.x,
+; CM-NEXT:     LSHR T0.X, PV.W, literal.x,
+; CM-NEXT:     MOV T7.Z, T5.X,
+; CM-NEXT:     MOV * T1.W, T4.X, BS:VEC_120/SCL_212
+; CM-NEXT:    2(2.802597e-45), 0(0.000000e+00)
+; CM-NEXT:     LSHR T2.X, KC0[2].Y, literal.x,
+; CM-NEXT:     MOV * T7.W, T3.X,
 ; CM-NEXT:    2(2.802597e-45), 0(0.000000e+00)
-; CM-NEXT:     LSHR T14.X, KC0[2].Y, literal.x,
-; CM-NEXT:     AND_INT T0.Z, T0.Y, literal.y,
-; CM-NEXT:     AND_INT * T0.W, T11.X, literal.z,
-; CM-NEXT:    2(2.802597e-45), -65536(nan)
-; CM-NEXT:    65535(9.183409e-41), 0(0.000000e+00)
-; CM-NEXT:     OR_INT * T11.X, PV.Z, PV.W,
-; CM-NEXT:     MOV T8.X, PV.X,
-; CM-NEXT:     MOV * T12.W, T3.X,
-; CM-NEXT:     MOV T12.Y, T5.X,
-; CM-NEXT:     MOV * T11.W, T7.X, BS:VEC_120/SCL_212
-; CM-NEXT:     MOV * T11.Y, T9.X,
 entry:
   store <16 x i16> %in, ptr addrspace(1) %out
   ret void
diff --git a/llvm/test/CodeGen/AMDGPU/licm-wwm.mir b/llvm/test/CodeGen/AMDGPU/licm-wwm.mir
index fc20674971a71..85525aa4dbb09 100644
--- a/llvm/test/CodeGen/AMDGPU/licm-wwm.mir
+++ b/llvm/test/CodeGen/AMDGPU/licm-wwm.mir
@@ -1,5 +1,6 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 5
 # RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1100 -run-pass=early-machinelicm,si-wqm -o - %s | FileCheck -check-prefix=GCN %s
+# RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1100 -passes=early-machinelicm,si-wqm -o - %s | FileCheck -check-prefix=GCN %s
 
 # Machine LICM may hoist an intruction from a WWM region, which will force SI-WQM pass
 # to create a second WWM region. This is an unwanted hoisting.
diff --git a/llvm/test/CodeGen/AMDGPU/llc-pipeline.ll b/llvm/test/CodeGen/AMDGPU/llc-pipeline.ll
index a7cc4cfc6707f..893b9fa6fb40d 100644
--- a/llvm/test/CodeGen/AMDGPU/llc-pipeline.ll
+++ b/llvm/test/CodeGen/AMDGPU/llc-pipeline.ll
@@ -143,6 +143,7 @@
 ; GCN-O0-NEXT:        Insert required mode register values
 ; GCN-O0-NEXT:        SI Final Branch Preparation
 ; GCN-O0-NEXT:        Post RA hazard recognizer
+; GCN-O0-NEXT:        AMDGPU Insert waits for SGPR read hazards
 ; GCN-O0-NEXT:        Branch relaxation pass
 ; GCN-O0-NEXT:        AMDGPU Preload Kernel Arguments Prolog
 ; GCN-O0-NEXT:        Register Usage Information Collector Pass
@@ -426,6 +427,7 @@
 ; GCN-O1-NEXT:        SI Final Branch Preparation
 ; GCN-O1-NEXT:        SI peephole optimizations
 ; GCN-O1-NEXT:        Post RA hazard recognizer
+; GCN-O1-NEXT:        AMDGPU Insert waits for SGPR read hazards
 ; GCN-O1-NEXT:        AMDGPU Insert Delay ALU
 ; GCN-O1-NEXT:        Branch relaxation pass
 ; GCN-O1-NEXT:        AMDGPU Preload Kernel Arguments Prolog
@@ -738,6 +740,7 @@
 ; GCN-O1-OPTS-NEXT:        SI Final Branch Preparation
 ; GCN-O1-OPTS-NEXT:        SI peephole optimizations
 ; GCN-O1-OPTS-NEXT:        Post RA hazard recognizer
+; GCN-O1-OPTS-NEXT:        AMDGPU Insert waits for SGPR read hazards
 ; GCN-O1-OPTS-NEXT:        AMDGPU Insert Delay ALU
 ; GCN-O1-OPTS-NEXT:        Branch relaxation pass
 ; GCN-O1-OPTS-NEXT:        AMDGPU Preload Kernel Arguments Prolog
@@ -1056,6 +1059,7 @@
 ; GCN-O2-NEXT:        SI Final Branch Preparation
 ; GCN-O2-NEXT:        SI peephole optimizations
 ; GCN-O2-NEXT:        Post RA hazard recognizer
+; GCN-O2-NEXT:        AMDGPU Insert waits for SGPR read hazards
 ; GCN-O2-NEXT:        AMDGPU Insert Delay ALU
 ; GCN-O2-NEXT:        Branch relaxation pass
 ; GCN-O2-NEXT:        AMDGPU Preload Kernel Arguments Prolog
@@ -1387,6 +1391,7 @@
 ; GCN-O3-NEXT:        SI Final Branch Preparation
 ; GCN-O3-NEXT:        SI peephole optimizations
 ; GCN-O3-NEXT:        Post RA hazard recognizer
+; GCN-O3-NEXT:        AMDGPU Insert waits for SGPR read hazards
 ; GCN-O3-NEXT:        AMDGPU Insert Delay ALU
 ; GCN-O3-NEXT:        Branch relaxation pass
 ; GCN-O3-NEXT:        AMDGPU Preload Kernel Arguments Prolog
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.alignbyte.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.alignbyte.ll
index 8b16fef915a79..07421afde7622 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.alignbyte.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.alignbyte.ll
@@ -1,14 +1,128 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
 ; RUN: llc -mtriple=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefix=GCN %s
+; RUN: llc -mtriple=amdgcn -mcpu=gfx1100 -mattr=+real-true16 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX11-TRUE16 %s
+; RUN: llc -mtriple=amdgcn -mcpu=gfx1100 -mattr=-real-true16 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX11-FAKE16 %s
 
 declare i32 @llvm.amdgcn.alignbyte(i32, i32, i32) #0
 
-; GCN-LABEL: {{^}}v_alignbyte_b32:
-; GCN: v_alignbyte_b32 {{[vs][0-9]+}}, {{[vs][0-9]+}}, {{[vs][0-9]+}}
 define amdgpu_kernel void @v_alignbyte_b32(ptr addrspace(1) %out, i32 %src1, i32 %src2, i32 %src3) #1 {
+; GCN-LABEL: v_alignbyte_b32:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0xb
+; GCN-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x9
+; GCN-NEXT:    s_mov_b32 s7, 0xf000
+; GCN-NEXT:    s_mov_b32 s6, -1
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    v_mov_b32_e32 v0, s1
+; GCN-NEXT:    v_mov_b32_e32 v1, s2
+; GCN-NEXT:    v_alignbyte_b32 v0, s0, v0, v1
+; GCN-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; GCN-NEXT:    s_endpgm
+;
+; GFX11-TRUE16-LABEL: v_alignbyte_b32:
+; GFX11-TRUE16:       ; %bb.0:
+; GFX11-TRUE16-NEXT:    s_clause 0x1
+; GFX11-TRUE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x2c
+; GFX11-TRUE16-NEXT:    s_load_b64 s[4:5], s[4:5], 0x24
+; GFX11-TRUE16-NEXT:    v_mov_b32_e32 v1, 0
+; GFX11-TRUE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v0.l, s2
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_alignbyte_b32 v0, s0, s1, v0.l
+; GFX11-TRUE16-NEXT:    global_store_b32 v1, v0, s[4:5]
+; GFX11-TRUE16-NEXT:    s_endpgm
+;
+; GFX11-FAKE16-LABEL: v_alignbyte_b32:
+; GFX11-FAKE16:       ; %bb.0:
+; GFX11-FAKE16-NEXT:    s_clause 0x1
+; GFX11-FAKE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x2c
+; GFX11-FAKE16-NEXT:    s_load_b64 s[4:5], s[4:5], 0x24
+; GFX11-FAKE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-FAKE16-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v0, s2
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT:    v_alignbyte_b32 v0, s0, s1, v0
+; GFX11-FAKE16-NEXT:    global_store_b32 v1, v0, s[4:5]
+; GFX11-FAKE16-NEXT:    s_endpgm
   %val = call i32 @llvm.amdgcn.alignbyte(i32 %src1, i32 %src2, i32 %src3) #0
   store i32 %val, ptr addrspace(1) %out
   ret void
 }
 
+define amdgpu_kernel void @v_alignbyte_b32_2(ptr addrspace(1) %out, ptr addrspace(1) %src1, ptr addrspace(1) %src2, i32 %src3) #1 {
+; GCN-LABEL: v_alignbyte_b32_2:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GCN-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0xd
+; GCN-NEXT:    s_load_dword s16, s[4:5], 0xf
+; GCN-NEXT:    s_mov_b32 s7, 0xf000
+; GCN-NEXT:    s_mov_b32 s14, 0
+; GCN-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; GCN-NEXT:    v_mov_b32_e32 v1, 0
+; GCN-NEXT:    s_mov_b32 s15, s7
+; GCN-NEXT:    s_mov_b64 s[10:11], s[14:15]
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    s_mov_b64 s[12:13], s[2:3]
+; GCN-NEXT:    buffer_load_dword v2, v[0:1], s[12:15], 0 addr64 glc
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    buffer_load_dword v0, v[0:1], s[8:11], 0 addr64 glc
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    s_mov_b32 s6, -1
+; GCN-NEXT:    s_mov_b32 s4, s0
+; GCN-NEXT:    s_mov_b32 s5, s1
+; GCN-NEXT:    v_alignbyte_b32 v0, v2, v0, s16
+; GCN-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; GCN-NEXT:    s_endpgm
+;
+; GFX11-TRUE16-LABEL: v_alignbyte_b32_2:
+; GFX11-TRUE16:       ; %bb.0:
+; GFX11-TRUE16-NEXT:    s_clause 0x1
+; GFX11-TRUE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-TRUE16-NEXT:    s_load_b64 s[6:7], s[4:5], 0x34
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v3, 0 :: v_dual_and_b32 v0, 0x3ff, v0
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; GFX11-TRUE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-TRUE16-NEXT:    global_load_b32 v1, v0, s[2:3] glc dlc
+; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT:    global_load_b32 v2, v0, s[6:7] glc dlc
+; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT:    s_load_b32 s2, s[4:5], 0x3c
+; GFX11-TRUE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v0.l, s2
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_alignbyte_b32 v0, v1, v2, v0.l
+; GFX11-TRUE16-NEXT:    global_store_b32 v3, v0, s[0:1]
+; GFX11-TRUE16-NEXT:    s_endpgm
+;
+; GFX11-FAKE16-LABEL: v_alignbyte_b32_2:
+; GFX11-FAKE16:       ; %bb.0:
+; GFX11-FAKE16-NEXT:    s_clause 0x1
+; GFX11-FAKE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-FAKE16-NEXT:    s_load_b64 s[6:7], s[4:5], 0x34
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-FAKE16-NEXT:    v_mov_b32_e32 v2, 0
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2)
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; GFX11-FAKE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-FAKE16-NEXT:    global_load_b32 v1, v0, s[2:3] glc dlc
+; GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT:    global_load_b32 v0, v0, s[6:7] glc dlc
+; GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT:    s_load_b32 s2, s[4:5], 0x3c
+; GFX11-FAKE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-FAKE16-NEXT:    v_alignbyte_b32 v0, v1, v0, s2
+; GFX11-FAKE16-NEXT:    global_store_b32 v2, v0, s[0:1]
+; GFX11-FAKE16-NEXT:    s_endpgm
+  %tid = call i32 @llvm.amdgcn.workitem.id.x()
+  %a.gep = getelementptr inbounds i32, ptr addrspace(1) %src1, i32 %tid
+  %b.gep = getelementptr inbounds i32, ptr addrspace(1) %src2, i32 %tid
+  %a.val = load volatile i32, ptr addrspace(1) %a.gep
+  %b.val = load volatile i32, ptr addrspace(1) %b.gep
+
+  %val = call i32 @llvm.amdgcn.alignbyte(i32 %a.val, i32 %b.val, i32 %src3) #0
+  store i32 %val, ptr addrspace(1) %out
+  ret void
+}
+
 attributes #0 = { nounwind readnone }
 attributes #1 = { nounwind }
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.atomic.cond.sub.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.atomic.cond.sub.ll
index f86c9365d0b79..6fbd5ff80b5cd 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.atomic.cond.sub.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.atomic.cond.sub.ll
@@ -64,7 +64,6 @@ define float @raw_buffer_atomic_cond_sub_imm_soff_return(<4 x i32> inreg %rsrc,
 ; GFX12-NEXT:    s_mov_b32 s4, 4
 ; GFX12-NEXT:    buffer_atomic_cond_sub_u32 v0, off, s[0:3], s4 th:TH_ATOMIC_RETURN
 ; GFX12-NEXT:    s_wait_loadcnt 0x0
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 main_body:
   %orig = call i32 @llvm.amdgcn.raw.buffer.atomic.cond.sub.u32.i32(i32 %data, <4 x i32> %rsrc, i32 0, i32 4, i32 0)
@@ -84,7 +83,6 @@ define void @raw_buffer_atomic_cond_sub_imm_soff_no_return(<4 x i32> inreg %rsrc
 ; GFX12-NEXT:    s_mov_b32 s4, 4
 ; GFX12-NEXT:    buffer_atomic_cond_sub_u32 v0, off, s[0:3], s4 th:TH_ATOMIC_RETURN
 ; GFX12-NEXT:    s_wait_loadcnt 0x0
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 main_body:
   %unused = call i32 @llvm.amdgcn.raw.buffer.atomic.cond.sub.u32.i32(i32 %data, <4 x i32> %rsrc, i32 0, i32 4, i32 0)
@@ -102,7 +100,6 @@ define void @raw_buffer_atomic_cond_sub_imm_soff_no_return_forced(<4 x i32> inre
 ; GFX12-NEXT:    v_mov_b32_e32 v0, s16
 ; GFX12-NEXT:    s_mov_b32 s4, 4
 ; GFX12-NEXT:    buffer_atomic_cond_sub_u32 v0, off, s[0:3], s4
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 main_body:
   %unused = call i32 @llvm.amdgcn.raw.buffer.atomic.cond.sub.u32.i32(i32 %data, <4 x i32> %rsrc, i32 0, i32 4, i32 0)
@@ -172,7 +169,6 @@ define float @struct_buffer_atomic_cond_sub_imm_soff_return(<4 x i32> inreg %rsr
 ; GFX12-NEXT:    s_mov_b32 s4, 4
 ; GFX12-NEXT:    buffer_atomic_cond_sub_u32 v0, v1, s[0:3], s4 idxen th:TH_ATOMIC_RETURN
 ; GFX12-NEXT:    s_wait_loadcnt 0x0
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 main_body:
   %orig = call i32 @llvm.amdgcn.struct.buffer.atomic.cond.sub.u32.i32(i32 %data, <4 x i32> %rsrc, i32 0, i32 0, i32 4, i32 0)
@@ -192,7 +188,6 @@ define void @struct_buffer_atomic_cond_sub_imm_soff_no_return(<4 x i32> inreg %r
 ; GFX12-NEXT:    s_mov_b32 s4, 4
 ; GFX12-NEXT:    buffer_atomic_cond_sub_u32 v1, v0, s[0:3], s4 idxen th:TH_ATOMIC_RETURN
 ; GFX12-NEXT:    s_wait_loadcnt 0x0
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 main_body:
   %unused = call i32 @llvm.amdgcn.struct.buffer.atomic.cond.sub.u32.i32(i32 %data, <4 x i32> %rsrc, i32 0, i32 0, i32 4, i32 0)
@@ -210,7 +205,6 @@ define void @struct_buffer_atomic_cond_sub_imm_soff_no_return_forced(<4 x i32> i
 ; GFX12-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s16
 ; GFX12-NEXT:    s_mov_b32 s4, 4
 ; GFX12-NEXT:    buffer_atomic_cond_sub_u32 v1, v0, s[0:3], s4 idxen
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 main_body:
   %unused = call i32 @llvm.amdgcn.struct.buffer.atomic.cond.sub.u32.i32(i32 %data, <4 x i32> %rsrc, i32 0, i32 0, i32 4, i32 0)
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.scalef32.pk.gfx950.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.scalef32.pk.gfx950.ll
index 046a72b9307d0..a0ba97d3b639c 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.scalef32.pk.gfx950.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.scalef32.pk.gfx950.ll
@@ -1,6 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 4
 ; RUN: llc -global-isel=0 -mtriple=amdgcn -mcpu=gfx950 -verify-machineinstrs < %s | FileCheck --check-prefixes=GCN,GFX950-SDAG %s
-; RUN: llc -global-isel=1 -mtriple=amdgcn -mcpu=gfx950 -verify-machineinstrs < %s | FileCheck --check-prefixes=GCN,GFX950-GISEL %s
+; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgcn -mcpu=gfx950 -verify-machineinstrs < %s | FileCheck --check-prefixes=GCN,GFX950-GISEL %s
 
 declare <6 x i32> @llvm.amdgcn.cvt.scalef32.2xpk16.bf6.f32(<16 x float> %src0, <16 x float> %src1, float %scale)
 declare <6 x i32> @llvm.amdgcn.cvt.scalef32.2xpk16.fp6.f32(<16 x float> %src0, <16 x float> %src1, float %scale)
@@ -983,85 +983,35 @@ define <32 x half> @test_cvt_scalef32_pk32_f16_fp6_sl(<6 x i32> inreg %src) {
 }
 
 define <32 x bfloat> @test_cvt_scalef32_pk32_bf16_fp6_vv(<6 x i32> %src, float %scale) {
-; GFX950-SDAG-LABEL: test_cvt_scalef32_pk32_bf16_fp6_vv:
-; GFX950-SDAG:       ; %bb.0:
-; GFX950-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX950-SDAG-NEXT:    v_mov_b32_e32 v22, v6
-; GFX950-SDAG-NEXT:    v_mov_b32_e32 v21, v5
-; GFX950-SDAG-NEXT:    v_mov_b32_e32 v20, v4
-; GFX950-SDAG-NEXT:    v_mov_b32_e32 v19, v3
-; GFX950-SDAG-NEXT:    v_mov_b32_e32 v18, v2
-; GFX950-SDAG-NEXT:    v_mov_b32_e32 v17, v1
-; GFX950-SDAG-NEXT:    v_mov_b32_e32 v16, v0
-; GFX950-SDAG-NEXT:    v_cvt_scalef32_pk32_bf16_fp6 v[0:15], v[16:21], v22
-; GFX950-SDAG-NEXT:    s_setpc_b64 s[30:31]
-;
-; GFX950-GISEL-LABEL: test_cvt_scalef32_pk32_bf16_fp6_vv:
-; GFX950-GISEL:       ; %bb.0:
-; GFX950-GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX950-GISEL-NEXT:    v_cvt_scalef32_pk32_bf16_fp6 v[16:31], v[0:5], v6
-; GFX950-GISEL-NEXT:    v_lshrrev_b32_e32 v1, 16, v16
-; GFX950-GISEL-NEXT:    v_lshrrev_b32_e32 v3, 16, v17
-; GFX950-GISEL-NEXT:    v_lshrrev_b32_e32 v5, 16, v18
-; GFX950-GISEL-NEXT:    v_lshrrev_b32_e32 v7, 16, v19
-; GFX950-GISEL-NEXT:    v_lshrrev_b32_e32 v9, 16, v20
-; GFX950-GISEL-NEXT:    v_lshrrev_b32_e32 v11, 16, v21
-; GFX950-GISEL-NEXT:    v_lshrrev_b32_e32 v13, 16, v22
-; GFX950-GISEL-NEXT:    v_lshrrev_b32_e32 v15, 16, v23
-; GFX950-GISEL-NEXT:    v_mov_b32_e32 v0, v16
-; GFX950-GISEL-NEXT:    v_mov_b32_e32 v2, v17
-; GFX950-GISEL-NEXT:    v_mov_b32_e32 v4, v18
-; GFX950-GISEL-NEXT:    v_mov_b32_e32 v6, v19
-; GFX950-GISEL-NEXT:    v_mov_b32_e32 v8, v20
-; GFX950-GISEL-NEXT:    v_mov_b32_e32 v10, v21
-; GFX950-GISEL-NEXT:    v_mov_b32_e32 v12, v22
-; GFX950-GISEL-NEXT:    v_mov_b32_e32 v14, v23
-; GFX950-GISEL-NEXT:    s_setpc_b64 s[30:31]
+; GCN-LABEL: test_cvt_scalef32_pk32_bf16_fp6_vv:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_mov_b32_e32 v22, v6
+; GCN-NEXT:    v_mov_b32_e32 v21, v5
+; GCN-NEXT:    v_mov_b32_e32 v20, v4
+; GCN-NEXT:    v_mov_b32_e32 v19, v3
+; GCN-NEXT:    v_mov_b32_e32 v18, v2
+; GCN-NEXT:    v_mov_b32_e32 v17, v1
+; GCN-NEXT:    v_mov_b32_e32 v16, v0
+; GCN-NEXT:    v_cvt_scalef32_pk32_bf16_fp6 v[0:15], v[16:21], v22
+; GCN-NEXT:    s_setpc_b64 s[30:31]
   %ret = tail call <32 x bfloat> @llvm.amdgcn.cvt.scalef32.pk32.bf16.fp6(<6 x i32> %src, float %scale)
   ret <32 x bfloat> %ret
 }
 
 define <32 x bfloat> @test_cvt_scalef32_pk32_bf16_fp6_sl(<6 x i32> inreg %src) {
-; GFX950-SDAG-LABEL: test_cvt_scalef32_pk32_bf16_fp6_sl:
-; GFX950-SDAG:       ; %bb.0:
-; GFX950-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX950-SDAG-NEXT:    v_mov_b32_e32 v16, s0
-; GFX950-SDAG-NEXT:    v_mov_b32_e32 v17, s1
-; GFX950-SDAG-NEXT:    v_mov_b32_e32 v18, s2
-; GFX950-SDAG-NEXT:    v_mov_b32_e32 v19, s3
-; GFX950-SDAG-NEXT:    v_mov_b32_e32 v20, s16
-; GFX950-SDAG-NEXT:    v_mov_b32_e32 v21, s17
-; GFX950-SDAG-NEXT:    s_mov_b32 s0, 0x42c80000
-; GFX950-SDAG-NEXT:    v_cvt_scalef32_pk32_bf16_fp6 v[0:15], v[16:21], s0
-; GFX950-SDAG-NEXT:    s_setpc_b64 s[30:31]
-;
-; GFX950-GISEL-LABEL: test_cvt_scalef32_pk32_bf16_fp6_sl:
-; GFX950-GISEL:       ; %bb.0:
-; GFX950-GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX950-GISEL-NEXT:    s_mov_b32 s4, s16
-; GFX950-GISEL-NEXT:    s_mov_b32 s5, s17
-; GFX950-GISEL-NEXT:    v_mov_b64_e32 v[0:1], s[0:1]
-; GFX950-GISEL-NEXT:    v_mov_b64_e32 v[2:3], s[2:3]
-; GFX950-GISEL-NEXT:    v_mov_b64_e32 v[4:5], s[4:5]
-; GFX950-GISEL-NEXT:    v_mov_b32_e32 v6, 0x42c80000
-; GFX950-GISEL-NEXT:    v_cvt_scalef32_pk32_bf16_fp6 v[16:31], v[0:5], v6
-; GFX950-GISEL-NEXT:    v_lshrrev_b32_e32 v1, 16, v16
-; GFX950-GISEL-NEXT:    v_lshrrev_b32_e32 v3, 16, v17
-; GFX950-GISEL-NEXT:    v_lshrrev_b32_e32 v5, 16, v18
-; GFX950-GISEL-NEXT:    v_lshrrev_b32_e32 v7, 16, v19
-; GFX950-GISEL-NEXT:    v_lshrrev_b32_e32 v9, 16, v20
-; GFX950-GISEL-NEXT:    v_lshrrev_b32_e32 v11, 16, v21
-; GFX950-GISEL-NEXT:    v_lshrrev_b32_e32 v13, 16, v22
-; GFX950-GISEL-NEXT:    v_lshrrev_b32_e32 v15, 16, v23
-; GFX950-GISEL-NEXT:    v_mov_b32_e32 v0, v16
-; GFX950-GISEL-NEXT:    v_mov_b32_e32 v2, v17
-; GFX950-GISEL-NEXT:    v_mov_b32_e32 v4, v18
-; GFX950-GISEL-NEXT:    v_mov_b32_e32 v6, v19
-; GFX950-GISEL-NEXT:    v_mov_b32_e32 v8, v20
-; GFX950-GISEL-NEXT:    v_mov_b32_e32 v10, v21
-; GFX950-GISEL-NEXT:    v_mov_b32_e32 v12, v22
-; GFX950-GISEL-NEXT:    v_mov_b32_e32 v14, v23
-; GFX950-GISEL-NEXT:    s_setpc_b64 s[30:31]
+; GCN-LABEL: test_cvt_scalef32_pk32_bf16_fp6_sl:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_mov_b32_e32 v16, s0
+; GCN-NEXT:    v_mov_b32_e32 v17, s1
+; GCN-NEXT:    v_mov_b32_e32 v18, s2
+; GCN-NEXT:    v_mov_b32_e32 v19, s3
+; GCN-NEXT:    v_mov_b32_e32 v20, s16
+; GCN-NEXT:    v_mov_b32_e32 v21, s17
+; GCN-NEXT:    s_mov_b32 s0, 0x42c80000
+; GCN-NEXT:    v_cvt_scalef32_pk32_bf16_fp6 v[0:15], v[16:21], s0
+; GCN-NEXT:    s_setpc_b64 s[30:31]
   %ret = tail call <32 x bfloat> @llvm.amdgcn.cvt.scalef32.pk32.bf16.fp6(<6 x i32> %src, float 100.0)
   ret <32 x bfloat> %ret
 }
@@ -1126,85 +1076,35 @@ define <32 x half> @test_cvt_scalef32_pk32_f16_bf6_sl(<6 x i32> inreg %src) {
 }
 
 define <32 x bfloat> @test_cvt_scalef32_pk32_bf16_bf6_vv(<6 x i32> %src, float %scale) {
-; GFX950-SDAG-LABEL: test_cvt_scalef32_pk32_bf16_bf6_vv:
-; GFX950-SDAG:       ; %bb.0:
-; GFX950-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX950-SDAG-NEXT:    v_mov_b32_e32 v22, v6
-; GFX950-SDAG-NEXT:    v_mov_b32_e32 v21, v5
-; GFX950-SDAG-NEXT:    v_mov_b32_e32 v20, v4
-; GFX950-SDAG-NEXT:    v_mov_b32_e32 v19, v3
-; GFX950-SDAG-NEXT:    v_mov_b32_e32 v18, v2
-; GFX950-SDAG-NEXT:    v_mov_b32_e32 v17, v1
-; GFX950-SDAG-NEXT:    v_mov_b32_e32 v16, v0
-; GFX950-SDAG-NEXT:    v_cvt_scalef32_pk32_bf16_bf6 v[0:15], v[16:21], v22
-; GFX950-SDAG-NEXT:    s_setpc_b64 s[30:31]
-;
-; GFX950-GISEL-LABEL: test_cvt_scalef32_pk32_bf16_bf6_vv:
-; GFX950-GISEL:       ; %bb.0:
-; GFX950-GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX950-GISEL-NEXT:    v_cvt_scalef32_pk32_bf16_bf6 v[16:31], v[0:5], v6
-; GFX950-GISEL-NEXT:    v_lshrrev_b32_e32 v1, 16, v16
-; GFX950-GISEL-NEXT:    v_lshrrev_b32_e32 v3, 16, v17
-; GFX950-GISEL-NEXT:    v_lshrrev_b32_e32 v5, 16, v18
-; GFX950-GISEL-NEXT:    v_lshrrev_b32_e32 v7, 16, v19
-; GFX950-GISEL-NEXT:    v_lshrrev_b32_e32 v9, 16, v20
-; GFX950-GISEL-NEXT:    v_lshrrev_b32_e32 v11, 16, v21
-; GFX950-GISEL-NEXT:    v_lshrrev_b32_e32 v13, 16, v22
-; GFX950-GISEL-NEXT:    v_lshrrev_b32_e32 v15, 16, v23
-; GFX950-GISEL-NEXT:    v_mov_b32_e32 v0, v16
-; GFX950-GISEL-NEXT:    v_mov_b32_e32 v2, v17
-; GFX950-GISEL-NEXT:    v_mov_b32_e32 v4, v18
-; GFX950-GISEL-NEXT:    v_mov_b32_e32 v6, v19
-; GFX950-GISEL-NEXT:    v_mov_b32_e32 v8, v20
-; GFX950-GISEL-NEXT:    v_mov_b32_e32 v10, v21
-; GFX950-GISEL-NEXT:    v_mov_b32_e32 v12, v22
-; GFX950-GISEL-NEXT:    v_mov_b32_e32 v14, v23
-; GFX950-GISEL-NEXT:    s_setpc_b64 s[30:31]
+; GCN-LABEL: test_cvt_scalef32_pk32_bf16_bf6_vv:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_mov_b32_e32 v22, v6
+; GCN-NEXT:    v_mov_b32_e32 v21, v5
+; GCN-NEXT:    v_mov_b32_e32 v20, v4
+; GCN-NEXT:    v_mov_b32_e32 v19, v3
+; GCN-NEXT:    v_mov_b32_e32 v18, v2
+; GCN-NEXT:    v_mov_b32_e32 v17, v1
+; GCN-NEXT:    v_mov_b32_e32 v16, v0
+; GCN-NEXT:    v_cvt_scalef32_pk32_bf16_bf6 v[0:15], v[16:21], v22
+; GCN-NEXT:    s_setpc_b64 s[30:31]
   %ret = tail call <32 x bfloat> @llvm.amdgcn.cvt.scalef32.pk32.bf16.bf6(<6 x i32> %src, float %scale)
   ret <32 x bfloat> %ret
 }
 
 define <32 x bfloat> @test_cvt_scalef32_pk32_bf16_bf6_sl(<6 x i32> inreg %src) {
-; GFX950-SDAG-LABEL: test_cvt_scalef32_pk32_bf16_bf6_sl:
-; GFX950-SDAG:       ; %bb.0:
-; GFX950-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX950-SDAG-NEXT:    v_mov_b32_e32 v16, s0
-; GFX950-SDAG-NEXT:    v_mov_b32_e32 v17, s1
-; GFX950-SDAG-NEXT:    v_mov_b32_e32 v18, s2
-; GFX950-SDAG-NEXT:    v_mov_b32_e32 v19, s3
-; GFX950-SDAG-NEXT:    v_mov_b32_e32 v20, s16
-; GFX950-SDAG-NEXT:    v_mov_b32_e32 v21, s17
-; GFX950-SDAG-NEXT:    s_mov_b32 s0, 0x42c80000
-; GFX950-SDAG-NEXT:    v_cvt_scalef32_pk32_bf16_bf6 v[0:15], v[16:21], s0
-; GFX950-SDAG-NEXT:    s_setpc_b64 s[30:31]
-;
-; GFX950-GISEL-LABEL: test_cvt_scalef32_pk32_bf16_bf6_sl:
-; GFX950-GISEL:       ; %bb.0:
-; GFX950-GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX950-GISEL-NEXT:    s_mov_b32 s4, s16
-; GFX950-GISEL-NEXT:    s_mov_b32 s5, s17
-; GFX950-GISEL-NEXT:    v_mov_b64_e32 v[0:1], s[0:1]
-; GFX950-GISEL-NEXT:    v_mov_b64_e32 v[2:3], s[2:3]
-; GFX950-GISEL-NEXT:    v_mov_b64_e32 v[4:5], s[4:5]
-; GFX950-GISEL-NEXT:    v_mov_b32_e32 v6, 0x42c80000
-; GFX950-GISEL-NEXT:    v_cvt_scalef32_pk32_bf16_bf6 v[16:31], v[0:5], v6
-; GFX950-GISEL-NEXT:    v_lshrrev_b32_e32 v1, 16, v16
-; GFX950-GISEL-NEXT:    v_lshrrev_b32_e32 v3, 16, v17
-; GFX950-GISEL-NEXT:    v_lshrrev_b32_e32 v5, 16, v18
-; GFX950-GISEL-NEXT:    v_lshrrev_b32_e32 v7, 16, v19
-; GFX950-GISEL-NEXT:    v_lshrrev_b32_e32 v9, 16, v20
-; GFX950-GISEL-NEXT:    v_lshrrev_b32_e32 v11, 16, v21
-; GFX950-GISEL-NEXT:    v_lshrrev_b32_e32 v13, 16, v22
-; GFX950-GISEL-NEXT:    v_lshrrev_b32_e32 v15, 16, v23
-; GFX950-GISEL-NEXT:    v_mov_b32_e32 v0, v16
-; GFX950-GISEL-NEXT:    v_mov_b32_e32 v2, v17
-; GFX950-GISEL-NEXT:    v_mov_b32_e32 v4, v18
-; GFX950-GISEL-NEXT:    v_mov_b32_e32 v6, v19
-; GFX950-GISEL-NEXT:    v_mov_b32_e32 v8, v20
-; GFX950-GISEL-NEXT:    v_mov_b32_e32 v10, v21
-; GFX950-GISEL-NEXT:    v_mov_b32_e32 v12, v22
-; GFX950-GISEL-NEXT:    v_mov_b32_e32 v14, v23
-; GFX950-GISEL-NEXT:    s_setpc_b64 s[30:31]
+; GCN-LABEL: test_cvt_scalef32_pk32_bf16_bf6_sl:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_mov_b32_e32 v16, s0
+; GCN-NEXT:    v_mov_b32_e32 v17, s1
+; GCN-NEXT:    v_mov_b32_e32 v18, s2
+; GCN-NEXT:    v_mov_b32_e32 v19, s3
+; GCN-NEXT:    v_mov_b32_e32 v20, s16
+; GCN-NEXT:    v_mov_b32_e32 v21, s17
+; GCN-NEXT:    s_mov_b32 s0, 0x42c80000
+; GCN-NEXT:    v_cvt_scalef32_pk32_bf16_bf6 v[0:15], v[16:21], s0
+; GCN-NEXT:    s_setpc_b64 s[30:31]
   %ret = tail call <32 x bfloat> @llvm.amdgcn.cvt.scalef32.pk32.bf16.bf6(<6 x i32> %src, float 100.0)
   ret <32 x bfloat> %ret
 }
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.scalef32.pk.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.scalef32.pk.ll
index f9fd7e253b124..517c87193598d 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.scalef32.pk.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.scalef32.pk.ll
@@ -1,6 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 4
 ; RUN: llc -global-isel=0 -mtriple=amdgcn -mcpu=gfx950 -verify-machineinstrs < %s | FileCheck -check-prefix=GFX950-SDAG %s
-; RUN: llc -global-isel=1 -mtriple=amdgcn -mcpu=gfx950 -verify-machineinstrs < %s | FileCheck -check-prefix=GFX950-GISEL %s
+; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgcn -mcpu=gfx950 -verify-machineinstrs < %s | FileCheck -check-prefix=GFX950-GISEL %s
 
 declare <6 x i32> @llvm.amdgcn.cvt.scalef32.pk32.bf6.bf16(<32 x bfloat> %src, float %scale)
 declare <6 x i32> @llvm.amdgcn.cvt.scalef32.pk32.bf6.f16(<32 x half> %src, float %scale)
@@ -19,44 +19,11 @@ define amdgpu_ps void @test_scalef32_pk32_bf6_bf16_vv(<32 x bfloat> %src, float
 ;
 ; GFX950-GISEL-LABEL: test_scalef32_pk32_bf6_bf16_vv:
 ; GFX950-GISEL:       ; %bb.0:
-; GFX950-GISEL-NEXT:    v_mov_b32_e32 v24, v17
 ; GFX950-GISEL-NEXT:    v_mov_b32_e32 v25, v18
-; GFX950-GISEL-NEXT:    v_lshrrev_b32_e32 v17, 16, v0
-; GFX950-GISEL-NEXT:    v_lshrrev_b32_e32 v18, 16, v1
-; GFX950-GISEL-NEXT:    v_lshrrev_b32_e32 v19, 16, v2
-; GFX950-GISEL-NEXT:    v_lshrrev_b32_e32 v20, 16, v3
-; GFX950-GISEL-NEXT:    v_lshrrev_b32_e32 v21, 16, v4
-; GFX950-GISEL-NEXT:    v_lshrrev_b32_e32 v22, 16, v5
-; GFX950-GISEL-NEXT:    v_lshrrev_b32_e32 v23, 16, v6
-; GFX950-GISEL-NEXT:    v_lshrrev_b32_e32 v26, 16, v7
-; GFX950-GISEL-NEXT:    v_lshrrev_b32_e32 v27, 16, v8
-; GFX950-GISEL-NEXT:    v_lshrrev_b32_e32 v28, 16, v9
-; GFX950-GISEL-NEXT:    v_lshrrev_b32_e32 v29, 16, v10
-; GFX950-GISEL-NEXT:    v_lshrrev_b32_e32 v30, 16, v11
-; GFX950-GISEL-NEXT:    v_lshrrev_b32_e32 v31, 16, v12
-; GFX950-GISEL-NEXT:    v_lshrrev_b32_e32 v32, 16, v13
-; GFX950-GISEL-NEXT:    v_lshrrev_b32_e32 v33, 16, v14
-; GFX950-GISEL-NEXT:    v_lshrrev_b32_e32 v34, 16, v15
-; GFX950-GISEL-NEXT:    v_mov_b32_sdwa v0, v17 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0
-; GFX950-GISEL-NEXT:    v_mov_b32_sdwa v1, v18 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0
-; GFX950-GISEL-NEXT:    v_mov_b32_sdwa v2, v19 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0
-; GFX950-GISEL-NEXT:    v_mov_b32_sdwa v3, v20 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0
-; GFX950-GISEL-NEXT:    v_mov_b32_sdwa v4, v21 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0
-; GFX950-GISEL-NEXT:    v_mov_b32_sdwa v5, v22 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0
-; GFX950-GISEL-NEXT:    v_mov_b32_sdwa v6, v23 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0
-; GFX950-GISEL-NEXT:    v_mov_b32_sdwa v7, v26 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0
-; GFX950-GISEL-NEXT:    v_mov_b32_sdwa v8, v27 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0
-; GFX950-GISEL-NEXT:    v_mov_b32_sdwa v9, v28 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0
-; GFX950-GISEL-NEXT:    v_mov_b32_sdwa v10, v29 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0
-; GFX950-GISEL-NEXT:    v_mov_b32_sdwa v11, v30 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0
-; GFX950-GISEL-NEXT:    v_mov_b32_sdwa v12, v31 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0
-; GFX950-GISEL-NEXT:    v_mov_b32_sdwa v13, v32 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0
-; GFX950-GISEL-NEXT:    v_mov_b32_sdwa v14, v33 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0
-; GFX950-GISEL-NEXT:    v_mov_b32_sdwa v15, v34 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0
-; GFX950-GISEL-NEXT:    s_nop 0
+; GFX950-GISEL-NEXT:    v_mov_b32_e32 v24, v17
 ; GFX950-GISEL-NEXT:    v_cvt_scalef32_pk32_bf6_bf16 v[18:23], v[0:15], v16
-; GFX950-GISEL-NEXT:    global_store_dwordx4 v[24:25], v[18:21], off
 ; GFX950-GISEL-NEXT:    global_store_dwordx2 v[24:25], v[22:23], off offset:16
+; GFX950-GISEL-NEXT:    global_store_dwordx4 v[24:25], v[18:21], off
 ; GFX950-GISEL-NEXT:    s_endpgm
   %cvt = tail call <6 x i32> @llvm.amdgcn.cvt.scalef32.pk32.bf6.bf16(<32 x bfloat> %src, float %scale)
   store <6 x i32> %cvt, ptr addrspace(1) %out, align 8
@@ -90,82 +57,26 @@ define amdgpu_ps void @test_scalef32_pk32_bf6_bf16_sl(<32 x bfloat> inreg %src,
 ;
 ; GFX950-GISEL-LABEL: test_scalef32_pk32_bf6_bf16_sl:
 ; GFX950-GISEL:       ; %bb.0:
-; GFX950-GISEL-NEXT:    s_lshr_b32 s16, s0, 16
-; GFX950-GISEL-NEXT:    s_lshr_b32 s17, s1, 16
-; GFX950-GISEL-NEXT:    s_lshl_b32 s16, s16, 16
-; GFX950-GISEL-NEXT:    s_and_b32 s0, s0, 0xffff
-; GFX950-GISEL-NEXT:    s_lshr_b32 s18, s2, 16
-; GFX950-GISEL-NEXT:    s_or_b32 s0, s16, s0
-; GFX950-GISEL-NEXT:    s_lshl_b32 s16, s17, 16
-; GFX950-GISEL-NEXT:    s_and_b32 s1, s1, 0xffff
-; GFX950-GISEL-NEXT:    s_lshr_b32 s19, s3, 16
-; GFX950-GISEL-NEXT:    s_or_b32 s1, s16, s1
-; GFX950-GISEL-NEXT:    s_lshl_b32 s16, s18, 16
-; GFX950-GISEL-NEXT:    s_and_b32 s2, s2, 0xffff
-; GFX950-GISEL-NEXT:    s_lshr_b32 s20, s4, 16
-; GFX950-GISEL-NEXT:    s_or_b32 s2, s16, s2
-; GFX950-GISEL-NEXT:    s_lshl_b32 s16, s19, 16
-; GFX950-GISEL-NEXT:    s_and_b32 s3, s3, 0xffff
-; GFX950-GISEL-NEXT:    s_lshr_b32 s21, s5, 16
-; GFX950-GISEL-NEXT:    s_or_b32 s3, s16, s3
-; GFX950-GISEL-NEXT:    s_lshl_b32 s16, s20, 16
-; GFX950-GISEL-NEXT:    s_and_b32 s4, s4, 0xffff
-; GFX950-GISEL-NEXT:    s_lshr_b32 s22, s6, 16
-; GFX950-GISEL-NEXT:    s_or_b32 s4, s16, s4
-; GFX950-GISEL-NEXT:    s_lshl_b32 s16, s21, 16
-; GFX950-GISEL-NEXT:    s_and_b32 s5, s5, 0xffff
-; GFX950-GISEL-NEXT:    s_lshr_b32 s23, s7, 16
-; GFX950-GISEL-NEXT:    s_or_b32 s5, s16, s5
-; GFX950-GISEL-NEXT:    s_lshl_b32 s16, s22, 16
-; GFX950-GISEL-NEXT:    s_and_b32 s6, s6, 0xffff
-; GFX950-GISEL-NEXT:    s_lshr_b32 s24, s8, 16
-; GFX950-GISEL-NEXT:    s_or_b32 s6, s16, s6
-; GFX950-GISEL-NEXT:    s_lshl_b32 s16, s23, 16
-; GFX950-GISEL-NEXT:    s_and_b32 s7, s7, 0xffff
-; GFX950-GISEL-NEXT:    s_lshr_b32 s25, s9, 16
-; GFX950-GISEL-NEXT:    s_or_b32 s7, s16, s7
-; GFX950-GISEL-NEXT:    s_lshl_b32 s16, s24, 16
-; GFX950-GISEL-NEXT:    s_and_b32 s8, s8, 0xffff
-; GFX950-GISEL-NEXT:    s_lshr_b32 s26, s10, 16
-; GFX950-GISEL-NEXT:    s_or_b32 s8, s16, s8
-; GFX950-GISEL-NEXT:    s_lshl_b32 s16, s25, 16
-; GFX950-GISEL-NEXT:    s_and_b32 s9, s9, 0xffff
-; GFX950-GISEL-NEXT:    s_lshr_b32 s27, s11, 16
-; GFX950-GISEL-NEXT:    s_or_b32 s9, s16, s9
-; GFX950-GISEL-NEXT:    s_lshl_b32 s16, s26, 16
-; GFX950-GISEL-NEXT:    s_and_b32 s10, s10, 0xffff
-; GFX950-GISEL-NEXT:    s_lshr_b32 s28, s12, 16
-; GFX950-GISEL-NEXT:    s_or_b32 s10, s16, s10
-; GFX950-GISEL-NEXT:    s_lshl_b32 s16, s27, 16
-; GFX950-GISEL-NEXT:    s_and_b32 s11, s11, 0xffff
-; GFX950-GISEL-NEXT:    s_lshr_b32 s29, s13, 16
-; GFX950-GISEL-NEXT:    s_or_b32 s11, s16, s11
-; GFX950-GISEL-NEXT:    s_lshl_b32 s16, s28, 16
-; GFX950-GISEL-NEXT:    s_and_b32 s12, s12, 0xffff
-; GFX950-GISEL-NEXT:    s_lshr_b32 s30, s14, 16
-; GFX950-GISEL-NEXT:    s_or_b32 s12, s16, s12
-; GFX950-GISEL-NEXT:    s_lshl_b32 s16, s29, 16
-; GFX950-GISEL-NEXT:    s_and_b32 s13, s13, 0xffff
-; GFX950-GISEL-NEXT:    s_lshr_b32 s31, s15, 16
-; GFX950-GISEL-NEXT:    s_or_b32 s13, s16, s13
-; GFX950-GISEL-NEXT:    s_lshl_b32 s16, s30, 16
-; GFX950-GISEL-NEXT:    s_and_b32 s14, s14, 0xffff
-; GFX950-GISEL-NEXT:    s_or_b32 s14, s16, s14
-; GFX950-GISEL-NEXT:    s_lshl_b32 s16, s31, 16
-; GFX950-GISEL-NEXT:    s_and_b32 s15, s15, 0xffff
-; GFX950-GISEL-NEXT:    s_or_b32 s15, s16, s15
-; GFX950-GISEL-NEXT:    v_mov_b64_e32 v[16:17], s[14:15]
-; GFX950-GISEL-NEXT:    v_mov_b64_e32 v[14:15], s[12:13]
-; GFX950-GISEL-NEXT:    v_mov_b64_e32 v[12:13], s[10:11]
-; GFX950-GISEL-NEXT:    v_mov_b64_e32 v[10:11], s[8:9]
-; GFX950-GISEL-NEXT:    v_mov_b64_e32 v[8:9], s[6:7]
-; GFX950-GISEL-NEXT:    v_mov_b64_e32 v[6:7], s[4:5]
-; GFX950-GISEL-NEXT:    v_mov_b64_e32 v[4:5], s[2:3]
-; GFX950-GISEL-NEXT:    v_mov_b64_e32 v[2:3], s[0:1]
-; GFX950-GISEL-NEXT:    v_mov_b32_e32 v24, 0x42c80000
-; GFX950-GISEL-NEXT:    v_cvt_scalef32_pk32_bf6_bf16 v[18:23], v[2:17], v24
-; GFX950-GISEL-NEXT:    global_store_dwordx4 v[0:1], v[18:21], off
+; GFX950-GISEL-NEXT:    v_mov_b32_e32 v2, s0
+; GFX950-GISEL-NEXT:    v_mov_b32_e32 v3, s1
+; GFX950-GISEL-NEXT:    v_mov_b32_e32 v4, s2
+; GFX950-GISEL-NEXT:    v_mov_b32_e32 v5, s3
+; GFX950-GISEL-NEXT:    v_mov_b32_e32 v6, s4
+; GFX950-GISEL-NEXT:    v_mov_b32_e32 v7, s5
+; GFX950-GISEL-NEXT:    v_mov_b32_e32 v8, s6
+; GFX950-GISEL-NEXT:    v_mov_b32_e32 v9, s7
+; GFX950-GISEL-NEXT:    v_mov_b32_e32 v10, s8
+; GFX950-GISEL-NEXT:    v_mov_b32_e32 v11, s9
+; GFX950-GISEL-NEXT:    v_mov_b32_e32 v12, s10
+; GFX950-GISEL-NEXT:    v_mov_b32_e32 v13, s11
+; GFX950-GISEL-NEXT:    v_mov_b32_e32 v14, s12
+; GFX950-GISEL-NEXT:    v_mov_b32_e32 v15, s13
+; GFX950-GISEL-NEXT:    v_mov_b32_e32 v16, s14
+; GFX950-GISEL-NEXT:    v_mov_b32_e32 v17, s15
+; GFX950-GISEL-NEXT:    s_mov_b32 s0, 0x42c80000
+; GFX950-GISEL-NEXT:    v_cvt_scalef32_pk32_bf6_bf16 v[18:23], v[2:17], s0
 ; GFX950-GISEL-NEXT:    global_store_dwordx2 v[0:1], v[22:23], off offset:16
+; GFX950-GISEL-NEXT:    global_store_dwordx4 v[0:1], v[18:21], off
 ; GFX950-GISEL-NEXT:    s_endpgm
   %cvt = tail call <6 x i32> @llvm.amdgcn.cvt.scalef32.pk32.bf6.bf16(<32 x bfloat> %src, float 100.0)
   store <6 x i32> %cvt, ptr addrspace(1) %out, align 8
@@ -252,44 +163,11 @@ define amdgpu_ps void @test_scalef32_pk32_fp6_bf16_vv(<32 x bfloat> %src, float
 ;
 ; GFX950-GISEL-LABEL: test_scalef32_pk32_fp6_bf16_vv:
 ; GFX950-GISEL:       ; %bb.0:
-; GFX950-GISEL-NEXT:    v_mov_b32_e32 v24, v17
 ; GFX950-GISEL-NEXT:    v_mov_b32_e32 v25, v18
-; GFX950-GISEL-NEXT:    v_lshrrev_b32_e32 v17, 16, v0
-; GFX950-GISEL-NEXT:    v_lshrrev_b32_e32 v18, 16, v1
-; GFX950-GISEL-NEXT:    v_lshrrev_b32_e32 v19, 16, v2
-; GFX950-GISEL-NEXT:    v_lshrrev_b32_e32 v20, 16, v3
-; GFX950-GISEL-NEXT:    v_lshrrev_b32_e32 v21, 16, v4
-; GFX950-GISEL-NEXT:    v_lshrrev_b32_e32 v22, 16, v5
-; GFX950-GISEL-NEXT:    v_lshrrev_b32_e32 v23, 16, v6
-; GFX950-GISEL-NEXT:    v_lshrrev_b32_e32 v26, 16, v7
-; GFX950-GISEL-NEXT:    v_lshrrev_b32_e32 v27, 16, v8
-; GFX950-GISEL-NEXT:    v_lshrrev_b32_e32 v28, 16, v9
-; GFX950-GISEL-NEXT:    v_lshrrev_b32_e32 v29, 16, v10
-; GFX950-GISEL-NEXT:    v_lshrrev_b32_e32 v30, 16, v11
-; GFX950-GISEL-NEXT:    v_lshrrev_b32_e32 v31, 16, v12
-; GFX950-GISEL-NEXT:    v_lshrrev_b32_e32 v32, 16, v13
-; GFX950-GISEL-NEXT:    v_lshrrev_b32_e32 v33, 16, v14
-; GFX950-GISEL-NEXT:    v_lshrrev_b32_e32 v34, 16, v15
-; GFX950-GISEL-NEXT:    v_mov_b32_sdwa v0, v17 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0
-; GFX950-GISEL-NEXT:    v_mov_b32_sdwa v1, v18 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0
-; GFX950-GISEL-NEXT:    v_mov_b32_sdwa v2, v19 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0
-; GFX950-GISEL-NEXT:    v_mov_b32_sdwa v3, v20 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0
-; GFX950-GISEL-NEXT:    v_mov_b32_sdwa v4, v21 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0
-; GFX950-GISEL-NEXT:    v_mov_b32_sdwa v5, v22 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0
-; GFX950-GISEL-NEXT:    v_mov_b32_sdwa v6, v23 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0
-; GFX950-GISEL-NEXT:    v_mov_b32_sdwa v7, v26 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0
-; GFX950-GISEL-NEXT:    v_mov_b32_sdwa v8, v27 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0
-; GFX950-GISEL-NEXT:    v_mov_b32_sdwa v9, v28 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0
-; GFX950-GISEL-NEXT:    v_mov_b32_sdwa v10, v29 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0
-; GFX950-GISEL-NEXT:    v_mov_b32_sdwa v11, v30 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0
-; GFX950-GISEL-NEXT:    v_mov_b32_sdwa v12, v31 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0
-; GFX950-GISEL-NEXT:    v_mov_b32_sdwa v13, v32 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0
-; GFX950-GISEL-NEXT:    v_mov_b32_sdwa v14, v33 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0
-; GFX950-GISEL-NEXT:    v_mov_b32_sdwa v15, v34 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0
-; GFX950-GISEL-NEXT:    s_nop 0
+; GFX950-GISEL-NEXT:    v_mov_b32_e32 v24, v17
 ; GFX950-GISEL-NEXT:    v_cvt_scalef32_pk32_fp6_bf16 v[18:23], v[0:15], v16
-; GFX950-GISEL-NEXT:    global_store_dwordx4 v[24:25], v[18:21], off
 ; GFX950-GISEL-NEXT:    global_store_dwordx2 v[24:25], v[22:23], off offset:16
+; GFX950-GISEL-NEXT:    global_store_dwordx4 v[24:25], v[18:21], off
 ; GFX950-GISEL-NEXT:    s_endpgm
   %cvt = tail call <6 x i32> @llvm.amdgcn.cvt.scalef32.pk32.fp6.bf16(<32 x bfloat> %src, float %scale)
   store <6 x i32> %cvt, ptr addrspace(1) %out, align 8
@@ -323,82 +201,26 @@ define amdgpu_ps void @test_scalef32_pk32_fp6_bf16_sl(<32 x bfloat> inreg %src,
 ;
 ; GFX950-GISEL-LABEL: test_scalef32_pk32_fp6_bf16_sl:
 ; GFX950-GISEL:       ; %bb.0:
-; GFX950-GISEL-NEXT:    s_lshr_b32 s16, s0, 16
-; GFX950-GISEL-NEXT:    s_lshr_b32 s17, s1, 16
-; GFX950-GISEL-NEXT:    s_lshl_b32 s16, s16, 16
-; GFX950-GISEL-NEXT:    s_and_b32 s0, s0, 0xffff
-; GFX950-GISEL-NEXT:    s_lshr_b32 s18, s2, 16
-; GFX950-GISEL-NEXT:    s_or_b32 s0, s16, s0
-; GFX950-GISEL-NEXT:    s_lshl_b32 s16, s17, 16
-; GFX950-GISEL-NEXT:    s_and_b32 s1, s1, 0xffff
-; GFX950-GISEL-NEXT:    s_lshr_b32 s19, s3, 16
-; GFX950-GISEL-NEXT:    s_or_b32 s1, s16, s1
-; GFX950-GISEL-NEXT:    s_lshl_b32 s16, s18, 16
-; GFX950-GISEL-NEXT:    s_and_b32 s2, s2, 0xffff
-; GFX950-GISEL-NEXT:    s_lshr_b32 s20, s4, 16
-; GFX950-GISEL-NEXT:    s_or_b32 s2, s16, s2
-; GFX950-GISEL-NEXT:    s_lshl_b32 s16, s19, 16
-; GFX950-GISEL-NEXT:    s_and_b32 s3, s3, 0xffff
-; GFX950-GISEL-NEXT:    s_lshr_b32 s21, s5, 16
-; GFX950-GISEL-NEXT:    s_or_b32 s3, s16, s3
-; GFX950-GISEL-NEXT:    s_lshl_b32 s16, s20, 16
-; GFX950-GISEL-NEXT:    s_and_b32 s4, s4, 0xffff
-; GFX950-GISEL-NEXT:    s_lshr_b32 s22, s6, 16
-; GFX950-GISEL-NEXT:    s_or_b32 s4, s16, s4
-; GFX950-GISEL-NEXT:    s_lshl_b32 s16, s21, 16
-; GFX950-GISEL-NEXT:    s_and_b32 s5, s5, 0xffff
-; GFX950-GISEL-NEXT:    s_lshr_b32 s23, s7, 16
-; GFX950-GISEL-NEXT:    s_or_b32 s5, s16, s5
-; GFX950-GISEL-NEXT:    s_lshl_b32 s16, s22, 16
-; GFX950-GISEL-NEXT:    s_and_b32 s6, s6, 0xffff
-; GFX950-GISEL-NEXT:    s_lshr_b32 s24, s8, 16
-; GFX950-GISEL-NEXT:    s_or_b32 s6, s16, s6
-; GFX950-GISEL-NEXT:    s_lshl_b32 s16, s23, 16
-; GFX950-GISEL-NEXT:    s_and_b32 s7, s7, 0xffff
-; GFX950-GISEL-NEXT:    s_lshr_b32 s25, s9, 16
-; GFX950-GISEL-NEXT:    s_or_b32 s7, s16, s7
-; GFX950-GISEL-NEXT:    s_lshl_b32 s16, s24, 16
-; GFX950-GISEL-NEXT:    s_and_b32 s8, s8, 0xffff
-; GFX950-GISEL-NEXT:    s_lshr_b32 s26, s10, 16
-; GFX950-GISEL-NEXT:    s_or_b32 s8, s16, s8
-; GFX950-GISEL-NEXT:    s_lshl_b32 s16, s25, 16
-; GFX950-GISEL-NEXT:    s_and_b32 s9, s9, 0xffff
-; GFX950-GISEL-NEXT:    s_lshr_b32 s27, s11, 16
-; GFX950-GISEL-NEXT:    s_or_b32 s9, s16, s9
-; GFX950-GISEL-NEXT:    s_lshl_b32 s16, s26, 16
-; GFX950-GISEL-NEXT:    s_and_b32 s10, s10, 0xffff
-; GFX950-GISEL-NEXT:    s_lshr_b32 s28, s12, 16
-; GFX950-GISEL-NEXT:    s_or_b32 s10, s16, s10
-; GFX950-GISEL-NEXT:    s_lshl_b32 s16, s27, 16
-; GFX950-GISEL-NEXT:    s_and_b32 s11, s11, 0xffff
-; GFX950-GISEL-NEXT:    s_lshr_b32 s29, s13, 16
-; GFX950-GISEL-NEXT:    s_or_b32 s11, s16, s11
-; GFX950-GISEL-NEXT:    s_lshl_b32 s16, s28, 16
-; GFX950-GISEL-NEXT:    s_and_b32 s12, s12, 0xffff
-; GFX950-GISEL-NEXT:    s_lshr_b32 s30, s14, 16
-; GFX950-GISEL-NEXT:    s_or_b32 s12, s16, s12
-; GFX950-GISEL-NEXT:    s_lshl_b32 s16, s29, 16
-; GFX950-GISEL-NEXT:    s_and_b32 s13, s13, 0xffff
-; GFX950-GISEL-NEXT:    s_lshr_b32 s31, s15, 16
-; GFX950-GISEL-NEXT:    s_or_b32 s13, s16, s13
-; GFX950-GISEL-NEXT:    s_lshl_b32 s16, s30, 16
-; GFX950-GISEL-NEXT:    s_and_b32 s14, s14, 0xffff
-; GFX950-GISEL-NEXT:    s_or_b32 s14, s16, s14
-; GFX950-GISEL-NEXT:    s_lshl_b32 s16, s31, 16
-; GFX950-GISEL-NEXT:    s_and_b32 s15, s15, 0xffff
-; GFX950-GISEL-NEXT:    s_or_b32 s15, s16, s15
-; GFX950-GISEL-NEXT:    v_mov_b64_e32 v[16:17], s[14:15]
-; GFX950-GISEL-NEXT:    v_mov_b64_e32 v[14:15], s[12:13]
-; GFX950-GISEL-NEXT:    v_mov_b64_e32 v[12:13], s[10:11]
-; GFX950-GISEL-NEXT:    v_mov_b64_e32 v[10:11], s[8:9]
-; GFX950-GISEL-NEXT:    v_mov_b64_e32 v[8:9], s[6:7]
-; GFX950-GISEL-NEXT:    v_mov_b64_e32 v[6:7], s[4:5]
-; GFX950-GISEL-NEXT:    v_mov_b64_e32 v[4:5], s[2:3]
-; GFX950-GISEL-NEXT:    v_mov_b64_e32 v[2:3], s[0:1]
-; GFX950-GISEL-NEXT:    v_mov_b32_e32 v24, 0x42c80000
-; GFX950-GISEL-NEXT:    v_cvt_scalef32_pk32_fp6_bf16 v[18:23], v[2:17], v24
-; GFX950-GISEL-NEXT:    global_store_dwordx4 v[0:1], v[18:21], off
+; GFX950-GISEL-NEXT:    v_mov_b32_e32 v2, s0
+; GFX950-GISEL-NEXT:    v_mov_b32_e32 v3, s1
+; GFX950-GISEL-NEXT:    v_mov_b32_e32 v4, s2
+; GFX950-GISEL-NEXT:    v_mov_b32_e32 v5, s3
+; GFX950-GISEL-NEXT:    v_mov_b32_e32 v6, s4
+; GFX950-GISEL-NEXT:    v_mov_b32_e32 v7, s5
+; GFX950-GISEL-NEXT:    v_mov_b32_e32 v8, s6
+; GFX950-GISEL-NEXT:    v_mov_b32_e32 v9, s7
+; GFX950-GISEL-NEXT:    v_mov_b32_e32 v10, s8
+; GFX950-GISEL-NEXT:    v_mov_b32_e32 v11, s9
+; GFX950-GISEL-NEXT:    v_mov_b32_e32 v12, s10
+; GFX950-GISEL-NEXT:    v_mov_b32_e32 v13, s11
+; GFX950-GISEL-NEXT:    v_mov_b32_e32 v14, s12
+; GFX950-GISEL-NEXT:    v_mov_b32_e32 v15, s13
+; GFX950-GISEL-NEXT:    v_mov_b32_e32 v16, s14
+; GFX950-GISEL-NEXT:    v_mov_b32_e32 v17, s15
+; GFX950-GISEL-NEXT:    s_mov_b32 s0, 0x42c80000
+; GFX950-GISEL-NEXT:    v_cvt_scalef32_pk32_fp6_bf16 v[18:23], v[2:17], s0
 ; GFX950-GISEL-NEXT:    global_store_dwordx2 v[0:1], v[22:23], off offset:16
+; GFX950-GISEL-NEXT:    global_store_dwordx4 v[0:1], v[18:21], off
 ; GFX950-GISEL-NEXT:    s_endpgm
   %cvt = tail call <6 x i32> @llvm.amdgcn.cvt.scalef32.pk32.fp6.bf16(<32 x bfloat> %src, float 100.0)
   store <6 x i32> %cvt, ptr addrspace(1) %out, align 8
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.scalef32.sr.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.scalef32.sr.ll
index e1bf9f0daa1ef..d3851b1a084d6 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.scalef32.sr.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.scalef32.sr.ll
@@ -1,6 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 4
 ; RUN: llc -global-isel=0 -mtriple=amdgcn -mcpu=gfx950 -verify-machineinstrs -o - %s | FileCheck -check-prefix=GFX950 %s
-; RUN: llc -global-isel=1 -mtriple=amdgcn -mcpu=gfx950 -verify-machineinstrs -o - %s | FileCheck -check-prefix=GFX950 %s
+; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgcn -mcpu=gfx950 -verify-machineinstrs -o - %s | FileCheck -check-prefix=GFX950 %s
 
 declare i32 @llvm.amdgcn.cvt.scalef32.sr.bf8.bf16(i32 %old, bfloat %src, i32 %seed, float %scale, i32 %dst_sel)
 declare i32 @llvm.amdgcn.cvt.scalef32.sr.bf8.f16(i32 %old, half %src, i32 %seed, float %scale, i32 %dst_sel)
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.scalef32.sr.pk.gfx950.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.scalef32.sr.pk.gfx950.ll
index 1107b46f8f6d3..7433f6611cd9b 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.scalef32.sr.pk.gfx950.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.scalef32.sr.pk.gfx950.ll
@@ -1,6 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 4
 ; RUN: llc -global-isel=0 -mtriple=amdgcn -mcpu=gfx950 -verify-machineinstrs < %s | FileCheck -check-prefix=GFX950 %s
-; RUN: llc -global-isel=1 -mtriple=amdgcn -mcpu=gfx950 -verify-machineinstrs < %s | FileCheck -check-prefix=GFX950 %s
+; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgcn -mcpu=gfx950 -verify-machineinstrs < %s | FileCheck -check-prefix=GFX950 %s
 
 declare i32 @llvm.amdgcn.cvt.scalef32.sr.pk.fp4.f16(i32 %old, <2 x half> %src, i32 %seed, float %scale, i32 %dst_sel)
 declare i32 @llvm.amdgcn.cvt.scalef32.sr.pk.fp4.bf16(i32 %old, <2 x bfloat> %src, i32 %seed, float %scale, i32 %dst_sel)
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.scalef32.sr.pk.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.scalef32.sr.pk.ll
index 0d4598f316c41..18b20e101a938 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.scalef32.sr.pk.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.scalef32.sr.pk.ll
@@ -1,6 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 4
 ; RUN: llc -global-isel=0 -mtriple=amdgcn -mcpu=gfx950 -verify-machineinstrs < %s | FileCheck -check-prefix=GFX950-SDAG %s
-; RUN: llc -global-isel=1 -mtriple=amdgcn -mcpu=gfx950 -verify-machineinstrs < %s | FileCheck -check-prefix=GFX950-GISEL %s
+; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgcn -mcpu=gfx950 -verify-machineinstrs < %s | FileCheck -check-prefix=GFX950-GISEL %s
 
 declare <6 x i32> @llvm.amdgcn.cvt.scalef32.sr.pk32.bf6.bf16(<32 x bfloat> %src, i32 %sr, float %scale)
 declare <6 x i32> @llvm.amdgcn.cvt.scalef32.sr.pk32.bf6.f16(<32 x half> %src, i32 %sr, float %scale)
@@ -19,42 +19,9 @@ define amdgpu_ps void @test_scalef32_sr_pk32_bf6_bf16_vv(<32 x bfloat> %src, i32
 ;
 ; GFX950-GISEL-LABEL: test_scalef32_sr_pk32_bf6_bf16_vv:
 ; GFX950-GISEL:       ; %bb.0:
-; GFX950-GISEL-NEXT:    v_lshrrev_b32_e32 v20, 16, v0
-; GFX950-GISEL-NEXT:    v_lshrrev_b32_e32 v21, 16, v1
-; GFX950-GISEL-NEXT:    v_lshrrev_b32_e32 v22, 16, v2
-; GFX950-GISEL-NEXT:    v_lshrrev_b32_e32 v23, 16, v3
-; GFX950-GISEL-NEXT:    v_lshrrev_b32_e32 v24, 16, v4
-; GFX950-GISEL-NEXT:    v_lshrrev_b32_e32 v25, 16, v5
-; GFX950-GISEL-NEXT:    v_lshrrev_b32_e32 v26, 16, v6
-; GFX950-GISEL-NEXT:    v_lshrrev_b32_e32 v27, 16, v7
-; GFX950-GISEL-NEXT:    v_lshrrev_b32_e32 v28, 16, v8
-; GFX950-GISEL-NEXT:    v_lshrrev_b32_e32 v29, 16, v9
-; GFX950-GISEL-NEXT:    v_lshrrev_b32_e32 v30, 16, v10
-; GFX950-GISEL-NEXT:    v_lshrrev_b32_e32 v31, 16, v11
-; GFX950-GISEL-NEXT:    v_lshrrev_b32_e32 v32, 16, v12
-; GFX950-GISEL-NEXT:    v_lshrrev_b32_e32 v33, 16, v13
-; GFX950-GISEL-NEXT:    v_lshrrev_b32_e32 v34, 16, v14
-; GFX950-GISEL-NEXT:    v_lshrrev_b32_e32 v35, 16, v15
-; GFX950-GISEL-NEXT:    v_mov_b32_sdwa v0, v20 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0
-; GFX950-GISEL-NEXT:    v_mov_b32_sdwa v1, v21 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0
-; GFX950-GISEL-NEXT:    v_mov_b32_sdwa v2, v22 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0
-; GFX950-GISEL-NEXT:    v_mov_b32_sdwa v3, v23 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0
-; GFX950-GISEL-NEXT:    v_mov_b32_sdwa v4, v24 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0
-; GFX950-GISEL-NEXT:    v_mov_b32_sdwa v5, v25 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0
-; GFX950-GISEL-NEXT:    v_mov_b32_sdwa v6, v26 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0
-; GFX950-GISEL-NEXT:    v_mov_b32_sdwa v7, v27 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0
-; GFX950-GISEL-NEXT:    v_mov_b32_sdwa v8, v28 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0
-; GFX950-GISEL-NEXT:    v_mov_b32_sdwa v9, v29 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0
-; GFX950-GISEL-NEXT:    v_mov_b32_sdwa v10, v30 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0
-; GFX950-GISEL-NEXT:    v_mov_b32_sdwa v11, v31 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0
-; GFX950-GISEL-NEXT:    v_mov_b32_sdwa v12, v32 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0
-; GFX950-GISEL-NEXT:    v_mov_b32_sdwa v13, v33 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0
-; GFX950-GISEL-NEXT:    v_mov_b32_sdwa v14, v34 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0
-; GFX950-GISEL-NEXT:    v_mov_b32_sdwa v15, v35 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0
-; GFX950-GISEL-NEXT:    s_nop 0
 ; GFX950-GISEL-NEXT:    v_cvt_scalef32_sr_pk32_bf6_bf16 v[20:25], v[0:15], v16, v17
-; GFX950-GISEL-NEXT:    global_store_dwordx4 v[18:19], v[20:23], off
 ; GFX950-GISEL-NEXT:    global_store_dwordx2 v[18:19], v[24:25], off offset:16
+; GFX950-GISEL-NEXT:    global_store_dwordx4 v[18:19], v[20:23], off
 ; GFX950-GISEL-NEXT:    s_endpgm
   %cvt = tail call <6 x i32> @llvm.amdgcn.cvt.scalef32.sr.pk32.bf6.bf16(<32 x bfloat> %src, i32 %sr, float %scale)
   store <6 x i32> %cvt, ptr addrspace(1) %out, align 8
@@ -88,82 +55,26 @@ define amdgpu_ps void @test_scalef32_sr_pk32_bf6_bf16_sl(<32 x bfloat> inreg %sr
 ;
 ; GFX950-GISEL-LABEL: test_scalef32_sr_pk32_bf6_bf16_sl:
 ; GFX950-GISEL:       ; %bb.0:
-; GFX950-GISEL-NEXT:    s_lshr_b32 s17, s0, 16
-; GFX950-GISEL-NEXT:    s_lshr_b32 s18, s1, 16
-; GFX950-GISEL-NEXT:    s_lshl_b32 s17, s17, 16
-; GFX950-GISEL-NEXT:    s_and_b32 s0, s0, 0xffff
-; GFX950-GISEL-NEXT:    s_lshr_b32 s19, s2, 16
-; GFX950-GISEL-NEXT:    s_or_b32 s0, s17, s0
-; GFX950-GISEL-NEXT:    s_lshl_b32 s17, s18, 16
-; GFX950-GISEL-NEXT:    s_and_b32 s1, s1, 0xffff
-; GFX950-GISEL-NEXT:    s_lshr_b32 s20, s3, 16
-; GFX950-GISEL-NEXT:    s_or_b32 s1, s17, s1
-; GFX950-GISEL-NEXT:    s_lshl_b32 s17, s19, 16
-; GFX950-GISEL-NEXT:    s_and_b32 s2, s2, 0xffff
-; GFX950-GISEL-NEXT:    s_lshr_b32 s21, s4, 16
-; GFX950-GISEL-NEXT:    s_or_b32 s2, s17, s2
-; GFX950-GISEL-NEXT:    s_lshl_b32 s17, s20, 16
-; GFX950-GISEL-NEXT:    s_and_b32 s3, s3, 0xffff
-; GFX950-GISEL-NEXT:    s_lshr_b32 s22, s5, 16
-; GFX950-GISEL-NEXT:    s_or_b32 s3, s17, s3
-; GFX950-GISEL-NEXT:    s_lshl_b32 s17, s21, 16
-; GFX950-GISEL-NEXT:    s_and_b32 s4, s4, 0xffff
-; GFX950-GISEL-NEXT:    s_lshr_b32 s23, s6, 16
-; GFX950-GISEL-NEXT:    s_or_b32 s4, s17, s4
-; GFX950-GISEL-NEXT:    s_lshl_b32 s17, s22, 16
-; GFX950-GISEL-NEXT:    s_and_b32 s5, s5, 0xffff
-; GFX950-GISEL-NEXT:    s_lshr_b32 s24, s7, 16
-; GFX950-GISEL-NEXT:    s_or_b32 s5, s17, s5
-; GFX950-GISEL-NEXT:    s_lshl_b32 s17, s23, 16
-; GFX950-GISEL-NEXT:    s_and_b32 s6, s6, 0xffff
-; GFX950-GISEL-NEXT:    s_lshr_b32 s25, s8, 16
-; GFX950-GISEL-NEXT:    s_or_b32 s6, s17, s6
-; GFX950-GISEL-NEXT:    s_lshl_b32 s17, s24, 16
-; GFX950-GISEL-NEXT:    s_and_b32 s7, s7, 0xffff
-; GFX950-GISEL-NEXT:    s_lshr_b32 s26, s9, 16
-; GFX950-GISEL-NEXT:    s_or_b32 s7, s17, s7
-; GFX950-GISEL-NEXT:    s_lshl_b32 s17, s25, 16
-; GFX950-GISEL-NEXT:    s_and_b32 s8, s8, 0xffff
-; GFX950-GISEL-NEXT:    s_lshr_b32 s27, s10, 16
-; GFX950-GISEL-NEXT:    s_or_b32 s8, s17, s8
-; GFX950-GISEL-NEXT:    s_lshl_b32 s17, s26, 16
-; GFX950-GISEL-NEXT:    s_and_b32 s9, s9, 0xffff
-; GFX950-GISEL-NEXT:    s_lshr_b32 s28, s11, 16
-; GFX950-GISEL-NEXT:    s_or_b32 s9, s17, s9
-; GFX950-GISEL-NEXT:    s_lshl_b32 s17, s27, 16
-; GFX950-GISEL-NEXT:    s_and_b32 s10, s10, 0xffff
-; GFX950-GISEL-NEXT:    s_lshr_b32 s29, s12, 16
-; GFX950-GISEL-NEXT:    s_or_b32 s10, s17, s10
-; GFX950-GISEL-NEXT:    s_lshl_b32 s17, s28, 16
-; GFX950-GISEL-NEXT:    s_and_b32 s11, s11, 0xffff
-; GFX950-GISEL-NEXT:    s_lshr_b32 s30, s13, 16
-; GFX950-GISEL-NEXT:    s_or_b32 s11, s17, s11
-; GFX950-GISEL-NEXT:    s_lshl_b32 s17, s29, 16
-; GFX950-GISEL-NEXT:    s_and_b32 s12, s12, 0xffff
-; GFX950-GISEL-NEXT:    s_lshr_b32 s31, s14, 16
-; GFX950-GISEL-NEXT:    s_or_b32 s12, s17, s12
-; GFX950-GISEL-NEXT:    s_lshl_b32 s17, s30, 16
-; GFX950-GISEL-NEXT:    s_and_b32 s13, s13, 0xffff
-; GFX950-GISEL-NEXT:    s_lshr_b32 s33, s15, 16
-; GFX950-GISEL-NEXT:    s_or_b32 s13, s17, s13
-; GFX950-GISEL-NEXT:    s_lshl_b32 s17, s31, 16
-; GFX950-GISEL-NEXT:    s_and_b32 s14, s14, 0xffff
-; GFX950-GISEL-NEXT:    s_or_b32 s14, s17, s14
-; GFX950-GISEL-NEXT:    s_lshl_b32 s17, s33, 16
-; GFX950-GISEL-NEXT:    s_and_b32 s15, s15, 0xffff
-; GFX950-GISEL-NEXT:    s_or_b32 s15, s17, s15
-; GFX950-GISEL-NEXT:    v_mov_b64_e32 v[16:17], s[14:15]
-; GFX950-GISEL-NEXT:    v_mov_b64_e32 v[14:15], s[12:13]
-; GFX950-GISEL-NEXT:    v_mov_b64_e32 v[12:13], s[10:11]
-; GFX950-GISEL-NEXT:    v_mov_b64_e32 v[10:11], s[8:9]
-; GFX950-GISEL-NEXT:    v_mov_b64_e32 v[8:9], s[6:7]
-; GFX950-GISEL-NEXT:    v_mov_b64_e32 v[6:7], s[4:5]
-; GFX950-GISEL-NEXT:    v_mov_b64_e32 v[4:5], s[2:3]
-; GFX950-GISEL-NEXT:    v_mov_b64_e32 v[2:3], s[0:1]
+; GFX950-GISEL-NEXT:    v_mov_b32_e32 v2, s0
+; GFX950-GISEL-NEXT:    v_mov_b32_e32 v3, s1
+; GFX950-GISEL-NEXT:    v_mov_b32_e32 v4, s2
+; GFX950-GISEL-NEXT:    v_mov_b32_e32 v5, s3
+; GFX950-GISEL-NEXT:    v_mov_b32_e32 v6, s4
+; GFX950-GISEL-NEXT:    v_mov_b32_e32 v7, s5
+; GFX950-GISEL-NEXT:    v_mov_b32_e32 v8, s6
+; GFX950-GISEL-NEXT:    v_mov_b32_e32 v9, s7
+; GFX950-GISEL-NEXT:    v_mov_b32_e32 v10, s8
+; GFX950-GISEL-NEXT:    v_mov_b32_e32 v11, s9
+; GFX950-GISEL-NEXT:    v_mov_b32_e32 v12, s10
+; GFX950-GISEL-NEXT:    v_mov_b32_e32 v13, s11
+; GFX950-GISEL-NEXT:    v_mov_b32_e32 v14, s12
+; GFX950-GISEL-NEXT:    v_mov_b32_e32 v15, s13
+; GFX950-GISEL-NEXT:    v_mov_b32_e32 v16, s14
+; GFX950-GISEL-NEXT:    v_mov_b32_e32 v17, s15
 ; GFX950-GISEL-NEXT:    v_mov_b32_e32 v24, 0x42c80000
 ; GFX950-GISEL-NEXT:    v_cvt_scalef32_sr_pk32_bf6_bf16 v[18:23], v[2:17], s16, v24
-; GFX950-GISEL-NEXT:    global_store_dwordx4 v[0:1], v[18:21], off
 ; GFX950-GISEL-NEXT:    global_store_dwordx2 v[0:1], v[22:23], off offset:16
+; GFX950-GISEL-NEXT:    global_store_dwordx4 v[0:1], v[18:21], off
 ; GFX950-GISEL-NEXT:    s_endpgm
   %cvt = tail call <6 x i32> @llvm.amdgcn.cvt.scalef32.sr.pk32.bf6.bf16(<32 x bfloat> %src, i32 %sr, float 100.0)
   store <6 x i32> %cvt, ptr addrspace(1) %out, align 8
@@ -244,42 +155,9 @@ define amdgpu_ps void @test_scalef32_sr_pk32_fp6_bf16_vv(<32 x bfloat> %src, i32
 ;
 ; GFX950-GISEL-LABEL: test_scalef32_sr_pk32_fp6_bf16_vv:
 ; GFX950-GISEL:       ; %bb.0:
-; GFX950-GISEL-NEXT:    v_lshrrev_b32_e32 v20, 16, v0
-; GFX950-GISEL-NEXT:    v_lshrrev_b32_e32 v21, 16, v1
-; GFX950-GISEL-NEXT:    v_lshrrev_b32_e32 v22, 16, v2
-; GFX950-GISEL-NEXT:    v_lshrrev_b32_e32 v23, 16, v3
-; GFX950-GISEL-NEXT:    v_lshrrev_b32_e32 v24, 16, v4
-; GFX950-GISEL-NEXT:    v_lshrrev_b32_e32 v25, 16, v5
-; GFX950-GISEL-NEXT:    v_lshrrev_b32_e32 v26, 16, v6
-; GFX950-GISEL-NEXT:    v_lshrrev_b32_e32 v27, 16, v7
-; GFX950-GISEL-NEXT:    v_lshrrev_b32_e32 v28, 16, v8
-; GFX950-GISEL-NEXT:    v_lshrrev_b32_e32 v29, 16, v9
-; GFX950-GISEL-NEXT:    v_lshrrev_b32_e32 v30, 16, v10
-; GFX950-GISEL-NEXT:    v_lshrrev_b32_e32 v31, 16, v11
-; GFX950-GISEL-NEXT:    v_lshrrev_b32_e32 v32, 16, v12
-; GFX950-GISEL-NEXT:    v_lshrrev_b32_e32 v33, 16, v13
-; GFX950-GISEL-NEXT:    v_lshrrev_b32_e32 v34, 16, v14
-; GFX950-GISEL-NEXT:    v_lshrrev_b32_e32 v35, 16, v15
-; GFX950-GISEL-NEXT:    v_mov_b32_sdwa v0, v20 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0
-; GFX950-GISEL-NEXT:    v_mov_b32_sdwa v1, v21 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0
-; GFX950-GISEL-NEXT:    v_mov_b32_sdwa v2, v22 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0
-; GFX950-GISEL-NEXT:    v_mov_b32_sdwa v3, v23 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0
-; GFX950-GISEL-NEXT:    v_mov_b32_sdwa v4, v24 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0
-; GFX950-GISEL-NEXT:    v_mov_b32_sdwa v5, v25 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0
-; GFX950-GISEL-NEXT:    v_mov_b32_sdwa v6, v26 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0
-; GFX950-GISEL-NEXT:    v_mov_b32_sdwa v7, v27 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0
-; GFX950-GISEL-NEXT:    v_mov_b32_sdwa v8, v28 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0
-; GFX950-GISEL-NEXT:    v_mov_b32_sdwa v9, v29 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0
-; GFX950-GISEL-NEXT:    v_mov_b32_sdwa v10, v30 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0
-; GFX950-GISEL-NEXT:    v_mov_b32_sdwa v11, v31 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0
-; GFX950-GISEL-NEXT:    v_mov_b32_sdwa v12, v32 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0
-; GFX950-GISEL-NEXT:    v_mov_b32_sdwa v13, v33 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0
-; GFX950-GISEL-NEXT:    v_mov_b32_sdwa v14, v34 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0
-; GFX950-GISEL-NEXT:    v_mov_b32_sdwa v15, v35 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0
-; GFX950-GISEL-NEXT:    s_nop 0
 ; GFX950-GISEL-NEXT:    v_cvt_scalef32_sr_pk32_fp6_bf16 v[20:25], v[0:15], v16, v17
-; GFX950-GISEL-NEXT:    global_store_dwordx4 v[18:19], v[20:23], off
 ; GFX950-GISEL-NEXT:    global_store_dwordx2 v[18:19], v[24:25], off offset:16
+; GFX950-GISEL-NEXT:    global_store_dwordx4 v[18:19], v[20:23], off
 ; GFX950-GISEL-NEXT:    s_endpgm
   %cvt = tail call <6 x i32> @llvm.amdgcn.cvt.scalef32.sr.pk32.fp6.bf16(<32 x bfloat> %src, i32 %sr, float %scale)
   store <6 x i32> %cvt, ptr addrspace(1) %out, align 8
@@ -313,82 +191,26 @@ define amdgpu_ps void @test_scalef32_sr_pk32_fp6_bf16_sl(<32 x bfloat> inreg %sr
 ;
 ; GFX950-GISEL-LABEL: test_scalef32_sr_pk32_fp6_bf16_sl:
 ; GFX950-GISEL:       ; %bb.0:
-; GFX950-GISEL-NEXT:    s_lshr_b32 s17, s0, 16
-; GFX950-GISEL-NEXT:    s_lshr_b32 s18, s1, 16
-; GFX950-GISEL-NEXT:    s_lshl_b32 s17, s17, 16
-; GFX950-GISEL-NEXT:    s_and_b32 s0, s0, 0xffff
-; GFX950-GISEL-NEXT:    s_lshr_b32 s19, s2, 16
-; GFX950-GISEL-NEXT:    s_or_b32 s0, s17, s0
-; GFX950-GISEL-NEXT:    s_lshl_b32 s17, s18, 16
-; GFX950-GISEL-NEXT:    s_and_b32 s1, s1, 0xffff
-; GFX950-GISEL-NEXT:    s_lshr_b32 s20, s3, 16
-; GFX950-GISEL-NEXT:    s_or_b32 s1, s17, s1
-; GFX950-GISEL-NEXT:    s_lshl_b32 s17, s19, 16
-; GFX950-GISEL-NEXT:    s_and_b32 s2, s2, 0xffff
-; GFX950-GISEL-NEXT:    s_lshr_b32 s21, s4, 16
-; GFX950-GISEL-NEXT:    s_or_b32 s2, s17, s2
-; GFX950-GISEL-NEXT:    s_lshl_b32 s17, s20, 16
-; GFX950-GISEL-NEXT:    s_and_b32 s3, s3, 0xffff
-; GFX950-GISEL-NEXT:    s_lshr_b32 s22, s5, 16
-; GFX950-GISEL-NEXT:    s_or_b32 s3, s17, s3
-; GFX950-GISEL-NEXT:    s_lshl_b32 s17, s21, 16
-; GFX950-GISEL-NEXT:    s_and_b32 s4, s4, 0xffff
-; GFX950-GISEL-NEXT:    s_lshr_b32 s23, s6, 16
-; GFX950-GISEL-NEXT:    s_or_b32 s4, s17, s4
-; GFX950-GISEL-NEXT:    s_lshl_b32 s17, s22, 16
-; GFX950-GISEL-NEXT:    s_and_b32 s5, s5, 0xffff
-; GFX950-GISEL-NEXT:    s_lshr_b32 s24, s7, 16
-; GFX950-GISEL-NEXT:    s_or_b32 s5, s17, s5
-; GFX950-GISEL-NEXT:    s_lshl_b32 s17, s23, 16
-; GFX950-GISEL-NEXT:    s_and_b32 s6, s6, 0xffff
-; GFX950-GISEL-NEXT:    s_lshr_b32 s25, s8, 16
-; GFX950-GISEL-NEXT:    s_or_b32 s6, s17, s6
-; GFX950-GISEL-NEXT:    s_lshl_b32 s17, s24, 16
-; GFX950-GISEL-NEXT:    s_and_b32 s7, s7, 0xffff
-; GFX950-GISEL-NEXT:    s_lshr_b32 s26, s9, 16
-; GFX950-GISEL-NEXT:    s_or_b32 s7, s17, s7
-; GFX950-GISEL-NEXT:    s_lshl_b32 s17, s25, 16
-; GFX950-GISEL-NEXT:    s_and_b32 s8, s8, 0xffff
-; GFX950-GISEL-NEXT:    s_lshr_b32 s27, s10, 16
-; GFX950-GISEL-NEXT:    s_or_b32 s8, s17, s8
-; GFX950-GISEL-NEXT:    s_lshl_b32 s17, s26, 16
-; GFX950-GISEL-NEXT:    s_and_b32 s9, s9, 0xffff
-; GFX950-GISEL-NEXT:    s_lshr_b32 s28, s11, 16
-; GFX950-GISEL-NEXT:    s_or_b32 s9, s17, s9
-; GFX950-GISEL-NEXT:    s_lshl_b32 s17, s27, 16
-; GFX950-GISEL-NEXT:    s_and_b32 s10, s10, 0xffff
-; GFX950-GISEL-NEXT:    s_lshr_b32 s29, s12, 16
-; GFX950-GISEL-NEXT:    s_or_b32 s10, s17, s10
-; GFX950-GISEL-NEXT:    s_lshl_b32 s17, s28, 16
-; GFX950-GISEL-NEXT:    s_and_b32 s11, s11, 0xffff
-; GFX950-GISEL-NEXT:    s_lshr_b32 s30, s13, 16
-; GFX950-GISEL-NEXT:    s_or_b32 s11, s17, s11
-; GFX950-GISEL-NEXT:    s_lshl_b32 s17, s29, 16
-; GFX950-GISEL-NEXT:    s_and_b32 s12, s12, 0xffff
-; GFX950-GISEL-NEXT:    s_lshr_b32 s31, s14, 16
-; GFX950-GISEL-NEXT:    s_or_b32 s12, s17, s12
-; GFX950-GISEL-NEXT:    s_lshl_b32 s17, s30, 16
-; GFX950-GISEL-NEXT:    s_and_b32 s13, s13, 0xffff
-; GFX950-GISEL-NEXT:    s_lshr_b32 s33, s15, 16
-; GFX950-GISEL-NEXT:    s_or_b32 s13, s17, s13
-; GFX950-GISEL-NEXT:    s_lshl_b32 s17, s31, 16
-; GFX950-GISEL-NEXT:    s_and_b32 s14, s14, 0xffff
-; GFX950-GISEL-NEXT:    s_or_b32 s14, s17, s14
-; GFX950-GISEL-NEXT:    s_lshl_b32 s17, s33, 16
-; GFX950-GISEL-NEXT:    s_and_b32 s15, s15, 0xffff
-; GFX950-GISEL-NEXT:    s_or_b32 s15, s17, s15
-; GFX950-GISEL-NEXT:    v_mov_b64_e32 v[16:17], s[14:15]
-; GFX950-GISEL-NEXT:    v_mov_b64_e32 v[14:15], s[12:13]
-; GFX950-GISEL-NEXT:    v_mov_b64_e32 v[12:13], s[10:11]
-; GFX950-GISEL-NEXT:    v_mov_b64_e32 v[10:11], s[8:9]
-; GFX950-GISEL-NEXT:    v_mov_b64_e32 v[8:9], s[6:7]
-; GFX950-GISEL-NEXT:    v_mov_b64_e32 v[6:7], s[4:5]
-; GFX950-GISEL-NEXT:    v_mov_b64_e32 v[4:5], s[2:3]
-; GFX950-GISEL-NEXT:    v_mov_b64_e32 v[2:3], s[0:1]
+; GFX950-GISEL-NEXT:    v_mov_b32_e32 v2, s0
+; GFX950-GISEL-NEXT:    v_mov_b32_e32 v3, s1
+; GFX950-GISEL-NEXT:    v_mov_b32_e32 v4, s2
+; GFX950-GISEL-NEXT:    v_mov_b32_e32 v5, s3
+; GFX950-GISEL-NEXT:    v_mov_b32_e32 v6, s4
+; GFX950-GISEL-NEXT:    v_mov_b32_e32 v7, s5
+; GFX950-GISEL-NEXT:    v_mov_b32_e32 v8, s6
+; GFX950-GISEL-NEXT:    v_mov_b32_e32 v9, s7
+; GFX950-GISEL-NEXT:    v_mov_b32_e32 v10, s8
+; GFX950-GISEL-NEXT:    v_mov_b32_e32 v11, s9
+; GFX950-GISEL-NEXT:    v_mov_b32_e32 v12, s10
+; GFX950-GISEL-NEXT:    v_mov_b32_e32 v13, s11
+; GFX950-GISEL-NEXT:    v_mov_b32_e32 v14, s12
+; GFX950-GISEL-NEXT:    v_mov_b32_e32 v15, s13
+; GFX950-GISEL-NEXT:    v_mov_b32_e32 v16, s14
+; GFX950-GISEL-NEXT:    v_mov_b32_e32 v17, s15
 ; GFX950-GISEL-NEXT:    v_mov_b32_e32 v24, 0x42c80000
 ; GFX950-GISEL-NEXT:    v_cvt_scalef32_sr_pk32_fp6_bf16 v[18:23], v[2:17], s16, v24
-; GFX950-GISEL-NEXT:    global_store_dwordx4 v[0:1], v[18:21], off
 ; GFX950-GISEL-NEXT:    global_store_dwordx2 v[0:1], v[22:23], off offset:16
+; GFX950-GISEL-NEXT:    global_store_dwordx4 v[0:1], v[18:21], off
 ; GFX950-GISEL-NEXT:    s_endpgm
   %cvt = tail call <6 x i32> @llvm.amdgcn.cvt.scalef32.sr.pk32.fp6.bf16(<32 x bfloat> %src, i32 %sr, float 100.0)
   store <6 x i32> %cvt, ptr addrspace(1) %out, align 8
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.sr.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.sr.ll
index 4a58d6346fc57..64a15bc102759 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.sr.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.sr.ll
@@ -1,6 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 4
 ; RUN: llc -global-isel=0 -mtriple=amdgcn -mcpu=gfx950 -o - %s | FileCheck -check-prefix=GFX950 %s
-; RUN: llc -global-isel=1 -mtriple=amdgcn -mcpu=gfx950 -o - %s | FileCheck -check-prefix=GFX950 %s
+; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgcn -mcpu=gfx950 -o - %s | FileCheck -check-prefix=GFX950 %s
 
 declare <2 x half> @llvm.amdgcn.cvt.sr.f16.f32(<2 x half>, float, i32, i1)
 declare <2 x bfloat> @llvm.amdgcn.cvt.sr.bf16.f32(<2 x bfloat>, float, i32, i1)
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ds.bpermute.fi.b32.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ds.bpermute.fi.b32.ll
new file mode 100644
index 0000000000000..ea85055b65d12
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ds.bpermute.fi.b32.ll
@@ -0,0 +1,154 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc -global-isel=0 -mtriple=amdgcn -mcpu=gfx1200 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX12,GFX12-SDAG %s
+; RUN: llc -global-isel=1 -mtriple=amdgcn -mcpu=gfx1200 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX12,GFX12-GISEL %s
+
+declare i32 @llvm.amdgcn.ds.bpermute.fi.b32(i32, i32) #0
+
+define amdgpu_kernel void @ds_bpermute_fi_b32(ptr addrspace(1) %out, i32 %index, i32 %src) nounwind {
+; GFX12-LABEL: ds_bpermute_fi_b32:
+; GFX12:       ; %bb.0:
+; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX12-NEXT:    s_wait_kmcnt 0x0
+; GFX12-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-NEXT:    ds_bpermute_fi_b32 v0, v0, v1
+; GFX12-NEXT:    v_mov_b32_e32 v1, 0
+; GFX12-NEXT:    s_wait_dscnt 0x0
+; GFX12-NEXT:    global_store_b32 v1, v0, s[0:1]
+; GFX12-NEXT:    s_endpgm
+  %bpermute = call i32 @llvm.amdgcn.ds.bpermute.fi.b32(i32 %index, i32 %src) #0
+  store i32 %bpermute, ptr addrspace(1) %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @ds_bpermute_fi_b32_imm_offset(ptr addrspace(1) %out, i32 %base_index, i32 %src) nounwind {
+; GFX12-SDAG-LABEL: ds_bpermute_fi_b32_imm_offset:
+; GFX12-SDAG:       ; %bb.0:
+; GFX12-SDAG-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX12-SDAG-NEXT:    s_wait_kmcnt 0x0
+; GFX12-SDAG-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-SDAG-NEXT:    ds_bpermute_fi_b32 v0, v0, v1 offset:4
+; GFX12-SDAG-NEXT:    v_mov_b32_e32 v1, 0
+; GFX12-SDAG-NEXT:    s_wait_dscnt 0x0
+; GFX12-SDAG-NEXT:    global_store_b32 v1, v0, s[0:1]
+; GFX12-SDAG-NEXT:    s_endpgm
+;
+; GFX12-GISEL-LABEL: ds_bpermute_fi_b32_imm_offset:
+; GFX12-GISEL:       ; %bb.0:
+; GFX12-GISEL-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX12-GISEL-NEXT:    s_wait_kmcnt 0x0
+; GFX12-GISEL-NEXT:    s_add_co_i32 s2, s2, 4
+; GFX12-GISEL-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-GISEL-NEXT:    v_dual_mov_b32 v1, s3 :: v_dual_mov_b32 v0, s2
+; GFX12-GISEL-NEXT:    ds_bpermute_fi_b32 v0, v0, v1
+; GFX12-GISEL-NEXT:    v_mov_b32_e32 v1, 0
+; GFX12-GISEL-NEXT:    s_wait_dscnt 0x0
+; GFX12-GISEL-NEXT:    global_store_b32 v1, v0, s[0:1]
+; GFX12-GISEL-NEXT:    s_endpgm
+  %index = add i32 %base_index, 4
+  %bpermute = call i32 @llvm.amdgcn.ds.bpermute.fi.b32(i32 %index, i32 %src) #0
+  store i32 %bpermute, ptr addrspace(1) %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @ds_bpermute_fi_b32_imm_index(ptr addrspace(1) %out, i32 %base_index, i32 %src) nounwind {
+; GFX12-SDAG-LABEL: ds_bpermute_fi_b32_imm_index:
+; GFX12-SDAG:       ; %bb.0:
+; GFX12-SDAG-NEXT:    s_load_b32 s0, s[4:5], 0x30
+; GFX12-SDAG-NEXT:    s_wait_kmcnt 0x0
+; GFX12-SDAG-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s0
+; GFX12-SDAG-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
+; GFX12-SDAG-NEXT:    ds_bpermute_fi_b32 v1, v0, v1 offset:64
+; GFX12-SDAG-NEXT:    s_wait_dscnt 0x0
+; GFX12-SDAG-NEXT:    s_wait_kmcnt 0x0
+; GFX12-SDAG-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX12-SDAG-NEXT:    s_endpgm
+;
+; GFX12-GISEL-LABEL: ds_bpermute_fi_b32_imm_index:
+; GFX12-GISEL:       ; %bb.0:
+; GFX12-GISEL-NEXT:    s_load_b32 s0, s[4:5], 0x30
+; GFX12-GISEL-NEXT:    s_wait_kmcnt 0x0
+; GFX12-GISEL-NEXT:    v_dual_mov_b32 v0, 64 :: v_dual_mov_b32 v1, s0
+; GFX12-GISEL-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
+; GFX12-GISEL-NEXT:    ds_bpermute_fi_b32 v0, v0, v1
+; GFX12-GISEL-NEXT:    v_mov_b32_e32 v1, 0
+; GFX12-GISEL-NEXT:    s_wait_dscnt 0x0
+; GFX12-GISEL-NEXT:    s_wait_kmcnt 0x0
+; GFX12-GISEL-NEXT:    global_store_b32 v1, v0, s[0:1]
+; GFX12-GISEL-NEXT:    s_endpgm
+  %bpermute = call i32 @llvm.amdgcn.ds.bpermute.fi.b32(i32 64, i32 %src) #0
+  store i32 %bpermute, ptr addrspace(1) %out, align 4
+  ret void
+}
+
+define void @ds_bpermute_fi_b32_add_shl(ptr addrspace(1) %out, i32 %base_index, i32 %src) nounwind {
+; GFX12-SDAG-LABEL: ds_bpermute_fi_b32_add_shl:
+; GFX12-SDAG:       ; %bb.0:
+; GFX12-SDAG-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX12-SDAG-NEXT:    s_wait_expcnt 0x0
+; GFX12-SDAG-NEXT:    s_wait_samplecnt 0x0
+; GFX12-SDAG-NEXT:    s_wait_bvhcnt 0x0
+; GFX12-SDAG-NEXT:    s_wait_kmcnt 0x0
+; GFX12-SDAG-NEXT:    v_lshlrev_b32_e32 v2, 2, v2
+; GFX12-SDAG-NEXT:    ds_bpermute_fi_b32 v2, v2, v3 offset:4
+; GFX12-SDAG-NEXT:    s_wait_dscnt 0x0
+; GFX12-SDAG-NEXT:    global_store_b32 v[0:1], v2, off
+; GFX12-SDAG-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX12-GISEL-LABEL: ds_bpermute_fi_b32_add_shl:
+; GFX12-GISEL:       ; %bb.0:
+; GFX12-GISEL-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX12-GISEL-NEXT:    s_wait_expcnt 0x0
+; GFX12-GISEL-NEXT:    s_wait_samplecnt 0x0
+; GFX12-GISEL-NEXT:    s_wait_bvhcnt 0x0
+; GFX12-GISEL-NEXT:    s_wait_kmcnt 0x0
+; GFX12-GISEL-NEXT:    v_lshl_add_u32 v2, v2, 2, 4
+; GFX12-GISEL-NEXT:    ds_bpermute_fi_b32 v2, v2, v3
+; GFX12-GISEL-NEXT:    s_wait_dscnt 0x0
+; GFX12-GISEL-NEXT:    global_store_b32 v[0:1], v2, off
+; GFX12-GISEL-NEXT:    s_setpc_b64 s[30:31]
+  %index = add i32 %base_index, 1
+  %byte_index = shl i32 %index, 2
+  %bpermute = call i32 @llvm.amdgcn.ds.bpermute.fi.b32(i32 %byte_index, i32 %src) #0
+  store i32 %bpermute, ptr addrspace(1) %out, align 4
+  ret void
+}
+
+define void @ds_bpermute_fi_b32_or_shl(ptr addrspace(1) %out, i32 %base_index, i32 %src) nounwind {
+; GFX12-SDAG-LABEL: ds_bpermute_fi_b32_or_shl:
+; GFX12-SDAG:       ; %bb.0:
+; GFX12-SDAG-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX12-SDAG-NEXT:    s_wait_expcnt 0x0
+; GFX12-SDAG-NEXT:    s_wait_samplecnt 0x0
+; GFX12-SDAG-NEXT:    s_wait_bvhcnt 0x0
+; GFX12-SDAG-NEXT:    s_wait_kmcnt 0x0
+; GFX12-SDAG-NEXT:    v_and_b32_e32 v2, 62, v2
+; GFX12-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX12-SDAG-NEXT:    v_lshlrev_b32_e32 v2, 2, v2
+; GFX12-SDAG-NEXT:    ds_bpermute_fi_b32 v2, v2, v3 offset:4
+; GFX12-SDAG-NEXT:    s_wait_dscnt 0x0
+; GFX12-SDAG-NEXT:    global_store_b32 v[0:1], v2, off
+; GFX12-SDAG-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX12-GISEL-LABEL: ds_bpermute_fi_b32_or_shl:
+; GFX12-GISEL:       ; %bb.0:
+; GFX12-GISEL-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX12-GISEL-NEXT:    s_wait_expcnt 0x0
+; GFX12-GISEL-NEXT:    s_wait_samplecnt 0x0
+; GFX12-GISEL-NEXT:    s_wait_bvhcnt 0x0
+; GFX12-GISEL-NEXT:    s_wait_kmcnt 0x0
+; GFX12-GISEL-NEXT:    v_and_b32_e32 v2, 62, v2
+; GFX12-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX12-GISEL-NEXT:    v_lshl_or_b32 v2, v2, 2, 4
+; GFX12-GISEL-NEXT:    ds_bpermute_fi_b32 v2, v2, v3
+; GFX12-GISEL-NEXT:    s_wait_dscnt 0x0
+; GFX12-GISEL-NEXT:    global_store_b32 v[0:1], v2, off
+; GFX12-GISEL-NEXT:    s_setpc_b64 s[30:31]
+  %masked = and i32 %base_index, 62
+  %index = or i32 %masked, 1
+  %byte_index = shl i32 %index, 2
+  %bpermute = call i32 @llvm.amdgcn.ds.bpermute.fi.b32(i32 %byte_index, i32 %src) #0
+  store i32 %bpermute, ptr addrspace(1) %out, align 4
+  ret void
+}
+
+attributes #0 = { nounwind readnone convergent }
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ds.read.tr.gfx950.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ds.read.tr.gfx950.ll
index f694d55f83b68..eb5bded6d2610 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ds.read.tr.gfx950.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ds.read.tr.gfx950.ll
@@ -1,6 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 4
 ; RUN: llc -global-isel=0 -mtriple=amdgcn -mcpu=gfx950 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX950-SDAG %s
-; RUN: llc -global-isel=1 -mtriple=amdgcn -mcpu=gfx950 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX950-GISEL %s
+; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgcn -mcpu=gfx950 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX950-GISEL %s
 
 declare <2 x i32>    @llvm.amdgcn.ds.read.tr4.b64.v2i32.p3(ptr addrspace(3))
 declare <2 x i32>    @llvm.amdgcn.ds.read.tr8.b64.v2i32.p3(ptr addrspace(3))
@@ -146,11 +146,11 @@ define amdgpu_ps void @ds_read_b64_tr_b16_v4bf16(ptr addrspace(3) %addr, ptr add
 ;
 ; GFX950-GISEL-LABEL: ds_read_b64_tr_b16_v4bf16:
 ; GFX950-GISEL:       ; %bb.0: ; %entry
-; GFX950-GISEL-NEXT:    v_mov_b32_e32 v4, v1
+; GFX950-GISEL-NEXT:    v_mov_b32_e32 v3, v2
+; GFX950-GISEL-NEXT:    v_mov_b32_e32 v2, v1
 ; GFX950-GISEL-NEXT:    ds_read_b64_tr_b16 v[0:1], v0 offset:32
-; GFX950-GISEL-NEXT:    v_mov_b32_e32 v5, v2
 ; GFX950-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX950-GISEL-NEXT:    global_store_dwordx2 v[4:5], v[0:1], off
+; GFX950-GISEL-NEXT:    global_store_dwordx2 v[2:3], v[0:1], off
 ; GFX950-GISEL-NEXT:    s_endpgm
 entry:
   %gep = getelementptr i64, ptr addrspace(3) %addr, i32 4
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.fdot2.f32.bf16.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.fdot2.f32.bf16.ll
index 42acf089e8648..159592cab6a34 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.fdot2.f32.bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.fdot2.f32.bf16.ll
@@ -1,8 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mtriple=amdgcn -mcpu=gfx1100 -verify-machineinstrs < %s | FileCheck %s --check-prefixes=GFX11
-; RUN: llc -global-isel -mtriple=amdgcn -mcpu=gfx1100 -verify-machineinstrs < %s | FileCheck %s --check-prefixes=GFX11
+; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgcn -mcpu=gfx1100 -verify-machineinstrs < %s | FileCheck %s --check-prefixes=GFX11
 ; RUN: llc -mtriple=amdgcn -mcpu=gfx950 -verify-machineinstrs < %s | FileCheck %s --check-prefixes=GFX950
-; RUN: llc -global-isel -mtriple=amdgcn -mcpu=gfx950 -verify-machineinstrs < %s | FileCheck %s --check-prefixes=GFX950-ISEL
+; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgcn -mcpu=gfx950 -verify-machineinstrs < %s | FileCheck %s --check-prefixes=GFX950-ISEL
 
 declare float @llvm.amdgcn.fdot2.f32.bf16(<2 x bfloat> %a, <2 x bfloat> %b, float %c, i1 %clamp)
 
@@ -40,17 +40,17 @@ define amdgpu_kernel void @test_llvm_amdgcn_fdot2_f32_bf16_clamp(
 ; GFX950-ISEL-LABEL: test_llvm_amdgcn_fdot2_f32_bf16_clamp:
 ; GFX950-ISEL:       ; %bb.0: ; %entry
 ; GFX950-ISEL-NEXT:    s_load_dwordx8 s[8:15], s[4:5], 0x24
+; GFX950-ISEL-NEXT:    v_mov_b32_e32 v0, 0
 ; GFX950-ISEL-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX950-ISEL-NEXT:    s_load_dword s0, s[12:13], 0x0
 ; GFX950-ISEL-NEXT:    s_load_dword s1, s[14:15], 0x0
 ; GFX950-ISEL-NEXT:    s_load_dword s2, s[10:11], 0x0
 ; GFX950-ISEL-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX950-ISEL-NEXT:    v_mov_b32_e32 v0, s0
-; GFX950-ISEL-NEXT:    v_mov_b32_e32 v1, s1
-; GFX950-ISEL-NEXT:    v_dot2_f32_bf16 v0, s2, v0, v1 clamp
-; GFX950-ISEL-NEXT:    v_mov_b32_e32 v1, 0
-; GFX950-ISEL-NEXT:    s_nop 1
-; GFX950-ISEL-NEXT:    global_store_dword v1, v0, s[8:9]
+; GFX950-ISEL-NEXT:    v_mov_b32_e32 v1, s0
+; GFX950-ISEL-NEXT:    v_mov_b32_e32 v2, s1
+; GFX950-ISEL-NEXT:    v_dot2_f32_bf16 v1, s2, v1, v2 clamp
+; GFX950-ISEL-NEXT:    s_nop 2
+; GFX950-ISEL-NEXT:    global_store_dword v0, v1, s[8:9]
 ; GFX950-ISEL-NEXT:    s_endpgm
     ptr addrspace(1) %r,
     ptr addrspace(1) %a,
@@ -100,17 +100,17 @@ define amdgpu_kernel void @test_llvm_amdgcn_fdot2_f32_bf16_no_clamp(
 ; GFX950-ISEL-LABEL: test_llvm_amdgcn_fdot2_f32_bf16_no_clamp:
 ; GFX950-ISEL:       ; %bb.0: ; %entry
 ; GFX950-ISEL-NEXT:    s_load_dwordx8 s[8:15], s[4:5], 0x24
+; GFX950-ISEL-NEXT:    v_mov_b32_e32 v0, 0
 ; GFX950-ISEL-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX950-ISEL-NEXT:    s_load_dword s0, s[12:13], 0x0
 ; GFX950-ISEL-NEXT:    s_load_dword s1, s[14:15], 0x0
 ; GFX950-ISEL-NEXT:    s_load_dword s2, s[10:11], 0x0
 ; GFX950-ISEL-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX950-ISEL-NEXT:    v_mov_b32_e32 v0, s0
-; GFX950-ISEL-NEXT:    v_mov_b32_e32 v1, s1
-; GFX950-ISEL-NEXT:    v_dot2c_f32_bf16_e32 v1, s2, v0
-; GFX950-ISEL-NEXT:    v_mov_b32_e32 v0, 0
-; GFX950-ISEL-NEXT:    s_nop 1
-; GFX950-ISEL-NEXT:    global_store_dword v0, v1, s[8:9]
+; GFX950-ISEL-NEXT:    v_mov_b32_e32 v1, s0
+; GFX950-ISEL-NEXT:    v_mov_b32_e32 v2, s1
+; GFX950-ISEL-NEXT:    v_dot2c_f32_bf16_e32 v2, s2, v1
+; GFX950-ISEL-NEXT:    s_nop 2
+; GFX950-ISEL-NEXT:    global_store_dword v0, v2, s[8:9]
 ; GFX950-ISEL-NEXT:    s_endpgm
     ptr addrspace(1) %r,
     ptr addrspace(1) %a,
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.global.load.tr-w32.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.global.load.tr-w32.ll
index 8427b4e7f6f35..537aab9a3e9c5 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.global.load.tr-w32.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.global.load.tr-w32.ll
@@ -1,6 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -global-isel=0 -mtriple=amdgcn -mcpu=gfx1200 -verify-machineinstrs -mattr=+wavefrontsize32 < %s | FileCheck -check-prefix=GFX12 %s
-; RUN: llc -global-isel=1 -mtriple=amdgcn -mcpu=gfx1200 -verify-machineinstrs -mattr=+wavefrontsize32 < %s | FileCheck -check-prefix=GFX12 %s
+; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgcn -mcpu=gfx1200 -verify-machineinstrs -mattr=+wavefrontsize32 < %s | FileCheck -check-prefix=GFX12 %s
 
 declare <2 x i32> @llvm.amdgcn.global.load.tr.b64.v2i32.p1(ptr addrspace(1))
 declare <8 x i16> @llvm.amdgcn.global.load.tr.b128.v8i16.p1(ptr addrspace(1))
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.global.load.tr-w64.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.global.load.tr-w64.ll
index be4fa79951daf..4db256de1ce1b 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.global.load.tr-w64.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.global.load.tr-w64.ll
@@ -1,6 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -global-isel=0 -mtriple=amdgcn -mcpu=gfx1200 -verify-machineinstrs -mattr=+wavefrontsize64 < %s | FileCheck -check-prefix=GFX12 %s
-; RUN: llc -global-isel=1 -mtriple=amdgcn -mcpu=gfx1200 -verify-machineinstrs -mattr=+wavefrontsize64 < %s | FileCheck -check-prefix=GFX12 %s
+; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgcn -mcpu=gfx1200 -verify-machineinstrs -mattr=+wavefrontsize64 < %s | FileCheck -check-prefix=GFX12 %s
 
 declare i32 @llvm.amdgcn.global.load.tr.b64.i32.p1(ptr addrspace(1))
 declare <4 x i16> @llvm.amdgcn.global.load.tr.b128.v4i16.p1(ptr addrspace(1))
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.atomic.pk.add.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.atomic.pk.add.ll
index 12f9029392a43..7be0d9ca329aa 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.atomic.pk.add.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.atomic.pk.add.ll
@@ -1,6 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mtriple=amdgcn -global-isel=0 -mcpu=gfx1200 -verify-machineinstrs < %s | FileCheck -check-prefix=GFX12-SDAG %s
-; RUN: llc -mtriple=amdgcn -global-isel=1 -mcpu=gfx1200 -verify-machineinstrs < %s | FileCheck -check-prefix=GFX12-GISEL %s
+; RUN: llc -mtriple=amdgcn -global-isel=1 -global-isel-abort=2 -mcpu=gfx1200 -verify-machineinstrs < %s | FileCheck -check-prefix=GFX12-GISEL %s
 
 define amdgpu_ps float @atomic_pk_add_f16_1d_v2(<8 x i32> inreg %rsrc, <2 x half> %data, i32 %s) {
 ; GFX12-SDAG-LABEL: atomic_pk_add_f16_1d_v2:
@@ -156,16 +156,6 @@ define amdgpu_ps float @atomic_pk_add_bf16_1d_v4(<8 x i32> inreg %rsrc, <4 x bfl
 ;
 ; GFX12-GISEL-LABEL: atomic_pk_add_bf16_1d_v4:
 ; GFX12-GISEL:       ; %bb.0: ; %main_body
-; GFX12-GISEL-NEXT:    v_lshrrev_b32_e32 v3, 16, v0
-; GFX12-GISEL-NEXT:    v_lshrrev_b32_e32 v4, 16, v1
-; GFX12-GISEL-NEXT:    v_and_b32_e32 v0, 0xffff, v0
-; GFX12-GISEL-NEXT:    v_and_b32_e32 v1, 0xffff, v1
-; GFX12-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX12-GISEL-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
-; GFX12-GISEL-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
-; GFX12-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-GISEL-NEXT:    v_or_b32_e32 v0, v3, v0
-; GFX12-GISEL-NEXT:    v_or_b32_e32 v1, v4, v1
 ; GFX12-GISEL-NEXT:    image_atomic_pk_add_bf16 v[0:1], v2, s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_1D th:TH_ATOMIC_RETURN
 ; GFX12-GISEL-NEXT:    v_mov_b32_e32 v2, 0
 ; GFX12-GISEL-NEXT:    v_mov_b32_e32 v3, 0
@@ -190,16 +180,6 @@ define amdgpu_ps float @atomic_pk_add_bf16_1d_v4_noret(<8 x i32> inreg %rsrc, <4
 ;
 ; GFX12-GISEL-LABEL: atomic_pk_add_bf16_1d_v4_noret:
 ; GFX12-GISEL:       ; %bb.0: ; %main_body
-; GFX12-GISEL-NEXT:    v_lshrrev_b32_e32 v3, 16, v0
-; GFX12-GISEL-NEXT:    v_lshrrev_b32_e32 v4, 16, v1
-; GFX12-GISEL-NEXT:    v_and_b32_e32 v0, 0xffff, v0
-; GFX12-GISEL-NEXT:    v_and_b32_e32 v1, 0xffff, v1
-; GFX12-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX12-GISEL-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
-; GFX12-GISEL-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
-; GFX12-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-GISEL-NEXT:    v_or_b32_e32 v0, v3, v0
-; GFX12-GISEL-NEXT:    v_or_b32_e32 v1, v4, v1
 ; GFX12-GISEL-NEXT:    image_atomic_pk_add_bf16 v[0:1], v2, s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_1D th:TH_ATOMIC_RETURN
 ; GFX12-GISEL-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-GISEL-NEXT:    v_mov_b32_e32 v0, 1.0
@@ -219,16 +199,6 @@ define amdgpu_ps float @atomic_pk_add_bf16_1d_v4_nt(<8 x i32> inreg %rsrc, <4 x
 ;
 ; GFX12-GISEL-LABEL: atomic_pk_add_bf16_1d_v4_nt:
 ; GFX12-GISEL:       ; %bb.0: ; %main_body
-; GFX12-GISEL-NEXT:    v_lshrrev_b32_e32 v3, 16, v0
-; GFX12-GISEL-NEXT:    v_lshrrev_b32_e32 v4, 16, v1
-; GFX12-GISEL-NEXT:    v_and_b32_e32 v0, 0xffff, v0
-; GFX12-GISEL-NEXT:    v_and_b32_e32 v1, 0xffff, v1
-; GFX12-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX12-GISEL-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
-; GFX12-GISEL-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
-; GFX12-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-GISEL-NEXT:    v_or_b32_e32 v0, v3, v0
-; GFX12-GISEL-NEXT:    v_or_b32_e32 v1, v4, v1
 ; GFX12-GISEL-NEXT:    image_atomic_pk_add_bf16 v[0:1], v2, s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_1D th:TH_ATOMIC_NT_RETURN
 ; GFX12-GISEL-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-GISEL-NEXT:    v_mov_b32_e32 v0, 1.0
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.init.whole.wave-w32.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.init.whole.wave-w32.ll
index 353f4d90cad1f..1bdaa4c98127d 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.init.whole.wave-w32.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.init.whole.wave-w32.ll
@@ -26,7 +26,6 @@ define amdgpu_cs_chain void @basic(<3 x i32> inreg %sgpr, ptr inreg %callee, i32
 ; GISEL12-NEXT:    s_delay_alu instid0(VALU_DEP_2)
 ; GISEL12-NEXT:    v_add_nc_u32_e32 v11, 32, v12
 ; GISEL12-NEXT:    s_mov_b32 exec_lo, s5
-; GISEL12-NEXT:    s_wait_alu 0xfffe
 ; GISEL12-NEXT:    s_setpc_b64 s[6:7]
 ;
 ; DAGISEL12-LABEL: basic:
@@ -50,7 +49,6 @@ define amdgpu_cs_chain void @basic(<3 x i32> inreg %sgpr, ptr inreg %callee, i32
 ; DAGISEL12-NEXT:    s_delay_alu instid0(VALU_DEP_2)
 ; DAGISEL12-NEXT:    v_add_nc_u32_e32 v11, 32, v12
 ; DAGISEL12-NEXT:    s_mov_b32 exec_lo, s5
-; DAGISEL12-NEXT:    s_wait_alu 0xfffe
 ; DAGISEL12-NEXT:    s_setpc_b64 s[6:7]
 ;
 ; GISEL10-LABEL: basic:
@@ -123,8 +121,9 @@ define amdgpu_cs_chain void @wwm_in_shader(<3 x i32> inreg %sgpr, ptr inreg %cal
 ; GISEL12-NEXT:    s_or_saveexec_b32 s4, -1
 ; GISEL12-NEXT:    s_wait_alu 0xfffe
 ; GISEL12-NEXT:    v_cndmask_b32_e64 v0, 0x47, v10, s4
-; GISEL12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GISEL12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
 ; GISEL12-NEXT:    v_cmp_ne_u32_e64 s8, 0, v0
+; GISEL12-NEXT:    s_wait_alu 0xf1ff
 ; GISEL12-NEXT:    v_mov_b32_e32 v0, s8
 ; GISEL12-NEXT:    s_mov_b32 exec_lo, s4
 ; GISEL12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
@@ -132,7 +131,6 @@ define amdgpu_cs_chain void @wwm_in_shader(<3 x i32> inreg %sgpr, ptr inreg %cal
 ; GISEL12-NEXT:  ; %bb.2: ; %tail
 ; GISEL12-NEXT:    s_or_b32 exec_lo, exec_lo, s3
 ; GISEL12-NEXT:    s_mov_b32 exec_lo, s5
-; GISEL12-NEXT:    s_wait_alu 0xfffe
 ; GISEL12-NEXT:    s_setpc_b64 s[6:7]
 ;
 ; DAGISEL12-LABEL: wwm_in_shader:
@@ -159,7 +157,6 @@ define amdgpu_cs_chain void @wwm_in_shader(<3 x i32> inreg %sgpr, ptr inreg %cal
 ; DAGISEL12-NEXT:  ; %bb.2: ; %tail
 ; DAGISEL12-NEXT:    s_or_b32 exec_lo, exec_lo, s3
 ; DAGISEL12-NEXT:    s_mov_b32 exec_lo, s5
-; DAGISEL12-NEXT:    s_wait_alu 0xfffe
 ; DAGISEL12-NEXT:    s_setpc_b64 s[6:7]
 ;
 ; GISEL10-LABEL: wwm_in_shader:
@@ -244,8 +241,9 @@ define amdgpu_cs_chain void @phi_whole_struct(<3 x i32> inreg %sgpr, ptr inreg %
 ; GISEL12-NEXT:    s_or_saveexec_b32 s4, -1
 ; GISEL12-NEXT:    s_wait_alu 0xfffe
 ; GISEL12-NEXT:    v_cndmask_b32_e64 v0, 0x47, v12, s4
-; GISEL12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GISEL12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
 ; GISEL12-NEXT:    v_cmp_ne_u32_e64 s8, 0, v0
+; GISEL12-NEXT:    s_wait_alu 0xf1ff
 ; GISEL12-NEXT:    v_mov_b32_e32 v0, s8
 ; GISEL12-NEXT:    s_mov_b32 exec_lo, s4
 ; GISEL12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
@@ -253,7 +251,6 @@ define amdgpu_cs_chain void @phi_whole_struct(<3 x i32> inreg %sgpr, ptr inreg %
 ; GISEL12-NEXT:  ; %bb.2: ; %tail
 ; GISEL12-NEXT:    s_or_b32 exec_lo, exec_lo, s3
 ; GISEL12-NEXT:    s_mov_b32 exec_lo, s5
-; GISEL12-NEXT:    s_wait_alu 0xfffe
 ; GISEL12-NEXT:    s_setpc_b64 s[6:7]
 ;
 ; DAGISEL12-LABEL: phi_whole_struct:
@@ -279,7 +276,6 @@ define amdgpu_cs_chain void @phi_whole_struct(<3 x i32> inreg %sgpr, ptr inreg %
 ; DAGISEL12-NEXT:  ; %bb.2: ; %tail
 ; DAGISEL12-NEXT:    s_or_b32 exec_lo, exec_lo, s3
 ; DAGISEL12-NEXT:    s_mov_b32 exec_lo, s5
-; DAGISEL12-NEXT:    s_wait_alu 0xfffe
 ; DAGISEL12-NEXT:    s_setpc_b64 s[6:7]
 ;
 ; GISEL10-LABEL: phi_whole_struct:
@@ -367,8 +363,9 @@ define amdgpu_cs_chain void @control_flow(<3 x i32> inreg %sgpr, ptr inreg %call
 ; GISEL12-NEXT:    s_or_saveexec_b32 s8, -1
 ; GISEL12-NEXT:    s_wait_alu 0xfffe
 ; GISEL12-NEXT:    v_cndmask_b32_e64 v0, 0x47, v1, s8
-; GISEL12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GISEL12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
 ; GISEL12-NEXT:    v_cmp_ne_u32_e64 s9, 0, v0
+; GISEL12-NEXT:    s_wait_alu 0xf1ff
 ; GISEL12-NEXT:    v_mov_b32_e32 v0, s9
 ; GISEL12-NEXT:    s_mov_b32 exec_lo, s8
 ; GISEL12-NEXT:    v_cmp_eq_u32_e32 vcc_lo, v13, v1
@@ -406,7 +403,6 @@ define amdgpu_cs_chain void @control_flow(<3 x i32> inreg %sgpr, ptr inreg %call
 ; GISEL12-NEXT:  ; %bb.8: ; %tail.end
 ; GISEL12-NEXT:    s_or_b32 exec_lo, exec_lo, s3
 ; GISEL12-NEXT:    s_mov_b32 exec_lo, s5
-; GISEL12-NEXT:    s_wait_alu 0xfffe
 ; GISEL12-NEXT:    s_setpc_b64 s[6:7]
 ;
 ; DAGISEL12-LABEL: control_flow:
@@ -432,18 +428,17 @@ define amdgpu_cs_chain void @control_flow(<3 x i32> inreg %sgpr, ptr inreg %call
 ; DAGISEL12-NEXT:    s_or_saveexec_b32 s8, -1
 ; DAGISEL12-NEXT:    s_wait_alu 0xfffe
 ; DAGISEL12-NEXT:    v_cndmask_b32_e64 v0, 0x47, v1, s8
-; DAGISEL12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
+; DAGISEL12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; DAGISEL12-NEXT:    v_cmp_ne_u32_e64 s9, 0, v0
 ; DAGISEL12-NEXT:    s_mov_b32 exec_lo, s8
 ; DAGISEL12-NEXT:    v_cmp_eq_u32_e32 vcc_lo, v13, v1
-; DAGISEL12-NEXT:    v_mov_b32_e32 v11, s9
 ; DAGISEL12-NEXT:    s_or_b32 s4, vcc_lo, s4
 ; DAGISEL12-NEXT:    s_wait_alu 0xfffe
 ; DAGISEL12-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s4
 ; DAGISEL12-NEXT:    s_cbranch_execnz .LBB3_2
 ; DAGISEL12-NEXT:  ; %bb.3: ; %tail.loopexit
 ; DAGISEL12-NEXT:    s_or_b32 exec_lo, exec_lo, s4
-; DAGISEL12-NEXT:    v_add_nc_u32_e32 v10, 42, v1
+; DAGISEL12-NEXT:    v_dual_mov_b32 v11, s9 :: v_dual_add_nc_u32 v10, 42, v1
 ; DAGISEL12-NEXT:  .LBB3_4: ; %Flow1
 ; DAGISEL12-NEXT:    s_wait_alu 0xfffe
 ; DAGISEL12-NEXT:    s_or_b32 exec_lo, exec_lo, s3
@@ -465,7 +460,6 @@ define amdgpu_cs_chain void @control_flow(<3 x i32> inreg %sgpr, ptr inreg %call
 ; DAGISEL12-NEXT:    s_wait_alu 0xfffe
 ; DAGISEL12-NEXT:    s_or_b32 exec_lo, exec_lo, s3
 ; DAGISEL12-NEXT:    s_mov_b32 exec_lo, s5
-; DAGISEL12-NEXT:    s_wait_alu 0xfffe
 ; DAGISEL12-NEXT:    s_setpc_b64 s[6:7]
 ;
 ; GISEL10-LABEL: control_flow:
@@ -535,13 +529,13 @@ define amdgpu_cs_chain void @control_flow(<3 x i32> inreg %sgpr, ptr inreg %call
 ; DAGISEL10-NEXT:    v_cmp_ne_u32_e64 s9, 0, v0
 ; DAGISEL10-NEXT:    s_mov_b32 exec_lo, s8
 ; DAGISEL10-NEXT:    v_cmp_eq_u32_e32 vcc_lo, v13, v1
-; DAGISEL10-NEXT:    v_mov_b32_e32 v11, s9
 ; DAGISEL10-NEXT:    s_or_b32 s4, vcc_lo, s4
 ; DAGISEL10-NEXT:    s_andn2_b32 exec_lo, exec_lo, s4
 ; DAGISEL10-NEXT:    s_cbranch_execnz .LBB3_2
 ; DAGISEL10-NEXT:  ; %bb.3: ; %tail.loopexit
 ; DAGISEL10-NEXT:    s_or_b32 exec_lo, exec_lo, s4
 ; DAGISEL10-NEXT:    v_add_nc_u32_e32 v10, 42, v1
+; DAGISEL10-NEXT:    v_mov_b32_e32 v11, s9
 ; DAGISEL10-NEXT:  .LBB3_4: ; %Flow1
 ; DAGISEL10-NEXT:    s_or_b32 exec_lo, exec_lo, s3
 ; DAGISEL10-NEXT:    s_mov_b32 s3, exec_lo
@@ -619,8 +613,9 @@ define amdgpu_cs_chain void @use_v0_7(<3 x i32> inreg %sgpr, ptr inreg %callee,
 ; GISEL12-NEXT:    s_or_saveexec_b32 s4, -1
 ; GISEL12-NEXT:    s_wait_alu 0xfffe
 ; GISEL12-NEXT:    v_cndmask_b32_e64 v13, 0x47, v12, s4
-; GISEL12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GISEL12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
 ; GISEL12-NEXT:    v_cmp_ne_u32_e64 s8, 0, v13
+; GISEL12-NEXT:    s_wait_alu 0xf1ff
 ; GISEL12-NEXT:    v_mov_b32_e32 v13, s8
 ; GISEL12-NEXT:    s_mov_b32 exec_lo, s4
 ; GISEL12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
@@ -632,7 +627,6 @@ define amdgpu_cs_chain void @use_v0_7(<3 x i32> inreg %sgpr, ptr inreg %callee,
 ; GISEL12-NEXT:    s_wait_alu 0xfffe
 ; GISEL12-NEXT:    s_or_b32 exec_lo, exec_lo, s3
 ; GISEL12-NEXT:    s_mov_b32 exec_lo, s5
-; GISEL12-NEXT:    s_wait_alu 0xfffe
 ; GISEL12-NEXT:    s_setpc_b64 s[6:7]
 ;
 ; DAGISEL12-LABEL: use_v0_7:
@@ -663,7 +657,6 @@ define amdgpu_cs_chain void @use_v0_7(<3 x i32> inreg %sgpr, ptr inreg %callee,
 ; DAGISEL12-NEXT:    s_wait_alu 0xfffe
 ; DAGISEL12-NEXT:    s_or_b32 exec_lo, exec_lo, s3
 ; DAGISEL12-NEXT:    s_mov_b32 exec_lo, s5
-; DAGISEL12-NEXT:    s_wait_alu 0xfffe
 ; DAGISEL12-NEXT:    s_setpc_b64 s[6:7]
 ;
 ; GISEL10-LABEL: use_v0_7:
@@ -786,7 +779,6 @@ define amdgpu_cs_chain void @wwm_write_to_arg_reg(<3 x i32> inreg %sgpr, ptr inr
 ; GISEL12-NEXT:    v_dual_mov_b32 v12, v36 :: v_dual_mov_b32 v13, v37
 ; GISEL12-NEXT:    v_dual_mov_b32 v14, v38 :: v_dual_mov_b32 v15, v39
 ; GISEL12-NEXT:    s_wait_kmcnt 0x0
-; GISEL12-NEXT:    s_wait_alu 0xfffe
 ; GISEL12-NEXT:    s_swappc_b64 s[30:31], s[0:1]
 ; GISEL12-NEXT:    v_dual_mov_b32 v24, v0 :: v_dual_mov_b32 v25, v1
 ; GISEL12-NEXT:    v_dual_mov_b32 v26, v2 :: v_dual_mov_b32 v27, v3
@@ -861,7 +853,6 @@ define amdgpu_cs_chain void @wwm_write_to_arg_reg(<3 x i32> inreg %sgpr, ptr inr
 ; DAGISEL12-NEXT:    v_dual_mov_b32 v12, v36 :: v_dual_mov_b32 v13, v37
 ; DAGISEL12-NEXT:    v_dual_mov_b32 v14, v38 :: v_dual_mov_b32 v15, v39
 ; DAGISEL12-NEXT:    s_wait_kmcnt 0x0
-; DAGISEL12-NEXT:    s_wait_alu 0xfffe
 ; DAGISEL12-NEXT:    s_swappc_b64 s[30:31], s[0:1]
 ; DAGISEL12-NEXT:    v_dual_mov_b32 v40, v0 :: v_dual_mov_b32 v41, v1
 ; DAGISEL12-NEXT:    v_dual_mov_b32 v42, v2 :: v_dual_mov_b32 v43, v3
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.init.whole.wave-w64.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.init.whole.wave-w64.ll
index 1b1c89d9f5ad2..e0a5d397bded4 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.init.whole.wave-w64.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.init.whole.wave-w64.ll
@@ -26,8 +26,9 @@ define amdgpu_cs_chain void @basic(<3 x i32> inreg %sgpr, ptr inreg %callee, i64
 ; GISEL12-NEXT:    s_or_saveexec_b64 s[10:11], -1
 ; GISEL12-NEXT:    s_wait_alu 0xfffe
 ; GISEL12-NEXT:    v_cndmask_b32_e64 v0, 0x47, v13, s[10:11]
-; GISEL12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GISEL12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
 ; GISEL12-NEXT:    v_cmp_ne_u32_e64 s[12:13], 0, v0
+; GISEL12-NEXT:    s_wait_alu 0xf1ff
 ; GISEL12-NEXT:    v_mov_b32_e32 v0, s12
 ; GISEL12-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GISEL12-NEXT:    v_mov_b32_e32 v1, s13
@@ -40,7 +41,6 @@ define amdgpu_cs_chain void @basic(<3 x i32> inreg %sgpr, ptr inreg %callee, i64
 ; GISEL12-NEXT:    s_wait_alu 0xfffe
 ; GISEL12-NEXT:    s_or_b64 exec, exec, s[6:7]
 ; GISEL12-NEXT:    s_mov_b64 exec, s[4:5]
-; GISEL12-NEXT:    s_wait_alu 0xfffe
 ; GISEL12-NEXT:    s_setpc_b64 s[8:9]
 ;
 ; DAGISEL12-LABEL: basic:
@@ -71,7 +71,6 @@ define amdgpu_cs_chain void @basic(<3 x i32> inreg %sgpr, ptr inreg %callee, i64
 ; DAGISEL12-NEXT:  ; %bb.2: ; %tail
 ; DAGISEL12-NEXT:    s_or_b64 exec, exec, s[8:9]
 ; DAGISEL12-NEXT:    s_mov_b64 exec, s[6:7]
-; DAGISEL12-NEXT:    s_wait_alu 0xfffe
 ; DAGISEL12-NEXT:    s_setpc_b64 s[4:5]
 ;
 ; GISEL10-LABEL: basic:
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.intersect_ray.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.intersect_ray.ll
index e592a4ac5e8fa..8af5db9f62908 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.intersect_ray.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.intersect_ray.ll
@@ -352,9 +352,10 @@ define amdgpu_kernel void @image_bvh_intersect_ray_nsa_reassign(ptr %p_node_ptr,
 ; GFX12-SDAG-NEXT:    v_mov_b32_e32 v4, 4.0
 ; GFX12-SDAG-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-SDAG-NEXT:    v_add_co_u32 v0, s0, s0, v2
-; GFX12-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-SDAG-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-SDAG-NEXT:    v_add_co_ci_u32_e64 v1, null, s1, 0, s0
 ; GFX12-SDAG-NEXT:    v_add_co_u32 v2, s0, s2, v2
+; GFX12-SDAG-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-SDAG-NEXT:    v_add_co_ci_u32_e64 v3, null, s3, 0, s0
 ; GFX12-SDAG-NEXT:    flat_load_b32 v9, v[0:1]
 ; GFX12-SDAG-NEXT:    flat_load_b32 v10, v[2:3]
@@ -391,6 +392,7 @@ define amdgpu_kernel void @image_bvh_intersect_ray_nsa_reassign(ptr %p_node_ptr,
 ; GFX12-GISEL-NEXT:    v_add_co_u32 v0, vcc_lo, v0, v4
 ; GFX12-GISEL-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
 ; GFX12-GISEL-NEXT:    v_add_co_u32 v2, vcc_lo, v2, v4
+; GFX12-GISEL-NEXT:    s_wait_alu 0xfffd
 ; GFX12-GISEL-NEXT:    v_add_co_ci_u32_e32 v3, vcc_lo, 0, v3, vcc_lo
 ; GFX12-GISEL-NEXT:    flat_load_b32 v9, v[0:1]
 ; GFX12-GISEL-NEXT:    flat_load_b32 v10, v[2:3]
@@ -503,9 +505,10 @@ define amdgpu_kernel void @image_bvh_intersect_ray_a16_nsa_reassign(ptr %p_node_
 ; GFX12-SDAG-NEXT:    v_lshlrev_b32_e32 v2, 2, v0
 ; GFX12-SDAG-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-SDAG-NEXT:    v_add_co_u32 v0, s0, s0, v2
-; GFX12-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-SDAG-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-SDAG-NEXT:    v_add_co_ci_u32_e64 v1, null, s1, 0, s0
 ; GFX12-SDAG-NEXT:    v_add_co_u32 v2, s0, s2, v2
+; GFX12-SDAG-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-SDAG-NEXT:    v_add_co_ci_u32_e64 v3, null, s3, 0, s0
 ; GFX12-SDAG-NEXT:    flat_load_b32 v6, v[0:1]
 ; GFX12-SDAG-NEXT:    flat_load_b32 v7, v[2:3]
@@ -537,6 +540,7 @@ define amdgpu_kernel void @image_bvh_intersect_ray_a16_nsa_reassign(ptr %p_node_
 ; GFX12-GISEL-NEXT:    v_add_co_u32 v0, vcc_lo, v0, v4
 ; GFX12-GISEL-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
 ; GFX12-GISEL-NEXT:    v_add_co_u32 v2, vcc_lo, v2, v4
+; GFX12-GISEL-NEXT:    s_wait_alu 0xfffd
 ; GFX12-GISEL-NEXT:    v_add_co_ci_u32_e32 v3, vcc_lo, 0, v3, vcc_lo
 ; GFX12-GISEL-NEXT:    flat_load_b32 v6, v[0:1]
 ; GFX12-GISEL-NEXT:    flat_load_b32 v7, v[2:3]
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.gfx950.bf16.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.gfx950.bf16.ll
index 8d380516df8b5..452033f332659 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.gfx950.bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.gfx950.bf16.ll
@@ -49,7 +49,7 @@ define amdgpu_kernel void @test_mfma_f32_32x32x16_bf16(<8 x bfloat> %arg0, <8 x
 ; GCN-NEXT:    v_mov_b32_e32 v9, s17
 ; GCN-NEXT:    v_mov_b32_e32 v10, s18
 ; GCN-NEXT:    v_mov_b32_e32 v11, s19
-; GCN-NEXT:    s_nop 3
+; GCN-NEXT:    s_nop 4
 ; GCN-NEXT:    global_store_dwordx4 v[12:13], a[28:31], off sc0 sc1
 ; GCN-NEXT:    s_waitcnt vmcnt(0)
 ; GCN-NEXT:    global_store_dwordx4 v[14:15], a[24:27], off sc0 sc1
@@ -122,7 +122,7 @@ define amdgpu_kernel void @test_mfma_f32_32x32x16_bf16__flags(<8 x bfloat> %arg0
 ; GCN-NEXT:    v_mov_b32_e32 v9, s17
 ; GCN-NEXT:    v_mov_b32_e32 v10, s18
 ; GCN-NEXT:    v_mov_b32_e32 v11, s19
-; GCN-NEXT:    s_nop 3
+; GCN-NEXT:    s_nop 4
 ; GCN-NEXT:    global_store_dwordx4 v[12:13], a[28:31], off sc0 sc1
 ; GCN-NEXT:    s_waitcnt vmcnt(0)
 ; GCN-NEXT:    global_store_dwordx4 v[14:15], a[24:27], off sc0 sc1
@@ -179,7 +179,7 @@ define <16 x float> @test_mfma_f32_32x32x16_bf16__mac(<8 x bfloat> %arg0, <8 x b
 ; GCN-NEXT:    s_nop 1
 ; GCN-NEXT:    v_mfma_f32_32x32x16_bf16 a[0:15], v[0:3], v[4:7], a[0:15]
 ; GCN-NEXT:    s_nop 7
-; GCN-NEXT:    s_nop 2
+; GCN-NEXT:    s_nop 3
 ; GCN-NEXT:    v_accvgpr_read_b32 v0, a0
 ; GCN-NEXT:    v_accvgpr_read_b32 v1, a1
 ; GCN-NEXT:    v_accvgpr_read_b32 v2, a2
@@ -224,7 +224,7 @@ define <16 x float> @test_mfma_f32_32x32x16_bf16__mac__flags(<8 x bfloat> %arg0,
 ; GCN-NEXT:    s_nop 1
 ; GCN-NEXT:    v_mfma_f32_32x32x16_bf16 a[0:15], v[0:3], v[4:7], a[0:15] cbsz:1 abid:1 blgp:1
 ; GCN-NEXT:    s_nop 7
-; GCN-NEXT:    s_nop 2
+; GCN-NEXT:    s_nop 3
 ; GCN-NEXT:    v_accvgpr_read_b32 v0, a0
 ; GCN-NEXT:    v_accvgpr_read_b32 v1, a1
 ; GCN-NEXT:    v_accvgpr_read_b32 v2, a2
@@ -417,7 +417,7 @@ define amdgpu_kernel void @test_mfma_f32_32x32x16_bf16__vgprcd_mac(<8 x bfloat>
 ; GCN-NEXT:    v_mfma_f32_32x32x16_bf16 a[0:15], v[0:3], v[4:7], a[0:15]
 ; GCN-NEXT:    v_mov_b32_e32 v0, 0
 ; GCN-NEXT:    s_nop 7
-; GCN-NEXT:    s_nop 1
+; GCN-NEXT:    s_nop 2
 ; GCN-NEXT:    global_store_dwordx4 v0, a[12:15], s[0:1] offset:48
 ; GCN-NEXT:    global_store_dwordx4 v0, a[8:11], s[0:1] offset:32
 ; GCN-NEXT:    global_store_dwordx4 v0, a[4:7], s[0:1] offset:16
@@ -459,7 +459,7 @@ define amdgpu_kernel void @test_mfma_f32_32x32x16_bf16__vgprcd_mac_flags(<8 x bf
 ; GCN-NEXT:    v_mfma_f32_32x32x16_bf16 a[0:15], v[0:3], v[4:7], a[0:15] cbsz:3 abid:2 blgp:1
 ; GCN-NEXT:    v_mov_b32_e32 v0, 0
 ; GCN-NEXT:    s_nop 7
-; GCN-NEXT:    s_nop 1
+; GCN-NEXT:    s_nop 2
 ; GCN-NEXT:    global_store_dwordx4 v0, a[12:15], s[0:1] offset:48
 ; GCN-NEXT:    global_store_dwordx4 v0, a[8:11], s[0:1] offset:32
 ; GCN-NEXT:    global_store_dwordx4 v0, a[4:7], s[0:1] offset:16
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.gfx950.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.gfx950.ll
index 722c53a9dd607..4628a9c15391b 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.gfx950.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.gfx950.ll
@@ -1,6 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 4
 ; RUN: llc -mtriple=amdgcn -mcpu=gfx950 -global-isel=0 < %s | FileCheck -enable-var-scope --check-prefixes=GCN,SDAG %s
-; RUN: llc -mtriple=amdgcn -mcpu=gfx950 -global-isel=1 < %s | FileCheck -enable-var-scope --check-prefixes=GCN,GISEL %s
+; RUN: llc -mtriple=amdgcn -mcpu=gfx950 -global-isel=1 -global-isel-abort=2 -verify-machineinstrs < %s | FileCheck -enable-var-scope --check-prefixes=GCN,GISEL %s
 
 declare <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.f16(<8 x half>, <8 x half>, <4 x float>, i32 immarg, i32 immarg, i32 immarg)
 declare <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.f16(<8 x half>, <8 x half>, <16 x float>, i32 immarg, i32 immarg, i32 immarg)
@@ -19,7 +19,7 @@ define <4 x float> @test_mfma_f32_16x16x32_f16(<8 x half> %arg0, <8 x half> %arg
 ; GCN-NEXT:    v_accvgpr_write_b32 a3, v11
 ; GCN-NEXT:    s_nop 1
 ; GCN-NEXT:    v_mfma_f32_16x16x32_f16 a[0:3], v[0:3], v[4:7], a[0:3]
-; GCN-NEXT:    s_nop 6
+; GCN-NEXT:    s_nop 7
 ; GCN-NEXT:    v_accvgpr_read_b32 v0, a0
 ; GCN-NEXT:    v_accvgpr_read_b32 v1, a1
 ; GCN-NEXT:    v_accvgpr_read_b32 v2, a2
@@ -39,7 +39,7 @@ define <4 x float> @test_mfma_f32_16x16x32_f16__flags(<8 x half> %arg0, <8 x hal
 ; GCN-NEXT:    v_accvgpr_write_b32 a3, v11
 ; GCN-NEXT:    s_nop 1
 ; GCN-NEXT:    v_mfma_f32_16x16x32_f16 a[0:3], v[0:3], v[4:7], a[0:3] cbsz:1 abid:1 blgp:1
-; GCN-NEXT:    s_nop 6
+; GCN-NEXT:    s_nop 7
 ; GCN-NEXT:    v_accvgpr_read_b32 v0, a0
 ; GCN-NEXT:    v_accvgpr_read_b32 v1, a1
 ; GCN-NEXT:    v_accvgpr_read_b32 v2, a2
@@ -67,7 +67,7 @@ define amdgpu_kernel void @test_mfma_f32_16x16x32_f16_no_agpr__vgprcd(ptr addrsp
 ; SDAG-NEXT:    v_accvgpr_write_b32 a3, s3
 ; SDAG-NEXT:    s_nop 1
 ; SDAG-NEXT:    v_mfma_f32_16x16x32_f16 a[0:3], v[0:3], v[4:7], a[0:3]
-; SDAG-NEXT:    s_nop 6
+; SDAG-NEXT:    s_nop 7
 ; SDAG-NEXT:    global_store_dwordx4 v8, a[0:3], s[6:7]
 ; SDAG-NEXT:    s_endpgm
 ;
@@ -88,7 +88,7 @@ define amdgpu_kernel void @test_mfma_f32_16x16x32_f16_no_agpr__vgprcd(ptr addrsp
 ; GISEL-NEXT:    s_nop 1
 ; GISEL-NEXT:    v_mfma_f32_16x16x32_f16 a[0:3], v[0:3], v[4:7], a[0:3]
 ; GISEL-NEXT:    v_mov_b32_e32 v0, 0
-; GISEL-NEXT:    s_nop 5
+; GISEL-NEXT:    s_nop 6
 ; GISEL-NEXT:    global_store_dwordx4 v0, a[0:3], s[6:7]
 ; GISEL-NEXT:    s_endpgm
   %result = call <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.f16(<8 x half> %arg0, <8 x half> %arg1, <4 x float> %arg2, i32 0, i32 0, i32 0)
@@ -114,7 +114,7 @@ define amdgpu_kernel void @test_mfma_f32_16x16x32_f16_no_agpr__vgprcd__flags(ptr
 ; SDAG-NEXT:    v_accvgpr_write_b32 a3, s3
 ; SDAG-NEXT:    s_nop 1
 ; SDAG-NEXT:    v_mfma_f32_16x16x32_f16 a[0:3], v[0:3], v[4:7], a[0:3] cbsz:3 abid:2 blgp:1
-; SDAG-NEXT:    s_nop 6
+; SDAG-NEXT:    s_nop 7
 ; SDAG-NEXT:    global_store_dwordx4 v8, a[0:3], s[6:7]
 ; SDAG-NEXT:    s_endpgm
 ;
@@ -135,7 +135,7 @@ define amdgpu_kernel void @test_mfma_f32_16x16x32_f16_no_agpr__vgprcd__flags(ptr
 ; GISEL-NEXT:    s_nop 1
 ; GISEL-NEXT:    v_mfma_f32_16x16x32_f16 a[0:3], v[0:3], v[4:7], a[0:3] cbsz:3 abid:2 blgp:1
 ; GISEL-NEXT:    v_mov_b32_e32 v0, 0
-; GISEL-NEXT:    s_nop 5
+; GISEL-NEXT:    s_nop 6
 ; GISEL-NEXT:    global_store_dwordx4 v0, a[0:3], s[6:7]
 ; GISEL-NEXT:    s_endpgm
   %result = call <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.f16(<8 x half> %arg0, <8 x half> %arg1, <4 x float> %arg2, i32 3, i32 2, i32 1)
@@ -186,7 +186,7 @@ define amdgpu_kernel void @test_mfma_f32_32x32x16_f16(<8 x half> %arg0, <8 x hal
 ; SDAG-NEXT:    v_mov_b32_e32 v9, s17
 ; SDAG-NEXT:    v_mov_b32_e32 v10, s18
 ; SDAG-NEXT:    v_mov_b32_e32 v11, s19
-; SDAG-NEXT:    s_nop 3
+; SDAG-NEXT:    s_nop 4
 ; SDAG-NEXT:    global_store_dwordx4 v[12:13], a[28:31], off sc0 sc1
 ; SDAG-NEXT:    s_waitcnt vmcnt(0)
 ; SDAG-NEXT:    global_store_dwordx4 v[14:15], a[24:27], off sc0 sc1
@@ -253,7 +253,7 @@ define amdgpu_kernel void @test_mfma_f32_32x32x16_f16(<8 x half> %arg0, <8 x hal
 ; GISEL-NEXT:    v_mov_b64_e32 v[10:11], s[10:11]
 ; GISEL-NEXT:    v_mov_b64_e32 v[14:15], s[14:15]
 ; GISEL-NEXT:    v_mov_b64_e32 v[18:19], s[18:19]
-; GISEL-NEXT:    s_nop 3
+; GISEL-NEXT:    s_nop 4
 ; GISEL-NEXT:    global_store_dwordx4 v[20:21], a[16:19], off sc0 sc1
 ; GISEL-NEXT:    s_waitcnt vmcnt(0)
 ; GISEL-NEXT:    global_store_dwordx4 v[22:23], a[20:23], off sc0 sc1
@@ -316,7 +316,7 @@ define amdgpu_kernel void @test_mfma_f32_32x32x16_f16__flags(<8 x half> %arg0, <
 ; SDAG-NEXT:    v_mov_b32_e32 v9, s17
 ; SDAG-NEXT:    v_mov_b32_e32 v10, s18
 ; SDAG-NEXT:    v_mov_b32_e32 v11, s19
-; SDAG-NEXT:    s_nop 3
+; SDAG-NEXT:    s_nop 4
 ; SDAG-NEXT:    global_store_dwordx4 v[12:13], a[28:31], off sc0 sc1
 ; SDAG-NEXT:    s_waitcnt vmcnt(0)
 ; SDAG-NEXT:    global_store_dwordx4 v[14:15], a[24:27], off sc0 sc1
@@ -383,7 +383,7 @@ define amdgpu_kernel void @test_mfma_f32_32x32x16_f16__flags(<8 x half> %arg0, <
 ; GISEL-NEXT:    v_mov_b64_e32 v[10:11], s[10:11]
 ; GISEL-NEXT:    v_mov_b64_e32 v[14:15], s[14:15]
 ; GISEL-NEXT:    v_mov_b64_e32 v[18:19], s[18:19]
-; GISEL-NEXT:    s_nop 3
+; GISEL-NEXT:    s_nop 4
 ; GISEL-NEXT:    global_store_dwordx4 v[20:21], a[16:19], off sc0 sc1
 ; GISEL-NEXT:    s_waitcnt vmcnt(0)
 ; GISEL-NEXT:    global_store_dwordx4 v[22:23], a[20:23], off sc0 sc1
@@ -430,7 +430,7 @@ define <16 x float> @test_mfma_f32_32x32x16_f16__mac(<8 x half> %arg0, <8 x half
 ; GCN-NEXT:    s_nop 1
 ; GCN-NEXT:    v_mfma_f32_32x32x16_f16 a[0:15], v[0:3], v[4:7], a[0:15]
 ; GCN-NEXT:    s_nop 7
-; GCN-NEXT:    s_nop 2
+; GCN-NEXT:    s_nop 3
 ; GCN-NEXT:    v_accvgpr_read_b32 v0, a0
 ; GCN-NEXT:    v_accvgpr_read_b32 v1, a1
 ; GCN-NEXT:    v_accvgpr_read_b32 v2, a2
@@ -475,7 +475,7 @@ define <16 x float> @test_mfma_f32_32x32x16_f16__mac__flags(<8 x half> %arg0, <8
 ; GCN-NEXT:    s_nop 1
 ; GCN-NEXT:    v_mfma_f32_32x32x16_f16 a[0:15], v[0:3], v[4:7], a[0:15] cbsz:1 abid:1 blgp:1
 ; GCN-NEXT:    s_nop 7
-; GCN-NEXT:    s_nop 2
+; GCN-NEXT:    s_nop 3
 ; GCN-NEXT:    v_accvgpr_read_b32 v0, a0
 ; GCN-NEXT:    v_accvgpr_read_b32 v1, a1
 ; GCN-NEXT:    v_accvgpr_read_b32 v2, a2
@@ -776,7 +776,7 @@ define amdgpu_kernel void @test_mfma_f32_32x32x16_f16__vgprcd_mac(<8 x half> %ar
 ; SDAG-NEXT:    v_mfma_f32_32x32x16_f16 a[0:15], v[0:3], v[4:7], a[0:15]
 ; SDAG-NEXT:    v_mov_b32_e32 v0, 0
 ; SDAG-NEXT:    s_nop 7
-; SDAG-NEXT:    s_nop 1
+; SDAG-NEXT:    s_nop 2
 ; SDAG-NEXT:    global_store_dwordx4 v0, a[12:15], s[0:1] offset:48
 ; SDAG-NEXT:    global_store_dwordx4 v0, a[8:11], s[0:1] offset:32
 ; SDAG-NEXT:    global_store_dwordx4 v0, a[4:7], s[0:1] offset:16
@@ -813,7 +813,7 @@ define amdgpu_kernel void @test_mfma_f32_32x32x16_f16__vgprcd_mac(<8 x half> %ar
 ; GISEL-NEXT:    v_mfma_f32_32x32x16_f16 a[0:15], v[0:3], v[4:7], a[0:15]
 ; GISEL-NEXT:    v_mov_b32_e32 v0, 0
 ; GISEL-NEXT:    s_nop 7
-; GISEL-NEXT:    s_nop 1
+; GISEL-NEXT:    s_nop 2
 ; GISEL-NEXT:    global_store_dwordx4 v0, a[0:3], s[0:1]
 ; GISEL-NEXT:    global_store_dwordx4 v0, a[4:7], s[0:1] offset:16
 ; GISEL-NEXT:    global_store_dwordx4 v0, a[8:11], s[0:1] offset:32
@@ -855,7 +855,7 @@ define amdgpu_kernel void @test_mfma_f32_32x32x16_f16__vgprcd_mac_flags(<8 x hal
 ; SDAG-NEXT:    v_mfma_f32_32x32x16_f16 a[0:15], v[0:3], v[4:7], a[0:15] cbsz:3 abid:2 blgp:1
 ; SDAG-NEXT:    v_mov_b32_e32 v0, 0
 ; SDAG-NEXT:    s_nop 7
-; SDAG-NEXT:    s_nop 1
+; SDAG-NEXT:    s_nop 2
 ; SDAG-NEXT:    global_store_dwordx4 v0, a[12:15], s[0:1] offset:48
 ; SDAG-NEXT:    global_store_dwordx4 v0, a[8:11], s[0:1] offset:32
 ; SDAG-NEXT:    global_store_dwordx4 v0, a[4:7], s[0:1] offset:16
@@ -892,7 +892,7 @@ define amdgpu_kernel void @test_mfma_f32_32x32x16_f16__vgprcd_mac_flags(<8 x hal
 ; GISEL-NEXT:    v_mfma_f32_32x32x16_f16 a[0:15], v[0:3], v[4:7], a[0:15] cbsz:3 abid:2 blgp:1
 ; GISEL-NEXT:    v_mov_b32_e32 v0, 0
 ; GISEL-NEXT:    s_nop 7
-; GISEL-NEXT:    s_nop 1
+; GISEL-NEXT:    s_nop 2
 ; GISEL-NEXT:    global_store_dwordx4 v0, a[0:3], s[0:1]
 ; GISEL-NEXT:    global_store_dwordx4 v0, a[4:7], s[0:1] offset:16
 ; GISEL-NEXT:    global_store_dwordx4 v0, a[8:11], s[0:1] offset:32
@@ -919,7 +919,7 @@ define <4 x i32> @test_mfma_i32_16x16x64_i8(<4 x i32> %arg0, <4 x i32> %arg1, <4
 ; GCN-NEXT:    v_accvgpr_write_b32 a3, v11
 ; GCN-NEXT:    s_nop 1
 ; GCN-NEXT:    v_mfma_i32_16x16x64_i8 a[0:3], v[0:3], v[4:7], a[0:3]
-; GCN-NEXT:    s_nop 6
+; GCN-NEXT:    s_nop 7
 ; GCN-NEXT:    v_accvgpr_read_b32 v0, a0
 ; GCN-NEXT:    v_accvgpr_read_b32 v1, a1
 ; GCN-NEXT:    v_accvgpr_read_b32 v2, a2
@@ -939,7 +939,7 @@ define <4 x i32> @test_mfma_i32_16x16x64_i8__flags(<4 x i32> %arg0, <4 x i32> %a
 ; GCN-NEXT:    v_accvgpr_write_b32 a3, v11
 ; GCN-NEXT:    s_nop 1
 ; GCN-NEXT:    v_mfma_i32_16x16x64_i8 a[0:3], v[0:3], v[4:7], a[0:3] cbsz:1 abid:1 blgp:1
-; GCN-NEXT:    s_nop 6
+; GCN-NEXT:    s_nop 7
 ; GCN-NEXT:    v_accvgpr_read_b32 v0, a0
 ; GCN-NEXT:    v_accvgpr_read_b32 v1, a1
 ; GCN-NEXT:    v_accvgpr_read_b32 v2, a2
@@ -971,7 +971,7 @@ define amdgpu_kernel void @test_mfma_i32_16x16x64_i8_no_agpr__vgprcd(ptr addrspa
 ; SDAG-NEXT:    v_accvgpr_write_b32 a3, s3
 ; SDAG-NEXT:    s_nop 1
 ; SDAG-NEXT:    v_mfma_i32_16x16x64_i8 a[0:3], v[0:3], v[4:7], a[0:3]
-; SDAG-NEXT:    s_nop 6
+; SDAG-NEXT:    s_nop 7
 ; SDAG-NEXT:    global_store_dwordx4 v8, a[0:3], s[6:7]
 ; SDAG-NEXT:    s_endpgm
 ;
@@ -992,7 +992,7 @@ define amdgpu_kernel void @test_mfma_i32_16x16x64_i8_no_agpr__vgprcd(ptr addrspa
 ; GISEL-NEXT:    s_nop 1
 ; GISEL-NEXT:    v_mfma_i32_16x16x64_i8 a[0:3], v[0:3], v[4:7], a[0:3]
 ; GISEL-NEXT:    v_mov_b32_e32 v0, 0
-; GISEL-NEXT:    s_nop 5
+; GISEL-NEXT:    s_nop 6
 ; GISEL-NEXT:    global_store_dwordx4 v0, a[0:3], s[6:7]
 ; GISEL-NEXT:    s_endpgm
   %result = call <4 x i32> @llvm.amdgcn.mfma.i32.16x16x64.i8(<4 x i32> %arg0, <4 x i32> %arg1, <4 x i32> %arg2, i32 0, i32 0, i32 0)
@@ -1022,7 +1022,7 @@ define amdgpu_kernel void @test_mfma_i32_16x16x64_i8_no_agpr__vgprcd__flags(ptr
 ; SDAG-NEXT:    v_accvgpr_write_b32 a3, s3
 ; SDAG-NEXT:    s_nop 1
 ; SDAG-NEXT:    v_mfma_i32_16x16x64_i8 a[0:3], v[0:3], v[4:7], a[0:3] cbsz:3 abid:2 blgp:1
-; SDAG-NEXT:    s_nop 6
+; SDAG-NEXT:    s_nop 7
 ; SDAG-NEXT:    global_store_dwordx4 v8, a[0:3], s[6:7]
 ; SDAG-NEXT:    s_endpgm
 ;
@@ -1043,7 +1043,7 @@ define amdgpu_kernel void @test_mfma_i32_16x16x64_i8_no_agpr__vgprcd__flags(ptr
 ; GISEL-NEXT:    s_nop 1
 ; GISEL-NEXT:    v_mfma_i32_16x16x64_i8 a[0:3], v[0:3], v[4:7], a[0:3] cbsz:3 abid:2 blgp:1
 ; GISEL-NEXT:    v_mov_b32_e32 v0, 0
-; GISEL-NEXT:    s_nop 5
+; GISEL-NEXT:    s_nop 6
 ; GISEL-NEXT:    global_store_dwordx4 v0, a[0:3], s[6:7]
 ; GISEL-NEXT:    s_endpgm
   %result = call <4 x i32> @llvm.amdgcn.mfma.i32.16x16x64.i8(<4 x i32> %arg0, <4 x i32> %arg1, <4 x i32> %arg2, i32 3, i32 2, i32 1)
@@ -1097,7 +1097,7 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8(<4 x i32> %arg0, <4 x i32>
 ; SDAG-NEXT:    v_mov_b32_e32 v1, s17
 ; SDAG-NEXT:    v_mov_b32_e32 v2, s18
 ; SDAG-NEXT:    v_mov_b32_e32 v3, s19
-; SDAG-NEXT:    s_nop 6
+; SDAG-NEXT:    s_nop 7
 ; SDAG-NEXT:    global_store_dwordx4 v[8:9], a[28:31], off sc0 sc1
 ; SDAG-NEXT:    s_waitcnt vmcnt(0)
 ; SDAG-NEXT:    global_store_dwordx4 v[10:11], a[24:27], off sc0 sc1
@@ -1169,7 +1169,7 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8(<4 x i32> %arg0, <4 x i32>
 ; GISEL-NEXT:    v_mov_b64_e32 v[10:11], s[10:11]
 ; GISEL-NEXT:    v_mov_b64_e32 v[14:15], s[14:15]
 ; GISEL-NEXT:    v_mov_b64_e32 v[18:19], s[18:19]
-; GISEL-NEXT:    s_nop 3
+; GISEL-NEXT:    s_nop 4
 ; GISEL-NEXT:    global_store_dwordx4 v[20:21], a[16:19], off sc0 sc1
 ; GISEL-NEXT:    s_waitcnt vmcnt(0)
 ; GISEL-NEXT:    global_store_dwordx4 v[22:23], a[20:23], off sc0 sc1
@@ -1233,7 +1233,7 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8__flags(<4 x i32> %arg0, <4
 ; SDAG-NEXT:    v_mov_b32_e32 v1, s17
 ; SDAG-NEXT:    v_mov_b32_e32 v2, s18
 ; SDAG-NEXT:    v_mov_b32_e32 v3, s19
-; SDAG-NEXT:    s_nop 6
+; SDAG-NEXT:    s_nop 7
 ; SDAG-NEXT:    global_store_dwordx4 v[8:9], a[28:31], off sc0 sc1
 ; SDAG-NEXT:    s_waitcnt vmcnt(0)
 ; SDAG-NEXT:    global_store_dwordx4 v[10:11], a[24:27], off sc0 sc1
@@ -1305,7 +1305,7 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8__flags(<4 x i32> %arg0, <4
 ; GISEL-NEXT:    v_mov_b64_e32 v[10:11], s[10:11]
 ; GISEL-NEXT:    v_mov_b64_e32 v[14:15], s[14:15]
 ; GISEL-NEXT:    v_mov_b64_e32 v[18:19], s[18:19]
-; GISEL-NEXT:    s_nop 3
+; GISEL-NEXT:    s_nop 4
 ; GISEL-NEXT:    global_store_dwordx4 v[20:21], a[16:19], off sc0 sc1
 ; GISEL-NEXT:    s_waitcnt vmcnt(0)
 ; GISEL-NEXT:    global_store_dwordx4 v[22:23], a[20:23], off sc0 sc1
@@ -1352,7 +1352,7 @@ define <16 x i32> @test_mfma_i32_32x32x32_i8__mac(<4 x i32> %arg0, <4 x i32> %ar
 ; GCN-NEXT:    s_nop 1
 ; GCN-NEXT:    v_mfma_i32_32x32x32_i8 a[0:15], v[0:3], v[4:7], a[0:15]
 ; GCN-NEXT:    s_nop 7
-; GCN-NEXT:    s_nop 2
+; GCN-NEXT:    s_nop 3
 ; GCN-NEXT:    v_accvgpr_read_b32 v0, a0
 ; GCN-NEXT:    v_accvgpr_read_b32 v1, a1
 ; GCN-NEXT:    v_accvgpr_read_b32 v2, a2
@@ -1397,7 +1397,7 @@ define <16 x i32> @test_mfma_i32_32x32x32_i8__mac__flags(<4 x i32> %arg0, <4 x i
 ; GCN-NEXT:    s_nop 1
 ; GCN-NEXT:    v_mfma_i32_32x32x32_i8 a[0:15], v[0:3], v[4:7], a[0:15] cbsz:1 abid:1 blgp:1
 ; GCN-NEXT:    s_nop 7
-; GCN-NEXT:    s_nop 2
+; GCN-NEXT:    s_nop 3
 ; GCN-NEXT:    v_accvgpr_read_b32 v0, a0
 ; GCN-NEXT:    v_accvgpr_read_b32 v1, a1
 ; GCN-NEXT:    v_accvgpr_read_b32 v2, a2
@@ -1717,7 +1717,7 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8__vgprcd_mac(<4 x i32> %arg0
 ; SDAG-NEXT:    v_mfma_i32_32x32x32_i8 a[0:15], v[0:3], v[4:7], a[0:15]
 ; SDAG-NEXT:    v_mov_b32_e32 v0, 0
 ; SDAG-NEXT:    s_nop 7
-; SDAG-NEXT:    s_nop 1
+; SDAG-NEXT:    s_nop 2
 ; SDAG-NEXT:    global_store_dwordx4 v0, a[12:15], s[0:1] offset:48
 ; SDAG-NEXT:    global_store_dwordx4 v0, a[8:11], s[0:1] offset:32
 ; SDAG-NEXT:    global_store_dwordx4 v0, a[4:7], s[0:1] offset:16
@@ -1754,7 +1754,7 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8__vgprcd_mac(<4 x i32> %arg0
 ; GISEL-NEXT:    v_mfma_i32_32x32x32_i8 a[0:15], v[0:3], v[4:7], a[0:15]
 ; GISEL-NEXT:    v_mov_b32_e32 v0, 0
 ; GISEL-NEXT:    s_nop 7
-; GISEL-NEXT:    s_nop 1
+; GISEL-NEXT:    s_nop 2
 ; GISEL-NEXT:    global_store_dwordx4 v0, a[0:3], s[0:1]
 ; GISEL-NEXT:    global_store_dwordx4 v0, a[4:7], s[0:1] offset:16
 ; GISEL-NEXT:    global_store_dwordx4 v0, a[8:11], s[0:1] offset:32
@@ -1801,7 +1801,7 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8__vgprcd_mac_flags(<4 x i32>
 ; SDAG-NEXT:    v_mfma_i32_32x32x32_i8 a[0:15], v[0:3], v[4:7], a[0:15] cbsz:3 abid:2 blgp:1
 ; SDAG-NEXT:    v_mov_b32_e32 v0, 0
 ; SDAG-NEXT:    s_nop 7
-; SDAG-NEXT:    s_nop 1
+; SDAG-NEXT:    s_nop 2
 ; SDAG-NEXT:    global_store_dwordx4 v0, a[12:15], s[0:1] offset:48
 ; SDAG-NEXT:    global_store_dwordx4 v0, a[8:11], s[0:1] offset:32
 ; SDAG-NEXT:    global_store_dwordx4 v0, a[4:7], s[0:1] offset:16
@@ -1838,7 +1838,7 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8__vgprcd_mac_flags(<4 x i32>
 ; GISEL-NEXT:    v_mfma_i32_32x32x32_i8 a[0:15], v[0:3], v[4:7], a[0:15] cbsz:3 abid:2 blgp:1
 ; GISEL-NEXT:    v_mov_b32_e32 v0, 0
 ; GISEL-NEXT:    s_nop 7
-; GISEL-NEXT:    s_nop 1
+; GISEL-NEXT:    s_nop 2
 ; GISEL-NEXT:    global_store_dwordx4 v0, a[0:3], s[0:1]
 ; GISEL-NEXT:    global_store_dwordx4 v0, a[4:7], s[0:1] offset:16
 ; GISEL-NEXT:    global_store_dwordx4 v0, a[8:11], s[0:1] offset:32
@@ -1856,198 +1856,92 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8__vgprcd_mac_flags(<4 x i32>
 declare <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.bf16(<8 x bfloat>, <8 x bfloat>, <4 x float>, i32 immarg, i32 immarg, i32 immarg)
 
 define <4 x float> @test_mfma_f32_16x16x32_bf16(<8 x bfloat> %arg0, <8 x bfloat> %arg1, <4 x float> %arg2) {
-; SDAG-LABEL: test_mfma_f32_16x16x32_bf16:
-; SDAG:       ; %bb.0:
-; SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SDAG-NEXT:    v_accvgpr_write_b32 a0, v8
-; SDAG-NEXT:    v_accvgpr_write_b32 a1, v9
-; SDAG-NEXT:    v_accvgpr_write_b32 a2, v10
-; SDAG-NEXT:    v_accvgpr_write_b32 a3, v11
-; SDAG-NEXT:    s_nop 1
-; SDAG-NEXT:    v_mfma_f32_16x16x32_bf16 a[0:3], v[0:3], v[4:7], a[0:3]
-; SDAG-NEXT:    s_nop 6
-; SDAG-NEXT:    v_accvgpr_read_b32 v0, a0
-; SDAG-NEXT:    v_accvgpr_read_b32 v1, a1
-; SDAG-NEXT:    v_accvgpr_read_b32 v2, a2
-; SDAG-NEXT:    v_accvgpr_read_b32 v3, a3
-; SDAG-NEXT:    s_setpc_b64 s[30:31]
-;
-; GISEL-LABEL: test_mfma_f32_16x16x32_bf16:
-; GISEL:       ; %bb.0:
-; GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GISEL-NEXT:    v_lshrrev_b32_e32 v12, 16, v0
-; GISEL-NEXT:    v_lshrrev_b32_e32 v13, 16, v1
-; GISEL-NEXT:    v_lshrrev_b32_e32 v14, 16, v2
-; GISEL-NEXT:    v_lshrrev_b32_e32 v15, 16, v3
-; GISEL-NEXT:    v_mov_b32_sdwa v0, v12 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0
-; GISEL-NEXT:    v_mov_b32_sdwa v1, v13 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0
-; GISEL-NEXT:    v_mov_b32_sdwa v2, v14 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0
-; GISEL-NEXT:    v_mov_b32_sdwa v3, v15 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0
-; GISEL-NEXT:    v_lshrrev_b32_e32 v12, 16, v4
-; GISEL-NEXT:    v_lshrrev_b32_e32 v13, 16, v5
-; GISEL-NEXT:    v_lshrrev_b32_e32 v14, 16, v6
-; GISEL-NEXT:    v_lshrrev_b32_e32 v15, 16, v7
-; GISEL-NEXT:    v_accvgpr_write_b32 a0, v8
-; GISEL-NEXT:    v_mov_b32_sdwa v4, v12 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0
-; GISEL-NEXT:    v_mov_b32_sdwa v5, v13 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0
-; GISEL-NEXT:    v_mov_b32_sdwa v6, v14 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0
-; GISEL-NEXT:    v_mov_b32_sdwa v7, v15 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0
-; GISEL-NEXT:    v_accvgpr_write_b32 a1, v9
-; GISEL-NEXT:    v_accvgpr_write_b32 a2, v10
-; GISEL-NEXT:    v_accvgpr_write_b32 a3, v11
-; GISEL-NEXT:    s_nop 1
-; GISEL-NEXT:    v_mfma_f32_16x16x32_bf16 a[0:3], v[0:3], v[4:7], a[0:3]
-; GISEL-NEXT:    s_nop 6
-; GISEL-NEXT:    v_accvgpr_read_b32 v0, a0
-; GISEL-NEXT:    v_accvgpr_read_b32 v1, a1
-; GISEL-NEXT:    v_accvgpr_read_b32 v2, a2
-; GISEL-NEXT:    v_accvgpr_read_b32 v3, a3
-; GISEL-NEXT:    s_setpc_b64 s[30:31]
+; GCN-LABEL: test_mfma_f32_16x16x32_bf16:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_accvgpr_write_b32 a0, v8
+; GCN-NEXT:    v_accvgpr_write_b32 a1, v9
+; GCN-NEXT:    v_accvgpr_write_b32 a2, v10
+; GCN-NEXT:    v_accvgpr_write_b32 a3, v11
+; GCN-NEXT:    s_nop 1
+; GCN-NEXT:    v_mfma_f32_16x16x32_bf16 a[0:3], v[0:3], v[4:7], a[0:3]
+; GCN-NEXT:    s_nop 7
+; GCN-NEXT:    v_accvgpr_read_b32 v0, a0
+; GCN-NEXT:    v_accvgpr_read_b32 v1, a1
+; GCN-NEXT:    v_accvgpr_read_b32 v2, a2
+; GCN-NEXT:    v_accvgpr_read_b32 v3, a3
+; GCN-NEXT:    s_setpc_b64 s[30:31]
   %result = call <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.bf16(<8 x bfloat> %arg0, <8 x bfloat> %arg1, <4 x float> %arg2, i32 0, i32 0, i32 0)
   ret <4 x float> %result
 }
 
 define <4 x float> @test_mfma_f32_16x16x32_bf16__flags(<8 x bfloat> %arg0, <8 x bfloat> %arg1, <4 x float> %arg2) {
-; SDAG-LABEL: test_mfma_f32_16x16x32_bf16__flags:
-; SDAG:       ; %bb.0:
-; SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SDAG-NEXT:    v_accvgpr_write_b32 a0, v8
-; SDAG-NEXT:    v_accvgpr_write_b32 a1, v9
-; SDAG-NEXT:    v_accvgpr_write_b32 a2, v10
-; SDAG-NEXT:    v_accvgpr_write_b32 a3, v11
-; SDAG-NEXT:    s_nop 1
-; SDAG-NEXT:    v_mfma_f32_16x16x32_bf16 a[0:3], v[0:3], v[4:7], a[0:3] cbsz:1 abid:1 blgp:1
-; SDAG-NEXT:    s_nop 6
-; SDAG-NEXT:    v_accvgpr_read_b32 v0, a0
-; SDAG-NEXT:    v_accvgpr_read_b32 v1, a1
-; SDAG-NEXT:    v_accvgpr_read_b32 v2, a2
-; SDAG-NEXT:    v_accvgpr_read_b32 v3, a3
-; SDAG-NEXT:    s_setpc_b64 s[30:31]
-;
-; GISEL-LABEL: test_mfma_f32_16x16x32_bf16__flags:
-; GISEL:       ; %bb.0:
-; GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GISEL-NEXT:    v_lshrrev_b32_e32 v12, 16, v0
-; GISEL-NEXT:    v_lshrrev_b32_e32 v13, 16, v1
-; GISEL-NEXT:    v_lshrrev_b32_e32 v14, 16, v2
-; GISEL-NEXT:    v_lshrrev_b32_e32 v15, 16, v3
-; GISEL-NEXT:    v_mov_b32_sdwa v0, v12 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0
-; GISEL-NEXT:    v_mov_b32_sdwa v1, v13 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0
-; GISEL-NEXT:    v_mov_b32_sdwa v2, v14 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0
-; GISEL-NEXT:    v_mov_b32_sdwa v3, v15 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0
-; GISEL-NEXT:    v_lshrrev_b32_e32 v12, 16, v4
-; GISEL-NEXT:    v_lshrrev_b32_e32 v13, 16, v5
-; GISEL-NEXT:    v_lshrrev_b32_e32 v14, 16, v6
-; GISEL-NEXT:    v_lshrrev_b32_e32 v15, 16, v7
-; GISEL-NEXT:    v_accvgpr_write_b32 a0, v8
-; GISEL-NEXT:    v_mov_b32_sdwa v4, v12 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0
-; GISEL-NEXT:    v_mov_b32_sdwa v5, v13 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0
-; GISEL-NEXT:    v_mov_b32_sdwa v6, v14 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0
-; GISEL-NEXT:    v_mov_b32_sdwa v7, v15 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0
-; GISEL-NEXT:    v_accvgpr_write_b32 a1, v9
-; GISEL-NEXT:    v_accvgpr_write_b32 a2, v10
-; GISEL-NEXT:    v_accvgpr_write_b32 a3, v11
-; GISEL-NEXT:    s_nop 1
-; GISEL-NEXT:    v_mfma_f32_16x16x32_bf16 a[0:3], v[0:3], v[4:7], a[0:3] cbsz:1 abid:1 blgp:1
-; GISEL-NEXT:    s_nop 6
-; GISEL-NEXT:    v_accvgpr_read_b32 v0, a0
-; GISEL-NEXT:    v_accvgpr_read_b32 v1, a1
-; GISEL-NEXT:    v_accvgpr_read_b32 v2, a2
-; GISEL-NEXT:    v_accvgpr_read_b32 v3, a3
-; GISEL-NEXT:    s_setpc_b64 s[30:31]
+; GCN-LABEL: test_mfma_f32_16x16x32_bf16__flags:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_accvgpr_write_b32 a0, v8
+; GCN-NEXT:    v_accvgpr_write_b32 a1, v9
+; GCN-NEXT:    v_accvgpr_write_b32 a2, v10
+; GCN-NEXT:    v_accvgpr_write_b32 a3, v11
+; GCN-NEXT:    s_nop 1
+; GCN-NEXT:    v_mfma_f32_16x16x32_bf16 a[0:3], v[0:3], v[4:7], a[0:3] cbsz:1 abid:1 blgp:1
+; GCN-NEXT:    s_nop 7
+; GCN-NEXT:    v_accvgpr_read_b32 v0, a0
+; GCN-NEXT:    v_accvgpr_read_b32 v1, a1
+; GCN-NEXT:    v_accvgpr_read_b32 v2, a2
+; GCN-NEXT:    v_accvgpr_read_b32 v3, a3
+; GCN-NEXT:    s_setpc_b64 s[30:31]
   %result = call <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.bf16(<8 x bfloat> %arg0, <8 x bfloat> %arg1, <4 x float> %arg2, i32 1, i32 1, i32 1)
   ret <4 x float> %result
 }
 
 define amdgpu_kernel void @test_mfma_f32_16x16x32_bf16_no_agpr__vgprcd(ptr addrspace(1) %out, <8 x bfloat> %arg0, <8 x bfloat> %arg1, <4 x float> %arg2) #0 {
-; SDAG-LABEL: test_mfma_f32_16x16x32_bf16_no_agpr__vgprcd:
-; SDAG:       ; %bb.0:
-; SDAG-NEXT:    s_load_dwordx8 s[8:15], s[4:5], 0x34
-; SDAG-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x54
-; SDAG-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x24
-; SDAG-NEXT:    v_mov_b32_e32 v8, 0
-; SDAG-NEXT:    s_waitcnt lgkmcnt(0)
-; SDAG-NEXT:    v_mov_b64_e32 v[0:1], s[8:9]
-; SDAG-NEXT:    v_mov_b64_e32 v[2:3], s[10:11]
-; SDAG-NEXT:    v_mov_b64_e32 v[4:5], s[12:13]
-; SDAG-NEXT:    v_accvgpr_write_b32 a0, s0
-; SDAG-NEXT:    v_mov_b64_e32 v[6:7], s[14:15]
-; SDAG-NEXT:    v_accvgpr_write_b32 a1, s1
-; SDAG-NEXT:    v_accvgpr_write_b32 a2, s2
-; SDAG-NEXT:    v_accvgpr_write_b32 a3, s3
-; SDAG-NEXT:    s_nop 1
-; SDAG-NEXT:    v_mfma_f32_16x16x32_bf16 a[0:3], v[0:3], v[4:7], a[0:3]
-; SDAG-NEXT:    s_nop 6
-; SDAG-NEXT:    global_store_dwordx4 v8, a[0:3], s[6:7]
-; SDAG-NEXT:    s_endpgm
-;
-; GISEL-LABEL: test_mfma_f32_16x16x32_bf16_no_agpr__vgprcd:
-; GISEL:       ; %bb.0:
-; GISEL-NEXT:    s_load_dwordx8 s[8:15], s[4:5], 0x34
-; GISEL-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x54
-; GISEL-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x24
-; GISEL-NEXT:    s_waitcnt lgkmcnt(0)
-; GISEL-NEXT:    v_mov_b64_e32 v[0:1], s[8:9]
-; GISEL-NEXT:    v_mov_b64_e32 v[2:3], s[10:11]
-; GISEL-NEXT:    v_mov_b64_e32 v[4:5], s[12:13]
-; GISEL-NEXT:    v_accvgpr_write_b32 a0, s0
-; GISEL-NEXT:    v_mov_b64_e32 v[6:7], s[14:15]
-; GISEL-NEXT:    v_accvgpr_write_b32 a1, s1
-; GISEL-NEXT:    v_accvgpr_write_b32 a2, s2
-; GISEL-NEXT:    v_accvgpr_write_b32 a3, s3
-; GISEL-NEXT:    s_nop 1
-; GISEL-NEXT:    v_mfma_f32_16x16x32_bf16 a[0:3], v[0:3], v[4:7], a[0:3]
-; GISEL-NEXT:    v_mov_b32_e32 v0, 0
-; GISEL-NEXT:    s_nop 5
-; GISEL-NEXT:    global_store_dwordx4 v0, a[0:3], s[6:7]
-; GISEL-NEXT:    s_endpgm
+; GCN-LABEL: test_mfma_f32_16x16x32_bf16_no_agpr__vgprcd:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_load_dwordx8 s[8:15], s[4:5], 0x34
+; GCN-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x54
+; GCN-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x24
+; GCN-NEXT:    v_mov_b32_e32 v8, 0
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    v_mov_b64_e32 v[0:1], s[8:9]
+; GCN-NEXT:    v_mov_b64_e32 v[2:3], s[10:11]
+; GCN-NEXT:    v_mov_b64_e32 v[4:5], s[12:13]
+; GCN-NEXT:    v_accvgpr_write_b32 a0, s0
+; GCN-NEXT:    v_mov_b64_e32 v[6:7], s[14:15]
+; GCN-NEXT:    v_accvgpr_write_b32 a1, s1
+; GCN-NEXT:    v_accvgpr_write_b32 a2, s2
+; GCN-NEXT:    v_accvgpr_write_b32 a3, s3
+; GCN-NEXT:    s_nop 1
+; GCN-NEXT:    v_mfma_f32_16x16x32_bf16 a[0:3], v[0:3], v[4:7], a[0:3]
+; GCN-NEXT:    s_nop 7
+; GCN-NEXT:    global_store_dwordx4 v8, a[0:3], s[6:7]
+; GCN-NEXT:    s_endpgm
   %result = call <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.bf16(<8 x bfloat> %arg0, <8 x bfloat> %arg1, <4 x float> %arg2, i32 0, i32 0, i32 0)
   store <4 x float> %result, ptr addrspace(1) %out
   ret void
 }
 
 define amdgpu_kernel void @test_mfma_f32_16x16x32_bf16_no_agpr__vgprcd__flags(ptr addrspace(1) %out, <8 x bfloat> %arg0, <8 x bfloat> %arg1, <4 x float> %arg2) #0 {
-; SDAG-LABEL: test_mfma_f32_16x16x32_bf16_no_agpr__vgprcd__flags:
-; SDAG:       ; %bb.0:
-; SDAG-NEXT:    s_load_dwordx8 s[8:15], s[4:5], 0x34
-; SDAG-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x54
-; SDAG-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x24
-; SDAG-NEXT:    v_mov_b32_e32 v8, 0
-; SDAG-NEXT:    s_waitcnt lgkmcnt(0)
-; SDAG-NEXT:    v_mov_b64_e32 v[0:1], s[8:9]
-; SDAG-NEXT:    v_mov_b64_e32 v[2:3], s[10:11]
-; SDAG-NEXT:    v_mov_b64_e32 v[4:5], s[12:13]
-; SDAG-NEXT:    v_accvgpr_write_b32 a0, s0
-; SDAG-NEXT:    v_mov_b64_e32 v[6:7], s[14:15]
-; SDAG-NEXT:    v_accvgpr_write_b32 a1, s1
-; SDAG-NEXT:    v_accvgpr_write_b32 a2, s2
-; SDAG-NEXT:    v_accvgpr_write_b32 a3, s3
-; SDAG-NEXT:    s_nop 1
-; SDAG-NEXT:    v_mfma_f32_16x16x32_bf16 a[0:3], v[0:3], v[4:7], a[0:3] cbsz:3 abid:2 blgp:1
-; SDAG-NEXT:    s_nop 6
-; SDAG-NEXT:    global_store_dwordx4 v8, a[0:3], s[6:7]
-; SDAG-NEXT:    s_endpgm
-;
-; GISEL-LABEL: test_mfma_f32_16x16x32_bf16_no_agpr__vgprcd__flags:
-; GISEL:       ; %bb.0:
-; GISEL-NEXT:    s_load_dwordx8 s[8:15], s[4:5], 0x34
-; GISEL-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x54
-; GISEL-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x24
-; GISEL-NEXT:    s_waitcnt lgkmcnt(0)
-; GISEL-NEXT:    v_mov_b64_e32 v[0:1], s[8:9]
-; GISEL-NEXT:    v_mov_b64_e32 v[2:3], s[10:11]
-; GISEL-NEXT:    v_mov_b64_e32 v[4:5], s[12:13]
-; GISEL-NEXT:    v_accvgpr_write_b32 a0, s0
-; GISEL-NEXT:    v_mov_b64_e32 v[6:7], s[14:15]
-; GISEL-NEXT:    v_accvgpr_write_b32 a1, s1
-; GISEL-NEXT:    v_accvgpr_write_b32 a2, s2
-; GISEL-NEXT:    v_accvgpr_write_b32 a3, s3
-; GISEL-NEXT:    s_nop 1
-; GISEL-NEXT:    v_mfma_f32_16x16x32_bf16 a[0:3], v[0:3], v[4:7], a[0:3] cbsz:3 abid:2 blgp:1
-; GISEL-NEXT:    v_mov_b32_e32 v0, 0
-; GISEL-NEXT:    s_nop 5
-; GISEL-NEXT:    global_store_dwordx4 v0, a[0:3], s[6:7]
-; GISEL-NEXT:    s_endpgm
+; GCN-LABEL: test_mfma_f32_16x16x32_bf16_no_agpr__vgprcd__flags:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_load_dwordx8 s[8:15], s[4:5], 0x34
+; GCN-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x54
+; GCN-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x24
+; GCN-NEXT:    v_mov_b32_e32 v8, 0
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    v_mov_b64_e32 v[0:1], s[8:9]
+; GCN-NEXT:    v_mov_b64_e32 v[2:3], s[10:11]
+; GCN-NEXT:    v_mov_b64_e32 v[4:5], s[12:13]
+; GCN-NEXT:    v_accvgpr_write_b32 a0, s0
+; GCN-NEXT:    v_mov_b64_e32 v[6:7], s[14:15]
+; GCN-NEXT:    v_accvgpr_write_b32 a1, s1
+; GCN-NEXT:    v_accvgpr_write_b32 a2, s2
+; GCN-NEXT:    v_accvgpr_write_b32 a3, s3
+; GCN-NEXT:    s_nop 1
+; GCN-NEXT:    v_mfma_f32_16x16x32_bf16 a[0:3], v[0:3], v[4:7], a[0:3] cbsz:3 abid:2 blgp:1
+; GCN-NEXT:    s_nop 7
+; GCN-NEXT:    global_store_dwordx4 v8, a[0:3], s[6:7]
+; GCN-NEXT:    s_endpgm
   %result = call <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.bf16(<8 x bfloat> %arg0, <8 x bfloat> %arg1, <4 x float> %arg2, i32 3, i32 2, i32 1)
   store <4 x float> %result, ptr addrspace(1) %out
   ret void
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.scale.f32.16x16x128.f8f6f4.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.scale.f32.16x16x128.f8f6f4.ll
index 9a8282231ac15..25b857f8f47dd 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.scale.f32.16x16x128.f8f6f4.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.scale.f32.16x16x128.f8f6f4.ll
@@ -24,7 +24,7 @@ define <4 x float> @test_mfma_scale_f32_16x16x128_f8f6f4_0_0__cbsz0__blgp0(<8 x
 ; GCN-NEXT:    s_nop 1
 ; GCN-NEXT:    v_mfma_scale_f32_16x16x128_f8f6f4 a[0:3], v[0:7], v[8:15], a[0:3], v20, v21 op_sel_hi:[0,0,0]
 ; GCN-NEXT:    s_nop 7
-; GCN-NEXT:    s_nop 2
+; GCN-NEXT:    s_nop 3
 ; GCN-NEXT:    v_accvgpr_read_b32 v0, a0
 ; GCN-NEXT:    v_accvgpr_read_b32 v1, a1
 ; GCN-NEXT:    v_accvgpr_read_b32 v2, a2
@@ -48,7 +48,7 @@ define <4 x float> @test_mfma_scale_f32_16x16x128_f8f6f4_1_1__cbsz1__blgp1(<8 x
 ; GCN-NEXT:    s_nop 1
 ; GCN-NEXT:    v_mfma_scale_f32_16x16x128_f8f6f4 a[0:3], v[0:7], v[8:15], a[0:3], v20, v21 op_sel_hi:[0,0,0]
 ; GCN-NEXT:    s_nop 7
-; GCN-NEXT:    s_nop 2
+; GCN-NEXT:    s_nop 3
 ; GCN-NEXT:    v_accvgpr_read_b32 v0, a0
 ; GCN-NEXT:    v_accvgpr_read_b32 v1, a1
 ; GCN-NEXT:    v_accvgpr_read_b32 v2, a2
@@ -72,7 +72,7 @@ define <4 x float> @test_mfma_scale_f32_16x16x128_f8f6f4_2_2__cbsz1__blgp1(<8 x
 ; GCN-NEXT:    s_nop 1
 ; GCN-NEXT:    v_mfma_scale_f32_16x16x128_f8f6f4 a[0:3], v[0:7], v[8:15], a[0:3], v20, v21 op_sel_hi:[0,0,0]
 ; GCN-NEXT:    s_nop 7
-; GCN-NEXT:    s_nop 2
+; GCN-NEXT:    s_nop 3
 ; GCN-NEXT:    v_accvgpr_read_b32 v0, a0
 ; GCN-NEXT:    v_accvgpr_read_b32 v1, a1
 ; GCN-NEXT:    v_accvgpr_read_b32 v2, a2
@@ -96,7 +96,7 @@ define <4 x float> @test_mfma_scale_f32_16x16x128_f8f6f4_3_3__cbsz1__blgp1(<8 x
 ; GCN-NEXT:    s_nop 1
 ; GCN-NEXT:    v_mfma_scale_f32_16x16x128_f8f6f4 a[0:3], v[0:7], v[8:15], a[0:3], v20, v21 op_sel_hi:[0,0,0]
 ; GCN-NEXT:    s_nop 7
-; GCN-NEXT:    s_nop 2
+; GCN-NEXT:    s_nop 3
 ; GCN-NEXT:    v_accvgpr_read_b32 v0, a0
 ; GCN-NEXT:    v_accvgpr_read_b32 v1, a1
 ; GCN-NEXT:    v_accvgpr_read_b32 v2, a2
@@ -120,7 +120,7 @@ define <4 x float> @test_mfma_scale_f32_16x16x128_f8f6f4_0_3__cbsz1__blgp1(<8 x
 ; GCN-NEXT:    s_nop 1
 ; GCN-NEXT:    v_mfma_scale_f32_16x16x128_f8f6f4 a[0:3], v[0:7], v[8:15], a[0:3], v20, v21 op_sel_hi:[0,0,0]
 ; GCN-NEXT:    s_nop 7
-; GCN-NEXT:    s_nop 2
+; GCN-NEXT:    s_nop 3
 ; GCN-NEXT:    v_accvgpr_read_b32 v0, a0
 ; GCN-NEXT:    v_accvgpr_read_b32 v1, a1
 ; GCN-NEXT:    v_accvgpr_read_b32 v2, a2
@@ -144,7 +144,7 @@ define <4 x float> @test_mfma_scale_f32_16x16x128_f8f6f4_3_0__cbsz1__blgp1(<8 x
 ; GCN-NEXT:    s_nop 1
 ; GCN-NEXT:    v_mfma_scale_f32_16x16x128_f8f6f4 a[0:3], v[0:7], v[8:15], a[0:3], v20, v21 op_sel_hi:[0,0,0]
 ; GCN-NEXT:    s_nop 7
-; GCN-NEXT:    s_nop 2
+; GCN-NEXT:    s_nop 3
 ; GCN-NEXT:    v_accvgpr_read_b32 v0, a0
 ; GCN-NEXT:    v_accvgpr_read_b32 v1, a1
 ; GCN-NEXT:    v_accvgpr_read_b32 v2, a2
@@ -168,7 +168,7 @@ define <4 x float> @test_mfma_scale_f32_16x16x128_f8f6f4_2_3__cbsz1__blgp1(<8 x
 ; GCN-NEXT:    s_nop 1
 ; GCN-NEXT:    v_mfma_scale_f32_16x16x128_f8f6f4 a[0:3], v[0:7], v[8:15], a[0:3], v20, v21 op_sel_hi:[0,0,0]
 ; GCN-NEXT:    s_nop 7
-; GCN-NEXT:    s_nop 2
+; GCN-NEXT:    s_nop 3
 ; GCN-NEXT:    v_accvgpr_read_b32 v0, a0
 ; GCN-NEXT:    v_accvgpr_read_b32 v1, a1
 ; GCN-NEXT:    v_accvgpr_read_b32 v2, a2
@@ -192,7 +192,7 @@ define <4 x float> @test_mfma_scale_f32_16x16x128_f8f6f4_3_2__cbsz1__blgp1(<8 x
 ; GCN-NEXT:    s_nop 1
 ; GCN-NEXT:    v_mfma_scale_f32_16x16x128_f8f6f4 a[0:3], v[0:7], v[8:15], a[0:3], v20, v21 op_sel_hi:[0,0,0]
 ; GCN-NEXT:    s_nop 7
-; GCN-NEXT:    s_nop 2
+; GCN-NEXT:    s_nop 3
 ; GCN-NEXT:    v_accvgpr_read_b32 v0, a0
 ; GCN-NEXT:    v_accvgpr_read_b32 v1, a1
 ; GCN-NEXT:    v_accvgpr_read_b32 v2, a2
@@ -217,7 +217,7 @@ define <4 x float> @test_mfma_scale_f32_16x16x128_f8f6f4_0_0__cbsz0__blgp0__cons
 ; GCN-NEXT:    s_nop 1
 ; GCN-NEXT:    v_mfma_f32_16x16x128_f8f6f4 a[0:3], v[0:7], v[8:15], a[0:3]
 ; GCN-NEXT:    s_nop 7
-; GCN-NEXT:    s_nop 2
+; GCN-NEXT:    s_nop 3
 ; GCN-NEXT:    v_accvgpr_read_b32 v0, a0
 ; GCN-NEXT:    v_accvgpr_read_b32 v1, a1
 ; GCN-NEXT:    v_accvgpr_read_b32 v2, a2
@@ -242,7 +242,7 @@ define <4 x float> @test_mfma_scale_f32_16x16x128_f8f6f4_0_0__cbsz0__blgp1(<8 x
 ; GCN-NEXT:    s_nop 1
 ; GCN-NEXT:    v_mfma_scale_f32_16x16x128_f8f6f4 a[0:3], v[0:7], v[8:15], a[0:3], v20, v21 op_sel_hi:[0,0,0] blgp:1
 ; GCN-NEXT:    s_nop 7
-; GCN-NEXT:    s_nop 2
+; GCN-NEXT:    s_nop 3
 ; GCN-NEXT:    v_accvgpr_read_b32 v0, a0
 ; GCN-NEXT:    v_accvgpr_read_b32 v1, a1
 ; GCN-NEXT:    v_accvgpr_read_b32 v2, a2
@@ -267,7 +267,7 @@ define <4 x float> @test_mfma_scale_f32_16x16x128_f8f6f4_0_0__cbsz0__blgp1__cons
 ; GCN-NEXT:    s_nop 1
 ; GCN-NEXT:    v_mfma_f32_16x16x128_f8f6f4 a[0:3], v[0:7], v[8:15], a[0:3] blgp:1
 ; GCN-NEXT:    s_nop 7
-; GCN-NEXT:    s_nop 2
+; GCN-NEXT:    s_nop 3
 ; GCN-NEXT:    v_accvgpr_read_b32 v0, a0
 ; GCN-NEXT:    v_accvgpr_read_b32 v1, a1
 ; GCN-NEXT:    v_accvgpr_read_b32 v2, a2
@@ -292,7 +292,7 @@ define <4 x float> @test_mfma_scale_f32_16x16x128_f8f6f4_0_0__cbsz0__blgp2(<8 x
 ; GCN-NEXT:    s_nop 1
 ; GCN-NEXT:    v_mfma_scale_f32_16x16x128_f8f6f4 a[0:3], v[0:7], v[8:13], a[0:3], v18, v19 op_sel_hi:[0,0,0] blgp:2
 ; GCN-NEXT:    s_nop 7
-; GCN-NEXT:    s_nop 2
+; GCN-NEXT:    s_nop 3
 ; GCN-NEXT:    v_accvgpr_read_b32 v0, a0
 ; GCN-NEXT:    v_accvgpr_read_b32 v1, a1
 ; GCN-NEXT:    v_accvgpr_read_b32 v2, a2
@@ -317,7 +317,7 @@ define <4 x float> @test_mfma_scale_f32_16x16x128_f8f6f4_0_0__cbsz0__blgp2__cons
 ; GCN-NEXT:    s_nop 1
 ; GCN-NEXT:    v_mfma_f32_16x16x128_f8f6f4 a[0:3], v[0:7], v[8:13], a[0:3] blgp:2
 ; GCN-NEXT:    s_nop 7
-; GCN-NEXT:    s_nop 2
+; GCN-NEXT:    s_nop 3
 ; GCN-NEXT:    v_accvgpr_read_b32 v0, a0
 ; GCN-NEXT:    v_accvgpr_read_b32 v1, a1
 ; GCN-NEXT:    v_accvgpr_read_b32 v2, a2
@@ -342,7 +342,7 @@ define <4 x float> @test_mfma_scale_f32_16x16x128_f8f6f4_0_0__cbsz0__blgp3(<8 x
 ; GCN-NEXT:    s_nop 1
 ; GCN-NEXT:    v_mfma_scale_f32_16x16x128_f8f6f4 a[0:3], v[0:7], v[8:13], a[0:3], v18, v19 op_sel_hi:[0,0,0] blgp:3
 ; GCN-NEXT:    s_nop 7
-; GCN-NEXT:    s_nop 2
+; GCN-NEXT:    s_nop 3
 ; GCN-NEXT:    v_accvgpr_read_b32 v0, a0
 ; GCN-NEXT:    v_accvgpr_read_b32 v1, a1
 ; GCN-NEXT:    v_accvgpr_read_b32 v2, a2
@@ -367,7 +367,7 @@ define <4 x float> @test_mfma_scale_f32_16x16x128_f8f6f4_0_0__cbsz0__blgp3__cons
 ; GCN-NEXT:    s_nop 1
 ; GCN-NEXT:    v_mfma_f32_16x16x128_f8f6f4 a[0:3], v[0:7], v[8:13], a[0:3] blgp:3
 ; GCN-NEXT:    s_nop 7
-; GCN-NEXT:    s_nop 2
+; GCN-NEXT:    s_nop 3
 ; GCN-NEXT:    v_accvgpr_read_b32 v0, a0
 ; GCN-NEXT:    v_accvgpr_read_b32 v1, a1
 ; GCN-NEXT:    v_accvgpr_read_b32 v2, a2
@@ -392,7 +392,7 @@ define <4 x float> @test_mfma_scale_f32_16x16x128_f8f6f4_0_0__cbsz0__blgp4(<8 x
 ; GCN-NEXT:    s_nop 1
 ; GCN-NEXT:    v_mfma_scale_f32_16x16x128_f8f6f4 a[0:3], v[0:7], v[8:11], a[0:3], v16, v17 op_sel_hi:[0,0,0] blgp:4
 ; GCN-NEXT:    s_nop 7
-; GCN-NEXT:    s_nop 2
+; GCN-NEXT:    s_nop 3
 ; GCN-NEXT:    v_accvgpr_read_b32 v0, a0
 ; GCN-NEXT:    v_accvgpr_read_b32 v1, a1
 ; GCN-NEXT:    v_accvgpr_read_b32 v2, a2
@@ -417,7 +417,7 @@ define <4 x float> @test_mfma_scale_f32_16x16x128_f8f6f4_0_0__cbsz0__blgp4__cons
 ; GCN-NEXT:    s_nop 1
 ; GCN-NEXT:    v_mfma_f32_16x16x128_f8f6f4 a[0:3], v[0:7], v[8:11], a[0:3] blgp:4
 ; GCN-NEXT:    s_nop 7
-; GCN-NEXT:    s_nop 2
+; GCN-NEXT:    s_nop 3
 ; GCN-NEXT:    v_accvgpr_read_b32 v0, a0
 ; GCN-NEXT:    v_accvgpr_read_b32 v1, a1
 ; GCN-NEXT:    v_accvgpr_read_b32 v2, a2
@@ -442,7 +442,7 @@ define <4 x float> @test_mfma_scale_f32_16x16x128_f8f6f4_0_0__cbsz1__blgp0(<8 x
 ; GCN-NEXT:    s_nop 1
 ; GCN-NEXT:    v_mfma_scale_f32_16x16x128_f8f6f4 a[0:3], v[0:7], v[8:15], a[0:3], v20, v21 op_sel_hi:[0,0,0] cbsz:1
 ; GCN-NEXT:    s_nop 7
-; GCN-NEXT:    s_nop 2
+; GCN-NEXT:    s_nop 3
 ; GCN-NEXT:    v_accvgpr_read_b32 v0, a0
 ; GCN-NEXT:    v_accvgpr_read_b32 v1, a1
 ; GCN-NEXT:    v_accvgpr_read_b32 v2, a2
@@ -467,7 +467,7 @@ define <4 x float> @test_mfma_scale_f32_16x16x128_f8f6f4_0_0__cbsz1__blgp0__cons
 ; GCN-NEXT:    s_nop 1
 ; GCN-NEXT:    v_mfma_f32_16x16x128_f8f6f4 a[0:3], v[0:7], v[8:15], a[0:3] cbsz:1
 ; GCN-NEXT:    s_nop 7
-; GCN-NEXT:    s_nop 2
+; GCN-NEXT:    s_nop 3
 ; GCN-NEXT:    v_accvgpr_read_b32 v0, a0
 ; GCN-NEXT:    v_accvgpr_read_b32 v1, a1
 ; GCN-NEXT:    v_accvgpr_read_b32 v2, a2
@@ -492,7 +492,7 @@ define <4 x float> @test_mfma_scale_f32_16x16x128_f8f6f4_0_0__cbsz1__blgp1(<8 x
 ; GCN-NEXT:    s_nop 1
 ; GCN-NEXT:    v_mfma_scale_f32_16x16x128_f8f6f4 a[0:3], v[0:7], v[8:15], a[0:3], v20, v21 op_sel_hi:[0,0,0] cbsz:1 blgp:1
 ; GCN-NEXT:    s_nop 7
-; GCN-NEXT:    s_nop 2
+; GCN-NEXT:    s_nop 3
 ; GCN-NEXT:    v_accvgpr_read_b32 v0, a0
 ; GCN-NEXT:    v_accvgpr_read_b32 v1, a1
 ; GCN-NEXT:    v_accvgpr_read_b32 v2, a2
@@ -518,7 +518,7 @@ define <4 x float> @test_mfma_scale_f32_16x16x128_f8f6f4_0_0__cbsz1__blgp1__cons
 ; GCN-NEXT:    s_nop 1
 ; GCN-NEXT:    v_mfma_f32_16x16x128_f8f6f4 a[0:3], v[0:7], v[8:15], a[0:3] cbsz:1 blgp:1
 ; GCN-NEXT:    s_nop 7
-; GCN-NEXT:    s_nop 2
+; GCN-NEXT:    s_nop 3
 ; GCN-NEXT:    v_accvgpr_read_b32 v0, a0
 ; GCN-NEXT:    v_accvgpr_read_b32 v1, a1
 ; GCN-NEXT:    v_accvgpr_read_b32 v2, a2
@@ -543,7 +543,7 @@ define <4 x float> @test_mfma_scale_f32_16x16x128_f8f6f4_0_0__cbsz1__blgp2(<8 x
 ; GCN-NEXT:    s_nop 1
 ; GCN-NEXT:    v_mfma_scale_f32_16x16x128_f8f6f4 a[0:3], v[0:7], v[8:13], a[0:3], v18, v19 op_sel_hi:[0,0,0] cbsz:1 blgp:2
 ; GCN-NEXT:    s_nop 7
-; GCN-NEXT:    s_nop 2
+; GCN-NEXT:    s_nop 3
 ; GCN-NEXT:    v_accvgpr_read_b32 v0, a0
 ; GCN-NEXT:    v_accvgpr_read_b32 v1, a1
 ; GCN-NEXT:    v_accvgpr_read_b32 v2, a2
@@ -567,7 +567,7 @@ define <4 x float> @test_mfma_scale_f32_16x16x128_f8f6f4_0_0__cbsz1__blgp2__cons
 ; GCN-NEXT:    s_nop 1
 ; GCN-NEXT:    v_mfma_f32_16x16x128_f8f6f4 a[0:3], v[0:7], v[8:13], a[0:3] cbsz:1 blgp:2
 ; GCN-NEXT:    s_nop 7
-; GCN-NEXT:    s_nop 2
+; GCN-NEXT:    s_nop 3
 ; GCN-NEXT:    v_accvgpr_read_b32 v0, a0
 ; GCN-NEXT:    v_accvgpr_read_b32 v1, a1
 ; GCN-NEXT:    v_accvgpr_read_b32 v2, a2
@@ -592,7 +592,7 @@ define <4 x float> @test_mfma_scale_f32_16x16x128_f8f6f4_0_0__cbsz1__blgp3(<8 x
 ; GCN-NEXT:    s_nop 1
 ; GCN-NEXT:    v_mfma_scale_f32_16x16x128_f8f6f4 a[0:3], v[0:7], v[8:13], a[0:3], v18, v19 op_sel_hi:[0,0,0] cbsz:1 blgp:3
 ; GCN-NEXT:    s_nop 7
-; GCN-NEXT:    s_nop 2
+; GCN-NEXT:    s_nop 3
 ; GCN-NEXT:    v_accvgpr_read_b32 v0, a0
 ; GCN-NEXT:    v_accvgpr_read_b32 v1, a1
 ; GCN-NEXT:    v_accvgpr_read_b32 v2, a2
@@ -617,7 +617,7 @@ define <4 x float> @test_mfma_scale_f32_16x16x128_f8f6f4_0_0__cbsz1__blgp3__cons
 ; GCN-NEXT:    s_nop 1
 ; GCN-NEXT:    v_mfma_f32_16x16x128_f8f6f4 a[0:3], v[0:7], v[8:13], a[0:3] cbsz:1 blgp:3
 ; GCN-NEXT:    s_nop 7
-; GCN-NEXT:    s_nop 2
+; GCN-NEXT:    s_nop 3
 ; GCN-NEXT:    v_accvgpr_read_b32 v0, a0
 ; GCN-NEXT:    v_accvgpr_read_b32 v1, a1
 ; GCN-NEXT:    v_accvgpr_read_b32 v2, a2
@@ -642,7 +642,7 @@ define <4 x float> @test_mfma_scale_f32_16x16x128_f8f6f4_0_0__cbsz1__blgp4(<8 x
 ; GCN-NEXT:    s_nop 1
 ; GCN-NEXT:    v_mfma_scale_f32_16x16x128_f8f6f4 a[0:3], v[0:7], v[8:11], a[0:3], v16, v17 op_sel_hi:[0,0,0] cbsz:1 blgp:4
 ; GCN-NEXT:    s_nop 7
-; GCN-NEXT:    s_nop 2
+; GCN-NEXT:    s_nop 3
 ; GCN-NEXT:    v_accvgpr_read_b32 v0, a0
 ; GCN-NEXT:    v_accvgpr_read_b32 v1, a1
 ; GCN-NEXT:    v_accvgpr_read_b32 v2, a2
@@ -667,7 +667,7 @@ define <4 x float> @test_mfma_scale_f32_16x16x128_f8f6f4_0_0__cbsz1__blgp4__cons
 ; GCN-NEXT:    s_nop 1
 ; GCN-NEXT:    v_mfma_f32_16x16x128_f8f6f4 a[0:3], v[0:7], v[8:11], a[0:3] cbsz:1 blgp:4
 ; GCN-NEXT:    s_nop 7
-; GCN-NEXT:    s_nop 2
+; GCN-NEXT:    s_nop 3
 ; GCN-NEXT:    v_accvgpr_read_b32 v0, a0
 ; GCN-NEXT:    v_accvgpr_read_b32 v1, a1
 ; GCN-NEXT:    v_accvgpr_read_b32 v2, a2
@@ -692,7 +692,7 @@ define <4 x float> @test_mfma_scale_f32_16x16x128_f8f6f4_0_0__cbsz2__blgp0(<6 x
 ; GCN-NEXT:    s_nop 1
 ; GCN-NEXT:    v_mfma_scale_f32_16x16x128_f8f6f4 a[0:3], v[0:5], v[6:13], a[0:3], v18, v19 op_sel_hi:[0,0,0] cbsz:2
 ; GCN-NEXT:    s_nop 7
-; GCN-NEXT:    s_nop 2
+; GCN-NEXT:    s_nop 3
 ; GCN-NEXT:    v_accvgpr_read_b32 v0, a0
 ; GCN-NEXT:    v_accvgpr_read_b32 v1, a1
 ; GCN-NEXT:    v_accvgpr_read_b32 v2, a2
@@ -717,7 +717,7 @@ define <4 x float> @test_mfma_scale_f32_16x16x128_f8f6f4_0_0__cbsz2__blgp0__cons
 ; GCN-NEXT:    s_nop 1
 ; GCN-NEXT:    v_mfma_f32_16x16x128_f8f6f4 a[0:3], v[0:5], v[6:13], a[0:3] cbsz:2
 ; GCN-NEXT:    s_nop 7
-; GCN-NEXT:    s_nop 2
+; GCN-NEXT:    s_nop 3
 ; GCN-NEXT:    v_accvgpr_read_b32 v0, a0
 ; GCN-NEXT:    v_accvgpr_read_b32 v1, a1
 ; GCN-NEXT:    v_accvgpr_read_b32 v2, a2
@@ -742,7 +742,7 @@ define <4 x float> @test_mfma_scale_f32_16x16x128_f8f6f4_0_0__cbsz2__blgp1(<6 x
 ; GCN-NEXT:    s_nop 1
 ; GCN-NEXT:    v_mfma_scale_f32_16x16x128_f8f6f4 a[0:3], v[0:5], v[6:13], a[0:3], v18, v19 op_sel_hi:[0,0,0] cbsz:2 blgp:1
 ; GCN-NEXT:    s_nop 7
-; GCN-NEXT:    s_nop 2
+; GCN-NEXT:    s_nop 3
 ; GCN-NEXT:    v_accvgpr_read_b32 v0, a0
 ; GCN-NEXT:    v_accvgpr_read_b32 v1, a1
 ; GCN-NEXT:    v_accvgpr_read_b32 v2, a2
@@ -767,7 +767,7 @@ define <4 x float> @test_mfma_scale_f32_16x16x128_f8f6f4_0_0__cbsz2__blgp1__cons
 ; GCN-NEXT:    s_nop 1
 ; GCN-NEXT:    v_mfma_f32_16x16x128_f8f6f4 a[0:3], v[0:5], v[6:13], a[0:3] cbsz:2 blgp:1
 ; GCN-NEXT:    s_nop 7
-; GCN-NEXT:    s_nop 2
+; GCN-NEXT:    s_nop 3
 ; GCN-NEXT:    v_accvgpr_read_b32 v0, a0
 ; GCN-NEXT:    v_accvgpr_read_b32 v1, a1
 ; GCN-NEXT:    v_accvgpr_read_b32 v2, a2
@@ -791,7 +791,7 @@ define <4 x float> @test_mfma_scale_f32_16x16x128_f8f6f4_0_0__cbsz2__blgp2(<6 x
 ; GCN-NEXT:    v_accvgpr_write_b32 a3, v15
 ; GCN-NEXT:    s_nop 1
 ; GCN-NEXT:    v_mfma_scale_f32_16x16x128_f8f6f4 a[0:3], v[0:5], v[6:11], a[0:3], v16, v17 op_sel_hi:[0,0,0] cbsz:2 blgp:2
-; GCN-NEXT:    s_nop 6
+; GCN-NEXT:    s_nop 7
 ; GCN-NEXT:    v_accvgpr_read_b32 v0, a0
 ; GCN-NEXT:    v_accvgpr_read_b32 v1, a1
 ; GCN-NEXT:    v_accvgpr_read_b32 v2, a2
@@ -815,7 +815,7 @@ define <4 x float> @test_mfma_scale_f32_16x16x128_f8f6f4_0_0__cbsz2__blgp2__cons
 ; GCN-NEXT:    v_accvgpr_write_b32 a3, v15
 ; GCN-NEXT:    s_nop 1
 ; GCN-NEXT:    v_mfma_f32_16x16x128_f8f6f4 a[0:3], v[0:5], v[6:11], a[0:3] cbsz:2 blgp:2
-; GCN-NEXT:    s_nop 6
+; GCN-NEXT:    s_nop 7
 ; GCN-NEXT:    v_accvgpr_read_b32 v0, a0
 ; GCN-NEXT:    v_accvgpr_read_b32 v1, a1
 ; GCN-NEXT:    v_accvgpr_read_b32 v2, a2
@@ -839,7 +839,7 @@ define <4 x float> @test_mfma_scale_f32_16x16x128_f8f6f4_0_0__cbsz2__blgp3(<6 x
 ; GCN-NEXT:    v_accvgpr_write_b32 a3, v15
 ; GCN-NEXT:    s_nop 1
 ; GCN-NEXT:    v_mfma_scale_f32_16x16x128_f8f6f4 a[0:3], v[0:5], v[6:11], a[0:3], v16, v17 op_sel_hi:[0,0,0] cbsz:2 blgp:3
-; GCN-NEXT:    s_nop 6
+; GCN-NEXT:    s_nop 7
 ; GCN-NEXT:    v_accvgpr_read_b32 v0, a0
 ; GCN-NEXT:    v_accvgpr_read_b32 v1, a1
 ; GCN-NEXT:    v_accvgpr_read_b32 v2, a2
@@ -863,7 +863,7 @@ define <4 x float> @test_mfma_scale_f32_16x16x128_f8f6f4_0_0__cbsz2__blgp3__cons
 ; GCN-NEXT:    v_accvgpr_write_b32 a3, v15
 ; GCN-NEXT:    s_nop 1
 ; GCN-NEXT:    v_mfma_f32_16x16x128_f8f6f4 a[0:3], v[0:5], v[6:11], a[0:3] cbsz:2 blgp:3
-; GCN-NEXT:    s_nop 6
+; GCN-NEXT:    s_nop 7
 ; GCN-NEXT:    v_accvgpr_read_b32 v0, a0
 ; GCN-NEXT:    v_accvgpr_read_b32 v1, a1
 ; GCN-NEXT:    v_accvgpr_read_b32 v2, a2
@@ -889,7 +889,7 @@ define <4 x float> @test_mfma_scale_f32_16x16x128_f8f6f4_0_0__cbsz3__blgp0(<6 x
 ; GCN-NEXT:    s_nop 1
 ; GCN-NEXT:    v_mfma_scale_f32_16x16x128_f8f6f4 a[0:3], v[0:5], v[6:13], a[0:3], v18, v19 op_sel_hi:[0,0,0] cbsz:3
 ; GCN-NEXT:    s_nop 7
-; GCN-NEXT:    s_nop 2
+; GCN-NEXT:    s_nop 3
 ; GCN-NEXT:    v_accvgpr_read_b32 v0, a0
 ; GCN-NEXT:    v_accvgpr_read_b32 v1, a1
 ; GCN-NEXT:    v_accvgpr_read_b32 v2, a2
@@ -914,7 +914,7 @@ define <4 x float> @test_mfma_scale_f32_16x16x128_f8f6f4_0_0__cbsz3__blgp0__cons
 ; GCN-NEXT:    s_nop 1
 ; GCN-NEXT:    v_mfma_f32_16x16x128_f8f6f4 a[0:3], v[0:5], v[6:13], a[0:3] cbsz:3
 ; GCN-NEXT:    s_nop 7
-; GCN-NEXT:    s_nop 2
+; GCN-NEXT:    s_nop 3
 ; GCN-NEXT:    v_accvgpr_read_b32 v0, a0
 ; GCN-NEXT:    v_accvgpr_read_b32 v1, a1
 ; GCN-NEXT:    v_accvgpr_read_b32 v2, a2
@@ -939,7 +939,7 @@ define <4 x float> @test_mfma_scale_f32_16x16x128_f8f6f4_0_0__cbsz3__blgp1(<6 x
 ; GCN-NEXT:    s_nop 1
 ; GCN-NEXT:    v_mfma_scale_f32_16x16x128_f8f6f4 a[0:3], v[0:5], v[6:13], a[0:3], v18, v19 op_sel_hi:[0,0,0] cbsz:3 blgp:1
 ; GCN-NEXT:    s_nop 7
-; GCN-NEXT:    s_nop 2
+; GCN-NEXT:    s_nop 3
 ; GCN-NEXT:    v_accvgpr_read_b32 v0, a0
 ; GCN-NEXT:    v_accvgpr_read_b32 v1, a1
 ; GCN-NEXT:    v_accvgpr_read_b32 v2, a2
@@ -964,7 +964,7 @@ define <4 x float> @test_mfma_scale_f32_16x16x128_f8f6f4_0_0__cbsz3__blgp1__cons
 ; GCN-NEXT:    s_nop 1
 ; GCN-NEXT:    v_mfma_f32_16x16x128_f8f6f4 a[0:3], v[0:5], v[6:13], a[0:3] cbsz:3 blgp:1
 ; GCN-NEXT:    s_nop 7
-; GCN-NEXT:    s_nop 2
+; GCN-NEXT:    s_nop 3
 ; GCN-NEXT:    v_accvgpr_read_b32 v0, a0
 ; GCN-NEXT:    v_accvgpr_read_b32 v1, a1
 ; GCN-NEXT:    v_accvgpr_read_b32 v2, a2
@@ -988,7 +988,7 @@ define <4 x float> @test_mfma_scale_f32_16x16x128_f8f6f4_0_0__cbsz3__blgp2(<6 x
 ; GCN-NEXT:    v_accvgpr_write_b32 a3, v15
 ; GCN-NEXT:    s_nop 1
 ; GCN-NEXT:    v_mfma_scale_f32_16x16x128_f8f6f4 a[0:3], v[0:5], v[6:11], a[0:3], v16, v17 op_sel_hi:[0,0,0] cbsz:3 blgp:2
-; GCN-NEXT:    s_nop 6
+; GCN-NEXT:    s_nop 7
 ; GCN-NEXT:    v_accvgpr_read_b32 v0, a0
 ; GCN-NEXT:    v_accvgpr_read_b32 v1, a1
 ; GCN-NEXT:    v_accvgpr_read_b32 v2, a2
@@ -1012,7 +1012,7 @@ define <4 x float> @test_mfma_scale_f32_16x16x128_f8f6f4_0_0__cbsz3__blgp2__cons
 ; GCN-NEXT:    v_accvgpr_write_b32 a3, v15
 ; GCN-NEXT:    s_nop 1
 ; GCN-NEXT:    v_mfma_f32_16x16x128_f8f6f4 a[0:3], v[0:5], v[6:11], a[0:3] cbsz:3 blgp:2
-; GCN-NEXT:    s_nop 6
+; GCN-NEXT:    s_nop 7
 ; GCN-NEXT:    v_accvgpr_read_b32 v0, a0
 ; GCN-NEXT:    v_accvgpr_read_b32 v1, a1
 ; GCN-NEXT:    v_accvgpr_read_b32 v2, a2
@@ -1036,7 +1036,7 @@ define <4 x float> @test_mfma_scale_f32_16x16x128_f8f6f4_0_0__cbsz3__blgp4(<6 x
 ; GCN-NEXT:    v_accvgpr_write_b32 a3, v13
 ; GCN-NEXT:    s_nop 1
 ; GCN-NEXT:    v_mfma_scale_f32_16x16x128_f8f6f4 a[0:3], v[0:5], v[6:9], a[0:3], v14, v15 op_sel_hi:[0,0,0] cbsz:3 blgp:4
-; GCN-NEXT:    s_nop 6
+; GCN-NEXT:    s_nop 7
 ; GCN-NEXT:    v_accvgpr_read_b32 v0, a0
 ; GCN-NEXT:    v_accvgpr_read_b32 v1, a1
 ; GCN-NEXT:    v_accvgpr_read_b32 v2, a2
@@ -1060,7 +1060,7 @@ define <4 x float> @test_mfma_scale_f32_16x16x128_f8f6f4_0_0__cbsz3__blgp4__cons
 ; GCN-NEXT:    v_accvgpr_write_b32 a3, v13
 ; GCN-NEXT:    s_nop 1
 ; GCN-NEXT:    v_mfma_f32_16x16x128_f8f6f4 a[0:3], v[0:5], v[6:9], a[0:3] cbsz:3 blgp:4
-; GCN-NEXT:    s_nop 6
+; GCN-NEXT:    s_nop 7
 ; GCN-NEXT:    v_accvgpr_read_b32 v0, a0
 ; GCN-NEXT:    v_accvgpr_read_b32 v1, a1
 ; GCN-NEXT:    v_accvgpr_read_b32 v2, a2
@@ -1084,7 +1084,7 @@ define <4 x float> @test_mfma_scale_f32_16x16x128_f8f6f4_0_0__cbsz3__blgp3(<6 x
 ; GCN-NEXT:    v_accvgpr_write_b32 a3, v15
 ; GCN-NEXT:    s_nop 1
 ; GCN-NEXT:    v_mfma_scale_f32_16x16x128_f8f6f4 a[0:3], v[0:5], v[6:11], a[0:3], v16, v17 op_sel_hi:[0,0,0] cbsz:3 blgp:3
-; GCN-NEXT:    s_nop 6
+; GCN-NEXT:    s_nop 7
 ; GCN-NEXT:    v_accvgpr_read_b32 v0, a0
 ; GCN-NEXT:    v_accvgpr_read_b32 v1, a1
 ; GCN-NEXT:    v_accvgpr_read_b32 v2, a2
@@ -1108,7 +1108,7 @@ define <4 x float> @test_mfma_scale_f32_16x16x128_f8f6f4_0_0__cbsz3__blgp3__cons
 ; GCN-NEXT:    v_accvgpr_write_b32 a3, v15
 ; GCN-NEXT:    s_nop 1
 ; GCN-NEXT:    v_mfma_f32_16x16x128_f8f6f4 a[0:3], v[0:5], v[6:11], a[0:3] cbsz:3 blgp:3
-; GCN-NEXT:    s_nop 6
+; GCN-NEXT:    s_nop 7
 ; GCN-NEXT:    v_accvgpr_read_b32 v0, a0
 ; GCN-NEXT:    v_accvgpr_read_b32 v1, a1
 ; GCN-NEXT:    v_accvgpr_read_b32 v2, a2
@@ -1132,7 +1132,7 @@ define <4 x float> @test_mfma_scale_f32_16x16x128_f8f6f4_0_0__cbsz2__blgp4(<6 x
 ; GCN-NEXT:    v_accvgpr_write_b32 a3, v13
 ; GCN-NEXT:    s_nop 1
 ; GCN-NEXT:    v_mfma_scale_f32_16x16x128_f8f6f4 a[0:3], v[0:5], v[6:9], a[0:3], v14, v15 op_sel_hi:[0,0,0] cbsz:2 blgp:4
-; GCN-NEXT:    s_nop 6
+; GCN-NEXT:    s_nop 7
 ; GCN-NEXT:    v_accvgpr_read_b32 v0, a0
 ; GCN-NEXT:    v_accvgpr_read_b32 v1, a1
 ; GCN-NEXT:    v_accvgpr_read_b32 v2, a2
@@ -1156,7 +1156,7 @@ define <4 x float> @test_mfma_scale_f32_16x16x128_f8f6f4_0_0__cbsz2__blgp4__cons
 ; GCN-NEXT:    v_accvgpr_write_b32 a3, v13
 ; GCN-NEXT:    s_nop 1
 ; GCN-NEXT:    v_mfma_f32_16x16x128_f8f6f4 a[0:3], v[0:5], v[6:9], a[0:3] cbsz:2 blgp:4
-; GCN-NEXT:    s_nop 6
+; GCN-NEXT:    s_nop 7
 ; GCN-NEXT:    v_accvgpr_read_b32 v0, a0
 ; GCN-NEXT:    v_accvgpr_read_b32 v1, a1
 ; GCN-NEXT:    v_accvgpr_read_b32 v2, a2
@@ -1181,7 +1181,7 @@ define <4 x float> @test_mfma_scale_f32_16x16x128_f8f6f4_0_0__cbsz4__blgp0(<4 x
 ; GCN-NEXT:    s_nop 1
 ; GCN-NEXT:    v_mfma_scale_f32_16x16x128_f8f6f4 a[0:3], v[0:3], v[4:11], a[0:3], v16, v17 op_sel_hi:[0,0,0] cbsz:4
 ; GCN-NEXT:    s_nop 7
-; GCN-NEXT:    s_nop 2
+; GCN-NEXT:    s_nop 3
 ; GCN-NEXT:    v_accvgpr_read_b32 v0, a0
 ; GCN-NEXT:    v_accvgpr_read_b32 v1, a1
 ; GCN-NEXT:    v_accvgpr_read_b32 v2, a2
@@ -1206,7 +1206,7 @@ define <4 x float> @test_mfma_scale_f32_16x16x128_f8f6f4_0_0__cbsz4__blgp0__cons
 ; GCN-NEXT:    s_nop 1
 ; GCN-NEXT:    v_mfma_f32_16x16x128_f8f6f4 a[0:3], v[0:3], v[4:11], a[0:3] cbsz:4
 ; GCN-NEXT:    s_nop 7
-; GCN-NEXT:    s_nop 2
+; GCN-NEXT:    s_nop 3
 ; GCN-NEXT:    v_accvgpr_read_b32 v0, a0
 ; GCN-NEXT:    v_accvgpr_read_b32 v1, a1
 ; GCN-NEXT:    v_accvgpr_read_b32 v2, a2
@@ -1231,7 +1231,7 @@ define <4 x float> @test_mfma_scale_f32_16x16x128_f8f6f4_0_0__cbsz4__blgp1(<4 x
 ; GCN-NEXT:    s_nop 1
 ; GCN-NEXT:    v_mfma_scale_f32_16x16x128_f8f6f4 a[0:3], v[0:3], v[4:11], a[0:3], v16, v17 op_sel_hi:[0,0,0] cbsz:4 blgp:1
 ; GCN-NEXT:    s_nop 7
-; GCN-NEXT:    s_nop 2
+; GCN-NEXT:    s_nop 3
 ; GCN-NEXT:    v_accvgpr_read_b32 v0, a0
 ; GCN-NEXT:    v_accvgpr_read_b32 v1, a1
 ; GCN-NEXT:    v_accvgpr_read_b32 v2, a2
@@ -1256,7 +1256,7 @@ define <4 x float> @test_mfma_scale_f32_16x16x128_f8f6f4_0_0__cbsz4__blgp1__cons
 ; GCN-NEXT:    s_nop 1
 ; GCN-NEXT:    v_mfma_f32_16x16x128_f8f6f4 a[0:3], v[0:3], v[4:11], a[0:3] cbsz:4 blgp:1
 ; GCN-NEXT:    s_nop 7
-; GCN-NEXT:    s_nop 2
+; GCN-NEXT:    s_nop 3
 ; GCN-NEXT:    v_accvgpr_read_b32 v0, a0
 ; GCN-NEXT:    v_accvgpr_read_b32 v1, a1
 ; GCN-NEXT:    v_accvgpr_read_b32 v2, a2
@@ -1280,7 +1280,7 @@ define <4 x float> @test_mfma_scale_f32_16x16x128_f8f6f4_0_0__cbsz4__blgp2(<4 x
 ; GCN-NEXT:    v_accvgpr_write_b32 a3, v13
 ; GCN-NEXT:    s_nop 1
 ; GCN-NEXT:    v_mfma_scale_f32_16x16x128_f8f6f4 a[0:3], v[0:3], v[4:9], a[0:3], v14, v15 op_sel_hi:[0,0,0] cbsz:4 blgp:2
-; GCN-NEXT:    s_nop 6
+; GCN-NEXT:    s_nop 7
 ; GCN-NEXT:    v_accvgpr_read_b32 v0, a0
 ; GCN-NEXT:    v_accvgpr_read_b32 v1, a1
 ; GCN-NEXT:    v_accvgpr_read_b32 v2, a2
@@ -1304,7 +1304,7 @@ define <4 x float> @test_mfma_scale_f32_16x16x128_f8f6f4_0_0__cbsz4__blgp2__cons
 ; GCN-NEXT:    v_accvgpr_write_b32 a3, v13
 ; GCN-NEXT:    s_nop 1
 ; GCN-NEXT:    v_mfma_f32_16x16x128_f8f6f4 a[0:3], v[0:3], v[4:9], a[0:3] cbsz:4 blgp:2
-; GCN-NEXT:    s_nop 6
+; GCN-NEXT:    s_nop 7
 ; GCN-NEXT:    v_accvgpr_read_b32 v0, a0
 ; GCN-NEXT:    v_accvgpr_read_b32 v1, a1
 ; GCN-NEXT:    v_accvgpr_read_b32 v2, a2
@@ -1328,7 +1328,7 @@ define <4 x float> @test_mfma_scale_f32_16x16x128_f8f6f4_0_0__cbsz4__blgp3(<4 x
 ; GCN-NEXT:    v_accvgpr_write_b32 a3, v13
 ; GCN-NEXT:    s_nop 1
 ; GCN-NEXT:    v_mfma_scale_f32_16x16x128_f8f6f4 a[0:3], v[0:3], v[4:9], a[0:3], v14, v15 op_sel_hi:[0,0,0] cbsz:4 blgp:3
-; GCN-NEXT:    s_nop 6
+; GCN-NEXT:    s_nop 7
 ; GCN-NEXT:    v_accvgpr_read_b32 v0, a0
 ; GCN-NEXT:    v_accvgpr_read_b32 v1, a1
 ; GCN-NEXT:    v_accvgpr_read_b32 v2, a2
@@ -1352,7 +1352,7 @@ define <4 x float> @test_mfma_scale_f32_16x16x128_f8f6f4_0_0__cbsz4__blgp3__cons
 ; GCN-NEXT:    v_accvgpr_write_b32 a3, v13
 ; GCN-NEXT:    s_nop 1
 ; GCN-NEXT:    v_mfma_f32_16x16x128_f8f6f4 a[0:3], v[0:3], v[4:9], a[0:3] cbsz:4 blgp:3
-; GCN-NEXT:    s_nop 6
+; GCN-NEXT:    s_nop 7
 ; GCN-NEXT:    v_accvgpr_read_b32 v0, a0
 ; GCN-NEXT:    v_accvgpr_read_b32 v1, a1
 ; GCN-NEXT:    v_accvgpr_read_b32 v2, a2
@@ -1376,7 +1376,7 @@ define <4 x float> @test_mfma_scale_f32_16x16x128_f8f6f4_0_0__cbsz4__blgp4(<4 x
 ; GCN-NEXT:    v_accvgpr_write_b32 a3, v11
 ; GCN-NEXT:    s_nop 1
 ; GCN-NEXT:    v_mfma_scale_f32_16x16x128_f8f6f4 a[0:3], v[0:3], v[4:7], a[0:3], v12, v13 op_sel_hi:[0,0,0] cbsz:4 blgp:4
-; GCN-NEXT:    s_nop 6
+; GCN-NEXT:    s_nop 7
 ; GCN-NEXT:    v_accvgpr_read_b32 v0, a0
 ; GCN-NEXT:    v_accvgpr_read_b32 v1, a1
 ; GCN-NEXT:    v_accvgpr_read_b32 v2, a2
@@ -1400,7 +1400,7 @@ define <4 x float> @test_mfma_scale_f32_16x16x128_f8f6f4_0_0__cbsz4__blgp4__cons
 ; GCN-NEXT:    v_accvgpr_write_b32 a3, v11
 ; GCN-NEXT:    s_nop 1
 ; GCN-NEXT:    v_mfma_f32_16x16x128_f8f6f4 a[0:3], v[0:3], v[4:7], a[0:3] cbsz:4 blgp:4
-; GCN-NEXT:    s_nop 6
+; GCN-NEXT:    s_nop 7
 ; GCN-NEXT:    v_accvgpr_read_b32 v0, a0
 ; GCN-NEXT:    v_accvgpr_read_b32 v1, a1
 ; GCN-NEXT:    v_accvgpr_read_b32 v2, a2
@@ -1429,7 +1429,7 @@ define <4 x float> @test_mfma_scale_f32_16x16x128_f8f6f4_0_0__sgpr_scaleA__sgpr_
 ; GCN-NEXT:    s_nop 1
 ; GCN-NEXT:    v_mfma_scale_f32_16x16x128_f8f6f4 a[0:3], v[0:7], v[8:15], a[0:3], s0, v16 op_sel_hi:[0,0,0]
 ; GCN-NEXT:    s_nop 7
-; GCN-NEXT:    s_nop 2
+; GCN-NEXT:    s_nop 3
 ; GCN-NEXT:    v_accvgpr_read_b32 v0, a0
 ; GCN-NEXT:    v_accvgpr_read_b32 v1, a1
 ; GCN-NEXT:    v_accvgpr_read_b32 v2, a2
@@ -1450,7 +1450,7 @@ define <4 x float> @test_mfma_scale_f32_16x16x128_f8f6f4_0_0__sgpr_scaleA__vgpr_
 ; GCN-NEXT:    s_nop 1
 ; GCN-NEXT:    v_mfma_scale_f32_16x16x128_f8f6f4 a[0:3], v[0:7], v[8:15], a[0:3], s0, v20 op_sel_hi:[0,0,0]
 ; GCN-NEXT:    s_nop 7
-; GCN-NEXT:    s_nop 2
+; GCN-NEXT:    s_nop 3
 ; GCN-NEXT:    v_accvgpr_read_b32 v0, a0
 ; GCN-NEXT:    v_accvgpr_read_b32 v1, a1
 ; GCN-NEXT:    v_accvgpr_read_b32 v2, a2
@@ -1471,7 +1471,7 @@ define <4 x float> @test_mfma_scale_f32_16x16x128_f8f6f4_0_0__vgpr_scaleA__sgpr_
 ; GCN-NEXT:    s_nop 1
 ; GCN-NEXT:    v_mfma_scale_f32_16x16x128_f8f6f4 a[0:3], v[0:7], v[8:15], a[0:3], v20, s0 op_sel_hi:[0,0,0]
 ; GCN-NEXT:    s_nop 7
-; GCN-NEXT:    s_nop 2
+; GCN-NEXT:    s_nop 3
 ; GCN-NEXT:    v_accvgpr_read_b32 v0, a0
 ; GCN-NEXT:    v_accvgpr_read_b32 v1, a1
 ; GCN-NEXT:    v_accvgpr_read_b32 v2, a2
@@ -1512,7 +1512,7 @@ define <4 x float> @test_mfma_scale_f32_16x16x128_f8f6f4_0_0_sgprs(<8 x i32> inr
 ; SDAG-NEXT:    s_nop 1
 ; SDAG-NEXT:    v_mfma_scale_f32_16x16x128_f8f6f4 a[0:3], v[12:19], v[4:11], a[0:3], v2, v3 op_sel_hi:[0,0,0]
 ; SDAG-NEXT:    s_nop 7
-; SDAG-NEXT:    s_nop 2
+; SDAG-NEXT:    s_nop 3
 ; SDAG-NEXT:    v_accvgpr_read_b32 v0, a0
 ; SDAG-NEXT:    v_accvgpr_read_b32 v1, a1
 ; SDAG-NEXT:    v_accvgpr_read_b32 v2, a2
@@ -1545,7 +1545,7 @@ define <4 x float> @test_mfma_scale_f32_16x16x128_f8f6f4_0_0_sgprs(<8 x i32> inr
 ; GISEL-NEXT:    s_nop 1
 ; GISEL-NEXT:    v_mfma_scale_f32_16x16x128_f8f6f4 a[0:3], v[4:11], v[12:19], a[0:3], v2, v3 op_sel_hi:[0,0,0]
 ; GISEL-NEXT:    s_nop 7
-; GISEL-NEXT:    s_nop 2
+; GISEL-NEXT:    s_nop 3
 ; GISEL-NEXT:    v_accvgpr_read_b32 v0, a0
 ; GISEL-NEXT:    v_accvgpr_read_b32 v1, a1
 ; GISEL-NEXT:    v_accvgpr_read_b32 v2, a2
@@ -1574,7 +1574,7 @@ define <4 x float> @test_mfma_scale_f32_16x16x128_f8f6f4_0_0_sgpr_vgpr_vgpr__sgp
 ; SDAG-NEXT:    s_nop 1
 ; SDAG-NEXT:    v_mfma_scale_f32_16x16x128_f8f6f4 a[0:3], v[14:21], v[0:7], a[0:3], s20, v12 op_sel_hi:[0,0,0]
 ; SDAG-NEXT:    s_nop 7
-; SDAG-NEXT:    s_nop 2
+; SDAG-NEXT:    s_nop 3
 ; SDAG-NEXT:    v_accvgpr_read_b32 v0, a0
 ; SDAG-NEXT:    v_accvgpr_read_b32 v1, a1
 ; SDAG-NEXT:    v_accvgpr_read_b32 v2, a2
@@ -1599,7 +1599,7 @@ define <4 x float> @test_mfma_scale_f32_16x16x128_f8f6f4_0_0_sgpr_vgpr_vgpr__sgp
 ; GISEL-NEXT:    s_nop 1
 ; GISEL-NEXT:    v_mfma_scale_f32_16x16x128_f8f6f4 a[0:3], v[14:21], v[0:7], a[0:3], s20, v12 op_sel_hi:[0,0,0]
 ; GISEL-NEXT:    s_nop 7
-; GISEL-NEXT:    s_nop 2
+; GISEL-NEXT:    s_nop 3
 ; GISEL-NEXT:    v_accvgpr_read_b32 v0, a0
 ; GISEL-NEXT:    v_accvgpr_read_b32 v1, a1
 ; GISEL-NEXT:    v_accvgpr_read_b32 v2, a2
@@ -1628,7 +1628,7 @@ define <4 x float> @test_mfma_scale_f32_16x16x128_f8f6f4_0_0_sgpr_vgpr_vgpr__vgp
 ; SDAG-NEXT:    s_nop 1
 ; SDAG-NEXT:    v_mfma_scale_f32_16x16x128_f8f6f4 a[0:3], v[14:21], v[0:7], a[0:3], v12, s20 op_sel_hi:[0,0,0]
 ; SDAG-NEXT:    s_nop 7
-; SDAG-NEXT:    s_nop 2
+; SDAG-NEXT:    s_nop 3
 ; SDAG-NEXT:    v_accvgpr_read_b32 v0, a0
 ; SDAG-NEXT:    v_accvgpr_read_b32 v1, a1
 ; SDAG-NEXT:    v_accvgpr_read_b32 v2, a2
@@ -1653,7 +1653,7 @@ define <4 x float> @test_mfma_scale_f32_16x16x128_f8f6f4_0_0_sgpr_vgpr_vgpr__vgp
 ; GISEL-NEXT:    s_nop 1
 ; GISEL-NEXT:    v_mfma_scale_f32_16x16x128_f8f6f4 a[0:3], v[14:21], v[0:7], a[0:3], v12, s20 op_sel_hi:[0,0,0]
 ; GISEL-NEXT:    s_nop 7
-; GISEL-NEXT:    s_nop 2
+; GISEL-NEXT:    s_nop 3
 ; GISEL-NEXT:    v_accvgpr_read_b32 v0, a0
 ; GISEL-NEXT:    v_accvgpr_read_b32 v1, a1
 ; GISEL-NEXT:    v_accvgpr_read_b32 v2, a2
@@ -1682,7 +1682,7 @@ define <4 x float> @test_mfma_scale_f32_16x16x128_f8f6f4_0_0_vgpr_sgpr_vgpr__vgp
 ; SDAG-NEXT:    s_nop 1
 ; SDAG-NEXT:    v_mfma_scale_f32_16x16x128_f8f6f4 a[0:3], v[0:7], v[14:21], a[0:3], v12, s20 op_sel_hi:[0,0,0]
 ; SDAG-NEXT:    s_nop 7
-; SDAG-NEXT:    s_nop 2
+; SDAG-NEXT:    s_nop 3
 ; SDAG-NEXT:    v_accvgpr_read_b32 v0, a0
 ; SDAG-NEXT:    v_accvgpr_read_b32 v1, a1
 ; SDAG-NEXT:    v_accvgpr_read_b32 v2, a2
@@ -1707,7 +1707,7 @@ define <4 x float> @test_mfma_scale_f32_16x16x128_f8f6f4_0_0_vgpr_sgpr_vgpr__vgp
 ; GISEL-NEXT:    s_nop 1
 ; GISEL-NEXT:    v_mfma_scale_f32_16x16x128_f8f6f4 a[0:3], v[0:7], v[14:21], a[0:3], v12, s20 op_sel_hi:[0,0,0]
 ; GISEL-NEXT:    s_nop 7
-; GISEL-NEXT:    s_nop 2
+; GISEL-NEXT:    s_nop 3
 ; GISEL-NEXT:    v_accvgpr_read_b32 v0, a0
 ; GISEL-NEXT:    v_accvgpr_read_b32 v1, a1
 ; GISEL-NEXT:    v_accvgpr_read_b32 v2, a2
@@ -1728,7 +1728,7 @@ define <4 x float> @test_mfma_scale_f32_16x16x128_f8f6f4_0_0_vgpr_vgpr_sgpr__vgp
 ; GCN-NEXT:    s_nop 1
 ; GCN-NEXT:    v_mfma_scale_f32_16x16x128_f8f6f4 a[0:3], v[0:7], v[8:15], a[0:3], v16, s16 op_sel_hi:[0,0,0]
 ; GCN-NEXT:    s_nop 7
-; GCN-NEXT:    s_nop 2
+; GCN-NEXT:    s_nop 3
 ; GCN-NEXT:    v_accvgpr_read_b32 v0, a0
 ; GCN-NEXT:    v_accvgpr_read_b32 v1, a1
 ; GCN-NEXT:    v_accvgpr_read_b32 v2, a2
@@ -1757,7 +1757,7 @@ define <4 x float> @test_mfma_scale_f32_16x16x128_f8f6f4_0_0_sgpr_vgpr_sgpr__vgp
 ; SDAG-NEXT:    s_nop 1
 ; SDAG-NEXT:    v_mfma_scale_f32_16x16x128_f8f6f4 a[0:3], v[10:17], v[0:7], a[0:3], v8, s24 op_sel_hi:[0,0,0]
 ; SDAG-NEXT:    s_nop 7
-; SDAG-NEXT:    s_nop 2
+; SDAG-NEXT:    s_nop 3
 ; SDAG-NEXT:    v_accvgpr_read_b32 v0, a0
 ; SDAG-NEXT:    v_accvgpr_read_b32 v1, a1
 ; SDAG-NEXT:    v_accvgpr_read_b32 v2, a2
@@ -1782,7 +1782,7 @@ define <4 x float> @test_mfma_scale_f32_16x16x128_f8f6f4_0_0_sgpr_vgpr_sgpr__vgp
 ; GISEL-NEXT:    s_nop 1
 ; GISEL-NEXT:    v_mfma_scale_f32_16x16x128_f8f6f4 a[0:3], v[10:17], v[0:7], a[0:3], v8, s24 op_sel_hi:[0,0,0]
 ; GISEL-NEXT:    s_nop 7
-; GISEL-NEXT:    s_nop 2
+; GISEL-NEXT:    s_nop 3
 ; GISEL-NEXT:    v_accvgpr_read_b32 v0, a0
 ; GISEL-NEXT:    v_accvgpr_read_b32 v1, a1
 ; GISEL-NEXT:    v_accvgpr_read_b32 v2, a2
@@ -1803,7 +1803,7 @@ define <4 x float> @test_mfma_scale_f32_16x16x128_f8f6f4_0_0__scaleA_inlineimm__
 ; GCN-NEXT:    s_nop 1
 ; GCN-NEXT:    v_mfma_scale_f32_16x16x128_f8f6f4 a[0:3], v[0:7], v[8:15], a[0:3], 33, -2 op_sel_hi:[0,0,0]
 ; GCN-NEXT:    s_nop 7
-; GCN-NEXT:    s_nop 2
+; GCN-NEXT:    s_nop 3
 ; GCN-NEXT:    v_accvgpr_read_b32 v0, a0
 ; GCN-NEXT:    v_accvgpr_read_b32 v1, a1
 ; GCN-NEXT:    v_accvgpr_read_b32 v2, a2
@@ -1825,7 +1825,7 @@ define <4 x float> @test_mfma_scale_f32_16x16x128_f8f6f4_0_0__scaleA_kimm__scale
 ; SDAG-NEXT:    s_nop 1
 ; SDAG-NEXT:    v_mfma_scale_f32_16x16x128_f8f6f4 a[0:3], v[0:7], v[8:15], a[0:3], s0, -2 op_sel_hi:[0,0,0]
 ; SDAG-NEXT:    s_nop 7
-; SDAG-NEXT:    s_nop 2
+; SDAG-NEXT:    s_nop 3
 ; SDAG-NEXT:    v_accvgpr_read_b32 v0, a0
 ; SDAG-NEXT:    v_accvgpr_read_b32 v1, a1
 ; SDAG-NEXT:    v_accvgpr_read_b32 v2, a2
@@ -1843,7 +1843,7 @@ define <4 x float> @test_mfma_scale_f32_16x16x128_f8f6f4_0_0__scaleA_kimm__scale
 ; GISEL-NEXT:    s_nop 1
 ; GISEL-NEXT:    v_mfma_scale_f32_16x16x128_f8f6f4 a[0:3], v[0:7], v[8:15], a[0:3], v16, -2 op_sel_hi:[0,0,0]
 ; GISEL-NEXT:    s_nop 7
-; GISEL-NEXT:    s_nop 2
+; GISEL-NEXT:    s_nop 3
 ; GISEL-NEXT:    v_accvgpr_read_b32 v0, a0
 ; GISEL-NEXT:    v_accvgpr_read_b32 v1, a1
 ; GISEL-NEXT:    v_accvgpr_read_b32 v2, a2
@@ -1866,7 +1866,7 @@ define <4 x float> @test_mfma_scale_f32_16x16x128_f8f6f4_0_0__scaleA_kimm__scale
 ; SDAG-NEXT:    s_nop 1
 ; SDAG-NEXT:    v_mfma_scale_f32_16x16x128_f8f6f4 a[0:3], v[0:7], v[8:15], a[0:3], s0, v16 op_sel_hi:[0,0,0]
 ; SDAG-NEXT:    s_nop 7
-; SDAG-NEXT:    s_nop 2
+; SDAG-NEXT:    s_nop 3
 ; SDAG-NEXT:    v_accvgpr_read_b32 v0, a0
 ; SDAG-NEXT:    v_accvgpr_read_b32 v1, a1
 ; SDAG-NEXT:    v_accvgpr_read_b32 v2, a2
@@ -1885,7 +1885,7 @@ define <4 x float> @test_mfma_scale_f32_16x16x128_f8f6f4_0_0__scaleA_kimm__scale
 ; GISEL-NEXT:    s_nop 1
 ; GISEL-NEXT:    v_mfma_scale_f32_16x16x128_f8f6f4 a[0:3], v[0:7], v[8:15], a[0:3], v16, v17 op_sel_hi:[0,0,0]
 ; GISEL-NEXT:    s_nop 7
-; GISEL-NEXT:    s_nop 2
+; GISEL-NEXT:    s_nop 3
 ; GISEL-NEXT:    v_accvgpr_read_b32 v0, a0
 ; GISEL-NEXT:    v_accvgpr_read_b32 v1, a1
 ; GISEL-NEXT:    v_accvgpr_read_b32 v2, a2
@@ -1927,7 +1927,7 @@ define amdgpu_kernel void @test_mfma_scale_f32_16x16x128_f8f6f4__vgprcd(<8 x i32
 ; SDAG-NEXT:    s_nop 1
 ; SDAG-NEXT:    v_mfma_scale_f32_16x16x128_f8f6f4 a[0:3], v[0:7], v[8:15], a[0:3], s12, v17 op_sel_hi:[0,0,0] blgp:2
 ; SDAG-NEXT:    s_nop 7
-; SDAG-NEXT:    s_nop 2
+; SDAG-NEXT:    s_nop 3
 ; SDAG-NEXT:    global_store_dwordx4 v16, a[0:3], s[14:15]
 ; SDAG-NEXT:    s_endpgm
 ;
@@ -1953,7 +1953,7 @@ define amdgpu_kernel void @test_mfma_scale_f32_16x16x128_f8f6f4__vgprcd(<8 x i32
 ; GISEL-NEXT:    v_mfma_scale_f32_16x16x128_f8f6f4 a[0:3], v[0:7], v[8:15], a[0:3], s28, v16 op_sel_hi:[0,0,0] blgp:2
 ; GISEL-NEXT:    v_mov_b32_e32 v0, 0
 ; GISEL-NEXT:    s_nop 7
-; GISEL-NEXT:    s_nop 1
+; GISEL-NEXT:    s_nop 2
 ; GISEL-NEXT:    global_store_dwordx4 v0, a[0:3], s[30:31]
 ; GISEL-NEXT:    s_endpgm
   %result = call <4 x float> @llvm.amdgcn.mfma.scale.f32.16x16x128.f8f6f4.v8i32.v8i32(<8 x i32> %arg0, <8 x i32> %arg1, <4 x float> %arg2, i32 0, i32 2, i32 3, i32 %scale0, i32 1, i32 %scale1)
@@ -1993,7 +1993,7 @@ define amdgpu_kernel void @test_mfma_scale_f32_16x16x128_f8f6f4__vgprcd___scaleA
 ; SDAG-NEXT:    s_nop 1
 ; SDAG-NEXT:    v_mfma_scale_f32_16x16x128_f8f6f4 a[0:3], v[0:7], v[8:15], a[0:3], s6, -2 op_sel_hi:[0,0,0]
 ; SDAG-NEXT:    s_nop 7
-; SDAG-NEXT:    s_nop 2
+; SDAG-NEXT:    s_nop 3
 ; SDAG-NEXT:    global_store_dwordx4 v16, a[0:3], s[4:5]
 ; SDAG-NEXT:    s_endpgm
 ;
@@ -2020,7 +2020,7 @@ define amdgpu_kernel void @test_mfma_scale_f32_16x16x128_f8f6f4__vgprcd___scaleA
 ; GISEL-NEXT:    v_mfma_scale_f32_16x16x128_f8f6f4 a[0:3], v[0:7], v[8:15], a[0:3], v16, -2 op_sel_hi:[0,0,0]
 ; GISEL-NEXT:    v_mov_b32_e32 v0, 0
 ; GISEL-NEXT:    s_nop 7
-; GISEL-NEXT:    s_nop 1
+; GISEL-NEXT:    s_nop 2
 ; GISEL-NEXT:    global_store_dwordx4 v0, a[0:3], s[4:5]
 ; GISEL-NEXT:    s_endpgm
   %result = call <4 x float> @llvm.amdgcn.mfma.scale.f32.16x16x128.f8f6f4.v8i32.v8i32(<8 x i32> %arg0, <8 x i32> %arg1, <4 x float> %arg2, i32 0, i32 0, i32 3, i32 65, i32 1, i32 -2)
@@ -2040,7 +2040,7 @@ define <4 x float> @test_mfma_scale_f32_16x16x128_f8f6f4___constant_scale_0_0_a(
 ; GCN-NEXT:    s_nop 1
 ; GCN-NEXT:    v_mfma_f32_16x16x128_f8f6f4 a[0:3], v[0:7], v[8:15], a[0:3]
 ; GCN-NEXT:    s_nop 7
-; GCN-NEXT:    s_nop 2
+; GCN-NEXT:    s_nop 3
 ; GCN-NEXT:    v_accvgpr_read_b32 v0, a0
 ; GCN-NEXT:    v_accvgpr_read_b32 v1, a1
 ; GCN-NEXT:    v_accvgpr_read_b32 v2, a2
@@ -2062,7 +2062,7 @@ define <4 x float> @test_mfma_scale_f32_16x16x128_f8f6f4___constant_scale_0_0_b(
 ; GCN-NEXT:    s_nop 1
 ; GCN-NEXT:    v_mfma_f32_16x16x128_f8f6f4 a[0:3], v[0:7], v[8:15], a[0:3]
 ; GCN-NEXT:    s_nop 7
-; GCN-NEXT:    s_nop 2
+; GCN-NEXT:    s_nop 3
 ; GCN-NEXT:    v_accvgpr_read_b32 v0, a0
 ; GCN-NEXT:    v_accvgpr_read_b32 v1, a1
 ; GCN-NEXT:    v_accvgpr_read_b32 v2, a2
@@ -2083,7 +2083,7 @@ define <4 x float> @test_mfma_scale_f32_16x16x128_f8f6f4___constant_scale_0_1(<8
 ; GCN-NEXT:    s_nop 1
 ; GCN-NEXT:    v_mfma_scale_f32_16x16x128_f8f6f4 a[0:3], v[0:7], v[8:15], a[0:3], 0, 1 op_sel_hi:[0,0,0]
 ; GCN-NEXT:    s_nop 7
-; GCN-NEXT:    s_nop 2
+; GCN-NEXT:    s_nop 3
 ; GCN-NEXT:    v_accvgpr_read_b32 v0, a0
 ; GCN-NEXT:    v_accvgpr_read_b32 v1, a1
 ; GCN-NEXT:    v_accvgpr_read_b32 v2, a2
@@ -2104,7 +2104,7 @@ define <4 x float> @test_mfma_scale_f32_16x16x128_f8f6f4___constant_scale_1_0_a(
 ; GCN-NEXT:    s_nop 1
 ; GCN-NEXT:    v_mfma_scale_f32_16x16x128_f8f6f4 a[0:3], v[0:7], v[8:15], a[0:3], 1, 0 op_sel_hi:[0,0,0]
 ; GCN-NEXT:    s_nop 7
-; GCN-NEXT:    s_nop 2
+; GCN-NEXT:    s_nop 3
 ; GCN-NEXT:    v_accvgpr_read_b32 v0, a0
 ; GCN-NEXT:    v_accvgpr_read_b32 v1, a1
 ; GCN-NEXT:    v_accvgpr_read_b32 v2, a2
@@ -2129,7 +2129,7 @@ define <4 x float> @test_mfma_scale_f32_16x16x128_f8f6f4___v8i32_fp8__v8i32_fp6(
 ; GCN-NEXT:    s_nop 1
 ; GCN-NEXT:    v_mfma_scale_f32_16x16x128_f8f6f4 a[0:3], v[0:7], v[8:15], a[0:3], v20, v21 op_sel_hi:[0,0,0] blgp:2
 ; GCN-NEXT:    s_nop 7
-; GCN-NEXT:    s_nop 2
+; GCN-NEXT:    s_nop 3
 ; GCN-NEXT:    v_accvgpr_read_b32 v0, a0
 ; GCN-NEXT:    v_accvgpr_read_b32 v1, a1
 ; GCN-NEXT:    v_accvgpr_read_b32 v2, a2
@@ -2153,7 +2153,7 @@ define <4 x float> @test_mfma_scale_f32_16x16x128_f8f6f4___v8i32_fp6__v8i32_fp8(
 ; GCN-NEXT:    s_nop 1
 ; GCN-NEXT:    v_mfma_scale_f32_16x16x128_f8f6f4 a[0:3], v[0:7], v[8:15], a[0:3], v20, v21 op_sel_hi:[0,0,0] cbsz:2
 ; GCN-NEXT:    s_nop 7
-; GCN-NEXT:    s_nop 2
+; GCN-NEXT:    s_nop 3
 ; GCN-NEXT:    v_accvgpr_read_b32 v0, a0
 ; GCN-NEXT:    v_accvgpr_read_b32 v1, a1
 ; GCN-NEXT:    v_accvgpr_read_b32 v2, a2
@@ -2176,7 +2176,7 @@ define <4 x float> @test_mfma_scale_f32_16x16x128_f8f6f4___v8i32_fp6__v8i32_fp6(
 ; GCN-NEXT:    v_accvgpr_write_b32 a3, v19
 ; GCN-NEXT:    s_nop 1
 ; GCN-NEXT:    v_mfma_scale_f32_16x16x128_f8f6f4 a[0:3], v[0:7], v[8:15], a[0:3], v20, v21 op_sel_hi:[0,0,0] cbsz:2 blgp:2
-; GCN-NEXT:    s_nop 6
+; GCN-NEXT:    s_nop 7
 ; GCN-NEXT:    v_accvgpr_read_b32 v0, a0
 ; GCN-NEXT:    v_accvgpr_read_b32 v1, a1
 ; GCN-NEXT:    v_accvgpr_read_b32 v2, a2
@@ -2199,7 +2199,7 @@ define <4 x float> @test_mfma_scale_f32_16x16x128_f8f6f4___v8i32_fp6__v8i32_fp6_
 ; GCN-NEXT:    v_accvgpr_write_b32 a3, v19
 ; GCN-NEXT:    s_nop 1
 ; GCN-NEXT:    v_mfma_f32_16x16x128_f8f6f4 a[0:3], v[0:7], v[8:15], a[0:3] cbsz:2 blgp:2
-; GCN-NEXT:    s_nop 6
+; GCN-NEXT:    s_nop 7
 ; GCN-NEXT:    v_accvgpr_read_b32 v0, a0
 ; GCN-NEXT:    v_accvgpr_read_b32 v1, a1
 ; GCN-NEXT:    v_accvgpr_read_b32 v2, a2
@@ -2223,7 +2223,7 @@ define <4 x float> @test_mfma_scale_f32_16x16x128_f8f6f4___v8i32_fp8__v8i32_fp4(
 ; GCN-NEXT:    s_nop 1
 ; GCN-NEXT:    v_mfma_scale_f32_16x16x128_f8f6f4 a[0:3], v[0:7], v[8:15], a[0:3], v20, v21 op_sel_hi:[0,0,0] blgp:4
 ; GCN-NEXT:    s_nop 7
-; GCN-NEXT:    s_nop 2
+; GCN-NEXT:    s_nop 3
 ; GCN-NEXT:    v_accvgpr_read_b32 v0, a0
 ; GCN-NEXT:    v_accvgpr_read_b32 v1, a1
 ; GCN-NEXT:    v_accvgpr_read_b32 v2, a2
@@ -2247,7 +2247,7 @@ define <4 x float> @test_mfma_scale_f32_16x16x128_f8f6f4___v8i32_fp4__v8i32_fp8(
 ; GCN-NEXT:    s_nop 1
 ; GCN-NEXT:    v_mfma_scale_f32_16x16x128_f8f6f4 a[0:3], v[0:7], v[8:15], a[0:3], v20, v21 op_sel_hi:[0,0,0] cbsz:4
 ; GCN-NEXT:    s_nop 7
-; GCN-NEXT:    s_nop 2
+; GCN-NEXT:    s_nop 3
 ; GCN-NEXT:    v_accvgpr_read_b32 v0, a0
 ; GCN-NEXT:    v_accvgpr_read_b32 v1, a1
 ; GCN-NEXT:    v_accvgpr_read_b32 v2, a2
@@ -2271,7 +2271,7 @@ define <4 x float> @test_mfma_scale_f32_16x16x128_f8f6f4___v8i32_fp8__v6i32_fp4(
 ; GCN-NEXT:    s_nop 1
 ; GCN-NEXT:    v_mfma_scale_f32_16x16x128_f8f6f4 a[0:3], v[0:7], v[8:13], a[0:3], v18, v19 op_sel_hi:[0,0,0] blgp:4
 ; GCN-NEXT:    s_nop 7
-; GCN-NEXT:    s_nop 2
+; GCN-NEXT:    s_nop 3
 ; GCN-NEXT:    v_accvgpr_read_b32 v0, a0
 ; GCN-NEXT:    v_accvgpr_read_b32 v1, a1
 ; GCN-NEXT:    v_accvgpr_read_b32 v2, a2
@@ -2295,7 +2295,7 @@ define <4 x float> @test_mfma_scale_f32_16x16x128_f8f6f4___v6i32_fp4__v8i32_fp8(
 ; GCN-NEXT:    s_nop 1
 ; GCN-NEXT:    v_mfma_scale_f32_16x16x128_f8f6f4 a[0:3], v[0:5], v[6:13], a[0:3], v18, v19 op_sel_hi:[0,0,0] cbsz:4
 ; GCN-NEXT:    s_nop 7
-; GCN-NEXT:    s_nop 2
+; GCN-NEXT:    s_nop 3
 ; GCN-NEXT:    v_accvgpr_read_b32 v0, a0
 ; GCN-NEXT:    v_accvgpr_read_b32 v1, a1
 ; GCN-NEXT:    v_accvgpr_read_b32 v2, a2
@@ -2318,7 +2318,7 @@ define <4 x float> @test_mfma_scale_f32_16x16x128_f8f6f4___v8i32_fp4__v8i32_fp4(
 ; GCN-NEXT:    v_accvgpr_write_b32 a3, v19
 ; GCN-NEXT:    s_nop 1
 ; GCN-NEXT:    v_mfma_scale_f32_16x16x128_f8f6f4 a[0:3], v[0:7], v[8:15], a[0:3], v20, v21 op_sel_hi:[0,0,0] cbsz:4 blgp:4
-; GCN-NEXT:    s_nop 6
+; GCN-NEXT:    s_nop 7
 ; GCN-NEXT:    v_accvgpr_read_b32 v0, a0
 ; GCN-NEXT:    v_accvgpr_read_b32 v1, a1
 ; GCN-NEXT:    v_accvgpr_read_b32 v2, a2
@@ -2341,7 +2341,7 @@ define <4 x float> @test_mfma_scale_f32_16x16x128_f8f6f4___v8i32_fp4__v8i32_fp4_
 ; GCN-NEXT:    v_accvgpr_write_b32 a3, v19
 ; GCN-NEXT:    s_nop 1
 ; GCN-NEXT:    v_mfma_f32_16x16x128_f8f6f4 a[0:3], v[0:7], v[8:15], a[0:3] cbsz:4 blgp:4
-; GCN-NEXT:    s_nop 6
+; GCN-NEXT:    s_nop 7
 ; GCN-NEXT:    v_accvgpr_read_b32 v0, a0
 ; GCN-NEXT:    v_accvgpr_read_b32 v1, a1
 ; GCN-NEXT:    v_accvgpr_read_b32 v2, a2
@@ -2365,4 +2365,4 @@ declare <4 x float> @llvm.amdgcn.mfma.scale.f32.16x16x128.f8f6f4.v8i32.v4i32(<8
 declare <4 x float> @llvm.amdgcn.mfma.scale.f32.16x16x128.f8f6f4.v8i32.v6i32(<8 x i32>, <6 x i32>, <4 x float>, i32 immarg, i32 immarg, i32 immarg, i32, i32 immarg, i32) #1
 
 attributes #0 = { "amdgpu-flat-work-group-size"="512,512" }
-attributes #1 = { convergent nocallback nofree nosync nounwind willreturn memory(none) }
+attributes #1 = { convergent nocallback nofree nosync nounwind willreturn memory(none) }
\ No newline at end of file
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.scale.f32.32x32x64.f8f6f4.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.scale.f32.32x32x64.f8f6f4.ll
index 05f8739e7cb89..3d959393a8fa7 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.scale.f32.32x32x64.f8f6f4.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.scale.f32.32x32x64.f8f6f4.ll
@@ -40,7 +40,7 @@ define <16 x float> @test_mfma_scale_f32_32x32x64_f8f6f4_0_0__cbsz0__blgp0(<8 x
 ; SDAG-NEXT:    v_mfma_scale_f32_32x32x64_f8f6f4 a[0:15], v[0:7], v[8:15], a[0:15], v32, v31 op_sel_hi:[0,0,0]
 ; SDAG-NEXT:    s_nop 7
 ; SDAG-NEXT:    s_nop 7
-; SDAG-NEXT:    s_nop 2
+; SDAG-NEXT:    s_nop 3
 ; SDAG-NEXT:    v_accvgpr_read_b32 v0, a0
 ; SDAG-NEXT:    v_accvgpr_read_b32 v1, a1
 ; SDAG-NEXT:    v_accvgpr_read_b32 v2, a2
@@ -85,7 +85,7 @@ define <16 x float> @test_mfma_scale_f32_32x32x64_f8f6f4_0_0__cbsz0__blgp0(<8 x
 ; GISEL-NEXT:    v_mfma_scale_f32_32x32x64_f8f6f4 a[0:15], v[0:7], v[8:15], a[0:15], v31, v32 op_sel_hi:[0,0,0]
 ; GISEL-NEXT:    s_nop 7
 ; GISEL-NEXT:    s_nop 7
-; GISEL-NEXT:    s_nop 2
+; GISEL-NEXT:    s_nop 3
 ; GISEL-NEXT:    v_accvgpr_read_b32 v0, a0
 ; GISEL-NEXT:    v_accvgpr_read_b32 v1, a1
 ; GISEL-NEXT:    v_accvgpr_read_b32 v2, a2
@@ -137,7 +137,7 @@ define <16 x float> @test_mfma_scale_f32_32x32x64_f8f6f4_1_1__cbsz1__blgp1(<8 x
 ; SDAG-NEXT:    v_mfma_scale_f32_32x32x64_f8f6f4 a[0:15], v[0:7], v[8:15], a[0:15], v32, v31 op_sel_hi:[0,0,0]
 ; SDAG-NEXT:    s_nop 7
 ; SDAG-NEXT:    s_nop 7
-; SDAG-NEXT:    s_nop 2
+; SDAG-NEXT:    s_nop 3
 ; SDAG-NEXT:    v_accvgpr_read_b32 v0, a0
 ; SDAG-NEXT:    v_accvgpr_read_b32 v1, a1
 ; SDAG-NEXT:    v_accvgpr_read_b32 v2, a2
@@ -182,7 +182,7 @@ define <16 x float> @test_mfma_scale_f32_32x32x64_f8f6f4_1_1__cbsz1__blgp1(<8 x
 ; GISEL-NEXT:    v_mfma_scale_f32_32x32x64_f8f6f4 a[0:15], v[0:7], v[8:15], a[0:15], v31, v32 op_sel_hi:[0,0,0]
 ; GISEL-NEXT:    s_nop 7
 ; GISEL-NEXT:    s_nop 7
-; GISEL-NEXT:    s_nop 2
+; GISEL-NEXT:    s_nop 3
 ; GISEL-NEXT:    v_accvgpr_read_b32 v0, a0
 ; GISEL-NEXT:    v_accvgpr_read_b32 v1, a1
 ; GISEL-NEXT:    v_accvgpr_read_b32 v2, a2
@@ -234,7 +234,7 @@ define <16 x float> @test_mfma_scale_f32_32x32x64_f8f6f4_2_2__cbsz1__blgp1(<8 x
 ; SDAG-NEXT:    v_mfma_scale_f32_32x32x64_f8f6f4 a[0:15], v[0:7], v[8:15], a[0:15], v32, v31 op_sel_hi:[0,0,0]
 ; SDAG-NEXT:    s_nop 7
 ; SDAG-NEXT:    s_nop 7
-; SDAG-NEXT:    s_nop 2
+; SDAG-NEXT:    s_nop 3
 ; SDAG-NEXT:    v_accvgpr_read_b32 v0, a0
 ; SDAG-NEXT:    v_accvgpr_read_b32 v1, a1
 ; SDAG-NEXT:    v_accvgpr_read_b32 v2, a2
@@ -279,7 +279,7 @@ define <16 x float> @test_mfma_scale_f32_32x32x64_f8f6f4_2_2__cbsz1__blgp1(<8 x
 ; GISEL-NEXT:    v_mfma_scale_f32_32x32x64_f8f6f4 a[0:15], v[0:7], v[8:15], a[0:15], v31, v32 op_sel_hi:[0,0,0]
 ; GISEL-NEXT:    s_nop 7
 ; GISEL-NEXT:    s_nop 7
-; GISEL-NEXT:    s_nop 2
+; GISEL-NEXT:    s_nop 3
 ; GISEL-NEXT:    v_accvgpr_read_b32 v0, a0
 ; GISEL-NEXT:    v_accvgpr_read_b32 v1, a1
 ; GISEL-NEXT:    v_accvgpr_read_b32 v2, a2
@@ -331,7 +331,7 @@ define <16 x float> @test_mfma_scale_f32_32x32x64_f8f6f4_3_3__cbsz1__blgp1(<8 x
 ; SDAG-NEXT:    v_mfma_scale_f32_32x32x64_f8f6f4 a[0:15], v[0:7], v[8:15], a[0:15], v32, v31 op_sel_hi:[0,0,0]
 ; SDAG-NEXT:    s_nop 7
 ; SDAG-NEXT:    s_nop 7
-; SDAG-NEXT:    s_nop 2
+; SDAG-NEXT:    s_nop 3
 ; SDAG-NEXT:    v_accvgpr_read_b32 v0, a0
 ; SDAG-NEXT:    v_accvgpr_read_b32 v1, a1
 ; SDAG-NEXT:    v_accvgpr_read_b32 v2, a2
@@ -376,7 +376,7 @@ define <16 x float> @test_mfma_scale_f32_32x32x64_f8f6f4_3_3__cbsz1__blgp1(<8 x
 ; GISEL-NEXT:    v_mfma_scale_f32_32x32x64_f8f6f4 a[0:15], v[0:7], v[8:15], a[0:15], v31, v32 op_sel_hi:[0,0,0]
 ; GISEL-NEXT:    s_nop 7
 ; GISEL-NEXT:    s_nop 7
-; GISEL-NEXT:    s_nop 2
+; GISEL-NEXT:    s_nop 3
 ; GISEL-NEXT:    v_accvgpr_read_b32 v0, a0
 ; GISEL-NEXT:    v_accvgpr_read_b32 v1, a1
 ; GISEL-NEXT:    v_accvgpr_read_b32 v2, a2
@@ -428,7 +428,7 @@ define <16 x float> @test_mfma_scale_f32_32x32x64_f8f6f4_0_3__cbsz1__blgp1(<8 x
 ; SDAG-NEXT:    v_mfma_scale_f32_32x32x64_f8f6f4 a[0:15], v[0:7], v[8:15], a[0:15], v32, v31 op_sel_hi:[0,0,0]
 ; SDAG-NEXT:    s_nop 7
 ; SDAG-NEXT:    s_nop 7
-; SDAG-NEXT:    s_nop 2
+; SDAG-NEXT:    s_nop 3
 ; SDAG-NEXT:    v_accvgpr_read_b32 v0, a0
 ; SDAG-NEXT:    v_accvgpr_read_b32 v1, a1
 ; SDAG-NEXT:    v_accvgpr_read_b32 v2, a2
@@ -473,7 +473,7 @@ define <16 x float> @test_mfma_scale_f32_32x32x64_f8f6f4_0_3__cbsz1__blgp1(<8 x
 ; GISEL-NEXT:    v_mfma_scale_f32_32x32x64_f8f6f4 a[0:15], v[0:7], v[8:15], a[0:15], v31, v32 op_sel_hi:[0,0,0]
 ; GISEL-NEXT:    s_nop 7
 ; GISEL-NEXT:    s_nop 7
-; GISEL-NEXT:    s_nop 2
+; GISEL-NEXT:    s_nop 3
 ; GISEL-NEXT:    v_accvgpr_read_b32 v0, a0
 ; GISEL-NEXT:    v_accvgpr_read_b32 v1, a1
 ; GISEL-NEXT:    v_accvgpr_read_b32 v2, a2
@@ -525,7 +525,7 @@ define <16 x float> @test_mfma_scale_f32_32x32x64_f8f6f4_3_0__cbsz1__blgp1(<8 x
 ; SDAG-NEXT:    v_mfma_scale_f32_32x32x64_f8f6f4 a[0:15], v[0:7], v[8:15], a[0:15], v32, v31 op_sel_hi:[0,0,0]
 ; SDAG-NEXT:    s_nop 7
 ; SDAG-NEXT:    s_nop 7
-; SDAG-NEXT:    s_nop 2
+; SDAG-NEXT:    s_nop 3
 ; SDAG-NEXT:    v_accvgpr_read_b32 v0, a0
 ; SDAG-NEXT:    v_accvgpr_read_b32 v1, a1
 ; SDAG-NEXT:    v_accvgpr_read_b32 v2, a2
@@ -570,7 +570,7 @@ define <16 x float> @test_mfma_scale_f32_32x32x64_f8f6f4_3_0__cbsz1__blgp1(<8 x
 ; GISEL-NEXT:    v_mfma_scale_f32_32x32x64_f8f6f4 a[0:15], v[0:7], v[8:15], a[0:15], v31, v32 op_sel_hi:[0,0,0]
 ; GISEL-NEXT:    s_nop 7
 ; GISEL-NEXT:    s_nop 7
-; GISEL-NEXT:    s_nop 2
+; GISEL-NEXT:    s_nop 3
 ; GISEL-NEXT:    v_accvgpr_read_b32 v0, a0
 ; GISEL-NEXT:    v_accvgpr_read_b32 v1, a1
 ; GISEL-NEXT:    v_accvgpr_read_b32 v2, a2
@@ -622,7 +622,7 @@ define <16 x float> @test_mfma_scale_f32_32x32x64_f8f6f4_2_3__cbsz1__blgp1(<8 x
 ; SDAG-NEXT:    v_mfma_scale_f32_32x32x64_f8f6f4 a[0:15], v[0:7], v[8:15], a[0:15], v32, v31 op_sel_hi:[0,0,0]
 ; SDAG-NEXT:    s_nop 7
 ; SDAG-NEXT:    s_nop 7
-; SDAG-NEXT:    s_nop 2
+; SDAG-NEXT:    s_nop 3
 ; SDAG-NEXT:    v_accvgpr_read_b32 v0, a0
 ; SDAG-NEXT:    v_accvgpr_read_b32 v1, a1
 ; SDAG-NEXT:    v_accvgpr_read_b32 v2, a2
@@ -667,7 +667,7 @@ define <16 x float> @test_mfma_scale_f32_32x32x64_f8f6f4_2_3__cbsz1__blgp1(<8 x
 ; GISEL-NEXT:    v_mfma_scale_f32_32x32x64_f8f6f4 a[0:15], v[0:7], v[8:15], a[0:15], v31, v32 op_sel_hi:[0,0,0]
 ; GISEL-NEXT:    s_nop 7
 ; GISEL-NEXT:    s_nop 7
-; GISEL-NEXT:    s_nop 2
+; GISEL-NEXT:    s_nop 3
 ; GISEL-NEXT:    v_accvgpr_read_b32 v0, a0
 ; GISEL-NEXT:    v_accvgpr_read_b32 v1, a1
 ; GISEL-NEXT:    v_accvgpr_read_b32 v2, a2
@@ -719,7 +719,7 @@ define <16 x float> @test_mfma_scale_f32_32x32x64_f8f6f4_3_2__cbsz1__blgp1(<8 x
 ; SDAG-NEXT:    v_mfma_scale_f32_32x32x64_f8f6f4 a[0:15], v[0:7], v[8:15], a[0:15], v32, v31 op_sel_hi:[0,0,0]
 ; SDAG-NEXT:    s_nop 7
 ; SDAG-NEXT:    s_nop 7
-; SDAG-NEXT:    s_nop 2
+; SDAG-NEXT:    s_nop 3
 ; SDAG-NEXT:    v_accvgpr_read_b32 v0, a0
 ; SDAG-NEXT:    v_accvgpr_read_b32 v1, a1
 ; SDAG-NEXT:    v_accvgpr_read_b32 v2, a2
@@ -764,7 +764,7 @@ define <16 x float> @test_mfma_scale_f32_32x32x64_f8f6f4_3_2__cbsz1__blgp1(<8 x
 ; GISEL-NEXT:    v_mfma_scale_f32_32x32x64_f8f6f4 a[0:15], v[0:7], v[8:15], a[0:15], v31, v32 op_sel_hi:[0,0,0]
 ; GISEL-NEXT:    s_nop 7
 ; GISEL-NEXT:    s_nop 7
-; GISEL-NEXT:    s_nop 2
+; GISEL-NEXT:    s_nop 3
 ; GISEL-NEXT:    v_accvgpr_read_b32 v0, a0
 ; GISEL-NEXT:    v_accvgpr_read_b32 v1, a1
 ; GISEL-NEXT:    v_accvgpr_read_b32 v2, a2
@@ -815,7 +815,7 @@ define <16 x float> @test_mfma_scale_f32_32x32x64_f8f6f4_0_0__cbsz0__blgp0__cons
 ; GCN-NEXT:    v_mfma_f32_32x32x64_f8f6f4 a[0:15], v[0:7], v[8:15], a[0:15]
 ; GCN-NEXT:    s_nop 7
 ; GCN-NEXT:    s_nop 7
-; GCN-NEXT:    s_nop 2
+; GCN-NEXT:    s_nop 3
 ; GCN-NEXT:    v_accvgpr_read_b32 v0, a0
 ; GCN-NEXT:    v_accvgpr_read_b32 v1, a1
 ; GCN-NEXT:    v_accvgpr_read_b32 v2, a2
@@ -868,7 +868,7 @@ define <16 x float> @test_mfma_scale_f32_32x32x64_f8f6f4_0_0__cbsz0__blgp1(<8 x
 ; SDAG-NEXT:    v_mfma_scale_f32_32x32x64_f8f6f4 a[0:15], v[0:7], v[8:15], a[0:15], v32, v31 op_sel_hi:[0,0,0] blgp:1
 ; SDAG-NEXT:    s_nop 7
 ; SDAG-NEXT:    s_nop 7
-; SDAG-NEXT:    s_nop 2
+; SDAG-NEXT:    s_nop 3
 ; SDAG-NEXT:    v_accvgpr_read_b32 v0, a0
 ; SDAG-NEXT:    v_accvgpr_read_b32 v1, a1
 ; SDAG-NEXT:    v_accvgpr_read_b32 v2, a2
@@ -913,7 +913,7 @@ define <16 x float> @test_mfma_scale_f32_32x32x64_f8f6f4_0_0__cbsz0__blgp1(<8 x
 ; GISEL-NEXT:    v_mfma_scale_f32_32x32x64_f8f6f4 a[0:15], v[0:7], v[8:15], a[0:15], v31, v32 op_sel_hi:[0,0,0] blgp:1
 ; GISEL-NEXT:    s_nop 7
 ; GISEL-NEXT:    s_nop 7
-; GISEL-NEXT:    s_nop 2
+; GISEL-NEXT:    s_nop 3
 ; GISEL-NEXT:    v_accvgpr_read_b32 v0, a0
 ; GISEL-NEXT:    v_accvgpr_read_b32 v1, a1
 ; GISEL-NEXT:    v_accvgpr_read_b32 v2, a2
@@ -963,7 +963,7 @@ define <16 x float> @test_mfma_scale_f32_32x32x64_f8f6f4_0_0__cbsz0__blgp1__cons
 ; GCN-NEXT:    v_mfma_f32_32x32x64_f8f6f4 a[0:15], v[0:7], v[8:15], a[0:15] blgp:1
 ; GCN-NEXT:    s_nop 7
 ; GCN-NEXT:    s_nop 7
-; GCN-NEXT:    s_nop 2
+; GCN-NEXT:    s_nop 3
 ; GCN-NEXT:    v_accvgpr_read_b32 v0, a0
 ; GCN-NEXT:    v_accvgpr_read_b32 v1, a1
 ; GCN-NEXT:    v_accvgpr_read_b32 v2, a2
@@ -1015,7 +1015,7 @@ define <16 x float> @test_mfma_scale_f32_32x32x64_f8f6f4_0_0__cbsz0__blgp2(<8 x
 ; GCN-NEXT:    v_mfma_scale_f32_32x32x64_f8f6f4 a[0:15], v[0:7], v[8:13], a[0:15], v30, v31 op_sel_hi:[0,0,0] blgp:2
 ; GCN-NEXT:    s_nop 7
 ; GCN-NEXT:    s_nop 7
-; GCN-NEXT:    s_nop 2
+; GCN-NEXT:    s_nop 3
 ; GCN-NEXT:    v_accvgpr_read_b32 v0, a0
 ; GCN-NEXT:    v_accvgpr_read_b32 v1, a1
 ; GCN-NEXT:    v_accvgpr_read_b32 v2, a2
@@ -1064,7 +1064,7 @@ define <16 x float> @test_mfma_scale_f32_32x32x64_f8f6f4_0_0__cbsz0__blgp2__cons
 ; GCN-NEXT:    v_mfma_f32_32x32x64_f8f6f4 a[0:15], v[0:7], v[8:13], a[0:15] blgp:2
 ; GCN-NEXT:    s_nop 7
 ; GCN-NEXT:    s_nop 7
-; GCN-NEXT:    s_nop 2
+; GCN-NEXT:    s_nop 3
 ; GCN-NEXT:    v_accvgpr_read_b32 v0, a0
 ; GCN-NEXT:    v_accvgpr_read_b32 v1, a1
 ; GCN-NEXT:    v_accvgpr_read_b32 v2, a2
@@ -1116,7 +1116,7 @@ define <16 x float> @test_mfma_scale_f32_32x32x64_f8f6f4_0_0__cbsz0__blgp3(<8 x
 ; GCN-NEXT:    v_mfma_scale_f32_32x32x64_f8f6f4 a[0:15], v[0:7], v[8:13], a[0:15], v30, v31 op_sel_hi:[0,0,0] blgp:3
 ; GCN-NEXT:    s_nop 7
 ; GCN-NEXT:    s_nop 7
-; GCN-NEXT:    s_nop 2
+; GCN-NEXT:    s_nop 3
 ; GCN-NEXT:    v_accvgpr_read_b32 v0, a0
 ; GCN-NEXT:    v_accvgpr_read_b32 v1, a1
 ; GCN-NEXT:    v_accvgpr_read_b32 v2, a2
@@ -1165,7 +1165,7 @@ define <16 x float> @test_mfma_scale_f32_32x32x64_f8f6f4_0_0__cbsz0__blgp3__cons
 ; GCN-NEXT:    v_mfma_f32_32x32x64_f8f6f4 a[0:15], v[0:7], v[8:13], a[0:15] blgp:3
 ; GCN-NEXT:    s_nop 7
 ; GCN-NEXT:    s_nop 7
-; GCN-NEXT:    s_nop 2
+; GCN-NEXT:    s_nop 3
 ; GCN-NEXT:    v_accvgpr_read_b32 v0, a0
 ; GCN-NEXT:    v_accvgpr_read_b32 v1, a1
 ; GCN-NEXT:    v_accvgpr_read_b32 v2, a2
@@ -1215,7 +1215,7 @@ define <16 x float> @test_mfma_scale_f32_32x32x64_f8f6f4_0_0__cbsz0__blgp4(<8 x
 ; GCN-NEXT:    v_mfma_scale_f32_32x32x64_f8f6f4 a[0:15], v[0:7], v[8:11], a[0:15], v28, v29 op_sel_hi:[0,0,0] blgp:4
 ; GCN-NEXT:    s_nop 7
 ; GCN-NEXT:    s_nop 7
-; GCN-NEXT:    s_nop 2
+; GCN-NEXT:    s_nop 3
 ; GCN-NEXT:    v_accvgpr_read_b32 v0, a0
 ; GCN-NEXT:    v_accvgpr_read_b32 v1, a1
 ; GCN-NEXT:    v_accvgpr_read_b32 v2, a2
@@ -1264,7 +1264,7 @@ define <16 x float> @test_mfma_scale_f32_32x32x64_f8f6f4_0_0__cbsz0__blgp4__cons
 ; GCN-NEXT:    v_mfma_f32_32x32x64_f8f6f4 a[0:15], v[0:7], v[8:11], a[0:15] blgp:4
 ; GCN-NEXT:    s_nop 7
 ; GCN-NEXT:    s_nop 7
-; GCN-NEXT:    s_nop 2
+; GCN-NEXT:    s_nop 3
 ; GCN-NEXT:    v_accvgpr_read_b32 v0, a0
 ; GCN-NEXT:    v_accvgpr_read_b32 v1, a1
 ; GCN-NEXT:    v_accvgpr_read_b32 v2, a2
@@ -1317,7 +1317,7 @@ define <16 x float> @test_mfma_scale_f32_32x32x64_f8f6f4_0_0__cbsz1__blgp0(<8 x
 ; SDAG-NEXT:    v_mfma_scale_f32_32x32x64_f8f6f4 a[0:15], v[0:7], v[8:15], a[0:15], v32, v31 op_sel_hi:[0,0,0] cbsz:1
 ; SDAG-NEXT:    s_nop 7
 ; SDAG-NEXT:    s_nop 7
-; SDAG-NEXT:    s_nop 2
+; SDAG-NEXT:    s_nop 3
 ; SDAG-NEXT:    v_accvgpr_read_b32 v0, a0
 ; SDAG-NEXT:    v_accvgpr_read_b32 v1, a1
 ; SDAG-NEXT:    v_accvgpr_read_b32 v2, a2
@@ -1362,7 +1362,7 @@ define <16 x float> @test_mfma_scale_f32_32x32x64_f8f6f4_0_0__cbsz1__blgp0(<8 x
 ; GISEL-NEXT:    v_mfma_scale_f32_32x32x64_f8f6f4 a[0:15], v[0:7], v[8:15], a[0:15], v31, v32 op_sel_hi:[0,0,0] cbsz:1
 ; GISEL-NEXT:    s_nop 7
 ; GISEL-NEXT:    s_nop 7
-; GISEL-NEXT:    s_nop 2
+; GISEL-NEXT:    s_nop 3
 ; GISEL-NEXT:    v_accvgpr_read_b32 v0, a0
 ; GISEL-NEXT:    v_accvgpr_read_b32 v1, a1
 ; GISEL-NEXT:    v_accvgpr_read_b32 v2, a2
@@ -1412,7 +1412,7 @@ define <16 x float> @test_mfma_scale_f32_32x32x64_f8f6f4_0_0__cbsz1__blgp0__cons
 ; GCN-NEXT:    v_mfma_f32_32x32x64_f8f6f4 a[0:15], v[0:7], v[8:15], a[0:15] cbsz:1
 ; GCN-NEXT:    s_nop 7
 ; GCN-NEXT:    s_nop 7
-; GCN-NEXT:    s_nop 2
+; GCN-NEXT:    s_nop 3
 ; GCN-NEXT:    v_accvgpr_read_b32 v0, a0
 ; GCN-NEXT:    v_accvgpr_read_b32 v1, a1
 ; GCN-NEXT:    v_accvgpr_read_b32 v2, a2
@@ -1465,7 +1465,7 @@ define <16 x float> @test_mfma_scale_f32_32x32x64_f8f6f4_0_0__cbsz1__blgp1(<8 x
 ; SDAG-NEXT:    v_mfma_scale_f32_32x32x64_f8f6f4 a[0:15], v[0:7], v[8:15], a[0:15], v32, v31 op_sel_hi:[0,0,0] cbsz:1 blgp:1
 ; SDAG-NEXT:    s_nop 7
 ; SDAG-NEXT:    s_nop 7
-; SDAG-NEXT:    s_nop 2
+; SDAG-NEXT:    s_nop 3
 ; SDAG-NEXT:    v_accvgpr_read_b32 v0, a0
 ; SDAG-NEXT:    v_accvgpr_read_b32 v1, a1
 ; SDAG-NEXT:    v_accvgpr_read_b32 v2, a2
@@ -1510,7 +1510,7 @@ define <16 x float> @test_mfma_scale_f32_32x32x64_f8f6f4_0_0__cbsz1__blgp1(<8 x
 ; GISEL-NEXT:    v_mfma_scale_f32_32x32x64_f8f6f4 a[0:15], v[0:7], v[8:15], a[0:15], v31, v32 op_sel_hi:[0,0,0] cbsz:1 blgp:1
 ; GISEL-NEXT:    s_nop 7
 ; GISEL-NEXT:    s_nop 7
-; GISEL-NEXT:    s_nop 2
+; GISEL-NEXT:    s_nop 3
 ; GISEL-NEXT:    v_accvgpr_read_b32 v0, a0
 ; GISEL-NEXT:    v_accvgpr_read_b32 v1, a1
 ; GISEL-NEXT:    v_accvgpr_read_b32 v2, a2
@@ -1561,7 +1561,7 @@ define <16 x float> @test_mfma_scale_f32_32x32x64_f8f6f4_0_0__cbsz1__blgp1__cons
 ; GCN-NEXT:    v_mfma_f32_32x32x64_f8f6f4 a[0:15], v[0:7], v[8:15], a[0:15] cbsz:1 blgp:1
 ; GCN-NEXT:    s_nop 7
 ; GCN-NEXT:    s_nop 7
-; GCN-NEXT:    s_nop 2
+; GCN-NEXT:    s_nop 3
 ; GCN-NEXT:    v_accvgpr_read_b32 v0, a0
 ; GCN-NEXT:    v_accvgpr_read_b32 v1, a1
 ; GCN-NEXT:    v_accvgpr_read_b32 v2, a2
@@ -1613,7 +1613,7 @@ define <16 x float> @test_mfma_scale_f32_32x32x64_f8f6f4_0_0__cbsz1__blgp2(<8 x
 ; GCN-NEXT:    v_mfma_scale_f32_32x32x64_f8f6f4 a[0:15], v[0:7], v[8:13], a[0:15], v30, v31 op_sel_hi:[0,0,0] cbsz:1 blgp:2
 ; GCN-NEXT:    s_nop 7
 ; GCN-NEXT:    s_nop 7
-; GCN-NEXT:    s_nop 2
+; GCN-NEXT:    s_nop 3
 ; GCN-NEXT:    v_accvgpr_read_b32 v0, a0
 ; GCN-NEXT:    v_accvgpr_read_b32 v1, a1
 ; GCN-NEXT:    v_accvgpr_read_b32 v2, a2
@@ -1662,7 +1662,7 @@ define <16 x float> @test_mfma_scale_f32_32x32x64_f8f6f4_0_0__cbsz1__blgp2__cons
 ; GCN-NEXT:    v_mfma_f32_32x32x64_f8f6f4 a[0:15], v[0:7], v[8:13], a[0:15] cbsz:1 blgp:2
 ; GCN-NEXT:    s_nop 7
 ; GCN-NEXT:    s_nop 7
-; GCN-NEXT:    s_nop 2
+; GCN-NEXT:    s_nop 3
 ; GCN-NEXT:    v_accvgpr_read_b32 v0, a0
 ; GCN-NEXT:    v_accvgpr_read_b32 v1, a1
 ; GCN-NEXT:    v_accvgpr_read_b32 v2, a2
@@ -1714,7 +1714,7 @@ define <16 x float> @test_mfma_scale_f32_32x32x64_f8f6f4_0_0__cbsz1__blgp3(<8 x
 ; GCN-NEXT:    v_mfma_scale_f32_32x32x64_f8f6f4 a[0:15], v[0:7], v[8:13], a[0:15], v30, v31 op_sel_hi:[0,0,0] cbsz:1 blgp:3
 ; GCN-NEXT:    s_nop 7
 ; GCN-NEXT:    s_nop 7
-; GCN-NEXT:    s_nop 2
+; GCN-NEXT:    s_nop 3
 ; GCN-NEXT:    v_accvgpr_read_b32 v0, a0
 ; GCN-NEXT:    v_accvgpr_read_b32 v1, a1
 ; GCN-NEXT:    v_accvgpr_read_b32 v2, a2
@@ -1763,7 +1763,7 @@ define <16 x float> @test_mfma_scale_f32_32x32x64_f8f6f4_0_0__cbsz1__blgp3__cons
 ; GCN-NEXT:    v_mfma_f32_32x32x64_f8f6f4 a[0:15], v[0:7], v[8:13], a[0:15] cbsz:1 blgp:3
 ; GCN-NEXT:    s_nop 7
 ; GCN-NEXT:    s_nop 7
-; GCN-NEXT:    s_nop 2
+; GCN-NEXT:    s_nop 3
 ; GCN-NEXT:    v_accvgpr_read_b32 v0, a0
 ; GCN-NEXT:    v_accvgpr_read_b32 v1, a1
 ; GCN-NEXT:    v_accvgpr_read_b32 v2, a2
@@ -1813,7 +1813,7 @@ define <16 x float> @test_mfma_scale_f32_32x32x64_f8f6f4_0_0__cbsz1__blgp4(<8 x
 ; GCN-NEXT:    v_mfma_scale_f32_32x32x64_f8f6f4 a[0:15], v[0:7], v[8:11], a[0:15], v28, v29 op_sel_hi:[0,0,0] cbsz:1 blgp:4
 ; GCN-NEXT:    s_nop 7
 ; GCN-NEXT:    s_nop 7
-; GCN-NEXT:    s_nop 2
+; GCN-NEXT:    s_nop 3
 ; GCN-NEXT:    v_accvgpr_read_b32 v0, a0
 ; GCN-NEXT:    v_accvgpr_read_b32 v1, a1
 ; GCN-NEXT:    v_accvgpr_read_b32 v2, a2
@@ -1862,7 +1862,7 @@ define <16 x float> @test_mfma_scale_f32_32x32x64_f8f6f4_0_0__cbsz1__blgp4__cons
 ; GCN-NEXT:    v_mfma_f32_32x32x64_f8f6f4 a[0:15], v[0:7], v[8:11], a[0:15] cbsz:1 blgp:4
 ; GCN-NEXT:    s_nop 7
 ; GCN-NEXT:    s_nop 7
-; GCN-NEXT:    s_nop 2
+; GCN-NEXT:    s_nop 3
 ; GCN-NEXT:    v_accvgpr_read_b32 v0, a0
 ; GCN-NEXT:    v_accvgpr_read_b32 v1, a1
 ; GCN-NEXT:    v_accvgpr_read_b32 v2, a2
@@ -1914,7 +1914,7 @@ define <16 x float> @test_mfma_scale_f32_32x32x64_f8f6f4_0_0__cbsz2__blgp0(<6 x
 ; GCN-NEXT:    v_mfma_scale_f32_32x32x64_f8f6f4 a[0:15], v[0:5], v[6:13], a[0:15], v30, v31 op_sel_hi:[0,0,0] cbsz:2
 ; GCN-NEXT:    s_nop 7
 ; GCN-NEXT:    s_nop 7
-; GCN-NEXT:    s_nop 2
+; GCN-NEXT:    s_nop 3
 ; GCN-NEXT:    v_accvgpr_read_b32 v0, a0
 ; GCN-NEXT:    v_accvgpr_read_b32 v1, a1
 ; GCN-NEXT:    v_accvgpr_read_b32 v2, a2
@@ -1963,7 +1963,7 @@ define <16 x float> @test_mfma_scale_f32_32x32x64_f8f6f4_0_0__cbsz2__blgp0__cons
 ; GCN-NEXT:    v_mfma_f32_32x32x64_f8f6f4 a[0:15], v[0:5], v[6:13], a[0:15] cbsz:2
 ; GCN-NEXT:    s_nop 7
 ; GCN-NEXT:    s_nop 7
-; GCN-NEXT:    s_nop 2
+; GCN-NEXT:    s_nop 3
 ; GCN-NEXT:    v_accvgpr_read_b32 v0, a0
 ; GCN-NEXT:    v_accvgpr_read_b32 v1, a1
 ; GCN-NEXT:    v_accvgpr_read_b32 v2, a2
@@ -2015,7 +2015,7 @@ define <16 x float> @test_mfma_scale_f32_32x32x64_f8f6f4_0_0__cbsz2__blgp1(<6 x
 ; GCN-NEXT:    v_mfma_scale_f32_32x32x64_f8f6f4 a[0:15], v[0:5], v[6:13], a[0:15], v30, v31 op_sel_hi:[0,0,0] cbsz:2 blgp:1
 ; GCN-NEXT:    s_nop 7
 ; GCN-NEXT:    s_nop 7
-; GCN-NEXT:    s_nop 2
+; GCN-NEXT:    s_nop 3
 ; GCN-NEXT:    v_accvgpr_read_b32 v0, a0
 ; GCN-NEXT:    v_accvgpr_read_b32 v1, a1
 ; GCN-NEXT:    v_accvgpr_read_b32 v2, a2
@@ -2064,7 +2064,7 @@ define <16 x float> @test_mfma_scale_f32_32x32x64_f8f6f4_0_0__cbsz2__blgp1__cons
 ; GCN-NEXT:    v_mfma_f32_32x32x64_f8f6f4 a[0:15], v[0:5], v[6:13], a[0:15] cbsz:2 blgp:1
 ; GCN-NEXT:    s_nop 7
 ; GCN-NEXT:    s_nop 7
-; GCN-NEXT:    s_nop 2
+; GCN-NEXT:    s_nop 3
 ; GCN-NEXT:    v_accvgpr_read_b32 v0, a0
 ; GCN-NEXT:    v_accvgpr_read_b32 v1, a1
 ; GCN-NEXT:    v_accvgpr_read_b32 v2, a2
@@ -2113,7 +2113,7 @@ define <16 x float> @test_mfma_scale_f32_32x32x64_f8f6f4_0_0__cbsz2__blgp2(<6 x
 ; GCN-NEXT:    s_nop 1
 ; GCN-NEXT:    v_mfma_scale_f32_32x32x64_f8f6f4 a[0:15], v[0:5], v[6:11], a[0:15], v28, v29 op_sel_hi:[0,0,0] cbsz:2 blgp:2
 ; GCN-NEXT:    s_nop 7
-; GCN-NEXT:    s_nop 2
+; GCN-NEXT:    s_nop 3
 ; GCN-NEXT:    v_accvgpr_read_b32 v0, a0
 ; GCN-NEXT:    v_accvgpr_read_b32 v1, a1
 ; GCN-NEXT:    v_accvgpr_read_b32 v2, a2
@@ -2161,7 +2161,7 @@ define <16 x float> @test_mfma_scale_f32_32x32x64_f8f6f4_0_0__cbsz2__blgp2__cons
 ; GCN-NEXT:    s_nop 1
 ; GCN-NEXT:    v_mfma_f32_32x32x64_f8f6f4 a[0:15], v[0:5], v[6:11], a[0:15] cbsz:2 blgp:2
 ; GCN-NEXT:    s_nop 7
-; GCN-NEXT:    s_nop 2
+; GCN-NEXT:    s_nop 3
 ; GCN-NEXT:    v_accvgpr_read_b32 v0, a0
 ; GCN-NEXT:    v_accvgpr_read_b32 v1, a1
 ; GCN-NEXT:    v_accvgpr_read_b32 v2, a2
@@ -2210,7 +2210,7 @@ define <16 x float> @test_mfma_scale_f32_32x32x64_f8f6f4_0_0__cbsz2__blgp3(<6 x
 ; GCN-NEXT:    s_nop 1
 ; GCN-NEXT:    v_mfma_scale_f32_32x32x64_f8f6f4 a[0:15], v[0:5], v[6:11], a[0:15], v28, v29 op_sel_hi:[0,0,0] cbsz:2 blgp:3
 ; GCN-NEXT:    s_nop 7
-; GCN-NEXT:    s_nop 2
+; GCN-NEXT:    s_nop 3
 ; GCN-NEXT:    v_accvgpr_read_b32 v0, a0
 ; GCN-NEXT:    v_accvgpr_read_b32 v1, a1
 ; GCN-NEXT:    v_accvgpr_read_b32 v2, a2
@@ -2258,7 +2258,7 @@ define <16 x float> @test_mfma_scale_f32_32x32x64_f8f6f4_0_0__cbsz2__blgp3__cons
 ; GCN-NEXT:    s_nop 1
 ; GCN-NEXT:    v_mfma_f32_32x32x64_f8f6f4 a[0:15], v[0:5], v[6:11], a[0:15] cbsz:2 blgp:3
 ; GCN-NEXT:    s_nop 7
-; GCN-NEXT:    s_nop 2
+; GCN-NEXT:    s_nop 3
 ; GCN-NEXT:    v_accvgpr_read_b32 v0, a0
 ; GCN-NEXT:    v_accvgpr_read_b32 v1, a1
 ; GCN-NEXT:    v_accvgpr_read_b32 v2, a2
@@ -2311,7 +2311,7 @@ define <16 x float> @test_mfma_scale_f32_32x32x64_f8f6f4_0_0__cbsz3__blgp0(<6 x
 ; GCN-NEXT:    v_mfma_scale_f32_32x32x64_f8f6f4 a[0:15], v[0:5], v[6:13], a[0:15], v30, v31 op_sel_hi:[0,0,0] cbsz:3
 ; GCN-NEXT:    s_nop 7
 ; GCN-NEXT:    s_nop 7
-; GCN-NEXT:    s_nop 2
+; GCN-NEXT:    s_nop 3
 ; GCN-NEXT:    v_accvgpr_read_b32 v0, a0
 ; GCN-NEXT:    v_accvgpr_read_b32 v1, a1
 ; GCN-NEXT:    v_accvgpr_read_b32 v2, a2
@@ -2360,7 +2360,7 @@ define <16 x float> @test_mfma_scale_f32_32x32x64_f8f6f4_0_0__cbsz3__blgp0__cons
 ; GCN-NEXT:    v_mfma_f32_32x32x64_f8f6f4 a[0:15], v[0:5], v[6:13], a[0:15] cbsz:3
 ; GCN-NEXT:    s_nop 7
 ; GCN-NEXT:    s_nop 7
-; GCN-NEXT:    s_nop 2
+; GCN-NEXT:    s_nop 3
 ; GCN-NEXT:    v_accvgpr_read_b32 v0, a0
 ; GCN-NEXT:    v_accvgpr_read_b32 v1, a1
 ; GCN-NEXT:    v_accvgpr_read_b32 v2, a2
@@ -2412,7 +2412,7 @@ define <16 x float> @test_mfma_scale_f32_32x32x64_f8f6f4_0_0__cbsz3__blgp1(<6 x
 ; GCN-NEXT:    v_mfma_scale_f32_32x32x64_f8f6f4 a[0:15], v[0:5], v[6:13], a[0:15], v30, v31 op_sel_hi:[0,0,0] cbsz:3 blgp:1
 ; GCN-NEXT:    s_nop 7
 ; GCN-NEXT:    s_nop 7
-; GCN-NEXT:    s_nop 2
+; GCN-NEXT:    s_nop 3
 ; GCN-NEXT:    v_accvgpr_read_b32 v0, a0
 ; GCN-NEXT:    v_accvgpr_read_b32 v1, a1
 ; GCN-NEXT:    v_accvgpr_read_b32 v2, a2
@@ -2461,7 +2461,7 @@ define <16 x float> @test_mfma_scale_f32_32x32x64_f8f6f4_0_0__cbsz3__blgp1__cons
 ; GCN-NEXT:    v_mfma_f32_32x32x64_f8f6f4 a[0:15], v[0:5], v[6:13], a[0:15] cbsz:3 blgp:1
 ; GCN-NEXT:    s_nop 7
 ; GCN-NEXT:    s_nop 7
-; GCN-NEXT:    s_nop 2
+; GCN-NEXT:    s_nop 3
 ; GCN-NEXT:    v_accvgpr_read_b32 v0, a0
 ; GCN-NEXT:    v_accvgpr_read_b32 v1, a1
 ; GCN-NEXT:    v_accvgpr_read_b32 v2, a2
@@ -2510,7 +2510,7 @@ define <16 x float> @test_mfma_scale_f32_32x32x64_f8f6f4_0_0__cbsz3__blgp2(<6 x
 ; GCN-NEXT:    s_nop 1
 ; GCN-NEXT:    v_mfma_scale_f32_32x32x64_f8f6f4 a[0:15], v[0:5], v[6:11], a[0:15], v28, v29 op_sel_hi:[0,0,0] cbsz:3 blgp:2
 ; GCN-NEXT:    s_nop 7
-; GCN-NEXT:    s_nop 2
+; GCN-NEXT:    s_nop 3
 ; GCN-NEXT:    v_accvgpr_read_b32 v0, a0
 ; GCN-NEXT:    v_accvgpr_read_b32 v1, a1
 ; GCN-NEXT:    v_accvgpr_read_b32 v2, a2
@@ -2558,7 +2558,7 @@ define <16 x float> @test_mfma_scale_f32_32x32x64_f8f6f4_0_0__cbsz3__blgp2__cons
 ; GCN-NEXT:    s_nop 1
 ; GCN-NEXT:    v_mfma_f32_32x32x64_f8f6f4 a[0:15], v[0:5], v[6:11], a[0:15] cbsz:3 blgp:2
 ; GCN-NEXT:    s_nop 7
-; GCN-NEXT:    s_nop 2
+; GCN-NEXT:    s_nop 3
 ; GCN-NEXT:    v_accvgpr_read_b32 v0, a0
 ; GCN-NEXT:    v_accvgpr_read_b32 v1, a1
 ; GCN-NEXT:    v_accvgpr_read_b32 v2, a2
@@ -2607,7 +2607,7 @@ define <16 x float> @test_mfma_scale_f32_32x32x64_f8f6f4_0_0__cbsz3__blgp4(<6 x
 ; GCN-NEXT:    s_nop 1
 ; GCN-NEXT:    v_mfma_scale_f32_32x32x64_f8f6f4 a[0:15], v[0:5], v[6:9], a[0:15], v26, v27 op_sel_hi:[0,0,0] cbsz:3 blgp:4
 ; GCN-NEXT:    s_nop 7
-; GCN-NEXT:    s_nop 2
+; GCN-NEXT:    s_nop 3
 ; GCN-NEXT:    v_accvgpr_read_b32 v0, a0
 ; GCN-NEXT:    v_accvgpr_read_b32 v1, a1
 ; GCN-NEXT:    v_accvgpr_read_b32 v2, a2
@@ -2655,7 +2655,7 @@ define <16 x float> @test_mfma_scale_f32_32x32x64_f8f6f4_0_0__cbsz3__blgp4__cons
 ; GCN-NEXT:    s_nop 1
 ; GCN-NEXT:    v_mfma_f32_32x32x64_f8f6f4 a[0:15], v[0:5], v[6:9], a[0:15] cbsz:3 blgp:4
 ; GCN-NEXT:    s_nop 7
-; GCN-NEXT:    s_nop 2
+; GCN-NEXT:    s_nop 3
 ; GCN-NEXT:    v_accvgpr_read_b32 v0, a0
 ; GCN-NEXT:    v_accvgpr_read_b32 v1, a1
 ; GCN-NEXT:    v_accvgpr_read_b32 v2, a2
@@ -2704,7 +2704,7 @@ define <16 x float> @test_mfma_scale_f32_32x32x64_f8f6f4_0_0__cbsz3__blgp3(<6 x
 ; GCN-NEXT:    s_nop 1
 ; GCN-NEXT:    v_mfma_scale_f32_32x32x64_f8f6f4 a[0:15], v[0:5], v[6:11], a[0:15], v28, v29 op_sel_hi:[0,0,0] cbsz:3 blgp:3
 ; GCN-NEXT:    s_nop 7
-; GCN-NEXT:    s_nop 2
+; GCN-NEXT:    s_nop 3
 ; GCN-NEXT:    v_accvgpr_read_b32 v0, a0
 ; GCN-NEXT:    v_accvgpr_read_b32 v1, a1
 ; GCN-NEXT:    v_accvgpr_read_b32 v2, a2
@@ -2752,7 +2752,7 @@ define <16 x float> @test_mfma_scale_f32_32x32x64_f8f6f4_0_0__cbsz3__blgp3__cons
 ; GCN-NEXT:    s_nop 1
 ; GCN-NEXT:    v_mfma_f32_32x32x64_f8f6f4 a[0:15], v[0:5], v[6:11], a[0:15] cbsz:3 blgp:3
 ; GCN-NEXT:    s_nop 7
-; GCN-NEXT:    s_nop 2
+; GCN-NEXT:    s_nop 3
 ; GCN-NEXT:    v_accvgpr_read_b32 v0, a0
 ; GCN-NEXT:    v_accvgpr_read_b32 v1, a1
 ; GCN-NEXT:    v_accvgpr_read_b32 v2, a2
@@ -2801,7 +2801,7 @@ define <16 x float> @test_mfma_scale_f32_32x32x64_f8f6f4_0_0__cbsz2__blgp4(<6 x
 ; GCN-NEXT:    s_nop 1
 ; GCN-NEXT:    v_mfma_scale_f32_32x32x64_f8f6f4 a[0:15], v[0:5], v[6:9], a[0:15], v26, v27 op_sel_hi:[0,0,0] cbsz:2 blgp:4
 ; GCN-NEXT:    s_nop 7
-; GCN-NEXT:    s_nop 2
+; GCN-NEXT:    s_nop 3
 ; GCN-NEXT:    v_accvgpr_read_b32 v0, a0
 ; GCN-NEXT:    v_accvgpr_read_b32 v1, a1
 ; GCN-NEXT:    v_accvgpr_read_b32 v2, a2
@@ -2849,7 +2849,7 @@ define <16 x float> @test_mfma_scale_f32_32x32x64_f8f6f4_0_0__cbsz2__blgp4__cons
 ; GCN-NEXT:    s_nop 1
 ; GCN-NEXT:    v_mfma_f32_32x32x64_f8f6f4 a[0:15], v[0:5], v[6:9], a[0:15] cbsz:2 blgp:4
 ; GCN-NEXT:    s_nop 7
-; GCN-NEXT:    s_nop 2
+; GCN-NEXT:    s_nop 3
 ; GCN-NEXT:    v_accvgpr_read_b32 v0, a0
 ; GCN-NEXT:    v_accvgpr_read_b32 v1, a1
 ; GCN-NEXT:    v_accvgpr_read_b32 v2, a2
@@ -2899,7 +2899,7 @@ define <16 x float> @test_mfma_scale_f32_32x32x64_f8f6f4_0_0__cbsz4__blgp0(<4 x
 ; GCN-NEXT:    v_mfma_scale_f32_32x32x64_f8f6f4 a[0:15], v[0:3], v[4:11], a[0:15], v28, v29 op_sel_hi:[0,0,0] cbsz:4
 ; GCN-NEXT:    s_nop 7
 ; GCN-NEXT:    s_nop 7
-; GCN-NEXT:    s_nop 2
+; GCN-NEXT:    s_nop 3
 ; GCN-NEXT:    v_accvgpr_read_b32 v0, a0
 ; GCN-NEXT:    v_accvgpr_read_b32 v1, a1
 ; GCN-NEXT:    v_accvgpr_read_b32 v2, a2
@@ -2948,7 +2948,7 @@ define <16 x float> @test_mfma_scale_f32_32x32x64_f8f6f4_0_0__cbsz4__blgp0__cons
 ; GCN-NEXT:    v_mfma_f32_32x32x64_f8f6f4 a[0:15], v[0:3], v[4:11], a[0:15] cbsz:4
 ; GCN-NEXT:    s_nop 7
 ; GCN-NEXT:    s_nop 7
-; GCN-NEXT:    s_nop 2
+; GCN-NEXT:    s_nop 3
 ; GCN-NEXT:    v_accvgpr_read_b32 v0, a0
 ; GCN-NEXT:    v_accvgpr_read_b32 v1, a1
 ; GCN-NEXT:    v_accvgpr_read_b32 v2, a2
@@ -2998,7 +2998,7 @@ define <16 x float> @test_mfma_scale_f32_32x32x64_f8f6f4_0_0__cbsz4__blgp1(<4 x
 ; GCN-NEXT:    v_mfma_scale_f32_32x32x64_f8f6f4 a[0:15], v[0:3], v[4:11], a[0:15], v28, v29 op_sel_hi:[0,0,0] cbsz:4 blgp:1
 ; GCN-NEXT:    s_nop 7
 ; GCN-NEXT:    s_nop 7
-; GCN-NEXT:    s_nop 2
+; GCN-NEXT:    s_nop 3
 ; GCN-NEXT:    v_accvgpr_read_b32 v0, a0
 ; GCN-NEXT:    v_accvgpr_read_b32 v1, a1
 ; GCN-NEXT:    v_accvgpr_read_b32 v2, a2
@@ -3047,7 +3047,7 @@ define <16 x float> @test_mfma_scale_f32_32x32x64_f8f6f4_0_0__cbsz4__blgp1__cons
 ; GCN-NEXT:    v_mfma_f32_32x32x64_f8f6f4 a[0:15], v[0:3], v[4:11], a[0:15] cbsz:4 blgp:1
 ; GCN-NEXT:    s_nop 7
 ; GCN-NEXT:    s_nop 7
-; GCN-NEXT:    s_nop 2
+; GCN-NEXT:    s_nop 3
 ; GCN-NEXT:    v_accvgpr_read_b32 v0, a0
 ; GCN-NEXT:    v_accvgpr_read_b32 v1, a1
 ; GCN-NEXT:    v_accvgpr_read_b32 v2, a2
@@ -3096,7 +3096,7 @@ define <16 x float> @test_mfma_scale_f32_32x32x64_f8f6f4_0_0__cbsz4__blgp2(<4 x
 ; GCN-NEXT:    s_nop 1
 ; GCN-NEXT:    v_mfma_scale_f32_32x32x64_f8f6f4 a[0:15], v[0:3], v[4:9], a[0:15], v26, v27 op_sel_hi:[0,0,0] cbsz:4 blgp:2
 ; GCN-NEXT:    s_nop 7
-; GCN-NEXT:    s_nop 2
+; GCN-NEXT:    s_nop 3
 ; GCN-NEXT:    v_accvgpr_read_b32 v0, a0
 ; GCN-NEXT:    v_accvgpr_read_b32 v1, a1
 ; GCN-NEXT:    v_accvgpr_read_b32 v2, a2
@@ -3144,7 +3144,7 @@ define <16 x float> @test_mfma_scale_f32_32x32x64_f8f6f4_0_0__cbsz4__blgp2__cons
 ; GCN-NEXT:    s_nop 1
 ; GCN-NEXT:    v_mfma_f32_32x32x64_f8f6f4 a[0:15], v[0:3], v[4:9], a[0:15] cbsz:4 blgp:2
 ; GCN-NEXT:    s_nop 7
-; GCN-NEXT:    s_nop 2
+; GCN-NEXT:    s_nop 3
 ; GCN-NEXT:    v_accvgpr_read_b32 v0, a0
 ; GCN-NEXT:    v_accvgpr_read_b32 v1, a1
 ; GCN-NEXT:    v_accvgpr_read_b32 v2, a2
@@ -3193,7 +3193,7 @@ define <16 x float> @test_mfma_scale_f32_32x32x64_f8f6f4_0_0__cbsz4__blgp3(<4 x
 ; GCN-NEXT:    s_nop 1
 ; GCN-NEXT:    v_mfma_scale_f32_32x32x64_f8f6f4 a[0:15], v[0:3], v[4:9], a[0:15], v26, v27 op_sel_hi:[0,0,0] cbsz:4 blgp:3
 ; GCN-NEXT:    s_nop 7
-; GCN-NEXT:    s_nop 2
+; GCN-NEXT:    s_nop 3
 ; GCN-NEXT:    v_accvgpr_read_b32 v0, a0
 ; GCN-NEXT:    v_accvgpr_read_b32 v1, a1
 ; GCN-NEXT:    v_accvgpr_read_b32 v2, a2
@@ -3241,7 +3241,7 @@ define <16 x float> @test_mfma_scale_f32_32x32x64_f8f6f4_0_0__cbsz4__blgp3__cons
 ; GCN-NEXT:    s_nop 1
 ; GCN-NEXT:    v_mfma_f32_32x32x64_f8f6f4 a[0:15], v[0:3], v[4:9], a[0:15] cbsz:4 blgp:3
 ; GCN-NEXT:    s_nop 7
-; GCN-NEXT:    s_nop 2
+; GCN-NEXT:    s_nop 3
 ; GCN-NEXT:    v_accvgpr_read_b32 v0, a0
 ; GCN-NEXT:    v_accvgpr_read_b32 v1, a1
 ; GCN-NEXT:    v_accvgpr_read_b32 v2, a2
@@ -3290,7 +3290,7 @@ define <16 x float> @test_mfma_scale_f32_32x32x64_f8f6f4_0_0__cbsz4__blgp4(<4 x
 ; GCN-NEXT:    s_nop 1
 ; GCN-NEXT:    v_mfma_scale_f32_32x32x64_f8f6f4 a[0:15], v[0:3], v[4:7], a[0:15], v24, v25 op_sel_hi:[0,0,0] cbsz:4 blgp:4
 ; GCN-NEXT:    s_nop 7
-; GCN-NEXT:    s_nop 2
+; GCN-NEXT:    s_nop 3
 ; GCN-NEXT:    v_accvgpr_read_b32 v0, a0
 ; GCN-NEXT:    v_accvgpr_read_b32 v1, a1
 ; GCN-NEXT:    v_accvgpr_read_b32 v2, a2
@@ -3338,7 +3338,7 @@ define <16 x float> @test_mfma_scale_f32_32x32x64_f8f6f4_0_0__cbsz4__blgp4__cons
 ; GCN-NEXT:    s_nop 1
 ; GCN-NEXT:    v_mfma_f32_32x32x64_f8f6f4 a[0:15], v[0:3], v[4:7], a[0:15] cbsz:4 blgp:4
 ; GCN-NEXT:    s_nop 7
-; GCN-NEXT:    s_nop 2
+; GCN-NEXT:    s_nop 3
 ; GCN-NEXT:    v_accvgpr_read_b32 v0, a0
 ; GCN-NEXT:    v_accvgpr_read_b32 v1, a1
 ; GCN-NEXT:    v_accvgpr_read_b32 v2, a2
@@ -3393,7 +3393,7 @@ define <16 x float> @test_mfma_scale_f32_32x32x64_f8f6f4_0_0__sgpr_scaleA__sgpr_
 ; GCN-NEXT:    v_mfma_scale_f32_32x32x64_f8f6f4 a[0:15], v[0:7], v[8:15], a[0:15], s0, v16 op_sel_hi:[0,0,0]
 ; GCN-NEXT:    s_nop 7
 ; GCN-NEXT:    s_nop 7
-; GCN-NEXT:    s_nop 2
+; GCN-NEXT:    s_nop 3
 ; GCN-NEXT:    v_accvgpr_read_b32 v0, a0
 ; GCN-NEXT:    v_accvgpr_read_b32 v1, a1
 ; GCN-NEXT:    v_accvgpr_read_b32 v2, a2
@@ -3441,7 +3441,7 @@ define <16 x float> @test_mfma_scale_f32_32x32x64_f8f6f4_0_0__sgpr_scaleA__vgpr_
 ; GCN-NEXT:    v_mfma_scale_f32_32x32x64_f8f6f4 a[0:15], v[0:7], v[8:15], a[0:15], s0, v31 op_sel_hi:[0,0,0]
 ; GCN-NEXT:    s_nop 7
 ; GCN-NEXT:    s_nop 7
-; GCN-NEXT:    s_nop 2
+; GCN-NEXT:    s_nop 3
 ; GCN-NEXT:    v_accvgpr_read_b32 v0, a0
 ; GCN-NEXT:    v_accvgpr_read_b32 v1, a1
 ; GCN-NEXT:    v_accvgpr_read_b32 v2, a2
@@ -3489,7 +3489,7 @@ define <16 x float> @test_mfma_scale_f32_32x32x64_f8f6f4_0_0__vgpr_scaleA__sgpr_
 ; GCN-NEXT:    v_mfma_scale_f32_32x32x64_f8f6f4 a[0:15], v[0:7], v[8:15], a[0:15], v31, s0 op_sel_hi:[0,0,0]
 ; GCN-NEXT:    s_nop 7
 ; GCN-NEXT:    s_nop 7
-; GCN-NEXT:    s_nop 2
+; GCN-NEXT:    s_nop 3
 ; GCN-NEXT:    v_accvgpr_read_b32 v0, a0
 ; GCN-NEXT:    v_accvgpr_read_b32 v1, a1
 ; GCN-NEXT:    v_accvgpr_read_b32 v2, a2
@@ -3567,7 +3567,7 @@ define <16 x float> @test_mfma_scale_f32_32x32x64_f8f6f4_0_0_sgprs(<8 x i32> inr
 ; SDAG-NEXT:    v_mfma_scale_f32_32x32x64_f8f6f4 a[0:15], v[32:39], v[0:7], a[0:15], v14, v15 op_sel_hi:[0,0,0]
 ; SDAG-NEXT:    s_nop 7
 ; SDAG-NEXT:    s_nop 7
-; SDAG-NEXT:    s_nop 2
+; SDAG-NEXT:    s_nop 3
 ; SDAG-NEXT:    v_accvgpr_read_b32 v0, a0
 ; SDAG-NEXT:    v_accvgpr_read_b32 v1, a1
 ; SDAG-NEXT:    v_accvgpr_read_b32 v2, a2
@@ -3637,7 +3637,7 @@ define <16 x float> @test_mfma_scale_f32_32x32x64_f8f6f4_0_0_sgprs(<8 x i32> inr
 ; GISEL-NEXT:    v_mfma_scale_f32_32x32x64_f8f6f4 a[0:15], v[0:7], v[32:39], a[0:15], v14, v15 op_sel_hi:[0,0,0]
 ; GISEL-NEXT:    s_nop 7
 ; GISEL-NEXT:    s_nop 7
-; GISEL-NEXT:    s_nop 2
+; GISEL-NEXT:    s_nop 3
 ; GISEL-NEXT:    v_accvgpr_read_b32 v0, a0
 ; GISEL-NEXT:    v_accvgpr_read_b32 v1, a1
 ; GISEL-NEXT:    v_accvgpr_read_b32 v2, a2
@@ -3691,7 +3691,7 @@ define <16 x float> @test_mfma_scale_f32_32x32x64_f8f6f4_0_0_sgpr_vgpr_vgpr__sgp
 ; SDAG-NEXT:    v_mfma_scale_f32_32x32x64_f8f6f4 a[0:15], v[26:33], v[0:7], a[0:15], s20, v24 op_sel_hi:[0,0,0]
 ; SDAG-NEXT:    s_nop 7
 ; SDAG-NEXT:    s_nop 7
-; SDAG-NEXT:    s_nop 2
+; SDAG-NEXT:    s_nop 3
 ; SDAG-NEXT:    v_accvgpr_read_b32 v0, a0
 ; SDAG-NEXT:    v_accvgpr_read_b32 v1, a1
 ; SDAG-NEXT:    v_accvgpr_read_b32 v2, a2
@@ -3741,7 +3741,7 @@ define <16 x float> @test_mfma_scale_f32_32x32x64_f8f6f4_0_0_sgpr_vgpr_vgpr__sgp
 ; GISEL-NEXT:    v_mfma_scale_f32_32x32x64_f8f6f4 a[0:15], v[26:33], v[0:7], a[0:15], s20, v24 op_sel_hi:[0,0,0]
 ; GISEL-NEXT:    s_nop 7
 ; GISEL-NEXT:    s_nop 7
-; GISEL-NEXT:    s_nop 2
+; GISEL-NEXT:    s_nop 3
 ; GISEL-NEXT:    v_accvgpr_read_b32 v0, a0
 ; GISEL-NEXT:    v_accvgpr_read_b32 v1, a1
 ; GISEL-NEXT:    v_accvgpr_read_b32 v2, a2
@@ -3795,7 +3795,7 @@ define <16 x float> @test_mfma_scale_f32_32x32x64_f8f6f4_0_0_sgpr_vgpr_vgpr__vgp
 ; SDAG-NEXT:    v_mfma_scale_f32_32x32x64_f8f6f4 a[0:15], v[26:33], v[0:7], a[0:15], v24, s20 op_sel_hi:[0,0,0]
 ; SDAG-NEXT:    s_nop 7
 ; SDAG-NEXT:    s_nop 7
-; SDAG-NEXT:    s_nop 2
+; SDAG-NEXT:    s_nop 3
 ; SDAG-NEXT:    v_accvgpr_read_b32 v0, a0
 ; SDAG-NEXT:    v_accvgpr_read_b32 v1, a1
 ; SDAG-NEXT:    v_accvgpr_read_b32 v2, a2
@@ -3845,7 +3845,7 @@ define <16 x float> @test_mfma_scale_f32_32x32x64_f8f6f4_0_0_sgpr_vgpr_vgpr__vgp
 ; GISEL-NEXT:    v_mfma_scale_f32_32x32x64_f8f6f4 a[0:15], v[26:33], v[0:7], a[0:15], v24, s20 op_sel_hi:[0,0,0]
 ; GISEL-NEXT:    s_nop 7
 ; GISEL-NEXT:    s_nop 7
-; GISEL-NEXT:    s_nop 2
+; GISEL-NEXT:    s_nop 3
 ; GISEL-NEXT:    v_accvgpr_read_b32 v0, a0
 ; GISEL-NEXT:    v_accvgpr_read_b32 v1, a1
 ; GISEL-NEXT:    v_accvgpr_read_b32 v2, a2
@@ -3899,7 +3899,7 @@ define <16 x float> @test_mfma_scale_f32_32x32x64_f8f6f4_0_0_vgpr_sgpr_vgpr__vgp
 ; SDAG-NEXT:    v_mfma_scale_f32_32x32x64_f8f6f4 a[0:15], v[0:7], v[26:33], a[0:15], v24, s20 op_sel_hi:[0,0,0]
 ; SDAG-NEXT:    s_nop 7
 ; SDAG-NEXT:    s_nop 7
-; SDAG-NEXT:    s_nop 2
+; SDAG-NEXT:    s_nop 3
 ; SDAG-NEXT:    v_accvgpr_read_b32 v0, a0
 ; SDAG-NEXT:    v_accvgpr_read_b32 v1, a1
 ; SDAG-NEXT:    v_accvgpr_read_b32 v2, a2
@@ -3949,7 +3949,7 @@ define <16 x float> @test_mfma_scale_f32_32x32x64_f8f6f4_0_0_vgpr_sgpr_vgpr__vgp
 ; GISEL-NEXT:    v_mfma_scale_f32_32x32x64_f8f6f4 a[0:15], v[0:7], v[26:33], a[0:15], v24, s20 op_sel_hi:[0,0,0]
 ; GISEL-NEXT:    s_nop 7
 ; GISEL-NEXT:    s_nop 7
-; GISEL-NEXT:    s_nop 2
+; GISEL-NEXT:    s_nop 3
 ; GISEL-NEXT:    v_accvgpr_read_b32 v0, a0
 ; GISEL-NEXT:    v_accvgpr_read_b32 v1, a1
 ; GISEL-NEXT:    v_accvgpr_read_b32 v2, a2
@@ -3995,7 +3995,7 @@ define <16 x float> @test_mfma_scale_f32_32x32x64_f8f6f4_0_0_vgpr_vgpr_sgpr__vgp
 ; SDAG-NEXT:    v_mfma_scale_f32_32x32x64_f8f6f4 a[0:15], v[0:7], v[8:15], a[0:15], v16, s28 op_sel_hi:[0,0,0]
 ; SDAG-NEXT:    s_nop 7
 ; SDAG-NEXT:    s_nop 7
-; SDAG-NEXT:    s_nop 2
+; SDAG-NEXT:    s_nop 3
 ; SDAG-NEXT:    v_accvgpr_read_b32 v0, a0
 ; SDAG-NEXT:    v_accvgpr_read_b32 v1, a1
 ; SDAG-NEXT:    v_accvgpr_read_b32 v2, a2
@@ -4041,7 +4041,7 @@ define <16 x float> @test_mfma_scale_f32_32x32x64_f8f6f4_0_0_vgpr_vgpr_sgpr__vgp
 ; GISEL-NEXT:    v_mfma_scale_f32_32x32x64_f8f6f4 a[0:15], v[0:7], v[8:15], a[0:15], v16, s28 op_sel_hi:[0,0,0]
 ; GISEL-NEXT:    s_nop 7
 ; GISEL-NEXT:    s_nop 7
-; GISEL-NEXT:    s_nop 2
+; GISEL-NEXT:    s_nop 3
 ; GISEL-NEXT:    v_accvgpr_read_b32 v0, a0
 ; GISEL-NEXT:    v_accvgpr_read_b32 v1, a1
 ; GISEL-NEXT:    v_accvgpr_read_b32 v2, a2
@@ -4111,7 +4111,7 @@ define <16 x float> @test_mfma_scale_f32_32x32x64_f8f6f4_0_0_sgpr_vgpr_sgpr__vgp
 ; SDAG-NEXT:    v_mfma_scale_f32_32x32x64_f8f6f4 a[0:15], v[32:39], v[0:7], a[0:15], v14, v15 op_sel_hi:[0,0,0]
 ; SDAG-NEXT:    s_nop 7
 ; SDAG-NEXT:    s_nop 7
-; SDAG-NEXT:    s_nop 2
+; SDAG-NEXT:    s_nop 3
 ; SDAG-NEXT:    v_accvgpr_read_b32 v0, a0
 ; SDAG-NEXT:    v_accvgpr_read_b32 v1, a1
 ; SDAG-NEXT:    v_accvgpr_read_b32 v2, a2
@@ -4177,7 +4177,7 @@ define <16 x float> @test_mfma_scale_f32_32x32x64_f8f6f4_0_0_sgpr_vgpr_sgpr__vgp
 ; GISEL-NEXT:    v_mfma_scale_f32_32x32x64_f8f6f4 a[0:15], v[32:39], v[0:7], a[0:15], v14, v15 op_sel_hi:[0,0,0]
 ; GISEL-NEXT:    s_nop 7
 ; GISEL-NEXT:    s_nop 7
-; GISEL-NEXT:    s_nop 2
+; GISEL-NEXT:    s_nop 3
 ; GISEL-NEXT:    v_accvgpr_read_b32 v0, a0
 ; GISEL-NEXT:    v_accvgpr_read_b32 v1, a1
 ; GISEL-NEXT:    v_accvgpr_read_b32 v2, a2
@@ -4224,7 +4224,7 @@ define <16 x float> @test_mfma_scale_f32_32x32x64_f8f6f4_0_0__scaleA_inlineimm__
 ; GCN-NEXT:    v_mfma_scale_f32_32x32x64_f8f6f4 a[0:15], v[0:7], v[8:15], a[0:15], 33, -2 op_sel_hi:[0,0,0]
 ; GCN-NEXT:    s_nop 7
 ; GCN-NEXT:    s_nop 7
-; GCN-NEXT:    s_nop 2
+; GCN-NEXT:    s_nop 3
 ; GCN-NEXT:    v_accvgpr_read_b32 v0, a0
 ; GCN-NEXT:    v_accvgpr_read_b32 v1, a1
 ; GCN-NEXT:    v_accvgpr_read_b32 v2, a2
@@ -4272,7 +4272,7 @@ define <16 x float> @test_mfma_scale_f32_32x32x64_f8f6f4_0_0__scaleA_kimm__scale
 ; SDAG-NEXT:    v_mfma_scale_f32_32x32x64_f8f6f4 a[0:15], v[0:7], v[8:15], a[0:15], s0, -2 op_sel_hi:[0,0,0]
 ; SDAG-NEXT:    s_nop 7
 ; SDAG-NEXT:    s_nop 7
-; SDAG-NEXT:    s_nop 2
+; SDAG-NEXT:    s_nop 3
 ; SDAG-NEXT:    v_accvgpr_read_b32 v0, a0
 ; SDAG-NEXT:    v_accvgpr_read_b32 v1, a1
 ; SDAG-NEXT:    v_accvgpr_read_b32 v2, a2
@@ -4316,7 +4316,7 @@ define <16 x float> @test_mfma_scale_f32_32x32x64_f8f6f4_0_0__scaleA_kimm__scale
 ; GISEL-NEXT:    v_mfma_scale_f32_32x32x64_f8f6f4 a[0:15], v[0:7], v[8:15], a[0:15], v31, -2 op_sel_hi:[0,0,0]
 ; GISEL-NEXT:    s_nop 7
 ; GISEL-NEXT:    s_nop 7
-; GISEL-NEXT:    s_nop 2
+; GISEL-NEXT:    s_nop 3
 ; GISEL-NEXT:    v_accvgpr_read_b32 v0, a0
 ; GISEL-NEXT:    v_accvgpr_read_b32 v1, a1
 ; GISEL-NEXT:    v_accvgpr_read_b32 v2, a2
@@ -4365,7 +4365,7 @@ define <16 x float> @test_mfma_scale_f32_32x32x64_f8f6f4_0_0__scaleA_kimm__scale
 ; SDAG-NEXT:    v_mfma_scale_f32_32x32x64_f8f6f4 a[0:15], v[0:7], v[8:15], a[0:15], s0, v31 op_sel_hi:[0,0,0]
 ; SDAG-NEXT:    s_nop 7
 ; SDAG-NEXT:    s_nop 7
-; SDAG-NEXT:    s_nop 2
+; SDAG-NEXT:    s_nop 3
 ; SDAG-NEXT:    v_accvgpr_read_b32 v0, a0
 ; SDAG-NEXT:    v_accvgpr_read_b32 v1, a1
 ; SDAG-NEXT:    v_accvgpr_read_b32 v2, a2
@@ -4410,7 +4410,7 @@ define <16 x float> @test_mfma_scale_f32_32x32x64_f8f6f4_0_0__scaleA_kimm__scale
 ; GISEL-NEXT:    v_mfma_scale_f32_32x32x64_f8f6f4 a[0:15], v[0:7], v[8:15], a[0:15], v31, v32 op_sel_hi:[0,0,0]
 ; GISEL-NEXT:    s_nop 7
 ; GISEL-NEXT:    s_nop 7
-; GISEL-NEXT:    s_nop 2
+; GISEL-NEXT:    s_nop 3
 ; GISEL-NEXT:    v_accvgpr_read_b32 v0, a0
 ; GISEL-NEXT:    v_accvgpr_read_b32 v1, a1
 ; GISEL-NEXT:    v_accvgpr_read_b32 v2, a2
@@ -4477,7 +4477,7 @@ define amdgpu_kernel void @test_mfma_scale_f32_32x32x64_f8f6f4__vgprcd(<8 x i32>
 ; SDAG-NEXT:    v_mov_b32_e32 v0, 0
 ; SDAG-NEXT:    s_nop 7
 ; SDAG-NEXT:    s_nop 7
-; SDAG-NEXT:    s_nop 1
+; SDAG-NEXT:    s_nop 2
 ; SDAG-NEXT:    global_store_dwordx4 v0, a[12:15], s[2:3] offset:48
 ; SDAG-NEXT:    global_store_dwordx4 v0, a[8:11], s[2:3] offset:32
 ; SDAG-NEXT:    global_store_dwordx4 v0, a[4:7], s[2:3] offset:16
@@ -4520,7 +4520,7 @@ define amdgpu_kernel void @test_mfma_scale_f32_32x32x64_f8f6f4__vgprcd(<8 x i32>
 ; GISEL-NEXT:    v_mov_b32_e32 v0, 0
 ; GISEL-NEXT:    s_nop 7
 ; GISEL-NEXT:    s_nop 7
-; GISEL-NEXT:    s_nop 1
+; GISEL-NEXT:    s_nop 2
 ; GISEL-NEXT:    global_store_dwordx4 v0, a[0:3], s[2:3]
 ; GISEL-NEXT:    global_store_dwordx4 v0, a[4:7], s[2:3] offset:16
 ; GISEL-NEXT:    global_store_dwordx4 v0, a[8:11], s[2:3] offset:32
@@ -4576,7 +4576,7 @@ define amdgpu_kernel void @test_mfma_scale_f32_32x32x64_f8f6f4__vgprcd___scaleA_
 ; SDAG-NEXT:    v_mov_b32_e32 v0, 0
 ; SDAG-NEXT:    s_nop 7
 ; SDAG-NEXT:    s_nop 7
-; SDAG-NEXT:    s_nop 1
+; SDAG-NEXT:    s_nop 2
 ; SDAG-NEXT:    global_store_dwordx4 v0, a[12:15], s[0:1] offset:48
 ; SDAG-NEXT:    global_store_dwordx4 v0, a[8:11], s[0:1] offset:32
 ; SDAG-NEXT:    global_store_dwordx4 v0, a[4:7], s[0:1] offset:16
@@ -4619,7 +4619,7 @@ define amdgpu_kernel void @test_mfma_scale_f32_32x32x64_f8f6f4__vgprcd___scaleA_
 ; GISEL-NEXT:    v_mov_b32_e32 v0, 0
 ; GISEL-NEXT:    s_nop 7
 ; GISEL-NEXT:    s_nop 7
-; GISEL-NEXT:    s_nop 1
+; GISEL-NEXT:    s_nop 2
 ; GISEL-NEXT:    global_store_dwordx4 v0, a[0:3], s[0:1]
 ; GISEL-NEXT:    global_store_dwordx4 v0, a[4:7], s[0:1] offset:16
 ; GISEL-NEXT:    global_store_dwordx4 v0, a[8:11], s[0:1] offset:32
@@ -4765,7 +4765,7 @@ define amdgpu_kernel void @test_mfma_scale_f32_32x32x64_f8f6f4_0_0__nonmac(<8 x
 ; GISEL-NEXT:    s_waitcnt vmcnt(0)
 ; GISEL-NEXT:    global_store_dwordx4 v[22:23], v[12:15], off sc0 sc1
 ; GISEL-NEXT:    s_waitcnt vmcnt(0)
-; GISEL-NEXT:    s_nop 1
+; GISEL-NEXT:    s_nop 2
 ; GISEL-NEXT:    global_store_dwordx4 v[16:17], a[0:3], off sc0 sc1
 ; GISEL-NEXT:    s_waitcnt vmcnt(0)
 ; GISEL-NEXT:    global_store_dwordx4 v[18:19], a[4:7], off sc0 sc1
@@ -4912,7 +4912,7 @@ define amdgpu_kernel void @test_mfma_scale_f32_32x32x64_f8f6f4_25_42__nonmac(<8
 ; GISEL-NEXT:    s_waitcnt vmcnt(0)
 ; GISEL-NEXT:    global_store_dwordx4 v[22:23], v[12:15], off sc0 sc1
 ; GISEL-NEXT:    s_waitcnt vmcnt(0)
-; GISEL-NEXT:    s_nop 2
+; GISEL-NEXT:    s_nop 3
 ; GISEL-NEXT:    global_store_dwordx4 v[16:17], a[0:3], off sc0 sc1
 ; GISEL-NEXT:    s_waitcnt vmcnt(0)
 ; GISEL-NEXT:    global_store_dwordx4 v[18:19], a[4:7], off sc0 sc1
@@ -5059,7 +5059,7 @@ define amdgpu_kernel void @test_mfma_scale_f32_32x32x64_f8f6f4_0_0__vgprcd_nonma
 ; GISEL-NEXT:    s_waitcnt vmcnt(0)
 ; GISEL-NEXT:    global_store_dwordx4 v[22:23], v[12:15], off sc0 sc1
 ; GISEL-NEXT:    s_waitcnt vmcnt(0)
-; GISEL-NEXT:    s_nop 2
+; GISEL-NEXT:    s_nop 3
 ; GISEL-NEXT:    global_store_dwordx4 v[16:17], a[0:3], off sc0 sc1
 ; GISEL-NEXT:    s_waitcnt vmcnt(0)
 ; GISEL-NEXT:    global_store_dwordx4 v[18:19], a[4:7], off sc0 sc1
@@ -5206,7 +5206,7 @@ define amdgpu_kernel void @test_mfma_scale_f32_32x32x64_f8f6f4_25_42__vgprcd_non
 ; GISEL-NEXT:    s_waitcnt vmcnt(0)
 ; GISEL-NEXT:    global_store_dwordx4 v[22:23], v[12:15], off sc0 sc1
 ; GISEL-NEXT:    s_waitcnt vmcnt(0)
-; GISEL-NEXT:    s_nop 2
+; GISEL-NEXT:    s_nop 3
 ; GISEL-NEXT:    global_store_dwordx4 v[16:17], a[0:3], off sc0 sc1
 ; GISEL-NEXT:    s_waitcnt vmcnt(0)
 ; GISEL-NEXT:    global_store_dwordx4 v[18:19], a[4:7], off sc0 sc1
@@ -5247,7 +5247,7 @@ define <16 x float> @test_mfma_scale_f32_32x32x64_f8f6f4___constant_scale_0_0_a(
 ; GCN-NEXT:    v_mfma_f32_32x32x64_f8f6f4 a[0:15], v[0:7], v[8:15], a[0:15]
 ; GCN-NEXT:    s_nop 7
 ; GCN-NEXT:    s_nop 7
-; GCN-NEXT:    s_nop 2
+; GCN-NEXT:    s_nop 3
 ; GCN-NEXT:    v_accvgpr_read_b32 v0, a0
 ; GCN-NEXT:    v_accvgpr_read_b32 v1, a1
 ; GCN-NEXT:    v_accvgpr_read_b32 v2, a2
@@ -5294,7 +5294,7 @@ define <16 x float> @test_mfma_scale_f32_32x32x64_f8f6f4___constant_scale_0_0_b(
 ; GCN-NEXT:    v_mfma_f32_32x32x64_f8f6f4 a[0:15], v[0:7], v[8:15], a[0:15]
 ; GCN-NEXT:    s_nop 7
 ; GCN-NEXT:    s_nop 7
-; GCN-NEXT:    s_nop 2
+; GCN-NEXT:    s_nop 3
 ; GCN-NEXT:    v_accvgpr_read_b32 v0, a0
 ; GCN-NEXT:    v_accvgpr_read_b32 v1, a1
 ; GCN-NEXT:    v_accvgpr_read_b32 v2, a2
@@ -5341,7 +5341,7 @@ define <16 x float> @test_mfma_scale_f32_32x32x64_f8f6f4___constant_scale_0_1(<8
 ; GCN-NEXT:    v_mfma_scale_f32_32x32x64_f8f6f4 a[0:15], v[0:7], v[8:15], a[0:15], 0, 1 op_sel_hi:[0,0,0]
 ; GCN-NEXT:    s_nop 7
 ; GCN-NEXT:    s_nop 7
-; GCN-NEXT:    s_nop 2
+; GCN-NEXT:    s_nop 3
 ; GCN-NEXT:    v_accvgpr_read_b32 v0, a0
 ; GCN-NEXT:    v_accvgpr_read_b32 v1, a1
 ; GCN-NEXT:    v_accvgpr_read_b32 v2, a2
@@ -5388,7 +5388,7 @@ define <16 x float> @test_mfma_scale_f32_32x32x64_f8f6f4___constant_scale_1_0_a(
 ; GCN-NEXT:    v_mfma_scale_f32_32x32x64_f8f6f4 a[0:15], v[0:7], v[8:15], a[0:15], 1, 0 op_sel_hi:[0,0,0]
 ; GCN-NEXT:    s_nop 7
 ; GCN-NEXT:    s_nop 7
-; GCN-NEXT:    s_nop 2
+; GCN-NEXT:    s_nop 3
 ; GCN-NEXT:    v_accvgpr_read_b32 v0, a0
 ; GCN-NEXT:    v_accvgpr_read_b32 v1, a1
 ; GCN-NEXT:    v_accvgpr_read_b32 v2, a2
@@ -5441,7 +5441,7 @@ define <16 x float> @test_mfma_scale_f32_32x32x64_f8f6f4___v8i32_fp8__v8i32_fp6(
 ; SDAG-NEXT:    v_mfma_scale_f32_32x32x64_f8f6f4 a[0:15], v[0:7], v[8:15], a[0:15], v32, v31 op_sel_hi:[0,0,0] blgp:2
 ; SDAG-NEXT:    s_nop 7
 ; SDAG-NEXT:    s_nop 7
-; SDAG-NEXT:    s_nop 2
+; SDAG-NEXT:    s_nop 3
 ; SDAG-NEXT:    v_accvgpr_read_b32 v0, a0
 ; SDAG-NEXT:    v_accvgpr_read_b32 v1, a1
 ; SDAG-NEXT:    v_accvgpr_read_b32 v2, a2
@@ -5486,7 +5486,7 @@ define <16 x float> @test_mfma_scale_f32_32x32x64_f8f6f4___v8i32_fp8__v8i32_fp6(
 ; GISEL-NEXT:    v_mfma_scale_f32_32x32x64_f8f6f4 a[0:15], v[0:7], v[8:15], a[0:15], v31, v32 op_sel_hi:[0,0,0] blgp:2
 ; GISEL-NEXT:    s_nop 7
 ; GISEL-NEXT:    s_nop 7
-; GISEL-NEXT:    s_nop 2
+; GISEL-NEXT:    s_nop 3
 ; GISEL-NEXT:    v_accvgpr_read_b32 v0, a0
 ; GISEL-NEXT:    v_accvgpr_read_b32 v1, a1
 ; GISEL-NEXT:    v_accvgpr_read_b32 v2, a2
@@ -5538,7 +5538,7 @@ define <16 x float> @test_mfma_scale_f32_32x32x64_f8f6f4___v8i32_fp6__v8i32_fp8(
 ; SDAG-NEXT:    v_mfma_scale_f32_32x32x64_f8f6f4 a[0:15], v[0:7], v[8:15], a[0:15], v32, v31 op_sel_hi:[0,0,0] cbsz:2
 ; SDAG-NEXT:    s_nop 7
 ; SDAG-NEXT:    s_nop 7
-; SDAG-NEXT:    s_nop 2
+; SDAG-NEXT:    s_nop 3
 ; SDAG-NEXT:    v_accvgpr_read_b32 v0, a0
 ; SDAG-NEXT:    v_accvgpr_read_b32 v1, a1
 ; SDAG-NEXT:    v_accvgpr_read_b32 v2, a2
@@ -5583,7 +5583,7 @@ define <16 x float> @test_mfma_scale_f32_32x32x64_f8f6f4___v8i32_fp6__v8i32_fp8(
 ; GISEL-NEXT:    v_mfma_scale_f32_32x32x64_f8f6f4 a[0:15], v[0:7], v[8:15], a[0:15], v31, v32 op_sel_hi:[0,0,0] cbsz:2
 ; GISEL-NEXT:    s_nop 7
 ; GISEL-NEXT:    s_nop 7
-; GISEL-NEXT:    s_nop 2
+; GISEL-NEXT:    s_nop 3
 ; GISEL-NEXT:    v_accvgpr_read_b32 v0, a0
 ; GISEL-NEXT:    v_accvgpr_read_b32 v1, a1
 ; GISEL-NEXT:    v_accvgpr_read_b32 v2, a2
@@ -5634,7 +5634,7 @@ define <16 x float> @test_mfma_scale_f32_32x32x64_f8f6f4___v8i32_fp6__v8i32_fp6(
 ; SDAG-NEXT:    s_nop 0
 ; SDAG-NEXT:    v_mfma_scale_f32_32x32x64_f8f6f4 a[0:15], v[0:7], v[8:15], a[0:15], v32, v31 op_sel_hi:[0,0,0] cbsz:2 blgp:2
 ; SDAG-NEXT:    s_nop 7
-; SDAG-NEXT:    s_nop 2
+; SDAG-NEXT:    s_nop 3
 ; SDAG-NEXT:    v_accvgpr_read_b32 v0, a0
 ; SDAG-NEXT:    v_accvgpr_read_b32 v1, a1
 ; SDAG-NEXT:    v_accvgpr_read_b32 v2, a2
@@ -5678,7 +5678,7 @@ define <16 x float> @test_mfma_scale_f32_32x32x64_f8f6f4___v8i32_fp6__v8i32_fp6(
 ; GISEL-NEXT:    s_nop 0
 ; GISEL-NEXT:    v_mfma_scale_f32_32x32x64_f8f6f4 a[0:15], v[0:7], v[8:15], a[0:15], v31, v32 op_sel_hi:[0,0,0] cbsz:2 blgp:2
 ; GISEL-NEXT:    s_nop 7
-; GISEL-NEXT:    s_nop 2
+; GISEL-NEXT:    s_nop 3
 ; GISEL-NEXT:    v_accvgpr_read_b32 v0, a0
 ; GISEL-NEXT:    v_accvgpr_read_b32 v1, a1
 ; GISEL-NEXT:    v_accvgpr_read_b32 v2, a2
@@ -5727,7 +5727,7 @@ define <16 x float> @test_mfma_scale_f32_32x32x64_f8f6f4___v8i32_fp6__v8i32_fp6_
 ; GCN-NEXT:    s_nop 0
 ; GCN-NEXT:    v_mfma_f32_32x32x64_f8f6f4 a[0:15], v[0:7], v[8:15], a[0:15] cbsz:2 blgp:2
 ; GCN-NEXT:    s_nop 7
-; GCN-NEXT:    s_nop 2
+; GCN-NEXT:    s_nop 3
 ; GCN-NEXT:    v_accvgpr_read_b32 v0, a0
 ; GCN-NEXT:    v_accvgpr_read_b32 v1, a1
 ; GCN-NEXT:    v_accvgpr_read_b32 v2, a2
@@ -5779,7 +5779,7 @@ define <16 x float> @test_mfma_scale_f32_32x32x64_f8f6f4___v8i32_fp8__v8i32_fp4(
 ; SDAG-NEXT:    v_mfma_scale_f32_32x32x64_f8f6f4 a[0:15], v[0:7], v[8:15], a[0:15], v32, v31 op_sel_hi:[0,0,0] blgp:4
 ; SDAG-NEXT:    s_nop 7
 ; SDAG-NEXT:    s_nop 7
-; SDAG-NEXT:    s_nop 2
+; SDAG-NEXT:    s_nop 3
 ; SDAG-NEXT:    v_accvgpr_read_b32 v0, a0
 ; SDAG-NEXT:    v_accvgpr_read_b32 v1, a1
 ; SDAG-NEXT:    v_accvgpr_read_b32 v2, a2
@@ -5824,7 +5824,7 @@ define <16 x float> @test_mfma_scale_f32_32x32x64_f8f6f4___v8i32_fp8__v8i32_fp4(
 ; GISEL-NEXT:    v_mfma_scale_f32_32x32x64_f8f6f4 a[0:15], v[0:7], v[8:15], a[0:15], v31, v32 op_sel_hi:[0,0,0] blgp:4
 ; GISEL-NEXT:    s_nop 7
 ; GISEL-NEXT:    s_nop 7
-; GISEL-NEXT:    s_nop 2
+; GISEL-NEXT:    s_nop 3
 ; GISEL-NEXT:    v_accvgpr_read_b32 v0, a0
 ; GISEL-NEXT:    v_accvgpr_read_b32 v1, a1
 ; GISEL-NEXT:    v_accvgpr_read_b32 v2, a2
@@ -5876,7 +5876,7 @@ define <16 x float> @test_mfma_scale_f32_32x32x64_f8f6f4___v8i32_fp4__v8i32_fp8(
 ; SDAG-NEXT:    v_mfma_scale_f32_32x32x64_f8f6f4 a[0:15], v[0:7], v[8:15], a[0:15], v32, v31 op_sel_hi:[0,0,0] cbsz:4
 ; SDAG-NEXT:    s_nop 7
 ; SDAG-NEXT:    s_nop 7
-; SDAG-NEXT:    s_nop 2
+; SDAG-NEXT:    s_nop 3
 ; SDAG-NEXT:    v_accvgpr_read_b32 v0, a0
 ; SDAG-NEXT:    v_accvgpr_read_b32 v1, a1
 ; SDAG-NEXT:    v_accvgpr_read_b32 v2, a2
@@ -5921,7 +5921,7 @@ define <16 x float> @test_mfma_scale_f32_32x32x64_f8f6f4___v8i32_fp4__v8i32_fp8(
 ; GISEL-NEXT:    v_mfma_scale_f32_32x32x64_f8f6f4 a[0:15], v[0:7], v[8:15], a[0:15], v31, v32 op_sel_hi:[0,0,0] cbsz:4
 ; GISEL-NEXT:    s_nop 7
 ; GISEL-NEXT:    s_nop 7
-; GISEL-NEXT:    s_nop 2
+; GISEL-NEXT:    s_nop 3
 ; GISEL-NEXT:    v_accvgpr_read_b32 v0, a0
 ; GISEL-NEXT:    v_accvgpr_read_b32 v1, a1
 ; GISEL-NEXT:    v_accvgpr_read_b32 v2, a2
@@ -5972,7 +5972,7 @@ define <16 x float> @test_mfma_scale_f32_32x32x64_f8f6f4___v8i32_fp8__v6i32_fp4(
 ; GCN-NEXT:    v_mfma_scale_f32_32x32x64_f8f6f4 a[0:15], v[0:7], v[8:13], a[0:15], v30, v31 op_sel_hi:[0,0,0] blgp:4
 ; GCN-NEXT:    s_nop 7
 ; GCN-NEXT:    s_nop 7
-; GCN-NEXT:    s_nop 2
+; GCN-NEXT:    s_nop 3
 ; GCN-NEXT:    v_accvgpr_read_b32 v0, a0
 ; GCN-NEXT:    v_accvgpr_read_b32 v1, a1
 ; GCN-NEXT:    v_accvgpr_read_b32 v2, a2
@@ -6023,7 +6023,7 @@ define <16 x float> @test_mfma_scale_f32_32x32x64_f8f6f4___v6i32_fp4__v8i32_fp8(
 ; GCN-NEXT:    v_mfma_scale_f32_32x32x64_f8f6f4 a[0:15], v[0:5], v[6:13], a[0:15], v30, v31 op_sel_hi:[0,0,0] cbsz:4
 ; GCN-NEXT:    s_nop 7
 ; GCN-NEXT:    s_nop 7
-; GCN-NEXT:    s_nop 2
+; GCN-NEXT:    s_nop 3
 ; GCN-NEXT:    v_accvgpr_read_b32 v0, a0
 ; GCN-NEXT:    v_accvgpr_read_b32 v1, a1
 ; GCN-NEXT:    v_accvgpr_read_b32 v2, a2
@@ -6074,7 +6074,7 @@ define <16 x float> @test_mfma_scale_f32_32x32x64_f8f6f4___v8i32_fp4__v8i32_fp4(
 ; SDAG-NEXT:    s_nop 0
 ; SDAG-NEXT:    v_mfma_scale_f32_32x32x64_f8f6f4 a[0:15], v[0:7], v[8:15], a[0:15], v32, v31 op_sel_hi:[0,0,0] cbsz:4 blgp:4
 ; SDAG-NEXT:    s_nop 7
-; SDAG-NEXT:    s_nop 2
+; SDAG-NEXT:    s_nop 3
 ; SDAG-NEXT:    v_accvgpr_read_b32 v0, a0
 ; SDAG-NEXT:    v_accvgpr_read_b32 v1, a1
 ; SDAG-NEXT:    v_accvgpr_read_b32 v2, a2
@@ -6118,7 +6118,7 @@ define <16 x float> @test_mfma_scale_f32_32x32x64_f8f6f4___v8i32_fp4__v8i32_fp4(
 ; GISEL-NEXT:    s_nop 0
 ; GISEL-NEXT:    v_mfma_scale_f32_32x32x64_f8f6f4 a[0:15], v[0:7], v[8:15], a[0:15], v31, v32 op_sel_hi:[0,0,0] cbsz:4 blgp:4
 ; GISEL-NEXT:    s_nop 7
-; GISEL-NEXT:    s_nop 2
+; GISEL-NEXT:    s_nop 3
 ; GISEL-NEXT:    v_accvgpr_read_b32 v0, a0
 ; GISEL-NEXT:    v_accvgpr_read_b32 v1, a1
 ; GISEL-NEXT:    v_accvgpr_read_b32 v2, a2
@@ -6167,7 +6167,7 @@ define <16 x float> @test_mfma_scale_f32_32x32x64_f8f6f4___v8i32_fp4__v8i32_fp4_
 ; GCN-NEXT:    s_nop 0
 ; GCN-NEXT:    v_mfma_f32_32x32x64_f8f6f4 a[0:15], v[0:7], v[8:15], a[0:15] cbsz:4 blgp:4
 ; GCN-NEXT:    s_nop 7
-; GCN-NEXT:    s_nop 2
+; GCN-NEXT:    s_nop 3
 ; GCN-NEXT:    v_accvgpr_read_b32 v0, a0
 ; GCN-NEXT:    v_accvgpr_read_b32 v1, a1
 ; GCN-NEXT:    v_accvgpr_read_b32 v2, a2
@@ -6204,4 +6204,4 @@ declare <16 x float> @llvm.amdgcn.mfma.scale.f32.32x32x64.f8f6f4.v8i32.v6i32(<8
 
 attributes #0 = { "amdgpu-flat-work-group-size"="512,512" }
 attributes #1 = { "amdgpu-flat-work-group-size"="128,128" }
-attributes #2 = { convergent nocallback nofree nosync nounwind willreturn memory(none) }
+attributes #2 = { convergent nocallback nofree nosync nounwind willreturn memory(none) }
\ No newline at end of file
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mov.dpp8.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mov.dpp8.ll
index 53e37479f68e6..481e721e3c21d 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mov.dpp8.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mov.dpp8.ll
@@ -1,9 +1,9 @@
 ; RUN: llc -global-isel=0 -mtriple=amdgcn -mcpu=gfx1010 -verify-machineinstrs < %s | FileCheck -check-prefix=GFX10PLUS %s
-; RUN: llc -global-isel=1 -mtriple=amdgcn -mcpu=gfx1010 -verify-machineinstrs < %s | FileCheck -check-prefix=GFX10PLUS %s
+; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgcn -mcpu=gfx1010 -verify-machineinstrs < %s | FileCheck -check-prefix=GFX10PLUS %s
 ; RUN: llc -global-isel=0 -mtriple=amdgcn -mcpu=gfx1100 -amdgpu-enable-vopd=0 -verify-machineinstrs < %s | FileCheck -check-prefix=GFX10PLUS %s
-; RUN: llc -global-isel=1 -mtriple=amdgcn -mcpu=gfx1100 -amdgpu-enable-vopd=0 -verify-machineinstrs < %s | FileCheck -check-prefix=GFX10PLUS %s
+; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgcn -mcpu=gfx1100 -amdgpu-enable-vopd=0 -verify-machineinstrs < %s | FileCheck -check-prefix=GFX10PLUS %s
 ; RUN: llc -global-isel=0 -mtriple=amdgcn -mcpu=gfx1200 -amdgpu-enable-vopd=0 -verify-machineinstrs < %s | FileCheck -check-prefix=GFX10PLUS %s
-; RUN: llc -global-isel=1 -mtriple=amdgcn -mcpu=gfx1200 -amdgpu-enable-vopd=0 -verify-machineinstrs < %s | FileCheck -check-prefix=GFX10PLUS %s
+; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgcn -mcpu=gfx1200 -amdgpu-enable-vopd=0 -verify-machineinstrs < %s | FileCheck -check-prefix=GFX10PLUS %s
 
 ; GFX10PLUS-LABEL: {{^}}dpp8_test:
 ; GFX10PLUS: v_mov_b32_e32 [[SRC:v[0-9]+]], s{{[0-9]+}}
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.permlane.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.permlane.ll
index 71961a57bd080..076cf09678b57 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.permlane.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.permlane.ll
@@ -1,10 +1,10 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -global-isel=0 -amdgpu-load-store-vectorizer=0 -mtriple=amdgcn -mcpu=gfx1010 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX10,GFX10-SDAG %s
-; RUN: llc -global-isel=1 -amdgpu-load-store-vectorizer=0 -mtriple=amdgcn -mcpu=gfx1010 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX10,GFX10-GISEL %s
+; RUN: llc -global-isel=1 -global-isel-abort=2 -amdgpu-load-store-vectorizer=0 -mtriple=amdgcn -mcpu=gfx1010 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX10,GFX10-GISEL %s
 ; RUN: llc -global-isel=0 -amdgpu-load-store-vectorizer=0 -mtriple=amdgcn -mcpu=gfx1100 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX11,GFX11-SDAG %s
-; RUN: llc -global-isel=1 -amdgpu-load-store-vectorizer=0 -mtriple=amdgcn -mcpu=gfx1100 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX11,GFX11-GISEL %s
+; RUN: llc -global-isel=1 -global-isel-abort=2 -amdgpu-load-store-vectorizer=0 -mtriple=amdgcn -mcpu=gfx1100 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX11,GFX11-GISEL %s
 ; RUN: llc -global-isel=0 -amdgpu-load-store-vectorizer=0 -mtriple=amdgcn -mcpu=gfx1200 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX12,GFX12-SDAG %s
-; RUN: llc -global-isel=1 -amdgpu-load-store-vectorizer=0 -mtriple=amdgcn -mcpu=gfx1200 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX12,GFX12-GISEL %s
+; RUN: llc -global-isel=1 -global-isel-abort=2 -amdgpu-load-store-vectorizer=0 -mtriple=amdgcn -mcpu=gfx1200 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX12,GFX12-GISEL %s
 
 declare i32 @llvm.amdgcn.permlane16(i32, i32, i32, i32, i1, i1)
 declare i32 @llvm.amdgcn.permlanex16(i32, i32, i32, i32, i1, i1)
@@ -840,6 +840,7 @@ define amdgpu_kernel void @v_permlane16_b32_vvv_i32(ptr addrspace(1) %out, i32 %
 ; GFX12-SDAG-NEXT:    v_mov_b32_e32 v1, s2
 ; GFX12-SDAG-NEXT:    v_readfirstlane_b32 s2, v0
 ; GFX12-SDAG-NEXT:    v_mov_b32_e32 v0, 0
+; GFX12-SDAG-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2)
 ; GFX12-SDAG-NEXT:    v_permlane16_b32 v1, v1, s3, s2
 ; GFX12-SDAG-NEXT:    global_store_b32 v0, v1, s[0:1]
@@ -850,12 +851,14 @@ define amdgpu_kernel void @v_permlane16_b32_vvv_i32(ptr addrspace(1) %out, i32 %
 ; GFX12-GISEL-NEXT:    s_load_b96 s[0:2], s[4:5], 0x24
 ; GFX12-GISEL-NEXT:    v_and_b32_e32 v1, 0x3ff, v0
 ; GFX12-GISEL-NEXT:    v_bfe_u32 v0, v0, 10, 10
-; GFX12-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_4) | instid1(VALU_DEP_2)
+; GFX12-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-GISEL-NEXT:    v_readfirstlane_b32 s4, v0
 ; GFX12-GISEL-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-GISEL-NEXT:    v_mov_b32_e32 v0, s2
 ; GFX12-GISEL-NEXT:    v_readfirstlane_b32 s3, v1
 ; GFX12-GISEL-NEXT:    v_mov_b32_e32 v1, 0
+; GFX12-GISEL-NEXT:    s_wait_alu 0xf1ff
+; GFX12-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_2)
 ; GFX12-GISEL-NEXT:    v_permlane16_b32 v0, v0, s3, s4
 ; GFX12-GISEL-NEXT:    global_store_b32 v1, v0, s[0:1]
 ; GFX12-GISEL-NEXT:    s_endpgm
@@ -1015,6 +1018,7 @@ define amdgpu_kernel void @v_permlane16_b32_vvv_f32(ptr addrspace(1) %out, float
 ; GFX12-SDAG-NEXT:    v_mov_b32_e32 v1, s2
 ; GFX12-SDAG-NEXT:    v_readfirstlane_b32 s2, v0
 ; GFX12-SDAG-NEXT:    v_mov_b32_e32 v0, 0
+; GFX12-SDAG-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2)
 ; GFX12-SDAG-NEXT:    v_permlane16_b32 v1, v1, s3, s2
 ; GFX12-SDAG-NEXT:    global_store_b32 v0, v1, s[0:1]
@@ -1025,12 +1029,14 @@ define amdgpu_kernel void @v_permlane16_b32_vvv_f32(ptr addrspace(1) %out, float
 ; GFX12-GISEL-NEXT:    s_load_b96 s[0:2], s[4:5], 0x24
 ; GFX12-GISEL-NEXT:    v_and_b32_e32 v1, 0x3ff, v0
 ; GFX12-GISEL-NEXT:    v_bfe_u32 v0, v0, 10, 10
-; GFX12-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_4) | instid1(VALU_DEP_2)
+; GFX12-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-GISEL-NEXT:    v_readfirstlane_b32 s4, v0
 ; GFX12-GISEL-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-GISEL-NEXT:    v_mov_b32_e32 v0, s2
 ; GFX12-GISEL-NEXT:    v_readfirstlane_b32 s3, v1
 ; GFX12-GISEL-NEXT:    v_mov_b32_e32 v1, 0
+; GFX12-GISEL-NEXT:    s_wait_alu 0xf1ff
+; GFX12-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_2)
 ; GFX12-GISEL-NEXT:    v_permlane16_b32 v0, v0, s3, s4
 ; GFX12-GISEL-NEXT:    global_store_b32 v1, v0, s[0:1]
 ; GFX12-GISEL-NEXT:    s_endpgm
@@ -1167,9 +1173,10 @@ define amdgpu_kernel void @v_permlane16_b32_vvs_i32(ptr addrspace(1) %out, i32 %
 ; GFX12-SDAG-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
 ; GFX12-SDAG-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-SDAG-NEXT:    v_dual_mov_b32 v1, s2 :: v_dual_and_b32 v0, 0x3ff, v0
-; GFX12-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
 ; GFX12-SDAG-NEXT:    v_readfirstlane_b32 s2, v0
 ; GFX12-SDAG-NEXT:    v_mov_b32_e32 v0, 0
+; GFX12-SDAG-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-SDAG-NEXT:    v_permlane16_b32 v1, v1, s2, s3
 ; GFX12-SDAG-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-SDAG-NEXT:    s_endpgm
@@ -1316,9 +1323,10 @@ define amdgpu_kernel void @v_permlane16_b32_vvs_f32(ptr addrspace(1) %out, float
 ; GFX12-SDAG-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
 ; GFX12-SDAG-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-SDAG-NEXT:    v_dual_mov_b32 v1, s2 :: v_dual_and_b32 v0, 0x3ff, v0
-; GFX12-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
 ; GFX12-SDAG-NEXT:    v_readfirstlane_b32 s2, v0
 ; GFX12-SDAG-NEXT:    v_mov_b32_e32 v0, 0
+; GFX12-SDAG-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-SDAG-NEXT:    v_permlane16_b32 v1, v1, s2, s3
 ; GFX12-SDAG-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-SDAG-NEXT:    s_endpgm
@@ -1468,9 +1476,10 @@ define amdgpu_kernel void @v_permlane16_b32_vsv_i32(ptr addrspace(1) %out, i32 %
 ; GFX12-SDAG-NEXT:    v_bfe_u32 v0, v0, 10, 10
 ; GFX12-SDAG-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-SDAG-NEXT:    v_mov_b32_e32 v1, s2
-; GFX12-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_2)
 ; GFX12-SDAG-NEXT:    v_readfirstlane_b32 s2, v0
 ; GFX12-SDAG-NEXT:    v_mov_b32_e32 v0, 0
+; GFX12-SDAG-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-SDAG-NEXT:    v_permlane16_b32 v1, v1, s3, s2
 ; GFX12-SDAG-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-SDAG-NEXT:    s_endpgm
@@ -1655,9 +1664,10 @@ define amdgpu_kernel void @v_permlane16_b32_vsv_f32(ptr addrspace(1) %out, float
 ; GFX12-SDAG-NEXT:    v_bfe_u32 v0, v0, 10, 10
 ; GFX12-SDAG-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-SDAG-NEXT:    v_mov_b32_e32 v1, s2
-; GFX12-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_2)
 ; GFX12-SDAG-NEXT:    v_readfirstlane_b32 s2, v0
 ; GFX12-SDAG-NEXT:    v_mov_b32_e32 v0, 0
+; GFX12-SDAG-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-SDAG-NEXT:    v_permlane16_b32 v1, v1, s3, s2
 ; GFX12-SDAG-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-SDAG-NEXT:    s_endpgm
@@ -3395,6 +3405,7 @@ define amdgpu_kernel void @v_permlanex16_b32_vvv_i32(ptr addrspace(1) %out, i32
 ; GFX12-SDAG-NEXT:    v_mov_b32_e32 v1, s2
 ; GFX12-SDAG-NEXT:    v_readfirstlane_b32 s2, v0
 ; GFX12-SDAG-NEXT:    v_mov_b32_e32 v0, 0
+; GFX12-SDAG-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2)
 ; GFX12-SDAG-NEXT:    v_permlanex16_b32 v1, v1, s3, s2
 ; GFX12-SDAG-NEXT:    global_store_b32 v0, v1, s[0:1]
@@ -3405,12 +3416,14 @@ define amdgpu_kernel void @v_permlanex16_b32_vvv_i32(ptr addrspace(1) %out, i32
 ; GFX12-GISEL-NEXT:    s_load_b96 s[0:2], s[4:5], 0x24
 ; GFX12-GISEL-NEXT:    v_and_b32_e32 v1, 0x3ff, v0
 ; GFX12-GISEL-NEXT:    v_bfe_u32 v0, v0, 10, 10
-; GFX12-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_4) | instid1(VALU_DEP_2)
+; GFX12-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-GISEL-NEXT:    v_readfirstlane_b32 s4, v0
 ; GFX12-GISEL-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-GISEL-NEXT:    v_mov_b32_e32 v0, s2
 ; GFX12-GISEL-NEXT:    v_readfirstlane_b32 s3, v1
 ; GFX12-GISEL-NEXT:    v_mov_b32_e32 v1, 0
+; GFX12-GISEL-NEXT:    s_wait_alu 0xf1ff
+; GFX12-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_2)
 ; GFX12-GISEL-NEXT:    v_permlanex16_b32 v0, v0, s3, s4
 ; GFX12-GISEL-NEXT:    global_store_b32 v1, v0, s[0:1]
 ; GFX12-GISEL-NEXT:    s_endpgm
@@ -3497,6 +3510,7 @@ define amdgpu_kernel void @v_permlanex16_b32_vvv_f32(ptr addrspace(1) %out, floa
 ; GFX12-SDAG-NEXT:    v_mov_b32_e32 v1, s2
 ; GFX12-SDAG-NEXT:    v_readfirstlane_b32 s2, v0
 ; GFX12-SDAG-NEXT:    v_mov_b32_e32 v0, 0
+; GFX12-SDAG-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2)
 ; GFX12-SDAG-NEXT:    v_permlanex16_b32 v1, v1, s3, s2
 ; GFX12-SDAG-NEXT:    global_store_b32 v0, v1, s[0:1]
@@ -3507,12 +3521,14 @@ define amdgpu_kernel void @v_permlanex16_b32_vvv_f32(ptr addrspace(1) %out, floa
 ; GFX12-GISEL-NEXT:    s_load_b96 s[0:2], s[4:5], 0x24
 ; GFX12-GISEL-NEXT:    v_and_b32_e32 v1, 0x3ff, v0
 ; GFX12-GISEL-NEXT:    v_bfe_u32 v0, v0, 10, 10
-; GFX12-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_4) | instid1(VALU_DEP_2)
+; GFX12-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-GISEL-NEXT:    v_readfirstlane_b32 s4, v0
 ; GFX12-GISEL-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-GISEL-NEXT:    v_mov_b32_e32 v0, s2
 ; GFX12-GISEL-NEXT:    v_readfirstlane_b32 s3, v1
 ; GFX12-GISEL-NEXT:    v_mov_b32_e32 v1, 0
+; GFX12-GISEL-NEXT:    s_wait_alu 0xf1ff
+; GFX12-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_2)
 ; GFX12-GISEL-NEXT:    v_permlanex16_b32 v0, v0, s3, s4
 ; GFX12-GISEL-NEXT:    global_store_b32 v1, v0, s[0:1]
 ; GFX12-GISEL-NEXT:    s_endpgm
@@ -3722,9 +3738,10 @@ define amdgpu_kernel void @v_permlanex16_b32_vvs_i32(ptr addrspace(1) %out, i32
 ; GFX12-SDAG-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
 ; GFX12-SDAG-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-SDAG-NEXT:    v_dual_mov_b32 v1, s2 :: v_dual_and_b32 v0, 0x3ff, v0
-; GFX12-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
 ; GFX12-SDAG-NEXT:    v_readfirstlane_b32 s2, v0
 ; GFX12-SDAG-NEXT:    v_mov_b32_e32 v0, 0
+; GFX12-SDAG-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-SDAG-NEXT:    v_permlanex16_b32 v1, v1, s2, s3
 ; GFX12-SDAG-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-SDAG-NEXT:    s_endpgm
@@ -3799,9 +3816,10 @@ define amdgpu_kernel void @v_permlanex16_b32_vvs_f32(ptr addrspace(1) %out, floa
 ; GFX12-SDAG-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
 ; GFX12-SDAG-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-SDAG-NEXT:    v_dual_mov_b32 v1, s2 :: v_dual_and_b32 v0, 0x3ff, v0
-; GFX12-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
 ; GFX12-SDAG-NEXT:    v_readfirstlane_b32 s2, v0
 ; GFX12-SDAG-NEXT:    v_mov_b32_e32 v0, 0
+; GFX12-SDAG-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-SDAG-NEXT:    v_permlanex16_b32 v1, v1, s2, s3
 ; GFX12-SDAG-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-SDAG-NEXT:    s_endpgm
@@ -4023,9 +4041,10 @@ define amdgpu_kernel void @v_permlanex16_b32_vsv_i32(ptr addrspace(1) %out, i32
 ; GFX12-SDAG-NEXT:    v_bfe_u32 v0, v0, 10, 10
 ; GFX12-SDAG-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-SDAG-NEXT:    v_mov_b32_e32 v1, s2
-; GFX12-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_2)
 ; GFX12-SDAG-NEXT:    v_readfirstlane_b32 s2, v0
 ; GFX12-SDAG-NEXT:    v_mov_b32_e32 v0, 0
+; GFX12-SDAG-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-SDAG-NEXT:    v_permlanex16_b32 v1, v1, s3, s2
 ; GFX12-SDAG-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-SDAG-NEXT:    s_endpgm
@@ -4104,9 +4123,10 @@ define amdgpu_kernel void @v_permlanex16_b32_vsv_f32(ptr addrspace(1) %out, floa
 ; GFX12-SDAG-NEXT:    v_bfe_u32 v0, v0, 10, 10
 ; GFX12-SDAG-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-SDAG-NEXT:    v_mov_b32_e32 v1, s2
-; GFX12-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_2)
 ; GFX12-SDAG-NEXT:    v_readfirstlane_b32 s2, v0
 ; GFX12-SDAG-NEXT:    v_mov_b32_e32 v0, 0
+; GFX12-SDAG-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-SDAG-NEXT:    v_permlanex16_b32 v1, v1, s3, s2
 ; GFX12-SDAG-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-SDAG-NEXT:    s_endpgm
@@ -8550,6 +8570,7 @@ define void @v_permlane16_half(ptr addrspace(1) %out, half %src0, i32 %src1, i32
 ; GFX12-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-NEXT:    v_readfirstlane_b32 s0, v3
 ; GFX12-NEXT:    v_readfirstlane_b32 s1, v4
+; GFX12-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-NEXT:    v_permlane16_b32 v2, v2, s0, s1
 ; GFX12-NEXT:    global_store_b16 v[0:1], v2, off
@@ -8588,6 +8609,7 @@ define void @v_permlanex16_half(ptr addrspace(1) %out, half %src0, i32 %src1, i3
 ; GFX12-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-NEXT:    v_readfirstlane_b32 s0, v3
 ; GFX12-NEXT:    v_readfirstlane_b32 s1, v4
+; GFX12-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-NEXT:    v_permlanex16_b32 v2, v2, s0, s1
 ; GFX12-NEXT:    global_store_b16 v[0:1], v2, off
@@ -8626,6 +8648,7 @@ define void @v_permlane16_bfloat(ptr addrspace(1) %out, bfloat %src0, i32 %src1,
 ; GFX12-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-NEXT:    v_readfirstlane_b32 s0, v3
 ; GFX12-NEXT:    v_readfirstlane_b32 s1, v4
+; GFX12-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-NEXT:    v_permlane16_b32 v2, v2, s0, s1
 ; GFX12-NEXT:    global_store_b16 v[0:1], v2, off
@@ -8664,6 +8687,7 @@ define void @v_permlanex16_bfloat(ptr addrspace(1) %out, bfloat %src0, i32 %src1
 ; GFX12-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-NEXT:    v_readfirstlane_b32 s0, v3
 ; GFX12-NEXT:    v_readfirstlane_b32 s1, v4
+; GFX12-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-NEXT:    v_permlanex16_b32 v2, v2, s0, s1
 ; GFX12-NEXT:    global_store_b16 v[0:1], v2, off
@@ -8702,6 +8726,7 @@ define void @v_permlane16_i16(ptr addrspace(1) %out, i16 %src0, i32 %src1, i32 %
 ; GFX12-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-NEXT:    v_readfirstlane_b32 s0, v3
 ; GFX12-NEXT:    v_readfirstlane_b32 s1, v4
+; GFX12-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-NEXT:    v_permlane16_b32 v2, v2, s0, s1
 ; GFX12-NEXT:    global_store_b16 v[0:1], v2, off
@@ -8740,6 +8765,7 @@ define void @v_permlanex16_i16(ptr addrspace(1) %out, i16 %src0, i32 %src1, i32
 ; GFX12-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-NEXT:    v_readfirstlane_b32 s0, v3
 ; GFX12-NEXT:    v_readfirstlane_b32 s1, v4
+; GFX12-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-NEXT:    v_permlanex16_b32 v2, v2, s0, s1
 ; GFX12-NEXT:    global_store_b16 v[0:1], v2, off
@@ -8778,6 +8804,7 @@ define void @v_permlane16_v2f16(ptr addrspace(1) %out, <2 x half> %src0, i32 %sr
 ; GFX12-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-NEXT:    v_readfirstlane_b32 s0, v3
 ; GFX12-NEXT:    v_readfirstlane_b32 s1, v4
+; GFX12-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-NEXT:    v_permlane16_b32 v2, v2, s0, s1
 ; GFX12-NEXT:    global_store_b32 v[0:1], v2, off
@@ -8816,6 +8843,7 @@ define void @v_permlanex16_v2f16(ptr addrspace(1) %out, <2 x half> %src0, i32 %s
 ; GFX12-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-NEXT:    v_readfirstlane_b32 s0, v3
 ; GFX12-NEXT:    v_readfirstlane_b32 s1, v4
+; GFX12-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-NEXT:    v_permlanex16_b32 v2, v2, s0, s1
 ; GFX12-NEXT:    global_store_b32 v[0:1], v2, off
@@ -8877,6 +8905,7 @@ define void @v_permlane16_v2f32(ptr addrspace(1) %out, <2 x float> %src0, i32 %s
 ; GFX12-SDAG-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-SDAG-NEXT:    v_readfirstlane_b32 s0, v4
 ; GFX12-SDAG-NEXT:    v_readfirstlane_b32 s1, v5
+; GFX12-SDAG-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-SDAG-NEXT:    v_permlane16_b32 v3, v3, s0, s1
 ; GFX12-SDAG-NEXT:    v_permlane16_b32 v2, v2, s0, s1
@@ -8892,6 +8921,7 @@ define void @v_permlane16_v2f32(ptr addrspace(1) %out, <2 x float> %src0, i32 %s
 ; GFX12-GISEL-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-GISEL-NEXT:    v_readfirstlane_b32 s0, v4
 ; GFX12-GISEL-NEXT:    v_readfirstlane_b32 s1, v5
+; GFX12-GISEL-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-GISEL-NEXT:    v_permlane16_b32 v2, v2, s0, s1
 ; GFX12-GISEL-NEXT:    v_permlane16_b32 v3, v3, s0, s1
@@ -8954,6 +8984,7 @@ define void @v_permlanex16_v2f32(ptr addrspace(1) %out, <2 x float> %src0, i32 %
 ; GFX12-SDAG-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-SDAG-NEXT:    v_readfirstlane_b32 s0, v4
 ; GFX12-SDAG-NEXT:    v_readfirstlane_b32 s1, v5
+; GFX12-SDAG-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-SDAG-NEXT:    v_permlanex16_b32 v3, v3, s0, s1
 ; GFX12-SDAG-NEXT:    v_permlanex16_b32 v2, v2, s0, s1
@@ -8969,6 +9000,7 @@ define void @v_permlanex16_v2f32(ptr addrspace(1) %out, <2 x float> %src0, i32 %
 ; GFX12-GISEL-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-GISEL-NEXT:    v_readfirstlane_b32 s0, v4
 ; GFX12-GISEL-NEXT:    v_readfirstlane_b32 s1, v5
+; GFX12-GISEL-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-GISEL-NEXT:    v_permlanex16_b32 v2, v2, s0, s1
 ; GFX12-GISEL-NEXT:    v_permlanex16_b32 v3, v3, s0, s1
@@ -9057,6 +9089,7 @@ define void @v_permlane16_v7i32(ptr addrspace(1) %out, <7 x i32> %src0, i32 %src
 ; GFX12-SDAG-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-SDAG-NEXT:    v_readfirstlane_b32 s0, v9
 ; GFX12-SDAG-NEXT:    v_readfirstlane_b32 s1, v10
+; GFX12-SDAG-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-SDAG-NEXT:    v_permlane16_b32 v8, v8, s0, s1
 ; GFX12-SDAG-NEXT:    v_permlane16_b32 v7, v7, s0, s1
@@ -9079,6 +9112,7 @@ define void @v_permlane16_v7i32(ptr addrspace(1) %out, <7 x i32> %src0, i32 %src
 ; GFX12-GISEL-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-GISEL-NEXT:    v_readfirstlane_b32 s0, v9
 ; GFX12-GISEL-NEXT:    v_readfirstlane_b32 s1, v10
+; GFX12-GISEL-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-GISEL-NEXT:    v_permlane16_b32 v2, v2, s0, s1
 ; GFX12-GISEL-NEXT:    v_permlane16_b32 v3, v3, s0, s1
@@ -9174,6 +9208,7 @@ define void @v_permlanex16_v7i32(ptr addrspace(1) %out, <7 x i32> %src0, i32 %sr
 ; GFX12-SDAG-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-SDAG-NEXT:    v_readfirstlane_b32 s0, v9
 ; GFX12-SDAG-NEXT:    v_readfirstlane_b32 s1, v10
+; GFX12-SDAG-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-SDAG-NEXT:    v_permlanex16_b32 v8, v8, s0, s1
 ; GFX12-SDAG-NEXT:    v_permlanex16_b32 v7, v7, s0, s1
@@ -9196,6 +9231,7 @@ define void @v_permlanex16_v7i32(ptr addrspace(1) %out, <7 x i32> %src0, i32 %sr
 ; GFX12-GISEL-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-GISEL-NEXT:    v_readfirstlane_b32 s0, v9
 ; GFX12-GISEL-NEXT:    v_readfirstlane_b32 s1, v10
+; GFX12-GISEL-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-GISEL-NEXT:    v_permlanex16_b32 v2, v2, s0, s1
 ; GFX12-GISEL-NEXT:    v_permlanex16_b32 v3, v3, s0, s1
@@ -9273,6 +9309,7 @@ define void @v_permlane16_v8i16(ptr addrspace(1) %out, <8 x i16> %src0, i32 %src
 ; GFX12-SDAG-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-SDAG-NEXT:    v_readfirstlane_b32 s0, v6
 ; GFX12-SDAG-NEXT:    v_readfirstlane_b32 s1, v7
+; GFX12-SDAG-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-SDAG-NEXT:    v_permlane16_b32 v5, v5, s0, s1
 ; GFX12-SDAG-NEXT:    v_permlane16_b32 v4, v4, s0, s1
@@ -9290,6 +9327,7 @@ define void @v_permlane16_v8i16(ptr addrspace(1) %out, <8 x i16> %src0, i32 %src
 ; GFX12-GISEL-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-GISEL-NEXT:    v_readfirstlane_b32 s0, v6
 ; GFX12-GISEL-NEXT:    v_readfirstlane_b32 s1, v7
+; GFX12-GISEL-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-GISEL-NEXT:    v_permlane16_b32 v2, v2, s0, s1
 ; GFX12-GISEL-NEXT:    v_permlane16_b32 v3, v3, s0, s1
@@ -9362,6 +9400,7 @@ define void @v_permlanex16_v8i16(ptr addrspace(1) %out, <8 x i16> %src0, i32 %sr
 ; GFX12-SDAG-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-SDAG-NEXT:    v_readfirstlane_b32 s0, v6
 ; GFX12-SDAG-NEXT:    v_readfirstlane_b32 s1, v7
+; GFX12-SDAG-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-SDAG-NEXT:    v_permlanex16_b32 v5, v5, s0, s1
 ; GFX12-SDAG-NEXT:    v_permlanex16_b32 v4, v4, s0, s1
@@ -9379,6 +9418,7 @@ define void @v_permlanex16_v8i16(ptr addrspace(1) %out, <8 x i16> %src0, i32 %sr
 ; GFX12-GISEL-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-GISEL-NEXT:    v_readfirstlane_b32 s0, v6
 ; GFX12-GISEL-NEXT:    v_readfirstlane_b32 s1, v7
+; GFX12-GISEL-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-GISEL-NEXT:    v_permlanex16_b32 v2, v2, s0, s1
 ; GFX12-GISEL-NEXT:    v_permlanex16_b32 v3, v3, s0, s1
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.permlane.ptr.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.permlane.ptr.ll
index bb42834221681..10c000095fe3d 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.permlane.ptr.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.permlane.ptr.ll
@@ -34,6 +34,7 @@ define void @v_permlane16_p0(ptr addrspace(1) %out, ptr %src0, i32 %src1, i32 %s
 ; GFX12-SDAG-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-SDAG-NEXT:    v_readfirstlane_b32 s0, v4
 ; GFX12-SDAG-NEXT:    v_readfirstlane_b32 s1, v5
+; GFX12-SDAG-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-SDAG-NEXT:    v_permlane16_b32 v3, v3, s0, s1
 ; GFX12-SDAG-NEXT:    v_permlane16_b32 v2, v2, s0, s1
@@ -75,6 +76,7 @@ define void @v_permlanex16_p0(ptr addrspace(1) %out, ptr %src0, i32 %src1, i32 %
 ; GFX12-SDAG-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-SDAG-NEXT:    v_readfirstlane_b32 s0, v4
 ; GFX12-SDAG-NEXT:    v_readfirstlane_b32 s1, v5
+; GFX12-SDAG-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-SDAG-NEXT:    v_permlanex16_b32 v3, v3, s0, s1
 ; GFX12-SDAG-NEXT:    v_permlanex16_b32 v2, v2, s0, s1
@@ -127,6 +129,7 @@ define void @v_permlane16_v3p0(ptr addrspace(1) %out, <3 x ptr> %src0, i32 %src1
 ; GFX12-SDAG-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-SDAG-NEXT:    v_readfirstlane_b32 s0, v8
 ; GFX12-SDAG-NEXT:    v_readfirstlane_b32 s1, v9
+; GFX12-SDAG-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-SDAG-NEXT:    v_permlane16_b32 v7, v7, s0, s1
 ; GFX12-SDAG-NEXT:    v_permlane16_b32 v6, v6, s0, s1
@@ -185,6 +188,7 @@ define void @v_permlanex16_v3p0(ptr addrspace(1) %out, <3 x ptr> %src0, i32 %src
 ; GFX12-SDAG-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-SDAG-NEXT:    v_readfirstlane_b32 s0, v8
 ; GFX12-SDAG-NEXT:    v_readfirstlane_b32 s1, v9
+; GFX12-SDAG-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-SDAG-NEXT:    v_permlanex16_b32 v7, v7, s0, s1
 ; GFX12-SDAG-NEXT:    v_permlanex16_b32 v6, v6, s0, s1
@@ -230,6 +234,7 @@ define void @v_permlane16_p3(ptr addrspace(1) %out, ptr addrspace(3) %src0, i32
 ; GFX12-SDAG-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-SDAG-NEXT:    v_readfirstlane_b32 s0, v3
 ; GFX12-SDAG-NEXT:    v_readfirstlane_b32 s1, v4
+; GFX12-SDAG-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-SDAG-NEXT:    v_permlane16_b32 v2, v2, s0, s1
 ; GFX12-SDAG-NEXT:    global_store_b32 v[0:1], v2, off
@@ -268,6 +273,7 @@ define void @v_permlanex16_p3(ptr addrspace(1) %out, ptr addrspace(3) %src0, i32
 ; GFX12-SDAG-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-SDAG-NEXT:    v_readfirstlane_b32 s0, v3
 ; GFX12-SDAG-NEXT:    v_readfirstlane_b32 s1, v4
+; GFX12-SDAG-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-SDAG-NEXT:    v_permlanex16_b32 v2, v2, s0, s1
 ; GFX12-SDAG-NEXT:    global_store_b32 v[0:1], v2, off
@@ -310,6 +316,7 @@ define void @v_permlane16_v3p3(ptr addrspace(1) %out, <3 x ptr addrspace(3)> %sr
 ; GFX12-SDAG-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-SDAG-NEXT:    v_readfirstlane_b32 s0, v5
 ; GFX12-SDAG-NEXT:    v_readfirstlane_b32 s1, v6
+; GFX12-SDAG-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-SDAG-NEXT:    v_permlane16_b32 v4, v4, s0, s1
 ; GFX12-SDAG-NEXT:    v_permlane16_b32 v3, v3, s0, s1
@@ -354,6 +361,7 @@ define void @v_permlanex16_v3p3(ptr addrspace(1) %out, <3 x ptr addrspace(3)> %s
 ; GFX12-SDAG-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-SDAG-NEXT:    v_readfirstlane_b32 s0, v5
 ; GFX12-SDAG-NEXT:    v_readfirstlane_b32 s1, v6
+; GFX12-SDAG-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-SDAG-NEXT:    v_permlanex16_b32 v4, v4, s0, s1
 ; GFX12-SDAG-NEXT:    v_permlanex16_b32 v3, v3, s0, s1
@@ -394,6 +402,7 @@ define void @v_permlane16_p5(ptr addrspace(1) %out, ptr addrspace(5) %src0, i32
 ; GFX12-SDAG-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-SDAG-NEXT:    v_readfirstlane_b32 s0, v3
 ; GFX12-SDAG-NEXT:    v_readfirstlane_b32 s1, v4
+; GFX12-SDAG-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-SDAG-NEXT:    v_permlane16_b32 v2, v2, s0, s1
 ; GFX12-SDAG-NEXT:    global_store_b32 v[0:1], v2, off
@@ -432,6 +441,7 @@ define void @v_permlanex16_p5(ptr addrspace(1) %out, ptr addrspace(5) %src0, i32
 ; GFX12-SDAG-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-SDAG-NEXT:    v_readfirstlane_b32 s0, v3
 ; GFX12-SDAG-NEXT:    v_readfirstlane_b32 s1, v4
+; GFX12-SDAG-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-SDAG-NEXT:    v_permlanex16_b32 v2, v2, s0, s1
 ; GFX12-SDAG-NEXT:    global_store_b32 v[0:1], v2, off
@@ -474,6 +484,7 @@ define void @v_permlane16_v3p5(ptr addrspace(1) %out, <3 x ptr addrspace(5)> %sr
 ; GFX12-SDAG-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-SDAG-NEXT:    v_readfirstlane_b32 s0, v5
 ; GFX12-SDAG-NEXT:    v_readfirstlane_b32 s1, v6
+; GFX12-SDAG-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-SDAG-NEXT:    v_permlane16_b32 v4, v4, s0, s1
 ; GFX12-SDAG-NEXT:    v_permlane16_b32 v3, v3, s0, s1
@@ -518,6 +529,7 @@ define void @v_permlanex16_v3p5(ptr addrspace(1) %out, <3 x ptr addrspace(5)> %s
 ; GFX12-SDAG-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-SDAG-NEXT:    v_readfirstlane_b32 s0, v5
 ; GFX12-SDAG-NEXT:    v_readfirstlane_b32 s1, v6
+; GFX12-SDAG-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-SDAG-NEXT:    v_permlanex16_b32 v4, v4, s0, s1
 ; GFX12-SDAG-NEXT:    v_permlanex16_b32 v3, v3, s0, s1
@@ -558,6 +570,7 @@ define void @v_permlane16_p6(ptr addrspace(1) %out, ptr addrspace(6) %src0, i32
 ; GFX12-SDAG-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-SDAG-NEXT:    v_readfirstlane_b32 s0, v3
 ; GFX12-SDAG-NEXT:    v_readfirstlane_b32 s1, v4
+; GFX12-SDAG-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-SDAG-NEXT:    v_permlane16_b32 v2, v2, s0, s1
 ; GFX12-SDAG-NEXT:    global_store_b32 v[0:1], v2, off
@@ -596,6 +609,7 @@ define void @v_permlanex16_p6(ptr addrspace(1) %out, ptr addrspace(6) %src0, i32
 ; GFX12-SDAG-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-SDAG-NEXT:    v_readfirstlane_b32 s0, v3
 ; GFX12-SDAG-NEXT:    v_readfirstlane_b32 s1, v4
+; GFX12-SDAG-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-SDAG-NEXT:    v_permlanex16_b32 v2, v2, s0, s1
 ; GFX12-SDAG-NEXT:    global_store_b32 v[0:1], v2, off
@@ -638,6 +652,7 @@ define void @v_permlane16_v3p6(ptr addrspace(1) %out, <3 x ptr addrspace(6)> %sr
 ; GFX12-SDAG-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-SDAG-NEXT:    v_readfirstlane_b32 s0, v5
 ; GFX12-SDAG-NEXT:    v_readfirstlane_b32 s1, v6
+; GFX12-SDAG-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-SDAG-NEXT:    v_permlane16_b32 v4, v4, s0, s1
 ; GFX12-SDAG-NEXT:    v_permlane16_b32 v3, v3, s0, s1
@@ -682,6 +697,7 @@ define void @v_permlanex16_v3p6(ptr addrspace(1) %out, <3 x ptr addrspace(6)> %s
 ; GFX12-SDAG-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-SDAG-NEXT:    v_readfirstlane_b32 s0, v5
 ; GFX12-SDAG-NEXT:    v_readfirstlane_b32 s1, v6
+; GFX12-SDAG-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-SDAG-NEXT:    v_permlanex16_b32 v4, v4, s0, s1
 ; GFX12-SDAG-NEXT:    v_permlanex16_b32 v3, v3, s0, s1
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ptr.buffer.atomic.fadd_rtn_errors.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ptr.buffer.atomic.fadd_rtn_errors.ll
index f8caf84d5c51a..09cc55b53539b 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ptr.buffer.atomic.fadd_rtn_errors.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ptr.buffer.atomic.fadd_rtn_errors.ll
@@ -15,14 +15,12 @@
 ; RUN: not --crash llc -global-isel=1 -mtriple=amdgcn-amd-amdhsa -mcpu=gfx908 -filetype=null %t/struct-ret-f32-error.ll 2>&1 | FileCheck -check-prefix=ERR-STRUCT-F32-GISEL %s
 ; RUN: not --crash llc -global-isel=1 -mtriple=amdgcn-amd-amdhsa -mcpu=gfx908 -filetype=null %t/raw-ret-v2f16-error.ll 2>&1 | FileCheck -check-prefix=ERR-RAW-V2F16-GISEL %s
 ; RUN: not --crash llc -global-isel=1 -mtriple=amdgcn-amd-amdhsa -mcpu=gfx908 -filetype=null %t/struct-ret-v2f16-error.ll 2>&1 | FileCheck -check-prefix=ERR-STRUCT-V2F16-GISEL %s
-; RUN: not --crash llc -global-isel=1 -mtriple=amdgcn-amd-amdhsa -mcpu=gfx908 -filetype=null %t/raw-ret-v2bf16-error.ll 2>&1 | FileCheck -check-prefix=ERR-RAW-V2BF16-GISEL %s
-; RUN: not --crash llc -global-isel=1 -mtriple=amdgcn-amd-amdhsa -mcpu=gfx908 -filetype=null %t/struct-ret-v2bf16-error.ll 2>&1 | FileCheck -check-prefix=ERR-STRUCT-V2BF16-GISEL %s
-
-; FIXME: These should fail when bfloat support is handled correctly
-; xUN: not --crash llc -global-isel=1 -mtriple=amdgcn-amd-amdhsa -mcpu=gfx90a -filetype=null %t/raw-ret-v2bf16-error.ll 2>&1 | FileCheck -check-prefix=ERR-RAW-V2BF16-GISEL %s
-; xUN: not --crash llc -global-isel=1 -mtriple=amdgcn-amd-amdhsa -mcpu=gfx90a -filetype=null %t/struct-ret-v2bf16-error.ll 2>&1 | FileCheck -check-prefix=ERR-STRUCT-V2BF16-GISEL %s
-; xUN: not --crash llc -global-isel=1 -mtriple=amdgcn-amd-amdhsa -mcpu=gfx940 -filetype=null %t/raw-ret-v2bf16-error.ll 2>&1 | FileCheck -check-prefix=ERR-RAW-V2BF16-GISEL %s
-; xUN: not --crash llc -global-isel=1 -mtriple=amdgcn-amd-amdhsa -mcpu=gfx940 -filetype=null %t/struct-ret-v2bf16-error.ll 2>&1 | FileCheck -check-prefix=ERR-STRUCT-V2BF16-GISEL %s
+; RUN: not --crash llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgcn-amd-amdhsa -mcpu=gfx908 -filetype=null %t/raw-ret-v2bf16-error.ll 2>&1 | FileCheck -check-prefix=ERR-RAW-V2BF16-GISEL %s
+; RUN: not --crash llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgcn-amd-amdhsa -mcpu=gfx908 -filetype=null %t/struct-ret-v2bf16-error.ll 2>&1 | FileCheck -check-prefix=ERR-STRUCT-V2BF16-GISEL %s
+; RUN: not --crash llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgcn-amd-amdhsa -mcpu=gfx90a -filetype=null %t/raw-ret-v2bf16-error.ll 2>&1 | FileCheck -check-prefix=ERR-RAW-V2BF16-GISEL %s
+; RUN: not --crash llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgcn-amd-amdhsa -mcpu=gfx90a -filetype=null %t/struct-ret-v2bf16-error.ll 2>&1 | FileCheck -check-prefix=ERR-STRUCT-V2BF16-GISEL %s
+; RUN: not --crash llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgcn-amd-amdhsa -mcpu=gfx940 -filetype=null %t/raw-ret-v2bf16-error.ll 2>&1 | FileCheck -check-prefix=ERR-RAW-V2BF16-GISEL %s
+; RUN: not --crash llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgcn-amd-amdhsa -mcpu=gfx940 -filetype=null %t/struct-ret-v2bf16-error.ll 2>&1 | FileCheck -check-prefix=ERR-STRUCT-V2BF16-GISEL %s
 
 ; Make sure buffer fadd atomics with return values are not selected
 ; for gfx908 where they do not work.
@@ -66,7 +64,7 @@ define <2 x half> @struct_ptr_buffer_atomic_fadd_v2f16_rtn(<2 x half> %val, ptr
 
 ;--- raw-ret-v2bf16-error.ll
 ; ERR-RAW-V2BF16-SDAG: LLVM ERROR: Cannot select: {{.+}}: v2bf16,ch = BUFFER_ATOMIC_FADD
-; ERR-RAW-V2BF16-GISEL: LLVM ERROR: cannot select: %{{[0-9]+}}:vgpr_32(<2 x s16>) = G_AMDGPU_BUFFER_ATOMIC_FADD
+; ERR-RAW-V2BF16-GISEL: LLVM ERROR: Cannot select: {{.+}}: v2bf16,ch = BUFFER_ATOMIC_FADD
 
 define <2 x bfloat> @raw_ptr_buffer_atomic_fadd_v2bf16_rtn(<2 x bfloat> %val, <4 x i32> inreg %rsrc, i32 inreg %soffset) {
   %ret = call <2 x bfloat> @llvm.amdgcn.raw.buffer.atomic.fadd.v2bf16(<2 x bfloat> %val, <4 x i32> %rsrc, i32 0, i32 %soffset, i32 0)
@@ -75,7 +73,7 @@ define <2 x bfloat> @raw_ptr_buffer_atomic_fadd_v2bf16_rtn(<2 x bfloat> %val, <4
 
 ;--- struct-ret-v2bf16-error.ll
 ; ERR-STRUCT-V2BF16-SDAG: LLVM ERROR: Cannot select: {{.+}}: v2bf16,ch = BUFFER_ATOMIC_FADD
-; ERR-STRUCT-V2BF16-GISEL: LLVM ERROR: cannot select: %{{[0-9]+}}:vgpr_32(<2 x s16>) = G_AMDGPU_BUFFER_ATOMIC_FADD
+; ERR-STRUCT-V2BF16-GISEL: LLVM ERROR: Cannot select: {{.+}}: v2bf16,ch = BUFFER_ATOMIC_FADD
 
 define <2 x bfloat> @struct_ptr_buffer_atomic_fadd_v2bf16_rtn(<2 x bfloat> %val, ptr addrspace(8) inreg %rsrc, i32 %vindex, i32 %voffset, i32 inreg %soffset) {
   %ret = call <2 x bfloat> @llvm.amdgcn.struct.ptr.buffer.atomic.fadd.v2bf16(<2 x bfloat> %val, ptr addrspace(8) %rsrc, i32 %vindex, i32 %voffset, i32 %soffset, i32 0)
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.ptr.buffer.atomic.fadd.v2bf16.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.ptr.buffer.atomic.fadd.v2bf16.ll
index 22a473e44b273..08d2201036c77 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.ptr.buffer.atomic.fadd.v2bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.ptr.buffer.atomic.fadd.v2bf16.ll
@@ -78,15 +78,14 @@ define <2 x bfloat> @raw_ptr_buffer_atomic_add_v2bf16_rtn__vgpr_val__vgpr_rsrc__
 ; GFX12-NEXT:    v_readfirstlane_b32 s6, v3
 ; GFX12-NEXT:    v_readfirstlane_b32 s7, v4
 ; GFX12-NEXT:    v_readfirstlane_b32 s3, v6
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-NEXT:    v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[1:2]
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
 ; GFX12-NEXT:    v_cmp_eq_u64_e64 s0, s[6:7], v[3:4]
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX12-NEXT:    v_cmp_eq_u32_e64 s1, s3, v6
-; GFX12-NEXT:    s_wait_alu 0xfffe
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    s_and_b32 s0, vcc_lo, s0
 ; GFX12-NEXT:    s_wait_alu 0xfffe
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-NEXT:    s_and_b32 s0, s0, s1
 ; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_and_saveexec_b32 s0, s0
@@ -95,13 +94,11 @@ define <2 x bfloat> @raw_ptr_buffer_atomic_add_v2bf16_rtn__vgpr_val__vgpr_rsrc__
 ; GFX12-NEXT:    ; implicit-def: $vgpr1_vgpr2_vgpr3_vgpr4
 ; GFX12-NEXT:    ; implicit-def: $vgpr6
 ; GFX12-NEXT:    ; implicit-def: $vgpr5
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_xor_b32 exec_lo, exec_lo, s0
 ; GFX12-NEXT:    s_cbranch_execnz .LBB4_1
 ; GFX12-NEXT:  ; %bb.2:
 ; GFX12-NEXT:    s_mov_b32 exec_lo, s2
 ; GFX12-NEXT:    s_wait_loadcnt 0x0
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
   %voffset.add = add i32 %voffset, 128
   %ret = call <2 x bfloat> @llvm.amdgcn.raw.ptr.buffer.atomic.fadd.v2bf16(<2 x bfloat> %val, ptr addrspace(8) %rsrc, i32 %voffset.add, i32 %soffset, i32 0)
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.readfirstlane.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.readfirstlane.ll
index 9a2f0aa5adb77..0605a158b974f 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.readfirstlane.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.readfirstlane.ll
@@ -1,6 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 4
 ; RUN: llc -mtriple=amdgcn--amdhsa -mcpu=fiji -verify-machineinstrs < %s | FileCheck -check-prefix=CHECK-SDAG -enable-var-scope %s
-; RUN: llc -mtriple=amdgcn--amdhsa -mcpu=fiji -verify-machineinstrs -global-isel < %s | FileCheck -check-prefix=CHECK-GISEL -enable-var-scope %s
+; RUN: llc -mtriple=amdgcn--amdhsa -mcpu=fiji -verify-machineinstrs -global-isel -global-isel-abort=2 < %s | FileCheck -check-prefix=CHECK-GISEL -enable-var-scope %s
 
 define void @test_readfirstlane_i1(ptr addrspace(1) %out, i1 %src) {
 ; CHECK-SDAG-LABEL: test_readfirstlane_i1:
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.readlane.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.readlane.ll
index a8560ff1aa2b0..edb6ebcee1325 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.readlane.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.readlane.ll
@@ -1,6 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 4
 ; RUN: llc -mtriple=amdgcn--amdhsa -mcpu=fiji -verify-machineinstrs < %s | FileCheck --check-prefix=CHECK-SDAG -enable-var-scope %s
-; RUN: llc -mtriple=amdgcn--amdhsa -mcpu=fiji -verify-machineinstrs -global-isel < %s | FileCheck --check-prefix=CHECK-GISEL -enable-var-scope %s
+; RUN: llc -mtriple=amdgcn--amdhsa -mcpu=fiji -verify-machineinstrs -global-isel -global-isel-abort=2 < %s | FileCheck --check-prefix=CHECK-GISEL -enable-var-scope %s
 
 declare i32 @llvm.amdgcn.readlane.i32(i32, i32) #0
 declare i64 @llvm.amdgcn.readlane.i64(i64, i32) #0
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.smfmac.gfx950.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.smfmac.gfx950.ll
index 0a330e91f8206..6b922fcd9b550 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.smfmac.gfx950.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.smfmac.gfx950.ll
@@ -1,6 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 4
 ; RUN: llc -mtriple=amdgcn -mcpu=gfx950 -global-isel=0 < %s | FileCheck -enable-var-scope --check-prefixes=GCN,SDAG %s
-; RUN: llc -mtriple=amdgcn -mcpu=gfx950 -global-isel=1 < %s | FileCheck -enable-var-scope --check-prefixes=GCN,GISEL %s
+; RUN: llc -mtriple=amdgcn -mcpu=gfx950 -global-isel=1 -global-isel-abort=2 < %s | FileCheck -enable-var-scope --check-prefixes=GCN,GISEL %s
 
 declare i32 @llvm.amdgcn.workitem.id.x()
 
@@ -33,7 +33,7 @@ define amdgpu_kernel void @test_smfmac_f32_16x16x64_f16__vgpr(ptr addrspace(1) %
 ; SDAG-NEXT:    s_waitcnt vmcnt(0)
 ; SDAG-NEXT:    s_nop 0
 ; SDAG-NEXT:    v_smfmac_f32_16x16x64_f16 v[8:11], v[12:15], v[0:7], v17 cbsz:1 abid:2
-; SDAG-NEXT:    s_nop 6
+; SDAG-NEXT:    s_nop 7
 ; SDAG-NEXT:    global_store_dwordx4 v16, v[8:11], s[6:7]
 ; SDAG-NEXT:    s_endpgm
 ;
@@ -59,7 +59,7 @@ define amdgpu_kernel void @test_smfmac_f32_16x16x64_f16__vgpr(ptr addrspace(1) %
 ; GISEL-NEXT:    s_nop 0
 ; GISEL-NEXT:    v_smfmac_f32_16x16x64_f16 v[8:11], v[12:15], v[0:7], v16 cbsz:1 abid:2
 ; GISEL-NEXT:    v_mov_b32_e32 v0, 0
-; GISEL-NEXT:    s_nop 5
+; GISEL-NEXT:    s_nop 6
 ; GISEL-NEXT:    global_store_dwordx4 v0, v[8:11], s[6:7]
 ; GISEL-NEXT:    s_endpgm
 bb:
@@ -81,7 +81,7 @@ define <4 x float> @test_smfmac_f32_16x16x64_f16(<8 x half> %arg0, <16 x half> %
 ; SDAG-NEXT:    v_accvgpr_write_b32 a3, v15
 ; SDAG-NEXT:    s_nop 1
 ; SDAG-NEXT:    v_smfmac_f32_16x16x64_f16 a[0:3], v[0:3], v[4:11], v16
-; SDAG-NEXT:    s_nop 6
+; SDAG-NEXT:    s_nop 7
 ; SDAG-NEXT:    v_accvgpr_read_b32 v0, a0
 ; SDAG-NEXT:    v_accvgpr_read_b32 v1, a1
 ; SDAG-NEXT:    v_accvgpr_read_b32 v2, a2
@@ -92,7 +92,7 @@ define <4 x float> @test_smfmac_f32_16x16x64_f16(<8 x half> %arg0, <16 x half> %
 ; GISEL:       ; %bb.0:
 ; GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GISEL-NEXT:    v_smfmac_f32_16x16x64_f16 v[12:15], v[0:3], v[4:11], v16
-; GISEL-NEXT:    s_nop 6
+; GISEL-NEXT:    s_nop 7
 ; GISEL-NEXT:    v_mov_b32_e32 v0, v12
 ; GISEL-NEXT:    v_mov_b32_e32 v1, v13
 ; GISEL-NEXT:    v_mov_b32_e32 v2, v14
@@ -111,697 +111,30 @@ define <4 x float> @test_smfmac_f32_16x16x64_f16__flags0(<8 x half> %arg0, <16 x
 ; SDAG-NEXT:    v_accvgpr_write_b32 a2, v14
 ; SDAG-NEXT:    v_accvgpr_write_b32 a3, v15
 ; SDAG-NEXT:    s_nop 1
-; SDAG-NEXT:    v_smfmac_f32_16x16x64_f16 a[0:3], v[0:3], v[4:11], v16 cbsz:1 abid:3
-; SDAG-NEXT:    s_nop 6
-; SDAG-NEXT:    v_accvgpr_read_b32 v0, a0
-; SDAG-NEXT:    v_accvgpr_read_b32 v1, a1
-; SDAG-NEXT:    v_accvgpr_read_b32 v2, a2
-; SDAG-NEXT:    v_accvgpr_read_b32 v3, a3
-; SDAG-NEXT:    s_setpc_b64 s[30:31]
-;
-; GISEL-LABEL: test_smfmac_f32_16x16x64_f16__flags0:
-; GISEL:       ; %bb.0:
-; GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GISEL-NEXT:    v_smfmac_f32_16x16x64_f16 v[12:15], v[0:3], v[4:11], v16 cbsz:1 abid:3
-; GISEL-NEXT:    s_nop 6
-; GISEL-NEXT:    v_mov_b32_e32 v0, v12
-; GISEL-NEXT:    v_mov_b32_e32 v1, v13
-; GISEL-NEXT:    v_mov_b32_e32 v2, v14
-; GISEL-NEXT:    v_mov_b32_e32 v3, v15
-; GISEL-NEXT:    s_setpc_b64 s[30:31]
-  %result = call <4 x float> @llvm.amdgcn.smfmac.f32.16x16x64.f16(<8 x half> %arg0, <16 x half> %arg1, <4 x float> %arg2, i32 %arg3, i32 immarg 1, i32 immarg 3)
-  ret <4 x float> %result
-}
-
-define <4 x float> @test_smfmac_f32_16x16x64_f16__flags1(<8 x half> %arg0, <16 x half> %arg1, <4 x float> %arg2, i32 %arg3) {
-; SDAG-LABEL: test_smfmac_f32_16x16x64_f16__flags1:
-; SDAG:       ; %bb.0:
-; SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SDAG-NEXT:    v_accvgpr_write_b32 a0, v12
-; SDAG-NEXT:    v_accvgpr_write_b32 a1, v13
-; SDAG-NEXT:    v_accvgpr_write_b32 a2, v14
-; SDAG-NEXT:    v_accvgpr_write_b32 a3, v15
-; SDAG-NEXT:    s_nop 1
-; SDAG-NEXT:    v_smfmac_f32_16x16x64_f16 a[0:3], v[0:3], v[4:11], v16 cbsz:3 abid:1
-; SDAG-NEXT:    s_nop 6
-; SDAG-NEXT:    v_accvgpr_read_b32 v0, a0
-; SDAG-NEXT:    v_accvgpr_read_b32 v1, a1
-; SDAG-NEXT:    v_accvgpr_read_b32 v2, a2
-; SDAG-NEXT:    v_accvgpr_read_b32 v3, a3
-; SDAG-NEXT:    s_setpc_b64 s[30:31]
-;
-; GISEL-LABEL: test_smfmac_f32_16x16x64_f16__flags1:
-; GISEL:       ; %bb.0:
-; GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GISEL-NEXT:    v_smfmac_f32_16x16x64_f16 v[12:15], v[0:3], v[4:11], v16 cbsz:3 abid:1
-; GISEL-NEXT:    s_nop 6
-; GISEL-NEXT:    v_mov_b32_e32 v0, v12
-; GISEL-NEXT:    v_mov_b32_e32 v1, v13
-; GISEL-NEXT:    v_mov_b32_e32 v2, v14
-; GISEL-NEXT:    v_mov_b32_e32 v3, v15
-; GISEL-NEXT:    s_setpc_b64 s[30:31]
-  %result = call <4 x float> @llvm.amdgcn.smfmac.f32.16x16x64.f16(<8 x half> %arg0, <16 x half> %arg1, <4 x float> %arg2, i32 %arg3, i32 immarg 3, i32 immarg 1)
-  ret <4 x float> %result
-}
-
-define <4 x float> @test_smfmac_f32_16x16x64_f16__sgpr(<8 x half> inreg %arg0, <16 x half> inreg %arg1, <4 x float> inreg %arg2, i32 inreg %arg3) {
-; SDAG-LABEL: test_smfmac_f32_16x16x64_f16__sgpr:
-; SDAG:       ; %bb.0:
-; SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SDAG-NEXT:    v_mov_b32_e32 v8, s0
-; SDAG-NEXT:    v_mov_b32_e32 v9, s1
-; SDAG-NEXT:    v_mov_b32_e32 v10, s2
-; SDAG-NEXT:    v_mov_b32_e32 v11, s3
-; SDAG-NEXT:    v_mov_b32_e32 v0, s16
-; SDAG-NEXT:    v_mov_b32_e32 v1, s17
-; SDAG-NEXT:    v_mov_b32_e32 v2, s18
-; SDAG-NEXT:    v_mov_b32_e32 v3, s19
-; SDAG-NEXT:    v_mov_b32_e32 v4, s20
-; SDAG-NEXT:    v_mov_b32_e32 v5, s21
-; SDAG-NEXT:    v_mov_b32_e32 v6, s22
-; SDAG-NEXT:    v_mov_b32_e32 v7, s23
-; SDAG-NEXT:    v_accvgpr_write_b32 a0, s24
-; SDAG-NEXT:    v_accvgpr_write_b32 a1, s25
-; SDAG-NEXT:    v_accvgpr_write_b32 a2, s26
-; SDAG-NEXT:    v_accvgpr_write_b32 a3, s27
-; SDAG-NEXT:    v_mov_b32_e32 v12, s28
-; SDAG-NEXT:    s_nop 1
-; SDAG-NEXT:    v_smfmac_f32_16x16x64_f16 a[0:3], v[8:11], v[0:7], v12
-; SDAG-NEXT:    s_nop 6
-; SDAG-NEXT:    v_accvgpr_read_b32 v0, a0
-; SDAG-NEXT:    v_accvgpr_read_b32 v1, a1
-; SDAG-NEXT:    v_accvgpr_read_b32 v2, a2
-; SDAG-NEXT:    v_accvgpr_read_b32 v3, a3
-; SDAG-NEXT:    s_setpc_b64 s[30:31]
-;
-; GISEL-LABEL: test_smfmac_f32_16x16x64_f16__sgpr:
-; GISEL:       ; %bb.0:
-; GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GISEL-NEXT:    v_mov_b64_e32 v[14:15], s[2:3]
-; GISEL-NEXT:    v_mov_b64_e32 v[12:13], s[0:1]
-; GISEL-NEXT:    v_mov_b64_e32 v[4:5], s[16:17]
-; GISEL-NEXT:    v_mov_b64_e32 v[0:1], s[24:25]
-; GISEL-NEXT:    v_mov_b64_e32 v[6:7], s[18:19]
-; GISEL-NEXT:    v_mov_b64_e32 v[8:9], s[20:21]
-; GISEL-NEXT:    v_mov_b64_e32 v[10:11], s[22:23]
-; GISEL-NEXT:    v_mov_b64_e32 v[2:3], s[26:27]
-; GISEL-NEXT:    v_mov_b32_e32 v16, s28
-; GISEL-NEXT:    s_nop 1
-; GISEL-NEXT:    v_smfmac_f32_16x16x64_f16 v[0:3], v[12:15], v[4:11], v16
-; GISEL-NEXT:    s_setpc_b64 s[30:31]
-  %result = call <4 x float> @llvm.amdgcn.smfmac.f32.16x16x64.f16(<8 x half> %arg0, <16 x half> %arg1, <4 x float> %arg2, i32 %arg3, i32 immarg 0, i32 immarg 0)
-  ret <4 x float> %result
-}
-
-; --------------------------------------------------------------------
-; llvm.amdgcn.smfmac.f32.32x32x32.f16
-; --------------------------------------------------------------------
-
-declare <16 x float> @llvm.amdgcn.smfmac.f32.32x32x32.f16(<8 x half>, <16 x half>, <16 x float>, i32, i32 immarg, i32 immarg)
-
-define amdgpu_kernel void @test_smfmac_f32_32x32x32_f16__vgpr(ptr addrspace(1) %arg, <8 x half> %a, <16 x half> %b, i32 %idx) #0 {
-; SDAG-LABEL: test_smfmac_f32_32x32x32_f16__vgpr:
-; SDAG:       ; %bb.0: ; %bb
-; SDAG-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x24
-; SDAG-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x34
-; SDAG-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
-; SDAG-NEXT:    v_lshlrev_b32_e32 v16, 6, v0
-; SDAG-NEXT:    s_waitcnt lgkmcnt(0)
-; SDAG-NEXT:    global_load_dwordx4 v[12:15], v16, s[6:7] offset:48
-; SDAG-NEXT:    global_load_dwordx4 v[8:11], v16, s[6:7] offset:32
-; SDAG-NEXT:    global_load_dwordx4 v[4:7], v16, s[6:7] offset:16
-; SDAG-NEXT:    global_load_dwordx4 v[0:3], v16, s[6:7]
-; SDAG-NEXT:    s_load_dwordx8 s[8:15], s[4:5], 0x44
-; SDAG-NEXT:    s_load_dword s16, s[4:5], 0x64
-; SDAG-NEXT:    v_mov_b64_e32 v[26:27], s[2:3]
-; SDAG-NEXT:    v_mov_b64_e32 v[24:25], s[0:1]
-; SDAG-NEXT:    s_waitcnt lgkmcnt(0)
-; SDAG-NEXT:    v_mov_b64_e32 v[22:23], s[14:15]
-; SDAG-NEXT:    v_mov_b64_e32 v[20:21], s[12:13]
-; SDAG-NEXT:    v_mov_b64_e32 v[18:19], s[10:11]
-; SDAG-NEXT:    v_mov_b64_e32 v[16:17], s[8:9]
-; SDAG-NEXT:    v_mov_b32_e32 v28, s16
-; SDAG-NEXT:    s_waitcnt vmcnt(0)
-; SDAG-NEXT:    s_nop 0
-; SDAG-NEXT:    v_smfmac_f32_32x32x32_f16 v[0:15], v[24:27], v[16:23], v28 cbsz:1 abid:2
-; SDAG-NEXT:    v_mov_b32_e32 v16, 0
-; SDAG-NEXT:    s_nop 7
-; SDAG-NEXT:    s_nop 1
-; SDAG-NEXT:    global_store_dwordx4 v16, v[8:11], s[6:7] offset:32
-; SDAG-NEXT:    global_store_dwordx4 v16, v[12:15], s[6:7] offset:48
-; SDAG-NEXT:    global_store_dwordx4 v16, v[0:3], s[6:7]
-; SDAG-NEXT:    global_store_dwordx4 v16, v[4:7], s[6:7] offset:16
-; SDAG-NEXT:    s_endpgm
-;
-; GISEL-LABEL: test_smfmac_f32_32x32x32_f16__vgpr:
-; GISEL:       ; %bb.0: ; %bb
-; GISEL-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x24
-; GISEL-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x34
-; GISEL-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
-; GISEL-NEXT:    v_lshlrev_b32_e32 v16, 6, v0
-; GISEL-NEXT:    s_waitcnt lgkmcnt(0)
-; GISEL-NEXT:    global_load_dwordx4 v[0:3], v16, s[6:7]
-; GISEL-NEXT:    global_load_dwordx4 v[4:7], v16, s[6:7] offset:16
-; GISEL-NEXT:    global_load_dwordx4 v[8:11], v16, s[6:7] offset:32
-; GISEL-NEXT:    global_load_dwordx4 v[12:15], v16, s[6:7] offset:48
-; GISEL-NEXT:    s_load_dwordx8 s[8:15], s[4:5], 0x44
-; GISEL-NEXT:    s_load_dword s16, s[4:5], 0x64
-; GISEL-NEXT:    v_mov_b64_e32 v[26:27], s[2:3]
-; GISEL-NEXT:    v_mov_b64_e32 v[24:25], s[0:1]
-; GISEL-NEXT:    s_waitcnt lgkmcnt(0)
-; GISEL-NEXT:    v_mov_b64_e32 v[22:23], s[14:15]
-; GISEL-NEXT:    v_mov_b64_e32 v[20:21], s[12:13]
-; GISEL-NEXT:    v_mov_b64_e32 v[18:19], s[10:11]
-; GISEL-NEXT:    v_mov_b64_e32 v[16:17], s[8:9]
-; GISEL-NEXT:    v_mov_b32_e32 v28, s16
-; GISEL-NEXT:    s_waitcnt vmcnt(0)
-; GISEL-NEXT:    s_nop 0
-; GISEL-NEXT:    v_smfmac_f32_32x32x32_f16 v[0:15], v[24:27], v[16:23], v28 cbsz:1 abid:2
-; GISEL-NEXT:    v_mov_b32_e32 v16, 0
-; GISEL-NEXT:    s_nop 7
-; GISEL-NEXT:    s_nop 1
-; GISEL-NEXT:    global_store_dwordx4 v16, v[0:3], s[6:7]
-; GISEL-NEXT:    global_store_dwordx4 v16, v[4:7], s[6:7] offset:16
-; GISEL-NEXT:    global_store_dwordx4 v16, v[8:11], s[6:7] offset:32
-; GISEL-NEXT:    global_store_dwordx4 v16, v[12:15], s[6:7] offset:48
-; GISEL-NEXT:    s_endpgm
-bb:
-  %id = call i32 @llvm.amdgcn.workitem.id.x()
-  %gep = getelementptr <16 x float>, ptr addrspace(1) %arg, i32 %id
-  %in.1 = load <16 x float>, ptr addrspace(1) %gep
-  %mai.1 = tail call <16 x float> @llvm.amdgcn.smfmac.f32.32x32x32.f16(<8 x half> %a, <16 x half> %b, <16 x float> %in.1, i32 %idx, i32 1, i32 2)
-  store <16 x float> %mai.1, ptr addrspace(1) %arg
-  ret void
-}
-
-define <16 x float> @test_smfmac_f32_32x32x32_f16(<8 x half> %arg0, <16 x half> %arg1, <16 x float> %arg2, i32 %arg3) {
-; SDAG-LABEL: test_smfmac_f32_32x32x32_f16:
-; SDAG:       ; %bb.0:
-; SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SDAG-NEXT:    v_accvgpr_write_b32 a0, v12
-; SDAG-NEXT:    v_accvgpr_write_b32 a1, v13
-; SDAG-NEXT:    v_accvgpr_write_b32 a2, v14
-; SDAG-NEXT:    v_accvgpr_write_b32 a3, v15
-; SDAG-NEXT:    v_accvgpr_write_b32 a4, v16
-; SDAG-NEXT:    v_accvgpr_write_b32 a5, v17
-; SDAG-NEXT:    v_accvgpr_write_b32 a6, v18
-; SDAG-NEXT:    v_accvgpr_write_b32 a7, v19
-; SDAG-NEXT:    v_accvgpr_write_b32 a8, v20
-; SDAG-NEXT:    v_accvgpr_write_b32 a9, v21
-; SDAG-NEXT:    v_accvgpr_write_b32 a10, v22
-; SDAG-NEXT:    v_accvgpr_write_b32 a11, v23
-; SDAG-NEXT:    v_accvgpr_write_b32 a12, v24
-; SDAG-NEXT:    v_accvgpr_write_b32 a13, v25
-; SDAG-NEXT:    v_accvgpr_write_b32 a14, v26
-; SDAG-NEXT:    v_accvgpr_write_b32 a15, v27
-; SDAG-NEXT:    s_nop 1
-; SDAG-NEXT:    v_smfmac_f32_32x32x32_f16 a[0:15], v[0:3], v[4:11], v28
-; SDAG-NEXT:    s_nop 7
-; SDAG-NEXT:    s_nop 2
-; SDAG-NEXT:    v_accvgpr_read_b32 v0, a0
-; SDAG-NEXT:    v_accvgpr_read_b32 v1, a1
-; SDAG-NEXT:    v_accvgpr_read_b32 v2, a2
-; SDAG-NEXT:    v_accvgpr_read_b32 v3, a3
-; SDAG-NEXT:    v_accvgpr_read_b32 v4, a4
-; SDAG-NEXT:    v_accvgpr_read_b32 v5, a5
-; SDAG-NEXT:    v_accvgpr_read_b32 v6, a6
-; SDAG-NEXT:    v_accvgpr_read_b32 v7, a7
-; SDAG-NEXT:    v_accvgpr_read_b32 v8, a8
-; SDAG-NEXT:    v_accvgpr_read_b32 v9, a9
-; SDAG-NEXT:    v_accvgpr_read_b32 v10, a10
-; SDAG-NEXT:    v_accvgpr_read_b32 v11, a11
-; SDAG-NEXT:    v_accvgpr_read_b32 v12, a12
-; SDAG-NEXT:    v_accvgpr_read_b32 v13, a13
-; SDAG-NEXT:    v_accvgpr_read_b32 v14, a14
-; SDAG-NEXT:    v_accvgpr_read_b32 v15, a15
-; SDAG-NEXT:    s_setpc_b64 s[30:31]
-;
-; GISEL-LABEL: test_smfmac_f32_32x32x32_f16:
-; GISEL:       ; %bb.0:
-; GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GISEL-NEXT:    v_mov_b32_e32 v48, v0
-; GISEL-NEXT:    v_mov_b32_e32 v49, v1
-; GISEL-NEXT:    v_mov_b32_e32 v50, v2
-; GISEL-NEXT:    v_mov_b32_e32 v51, v3
-; GISEL-NEXT:    v_mov_b32_e32 v30, v4
-; GISEL-NEXT:    v_mov_b32_e32 v31, v5
-; GISEL-NEXT:    v_mov_b32_e32 v32, v6
-; GISEL-NEXT:    v_mov_b32_e32 v33, v7
-; GISEL-NEXT:    v_mov_b32_e32 v34, v8
-; GISEL-NEXT:    v_mov_b32_e32 v35, v9
-; GISEL-NEXT:    v_mov_b32_e32 v36, v10
-; GISEL-NEXT:    v_mov_b32_e32 v37, v11
-; GISEL-NEXT:    v_mov_b64_e32 v[0:1], v[12:13]
-; GISEL-NEXT:    v_mov_b64_e32 v[2:3], v[14:15]
-; GISEL-NEXT:    v_mov_b64_e32 v[4:5], v[16:17]
-; GISEL-NEXT:    v_mov_b64_e32 v[6:7], v[18:19]
-; GISEL-NEXT:    v_mov_b64_e32 v[8:9], v[20:21]
-; GISEL-NEXT:    v_mov_b64_e32 v[10:11], v[22:23]
-; GISEL-NEXT:    v_mov_b64_e32 v[12:13], v[24:25]
-; GISEL-NEXT:    v_mov_b64_e32 v[14:15], v[26:27]
-; GISEL-NEXT:    s_nop 1
-; GISEL-NEXT:    v_smfmac_f32_32x32x32_f16 v[0:15], v[48:51], v[30:37], v28
-; GISEL-NEXT:    s_setpc_b64 s[30:31]
-  %result = call <16 x float> @llvm.amdgcn.smfmac.f32.32x32x32.f16(<8 x half> %arg0, <16 x half> %arg1, <16 x float> %arg2, i32 %arg3, i32 immarg 0, i32 immarg 0)
-  ret <16 x float> %result
-}
-
-define <16 x float> @test_smfmac_f32_32x32x32_f16__flags0(<8 x half> %arg0, <16 x half> %arg1, <16 x float> %arg2, i32 %arg3) {
-; SDAG-LABEL: test_smfmac_f32_32x32x32_f16__flags0:
-; SDAG:       ; %bb.0:
-; SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SDAG-NEXT:    v_accvgpr_write_b32 a0, v12
-; SDAG-NEXT:    v_accvgpr_write_b32 a1, v13
-; SDAG-NEXT:    v_accvgpr_write_b32 a2, v14
-; SDAG-NEXT:    v_accvgpr_write_b32 a3, v15
-; SDAG-NEXT:    v_accvgpr_write_b32 a4, v16
-; SDAG-NEXT:    v_accvgpr_write_b32 a5, v17
-; SDAG-NEXT:    v_accvgpr_write_b32 a6, v18
-; SDAG-NEXT:    v_accvgpr_write_b32 a7, v19
-; SDAG-NEXT:    v_accvgpr_write_b32 a8, v20
-; SDAG-NEXT:    v_accvgpr_write_b32 a9, v21
-; SDAG-NEXT:    v_accvgpr_write_b32 a10, v22
-; SDAG-NEXT:    v_accvgpr_write_b32 a11, v23
-; SDAG-NEXT:    v_accvgpr_write_b32 a12, v24
-; SDAG-NEXT:    v_accvgpr_write_b32 a13, v25
-; SDAG-NEXT:    v_accvgpr_write_b32 a14, v26
-; SDAG-NEXT:    v_accvgpr_write_b32 a15, v27
-; SDAG-NEXT:    s_nop 1
-; SDAG-NEXT:    v_smfmac_f32_32x32x32_f16 a[0:15], v[0:3], v[4:11], v28 cbsz:1 abid:3
-; SDAG-NEXT:    s_nop 7
-; SDAG-NEXT:    s_nop 2
-; SDAG-NEXT:    v_accvgpr_read_b32 v0, a0
-; SDAG-NEXT:    v_accvgpr_read_b32 v1, a1
-; SDAG-NEXT:    v_accvgpr_read_b32 v2, a2
-; SDAG-NEXT:    v_accvgpr_read_b32 v3, a3
-; SDAG-NEXT:    v_accvgpr_read_b32 v4, a4
-; SDAG-NEXT:    v_accvgpr_read_b32 v5, a5
-; SDAG-NEXT:    v_accvgpr_read_b32 v6, a6
-; SDAG-NEXT:    v_accvgpr_read_b32 v7, a7
-; SDAG-NEXT:    v_accvgpr_read_b32 v8, a8
-; SDAG-NEXT:    v_accvgpr_read_b32 v9, a9
-; SDAG-NEXT:    v_accvgpr_read_b32 v10, a10
-; SDAG-NEXT:    v_accvgpr_read_b32 v11, a11
-; SDAG-NEXT:    v_accvgpr_read_b32 v12, a12
-; SDAG-NEXT:    v_accvgpr_read_b32 v13, a13
-; SDAG-NEXT:    v_accvgpr_read_b32 v14, a14
-; SDAG-NEXT:    v_accvgpr_read_b32 v15, a15
-; SDAG-NEXT:    s_setpc_b64 s[30:31]
-;
-; GISEL-LABEL: test_smfmac_f32_32x32x32_f16__flags0:
-; GISEL:       ; %bb.0:
-; GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GISEL-NEXT:    v_mov_b32_e32 v48, v0
-; GISEL-NEXT:    v_mov_b32_e32 v49, v1
-; GISEL-NEXT:    v_mov_b32_e32 v50, v2
-; GISEL-NEXT:    v_mov_b32_e32 v51, v3
-; GISEL-NEXT:    v_mov_b32_e32 v30, v4
-; GISEL-NEXT:    v_mov_b32_e32 v31, v5
-; GISEL-NEXT:    v_mov_b32_e32 v32, v6
-; GISEL-NEXT:    v_mov_b32_e32 v33, v7
-; GISEL-NEXT:    v_mov_b32_e32 v34, v8
-; GISEL-NEXT:    v_mov_b32_e32 v35, v9
-; GISEL-NEXT:    v_mov_b32_e32 v36, v10
-; GISEL-NEXT:    v_mov_b32_e32 v37, v11
-; GISEL-NEXT:    v_mov_b64_e32 v[0:1], v[12:13]
-; GISEL-NEXT:    v_mov_b64_e32 v[2:3], v[14:15]
-; GISEL-NEXT:    v_mov_b64_e32 v[4:5], v[16:17]
-; GISEL-NEXT:    v_mov_b64_e32 v[6:7], v[18:19]
-; GISEL-NEXT:    v_mov_b64_e32 v[8:9], v[20:21]
-; GISEL-NEXT:    v_mov_b64_e32 v[10:11], v[22:23]
-; GISEL-NEXT:    v_mov_b64_e32 v[12:13], v[24:25]
-; GISEL-NEXT:    v_mov_b64_e32 v[14:15], v[26:27]
-; GISEL-NEXT:    s_nop 1
-; GISEL-NEXT:    v_smfmac_f32_32x32x32_f16 v[0:15], v[48:51], v[30:37], v28 cbsz:1 abid:3
-; GISEL-NEXT:    s_setpc_b64 s[30:31]
-  %result = call <16 x float> @llvm.amdgcn.smfmac.f32.32x32x32.f16(<8 x half> %arg0, <16 x half> %arg1, <16 x float> %arg2, i32 %arg3, i32 immarg 1, i32 immarg 3)
-  ret <16 x float> %result
-}
-
-define <16 x float> @test_smfmac_f32_32x32x32_f16__flags1(<8 x half> %arg0, <16 x half> %arg1, <16 x float> %arg2, i32 %arg3) {
-; SDAG-LABEL: test_smfmac_f32_32x32x32_f16__flags1:
-; SDAG:       ; %bb.0:
-; SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SDAG-NEXT:    v_accvgpr_write_b32 a0, v12
-; SDAG-NEXT:    v_accvgpr_write_b32 a1, v13
-; SDAG-NEXT:    v_accvgpr_write_b32 a2, v14
-; SDAG-NEXT:    v_accvgpr_write_b32 a3, v15
-; SDAG-NEXT:    v_accvgpr_write_b32 a4, v16
-; SDAG-NEXT:    v_accvgpr_write_b32 a5, v17
-; SDAG-NEXT:    v_accvgpr_write_b32 a6, v18
-; SDAG-NEXT:    v_accvgpr_write_b32 a7, v19
-; SDAG-NEXT:    v_accvgpr_write_b32 a8, v20
-; SDAG-NEXT:    v_accvgpr_write_b32 a9, v21
-; SDAG-NEXT:    v_accvgpr_write_b32 a10, v22
-; SDAG-NEXT:    v_accvgpr_write_b32 a11, v23
-; SDAG-NEXT:    v_accvgpr_write_b32 a12, v24
-; SDAG-NEXT:    v_accvgpr_write_b32 a13, v25
-; SDAG-NEXT:    v_accvgpr_write_b32 a14, v26
-; SDAG-NEXT:    v_accvgpr_write_b32 a15, v27
-; SDAG-NEXT:    s_nop 1
-; SDAG-NEXT:    v_smfmac_f32_32x32x32_f16 a[0:15], v[0:3], v[4:11], v28 cbsz:3 abid:1
-; SDAG-NEXT:    s_nop 7
-; SDAG-NEXT:    s_nop 2
-; SDAG-NEXT:    v_accvgpr_read_b32 v0, a0
-; SDAG-NEXT:    v_accvgpr_read_b32 v1, a1
-; SDAG-NEXT:    v_accvgpr_read_b32 v2, a2
-; SDAG-NEXT:    v_accvgpr_read_b32 v3, a3
-; SDAG-NEXT:    v_accvgpr_read_b32 v4, a4
-; SDAG-NEXT:    v_accvgpr_read_b32 v5, a5
-; SDAG-NEXT:    v_accvgpr_read_b32 v6, a6
-; SDAG-NEXT:    v_accvgpr_read_b32 v7, a7
-; SDAG-NEXT:    v_accvgpr_read_b32 v8, a8
-; SDAG-NEXT:    v_accvgpr_read_b32 v9, a9
-; SDAG-NEXT:    v_accvgpr_read_b32 v10, a10
-; SDAG-NEXT:    v_accvgpr_read_b32 v11, a11
-; SDAG-NEXT:    v_accvgpr_read_b32 v12, a12
-; SDAG-NEXT:    v_accvgpr_read_b32 v13, a13
-; SDAG-NEXT:    v_accvgpr_read_b32 v14, a14
-; SDAG-NEXT:    v_accvgpr_read_b32 v15, a15
-; SDAG-NEXT:    s_setpc_b64 s[30:31]
-;
-; GISEL-LABEL: test_smfmac_f32_32x32x32_f16__flags1:
-; GISEL:       ; %bb.0:
-; GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GISEL-NEXT:    v_mov_b32_e32 v48, v0
-; GISEL-NEXT:    v_mov_b32_e32 v49, v1
-; GISEL-NEXT:    v_mov_b32_e32 v50, v2
-; GISEL-NEXT:    v_mov_b32_e32 v51, v3
-; GISEL-NEXT:    v_mov_b32_e32 v30, v4
-; GISEL-NEXT:    v_mov_b32_e32 v31, v5
-; GISEL-NEXT:    v_mov_b32_e32 v32, v6
-; GISEL-NEXT:    v_mov_b32_e32 v33, v7
-; GISEL-NEXT:    v_mov_b32_e32 v34, v8
-; GISEL-NEXT:    v_mov_b32_e32 v35, v9
-; GISEL-NEXT:    v_mov_b32_e32 v36, v10
-; GISEL-NEXT:    v_mov_b32_e32 v37, v11
-; GISEL-NEXT:    v_mov_b64_e32 v[0:1], v[12:13]
-; GISEL-NEXT:    v_mov_b64_e32 v[2:3], v[14:15]
-; GISEL-NEXT:    v_mov_b64_e32 v[4:5], v[16:17]
-; GISEL-NEXT:    v_mov_b64_e32 v[6:7], v[18:19]
-; GISEL-NEXT:    v_mov_b64_e32 v[8:9], v[20:21]
-; GISEL-NEXT:    v_mov_b64_e32 v[10:11], v[22:23]
-; GISEL-NEXT:    v_mov_b64_e32 v[12:13], v[24:25]
-; GISEL-NEXT:    v_mov_b64_e32 v[14:15], v[26:27]
-; GISEL-NEXT:    s_nop 1
-; GISEL-NEXT:    v_smfmac_f32_32x32x32_f16 v[0:15], v[48:51], v[30:37], v28 cbsz:3 abid:1
-; GISEL-NEXT:    s_setpc_b64 s[30:31]
-  %result = call <16 x float> @llvm.amdgcn.smfmac.f32.32x32x32.f16(<8 x half> %arg0, <16 x half> %arg1, <16 x float> %arg2, i32 %arg3, i32 immarg 3, i32 immarg 1)
-  ret <16 x float> %result
-}
-
-define <16 x float> @test_smfmac_f32_32x32x32_f16__sgpr(<8 x half> inreg %arg0, <16 x half> inreg %arg1, <16 x float> inreg %arg2, i32 inreg %arg3) {
-; SDAG-LABEL: test_smfmac_f32_32x32x32_f16__sgpr:
-; SDAG:       ; %bb.0:
-; SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SDAG-NEXT:    v_mov_b32_e32 v28, s0
-; SDAG-NEXT:    v_mov_b32_e32 v29, s1
-; SDAG-NEXT:    v_mov_b32_e32 v30, s2
-; SDAG-NEXT:    v_mov_b32_e32 v31, s3
-; SDAG-NEXT:    v_mov_b32_e32 v12, s24
-; SDAG-NEXT:    v_mov_b32_e32 v27, v9
-; SDAG-NEXT:    v_mov_b32_e32 v26, v8
-; SDAG-NEXT:    v_mov_b32_e32 v25, v7
-; SDAG-NEXT:    v_mov_b32_e32 v24, v6
-; SDAG-NEXT:    v_mov_b32_e32 v23, v5
-; SDAG-NEXT:    v_mov_b32_e32 v22, v4
-; SDAG-NEXT:    v_mov_b32_e32 v21, v3
-; SDAG-NEXT:    v_mov_b32_e32 v20, v2
-; SDAG-NEXT:    v_mov_b32_e32 v19, v1
-; SDAG-NEXT:    v_mov_b32_e32 v18, v0
-; SDAG-NEXT:    v_mov_b32_e32 v13, s25
-; SDAG-NEXT:    v_mov_b32_e32 v14, s26
-; SDAG-NEXT:    v_mov_b32_e32 v15, s27
-; SDAG-NEXT:    v_mov_b32_e32 v16, s28
-; SDAG-NEXT:    v_mov_b32_e32 v17, s29
-; SDAG-NEXT:    v_accvgpr_write_b32 a0, v12
-; SDAG-NEXT:    v_mov_b32_e32 v0, s16
-; SDAG-NEXT:    v_mov_b32_e32 v1, s17
-; SDAG-NEXT:    v_mov_b32_e32 v2, s18
-; SDAG-NEXT:    v_mov_b32_e32 v3, s19
-; SDAG-NEXT:    v_mov_b32_e32 v4, s20
-; SDAG-NEXT:    v_mov_b32_e32 v5, s21
-; SDAG-NEXT:    v_mov_b32_e32 v6, s22
-; SDAG-NEXT:    v_mov_b32_e32 v7, s23
-; SDAG-NEXT:    v_accvgpr_write_b32 a1, v13
-; SDAG-NEXT:    v_accvgpr_write_b32 a2, v14
-; SDAG-NEXT:    v_accvgpr_write_b32 a3, v15
-; SDAG-NEXT:    v_accvgpr_write_b32 a4, v16
-; SDAG-NEXT:    v_accvgpr_write_b32 a5, v17
-; SDAG-NEXT:    v_accvgpr_write_b32 a6, v18
-; SDAG-NEXT:    v_accvgpr_write_b32 a7, v19
-; SDAG-NEXT:    v_accvgpr_write_b32 a8, v20
-; SDAG-NEXT:    v_accvgpr_write_b32 a9, v21
-; SDAG-NEXT:    v_accvgpr_write_b32 a10, v22
-; SDAG-NEXT:    v_accvgpr_write_b32 a11, v23
-; SDAG-NEXT:    v_accvgpr_write_b32 a12, v24
-; SDAG-NEXT:    v_accvgpr_write_b32 a13, v25
-; SDAG-NEXT:    v_accvgpr_write_b32 a14, v26
-; SDAG-NEXT:    v_accvgpr_write_b32 a15, v27
-; SDAG-NEXT:    s_nop 1
-; SDAG-NEXT:    v_smfmac_f32_32x32x32_f16 a[0:15], v[28:31], v[0:7], v10
-; SDAG-NEXT:    s_nop 7
-; SDAG-NEXT:    s_nop 2
-; SDAG-NEXT:    v_accvgpr_read_b32 v0, a0
-; SDAG-NEXT:    v_accvgpr_read_b32 v1, a1
-; SDAG-NEXT:    v_accvgpr_read_b32 v2, a2
-; SDAG-NEXT:    v_accvgpr_read_b32 v3, a3
-; SDAG-NEXT:    v_accvgpr_read_b32 v4, a4
-; SDAG-NEXT:    v_accvgpr_read_b32 v5, a5
-; SDAG-NEXT:    v_accvgpr_read_b32 v6, a6
-; SDAG-NEXT:    v_accvgpr_read_b32 v7, a7
-; SDAG-NEXT:    v_accvgpr_read_b32 v8, a8
-; SDAG-NEXT:    v_accvgpr_read_b32 v9, a9
-; SDAG-NEXT:    v_accvgpr_read_b32 v10, a10
-; SDAG-NEXT:    v_accvgpr_read_b32 v11, a11
-; SDAG-NEXT:    v_accvgpr_read_b32 v12, a12
-; SDAG-NEXT:    v_accvgpr_read_b32 v13, a13
-; SDAG-NEXT:    v_accvgpr_read_b32 v14, a14
-; SDAG-NEXT:    v_accvgpr_read_b32 v15, a15
-; SDAG-NEXT:    s_setpc_b64 s[30:31]
-;
-; GISEL-LABEL: test_smfmac_f32_32x32x32_f16__sgpr:
-; GISEL:       ; %bb.0:
-; GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GISEL-NEXT:    v_mov_b64_e32 v[36:37], s[2:3]
-; GISEL-NEXT:    v_mov_b64_e32 v[34:35], s[0:1]
-; GISEL-NEXT:    v_mov_b32_e32 v18, s24
-; GISEL-NEXT:    v_mov_b32_e32 v19, s25
-; GISEL-NEXT:    v_mov_b32_e32 v24, v0
-; GISEL-NEXT:    v_mov_b32_e32 v25, v1
-; GISEL-NEXT:    v_mov_b32_e32 v26, v2
-; GISEL-NEXT:    v_mov_b32_e32 v27, v3
-; GISEL-NEXT:    v_mov_b32_e32 v28, v4
-; GISEL-NEXT:    v_mov_b32_e32 v29, v5
-; GISEL-NEXT:    v_mov_b32_e32 v30, v6
-; GISEL-NEXT:    v_mov_b32_e32 v31, v7
-; GISEL-NEXT:    v_mov_b32_e32 v32, v8
-; GISEL-NEXT:    v_mov_b32_e32 v33, v9
-; GISEL-NEXT:    v_mov_b32_e32 v16, v10
-; GISEL-NEXT:    v_mov_b32_e32 v20, s26
-; GISEL-NEXT:    v_mov_b32_e32 v21, s27
-; GISEL-NEXT:    v_mov_b32_e32 v22, s28
-; GISEL-NEXT:    v_mov_b32_e32 v23, s29
-; GISEL-NEXT:    v_mov_b64_e32 v[54:55], s[22:23]
-; GISEL-NEXT:    v_mov_b64_e32 v[0:1], v[18:19]
-; GISEL-NEXT:    v_mov_b64_e32 v[52:53], s[20:21]
-; GISEL-NEXT:    v_mov_b64_e32 v[50:51], s[18:19]
-; GISEL-NEXT:    v_mov_b64_e32 v[48:49], s[16:17]
-; GISEL-NEXT:    v_mov_b64_e32 v[2:3], v[20:21]
-; GISEL-NEXT:    v_mov_b64_e32 v[4:5], v[22:23]
-; GISEL-NEXT:    v_mov_b64_e32 v[6:7], v[24:25]
-; GISEL-NEXT:    v_mov_b64_e32 v[8:9], v[26:27]
-; GISEL-NEXT:    v_mov_b64_e32 v[10:11], v[28:29]
-; GISEL-NEXT:    v_mov_b64_e32 v[12:13], v[30:31]
-; GISEL-NEXT:    v_mov_b64_e32 v[14:15], v[32:33]
-; GISEL-NEXT:    s_nop 1
-; GISEL-NEXT:    v_smfmac_f32_32x32x32_f16 v[0:15], v[34:37], v[48:55], v16
-; GISEL-NEXT:    s_setpc_b64 s[30:31]
-  %result = call <16 x float> @llvm.amdgcn.smfmac.f32.32x32x32.f16(<8 x half> %arg0, <16 x half> %arg1, <16 x float> %arg2, i32 %arg3, i32 immarg 0, i32 immarg 0)
-  ret <16 x float> %result
-}
-
-; --------------------------------------------------------------------
-; llvm.amdgcn.smfmac.f32.16x16x64.bf16
-; --------------------------------------------------------------------
-
-declare <4 x float> @llvm.amdgcn.smfmac.f32.16x16x64.bf16(<8 x bfloat>, <16 x bfloat>, <4 x float>, i32, i32 immarg, i32 immarg)
-
-define amdgpu_kernel void @test_smfmac_f32_16x16x64_bf16__vgpr(ptr addrspace(1) %arg, <8 x bfloat> %a, <16 x bfloat> %b, i32 %idx) #0 {
-; SDAG-LABEL: test_smfmac_f32_16x16x64_bf16__vgpr:
-; SDAG:       ; %bb.0: ; %bb
-; SDAG-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x24
-; SDAG-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x34
-; SDAG-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
-; SDAG-NEXT:    v_lshlrev_b32_e32 v0, 4, v0
-; SDAG-NEXT:    v_mov_b32_e32 v16, 0
-; SDAG-NEXT:    s_waitcnt lgkmcnt(0)
-; SDAG-NEXT:    global_load_dwordx4 v[8:11], v0, s[6:7]
-; SDAG-NEXT:    s_load_dwordx8 s[8:15], s[4:5], 0x44
-; SDAG-NEXT:    s_load_dword s16, s[4:5], 0x64
-; SDAG-NEXT:    v_mov_b64_e32 v[14:15], s[2:3]
-; SDAG-NEXT:    v_mov_b64_e32 v[12:13], s[0:1]
-; SDAG-NEXT:    s_waitcnt lgkmcnt(0)
-; SDAG-NEXT:    v_mov_b64_e32 v[0:1], s[8:9]
-; SDAG-NEXT:    v_mov_b64_e32 v[2:3], s[10:11]
-; SDAG-NEXT:    v_mov_b64_e32 v[4:5], s[12:13]
-; SDAG-NEXT:    v_mov_b64_e32 v[6:7], s[14:15]
-; SDAG-NEXT:    v_mov_b32_e32 v17, s16
-; SDAG-NEXT:    s_waitcnt vmcnt(0)
-; SDAG-NEXT:    s_nop 0
-; SDAG-NEXT:    v_smfmac_f32_16x16x64_bf16 v[8:11], v[12:15], v[0:7], v17 cbsz:1 abid:2
-; SDAG-NEXT:    s_nop 6
-; SDAG-NEXT:    global_store_dwordx4 v16, v[8:11], s[6:7]
-; SDAG-NEXT:    s_endpgm
-;
-; GISEL-LABEL: test_smfmac_f32_16x16x64_bf16__vgpr:
-; GISEL:       ; %bb.0: ; %bb
-; GISEL-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x24
-; GISEL-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x34
-; GISEL-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
-; GISEL-NEXT:    v_lshlrev_b32_e32 v0, 4, v0
-; GISEL-NEXT:    s_waitcnt lgkmcnt(0)
-; GISEL-NEXT:    global_load_dwordx4 v[8:11], v0, s[6:7]
-; GISEL-NEXT:    s_load_dwordx8 s[8:15], s[4:5], 0x44
-; GISEL-NEXT:    s_load_dword s16, s[4:5], 0x64
-; GISEL-NEXT:    v_mov_b64_e32 v[14:15], s[2:3]
-; GISEL-NEXT:    v_mov_b64_e32 v[12:13], s[0:1]
-; GISEL-NEXT:    s_waitcnt lgkmcnt(0)
-; GISEL-NEXT:    v_mov_b64_e32 v[0:1], s[8:9]
-; GISEL-NEXT:    v_mov_b64_e32 v[2:3], s[10:11]
-; GISEL-NEXT:    v_mov_b64_e32 v[4:5], s[12:13]
-; GISEL-NEXT:    v_mov_b64_e32 v[6:7], s[14:15]
-; GISEL-NEXT:    v_mov_b32_e32 v16, s16
-; GISEL-NEXT:    s_waitcnt vmcnt(0)
-; GISEL-NEXT:    s_nop 0
-; GISEL-NEXT:    v_smfmac_f32_16x16x64_bf16 v[8:11], v[12:15], v[0:7], v16 cbsz:1 abid:2
-; GISEL-NEXT:    v_mov_b32_e32 v0, 0
-; GISEL-NEXT:    s_nop 5
-; GISEL-NEXT:    global_store_dwordx4 v0, v[8:11], s[6:7]
-; GISEL-NEXT:    s_endpgm
-bb:
-  %id = call i32 @llvm.amdgcn.workitem.id.x()
-  %gep = getelementptr <4 x float>, ptr addrspace(1) %arg, i32 %id
-  %in.1 = load <4 x float>, ptr addrspace(1) %gep
-  %mai.1 = tail call <4 x float> @llvm.amdgcn.smfmac.f32.16x16x64.bf16(<8 x bfloat> %a, <16 x bfloat> %b, <4 x float> %in.1, i32 %idx, i32 1, i32 2)
-  store <4 x float> %mai.1, ptr addrspace(1) %arg
-  ret void
-}
-
-define <4 x float> @test_smfmac_f32_16x16x64_bf16(<8 x bfloat> %arg0, <16 x bfloat> %arg1, <4 x float> %arg2, i32 %arg3) {
-; SDAG-LABEL: test_smfmac_f32_16x16x64_bf16:
-; SDAG:       ; %bb.0:
-; SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SDAG-NEXT:    v_accvgpr_write_b32 a0, v12
-; SDAG-NEXT:    v_accvgpr_write_b32 a1, v13
-; SDAG-NEXT:    v_accvgpr_write_b32 a2, v14
-; SDAG-NEXT:    v_accvgpr_write_b32 a3, v15
-; SDAG-NEXT:    s_nop 1
-; SDAG-NEXT:    v_smfmac_f32_16x16x64_bf16 a[0:3], v[0:3], v[4:11], v16
-; SDAG-NEXT:    s_nop 6
-; SDAG-NEXT:    v_accvgpr_read_b32 v0, a0
-; SDAG-NEXT:    v_accvgpr_read_b32 v1, a1
-; SDAG-NEXT:    v_accvgpr_read_b32 v2, a2
-; SDAG-NEXT:    v_accvgpr_read_b32 v3, a3
-; SDAG-NEXT:    s_setpc_b64 s[30:31]
-;
-; GISEL-LABEL: test_smfmac_f32_16x16x64_bf16:
-; GISEL:       ; %bb.0:
-; GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GISEL-NEXT:    v_lshrrev_b32_e32 v17, 16, v0
-; GISEL-NEXT:    v_lshrrev_b32_e32 v18, 16, v1
-; GISEL-NEXT:    v_lshrrev_b32_e32 v19, 16, v2
-; GISEL-NEXT:    v_lshrrev_b32_e32 v20, 16, v3
-; GISEL-NEXT:    v_mov_b32_sdwa v0, v17 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0
-; GISEL-NEXT:    v_mov_b32_sdwa v1, v18 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0
-; GISEL-NEXT:    v_mov_b32_sdwa v2, v19 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0
-; GISEL-NEXT:    v_mov_b32_sdwa v3, v20 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0
-; GISEL-NEXT:    v_lshrrev_b32_e32 v17, 16, v4
-; GISEL-NEXT:    v_lshrrev_b32_e32 v18, 16, v5
-; GISEL-NEXT:    v_lshrrev_b32_e32 v19, 16, v6
-; GISEL-NEXT:    v_lshrrev_b32_e32 v20, 16, v7
-; GISEL-NEXT:    v_lshrrev_b32_e32 v21, 16, v8
-; GISEL-NEXT:    v_lshrrev_b32_e32 v22, 16, v9
-; GISEL-NEXT:    v_lshrrev_b32_e32 v23, 16, v10
-; GISEL-NEXT:    v_lshrrev_b32_e32 v24, 16, v11
-; GISEL-NEXT:    v_mov_b32_sdwa v4, v17 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0
-; GISEL-NEXT:    v_mov_b32_sdwa v5, v18 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0
-; GISEL-NEXT:    v_mov_b32_sdwa v6, v19 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0
-; GISEL-NEXT:    v_mov_b32_sdwa v7, v20 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0
-; GISEL-NEXT:    v_mov_b32_sdwa v8, v21 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0
-; GISEL-NEXT:    v_mov_b32_sdwa v9, v22 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0
-; GISEL-NEXT:    v_mov_b32_sdwa v10, v23 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0
-; GISEL-NEXT:    v_mov_b32_sdwa v11, v24 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0
-; GISEL-NEXT:    s_nop 1
-; GISEL-NEXT:    v_smfmac_f32_16x16x64_bf16 v[12:15], v[0:3], v[4:11], v16
-; GISEL-NEXT:    s_nop 6
-; GISEL-NEXT:    v_mov_b32_e32 v0, v12
-; GISEL-NEXT:    v_mov_b32_e32 v1, v13
-; GISEL-NEXT:    v_mov_b32_e32 v2, v14
-; GISEL-NEXT:    v_mov_b32_e32 v3, v15
-; GISEL-NEXT:    s_setpc_b64 s[30:31]
-  %result = call <4 x float> @llvm.amdgcn.smfmac.f32.16x16x64.bf16(<8 x bfloat> %arg0, <16 x bfloat> %arg1, <4 x float> %arg2, i32 %arg3, i32 immarg 0, i32 immarg 0)
-  ret <4 x float> %result
-}
-
-define <4 x float> @test_smfmac_f32_16x16x64_bf16__flags0(<8 x bfloat> %arg0, <16 x bfloat> %arg1, <4 x float> %arg2, i32 %arg3) {
-; SDAG-LABEL: test_smfmac_f32_16x16x64_bf16__flags0:
-; SDAG:       ; %bb.0:
-; SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SDAG-NEXT:    v_accvgpr_write_b32 a0, v12
-; SDAG-NEXT:    v_accvgpr_write_b32 a1, v13
-; SDAG-NEXT:    v_accvgpr_write_b32 a2, v14
-; SDAG-NEXT:    v_accvgpr_write_b32 a3, v15
-; SDAG-NEXT:    s_nop 1
-; SDAG-NEXT:    v_smfmac_f32_16x16x64_bf16 a[0:3], v[0:3], v[4:11], v16 cbsz:1 abid:3
-; SDAG-NEXT:    s_nop 6
+; SDAG-NEXT:    v_smfmac_f32_16x16x64_f16 a[0:3], v[0:3], v[4:11], v16 cbsz:1 abid:3
+; SDAG-NEXT:    s_nop 7
 ; SDAG-NEXT:    v_accvgpr_read_b32 v0, a0
 ; SDAG-NEXT:    v_accvgpr_read_b32 v1, a1
 ; SDAG-NEXT:    v_accvgpr_read_b32 v2, a2
 ; SDAG-NEXT:    v_accvgpr_read_b32 v3, a3
 ; SDAG-NEXT:    s_setpc_b64 s[30:31]
 ;
-; GISEL-LABEL: test_smfmac_f32_16x16x64_bf16__flags0:
+; GISEL-LABEL: test_smfmac_f32_16x16x64_f16__flags0:
 ; GISEL:       ; %bb.0:
 ; GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GISEL-NEXT:    v_lshrrev_b32_e32 v17, 16, v0
-; GISEL-NEXT:    v_lshrrev_b32_e32 v18, 16, v1
-; GISEL-NEXT:    v_lshrrev_b32_e32 v19, 16, v2
-; GISEL-NEXT:    v_lshrrev_b32_e32 v20, 16, v3
-; GISEL-NEXT:    v_mov_b32_sdwa v0, v17 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0
-; GISEL-NEXT:    v_mov_b32_sdwa v1, v18 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0
-; GISEL-NEXT:    v_mov_b32_sdwa v2, v19 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0
-; GISEL-NEXT:    v_mov_b32_sdwa v3, v20 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0
-; GISEL-NEXT:    v_lshrrev_b32_e32 v17, 16, v4
-; GISEL-NEXT:    v_lshrrev_b32_e32 v18, 16, v5
-; GISEL-NEXT:    v_lshrrev_b32_e32 v19, 16, v6
-; GISEL-NEXT:    v_lshrrev_b32_e32 v20, 16, v7
-; GISEL-NEXT:    v_lshrrev_b32_e32 v21, 16, v8
-; GISEL-NEXT:    v_lshrrev_b32_e32 v22, 16, v9
-; GISEL-NEXT:    v_lshrrev_b32_e32 v23, 16, v10
-; GISEL-NEXT:    v_lshrrev_b32_e32 v24, 16, v11
-; GISEL-NEXT:    v_mov_b32_sdwa v4, v17 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0
-; GISEL-NEXT:    v_mov_b32_sdwa v5, v18 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0
-; GISEL-NEXT:    v_mov_b32_sdwa v6, v19 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0
-; GISEL-NEXT:    v_mov_b32_sdwa v7, v20 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0
-; GISEL-NEXT:    v_mov_b32_sdwa v8, v21 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0
-; GISEL-NEXT:    v_mov_b32_sdwa v9, v22 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0
-; GISEL-NEXT:    v_mov_b32_sdwa v10, v23 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0
-; GISEL-NEXT:    v_mov_b32_sdwa v11, v24 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0
-; GISEL-NEXT:    s_nop 1
-; GISEL-NEXT:    v_smfmac_f32_16x16x64_bf16 v[12:15], v[0:3], v[4:11], v16 cbsz:1 abid:3
-; GISEL-NEXT:    s_nop 6
+; GISEL-NEXT:    v_smfmac_f32_16x16x64_f16 v[12:15], v[0:3], v[4:11], v16 cbsz:1 abid:3
+; GISEL-NEXT:    s_nop 7
 ; GISEL-NEXT:    v_mov_b32_e32 v0, v12
 ; GISEL-NEXT:    v_mov_b32_e32 v1, v13
 ; GISEL-NEXT:    v_mov_b32_e32 v2, v14
 ; GISEL-NEXT:    v_mov_b32_e32 v3, v15
 ; GISEL-NEXT:    s_setpc_b64 s[30:31]
-  %result = call <4 x float> @llvm.amdgcn.smfmac.f32.16x16x64.bf16(<8 x bfloat> %arg0, <16 x bfloat> %arg1, <4 x float> %arg2, i32 %arg3, i32 immarg 1, i32 immarg 3)
+  %result = call <4 x float> @llvm.amdgcn.smfmac.f32.16x16x64.f16(<8 x half> %arg0, <16 x half> %arg1, <4 x float> %arg2, i32 %arg3, i32 immarg 1, i32 immarg 3)
   ret <4 x float> %result
 }
 
-define <4 x float> @test_smfmac_f32_16x16x64_bf16__flags1(<8 x bfloat> %arg0, <16 x bfloat> %arg1, <4 x float> %arg2, i32 %arg3) {
-; SDAG-LABEL: test_smfmac_f32_16x16x64_bf16__flags1:
+define <4 x float> @test_smfmac_f32_16x16x64_f16__flags1(<8 x half> %arg0, <16 x half> %arg1, <4 x float> %arg2, i32 %arg3) {
+; SDAG-LABEL: test_smfmac_f32_16x16x64_f16__flags1:
 ; SDAG:       ; %bb.0:
 ; SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; SDAG-NEXT:    v_accvgpr_write_b32 a0, v12
@@ -809,55 +142,30 @@ define <4 x float> @test_smfmac_f32_16x16x64_bf16__flags1(<8 x bfloat> %arg0, <1
 ; SDAG-NEXT:    v_accvgpr_write_b32 a2, v14
 ; SDAG-NEXT:    v_accvgpr_write_b32 a3, v15
 ; SDAG-NEXT:    s_nop 1
-; SDAG-NEXT:    v_smfmac_f32_16x16x64_bf16 a[0:3], v[0:3], v[4:11], v16 cbsz:3 abid:1
-; SDAG-NEXT:    s_nop 6
+; SDAG-NEXT:    v_smfmac_f32_16x16x64_f16 a[0:3], v[0:3], v[4:11], v16 cbsz:3 abid:1
+; SDAG-NEXT:    s_nop 7
 ; SDAG-NEXT:    v_accvgpr_read_b32 v0, a0
 ; SDAG-NEXT:    v_accvgpr_read_b32 v1, a1
 ; SDAG-NEXT:    v_accvgpr_read_b32 v2, a2
 ; SDAG-NEXT:    v_accvgpr_read_b32 v3, a3
 ; SDAG-NEXT:    s_setpc_b64 s[30:31]
 ;
-; GISEL-LABEL: test_smfmac_f32_16x16x64_bf16__flags1:
+; GISEL-LABEL: test_smfmac_f32_16x16x64_f16__flags1:
 ; GISEL:       ; %bb.0:
 ; GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GISEL-NEXT:    v_lshrrev_b32_e32 v17, 16, v0
-; GISEL-NEXT:    v_lshrrev_b32_e32 v18, 16, v1
-; GISEL-NEXT:    v_lshrrev_b32_e32 v19, 16, v2
-; GISEL-NEXT:    v_lshrrev_b32_e32 v20, 16, v3
-; GISEL-NEXT:    v_mov_b32_sdwa v0, v17 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0
-; GISEL-NEXT:    v_mov_b32_sdwa v1, v18 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0
-; GISEL-NEXT:    v_mov_b32_sdwa v2, v19 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0
-; GISEL-NEXT:    v_mov_b32_sdwa v3, v20 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0
-; GISEL-NEXT:    v_lshrrev_b32_e32 v17, 16, v4
-; GISEL-NEXT:    v_lshrrev_b32_e32 v18, 16, v5
-; GISEL-NEXT:    v_lshrrev_b32_e32 v19, 16, v6
-; GISEL-NEXT:    v_lshrrev_b32_e32 v20, 16, v7
-; GISEL-NEXT:    v_lshrrev_b32_e32 v21, 16, v8
-; GISEL-NEXT:    v_lshrrev_b32_e32 v22, 16, v9
-; GISEL-NEXT:    v_lshrrev_b32_e32 v23, 16, v10
-; GISEL-NEXT:    v_lshrrev_b32_e32 v24, 16, v11
-; GISEL-NEXT:    v_mov_b32_sdwa v4, v17 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0
-; GISEL-NEXT:    v_mov_b32_sdwa v5, v18 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0
-; GISEL-NEXT:    v_mov_b32_sdwa v6, v19 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0
-; GISEL-NEXT:    v_mov_b32_sdwa v7, v20 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0
-; GISEL-NEXT:    v_mov_b32_sdwa v8, v21 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0
-; GISEL-NEXT:    v_mov_b32_sdwa v9, v22 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0
-; GISEL-NEXT:    v_mov_b32_sdwa v10, v23 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0
-; GISEL-NEXT:    v_mov_b32_sdwa v11, v24 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0
-; GISEL-NEXT:    s_nop 1
-; GISEL-NEXT:    v_smfmac_f32_16x16x64_bf16 v[12:15], v[0:3], v[4:11], v16 cbsz:3 abid:1
-; GISEL-NEXT:    s_nop 6
+; GISEL-NEXT:    v_smfmac_f32_16x16x64_f16 v[12:15], v[0:3], v[4:11], v16 cbsz:3 abid:1
+; GISEL-NEXT:    s_nop 7
 ; GISEL-NEXT:    v_mov_b32_e32 v0, v12
 ; GISEL-NEXT:    v_mov_b32_e32 v1, v13
 ; GISEL-NEXT:    v_mov_b32_e32 v2, v14
 ; GISEL-NEXT:    v_mov_b32_e32 v3, v15
 ; GISEL-NEXT:    s_setpc_b64 s[30:31]
-  %result = call <4 x float> @llvm.amdgcn.smfmac.f32.16x16x64.bf16(<8 x bfloat> %arg0, <16 x bfloat> %arg1, <4 x float> %arg2, i32 %arg3, i32 immarg 3, i32 immarg 1)
+  %result = call <4 x float> @llvm.amdgcn.smfmac.f32.16x16x64.f16(<8 x half> %arg0, <16 x half> %arg1, <4 x float> %arg2, i32 %arg3, i32 immarg 3, i32 immarg 1)
   ret <4 x float> %result
 }
 
-define <4 x float> @test_smfmac_f32_16x16x64_bf16__sgpr(<8 x bfloat> inreg %arg0, <16 x bfloat> inreg %arg1, <4 x float> inreg %arg2, i32 inreg %arg3) {
-; SDAG-LABEL: test_smfmac_f32_16x16x64_bf16__sgpr:
+define <4 x float> @test_smfmac_f32_16x16x64_f16__sgpr(<8 x half> inreg %arg0, <16 x half> inreg %arg1, <4 x float> inreg %arg2, i32 inreg %arg3) {
+; SDAG-LABEL: test_smfmac_f32_16x16x64_f16__sgpr:
 ; SDAG:       ; %bb.0:
 ; SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; SDAG-NEXT:    v_mov_b32_e32 v8, s0
@@ -878,89 +186,41 @@ define <4 x float> @test_smfmac_f32_16x16x64_bf16__sgpr(<8 x bfloat> inreg %arg0
 ; SDAG-NEXT:    v_accvgpr_write_b32 a3, s27
 ; SDAG-NEXT:    v_mov_b32_e32 v12, s28
 ; SDAG-NEXT:    s_nop 1
-; SDAG-NEXT:    v_smfmac_f32_16x16x64_bf16 a[0:3], v[8:11], v[0:7], v12
-; SDAG-NEXT:    s_nop 6
+; SDAG-NEXT:    v_smfmac_f32_16x16x64_f16 a[0:3], v[8:11], v[0:7], v12
+; SDAG-NEXT:    s_nop 7
 ; SDAG-NEXT:    v_accvgpr_read_b32 v0, a0
 ; SDAG-NEXT:    v_accvgpr_read_b32 v1, a1
 ; SDAG-NEXT:    v_accvgpr_read_b32 v2, a2
 ; SDAG-NEXT:    v_accvgpr_read_b32 v3, a3
 ; SDAG-NEXT:    s_setpc_b64 s[30:31]
 ;
-; GISEL-LABEL: test_smfmac_f32_16x16x64_bf16__sgpr:
+; GISEL-LABEL: test_smfmac_f32_16x16x64_f16__sgpr:
 ; GISEL:       ; %bb.0:
 ; GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GISEL-NEXT:    s_lshr_b32 s4, s0, 16
-; GISEL-NEXT:    s_lshr_b32 s5, s1, 16
-; GISEL-NEXT:    s_lshl_b32 s4, s4, 16
-; GISEL-NEXT:    s_and_b32 s0, s0, 0xffff
-; GISEL-NEXT:    s_lshr_b32 s6, s2, 16
-; GISEL-NEXT:    s_or_b32 s0, s4, s0
-; GISEL-NEXT:    s_lshl_b32 s4, s5, 16
-; GISEL-NEXT:    s_and_b32 s1, s1, 0xffff
-; GISEL-NEXT:    s_lshr_b32 s7, s3, 16
-; GISEL-NEXT:    s_or_b32 s1, s4, s1
-; GISEL-NEXT:    s_lshl_b32 s4, s6, 16
-; GISEL-NEXT:    s_and_b32 s2, s2, 0xffff
-; GISEL-NEXT:    s_or_b32 s2, s4, s2
-; GISEL-NEXT:    s_lshl_b32 s4, s7, 16
-; GISEL-NEXT:    s_and_b32 s3, s3, 0xffff
-; GISEL-NEXT:    s_or_b32 s3, s4, s3
-; GISEL-NEXT:    s_lshr_b32 s4, s16, 16
-; GISEL-NEXT:    s_lshr_b32 s5, s17, 16
-; GISEL-NEXT:    s_lshl_b32 s4, s4, 16
-; GISEL-NEXT:    s_and_b32 s12, s16, 0xffff
-; GISEL-NEXT:    s_lshr_b32 s6, s18, 16
-; GISEL-NEXT:    s_or_b32 s4, s4, s12
-; GISEL-NEXT:    s_lshl_b32 s5, s5, 16
-; GISEL-NEXT:    s_and_b32 s12, s17, 0xffff
-; GISEL-NEXT:    s_lshr_b32 s7, s19, 16
-; GISEL-NEXT:    s_or_b32 s5, s5, s12
-; GISEL-NEXT:    s_lshl_b32 s6, s6, 16
-; GISEL-NEXT:    s_and_b32 s12, s18, 0xffff
-; GISEL-NEXT:    s_lshr_b32 s8, s20, 16
-; GISEL-NEXT:    s_or_b32 s6, s6, s12
-; GISEL-NEXT:    s_lshl_b32 s7, s7, 16
-; GISEL-NEXT:    s_and_b32 s12, s19, 0xffff
-; GISEL-NEXT:    s_lshr_b32 s9, s21, 16
-; GISEL-NEXT:    s_or_b32 s7, s7, s12
-; GISEL-NEXT:    s_lshl_b32 s8, s8, 16
-; GISEL-NEXT:    s_and_b32 s12, s20, 0xffff
 ; GISEL-NEXT:    v_mov_b64_e32 v[14:15], s[2:3]
-; GISEL-NEXT:    s_lshr_b32 s10, s22, 16
-; GISEL-NEXT:    s_or_b32 s8, s8, s12
-; GISEL-NEXT:    s_lshl_b32 s9, s9, 16
-; GISEL-NEXT:    s_and_b32 s12, s21, 0xffff
 ; GISEL-NEXT:    v_mov_b64_e32 v[12:13], s[0:1]
-; GISEL-NEXT:    s_lshr_b32 s11, s23, 16
-; GISEL-NEXT:    s_or_b32 s9, s9, s12
-; GISEL-NEXT:    s_lshl_b32 s10, s10, 16
-; GISEL-NEXT:    s_and_b32 s12, s22, 0xffff
-; GISEL-NEXT:    s_or_b32 s10, s10, s12
-; GISEL-NEXT:    s_lshl_b32 s11, s11, 16
-; GISEL-NEXT:    s_and_b32 s12, s23, 0xffff
-; GISEL-NEXT:    s_or_b32 s11, s11, s12
-; GISEL-NEXT:    v_mov_b64_e32 v[4:5], s[4:5]
+; GISEL-NEXT:    v_mov_b64_e32 v[4:5], s[16:17]
 ; GISEL-NEXT:    v_mov_b64_e32 v[0:1], s[24:25]
-; GISEL-NEXT:    v_mov_b64_e32 v[6:7], s[6:7]
-; GISEL-NEXT:    v_mov_b64_e32 v[8:9], s[8:9]
-; GISEL-NEXT:    v_mov_b64_e32 v[10:11], s[10:11]
+; GISEL-NEXT:    v_mov_b64_e32 v[6:7], s[18:19]
+; GISEL-NEXT:    v_mov_b64_e32 v[8:9], s[20:21]
+; GISEL-NEXT:    v_mov_b64_e32 v[10:11], s[22:23]
 ; GISEL-NEXT:    v_mov_b64_e32 v[2:3], s[26:27]
 ; GISEL-NEXT:    v_mov_b32_e32 v16, s28
 ; GISEL-NEXT:    s_nop 1
-; GISEL-NEXT:    v_smfmac_f32_16x16x64_bf16 v[0:3], v[12:15], v[4:11], v16
+; GISEL-NEXT:    v_smfmac_f32_16x16x64_f16 v[0:3], v[12:15], v[4:11], v16
 ; GISEL-NEXT:    s_setpc_b64 s[30:31]
-  %result = call <4 x float> @llvm.amdgcn.smfmac.f32.16x16x64.bf16(<8 x bfloat> %arg0, <16 x bfloat> %arg1, <4 x float> %arg2, i32 %arg3, i32 immarg 0, i32 immarg 0)
+  %result = call <4 x float> @llvm.amdgcn.smfmac.f32.16x16x64.f16(<8 x half> %arg0, <16 x half> %arg1, <4 x float> %arg2, i32 %arg3, i32 immarg 0, i32 immarg 0)
   ret <4 x float> %result
 }
 
 ; --------------------------------------------------------------------
-; llvm.amdgcn.smfmac.f32.32x32x32.bf16
+; llvm.amdgcn.smfmac.f32.32x32x32.f16
 ; --------------------------------------------------------------------
 
-declare <16 x float> @llvm.amdgcn.smfmac.f32.32x32x32.bf16(<8 x bfloat>, <16 x bfloat>, <16 x float>, i32, i32 immarg, i32 immarg)
+declare <16 x float> @llvm.amdgcn.smfmac.f32.32x32x32.f16(<8 x half>, <16 x half>, <16 x float>, i32, i32 immarg, i32 immarg)
 
-define amdgpu_kernel void @test_smfmac_f32_32x32x32_bf16__vgpr(ptr addrspace(1) %arg, <8 x bfloat> %a, <16 x bfloat> %b, i32 %idx) #0 {
-; SDAG-LABEL: test_smfmac_f32_32x32x32_bf16__vgpr:
+define amdgpu_kernel void @test_smfmac_f32_32x32x32_f16__vgpr(ptr addrspace(1) %arg, <8 x half> %a, <16 x half> %b, i32 %idx) #0 {
+; SDAG-LABEL: test_smfmac_f32_32x32x32_f16__vgpr:
 ; SDAG:       ; %bb.0: ; %bb
 ; SDAG-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x24
 ; SDAG-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x34
@@ -983,17 +243,17 @@ define amdgpu_kernel void @test_smfmac_f32_32x32x32_bf16__vgpr(ptr addrspace(1)
 ; SDAG-NEXT:    v_mov_b32_e32 v28, s16
 ; SDAG-NEXT:    s_waitcnt vmcnt(0)
 ; SDAG-NEXT:    s_nop 0
-; SDAG-NEXT:    v_smfmac_f32_32x32x32_bf16 v[0:15], v[24:27], v[16:23], v28 cbsz:1 abid:2
+; SDAG-NEXT:    v_smfmac_f32_32x32x32_f16 v[0:15], v[24:27], v[16:23], v28 cbsz:1 abid:2
 ; SDAG-NEXT:    v_mov_b32_e32 v16, 0
 ; SDAG-NEXT:    s_nop 7
-; SDAG-NEXT:    s_nop 1
+; SDAG-NEXT:    s_nop 2
 ; SDAG-NEXT:    global_store_dwordx4 v16, v[8:11], s[6:7] offset:32
 ; SDAG-NEXT:    global_store_dwordx4 v16, v[12:15], s[6:7] offset:48
 ; SDAG-NEXT:    global_store_dwordx4 v16, v[0:3], s[6:7]
 ; SDAG-NEXT:    global_store_dwordx4 v16, v[4:7], s[6:7] offset:16
 ; SDAG-NEXT:    s_endpgm
 ;
-; GISEL-LABEL: test_smfmac_f32_32x32x32_bf16__vgpr:
+; GISEL-LABEL: test_smfmac_f32_32x32x32_f16__vgpr:
 ; GISEL:       ; %bb.0: ; %bb
 ; GISEL-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x24
 ; GISEL-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x34
@@ -1016,10 +276,10 @@ define amdgpu_kernel void @test_smfmac_f32_32x32x32_bf16__vgpr(ptr addrspace(1)
 ; GISEL-NEXT:    v_mov_b32_e32 v28, s16
 ; GISEL-NEXT:    s_waitcnt vmcnt(0)
 ; GISEL-NEXT:    s_nop 0
-; GISEL-NEXT:    v_smfmac_f32_32x32x32_bf16 v[0:15], v[24:27], v[16:23], v28 cbsz:1 abid:2
+; GISEL-NEXT:    v_smfmac_f32_32x32x32_f16 v[0:15], v[24:27], v[16:23], v28 cbsz:1 abid:2
 ; GISEL-NEXT:    v_mov_b32_e32 v16, 0
 ; GISEL-NEXT:    s_nop 7
-; GISEL-NEXT:    s_nop 1
+; GISEL-NEXT:    s_nop 2
 ; GISEL-NEXT:    global_store_dwordx4 v16, v[0:3], s[6:7]
 ; GISEL-NEXT:    global_store_dwordx4 v16, v[4:7], s[6:7] offset:16
 ; GISEL-NEXT:    global_store_dwordx4 v16, v[8:11], s[6:7] offset:32
@@ -1029,13 +289,13 @@ bb:
   %id = call i32 @llvm.amdgcn.workitem.id.x()
   %gep = getelementptr <16 x float>, ptr addrspace(1) %arg, i32 %id
   %in.1 = load <16 x float>, ptr addrspace(1) %gep
-  %mai.1 = tail call <16 x float> @llvm.amdgcn.smfmac.f32.32x32x32.bf16(<8 x bfloat> %a, <16 x bfloat> %b, <16 x float> %in.1, i32 %idx, i32 1, i32 2)
+  %mai.1 = tail call <16 x float> @llvm.amdgcn.smfmac.f32.32x32x32.f16(<8 x half> %a, <16 x half> %b, <16 x float> %in.1, i32 %idx, i32 1, i32 2)
   store <16 x float> %mai.1, ptr addrspace(1) %arg
   ret void
 }
 
-define <16 x float> @test_smfmac_f32_32x32x32_bf16(<8 x bfloat> %arg0, <16 x bfloat> %arg1, <16 x float> %arg2, i32 %arg3) {
-; SDAG-LABEL: test_smfmac_f32_32x32x32_bf16:
+define <16 x float> @test_smfmac_f32_32x32x32_f16(<8 x half> %arg0, <16 x half> %arg1, <16 x float> %arg2, i32 %arg3) {
+; SDAG-LABEL: test_smfmac_f32_32x32x32_f16:
 ; SDAG:       ; %bb.0:
 ; SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; SDAG-NEXT:    v_accvgpr_write_b32 a0, v12
@@ -1055,9 +315,9 @@ define <16 x float> @test_smfmac_f32_32x32x32_bf16(<8 x bfloat> %arg0, <16 x bfl
 ; SDAG-NEXT:    v_accvgpr_write_b32 a14, v26
 ; SDAG-NEXT:    v_accvgpr_write_b32 a15, v27
 ; SDAG-NEXT:    s_nop 1
-; SDAG-NEXT:    v_smfmac_f32_32x32x32_bf16 a[0:15], v[0:3], v[4:11], v28
+; SDAG-NEXT:    v_smfmac_f32_32x32x32_f16 a[0:15], v[0:3], v[4:11], v28
 ; SDAG-NEXT:    s_nop 7
-; SDAG-NEXT:    s_nop 2
+; SDAG-NEXT:    s_nop 3
 ; SDAG-NEXT:    v_accvgpr_read_b32 v0, a0
 ; SDAG-NEXT:    v_accvgpr_read_b32 v1, a1
 ; SDAG-NEXT:    v_accvgpr_read_b32 v2, a2
@@ -1076,21 +336,13 @@ define <16 x float> @test_smfmac_f32_32x32x32_bf16(<8 x bfloat> %arg0, <16 x bfl
 ; SDAG-NEXT:    v_accvgpr_read_b32 v15, a15
 ; SDAG-NEXT:    s_setpc_b64 s[30:31]
 ;
-; GISEL-LABEL: test_smfmac_f32_32x32x32_bf16:
+; GISEL-LABEL: test_smfmac_f32_32x32x32_f16:
 ; GISEL:       ; %bb.0:
 ; GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GISEL-NEXT:    v_mov_b32_e32 v48, v0
 ; GISEL-NEXT:    v_mov_b32_e32 v49, v1
 ; GISEL-NEXT:    v_mov_b32_e32 v50, v2
 ; GISEL-NEXT:    v_mov_b32_e32 v51, v3
-; GISEL-NEXT:    v_lshrrev_b32_e32 v0, 16, v48
-; GISEL-NEXT:    v_lshrrev_b32_e32 v1, 16, v49
-; GISEL-NEXT:    v_lshrrev_b32_e32 v2, 16, v50
-; GISEL-NEXT:    v_lshrrev_b32_e32 v3, 16, v51
-; GISEL-NEXT:    v_mov_b32_sdwa v48, v0 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0
-; GISEL-NEXT:    v_mov_b32_sdwa v49, v1 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0
-; GISEL-NEXT:    v_mov_b32_sdwa v50, v2 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0
-; GISEL-NEXT:    v_mov_b32_sdwa v51, v3 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0
 ; GISEL-NEXT:    v_mov_b32_e32 v30, v4
 ; GISEL-NEXT:    v_mov_b32_e32 v31, v5
 ; GISEL-NEXT:    v_mov_b32_e32 v32, v6
@@ -1099,22 +351,6 @@ define <16 x float> @test_smfmac_f32_32x32x32_bf16(<8 x bfloat> %arg0, <16 x bfl
 ; GISEL-NEXT:    v_mov_b32_e32 v35, v9
 ; GISEL-NEXT:    v_mov_b32_e32 v36, v10
 ; GISEL-NEXT:    v_mov_b32_e32 v37, v11
-; GISEL-NEXT:    v_lshrrev_b32_e32 v0, 16, v30
-; GISEL-NEXT:    v_lshrrev_b32_e32 v1, 16, v31
-; GISEL-NEXT:    v_lshrrev_b32_e32 v2, 16, v32
-; GISEL-NEXT:    v_lshrrev_b32_e32 v3, 16, v33
-; GISEL-NEXT:    v_lshrrev_b32_e32 v4, 16, v34
-; GISEL-NEXT:    v_lshrrev_b32_e32 v5, 16, v35
-; GISEL-NEXT:    v_lshrrev_b32_e32 v6, 16, v36
-; GISEL-NEXT:    v_lshrrev_b32_e32 v7, 16, v37
-; GISEL-NEXT:    v_mov_b32_sdwa v30, v0 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0
-; GISEL-NEXT:    v_mov_b32_sdwa v31, v1 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0
-; GISEL-NEXT:    v_mov_b32_sdwa v32, v2 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0
-; GISEL-NEXT:    v_mov_b32_sdwa v33, v3 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0
-; GISEL-NEXT:    v_mov_b32_sdwa v34, v4 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0
-; GISEL-NEXT:    v_mov_b32_sdwa v35, v5 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0
-; GISEL-NEXT:    v_mov_b32_sdwa v36, v6 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0
-; GISEL-NEXT:    v_mov_b32_sdwa v37, v7 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0
 ; GISEL-NEXT:    v_mov_b64_e32 v[0:1], v[12:13]
 ; GISEL-NEXT:    v_mov_b64_e32 v[2:3], v[14:15]
 ; GISEL-NEXT:    v_mov_b64_e32 v[4:5], v[16:17]
@@ -1124,14 +360,14 @@ define <16 x float> @test_smfmac_f32_32x32x32_bf16(<8 x bfloat> %arg0, <16 x bfl
 ; GISEL-NEXT:    v_mov_b64_e32 v[12:13], v[24:25]
 ; GISEL-NEXT:    v_mov_b64_e32 v[14:15], v[26:27]
 ; GISEL-NEXT:    s_nop 1
-; GISEL-NEXT:    v_smfmac_f32_32x32x32_bf16 v[0:15], v[48:51], v[30:37], v28
+; GISEL-NEXT:    v_smfmac_f32_32x32x32_f16 v[0:15], v[48:51], v[30:37], v28
 ; GISEL-NEXT:    s_setpc_b64 s[30:31]
-  %result = call <16 x float> @llvm.amdgcn.smfmac.f32.32x32x32.bf16(<8 x bfloat> %arg0, <16 x bfloat> %arg1, <16 x float> %arg2, i32 %arg3, i32 immarg 0, i32 immarg 0)
+  %result = call <16 x float> @llvm.amdgcn.smfmac.f32.32x32x32.f16(<8 x half> %arg0, <16 x half> %arg1, <16 x float> %arg2, i32 %arg3, i32 immarg 0, i32 immarg 0)
   ret <16 x float> %result
 }
 
-define <16 x float> @test_smfmac_f32_32x32x32_bf16__flags0(<8 x bfloat> %arg0, <16 x bfloat> %arg1, <16 x float> %arg2, i32 %arg3) {
-; SDAG-LABEL: test_smfmac_f32_32x32x32_bf16__flags0:
+define <16 x float> @test_smfmac_f32_32x32x32_f16__flags0(<8 x half> %arg0, <16 x half> %arg1, <16 x float> %arg2, i32 %arg3) {
+; SDAG-LABEL: test_smfmac_f32_32x32x32_f16__flags0:
 ; SDAG:       ; %bb.0:
 ; SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; SDAG-NEXT:    v_accvgpr_write_b32 a0, v12
@@ -1151,9 +387,9 @@ define <16 x float> @test_smfmac_f32_32x32x32_bf16__flags0(<8 x bfloat> %arg0, <
 ; SDAG-NEXT:    v_accvgpr_write_b32 a14, v26
 ; SDAG-NEXT:    v_accvgpr_write_b32 a15, v27
 ; SDAG-NEXT:    s_nop 1
-; SDAG-NEXT:    v_smfmac_f32_32x32x32_bf16 a[0:15], v[0:3], v[4:11], v28 cbsz:1 abid:3
+; SDAG-NEXT:    v_smfmac_f32_32x32x32_f16 a[0:15], v[0:3], v[4:11], v28 cbsz:1 abid:3
 ; SDAG-NEXT:    s_nop 7
-; SDAG-NEXT:    s_nop 2
+; SDAG-NEXT:    s_nop 3
 ; SDAG-NEXT:    v_accvgpr_read_b32 v0, a0
 ; SDAG-NEXT:    v_accvgpr_read_b32 v1, a1
 ; SDAG-NEXT:    v_accvgpr_read_b32 v2, a2
@@ -1172,21 +408,13 @@ define <16 x float> @test_smfmac_f32_32x32x32_bf16__flags0(<8 x bfloat> %arg0, <
 ; SDAG-NEXT:    v_accvgpr_read_b32 v15, a15
 ; SDAG-NEXT:    s_setpc_b64 s[30:31]
 ;
-; GISEL-LABEL: test_smfmac_f32_32x32x32_bf16__flags0:
+; GISEL-LABEL: test_smfmac_f32_32x32x32_f16__flags0:
 ; GISEL:       ; %bb.0:
 ; GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GISEL-NEXT:    v_mov_b32_e32 v48, v0
 ; GISEL-NEXT:    v_mov_b32_e32 v49, v1
 ; GISEL-NEXT:    v_mov_b32_e32 v50, v2
 ; GISEL-NEXT:    v_mov_b32_e32 v51, v3
-; GISEL-NEXT:    v_lshrrev_b32_e32 v0, 16, v48
-; GISEL-NEXT:    v_lshrrev_b32_e32 v1, 16, v49
-; GISEL-NEXT:    v_lshrrev_b32_e32 v2, 16, v50
-; GISEL-NEXT:    v_lshrrev_b32_e32 v3, 16, v51
-; GISEL-NEXT:    v_mov_b32_sdwa v48, v0 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0
-; GISEL-NEXT:    v_mov_b32_sdwa v49, v1 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0
-; GISEL-NEXT:    v_mov_b32_sdwa v50, v2 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0
-; GISEL-NEXT:    v_mov_b32_sdwa v51, v3 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0
 ; GISEL-NEXT:    v_mov_b32_e32 v30, v4
 ; GISEL-NEXT:    v_mov_b32_e32 v31, v5
 ; GISEL-NEXT:    v_mov_b32_e32 v32, v6
@@ -1195,22 +423,6 @@ define <16 x float> @test_smfmac_f32_32x32x32_bf16__flags0(<8 x bfloat> %arg0, <
 ; GISEL-NEXT:    v_mov_b32_e32 v35, v9
 ; GISEL-NEXT:    v_mov_b32_e32 v36, v10
 ; GISEL-NEXT:    v_mov_b32_e32 v37, v11
-; GISEL-NEXT:    v_lshrrev_b32_e32 v0, 16, v30
-; GISEL-NEXT:    v_lshrrev_b32_e32 v1, 16, v31
-; GISEL-NEXT:    v_lshrrev_b32_e32 v2, 16, v32
-; GISEL-NEXT:    v_lshrrev_b32_e32 v3, 16, v33
-; GISEL-NEXT:    v_lshrrev_b32_e32 v4, 16, v34
-; GISEL-NEXT:    v_lshrrev_b32_e32 v5, 16, v35
-; GISEL-NEXT:    v_lshrrev_b32_e32 v6, 16, v36
-; GISEL-NEXT:    v_lshrrev_b32_e32 v7, 16, v37
-; GISEL-NEXT:    v_mov_b32_sdwa v30, v0 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0
-; GISEL-NEXT:    v_mov_b32_sdwa v31, v1 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0
-; GISEL-NEXT:    v_mov_b32_sdwa v32, v2 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0
-; GISEL-NEXT:    v_mov_b32_sdwa v33, v3 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0
-; GISEL-NEXT:    v_mov_b32_sdwa v34, v4 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0
-; GISEL-NEXT:    v_mov_b32_sdwa v35, v5 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0
-; GISEL-NEXT:    v_mov_b32_sdwa v36, v6 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0
-; GISEL-NEXT:    v_mov_b32_sdwa v37, v7 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0
 ; GISEL-NEXT:    v_mov_b64_e32 v[0:1], v[12:13]
 ; GISEL-NEXT:    v_mov_b64_e32 v[2:3], v[14:15]
 ; GISEL-NEXT:    v_mov_b64_e32 v[4:5], v[16:17]
@@ -1220,14 +432,14 @@ define <16 x float> @test_smfmac_f32_32x32x32_bf16__flags0(<8 x bfloat> %arg0, <
 ; GISEL-NEXT:    v_mov_b64_e32 v[12:13], v[24:25]
 ; GISEL-NEXT:    v_mov_b64_e32 v[14:15], v[26:27]
 ; GISEL-NEXT:    s_nop 1
-; GISEL-NEXT:    v_smfmac_f32_32x32x32_bf16 v[0:15], v[48:51], v[30:37], v28 cbsz:1 abid:3
+; GISEL-NEXT:    v_smfmac_f32_32x32x32_f16 v[0:15], v[48:51], v[30:37], v28 cbsz:1 abid:3
 ; GISEL-NEXT:    s_setpc_b64 s[30:31]
-  %result = call <16 x float> @llvm.amdgcn.smfmac.f32.32x32x32.bf16(<8 x bfloat> %arg0, <16 x bfloat> %arg1, <16 x float> %arg2, i32 %arg3, i32 immarg 1, i32 immarg 3)
+  %result = call <16 x float> @llvm.amdgcn.smfmac.f32.32x32x32.f16(<8 x half> %arg0, <16 x half> %arg1, <16 x float> %arg2, i32 %arg3, i32 immarg 1, i32 immarg 3)
   ret <16 x float> %result
 }
 
-define <16 x float> @test_smfmac_f32_32x32x32_bf16__flags1(<8 x bfloat> %arg0, <16 x bfloat> %arg1, <16 x float> %arg2, i32 %arg3) {
-; SDAG-LABEL: test_smfmac_f32_32x32x32_bf16__flags1:
+define <16 x float> @test_smfmac_f32_32x32x32_f16__flags1(<8 x half> %arg0, <16 x half> %arg1, <16 x float> %arg2, i32 %arg3) {
+; SDAG-LABEL: test_smfmac_f32_32x32x32_f16__flags1:
 ; SDAG:       ; %bb.0:
 ; SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; SDAG-NEXT:    v_accvgpr_write_b32 a0, v12
@@ -1247,9 +459,9 @@ define <16 x float> @test_smfmac_f32_32x32x32_bf16__flags1(<8 x bfloat> %arg0, <
 ; SDAG-NEXT:    v_accvgpr_write_b32 a14, v26
 ; SDAG-NEXT:    v_accvgpr_write_b32 a15, v27
 ; SDAG-NEXT:    s_nop 1
-; SDAG-NEXT:    v_smfmac_f32_32x32x32_bf16 a[0:15], v[0:3], v[4:11], v28 cbsz:3 abid:1
+; SDAG-NEXT:    v_smfmac_f32_32x32x32_f16 a[0:15], v[0:3], v[4:11], v28 cbsz:3 abid:1
 ; SDAG-NEXT:    s_nop 7
-; SDAG-NEXT:    s_nop 2
+; SDAG-NEXT:    s_nop 3
 ; SDAG-NEXT:    v_accvgpr_read_b32 v0, a0
 ; SDAG-NEXT:    v_accvgpr_read_b32 v1, a1
 ; SDAG-NEXT:    v_accvgpr_read_b32 v2, a2
@@ -1268,21 +480,13 @@ define <16 x float> @test_smfmac_f32_32x32x32_bf16__flags1(<8 x bfloat> %arg0, <
 ; SDAG-NEXT:    v_accvgpr_read_b32 v15, a15
 ; SDAG-NEXT:    s_setpc_b64 s[30:31]
 ;
-; GISEL-LABEL: test_smfmac_f32_32x32x32_bf16__flags1:
+; GISEL-LABEL: test_smfmac_f32_32x32x32_f16__flags1:
 ; GISEL:       ; %bb.0:
 ; GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GISEL-NEXT:    v_mov_b32_e32 v48, v0
 ; GISEL-NEXT:    v_mov_b32_e32 v49, v1
 ; GISEL-NEXT:    v_mov_b32_e32 v50, v2
 ; GISEL-NEXT:    v_mov_b32_e32 v51, v3
-; GISEL-NEXT:    v_lshrrev_b32_e32 v0, 16, v48
-; GISEL-NEXT:    v_lshrrev_b32_e32 v1, 16, v49
-; GISEL-NEXT:    v_lshrrev_b32_e32 v2, 16, v50
-; GISEL-NEXT:    v_lshrrev_b32_e32 v3, 16, v51
-; GISEL-NEXT:    v_mov_b32_sdwa v48, v0 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0
-; GISEL-NEXT:    v_mov_b32_sdwa v49, v1 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0
-; GISEL-NEXT:    v_mov_b32_sdwa v50, v2 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0
-; GISEL-NEXT:    v_mov_b32_sdwa v51, v3 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0
 ; GISEL-NEXT:    v_mov_b32_e32 v30, v4
 ; GISEL-NEXT:    v_mov_b32_e32 v31, v5
 ; GISEL-NEXT:    v_mov_b32_e32 v32, v6
@@ -1291,22 +495,6 @@ define <16 x float> @test_smfmac_f32_32x32x32_bf16__flags1(<8 x bfloat> %arg0, <
 ; GISEL-NEXT:    v_mov_b32_e32 v35, v9
 ; GISEL-NEXT:    v_mov_b32_e32 v36, v10
 ; GISEL-NEXT:    v_mov_b32_e32 v37, v11
-; GISEL-NEXT:    v_lshrrev_b32_e32 v0, 16, v30
-; GISEL-NEXT:    v_lshrrev_b32_e32 v1, 16, v31
-; GISEL-NEXT:    v_lshrrev_b32_e32 v2, 16, v32
-; GISEL-NEXT:    v_lshrrev_b32_e32 v3, 16, v33
-; GISEL-NEXT:    v_lshrrev_b32_e32 v4, 16, v34
-; GISEL-NEXT:    v_lshrrev_b32_e32 v5, 16, v35
-; GISEL-NEXT:    v_lshrrev_b32_e32 v6, 16, v36
-; GISEL-NEXT:    v_lshrrev_b32_e32 v7, 16, v37
-; GISEL-NEXT:    v_mov_b32_sdwa v30, v0 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0
-; GISEL-NEXT:    v_mov_b32_sdwa v31, v1 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0
-; GISEL-NEXT:    v_mov_b32_sdwa v32, v2 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0
-; GISEL-NEXT:    v_mov_b32_sdwa v33, v3 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0
-; GISEL-NEXT:    v_mov_b32_sdwa v34, v4 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0
-; GISEL-NEXT:    v_mov_b32_sdwa v35, v5 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0
-; GISEL-NEXT:    v_mov_b32_sdwa v36, v6 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0
-; GISEL-NEXT:    v_mov_b32_sdwa v37, v7 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0
 ; GISEL-NEXT:    v_mov_b64_e32 v[0:1], v[12:13]
 ; GISEL-NEXT:    v_mov_b64_e32 v[2:3], v[14:15]
 ; GISEL-NEXT:    v_mov_b64_e32 v[4:5], v[16:17]
@@ -1316,14 +504,14 @@ define <16 x float> @test_smfmac_f32_32x32x32_bf16__flags1(<8 x bfloat> %arg0, <
 ; GISEL-NEXT:    v_mov_b64_e32 v[12:13], v[24:25]
 ; GISEL-NEXT:    v_mov_b64_e32 v[14:15], v[26:27]
 ; GISEL-NEXT:    s_nop 1
-; GISEL-NEXT:    v_smfmac_f32_32x32x32_bf16 v[0:15], v[48:51], v[30:37], v28 cbsz:3 abid:1
+; GISEL-NEXT:    v_smfmac_f32_32x32x32_f16 v[0:15], v[48:51], v[30:37], v28 cbsz:3 abid:1
 ; GISEL-NEXT:    s_setpc_b64 s[30:31]
-  %result = call <16 x float> @llvm.amdgcn.smfmac.f32.32x32x32.bf16(<8 x bfloat> %arg0, <16 x bfloat> %arg1, <16 x float> %arg2, i32 %arg3, i32 immarg 3, i32 immarg 1)
+  %result = call <16 x float> @llvm.amdgcn.smfmac.f32.32x32x32.f16(<8 x half> %arg0, <16 x half> %arg1, <16 x float> %arg2, i32 %arg3, i32 immarg 3, i32 immarg 1)
   ret <16 x float> %result
 }
 
-define <16 x float> @test_smfmac_f32_32x32x32_bf16__sgpr(<8 x bfloat> inreg %arg0, <16 x bfloat> inreg %arg1, <16 x float> inreg %arg2, i32 inreg %arg3) {
-; SDAG-LABEL: test_smfmac_f32_32x32x32_bf16__sgpr:
+define <16 x float> @test_smfmac_f32_32x32x32_f16__sgpr(<8 x half> inreg %arg0, <16 x half> inreg %arg1, <16 x float> inreg %arg2, i32 inreg %arg3) {
+; SDAG-LABEL: test_smfmac_f32_32x32x32_f16__sgpr:
 ; SDAG:       ; %bb.0:
 ; SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; SDAG-NEXT:    v_mov_b32_e32 v28, s0
@@ -1371,9 +559,9 @@ define <16 x float> @test_smfmac_f32_32x32x32_bf16__sgpr(<8 x bfloat> inreg %arg
 ; SDAG-NEXT:    v_accvgpr_write_b32 a14, v26
 ; SDAG-NEXT:    v_accvgpr_write_b32 a15, v27
 ; SDAG-NEXT:    s_nop 1
-; SDAG-NEXT:    v_smfmac_f32_32x32x32_bf16 a[0:15], v[28:31], v[0:7], v10
+; SDAG-NEXT:    v_smfmac_f32_32x32x32_f16 a[0:15], v[28:31], v[0:7], v10
 ; SDAG-NEXT:    s_nop 7
-; SDAG-NEXT:    s_nop 2
+; SDAG-NEXT:    s_nop 3
 ; SDAG-NEXT:    v_accvgpr_read_b32 v0, a0
 ; SDAG-NEXT:    v_accvgpr_read_b32 v1, a1
 ; SDAG-NEXT:    v_accvgpr_read_b32 v2, a2
@@ -1392,59 +580,11 @@ define <16 x float> @test_smfmac_f32_32x32x32_bf16__sgpr(<8 x bfloat> inreg %arg
 ; SDAG-NEXT:    v_accvgpr_read_b32 v15, a15
 ; SDAG-NEXT:    s_setpc_b64 s[30:31]
 ;
-; GISEL-LABEL: test_smfmac_f32_32x32x32_bf16__sgpr:
+; GISEL-LABEL: test_smfmac_f32_32x32x32_f16__sgpr:
 ; GISEL:       ; %bb.0:
 ; GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GISEL-NEXT:    s_lshr_b32 s4, s0, 16
-; GISEL-NEXT:    s_lshr_b32 s5, s1, 16
-; GISEL-NEXT:    s_lshl_b32 s4, s4, 16
-; GISEL-NEXT:    s_and_b32 s0, s0, 0xffff
-; GISEL-NEXT:    s_lshr_b32 s6, s2, 16
-; GISEL-NEXT:    s_or_b32 s8, s4, s0
-; GISEL-NEXT:    s_lshl_b32 s0, s5, 16
-; GISEL-NEXT:    s_and_b32 s1, s1, 0xffff
-; GISEL-NEXT:    s_lshr_b32 s7, s3, 16
-; GISEL-NEXT:    s_or_b32 s9, s0, s1
-; GISEL-NEXT:    s_lshl_b32 s0, s6, 16
-; GISEL-NEXT:    s_and_b32 s1, s2, 0xffff
-; GISEL-NEXT:    s_or_b32 s10, s0, s1
-; GISEL-NEXT:    s_lshl_b32 s0, s7, 16
-; GISEL-NEXT:    s_and_b32 s1, s3, 0xffff
-; GISEL-NEXT:    s_or_b32 s11, s0, s1
-; GISEL-NEXT:    s_lshr_b32 s0, s16, 16
-; GISEL-NEXT:    s_lshr_b32 s1, s17, 16
-; GISEL-NEXT:    s_lshl_b32 s0, s0, 16
-; GISEL-NEXT:    s_and_b32 s12, s16, 0xffff
-; GISEL-NEXT:    s_lshr_b32 s2, s18, 16
-; GISEL-NEXT:    s_or_b32 s0, s0, s12
-; GISEL-NEXT:    s_lshl_b32 s1, s1, 16
-; GISEL-NEXT:    s_and_b32 s12, s17, 0xffff
-; GISEL-NEXT:    s_lshr_b32 s3, s19, 16
-; GISEL-NEXT:    s_or_b32 s1, s1, s12
-; GISEL-NEXT:    s_lshl_b32 s2, s2, 16
-; GISEL-NEXT:    s_and_b32 s12, s18, 0xffff
-; GISEL-NEXT:    s_lshr_b32 s4, s20, 16
-; GISEL-NEXT:    s_or_b32 s2, s2, s12
-; GISEL-NEXT:    s_lshl_b32 s3, s3, 16
-; GISEL-NEXT:    s_and_b32 s12, s19, 0xffff
-; GISEL-NEXT:    s_lshr_b32 s5, s21, 16
-; GISEL-NEXT:    s_or_b32 s3, s3, s12
-; GISEL-NEXT:    s_lshl_b32 s4, s4, 16
-; GISEL-NEXT:    s_and_b32 s12, s20, 0xffff
-; GISEL-NEXT:    s_lshr_b32 s6, s22, 16
-; GISEL-NEXT:    s_or_b32 s4, s4, s12
-; GISEL-NEXT:    s_lshl_b32 s5, s5, 16
-; GISEL-NEXT:    s_and_b32 s12, s21, 0xffff
-; GISEL-NEXT:    v_mov_b64_e32 v[36:37], s[10:11]
-; GISEL-NEXT:    s_lshr_b32 s7, s23, 16
-; GISEL-NEXT:    s_or_b32 s5, s5, s12
-; GISEL-NEXT:    s_lshl_b32 s6, s6, 16
-; GISEL-NEXT:    s_and_b32 s12, s22, 0xffff
-; GISEL-NEXT:    v_mov_b64_e32 v[34:35], s[8:9]
-; GISEL-NEXT:    s_or_b32 s6, s6, s12
-; GISEL-NEXT:    s_lshl_b32 s7, s7, 16
-; GISEL-NEXT:    s_and_b32 s12, s23, 0xffff
-; GISEL-NEXT:    s_or_b32 s7, s7, s12
+; GISEL-NEXT:    v_mov_b64_e32 v[36:37], s[2:3]
+; GISEL-NEXT:    v_mov_b64_e32 v[34:35], s[0:1]
 ; GISEL-NEXT:    v_mov_b32_e32 v18, s24
 ; GISEL-NEXT:    v_mov_b32_e32 v19, s25
 ; GISEL-NEXT:    v_mov_b32_e32 v24, v0
@@ -1462,11 +602,11 @@ define <16 x float> @test_smfmac_f32_32x32x32_bf16__sgpr(<8 x bfloat> inreg %arg
 ; GISEL-NEXT:    v_mov_b32_e32 v21, s27
 ; GISEL-NEXT:    v_mov_b32_e32 v22, s28
 ; GISEL-NEXT:    v_mov_b32_e32 v23, s29
-; GISEL-NEXT:    v_mov_b64_e32 v[54:55], s[6:7]
+; GISEL-NEXT:    v_mov_b64_e32 v[54:55], s[22:23]
 ; GISEL-NEXT:    v_mov_b64_e32 v[0:1], v[18:19]
-; GISEL-NEXT:    v_mov_b64_e32 v[52:53], s[4:5]
-; GISEL-NEXT:    v_mov_b64_e32 v[50:51], s[2:3]
-; GISEL-NEXT:    v_mov_b64_e32 v[48:49], s[0:1]
+; GISEL-NEXT:    v_mov_b64_e32 v[52:53], s[20:21]
+; GISEL-NEXT:    v_mov_b64_e32 v[50:51], s[18:19]
+; GISEL-NEXT:    v_mov_b64_e32 v[48:49], s[16:17]
 ; GISEL-NEXT:    v_mov_b64_e32 v[2:3], v[20:21]
 ; GISEL-NEXT:    v_mov_b64_e32 v[4:5], v[22:23]
 ; GISEL-NEXT:    v_mov_b64_e32 v[6:7], v[24:25]
@@ -1475,8 +615,398 @@ define <16 x float> @test_smfmac_f32_32x32x32_bf16__sgpr(<8 x bfloat> inreg %arg
 ; GISEL-NEXT:    v_mov_b64_e32 v[12:13], v[30:31]
 ; GISEL-NEXT:    v_mov_b64_e32 v[14:15], v[32:33]
 ; GISEL-NEXT:    s_nop 1
-; GISEL-NEXT:    v_smfmac_f32_32x32x32_bf16 v[0:15], v[34:37], v[48:55], v16
+; GISEL-NEXT:    v_smfmac_f32_32x32x32_f16 v[0:15], v[34:37], v[48:55], v16
 ; GISEL-NEXT:    s_setpc_b64 s[30:31]
+  %result = call <16 x float> @llvm.amdgcn.smfmac.f32.32x32x32.f16(<8 x half> %arg0, <16 x half> %arg1, <16 x float> %arg2, i32 %arg3, i32 immarg 0, i32 immarg 0)
+  ret <16 x float> %result
+}
+
+; --------------------------------------------------------------------
+; llvm.amdgcn.smfmac.f32.16x16x64.bf16
+; --------------------------------------------------------------------
+
+declare <4 x float> @llvm.amdgcn.smfmac.f32.16x16x64.bf16(<8 x bfloat>, <16 x bfloat>, <4 x float>, i32, i32 immarg, i32 immarg)
+
+define amdgpu_kernel void @test_smfmac_f32_16x16x64_bf16__vgpr(ptr addrspace(1) %arg, <8 x bfloat> %a, <16 x bfloat> %b, i32 %idx) #0 {
+; GCN-LABEL: test_smfmac_f32_16x16x64_bf16__vgpr:
+; GCN:       ; %bb.0: ; %bb
+; GCN-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x24
+; GCN-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x34
+; GCN-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
+; GCN-NEXT:    v_lshlrev_b32_e32 v0, 4, v0
+; GCN-NEXT:    v_mov_b32_e32 v16, 0
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    global_load_dwordx4 v[8:11], v0, s[6:7]
+; GCN-NEXT:    s_load_dwordx8 s[8:15], s[4:5], 0x44
+; GCN-NEXT:    s_load_dword s16, s[4:5], 0x64
+; GCN-NEXT:    v_mov_b64_e32 v[14:15], s[2:3]
+; GCN-NEXT:    v_mov_b64_e32 v[12:13], s[0:1]
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    v_mov_b64_e32 v[0:1], s[8:9]
+; GCN-NEXT:    v_mov_b64_e32 v[2:3], s[10:11]
+; GCN-NEXT:    v_mov_b64_e32 v[4:5], s[12:13]
+; GCN-NEXT:    v_mov_b64_e32 v[6:7], s[14:15]
+; GCN-NEXT:    v_mov_b32_e32 v17, s16
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    s_nop 0
+; GCN-NEXT:    v_smfmac_f32_16x16x64_bf16 v[8:11], v[12:15], v[0:7], v17 cbsz:1 abid:2
+; GCN-NEXT:    s_nop 7
+; GCN-NEXT:    global_store_dwordx4 v16, v[8:11], s[6:7]
+; GCN-NEXT:    s_endpgm
+bb:
+  %id = call i32 @llvm.amdgcn.workitem.id.x()
+  %gep = getelementptr <4 x float>, ptr addrspace(1) %arg, i32 %id
+  %in.1 = load <4 x float>, ptr addrspace(1) %gep
+  %mai.1 = tail call <4 x float> @llvm.amdgcn.smfmac.f32.16x16x64.bf16(<8 x bfloat> %a, <16 x bfloat> %b, <4 x float> %in.1, i32 %idx, i32 1, i32 2)
+  store <4 x float> %mai.1, ptr addrspace(1) %arg
+  ret void
+}
+
+define <4 x float> @test_smfmac_f32_16x16x64_bf16(<8 x bfloat> %arg0, <16 x bfloat> %arg1, <4 x float> %arg2, i32 %arg3) {
+; GCN-LABEL: test_smfmac_f32_16x16x64_bf16:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_accvgpr_write_b32 a0, v12
+; GCN-NEXT:    v_accvgpr_write_b32 a1, v13
+; GCN-NEXT:    v_accvgpr_write_b32 a2, v14
+; GCN-NEXT:    v_accvgpr_write_b32 a3, v15
+; GCN-NEXT:    s_nop 1
+; GCN-NEXT:    v_smfmac_f32_16x16x64_bf16 a[0:3], v[0:3], v[4:11], v16
+; GCN-NEXT:    s_nop 7
+; GCN-NEXT:    v_accvgpr_read_b32 v0, a0
+; GCN-NEXT:    v_accvgpr_read_b32 v1, a1
+; GCN-NEXT:    v_accvgpr_read_b32 v2, a2
+; GCN-NEXT:    v_accvgpr_read_b32 v3, a3
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+  %result = call <4 x float> @llvm.amdgcn.smfmac.f32.16x16x64.bf16(<8 x bfloat> %arg0, <16 x bfloat> %arg1, <4 x float> %arg2, i32 %arg3, i32 immarg 0, i32 immarg 0)
+  ret <4 x float> %result
+}
+
+define <4 x float> @test_smfmac_f32_16x16x64_bf16__flags0(<8 x bfloat> %arg0, <16 x bfloat> %arg1, <4 x float> %arg2, i32 %arg3) {
+; GCN-LABEL: test_smfmac_f32_16x16x64_bf16__flags0:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_accvgpr_write_b32 a0, v12
+; GCN-NEXT:    v_accvgpr_write_b32 a1, v13
+; GCN-NEXT:    v_accvgpr_write_b32 a2, v14
+; GCN-NEXT:    v_accvgpr_write_b32 a3, v15
+; GCN-NEXT:    s_nop 1
+; GCN-NEXT:    v_smfmac_f32_16x16x64_bf16 a[0:3], v[0:3], v[4:11], v16 cbsz:1 abid:3
+; GCN-NEXT:    s_nop 7
+; GCN-NEXT:    v_accvgpr_read_b32 v0, a0
+; GCN-NEXT:    v_accvgpr_read_b32 v1, a1
+; GCN-NEXT:    v_accvgpr_read_b32 v2, a2
+; GCN-NEXT:    v_accvgpr_read_b32 v3, a3
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+  %result = call <4 x float> @llvm.amdgcn.smfmac.f32.16x16x64.bf16(<8 x bfloat> %arg0, <16 x bfloat> %arg1, <4 x float> %arg2, i32 %arg3, i32 immarg 1, i32 immarg 3)
+  ret <4 x float> %result
+}
+
+define <4 x float> @test_smfmac_f32_16x16x64_bf16__flags1(<8 x bfloat> %arg0, <16 x bfloat> %arg1, <4 x float> %arg2, i32 %arg3) {
+; GCN-LABEL: test_smfmac_f32_16x16x64_bf16__flags1:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_accvgpr_write_b32 a0, v12
+; GCN-NEXT:    v_accvgpr_write_b32 a1, v13
+; GCN-NEXT:    v_accvgpr_write_b32 a2, v14
+; GCN-NEXT:    v_accvgpr_write_b32 a3, v15
+; GCN-NEXT:    s_nop 1
+; GCN-NEXT:    v_smfmac_f32_16x16x64_bf16 a[0:3], v[0:3], v[4:11], v16 cbsz:3 abid:1
+; GCN-NEXT:    s_nop 7
+; GCN-NEXT:    v_accvgpr_read_b32 v0, a0
+; GCN-NEXT:    v_accvgpr_read_b32 v1, a1
+; GCN-NEXT:    v_accvgpr_read_b32 v2, a2
+; GCN-NEXT:    v_accvgpr_read_b32 v3, a3
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+  %result = call <4 x float> @llvm.amdgcn.smfmac.f32.16x16x64.bf16(<8 x bfloat> %arg0, <16 x bfloat> %arg1, <4 x float> %arg2, i32 %arg3, i32 immarg 3, i32 immarg 1)
+  ret <4 x float> %result
+}
+
+define <4 x float> @test_smfmac_f32_16x16x64_bf16__sgpr(<8 x bfloat> inreg %arg0, <16 x bfloat> inreg %arg1, <4 x float> inreg %arg2, i32 inreg %arg3) {
+; GCN-LABEL: test_smfmac_f32_16x16x64_bf16__sgpr:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_mov_b32_e32 v8, s0
+; GCN-NEXT:    v_mov_b32_e32 v9, s1
+; GCN-NEXT:    v_mov_b32_e32 v10, s2
+; GCN-NEXT:    v_mov_b32_e32 v11, s3
+; GCN-NEXT:    v_mov_b32_e32 v0, s16
+; GCN-NEXT:    v_mov_b32_e32 v1, s17
+; GCN-NEXT:    v_mov_b32_e32 v2, s18
+; GCN-NEXT:    v_mov_b32_e32 v3, s19
+; GCN-NEXT:    v_mov_b32_e32 v4, s20
+; GCN-NEXT:    v_mov_b32_e32 v5, s21
+; GCN-NEXT:    v_mov_b32_e32 v6, s22
+; GCN-NEXT:    v_mov_b32_e32 v7, s23
+; GCN-NEXT:    v_accvgpr_write_b32 a0, s24
+; GCN-NEXT:    v_accvgpr_write_b32 a1, s25
+; GCN-NEXT:    v_accvgpr_write_b32 a2, s26
+; GCN-NEXT:    v_accvgpr_write_b32 a3, s27
+; GCN-NEXT:    v_mov_b32_e32 v12, s28
+; GCN-NEXT:    s_nop 1
+; GCN-NEXT:    v_smfmac_f32_16x16x64_bf16 a[0:3], v[8:11], v[0:7], v12
+; GCN-NEXT:    s_nop 7
+; GCN-NEXT:    v_accvgpr_read_b32 v0, a0
+; GCN-NEXT:    v_accvgpr_read_b32 v1, a1
+; GCN-NEXT:    v_accvgpr_read_b32 v2, a2
+; GCN-NEXT:    v_accvgpr_read_b32 v3, a3
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+  %result = call <4 x float> @llvm.amdgcn.smfmac.f32.16x16x64.bf16(<8 x bfloat> %arg0, <16 x bfloat> %arg1, <4 x float> %arg2, i32 %arg3, i32 immarg 0, i32 immarg 0)
+  ret <4 x float> %result
+}
+
+; --------------------------------------------------------------------
+; llvm.amdgcn.smfmac.f32.32x32x32.bf16
+; --------------------------------------------------------------------
+
+declare <16 x float> @llvm.amdgcn.smfmac.f32.32x32x32.bf16(<8 x bfloat>, <16 x bfloat>, <16 x float>, i32, i32 immarg, i32 immarg)
+
+define amdgpu_kernel void @test_smfmac_f32_32x32x32_bf16__vgpr(ptr addrspace(1) %arg, <8 x bfloat> %a, <16 x bfloat> %b, i32 %idx) #0 {
+; GCN-LABEL: test_smfmac_f32_32x32x32_bf16__vgpr:
+; GCN:       ; %bb.0: ; %bb
+; GCN-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x24
+; GCN-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x34
+; GCN-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
+; GCN-NEXT:    v_lshlrev_b32_e32 v16, 6, v0
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    global_load_dwordx4 v[12:15], v16, s[6:7] offset:48
+; GCN-NEXT:    global_load_dwordx4 v[8:11], v16, s[6:7] offset:32
+; GCN-NEXT:    global_load_dwordx4 v[4:7], v16, s[6:7] offset:16
+; GCN-NEXT:    global_load_dwordx4 v[0:3], v16, s[6:7]
+; GCN-NEXT:    s_load_dwordx8 s[8:15], s[4:5], 0x44
+; GCN-NEXT:    s_load_dword s16, s[4:5], 0x64
+; GCN-NEXT:    v_mov_b64_e32 v[26:27], s[2:3]
+; GCN-NEXT:    v_mov_b64_e32 v[24:25], s[0:1]
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    v_mov_b64_e32 v[22:23], s[14:15]
+; GCN-NEXT:    v_mov_b64_e32 v[20:21], s[12:13]
+; GCN-NEXT:    v_mov_b64_e32 v[18:19], s[10:11]
+; GCN-NEXT:    v_mov_b64_e32 v[16:17], s[8:9]
+; GCN-NEXT:    v_mov_b32_e32 v28, s16
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    s_nop 0
+; GCN-NEXT:    v_smfmac_f32_32x32x32_bf16 v[0:15], v[24:27], v[16:23], v28 cbsz:1 abid:2
+; GCN-NEXT:    v_mov_b32_e32 v16, 0
+; GCN-NEXT:    s_nop 7
+; GCN-NEXT:    s_nop 2
+; GCN-NEXT:    global_store_dwordx4 v16, v[8:11], s[6:7] offset:32
+; GCN-NEXT:    global_store_dwordx4 v16, v[12:15], s[6:7] offset:48
+; GCN-NEXT:    global_store_dwordx4 v16, v[0:3], s[6:7]
+; GCN-NEXT:    global_store_dwordx4 v16, v[4:7], s[6:7] offset:16
+; GCN-NEXT:    s_endpgm
+bb:
+  %id = call i32 @llvm.amdgcn.workitem.id.x()
+  %gep = getelementptr <16 x float>, ptr addrspace(1) %arg, i32 %id
+  %in.1 = load <16 x float>, ptr addrspace(1) %gep
+  %mai.1 = tail call <16 x float> @llvm.amdgcn.smfmac.f32.32x32x32.bf16(<8 x bfloat> %a, <16 x bfloat> %b, <16 x float> %in.1, i32 %idx, i32 1, i32 2)
+  store <16 x float> %mai.1, ptr addrspace(1) %arg
+  ret void
+}
+
+define <16 x float> @test_smfmac_f32_32x32x32_bf16(<8 x bfloat> %arg0, <16 x bfloat> %arg1, <16 x float> %arg2, i32 %arg3) {
+; GCN-LABEL: test_smfmac_f32_32x32x32_bf16:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_accvgpr_write_b32 a0, v12
+; GCN-NEXT:    v_accvgpr_write_b32 a1, v13
+; GCN-NEXT:    v_accvgpr_write_b32 a2, v14
+; GCN-NEXT:    v_accvgpr_write_b32 a3, v15
+; GCN-NEXT:    v_accvgpr_write_b32 a4, v16
+; GCN-NEXT:    v_accvgpr_write_b32 a5, v17
+; GCN-NEXT:    v_accvgpr_write_b32 a6, v18
+; GCN-NEXT:    v_accvgpr_write_b32 a7, v19
+; GCN-NEXT:    v_accvgpr_write_b32 a8, v20
+; GCN-NEXT:    v_accvgpr_write_b32 a9, v21
+; GCN-NEXT:    v_accvgpr_write_b32 a10, v22
+; GCN-NEXT:    v_accvgpr_write_b32 a11, v23
+; GCN-NEXT:    v_accvgpr_write_b32 a12, v24
+; GCN-NEXT:    v_accvgpr_write_b32 a13, v25
+; GCN-NEXT:    v_accvgpr_write_b32 a14, v26
+; GCN-NEXT:    v_accvgpr_write_b32 a15, v27
+; GCN-NEXT:    s_nop 1
+; GCN-NEXT:    v_smfmac_f32_32x32x32_bf16 a[0:15], v[0:3], v[4:11], v28
+; GCN-NEXT:    s_nop 7
+; GCN-NEXT:    s_nop 3
+; GCN-NEXT:    v_accvgpr_read_b32 v0, a0
+; GCN-NEXT:    v_accvgpr_read_b32 v1, a1
+; GCN-NEXT:    v_accvgpr_read_b32 v2, a2
+; GCN-NEXT:    v_accvgpr_read_b32 v3, a3
+; GCN-NEXT:    v_accvgpr_read_b32 v4, a4
+; GCN-NEXT:    v_accvgpr_read_b32 v5, a5
+; GCN-NEXT:    v_accvgpr_read_b32 v6, a6
+; GCN-NEXT:    v_accvgpr_read_b32 v7, a7
+; GCN-NEXT:    v_accvgpr_read_b32 v8, a8
+; GCN-NEXT:    v_accvgpr_read_b32 v9, a9
+; GCN-NEXT:    v_accvgpr_read_b32 v10, a10
+; GCN-NEXT:    v_accvgpr_read_b32 v11, a11
+; GCN-NEXT:    v_accvgpr_read_b32 v12, a12
+; GCN-NEXT:    v_accvgpr_read_b32 v13, a13
+; GCN-NEXT:    v_accvgpr_read_b32 v14, a14
+; GCN-NEXT:    v_accvgpr_read_b32 v15, a15
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+  %result = call <16 x float> @llvm.amdgcn.smfmac.f32.32x32x32.bf16(<8 x bfloat> %arg0, <16 x bfloat> %arg1, <16 x float> %arg2, i32 %arg3, i32 immarg 0, i32 immarg 0)
+  ret <16 x float> %result
+}
+
+define <16 x float> @test_smfmac_f32_32x32x32_bf16__flags0(<8 x bfloat> %arg0, <16 x bfloat> %arg1, <16 x float> %arg2, i32 %arg3) {
+; GCN-LABEL: test_smfmac_f32_32x32x32_bf16__flags0:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_accvgpr_write_b32 a0, v12
+; GCN-NEXT:    v_accvgpr_write_b32 a1, v13
+; GCN-NEXT:    v_accvgpr_write_b32 a2, v14
+; GCN-NEXT:    v_accvgpr_write_b32 a3, v15
+; GCN-NEXT:    v_accvgpr_write_b32 a4, v16
+; GCN-NEXT:    v_accvgpr_write_b32 a5, v17
+; GCN-NEXT:    v_accvgpr_write_b32 a6, v18
+; GCN-NEXT:    v_accvgpr_write_b32 a7, v19
+; GCN-NEXT:    v_accvgpr_write_b32 a8, v20
+; GCN-NEXT:    v_accvgpr_write_b32 a9, v21
+; GCN-NEXT:    v_accvgpr_write_b32 a10, v22
+; GCN-NEXT:    v_accvgpr_write_b32 a11, v23
+; GCN-NEXT:    v_accvgpr_write_b32 a12, v24
+; GCN-NEXT:    v_accvgpr_write_b32 a13, v25
+; GCN-NEXT:    v_accvgpr_write_b32 a14, v26
+; GCN-NEXT:    v_accvgpr_write_b32 a15, v27
+; GCN-NEXT:    s_nop 1
+; GCN-NEXT:    v_smfmac_f32_32x32x32_bf16 a[0:15], v[0:3], v[4:11], v28 cbsz:1 abid:3
+; GCN-NEXT:    s_nop 7
+; GCN-NEXT:    s_nop 3
+; GCN-NEXT:    v_accvgpr_read_b32 v0, a0
+; GCN-NEXT:    v_accvgpr_read_b32 v1, a1
+; GCN-NEXT:    v_accvgpr_read_b32 v2, a2
+; GCN-NEXT:    v_accvgpr_read_b32 v3, a3
+; GCN-NEXT:    v_accvgpr_read_b32 v4, a4
+; GCN-NEXT:    v_accvgpr_read_b32 v5, a5
+; GCN-NEXT:    v_accvgpr_read_b32 v6, a6
+; GCN-NEXT:    v_accvgpr_read_b32 v7, a7
+; GCN-NEXT:    v_accvgpr_read_b32 v8, a8
+; GCN-NEXT:    v_accvgpr_read_b32 v9, a9
+; GCN-NEXT:    v_accvgpr_read_b32 v10, a10
+; GCN-NEXT:    v_accvgpr_read_b32 v11, a11
+; GCN-NEXT:    v_accvgpr_read_b32 v12, a12
+; GCN-NEXT:    v_accvgpr_read_b32 v13, a13
+; GCN-NEXT:    v_accvgpr_read_b32 v14, a14
+; GCN-NEXT:    v_accvgpr_read_b32 v15, a15
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+  %result = call <16 x float> @llvm.amdgcn.smfmac.f32.32x32x32.bf16(<8 x bfloat> %arg0, <16 x bfloat> %arg1, <16 x float> %arg2, i32 %arg3, i32 immarg 1, i32 immarg 3)
+  ret <16 x float> %result
+}
+
+define <16 x float> @test_smfmac_f32_32x32x32_bf16__flags1(<8 x bfloat> %arg0, <16 x bfloat> %arg1, <16 x float> %arg2, i32 %arg3) {
+; GCN-LABEL: test_smfmac_f32_32x32x32_bf16__flags1:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_accvgpr_write_b32 a0, v12
+; GCN-NEXT:    v_accvgpr_write_b32 a1, v13
+; GCN-NEXT:    v_accvgpr_write_b32 a2, v14
+; GCN-NEXT:    v_accvgpr_write_b32 a3, v15
+; GCN-NEXT:    v_accvgpr_write_b32 a4, v16
+; GCN-NEXT:    v_accvgpr_write_b32 a5, v17
+; GCN-NEXT:    v_accvgpr_write_b32 a6, v18
+; GCN-NEXT:    v_accvgpr_write_b32 a7, v19
+; GCN-NEXT:    v_accvgpr_write_b32 a8, v20
+; GCN-NEXT:    v_accvgpr_write_b32 a9, v21
+; GCN-NEXT:    v_accvgpr_write_b32 a10, v22
+; GCN-NEXT:    v_accvgpr_write_b32 a11, v23
+; GCN-NEXT:    v_accvgpr_write_b32 a12, v24
+; GCN-NEXT:    v_accvgpr_write_b32 a13, v25
+; GCN-NEXT:    v_accvgpr_write_b32 a14, v26
+; GCN-NEXT:    v_accvgpr_write_b32 a15, v27
+; GCN-NEXT:    s_nop 1
+; GCN-NEXT:    v_smfmac_f32_32x32x32_bf16 a[0:15], v[0:3], v[4:11], v28 cbsz:3 abid:1
+; GCN-NEXT:    s_nop 7
+; GCN-NEXT:    s_nop 3
+; GCN-NEXT:    v_accvgpr_read_b32 v0, a0
+; GCN-NEXT:    v_accvgpr_read_b32 v1, a1
+; GCN-NEXT:    v_accvgpr_read_b32 v2, a2
+; GCN-NEXT:    v_accvgpr_read_b32 v3, a3
+; GCN-NEXT:    v_accvgpr_read_b32 v4, a4
+; GCN-NEXT:    v_accvgpr_read_b32 v5, a5
+; GCN-NEXT:    v_accvgpr_read_b32 v6, a6
+; GCN-NEXT:    v_accvgpr_read_b32 v7, a7
+; GCN-NEXT:    v_accvgpr_read_b32 v8, a8
+; GCN-NEXT:    v_accvgpr_read_b32 v9, a9
+; GCN-NEXT:    v_accvgpr_read_b32 v10, a10
+; GCN-NEXT:    v_accvgpr_read_b32 v11, a11
+; GCN-NEXT:    v_accvgpr_read_b32 v12, a12
+; GCN-NEXT:    v_accvgpr_read_b32 v13, a13
+; GCN-NEXT:    v_accvgpr_read_b32 v14, a14
+; GCN-NEXT:    v_accvgpr_read_b32 v15, a15
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+  %result = call <16 x float> @llvm.amdgcn.smfmac.f32.32x32x32.bf16(<8 x bfloat> %arg0, <16 x bfloat> %arg1, <16 x float> %arg2, i32 %arg3, i32 immarg 3, i32 immarg 1)
+  ret <16 x float> %result
+}
+
+define <16 x float> @test_smfmac_f32_32x32x32_bf16__sgpr(<8 x bfloat> inreg %arg0, <16 x bfloat> inreg %arg1, <16 x float> inreg %arg2, i32 inreg %arg3) {
+; GCN-LABEL: test_smfmac_f32_32x32x32_bf16__sgpr:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_mov_b32_e32 v28, s0
+; GCN-NEXT:    v_mov_b32_e32 v29, s1
+; GCN-NEXT:    v_mov_b32_e32 v30, s2
+; GCN-NEXT:    v_mov_b32_e32 v31, s3
+; GCN-NEXT:    v_mov_b32_e32 v12, s24
+; GCN-NEXT:    v_mov_b32_e32 v27, v9
+; GCN-NEXT:    v_mov_b32_e32 v26, v8
+; GCN-NEXT:    v_mov_b32_e32 v25, v7
+; GCN-NEXT:    v_mov_b32_e32 v24, v6
+; GCN-NEXT:    v_mov_b32_e32 v23, v5
+; GCN-NEXT:    v_mov_b32_e32 v22, v4
+; GCN-NEXT:    v_mov_b32_e32 v21, v3
+; GCN-NEXT:    v_mov_b32_e32 v20, v2
+; GCN-NEXT:    v_mov_b32_e32 v19, v1
+; GCN-NEXT:    v_mov_b32_e32 v18, v0
+; GCN-NEXT:    v_mov_b32_e32 v13, s25
+; GCN-NEXT:    v_mov_b32_e32 v14, s26
+; GCN-NEXT:    v_mov_b32_e32 v15, s27
+; GCN-NEXT:    v_mov_b32_e32 v16, s28
+; GCN-NEXT:    v_mov_b32_e32 v17, s29
+; GCN-NEXT:    v_accvgpr_write_b32 a0, v12
+; GCN-NEXT:    v_mov_b32_e32 v0, s16
+; GCN-NEXT:    v_mov_b32_e32 v1, s17
+; GCN-NEXT:    v_mov_b32_e32 v2, s18
+; GCN-NEXT:    v_mov_b32_e32 v3, s19
+; GCN-NEXT:    v_mov_b32_e32 v4, s20
+; GCN-NEXT:    v_mov_b32_e32 v5, s21
+; GCN-NEXT:    v_mov_b32_e32 v6, s22
+; GCN-NEXT:    v_mov_b32_e32 v7, s23
+; GCN-NEXT:    v_accvgpr_write_b32 a1, v13
+; GCN-NEXT:    v_accvgpr_write_b32 a2, v14
+; GCN-NEXT:    v_accvgpr_write_b32 a3, v15
+; GCN-NEXT:    v_accvgpr_write_b32 a4, v16
+; GCN-NEXT:    v_accvgpr_write_b32 a5, v17
+; GCN-NEXT:    v_accvgpr_write_b32 a6, v18
+; GCN-NEXT:    v_accvgpr_write_b32 a7, v19
+; GCN-NEXT:    v_accvgpr_write_b32 a8, v20
+; GCN-NEXT:    v_accvgpr_write_b32 a9, v21
+; GCN-NEXT:    v_accvgpr_write_b32 a10, v22
+; GCN-NEXT:    v_accvgpr_write_b32 a11, v23
+; GCN-NEXT:    v_accvgpr_write_b32 a12, v24
+; GCN-NEXT:    v_accvgpr_write_b32 a13, v25
+; GCN-NEXT:    v_accvgpr_write_b32 a14, v26
+; GCN-NEXT:    v_accvgpr_write_b32 a15, v27
+; GCN-NEXT:    s_nop 1
+; GCN-NEXT:    v_smfmac_f32_32x32x32_bf16 a[0:15], v[28:31], v[0:7], v10
+; GCN-NEXT:    s_nop 7
+; GCN-NEXT:    s_nop 3
+; GCN-NEXT:    v_accvgpr_read_b32 v0, a0
+; GCN-NEXT:    v_accvgpr_read_b32 v1, a1
+; GCN-NEXT:    v_accvgpr_read_b32 v2, a2
+; GCN-NEXT:    v_accvgpr_read_b32 v3, a3
+; GCN-NEXT:    v_accvgpr_read_b32 v4, a4
+; GCN-NEXT:    v_accvgpr_read_b32 v5, a5
+; GCN-NEXT:    v_accvgpr_read_b32 v6, a6
+; GCN-NEXT:    v_accvgpr_read_b32 v7, a7
+; GCN-NEXT:    v_accvgpr_read_b32 v8, a8
+; GCN-NEXT:    v_accvgpr_read_b32 v9, a9
+; GCN-NEXT:    v_accvgpr_read_b32 v10, a10
+; GCN-NEXT:    v_accvgpr_read_b32 v11, a11
+; GCN-NEXT:    v_accvgpr_read_b32 v12, a12
+; GCN-NEXT:    v_accvgpr_read_b32 v13, a13
+; GCN-NEXT:    v_accvgpr_read_b32 v14, a14
+; GCN-NEXT:    v_accvgpr_read_b32 v15, a15
+; GCN-NEXT:    s_setpc_b64 s[30:31]
   %result = call <16 x float> @llvm.amdgcn.smfmac.f32.32x32x32.bf16(<8 x bfloat> %arg0, <16 x bfloat> %arg1, <16 x float> %arg2, i32 %arg3, i32 immarg 0, i32 immarg 0)
   ret <16 x float> %result
 }
@@ -1516,7 +1046,7 @@ define amdgpu_kernel void @test_smfmac_i32_16x16x128_i8__vgpr(ptr addrspace(1) %
 ; SDAG-NEXT:    s_waitcnt vmcnt(0)
 ; SDAG-NEXT:    s_nop 0
 ; SDAG-NEXT:    v_smfmac_i32_16x16x128_i8 v[8:11], v[12:15], v[0:7], v17 cbsz:1 abid:2
-; SDAG-NEXT:    s_nop 6
+; SDAG-NEXT:    s_nop 7
 ; SDAG-NEXT:    global_store_dwordx4 v16, v[8:11], s[6:7]
 ; SDAG-NEXT:    s_endpgm
 ;
@@ -1542,7 +1072,7 @@ define amdgpu_kernel void @test_smfmac_i32_16x16x128_i8__vgpr(ptr addrspace(1) %
 ; GISEL-NEXT:    s_nop 0
 ; GISEL-NEXT:    v_smfmac_i32_16x16x128_i8 v[8:11], v[12:15], v[0:7], v16 cbsz:1 abid:2
 ; GISEL-NEXT:    v_mov_b32_e32 v0, 0
-; GISEL-NEXT:    s_nop 5
+; GISEL-NEXT:    s_nop 6
 ; GISEL-NEXT:    global_store_dwordx4 v0, v[8:11], s[0:1]
 ; GISEL-NEXT:    s_endpgm
 bb:
@@ -1564,7 +1094,7 @@ define <4 x i32> @test_smfmac_i32_16x16x128_i8(<4 x i32> %arg0, <8 x i32> %arg1,
 ; SDAG-NEXT:    v_accvgpr_write_b32 a3, v15
 ; SDAG-NEXT:    s_nop 1
 ; SDAG-NEXT:    v_smfmac_i32_16x16x128_i8 a[0:3], v[0:3], v[4:11], v16
-; SDAG-NEXT:    s_nop 6
+; SDAG-NEXT:    s_nop 7
 ; SDAG-NEXT:    v_accvgpr_read_b32 v0, a0
 ; SDAG-NEXT:    v_accvgpr_read_b32 v1, a1
 ; SDAG-NEXT:    v_accvgpr_read_b32 v2, a2
@@ -1575,7 +1105,7 @@ define <4 x i32> @test_smfmac_i32_16x16x128_i8(<4 x i32> %arg0, <8 x i32> %arg1,
 ; GISEL:       ; %bb.0:
 ; GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GISEL-NEXT:    v_smfmac_i32_16x16x128_i8 v[12:15], v[0:3], v[4:11], v16
-; GISEL-NEXT:    s_nop 6
+; GISEL-NEXT:    s_nop 7
 ; GISEL-NEXT:    v_mov_b32_e32 v0, v12
 ; GISEL-NEXT:    v_mov_b32_e32 v1, v13
 ; GISEL-NEXT:    v_mov_b32_e32 v2, v14
@@ -1595,7 +1125,7 @@ define <4 x i32> @test_smfmac_i32_16x16x128_i8__flags0(<4 x i32> %arg0, <8 x i32
 ; SDAG-NEXT:    v_accvgpr_write_b32 a3, v15
 ; SDAG-NEXT:    s_nop 1
 ; SDAG-NEXT:    v_smfmac_i32_16x16x128_i8 a[0:3], v[0:3], v[4:11], v16 cbsz:1 abid:3
-; SDAG-NEXT:    s_nop 6
+; SDAG-NEXT:    s_nop 7
 ; SDAG-NEXT:    v_accvgpr_read_b32 v0, a0
 ; SDAG-NEXT:    v_accvgpr_read_b32 v1, a1
 ; SDAG-NEXT:    v_accvgpr_read_b32 v2, a2
@@ -1606,7 +1136,7 @@ define <4 x i32> @test_smfmac_i32_16x16x128_i8__flags0(<4 x i32> %arg0, <8 x i32
 ; GISEL:       ; %bb.0:
 ; GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GISEL-NEXT:    v_smfmac_i32_16x16x128_i8 v[12:15], v[0:3], v[4:11], v16 cbsz:1 abid:3
-; GISEL-NEXT:    s_nop 6
+; GISEL-NEXT:    s_nop 7
 ; GISEL-NEXT:    v_mov_b32_e32 v0, v12
 ; GISEL-NEXT:    v_mov_b32_e32 v1, v13
 ; GISEL-NEXT:    v_mov_b32_e32 v2, v14
@@ -1626,7 +1156,7 @@ define <4 x i32> @test_smfmac_i32_16x16x128_i8__flags1(<4 x i32> %arg0, <8 x i32
 ; SDAG-NEXT:    v_accvgpr_write_b32 a3, v15
 ; SDAG-NEXT:    s_nop 1
 ; SDAG-NEXT:    v_smfmac_i32_16x16x128_i8 a[0:3], v[0:3], v[4:11], v16 cbsz:3 abid:1
-; SDAG-NEXT:    s_nop 6
+; SDAG-NEXT:    s_nop 7
 ; SDAG-NEXT:    v_accvgpr_read_b32 v0, a0
 ; SDAG-NEXT:    v_accvgpr_read_b32 v1, a1
 ; SDAG-NEXT:    v_accvgpr_read_b32 v2, a2
@@ -1637,7 +1167,7 @@ define <4 x i32> @test_smfmac_i32_16x16x128_i8__flags1(<4 x i32> %arg0, <8 x i32
 ; GISEL:       ; %bb.0:
 ; GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GISEL-NEXT:    v_smfmac_i32_16x16x128_i8 v[12:15], v[0:3], v[4:11], v16 cbsz:3 abid:1
-; GISEL-NEXT:    s_nop 6
+; GISEL-NEXT:    s_nop 7
 ; GISEL-NEXT:    v_mov_b32_e32 v0, v12
 ; GISEL-NEXT:    v_mov_b32_e32 v1, v13
 ; GISEL-NEXT:    v_mov_b32_e32 v2, v14
@@ -1670,7 +1200,7 @@ define <4 x i32> @test_smfmac_i32_16x16x128_i8__sgpr(<4 x i32> inreg %arg0, <8 x
 ; SDAG-NEXT:    v_mov_b32_e32 v12, s28
 ; SDAG-NEXT:    s_nop 1
 ; SDAG-NEXT:    v_smfmac_i32_16x16x128_i8 a[0:3], v[8:11], v[0:7], v12
-; SDAG-NEXT:    s_nop 6
+; SDAG-NEXT:    s_nop 7
 ; SDAG-NEXT:    v_accvgpr_read_b32 v0, a0
 ; SDAG-NEXT:    v_accvgpr_read_b32 v1, a1
 ; SDAG-NEXT:    v_accvgpr_read_b32 v2, a2
@@ -1735,7 +1265,7 @@ define amdgpu_kernel void @test_smfmac_i32_32x32x64_i8__vgpr(ptr addrspace(1) %a
 ; SDAG-NEXT:    v_smfmac_i32_32x32x64_i8 v[0:15], v[24:27], v[16:23], v28 cbsz:1 abid:2
 ; SDAG-NEXT:    v_mov_b32_e32 v16, 0
 ; SDAG-NEXT:    s_nop 7
-; SDAG-NEXT:    s_nop 1
+; SDAG-NEXT:    s_nop 2
 ; SDAG-NEXT:    global_store_dwordx4 v16, v[8:11], s[0:1] offset:32
 ; SDAG-NEXT:    global_store_dwordx4 v16, v[12:15], s[0:1] offset:48
 ; SDAG-NEXT:    global_store_dwordx4 v16, v[0:3], s[0:1]
@@ -1768,7 +1298,7 @@ define amdgpu_kernel void @test_smfmac_i32_32x32x64_i8__vgpr(ptr addrspace(1) %a
 ; GISEL-NEXT:    v_smfmac_i32_32x32x64_i8 v[0:15], v[24:27], v[16:23], v28 cbsz:1 abid:2
 ; GISEL-NEXT:    v_mov_b32_e32 v16, 0
 ; GISEL-NEXT:    s_nop 7
-; GISEL-NEXT:    s_nop 1
+; GISEL-NEXT:    s_nop 2
 ; GISEL-NEXT:    global_store_dwordx4 v16, v[0:3], s[0:1]
 ; GISEL-NEXT:    global_store_dwordx4 v16, v[4:7], s[0:1] offset:16
 ; GISEL-NEXT:    global_store_dwordx4 v16, v[8:11], s[0:1] offset:32
@@ -1806,7 +1336,7 @@ define <16 x i32> @test_smfmac_i32_32x32x64_i8(<4 x i32> %arg0, <8 x i32> %arg1,
 ; SDAG-NEXT:    s_nop 1
 ; SDAG-NEXT:    v_smfmac_i32_32x32x64_i8 a[0:15], v[0:3], v[4:11], v28
 ; SDAG-NEXT:    s_nop 7
-; SDAG-NEXT:    s_nop 2
+; SDAG-NEXT:    s_nop 3
 ; SDAG-NEXT:    v_accvgpr_read_b32 v0, a0
 ; SDAG-NEXT:    v_accvgpr_read_b32 v1, a1
 ; SDAG-NEXT:    v_accvgpr_read_b32 v2, a2
@@ -1878,7 +1408,7 @@ define <16 x i32> @test_smfmac_i32_32x32x64_i8__flags0(<4 x i32> %arg0, <8 x i32
 ; SDAG-NEXT:    s_nop 1
 ; SDAG-NEXT:    v_smfmac_i32_32x32x64_i8 a[0:15], v[0:3], v[4:11], v28 cbsz:1 abid:3
 ; SDAG-NEXT:    s_nop 7
-; SDAG-NEXT:    s_nop 2
+; SDAG-NEXT:    s_nop 3
 ; SDAG-NEXT:    v_accvgpr_read_b32 v0, a0
 ; SDAG-NEXT:    v_accvgpr_read_b32 v1, a1
 ; SDAG-NEXT:    v_accvgpr_read_b32 v2, a2
@@ -1950,7 +1480,7 @@ define <16 x i32> @test_smfmac_i32_32x32x64_i8__flags1(<4 x i32> %arg0, <8 x i32
 ; SDAG-NEXT:    s_nop 1
 ; SDAG-NEXT:    v_smfmac_i32_32x32x64_i8 a[0:15], v[0:3], v[4:11], v28 cbsz:3 abid:1
 ; SDAG-NEXT:    s_nop 7
-; SDAG-NEXT:    s_nop 2
+; SDAG-NEXT:    s_nop 3
 ; SDAG-NEXT:    v_accvgpr_read_b32 v0, a0
 ; SDAG-NEXT:    v_accvgpr_read_b32 v1, a1
 ; SDAG-NEXT:    v_accvgpr_read_b32 v2, a2
@@ -2050,7 +1580,7 @@ define <16 x i32> @test_smfmac_i32_32x32x64_i8__sgpr(<4 x i32> inreg %arg0, <8 x
 ; SDAG-NEXT:    s_nop 1
 ; SDAG-NEXT:    v_smfmac_i32_32x32x64_i8 a[0:15], v[28:31], v[0:7], v10
 ; SDAG-NEXT:    s_nop 7
-; SDAG-NEXT:    s_nop 2
+; SDAG-NEXT:    s_nop 3
 ; SDAG-NEXT:    v_accvgpr_read_b32 v0, a0
 ; SDAG-NEXT:    v_accvgpr_read_b32 v1, a1
 ; SDAG-NEXT:    v_accvgpr_read_b32 v2, a2
@@ -2145,7 +1675,7 @@ define amdgpu_kernel void @test_smfmac_f32_16x16x128_bf8_bf8__vgpr(ptr addrspace
 ; SDAG-NEXT:    s_waitcnt vmcnt(0)
 ; SDAG-NEXT:    s_nop 0
 ; SDAG-NEXT:    v_smfmac_f32_16x16x128_bf8_bf8 v[8:11], v[12:15], v[0:7], v17 cbsz:1 abid:2
-; SDAG-NEXT:    s_nop 6
+; SDAG-NEXT:    s_nop 7
 ; SDAG-NEXT:    global_store_dwordx4 v16, v[8:11], s[6:7]
 ; SDAG-NEXT:    s_endpgm
 ;
@@ -2171,7 +1701,7 @@ define amdgpu_kernel void @test_smfmac_f32_16x16x128_bf8_bf8__vgpr(ptr addrspace
 ; GISEL-NEXT:    s_nop 0
 ; GISEL-NEXT:    v_smfmac_f32_16x16x128_bf8_bf8 v[8:11], v[12:15], v[0:7], v16 cbsz:1 abid:2
 ; GISEL-NEXT:    v_mov_b32_e32 v0, 0
-; GISEL-NEXT:    s_nop 5
+; GISEL-NEXT:    s_nop 6
 ; GISEL-NEXT:    global_store_dwordx4 v0, v[8:11], s[0:1]
 ; GISEL-NEXT:    s_endpgm
 bb:
@@ -2193,7 +1723,7 @@ define <4 x float> @test_smfmac_f32_16x16x128_bf8_bf8(<4 x i32> %arg0, <8 x i32>
 ; SDAG-NEXT:    v_accvgpr_write_b32 a3, v15
 ; SDAG-NEXT:    s_nop 1
 ; SDAG-NEXT:    v_smfmac_f32_16x16x128_bf8_bf8 a[0:3], v[0:3], v[4:11], v16
-; SDAG-NEXT:    s_nop 6
+; SDAG-NEXT:    s_nop 7
 ; SDAG-NEXT:    v_accvgpr_read_b32 v0, a0
 ; SDAG-NEXT:    v_accvgpr_read_b32 v1, a1
 ; SDAG-NEXT:    v_accvgpr_read_b32 v2, a2
@@ -2204,7 +1734,7 @@ define <4 x float> @test_smfmac_f32_16x16x128_bf8_bf8(<4 x i32> %arg0, <8 x i32>
 ; GISEL:       ; %bb.0:
 ; GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GISEL-NEXT:    v_smfmac_f32_16x16x128_bf8_bf8 v[12:15], v[0:3], v[4:11], v16
-; GISEL-NEXT:    s_nop 6
+; GISEL-NEXT:    s_nop 7
 ; GISEL-NEXT:    v_mov_b32_e32 v0, v12
 ; GISEL-NEXT:    v_mov_b32_e32 v1, v13
 ; GISEL-NEXT:    v_mov_b32_e32 v2, v14
@@ -2224,7 +1754,7 @@ define <4 x float> @test_smfmac_f32_16x16x128_bf8_bf8__flags0(<4 x i32> %arg0, <
 ; SDAG-NEXT:    v_accvgpr_write_b32 a3, v15
 ; SDAG-NEXT:    s_nop 1
 ; SDAG-NEXT:    v_smfmac_f32_16x16x128_bf8_bf8 a[0:3], v[0:3], v[4:11], v16 cbsz:1 abid:3
-; SDAG-NEXT:    s_nop 6
+; SDAG-NEXT:    s_nop 7
 ; SDAG-NEXT:    v_accvgpr_read_b32 v0, a0
 ; SDAG-NEXT:    v_accvgpr_read_b32 v1, a1
 ; SDAG-NEXT:    v_accvgpr_read_b32 v2, a2
@@ -2235,7 +1765,7 @@ define <4 x float> @test_smfmac_f32_16x16x128_bf8_bf8__flags0(<4 x i32> %arg0, <
 ; GISEL:       ; %bb.0:
 ; GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GISEL-NEXT:    v_smfmac_f32_16x16x128_bf8_bf8 v[12:15], v[0:3], v[4:11], v16 cbsz:1 abid:3
-; GISEL-NEXT:    s_nop 6
+; GISEL-NEXT:    s_nop 7
 ; GISEL-NEXT:    v_mov_b32_e32 v0, v12
 ; GISEL-NEXT:    v_mov_b32_e32 v1, v13
 ; GISEL-NEXT:    v_mov_b32_e32 v2, v14
@@ -2255,7 +1785,7 @@ define <4 x float> @test_smfmac_f32_16x16x128_bf8_bf8__flags1(<4 x i32> %arg0, <
 ; SDAG-NEXT:    v_accvgpr_write_b32 a3, v15
 ; SDAG-NEXT:    s_nop 1
 ; SDAG-NEXT:    v_smfmac_f32_16x16x128_bf8_bf8 a[0:3], v[0:3], v[4:11], v16 cbsz:3 abid:1
-; SDAG-NEXT:    s_nop 6
+; SDAG-NEXT:    s_nop 7
 ; SDAG-NEXT:    v_accvgpr_read_b32 v0, a0
 ; SDAG-NEXT:    v_accvgpr_read_b32 v1, a1
 ; SDAG-NEXT:    v_accvgpr_read_b32 v2, a2
@@ -2266,7 +1796,7 @@ define <4 x float> @test_smfmac_f32_16x16x128_bf8_bf8__flags1(<4 x i32> %arg0, <
 ; GISEL:       ; %bb.0:
 ; GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GISEL-NEXT:    v_smfmac_f32_16x16x128_bf8_bf8 v[12:15], v[0:3], v[4:11], v16 cbsz:3 abid:1
-; GISEL-NEXT:    s_nop 6
+; GISEL-NEXT:    s_nop 7
 ; GISEL-NEXT:    v_mov_b32_e32 v0, v12
 ; GISEL-NEXT:    v_mov_b32_e32 v1, v13
 ; GISEL-NEXT:    v_mov_b32_e32 v2, v14
@@ -2299,7 +1829,7 @@ define <4 x float> @test_smfmac_f32_16x16x128_bf8_bf8__sgpr(<4 x i32> inreg %arg
 ; SDAG-NEXT:    v_mov_b32_e32 v12, s28
 ; SDAG-NEXT:    s_nop 1
 ; SDAG-NEXT:    v_smfmac_f32_16x16x128_bf8_bf8 a[0:3], v[8:11], v[0:7], v12
-; SDAG-NEXT:    s_nop 6
+; SDAG-NEXT:    s_nop 7
 ; SDAG-NEXT:    v_accvgpr_read_b32 v0, a0
 ; SDAG-NEXT:    v_accvgpr_read_b32 v1, a1
 ; SDAG-NEXT:    v_accvgpr_read_b32 v2, a2
@@ -2360,7 +1890,7 @@ define amdgpu_kernel void @test_smfmac_f32_16x16x128_bf8_fp8__vgpr(ptr addrspace
 ; SDAG-NEXT:    s_waitcnt vmcnt(0)
 ; SDAG-NEXT:    s_nop 0
 ; SDAG-NEXT:    v_smfmac_f32_16x16x128_bf8_fp8 v[8:11], v[12:15], v[0:7], v17 cbsz:1 abid:2
-; SDAG-NEXT:    s_nop 6
+; SDAG-NEXT:    s_nop 7
 ; SDAG-NEXT:    global_store_dwordx4 v16, v[8:11], s[6:7]
 ; SDAG-NEXT:    s_endpgm
 ;
@@ -2386,7 +1916,7 @@ define amdgpu_kernel void @test_smfmac_f32_16x16x128_bf8_fp8__vgpr(ptr addrspace
 ; GISEL-NEXT:    s_nop 0
 ; GISEL-NEXT:    v_smfmac_f32_16x16x128_bf8_fp8 v[8:11], v[12:15], v[0:7], v16 cbsz:1 abid:2
 ; GISEL-NEXT:    v_mov_b32_e32 v0, 0
-; GISEL-NEXT:    s_nop 5
+; GISEL-NEXT:    s_nop 6
 ; GISEL-NEXT:    global_store_dwordx4 v0, v[8:11], s[0:1]
 ; GISEL-NEXT:    s_endpgm
 bb:
@@ -2408,7 +1938,7 @@ define <4 x float> @test_smfmac_f32_16x16x128_bf8_fp8(<4 x i32> %arg0, <8 x i32>
 ; SDAG-NEXT:    v_accvgpr_write_b32 a3, v15
 ; SDAG-NEXT:    s_nop 1
 ; SDAG-NEXT:    v_smfmac_f32_16x16x128_bf8_fp8 a[0:3], v[0:3], v[4:11], v16
-; SDAG-NEXT:    s_nop 6
+; SDAG-NEXT:    s_nop 7
 ; SDAG-NEXT:    v_accvgpr_read_b32 v0, a0
 ; SDAG-NEXT:    v_accvgpr_read_b32 v1, a1
 ; SDAG-NEXT:    v_accvgpr_read_b32 v2, a2
@@ -2419,7 +1949,7 @@ define <4 x float> @test_smfmac_f32_16x16x128_bf8_fp8(<4 x i32> %arg0, <8 x i32>
 ; GISEL:       ; %bb.0:
 ; GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GISEL-NEXT:    v_smfmac_f32_16x16x128_bf8_fp8 v[12:15], v[0:3], v[4:11], v16
-; GISEL-NEXT:    s_nop 6
+; GISEL-NEXT:    s_nop 7
 ; GISEL-NEXT:    v_mov_b32_e32 v0, v12
 ; GISEL-NEXT:    v_mov_b32_e32 v1, v13
 ; GISEL-NEXT:    v_mov_b32_e32 v2, v14
@@ -2439,7 +1969,7 @@ define <4 x float> @test_smfmac_f32_16x16x128_bf8_fp8__flags0(<4 x i32> %arg0, <
 ; SDAG-NEXT:    v_accvgpr_write_b32 a3, v15
 ; SDAG-NEXT:    s_nop 1
 ; SDAG-NEXT:    v_smfmac_f32_16x16x128_bf8_fp8 a[0:3], v[0:3], v[4:11], v16 cbsz:1 abid:3
-; SDAG-NEXT:    s_nop 6
+; SDAG-NEXT:    s_nop 7
 ; SDAG-NEXT:    v_accvgpr_read_b32 v0, a0
 ; SDAG-NEXT:    v_accvgpr_read_b32 v1, a1
 ; SDAG-NEXT:    v_accvgpr_read_b32 v2, a2
@@ -2450,7 +1980,7 @@ define <4 x float> @test_smfmac_f32_16x16x128_bf8_fp8__flags0(<4 x i32> %arg0, <
 ; GISEL:       ; %bb.0:
 ; GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GISEL-NEXT:    v_smfmac_f32_16x16x128_bf8_fp8 v[12:15], v[0:3], v[4:11], v16 cbsz:1 abid:3
-; GISEL-NEXT:    s_nop 6
+; GISEL-NEXT:    s_nop 7
 ; GISEL-NEXT:    v_mov_b32_e32 v0, v12
 ; GISEL-NEXT:    v_mov_b32_e32 v1, v13
 ; GISEL-NEXT:    v_mov_b32_e32 v2, v14
@@ -2470,7 +2000,7 @@ define <4 x float> @test_smfmac_f32_16x16x128_bf8_fp8__flags1(<4 x i32> %arg0, <
 ; SDAG-NEXT:    v_accvgpr_write_b32 a3, v15
 ; SDAG-NEXT:    s_nop 1
 ; SDAG-NEXT:    v_smfmac_f32_16x16x128_bf8_fp8 a[0:3], v[0:3], v[4:11], v16 cbsz:3 abid:1
-; SDAG-NEXT:    s_nop 6
+; SDAG-NEXT:    s_nop 7
 ; SDAG-NEXT:    v_accvgpr_read_b32 v0, a0
 ; SDAG-NEXT:    v_accvgpr_read_b32 v1, a1
 ; SDAG-NEXT:    v_accvgpr_read_b32 v2, a2
@@ -2481,7 +2011,7 @@ define <4 x float> @test_smfmac_f32_16x16x128_bf8_fp8__flags1(<4 x i32> %arg0, <
 ; GISEL:       ; %bb.0:
 ; GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GISEL-NEXT:    v_smfmac_f32_16x16x128_bf8_fp8 v[12:15], v[0:3], v[4:11], v16 cbsz:3 abid:1
-; GISEL-NEXT:    s_nop 6
+; GISEL-NEXT:    s_nop 7
 ; GISEL-NEXT:    v_mov_b32_e32 v0, v12
 ; GISEL-NEXT:    v_mov_b32_e32 v1, v13
 ; GISEL-NEXT:    v_mov_b32_e32 v2, v14
@@ -2514,7 +2044,7 @@ define <4 x float> @test_smfmac_f32_16x16x128_bf8_fp8__sgpr(<4 x i32> inreg %arg
 ; SDAG-NEXT:    v_mov_b32_e32 v12, s28
 ; SDAG-NEXT:    s_nop 1
 ; SDAG-NEXT:    v_smfmac_f32_16x16x128_bf8_fp8 a[0:3], v[8:11], v[0:7], v12
-; SDAG-NEXT:    s_nop 6
+; SDAG-NEXT:    s_nop 7
 ; SDAG-NEXT:    v_accvgpr_read_b32 v0, a0
 ; SDAG-NEXT:    v_accvgpr_read_b32 v1, a1
 ; SDAG-NEXT:    v_accvgpr_read_b32 v2, a2
@@ -2575,7 +2105,7 @@ define amdgpu_kernel void @test_smfmac_f32_16x16x128_fp8_bf8__vgpr(ptr addrspace
 ; SDAG-NEXT:    s_waitcnt vmcnt(0)
 ; SDAG-NEXT:    s_nop 0
 ; SDAG-NEXT:    v_smfmac_f32_16x16x128_fp8_bf8 v[8:11], v[12:15], v[0:7], v17 cbsz:1 abid:2
-; SDAG-NEXT:    s_nop 6
+; SDAG-NEXT:    s_nop 7
 ; SDAG-NEXT:    global_store_dwordx4 v16, v[8:11], s[6:7]
 ; SDAG-NEXT:    s_endpgm
 ;
@@ -2601,7 +2131,7 @@ define amdgpu_kernel void @test_smfmac_f32_16x16x128_fp8_bf8__vgpr(ptr addrspace
 ; GISEL-NEXT:    s_nop 0
 ; GISEL-NEXT:    v_smfmac_f32_16x16x128_fp8_bf8 v[8:11], v[12:15], v[0:7], v16 cbsz:1 abid:2
 ; GISEL-NEXT:    v_mov_b32_e32 v0, 0
-; GISEL-NEXT:    s_nop 5
+; GISEL-NEXT:    s_nop 6
 ; GISEL-NEXT:    global_store_dwordx4 v0, v[8:11], s[0:1]
 ; GISEL-NEXT:    s_endpgm
 bb:
@@ -2623,7 +2153,7 @@ define <4 x float> @test_smfmac_f32_16x16x128_fp8_bf8(<4 x i32> %arg0, <8 x i32>
 ; SDAG-NEXT:    v_accvgpr_write_b32 a3, v15
 ; SDAG-NEXT:    s_nop 1
 ; SDAG-NEXT:    v_smfmac_f32_16x16x128_fp8_bf8 a[0:3], v[0:3], v[4:11], v16
-; SDAG-NEXT:    s_nop 6
+; SDAG-NEXT:    s_nop 7
 ; SDAG-NEXT:    v_accvgpr_read_b32 v0, a0
 ; SDAG-NEXT:    v_accvgpr_read_b32 v1, a1
 ; SDAG-NEXT:    v_accvgpr_read_b32 v2, a2
@@ -2634,7 +2164,7 @@ define <4 x float> @test_smfmac_f32_16x16x128_fp8_bf8(<4 x i32> %arg0, <8 x i32>
 ; GISEL:       ; %bb.0:
 ; GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GISEL-NEXT:    v_smfmac_f32_16x16x128_fp8_bf8 v[12:15], v[0:3], v[4:11], v16
-; GISEL-NEXT:    s_nop 6
+; GISEL-NEXT:    s_nop 7
 ; GISEL-NEXT:    v_mov_b32_e32 v0, v12
 ; GISEL-NEXT:    v_mov_b32_e32 v1, v13
 ; GISEL-NEXT:    v_mov_b32_e32 v2, v14
@@ -2654,7 +2184,7 @@ define <4 x float> @test_smfmac_f32_16x16x128_fp8_bf8__flags0(<4 x i32> %arg0, <
 ; SDAG-NEXT:    v_accvgpr_write_b32 a3, v15
 ; SDAG-NEXT:    s_nop 1
 ; SDAG-NEXT:    v_smfmac_f32_16x16x128_fp8_bf8 a[0:3], v[0:3], v[4:11], v16 cbsz:1 abid:3
-; SDAG-NEXT:    s_nop 6
+; SDAG-NEXT:    s_nop 7
 ; SDAG-NEXT:    v_accvgpr_read_b32 v0, a0
 ; SDAG-NEXT:    v_accvgpr_read_b32 v1, a1
 ; SDAG-NEXT:    v_accvgpr_read_b32 v2, a2
@@ -2665,7 +2195,7 @@ define <4 x float> @test_smfmac_f32_16x16x128_fp8_bf8__flags0(<4 x i32> %arg0, <
 ; GISEL:       ; %bb.0:
 ; GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GISEL-NEXT:    v_smfmac_f32_16x16x128_fp8_bf8 v[12:15], v[0:3], v[4:11], v16 cbsz:1 abid:3
-; GISEL-NEXT:    s_nop 6
+; GISEL-NEXT:    s_nop 7
 ; GISEL-NEXT:    v_mov_b32_e32 v0, v12
 ; GISEL-NEXT:    v_mov_b32_e32 v1, v13
 ; GISEL-NEXT:    v_mov_b32_e32 v2, v14
@@ -2685,7 +2215,7 @@ define <4 x float> @test_smfmac_f32_16x16x128_fp8_bf8__flags1(<4 x i32> %arg0, <
 ; SDAG-NEXT:    v_accvgpr_write_b32 a3, v15
 ; SDAG-NEXT:    s_nop 1
 ; SDAG-NEXT:    v_smfmac_f32_16x16x128_fp8_bf8 a[0:3], v[0:3], v[4:11], v16 cbsz:3 abid:1
-; SDAG-NEXT:    s_nop 6
+; SDAG-NEXT:    s_nop 7
 ; SDAG-NEXT:    v_accvgpr_read_b32 v0, a0
 ; SDAG-NEXT:    v_accvgpr_read_b32 v1, a1
 ; SDAG-NEXT:    v_accvgpr_read_b32 v2, a2
@@ -2696,7 +2226,7 @@ define <4 x float> @test_smfmac_f32_16x16x128_fp8_bf8__flags1(<4 x i32> %arg0, <
 ; GISEL:       ; %bb.0:
 ; GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GISEL-NEXT:    v_smfmac_f32_16x16x128_fp8_bf8 v[12:15], v[0:3], v[4:11], v16 cbsz:3 abid:1
-; GISEL-NEXT:    s_nop 6
+; GISEL-NEXT:    s_nop 7
 ; GISEL-NEXT:    v_mov_b32_e32 v0, v12
 ; GISEL-NEXT:    v_mov_b32_e32 v1, v13
 ; GISEL-NEXT:    v_mov_b32_e32 v2, v14
@@ -2729,7 +2259,7 @@ define <4 x float> @test_smfmac_f32_16x16x128_fp8_bf8__sgpr(<4 x i32> inreg %arg
 ; SDAG-NEXT:    v_mov_b32_e32 v12, s28
 ; SDAG-NEXT:    s_nop 1
 ; SDAG-NEXT:    v_smfmac_f32_16x16x128_fp8_bf8 a[0:3], v[8:11], v[0:7], v12
-; SDAG-NEXT:    s_nop 6
+; SDAG-NEXT:    s_nop 7
 ; SDAG-NEXT:    v_accvgpr_read_b32 v0, a0
 ; SDAG-NEXT:    v_accvgpr_read_b32 v1, a1
 ; SDAG-NEXT:    v_accvgpr_read_b32 v2, a2
@@ -2790,7 +2320,7 @@ define amdgpu_kernel void @test_smfmac_f32_16x16x128_fp8_fp8__vgpr(ptr addrspace
 ; SDAG-NEXT:    s_waitcnt vmcnt(0)
 ; SDAG-NEXT:    s_nop 0
 ; SDAG-NEXT:    v_smfmac_f32_16x16x128_fp8_fp8 v[8:11], v[12:15], v[0:7], v17 cbsz:1 abid:2
-; SDAG-NEXT:    s_nop 6
+; SDAG-NEXT:    s_nop 7
 ; SDAG-NEXT:    global_store_dwordx4 v16, v[8:11], s[6:7]
 ; SDAG-NEXT:    s_endpgm
 ;
@@ -2816,7 +2346,7 @@ define amdgpu_kernel void @test_smfmac_f32_16x16x128_fp8_fp8__vgpr(ptr addrspace
 ; GISEL-NEXT:    s_nop 0
 ; GISEL-NEXT:    v_smfmac_f32_16x16x128_fp8_fp8 v[8:11], v[12:15], v[0:7], v16 cbsz:1 abid:2
 ; GISEL-NEXT:    v_mov_b32_e32 v0, 0
-; GISEL-NEXT:    s_nop 5
+; GISEL-NEXT:    s_nop 6
 ; GISEL-NEXT:    global_store_dwordx4 v0, v[8:11], s[0:1]
 ; GISEL-NEXT:    s_endpgm
 bb:
@@ -2838,7 +2368,7 @@ define <4 x float> @test_smfmac_f32_16x16x128_fp8_fp8(<4 x i32> %arg0, <8 x i32>
 ; SDAG-NEXT:    v_accvgpr_write_b32 a3, v15
 ; SDAG-NEXT:    s_nop 1
 ; SDAG-NEXT:    v_smfmac_f32_16x16x128_fp8_fp8 a[0:3], v[0:3], v[4:11], v16
-; SDAG-NEXT:    s_nop 6
+; SDAG-NEXT:    s_nop 7
 ; SDAG-NEXT:    v_accvgpr_read_b32 v0, a0
 ; SDAG-NEXT:    v_accvgpr_read_b32 v1, a1
 ; SDAG-NEXT:    v_accvgpr_read_b32 v2, a2
@@ -2849,7 +2379,7 @@ define <4 x float> @test_smfmac_f32_16x16x128_fp8_fp8(<4 x i32> %arg0, <8 x i32>
 ; GISEL:       ; %bb.0:
 ; GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GISEL-NEXT:    v_smfmac_f32_16x16x128_fp8_fp8 v[12:15], v[0:3], v[4:11], v16
-; GISEL-NEXT:    s_nop 6
+; GISEL-NEXT:    s_nop 7
 ; GISEL-NEXT:    v_mov_b32_e32 v0, v12
 ; GISEL-NEXT:    v_mov_b32_e32 v1, v13
 ; GISEL-NEXT:    v_mov_b32_e32 v2, v14
@@ -2869,7 +2399,7 @@ define <4 x float> @test_smfmac_f32_16x16x128_fp8_fp8__flags0(<4 x i32> %arg0, <
 ; SDAG-NEXT:    v_accvgpr_write_b32 a3, v15
 ; SDAG-NEXT:    s_nop 1
 ; SDAG-NEXT:    v_smfmac_f32_16x16x128_fp8_fp8 a[0:3], v[0:3], v[4:11], v16 cbsz:1 abid:3
-; SDAG-NEXT:    s_nop 6
+; SDAG-NEXT:    s_nop 7
 ; SDAG-NEXT:    v_accvgpr_read_b32 v0, a0
 ; SDAG-NEXT:    v_accvgpr_read_b32 v1, a1
 ; SDAG-NEXT:    v_accvgpr_read_b32 v2, a2
@@ -2880,7 +2410,7 @@ define <4 x float> @test_smfmac_f32_16x16x128_fp8_fp8__flags0(<4 x i32> %arg0, <
 ; GISEL:       ; %bb.0:
 ; GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GISEL-NEXT:    v_smfmac_f32_16x16x128_fp8_fp8 v[12:15], v[0:3], v[4:11], v16 cbsz:1 abid:3
-; GISEL-NEXT:    s_nop 6
+; GISEL-NEXT:    s_nop 7
 ; GISEL-NEXT:    v_mov_b32_e32 v0, v12
 ; GISEL-NEXT:    v_mov_b32_e32 v1, v13
 ; GISEL-NEXT:    v_mov_b32_e32 v2, v14
@@ -2900,7 +2430,7 @@ define <4 x float> @test_smfmac_f32_16x16x128_fp8_fp8__flags1(<4 x i32> %arg0, <
 ; SDAG-NEXT:    v_accvgpr_write_b32 a3, v15
 ; SDAG-NEXT:    s_nop 1
 ; SDAG-NEXT:    v_smfmac_f32_16x16x128_fp8_fp8 a[0:3], v[0:3], v[4:11], v16 cbsz:3 abid:1
-; SDAG-NEXT:    s_nop 6
+; SDAG-NEXT:    s_nop 7
 ; SDAG-NEXT:    v_accvgpr_read_b32 v0, a0
 ; SDAG-NEXT:    v_accvgpr_read_b32 v1, a1
 ; SDAG-NEXT:    v_accvgpr_read_b32 v2, a2
@@ -2911,7 +2441,7 @@ define <4 x float> @test_smfmac_f32_16x16x128_fp8_fp8__flags1(<4 x i32> %arg0, <
 ; GISEL:       ; %bb.0:
 ; GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GISEL-NEXT:    v_smfmac_f32_16x16x128_fp8_fp8 v[12:15], v[0:3], v[4:11], v16 cbsz:3 abid:1
-; GISEL-NEXT:    s_nop 6
+; GISEL-NEXT:    s_nop 7
 ; GISEL-NEXT:    v_mov_b32_e32 v0, v12
 ; GISEL-NEXT:    v_mov_b32_e32 v1, v13
 ; GISEL-NEXT:    v_mov_b32_e32 v2, v14
@@ -2944,7 +2474,7 @@ define <4 x float> @test_smfmac_f32_16x16x128_fp8_fp8__sgpr(<4 x i32> inreg %arg
 ; SDAG-NEXT:    v_mov_b32_e32 v12, s28
 ; SDAG-NEXT:    s_nop 1
 ; SDAG-NEXT:    v_smfmac_f32_16x16x128_fp8_fp8 a[0:3], v[8:11], v[0:7], v12
-; SDAG-NEXT:    s_nop 6
+; SDAG-NEXT:    s_nop 7
 ; SDAG-NEXT:    v_accvgpr_read_b32 v0, a0
 ; SDAG-NEXT:    v_accvgpr_read_b32 v1, a1
 ; SDAG-NEXT:    v_accvgpr_read_b32 v2, a2
@@ -3009,7 +2539,7 @@ define amdgpu_kernel void @test_smfmac_f32_32x32x64_bf8_bf8__vgpr(ptr addrspace(
 ; SDAG-NEXT:    v_smfmac_f32_32x32x64_bf8_bf8 v[0:15], v[24:27], v[16:23], v28 cbsz:1 abid:2
 ; SDAG-NEXT:    v_mov_b32_e32 v16, 0
 ; SDAG-NEXT:    s_nop 7
-; SDAG-NEXT:    s_nop 1
+; SDAG-NEXT:    s_nop 2
 ; SDAG-NEXT:    global_store_dwordx4 v16, v[8:11], s[0:1] offset:32
 ; SDAG-NEXT:    global_store_dwordx4 v16, v[12:15], s[0:1] offset:48
 ; SDAG-NEXT:    global_store_dwordx4 v16, v[0:3], s[0:1]
@@ -3042,7 +2572,7 @@ define amdgpu_kernel void @test_smfmac_f32_32x32x64_bf8_bf8__vgpr(ptr addrspace(
 ; GISEL-NEXT:    v_smfmac_f32_32x32x64_bf8_bf8 v[0:15], v[24:27], v[16:23], v28 cbsz:1 abid:2
 ; GISEL-NEXT:    v_mov_b32_e32 v16, 0
 ; GISEL-NEXT:    s_nop 7
-; GISEL-NEXT:    s_nop 1
+; GISEL-NEXT:    s_nop 2
 ; GISEL-NEXT:    global_store_dwordx4 v16, v[0:3], s[0:1]
 ; GISEL-NEXT:    global_store_dwordx4 v16, v[4:7], s[0:1] offset:16
 ; GISEL-NEXT:    global_store_dwordx4 v16, v[8:11], s[0:1] offset:32
@@ -3080,7 +2610,7 @@ define <16 x float> @test_smfmac_f32_32x32x64_bf8_bf8(<4 x i32> %arg0, <8 x i32>
 ; SDAG-NEXT:    s_nop 1
 ; SDAG-NEXT:    v_smfmac_f32_32x32x64_bf8_bf8 a[0:15], v[0:3], v[4:11], v28
 ; SDAG-NEXT:    s_nop 7
-; SDAG-NEXT:    s_nop 2
+; SDAG-NEXT:    s_nop 3
 ; SDAG-NEXT:    v_accvgpr_read_b32 v0, a0
 ; SDAG-NEXT:    v_accvgpr_read_b32 v1, a1
 ; SDAG-NEXT:    v_accvgpr_read_b32 v2, a2
@@ -3152,7 +2682,7 @@ define <16 x float> @test_smfmac_f32_32x32x64_bf8_bf8__flags0(<4 x i32> %arg0, <
 ; SDAG-NEXT:    s_nop 1
 ; SDAG-NEXT:    v_smfmac_f32_32x32x64_bf8_bf8 a[0:15], v[0:3], v[4:11], v28 cbsz:1 abid:3
 ; SDAG-NEXT:    s_nop 7
-; SDAG-NEXT:    s_nop 2
+; SDAG-NEXT:    s_nop 3
 ; SDAG-NEXT:    v_accvgpr_read_b32 v0, a0
 ; SDAG-NEXT:    v_accvgpr_read_b32 v1, a1
 ; SDAG-NEXT:    v_accvgpr_read_b32 v2, a2
@@ -3224,7 +2754,7 @@ define <16 x float> @test_smfmac_f32_32x32x64_bf8_bf8__flags1(<4 x i32> %arg0, <
 ; SDAG-NEXT:    s_nop 1
 ; SDAG-NEXT:    v_smfmac_f32_32x32x64_bf8_bf8 a[0:15], v[0:3], v[4:11], v28 cbsz:3 abid:1
 ; SDAG-NEXT:    s_nop 7
-; SDAG-NEXT:    s_nop 2
+; SDAG-NEXT:    s_nop 3
 ; SDAG-NEXT:    v_accvgpr_read_b32 v0, a0
 ; SDAG-NEXT:    v_accvgpr_read_b32 v1, a1
 ; SDAG-NEXT:    v_accvgpr_read_b32 v2, a2
@@ -3324,7 +2854,7 @@ define <16 x float> @test_smfmac_f32_32x32x64_bf8_bf8__sgpr(<4 x i32> inreg %arg
 ; SDAG-NEXT:    s_nop 1
 ; SDAG-NEXT:    v_smfmac_f32_32x32x64_bf8_bf8 a[0:15], v[28:31], v[0:7], v10
 ; SDAG-NEXT:    s_nop 7
-; SDAG-NEXT:    s_nop 2
+; SDAG-NEXT:    s_nop 3
 ; SDAG-NEXT:    v_accvgpr_read_b32 v0, a0
 ; SDAG-NEXT:    v_accvgpr_read_b32 v1, a1
 ; SDAG-NEXT:    v_accvgpr_read_b32 v2, a2
@@ -3423,7 +2953,7 @@ define amdgpu_kernel void @test_smfmac_f32_32x32x64_bf8_fp8__vgpr(ptr addrspace(
 ; SDAG-NEXT:    v_smfmac_f32_32x32x64_bf8_fp8 v[0:15], v[24:27], v[16:23], v28 cbsz:1 abid:2
 ; SDAG-NEXT:    v_mov_b32_e32 v16, 0
 ; SDAG-NEXT:    s_nop 7
-; SDAG-NEXT:    s_nop 1
+; SDAG-NEXT:    s_nop 2
 ; SDAG-NEXT:    global_store_dwordx4 v16, v[8:11], s[0:1] offset:32
 ; SDAG-NEXT:    global_store_dwordx4 v16, v[12:15], s[0:1] offset:48
 ; SDAG-NEXT:    global_store_dwordx4 v16, v[0:3], s[0:1]
@@ -3456,7 +2986,7 @@ define amdgpu_kernel void @test_smfmac_f32_32x32x64_bf8_fp8__vgpr(ptr addrspace(
 ; GISEL-NEXT:    v_smfmac_f32_32x32x64_bf8_fp8 v[0:15], v[24:27], v[16:23], v28 cbsz:1 abid:2
 ; GISEL-NEXT:    v_mov_b32_e32 v16, 0
 ; GISEL-NEXT:    s_nop 7
-; GISEL-NEXT:    s_nop 1
+; GISEL-NEXT:    s_nop 2
 ; GISEL-NEXT:    global_store_dwordx4 v16, v[0:3], s[0:1]
 ; GISEL-NEXT:    global_store_dwordx4 v16, v[4:7], s[0:1] offset:16
 ; GISEL-NEXT:    global_store_dwordx4 v16, v[8:11], s[0:1] offset:32
@@ -3494,7 +3024,7 @@ define <16 x float> @test_smfmac_f32_32x32x64_bf8_fp8(<4 x i32> %arg0, <8 x i32>
 ; SDAG-NEXT:    s_nop 1
 ; SDAG-NEXT:    v_smfmac_f32_32x32x64_bf8_fp8 a[0:15], v[0:3], v[4:11], v28
 ; SDAG-NEXT:    s_nop 7
-; SDAG-NEXT:    s_nop 2
+; SDAG-NEXT:    s_nop 3
 ; SDAG-NEXT:    v_accvgpr_read_b32 v0, a0
 ; SDAG-NEXT:    v_accvgpr_read_b32 v1, a1
 ; SDAG-NEXT:    v_accvgpr_read_b32 v2, a2
@@ -3566,7 +3096,7 @@ define <16 x float> @test_smfmac_f32_32x32x64_bf8_fp8__flags0(<4 x i32> %arg0, <
 ; SDAG-NEXT:    s_nop 1
 ; SDAG-NEXT:    v_smfmac_f32_32x32x64_bf8_fp8 a[0:15], v[0:3], v[4:11], v28 cbsz:1 abid:3
 ; SDAG-NEXT:    s_nop 7
-; SDAG-NEXT:    s_nop 2
+; SDAG-NEXT:    s_nop 3
 ; SDAG-NEXT:    v_accvgpr_read_b32 v0, a0
 ; SDAG-NEXT:    v_accvgpr_read_b32 v1, a1
 ; SDAG-NEXT:    v_accvgpr_read_b32 v2, a2
@@ -3638,7 +3168,7 @@ define <16 x float> @test_smfmac_f32_32x32x64_bf8_fp8__flags1(<4 x i32> %arg0, <
 ; SDAG-NEXT:    s_nop 1
 ; SDAG-NEXT:    v_smfmac_f32_32x32x64_bf8_fp8 a[0:15], v[0:3], v[4:11], v28 cbsz:3 abid:1
 ; SDAG-NEXT:    s_nop 7
-; SDAG-NEXT:    s_nop 2
+; SDAG-NEXT:    s_nop 3
 ; SDAG-NEXT:    v_accvgpr_read_b32 v0, a0
 ; SDAG-NEXT:    v_accvgpr_read_b32 v1, a1
 ; SDAG-NEXT:    v_accvgpr_read_b32 v2, a2
@@ -3738,7 +3268,7 @@ define <16 x float> @test_smfmac_f32_32x32x64_bf8_fp8__sgpr(<4 x i32> inreg %arg
 ; SDAG-NEXT:    s_nop 1
 ; SDAG-NEXT:    v_smfmac_f32_32x32x64_bf8_fp8 a[0:15], v[28:31], v[0:7], v10
 ; SDAG-NEXT:    s_nop 7
-; SDAG-NEXT:    s_nop 2
+; SDAG-NEXT:    s_nop 3
 ; SDAG-NEXT:    v_accvgpr_read_b32 v0, a0
 ; SDAG-NEXT:    v_accvgpr_read_b32 v1, a1
 ; SDAG-NEXT:    v_accvgpr_read_b32 v2, a2
@@ -3837,7 +3367,7 @@ define amdgpu_kernel void @test_smfmac_f32_32x32x64_fp8_bf8__vgpr(ptr addrspace(
 ; SDAG-NEXT:    v_smfmac_f32_32x32x64_fp8_bf8 v[0:15], v[24:27], v[16:23], v28 cbsz:1 abid:2
 ; SDAG-NEXT:    v_mov_b32_e32 v16, 0
 ; SDAG-NEXT:    s_nop 7
-; SDAG-NEXT:    s_nop 1
+; SDAG-NEXT:    s_nop 2
 ; SDAG-NEXT:    global_store_dwordx4 v16, v[8:11], s[0:1] offset:32
 ; SDAG-NEXT:    global_store_dwordx4 v16, v[12:15], s[0:1] offset:48
 ; SDAG-NEXT:    global_store_dwordx4 v16, v[0:3], s[0:1]
@@ -3870,7 +3400,7 @@ define amdgpu_kernel void @test_smfmac_f32_32x32x64_fp8_bf8__vgpr(ptr addrspace(
 ; GISEL-NEXT:    v_smfmac_f32_32x32x64_fp8_bf8 v[0:15], v[24:27], v[16:23], v28 cbsz:1 abid:2
 ; GISEL-NEXT:    v_mov_b32_e32 v16, 0
 ; GISEL-NEXT:    s_nop 7
-; GISEL-NEXT:    s_nop 1
+; GISEL-NEXT:    s_nop 2
 ; GISEL-NEXT:    global_store_dwordx4 v16, v[0:3], s[0:1]
 ; GISEL-NEXT:    global_store_dwordx4 v16, v[4:7], s[0:1] offset:16
 ; GISEL-NEXT:    global_store_dwordx4 v16, v[8:11], s[0:1] offset:32
@@ -3908,7 +3438,7 @@ define <16 x float> @test_smfmac_f32_32x32x64_fp8_bf8(<4 x i32> %arg0, <8 x i32>
 ; SDAG-NEXT:    s_nop 1
 ; SDAG-NEXT:    v_smfmac_f32_32x32x64_fp8_bf8 a[0:15], v[0:3], v[4:11], v28
 ; SDAG-NEXT:    s_nop 7
-; SDAG-NEXT:    s_nop 2
+; SDAG-NEXT:    s_nop 3
 ; SDAG-NEXT:    v_accvgpr_read_b32 v0, a0
 ; SDAG-NEXT:    v_accvgpr_read_b32 v1, a1
 ; SDAG-NEXT:    v_accvgpr_read_b32 v2, a2
@@ -3980,7 +3510,7 @@ define <16 x float> @test_smfmac_f32_32x32x64_fp8_bf8__flags0(<4 x i32> %arg0, <
 ; SDAG-NEXT:    s_nop 1
 ; SDAG-NEXT:    v_smfmac_f32_32x32x64_fp8_bf8 a[0:15], v[0:3], v[4:11], v28 cbsz:1 abid:3
 ; SDAG-NEXT:    s_nop 7
-; SDAG-NEXT:    s_nop 2
+; SDAG-NEXT:    s_nop 3
 ; SDAG-NEXT:    v_accvgpr_read_b32 v0, a0
 ; SDAG-NEXT:    v_accvgpr_read_b32 v1, a1
 ; SDAG-NEXT:    v_accvgpr_read_b32 v2, a2
@@ -4052,7 +3582,7 @@ define <16 x float> @test_smfmac_f32_32x32x64_fp8_bf8__flags1(<4 x i32> %arg0, <
 ; SDAG-NEXT:    s_nop 1
 ; SDAG-NEXT:    v_smfmac_f32_32x32x64_fp8_bf8 a[0:15], v[0:3], v[4:11], v28 cbsz:3 abid:1
 ; SDAG-NEXT:    s_nop 7
-; SDAG-NEXT:    s_nop 2
+; SDAG-NEXT:    s_nop 3
 ; SDAG-NEXT:    v_accvgpr_read_b32 v0, a0
 ; SDAG-NEXT:    v_accvgpr_read_b32 v1, a1
 ; SDAG-NEXT:    v_accvgpr_read_b32 v2, a2
@@ -4152,7 +3682,7 @@ define <16 x float> @test_smfmac_f32_32x32x64_fp8_bf8__sgpr(<4 x i32> inreg %arg
 ; SDAG-NEXT:    s_nop 1
 ; SDAG-NEXT:    v_smfmac_f32_32x32x64_fp8_bf8 a[0:15], v[28:31], v[0:7], v10
 ; SDAG-NEXT:    s_nop 7
-; SDAG-NEXT:    s_nop 2
+; SDAG-NEXT:    s_nop 3
 ; SDAG-NEXT:    v_accvgpr_read_b32 v0, a0
 ; SDAG-NEXT:    v_accvgpr_read_b32 v1, a1
 ; SDAG-NEXT:    v_accvgpr_read_b32 v2, a2
@@ -4251,7 +3781,7 @@ define amdgpu_kernel void @test_smfmac_f32_32x32x64_fp8_fp8__vgpr(ptr addrspace(
 ; SDAG-NEXT:    v_smfmac_f32_32x32x64_fp8_fp8 v[0:15], v[24:27], v[16:23], v28 cbsz:1 abid:2
 ; SDAG-NEXT:    v_mov_b32_e32 v16, 0
 ; SDAG-NEXT:    s_nop 7
-; SDAG-NEXT:    s_nop 1
+; SDAG-NEXT:    s_nop 2
 ; SDAG-NEXT:    global_store_dwordx4 v16, v[8:11], s[0:1] offset:32
 ; SDAG-NEXT:    global_store_dwordx4 v16, v[12:15], s[0:1] offset:48
 ; SDAG-NEXT:    global_store_dwordx4 v16, v[0:3], s[0:1]
@@ -4284,7 +3814,7 @@ define amdgpu_kernel void @test_smfmac_f32_32x32x64_fp8_fp8__vgpr(ptr addrspace(
 ; GISEL-NEXT:    v_smfmac_f32_32x32x64_fp8_fp8 v[0:15], v[24:27], v[16:23], v28 cbsz:1 abid:2
 ; GISEL-NEXT:    v_mov_b32_e32 v16, 0
 ; GISEL-NEXT:    s_nop 7
-; GISEL-NEXT:    s_nop 1
+; GISEL-NEXT:    s_nop 2
 ; GISEL-NEXT:    global_store_dwordx4 v16, v[0:3], s[0:1]
 ; GISEL-NEXT:    global_store_dwordx4 v16, v[4:7], s[0:1] offset:16
 ; GISEL-NEXT:    global_store_dwordx4 v16, v[8:11], s[0:1] offset:32
@@ -4322,7 +3852,7 @@ define <16 x float> @test_smfmac_f32_32x32x64_fp8_fp8(<4 x i32> %arg0, <8 x i32>
 ; SDAG-NEXT:    s_nop 1
 ; SDAG-NEXT:    v_smfmac_f32_32x32x64_fp8_fp8 a[0:15], v[0:3], v[4:11], v28
 ; SDAG-NEXT:    s_nop 7
-; SDAG-NEXT:    s_nop 2
+; SDAG-NEXT:    s_nop 3
 ; SDAG-NEXT:    v_accvgpr_read_b32 v0, a0
 ; SDAG-NEXT:    v_accvgpr_read_b32 v1, a1
 ; SDAG-NEXT:    v_accvgpr_read_b32 v2, a2
@@ -4394,7 +3924,7 @@ define <16 x float> @test_smfmac_f32_32x32x64_fp8_fp8__flags0(<4 x i32> %arg0, <
 ; SDAG-NEXT:    s_nop 1
 ; SDAG-NEXT:    v_smfmac_f32_32x32x64_fp8_fp8 a[0:15], v[0:3], v[4:11], v28 cbsz:1 abid:3
 ; SDAG-NEXT:    s_nop 7
-; SDAG-NEXT:    s_nop 2
+; SDAG-NEXT:    s_nop 3
 ; SDAG-NEXT:    v_accvgpr_read_b32 v0, a0
 ; SDAG-NEXT:    v_accvgpr_read_b32 v1, a1
 ; SDAG-NEXT:    v_accvgpr_read_b32 v2, a2
@@ -4466,7 +3996,7 @@ define <16 x float> @test_smfmac_f32_32x32x64_fp8_fp8__flags1(<4 x i32> %arg0, <
 ; SDAG-NEXT:    s_nop 1
 ; SDAG-NEXT:    v_smfmac_f32_32x32x64_fp8_fp8 a[0:15], v[0:3], v[4:11], v28 cbsz:3 abid:1
 ; SDAG-NEXT:    s_nop 7
-; SDAG-NEXT:    s_nop 2
+; SDAG-NEXT:    s_nop 3
 ; SDAG-NEXT:    v_accvgpr_read_b32 v0, a0
 ; SDAG-NEXT:    v_accvgpr_read_b32 v1, a1
 ; SDAG-NEXT:    v_accvgpr_read_b32 v2, a2
@@ -4566,7 +4096,7 @@ define <16 x float> @test_smfmac_f32_32x32x64_fp8_fp8__sgpr(<4 x i32> inreg %arg
 ; SDAG-NEXT:    s_nop 1
 ; SDAG-NEXT:    v_smfmac_f32_32x32x64_fp8_fp8 a[0:15], v[28:31], v[0:7], v10
 ; SDAG-NEXT:    s_nop 7
-; SDAG-NEXT:    s_nop 2
+; SDAG-NEXT:    s_nop 3
 ; SDAG-NEXT:    v_accvgpr_read_b32 v0, a0
 ; SDAG-NEXT:    v_accvgpr_read_b32 v1, a1
 ; SDAG-NEXT:    v_accvgpr_read_b32 v2, a2
@@ -4626,6 +4156,4 @@ define <16 x float> @test_smfmac_f32_32x32x64_fp8_fp8__sgpr(<4 x i32> inreg %arg
   ret <16 x float> %result
 }
 
-attributes #0 = { "amdgpu-flat-work-group-size"="1,256" }
-;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
-; GCN: {{.*}}
+attributes #0 = { "amdgpu-flat-work-group-size"="1,256" }
\ No newline at end of file
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.buffer.load.format.v3f16.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.buffer.load.format.v3f16.ll
index 0522d5258b9b5..fed7a8ec105fd 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.buffer.load.format.v3f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.buffer.load.format.v3f16.ll
@@ -122,6 +122,7 @@ define amdgpu_gs void @main(<4 x i32> %arg, i32 %arg1) {
 ; GFX12-NEXT:    v_readfirstlane_b32 s5, v1
 ; GFX12-NEXT:    v_readfirstlane_b32 s6, v2
 ; GFX12-NEXT:    v_readfirstlane_b32 s7, v3
+; GFX12-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
 ; GFX12-NEXT:    v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
 ; GFX12-NEXT:    v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.atomic.fadd.v2bf16.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.atomic.fadd.v2bf16.ll
index 2698ce1dc3fe3..a2b9c869c9c9a 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.atomic.fadd.v2bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.atomic.fadd.v2bf16.ll
@@ -47,15 +47,14 @@ define <2 x bfloat> @struct_ptr_buffer_atomic_add_v2bf16_rtn__vgpr_val__vgpr_rsr
 ; GFX1200-NEXT:    v_readfirstlane_b32 s6, v3
 ; GFX1200-NEXT:    v_readfirstlane_b32 s7, v4
 ; GFX1200-NEXT:    v_readfirstlane_b32 s3, v7
-; GFX1200-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX1200-NEXT:    s_wait_alu 0xf1ff
 ; GFX1200-NEXT:    v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[1:2]
+; GFX1200-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
 ; GFX1200-NEXT:    v_cmp_eq_u64_e64 s0, s[6:7], v[3:4]
-; GFX1200-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX1200-NEXT:    v_cmp_eq_u32_e64 s1, s3, v7
-; GFX1200-NEXT:    s_wait_alu 0xfffe
+; GFX1200-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
 ; GFX1200-NEXT:    s_and_b32 s0, vcc_lo, s0
 ; GFX1200-NEXT:    s_wait_alu 0xfffe
-; GFX1200-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX1200-NEXT:    s_and_b32 s0, s0, s1
 ; GFX1200-NEXT:    s_wait_alu 0xfffe
 ; GFX1200-NEXT:    s_and_saveexec_b32 s0, s0
@@ -64,13 +63,11 @@ define <2 x bfloat> @struct_ptr_buffer_atomic_add_v2bf16_rtn__vgpr_val__vgpr_rsr
 ; GFX1200-NEXT:    ; implicit-def: $vgpr1_vgpr2_vgpr3_vgpr4
 ; GFX1200-NEXT:    ; implicit-def: $vgpr7
 ; GFX1200-NEXT:    ; implicit-def: $vgpr5_vgpr6
-; GFX1200-NEXT:    s_wait_alu 0xfffe
 ; GFX1200-NEXT:    s_xor_b32 exec_lo, exec_lo, s0
 ; GFX1200-NEXT:    s_cbranch_execnz .LBB2_1
 ; GFX1200-NEXT:  ; %bb.2:
 ; GFX1200-NEXT:    s_mov_b32 exec_lo, s2
 ; GFX1200-NEXT:    s_wait_loadcnt 0x0
-; GFX1200-NEXT:    s_wait_alu 0xfffe
 ; GFX1200-NEXT:    s_setpc_b64 s[30:31]
   %ret = call <2 x bfloat> @llvm.amdgcn.struct.ptr.buffer.atomic.fadd.v2bf16(<2 x bfloat> %val, ptr addrspace(8) %rsrc, i32 %vindex, i32 %voffset, i32 %soffset, i32 0)
   ret <2 x bfloat> %ret
@@ -91,15 +88,14 @@ define void @struct_ptr_buffer_atomic_add_v2bf16_noret__vgpr_val__vgpr_rsrc__vgp
 ; GFX1200-NEXT:    v_readfirstlane_b32 s6, v3
 ; GFX1200-NEXT:    v_readfirstlane_b32 s7, v4
 ; GFX1200-NEXT:    v_readfirstlane_b32 s3, v7
-; GFX1200-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX1200-NEXT:    s_wait_alu 0xf1ff
 ; GFX1200-NEXT:    v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[1:2]
+; GFX1200-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
 ; GFX1200-NEXT:    v_cmp_eq_u64_e64 s0, s[6:7], v[3:4]
-; GFX1200-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX1200-NEXT:    v_cmp_eq_u32_e64 s1, s3, v7
-; GFX1200-NEXT:    s_wait_alu 0xfffe
+; GFX1200-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
 ; GFX1200-NEXT:    s_and_b32 s0, vcc_lo, s0
 ; GFX1200-NEXT:    s_wait_alu 0xfffe
-; GFX1200-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX1200-NEXT:    s_and_b32 s0, s0, s1
 ; GFX1200-NEXT:    s_wait_alu 0xfffe
 ; GFX1200-NEXT:    s_and_saveexec_b32 s0, s0
@@ -108,12 +104,10 @@ define void @struct_ptr_buffer_atomic_add_v2bf16_noret__vgpr_val__vgpr_rsrc__vgp
 ; GFX1200-NEXT:    ; implicit-def: $vgpr7
 ; GFX1200-NEXT:    ; implicit-def: $vgpr0
 ; GFX1200-NEXT:    ; implicit-def: $vgpr5_vgpr6
-; GFX1200-NEXT:    s_wait_alu 0xfffe
 ; GFX1200-NEXT:    s_xor_b32 exec_lo, exec_lo, s0
 ; GFX1200-NEXT:    s_cbranch_execnz .LBB3_1
 ; GFX1200-NEXT:  ; %bb.2:
 ; GFX1200-NEXT:    s_mov_b32 exec_lo, s2
-; GFX1200-NEXT:    s_wait_alu 0xfffe
 ; GFX1200-NEXT:    s_setpc_b64 s[30:31]
   %ret = call <2 x bfloat> @llvm.amdgcn.struct.ptr.buffer.atomic.fadd.v2bf16(<2 x bfloat> %val, ptr addrspace(8) %rsrc, i32 %vindex, i32 %voffset, i32 %soffset, i32 0)
   ret void
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.atomic.fadd_nortn.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.atomic.fadd_nortn.ll
index 6e94d4fe9fa27..91217c219c451 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.atomic.fadd_nortn.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.atomic.fadd_nortn.ll
@@ -271,15 +271,14 @@ define void @struct_ptr_buffer_atomic_add_f32_noret__vgpr_val__vgpr_rsrc__vgpr_v
 ; GFX1200-NEXT:    v_readfirstlane_b32 s6, v3
 ; GFX1200-NEXT:    v_readfirstlane_b32 s7, v4
 ; GFX1200-NEXT:    v_readfirstlane_b32 s3, v7
-; GFX1200-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX1200-NEXT:    s_wait_alu 0xf1ff
 ; GFX1200-NEXT:    v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[1:2]
+; GFX1200-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
 ; GFX1200-NEXT:    v_cmp_eq_u64_e64 s0, s[6:7], v[3:4]
-; GFX1200-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX1200-NEXT:    v_cmp_eq_u32_e64 s1, s3, v7
-; GFX1200-NEXT:    s_wait_alu 0xfffe
+; GFX1200-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
 ; GFX1200-NEXT:    s_and_b32 s0, vcc_lo, s0
 ; GFX1200-NEXT:    s_wait_alu 0xfffe
-; GFX1200-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX1200-NEXT:    s_and_b32 s0, s0, s1
 ; GFX1200-NEXT:    s_wait_alu 0xfffe
 ; GFX1200-NEXT:    s_and_saveexec_b32 s0, s0
@@ -288,12 +287,10 @@ define void @struct_ptr_buffer_atomic_add_f32_noret__vgpr_val__vgpr_rsrc__vgpr_v
 ; GFX1200-NEXT:    ; implicit-def: $vgpr7
 ; GFX1200-NEXT:    ; implicit-def: $vgpr0
 ; GFX1200-NEXT:    ; implicit-def: $vgpr5_vgpr6
-; GFX1200-NEXT:    s_wait_alu 0xfffe
 ; GFX1200-NEXT:    s_xor_b32 exec_lo, exec_lo, s0
 ; GFX1200-NEXT:    s_cbranch_execnz .LBB4_1
 ; GFX1200-NEXT:  ; %bb.2:
 ; GFX1200-NEXT:    s_mov_b32 exec_lo, s2
-; GFX1200-NEXT:    s_wait_alu 0xfffe
 ; GFX1200-NEXT:    s_setpc_b64 s[30:31]
   %ret = call float @llvm.amdgcn.struct.ptr.buffer.atomic.fadd.f32(float %val, ptr addrspace(8) %rsrc, i32 %vindex, i32 %voffset, i32 %soffset, i32 0)
   ret void
@@ -413,15 +410,14 @@ define void @struct_ptr_buffer_atomic_add_v2f16_noret__vgpr_val__vgpr_rsrc__vgpr
 ; GFX1200-NEXT:    v_readfirstlane_b32 s6, v3
 ; GFX1200-NEXT:    v_readfirstlane_b32 s7, v4
 ; GFX1200-NEXT:    v_readfirstlane_b32 s3, v7
-; GFX1200-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX1200-NEXT:    s_wait_alu 0xf1ff
 ; GFX1200-NEXT:    v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[1:2]
+; GFX1200-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
 ; GFX1200-NEXT:    v_cmp_eq_u64_e64 s0, s[6:7], v[3:4]
-; GFX1200-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX1200-NEXT:    v_cmp_eq_u32_e64 s1, s3, v7
-; GFX1200-NEXT:    s_wait_alu 0xfffe
+; GFX1200-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
 ; GFX1200-NEXT:    s_and_b32 s0, vcc_lo, s0
 ; GFX1200-NEXT:    s_wait_alu 0xfffe
-; GFX1200-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX1200-NEXT:    s_and_b32 s0, s0, s1
 ; GFX1200-NEXT:    s_wait_alu 0xfffe
 ; GFX1200-NEXT:    s_and_saveexec_b32 s0, s0
@@ -430,12 +426,10 @@ define void @struct_ptr_buffer_atomic_add_v2f16_noret__vgpr_val__vgpr_rsrc__vgpr
 ; GFX1200-NEXT:    ; implicit-def: $vgpr7
 ; GFX1200-NEXT:    ; implicit-def: $vgpr0
 ; GFX1200-NEXT:    ; implicit-def: $vgpr5_vgpr6
-; GFX1200-NEXT:    s_wait_alu 0xfffe
 ; GFX1200-NEXT:    s_xor_b32 exec_lo, exec_lo, s0
 ; GFX1200-NEXT:    s_cbranch_execnz .LBB5_1
 ; GFX1200-NEXT:  ; %bb.2:
 ; GFX1200-NEXT:    s_mov_b32 exec_lo, s2
-; GFX1200-NEXT:    s_wait_alu 0xfffe
 ; GFX1200-NEXT:    s_setpc_b64 s[30:31]
   %ret = call <2 x half> @llvm.amdgcn.struct.ptr.buffer.atomic.fadd.v2f16(<2 x half> %val, ptr addrspace(8) %rsrc, i32 %vindex, i32 %voffset, i32 %soffset, i32 0)
   ret void
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.atomic.fadd_rtn.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.atomic.fadd_rtn.ll
index 9d8572493b456..80fd1e05477f1 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.atomic.fadd_rtn.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.atomic.fadd_rtn.ll
@@ -216,15 +216,14 @@ define float @struct_ptr_buffer_atomic_add_f32_rtn__vgpr_val__vgpr_rsrc__vgpr_vo
 ; GFX1200-NEXT:    v_readfirstlane_b32 s6, v3
 ; GFX1200-NEXT:    v_readfirstlane_b32 s7, v4
 ; GFX1200-NEXT:    v_readfirstlane_b32 s3, v7
-; GFX1200-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX1200-NEXT:    s_wait_alu 0xf1ff
 ; GFX1200-NEXT:    v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[1:2]
+; GFX1200-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
 ; GFX1200-NEXT:    v_cmp_eq_u64_e64 s0, s[6:7], v[3:4]
-; GFX1200-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX1200-NEXT:    v_cmp_eq_u32_e64 s1, s3, v7
-; GFX1200-NEXT:    s_wait_alu 0xfffe
+; GFX1200-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
 ; GFX1200-NEXT:    s_and_b32 s0, vcc_lo, s0
 ; GFX1200-NEXT:    s_wait_alu 0xfffe
-; GFX1200-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX1200-NEXT:    s_and_b32 s0, s0, s1
 ; GFX1200-NEXT:    s_wait_alu 0xfffe
 ; GFX1200-NEXT:    s_and_saveexec_b32 s0, s0
@@ -233,13 +232,11 @@ define float @struct_ptr_buffer_atomic_add_f32_rtn__vgpr_val__vgpr_rsrc__vgpr_vo
 ; GFX1200-NEXT:    ; implicit-def: $vgpr1_vgpr2_vgpr3_vgpr4
 ; GFX1200-NEXT:    ; implicit-def: $vgpr7
 ; GFX1200-NEXT:    ; implicit-def: $vgpr5_vgpr6
-; GFX1200-NEXT:    s_wait_alu 0xfffe
 ; GFX1200-NEXT:    s_xor_b32 exec_lo, exec_lo, s0
 ; GFX1200-NEXT:    s_cbranch_execnz .LBB4_1
 ; GFX1200-NEXT:  ; %bb.2:
 ; GFX1200-NEXT:    s_mov_b32 exec_lo, s2
 ; GFX1200-NEXT:    s_wait_loadcnt 0x0
-; GFX1200-NEXT:    s_wait_alu 0xfffe
 ; GFX1200-NEXT:    s_setpc_b64 s[30:31]
   %ret = call float @llvm.amdgcn.struct.ptr.buffer.atomic.fadd.f32(float %val, ptr addrspace(8) %rsrc, i32 %vindex, i32 %voffset, i32 %soffset, i32 0)
   ret float %ret
@@ -329,15 +326,14 @@ define <2 x half> @struct_ptr_buffer_atomic_add_v2f16_rtn__vgpr_val__vgpr_rsrc__
 ; GFX1200-NEXT:    v_readfirstlane_b32 s6, v3
 ; GFX1200-NEXT:    v_readfirstlane_b32 s7, v4
 ; GFX1200-NEXT:    v_readfirstlane_b32 s3, v7
-; GFX1200-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX1200-NEXT:    s_wait_alu 0xf1ff
 ; GFX1200-NEXT:    v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[1:2]
+; GFX1200-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
 ; GFX1200-NEXT:    v_cmp_eq_u64_e64 s0, s[6:7], v[3:4]
-; GFX1200-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX1200-NEXT:    v_cmp_eq_u32_e64 s1, s3, v7
-; GFX1200-NEXT:    s_wait_alu 0xfffe
+; GFX1200-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
 ; GFX1200-NEXT:    s_and_b32 s0, vcc_lo, s0
 ; GFX1200-NEXT:    s_wait_alu 0xfffe
-; GFX1200-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX1200-NEXT:    s_and_b32 s0, s0, s1
 ; GFX1200-NEXT:    s_wait_alu 0xfffe
 ; GFX1200-NEXT:    s_and_saveexec_b32 s0, s0
@@ -346,13 +342,11 @@ define <2 x half> @struct_ptr_buffer_atomic_add_v2f16_rtn__vgpr_val__vgpr_rsrc__
 ; GFX1200-NEXT:    ; implicit-def: $vgpr1_vgpr2_vgpr3_vgpr4
 ; GFX1200-NEXT:    ; implicit-def: $vgpr7
 ; GFX1200-NEXT:    ; implicit-def: $vgpr5_vgpr6
-; GFX1200-NEXT:    s_wait_alu 0xfffe
 ; GFX1200-NEXT:    s_xor_b32 exec_lo, exec_lo, s0
 ; GFX1200-NEXT:    s_cbranch_execnz .LBB5_1
 ; GFX1200-NEXT:  ; %bb.2:
 ; GFX1200-NEXT:    s_mov_b32 exec_lo, s2
 ; GFX1200-NEXT:    s_wait_loadcnt 0x0
-; GFX1200-NEXT:    s_wait_alu 0xfffe
 ; GFX1200-NEXT:    s_setpc_b64 s[30:31]
   %ret = call <2 x half> @llvm.amdgcn.struct.ptr.buffer.atomic.fadd.v2f16(<2 x half> %val, ptr addrspace(8) %rsrc, i32 %vindex, i32 %voffset, i32 %soffset, i32 0)
   ret <2 x half> %ret
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.atomic.fmax.f32.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.atomic.fmax.f32.ll
index 6da16f0a3b053..13bb72a96142f 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.atomic.fmax.f32.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.atomic.fmax.f32.ll
@@ -455,10 +455,10 @@ define float @struct_ptr_buffer_atomic_add_f32_ret__vgpr_val__vgpr_rsrc__vgpr_vo
 ; GFX12-NEXT:    v_readfirstlane_b32 s5, v2
 ; GFX12-NEXT:    v_readfirstlane_b32 s6, v3
 ; GFX12-NEXT:    v_readfirstlane_b32 s7, v4
+; GFX12-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
 ; GFX12-NEXT:    v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[1:2]
 ; GFX12-NEXT:    v_cmp_eq_u64_e64 s1, s[6:7], v[3:4]
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-NEXT:    s_and_b32 s1, vcc_lo, s1
 ; GFX12-NEXT:    s_wait_alu 0xfffe
@@ -467,13 +467,11 @@ define float @struct_ptr_buffer_atomic_add_f32_ret__vgpr_val__vgpr_rsrc__vgpr_vo
 ; GFX12-NEXT:    buffer_atomic_max_num_f32 v0, v[5:6], s[4:7], s0 idxen offen offset:256 th:TH_ATOMIC_RETURN
 ; GFX12-NEXT:    ; implicit-def: $vgpr1_vgpr2_vgpr3_vgpr4
 ; GFX12-NEXT:    ; implicit-def: $vgpr5_vgpr6
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_xor_b32 exec_lo, exec_lo, s1
 ; GFX12-NEXT:    s_cbranch_execnz .LBB8_1
 ; GFX12-NEXT:  ; %bb.2:
 ; GFX12-NEXT:    s_mov_b32 exec_lo, s2
 ; GFX12-NEXT:    s_wait_loadcnt 0x0
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
   %voffset.add = add i32 %voffset, 256
   %ret = call float @llvm.amdgcn.struct.ptr.buffer.atomic.fmax.f32(float %val, ptr addrspace(8) %rsrc, i32 %vindex, i32 %voffset.add, i32 %soffset, i32 0)
@@ -613,15 +611,14 @@ define float @struct_ptr_buffer_atomic_add_f32_ret__vgpr_val__sgpr_rsrc__vgpr_vo
 ; GFX12-NEXT:    v_readfirstlane_b32 s6, v3
 ; GFX12-NEXT:    v_readfirstlane_b32 s7, v4
 ; GFX12-NEXT:    v_readfirstlane_b32 s3, v7
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-NEXT:    v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[1:2]
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
 ; GFX12-NEXT:    v_cmp_eq_u64_e64 s0, s[6:7], v[3:4]
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX12-NEXT:    v_cmp_eq_u32_e64 s1, s3, v7
-; GFX12-NEXT:    s_wait_alu 0xfffe
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    s_and_b32 s0, vcc_lo, s0
 ; GFX12-NEXT:    s_wait_alu 0xfffe
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-NEXT:    s_and_b32 s0, s0, s1
 ; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_and_saveexec_b32 s0, s0
@@ -630,13 +627,11 @@ define float @struct_ptr_buffer_atomic_add_f32_ret__vgpr_val__sgpr_rsrc__vgpr_vo
 ; GFX12-NEXT:    ; implicit-def: $vgpr1_vgpr2_vgpr3_vgpr4
 ; GFX12-NEXT:    ; implicit-def: $vgpr7
 ; GFX12-NEXT:    ; implicit-def: $vgpr5_vgpr6
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_xor_b32 exec_lo, exec_lo, s0
 ; GFX12-NEXT:    s_cbranch_execnz .LBB9_1
 ; GFX12-NEXT:  ; %bb.2:
 ; GFX12-NEXT:    s_mov_b32 exec_lo, s2
 ; GFX12-NEXT:    s_wait_loadcnt 0x0
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
   %voffset.add = add i32 %voffset, 256
   %ret = call float @llvm.amdgcn.struct.ptr.buffer.atomic.fmax.f32(float %val, ptr addrspace(8) %rsrc, i32 %vindex, i32 %voffset.add, i32 %soffset, i32 0)
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.atomic.fmin.f32.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.atomic.fmin.f32.ll
index aa41ef024d6e0..e75dd7409d51b 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.atomic.fmin.f32.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.atomic.fmin.f32.ll
@@ -455,10 +455,10 @@ define float @struct_ptr_buffer_atomic_fmin_f32_ret__vgpr_val__vgpr_rsrc__vgpr_v
 ; GFX12-NEXT:    v_readfirstlane_b32 s5, v2
 ; GFX12-NEXT:    v_readfirstlane_b32 s6, v3
 ; GFX12-NEXT:    v_readfirstlane_b32 s7, v4
+; GFX12-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
 ; GFX12-NEXT:    v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[1:2]
 ; GFX12-NEXT:    v_cmp_eq_u64_e64 s1, s[6:7], v[3:4]
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-NEXT:    s_and_b32 s1, vcc_lo, s1
 ; GFX12-NEXT:    s_wait_alu 0xfffe
@@ -467,13 +467,11 @@ define float @struct_ptr_buffer_atomic_fmin_f32_ret__vgpr_val__vgpr_rsrc__vgpr_v
 ; GFX12-NEXT:    buffer_atomic_min_num_f32 v0, v[5:6], s[4:7], s0 idxen offen offset:256 th:TH_ATOMIC_RETURN
 ; GFX12-NEXT:    ; implicit-def: $vgpr1_vgpr2_vgpr3_vgpr4
 ; GFX12-NEXT:    ; implicit-def: $vgpr5_vgpr6
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_xor_b32 exec_lo, exec_lo, s1
 ; GFX12-NEXT:    s_cbranch_execnz .LBB8_1
 ; GFX12-NEXT:  ; %bb.2:
 ; GFX12-NEXT:    s_mov_b32 exec_lo, s2
 ; GFX12-NEXT:    s_wait_loadcnt 0x0
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
   %voffset.add = add i32 %voffset, 256
   %ret = call float @llvm.amdgcn.struct.ptr.buffer.atomic.fmin.f32(float %val, ptr addrspace(8) %rsrc, i32 %vindex, i32 %voffset.add, i32 %soffset, i32 0)
@@ -613,15 +611,14 @@ define float @struct_ptr_buffer_atomic_fmin_f32_ret__vgpr_val__sgpr_rsrc__vgpr_v
 ; GFX12-NEXT:    v_readfirstlane_b32 s6, v3
 ; GFX12-NEXT:    v_readfirstlane_b32 s7, v4
 ; GFX12-NEXT:    v_readfirstlane_b32 s3, v7
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-NEXT:    v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[1:2]
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
 ; GFX12-NEXT:    v_cmp_eq_u64_e64 s0, s[6:7], v[3:4]
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX12-NEXT:    v_cmp_eq_u32_e64 s1, s3, v7
-; GFX12-NEXT:    s_wait_alu 0xfffe
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    s_and_b32 s0, vcc_lo, s0
 ; GFX12-NEXT:    s_wait_alu 0xfffe
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-NEXT:    s_and_b32 s0, s0, s1
 ; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_and_saveexec_b32 s0, s0
@@ -630,13 +627,11 @@ define float @struct_ptr_buffer_atomic_fmin_f32_ret__vgpr_val__sgpr_rsrc__vgpr_v
 ; GFX12-NEXT:    ; implicit-def: $vgpr1_vgpr2_vgpr3_vgpr4
 ; GFX12-NEXT:    ; implicit-def: $vgpr7
 ; GFX12-NEXT:    ; implicit-def: $vgpr5_vgpr6
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_xor_b32 exec_lo, exec_lo, s0
 ; GFX12-NEXT:    s_cbranch_execnz .LBB9_1
 ; GFX12-NEXT:  ; %bb.2:
 ; GFX12-NEXT:    s_mov_b32 exec_lo, s2
 ; GFX12-NEXT:    s_wait_loadcnt 0x0
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
   %voffset.add = add i32 %voffset, 256
   %ret = call float @llvm.amdgcn.struct.ptr.buffer.atomic.fmin.f32(float %val, ptr addrspace(8) %rsrc, i32 %vindex, i32 %voffset.add, i32 %soffset, i32 0)
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.update.dpp.gfx90a.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.update.dpp.gfx90a.ll
index 0ca96d5a1eb19..fa32ee108d382 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.update.dpp.gfx90a.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.update.dpp.gfx90a.ll
@@ -1,8 +1,8 @@
 ; RUN: llc -global-isel=0 -mtriple=amdgcn -mcpu=gfx90a < %s | FileCheck --check-prefixes=GCN,GFX90A %s
-; RUN: llc -global-isel=1 -mtriple=amdgcn -mcpu=gfx90a < %s | FileCheck --check-prefixes=GCN,GFX90A %s
+; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgcn -mcpu=gfx90a < %s | FileCheck --check-prefixes=GCN,GFX90A %s
 ; RUN: llc -global-isel=0 -mtriple=amdgcn -mcpu=gfx942 < %s | FileCheck --check-prefixes=GCN,GFX942 %s
-; RUN: llc -global-isel=1 -mtriple=amdgcn -mcpu=gfx942 < %s | FileCheck --check-prefixes=GCN,GFX942 %s
-; RUN: llc -global-isel=1 -mtriple=amdgcn -mcpu=gfx9-4-generic --amdhsa-code-object-version=6 < %s | FileCheck --check-prefixes=GCN,GFX942 %s
+; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgcn -mcpu=gfx942 < %s | FileCheck --check-prefixes=GCN,GFX942 %s
+; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgcn -mcpu=gfx9-4-generic --amdhsa-code-object-version=6 < %s | FileCheck --check-prefixes=GCN,GFX942 %s
 
 ; DPP control value 337 is valid for 64-bit DPP on gfx942
 
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.writelane.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.writelane.ll
index cbc76a32a75e4..7342c366799e9 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.writelane.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.writelane.ll
@@ -3,9 +3,9 @@
 ; RUN: llc -mtriple=amdgcn--amdhsa -mcpu=gfx1010 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX1010-SDAG %s
 ; RUN: llc -mtriple=amdgcn--amdhsa -mcpu=gfx1100 -verify-machineinstrs -amdgpu-enable-vopd=0 < %s | FileCheck -check-prefixes=GFX1100-SDAG %s
 
-; RUN: llc -mtriple=amdgcn--amdhsa -mcpu=gfx802 -verify-machineinstrs -global-isel < %s | FileCheck -check-prefixes=GFX802-GISEL %s
-; RUN: llc -mtriple=amdgcn--amdhsa -mcpu=gfx1010 -verify-machineinstrs -global-isel < %s | FileCheck -check-prefixes=GFX1010-GISEL %s
-; RUN: llc -mtriple=amdgcn--amdhsa -mcpu=gfx1100 -verify-machineinstrs -amdgpu-enable-vopd=0 -global-isel < %s | FileCheck -check-prefixes=GFX1100-GISEL %s
+; RUN: llc -mtriple=amdgcn--amdhsa -mcpu=gfx802 -verify-machineinstrs -global-isel -global-isel-abort=2 < %s | FileCheck -check-prefixes=GFX802-GISEL %s
+; RUN: llc -mtriple=amdgcn--amdhsa -mcpu=gfx1010 -verify-machineinstrs -global-isel -global-isel-abort=2 < %s | FileCheck -check-prefixes=GFX1010-GISEL %s
+; RUN: llc -mtriple=amdgcn--amdhsa -mcpu=gfx1100 -verify-machineinstrs -amdgpu-enable-vopd=0 -global-isel -global-isel-abort=2 < %s | FileCheck -check-prefixes=GFX1100-GISEL %s
 
 declare i32 @llvm.amdgcn.writelane(i32, i32, i32) #0
 declare i64 @llvm.amdgcn.writelane.i64(i64, i32, i64) #0
@@ -2128,10 +2128,10 @@ define void @test_writelane_bfloat(ptr addrspace(1) %out, bfloat %src, i32 %src1
 ; GFX802-GISEL:       ; %bb.0:
 ; GFX802-GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX802-GISEL-NEXT:    flat_load_ushort v4, v[0:1]
-; GFX802-GISEL-NEXT:    v_readfirstlane_b32 s5, v3
+; GFX802-GISEL-NEXT:    v_readfirstlane_b32 m0, v3
 ; GFX802-GISEL-NEXT:    v_readfirstlane_b32 s4, v2
-; GFX802-GISEL-NEXT:    s_mov_b32 m0, s5
 ; GFX802-GISEL-NEXT:    s_waitcnt vmcnt(0)
+; GFX802-GISEL-NEXT:    s_nop 1
 ; GFX802-GISEL-NEXT:    v_writelane_b32 v4, s4, m0
 ; GFX802-GISEL-NEXT:    flat_store_short v[0:1], v4
 ; GFX802-GISEL-NEXT:    s_waitcnt vmcnt(0)
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.mulo.ll b/llvm/test/CodeGen/AMDGPU/llvm.mulo.ll
index d46622ef45f43..8f7456b788f81 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.mulo.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.mulo.ll
@@ -32,13 +32,11 @@ define { i64, i1 } @umulo_i64_v_v(i64 %x, i64 %y) {
 ; GFX9-LABEL: umulo_i64_v_v:
 ; GFX9:       ; %bb.0: ; %bb
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_mov_b32_e32 v5, v0
 ; GFX9-NEXT:    v_mov_b32_e32 v4, v1
-; GFX9-NEXT:    v_mad_u64_u32 v[0:1], s[4:5], v5, v2, 0
-; GFX9-NEXT:    v_mad_u64_u32 v[5:6], s[4:5], v5, v3, 0
+; GFX9-NEXT:    v_mad_u64_u32 v[5:6], s[4:5], v0, v3, 0
+; GFX9-NEXT:    v_mad_u64_u32 v[0:1], s[4:5], v0, v2, 0
 ; GFX9-NEXT:    v_mad_u64_u32 v[7:8], s[4:5], v4, v2, 0
-; GFX9-NEXT:    v_mov_b32_e32 v2, v1
-; GFX9-NEXT:    v_add_co_u32_e32 v9, vcc, v2, v5
+; GFX9-NEXT:    v_add_co_u32_e32 v9, vcc, v1, v5
 ; GFX9-NEXT:    v_mad_u64_u32 v[2:3], s[4:5], v4, v3, 0
 ; GFX9-NEXT:    v_addc_co_u32_e32 v6, vcc, 0, v6, vcc
 ; GFX9-NEXT:    v_add_co_u32_e32 v4, vcc, v9, v7
@@ -54,18 +52,16 @@ define { i64, i1 } @umulo_i64_v_v(i64 %x, i64 %y) {
 ; GFX10-LABEL: umulo_i64_v_v:
 ; GFX10:       ; %bb.0: ; %bb
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT:    v_mov_b32_e32 v4, v0
-; GFX10-NEXT:    v_mov_b32_e32 v5, v1
-; GFX10-NEXT:    v_mad_u64_u32 v[0:1], s4, v4, v2, 0
-; GFX10-NEXT:    v_mad_u64_u32 v[6:7], s4, v4, v3, 0
-; GFX10-NEXT:    v_mad_u64_u32 v[8:9], s4, v5, v2, 0
-; GFX10-NEXT:    v_mad_u64_u32 v[2:3], s4, v5, v3, 0
 ; GFX10-NEXT:    v_mov_b32_e32 v4, v1
-; GFX10-NEXT:    v_add3_u32 v1, v1, v6, v8
-; GFX10-NEXT:    v_add_co_u32 v4, vcc_lo, v4, v6
-; GFX10-NEXT:    v_add_co_ci_u32_e32 v5, vcc_lo, 0, v7, vcc_lo
-; GFX10-NEXT:    v_add_co_u32 v4, vcc_lo, v4, v8
-; GFX10-NEXT:    v_add_co_ci_u32_e32 v4, vcc_lo, v5, v9, vcc_lo
+; GFX10-NEXT:    v_mad_u64_u32 v[5:6], s4, v0, v3, 0
+; GFX10-NEXT:    v_mad_u64_u32 v[0:1], s4, v0, v2, 0
+; GFX10-NEXT:    v_mad_u64_u32 v[7:8], s4, v4, v2, 0
+; GFX10-NEXT:    v_mad_u64_u32 v[2:3], s4, v4, v3, 0
+; GFX10-NEXT:    v_add_co_u32 v4, vcc_lo, v1, v5
+; GFX10-NEXT:    v_add_co_ci_u32_e32 v6, vcc_lo, 0, v6, vcc_lo
+; GFX10-NEXT:    v_add3_u32 v1, v1, v5, v7
+; GFX10-NEXT:    v_add_co_u32 v4, vcc_lo, v4, v7
+; GFX10-NEXT:    v_add_co_ci_u32_e32 v4, vcc_lo, v6, v8, vcc_lo
 ; GFX10-NEXT:    v_add_co_ci_u32_e32 v3, vcc_lo, 0, v3, vcc_lo
 ; GFX10-NEXT:    v_add_co_u32 v2, vcc_lo, v4, v2
 ; GFX10-NEXT:    v_add_co_ci_u32_e32 v3, vcc_lo, 0, v3, vcc_lo
@@ -78,23 +74,22 @@ define { i64, i1 } @umulo_i64_v_v(i64 %x, i64 %y) {
 ; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX11-NEXT:    v_dual_mov_b32 v4, v0 :: v_dual_mov_b32 v5, v1
 ; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-NEXT:    v_mad_u64_u32 v[0:1], null, v4, v2, 0
 ; GFX11-NEXT:    v_mad_u64_u32 v[6:7], null, v4, v3, 0
+; GFX11-NEXT:    v_mad_u64_u32 v[0:1], null, v4, v2, 0
 ; GFX11-NEXT:    v_mad_u64_u32 v[8:9], null, v5, v2, 0
 ; GFX11-NEXT:    v_mad_u64_u32 v[10:11], null, v5, v3, 0
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-NEXT:    v_mov_b32_e32 v4, v1
-; GFX11-NEXT:    v_add3_u32 v1, v1, v6, v8
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-NEXT:    v_add_co_u32 v2, vcc_lo, v4, v6
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-NEXT:    v_add_co_u32 v2, vcc_lo, v1, v6
 ; GFX11-NEXT:    v_add_co_ci_u32_e32 v3, vcc_lo, 0, v7, vcc_lo
+; GFX11-NEXT:    v_add3_u32 v1, v1, v6, v8
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
 ; GFX11-NEXT:    v_add_co_u32 v2, vcc_lo, v2, v8
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX11-NEXT:    v_add_co_ci_u32_e32 v2, vcc_lo, v3, v9, vcc_lo
 ; GFX11-NEXT:    v_add_co_ci_u32_e32 v3, vcc_lo, 0, v11, vcc_lo
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
 ; GFX11-NEXT:    v_add_co_u32 v2, vcc_lo, v2, v10
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX11-NEXT:    v_add_co_ci_u32_e32 v3, vcc_lo, 0, v3, vcc_lo
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX11-NEXT:    v_cmp_ne_u64_e32 vcc_lo, 0, v[2:3]
 ; GFX11-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc_lo
 ; GFX11-NEXT:    s_setpc_b64 s[30:31]
@@ -106,26 +101,29 @@ define { i64, i1 } @umulo_i64_v_v(i64 %x, i64 %y) {
 ; GFX12-NEXT:    s_wait_samplecnt 0x0
 ; GFX12-NEXT:    s_wait_bvhcnt 0x0
 ; GFX12-NEXT:    s_wait_kmcnt 0x0
-; GFX12-NEXT:    v_dual_mov_b32 v4, v0 :: v_dual_mov_b32 v5, v1
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX12-NEXT:    v_mad_co_u64_u32 v[0:1], null, v4, v2, 0
-; GFX12-NEXT:    v_mad_co_u64_u32 v[6:7], null, v4, v3, 0
-; GFX12-NEXT:    v_mad_co_u64_u32 v[8:9], null, v5, v2, 0
-; GFX12-NEXT:    v_mad_co_u64_u32 v[2:3], null, v5, v3, 0
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
 ; GFX12-NEXT:    v_mov_b32_e32 v4, v1
-; GFX12-NEXT:    v_add3_u32 v1, v1, v6, v8
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-NEXT:    v_add_co_u32 v4, vcc_lo, v4, v6
-; GFX12-NEXT:    v_add_co_ci_u32_e32 v5, vcc_lo, 0, v7, vcc_lo
-; GFX12-NEXT:    v_add_co_u32 v4, vcc_lo, v4, v8
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-NEXT:    v_add_co_ci_u32_e32 v4, vcc_lo, v5, v9, vcc_lo
+; GFX12-NEXT:    v_mad_co_u64_u32 v[5:6], null, v0, v3, 0
+; GFX12-NEXT:    v_mad_co_u64_u32 v[0:1], null, v0, v2, 0
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX12-NEXT:    v_mad_co_u64_u32 v[7:8], null, v4, v2, 0
+; GFX12-NEXT:    v_mad_co_u64_u32 v[2:3], null, v4, v3, 0
+; GFX12-NEXT:    v_add_co_u32 v4, vcc_lo, v1, v5
+; GFX12-NEXT:    s_wait_alu 0xfffd
+; GFX12-NEXT:    v_add_co_ci_u32_e32 v6, vcc_lo, 0, v6, vcc_lo
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-NEXT:    v_add3_u32 v1, v1, v5, v7
+; GFX12-NEXT:    v_add_co_u32 v4, vcc_lo, v4, v7
+; GFX12-NEXT:    s_wait_alu 0xfffd
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_2)
+; GFX12-NEXT:    v_add_co_ci_u32_e32 v4, vcc_lo, v6, v8, vcc_lo
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_add_co_ci_u32_e32 v3, vcc_lo, 0, v3, vcc_lo
 ; GFX12-NEXT:    v_add_co_u32 v2, vcc_lo, v4, v2
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_add_co_ci_u32_e32 v3, vcc_lo, 0, v3, vcc_lo
 ; GFX12-NEXT:    v_cmp_ne_u64_e32 vcc_lo, 0, v[2:3]
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc_lo
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 bb:
@@ -176,11 +174,10 @@ define { i64, i1 } @smulo_i64_v_v(i64 %x, i64 %y) {
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX9-NEXT:    v_mov_b32_e32 v5, v0
 ; GFX9-NEXT:    v_mov_b32_e32 v4, v1
-; GFX9-NEXT:    v_mad_u64_u32 v[0:1], s[4:5], v5, v2, 0
 ; GFX9-NEXT:    v_mad_u64_u32 v[6:7], s[4:5], v5, v3, 0
+; GFX9-NEXT:    v_mad_u64_u32 v[0:1], s[4:5], v5, v2, 0
 ; GFX9-NEXT:    v_mad_u64_u32 v[8:9], s[4:5], v4, v2, 0
-; GFX9-NEXT:    v_mov_b32_e32 v10, v1
-; GFX9-NEXT:    v_add_co_u32_e32 v12, vcc, v10, v6
+; GFX9-NEXT:    v_add_co_u32_e32 v12, vcc, v1, v6
 ; GFX9-NEXT:    v_mad_i64_i32 v[10:11], s[4:5], v4, v3, 0
 ; GFX9-NEXT:    v_addc_co_u32_e32 v7, vcc, 0, v7, vcc
 ; GFX9-NEXT:    v_add_co_u32_e32 v12, vcc, v12, v8
@@ -210,14 +207,13 @@ define { i64, i1 } @smulo_i64_v_v(i64 %x, i64 %y) {
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    v_mov_b32_e32 v4, v0
 ; GFX10-NEXT:    v_mov_b32_e32 v5, v1
-; GFX10-NEXT:    v_mad_u64_u32 v[0:1], s4, v4, v2, 0
 ; GFX10-NEXT:    v_mad_u64_u32 v[6:7], s4, v4, v3, 0
+; GFX10-NEXT:    v_mad_u64_u32 v[0:1], s4, v4, v2, 0
 ; GFX10-NEXT:    v_mad_u64_u32 v[8:9], s4, v5, v2, 0
 ; GFX10-NEXT:    v_mad_i64_i32 v[10:11], s4, v5, v3, 0
-; GFX10-NEXT:    v_mov_b32_e32 v12, v1
-; GFX10-NEXT:    v_add3_u32 v1, v1, v6, v8
-; GFX10-NEXT:    v_add_co_u32 v12, vcc_lo, v12, v6
+; GFX10-NEXT:    v_add_co_u32 v12, vcc_lo, v1, v6
 ; GFX10-NEXT:    v_add_co_ci_u32_e32 v7, vcc_lo, 0, v7, vcc_lo
+; GFX10-NEXT:    v_add3_u32 v1, v1, v6, v8
 ; GFX10-NEXT:    v_add_co_u32 v12, vcc_lo, v12, v8
 ; GFX10-NEXT:    v_add_co_ci_u32_e32 v7, vcc_lo, v7, v9, vcc_lo
 ; GFX10-NEXT:    v_add_co_ci_u32_e32 v9, vcc_lo, 0, v11, vcc_lo
@@ -244,38 +240,37 @@ define { i64, i1 } @smulo_i64_v_v(i64 %x, i64 %y) {
 ; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX11-NEXT:    v_dual_mov_b32 v4, v0 :: v_dual_mov_b32 v5, v1
 ; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-NEXT:    v_mad_u64_u32 v[0:1], null, v4, v2, 0
 ; GFX11-NEXT:    v_mad_u64_u32 v[6:7], null, v4, v3, 0
+; GFX11-NEXT:    v_mad_u64_u32 v[0:1], null, v4, v2, 0
 ; GFX11-NEXT:    v_mad_u64_u32 v[8:9], null, v5, v2, 0
 ; GFX11-NEXT:    v_mad_i64_i32 v[10:11], null, v5, v3, 0
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-NEXT:    v_mov_b32_e32 v12, v1
-; GFX11-NEXT:    v_add3_u32 v1, v1, v6, v8
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-NEXT:    v_add_co_u32 v12, vcc_lo, v12, v6
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-NEXT:    v_add_co_u32 v12, vcc_lo, v1, v6
 ; GFX11-NEXT:    v_add_co_ci_u32_e32 v7, vcc_lo, 0, v7, vcc_lo
+; GFX11-NEXT:    v_add3_u32 v1, v1, v6, v8
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
 ; GFX11-NEXT:    v_add_co_u32 v12, vcc_lo, v12, v8
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX11-NEXT:    v_add_co_ci_u32_e32 v7, vcc_lo, v7, v9, vcc_lo
 ; GFX11-NEXT:    v_add_co_ci_u32_e32 v9, vcc_lo, 0, v11, vcc_lo
-; GFX11-NEXT:    v_add_co_u32 v7, vcc_lo, v7, v10
 ; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-NEXT:    v_add_co_u32 v7, vcc_lo, v7, v10
 ; GFX11-NEXT:    v_add_co_ci_u32_e32 v9, vcc_lo, 0, v9, vcc_lo
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
 ; GFX11-NEXT:    v_sub_co_u32 v2, vcc_lo, v7, v2
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
 ; GFX11-NEXT:    v_subrev_co_ci_u32_e32 v10, vcc_lo, 0, v9, vcc_lo
 ; GFX11-NEXT:    v_cmp_gt_i32_e32 vcc_lo, 0, v5
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
 ; GFX11-NEXT:    v_cndmask_b32_e32 v6, v7, v2, vcc_lo
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
 ; GFX11-NEXT:    v_cndmask_b32_e32 v5, v9, v10, vcc_lo
 ; GFX11-NEXT:    v_ashrrev_i32_e32 v2, 31, v1
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
 ; GFX11-NEXT:    v_sub_co_u32 v4, vcc_lo, v6, v4
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
 ; GFX11-NEXT:    v_subrev_co_ci_u32_e32 v7, vcc_lo, 0, v5, vcc_lo
 ; GFX11-NEXT:    v_cmp_gt_i32_e32 vcc_lo, 0, v3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
 ; GFX11-NEXT:    v_mov_b32_e32 v3, v2
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX11-NEXT:    v_dual_cndmask_b32 v4, v6, v4 :: v_dual_cndmask_b32 v5, v5, v7
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX11-NEXT:    v_cmp_ne_u64_e32 vcc_lo, v[4:5], v[2:3]
 ; GFX11-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc_lo
 ; GFX11-NEXT:    s_setpc_b64 s[30:31]
@@ -289,39 +284,46 @@ define { i64, i1 } @smulo_i64_v_v(i64 %x, i64 %y) {
 ; GFX12-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-NEXT:    v_dual_mov_b32 v4, v0 :: v_dual_mov_b32 v5, v1
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX12-NEXT:    v_mad_co_u64_u32 v[0:1], null, v4, v2, 0
 ; GFX12-NEXT:    v_mad_co_u64_u32 v[6:7], null, v4, v3, 0
+; GFX12-NEXT:    v_mad_co_u64_u32 v[0:1], null, v4, v2, 0
 ; GFX12-NEXT:    v_mad_co_u64_u32 v[8:9], null, v5, v2, 0
 ; GFX12-NEXT:    v_mad_co_i64_i32 v[10:11], null, v5, v3, 0
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX12-NEXT:    v_mov_b32_e32 v12, v1
-; GFX12-NEXT:    v_add3_u32 v1, v1, v6, v8
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-NEXT:    v_add_co_u32 v12, vcc_lo, v12, v6
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; GFX12-NEXT:    v_add_co_u32 v12, vcc_lo, v1, v6
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_add_co_ci_u32_e32 v7, vcc_lo, 0, v7, vcc_lo
+; GFX12-NEXT:    v_add3_u32 v1, v1, v6, v8
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
 ; GFX12-NEXT:    v_add_co_u32 v12, vcc_lo, v12, v8
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_add_co_ci_u32_e32 v7, vcc_lo, v7, v9, vcc_lo
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_add_co_ci_u32_e32 v9, vcc_lo, 0, v11, vcc_lo
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX12-NEXT:    v_add_co_u32 v7, vcc_lo, v7, v10
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_add_co_ci_u32_e32 v9, vcc_lo, 0, v9, vcc_lo
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX12-NEXT:    v_sub_co_u32 v2, vcc_lo, v7, v2
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_subrev_co_ci_u32_e32 v10, vcc_lo, 0, v9, vcc_lo
 ; GFX12-NEXT:    v_cmp_gt_i32_e32 vcc_lo, 0, v5
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_cndmask_b32_e32 v6, v7, v2, vcc_lo
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
 ; GFX12-NEXT:    v_cndmask_b32_e32 v5, v9, v10, vcc_lo
 ; GFX12-NEXT:    v_ashrrev_i32_e32 v2, 31, v1
 ; GFX12-NEXT:    v_sub_co_u32 v4, vcc_lo, v6, v4
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX12-NEXT:    s_wait_alu 0xfffd
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_3)
 ; GFX12-NEXT:    v_subrev_co_ci_u32_e32 v7, vcc_lo, 0, v5, vcc_lo
 ; GFX12-NEXT:    v_cmp_gt_i32_e32 vcc_lo, 0, v3
 ; GFX12-NEXT:    v_mov_b32_e32 v3, v2
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_dual_cndmask_b32 v4, v6, v4 :: v_dual_cndmask_b32 v5, v5, v7
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-NEXT:    v_cmp_ne_u64_e32 vcc_lo, v[4:5], v[2:3]
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc_lo
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 bb:
@@ -771,6 +773,7 @@ define { i64, i1 } @smulo_i64_v_4(i64 %i) {
 ; GFX12-NEXT:    v_cmp_ne_u64_e32 vcc_lo, v[5:6], v[0:1]
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3)
 ; GFX12-NEXT:    v_dual_mov_b32 v0, v4 :: v_dual_mov_b32 v1, v3
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc_lo
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 bb:
@@ -843,6 +846,7 @@ define { i64, i1 } @umulo_i64_v_4(i64 %i) {
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
 ; GFX12-NEXT:    v_cmp_ne_u64_e32 vcc_lo, v[6:7], v[0:1]
 ; GFX12-NEXT:    v_dual_mov_b32 v0, v4 :: v_dual_mov_b32 v1, v3
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc_lo
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 bb:
diff --git a/llvm/test/CodeGen/AMDGPU/load-constant-i1.ll b/llvm/test/CodeGen/AMDGPU/load-constant-i1.ll
index 223870950e4b7..1fc7349882ba1 100644
--- a/llvm/test/CodeGen/AMDGPU/load-constant-i1.ll
+++ b/llvm/test/CodeGen/AMDGPU/load-constant-i1.ll
@@ -6026,7 +6026,6 @@ define amdgpu_kernel void @constant_sextload_v16i1_to_v16i64(ptr addrspace(1) %o
 ; GFX12-NEXT:    s_bfe_i64 s[10:11], s[10:11], 0x10000
 ; GFX12-NEXT:    s_bfe_i64 s[8:9], s[8:9], 0x10000
 ; GFX12-NEXT:    s_bfe_i64 s[6:7], s[6:7], 0x10000
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_bfe_i64 s[2:3], s[2:3], 0x10000
 ; GFX12-NEXT:    s_bfe_i64 s[16:17], s[16:17], 0x10000
 ; GFX12-NEXT:    s_bfe_i64 s[14:15], s[14:15], 0x10000
@@ -8138,12 +8137,11 @@ define amdgpu_kernel void @constant_zextload_v64i1_to_v64i64(ptr addrspace(1) %o
 ; GFX12-NEXT:    s_load_b64 s[2:3], s[2:3], 0x0
 ; GFX12-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-NEXT:    s_bfe_u32 s4, s3, 0x10014
-; GFX12-NEXT:    s_wait_alu 0xfffe
+; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v0, s4
 ; GFX12-NEXT:    s_bfe_u32 s5, s3, 0x10015
 ; GFX12-NEXT:    s_lshr_b32 s4, s3, 31
 ; GFX12-NEXT:    s_wait_alu 0xfffe
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-NEXT:    v_dual_mov_b32 v2, s5 :: v_dual_mov_b32 v3, v1
 ; GFX12-NEXT:    s_bfe_u32 s5, s3, 0x1001e
 ; GFX12-NEXT:    global_store_b128 v1, v[0:3], s[0:1] offset:416
@@ -8231,8 +8229,8 @@ define amdgpu_kernel void @constant_zextload_v64i1_to_v64i64(ptr addrspace(1) %o
 ; GFX12-NEXT:    s_bfe_u32 s4, s3, 0x10001
 ; GFX12-NEXT:    s_and_b32 s3, s3, 1
 ; GFX12-NEXT:    global_store_b128 v1, v[0:3], s[0:1] offset:272
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    v_mov_b32_e32 v0, s3
+; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    v_mov_b32_e32 v2, s4
 ; GFX12-NEXT:    s_lshr_b32 s3, s2, 31
 ; GFX12-NEXT:    s_bfe_u32 s4, s2, 0x1001e
@@ -9467,53 +9465,48 @@ define amdgpu_kernel void @constant_sextload_v64i1_to_v64i64(ptr addrspace(1) %o
 ; GFX12:       ; %bb.0:
 ; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
 ; GFX12-NEXT:    s_wait_kmcnt 0x0
-; GFX12-NEXT:    s_load_b64 s[12:13], s[2:3], 0x0
+; GFX12-NEXT:    s_load_b64 s[10:11], s[2:3], 0x0
 ; GFX12-NEXT:    s_wait_kmcnt 0x0
-; GFX12-NEXT:    s_lshr_b32 s96, s13, 30
-; GFX12-NEXT:    s_lshr_b32 s98, s13, 31
-; GFX12-NEXT:    s_lshr_b32 s92, s13, 28
-; GFX12-NEXT:    s_lshr_b32 s94, s13, 29
-; GFX12-NEXT:    s_lshr_b32 s78, s13, 26
-; GFX12-NEXT:    s_lshr_b32 s88, s13, 27
-; GFX12-NEXT:    s_wait_alu 0xfffe
+; GFX12-NEXT:    s_lshr_b32 s96, s11, 30
+; GFX12-NEXT:    s_lshr_b32 s98, s11, 31
+; GFX12-NEXT:    s_lshr_b32 s92, s11, 28
+; GFX12-NEXT:    s_lshr_b32 s94, s11, 29
+; GFX12-NEXT:    s_lshr_b32 s78, s11, 26
+; GFX12-NEXT:    s_lshr_b32 s88, s11, 27
 ; GFX12-NEXT:    s_bfe_i64 s[96:97], s[96:97], 0x10000
 ; GFX12-NEXT:    s_bfe_i64 s[100:101], s[98:99], 0x10000
-; GFX12-NEXT:    s_lshr_b32 s66, s13, 24
-; GFX12-NEXT:    s_lshr_b32 s74, s13, 25
+; GFX12-NEXT:    s_lshr_b32 s66, s11, 24
+; GFX12-NEXT:    s_lshr_b32 s74, s11, 25
 ; GFX12-NEXT:    s_bfe_i64 s[92:93], s[92:93], 0x10000
 ; GFX12-NEXT:    s_bfe_i64 s[94:95], s[94:95], 0x10000
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s96
-; GFX12-NEXT:    s_lshr_b32 s56, s13, 22
-; GFX12-NEXT:    s_lshr_b32 s62, s13, 23
+; GFX12-NEXT:    s_lshr_b32 s56, s11, 22
+; GFX12-NEXT:    s_lshr_b32 s62, s11, 23
 ; GFX12-NEXT:    v_dual_mov_b32 v2, s97 :: v_dual_mov_b32 v3, s100
 ; GFX12-NEXT:    v_dual_mov_b32 v4, s101 :: v_dual_mov_b32 v5, s92
 ; GFX12-NEXT:    s_bfe_i64 s[78:79], s[78:79], 0x10000
 ; GFX12-NEXT:    s_bfe_i64 s[88:89], s[88:89], 0x10000
-; GFX12-NEXT:    s_lshr_b32 s44, s13, 20
-; GFX12-NEXT:    s_lshr_b32 s52, s13, 21
-; GFX12-NEXT:    s_lshr_b32 s30, s13, 18
-; GFX12-NEXT:    s_lshr_b32 s40, s13, 19
-; GFX12-NEXT:    s_lshr_b32 s18, s13, 16
-; GFX12-NEXT:    s_lshr_b32 s26, s13, 17
-; GFX12-NEXT:    s_lshr_b32 s2, s13, 14
-; GFX12-NEXT:    s_lshr_b32 s4, s13, 15
+; GFX12-NEXT:    s_lshr_b32 s44, s11, 20
+; GFX12-NEXT:    s_lshr_b32 s52, s11, 21
+; GFX12-NEXT:    s_lshr_b32 s30, s11, 18
+; GFX12-NEXT:    s_lshr_b32 s40, s11, 19
+; GFX12-NEXT:    s_lshr_b32 s18, s11, 16
+; GFX12-NEXT:    s_lshr_b32 s26, s11, 17
+; GFX12-NEXT:    s_lshr_b32 s2, s11, 14
+; GFX12-NEXT:    s_lshr_b32 s4, s11, 15
 ; GFX12-NEXT:    v_dual_mov_b32 v6, s93 :: v_dual_mov_b32 v7, s94
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    v_dual_mov_b32 v8, s95 :: v_dual_mov_b32 v9, s78
 ; GFX12-NEXT:    s_bfe_i64 s[66:67], s[66:67], 0x10000
 ; GFX12-NEXT:    s_bfe_i64 s[74:75], s[74:75], 0x10000
-; GFX12-NEXT:    s_lshr_b32 s6, s13, 12
-; GFX12-NEXT:    s_lshr_b32 s8, s13, 13
+; GFX12-NEXT:    s_lshr_b32 s6, s11, 12
+; GFX12-NEXT:    s_lshr_b32 s8, s11, 13
 ; GFX12-NEXT:    v_dual_mov_b32 v10, s79 :: v_dual_mov_b32 v11, s88
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    v_dual_mov_b32 v12, s89 :: v_dual_mov_b32 v13, s66
 ; GFX12-NEXT:    s_bfe_i64 s[56:57], s[56:57], 0x10000
 ; GFX12-NEXT:    s_bfe_i64 s[62:63], s[62:63], 0x10000
-; GFX12-NEXT:    s_lshr_b32 s10, s13, 10
-; GFX12-NEXT:    s_lshr_b32 s14, s13, 11
+; GFX12-NEXT:    s_lshr_b32 s12, s11, 10
+; GFX12-NEXT:    s_lshr_b32 s14, s11, 11
 ; GFX12-NEXT:    v_dual_mov_b32 v14, s67 :: v_dual_mov_b32 v15, s74
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    v_dual_mov_b32 v16, s75 :: v_dual_mov_b32 v17, s56
 ; GFX12-NEXT:    s_bfe_i64 s[44:45], s[44:45], 0x10000
 ; GFX12-NEXT:    s_bfe_i64 s[52:53], s[52:53], 0x10000
@@ -9523,15 +9516,14 @@ define amdgpu_kernel void @constant_sextload_v64i1_to_v64i64(ptr addrspace(1) %o
 ; GFX12-NEXT:    s_bfe_i64 s[26:27], s[26:27], 0x10000
 ; GFX12-NEXT:    s_bfe_i64 s[4:5], s[4:5], 0x10000
 ; GFX12-NEXT:    s_bfe_i64 s[2:3], s[2:3], 0x10000
-; GFX12-NEXT:    s_lshr_b32 s16, s13, 8
-; GFX12-NEXT:    s_lshr_b32 s20, s13, 9
+; GFX12-NEXT:    s_lshr_b32 s16, s11, 8
+; GFX12-NEXT:    s_lshr_b32 s20, s11, 9
 ; GFX12-NEXT:    v_dual_mov_b32 v18, s57 :: v_dual_mov_b32 v19, s62
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    v_dual_mov_b32 v20, s63 :: v_dual_mov_b32 v21, s44
 ; GFX12-NEXT:    s_bfe_i64 s[8:9], s[8:9], 0x10000
 ; GFX12-NEXT:    s_bfe_i64 s[6:7], s[6:7], 0x10000
-; GFX12-NEXT:    s_lshr_b32 s22, s13, 6
-; GFX12-NEXT:    s_lshr_b32 s24, s13, 7
+; GFX12-NEXT:    s_lshr_b32 s22, s11, 6
+; GFX12-NEXT:    s_lshr_b32 s24, s11, 7
 ; GFX12-NEXT:    v_dual_mov_b32 v22, s45 :: v_dual_mov_b32 v23, s52
 ; GFX12-NEXT:    v_dual_mov_b32 v24, s53 :: v_dual_mov_b32 v25, s30
 ; GFX12-NEXT:    v_dual_mov_b32 v26, s31 :: v_dual_mov_b32 v27, s40
@@ -9539,7 +9531,7 @@ define amdgpu_kernel void @constant_sextload_v64i1_to_v64i64(ptr addrspace(1) %o
 ; GFX12-NEXT:    v_dual_mov_b32 v30, s19 :: v_dual_mov_b32 v31, s26
 ; GFX12-NEXT:    v_mov_b32_e32 v32, s27
 ; GFX12-NEXT:    s_bfe_i64 s[14:15], s[14:15], 0x10000
-; GFX12-NEXT:    s_bfe_i64 s[10:11], s[10:11], 0x10000
+; GFX12-NEXT:    s_bfe_i64 s[12:13], s[12:13], 0x10000
 ; GFX12-NEXT:    s_clause 0x7
 ; GFX12-NEXT:    global_store_b128 v0, v[1:4], s[0:1] offset:496
 ; GFX12-NEXT:    global_store_b128 v0, v[5:8], s[0:1] offset:480
@@ -9552,36 +9544,36 @@ define amdgpu_kernel void @constant_sextload_v64i1_to_v64i64(ptr addrspace(1) %o
 ; GFX12-NEXT:    v_dual_mov_b32 v1, s2 :: v_dual_mov_b32 v2, s3
 ; GFX12-NEXT:    v_dual_mov_b32 v3, s4 :: v_dual_mov_b32 v4, s5
 ; GFX12-NEXT:    v_mov_b32_e32 v5, s6
-; GFX12-NEXT:    s_lshr_b32 s28, s13, 4
-; GFX12-NEXT:    s_lshr_b32 s34, s13, 5
-; GFX12-NEXT:    s_lshr_b32 s36, s13, 2
-; GFX12-NEXT:    s_lshr_b32 s38, s13, 3
+; GFX12-NEXT:    s_lshr_b32 s28, s11, 4
+; GFX12-NEXT:    s_lshr_b32 s34, s11, 5
+; GFX12-NEXT:    s_lshr_b32 s36, s11, 2
+; GFX12-NEXT:    s_lshr_b32 s38, s11, 3
 ; GFX12-NEXT:    s_bfe_i64 s[20:21], s[20:21], 0x10000
 ; GFX12-NEXT:    s_bfe_i64 s[16:17], s[16:17], 0x10000
 ; GFX12-NEXT:    v_dual_mov_b32 v6, s7 :: v_dual_mov_b32 v7, s8
-; GFX12-NEXT:    v_dual_mov_b32 v8, s9 :: v_dual_mov_b32 v9, s10
-; GFX12-NEXT:    s_lshr_b32 s42, s13, 1
-; GFX12-NEXT:    s_mov_b32 s46, s13
+; GFX12-NEXT:    v_dual_mov_b32 v8, s9 :: v_dual_mov_b32 v9, s12
+; GFX12-NEXT:    s_lshr_b32 s42, s11, 1
+; GFX12-NEXT:    s_mov_b32 s46, s11
 ; GFX12-NEXT:    s_bfe_i64 s[24:25], s[24:25], 0x10000
 ; GFX12-NEXT:    s_bfe_i64 s[22:23], s[22:23], 0x10000
-; GFX12-NEXT:    v_dual_mov_b32 v10, s11 :: v_dual_mov_b32 v11, s14
+; GFX12-NEXT:    v_dual_mov_b32 v10, s13 :: v_dual_mov_b32 v11, s14
 ; GFX12-NEXT:    v_dual_mov_b32 v12, s15 :: v_dual_mov_b32 v13, s16
-; GFX12-NEXT:    s_lshr_b32 s48, s12, 30
-; GFX12-NEXT:    s_lshr_b32 s50, s12, 31
+; GFX12-NEXT:    s_lshr_b32 s48, s10, 30
+; GFX12-NEXT:    s_lshr_b32 s50, s10, 31
 ; GFX12-NEXT:    s_bfe_i64 s[38:39], s[38:39], 0x10000
 ; GFX12-NEXT:    s_bfe_i64 s[36:37], s[36:37], 0x10000
 ; GFX12-NEXT:    s_bfe_i64 s[34:35], s[34:35], 0x10000
 ; GFX12-NEXT:    s_bfe_i64 s[28:29], s[28:29], 0x10000
 ; GFX12-NEXT:    v_dual_mov_b32 v14, s17 :: v_dual_mov_b32 v15, s20
 ; GFX12-NEXT:    v_dual_mov_b32 v16, s21 :: v_dual_mov_b32 v17, s22
-; GFX12-NEXT:    s_lshr_b32 s54, s12, 28
-; GFX12-NEXT:    s_lshr_b32 s58, s12, 29
+; GFX12-NEXT:    s_lshr_b32 s54, s10, 28
+; GFX12-NEXT:    s_lshr_b32 s58, s10, 29
 ; GFX12-NEXT:    s_bfe_i64 s[46:47], s[46:47], 0x10000
 ; GFX12-NEXT:    s_bfe_i64 s[42:43], s[42:43], 0x10000
 ; GFX12-NEXT:    v_dual_mov_b32 v18, s23 :: v_dual_mov_b32 v19, s24
 ; GFX12-NEXT:    v_dual_mov_b32 v20, s25 :: v_dual_mov_b32 v21, s28
-; GFX12-NEXT:    s_lshr_b32 s60, s12, 26
-; GFX12-NEXT:    s_lshr_b32 s64, s12, 27
+; GFX12-NEXT:    s_lshr_b32 s60, s10, 26
+; GFX12-NEXT:    s_lshr_b32 s64, s10, 27
 ; GFX12-NEXT:    s_bfe_i64 s[50:51], s[50:51], 0x10000
 ; GFX12-NEXT:    s_bfe_i64 s[48:49], s[48:49], 0x10000
 ; GFX12-NEXT:    v_dual_mov_b32 v22, s29 :: v_dual_mov_b32 v23, s34
@@ -9596,43 +9588,43 @@ define amdgpu_kernel void @constant_sextload_v64i1_to_v64i64(ptr addrspace(1) %o
 ; GFX12-NEXT:    v_dual_mov_b32 v1, s36 :: v_dual_mov_b32 v2, s37
 ; GFX12-NEXT:    v_dual_mov_b32 v3, s38 :: v_dual_mov_b32 v4, s39
 ; GFX12-NEXT:    v_mov_b32_e32 v5, s46
-; GFX12-NEXT:    s_lshr_b32 s68, s12, 24
-; GFX12-NEXT:    s_lshr_b32 s70, s12, 25
-; GFX12-NEXT:    s_lshr_b32 s72, s12, 22
-; GFX12-NEXT:    s_lshr_b32 s76, s12, 23
+; GFX12-NEXT:    s_lshr_b32 s68, s10, 24
+; GFX12-NEXT:    s_lshr_b32 s70, s10, 25
+; GFX12-NEXT:    s_lshr_b32 s72, s10, 22
+; GFX12-NEXT:    s_lshr_b32 s76, s10, 23
 ; GFX12-NEXT:    s_bfe_i64 s[58:59], s[58:59], 0x10000
 ; GFX12-NEXT:    s_bfe_i64 s[54:55], s[54:55], 0x10000
 ; GFX12-NEXT:    v_dual_mov_b32 v6, s47 :: v_dual_mov_b32 v7, s42
 ; GFX12-NEXT:    v_dual_mov_b32 v8, s43 :: v_dual_mov_b32 v9, s48
-; GFX12-NEXT:    s_lshr_b32 s80, s12, 20
-; GFX12-NEXT:    s_lshr_b32 s82, s12, 21
+; GFX12-NEXT:    s_lshr_b32 s80, s10, 20
+; GFX12-NEXT:    s_lshr_b32 s82, s10, 21
 ; GFX12-NEXT:    s_bfe_i64 s[64:65], s[64:65], 0x10000
 ; GFX12-NEXT:    s_bfe_i64 s[60:61], s[60:61], 0x10000
 ; GFX12-NEXT:    v_dual_mov_b32 v10, s49 :: v_dual_mov_b32 v11, s50
 ; GFX12-NEXT:    v_dual_mov_b32 v12, s51 :: v_dual_mov_b32 v13, s54
-; GFX12-NEXT:    s_lshr_b32 s84, s12, 18
-; GFX12-NEXT:    s_lshr_b32 s86, s12, 19
+; GFX12-NEXT:    s_lshr_b32 s84, s10, 18
+; GFX12-NEXT:    s_lshr_b32 s86, s10, 19
 ; GFX12-NEXT:    s_bfe_i64 s[76:77], s[76:77], 0x10000
 ; GFX12-NEXT:    s_bfe_i64 s[72:73], s[72:73], 0x10000
 ; GFX12-NEXT:    s_bfe_i64 s[70:71], s[70:71], 0x10000
 ; GFX12-NEXT:    s_bfe_i64 s[68:69], s[68:69], 0x10000
 ; GFX12-NEXT:    v_dual_mov_b32 v14, s55 :: v_dual_mov_b32 v15, s58
 ; GFX12-NEXT:    v_dual_mov_b32 v16, s59 :: v_dual_mov_b32 v17, s60
-; GFX12-NEXT:    s_lshr_b32 s90, s12, 16
-; GFX12-NEXT:    s_lshr_b32 s98, s12, 17
+; GFX12-NEXT:    s_lshr_b32 s90, s10, 16
+; GFX12-NEXT:    s_lshr_b32 s98, s10, 17
 ; GFX12-NEXT:    s_bfe_i64 s[82:83], s[82:83], 0x10000
 ; GFX12-NEXT:    s_bfe_i64 s[80:81], s[80:81], 0x10000
 ; GFX12-NEXT:    v_dual_mov_b32 v18, s61 :: v_dual_mov_b32 v19, s64
 ; GFX12-NEXT:    v_dual_mov_b32 v20, s65 :: v_dual_mov_b32 v21, s68
-; GFX12-NEXT:    s_lshr_b32 s96, s12, 14
-; GFX12-NEXT:    s_lshr_b32 s100, s12, 15
-; GFX12-NEXT:    s_lshr_b32 s94, s12, 13
-; GFX12-NEXT:    s_lshr_b32 s88, s12, 11
-; GFX12-NEXT:    s_lshr_b32 s74, s12, 9
-; GFX12-NEXT:    s_lshr_b32 s62, s12, 7
-; GFX12-NEXT:    s_lshr_b32 s52, s12, 5
-; GFX12-NEXT:    s_lshr_b32 s40, s12, 3
-; GFX12-NEXT:    s_lshr_b32 s26, s12, 1
+; GFX12-NEXT:    s_lshr_b32 s96, s10, 14
+; GFX12-NEXT:    s_lshr_b32 s100, s10, 15
+; GFX12-NEXT:    s_lshr_b32 s94, s10, 13
+; GFX12-NEXT:    s_lshr_b32 s88, s10, 11
+; GFX12-NEXT:    s_lshr_b32 s74, s10, 9
+; GFX12-NEXT:    s_lshr_b32 s62, s10, 7
+; GFX12-NEXT:    s_lshr_b32 s52, s10, 5
+; GFX12-NEXT:    s_lshr_b32 s40, s10, 3
+; GFX12-NEXT:    s_lshr_b32 s26, s10, 1
 ; GFX12-NEXT:    s_bfe_i64 s[86:87], s[86:87], 0x10000
 ; GFX12-NEXT:    s_bfe_i64 s[84:85], s[84:85], 0x10000
 ; GFX12-NEXT:    v_dual_mov_b32 v22, s69 :: v_dual_mov_b32 v23, s70
@@ -9647,18 +9639,19 @@ define amdgpu_kernel void @constant_sextload_v64i1_to_v64i64(ptr addrspace(1) %o
 ; GFX12-NEXT:    v_dual_mov_b32 v1, s72 :: v_dual_mov_b32 v2, s73
 ; GFX12-NEXT:    v_dual_mov_b32 v3, s76 :: v_dual_mov_b32 v4, s77
 ; GFX12-NEXT:    v_mov_b32_e32 v5, s80
-; GFX12-NEXT:    s_lshr_b32 s92, s12, 12
-; GFX12-NEXT:    s_lshr_b32 s78, s12, 10
+; GFX12-NEXT:    s_lshr_b32 s92, s10, 12
+; GFX12-NEXT:    s_lshr_b32 s78, s10, 10
 ; GFX12-NEXT:    s_bfe_i64 s[98:99], s[98:99], 0x10000
 ; GFX12-NEXT:    s_bfe_i64 s[90:91], s[90:91], 0x10000
 ; GFX12-NEXT:    v_dual_mov_b32 v6, s81 :: v_dual_mov_b32 v7, s82
 ; GFX12-NEXT:    v_dual_mov_b32 v8, s83 :: v_dual_mov_b32 v9, s84
-; GFX12-NEXT:    s_lshr_b32 s66, s12, 8
-; GFX12-NEXT:    s_lshr_b32 s56, s12, 6
-; GFX12-NEXT:    s_lshr_b32 s44, s12, 4
-; GFX12-NEXT:    s_lshr_b32 s30, s12, 2
-; GFX12-NEXT:    s_bfe_i64 s[18:19], s[12:13], 0x10000
-; GFX12-NEXT:    s_bfe_i64 s[12:13], s[26:27], 0x10000
+; GFX12-NEXT:    s_lshr_b32 s66, s10, 8
+; GFX12-NEXT:    s_lshr_b32 s56, s10, 6
+; GFX12-NEXT:    s_lshr_b32 s44, s10, 4
+; GFX12-NEXT:    s_lshr_b32 s30, s10, 2
+; GFX12-NEXT:    s_bfe_i64 s[18:19], s[10:11], 0x10000
+; GFX12-NEXT:    s_wait_alu 0xfffe
+; GFX12-NEXT:    s_bfe_i64 s[10:11], s[26:27], 0x10000
 ; GFX12-NEXT:    s_bfe_i64 s[26:27], s[40:41], 0x10000
 ; GFX12-NEXT:    s_bfe_i64 s[40:41], s[52:53], 0x10000
 ; GFX12-NEXT:    s_bfe_i64 s[52:53], s[62:63], 0x10000
@@ -9702,8 +9695,8 @@ define amdgpu_kernel void @constant_sextload_v64i1_to_v64i64(ptr addrspace(1) %o
 ; GFX12-NEXT:    v_dual_mov_b32 v16, s41 :: v_dual_mov_b32 v17, s30
 ; GFX12-NEXT:    v_dual_mov_b32 v18, s31 :: v_dual_mov_b32 v19, s26
 ; GFX12-NEXT:    v_dual_mov_b32 v20, s27 :: v_dual_mov_b32 v21, s18
-; GFX12-NEXT:    v_dual_mov_b32 v22, s19 :: v_dual_mov_b32 v23, s12
-; GFX12-NEXT:    v_mov_b32_e32 v24, s13
+; GFX12-NEXT:    v_dual_mov_b32 v22, s19 :: v_dual_mov_b32 v23, s10
+; GFX12-NEXT:    v_mov_b32_e32 v24, s11
 ; GFX12-NEXT:    s_clause 0x5
 ; GFX12-NEXT:    global_store_b128 v0, v[1:4], s[0:1] offset:80
 ; GFX12-NEXT:    global_store_b128 v0, v[5:8], s[0:1] offset:64
diff --git a/llvm/test/CodeGen/AMDGPU/load-constant-i16.ll b/llvm/test/CodeGen/AMDGPU/load-constant-i16.ll
index 255a1acbe0086..458afa4d6aad2 100644
--- a/llvm/test/CodeGen/AMDGPU/load-constant-i16.ll
+++ b/llvm/test/CodeGen/AMDGPU/load-constant-i16.ll
@@ -212,38 +212,32 @@ define amdgpu_kernel void @constant_load_v3i16(ptr addrspace(1) %out, ptr addrsp
 ;
 ; EG-LABEL: constant_load_v3i16:
 ; EG:       ; %bb.0: ; %entry
-; EG-NEXT:    ALU 0, @12, KC0[CB0:0-32], KC1[]
-; EG-NEXT:    TEX 2 @6
-; EG-NEXT:    ALU 19, @13, KC0[CB0:0-32], KC1[]
-; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T6.X, T7.X, 0
-; EG-NEXT:    MEM_RAT MSKOR T5.XW, T8.X
+; EG-NEXT:    ALU 0, @10, KC0[CB0:0-32], KC1[]
+; EG-NEXT:    TEX 1 @6
+; EG-NEXT:    ALU 14, @11, KC0[CB0:0-32], KC1[]
+; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T1.X, T3.X, 0
+; EG-NEXT:    MEM_RAT MSKOR T2.XW, T0.X
 ; EG-NEXT:    CF_END
 ; EG-NEXT:    Fetch clause starting at 6:
-; EG-NEXT:     VTX_READ_16 T6.X, T5.X, 0, #1
-; EG-NEXT:     VTX_READ_16 T7.X, T5.X, 2, #1
-; EG-NEXT:     VTX_READ_16 T5.X, T5.X, 4, #1
-; EG-NEXT:    ALU clause starting at 12:
-; EG-NEXT:     MOV * T5.X, KC0[2].Z,
-; EG-NEXT:    ALU clause starting at 13:
+; EG-NEXT:     VTX_READ_16 T1.X, T0.X, 0, #1
+; EG-NEXT:     VTX_READ_16 T0.X, T0.X, 4, #1
+; EG-NEXT:    ALU clause starting at 10:
+; EG-NEXT:     MOV * T0.X, KC0[2].Z,
+; EG-NEXT:    ALU clause starting at 11:
 ; EG-NEXT:     ADD_INT * T0.W, KC0[2].Y, literal.x,
 ; EG-NEXT:    4(5.605194e-45), 0(0.000000e+00)
 ; EG-NEXT:     AND_INT T1.W, PV.W, literal.x,
-; EG-NEXT:     AND_INT * T2.W, T5.X, literal.y,
+; EG-NEXT:     AND_INT * T2.W, T0.X, literal.y,
 ; EG-NEXT:    3(4.203895e-45), 65535(9.183409e-41)
 ; EG-NEXT:     LSHL * T1.W, PV.W, literal.x,
 ; EG-NEXT:    3(4.203895e-45), 0(0.000000e+00)
-; EG-NEXT:     LSHL T5.X, T2.W, PV.W,
-; EG-NEXT:     LSHL * T5.W, literal.x, PV.W,
+; EG-NEXT:     LSHL T2.X, T2.W, PV.W,
+; EG-NEXT:     LSHL * T2.W, literal.x, PV.W,
 ; EG-NEXT:    65535(9.183409e-41), 0(0.000000e+00)
-; EG-NEXT:     MOV T5.Y, 0.0,
-; EG-NEXT:     MOV * T5.Z, 0.0,
-; EG-NEXT:     LSHR T8.X, T0.W, literal.x,
-; EG-NEXT:     LSHL T0.W, T7.X, literal.y,
-; EG-NEXT:     AND_INT * T1.W, T6.X, literal.z,
-; EG-NEXT:    2(2.802597e-45), 16(2.242078e-44)
-; EG-NEXT:    65535(9.183409e-41), 0(0.000000e+00)
-; EG-NEXT:     OR_INT T6.X, PV.W, PS,
-; EG-NEXT:     LSHR * T7.X, KC0[2].Y, literal.x,
+; EG-NEXT:     MOV T2.Y, 0.0,
+; EG-NEXT:     MOV * T2.Z, 0.0,
+; EG-NEXT:     LSHR T0.X, T0.W, literal.x,
+; EG-NEXT:     LSHR * T3.X, KC0[2].Y, literal.x,
 ; EG-NEXT:    2(2.802597e-45), 0(0.000000e+00)
 ;
 ; GFX12-LABEL: constant_load_v3i16:
@@ -8875,7 +8869,6 @@ define amdgpu_kernel void @constant_sextload_v32i16_to_v32i64(ptr addrspace(1) %
 ; GFX12-NEXT:    s_bfe_i64 s[54:55], s[54:55], 0x100000
 ; GFX12-NEXT:    v_dual_mov_b32 v4, s64 :: v_dual_mov_b32 v7, s61
 ; GFX12-NEXT:    v_dual_mov_b32 v6, s60 :: v_dual_mov_b32 v9, s13
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    v_dual_mov_b32 v8, s12 :: v_dual_mov_b32 v11, s59
 ; GFX12-NEXT:    v_dual_mov_b32 v10, s58 :: v_dual_mov_b32 v13, s57
 ; GFX12-NEXT:    v_dual_mov_b32 v12, s56 :: v_dual_mov_b32 v15, s55
diff --git a/llvm/test/CodeGen/AMDGPU/load-constant-i32.ll b/llvm/test/CodeGen/AMDGPU/load-constant-i32.ll
index 341332e60b5c0..4ce3b46211e64 100644
--- a/llvm/test/CodeGen/AMDGPU/load-constant-i32.ll
+++ b/llvm/test/CodeGen/AMDGPU/load-constant-i32.ll
@@ -1843,11 +1843,10 @@ define amdgpu_kernel void @constant_sextload_v2i32_to_v2i64(ptr addrspace(1) %ou
 ; GFX7-HSA-NEXT:    v_mov_b32_e32 v5, s1
 ; GFX7-HSA-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7-HSA-NEXT:    s_ashr_i32 s0, s3, 31
-; GFX7-HSA-NEXT:    s_mov_b32 s1, s3
-; GFX7-HSA-NEXT:    s_ashr_i32 s3, s2, 31
+; GFX7-HSA-NEXT:    s_ashr_i32 s1, s2, 31
 ; GFX7-HSA-NEXT:    v_mov_b32_e32 v0, s2
-; GFX7-HSA-NEXT:    v_mov_b32_e32 v1, s3
-; GFX7-HSA-NEXT:    v_mov_b32_e32 v2, s1
+; GFX7-HSA-NEXT:    v_mov_b32_e32 v2, s3
+; GFX7-HSA-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX7-HSA-NEXT:    v_mov_b32_e32 v3, s0
 ; GFX7-HSA-NEXT:    flat_store_dwordx4 v[4:5], v[0:3]
 ; GFX7-HSA-NEXT:    s_endpgm
@@ -1861,11 +1860,10 @@ define amdgpu_kernel void @constant_sextload_v2i32_to_v2i64(ptr addrspace(1) %ou
 ; GFX8-NOHSA-NEXT:    v_mov_b32_e32 v5, s1
 ; GFX8-NOHSA-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX8-NOHSA-NEXT:    s_ashr_i32 s0, s3, 31
-; GFX8-NOHSA-NEXT:    s_mov_b32 s1, s3
-; GFX8-NOHSA-NEXT:    s_ashr_i32 s3, s2, 31
+; GFX8-NOHSA-NEXT:    s_ashr_i32 s1, s2, 31
 ; GFX8-NOHSA-NEXT:    v_mov_b32_e32 v0, s2
-; GFX8-NOHSA-NEXT:    v_mov_b32_e32 v1, s3
-; GFX8-NOHSA-NEXT:    v_mov_b32_e32 v2, s1
+; GFX8-NOHSA-NEXT:    v_mov_b32_e32 v2, s3
+; GFX8-NOHSA-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX8-NOHSA-NEXT:    v_mov_b32_e32 v3, s0
 ; GFX8-NOHSA-NEXT:    flat_store_dwordx4 v[4:5], v[0:3]
 ; GFX8-NOHSA-NEXT:    s_endpgm
@@ -1902,8 +1900,8 @@ define amdgpu_kernel void @constant_sextload_v2i32_to_v2i64(ptr addrspace(1) %ou
 ; GFX9-HSA-NEXT:    s_ashr_i32 s4, s3, 31
 ; GFX9-HSA-NEXT:    s_ashr_i32 s5, s2, 31
 ; GFX9-HSA-NEXT:    v_mov_b32_e32 v0, s2
-; GFX9-HSA-NEXT:    v_mov_b32_e32 v1, s5
 ; GFX9-HSA-NEXT:    v_mov_b32_e32 v2, s3
+; GFX9-HSA-NEXT:    v_mov_b32_e32 v1, s5
 ; GFX9-HSA-NEXT:    v_mov_b32_e32 v3, s4
 ; GFX9-HSA-NEXT:    global_store_dwordx4 v4, v[0:3], s[0:1]
 ; GFX9-HSA-NEXT:    s_endpgm
diff --git a/llvm/test/CodeGen/AMDGPU/load-constant-i8.ll b/llvm/test/CodeGen/AMDGPU/load-constant-i8.ll
index efc31fbd5ed9e..c608bef3f726e 100644
--- a/llvm/test/CodeGen/AMDGPU/load-constant-i8.ll
+++ b/llvm/test/CodeGen/AMDGPU/load-constant-i8.ll
@@ -8756,7 +8756,6 @@ define amdgpu_kernel void @constant_sextload_v32i8_to_v32i64(ptr addrspace(1) %o
 ; GFX12-NEXT:    v_dual_mov_b32 v24, 0 :: v_dual_mov_b32 v1, s37
 ; GFX12-NEXT:    s_bfe_i64 s[44:45], s[44:45], 0x80000
 ; GFX12-NEXT:    s_bfe_i64 s[42:43], s[42:43], 0x80000
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    v_dual_mov_b32 v0, s36 :: v_dual_mov_b32 v3, s75
 ; GFX12-NEXT:    v_dual_mov_b32 v2, s74 :: v_dual_mov_b32 v5, s41
 ; GFX12-NEXT:    s_lshr_b32 s48, s5, 16
@@ -9492,50 +9491,24 @@ define amdgpu_kernel void @constant_zextload_v4i8_to_v4i16(ptr addrspace(1) %out
 ;
 ; EG-LABEL: constant_zextload_v4i8_to_v4i16:
 ; EG:       ; %bb.0:
-; EG-NEXT:    ALU 1, @8, KC0[CB0:0-32], KC1[]
+; EG-NEXT:    ALU 0, @8, KC0[CB0:0-32], KC1[]
 ; EG-NEXT:    TEX 0 @6
-; EG-NEXT:    ALU 31, @10, KC0[CB0:0-32], KC1[]
-; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T8.XY, T7.X, 1
+; EG-NEXT:    ALU 6, @9, KC0[CB0:0-32], KC1[]
+; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T4.XY, T5.X, 1
 ; EG-NEXT:    CF_END
 ; EG-NEXT:    PAD
 ; EG-NEXT:    Fetch clause starting at 6:
-; EG-NEXT:     VTX_READ_32 T7.X, T7.X, 0, #1
+; EG-NEXT:     VTX_READ_32 T4.X, T4.X, 0, #1
 ; EG-NEXT:    ALU clause starting at 8:
-; EG-NEXT:     MOV * T0.Y, T4.X,
-; EG-NEXT:     MOV * T7.X, KC0[2].Z,
-; EG-NEXT:    ALU clause starting at 10:
-; EG-NEXT:     AND_INT T0.W, T7.X, literal.x,
-; EG-NEXT:     AND_INT * T1.W, T0.Y, literal.y,
-; EG-NEXT:    255(3.573311e-43), -65536(nan)
-; EG-NEXT:     OR_INT * T0.W, PS, PV.W,
-; EG-NEXT:     MOV * T4.X, PV.W,
-; EG-NEXT:     MOV T0.Y, PV.X,
-; EG-NEXT:     LSHL * T0.W, T7.X, literal.x,
-; EG-NEXT:    8(1.121039e-44), 0(0.000000e+00)
-; EG-NEXT:     AND_INT T1.W, PV.Y, literal.x,
-; EG-NEXT:     AND_INT * T0.W, PV.W, literal.y,
-; EG-NEXT:    65535(9.183409e-41), 16711680(2.341805e-38)
-; EG-NEXT:     OR_INT * T0.W, PV.W, PS,
-; EG-NEXT:     MOV T4.X, PV.W,
-; EG-NEXT:     MOV T0.Y, T5.X,
+; EG-NEXT:     MOV * T4.X, KC0[2].Z,
+; EG-NEXT:    ALU clause starting at 9:
 ; EG-NEXT:     MOV * T0.W, literal.x,
 ; EG-NEXT:    8(1.121039e-44), 0(0.000000e+00)
-; EG-NEXT:     BFE_UINT T0.W, T7.X, literal.x, PV.W,
-; EG-NEXT:     AND_INT * T1.W, PV.Y, literal.y,
-; EG-NEXT:    16(2.242078e-44), -65536(nan)
-; EG-NEXT:     OR_INT * T0.W, PS, PV.W,
-; EG-NEXT:     MOV * T5.X, PV.W,
-; EG-NEXT:     MOV T0.Y, PV.X,
-; EG-NEXT:     LSHR * T0.W, T7.X, literal.x,
+; EG-NEXT:     BFE_UINT * T4.Y, T4.X, literal.x, PV.W,
 ; EG-NEXT:    8(1.121039e-44), 0(0.000000e+00)
-; EG-NEXT:     AND_INT T1.W, PV.Y, literal.x,
-; EG-NEXT:     AND_INT * T0.W, PV.W, literal.y,
-; EG-NEXT:    65535(9.183409e-41), 16711680(2.341805e-38)
-; EG-NEXT:     LSHR T7.X, KC0[2].Y, literal.x,
-; EG-NEXT:     OR_INT * T8.Y, PV.W, PS,
-; EG-NEXT:    2(2.802597e-45), 0(0.000000e+00)
-; EG-NEXT:     MOV T5.X, PV.Y,
-; EG-NEXT:     MOV * T8.X, T4.X,
+; EG-NEXT:     AND_INT T4.X, T4.X, literal.x,
+; EG-NEXT:     LSHR * T5.X, KC0[2].Y, literal.y,
+; EG-NEXT:    255(3.573311e-43), 2(2.802597e-45)
 ;
 ; GFX12-LABEL: constant_zextload_v4i8_to_v4i16:
 ; GFX12:       ; %bb.0:
@@ -9634,56 +9607,23 @@ define amdgpu_kernel void @constant_sextload_v4i8_to_v4i16(ptr addrspace(1) %out
 ;
 ; EG-LABEL: constant_sextload_v4i8_to_v4i16:
 ; EG:       ; %bb.0:
-; EG-NEXT:    ALU 1, @8, KC0[CB0:0-32], KC1[]
+; EG-NEXT:    ALU 0, @8, KC0[CB0:0-32], KC1[]
 ; EG-NEXT:    TEX 0 @6
-; EG-NEXT:    ALU 37, @10, KC0[CB0:0-32], KC1[]
-; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T8.XY, T7.X, 1
+; EG-NEXT:    ALU 5, @9, KC0[CB0:0-32], KC1[]
+; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T5.XY, T4.X, 1
 ; EG-NEXT:    CF_END
 ; EG-NEXT:    PAD
 ; EG-NEXT:    Fetch clause starting at 6:
-; EG-NEXT:     VTX_READ_32 T7.X, T7.X, 0, #1
+; EG-NEXT:     VTX_READ_32 T4.X, T4.X, 0, #1
 ; EG-NEXT:    ALU clause starting at 8:
-; EG-NEXT:     MOV * T0.Y, T4.X,
-; EG-NEXT:     MOV * T7.X, KC0[2].Z,
-; EG-NEXT:    ALU clause starting at 10:
-; EG-NEXT:     BFE_INT * T0.W, T7.X, 0.0, literal.x,
-; EG-NEXT:    8(1.121039e-44), 0(0.000000e+00)
-; EG-NEXT:     AND_INT T0.W, PV.W, literal.x,
-; EG-NEXT:     AND_INT * T1.W, T0.Y, literal.y,
-; EG-NEXT:    65535(9.183409e-41), -65536(nan)
-; EG-NEXT:     OR_INT * T0.W, PS, PV.W,
-; EG-NEXT:     MOV * T4.X, PV.W,
-; EG-NEXT:     MOV T0.Y, PV.X,
-; EG-NEXT:     LSHR * T0.W, T7.X, literal.x,
+; EG-NEXT:     MOV * T4.X, KC0[2].Z,
+; EG-NEXT:    ALU clause starting at 9:
+; EG-NEXT:     BFE_INT T5.X, T4.X, 0.0, literal.x,
+; EG-NEXT:     LSHR T0.W, T4.X, literal.x,
+; EG-NEXT:     LSHR * T4.X, KC0[2].Y, literal.y,
+; EG-NEXT:    8(1.121039e-44), 2(2.802597e-45)
+; EG-NEXT:     BFE_INT * T5.Y, PV.W, 0.0, literal.x,
 ; EG-NEXT:    8(1.121039e-44), 0(0.000000e+00)
-; EG-NEXT:     BFE_INT T0.W, PV.W, 0.0, literal.x,
-; EG-NEXT:     AND_INT * T1.W, PV.Y, literal.y,
-; EG-NEXT:    8(1.121039e-44), 65535(9.183409e-41)
-; EG-NEXT:     LSHL * T0.W, PV.W, literal.x,
-; EG-NEXT:    16(2.242078e-44), 0(0.000000e+00)
-; EG-NEXT:     OR_INT * T0.W, T1.W, PV.W,
-; EG-NEXT:     MOV T4.X, PV.W,
-; EG-NEXT:     MOV T0.Y, T5.X,
-; EG-NEXT:     LSHR * T0.W, T7.X, literal.x, BS:VEC_120/SCL_212
-; EG-NEXT:    16(2.242078e-44), 0(0.000000e+00)
-; EG-NEXT:     BFE_INT T0.W, PV.W, 0.0, literal.x,
-; EG-NEXT:     AND_INT * T1.W, PV.Y, literal.y,
-; EG-NEXT:    8(1.121039e-44), -65536(nan)
-; EG-NEXT:     AND_INT * T0.W, PV.W, literal.x,
-; EG-NEXT:    65535(9.183409e-41), 0(0.000000e+00)
-; EG-NEXT:     OR_INT * T0.W, T1.W, PV.W,
-; EG-NEXT:     MOV * T5.X, PV.W,
-; EG-NEXT:     MOV T0.Y, PV.X,
-; EG-NEXT:     ASHR * T0.W, T7.X, literal.x,
-; EG-NEXT:    24(3.363116e-44), 0(0.000000e+00)
-; EG-NEXT:     AND_INT T1.W, PV.Y, literal.x,
-; EG-NEXT:     LSHL * T0.W, PV.W, literal.y,
-; EG-NEXT:    65535(9.183409e-41), 16(2.242078e-44)
-; EG-NEXT:     LSHR T7.X, KC0[2].Y, literal.x,
-; EG-NEXT:     OR_INT * T8.Y, PV.W, PS,
-; EG-NEXT:    2(2.802597e-45), 0(0.000000e+00)
-; EG-NEXT:     MOV T5.X, PV.Y,
-; EG-NEXT:     MOV * T8.X, T4.X,
 ;
 ; GFX12-LABEL: constant_sextload_v4i8_to_v4i16:
 ; GFX12:       ; %bb.0:
@@ -9801,80 +9741,27 @@ define amdgpu_kernel void @constant_zextload_v8i8_to_v8i16(ptr addrspace(1) %out
 ;
 ; EG-LABEL: constant_zextload_v8i8_to_v8i16:
 ; EG:       ; %bb.0:
-; EG-NEXT:    ALU 1, @8, KC0[CB0:0-32], KC1[]
+; EG-NEXT:    ALU 0, @8, KC0[CB0:0-32], KC1[]
 ; EG-NEXT:    TEX 0 @6
-; EG-NEXT:    ALU 61, @10, KC0[CB0:0-32], KC1[]
-; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T12.XYZW, T11.X, 1
+; EG-NEXT:    ALU 9, @9, KC0[CB0:0-32], KC1[]
+; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T6.XYZW, T5.X, 1
 ; EG-NEXT:    CF_END
 ; EG-NEXT:    PAD
 ; EG-NEXT:    Fetch clause starting at 6:
-; EG-NEXT:     VTX_READ_64 T11.XY, T11.X, 0, #1
+; EG-NEXT:     VTX_READ_64 T5.XY, T5.X, 0, #1
 ; EG-NEXT:    ALU clause starting at 8:
-; EG-NEXT:     MOV * T0.Y, T8.X,
-; EG-NEXT:     MOV * T11.X, KC0[2].Z,
-; EG-NEXT:    ALU clause starting at 10:
-; EG-NEXT:     AND_INT T0.W, T11.X, literal.x,
-; EG-NEXT:     AND_INT * T1.W, T0.Y, literal.y,
-; EG-NEXT:    255(3.573311e-43), -65536(nan)
-; EG-NEXT:     OR_INT * T0.W, PS, PV.W,
-; EG-NEXT:     MOV * T8.X, PV.W,
-; EG-NEXT:     MOV T0.Y, PV.X,
-; EG-NEXT:     LSHL * T0.W, T11.X, literal.x,
-; EG-NEXT:    8(1.121039e-44), 0(0.000000e+00)
-; EG-NEXT:     AND_INT T1.W, PV.Y, literal.x,
-; EG-NEXT:     AND_INT * T0.W, PV.W, literal.y,
-; EG-NEXT:    65535(9.183409e-41), 16711680(2.341805e-38)
-; EG-NEXT:     OR_INT * T0.W, PV.W, PS,
-; EG-NEXT:     MOV T8.X, PV.W,
-; EG-NEXT:     MOV T0.Y, T9.X,
+; EG-NEXT:     MOV * T5.X, KC0[2].Z,
+; EG-NEXT:    ALU clause starting at 9:
 ; EG-NEXT:     MOV * T0.W, literal.x,
 ; EG-NEXT:    8(1.121039e-44), 0(0.000000e+00)
-; EG-NEXT:     BFE_UINT T1.W, T11.X, literal.x, PV.W,
-; EG-NEXT:     AND_INT * T2.W, PV.Y, literal.y,
-; EG-NEXT:    16(2.242078e-44), -65536(nan)
-; EG-NEXT:     OR_INT * T1.W, PS, PV.W,
-; EG-NEXT:     MOV * T9.X, PV.W,
-; EG-NEXT:     MOV T0.Y, PV.X,
-; EG-NEXT:     LSHR * T1.W, T11.X, literal.x,
-; EG-NEXT:    8(1.121039e-44), 0(0.000000e+00)
-; EG-NEXT:     AND_INT T2.W, PV.Y, literal.x,
-; EG-NEXT:     AND_INT * T1.W, PV.W, literal.y,
-; EG-NEXT:    65535(9.183409e-41), 16711680(2.341805e-38)
-; EG-NEXT:     OR_INT * T12.Y, PV.W, PS,
-; EG-NEXT:     MOV T9.X, PV.Y,
-; EG-NEXT:     MOV * T0.Y, T4.X,
-; EG-NEXT:     AND_INT T1.W, PV.Y, literal.x,
-; EG-NEXT:     AND_INT * T2.W, T11.Y, literal.y,
-; EG-NEXT:    -65536(nan), 255(3.573311e-43)
-; EG-NEXT:     OR_INT * T1.W, PV.W, PS,
-; EG-NEXT:     MOV * T4.X, PV.W,
-; EG-NEXT:     MOV T0.Y, PV.X,
-; EG-NEXT:     LSHL * T1.W, T11.Y, literal.x,
-; EG-NEXT:    8(1.121039e-44), 0(0.000000e+00)
-; EG-NEXT:     AND_INT T2.W, PV.Y, literal.x,
-; EG-NEXT:     AND_INT * T1.W, PV.W, literal.y,
-; EG-NEXT:    65535(9.183409e-41), 16711680(2.341805e-38)
-; EG-NEXT:     OR_INT * T1.W, PV.W, PS,
-; EG-NEXT:     MOV T4.X, PV.W,
-; EG-NEXT:     MOV T0.Y, T5.X,
-; EG-NEXT:     BFE_UINT * T0.W, T11.Y, literal.x, T0.W,
-; EG-NEXT:    16(2.242078e-44), 0(0.000000e+00)
-; EG-NEXT:     AND_INT * T1.W, PV.Y, literal.x,
-; EG-NEXT:    -65536(nan), 0(0.000000e+00)
-; EG-NEXT:     OR_INT * T0.W, PV.W, T0.W,
-; EG-NEXT:     MOV * T5.X, PV.W,
-; EG-NEXT:     MOV T0.Y, PV.X,
-; EG-NEXT:     LSHR * T0.W, T11.Y, literal.x,
+; EG-NEXT:     BFE_UINT * T6.W, T5.Y, literal.x, PV.W,
 ; EG-NEXT:    8(1.121039e-44), 0(0.000000e+00)
-; EG-NEXT:     AND_INT T1.W, PV.Y, literal.x,
-; EG-NEXT:     AND_INT * T0.W, PV.W, literal.y,
-; EG-NEXT:    65535(9.183409e-41), 16711680(2.341805e-38)
-; EG-NEXT:     LSHR T11.X, KC0[2].Y, literal.x,
-; EG-NEXT:     OR_INT * T12.W, PV.W, PS,
-; EG-NEXT:    2(2.802597e-45), 0(0.000000e+00)
-; EG-NEXT:     MOV T5.X, PV.W,
-; EG-NEXT:     MOV * T12.X, T8.X,
-; EG-NEXT:     MOV * T12.Z, T4.X,
+; EG-NEXT:     BFE_UINT T6.Y, T5.X, literal.x, T0.W,
+; EG-NEXT:     AND_INT * T6.Z, T5.Y, literal.y,
+; EG-NEXT:    8(1.121039e-44), 255(3.573311e-43)
+; EG-NEXT:     AND_INT T6.X, T5.X, literal.x,
+; EG-NEXT:     LSHR * T5.X, KC0[2].Y, literal.y,
+; EG-NEXT:    255(3.573311e-43), 2(2.802597e-45)
 ;
 ; GFX12-LABEL: constant_zextload_v8i8_to_v8i16:
 ; GFX12:       ; %bb.0:
@@ -10018,93 +9905,28 @@ define amdgpu_kernel void @constant_sextload_v8i8_to_v8i16(ptr addrspace(1) %out
 ;
 ; EG-LABEL: constant_sextload_v8i8_to_v8i16:
 ; EG:       ; %bb.0:
-; EG-NEXT:    ALU 1, @8, KC0[CB0:0-32], KC1[]
+; EG-NEXT:    ALU 0, @8, KC0[CB0:0-32], KC1[]
 ; EG-NEXT:    TEX 0 @6
-; EG-NEXT:    ALU 74, @10, KC0[CB0:0-32], KC1[]
-; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T12.XYZW, T11.X, 1
+; EG-NEXT:    ALU 10, @9, KC0[CB0:0-32], KC1[]
+; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T6.XYZW, T5.X, 1
 ; EG-NEXT:    CF_END
 ; EG-NEXT:    PAD
 ; EG-NEXT:    Fetch clause starting at 6:
-; EG-NEXT:     VTX_READ_64 T11.XY, T11.X, 0, #1
+; EG-NEXT:     VTX_READ_64 T5.XY, T5.X, 0, #1
 ; EG-NEXT:    ALU clause starting at 8:
-; EG-NEXT:     MOV * T0.Y, T8.X,
-; EG-NEXT:     MOV * T11.X, KC0[2].Z,
-; EG-NEXT:    ALU clause starting at 10:
-; EG-NEXT:     BFE_INT * T0.W, T11.X, 0.0, literal.x,
-; EG-NEXT:    8(1.121039e-44), 0(0.000000e+00)
-; EG-NEXT:     AND_INT T0.W, PV.W, literal.x,
-; EG-NEXT:     AND_INT * T1.W, T0.Y, literal.y,
-; EG-NEXT:    65535(9.183409e-41), -65536(nan)
-; EG-NEXT:     OR_INT * T0.W, PS, PV.W,
-; EG-NEXT:     MOV * T8.X, PV.W,
-; EG-NEXT:     MOV T0.Y, PV.X,
-; EG-NEXT:     LSHR * T0.W, T11.X, literal.x,
+; EG-NEXT:     MOV * T5.X, KC0[2].Z,
+; EG-NEXT:    ALU clause starting at 9:
+; EG-NEXT:     BFE_INT * T6.Z, T5.Y, 0.0, literal.x,
 ; EG-NEXT:    8(1.121039e-44), 0(0.000000e+00)
-; EG-NEXT:     BFE_INT T0.W, PV.W, 0.0, literal.x,
-; EG-NEXT:     AND_INT * T1.W, PV.Y, literal.y,
-; EG-NEXT:    8(1.121039e-44), 65535(9.183409e-41)
-; EG-NEXT:     LSHL * T0.W, PV.W, literal.x,
-; EG-NEXT:    16(2.242078e-44), 0(0.000000e+00)
-; EG-NEXT:     OR_INT * T0.W, T1.W, PV.W,
-; EG-NEXT:     MOV T8.X, PV.W,
-; EG-NEXT:     MOV T0.Y, T9.X,
-; EG-NEXT:     LSHR * T0.W, T11.X, literal.x, BS:VEC_120/SCL_212
-; EG-NEXT:    16(2.242078e-44), 0(0.000000e+00)
-; EG-NEXT:     BFE_INT T0.W, PV.W, 0.0, literal.x,
-; EG-NEXT:     AND_INT * T1.W, PV.Y, literal.y,
-; EG-NEXT:    8(1.121039e-44), -65536(nan)
-; EG-NEXT:     AND_INT * T0.W, PV.W, literal.x,
-; EG-NEXT:    65535(9.183409e-41), 0(0.000000e+00)
-; EG-NEXT:     OR_INT * T0.W, T1.W, PV.W,
-; EG-NEXT:     MOV * T9.X, PV.W,
-; EG-NEXT:     MOV T0.Y, PV.X,
-; EG-NEXT:     ASHR * T0.W, T11.X, literal.x,
-; EG-NEXT:    24(3.363116e-44), 0(0.000000e+00)
-; EG-NEXT:     AND_INT T1.W, PV.Y, literal.x,
-; EG-NEXT:     LSHL * T0.W, PV.W, literal.y,
-; EG-NEXT:    65535(9.183409e-41), 16(2.242078e-44)
-; EG-NEXT:     OR_INT * T12.Y, PV.W, PS,
-; EG-NEXT:     MOV T9.X, PV.Y,
-; EG-NEXT:     MOV T0.Y, T4.X,
-; EG-NEXT:     BFE_INT * T0.W, T11.Y, 0.0, literal.x,
+; EG-NEXT:     BFE_INT T6.X, T5.X, 0.0, literal.x,
+; EG-NEXT:     LSHR * T0.W, T5.Y, literal.x,
 ; EG-NEXT:    8(1.121039e-44), 0(0.000000e+00)
-; EG-NEXT:     AND_INT T1.W, PV.Y, literal.x,
-; EG-NEXT:     AND_INT * T0.W, PV.W, literal.y,
-; EG-NEXT:    -65536(nan), 65535(9.183409e-41)
-; EG-NEXT:     OR_INT * T0.W, PV.W, PS,
-; EG-NEXT:     MOV * T4.X, PV.W,
-; EG-NEXT:     MOV T0.Y, PV.X,
-; EG-NEXT:     LSHR * T0.W, T11.Y, literal.x,
+; EG-NEXT:     BFE_INT T6.W, PV.W, 0.0, literal.x,
+; EG-NEXT:     LSHR * T0.W, T5.X, literal.x,
 ; EG-NEXT:    8(1.121039e-44), 0(0.000000e+00)
-; EG-NEXT:     BFE_INT T0.W, PV.W, 0.0, literal.x,
-; EG-NEXT:     AND_INT * T1.W, PV.Y, literal.y,
-; EG-NEXT:    8(1.121039e-44), 65535(9.183409e-41)
-; EG-NEXT:     LSHL * T0.W, PV.W, literal.x,
-; EG-NEXT:    16(2.242078e-44), 0(0.000000e+00)
-; EG-NEXT:     OR_INT * T0.W, T1.W, PV.W,
-; EG-NEXT:     MOV T4.X, PV.W,
-; EG-NEXT:     MOV T0.Y, T5.X,
-; EG-NEXT:     LSHR * T0.W, T11.Y, literal.x,
-; EG-NEXT:    16(2.242078e-44), 0(0.000000e+00)
-; EG-NEXT:     BFE_INT T0.W, PV.W, 0.0, literal.x,
-; EG-NEXT:     AND_INT * T1.W, PV.Y, literal.y,
-; EG-NEXT:    8(1.121039e-44), -65536(nan)
-; EG-NEXT:     AND_INT * T0.W, PV.W, literal.x,
-; EG-NEXT:    65535(9.183409e-41), 0(0.000000e+00)
-; EG-NEXT:     OR_INT * T0.W, T1.W, PV.W,
-; EG-NEXT:     MOV * T5.X, PV.W,
-; EG-NEXT:     MOV T0.Y, PV.X,
-; EG-NEXT:     ASHR * T0.W, T11.Y, literal.x,
-; EG-NEXT:    24(3.363116e-44), 0(0.000000e+00)
-; EG-NEXT:     AND_INT T1.W, PV.Y, literal.x,
-; EG-NEXT:     LSHL * T0.W, PV.W, literal.y,
-; EG-NEXT:    65535(9.183409e-41), 16(2.242078e-44)
-; EG-NEXT:     LSHR T11.X, KC0[2].Y, literal.x,
-; EG-NEXT:     OR_INT * T12.W, PV.W, PS,
-; EG-NEXT:    2(2.802597e-45), 0(0.000000e+00)
-; EG-NEXT:     MOV T5.X, PV.W,
-; EG-NEXT:     MOV * T12.X, T8.X,
-; EG-NEXT:     MOV * T12.Z, T4.X,
+; EG-NEXT:     LSHR T5.X, KC0[2].Y, literal.x,
+; EG-NEXT:     BFE_INT * T6.Y, PS, 0.0, literal.y,
+; EG-NEXT:    2(2.802597e-45), 8(1.121039e-44)
 ;
 ; GFX12-LABEL: constant_sextload_v8i8_to_v8i16:
 ; GFX12:       ; %bb.0:
@@ -10297,146 +10119,37 @@ define amdgpu_kernel void @constant_zextload_v16i8_to_v16i16(ptr addrspace(1) %o
 ;
 ; EG-LABEL: constant_zextload_v16i8_to_v16i16:
 ; EG:       ; %bb.0:
-; EG-NEXT:    ALU 1, @10, KC0[CB0:0-32], KC1[]
-; EG-NEXT:    TEX 0 @8
-; EG-NEXT:    ALU 103, @12, KC0[], KC1[]
-; EG-NEXT:    ALU 20, @116, KC0[CB0:0-32], KC1[]
-; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T20.XYZW, T22.X, 0
-; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T19.XYZW, T21.X, 1
+; EG-NEXT:    ALU 0, @8, KC0[CB0:0-32], KC1[]
+; EG-NEXT:    TEX 0 @6
+; EG-NEXT:    ALU 19, @9, KC0[CB0:0-32], KC1[]
+; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T9.XYZW, T10.X, 0
+; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T8.XYZW, T7.X, 1
 ; EG-NEXT:    CF_END
-; EG-NEXT:    PAD
-; EG-NEXT:    Fetch clause starting at 8:
-; EG-NEXT:     VTX_READ_128 T19.XYZW, T19.X, 0, #1
-; EG-NEXT:    ALU clause starting at 10:
-; EG-NEXT:     MOV * T0.Y, T16.X,
-; EG-NEXT:     MOV * T19.X, KC0[2].Z,
-; EG-NEXT:    ALU clause starting at 12:
-; EG-NEXT:     AND_INT T0.W, T19.X, literal.x,
-; EG-NEXT:     AND_INT * T1.W, T0.Y, literal.y,
-; EG-NEXT:    255(3.573311e-43), -65536(nan)
-; EG-NEXT:     OR_INT * T0.W, PS, PV.W,
-; EG-NEXT:     MOV * T16.X, PV.W,
-; EG-NEXT:     MOV T0.Y, PV.X,
-; EG-NEXT:     LSHL * T0.W, T19.X, literal.x,
-; EG-NEXT:    8(1.121039e-44), 0(0.000000e+00)
-; EG-NEXT:     AND_INT T1.W, PV.Y, literal.x,
-; EG-NEXT:     AND_INT * T0.W, PV.W, literal.y,
-; EG-NEXT:    65535(9.183409e-41), 16711680(2.341805e-38)
-; EG-NEXT:     OR_INT * T0.W, PV.W, PS,
-; EG-NEXT:     MOV T16.X, PV.W,
-; EG-NEXT:     MOV T0.Y, T17.X,
+; EG-NEXT:    Fetch clause starting at 6:
+; EG-NEXT:     VTX_READ_128 T7.XYZW, T7.X, 0, #1
+; EG-NEXT:    ALU clause starting at 8:
+; EG-NEXT:     MOV * T7.X, KC0[2].Z,
+; EG-NEXT:    ALU clause starting at 9:
 ; EG-NEXT:     MOV * T0.W, literal.x,
 ; EG-NEXT:    8(1.121039e-44), 0(0.000000e+00)
-; EG-NEXT:     BFE_UINT T1.W, T19.X, literal.x, PV.W,
-; EG-NEXT:     AND_INT * T2.W, PV.Y, literal.y,
-; EG-NEXT:    16(2.242078e-44), -65536(nan)
-; EG-NEXT:     OR_INT * T1.W, PS, PV.W,
-; EG-NEXT:     MOV * T17.X, PV.W,
-; EG-NEXT:     MOV T0.Y, PV.X,
-; EG-NEXT:     LSHR * T1.W, T19.X, literal.x,
-; EG-NEXT:    8(1.121039e-44), 0(0.000000e+00)
-; EG-NEXT:     AND_INT T2.W, PV.Y, literal.x,
-; EG-NEXT:     AND_INT * T1.W, PV.W, literal.y,
-; EG-NEXT:    65535(9.183409e-41), 16711680(2.341805e-38)
-; EG-NEXT:     OR_INT * T20.Y, PV.W, PS,
-; EG-NEXT:     MOV T17.X, PV.Y,
-; EG-NEXT:     MOV * T0.Y, T12.X,
-; EG-NEXT:     AND_INT T1.W, PV.Y, literal.x,
-; EG-NEXT:     AND_INT * T2.W, T19.Y, literal.y,
-; EG-NEXT:    -65536(nan), 255(3.573311e-43)
-; EG-NEXT:     OR_INT * T1.W, PV.W, PS,
-; EG-NEXT:     MOV * T12.X, PV.W,
-; EG-NEXT:     MOV T0.Y, PV.X,
-; EG-NEXT:     LSHL * T1.W, T19.Y, literal.x,
-; EG-NEXT:    8(1.121039e-44), 0(0.000000e+00)
-; EG-NEXT:     AND_INT T2.W, PV.Y, literal.x,
-; EG-NEXT:     AND_INT * T1.W, PV.W, literal.y,
-; EG-NEXT:    65535(9.183409e-41), 16711680(2.341805e-38)
-; EG-NEXT:     OR_INT * T1.W, PV.W, PS,
-; EG-NEXT:     MOV T12.X, PV.W,
-; EG-NEXT:     MOV T0.Y, T13.X,
-; EG-NEXT:     BFE_UINT * T1.W, T19.Y, literal.x, T0.W,
-; EG-NEXT:    16(2.242078e-44), 0(0.000000e+00)
-; EG-NEXT:     AND_INT * T2.W, PV.Y, literal.x,
-; EG-NEXT:    -65536(nan), 0(0.000000e+00)
-; EG-NEXT:     OR_INT * T1.W, PV.W, T1.W,
-; EG-NEXT:     MOV * T13.X, PV.W,
-; EG-NEXT:     MOV T0.Y, PV.X,
-; EG-NEXT:     LSHR * T1.W, T19.Y, literal.x,
+; EG-NEXT:     BFE_UINT * T8.W, T7.Y, literal.x, PV.W,
 ; EG-NEXT:    8(1.121039e-44), 0(0.000000e+00)
-; EG-NEXT:     AND_INT T2.W, PV.Y, literal.x,
-; EG-NEXT:     AND_INT * T1.W, PV.W, literal.y,
-; EG-NEXT:    65535(9.183409e-41), 16711680(2.341805e-38)
-; EG-NEXT:     OR_INT * T20.W, PV.W, PS,
-; EG-NEXT:     MOV T13.X, PV.W,
-; EG-NEXT:     MOV * T0.Y, T8.X,
-; EG-NEXT:     AND_INT T1.W, PV.Y, literal.x,
-; EG-NEXT:     AND_INT * T2.W, T19.Z, literal.y,
-; EG-NEXT:    -65536(nan), 255(3.573311e-43)
-; EG-NEXT:     OR_INT * T1.W, PV.W, PS,
-; EG-NEXT:     MOV * T8.X, PV.W,
-; EG-NEXT:     MOV T0.Y, PV.X,
-; EG-NEXT:     LSHL * T1.W, T19.Z, literal.x,
-; EG-NEXT:    8(1.121039e-44), 0(0.000000e+00)
-; EG-NEXT:     AND_INT T2.W, PV.Y, literal.x,
-; EG-NEXT:     AND_INT * T1.W, PV.W, literal.y,
-; EG-NEXT:    65535(9.183409e-41), 16711680(2.341805e-38)
-; EG-NEXT:     OR_INT * T1.W, PV.W, PS,
-; EG-NEXT:     MOV T8.X, PV.W,
-; EG-NEXT:     MOV T0.Y, T9.X,
-; EG-NEXT:     BFE_UINT * T1.W, T19.Z, literal.x, T0.W,
-; EG-NEXT:    16(2.242078e-44), 0(0.000000e+00)
-; EG-NEXT:     AND_INT * T2.W, PV.Y, literal.x,
-; EG-NEXT:    -65536(nan), 0(0.000000e+00)
-; EG-NEXT:     OR_INT * T1.W, PV.W, T1.W,
-; EG-NEXT:     MOV * T9.X, PV.W,
-; EG-NEXT:     MOV T0.Y, PV.X,
-; EG-NEXT:     LSHR * T1.W, T19.Z, literal.x,
-; EG-NEXT:    8(1.121039e-44), 0(0.000000e+00)
-; EG-NEXT:     AND_INT T2.W, PV.Y, literal.x,
-; EG-NEXT:     AND_INT * T1.W, PV.W, literal.y,
-; EG-NEXT:    65535(9.183409e-41), 16711680(2.341805e-38)
-; EG-NEXT:     OR_INT * T19.Y, PV.W, PS,
-; EG-NEXT:     MOV T9.X, PV.Y,
-; EG-NEXT:     MOV * T0.Y, T4.X,
-; EG-NEXT:     AND_INT T1.W, PV.Y, literal.x,
-; EG-NEXT:     AND_INT * T2.W, T19.W, literal.y,
-; EG-NEXT:    -65536(nan), 255(3.573311e-43)
-; EG-NEXT:     OR_INT * T1.W, PV.W, PS,
-; EG-NEXT:     MOV * T4.X, PV.W,
-; EG-NEXT:     MOV T0.Y, PV.X,
-; EG-NEXT:     LSHL * T1.W, T19.W, literal.x,
-; EG-NEXT:    8(1.121039e-44), 0(0.000000e+00)
-; EG-NEXT:     AND_INT T2.W, PV.Y, literal.x,
-; EG-NEXT:     AND_INT * T1.W, PV.W, literal.y,
-; EG-NEXT:    65535(9.183409e-41), 16711680(2.341805e-38)
-; EG-NEXT:     OR_INT * T1.W, PV.W, PS,
-; EG-NEXT:     MOV T4.X, PV.W,
-; EG-NEXT:     MOV T0.Y, T5.X,
-; EG-NEXT:     BFE_UINT * T0.W, T19.W, literal.x, T0.W,
-; EG-NEXT:    16(2.242078e-44), 0(0.000000e+00)
-; EG-NEXT:    ALU clause starting at 116:
-; EG-NEXT:     AND_INT * T1.W, T0.Y, literal.x,
-; EG-NEXT:    -65536(nan), 0(0.000000e+00)
-; EG-NEXT:     OR_INT * T0.W, PV.W, T0.W,
-; EG-NEXT:     MOV * T5.X, PV.W,
-; EG-NEXT:     MOV T0.Y, PV.X,
-; EG-NEXT:     LSHR T0.W, T19.W, literal.x,
-; EG-NEXT:     ADD_INT * T1.W, KC0[2].Y, literal.y,
-; EG-NEXT:    8(1.121039e-44), 16(2.242078e-44)
-; EG-NEXT:     LSHR T21.X, PS, literal.x,
-; EG-NEXT:     AND_INT T1.W, PV.Y, literal.y,
-; EG-NEXT:     AND_INT * T0.W, PV.W, literal.z,
-; EG-NEXT:    2(2.802597e-45), 65535(9.183409e-41)
-; EG-NEXT:    16711680(2.341805e-38), 0(0.000000e+00)
-; EG-NEXT:     LSHR T22.X, KC0[2].Y, literal.x,
-; EG-NEXT:     OR_INT * T19.W, PV.W, PS,
+; EG-NEXT:     BFE_UINT T8.Y, T7.X, literal.x, T0.W,
+; EG-NEXT:     AND_INT T8.Z, T7.Y, literal.y,
+; EG-NEXT:     BFE_UINT * T9.W, T7.W, literal.x, T0.W,
+; EG-NEXT:    8(1.121039e-44), 255(3.573311e-43)
+; EG-NEXT:     AND_INT T8.X, T7.X, literal.x,
+; EG-NEXT:     BFE_UINT T9.Y, T7.Z, literal.y, T0.W,
+; EG-NEXT:     LSHR * T7.X, KC0[2].Y, literal.z,
+; EG-NEXT:    255(3.573311e-43), 8(1.121039e-44)
+; EG-NEXT:    2(2.802597e-45), 0(0.000000e+00)
+; EG-NEXT:     AND_INT * T9.Z, T7.W, literal.x,
+; EG-NEXT:    255(3.573311e-43), 0(0.000000e+00)
+; EG-NEXT:     AND_INT T9.X, T7.Z, literal.x,
+; EG-NEXT:     ADD_INT * T0.W, KC0[2].Y, literal.y,
+; EG-NEXT:    255(3.573311e-43), 16(2.242078e-44)
+; EG-NEXT:     LSHR * T10.X, PV.W, literal.x,
 ; EG-NEXT:    2(2.802597e-45), 0(0.000000e+00)
-; EG-NEXT:     MOV T5.X, PV.W,
-; EG-NEXT:     MOV * T20.X, T16.X,
-; EG-NEXT:     MOV * T20.Z, T12.X,
-; EG-NEXT:     MOV T19.X, T8.X,
-; EG-NEXT:     MOV * T19.Z, T4.X, BS:VEC_120/SCL_212
 ;
 ; GFX12-LABEL: constant_zextload_v16i8_to_v16i16:
 ; GFX12:       ; %bb.0:
@@ -10684,173 +10397,38 @@ define amdgpu_kernel void @constant_sextload_v16i8_to_v16i16(ptr addrspace(1) %o
 ;
 ; EG-LABEL: constant_sextload_v16i8_to_v16i16:
 ; EG:       ; %bb.0:
-; EG-NEXT:    ALU 1, @10, KC0[CB0:0-32], KC1[]
-; EG-NEXT:    TEX 0 @8
-; EG-NEXT:    ALU 104, @12, KC0[], KC1[]
-; EG-NEXT:    ALU 46, @117, KC0[CB0:0-32], KC1[]
-; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T20.XYZW, T22.X, 0
-; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T19.XYZW, T21.X, 1
+; EG-NEXT:    ALU 0, @8, KC0[CB0:0-32], KC1[]
+; EG-NEXT:    TEX 0 @6
+; EG-NEXT:    ALU 20, @9, KC0[CB0:0-32], KC1[]
+; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T9.XYZW, T10.X, 0
+; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T8.XYZW, T7.X, 1
 ; EG-NEXT:    CF_END
-; EG-NEXT:    PAD
-; EG-NEXT:    Fetch clause starting at 8:
-; EG-NEXT:     VTX_READ_128 T19.XYZW, T19.X, 0, #1
-; EG-NEXT:    ALU clause starting at 10:
-; EG-NEXT:     MOV * T0.Y, T16.X,
-; EG-NEXT:     MOV * T19.X, KC0[2].Z,
-; EG-NEXT:    ALU clause starting at 12:
-; EG-NEXT:     BFE_INT * T0.W, T19.X, 0.0, literal.x,
-; EG-NEXT:    8(1.121039e-44), 0(0.000000e+00)
-; EG-NEXT:     AND_INT T0.W, PV.W, literal.x,
-; EG-NEXT:     AND_INT * T1.W, T0.Y, literal.y,
-; EG-NEXT:    65535(9.183409e-41), -65536(nan)
-; EG-NEXT:     OR_INT * T0.W, PS, PV.W,
-; EG-NEXT:     MOV * T16.X, PV.W,
-; EG-NEXT:     MOV T0.Y, PV.X,
-; EG-NEXT:     LSHR * T0.W, T19.X, literal.x,
-; EG-NEXT:    8(1.121039e-44), 0(0.000000e+00)
-; EG-NEXT:     BFE_INT T0.W, PV.W, 0.0, literal.x,
-; EG-NEXT:     AND_INT * T1.W, PV.Y, literal.y,
-; EG-NEXT:    8(1.121039e-44), 65535(9.183409e-41)
-; EG-NEXT:     LSHL * T0.W, PV.W, literal.x,
-; EG-NEXT:    16(2.242078e-44), 0(0.000000e+00)
-; EG-NEXT:     OR_INT * T0.W, T1.W, PV.W,
-; EG-NEXT:     MOV T16.X, PV.W,
-; EG-NEXT:     MOV T0.Y, T17.X,
-; EG-NEXT:     LSHR * T0.W, T19.X, literal.x, BS:VEC_120/SCL_212
-; EG-NEXT:    16(2.242078e-44), 0(0.000000e+00)
-; EG-NEXT:     BFE_INT T0.W, PV.W, 0.0, literal.x,
-; EG-NEXT:     AND_INT * T1.W, PV.Y, literal.y,
-; EG-NEXT:    8(1.121039e-44), -65536(nan)
-; EG-NEXT:     AND_INT * T0.W, PV.W, literal.x,
-; EG-NEXT:    65535(9.183409e-41), 0(0.000000e+00)
-; EG-NEXT:     OR_INT * T0.W, T1.W, PV.W,
-; EG-NEXT:     MOV * T17.X, PV.W,
-; EG-NEXT:     MOV T0.Y, PV.X,
-; EG-NEXT:     ASHR * T0.W, T19.X, literal.x,
-; EG-NEXT:    24(3.363116e-44), 0(0.000000e+00)
-; EG-NEXT:     AND_INT T1.W, PV.Y, literal.x,
-; EG-NEXT:     LSHL * T0.W, PV.W, literal.y,
-; EG-NEXT:    65535(9.183409e-41), 16(2.242078e-44)
-; EG-NEXT:     OR_INT * T20.Y, PV.W, PS,
-; EG-NEXT:     MOV T17.X, PV.Y,
-; EG-NEXT:     MOV T0.Y, T12.X,
-; EG-NEXT:     BFE_INT * T0.W, T19.Y, 0.0, literal.x,
-; EG-NEXT:    8(1.121039e-44), 0(0.000000e+00)
-; EG-NEXT:     AND_INT T1.W, PV.Y, literal.x,
-; EG-NEXT:     AND_INT * T0.W, PV.W, literal.y,
-; EG-NEXT:    -65536(nan), 65535(9.183409e-41)
-; EG-NEXT:     OR_INT * T0.W, PV.W, PS,
-; EG-NEXT:     MOV * T12.X, PV.W,
-; EG-NEXT:     MOV T0.Y, PV.X,
-; EG-NEXT:     LSHR * T0.W, T19.Y, literal.x,
-; EG-NEXT:    8(1.121039e-44), 0(0.000000e+00)
-; EG-NEXT:     BFE_INT T0.W, PV.W, 0.0, literal.x,
-; EG-NEXT:     AND_INT * T1.W, PV.Y, literal.y,
-; EG-NEXT:    8(1.121039e-44), 65535(9.183409e-41)
-; EG-NEXT:     LSHL * T0.W, PV.W, literal.x,
-; EG-NEXT:    16(2.242078e-44), 0(0.000000e+00)
-; EG-NEXT:     OR_INT * T0.W, T1.W, PV.W,
-; EG-NEXT:     MOV T12.X, PV.W,
-; EG-NEXT:     MOV T0.Y, T13.X,
-; EG-NEXT:     LSHR * T0.W, T19.Y, literal.x,
-; EG-NEXT:    16(2.242078e-44), 0(0.000000e+00)
-; EG-NEXT:     BFE_INT T0.W, PV.W, 0.0, literal.x,
-; EG-NEXT:     AND_INT * T1.W, PV.Y, literal.y,
-; EG-NEXT:    8(1.121039e-44), -65536(nan)
-; EG-NEXT:     AND_INT * T0.W, PV.W, literal.x,
-; EG-NEXT:    65535(9.183409e-41), 0(0.000000e+00)
-; EG-NEXT:     OR_INT * T0.W, T1.W, PV.W,
-; EG-NEXT:     MOV * T13.X, PV.W,
-; EG-NEXT:     MOV T0.Y, PV.X,
-; EG-NEXT:     ASHR * T0.W, T19.Y, literal.x,
-; EG-NEXT:    24(3.363116e-44), 0(0.000000e+00)
-; EG-NEXT:     AND_INT T1.W, PV.Y, literal.x,
-; EG-NEXT:     LSHL * T0.W, PV.W, literal.y,
-; EG-NEXT:    65535(9.183409e-41), 16(2.242078e-44)
-; EG-NEXT:     OR_INT * T20.W, PV.W, PS,
-; EG-NEXT:     MOV T13.X, PV.W,
-; EG-NEXT:     MOV T0.Y, T8.X,
-; EG-NEXT:     BFE_INT * T0.W, T19.Z, 0.0, literal.x,
-; EG-NEXT:    8(1.121039e-44), 0(0.000000e+00)
-; EG-NEXT:     AND_INT T1.W, PV.Y, literal.x,
-; EG-NEXT:     AND_INT * T0.W, PV.W, literal.y,
-; EG-NEXT:    -65536(nan), 65535(9.183409e-41)
-; EG-NEXT:     OR_INT * T0.W, PV.W, PS,
-; EG-NEXT:     MOV * T8.X, PV.W,
-; EG-NEXT:     MOV T0.Y, PV.X,
-; EG-NEXT:     LSHR * T0.W, T19.Z, literal.x,
+; EG-NEXT:    Fetch clause starting at 6:
+; EG-NEXT:     VTX_READ_128 T7.XYZW, T7.X, 0, #1
+; EG-NEXT:    ALU clause starting at 8:
+; EG-NEXT:     MOV * T7.X, KC0[2].Z,
+; EG-NEXT:    ALU clause starting at 9:
+; EG-NEXT:     BFE_INT * T8.Z, T7.Y, 0.0, literal.x,
 ; EG-NEXT:    8(1.121039e-44), 0(0.000000e+00)
-; EG-NEXT:     BFE_INT T0.W, PV.W, 0.0, literal.x,
-; EG-NEXT:     AND_INT * T1.W, PV.Y, literal.y,
-; EG-NEXT:    8(1.121039e-44), 65535(9.183409e-41)
-; EG-NEXT:     LSHL * T0.W, PV.W, literal.x,
-; EG-NEXT:    16(2.242078e-44), 0(0.000000e+00)
-; EG-NEXT:     OR_INT * T0.W, T1.W, PV.W,
-; EG-NEXT:     MOV T8.X, PV.W,
-; EG-NEXT:     MOV T0.Y, T9.X,
-; EG-NEXT:     LSHR * T0.W, T19.Z, literal.x,
-; EG-NEXT:    16(2.242078e-44), 0(0.000000e+00)
-; EG-NEXT:     BFE_INT T0.W, PV.W, 0.0, literal.x,
-; EG-NEXT:     AND_INT * T1.W, PV.Y, literal.y,
-; EG-NEXT:    8(1.121039e-44), -65536(nan)
-; EG-NEXT:     AND_INT * T0.W, PV.W, literal.x,
-; EG-NEXT:    65535(9.183409e-41), 0(0.000000e+00)
-; EG-NEXT:     OR_INT * T0.W, T1.W, PV.W,
-; EG-NEXT:     MOV * T9.X, PV.W,
-; EG-NEXT:     MOV T0.Y, PV.X,
-; EG-NEXT:     ASHR * T0.W, T19.Z, literal.x,
-; EG-NEXT:    24(3.363116e-44), 0(0.000000e+00)
-; EG-NEXT:     AND_INT T1.W, PV.Y, literal.x,
-; EG-NEXT:     LSHL * T0.W, PV.W, literal.y,
-; EG-NEXT:    65535(9.183409e-41), 16(2.242078e-44)
-; EG-NEXT:    ALU clause starting at 117:
-; EG-NEXT:     OR_INT * T19.Y, T1.W, T0.W,
-; EG-NEXT:     MOV T9.X, PV.Y,
-; EG-NEXT:     MOV T0.Y, T4.X,
-; EG-NEXT:     BFE_INT * T0.W, T19.W, 0.0, literal.x,
+; EG-NEXT:     BFE_INT T8.X, T7.X, 0.0, literal.x,
+; EG-NEXT:     BFE_INT T9.Z, T7.W, 0.0, literal.x,
+; EG-NEXT:     LSHR * T0.W, T7.Y, literal.x,
 ; EG-NEXT:    8(1.121039e-44), 0(0.000000e+00)
-; EG-NEXT:     AND_INT T1.W, PV.Y, literal.x,
-; EG-NEXT:     AND_INT * T0.W, PV.W, literal.y,
-; EG-NEXT:    -65536(nan), 65535(9.183409e-41)
-; EG-NEXT:     OR_INT * T0.W, PV.W, PS,
-; EG-NEXT:     MOV * T4.X, PV.W,
-; EG-NEXT:     MOV T0.Y, PV.X,
-; EG-NEXT:     LSHR * T0.W, T19.W, literal.x,
+; EG-NEXT:     BFE_INT T9.X, T7.Z, 0.0, literal.x,
+; EG-NEXT:     LSHR T0.Z, T7.W, literal.x,
+; EG-NEXT:     BFE_INT T8.W, PV.W, 0.0, literal.x,
+; EG-NEXT:     LSHR * T0.W, T7.X, literal.x,
 ; EG-NEXT:    8(1.121039e-44), 0(0.000000e+00)
-; EG-NEXT:     BFE_INT T0.W, PV.W, 0.0, literal.x,
-; EG-NEXT:     AND_INT * T1.W, PV.Y, literal.y,
-; EG-NEXT:    8(1.121039e-44), 65535(9.183409e-41)
-; EG-NEXT:     LSHL * T0.W, PV.W, literal.x,
-; EG-NEXT:    16(2.242078e-44), 0(0.000000e+00)
-; EG-NEXT:     OR_INT * T0.W, T1.W, PV.W,
-; EG-NEXT:     MOV T4.X, PV.W,
-; EG-NEXT:     MOV T0.Y, T5.X,
-; EG-NEXT:     LSHR * T0.W, T19.W, literal.x,
-; EG-NEXT:    16(2.242078e-44), 0(0.000000e+00)
-; EG-NEXT:     BFE_INT T0.W, PV.W, 0.0, literal.x,
-; EG-NEXT:     AND_INT * T1.W, PV.Y, literal.y,
-; EG-NEXT:    8(1.121039e-44), -65536(nan)
-; EG-NEXT:     AND_INT * T0.W, PV.W, literal.x,
-; EG-NEXT:    65535(9.183409e-41), 0(0.000000e+00)
-; EG-NEXT:     OR_INT * T0.W, T1.W, PV.W,
-; EG-NEXT:     MOV * T5.X, PV.W,
-; EG-NEXT:     MOV T0.Y, PV.X,
-; EG-NEXT:     ASHR T0.W, T19.W, literal.x,
-; EG-NEXT:     ADD_INT * T1.W, KC0[2].Y, literal.y,
-; EG-NEXT:    24(3.363116e-44), 16(2.242078e-44)
-; EG-NEXT:     LSHR T21.X, PS, literal.x,
-; EG-NEXT:     AND_INT T1.W, PV.Y, literal.y,
-; EG-NEXT:     LSHL * T0.W, PV.W, literal.z,
-; EG-NEXT:    2(2.802597e-45), 65535(9.183409e-41)
+; EG-NEXT:     LSHR T7.X, KC0[2].Y, literal.x,
+; EG-NEXT:     BFE_INT T8.Y, PS, 0.0, literal.y,
+; EG-NEXT:     LSHR T1.Z, T7.Z, literal.y,
+; EG-NEXT:     BFE_INT T9.W, PV.Z, 0.0, literal.y,
+; EG-NEXT:     ADD_INT * T0.W, KC0[2].Y, literal.z,
+; EG-NEXT:    2(2.802597e-45), 8(1.121039e-44)
 ; EG-NEXT:    16(2.242078e-44), 0(0.000000e+00)
-; EG-NEXT:     LSHR T22.X, KC0[2].Y, literal.x,
-; EG-NEXT:     OR_INT * T19.W, PV.W, PS,
-; EG-NEXT:    2(2.802597e-45), 0(0.000000e+00)
-; EG-NEXT:     MOV T5.X, PV.W,
-; EG-NEXT:     MOV * T20.X, T16.X,
-; EG-NEXT:     MOV * T20.Z, T12.X,
-; EG-NEXT:     MOV T19.X, T8.X,
-; EG-NEXT:     MOV * T19.Z, T4.X, BS:VEC_120/SCL_212
+; EG-NEXT:     LSHR T10.X, PS, literal.x,
+; EG-NEXT:     BFE_INT * T9.Y, PV.Z, 0.0, literal.y,
+; EG-NEXT:    2(2.802597e-45), 8(1.121039e-44)
 ;
 ; GFX12-LABEL: constant_sextload_v16i8_to_v16i16:
 ; GFX12:       ; %bb.0:
@@ -11195,276 +10773,58 @@ define amdgpu_kernel void @constant_zextload_v32i8_to_v32i16(ptr addrspace(1) %o
 ;
 ; EG-LABEL: constant_zextload_v32i8_to_v32i16:
 ; EG:       ; %bb.0:
-; EG-NEXT:    ALU 1, @14, KC0[CB0:0-32], KC1[]
-; EG-NEXT:    TEX 1 @10
-; EG-NEXT:    ALU 103, @16, KC0[], KC1[]
-; EG-NEXT:    ALU 104, @120, KC0[], KC1[]
-; EG-NEXT:    ALU 41, @225, KC0[CB0:0-32], KC1[]
-; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T36.XYZW, T42.X, 0
-; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T37.XYZW, T41.X, 0
-; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T38.XYZW, T40.X, 0
-; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T35.XYZW, T39.X, 1
+; EG-NEXT:    ALU 0, @12, KC0[CB0:0-32], KC1[]
+; EG-NEXT:    TEX 1 @8
+; EG-NEXT:    ALU 37, @13, KC0[CB0:0-32], KC1[]
+; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T17.XYZW, T18.X, 0
+; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T15.XYZW, T12.X, 0
+; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T14.XYZW, T16.X, 0
+; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T13.XYZW, T11.X, 1
 ; EG-NEXT:    CF_END
-; EG-NEXT:    Fetch clause starting at 10:
-; EG-NEXT:     VTX_READ_128 T37.XYZW, T35.X, 16, #1
-; EG-NEXT:     VTX_READ_128 T35.XYZW, T35.X, 0, #1
-; EG-NEXT:    ALU clause starting at 14:
-; EG-NEXT:     MOV * T0.Y, T16.X,
-; EG-NEXT:     MOV * T35.X, KC0[2].Z,
-; EG-NEXT:    ALU clause starting at 16:
-; EG-NEXT:     AND_INT T0.W, T37.X, literal.x,
-; EG-NEXT:     AND_INT * T1.W, T0.Y, literal.y,
-; EG-NEXT:    255(3.573311e-43), -65536(nan)
-; EG-NEXT:     OR_INT * T0.W, PS, PV.W,
-; EG-NEXT:     MOV * T16.X, PV.W,
-; EG-NEXT:     MOV T0.Y, PV.X,
-; EG-NEXT:     LSHL * T0.W, T37.X, literal.x,
-; EG-NEXT:    8(1.121039e-44), 0(0.000000e+00)
-; EG-NEXT:     AND_INT T1.W, PV.Y, literal.x,
-; EG-NEXT:     AND_INT * T0.W, PV.W, literal.y,
-; EG-NEXT:    65535(9.183409e-41), 16711680(2.341805e-38)
-; EG-NEXT:     OR_INT * T0.W, PV.W, PS,
-; EG-NEXT:     MOV T16.X, PV.W,
-; EG-NEXT:     MOV T0.Y, T17.X,
+; EG-NEXT:    Fetch clause starting at 8:
+; EG-NEXT:     VTX_READ_128 T12.XYZW, T11.X, 16, #1
+; EG-NEXT:     VTX_READ_128 T11.XYZW, T11.X, 0, #1
+; EG-NEXT:    ALU clause starting at 12:
+; EG-NEXT:     MOV * T11.X, KC0[2].Z,
+; EG-NEXT:    ALU clause starting at 13:
 ; EG-NEXT:     MOV * T0.W, literal.x,
 ; EG-NEXT:    8(1.121039e-44), 0(0.000000e+00)
-; EG-NEXT:     BFE_UINT T1.W, T37.X, literal.x, PV.W,
-; EG-NEXT:     AND_INT * T2.W, PV.Y, literal.y,
-; EG-NEXT:    16(2.242078e-44), -65536(nan)
-; EG-NEXT:     OR_INT * T1.W, PS, PV.W,
-; EG-NEXT:     MOV * T17.X, PV.W,
-; EG-NEXT:     MOV T0.Y, PV.X,
-; EG-NEXT:     LSHR * T1.W, T37.X, literal.x,
-; EG-NEXT:    8(1.121039e-44), 0(0.000000e+00)
-; EG-NEXT:     AND_INT T2.W, PV.Y, literal.x,
-; EG-NEXT:     AND_INT * T1.W, PV.W, literal.y,
-; EG-NEXT:    65535(9.183409e-41), 16711680(2.341805e-38)
-; EG-NEXT:     OR_INT * T36.Y, PV.W, PS,
-; EG-NEXT:     MOV T17.X, PV.Y,
-; EG-NEXT:     MOV * T0.Y, T12.X,
-; EG-NEXT:     AND_INT T1.W, PV.Y, literal.x,
-; EG-NEXT:     AND_INT * T2.W, T37.Y, literal.y,
-; EG-NEXT:    -65536(nan), 255(3.573311e-43)
-; EG-NEXT:     OR_INT * T1.W, PV.W, PS,
-; EG-NEXT:     MOV * T12.X, PV.W,
-; EG-NEXT:     MOV T0.Y, PV.X,
-; EG-NEXT:     LSHL * T1.W, T37.Y, literal.x,
-; EG-NEXT:    8(1.121039e-44), 0(0.000000e+00)
-; EG-NEXT:     AND_INT T2.W, PV.Y, literal.x,
-; EG-NEXT:     AND_INT * T1.W, PV.W, literal.y,
-; EG-NEXT:    65535(9.183409e-41), 16711680(2.341805e-38)
-; EG-NEXT:     OR_INT * T1.W, PV.W, PS,
-; EG-NEXT:     MOV T12.X, PV.W,
-; EG-NEXT:     MOV T0.Y, T13.X,
-; EG-NEXT:     BFE_UINT * T1.W, T37.Y, literal.x, T0.W,
-; EG-NEXT:    16(2.242078e-44), 0(0.000000e+00)
-; EG-NEXT:     AND_INT * T2.W, PV.Y, literal.x,
-; EG-NEXT:    -65536(nan), 0(0.000000e+00)
-; EG-NEXT:     OR_INT * T1.W, PV.W, T1.W,
-; EG-NEXT:     MOV * T13.X, PV.W,
-; EG-NEXT:     MOV T0.Y, PV.X,
-; EG-NEXT:     LSHR * T1.W, T37.Y, literal.x,
-; EG-NEXT:    8(1.121039e-44), 0(0.000000e+00)
-; EG-NEXT:     AND_INT T2.W, PV.Y, literal.x,
-; EG-NEXT:     AND_INT * T1.W, PV.W, literal.y,
-; EG-NEXT:    65535(9.183409e-41), 16711680(2.341805e-38)
-; EG-NEXT:     OR_INT * T36.W, PV.W, PS,
-; EG-NEXT:     MOV T13.X, PV.W,
-; EG-NEXT:     MOV * T0.Y, T8.X,
-; EG-NEXT:     AND_INT T1.W, PV.Y, literal.x,
-; EG-NEXT:     AND_INT * T2.W, T37.Z, literal.y,
-; EG-NEXT:    -65536(nan), 255(3.573311e-43)
-; EG-NEXT:     OR_INT * T1.W, PV.W, PS,
-; EG-NEXT:     MOV * T8.X, PV.W,
-; EG-NEXT:     MOV T0.Y, PV.X,
-; EG-NEXT:     LSHL * T1.W, T37.Z, literal.x,
-; EG-NEXT:    8(1.121039e-44), 0(0.000000e+00)
-; EG-NEXT:     AND_INT T2.W, PV.Y, literal.x,
-; EG-NEXT:     AND_INT * T1.W, PV.W, literal.y,
-; EG-NEXT:    65535(9.183409e-41), 16711680(2.341805e-38)
-; EG-NEXT:     OR_INT * T1.W, PV.W, PS,
-; EG-NEXT:     MOV T8.X, PV.W,
-; EG-NEXT:     MOV T0.Y, T9.X,
-; EG-NEXT:     BFE_UINT * T1.W, T37.Z, literal.x, T0.W,
-; EG-NEXT:    16(2.242078e-44), 0(0.000000e+00)
-; EG-NEXT:     AND_INT * T2.W, PV.Y, literal.x,
-; EG-NEXT:    -65536(nan), 0(0.000000e+00)
-; EG-NEXT:     OR_INT * T1.W, PV.W, T1.W,
-; EG-NEXT:     MOV * T9.X, PV.W,
-; EG-NEXT:     MOV T0.Y, PV.X,
-; EG-NEXT:     LSHR * T1.W, T37.Z, literal.x,
-; EG-NEXT:    8(1.121039e-44), 0(0.000000e+00)
-; EG-NEXT:     AND_INT T2.W, PV.Y, literal.x,
-; EG-NEXT:     AND_INT * T1.W, PV.W, literal.y,
-; EG-NEXT:    65535(9.183409e-41), 16711680(2.341805e-38)
-; EG-NEXT:     OR_INT * T37.Y, PV.W, PS,
-; EG-NEXT:     MOV T9.X, PV.Y,
-; EG-NEXT:     MOV * T0.Y, T4.X,
-; EG-NEXT:     AND_INT T1.W, PV.Y, literal.x,
-; EG-NEXT:     AND_INT * T2.W, T37.W, literal.y,
-; EG-NEXT:    -65536(nan), 255(3.573311e-43)
-; EG-NEXT:     OR_INT * T1.W, PV.W, PS,
-; EG-NEXT:     MOV * T4.X, PV.W,
-; EG-NEXT:     MOV T0.Y, PV.X,
-; EG-NEXT:     LSHL * T1.W, T37.W, literal.x,
-; EG-NEXT:    8(1.121039e-44), 0(0.000000e+00)
-; EG-NEXT:     AND_INT T2.W, PV.Y, literal.x,
-; EG-NEXT:     AND_INT * T1.W, PV.W, literal.y,
-; EG-NEXT:    65535(9.183409e-41), 16711680(2.341805e-38)
-; EG-NEXT:     OR_INT * T1.W, PV.W, PS,
-; EG-NEXT:     MOV T4.X, PV.W,
-; EG-NEXT:     MOV T0.Y, T5.X,
-; EG-NEXT:     BFE_UINT * T1.W, T37.W, literal.x, T0.W,
-; EG-NEXT:    16(2.242078e-44), 0(0.000000e+00)
-; EG-NEXT:    ALU clause starting at 120:
-; EG-NEXT:     AND_INT * T2.W, T0.Y, literal.x,
-; EG-NEXT:    -65536(nan), 0(0.000000e+00)
-; EG-NEXT:     OR_INT * T1.W, PV.W, T1.W,
-; EG-NEXT:     MOV * T5.X, PV.W,
-; EG-NEXT:     MOV T0.Y, PV.X,
-; EG-NEXT:     LSHR * T1.W, T37.W, literal.x,
-; EG-NEXT:    8(1.121039e-44), 0(0.000000e+00)
-; EG-NEXT:     AND_INT T2.W, PV.Y, literal.x,
-; EG-NEXT:     AND_INT * T1.W, PV.W, literal.y,
-; EG-NEXT:    65535(9.183409e-41), 16711680(2.341805e-38)
-; EG-NEXT:     OR_INT * T37.W, PV.W, PS,
-; EG-NEXT:     MOV T5.X, PV.W,
-; EG-NEXT:     MOV * T0.Y, T32.X,
-; EG-NEXT:     AND_INT T1.W, PV.Y, literal.x,
-; EG-NEXT:     AND_INT * T2.W, T35.X, literal.y,
-; EG-NEXT:    -65536(nan), 255(3.573311e-43)
-; EG-NEXT:     OR_INT * T1.W, PV.W, PS,
-; EG-NEXT:     MOV * T32.X, PV.W,
-; EG-NEXT:     MOV T0.Y, PV.X,
-; EG-NEXT:     LSHL * T1.W, T35.X, literal.x,
-; EG-NEXT:    8(1.121039e-44), 0(0.000000e+00)
-; EG-NEXT:     AND_INT T2.W, PV.Y, literal.x,
-; EG-NEXT:     AND_INT * T1.W, PV.W, literal.y,
-; EG-NEXT:    65535(9.183409e-41), 16711680(2.341805e-38)
-; EG-NEXT:     OR_INT * T1.W, PV.W, PS,
-; EG-NEXT:     MOV T32.X, PV.W,
-; EG-NEXT:     MOV T0.Y, T33.X,
-; EG-NEXT:     BFE_UINT * T1.W, T35.X, literal.x, T0.W, BS:VEC_120/SCL_212
-; EG-NEXT:    16(2.242078e-44), 0(0.000000e+00)
-; EG-NEXT:     AND_INT * T2.W, PV.Y, literal.x,
-; EG-NEXT:    -65536(nan), 0(0.000000e+00)
-; EG-NEXT:     OR_INT * T1.W, PV.W, T1.W,
-; EG-NEXT:     MOV * T33.X, PV.W,
-; EG-NEXT:     MOV T0.Y, PV.X,
-; EG-NEXT:     LSHR * T1.W, T35.X, literal.x,
-; EG-NEXT:    8(1.121039e-44), 0(0.000000e+00)
-; EG-NEXT:     AND_INT T2.W, PV.Y, literal.x,
-; EG-NEXT:     AND_INT * T1.W, PV.W, literal.y,
-; EG-NEXT:    65535(9.183409e-41), 16711680(2.341805e-38)
-; EG-NEXT:     OR_INT * T38.Y, PV.W, PS,
-; EG-NEXT:     MOV T33.X, PV.Y,
-; EG-NEXT:     MOV * T0.Y, T28.X,
-; EG-NEXT:     AND_INT T1.W, PV.Y, literal.x,
-; EG-NEXT:     AND_INT * T2.W, T35.Y, literal.y,
-; EG-NEXT:    -65536(nan), 255(3.573311e-43)
-; EG-NEXT:     OR_INT * T1.W, PV.W, PS,
-; EG-NEXT:     MOV * T28.X, PV.W,
-; EG-NEXT:     MOV T0.Y, PV.X,
-; EG-NEXT:     LSHL * T1.W, T35.Y, literal.x,
-; EG-NEXT:    8(1.121039e-44), 0(0.000000e+00)
-; EG-NEXT:     AND_INT T2.W, PV.Y, literal.x,
-; EG-NEXT:     AND_INT * T1.W, PV.W, literal.y,
-; EG-NEXT:    65535(9.183409e-41), 16711680(2.341805e-38)
-; EG-NEXT:     OR_INT * T1.W, PV.W, PS,
-; EG-NEXT:     MOV T28.X, PV.W,
-; EG-NEXT:     MOV T0.Y, T29.X,
-; EG-NEXT:     BFE_UINT * T1.W, T35.Y, literal.x, T0.W,
-; EG-NEXT:    16(2.242078e-44), 0(0.000000e+00)
-; EG-NEXT:     AND_INT * T2.W, PV.Y, literal.x,
-; EG-NEXT:    -65536(nan), 0(0.000000e+00)
-; EG-NEXT:     OR_INT * T1.W, PV.W, T1.W,
-; EG-NEXT:     MOV * T29.X, PV.W,
-; EG-NEXT:     MOV T0.Y, PV.X,
-; EG-NEXT:     LSHR * T1.W, T35.Y, literal.x,
-; EG-NEXT:    8(1.121039e-44), 0(0.000000e+00)
-; EG-NEXT:     AND_INT T2.W, PV.Y, literal.x,
-; EG-NEXT:     AND_INT * T1.W, PV.W, literal.y,
-; EG-NEXT:    65535(9.183409e-41), 16711680(2.341805e-38)
-; EG-NEXT:     OR_INT * T38.W, PV.W, PS,
-; EG-NEXT:     MOV T29.X, PV.W,
-; EG-NEXT:     MOV * T0.Y, T24.X,
-; EG-NEXT:     AND_INT T1.W, PV.Y, literal.x,
-; EG-NEXT:     AND_INT * T2.W, T35.Z, literal.y,
-; EG-NEXT:    -65536(nan), 255(3.573311e-43)
-; EG-NEXT:     OR_INT * T1.W, PV.W, PS,
-; EG-NEXT:     MOV * T24.X, PV.W,
-; EG-NEXT:     MOV T0.Y, PV.X,
-; EG-NEXT:     LSHL * T1.W, T35.Z, literal.x,
+; EG-NEXT:     BFE_UINT * T13.W, T11.Y, literal.x, PV.W,
 ; EG-NEXT:    8(1.121039e-44), 0(0.000000e+00)
-; EG-NEXT:     AND_INT T2.W, PV.Y, literal.x,
-; EG-NEXT:     AND_INT * T1.W, PV.W, literal.y,
-; EG-NEXT:    65535(9.183409e-41), 16711680(2.341805e-38)
-; EG-NEXT:     OR_INT * T1.W, PV.W, PS,
-; EG-NEXT:     MOV T24.X, PV.W,
-; EG-NEXT:     MOV T0.Y, T25.X,
-; EG-NEXT:     BFE_UINT * T1.W, T35.Z, literal.x, T0.W,
+; EG-NEXT:     BFE_UINT T13.Y, T11.X, literal.x, T0.W,
+; EG-NEXT:     AND_INT T13.Z, T11.Y, literal.y,
+; EG-NEXT:     BFE_UINT * T14.W, T11.W, literal.x, T0.W,
+; EG-NEXT:    8(1.121039e-44), 255(3.573311e-43)
+; EG-NEXT:     AND_INT T13.X, T11.X, literal.x,
+; EG-NEXT:     BFE_UINT T14.Y, T11.Z, literal.y, T0.W,
+; EG-NEXT:     LSHR * T11.X, KC0[2].Y, literal.z,
+; EG-NEXT:    255(3.573311e-43), 8(1.121039e-44)
+; EG-NEXT:    2(2.802597e-45), 0(0.000000e+00)
+; EG-NEXT:     AND_INT T14.Z, T11.W, literal.x,
+; EG-NEXT:     BFE_UINT * T15.W, T12.Y, literal.y, T0.W,
+; EG-NEXT:    255(3.573311e-43), 8(1.121039e-44)
+; EG-NEXT:     AND_INT T14.X, T11.Z, literal.x,
+; EG-NEXT:     BFE_UINT T15.Y, T12.X, literal.y, T0.W,
+; EG-NEXT:     ADD_INT * T1.W, KC0[2].Y, literal.z,
+; EG-NEXT:    255(3.573311e-43), 8(1.121039e-44)
 ; EG-NEXT:    16(2.242078e-44), 0(0.000000e+00)
-; EG-NEXT:     AND_INT * T2.W, PV.Y, literal.x,
-; EG-NEXT:    -65536(nan), 0(0.000000e+00)
-; EG-NEXT:     OR_INT * T1.W, PV.W, T1.W,
-; EG-NEXT:     MOV * T25.X, PV.W,
-; EG-NEXT:     MOV T0.Y, PV.X,
-; EG-NEXT:     LSHR * T1.W, T35.Z, literal.x,
-; EG-NEXT:    8(1.121039e-44), 0(0.000000e+00)
-; EG-NEXT:     AND_INT T2.W, PV.Y, literal.x,
-; EG-NEXT:     AND_INT * T1.W, PV.W, literal.y,
-; EG-NEXT:    65535(9.183409e-41), 16711680(2.341805e-38)
-; EG-NEXT:     OR_INT * T35.Y, PV.W, PS,
-; EG-NEXT:     MOV T25.X, PV.Y,
-; EG-NEXT:     MOV * T0.Y, T20.X,
-; EG-NEXT:     AND_INT T1.W, PV.Y, literal.x,
-; EG-NEXT:     AND_INT * T2.W, T35.W, literal.y,
-; EG-NEXT:    -65536(nan), 255(3.573311e-43)
-; EG-NEXT:     OR_INT * T1.W, PV.W, PS,
-; EG-NEXT:     MOV * T20.X, PV.W,
-; EG-NEXT:    ALU clause starting at 225:
-; EG-NEXT:     MOV T0.Y, T20.X,
-; EG-NEXT:     LSHL * T1.W, T35.W, literal.x,
+; EG-NEXT:     LSHR T16.X, PV.W, literal.x,
+; EG-NEXT:     AND_INT T15.Z, T12.Y, literal.y,
+; EG-NEXT:     BFE_UINT T17.W, T12.W, literal.z, T0.W,
+; EG-NEXT:     AND_INT * T15.X, T12.X, literal.y,
+; EG-NEXT:    2(2.802597e-45), 255(3.573311e-43)
 ; EG-NEXT:    8(1.121039e-44), 0(0.000000e+00)
-; EG-NEXT:     AND_INT T2.W, PV.Y, literal.x,
-; EG-NEXT:     AND_INT * T1.W, PV.W, literal.y,
-; EG-NEXT:    65535(9.183409e-41), 16711680(2.341805e-38)
-; EG-NEXT:     OR_INT * T1.W, PV.W, PS,
-; EG-NEXT:     MOV T20.X, PV.W,
-; EG-NEXT:     MOV T0.Y, T21.X,
-; EG-NEXT:     BFE_UINT * T0.W, T35.W, literal.x, T0.W,
-; EG-NEXT:    16(2.242078e-44), 0(0.000000e+00)
-; EG-NEXT:     AND_INT * T1.W, PV.Y, literal.x,
-; EG-NEXT:    -65536(nan), 0(0.000000e+00)
-; EG-NEXT:     OR_INT * T0.W, PV.W, T0.W,
-; EG-NEXT:     MOV * T21.X, PV.W,
-; EG-NEXT:     MOV T0.Y, PV.X,
+; EG-NEXT:     BFE_UINT T17.Y, T12.Z, literal.x, T0.W,
+; EG-NEXT:     ADD_INT * T0.W, KC0[2].Y, literal.y,
+; EG-NEXT:    8(1.121039e-44), 32(4.484155e-44)
+; EG-NEXT:     LSHR T12.X, PV.W, literal.x,
+; EG-NEXT:     AND_INT T17.Z, T12.W, literal.y,
+; EG-NEXT:     AND_INT * T17.X, T12.Z, literal.y,
+; EG-NEXT:    2(2.802597e-45), 255(3.573311e-43)
 ; EG-NEXT:     ADD_INT * T0.W, KC0[2].Y, literal.x,
-; EG-NEXT:    16(2.242078e-44), 0(0.000000e+00)
-; EG-NEXT:     LSHR T39.X, PV.W, literal.x,
-; EG-NEXT:     LSHR * T40.X, KC0[2].Y, literal.x,
-; EG-NEXT:    2(2.802597e-45), 0(0.000000e+00)
-; EG-NEXT:     LSHR T0.W, T35.W, literal.x,
-; EG-NEXT:     ADD_INT * T1.W, KC0[2].Y, literal.y,
-; EG-NEXT:    8(1.121039e-44), 48(6.726233e-44)
-; EG-NEXT:     LSHR T41.X, PS, literal.x,
-; EG-NEXT:     AND_INT T0.Z, T0.Y, literal.y,
-; EG-NEXT:     AND_INT T0.W, PV.W, literal.z,
-; EG-NEXT:     ADD_INT * T1.W, KC0[2].Y, literal.w,
-; EG-NEXT:    2(2.802597e-45), 65535(9.183409e-41)
-; EG-NEXT:    16711680(2.341805e-38), 32(4.484155e-44)
-; EG-NEXT:     LSHR T42.X, PS, literal.x,
-; EG-NEXT:     OR_INT * T35.W, PV.Z, PV.W,
+; EG-NEXT:    48(6.726233e-44), 0(0.000000e+00)
+; EG-NEXT:     LSHR * T18.X, PV.W, literal.x,
 ; EG-NEXT:    2(2.802597e-45), 0(0.000000e+00)
-; EG-NEXT:     MOV T21.X, PV.W,
-; EG-NEXT:     MOV * T36.X, T16.X,
-; EG-NEXT:     MOV * T36.Z, T12.X,
-; EG-NEXT:     MOV T37.X, T8.X,
-; EG-NEXT:     MOV T37.Z, T4.X, BS:VEC_120/SCL_212
-; EG-NEXT:     MOV * T38.X, T32.X,
-; EG-NEXT:     MOV * T38.Z, T28.X,
-; EG-NEXT:     MOV T35.X, T24.X,
-; EG-NEXT:     MOV * T35.Z, T20.X, BS:VEC_120/SCL_212
 ;
 ; GFX12-LABEL: constant_zextload_v32i8_to_v32i16:
 ; GFX12:       ; %bb.0:
@@ -11920,331 +11280,60 @@ define amdgpu_kernel void @constant_sextload_v32i8_to_v32i16(ptr addrspace(1) %o
 ;
 ; EG-LABEL: constant_sextload_v32i8_to_v32i16:
 ; EG:       ; %bb.0:
-; EG-NEXT:    ALU 1, @14, KC0[CB0:0-32], KC1[]
-; EG-NEXT:    TEX 1 @10
-; EG-NEXT:    ALU 104, @16, KC0[], KC1[]
-; EG-NEXT:    ALU 104, @121, KC0[], KC1[]
-; EG-NEXT:    ALU 95, @226, KC0[CB0:0-32], KC1[]
-; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T36.XYZW, T42.X, 0
-; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T37.XYZW, T41.X, 0
-; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T38.XYZW, T40.X, 0
-; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T35.XYZW, T39.X, 1
+; EG-NEXT:    ALU 0, @12, KC0[CB0:0-32], KC1[]
+; EG-NEXT:    TEX 1 @8
+; EG-NEXT:    ALU 39, @13, KC0[CB0:0-32], KC1[]
+; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T18.XYZW, T12.X, 0
+; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T17.XYZW, T11.X, 0
+; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T16.XYZW, T14.X, 0
+; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T15.XYZW, T13.X, 1
 ; EG-NEXT:    CF_END
-; EG-NEXT:    Fetch clause starting at 10:
-; EG-NEXT:     VTX_READ_128 T37.XYZW, T35.X, 16, #1
-; EG-NEXT:     VTX_READ_128 T35.XYZW, T35.X, 0, #1
-; EG-NEXT:    ALU clause starting at 14:
-; EG-NEXT:     MOV * T0.Y, T16.X,
-; EG-NEXT:     MOV * T35.X, KC0[2].Z,
-; EG-NEXT:    ALU clause starting at 16:
-; EG-NEXT:     BFE_INT * T0.W, T37.X, 0.0, literal.x,
-; EG-NEXT:    8(1.121039e-44), 0(0.000000e+00)
-; EG-NEXT:     AND_INT T0.W, PV.W, literal.x,
-; EG-NEXT:     AND_INT * T1.W, T0.Y, literal.y,
-; EG-NEXT:    65535(9.183409e-41), -65536(nan)
-; EG-NEXT:     OR_INT * T0.W, PS, PV.W,
-; EG-NEXT:     MOV * T16.X, PV.W,
-; EG-NEXT:     MOV T0.Y, PV.X,
-; EG-NEXT:     LSHR * T0.W, T37.X, literal.x,
-; EG-NEXT:    8(1.121039e-44), 0(0.000000e+00)
-; EG-NEXT:     BFE_INT T0.W, PV.W, 0.0, literal.x,
-; EG-NEXT:     AND_INT * T1.W, PV.Y, literal.y,
-; EG-NEXT:    8(1.121039e-44), 65535(9.183409e-41)
-; EG-NEXT:     LSHL * T0.W, PV.W, literal.x,
-; EG-NEXT:    16(2.242078e-44), 0(0.000000e+00)
-; EG-NEXT:     OR_INT * T0.W, T1.W, PV.W,
-; EG-NEXT:     MOV T16.X, PV.W,
-; EG-NEXT:     MOV T0.Y, T17.X,
-; EG-NEXT:     LSHR * T0.W, T37.X, literal.x, BS:VEC_120/SCL_212
-; EG-NEXT:    16(2.242078e-44), 0(0.000000e+00)
-; EG-NEXT:     BFE_INT T0.W, PV.W, 0.0, literal.x,
-; EG-NEXT:     AND_INT * T1.W, PV.Y, literal.y,
-; EG-NEXT:    8(1.121039e-44), -65536(nan)
-; EG-NEXT:     AND_INT * T0.W, PV.W, literal.x,
-; EG-NEXT:    65535(9.183409e-41), 0(0.000000e+00)
-; EG-NEXT:     OR_INT * T0.W, T1.W, PV.W,
-; EG-NEXT:     MOV * T17.X, PV.W,
-; EG-NEXT:     MOV T0.Y, PV.X,
-; EG-NEXT:     ASHR * T0.W, T37.X, literal.x,
-; EG-NEXT:    24(3.363116e-44), 0(0.000000e+00)
-; EG-NEXT:     AND_INT T1.W, PV.Y, literal.x,
-; EG-NEXT:     LSHL * T0.W, PV.W, literal.y,
-; EG-NEXT:    65535(9.183409e-41), 16(2.242078e-44)
-; EG-NEXT:     OR_INT * T36.Y, PV.W, PS,
-; EG-NEXT:     MOV T17.X, PV.Y,
-; EG-NEXT:     MOV T0.Y, T12.X,
-; EG-NEXT:     BFE_INT * T0.W, T37.Y, 0.0, literal.x,
-; EG-NEXT:    8(1.121039e-44), 0(0.000000e+00)
-; EG-NEXT:     AND_INT T1.W, PV.Y, literal.x,
-; EG-NEXT:     AND_INT * T0.W, PV.W, literal.y,
-; EG-NEXT:    -65536(nan), 65535(9.183409e-41)
-; EG-NEXT:     OR_INT * T0.W, PV.W, PS,
-; EG-NEXT:     MOV * T12.X, PV.W,
-; EG-NEXT:     MOV T0.Y, PV.X,
-; EG-NEXT:     LSHR * T0.W, T37.Y, literal.x,
-; EG-NEXT:    8(1.121039e-44), 0(0.000000e+00)
-; EG-NEXT:     BFE_INT T0.W, PV.W, 0.0, literal.x,
-; EG-NEXT:     AND_INT * T1.W, PV.Y, literal.y,
-; EG-NEXT:    8(1.121039e-44), 65535(9.183409e-41)
-; EG-NEXT:     LSHL * T0.W, PV.W, literal.x,
-; EG-NEXT:    16(2.242078e-44), 0(0.000000e+00)
-; EG-NEXT:     OR_INT * T0.W, T1.W, PV.W,
-; EG-NEXT:     MOV T12.X, PV.W,
-; EG-NEXT:     MOV T0.Y, T13.X,
-; EG-NEXT:     LSHR * T0.W, T37.Y, literal.x,
-; EG-NEXT:    16(2.242078e-44), 0(0.000000e+00)
-; EG-NEXT:     BFE_INT T0.W, PV.W, 0.0, literal.x,
-; EG-NEXT:     AND_INT * T1.W, PV.Y, literal.y,
-; EG-NEXT:    8(1.121039e-44), -65536(nan)
-; EG-NEXT:     AND_INT * T0.W, PV.W, literal.x,
-; EG-NEXT:    65535(9.183409e-41), 0(0.000000e+00)
-; EG-NEXT:     OR_INT * T0.W, T1.W, PV.W,
-; EG-NEXT:     MOV * T13.X, PV.W,
-; EG-NEXT:     MOV T0.Y, PV.X,
-; EG-NEXT:     ASHR * T0.W, T37.Y, literal.x,
-; EG-NEXT:    24(3.363116e-44), 0(0.000000e+00)
-; EG-NEXT:     AND_INT T1.W, PV.Y, literal.x,
-; EG-NEXT:     LSHL * T0.W, PV.W, literal.y,
-; EG-NEXT:    65535(9.183409e-41), 16(2.242078e-44)
-; EG-NEXT:     OR_INT * T36.W, PV.W, PS,
-; EG-NEXT:     MOV T13.X, PV.W,
-; EG-NEXT:     MOV T0.Y, T8.X,
-; EG-NEXT:     BFE_INT * T0.W, T37.Z, 0.0, literal.x,
-; EG-NEXT:    8(1.121039e-44), 0(0.000000e+00)
-; EG-NEXT:     AND_INT T1.W, PV.Y, literal.x,
-; EG-NEXT:     AND_INT * T0.W, PV.W, literal.y,
-; EG-NEXT:    -65536(nan), 65535(9.183409e-41)
-; EG-NEXT:     OR_INT * T0.W, PV.W, PS,
-; EG-NEXT:     MOV * T8.X, PV.W,
-; EG-NEXT:     MOV T0.Y, PV.X,
-; EG-NEXT:     LSHR * T0.W, T37.Z, literal.x,
-; EG-NEXT:    8(1.121039e-44), 0(0.000000e+00)
-; EG-NEXT:     BFE_INT T0.W, PV.W, 0.0, literal.x,
-; EG-NEXT:     AND_INT * T1.W, PV.Y, literal.y,
-; EG-NEXT:    8(1.121039e-44), 65535(9.183409e-41)
-; EG-NEXT:     LSHL * T0.W, PV.W, literal.x,
-; EG-NEXT:    16(2.242078e-44), 0(0.000000e+00)
-; EG-NEXT:     OR_INT * T0.W, T1.W, PV.W,
-; EG-NEXT:     MOV T8.X, PV.W,
-; EG-NEXT:     MOV T0.Y, T9.X,
-; EG-NEXT:     LSHR * T0.W, T37.Z, literal.x,
-; EG-NEXT:    16(2.242078e-44), 0(0.000000e+00)
-; EG-NEXT:     BFE_INT T0.W, PV.W, 0.0, literal.x,
-; EG-NEXT:     AND_INT * T1.W, PV.Y, literal.y,
-; EG-NEXT:    8(1.121039e-44), -65536(nan)
-; EG-NEXT:     AND_INT * T0.W, PV.W, literal.x,
-; EG-NEXT:    65535(9.183409e-41), 0(0.000000e+00)
-; EG-NEXT:     OR_INT * T0.W, T1.W, PV.W,
-; EG-NEXT:     MOV * T9.X, PV.W,
-; EG-NEXT:     MOV T0.Y, PV.X,
-; EG-NEXT:     ASHR * T0.W, T37.Z, literal.x,
-; EG-NEXT:    24(3.363116e-44), 0(0.000000e+00)
-; EG-NEXT:     AND_INT T1.W, PV.Y, literal.x,
-; EG-NEXT:     LSHL * T0.W, PV.W, literal.y,
-; EG-NEXT:    65535(9.183409e-41), 16(2.242078e-44)
-; EG-NEXT:    ALU clause starting at 121:
-; EG-NEXT:     OR_INT * T37.Y, T1.W, T0.W,
-; EG-NEXT:     MOV T9.X, PV.Y,
-; EG-NEXT:     MOV T0.Y, T4.X,
-; EG-NEXT:     BFE_INT * T0.W, T37.W, 0.0, literal.x,
-; EG-NEXT:    8(1.121039e-44), 0(0.000000e+00)
-; EG-NEXT:     AND_INT T1.W, PV.Y, literal.x,
-; EG-NEXT:     AND_INT * T0.W, PV.W, literal.y,
-; EG-NEXT:    -65536(nan), 65535(9.183409e-41)
-; EG-NEXT:     OR_INT * T0.W, PV.W, PS,
-; EG-NEXT:     MOV * T4.X, PV.W,
-; EG-NEXT:     MOV T0.Y, PV.X,
-; EG-NEXT:     LSHR * T0.W, T37.W, literal.x,
-; EG-NEXT:    8(1.121039e-44), 0(0.000000e+00)
-; EG-NEXT:     BFE_INT T0.W, PV.W, 0.0, literal.x,
-; EG-NEXT:     AND_INT * T1.W, PV.Y, literal.y,
-; EG-NEXT:    8(1.121039e-44), 65535(9.183409e-41)
-; EG-NEXT:     LSHL * T0.W, PV.W, literal.x,
-; EG-NEXT:    16(2.242078e-44), 0(0.000000e+00)
-; EG-NEXT:     OR_INT * T0.W, T1.W, PV.W,
-; EG-NEXT:     MOV T4.X, PV.W,
-; EG-NEXT:     MOV T0.Y, T5.X,
-; EG-NEXT:     LSHR * T0.W, T37.W, literal.x,
-; EG-NEXT:    16(2.242078e-44), 0(0.000000e+00)
-; EG-NEXT:     BFE_INT T0.W, PV.W, 0.0, literal.x,
-; EG-NEXT:     AND_INT * T1.W, PV.Y, literal.y,
-; EG-NEXT:    8(1.121039e-44), -65536(nan)
-; EG-NEXT:     AND_INT * T0.W, PV.W, literal.x,
-; EG-NEXT:    65535(9.183409e-41), 0(0.000000e+00)
-; EG-NEXT:     OR_INT * T0.W, T1.W, PV.W,
-; EG-NEXT:     MOV * T5.X, PV.W,
-; EG-NEXT:     MOV T0.Y, PV.X,
-; EG-NEXT:     ASHR * T0.W, T37.W, literal.x,
-; EG-NEXT:    24(3.363116e-44), 0(0.000000e+00)
-; EG-NEXT:     AND_INT T1.W, PV.Y, literal.x,
-; EG-NEXT:     LSHL * T0.W, PV.W, literal.y,
-; EG-NEXT:    65535(9.183409e-41), 16(2.242078e-44)
-; EG-NEXT:     OR_INT * T37.W, PV.W, PS,
-; EG-NEXT:     MOV T5.X, PV.W,
-; EG-NEXT:     MOV T0.Y, T32.X,
-; EG-NEXT:     BFE_INT * T0.W, T35.X, 0.0, literal.x, BS:VEC_120/SCL_212
-; EG-NEXT:    8(1.121039e-44), 0(0.000000e+00)
-; EG-NEXT:     AND_INT T1.W, PV.Y, literal.x,
-; EG-NEXT:     AND_INT * T0.W, PV.W, literal.y,
-; EG-NEXT:    -65536(nan), 65535(9.183409e-41)
-; EG-NEXT:     OR_INT * T0.W, PV.W, PS,
-; EG-NEXT:     MOV * T32.X, PV.W,
-; EG-NEXT:     MOV T0.Y, PV.X,
-; EG-NEXT:     LSHR * T0.W, T35.X, literal.x,
-; EG-NEXT:    8(1.121039e-44), 0(0.000000e+00)
-; EG-NEXT:     BFE_INT T0.W, PV.W, 0.0, literal.x,
-; EG-NEXT:     AND_INT * T1.W, PV.Y, literal.y,
-; EG-NEXT:    8(1.121039e-44), 65535(9.183409e-41)
-; EG-NEXT:     LSHL * T0.W, PV.W, literal.x,
-; EG-NEXT:    16(2.242078e-44), 0(0.000000e+00)
-; EG-NEXT:     OR_INT * T0.W, T1.W, PV.W,
-; EG-NEXT:     MOV T32.X, PV.W,
-; EG-NEXT:     MOV T0.Y, T33.X,
-; EG-NEXT:     LSHR * T0.W, T35.X, literal.x, BS:VEC_120/SCL_212
-; EG-NEXT:    16(2.242078e-44), 0(0.000000e+00)
-; EG-NEXT:     BFE_INT T0.W, PV.W, 0.0, literal.x,
-; EG-NEXT:     AND_INT * T1.W, PV.Y, literal.y,
-; EG-NEXT:    8(1.121039e-44), -65536(nan)
-; EG-NEXT:     AND_INT * T0.W, PV.W, literal.x,
-; EG-NEXT:    65535(9.183409e-41), 0(0.000000e+00)
-; EG-NEXT:     OR_INT * T0.W, T1.W, PV.W,
-; EG-NEXT:     MOV * T33.X, PV.W,
-; EG-NEXT:     MOV T0.Y, PV.X,
-; EG-NEXT:     ASHR * T0.W, T35.X, literal.x,
-; EG-NEXT:    24(3.363116e-44), 0(0.000000e+00)
-; EG-NEXT:     AND_INT T1.W, PV.Y, literal.x,
-; EG-NEXT:     LSHL * T0.W, PV.W, literal.y,
-; EG-NEXT:    65535(9.183409e-41), 16(2.242078e-44)
-; EG-NEXT:     OR_INT * T38.Y, PV.W, PS,
-; EG-NEXT:     MOV T33.X, PV.Y,
-; EG-NEXT:     MOV T0.Y, T28.X,
-; EG-NEXT:     BFE_INT * T0.W, T35.Y, 0.0, literal.x,
-; EG-NEXT:    8(1.121039e-44), 0(0.000000e+00)
-; EG-NEXT:     AND_INT T1.W, PV.Y, literal.x,
-; EG-NEXT:     AND_INT * T0.W, PV.W, literal.y,
-; EG-NEXT:    -65536(nan), 65535(9.183409e-41)
-; EG-NEXT:     OR_INT * T0.W, PV.W, PS,
-; EG-NEXT:     MOV * T28.X, PV.W,
-; EG-NEXT:     MOV T0.Y, PV.X,
-; EG-NEXT:     LSHR * T0.W, T35.Y, literal.x,
-; EG-NEXT:    8(1.121039e-44), 0(0.000000e+00)
-; EG-NEXT:     BFE_INT T0.W, PV.W, 0.0, literal.x,
-; EG-NEXT:     AND_INT * T1.W, PV.Y, literal.y,
-; EG-NEXT:    8(1.121039e-44), 65535(9.183409e-41)
-; EG-NEXT:     LSHL * T0.W, PV.W, literal.x,
-; EG-NEXT:    16(2.242078e-44), 0(0.000000e+00)
-; EG-NEXT:     OR_INT * T0.W, T1.W, PV.W,
-; EG-NEXT:     MOV T28.X, PV.W,
-; EG-NEXT:     MOV T0.Y, T29.X,
-; EG-NEXT:     LSHR * T0.W, T35.Y, literal.x,
-; EG-NEXT:    16(2.242078e-44), 0(0.000000e+00)
-; EG-NEXT:     BFE_INT T0.W, PV.W, 0.0, literal.x,
-; EG-NEXT:     AND_INT * T1.W, PV.Y, literal.y,
-; EG-NEXT:    8(1.121039e-44), -65536(nan)
-; EG-NEXT:     AND_INT * T0.W, PV.W, literal.x,
-; EG-NEXT:    65535(9.183409e-41), 0(0.000000e+00)
-; EG-NEXT:     OR_INT * T0.W, T1.W, PV.W,
-; EG-NEXT:     MOV * T29.X, PV.W,
-; EG-NEXT:     MOV T0.Y, PV.X,
-; EG-NEXT:     ASHR * T0.W, T35.Y, literal.x,
-; EG-NEXT:    24(3.363116e-44), 0(0.000000e+00)
-; EG-NEXT:    ALU clause starting at 226:
-; EG-NEXT:     AND_INT T1.W, T0.Y, literal.x,
-; EG-NEXT:     LSHL * T0.W, T0.W, literal.y,
-; EG-NEXT:    65535(9.183409e-41), 16(2.242078e-44)
-; EG-NEXT:     OR_INT * T38.W, PV.W, PS,
-; EG-NEXT:     MOV T29.X, PV.W,
-; EG-NEXT:     MOV T0.Y, T24.X,
-; EG-NEXT:     BFE_INT * T0.W, T35.Z, 0.0, literal.x,
-; EG-NEXT:    8(1.121039e-44), 0(0.000000e+00)
-; EG-NEXT:     AND_INT T1.W, PV.Y, literal.x,
-; EG-NEXT:     AND_INT * T0.W, PV.W, literal.y,
-; EG-NEXT:    -65536(nan), 65535(9.183409e-41)
-; EG-NEXT:     OR_INT * T0.W, PV.W, PS,
-; EG-NEXT:     MOV * T24.X, PV.W,
-; EG-NEXT:     MOV T0.Y, PV.X,
-; EG-NEXT:     LSHR * T0.W, T35.Z, literal.x,
+; EG-NEXT:    Fetch clause starting at 8:
+; EG-NEXT:     VTX_READ_128 T12.XYZW, T11.X, 16, #1
+; EG-NEXT:     VTX_READ_128 T11.XYZW, T11.X, 0, #1
+; EG-NEXT:    ALU clause starting at 12:
+; EG-NEXT:     MOV * T11.X, KC0[2].Z,
+; EG-NEXT:    ALU clause starting at 13:
+; EG-NEXT:     LSHR T13.X, KC0[2].Y, literal.x,
+; EG-NEXT:     ADD_INT * T0.W, KC0[2].Y, literal.y,
+; EG-NEXT:    2(2.802597e-45), 16(2.242078e-44)
+; EG-NEXT:     LSHR T14.X, PV.W, literal.x,
+; EG-NEXT:     BFE_INT * T15.Z, T11.Y, 0.0, literal.y,
+; EG-NEXT:    2(2.802597e-45), 8(1.121039e-44)
+; EG-NEXT:     BFE_INT T15.X, T11.X, 0.0, literal.x,
+; EG-NEXT:     LSHR T0.Y, T12.W, literal.x,
+; EG-NEXT:     BFE_INT T16.Z, T11.W, 0.0, literal.x, BS:VEC_120/SCL_212
+; EG-NEXT:     LSHR T0.W, T12.Y, literal.x,
+; EG-NEXT:     LSHR * T1.W, T11.Y, literal.x,
 ; EG-NEXT:    8(1.121039e-44), 0(0.000000e+00)
-; EG-NEXT:     BFE_INT T0.W, PV.W, 0.0, literal.x,
-; EG-NEXT:     AND_INT * T1.W, PV.Y, literal.y,
-; EG-NEXT:    8(1.121039e-44), 65535(9.183409e-41)
-; EG-NEXT:     LSHL * T0.W, PV.W, literal.x,
-; EG-NEXT:    16(2.242078e-44), 0(0.000000e+00)
-; EG-NEXT:     OR_INT * T0.W, T1.W, PV.W,
-; EG-NEXT:     MOV T24.X, PV.W,
-; EG-NEXT:     MOV T0.Y, T25.X,
-; EG-NEXT:     LSHR * T0.W, T35.Z, literal.x,
-; EG-NEXT:    16(2.242078e-44), 0(0.000000e+00)
-; EG-NEXT:     BFE_INT T0.W, PV.W, 0.0, literal.x,
-; EG-NEXT:     AND_INT * T1.W, PV.Y, literal.y,
-; EG-NEXT:    8(1.121039e-44), -65536(nan)
-; EG-NEXT:     AND_INT * T0.W, PV.W, literal.x,
-; EG-NEXT:    65535(9.183409e-41), 0(0.000000e+00)
-; EG-NEXT:     OR_INT * T0.W, T1.W, PV.W,
-; EG-NEXT:     MOV * T25.X, PV.W,
-; EG-NEXT:     MOV T0.Y, PV.X,
-; EG-NEXT:     ASHR * T0.W, T35.Z, literal.x,
-; EG-NEXT:    24(3.363116e-44), 0(0.000000e+00)
-; EG-NEXT:     AND_INT T1.W, PV.Y, literal.x,
-; EG-NEXT:     LSHL * T0.W, PV.W, literal.y,
-; EG-NEXT:    65535(9.183409e-41), 16(2.242078e-44)
-; EG-NEXT:     OR_INT * T35.Y, PV.W, PS,
-; EG-NEXT:     MOV T25.X, PV.Y,
-; EG-NEXT:     MOV T0.Y, T20.X,
-; EG-NEXT:     BFE_INT * T0.W, T35.W, 0.0, literal.x,
+; EG-NEXT:     BFE_INT T16.X, T11.Z, 0.0, literal.x,
+; EG-NEXT:     LSHR T1.Y, T11.W, literal.x,
+; EG-NEXT:     BFE_INT T17.Z, T12.Y, 0.0, literal.x,
+; EG-NEXT:     BFE_INT T15.W, PS, 0.0, literal.x,
+; EG-NEXT:     LSHR * T1.W, T11.X, literal.x,
 ; EG-NEXT:    8(1.121039e-44), 0(0.000000e+00)
-; EG-NEXT:     AND_INT T1.W, PV.Y, literal.x,
-; EG-NEXT:     AND_INT * T0.W, PV.W, literal.y,
-; EG-NEXT:    -65536(nan), 65535(9.183409e-41)
-; EG-NEXT:     OR_INT * T0.W, PV.W, PS,
-; EG-NEXT:     MOV * T20.X, PV.W,
-; EG-NEXT:     MOV T0.Y, PV.X,
-; EG-NEXT:     LSHR * T0.W, T35.W, literal.x,
+; EG-NEXT:     BFE_INT T17.X, T12.X, 0.0, literal.x,
+; EG-NEXT:     BFE_INT T15.Y, PS, 0.0, literal.x,
+; EG-NEXT:     BFE_INT T18.Z, T12.W, 0.0, literal.x,
+; EG-NEXT:     BFE_INT T16.W, PV.Y, 0.0, literal.x,
+; EG-NEXT:     LSHR * T1.W, T11.Z, literal.x,
 ; EG-NEXT:    8(1.121039e-44), 0(0.000000e+00)
-; EG-NEXT:     BFE_INT T0.W, PV.W, 0.0, literal.x,
-; EG-NEXT:     AND_INT * T1.W, PV.Y, literal.y,
-; EG-NEXT:    8(1.121039e-44), 65535(9.183409e-41)
-; EG-NEXT:     LSHL * T0.W, PV.W, literal.x,
-; EG-NEXT:    16(2.242078e-44), 0(0.000000e+00)
-; EG-NEXT:     OR_INT * T0.W, T1.W, PV.W,
-; EG-NEXT:     MOV T20.X, PV.W,
-; EG-NEXT:     MOV T0.Y, T21.X,
-; EG-NEXT:     LSHR * T0.W, T35.W, literal.x,
-; EG-NEXT:    16(2.242078e-44), 0(0.000000e+00)
-; EG-NEXT:     BFE_INT T0.W, PV.W, 0.0, literal.x,
-; EG-NEXT:     AND_INT * T1.W, PV.Y, literal.y,
-; EG-NEXT:    8(1.121039e-44), -65536(nan)
-; EG-NEXT:     AND_INT * T0.W, PV.W, literal.x,
-; EG-NEXT:    65535(9.183409e-41), 0(0.000000e+00)
-; EG-NEXT:     OR_INT * T0.W, T1.W, PV.W,
-; EG-NEXT:     MOV * T21.X, PV.W,
-; EG-NEXT:     MOV T0.Y, PV.X,
-; EG-NEXT:     ADD_INT * T0.W, KC0[2].Y, literal.x,
-; EG-NEXT:    16(2.242078e-44), 0(0.000000e+00)
-; EG-NEXT:     LSHR T39.X, PV.W, literal.x,
-; EG-NEXT:     LSHR * T40.X, KC0[2].Y, literal.x,
-; EG-NEXT:    2(2.802597e-45), 0(0.000000e+00)
-; EG-NEXT:     ASHR T0.W, T35.W, literal.x,
-; EG-NEXT:     ADD_INT * T1.W, KC0[2].Y, literal.y,
-; EG-NEXT:    24(3.363116e-44), 48(6.726233e-44)
-; EG-NEXT:     LSHR T41.X, PS, literal.x,
-; EG-NEXT:     AND_INT T0.Z, T0.Y, literal.y,
-; EG-NEXT:     LSHL T0.W, PV.W, literal.z,
-; EG-NEXT:     ADD_INT * T1.W, KC0[2].Y, literal.w,
-; EG-NEXT:    2(2.802597e-45), 65535(9.183409e-41)
-; EG-NEXT:    16(2.242078e-44), 32(4.484155e-44)
-; EG-NEXT:     LSHR T42.X, PS, literal.x,
-; EG-NEXT:     OR_INT * T35.W, PV.Z, PV.W,
-; EG-NEXT:    2(2.802597e-45), 0(0.000000e+00)
-; EG-NEXT:     MOV T21.X, PV.W,
-; EG-NEXT:     MOV * T36.X, T16.X,
-; EG-NEXT:     MOV * T36.Z, T12.X,
-; EG-NEXT:     MOV T37.X, T8.X,
-; EG-NEXT:     MOV T37.Z, T4.X, BS:VEC_120/SCL_212
-; EG-NEXT:     MOV * T38.X, T32.X,
-; EG-NEXT:     MOV * T38.Z, T28.X,
-; EG-NEXT:     MOV T35.X, T24.X,
-; EG-NEXT:     MOV * T35.Z, T20.X, BS:VEC_120/SCL_212
+; EG-NEXT:     BFE_INT T18.X, T12.Z, 0.0, literal.x,
+; EG-NEXT:     BFE_INT T16.Y, PS, 0.0, literal.x,
+; EG-NEXT:     LSHR T0.Z, T12.X, literal.x,
+; EG-NEXT:     BFE_INT T17.W, T0.W, 0.0, literal.x,
+; EG-NEXT:     ADD_INT * T0.W, KC0[2].Y, literal.y,
+; EG-NEXT:    8(1.121039e-44), 32(4.484155e-44)
+; EG-NEXT:     LSHR T11.X, PS, literal.x,
+; EG-NEXT:     BFE_INT T17.Y, PV.Z, 0.0, literal.y,
+; EG-NEXT:     LSHR T0.Z, T12.Z, literal.y,
+; EG-NEXT:     BFE_INT T18.W, T0.Y, 0.0, literal.y,
+; EG-NEXT:     ADD_INT * T0.W, KC0[2].Y, literal.z,
+; EG-NEXT:    2(2.802597e-45), 8(1.121039e-44)
+; EG-NEXT:    48(6.726233e-44), 0(0.000000e+00)
+; EG-NEXT:     LSHR T12.X, PS, literal.x,
+; EG-NEXT:     BFE_INT * T18.Y, PV.Z, 0.0, literal.y,
+; EG-NEXT:    2(2.802597e-45), 8(1.121039e-44)
 ;
 ; GFX12-LABEL: constant_sextload_v32i8_to_v32i16:
 ; GFX12:       ; %bb.0:
diff --git a/llvm/test/CodeGen/AMDGPU/load-global-i16.ll b/llvm/test/CodeGen/AMDGPU/load-global-i16.ll
index 4217384cdd5ce..573338231bd57 100644
--- a/llvm/test/CodeGen/AMDGPU/load-global-i16.ll
+++ b/llvm/test/CodeGen/AMDGPU/load-global-i16.ll
@@ -254,74 +254,63 @@ define amdgpu_kernel void @global_load_v3i16(ptr addrspace(1) %out, ptr addrspac
 ;
 ; EG-LABEL: global_load_v3i16:
 ; EG:       ; %bb.0: ; %entry
-; EG-NEXT:    ALU 0, @12, KC0[CB0:0-32], KC1[]
-; EG-NEXT:    TEX 2 @6
-; EG-NEXT:    ALU 19, @13, KC0[CB0:0-32], KC1[]
-; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T6.X, T7.X, 0
-; EG-NEXT:    MEM_RAT MSKOR T5.XW, T8.X
+; EG-NEXT:    ALU 0, @10, KC0[CB0:0-32], KC1[]
+; EG-NEXT:    TEX 1 @6
+; EG-NEXT:    ALU 14, @11, KC0[CB0:0-32], KC1[]
+; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T1.X, T3.X, 0
+; EG-NEXT:    MEM_RAT MSKOR T2.XW, T0.X
 ; EG-NEXT:    CF_END
 ; EG-NEXT:    Fetch clause starting at 6:
-; EG-NEXT:     VTX_READ_16 T6.X, T5.X, 0, #1
-; EG-NEXT:     VTX_READ_16 T7.X, T5.X, 2, #1
-; EG-NEXT:     VTX_READ_16 T5.X, T5.X, 4, #1
-; EG-NEXT:    ALU clause starting at 12:
-; EG-NEXT:     MOV * T5.X, KC0[2].Z,
-; EG-NEXT:    ALU clause starting at 13:
+; EG-NEXT:     VTX_READ_16 T1.X, T0.X, 0, #1
+; EG-NEXT:     VTX_READ_16 T0.X, T0.X, 4, #1
+; EG-NEXT:    ALU clause starting at 10:
+; EG-NEXT:     MOV * T0.X, KC0[2].Z,
+; EG-NEXT:    ALU clause starting at 11:
 ; EG-NEXT:     ADD_INT * T0.W, KC0[2].Y, literal.x,
 ; EG-NEXT:    4(5.605194e-45), 0(0.000000e+00)
 ; EG-NEXT:     AND_INT T1.W, PV.W, literal.x,
-; EG-NEXT:     AND_INT * T2.W, T5.X, literal.y,
+; EG-NEXT:     AND_INT * T2.W, T0.X, literal.y,
 ; EG-NEXT:    3(4.203895e-45), 65535(9.183409e-41)
 ; EG-NEXT:     LSHL * T1.W, PV.W, literal.x,
 ; EG-NEXT:    3(4.203895e-45), 0(0.000000e+00)
-; EG-NEXT:     LSHL T5.X, T2.W, PV.W,
-; EG-NEXT:     LSHL * T5.W, literal.x, PV.W,
+; EG-NEXT:     LSHL T2.X, T2.W, PV.W,
+; EG-NEXT:     LSHL * T2.W, literal.x, PV.W,
 ; EG-NEXT:    65535(9.183409e-41), 0(0.000000e+00)
-; EG-NEXT:     MOV T5.Y, 0.0,
-; EG-NEXT:     MOV * T5.Z, 0.0,
-; EG-NEXT:     LSHR T8.X, T0.W, literal.x,
-; EG-NEXT:     LSHL T0.W, T7.X, literal.y,
-; EG-NEXT:     AND_INT * T1.W, T6.X, literal.z,
-; EG-NEXT:    2(2.802597e-45), 16(2.242078e-44)
-; EG-NEXT:    65535(9.183409e-41), 0(0.000000e+00)
-; EG-NEXT:     OR_INT T6.X, PV.W, PS,
-; EG-NEXT:     LSHR * T7.X, KC0[2].Y, literal.x,
+; EG-NEXT:     MOV T2.Y, 0.0,
+; EG-NEXT:     MOV * T2.Z, 0.0,
+; EG-NEXT:     LSHR T0.X, T0.W, literal.x,
+; EG-NEXT:     LSHR * T3.X, KC0[2].Y, literal.x,
 ; EG-NEXT:    2(2.802597e-45), 0(0.000000e+00)
 ;
 ; CM-LABEL: global_load_v3i16:
 ; CM:       ; %bb.0: ; %entry
-; CM-NEXT:    ALU 0, @12, KC0[CB0:0-32], KC1[]
-; CM-NEXT:    TEX 2 @6
-; CM-NEXT:    ALU 19, @13, KC0[CB0:0-32], KC1[]
-; CM-NEXT:    MEM_RAT MSKOR T5.XW, T8.X
-; CM-NEXT:    MEM_RAT_CACHELESS STORE_DWORD T6.X, T7.X
+; CM-NEXT:    ALU 0, @10, KC0[CB0:0-32], KC1[]
+; CM-NEXT:    TEX 1 @6
+; CM-NEXT:    ALU 15, @11, KC0[CB0:0-32], KC1[]
+; CM-NEXT:    MEM_RAT MSKOR T2.XW, T3.X
+; CM-NEXT:    MEM_RAT_CACHELESS STORE_DWORD T1.X, T0.X
 ; CM-NEXT:    CF_END
 ; CM-NEXT:    Fetch clause starting at 6:
-; CM-NEXT:     VTX_READ_16 T6.X, T5.X, 0, #1
-; CM-NEXT:     VTX_READ_16 T7.X, T5.X, 2, #1
-; CM-NEXT:     VTX_READ_16 T5.X, T5.X, 4, #1
-; CM-NEXT:    ALU clause starting at 12:
-; CM-NEXT:     MOV * T5.X, KC0[2].Z,
-; CM-NEXT:    ALU clause starting at 13:
+; CM-NEXT:     VTX_READ_16 T1.X, T0.X, 0, #1
+; CM-NEXT:     VTX_READ_16 T0.X, T0.X, 4, #1
+; CM-NEXT:    ALU clause starting at 10:
+; CM-NEXT:     MOV * T0.X, KC0[2].Z,
+; CM-NEXT:    ALU clause starting at 11:
 ; CM-NEXT:     ADD_INT * T0.W, KC0[2].Y, literal.x,
 ; CM-NEXT:    4(5.605194e-45), 0(0.000000e+00)
 ; CM-NEXT:     AND_INT * T1.W, PV.W, literal.x,
 ; CM-NEXT:    3(4.203895e-45), 0(0.000000e+00)
-; CM-NEXT:     AND_INT T0.Z, T5.X, literal.x,
+; CM-NEXT:     AND_INT T0.Z, T0.X, literal.x,
 ; CM-NEXT:     LSHL * T1.W, PV.W, literal.y,
 ; CM-NEXT:    65535(9.183409e-41), 3(4.203895e-45)
-; CM-NEXT:     LSHL T5.X, PV.Z, PV.W,
-; CM-NEXT:     LSHL * T5.W, literal.x, PV.W,
+; CM-NEXT:     LSHL T2.X, PV.Z, PV.W,
+; CM-NEXT:     LSHL * T2.W, literal.x, PV.W,
 ; CM-NEXT:    65535(9.183409e-41), 0(0.000000e+00)
-; CM-NEXT:     MOV T5.Y, 0.0,
-; CM-NEXT:     MOV * T5.Z, 0.0,
-; CM-NEXT:     LSHL T0.Z, T7.X, literal.x,
-; CM-NEXT:     AND_INT * T1.W, T6.X, literal.y, BS:VEC_120/SCL_212
-; CM-NEXT:    16(2.242078e-44), 65535(9.183409e-41)
-; CM-NEXT:     OR_INT * T6.X, PV.Z, PV.W,
-; CM-NEXT:     LSHR * T7.X, KC0[2].Y, literal.x,
+; CM-NEXT:     MOV T2.Y, 0.0,
+; CM-NEXT:     MOV * T2.Z, 0.0,
+; CM-NEXT:     LSHR * T0.X, KC0[2].Y, literal.x,
 ; CM-NEXT:    2(2.802597e-45), 0(0.000000e+00)
-; CM-NEXT:     LSHR * T8.X, T0.W, literal.x,
+; CM-NEXT:     LSHR * T3.X, T0.W, literal.x,
 ; CM-NEXT:    2(2.802597e-45), 0(0.000000e+00)
 entry:
   %ld = load <3 x i16>, ptr addrspace(1) %in
@@ -5933,17 +5922,17 @@ define amdgpu_kernel void @global_sextload_v4i16_to_v4i64(ptr addrspace(1) %out,
 ;
 ; GCN-NOHSA-VI-LABEL: global_sextload_v4i16_to_v4i64:
 ; GCN-NOHSA-VI:       ; %bb.0:
-; GCN-NOHSA-VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GCN-NOHSA-VI-NEXT:    s_mov_b32 s7, 0xf000
-; GCN-NOHSA-VI-NEXT:    s_mov_b32 s6, -1
-; GCN-NOHSA-VI-NEXT:    s_mov_b32 s10, s6
-; GCN-NOHSA-VI-NEXT:    s_mov_b32 s11, s7
+; GCN-NOHSA-VI-NEXT:    s_load_dwordx4 s[4:7], s[4:5], 0x24
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s3, 0xf000
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s2, -1
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s10, s2
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s11, s3
 ; GCN-NOHSA-VI-NEXT:    s_waitcnt lgkmcnt(0)
-; GCN-NOHSA-VI-NEXT:    s_mov_b32 s8, s2
-; GCN-NOHSA-VI-NEXT:    s_mov_b32 s9, s3
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s8, s6
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s9, s7
 ; GCN-NOHSA-VI-NEXT:    buffer_load_dwordx2 v[1:2], off, s[8:11], 0
-; GCN-NOHSA-VI-NEXT:    s_mov_b32 s4, s0
-; GCN-NOHSA-VI-NEXT:    s_mov_b32 s5, s1
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s0, s4
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s1, s5
 ; GCN-NOHSA-VI-NEXT:    s_waitcnt vmcnt(0)
 ; GCN-NOHSA-VI-NEXT:    v_mov_b32_e32 v4, v2
 ; GCN-NOHSA-VI-NEXT:    v_lshrrev_b32_e32 v5, 16, v2
@@ -5956,8 +5945,8 @@ define amdgpu_kernel void @global_sextload_v4i16_to_v4i64(ptr addrspace(1) %out,
 ; GCN-NOHSA-VI-NEXT:    v_ashrrev_i32_e32 v7, 31, v6
 ; GCN-NOHSA-VI-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
 ; GCN-NOHSA-VI-NEXT:    v_ashrrev_i32_e32 v3, 31, v2
-; GCN-NOHSA-VI-NEXT:    buffer_store_dwordx4 v[4:7], off, s[4:7], 0 offset:16
-; GCN-NOHSA-VI-NEXT:    buffer_store_dwordx4 v[0:3], off, s[4:7], 0
+; GCN-NOHSA-VI-NEXT:    buffer_store_dwordx4 v[4:7], off, s[0:3], 0 offset:16
+; GCN-NOHSA-VI-NEXT:    buffer_store_dwordx4 v[0:3], off, s[0:3], 0
 ; GCN-NOHSA-VI-NEXT:    s_endpgm
 ;
 ; EG-LABEL: global_sextload_v4i16_to_v4i64:
diff --git a/llvm/test/CodeGen/AMDGPU/load-global-i32.ll b/llvm/test/CodeGen/AMDGPU/load-global-i32.ll
index 5ce8a2b5f862e..fe693b4af67f3 100644
--- a/llvm/test/CodeGen/AMDGPU/load-global-i32.ll
+++ b/llvm/test/CodeGen/AMDGPU/load-global-i32.ll
@@ -1555,13 +1555,14 @@ define amdgpu_kernel void @global_sextload_v2i32_to_v2i64(ptr addrspace(1) %out,
 ; SI-NOHSA-NEXT:    s_waitcnt lgkmcnt(0)
 ; SI-NOHSA-NEXT:    s_mov_b32 s8, s2
 ; SI-NOHSA-NEXT:    s_mov_b32 s9, s3
-; SI-NOHSA-NEXT:    buffer_load_dwordx2 v[0:1], off, s[8:11], 0
+; SI-NOHSA-NEXT:    buffer_load_dwordx2 v[4:5], off, s[8:11], 0
 ; SI-NOHSA-NEXT:    s_mov_b32 s4, s0
 ; SI-NOHSA-NEXT:    s_mov_b32 s5, s1
 ; SI-NOHSA-NEXT:    s_waitcnt vmcnt(0)
-; SI-NOHSA-NEXT:    v_ashrrev_i32_e32 v3, 31, v1
-; SI-NOHSA-NEXT:    v_mov_b32_e32 v2, v1
-; SI-NOHSA-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
+; SI-NOHSA-NEXT:    v_ashrrev_i32_e32 v3, 31, v5
+; SI-NOHSA-NEXT:    v_ashrrev_i32_e32 v1, 31, v4
+; SI-NOHSA-NEXT:    v_mov_b32_e32 v0, v4
+; SI-NOHSA-NEXT:    v_mov_b32_e32 v2, v5
 ; SI-NOHSA-NEXT:    buffer_store_dwordx4 v[0:3], off, s[4:7], 0
 ; SI-NOHSA-NEXT:    s_endpgm
 ;
@@ -1571,14 +1572,15 @@ define amdgpu_kernel void @global_sextload_v2i32_to_v2i64(ptr addrspace(1) %out,
 ; GCNX3-HSA-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCNX3-HSA-NEXT:    v_mov_b32_e32 v0, s2
 ; GCNX3-HSA-NEXT:    v_mov_b32_e32 v1, s3
-; GCNX3-HSA-NEXT:    flat_load_dwordx2 v[0:1], v[0:1]
-; GCNX3-HSA-NEXT:    v_mov_b32_e32 v4, s0
-; GCNX3-HSA-NEXT:    v_mov_b32_e32 v5, s1
+; GCNX3-HSA-NEXT:    flat_load_dwordx2 v[4:5], v[0:1]
+; GCNX3-HSA-NEXT:    v_mov_b32_e32 v6, s0
+; GCNX3-HSA-NEXT:    v_mov_b32_e32 v7, s1
 ; GCNX3-HSA-NEXT:    s_waitcnt vmcnt(0)
-; GCNX3-HSA-NEXT:    v_ashrrev_i32_e32 v3, 31, v1
-; GCNX3-HSA-NEXT:    v_mov_b32_e32 v2, v1
-; GCNX3-HSA-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
-; GCNX3-HSA-NEXT:    flat_store_dwordx4 v[4:5], v[0:3]
+; GCNX3-HSA-NEXT:    v_ashrrev_i32_e32 v3, 31, v5
+; GCNX3-HSA-NEXT:    v_ashrrev_i32_e32 v1, 31, v4
+; GCNX3-HSA-NEXT:    v_mov_b32_e32 v0, v4
+; GCNX3-HSA-NEXT:    v_mov_b32_e32 v2, v5
+; GCNX3-HSA-NEXT:    flat_store_dwordx4 v[6:7], v[0:3]
 ; GCNX3-HSA-NEXT:    s_endpgm
 ;
 ; GCNX3-NOHSA-LABEL: global_sextload_v2i32_to_v2i64:
@@ -1591,13 +1593,14 @@ define amdgpu_kernel void @global_sextload_v2i32_to_v2i64(ptr addrspace(1) %out,
 ; GCNX3-NOHSA-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCNX3-NOHSA-NEXT:    s_mov_b32 s8, s2
 ; GCNX3-NOHSA-NEXT:    s_mov_b32 s9, s3
-; GCNX3-NOHSA-NEXT:    buffer_load_dwordx2 v[0:1], off, s[8:11], 0
+; GCNX3-NOHSA-NEXT:    buffer_load_dwordx2 v[4:5], off, s[8:11], 0
 ; GCNX3-NOHSA-NEXT:    s_mov_b32 s4, s0
 ; GCNX3-NOHSA-NEXT:    s_mov_b32 s5, s1
 ; GCNX3-NOHSA-NEXT:    s_waitcnt vmcnt(0)
-; GCNX3-NOHSA-NEXT:    v_ashrrev_i32_e32 v3, 31, v1
-; GCNX3-NOHSA-NEXT:    v_mov_b32_e32 v2, v1
-; GCNX3-NOHSA-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
+; GCNX3-NOHSA-NEXT:    v_ashrrev_i32_e32 v3, 31, v5
+; GCNX3-NOHSA-NEXT:    v_ashrrev_i32_e32 v1, 31, v4
+; GCNX3-NOHSA-NEXT:    v_mov_b32_e32 v0, v4
+; GCNX3-NOHSA-NEXT:    v_mov_b32_e32 v2, v5
 ; GCNX3-NOHSA-NEXT:    buffer_store_dwordx4 v[0:3], off, s[4:7], 0
 ; GCNX3-NOHSA-NEXT:    s_endpgm
 ;
@@ -1626,14 +1629,15 @@ define amdgpu_kernel void @global_sextload_v2i32_to_v2i64(ptr addrspace(1) %out,
 ; GCN-HSA-LABEL: global_sextload_v2i32_to_v2i64:
 ; GCN-HSA:       ; %bb.0:
 ; GCN-HSA-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
-; GCN-HSA-NEXT:    v_mov_b32_e32 v4, 0
+; GCN-HSA-NEXT:    v_mov_b32_e32 v6, 0
 ; GCN-HSA-NEXT:    s_waitcnt lgkmcnt(0)
-; GCN-HSA-NEXT:    global_load_dwordx2 v[0:1], v4, s[2:3]
+; GCN-HSA-NEXT:    global_load_dwordx2 v[4:5], v6, s[2:3]
 ; GCN-HSA-NEXT:    s_waitcnt vmcnt(0)
-; GCN-HSA-NEXT:    v_ashrrev_i32_e32 v3, 31, v1
-; GCN-HSA-NEXT:    v_mov_b32_e32 v2, v1
-; GCN-HSA-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
-; GCN-HSA-NEXT:    global_store_dwordx4 v4, v[0:3], s[0:1]
+; GCN-HSA-NEXT:    v_ashrrev_i32_e32 v3, 31, v5
+; GCN-HSA-NEXT:    v_ashrrev_i32_e32 v1, 31, v4
+; GCN-HSA-NEXT:    v_mov_b32_e32 v0, v4
+; GCN-HSA-NEXT:    v_mov_b32_e32 v2, v5
+; GCN-HSA-NEXT:    global_store_dwordx4 v6, v[0:3], s[0:1]
 ; GCN-HSA-NEXT:    s_endpgm
   %ld = load <2 x i32>, ptr addrspace(1) %in
   %ext = sext <2 x i32> %ld to <2 x i64>
@@ -1902,36 +1906,36 @@ define amdgpu_kernel void @global_sextload_v4i32_to_v4i64(ptr addrspace(1) %out,
 define amdgpu_kernel void @global_zextload_v8i32_to_v8i64(ptr addrspace(1) %out, ptr addrspace(1) %in) #0 {
 ; SI-NOHSA-LABEL: global_zextload_v8i32_to_v8i64:
 ; SI-NOHSA:       ; %bb.0:
-; SI-NOHSA-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
-; SI-NOHSA-NEXT:    s_mov_b32 s7, 0xf000
-; SI-NOHSA-NEXT:    s_mov_b32 s6, -1
-; SI-NOHSA-NEXT:    s_mov_b32 s10, s6
-; SI-NOHSA-NEXT:    s_mov_b32 s11, s7
+; SI-NOHSA-NEXT:    s_load_dwordx4 s[4:7], s[4:5], 0x9
+; SI-NOHSA-NEXT:    s_mov_b32 s3, 0xf000
+; SI-NOHSA-NEXT:    s_mov_b32 s2, -1
+; SI-NOHSA-NEXT:    s_mov_b32 s10, s2
+; SI-NOHSA-NEXT:    s_mov_b32 s11, s3
 ; SI-NOHSA-NEXT:    s_waitcnt lgkmcnt(0)
-; SI-NOHSA-NEXT:    s_mov_b32 s8, s2
-; SI-NOHSA-NEXT:    s_mov_b32 s9, s3
+; SI-NOHSA-NEXT:    s_mov_b32 s8, s6
+; SI-NOHSA-NEXT:    s_mov_b32 s9, s7
 ; SI-NOHSA-NEXT:    buffer_load_dwordx4 v[0:3], off, s[8:11], 0 offset:16
 ; SI-NOHSA-NEXT:    buffer_load_dwordx4 v[4:7], off, s[8:11], 0
 ; SI-NOHSA-NEXT:    v_mov_b32_e32 v9, 0
 ; SI-NOHSA-NEXT:    v_mov_b32_e32 v11, v9
-; SI-NOHSA-NEXT:    s_mov_b32 s4, s0
-; SI-NOHSA-NEXT:    s_mov_b32 s5, s1
+; SI-NOHSA-NEXT:    s_mov_b32 s0, s4
+; SI-NOHSA-NEXT:    s_mov_b32 s1, s5
 ; SI-NOHSA-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NOHSA-NEXT:    v_mov_b32_e32 v8, v2
 ; SI-NOHSA-NEXT:    v_mov_b32_e32 v10, v3
-; SI-NOHSA-NEXT:    buffer_store_dwordx4 v[8:11], off, s[4:7], 0 offset:48
+; SI-NOHSA-NEXT:    buffer_store_dwordx4 v[8:11], off, s[0:3], 0 offset:48
 ; SI-NOHSA-NEXT:    s_waitcnt expcnt(0)
 ; SI-NOHSA-NEXT:    v_mov_b32_e32 v8, v0
 ; SI-NOHSA-NEXT:    v_mov_b32_e32 v10, v1
-; SI-NOHSA-NEXT:    buffer_store_dwordx4 v[8:11], off, s[4:7], 0 offset:32
+; SI-NOHSA-NEXT:    buffer_store_dwordx4 v[8:11], off, s[0:3], 0 offset:32
 ; SI-NOHSA-NEXT:    s_waitcnt vmcnt(2) expcnt(0)
 ; SI-NOHSA-NEXT:    v_mov_b32_e32 v8, v6
 ; SI-NOHSA-NEXT:    v_mov_b32_e32 v10, v7
-; SI-NOHSA-NEXT:    buffer_store_dwordx4 v[8:11], off, s[4:7], 0 offset:16
+; SI-NOHSA-NEXT:    buffer_store_dwordx4 v[8:11], off, s[0:3], 0 offset:16
 ; SI-NOHSA-NEXT:    s_waitcnt expcnt(0)
 ; SI-NOHSA-NEXT:    v_mov_b32_e32 v8, v4
 ; SI-NOHSA-NEXT:    v_mov_b32_e32 v10, v5
-; SI-NOHSA-NEXT:    buffer_store_dwordx4 v[8:11], off, s[4:7], 0
+; SI-NOHSA-NEXT:    buffer_store_dwordx4 v[8:11], off, s[0:3], 0
 ; SI-NOHSA-NEXT:    s_endpgm
 ;
 ; GCNX3-HSA-LABEL: global_zextload_v8i32_to_v8i64:
@@ -1981,36 +1985,36 @@ define amdgpu_kernel void @global_zextload_v8i32_to_v8i64(ptr addrspace(1) %out,
 ;
 ; GCNX3-NOHSA-LABEL: global_zextload_v8i32_to_v8i64:
 ; GCNX3-NOHSA:       ; %bb.0:
-; GCNX3-NOHSA-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GCNX3-NOHSA-NEXT:    s_mov_b32 s7, 0xf000
-; GCNX3-NOHSA-NEXT:    s_mov_b32 s6, -1
-; GCNX3-NOHSA-NEXT:    s_mov_b32 s10, s6
-; GCNX3-NOHSA-NEXT:    s_mov_b32 s11, s7
+; GCNX3-NOHSA-NEXT:    s_load_dwordx4 s[4:7], s[4:5], 0x24
+; GCNX3-NOHSA-NEXT:    s_mov_b32 s3, 0xf000
+; GCNX3-NOHSA-NEXT:    s_mov_b32 s2, -1
+; GCNX3-NOHSA-NEXT:    s_mov_b32 s10, s2
+; GCNX3-NOHSA-NEXT:    s_mov_b32 s11, s3
 ; GCNX3-NOHSA-NEXT:    s_waitcnt lgkmcnt(0)
-; GCNX3-NOHSA-NEXT:    s_mov_b32 s8, s2
-; GCNX3-NOHSA-NEXT:    s_mov_b32 s9, s3
+; GCNX3-NOHSA-NEXT:    s_mov_b32 s8, s6
+; GCNX3-NOHSA-NEXT:    s_mov_b32 s9, s7
 ; GCNX3-NOHSA-NEXT:    buffer_load_dwordx4 v[0:3], off, s[8:11], 0 offset:16
 ; GCNX3-NOHSA-NEXT:    buffer_load_dwordx4 v[4:7], off, s[8:11], 0
 ; GCNX3-NOHSA-NEXT:    v_mov_b32_e32 v9, 0
 ; GCNX3-NOHSA-NEXT:    v_mov_b32_e32 v11, v9
-; GCNX3-NOHSA-NEXT:    s_mov_b32 s4, s0
-; GCNX3-NOHSA-NEXT:    s_mov_b32 s5, s1
+; GCNX3-NOHSA-NEXT:    s_mov_b32 s0, s4
+; GCNX3-NOHSA-NEXT:    s_mov_b32 s1, s5
 ; GCNX3-NOHSA-NEXT:    s_waitcnt vmcnt(1)
 ; GCNX3-NOHSA-NEXT:    v_mov_b32_e32 v8, v2
 ; GCNX3-NOHSA-NEXT:    v_mov_b32_e32 v10, v3
-; GCNX3-NOHSA-NEXT:    buffer_store_dwordx4 v[8:11], off, s[4:7], 0 offset:48
+; GCNX3-NOHSA-NEXT:    buffer_store_dwordx4 v[8:11], off, s[0:3], 0 offset:48
 ; GCNX3-NOHSA-NEXT:    s_nop 0
 ; GCNX3-NOHSA-NEXT:    v_mov_b32_e32 v8, v0
 ; GCNX3-NOHSA-NEXT:    v_mov_b32_e32 v10, v1
-; GCNX3-NOHSA-NEXT:    buffer_store_dwordx4 v[8:11], off, s[4:7], 0 offset:32
+; GCNX3-NOHSA-NEXT:    buffer_store_dwordx4 v[8:11], off, s[0:3], 0 offset:32
 ; GCNX3-NOHSA-NEXT:    s_waitcnt vmcnt(2)
 ; GCNX3-NOHSA-NEXT:    v_mov_b32_e32 v8, v6
 ; GCNX3-NOHSA-NEXT:    v_mov_b32_e32 v10, v7
-; GCNX3-NOHSA-NEXT:    buffer_store_dwordx4 v[8:11], off, s[4:7], 0 offset:16
+; GCNX3-NOHSA-NEXT:    buffer_store_dwordx4 v[8:11], off, s[0:3], 0 offset:16
 ; GCNX3-NOHSA-NEXT:    s_nop 0
 ; GCNX3-NOHSA-NEXT:    v_mov_b32_e32 v8, v4
 ; GCNX3-NOHSA-NEXT:    v_mov_b32_e32 v10, v5
-; GCNX3-NOHSA-NEXT:    buffer_store_dwordx4 v[8:11], off, s[4:7], 0
+; GCNX3-NOHSA-NEXT:    buffer_store_dwordx4 v[8:11], off, s[0:3], 0
 ; GCNX3-NOHSA-NEXT:    s_endpgm
 ;
 ; EG-LABEL: global_zextload_v8i32_to_v8i64:
@@ -2091,17 +2095,17 @@ define amdgpu_kernel void @global_zextload_v8i32_to_v8i64(ptr addrspace(1) %out,
 define amdgpu_kernel void @global_sextload_v8i32_to_v8i64(ptr addrspace(1) %out, ptr addrspace(1) %in) #0 {
 ; SI-NOHSA-LABEL: global_sextload_v8i32_to_v8i64:
 ; SI-NOHSA:       ; %bb.0:
-; SI-NOHSA-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
-; SI-NOHSA-NEXT:    s_mov_b32 s7, 0xf000
-; SI-NOHSA-NEXT:    s_mov_b32 s6, -1
-; SI-NOHSA-NEXT:    s_mov_b32 s10, s6
-; SI-NOHSA-NEXT:    s_mov_b32 s11, s7
+; SI-NOHSA-NEXT:    s_load_dwordx4 s[4:7], s[4:5], 0x9
+; SI-NOHSA-NEXT:    s_mov_b32 s3, 0xf000
+; SI-NOHSA-NEXT:    s_mov_b32 s2, -1
+; SI-NOHSA-NEXT:    s_mov_b32 s10, s2
+; SI-NOHSA-NEXT:    s_mov_b32 s11, s3
 ; SI-NOHSA-NEXT:    s_waitcnt lgkmcnt(0)
-; SI-NOHSA-NEXT:    s_mov_b32 s8, s2
-; SI-NOHSA-NEXT:    s_mov_b32 s9, s3
+; SI-NOHSA-NEXT:    s_mov_b32 s8, s6
+; SI-NOHSA-NEXT:    s_mov_b32 s9, s7
 ; SI-NOHSA-NEXT:    buffer_load_dwordx4 v[0:3], off, s[8:11], 0
-; SI-NOHSA-NEXT:    s_mov_b32 s4, s0
-; SI-NOHSA-NEXT:    s_mov_b32 s5, s1
+; SI-NOHSA-NEXT:    s_mov_b32 s0, s4
+; SI-NOHSA-NEXT:    s_mov_b32 s1, s5
 ; SI-NOHSA-NEXT:    buffer_load_dwordx4 v[4:7], off, s[8:11], 0 offset:16
 ; SI-NOHSA-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NOHSA-NEXT:    v_ashrrev_i32_e32 v10, 31, v1
@@ -2121,10 +2125,10 @@ define amdgpu_kernel void @global_sextload_v8i32_to_v8i64(ptr addrspace(1) %out,
 ; SI-NOHSA-NEXT:    v_mov_b32_e32 v13, v3
 ; SI-NOHSA-NEXT:    v_mov_b32_e32 v7, v0
 ; SI-NOHSA-NEXT:    v_mov_b32_e32 v9, v1
-; SI-NOHSA-NEXT:    buffer_store_dwordx4 v[19:22], off, s[4:7], 0 offset:48
-; SI-NOHSA-NEXT:    buffer_store_dwordx4 v[15:18], off, s[4:7], 0 offset:32
-; SI-NOHSA-NEXT:    buffer_store_dwordx4 v[11:14], off, s[4:7], 0 offset:16
-; SI-NOHSA-NEXT:    buffer_store_dwordx4 v[7:10], off, s[4:7], 0
+; SI-NOHSA-NEXT:    buffer_store_dwordx4 v[19:22], off, s[0:3], 0 offset:48
+; SI-NOHSA-NEXT:    buffer_store_dwordx4 v[15:18], off, s[0:3], 0 offset:32
+; SI-NOHSA-NEXT:    buffer_store_dwordx4 v[11:14], off, s[0:3], 0 offset:16
+; SI-NOHSA-NEXT:    buffer_store_dwordx4 v[7:10], off, s[0:3], 0
 ; SI-NOHSA-NEXT:    s_endpgm
 ;
 ; GCNX3-HSA-LABEL: global_sextload_v8i32_to_v8i64:
@@ -2369,21 +2373,21 @@ define amdgpu_kernel void @global_sextload_v16i32_to_v16i64(ptr addrspace(1) %ou
 ; GCNX3-HSA-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCNX3-HSA-NEXT:    v_mov_b32_e32 v0, s2
 ; GCNX3-HSA-NEXT:    v_mov_b32_e32 v1, s3
-; GCNX3-HSA-NEXT:    flat_load_dwordx4 v[8:11], v[0:1]
 ; GCNX3-HSA-NEXT:    s_add_u32 s4, s2, 48
+; GCNX3-HSA-NEXT:    flat_load_dwordx4 v[8:11], v[0:1]
 ; GCNX3-HSA-NEXT:    s_addc_u32 s5, s3, 0
 ; GCNX3-HSA-NEXT:    v_mov_b32_e32 v0, s4
 ; GCNX3-HSA-NEXT:    v_mov_b32_e32 v1, s5
 ; GCNX3-HSA-NEXT:    s_add_u32 s4, s2, 32
-; GCNX3-HSA-NEXT:    flat_load_dwordx4 v[12:15], v[0:1]
 ; GCNX3-HSA-NEXT:    s_addc_u32 s5, s3, 0
-; GCNX3-HSA-NEXT:    v_mov_b32_e32 v0, s4
-; GCNX3-HSA-NEXT:    v_mov_b32_e32 v1, s5
 ; GCNX3-HSA-NEXT:    s_add_u32 s2, s2, 16
-; GCNX3-HSA-NEXT:    flat_load_dwordx4 v[4:7], v[0:1]
 ; GCNX3-HSA-NEXT:    s_addc_u32 s3, s3, 0
-; GCNX3-HSA-NEXT:    v_mov_b32_e32 v0, s2
-; GCNX3-HSA-NEXT:    v_mov_b32_e32 v1, s3
+; GCNX3-HSA-NEXT:    v_mov_b32_e32 v5, s3
+; GCNX3-HSA-NEXT:    v_mov_b32_e32 v4, s2
+; GCNX3-HSA-NEXT:    flat_load_dwordx4 v[12:15], v[0:1]
+; GCNX3-HSA-NEXT:    flat_load_dwordx4 v[4:7], v[4:5]
+; GCNX3-HSA-NEXT:    v_mov_b32_e32 v0, s4
+; GCNX3-HSA-NEXT:    v_mov_b32_e32 v1, s5
 ; GCNX3-HSA-NEXT:    flat_load_dwordx4 v[0:3], v[0:1]
 ; GCNX3-HSA-NEXT:    s_add_u32 s2, s0, 16
 ; GCNX3-HSA-NEXT:    s_addc_u32 s3, s1, 0
@@ -2434,31 +2438,30 @@ define amdgpu_kernel void @global_sextload_v16i32_to_v16i64(ptr addrspace(1) %ou
 ; GCNX3-HSA-NEXT:    s_addc_u32 s1, s1, 0
 ; GCNX3-HSA-NEXT:    flat_store_dwordx4 v[24:25], v[16:19]
 ; GCNX3-HSA-NEXT:    flat_store_dwordx4 v[26:27], v[8:11]
-; GCNX3-HSA-NEXT:    s_waitcnt vmcnt(5)
-; GCNX3-HSA-NEXT:    v_ashrrev_i32_e32 v15, 31, v5
-; GCNX3-HSA-NEXT:    v_ashrrev_i32_e32 v11, 31, v7
-; GCNX3-HSA-NEXT:    v_ashrrev_i32_e32 v9, 31, v6
-; GCNX3-HSA-NEXT:    v_ashrrev_i32_e32 v13, 31, v4
-; GCNX3-HSA-NEXT:    v_mov_b32_e32 v12, v4
-; GCNX3-HSA-NEXT:    v_mov_b32_e32 v14, v5
-; GCNX3-HSA-NEXT:    v_mov_b32_e32 v8, v6
-; GCNX3-HSA-NEXT:    v_mov_b32_e32 v10, v7
+; GCNX3-HSA-NEXT:    s_waitcnt vmcnt(4)
+; GCNX3-HSA-NEXT:    v_ashrrev_i32_e32 v15, 31, v1
+; GCNX3-HSA-NEXT:    v_ashrrev_i32_e32 v11, 31, v3
+; GCNX3-HSA-NEXT:    v_ashrrev_i32_e32 v9, 31, v2
+; GCNX3-HSA-NEXT:    v_ashrrev_i32_e32 v13, 31, v0
+; GCNX3-HSA-NEXT:    v_mov_b32_e32 v12, v0
+; GCNX3-HSA-NEXT:    v_mov_b32_e32 v14, v1
+; GCNX3-HSA-NEXT:    v_mov_b32_e32 v8, v2
+; GCNX3-HSA-NEXT:    v_mov_b32_e32 v10, v3
 ; GCNX3-HSA-NEXT:    v_mov_b32_e32 v22, s2
 ; GCNX3-HSA-NEXT:    v_mov_b32_e32 v31, s1
 ; GCNX3-HSA-NEXT:    flat_store_dwordx4 v[20:21], v[12:15]
 ; GCNX3-HSA-NEXT:    flat_store_dwordx4 v[28:29], v[8:11]
 ; GCNX3-HSA-NEXT:    v_mov_b32_e32 v30, s0
-; GCNX3-HSA-NEXT:    s_waitcnt vmcnt(6)
-; GCNX3-HSA-NEXT:    v_ashrrev_i32_e32 v11, 31, v1
-; GCNX3-HSA-NEXT:    v_ashrrev_i32_e32 v9, 31, v0
-; GCNX3-HSA-NEXT:    v_mov_b32_e32 v8, v0
-; GCNX3-HSA-NEXT:    v_mov_b32_e32 v10, v1
-; GCNX3-HSA-NEXT:    v_ashrrev_i32_e32 v7, 31, v3
-; GCNX3-HSA-NEXT:    v_ashrrev_i32_e32 v5, 31, v2
-; GCNX3-HSA-NEXT:    v_mov_b32_e32 v4, v2
-; GCNX3-HSA-NEXT:    v_mov_b32_e32 v6, v3
+; GCNX3-HSA-NEXT:    v_ashrrev_i32_e32 v11, 31, v5
+; GCNX3-HSA-NEXT:    v_ashrrev_i32_e32 v9, 31, v4
+; GCNX3-HSA-NEXT:    v_mov_b32_e32 v8, v4
+; GCNX3-HSA-NEXT:    v_mov_b32_e32 v10, v5
+; GCNX3-HSA-NEXT:    v_ashrrev_i32_e32 v3, 31, v7
+; GCNX3-HSA-NEXT:    v_ashrrev_i32_e32 v1, 31, v6
+; GCNX3-HSA-NEXT:    v_mov_b32_e32 v0, v6
+; GCNX3-HSA-NEXT:    v_mov_b32_e32 v2, v7
 ; GCNX3-HSA-NEXT:    flat_store_dwordx4 v[22:23], v[8:11]
-; GCNX3-HSA-NEXT:    flat_store_dwordx4 v[30:31], v[4:7]
+; GCNX3-HSA-NEXT:    flat_store_dwordx4 v[30:31], v[0:3]
 ; GCNX3-HSA-NEXT:    s_endpgm
 ;
 ; GCNX3-NOHSA-LABEL: global_sextload_v16i32_to_v16i64:
@@ -2611,60 +2614,115 @@ define amdgpu_kernel void @global_sextload_v16i32_to_v16i64(ptr addrspace(1) %ou
 ; EG-NEXT:     MOV * T16.Z, T1.Y,
 ; EG-NEXT:    2(2.802597e-45), 0(0.000000e+00)
 ;
-; GCN-HSA-LABEL: global_sextload_v16i32_to_v16i64:
-; GCN-HSA:       ; %bb.0:
-; GCN-HSA-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
-; GCN-HSA-NEXT:    v_mov_b32_e32 v36, 0
-; GCN-HSA-NEXT:    s_waitcnt lgkmcnt(0)
-; GCN-HSA-NEXT:    global_load_dwordx4 v[0:3], v36, s[2:3] offset:32
-; GCN-HSA-NEXT:    global_load_dwordx4 v[4:7], v36, s[2:3] offset:48
-; GCN-HSA-NEXT:    global_load_dwordx4 v[8:11], v36, s[2:3] offset:16
-; GCN-HSA-NEXT:    global_load_dwordx4 v[12:15], v36, s[2:3]
-; GCN-HSA-NEXT:    s_waitcnt vmcnt(3)
-; GCN-HSA-NEXT:    v_ashrrev_i32_e32 v19, 31, v3
-; GCN-HSA-NEXT:    s_waitcnt vmcnt(2)
-; GCN-HSA-NEXT:    v_ashrrev_i32_e32 v27, 31, v5
-; GCN-HSA-NEXT:    v_ashrrev_i32_e32 v25, 31, v4
-; GCN-HSA-NEXT:    v_mov_b32_e32 v24, v4
-; GCN-HSA-NEXT:    v_mov_b32_e32 v26, v5
-; GCN-HSA-NEXT:    v_ashrrev_i32_e32 v17, 31, v2
-; GCN-HSA-NEXT:    v_ashrrev_i32_e32 v23, 31, v7
-; GCN-HSA-NEXT:    v_ashrrev_i32_e32 v21, 31, v6
-; GCN-HSA-NEXT:    v_mov_b32_e32 v20, v6
-; GCN-HSA-NEXT:    v_mov_b32_e32 v22, v7
-; GCN-HSA-NEXT:    v_ashrrev_i32_e32 v7, 31, v1
-; GCN-HSA-NEXT:    v_ashrrev_i32_e32 v5, 31, v0
-; GCN-HSA-NEXT:    v_mov_b32_e32 v4, v0
-; GCN-HSA-NEXT:    v_mov_b32_e32 v6, v1
-; GCN-HSA-NEXT:    v_mov_b32_e32 v16, v2
-; GCN-HSA-NEXT:    v_mov_b32_e32 v18, v3
-; GCN-HSA-NEXT:    s_waitcnt vmcnt(1)
-; GCN-HSA-NEXT:    v_ashrrev_i32_e32 v3, 31, v11
-; GCN-HSA-NEXT:    v_ashrrev_i32_e32 v1, 31, v10
-; GCN-HSA-NEXT:    v_ashrrev_i32_e32 v31, 31, v9
-; GCN-HSA-NEXT:    v_ashrrev_i32_e32 v29, 31, v8
-; GCN-HSA-NEXT:    v_mov_b32_e32 v28, v8
-; GCN-HSA-NEXT:    v_mov_b32_e32 v30, v9
-; GCN-HSA-NEXT:    v_mov_b32_e32 v0, v10
-; GCN-HSA-NEXT:    v_mov_b32_e32 v2, v11
-; GCN-HSA-NEXT:    s_waitcnt vmcnt(0)
-; GCN-HSA-NEXT:    v_ashrrev_i32_e32 v11, 31, v15
-; GCN-HSA-NEXT:    v_ashrrev_i32_e32 v9, 31, v14
-; GCN-HSA-NEXT:    v_ashrrev_i32_e32 v35, 31, v13
-; GCN-HSA-NEXT:    v_ashrrev_i32_e32 v33, 31, v12
-; GCN-HSA-NEXT:    v_mov_b32_e32 v32, v12
-; GCN-HSA-NEXT:    v_mov_b32_e32 v34, v13
-; GCN-HSA-NEXT:    v_mov_b32_e32 v8, v14
-; GCN-HSA-NEXT:    v_mov_b32_e32 v10, v15
-; GCN-HSA-NEXT:    global_store_dwordx4 v36, v[24:27], s[0:1] offset:96
-; GCN-HSA-NEXT:    global_store_dwordx4 v36, v[20:23], s[0:1] offset:112
-; GCN-HSA-NEXT:    global_store_dwordx4 v36, v[4:7], s[0:1] offset:64
-; GCN-HSA-NEXT:    global_store_dwordx4 v36, v[16:19], s[0:1] offset:80
-; GCN-HSA-NEXT:    global_store_dwordx4 v36, v[28:31], s[0:1] offset:32
-; GCN-HSA-NEXT:    global_store_dwordx4 v36, v[0:3], s[0:1] offset:48
-; GCN-HSA-NEXT:    global_store_dwordx4 v36, v[32:35], s[0:1]
-; GCN-HSA-NEXT:    global_store_dwordx4 v36, v[8:11], s[0:1] offset:16
-; GCN-HSA-NEXT:    s_endpgm
+; GCN-GFX900-HSA-LABEL: global_sextload_v16i32_to_v16i64:
+; GCN-GFX900-HSA:       ; %bb.0:
+; GCN-GFX900-HSA-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GCN-GFX900-HSA-NEXT:    v_mov_b32_e32 v36, 0
+; GCN-GFX900-HSA-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-GFX900-HSA-NEXT:    global_load_dwordx4 v[0:3], v36, s[2:3] offset:32
+; GCN-GFX900-HSA-NEXT:    global_load_dwordx4 v[4:7], v36, s[2:3] offset:48
+; GCN-GFX900-HSA-NEXT:    global_load_dwordx4 v[8:11], v36, s[2:3] offset:16
+; GCN-GFX900-HSA-NEXT:    global_load_dwordx4 v[12:15], v36, s[2:3]
+; GCN-GFX900-HSA-NEXT:    s_waitcnt vmcnt(3)
+; GCN-GFX900-HSA-NEXT:    v_ashrrev_i32_e32 v19, 31, v3
+; GCN-GFX900-HSA-NEXT:    s_waitcnt vmcnt(2)
+; GCN-GFX900-HSA-NEXT:    v_ashrrev_i32_e32 v27, 31, v5
+; GCN-GFX900-HSA-NEXT:    v_ashrrev_i32_e32 v25, 31, v4
+; GCN-GFX900-HSA-NEXT:    v_mov_b32_e32 v24, v4
+; GCN-GFX900-HSA-NEXT:    v_mov_b32_e32 v26, v5
+; GCN-GFX900-HSA-NEXT:    v_ashrrev_i32_e32 v17, 31, v2
+; GCN-GFX900-HSA-NEXT:    v_ashrrev_i32_e32 v23, 31, v7
+; GCN-GFX900-HSA-NEXT:    v_ashrrev_i32_e32 v21, 31, v6
+; GCN-GFX900-HSA-NEXT:    v_mov_b32_e32 v20, v6
+; GCN-GFX900-HSA-NEXT:    v_mov_b32_e32 v22, v7
+; GCN-GFX900-HSA-NEXT:    v_ashrrev_i32_e32 v7, 31, v1
+; GCN-GFX900-HSA-NEXT:    v_ashrrev_i32_e32 v5, 31, v0
+; GCN-GFX900-HSA-NEXT:    v_mov_b32_e32 v4, v0
+; GCN-GFX900-HSA-NEXT:    v_mov_b32_e32 v6, v1
+; GCN-GFX900-HSA-NEXT:    v_mov_b32_e32 v16, v2
+; GCN-GFX900-HSA-NEXT:    v_mov_b32_e32 v18, v3
+; GCN-GFX900-HSA-NEXT:    s_waitcnt vmcnt(1)
+; GCN-GFX900-HSA-NEXT:    v_ashrrev_i32_e32 v3, 31, v11
+; GCN-GFX900-HSA-NEXT:    v_ashrrev_i32_e32 v1, 31, v10
+; GCN-GFX900-HSA-NEXT:    v_ashrrev_i32_e32 v31, 31, v9
+; GCN-GFX900-HSA-NEXT:    v_ashrrev_i32_e32 v29, 31, v8
+; GCN-GFX900-HSA-NEXT:    v_mov_b32_e32 v28, v8
+; GCN-GFX900-HSA-NEXT:    v_mov_b32_e32 v30, v9
+; GCN-GFX900-HSA-NEXT:    v_mov_b32_e32 v0, v10
+; GCN-GFX900-HSA-NEXT:    v_mov_b32_e32 v2, v11
+; GCN-GFX900-HSA-NEXT:    s_waitcnt vmcnt(0)
+; GCN-GFX900-HSA-NEXT:    v_ashrrev_i32_e32 v11, 31, v15
+; GCN-GFX900-HSA-NEXT:    v_ashrrev_i32_e32 v9, 31, v14
+; GCN-GFX900-HSA-NEXT:    v_ashrrev_i32_e32 v35, 31, v13
+; GCN-GFX900-HSA-NEXT:    v_ashrrev_i32_e32 v33, 31, v12
+; GCN-GFX900-HSA-NEXT:    v_mov_b32_e32 v32, v12
+; GCN-GFX900-HSA-NEXT:    v_mov_b32_e32 v34, v13
+; GCN-GFX900-HSA-NEXT:    v_mov_b32_e32 v8, v14
+; GCN-GFX900-HSA-NEXT:    v_mov_b32_e32 v10, v15
+; GCN-GFX900-HSA-NEXT:    global_store_dwordx4 v36, v[24:27], s[0:1] offset:96
+; GCN-GFX900-HSA-NEXT:    global_store_dwordx4 v36, v[20:23], s[0:1] offset:112
+; GCN-GFX900-HSA-NEXT:    global_store_dwordx4 v36, v[4:7], s[0:1] offset:64
+; GCN-GFX900-HSA-NEXT:    global_store_dwordx4 v36, v[16:19], s[0:1] offset:80
+; GCN-GFX900-HSA-NEXT:    global_store_dwordx4 v36, v[28:31], s[0:1] offset:32
+; GCN-GFX900-HSA-NEXT:    global_store_dwordx4 v36, v[0:3], s[0:1] offset:48
+; GCN-GFX900-HSA-NEXT:    global_store_dwordx4 v36, v[32:35], s[0:1]
+; GCN-GFX900-HSA-NEXT:    global_store_dwordx4 v36, v[8:11], s[0:1] offset:16
+; GCN-GFX900-HSA-NEXT:    s_endpgm
+;
+; GCN-GFX908-HSA-LABEL: global_sextload_v16i32_to_v16i64:
+; GCN-GFX908-HSA:       ; %bb.0:
+; GCN-GFX908-HSA-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GCN-GFX908-HSA-NEXT:    v_mov_b32_e32 v0, 0
+; GCN-GFX908-HSA-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-GFX908-HSA-NEXT:    global_load_dwordx4 v[1:4], v0, s[2:3] offset:32
+; GCN-GFX908-HSA-NEXT:    global_load_dwordx4 v[5:8], v0, s[2:3] offset:48
+; GCN-GFX908-HSA-NEXT:    global_load_dwordx4 v[9:12], v0, s[2:3] offset:16
+; GCN-GFX908-HSA-NEXT:    global_load_dwordx4 v[13:16], v0, s[2:3]
+; GCN-GFX908-HSA-NEXT:    s_waitcnt vmcnt(3)
+; GCN-GFX908-HSA-NEXT:    v_ashrrev_i32_e32 v20, 31, v4
+; GCN-GFX908-HSA-NEXT:    s_waitcnt vmcnt(2)
+; GCN-GFX908-HSA-NEXT:    v_ashrrev_i32_e32 v28, 31, v6
+; GCN-GFX908-HSA-NEXT:    v_ashrrev_i32_e32 v26, 31, v5
+; GCN-GFX908-HSA-NEXT:    v_mov_b32_e32 v25, v5
+; GCN-GFX908-HSA-NEXT:    v_mov_b32_e32 v27, v6
+; GCN-GFX908-HSA-NEXT:    v_ashrrev_i32_e32 v18, 31, v3
+; GCN-GFX908-HSA-NEXT:    v_ashrrev_i32_e32 v24, 31, v8
+; GCN-GFX908-HSA-NEXT:    v_ashrrev_i32_e32 v22, 31, v7
+; GCN-GFX908-HSA-NEXT:    v_mov_b32_e32 v21, v7
+; GCN-GFX908-HSA-NEXT:    v_mov_b32_e32 v23, v8
+; GCN-GFX908-HSA-NEXT:    v_ashrrev_i32_e32 v8, 31, v2
+; GCN-GFX908-HSA-NEXT:    v_ashrrev_i32_e32 v6, 31, v1
+; GCN-GFX908-HSA-NEXT:    v_mov_b32_e32 v5, v1
+; GCN-GFX908-HSA-NEXT:    v_mov_b32_e32 v7, v2
+; GCN-GFX908-HSA-NEXT:    v_mov_b32_e32 v17, v3
+; GCN-GFX908-HSA-NEXT:    v_mov_b32_e32 v19, v4
+; GCN-GFX908-HSA-NEXT:    s_waitcnt vmcnt(1)
+; GCN-GFX908-HSA-NEXT:    v_ashrrev_i32_e32 v4, 31, v12
+; GCN-GFX908-HSA-NEXT:    v_ashrrev_i32_e32 v2, 31, v11
+; GCN-GFX908-HSA-NEXT:    v_ashrrev_i32_e32 v32, 31, v10
+; GCN-GFX908-HSA-NEXT:    v_ashrrev_i32_e32 v30, 31, v9
+; GCN-GFX908-HSA-NEXT:    v_mov_b32_e32 v29, v9
+; GCN-GFX908-HSA-NEXT:    v_mov_b32_e32 v31, v10
+; GCN-GFX908-HSA-NEXT:    v_mov_b32_e32 v1, v11
+; GCN-GFX908-HSA-NEXT:    v_mov_b32_e32 v3, v12
+; GCN-GFX908-HSA-NEXT:    s_waitcnt vmcnt(0)
+; GCN-GFX908-HSA-NEXT:    v_ashrrev_i32_e32 v12, 31, v16
+; GCN-GFX908-HSA-NEXT:    v_ashrrev_i32_e32 v10, 31, v15
+; GCN-GFX908-HSA-NEXT:    v_ashrrev_i32_e32 v36, 31, v14
+; GCN-GFX908-HSA-NEXT:    v_ashrrev_i32_e32 v34, 31, v13
+; GCN-GFX908-HSA-NEXT:    v_mov_b32_e32 v33, v13
+; GCN-GFX908-HSA-NEXT:    v_mov_b32_e32 v35, v14
+; GCN-GFX908-HSA-NEXT:    v_mov_b32_e32 v9, v15
+; GCN-GFX908-HSA-NEXT:    v_mov_b32_e32 v11, v16
+; GCN-GFX908-HSA-NEXT:    global_store_dwordx4 v0, v[25:28], s[0:1] offset:96
+; GCN-GFX908-HSA-NEXT:    global_store_dwordx4 v0, v[21:24], s[0:1] offset:112
+; GCN-GFX908-HSA-NEXT:    global_store_dwordx4 v0, v[5:8], s[0:1] offset:64
+; GCN-GFX908-HSA-NEXT:    global_store_dwordx4 v0, v[17:20], s[0:1] offset:80
+; GCN-GFX908-HSA-NEXT:    global_store_dwordx4 v0, v[29:32], s[0:1] offset:32
+; GCN-GFX908-HSA-NEXT:    global_store_dwordx4 v0, v[1:4], s[0:1] offset:48
+; GCN-GFX908-HSA-NEXT:    global_store_dwordx4 v0, v[33:36], s[0:1]
+; GCN-GFX908-HSA-NEXT:    global_store_dwordx4 v0, v[9:12], s[0:1] offset:16
+; GCN-GFX908-HSA-NEXT:    s_endpgm
   %ld = load <16 x i32>, ptr addrspace(1) %in
   %ext = sext <16 x i32> %ld to <16 x i64>
   store <16 x i64> %ext, ptr addrspace(1) %out
@@ -3137,26 +3195,26 @@ define amdgpu_kernel void @global_sextload_v32i32_to_v32i64(ptr addrspace(1) %ou
 ; GCNX3-HSA-NEXT:    v_mov_b32_e32 v0, s4
 ; GCNX3-HSA-NEXT:    v_mov_b32_e32 v1, s5
 ; GCNX3-HSA-NEXT:    s_add_u32 s4, s2, 64
-; GCNX3-HSA-NEXT:    flat_load_dwordx4 v[12:15], v[0:1]
+; GCNX3-HSA-NEXT:    flat_load_dwordx4 v[8:11], v[0:1]
 ; GCNX3-HSA-NEXT:    s_addc_u32 s5, s3, 0
 ; GCNX3-HSA-NEXT:    v_mov_b32_e32 v0, s4
 ; GCNX3-HSA-NEXT:    v_mov_b32_e32 v1, s5
 ; GCNX3-HSA-NEXT:    flat_load_dwordx4 v[4:7], v[0:1]
 ; GCNX3-HSA-NEXT:    s_add_u32 s4, s2, 48
 ; GCNX3-HSA-NEXT:    s_addc_u32 s5, s3, 0
-; GCNX3-HSA-NEXT:    v_mov_b32_e32 v9, s5
-; GCNX3-HSA-NEXT:    v_mov_b32_e32 v8, s4
-; GCNX3-HSA-NEXT:    flat_load_dwordx4 v[16:19], v[8:9]
+; GCNX3-HSA-NEXT:    v_mov_b32_e32 v13, s5
+; GCNX3-HSA-NEXT:    v_mov_b32_e32 v12, s4
+; GCNX3-HSA-NEXT:    flat_load_dwordx4 v[16:19], v[12:13]
 ; GCNX3-HSA-NEXT:    s_add_u32 s6, s2, 32
 ; GCNX3-HSA-NEXT:    s_addc_u32 s7, s3, 0
 ; GCNX3-HSA-NEXT:    s_add_u32 s2, s2, 16
 ; GCNX3-HSA-NEXT:    s_addc_u32 s3, s3, 0
 ; GCNX3-HSA-NEXT:    v_mov_b32_e32 v0, s2
-; GCNX3-HSA-NEXT:    v_mov_b32_e32 v9, s7
+; GCNX3-HSA-NEXT:    v_mov_b32_e32 v13, s7
 ; GCNX3-HSA-NEXT:    v_mov_b32_e32 v1, s3
-; GCNX3-HSA-NEXT:    v_mov_b32_e32 v8, s6
+; GCNX3-HSA-NEXT:    v_mov_b32_e32 v12, s6
 ; GCNX3-HSA-NEXT:    flat_load_dwordx4 v[0:3], v[0:1]
-; GCNX3-HSA-NEXT:    flat_load_dwordx4 v[8:11], v[8:9]
+; GCNX3-HSA-NEXT:    flat_load_dwordx4 v[12:15], v[12:13]
 ; GCNX3-HSA-NEXT:    s_add_u32 s2, s0, 16
 ; GCNX3-HSA-NEXT:    s_addc_u32 s3, s1, 0
 ; GCNX3-HSA-NEXT:    v_mov_b32_e32 v37, s1
@@ -3223,26 +3281,26 @@ define amdgpu_kernel void @global_sextload_v32i32_to_v32i64(ptr addrspace(1) %ou
 ; GCNX3-HSA-NEXT:    flat_store_dwordx4 v[34:35], v[24:27]
 ; GCNX3-HSA-NEXT:    flat_store_dwordx4 v[36:37], v[28:31]
 ; GCNX3-HSA-NEXT:    s_waitcnt vmcnt(10)
-; GCNX3-HSA-NEXT:    v_ashrrev_i32_e32 v23, 31, v15
-; GCNX3-HSA-NEXT:    v_ashrrev_i32_e32 v21, 31, v14
-; GCNX3-HSA-NEXT:    v_ashrrev_i32_e32 v27, 31, v13
-; GCNX3-HSA-NEXT:    v_ashrrev_i32_e32 v25, 31, v12
-; GCNX3-HSA-NEXT:    v_mov_b32_e32 v24, v12
-; GCNX3-HSA-NEXT:    v_mov_b32_e32 v26, v13
-; GCNX3-HSA-NEXT:    v_mov_b32_e32 v20, v14
-; GCNX3-HSA-NEXT:    v_mov_b32_e32 v22, v15
+; GCNX3-HSA-NEXT:    v_ashrrev_i32_e32 v23, 31, v11
+; GCNX3-HSA-NEXT:    v_ashrrev_i32_e32 v21, 31, v10
+; GCNX3-HSA-NEXT:    v_ashrrev_i32_e32 v27, 31, v9
+; GCNX3-HSA-NEXT:    v_ashrrev_i32_e32 v25, 31, v8
+; GCNX3-HSA-NEXT:    v_mov_b32_e32 v24, v8
+; GCNX3-HSA-NEXT:    v_mov_b32_e32 v26, v9
+; GCNX3-HSA-NEXT:    v_mov_b32_e32 v20, v10
+; GCNX3-HSA-NEXT:    v_mov_b32_e32 v22, v11
 ; GCNX3-HSA-NEXT:    s_waitcnt vmcnt(9)
-; GCNX3-HSA-NEXT:    v_ashrrev_i32_e32 v15, 31, v5
-; GCNX3-HSA-NEXT:    v_ashrrev_i32_e32 v13, 31, v4
-; GCNX3-HSA-NEXT:    v_mov_b32_e32 v12, v4
-; GCNX3-HSA-NEXT:    v_mov_b32_e32 v14, v5
+; GCNX3-HSA-NEXT:    v_ashrrev_i32_e32 v11, 31, v5
+; GCNX3-HSA-NEXT:    v_ashrrev_i32_e32 v9, 31, v4
+; GCNX3-HSA-NEXT:    v_mov_b32_e32 v8, v4
+; GCNX3-HSA-NEXT:    v_mov_b32_e32 v10, v5
 ; GCNX3-HSA-NEXT:    v_mov_b32_e32 v5, s3
 ; GCNX3-HSA-NEXT:    v_mov_b32_e32 v4, s2
 ; GCNX3-HSA-NEXT:    s_add_u32 s2, s0, 0x90
 ; GCNX3-HSA-NEXT:    s_addc_u32 s3, s1, 0
 ; GCNX3-HSA-NEXT:    flat_store_dwordx4 v[32:33], v[24:27]
 ; GCNX3-HSA-NEXT:    flat_store_dwordx4 v[38:39], v[20:23]
-; GCNX3-HSA-NEXT:    flat_store_dwordx4 v[4:5], v[12:15]
+; GCNX3-HSA-NEXT:    flat_store_dwordx4 v[4:5], v[8:11]
 ; GCNX3-HSA-NEXT:    v_mov_b32_e32 v5, s3
 ; GCNX3-HSA-NEXT:    v_mov_b32_e32 v4, s2
 ; GCNX3-HSA-NEXT:    s_add_u32 s2, s0, 0x60
@@ -3253,58 +3311,58 @@ define amdgpu_kernel void @global_sextload_v32i32_to_v32i64(ptr addrspace(1) %ou
 ; GCNX3-HSA-NEXT:    s_addc_u32 s3, s1, 0
 ; GCNX3-HSA-NEXT:    flat_store_dwordx4 v[4:5], v[23:26]
 ; GCNX3-HSA-NEXT:    s_waitcnt vmcnt(12)
+; GCNX3-HSA-NEXT:    v_ashrrev_i32_e32 v7, 31, v17
 ; GCNX3-HSA-NEXT:    v_ashrrev_i32_e32 v5, 31, v16
 ; GCNX3-HSA-NEXT:    v_mov_b32_e32 v4, v16
-; GCNX3-HSA-NEXT:    v_mov_b32_e32 v16, s3
-; GCNX3-HSA-NEXT:    v_mov_b32_e32 v15, s2
-; GCNX3-HSA-NEXT:    s_add_u32 s2, s0, 0x70
-; GCNX3-HSA-NEXT:    v_ashrrev_i32_e32 v7, 31, v17
 ; GCNX3-HSA-NEXT:    v_mov_b32_e32 v6, v17
+; GCNX3-HSA-NEXT:    v_mov_b32_e32 v17, s3
+; GCNX3-HSA-NEXT:    v_mov_b32_e32 v16, s2
+; GCNX3-HSA-NEXT:    s_add_u32 s2, s0, 0x70
 ; GCNX3-HSA-NEXT:    s_addc_u32 s3, s1, 0
-; GCNX3-HSA-NEXT:    flat_store_dwordx4 v[15:16], v[4:7]
-; GCNX3-HSA-NEXT:    v_mov_b32_e32 v16, s3
-; GCNX3-HSA-NEXT:    v_mov_b32_e32 v15, s2
+; GCNX3-HSA-NEXT:    flat_store_dwordx4 v[16:17], v[4:7]
+; GCNX3-HSA-NEXT:    v_mov_b32_e32 v17, s3
+; GCNX3-HSA-NEXT:    v_mov_b32_e32 v16, s2
 ; GCNX3-HSA-NEXT:    s_add_u32 s2, s0, 64
 ; GCNX3-HSA-NEXT:    v_ashrrev_i32_e32 v26, 31, v19
 ; GCNX3-HSA-NEXT:    v_ashrrev_i32_e32 v24, 31, v18
 ; GCNX3-HSA-NEXT:    v_mov_b32_e32 v23, v18
 ; GCNX3-HSA-NEXT:    v_mov_b32_e32 v25, v19
 ; GCNX3-HSA-NEXT:    s_addc_u32 s3, s1, 0
-; GCNX3-HSA-NEXT:    flat_store_dwordx4 v[15:16], v[23:26]
+; GCNX3-HSA-NEXT:    flat_store_dwordx4 v[16:17], v[23:26]
 ; GCNX3-HSA-NEXT:    s_waitcnt vmcnt(12)
-; GCNX3-HSA-NEXT:    v_ashrrev_i32_e32 v18, 31, v9
-; GCNX3-HSA-NEXT:    v_ashrrev_i32_e32 v16, 31, v8
-; GCNX3-HSA-NEXT:    v_mov_b32_e32 v15, v8
-; GCNX3-HSA-NEXT:    v_mov_b32_e32 v17, v9
-; GCNX3-HSA-NEXT:    v_mov_b32_e32 v9, s3
-; GCNX3-HSA-NEXT:    v_mov_b32_e32 v8, s2
+; GCNX3-HSA-NEXT:    v_ashrrev_i32_e32 v17, 31, v12
+; GCNX3-HSA-NEXT:    v_mov_b32_e32 v16, v12
+; GCNX3-HSA-NEXT:    v_mov_b32_e32 v12, s3
+; GCNX3-HSA-NEXT:    v_mov_b32_e32 v11, s2
 ; GCNX3-HSA-NEXT:    s_add_u32 s2, s0, 0x50
+; GCNX3-HSA-NEXT:    v_ashrrev_i32_e32 v19, 31, v13
+; GCNX3-HSA-NEXT:    v_mov_b32_e32 v18, v13
 ; GCNX3-HSA-NEXT:    s_addc_u32 s3, s1, 0
-; GCNX3-HSA-NEXT:    flat_store_dwordx4 v[8:9], v[15:18]
-; GCNX3-HSA-NEXT:    v_mov_b32_e32 v9, s3
-; GCNX3-HSA-NEXT:    v_mov_b32_e32 v8, s2
+; GCNX3-HSA-NEXT:    flat_store_dwordx4 v[11:12], v[16:19]
+; GCNX3-HSA-NEXT:    v_mov_b32_e32 v12, s3
+; GCNX3-HSA-NEXT:    v_ashrrev_i32_e32 v7, 31, v15
+; GCNX3-HSA-NEXT:    v_ashrrev_i32_e32 v5, 31, v14
+; GCNX3-HSA-NEXT:    v_mov_b32_e32 v4, v14
+; GCNX3-HSA-NEXT:    v_mov_b32_e32 v6, v15
+; GCNX3-HSA-NEXT:    v_mov_b32_e32 v11, s2
 ; GCNX3-HSA-NEXT:    s_add_u32 s2, s0, 32
-; GCNX3-HSA-NEXT:    v_ashrrev_i32_e32 v14, 31, v1
-; GCNX3-HSA-NEXT:    v_ashrrev_i32_e32 v12, 31, v0
-; GCNX3-HSA-NEXT:    v_ashrrev_i32_e32 v7, 31, v11
-; GCNX3-HSA-NEXT:    v_mov_b32_e32 v6, v11
-; GCNX3-HSA-NEXT:    v_mov_b32_e32 v11, v0
-; GCNX3-HSA-NEXT:    v_mov_b32_e32 v13, v1
+; GCNX3-HSA-NEXT:    v_ashrrev_i32_e32 v10, 31, v1
+; GCNX3-HSA-NEXT:    v_ashrrev_i32_e32 v8, 31, v0
+; GCNX3-HSA-NEXT:    flat_store_dwordx4 v[11:12], v[4:7]
+; GCNX3-HSA-NEXT:    v_mov_b32_e32 v9, v1
+; GCNX3-HSA-NEXT:    v_mov_b32_e32 v7, v0
 ; GCNX3-HSA-NEXT:    s_addc_u32 s3, s1, 0
 ; GCNX3-HSA-NEXT:    v_mov_b32_e32 v0, s2
 ; GCNX3-HSA-NEXT:    v_mov_b32_e32 v1, s3
 ; GCNX3-HSA-NEXT:    s_add_u32 s0, s0, 48
-; GCNX3-HSA-NEXT:    flat_store_dwordx4 v[0:1], v[11:14]
+; GCNX3-HSA-NEXT:    flat_store_dwordx4 v[0:1], v[7:10]
 ; GCNX3-HSA-NEXT:    s_addc_u32 s1, s1, 0
 ; GCNX3-HSA-NEXT:    v_mov_b32_e32 v0, s0
 ; GCNX3-HSA-NEXT:    v_ashrrev_i32_e32 v22, 31, v3
 ; GCNX3-HSA-NEXT:    v_ashrrev_i32_e32 v20, 31, v2
-; GCNX3-HSA-NEXT:    v_ashrrev_i32_e32 v5, 31, v10
-; GCNX3-HSA-NEXT:    v_mov_b32_e32 v4, v10
 ; GCNX3-HSA-NEXT:    v_mov_b32_e32 v19, v2
 ; GCNX3-HSA-NEXT:    v_mov_b32_e32 v21, v3
 ; GCNX3-HSA-NEXT:    v_mov_b32_e32 v1, s1
-; GCNX3-HSA-NEXT:    flat_store_dwordx4 v[8:9], v[4:7]
 ; GCNX3-HSA-NEXT:    flat_store_dwordx4 v[0:1], v[19:22]
 ; GCNX3-HSA-NEXT:    s_endpgm
 ;
@@ -3941,13 +3999,13 @@ define amdgpu_kernel void @global_zextload_v32i32_to_v32i64(ptr addrspace(1) %ou
 ; GCNX3-HSA-NEXT:    flat_load_dwordx4 v[16:19], v[0:1]
 ; GCNX3-HSA-NEXT:    v_mov_b32_e32 v0, s8
 ; GCNX3-HSA-NEXT:    v_mov_b32_e32 v1, s9
-; GCNX3-HSA-NEXT:    flat_load_dwordx4 v[12:15], v[0:1]
+; GCNX3-HSA-NEXT:    flat_load_dwordx4 v[8:11], v[0:1]
 ; GCNX3-HSA-NEXT:    v_mov_b32_e32 v4, s4
-; GCNX3-HSA-NEXT:    v_mov_b32_e32 v9, s7
+; GCNX3-HSA-NEXT:    v_mov_b32_e32 v13, s7
 ; GCNX3-HSA-NEXT:    v_mov_b32_e32 v5, s5
-; GCNX3-HSA-NEXT:    v_mov_b32_e32 v8, s6
+; GCNX3-HSA-NEXT:    v_mov_b32_e32 v12, s6
 ; GCNX3-HSA-NEXT:    flat_load_dwordx4 v[4:7], v[4:5]
-; GCNX3-HSA-NEXT:    flat_load_dwordx4 v[8:11], v[8:9]
+; GCNX3-HSA-NEXT:    flat_load_dwordx4 v[12:15], v[12:13]
 ; GCNX3-HSA-NEXT:    v_mov_b32_e32 v1, 0
 ; GCNX3-HSA-NEXT:    s_add_u32 s2, s0, 16
 ; GCNX3-HSA-NEXT:    v_mov_b32_e32 v3, v1
@@ -4032,29 +4090,29 @@ define amdgpu_kernel void @global_zextload_v32i32_to_v32i64(ptr addrspace(1) %ou
 ; GCNX3-HSA-NEXT:    v_mov_b32_e32 v2, v19
 ; GCNX3-HSA-NEXT:    flat_store_dwordx4 v[26:27], v[0:3]
 ; GCNX3-HSA-NEXT:    s_waitcnt vmcnt(12)
-; GCNX3-HSA-NEXT:    v_mov_b32_e32 v0, v12
-; GCNX3-HSA-NEXT:    v_mov_b32_e32 v2, v13
-; GCNX3-HSA-NEXT:    v_mov_b32_e32 v13, s3
-; GCNX3-HSA-NEXT:    v_mov_b32_e32 v12, s2
+; GCNX3-HSA-NEXT:    v_mov_b32_e32 v0, v8
+; GCNX3-HSA-NEXT:    v_mov_b32_e32 v2, v9
+; GCNX3-HSA-NEXT:    v_mov_b32_e32 v9, s3
+; GCNX3-HSA-NEXT:    v_mov_b32_e32 v8, s2
 ; GCNX3-HSA-NEXT:    s_add_u32 s2, s0, 64
 ; GCNX3-HSA-NEXT:    flat_store_dwordx4 v[30:31], v[0:3]
 ; GCNX3-HSA-NEXT:    s_addc_u32 s3, s1, 0
-; GCNX3-HSA-NEXT:    v_mov_b32_e32 v0, v14
-; GCNX3-HSA-NEXT:    v_mov_b32_e32 v2, v15
-; GCNX3-HSA-NEXT:    flat_store_dwordx4 v[12:13], v[0:3]
-; GCNX3-HSA-NEXT:    s_waitcnt vmcnt(12)
-; GCNX3-HSA-NEXT:    v_mov_b32_e32 v0, v8
-; GCNX3-HSA-NEXT:    v_mov_b32_e32 v2, v9
+; GCNX3-HSA-NEXT:    v_mov_b32_e32 v0, v10
+; GCNX3-HSA-NEXT:    v_mov_b32_e32 v2, v11
+; GCNX3-HSA-NEXT:    flat_store_dwordx4 v[8:9], v[0:3]
 ; GCNX3-HSA-NEXT:    v_mov_b32_e32 v9, s3
 ; GCNX3-HSA-NEXT:    v_mov_b32_e32 v8, s2
 ; GCNX3-HSA-NEXT:    s_add_u32 s2, s0, 0x50
+; GCNX3-HSA-NEXT:    s_waitcnt vmcnt(12)
+; GCNX3-HSA-NEXT:    v_mov_b32_e32 v0, v12
+; GCNX3-HSA-NEXT:    v_mov_b32_e32 v2, v13
 ; GCNX3-HSA-NEXT:    s_addc_u32 s3, s1, 0
 ; GCNX3-HSA-NEXT:    flat_store_dwordx4 v[8:9], v[0:3]
 ; GCNX3-HSA-NEXT:    v_mov_b32_e32 v9, s3
 ; GCNX3-HSA-NEXT:    v_mov_b32_e32 v8, s2
 ; GCNX3-HSA-NEXT:    s_add_u32 s2, s0, 32
-; GCNX3-HSA-NEXT:    v_mov_b32_e32 v0, v10
-; GCNX3-HSA-NEXT:    v_mov_b32_e32 v2, v11
+; GCNX3-HSA-NEXT:    v_mov_b32_e32 v0, v14
+; GCNX3-HSA-NEXT:    v_mov_b32_e32 v2, v15
 ; GCNX3-HSA-NEXT:    s_addc_u32 s3, s1, 0
 ; GCNX3-HSA-NEXT:    flat_store_dwordx4 v[8:9], v[0:3]
 ; GCNX3-HSA-NEXT:    s_add_u32 s0, s0, 48
diff --git a/llvm/test/CodeGen/AMDGPU/load-global-i8.ll b/llvm/test/CodeGen/AMDGPU/load-global-i8.ll
index fb34b5e1f3af6..896e60900c745 100644
--- a/llvm/test/CodeGen/AMDGPU/load-global-i8.ll
+++ b/llvm/test/CodeGen/AMDGPU/load-global-i8.ll
@@ -916,38 +916,22 @@ define amdgpu_kernel void @global_zextload_v32i8_to_v32i16(ptr addrspace(1) %out
 
 ; EG-DAG: VTX_READ_128 T{{[0-9]+}}.XYZW, T{{[0-9]+}}.X, 0, #1
 ; EG-DAG: VTX_READ_128 T{{[0-9]+}}.XYZW, T{{[0-9]+}}.X, 16, #1
-; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal
-; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal
-; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal
-; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal
-; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal
-; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal
-; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal
-; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal
-; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal
-; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal
-; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal
-; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal
-; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal
-; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal
-; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal
-; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal
-; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal
-; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal
-; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal
-; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal
-; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal
-; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal
-; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal
-; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal
-; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal
-; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal
-; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal
-; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal
-; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal
-; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal
-; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal
-; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal
+; EG: BFE_{{U?}}INT
+; EG: BFE_{{U?}}INT
+; EG: BFE_{{U?}}INT
+; EG: BFE_{{U?}}INT
+; EG: BFE_{{U?}}INT
+; EG: BFE_{{U?}}INT
+; EG: BFE_{{U?}}INT
+; EG: BFE_{{U?}}INT
+; EG: BFE_{{U?}}INT
+; EG: BFE_{{U?}}INT
+; EG: BFE_{{U?}}INT
+; EG: BFE_{{U?}}INT
+; EG: BFE_{{U?}}INT
+; EG: BFE_{{U?}}INT
+; EG: BFE_{{U?}}INT
+; EG: BFE_{{U?}}INT
 define amdgpu_kernel void @global_sextload_v32i8_to_v32i16(ptr addrspace(1) %out, ptr addrspace(1) %in) #0 {
   %load = load <32 x i8>, ptr addrspace(1) %in
   %ext = sext <32 x i8> %load to <32 x i16>
diff --git a/llvm/test/CodeGen/AMDGPU/load-store-cnt.ll b/llvm/test/CodeGen/AMDGPU/load-store-cnt.ll
new file mode 100644
index 0000000000000..a7fccde416671
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/load-store-cnt.ll
@@ -0,0 +1,48 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc -march=amdgcn -mcpu=gfx1100 < %s | FileCheck --check-prefixes=DEFAULT %s
+; RUN: llc -march=amdgcn -mcpu=gfx1100 -amdgpu-waitcnt-load-forcezero < %s | FileCheck --check-prefixes=LDZERO %s
+
+define amdgpu_kernel void @copy(ptr addrspace(1) noalias nocapture readonly %src1, ptr addrspace(1) noalias nocapture readonly %src2, ptr addrspace(1) noalias nocapture writeonly %dst1, ptr addrspace(1) noalias nocapture writeonly %dst2) {
+; DEFAULT-LABEL: copy:
+; DEFAULT:       ; %bb.0:
+; DEFAULT-NEXT:    s_load_b256 s[0:7], s[4:5], 0x24
+; DEFAULT-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
+; DEFAULT-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; DEFAULT-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; DEFAULT-NEXT:    s_waitcnt lgkmcnt(0)
+; DEFAULT-NEXT:    s_clause 0x1
+; DEFAULT-NEXT:    global_load_b32 v1, v0, s[0:1]
+; DEFAULT-NEXT:    global_load_b32 v2, v0, s[2:3]
+; DEFAULT-NEXT:    s_waitcnt vmcnt(1)
+; DEFAULT-NEXT:    global_store_b32 v0, v1, s[4:5]
+; DEFAULT-NEXT:    s_waitcnt vmcnt(0)
+; DEFAULT-NEXT:    global_store_b32 v0, v2, s[6:7]
+; DEFAULT-NEXT:    s_endpgm
+;
+; LDZERO-LABEL: copy:
+; LDZERO:       ; %bb.0:
+; LDZERO-NEXT:    s_load_b256 s[0:7], s[4:5], 0x24
+; LDZERO-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
+; LDZERO-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; LDZERO-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; LDZERO-NEXT:    s_waitcnt lgkmcnt(0)
+; LDZERO-NEXT:    s_clause 0x1
+; LDZERO-NEXT:    global_load_b32 v1, v0, s[0:1]
+; LDZERO-NEXT:    global_load_b32 v2, v0, s[2:3]
+; LDZERO-NEXT:    s_waitcnt vmcnt(0)
+; LDZERO-NEXT:    s_clause 0x1
+; LDZERO-NEXT:    global_store_b32 v0, v1, s[4:5]
+; LDZERO-NEXT:    global_store_b32 v0, v2, s[6:7]
+; LDZERO-NEXT:    s_endpgm
+  %id = tail call i32 @llvm.amdgcn.workitem.id.x()
+  %idx = zext i32 %id to i64
+  %gep.ld1 = getelementptr inbounds nuw float, ptr addrspace(1) %src1, i64 %idx
+  %v1 = load float, ptr addrspace(1) %gep.ld1, align 4
+  %gep.ld2 = getelementptr inbounds nuw float, ptr addrspace(1) %src2, i64 %idx
+  %v2 = load float, ptr addrspace(1) %gep.ld2, align 4
+  %gep.st1 = getelementptr inbounds nuw float, ptr addrspace(1) %dst1, i64 %idx
+  store float %v1, ptr addrspace(1) %gep.st1, align 4
+  %gep.st2 = getelementptr inbounds nuw float, ptr addrspace(1) %dst2, i64 %idx
+  store float %v2, ptr addrspace(1) %gep.st2, align 4
+  ret void
+}
diff --git a/llvm/test/CodeGen/AMDGPU/local-atomicrmw-fadd.ll b/llvm/test/CodeGen/AMDGPU/local-atomicrmw-fadd.ll
index e4602f20f8a37..5fd6deff0fbbb 100644
--- a/llvm/test/CodeGen/AMDGPU/local-atomicrmw-fadd.ll
+++ b/llvm/test/CodeGen/AMDGPU/local-atomicrmw-fadd.ll
@@ -505,7 +505,6 @@ define double @local_atomic_fadd_ret_f64(ptr addrspace(3) %ptr) nounwind {
 ; GFX12-NEXT:    s_cbranch_execnz .LBB4_1
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: local_atomic_fadd_ret_f64:
@@ -697,7 +696,6 @@ define double @local_atomic_fadd_ret_f64__offset(ptr addrspace(3) %ptr) nounwind
 ; GFX12-NEXT:    s_cbranch_execnz .LBB5_1
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: local_atomic_fadd_ret_f64__offset:
@@ -888,7 +886,6 @@ define void @local_atomic_fadd_noret_f64(ptr addrspace(3) %ptr) nounwind {
 ; GFX12-NEXT:    s_cbranch_execnz .LBB6_1
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: local_atomic_fadd_noret_f64:
@@ -1071,7 +1068,6 @@ define void @local_atomic_fadd_noret_f64__offset(ptr addrspace(3) %ptr) nounwind
 ; GFX12-NEXT:    s_cbranch_execnz .LBB7_1
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: local_atomic_fadd_noret_f64__offset:
@@ -1274,7 +1270,6 @@ define half @local_atomic_fadd_ret_f16(ptr addrspace(3) %ptr) nounwind {
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX12-NEXT:    v_lshrrev_b32_e32 v0, v0, v2
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: local_atomic_fadd_ret_f16:
@@ -1581,7 +1576,6 @@ define half @local_atomic_fadd_ret_f16__offset(ptr addrspace(3) %ptr) nounwind {
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX12-NEXT:    v_lshrrev_b32_e32 v0, v1, v3
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: local_atomic_fadd_ret_f16__offset:
@@ -1895,7 +1889,6 @@ define void @local_atomic_fadd_noret_f16(ptr addrspace(3) %ptr) nounwind {
 ; GFX12-NEXT:    s_cbranch_execnz .LBB10_1
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: local_atomic_fadd_noret_f16:
@@ -2190,7 +2183,6 @@ define void @local_atomic_fadd_noret_f16__offset(ptr addrspace(3) %ptr) nounwind
 ; GFX12-NEXT:    s_cbranch_execnz .LBB11_1
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: local_atomic_fadd_noret_f16__offset:
@@ -2485,7 +2477,6 @@ define half @local_atomic_fadd_ret_f16__offset__align4(ptr addrspace(3) %ptr) no
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX12-NEXT:    v_mov_b32_e32 v0, v1
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: local_atomic_fadd_ret_f16__offset__align4:
@@ -2721,7 +2712,6 @@ define void @local_atomic_fadd_noret_f16__offset__align4(ptr addrspace(3) %ptr)
 ; GFX12-NEXT:    s_cbranch_execnz .LBB13_1
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: local_atomic_fadd_noret_f16__offset__align4:
@@ -2950,8 +2940,9 @@ define bfloat @local_atomic_fadd_ret_bf16(ptr addrspace(3) %ptr) nounwind {
 ; GFX12-NEXT:    v_bfe_u32 v5, v3, 16, 1
 ; GFX12-NEXT:    v_or_b32_e32 v6, 0x400000, v3
 ; GFX12-NEXT:    v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_add3_u32 v5, v5, v3, 0x7fff
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_cndmask_b32_e32 v3, v5, v6, vcc_lo
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_lshrrev_b32_e32 v3, 16, v3
@@ -2971,7 +2962,6 @@ define bfloat @local_atomic_fadd_ret_bf16(ptr addrspace(3) %ptr) nounwind {
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX12-NEXT:    v_lshrrev_b32_e32 v0, v0, v3
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: local_atomic_fadd_ret_bf16:
@@ -3305,8 +3295,9 @@ define bfloat @local_atomic_fadd_ret_bf16__offset(ptr addrspace(3) %ptr) nounwin
 ; GFX12-NEXT:    v_bfe_u32 v5, v3, 16, 1
 ; GFX12-NEXT:    v_or_b32_e32 v6, 0x400000, v3
 ; GFX12-NEXT:    v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_add3_u32 v5, v5, v3, 0x7fff
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_cndmask_b32_e32 v3, v5, v6, vcc_lo
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_lshrrev_b32_e32 v3, 16, v3
@@ -3326,7 +3317,6 @@ define bfloat @local_atomic_fadd_ret_bf16__offset(ptr addrspace(3) %ptr) nounwin
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX12-NEXT:    v_lshrrev_b32_e32 v0, v1, v3
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: local_atomic_fadd_ret_bf16__offset:
@@ -3667,8 +3657,9 @@ define void @local_atomic_fadd_noret_bf16(ptr addrspace(3) %ptr) nounwind {
 ; GFX12-NEXT:    v_bfe_u32 v5, v4, 16, 1
 ; GFX12-NEXT:    v_or_b32_e32 v6, 0x400000, v4
 ; GFX12-NEXT:    v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_add3_u32 v5, v5, v4, 0x7fff
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_cndmask_b32_e32 v4, v5, v6, vcc_lo
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_lshrrev_b32_e32 v4, 16, v4
@@ -3688,7 +3679,6 @@ define void @local_atomic_fadd_noret_bf16(ptr addrspace(3) %ptr) nounwind {
 ; GFX12-NEXT:    s_cbranch_execnz .LBB16_1
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: local_atomic_fadd_noret_bf16:
@@ -4012,6 +4002,7 @@ define void @local_atomic_fadd_noret_bf16__offset(ptr addrspace(3) %ptr) nounwin
 ; GFX12-NEXT:    v_or_b32_e32 v6, 0x400000, v4
 ; GFX12-NEXT:    v_cmp_u_f32_e32 vcc_lo, v4, v4
 ; GFX12-NEXT:    v_add3_u32 v5, v5, v4, 0x7fff
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_cndmask_b32_e32 v4, v5, v6, vcc_lo
 ; GFX12-NEXT:    v_lshrrev_b32_e32 v4, 16, v4
@@ -4031,7 +4022,6 @@ define void @local_atomic_fadd_noret_bf16__offset(ptr addrspace(3) %ptr) nounwin
 ; GFX12-NEXT:    s_cbranch_execnz .LBB17_1
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: local_atomic_fadd_noret_bf16__offset:
@@ -4356,6 +4346,7 @@ define bfloat @local_atomic_fadd_ret_bf16__offset__align4(ptr addrspace(3) %ptr)
 ; GFX12-NEXT:    v_or_b32_e32 v4, 0x400000, v1
 ; GFX12-NEXT:    v_cmp_u_f32_e32 vcc_lo, v1, v1
 ; GFX12-NEXT:    v_add3_u32 v3, v3, v1, 0x7fff
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_cndmask_b32_e32 v1, v3, v4, vcc_lo
 ; GFX12-NEXT:    v_lshrrev_b32_e32 v1, 16, v1
@@ -4374,7 +4365,6 @@ define bfloat @local_atomic_fadd_ret_bf16__offset__align4(ptr addrspace(3) %ptr)
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX12-NEXT:    v_mov_b32_e32 v0, v1
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: local_atomic_fadd_ret_bf16__offset__align4:
@@ -4646,8 +4636,9 @@ define void @local_atomic_fadd_noret_bf16__offset__align4(ptr addrspace(3) %ptr)
 ; GFX12-NEXT:    v_bfe_u32 v3, v2, 16, 1
 ; GFX12-NEXT:    v_or_b32_e32 v4, 0x400000, v2
 ; GFX12-NEXT:    v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_add3_u32 v3, v3, v2, 0x7fff
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_cndmask_b32_e32 v2, v3, v4, vcc_lo
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_lshrrev_b32_e32 v2, 16, v2
@@ -4665,7 +4656,6 @@ define void @local_atomic_fadd_noret_bf16__offset__align4(ptr addrspace(3) %ptr)
 ; GFX12-NEXT:    s_cbranch_execnz .LBB19_1
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: local_atomic_fadd_noret_bf16__offset__align4:
@@ -7030,7 +7020,6 @@ define amdgpu_kernel void @local_ds_fadd(ptr addrspace(1) %out, ptr addrspace(3)
 ; GFX12-NEXT:    s_and_saveexec_b32 s0, vcc_lo
 ; GFX12-NEXT:    s_cbranch_execz .LBB28_2
 ; GFX12-NEXT:  ; %bb.1:
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_bcnt1_i32_b32 s3, s6
 ; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_cvt_f32_ubyte0_e32 v1, s3
@@ -7106,6 +7095,7 @@ define amdgpu_kernel void @local_ds_fadd(ptr addrspace(1) %out, ptr addrspace(3)
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s1
 ; GFX12-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
 ; GFX12-NEXT:    v_readfirstlane_b32 s2, v1
+; GFX12-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_add_f32 v0, s2, v0
 ; GFX12-NEXT:    v_cndmask_b32_e64 v0, v0, s2, vcc_lo
@@ -7897,7 +7887,6 @@ define amdgpu_kernel void @local_ds_fadd_one_as(ptr addrspace(1) %out, ptr addrs
 ; GFX12-NEXT:    s_and_saveexec_b32 s0, vcc_lo
 ; GFX12-NEXT:    s_cbranch_execz .LBB29_2
 ; GFX12-NEXT:  ; %bb.1:
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_bcnt1_i32_b32 s3, s6
 ; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_cvt_f32_ubyte0_e32 v1, s3
@@ -7969,6 +7958,7 @@ define amdgpu_kernel void @local_ds_fadd_one_as(ptr addrspace(1) %out, ptr addrs
 ; GFX12-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
 ; GFX12-NEXT:    s_wait_dscnt 0x0
 ; GFX12-NEXT:    v_readfirstlane_b32 s2, v1
+; GFX12-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_add_f32 v0, s2, v0
 ; GFX12-NEXT:    v_cndmask_b32_e64 v0, v0, s2, vcc_lo
diff --git a/llvm/test/CodeGen/AMDGPU/local-atomicrmw-fmax.ll b/llvm/test/CodeGen/AMDGPU/local-atomicrmw-fmax.ll
index d419b0cdfdd1a..1e8072460c7a3 100644
--- a/llvm/test/CodeGen/AMDGPU/local-atomicrmw-fmax.ll
+++ b/llvm/test/CodeGen/AMDGPU/local-atomicrmw-fmax.ll
@@ -815,7 +815,6 @@ define half @local_atomic_fmax_ret_f16(ptr addrspace(3) %ptr) nounwind {
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX12-NEXT:    v_lshrrev_b32_e32 v0, v0, v3
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: local_atomic_fmax_ret_f16:
@@ -1129,7 +1128,6 @@ define half @local_atomic_fmax_ret_f16__offset(ptr addrspace(3) %ptr) nounwind {
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX12-NEXT:    v_lshrrev_b32_e32 v0, v1, v3
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: local_atomic_fmax_ret_f16__offset:
@@ -1450,7 +1448,6 @@ define void @local_atomic_fmax_noret_f16(ptr addrspace(3) %ptr) nounwind {
 ; GFX12-NEXT:    s_cbranch_execnz .LBB10_1
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: local_atomic_fmax_noret_f16:
@@ -1753,7 +1750,6 @@ define void @local_atomic_fmax_noret_f16__offset(ptr addrspace(3) %ptr) nounwind
 ; GFX12-NEXT:    s_cbranch_execnz .LBB11_1
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: local_atomic_fmax_noret_f16__offset:
@@ -2056,7 +2052,6 @@ define half @local_atomic_fmax_ret_f16__offset__align4(ptr addrspace(3) %ptr) no
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX12-NEXT:    v_mov_b32_e32 v0, v1
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: local_atomic_fmax_ret_f16__offset__align4:
@@ -2300,7 +2295,6 @@ define void @local_atomic_fmax_noret_f16__offset__align4(ptr addrspace(3) %ptr)
 ; GFX12-NEXT:    s_cbranch_execnz .LBB13_1
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: local_atomic_fmax_noret_f16__offset__align4:
@@ -2536,8 +2530,9 @@ define bfloat @local_atomic_fmax_ret_bf16(ptr addrspace(3) %ptr) nounwind {
 ; GFX12-NEXT:    v_bfe_u32 v5, v3, 16, 1
 ; GFX12-NEXT:    v_or_b32_e32 v6, 0x400000, v3
 ; GFX12-NEXT:    v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_add3_u32 v5, v5, v3, 0x7fff
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_cndmask_b32_e32 v3, v5, v6, vcc_lo
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_lshrrev_b32_e32 v3, 16, v3
@@ -2557,7 +2552,6 @@ define bfloat @local_atomic_fmax_ret_bf16(ptr addrspace(3) %ptr) nounwind {
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX12-NEXT:    v_lshrrev_b32_e32 v0, v0, v3
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: local_atomic_fmax_ret_bf16:
@@ -2893,8 +2887,9 @@ define bfloat @local_atomic_fmax_ret_bf16__offset(ptr addrspace(3) %ptr) nounwin
 ; GFX12-NEXT:    v_bfe_u32 v5, v3, 16, 1
 ; GFX12-NEXT:    v_or_b32_e32 v6, 0x400000, v3
 ; GFX12-NEXT:    v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_add3_u32 v5, v5, v3, 0x7fff
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_cndmask_b32_e32 v3, v5, v6, vcc_lo
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_lshrrev_b32_e32 v3, 16, v3
@@ -2914,7 +2909,6 @@ define bfloat @local_atomic_fmax_ret_bf16__offset(ptr addrspace(3) %ptr) nounwin
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX12-NEXT:    v_lshrrev_b32_e32 v0, v1, v3
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: local_atomic_fmax_ret_bf16__offset:
@@ -3257,8 +3251,9 @@ define void @local_atomic_fmax_noret_bf16(ptr addrspace(3) %ptr) nounwind {
 ; GFX12-NEXT:    v_bfe_u32 v5, v4, 16, 1
 ; GFX12-NEXT:    v_or_b32_e32 v6, 0x400000, v4
 ; GFX12-NEXT:    v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_add3_u32 v5, v5, v4, 0x7fff
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_cndmask_b32_e32 v4, v5, v6, vcc_lo
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_lshrrev_b32_e32 v4, 16, v4
@@ -3278,7 +3273,6 @@ define void @local_atomic_fmax_noret_bf16(ptr addrspace(3) %ptr) nounwind {
 ; GFX12-NEXT:    s_cbranch_execnz .LBB16_1
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: local_atomic_fmax_noret_bf16:
@@ -3604,6 +3598,7 @@ define void @local_atomic_fmax_noret_bf16__offset(ptr addrspace(3) %ptr) nounwin
 ; GFX12-NEXT:    v_or_b32_e32 v6, 0x400000, v4
 ; GFX12-NEXT:    v_cmp_u_f32_e32 vcc_lo, v4, v4
 ; GFX12-NEXT:    v_add3_u32 v5, v5, v4, 0x7fff
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_cndmask_b32_e32 v4, v5, v6, vcc_lo
 ; GFX12-NEXT:    v_lshrrev_b32_e32 v4, 16, v4
@@ -3623,7 +3618,6 @@ define void @local_atomic_fmax_noret_bf16__offset(ptr addrspace(3) %ptr) nounwin
 ; GFX12-NEXT:    s_cbranch_execnz .LBB17_1
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: local_atomic_fmax_noret_bf16__offset:
@@ -3950,6 +3944,7 @@ define bfloat @local_atomic_fmax_ret_bf16__offset__align4(ptr addrspace(3) %ptr)
 ; GFX12-NEXT:    v_or_b32_e32 v4, 0x400000, v1
 ; GFX12-NEXT:    v_cmp_u_f32_e32 vcc_lo, v1, v1
 ; GFX12-NEXT:    v_add3_u32 v3, v3, v1, 0x7fff
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_cndmask_b32_e32 v1, v3, v4, vcc_lo
 ; GFX12-NEXT:    v_lshrrev_b32_e32 v1, 16, v1
@@ -3968,7 +3963,6 @@ define bfloat @local_atomic_fmax_ret_bf16__offset__align4(ptr addrspace(3) %ptr)
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX12-NEXT:    v_mov_b32_e32 v0, v1
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: local_atomic_fmax_ret_bf16__offset__align4:
@@ -4242,8 +4236,9 @@ define void @local_atomic_fmax_noret_bf16__offset__align4(ptr addrspace(3) %ptr)
 ; GFX12-NEXT:    v_bfe_u32 v3, v2, 16, 1
 ; GFX12-NEXT:    v_or_b32_e32 v4, 0x400000, v2
 ; GFX12-NEXT:    v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_add3_u32 v3, v3, v2, 0x7fff
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_cndmask_b32_e32 v2, v3, v4, vcc_lo
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_lshrrev_b32_e32 v2, 16, v2
@@ -4261,7 +4256,6 @@ define void @local_atomic_fmax_noret_bf16__offset__align4(ptr addrspace(3) %ptr)
 ; GFX12-NEXT:    s_cbranch_execnz .LBB19_1
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: local_atomic_fmax_noret_bf16__offset__align4:
@@ -4542,7 +4536,6 @@ define <2 x half> @local_atomic_fmax_ret_v2f16(ptr addrspace(3) %ptr, <2 x half>
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX12-NEXT:    v_mov_b32_e32 v0, v2
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: local_atomic_fmax_ret_v2f16:
@@ -4814,7 +4807,6 @@ define <2 x half> @local_atomic_fmax_ret_v2f16__offset(ptr addrspace(3) %ptr, <2
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX12-NEXT:    v_mov_b32_e32 v0, v2
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: local_atomic_fmax_ret_v2f16__offset:
@@ -5085,7 +5077,6 @@ define void @local_atomic_fmax_noret_v2f16(ptr addrspace(3) %ptr, <2 x half> %va
 ; GFX12-NEXT:    s_cbranch_execnz .LBB22_1
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: local_atomic_fmax_noret_v2f16:
@@ -5347,7 +5338,6 @@ define void @local_atomic_fmax_noret_v2f16__offset(ptr addrspace(3) %ptr, <2 x h
 ; GFX12-NEXT:    s_cbranch_execnz .LBB23_1
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: local_atomic_fmax_noret_v2f16__offset:
@@ -5609,6 +5599,7 @@ define <2 x bfloat> @local_atomic_fmax_ret_v2bf16(ptr addrspace(3) %ptr, <2 x bf
 ; GFX12-NEXT:    v_or_b32_e32 v9, 0x400000, v5
 ; GFX12-NEXT:    v_cmp_u_f32_e32 vcc_lo, v5, v5
 ; GFX12-NEXT:    v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_dual_cndmask_b32 v5, v7, v9 :: v_dual_lshlrev_b32 v2, 16, v4
 ; GFX12-NEXT:    v_max_num_f32_e32 v2, v2, v3
@@ -5617,6 +5608,7 @@ define <2 x bfloat> @local_atomic_fmax_ret_v2bf16(ptr addrspace(3) %ptr, <2 x bf
 ; GFX12-NEXT:    v_or_b32_e32 v8, 0x400000, v2
 ; GFX12-NEXT:    v_cmp_u_f32_e64 s0, v2, v2
 ; GFX12-NEXT:    v_add3_u32 v6, v6, v2, 0x7fff
+; GFX12-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_cndmask_b32_e64 v2, v6, v8, s0
 ; GFX12-NEXT:    v_perm_b32 v2, v5, v2, 0x7060302
@@ -5633,7 +5625,6 @@ define <2 x bfloat> @local_atomic_fmax_ret_v2bf16(ptr addrspace(3) %ptr, <2 x bf
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s1
 ; GFX12-NEXT:    v_mov_b32_e32 v0, v2
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: local_atomic_fmax_ret_v2bf16:
@@ -5986,6 +5977,7 @@ define <2 x bfloat> @local_atomic_fmax_ret_v2bf16__offset(ptr addrspace(3) %ptr,
 ; GFX12-NEXT:    v_or_b32_e32 v9, 0x400000, v5
 ; GFX12-NEXT:    v_cmp_u_f32_e32 vcc_lo, v5, v5
 ; GFX12-NEXT:    v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_dual_cndmask_b32 v5, v7, v9 :: v_dual_lshlrev_b32 v2, 16, v4
 ; GFX12-NEXT:    v_max_num_f32_e32 v2, v2, v3
@@ -5994,6 +5986,7 @@ define <2 x bfloat> @local_atomic_fmax_ret_v2bf16__offset(ptr addrspace(3) %ptr,
 ; GFX12-NEXT:    v_or_b32_e32 v8, 0x400000, v2
 ; GFX12-NEXT:    v_cmp_u_f32_e64 s0, v2, v2
 ; GFX12-NEXT:    v_add3_u32 v6, v6, v2, 0x7fff
+; GFX12-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_cndmask_b32_e64 v2, v6, v8, s0
 ; GFX12-NEXT:    v_perm_b32 v2, v5, v2, 0x7060302
@@ -6010,7 +6003,6 @@ define <2 x bfloat> @local_atomic_fmax_ret_v2bf16__offset(ptr addrspace(3) %ptr,
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s1
 ; GFX12-NEXT:    v_mov_b32_e32 v0, v2
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: local_atomic_fmax_ret_v2bf16__offset:
@@ -6368,8 +6360,10 @@ define void @local_atomic_fmax_noret_v2bf16(ptr addrspace(3) %ptr, <2 x bfloat>
 ; GFX12-NEXT:    v_add3_u32 v7, v7, v5, 0x7fff
 ; GFX12-NEXT:    v_add3_u32 v6, v6, v4, 0x7fff
 ; GFX12-NEXT:    v_cmp_u_f32_e64 s0, v4, v4
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-NEXT:    s_wait_alu 0xfffd
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX12-NEXT:    v_cndmask_b32_e32 v5, v7, v9, vcc_lo
+; GFX12-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-NEXT:    v_cndmask_b32_e64 v4, v6, v8, s0
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-NEXT:    v_perm_b32 v4, v5, v4, 0x7060302
@@ -6386,7 +6380,6 @@ define void @local_atomic_fmax_noret_v2bf16(ptr addrspace(3) %ptr, <2 x bfloat>
 ; GFX12-NEXT:    s_cbranch_execnz .LBB26_1
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s1
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: local_atomic_fmax_noret_v2bf16:
@@ -6732,8 +6725,10 @@ define void @local_atomic_fmax_noret_v2bf16__ofset(ptr addrspace(3) %ptr, <2 x b
 ; GFX12-NEXT:    v_add3_u32 v7, v7, v5, 0x7fff
 ; GFX12-NEXT:    v_add3_u32 v6, v6, v4, 0x7fff
 ; GFX12-NEXT:    v_cmp_u_f32_e64 s0, v4, v4
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-NEXT:    s_wait_alu 0xfffd
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX12-NEXT:    v_cndmask_b32_e32 v5, v7, v9, vcc_lo
+; GFX12-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-NEXT:    v_cndmask_b32_e64 v4, v6, v8, s0
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-NEXT:    v_perm_b32 v4, v5, v4, 0x7060302
@@ -6750,7 +6745,6 @@ define void @local_atomic_fmax_noret_v2bf16__ofset(ptr addrspace(3) %ptr, <2 x b
 ; GFX12-NEXT:    s_cbranch_execnz .LBB27_1
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s1
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: local_atomic_fmax_noret_v2bf16__ofset:
diff --git a/llvm/test/CodeGen/AMDGPU/local-atomicrmw-fmin.ll b/llvm/test/CodeGen/AMDGPU/local-atomicrmw-fmin.ll
index 282947afa409a..7249b0b1fc0e3 100644
--- a/llvm/test/CodeGen/AMDGPU/local-atomicrmw-fmin.ll
+++ b/llvm/test/CodeGen/AMDGPU/local-atomicrmw-fmin.ll
@@ -815,7 +815,6 @@ define half @local_atomic_fmin_ret_f16(ptr addrspace(3) %ptr) nounwind {
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX12-NEXT:    v_lshrrev_b32_e32 v0, v0, v3
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: local_atomic_fmin_ret_f16:
@@ -1129,7 +1128,6 @@ define half @local_atomic_fmin_ret_f16__offset(ptr addrspace(3) %ptr) nounwind {
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX12-NEXT:    v_lshrrev_b32_e32 v0, v1, v3
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: local_atomic_fmin_ret_f16__offset:
@@ -1450,7 +1448,6 @@ define void @local_atomic_fmin_noret_f16(ptr addrspace(3) %ptr) nounwind {
 ; GFX12-NEXT:    s_cbranch_execnz .LBB10_1
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: local_atomic_fmin_noret_f16:
@@ -1753,7 +1750,6 @@ define void @local_atomic_fmin_noret_f16__offset(ptr addrspace(3) %ptr) nounwind
 ; GFX12-NEXT:    s_cbranch_execnz .LBB11_1
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: local_atomic_fmin_noret_f16__offset:
@@ -2056,7 +2052,6 @@ define half @local_atomic_fmin_ret_f16__offset__align4(ptr addrspace(3) %ptr) no
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX12-NEXT:    v_mov_b32_e32 v0, v1
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: local_atomic_fmin_ret_f16__offset__align4:
@@ -2300,7 +2295,6 @@ define void @local_atomic_fmin_noret_f16__offset__align4(ptr addrspace(3) %ptr)
 ; GFX12-NEXT:    s_cbranch_execnz .LBB13_1
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: local_atomic_fmin_noret_f16__offset__align4:
@@ -2536,8 +2530,9 @@ define bfloat @local_atomic_fmin_ret_bf16(ptr addrspace(3) %ptr) nounwind {
 ; GFX12-NEXT:    v_bfe_u32 v5, v3, 16, 1
 ; GFX12-NEXT:    v_or_b32_e32 v6, 0x400000, v3
 ; GFX12-NEXT:    v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_add3_u32 v5, v5, v3, 0x7fff
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_cndmask_b32_e32 v3, v5, v6, vcc_lo
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_lshrrev_b32_e32 v3, 16, v3
@@ -2557,7 +2552,6 @@ define bfloat @local_atomic_fmin_ret_bf16(ptr addrspace(3) %ptr) nounwind {
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX12-NEXT:    v_lshrrev_b32_e32 v0, v0, v3
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: local_atomic_fmin_ret_bf16:
@@ -2893,8 +2887,9 @@ define bfloat @local_atomic_fmin_ret_bf16__offset(ptr addrspace(3) %ptr) nounwin
 ; GFX12-NEXT:    v_bfe_u32 v5, v3, 16, 1
 ; GFX12-NEXT:    v_or_b32_e32 v6, 0x400000, v3
 ; GFX12-NEXT:    v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_add3_u32 v5, v5, v3, 0x7fff
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_cndmask_b32_e32 v3, v5, v6, vcc_lo
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_lshrrev_b32_e32 v3, 16, v3
@@ -2914,7 +2909,6 @@ define bfloat @local_atomic_fmin_ret_bf16__offset(ptr addrspace(3) %ptr) nounwin
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX12-NEXT:    v_lshrrev_b32_e32 v0, v1, v3
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: local_atomic_fmin_ret_bf16__offset:
@@ -3257,8 +3251,9 @@ define void @local_atomic_fmin_noret_bf16(ptr addrspace(3) %ptr) nounwind {
 ; GFX12-NEXT:    v_bfe_u32 v5, v4, 16, 1
 ; GFX12-NEXT:    v_or_b32_e32 v6, 0x400000, v4
 ; GFX12-NEXT:    v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_add3_u32 v5, v5, v4, 0x7fff
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_cndmask_b32_e32 v4, v5, v6, vcc_lo
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_lshrrev_b32_e32 v4, 16, v4
@@ -3278,7 +3273,6 @@ define void @local_atomic_fmin_noret_bf16(ptr addrspace(3) %ptr) nounwind {
 ; GFX12-NEXT:    s_cbranch_execnz .LBB16_1
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: local_atomic_fmin_noret_bf16:
@@ -3604,6 +3598,7 @@ define void @local_atomic_fmin_noret_bf16__offset(ptr addrspace(3) %ptr) nounwin
 ; GFX12-NEXT:    v_or_b32_e32 v6, 0x400000, v4
 ; GFX12-NEXT:    v_cmp_u_f32_e32 vcc_lo, v4, v4
 ; GFX12-NEXT:    v_add3_u32 v5, v5, v4, 0x7fff
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_cndmask_b32_e32 v4, v5, v6, vcc_lo
 ; GFX12-NEXT:    v_lshrrev_b32_e32 v4, 16, v4
@@ -3623,7 +3618,6 @@ define void @local_atomic_fmin_noret_bf16__offset(ptr addrspace(3) %ptr) nounwin
 ; GFX12-NEXT:    s_cbranch_execnz .LBB17_1
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: local_atomic_fmin_noret_bf16__offset:
@@ -3950,6 +3944,7 @@ define bfloat @local_atomic_fmin_ret_bf16__offset__align4(ptr addrspace(3) %ptr)
 ; GFX12-NEXT:    v_or_b32_e32 v4, 0x400000, v1
 ; GFX12-NEXT:    v_cmp_u_f32_e32 vcc_lo, v1, v1
 ; GFX12-NEXT:    v_add3_u32 v3, v3, v1, 0x7fff
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_cndmask_b32_e32 v1, v3, v4, vcc_lo
 ; GFX12-NEXT:    v_lshrrev_b32_e32 v1, 16, v1
@@ -3968,7 +3963,6 @@ define bfloat @local_atomic_fmin_ret_bf16__offset__align4(ptr addrspace(3) %ptr)
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX12-NEXT:    v_mov_b32_e32 v0, v1
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: local_atomic_fmin_ret_bf16__offset__align4:
@@ -4242,8 +4236,9 @@ define void @local_atomic_fmin_noret_bf16__offset__align4(ptr addrspace(3) %ptr)
 ; GFX12-NEXT:    v_bfe_u32 v3, v2, 16, 1
 ; GFX12-NEXT:    v_or_b32_e32 v4, 0x400000, v2
 ; GFX12-NEXT:    v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_add3_u32 v3, v3, v2, 0x7fff
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_cndmask_b32_e32 v2, v3, v4, vcc_lo
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_lshrrev_b32_e32 v2, 16, v2
@@ -4261,7 +4256,6 @@ define void @local_atomic_fmin_noret_bf16__offset__align4(ptr addrspace(3) %ptr)
 ; GFX12-NEXT:    s_cbranch_execnz .LBB19_1
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: local_atomic_fmin_noret_bf16__offset__align4:
@@ -4542,7 +4536,6 @@ define <2 x half> @local_atomic_fmin_ret_v2f16(ptr addrspace(3) %ptr, <2 x half>
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX12-NEXT:    v_mov_b32_e32 v0, v2
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: local_atomic_fmin_ret_v2f16:
@@ -4814,7 +4807,6 @@ define <2 x half> @local_atomic_fmin_ret_v2f16__offset(ptr addrspace(3) %ptr, <2
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX12-NEXT:    v_mov_b32_e32 v0, v2
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: local_atomic_fmin_ret_v2f16__offset:
@@ -5085,7 +5077,6 @@ define void @local_atomic_fmin_noret_v2f16(ptr addrspace(3) %ptr, <2 x half> %va
 ; GFX12-NEXT:    s_cbranch_execnz .LBB22_1
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: local_atomic_fmin_noret_v2f16:
@@ -5347,7 +5338,6 @@ define void @local_atomic_fmin_noret_v2f16__offset(ptr addrspace(3) %ptr, <2 x h
 ; GFX12-NEXT:    s_cbranch_execnz .LBB23_1
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: local_atomic_fmin_noret_v2f16__offset:
@@ -5609,6 +5599,7 @@ define <2 x bfloat> @local_atomic_fmin_ret_v2bf16(ptr addrspace(3) %ptr, <2 x bf
 ; GFX12-NEXT:    v_or_b32_e32 v9, 0x400000, v5
 ; GFX12-NEXT:    v_cmp_u_f32_e32 vcc_lo, v5, v5
 ; GFX12-NEXT:    v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_dual_cndmask_b32 v5, v7, v9 :: v_dual_lshlrev_b32 v2, 16, v4
 ; GFX12-NEXT:    v_min_num_f32_e32 v2, v2, v3
@@ -5617,6 +5608,7 @@ define <2 x bfloat> @local_atomic_fmin_ret_v2bf16(ptr addrspace(3) %ptr, <2 x bf
 ; GFX12-NEXT:    v_or_b32_e32 v8, 0x400000, v2
 ; GFX12-NEXT:    v_cmp_u_f32_e64 s0, v2, v2
 ; GFX12-NEXT:    v_add3_u32 v6, v6, v2, 0x7fff
+; GFX12-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_cndmask_b32_e64 v2, v6, v8, s0
 ; GFX12-NEXT:    v_perm_b32 v2, v5, v2, 0x7060302
@@ -5633,7 +5625,6 @@ define <2 x bfloat> @local_atomic_fmin_ret_v2bf16(ptr addrspace(3) %ptr, <2 x bf
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s1
 ; GFX12-NEXT:    v_mov_b32_e32 v0, v2
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: local_atomic_fmin_ret_v2bf16:
@@ -5986,6 +5977,7 @@ define <2 x bfloat> @local_atomic_fmin_ret_v2bf16__offset(ptr addrspace(3) %ptr,
 ; GFX12-NEXT:    v_or_b32_e32 v9, 0x400000, v5
 ; GFX12-NEXT:    v_cmp_u_f32_e32 vcc_lo, v5, v5
 ; GFX12-NEXT:    v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_dual_cndmask_b32 v5, v7, v9 :: v_dual_lshlrev_b32 v2, 16, v4
 ; GFX12-NEXT:    v_min_num_f32_e32 v2, v2, v3
@@ -5994,6 +5986,7 @@ define <2 x bfloat> @local_atomic_fmin_ret_v2bf16__offset(ptr addrspace(3) %ptr,
 ; GFX12-NEXT:    v_or_b32_e32 v8, 0x400000, v2
 ; GFX12-NEXT:    v_cmp_u_f32_e64 s0, v2, v2
 ; GFX12-NEXT:    v_add3_u32 v6, v6, v2, 0x7fff
+; GFX12-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_cndmask_b32_e64 v2, v6, v8, s0
 ; GFX12-NEXT:    v_perm_b32 v2, v5, v2, 0x7060302
@@ -6010,7 +6003,6 @@ define <2 x bfloat> @local_atomic_fmin_ret_v2bf16__offset(ptr addrspace(3) %ptr,
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s1
 ; GFX12-NEXT:    v_mov_b32_e32 v0, v2
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: local_atomic_fmin_ret_v2bf16__offset:
@@ -6368,8 +6360,10 @@ define void @local_atomic_fmin_noret_v2bf16(ptr addrspace(3) %ptr, <2 x bfloat>
 ; GFX12-NEXT:    v_add3_u32 v7, v7, v5, 0x7fff
 ; GFX12-NEXT:    v_add3_u32 v6, v6, v4, 0x7fff
 ; GFX12-NEXT:    v_cmp_u_f32_e64 s0, v4, v4
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-NEXT:    s_wait_alu 0xfffd
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX12-NEXT:    v_cndmask_b32_e32 v5, v7, v9, vcc_lo
+; GFX12-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-NEXT:    v_cndmask_b32_e64 v4, v6, v8, s0
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-NEXT:    v_perm_b32 v4, v5, v4, 0x7060302
@@ -6386,7 +6380,6 @@ define void @local_atomic_fmin_noret_v2bf16(ptr addrspace(3) %ptr, <2 x bfloat>
 ; GFX12-NEXT:    s_cbranch_execnz .LBB26_1
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s1
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: local_atomic_fmin_noret_v2bf16:
@@ -6732,8 +6725,10 @@ define void @local_atomic_fmin_noret_v2bf16__ofset(ptr addrspace(3) %ptr, <2 x b
 ; GFX12-NEXT:    v_add3_u32 v7, v7, v5, 0x7fff
 ; GFX12-NEXT:    v_add3_u32 v6, v6, v4, 0x7fff
 ; GFX12-NEXT:    v_cmp_u_f32_e64 s0, v4, v4
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-NEXT:    s_wait_alu 0xfffd
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX12-NEXT:    v_cndmask_b32_e32 v5, v7, v9, vcc_lo
+; GFX12-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-NEXT:    v_cndmask_b32_e64 v4, v6, v8, s0
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-NEXT:    v_perm_b32 v4, v5, v4, 0x7060302
@@ -6750,7 +6745,6 @@ define void @local_atomic_fmin_noret_v2bf16__ofset(ptr addrspace(3) %ptr, <2 x b
 ; GFX12-NEXT:    s_cbranch_execnz .LBB27_1
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s1
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: local_atomic_fmin_noret_v2bf16__ofset:
diff --git a/llvm/test/CodeGen/AMDGPU/local-atomicrmw-fsub.ll b/llvm/test/CodeGen/AMDGPU/local-atomicrmw-fsub.ll
index 967e972e53e29..65e00c50292dc 100644
--- a/llvm/test/CodeGen/AMDGPU/local-atomicrmw-fsub.ll
+++ b/llvm/test/CodeGen/AMDGPU/local-atomicrmw-fsub.ll
@@ -42,7 +42,6 @@ define float @local_atomic_fsub_ret_f32(ptr addrspace(3) %ptr) nounwind {
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX12-NEXT:    v_mov_b32_e32 v0, v1
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: local_atomic_fsub_ret_f32:
@@ -254,7 +253,6 @@ define float @local_atomic_fsub_ret_f32__offset(ptr addrspace(3) %ptr) nounwind
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX12-NEXT:    v_mov_b32_e32 v0, v1
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: local_atomic_fsub_ret_f32__offset:
@@ -465,7 +463,6 @@ define void @local_atomic_fsub_noret_f32(ptr addrspace(3) %ptr) nounwind {
 ; GFX12-NEXT:    s_cbranch_execnz .LBB2_1
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: local_atomic_fsub_noret_f32:
@@ -666,7 +663,6 @@ define void @local_atomic_fsub_noret_f32__offset(ptr addrspace(3) %ptr) nounwind
 ; GFX12-NEXT:    s_cbranch_execnz .LBB3_1
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: local_atomic_fsub_noret_f32__offset:
@@ -875,7 +871,6 @@ define double @local_atomic_fsub_ret_f64(ptr addrspace(3) %ptr) nounwind {
 ; GFX12-NEXT:    s_cbranch_execnz .LBB4_1
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: local_atomic_fsub_ret_f64:
@@ -1092,7 +1087,6 @@ define double @local_atomic_fsub_ret_f64__offset(ptr addrspace(3) %ptr) nounwind
 ; GFX12-NEXT:    s_cbranch_execnz .LBB5_1
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: local_atomic_fsub_ret_f64__offset:
@@ -1308,7 +1302,6 @@ define void @local_atomic_fsub_noret_f64(ptr addrspace(3) %ptr) nounwind {
 ; GFX12-NEXT:    s_cbranch_execnz .LBB6_1
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: local_atomic_fsub_noret_f64:
@@ -1514,7 +1507,6 @@ define void @local_atomic_fsub_noret_f64__offset(ptr addrspace(3) %ptr) nounwind
 ; GFX12-NEXT:    s_cbranch_execnz .LBB7_1
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: local_atomic_fsub_noret_f64__offset:
@@ -1740,7 +1732,6 @@ define half @local_atomic_fsub_ret_f16(ptr addrspace(3) %ptr) nounwind {
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX12-NEXT:    v_lshrrev_b32_e32 v0, v0, v2
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: local_atomic_fsub_ret_f16:
@@ -2047,7 +2038,6 @@ define half @local_atomic_fsub_ret_f16__offset(ptr addrspace(3) %ptr) nounwind {
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX12-NEXT:    v_lshrrev_b32_e32 v0, v1, v3
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: local_atomic_fsub_ret_f16__offset:
@@ -2361,7 +2351,6 @@ define void @local_atomic_fsub_noret_f16(ptr addrspace(3) %ptr) nounwind {
 ; GFX12-NEXT:    s_cbranch_execnz .LBB10_1
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: local_atomic_fsub_noret_f16:
@@ -2656,7 +2645,6 @@ define void @local_atomic_fsub_noret_f16__offset(ptr addrspace(3) %ptr) nounwind
 ; GFX12-NEXT:    s_cbranch_execnz .LBB11_1
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: local_atomic_fsub_noret_f16__offset:
@@ -2951,7 +2939,6 @@ define half @local_atomic_fsub_ret_f16__offset__align4(ptr addrspace(3) %ptr) no
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX12-NEXT:    v_mov_b32_e32 v0, v1
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: local_atomic_fsub_ret_f16__offset__align4:
@@ -3187,7 +3174,6 @@ define void @local_atomic_fsub_noret_f16__offset__align4(ptr addrspace(3) %ptr)
 ; GFX12-NEXT:    s_cbranch_execnz .LBB13_1
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: local_atomic_fsub_noret_f16__offset__align4:
@@ -3416,8 +3402,9 @@ define bfloat @local_atomic_fsub_ret_bf16(ptr addrspace(3) %ptr) nounwind {
 ; GFX12-NEXT:    v_bfe_u32 v5, v3, 16, 1
 ; GFX12-NEXT:    v_or_b32_e32 v6, 0x400000, v3
 ; GFX12-NEXT:    v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_add3_u32 v5, v5, v3, 0x7fff
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_cndmask_b32_e32 v3, v5, v6, vcc_lo
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_lshrrev_b32_e32 v3, 16, v3
@@ -3437,7 +3424,6 @@ define bfloat @local_atomic_fsub_ret_bf16(ptr addrspace(3) %ptr) nounwind {
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX12-NEXT:    v_lshrrev_b32_e32 v0, v0, v3
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: local_atomic_fsub_ret_bf16:
@@ -3771,8 +3757,9 @@ define bfloat @local_atomic_fsub_ret_bf16__offset(ptr addrspace(3) %ptr) nounwin
 ; GFX12-NEXT:    v_bfe_u32 v5, v3, 16, 1
 ; GFX12-NEXT:    v_or_b32_e32 v6, 0x400000, v3
 ; GFX12-NEXT:    v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_add3_u32 v5, v5, v3, 0x7fff
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_cndmask_b32_e32 v3, v5, v6, vcc_lo
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_lshrrev_b32_e32 v3, 16, v3
@@ -3792,7 +3779,6 @@ define bfloat @local_atomic_fsub_ret_bf16__offset(ptr addrspace(3) %ptr) nounwin
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX12-NEXT:    v_lshrrev_b32_e32 v0, v1, v3
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: local_atomic_fsub_ret_bf16__offset:
@@ -4133,8 +4119,9 @@ define void @local_atomic_fsub_noret_bf16(ptr addrspace(3) %ptr) nounwind {
 ; GFX12-NEXT:    v_bfe_u32 v5, v4, 16, 1
 ; GFX12-NEXT:    v_or_b32_e32 v6, 0x400000, v4
 ; GFX12-NEXT:    v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_add3_u32 v5, v5, v4, 0x7fff
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_cndmask_b32_e32 v4, v5, v6, vcc_lo
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_lshrrev_b32_e32 v4, 16, v4
@@ -4154,7 +4141,6 @@ define void @local_atomic_fsub_noret_bf16(ptr addrspace(3) %ptr) nounwind {
 ; GFX12-NEXT:    s_cbranch_execnz .LBB16_1
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: local_atomic_fsub_noret_bf16:
@@ -4478,6 +4464,7 @@ define void @local_atomic_fsub_noret_bf16__offset(ptr addrspace(3) %ptr) nounwin
 ; GFX12-NEXT:    v_or_b32_e32 v6, 0x400000, v4
 ; GFX12-NEXT:    v_cmp_u_f32_e32 vcc_lo, v4, v4
 ; GFX12-NEXT:    v_add3_u32 v5, v5, v4, 0x7fff
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_cndmask_b32_e32 v4, v5, v6, vcc_lo
 ; GFX12-NEXT:    v_lshrrev_b32_e32 v4, 16, v4
@@ -4497,7 +4484,6 @@ define void @local_atomic_fsub_noret_bf16__offset(ptr addrspace(3) %ptr) nounwin
 ; GFX12-NEXT:    s_cbranch_execnz .LBB17_1
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: local_atomic_fsub_noret_bf16__offset:
@@ -4822,6 +4808,7 @@ define bfloat @local_atomic_fsub_ret_bf16__offset__align4(ptr addrspace(3) %ptr)
 ; GFX12-NEXT:    v_or_b32_e32 v4, 0x400000, v1
 ; GFX12-NEXT:    v_cmp_u_f32_e32 vcc_lo, v1, v1
 ; GFX12-NEXT:    v_add3_u32 v3, v3, v1, 0x7fff
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_cndmask_b32_e32 v1, v3, v4, vcc_lo
 ; GFX12-NEXT:    v_lshrrev_b32_e32 v1, 16, v1
@@ -4840,7 +4827,6 @@ define bfloat @local_atomic_fsub_ret_bf16__offset__align4(ptr addrspace(3) %ptr)
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX12-NEXT:    v_mov_b32_e32 v0, v1
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: local_atomic_fsub_ret_bf16__offset__align4:
@@ -5112,8 +5098,9 @@ define void @local_atomic_fsub_noret_bf16__offset__align4(ptr addrspace(3) %ptr)
 ; GFX12-NEXT:    v_bfe_u32 v3, v2, 16, 1
 ; GFX12-NEXT:    v_or_b32_e32 v4, 0x400000, v2
 ; GFX12-NEXT:    v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_add3_u32 v3, v3, v2, 0x7fff
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_cndmask_b32_e32 v2, v3, v4, vcc_lo
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_lshrrev_b32_e32 v2, 16, v2
@@ -5131,7 +5118,6 @@ define void @local_atomic_fsub_noret_bf16__offset__align4(ptr addrspace(3) %ptr)
 ; GFX12-NEXT:    s_cbranch_execnz .LBB19_1
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: local_atomic_fsub_noret_bf16__offset__align4:
@@ -5408,7 +5394,6 @@ define <2 x half> @local_atomic_fsub_ret_v2f16(ptr addrspace(3) %ptr, <2 x half>
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX12-NEXT:    v_mov_b32_e32 v0, v2
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: local_atomic_fsub_ret_v2f16:
@@ -5665,7 +5650,6 @@ define <2 x half> @local_atomic_fsub_ret_v2f16__offset(ptr addrspace(3) %ptr, <2
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX12-NEXT:    v_mov_b32_e32 v0, v2
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: local_atomic_fsub_ret_v2f16__offset:
@@ -5920,7 +5904,6 @@ define void @local_atomic_fsub_noret_v2f16(ptr addrspace(3) %ptr, <2 x half> %va
 ; GFX12-NEXT:    s_cbranch_execnz .LBB22_1
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: local_atomic_fsub_noret_v2f16:
@@ -6165,7 +6148,6 @@ define void @local_atomic_fsub_noret_v2f16__offset(ptr addrspace(3) %ptr, <2 x h
 ; GFX12-NEXT:    s_cbranch_execnz .LBB23_1
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: local_atomic_fsub_noret_v2f16__offset:
@@ -6413,6 +6395,7 @@ define <2 x bfloat> @local_atomic_fsub_ret_v2bf16(ptr addrspace(3) %ptr, <2 x bf
 ; GFX12-NEXT:    v_or_b32_e32 v9, 0x400000, v5
 ; GFX12-NEXT:    v_cmp_u_f32_e32 vcc_lo, v5, v5
 ; GFX12-NEXT:    v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_dual_cndmask_b32 v5, v7, v9 :: v_dual_lshlrev_b32 v2, 16, v4
 ; GFX12-NEXT:    v_sub_f32_e32 v2, v2, v3
@@ -6421,6 +6404,7 @@ define <2 x bfloat> @local_atomic_fsub_ret_v2bf16(ptr addrspace(3) %ptr, <2 x bf
 ; GFX12-NEXT:    v_or_b32_e32 v8, 0x400000, v2
 ; GFX12-NEXT:    v_cmp_u_f32_e64 s0, v2, v2
 ; GFX12-NEXT:    v_add3_u32 v6, v6, v2, 0x7fff
+; GFX12-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_cndmask_b32_e64 v2, v6, v8, s0
 ; GFX12-NEXT:    v_perm_b32 v2, v5, v2, 0x7060302
@@ -6437,7 +6421,6 @@ define <2 x bfloat> @local_atomic_fsub_ret_v2bf16(ptr addrspace(3) %ptr, <2 x bf
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s1
 ; GFX12-NEXT:    v_mov_b32_e32 v0, v2
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: local_atomic_fsub_ret_v2bf16:
@@ -6790,6 +6773,7 @@ define <2 x bfloat> @local_atomic_fsub_ret_v2bf16__offset(ptr addrspace(3) %ptr,
 ; GFX12-NEXT:    v_or_b32_e32 v9, 0x400000, v5
 ; GFX12-NEXT:    v_cmp_u_f32_e32 vcc_lo, v5, v5
 ; GFX12-NEXT:    v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_dual_cndmask_b32 v5, v7, v9 :: v_dual_lshlrev_b32 v2, 16, v4
 ; GFX12-NEXT:    v_sub_f32_e32 v2, v2, v3
@@ -6798,6 +6782,7 @@ define <2 x bfloat> @local_atomic_fsub_ret_v2bf16__offset(ptr addrspace(3) %ptr,
 ; GFX12-NEXT:    v_or_b32_e32 v8, 0x400000, v2
 ; GFX12-NEXT:    v_cmp_u_f32_e64 s0, v2, v2
 ; GFX12-NEXT:    v_add3_u32 v6, v6, v2, 0x7fff
+; GFX12-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_cndmask_b32_e64 v2, v6, v8, s0
 ; GFX12-NEXT:    v_perm_b32 v2, v5, v2, 0x7060302
@@ -6814,7 +6799,6 @@ define <2 x bfloat> @local_atomic_fsub_ret_v2bf16__offset(ptr addrspace(3) %ptr,
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s1
 ; GFX12-NEXT:    v_mov_b32_e32 v0, v2
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: local_atomic_fsub_ret_v2bf16__offset:
@@ -7172,8 +7156,10 @@ define void @local_atomic_fsub_noret_v2bf16(ptr addrspace(3) %ptr, <2 x bfloat>
 ; GFX12-NEXT:    v_add3_u32 v7, v7, v5, 0x7fff
 ; GFX12-NEXT:    v_add3_u32 v6, v6, v4, 0x7fff
 ; GFX12-NEXT:    v_cmp_u_f32_e64 s0, v4, v4
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-NEXT:    s_wait_alu 0xfffd
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX12-NEXT:    v_cndmask_b32_e32 v5, v7, v9, vcc_lo
+; GFX12-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-NEXT:    v_cndmask_b32_e64 v4, v6, v8, s0
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-NEXT:    v_perm_b32 v4, v5, v4, 0x7060302
@@ -7190,7 +7176,6 @@ define void @local_atomic_fsub_noret_v2bf16(ptr addrspace(3) %ptr, <2 x bfloat>
 ; GFX12-NEXT:    s_cbranch_execnz .LBB26_1
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s1
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: local_atomic_fsub_noret_v2bf16:
@@ -7536,8 +7521,10 @@ define void @local_atomic_fsub_noret_v2bf16__ofset(ptr addrspace(3) %ptr, <2 x b
 ; GFX12-NEXT:    v_add3_u32 v7, v7, v5, 0x7fff
 ; GFX12-NEXT:    v_add3_u32 v6, v6, v4, 0x7fff
 ; GFX12-NEXT:    v_cmp_u_f32_e64 s0, v4, v4
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-NEXT:    s_wait_alu 0xfffd
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX12-NEXT:    v_cndmask_b32_e32 v5, v7, v9, vcc_lo
+; GFX12-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-NEXT:    v_cndmask_b32_e64 v4, v6, v8, s0
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-NEXT:    v_perm_b32 v4, v5, v4, 0x7060302
@@ -7554,7 +7541,6 @@ define void @local_atomic_fsub_noret_v2bf16__ofset(ptr addrspace(3) %ptr, <2 x b
 ; GFX12-NEXT:    s_cbranch_execnz .LBB27_1
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s1
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: local_atomic_fsub_noret_v2bf16__ofset:
@@ -7907,7 +7893,6 @@ define float @local_atomic_fsub_ret_f32__amdgpu_ignore_denormal_mode(ptr addrspa
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX12-NEXT:    v_mov_b32_e32 v0, v1
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: local_atomic_fsub_ret_f32__amdgpu_ignore_denormal_mode:
@@ -8117,7 +8102,6 @@ define void @local_atomic_fsub_noret_f32__amdgpu_ignore_denormal_mode(ptr addrsp
 ; GFX12-NEXT:    s_cbranch_execnz .LBB29_1
 ; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: local_atomic_fsub_noret_f32__amdgpu_ignore_denormal_mode:
diff --git a/llvm/test/CodeGen/AMDGPU/lower-buffer-fat-pointers-lastuse-metadata.ll b/llvm/test/CodeGen/AMDGPU/lower-buffer-fat-pointers-lastuse-metadata.ll
index eaf8809d33fc3..2b10d469acf5c 100644
--- a/llvm/test/CodeGen/AMDGPU/lower-buffer-fat-pointers-lastuse-metadata.ll
+++ b/llvm/test/CodeGen/AMDGPU/lower-buffer-fat-pointers-lastuse-metadata.ll
@@ -6,77 +6,37 @@
 define amdgpu_kernel void @buffer_last_use_load_0(ptr addrspace(7) %in, ptr addrspace(7) %out) {
 ; GFX12-LABEL: buffer_last_use_load_0:
 ; GFX12:       ; %bb.0: ; %entry
-; GFX12-NEXT:    s_clause 0x2
+; GFX12-NEXT:    s_clause 0x1
 ; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x0
-; GFX12-NEXT:    s_load_b128 s[8:11], s[4:5], 0x20
-; GFX12-NEXT:    s_load_b32 s6, s[4:5], 0x10
+; GFX12-NEXT:    s_load_b32 s13, s[4:5], 0x10
+; GFX12-NEXT:    s_mov_b32 s12, 0
+; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-NEXT:    s_mov_b32 s7, s12
+; GFX12-NEXT:    s_mov_b32 s9, s12
 ; GFX12-NEXT:    s_wait_kmcnt 0x0
-; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
-; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
-; GFX12-NEXT:    v_dual_mov_b32 v7, s8 :: v_dual_mov_b32 v8, s9
-; GFX12-NEXT:    v_dual_mov_b32 v9, s10 :: v_dual_mov_b32 v10, s11
-; GFX12-NEXT:    scratch_store_b128 off, v[0:3], off offset:32
-; GFX12-NEXT:    s_clause 0x1
-; GFX12-NEXT:    scratch_load_b64 v[5:6], off, off offset:40
-; GFX12-NEXT:    scratch_load_b32 v4, off, off offset:36
-; GFX12-NEXT:    s_load_b32 s1, s[4:5], 0x30
-; GFX12-NEXT:    scratch_store_b128 off, v[7:10], off
+; GFX12-NEXT:    s_mov_b32 s6, s3
+; GFX12-NEXT:    v_mov_b32_e32 v0, s0
+; GFX12-NEXT:    s_mov_b32 s8, s1
+; GFX12-NEXT:    s_or_b64 s[10:11], s[6:7], s[12:13]
+; GFX12-NEXT:    s_mov_b32 s13, s2
+; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-NEXT:    s_or_b64 s[8:9], s[8:9], s[12:13]
+; GFX12-NEXT:    buffer_load_b32 v0, v0, s[8:11], null offen th:TH_LOAD_LU
 ; GFX12-NEXT:    s_clause 0x1
-; GFX12-NEXT:    scratch_load_b64 v[1:2], off, off offset:8
-; GFX12-NEXT:    scratch_load_b32 v0, off, off offset:4
-; GFX12-NEXT:    v_mov_b32_e32 v7, s6
-; GFX12-NEXT:    v_mov_b32_e32 v9, s0
+; GFX12-NEXT:    s_load_b32 s13, s[4:5], 0x30
+; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x20
+; GFX12-NEXT:    s_mov_b32 s5, s12
 ; GFX12-NEXT:    s_wait_kmcnt 0x0
-; GFX12-NEXT:    v_mov_b32_e32 v3, s1
-; GFX12-NEXT:    s_mov_b32 s1, exec_lo
-; GFX12-NEXT:  .LBB0_1: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT:    s_wait_loadcnt 0x2
-; GFX12-NEXT:    v_readfirstlane_b32 s4, v4
-; GFX12-NEXT:    v_readfirstlane_b32 s5, v5
-; GFX12-NEXT:    v_readfirstlane_b32 s6, v6
-; GFX12-NEXT:    v_readfirstlane_b32 s7, v7
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-NEXT:    v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[4:5]
-; GFX12-NEXT:    v_cmp_eq_u64_e64 s0, s[6:7], v[6:7]
-; GFX12-NEXT:    s_wait_alu 0xfffe
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT:    s_and_b32 s0, vcc_lo, s0
-; GFX12-NEXT:    s_wait_alu 0xfffe
-; GFX12-NEXT:    s_and_saveexec_b32 s0, s0
-; GFX12-NEXT:    s_wait_loadcnt 0x0
-; GFX12-NEXT:    buffer_load_b32 v8, v9, s[4:7], null offen th:TH_LOAD_LU
-; GFX12-NEXT:    ; implicit-def: $vgpr4_vgpr5_vgpr6_vgpr7
-; GFX12-NEXT:    ; implicit-def: $vgpr9
-; GFX12-NEXT:    s_wait_alu 0xfffe
-; GFX12-NEXT:    s_xor_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT:    s_cbranch_execnz .LBB0_1
-; GFX12-NEXT:  ; %bb.2:
-; GFX12-NEXT:    s_mov_b32 exec_lo, s1
-; GFX12-NEXT:    v_mov_b32_e32 v4, s8
-; GFX12-NEXT:    s_mov_b32 s0, exec_lo
-; GFX12-NEXT:  .LBB0_3: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT:    s_wait_loadcnt 0x1
-; GFX12-NEXT:    v_readfirstlane_b32 s4, v0
-; GFX12-NEXT:    v_readfirstlane_b32 s5, v1
-; GFX12-NEXT:    v_readfirstlane_b32 s6, v2
-; GFX12-NEXT:    v_readfirstlane_b32 s7, v3
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-NEXT:    v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
-; GFX12-NEXT:    v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
-; GFX12-NEXT:    s_wait_alu 0xfffe
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT:    s_and_b32 s0, vcc_lo, s0
-; GFX12-NEXT:    s_wait_alu 0xfffe
-; GFX12-NEXT:    s_and_saveexec_b32 s0, s0
+; GFX12-NEXT:    v_mov_b32_e32 v1, s0
+; GFX12-NEXT:    s_mov_b32 s4, s3
+; GFX12-NEXT:    s_mov_b32 s3, s12
+; GFX12-NEXT:    s_or_b64 s[6:7], s[4:5], s[12:13]
+; GFX12-NEXT:    s_mov_b32 s13, s2
+; GFX12-NEXT:    s_mov_b32 s2, s1
+; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-NEXT:    s_or_b64 s[4:5], s[2:3], s[12:13]
 ; GFX12-NEXT:    s_wait_loadcnt 0x0
-; GFX12-NEXT:    buffer_store_b32 v8, v4, s[4:7], null offen
-; GFX12-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
-; GFX12-NEXT:    ; implicit-def: $vgpr8
-; GFX12-NEXT:    ; implicit-def: $vgpr4
-; GFX12-NEXT:    s_wait_alu 0xfffe
-; GFX12-NEXT:    s_xor_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT:    s_cbranch_execnz .LBB0_3
-; GFX12-NEXT:  ; %bb.4:
+; GFX12-NEXT:    buffer_store_b32 v0, v1, s[4:7], null offen
 ; GFX12-NEXT:    s_endpgm
 entry:
   %val = load i32, ptr addrspace(7) %in, !amdgpu.last.use !{}
@@ -87,78 +47,38 @@ entry:
 define amdgpu_kernel void @buffer_last_use_load_1(ptr addrspace(7) %in, ptr addrspace(7) %out) {
 ; GFX12-LABEL: buffer_last_use_load_1:
 ; GFX12:       ; %bb.0: ; %entry
-; GFX12-NEXT:    s_clause 0x2
+; GFX12-NEXT:    s_clause 0x1
 ; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x0
-; GFX12-NEXT:    s_load_b128 s[8:11], s[4:5], 0x20
-; GFX12-NEXT:    s_load_b32 s6, s[4:5], 0x10
+; GFX12-NEXT:    s_load_b32 s13, s[4:5], 0x10
 ; GFX12-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
+; GFX12-NEXT:    s_mov_b32 s12, 0
+; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-NEXT:    s_mov_b32 s7, s12
+; GFX12-NEXT:    s_mov_b32 s9, s12
 ; GFX12-NEXT:    s_wait_kmcnt 0x0
-; GFX12-NEXT:    v_dual_mov_b32 v4, s3 :: v_dual_mov_b32 v3, s2
-; GFX12-NEXT:    v_dual_mov_b32 v2, s1 :: v_dual_mov_b32 v1, s0
-; GFX12-NEXT:    v_dual_mov_b32 v8, s8 :: v_dual_mov_b32 v9, s9
-; GFX12-NEXT:    v_dual_mov_b32 v10, s10 :: v_dual_mov_b32 v11, s11
-; GFX12-NEXT:    scratch_store_b128 off, v[1:4], off offset:32
-; GFX12-NEXT:    s_clause 0x1
-; GFX12-NEXT:    scratch_load_b64 v[6:7], off, off offset:40
-; GFX12-NEXT:    scratch_load_b32 v5, off, off offset:36
-; GFX12-NEXT:    s_load_b32 s1, s[4:5], 0x30
-; GFX12-NEXT:    scratch_store_b128 off, v[8:11], off
+; GFX12-NEXT:    s_mov_b32 s6, s3
+; GFX12-NEXT:    v_lshl_add_u32 v0, v0, 2, s0
+; GFX12-NEXT:    s_mov_b32 s8, s1
+; GFX12-NEXT:    s_or_b64 s[10:11], s[6:7], s[12:13]
+; GFX12-NEXT:    s_mov_b32 s13, s2
+; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-NEXT:    s_or_b64 s[8:9], s[8:9], s[12:13]
+; GFX12-NEXT:    buffer_load_b32 v0, v0, s[8:11], null offen th:TH_LOAD_LU
 ; GFX12-NEXT:    s_clause 0x1
-; GFX12-NEXT:    scratch_load_b64 v[2:3], off, off offset:8
-; GFX12-NEXT:    scratch_load_b32 v1, off, off offset:4
-; GFX12-NEXT:    v_mov_b32_e32 v8, s6
-; GFX12-NEXT:    v_lshl_add_u32 v9, v0, 2, s0
+; GFX12-NEXT:    s_load_b32 s13, s[4:5], 0x30
+; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x20
+; GFX12-NEXT:    s_mov_b32 s5, s12
 ; GFX12-NEXT:    s_wait_kmcnt 0x0
-; GFX12-NEXT:    v_mov_b32_e32 v4, s1
-; GFX12-NEXT:    s_mov_b32 s1, exec_lo
-; GFX12-NEXT:  .LBB1_1: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT:    s_wait_loadcnt 0x2
-; GFX12-NEXT:    v_readfirstlane_b32 s4, v5
-; GFX12-NEXT:    v_readfirstlane_b32 s5, v6
-; GFX12-NEXT:    v_readfirstlane_b32 s6, v7
-; GFX12-NEXT:    v_readfirstlane_b32 s7, v8
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-NEXT:    v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[5:6]
-; GFX12-NEXT:    v_cmp_eq_u64_e64 s0, s[6:7], v[7:8]
-; GFX12-NEXT:    s_wait_alu 0xfffe
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT:    s_and_b32 s0, vcc_lo, s0
-; GFX12-NEXT:    s_wait_alu 0xfffe
-; GFX12-NEXT:    s_and_saveexec_b32 s0, s0
-; GFX12-NEXT:    s_wait_loadcnt 0x0
-; GFX12-NEXT:    buffer_load_b32 v0, v9, s[4:7], null offen th:TH_LOAD_LU
-; GFX12-NEXT:    ; implicit-def: $vgpr5_vgpr6_vgpr7_vgpr8
-; GFX12-NEXT:    ; implicit-def: $vgpr9
-; GFX12-NEXT:    s_wait_alu 0xfffe
-; GFX12-NEXT:    s_xor_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT:    s_cbranch_execnz .LBB1_1
-; GFX12-NEXT:  ; %bb.2:
-; GFX12-NEXT:    s_mov_b32 exec_lo, s1
-; GFX12-NEXT:    v_mov_b32_e32 v5, s8
-; GFX12-NEXT:    s_mov_b32 s0, exec_lo
-; GFX12-NEXT:  .LBB1_3: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT:    s_wait_loadcnt 0x1
-; GFX12-NEXT:    v_readfirstlane_b32 s4, v1
-; GFX12-NEXT:    v_readfirstlane_b32 s5, v2
-; GFX12-NEXT:    v_readfirstlane_b32 s6, v3
-; GFX12-NEXT:    v_readfirstlane_b32 s7, v4
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-NEXT:    v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[1:2]
-; GFX12-NEXT:    v_cmp_eq_u64_e64 s0, s[6:7], v[3:4]
-; GFX12-NEXT:    s_wait_alu 0xfffe
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT:    s_and_b32 s0, vcc_lo, s0
-; GFX12-NEXT:    s_wait_alu 0xfffe
-; GFX12-NEXT:    s_and_saveexec_b32 s0, s0
+; GFX12-NEXT:    v_mov_b32_e32 v1, s0
+; GFX12-NEXT:    s_mov_b32 s4, s3
+; GFX12-NEXT:    s_mov_b32 s3, s12
+; GFX12-NEXT:    s_or_b64 s[6:7], s[4:5], s[12:13]
+; GFX12-NEXT:    s_mov_b32 s13, s2
+; GFX12-NEXT:    s_mov_b32 s2, s1
+; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-NEXT:    s_or_b64 s[4:5], s[2:3], s[12:13]
 ; GFX12-NEXT:    s_wait_loadcnt 0x0
-; GFX12-NEXT:    buffer_store_b32 v0, v5, s[4:7], null offen
-; GFX12-NEXT:    ; implicit-def: $vgpr1_vgpr2_vgpr3_vgpr4
-; GFX12-NEXT:    ; implicit-def: $vgpr0
-; GFX12-NEXT:    ; implicit-def: $vgpr5
-; GFX12-NEXT:    s_wait_alu 0xfffe
-; GFX12-NEXT:    s_xor_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT:    s_cbranch_execnz .LBB1_3
-; GFX12-NEXT:  ; %bb.4:
+; GFX12-NEXT:    buffer_store_b32 v0, v1, s[4:7], null offen
 ; GFX12-NEXT:    s_endpgm
 entry:
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
@@ -171,77 +91,37 @@ entry:
 define amdgpu_kernel void @buffer_last_use_and_volatile_load(ptr addrspace(7) %in, ptr addrspace(7) %out) {
 ; GFX12-LABEL: buffer_last_use_and_volatile_load:
 ; GFX12:       ; %bb.0: ; %entry
-; GFX12-NEXT:    s_clause 0x2
+; GFX12-NEXT:    s_clause 0x1
 ; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x0
-; GFX12-NEXT:    s_load_b128 s[8:11], s[4:5], 0x20
-; GFX12-NEXT:    s_load_b32 s6, s[4:5], 0x10
+; GFX12-NEXT:    s_load_b32 s13, s[4:5], 0x10
+; GFX12-NEXT:    s_mov_b32 s12, 0
+; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-NEXT:    s_mov_b32 s7, s12
+; GFX12-NEXT:    s_mov_b32 s9, s12
 ; GFX12-NEXT:    s_wait_kmcnt 0x0
-; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
-; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
-; GFX12-NEXT:    v_dual_mov_b32 v7, s8 :: v_dual_mov_b32 v8, s9
-; GFX12-NEXT:    v_dual_mov_b32 v9, s10 :: v_dual_mov_b32 v10, s11
-; GFX12-NEXT:    scratch_store_b128 off, v[0:3], off offset:32
-; GFX12-NEXT:    s_clause 0x1
-; GFX12-NEXT:    scratch_load_b64 v[5:6], off, off offset:40
-; GFX12-NEXT:    scratch_load_b32 v4, off, off offset:36
-; GFX12-NEXT:    s_load_b32 s1, s[4:5], 0x30
-; GFX12-NEXT:    scratch_store_b128 off, v[7:10], off
+; GFX12-NEXT:    s_mov_b32 s6, s3
+; GFX12-NEXT:    v_mov_b32_e32 v0, s0
+; GFX12-NEXT:    s_mov_b32 s8, s1
+; GFX12-NEXT:    s_or_b64 s[10:11], s[6:7], s[12:13]
+; GFX12-NEXT:    s_mov_b32 s13, s2
+; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-NEXT:    s_or_b64 s[8:9], s[8:9], s[12:13]
+; GFX12-NEXT:    buffer_load_b32 v0, v0, s[8:11], null offen th:TH_LOAD_BYPASS scope:SCOPE_SYS
 ; GFX12-NEXT:    s_clause 0x1
-; GFX12-NEXT:    scratch_load_b64 v[1:2], off, off offset:8
-; GFX12-NEXT:    scratch_load_b32 v0, off, off offset:4
-; GFX12-NEXT:    v_mov_b32_e32 v7, s6
-; GFX12-NEXT:    v_mov_b32_e32 v9, s0
+; GFX12-NEXT:    s_load_b32 s13, s[4:5], 0x30
+; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x20
+; GFX12-NEXT:    s_mov_b32 s5, s12
 ; GFX12-NEXT:    s_wait_kmcnt 0x0
-; GFX12-NEXT:    v_mov_b32_e32 v3, s1
-; GFX12-NEXT:    s_mov_b32 s1, exec_lo
-; GFX12-NEXT:  .LBB2_1: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT:    s_wait_loadcnt 0x2
-; GFX12-NEXT:    v_readfirstlane_b32 s4, v4
-; GFX12-NEXT:    v_readfirstlane_b32 s5, v5
-; GFX12-NEXT:    v_readfirstlane_b32 s6, v6
-; GFX12-NEXT:    v_readfirstlane_b32 s7, v7
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-NEXT:    v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[4:5]
-; GFX12-NEXT:    v_cmp_eq_u64_e64 s0, s[6:7], v[6:7]
-; GFX12-NEXT:    s_wait_alu 0xfffe
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT:    s_and_b32 s0, vcc_lo, s0
-; GFX12-NEXT:    s_wait_alu 0xfffe
-; GFX12-NEXT:    s_and_saveexec_b32 s0, s0
-; GFX12-NEXT:    s_wait_loadcnt 0x0
-; GFX12-NEXT:    buffer_load_b32 v8, v9, s[4:7], null offen th:TH_LOAD_BYPASS scope:SCOPE_SYS
-; GFX12-NEXT:    ; implicit-def: $vgpr4_vgpr5_vgpr6_vgpr7
-; GFX12-NEXT:    ; implicit-def: $vgpr9
-; GFX12-NEXT:    s_wait_alu 0xfffe
-; GFX12-NEXT:    s_xor_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT:    s_cbranch_execnz .LBB2_1
-; GFX12-NEXT:  ; %bb.2:
-; GFX12-NEXT:    s_mov_b32 exec_lo, s1
-; GFX12-NEXT:    v_mov_b32_e32 v4, s8
-; GFX12-NEXT:    s_mov_b32 s0, exec_lo
-; GFX12-NEXT:  .LBB2_3: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT:    s_wait_loadcnt 0x1
-; GFX12-NEXT:    v_readfirstlane_b32 s4, v0
-; GFX12-NEXT:    v_readfirstlane_b32 s5, v1
-; GFX12-NEXT:    v_readfirstlane_b32 s6, v2
-; GFX12-NEXT:    v_readfirstlane_b32 s7, v3
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-NEXT:    v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
-; GFX12-NEXT:    v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
-; GFX12-NEXT:    s_wait_alu 0xfffe
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT:    s_and_b32 s0, vcc_lo, s0
-; GFX12-NEXT:    s_wait_alu 0xfffe
-; GFX12-NEXT:    s_and_saveexec_b32 s0, s0
+; GFX12-NEXT:    v_mov_b32_e32 v1, s0
+; GFX12-NEXT:    s_mov_b32 s4, s3
+; GFX12-NEXT:    s_mov_b32 s3, s12
+; GFX12-NEXT:    s_or_b64 s[6:7], s[4:5], s[12:13]
+; GFX12-NEXT:    s_mov_b32 s13, s2
+; GFX12-NEXT:    s_mov_b32 s2, s1
+; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-NEXT:    s_or_b64 s[4:5], s[2:3], s[12:13]
 ; GFX12-NEXT:    s_wait_loadcnt 0x0
-; GFX12-NEXT:    buffer_store_b32 v8, v4, s[4:7], null offen
-; GFX12-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
-; GFX12-NEXT:    ; implicit-def: $vgpr8
-; GFX12-NEXT:    ; implicit-def: $vgpr4
-; GFX12-NEXT:    s_wait_alu 0xfffe
-; GFX12-NEXT:    s_xor_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT:    s_cbranch_execnz .LBB2_3
-; GFX12-NEXT:  ; %bb.4:
+; GFX12-NEXT:    buffer_store_b32 v0, v1, s[4:7], null offen
 ; GFX12-NEXT:    s_endpgm
 entry:
   %val = load volatile i32, ptr addrspace(7) %in, !amdgpu.last.use !{}
@@ -252,77 +132,37 @@ entry:
 define amdgpu_kernel void @buffer_last_use_and_nontemporal_load(ptr addrspace(7) %in, ptr addrspace(7) %out) {
 ; GFX12-LABEL: buffer_last_use_and_nontemporal_load:
 ; GFX12:       ; %bb.0: ; %entry
-; GFX12-NEXT:    s_clause 0x2
+; GFX12-NEXT:    s_clause 0x1
 ; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x0
-; GFX12-NEXT:    s_load_b128 s[8:11], s[4:5], 0x20
-; GFX12-NEXT:    s_load_b32 s6, s[4:5], 0x10
+; GFX12-NEXT:    s_load_b32 s13, s[4:5], 0x10
+; GFX12-NEXT:    s_mov_b32 s12, 0
+; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-NEXT:    s_mov_b32 s7, s12
+; GFX12-NEXT:    s_mov_b32 s9, s12
 ; GFX12-NEXT:    s_wait_kmcnt 0x0
-; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
-; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
-; GFX12-NEXT:    v_dual_mov_b32 v7, s8 :: v_dual_mov_b32 v8, s9
-; GFX12-NEXT:    v_dual_mov_b32 v9, s10 :: v_dual_mov_b32 v10, s11
-; GFX12-NEXT:    scratch_store_b128 off, v[0:3], off offset:32
-; GFX12-NEXT:    s_clause 0x1
-; GFX12-NEXT:    scratch_load_b64 v[5:6], off, off offset:40
-; GFX12-NEXT:    scratch_load_b32 v4, off, off offset:36
-; GFX12-NEXT:    s_load_b32 s1, s[4:5], 0x30
-; GFX12-NEXT:    scratch_store_b128 off, v[7:10], off
+; GFX12-NEXT:    s_mov_b32 s6, s3
+; GFX12-NEXT:    v_mov_b32_e32 v0, s0
+; GFX12-NEXT:    s_mov_b32 s8, s1
+; GFX12-NEXT:    s_or_b64 s[10:11], s[6:7], s[12:13]
+; GFX12-NEXT:    s_mov_b32 s13, s2
+; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-NEXT:    s_or_b64 s[8:9], s[8:9], s[12:13]
+; GFX12-NEXT:    buffer_load_b32 v0, v0, s[8:11], null offen th:TH_LOAD_LU
 ; GFX12-NEXT:    s_clause 0x1
-; GFX12-NEXT:    scratch_load_b64 v[1:2], off, off offset:8
-; GFX12-NEXT:    scratch_load_b32 v0, off, off offset:4
-; GFX12-NEXT:    v_mov_b32_e32 v7, s6
-; GFX12-NEXT:    v_mov_b32_e32 v9, s0
+; GFX12-NEXT:    s_load_b32 s13, s[4:5], 0x30
+; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x20
+; GFX12-NEXT:    s_mov_b32 s5, s12
 ; GFX12-NEXT:    s_wait_kmcnt 0x0
-; GFX12-NEXT:    v_mov_b32_e32 v3, s1
-; GFX12-NEXT:    s_mov_b32 s1, exec_lo
-; GFX12-NEXT:  .LBB3_1: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT:    s_wait_loadcnt 0x2
-; GFX12-NEXT:    v_readfirstlane_b32 s4, v4
-; GFX12-NEXT:    v_readfirstlane_b32 s5, v5
-; GFX12-NEXT:    v_readfirstlane_b32 s6, v6
-; GFX12-NEXT:    v_readfirstlane_b32 s7, v7
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-NEXT:    v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[4:5]
-; GFX12-NEXT:    v_cmp_eq_u64_e64 s0, s[6:7], v[6:7]
-; GFX12-NEXT:    s_wait_alu 0xfffe
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT:    s_and_b32 s0, vcc_lo, s0
-; GFX12-NEXT:    s_wait_alu 0xfffe
-; GFX12-NEXT:    s_and_saveexec_b32 s0, s0
-; GFX12-NEXT:    s_wait_loadcnt 0x0
-; GFX12-NEXT:    buffer_load_b32 v8, v9, s[4:7], null offen th:TH_LOAD_LU
-; GFX12-NEXT:    ; implicit-def: $vgpr4_vgpr5_vgpr6_vgpr7
-; GFX12-NEXT:    ; implicit-def: $vgpr9
-; GFX12-NEXT:    s_wait_alu 0xfffe
-; GFX12-NEXT:    s_xor_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT:    s_cbranch_execnz .LBB3_1
-; GFX12-NEXT:  ; %bb.2:
-; GFX12-NEXT:    s_mov_b32 exec_lo, s1
-; GFX12-NEXT:    v_mov_b32_e32 v4, s8
-; GFX12-NEXT:    s_mov_b32 s0, exec_lo
-; GFX12-NEXT:  .LBB3_3: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT:    s_wait_loadcnt 0x1
-; GFX12-NEXT:    v_readfirstlane_b32 s4, v0
-; GFX12-NEXT:    v_readfirstlane_b32 s5, v1
-; GFX12-NEXT:    v_readfirstlane_b32 s6, v2
-; GFX12-NEXT:    v_readfirstlane_b32 s7, v3
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-NEXT:    v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
-; GFX12-NEXT:    v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
-; GFX12-NEXT:    s_wait_alu 0xfffe
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT:    s_and_b32 s0, vcc_lo, s0
-; GFX12-NEXT:    s_wait_alu 0xfffe
-; GFX12-NEXT:    s_and_saveexec_b32 s0, s0
+; GFX12-NEXT:    v_mov_b32_e32 v1, s0
+; GFX12-NEXT:    s_mov_b32 s4, s3
+; GFX12-NEXT:    s_mov_b32 s3, s12
+; GFX12-NEXT:    s_or_b64 s[6:7], s[4:5], s[12:13]
+; GFX12-NEXT:    s_mov_b32 s13, s2
+; GFX12-NEXT:    s_mov_b32 s2, s1
+; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-NEXT:    s_or_b64 s[4:5], s[2:3], s[12:13]
 ; GFX12-NEXT:    s_wait_loadcnt 0x0
-; GFX12-NEXT:    buffer_store_b32 v8, v4, s[4:7], null offen
-; GFX12-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
-; GFX12-NEXT:    ; implicit-def: $vgpr8
-; GFX12-NEXT:    ; implicit-def: $vgpr4
-; GFX12-NEXT:    s_wait_alu 0xfffe
-; GFX12-NEXT:    s_xor_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT:    s_cbranch_execnz .LBB3_3
-; GFX12-NEXT:  ; %bb.4:
+; GFX12-NEXT:    buffer_store_b32 v0, v1, s[4:7], null offen
 ; GFX12-NEXT:    s_endpgm
 entry:
   %val = load i32, ptr addrspace(7) %in, !amdgpu.last.use !{}, !nontemporal !0
diff --git a/llvm/test/CodeGen/AMDGPU/lower-buffer-fat-pointers-nontemporal-metadata.ll b/llvm/test/CodeGen/AMDGPU/lower-buffer-fat-pointers-nontemporal-metadata.ll
index 6bd0498a2a4e4..a62910e4e5711 100644
--- a/llvm/test/CodeGen/AMDGPU/lower-buffer-fat-pointers-nontemporal-metadata.ll
+++ b/llvm/test/CodeGen/AMDGPU/lower-buffer-fat-pointers-nontemporal-metadata.ll
@@ -13,30 +13,32 @@
 define amdgpu_kernel void @buffer_nontemporal_load_store(ptr addrspace(7) %in, ptr addrspace(7) %out) {
 ; GFX9-SDAG-LABEL: buffer_nontemporal_load_store:
 ; GFX9-SDAG:       ; %bb.0: ; %entry
-; GFX9-SDAG-NEXT:    s_load_dwordx4 s[4:7], s[8:9], 0x0
+; GFX9-SDAG-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
 ; GFX9-SDAG-NEXT:    s_load_dword s11, s[8:9], 0x10
 ; GFX9-SDAG-NEXT:    s_mov_b32 s10, 0
-; GFX9-SDAG-NEXT:    s_add_u32 s0, s0, s15
-; GFX9-SDAG-NEXT:    s_mov_b32 s15, s10
+; GFX9-SDAG-NEXT:    s_mov_b32 s5, s10
 ; GFX9-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-SDAG-NEXT:    s_mov_b32 s14, s7
-; GFX9-SDAG-NEXT:    s_addc_u32 s1, s1, 0
-; GFX9-SDAG-NEXT:    s_mov_b32 s12, s5
-; GFX9-SDAG-NEXT:    s_or_b64 s[14:15], s[14:15], s[10:11]
-; GFX9-SDAG-NEXT:    s_mov_b32 s13, s6
-; GFX9-SDAG-NEXT:    v_mov_b32_e32 v0, s4
-; GFX9-SDAG-NEXT:    buffer_load_dword v0, v0, s[12:15], 0 offen glc slc
+; GFX9-SDAG-NEXT:    s_mov_b32 s4, s3
+; GFX9-SDAG-NEXT:    s_or_b64 s[6:7], s[4:5], s[10:11]
+; GFX9-SDAG-NEXT:    s_mov_b32 s11, s2
+; GFX9-SDAG-NEXT:    s_mov_b32 s2, s1
+; GFX9-SDAG-NEXT:    s_mov_b32 s3, s10
+; GFX9-SDAG-NEXT:    s_or_b64 s[4:5], s[2:3], s[10:11]
+; GFX9-SDAG-NEXT:    v_mov_b32_e32 v0, s0
+; GFX9-SDAG-NEXT:    buffer_load_dword v0, v0, s[4:7], 0 offen glc slc
 ; GFX9-SDAG-NEXT:    s_load_dword s11, s[8:9], 0x30
-; GFX9-SDAG-NEXT:    s_load_dwordx4 s[4:7], s[8:9], 0x20
-; GFX9-SDAG-NEXT:    s_mov_b32 s9, s10
+; GFX9-SDAG-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x20
+; GFX9-SDAG-NEXT:    s_mov_b32 s5, s10
 ; GFX9-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-SDAG-NEXT:    s_mov_b32 s8, s7
-; GFX9-SDAG-NEXT:    s_or_b64 s[10:11], s[8:9], s[10:11]
-; GFX9-SDAG-NEXT:    s_mov_b32 s8, s5
-; GFX9-SDAG-NEXT:    s_mov_b32 s9, s6
-; GFX9-SDAG-NEXT:    v_mov_b32_e32 v1, s4
+; GFX9-SDAG-NEXT:    s_mov_b32 s4, s3
+; GFX9-SDAG-NEXT:    s_or_b64 s[6:7], s[4:5], s[10:11]
+; GFX9-SDAG-NEXT:    s_mov_b32 s11, s2
+; GFX9-SDAG-NEXT:    s_mov_b32 s2, s1
+; GFX9-SDAG-NEXT:    s_mov_b32 s3, s10
+; GFX9-SDAG-NEXT:    s_or_b64 s[4:5], s[2:3], s[10:11]
+; GFX9-SDAG-NEXT:    v_mov_b32_e32 v1, s0
 ; GFX9-SDAG-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-SDAG-NEXT:    buffer_store_dword v0, v1, s[8:11], 0 offen glc slc
+; GFX9-SDAG-NEXT:    buffer_store_dword v0, v1, s[4:7], 0 offen glc slc
 ; GFX9-SDAG-NEXT:    s_endpgm
 ;
 ; GFX9-GISEL-LABEL: buffer_nontemporal_load_store:
@@ -72,68 +74,31 @@ define amdgpu_kernel void @buffer_nontemporal_load_store(ptr addrspace(7) %in, p
 ; GFX940-SDAG-LABEL: buffer_nontemporal_load_store:
 ; GFX940-SDAG:       ; %bb.0: ; %entry
 ; GFX940-SDAG-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
-; GFX940-SDAG-NEXT:    s_load_dword s6, s[4:5], 0x10
-; GFX940-SDAG-NEXT:    s_load_dwordx4 s[8:11], s[4:5], 0x20
-; GFX940-SDAG-NEXT:    s_load_dword s7, s[4:5], 0x30
+; GFX940-SDAG-NEXT:    s_load_dword s13, s[4:5], 0x10
+; GFX940-SDAG-NEXT:    s_mov_b32 s12, 0
+; GFX940-SDAG-NEXT:    s_mov_b32 s7, s12
 ; GFX940-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX940-SDAG-NEXT:    v_mov_b64_e32 v[0:1], s[0:1]
-; GFX940-SDAG-NEXT:    v_mov_b64_e32 v[2:3], s[2:3]
-; GFX940-SDAG-NEXT:    scratch_store_dwordx4 off, v[0:3], off offset:32 sc0 sc1
-; GFX940-SDAG-NEXT:    scratch_load_dwordx2 v[10:11], off, off offset:40
-; GFX940-SDAG-NEXT:    scratch_load_dword v4, off, off offset:36
-; GFX940-SDAG-NEXT:    v_mov_b64_e32 v[0:1], s[8:9]
-; GFX940-SDAG-NEXT:    v_mov_b64_e32 v[2:3], s[10:11]
-; GFX940-SDAG-NEXT:    scratch_store_dwordx4 off, v[0:3], off sc0 sc1
-; GFX940-SDAG-NEXT:    scratch_load_dwordx2 v[12:13], off, off offset:8
-; GFX940-SDAG-NEXT:    s_nop 0
-; GFX940-SDAG-NEXT:    scratch_load_dword v0, off, off offset:4
-; GFX940-SDAG-NEXT:    v_mov_b32_e32 v7, s6
-; GFX940-SDAG-NEXT:    v_mov_b32_e32 v3, s7
-; GFX940-SDAG-NEXT:    v_mov_b32_e32 v9, s0
-; GFX940-SDAG-NEXT:    s_mov_b64 s[2:3], exec
-; GFX940-SDAG-NEXT:    s_waitcnt vmcnt(4)
-; GFX940-SDAG-NEXT:    v_mov_b32_e32 v6, v11
-; GFX940-SDAG-NEXT:    v_mov_b32_e32 v5, v10
-; GFX940-SDAG-NEXT:    s_waitcnt vmcnt(1)
-; GFX940-SDAG-NEXT:    v_mov_b32_e32 v2, v13
-; GFX940-SDAG-NEXT:    v_mov_b32_e32 v1, v12
-; GFX940-SDAG-NEXT:  .LBB0_1: ; =>This Inner Loop Header: Depth=1
-; GFX940-SDAG-NEXT:    v_readfirstlane_b32 s4, v4
-; GFX940-SDAG-NEXT:    v_readfirstlane_b32 s5, v5
-; GFX940-SDAG-NEXT:    v_readfirstlane_b32 s6, v6
-; GFX940-SDAG-NEXT:    v_readfirstlane_b32 s7, v7
-; GFX940-SDAG-NEXT:    v_cmp_eq_u64_e32 vcc, s[4:5], v[4:5]
-; GFX940-SDAG-NEXT:    s_nop 0
-; GFX940-SDAG-NEXT:    v_cmp_eq_u64_e64 s[0:1], s[6:7], v[6:7]
-; GFX940-SDAG-NEXT:    s_and_b64 s[0:1], vcc, s[0:1]
-; GFX940-SDAG-NEXT:    s_and_saveexec_b64 s[0:1], s[0:1]
-; GFX940-SDAG-NEXT:    buffer_load_dword v8, v9, s[4:7], 0 offen nt
-; GFX940-SDAG-NEXT:    ; implicit-def: $vgpr4_vgpr5_vgpr6_vgpr7
-; GFX940-SDAG-NEXT:    ; implicit-def: $vgpr9
-; GFX940-SDAG-NEXT:    s_xor_b64 exec, exec, s[0:1]
-; GFX940-SDAG-NEXT:    s_cbranch_execnz .LBB0_1
-; GFX940-SDAG-NEXT:  ; %bb.2:
-; GFX940-SDAG-NEXT:    s_mov_b64 exec, s[2:3]
-; GFX940-SDAG-NEXT:    v_mov_b32_e32 v4, s8
-; GFX940-SDAG-NEXT:    s_mov_b64 s[0:1], exec
+; GFX940-SDAG-NEXT:    s_mov_b32 s6, s3
+; GFX940-SDAG-NEXT:    s_or_b64 s[10:11], s[6:7], s[12:13]
+; GFX940-SDAG-NEXT:    s_mov_b32 s13, s2
+; GFX940-SDAG-NEXT:    s_mov_b32 s2, s1
+; GFX940-SDAG-NEXT:    s_mov_b32 s3, s12
+; GFX940-SDAG-NEXT:    s_or_b64 s[8:9], s[2:3], s[12:13]
+; GFX940-SDAG-NEXT:    v_mov_b32_e32 v0, s0
+; GFX940-SDAG-NEXT:    buffer_load_dword v0, v0, s[8:11], 0 offen nt
+; GFX940-SDAG-NEXT:    s_load_dword s13, s[4:5], 0x30
+; GFX940-SDAG-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x20
+; GFX940-SDAG-NEXT:    s_mov_b32 s5, s12
+; GFX940-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX940-SDAG-NEXT:    s_mov_b32 s4, s3
+; GFX940-SDAG-NEXT:    s_or_b64 s[6:7], s[4:5], s[12:13]
+; GFX940-SDAG-NEXT:    s_mov_b32 s13, s2
+; GFX940-SDAG-NEXT:    s_mov_b32 s2, s1
+; GFX940-SDAG-NEXT:    s_mov_b32 s3, s12
+; GFX940-SDAG-NEXT:    s_or_b64 s[4:5], s[2:3], s[12:13]
+; GFX940-SDAG-NEXT:    v_mov_b32_e32 v1, s0
 ; GFX940-SDAG-NEXT:    s_waitcnt vmcnt(0)
-; GFX940-SDAG-NEXT:  .LBB0_3: ; =>This Inner Loop Header: Depth=1
-; GFX940-SDAG-NEXT:    v_readfirstlane_b32 s4, v0
-; GFX940-SDAG-NEXT:    v_readfirstlane_b32 s5, v1
-; GFX940-SDAG-NEXT:    v_readfirstlane_b32 s6, v2
-; GFX940-SDAG-NEXT:    v_readfirstlane_b32 s7, v3
-; GFX940-SDAG-NEXT:    v_cmp_eq_u64_e32 vcc, s[4:5], v[0:1]
-; GFX940-SDAG-NEXT:    s_nop 0
-; GFX940-SDAG-NEXT:    v_cmp_eq_u64_e64 s[0:1], s[6:7], v[2:3]
-; GFX940-SDAG-NEXT:    s_and_b64 s[0:1], vcc, s[0:1]
-; GFX940-SDAG-NEXT:    s_and_saveexec_b64 s[0:1], s[0:1]
-; GFX940-SDAG-NEXT:    buffer_store_dword v8, v4, s[4:7], 0 offen sc0 nt sc1
-; GFX940-SDAG-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
-; GFX940-SDAG-NEXT:    ; implicit-def: $vgpr8
-; GFX940-SDAG-NEXT:    ; implicit-def: $vgpr4
-; GFX940-SDAG-NEXT:    s_xor_b64 exec, exec, s[0:1]
-; GFX940-SDAG-NEXT:    s_cbranch_execnz .LBB0_3
-; GFX940-SDAG-NEXT:  ; %bb.4:
+; GFX940-SDAG-NEXT:    buffer_store_dword v0, v1, s[4:7], 0 offen sc0 nt sc1
 ; GFX940-SDAG-NEXT:    s_endpgm
 ;
 ; GFX940-GISEL-LABEL: buffer_nontemporal_load_store:
@@ -169,31 +134,34 @@ define amdgpu_kernel void @buffer_nontemporal_load_store(ptr addrspace(7) %in, p
 ; GFX10-SDAG-LABEL: buffer_nontemporal_load_store:
 ; GFX10-SDAG:       ; %bb.0: ; %entry
 ; GFX10-SDAG-NEXT:    s_clause 0x1
-; GFX10-SDAG-NEXT:    s_load_dwordx4 s[4:7], s[8:9], 0x0
+; GFX10-SDAG-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
 ; GFX10-SDAG-NEXT:    s_load_dword s11, s[8:9], 0x10
 ; GFX10-SDAG-NEXT:    s_mov_b32 s10, 0
-; GFX10-SDAG-NEXT:    s_add_u32 s0, s0, s15
+; GFX10-SDAG-NEXT:    s_mov_b32 s5, s10
 ; GFX10-SDAG-NEXT:    s_mov_b32 s13, s10
-; GFX10-SDAG-NEXT:    s_addc_u32 s1, s1, 0
 ; GFX10-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX10-SDAG-NEXT:    v_mov_b32_e32 v0, s4
-; GFX10-SDAG-NEXT:    s_mov_b32 s12, s7
-; GFX10-SDAG-NEXT:    s_or_b64 s[14:15], s[12:13], s[10:11]
-; GFX10-SDAG-NEXT:    s_mov_b32 s12, s5
-; GFX10-SDAG-NEXT:    s_mov_b32 s13, s6
-; GFX10-SDAG-NEXT:    buffer_load_dword v0, v0, s[12:15], 0 offen slc
+; GFX10-SDAG-NEXT:    s_mov_b32 s4, s3
+; GFX10-SDAG-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-SDAG-NEXT:    s_mov_b32 s12, s1
+; GFX10-SDAG-NEXT:    s_or_b64 s[6:7], s[4:5], s[10:11]
+; GFX10-SDAG-NEXT:    s_mov_b32 s11, s2
+; GFX10-SDAG-NEXT:    s_or_b64 s[4:5], s[12:13], s[10:11]
+; GFX10-SDAG-NEXT:    buffer_load_dword v0, v0, s[4:7], 0 offen slc
 ; GFX10-SDAG-NEXT:    s_clause 0x1
 ; GFX10-SDAG-NEXT:    s_load_dword s11, s[8:9], 0x30
-; GFX10-SDAG-NEXT:    s_load_dwordx4 s[4:7], s[8:9], 0x20
-; GFX10-SDAG-NEXT:    s_mov_b32 s9, s10
+; GFX10-SDAG-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x20
+; GFX10-SDAG-NEXT:    s_waitcnt_depctr 0xffe3
+; GFX10-SDAG-NEXT:    s_mov_b32 s5, s10
 ; GFX10-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX10-SDAG-NEXT:    v_mov_b32_e32 v1, s4
-; GFX10-SDAG-NEXT:    s_mov_b32 s8, s7
-; GFX10-SDAG-NEXT:    s_or_b64 s[10:11], s[8:9], s[10:11]
-; GFX10-SDAG-NEXT:    s_mov_b32 s8, s5
-; GFX10-SDAG-NEXT:    s_mov_b32 s9, s6
+; GFX10-SDAG-NEXT:    s_mov_b32 s4, s3
+; GFX10-SDAG-NEXT:    v_mov_b32_e32 v1, s0
+; GFX10-SDAG-NEXT:    s_or_b64 s[6:7], s[4:5], s[10:11]
+; GFX10-SDAG-NEXT:    s_mov_b32 s11, s2
+; GFX10-SDAG-NEXT:    s_mov_b32 s2, s1
+; GFX10-SDAG-NEXT:    s_mov_b32 s3, s10
+; GFX10-SDAG-NEXT:    s_or_b64 s[4:5], s[2:3], s[10:11]
 ; GFX10-SDAG-NEXT:    s_waitcnt vmcnt(0)
-; GFX10-SDAG-NEXT:    buffer_store_dword v0, v1, s[8:11], 0 offen glc slc
+; GFX10-SDAG-NEXT:    buffer_store_dword v0, v1, s[4:7], 0 offen glc slc
 ; GFX10-SDAG-NEXT:    s_endpgm
 ;
 ; GFX10-GISEL-LABEL: buffer_nontemporal_load_store:
@@ -229,69 +197,37 @@ define amdgpu_kernel void @buffer_nontemporal_load_store(ptr addrspace(7) %in, p
 ;
 ; GFX11-SDAG-LABEL: buffer_nontemporal_load_store:
 ; GFX11-SDAG:       ; %bb.0: ; %entry
-; GFX11-SDAG-NEXT:    s_clause 0x2
+; GFX11-SDAG-NEXT:    s_clause 0x1
 ; GFX11-SDAG-NEXT:    s_load_b128 s[0:3], s[4:5], 0x0
-; GFX11-SDAG-NEXT:    s_load_b128 s[8:11], s[4:5], 0x20
-; GFX11-SDAG-NEXT:    s_load_b32 s6, s[4:5], 0x10
+; GFX11-SDAG-NEXT:    s_load_b32 s13, s[4:5], 0x10
+; GFX11-SDAG-NEXT:    s_mov_b32 s12, 0
+; GFX11-SDAG-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-SDAG-NEXT:    s_mov_b32 s7, s12
+; GFX11-SDAG-NEXT:    s_mov_b32 s9, s12
 ; GFX11-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX11-SDAG-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
-; GFX11-SDAG-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
-; GFX11-SDAG-NEXT:    v_dual_mov_b32 v7, s8 :: v_dual_mov_b32 v8, s9
-; GFX11-SDAG-NEXT:    v_dual_mov_b32 v9, s10 :: v_dual_mov_b32 v10, s11
-; GFX11-SDAG-NEXT:    scratch_store_b128 off, v[0:3], off offset:32
+; GFX11-SDAG-NEXT:    s_mov_b32 s6, s3
+; GFX11-SDAG-NEXT:    v_mov_b32_e32 v0, s0
+; GFX11-SDAG-NEXT:    s_mov_b32 s8, s1
+; GFX11-SDAG-NEXT:    s_or_b64 s[10:11], s[6:7], s[12:13]
+; GFX11-SDAG-NEXT:    s_mov_b32 s13, s2
+; GFX11-SDAG-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-SDAG-NEXT:    s_or_b64 s[8:9], s[8:9], s[12:13]
+; GFX11-SDAG-NEXT:    buffer_load_b32 v0, v0, s[8:11], 0 offen slc dlc
 ; GFX11-SDAG-NEXT:    s_clause 0x1
-; GFX11-SDAG-NEXT:    scratch_load_b64 v[5:6], off, off offset:40
-; GFX11-SDAG-NEXT:    scratch_load_b32 v4, off, off offset:36
-; GFX11-SDAG-NEXT:    s_load_b32 s1, s[4:5], 0x30
-; GFX11-SDAG-NEXT:    scratch_store_b128 off, v[7:10], off
-; GFX11-SDAG-NEXT:    s_clause 0x1
-; GFX11-SDAG-NEXT:    scratch_load_b64 v[1:2], off, off offset:8
-; GFX11-SDAG-NEXT:    scratch_load_b32 v0, off, off offset:4
-; GFX11-SDAG-NEXT:    v_mov_b32_e32 v7, s6
-; GFX11-SDAG-NEXT:    v_mov_b32_e32 v9, s0
+; GFX11-SDAG-NEXT:    s_load_b32 s13, s[4:5], 0x30
+; GFX11-SDAG-NEXT:    s_load_b128 s[0:3], s[4:5], 0x20
+; GFX11-SDAG-NEXT:    s_mov_b32 s5, s12
 ; GFX11-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX11-SDAG-NEXT:    v_mov_b32_e32 v3, s1
-; GFX11-SDAG-NEXT:    s_mov_b32 s1, exec_lo
-; GFX11-SDAG-NEXT:    s_waitcnt vmcnt(0)
-; GFX11-SDAG-NEXT:  .LBB0_1: ; =>This Inner Loop Header: Depth=1
-; GFX11-SDAG-NEXT:    v_readfirstlane_b32 s4, v4
-; GFX11-SDAG-NEXT:    v_readfirstlane_b32 s5, v5
-; GFX11-SDAG-NEXT:    v_readfirstlane_b32 s6, v6
-; GFX11-SDAG-NEXT:    v_readfirstlane_b32 s7, v7
-; GFX11-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-SDAG-NEXT:    v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[4:5]
-; GFX11-SDAG-NEXT:    v_cmp_eq_u64_e64 s0, s[6:7], v[6:7]
-; GFX11-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX11-SDAG-NEXT:    s_and_b32 s0, vcc_lo, s0
-; GFX11-SDAG-NEXT:    s_and_saveexec_b32 s0, s0
-; GFX11-SDAG-NEXT:    buffer_load_b32 v8, v9, s[4:7], 0 offen slc dlc
-; GFX11-SDAG-NEXT:    ; implicit-def: $vgpr4_vgpr5_vgpr6_vgpr7
-; GFX11-SDAG-NEXT:    ; implicit-def: $vgpr9
-; GFX11-SDAG-NEXT:    s_xor_b32 exec_lo, exec_lo, s0
-; GFX11-SDAG-NEXT:    s_cbranch_execnz .LBB0_1
-; GFX11-SDAG-NEXT:  ; %bb.2:
-; GFX11-SDAG-NEXT:    s_mov_b32 exec_lo, s1
-; GFX11-SDAG-NEXT:    v_mov_b32_e32 v4, s8
-; GFX11-SDAG-NEXT:    s_mov_b32 s0, exec_lo
-; GFX11-SDAG-NEXT:  .LBB0_3: ; =>This Inner Loop Header: Depth=1
-; GFX11-SDAG-NEXT:    v_readfirstlane_b32 s4, v0
-; GFX11-SDAG-NEXT:    v_readfirstlane_b32 s5, v1
-; GFX11-SDAG-NEXT:    v_readfirstlane_b32 s6, v2
-; GFX11-SDAG-NEXT:    v_readfirstlane_b32 s7, v3
-; GFX11-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-SDAG-NEXT:    v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
-; GFX11-SDAG-NEXT:    v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
-; GFX11-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX11-SDAG-NEXT:    s_and_b32 s0, vcc_lo, s0
-; GFX11-SDAG-NEXT:    s_and_saveexec_b32 s0, s0
+; GFX11-SDAG-NEXT:    v_mov_b32_e32 v1, s0
+; GFX11-SDAG-NEXT:    s_mov_b32 s4, s3
+; GFX11-SDAG-NEXT:    s_mov_b32 s3, s12
+; GFX11-SDAG-NEXT:    s_or_b64 s[6:7], s[4:5], s[12:13]
+; GFX11-SDAG-NEXT:    s_mov_b32 s13, s2
+; GFX11-SDAG-NEXT:    s_mov_b32 s2, s1
+; GFX11-SDAG-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-SDAG-NEXT:    s_or_b64 s[4:5], s[2:3], s[12:13]
 ; GFX11-SDAG-NEXT:    s_waitcnt vmcnt(0)
-; GFX11-SDAG-NEXT:    buffer_store_b32 v8, v4, s[4:7], 0 offen glc slc dlc
-; GFX11-SDAG-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
-; GFX11-SDAG-NEXT:    ; implicit-def: $vgpr8
-; GFX11-SDAG-NEXT:    ; implicit-def: $vgpr4
-; GFX11-SDAG-NEXT:    s_xor_b32 exec_lo, exec_lo, s0
-; GFX11-SDAG-NEXT:    s_cbranch_execnz .LBB0_3
-; GFX11-SDAG-NEXT:  ; %bb.4:
+; GFX11-SDAG-NEXT:    buffer_store_b32 v0, v1, s[4:7], 0 offen glc slc dlc
 ; GFX11-SDAG-NEXT:    s_endpgm
 ;
 ; GFX11-GISEL-LABEL: buffer_nontemporal_load_store:
@@ -330,77 +266,37 @@ define amdgpu_kernel void @buffer_nontemporal_load_store(ptr addrspace(7) %in, p
 ;
 ; GFX12-SDAG-LABEL: buffer_nontemporal_load_store:
 ; GFX12-SDAG:       ; %bb.0: ; %entry
-; GFX12-SDAG-NEXT:    s_clause 0x2
+; GFX12-SDAG-NEXT:    s_clause 0x1
 ; GFX12-SDAG-NEXT:    s_load_b128 s[0:3], s[4:5], 0x0
-; GFX12-SDAG-NEXT:    s_load_b128 s[8:11], s[4:5], 0x20
-; GFX12-SDAG-NEXT:    s_load_b32 s6, s[4:5], 0x10
+; GFX12-SDAG-NEXT:    s_load_b32 s13, s[4:5], 0x10
+; GFX12-SDAG-NEXT:    s_mov_b32 s12, 0
+; GFX12-SDAG-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-SDAG-NEXT:    s_mov_b32 s7, s12
+; GFX12-SDAG-NEXT:    s_mov_b32 s9, s12
 ; GFX12-SDAG-NEXT:    s_wait_kmcnt 0x0
-; GFX12-SDAG-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
-; GFX12-SDAG-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
-; GFX12-SDAG-NEXT:    v_dual_mov_b32 v7, s8 :: v_dual_mov_b32 v8, s9
-; GFX12-SDAG-NEXT:    v_dual_mov_b32 v9, s10 :: v_dual_mov_b32 v10, s11
-; GFX12-SDAG-NEXT:    scratch_store_b128 off, v[0:3], off offset:32
-; GFX12-SDAG-NEXT:    s_clause 0x1
-; GFX12-SDAG-NEXT:    scratch_load_b64 v[5:6], off, off offset:40
-; GFX12-SDAG-NEXT:    scratch_load_b32 v4, off, off offset:36
-; GFX12-SDAG-NEXT:    s_load_b32 s1, s[4:5], 0x30
-; GFX12-SDAG-NEXT:    scratch_store_b128 off, v[7:10], off
+; GFX12-SDAG-NEXT:    s_mov_b32 s6, s3
+; GFX12-SDAG-NEXT:    v_mov_b32_e32 v0, s0
+; GFX12-SDAG-NEXT:    s_mov_b32 s8, s1
+; GFX12-SDAG-NEXT:    s_or_b64 s[10:11], s[6:7], s[12:13]
+; GFX12-SDAG-NEXT:    s_mov_b32 s13, s2
+; GFX12-SDAG-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-SDAG-NEXT:    s_or_b64 s[8:9], s[8:9], s[12:13]
+; GFX12-SDAG-NEXT:    buffer_load_b32 v0, v0, s[8:11], null offen th:TH_LOAD_NT
 ; GFX12-SDAG-NEXT:    s_clause 0x1
-; GFX12-SDAG-NEXT:    scratch_load_b64 v[1:2], off, off offset:8
-; GFX12-SDAG-NEXT:    scratch_load_b32 v0, off, off offset:4
-; GFX12-SDAG-NEXT:    v_mov_b32_e32 v7, s6
-; GFX12-SDAG-NEXT:    v_mov_b32_e32 v9, s0
+; GFX12-SDAG-NEXT:    s_load_b32 s13, s[4:5], 0x30
+; GFX12-SDAG-NEXT:    s_load_b128 s[0:3], s[4:5], 0x20
+; GFX12-SDAG-NEXT:    s_mov_b32 s5, s12
 ; GFX12-SDAG-NEXT:    s_wait_kmcnt 0x0
-; GFX12-SDAG-NEXT:    v_mov_b32_e32 v3, s1
-; GFX12-SDAG-NEXT:    s_mov_b32 s1, exec_lo
-; GFX12-SDAG-NEXT:  .LBB0_1: ; =>This Inner Loop Header: Depth=1
-; GFX12-SDAG-NEXT:    s_wait_loadcnt 0x2
-; GFX12-SDAG-NEXT:    v_readfirstlane_b32 s4, v4
-; GFX12-SDAG-NEXT:    v_readfirstlane_b32 s5, v5
-; GFX12-SDAG-NEXT:    v_readfirstlane_b32 s6, v6
-; GFX12-SDAG-NEXT:    v_readfirstlane_b32 s7, v7
-; GFX12-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-SDAG-NEXT:    v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[4:5]
-; GFX12-SDAG-NEXT:    v_cmp_eq_u64_e64 s0, s[6:7], v[6:7]
-; GFX12-SDAG-NEXT:    s_wait_alu 0xfffe
-; GFX12-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX12-SDAG-NEXT:    s_and_b32 s0, vcc_lo, s0
-; GFX12-SDAG-NEXT:    s_wait_alu 0xfffe
-; GFX12-SDAG-NEXT:    s_and_saveexec_b32 s0, s0
-; GFX12-SDAG-NEXT:    s_wait_loadcnt 0x0
-; GFX12-SDAG-NEXT:    buffer_load_b32 v8, v9, s[4:7], null offen th:TH_LOAD_NT
-; GFX12-SDAG-NEXT:    ; implicit-def: $vgpr4_vgpr5_vgpr6_vgpr7
-; GFX12-SDAG-NEXT:    ; implicit-def: $vgpr9
-; GFX12-SDAG-NEXT:    s_wait_alu 0xfffe
-; GFX12-SDAG-NEXT:    s_xor_b32 exec_lo, exec_lo, s0
-; GFX12-SDAG-NEXT:    s_cbranch_execnz .LBB0_1
-; GFX12-SDAG-NEXT:  ; %bb.2:
-; GFX12-SDAG-NEXT:    s_mov_b32 exec_lo, s1
-; GFX12-SDAG-NEXT:    v_mov_b32_e32 v4, s8
-; GFX12-SDAG-NEXT:    s_mov_b32 s0, exec_lo
-; GFX12-SDAG-NEXT:  .LBB0_3: ; =>This Inner Loop Header: Depth=1
-; GFX12-SDAG-NEXT:    s_wait_loadcnt 0x1
-; GFX12-SDAG-NEXT:    v_readfirstlane_b32 s4, v0
-; GFX12-SDAG-NEXT:    v_readfirstlane_b32 s5, v1
-; GFX12-SDAG-NEXT:    v_readfirstlane_b32 s6, v2
-; GFX12-SDAG-NEXT:    v_readfirstlane_b32 s7, v3
-; GFX12-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-SDAG-NEXT:    v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
-; GFX12-SDAG-NEXT:    v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
-; GFX12-SDAG-NEXT:    s_wait_alu 0xfffe
-; GFX12-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX12-SDAG-NEXT:    s_and_b32 s0, vcc_lo, s0
-; GFX12-SDAG-NEXT:    s_wait_alu 0xfffe
-; GFX12-SDAG-NEXT:    s_and_saveexec_b32 s0, s0
+; GFX12-SDAG-NEXT:    v_mov_b32_e32 v1, s0
+; GFX12-SDAG-NEXT:    s_mov_b32 s4, s3
+; GFX12-SDAG-NEXT:    s_mov_b32 s3, s12
+; GFX12-SDAG-NEXT:    s_or_b64 s[6:7], s[4:5], s[12:13]
+; GFX12-SDAG-NEXT:    s_mov_b32 s13, s2
+; GFX12-SDAG-NEXT:    s_mov_b32 s2, s1
+; GFX12-SDAG-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-SDAG-NEXT:    s_or_b64 s[4:5], s[2:3], s[12:13]
 ; GFX12-SDAG-NEXT:    s_wait_loadcnt 0x0
-; GFX12-SDAG-NEXT:    buffer_store_b32 v8, v4, s[4:7], null offen th:TH_STORE_NT
-; GFX12-SDAG-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
-; GFX12-SDAG-NEXT:    ; implicit-def: $vgpr8
-; GFX12-SDAG-NEXT:    ; implicit-def: $vgpr4
-; GFX12-SDAG-NEXT:    s_wait_alu 0xfffe
-; GFX12-SDAG-NEXT:    s_xor_b32 exec_lo, exec_lo, s0
-; GFX12-SDAG-NEXT:    s_cbranch_execnz .LBB0_3
-; GFX12-SDAG-NEXT:  ; %bb.4:
+; GFX12-SDAG-NEXT:    buffer_store_b32 v0, v1, s[4:7], null offen th:TH_STORE_NT
 ; GFX12-SDAG-NEXT:    s_endpgm
 ;
 ; GFX12-GISEL-LABEL: buffer_nontemporal_load_store:
@@ -426,7 +322,6 @@ define amdgpu_kernel void @buffer_nontemporal_load_store(ptr addrspace(7) %in, p
 ; GFX12-GISEL-NEXT:    s_load_b32 s7, s[4:5], 0x30
 ; GFX12-GISEL-NEXT:    s_mov_b32 s4, s9
 ; GFX12-GISEL-NEXT:    s_wait_kmcnt 0x0
-; GFX12-GISEL-NEXT:    s_wait_alu 0xfffe
 ; GFX12-GISEL-NEXT:    v_mov_b32_e32 v1, s0
 ; GFX12-GISEL-NEXT:    s_mov_b32 s8, s1
 ; GFX12-GISEL-NEXT:    s_mov_b32 s5, s2
@@ -446,30 +341,32 @@ entry:
 define amdgpu_kernel void @buffer_nontemporal_and_volatile_load_store(ptr addrspace(7) %in, ptr addrspace(7) %out) {
 ; GFX9-SDAG-LABEL: buffer_nontemporal_and_volatile_load_store:
 ; GFX9-SDAG:       ; %bb.0: ; %entry
-; GFX9-SDAG-NEXT:    s_load_dwordx4 s[4:7], s[8:9], 0x0
+; GFX9-SDAG-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
 ; GFX9-SDAG-NEXT:    s_load_dword s11, s[8:9], 0x10
 ; GFX9-SDAG-NEXT:    s_mov_b32 s10, 0
-; GFX9-SDAG-NEXT:    s_add_u32 s0, s0, s15
-; GFX9-SDAG-NEXT:    s_mov_b32 s15, s10
+; GFX9-SDAG-NEXT:    s_mov_b32 s5, s10
 ; GFX9-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-SDAG-NEXT:    s_mov_b32 s14, s7
-; GFX9-SDAG-NEXT:    s_addc_u32 s1, s1, 0
-; GFX9-SDAG-NEXT:    s_mov_b32 s12, s5
-; GFX9-SDAG-NEXT:    s_or_b64 s[14:15], s[14:15], s[10:11]
-; GFX9-SDAG-NEXT:    s_mov_b32 s13, s6
-; GFX9-SDAG-NEXT:    v_mov_b32_e32 v0, s4
-; GFX9-SDAG-NEXT:    buffer_load_dword v0, v0, s[12:15], 0 offen glc
+; GFX9-SDAG-NEXT:    s_mov_b32 s4, s3
+; GFX9-SDAG-NEXT:    s_or_b64 s[6:7], s[4:5], s[10:11]
+; GFX9-SDAG-NEXT:    s_mov_b32 s11, s2
+; GFX9-SDAG-NEXT:    s_mov_b32 s2, s1
+; GFX9-SDAG-NEXT:    s_mov_b32 s3, s10
+; GFX9-SDAG-NEXT:    s_or_b64 s[4:5], s[2:3], s[10:11]
+; GFX9-SDAG-NEXT:    v_mov_b32_e32 v0, s0
+; GFX9-SDAG-NEXT:    buffer_load_dword v0, v0, s[4:7], 0 offen glc
 ; GFX9-SDAG-NEXT:    s_load_dword s11, s[8:9], 0x30
-; GFX9-SDAG-NEXT:    s_load_dwordx4 s[4:7], s[8:9], 0x20
-; GFX9-SDAG-NEXT:    s_mov_b32 s9, s10
+; GFX9-SDAG-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x20
+; GFX9-SDAG-NEXT:    s_mov_b32 s5, s10
 ; GFX9-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-SDAG-NEXT:    s_mov_b32 s8, s7
-; GFX9-SDAG-NEXT:    s_or_b64 s[10:11], s[8:9], s[10:11]
-; GFX9-SDAG-NEXT:    s_mov_b32 s8, s5
-; GFX9-SDAG-NEXT:    s_mov_b32 s9, s6
-; GFX9-SDAG-NEXT:    v_mov_b32_e32 v1, s4
+; GFX9-SDAG-NEXT:    s_mov_b32 s4, s3
+; GFX9-SDAG-NEXT:    s_or_b64 s[6:7], s[4:5], s[10:11]
+; GFX9-SDAG-NEXT:    s_mov_b32 s11, s2
+; GFX9-SDAG-NEXT:    s_mov_b32 s2, s1
+; GFX9-SDAG-NEXT:    s_mov_b32 s3, s10
+; GFX9-SDAG-NEXT:    s_or_b64 s[4:5], s[2:3], s[10:11]
+; GFX9-SDAG-NEXT:    v_mov_b32_e32 v1, s0
 ; GFX9-SDAG-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-SDAG-NEXT:    buffer_store_dword v0, v1, s[8:11], 0 offen
+; GFX9-SDAG-NEXT:    buffer_store_dword v0, v1, s[4:7], 0 offen
 ; GFX9-SDAG-NEXT:    s_endpgm
 ;
 ; GFX9-GISEL-LABEL: buffer_nontemporal_and_volatile_load_store:
@@ -505,68 +402,31 @@ define amdgpu_kernel void @buffer_nontemporal_and_volatile_load_store(ptr addrsp
 ; GFX940-SDAG-LABEL: buffer_nontemporal_and_volatile_load_store:
 ; GFX940-SDAG:       ; %bb.0: ; %entry
 ; GFX940-SDAG-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
-; GFX940-SDAG-NEXT:    s_load_dword s6, s[4:5], 0x10
-; GFX940-SDAG-NEXT:    s_load_dwordx4 s[8:11], s[4:5], 0x20
-; GFX940-SDAG-NEXT:    s_load_dword s7, s[4:5], 0x30
+; GFX940-SDAG-NEXT:    s_load_dword s13, s[4:5], 0x10
+; GFX940-SDAG-NEXT:    s_mov_b32 s12, 0
+; GFX940-SDAG-NEXT:    s_mov_b32 s7, s12
 ; GFX940-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX940-SDAG-NEXT:    v_mov_b64_e32 v[0:1], s[0:1]
-; GFX940-SDAG-NEXT:    v_mov_b64_e32 v[2:3], s[2:3]
-; GFX940-SDAG-NEXT:    scratch_store_dwordx4 off, v[0:3], off offset:32 sc0 sc1
-; GFX940-SDAG-NEXT:    scratch_load_dwordx2 v[10:11], off, off offset:40
-; GFX940-SDAG-NEXT:    scratch_load_dword v4, off, off offset:36
-; GFX940-SDAG-NEXT:    v_mov_b64_e32 v[0:1], s[8:9]
-; GFX940-SDAG-NEXT:    v_mov_b64_e32 v[2:3], s[10:11]
-; GFX940-SDAG-NEXT:    scratch_store_dwordx4 off, v[0:3], off sc0 sc1
-; GFX940-SDAG-NEXT:    scratch_load_dwordx2 v[12:13], off, off offset:8
-; GFX940-SDAG-NEXT:    s_nop 0
-; GFX940-SDAG-NEXT:    scratch_load_dword v0, off, off offset:4
-; GFX940-SDAG-NEXT:    v_mov_b32_e32 v7, s6
-; GFX940-SDAG-NEXT:    v_mov_b32_e32 v3, s7
-; GFX940-SDAG-NEXT:    v_mov_b32_e32 v9, s0
-; GFX940-SDAG-NEXT:    s_mov_b64 s[2:3], exec
-; GFX940-SDAG-NEXT:    s_waitcnt vmcnt(4)
-; GFX940-SDAG-NEXT:    v_mov_b32_e32 v6, v11
-; GFX940-SDAG-NEXT:    v_mov_b32_e32 v5, v10
-; GFX940-SDAG-NEXT:    s_waitcnt vmcnt(1)
-; GFX940-SDAG-NEXT:    v_mov_b32_e32 v2, v13
-; GFX940-SDAG-NEXT:    v_mov_b32_e32 v1, v12
-; GFX940-SDAG-NEXT:  .LBB1_1: ; =>This Inner Loop Header: Depth=1
-; GFX940-SDAG-NEXT:    v_readfirstlane_b32 s4, v4
-; GFX940-SDAG-NEXT:    v_readfirstlane_b32 s5, v5
-; GFX940-SDAG-NEXT:    v_readfirstlane_b32 s6, v6
-; GFX940-SDAG-NEXT:    v_readfirstlane_b32 s7, v7
-; GFX940-SDAG-NEXT:    v_cmp_eq_u64_e32 vcc, s[4:5], v[4:5]
-; GFX940-SDAG-NEXT:    s_nop 0
-; GFX940-SDAG-NEXT:    v_cmp_eq_u64_e64 s[0:1], s[6:7], v[6:7]
-; GFX940-SDAG-NEXT:    s_and_b64 s[0:1], vcc, s[0:1]
-; GFX940-SDAG-NEXT:    s_and_saveexec_b64 s[0:1], s[0:1]
-; GFX940-SDAG-NEXT:    buffer_load_dword v8, v9, s[4:7], 0 offen sc0 sc1
-; GFX940-SDAG-NEXT:    ; implicit-def: $vgpr4_vgpr5_vgpr6_vgpr7
-; GFX940-SDAG-NEXT:    ; implicit-def: $vgpr9
-; GFX940-SDAG-NEXT:    s_xor_b64 exec, exec, s[0:1]
-; GFX940-SDAG-NEXT:    s_cbranch_execnz .LBB1_1
-; GFX940-SDAG-NEXT:  ; %bb.2:
-; GFX940-SDAG-NEXT:    s_mov_b64 exec, s[2:3]
-; GFX940-SDAG-NEXT:    v_mov_b32_e32 v4, s8
-; GFX940-SDAG-NEXT:    s_mov_b64 s[0:1], exec
+; GFX940-SDAG-NEXT:    s_mov_b32 s6, s3
+; GFX940-SDAG-NEXT:    s_or_b64 s[10:11], s[6:7], s[12:13]
+; GFX940-SDAG-NEXT:    s_mov_b32 s13, s2
+; GFX940-SDAG-NEXT:    s_mov_b32 s2, s1
+; GFX940-SDAG-NEXT:    s_mov_b32 s3, s12
+; GFX940-SDAG-NEXT:    s_or_b64 s[8:9], s[2:3], s[12:13]
+; GFX940-SDAG-NEXT:    v_mov_b32_e32 v0, s0
+; GFX940-SDAG-NEXT:    buffer_load_dword v0, v0, s[8:11], 0 offen sc0 sc1
+; GFX940-SDAG-NEXT:    s_load_dword s13, s[4:5], 0x30
+; GFX940-SDAG-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x20
+; GFX940-SDAG-NEXT:    s_mov_b32 s5, s12
+; GFX940-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX940-SDAG-NEXT:    s_mov_b32 s4, s3
+; GFX940-SDAG-NEXT:    s_or_b64 s[6:7], s[4:5], s[12:13]
+; GFX940-SDAG-NEXT:    s_mov_b32 s13, s2
+; GFX940-SDAG-NEXT:    s_mov_b32 s2, s1
+; GFX940-SDAG-NEXT:    s_mov_b32 s3, s12
+; GFX940-SDAG-NEXT:    s_or_b64 s[4:5], s[2:3], s[12:13]
+; GFX940-SDAG-NEXT:    v_mov_b32_e32 v1, s0
 ; GFX940-SDAG-NEXT:    s_waitcnt vmcnt(0)
-; GFX940-SDAG-NEXT:  .LBB1_3: ; =>This Inner Loop Header: Depth=1
-; GFX940-SDAG-NEXT:    v_readfirstlane_b32 s4, v0
-; GFX940-SDAG-NEXT:    v_readfirstlane_b32 s5, v1
-; GFX940-SDAG-NEXT:    v_readfirstlane_b32 s6, v2
-; GFX940-SDAG-NEXT:    v_readfirstlane_b32 s7, v3
-; GFX940-SDAG-NEXT:    v_cmp_eq_u64_e32 vcc, s[4:5], v[0:1]
-; GFX940-SDAG-NEXT:    s_nop 0
-; GFX940-SDAG-NEXT:    v_cmp_eq_u64_e64 s[0:1], s[6:7], v[2:3]
-; GFX940-SDAG-NEXT:    s_and_b64 s[0:1], vcc, s[0:1]
-; GFX940-SDAG-NEXT:    s_and_saveexec_b64 s[0:1], s[0:1]
-; GFX940-SDAG-NEXT:    buffer_store_dword v8, v4, s[4:7], 0 offen sc0 sc1
-; GFX940-SDAG-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
-; GFX940-SDAG-NEXT:    ; implicit-def: $vgpr8
-; GFX940-SDAG-NEXT:    ; implicit-def: $vgpr4
-; GFX940-SDAG-NEXT:    s_xor_b64 exec, exec, s[0:1]
-; GFX940-SDAG-NEXT:    s_cbranch_execnz .LBB1_3
-; GFX940-SDAG-NEXT:  ; %bb.4:
+; GFX940-SDAG-NEXT:    buffer_store_dword v0, v1, s[4:7], 0 offen sc0 sc1
 ; GFX940-SDAG-NEXT:    s_endpgm
 ;
 ; GFX940-GISEL-LABEL: buffer_nontemporal_and_volatile_load_store:
@@ -602,31 +462,34 @@ define amdgpu_kernel void @buffer_nontemporal_and_volatile_load_store(ptr addrsp
 ; GFX10-SDAG-LABEL: buffer_nontemporal_and_volatile_load_store:
 ; GFX10-SDAG:       ; %bb.0: ; %entry
 ; GFX10-SDAG-NEXT:    s_clause 0x1
-; GFX10-SDAG-NEXT:    s_load_dwordx4 s[4:7], s[8:9], 0x0
+; GFX10-SDAG-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
 ; GFX10-SDAG-NEXT:    s_load_dword s11, s[8:9], 0x10
 ; GFX10-SDAG-NEXT:    s_mov_b32 s10, 0
-; GFX10-SDAG-NEXT:    s_add_u32 s0, s0, s15
+; GFX10-SDAG-NEXT:    s_mov_b32 s5, s10
 ; GFX10-SDAG-NEXT:    s_mov_b32 s13, s10
-; GFX10-SDAG-NEXT:    s_addc_u32 s1, s1, 0
 ; GFX10-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX10-SDAG-NEXT:    v_mov_b32_e32 v0, s4
-; GFX10-SDAG-NEXT:    s_mov_b32 s12, s7
-; GFX10-SDAG-NEXT:    s_or_b64 s[14:15], s[12:13], s[10:11]
-; GFX10-SDAG-NEXT:    s_mov_b32 s12, s5
-; GFX10-SDAG-NEXT:    s_mov_b32 s13, s6
-; GFX10-SDAG-NEXT:    buffer_load_dword v0, v0, s[12:15], 0 offen glc dlc
+; GFX10-SDAG-NEXT:    s_mov_b32 s4, s3
+; GFX10-SDAG-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-SDAG-NEXT:    s_mov_b32 s12, s1
+; GFX10-SDAG-NEXT:    s_or_b64 s[6:7], s[4:5], s[10:11]
+; GFX10-SDAG-NEXT:    s_mov_b32 s11, s2
+; GFX10-SDAG-NEXT:    s_or_b64 s[4:5], s[12:13], s[10:11]
+; GFX10-SDAG-NEXT:    buffer_load_dword v0, v0, s[4:7], 0 offen glc dlc
 ; GFX10-SDAG-NEXT:    s_clause 0x1
 ; GFX10-SDAG-NEXT:    s_load_dword s11, s[8:9], 0x30
-; GFX10-SDAG-NEXT:    s_load_dwordx4 s[4:7], s[8:9], 0x20
-; GFX10-SDAG-NEXT:    s_mov_b32 s9, s10
+; GFX10-SDAG-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x20
+; GFX10-SDAG-NEXT:    s_waitcnt_depctr 0xffe3
+; GFX10-SDAG-NEXT:    s_mov_b32 s5, s10
 ; GFX10-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX10-SDAG-NEXT:    v_mov_b32_e32 v1, s4
-; GFX10-SDAG-NEXT:    s_mov_b32 s8, s7
-; GFX10-SDAG-NEXT:    s_or_b64 s[10:11], s[8:9], s[10:11]
-; GFX10-SDAG-NEXT:    s_mov_b32 s8, s5
-; GFX10-SDAG-NEXT:    s_mov_b32 s9, s6
+; GFX10-SDAG-NEXT:    s_mov_b32 s4, s3
+; GFX10-SDAG-NEXT:    v_mov_b32_e32 v1, s0
+; GFX10-SDAG-NEXT:    s_or_b64 s[6:7], s[4:5], s[10:11]
+; GFX10-SDAG-NEXT:    s_mov_b32 s11, s2
+; GFX10-SDAG-NEXT:    s_mov_b32 s2, s1
+; GFX10-SDAG-NEXT:    s_mov_b32 s3, s10
+; GFX10-SDAG-NEXT:    s_or_b64 s[4:5], s[2:3], s[10:11]
 ; GFX10-SDAG-NEXT:    s_waitcnt vmcnt(0)
-; GFX10-SDAG-NEXT:    buffer_store_dword v0, v1, s[8:11], 0 offen
+; GFX10-SDAG-NEXT:    buffer_store_dword v0, v1, s[4:7], 0 offen
 ; GFX10-SDAG-NEXT:    s_endpgm
 ;
 ; GFX10-GISEL-LABEL: buffer_nontemporal_and_volatile_load_store:
@@ -662,69 +525,37 @@ define amdgpu_kernel void @buffer_nontemporal_and_volatile_load_store(ptr addrsp
 ;
 ; GFX11-SDAG-LABEL: buffer_nontemporal_and_volatile_load_store:
 ; GFX11-SDAG:       ; %bb.0: ; %entry
-; GFX11-SDAG-NEXT:    s_clause 0x2
+; GFX11-SDAG-NEXT:    s_clause 0x1
 ; GFX11-SDAG-NEXT:    s_load_b128 s[0:3], s[4:5], 0x0
-; GFX11-SDAG-NEXT:    s_load_b128 s[8:11], s[4:5], 0x20
-; GFX11-SDAG-NEXT:    s_load_b32 s6, s[4:5], 0x10
+; GFX11-SDAG-NEXT:    s_load_b32 s13, s[4:5], 0x10
+; GFX11-SDAG-NEXT:    s_mov_b32 s12, 0
+; GFX11-SDAG-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-SDAG-NEXT:    s_mov_b32 s7, s12
+; GFX11-SDAG-NEXT:    s_mov_b32 s9, s12
 ; GFX11-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX11-SDAG-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
-; GFX11-SDAG-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
-; GFX11-SDAG-NEXT:    v_dual_mov_b32 v7, s8 :: v_dual_mov_b32 v8, s9
-; GFX11-SDAG-NEXT:    v_dual_mov_b32 v9, s10 :: v_dual_mov_b32 v10, s11
-; GFX11-SDAG-NEXT:    scratch_store_b128 off, v[0:3], off offset:32
+; GFX11-SDAG-NEXT:    s_mov_b32 s6, s3
+; GFX11-SDAG-NEXT:    v_mov_b32_e32 v0, s0
+; GFX11-SDAG-NEXT:    s_mov_b32 s8, s1
+; GFX11-SDAG-NEXT:    s_or_b64 s[10:11], s[6:7], s[12:13]
+; GFX11-SDAG-NEXT:    s_mov_b32 s13, s2
+; GFX11-SDAG-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-SDAG-NEXT:    s_or_b64 s[8:9], s[8:9], s[12:13]
+; GFX11-SDAG-NEXT:    buffer_load_b32 v0, v0, s[8:11], 0 offen glc dlc
 ; GFX11-SDAG-NEXT:    s_clause 0x1
-; GFX11-SDAG-NEXT:    scratch_load_b64 v[5:6], off, off offset:40
-; GFX11-SDAG-NEXT:    scratch_load_b32 v4, off, off offset:36
-; GFX11-SDAG-NEXT:    s_load_b32 s1, s[4:5], 0x30
-; GFX11-SDAG-NEXT:    scratch_store_b128 off, v[7:10], off
-; GFX11-SDAG-NEXT:    s_clause 0x1
-; GFX11-SDAG-NEXT:    scratch_load_b64 v[1:2], off, off offset:8
-; GFX11-SDAG-NEXT:    scratch_load_b32 v0, off, off offset:4
-; GFX11-SDAG-NEXT:    v_mov_b32_e32 v7, s6
-; GFX11-SDAG-NEXT:    v_mov_b32_e32 v9, s0
+; GFX11-SDAG-NEXT:    s_load_b32 s13, s[4:5], 0x30
+; GFX11-SDAG-NEXT:    s_load_b128 s[0:3], s[4:5], 0x20
+; GFX11-SDAG-NEXT:    s_mov_b32 s5, s12
 ; GFX11-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX11-SDAG-NEXT:    v_mov_b32_e32 v3, s1
-; GFX11-SDAG-NEXT:    s_mov_b32 s1, exec_lo
-; GFX11-SDAG-NEXT:    s_waitcnt vmcnt(0)
-; GFX11-SDAG-NEXT:  .LBB1_1: ; =>This Inner Loop Header: Depth=1
-; GFX11-SDAG-NEXT:    v_readfirstlane_b32 s4, v4
-; GFX11-SDAG-NEXT:    v_readfirstlane_b32 s5, v5
-; GFX11-SDAG-NEXT:    v_readfirstlane_b32 s6, v6
-; GFX11-SDAG-NEXT:    v_readfirstlane_b32 s7, v7
-; GFX11-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-SDAG-NEXT:    v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[4:5]
-; GFX11-SDAG-NEXT:    v_cmp_eq_u64_e64 s0, s[6:7], v[6:7]
-; GFX11-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX11-SDAG-NEXT:    s_and_b32 s0, vcc_lo, s0
-; GFX11-SDAG-NEXT:    s_and_saveexec_b32 s0, s0
-; GFX11-SDAG-NEXT:    buffer_load_b32 v8, v9, s[4:7], 0 offen glc dlc
-; GFX11-SDAG-NEXT:    ; implicit-def: $vgpr4_vgpr5_vgpr6_vgpr7
-; GFX11-SDAG-NEXT:    ; implicit-def: $vgpr9
-; GFX11-SDAG-NEXT:    s_xor_b32 exec_lo, exec_lo, s0
-; GFX11-SDAG-NEXT:    s_cbranch_execnz .LBB1_1
-; GFX11-SDAG-NEXT:  ; %bb.2:
-; GFX11-SDAG-NEXT:    s_mov_b32 exec_lo, s1
-; GFX11-SDAG-NEXT:    v_mov_b32_e32 v4, s8
-; GFX11-SDAG-NEXT:    s_mov_b32 s0, exec_lo
-; GFX11-SDAG-NEXT:  .LBB1_3: ; =>This Inner Loop Header: Depth=1
-; GFX11-SDAG-NEXT:    v_readfirstlane_b32 s4, v0
-; GFX11-SDAG-NEXT:    v_readfirstlane_b32 s5, v1
-; GFX11-SDAG-NEXT:    v_readfirstlane_b32 s6, v2
-; GFX11-SDAG-NEXT:    v_readfirstlane_b32 s7, v3
-; GFX11-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-SDAG-NEXT:    v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
-; GFX11-SDAG-NEXT:    v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
-; GFX11-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX11-SDAG-NEXT:    s_and_b32 s0, vcc_lo, s0
-; GFX11-SDAG-NEXT:    s_and_saveexec_b32 s0, s0
+; GFX11-SDAG-NEXT:    v_mov_b32_e32 v1, s0
+; GFX11-SDAG-NEXT:    s_mov_b32 s4, s3
+; GFX11-SDAG-NEXT:    s_mov_b32 s3, s12
+; GFX11-SDAG-NEXT:    s_or_b64 s[6:7], s[4:5], s[12:13]
+; GFX11-SDAG-NEXT:    s_mov_b32 s13, s2
+; GFX11-SDAG-NEXT:    s_mov_b32 s2, s1
+; GFX11-SDAG-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-SDAG-NEXT:    s_or_b64 s[4:5], s[2:3], s[12:13]
 ; GFX11-SDAG-NEXT:    s_waitcnt vmcnt(0)
-; GFX11-SDAG-NEXT:    buffer_store_b32 v8, v4, s[4:7], 0 offen dlc
-; GFX11-SDAG-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
-; GFX11-SDAG-NEXT:    ; implicit-def: $vgpr8
-; GFX11-SDAG-NEXT:    ; implicit-def: $vgpr4
-; GFX11-SDAG-NEXT:    s_xor_b32 exec_lo, exec_lo, s0
-; GFX11-SDAG-NEXT:    s_cbranch_execnz .LBB1_3
-; GFX11-SDAG-NEXT:  ; %bb.4:
+; GFX11-SDAG-NEXT:    buffer_store_b32 v0, v1, s[4:7], 0 offen dlc
 ; GFX11-SDAG-NEXT:    s_endpgm
 ;
 ; GFX11-GISEL-LABEL: buffer_nontemporal_and_volatile_load_store:
@@ -763,78 +594,37 @@ define amdgpu_kernel void @buffer_nontemporal_and_volatile_load_store(ptr addrsp
 ;
 ; GFX12-SDAG-LABEL: buffer_nontemporal_and_volatile_load_store:
 ; GFX12-SDAG:       ; %bb.0: ; %entry
-; GFX12-SDAG-NEXT:    s_clause 0x2
+; GFX12-SDAG-NEXT:    s_clause 0x1
 ; GFX12-SDAG-NEXT:    s_load_b128 s[0:3], s[4:5], 0x0
-; GFX12-SDAG-NEXT:    s_load_b128 s[8:11], s[4:5], 0x20
-; GFX12-SDAG-NEXT:    s_load_b32 s6, s[4:5], 0x10
+; GFX12-SDAG-NEXT:    s_load_b32 s13, s[4:5], 0x10
+; GFX12-SDAG-NEXT:    s_mov_b32 s12, 0
+; GFX12-SDAG-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-SDAG-NEXT:    s_mov_b32 s7, s12
+; GFX12-SDAG-NEXT:    s_mov_b32 s9, s12
 ; GFX12-SDAG-NEXT:    s_wait_kmcnt 0x0
-; GFX12-SDAG-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
-; GFX12-SDAG-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
-; GFX12-SDAG-NEXT:    v_dual_mov_b32 v7, s8 :: v_dual_mov_b32 v8, s9
-; GFX12-SDAG-NEXT:    v_dual_mov_b32 v9, s10 :: v_dual_mov_b32 v10, s11
-; GFX12-SDAG-NEXT:    scratch_store_b128 off, v[0:3], off offset:32
-; GFX12-SDAG-NEXT:    s_clause 0x1
-; GFX12-SDAG-NEXT:    scratch_load_b64 v[5:6], off, off offset:40
-; GFX12-SDAG-NEXT:    scratch_load_b32 v4, off, off offset:36
-; GFX12-SDAG-NEXT:    s_load_b32 s1, s[4:5], 0x30
-; GFX12-SDAG-NEXT:    scratch_store_b128 off, v[7:10], off
+; GFX12-SDAG-NEXT:    s_mov_b32 s6, s3
+; GFX12-SDAG-NEXT:    v_mov_b32_e32 v0, s0
+; GFX12-SDAG-NEXT:    s_mov_b32 s8, s1
+; GFX12-SDAG-NEXT:    s_or_b64 s[10:11], s[6:7], s[12:13]
+; GFX12-SDAG-NEXT:    s_mov_b32 s13, s2
+; GFX12-SDAG-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-SDAG-NEXT:    s_or_b64 s[8:9], s[8:9], s[12:13]
+; GFX12-SDAG-NEXT:    buffer_load_b32 v0, v0, s[8:11], null offen th:TH_LOAD_NT scope:SCOPE_SYS
 ; GFX12-SDAG-NEXT:    s_clause 0x1
-; GFX12-SDAG-NEXT:    scratch_load_b64 v[1:2], off, off offset:8
-; GFX12-SDAG-NEXT:    scratch_load_b32 v0, off, off offset:4
-; GFX12-SDAG-NEXT:    v_mov_b32_e32 v7, s6
-; GFX12-SDAG-NEXT:    v_mov_b32_e32 v9, s0
+; GFX12-SDAG-NEXT:    s_load_b32 s13, s[4:5], 0x30
+; GFX12-SDAG-NEXT:    s_load_b128 s[0:3], s[4:5], 0x20
+; GFX12-SDAG-NEXT:    s_mov_b32 s5, s12
 ; GFX12-SDAG-NEXT:    s_wait_kmcnt 0x0
-; GFX12-SDAG-NEXT:    v_mov_b32_e32 v3, s1
-; GFX12-SDAG-NEXT:    s_mov_b32 s1, exec_lo
-; GFX12-SDAG-NEXT:  .LBB1_1: ; =>This Inner Loop Header: Depth=1
-; GFX12-SDAG-NEXT:    s_wait_loadcnt 0x2
-; GFX12-SDAG-NEXT:    v_readfirstlane_b32 s4, v4
-; GFX12-SDAG-NEXT:    v_readfirstlane_b32 s5, v5
-; GFX12-SDAG-NEXT:    v_readfirstlane_b32 s6, v6
-; GFX12-SDAG-NEXT:    v_readfirstlane_b32 s7, v7
-; GFX12-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-SDAG-NEXT:    v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[4:5]
-; GFX12-SDAG-NEXT:    v_cmp_eq_u64_e64 s0, s[6:7], v[6:7]
-; GFX12-SDAG-NEXT:    s_wait_alu 0xfffe
-; GFX12-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX12-SDAG-NEXT:    s_and_b32 s0, vcc_lo, s0
-; GFX12-SDAG-NEXT:    s_wait_alu 0xfffe
-; GFX12-SDAG-NEXT:    s_and_saveexec_b32 s0, s0
-; GFX12-SDAG-NEXT:    s_wait_loadcnt 0x0
-; GFX12-SDAG-NEXT:    buffer_load_b32 v8, v9, s[4:7], null offen th:TH_LOAD_NT scope:SCOPE_SYS
-; GFX12-SDAG-NEXT:    ; implicit-def: $vgpr4_vgpr5_vgpr6_vgpr7
-; GFX12-SDAG-NEXT:    ; implicit-def: $vgpr9
-; GFX12-SDAG-NEXT:    s_wait_alu 0xfffe
-; GFX12-SDAG-NEXT:    s_xor_b32 exec_lo, exec_lo, s0
-; GFX12-SDAG-NEXT:    s_cbranch_execnz .LBB1_1
-; GFX12-SDAG-NEXT:  ; %bb.2:
-; GFX12-SDAG-NEXT:    s_mov_b32 exec_lo, s1
-; GFX12-SDAG-NEXT:    v_mov_b32_e32 v4, s8
-; GFX12-SDAG-NEXT:    s_mov_b32 s0, exec_lo
-; GFX12-SDAG-NEXT:  .LBB1_3: ; =>This Inner Loop Header: Depth=1
-; GFX12-SDAG-NEXT:    s_wait_loadcnt 0x1
-; GFX12-SDAG-NEXT:    v_readfirstlane_b32 s4, v0
-; GFX12-SDAG-NEXT:    v_readfirstlane_b32 s5, v1
-; GFX12-SDAG-NEXT:    v_readfirstlane_b32 s6, v2
-; GFX12-SDAG-NEXT:    v_readfirstlane_b32 s7, v3
-; GFX12-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-SDAG-NEXT:    v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
-; GFX12-SDAG-NEXT:    v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
-; GFX12-SDAG-NEXT:    s_wait_alu 0xfffe
-; GFX12-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX12-SDAG-NEXT:    s_and_b32 s0, vcc_lo, s0
-; GFX12-SDAG-NEXT:    s_wait_alu 0xfffe
-; GFX12-SDAG-NEXT:    s_and_saveexec_b32 s0, s0
+; GFX12-SDAG-NEXT:    v_mov_b32_e32 v1, s0
+; GFX12-SDAG-NEXT:    s_mov_b32 s4, s3
+; GFX12-SDAG-NEXT:    s_mov_b32 s3, s12
+; GFX12-SDAG-NEXT:    s_or_b64 s[6:7], s[4:5], s[12:13]
+; GFX12-SDAG-NEXT:    s_mov_b32 s13, s2
+; GFX12-SDAG-NEXT:    s_mov_b32 s2, s1
+; GFX12-SDAG-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-SDAG-NEXT:    s_or_b64 s[4:5], s[2:3], s[12:13]
 ; GFX12-SDAG-NEXT:    s_wait_loadcnt 0x0
-; GFX12-SDAG-NEXT:    s_wait_storecnt 0x0
-; GFX12-SDAG-NEXT:    buffer_store_b32 v8, v4, s[4:7], null offen th:TH_STORE_NT scope:SCOPE_SYS
-; GFX12-SDAG-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
-; GFX12-SDAG-NEXT:    ; implicit-def: $vgpr8
-; GFX12-SDAG-NEXT:    ; implicit-def: $vgpr4
-; GFX12-SDAG-NEXT:    s_wait_alu 0xfffe
-; GFX12-SDAG-NEXT:    s_xor_b32 exec_lo, exec_lo, s0
-; GFX12-SDAG-NEXT:    s_cbranch_execnz .LBB1_3
-; GFX12-SDAG-NEXT:  ; %bb.4:
+; GFX12-SDAG-NEXT:    buffer_store_b32 v0, v1, s[4:7], null offen th:TH_STORE_NT scope:SCOPE_SYS
 ; GFX12-SDAG-NEXT:    s_endpgm
 ;
 ; GFX12-GISEL-LABEL: buffer_nontemporal_and_volatile_load_store:
@@ -860,7 +650,6 @@ define amdgpu_kernel void @buffer_nontemporal_and_volatile_load_store(ptr addrsp
 ; GFX12-GISEL-NEXT:    s_load_b32 s7, s[4:5], 0x30
 ; GFX12-GISEL-NEXT:    s_mov_b32 s4, s9
 ; GFX12-GISEL-NEXT:    s_wait_kmcnt 0x0
-; GFX12-GISEL-NEXT:    s_wait_alu 0xfffe
 ; GFX12-GISEL-NEXT:    v_mov_b32_e32 v1, s0
 ; GFX12-GISEL-NEXT:    s_mov_b32 s8, s1
 ; GFX12-GISEL-NEXT:    s_mov_b32 s5, s2
diff --git a/llvm/test/CodeGen/AMDGPU/lower-lds-struct-aa-memcpy.ll b/llvm/test/CodeGen/AMDGPU/lower-lds-struct-aa-memcpy.ll
index 52e64197d6815..4ab05c2923fdb 100644
--- a/llvm/test/CodeGen/AMDGPU/lower-lds-struct-aa-memcpy.ll
+++ b/llvm/test/CodeGen/AMDGPU/lower-lds-struct-aa-memcpy.ll
@@ -33,7 +33,7 @@ define protected amdgpu_kernel void @test(ptr addrspace(1) nocapture %ptr.coerce
 ; GCN-NEXT:    global_store_byte v1, v0, s[0:1]
 ; GCN-NEXT:    s_endpgm
 ; CHECK-LABEL: define protected amdgpu_kernel void @test(
-; CHECK-SAME: ptr addrspace(1) nocapture [[PTR_COERCE:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
+; CHECK-SAME: ptr addrspace(1) captures(none) [[PTR_COERCE:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    store i8 3, ptr addrspace(3) @llvm.amdgcn.kernel.test.lds, align 4, !alias.scope !1, !noalias !4
 ; CHECK-NEXT:    tail call void @llvm.memcpy.p3.p3.i64(ptr addrspace(3) noundef align 1 dereferenceable(3) getelementptr inbounds ([[LLVM_AMDGCN_KERNEL_TEST_LDS_T:%.*]], ptr addrspace(3) @llvm.amdgcn.kernel.test.lds, i32 0, i32 2), ptr addrspace(3) noundef align 1 dereferenceable(3) @llvm.amdgcn.kernel.test.lds, i64 3, i1 false), !alias.scope !6, !noalias !7
diff --git a/llvm/test/CodeGen/AMDGPU/lower-mem-intrinsics.ll b/llvm/test/CodeGen/AMDGPU/lower-mem-intrinsics.ll
index ffe9e06c04ae4..5a9f53ec0077d 100644
--- a/llvm/test/CodeGen/AMDGPU/lower-mem-intrinsics.ll
+++ b/llvm/test/CodeGen/AMDGPU/lower-mem-intrinsics.ll
@@ -330,17 +330,17 @@ define amdgpu_kernel void @memcpy_multi_use_one_function(ptr addrspace(1) %dst0,
 
 define amdgpu_kernel void @memcpy_alt_type(ptr addrspace(1) %dst, ptr addrspace(3) %src, i32 %n) #0 {
 ; OPT-LABEL: @memcpy_alt_type(
-; OPT-NEXT:    [[TMP2:%.*]] = and i32 [[N:%.*]], 7
+; OPT-NEXT:    [[TMP2:%.*]] = and i32 [[N:%.*]], 15
 ; OPT-NEXT:    [[TMP3:%.*]] = sub i32 [[N]], [[TMP2]]
 ; OPT-NEXT:    [[TMP4:%.*]] = icmp ne i32 [[TMP3]], 0
 ; OPT-NEXT:    br i1 [[TMP4]], label [[LOOP_MEMCPY_EXPANSION:%.*]], label [[LOOP_MEMCPY_RESIDUAL_HEADER:%.*]]
 ; OPT:       loop-memcpy-expansion:
 ; OPT-NEXT:    [[LOOP_INDEX:%.*]] = phi i32 [ 0, [[TMP0:%.*]] ], [ [[TMP8:%.*]], [[LOOP_MEMCPY_EXPANSION]] ]
 ; OPT-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i8, ptr addrspace(3) [[SRC:%.*]], i32 [[LOOP_INDEX]]
-; OPT-NEXT:    [[TMP6:%.*]] = load <2 x i32>, ptr addrspace(3) [[TMP5]], align 1
+; OPT-NEXT:    [[TMP6:%.*]] = load <4 x i32>, ptr addrspace(3) [[TMP5]], align 1
 ; OPT-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[DST:%.*]], i32 [[LOOP_INDEX]]
-; OPT-NEXT:    store <2 x i32> [[TMP6]], ptr addrspace(1) [[TMP7]], align 1
-; OPT-NEXT:    [[TMP8]] = add i32 [[LOOP_INDEX]], 8
+; OPT-NEXT:    store <4 x i32> [[TMP6]], ptr addrspace(1) [[TMP7]], align 1
+; OPT-NEXT:    [[TMP8]] = add i32 [[LOOP_INDEX]], 16
 ; OPT-NEXT:    [[TMP9:%.*]] = icmp ult i32 [[TMP8]], [[TMP3]]
 ; OPT-NEXT:    br i1 [[TMP9]], label [[LOOP_MEMCPY_EXPANSION]], label [[LOOP_MEMCPY_RESIDUAL_HEADER]]
 ; OPT:       loop-memcpy-residual:
@@ -681,13 +681,25 @@ define amdgpu_kernel void @memcpy_global_align2_global_align2_1039(ptr addrspace
 ; OPT:       load-store-loop:
 ; OPT-NEXT:    [[LOOP_INDEX:%.*]] = phi i64 [ 0, [[TMP0:%.*]] ], [ [[TMP4:%.*]], [[LOAD_STORE_LOOP]] ]
 ; OPT-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[SRC:%.*]], i64 [[LOOP_INDEX]]
-; OPT-NEXT:    [[TMP2:%.*]] = load i16, ptr addrspace(1) [[TMP1]], align 2
+; OPT-NEXT:    [[TMP2:%.*]] = load <64 x i32>, ptr addrspace(1) [[TMP1]], align 2
 ; OPT-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[DST:%.*]], i64 [[LOOP_INDEX]]
-; OPT-NEXT:    store i16 [[TMP2]], ptr addrspace(1) [[TMP3]], align 2
-; OPT-NEXT:    [[TMP4]] = add i64 [[LOOP_INDEX]], 2
-; OPT-NEXT:    [[TMP5:%.*]] = icmp ult i64 [[TMP4]], 1038
+; OPT-NEXT:    store <64 x i32> [[TMP2]], ptr addrspace(1) [[TMP3]], align 2
+; OPT-NEXT:    [[TMP4]] = add i64 [[LOOP_INDEX]], 256
+; OPT-NEXT:    [[TMP5:%.*]] = icmp ult i64 [[TMP4]], 1024
 ; OPT-NEXT:    br i1 [[TMP5]], label [[LOAD_STORE_LOOP]], label [[MEMCPY_SPLIT:%.*]]
 ; OPT:       memcpy-split:
+; OPT-NEXT:    [[TMP15:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[SRC]], i64 1024
+; OPT-NEXT:    [[TMP16:%.*]] = load i64, ptr addrspace(1) [[TMP15]], align 2
+; OPT-NEXT:    [[TMP17:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[DST]], i64 1024
+; OPT-NEXT:    store i64 [[TMP16]], ptr addrspace(1) [[TMP17]], align 2
+; OPT-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[SRC]], i64 1032
+; OPT-NEXT:    [[TMP10:%.*]] = load i32, ptr addrspace(1) [[TMP9]], align 2
+; OPT-NEXT:    [[TMP11:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[DST]], i64 1032
+; OPT-NEXT:    store i32 [[TMP10]], ptr addrspace(1) [[TMP11]], align 2
+; OPT-NEXT:    [[TMP12:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[SRC]], i64 1036
+; OPT-NEXT:    [[TMP13:%.*]] = load i16, ptr addrspace(1) [[TMP12]], align 2
+; OPT-NEXT:    [[TMP14:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[DST]], i64 1036
+; OPT-NEXT:    store i16 [[TMP13]], ptr addrspace(1) [[TMP14]], align 2
 ; OPT-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[SRC]], i64 1038
 ; OPT-NEXT:    [[TMP7:%.*]] = load i8, ptr addrspace(1) [[TMP6]], align 2
 ; OPT-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[DST]], i64 1038
@@ -731,13 +743,17 @@ define amdgpu_kernel void @memcpy_global_align2_global_align4_1027(ptr addrspace
 ; OPT:       load-store-loop:
 ; OPT-NEXT:    [[LOOP_INDEX:%.*]] = phi i64 [ 0, [[TMP0:%.*]] ], [ [[TMP4:%.*]], [[LOAD_STORE_LOOP]] ]
 ; OPT-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[SRC:%.*]], i64 [[LOOP_INDEX]]
-; OPT-NEXT:    [[TMP2:%.*]] = load i16, ptr addrspace(1) [[TMP1]], align 2
+; OPT-NEXT:    [[TMP2:%.*]] = load <64 x i32>, ptr addrspace(1) [[TMP1]], align 4
 ; OPT-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[DST:%.*]], i64 [[LOOP_INDEX]]
-; OPT-NEXT:    store i16 [[TMP2]], ptr addrspace(1) [[TMP3]], align 2
-; OPT-NEXT:    [[TMP4]] = add i64 [[LOOP_INDEX]], 2
-; OPT-NEXT:    [[TMP5:%.*]] = icmp ult i64 [[TMP4]], 1026
+; OPT-NEXT:    store <64 x i32> [[TMP2]], ptr addrspace(1) [[TMP3]], align 2
+; OPT-NEXT:    [[TMP4]] = add i64 [[LOOP_INDEX]], 256
+; OPT-NEXT:    [[TMP5:%.*]] = icmp ult i64 [[TMP4]], 1024
 ; OPT-NEXT:    br i1 [[TMP5]], label [[LOAD_STORE_LOOP]], label [[MEMCPY_SPLIT:%.*]]
 ; OPT:       memcpy-split:
+; OPT-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[SRC]], i64 1024
+; OPT-NEXT:    [[TMP10:%.*]] = load i16, ptr addrspace(1) [[TMP9]], align 4
+; OPT-NEXT:    [[TMP11:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[DST]], i64 1024
+; OPT-NEXT:    store i16 [[TMP10]], ptr addrspace(1) [[TMP11]], align 2
 ; OPT-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[SRC]], i64 1026
 ; OPT-NEXT:    [[TMP7:%.*]] = load i8, ptr addrspace(1) [[TMP6]], align 2
 ; OPT-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[DST]], i64 1026
@@ -754,13 +770,17 @@ define amdgpu_kernel void @memcpy_global_align4_global_align2_1027(ptr addrspace
 ; OPT:       load-store-loop:
 ; OPT-NEXT:    [[LOOP_INDEX:%.*]] = phi i64 [ 0, [[TMP0:%.*]] ], [ [[TMP4:%.*]], [[LOAD_STORE_LOOP]] ]
 ; OPT-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[SRC:%.*]], i64 [[LOOP_INDEX]]
-; OPT-NEXT:    [[TMP2:%.*]] = load i16, ptr addrspace(1) [[TMP1]], align 2
+; OPT-NEXT:    [[TMP2:%.*]] = load <64 x i32>, ptr addrspace(1) [[TMP1]], align 2
 ; OPT-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[DST:%.*]], i64 [[LOOP_INDEX]]
-; OPT-NEXT:    store i16 [[TMP2]], ptr addrspace(1) [[TMP3]], align 2
-; OPT-NEXT:    [[TMP4]] = add i64 [[LOOP_INDEX]], 2
-; OPT-NEXT:    [[TMP5:%.*]] = icmp ult i64 [[TMP4]], 1026
+; OPT-NEXT:    store <64 x i32> [[TMP2]], ptr addrspace(1) [[TMP3]], align 4
+; OPT-NEXT:    [[TMP4]] = add i64 [[LOOP_INDEX]], 256
+; OPT-NEXT:    [[TMP5:%.*]] = icmp ult i64 [[TMP4]], 1024
 ; OPT-NEXT:    br i1 [[TMP5]], label [[LOAD_STORE_LOOP]], label [[MEMCPY_SPLIT:%.*]]
 ; OPT:       memcpy-split:
+; OPT-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[SRC]], i64 1024
+; OPT-NEXT:    [[TMP10:%.*]] = load i16, ptr addrspace(1) [[TMP9]], align 2
+; OPT-NEXT:    [[TMP11:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[DST]], i64 1024
+; OPT-NEXT:    store i16 [[TMP10]], ptr addrspace(1) [[TMP11]], align 4
 ; OPT-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[SRC]], i64 1026
 ; OPT-NEXT:    [[TMP7:%.*]] = load i8, ptr addrspace(1) [[TMP6]], align 2
 ; OPT-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[DST]], i64 1026
@@ -804,13 +824,17 @@ define amdgpu_kernel void @memcpy_private_align2_private_align4_1027(ptr addrspa
 ; OPT:       load-store-loop:
 ; OPT-NEXT:    [[LOOP_INDEX:%.*]] = phi i32 [ 0, [[TMP0:%.*]] ], [ [[TMP4:%.*]], [[LOAD_STORE_LOOP]] ]
 ; OPT-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i8, ptr addrspace(5) [[SRC:%.*]], i32 [[LOOP_INDEX]]
-; OPT-NEXT:    [[TMP2:%.*]] = load i16, ptr addrspace(5) [[TMP1]], align 2
+; OPT-NEXT:    [[TMP2:%.*]] = load <64 x i32>, ptr addrspace(5) [[TMP1]], align 4
 ; OPT-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i8, ptr addrspace(5) [[DST:%.*]], i32 [[LOOP_INDEX]]
-; OPT-NEXT:    store i16 [[TMP2]], ptr addrspace(5) [[TMP3]], align 2
-; OPT-NEXT:    [[TMP4]] = add i32 [[LOOP_INDEX]], 2
-; OPT-NEXT:    [[TMP5:%.*]] = icmp ult i32 [[TMP4]], 1026
+; OPT-NEXT:    store <64 x i32> [[TMP2]], ptr addrspace(5) [[TMP3]], align 2
+; OPT-NEXT:    [[TMP4]] = add i32 [[LOOP_INDEX]], 256
+; OPT-NEXT:    [[TMP5:%.*]] = icmp ult i32 [[TMP4]], 1024
 ; OPT-NEXT:    br i1 [[TMP5]], label [[LOAD_STORE_LOOP]], label [[MEMCPY_SPLIT:%.*]]
 ; OPT:       memcpy-split:
+; OPT-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i8, ptr addrspace(5) [[SRC]], i32 1024
+; OPT-NEXT:    [[TMP10:%.*]] = load i16, ptr addrspace(5) [[TMP9]], align 4
+; OPT-NEXT:    [[TMP11:%.*]] = getelementptr inbounds i8, ptr addrspace(5) [[DST]], i32 1024
+; OPT-NEXT:    store i16 [[TMP10]], ptr addrspace(5) [[TMP11]], align 2
 ; OPT-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i8, ptr addrspace(5) [[SRC]], i32 1026
 ; OPT-NEXT:    [[TMP7:%.*]] = load i8, ptr addrspace(5) [[TMP6]], align 2
 ; OPT-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i8, ptr addrspace(5) [[DST]], i32 1026
@@ -854,13 +878,17 @@ define amdgpu_kernel void @memcpy_private_align4_private_align2_1027(ptr addrspa
 ; OPT:       load-store-loop:
 ; OPT-NEXT:    [[LOOP_INDEX:%.*]] = phi i32 [ 0, [[TMP0:%.*]] ], [ [[TMP4:%.*]], [[LOAD_STORE_LOOP]] ]
 ; OPT-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i8, ptr addrspace(5) [[SRC:%.*]], i32 [[LOOP_INDEX]]
-; OPT-NEXT:    [[TMP2:%.*]] = load i16, ptr addrspace(5) [[TMP1]], align 2
+; OPT-NEXT:    [[TMP2:%.*]] = load <64 x i32>, ptr addrspace(5) [[TMP1]], align 2
 ; OPT-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i8, ptr addrspace(5) [[DST:%.*]], i32 [[LOOP_INDEX]]
-; OPT-NEXT:    store i16 [[TMP2]], ptr addrspace(5) [[TMP3]], align 2
-; OPT-NEXT:    [[TMP4]] = add i32 [[LOOP_INDEX]], 2
-; OPT-NEXT:    [[TMP5:%.*]] = icmp ult i32 [[TMP4]], 1026
+; OPT-NEXT:    store <64 x i32> [[TMP2]], ptr addrspace(5) [[TMP3]], align 4
+; OPT-NEXT:    [[TMP4]] = add i32 [[LOOP_INDEX]], 256
+; OPT-NEXT:    [[TMP5:%.*]] = icmp ult i32 [[TMP4]], 1024
 ; OPT-NEXT:    br i1 [[TMP5]], label [[LOAD_STORE_LOOP]], label [[MEMCPY_SPLIT:%.*]]
 ; OPT:       memcpy-split:
+; OPT-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i8, ptr addrspace(5) [[SRC]], i32 1024
+; OPT-NEXT:    [[TMP10:%.*]] = load i16, ptr addrspace(5) [[TMP9]], align 2
+; OPT-NEXT:    [[TMP11:%.*]] = getelementptr inbounds i8, ptr addrspace(5) [[DST]], i32 1024
+; OPT-NEXT:    store i16 [[TMP10]], ptr addrspace(5) [[TMP11]], align 4
 ; OPT-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i8, ptr addrspace(5) [[SRC]], i32 1026
 ; OPT-NEXT:    [[TMP7:%.*]] = load i8, ptr addrspace(5) [[TMP6]], align 2
 ; OPT-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i8, ptr addrspace(5) [[DST]], i32 1026
@@ -904,13 +932,17 @@ define amdgpu_kernel void @memcpy_private_align2_private_align2_1027(ptr addrspa
 ; OPT:       load-store-loop:
 ; OPT-NEXT:    [[LOOP_INDEX:%.*]] = phi i32 [ 0, [[TMP0:%.*]] ], [ [[TMP4:%.*]], [[LOAD_STORE_LOOP]] ]
 ; OPT-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i8, ptr addrspace(5) [[SRC:%.*]], i32 [[LOOP_INDEX]]
-; OPT-NEXT:    [[TMP2:%.*]] = load i16, ptr addrspace(5) [[TMP1]], align 2
+; OPT-NEXT:    [[TMP2:%.*]] = load <64 x i32>, ptr addrspace(5) [[TMP1]], align 2
 ; OPT-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i8, ptr addrspace(5) [[DST:%.*]], i32 [[LOOP_INDEX]]
-; OPT-NEXT:    store i16 [[TMP2]], ptr addrspace(5) [[TMP3]], align 2
-; OPT-NEXT:    [[TMP4]] = add i32 [[LOOP_INDEX]], 2
-; OPT-NEXT:    [[TMP5:%.*]] = icmp ult i32 [[TMP4]], 1026
+; OPT-NEXT:    store <64 x i32> [[TMP2]], ptr addrspace(5) [[TMP3]], align 2
+; OPT-NEXT:    [[TMP4]] = add i32 [[LOOP_INDEX]], 256
+; OPT-NEXT:    [[TMP5:%.*]] = icmp ult i32 [[TMP4]], 1024
 ; OPT-NEXT:    br i1 [[TMP5]], label [[LOAD_STORE_LOOP]], label [[MEMCPY_SPLIT:%.*]]
 ; OPT:       memcpy-split:
+; OPT-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i8, ptr addrspace(5) [[SRC]], i32 1024
+; OPT-NEXT:    [[TMP10:%.*]] = load i16, ptr addrspace(5) [[TMP9]], align 2
+; OPT-NEXT:    [[TMP11:%.*]] = getelementptr inbounds i8, ptr addrspace(5) [[DST]], i32 1024
+; OPT-NEXT:    store i16 [[TMP10]], ptr addrspace(5) [[TMP11]], align 2
 ; OPT-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i8, ptr addrspace(5) [[SRC]], i32 1026
 ; OPT-NEXT:    [[TMP7:%.*]] = load i8, ptr addrspace(5) [[TMP6]], align 2
 ; OPT-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i8, ptr addrspace(5) [[DST]], i32 1026
@@ -958,17 +990,17 @@ define amdgpu_kernel void @memcpy_global_align4_global_align4_variable(ptr addrs
 
 define amdgpu_kernel void @memcpy_global_align2_global_align2_variable(ptr addrspace(1) %dst, ptr addrspace(1) %src, i64 %n) #0 {
 ; OPT-LABEL: @memcpy_global_align2_global_align2_variable(
-; OPT-NEXT:    [[TMP2:%.*]] = and i64 [[N:%.*]], 1
+; OPT-NEXT:    [[TMP2:%.*]] = and i64 [[N:%.*]], 15
 ; OPT-NEXT:    [[TMP3:%.*]] = sub i64 [[N]], [[TMP2]]
 ; OPT-NEXT:    [[TMP4:%.*]] = icmp ne i64 [[TMP3]], 0
 ; OPT-NEXT:    br i1 [[TMP4]], label [[LOOP_MEMCPY_EXPANSION:%.*]], label [[LOOP_MEMCPY_RESIDUAL_HEADER:%.*]]
 ; OPT:       loop-memcpy-expansion:
 ; OPT-NEXT:    [[LOOP_INDEX:%.*]] = phi i64 [ 0, [[TMP0:%.*]] ], [ [[TMP8:%.*]], [[LOOP_MEMCPY_EXPANSION]] ]
 ; OPT-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[SRC:%.*]], i64 [[LOOP_INDEX]]
-; OPT-NEXT:    [[TMP6:%.*]] = load i16, ptr addrspace(1) [[TMP5]], align 2
+; OPT-NEXT:    [[TMP6:%.*]] = load <4 x i32>, ptr addrspace(1) [[TMP5]], align 2
 ; OPT-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[DST:%.*]], i64 [[LOOP_INDEX]]
-; OPT-NEXT:    store i16 [[TMP6]], ptr addrspace(1) [[TMP7]], align 2
-; OPT-NEXT:    [[TMP8]] = add i64 [[LOOP_INDEX]], 2
+; OPT-NEXT:    store <4 x i32> [[TMP6]], ptr addrspace(1) [[TMP7]], align 2
+; OPT-NEXT:    [[TMP8]] = add i64 [[LOOP_INDEX]], 16
 ; OPT-NEXT:    [[TMP9:%.*]] = icmp ult i64 [[TMP8]], [[TMP3]]
 ; OPT-NEXT:    br i1 [[TMP9]], label [[LOOP_MEMCPY_EXPANSION]], label [[LOOP_MEMCPY_RESIDUAL_HEADER]]
 ; OPT:       loop-memcpy-residual:
@@ -1028,17 +1060,17 @@ define amdgpu_kernel void @memcpy_global_align1_global_align1_variable(ptr addrs
 
 define amdgpu_kernel void @memcpy_local_align4_local_align4_variable(ptr addrspace(3) %dst, ptr addrspace(3) %src, i32 %n) #0 {
 ; OPT-LABEL: @memcpy_local_align4_local_align4_variable(
-; OPT-NEXT:    [[TMP2:%.*]] = and i32 [[N:%.*]], 7
+; OPT-NEXT:    [[TMP2:%.*]] = and i32 [[N:%.*]], 15
 ; OPT-NEXT:    [[TMP3:%.*]] = sub i32 [[N]], [[TMP2]]
 ; OPT-NEXT:    [[TMP4:%.*]] = icmp ne i32 [[TMP3]], 0
 ; OPT-NEXT:    br i1 [[TMP4]], label [[LOOP_MEMCPY_EXPANSION:%.*]], label [[LOOP_MEMCPY_RESIDUAL_HEADER:%.*]]
 ; OPT:       loop-memcpy-expansion:
 ; OPT-NEXT:    [[LOOP_INDEX:%.*]] = phi i32 [ 0, [[TMP0:%.*]] ], [ [[TMP8:%.*]], [[LOOP_MEMCPY_EXPANSION]] ]
 ; OPT-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i8, ptr addrspace(3) [[SRC:%.*]], i32 [[LOOP_INDEX]]
-; OPT-NEXT:    [[TMP6:%.*]] = load <2 x i32>, ptr addrspace(3) [[TMP5]], align 4
+; OPT-NEXT:    [[TMP6:%.*]] = load <4 x i32>, ptr addrspace(3) [[TMP5]], align 4
 ; OPT-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i8, ptr addrspace(3) [[DST:%.*]], i32 [[LOOP_INDEX]]
-; OPT-NEXT:    store <2 x i32> [[TMP6]], ptr addrspace(3) [[TMP7]], align 4
-; OPT-NEXT:    [[TMP8]] = add i32 [[LOOP_INDEX]], 8
+; OPT-NEXT:    store <4 x i32> [[TMP6]], ptr addrspace(3) [[TMP7]], align 4
+; OPT-NEXT:    [[TMP8]] = add i32 [[LOOP_INDEX]], 16
 ; OPT-NEXT:    [[TMP9:%.*]] = icmp ult i32 [[TMP8]], [[TMP3]]
 ; OPT-NEXT:    br i1 [[TMP9]], label [[LOOP_MEMCPY_EXPANSION]], label [[LOOP_MEMCPY_RESIDUAL_HEADER]]
 ; OPT:       loop-memcpy-residual:
@@ -1063,17 +1095,17 @@ define amdgpu_kernel void @memcpy_local_align4_local_align4_variable(ptr addrspa
 
 define amdgpu_kernel void @memcpy_local_align2_local_align2_variable(ptr addrspace(3) %dst, ptr addrspace(3) %src, i32 %n) #0 {
 ; OPT-LABEL: @memcpy_local_align2_local_align2_variable(
-; OPT-NEXT:    [[TMP2:%.*]] = and i32 [[N:%.*]], 1
+; OPT-NEXT:    [[TMP2:%.*]] = and i32 [[N:%.*]], 15
 ; OPT-NEXT:    [[TMP3:%.*]] = sub i32 [[N]], [[TMP2]]
 ; OPT-NEXT:    [[TMP4:%.*]] = icmp ne i32 [[TMP3]], 0
 ; OPT-NEXT:    br i1 [[TMP4]], label [[LOOP_MEMCPY_EXPANSION:%.*]], label [[LOOP_MEMCPY_RESIDUAL_HEADER:%.*]]
 ; OPT:       loop-memcpy-expansion:
 ; OPT-NEXT:    [[LOOP_INDEX:%.*]] = phi i32 [ 0, [[TMP0:%.*]] ], [ [[TMP8:%.*]], [[LOOP_MEMCPY_EXPANSION]] ]
 ; OPT-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i8, ptr addrspace(3) [[SRC:%.*]], i32 [[LOOP_INDEX]]
-; OPT-NEXT:    [[TMP6:%.*]] = load i16, ptr addrspace(3) [[TMP5]], align 2
+; OPT-NEXT:    [[TMP6:%.*]] = load <4 x i32>, ptr addrspace(3) [[TMP5]], align 2
 ; OPT-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i8, ptr addrspace(3) [[DST:%.*]], i32 [[LOOP_INDEX]]
-; OPT-NEXT:    store i16 [[TMP6]], ptr addrspace(3) [[TMP7]], align 2
-; OPT-NEXT:    [[TMP8]] = add i32 [[LOOP_INDEX]], 2
+; OPT-NEXT:    store <4 x i32> [[TMP6]], ptr addrspace(3) [[TMP7]], align 2
+; OPT-NEXT:    [[TMP8]] = add i32 [[LOOP_INDEX]], 16
 ; OPT-NEXT:    [[TMP9:%.*]] = icmp ult i32 [[TMP8]], [[TMP3]]
 ; OPT-NEXT:    br i1 [[TMP9]], label [[LOOP_MEMCPY_EXPANSION]], label [[LOOP_MEMCPY_RESIDUAL_HEADER]]
 ; OPT:       loop-memcpy-residual:
@@ -1098,17 +1130,17 @@ define amdgpu_kernel void @memcpy_local_align2_local_align2_variable(ptr addrspa
 
 define amdgpu_kernel void @memcpy_local_align1_local_align1_variable(ptr addrspace(3) %dst, ptr addrspace(3) %src, i32 %n) #0 {
 ; OPT-LABEL: @memcpy_local_align1_local_align1_variable(
-; OPT-NEXT:    [[TMP2:%.*]] = and i32 [[N:%.*]], 7
+; OPT-NEXT:    [[TMP2:%.*]] = and i32 [[N:%.*]], 15
 ; OPT-NEXT:    [[TMP3:%.*]] = sub i32 [[N]], [[TMP2]]
 ; OPT-NEXT:    [[TMP4:%.*]] = icmp ne i32 [[TMP3]], 0
 ; OPT-NEXT:    br i1 [[TMP4]], label [[LOOP_MEMCPY_EXPANSION:%.*]], label [[LOOP_MEMCPY_RESIDUAL_HEADER:%.*]]
 ; OPT:       loop-memcpy-expansion:
 ; OPT-NEXT:    [[LOOP_INDEX:%.*]] = phi i32 [ 0, [[TMP0:%.*]] ], [ [[TMP8:%.*]], [[LOOP_MEMCPY_EXPANSION]] ]
 ; OPT-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i8, ptr addrspace(3) [[SRC:%.*]], i32 [[LOOP_INDEX]]
-; OPT-NEXT:    [[TMP6:%.*]] = load <2 x i32>, ptr addrspace(3) [[TMP5]], align 1
+; OPT-NEXT:    [[TMP6:%.*]] = load <4 x i32>, ptr addrspace(3) [[TMP5]], align 1
 ; OPT-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i8, ptr addrspace(3) [[DST:%.*]], i32 [[LOOP_INDEX]]
-; OPT-NEXT:    store <2 x i32> [[TMP6]], ptr addrspace(3) [[TMP7]], align 1
-; OPT-NEXT:    [[TMP8]] = add i32 [[LOOP_INDEX]], 8
+; OPT-NEXT:    store <4 x i32> [[TMP6]], ptr addrspace(3) [[TMP7]], align 1
+; OPT-NEXT:    [[TMP8]] = add i32 [[LOOP_INDEX]], 16
 ; OPT-NEXT:    [[TMP9:%.*]] = icmp ult i32 [[TMP8]], [[TMP3]]
 ; OPT-NEXT:    br i1 [[TMP9]], label [[LOOP_MEMCPY_EXPANSION]], label [[LOOP_MEMCPY_RESIDUAL_HEADER]]
 ; OPT:       loop-memcpy-residual:
@@ -1133,17 +1165,17 @@ define amdgpu_kernel void @memcpy_local_align1_local_align1_variable(ptr addrspa
 
 define amdgpu_kernel void @memcpy_local_align4_global_align4_variable(ptr addrspace(3) %dst, ptr addrspace(1) %src, i32 %n) #0 {
 ; OPT-LABEL: @memcpy_local_align4_global_align4_variable(
-; OPT-NEXT:    [[TMP2:%.*]] = and i32 [[N:%.*]], 7
+; OPT-NEXT:    [[TMP2:%.*]] = and i32 [[N:%.*]], 15
 ; OPT-NEXT:    [[TMP3:%.*]] = sub i32 [[N]], [[TMP2]]
 ; OPT-NEXT:    [[TMP4:%.*]] = icmp ne i32 [[TMP3]], 0
 ; OPT-NEXT:    br i1 [[TMP4]], label [[LOOP_MEMCPY_EXPANSION:%.*]], label [[LOOP_MEMCPY_RESIDUAL_HEADER:%.*]]
 ; OPT:       loop-memcpy-expansion:
 ; OPT-NEXT:    [[LOOP_INDEX:%.*]] = phi i32 [ 0, [[TMP0:%.*]] ], [ [[TMP8:%.*]], [[LOOP_MEMCPY_EXPANSION]] ]
 ; OPT-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[SRC:%.*]], i32 [[LOOP_INDEX]]
-; OPT-NEXT:    [[TMP6:%.*]] = load <2 x i32>, ptr addrspace(1) [[TMP5]], align 4
+; OPT-NEXT:    [[TMP6:%.*]] = load <4 x i32>, ptr addrspace(1) [[TMP5]], align 4
 ; OPT-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i8, ptr addrspace(3) [[DST:%.*]], i32 [[LOOP_INDEX]]
-; OPT-NEXT:    store <2 x i32> [[TMP6]], ptr addrspace(3) [[TMP7]], align 4
-; OPT-NEXT:    [[TMP8]] = add i32 [[LOOP_INDEX]], 8
+; OPT-NEXT:    store <4 x i32> [[TMP6]], ptr addrspace(3) [[TMP7]], align 4
+; OPT-NEXT:    [[TMP8]] = add i32 [[LOOP_INDEX]], 16
 ; OPT-NEXT:    [[TMP9:%.*]] = icmp ult i32 [[TMP8]], [[TMP3]]
 ; OPT-NEXT:    br i1 [[TMP9]], label [[LOOP_MEMCPY_EXPANSION]], label [[LOOP_MEMCPY_RESIDUAL_HEADER]]
 ; OPT:       loop-memcpy-residual:
@@ -1168,17 +1200,17 @@ define amdgpu_kernel void @memcpy_local_align4_global_align4_variable(ptr addrsp
 
 define amdgpu_kernel void @memcpy_global_align4_local_align4_variable(ptr addrspace(1) %dst, ptr addrspace(3) %src, i32 %n) #0 {
 ; OPT-LABEL: @memcpy_global_align4_local_align4_variable(
-; OPT-NEXT:    [[TMP2:%.*]] = and i32 [[N:%.*]], 7
+; OPT-NEXT:    [[TMP2:%.*]] = and i32 [[N:%.*]], 15
 ; OPT-NEXT:    [[TMP3:%.*]] = sub i32 [[N]], [[TMP2]]
 ; OPT-NEXT:    [[TMP4:%.*]] = icmp ne i32 [[TMP3]], 0
 ; OPT-NEXT:    br i1 [[TMP4]], label [[LOOP_MEMCPY_EXPANSION:%.*]], label [[LOOP_MEMCPY_RESIDUAL_HEADER:%.*]]
 ; OPT:       loop-memcpy-expansion:
 ; OPT-NEXT:    [[LOOP_INDEX:%.*]] = phi i32 [ 0, [[TMP0:%.*]] ], [ [[TMP8:%.*]], [[LOOP_MEMCPY_EXPANSION]] ]
 ; OPT-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i8, ptr addrspace(3) [[SRC:%.*]], i32 [[LOOP_INDEX]]
-; OPT-NEXT:    [[TMP6:%.*]] = load <2 x i32>, ptr addrspace(3) [[TMP5]], align 4
+; OPT-NEXT:    [[TMP6:%.*]] = load <4 x i32>, ptr addrspace(3) [[TMP5]], align 4
 ; OPT-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[DST:%.*]], i32 [[LOOP_INDEX]]
-; OPT-NEXT:    store <2 x i32> [[TMP6]], ptr addrspace(1) [[TMP7]], align 4
-; OPT-NEXT:    [[TMP8]] = add i32 [[LOOP_INDEX]], 8
+; OPT-NEXT:    store <4 x i32> [[TMP6]], ptr addrspace(1) [[TMP7]], align 4
+; OPT-NEXT:    [[TMP8]] = add i32 [[LOOP_INDEX]], 16
 ; OPT-NEXT:    [[TMP9:%.*]] = icmp ult i32 [[TMP8]], [[TMP3]]
 ; OPT-NEXT:    br i1 [[TMP9]], label [[LOOP_MEMCPY_EXPANSION]], label [[LOOP_MEMCPY_RESIDUAL_HEADER]]
 ; OPT:       loop-memcpy-residual:
@@ -1693,10 +1725,10 @@ define amdgpu_kernel void @memmove_local_align1_private_align1(ptr addrspace(3)
 ; ALL:       load-store-loop:
 ; ALL-NEXT:    [[LOOP_INDEX:%.*]] = phi i32 [ 0, [[TMP0:%.*]] ], [ [[TMP4:%.*]], [[LOAD_STORE_LOOP]] ]
 ; ALL-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i8, ptr addrspace(5) [[SRC:%.*]], i32 [[LOOP_INDEX]]
-; ALL-NEXT:    [[TMP2:%.*]] = load <2 x i32>, ptr addrspace(5) [[TMP1]], align 1, !alias.scope [[META6:![0-9]+]]
+; ALL-NEXT:    [[TMP2:%.*]] = load <64 x i32>, ptr addrspace(5) [[TMP1]], align 1, !alias.scope [[META6:![0-9]+]]
 ; ALL-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i8, ptr addrspace(3) [[DST:%.*]], i32 [[LOOP_INDEX]]
-; ALL-NEXT:    store <2 x i32> [[TMP2]], ptr addrspace(3) [[TMP3]], align 1, !noalias [[META6]]
-; ALL-NEXT:    [[TMP4]] = add i32 [[LOOP_INDEX]], 8
+; ALL-NEXT:    store <64 x i32> [[TMP2]], ptr addrspace(3) [[TMP3]], align 1, !noalias [[META6]]
+; ALL-NEXT:    [[TMP4]] = add i32 [[LOOP_INDEX]], 256
 ; ALL-NEXT:    [[TMP5:%.*]] = icmp ult i32 [[TMP4]], 256
 ; ALL-NEXT:    br i1 [[TMP5]], label [[LOAD_STORE_LOOP]], label [[MEMCPY_SPLIT:%.*]]
 ; ALL:       memcpy-split:
@@ -1708,17 +1740,17 @@ define amdgpu_kernel void @memmove_local_align1_private_align1(ptr addrspace(3)
 
 define amdgpu_kernel void @memmove_local_align1_private_align1_unknown_size(ptr addrspace(3) %dst, ptr addrspace(5) %src, i32 %size) {
 ; MAX1024-LABEL: @memmove_local_align1_private_align1_unknown_size(
-; MAX1024-NEXT:    [[TMP2:%.*]] = and i32 [[SIZE:%.*]], 7
+; MAX1024-NEXT:    [[TMP2:%.*]] = and i32 [[SIZE:%.*]], 15
 ; MAX1024-NEXT:    [[TMP3:%.*]] = sub i32 [[SIZE]], [[TMP2]]
 ; MAX1024-NEXT:    [[TMP4:%.*]] = icmp ne i32 [[TMP3]], 0
 ; MAX1024-NEXT:    br i1 [[TMP4]], label [[LOOP_MEMCPY_EXPANSION:%.*]], label [[LOOP_MEMCPY_RESIDUAL_HEADER:%.*]]
 ; MAX1024:       loop-memcpy-expansion:
 ; MAX1024-NEXT:    [[LOOP_INDEX:%.*]] = phi i32 [ 0, [[TMP0:%.*]] ], [ [[TMP8:%.*]], [[LOOP_MEMCPY_EXPANSION]] ]
 ; MAX1024-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i8, ptr addrspace(5) [[SRC:%.*]], i32 [[LOOP_INDEX]]
-; MAX1024-NEXT:    [[TMP5:%.*]] = load <2 x i32>, ptr addrspace(5) [[TMP7]], align 1, !alias.scope [[META0:![0-9]+]]
+; MAX1024-NEXT:    [[TMP5:%.*]] = load <4 x i32>, ptr addrspace(5) [[TMP7]], align 1, !alias.scope [[META0:![0-9]+]]
 ; MAX1024-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i8, ptr addrspace(3) [[DST:%.*]], i32 [[LOOP_INDEX]]
-; MAX1024-NEXT:    store <2 x i32> [[TMP5]], ptr addrspace(3) [[TMP6]], align 1, !noalias [[META0]]
-; MAX1024-NEXT:    [[TMP8]] = add i32 [[LOOP_INDEX]], 8
+; MAX1024-NEXT:    store <4 x i32> [[TMP5]], ptr addrspace(3) [[TMP6]], align 1, !noalias [[META0]]
+; MAX1024-NEXT:    [[TMP8]] = add i32 [[LOOP_INDEX]], 16
 ; MAX1024-NEXT:    [[TMP9:%.*]] = icmp ult i32 [[TMP8]], [[TMP3]]
 ; MAX1024-NEXT:    br i1 [[TMP9]], label [[LOOP_MEMCPY_EXPANSION]], label [[LOOP_MEMCPY_RESIDUAL_HEADER]]
 ; MAX1024:       loop-memcpy-residual:
@@ -1738,17 +1770,17 @@ define amdgpu_kernel void @memmove_local_align1_private_align1_unknown_size(ptr
 ; MAX1024-NEXT:    br i1 [[TMP16]], label [[LOOP_MEMCPY_RESIDUAL]], label [[POST_LOOP_MEMCPY_EXPANSION]]
 ;
 ; ALL-LABEL: @memmove_local_align1_private_align1_unknown_size(
-; ALL-NEXT:    [[TMP2:%.*]] = and i32 [[SIZE:%.*]], 7
+; ALL-NEXT:    [[TMP2:%.*]] = and i32 [[SIZE:%.*]], 15
 ; ALL-NEXT:    [[TMP3:%.*]] = sub i32 [[SIZE]], [[TMP2]]
 ; ALL-NEXT:    [[TMP4:%.*]] = icmp ne i32 [[TMP3]], 0
 ; ALL-NEXT:    br i1 [[TMP4]], label [[LOOP_MEMCPY_EXPANSION:%.*]], label [[LOOP_MEMCPY_RESIDUAL_HEADER:%.*]]
 ; ALL:       loop-memcpy-expansion:
 ; ALL-NEXT:    [[LOOP_INDEX:%.*]] = phi i32 [ 0, [[TMP0:%.*]] ], [ [[TMP8:%.*]], [[LOOP_MEMCPY_EXPANSION]] ]
 ; ALL-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i8, ptr addrspace(5) [[SRC:%.*]], i32 [[LOOP_INDEX]]
-; ALL-NEXT:    [[TMP5:%.*]] = load <2 x i32>, ptr addrspace(5) [[TMP7]], align 1, !alias.scope [[META9:![0-9]+]]
+; ALL-NEXT:    [[TMP5:%.*]] = load <4 x i32>, ptr addrspace(5) [[TMP7]], align 1, !alias.scope [[META9:![0-9]+]]
 ; ALL-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i8, ptr addrspace(3) [[DST:%.*]], i32 [[LOOP_INDEX]]
-; ALL-NEXT:    store <2 x i32> [[TMP5]], ptr addrspace(3) [[TMP6]], align 1, !noalias [[META9]]
-; ALL-NEXT:    [[TMP8]] = add i32 [[LOOP_INDEX]], 8
+; ALL-NEXT:    store <4 x i32> [[TMP5]], ptr addrspace(3) [[TMP6]], align 1, !noalias [[META9]]
+; ALL-NEXT:    [[TMP8]] = add i32 [[LOOP_INDEX]], 16
 ; ALL-NEXT:    [[TMP9:%.*]] = icmp ult i32 [[TMP8]], [[TMP3]]
 ; ALL-NEXT:    br i1 [[TMP9]], label [[LOOP_MEMCPY_EXPANSION]], label [[LOOP_MEMCPY_RESIDUAL_HEADER]]
 ; ALL:       loop-memcpy-residual:
@@ -1781,10 +1813,10 @@ define amdgpu_kernel void @memmove_private_align1_local_align1(ptr addrspace(5)
 ; ALL:       load-store-loop:
 ; ALL-NEXT:    [[LOOP_INDEX:%.*]] = phi i32 [ 0, [[TMP0:%.*]] ], [ [[TMP4:%.*]], [[LOAD_STORE_LOOP]] ]
 ; ALL-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i8, ptr addrspace(3) [[SRC:%.*]], i32 [[LOOP_INDEX]]
-; ALL-NEXT:    [[TMP2:%.*]] = load <2 x i32>, ptr addrspace(3) [[TMP1]], align 1, !alias.scope [[META12:![0-9]+]]
+; ALL-NEXT:    [[TMP2:%.*]] = load <64 x i32>, ptr addrspace(3) [[TMP1]], align 1, !alias.scope [[META12:![0-9]+]]
 ; ALL-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i8, ptr addrspace(5) [[DST:%.*]], i32 [[LOOP_INDEX]]
-; ALL-NEXT:    store <2 x i32> [[TMP2]], ptr addrspace(5) [[TMP3]], align 1, !noalias [[META12]]
-; ALL-NEXT:    [[TMP4]] = add i32 [[LOOP_INDEX]], 8
+; ALL-NEXT:    store <64 x i32> [[TMP2]], ptr addrspace(5) [[TMP3]], align 1, !noalias [[META12]]
+; ALL-NEXT:    [[TMP4]] = add i32 [[LOOP_INDEX]], 256
 ; ALL-NEXT:    [[TMP5:%.*]] = icmp ult i32 [[TMP4]], 256
 ; ALL-NEXT:    br i1 [[TMP5]], label [[LOAD_STORE_LOOP]], label [[MEMCPY_SPLIT:%.*]]
 ; ALL:       memcpy-split:
@@ -1796,17 +1828,17 @@ define amdgpu_kernel void @memmove_private_align1_local_align1(ptr addrspace(5)
 
 define amdgpu_kernel void @memmove_private_align1_local_align1_unknown_size(ptr addrspace(5) %dst, ptr addrspace(3) %src, i32 %size) {
 ; MAX1024-LABEL: @memmove_private_align1_local_align1_unknown_size(
-; MAX1024-NEXT:    [[TMP2:%.*]] = and i32 [[SIZE:%.*]], 7
+; MAX1024-NEXT:    [[TMP2:%.*]] = and i32 [[SIZE:%.*]], 15
 ; MAX1024-NEXT:    [[TMP3:%.*]] = sub i32 [[SIZE]], [[TMP2]]
 ; MAX1024-NEXT:    [[TMP4:%.*]] = icmp ne i32 [[TMP3]], 0
 ; MAX1024-NEXT:    br i1 [[TMP4]], label [[LOOP_MEMCPY_EXPANSION:%.*]], label [[LOOP_MEMCPY_RESIDUAL_HEADER:%.*]]
 ; MAX1024:       loop-memcpy-expansion:
 ; MAX1024-NEXT:    [[LOOP_INDEX:%.*]] = phi i32 [ 0, [[TMP0:%.*]] ], [ [[TMP8:%.*]], [[LOOP_MEMCPY_EXPANSION]] ]
 ; MAX1024-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i8, ptr addrspace(3) [[SRC:%.*]], i32 [[LOOP_INDEX]]
-; MAX1024-NEXT:    [[TMP5:%.*]] = load <2 x i32>, ptr addrspace(3) [[TMP7]], align 1, !alias.scope [[META3:![0-9]+]]
+; MAX1024-NEXT:    [[TMP5:%.*]] = load <4 x i32>, ptr addrspace(3) [[TMP7]], align 1, !alias.scope [[META3:![0-9]+]]
 ; MAX1024-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i8, ptr addrspace(5) [[DST:%.*]], i32 [[LOOP_INDEX]]
-; MAX1024-NEXT:    store <2 x i32> [[TMP5]], ptr addrspace(5) [[TMP6]], align 1, !noalias [[META3]]
-; MAX1024-NEXT:    [[TMP8]] = add i32 [[LOOP_INDEX]], 8
+; MAX1024-NEXT:    store <4 x i32> [[TMP5]], ptr addrspace(5) [[TMP6]], align 1, !noalias [[META3]]
+; MAX1024-NEXT:    [[TMP8]] = add i32 [[LOOP_INDEX]], 16
 ; MAX1024-NEXT:    [[TMP9:%.*]] = icmp ult i32 [[TMP8]], [[TMP3]]
 ; MAX1024-NEXT:    br i1 [[TMP9]], label [[LOOP_MEMCPY_EXPANSION]], label [[LOOP_MEMCPY_RESIDUAL_HEADER]]
 ; MAX1024:       loop-memcpy-residual:
@@ -1826,17 +1858,17 @@ define amdgpu_kernel void @memmove_private_align1_local_align1_unknown_size(ptr
 ; MAX1024-NEXT:    br i1 [[TMP16]], label [[LOOP_MEMCPY_RESIDUAL]], label [[POST_LOOP_MEMCPY_EXPANSION]]
 ;
 ; ALL-LABEL: @memmove_private_align1_local_align1_unknown_size(
-; ALL-NEXT:    [[TMP2:%.*]] = and i32 [[SIZE:%.*]], 7
+; ALL-NEXT:    [[TMP2:%.*]] = and i32 [[SIZE:%.*]], 15
 ; ALL-NEXT:    [[TMP3:%.*]] = sub i32 [[SIZE]], [[TMP2]]
 ; ALL-NEXT:    [[TMP4:%.*]] = icmp ne i32 [[TMP3]], 0
 ; ALL-NEXT:    br i1 [[TMP4]], label [[LOOP_MEMCPY_EXPANSION:%.*]], label [[LOOP_MEMCPY_RESIDUAL_HEADER:%.*]]
 ; ALL:       loop-memcpy-expansion:
 ; ALL-NEXT:    [[LOOP_INDEX:%.*]] = phi i32 [ 0, [[TMP0:%.*]] ], [ [[TMP8:%.*]], [[LOOP_MEMCPY_EXPANSION]] ]
 ; ALL-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i8, ptr addrspace(3) [[SRC:%.*]], i32 [[LOOP_INDEX]]
-; ALL-NEXT:    [[TMP5:%.*]] = load <2 x i32>, ptr addrspace(3) [[TMP7]], align 1, !alias.scope [[META15:![0-9]+]]
+; ALL-NEXT:    [[TMP5:%.*]] = load <4 x i32>, ptr addrspace(3) [[TMP7]], align 1, !alias.scope [[META15:![0-9]+]]
 ; ALL-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i8, ptr addrspace(5) [[DST:%.*]], i32 [[LOOP_INDEX]]
-; ALL-NEXT:    store <2 x i32> [[TMP5]], ptr addrspace(5) [[TMP6]], align 1, !noalias [[META15]]
-; ALL-NEXT:    [[TMP8]] = add i32 [[LOOP_INDEX]], 8
+; ALL-NEXT:    store <4 x i32> [[TMP5]], ptr addrspace(5) [[TMP6]], align 1, !noalias [[META15]]
+; ALL-NEXT:    [[TMP8]] = add i32 [[LOOP_INDEX]], 16
 ; ALL-NEXT:    [[TMP9:%.*]] = icmp ult i32 [[TMP8]], [[TMP3]]
 ; ALL-NEXT:    br i1 [[TMP9]], label [[LOOP_MEMCPY_EXPANSION]], label [[LOOP_MEMCPY_RESIDUAL_HEADER]]
 ; ALL:       loop-memcpy-residual:
@@ -1871,20 +1903,20 @@ define amdgpu_kernel void @memmove_flat_align1_local_align1(ptr addrspace(0) %ds
 ; ALL-NEXT:    br i1 [[COMPARE_SRC_DST]], label [[MEMMOVE_BWD_LOOP:%.*]], label [[MEMMOVE_FWD_LOOP:%.*]]
 ; ALL:       memmove_bwd_loop:
 ; ALL-NEXT:    [[TMP2:%.*]] = phi i32 [ [[BWD_INDEX:%.*]], [[MEMMOVE_BWD_LOOP]] ], [ 256, [[TMP0:%.*]] ]
-; ALL-NEXT:    [[BWD_INDEX]] = sub i32 [[TMP2]], 8
+; ALL-NEXT:    [[BWD_INDEX]] = sub i32 [[TMP2]], 256
 ; ALL-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i8, ptr addrspace(3) [[SRC]], i32 [[BWD_INDEX]]
-; ALL-NEXT:    [[ELEMENT:%.*]] = load <2 x i32>, ptr addrspace(3) [[TMP3]], align 1
+; ALL-NEXT:    [[ELEMENT:%.*]] = load <64 x i32>, ptr addrspace(3) [[TMP3]], align 1
 ; ALL-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i8, ptr [[DST]], i32 [[BWD_INDEX]]
-; ALL-NEXT:    store <2 x i32> [[ELEMENT]], ptr [[TMP4]], align 1
+; ALL-NEXT:    store <64 x i32> [[ELEMENT]], ptr [[TMP4]], align 1
 ; ALL-NEXT:    [[TMP5:%.*]] = icmp eq i32 [[BWD_INDEX]], 0
 ; ALL-NEXT:    br i1 [[TMP5]], label [[MEMMOVE_DONE:%.*]], label [[MEMMOVE_BWD_LOOP]]
 ; ALL:       memmove_fwd_loop:
 ; ALL-NEXT:    [[FWD_INDEX:%.*]] = phi i32 [ [[TMP8:%.*]], [[MEMMOVE_FWD_LOOP]] ], [ 0, [[TMP0]] ]
 ; ALL-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i8, ptr addrspace(3) [[SRC]], i32 [[FWD_INDEX]]
-; ALL-NEXT:    [[ELEMENT1:%.*]] = load <2 x i32>, ptr addrspace(3) [[TMP6]], align 1
+; ALL-NEXT:    [[ELEMENT1:%.*]] = load <64 x i32>, ptr addrspace(3) [[TMP6]], align 1
 ; ALL-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i8, ptr [[DST]], i32 [[FWD_INDEX]]
-; ALL-NEXT:    store <2 x i32> [[ELEMENT1]], ptr [[TMP7]], align 1
-; ALL-NEXT:    [[TMP8]] = add i32 [[FWD_INDEX]], 8
+; ALL-NEXT:    store <64 x i32> [[ELEMENT1]], ptr [[TMP7]], align 1
+; ALL-NEXT:    [[TMP8]] = add i32 [[FWD_INDEX]], 256
 ; ALL-NEXT:    [[TMP9:%.*]] = icmp eq i32 [[TMP8]], 256
 ; ALL-NEXT:    br i1 [[TMP9]], label [[MEMMOVE_DONE]], label [[MEMMOVE_FWD_LOOP]]
 ; ALL:       memmove_done:
@@ -1896,7 +1928,7 @@ define amdgpu_kernel void @memmove_flat_align1_local_align1(ptr addrspace(0) %ds
 
 define amdgpu_kernel void @memmove_flat_align1_local_align1_unknown_size(ptr addrspace(0) %dst, ptr addrspace(3) %src, i32 %size) {
 ; OPT-LABEL: @memmove_flat_align1_local_align1_unknown_size(
-; OPT-NEXT:    [[TMP2:%.*]] = and i32 [[SIZE:%.*]], 7
+; OPT-NEXT:    [[TMP2:%.*]] = and i32 [[SIZE:%.*]], 15
 ; OPT-NEXT:    [[TMP3:%.*]] = sub i32 [[SIZE]], [[TMP2]]
 ; OPT-NEXT:    [[SKIP_RESIDUAL:%.*]] = icmp eq i32 [[TMP2]], 0
 ; OPT-NEXT:    [[SKIP_MAIN:%.*]] = icmp eq i32 [[TMP3]], 0
@@ -1918,11 +1950,11 @@ define amdgpu_kernel void @memmove_flat_align1_local_align1_unknown_size(ptr add
 ; OPT-NEXT:    br i1 [[SKIP_MAIN]], label [[MEMMOVE_DONE:%.*]], label [[MEMMOVE_BWD_MAIN_LOOP:%.*]]
 ; OPT:       memmove_bwd_main_loop:
 ; OPT-NEXT:    [[TMP9:%.*]] = phi i32 [ [[BWD_MAIN_INDEX:%.*]], [[MEMMOVE_BWD_MAIN_LOOP]] ], [ [[TMP3]], [[MEMMOVE_BWD_MIDDLE]] ]
-; OPT-NEXT:    [[BWD_MAIN_INDEX]] = sub i32 [[TMP9]], 8
+; OPT-NEXT:    [[BWD_MAIN_INDEX]] = sub i32 [[TMP9]], 16
 ; OPT-NEXT:    [[TMP10:%.*]] = getelementptr inbounds i8, ptr addrspace(3) [[SRC]], i32 [[BWD_MAIN_INDEX]]
-; OPT-NEXT:    [[ELEMENT1:%.*]] = load <2 x i32>, ptr addrspace(3) [[TMP10]], align 1
+; OPT-NEXT:    [[ELEMENT1:%.*]] = load <4 x i32>, ptr addrspace(3) [[TMP10]], align 1
 ; OPT-NEXT:    [[TMP11:%.*]] = getelementptr inbounds i8, ptr [[DST]], i32 [[BWD_MAIN_INDEX]]
-; OPT-NEXT:    store <2 x i32> [[ELEMENT1]], ptr [[TMP11]], align 1
+; OPT-NEXT:    store <4 x i32> [[ELEMENT1]], ptr [[TMP11]], align 1
 ; OPT-NEXT:    [[TMP12:%.*]] = icmp eq i32 [[BWD_MAIN_INDEX]], 0
 ; OPT-NEXT:    br i1 [[TMP12]], label [[MEMMOVE_DONE]], label [[MEMMOVE_BWD_MAIN_LOOP]]
 ; OPT:       memmove_copy_forward:
@@ -1930,10 +1962,10 @@ define amdgpu_kernel void @memmove_flat_align1_local_align1_unknown_size(ptr add
 ; OPT:       memmove_fwd_main_loop:
 ; OPT-NEXT:    [[FWD_MAIN_INDEX:%.*]] = phi i32 [ [[TMP15:%.*]], [[MEMMOVE_FWD_MAIN_LOOP]] ], [ 0, [[MEMMOVE_COPY_FORWARD]] ]
 ; OPT-NEXT:    [[TMP13:%.*]] = getelementptr inbounds i8, ptr addrspace(3) [[SRC]], i32 [[FWD_MAIN_INDEX]]
-; OPT-NEXT:    [[ELEMENT2:%.*]] = load <2 x i32>, ptr addrspace(3) [[TMP13]], align 1
+; OPT-NEXT:    [[ELEMENT2:%.*]] = load <4 x i32>, ptr addrspace(3) [[TMP13]], align 1
 ; OPT-NEXT:    [[TMP14:%.*]] = getelementptr inbounds i8, ptr [[DST]], i32 [[FWD_MAIN_INDEX]]
-; OPT-NEXT:    store <2 x i32> [[ELEMENT2]], ptr [[TMP14]], align 1
-; OPT-NEXT:    [[TMP15]] = add i32 [[FWD_MAIN_INDEX]], 8
+; OPT-NEXT:    store <4 x i32> [[ELEMENT2]], ptr [[TMP14]], align 1
+; OPT-NEXT:    [[TMP15]] = add i32 [[FWD_MAIN_INDEX]], 16
 ; OPT-NEXT:    [[TMP16:%.*]] = icmp eq i32 [[TMP15]], [[TMP3]]
 ; OPT-NEXT:    br i1 [[TMP16]], label [[MEMMOVE_FWD_MIDDLE]], label [[MEMMOVE_FWD_MAIN_LOOP]]
 ; OPT:       memmove_fwd_middle:
@@ -1965,20 +1997,20 @@ define amdgpu_kernel void @memmove_local_align1_flat_align1(ptr addrspace(3) %ds
 ; ALL-NEXT:    br i1 [[COMPARE_SRC_DST]], label [[MEMMOVE_BWD_LOOP:%.*]], label [[MEMMOVE_FWD_LOOP:%.*]]
 ; ALL:       memmove_bwd_loop:
 ; ALL-NEXT:    [[TMP2:%.*]] = phi i32 [ [[BWD_INDEX:%.*]], [[MEMMOVE_BWD_LOOP]] ], [ 256, [[TMP0:%.*]] ]
-; ALL-NEXT:    [[BWD_INDEX]] = sub i32 [[TMP2]], 8
+; ALL-NEXT:    [[BWD_INDEX]] = sub i32 [[TMP2]], 256
 ; ALL-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i8, ptr [[SRC]], i32 [[BWD_INDEX]]
-; ALL-NEXT:    [[ELEMENT:%.*]] = load <2 x i32>, ptr [[TMP3]], align 1
+; ALL-NEXT:    [[ELEMENT:%.*]] = load <64 x i32>, ptr [[TMP3]], align 1
 ; ALL-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i8, ptr addrspace(3) [[DST]], i32 [[BWD_INDEX]]
-; ALL-NEXT:    store <2 x i32> [[ELEMENT]], ptr addrspace(3) [[TMP4]], align 1
+; ALL-NEXT:    store <64 x i32> [[ELEMENT]], ptr addrspace(3) [[TMP4]], align 1
 ; ALL-NEXT:    [[TMP5:%.*]] = icmp eq i32 [[BWD_INDEX]], 0
 ; ALL-NEXT:    br i1 [[TMP5]], label [[MEMMOVE_DONE:%.*]], label [[MEMMOVE_BWD_LOOP]]
 ; ALL:       memmove_fwd_loop:
 ; ALL-NEXT:    [[FWD_INDEX:%.*]] = phi i32 [ [[TMP8:%.*]], [[MEMMOVE_FWD_LOOP]] ], [ 0, [[TMP0]] ]
 ; ALL-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i8, ptr [[SRC]], i32 [[FWD_INDEX]]
-; ALL-NEXT:    [[ELEMENT1:%.*]] = load <2 x i32>, ptr [[TMP6]], align 1
+; ALL-NEXT:    [[ELEMENT1:%.*]] = load <64 x i32>, ptr [[TMP6]], align 1
 ; ALL-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i8, ptr addrspace(3) [[DST]], i32 [[FWD_INDEX]]
-; ALL-NEXT:    store <2 x i32> [[ELEMENT1]], ptr addrspace(3) [[TMP7]], align 1
-; ALL-NEXT:    [[TMP8]] = add i32 [[FWD_INDEX]], 8
+; ALL-NEXT:    store <64 x i32> [[ELEMENT1]], ptr addrspace(3) [[TMP7]], align 1
+; ALL-NEXT:    [[TMP8]] = add i32 [[FWD_INDEX]], 256
 ; ALL-NEXT:    [[TMP9:%.*]] = icmp eq i32 [[TMP8]], 256
 ; ALL-NEXT:    br i1 [[TMP9]], label [[MEMMOVE_DONE]], label [[MEMMOVE_FWD_LOOP]]
 ; ALL:       memmove_done:
@@ -1990,7 +2022,7 @@ define amdgpu_kernel void @memmove_local_align1_flat_align1(ptr addrspace(3) %ds
 
 define amdgpu_kernel void @memmove_local_align1_flat_align1_unknown_size(ptr addrspace(3) %dst, ptr addrspace(0) %src, i32 %size) {
 ; OPT-LABEL: @memmove_local_align1_flat_align1_unknown_size(
-; OPT-NEXT:    [[TMP2:%.*]] = and i32 [[SIZE:%.*]], 7
+; OPT-NEXT:    [[TMP2:%.*]] = and i32 [[SIZE:%.*]], 15
 ; OPT-NEXT:    [[TMP3:%.*]] = sub i32 [[SIZE]], [[TMP2]]
 ; OPT-NEXT:    [[SKIP_RESIDUAL:%.*]] = icmp eq i32 [[TMP2]], 0
 ; OPT-NEXT:    [[SKIP_MAIN:%.*]] = icmp eq i32 [[TMP3]], 0
@@ -2012,11 +2044,11 @@ define amdgpu_kernel void @memmove_local_align1_flat_align1_unknown_size(ptr add
 ; OPT-NEXT:    br i1 [[SKIP_MAIN]], label [[MEMMOVE_DONE:%.*]], label [[MEMMOVE_BWD_MAIN_LOOP:%.*]]
 ; OPT:       memmove_bwd_main_loop:
 ; OPT-NEXT:    [[TMP9:%.*]] = phi i32 [ [[BWD_MAIN_INDEX:%.*]], [[MEMMOVE_BWD_MAIN_LOOP]] ], [ [[TMP3]], [[MEMMOVE_BWD_MIDDLE]] ]
-; OPT-NEXT:    [[BWD_MAIN_INDEX]] = sub i32 [[TMP9]], 8
+; OPT-NEXT:    [[BWD_MAIN_INDEX]] = sub i32 [[TMP9]], 16
 ; OPT-NEXT:    [[TMP10:%.*]] = getelementptr inbounds i8, ptr [[SRC]], i32 [[BWD_MAIN_INDEX]]
-; OPT-NEXT:    [[ELEMENT1:%.*]] = load <2 x i32>, ptr [[TMP10]], align 1
+; OPT-NEXT:    [[ELEMENT1:%.*]] = load <4 x i32>, ptr [[TMP10]], align 1
 ; OPT-NEXT:    [[TMP11:%.*]] = getelementptr inbounds i8, ptr addrspace(3) [[DST]], i32 [[BWD_MAIN_INDEX]]
-; OPT-NEXT:    store <2 x i32> [[ELEMENT1]], ptr addrspace(3) [[TMP11]], align 1
+; OPT-NEXT:    store <4 x i32> [[ELEMENT1]], ptr addrspace(3) [[TMP11]], align 1
 ; OPT-NEXT:    [[TMP12:%.*]] = icmp eq i32 [[BWD_MAIN_INDEX]], 0
 ; OPT-NEXT:    br i1 [[TMP12]], label [[MEMMOVE_DONE]], label [[MEMMOVE_BWD_MAIN_LOOP]]
 ; OPT:       memmove_copy_forward:
@@ -2024,10 +2056,10 @@ define amdgpu_kernel void @memmove_local_align1_flat_align1_unknown_size(ptr add
 ; OPT:       memmove_fwd_main_loop:
 ; OPT-NEXT:    [[FWD_MAIN_INDEX:%.*]] = phi i32 [ [[TMP15:%.*]], [[MEMMOVE_FWD_MAIN_LOOP]] ], [ 0, [[MEMMOVE_COPY_FORWARD]] ]
 ; OPT-NEXT:    [[TMP13:%.*]] = getelementptr inbounds i8, ptr [[SRC]], i32 [[FWD_MAIN_INDEX]]
-; OPT-NEXT:    [[ELEMENT2:%.*]] = load <2 x i32>, ptr [[TMP13]], align 1
+; OPT-NEXT:    [[ELEMENT2:%.*]] = load <4 x i32>, ptr [[TMP13]], align 1
 ; OPT-NEXT:    [[TMP14:%.*]] = getelementptr inbounds i8, ptr addrspace(3) [[DST]], i32 [[FWD_MAIN_INDEX]]
-; OPT-NEXT:    store <2 x i32> [[ELEMENT2]], ptr addrspace(3) [[TMP14]], align 1
-; OPT-NEXT:    [[TMP15]] = add i32 [[FWD_MAIN_INDEX]], 8
+; OPT-NEXT:    store <4 x i32> [[ELEMENT2]], ptr addrspace(3) [[TMP14]], align 1
+; OPT-NEXT:    [[TMP15]] = add i32 [[FWD_MAIN_INDEX]], 16
 ; OPT-NEXT:    [[TMP16:%.*]] = icmp eq i32 [[TMP15]], [[TMP3]]
 ; OPT-NEXT:    br i1 [[TMP16]], label [[MEMMOVE_FWD_MIDDLE]], label [[MEMMOVE_FWD_MAIN_LOOP]]
 ; OPT:       memmove_fwd_middle:
@@ -2058,20 +2090,20 @@ define amdgpu_kernel void @memmove_local_align1_local_align1(ptr addrspace(3) %d
 ; ALL-NEXT:    br i1 [[COMPARE_SRC_DST]], label [[MEMMOVE_BWD_LOOP:%.*]], label [[MEMMOVE_FWD_LOOP:%.*]]
 ; ALL:       memmove_bwd_loop:
 ; ALL-NEXT:    [[TMP1:%.*]] = phi i32 [ [[BWD_INDEX:%.*]], [[MEMMOVE_BWD_LOOP]] ], [ 256, [[TMP0:%.*]] ]
-; ALL-NEXT:    [[BWD_INDEX]] = sub i32 [[TMP1]], 8
+; ALL-NEXT:    [[BWD_INDEX]] = sub i32 [[TMP1]], 256
 ; ALL-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i8, ptr addrspace(3) [[SRC]], i32 [[BWD_INDEX]]
-; ALL-NEXT:    [[ELEMENT:%.*]] = load <2 x i32>, ptr addrspace(3) [[TMP2]], align 1
+; ALL-NEXT:    [[ELEMENT:%.*]] = load <64 x i32>, ptr addrspace(3) [[TMP2]], align 1
 ; ALL-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i8, ptr addrspace(3) [[DST]], i32 [[BWD_INDEX]]
-; ALL-NEXT:    store <2 x i32> [[ELEMENT]], ptr addrspace(3) [[TMP3]], align 1
+; ALL-NEXT:    store <64 x i32> [[ELEMENT]], ptr addrspace(3) [[TMP3]], align 1
 ; ALL-NEXT:    [[TMP4:%.*]] = icmp eq i32 [[BWD_INDEX]], 0
 ; ALL-NEXT:    br i1 [[TMP4]], label [[MEMMOVE_DONE:%.*]], label [[MEMMOVE_BWD_LOOP]]
 ; ALL:       memmove_fwd_loop:
 ; ALL-NEXT:    [[FWD_INDEX:%.*]] = phi i32 [ [[TMP7:%.*]], [[MEMMOVE_FWD_LOOP]] ], [ 0, [[TMP0]] ]
 ; ALL-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i8, ptr addrspace(3) [[SRC]], i32 [[FWD_INDEX]]
-; ALL-NEXT:    [[ELEMENT1:%.*]] = load <2 x i32>, ptr addrspace(3) [[TMP5]], align 1
+; ALL-NEXT:    [[ELEMENT1:%.*]] = load <64 x i32>, ptr addrspace(3) [[TMP5]], align 1
 ; ALL-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i8, ptr addrspace(3) [[DST]], i32 [[FWD_INDEX]]
-; ALL-NEXT:    store <2 x i32> [[ELEMENT1]], ptr addrspace(3) [[TMP6]], align 1
-; ALL-NEXT:    [[TMP7]] = add i32 [[FWD_INDEX]], 8
+; ALL-NEXT:    store <64 x i32> [[ELEMENT1]], ptr addrspace(3) [[TMP6]], align 1
+; ALL-NEXT:    [[TMP7]] = add i32 [[FWD_INDEX]], 256
 ; ALL-NEXT:    [[TMP8:%.*]] = icmp eq i32 [[TMP7]], 256
 ; ALL-NEXT:    br i1 [[TMP8]], label [[MEMMOVE_DONE]], label [[MEMMOVE_FWD_LOOP]]
 ; ALL:       memmove_done:
@@ -2083,7 +2115,7 @@ define amdgpu_kernel void @memmove_local_align1_local_align1(ptr addrspace(3) %d
 
 define amdgpu_kernel void @memmove_local_align1_local_align1_unknown_size(ptr addrspace(3) %dst, ptr addrspace(3) %src, i32 %size) {
 ; OPT-LABEL: @memmove_local_align1_local_align1_unknown_size(
-; OPT-NEXT:    [[TMP2:%.*]] = and i32 [[SIZE:%.*]], 7
+; OPT-NEXT:    [[TMP2:%.*]] = and i32 [[SIZE:%.*]], 15
 ; OPT-NEXT:    [[TMP3:%.*]] = sub i32 [[SIZE]], [[TMP2]]
 ; OPT-NEXT:    [[SKIP_RESIDUAL:%.*]] = icmp eq i32 [[TMP2]], 0
 ; OPT-NEXT:    [[SKIP_MAIN:%.*]] = icmp eq i32 [[TMP3]], 0
@@ -2104,11 +2136,11 @@ define amdgpu_kernel void @memmove_local_align1_local_align1_unknown_size(ptr ad
 ; OPT-NEXT:    br i1 [[SKIP_MAIN]], label [[MEMMOVE_DONE:%.*]], label [[MEMMOVE_BWD_MAIN_LOOP:%.*]]
 ; OPT:       memmove_bwd_main_loop:
 ; OPT-NEXT:    [[TMP8:%.*]] = phi i32 [ [[BWD_MAIN_INDEX:%.*]], [[MEMMOVE_BWD_MAIN_LOOP]] ], [ [[TMP3]], [[MEMMOVE_BWD_MIDDLE]] ]
-; OPT-NEXT:    [[BWD_MAIN_INDEX]] = sub i32 [[TMP8]], 8
+; OPT-NEXT:    [[BWD_MAIN_INDEX]] = sub i32 [[TMP8]], 16
 ; OPT-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i8, ptr addrspace(3) [[SRC]], i32 [[BWD_MAIN_INDEX]]
-; OPT-NEXT:    [[ELEMENT1:%.*]] = load <2 x i32>, ptr addrspace(3) [[TMP9]], align 1
+; OPT-NEXT:    [[ELEMENT1:%.*]] = load <4 x i32>, ptr addrspace(3) [[TMP9]], align 1
 ; OPT-NEXT:    [[TMP10:%.*]] = getelementptr inbounds i8, ptr addrspace(3) [[DST]], i32 [[BWD_MAIN_INDEX]]
-; OPT-NEXT:    store <2 x i32> [[ELEMENT1]], ptr addrspace(3) [[TMP10]], align 1
+; OPT-NEXT:    store <4 x i32> [[ELEMENT1]], ptr addrspace(3) [[TMP10]], align 1
 ; OPT-NEXT:    [[TMP11:%.*]] = icmp eq i32 [[BWD_MAIN_INDEX]], 0
 ; OPT-NEXT:    br i1 [[TMP11]], label [[MEMMOVE_DONE]], label [[MEMMOVE_BWD_MAIN_LOOP]]
 ; OPT:       memmove_copy_forward:
@@ -2116,10 +2148,10 @@ define amdgpu_kernel void @memmove_local_align1_local_align1_unknown_size(ptr ad
 ; OPT:       memmove_fwd_main_loop:
 ; OPT-NEXT:    [[FWD_MAIN_INDEX:%.*]] = phi i32 [ [[TMP14:%.*]], [[MEMMOVE_FWD_MAIN_LOOP]] ], [ 0, [[MEMMOVE_COPY_FORWARD]] ]
 ; OPT-NEXT:    [[TMP12:%.*]] = getelementptr inbounds i8, ptr addrspace(3) [[SRC]], i32 [[FWD_MAIN_INDEX]]
-; OPT-NEXT:    [[ELEMENT2:%.*]] = load <2 x i32>, ptr addrspace(3) [[TMP12]], align 1
+; OPT-NEXT:    [[ELEMENT2:%.*]] = load <4 x i32>, ptr addrspace(3) [[TMP12]], align 1
 ; OPT-NEXT:    [[TMP13:%.*]] = getelementptr inbounds i8, ptr addrspace(3) [[DST]], i32 [[FWD_MAIN_INDEX]]
-; OPT-NEXT:    store <2 x i32> [[ELEMENT2]], ptr addrspace(3) [[TMP13]], align 1
-; OPT-NEXT:    [[TMP14]] = add i32 [[FWD_MAIN_INDEX]], 8
+; OPT-NEXT:    store <4 x i32> [[ELEMENT2]], ptr addrspace(3) [[TMP13]], align 1
+; OPT-NEXT:    [[TMP14]] = add i32 [[FWD_MAIN_INDEX]], 16
 ; OPT-NEXT:    [[TMP15:%.*]] = icmp eq i32 [[TMP14]], [[TMP3]]
 ; OPT-NEXT:    br i1 [[TMP15]], label [[MEMMOVE_FWD_MIDDLE]], label [[MEMMOVE_FWD_MAIN_LOOP]]
 ; OPT:       memmove_fwd_middle:
diff --git a/llvm/test/CodeGen/AMDGPU/lower-work-group-id-intrinsics-hsa.ll b/llvm/test/CodeGen/AMDGPU/lower-work-group-id-intrinsics-hsa.ll
index 382f1a8c3f431..989ef6f981d9d 100644
--- a/llvm/test/CodeGen/AMDGPU/lower-work-group-id-intrinsics-hsa.ll
+++ b/llvm/test/CodeGen/AMDGPU/lower-work-group-id-intrinsics-hsa.ll
@@ -193,7 +193,6 @@ define amdgpu_kernel void @caller() {
 ; GFX12-SDAG-NEXT:    s_mov_b64 s[4:5], s[0:1]
 ; GFX12-SDAG-NEXT:    s_mov_b64 s[6:7], s[2:3]
 ; GFX12-SDAG-NEXT:    s_mov_b32 s32, 0
-; GFX12-SDAG-NEXT:    s_wait_alu 0xfffe
 ; GFX12-SDAG-NEXT:    s_swappc_b64 s[30:31], s[12:13]
 ; GFX12-SDAG-NEXT:    s_endpgm
 ;
@@ -207,7 +206,6 @@ define amdgpu_kernel void @caller() {
 ; GFX12-GISEL-NEXT:    s_mov_b64 s[4:5], s[0:1]
 ; GFX12-GISEL-NEXT:    s_mov_b64 s[6:7], s[2:3]
 ; GFX12-GISEL-NEXT:    s_mov_b32 s32, 0
-; GFX12-GISEL-NEXT:    s_wait_alu 0xfffe
 ; GFX12-GISEL-NEXT:    s_swappc_b64 s[30:31], s[12:13]
 ; GFX12-GISEL-NEXT:    s_endpgm
   %idx = call i32 @llvm.amdgcn.workgroup.id.x()
diff --git a/llvm/test/CodeGen/AMDGPU/machine-scheduler-sink-trivial-remats.mir b/llvm/test/CodeGen/AMDGPU/machine-scheduler-sink-trivial-remats.mir
index 9f264de531950..fb65d80c46e06 100644
--- a/llvm/test/CodeGen/AMDGPU/machine-scheduler-sink-trivial-remats.mir
+++ b/llvm/test/CodeGen/AMDGPU/machine-scheduler-sink-trivial-remats.mir
@@ -84,13 +84,11 @@ body:             |
     %23:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 23, implicit $exec, implicit $mode
 
   bb.1:
-  ; predecessors: %bb.0
     successors: %bb.2
 
     S_NOP 0
 
   bb.2:
-  ; predcessors: %bb.1
 
     S_NOP 0, implicit %0, implicit %1
     S_NOP 0, implicit %2, implicit %3
@@ -191,14 +189,12 @@ body:             |
     %23:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 23, implicit $exec, implicit $mode
 
   bb.1:
-  ; predecessors: %bb.0
     successors: %bb.2
 
     %24:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 24, implicit $exec, implicit $mode, implicit-def $m0
     S_NOP 0, implicit %24
 
   bb.2:
-  ; predcessors: %bb.1
 
     S_NOP 0, implicit %23
     S_NOP 0, implicit %0, implicit %1
@@ -300,7 +296,6 @@ body:             |
     %23:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 23, implicit $exec, implicit $mode
 
   bb.1:
-  ; predecessors: %bb.0
     successors: %bb.2
 
     %24:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 24, implicit $exec, implicit $mode, implicit-def $m0
@@ -308,7 +303,6 @@ body:             |
     S_NOP 0, implicit %23
 
   bb.2:
-  ; predcessors: %bb.1
 
     S_NOP 0, implicit %0, implicit %1
     S_NOP 0, implicit %2, implicit %3
@@ -408,7 +402,6 @@ body:             |
     %23:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 23, implicit $exec, implicit $mode
 
   bb.1:
-  ; predecessors: %bb.0
     successors: %bb.2
 
     %24:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 24, implicit $exec, implicit $mode, implicit-def $m0
@@ -416,7 +409,6 @@ body:             |
     S_NOP 0, implicit %22, implicit %23
 
   bb.2:
-  ; predcessors: %bb.1
 
     S_NOP 0, implicit %0, implicit %1
     S_NOP 0, implicit %2, implicit %3
@@ -529,7 +521,6 @@ body:             |
     %23:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 23, implicit $exec, implicit $mode
 
   bb.1:
-  ; predecessors: %bb.0
     successors: %bb.2
 
     %24:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 24, implicit $exec, implicit $mode, implicit-def $m0
@@ -537,14 +528,12 @@ body:             |
     S_NOP 0, implicit %23
 
   bb.2:
-  ; predcessors: %bb.1
     successors: %bb.3
 
     %25:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 25, implicit $exec, implicit $mode
     S_NOP 0
 
   bb.3:
-  ; predecessors: %bb.2
     successors: %bb.4
 
     %26:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 26, implicit $exec, implicit $mode, implicit-def $m0
@@ -552,7 +541,6 @@ body:             |
     S_NOP 0, implicit %25
 
   bb.4:
-  ; predcessors: %bb.3
 
     S_NOP 0, implicit %0, implicit %1
     S_NOP 0, implicit %2, implicit %3
@@ -666,7 +654,6 @@ body:             |
     %23:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 23, implicit $exec, implicit $mode
 
   bb.1:
-  ; predecessors: %bb.0
     successors: %bb.2
 
     %24:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 24, implicit $exec, implicit $mode, implicit-def $m0
@@ -674,7 +661,6 @@ body:             |
     S_NOP 0, implicit %23, implicit %22
 
   bb.2:
-  ; predcessors: %bb.1
     successors: %bb.3
 
     %25:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 25, implicit $exec, implicit $mode
@@ -682,7 +668,6 @@ body:             |
     S_NOP 0
 
   bb.3:
-  ; predecessors: %bb.2
     successors: %bb.4
 
     %27:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 27, implicit $exec, implicit $mode, implicit-def $m0
@@ -690,7 +675,6 @@ body:             |
     S_NOP 0, implicit %25, implicit %26
 
   bb.4:
-  ; predcessors: %bb.3
 
     S_NOP 0, implicit %0, implicit %1
     S_NOP 0, implicit %2, implicit %3
@@ -949,14 +933,12 @@ body:             |
     undef %23.sub0:vreg_64 = nofpexcept V_CVT_I32_F64_e32 23, implicit $exec, implicit $mode
 
   bb.1:
-  ; predecessors: %bb.0
     successors: %bb.2
 
     %23.sub1:vreg_64 = nofpexcept V_CVT_I32_F64_e32 24, implicit $exec, implicit $mode, implicit-def $m0
     S_NOP 0, implicit %23
 
   bb.2:
-  ; predcessors: %bb.1
 
     S_NOP 0, implicit %0, implicit %1
     S_NOP 0, implicit %2, implicit %3
@@ -1053,7 +1035,6 @@ body:             |
     undef %21.sub0:vreg_128 = nofpexcept V_CVT_I32_F64_e32 21, implicit $exec, implicit $mode
 
   bb.1:
-  ; predecessors: %bb.0
     successors: %bb.2
 
     %21.sub1:vreg_128 = nofpexcept V_CVT_I32_F64_e32 22, implicit $exec, implicit $mode, implicit-def $m0
@@ -1062,7 +1043,6 @@ body:             |
     S_NOP 0, implicit %21
 
   bb.2:
-  ; predcessors: %bb.1
 
     S_NOP 0, implicit %0, implicit %1
     S_NOP 0, implicit %2, implicit %3
@@ -1581,7 +1561,6 @@ body:             |
     %23:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 23, implicit $exec, implicit $mode
 
   bb.1:
-  ; predecessors: %bb.0
     successors: %bb.2
 
     %24:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 24, implicit $exec, implicit $mode, implicit-def $m0
@@ -1589,7 +1568,6 @@ body:             |
     S_NOP 0, implicit %24, implicit %25
 
   bb.2:
-  ; predcessors: %bb.1
 
     S_NOP 0, implicit %23
     S_NOP 0, implicit %0, implicit %1
@@ -2528,14 +2506,12 @@ body:             |
     %23:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 23, implicit $exec, implicit $mode
 
   bb.1:
-  ; predecessors: %bb.0
     successors: %bb.2
 
     %24:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 24, implicit $exec, implicit $mode, implicit-def $m0
     S_NOP 0, implicit %24
 
   bb.2:
-  ; predcessors: %bb.1
     successors: %bb.3
 
     S_NOP 0, implicit %23
@@ -2543,7 +2519,6 @@ body:             |
     S_NOP 0
 
   bb.3:
-  ; predecessors: %bb.2
     successors: %bb.4
 
     %26:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 26, implicit $exec, implicit $mode, implicit-def $m0
@@ -2551,7 +2526,6 @@ body:             |
     S_NOP 0, implicit %26, implicit %27
 
   bb.4:
-  ; predcessors: %bb.3
 
     S_NOP 0, implicit %25
     S_NOP 0, implicit %0, implicit %1
@@ -2650,7 +2624,6 @@ body:             |
     %21.sub1:vreg_128 = nofpexcept V_CVT_I32_F64_e32 22, implicit $exec, implicit $mode, implicit-def $m0
 
   bb.1:
-  ; predecessors: %bb.0
     successors: %bb.2
 
     %21.sub2:vreg_128 = nofpexcept V_CVT_I32_F64_e32 23, implicit $exec, implicit $mode, implicit-def $m0
@@ -2658,7 +2631,6 @@ body:             |
     S_NOP 0, implicit %21
 
   bb.2:
-  ; predcessors: %bb.1
 
     S_NOP 0, implicit %0, implicit %1
     S_NOP 0, implicit %2, implicit %3
@@ -2759,7 +2731,6 @@ body:             |
     %23:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 23, implicit $exec, implicit $mode
 
   bb.1:
-  ; predecessors: %bb.0
     successors: %bb.2
 
     %24:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 24, implicit $exec, implicit $mode, implicit-def $m0
@@ -2767,7 +2738,6 @@ body:             |
     S_NOP 0, implicit %23
 
   bb.2:
-  ; predcessors: %bb.1
 
     S_NOP 0, implicit %23
     S_NOP 0, implicit %0, implicit %1
@@ -5030,7 +5000,6 @@ body:             |
     %21.sub1:vreg_128 = nofpexcept V_CVT_I32_F64_e32 22, implicit $exec, implicit $mode
 
   bb.1:
-  ; predecessors: %bb.0
     successors: %bb.2
 
     %21.sub2:vreg_128 = nofpexcept V_CVT_I32_F64_e32 23, implicit $exec, implicit $mode, implicit-def $m0
@@ -5038,7 +5007,6 @@ body:             |
     S_NOP 0, implicit %21
 
   bb.2:
-  ; predcessors: %bb.1
 
     S_NOP 0, implicit %0, implicit %1
     S_NOP 0, implicit %2, implicit %3
@@ -5137,14 +5105,12 @@ body:             |
     %23:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 23, implicit $exec, implicit $mode
 
   bb.1:
-  ; predecessors: %bb.0
     successors: %bb.2
 
     %24:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 24, implicit $exec, implicit $mode, implicit-def $m0
     S_NOP 0, implicit %23, implicit %24
 
   bb.2:
-  ; predcessors: %bb.1
 
     S_NOP 0, implicit %0, implicit %1
     S_NOP 0, implicit %2, implicit %3
@@ -5242,7 +5208,6 @@ body:             |
     %22:vreg_64 = nofpexcept V_CVT_F64_I32_e32 22, implicit $exec, implicit $mode
 
   bb.1:
-  ; predecessors: %bb.0
     successors: %bb.2
 
     %23:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 23, implicit $exec, implicit $mode, implicit-def $m0
@@ -5250,7 +5215,6 @@ body:             |
     S_NOP 0, implicit %22
 
   bb.2:
-  ; predcessors: %bb.1
 
     S_NOP 0, implicit %0, implicit %1
     S_NOP 0, implicit %2, implicit %3
@@ -5348,7 +5312,6 @@ body:             |
     %22:vreg_64 = nofpexcept V_CVT_F64_I32_e32 22, implicit $exec, implicit $mode
 
   bb.1:
-  ; predecessors: %bb.0
     successors: %bb.2
 
     %23:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 23, implicit $exec, implicit $mode, implicit-def $m0
@@ -5357,7 +5320,6 @@ body:             |
     S_NOP 0, implicit %22
 
   bb.2:
-  ; predcessors: %bb.1
 
     S_NOP 0, implicit %0, implicit %1
     S_NOP 0, implicit %2, implicit %3
@@ -5456,7 +5418,6 @@ body:             |
     %22:vreg_64 = nofpexcept V_CVT_F64_I32_e32 22, implicit $exec, implicit $mode
 
   bb.1:
-  ; predecessors: %bb.0
     successors: %bb.2
 
     %23:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 23, implicit $exec, implicit $mode, implicit-def $m0
@@ -5466,7 +5427,6 @@ body:             |
     S_NOP 0, implicit %22
 
   bb.2:
-  ; predcessors: %bb.1
 
     S_NOP 0, implicit %0, implicit %1
     S_NOP 0, implicit %2, implicit %3
@@ -5562,14 +5522,12 @@ body:             |
     %22:vreg_64 = nofpexcept V_CVT_F64_I32_e32 22, implicit $exec, implicit $mode
 
   bb.1:
-  ; predecessors: %bb.0
     successors: %bb.2
 
     %23:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 23, implicit $exec, implicit $mode, implicit-def $m0
     S_NOP 0, implicit %22, implicit %23
 
   bb.2:
-  ; predcessors: %bb.1
 
     S_NOP 0, implicit %0, implicit %1
     S_NOP 0, implicit %2, implicit %3
@@ -5669,14 +5627,12 @@ body:             |
     undef %23.sub1:vreg_64_align2 = V_MOV_B32_e32 23, implicit $exec
 
   bb.1:
-  ; predecessors: %bb.0
     successors: %bb.2
 
     %24:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 24, implicit $exec, implicit $mode, implicit-def $m0
     S_NOP 0, implicit %24
 
   bb.2:
-  ; predcessors: %bb.1
 
     S_NOP 0, implicit %23.sub1
     S_NOP 0, implicit %0, implicit %1
@@ -5779,14 +5735,12 @@ body:             |
     %23:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 23, implicit $exec, implicit $mode
 
   bb.1:
-  ; predecessors: %bb.0
     successors: %bb.2
 
     %24:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 24, implicit $exec, implicit $mode, implicit-def $m0
     S_NOP 0, implicit %24
 
   bb.2:
-  ; predcessors: %bb.1
 
     DBG_VALUE %23, 0, 0
     S_NOP 0, implicit %23
@@ -5889,14 +5843,12 @@ body:             |
     %23:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 23, implicit $exec, implicit $mode
 
   bb.1:
-  ; predecessors: %bb.0
     successors: %bb.2
 
     %24:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 24, implicit $exec, implicit $mode, implicit-def $m0
     S_NOP 0, implicit %24
 
   bb.2:
-  ; predcessors: %bb.1
 
     S_NOP 0, implicit %23
     S_NOP 0, implicit %0, implicit %1
@@ -5914,3 +5866,1671 @@ body:             |
     S_ENDPGM 0
 ...
 
+---
+name:            test_occ_8_physreg_use
+tracksRegLiveness: true
+machineFunctionInfo:
+  isEntryFunction: true
+body:             |
+  ; GFX908-LABEL: name: test_occ_8_physreg_use
+  ; GFX908: bb.0:
+  ; GFX908-NEXT:   successors: %bb.1(0x80000000)
+  ; GFX908-NEXT:   liveins: $vgpr0, $sgpr0_sgpr1
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[COPY:%[0-9]+]]:sgpr_64(p4) = COPY $sgpr0_sgpr1
+  ; GFX908-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr0
+  ; GFX908-NEXT:   [[S_LOAD_DWORDX2_IMM:%[0-9]+]]:sreg_64_xexec = S_LOAD_DWORDX2_IMM [[COPY]](p4), 52, 0 :: (dereferenceable invariant load (s64), align 4, addrspace 4)
+  ; GFX908-NEXT:   undef [[S_MOV_B32_:%[0-9]+]].sub1:sreg_64 = S_MOV_B32 0
+  ; GFX908-NEXT:   $vgpr8 = IMPLICIT_DEF
+  ; GFX908-NEXT:   $vgpr9 = IMPLICIT_DEF
+  ; GFX908-NEXT:   dead [[DEF:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF1:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF2:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF3:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF4:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF5:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF6:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF7:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF8:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF9:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF10:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF11:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF12:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF13:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF14:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF15:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF16:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF17:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   dead [[V_CVT_I32_F32_e32_:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF17]], implicit $exec, implicit $mode
+  ; GFX908-NEXT:   [[DEF18:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF19:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   dead [[V_CMP_GT_U32_e64_:%[0-9]+]]:sreg_64 = V_CMP_GT_U32_e64 [[S_LOAD_DWORDX2_IMM]].sub0, [[COPY1]](s32), implicit $exec
+  ; GFX908-NEXT:   dead undef [[S_MOV_B32_:%[0-9]+]].sub0:sreg_64 = COPY [[S_LOAD_DWORDX2_IMM]].sub1
+  ; GFX908-NEXT:   [[V_CVT_I32_F32_e32_1:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 $vgpr8, implicit $exec, implicit $mode
+  ; GFX908-NEXT:   [[V_CVT_I32_F32_e32_2:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 $vgpr9, implicit $exec, implicit $mode
+  ; GFX908-NEXT:   [[V_CVT_I32_F32_e32_3:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF1]], implicit $exec, implicit $mode
+  ; GFX908-NEXT:   [[V_CVT_I32_F32_e32_4:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF2]], implicit $exec, implicit $mode
+  ; GFX908-NEXT:   [[V_CVT_I32_F32_e32_5:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF3]], implicit $exec, implicit $mode
+  ; GFX908-NEXT:   [[V_CVT_I32_F32_e32_6:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF4]], implicit $exec, implicit $mode
+  ; GFX908-NEXT:   [[V_CVT_I32_F32_e32_7:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF5]], implicit $exec, implicit $mode
+  ; GFX908-NEXT:   [[V_CVT_I32_F32_e32_8:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF6]], implicit $exec, implicit $mode
+  ; GFX908-NEXT:   [[V_CVT_I32_F32_e32_9:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF7]], implicit $exec, implicit $mode
+  ; GFX908-NEXT:   [[V_CVT_I32_F32_e32_10:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF8]], implicit $exec, implicit $mode
+  ; GFX908-NEXT:   [[V_CVT_I32_F32_e32_11:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF9]], implicit $exec, implicit $mode
+  ; GFX908-NEXT:   [[V_CVT_I32_F32_e32_12:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF10]], implicit $exec, implicit $mode
+  ; GFX908-NEXT:   [[V_CVT_I32_F32_e32_13:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF11]], implicit $exec, implicit $mode
+  ; GFX908-NEXT:   [[V_CVT_I32_F32_e32_14:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF12]], implicit $exec, implicit $mode
+  ; GFX908-NEXT:   [[V_CVT_I32_F32_e32_15:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF13]], implicit $exec, implicit $mode
+  ; GFX908-NEXT:   [[V_CVT_I32_F32_e32_16:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF14]], implicit $exec, implicit $mode
+  ; GFX908-NEXT:   [[V_CVT_I32_F32_e32_17:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF15]], implicit $exec, implicit $mode
+  ; GFX908-NEXT:   [[V_CVT_I32_F32_e32_18:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF16]], implicit $exec, implicit $mode
+  ; GFX908-NEXT:   [[V_CVT_I32_F32_e32_19:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF18]], implicit $exec, implicit $mode
+  ; GFX908-NEXT:   [[V_CVT_I32_F32_e32_20:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF19]], implicit $exec, implicit $mode
+  ; GFX908-NEXT:   [[DEF20:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[V_CVT_I32_F32_e32_21:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF20]], implicit $exec, implicit $mode
+  ; GFX908-NEXT:   [[DEF21:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[V_CVT_I32_F32_e32_22:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF21]], implicit $exec, implicit $mode
+  ; GFX908-NEXT:   [[DEF22:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[V_CVT_I32_F32_e32_23:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF22]], implicit $exec, implicit $mode
+  ; GFX908-NEXT:   [[DEF23:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[V_CVT_I32_F32_e32_24:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF23]], implicit $exec, implicit $mode
+  ; GFX908-NEXT:   [[DEF24:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[V_CVT_I32_F32_e32_25:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF24]], implicit $exec, implicit $mode
+  ; GFX908-NEXT:   [[DEF25:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[V_CVT_I32_F32_e32_26:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF25]], implicit $exec, implicit $mode
+  ; GFX908-NEXT:   [[DEF26:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[V_CVT_I32_F32_e32_27:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF26]], implicit $exec, implicit $mode
+  ; GFX908-NEXT:   [[DEF27:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[V_CVT_I32_F32_e32_28:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF27]], implicit $exec, implicit $mode
+  ; GFX908-NEXT:   [[DEF28:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[V_CVT_I32_F32_e32_29:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF28]], implicit $exec, implicit $mode
+  ; GFX908-NEXT:   [[DEF29:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF30:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   S_BRANCH %bb.1
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.1:
+  ; GFX908-NEXT:   [[V_CVT_I32_F32_e32_30:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF29]], implicit $exec, implicit $mode
+  ; GFX908-NEXT:   S_NOP 0, implicit [[V_CVT_I32_F32_e32_1]], implicit [[V_CVT_I32_F32_e32_30]], implicit [[DEF29]]
+  ; GFX908-NEXT:   [[V_CVT_I32_F32_e32_31:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF30]], implicit $exec, implicit $mode
+  ; GFX908-NEXT:   S_NOP 0, implicit [[V_CVT_I32_F32_e32_2]], implicit [[V_CVT_I32_F32_e32_31]], implicit [[DEF30]]
+  ; GFX908-NEXT:   S_NOP 0, implicit [[V_CVT_I32_F32_e32_3]], implicit [[V_CVT_I32_F32_e32_11]]
+  ; GFX908-NEXT:   S_NOP 0, implicit [[V_CVT_I32_F32_e32_4]], implicit [[V_CVT_I32_F32_e32_12]]
+  ; GFX908-NEXT:   S_NOP 0, implicit [[V_CVT_I32_F32_e32_5]], implicit [[V_CVT_I32_F32_e32_13]]
+  ; GFX908-NEXT:   S_NOP 0, implicit [[V_CVT_I32_F32_e32_6]], implicit [[V_CVT_I32_F32_e32_14]]
+  ; GFX908-NEXT:   S_NOP 0, implicit [[V_CVT_I32_F32_e32_7]], implicit [[V_CVT_I32_F32_e32_15]]
+  ; GFX908-NEXT:   S_NOP 0, implicit [[V_CVT_I32_F32_e32_8]], implicit [[V_CVT_I32_F32_e32_16]]
+  ; GFX908-NEXT:   S_NOP 0, implicit [[V_CVT_I32_F32_e32_9]], implicit [[V_CVT_I32_F32_e32_17]]
+  ; GFX908-NEXT:   S_NOP 0, implicit [[V_CVT_I32_F32_e32_10]], implicit [[V_CVT_I32_F32_e32_18]]
+  ; GFX908-NEXT:   S_NOP 0, implicit [[V_CVT_I32_F32_e32_19]], implicit [[V_CVT_I32_F32_e32_20]]
+  ; GFX908-NEXT:   S_NOP 0, implicit [[V_CVT_I32_F32_e32_21]], implicit [[V_CVT_I32_F32_e32_22]]
+  ; GFX908-NEXT:   S_NOP 0, implicit [[V_CVT_I32_F32_e32_21]], implicit [[V_CVT_I32_F32_e32_22]]
+  ; GFX908-NEXT:   S_NOP 0, implicit [[V_CVT_I32_F32_e32_23]], implicit [[V_CVT_I32_F32_e32_24]]
+  ; GFX908-NEXT:   S_NOP 0, implicit [[V_CVT_I32_F32_e32_25]], implicit [[V_CVT_I32_F32_e32_26]]
+  ; GFX908-NEXT:   S_NOP 0, implicit [[V_CVT_I32_F32_e32_27]], implicit [[V_CVT_I32_F32_e32_28]]
+  ; GFX908-NEXT:   S_NOP 0, implicit [[V_CVT_I32_F32_e32_29]]
+  ; GFX908-NEXT:   S_ENDPGM 0
+  bb.0:
+    liveins: $vgpr0, $sgpr0_sgpr1
+
+    %1:sgpr_64(p4) = COPY $sgpr0_sgpr1
+    %2:vgpr_32(s32) = COPY $vgpr0
+    %3:sreg_64_xexec = S_LOAD_DWORDX2_IMM %1(p4), 52, 0 :: (dereferenceable invariant load (s64), align 4, addrspace 4)
+    %4:sreg_64 = V_CMP_GT_U32_e64 %3.sub0, %2(s32), implicit $exec
+    undef %5.sub1:sreg_64 = S_MOV_B32 0
+    %5.sub0:sreg_64 = COPY %3.sub1
+    $vgpr8 = IMPLICIT_DEF
+    $vgpr9 = IMPLICIT_DEF
+    %11:vgpr_32 = IMPLICIT_DEF
+    %12:vgpr_32 = IMPLICIT_DEF
+    %13:vgpr_32 = IMPLICIT_DEF
+    %14:vgpr_32 = IMPLICIT_DEF
+    %15:vgpr_32 = IMPLICIT_DEF
+    %16:vgpr_32 = IMPLICIT_DEF
+    %17:vgpr_32 = IMPLICIT_DEF
+    %18:vgpr_32 = IMPLICIT_DEF
+    %19:vgpr_32 = IMPLICIT_DEF
+    %20:vgpr_32 = IMPLICIT_DEF
+    %21:vgpr_32 = IMPLICIT_DEF
+    %22:vgpr_32 = IMPLICIT_DEF
+    %23:vgpr_32 = IMPLICIT_DEF
+    %24:vgpr_32 = IMPLICIT_DEF
+    %25:vgpr_32 = IMPLICIT_DEF
+    %26:vgpr_32 = IMPLICIT_DEF
+    %27:vgpr_32 = IMPLICIT_DEF
+    %28:vgpr_32 = IMPLICIT_DEF
+    %29:vgpr_32 = IMPLICIT_DEF
+    %30:vgpr_32 = IMPLICIT_DEF
+    %31:vgpr_32 = IMPLICIT_DEF
+    %32:vgpr_32 = IMPLICIT_DEF
+    %33:vgpr_32 = IMPLICIT_DEF
+    %34:vgpr_32 = IMPLICIT_DEF
+    %35:vgpr_32 = IMPLICIT_DEF
+    %36:vgpr_32 = IMPLICIT_DEF
+    %37:vgpr_32 = IMPLICIT_DEF
+    %38:vgpr_32 = IMPLICIT_DEF
+    %39:vgpr_32 = IMPLICIT_DEF
+    %40:vgpr_32 = IMPLICIT_DEF
+    %41:vgpr_32 = IMPLICIT_DEF
+    %50:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 $vgpr8, implicit $exec, implicit $mode
+    %51:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 $vgpr9, implicit $exec, implicit $mode
+    %52:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 %12, implicit $exec, implicit $mode
+    %53:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 %13, implicit $exec, implicit $mode
+    %54:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 %14, implicit $exec, implicit $mode
+    %55:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 %15, implicit $exec, implicit $mode
+    %56:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 %16, implicit $exec, implicit $mode
+    %57:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 %17, implicit $exec, implicit $mode
+    %58:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 %18, implicit $exec, implicit $mode
+    %59:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 %19, implicit $exec, implicit $mode
+    %60:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 %20, implicit $exec, implicit $mode
+    %61:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 %21, implicit $exec, implicit $mode
+    %62:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 %22, implicit $exec, implicit $mode
+    %63:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 %23, implicit $exec, implicit $mode
+    %64:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 %24, implicit $exec, implicit $mode
+    %65:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 %25, implicit $exec, implicit $mode
+    %66:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 %26, implicit $exec, implicit $mode
+    %67:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 %27, implicit $exec, implicit $mode
+    %68:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 %28, implicit $exec, implicit $mode
+    %69:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 %29, implicit $exec, implicit $mode
+    %70:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 %30, implicit $exec, implicit $mode
+    %71:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 %31, implicit $exec, implicit $mode
+    %72:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 %32, implicit $exec, implicit $mode
+    %73:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 %33, implicit $exec, implicit $mode
+    %74:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 %34, implicit $exec, implicit $mode
+    %75:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 %35, implicit $exec, implicit $mode
+    %76:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 %36, implicit $exec, implicit $mode
+    %77:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 %37, implicit $exec, implicit $mode
+    %78:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 %38, implicit $exec, implicit $mode
+    %79:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 %39, implicit $exec, implicit $mode
+    %80:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 %40, implicit $exec, implicit $mode
+    %81:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 %41, implicit $exec, implicit $mode
+    S_BRANCH %bb.4
+
+  bb.4:
+
+    S_NOP 0, implicit %50, implicit %60, implicit %20
+    S_NOP 0, implicit %51, implicit %61, implicit %21
+    S_NOP 0, implicit %52, implicit %62
+    S_NOP 0, implicit %53, implicit %63
+    S_NOP 0, implicit %54, implicit %64
+    S_NOP 0, implicit %55, implicit %65
+    S_NOP 0, implicit %56, implicit %66
+    S_NOP 0, implicit %57, implicit %67
+    S_NOP 0, implicit %58, implicit %68
+    S_NOP 0, implicit %59, implicit %69
+    S_NOP 0, implicit %70, implicit %71
+    S_NOP 0, implicit %72, implicit %73
+    S_NOP 0, implicit %72, implicit %73
+    S_NOP 0, implicit %74, implicit %75
+    S_NOP 0, implicit %76, implicit %77
+    S_NOP 0, implicit %78, implicit %79
+    S_NOP 0, implicit %80
+    S_ENDPGM 0
+...
+
+---
+name:            test_occ_8_exec_use
+tracksRegLiveness: true
+machineFunctionInfo:
+  isEntryFunction: true
+body:             |
+  ; GFX908-LABEL: name: test_occ_8_exec_use
+  ; GFX908: bb.0:
+  ; GFX908-NEXT:   successors: %bb.1(0x80000000)
+  ; GFX908-NEXT:   liveins: $vgpr0, $sgpr0_sgpr1
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[COPY:%[0-9]+]]:sgpr_64(p4) = COPY $sgpr0_sgpr1
+  ; GFX908-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr0
+  ; GFX908-NEXT:   [[S_LOAD_DWORDX2_IMM:%[0-9]+]]:sreg_64_xexec = S_LOAD_DWORDX2_IMM [[COPY]](p4), 52, 0 :: (dereferenceable invariant load (s64), align 4, addrspace 4)
+  ; GFX908-NEXT:   undef [[S_MOV_B32_:%[0-9]+]].sub1:sreg_64 = S_MOV_B32 0
+  ; GFX908-NEXT:   dead [[DEF:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF1:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF2:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF3:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF4:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF5:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF6:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF7:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF8:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF9:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF10:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF11:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF12:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF13:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF14:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF15:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF16:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF17:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   dead [[V_CVT_I32_F32_e32_:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF17]], implicit $exec, implicit $mode
+  ; GFX908-NEXT:   [[DEF18:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF19:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   dead [[V_CMP_GT_U32_e64_:%[0-9]+]]:sreg_64 = V_CMP_GT_U32_e64 [[S_LOAD_DWORDX2_IMM]].sub0, [[COPY1]](s32), implicit $exec
+  ; GFX908-NEXT:   dead undef [[S_MOV_B32_:%[0-9]+]].sub0:sreg_64 = COPY [[S_LOAD_DWORDX2_IMM]].sub1
+  ; GFX908-NEXT:   [[V_CVT_I32_F32_e32_1:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF1]], implicit $exec, implicit $mode
+  ; GFX908-NEXT:   [[V_CVT_I32_F32_e32_2:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF2]], implicit $exec, implicit $mode
+  ; GFX908-NEXT:   [[V_CVT_I32_F32_e32_3:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF3]], implicit $exec, implicit $mode
+  ; GFX908-NEXT:   [[V_CVT_I32_F32_e32_4:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF4]], implicit $exec, implicit $mode
+  ; GFX908-NEXT:   [[V_CVT_I32_F32_e32_5:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF5]], implicit $exec, implicit $mode
+  ; GFX908-NEXT:   [[V_CVT_I32_F32_e32_6:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF6]], implicit $exec, implicit $mode
+  ; GFX908-NEXT:   [[V_CVT_I32_F32_e32_7:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF7]], implicit $exec, implicit $mode
+  ; GFX908-NEXT:   [[V_CVT_I32_F32_e32_8:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF8]], implicit $exec, implicit $mode
+  ; GFX908-NEXT:   [[V_CVT_I32_F32_e32_9:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF9]], implicit $exec, implicit $mode
+  ; GFX908-NEXT:   [[V_CVT_I32_F32_e32_10:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF10]], implicit $exec, implicit $mode
+  ; GFX908-NEXT:   [[V_CVT_I32_F32_e32_11:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF11]], implicit $exec, implicit $mode
+  ; GFX908-NEXT:   [[V_CVT_I32_F32_e32_12:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF12]], implicit $exec, implicit $mode
+  ; GFX908-NEXT:   [[V_CVT_I32_F32_e32_13:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF13]], implicit $exec, implicit $mode
+  ; GFX908-NEXT:   [[V_CVT_I32_F32_e32_14:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF14]], implicit $exec, implicit $mode
+  ; GFX908-NEXT:   [[V_CVT_I32_F32_e32_15:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF15]], implicit $exec, implicit $mode
+  ; GFX908-NEXT:   [[V_CVT_I32_F32_e32_16:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF16]], implicit $exec, implicit $mode
+  ; GFX908-NEXT:   [[V_CVT_I32_F32_e32_17:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF18]], implicit $exec, implicit $mode
+  ; GFX908-NEXT:   [[V_CVT_I32_F32_e32_18:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF19]], implicit $exec, implicit $mode
+  ; GFX908-NEXT:   [[DEF20:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[V_CVT_I32_F32_e32_19:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF20]], implicit $exec, implicit $mode
+  ; GFX908-NEXT:   [[DEF21:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[V_CVT_I32_F32_e32_20:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF21]], implicit $exec, implicit $mode
+  ; GFX908-NEXT:   [[DEF22:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[V_CVT_I32_F32_e32_21:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF22]], implicit $exec, implicit $mode
+  ; GFX908-NEXT:   [[DEF23:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[V_CVT_I32_F32_e32_22:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF23]], implicit $exec, implicit $mode
+  ; GFX908-NEXT:   [[DEF24:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[V_CVT_I32_F32_e32_23:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF24]], implicit $exec, implicit $mode
+  ; GFX908-NEXT:   [[DEF25:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[V_CVT_I32_F32_e32_24:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF25]], implicit $exec, implicit $mode
+  ; GFX908-NEXT:   [[DEF26:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[V_CVT_I32_F32_e32_25:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF26]], implicit $exec, implicit $mode
+  ; GFX908-NEXT:   [[DEF27:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[V_CVT_I32_F32_e32_26:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF27]], implicit $exec, implicit $mode
+  ; GFX908-NEXT:   [[DEF28:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[V_CVT_I32_F32_e32_27:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF28]], implicit $exec, implicit $mode
+  ; GFX908-NEXT:   [[DEF29:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF30:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[V_CVT_I32_F32_e32_28:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF29]], implicit $exec, implicit $mode
+  ; GFX908-NEXT:   $vgpr8 = IMPLICIT_DEF
+  ; GFX908-NEXT:   $vgpr9 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[V_CVT_I32_F32_e32_29:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF30]], implicit $exec, implicit $mode
+  ; GFX908-NEXT:   S_BRANCH %bb.1
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.1:
+  ; GFX908-NEXT:   [[S_MOV_B64_:%[0-9]+]]:sreg_64 = S_MOV_B64 255
+  ; GFX908-NEXT:   [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 [[S_MOV_B64_]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX908-NEXT:   [[V_CVT_I32_F32_e32_30:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 1, implicit $exec, implicit $mode
+  ; GFX908-NEXT:   S_NOP 0, implicit [[V_CVT_I32_F32_e32_30]], implicit [[V_CVT_I32_F32_e32_28]], implicit [[DEF29]]
+  ; GFX908-NEXT:   [[V_CVT_I32_F32_e32_31:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 2, implicit $exec, implicit $mode
+  ; GFX908-NEXT:   S_NOP 0, implicit [[V_CVT_I32_F32_e32_31]], implicit [[V_CVT_I32_F32_e32_29]], implicit [[DEF30]]
+  ; GFX908-NEXT:   S_NOP 0, implicit [[V_CVT_I32_F32_e32_1]], implicit [[V_CVT_I32_F32_e32_9]]
+  ; GFX908-NEXT:   S_NOP 0, implicit [[V_CVT_I32_F32_e32_2]], implicit [[V_CVT_I32_F32_e32_10]]
+  ; GFX908-NEXT:   S_NOP 0, implicit [[V_CVT_I32_F32_e32_3]], implicit [[V_CVT_I32_F32_e32_11]]
+  ; GFX908-NEXT:   S_NOP 0, implicit [[V_CVT_I32_F32_e32_4]], implicit [[V_CVT_I32_F32_e32_12]]
+  ; GFX908-NEXT:   S_NOP 0, implicit [[V_CVT_I32_F32_e32_5]], implicit [[V_CVT_I32_F32_e32_13]]
+  ; GFX908-NEXT:   S_NOP 0, implicit [[V_CVT_I32_F32_e32_6]], implicit [[V_CVT_I32_F32_e32_14]]
+  ; GFX908-NEXT:   S_NOP 0, implicit [[V_CVT_I32_F32_e32_7]], implicit [[V_CVT_I32_F32_e32_15]]
+  ; GFX908-NEXT:   S_NOP 0, implicit [[V_CVT_I32_F32_e32_8]], implicit [[V_CVT_I32_F32_e32_16]]
+  ; GFX908-NEXT:   S_NOP 0, implicit [[V_CVT_I32_F32_e32_17]], implicit [[V_CVT_I32_F32_e32_18]]
+  ; GFX908-NEXT:   S_NOP 0, implicit [[V_CVT_I32_F32_e32_19]], implicit [[V_CVT_I32_F32_e32_20]]
+  ; GFX908-NEXT:   S_NOP 0, implicit [[V_CVT_I32_F32_e32_19]], implicit [[V_CVT_I32_F32_e32_20]]
+  ; GFX908-NEXT:   S_NOP 0, implicit [[V_CVT_I32_F32_e32_21]], implicit [[V_CVT_I32_F32_e32_22]]
+  ; GFX908-NEXT:   S_NOP 0, implicit [[V_CVT_I32_F32_e32_23]], implicit [[V_CVT_I32_F32_e32_24]]
+  ; GFX908-NEXT:   S_NOP 0, implicit [[V_CVT_I32_F32_e32_25]], implicit [[V_CVT_I32_F32_e32_26]]
+  ; GFX908-NEXT:   S_NOP 0, implicit [[V_CVT_I32_F32_e32_27]]
+  ; GFX908-NEXT:   $exec = S_MOV_B64 [[S_AND_SAVEEXEC_B64_]]
+  ; GFX908-NEXT:   S_ENDPGM 0
+  bb.0:
+    liveins: $vgpr0, $sgpr0_sgpr1
+
+    %1:sgpr_64(p4) = COPY $sgpr0_sgpr1
+    %2:vgpr_32(s32) = COPY $vgpr0
+    %3:sreg_64_xexec = S_LOAD_DWORDX2_IMM %1(p4), 52, 0 :: (dereferenceable invariant load (s64), align 4, addrspace 4)
+    %4:sreg_64 = V_CMP_GT_U32_e64 %3.sub0, %2(s32), implicit $exec
+    undef %5.sub1:sreg_64 = S_MOV_B32 0
+    %5.sub0:sreg_64 = COPY %3.sub1
+    $vgpr8 = IMPLICIT_DEF
+    $vgpr9 = IMPLICIT_DEF
+    %11:vgpr_32 = IMPLICIT_DEF
+    %12:vgpr_32 = IMPLICIT_DEF
+    %13:vgpr_32 = IMPLICIT_DEF
+    %14:vgpr_32 = IMPLICIT_DEF
+    %15:vgpr_32 = IMPLICIT_DEF
+    %16:vgpr_32 = IMPLICIT_DEF
+    %17:vgpr_32 = IMPLICIT_DEF
+    %18:vgpr_32 = IMPLICIT_DEF
+    %19:vgpr_32 = IMPLICIT_DEF
+    %20:vgpr_32 = IMPLICIT_DEF
+    %21:vgpr_32 = IMPLICIT_DEF
+    %22:vgpr_32 = IMPLICIT_DEF
+    %23:vgpr_32 = IMPLICIT_DEF
+    %24:vgpr_32 = IMPLICIT_DEF
+    %25:vgpr_32 = IMPLICIT_DEF
+    %26:vgpr_32 = IMPLICIT_DEF
+    %27:vgpr_32 = IMPLICIT_DEF
+    %28:vgpr_32 = IMPLICIT_DEF
+    %29:vgpr_32 = IMPLICIT_DEF
+    %30:vgpr_32 = IMPLICIT_DEF
+    %31:vgpr_32 = IMPLICIT_DEF
+    %32:vgpr_32 = IMPLICIT_DEF
+    %33:vgpr_32 = IMPLICIT_DEF
+    %34:vgpr_32 = IMPLICIT_DEF
+    %35:vgpr_32 = IMPLICIT_DEF
+    %36:vgpr_32 = IMPLICIT_DEF
+    %37:vgpr_32 = IMPLICIT_DEF
+    %38:vgpr_32 = IMPLICIT_DEF
+    %39:vgpr_32 = IMPLICIT_DEF
+    %40:vgpr_32 = IMPLICIT_DEF
+    %41:vgpr_32 = IMPLICIT_DEF
+    %50:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 1, implicit $exec, implicit $mode
+    %51:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 2, implicit $exec, implicit $mode
+    %52:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 %12, implicit $exec, implicit $mode
+    %53:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 %13, implicit $exec, implicit $mode
+    %54:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 %14, implicit $exec, implicit $mode
+    %55:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 %15, implicit $exec, implicit $mode
+    %56:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 %16, implicit $exec, implicit $mode
+    %57:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 %17, implicit $exec, implicit $mode
+    %58:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 %18, implicit $exec, implicit $mode
+    %59:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 %19, implicit $exec, implicit $mode
+    %60:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 %20, implicit $exec, implicit $mode
+    %61:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 %21, implicit $exec, implicit $mode
+    %62:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 %22, implicit $exec, implicit $mode
+    %63:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 %23, implicit $exec, implicit $mode
+    %64:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 %24, implicit $exec, implicit $mode
+    %65:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 %25, implicit $exec, implicit $mode
+    %66:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 %26, implicit $exec, implicit $mode
+    %67:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 %27, implicit $exec, implicit $mode
+    %68:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 %28, implicit $exec, implicit $mode
+    %69:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 %29, implicit $exec, implicit $mode
+    %70:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 %30, implicit $exec, implicit $mode
+    %71:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 %31, implicit $exec, implicit $mode
+    %72:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 %32, implicit $exec, implicit $mode
+    %73:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 %33, implicit $exec, implicit $mode
+    %74:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 %34, implicit $exec, implicit $mode
+    %75:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 %35, implicit $exec, implicit $mode
+    %76:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 %36, implicit $exec, implicit $mode
+    %77:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 %37, implicit $exec, implicit $mode
+    %78:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 %38, implicit $exec, implicit $mode
+    %79:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 %39, implicit $exec, implicit $mode
+    %80:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 %40, implicit $exec, implicit $mode
+    %81:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 %41, implicit $exec, implicit $mode
+    S_BRANCH %bb.4
+
+  bb.4:
+
+    %100:sreg_64 = S_MOV_B64 255
+    %101:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed %100, implicit-def $exec, implicit-def $scc, implicit $exec
+    S_NOP 0, implicit %50, implicit %60, implicit %20
+    S_NOP 0, implicit %51, implicit %61, implicit %21
+    S_NOP 0, implicit %52, implicit %62
+    S_NOP 0, implicit %53, implicit %63
+    S_NOP 0, implicit %54, implicit %64
+    S_NOP 0, implicit %55, implicit %65
+    S_NOP 0, implicit %56, implicit %66
+    S_NOP 0, implicit %57, implicit %67
+    S_NOP 0, implicit %58, implicit %68
+    S_NOP 0, implicit %59, implicit %69
+    S_NOP 0, implicit %70, implicit %71
+    S_NOP 0, implicit %72, implicit %73
+    S_NOP 0, implicit %72, implicit %73
+    S_NOP 0, implicit %74, implicit %75
+    S_NOP 0, implicit %76, implicit %77
+    S_NOP 0, implicit %78, implicit %79
+    S_NOP 0, implicit %80
+    $exec = S_MOV_B64 %101:sreg_64_xexec
+    S_ENDPGM 0
+...
+
+---
+name:            remat_virtual_vgpr_occ_6
+tracksRegLiveness: true
+machineFunctionInfo:
+  isEntryFunction: true
+body:             |
+  ; GFX908-LABEL: name: remat_virtual_vgpr_occ_6
+  ; GFX908: bb.0:
+  ; GFX908-NEXT:   successors: %bb.1(0x80000000)
+  ; GFX908-NEXT:   liveins: $vgpr0, $sgpr0_sgpr1
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[COPY:%[0-9]+]]:sgpr_64(p4) = COPY $sgpr0_sgpr1
+  ; GFX908-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr0
+  ; GFX908-NEXT:   [[S_LOAD_DWORDX2_IMM:%[0-9]+]]:sreg_64_xexec = S_LOAD_DWORDX2_IMM [[COPY]](p4), 52, 0 :: (dereferenceable invariant load (s64), align 4, addrspace 4)
+  ; GFX908-NEXT:   [[DEF:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF1:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF2:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF3:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF4:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF5:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF6:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF7:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF8:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF9:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF10:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF11:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF12:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF13:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF14:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF15:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF16:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF17:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF18:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   dead [[V_CVT_I32_F32_e32_:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF18]], implicit $exec, implicit $mode
+  ; GFX908-NEXT:   [[DEF19:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF20:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[V_CVT_I32_F32_e32_1:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF]], implicit $exec, implicit $mode
+  ; GFX908-NEXT:   [[V_CVT_I32_F32_e32_2:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF1]], implicit $exec, implicit $mode
+  ; GFX908-NEXT:   [[V_CVT_I32_F32_e32_3:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF2]], implicit $exec, implicit $mode
+  ; GFX908-NEXT:   [[V_CVT_I32_F32_e32_4:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF3]], implicit $exec, implicit $mode
+  ; GFX908-NEXT:   [[V_CVT_I32_F32_e32_5:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF4]], implicit $exec, implicit $mode
+  ; GFX908-NEXT:   [[V_CVT_I32_F32_e32_6:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF5]], implicit $exec, implicit $mode
+  ; GFX908-NEXT:   [[V_CVT_I32_F32_e32_7:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF6]], implicit $exec, implicit $mode
+  ; GFX908-NEXT:   [[V_CVT_I32_F32_e32_8:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF7]], implicit $exec, implicit $mode
+  ; GFX908-NEXT:   [[V_CVT_I32_F32_e32_9:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF8]], implicit $exec, implicit $mode
+  ; GFX908-NEXT:   [[V_CVT_I32_F32_e32_10:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF9]], implicit $exec, implicit $mode
+  ; GFX908-NEXT:   [[V_CVT_I32_F32_e32_11:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF10]], implicit $exec, implicit $mode
+  ; GFX908-NEXT:   [[V_CVT_I32_F32_e32_12:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF11]], implicit $exec, implicit $mode
+  ; GFX908-NEXT:   [[V_CVT_I32_F32_e32_13:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF12]], implicit $exec, implicit $mode
+  ; GFX908-NEXT:   [[V_CVT_I32_F32_e32_14:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF13]], implicit $exec, implicit $mode
+  ; GFX908-NEXT:   [[V_CVT_I32_F32_e32_15:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF14]], implicit $exec, implicit $mode
+  ; GFX908-NEXT:   [[V_CVT_I32_F32_e32_16:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF15]], implicit $exec, implicit $mode
+  ; GFX908-NEXT:   [[V_CVT_I32_F32_e32_17:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF16]], implicit $exec, implicit $mode
+  ; GFX908-NEXT:   [[V_CVT_I32_F32_e32_18:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF17]], implicit $exec, implicit $mode
+  ; GFX908-NEXT:   [[V_CVT_I32_F32_e32_19:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF19]], implicit $exec, implicit $mode
+  ; GFX908-NEXT:   [[V_CVT_I32_F32_e32_20:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF20]], implicit $exec, implicit $mode
+  ; GFX908-NEXT:   [[DEF21:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[V_CVT_I32_F32_e32_21:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF21]], implicit $exec, implicit $mode
+  ; GFX908-NEXT:   [[DEF22:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF23:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF24:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF25:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF26:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF27:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF28:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF29:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF30:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF31:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[V_CVT_I32_F32_e32_22:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF24]], implicit $exec, implicit $mode
+  ; GFX908-NEXT:   [[V_CVT_I32_F32_e32_23:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF25]], implicit $exec, implicit $mode
+  ; GFX908-NEXT:   [[V_CVT_I32_F32_e32_24:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF26]], implicit $exec, implicit $mode
+  ; GFX908-NEXT:   [[V_CVT_I32_F32_e32_25:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF27]], implicit $exec, implicit $mode
+  ; GFX908-NEXT:   [[V_CVT_I32_F32_e32_26:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF28]], implicit $exec, implicit $mode
+  ; GFX908-NEXT:   [[V_CVT_I32_F32_e32_27:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF29]], implicit $exec, implicit $mode
+  ; GFX908-NEXT:   [[V_CVT_I32_F32_e32_28:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF30]], implicit $exec, implicit $mode
+  ; GFX908-NEXT:   [[V_CVT_I32_F32_e32_29:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF31]], implicit $exec, implicit $mode
+  ; GFX908-NEXT:   undef [[S_MOV_B32_:%[0-9]+]].sub1:sreg_64 = S_MOV_B32 0
+  ; GFX908-NEXT:   dead [[V_CMP_GT_U32_e64_:%[0-9]+]]:sreg_64 = V_CMP_GT_U32_e64 [[S_LOAD_DWORDX2_IMM]].sub0, [[COPY1]](s32), implicit $exec
+  ; GFX908-NEXT:   dead undef [[S_MOV_B32_:%[0-9]+]].sub0:sreg_64 = COPY [[S_LOAD_DWORDX2_IMM]].sub1
+  ; GFX908-NEXT:   S_BRANCH %bb.1
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.1:
+  ; GFX908-NEXT:   [[V_CVT_I32_F32_e32_30:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF22]], implicit $exec, implicit $mode
+  ; GFX908-NEXT:   S_NOP 0, implicit [[V_CVT_I32_F32_e32_30]], implicit [[V_CVT_I32_F32_e32_25]], implicit [[DEF22]], implicit [[DEF27]]
+  ; GFX908-NEXT:   [[V_CVT_I32_F32_e32_31:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF23]], implicit $exec, implicit $mode
+  ; GFX908-NEXT:   S_NOP 0, implicit [[V_CVT_I32_F32_e32_31]], implicit [[V_CVT_I32_F32_e32_26]], implicit [[DEF23]], implicit [[DEF28]]
+  ; GFX908-NEXT:   S_NOP 0, implicit [[V_CVT_I32_F32_e32_22]], implicit [[V_CVT_I32_F32_e32_27]], implicit [[DEF24]], implicit [[DEF29]]
+  ; GFX908-NEXT:   S_NOP 0, implicit [[V_CVT_I32_F32_e32_23]], implicit [[V_CVT_I32_F32_e32_28]], implicit [[DEF25]], implicit [[DEF30]]
+  ; GFX908-NEXT:   S_NOP 0, implicit [[V_CVT_I32_F32_e32_24]], implicit [[V_CVT_I32_F32_e32_29]], implicit [[DEF26]], implicit [[DEF31]]
+  ; GFX908-NEXT:   S_NOP 0, implicit [[V_CVT_I32_F32_e32_1]], implicit [[V_CVT_I32_F32_e32_6]]
+  ; GFX908-NEXT:   S_NOP 0, implicit [[V_CVT_I32_F32_e32_2]], implicit [[V_CVT_I32_F32_e32_7]]
+  ; GFX908-NEXT:   S_NOP 0, implicit [[V_CVT_I32_F32_e32_3]], implicit [[V_CVT_I32_F32_e32_8]]
+  ; GFX908-NEXT:   S_NOP 0, implicit [[V_CVT_I32_F32_e32_4]], implicit [[V_CVT_I32_F32_e32_9]]
+  ; GFX908-NEXT:   S_NOP 0, implicit [[V_CVT_I32_F32_e32_5]], implicit [[V_CVT_I32_F32_e32_10]]
+  ; GFX908-NEXT:   S_NOP 0, implicit [[V_CVT_I32_F32_e32_11]], implicit [[V_CVT_I32_F32_e32_12]]
+  ; GFX908-NEXT:   S_NOP 0, implicit [[V_CVT_I32_F32_e32_13]], implicit [[V_CVT_I32_F32_e32_14]]
+  ; GFX908-NEXT:   S_NOP 0, implicit [[V_CVT_I32_F32_e32_13]], implicit [[V_CVT_I32_F32_e32_14]]
+  ; GFX908-NEXT:   S_NOP 0, implicit [[V_CVT_I32_F32_e32_15]], implicit [[V_CVT_I32_F32_e32_16]]
+  ; GFX908-NEXT:   S_NOP 0, implicit [[V_CVT_I32_F32_e32_17]], implicit [[V_CVT_I32_F32_e32_18]]
+  ; GFX908-NEXT:   S_NOP 0, implicit [[V_CVT_I32_F32_e32_19]], implicit [[V_CVT_I32_F32_e32_20]]
+  ; GFX908-NEXT:   S_NOP 0, implicit [[V_CVT_I32_F32_e32_21]]
+  ; GFX908-NEXT:   S_ENDPGM 0
+  bb.0:
+    liveins: $vgpr0, $sgpr0_sgpr1
+
+    %1:sgpr_64(p4) = COPY $sgpr0_sgpr1
+    %2:vgpr_32(s32) = COPY $vgpr0
+    %3:sreg_64_xexec = S_LOAD_DWORDX2_IMM %1(p4), 52, 0 :: (dereferenceable invariant load (s64), align 4, addrspace 4)
+    %4:sreg_64 = V_CMP_GT_U32_e64 %3.sub0, %2(s32), implicit $exec
+    undef %5.sub1:sreg_64 = S_MOV_B32 0
+    %5.sub0:sreg_64 = COPY %3.sub1
+    %10:vgpr_32 = IMPLICIT_DEF
+    %11:vgpr_32 = IMPLICIT_DEF
+    %12:vgpr_32 = IMPLICIT_DEF
+    %13:vgpr_32 = IMPLICIT_DEF
+    %14:vgpr_32 = IMPLICIT_DEF
+    %15:vgpr_32 = IMPLICIT_DEF
+    %16:vgpr_32 = IMPLICIT_DEF
+    %17:vgpr_32 = IMPLICIT_DEF
+    %18:vgpr_32 = IMPLICIT_DEF
+    %19:vgpr_32 = IMPLICIT_DEF
+    %20:vgpr_32 = IMPLICIT_DEF
+    %21:vgpr_32 = IMPLICIT_DEF
+    %22:vgpr_32 = IMPLICIT_DEF
+    %23:vgpr_32 = IMPLICIT_DEF
+    %24:vgpr_32 = IMPLICIT_DEF
+    %25:vgpr_32 = IMPLICIT_DEF
+    %26:vgpr_32 = IMPLICIT_DEF
+    %27:vgpr_32 = IMPLICIT_DEF
+    %28:vgpr_32 = IMPLICIT_DEF
+    %29:vgpr_32 = IMPLICIT_DEF
+    %30:vgpr_32 = IMPLICIT_DEF
+    %31:vgpr_32 = IMPLICIT_DEF
+    %32:vgpr_32 = IMPLICIT_DEF
+    %33:vgpr_32 = IMPLICIT_DEF
+    %34:vgpr_32 = IMPLICIT_DEF
+    %35:vgpr_32 = IMPLICIT_DEF
+    %36:vgpr_32 = IMPLICIT_DEF
+    %37:vgpr_32 = IMPLICIT_DEF
+    %38:vgpr_32 = IMPLICIT_DEF
+    %39:vgpr_32 = IMPLICIT_DEF
+    %40:vgpr_32 = IMPLICIT_DEF
+    %41:vgpr_32 = IMPLICIT_DEF
+    %50:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 %10, implicit $exec, implicit $mode
+    %51:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 %11, implicit $exec, implicit $mode
+    %52:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 %12, implicit $exec, implicit $mode
+    %53:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 %13, implicit $exec, implicit $mode
+    %54:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 %14, implicit $exec, implicit $mode
+    %55:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 %15, implicit $exec, implicit $mode
+    %56:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 %16, implicit $exec, implicit $mode
+    %57:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 %17, implicit $exec, implicit $mode
+    %58:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 %18, implicit $exec, implicit $mode
+    %59:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 %19, implicit $exec, implicit $mode
+    %60:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 %20, implicit $exec, implicit $mode
+    %61:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 %21, implicit $exec, implicit $mode
+    %62:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 %22, implicit $exec, implicit $mode
+    %63:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 %23, implicit $exec, implicit $mode
+    %64:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 %24, implicit $exec, implicit $mode
+    %65:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 %25, implicit $exec, implicit $mode
+    %66:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 %26, implicit $exec, implicit $mode
+    %67:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 %27, implicit $exec, implicit $mode
+    %68:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 %28, implicit $exec, implicit $mode
+    %69:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 %29, implicit $exec, implicit $mode
+    %70:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 %30, implicit $exec, implicit $mode
+    %71:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 %31, implicit $exec, implicit $mode
+    %72:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 %32, implicit $exec, implicit $mode
+    %73:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 %33, implicit $exec, implicit $mode
+    %74:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 %34, implicit $exec, implicit $mode
+    %75:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 %35, implicit $exec, implicit $mode
+    %76:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 %36, implicit $exec, implicit $mode
+    %77:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 %37, implicit $exec, implicit $mode
+    %78:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 %38, implicit $exec, implicit $mode
+    %79:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 %39, implicit $exec, implicit $mode
+    %80:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 %40, implicit $exec, implicit $mode
+    %81:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 %41, implicit $exec, implicit $mode
+    S_BRANCH %bb.4
+
+  bb.4:
+    S_NOP 0, implicit %50, implicit %60, implicit %10, implicit %20
+    S_NOP 0, implicit %51, implicit %61, implicit %11, implicit %21
+    S_NOP 0, implicit %52, implicit %62, implicit %12, implicit %22
+    S_NOP 0, implicit %53, implicit %63, implicit %13, implicit %23
+    S_NOP 0, implicit %54, implicit %64, implicit %14, implicit %24
+    S_NOP 0, implicit %55, implicit %65
+    S_NOP 0, implicit %56, implicit %66
+    S_NOP 0, implicit %57, implicit %67
+    S_NOP 0, implicit %58, implicit %68
+    S_NOP 0, implicit %59, implicit %69
+    S_NOP 0, implicit %70, implicit %71
+    S_NOP 0, implicit %72, implicit %73
+    S_NOP 0, implicit %72, implicit %73
+    S_NOP 0, implicit %74, implicit %75
+    S_NOP 0, implicit %76, implicit %77
+    S_NOP 0, implicit %78, implicit %79
+    S_NOP 0, implicit %80
+    S_ENDPGM 0
+...
+
+---
+name:            remat_virtual_vgpr_uses_not_available
+tracksRegLiveness: true
+machineFunctionInfo:
+  isEntryFunction: true
+body:             |
+  ; GFX908-LABEL: name: remat_virtual_vgpr_uses_not_available
+  ; GFX908: bb.0:
+  ; GFX908-NEXT:   successors: %bb.1(0x80000000)
+  ; GFX908-NEXT:   liveins: $vgpr0, $sgpr0_sgpr1
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[COPY:%[0-9]+]]:sgpr_64(p4) = COPY $sgpr0_sgpr1
+  ; GFX908-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr0
+  ; GFX908-NEXT:   [[S_LOAD_DWORDX2_IMM:%[0-9]+]]:sreg_64_xexec = S_LOAD_DWORDX2_IMM [[COPY]](p4), 52, 0 :: (dereferenceable invariant load (s64), align 4, addrspace 4)
+  ; GFX908-NEXT:   [[DEF:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF1:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF2:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF3:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF4:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF5:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF6:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF7:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF8:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF9:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF10:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF11:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF12:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF13:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF14:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF15:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF16:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF17:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF18:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   dead [[V_CVT_I32_F32_e32_:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF18]], implicit $exec, implicit $mode
+  ; GFX908-NEXT:   [[DEF19:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF20:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[V_CVT_I32_F32_e32_1:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF]], implicit $exec, implicit $mode
+  ; GFX908-NEXT:   [[V_CVT_I32_F32_e32_2:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF1]], implicit $exec, implicit $mode
+  ; GFX908-NEXT:   [[V_CVT_I32_F32_e32_3:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF2]], implicit $exec, implicit $mode
+  ; GFX908-NEXT:   [[V_CVT_I32_F32_e32_4:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF3]], implicit $exec, implicit $mode
+  ; GFX908-NEXT:   [[V_CVT_I32_F32_e32_5:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF4]], implicit $exec, implicit $mode
+  ; GFX908-NEXT:   [[V_CVT_I32_F32_e32_6:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF5]], implicit $exec, implicit $mode
+  ; GFX908-NEXT:   [[V_CVT_I32_F32_e32_7:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF6]], implicit $exec, implicit $mode
+  ; GFX908-NEXT:   [[V_CVT_I32_F32_e32_8:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF7]], implicit $exec, implicit $mode
+  ; GFX908-NEXT:   [[V_CVT_I32_F32_e32_9:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF8]], implicit $exec, implicit $mode
+  ; GFX908-NEXT:   [[V_CVT_I32_F32_e32_10:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF9]], implicit $exec, implicit $mode
+  ; GFX908-NEXT:   [[V_CVT_I32_F32_e32_11:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF10]], implicit $exec, implicit $mode
+  ; GFX908-NEXT:   [[V_CVT_I32_F32_e32_12:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF11]], implicit $exec, implicit $mode
+  ; GFX908-NEXT:   [[V_CVT_I32_F32_e32_13:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF12]], implicit $exec, implicit $mode
+  ; GFX908-NEXT:   [[V_CVT_I32_F32_e32_14:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF13]], implicit $exec, implicit $mode
+  ; GFX908-NEXT:   [[V_CVT_I32_F32_e32_15:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF14]], implicit $exec, implicit $mode
+  ; GFX908-NEXT:   [[V_CVT_I32_F32_e32_16:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF15]], implicit $exec, implicit $mode
+  ; GFX908-NEXT:   [[V_CVT_I32_F32_e32_17:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF16]], implicit $exec, implicit $mode
+  ; GFX908-NEXT:   [[V_CVT_I32_F32_e32_18:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF17]], implicit $exec, implicit $mode
+  ; GFX908-NEXT:   [[V_CVT_I32_F32_e32_19:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF19]], implicit $exec, implicit $mode
+  ; GFX908-NEXT:   [[V_CVT_I32_F32_e32_20:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF20]], implicit $exec, implicit $mode
+  ; GFX908-NEXT:   [[DEF21:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[V_CVT_I32_F32_e32_21:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF21]], implicit $exec, implicit $mode
+  ; GFX908-NEXT:   [[DEF22:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF23:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF24:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF25:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF26:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF27:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF28:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF29:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF30:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF31:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[V_CVT_I32_F32_e32_22:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF22]], implicit $exec, implicit $mode
+  ; GFX908-NEXT:   [[V_CVT_I32_F32_e32_23:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF23]], implicit $exec, implicit $mode
+  ; GFX908-NEXT:   [[V_CVT_I32_F32_e32_24:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF24]], implicit $exec, implicit $mode
+  ; GFX908-NEXT:   [[V_CVT_I32_F32_e32_25:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF25]], implicit $exec, implicit $mode
+  ; GFX908-NEXT:   [[V_CVT_I32_F32_e32_26:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF26]], implicit $exec, implicit $mode
+  ; GFX908-NEXT:   [[V_CVT_I32_F32_e32_27:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF27]], implicit $exec, implicit $mode
+  ; GFX908-NEXT:   [[V_CVT_I32_F32_e32_28:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF28]], implicit $exec, implicit $mode
+  ; GFX908-NEXT:   [[V_CVT_I32_F32_e32_29:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF29]], implicit $exec, implicit $mode
+  ; GFX908-NEXT:   [[V_CVT_I32_F32_e32_30:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF30]], implicit $exec, implicit $mode
+  ; GFX908-NEXT:   [[V_CVT_I32_F32_e32_31:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF31]], implicit $exec, implicit $mode
+  ; GFX908-NEXT:   undef [[S_MOV_B32_:%[0-9]+]].sub1:sreg_64 = S_MOV_B32 0
+  ; GFX908-NEXT:   dead [[V_CMP_GT_U32_e64_:%[0-9]+]]:sreg_64 = V_CMP_GT_U32_e64 [[S_LOAD_DWORDX2_IMM]].sub0, [[COPY1]](s32), implicit $exec
+  ; GFX908-NEXT:   dead undef [[S_MOV_B32_:%[0-9]+]].sub0:sreg_64 = COPY [[S_LOAD_DWORDX2_IMM]].sub1
+  ; GFX908-NEXT:   S_BRANCH %bb.1
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.1:
+  ; GFX908-NEXT:   S_NOP 0, implicit [[V_CVT_I32_F32_e32_1]], implicit [[V_CVT_I32_F32_e32_6]], implicit [[DEF22]], implicit [[DEF27]]
+  ; GFX908-NEXT:   S_NOP 0, implicit [[V_CVT_I32_F32_e32_22]], implicit [[V_CVT_I32_F32_e32_27]], implicit [[DEF23]], implicit [[DEF28]]
+  ; GFX908-NEXT:   S_NOP 0, implicit [[V_CVT_I32_F32_e32_23]], implicit [[V_CVT_I32_F32_e32_28]], implicit [[DEF24]], implicit [[DEF29]]
+  ; GFX908-NEXT:   S_NOP 0, implicit [[V_CVT_I32_F32_e32_24]], implicit [[V_CVT_I32_F32_e32_29]], implicit [[DEF25]], implicit [[DEF30]]
+  ; GFX908-NEXT:   S_NOP 0, implicit [[V_CVT_I32_F32_e32_25]], implicit [[V_CVT_I32_F32_e32_30]], implicit [[DEF26]], implicit [[DEF31]]
+  ; GFX908-NEXT:   S_NOP 0, implicit [[V_CVT_I32_F32_e32_26]], implicit [[V_CVT_I32_F32_e32_31]]
+  ; GFX908-NEXT:   S_NOP 0, implicit [[V_CVT_I32_F32_e32_2]], implicit [[V_CVT_I32_F32_e32_7]]
+  ; GFX908-NEXT:   S_NOP 0, implicit [[V_CVT_I32_F32_e32_3]], implicit [[V_CVT_I32_F32_e32_8]]
+  ; GFX908-NEXT:   S_NOP 0, implicit [[V_CVT_I32_F32_e32_4]], implicit [[V_CVT_I32_F32_e32_9]]
+  ; GFX908-NEXT:   S_NOP 0, implicit [[V_CVT_I32_F32_e32_5]], implicit [[V_CVT_I32_F32_e32_10]]
+  ; GFX908-NEXT:   S_NOP 0, implicit [[V_CVT_I32_F32_e32_11]], implicit [[V_CVT_I32_F32_e32_12]]
+  ; GFX908-NEXT:   S_NOP 0, implicit [[V_CVT_I32_F32_e32_13]], implicit [[V_CVT_I32_F32_e32_14]]
+  ; GFX908-NEXT:   S_NOP 0, implicit [[V_CVT_I32_F32_e32_13]], implicit [[V_CVT_I32_F32_e32_14]]
+  ; GFX908-NEXT:   S_NOP 0, implicit [[V_CVT_I32_F32_e32_15]], implicit [[V_CVT_I32_F32_e32_16]]
+  ; GFX908-NEXT:   S_NOP 0, implicit [[V_CVT_I32_F32_e32_17]], implicit [[V_CVT_I32_F32_e32_18]]
+  ; GFX908-NEXT:   S_NOP 0, implicit [[V_CVT_I32_F32_e32_19]], implicit [[V_CVT_I32_F32_e32_20]]
+  ; GFX908-NEXT:   S_NOP 0, implicit [[V_CVT_I32_F32_e32_21]]
+  ; GFX908-NEXT:   S_ENDPGM 0
+  bb.0:
+    liveins: $vgpr0, $sgpr0_sgpr1
+
+    %1:sgpr_64(p4) = COPY $sgpr0_sgpr1
+    %2:vgpr_32(s32) = COPY $vgpr0
+    %3:sreg_64_xexec = S_LOAD_DWORDX2_IMM %1(p4), 52, 0 :: (dereferenceable invariant load (s64), align 4, addrspace 4)
+    %4:sreg_64 = V_CMP_GT_U32_e64 %3.sub0, %2(s32), implicit $exec
+    undef %5.sub1:sreg_64 = S_MOV_B32 0
+    %5.sub0:sreg_64 = COPY %3.sub1
+    %10:vgpr_32 = IMPLICIT_DEF
+    %11:vgpr_32 = IMPLICIT_DEF
+    %12:vgpr_32 = IMPLICIT_DEF
+    %13:vgpr_32 = IMPLICIT_DEF
+    %14:vgpr_32 = IMPLICIT_DEF
+    %15:vgpr_32 = IMPLICIT_DEF
+    %16:vgpr_32 = IMPLICIT_DEF
+    %17:vgpr_32 = IMPLICIT_DEF
+    %18:vgpr_32 = IMPLICIT_DEF
+    %19:vgpr_32 = IMPLICIT_DEF
+    %20:vgpr_32 = IMPLICIT_DEF
+    %21:vgpr_32 = IMPLICIT_DEF
+    %22:vgpr_32 = IMPLICIT_DEF
+    %23:vgpr_32 = IMPLICIT_DEF
+    %24:vgpr_32 = IMPLICIT_DEF
+    %25:vgpr_32 = IMPLICIT_DEF
+    %26:vgpr_32 = IMPLICIT_DEF
+    %27:vgpr_32 = IMPLICIT_DEF
+    %28:vgpr_32 = IMPLICIT_DEF
+    %29:vgpr_32 = IMPLICIT_DEF
+    %30:vgpr_32 = IMPLICIT_DEF
+    %31:vgpr_32 = IMPLICIT_DEF
+    %32:vgpr_32 = IMPLICIT_DEF
+    %33:vgpr_32 = IMPLICIT_DEF
+    %34:vgpr_32 = IMPLICIT_DEF
+    %35:vgpr_32 = IMPLICIT_DEF
+    %36:vgpr_32 = IMPLICIT_DEF
+    %37:vgpr_32 = IMPLICIT_DEF
+    %38:vgpr_32 = IMPLICIT_DEF
+    %39:vgpr_32 = IMPLICIT_DEF
+    %40:vgpr_32 = IMPLICIT_DEF
+    %41:vgpr_32 = IMPLICIT_DEF
+    %50:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 %10, implicit $exec, implicit $mode
+    %51:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 %11, implicit $exec, implicit $mode
+    %52:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 %12, implicit $exec, implicit $mode
+    %53:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 %13, implicit $exec, implicit $mode
+    %54:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 %14, implicit $exec, implicit $mode
+    %55:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 %15, implicit $exec, implicit $mode
+    %56:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 %16, implicit $exec, implicit $mode
+    %57:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 %17, implicit $exec, implicit $mode
+    %58:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 %18, implicit $exec, implicit $mode
+    %59:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 %19, implicit $exec, implicit $mode
+    %60:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 %20, implicit $exec, implicit $mode
+    %61:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 %21, implicit $exec, implicit $mode
+    %62:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 %22, implicit $exec, implicit $mode
+    %63:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 %23, implicit $exec, implicit $mode
+    %64:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 %24, implicit $exec, implicit $mode
+    %65:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 %25, implicit $exec, implicit $mode
+    %66:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 %26, implicit $exec, implicit $mode
+    %67:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 %27, implicit $exec, implicit $mode
+    %68:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 %28, implicit $exec, implicit $mode
+    %69:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 %29, implicit $exec, implicit $mode
+    %70:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 %30, implicit $exec, implicit $mode
+    %71:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 %31, implicit $exec, implicit $mode
+    %72:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 %32, implicit $exec, implicit $mode
+    %73:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 %33, implicit $exec, implicit $mode
+    %74:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 %34, implicit $exec, implicit $mode
+    %75:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 %35, implicit $exec, implicit $mode
+    %76:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 %36, implicit $exec, implicit $mode
+    %77:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 %37, implicit $exec, implicit $mode
+    %78:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 %38, implicit $exec, implicit $mode
+    %79:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 %39, implicit $exec, implicit $mode
+    %80:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 %40, implicit $exec, implicit $mode
+    %81:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 %41, implicit $exec, implicit $mode
+    S_BRANCH %bb.4
+
+  bb.4:
+    S_NOP 0, implicit %50, implicit %60, implicit %11, implicit %21
+    S_NOP 0, implicit %51, implicit %61, implicit %12, implicit %22
+    S_NOP 0, implicit %52, implicit %62, implicit %13, implicit %23
+    S_NOP 0, implicit %53, implicit %63, implicit %14, implicit %24
+    S_NOP 0, implicit %54, implicit %64, implicit %15, implicit %25
+    S_NOP 0, implicit %55, implicit %65
+    S_NOP 0, implicit %56, implicit %66
+    S_NOP 0, implicit %57, implicit %67
+    S_NOP 0, implicit %58, implicit %68
+    S_NOP 0, implicit %59, implicit %69
+    S_NOP 0, implicit %70, implicit %71
+    S_NOP 0, implicit %72, implicit %73
+    S_NOP 0, implicit %72, implicit %73
+    S_NOP 0, implicit %74, implicit %75
+    S_NOP 0, implicit %76, implicit %77
+    S_NOP 0, implicit %78, implicit %79
+    S_NOP 0, implicit %80
+    S_ENDPGM 0
+...
+
+---
+name:            remat_virtual_vgpr_subreg_occ_6
+tracksRegLiveness: true
+machineFunctionInfo:
+  isEntryFunction: true
+body:             |
+  ; GFX908-LABEL: name: remat_virtual_vgpr_subreg_occ_6
+  ; GFX908: bb.0:
+  ; GFX908-NEXT:   successors: %bb.1(0x80000000)
+  ; GFX908-NEXT:   liveins: $vgpr0, $sgpr0_sgpr1
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[COPY:%[0-9]+]]:sgpr_64(p4) = COPY $sgpr0_sgpr1
+  ; GFX908-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr0
+  ; GFX908-NEXT:   [[S_LOAD_DWORDX2_IMM:%[0-9]+]]:sreg_64_xexec = S_LOAD_DWORDX2_IMM [[COPY]](p4), 52, 0 :: (dereferenceable invariant load (s64), align 4, addrspace 4)
+  ; GFX908-NEXT:   undef [[S_MOV_B32_:%[0-9]+]].sub1:sreg_64 = S_MOV_B32 0
+  ; GFX908-NEXT:   undef [[DEF:%[0-9]+]].sub0:vreg_512 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF:%[0-9]+]].sub1:vreg_512 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF:%[0-9]+]].sub2:vreg_512 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF:%[0-9]+]].sub3:vreg_512 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF1:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF2:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF3:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   dead [[V_CVT_I32_F32_e32_:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF3]], implicit $exec, implicit $mode
+  ; GFX908-NEXT:   [[DEF4:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF5:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   dead [[DEF6:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   dead [[V_CMP_GT_U32_e64_:%[0-9]+]]:sreg_64 = V_CMP_GT_U32_e64 [[S_LOAD_DWORDX2_IMM]].sub0, [[COPY1]](s32), implicit $exec
+  ; GFX908-NEXT:   dead undef [[S_MOV_B32_:%[0-9]+]].sub0:sreg_64 = COPY [[S_LOAD_DWORDX2_IMM]].sub1
+  ; GFX908-NEXT:   [[V_CVT_I32_F32_e32_1:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF1]], implicit $exec, implicit $mode
+  ; GFX908-NEXT:   [[V_CVT_I32_F32_e32_2:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF2]], implicit $exec, implicit $mode
+  ; GFX908-NEXT:   [[V_CVT_I32_F32_e32_3:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF4]], implicit $exec, implicit $mode
+  ; GFX908-NEXT:   [[V_CVT_I32_F32_e32_4:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF5]], implicit $exec, implicit $mode
+  ; GFX908-NEXT:   [[DEF7:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[V_CVT_I32_F32_e32_5:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF7]], implicit $exec, implicit $mode
+  ; GFX908-NEXT:   [[DEF8:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[V_CVT_I32_F32_e32_6:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF8]], implicit $exec, implicit $mode
+  ; GFX908-NEXT:   [[DEF9:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[V_CVT_I32_F32_e32_7:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF9]], implicit $exec, implicit $mode
+  ; GFX908-NEXT:   [[DEF10:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[V_CVT_I32_F32_e32_8:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF10]], implicit $exec, implicit $mode
+  ; GFX908-NEXT:   [[DEF11:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[V_CVT_I32_F32_e32_9:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF11]], implicit $exec, implicit $mode
+  ; GFX908-NEXT:   [[DEF12:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[V_CVT_I32_F32_e32_10:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF12]], implicit $exec, implicit $mode
+  ; GFX908-NEXT:   [[DEF13:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[V_CVT_I32_F32_e32_11:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF13]], implicit $exec, implicit $mode
+  ; GFX908-NEXT:   [[DEF14:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[V_CVT_I32_F32_e32_12:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF14]], implicit $exec, implicit $mode
+  ; GFX908-NEXT:   [[DEF15:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[V_CVT_I32_F32_e32_13:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF15]], implicit $exec, implicit $mode
+  ; GFX908-NEXT:   [[DEF16:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[V_CVT_I32_F32_e32_14:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF16]], implicit $exec, implicit $mode
+  ; GFX908-NEXT:   [[DEF17:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[V_CVT_I32_F32_e32_15:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF17]], implicit $exec, implicit $mode
+  ; GFX908-NEXT:   [[DEF18:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[V_CVT_I32_F32_e32_16:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF18]], implicit $exec, implicit $mode
+  ; GFX908-NEXT:   [[DEF19:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[V_CVT_I32_F32_e32_17:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF19]], implicit $exec, implicit $mode
+  ; GFX908-NEXT:   [[DEF20:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[V_CVT_I32_F32_e32_18:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF20]], implicit $exec, implicit $mode
+  ; GFX908-NEXT:   [[DEF21:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[V_CVT_I32_F32_e32_19:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF21]], implicit $exec, implicit $mode
+  ; GFX908-NEXT:   [[DEF22:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[V_CVT_I32_F32_e32_20:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF22]], implicit $exec, implicit $mode
+  ; GFX908-NEXT:   [[DEF23:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[V_CVT_I32_F32_e32_21:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF23]], implicit $exec, implicit $mode
+  ; GFX908-NEXT:   [[DEF24:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF25:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF26:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF27:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF28:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF29:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF30:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF31:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[V_CVT_I32_F32_e32_22:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF25]], implicit $exec, implicit $mode
+  ; GFX908-NEXT:   [[V_CVT_I32_F32_e32_23:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF26]], implicit $exec, implicit $mode
+  ; GFX908-NEXT:   [[V_CVT_I32_F32_e32_24:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF27]], implicit $exec, implicit $mode
+  ; GFX908-NEXT:   [[V_CVT_I32_F32_e32_25:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF]].sub2, implicit $exec, implicit $mode
+  ; GFX908-NEXT:   [[V_CVT_I32_F32_e32_26:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF28]], implicit $exec, implicit $mode
+  ; GFX908-NEXT:   [[V_CVT_I32_F32_e32_27:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF29]], implicit $exec, implicit $mode
+  ; GFX908-NEXT:   [[V_CVT_I32_F32_e32_28:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF30]], implicit $exec, implicit $mode
+  ; GFX908-NEXT:   [[V_CVT_I32_F32_e32_29:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF31]], implicit $exec, implicit $mode
+  ; GFX908-NEXT:   S_BRANCH %bb.1
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.1:
+  ; GFX908-NEXT:   [[V_CVT_I32_F32_e32_30:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF]].sub0, implicit $exec, implicit $mode
+  ; GFX908-NEXT:   S_NOP 0, implicit [[V_CVT_I32_F32_e32_30]], implicit [[V_CVT_I32_F32_e32_25]], implicit [[DEF]].sub0, implicit [[DEF]].sub2
+  ; GFX908-NEXT:   [[V_CVT_I32_F32_e32_31:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF24]], implicit $exec, implicit $mode
+  ; GFX908-NEXT:   S_NOP 0, implicit [[V_CVT_I32_F32_e32_31]], implicit [[V_CVT_I32_F32_e32_26]], implicit [[DEF24]], implicit [[DEF28]]
+  ; GFX908-NEXT:   S_NOP 0, implicit [[V_CVT_I32_F32_e32_22]], implicit [[V_CVT_I32_F32_e32_27]], implicit [[DEF25]], implicit [[DEF29]]
+  ; GFX908-NEXT:   S_NOP 0, implicit [[V_CVT_I32_F32_e32_23]], implicit [[V_CVT_I32_F32_e32_28]], implicit [[DEF26]], implicit [[DEF30]]
+  ; GFX908-NEXT:   S_NOP 0, implicit [[V_CVT_I32_F32_e32_24]], implicit [[V_CVT_I32_F32_e32_29]], implicit [[DEF27]], implicit [[DEF31]]
+  ; GFX908-NEXT:   S_NOP 0, implicit [[V_CVT_I32_F32_e32_1]], implicit [[V_CVT_I32_F32_e32_6]]
+  ; GFX908-NEXT:   S_NOP 0, implicit [[V_CVT_I32_F32_e32_2]], implicit [[V_CVT_I32_F32_e32_7]]
+  ; GFX908-NEXT:   S_NOP 0, implicit [[V_CVT_I32_F32_e32_3]], implicit [[V_CVT_I32_F32_e32_8]]
+  ; GFX908-NEXT:   S_NOP 0, implicit [[V_CVT_I32_F32_e32_4]], implicit [[V_CVT_I32_F32_e32_9]]
+  ; GFX908-NEXT:   S_NOP 0, implicit [[V_CVT_I32_F32_e32_5]], implicit [[V_CVT_I32_F32_e32_10]]
+  ; GFX908-NEXT:   S_NOP 0, implicit [[V_CVT_I32_F32_e32_11]], implicit [[V_CVT_I32_F32_e32_12]]
+  ; GFX908-NEXT:   S_NOP 0, implicit [[V_CVT_I32_F32_e32_13]], implicit [[V_CVT_I32_F32_e32_14]]
+  ; GFX908-NEXT:   S_NOP 0, implicit [[V_CVT_I32_F32_e32_13]], implicit [[V_CVT_I32_F32_e32_14]]
+  ; GFX908-NEXT:   S_NOP 0, implicit [[V_CVT_I32_F32_e32_15]], implicit [[V_CVT_I32_F32_e32_16]]
+  ; GFX908-NEXT:   S_NOP 0, implicit [[V_CVT_I32_F32_e32_17]], implicit [[V_CVT_I32_F32_e32_18]]
+  ; GFX908-NEXT:   S_NOP 0, implicit [[V_CVT_I32_F32_e32_19]], implicit [[V_CVT_I32_F32_e32_20]]
+  ; GFX908-NEXT:   S_NOP 0, implicit [[V_CVT_I32_F32_e32_21]]
+  ; GFX908-NEXT:   S_ENDPGM 0
+  bb.0:
+    liveins: $vgpr0, $sgpr0_sgpr1
+
+    %1:sgpr_64(p4) = COPY $sgpr0_sgpr1
+    %2:vgpr_32(s32) = COPY $vgpr0
+    %3:sreg_64_xexec = S_LOAD_DWORDX2_IMM %1(p4), 52, 0 :: (dereferenceable invariant load (s64), align 4, addrspace 4)
+    %4:sreg_64 = V_CMP_GT_U32_e64 %3.sub0, %2(s32), implicit $exec
+    undef %5.sub1:sreg_64 = S_MOV_B32 0
+    %5.sub0:sreg_64 = COPY %3.sub1
+    undef %10.sub0:vreg_512 = IMPLICIT_DEF
+    %10.sub1:vreg_512 = IMPLICIT_DEF
+    %10.sub2:vreg_512 = IMPLICIT_DEF
+    %10.sub3:vreg_512 = IMPLICIT_DEF
+    %11:vgpr_32 = IMPLICIT_DEF
+    %12:vgpr_32 = IMPLICIT_DEF
+    %13:vgpr_32 = IMPLICIT_DEF
+    %14:vgpr_32 = IMPLICIT_DEF
+    %15:vgpr_32 = IMPLICIT_DEF
+    %16:vgpr_32 = IMPLICIT_DEF
+    %17:vgpr_32 = IMPLICIT_DEF
+    %18:vgpr_32 = IMPLICIT_DEF
+    %19:vgpr_32 = IMPLICIT_DEF
+    %20:vgpr_32 = IMPLICIT_DEF
+    %21:vgpr_32 = IMPLICIT_DEF
+    %22:vgpr_32 = IMPLICIT_DEF
+    %23:vgpr_32 = IMPLICIT_DEF
+    %24:vgpr_32 = IMPLICIT_DEF
+    %25:vgpr_32 = IMPLICIT_DEF
+    %26:vgpr_32 = IMPLICIT_DEF
+    %27:vgpr_32 = IMPLICIT_DEF
+    %28:vgpr_32 = IMPLICIT_DEF
+    %29:vgpr_32 = IMPLICIT_DEF
+    %30:vgpr_32 = IMPLICIT_DEF
+    %31:vgpr_32 = IMPLICIT_DEF
+    %32:vgpr_32 = IMPLICIT_DEF
+    %33:vgpr_32 = IMPLICIT_DEF
+    %34:vgpr_32 = IMPLICIT_DEF
+    %35:vgpr_32 = IMPLICIT_DEF
+    %36:vgpr_32 = IMPLICIT_DEF
+    %37:vgpr_32 = IMPLICIT_DEF
+    %38:vgpr_32 = IMPLICIT_DEF
+    %39:vgpr_32 = IMPLICIT_DEF
+    %40:vgpr_32 = IMPLICIT_DEF
+    %41:vgpr_32 = IMPLICIT_DEF
+    %50:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 %10.sub0, implicit $exec, implicit $mode
+    %51:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 %11, implicit $exec, implicit $mode
+    %52:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 %12, implicit $exec, implicit $mode
+    %53:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 %13, implicit $exec, implicit $mode
+    %54:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 %14, implicit $exec, implicit $mode
+    %55:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 %15, implicit $exec, implicit $mode
+    %56:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 %16, implicit $exec, implicit $mode
+    %57:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 %17, implicit $exec, implicit $mode
+    %58:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 %18, implicit $exec, implicit $mode
+    %59:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 %19, implicit $exec, implicit $mode
+    %60:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 %10.sub2, implicit $exec, implicit $mode
+    %61:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 %21, implicit $exec, implicit $mode
+    %62:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 %22, implicit $exec, implicit $mode
+    %63:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 %23, implicit $exec, implicit $mode
+    %64:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 %24, implicit $exec, implicit $mode
+    %65:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 %25, implicit $exec, implicit $mode
+    %66:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 %26, implicit $exec, implicit $mode
+    %67:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 %27, implicit $exec, implicit $mode
+    %68:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 %28, implicit $exec, implicit $mode
+    %69:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 %29, implicit $exec, implicit $mode
+    %70:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 %30, implicit $exec, implicit $mode
+    %71:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 %31, implicit $exec, implicit $mode
+    %72:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 %32, implicit $exec, implicit $mode
+    %73:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 %33, implicit $exec, implicit $mode
+    %74:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 %34, implicit $exec, implicit $mode
+    %75:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 %35, implicit $exec, implicit $mode
+    %76:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 %36, implicit $exec, implicit $mode
+    %77:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 %37, implicit $exec, implicit $mode
+    %78:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 %38, implicit $exec, implicit $mode
+    %79:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 %39, implicit $exec, implicit $mode
+    %80:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 %40, implicit $exec, implicit $mode
+    %81:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 %41, implicit $exec, implicit $mode
+    S_BRANCH %bb.4
+
+  bb.4:
+    S_NOP 0, implicit %50, implicit %60, implicit %10.sub0, implicit %10.sub2
+    S_NOP 0, implicit %51, implicit %61, implicit %11, implicit %21
+    S_NOP 0, implicit %52, implicit %62, implicit %12, implicit %22
+    S_NOP 0, implicit %53, implicit %63, implicit %13, implicit %23
+    S_NOP 0, implicit %54, implicit %64, implicit %14, implicit %24
+    S_NOP 0, implicit %55, implicit %65
+    S_NOP 0, implicit %56, implicit %66
+    S_NOP 0, implicit %57, implicit %67
+    S_NOP 0, implicit %58, implicit %68
+    S_NOP 0, implicit %59, implicit %69
+    S_NOP 0, implicit %70, implicit %71
+    S_NOP 0, implicit %72, implicit %73
+    S_NOP 0, implicit %72, implicit %73
+    S_NOP 0, implicit %74, implicit %75
+    S_NOP 0, implicit %76, implicit %77
+    S_NOP 0, implicit %78, implicit %79
+    S_NOP 0, implicit %80
+    S_ENDPGM 0
+...
+
+---
+name:            remat_virtual_vgpr_subreg_uses_not_available
+tracksRegLiveness: true
+machineFunctionInfo:
+  isEntryFunction: true
+body:             |
+  ; GFX908-LABEL: name: remat_virtual_vgpr_subreg_uses_not_available
+  ; GFX908: bb.0:
+  ; GFX908-NEXT:   successors: %bb.1(0x80000000)
+  ; GFX908-NEXT:   liveins: $vgpr0, $sgpr0_sgpr1
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[COPY:%[0-9]+]]:sgpr_64(p4) = COPY $sgpr0_sgpr1
+  ; GFX908-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr0
+  ; GFX908-NEXT:   [[S_LOAD_DWORDX2_IMM:%[0-9]+]]:sreg_64_xexec = S_LOAD_DWORDX2_IMM [[COPY]](p4), 52, 0 :: (dereferenceable invariant load (s64), align 4, addrspace 4)
+  ; GFX908-NEXT:   undef [[S_MOV_B32_:%[0-9]+]].sub1:sreg_64 = S_MOV_B32 0
+  ; GFX908-NEXT:   undef [[DEF:%[0-9]+]].sub0:vreg_512 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF:%[0-9]+]].sub1:vreg_512 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF:%[0-9]+]].sub2:vreg_512 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF:%[0-9]+]].sub3:vreg_512 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF1:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF2:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF3:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   dead [[V_CVT_I32_F32_e32_:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF3]], implicit $exec, implicit $mode
+  ; GFX908-NEXT:   [[DEF4:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF5:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   dead [[DEF6:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   dead [[V_CMP_GT_U32_e64_:%[0-9]+]]:sreg_64 = V_CMP_GT_U32_e64 [[S_LOAD_DWORDX2_IMM]].sub0, [[COPY1]](s32), implicit $exec
+  ; GFX908-NEXT:   dead undef [[S_MOV_B32_:%[0-9]+]].sub0:sreg_64 = COPY [[S_LOAD_DWORDX2_IMM]].sub1
+  ; GFX908-NEXT:   [[V_CVT_I32_F32_e32_1:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF1]], implicit $exec, implicit $mode
+  ; GFX908-NEXT:   [[V_CVT_I32_F32_e32_2:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF2]], implicit $exec, implicit $mode
+  ; GFX908-NEXT:   [[V_CVT_I32_F32_e32_3:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF4]], implicit $exec, implicit $mode
+  ; GFX908-NEXT:   [[V_CVT_I32_F32_e32_4:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF5]], implicit $exec, implicit $mode
+  ; GFX908-NEXT:   [[DEF7:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[V_CVT_I32_F32_e32_5:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF7]], implicit $exec, implicit $mode
+  ; GFX908-NEXT:   [[DEF8:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[V_CVT_I32_F32_e32_6:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF8]], implicit $exec, implicit $mode
+  ; GFX908-NEXT:   [[DEF9:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[V_CVT_I32_F32_e32_7:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF9]], implicit $exec, implicit $mode
+  ; GFX908-NEXT:   [[DEF10:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[V_CVT_I32_F32_e32_8:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF10]], implicit $exec, implicit $mode
+  ; GFX908-NEXT:   [[DEF11:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[V_CVT_I32_F32_e32_9:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF11]], implicit $exec, implicit $mode
+  ; GFX908-NEXT:   [[DEF12:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[V_CVT_I32_F32_e32_10:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF12]], implicit $exec, implicit $mode
+  ; GFX908-NEXT:   [[DEF13:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[V_CVT_I32_F32_e32_11:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF13]], implicit $exec, implicit $mode
+  ; GFX908-NEXT:   [[DEF14:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[V_CVT_I32_F32_e32_12:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF14]], implicit $exec, implicit $mode
+  ; GFX908-NEXT:   [[DEF15:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[V_CVT_I32_F32_e32_13:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF15]], implicit $exec, implicit $mode
+  ; GFX908-NEXT:   [[DEF16:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[V_CVT_I32_F32_e32_14:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF16]], implicit $exec, implicit $mode
+  ; GFX908-NEXT:   [[DEF17:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[V_CVT_I32_F32_e32_15:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF17]], implicit $exec, implicit $mode
+  ; GFX908-NEXT:   [[DEF18:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[V_CVT_I32_F32_e32_16:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF18]], implicit $exec, implicit $mode
+  ; GFX908-NEXT:   [[DEF19:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[V_CVT_I32_F32_e32_17:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF19]], implicit $exec, implicit $mode
+  ; GFX908-NEXT:   [[DEF20:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[V_CVT_I32_F32_e32_18:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF20]], implicit $exec, implicit $mode
+  ; GFX908-NEXT:   [[DEF21:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[V_CVT_I32_F32_e32_19:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF21]], implicit $exec, implicit $mode
+  ; GFX908-NEXT:   [[DEF22:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[V_CVT_I32_F32_e32_20:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF22]], implicit $exec, implicit $mode
+  ; GFX908-NEXT:   [[DEF23:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[V_CVT_I32_F32_e32_21:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF23]], implicit $exec, implicit $mode
+  ; GFX908-NEXT:   [[DEF24:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF25:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF26:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF27:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF28:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF29:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF30:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF31:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[V_CVT_I32_F32_e32_22:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF]].sub0, implicit $exec, implicit $mode
+  ; GFX908-NEXT:   [[V_CVT_I32_F32_e32_23:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF24]], implicit $exec, implicit $mode
+  ; GFX908-NEXT:   [[V_CVT_I32_F32_e32_24:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF25]], implicit $exec, implicit $mode
+  ; GFX908-NEXT:   [[V_CVT_I32_F32_e32_25:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF26]], implicit $exec, implicit $mode
+  ; GFX908-NEXT:   [[V_CVT_I32_F32_e32_26:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF27]], implicit $exec, implicit $mode
+  ; GFX908-NEXT:   [[V_CVT_I32_F32_e32_27:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF]].sub2, implicit $exec, implicit $mode
+  ; GFX908-NEXT:   [[V_CVT_I32_F32_e32_28:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF28]], implicit $exec, implicit $mode
+  ; GFX908-NEXT:   [[V_CVT_I32_F32_e32_29:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF29]], implicit $exec, implicit $mode
+  ; GFX908-NEXT:   [[V_CVT_I32_F32_e32_30:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF30]], implicit $exec, implicit $mode
+  ; GFX908-NEXT:   [[V_CVT_I32_F32_e32_31:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF31]], implicit $exec, implicit $mode
+  ; GFX908-NEXT:   S_BRANCH %bb.1
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.1:
+  ; GFX908-NEXT:   S_NOP 0, implicit [[V_CVT_I32_F32_e32_22]], implicit [[V_CVT_I32_F32_e32_27]], implicit [[DEF]].sub1, implicit [[DEF]].sub3
+  ; GFX908-NEXT:   S_NOP 0, implicit [[V_CVT_I32_F32_e32_1]], implicit [[V_CVT_I32_F32_e32_6]], implicit [[DEF24]], implicit [[DEF28]]
+  ; GFX908-NEXT:   S_NOP 0, implicit [[V_CVT_I32_F32_e32_23]], implicit [[V_CVT_I32_F32_e32_28]], implicit [[DEF25]], implicit [[DEF29]]
+  ; GFX908-NEXT:   S_NOP 0, implicit [[V_CVT_I32_F32_e32_24]], implicit [[V_CVT_I32_F32_e32_29]], implicit [[DEF26]], implicit [[DEF30]]
+  ; GFX908-NEXT:   S_NOP 0, implicit [[V_CVT_I32_F32_e32_25]], implicit [[V_CVT_I32_F32_e32_30]], implicit [[DEF27]], implicit [[DEF31]]
+  ; GFX908-NEXT:   S_NOP 0, implicit [[V_CVT_I32_F32_e32_26]], implicit [[V_CVT_I32_F32_e32_31]]
+  ; GFX908-NEXT:   S_NOP 0, implicit [[V_CVT_I32_F32_e32_2]], implicit [[V_CVT_I32_F32_e32_7]]
+  ; GFX908-NEXT:   S_NOP 0, implicit [[V_CVT_I32_F32_e32_3]], implicit [[V_CVT_I32_F32_e32_8]]
+  ; GFX908-NEXT:   S_NOP 0, implicit [[V_CVT_I32_F32_e32_4]], implicit [[V_CVT_I32_F32_e32_9]]
+  ; GFX908-NEXT:   S_NOP 0, implicit [[V_CVT_I32_F32_e32_5]], implicit [[V_CVT_I32_F32_e32_10]]
+  ; GFX908-NEXT:   S_NOP 0, implicit [[V_CVT_I32_F32_e32_11]], implicit [[V_CVT_I32_F32_e32_12]]
+  ; GFX908-NEXT:   S_NOP 0, implicit [[V_CVT_I32_F32_e32_13]], implicit [[V_CVT_I32_F32_e32_14]]
+  ; GFX908-NEXT:   S_NOP 0, implicit [[V_CVT_I32_F32_e32_13]], implicit [[V_CVT_I32_F32_e32_14]]
+  ; GFX908-NEXT:   S_NOP 0, implicit [[V_CVT_I32_F32_e32_15]], implicit [[V_CVT_I32_F32_e32_16]]
+  ; GFX908-NEXT:   S_NOP 0, implicit [[V_CVT_I32_F32_e32_17]], implicit [[V_CVT_I32_F32_e32_18]]
+  ; GFX908-NEXT:   S_NOP 0, implicit [[V_CVT_I32_F32_e32_19]], implicit [[V_CVT_I32_F32_e32_20]]
+  ; GFX908-NEXT:   S_NOP 0, implicit [[V_CVT_I32_F32_e32_21]]
+  ; GFX908-NEXT:   S_ENDPGM 0
+  bb.0:
+    liveins: $vgpr0, $sgpr0_sgpr1
+
+    %1:sgpr_64(p4) = COPY $sgpr0_sgpr1
+    %2:vgpr_32(s32) = COPY $vgpr0
+    %3:sreg_64_xexec = S_LOAD_DWORDX2_IMM %1(p4), 52, 0 :: (dereferenceable invariant load (s64), align 4, addrspace 4)
+    %4:sreg_64 = V_CMP_GT_U32_e64 %3.sub0, %2(s32), implicit $exec
+    undef %5.sub1:sreg_64 = S_MOV_B32 0
+    %5.sub0:sreg_64 = COPY %3.sub1
+    undef %10.sub0:vreg_512 = IMPLICIT_DEF
+    %10.sub1:vreg_512 = IMPLICIT_DEF
+    %10.sub2:vreg_512 = IMPLICIT_DEF
+    %10.sub3:vreg_512 = IMPLICIT_DEF
+    %11:vgpr_32 = IMPLICIT_DEF
+    %12:vgpr_32 = IMPLICIT_DEF
+    %13:vgpr_32 = IMPLICIT_DEF
+    %14:vgpr_32 = IMPLICIT_DEF
+    %15:vgpr_32 = IMPLICIT_DEF
+    %16:vgpr_32 = IMPLICIT_DEF
+    %17:vgpr_32 = IMPLICIT_DEF
+    %18:vgpr_32 = IMPLICIT_DEF
+    %19:vgpr_32 = IMPLICIT_DEF
+    %20:vgpr_32 = IMPLICIT_DEF
+    %21:vgpr_32 = IMPLICIT_DEF
+    %22:vgpr_32 = IMPLICIT_DEF
+    %23:vgpr_32 = IMPLICIT_DEF
+    %24:vgpr_32 = IMPLICIT_DEF
+    %25:vgpr_32 = IMPLICIT_DEF
+    %26:vgpr_32 = IMPLICIT_DEF
+    %27:vgpr_32 = IMPLICIT_DEF
+    %28:vgpr_32 = IMPLICIT_DEF
+    %29:vgpr_32 = IMPLICIT_DEF
+    %30:vgpr_32 = IMPLICIT_DEF
+    %31:vgpr_32 = IMPLICIT_DEF
+    %32:vgpr_32 = IMPLICIT_DEF
+    %33:vgpr_32 = IMPLICIT_DEF
+    %34:vgpr_32 = IMPLICIT_DEF
+    %35:vgpr_32 = IMPLICIT_DEF
+    %36:vgpr_32 = IMPLICIT_DEF
+    %37:vgpr_32 = IMPLICIT_DEF
+    %38:vgpr_32 = IMPLICIT_DEF
+    %39:vgpr_32 = IMPLICIT_DEF
+    %40:vgpr_32 = IMPLICIT_DEF
+    %41:vgpr_32 = IMPLICIT_DEF
+    %50:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 %10.sub0, implicit $exec, implicit $mode
+    %51:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 %11, implicit $exec, implicit $mode
+    %52:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 %12, implicit $exec, implicit $mode
+    %53:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 %13, implicit $exec, implicit $mode
+    %54:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 %14, implicit $exec, implicit $mode
+    %55:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 %15, implicit $exec, implicit $mode
+    %56:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 %16, implicit $exec, implicit $mode
+    %57:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 %17, implicit $exec, implicit $mode
+    %58:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 %18, implicit $exec, implicit $mode
+    %59:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 %19, implicit $exec, implicit $mode
+    %60:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 %10.sub2, implicit $exec, implicit $mode
+    %61:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 %21, implicit $exec, implicit $mode
+    %62:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 %22, implicit $exec, implicit $mode
+    %63:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 %23, implicit $exec, implicit $mode
+    %64:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 %24, implicit $exec, implicit $mode
+    %65:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 %25, implicit $exec, implicit $mode
+    %66:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 %26, implicit $exec, implicit $mode
+    %67:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 %27, implicit $exec, implicit $mode
+    %68:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 %28, implicit $exec, implicit $mode
+    %69:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 %29, implicit $exec, implicit $mode
+    %70:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 %30, implicit $exec, implicit $mode
+    %71:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 %31, implicit $exec, implicit $mode
+    %72:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 %32, implicit $exec, implicit $mode
+    %73:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 %33, implicit $exec, implicit $mode
+    %74:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 %34, implicit $exec, implicit $mode
+    %75:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 %35, implicit $exec, implicit $mode
+    %76:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 %36, implicit $exec, implicit $mode
+    %77:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 %37, implicit $exec, implicit $mode
+    %78:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 %38, implicit $exec, implicit $mode
+    %79:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 %39, implicit $exec, implicit $mode
+    %80:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 %40, implicit $exec, implicit $mode
+    %81:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 %41, implicit $exec, implicit $mode
+    S_BRANCH %bb.4
+
+  bb.4:
+    S_NOP 0, implicit %50, implicit %60, implicit %10.sub1, implicit %10.sub3
+    S_NOP 0, implicit %51, implicit %61, implicit %12, implicit %22
+    S_NOP 0, implicit %52, implicit %62, implicit %13, implicit %23
+    S_NOP 0, implicit %53, implicit %63, implicit %14, implicit %24
+    S_NOP 0, implicit %54, implicit %64, implicit %15, implicit %25
+    S_NOP 0, implicit %55, implicit %65
+    S_NOP 0, implicit %56, implicit %66
+    S_NOP 0, implicit %57, implicit %67
+    S_NOP 0, implicit %58, implicit %68
+    S_NOP 0, implicit %59, implicit %69
+    S_NOP 0, implicit %70, implicit %71
+    S_NOP 0, implicit %72, implicit %73
+    S_NOP 0, implicit %72, implicit %73
+    S_NOP 0, implicit %74, implicit %75
+    S_NOP 0, implicit %76, implicit %77
+    S_NOP 0, implicit %78, implicit %79
+    S_NOP 0, implicit %80
+    S_ENDPGM 0
+...
+
+
+---
+name:            remat_virtual_vgpr_subreg_def
+tracksRegLiveness: true
+machineFunctionInfo:
+  isEntryFunction: true
+body:             |
+  ; GFX908-LABEL: name: remat_virtual_vgpr_subreg_def
+  ; GFX908: bb.0:
+  ; GFX908-NEXT:   successors: %bb.1(0x80000000)
+  ; GFX908-NEXT:   liveins: $vgpr0, $sgpr0_sgpr1
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[COPY:%[0-9]+]]:sgpr_64(p4) = COPY $sgpr0_sgpr1
+  ; GFX908-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr0
+  ; GFX908-NEXT:   [[S_LOAD_DWORDX2_IMM:%[0-9]+]]:sreg_64_xexec = S_LOAD_DWORDX2_IMM [[COPY]](p4), 52, 0 :: (dereferenceable invariant load (s64), align 4, addrspace 4)
+  ; GFX908-NEXT:   undef [[S_MOV_B32_:%[0-9]+]].sub1:sreg_64 = S_MOV_B32 0
+  ; GFX908-NEXT:   undef [[DEF:%[0-9]+]].sub0:vreg_512 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF:%[0-9]+]].sub1:vreg_512 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF:%[0-9]+]].sub2:vreg_512 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF:%[0-9]+]].sub3:vreg_512 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF1:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF2:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF3:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   dead [[V_CVT_I32_F32_e32_:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF3]], implicit $exec, implicit $mode
+  ; GFX908-NEXT:   [[DEF4:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF5:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   dead [[DEF6:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   dead [[V_CMP_GT_U32_e64_:%[0-9]+]]:sreg_64 = V_CMP_GT_U32_e64 [[S_LOAD_DWORDX2_IMM]].sub0, [[COPY1]](s32), implicit $exec
+  ; GFX908-NEXT:   dead undef [[S_MOV_B32_:%[0-9]+]].sub0:sreg_64 = COPY [[S_LOAD_DWORDX2_IMM]].sub1
+  ; GFX908-NEXT:   undef [[V_CVT_I32_F32_e32_1:%[0-9]+]].sub1:vreg_64 = nofpexcept V_CVT_I32_F32_e32 [[DEF1]], implicit $exec, implicit $mode
+  ; GFX908-NEXT:   [[V_CVT_I32_F32_e32_2:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF2]], implicit $exec, implicit $mode
+  ; GFX908-NEXT:   [[V_CVT_I32_F32_e32_3:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF4]], implicit $exec, implicit $mode
+  ; GFX908-NEXT:   [[V_CVT_I32_F32_e32_4:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF5]], implicit $exec, implicit $mode
+  ; GFX908-NEXT:   [[DEF7:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[V_CVT_I32_F32_e32_5:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF7]], implicit $exec, implicit $mode
+  ; GFX908-NEXT:   [[DEF8:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[V_CVT_I32_F32_e32_6:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF8]], implicit $exec, implicit $mode
+  ; GFX908-NEXT:   [[DEF9:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[V_CVT_I32_F32_e32_7:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF9]], implicit $exec, implicit $mode
+  ; GFX908-NEXT:   [[DEF10:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[V_CVT_I32_F32_e32_8:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF10]], implicit $exec, implicit $mode
+  ; GFX908-NEXT:   [[DEF11:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[V_CVT_I32_F32_e32_9:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF11]], implicit $exec, implicit $mode
+  ; GFX908-NEXT:   [[DEF12:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[V_CVT_I32_F32_e32_10:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF12]], implicit $exec, implicit $mode
+  ; GFX908-NEXT:   [[DEF13:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[V_CVT_I32_F32_e32_11:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF13]], implicit $exec, implicit $mode
+  ; GFX908-NEXT:   [[DEF14:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[V_CVT_I32_F32_e32_12:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF14]], implicit $exec, implicit $mode
+  ; GFX908-NEXT:   [[DEF15:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[V_CVT_I32_F32_e32_13:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF15]], implicit $exec, implicit $mode
+  ; GFX908-NEXT:   [[DEF16:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[V_CVT_I32_F32_e32_14:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF16]], implicit $exec, implicit $mode
+  ; GFX908-NEXT:   [[DEF17:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[V_CVT_I32_F32_e32_15:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF17]], implicit $exec, implicit $mode
+  ; GFX908-NEXT:   [[DEF18:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[V_CVT_I32_F32_e32_16:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF18]], implicit $exec, implicit $mode
+  ; GFX908-NEXT:   [[DEF19:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[V_CVT_I32_F32_e32_17:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF19]], implicit $exec, implicit $mode
+  ; GFX908-NEXT:   [[DEF20:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[V_CVT_I32_F32_e32_18:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF20]], implicit $exec, implicit $mode
+  ; GFX908-NEXT:   [[DEF21:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[V_CVT_I32_F32_e32_19:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF21]], implicit $exec, implicit $mode
+  ; GFX908-NEXT:   [[DEF22:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[V_CVT_I32_F32_e32_20:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF22]], implicit $exec, implicit $mode
+  ; GFX908-NEXT:   [[DEF23:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[V_CVT_I32_F32_e32_21:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF23]], implicit $exec, implicit $mode
+  ; GFX908-NEXT:   [[DEF24:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[V_CVT_I32_F32_e32_22:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF24]], implicit $exec, implicit $mode
+  ; GFX908-NEXT:   [[DEF25:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF26:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF27:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF28:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF29:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF30:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF31:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[V_CVT_I32_F32_e32_23:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF25]], implicit $exec, implicit $mode
+  ; GFX908-NEXT:   [[V_CVT_I32_F32_e32_24:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF30]], implicit $exec, implicit $mode
+  ; GFX908-NEXT:   [[V_CVT_I32_F32_e32_25:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF31]], implicit $exec, implicit $mode
+  ; GFX908-NEXT:   undef [[V_CVT_I32_F32_e32_1:%[0-9]+]].sub0:vreg_64 = nofpexcept V_CVT_I32_F32_e32 [[DEF]].sub0, implicit $exec, implicit $mode
+  ; GFX908-NEXT:   S_BRANCH %bb.1
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.1:
+  ; GFX908-NEXT:   [[V_CVT_I32_F32_e32_26:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF]].sub2, implicit $exec, implicit $mode
+  ; GFX908-NEXT:   S_NOP 0, implicit [[V_CVT_I32_F32_e32_1]].sub0, implicit [[V_CVT_I32_F32_e32_26]], implicit [[DEF]].sub0, implicit [[DEF]].sub2
+  ; GFX908-NEXT:   [[V_CVT_I32_F32_e32_27:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF28]], implicit $exec, implicit $mode
+  ; GFX908-NEXT:   S_NOP 0, implicit [[V_CVT_I32_F32_e32_23]], implicit [[V_CVT_I32_F32_e32_27]], implicit [[DEF1]], implicit [[DEF28]]
+  ; GFX908-NEXT:   [[V_CVT_I32_F32_e32_28:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF29]], implicit $exec, implicit $mode
+  ; GFX908-NEXT:   S_NOP 0, implicit [[V_CVT_I32_F32_e32_23]], implicit [[V_CVT_I32_F32_e32_28]], implicit [[DEF25]], implicit [[DEF29]]
+  ; GFX908-NEXT:   [[V_CVT_I32_F32_e32_29:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF26]], implicit $exec, implicit $mode
+  ; GFX908-NEXT:   [[V_CVT_I32_F32_e32_30:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF27]], implicit $exec, implicit $mode
+  ; GFX908-NEXT:   S_NOP 0, implicit [[V_CVT_I32_F32_e32_29]], implicit [[V_CVT_I32_F32_e32_24]], implicit [[DEF26]], implicit [[DEF30]]
+  ; GFX908-NEXT:   S_NOP 0, implicit [[V_CVT_I32_F32_e32_30]], implicit [[V_CVT_I32_F32_e32_25]], implicit [[DEF27]], implicit [[DEF31]]
+  ; GFX908-NEXT:   S_NOP 0, implicit [[V_CVT_I32_F32_e32_2]], implicit [[V_CVT_I32_F32_e32_7]]
+  ; GFX908-NEXT:   S_NOP 0, implicit [[V_CVT_I32_F32_e32_3]], implicit [[V_CVT_I32_F32_e32_8]]
+  ; GFX908-NEXT:   S_NOP 0, implicit [[V_CVT_I32_F32_e32_4]], implicit [[V_CVT_I32_F32_e32_9]]
+  ; GFX908-NEXT:   S_NOP 0, implicit [[V_CVT_I32_F32_e32_5]], implicit [[V_CVT_I32_F32_e32_10]]
+  ; GFX908-NEXT:   S_NOP 0, implicit [[V_CVT_I32_F32_e32_6]], implicit [[V_CVT_I32_F32_e32_11]]
+  ; GFX908-NEXT:   S_NOP 0, implicit [[V_CVT_I32_F32_e32_12]], implicit [[V_CVT_I32_F32_e32_13]]
+  ; GFX908-NEXT:   S_NOP 0, implicit [[V_CVT_I32_F32_e32_14]], implicit [[V_CVT_I32_F32_e32_15]]
+  ; GFX908-NEXT:   S_NOP 0, implicit [[V_CVT_I32_F32_e32_14]], implicit [[V_CVT_I32_F32_e32_15]]
+  ; GFX908-NEXT:   S_NOP 0, implicit [[V_CVT_I32_F32_e32_16]], implicit [[V_CVT_I32_F32_e32_17]]
+  ; GFX908-NEXT:   S_NOP 0, implicit [[V_CVT_I32_F32_e32_18]], implicit [[V_CVT_I32_F32_e32_19]]
+  ; GFX908-NEXT:   S_NOP 0, implicit [[V_CVT_I32_F32_e32_20]], implicit [[V_CVT_I32_F32_e32_21]]
+  ; GFX908-NEXT:   S_NOP 0, implicit [[V_CVT_I32_F32_e32_22]]
+  ; GFX908-NEXT:   S_ENDPGM 0
+  bb.0:
+    liveins: $vgpr0, $sgpr0_sgpr1
+
+    %1:sgpr_64(p4) = COPY $sgpr0_sgpr1
+    %2:vgpr_32(s32) = COPY $vgpr0
+    %3:sreg_64_xexec = S_LOAD_DWORDX2_IMM %1(p4), 52, 0 :: (dereferenceable invariant load (s64), align 4, addrspace 4)
+    %4:sreg_64 = V_CMP_GT_U32_e64 %3.sub0, %2(s32), implicit $exec
+    undef %5.sub1:sreg_64 = S_MOV_B32 0
+    %5.sub0:sreg_64 = COPY %3.sub1
+    undef %10.sub0:vreg_512 = IMPLICIT_DEF
+    %10.sub1:vreg_512 = IMPLICIT_DEF
+    %10.sub2:vreg_512 = IMPLICIT_DEF
+    %10.sub3:vreg_512 = IMPLICIT_DEF
+    %11:vgpr_32 = IMPLICIT_DEF
+    %12:vgpr_32 = IMPLICIT_DEF
+    %13:vgpr_32 = IMPLICIT_DEF
+    %14:vgpr_32 = IMPLICIT_DEF
+    %15:vgpr_32 = IMPLICIT_DEF
+    %16:vgpr_32 = IMPLICIT_DEF
+    %17:vgpr_32 = IMPLICIT_DEF
+    %18:vgpr_32 = IMPLICIT_DEF
+    %19:vgpr_32 = IMPLICIT_DEF
+    %20:vgpr_32 = IMPLICIT_DEF
+    %21:vgpr_32 = IMPLICIT_DEF
+    %22:vgpr_32 = IMPLICIT_DEF
+    %23:vgpr_32 = IMPLICIT_DEF
+    %24:vgpr_32 = IMPLICIT_DEF
+    %25:vgpr_32 = IMPLICIT_DEF
+    %26:vgpr_32 = IMPLICIT_DEF
+    %27:vgpr_32 = IMPLICIT_DEF
+    %28:vgpr_32 = IMPLICIT_DEF
+    %29:vgpr_32 = IMPLICIT_DEF
+    %30:vgpr_32 = IMPLICIT_DEF
+    %31:vgpr_32 = IMPLICIT_DEF
+    %32:vgpr_32 = IMPLICIT_DEF
+    %33:vgpr_32 = IMPLICIT_DEF
+    %34:vgpr_32 = IMPLICIT_DEF
+    %35:vgpr_32 = IMPLICIT_DEF
+    %36:vgpr_32 = IMPLICIT_DEF
+    %37:vgpr_32 = IMPLICIT_DEF
+    %38:vgpr_32 = IMPLICIT_DEF
+    %39:vgpr_32 = IMPLICIT_DEF
+    %40:vgpr_32 = IMPLICIT_DEF
+    %41:vgpr_32 = IMPLICIT_DEF
+    undef %50.sub0:vreg_64 = nofpexcept V_CVT_I32_F32_e32 %10.sub0, implicit $exec, implicit $mode
+    %50.sub1:vreg_64 = nofpexcept V_CVT_I32_F32_e32 %11, implicit $exec, implicit $mode
+    %52:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 %12, implicit $exec, implicit $mode
+    %53:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 %13, implicit $exec, implicit $mode
+    %54:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 %14, implicit $exec, implicit $mode
+    %55:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 %15, implicit $exec, implicit $mode
+    %56:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 %16, implicit $exec, implicit $mode
+    %57:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 %17, implicit $exec, implicit $mode
+    %58:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 %18, implicit $exec, implicit $mode
+    %59:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 %19, implicit $exec, implicit $mode
+    %60:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 %10.sub2, implicit $exec, implicit $mode
+    %61:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 %21, implicit $exec, implicit $mode
+    %62:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 %22, implicit $exec, implicit $mode
+    %63:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 %23, implicit $exec, implicit $mode
+    %64:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 %24, implicit $exec, implicit $mode
+    %65:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 %25, implicit $exec, implicit $mode
+    %66:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 %26, implicit $exec, implicit $mode
+    %67:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 %27, implicit $exec, implicit $mode
+    %68:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 %28, implicit $exec, implicit $mode
+    %69:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 %29, implicit $exec, implicit $mode
+    %70:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 %30, implicit $exec, implicit $mode
+    %71:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 %31, implicit $exec, implicit $mode
+    %72:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 %32, implicit $exec, implicit $mode
+    %73:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 %33, implicit $exec, implicit $mode
+    %74:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 %34, implicit $exec, implicit $mode
+    %75:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 %35, implicit $exec, implicit $mode
+    %76:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 %36, implicit $exec, implicit $mode
+    %77:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 %37, implicit $exec, implicit $mode
+    %78:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 %38, implicit $exec, implicit $mode
+    %79:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 %39, implicit $exec, implicit $mode
+    %80:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 %40, implicit $exec, implicit $mode
+    %81:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 %41, implicit $exec, implicit $mode
+    S_BRANCH %bb.4
+
+  bb.4:
+    S_NOP 0, implicit %50.sub0, implicit %60, implicit %10.sub0, implicit %10.sub2
+    S_NOP 0, implicit %52, implicit %61, implicit %11, implicit %21
+    S_NOP 0, implicit %52, implicit %62, implicit %12, implicit %22
+    S_NOP 0, implicit %53, implicit %63, implicit %13, implicit %23
+    S_NOP 0, implicit %54, implicit %64, implicit %14, implicit %24
+    S_NOP 0, implicit %55, implicit %65
+    S_NOP 0, implicit %56, implicit %66
+    S_NOP 0, implicit %57, implicit %67
+    S_NOP 0, implicit %58, implicit %68
+    S_NOP 0, implicit %59, implicit %69
+    S_NOP 0, implicit %70, implicit %71
+    S_NOP 0, implicit %72, implicit %73
+    S_NOP 0, implicit %72, implicit %73
+    S_NOP 0, implicit %74, implicit %75
+    S_NOP 0, implicit %76, implicit %77
+    S_NOP 0, implicit %78, implicit %79
+    S_NOP 0, implicit %80
+    S_ENDPGM 0
+...
+
+# The remat candidate (instruction defining %3) uses a register (%2) which does not have a subrange for every subregister (sub1 is undefined).
+# Be sure that when checking if we can rematerialize %3, that we handle the liveness checking of %2 properly.
+
+---
+name:            omitted_subrange
+tracksRegLiveness: true
+body:             |
+  ; GFX908-LABEL: name: omitted_subrange
+  ; GFX908: bb.0:
+  ; GFX908-NEXT:   successors: %bb.1(0x80000000)
+  ; GFX908-NEXT:   liveins: $sgpr3, $sgpr4
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[DEF:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF1:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF2:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF3:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF4:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF5:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF6:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF7:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF8:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF9:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF10:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF11:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF12:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF13:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF14:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF15:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF16:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF17:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF18:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF19:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   dead [[V_CVT_I32_F32_e32_:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF19]], implicit $exec, implicit $mode
+  ; GFX908-NEXT:   [[DEF20:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[V_CVT_I32_F32_e32_1:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF]], implicit $exec, implicit $mode
+  ; GFX908-NEXT:   [[V_CVT_I32_F32_e32_2:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF1]], implicit $exec, implicit $mode
+  ; GFX908-NEXT:   [[V_CVT_I32_F32_e32_3:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF2]], implicit $exec, implicit $mode
+  ; GFX908-NEXT:   [[V_CVT_I32_F32_e32_4:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF3]], implicit $exec, implicit $mode
+  ; GFX908-NEXT:   [[V_CVT_I32_F32_e32_5:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF4]], implicit $exec, implicit $mode
+  ; GFX908-NEXT:   [[V_CVT_I32_F32_e32_6:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF5]], implicit $exec, implicit $mode
+  ; GFX908-NEXT:   [[V_CVT_I32_F32_e32_7:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF6]], implicit $exec, implicit $mode
+  ; GFX908-NEXT:   [[V_CVT_I32_F32_e32_8:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF7]], implicit $exec, implicit $mode
+  ; GFX908-NEXT:   [[V_CVT_I32_F32_e32_9:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF8]], implicit $exec, implicit $mode
+  ; GFX908-NEXT:   [[V_CVT_I32_F32_e32_10:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF9]], implicit $exec, implicit $mode
+  ; GFX908-NEXT:   [[V_CVT_I32_F32_e32_11:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF10]], implicit $exec, implicit $mode
+  ; GFX908-NEXT:   [[V_CVT_I32_F32_e32_12:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF11]], implicit $exec, implicit $mode
+  ; GFX908-NEXT:   [[V_CVT_I32_F32_e32_13:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF12]], implicit $exec, implicit $mode
+  ; GFX908-NEXT:   [[V_CVT_I32_F32_e32_14:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF13]], implicit $exec, implicit $mode
+  ; GFX908-NEXT:   [[V_CVT_I32_F32_e32_15:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF14]], implicit $exec, implicit $mode
+  ; GFX908-NEXT:   [[V_CVT_I32_F32_e32_16:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF15]], implicit $exec, implicit $mode
+  ; GFX908-NEXT:   [[V_CVT_I32_F32_e32_17:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF16]], implicit $exec, implicit $mode
+  ; GFX908-NEXT:   [[V_CVT_I32_F32_e32_18:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF17]], implicit $exec, implicit $mode
+  ; GFX908-NEXT:   [[V_CVT_I32_F32_e32_19:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF18]], implicit $exec, implicit $mode
+  ; GFX908-NEXT:   [[V_CVT_I32_F32_e32_20:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF20]], implicit $exec, implicit $mode
+  ; GFX908-NEXT:   [[DEF21:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[V_CVT_I32_F32_e32_21:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF21]], implicit $exec, implicit $mode
+  ; GFX908-NEXT:   [[DEF22:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF23:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF24:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF25:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF26:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF27:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF28:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF29:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF30:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF31:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[V_CVT_I32_F32_e32_22:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF26]], implicit $exec, implicit $mode
+  ; GFX908-NEXT:   [[V_CVT_I32_F32_e32_23:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF27]], implicit $exec, implicit $mode
+  ; GFX908-NEXT:   [[V_CVT_I32_F32_e32_24:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF28]], implicit $exec, implicit $mode
+  ; GFX908-NEXT:   [[V_CVT_I32_F32_e32_25:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF29]], implicit $exec, implicit $mode
+  ; GFX908-NEXT:   [[V_CVT_I32_F32_e32_26:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF30]], implicit $exec, implicit $mode
+  ; GFX908-NEXT:   [[DEF32:%[0-9]+]]:vreg_64_align2 = IMPLICIT_DEF
+  ; GFX908-NEXT:   undef [[V_RCP_F32_e32_:%[0-9]+]].sub0:vreg_64_align2 = nnan ninf nsz arcp contract afn reassoc nofpexcept V_RCP_F32_e32 [[DEF32]].sub0, implicit $mode, implicit $exec
+  ; GFX908-NEXT:   [[V_CVT_I32_F32_e32_27:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF31]], implicit $exec, implicit $mode
+  ; GFX908-NEXT:   dead [[DEF33:%[0-9]+]]:vreg_64 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF34:%[0-9]+]]:vreg_64_align2 = IMPLICIT_DEF
+  ; GFX908-NEXT:   S_BRANCH %bb.1
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.1:
+  ; GFX908-NEXT:   successors: %bb.2(0x40000000), %bb.3(0x40000000)
+  ; GFX908-NEXT:   liveins: $sgpr3, $sgpr4
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   %temp:vgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   S_CMP_LG_U32 $sgpr3, $sgpr4, implicit-def $scc
+  ; GFX908-NEXT:   [[DEF34:%[0-9]+]].sub0:vreg_64_align2 = nnan ninf nsz arcp contract afn reassoc nofpexcept V_MUL_F32_e64 0, [[DEF32]].sub0, 1, %temp, 0, 0, implicit $mode, implicit $exec
+  ; GFX908-NEXT:   S_CBRANCH_SCC1 %bb.2, implicit killed $scc
+  ; GFX908-NEXT:   S_BRANCH %bb.3
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.2:
+  ; GFX908-NEXT:   successors: %bb.3(0x80000000)
+  ; GFX908-NEXT:   liveins: $sgpr3, $sgpr4
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   undef [[V_FMA_F32_e64_:%[0-9]+]].sub0:vreg_64_align2 = nnan ninf nsz arcp contract afn reassoc nofpexcept V_FMA_F32_e64 0, [[DEF34]].sub1, 0, [[V_RCP_F32_e32_]].sub0, 0, [[DEF34]].sub0, 0, 0, implicit $mode, implicit $exec
+  ; GFX908-NEXT:   %temp2:vreg_64_align2 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[V_PK_MUL_F32_:%[0-9]+]]:vreg_64_align2 = nnan ninf nsz arcp contract afn reassoc nofpexcept V_PK_MUL_F32 0, [[V_RCP_F32_e32_]], 8, [[DEF32]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+  ; GFX908-NEXT:   dead [[V_PK_FMA_F32_:%[0-9]+]]:vreg_64_align2 = nnan ninf nsz arcp contract afn reassoc nofpexcept V_PK_FMA_F32 0, [[V_FMA_F32_e64_]], 8, %temp2, 11, [[V_PK_MUL_F32_]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+  ; GFX908-NEXT:   [[V_CVT_I32_F32_e32_28:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF22]], implicit $exec, implicit $mode
+  ; GFX908-NEXT:   S_NOP 0, implicit [[V_CVT_I32_F32_e32_28]], implicit [[V_CVT_I32_F32_e32_23]], implicit [[DEF22]], implicit [[DEF27]]
+  ; GFX908-NEXT:   [[V_CVT_I32_F32_e32_29:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF23]], implicit $exec, implicit $mode
+  ; GFX908-NEXT:   S_NOP 0, implicit [[V_CVT_I32_F32_e32_29]], implicit [[V_CVT_I32_F32_e32_24]], implicit [[DEF23]], implicit [[DEF28]]
+  ; GFX908-NEXT:   [[V_CVT_I32_F32_e32_30:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF24]], implicit $exec, implicit $mode
+  ; GFX908-NEXT:   S_NOP 0, implicit [[V_CVT_I32_F32_e32_30]], implicit [[V_CVT_I32_F32_e32_25]], implicit [[DEF24]], implicit [[DEF29]]
+  ; GFX908-NEXT:   [[V_CVT_I32_F32_e32_31:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF25]], implicit $exec, implicit $mode
+  ; GFX908-NEXT:   dead [[DEF35:%[0-9]+]]:vreg_64_align2 = IMPLICIT_DEF
+  ; GFX908-NEXT:   S_NOP 0, implicit [[V_CVT_I32_F32_e32_31]], implicit [[V_CVT_I32_F32_e32_26]], implicit [[DEF25]], implicit [[DEF30]]
+  ; GFX908-NEXT:   S_NOP 0, implicit [[V_CVT_I32_F32_e32_22]], implicit [[V_CVT_I32_F32_e32_27]], implicit [[DEF26]], implicit [[DEF31]]
+  ; GFX908-NEXT:   S_NOP 0, implicit [[V_CVT_I32_F32_e32_1]], implicit [[V_CVT_I32_F32_e32_6]], implicit [[DEF32]]
+  ; GFX908-NEXT:   S_NOP 0, implicit [[V_CVT_I32_F32_e32_2]], implicit [[V_CVT_I32_F32_e32_7]]
+  ; GFX908-NEXT:   S_NOP 0, implicit [[V_CVT_I32_F32_e32_3]], implicit [[V_CVT_I32_F32_e32_8]]
+  ; GFX908-NEXT:   S_NOP 0, implicit [[V_CVT_I32_F32_e32_4]], implicit [[V_CVT_I32_F32_e32_9]]
+  ; GFX908-NEXT:   S_NOP 0, implicit [[V_CVT_I32_F32_e32_5]], implicit [[V_CVT_I32_F32_e32_10]]
+  ; GFX908-NEXT:   S_NOP 0, implicit [[V_CVT_I32_F32_e32_11]], implicit [[V_CVT_I32_F32_e32_12]]
+  ; GFX908-NEXT:   S_NOP 0, implicit [[V_CVT_I32_F32_e32_13]], implicit [[V_CVT_I32_F32_e32_14]]
+  ; GFX908-NEXT:   S_NOP 0, implicit [[V_CVT_I32_F32_e32_13]], implicit [[V_CVT_I32_F32_e32_14]]
+  ; GFX908-NEXT:   S_NOP 0, implicit [[V_CVT_I32_F32_e32_15]], implicit [[V_CVT_I32_F32_e32_16]]
+  ; GFX908-NEXT:   S_NOP 0, implicit [[V_CVT_I32_F32_e32_17]], implicit [[V_CVT_I32_F32_e32_18]]
+  ; GFX908-NEXT:   S_NOP 0, implicit [[V_CVT_I32_F32_e32_19]], implicit [[V_CVT_I32_F32_e32_20]]
+  ; GFX908-NEXT:   S_NOP 0, implicit [[V_CVT_I32_F32_e32_21]]
+  ; GFX908-NEXT:   S_BRANCH %bb.3
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.3:
+  ; GFX908-NEXT:   successors: %bb.1(0x40000000), %bb.4(0x40000000)
+  ; GFX908-NEXT:   liveins: $sgpr3, $sgpr4
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   S_CMP_LG_U32 $sgpr4, 0, implicit-def $scc
+  ; GFX908-NEXT:   S_CBRANCH_SCC1 %bb.1, implicit killed $scc
+  ; GFX908-NEXT:   S_BRANCH %bb.4
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.4:
+  ; GFX908-NEXT:   S_ENDPGM 0
+  bb.0:
+    liveins: $sgpr3, $sgpr4
+
+    %0:vreg_64 = IMPLICIT_DEF
+    %1:vreg_64_align2 = IMPLICIT_DEF
+    undef %2.sub0:vreg_64_align2 = nnan ninf nsz arcp contract afn reassoc nofpexcept V_RCP_F32_e32 %1.sub0:vreg_64_align2, implicit $mode, implicit $exec
+    %3:vreg_64_align2 =  nnan ninf nsz arcp contract afn reassoc nofpexcept V_PK_MUL_F32 0, %2, 8, %1, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+    %5:vreg_64_align2 = IMPLICIT_DEF
+    %10:vgpr_32 = IMPLICIT_DEF
+    %11:vgpr_32 = IMPLICIT_DEF
+    %12:vgpr_32 = IMPLICIT_DEF
+    %13:vgpr_32 = IMPLICIT_DEF
+    %14:vgpr_32 = IMPLICIT_DEF
+    %15:vgpr_32 = IMPLICIT_DEF
+    %16:vgpr_32 = IMPLICIT_DEF
+    %17:vgpr_32 = IMPLICIT_DEF
+    %18:vgpr_32 = IMPLICIT_DEF
+    %19:vgpr_32 = IMPLICIT_DEF
+    %20:vgpr_32 = IMPLICIT_DEF
+    %21:vgpr_32 = IMPLICIT_DEF
+    %22:vgpr_32 = IMPLICIT_DEF
+    %23:vgpr_32 = IMPLICIT_DEF
+    %24:vgpr_32 = IMPLICIT_DEF
+    %25:vgpr_32 = IMPLICIT_DEF
+    %26:vgpr_32 = IMPLICIT_DEF
+    %27:vgpr_32 = IMPLICIT_DEF
+    %28:vgpr_32 = IMPLICIT_DEF
+    %29:vgpr_32 = IMPLICIT_DEF
+    %30:vgpr_32 = IMPLICIT_DEF
+    %31:vgpr_32 = IMPLICIT_DEF
+    %32:vgpr_32 = IMPLICIT_DEF
+    %33:vgpr_32 = IMPLICIT_DEF
+    %34:vgpr_32 = IMPLICIT_DEF
+    %35:vgpr_32 = IMPLICIT_DEF
+    %36:vgpr_32 = IMPLICIT_DEF
+    %37:vgpr_32 = IMPLICIT_DEF
+    %38:vgpr_32 = IMPLICIT_DEF
+    %39:vgpr_32 = IMPLICIT_DEF
+    %40:vgpr_32 = IMPLICIT_DEF
+    %41:vgpr_32 = IMPLICIT_DEF
+    %50:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 %10, implicit $exec, implicit $mode
+    %51:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 %11, implicit $exec, implicit $mode
+    %52:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 %12, implicit $exec, implicit $mode
+    %53:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 %13, implicit $exec, implicit $mode
+    %54:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 %14, implicit $exec, implicit $mode
+    %55:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 %15, implicit $exec, implicit $mode
+    %56:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 %16, implicit $exec, implicit $mode
+    %57:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 %17, implicit $exec, implicit $mode
+    %58:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 %18, implicit $exec, implicit $mode
+    %59:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 %19, implicit $exec, implicit $mode
+    %60:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 %20, implicit $exec, implicit $mode
+    %61:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 %21, implicit $exec, implicit $mode
+    %62:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 %22, implicit $exec, implicit $mode
+    %63:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 %23, implicit $exec, implicit $mode
+    %64:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 %24, implicit $exec, implicit $mode
+    %65:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 %25, implicit $exec, implicit $mode
+    %66:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 %26, implicit $exec, implicit $mode
+    %67:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 %27, implicit $exec, implicit $mode
+    %68:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 %28, implicit $exec, implicit $mode
+    %69:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 %29, implicit $exec, implicit $mode
+    %70:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 %30, implicit $exec, implicit $mode
+    %71:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 %31, implicit $exec, implicit $mode
+    %72:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 %32, implicit $exec, implicit $mode
+    %73:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 %33, implicit $exec, implicit $mode
+    %74:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 %34, implicit $exec, implicit $mode
+    %75:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 %35, implicit $exec, implicit $mode
+    %76:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 %36, implicit $exec, implicit $mode
+    %77:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 %37, implicit $exec, implicit $mode
+    %78:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 %38, implicit $exec, implicit $mode
+    %79:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 %39, implicit $exec, implicit $mode
+    %80:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 %40, implicit $exec, implicit $mode
+    %81:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 %41, implicit $exec, implicit $mode
+    S_BRANCH %bb.1
+
+  bb.1:
+    liveins: $sgpr3, $sgpr4
+
+    %temp:vgpr_32 = IMPLICIT_DEF
+    %5.sub0:vreg_64_align2 = nnan ninf nsz arcp contract afn reassoc nofpexcept V_MUL_F32_e64 0, %1.sub0:vreg_64_align2, 1, %temp, 0, 0, implicit $mode, implicit $exec
+
+    S_CMP_LG_U32 killed $sgpr3, $sgpr4, implicit-def $scc
+    S_CBRANCH_SCC1 %bb.3, implicit killed $scc
+    S_BRANCH %bb.4
+
+  bb.3:
+    liveins: $sgpr3, $sgpr4
+
+    %6:vreg_64_align2 = IMPLICIT_DEF
+    undef %7.sub0:vreg_64_align2 = nnan ninf nsz arcp contract afn reassoc nofpexcept V_FMA_F32_e64 0, %5.sub1, 0, %2.sub0:vreg_64_align2, 0, %5.sub0:vreg_64_align2, 0, 0, implicit $mode, implicit $exec
+    %temp2:vreg_64_align2 = IMPLICIT_DEF
+    %8:vreg_64_align2 = nnan ninf nsz arcp contract afn reassoc nofpexcept V_PK_FMA_F32 0, %7:vreg_64_align2, 8, %temp2, 11, %3:vreg_64_align2, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+    S_NOP 0, implicit %50, implicit %60, implicit %10, implicit %20
+    S_NOP 0, implicit %51, implicit %61, implicit %11, implicit %21
+    S_NOP 0, implicit %52, implicit %62, implicit %12, implicit %22
+    S_NOP 0, implicit %53, implicit %63, implicit %13, implicit %23
+    S_NOP 0, implicit %54, implicit %64, implicit %14, implicit %24
+    S_NOP 0, implicit %55, implicit %65, implicit %1
+    S_NOP 0, implicit %56, implicit %66
+    S_NOP 0, implicit %57, implicit %67
+    S_NOP 0, implicit %58, implicit %68
+    S_NOP 0, implicit %59, implicit %69
+    S_NOP 0, implicit %70, implicit %71
+    S_NOP 0, implicit %72, implicit %73
+    S_NOP 0, implicit %72, implicit %73
+    S_NOP 0, implicit %74, implicit %75
+    S_NOP 0, implicit %76, implicit %77
+    S_NOP 0, implicit %78, implicit %79
+    S_NOP 0, implicit %80
+    S_BRANCH %bb.4
+
+  bb.4:
+    liveins: $sgpr3, $sgpr4
+
+    S_CMP_LG_U32 $sgpr4, 0, implicit-def $scc
+    S_CBRANCH_SCC1 %bb.1, implicit killed $scc
+    S_BRANCH %bb.2
+
+  bb.2:
+    S_ENDPGM 0
+
+...
diff --git a/llvm/test/CodeGen/AMDGPU/machine-sink-temporal-divergence-swdev407790.ll b/llvm/test/CodeGen/AMDGPU/machine-sink-temporal-divergence-swdev407790.ll
index c826980991f94..acb706cee04d0 100644
--- a/llvm/test/CodeGen/AMDGPU/machine-sink-temporal-divergence-swdev407790.ll
+++ b/llvm/test/CodeGen/AMDGPU/machine-sink-temporal-divergence-swdev407790.ll
@@ -54,6 +54,9 @@ define protected amdgpu_kernel void @kernel_round1(ptr addrspace(1) nocapture no
 ; CHECK-NEXT:    s_mov_b32 s33, s16
 ; CHECK-NEXT:    s_addc_u32 s45, s35, 0
 ; CHECK-NEXT:    s_mov_b32 s43, s14
+; CHECK-NEXT:    s_getpc_b64 s[16:17]
+; CHECK-NEXT:    s_add_u32 s16, s16, _Z13get_global_idj@rel32@lo+4
+; CHECK-NEXT:    s_addc_u32 s17, s17, _Z13get_global_idj@rel32@hi+12
 ; CHECK-NEXT:    s_mov_b64 s[8:9], s[44:45]
 ; CHECK-NEXT:    s_mov_b32 s12, s14
 ; CHECK-NEXT:    s_mov_b32 s13, s15
@@ -62,14 +65,14 @@ define protected amdgpu_kernel void @kernel_round1(ptr addrspace(1) nocapture no
 ; CHECK-NEXT:    s_mov_b64 s[36:37], s[10:11]
 ; CHECK-NEXT:    s_mov_b64 s[38:39], s[6:7]
 ; CHECK-NEXT:    s_mov_b64 s[40:41], s[4:5]
-; CHECK-NEXT:    s_getpc_b64 s[16:17]
-; CHECK-NEXT:    s_add_u32 s16, s16, _Z13get_global_idj@rel32@lo+4
-; CHECK-NEXT:    s_addc_u32 s17, s17, _Z13get_global_idj@rel32@hi+12
 ; CHECK-NEXT:    v_mov_b32_e32 v45, 0
 ; CHECK-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; CHECK-NEXT:    v_mov_b32_e32 v43, v0
 ; CHECK-NEXT:    v_mov_b32_e32 v31, v40
 ; CHECK-NEXT:    v_mov_b32_e32 v0, 0
+; CHECK-NEXT:    s_getpc_b64 s[16:17]
+; CHECK-NEXT:    s_add_u32 s16, s16, _Z12get_local_idj@rel32@lo+4
+; CHECK-NEXT:    s_addc_u32 s17, s17, _Z12get_local_idj@rel32@hi+12
 ; CHECK-NEXT:    s_mov_b64 s[4:5], s[40:41]
 ; CHECK-NEXT:    s_mov_b64 s[6:7], s[38:39]
 ; CHECK-NEXT:    s_mov_b64 s[8:9], s[44:45]
@@ -77,13 +80,13 @@ define protected amdgpu_kernel void @kernel_round1(ptr addrspace(1) nocapture no
 ; CHECK-NEXT:    s_mov_b32 s12, s43
 ; CHECK-NEXT:    s_mov_b32 s13, s42
 ; CHECK-NEXT:    s_mov_b32 s14, s33
-; CHECK-NEXT:    s_getpc_b64 s[16:17]
-; CHECK-NEXT:    s_add_u32 s16, s16, _Z12get_local_idj@rel32@lo+4
-; CHECK-NEXT:    s_addc_u32 s17, s17, _Z12get_local_idj@rel32@hi+12
 ; CHECK-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; CHECK-NEXT:    v_mov_b32_e32 v41, v0
 ; CHECK-NEXT:    v_mov_b32_e32 v31, v40
 ; CHECK-NEXT:    v_mov_b32_e32 v0, 1
+; CHECK-NEXT:    s_getpc_b64 s[16:17]
+; CHECK-NEXT:    s_add_u32 s16, s16, _Z7barrierj@rel32@lo+4
+; CHECK-NEXT:    s_addc_u32 s17, s17, _Z7barrierj@rel32@hi+12
 ; CHECK-NEXT:    s_mov_b64 s[4:5], s[40:41]
 ; CHECK-NEXT:    s_mov_b64 s[6:7], s[38:39]
 ; CHECK-NEXT:    s_mov_b64 s[8:9], s[44:45]
@@ -92,26 +95,23 @@ define protected amdgpu_kernel void @kernel_round1(ptr addrspace(1) nocapture no
 ; CHECK-NEXT:    s_mov_b32 s13, s42
 ; CHECK-NEXT:    s_mov_b32 s14, s33
 ; CHECK-NEXT:    ds_write_b32 v45, v45 offset:15360
-; CHECK-NEXT:    s_getpc_b64 s[16:17]
-; CHECK-NEXT:    s_add_u32 s16, s16, _Z7barrierj@rel32@lo+4
-; CHECK-NEXT:    s_addc_u32 s17, s17, _Z7barrierj@rel32@hi+12
 ; CHECK-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; CHECK-NEXT:    v_lshrrev_b32_e32 v0, 1, v43
 ; CHECK-NEXT:    v_lshlrev_b32_e32 v1, 2, v43
 ; CHECK-NEXT:    v_mov_b32_e32 v31, v40
+; CHECK-NEXT:    s_getpc_b64 s[16:17]
+; CHECK-NEXT:    s_add_u32 s16, s16, _Z3minjj@rel32@lo+4
+; CHECK-NEXT:    s_addc_u32 s17, s17, _Z3minjj@rel32@hi+12
 ; CHECK-NEXT:    s_mov_b64 s[4:5], s[40:41]
-; CHECK-NEXT:    s_mov_b64 s[6:7], s[38:39]
 ; CHECK-NEXT:    v_and_b32_e32 v0, 0x7ffffffc, v0
 ; CHECK-NEXT:    v_and_b32_e32 v1, 28, v1
+; CHECK-NEXT:    s_mov_b64 s[6:7], s[38:39]
 ; CHECK-NEXT:    s_mov_b64 s[8:9], s[44:45]
 ; CHECK-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; CHECK-NEXT:    s_mov_b32 s12, s43
 ; CHECK-NEXT:    global_load_dword v0, v0, s[52:53]
+; CHECK-NEXT:    s_mov_b32 s12, s43
 ; CHECK-NEXT:    s_mov_b32 s13, s42
 ; CHECK-NEXT:    s_mov_b32 s14, s33
-; CHECK-NEXT:    s_getpc_b64 s[16:17]
-; CHECK-NEXT:    s_add_u32 s16, s16, _Z3minjj@rel32@lo+4
-; CHECK-NEXT:    s_addc_u32 s17, s17, _Z3minjj@rel32@hi+12
 ; CHECK-NEXT:    s_waitcnt vmcnt(0)
 ; CHECK-NEXT:    v_bfe_u32 v0, v0, v1, 4
 ; CHECK-NEXT:    v_mov_b32_e32 v1, 12
@@ -190,6 +190,9 @@ define protected amdgpu_kernel void @kernel_round1(ptr addrspace(1) nocapture no
 ; CHECK-NEXT:    v_mov_b32_e32 v0, 0x3c00
 ; CHECK-NEXT:    s_add_u32 s8, s34, 40
 ; CHECK-NEXT:    s_addc_u32 s9, s35, 0
+; CHECK-NEXT:    s_getpc_b64 s[16:17]
+; CHECK-NEXT:    s_add_u32 s16, s16, _Z10atomic_incPU3AS3Vj@rel32@lo+4
+; CHECK-NEXT:    s_addc_u32 s17, s17, _Z10atomic_incPU3AS3Vj@rel32@hi+12
 ; CHECK-NEXT:    s_mov_b64 s[4:5], s[40:41]
 ; CHECK-NEXT:    s_mov_b64 s[6:7], s[38:39]
 ; CHECK-NEXT:    s_mov_b64 s[10:11], s[36:37]
@@ -197,9 +200,6 @@ define protected amdgpu_kernel void @kernel_round1(ptr addrspace(1) nocapture no
 ; CHECK-NEXT:    s_mov_b32 s13, s42
 ; CHECK-NEXT:    s_mov_b32 s14, s33
 ; CHECK-NEXT:    v_add_nc_u32_e32 v47, 1, v47
-; CHECK-NEXT:    s_getpc_b64 s[16:17]
-; CHECK-NEXT:    s_add_u32 s16, s16, _Z10atomic_incPU3AS3Vj@rel32@lo+4
-; CHECK-NEXT:    s_addc_u32 s17, s17, _Z10atomic_incPU3AS3Vj@rel32@hi+12
 ; CHECK-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; CHECK-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
 ; CHECK-NEXT:    ds_write_b32 v0, v58
@@ -215,6 +215,9 @@ define protected amdgpu_kernel void @kernel_round1(ptr addrspace(1) nocapture no
 ; CHECK-NEXT:    v_mov_b32_e32 v0, 0x3c00
 ; CHECK-NEXT:    s_add_u32 s8, s34, 40
 ; CHECK-NEXT:    s_addc_u32 s9, s35, 0
+; CHECK-NEXT:    s_getpc_b64 s[16:17]
+; CHECK-NEXT:    s_add_u32 s16, s16, _Z10atomic_incPU3AS3Vj@rel32@lo+4
+; CHECK-NEXT:    s_addc_u32 s17, s17, _Z10atomic_incPU3AS3Vj@rel32@hi+12
 ; CHECK-NEXT:    s_mov_b64 s[4:5], s[40:41]
 ; CHECK-NEXT:    s_mov_b64 s[6:7], s[38:39]
 ; CHECK-NEXT:    s_mov_b64 s[10:11], s[36:37]
@@ -223,9 +226,6 @@ define protected amdgpu_kernel void @kernel_round1(ptr addrspace(1) nocapture no
 ; CHECK-NEXT:    s_mov_b32 s14, s33
 ; CHECK-NEXT:    v_add_nc_u32_e32 v60, 1, v58
 ; CHECK-NEXT:    v_add_nc_u32_e32 v47, 1, v47
-; CHECK-NEXT:    s_getpc_b64 s[16:17]
-; CHECK-NEXT:    s_add_u32 s16, s16, _Z10atomic_incPU3AS3Vj@rel32@lo+4
-; CHECK-NEXT:    s_addc_u32 s17, s17, _Z10atomic_incPU3AS3Vj@rel32@hi+12
 ; CHECK-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; CHECK-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
 ; CHECK-NEXT:    ds_write_b32 v0, v60
@@ -241,6 +241,9 @@ define protected amdgpu_kernel void @kernel_round1(ptr addrspace(1) nocapture no
 ; CHECK-NEXT:    v_mov_b32_e32 v0, 0x3c00
 ; CHECK-NEXT:    s_add_u32 s8, s34, 40
 ; CHECK-NEXT:    s_addc_u32 s9, s35, 0
+; CHECK-NEXT:    s_getpc_b64 s[16:17]
+; CHECK-NEXT:    s_add_u32 s16, s16, _Z10atomic_incPU3AS3Vj@rel32@lo+4
+; CHECK-NEXT:    s_addc_u32 s17, s17, _Z10atomic_incPU3AS3Vj@rel32@hi+12
 ; CHECK-NEXT:    s_mov_b64 s[4:5], s[40:41]
 ; CHECK-NEXT:    s_mov_b64 s[6:7], s[38:39]
 ; CHECK-NEXT:    s_mov_b64 s[10:11], s[36:37]
@@ -249,9 +252,6 @@ define protected amdgpu_kernel void @kernel_round1(ptr addrspace(1) nocapture no
 ; CHECK-NEXT:    s_mov_b32 s14, s33
 ; CHECK-NEXT:    v_add_nc_u32_e32 v60, 2, v58
 ; CHECK-NEXT:    v_add_nc_u32_e32 v47, 1, v47
-; CHECK-NEXT:    s_getpc_b64 s[16:17]
-; CHECK-NEXT:    s_add_u32 s16, s16, _Z10atomic_incPU3AS3Vj@rel32@lo+4
-; CHECK-NEXT:    s_addc_u32 s17, s17, _Z10atomic_incPU3AS3Vj@rel32@hi+12
 ; CHECK-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; CHECK-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
 ; CHECK-NEXT:    ds_write_b32 v0, v60
@@ -267,6 +267,9 @@ define protected amdgpu_kernel void @kernel_round1(ptr addrspace(1) nocapture no
 ; CHECK-NEXT:    v_mov_b32_e32 v0, 0x3c00
 ; CHECK-NEXT:    s_add_u32 s8, s34, 40
 ; CHECK-NEXT:    s_addc_u32 s9, s35, 0
+; CHECK-NEXT:    s_getpc_b64 s[16:17]
+; CHECK-NEXT:    s_add_u32 s16, s16, _Z10atomic_incPU3AS3Vj@rel32@lo+4
+; CHECK-NEXT:    s_addc_u32 s17, s17, _Z10atomic_incPU3AS3Vj@rel32@hi+12
 ; CHECK-NEXT:    s_mov_b64 s[4:5], s[40:41]
 ; CHECK-NEXT:    s_mov_b64 s[6:7], s[38:39]
 ; CHECK-NEXT:    s_mov_b64 s[10:11], s[36:37]
@@ -275,9 +278,6 @@ define protected amdgpu_kernel void @kernel_round1(ptr addrspace(1) nocapture no
 ; CHECK-NEXT:    s_mov_b32 s14, s33
 ; CHECK-NEXT:    v_add_nc_u32_e32 v58, 3, v58
 ; CHECK-NEXT:    v_add_nc_u32_e32 v47, 1, v47
-; CHECK-NEXT:    s_getpc_b64 s[16:17]
-; CHECK-NEXT:    s_add_u32 s16, s16, _Z10atomic_incPU3AS3Vj@rel32@lo+4
-; CHECK-NEXT:    s_addc_u32 s17, s17, _Z10atomic_incPU3AS3Vj@rel32@hi+12
 ; CHECK-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; CHECK-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
 ; CHECK-NEXT:    ds_write_b32 v0, v58
@@ -319,6 +319,9 @@ define protected amdgpu_kernel void @kernel_round1(ptr addrspace(1) nocapture no
 ; CHECK-NEXT:    v_mov_b32_e32 v0, 0x3c00
 ; CHECK-NEXT:    s_add_u32 s8, s34, 40
 ; CHECK-NEXT:    s_addc_u32 s9, s35, 0
+; CHECK-NEXT:    s_getpc_b64 s[16:17]
+; CHECK-NEXT:    s_add_u32 s16, s16, _Z10atomic_incPU3AS3Vj@rel32@lo+4
+; CHECK-NEXT:    s_addc_u32 s17, s17, _Z10atomic_incPU3AS3Vj@rel32@hi+12
 ; CHECK-NEXT:    s_mov_b64 s[4:5], s[40:41]
 ; CHECK-NEXT:    s_mov_b64 s[6:7], s[38:39]
 ; CHECK-NEXT:    s_mov_b64 s[10:11], s[36:37]
@@ -326,9 +329,6 @@ define protected amdgpu_kernel void @kernel_round1(ptr addrspace(1) nocapture no
 ; CHECK-NEXT:    s_mov_b32 s13, s42
 ; CHECK-NEXT:    s_mov_b32 s14, s33
 ; CHECK-NEXT:    v_add_nc_u32_e32 v47, 1, v47
-; CHECK-NEXT:    s_getpc_b64 s[16:17]
-; CHECK-NEXT:    s_add_u32 s16, s16, _Z10atomic_incPU3AS3Vj@rel32@lo+4
-; CHECK-NEXT:    s_addc_u32 s17, s17, _Z10atomic_incPU3AS3Vj@rel32@hi+12
 ; CHECK-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; CHECK-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
 ; CHECK-NEXT:    ds_write_b32 v0, v57
@@ -356,15 +356,15 @@ define protected amdgpu_kernel void @kernel_round1(ptr addrspace(1) nocapture no
 ; CHECK-NEXT:    v_mov_b32_e32 v0, 1
 ; CHECK-NEXT:    s_add_u32 s8, s34, 40
 ; CHECK-NEXT:    s_addc_u32 s9, s35, 0
+; CHECK-NEXT:    s_getpc_b64 s[16:17]
+; CHECK-NEXT:    s_add_u32 s16, s16, _Z7barrierj@rel32@lo+4
+; CHECK-NEXT:    s_addc_u32 s17, s17, _Z7barrierj@rel32@hi+12
 ; CHECK-NEXT:    s_mov_b64 s[4:5], s[40:41]
 ; CHECK-NEXT:    s_mov_b64 s[6:7], s[38:39]
 ; CHECK-NEXT:    s_mov_b64 s[10:11], s[36:37]
 ; CHECK-NEXT:    s_mov_b32 s12, s43
 ; CHECK-NEXT:    s_mov_b32 s13, s42
 ; CHECK-NEXT:    s_mov_b32 s14, s33
-; CHECK-NEXT:    s_getpc_b64 s[16:17]
-; CHECK-NEXT:    s_add_u32 s16, s16, _Z7barrierj@rel32@lo+4
-; CHECK-NEXT:    s_addc_u32 s17, s17, _Z7barrierj@rel32@hi+12
 ; CHECK-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; CHECK-NEXT:    v_mov_b32_e32 v0, 0
 ; CHECK-NEXT:    s_mov_b32 s4, exec_lo
@@ -381,15 +381,15 @@ define protected amdgpu_kernel void @kernel_round1(ptr addrspace(1) nocapture no
 ; CHECK-NEXT:    v_mov_b32_e32 v0, 0
 ; CHECK-NEXT:    s_add_u32 s8, s34, 40
 ; CHECK-NEXT:    s_addc_u32 s9, s35, 0
+; CHECK-NEXT:    s_getpc_b64 s[16:17]
+; CHECK-NEXT:    s_add_u32 s16, s16, _Z14get_local_sizej@rel32@lo+4
+; CHECK-NEXT:    s_addc_u32 s17, s17, _Z14get_local_sizej@rel32@hi+12
 ; CHECK-NEXT:    s_mov_b64 s[4:5], s[40:41]
 ; CHECK-NEXT:    s_mov_b64 s[6:7], s[38:39]
 ; CHECK-NEXT:    s_mov_b64 s[10:11], s[36:37]
 ; CHECK-NEXT:    s_mov_b32 s12, s43
 ; CHECK-NEXT:    s_mov_b32 s13, s42
 ; CHECK-NEXT:    s_mov_b32 s14, s33
-; CHECK-NEXT:    s_getpc_b64 s[16:17]
-; CHECK-NEXT:    s_add_u32 s16, s16, _Z14get_local_sizej@rel32@lo+4
-; CHECK-NEXT:    s_addc_u32 s17, s17, _Z14get_local_sizej@rel32@hi+12
 ; CHECK-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; CHECK-NEXT:    v_add_co_u32 v41, vcc_lo, v0, v41
 ; CHECK-NEXT:    v_cmp_le_u32_e32 vcc_lo, v47, v41
@@ -439,16 +439,16 @@ define protected amdgpu_kernel void @kernel_round1(ptr addrspace(1) nocapture no
 ; CHECK-NEXT:    v_and_b32_e32 v0, 0xf0, v0
 ; CHECK-NEXT:    v_and_b32_e32 v1, 15, v1
 ; CHECK-NEXT:    s_addc_u32 s9, s35, 0
-; CHECK-NEXT:    s_mov_b64 s[6:7], s[38:39]
+; CHECK-NEXT:    s_getpc_b64 s[16:17]
+; CHECK-NEXT:    s_add_u32 s16, s16, _Z10atomic_addPU3AS1Vjj@rel32@lo+4
+; CHECK-NEXT:    s_addc_u32 s17, s17, _Z10atomic_addPU3AS1Vjj@rel32@hi+12
 ; CHECK-NEXT:    v_or3_b32 v2, v3, v2, v4
+; CHECK-NEXT:    s_mov_b64 s[6:7], s[38:39]
 ; CHECK-NEXT:    s_mov_b64 s[10:11], s[36:37]
 ; CHECK-NEXT:    s_mov_b32 s12, s43
 ; CHECK-NEXT:    s_mov_b32 s13, s42
-; CHECK-NEXT:    s_mov_b32 s14, s33
 ; CHECK-NEXT:    v_or3_b32 v73, v2, v0, v1
-; CHECK-NEXT:    s_getpc_b64 s[16:17]
-; CHECK-NEXT:    s_add_u32 s16, s16, _Z10atomic_addPU3AS1Vjj@rel32@lo+4
-; CHECK-NEXT:    s_addc_u32 s17, s17, _Z10atomic_addPU3AS1Vjj@rel32@hi+12
+; CHECK-NEXT:    s_mov_b32 s14, s33
 ; CHECK-NEXT:    v_lshrrev_b32_e32 v0, 1, v73
 ; CHECK-NEXT:    v_lshlrev_b32_e32 v1, 2, v73
 ; CHECK-NEXT:    v_and_b32_e32 v0, 0x7fffc, v0
@@ -500,15 +500,15 @@ define protected amdgpu_kernel void @kernel_round1(ptr addrspace(1) nocapture no
 ; CHECK-NEXT:    v_mov_b32_e32 v2, v44
 ; CHECK-NEXT:    s_add_u32 s8, s34, 40
 ; CHECK-NEXT:    s_addc_u32 s9, s35, 0
+; CHECK-NEXT:    s_getpc_b64 s[16:17]
+; CHECK-NEXT:    s_add_u32 s16, s16, _Z10atomic_subPU3AS1Vjj@rel32@lo+4
+; CHECK-NEXT:    s_addc_u32 s17, s17, _Z10atomic_subPU3AS1Vjj@rel32@hi+12
 ; CHECK-NEXT:    s_mov_b64 s[4:5], s[40:41]
 ; CHECK-NEXT:    s_mov_b64 s[6:7], s[38:39]
 ; CHECK-NEXT:    s_mov_b64 s[10:11], s[36:37]
 ; CHECK-NEXT:    s_mov_b32 s12, s43
 ; CHECK-NEXT:    s_mov_b32 s13, s42
 ; CHECK-NEXT:    s_mov_b32 s14, s33
-; CHECK-NEXT:    s_getpc_b64 s[16:17]
-; CHECK-NEXT:    s_add_u32 s16, s16, _Z10atomic_subPU3AS1Vjj@rel32@lo+4
-; CHECK-NEXT:    s_addc_u32 s17, s17, _Z10atomic_subPU3AS1Vjj@rel32@hi+12
 ; CHECK-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; CHECK-NEXT:    s_branch .LBB0_27
 ; CHECK-NEXT:  .LBB0_33:
@@ -803,6 +803,9 @@ define protected amdgpu_kernel void @kernel_round1_short(ptr addrspace(1) nocapt
 ; CHECK-NEXT:    s_mov_b32 s33, s16
 ; CHECK-NEXT:    s_addc_u32 s45, s39, 0
 ; CHECK-NEXT:    s_mov_b32 s43, s14
+; CHECK-NEXT:    s_getpc_b64 s[16:17]
+; CHECK-NEXT:    s_add_u32 s16, s16, _Z13get_global_idj@rel32@lo+4
+; CHECK-NEXT:    s_addc_u32 s17, s17, _Z13get_global_idj@rel32@hi+12
 ; CHECK-NEXT:    s_mov_b64 s[8:9], s[44:45]
 ; CHECK-NEXT:    s_mov_b32 s12, s14
 ; CHECK-NEXT:    s_mov_b32 s13, s15
@@ -811,14 +814,14 @@ define protected amdgpu_kernel void @kernel_round1_short(ptr addrspace(1) nocapt
 ; CHECK-NEXT:    s_mov_b64 s[34:35], s[10:11]
 ; CHECK-NEXT:    s_mov_b64 s[36:37], s[6:7]
 ; CHECK-NEXT:    s_mov_b64 s[40:41], s[4:5]
-; CHECK-NEXT:    s_getpc_b64 s[16:17]
-; CHECK-NEXT:    s_add_u32 s16, s16, _Z13get_global_idj@rel32@lo+4
-; CHECK-NEXT:    s_addc_u32 s17, s17, _Z13get_global_idj@rel32@hi+12
 ; CHECK-NEXT:    v_mov_b32_e32 v43, 0
 ; CHECK-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; CHECK-NEXT:    v_mov_b32_e32 v42, v0
 ; CHECK-NEXT:    v_mov_b32_e32 v31, v40
 ; CHECK-NEXT:    v_mov_b32_e32 v0, 0
+; CHECK-NEXT:    s_getpc_b64 s[16:17]
+; CHECK-NEXT:    s_add_u32 s16, s16, _Z12get_local_idj@rel32@lo+4
+; CHECK-NEXT:    s_addc_u32 s17, s17, _Z12get_local_idj@rel32@hi+12
 ; CHECK-NEXT:    s_mov_b64 s[4:5], s[40:41]
 ; CHECK-NEXT:    s_mov_b64 s[6:7], s[36:37]
 ; CHECK-NEXT:    s_mov_b64 s[8:9], s[44:45]
@@ -826,13 +829,13 @@ define protected amdgpu_kernel void @kernel_round1_short(ptr addrspace(1) nocapt
 ; CHECK-NEXT:    s_mov_b32 s12, s43
 ; CHECK-NEXT:    s_mov_b32 s13, s42
 ; CHECK-NEXT:    s_mov_b32 s14, s33
-; CHECK-NEXT:    s_getpc_b64 s[16:17]
-; CHECK-NEXT:    s_add_u32 s16, s16, _Z12get_local_idj@rel32@lo+4
-; CHECK-NEXT:    s_addc_u32 s17, s17, _Z12get_local_idj@rel32@hi+12
 ; CHECK-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; CHECK-NEXT:    v_mul_lo_u32 v46, v0, 14
 ; CHECK-NEXT:    v_mov_b32_e32 v31, v40
 ; CHECK-NEXT:    v_mov_b32_e32 v0, 1
+; CHECK-NEXT:    s_getpc_b64 s[16:17]
+; CHECK-NEXT:    s_add_u32 s16, s16, _Z7barrierj@rel32@lo+4
+; CHECK-NEXT:    s_addc_u32 s17, s17, _Z7barrierj@rel32@hi+12
 ; CHECK-NEXT:    s_mov_b64 s[4:5], s[40:41]
 ; CHECK-NEXT:    s_mov_b64 s[6:7], s[36:37]
 ; CHECK-NEXT:    s_mov_b64 s[8:9], s[44:45]
@@ -841,27 +844,24 @@ define protected amdgpu_kernel void @kernel_round1_short(ptr addrspace(1) nocapt
 ; CHECK-NEXT:    s_mov_b32 s13, s42
 ; CHECK-NEXT:    s_mov_b32 s14, s33
 ; CHECK-NEXT:    ds_write_b32 v43, v43 offset:15360
-; CHECK-NEXT:    s_getpc_b64 s[16:17]
-; CHECK-NEXT:    s_add_u32 s16, s16, _Z7barrierj@rel32@lo+4
-; CHECK-NEXT:    s_addc_u32 s17, s17, _Z7barrierj@rel32@hi+12
 ; CHECK-NEXT:    v_add_nc_u32_e32 v44, 0x3c04, v46
 ; CHECK-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; CHECK-NEXT:    v_lshrrev_b32_e32 v0, 1, v42
 ; CHECK-NEXT:    v_lshlrev_b32_e32 v1, 2, v42
 ; CHECK-NEXT:    v_mov_b32_e32 v31, v40
+; CHECK-NEXT:    s_getpc_b64 s[16:17]
+; CHECK-NEXT:    s_add_u32 s16, s16, _Z3minjj@rel32@lo+4
+; CHECK-NEXT:    s_addc_u32 s17, s17, _Z3minjj@rel32@hi+12
 ; CHECK-NEXT:    s_mov_b64 s[4:5], s[40:41]
-; CHECK-NEXT:    s_mov_b64 s[6:7], s[36:37]
 ; CHECK-NEXT:    v_and_b32_e32 v0, 0x7ffffffc, v0
 ; CHECK-NEXT:    v_and_b32_e32 v1, 28, v1
+; CHECK-NEXT:    s_mov_b64 s[6:7], s[36:37]
 ; CHECK-NEXT:    s_mov_b64 s[8:9], s[44:45]
 ; CHECK-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; CHECK-NEXT:    s_mov_b32 s12, s43
 ; CHECK-NEXT:    global_load_dword v0, v0, s[46:47]
+; CHECK-NEXT:    s_mov_b32 s12, s43
 ; CHECK-NEXT:    s_mov_b32 s13, s42
 ; CHECK-NEXT:    s_mov_b32 s14, s33
-; CHECK-NEXT:    s_getpc_b64 s[16:17]
-; CHECK-NEXT:    s_add_u32 s16, s16, _Z3minjj@rel32@lo+4
-; CHECK-NEXT:    s_addc_u32 s17, s17, _Z3minjj@rel32@hi+12
 ; CHECK-NEXT:    s_waitcnt vmcnt(0)
 ; CHECK-NEXT:    v_bfe_u32 v0, v0, v1, 4
 ; CHECK-NEXT:    v_mov_b32_e32 v1, 12
@@ -945,6 +945,9 @@ define protected amdgpu_kernel void @kernel_round1_short(ptr addrspace(1) nocapt
 ; CHECK-NEXT:    v_mov_b32_e32 v0, 0x3c00
 ; CHECK-NEXT:    s_add_u32 s8, s38, 40
 ; CHECK-NEXT:    s_addc_u32 s9, s39, 0
+; CHECK-NEXT:    s_getpc_b64 s[16:17]
+; CHECK-NEXT:    s_add_u32 s16, s16, _Z10atomic_incPU3AS3Vj@rel32@lo+4
+; CHECK-NEXT:    s_addc_u32 s17, s17, _Z10atomic_incPU3AS3Vj@rel32@hi+12
 ; CHECK-NEXT:    s_mov_b64 s[4:5], s[40:41]
 ; CHECK-NEXT:    s_mov_b64 s[6:7], s[36:37]
 ; CHECK-NEXT:    s_mov_b64 s[10:11], s[34:35]
@@ -952,9 +955,6 @@ define protected amdgpu_kernel void @kernel_round1_short(ptr addrspace(1) nocapt
 ; CHECK-NEXT:    s_mov_b32 s13, s42
 ; CHECK-NEXT:    s_mov_b32 s14, s33
 ; CHECK-NEXT:    v_add_nc_u32_e32 v43, 1, v43
-; CHECK-NEXT:    s_getpc_b64 s[16:17]
-; CHECK-NEXT:    s_add_u32 s16, s16, _Z10atomic_incPU3AS3Vj@rel32@lo+4
-; CHECK-NEXT:    s_addc_u32 s17, s17, _Z10atomic_incPU3AS3Vj@rel32@hi+12
 ; CHECK-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; CHECK-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
 ; CHECK-NEXT:    ds_write_b32 v0, v47
@@ -982,15 +982,15 @@ define protected amdgpu_kernel void @kernel_round1_short(ptr addrspace(1) nocapt
 ; CHECK-NEXT:    v_mov_b32_e32 v0, 1
 ; CHECK-NEXT:    s_add_u32 s8, s38, 40
 ; CHECK-NEXT:    s_addc_u32 s9, s39, 0
+; CHECK-NEXT:    s_getpc_b64 s[16:17]
+; CHECK-NEXT:    s_add_u32 s16, s16, _Z7barrierj@rel32@lo+4
+; CHECK-NEXT:    s_addc_u32 s17, s17, _Z7barrierj@rel32@hi+12
 ; CHECK-NEXT:    s_mov_b64 s[4:5], s[40:41]
 ; CHECK-NEXT:    s_mov_b64 s[6:7], s[36:37]
 ; CHECK-NEXT:    s_mov_b64 s[10:11], s[34:35]
 ; CHECK-NEXT:    s_mov_b32 s12, s43
 ; CHECK-NEXT:    s_mov_b32 s13, s42
 ; CHECK-NEXT:    s_mov_b32 s14, s33
-; CHECK-NEXT:    s_getpc_b64 s[16:17]
-; CHECK-NEXT:    s_add_u32 s16, s16, _Z7barrierj@rel32@lo+4
-; CHECK-NEXT:    s_addc_u32 s17, s17, _Z7barrierj@rel32@hi+12
 ; CHECK-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; CHECK-NEXT:    s_endpgm
 .5:
diff --git a/llvm/test/CodeGen/AMDGPU/mad-mix-hi.ll b/llvm/test/CodeGen/AMDGPU/mad-mix-hi.ll
index 819b6ca98b3a8..a7060e4f198f1 100644
--- a/llvm/test/CodeGen/AMDGPU/mad-mix-hi.ll
+++ b/llvm/test/CodeGen/AMDGPU/mad-mix-hi.ll
@@ -1,5 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=amdgcn -mcpu=gfx1100 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX11,SDAG-GFX11 %s
+; RUN: llc -mtriple=amdgcn -mcpu=gfx1100 -mattr=+real-true16 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX11,SDAG-GFX11,SDAG-GFX11-TRUE16 %s
+; RUN: llc -mtriple=amdgcn -mcpu=gfx1100 -mattr=-real-true16 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX11,SDAG-GFX11,SDAG-GFX11-FAKE16 %s
 ; RUN: llc -mtriple=amdgcn -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX9,SDAG-GFX9 %s
 ; RUN: llc -mtriple=amdgcn -mcpu=fiji -verify-machineinstrs < %s | FileCheck -check-prefixes=VI,SDAG-VI %s
 ; RUN: llc -mtriple=amdgcn -mcpu=hawaii -verify-machineinstrs < %s | FileCheck -check-prefixes=SDAG-CI %s
@@ -329,14 +330,23 @@ define i32 @v_mad_mixhi_f16_f16lo_f16lo_f16lo_intpack_sext(half %src0, half %src
 }
 
 define <2 x half> @v_mad_mixhi_f16_f16lo_f16lo_f16lo_undeflo_clamp_precvt(half %src0, half %src1, half %src2) #0 {
-; GFX11-LABEL: v_mad_mixhi_f16_f16lo_f16lo_f16lo_undeflo_clamp_precvt:
-; GFX11:       ; %bb.0:
-; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT:    v_fma_mix_f32 v0, v0, v1, v2 op_sel_hi:[1,1,1] clamp
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT:    v_cvt_f16_f32_e32 v0, v0
-; GFX11-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
-; GFX11-NEXT:    s_setpc_b64 s[30:31]
+; SDAG-GFX11-TRUE16-LABEL: v_mad_mixhi_f16_f16lo_f16lo_f16lo_undeflo_clamp_precvt:
+; SDAG-GFX11-TRUE16:       ; %bb.0:
+; SDAG-GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SDAG-GFX11-TRUE16-NEXT:    v_fma_mix_f32 v0, v0, v1, v2 op_sel_hi:[1,1,1] clamp
+; SDAG-GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; SDAG-GFX11-TRUE16-NEXT:    v_cvt_f16_f32_e32 v0.l, v0
+; SDAG-GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
+; SDAG-GFX11-TRUE16-NEXT:    s_setpc_b64 s[30:31]
+;
+; SDAG-GFX11-FAKE16-LABEL: v_mad_mixhi_f16_f16lo_f16lo_f16lo_undeflo_clamp_precvt:
+; SDAG-GFX11-FAKE16:       ; %bb.0:
+; SDAG-GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SDAG-GFX11-FAKE16-NEXT:    v_fma_mix_f32 v0, v0, v1, v2 op_sel_hi:[1,1,1] clamp
+; SDAG-GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; SDAG-GFX11-FAKE16-NEXT:    v_cvt_f16_f32_e32 v0, v0
+; SDAG-GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
+; SDAG-GFX11-FAKE16-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: v_mad_mixhi_f16_f16lo_f16lo_f16lo_undeflo_clamp_precvt:
 ; GFX9:       ; %bb.0:
@@ -363,6 +373,15 @@ define <2 x half> @v_mad_mixhi_f16_f16lo_f16lo_f16lo_undeflo_clamp_precvt(half %
 ; SDAG-CI-NEXT:    v_cvt_f32_f16_e32 v1, v0
 ; SDAG-CI-NEXT:    s_setpc_b64 s[30:31]
 ;
+; GISEL-GFX11-LABEL: v_mad_mixhi_f16_f16lo_f16lo_f16lo_undeflo_clamp_precvt:
+; GISEL-GFX11:       ; %bb.0:
+; GISEL-GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GISEL-GFX11-NEXT:    v_fma_mix_f32 v0, v0, v1, v2 op_sel_hi:[1,1,1] clamp
+; GISEL-GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GISEL-GFX11-NEXT:    v_cvt_f16_f32_e32 v0, v0
+; GISEL-GFX11-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
+; GISEL-GFX11-NEXT:    s_setpc_b64 s[30:31]
+;
 ; GISEL-CI-LABEL: v_mad_mixhi_f16_f16lo_f16lo_f16lo_undeflo_clamp_precvt:
 ; GISEL-CI:       ; %bb.0:
 ; GISEL-CI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
diff --git a/llvm/test/CodeGen/AMDGPU/mad-mix-lo.ll b/llvm/test/CodeGen/AMDGPU/mad-mix-lo.ll
index 95d579be04ed2..dd4ad0a70b254 100644
--- a/llvm/test/CodeGen/AMDGPU/mad-mix-lo.ll
+++ b/llvm/test/CodeGen/AMDGPU/mad-mix-lo.ll
@@ -1,5 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=amdgcn -mcpu=gfx1100 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX1100,SDAG-GFX1100 %s
+; RUN: llc -mtriple=amdgcn -mcpu=gfx1100 -mattr=+real-true16 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX1100,SDAG-GFX1100-TRUE16 %s
+; RUN: llc -mtriple=amdgcn -mcpu=gfx1100 -mattr=-real-true16 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX1100,SDAG-GFX1100-FAKE16 %s
 ; RUN: llc -mtriple=amdgcn -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX900,SDAG-GFX900 %s
 ; RUN: llc -mtriple=amdgcn -mcpu=gfx906 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX906,SDAG-GFX906 %s
 ; RUN: llc -mtriple=amdgcn -mcpu=fiji -verify-machineinstrs < %s | FileCheck -check-prefixes=VI,SDAG-VI %s
@@ -268,11 +269,19 @@ define half @v_mad_mixlo_f16_f16lo_f16lo_f32(half %src0, half %src1, float %src2
 }
 
 define half @v_mad_mixlo_f16_f16lo_f16lo_f32_clamp_post_cvt(half %src0, half %src1, float %src2) #0 {
-; GFX1100-LABEL: v_mad_mixlo_f16_f16lo_f16lo_f32_clamp_post_cvt:
-; GFX1100:       ; %bb.0:
-; GFX1100-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX1100-NEXT:    v_fma_mixlo_f16 v0, v0, v1, v2 op_sel_hi:[1,1,0] clamp
-; GFX1100-NEXT:    s_setpc_b64 s[30:31]
+; SDAG-GFX1100-TRUE16-LABEL: v_mad_mixlo_f16_f16lo_f16lo_f32_clamp_post_cvt:
+; SDAG-GFX1100-TRUE16:       ; %bb.0:
+; SDAG-GFX1100-TRUE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SDAG-GFX1100-TRUE16-NEXT:    v_fma_mixlo_f16 v0, v0, v1, v2 op_sel_hi:[1,1,0]
+; SDAG-GFX1100-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; SDAG-GFX1100-TRUE16-NEXT:    v_max_f16_e64 v0.l, v0.l, v0.l clamp
+; SDAG-GFX1100-TRUE16-NEXT:    s_setpc_b64 s[30:31]
+;
+; SDAG-GFX1100-FAKE16-LABEL: v_mad_mixlo_f16_f16lo_f16lo_f32_clamp_post_cvt:
+; SDAG-GFX1100-FAKE16:       ; %bb.0:
+; SDAG-GFX1100-FAKE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SDAG-GFX1100-FAKE16-NEXT:    v_fma_mixlo_f16 v0, v0, v1, v2 op_sel_hi:[1,1,0] clamp
+; SDAG-GFX1100-FAKE16-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX900-LABEL: v_mad_mixlo_f16_f16lo_f16lo_f32_clamp_post_cvt:
 ; GFX900:       ; %bb.0:
@@ -303,6 +312,12 @@ define half @v_mad_mixlo_f16_f16lo_f16lo_f32_clamp_post_cvt(half %src0, half %sr
 ; SDAG-CI-NEXT:    v_cvt_f32_f16_e64 v0, v0 clamp
 ; SDAG-CI-NEXT:    s_setpc_b64 s[30:31]
 ;
+; GISEL-GFX1100-LABEL: v_mad_mixlo_f16_f16lo_f16lo_f32_clamp_post_cvt:
+; GISEL-GFX1100:       ; %bb.0:
+; GISEL-GFX1100-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GISEL-GFX1100-NEXT:    v_fma_mixlo_f16 v0, v0, v1, v2 op_sel_hi:[1,1,0] clamp
+; GISEL-GFX1100-NEXT:    s_setpc_b64 s[30:31]
+;
 ; GISEL-CI-LABEL: v_mad_mixlo_f16_f16lo_f16lo_f32_clamp_post_cvt:
 ; GISEL-CI:       ; %bb.0:
 ; GISEL-CI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -329,13 +344,21 @@ define half @v_mad_mixlo_f16_f16lo_f16lo_f32_clamp_post_cvt(half %src0, half %sr
 }
 
 define half @v_mad_mixlo_f16_f16lo_f16lo_f32_clamp_pre_cvt(half %src0, half %src1, float %src2) #0 {
-; GFX1100-LABEL: v_mad_mixlo_f16_f16lo_f16lo_f32_clamp_pre_cvt:
-; GFX1100:       ; %bb.0:
-; GFX1100-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX1100-NEXT:    v_fma_mix_f32 v0, v0, v1, v2 op_sel_hi:[1,1,0] clamp
-; GFX1100-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX1100-NEXT:    v_cvt_f16_f32_e32 v0, v0
-; GFX1100-NEXT:    s_setpc_b64 s[30:31]
+; SDAG-GFX1100-TRUE16-LABEL: v_mad_mixlo_f16_f16lo_f16lo_f32_clamp_pre_cvt:
+; SDAG-GFX1100-TRUE16:       ; %bb.0:
+; SDAG-GFX1100-TRUE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SDAG-GFX1100-TRUE16-NEXT:    v_fma_mix_f32 v0, v0, v1, v2 op_sel_hi:[1,1,0] clamp
+; SDAG-GFX1100-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; SDAG-GFX1100-TRUE16-NEXT:    v_cvt_f16_f32_e32 v0.l, v0
+; SDAG-GFX1100-TRUE16-NEXT:    s_setpc_b64 s[30:31]
+;
+; SDAG-GFX1100-FAKE16-LABEL: v_mad_mixlo_f16_f16lo_f16lo_f32_clamp_pre_cvt:
+; SDAG-GFX1100-FAKE16:       ; %bb.0:
+; SDAG-GFX1100-FAKE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SDAG-GFX1100-FAKE16-NEXT:    v_fma_mix_f32 v0, v0, v1, v2 op_sel_hi:[1,1,0] clamp
+; SDAG-GFX1100-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; SDAG-GFX1100-FAKE16-NEXT:    v_cvt_f16_f32_e32 v0, v0
+; SDAG-GFX1100-FAKE16-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX900-LABEL: v_mad_mixlo_f16_f16lo_f16lo_f32_clamp_pre_cvt:
 ; GFX900:       ; %bb.0:
@@ -368,6 +391,14 @@ define half @v_mad_mixlo_f16_f16lo_f16lo_f32_clamp_pre_cvt(half %src0, half %src
 ; SDAG-CI-NEXT:    v_cvt_f32_f16_e32 v0, v0
 ; SDAG-CI-NEXT:    s_setpc_b64 s[30:31]
 ;
+; GISEL-GFX1100-LABEL: v_mad_mixlo_f16_f16lo_f16lo_f32_clamp_pre_cvt:
+; GISEL-GFX1100:       ; %bb.0:
+; GISEL-GFX1100-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GISEL-GFX1100-NEXT:    v_fma_mix_f32 v0, v0, v1, v2 op_sel_hi:[1,1,0] clamp
+; GISEL-GFX1100-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GISEL-GFX1100-NEXT:    v_cvt_f16_f32_e32 v0, v0
+; GISEL-GFX1100-NEXT:    s_setpc_b64 s[30:31]
+;
 ; GISEL-CI-LABEL: v_mad_mixlo_f16_f16lo_f16lo_f32_clamp_pre_cvt:
 ; GISEL-CI:       ; %bb.0:
 ; GISEL-CI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -389,14 +420,24 @@ define half @v_mad_mixlo_f16_f16lo_f16lo_f32_clamp_pre_cvt(half %src0, half %src
 ; operation only clobbers relevant lane.
 
 define <2 x half> @v_mad_mix_v2f32(<2 x half> %src0, <2 x half> %src1, <2 x half> %src2) #0 {
-; GFX1100-LABEL: v_mad_mix_v2f32:
-; GFX1100:       ; %bb.0:
-; GFX1100-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX1100-NEXT:    v_fma_mixlo_f16 v3, v0, v1, v2 op_sel_hi:[1,1,1]
-; GFX1100-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1100-NEXT:    v_fma_mixhi_f16 v3, v0, v1, v2 op_sel:[1,1,1] op_sel_hi:[1,1,1]
-; GFX1100-NEXT:    v_mov_b32_e32 v0, v3
-; GFX1100-NEXT:    s_setpc_b64 s[30:31]
+; SDAG-GFX1100-TRUE16-LABEL: v_mad_mix_v2f32:
+; SDAG-GFX1100-TRUE16:       ; %bb.0:
+; SDAG-GFX1100-TRUE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SDAG-GFX1100-TRUE16-NEXT:    v_fma_mixlo_f16 v3, v0, v1, v2 op_sel:[1,1,1] op_sel_hi:[1,1,1]
+; SDAG-GFX1100-TRUE16-NEXT:    v_mov_b16_e32 v4.l, v0.l
+; SDAG-GFX1100-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; SDAG-GFX1100-TRUE16-NEXT:    v_mov_b16_e32 v0.h, v3.l
+; SDAG-GFX1100-TRUE16-NEXT:    v_fma_mixlo_f16 v0, v4, v1, v2 op_sel_hi:[1,1,1]
+; SDAG-GFX1100-TRUE16-NEXT:    s_setpc_b64 s[30:31]
+;
+; SDAG-GFX1100-FAKE16-LABEL: v_mad_mix_v2f32:
+; SDAG-GFX1100-FAKE16:       ; %bb.0:
+; SDAG-GFX1100-FAKE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SDAG-GFX1100-FAKE16-NEXT:    v_fma_mixlo_f16 v3, v0, v1, v2 op_sel_hi:[1,1,1]
+; SDAG-GFX1100-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; SDAG-GFX1100-FAKE16-NEXT:    v_fma_mixhi_f16 v3, v0, v1, v2 op_sel:[1,1,1] op_sel_hi:[1,1,1]
+; SDAG-GFX1100-FAKE16-NEXT:    v_mov_b32_e32 v0, v3
+; SDAG-GFX1100-FAKE16-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX900-LABEL: v_mad_mix_v2f32:
 ; GFX900:       ; %bb.0:
@@ -453,6 +494,15 @@ define <2 x half> @v_mad_mix_v2f32(<2 x half> %src0, <2 x half> %src1, <2 x half
 ; SDAG-CI-NEXT:    v_cvt_f32_f16_e32 v1, v1
 ; SDAG-CI-NEXT:    s_setpc_b64 s[30:31]
 ;
+; GISEL-GFX1100-LABEL: v_mad_mix_v2f32:
+; GISEL-GFX1100:       ; %bb.0:
+; GISEL-GFX1100-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GISEL-GFX1100-NEXT:    v_fma_mixlo_f16 v3, v0, v1, v2 op_sel_hi:[1,1,1]
+; GISEL-GFX1100-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GISEL-GFX1100-NEXT:    v_fma_mixhi_f16 v3, v0, v1, v2 op_sel:[1,1,1] op_sel_hi:[1,1,1]
+; GISEL-GFX1100-NEXT:    v_mov_b32_e32 v0, v3
+; GISEL-GFX1100-NEXT:    s_setpc_b64 s[30:31]
+;
 ; GISEL-VI-LABEL: v_mad_mix_v2f32:
 ; GISEL-VI:       ; %bb.0:
 ; GISEL-VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -492,15 +542,26 @@ define <2 x half> @v_mad_mix_v2f32(<2 x half> %src0, <2 x half> %src1, <2 x half
 }
 
 define <3 x half> @v_mad_mix_v3f32(<3 x half> %src0, <3 x half> %src1, <3 x half> %src2) #0 {
-; GFX1100-LABEL: v_mad_mix_v3f32:
-; GFX1100:       ; %bb.0:
-; GFX1100-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX1100-NEXT:    v_fma_mixlo_f16 v6, v0, v2, v4 op_sel_hi:[1,1,1]
-; GFX1100-NEXT:    v_fma_mixlo_f16 v1, v1, v3, v5 op_sel_hi:[1,1,1]
-; GFX1100-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1100-NEXT:    v_fma_mixhi_f16 v6, v0, v2, v4 op_sel:[1,1,1] op_sel_hi:[1,1,1]
-; GFX1100-NEXT:    v_mov_b32_e32 v0, v6
-; GFX1100-NEXT:    s_setpc_b64 s[30:31]
+; SDAG-GFX1100-TRUE16-LABEL: v_mad_mix_v3f32:
+; SDAG-GFX1100-TRUE16:       ; %bb.0:
+; SDAG-GFX1100-TRUE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SDAG-GFX1100-TRUE16-NEXT:    v_fma_mixlo_f16 v6, v0, v2, v4 op_sel:[1,1,1] op_sel_hi:[1,1,1]
+; SDAG-GFX1100-TRUE16-NEXT:    v_mov_b16_e32 v7.l, v0.l
+; SDAG-GFX1100-TRUE16-NEXT:    v_fma_mixlo_f16 v1, v1, v3, v5 op_sel_hi:[1,1,1]
+; SDAG-GFX1100-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; SDAG-GFX1100-TRUE16-NEXT:    v_mov_b16_e32 v0.h, v6.l
+; SDAG-GFX1100-TRUE16-NEXT:    v_fma_mixlo_f16 v0, v7, v2, v4 op_sel_hi:[1,1,1]
+; SDAG-GFX1100-TRUE16-NEXT:    s_setpc_b64 s[30:31]
+;
+; SDAG-GFX1100-FAKE16-LABEL: v_mad_mix_v3f32:
+; SDAG-GFX1100-FAKE16:       ; %bb.0:
+; SDAG-GFX1100-FAKE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SDAG-GFX1100-FAKE16-NEXT:    v_fma_mixlo_f16 v6, v0, v2, v4 op_sel_hi:[1,1,1]
+; SDAG-GFX1100-FAKE16-NEXT:    v_fma_mixlo_f16 v1, v1, v3, v5 op_sel_hi:[1,1,1]
+; SDAG-GFX1100-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; SDAG-GFX1100-FAKE16-NEXT:    v_fma_mixhi_f16 v6, v0, v2, v4 op_sel:[1,1,1] op_sel_hi:[1,1,1]
+; SDAG-GFX1100-FAKE16-NEXT:    v_mov_b32_e32 v0, v6
+; SDAG-GFX1100-FAKE16-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; SDAG-GFX900-LABEL: v_mad_mix_v3f32:
 ; SDAG-GFX900:       ; %bb.0:
@@ -573,6 +634,16 @@ define <3 x half> @v_mad_mix_v3f32(<3 x half> %src0, <3 x half> %src1, <3 x half
 ; SDAG-CI-NEXT:    v_cvt_f32_f16_e32 v2, v2
 ; SDAG-CI-NEXT:    s_setpc_b64 s[30:31]
 ;
+; GISEL-GFX1100-LABEL: v_mad_mix_v3f32:
+; GISEL-GFX1100:       ; %bb.0:
+; GISEL-GFX1100-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GISEL-GFX1100-NEXT:    v_fma_mixlo_f16 v6, v0, v2, v4 op_sel_hi:[1,1,1]
+; GISEL-GFX1100-NEXT:    v_fma_mixlo_f16 v1, v1, v3, v5 op_sel_hi:[1,1,1]
+; GISEL-GFX1100-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GISEL-GFX1100-NEXT:    v_fma_mixhi_f16 v6, v0, v2, v4 op_sel:[1,1,1] op_sel_hi:[1,1,1]
+; GISEL-GFX1100-NEXT:    v_mov_b32_e32 v0, v6
+; GISEL-GFX1100-NEXT:    s_setpc_b64 s[30:31]
+;
 ; GISEL-GFX900-LABEL: v_mad_mix_v3f32:
 ; GISEL-GFX900:       ; %bb.0:
 ; GISEL-GFX900-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -640,17 +711,32 @@ define <3 x half> @v_mad_mix_v3f32(<3 x half> %src0, <3 x half> %src1, <3 x half
 }
 
 define <4 x half> @v_mad_mix_v4f32(<4 x half> %src0, <4 x half> %src1, <4 x half> %src2) #0 {
-; GFX1100-LABEL: v_mad_mix_v4f32:
-; GFX1100:       ; %bb.0:
-; GFX1100-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX1100-NEXT:    v_fma_mixlo_f16 v6, v0, v2, v4 op_sel_hi:[1,1,1]
-; GFX1100-NEXT:    v_fma_mixlo_f16 v7, v1, v3, v5 op_sel_hi:[1,1,1]
-; GFX1100-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1100-NEXT:    v_fma_mixhi_f16 v6, v0, v2, v4 op_sel:[1,1,1] op_sel_hi:[1,1,1]
-; GFX1100-NEXT:    v_fma_mixhi_f16 v7, v1, v3, v5 op_sel:[1,1,1] op_sel_hi:[1,1,1]
-; GFX1100-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX1100-NEXT:    v_dual_mov_b32 v0, v6 :: v_dual_mov_b32 v1, v7
-; GFX1100-NEXT:    s_setpc_b64 s[30:31]
+; SDAG-GFX1100-TRUE16-LABEL: v_mad_mix_v4f32:
+; SDAG-GFX1100-TRUE16:       ; %bb.0:
+; SDAG-GFX1100-TRUE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SDAG-GFX1100-TRUE16-NEXT:    v_fma_mixlo_f16 v6, v0, v2, v4 op_sel:[1,1,1] op_sel_hi:[1,1,1]
+; SDAG-GFX1100-TRUE16-NEXT:    v_fma_mixlo_f16 v7, v1, v3, v5 op_sel:[1,1,1] op_sel_hi:[1,1,1]
+; SDAG-GFX1100-TRUE16-NEXT:    v_mov_b16_e32 v8.l, v0.l
+; SDAG-GFX1100-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; SDAG-GFX1100-TRUE16-NEXT:    v_mov_b16_e32 v0.h, v6.l
+; SDAG-GFX1100-TRUE16-NEXT:    v_mov_b16_e32 v1.h, v7.l
+; SDAG-GFX1100-TRUE16-NEXT:    v_mov_b16_e32 v6.l, v1.l
+; SDAG-GFX1100-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; SDAG-GFX1100-TRUE16-NEXT:    v_fma_mixlo_f16 v0, v8, v2, v4 op_sel_hi:[1,1,1]
+; SDAG-GFX1100-TRUE16-NEXT:    v_fma_mixlo_f16 v1, v6, v3, v5 op_sel_hi:[1,1,1]
+; SDAG-GFX1100-TRUE16-NEXT:    s_setpc_b64 s[30:31]
+;
+; SDAG-GFX1100-FAKE16-LABEL: v_mad_mix_v4f32:
+; SDAG-GFX1100-FAKE16:       ; %bb.0:
+; SDAG-GFX1100-FAKE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SDAG-GFX1100-FAKE16-NEXT:    v_fma_mixlo_f16 v6, v0, v2, v4 op_sel_hi:[1,1,1]
+; SDAG-GFX1100-FAKE16-NEXT:    v_fma_mixlo_f16 v7, v1, v3, v5 op_sel_hi:[1,1,1]
+; SDAG-GFX1100-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; SDAG-GFX1100-FAKE16-NEXT:    v_fma_mixhi_f16 v6, v0, v2, v4 op_sel:[1,1,1] op_sel_hi:[1,1,1]
+; SDAG-GFX1100-FAKE16-NEXT:    v_fma_mixhi_f16 v7, v1, v3, v5 op_sel:[1,1,1] op_sel_hi:[1,1,1]
+; SDAG-GFX1100-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; SDAG-GFX1100-FAKE16-NEXT:    v_dual_mov_b32 v0, v6 :: v_dual_mov_b32 v1, v7
+; SDAG-GFX1100-FAKE16-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; SDAG-GFX900-LABEL: v_mad_mix_v4f32:
 ; SDAG-GFX900:       ; %bb.0:
@@ -742,6 +828,18 @@ define <4 x half> @v_mad_mix_v4f32(<4 x half> %src0, <4 x half> %src1, <4 x half
 ; SDAG-CI-NEXT:    v_cvt_f32_f16_e32 v3, v3
 ; SDAG-CI-NEXT:    s_setpc_b64 s[30:31]
 ;
+; GISEL-GFX1100-LABEL: v_mad_mix_v4f32:
+; GISEL-GFX1100:       ; %bb.0:
+; GISEL-GFX1100-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GISEL-GFX1100-NEXT:    v_fma_mixlo_f16 v6, v0, v2, v4 op_sel_hi:[1,1,1]
+; GISEL-GFX1100-NEXT:    v_fma_mixlo_f16 v7, v1, v3, v5 op_sel_hi:[1,1,1]
+; GISEL-GFX1100-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GISEL-GFX1100-NEXT:    v_fma_mixhi_f16 v6, v0, v2, v4 op_sel:[1,1,1] op_sel_hi:[1,1,1]
+; GISEL-GFX1100-NEXT:    v_fma_mixhi_f16 v7, v1, v3, v5 op_sel:[1,1,1] op_sel_hi:[1,1,1]
+; GISEL-GFX1100-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GISEL-GFX1100-NEXT:    v_dual_mov_b32 v0, v6 :: v_dual_mov_b32 v1, v7
+; GISEL-GFX1100-NEXT:    s_setpc_b64 s[30:31]
+;
 ; GISEL-GFX900-LABEL: v_mad_mix_v4f32:
 ; GISEL-GFX900:       ; %bb.0:
 ; GISEL-GFX900-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -826,14 +924,27 @@ define <4 x half> @v_mad_mix_v4f32(<4 x half> %src0, <4 x half> %src1, <4 x half
 ; FIXME (DAG): Fold clamp
 
 define <2 x half> @v_mad_mix_v2f32_clamp_postcvt(<2 x half> %src0, <2 x half> %src1, <2 x half> %src2) #0 {
-; GFX1100-LABEL: v_mad_mix_v2f32_clamp_postcvt:
-; GFX1100:       ; %bb.0:
-; GFX1100-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX1100-NEXT:    v_fma_mixlo_f16 v3, v0, v1, v2 op_sel_hi:[1,1,1] clamp
-; GFX1100-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1100-NEXT:    v_fma_mixhi_f16 v3, v0, v1, v2 op_sel:[1,1,1] op_sel_hi:[1,1,1] clamp
-; GFX1100-NEXT:    v_mov_b32_e32 v0, v3
-; GFX1100-NEXT:    s_setpc_b64 s[30:31]
+; SDAG-GFX1100-TRUE16-LABEL: v_mad_mix_v2f32_clamp_postcvt:
+; SDAG-GFX1100-TRUE16:       ; %bb.0:
+; SDAG-GFX1100-TRUE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SDAG-GFX1100-TRUE16-NEXT:    v_mov_b16_e32 v3.l, v0.l
+; SDAG-GFX1100-TRUE16-NEXT:    v_mov_b16_e32 v4.l, v1.l
+; SDAG-GFX1100-TRUE16-NEXT:    v_mov_b16_e32 v5.l, v2.l
+; SDAG-GFX1100-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; SDAG-GFX1100-TRUE16-NEXT:    v_fma_mixlo_f16 v3, v3, v4, v5 op_sel_hi:[1,1,1] clamp
+; SDAG-GFX1100-TRUE16-NEXT:    v_fma_mixhi_f16 v3, v0, v1, v2 op_sel:[1,1,1] op_sel_hi:[1,1,1] clamp
+; SDAG-GFX1100-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; SDAG-GFX1100-TRUE16-NEXT:    v_mov_b32_e32 v0, v3
+; SDAG-GFX1100-TRUE16-NEXT:    s_setpc_b64 s[30:31]
+;
+; SDAG-GFX1100-FAKE16-LABEL: v_mad_mix_v2f32_clamp_postcvt:
+; SDAG-GFX1100-FAKE16:       ; %bb.0:
+; SDAG-GFX1100-FAKE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SDAG-GFX1100-FAKE16-NEXT:    v_fma_mixlo_f16 v3, v0, v1, v2 op_sel_hi:[1,1,1] clamp
+; SDAG-GFX1100-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; SDAG-GFX1100-FAKE16-NEXT:    v_fma_mixhi_f16 v3, v0, v1, v2 op_sel:[1,1,1] op_sel_hi:[1,1,1] clamp
+; SDAG-GFX1100-FAKE16-NEXT:    v_mov_b32_e32 v0, v3
+; SDAG-GFX1100-FAKE16-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX900-LABEL: v_mad_mix_v2f32_clamp_postcvt:
 ; GFX900:       ; %bb.0:
@@ -890,6 +1001,15 @@ define <2 x half> @v_mad_mix_v2f32_clamp_postcvt(<2 x half> %src0, <2 x half> %s
 ; SDAG-CI-NEXT:    v_cvt_f32_f16_e64 v1, v1 clamp
 ; SDAG-CI-NEXT:    s_setpc_b64 s[30:31]
 ;
+; GISEL-GFX1100-LABEL: v_mad_mix_v2f32_clamp_postcvt:
+; GISEL-GFX1100:       ; %bb.0:
+; GISEL-GFX1100-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GISEL-GFX1100-NEXT:    v_fma_mixlo_f16 v3, v0, v1, v2 op_sel_hi:[1,1,1] clamp
+; GISEL-GFX1100-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GISEL-GFX1100-NEXT:    v_fma_mixhi_f16 v3, v0, v1, v2 op_sel:[1,1,1] op_sel_hi:[1,1,1] clamp
+; GISEL-GFX1100-NEXT:    v_mov_b32_e32 v0, v3
+; GISEL-GFX1100-NEXT:    s_setpc_b64 s[30:31]
+;
 ; GISEL-VI-LABEL: v_mad_mix_v2f32_clamp_postcvt:
 ; GISEL-VI:       ; %bb.0:
 ; GISEL-VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -948,18 +1068,36 @@ define <2 x half> @v_mad_mix_v2f32_clamp_postcvt(<2 x half> %src0, <2 x half> %s
 ; FIXME (GIsel): V_PK_MAX clamp could be folded into mixlo
 
 define <3 x half> @v_mad_mix_v3f32_clamp_postcvt(<3 x half> %src0, <3 x half> %src1, <3 x half> %src2) #0 {
-; SDAG-GFX1100-LABEL: v_mad_mix_v3f32_clamp_postcvt:
-; SDAG-GFX1100:       ; %bb.0:
-; SDAG-GFX1100-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SDAG-GFX1100-NEXT:    v_fma_mixlo_f16 v6, v0, v2, v4 op_sel_hi:[1,1,1]
-; SDAG-GFX1100-NEXT:    v_fma_mixlo_f16 v1, v1, v3, v5 op_sel_hi:[1,1,1]
-; SDAG-GFX1100-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; SDAG-GFX1100-NEXT:    v_fma_mixhi_f16 v6, v0, v2, v4 op_sel:[1,1,1] op_sel_hi:[1,1,1]
-; SDAG-GFX1100-NEXT:    v_pack_b32_f16 v1, v1, 0
-; SDAG-GFX1100-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; SDAG-GFX1100-NEXT:    v_pk_max_f16 v0, v6, v6 clamp
-; SDAG-GFX1100-NEXT:    v_pk_max_f16 v1, v1, v1 clamp
-; SDAG-GFX1100-NEXT:    s_setpc_b64 s[30:31]
+; SDAG-GFX1100-TRUE16-LABEL: v_mad_mix_v3f32_clamp_postcvt:
+; SDAG-GFX1100-TRUE16:       ; %bb.0:
+; SDAG-GFX1100-TRUE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SDAG-GFX1100-TRUE16-NEXT:    v_fma_mixlo_f16 v6, v0, v2, v4 op_sel:[1,1,1] op_sel_hi:[1,1,1]
+; SDAG-GFX1100-TRUE16-NEXT:    v_fma_mixlo_f16 v1, v1, v3, v5 op_sel_hi:[1,1,1]
+; SDAG-GFX1100-TRUE16-NEXT:    v_mov_b16_e32 v3.l, v0.l
+; SDAG-GFX1100-TRUE16-NEXT:    v_mov_b16_e32 v5.l, v2.l
+; SDAG-GFX1100-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; SDAG-GFX1100-TRUE16-NEXT:    v_mov_b16_e32 v2.h, v6.l
+; SDAG-GFX1100-TRUE16-NEXT:    v_mov_b16_e32 v0.l, v1.l
+; SDAG-GFX1100-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; SDAG-GFX1100-TRUE16-NEXT:    v_fma_mixlo_f16 v2, v3, v5, v4 op_sel_hi:[1,1,1]
+; SDAG-GFX1100-TRUE16-NEXT:    v_pack_b32_f16 v1, v0.l, 0
+; SDAG-GFX1100-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; SDAG-GFX1100-TRUE16-NEXT:    v_pk_max_f16 v0, v2, v2 clamp
+; SDAG-GFX1100-TRUE16-NEXT:    v_pk_max_f16 v1, v1, v1 clamp
+; SDAG-GFX1100-TRUE16-NEXT:    s_setpc_b64 s[30:31]
+;
+; SDAG-GFX1100-FAKE16-LABEL: v_mad_mix_v3f32_clamp_postcvt:
+; SDAG-GFX1100-FAKE16:       ; %bb.0:
+; SDAG-GFX1100-FAKE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SDAG-GFX1100-FAKE16-NEXT:    v_fma_mixlo_f16 v6, v0, v2, v4 op_sel_hi:[1,1,1]
+; SDAG-GFX1100-FAKE16-NEXT:    v_fma_mixlo_f16 v1, v1, v3, v5 op_sel_hi:[1,1,1]
+; SDAG-GFX1100-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; SDAG-GFX1100-FAKE16-NEXT:    v_fma_mixhi_f16 v6, v0, v2, v4 op_sel:[1,1,1] op_sel_hi:[1,1,1]
+; SDAG-GFX1100-FAKE16-NEXT:    v_pack_b32_f16 v1, v1, 0
+; SDAG-GFX1100-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; SDAG-GFX1100-FAKE16-NEXT:    v_pk_max_f16 v0, v6, v6 clamp
+; SDAG-GFX1100-FAKE16-NEXT:    v_pk_max_f16 v1, v1, v1 clamp
+; SDAG-GFX1100-FAKE16-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; SDAG-GFX900-LABEL: v_mad_mix_v3f32_clamp_postcvt:
 ; SDAG-GFX900:       ; %bb.0:
@@ -1146,18 +1284,36 @@ define <3 x half> @v_mad_mix_v3f32_clamp_postcvt(<3 x half> %src0, <3 x half> %s
 }
 
 define <4 x half> @v_mad_mix_v4f32_clamp_postcvt(<4 x half> %src0, <4 x half> %src1, <4 x half> %src2) #0 {
-; SDAG-GFX1100-LABEL: v_mad_mix_v4f32_clamp_postcvt:
-; SDAG-GFX1100:       ; %bb.0:
-; SDAG-GFX1100-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SDAG-GFX1100-NEXT:    v_fma_mixlo_f16 v6, v0, v2, v4 op_sel_hi:[1,1,1]
-; SDAG-GFX1100-NEXT:    v_fma_mixlo_f16 v7, v1, v3, v5 op_sel_hi:[1,1,1]
-; SDAG-GFX1100-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; SDAG-GFX1100-NEXT:    v_fma_mixhi_f16 v6, v0, v2, v4 op_sel:[1,1,1] op_sel_hi:[1,1,1]
-; SDAG-GFX1100-NEXT:    v_fma_mixhi_f16 v7, v1, v3, v5 op_sel:[1,1,1] op_sel_hi:[1,1,1]
-; SDAG-GFX1100-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; SDAG-GFX1100-NEXT:    v_pk_max_f16 v0, v6, v6 clamp
-; SDAG-GFX1100-NEXT:    v_pk_max_f16 v1, v7, v7 clamp
-; SDAG-GFX1100-NEXT:    s_setpc_b64 s[30:31]
+; SDAG-GFX1100-TRUE16-LABEL: v_mad_mix_v4f32_clamp_postcvt:
+; SDAG-GFX1100-TRUE16:       ; %bb.0:
+; SDAG-GFX1100-TRUE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SDAG-GFX1100-TRUE16-NEXT:    v_fma_mixlo_f16 v6, v1, v3, v5 op_sel:[1,1,1] op_sel_hi:[1,1,1]
+; SDAG-GFX1100-TRUE16-NEXT:    v_fma_mixlo_f16 v7, v0, v2, v4 op_sel:[1,1,1] op_sel_hi:[1,1,1]
+; SDAG-GFX1100-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; SDAG-GFX1100-TRUE16-NEXT:    v_mov_b16_e32 v2.h, v6.l
+; SDAG-GFX1100-TRUE16-NEXT:    v_mov_b16_e32 v0.h, v7.l
+; SDAG-GFX1100-TRUE16-NEXT:    v_mov_b16_e32 v6.l, v0.l
+; SDAG-GFX1100-TRUE16-NEXT:    v_mov_b16_e32 v7.l, v2.l
+; SDAG-GFX1100-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
+; SDAG-GFX1100-TRUE16-NEXT:    v_fma_mixlo_f16 v2, v1, v3, v5 op_sel_hi:[1,1,1]
+; SDAG-GFX1100-TRUE16-NEXT:    v_fma_mixlo_f16 v0, v6, v7, v4 op_sel_hi:[1,1,1]
+; SDAG-GFX1100-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; SDAG-GFX1100-TRUE16-NEXT:    v_pk_max_f16 v1, v2, v2 clamp
+; SDAG-GFX1100-TRUE16-NEXT:    v_pk_max_f16 v0, v0, v0 clamp
+; SDAG-GFX1100-TRUE16-NEXT:    s_setpc_b64 s[30:31]
+;
+; SDAG-GFX1100-FAKE16-LABEL: v_mad_mix_v4f32_clamp_postcvt:
+; SDAG-GFX1100-FAKE16:       ; %bb.0:
+; SDAG-GFX1100-FAKE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SDAG-GFX1100-FAKE16-NEXT:    v_fma_mixlo_f16 v6, v0, v2, v4 op_sel_hi:[1,1,1]
+; SDAG-GFX1100-FAKE16-NEXT:    v_fma_mixlo_f16 v7, v1, v3, v5 op_sel_hi:[1,1,1]
+; SDAG-GFX1100-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; SDAG-GFX1100-FAKE16-NEXT:    v_fma_mixhi_f16 v6, v0, v2, v4 op_sel:[1,1,1] op_sel_hi:[1,1,1]
+; SDAG-GFX1100-FAKE16-NEXT:    v_fma_mixhi_f16 v7, v1, v3, v5 op_sel:[1,1,1] op_sel_hi:[1,1,1]
+; SDAG-GFX1100-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; SDAG-GFX1100-FAKE16-NEXT:    v_pk_max_f16 v0, v6, v6 clamp
+; SDAG-GFX1100-FAKE16-NEXT:    v_pk_max_f16 v1, v7, v7 clamp
+; SDAG-GFX1100-FAKE16-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; SDAG-GFX900-LABEL: v_mad_mix_v4f32_clamp_postcvt:
 ; SDAG-GFX900:       ; %bb.0:
@@ -1383,14 +1539,28 @@ define <4 x half> @v_mad_mix_v4f32_clamp_postcvt(<4 x half> %src0, <4 x half> %s
 ;  a build_vector to select the mixhi. Issue is more specifically with how insert_vector_elt is being
 ;  legalized (bitwise ops instead of shuffle/build_vector for instance).
 define <2 x half> @v_mad_mix_v2f32_clamp_postcvt_lo(<2 x half> %src0, <2 x half> %src1, <2 x half> %src2) #0 {
-; SDAG-GFX1100-LABEL: v_mad_mix_v2f32_clamp_postcvt_lo:
-; SDAG-GFX1100:       ; %bb.0:
-; SDAG-GFX1100-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SDAG-GFX1100-NEXT:    v_fma_mixlo_f16 v3, v0, v1, v2 op_sel_hi:[1,1,1] clamp
-; SDAG-GFX1100-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; SDAG-GFX1100-NEXT:    v_fma_mixhi_f16 v3, v0, v1, v2 op_sel:[1,1,1] op_sel_hi:[1,1,1]
-; SDAG-GFX1100-NEXT:    v_mov_b32_e32 v0, v3
-; SDAG-GFX1100-NEXT:    s_setpc_b64 s[30:31]
+; SDAG-GFX1100-TRUE16-LABEL: v_mad_mix_v2f32_clamp_postcvt_lo:
+; SDAG-GFX1100-TRUE16:       ; %bb.0:
+; SDAG-GFX1100-TRUE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SDAG-GFX1100-TRUE16-NEXT:    v_mov_b16_e32 v3.l, v0.l
+; SDAG-GFX1100-TRUE16-NEXT:    v_mov_b16_e32 v4.l, v1.l
+; SDAG-GFX1100-TRUE16-NEXT:    v_mov_b16_e32 v5.l, v2.l
+; SDAG-GFX1100-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; SDAG-GFX1100-TRUE16-NEXT:    v_fma_mixlo_f16 v3, v3, v4, v5 op_sel_hi:[1,1,1]
+; SDAG-GFX1100-TRUE16-NEXT:    v_max_f16_e64 v3.l, v3.l, v3.l clamp
+; SDAG-GFX1100-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; SDAG-GFX1100-TRUE16-NEXT:    v_fma_mixhi_f16 v3, v0, v1, v2 op_sel:[1,1,1] op_sel_hi:[1,1,1]
+; SDAG-GFX1100-TRUE16-NEXT:    v_mov_b32_e32 v0, v3
+; SDAG-GFX1100-TRUE16-NEXT:    s_setpc_b64 s[30:31]
+;
+; SDAG-GFX1100-FAKE16-LABEL: v_mad_mix_v2f32_clamp_postcvt_lo:
+; SDAG-GFX1100-FAKE16:       ; %bb.0:
+; SDAG-GFX1100-FAKE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SDAG-GFX1100-FAKE16-NEXT:    v_fma_mixlo_f16 v3, v0, v1, v2 op_sel_hi:[1,1,1] clamp
+; SDAG-GFX1100-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; SDAG-GFX1100-FAKE16-NEXT:    v_fma_mixhi_f16 v3, v0, v1, v2 op_sel:[1,1,1] op_sel_hi:[1,1,1]
+; SDAG-GFX1100-FAKE16-NEXT:    v_mov_b32_e32 v0, v3
+; SDAG-GFX1100-FAKE16-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; SDAG-GFX900-LABEL: v_mad_mix_v2f32_clamp_postcvt_lo:
 ; SDAG-GFX900:       ; %bb.0:
@@ -1539,14 +1709,27 @@ define <2 x half> @v_mad_mix_v2f32_clamp_postcvt_lo(<2 x half> %src0, <2 x half>
 }
 
 define <2 x half> @v_mad_mix_v2f32_clamp_postcvt_hi(<2 x half> %src0, <2 x half> %src1, <2 x half> %src2) #0 {
-; SDAG-GFX1100-LABEL: v_mad_mix_v2f32_clamp_postcvt_hi:
-; SDAG-GFX1100:       ; %bb.0:
-; SDAG-GFX1100-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SDAG-GFX1100-NEXT:    v_fma_mixlo_f16 v3, v0, v1, v2 op_sel_hi:[1,1,1]
-; SDAG-GFX1100-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; SDAG-GFX1100-NEXT:    v_fma_mixhi_f16 v3, v0, v1, v2 op_sel:[1,1,1] op_sel_hi:[1,1,1] clamp
-; SDAG-GFX1100-NEXT:    v_mov_b32_e32 v0, v3
-; SDAG-GFX1100-NEXT:    s_setpc_b64 s[30:31]
+; SDAG-GFX1100-TRUE16-LABEL: v_mad_mix_v2f32_clamp_postcvt_hi:
+; SDAG-GFX1100-TRUE16:       ; %bb.0:
+; SDAG-GFX1100-TRUE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SDAG-GFX1100-TRUE16-NEXT:    v_mov_b16_e32 v3.l, v0.l
+; SDAG-GFX1100-TRUE16-NEXT:    v_mov_b16_e32 v4.l, v1.l
+; SDAG-GFX1100-TRUE16-NEXT:    v_mov_b16_e32 v5.l, v2.l
+; SDAG-GFX1100-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; SDAG-GFX1100-TRUE16-NEXT:    v_fma_mixlo_f16 v3, v3, v4, v5 op_sel_hi:[1,1,1]
+; SDAG-GFX1100-TRUE16-NEXT:    v_fma_mixhi_f16 v3, v0, v1, v2 op_sel:[1,1,1] op_sel_hi:[1,1,1] clamp
+; SDAG-GFX1100-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; SDAG-GFX1100-TRUE16-NEXT:    v_mov_b32_e32 v0, v3
+; SDAG-GFX1100-TRUE16-NEXT:    s_setpc_b64 s[30:31]
+;
+; SDAG-GFX1100-FAKE16-LABEL: v_mad_mix_v2f32_clamp_postcvt_hi:
+; SDAG-GFX1100-FAKE16:       ; %bb.0:
+; SDAG-GFX1100-FAKE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SDAG-GFX1100-FAKE16-NEXT:    v_fma_mixlo_f16 v3, v0, v1, v2 op_sel_hi:[1,1,1]
+; SDAG-GFX1100-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; SDAG-GFX1100-FAKE16-NEXT:    v_fma_mixhi_f16 v3, v0, v1, v2 op_sel:[1,1,1] op_sel_hi:[1,1,1] clamp
+; SDAG-GFX1100-FAKE16-NEXT:    v_mov_b32_e32 v0, v3
+; SDAG-GFX1100-FAKE16-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; SDAG-GFX900-LABEL: v_mad_mix_v2f32_clamp_postcvt_hi:
 ; SDAG-GFX900:       ; %bb.0:
@@ -1702,17 +1885,32 @@ define <2 x half> @v_mad_mix_v2f32_clamp_postcvt_hi(<2 x half> %src0, <2 x half>
 ; FIXME (DAG): Should be able to use mixlo/mixhi
 
 define <2 x half> @v_mad_mix_v2f32_clamp_precvt(<2 x half> %src0, <2 x half> %src1, <2 x half> %src2) #0 {
-; SDAG-GFX1100-LABEL: v_mad_mix_v2f32_clamp_precvt:
-; SDAG-GFX1100:       ; %bb.0:
-; SDAG-GFX1100-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SDAG-GFX1100-NEXT:    v_fma_mix_f32 v3, v0, v1, v2 op_sel:[1,1,1] op_sel_hi:[1,1,1] clamp
-; SDAG-GFX1100-NEXT:    v_fma_mix_f32 v0, v0, v1, v2 op_sel_hi:[1,1,1] clamp
-; SDAG-GFX1100-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; SDAG-GFX1100-NEXT:    v_cvt_f16_f32_e32 v1, v3
-; SDAG-GFX1100-NEXT:    v_cvt_f16_f32_e32 v0, v0
-; SDAG-GFX1100-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; SDAG-GFX1100-NEXT:    v_pack_b32_f16 v0, v0, v1
-; SDAG-GFX1100-NEXT:    s_setpc_b64 s[30:31]
+; SDAG-GFX1100-TRUE16-LABEL: v_mad_mix_v2f32_clamp_precvt:
+; SDAG-GFX1100-TRUE16:       ; %bb.0:
+; SDAG-GFX1100-TRUE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SDAG-GFX1100-TRUE16-NEXT:    v_mov_b16_e32 v3.l, v0.l
+; SDAG-GFX1100-TRUE16-NEXT:    v_mov_b16_e32 v4.l, v1.l
+; SDAG-GFX1100-TRUE16-NEXT:    v_mov_b16_e32 v5.l, v2.l
+; SDAG-GFX1100-TRUE16-NEXT:    v_fma_mix_f32 v0, v0, v1, v2 op_sel:[1,1,1] op_sel_hi:[1,1,1] clamp
+; SDAG-GFX1100-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; SDAG-GFX1100-TRUE16-NEXT:    v_fma_mix_f32 v1, v3, v4, v5 op_sel_hi:[1,1,1] clamp
+; SDAG-GFX1100-TRUE16-NEXT:    v_cvt_f16_f32_e32 v0.l, v0
+; SDAG-GFX1100-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; SDAG-GFX1100-TRUE16-NEXT:    v_cvt_f16_f32_e32 v0.h, v1
+; SDAG-GFX1100-TRUE16-NEXT:    v_pack_b32_f16 v0, v0.h, v0.l
+; SDAG-GFX1100-TRUE16-NEXT:    s_setpc_b64 s[30:31]
+;
+; SDAG-GFX1100-FAKE16-LABEL: v_mad_mix_v2f32_clamp_precvt:
+; SDAG-GFX1100-FAKE16:       ; %bb.0:
+; SDAG-GFX1100-FAKE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SDAG-GFX1100-FAKE16-NEXT:    v_fma_mix_f32 v3, v0, v1, v2 op_sel:[1,1,1] op_sel_hi:[1,1,1] clamp
+; SDAG-GFX1100-FAKE16-NEXT:    v_fma_mix_f32 v0, v0, v1, v2 op_sel_hi:[1,1,1] clamp
+; SDAG-GFX1100-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; SDAG-GFX1100-FAKE16-NEXT:    v_cvt_f16_f32_e32 v1, v3
+; SDAG-GFX1100-FAKE16-NEXT:    v_cvt_f16_f32_e32 v0, v0
+; SDAG-GFX1100-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; SDAG-GFX1100-FAKE16-NEXT:    v_pack_b32_f16 v0, v0, v1
+; SDAG-GFX1100-FAKE16-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; SDAG-GFX900-LABEL: v_mad_mix_v2f32_clamp_precvt:
 ; SDAG-GFX900:       ; %bb.0:
@@ -1848,19 +2046,37 @@ define <2 x half> @v_mad_mix_v2f32_clamp_precvt(<2 x half> %src0, <2 x half> %sr
 ; FIXME (DAG): Handling undef 4th component
 
 define <3 x half> @v_mad_mix_v3f32_clamp_precvt(<3 x half> %src0, <3 x half> %src1, <3 x half> %src2) #0 {
-; SDAG-GFX1100-LABEL: v_mad_mix_v3f32_clamp_precvt:
-; SDAG-GFX1100:       ; %bb.0:
-; SDAG-GFX1100-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SDAG-GFX1100-NEXT:    v_fma_mix_f32 v6, v0, v2, v4 op_sel:[1,1,1] op_sel_hi:[1,1,1] clamp
-; SDAG-GFX1100-NEXT:    v_fma_mix_f32 v0, v0, v2, v4 op_sel_hi:[1,1,1] clamp
-; SDAG-GFX1100-NEXT:    v_fma_mix_f32 v1, v1, v3, v5 op_sel_hi:[1,1,1] clamp
-; SDAG-GFX1100-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; SDAG-GFX1100-NEXT:    v_cvt_f16_f32_e32 v2, v6
-; SDAG-GFX1100-NEXT:    v_cvt_f16_f32_e32 v0, v0
-; SDAG-GFX1100-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; SDAG-GFX1100-NEXT:    v_cvt_f16_f32_e32 v1, v1
-; SDAG-GFX1100-NEXT:    v_pack_b32_f16 v0, v0, v2
-; SDAG-GFX1100-NEXT:    s_setpc_b64 s[30:31]
+; SDAG-GFX1100-TRUE16-LABEL: v_mad_mix_v3f32_clamp_precvt:
+; SDAG-GFX1100-TRUE16:       ; %bb.0:
+; SDAG-GFX1100-TRUE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SDAG-GFX1100-TRUE16-NEXT:    v_mov_b16_e32 v6.l, v0.l
+; SDAG-GFX1100-TRUE16-NEXT:    v_mov_b16_e32 v7.l, v2.l
+; SDAG-GFX1100-TRUE16-NEXT:    v_mov_b16_e32 v8.l, v4.l
+; SDAG-GFX1100-TRUE16-NEXT:    v_fma_mix_f32 v0, v0, v2, v4 op_sel:[1,1,1] op_sel_hi:[1,1,1] clamp
+; SDAG-GFX1100-TRUE16-NEXT:    v_fma_mix_f32 v1, v1, v3, v5 op_sel_hi:[1,1,1] clamp
+; SDAG-GFX1100-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; SDAG-GFX1100-TRUE16-NEXT:    v_fma_mix_f32 v2, v6, v7, v8 op_sel_hi:[1,1,1] clamp
+; SDAG-GFX1100-TRUE16-NEXT:    v_cvt_f16_f32_e32 v0.l, v0
+; SDAG-GFX1100-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; SDAG-GFX1100-TRUE16-NEXT:    v_cvt_f16_f32_e32 v1.l, v1
+; SDAG-GFX1100-TRUE16-NEXT:    v_cvt_f16_f32_e32 v0.h, v2
+; SDAG-GFX1100-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; SDAG-GFX1100-TRUE16-NEXT:    v_pack_b32_f16 v0, v0.h, v0.l
+; SDAG-GFX1100-TRUE16-NEXT:    s_setpc_b64 s[30:31]
+;
+; SDAG-GFX1100-FAKE16-LABEL: v_mad_mix_v3f32_clamp_precvt:
+; SDAG-GFX1100-FAKE16:       ; %bb.0:
+; SDAG-GFX1100-FAKE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SDAG-GFX1100-FAKE16-NEXT:    v_fma_mix_f32 v6, v0, v2, v4 op_sel:[1,1,1] op_sel_hi:[1,1,1] clamp
+; SDAG-GFX1100-FAKE16-NEXT:    v_fma_mix_f32 v0, v0, v2, v4 op_sel_hi:[1,1,1] clamp
+; SDAG-GFX1100-FAKE16-NEXT:    v_fma_mix_f32 v1, v1, v3, v5 op_sel_hi:[1,1,1] clamp
+; SDAG-GFX1100-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; SDAG-GFX1100-FAKE16-NEXT:    v_cvt_f16_f32_e32 v2, v6
+; SDAG-GFX1100-FAKE16-NEXT:    v_cvt_f16_f32_e32 v0, v0
+; SDAG-GFX1100-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; SDAG-GFX1100-FAKE16-NEXT:    v_cvt_f16_f32_e32 v1, v1
+; SDAG-GFX1100-FAKE16-NEXT:    v_pack_b32_f16 v0, v0, v2
+; SDAG-GFX1100-FAKE16-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; SDAG-GFX900-LABEL: v_mad_mix_v3f32_clamp_precvt:
 ; SDAG-GFX900:       ; %bb.0:
@@ -2028,23 +2244,48 @@ define <3 x half> @v_mad_mix_v3f32_clamp_precvt(<3 x half> %src0, <3 x half> %sr
 }
 
 define <4 x half> @v_mad_mix_v4f32_clamp_precvt(<4 x half> %src0, <4 x half> %src1, <4 x half> %src2) #0 {
-; SDAG-GFX1100-LABEL: v_mad_mix_v4f32_clamp_precvt:
-; SDAG-GFX1100:       ; %bb.0:
-; SDAG-GFX1100-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SDAG-GFX1100-NEXT:    v_fma_mix_f32 v6, v1, v3, v5 op_sel:[1,1,1] op_sel_hi:[1,1,1] clamp
-; SDAG-GFX1100-NEXT:    v_fma_mix_f32 v7, v0, v2, v4 op_sel:[1,1,1] op_sel_hi:[1,1,1] clamp
-; SDAG-GFX1100-NEXT:    v_fma_mix_f32 v0, v0, v2, v4 op_sel_hi:[1,1,1] clamp
-; SDAG-GFX1100-NEXT:    v_fma_mix_f32 v1, v1, v3, v5 op_sel_hi:[1,1,1] clamp
-; SDAG-GFX1100-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; SDAG-GFX1100-NEXT:    v_cvt_f16_f32_e32 v2, v6
-; SDAG-GFX1100-NEXT:    v_cvt_f16_f32_e32 v3, v7
-; SDAG-GFX1100-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; SDAG-GFX1100-NEXT:    v_cvt_f16_f32_e32 v0, v0
-; SDAG-GFX1100-NEXT:    v_cvt_f16_f32_e32 v1, v1
-; SDAG-GFX1100-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; SDAG-GFX1100-NEXT:    v_pack_b32_f16 v0, v0, v3
-; SDAG-GFX1100-NEXT:    v_pack_b32_f16 v1, v1, v2
-; SDAG-GFX1100-NEXT:    s_setpc_b64 s[30:31]
+; SDAG-GFX1100-TRUE16-LABEL: v_mad_mix_v4f32_clamp_precvt:
+; SDAG-GFX1100-TRUE16:       ; %bb.0:
+; SDAG-GFX1100-TRUE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SDAG-GFX1100-TRUE16-NEXT:    v_mov_b16_e32 v6.l, v1.l
+; SDAG-GFX1100-TRUE16-NEXT:    v_mov_b16_e32 v7.l, v3.l
+; SDAG-GFX1100-TRUE16-NEXT:    v_mov_b16_e32 v8.l, v0.l
+; SDAG-GFX1100-TRUE16-NEXT:    v_mov_b16_e32 v9.l, v2.l
+; SDAG-GFX1100-TRUE16-NEXT:    v_mov_b16_e32 v10.l, v4.l
+; SDAG-GFX1100-TRUE16-NEXT:    v_mov_b16_e32 v11.l, v5.l
+; SDAG-GFX1100-TRUE16-NEXT:    v_fma_mix_f32 v1, v1, v3, v5 op_sel:[1,1,1] op_sel_hi:[1,1,1] clamp
+; SDAG-GFX1100-TRUE16-NEXT:    v_fma_mix_f32 v0, v0, v2, v4 op_sel:[1,1,1] op_sel_hi:[1,1,1] clamp
+; SDAG-GFX1100-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; SDAG-GFX1100-TRUE16-NEXT:    v_fma_mix_f32 v2, v8, v9, v10 op_sel_hi:[1,1,1] clamp
+; SDAG-GFX1100-TRUE16-NEXT:    v_fma_mix_f32 v3, v6, v7, v11 op_sel_hi:[1,1,1] clamp
+; SDAG-GFX1100-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; SDAG-GFX1100-TRUE16-NEXT:    v_cvt_f16_f32_e32 v1.l, v1
+; SDAG-GFX1100-TRUE16-NEXT:    v_cvt_f16_f32_e32 v0.l, v0
+; SDAG-GFX1100-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; SDAG-GFX1100-TRUE16-NEXT:    v_cvt_f16_f32_e32 v0.h, v2
+; SDAG-GFX1100-TRUE16-NEXT:    v_cvt_f16_f32_e32 v1.h, v3
+; SDAG-GFX1100-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; SDAG-GFX1100-TRUE16-NEXT:    v_pack_b32_f16 v0, v0.h, v0.l
+; SDAG-GFX1100-TRUE16-NEXT:    v_pack_b32_f16 v1, v1.h, v1.l
+; SDAG-GFX1100-TRUE16-NEXT:    s_setpc_b64 s[30:31]
+;
+; SDAG-GFX1100-FAKE16-LABEL: v_mad_mix_v4f32_clamp_precvt:
+; SDAG-GFX1100-FAKE16:       ; %bb.0:
+; SDAG-GFX1100-FAKE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SDAG-GFX1100-FAKE16-NEXT:    v_fma_mix_f32 v6, v1, v3, v5 op_sel:[1,1,1] op_sel_hi:[1,1,1] clamp
+; SDAG-GFX1100-FAKE16-NEXT:    v_fma_mix_f32 v7, v0, v2, v4 op_sel:[1,1,1] op_sel_hi:[1,1,1] clamp
+; SDAG-GFX1100-FAKE16-NEXT:    v_fma_mix_f32 v0, v0, v2, v4 op_sel_hi:[1,1,1] clamp
+; SDAG-GFX1100-FAKE16-NEXT:    v_fma_mix_f32 v1, v1, v3, v5 op_sel_hi:[1,1,1] clamp
+; SDAG-GFX1100-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; SDAG-GFX1100-FAKE16-NEXT:    v_cvt_f16_f32_e32 v2, v6
+; SDAG-GFX1100-FAKE16-NEXT:    v_cvt_f16_f32_e32 v3, v7
+; SDAG-GFX1100-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; SDAG-GFX1100-FAKE16-NEXT:    v_cvt_f16_f32_e32 v0, v0
+; SDAG-GFX1100-FAKE16-NEXT:    v_cvt_f16_f32_e32 v1, v1
+; SDAG-GFX1100-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; SDAG-GFX1100-FAKE16-NEXT:    v_pack_b32_f16 v0, v0, v3
+; SDAG-GFX1100-FAKE16-NEXT:    v_pack_b32_f16 v1, v1, v2
+; SDAG-GFX1100-FAKE16-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; SDAG-GFX900-LABEL: v_mad_mix_v4f32_clamp_precvt:
 ; SDAG-GFX900:       ; %bb.0:
diff --git a/llvm/test/CodeGen/AMDGPU/mad-mix.ll b/llvm/test/CodeGen/AMDGPU/mad-mix.ll
index 30e3bc3ba5da8..4c2a16c17b38a 100644
--- a/llvm/test/CodeGen/AMDGPU/mad-mix.ll
+++ b/llvm/test/CodeGen/AMDGPU/mad-mix.ll
@@ -1,5 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=amdgcn -mcpu=gfx1100 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX1100,SDAG-GFX1100 %s
+; RUN: llc -mtriple=amdgcn -mcpu=gfx1100 -mattr=+real-true16 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX1100,SDAG-GFX1100,SDAG-GFX1100-TRUE16 %s
+; RUN: llc -mtriple=amdgcn -mcpu=gfx1100 -mattr=-real-true16 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX1100,SDAG-GFX1100,SDAG-GFX1100-FAKE16 %s
 ; RUN: llc -mtriple=amdgcn -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX900,SDAG-GFX900 %s
 ; RUN: llc -mtriple=amdgcn -mcpu=gfx906 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX906,SDAG-GFX906 %s
 ; RUN: llc -mtriple=amdgcn -mcpu=gfx9-generic -verify-machineinstrs --amdhsa-code-object-version=6 < %s | FileCheck -check-prefixes=GFX9GEN,SDAG-GFX9GEN %s
@@ -197,14 +198,26 @@ define float @v_mad_mix_f32_f16hi_f16hi_f16hi_elt(<2 x half> %src0, <2 x half> %
 }
 
 define <2 x float> @v_mad_mix_v2f32(<2 x half> %src0, <2 x half> %src1, <2 x half> %src2) #0 {
-; GFX1100-LABEL: v_mad_mix_v2f32:
-; GFX1100:       ; %bb.0:
-; GFX1100-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX1100-NEXT:    v_fma_mix_f32 v3, v0, v1, v2 op_sel_hi:[1,1,1]
-; GFX1100-NEXT:    v_fma_mix_f32 v1, v0, v1, v2 op_sel:[1,1,1] op_sel_hi:[1,1,1]
-; GFX1100-NEXT:    s_delay_alu instid0(VALU_DEP_2)
-; GFX1100-NEXT:    v_mov_b32_e32 v0, v3
-; GFX1100-NEXT:    s_setpc_b64 s[30:31]
+; SDAG-GFX1100-TRUE16-LABEL: v_mad_mix_v2f32:
+; SDAG-GFX1100-TRUE16:       ; %bb.0:
+; SDAG-GFX1100-TRUE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SDAG-GFX1100-TRUE16-NEXT:    v_mov_b16_e32 v3.l, v0.l
+; SDAG-GFX1100-TRUE16-NEXT:    v_mov_b16_e32 v4.l, v1.l
+; SDAG-GFX1100-TRUE16-NEXT:    v_mov_b16_e32 v5.l, v2.l
+; SDAG-GFX1100-TRUE16-NEXT:    v_fma_mix_f32 v1, v0, v1, v2 op_sel:[1,1,1] op_sel_hi:[1,1,1]
+; SDAG-GFX1100-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; SDAG-GFX1100-TRUE16-NEXT:    v_fma_mix_f32 v3, v3, v4, v5 op_sel_hi:[1,1,1]
+; SDAG-GFX1100-TRUE16-NEXT:    v_mov_b32_e32 v0, v3
+; SDAG-GFX1100-TRUE16-NEXT:    s_setpc_b64 s[30:31]
+;
+; SDAG-GFX1100-FAKE16-LABEL: v_mad_mix_v2f32:
+; SDAG-GFX1100-FAKE16:       ; %bb.0:
+; SDAG-GFX1100-FAKE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SDAG-GFX1100-FAKE16-NEXT:    v_fma_mix_f32 v3, v0, v1, v2 op_sel_hi:[1,1,1]
+; SDAG-GFX1100-FAKE16-NEXT:    v_fma_mix_f32 v1, v0, v1, v2 op_sel:[1,1,1] op_sel_hi:[1,1,1]
+; SDAG-GFX1100-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2)
+; SDAG-GFX1100-FAKE16-NEXT:    v_mov_b32_e32 v0, v3
+; SDAG-GFX1100-FAKE16-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; SDAG-GFX900-LABEL: v_mad_mix_v2f32:
 ; SDAG-GFX900:       ; %bb.0:
@@ -268,6 +281,15 @@ define <2 x float> @v_mad_mix_v2f32(<2 x half> %src0, <2 x half> %src1, <2 x hal
 ; SDAG-CI-NEXT:    v_mac_f32_e32 v0, v4, v2
 ; SDAG-CI-NEXT:    s_setpc_b64 s[30:31]
 ;
+; GISEL-GFX1100-LABEL: v_mad_mix_v2f32:
+; GISEL-GFX1100:       ; %bb.0:
+; GISEL-GFX1100-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GISEL-GFX1100-NEXT:    v_fma_mix_f32 v3, v0, v1, v2 op_sel_hi:[1,1,1]
+; GISEL-GFX1100-NEXT:    v_fma_mix_f32 v1, v0, v1, v2 op_sel:[1,1,1] op_sel_hi:[1,1,1]
+; GISEL-GFX1100-NEXT:    s_delay_alu instid0(VALU_DEP_2)
+; GISEL-GFX1100-NEXT:    v_mov_b32_e32 v0, v3
+; GISEL-GFX1100-NEXT:    s_setpc_b64 s[30:31]
+;
 ; GISEL-GFX900-LABEL: v_mad_mix_v2f32:
 ; GISEL-GFX900:       ; %bb.0:
 ; GISEL-GFX900-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -330,14 +352,24 @@ define <2 x float> @v_mad_mix_v2f32(<2 x half> %src0, <2 x half> %src1, <2 x hal
 }
 
 define <2 x float> @v_mad_mix_v2f32_shuffle(<2 x half> %src0, <2 x half> %src1, <2 x half> %src2) #0 {
-; GFX1100-LABEL: v_mad_mix_v2f32_shuffle:
-; GFX1100:       ; %bb.0:
-; GFX1100-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX1100-NEXT:    v_fma_mix_f32 v3, v0, v1, v2 op_sel:[1,0,1] op_sel_hi:[1,1,1]
-; GFX1100-NEXT:    v_fma_mix_f32 v1, v0, v1, v2 op_sel:[0,1,1] op_sel_hi:[1,1,1]
-; GFX1100-NEXT:    s_delay_alu instid0(VALU_DEP_2)
-; GFX1100-NEXT:    v_mov_b32_e32 v0, v3
-; GFX1100-NEXT:    s_setpc_b64 s[30:31]
+; SDAG-GFX1100-TRUE16-LABEL: v_mad_mix_v2f32_shuffle:
+; SDAG-GFX1100-TRUE16:       ; %bb.0:
+; SDAG-GFX1100-TRUE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SDAG-GFX1100-TRUE16-NEXT:    v_mov_b16_e32 v3.l, v1.l
+; SDAG-GFX1100-TRUE16-NEXT:    v_mov_b16_e32 v4.l, v0.l
+; SDAG-GFX1100-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; SDAG-GFX1100-TRUE16-NEXT:    v_fma_mix_f32 v0, v0, v3, v2 op_sel:[1,0,1] op_sel_hi:[1,1,1]
+; SDAG-GFX1100-TRUE16-NEXT:    v_fma_mix_f32 v1, v4, v1, v2 op_sel:[0,1,1] op_sel_hi:[1,1,1]
+; SDAG-GFX1100-TRUE16-NEXT:    s_setpc_b64 s[30:31]
+;
+; SDAG-GFX1100-FAKE16-LABEL: v_mad_mix_v2f32_shuffle:
+; SDAG-GFX1100-FAKE16:       ; %bb.0:
+; SDAG-GFX1100-FAKE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SDAG-GFX1100-FAKE16-NEXT:    v_fma_mix_f32 v3, v0, v1, v2 op_sel:[1,0,1] op_sel_hi:[1,1,1]
+; SDAG-GFX1100-FAKE16-NEXT:    v_fma_mix_f32 v1, v0, v1, v2 op_sel:[0,1,1] op_sel_hi:[1,1,1]
+; SDAG-GFX1100-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2)
+; SDAG-GFX1100-FAKE16-NEXT:    v_mov_b32_e32 v0, v3
+; SDAG-GFX1100-FAKE16-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX900-LABEL: v_mad_mix_v2f32_shuffle:
 ; GFX900:       ; %bb.0:
@@ -396,6 +428,15 @@ define <2 x float> @v_mad_mix_v2f32_shuffle(<2 x half> %src0, <2 x half> %src1,
 ; SDAG-CI-NEXT:    v_mad_f32 v1, v4, v3, v5
 ; SDAG-CI-NEXT:    s_setpc_b64 s[30:31]
 ;
+; GISEL-GFX1100-LABEL: v_mad_mix_v2f32_shuffle:
+; GISEL-GFX1100:       ; %bb.0:
+; GISEL-GFX1100-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GISEL-GFX1100-NEXT:    v_fma_mix_f32 v3, v0, v1, v2 op_sel:[1,0,1] op_sel_hi:[1,1,1]
+; GISEL-GFX1100-NEXT:    v_fma_mix_f32 v1, v0, v1, v2 op_sel:[0,1,1] op_sel_hi:[1,1,1]
+; GISEL-GFX1100-NEXT:    s_delay_alu instid0(VALU_DEP_2)
+; GISEL-GFX1100-NEXT:    v_mov_b32_e32 v0, v3
+; GISEL-GFX1100-NEXT:    s_setpc_b64 s[30:31]
+;
 ; GISEL-CI-LABEL: v_mad_mix_v2f32_shuffle:
 ; GISEL-CI:       ; %bb.0:
 ; GISEL-CI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -1214,15 +1255,28 @@ define float @v_mad_mix_f32_f16lo_f16lo_cvtf16imm63(half %src0, half %src1) #0 {
 }
 
 define <2 x float> @v_mad_mix_v2f32_f32imm1(<2 x half> %src0, <2 x half> %src1) #0 {
-; SDAG-GFX1100-LABEL: v_mad_mix_v2f32_f32imm1:
-; SDAG-GFX1100:       ; %bb.0:
-; SDAG-GFX1100-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SDAG-GFX1100-NEXT:    s_mov_b32 s0, 1.0
-; SDAG-GFX1100-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; SDAG-GFX1100-NEXT:    v_fma_mix_f32 v2, v0, v1, s0 op_sel_hi:[1,1,0]
-; SDAG-GFX1100-NEXT:    v_fma_mix_f32 v1, v0, v1, s0 op_sel:[1,1,0] op_sel_hi:[1,1,0]
-; SDAG-GFX1100-NEXT:    v_mov_b32_e32 v0, v2
-; SDAG-GFX1100-NEXT:    s_setpc_b64 s[30:31]
+; SDAG-GFX1100-TRUE16-LABEL: v_mad_mix_v2f32_f32imm1:
+; SDAG-GFX1100-TRUE16:       ; %bb.0:
+; SDAG-GFX1100-TRUE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SDAG-GFX1100-TRUE16-NEXT:    v_mov_b16_e32 v2.l, v0.l
+; SDAG-GFX1100-TRUE16-NEXT:    v_mov_b16_e32 v3.l, v1.l
+; SDAG-GFX1100-TRUE16-NEXT:    s_mov_b32 s0, 1.0
+; SDAG-GFX1100-TRUE16-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; SDAG-GFX1100-TRUE16-NEXT:    v_fma_mix_f32 v1, v0, v1, s0 op_sel:[1,1,0] op_sel_hi:[1,1,0]
+; SDAG-GFX1100-TRUE16-NEXT:    v_fma_mix_f32 v2, v2, v3, s0 op_sel_hi:[1,1,0]
+; SDAG-GFX1100-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; SDAG-GFX1100-TRUE16-NEXT:    v_mov_b32_e32 v0, v2
+; SDAG-GFX1100-TRUE16-NEXT:    s_setpc_b64 s[30:31]
+;
+; SDAG-GFX1100-FAKE16-LABEL: v_mad_mix_v2f32_f32imm1:
+; SDAG-GFX1100-FAKE16:       ; %bb.0:
+; SDAG-GFX1100-FAKE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SDAG-GFX1100-FAKE16-NEXT:    s_mov_b32 s0, 1.0
+; SDAG-GFX1100-FAKE16-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; SDAG-GFX1100-FAKE16-NEXT:    v_fma_mix_f32 v2, v0, v1, s0 op_sel_hi:[1,1,0]
+; SDAG-GFX1100-FAKE16-NEXT:    v_fma_mix_f32 v1, v0, v1, s0 op_sel:[1,1,0] op_sel_hi:[1,1,0]
+; SDAG-GFX1100-FAKE16-NEXT:    v_mov_b32_e32 v0, v2
+; SDAG-GFX1100-FAKE16-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; SDAG-GFX900-LABEL: v_mad_mix_v2f32_f32imm1:
 ; SDAG-GFX900:       ; %bb.0:
@@ -1346,15 +1400,28 @@ define <2 x float> @v_mad_mix_v2f32_f32imm1(<2 x half> %src0, <2 x half> %src1)
 }
 
 define <2 x float> @v_mad_mix_v2f32_cvtf16imminv2pi(<2 x half> %src0, <2 x half> %src1) #0 {
-; SDAG-GFX1100-LABEL: v_mad_mix_v2f32_cvtf16imminv2pi:
-; SDAG-GFX1100:       ; %bb.0:
-; SDAG-GFX1100-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SDAG-GFX1100-NEXT:    s_mov_b32 s0, 0x3e230000
-; SDAG-GFX1100-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; SDAG-GFX1100-NEXT:    v_fma_mix_f32 v2, v0, v1, s0 op_sel_hi:[1,1,0]
-; SDAG-GFX1100-NEXT:    v_fma_mix_f32 v1, v0, v1, s0 op_sel:[1,1,0] op_sel_hi:[1,1,0]
-; SDAG-GFX1100-NEXT:    v_mov_b32_e32 v0, v2
-; SDAG-GFX1100-NEXT:    s_setpc_b64 s[30:31]
+; SDAG-GFX1100-TRUE16-LABEL: v_mad_mix_v2f32_cvtf16imminv2pi:
+; SDAG-GFX1100-TRUE16:       ; %bb.0:
+; SDAG-GFX1100-TRUE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SDAG-GFX1100-TRUE16-NEXT:    v_mov_b16_e32 v2.l, v0.l
+; SDAG-GFX1100-TRUE16-NEXT:    v_mov_b16_e32 v3.l, v1.l
+; SDAG-GFX1100-TRUE16-NEXT:    s_mov_b32 s0, 0x3e230000
+; SDAG-GFX1100-TRUE16-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; SDAG-GFX1100-TRUE16-NEXT:    v_fma_mix_f32 v1, v0, v1, s0 op_sel:[1,1,0] op_sel_hi:[1,1,0]
+; SDAG-GFX1100-TRUE16-NEXT:    v_fma_mix_f32 v2, v2, v3, s0 op_sel_hi:[1,1,0]
+; SDAG-GFX1100-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; SDAG-GFX1100-TRUE16-NEXT:    v_mov_b32_e32 v0, v2
+; SDAG-GFX1100-TRUE16-NEXT:    s_setpc_b64 s[30:31]
+;
+; SDAG-GFX1100-FAKE16-LABEL: v_mad_mix_v2f32_cvtf16imminv2pi:
+; SDAG-GFX1100-FAKE16:       ; %bb.0:
+; SDAG-GFX1100-FAKE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SDAG-GFX1100-FAKE16-NEXT:    s_mov_b32 s0, 0x3e230000
+; SDAG-GFX1100-FAKE16-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; SDAG-GFX1100-FAKE16-NEXT:    v_fma_mix_f32 v2, v0, v1, s0 op_sel_hi:[1,1,0]
+; SDAG-GFX1100-FAKE16-NEXT:    v_fma_mix_f32 v1, v0, v1, s0 op_sel:[1,1,0] op_sel_hi:[1,1,0]
+; SDAG-GFX1100-FAKE16-NEXT:    v_mov_b32_e32 v0, v2
+; SDAG-GFX1100-FAKE16-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; SDAG-GFX900-LABEL: v_mad_mix_v2f32_cvtf16imminv2pi:
 ; SDAG-GFX900:       ; %bb.0:
@@ -1485,15 +1552,28 @@ define <2 x float> @v_mad_mix_v2f32_cvtf16imminv2pi(<2 x half> %src0, <2 x half>
 }
 
 define <2 x float> @v_mad_mix_v2f32_f32imminv2pi(<2 x half> %src0, <2 x half> %src1) #0 {
-; SDAG-GFX1100-LABEL: v_mad_mix_v2f32_f32imminv2pi:
-; SDAG-GFX1100:       ; %bb.0:
-; SDAG-GFX1100-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SDAG-GFX1100-NEXT:    s_mov_b32 s0, 0.15915494
-; SDAG-GFX1100-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; SDAG-GFX1100-NEXT:    v_fma_mix_f32 v2, v0, v1, s0 op_sel_hi:[1,1,0]
-; SDAG-GFX1100-NEXT:    v_fma_mix_f32 v1, v0, v1, s0 op_sel:[1,1,0] op_sel_hi:[1,1,0]
-; SDAG-GFX1100-NEXT:    v_mov_b32_e32 v0, v2
-; SDAG-GFX1100-NEXT:    s_setpc_b64 s[30:31]
+; SDAG-GFX1100-TRUE16-LABEL: v_mad_mix_v2f32_f32imminv2pi:
+; SDAG-GFX1100-TRUE16:       ; %bb.0:
+; SDAG-GFX1100-TRUE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SDAG-GFX1100-TRUE16-NEXT:    v_mov_b16_e32 v2.l, v0.l
+; SDAG-GFX1100-TRUE16-NEXT:    v_mov_b16_e32 v3.l, v1.l
+; SDAG-GFX1100-TRUE16-NEXT:    s_mov_b32 s0, 0.15915494
+; SDAG-GFX1100-TRUE16-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; SDAG-GFX1100-TRUE16-NEXT:    v_fma_mix_f32 v1, v0, v1, s0 op_sel:[1,1,0] op_sel_hi:[1,1,0]
+; SDAG-GFX1100-TRUE16-NEXT:    v_fma_mix_f32 v2, v2, v3, s0 op_sel_hi:[1,1,0]
+; SDAG-GFX1100-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; SDAG-GFX1100-TRUE16-NEXT:    v_mov_b32_e32 v0, v2
+; SDAG-GFX1100-TRUE16-NEXT:    s_setpc_b64 s[30:31]
+;
+; SDAG-GFX1100-FAKE16-LABEL: v_mad_mix_v2f32_f32imminv2pi:
+; SDAG-GFX1100-FAKE16:       ; %bb.0:
+; SDAG-GFX1100-FAKE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SDAG-GFX1100-FAKE16-NEXT:    s_mov_b32 s0, 0.15915494
+; SDAG-GFX1100-FAKE16-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; SDAG-GFX1100-FAKE16-NEXT:    v_fma_mix_f32 v2, v0, v1, s0 op_sel_hi:[1,1,0]
+; SDAG-GFX1100-FAKE16-NEXT:    v_fma_mix_f32 v1, v0, v1, s0 op_sel:[1,1,0] op_sel_hi:[1,1,0]
+; SDAG-GFX1100-FAKE16-NEXT:    v_mov_b32_e32 v0, v2
+; SDAG-GFX1100-FAKE16-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; SDAG-GFX900-LABEL: v_mad_mix_v2f32_f32imminv2pi:
 ; SDAG-GFX900:       ; %bb.0:
@@ -1887,16 +1967,30 @@ define float @v_mad_mix_f32_f16lo_f16lo_f32_denormals(half %src0, half %src1, fl
 }
 
 define float @v_mad_mix_f32_f16lo_f16lo_f16lo_f32_denormals_fmulfadd(half %src0, half %src1, half %src2) #1 {
-; GFX1100-LABEL: v_mad_mix_f32_f16lo_f16lo_f16lo_f32_denormals_fmulfadd:
-; GFX1100:       ; %bb.0:
-; GFX1100-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX1100-NEXT:    v_cvt_f32_f16_e32 v0, v0
-; GFX1100-NEXT:    v_cvt_f32_f16_e32 v1, v1
-; GFX1100-NEXT:    v_cvt_f32_f16_e32 v2, v2
-; GFX1100-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1100-NEXT:    v_mul_f32_e32 v0, v0, v1
-; GFX1100-NEXT:    v_add_f32_e32 v0, v0, v2
-; GFX1100-NEXT:    s_setpc_b64 s[30:31]
+; SDAG-GFX1100-TRUE16-LABEL: v_mad_mix_f32_f16lo_f16lo_f16lo_f32_denormals_fmulfadd:
+; SDAG-GFX1100-TRUE16:       ; %bb.0:
+; SDAG-GFX1100-TRUE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SDAG-GFX1100-TRUE16-NEXT:    v_mov_b16_e32 v0.h, v1.l
+; SDAG-GFX1100-TRUE16-NEXT:    v_mov_b16_e32 v1.l, v2.l
+; SDAG-GFX1100-TRUE16-NEXT:    v_cvt_f32_f16_e32 v2, v0.l
+; SDAG-GFX1100-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; SDAG-GFX1100-TRUE16-NEXT:    v_cvt_f32_f16_e32 v0, v0.h
+; SDAG-GFX1100-TRUE16-NEXT:    v_cvt_f32_f16_e32 v1, v1.l
+; SDAG-GFX1100-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; SDAG-GFX1100-TRUE16-NEXT:    v_mul_f32_e32 v0, v2, v0
+; SDAG-GFX1100-TRUE16-NEXT:    v_add_f32_e32 v0, v0, v1
+; SDAG-GFX1100-TRUE16-NEXT:    s_setpc_b64 s[30:31]
+;
+; SDAG-GFX1100-FAKE16-LABEL: v_mad_mix_f32_f16lo_f16lo_f16lo_f32_denormals_fmulfadd:
+; SDAG-GFX1100-FAKE16:       ; %bb.0:
+; SDAG-GFX1100-FAKE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SDAG-GFX1100-FAKE16-NEXT:    v_cvt_f32_f16_e32 v0, v0
+; SDAG-GFX1100-FAKE16-NEXT:    v_cvt_f32_f16_e32 v1, v1
+; SDAG-GFX1100-FAKE16-NEXT:    v_cvt_f32_f16_e32 v2, v2
+; SDAG-GFX1100-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; SDAG-GFX1100-FAKE16-NEXT:    v_mul_f32_e32 v0, v0, v1
+; SDAG-GFX1100-FAKE16-NEXT:    v_add_f32_e32 v0, v0, v2
+; SDAG-GFX1100-FAKE16-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX900-LABEL: v_mad_mix_f32_f16lo_f16lo_f16lo_f32_denormals_fmulfadd:
 ; GFX900:       ; %bb.0:
@@ -1945,6 +2039,17 @@ define float @v_mad_mix_f32_f16lo_f16lo_f16lo_f32_denormals_fmulfadd(half %src0,
 ; SDAG-CI-NEXT:    v_add_f32_e32 v0, v0, v2
 ; SDAG-CI-NEXT:    s_setpc_b64 s[30:31]
 ;
+; GISEL-GFX1100-LABEL: v_mad_mix_f32_f16lo_f16lo_f16lo_f32_denormals_fmulfadd:
+; GISEL-GFX1100:       ; %bb.0:
+; GISEL-GFX1100-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GISEL-GFX1100-NEXT:    v_cvt_f32_f16_e32 v0, v0
+; GISEL-GFX1100-NEXT:    v_cvt_f32_f16_e32 v1, v1
+; GISEL-GFX1100-NEXT:    v_cvt_f32_f16_e32 v2, v2
+; GISEL-GFX1100-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GISEL-GFX1100-NEXT:    v_mul_f32_e32 v0, v0, v1
+; GISEL-GFX1100-NEXT:    v_add_f32_e32 v0, v0, v2
+; GISEL-GFX1100-NEXT:    s_setpc_b64 s[30:31]
+;
 ; GISEL-CI-LABEL: v_mad_mix_f32_f16lo_f16lo_f16lo_f32_denormals_fmulfadd:
 ; GISEL-CI:       ; %bb.0:
 ; GISEL-CI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -1963,15 +2068,27 @@ define float @v_mad_mix_f32_f16lo_f16lo_f16lo_f32_denormals_fmulfadd(half %src0,
 }
 
 define float @v_mad_mix_f32_f16lo_f16lo_f32_denormals_fmulfadd(half %src0, half %src1, float %src2) #1 {
-; GFX1100-LABEL: v_mad_mix_f32_f16lo_f16lo_f32_denormals_fmulfadd:
-; GFX1100:       ; %bb.0:
-; GFX1100-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX1100-NEXT:    v_cvt_f32_f16_e32 v0, v0
-; GFX1100-NEXT:    v_cvt_f32_f16_e32 v1, v1
-; GFX1100-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1100-NEXT:    v_mul_f32_e32 v0, v0, v1
-; GFX1100-NEXT:    v_add_f32_e32 v0, v0, v2
-; GFX1100-NEXT:    s_setpc_b64 s[30:31]
+; SDAG-GFX1100-TRUE16-LABEL: v_mad_mix_f32_f16lo_f16lo_f32_denormals_fmulfadd:
+; SDAG-GFX1100-TRUE16:       ; %bb.0:
+; SDAG-GFX1100-TRUE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SDAG-GFX1100-TRUE16-NEXT:    v_mov_b16_e32 v0.h, v1.l
+; SDAG-GFX1100-TRUE16-NEXT:    v_cvt_f32_f16_e32 v1, v0.l
+; SDAG-GFX1100-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; SDAG-GFX1100-TRUE16-NEXT:    v_cvt_f32_f16_e32 v0, v0.h
+; SDAG-GFX1100-TRUE16-NEXT:    v_mul_f32_e32 v0, v1, v0
+; SDAG-GFX1100-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; SDAG-GFX1100-TRUE16-NEXT:    v_add_f32_e32 v0, v0, v2
+; SDAG-GFX1100-TRUE16-NEXT:    s_setpc_b64 s[30:31]
+;
+; SDAG-GFX1100-FAKE16-LABEL: v_mad_mix_f32_f16lo_f16lo_f32_denormals_fmulfadd:
+; SDAG-GFX1100-FAKE16:       ; %bb.0:
+; SDAG-GFX1100-FAKE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SDAG-GFX1100-FAKE16-NEXT:    v_cvt_f32_f16_e32 v0, v0
+; SDAG-GFX1100-FAKE16-NEXT:    v_cvt_f32_f16_e32 v1, v1
+; SDAG-GFX1100-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; SDAG-GFX1100-FAKE16-NEXT:    v_mul_f32_e32 v0, v0, v1
+; SDAG-GFX1100-FAKE16-NEXT:    v_add_f32_e32 v0, v0, v2
+; SDAG-GFX1100-FAKE16-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX900-LABEL: v_mad_mix_f32_f16lo_f16lo_f32_denormals_fmulfadd:
 ; GFX900:       ; %bb.0:
@@ -2016,6 +2133,16 @@ define float @v_mad_mix_f32_f16lo_f16lo_f32_denormals_fmulfadd(half %src0, half
 ; SDAG-CI-NEXT:    v_add_f32_e32 v0, v0, v2
 ; SDAG-CI-NEXT:    s_setpc_b64 s[30:31]
 ;
+; GISEL-GFX1100-LABEL: v_mad_mix_f32_f16lo_f16lo_f32_denormals_fmulfadd:
+; GISEL-GFX1100:       ; %bb.0:
+; GISEL-GFX1100-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GISEL-GFX1100-NEXT:    v_cvt_f32_f16_e32 v0, v0
+; GISEL-GFX1100-NEXT:    v_cvt_f32_f16_e32 v1, v1
+; GISEL-GFX1100-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GISEL-GFX1100-NEXT:    v_mul_f32_e32 v0, v0, v1
+; GISEL-GFX1100-NEXT:    v_add_f32_e32 v0, v0, v2
+; GISEL-GFX1100-NEXT:    s_setpc_b64 s[30:31]
+;
 ; GISEL-CI-LABEL: v_mad_mix_f32_f16lo_f16lo_f32_denormals_fmulfadd:
 ; GISEL-CI:       ; %bb.0:
 ; GISEL-CI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -2229,14 +2356,23 @@ define float @v_mad_mix_f32_negprecvtf16lo_f16lo_f16lo(i32 %src0.arg, half %src1
 ; Make sure we don't fold pre-cvt fneg if we already have a fabs
 
 define float @v_mad_mix_f32_precvtnegf16hi_abs_f16lo_f16lo(i32 %src0.arg, half %src1, half %src2) #0 {
-; GFX1100-LABEL: v_mad_mix_f32_precvtnegf16hi_abs_f16lo_f16lo:
-; GFX1100:       ; %bb.0:
-; GFX1100-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX1100-NEXT:    v_lshrrev_b32_e32 v0, 16, v0
-; GFX1100-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1100-NEXT:    v_xor_b32_e32 v0, 0x8000, v0
-; GFX1100-NEXT:    v_fma_mix_f32 v0, |v0|, v1, v2 op_sel_hi:[1,1,1]
-; GFX1100-NEXT:    s_setpc_b64 s[30:31]
+; SDAG-GFX1100-TRUE16-LABEL: v_mad_mix_f32_precvtnegf16hi_abs_f16lo_f16lo:
+; SDAG-GFX1100-TRUE16:       ; %bb.0:
+; SDAG-GFX1100-TRUE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SDAG-GFX1100-TRUE16-NEXT:    v_lshrrev_b32_e32 v0, 16, v0
+; SDAG-GFX1100-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; SDAG-GFX1100-TRUE16-NEXT:    v_xor_b16 v0.l, 0x8000, v0.l
+; SDAG-GFX1100-TRUE16-NEXT:    v_fma_mix_f32 v0, |v0|, v1, v2 op_sel_hi:[1,1,1]
+; SDAG-GFX1100-TRUE16-NEXT:    s_setpc_b64 s[30:31]
+;
+; SDAG-GFX1100-FAKE16-LABEL: v_mad_mix_f32_precvtnegf16hi_abs_f16lo_f16lo:
+; SDAG-GFX1100-FAKE16:       ; %bb.0:
+; SDAG-GFX1100-FAKE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SDAG-GFX1100-FAKE16-NEXT:    v_lshrrev_b32_e32 v0, 16, v0
+; SDAG-GFX1100-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; SDAG-GFX1100-FAKE16-NEXT:    v_xor_b32_e32 v0, 0x8000, v0
+; SDAG-GFX1100-FAKE16-NEXT:    v_fma_mix_f32 v0, |v0|, v1, v2 op_sel_hi:[1,1,1]
+; SDAG-GFX1100-FAKE16-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX900-LABEL: v_mad_mix_f32_precvtnegf16hi_abs_f16lo_f16lo:
 ; GFX900:       ; %bb.0:
@@ -2280,6 +2416,15 @@ define float @v_mad_mix_f32_precvtnegf16hi_abs_f16lo_f16lo(i32 %src0.arg, half %
 ; SDAG-CI-NEXT:    v_mad_f32 v0, v0, v1, v2
 ; SDAG-CI-NEXT:    s_setpc_b64 s[30:31]
 ;
+; GISEL-GFX1100-LABEL: v_mad_mix_f32_precvtnegf16hi_abs_f16lo_f16lo:
+; GISEL-GFX1100:       ; %bb.0:
+; GISEL-GFX1100-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GISEL-GFX1100-NEXT:    v_lshrrev_b32_e32 v0, 16, v0
+; GISEL-GFX1100-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GISEL-GFX1100-NEXT:    v_xor_b32_e32 v0, 0x8000, v0
+; GISEL-GFX1100-NEXT:    v_fma_mix_f32 v0, |v0|, v1, v2 op_sel_hi:[1,1,1]
+; GISEL-GFX1100-NEXT:    s_setpc_b64 s[30:31]
+;
 ; GISEL-CI-LABEL: v_mad_mix_f32_precvtnegf16hi_abs_f16lo_f16lo:
 ; GISEL-CI:       ; %bb.0:
 ; GISEL-CI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
diff --git a/llvm/test/CodeGen/AMDGPU/mad.u16.ll b/llvm/test/CodeGen/AMDGPU/mad.u16.ll
index c500942840053..c4c0dc6998265 100644
--- a/llvm/test/CodeGen/AMDGPU/mad.u16.ll
+++ b/llvm/test/CodeGen/AMDGPU/mad.u16.ll
@@ -2,7 +2,8 @@
 ; RUN: llc -mtriple=amdgcn -mcpu=gfx803 -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=GFX8 %s
 ; RUN: llc -mtriple=amdgcn -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=GFX9 %s
 ; RUN: llc -mtriple=amdgcn -mcpu=gfx1010 -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=GFX10 %s
-; RUN: llc -mtriple=amdgcn -mcpu=gfx1100 -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=GFX11 %s
+; RUN: llc -mtriple=amdgcn -mcpu=gfx1100 -mattr=+real-true16 -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefixes=GFX11,GFX11-TRUE16 %s
+; RUN: llc -mtriple=amdgcn -mcpu=gfx1100 -mattr=-real-true16 -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefixes=GFX11,GFX11-FAKE16 %s
 
 ; FIXME: GFX9 should be producing v_mad_u16 instead of v_mad_legacy_u16.
 
@@ -65,22 +66,44 @@ define amdgpu_kernel void @mad_u16(
 ; GFX10-NEXT:    global_store_short v0, v1, s[8:9]
 ; GFX10-NEXT:    s_endpgm
 ;
-; GFX11-LABEL: mad_u16:
-; GFX11:       ; %bb.0: ; %entry
-; GFX11-NEXT:    s_load_b256 s[0:7], s[4:5], 0x24
-; GFX11-NEXT:    v_dual_mov_b32 v3, 0 :: v_dual_and_b32 v0, 0x3ff, v0
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT:    v_lshlrev_b32_e32 v0, 1, v0
-; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX11-NEXT:    global_load_u16 v1, v0, s[2:3] glc dlc
-; GFX11-NEXT:    s_waitcnt vmcnt(0)
-; GFX11-NEXT:    global_load_u16 v2, v0, s[4:5] glc dlc
-; GFX11-NEXT:    s_waitcnt vmcnt(0)
-; GFX11-NEXT:    global_load_u16 v0, v0, s[6:7] glc dlc
-; GFX11-NEXT:    s_waitcnt vmcnt(0)
-; GFX11-NEXT:    v_mad_u16 v0, v1, v2, v0
-; GFX11-NEXT:    global_store_b16 v3, v0, s[0:1]
-; GFX11-NEXT:    s_endpgm
+; GFX11-TRUE16-LABEL: mad_u16:
+; GFX11-TRUE16:       ; %bb.0: ; %entry
+; GFX11-TRUE16-NEXT:    s_load_b256 s[0:7], s[4:5], 0x24
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v0, 1, v0
+; GFX11-TRUE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-TRUE16-NEXT:    global_load_u16 v1, v0, s[2:3] glc dlc
+; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT:    global_load_u16 v2, v0, s[4:5] glc dlc
+; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT:    global_load_u16 v3, v0, s[6:7] glc dlc
+; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v0.l, v1.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v0.h, v2.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v1.l, v3.l
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_mad_u16 v0.l, v0.l, v0.h, v1.l
+; GFX11-TRUE16-NEXT:    v_mov_b32_e32 v1, 0
+; GFX11-TRUE16-NEXT:    global_store_b16 v1, v0, s[0:1]
+; GFX11-TRUE16-NEXT:    s_endpgm
+;
+; GFX11-FAKE16-LABEL: mad_u16:
+; GFX11-FAKE16:       ; %bb.0: ; %entry
+; GFX11-FAKE16-NEXT:    s_load_b256 s[0:7], s[4:5], 0x24
+; GFX11-FAKE16-NEXT:    v_dual_mov_b32 v3, 0 :: v_dual_and_b32 v0, 0x3ff, v0
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v0, 1, v0
+; GFX11-FAKE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-FAKE16-NEXT:    global_load_u16 v1, v0, s[2:3] glc dlc
+; GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT:    global_load_u16 v2, v0, s[4:5] glc dlc
+; GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT:    global_load_u16 v0, v0, s[6:7] glc dlc
+; GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT:    v_mad_u16 v0, v1, v2, v0
+; GFX11-FAKE16-NEXT:    global_store_b16 v3, v0, s[0:1]
+; GFX11-FAKE16-NEXT:    s_endpgm
     ptr addrspace(1) %r,
     ptr addrspace(1) %a,
     ptr addrspace(1) %b,
@@ -121,11 +144,20 @@ define i16 @v_mad_u16(i16 %arg0, i16 %arg1, i16 %arg2) {
 ; GFX10-NEXT:    v_mad_u16 v0, v0, v1, v2
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
 ;
-; GFX11-LABEL: v_mad_u16:
-; GFX11:       ; %bb.0:
-; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT:    v_mad_u16 v0, v0, v1, v2
-; GFX11-NEXT:    s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: v_mad_u16:
+; GFX11-TRUE16:       ; %bb.0:
+; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v0.h, v1.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v1.l, v2.l
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_mad_u16 v0.l, v0.l, v0.h, v1.l
+; GFX11-TRUE16-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: v_mad_u16:
+; GFX11-FAKE16:       ; %bb.0:
+; GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT:    v_mad_u16 v0, v0, v1, v2
+; GFX11-FAKE16-NEXT:    s_setpc_b64 s[30:31]
   %mul = mul i16 %arg0, %arg1
   %add = add i16 %mul, %arg2
   ret i16 %add
@@ -151,13 +183,23 @@ define i32 @v_mad_u16_zext(i16 %arg0, i16 %arg1, i16 %arg2) {
 ; GFX10-NEXT:    v_and_b32_e32 v0, 0xffff, v0
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
 ;
-; GFX11-LABEL: v_mad_u16_zext:
-; GFX11:       ; %bb.0:
-; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT:    v_mad_u16 v0, v0, v1, v2
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT:    v_and_b32_e32 v0, 0xffff, v0
-; GFX11-NEXT:    s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: v_mad_u16_zext:
+; GFX11-TRUE16:       ; %bb.0:
+; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v0.h, v1.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v1.l, v2.l
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_mad_u16 v0.l, v0.l, v0.h, v1.l
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xffff, v0
+; GFX11-TRUE16-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: v_mad_u16_zext:
+; GFX11-FAKE16:       ; %bb.0:
+; GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT:    v_mad_u16 v0, v0, v1, v2
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v0, 0xffff, v0
+; GFX11-FAKE16-NEXT:    s_setpc_b64 s[30:31]
   %mul = mul i16 %arg0, %arg1
   %add = add i16 %mul, %arg2
   %zext = zext i16 %add to i32
@@ -187,13 +229,23 @@ define i64 @v_mad_u16_zext64(i16 %arg0, i16 %arg1, i16 %arg2) {
 ; GFX10-NEXT:    v_and_b32_e32 v0, 0xffff, v0
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
 ;
-; GFX11-LABEL: v_mad_u16_zext64:
-; GFX11:       ; %bb.0:
-; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT:    v_mad_u16 v0, v0, v1, v2
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_and_b32 v0, 0xffff, v0
-; GFX11-NEXT:    s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: v_mad_u16_zext64:
+; GFX11-TRUE16:       ; %bb.0:
+; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v0.h, v1.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v1.l, v2.l
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_mad_u16 v0.l, v0.l, v0.h, v1.l
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_and_b32 v0, 0xffff, v0
+; GFX11-TRUE16-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: v_mad_u16_zext64:
+; GFX11-FAKE16:       ; %bb.0:
+; GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT:    v_mad_u16 v0, v0, v1, v2
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_and_b32 v0, 0xffff, v0
+; GFX11-FAKE16-NEXT:    s_setpc_b64 s[30:31]
   %mul = mul i16 %arg0, %arg1
   %add = add i16 %mul, %arg2
   %zext = zext i16 %add to i64
diff --git a/llvm/test/CodeGen/AMDGPU/mad_64_32.ll b/llvm/test/CodeGen/AMDGPU/mad_64_32.ll
index 7d18739fd0c32..d4f75051b04d4 100644
--- a/llvm/test/CodeGen/AMDGPU/mad_64_32.ll
+++ b/llvm/test/CodeGen/AMDGPU/mad_64_32.ll
@@ -226,19 +226,17 @@ define i128 @mad_i64_i32_sextops_i32_i128(i32 %arg0, i32 %arg1, i128 %arg2) #0 {
 ; CI-NEXT:    v_mov_b32_e32 v8, 0
 ; CI-NEXT:    v_mad_u64_u32 v[9:10], s[4:5], v12, v1, v[7:8]
 ; CI-NEXT:    v_ashrrev_i32_e32 v13, 31, v1
-; CI-NEXT:    v_mov_b32_e32 v11, v10
-; CI-NEXT:    v_mov_b32_e32 v10, v8
-; CI-NEXT:    v_mad_u64_u32 v[7:8], s[4:5], v0, v13, v[9:10]
-; CI-NEXT:    v_add_i32_e32 v8, vcc, v11, v8
+; CI-NEXT:    v_mov_b32_e32 v7, v9
+; CI-NEXT:    v_mad_u64_u32 v[7:8], s[4:5], v0, v13, v[7:8]
+; CI-NEXT:    v_add_i32_e32 v8, vcc, v10, v8
 ; CI-NEXT:    v_mad_i64_i32 v[10:11], s[4:5], v1, v12, 0
 ; CI-NEXT:    v_addc_u32_e64 v9, s[4:5], 0, 0, vcc
 ; CI-NEXT:    v_mad_u64_u32 v[8:9], s[4:5], v12, v13, v[8:9]
 ; CI-NEXT:    v_mad_i64_i32 v[0:1], s[4:5], v13, v0, v[10:11]
 ; CI-NEXT:    v_add_i32_e32 v8, vcc, v8, v0
 ; CI-NEXT:    v_addc_u32_e32 v9, vcc, v9, v1, vcc
-; CI-NEXT:    v_mov_b32_e32 v1, v7
 ; CI-NEXT:    v_add_i32_e32 v0, vcc, v6, v2
-; CI-NEXT:    v_addc_u32_e32 v1, vcc, v1, v3, vcc
+; CI-NEXT:    v_addc_u32_e32 v1, vcc, v7, v3, vcc
 ; CI-NEXT:    v_addc_u32_e32 v2, vcc, v8, v4, vcc
 ; CI-NEXT:    v_addc_u32_e32 v3, vcc, v9, v5, vcc
 ; CI-NEXT:    s_setpc_b64 s[30:31]
@@ -280,27 +278,24 @@ define i128 @mad_i64_i32_sextops_i32_i128(i32 %arg0, i32 %arg1, i128 %arg2) #0 {
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX9-NEXT:    v_mad_u64_u32 v[6:7], s[4:5], v0, v1, 0
-; GFX9-NEXT:    v_ashrrev_i32_e32 v13, 31, v0
+; GFX9-NEXT:    v_ashrrev_i32_e32 v12, 31, v0
 ; GFX9-NEXT:    v_mov_b32_e32 v9, 0
 ; GFX9-NEXT:    v_mov_b32_e32 v8, v7
-; GFX9-NEXT:    v_mad_u64_u32 v[10:11], s[4:5], v13, v1, v[8:9]
+; GFX9-NEXT:    v_mad_u64_u32 v[10:11], s[4:5], v12, v1, v[8:9]
 ; GFX9-NEXT:    v_ashrrev_i32_e32 v14, 31, v1
-; GFX9-NEXT:    v_mov_b32_e32 v8, v11
-; GFX9-NEXT:    v_mov_b32_e32 v11, v9
-; GFX9-NEXT:    v_mad_u64_u32 v[10:11], s[4:5], v0, v14, v[10:11]
-; GFX9-NEXT:    v_mov_b32_e32 v12, v11
-; GFX9-NEXT:    v_add_co_u32_e32 v8, vcc, v8, v12
-; GFX9-NEXT:    v_addc_co_u32_e64 v9, s[4:5], 0, 0, vcc
-; GFX9-NEXT:    v_mad_u64_u32 v[8:9], s[4:5], v13, v14, v[8:9]
-; GFX9-NEXT:    v_mad_i64_i32 v[12:13], s[4:5], v1, v13, 0
+; GFX9-NEXT:    v_mov_b32_e32 v8, v10
+; GFX9-NEXT:    v_mad_u64_u32 v[8:9], s[4:5], v0, v14, v[8:9]
+; GFX9-NEXT:    v_add_co_u32_e32 v10, vcc, v11, v9
+; GFX9-NEXT:    v_addc_co_u32_e64 v11, s[4:5], 0, 0, vcc
+; GFX9-NEXT:    v_mad_u64_u32 v[10:11], s[4:5], v12, v14, v[10:11]
+; GFX9-NEXT:    v_mad_i64_i32 v[12:13], s[4:5], v1, v12, 0
 ; GFX9-NEXT:    v_mad_i64_i32 v[0:1], s[4:5], v14, v0, v[12:13]
-; GFX9-NEXT:    v_add_co_u32_e32 v7, vcc, v8, v0
-; GFX9-NEXT:    v_addc_co_u32_e32 v8, vcc, v9, v1, vcc
-; GFX9-NEXT:    v_mov_b32_e32 v1, v10
+; GFX9-NEXT:    v_add_co_u32_e32 v7, vcc, v10, v0
+; GFX9-NEXT:    v_addc_co_u32_e32 v9, vcc, v11, v1, vcc
 ; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, v6, v2
-; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, v1, v3, vcc
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, v8, v3, vcc
 ; GFX9-NEXT:    v_addc_co_u32_e32 v2, vcc, v7, v4, vcc
-; GFX9-NEXT:    v_addc_co_u32_e32 v3, vcc, v8, v5, vcc
+; GFX9-NEXT:    v_addc_co_u32_e32 v3, vcc, v9, v5, vcc
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX1100-LABEL: mad_i64_i32_sextops_i32_i128:
@@ -312,25 +307,24 @@ define i128 @mad_i64_i32_sextops_i32_i128(i32 %arg0, i32 %arg1, i128 %arg2) #0 {
 ; GFX1100-NEXT:    v_ashrrev_i32_e32 v15, 31, v1
 ; GFX1100-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX1100-NEXT:    v_mad_u64_u32 v[9:10], null, v14, v1, v[7:8]
-; GFX1100-NEXT:    v_dual_mov_b32 v11, v10 :: v_dual_mov_b32 v10, v8
-; GFX1100-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1100-NEXT:    v_mad_u64_u32 v[7:8], null, v0, v15, v[9:10]
-; GFX1100-NEXT:    v_mov_b32_e32 v10, v8
-; GFX1100-NEXT:    v_mad_i64_i32 v[8:9], null, v1, v14, 0
+; GFX1100-NEXT:    v_mov_b32_e32 v7, v9
+; GFX1100-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1100-NEXT:    v_mad_u64_u32 v[11:12], null, v0, v15, v[7:8]
+; GFX1100-NEXT:    v_mad_i64_i32 v[7:8], null, v1, v14, 0
+; GFX1100-NEXT:    v_add_co_u32 v9, s0, v10, v12
+; GFX1100-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX1100-NEXT:    v_add_co_ci_u32_e64 v10, null, 0, 0, s0
+; GFX1100-NEXT:    v_mad_i64_i32 v[12:13], null, v15, v0, v[7:8]
 ; GFX1100-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1100-NEXT:    v_add_co_u32 v10, s0, v11, v10
-; GFX1100-NEXT:    v_add_co_ci_u32_e64 v11, null, 0, 0, s0
-; GFX1100-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1100-NEXT:    v_mad_i64_i32 v[12:13], null, v15, v0, v[8:9]
-; GFX1100-NEXT:    v_mad_u64_u32 v[0:1], null, v14, v15, v[10:11]
-; GFX1100-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1100-NEXT:    v_add_co_u32 v8, vcc_lo, v0, v12
-; GFX1100-NEXT:    v_add_co_ci_u32_e32 v9, vcc_lo, v1, v13, vcc_lo
+; GFX1100-NEXT:    v_mad_u64_u32 v[0:1], null, v14, v15, v[9:10]
+; GFX1100-NEXT:    v_add_co_u32 v7, vcc_lo, v0, v12
+; GFX1100-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; GFX1100-NEXT:    v_add_co_ci_u32_e32 v8, vcc_lo, v1, v13, vcc_lo
 ; GFX1100-NEXT:    v_add_co_u32 v0, vcc_lo, v6, v2
-; GFX1100-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, v7, v3, vcc_lo
-; GFX1100-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX1100-NEXT:    v_add_co_ci_u32_e32 v2, vcc_lo, v8, v4, vcc_lo
-; GFX1100-NEXT:    v_add_co_ci_u32_e32 v3, vcc_lo, v9, v5, vcc_lo
+; GFX1100-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, v11, v3, vcc_lo
+; GFX1100-NEXT:    v_add_co_ci_u32_e32 v2, vcc_lo, v7, v4, vcc_lo
+; GFX1100-NEXT:    s_delay_alu instid0(VALU_DEP_4)
+; GFX1100-NEXT:    v_add_co_ci_u32_e32 v3, vcc_lo, v8, v5, vcc_lo
 ; GFX1100-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX1150-LABEL: mad_i64_i32_sextops_i32_i128:
@@ -338,21 +332,20 @@ define i128 @mad_i64_i32_sextops_i32_i128(i32 %arg0, i32 %arg1, i128 %arg2) #0 {
 ; GFX1150-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX1150-NEXT:    v_mad_u64_u32 v[6:7], null, v0, v1, 0
 ; GFX1150-NEXT:    v_mov_b32_e32 v8, 0
-; GFX1150-NEXT:    v_ashrrev_i32_e32 v12, 31, v0
-; GFX1150-NEXT:    v_ashrrev_i32_e32 v13, 31, v1
-; GFX1150-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1150-NEXT:    v_mad_u64_u32 v[9:10], null, v12, v1, v[7:8]
-; GFX1150-NEXT:    v_dual_mov_b32 v11, v10 :: v_dual_mov_b32 v10, v8
+; GFX1150-NEXT:    v_ashrrev_i32_e32 v13, 31, v0
+; GFX1150-NEXT:    v_ashrrev_i32_e32 v14, 31, v1
+; GFX1150-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX1150-NEXT:    v_mad_i64_i32 v[11:12], null, v1, v13, 0
+; GFX1150-NEXT:    v_mad_u64_u32 v[9:10], null, v13, v1, v[7:8]
 ; GFX1150-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1150-NEXT:    v_mad_u64_u32 v[7:8], null, v0, v13, v[9:10]
-; GFX1150-NEXT:    v_mov_b32_e32 v10, v8
-; GFX1150-NEXT:    v_mad_i64_i32 v[8:9], null, v1, v12, 0
-; GFX1150-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1150-NEXT:    v_add_co_u32 v10, s0, v11, v10
-; GFX1150-NEXT:    v_add_co_ci_u32_e64 v11, null, 0, 0, s0
-; GFX1150-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1150-NEXT:    v_mad_i64_i32 v[0:1], null, v13, v0, v[8:9]
-; GFX1150-NEXT:    v_mad_u64_u32 v[8:9], null, v12, v13, v[10:11]
+; GFX1150-NEXT:    v_mov_b32_e32 v7, v9
+; GFX1150-NEXT:    v_mad_u64_u32 v[7:8], null, v0, v14, v[7:8]
+; GFX1150-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1150-NEXT:    v_mad_i64_i32 v[0:1], null, v14, v0, v[11:12]
+; GFX1150-NEXT:    v_add_co_u32 v8, s0, v10, v8
+; GFX1150-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1150-NEXT:    v_add_co_ci_u32_e64 v9, null, 0, 0, s0
+; GFX1150-NEXT:    v_mad_u64_u32 v[8:9], null, v13, v14, v[8:9]
 ; GFX1150-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
 ; GFX1150-NEXT:    v_add_co_u32 v8, vcc_lo, v8, v0
 ; GFX1150-NEXT:    v_add_co_ci_u32_e32 v9, vcc_lo, v9, v1, vcc_lo
@@ -372,29 +365,33 @@ define i128 @mad_i64_i32_sextops_i32_i128(i32 %arg0, i32 %arg1, i128 %arg2) #0 {
 ; GFX12-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-NEXT:    v_mad_co_u64_u32 v[6:7], null, v0, v1, 0
 ; GFX12-NEXT:    v_mov_b32_e32 v8, 0
-; GFX12-NEXT:    v_ashrrev_i32_e32 v12, 31, v0
-; GFX12-NEXT:    v_ashrrev_i32_e32 v13, 31, v1
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT:    v_mad_co_u64_u32 v[9:10], null, v12, v1, v[7:8]
-; GFX12-NEXT:    v_dual_mov_b32 v11, v10 :: v_dual_mov_b32 v10, v8
+; GFX12-NEXT:    v_ashrrev_i32_e32 v13, 31, v0
+; GFX12-NEXT:    v_ashrrev_i32_e32 v14, 31, v1
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX12-NEXT:    v_mad_co_i64_i32 v[11:12], null, v1, v13, 0
+; GFX12-NEXT:    v_mad_co_u64_u32 v[9:10], null, v13, v1, v[7:8]
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT:    v_mad_co_u64_u32 v[7:8], null, v0, v13, v[9:10]
-; GFX12-NEXT:    v_mov_b32_e32 v10, v8
-; GFX12-NEXT:    v_mad_co_i64_i32 v[8:9], null, v1, v12, 0
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT:    v_add_co_u32 v10, s0, v11, v10
-; GFX12-NEXT:    v_add_co_ci_u32_e64 v11, null, 0, 0, s0
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-NEXT:    v_mad_co_i64_i32 v[0:1], null, v13, v0, v[8:9]
-; GFX12-NEXT:    v_mad_co_u64_u32 v[8:9], null, v12, v13, v[10:11]
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-NEXT:    v_mov_b32_e32 v7, v9
+; GFX12-NEXT:    v_mad_co_u64_u32 v[7:8], null, v0, v14, v[7:8]
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-NEXT:    v_mad_co_i64_i32 v[0:1], null, v14, v0, v[11:12]
+; GFX12-NEXT:    v_add_co_u32 v8, s0, v10, v8
+; GFX12-NEXT:    s_wait_alu 0xf1ff
+; GFX12-NEXT:    v_add_co_ci_u32_e64 v9, null, 0, 0, s0
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT:    v_mad_co_u64_u32 v[8:9], null, v13, v14, v[8:9]
 ; GFX12-NEXT:    v_add_co_u32 v8, vcc_lo, v8, v0
+; GFX12-NEXT:    s_wait_alu 0xfffd
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2)
 ; GFX12-NEXT:    v_add_co_ci_u32_e32 v9, vcc_lo, v9, v1, vcc_lo
 ; GFX12-NEXT:    v_add_co_u32 v0, vcc_lo, v6, v2
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, v7, v3, vcc_lo
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_add_co_ci_u32_e32 v2, vcc_lo, v8, v4, vcc_lo
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_add_co_ci_u32_e32 v3, vcc_lo, v9, v5, vcc_lo
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
   %sext0 = sext i32 %arg0 to i128
   %sext1 = sext i32 %arg1 to i128
@@ -1161,19 +1158,22 @@ define i64 @mad_i64_i32_thrice(i32 %arg0, i32 %arg1, i64 %arg2, i64 %arg3, i64 %
 ; GFX12-NEXT:    s_wait_bvhcnt 0x0
 ; GFX12-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-NEXT:    v_mad_co_i64_i32 v[0:1], null, v0, v1, 0
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX12-NEXT:    v_add_co_u32 v2, vcc_lo, v0, v2
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_add_co_ci_u32_e32 v3, vcc_lo, v1, v3, vcc_lo
 ; GFX12-NEXT:    v_add_co_u32 v4, vcc_lo, v0, v4
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_add_co_ci_u32_e32 v5, vcc_lo, v1, v5, vcc_lo
 ; GFX12-NEXT:    v_add_co_u32 v0, vcc_lo, v0, v6
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, v1, v7, vcc_lo
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
 ; GFX12-NEXT:    v_xor_b32_e32 v2, v2, v4
 ; GFX12-NEXT:    v_xor_b32_e32 v3, v3, v5
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
 ; GFX12-NEXT:    v_xor_b32_e32 v0, v2, v0
 ; GFX12-NEXT:    v_xor_b32_e32 v1, v3, v1
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
   %sext0 = sext i32 %arg0 to i64
   %sext1 = sext i32 %arg1 to i64
@@ -1249,12 +1249,14 @@ define i64 @mad_i64_i32_secondary_use(i32 %arg0, i32 %arg1, i64 %arg2) #0 {
 ; GFX12-NEXT:    s_wait_bvhcnt 0x0
 ; GFX12-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-NEXT:    v_mad_co_i64_i32 v[0:1], null, v0, v1, 0
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX12-NEXT:    v_add_co_u32 v2, vcc_lo, v0, v2
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_add_co_ci_u32_e32 v3, vcc_lo, v1, v3, vcc_lo
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
 ; GFX12-NEXT:    v_xor_b32_e32 v0, v2, v0
 ; GFX12-NEXT:    v_xor_b32_e32 v1, v3, v1
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
   %sext0 = sext i32 %arg0 to i64
   %sext1 = sext i32 %arg1 to i64
@@ -1795,11 +1797,14 @@ define i64 @lshr_mad_i64_negative_3(i64 %arg0) #0 {
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_and_b32_e32 v2, 0xfffffc00, v2
 ; GFX12-NEXT:    v_sub_co_u32 v0, vcc_lo, v0, v2
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
 ; GFX12-NEXT:    v_sub_co_ci_u32_e32 v1, vcc_lo, v1, v3, vcc_lo
 ; GFX12-NEXT:    v_add_co_u32 v0, vcc_lo, v0, 1
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2)
 ; GFX12-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
   %op = add i64 %arg0, 1
   %lsh = lshr i64 %arg0, 32
diff --git a/llvm/test/CodeGen/AMDGPU/mai-hazards-gfx940.mir b/llvm/test/CodeGen/AMDGPU/mai-hazards-gfx940.mir
index 52891989b88fb..0af37ad8c896e 100644
--- a/llvm/test/CodeGen/AMDGPU/mai-hazards-gfx940.mir
+++ b/llvm/test/CodeGen/AMDGPU/mai-hazards-gfx940.mir
@@ -417,7 +417,8 @@ body:             |
 # GCN-LABEL: name: xdl_sgemm16x16_mfma_write_agpr_mfma_srca_read_overlap
 # GCN:      V_MFMA
 # GCN-NEXT: S_NOP 7
-# GCN-NEXT: S_NOP 2
+# GFX940-NEXT: S_NOP 2
+# GFX950-NEXT: S_NOP 3
 # GCN-NEXT: V_MFMA
 name:            xdl_sgemm16x16_mfma_write_agpr_mfma_srca_read_overlap
 body:             |
@@ -439,7 +440,8 @@ body:             |
 # GCN-LABEL: name: smfmac32x32_write_agpr_mfma_srca_read_overlap
 # GCN:      V_SMFMAC
 # GCN-NEXT: S_NOP 7
-# GCN-NEXT: S_NOP 2
+# GFX940-NEXT: S_NOP 2
+# GFX950-NEXT: S_NOP 3
 # GCN-NEXT: V_MFMA
 name:            smfmac32x32_write_agpr_mfma_srca_read_overlap
 body:             |
@@ -450,7 +452,8 @@ body:             |
 # GCN-LABEL: name: smfmac32x32_write_agpr_smfmac_srcc_read_overlap
 # GCN:      V_SMFMAC
 # GCN-NEXT: S_NOP 7
-# GCN-NEXT: S_NOP 2
+# GFX940-NEXT: S_NOP 2
+# GFX950-NEXT: S_NOP 3
 # GCN-NEXT: V_SMFMAC
 name:            smfmac32x32_write_agpr_smfmac_srcc_read_overlap
 body:             |
@@ -462,7 +465,8 @@ body:             |
 # GCN:      V_MFMA
 # GCN-NEXT: S_NOP 7
 # GCN-NEXT: S_NOP 7
-# GCN-NEXT: S_NOP 2
+# GFX940-NEXT: S_NOP 2
+# GFX950-NEXT: S_NOP 3
 # GCN-NEXT: V_MFMA
 name:            xdl_sgemm32x32_mfma_write_agpr_mfma_srca_read_overlap
 body:             |
@@ -730,7 +734,8 @@ body:             |
 ...
 # GCN-LABEL: name: smfmac16x16_write_vgpr_flat_read
 # GCN:      V_SMFMAC
-# GCN-NEXT: S_NOP 6
+# GFX940-NEXT: S_NOP 6
+# GFX950-NEXT: S_NOP 7
 # GCN-NEXT: FLAT_STORE_DWORD
 name:            smfmac16x16_write_vgpr_flat_read
 body:             |
@@ -741,7 +746,8 @@ body:             |
 # GCN-LABEL: name: xdl_smfma16x16_write_vgpr_flat_read
 # GCN:      V_MFMA
 # GCN-NEXT: S_NOP 7
-# GCN-NEXT: S_NOP 2
+# GFX940-NEXT: S_NOP 2
+# GFX950-NEXT: S_NOP 3
 # GCN-NEXT: FLAT_STORE_DWORD
 name:            xdl_smfma16x16_write_vgpr_flat_read
 body:             |
@@ -752,7 +758,8 @@ body:             |
 # GCN-LABEL: name: smfmac32x32_write_vgpr_flat_read
 # GCN:      V_SMFMAC
 # GCN-NEXT: S_NOP 7
-# GCN-NEXT: S_NOP 2
+# GFX940-NEXT: S_NOP 2
+# GFX950-NEXT: S_NOP 3
 # GCN-NEXT: FLAT_STORE_DWORD
 name:            smfmac32x32_write_vgpr_flat_read
 body:             |
@@ -764,7 +771,8 @@ body:             |
 # GCN:      V_MFMA
 # GCN-NEXT: S_NOP 7
 # GCN-NEXT: S_NOP 7
-# GCN-NEXT: S_NOP 2
+# GFX940-NEXT: S_NOP 2
+# GFX950-NEXT: S_NOP 3
 # GCN-NEXT: FLAT_STORE_DWORD
 name:            xdl_smfma32x32_write_vgpr_flat_read
 body:             |
@@ -819,7 +827,8 @@ body:             |
 # GCN-LABEL: name: xdl_smfma16x16_write_vgpr_valu_read
 # GCN:      V_MFMA
 # GCN-NEXT: S_NOP 7
-# GCN-NEXT: S_NOP 2
+# GFX940-NEXT: S_NOP 2
+# GFX950-NEXT: S_NOP 3
 # GCN-NEXT: V_MOV_B32
 name:            xdl_smfma16x16_write_vgpr_valu_read
 body:             |
@@ -831,7 +840,8 @@ body:             |
 # GCN:      V_MFMA
 # GCN-NEXT: S_NOP 7
 # GCN-NEXT: S_NOP 7
-# GCN-NEXT: S_NOP 2
+# GFX940-NEXT: S_NOP 2
+# GFX950-NEXT: S_NOP 3
 # GCN-NEXT: V_MOV_B32
 name:            xdl_smfma32x32_write_vgpr_valu_read
 body:             |
@@ -877,7 +887,8 @@ body:             |
 # GCN-LABEL: name: xdl_smfma16x16_write_vgpr_accv_read
 # GCN:      V_MFMA
 # GCN-NEXT: S_NOP 7
-# GCN-NEXT: S_NOP 2
+# GFX940-NEXT: S_NOP 2
+# GFX950-NEXT: S_NOP 3
 # GCN-NEXT: V_ACCVGPR_WRITE_B32_e64
 name:            xdl_smfma16x16_write_vgpr_accv_read
 body:             |
@@ -889,7 +900,8 @@ body:             |
 # GCN:      V_MFMA
 # GCN-NEXT: S_NOP 7
 # GCN-NEXT: S_NOP 7
-# GCN-NEXT: S_NOP 2
+# GFX940-NEXT: S_NOP 2
+# GFX950-NEXT: S_NOP 3
 # GCN-NEXT: V_ACCVGPR_WRITE_B32_e64
 name:            xdl_smfma32x32_write_vgpr_accv_read
 body:             |
@@ -946,7 +958,8 @@ body:             |
 # GCN-LABEL: name: xdl_smfma16x16_write_vgpr_valu_write
 # GCN:      V_MFMA
 # GCN-NEXT: S_NOP 7
-# GCN-NEXT: S_NOP 2
+# GFX940-NEXT: S_NOP 2
+# GFX950-NEXT: S_NOP 3
 # GCN-NEXT: V_MOV_B32
 name:            xdl_smfma16x16_write_vgpr_valu_write
 body:             |
@@ -958,7 +971,8 @@ body:             |
 # GCN:      V_MFMA
 # GCN-NEXT: S_NOP 7
 # GCN-NEXT: S_NOP 7
-# GCN-NEXT: S_NOP 2
+# GFX940-NEXT: S_NOP 2
+# GFX950-NEXT: S_NOP 3
 # GCN-NEXT: V_MOV_B32
 name:            xdl_smfma32x32_write_vgpr_valu_write
 body:             |
@@ -979,7 +993,8 @@ body:             |
 # GCN-LABEL: name: xdl_smfma16x16_write_vgpr_valu_f16_write
 # GCN:      V_MFMA
 # GCN-NEXT: S_NOP 7
-# GCN-NEXT: S_NOP 2
+# GFX940-NEXT: S_NOP 2
+# GFX950-NEXT: S_NOP 3
 # GCN-NEXT: V_FMA_F16_e64
 name:            xdl_smfma16x16_write_vgpr_valu_f16_write
 body:             |
@@ -991,7 +1006,8 @@ body:             |
 # GCN:      V_MFMA
 # GCN-NEXT: S_NOP 7
 # GCN-NEXT: S_NOP 7
-# GCN-NEXT: S_NOP 2
+# GFX940-NEXT: S_NOP 2
+# GFX950-NEXT: S_NOP 3
 # GCN-NEXT: V_FMA_F16_e64
 name:            xdl_smfma32x32_write_vgpr_valu_f16_write
 body:             |
@@ -1012,7 +1028,8 @@ body:             |
 # GCN-LABEL: name: xdl_smfma16x16_write_vgpr_valu_sdwa_write
 # GCN:      V_MFMA
 # GCN-NEXT: S_NOP 7
-# GCN-NEXT: S_NOP 2
+# GFX940-NEXT: S_NOP 2
+# GFX950-NEXT: S_NOP 3
 # GCN-NEXT: V_MOV_B32_sdwa
 name:            xdl_smfma16x16_write_vgpr_valu_sdwa_write
 body:             |
@@ -1024,7 +1041,8 @@ body:             |
 # GCN:      V_MFMA
 # GCN-NEXT: S_NOP 7
 # GCN-NEXT: S_NOP 7
-# GCN-NEXT: S_NOP 2
+# GFX940-NEXT: S_NOP 2
+# GFX950-NEXT: S_NOP 3
 # GCN-NEXT: V_MOV_B32_sdwa
 name:            xdl_smfma32x32_write_vgpr_valu_sdwa_write
 body:             |
@@ -1715,7 +1733,8 @@ body:             |
 ...
 # GCN-LABEL: name: xdl_sgemm16X16X16_mfma_write_agpr_mfma_srca_read_overlap
 # GCN:      V_MFMA
-# GCN-NEXT: S_NOP 6
+# GFX940-NEXT: S_NOP 6
+# GFX950-NEXT: S_NOP 7
 # GCN-NEXT: V_MFMA
 name:            xdl_sgemm16X16X16_mfma_write_agpr_mfma_srca_read_overlap
 body:             |
@@ -1725,7 +1744,8 @@ body:             |
 ...
 # GCN-LABEL: name: xdl_sgemm16X16X32_mfma_write_agpr_mfma_srcb_read_overlap
 # GCN:      V_MFMA
-# GCN-NEXT: S_NOP 6
+# GFX940-NEXT: S_NOP 6
+# GFX950-NEXT: S_NOP 7
 # GCN-NEXT: V_MFMA
 name:            xdl_sgemm16X16X32_mfma_write_agpr_mfma_srcb_read_overlap
 body:             |
@@ -1735,7 +1755,8 @@ body:             |
 ...
 # GCN-LABEL: name: xdl_sgemm16X16X16_mfma_write_vgpr_dmfma16x16_srca_read_overlap
 # GCN:      V_MFMA
-# GCN-NEXT: S_NOP 6
+# GFX940-NEXT: S_NOP 6
+# GFX950-NEXT: S_NOP 7
 # GCN-NEXT: V_MFMA
 name:            xdl_sgemm16X16X16_mfma_write_vgpr_dmfma16x16_srca_read_overlap
 body:             |
@@ -1745,7 +1766,8 @@ body:             |
 ...
 # GCN-LABEL: name: xdl_sgemm16X16X16_mfma_write_vgpr_valu_write
 # GCN:      V_MFMA
-# GCN-NEXT: S_NOP 6
+# GFX940-NEXT: S_NOP 6
+# GFX950-NEXT: S_NOP 7
 # GCN-NEXT: V_MOV_B32
 name:            xdl_sgemm16X16X16_mfma_write_vgpr_valu_write
 body:             |
@@ -1755,7 +1777,8 @@ body:             |
 ...
 # GCN-LABEL: name: xdl_sgemm16X16X16_mfma_write_vgpr_vm_read
 # GCN:      V_MFMA
-# GCN-NEXT: S_NOP 6
+# GFX940-NEXT: S_NOP 6
+# GFX950-NEXT: S_NOP 7
 # GCN-NEXT: BUFFER_STORE_DWORD
 name:            xdl_sgemm16X16X16_mfma_write_vgpr_vm_read
 body:             |
@@ -1765,7 +1788,8 @@ body:             |
 ...
 # GCN-LABEL: name: xdl_sgemm16X16X16_mfma_write_vgpr_valu_read
 # GCN:      V_MFMA
-# GCN-NEXT: S_NOP 6
+# GFX940-NEXT: S_NOP 6
+# GFX950-NEXT: S_NOP 7
 # GCN-NEXT: V_MOV_B32
 name:            xdl_sgemm16X16X16_mfma_write_vgpr_valu_read
 body:             |
@@ -1775,7 +1799,8 @@ body:             |
 ...
 # GCN-LABEL: name: xdl_sgemm16X16X16_mfma_write_vgpr_dot_read
 # GCN:      V_MFMA
-# GCN-NEXT: S_NOP 6
+# GFX940-NEXT: S_NOP 6
+# GFX950-NEXT: S_NOP 7
 # GCN-NEXT: V_DOT
 name:            xdl_sgemm16X16X16_mfma_write_vgpr_dot_read
 body:             |
@@ -1826,7 +1851,8 @@ body:             |
 ...
 # GCN-LABEL: name: smfmac16x16x32_mfma_write_vgpr_smfmac_read_idx
 # GCN:      V_SMFMAC
-# GCN-NEXT: S_NOP 6
+# GFX940-NEXT: S_NOP 6
+# GFX950-NEXT: S_NOP 7
 # GCN-NEXT: V_SMFMAC
 name:            smfmac16x16x32_mfma_write_vgpr_smfmac_read_idx
 body:             |
@@ -2052,7 +2078,8 @@ body:             |
 ...
 # GCN-LABEL: name: smfmac16x16_read_vgpr_srcc_valu_write
 # GCN:      V_SMFMAC
-# GCN-NEXT: S_NOP 6
+# GFX940-NEXT: S_NOP 6
+# GFX950-NEXT: S_NOP 7
 # GCN-NEXT: V_MOV_B32
 name:            smfmac16x16_read_vgpr_srcc_valu_write
 body:             |
@@ -2082,7 +2109,8 @@ body:             |
 # GCN-LABEL: name: smfmac32x32_read_vgpr_srcc_valu_write
 # GCN:      V_SMFMAC
 # GCN-NEXT: S_NOP 7
-# GCN-NEXT: S_NOP 2
+# GFX940-NEXT: S_NOP 2
+# GFX950-NEXT: S_NOP 3
 # GCN-NEXT: V_MOV_B32
 name:            smfmac32x32_read_vgpr_srcc_valu_write
 body:             |
@@ -2188,7 +2216,8 @@ body:             |
 # 4 pass source
 # GCN-LABEL: name: xdl_mfma_4pass_write_vgpr_xdl_mfma_read_overlap_srca
 # GCN:      V_MFMA
-# GCN-NEXT: S_NOP 6
+# GFX940-NEXT: S_NOP 6
+# GFX950-NEXT: S_NOP 7
 # GCN-NEXT: V_MFMA
 name:            xdl_mfma_4pass_write_vgpr_xdl_mfma_read_overlap_srca
 body:             |
@@ -2202,7 +2231,8 @@ body:             |
 # 4 pass source
 # GCN-LABEL: name: xdl_mfma_4pass_write_vgpr_xdl_mfma_read_overlap_srcb
 # GCN:      V_MFMA
-# GCN-NEXT: S_NOP 6
+# GFX940-NEXT: S_NOP 6
+# GFX950-NEXT: S_NOP 7
 # GCN-NEXT: V_MFMA
 name:            xdl_mfma_4pass_write_vgpr_xdl_mfma_read_overlap_srcb
 body:             |
@@ -2276,7 +2306,8 @@ body:             |
 # 4 pass source
 # GCN-LABEL: name: xdl_mfma_4pass_write_vgpr_sgemm_mfma_read_overlap_srca
 # GCN:      V_MFMA
-# GCN-NEXT: S_NOP 6
+# GFX940-NEXT: S_NOP 6
+# GFX950-NEXT: S_NOP 7
 # GCN-NEXT: V_MFMA
 name:            xdl_mfma_4pass_write_vgpr_sgemm_mfma_read_overlap_srca
 body:             |
@@ -2290,7 +2321,8 @@ body:             |
 # 4 pass source
 # GCN-LABEL: name: xdl_mfma_4pass_write_vgpr_sgemm_mfma_read_overlap_srcb
 # GCN:      V_MFMA
-# GCN-NEXT: S_NOP 6
+# GFX940-NEXT: S_NOP 6
+# GFX950-NEXT: S_NOP 7
 # GCN-NEXT: V_MFMA
 name:            xdl_mfma_4pass_write_vgpr_sgemm_mfma_read_overlap_srcb
 body:             |
@@ -2321,7 +2353,8 @@ body:             |
 # GCN-LABEL: name: xdl_mfma_8pass_write_vgpr_nonxdl_sgemm_mfma_read_overlap_srca
 # GCN:      V_MFMA
 # GCN-NEXT: S_NOP 7
-# GCN-NEXT: S_NOP 2
+# GFX940-NEXT: S_NOP 2
+# GFX950-NEXT: S_NOP 3
 # GCN-NEXT: V_MFMA
 name: xdl_mfma_8pass_write_vgpr_nonxdl_sgemm_mfma_read_overlap_srca
 body:             |
@@ -2336,7 +2369,8 @@ body:             |
 # GCN-LABEL: name: xdl_mfma_8pass_write_vgpr_nonxdl_sgemm_mfma_read_overlap_srcb
 # GCN:      V_MFMA
 # GCN-NEXT: S_NOP 7
-# GCN-NEXT: S_NOP 2
+# GFX940-NEXT: S_NOP 2
+# GFX950-NEXT: S_NOP 3
 # GCN-NEXT: V_MFMA
 name: xdl_mfma_8pass_write_vgpr_nonxdl_sgemm_mfma_read_overlap_srcb
 body:             |
@@ -2370,7 +2404,8 @@ body:             |
 # GCN:      V_MFMA
 # GCN-NEXT: S_NOP 7
 # GCN-NEXT: S_NOP 7
-# GCN-NEXT: S_NOP 2
+# GFX940-NEXT: S_NOP 2
+# GFX950-NEXT: S_NOP 3
 # GCN-NEXT: V_MFMA
 name:            xdl_16pass_write_vgpr_nonxdl_sgemm_mfma_read_overlap_srca
 body:             |
@@ -2386,7 +2421,8 @@ body:             |
 # GCN:      V_MFMA
 # GCN-NEXT: S_NOP 7
 # GCN-NEXT: S_NOP 7
-# GCN-NEXT: S_NOP 2
+# GFX940-NEXT: S_NOP 2
+# GFX950-NEXT: S_NOP 3
 # GCN-NEXT: V_MFMA
 name:            xdl_16pass_write_vgpr_nonxdl_sgemm_mfma_read_overlap_srcb
 body:             |
@@ -2456,7 +2492,8 @@ body:             |
 # GCN-LABEL: name: xdl_mfma_8pass_write_vgpr_xdl_mfma_read_overlap_srca
 # GCN:      V_MFMA
 # GCN-NEXT: S_NOP 7
-# GCN-NEXT: S_NOP 2
+# GFX940-NEXT: S_NOP 2
+# GFX950-NEXT: S_NOP 3
 # GCN-NEXT: V_MFMA
 name:            xdl_mfma_8pass_write_vgpr_xdl_mfma_read_overlap_srca
 body:             |
@@ -2470,7 +2507,8 @@ body:             |
 # GCN-LABEL: name: xdl_mfma_8pass_write_vgpr_xdl_mfma_read_overlap_srcb
 # GCN:      V_MFMA
 # GCN-NEXT: S_NOP 7
-# GCN-NEXT: S_NOP 2
+# GFX940-NEXT: S_NOP 2
+# GFX950-NEXT: S_NOP 3
 # GCN-NEXT: V_MFMA
 name:            xdl_mfma_8pass_write_vgpr_xdl_mfma_read_overlap_srcb
 body:             |
@@ -2502,7 +2540,8 @@ body:             |
 # GCN:      V_MFMA
 # GCN-NEXT: S_NOP 7
 # GCN-NEXT: S_NOP 7
-# GCN-NEXT: S_NOP 2
+# GFX940-NEXT: S_NOP 2
+# GFX950-NEXT: S_NOP 3
 # GCN-NEXT: V_MFMA
 name:            xdl_16pass_write_vgpr_xdl_mfma_read_overlap_srca
 body:             |
@@ -2519,7 +2558,8 @@ body:             |
 # GCN:      V_MFMA
 # GCN-NEXT: S_NOP 7
 # GCN-NEXT: S_NOP 7
-# GCN-NEXT: S_NOP 2
+# GFX940-NEXT: S_NOP 2
+# GFX950-NEXT: S_NOP 3
 # GCN-NEXT: V_MFMA
 name:            xdl_16pass_write_vgpr_xdl_mfma_read_overlap_srcb
 body:             |
diff --git a/llvm/test/CodeGen/AMDGPU/mai-hazards-mfma-scale.gfx950.mir b/llvm/test/CodeGen/AMDGPU/mai-hazards-mfma-scale.gfx950.mir
index 433236180b137..4585eca8fe894 100644
--- a/llvm/test/CodeGen/AMDGPU/mai-hazards-mfma-scale.gfx950.mir
+++ b/llvm/test/CodeGen/AMDGPU/mai-hazards-mfma-scale.gfx950.mir
@@ -254,7 +254,7 @@ body:             |
     ; GCN-NEXT: {{  $}}
     ; GCN-NEXT: renamable $vgpr0_vgpr1_vgpr2_vgpr3 = nofpexcept V_MFMA_SCALE_F32_16X16X128_F8F6F4_f8_f8_vgprcd_e64 $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7, $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, $vgpr16_vgpr17_vgpr18_vgpr19, 0, 0, $sgpr4, $vgpr21, 12, 4, implicit $mode, implicit $exec
     ; GCN-NEXT: S_NOP 7
-    ; GCN-NEXT: S_NOP 2
+    ; GCN-NEXT: S_NOP 3
     ; GCN-NEXT: renamable $vgpr0_vgpr1_vgpr2_vgpr3 = nofpexcept V_MFMA_SCALE_F32_16X16X128_F8F6F4_f8_f8_vgprcd_e64 killed $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7, killed $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, killed $vgpr2_vgpr3_vgpr4_vgpr5, 0, 0, killed $sgpr4, killed $vgpr21, 12, 4, implicit $mode, implicit $exec
     ; GCN-NEXT: S_SETPC_B64_return undef $sgpr30_sgpr31, implicit $vgpr0, implicit $vgpr1, implicit $vgpr2, implicit $vgpr3
     renamable $vgpr0_vgpr1_vgpr2_vgpr3 = nofpexcept V_MFMA_SCALE_F32_16X16X128_F8F6F4_f8_f8_vgprcd_e64 $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7, $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, $vgpr16_vgpr17_vgpr18_vgpr19, 0, 0, $sgpr4, $vgpr21, 12, 4, implicit $mode, implicit $exec
@@ -275,7 +275,7 @@ body:             |
     ; GCN: liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6, $vgpr7, $vgpr8, $vgpr9, $vgpr10, $vgpr11, $vgpr12, $vgpr13, $vgpr14, $vgpr15, $vgpr16, $vgpr17, $vgpr18, $vgpr19, $vgpr20, $vgpr21, $vgpr22, $vgpr23, $vgpr24, $vgpr25, $vgpr26, $vgpr27, $vgpr28, $vgpr29, $vgpr30, $vgpr31, $vgpr32, $sgpr4
     ; GCN-NEXT: {{  $}}
     ; GCN-NEXT: renamable $vgpr0_vgpr1_vgpr2_vgpr3 = nofpexcept V_MFMA_SCALE_F32_16X16X128_F8F6F4_f8_f8_vgprcd_e64 $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7, $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, $vgpr16_vgpr17_vgpr18_vgpr19, 2, 2, $sgpr4, $vgpr21, 12, 4, implicit $mode, implicit $exec
-    ; GCN-NEXT: S_NOP 6
+    ; GCN-NEXT: S_NOP 7
     ; GCN-NEXT: renamable $vgpr0_vgpr1_vgpr2_vgpr3 = nofpexcept V_MFMA_SCALE_F32_16X16X128_F8F6F4_f8_f8_vgprcd_e64 killed $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7, killed $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, killed $vgpr2_vgpr3_vgpr4_vgpr5, 0, 0, killed $sgpr4, killed $vgpr21, 12, 4, implicit $mode, implicit $exec
     ; GCN-NEXT: S_SETPC_B64_return undef $sgpr30_sgpr31, implicit $vgpr0, implicit $vgpr1, implicit $vgpr2, implicit $vgpr3
     renamable $vgpr0_vgpr1_vgpr2_vgpr3 = nofpexcept V_MFMA_SCALE_F32_16X16X128_F8F6F4_f8_f8_vgprcd_e64 $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7, $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, $vgpr16_vgpr17_vgpr18_vgpr19, 2, 2, $sgpr4, $vgpr21, 12, 4, implicit $mode, implicit $exec
diff --git a/llvm/test/CodeGen/AMDGPU/materialize-frame-index-sgpr.gfx10.ll b/llvm/test/CodeGen/AMDGPU/materialize-frame-index-sgpr.gfx10.ll
index 8e436b327cda1..5e74a380e0748 100644
--- a/llvm/test/CodeGen/AMDGPU/materialize-frame-index-sgpr.gfx10.ll
+++ b/llvm/test/CodeGen/AMDGPU/materialize-frame-index-sgpr.gfx10.ll
@@ -130,7 +130,6 @@ define void @scalar_mov_materializes_frame_index_unavailable_scc() #0 {
 ; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_mov_b32 exec_lo, s0
 ; GFX12-NEXT:    s_wait_loadcnt 0x0
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX8-LABEL: scalar_mov_materializes_frame_index_unavailable_scc:
@@ -337,7 +336,6 @@ define void @scalar_mov_materializes_frame_index_dead_scc() #0 {
 ; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_mov_b32 exec_lo, s0
 ; GFX12-NEXT:    s_wait_loadcnt 0x0
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX8-LABEL: scalar_mov_materializes_frame_index_dead_scc:
@@ -773,7 +771,6 @@ define void @scalar_mov_materializes_frame_index_unavailable_scc_small_offset()
 ; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_mov_b32 exec_lo, s0
 ; GFX12-NEXT:    s_wait_loadcnt 0x0
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX8-LABEL: scalar_mov_materializes_frame_index_unavailable_scc_small_offset:
@@ -941,7 +938,6 @@ define void @scalar_mov_materializes_frame_index_available_scc_small_offset() #0
 ; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_mov_b32 exec_lo, s0
 ; GFX12-NEXT:    s_wait_loadcnt 0x0
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX8-LABEL: scalar_mov_materializes_frame_index_available_scc_small_offset:
@@ -1524,7 +1520,6 @@ define void @scalar_mov_materializes_frame_index_unavailable_scc__gep_immoffset(
 ; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_mov_b32 exec_lo, s0
 ; GFX12-NEXT:    s_wait_loadcnt 0x0
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX8-LABEL: scalar_mov_materializes_frame_index_unavailable_scc__gep_immoffset:
@@ -1734,7 +1729,6 @@ define void @scalar_mov_materializes_frame_index_unavailable_scc__gep_sgpr_offse
 ; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_mov_b32 exec_lo, s0
 ; GFX12-NEXT:    s_wait_loadcnt 0x0
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX8-LABEL: scalar_mov_materializes_frame_index_unavailable_scc__gep_sgpr_offset:
diff --git a/llvm/test/CodeGen/AMDGPU/materialize-frame-index-sgpr.ll b/llvm/test/CodeGen/AMDGPU/materialize-frame-index-sgpr.ll
index 7646197f13175..a2f7d24f9ec87 100644
--- a/llvm/test/CodeGen/AMDGPU/materialize-frame-index-sgpr.ll
+++ b/llvm/test/CodeGen/AMDGPU/materialize-frame-index-sgpr.ll
@@ -759,7 +759,6 @@ define void @scalar_mov_materializes_frame_index_no_live_scc_no_live_sgprs() #0
 ; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_mov_b32 exec_lo, s0
 ; GFX12-NEXT:    s_wait_loadcnt 0x0
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
   %alloca0 = alloca [4096 x i32], align 64, addrspace(5)
   %alloca1 = alloca i32, align 4, addrspace(5)
@@ -1474,7 +1473,6 @@ define void @scalar_mov_materializes_frame_index_no_live_scc_no_live_sgprs__lowe
 ; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_mov_b32 exec_lo, s0
 ; GFX12-NEXT:    s_wait_loadcnt 0x0
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
   %alloca0 = alloca [4096 x i32], align 16, addrspace(5)
 
@@ -2204,7 +2202,6 @@ define void @scalar_mov_materializes_frame_index_no_live_scc_no_live_sgprs_gep_i
 ; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_mov_b32 exec_lo, s0
 ; GFX12-NEXT:    s_wait_loadcnt 0x0
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
   %alloca0 = alloca [4096 x i32], align 64, addrspace(5)
   %alloca1 = alloca [4096 x i32], align 4, addrspace(5)
diff --git a/llvm/test/CodeGen/AMDGPU/maximumnum.ll b/llvm/test/CodeGen/AMDGPU/maximumnum.ll
index fa15a42aef2ac..64afe3cd01255 100644
--- a/llvm/test/CodeGen/AMDGPU/maximumnum.ll
+++ b/llvm/test/CodeGen/AMDGPU/maximumnum.ll
@@ -278,17 +278,20 @@ define bfloat @v_maximumnum_bf16(bfloat %x, bfloat %y) {
 ; GFX12-NEXT:    s_wait_bvhcnt 0x0
 ; GFX12-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-NEXT:    v_lshlrev_b32_e32 v2, 16, v0
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)
 ; GFX12-NEXT:    v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc_lo
 ; GFX12-NEXT:    v_lshlrev_b32_e32 v3, 16, v1
 ; GFX12-NEXT:    v_lshlrev_b32_e32 v2, 16, v0
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_cndmask_b32_e32 v1, v1, v0, vcc_lo
 ; GFX12-NEXT:    v_lshlrev_b32_e32 v3, 16, v1
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v2, v3
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_cndmask_b32_e32 v2, v1, v0, vcc_lo
 ; GFX12-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
@@ -296,19 +299,22 @@ define bfloat @v_maximumnum_bf16(bfloat %x, bfloat %y) {
 ; GFX12-NEXT:    v_bfe_u32 v3, v2, 16, 1
 ; GFX12-NEXT:    v_or_b32_e32 v4, 0x400000, v2
 ; GFX12-NEXT:    v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_add3_u32 v3, v3, v2, 0x7fff
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_cndmask_b32_e32 v2, v3, v4, vcc_lo
 ; GFX12-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v0
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_2)
 ; GFX12-NEXT:    v_lshrrev_b32_e32 v3, 16, v2
 ; GFX12-NEXT:    v_and_b32_e32 v2, 0xffff0000, v2
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc_lo
 ; GFX12-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v1
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX12-NEXT:    s_wait_alu 0xfffd
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_2)
 ; GFX12-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc_lo
 ; GFX12-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v2
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2)
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc_lo
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
   %result = call bfloat @llvm.maximumnum.bf16(bfloat %x, bfloat %y)
@@ -391,15 +397,19 @@ define bfloat @v_maximumnum_bf16_nnan(bfloat %x, bfloat %y) {
 ; GFX12-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-NEXT:    v_lshlrev_b32_e32 v2, 16, v1
 ; GFX12-NEXT:    v_lshlrev_b32_e32 v3, 16, v0
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_4) | instid1(VALU_DEP_2)
 ; GFX12-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v3, v2
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_cndmask_b32_e32 v2, v1, v0, vcc_lo
 ; GFX12-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v0
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_dual_cndmask_b32 v0, v2, v0 :: v_dual_lshlrev_b32 v3, 16, v2
 ; GFX12-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v1
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
 ; GFX12-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc_lo
 ; GFX12-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v3
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2)
 ; GFX12-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc_lo
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
diff --git a/llvm/test/CodeGen/AMDGPU/memmove-var-size.ll b/llvm/test/CodeGen/AMDGPU/memmove-var-size.ll
index a68d2e575607d..bc8bcc622810f 100644
--- a/llvm/test/CodeGen/AMDGPU/memmove-var-size.ll
+++ b/llvm/test/CodeGen/AMDGPU/memmove-var-size.ll
@@ -306,10 +306,10 @@ define void @memmove_p0_p3(ptr addrspace(0) align 1 %dst, ptr addrspace(3) align
 ; CHECK-LABEL: memmove_p0_p3:
 ; CHECK:       ; %bb.0: ; %entry
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    v_and_b32_e32 v7, 7, v3
+; CHECK-NEXT:    v_and_b32_e32 v7, 15, v3
 ; CHECK-NEXT:    v_mov_b32_e32 v8, 0
 ; CHECK-NEXT:    v_cmp_ne_u64_e32 vcc_lo, 0, v[0:1]
-; CHECK-NEXT:    v_and_b32_e32 v5, -8, v3
+; CHECK-NEXT:    v_and_b32_e32 v5, -16, v3
 ; CHECK-NEXT:    v_mov_b32_e32 v6, v4
 ; CHECK-NEXT:    s_mov_b32 s6, exec_lo
 ; CHECK-NEXT:    v_cmp_ne_u64_e64 s4, 0, v[7:8]
@@ -338,15 +338,15 @@ define void @memmove_p0_p3(ptr addrspace(0) align 1 %dst, ptr addrspace(3) align
 ; CHECK-NEXT:    .p2align 6
 ; CHECK-NEXT:  .LBB2_5: ; %memmove_fwd_main_loop
 ; CHECK-NEXT:    ; =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    ds_read_b64 v[13:14], v4
-; CHECK-NEXT:    v_add_co_u32 v11, s5, v11, -8
+; CHECK-NEXT:    ds_read_b128 v[13:16], v4
+; CHECK-NEXT:    v_add_co_u32 v11, s5, v11, -16
 ; CHECK-NEXT:    v_add_co_ci_u32_e64 v12, s5, -1, v12, s5
-; CHECK-NEXT:    v_add_nc_u32_e32 v4, 8, v4
+; CHECK-NEXT:    v_add_nc_u32_e32 v4, 16, v4
 ; CHECK-NEXT:    v_cmp_eq_u64_e64 s5, 0, v[11:12]
 ; CHECK-NEXT:    s_or_b32 s9, s5, s9
 ; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
-; CHECK-NEXT:    flat_store_dwordx2 v[9:10], v[13:14]
-; CHECK-NEXT:    v_add_co_u32 v9, s6, v9, 8
+; CHECK-NEXT:    flat_store_dwordx4 v[9:10], v[13:16]
+; CHECK-NEXT:    v_add_co_u32 v9, s6, v9, 16
 ; CHECK-NEXT:    v_add_co_ci_u32_e64 v10, s6, 0, v10, s6
 ; CHECK-NEXT:    s_andn2_b32 exec_lo, exec_lo, s9
 ; CHECK-NEXT:    s_cbranch_execnz .LBB2_5
@@ -355,7 +355,7 @@ define void @memmove_p0_p3(ptr addrspace(0) align 1 %dst, ptr addrspace(3) align
 ; CHECK-NEXT:    s_and_saveexec_b32 s8, s4
 ; CHECK-NEXT:    s_cbranch_execz .LBB2_9
 ; CHECK-NEXT:  ; %bb.7: ; %memmove_fwd_residual_loop.preheader
-; CHECK-NEXT:    v_and_b32_e32 v3, -8, v3
+; CHECK-NEXT:    v_and_b32_e32 v3, -16, v3
 ; CHECK-NEXT:    v_add_co_u32 v0, s5, v0, v5
 ; CHECK-NEXT:    v_add_co_ci_u32_e64 v1, s5, v1, v6, s5
 ; CHECK-NEXT:    v_add_nc_u32_e32 v2, v2, v3
@@ -414,26 +414,26 @@ define void @memmove_p0_p3(ptr addrspace(0) align 1 %dst, ptr addrspace(3) align
 ; CHECK-NEXT:    s_and_saveexec_b32 s5, vcc_lo
 ; CHECK-NEXT:    s_cbranch_execz .LBB2_16
 ; CHECK-NEXT:  ; %bb.14: ; %memmove_bwd_main_loop.preheader
-; CHECK-NEXT:    v_and_b32_e32 v3, -8, v3
-; CHECK-NEXT:    v_add_co_u32 v0, vcc_lo, v0, -8
+; CHECK-NEXT:    v_and_b32_e32 v3, -16, v3
+; CHECK-NEXT:    v_add_co_u32 v0, vcc_lo, v0, -16
 ; CHECK-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, -1, v1, vcc_lo
-; CHECK-NEXT:    v_add3_u32 v2, v3, v2, -8
+; CHECK-NEXT:    v_add3_u32 v2, v3, v2, -16
 ; CHECK-NEXT:    s_mov_b32 s7, 0
 ; CHECK-NEXT:    .p2align 6
 ; CHECK-NEXT:  .LBB2_15: ; %memmove_bwd_main_loop
 ; CHECK-NEXT:    ; =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    ds_read_b64 v[3:4], v2
-; CHECK-NEXT:    v_add_co_u32 v7, vcc_lo, v5, -8
-; CHECK-NEXT:    v_add_co_ci_u32_e32 v8, vcc_lo, -1, v6, vcc_lo
-; CHECK-NEXT:    v_add_co_u32 v9, vcc_lo, v0, v5
-; CHECK-NEXT:    v_add_co_ci_u32_e32 v10, vcc_lo, v1, v6, vcc_lo
-; CHECK-NEXT:    v_cmp_eq_u64_e64 s4, 0, v[7:8]
-; CHECK-NEXT:    v_mov_b32_e32 v5, v7
-; CHECK-NEXT:    v_add_nc_u32_e32 v2, -8, v2
-; CHECK-NEXT:    v_mov_b32_e32 v6, v8
+; CHECK-NEXT:    ds_read_b128 v[7:10], v2
+; CHECK-NEXT:    v_add_co_u32 v3, vcc_lo, v5, -16
+; CHECK-NEXT:    v_add_co_ci_u32_e32 v4, vcc_lo, -1, v6, vcc_lo
+; CHECK-NEXT:    v_add_co_u32 v11, vcc_lo, v0, v5
+; CHECK-NEXT:    v_add_co_ci_u32_e32 v12, vcc_lo, v1, v6, vcc_lo
+; CHECK-NEXT:    v_cmp_eq_u64_e64 s4, 0, v[3:4]
+; CHECK-NEXT:    v_mov_b32_e32 v6, v4
+; CHECK-NEXT:    v_add_nc_u32_e32 v2, -16, v2
+; CHECK-NEXT:    v_mov_b32_e32 v5, v3
 ; CHECK-NEXT:    s_or_b32 s7, s4, s7
 ; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
-; CHECK-NEXT:    flat_store_dwordx2 v[9:10], v[3:4]
+; CHECK-NEXT:    flat_store_dwordx4 v[11:12], v[7:10]
 ; CHECK-NEXT:    s_andn2_b32 exec_lo, exec_lo, s7
 ; CHECK-NEXT:    s_cbranch_execnz .LBB2_15
 ; CHECK-NEXT:  .LBB2_16: ; %Flow36
@@ -1043,9 +1043,9 @@ define void @memmove_p1_p3(ptr addrspace(1) align 1 %dst, ptr addrspace(3) align
 ; CHECK-LABEL: memmove_p1_p3:
 ; CHECK:       ; %bb.0: ; %entry
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    v_and_b32_e32 v7, -8, v3
+; CHECK-NEXT:    v_and_b32_e32 v7, -16, v3
 ; CHECK-NEXT:    v_mov_b32_e32 v8, v4
-; CHECK-NEXT:    v_and_b32_e32 v5, 7, v3
+; CHECK-NEXT:    v_and_b32_e32 v5, 15, v3
 ; CHECK-NEXT:    v_mov_b32_e32 v6, 0
 ; CHECK-NEXT:    s_mov_b64 s[4:5], 0
 ; CHECK-NEXT:    s_mov_b32 s6, exec_lo
@@ -1056,16 +1056,16 @@ define void @memmove_p1_p3(ptr addrspace(1) align 1 %dst, ptr addrspace(3) align
 ; CHECK-NEXT:    s_mov_b32 s7, 0
 ; CHECK-NEXT:  .LBB7_2: ; %loop-memcpy-expansion
 ; CHECK-NEXT:    ; =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    ds_read_b64 v[10:11], v9
-; CHECK-NEXT:    v_add_co_u32 v12, vcc_lo, v0, s4
-; CHECK-NEXT:    s_add_u32 s4, s4, 8
-; CHECK-NEXT:    v_add_co_ci_u32_e32 v13, vcc_lo, s5, v1, vcc_lo
+; CHECK-NEXT:    ds_read_b128 v[10:13], v9
+; CHECK-NEXT:    v_add_co_u32 v14, vcc_lo, v0, s4
+; CHECK-NEXT:    s_add_u32 s4, s4, 16
+; CHECK-NEXT:    v_add_co_ci_u32_e32 v15, vcc_lo, s5, v1, vcc_lo
 ; CHECK-NEXT:    s_addc_u32 s5, s5, 0
-; CHECK-NEXT:    v_add_nc_u32_e32 v9, 8, v9
+; CHECK-NEXT:    v_add_nc_u32_e32 v9, 16, v9
 ; CHECK-NEXT:    v_cmp_ge_u64_e32 vcc_lo, s[4:5], v[7:8]
 ; CHECK-NEXT:    s_or_b32 s7, vcc_lo, s7
 ; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
-; CHECK-NEXT:    global_store_dwordx2 v[12:13], v[10:11], off
+; CHECK-NEXT:    global_store_dwordx4 v[14:15], v[10:13], off
 ; CHECK-NEXT:    s_andn2_b32 exec_lo, exec_lo, s7
 ; CHECK-NEXT:    s_cbranch_execnz .LBB7_2
 ; CHECK-NEXT:  .LBB7_3: ; %Flow9
@@ -1076,7 +1076,7 @@ define void @memmove_p1_p3(ptr addrspace(1) align 1 %dst, ptr addrspace(3) align
 ; CHECK-NEXT:    s_xor_b32 s6, exec_lo, s6
 ; CHECK-NEXT:    s_cbranch_execz .LBB7_7
 ; CHECK-NEXT:  ; %bb.4: ; %loop-memcpy-residual.preheader
-; CHECK-NEXT:    v_and_b32_e32 v3, -8, v3
+; CHECK-NEXT:    v_and_b32_e32 v3, -16, v3
 ; CHECK-NEXT:    s_mov_b32 s7, 0
 ; CHECK-NEXT:    v_add_co_u32 v0, vcc_lo, v0, v3
 ; CHECK-NEXT:    v_add_nc_u32_e32 v2, v2, v3
@@ -1327,11 +1327,11 @@ define void @memmove_p3_p0(ptr addrspace(3) align 1 %dst, ptr addrspace(0) align
 ; CHECK-LABEL: memmove_p3_p0:
 ; CHECK:       ; %bb.0: ; %entry
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    v_and_b32_e32 v5, 7, v3
+; CHECK-NEXT:    v_and_b32_e32 v5, 15, v3
 ; CHECK-NEXT:    v_mov_b32_e32 v6, 0
 ; CHECK-NEXT:    v_cmp_ne_u32_e32 vcc_lo, -1, v0
 ; CHECK-NEXT:    s_mov_b64 s[4:5], src_shared_base
-; CHECK-NEXT:    v_and_b32_e32 v7, -8, v3
+; CHECK-NEXT:    v_and_b32_e32 v7, -16, v3
 ; CHECK-NEXT:    v_mov_b32_e32 v8, v4
 ; CHECK-NEXT:    v_cmp_ne_u64_e64 s4, 0, v[5:6]
 ; CHECK-NEXT:    v_cndmask_b32_e64 v10, 0, s5, vcc_lo
@@ -1361,16 +1361,16 @@ define void @memmove_p3_p0(ptr addrspace(3) align 1 %dst, ptr addrspace(0) align
 ; CHECK-NEXT:    .p2align 6
 ; CHECK-NEXT:  .LBB10_5: ; %memmove_fwd_main_loop
 ; CHECK-NEXT:    ; =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    flat_load_dwordx2 v[13:14], v[9:10]
-; CHECK-NEXT:    v_add_co_u32 v11, s5, v11, -8
+; CHECK-NEXT:    flat_load_dwordx4 v[13:16], v[9:10]
+; CHECK-NEXT:    v_add_co_u32 v11, s5, v11, -16
 ; CHECK-NEXT:    v_add_co_ci_u32_e64 v12, s5, -1, v12, s5
-; CHECK-NEXT:    v_add_co_u32 v9, s5, v9, 8
+; CHECK-NEXT:    v_add_co_u32 v9, s5, v9, 16
 ; CHECK-NEXT:    v_add_co_ci_u32_e64 v10, s5, 0, v10, s5
 ; CHECK-NEXT:    v_cmp_eq_u64_e64 s6, 0, v[11:12]
 ; CHECK-NEXT:    s_or_b32 s9, s6, s9
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    ds_write_b64 v4, v[13:14]
-; CHECK-NEXT:    v_add_nc_u32_e32 v4, 8, v4
+; CHECK-NEXT:    ds_write_b128 v4, v[13:16]
+; CHECK-NEXT:    v_add_nc_u32_e32 v4, 16, v4
 ; CHECK-NEXT:    s_andn2_b32 exec_lo, exec_lo, s9
 ; CHECK-NEXT:    s_cbranch_execnz .LBB10_5
 ; CHECK-NEXT:  .LBB10_6: ; %Flow34
@@ -1378,7 +1378,7 @@ define void @memmove_p3_p0(ptr addrspace(3) align 1 %dst, ptr addrspace(0) align
 ; CHECK-NEXT:    s_and_saveexec_b32 s8, s4
 ; CHECK-NEXT:    s_cbranch_execz .LBB10_9
 ; CHECK-NEXT:  ; %bb.7: ; %memmove_fwd_residual_loop.preheader
-; CHECK-NEXT:    v_and_b32_e32 v3, -8, v3
+; CHECK-NEXT:    v_and_b32_e32 v3, -16, v3
 ; CHECK-NEXT:    s_mov_b32 s9, 0
 ; CHECK-NEXT:    v_add_nc_u32_e32 v3, v0, v3
 ; CHECK-NEXT:    v_add_co_u32 v0, s5, v1, v7
@@ -1437,23 +1437,23 @@ define void @memmove_p3_p0(ptr addrspace(3) align 1 %dst, ptr addrspace(0) align
 ; CHECK-NEXT:    s_and_saveexec_b32 s4, vcc_lo
 ; CHECK-NEXT:    s_cbranch_execz .LBB10_16
 ; CHECK-NEXT:  ; %bb.14: ; %memmove_bwd_main_loop.preheader
-; CHECK-NEXT:    v_and_b32_e32 v3, -8, v3
-; CHECK-NEXT:    v_add_co_u32 v1, vcc_lo, v1, -8
+; CHECK-NEXT:    v_and_b32_e32 v3, -16, v3
+; CHECK-NEXT:    v_add_co_u32 v1, vcc_lo, v1, -16
 ; CHECK-NEXT:    v_add_co_ci_u32_e32 v2, vcc_lo, -1, v2, vcc_lo
-; CHECK-NEXT:    v_add3_u32 v0, v3, v0, -8
+; CHECK-NEXT:    v_add3_u32 v0, v3, v0, -16
 ; CHECK-NEXT:    s_mov_b32 s5, 0
 ; CHECK-NEXT:  .LBB10_15: ; %memmove_bwd_main_loop
 ; CHECK-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    v_add_co_u32 v3, vcc_lo, v1, v7
 ; CHECK-NEXT:    v_add_co_ci_u32_e32 v4, vcc_lo, v2, v8, vcc_lo
-; CHECK-NEXT:    v_add_co_u32 v7, vcc_lo, v7, -8
+; CHECK-NEXT:    v_add_co_u32 v7, vcc_lo, v7, -16
 ; CHECK-NEXT:    v_add_co_ci_u32_e32 v8, vcc_lo, -1, v8, vcc_lo
-; CHECK-NEXT:    flat_load_dwordx2 v[3:4], v[3:4]
+; CHECK-NEXT:    flat_load_dwordx4 v[3:6], v[3:4]
 ; CHECK-NEXT:    v_cmp_eq_u64_e32 vcc_lo, 0, v[7:8]
 ; CHECK-NEXT:    s_or_b32 s5, vcc_lo, s5
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    ds_write_b64 v0, v[3:4]
-; CHECK-NEXT:    v_add_nc_u32_e32 v0, -8, v0
+; CHECK-NEXT:    ds_write_b128 v0, v[3:6]
+; CHECK-NEXT:    v_add_nc_u32_e32 v0, -16, v0
 ; CHECK-NEXT:    s_andn2_b32 exec_lo, exec_lo, s5
 ; CHECK-NEXT:    s_cbranch_execnz .LBB10_15
 ; CHECK-NEXT:  .LBB10_16: ; %Flow36
@@ -1470,9 +1470,9 @@ define void @memmove_p3_p1(ptr addrspace(3) align 1 %dst, ptr addrspace(1) align
 ; CHECK-LABEL: memmove_p3_p1:
 ; CHECK:       ; %bb.0: ; %entry
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    v_and_b32_e32 v7, -8, v3
+; CHECK-NEXT:    v_and_b32_e32 v7, -16, v3
 ; CHECK-NEXT:    v_mov_b32_e32 v8, v4
-; CHECK-NEXT:    v_and_b32_e32 v5, 7, v3
+; CHECK-NEXT:    v_and_b32_e32 v5, 15, v3
 ; CHECK-NEXT:    v_mov_b32_e32 v6, 0
 ; CHECK-NEXT:    s_mov_b64 s[4:5], 0
 ; CHECK-NEXT:    s_mov_b32 s6, exec_lo
@@ -1485,14 +1485,14 @@ define void @memmove_p3_p1(ptr addrspace(3) align 1 %dst, ptr addrspace(1) align
 ; CHECK-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    v_add_co_u32 v10, vcc_lo, v1, s4
 ; CHECK-NEXT:    v_add_co_ci_u32_e32 v11, vcc_lo, s5, v2, vcc_lo
-; CHECK-NEXT:    s_add_u32 s4, s4, 8
+; CHECK-NEXT:    s_add_u32 s4, s4, 16
 ; CHECK-NEXT:    s_addc_u32 s5, s5, 0
 ; CHECK-NEXT:    v_cmp_ge_u64_e32 vcc_lo, s[4:5], v[7:8]
-; CHECK-NEXT:    global_load_dwordx2 v[10:11], v[10:11], off
+; CHECK-NEXT:    global_load_dwordx4 v[10:13], v[10:11], off
 ; CHECK-NEXT:    s_or_b32 s7, vcc_lo, s7
 ; CHECK-NEXT:    s_waitcnt vmcnt(0)
-; CHECK-NEXT:    ds_write_b64 v9, v[10:11]
-; CHECK-NEXT:    v_add_nc_u32_e32 v9, 8, v9
+; CHECK-NEXT:    ds_write_b128 v9, v[10:13]
+; CHECK-NEXT:    v_add_nc_u32_e32 v9, 16, v9
 ; CHECK-NEXT:    s_andn2_b32 exec_lo, exec_lo, s7
 ; CHECK-NEXT:    s_cbranch_execnz .LBB11_2
 ; CHECK-NEXT:  .LBB11_3: ; %Flow9
@@ -1503,7 +1503,7 @@ define void @memmove_p3_p1(ptr addrspace(3) align 1 %dst, ptr addrspace(1) align
 ; CHECK-NEXT:    s_xor_b32 s6, exec_lo, s6
 ; CHECK-NEXT:    s_cbranch_execz .LBB11_7
 ; CHECK-NEXT:  ; %bb.4: ; %loop-memcpy-residual.preheader
-; CHECK-NEXT:    v_and_b32_e32 v3, -8, v3
+; CHECK-NEXT:    v_and_b32_e32 v3, -16, v3
 ; CHECK-NEXT:    s_mov_b32 s7, 0
 ; CHECK-NEXT:    v_add_co_u32 v1, vcc_lo, v1, v3
 ; CHECK-NEXT:    v_add_nc_u32_e32 v0, v0, v3
@@ -1538,8 +1538,8 @@ define void @memmove_p3_p3(ptr addrspace(3) align 1 %dst, ptr addrspace(3) align
 ; CHECK:       ; %bb.0: ; %entry
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; CHECK-NEXT:    v_mov_b32_e32 v5, 0
-; CHECK-NEXT:    v_and_b32_e32 v4, 7, v2
-; CHECK-NEXT:    v_and_b32_e32 v6, -8, v2
+; CHECK-NEXT:    v_and_b32_e32 v4, 15, v2
+; CHECK-NEXT:    v_and_b32_e32 v6, -16, v2
 ; CHECK-NEXT:    v_mov_b32_e32 v7, v3
 ; CHECK-NEXT:    s_mov_b32 s6, exec_lo
 ; CHECK-NEXT:    v_cmp_ne_u64_e64 s4, 0, v[4:5]
@@ -1563,15 +1563,15 @@ define void @memmove_p3_p3(ptr addrspace(3) align 1 %dst, ptr addrspace(3) align
 ; CHECK-NEXT:    s_mov_b32 s8, 0
 ; CHECK-NEXT:  .LBB12_5: ; %memmove_fwd_main_loop
 ; CHECK-NEXT:    ; =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    ds_read_b64 v[9:10], v3
-; CHECK-NEXT:    v_add_co_u32 v6, s5, v6, -8
+; CHECK-NEXT:    ds_read_b128 v[9:12], v3
+; CHECK-NEXT:    v_add_co_u32 v6, s5, v6, -16
 ; CHECK-NEXT:    v_add_co_ci_u32_e64 v7, s5, -1, v7, s5
-; CHECK-NEXT:    v_add_nc_u32_e32 v3, 8, v3
+; CHECK-NEXT:    v_add_nc_u32_e32 v3, 16, v3
 ; CHECK-NEXT:    v_cmp_eq_u64_e64 s5, 0, v[6:7]
 ; CHECK-NEXT:    s_or_b32 s8, s5, s8
 ; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
-; CHECK-NEXT:    ds_write_b64 v8, v[9:10]
-; CHECK-NEXT:    v_add_nc_u32_e32 v8, 8, v8
+; CHECK-NEXT:    ds_write_b128 v8, v[9:12]
+; CHECK-NEXT:    v_add_nc_u32_e32 v8, 16, v8
 ; CHECK-NEXT:    s_andn2_b32 exec_lo, exec_lo, s8
 ; CHECK-NEXT:    s_cbranch_execnz .LBB12_5
 ; CHECK-NEXT:  .LBB12_6: ; %Flow41
@@ -1579,7 +1579,7 @@ define void @memmove_p3_p3(ptr addrspace(3) align 1 %dst, ptr addrspace(3) align
 ; CHECK-NEXT:    s_and_saveexec_b32 s7, s4
 ; CHECK-NEXT:    s_cbranch_execz .LBB12_9
 ; CHECK-NEXT:  ; %bb.7: ; %memmove_fwd_residual_loop.preheader
-; CHECK-NEXT:    v_and_b32_e32 v2, -8, v2
+; CHECK-NEXT:    v_and_b32_e32 v2, -16, v2
 ; CHECK-NEXT:    s_mov_b32 s8, 0
 ; CHECK-NEXT:    v_add_nc_u32_e32 v0, v0, v2
 ; CHECK-NEXT:    v_add_nc_u32_e32 v1, v1, v2
@@ -1630,24 +1630,24 @@ define void @memmove_p3_p3(ptr addrspace(3) align 1 %dst, ptr addrspace(3) align
 ; CHECK-NEXT:    s_and_saveexec_b32 s4, vcc_lo
 ; CHECK-NEXT:    s_cbranch_execz .LBB12_16
 ; CHECK-NEXT:  ; %bb.14: ; %memmove_bwd_main_loop.preheader
-; CHECK-NEXT:    v_and_b32_e32 v5, -8, v2
+; CHECK-NEXT:    v_and_b32_e32 v5, -16, v2
 ; CHECK-NEXT:    s_mov_b32 s6, 0
-; CHECK-NEXT:    v_add_nc_u32_e32 v4, -8, v5
+; CHECK-NEXT:    v_add_nc_u32_e32 v4, -16, v5
 ; CHECK-NEXT:    v_add_nc_u32_e32 v2, v0, v4
 ; CHECK-NEXT:    v_sub_co_u32 v0, vcc_lo, 0, v5
 ; CHECK-NEXT:    v_add_nc_u32_e32 v4, v1, v4
 ; CHECK-NEXT:    v_sub_co_ci_u32_e32 v1, vcc_lo, 0, v3, vcc_lo
 ; CHECK-NEXT:  .LBB12_15: ; %memmove_bwd_main_loop
 ; CHECK-NEXT:    ; =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    ds_read_b64 v[5:6], v4
-; CHECK-NEXT:    v_add_co_u32 v0, vcc_lo, v0, 8
+; CHECK-NEXT:    ds_read_b128 v[5:8], v4
+; CHECK-NEXT:    v_add_co_u32 v0, vcc_lo, v0, 16
 ; CHECK-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
-; CHECK-NEXT:    v_add_nc_u32_e32 v4, -8, v4
+; CHECK-NEXT:    v_add_nc_u32_e32 v4, -16, v4
 ; CHECK-NEXT:    v_cmp_eq_u64_e32 vcc_lo, 0, v[0:1]
 ; CHECK-NEXT:    s_or_b32 s6, vcc_lo, s6
 ; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
-; CHECK-NEXT:    ds_write_b64 v2, v[5:6]
-; CHECK-NEXT:    v_add_nc_u32_e32 v2, -8, v2
+; CHECK-NEXT:    ds_write_b128 v2, v[5:8]
+; CHECK-NEXT:    v_add_nc_u32_e32 v2, -16, v2
 ; CHECK-NEXT:    s_andn2_b32 exec_lo, exec_lo, s6
 ; CHECK-NEXT:    s_cbranch_execnz .LBB12_15
 ; CHECK-NEXT:  .LBB12_16: ; %Flow43
@@ -1664,9 +1664,9 @@ define void @memmove_p3_p4(ptr addrspace(3) align 1 %dst, ptr addrspace(4) align
 ; CHECK-LABEL: memmove_p3_p4:
 ; CHECK:       ; %bb.0: ; %entry
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    v_and_b32_e32 v7, -8, v3
+; CHECK-NEXT:    v_and_b32_e32 v7, -16, v3
 ; CHECK-NEXT:    v_mov_b32_e32 v8, v4
-; CHECK-NEXT:    v_and_b32_e32 v5, 7, v3
+; CHECK-NEXT:    v_and_b32_e32 v5, 15, v3
 ; CHECK-NEXT:    v_mov_b32_e32 v6, 0
 ; CHECK-NEXT:    s_mov_b64 s[4:5], 0
 ; CHECK-NEXT:    s_mov_b32 s6, exec_lo
@@ -1679,14 +1679,14 @@ define void @memmove_p3_p4(ptr addrspace(3) align 1 %dst, ptr addrspace(4) align
 ; CHECK-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    v_add_co_u32 v10, vcc_lo, v1, s4
 ; CHECK-NEXT:    v_add_co_ci_u32_e32 v11, vcc_lo, s5, v2, vcc_lo
-; CHECK-NEXT:    s_add_u32 s4, s4, 8
+; CHECK-NEXT:    s_add_u32 s4, s4, 16
 ; CHECK-NEXT:    s_addc_u32 s5, s5, 0
 ; CHECK-NEXT:    v_cmp_ge_u64_e32 vcc_lo, s[4:5], v[7:8]
-; CHECK-NEXT:    global_load_dwordx2 v[10:11], v[10:11], off
+; CHECK-NEXT:    global_load_dwordx4 v[10:13], v[10:11], off
 ; CHECK-NEXT:    s_or_b32 s7, vcc_lo, s7
 ; CHECK-NEXT:    s_waitcnt vmcnt(0)
-; CHECK-NEXT:    ds_write_b64 v9, v[10:11]
-; CHECK-NEXT:    v_add_nc_u32_e32 v9, 8, v9
+; CHECK-NEXT:    ds_write_b128 v9, v[10:13]
+; CHECK-NEXT:    v_add_nc_u32_e32 v9, 16, v9
 ; CHECK-NEXT:    s_andn2_b32 exec_lo, exec_lo, s7
 ; CHECK-NEXT:    s_cbranch_execnz .LBB13_2
 ; CHECK-NEXT:  .LBB13_3: ; %Flow9
@@ -1697,7 +1697,7 @@ define void @memmove_p3_p4(ptr addrspace(3) align 1 %dst, ptr addrspace(4) align
 ; CHECK-NEXT:    s_xor_b32 s6, exec_lo, s6
 ; CHECK-NEXT:    s_cbranch_execz .LBB13_7
 ; CHECK-NEXT:  ; %bb.4: ; %loop-memcpy-residual.preheader
-; CHECK-NEXT:    v_and_b32_e32 v3, -8, v3
+; CHECK-NEXT:    v_and_b32_e32 v3, -16, v3
 ; CHECK-NEXT:    s_mov_b32 s7, 0
 ; CHECK-NEXT:    v_add_co_u32 v1, vcc_lo, v1, v3
 ; CHECK-NEXT:    v_add_nc_u32_e32 v0, v0, v3
@@ -1735,27 +1735,30 @@ define void @memmove_p3_p5(ptr addrspace(3) align 1 %dst, ptr addrspace(5) align
 ; CHECK-NEXT:    v_mov_b32_e32 v6, 0
 ; CHECK-NEXT:    s_mov_b64 s[4:5], 0
 ; CHECK-NEXT:    s_mov_b32 s6, exec_lo
-; CHECK-NEXT:    v_and_b32_e32 v2, -8, v4
-; CHECK-NEXT:    v_and_b32_e32 v5, 7, v4
+; CHECK-NEXT:    v_and_b32_e32 v2, -16, v4
+; CHECK-NEXT:    v_and_b32_e32 v5, 15, v4
 ; CHECK-NEXT:    v_cmpx_ne_u64_e32 0, v[2:3]
 ; CHECK-NEXT:    s_cbranch_execz .LBB14_3
 ; CHECK-NEXT:  ; %bb.1: ; %loop-memcpy-expansion.preheader
 ; CHECK-NEXT:    v_mov_b32_e32 v7, v1
 ; CHECK-NEXT:    v_mov_b32_e32 v8, v0
 ; CHECK-NEXT:    s_mov_b32 s7, 0
+; CHECK-NEXT:    .p2align 6
 ; CHECK-NEXT:  .LBB14_2: ; %loop-memcpy-expansion
 ; CHECK-NEXT:    ; =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    s_clause 0x1
+; CHECK-NEXT:    s_clause 0x3
 ; CHECK-NEXT:    buffer_load_dword v9, v7, s[0:3], 0 offen
 ; CHECK-NEXT:    buffer_load_dword v10, v7, s[0:3], 0 offen offset:4
-; CHECK-NEXT:    s_add_u32 s4, s4, 8
+; CHECK-NEXT:    buffer_load_dword v11, v7, s[0:3], 0 offen offset:8
+; CHECK-NEXT:    buffer_load_dword v12, v7, s[0:3], 0 offen offset:12
+; CHECK-NEXT:    s_add_u32 s4, s4, 16
 ; CHECK-NEXT:    s_addc_u32 s5, s5, 0
-; CHECK-NEXT:    v_add_nc_u32_e32 v7, 8, v7
+; CHECK-NEXT:    v_add_nc_u32_e32 v7, 16, v7
 ; CHECK-NEXT:    v_cmp_ge_u64_e32 vcc_lo, s[4:5], v[2:3]
 ; CHECK-NEXT:    s_or_b32 s7, vcc_lo, s7
 ; CHECK-NEXT:    s_waitcnt vmcnt(0)
-; CHECK-NEXT:    ds_write_b64 v8, v[9:10]
-; CHECK-NEXT:    v_add_nc_u32_e32 v8, 8, v8
+; CHECK-NEXT:    ds_write_b128 v8, v[9:12]
+; CHECK-NEXT:    v_add_nc_u32_e32 v8, 16, v8
 ; CHECK-NEXT:    s_andn2_b32 exec_lo, exec_lo, s7
 ; CHECK-NEXT:    s_cbranch_execnz .LBB14_2
 ; CHECK-NEXT:  .LBB14_3: ; %Flow14
@@ -1766,7 +1769,7 @@ define void @memmove_p3_p5(ptr addrspace(3) align 1 %dst, ptr addrspace(5) align
 ; CHECK-NEXT:    s_xor_b32 s6, exec_lo, s6
 ; CHECK-NEXT:    s_cbranch_execz .LBB14_7
 ; CHECK-NEXT:  ; %bb.4: ; %loop-memcpy-residual.preheader
-; CHECK-NEXT:    v_and_b32_e32 v2, -8, v4
+; CHECK-NEXT:    v_and_b32_e32 v2, -16, v4
 ; CHECK-NEXT:    s_mov_b32 s7, 0
 ; CHECK-NEXT:    v_add_nc_u32_e32 v0, v0, v2
 ; CHECK-NEXT:    v_add_nc_u32_e32 v1, v1, v2
@@ -2021,25 +2024,28 @@ define void @memmove_p5_p3(ptr addrspace(5) align 1 %dst, ptr addrspace(3) align
 ; CHECK-NEXT:    v_mov_b32_e32 v6, 0
 ; CHECK-NEXT:    s_mov_b64 s[4:5], 0
 ; CHECK-NEXT:    s_mov_b32 s6, exec_lo
-; CHECK-NEXT:    v_and_b32_e32 v2, -8, v4
-; CHECK-NEXT:    v_and_b32_e32 v5, 7, v4
+; CHECK-NEXT:    v_and_b32_e32 v2, -16, v4
+; CHECK-NEXT:    v_and_b32_e32 v5, 15, v4
 ; CHECK-NEXT:    v_cmpx_ne_u64_e32 0, v[2:3]
 ; CHECK-NEXT:    s_cbranch_execz .LBB17_3
 ; CHECK-NEXT:  ; %bb.1: ; %loop-memcpy-expansion.preheader
 ; CHECK-NEXT:    v_mov_b32_e32 v7, v1
 ; CHECK-NEXT:    v_mov_b32_e32 v8, v0
 ; CHECK-NEXT:    s_mov_b32 s7, 0
+; CHECK-NEXT:    .p2align 6
 ; CHECK-NEXT:  .LBB17_2: ; %loop-memcpy-expansion
 ; CHECK-NEXT:    ; =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    ds_read_b64 v[9:10], v7
-; CHECK-NEXT:    s_add_u32 s4, s4, 8
+; CHECK-NEXT:    ds_read_b128 v[9:12], v7
+; CHECK-NEXT:    s_add_u32 s4, s4, 16
 ; CHECK-NEXT:    s_addc_u32 s5, s5, 0
-; CHECK-NEXT:    v_add_nc_u32_e32 v7, 8, v7
+; CHECK-NEXT:    v_add_nc_u32_e32 v7, 16, v7
 ; CHECK-NEXT:    v_cmp_ge_u64_e32 vcc_lo, s[4:5], v[2:3]
 ; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
+; CHECK-NEXT:    buffer_store_dword v12, v8, s[0:3], 0 offen offset:12
+; CHECK-NEXT:    buffer_store_dword v11, v8, s[0:3], 0 offen offset:8
 ; CHECK-NEXT:    buffer_store_dword v10, v8, s[0:3], 0 offen offset:4
 ; CHECK-NEXT:    buffer_store_dword v9, v8, s[0:3], 0 offen
-; CHECK-NEXT:    v_add_nc_u32_e32 v8, 8, v8
+; CHECK-NEXT:    v_add_nc_u32_e32 v8, 16, v8
 ; CHECK-NEXT:    s_or_b32 s7, vcc_lo, s7
 ; CHECK-NEXT:    s_andn2_b32 exec_lo, exec_lo, s7
 ; CHECK-NEXT:    s_cbranch_execnz .LBB17_2
@@ -2051,7 +2057,7 @@ define void @memmove_p5_p3(ptr addrspace(5) align 1 %dst, ptr addrspace(3) align
 ; CHECK-NEXT:    s_xor_b32 s6, exec_lo, s6
 ; CHECK-NEXT:    s_cbranch_execz .LBB17_7
 ; CHECK-NEXT:  ; %bb.4: ; %loop-memcpy-residual.preheader
-; CHECK-NEXT:    v_and_b32_e32 v2, -8, v4
+; CHECK-NEXT:    v_and_b32_e32 v2, -16, v4
 ; CHECK-NEXT:    s_mov_b32 s7, 0
 ; CHECK-NEXT:    v_add_nc_u32_e32 v0, v0, v2
 ; CHECK-NEXT:    v_add_nc_u32_e32 v1, v1, v2
diff --git a/llvm/test/CodeGen/AMDGPU/memory-legalizer-flat-lastuse.ll b/llvm/test/CodeGen/AMDGPU/memory-legalizer-flat-lastuse.ll
index ab485b1799470..a00af8e5b6582 100644
--- a/llvm/test/CodeGen/AMDGPU/memory-legalizer-flat-lastuse.ll
+++ b/llvm/test/CodeGen/AMDGPU/memory-legalizer-flat-lastuse.ll
@@ -26,29 +26,26 @@ define amdgpu_kernel void @flat_last_use_load_1(ptr %in, ptr %out) {
 ; GFX12-LABEL: flat_last_use_load_1:
 ; GFX12:       ; %bb.0: ; %entry
 ; GFX12-NEXT:    s_mov_b64 s[0:1], s[4:5]
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_load_b64 s[4:5], s[0:1], 0x0
 ; GFX12-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
 ; GFX12-NEXT:    s_mov_b32 s2, 0x3ff
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    v_and_b32_e64 v0, v0, s2
 ; GFX12-NEXT:    s_mov_b32 s2, 2
 ; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    v_lshlrev_b32_e64 v1, s2, v0
 ; GFX12-NEXT:    s_mov_b32 s2, 0
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    ; implicit-def: $sgpr2
 ; GFX12-NEXT:    v_mov_b32_e32 v0, 0
 ; GFX12-NEXT:    ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
 ; GFX12-NEXT:    v_mov_b32_e32 v2, v0
 ; GFX12-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-NEXT:    s_mov_b32 s3, s4
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    v_mov_b32_e32 v0, v1
 ; GFX12-NEXT:    s_mov_b32 s2, s5
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    v_mov_b32_e32 v1, v2
+; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    v_add_co_u32 v0, s3, s3, v0
+; GFX12-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-NEXT:    v_add_co_ci_u32_e64 v2, s2, s2, v1, s3
 ; GFX12-NEXT:    ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec
 ; GFX12-NEXT:    v_mov_b32_e32 v1, v2
diff --git a/llvm/test/CodeGen/AMDGPU/memory-legalizer-flat-nontemporal.ll b/llvm/test/CodeGen/AMDGPU/memory-legalizer-flat-nontemporal.ll
index ddc4673a290fe..df5b45dea0c2f 100644
--- a/llvm/test/CodeGen/AMDGPU/memory-legalizer-flat-nontemporal.ll
+++ b/llvm/test/CodeGen/AMDGPU/memory-legalizer-flat-nontemporal.ll
@@ -463,29 +463,26 @@ define amdgpu_kernel void @flat_nontemporal_load_1(
 ; GFX12-WGP-LABEL: flat_nontemporal_load_1:
 ; GFX12-WGP:       ; %bb.0: ; %entry
 ; GFX12-WGP-NEXT:    s_mov_b64 s[0:1], s[4:5]
-; GFX12-WGP-NEXT:    s_wait_alu 0xfffe
 ; GFX12-WGP-NEXT:    s_load_b64 s[4:5], s[0:1], 0x0
 ; GFX12-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
 ; GFX12-WGP-NEXT:    s_mov_b32 s2, 0x3ff
-; GFX12-WGP-NEXT:    s_wait_alu 0xfffe
 ; GFX12-WGP-NEXT:    v_and_b32_e64 v0, v0, s2
 ; GFX12-WGP-NEXT:    s_mov_b32 s2, 2
 ; GFX12-WGP-NEXT:    s_wait_alu 0xfffe
 ; GFX12-WGP-NEXT:    v_lshlrev_b32_e64 v1, s2, v0
 ; GFX12-WGP-NEXT:    s_mov_b32 s2, 0
-; GFX12-WGP-NEXT:    s_wait_alu 0xfffe
 ; GFX12-WGP-NEXT:    ; implicit-def: $sgpr2
 ; GFX12-WGP-NEXT:    v_mov_b32_e32 v0, 0
 ; GFX12-WGP-NEXT:    ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
 ; GFX12-WGP-NEXT:    v_mov_b32_e32 v2, v0
 ; GFX12-WGP-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-WGP-NEXT:    s_mov_b32 s3, s4
-; GFX12-WGP-NEXT:    s_wait_alu 0xfffe
 ; GFX12-WGP-NEXT:    v_mov_b32_e32 v0, v1
 ; GFX12-WGP-NEXT:    s_mov_b32 s2, s5
-; GFX12-WGP-NEXT:    s_wait_alu 0xfffe
 ; GFX12-WGP-NEXT:    v_mov_b32_e32 v1, v2
+; GFX12-WGP-NEXT:    s_wait_alu 0xfffe
 ; GFX12-WGP-NEXT:    v_add_co_u32 v0, s3, s3, v0
+; GFX12-WGP-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-WGP-NEXT:    v_add_co_ci_u32_e64 v2, s2, s2, v1, s3
 ; GFX12-WGP-NEXT:    ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec
 ; GFX12-WGP-NEXT:    v_mov_b32_e32 v1, v2
@@ -499,29 +496,26 @@ define amdgpu_kernel void @flat_nontemporal_load_1(
 ; GFX12-CU-LABEL: flat_nontemporal_load_1:
 ; GFX12-CU:       ; %bb.0: ; %entry
 ; GFX12-CU-NEXT:    s_mov_b64 s[0:1], s[4:5]
-; GFX12-CU-NEXT:    s_wait_alu 0xfffe
 ; GFX12-CU-NEXT:    s_load_b64 s[4:5], s[0:1], 0x0
 ; GFX12-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
 ; GFX12-CU-NEXT:    s_mov_b32 s2, 0x3ff
-; GFX12-CU-NEXT:    s_wait_alu 0xfffe
 ; GFX12-CU-NEXT:    v_and_b32_e64 v0, v0, s2
 ; GFX12-CU-NEXT:    s_mov_b32 s2, 2
 ; GFX12-CU-NEXT:    s_wait_alu 0xfffe
 ; GFX12-CU-NEXT:    v_lshlrev_b32_e64 v1, s2, v0
 ; GFX12-CU-NEXT:    s_mov_b32 s2, 0
-; GFX12-CU-NEXT:    s_wait_alu 0xfffe
 ; GFX12-CU-NEXT:    ; implicit-def: $sgpr2
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v0, 0
 ; GFX12-CU-NEXT:    ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v2, v0
 ; GFX12-CU-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-CU-NEXT:    s_mov_b32 s3, s4
-; GFX12-CU-NEXT:    s_wait_alu 0xfffe
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v0, v1
 ; GFX12-CU-NEXT:    s_mov_b32 s2, s5
-; GFX12-CU-NEXT:    s_wait_alu 0xfffe
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, v2
+; GFX12-CU-NEXT:    s_wait_alu 0xfffe
 ; GFX12-CU-NEXT:    v_add_co_u32 v0, s3, s3, v0
+; GFX12-CU-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-CU-NEXT:    v_add_co_ci_u32_e64 v2, s2, s2, v1, s3
 ; GFX12-CU-NEXT:    ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, v2
@@ -991,18 +985,17 @@ define amdgpu_kernel void @flat_nontemporal_store_1(
 ; GFX12-WGP-NEXT:    s_wait_alu 0xfffe
 ; GFX12-WGP-NEXT:    v_lshlrev_b32_e64 v3, s0, v0
 ; GFX12-WGP-NEXT:    s_mov_b32 s0, 0
-; GFX12-WGP-NEXT:    s_wait_alu 0xfffe
 ; GFX12-WGP-NEXT:    ; implicit-def: $sgpr0
 ; GFX12-WGP-NEXT:    v_mov_b32_e32 v0, 0
 ; GFX12-WGP-NEXT:    ; kill: def $vgpr3 killed $vgpr3 def $vgpr3_vgpr4 killed $exec
 ; GFX12-WGP-NEXT:    v_mov_b32_e32 v4, v0
 ; GFX12-WGP-NEXT:    s_mov_b32 s1, s2
-; GFX12-WGP-NEXT:    s_wait_alu 0xfffe
 ; GFX12-WGP-NEXT:    v_mov_b32_e32 v0, v3
 ; GFX12-WGP-NEXT:    s_mov_b32 s0, s3
-; GFX12-WGP-NEXT:    s_wait_alu 0xfffe
 ; GFX12-WGP-NEXT:    v_mov_b32_e32 v1, v4
+; GFX12-WGP-NEXT:    s_wait_alu 0xfffe
 ; GFX12-WGP-NEXT:    v_add_co_u32 v0, s1, s1, v0
+; GFX12-WGP-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-WGP-NEXT:    v_add_co_ci_u32_e64 v3, s0, s0, v1, s1
 ; GFX12-WGP-NEXT:    ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec
 ; GFX12-WGP-NEXT:    v_mov_b32_e32 v1, v3
@@ -1025,18 +1018,17 @@ define amdgpu_kernel void @flat_nontemporal_store_1(
 ; GFX12-CU-NEXT:    s_wait_alu 0xfffe
 ; GFX12-CU-NEXT:    v_lshlrev_b32_e64 v3, s0, v0
 ; GFX12-CU-NEXT:    s_mov_b32 s0, 0
-; GFX12-CU-NEXT:    s_wait_alu 0xfffe
 ; GFX12-CU-NEXT:    ; implicit-def: $sgpr0
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v0, 0
 ; GFX12-CU-NEXT:    ; kill: def $vgpr3 killed $vgpr3 def $vgpr3_vgpr4 killed $exec
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v4, v0
 ; GFX12-CU-NEXT:    s_mov_b32 s1, s2
-; GFX12-CU-NEXT:    s_wait_alu 0xfffe
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v0, v3
 ; GFX12-CU-NEXT:    s_mov_b32 s0, s3
-; GFX12-CU-NEXT:    s_wait_alu 0xfffe
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, v4
+; GFX12-CU-NEXT:    s_wait_alu 0xfffe
 ; GFX12-CU-NEXT:    v_add_co_u32 v0, s1, s1, v0
+; GFX12-CU-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-CU-NEXT:    v_add_co_ci_u32_e64 v3, s0, s0, v1, s1
 ; GFX12-CU-NEXT:    ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, v3
diff --git a/llvm/test/CodeGen/AMDGPU/memory-legalizer-flat-volatile.ll b/llvm/test/CodeGen/AMDGPU/memory-legalizer-flat-volatile.ll
index f10715033e433..e1f82a70b4c0a 100644
--- a/llvm/test/CodeGen/AMDGPU/memory-legalizer-flat-volatile.ll
+++ b/llvm/test/CodeGen/AMDGPU/memory-legalizer-flat-volatile.ll
@@ -325,29 +325,26 @@ define amdgpu_kernel void @flat_nontemporal_load_1(
 ; GFX12-WGP-LABEL: flat_nontemporal_load_1:
 ; GFX12-WGP:       ; %bb.0: ; %entry
 ; GFX12-WGP-NEXT:    s_mov_b64 s[0:1], s[4:5]
-; GFX12-WGP-NEXT:    s_wait_alu 0xfffe
 ; GFX12-WGP-NEXT:    s_load_b64 s[4:5], s[0:1], 0x0
 ; GFX12-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
 ; GFX12-WGP-NEXT:    s_mov_b32 s2, 0x3ff
-; GFX12-WGP-NEXT:    s_wait_alu 0xfffe
 ; GFX12-WGP-NEXT:    v_and_b32_e64 v0, v0, s2
 ; GFX12-WGP-NEXT:    s_mov_b32 s2, 2
 ; GFX12-WGP-NEXT:    s_wait_alu 0xfffe
 ; GFX12-WGP-NEXT:    v_lshlrev_b32_e64 v1, s2, v0
 ; GFX12-WGP-NEXT:    s_mov_b32 s2, 0
-; GFX12-WGP-NEXT:    s_wait_alu 0xfffe
 ; GFX12-WGP-NEXT:    ; implicit-def: $sgpr2
 ; GFX12-WGP-NEXT:    v_mov_b32_e32 v0, 0
 ; GFX12-WGP-NEXT:    ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
 ; GFX12-WGP-NEXT:    v_mov_b32_e32 v2, v0
 ; GFX12-WGP-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-WGP-NEXT:    s_mov_b32 s3, s4
-; GFX12-WGP-NEXT:    s_wait_alu 0xfffe
 ; GFX12-WGP-NEXT:    v_mov_b32_e32 v0, v1
 ; GFX12-WGP-NEXT:    s_mov_b32 s2, s5
-; GFX12-WGP-NEXT:    s_wait_alu 0xfffe
 ; GFX12-WGP-NEXT:    v_mov_b32_e32 v1, v2
+; GFX12-WGP-NEXT:    s_wait_alu 0xfffe
 ; GFX12-WGP-NEXT:    v_add_co_u32 v0, s3, s3, v0
+; GFX12-WGP-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-WGP-NEXT:    v_add_co_ci_u32_e64 v2, s2, s2, v1, s3
 ; GFX12-WGP-NEXT:    ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec
 ; GFX12-WGP-NEXT:    v_mov_b32_e32 v1, v2
@@ -364,29 +361,26 @@ define amdgpu_kernel void @flat_nontemporal_load_1(
 ; GFX12-CU-LABEL: flat_nontemporal_load_1:
 ; GFX12-CU:       ; %bb.0: ; %entry
 ; GFX12-CU-NEXT:    s_mov_b64 s[0:1], s[4:5]
-; GFX12-CU-NEXT:    s_wait_alu 0xfffe
 ; GFX12-CU-NEXT:    s_load_b64 s[4:5], s[0:1], 0x0
 ; GFX12-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
 ; GFX12-CU-NEXT:    s_mov_b32 s2, 0x3ff
-; GFX12-CU-NEXT:    s_wait_alu 0xfffe
 ; GFX12-CU-NEXT:    v_and_b32_e64 v0, v0, s2
 ; GFX12-CU-NEXT:    s_mov_b32 s2, 2
 ; GFX12-CU-NEXT:    s_wait_alu 0xfffe
 ; GFX12-CU-NEXT:    v_lshlrev_b32_e64 v1, s2, v0
 ; GFX12-CU-NEXT:    s_mov_b32 s2, 0
-; GFX12-CU-NEXT:    s_wait_alu 0xfffe
 ; GFX12-CU-NEXT:    ; implicit-def: $sgpr2
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v0, 0
 ; GFX12-CU-NEXT:    ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v2, v0
 ; GFX12-CU-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-CU-NEXT:    s_mov_b32 s3, s4
-; GFX12-CU-NEXT:    s_wait_alu 0xfffe
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v0, v1
 ; GFX12-CU-NEXT:    s_mov_b32 s2, s5
-; GFX12-CU-NEXT:    s_wait_alu 0xfffe
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, v2
+; GFX12-CU-NEXT:    s_wait_alu 0xfffe
 ; GFX12-CU-NEXT:    v_add_co_u32 v0, s3, s3, v0
+; GFX12-CU-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-CU-NEXT:    v_add_co_ci_u32_e64 v2, s2, s2, v1, s3
 ; GFX12-CU-NEXT:    ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, v2
@@ -733,18 +727,17 @@ define amdgpu_kernel void @flat_nontemporal_store_1(
 ; GFX12-WGP-NEXT:    s_wait_alu 0xfffe
 ; GFX12-WGP-NEXT:    v_lshlrev_b32_e64 v3, s0, v0
 ; GFX12-WGP-NEXT:    s_mov_b32 s0, 0
-; GFX12-WGP-NEXT:    s_wait_alu 0xfffe
 ; GFX12-WGP-NEXT:    ; implicit-def: $sgpr0
 ; GFX12-WGP-NEXT:    v_mov_b32_e32 v0, 0
 ; GFX12-WGP-NEXT:    ; kill: def $vgpr3 killed $vgpr3 def $vgpr3_vgpr4 killed $exec
 ; GFX12-WGP-NEXT:    v_mov_b32_e32 v4, v0
 ; GFX12-WGP-NEXT:    s_mov_b32 s1, s2
-; GFX12-WGP-NEXT:    s_wait_alu 0xfffe
 ; GFX12-WGP-NEXT:    v_mov_b32_e32 v0, v3
 ; GFX12-WGP-NEXT:    s_mov_b32 s0, s3
-; GFX12-WGP-NEXT:    s_wait_alu 0xfffe
 ; GFX12-WGP-NEXT:    v_mov_b32_e32 v1, v4
+; GFX12-WGP-NEXT:    s_wait_alu 0xfffe
 ; GFX12-WGP-NEXT:    v_add_co_u32 v0, s1, s1, v0
+; GFX12-WGP-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-WGP-NEXT:    v_add_co_ci_u32_e64 v3, s0, s0, v1, s1
 ; GFX12-WGP-NEXT:    ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec
 ; GFX12-WGP-NEXT:    v_mov_b32_e32 v1, v3
@@ -772,18 +765,17 @@ define amdgpu_kernel void @flat_nontemporal_store_1(
 ; GFX12-CU-NEXT:    s_wait_alu 0xfffe
 ; GFX12-CU-NEXT:    v_lshlrev_b32_e64 v3, s0, v0
 ; GFX12-CU-NEXT:    s_mov_b32 s0, 0
-; GFX12-CU-NEXT:    s_wait_alu 0xfffe
 ; GFX12-CU-NEXT:    ; implicit-def: $sgpr0
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v0, 0
 ; GFX12-CU-NEXT:    ; kill: def $vgpr3 killed $vgpr3 def $vgpr3_vgpr4 killed $exec
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v4, v0
 ; GFX12-CU-NEXT:    s_mov_b32 s1, s2
-; GFX12-CU-NEXT:    s_wait_alu 0xfffe
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v0, v3
 ; GFX12-CU-NEXT:    s_mov_b32 s0, s3
-; GFX12-CU-NEXT:    s_wait_alu 0xfffe
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, v4
+; GFX12-CU-NEXT:    s_wait_alu 0xfffe
 ; GFX12-CU-NEXT:    v_add_co_u32 v0, s1, s1, v0
+; GFX12-CU-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-CU-NEXT:    v_add_co_ci_u32_e64 v3, s0, s0, v1, s1
 ; GFX12-CU-NEXT:    ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, v3
diff --git a/llvm/test/CodeGen/AMDGPU/memory-legalizer-global-lastuse.ll b/llvm/test/CodeGen/AMDGPU/memory-legalizer-global-lastuse.ll
index 465626078f6c6..5f952b98041f3 100644
--- a/llvm/test/CodeGen/AMDGPU/memory-legalizer-global-lastuse.ll
+++ b/llvm/test/CodeGen/AMDGPU/memory-legalizer-global-lastuse.ll
@@ -28,7 +28,6 @@ define amdgpu_kernel void @global_last_use_load_1(ptr addrspace(1) %in, ptr addr
 ; GFX12-NEXT:    s_load_b64 s[0:1], s[4:5], 0x8
 ; GFX12-NEXT:    v_mov_b32_e32 v0, 0
 ; GFX12-NEXT:    s_mov_b32 s4, 0x3ff
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    v_and_b32_e64 v1, v1, s4
 ; GFX12-NEXT:    s_mov_b32 s4, 2
 ; GFX12-NEXT:    s_wait_alu 0xfffe
@@ -73,7 +72,6 @@ define amdgpu_kernel void @global_last_use_and_nontemporal_load(ptr addrspace(1)
 ; GFX12-NEXT:    s_load_b64 s[0:1], s[4:5], 0x8
 ; GFX12-NEXT:    v_mov_b32_e32 v0, 0
 ; GFX12-NEXT:    s_mov_b32 s4, 0x3ff
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    v_and_b32_e64 v1, v1, s4
 ; GFX12-NEXT:    s_mov_b32 s4, 2
 ; GFX12-NEXT:    s_wait_alu 0xfffe
diff --git a/llvm/test/CodeGen/AMDGPU/memory-legalizer-global-nontemporal.ll b/llvm/test/CodeGen/AMDGPU/memory-legalizer-global-nontemporal.ll
index f06118a7a6dc9..ebcc900307c46 100644
--- a/llvm/test/CodeGen/AMDGPU/memory-legalizer-global-nontemporal.ll
+++ b/llvm/test/CodeGen/AMDGPU/memory-legalizer-global-nontemporal.ll
@@ -416,7 +416,6 @@ define amdgpu_kernel void @global_nontemporal_load_1(
 ; GFX12-WGP-NEXT:    s_load_b64 s[0:1], s[4:5], 0x8
 ; GFX12-WGP-NEXT:    v_mov_b32_e32 v0, 0
 ; GFX12-WGP-NEXT:    s_mov_b32 s4, 0x3ff
-; GFX12-WGP-NEXT:    s_wait_alu 0xfffe
 ; GFX12-WGP-NEXT:    v_and_b32_e64 v1, v1, s4
 ; GFX12-WGP-NEXT:    s_mov_b32 s4, 2
 ; GFX12-WGP-NEXT:    s_wait_alu 0xfffe
@@ -434,7 +433,6 @@ define amdgpu_kernel void @global_nontemporal_load_1(
 ; GFX12-CU-NEXT:    s_load_b64 s[0:1], s[4:5], 0x8
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v0, 0
 ; GFX12-CU-NEXT:    s_mov_b32 s4, 0x3ff
-; GFX12-CU-NEXT:    s_wait_alu 0xfffe
 ; GFX12-CU-NEXT:    v_and_b32_e64 v1, v1, s4
 ; GFX12-CU-NEXT:    s_mov_b32 s4, 2
 ; GFX12-CU-NEXT:    s_wait_alu 0xfffe
@@ -832,7 +830,6 @@ define amdgpu_kernel void @global_nontemporal_store_1(
 ; GFX12-WGP-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-WGP-NEXT:    s_load_b32 s2, s[2:3], 0x0
 ; GFX12-WGP-NEXT:    s_mov_b32 s3, 0x3ff
-; GFX12-WGP-NEXT:    s_wait_alu 0xfffe
 ; GFX12-WGP-NEXT:    v_and_b32_e64 v0, v0, s3
 ; GFX12-WGP-NEXT:    s_mov_b32 s3, 2
 ; GFX12-WGP-NEXT:    s_wait_alu 0xfffe
@@ -849,7 +846,6 @@ define amdgpu_kernel void @global_nontemporal_store_1(
 ; GFX12-CU-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-CU-NEXT:    s_load_b32 s2, s[2:3], 0x0
 ; GFX12-CU-NEXT:    s_mov_b32 s3, 0x3ff
-; GFX12-CU-NEXT:    s_wait_alu 0xfffe
 ; GFX12-CU-NEXT:    v_and_b32_e64 v0, v0, s3
 ; GFX12-CU-NEXT:    s_mov_b32 s3, 2
 ; GFX12-CU-NEXT:    s_wait_alu 0xfffe
diff --git a/llvm/test/CodeGen/AMDGPU/memory-legalizer-global-volatile.ll b/llvm/test/CodeGen/AMDGPU/memory-legalizer-global-volatile.ll
index 2bf2e03cb0bd7..7dfd5e60c24f8 100644
--- a/llvm/test/CodeGen/AMDGPU/memory-legalizer-global-volatile.ll
+++ b/llvm/test/CodeGen/AMDGPU/memory-legalizer-global-volatile.ll
@@ -309,7 +309,6 @@ define amdgpu_kernel void @global_volatile_load_1(
 ; GFX12-WGP-NEXT:    s_load_b64 s[0:1], s[4:5], 0x8
 ; GFX12-WGP-NEXT:    v_mov_b32_e32 v0, 0
 ; GFX12-WGP-NEXT:    s_mov_b32 s4, 0x3ff
-; GFX12-WGP-NEXT:    s_wait_alu 0xfffe
 ; GFX12-WGP-NEXT:    v_and_b32_e64 v1, v1, s4
 ; GFX12-WGP-NEXT:    s_mov_b32 s4, 2
 ; GFX12-WGP-NEXT:    s_wait_alu 0xfffe
@@ -329,7 +328,6 @@ define amdgpu_kernel void @global_volatile_load_1(
 ; GFX12-CU-NEXT:    s_load_b64 s[0:1], s[4:5], 0x8
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v0, 0
 ; GFX12-CU-NEXT:    s_mov_b32 s4, 0x3ff
-; GFX12-CU-NEXT:    s_wait_alu 0xfffe
 ; GFX12-CU-NEXT:    v_and_b32_e64 v1, v1, s4
 ; GFX12-CU-NEXT:    s_mov_b32 s4, 2
 ; GFX12-CU-NEXT:    s_wait_alu 0xfffe
@@ -647,7 +645,6 @@ define amdgpu_kernel void @global_volatile_store_1(
 ; GFX12-WGP-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-WGP-NEXT:    s_load_b32 s2, s[2:3], 0x0
 ; GFX12-WGP-NEXT:    s_mov_b32 s3, 0x3ff
-; GFX12-WGP-NEXT:    s_wait_alu 0xfffe
 ; GFX12-WGP-NEXT:    v_and_b32_e64 v0, v0, s3
 ; GFX12-WGP-NEXT:    s_mov_b32 s3, 2
 ; GFX12-WGP-NEXT:    s_wait_alu 0xfffe
@@ -670,7 +667,6 @@ define amdgpu_kernel void @global_volatile_store_1(
 ; GFX12-CU-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-CU-NEXT:    s_load_b32 s2, s[2:3], 0x0
 ; GFX12-CU-NEXT:    s_mov_b32 s3, 0x3ff
-; GFX12-CU-NEXT:    s_wait_alu 0xfffe
 ; GFX12-CU-NEXT:    v_and_b32_e64 v0, v0, s3
 ; GFX12-CU-NEXT:    s_mov_b32 s3, 2
 ; GFX12-CU-NEXT:    s_wait_alu 0xfffe
diff --git a/llvm/test/CodeGen/AMDGPU/memory-legalizer-local-nontemporal.ll b/llvm/test/CodeGen/AMDGPU/memory-legalizer-local-nontemporal.ll
index 3c485af18166f..e9be38d6d17a3 100644
--- a/llvm/test/CodeGen/AMDGPU/memory-legalizer-local-nontemporal.ll
+++ b/llvm/test/CodeGen/AMDGPU/memory-legalizer-local-nontemporal.ll
@@ -396,11 +396,10 @@ define amdgpu_kernel void @local_nontemporal_load_1(
 ; GFX12-WGP-NEXT:    s_load_b64 s[0:1], s[4:5], 0x8
 ; GFX12-WGP-NEXT:    v_mov_b32_e32 v0, 0
 ; GFX12-WGP-NEXT:    s_mov_b32 s2, 0x3ff
-; GFX12-WGP-NEXT:    s_wait_alu 0xfffe
 ; GFX12-WGP-NEXT:    v_and_b32_e64 v1, v1, s2
 ; GFX12-WGP-NEXT:    s_mov_b32 s2, 2
-; GFX12-WGP-NEXT:    s_wait_alu 0xfffe
 ; GFX12-WGP-NEXT:    s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT:    s_wait_alu 0xfffe
 ; GFX12-WGP-NEXT:    v_lshl_add_u32 v1, v1, s2, s3
 ; GFX12-WGP-NEXT:    ds_load_b32 v1, v1
 ; GFX12-WGP-NEXT:    s_wait_dscnt 0x0
@@ -414,11 +413,10 @@ define amdgpu_kernel void @local_nontemporal_load_1(
 ; GFX12-CU-NEXT:    s_load_b64 s[0:1], s[4:5], 0x8
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v0, 0
 ; GFX12-CU-NEXT:    s_mov_b32 s2, 0x3ff
-; GFX12-CU-NEXT:    s_wait_alu 0xfffe
 ; GFX12-CU-NEXT:    v_and_b32_e64 v1, v1, s2
 ; GFX12-CU-NEXT:    s_mov_b32 s2, 2
-; GFX12-CU-NEXT:    s_wait_alu 0xfffe
 ; GFX12-CU-NEXT:    s_wait_kmcnt 0x0
+; GFX12-CU-NEXT:    s_wait_alu 0xfffe
 ; GFX12-CU-NEXT:    v_lshl_add_u32 v1, v1, s2, s3
 ; GFX12-CU-NEXT:    ds_load_b32 v1, v1
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
@@ -774,7 +772,6 @@ define amdgpu_kernel void @local_nontemporal_store_1(
 ; GFX12-WGP-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-WGP-NEXT:    s_load_b32 s0, s[0:1], 0x0
 ; GFX12-WGP-NEXT:    s_mov_b32 s1, 0x3ff
-; GFX12-WGP-NEXT:    s_wait_alu 0xfffe
 ; GFX12-WGP-NEXT:    v_and_b32_e64 v0, v0, s1
 ; GFX12-WGP-NEXT:    s_mov_b32 s1, 2
 ; GFX12-WGP-NEXT:    s_wait_alu 0xfffe
@@ -791,7 +788,6 @@ define amdgpu_kernel void @local_nontemporal_store_1(
 ; GFX12-CU-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-CU-NEXT:    s_load_b32 s0, s[0:1], 0x0
 ; GFX12-CU-NEXT:    s_mov_b32 s1, 0x3ff
-; GFX12-CU-NEXT:    s_wait_alu 0xfffe
 ; GFX12-CU-NEXT:    v_and_b32_e64 v0, v0, s1
 ; GFX12-CU-NEXT:    s_mov_b32 s1, 2
 ; GFX12-CU-NEXT:    s_wait_alu 0xfffe
diff --git a/llvm/test/CodeGen/AMDGPU/memory-legalizer-local-volatile.ll b/llvm/test/CodeGen/AMDGPU/memory-legalizer-local-volatile.ll
index 9740e0ae1d167..9e5f5fcffca9f 100644
--- a/llvm/test/CodeGen/AMDGPU/memory-legalizer-local-volatile.ll
+++ b/llvm/test/CodeGen/AMDGPU/memory-legalizer-local-volatile.ll
@@ -276,11 +276,10 @@ define amdgpu_kernel void @local_volatile_load_1(
 ; GFX12-WGP-NEXT:    s_load_b64 s[0:1], s[4:5], 0x8
 ; GFX12-WGP-NEXT:    v_mov_b32_e32 v0, 0
 ; GFX12-WGP-NEXT:    s_mov_b32 s2, 0x3ff
-; GFX12-WGP-NEXT:    s_wait_alu 0xfffe
 ; GFX12-WGP-NEXT:    v_and_b32_e64 v1, v1, s2
 ; GFX12-WGP-NEXT:    s_mov_b32 s2, 2
-; GFX12-WGP-NEXT:    s_wait_alu 0xfffe
 ; GFX12-WGP-NEXT:    s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT:    s_wait_alu 0xfffe
 ; GFX12-WGP-NEXT:    v_lshl_add_u32 v1, v1, s2, s3
 ; GFX12-WGP-NEXT:    ds_load_b32 v1, v1
 ; GFX12-WGP-NEXT:    s_wait_dscnt 0x0
@@ -294,11 +293,10 @@ define amdgpu_kernel void @local_volatile_load_1(
 ; GFX12-CU-NEXT:    s_load_b64 s[0:1], s[4:5], 0x8
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v0, 0
 ; GFX12-CU-NEXT:    s_mov_b32 s2, 0x3ff
-; GFX12-CU-NEXT:    s_wait_alu 0xfffe
 ; GFX12-CU-NEXT:    v_and_b32_e64 v1, v1, s2
 ; GFX12-CU-NEXT:    s_mov_b32 s2, 2
-; GFX12-CU-NEXT:    s_wait_alu 0xfffe
 ; GFX12-CU-NEXT:    s_wait_kmcnt 0x0
+; GFX12-CU-NEXT:    s_wait_alu 0xfffe
 ; GFX12-CU-NEXT:    v_lshl_add_u32 v1, v1, s2, s3
 ; GFX12-CU-NEXT:    ds_load_b32 v1, v1
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
@@ -552,7 +550,6 @@ define amdgpu_kernel void @local_volatile_store_1(
 ; GFX12-WGP-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-WGP-NEXT:    s_load_b32 s0, s[0:1], 0x0
 ; GFX12-WGP-NEXT:    s_mov_b32 s1, 0x3ff
-; GFX12-WGP-NEXT:    s_wait_alu 0xfffe
 ; GFX12-WGP-NEXT:    v_and_b32_e64 v0, v0, s1
 ; GFX12-WGP-NEXT:    s_mov_b32 s1, 2
 ; GFX12-WGP-NEXT:    s_wait_alu 0xfffe
@@ -574,7 +571,6 @@ define amdgpu_kernel void @local_volatile_store_1(
 ; GFX12-CU-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-CU-NEXT:    s_load_b32 s0, s[0:1], 0x0
 ; GFX12-CU-NEXT:    s_mov_b32 s1, 0x3ff
-; GFX12-CU-NEXT:    s_wait_alu 0xfffe
 ; GFX12-CU-NEXT:    v_and_b32_e64 v0, v0, s1
 ; GFX12-CU-NEXT:    s_mov_b32 s1, 2
 ; GFX12-CU-NEXT:    s_wait_alu 0xfffe
diff --git a/llvm/test/CodeGen/AMDGPU/memory-legalizer-private-lastuse.ll b/llvm/test/CodeGen/AMDGPU/memory-legalizer-private-lastuse.ll
index f988a4d33add9..bc905fa564f8a 100644
--- a/llvm/test/CodeGen/AMDGPU/memory-legalizer-private-lastuse.ll
+++ b/llvm/test/CodeGen/AMDGPU/memory-legalizer-private-lastuse.ll
@@ -27,7 +27,6 @@ define amdgpu_kernel void @private_last_use_load_1(ptr addrspace(5) %in, ptr add
 ; GFX12-NEXT:    s_load_b64 s[0:1], s[4:5], 0x8
 ; GFX12-NEXT:    v_mov_b32_e32 v0, 0
 ; GFX12-NEXT:    s_mov_b32 s3, 0x3ff
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    v_and_b32_e64 v1, v1, s3
 ; GFX12-NEXT:    s_mov_b32 s3, 2
 ; GFX12-NEXT:    s_wait_alu 0xfffe
diff --git a/llvm/test/CodeGen/AMDGPU/memory-legalizer-private-nontemporal.ll b/llvm/test/CodeGen/AMDGPU/memory-legalizer-private-nontemporal.ll
index 71f28efd47811..6feab49ed86b6 100644
--- a/llvm/test/CodeGen/AMDGPU/memory-legalizer-private-nontemporal.ll
+++ b/llvm/test/CodeGen/AMDGPU/memory-legalizer-private-nontemporal.ll
@@ -418,7 +418,6 @@ define amdgpu_kernel void @private_nontemporal_load_1(
 ; GFX12-WGP-NEXT:    s_load_b64 s[0:1], s[4:5], 0x8
 ; GFX12-WGP-NEXT:    v_mov_b32_e32 v0, 0
 ; GFX12-WGP-NEXT:    s_mov_b32 s3, 0x3ff
-; GFX12-WGP-NEXT:    s_wait_alu 0xfffe
 ; GFX12-WGP-NEXT:    v_and_b32_e64 v1, v1, s3
 ; GFX12-WGP-NEXT:    s_mov_b32 s3, 2
 ; GFX12-WGP-NEXT:    s_wait_alu 0xfffe
@@ -436,7 +435,6 @@ define amdgpu_kernel void @private_nontemporal_load_1(
 ; GFX12-CU-NEXT:    s_load_b64 s[0:1], s[4:5], 0x8
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v0, 0
 ; GFX12-CU-NEXT:    s_mov_b32 s3, 0x3ff
-; GFX12-CU-NEXT:    s_wait_alu 0xfffe
 ; GFX12-CU-NEXT:    v_and_b32_e64 v1, v1, s3
 ; GFX12-CU-NEXT:    s_mov_b32 s3, 2
 ; GFX12-CU-NEXT:    s_wait_alu 0xfffe
@@ -818,7 +816,6 @@ define amdgpu_kernel void @private_nontemporal_store_1(
 ; GFX12-WGP-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-WGP-NEXT:    s_load_b32 s1, s[2:3], 0x0
 ; GFX12-WGP-NEXT:    s_mov_b32 s2, 0x3ff
-; GFX12-WGP-NEXT:    s_wait_alu 0xfffe
 ; GFX12-WGP-NEXT:    v_and_b32_e64 v0, v0, s2
 ; GFX12-WGP-NEXT:    s_mov_b32 s2, 2
 ; GFX12-WGP-NEXT:    s_wait_alu 0xfffe
@@ -835,7 +832,6 @@ define amdgpu_kernel void @private_nontemporal_store_1(
 ; GFX12-CU-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-CU-NEXT:    s_load_b32 s1, s[2:3], 0x0
 ; GFX12-CU-NEXT:    s_mov_b32 s2, 0x3ff
-; GFX12-CU-NEXT:    s_wait_alu 0xfffe
 ; GFX12-CU-NEXT:    v_and_b32_e64 v0, v0, s2
 ; GFX12-CU-NEXT:    s_mov_b32 s2, 2
 ; GFX12-CU-NEXT:    s_wait_alu 0xfffe
diff --git a/llvm/test/CodeGen/AMDGPU/memory-legalizer-private-volatile.ll b/llvm/test/CodeGen/AMDGPU/memory-legalizer-private-volatile.ll
index 3346a034f963f..f8fb7986938f2 100644
--- a/llvm/test/CodeGen/AMDGPU/memory-legalizer-private-volatile.ll
+++ b/llvm/test/CodeGen/AMDGPU/memory-legalizer-private-volatile.ll
@@ -304,7 +304,6 @@ define amdgpu_kernel void @private_volatile_load_1(
 ; GFX12-WGP-NEXT:    s_load_b64 s[0:1], s[4:5], 0x8
 ; GFX12-WGP-NEXT:    v_mov_b32_e32 v0, 0
 ; GFX12-WGP-NEXT:    s_mov_b32 s3, 0x3ff
-; GFX12-WGP-NEXT:    s_wait_alu 0xfffe
 ; GFX12-WGP-NEXT:    v_and_b32_e64 v1, v1, s3
 ; GFX12-WGP-NEXT:    s_mov_b32 s3, 2
 ; GFX12-WGP-NEXT:    s_wait_alu 0xfffe
@@ -324,7 +323,6 @@ define amdgpu_kernel void @private_volatile_load_1(
 ; GFX12-CU-NEXT:    s_load_b64 s[0:1], s[4:5], 0x8
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v0, 0
 ; GFX12-CU-NEXT:    s_mov_b32 s3, 0x3ff
-; GFX12-CU-NEXT:    s_wait_alu 0xfffe
 ; GFX12-CU-NEXT:    v_and_b32_e64 v1, v1, s3
 ; GFX12-CU-NEXT:    s_mov_b32 s3, 2
 ; GFX12-CU-NEXT:    s_wait_alu 0xfffe
@@ -624,7 +622,6 @@ define amdgpu_kernel void @private_volatile_store_1(
 ; GFX12-WGP-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-WGP-NEXT:    s_load_b32 s1, s[2:3], 0x0
 ; GFX12-WGP-NEXT:    s_mov_b32 s2, 0x3ff
-; GFX12-WGP-NEXT:    s_wait_alu 0xfffe
 ; GFX12-WGP-NEXT:    v_and_b32_e64 v0, v0, s2
 ; GFX12-WGP-NEXT:    s_mov_b32 s2, 2
 ; GFX12-WGP-NEXT:    s_wait_alu 0xfffe
@@ -647,7 +644,6 @@ define amdgpu_kernel void @private_volatile_store_1(
 ; GFX12-CU-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-CU-NEXT:    s_load_b32 s1, s[2:3], 0x0
 ; GFX12-CU-NEXT:    s_mov_b32 s2, 0x3ff
-; GFX12-CU-NEXT:    s_wait_alu 0xfffe
 ; GFX12-CU-NEXT:    v_and_b32_e64 v0, v0, s2
 ; GFX12-CU-NEXT:    s_mov_b32 s2, 2
 ; GFX12-CU-NEXT:    s_wait_alu 0xfffe
diff --git a/llvm/test/CodeGen/AMDGPU/memory-legalizer-store-infinite-loop.ll b/llvm/test/CodeGen/AMDGPU/memory-legalizer-store-infinite-loop.ll
index 7059e80d5c3d1..db1399cc74dc6 100644
--- a/llvm/test/CodeGen/AMDGPU/memory-legalizer-store-infinite-loop.ll
+++ b/llvm/test/CodeGen/AMDGPU/memory-legalizer-store-infinite-loop.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
 ; RUN: llc -mtriple=amdgcn--amdhsa < %s | FileCheck -check-prefix=GCN %s
 
 ; Effectively, check that the compile finishes; in the case
diff --git a/llvm/test/CodeGen/AMDGPU/min.ll b/llvm/test/CodeGen/AMDGPU/min.ll
index af7f92798a931..a6db7d331cef3 100644
--- a/llvm/test/CodeGen/AMDGPU/min.ll
+++ b/llvm/test/CodeGen/AMDGPU/min.ll
@@ -828,81 +828,30 @@ define amdgpu_kernel void @s_test_imin_sle_v2i16(ptr addrspace(1) %out, <2 x i16
 define amdgpu_kernel void @s_test_imin_sle_v4i16(ptr addrspace(1) %out, <4 x i16> %a, <4 x i16> %b) #0 {
 ; EG-LABEL: s_test_imin_sle_v4i16:
 ; EG:       ; %bb.0:
-; EG-NEXT:    ALU 1, @28, KC0[], KC1[]
-; EG-NEXT:    TEX 1 @12
-; EG-NEXT:    ALU 9, @30, KC0[], KC1[]
-; EG-NEXT:    TEX 1 @16
-; EG-NEXT:    ALU 10, @40, KC0[], KC1[]
-; EG-NEXT:    TEX 1 @20
-; EG-NEXT:    ALU 10, @51, KC0[], KC1[]
-; EG-NEXT:    TEX 1 @24
-; EG-NEXT:    ALU 11, @62, KC0[CB0:0-32], KC1[]
-; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T6.XY, T5.X, 1
+; EG-NEXT:    ALU 0, @14, KC0[], KC1[]
+; EG-NEXT:    TEX 3 @6
+; EG-NEXT:    ALU 9, @15, KC0[CB0:0-32], KC1[]
+; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T0.XY, T1.X, 1
 ; EG-NEXT:    CF_END
 ; EG-NEXT:    PAD
-; EG-NEXT:    Fetch clause starting at 12:
-; EG-NEXT:     VTX_READ_16 T6.X, T5.X, 50, #3
-; EG-NEXT:     VTX_READ_16 T7.X, T5.X, 58, #3
-; EG-NEXT:    Fetch clause starting at 16:
-; EG-NEXT:     VTX_READ_16 T6.X, T5.X, 48, #3
-; EG-NEXT:     VTX_READ_16 T7.X, T5.X, 56, #3
-; EG-NEXT:    Fetch clause starting at 20:
-; EG-NEXT:     VTX_READ_16 T6.X, T5.X, 46, #3
-; EG-NEXT:     VTX_READ_16 T7.X, T5.X, 54, #3
-; EG-NEXT:    Fetch clause starting at 24:
-; EG-NEXT:     VTX_READ_16 T6.X, T5.X, 44, #3
-; EG-NEXT:     VTX_READ_16 T5.X, T5.X, 52, #3
-; EG-NEXT:    ALU clause starting at 28:
-; EG-NEXT:     MOV * T0.Y, T3.X,
-; EG-NEXT:     MOV * T5.X, 0.0,
-; EG-NEXT:    ALU clause starting at 30:
-; EG-NEXT:     BFE_INT T0.Z, T6.X, 0.0, literal.x,
-; EG-NEXT:     BFE_INT * T0.W, T7.X, 0.0, literal.x, BS:VEC_120/SCL_212
-; EG-NEXT:    16(2.242078e-44), 0(0.000000e+00)
-; EG-NEXT:     MIN_INT * T0.W, PV.Z, PV.W,
-; EG-NEXT:     LSHL T0.W, PV.W, literal.x,
-; EG-NEXT:     AND_INT * T1.W, T0.Y, literal.y,
-; EG-NEXT:    16(2.242078e-44), 65535(9.183409e-41)
-; EG-NEXT:     OR_INT * T0.W, PS, PV.W,
-; EG-NEXT:     MOV * T3.X, PV.W,
-; EG-NEXT:     MOV * T0.Y, PV.X,
-; EG-NEXT:    ALU clause starting at 40:
-; EG-NEXT:     BFE_INT T0.Z, T6.X, 0.0, literal.x,
-; EG-NEXT:     BFE_INT * T0.W, T7.X, 0.0, literal.x, BS:VEC_120/SCL_212
-; EG-NEXT:    16(2.242078e-44), 0(0.000000e+00)
-; EG-NEXT:     MIN_INT T0.W, PV.Z, PV.W,
-; EG-NEXT:     AND_INT * T1.W, T0.Y, literal.x,
-; EG-NEXT:    -65536(nan), 0(0.000000e+00)
-; EG-NEXT:     AND_INT * T0.W, PV.W, literal.x,
-; EG-NEXT:    65535(9.183409e-41), 0(0.000000e+00)
-; EG-NEXT:     OR_INT * T0.W, T1.W, PV.W,
-; EG-NEXT:     MOV T3.X, PV.W,
-; EG-NEXT:     MOV * T0.Y, T2.X,
-; EG-NEXT:    ALU clause starting at 51:
-; EG-NEXT:     BFE_INT T0.Z, T6.X, 0.0, literal.x,
-; EG-NEXT:     BFE_INT * T0.W, T7.X, 0.0, literal.x, BS:VEC_120/SCL_212
-; EG-NEXT:    16(2.242078e-44), 0(0.000000e+00)
-; EG-NEXT:     MIN_INT T0.W, PV.Z, PV.W,
-; EG-NEXT:     AND_INT * T1.W, T0.Y, literal.x,
-; EG-NEXT:    65535(9.183409e-41), 0(0.000000e+00)
-; EG-NEXT:     LSHL * T0.W, PV.W, literal.x,
+; EG-NEXT:    Fetch clause starting at 6:
+; EG-NEXT:     VTX_READ_16 T1.X, T0.X, 46, #3
+; EG-NEXT:     VTX_READ_16 T2.X, T0.X, 52, #3
+; EG-NEXT:     VTX_READ_16 T3.X, T0.X, 44, #3
+; EG-NEXT:     VTX_READ_16 T0.X, T0.X, 54, #3
+; EG-NEXT:    ALU clause starting at 14:
+; EG-NEXT:     MOV * T0.X, 0.0,
+; EG-NEXT:    ALU clause starting at 15:
+; EG-NEXT:     BFE_INT T0.Z, T1.X, 0.0, literal.x,
+; EG-NEXT:     BFE_INT * T0.W, T0.X, 0.0, literal.x, BS:VEC_120/SCL_212
 ; EG-NEXT:    16(2.242078e-44), 0(0.000000e+00)
-; EG-NEXT:     OR_INT * T0.W, T1.W, PV.W,
-; EG-NEXT:     MOV * T2.X, PV.W,
-; EG-NEXT:     MOV * T0.Y, PV.X,
-; EG-NEXT:    ALU clause starting at 62:
-; EG-NEXT:     BFE_INT T0.Z, T6.X, 0.0, literal.x,
-; EG-NEXT:     BFE_INT * T0.W, T5.X, 0.0, literal.x, BS:VEC_120/SCL_212
+; EG-NEXT:     MIN_INT T0.Y, PV.Z, PV.W,
+; EG-NEXT:     BFE_INT T0.Z, T3.X, 0.0, literal.x,
+; EG-NEXT:     BFE_INT * T0.W, T2.X, 0.0, literal.x, BS:VEC_120/SCL_212
 ; EG-NEXT:    16(2.242078e-44), 0(0.000000e+00)
-; EG-NEXT:     MIN_INT * T0.W, PV.Z, PV.W,
-; EG-NEXT:     LSHR T5.X, KC0[2].Y, literal.x,
-; EG-NEXT:     AND_INT T1.W, T0.Y, literal.y,
-; EG-NEXT:     AND_INT * T0.W, PV.W, literal.z,
-; EG-NEXT:    2(2.802597e-45), -65536(nan)
-; EG-NEXT:    65535(9.183409e-41), 0(0.000000e+00)
-; EG-NEXT:     OR_INT * T6.X, PV.W, PS,
-; EG-NEXT:     MOV T2.X, PV.X,
-; EG-NEXT:     MOV * T6.Y, T3.X,
+; EG-NEXT:     MIN_INT T0.X, PV.Z, PV.W,
+; EG-NEXT:     LSHR * T1.X, KC0[2].Y, literal.x,
+; EG-NEXT:    2(2.802597e-45), 0(0.000000e+00)
 ;
 ; CI-LABEL: s_test_imin_sle_v4i16:
 ; CI:       ; %bb.0:
@@ -1848,49 +1797,40 @@ define amdgpu_kernel void @v_test_umin_ule_v3i32(ptr addrspace(1) %out, ptr addr
 define amdgpu_kernel void @v_test_umin_ule_v3i16(ptr addrspace(1) %out, ptr addrspace(1) %a.ptr, ptr addrspace(1) %b.ptr) #0 {
 ; EG-LABEL: v_test_umin_ule_v3i16:
 ; EG:       ; %bb.0:
-; EG-NEXT:    ALU 3, @20, KC0[CB0:0-32], KC1[]
-; EG-NEXT:    TEX 1 @8
-; EG-NEXT:    ALU 11, @24, KC0[CB0:0-32], KC1[]
-; EG-NEXT:    TEX 3 @12
-; EG-NEXT:    ALU 8, @36, KC0[], KC1[]
-; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T6.X, T8.X, 0
-; EG-NEXT:    MEM_RAT MSKOR T7.XW, T0.X
+; EG-NEXT:    ALU 3, @14, KC0[CB0:0-32], KC1[]
+; EG-NEXT:    TEX 3 @6
+; EG-NEXT:    ALU 17, @18, KC0[CB0:0-32], KC1[]
+; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T1.X, T2.X, 0
+; EG-NEXT:    MEM_RAT MSKOR T4.XW, T0.X
 ; EG-NEXT:    CF_END
-; EG-NEXT:    Fetch clause starting at 8:
-; EG-NEXT:     VTX_READ_16 T7.X, T6.X, 4, #1
-; EG-NEXT:     VTX_READ_16 T8.X, T0.X, 4, #1
-; EG-NEXT:    Fetch clause starting at 12:
-; EG-NEXT:     VTX_READ_16 T8.X, T6.X, 0, #1
-; EG-NEXT:     VTX_READ_16 T9.X, T0.X, 0, #1
-; EG-NEXT:     VTX_READ_16 T6.X, T6.X, 2, #1
-; EG-NEXT:     VTX_READ_16 T0.X, T0.X, 2, #1
-; EG-NEXT:    ALU clause starting at 20:
+; EG-NEXT:    Fetch clause starting at 6:
+; EG-NEXT:     VTX_READ_16 T2.X, T1.X, 0, #1
+; EG-NEXT:     VTX_READ_16 T3.X, T0.X, 0, #1
+; EG-NEXT:     VTX_READ_16 T1.X, T1.X, 4, #1
+; EG-NEXT:     VTX_READ_16 T0.X, T0.X, 4, #1
+; EG-NEXT:    ALU clause starting at 14:
 ; EG-NEXT:     LSHL * T0.W, T0.X, literal.x,
 ; EG-NEXT:    3(4.203895e-45), 0(0.000000e+00)
 ; EG-NEXT:     ADD_INT T0.X, KC0[2].Z, PV.W,
-; EG-NEXT:     ADD_INT * T6.X, KC0[2].W, PV.W,
-; EG-NEXT:    ALU clause starting at 24:
+; EG-NEXT:     ADD_INT * T1.X, KC0[2].W, PV.W,
+; EG-NEXT:    ALU clause starting at 18:
 ; EG-NEXT:     ADD_INT * T0.W, KC0[2].Y, T0.W,
 ; EG-NEXT:     ADD_INT * T1.W, PV.W, literal.x,
 ; EG-NEXT:    4(5.605194e-45), 0(0.000000e+00)
 ; EG-NEXT:     AND_INT * T2.W, PV.W, literal.x,
 ; EG-NEXT:    3(4.203895e-45), 0(0.000000e+00)
 ; EG-NEXT:     LSHL T2.W, PV.W, literal.x,
-; EG-NEXT:     MIN_UINT * T3.W, T8.X, T7.X,
+; EG-NEXT:     MIN_UINT * T3.W, T0.X, T1.X,
 ; EG-NEXT:    3(4.203895e-45), 0(0.000000e+00)
-; EG-NEXT:     LSHL T7.X, PS, PV.W,
-; EG-NEXT:     LSHL * T7.W, literal.x, PV.W,
+; EG-NEXT:     LSHL T4.X, PS, PV.W,
+; EG-NEXT:     LSHL * T4.W, literal.x, PV.W,
 ; EG-NEXT:    65535(9.183409e-41), 0(0.000000e+00)
-; EG-NEXT:     MOV * T7.Y, 0.0,
-; EG-NEXT:    ALU clause starting at 36:
-; EG-NEXT:     MOV T7.Z, 0.0,
-; EG-NEXT:     MIN_UINT * T2.W, T0.X, T6.X,
+; EG-NEXT:     MOV T4.Y, 0.0,
+; EG-NEXT:     MOV * T4.Z, 0.0,
 ; EG-NEXT:     LSHR T0.X, T1.W, literal.x,
-; EG-NEXT:     LSHL T1.W, PV.W, literal.y,
-; EG-NEXT:     MIN_UINT * T2.W, T9.X, T8.X,
-; EG-NEXT:    2(2.802597e-45), 16(2.242078e-44)
-; EG-NEXT:     OR_INT T6.X, PV.W, PS,
-; EG-NEXT:     LSHR * T8.X, T0.W, literal.x,
+; EG-NEXT:     MIN_UINT * T1.X, T3.X, T2.X,
+; EG-NEXT:    2(2.802597e-45), 0(0.000000e+00)
+; EG-NEXT:     LSHR * T2.X, T0.W, literal.x,
 ; EG-NEXT:    2(2.802597e-45), 0(0.000000e+00)
 ;
 ; CI-LABEL: v_test_umin_ule_v3i16:
@@ -2936,142 +2876,46 @@ define amdgpu_kernel void @s_test_umin_ult_v8i32(ptr addrspace(1) %out, <8 x i32
 define amdgpu_kernel void @s_test_umin_ult_v8i16(ptr addrspace(1) %out, <8 x i16> %a, <8 x i16> %b) #0 {
 ; EG-LABEL: s_test_umin_ult_v8i16:
 ; EG:       ; %bb.0:
-; EG-NEXT:    ALU 1, @52, KC0[], KC1[]
-; EG-NEXT:    TEX 1 @20
-; EG-NEXT:    ALU 9, @54, KC0[], KC1[]
-; EG-NEXT:    TEX 1 @24
-; EG-NEXT:    ALU 8, @64, KC0[], KC1[]
-; EG-NEXT:    TEX 1 @28
-; EG-NEXT:    ALU 10, @73, KC0[], KC1[]
-; EG-NEXT:    TEX 1 @32
-; EG-NEXT:    ALU 8, @84, KC0[], KC1[]
-; EG-NEXT:    TEX 1 @36
-; EG-NEXT:    ALU 10, @93, KC0[], KC1[]
-; EG-NEXT:    TEX 1 @40
-; EG-NEXT:    ALU 8, @104, KC0[], KC1[]
-; EG-NEXT:    TEX 1 @44
-; EG-NEXT:    ALU 10, @113, KC0[], KC1[]
-; EG-NEXT:    TEX 1 @48
-; EG-NEXT:    ALU 10, @124, KC0[CB0:0-32], KC1[]
-; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T7.XYZW, T8.X, 1
+; EG-NEXT:    ALU 0, @24, KC0[], KC1[]
+; EG-NEXT:    TEX 2 @8
+; EG-NEXT:    ALU 2, @25, KC0[], KC1[]
+; EG-NEXT:    TEX 4 @14
+; EG-NEXT:    ALU 14, @28, KC0[CB0:0-32], KC1[]
+; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T0.XYZW, T1.X, 1
 ; EG-NEXT:    CF_END
 ; EG-NEXT:    PAD
-; EG-NEXT:    Fetch clause starting at 20:
-; EG-NEXT:     VTX_READ_16 T8.X, T7.X, 66, #3
-; EG-NEXT:     VTX_READ_16 T9.X, T7.X, 82, #3
-; EG-NEXT:    Fetch clause starting at 24:
-; EG-NEXT:     VTX_READ_16 T8.X, T7.X, 64, #3
-; EG-NEXT:     VTX_READ_16 T9.X, T7.X, 80, #3
-; EG-NEXT:    Fetch clause starting at 28:
-; EG-NEXT:     VTX_READ_16 T8.X, T7.X, 62, #3
-; EG-NEXT:     VTX_READ_16 T9.X, T7.X, 78, #3
-; EG-NEXT:    Fetch clause starting at 32:
-; EG-NEXT:     VTX_READ_16 T8.X, T7.X, 60, #3
-; EG-NEXT:     VTX_READ_16 T9.X, T7.X, 76, #3
-; EG-NEXT:    Fetch clause starting at 36:
-; EG-NEXT:     VTX_READ_16 T8.X, T7.X, 58, #3
-; EG-NEXT:     VTX_READ_16 T9.X, T7.X, 74, #3
-; EG-NEXT:    Fetch clause starting at 40:
-; EG-NEXT:     VTX_READ_16 T8.X, T7.X, 56, #3
-; EG-NEXT:     VTX_READ_16 T9.X, T7.X, 72, #3
-; EG-NEXT:    Fetch clause starting at 44:
-; EG-NEXT:     VTX_READ_16 T8.X, T7.X, 54, #3
-; EG-NEXT:     VTX_READ_16 T9.X, T7.X, 70, #3
-; EG-NEXT:    Fetch clause starting at 48:
-; EG-NEXT:     VTX_READ_16 T8.X, T7.X, 52, #3
-; EG-NEXT:     VTX_READ_16 T7.X, T7.X, 68, #3
-; EG-NEXT:    ALU clause starting at 52:
-; EG-NEXT:     MOV * T0.Y, T3.X,
-; EG-NEXT:     MOV * T7.X, 0.0,
-; EG-NEXT:    ALU clause starting at 54:
-; EG-NEXT:     AND_INT T0.W, T8.X, literal.x,
-; EG-NEXT:     AND_INT * T1.W, T9.X, literal.x,
-; EG-NEXT:    65535(9.183409e-41), 0(0.000000e+00)
-; EG-NEXT:     MIN_UINT * T0.W, PV.W, PS,
-; EG-NEXT:     LSHL T0.W, PV.W, literal.x,
-; EG-NEXT:     AND_INT * T1.W, T0.Y, literal.y,
-; EG-NEXT:    16(2.242078e-44), 65535(9.183409e-41)
-; EG-NEXT:     OR_INT * T0.W, PS, PV.W,
-; EG-NEXT:     MOV * T3.X, PV.W,
-; EG-NEXT:     MOV * T0.Y, PV.X,
-; EG-NEXT:    ALU clause starting at 64:
-; EG-NEXT:     AND_INT T0.W, T8.X, literal.x,
-; EG-NEXT:     AND_INT * T1.W, T9.X, literal.x,
-; EG-NEXT:    65535(9.183409e-41), 0(0.000000e+00)
-; EG-NEXT:     AND_INT T2.W, T0.Y, literal.x,
-; EG-NEXT:     MIN_UINT * T0.W, PV.W, PS,
-; EG-NEXT:    -65536(nan), 0(0.000000e+00)
-; EG-NEXT:     OR_INT * T0.W, PV.W, PS,
-; EG-NEXT:     MOV T3.X, PV.W,
-; EG-NEXT:     MOV * T0.Y, T2.X,
-; EG-NEXT:    ALU clause starting at 73:
-; EG-NEXT:     AND_INT T0.W, T8.X, literal.x,
-; EG-NEXT:     AND_INT * T1.W, T9.X, literal.x,
-; EG-NEXT:    65535(9.183409e-41), 0(0.000000e+00)
-; EG-NEXT:     MIN_UINT T0.W, PV.W, PS,
-; EG-NEXT:     AND_INT * T1.W, T0.Y, literal.x,
-; EG-NEXT:    65535(9.183409e-41), 0(0.000000e+00)
-; EG-NEXT:     LSHL * T0.W, PV.W, literal.x,
-; EG-NEXT:    16(2.242078e-44), 0(0.000000e+00)
-; EG-NEXT:     OR_INT * T0.W, T1.W, PV.W,
-; EG-NEXT:     MOV * T2.X, PV.W,
-; EG-NEXT:     MOV * T0.Y, PV.X,
-; EG-NEXT:    ALU clause starting at 84:
-; EG-NEXT:     AND_INT T0.W, T8.X, literal.x,
-; EG-NEXT:     AND_INT * T1.W, T9.X, literal.x,
-; EG-NEXT:    65535(9.183409e-41), 0(0.000000e+00)
-; EG-NEXT:     AND_INT T2.W, T0.Y, literal.x,
-; EG-NEXT:     MIN_UINT * T0.W, PV.W, PS,
-; EG-NEXT:    -65536(nan), 0(0.000000e+00)
-; EG-NEXT:     OR_INT * T7.Z, PV.W, PS,
-; EG-NEXT:     MOV T2.X, PV.Z,
-; EG-NEXT:     MOV * T0.Y, T5.X,
-; EG-NEXT:    ALU clause starting at 93:
-; EG-NEXT:     AND_INT T0.W, T8.X, literal.x,
-; EG-NEXT:     AND_INT * T1.W, T9.X, literal.x,
-; EG-NEXT:    65535(9.183409e-41), 0(0.000000e+00)
-; EG-NEXT:     MIN_UINT T0.W, PV.W, PS,
-; EG-NEXT:     AND_INT * T1.W, T0.Y, literal.x,
-; EG-NEXT:    65535(9.183409e-41), 0(0.000000e+00)
-; EG-NEXT:     LSHL * T0.W, PV.W, literal.x,
-; EG-NEXT:    16(2.242078e-44), 0(0.000000e+00)
-; EG-NEXT:     OR_INT * T0.W, T1.W, PV.W,
-; EG-NEXT:     MOV * T5.X, PV.W,
-; EG-NEXT:     MOV * T0.Y, PV.X,
-; EG-NEXT:    ALU clause starting at 104:
-; EG-NEXT:     AND_INT T0.W, T8.X, literal.x,
-; EG-NEXT:     AND_INT * T1.W, T9.X, literal.x,
+; EG-NEXT:    Fetch clause starting at 8:
+; EG-NEXT:     VTX_READ_16 T1.X, T0.X, 62, #3
+; EG-NEXT:     VTX_READ_16 T2.X, T0.X, 60, #3
+; EG-NEXT:     VTX_READ_16 T3.X, T0.X, 78, #3
+; EG-NEXT:    Fetch clause starting at 14:
+; EG-NEXT:     VTX_READ_16 T1.X, T0.X, 68, #3
+; EG-NEXT:     VTX_READ_16 T3.X, T0.X, 52, #3
+; EG-NEXT:     VTX_READ_16 T4.X, T0.X, 70, #3
+; EG-NEXT:     VTX_READ_16 T5.X, T0.X, 54, #3
+; EG-NEXT:     VTX_READ_16 T0.X, T0.X, 76, #3
+; EG-NEXT:    ALU clause starting at 24:
+; EG-NEXT:     MOV * T0.X, 0.0,
+; EG-NEXT:    ALU clause starting at 25:
+; EG-NEXT:     AND_INT T0.W, T1.X, literal.x,
+; EG-NEXT:     AND_INT * T1.W, T3.X, literal.x,
 ; EG-NEXT:    65535(9.183409e-41), 0(0.000000e+00)
-; EG-NEXT:     AND_INT T2.W, T0.Y, literal.x,
-; EG-NEXT:     MIN_UINT * T0.W, PV.W, PS,
-; EG-NEXT:    -65536(nan), 0(0.000000e+00)
-; EG-NEXT:     OR_INT * T0.W, PV.W, PS,
-; EG-NEXT:     MOV T5.X, PV.W,
-; EG-NEXT:     MOV * T0.Y, T4.X,
-; EG-NEXT:    ALU clause starting at 113:
-; EG-NEXT:     AND_INT T0.W, T8.X, literal.x,
-; EG-NEXT:     AND_INT * T1.W, T9.X, literal.x,
+; EG-NEXT:    ALU clause starting at 28:
+; EG-NEXT:     AND_INT T0.Z, T2.X, literal.x,
+; EG-NEXT:     AND_INT T2.W, T0.X, literal.x, BS:VEC_120/SCL_212
+; EG-NEXT:     MIN_UINT * T0.W, T0.W, T1.W,
 ; EG-NEXT:    65535(9.183409e-41), 0(0.000000e+00)
-; EG-NEXT:     MIN_UINT T0.W, PV.W, PS,
-; EG-NEXT:     AND_INT * T1.W, T0.Y, literal.x,
+; EG-NEXT:     MIN_UINT T0.Z, PV.Z, PV.W,
+; EG-NEXT:     AND_INT T1.W, T5.X, literal.x,
+; EG-NEXT:     AND_INT * T2.W, T4.X, literal.x,
 ; EG-NEXT:    65535(9.183409e-41), 0(0.000000e+00)
-; EG-NEXT:     LSHL * T0.W, PV.W, literal.x,
-; EG-NEXT:    16(2.242078e-44), 0(0.000000e+00)
-; EG-NEXT:     OR_INT * T0.W, T1.W, PV.W,
-; EG-NEXT:     MOV * T4.X, PV.W,
-; EG-NEXT:     MOV * T0.Y, PV.X,
-; EG-NEXT:    ALU clause starting at 124:
-; EG-NEXT:     AND_INT T0.W, T8.X, literal.x,
-; EG-NEXT:     AND_INT * T1.W, T7.X, literal.x,
+; EG-NEXT:     MIN_UINT T0.Y, PV.W, PS,
+; EG-NEXT:     AND_INT T1.W, T3.X, literal.x,
+; EG-NEXT:     AND_INT * T2.W, T1.X, literal.x,
 ; EG-NEXT:    65535(9.183409e-41), 0(0.000000e+00)
-; EG-NEXT:     LSHR T8.X, KC0[2].Y, literal.x,
-; EG-NEXT:     AND_INT T2.W, T0.Y, literal.y,
-; EG-NEXT:     MIN_UINT * T0.W, PV.W, PS,
-; EG-NEXT:    2(2.802597e-45), -65536(nan)
-; EG-NEXT:     OR_INT * T7.X, PV.W, PS,
-; EG-NEXT:     MOV T4.X, PV.X,
-; EG-NEXT:     MOV * T7.W, T3.X,
-; EG-NEXT:     MOV * T7.Y, T5.X,
+; EG-NEXT:     MIN_UINT T0.X, PV.W, PS,
+; EG-NEXT:     LSHR * T1.X, KC0[2].Y, literal.x,
+; EG-NEXT:    2(2.802597e-45), 0(0.000000e+00)
 ;
 ; CI-LABEL: s_test_umin_ult_v8i16:
 ; CI:       ; %bb.0:
diff --git a/llvm/test/CodeGen/AMDGPU/minimummaximum.ll b/llvm/test/CodeGen/AMDGPU/minimummaximum.ll
index c375b16ee3809..7e867a5372986 100644
--- a/llvm/test/CodeGen/AMDGPU/minimummaximum.ll
+++ b/llvm/test/CodeGen/AMDGPU/minimummaximum.ll
@@ -1,6 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=amdgcn -mcpu=gfx1200 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX12,SDAG %s
-; RUN: llc -global-isel -mtriple=amdgcn -mcpu=gfx1200 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX12,GISEL %s
+; RUN: llc -mtriple=amdgcn -mcpu=gfx1200 -mattr=+real-true16 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX12,SDAG,SDAG-TRUE16 %s
+; RUN: llc -mtriple=amdgcn -mcpu=gfx1200 -mattr=-real-true16 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX12,SDAG,SDAG-FAKE16 %s
+; RUN: llc -global-isel -mtriple=amdgcn -mcpu=gfx1200 -mattr=+real-true16 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX12,GISEL,GISEL-TRUE16 %s
+; RUN: llc -global-isel -mtriple=amdgcn -mcpu=gfx1200 -mattr=-real-true16 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX12,GISEL,GISEL-FAKE16 %s
 
 define amdgpu_ps float @test_minmax_f32(float %a, float %b, float %c) {
 ; GFX12-LABEL: test_minmax_f32:
@@ -72,30 +74,84 @@ define amdgpu_ps float @test_maxmin_commuted_f32(float %a, float %b, float %c) {
 }
 
 define amdgpu_ps half @test_minmax_f16(half %a, half %b, half %c) {
-; GFX12-LABEL: test_minmax_f16:
-; GFX12:       ; %bb.0:
-; GFX12-NEXT:    v_maximumminimum_f16 v0, v0, v1, v2
-; GFX12-NEXT:    ; return to shader part epilog
+; SDAG-TRUE16-LABEL: test_minmax_f16:
+; SDAG-TRUE16:       ; %bb.0:
+; SDAG-TRUE16-NEXT:    v_mov_b16_e32 v0.h, v1.l
+; SDAG-TRUE16-NEXT:    v_mov_b16_e32 v1.l, v2.l
+; SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; SDAG-TRUE16-NEXT:    v_maximumminimum_f16 v0.l, v0.l, v0.h, v1.l
+; SDAG-TRUE16-NEXT:    ; return to shader part epilog
+;
+; SDAG-FAKE16-LABEL: test_minmax_f16:
+; SDAG-FAKE16:       ; %bb.0:
+; SDAG-FAKE16-NEXT:    v_maximumminimum_f16 v0, v0, v1, v2
+; SDAG-FAKE16-NEXT:    ; return to shader part epilog
+;
+; GISEL-TRUE16-LABEL: test_minmax_f16:
+; GISEL-TRUE16:       ; %bb.0:
+; GISEL-TRUE16-NEXT:    v_maximumminimum_f16 v0.l, v0.l, v1.l, v2.l
+; GISEL-TRUE16-NEXT:    ; return to shader part epilog
+;
+; GISEL-FAKE16-LABEL: test_minmax_f16:
+; GISEL-FAKE16:       ; %bb.0:
+; GISEL-FAKE16-NEXT:    v_maximumminimum_f16 v0, v0, v1, v2
+; GISEL-FAKE16-NEXT:    ; return to shader part epilog
   %max = call half @llvm.maximum.f16(half %a, half %b)
   %minmax = call half @llvm.minimum.f16(half %max, half %c)
   ret half %minmax
 }
 
 define amdgpu_ps half @test_minmax_commuted_f16(half %a, half %b, half %c) {
-; GFX12-LABEL: test_minmax_commuted_f16:
-; GFX12:       ; %bb.0:
-; GFX12-NEXT:    v_maximumminimum_f16 v0, v0, v1, v2
-; GFX12-NEXT:    ; return to shader part epilog
+; SDAG-TRUE16-LABEL: test_minmax_commuted_f16:
+; SDAG-TRUE16:       ; %bb.0:
+; SDAG-TRUE16-NEXT:    v_mov_b16_e32 v0.h, v1.l
+; SDAG-TRUE16-NEXT:    v_mov_b16_e32 v1.l, v2.l
+; SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; SDAG-TRUE16-NEXT:    v_maximumminimum_f16 v0.l, v0.l, v0.h, v1.l
+; SDAG-TRUE16-NEXT:    ; return to shader part epilog
+;
+; SDAG-FAKE16-LABEL: test_minmax_commuted_f16:
+; SDAG-FAKE16:       ; %bb.0:
+; SDAG-FAKE16-NEXT:    v_maximumminimum_f16 v0, v0, v1, v2
+; SDAG-FAKE16-NEXT:    ; return to shader part epilog
+;
+; GISEL-TRUE16-LABEL: test_minmax_commuted_f16:
+; GISEL-TRUE16:       ; %bb.0:
+; GISEL-TRUE16-NEXT:    v_maximumminimum_f16 v0.l, v0.l, v1.l, v2.l
+; GISEL-TRUE16-NEXT:    ; return to shader part epilog
+;
+; GISEL-FAKE16-LABEL: test_minmax_commuted_f16:
+; GISEL-FAKE16:       ; %bb.0:
+; GISEL-FAKE16-NEXT:    v_maximumminimum_f16 v0, v0, v1, v2
+; GISEL-FAKE16-NEXT:    ; return to shader part epilog
   %max = call half @llvm.maximum.f16(half %a, half %b)
   %minmax = call half @llvm.minimum.f16(half %c, half %max)
   ret half %minmax
 }
 
 define amdgpu_ps half @test_maxmin_commuted_f16(half %a, half %b, half %c) {
-; GFX12-LABEL: test_maxmin_commuted_f16:
-; GFX12:       ; %bb.0:
-; GFX12-NEXT:    v_minimummaximum_f16 v0, v0, v1, v2
-; GFX12-NEXT:    ; return to shader part epilog
+; SDAG-TRUE16-LABEL: test_maxmin_commuted_f16:
+; SDAG-TRUE16:       ; %bb.0:
+; SDAG-TRUE16-NEXT:    v_mov_b16_e32 v0.h, v1.l
+; SDAG-TRUE16-NEXT:    v_mov_b16_e32 v1.l, v2.l
+; SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; SDAG-TRUE16-NEXT:    v_minimummaximum_f16 v0.l, v0.l, v0.h, v1.l
+; SDAG-TRUE16-NEXT:    ; return to shader part epilog
+;
+; SDAG-FAKE16-LABEL: test_maxmin_commuted_f16:
+; SDAG-FAKE16:       ; %bb.0:
+; SDAG-FAKE16-NEXT:    v_minimummaximum_f16 v0, v0, v1, v2
+; SDAG-FAKE16-NEXT:    ; return to shader part epilog
+;
+; GISEL-TRUE16-LABEL: test_maxmin_commuted_f16:
+; GISEL-TRUE16:       ; %bb.0:
+; GISEL-TRUE16-NEXT:    v_minimummaximum_f16 v0.l, v0.l, v1.l, v2.l
+; GISEL-TRUE16-NEXT:    ; return to shader part epilog
+;
+; GISEL-FAKE16-LABEL: test_maxmin_commuted_f16:
+; GISEL-FAKE16:       ; %bb.0:
+; GISEL-FAKE16-NEXT:    v_minimummaximum_f16 v0, v0, v1, v2
+; GISEL-FAKE16-NEXT:    ; return to shader part epilog
   %min = call half @llvm.minimum.f16(half %a, half %b)
   %maxmin = call half @llvm.maximum.f16(half %c, half %min)
   ret half %maxmin
diff --git a/llvm/test/CodeGen/AMDGPU/minimumnum.ll b/llvm/test/CodeGen/AMDGPU/minimumnum.ll
index f5fb85d63b8e4..a32b3b71cd606 100644
--- a/llvm/test/CodeGen/AMDGPU/minimumnum.ll
+++ b/llvm/test/CodeGen/AMDGPU/minimumnum.ll
@@ -280,17 +280,20 @@ define bfloat @v_minimumnum_bf16(bfloat %x, bfloat %y) {
 ; GFX12-NEXT:    s_wait_bvhcnt 0x0
 ; GFX12-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-NEXT:    v_lshlrev_b32_e32 v2, 16, v0
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)
 ; GFX12-NEXT:    v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc_lo
 ; GFX12-NEXT:    v_lshlrev_b32_e32 v3, 16, v1
 ; GFX12-NEXT:    v_lshlrev_b32_e32 v2, 16, v0
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_cndmask_b32_e32 v1, v1, v0, vcc_lo
 ; GFX12-NEXT:    v_lshlrev_b32_e32 v3, 16, v1
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v2, v3
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_cndmask_b32_e32 v2, v1, v0, vcc_lo
 ; GFX12-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
@@ -298,19 +301,22 @@ define bfloat @v_minimumnum_bf16(bfloat %x, bfloat %y) {
 ; GFX12-NEXT:    v_bfe_u32 v3, v2, 16, 1
 ; GFX12-NEXT:    v_or_b32_e32 v4, 0x400000, v2
 ; GFX12-NEXT:    v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_add3_u32 v3, v3, v2, 0x7fff
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_cndmask_b32_e32 v2, v3, v4, vcc_lo
 ; GFX12-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v0
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_2)
 ; GFX12-NEXT:    v_lshrrev_b32_e32 v3, 16, v2
 ; GFX12-NEXT:    v_and_b32_e32 v2, 0xffff0000, v2
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc_lo
 ; GFX12-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v1
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX12-NEXT:    s_wait_alu 0xfffd
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_2)
 ; GFX12-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc_lo
 ; GFX12-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v2
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2)
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc_lo
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
   %result = call bfloat @llvm.minimumnum.bf16(bfloat %x, bfloat %y)
@@ -395,15 +401,19 @@ define bfloat @v_minimumnum_bf16_nnan(bfloat %x, bfloat %y) {
 ; GFX12-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-NEXT:    v_lshlrev_b32_e32 v2, 16, v1
 ; GFX12-NEXT:    v_lshlrev_b32_e32 v3, 16, v0
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_4) | instid1(VALU_DEP_2)
 ; GFX12-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v3, v2
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_cndmask_b32_e32 v2, v1, v0, vcc_lo
 ; GFX12-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v0
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_dual_cndmask_b32 v0, v2, v0 :: v_dual_lshlrev_b32 v3, 16, v2
 ; GFX12-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v1
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
 ; GFX12-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc_lo
 ; GFX12-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v3
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2)
 ; GFX12-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc_lo
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
diff --git a/llvm/test/CodeGen/AMDGPU/minmax.ll b/llvm/test/CodeGen/AMDGPU/minmax.ll
index 774a22fb907db..954dab3d0fc6f 100644
--- a/llvm/test/CodeGen/AMDGPU/minmax.ll
+++ b/llvm/test/CodeGen/AMDGPU/minmax.ll
@@ -1,8 +1,12 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -global-isel=0 -mtriple=amdgcn -mcpu=gfx1100 -verify-machineinstrs -amdgpu-enable-delay-alu=0 < %s | FileCheck -check-prefixes=GFX11,SDAG,SDAG-GFX11 %s
-; RUN: llc -global-isel=1 -mtriple=amdgcn -mcpu=gfx1100 -verify-machineinstrs -amdgpu-enable-delay-alu=0 < %s | FileCheck -check-prefixes=GFX11,GISEL,GISEL-GFX11 %s
-; RUN: llc -global-isel=0 -mtriple=amdgcn -mcpu=gfx1200 -verify-machineinstrs -amdgpu-enable-delay-alu=0 < %s | FileCheck -check-prefixes=GFX12,SDAG,SDAG-GFX12 %s
-; RUN: llc -global-isel=1 -mtriple=amdgcn -mcpu=gfx1200 -verify-machineinstrs -amdgpu-enable-delay-alu=0 < %s | FileCheck -check-prefixes=GFX12,GISEL,GISEL-GFX12 %s
+; RUN: llc -global-isel=0 -mtriple=amdgcn -mcpu=gfx1100 -mattr=+real-true16 -verify-machineinstrs -amdgpu-enable-delay-alu=0 < %s | FileCheck -check-prefixes=GFX11,SDAG,SDAG-GFX11,SDAG-GFX11-TRUE16 %s
+; RUN: llc -global-isel=0 -mtriple=amdgcn -mcpu=gfx1100 -mattr=-real-true16 -verify-machineinstrs -amdgpu-enable-delay-alu=0 < %s | FileCheck -check-prefixes=GFX11,SDAG,SDAG-GFX11,SDAG-GFX11-FAKE16 %s
+; RUN: llc -global-isel=1 -mtriple=amdgcn -mcpu=gfx1100 -mattr=+real-true16 -verify-machineinstrs -amdgpu-enable-delay-alu=0 < %s | FileCheck -check-prefixes=GFX11,GISEL,GISEL-GFX11,GISEL-GFX11-TRUE16 %s
+; RUN: llc -global-isel=1 -mtriple=amdgcn -mcpu=gfx1100 -mattr=-real-true16 -verify-machineinstrs -amdgpu-enable-delay-alu=0 < %s | FileCheck -check-prefixes=GFX11,GISEL,GISEL-GFX11,GISEL-GFX11-FAKE16 %s
+; RUN: llc -global-isel=0 -mtriple=amdgcn -mcpu=gfx1200 -mattr=+real-true16 -verify-machineinstrs -amdgpu-enable-delay-alu=0 < %s | FileCheck -check-prefixes=GFX12,SDAG,SDAG-GFX12,SDAG-GFX12-TRUE16 %s
+; RUN: llc -global-isel=0 -mtriple=amdgcn -mcpu=gfx1200 -mattr=-real-true16 -verify-machineinstrs -amdgpu-enable-delay-alu=0 < %s | FileCheck -check-prefixes=GFX12,SDAG,SDAG-GFX12,SDAG-GFX12-FAKE16 %s
+; RUN: llc -global-isel=1 -mtriple=amdgcn -mcpu=gfx1200 -mattr=+real-true16 -verify-machineinstrs -amdgpu-enable-delay-alu=0 < %s | FileCheck -check-prefixes=GFX12,GISEL,GISEL-GFX12,GISEL-GFX12-TRUE16 %s
+; RUN: llc -global-isel=1 -mtriple=amdgcn -mcpu=gfx1200 -mattr=-real-true16 -verify-machineinstrs -amdgpu-enable-delay-alu=0 < %s | FileCheck -check-prefixes=GFX12,GISEL,GISEL-GFX12,GISEL-GFX12-FAKE16 %s
 
 define i32 @test_minmax_i32(i32 %a, i32 %b, i32 %c) {
 ; GFX11-LABEL: test_minmax_i32:
@@ -467,47 +471,111 @@ define void @test_med3_f32(ptr addrspace(1) %arg, float %x, float %y, float %z)
 }
 
 define amdgpu_ps half @test_minmax_f16_ieee_false(half %a, half %b, half %c) {
-; GFX11-LABEL: test_minmax_f16_ieee_false:
-; GFX11:       ; %bb.0:
-; GFX11-NEXT:    v_maxmin_f16 v0, v0, v1, v2
-; GFX11-NEXT:    ; return to shader part epilog
-;
-; GFX12-LABEL: test_minmax_f16_ieee_false:
-; GFX12:       ; %bb.0:
-; GFX12-NEXT:    v_maxmin_num_f16 v0, v0, v1, v2
-; GFX12-NEXT:    ; return to shader part epilog
+; SDAG-GFX11-TRUE16-LABEL: test_minmax_f16_ieee_false:
+; SDAG-GFX11-TRUE16:       ; %bb.0:
+; SDAG-GFX11-TRUE16-NEXT:    v_mov_b16_e32 v0.h, v1.l
+; SDAG-GFX11-TRUE16-NEXT:    v_mov_b16_e32 v1.l, v2.l
+; SDAG-GFX11-TRUE16-NEXT:    v_maxmin_f16 v0.l, v0.l, v0.h, v1.l
+; SDAG-GFX11-TRUE16-NEXT:    ; return to shader part epilog
+;
+; SDAG-GFX11-FAKE16-LABEL: test_minmax_f16_ieee_false:
+; SDAG-GFX11-FAKE16:       ; %bb.0:
+; SDAG-GFX11-FAKE16-NEXT:    v_maxmin_f16 v0, v0, v1, v2
+; SDAG-GFX11-FAKE16-NEXT:    ; return to shader part epilog
+;
+; GISEL-GFX11-TRUE16-LABEL: test_minmax_f16_ieee_false:
+; GISEL-GFX11-TRUE16:       ; %bb.0:
+; GISEL-GFX11-TRUE16-NEXT:    v_maxmin_f16 v0.l, v0.l, v1.l, v2.l
+; GISEL-GFX11-TRUE16-NEXT:    ; return to shader part epilog
+;
+; GISEL-GFX11-FAKE16-LABEL: test_minmax_f16_ieee_false:
+; GISEL-GFX11-FAKE16:       ; %bb.0:
+; GISEL-GFX11-FAKE16-NEXT:    v_maxmin_f16 v0, v0, v1, v2
+; GISEL-GFX11-FAKE16-NEXT:    ; return to shader part epilog
+;
+; SDAG-GFX12-TRUE16-LABEL: test_minmax_f16_ieee_false:
+; SDAG-GFX12-TRUE16:       ; %bb.0:
+; SDAG-GFX12-TRUE16-NEXT:    v_mov_b16_e32 v0.h, v1.l
+; SDAG-GFX12-TRUE16-NEXT:    v_mov_b16_e32 v1.l, v2.l
+; SDAG-GFX12-TRUE16-NEXT:    v_maxmin_num_f16 v0.l, v0.l, v0.h, v1.l
+; SDAG-GFX12-TRUE16-NEXT:    ; return to shader part epilog
+;
+; SDAG-GFX12-FAKE16-LABEL: test_minmax_f16_ieee_false:
+; SDAG-GFX12-FAKE16:       ; %bb.0:
+; SDAG-GFX12-FAKE16-NEXT:    v_maxmin_num_f16 v0, v0, v1, v2
+; SDAG-GFX12-FAKE16-NEXT:    ; return to shader part epilog
+;
+; GISEL-GFX12-TRUE16-LABEL: test_minmax_f16_ieee_false:
+; GISEL-GFX12-TRUE16:       ; %bb.0:
+; GISEL-GFX12-TRUE16-NEXT:    v_maxmin_num_f16 v0.l, v0.l, v1.l, v2.l
+; GISEL-GFX12-TRUE16-NEXT:    ; return to shader part epilog
+;
+; GISEL-GFX12-FAKE16-LABEL: test_minmax_f16_ieee_false:
+; GISEL-GFX12-FAKE16:       ; %bb.0:
+; GISEL-GFX12-FAKE16-NEXT:    v_maxmin_num_f16 v0, v0, v1, v2
+; GISEL-GFX12-FAKE16-NEXT:    ; return to shader part epilog
   %max = call half @llvm.maxnum.f16(half %a, half %b)
   %minmax = call half @llvm.minnum.f16(half %max, half %c)
   ret half %minmax
 }
 
 define amdgpu_ps void @s_test_minmax_f16_ieee_false(half inreg %a, half inreg %b, half inreg %c, ptr addrspace(1) inreg %out) {
-; SDAG-GFX11-LABEL: s_test_minmax_f16_ieee_false:
-; SDAG-GFX11:       ; %bb.0:
-; SDAG-GFX11-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, 0
-; SDAG-GFX11-NEXT:    s_mov_b32 s5, s4
-; SDAG-GFX11-NEXT:    s_mov_b32 s4, s3
-; SDAG-GFX11-NEXT:    v_maxmin_f16 v0, s0, s1, v0
-; SDAG-GFX11-NEXT:    global_store_b16 v1, v0, s[4:5]
-; SDAG-GFX11-NEXT:    s_endpgm
-;
-; GISEL-GFX11-LABEL: s_test_minmax_f16_ieee_false:
-; GISEL-GFX11:       ; %bb.0:
-; GISEL-GFX11-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, 0
-; GISEL-GFX11-NEXT:    s_mov_b32 s6, s3
-; GISEL-GFX11-NEXT:    s_mov_b32 s7, s4
-; GISEL-GFX11-NEXT:    v_maxmin_f16 v0, s0, s1, v0
-; GISEL-GFX11-NEXT:    global_store_b16 v1, v0, s[6:7]
-; GISEL-GFX11-NEXT:    s_endpgm
-;
-; SDAG-GFX12-LABEL: s_test_minmax_f16_ieee_false:
-; SDAG-GFX12:       ; %bb.0:
-; SDAG-GFX12-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, 0
-; SDAG-GFX12-NEXT:    s_mov_b32 s5, s4
-; SDAG-GFX12-NEXT:    s_mov_b32 s4, s3
-; SDAG-GFX12-NEXT:    v_maxmin_num_f16 v0, s0, s1, v0
-; SDAG-GFX12-NEXT:    global_store_b16 v1, v0, s[4:5]
-; SDAG-GFX12-NEXT:    s_endpgm
+; SDAG-GFX11-TRUE16-LABEL: s_test_minmax_f16_ieee_false:
+; SDAG-GFX11-TRUE16:       ; %bb.0:
+; SDAG-GFX11-TRUE16-NEXT:    v_mov_b16_e32 v0.l, s2
+; SDAG-GFX11-TRUE16-NEXT:    v_mov_b32_e32 v1, 0
+; SDAG-GFX11-TRUE16-NEXT:    s_mov_b32 s5, s4
+; SDAG-GFX11-TRUE16-NEXT:    s_mov_b32 s4, s3
+; SDAG-GFX11-TRUE16-NEXT:    v_maxmin_f16 v0.l, s0, s1, v0.l
+; SDAG-GFX11-TRUE16-NEXT:    global_store_b16 v1, v0, s[4:5]
+; SDAG-GFX11-TRUE16-NEXT:    s_endpgm
+;
+; SDAG-GFX11-FAKE16-LABEL: s_test_minmax_f16_ieee_false:
+; SDAG-GFX11-FAKE16:       ; %bb.0:
+; SDAG-GFX11-FAKE16-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, 0
+; SDAG-GFX11-FAKE16-NEXT:    s_mov_b32 s5, s4
+; SDAG-GFX11-FAKE16-NEXT:    s_mov_b32 s4, s3
+; SDAG-GFX11-FAKE16-NEXT:    v_maxmin_f16 v0, s0, s1, v0
+; SDAG-GFX11-FAKE16-NEXT:    global_store_b16 v1, v0, s[4:5]
+; SDAG-GFX11-FAKE16-NEXT:    s_endpgm
+;
+; GISEL-GFX11-TRUE16-LABEL: s_test_minmax_f16_ieee_false:
+; GISEL-GFX11-TRUE16:       ; %bb.0:
+; GISEL-GFX11-TRUE16-NEXT:    v_mov_b16_e32 v0.l, s2
+; GISEL-GFX11-TRUE16-NEXT:    v_mov_b32_e32 v1, 0
+; GISEL-GFX11-TRUE16-NEXT:    s_mov_b32 s6, s3
+; GISEL-GFX11-TRUE16-NEXT:    s_mov_b32 s7, s4
+; GISEL-GFX11-TRUE16-NEXT:    v_maxmin_f16 v0.l, s0, s1, v0.l
+; GISEL-GFX11-TRUE16-NEXT:    global_store_b16 v1, v0, s[6:7]
+; GISEL-GFX11-TRUE16-NEXT:    s_endpgm
+;
+; GISEL-GFX11-FAKE16-LABEL: s_test_minmax_f16_ieee_false:
+; GISEL-GFX11-FAKE16:       ; %bb.0:
+; GISEL-GFX11-FAKE16-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, 0
+; GISEL-GFX11-FAKE16-NEXT:    s_mov_b32 s6, s3
+; GISEL-GFX11-FAKE16-NEXT:    s_mov_b32 s7, s4
+; GISEL-GFX11-FAKE16-NEXT:    v_maxmin_f16 v0, s0, s1, v0
+; GISEL-GFX11-FAKE16-NEXT:    global_store_b16 v1, v0, s[6:7]
+; GISEL-GFX11-FAKE16-NEXT:    s_endpgm
+;
+; SDAG-GFX12-TRUE16-LABEL: s_test_minmax_f16_ieee_false:
+; SDAG-GFX12-TRUE16:       ; %bb.0:
+; SDAG-GFX12-TRUE16-NEXT:    v_mov_b16_e32 v0.l, s2
+; SDAG-GFX12-TRUE16-NEXT:    v_mov_b32_e32 v1, 0
+; SDAG-GFX12-TRUE16-NEXT:    s_mov_b32 s5, s4
+; SDAG-GFX12-TRUE16-NEXT:    s_mov_b32 s4, s3
+; SDAG-GFX12-TRUE16-NEXT:    v_maxmin_num_f16 v0.l, s0, s1, v0.l
+; SDAG-GFX12-TRUE16-NEXT:    global_store_b16 v1, v0, s[4:5]
+; SDAG-GFX12-TRUE16-NEXT:    s_endpgm
+;
+; SDAG-GFX12-FAKE16-LABEL: s_test_minmax_f16_ieee_false:
+; SDAG-GFX12-FAKE16:       ; %bb.0:
+; SDAG-GFX12-FAKE16-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, 0
+; SDAG-GFX12-FAKE16-NEXT:    s_mov_b32 s5, s4
+; SDAG-GFX12-FAKE16-NEXT:    s_mov_b32 s4, s3
+; SDAG-GFX12-FAKE16-NEXT:    v_maxmin_num_f16 v0, s0, s1, v0
+; SDAG-GFX12-FAKE16-NEXT:    global_store_b16 v1, v0, s[4:5]
+; SDAG-GFX12-FAKE16-NEXT:    s_endpgm
 ;
 ; GISEL-GFX12-LABEL: s_test_minmax_f16_ieee_false:
 ; GISEL-GFX12:       ; %bb.0:
@@ -526,136 +594,320 @@ define amdgpu_ps void @s_test_minmax_f16_ieee_false(half inreg %a, half inreg %b
 }
 
 define half @test_minmax_commuted_f16_ieee_true(half %a, half %b, half %c) {
-; SDAG-GFX11-LABEL: test_minmax_commuted_f16_ieee_true:
-; SDAG-GFX11:       ; %bb.0:
-; SDAG-GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SDAG-GFX11-NEXT:    v_max_f16_e32 v1, v1, v1
-; SDAG-GFX11-NEXT:    v_max_f16_e32 v0, v0, v0
-; SDAG-GFX11-NEXT:    v_max_f16_e32 v2, v2, v2
-; SDAG-GFX11-NEXT:    v_maxmin_f16 v0, v0, v1, v2
-; SDAG-GFX11-NEXT:    s_setpc_b64 s[30:31]
-;
-; GISEL-GFX11-LABEL: test_minmax_commuted_f16_ieee_true:
-; GISEL-GFX11:       ; %bb.0:
-; GISEL-GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GISEL-GFX11-NEXT:    v_max_f16_e32 v0, v0, v0
-; GISEL-GFX11-NEXT:    v_max_f16_e32 v1, v1, v1
-; GISEL-GFX11-NEXT:    v_max_f16_e32 v2, v2, v2
-; GISEL-GFX11-NEXT:    v_maxmin_f16 v0, v0, v1, v2
-; GISEL-GFX11-NEXT:    s_setpc_b64 s[30:31]
-;
-; SDAG-GFX12-LABEL: test_minmax_commuted_f16_ieee_true:
-; SDAG-GFX12:       ; %bb.0:
-; SDAG-GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0
-; SDAG-GFX12-NEXT:    s_wait_expcnt 0x0
-; SDAG-GFX12-NEXT:    s_wait_samplecnt 0x0
-; SDAG-GFX12-NEXT:    s_wait_bvhcnt 0x0
-; SDAG-GFX12-NEXT:    s_wait_kmcnt 0x0
-; SDAG-GFX12-NEXT:    v_max_num_f16_e32 v1, v1, v1
-; SDAG-GFX12-NEXT:    v_max_num_f16_e32 v0, v0, v0
-; SDAG-GFX12-NEXT:    v_max_num_f16_e32 v2, v2, v2
-; SDAG-GFX12-NEXT:    v_maxmin_num_f16 v0, v0, v1, v2
-; SDAG-GFX12-NEXT:    s_setpc_b64 s[30:31]
-;
-; GISEL-GFX12-LABEL: test_minmax_commuted_f16_ieee_true:
-; GISEL-GFX12:       ; %bb.0:
-; GISEL-GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0
-; GISEL-GFX12-NEXT:    s_wait_expcnt 0x0
-; GISEL-GFX12-NEXT:    s_wait_samplecnt 0x0
-; GISEL-GFX12-NEXT:    s_wait_bvhcnt 0x0
-; GISEL-GFX12-NEXT:    s_wait_kmcnt 0x0
-; GISEL-GFX12-NEXT:    v_max_num_f16_e32 v0, v0, v0
-; GISEL-GFX12-NEXT:    v_max_num_f16_e32 v1, v1, v1
-; GISEL-GFX12-NEXT:    v_max_num_f16_e32 v2, v2, v2
-; GISEL-GFX12-NEXT:    v_maxmin_num_f16 v0, v0, v1, v2
-; GISEL-GFX12-NEXT:    s_setpc_b64 s[30:31]
+; SDAG-GFX11-TRUE16-LABEL: test_minmax_commuted_f16_ieee_true:
+; SDAG-GFX11-TRUE16:       ; %bb.0:
+; SDAG-GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SDAG-GFX11-TRUE16-NEXT:    v_mov_b16_e32 v0.h, v2.l
+; SDAG-GFX11-TRUE16-NEXT:    v_max_f16_e32 v1.l, v1.l, v1.l
+; SDAG-GFX11-TRUE16-NEXT:    v_max_f16_e32 v0.l, v0.l, v0.l
+; SDAG-GFX11-TRUE16-NEXT:    v_max_f16_e32 v0.h, v0.h, v0.h
+; SDAG-GFX11-TRUE16-NEXT:    v_maxmin_f16 v0.l, v0.l, v1.l, v0.h
+; SDAG-GFX11-TRUE16-NEXT:    s_setpc_b64 s[30:31]
+;
+; SDAG-GFX11-FAKE16-LABEL: test_minmax_commuted_f16_ieee_true:
+; SDAG-GFX11-FAKE16:       ; %bb.0:
+; SDAG-GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SDAG-GFX11-FAKE16-NEXT:    v_max_f16_e32 v1, v1, v1
+; SDAG-GFX11-FAKE16-NEXT:    v_max_f16_e32 v0, v0, v0
+; SDAG-GFX11-FAKE16-NEXT:    v_max_f16_e32 v2, v2, v2
+; SDAG-GFX11-FAKE16-NEXT:    v_maxmin_f16 v0, v0, v1, v2
+; SDAG-GFX11-FAKE16-NEXT:    s_setpc_b64 s[30:31]
+;
+; GISEL-GFX11-TRUE16-LABEL: test_minmax_commuted_f16_ieee_true:
+; GISEL-GFX11-TRUE16:       ; %bb.0:
+; GISEL-GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GISEL-GFX11-TRUE16-NEXT:    v_max_f16_e32 v0.l, v0.l, v0.l
+; GISEL-GFX11-TRUE16-NEXT:    v_max_f16_e32 v0.h, v1.l, v1.l
+; GISEL-GFX11-TRUE16-NEXT:    v_max_f16_e32 v1.l, v2.l, v2.l
+; GISEL-GFX11-TRUE16-NEXT:    v_maxmin_f16 v0.l, v0.l, v0.h, v1.l
+; GISEL-GFX11-TRUE16-NEXT:    s_setpc_b64 s[30:31]
+;
+; GISEL-GFX11-FAKE16-LABEL: test_minmax_commuted_f16_ieee_true:
+; GISEL-GFX11-FAKE16:       ; %bb.0:
+; GISEL-GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GISEL-GFX11-FAKE16-NEXT:    v_max_f16_e32 v0, v0, v0
+; GISEL-GFX11-FAKE16-NEXT:    v_max_f16_e32 v1, v1, v1
+; GISEL-GFX11-FAKE16-NEXT:    v_max_f16_e32 v2, v2, v2
+; GISEL-GFX11-FAKE16-NEXT:    v_maxmin_f16 v0, v0, v1, v2
+; GISEL-GFX11-FAKE16-NEXT:    s_setpc_b64 s[30:31]
+;
+; SDAG-GFX12-TRUE16-LABEL: test_minmax_commuted_f16_ieee_true:
+; SDAG-GFX12-TRUE16:       ; %bb.0:
+; SDAG-GFX12-TRUE16-NEXT:    s_wait_loadcnt_dscnt 0x0
+; SDAG-GFX12-TRUE16-NEXT:    s_wait_expcnt 0x0
+; SDAG-GFX12-TRUE16-NEXT:    s_wait_samplecnt 0x0
+; SDAG-GFX12-TRUE16-NEXT:    s_wait_bvhcnt 0x0
+; SDAG-GFX12-TRUE16-NEXT:    s_wait_kmcnt 0x0
+; SDAG-GFX12-TRUE16-NEXT:    v_mov_b16_e32 v0.h, v2.l
+; SDAG-GFX12-TRUE16-NEXT:    v_max_num_f16_e32 v1.l, v1.l, v1.l
+; SDAG-GFX12-TRUE16-NEXT:    v_max_num_f16_e32 v0.l, v0.l, v0.l
+; SDAG-GFX12-TRUE16-NEXT:    v_max_num_f16_e32 v0.h, v0.h, v0.h
+; SDAG-GFX12-TRUE16-NEXT:    v_maxmin_num_f16 v0.l, v0.l, v1.l, v0.h
+; SDAG-GFX12-TRUE16-NEXT:    s_setpc_b64 s[30:31]
+;
+; SDAG-GFX12-FAKE16-LABEL: test_minmax_commuted_f16_ieee_true:
+; SDAG-GFX12-FAKE16:       ; %bb.0:
+; SDAG-GFX12-FAKE16-NEXT:    s_wait_loadcnt_dscnt 0x0
+; SDAG-GFX12-FAKE16-NEXT:    s_wait_expcnt 0x0
+; SDAG-GFX12-FAKE16-NEXT:    s_wait_samplecnt 0x0
+; SDAG-GFX12-FAKE16-NEXT:    s_wait_bvhcnt 0x0
+; SDAG-GFX12-FAKE16-NEXT:    s_wait_kmcnt 0x0
+; SDAG-GFX12-FAKE16-NEXT:    v_max_num_f16_e32 v1, v1, v1
+; SDAG-GFX12-FAKE16-NEXT:    v_max_num_f16_e32 v0, v0, v0
+; SDAG-GFX12-FAKE16-NEXT:    v_max_num_f16_e32 v2, v2, v2
+; SDAG-GFX12-FAKE16-NEXT:    v_maxmin_num_f16 v0, v0, v1, v2
+; SDAG-GFX12-FAKE16-NEXT:    s_setpc_b64 s[30:31]
+;
+; GISEL-GFX12-TRUE16-LABEL: test_minmax_commuted_f16_ieee_true:
+; GISEL-GFX12-TRUE16:       ; %bb.0:
+; GISEL-GFX12-TRUE16-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GISEL-GFX12-TRUE16-NEXT:    s_wait_expcnt 0x0
+; GISEL-GFX12-TRUE16-NEXT:    s_wait_samplecnt 0x0
+; GISEL-GFX12-TRUE16-NEXT:    s_wait_bvhcnt 0x0
+; GISEL-GFX12-TRUE16-NEXT:    s_wait_kmcnt 0x0
+; GISEL-GFX12-TRUE16-NEXT:    v_max_num_f16_e32 v0.l, v0.l, v0.l
+; GISEL-GFX12-TRUE16-NEXT:    v_max_num_f16_e32 v0.h, v1.l, v1.l
+; GISEL-GFX12-TRUE16-NEXT:    v_max_num_f16_e32 v1.l, v2.l, v2.l
+; GISEL-GFX12-TRUE16-NEXT:    v_maxmin_num_f16 v0.l, v0.l, v0.h, v1.l
+; GISEL-GFX12-TRUE16-NEXT:    s_setpc_b64 s[30:31]
+;
+; GISEL-GFX12-FAKE16-LABEL: test_minmax_commuted_f16_ieee_true:
+; GISEL-GFX12-FAKE16:       ; %bb.0:
+; GISEL-GFX12-FAKE16-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GISEL-GFX12-FAKE16-NEXT:    s_wait_expcnt 0x0
+; GISEL-GFX12-FAKE16-NEXT:    s_wait_samplecnt 0x0
+; GISEL-GFX12-FAKE16-NEXT:    s_wait_bvhcnt 0x0
+; GISEL-GFX12-FAKE16-NEXT:    s_wait_kmcnt 0x0
+; GISEL-GFX12-FAKE16-NEXT:    v_max_num_f16_e32 v0, v0, v0
+; GISEL-GFX12-FAKE16-NEXT:    v_max_num_f16_e32 v1, v1, v1
+; GISEL-GFX12-FAKE16-NEXT:    v_max_num_f16_e32 v2, v2, v2
+; GISEL-GFX12-FAKE16-NEXT:    v_maxmin_num_f16 v0, v0, v1, v2
+; GISEL-GFX12-FAKE16-NEXT:    s_setpc_b64 s[30:31]
   %max = call half @llvm.maxnum.f16(half %a, half %b)
   %minmax = call half @llvm.minnum.f16(half %c, half %max)
   ret half %minmax
 }
 
 define amdgpu_ps half @test_maxmin_f16_ieee_false(half %a, half %b, half %c) {
-; GFX11-LABEL: test_maxmin_f16_ieee_false:
-; GFX11:       ; %bb.0:
-; GFX11-NEXT:    v_minmax_f16 v0, v0, v1, v2
-; GFX11-NEXT:    ; return to shader part epilog
-;
-; GFX12-LABEL: test_maxmin_f16_ieee_false:
-; GFX12:       ; %bb.0:
-; GFX12-NEXT:    v_minmax_num_f16 v0, v0, v1, v2
-; GFX12-NEXT:    ; return to shader part epilog
+; SDAG-GFX11-TRUE16-LABEL: test_maxmin_f16_ieee_false:
+; SDAG-GFX11-TRUE16:       ; %bb.0:
+; SDAG-GFX11-TRUE16-NEXT:    v_mov_b16_e32 v0.h, v1.l
+; SDAG-GFX11-TRUE16-NEXT:    v_mov_b16_e32 v1.l, v2.l
+; SDAG-GFX11-TRUE16-NEXT:    v_minmax_f16 v0.l, v0.l, v0.h, v1.l
+; SDAG-GFX11-TRUE16-NEXT:    ; return to shader part epilog
+;
+; SDAG-GFX11-FAKE16-LABEL: test_maxmin_f16_ieee_false:
+; SDAG-GFX11-FAKE16:       ; %bb.0:
+; SDAG-GFX11-FAKE16-NEXT:    v_minmax_f16 v0, v0, v1, v2
+; SDAG-GFX11-FAKE16-NEXT:    ; return to shader part epilog
+;
+; GISEL-GFX11-TRUE16-LABEL: test_maxmin_f16_ieee_false:
+; GISEL-GFX11-TRUE16:       ; %bb.0:
+; GISEL-GFX11-TRUE16-NEXT:    v_minmax_f16 v0.l, v0.l, v1.l, v2.l
+; GISEL-GFX11-TRUE16-NEXT:    ; return to shader part epilog
+;
+; GISEL-GFX11-FAKE16-LABEL: test_maxmin_f16_ieee_false:
+; GISEL-GFX11-FAKE16:       ; %bb.0:
+; GISEL-GFX11-FAKE16-NEXT:    v_minmax_f16 v0, v0, v1, v2
+; GISEL-GFX11-FAKE16-NEXT:    ; return to shader part epilog
+;
+; SDAG-GFX12-TRUE16-LABEL: test_maxmin_f16_ieee_false:
+; SDAG-GFX12-TRUE16:       ; %bb.0:
+; SDAG-GFX12-TRUE16-NEXT:    v_mov_b16_e32 v0.h, v1.l
+; SDAG-GFX12-TRUE16-NEXT:    v_mov_b16_e32 v1.l, v2.l
+; SDAG-GFX12-TRUE16-NEXT:    v_minmax_num_f16 v0.l, v0.l, v0.h, v1.l
+; SDAG-GFX12-TRUE16-NEXT:    ; return to shader part epilog
+;
+; SDAG-GFX12-FAKE16-LABEL: test_maxmin_f16_ieee_false:
+; SDAG-GFX12-FAKE16:       ; %bb.0:
+; SDAG-GFX12-FAKE16-NEXT:    v_minmax_num_f16 v0, v0, v1, v2
+; SDAG-GFX12-FAKE16-NEXT:    ; return to shader part epilog
+;
+; GISEL-GFX12-TRUE16-LABEL: test_maxmin_f16_ieee_false:
+; GISEL-GFX12-TRUE16:       ; %bb.0:
+; GISEL-GFX12-TRUE16-NEXT:    v_minmax_num_f16 v0.l, v0.l, v1.l, v2.l
+; GISEL-GFX12-TRUE16-NEXT:    ; return to shader part epilog
+;
+; GISEL-GFX12-FAKE16-LABEL: test_maxmin_f16_ieee_false:
+; GISEL-GFX12-FAKE16:       ; %bb.0:
+; GISEL-GFX12-FAKE16-NEXT:    v_minmax_num_f16 v0, v0, v1, v2
+; GISEL-GFX12-FAKE16-NEXT:    ; return to shader part epilog
   %min = call half @llvm.minnum.f16(half %a, half %b)
   %maxmin = call half @llvm.maxnum.f16(half %min, half %c)
   ret half %maxmin
 }
 
 define half @test_maxmin_commuted_f16_ieee_true(half %a, half %b, half %c) {
-; SDAG-GFX11-LABEL: test_maxmin_commuted_f16_ieee_true:
-; SDAG-GFX11:       ; %bb.0:
-; SDAG-GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SDAG-GFX11-NEXT:    v_max_f16_e32 v1, v1, v1
-; SDAG-GFX11-NEXT:    v_max_f16_e32 v0, v0, v0
-; SDAG-GFX11-NEXT:    v_max_f16_e32 v2, v2, v2
-; SDAG-GFX11-NEXT:    v_minmax_f16 v0, v0, v1, v2
-; SDAG-GFX11-NEXT:    s_setpc_b64 s[30:31]
-;
-; GISEL-GFX11-LABEL: test_maxmin_commuted_f16_ieee_true:
-; GISEL-GFX11:       ; %bb.0:
-; GISEL-GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GISEL-GFX11-NEXT:    v_max_f16_e32 v0, v0, v0
-; GISEL-GFX11-NEXT:    v_max_f16_e32 v1, v1, v1
-; GISEL-GFX11-NEXT:    v_max_f16_e32 v2, v2, v2
-; GISEL-GFX11-NEXT:    v_minmax_f16 v0, v0, v1, v2
-; GISEL-GFX11-NEXT:    s_setpc_b64 s[30:31]
-;
-; SDAG-GFX12-LABEL: test_maxmin_commuted_f16_ieee_true:
-; SDAG-GFX12:       ; %bb.0:
-; SDAG-GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0
-; SDAG-GFX12-NEXT:    s_wait_expcnt 0x0
-; SDAG-GFX12-NEXT:    s_wait_samplecnt 0x0
-; SDAG-GFX12-NEXT:    s_wait_bvhcnt 0x0
-; SDAG-GFX12-NEXT:    s_wait_kmcnt 0x0
-; SDAG-GFX12-NEXT:    v_max_num_f16_e32 v1, v1, v1
-; SDAG-GFX12-NEXT:    v_max_num_f16_e32 v0, v0, v0
-; SDAG-GFX12-NEXT:    v_max_num_f16_e32 v2, v2, v2
-; SDAG-GFX12-NEXT:    v_minmax_num_f16 v0, v0, v1, v2
-; SDAG-GFX12-NEXT:    s_setpc_b64 s[30:31]
-;
-; GISEL-GFX12-LABEL: test_maxmin_commuted_f16_ieee_true:
-; GISEL-GFX12:       ; %bb.0:
-; GISEL-GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0
-; GISEL-GFX12-NEXT:    s_wait_expcnt 0x0
-; GISEL-GFX12-NEXT:    s_wait_samplecnt 0x0
-; GISEL-GFX12-NEXT:    s_wait_bvhcnt 0x0
-; GISEL-GFX12-NEXT:    s_wait_kmcnt 0x0
-; GISEL-GFX12-NEXT:    v_max_num_f16_e32 v0, v0, v0
-; GISEL-GFX12-NEXT:    v_max_num_f16_e32 v1, v1, v1
-; GISEL-GFX12-NEXT:    v_max_num_f16_e32 v2, v2, v2
-; GISEL-GFX12-NEXT:    v_minmax_num_f16 v0, v0, v1, v2
-; GISEL-GFX12-NEXT:    s_setpc_b64 s[30:31]
+; SDAG-GFX11-TRUE16-LABEL: test_maxmin_commuted_f16_ieee_true:
+; SDAG-GFX11-TRUE16:       ; %bb.0:
+; SDAG-GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SDAG-GFX11-TRUE16-NEXT:    v_mov_b16_e32 v0.h, v2.l
+; SDAG-GFX11-TRUE16-NEXT:    v_max_f16_e32 v1.l, v1.l, v1.l
+; SDAG-GFX11-TRUE16-NEXT:    v_max_f16_e32 v0.l, v0.l, v0.l
+; SDAG-GFX11-TRUE16-NEXT:    v_max_f16_e32 v0.h, v0.h, v0.h
+; SDAG-GFX11-TRUE16-NEXT:    v_minmax_f16 v0.l, v0.l, v1.l, v0.h
+; SDAG-GFX11-TRUE16-NEXT:    s_setpc_b64 s[30:31]
+;
+; SDAG-GFX11-FAKE16-LABEL: test_maxmin_commuted_f16_ieee_true:
+; SDAG-GFX11-FAKE16:       ; %bb.0:
+; SDAG-GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SDAG-GFX11-FAKE16-NEXT:    v_max_f16_e32 v1, v1, v1
+; SDAG-GFX11-FAKE16-NEXT:    v_max_f16_e32 v0, v0, v0
+; SDAG-GFX11-FAKE16-NEXT:    v_max_f16_e32 v2, v2, v2
+; SDAG-GFX11-FAKE16-NEXT:    v_minmax_f16 v0, v0, v1, v2
+; SDAG-GFX11-FAKE16-NEXT:    s_setpc_b64 s[30:31]
+;
+; GISEL-GFX11-TRUE16-LABEL: test_maxmin_commuted_f16_ieee_true:
+; GISEL-GFX11-TRUE16:       ; %bb.0:
+; GISEL-GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GISEL-GFX11-TRUE16-NEXT:    v_max_f16_e32 v0.l, v0.l, v0.l
+; GISEL-GFX11-TRUE16-NEXT:    v_max_f16_e32 v0.h, v1.l, v1.l
+; GISEL-GFX11-TRUE16-NEXT:    v_max_f16_e32 v1.l, v2.l, v2.l
+; GISEL-GFX11-TRUE16-NEXT:    v_minmax_f16 v0.l, v0.l, v0.h, v1.l
+; GISEL-GFX11-TRUE16-NEXT:    s_setpc_b64 s[30:31]
+;
+; GISEL-GFX11-FAKE16-LABEL: test_maxmin_commuted_f16_ieee_true:
+; GISEL-GFX11-FAKE16:       ; %bb.0:
+; GISEL-GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GISEL-GFX11-FAKE16-NEXT:    v_max_f16_e32 v0, v0, v0
+; GISEL-GFX11-FAKE16-NEXT:    v_max_f16_e32 v1, v1, v1
+; GISEL-GFX11-FAKE16-NEXT:    v_max_f16_e32 v2, v2, v2
+; GISEL-GFX11-FAKE16-NEXT:    v_minmax_f16 v0, v0, v1, v2
+; GISEL-GFX11-FAKE16-NEXT:    s_setpc_b64 s[30:31]
+;
+; SDAG-GFX12-TRUE16-LABEL: test_maxmin_commuted_f16_ieee_true:
+; SDAG-GFX12-TRUE16:       ; %bb.0:
+; SDAG-GFX12-TRUE16-NEXT:    s_wait_loadcnt_dscnt 0x0
+; SDAG-GFX12-TRUE16-NEXT:    s_wait_expcnt 0x0
+; SDAG-GFX12-TRUE16-NEXT:    s_wait_samplecnt 0x0
+; SDAG-GFX12-TRUE16-NEXT:    s_wait_bvhcnt 0x0
+; SDAG-GFX12-TRUE16-NEXT:    s_wait_kmcnt 0x0
+; SDAG-GFX12-TRUE16-NEXT:    v_mov_b16_e32 v0.h, v2.l
+; SDAG-GFX12-TRUE16-NEXT:    v_max_num_f16_e32 v1.l, v1.l, v1.l
+; SDAG-GFX12-TRUE16-NEXT:    v_max_num_f16_e32 v0.l, v0.l, v0.l
+; SDAG-GFX12-TRUE16-NEXT:    v_max_num_f16_e32 v0.h, v0.h, v0.h
+; SDAG-GFX12-TRUE16-NEXT:    v_minmax_num_f16 v0.l, v0.l, v1.l, v0.h
+; SDAG-GFX12-TRUE16-NEXT:    s_setpc_b64 s[30:31]
+;
+; SDAG-GFX12-FAKE16-LABEL: test_maxmin_commuted_f16_ieee_true:
+; SDAG-GFX12-FAKE16:       ; %bb.0:
+; SDAG-GFX12-FAKE16-NEXT:    s_wait_loadcnt_dscnt 0x0
+; SDAG-GFX12-FAKE16-NEXT:    s_wait_expcnt 0x0
+; SDAG-GFX12-FAKE16-NEXT:    s_wait_samplecnt 0x0
+; SDAG-GFX12-FAKE16-NEXT:    s_wait_bvhcnt 0x0
+; SDAG-GFX12-FAKE16-NEXT:    s_wait_kmcnt 0x0
+; SDAG-GFX12-FAKE16-NEXT:    v_max_num_f16_e32 v1, v1, v1
+; SDAG-GFX12-FAKE16-NEXT:    v_max_num_f16_e32 v0, v0, v0
+; SDAG-GFX12-FAKE16-NEXT:    v_max_num_f16_e32 v2, v2, v2
+; SDAG-GFX12-FAKE16-NEXT:    v_minmax_num_f16 v0, v0, v1, v2
+; SDAG-GFX12-FAKE16-NEXT:    s_setpc_b64 s[30:31]
+;
+; GISEL-GFX12-TRUE16-LABEL: test_maxmin_commuted_f16_ieee_true:
+; GISEL-GFX12-TRUE16:       ; %bb.0:
+; GISEL-GFX12-TRUE16-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GISEL-GFX12-TRUE16-NEXT:    s_wait_expcnt 0x0
+; GISEL-GFX12-TRUE16-NEXT:    s_wait_samplecnt 0x0
+; GISEL-GFX12-TRUE16-NEXT:    s_wait_bvhcnt 0x0
+; GISEL-GFX12-TRUE16-NEXT:    s_wait_kmcnt 0x0
+; GISEL-GFX12-TRUE16-NEXT:    v_max_num_f16_e32 v0.l, v0.l, v0.l
+; GISEL-GFX12-TRUE16-NEXT:    v_max_num_f16_e32 v0.h, v1.l, v1.l
+; GISEL-GFX12-TRUE16-NEXT:    v_max_num_f16_e32 v1.l, v2.l, v2.l
+; GISEL-GFX12-TRUE16-NEXT:    v_minmax_num_f16 v0.l, v0.l, v0.h, v1.l
+; GISEL-GFX12-TRUE16-NEXT:    s_setpc_b64 s[30:31]
+;
+; GISEL-GFX12-FAKE16-LABEL: test_maxmin_commuted_f16_ieee_true:
+; GISEL-GFX12-FAKE16:       ; %bb.0:
+; GISEL-GFX12-FAKE16-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GISEL-GFX12-FAKE16-NEXT:    s_wait_expcnt 0x0
+; GISEL-GFX12-FAKE16-NEXT:    s_wait_samplecnt 0x0
+; GISEL-GFX12-FAKE16-NEXT:    s_wait_bvhcnt 0x0
+; GISEL-GFX12-FAKE16-NEXT:    s_wait_kmcnt 0x0
+; GISEL-GFX12-FAKE16-NEXT:    v_max_num_f16_e32 v0, v0, v0
+; GISEL-GFX12-FAKE16-NEXT:    v_max_num_f16_e32 v1, v1, v1
+; GISEL-GFX12-FAKE16-NEXT:    v_max_num_f16_e32 v2, v2, v2
+; GISEL-GFX12-FAKE16-NEXT:    v_minmax_num_f16 v0, v0, v1, v2
+; GISEL-GFX12-FAKE16-NEXT:    s_setpc_b64 s[30:31]
   %min = call half @llvm.minnum.f16(half %a, half %b)
   %maxmin = call half @llvm.maxnum.f16(half %c, half %min)
   ret half %maxmin
 }
 
 define void @test_med3_f16(ptr addrspace(1) %arg, half %x, half %y, half %z) #0 {
-; GFX11-LABEL: test_med3_f16:
-; GFX11:       ; %bb.0:
-; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT:    v_med3_f16 v2, v2, v3, v4
-; GFX11-NEXT:    global_store_b16 v[0:1], v2, off
-; GFX11-NEXT:    s_setpc_b64 s[30:31]
-;
-; GFX12-LABEL: test_med3_f16:
-; GFX12:       ; %bb.0:
-; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT:    s_wait_expcnt 0x0
-; GFX12-NEXT:    s_wait_samplecnt 0x0
-; GFX12-NEXT:    s_wait_bvhcnt 0x0
-; GFX12-NEXT:    s_wait_kmcnt 0x0
-; GFX12-NEXT:    v_med3_num_f16 v2, v2, v3, v4
-; GFX12-NEXT:    global_store_b16 v[0:1], v2, off
-; GFX12-NEXT:    s_setpc_b64 s[30:31]
+; SDAG-GFX11-TRUE16-LABEL: test_med3_f16:
+; SDAG-GFX11-TRUE16:       ; %bb.0:
+; SDAG-GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SDAG-GFX11-TRUE16-NEXT:    v_mov_b16_e32 v2.h, v3.l
+; SDAG-GFX11-TRUE16-NEXT:    v_mov_b16_e32 v3.l, v4.l
+; SDAG-GFX11-TRUE16-NEXT:    v_med3_f16 v2.l, v2.l, v2.h, v3.l
+; SDAG-GFX11-TRUE16-NEXT:    global_store_b16 v[0:1], v2, off
+; SDAG-GFX11-TRUE16-NEXT:    s_setpc_b64 s[30:31]
+;
+; SDAG-GFX11-FAKE16-LABEL: test_med3_f16:
+; SDAG-GFX11-FAKE16:       ; %bb.0:
+; SDAG-GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SDAG-GFX11-FAKE16-NEXT:    v_med3_f16 v2, v2, v3, v4
+; SDAG-GFX11-FAKE16-NEXT:    global_store_b16 v[0:1], v2, off
+; SDAG-GFX11-FAKE16-NEXT:    s_setpc_b64 s[30:31]
+;
+; GISEL-GFX11-TRUE16-LABEL: test_med3_f16:
+; GISEL-GFX11-TRUE16:       ; %bb.0:
+; GISEL-GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GISEL-GFX11-TRUE16-NEXT:    v_med3_f16 v2.l, v2.l, v3.l, v4.l
+; GISEL-GFX11-TRUE16-NEXT:    global_store_b16 v[0:1], v2, off
+; GISEL-GFX11-TRUE16-NEXT:    s_setpc_b64 s[30:31]
+;
+; GISEL-GFX11-FAKE16-LABEL: test_med3_f16:
+; GISEL-GFX11-FAKE16:       ; %bb.0:
+; GISEL-GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GISEL-GFX11-FAKE16-NEXT:    v_med3_f16 v2, v2, v3, v4
+; GISEL-GFX11-FAKE16-NEXT:    global_store_b16 v[0:1], v2, off
+; GISEL-GFX11-FAKE16-NEXT:    s_setpc_b64 s[30:31]
+;
+; SDAG-GFX12-TRUE16-LABEL: test_med3_f16:
+; SDAG-GFX12-TRUE16:       ; %bb.0:
+; SDAG-GFX12-TRUE16-NEXT:    s_wait_loadcnt_dscnt 0x0
+; SDAG-GFX12-TRUE16-NEXT:    s_wait_expcnt 0x0
+; SDAG-GFX12-TRUE16-NEXT:    s_wait_samplecnt 0x0
+; SDAG-GFX12-TRUE16-NEXT:    s_wait_bvhcnt 0x0
+; SDAG-GFX12-TRUE16-NEXT:    s_wait_kmcnt 0x0
+; SDAG-GFX12-TRUE16-NEXT:    v_mov_b16_e32 v2.h, v3.l
+; SDAG-GFX12-TRUE16-NEXT:    v_mov_b16_e32 v3.l, v4.l
+; SDAG-GFX12-TRUE16-NEXT:    v_med3_num_f16 v2.l, v2.l, v2.h, v3.l
+; SDAG-GFX12-TRUE16-NEXT:    global_store_b16 v[0:1], v2, off
+; SDAG-GFX12-TRUE16-NEXT:    s_setpc_b64 s[30:31]
+;
+; SDAG-GFX12-FAKE16-LABEL: test_med3_f16:
+; SDAG-GFX12-FAKE16:       ; %bb.0:
+; SDAG-GFX12-FAKE16-NEXT:    s_wait_loadcnt_dscnt 0x0
+; SDAG-GFX12-FAKE16-NEXT:    s_wait_expcnt 0x0
+; SDAG-GFX12-FAKE16-NEXT:    s_wait_samplecnt 0x0
+; SDAG-GFX12-FAKE16-NEXT:    s_wait_bvhcnt 0x0
+; SDAG-GFX12-FAKE16-NEXT:    s_wait_kmcnt 0x0
+; SDAG-GFX12-FAKE16-NEXT:    v_med3_num_f16 v2, v2, v3, v4
+; SDAG-GFX12-FAKE16-NEXT:    global_store_b16 v[0:1], v2, off
+; SDAG-GFX12-FAKE16-NEXT:    s_setpc_b64 s[30:31]
+;
+; GISEL-GFX12-TRUE16-LABEL: test_med3_f16:
+; GISEL-GFX12-TRUE16:       ; %bb.0:
+; GISEL-GFX12-TRUE16-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GISEL-GFX12-TRUE16-NEXT:    s_wait_expcnt 0x0
+; GISEL-GFX12-TRUE16-NEXT:    s_wait_samplecnt 0x0
+; GISEL-GFX12-TRUE16-NEXT:    s_wait_bvhcnt 0x0
+; GISEL-GFX12-TRUE16-NEXT:    s_wait_kmcnt 0x0
+; GISEL-GFX12-TRUE16-NEXT:    v_med3_num_f16 v2.l, v2.l, v3.l, v4.l
+; GISEL-GFX12-TRUE16-NEXT:    global_store_b16 v[0:1], v2, off
+; GISEL-GFX12-TRUE16-NEXT:    s_setpc_b64 s[30:31]
+;
+; GISEL-GFX12-FAKE16-LABEL: test_med3_f16:
+; GISEL-GFX12-FAKE16:       ; %bb.0:
+; GISEL-GFX12-FAKE16-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GISEL-GFX12-FAKE16-NEXT:    s_wait_expcnt 0x0
+; GISEL-GFX12-FAKE16-NEXT:    s_wait_samplecnt 0x0
+; GISEL-GFX12-FAKE16-NEXT:    s_wait_bvhcnt 0x0
+; GISEL-GFX12-FAKE16-NEXT:    s_wait_kmcnt 0x0
+; GISEL-GFX12-FAKE16-NEXT:    v_med3_num_f16 v2, v2, v3, v4
+; GISEL-GFX12-FAKE16-NEXT:    global_store_b16 v[0:1], v2, off
+; GISEL-GFX12-FAKE16-NEXT:    s_setpc_b64 s[30:31]
   %tmp0 = call half @llvm.minnum.f16(half %x, half %y)
   %tmp1 = call half @llvm.maxnum.f16(half %x, half %y)
   %tmp2 = call half @llvm.minnum.f16(half %tmp1, half %z)
diff --git a/llvm/test/CodeGen/AMDGPU/misched-killflags.mir b/llvm/test/CodeGen/AMDGPU/misched-killflags.mir
index 9d0c32214100f..a7fa8ad5389b0 100644
--- a/llvm/test/CodeGen/AMDGPU/misched-killflags.mir
+++ b/llvm/test/CodeGen/AMDGPU/misched-killflags.mir
@@ -1,4 +1,5 @@
 # RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=fiji -verify-machineinstrs -run-pass=post-RA-sched -o - %s | FileCheck %s
+# RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=fiji -passes=post-RA-sched -o - %s | FileCheck %s
 # Make sure ScheduleDAGInstrs::fixupKills does not produce invalid kill flags.
 ---
 name: func0
diff --git a/llvm/test/CodeGen/AMDGPU/mmra.ll b/llvm/test/CodeGen/AMDGPU/mmra.ll
index 39650f4295c76..d0696bf329af8 100644
--- a/llvm/test/CodeGen/AMDGPU/mmra.ll
+++ b/llvm/test/CodeGen/AMDGPU/mmra.ll
@@ -17,7 +17,7 @@ define void @fence_loads(ptr %ptr) {
   ; CHECK-NEXT:   ATOMIC_FENCE 5, 1, mmra !0
   ; CHECK-NEXT:   [[COPY2:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]], mmra !1
   ; CHECK-NEXT:   [[FLAT_LOAD_UBYTE:%[0-9]+]]:vgpr_32 = FLAT_LOAD_UBYTE [[COPY2]], 0, 0, implicit $exec, implicit $flat_scr, mmra !1 :: (load acquire (s8) from %ir.ptr, align 4)
-  ; CHECK-NEXT:   [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 1, implicit $exec, mmra !2
+  ; CHECK-NEXT:   [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 1, implicit $exec
   ; CHECK-NEXT:   [[COPY3:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]], mmra !2
   ; CHECK-NEXT:   FLAT_STORE_BYTE [[COPY3]], killed [[V_MOV_B32_e32_]], 0, 0, implicit $exec, implicit $flat_scr, mmra !2 :: (store release (s8) into %ir.ptr, align 4)
   ; CHECK-NEXT:   SI_RETURN
@@ -82,7 +82,7 @@ define void @atomicrmw_rel(ptr %ptr) {
   ; CHECK-NEXT:   [[V_AND_B32_e64_2:%[0-9]+]]:vgpr_32 = V_AND_B32_e64 [[PHI1]], killed [[V_OR_B32_e64_]], implicit $exec
   ; CHECK-NEXT:   [[DEF4:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
   ; CHECK-NEXT:   [[DEF5:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
-  ; CHECK-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[V_AND_B32_e64_2]], %subreg.sub0, [[PHI1]], %subreg.sub1, mmra !2
+  ; CHECK-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[V_AND_B32_e64_2]], %subreg.sub0, [[PHI1]], %subreg.sub1
   ; CHECK-NEXT:   [[COPY6:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE2]], mmra !2
   ; CHECK-NEXT:   [[FLAT_ATOMIC_CMPSWAP_RTN:%[0-9]+]]:vgpr_32 = FLAT_ATOMIC_CMPSWAP_RTN [[COPY4]], killed [[COPY6]], 0, 1, implicit $exec, implicit $flat_scr, mmra !2 :: (load store release monotonic (s32) on %ir.AlignedAddr)
   ; CHECK-NEXT:   [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64 = V_CMP_EQ_U32_e64 [[FLAT_ATOMIC_CMPSWAP_RTN]], [[PHI1]], implicit $exec, mmra !2
@@ -140,7 +140,7 @@ define void @cmpxchg(ptr %ptr) {
   ; CHECK-NEXT:   [[V_OR_B32_e64_:%[0-9]+]]:vgpr_32 = V_OR_B32_e64 [[PHI2]], [[V_LSHLREV_B32_e64_2]], implicit $exec
   ; CHECK-NEXT:   [[DEF5:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
   ; CHECK-NEXT:   [[DEF6:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
-  ; CHECK-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[V_OR_B32_e64_]], %subreg.sub0, [[PHI2]], %subreg.sub1, mmra !1
+  ; CHECK-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[V_OR_B32_e64_]], %subreg.sub0, [[PHI2]], %subreg.sub1
   ; CHECK-NEXT:   [[COPY6:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE2]], mmra !1
   ; CHECK-NEXT:   [[FLAT_ATOMIC_CMPSWAP_RTN:%[0-9]+]]:vgpr_32 = FLAT_ATOMIC_CMPSWAP_RTN [[COPY4]], killed [[COPY6]], 0, 1, implicit $exec, implicit $flat_scr, mmra !1 :: (load store acquire acquire (s32) on %ir.AlignedAddr)
   ; CHECK-NEXT:   [[V_CMP_NE_U32_e64_:%[0-9]+]]:sreg_64 = V_CMP_NE_U32_e64 [[FLAT_ATOMIC_CMPSWAP_RTN]], [[PHI2]], implicit $exec
@@ -180,6 +180,105 @@ define void @cmpxchg(ptr %ptr) {
   ret void
 }
 
+declare <32 x half> @f()
+
+; Variant of atomicrmw_rel that provoked a crash in SelectionDAG::copyExtraInfo
+define void @atomicrmw_rel_deepcopy(ptr %ptr) {
+  ; CHECK-LABEL: name: atomicrmw_rel_deepcopy
+  ; CHECK: bb.0 (%ir-block.0):
+  ; CHECK-NEXT:   successors: %bb.1(0x80000000)
+  ; CHECK-NEXT:   liveins: $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9, $sgpr10_sgpr11, $sgpr12, $sgpr13, $sgpr14, $sgpr15, $vgpr0, $vgpr1, $vgpr31
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr31
+  ; CHECK-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; CHECK-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; CHECK-NEXT:   [[COPY3:%[0-9]+]]:sgpr_32 = COPY $sgpr15
+  ; CHECK-NEXT:   [[COPY4:%[0-9]+]]:sgpr_32 = COPY $sgpr14
+  ; CHECK-NEXT:   [[COPY5:%[0-9]+]]:sgpr_32 = COPY $sgpr13
+  ; CHECK-NEXT:   [[COPY6:%[0-9]+]]:sgpr_32 = COPY $sgpr12
+  ; CHECK-NEXT:   [[COPY7:%[0-9]+]]:sgpr_64 = COPY $sgpr10_sgpr11
+  ; CHECK-NEXT:   [[COPY8:%[0-9]+]]:sgpr_64 = COPY $sgpr8_sgpr9
+  ; CHECK-NEXT:   [[COPY9:%[0-9]+]]:sgpr_64 = COPY $sgpr6_sgpr7
+  ; CHECK-NEXT:   [[COPY10:%[0-9]+]]:sgpr_64 = COPY $sgpr4_sgpr5
+  ; CHECK-NEXT:   [[DEF:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; CHECK-NEXT:   [[DEF1:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; CHECK-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY2]], %subreg.sub0, [[COPY1]], %subreg.sub1
+  ; CHECK-NEXT:   [[COPY11:%[0-9]+]]:vgpr_32 = COPY [[REG_SEQUENCE]].sub1
+  ; CHECK-NEXT:   ADJCALLSTACKUP 0, 0, implicit-def dead $scc, implicit-def $sgpr32, implicit $sgpr32
+  ; CHECK-NEXT:   [[SI_PC_ADD_REL_OFFSET:%[0-9]+]]:sreg_64 = SI_PC_ADD_REL_OFFSET target-flags(amdgpu-gotprel32-lo) @f, target-flags(amdgpu-gotprel32-hi) @f, implicit-def dead $scc
+  ; CHECK-NEXT:   [[S_LOAD_DWORDX2_IMM:%[0-9]+]]:sreg_64_xexec = S_LOAD_DWORDX2_IMM killed [[SI_PC_ADD_REL_OFFSET]], 0, 0 :: (dereferenceable invariant load (s64) from got, addrspace 4)
+  ; CHECK-NEXT:   [[COPY12:%[0-9]+]]:sgpr_128 = COPY $sgpr0_sgpr1_sgpr2_sgpr3
+  ; CHECK-NEXT:   $sgpr4_sgpr5 = COPY [[COPY10]]
+  ; CHECK-NEXT:   $sgpr6_sgpr7 = COPY [[COPY9]]
+  ; CHECK-NEXT:   $sgpr8_sgpr9 = COPY [[COPY8]]
+  ; CHECK-NEXT:   $sgpr10_sgpr11 = COPY [[COPY7]]
+  ; CHECK-NEXT:   $sgpr12 = COPY [[COPY6]]
+  ; CHECK-NEXT:   $sgpr13 = COPY [[COPY5]]
+  ; CHECK-NEXT:   $sgpr14 = COPY [[COPY4]]
+  ; CHECK-NEXT:   $sgpr15 = COPY [[COPY3]]
+  ; CHECK-NEXT:   $vgpr31 = COPY [[COPY]]
+  ; CHECK-NEXT:   $sgpr0_sgpr1_sgpr2_sgpr3 = COPY [[COPY12]]
+  ; CHECK-NEXT:   $sgpr30_sgpr31 = SI_CALL killed [[S_LOAD_DWORDX2_IMM]], @f, csr_amdgpu, implicit $sgpr4_sgpr5, implicit $sgpr6_sgpr7, implicit $sgpr8_sgpr9, implicit $sgpr10_sgpr11, implicit $sgpr12, implicit $sgpr13, implicit $sgpr14, implicit $sgpr15, implicit $vgpr31, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit-def $vgpr0, implicit-def $vgpr1, implicit-def $vgpr2, implicit-def $vgpr3, implicit-def $vgpr4, implicit-def $vgpr5, implicit-def $vgpr6, implicit-def $vgpr7, implicit-def $vgpr8, implicit-def $vgpr9, implicit-def $vgpr10, implicit-def $vgpr11, implicit-def $vgpr12, implicit-def $vgpr13, implicit-def $vgpr14, implicit-def $vgpr15
+  ; CHECK-NEXT:   ADJCALLSTACKDOWN 0, 0, implicit-def dead $scc, implicit-def $sgpr32, implicit $sgpr32
+  ; CHECK-NEXT:   [[COPY13:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; CHECK-NEXT:   [[COPY14:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; CHECK-NEXT:   [[COPY15:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; CHECK-NEXT:   [[COPY16:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; CHECK-NEXT:   [[COPY17:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; CHECK-NEXT:   [[COPY18:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+  ; CHECK-NEXT:   [[COPY19:%[0-9]+]]:vgpr_32 = COPY $vgpr6
+  ; CHECK-NEXT:   [[COPY20:%[0-9]+]]:vgpr_32 = COPY $vgpr7
+  ; CHECK-NEXT:   [[COPY21:%[0-9]+]]:vgpr_32 = COPY $vgpr8
+  ; CHECK-NEXT:   [[COPY22:%[0-9]+]]:vgpr_32 = COPY $vgpr9
+  ; CHECK-NEXT:   [[COPY23:%[0-9]+]]:vgpr_32 = COPY $vgpr10
+  ; CHECK-NEXT:   [[COPY24:%[0-9]+]]:vgpr_32 = COPY $vgpr11
+  ; CHECK-NEXT:   [[COPY25:%[0-9]+]]:vgpr_32 = COPY $vgpr12
+  ; CHECK-NEXT:   [[COPY26:%[0-9]+]]:vgpr_32 = COPY $vgpr13
+  ; CHECK-NEXT:   [[COPY27:%[0-9]+]]:vgpr_32 = COPY $vgpr14
+  ; CHECK-NEXT:   [[COPY28:%[0-9]+]]:vgpr_32 = COPY $vgpr15
+  ; CHECK-NEXT:   [[COPY29:%[0-9]+]]:vgpr_32 = COPY [[REG_SEQUENCE]].sub0
+  ; CHECK-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 -4
+  ; CHECK-NEXT:   [[V_AND_B32_e64_:%[0-9]+]]:vgpr_32 = V_AND_B32_e64 [[COPY29]], killed [[S_MOV_B32_]], implicit $exec
+  ; CHECK-NEXT:   [[DEF2:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; CHECK-NEXT:   [[DEF3:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; CHECK-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[V_AND_B32_e64_]], %subreg.sub0, [[COPY11]], %subreg.sub1
+  ; CHECK-NEXT:   [[COPY30:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE1]]
+  ; CHECK-NEXT:   [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 3
+  ; CHECK-NEXT:   [[V_AND_B32_e64_1:%[0-9]+]]:vgpr_32 = V_AND_B32_e64 [[COPY29]], [[S_MOV_B32_1]], implicit $exec
+  ; CHECK-NEXT:   [[V_LSHLREV_B32_e64_:%[0-9]+]]:vgpr_32 = V_LSHLREV_B32_e64 [[S_MOV_B32_1]], killed [[V_AND_B32_e64_1]], implicit $exec
+  ; CHECK-NEXT:   [[S_MOV_B32_2:%[0-9]+]]:sreg_32 = S_MOV_B32 255
+  ; CHECK-NEXT:   [[V_LSHLREV_B32_e64_1:%[0-9]+]]:vgpr_32 = V_LSHLREV_B32_e64 killed [[V_LSHLREV_B32_e64_]], killed [[S_MOV_B32_2]], implicit $exec
+  ; CHECK-NEXT:   [[V_NOT_B32_e32_:%[0-9]+]]:vgpr_32 = V_NOT_B32_e32 [[V_LSHLREV_B32_e64_1]], implicit $exec
+  ; CHECK-NEXT:   [[COPY31:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE1]], mmra !0
+  ; CHECK-NEXT:   [[FLAT_LOAD_DWORD:%[0-9]+]]:vgpr_32 = FLAT_LOAD_DWORD [[COPY31]], 0, 0, implicit $exec, implicit $flat_scr, mmra !0 :: (load (s32) from %ir.AlignedAddr)
+  ; CHECK-NEXT:   [[S_MOV_B64_:%[0-9]+]]:sreg_64 = S_MOV_B64 0
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.1.atomicrmw.start:
+  ; CHECK-NEXT:   successors: %bb.2(0x04000000), %bb.1(0x7c000000)
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   [[PHI:%[0-9]+]]:sreg_64 = PHI [[S_MOV_B64_]], %bb.0, %7, %bb.1
+  ; CHECK-NEXT:   [[PHI1:%[0-9]+]]:vgpr_32 = PHI [[FLAT_LOAD_DWORD]], %bb.0, %6, %bb.1
+  ; CHECK-NEXT:   [[V_OR_B32_e64_:%[0-9]+]]:vgpr_32 = V_OR_B32_e64 [[V_NOT_B32_e32_]], [[V_LSHLREV_B32_e64_1]], implicit $exec
+  ; CHECK-NEXT:   [[V_AND_B32_e64_2:%[0-9]+]]:vgpr_32 = V_AND_B32_e64 [[PHI1]], killed [[V_OR_B32_e64_]], implicit $exec
+  ; CHECK-NEXT:   [[DEF4:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; CHECK-NEXT:   [[DEF5:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; CHECK-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[V_AND_B32_e64_2]], %subreg.sub0, [[PHI1]], %subreg.sub1
+  ; CHECK-NEXT:   [[COPY32:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE2]], mmra !0
+  ; CHECK-NEXT:   [[FLAT_ATOMIC_CMPSWAP_RTN:%[0-9]+]]:vgpr_32 = FLAT_ATOMIC_CMPSWAP_RTN [[COPY30]], killed [[COPY32]], 0, 1, implicit $exec, implicit $flat_scr, mmra !0 :: (load store release monotonic (s32) on %ir.AlignedAddr)
+  ; CHECK-NEXT:   [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64 = V_CMP_EQ_U32_e64 [[FLAT_ATOMIC_CMPSWAP_RTN]], [[PHI1]], implicit $exec, mmra !0
+  ; CHECK-NEXT:   [[SI_IF_BREAK:%[0-9]+]]:sreg_64 = SI_IF_BREAK killed [[V_CMP_EQ_U32_e64_]], [[PHI]], implicit-def dead $scc
+  ; CHECK-NEXT:   SI_LOOP [[SI_IF_BREAK]], %bb.1, implicit-def dead $exec, implicit-def dead $scc, implicit $exec
+  ; CHECK-NEXT:   S_BRANCH %bb.2
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.2.atomicrmw.end:
+  ; CHECK-NEXT:   [[PHI2:%[0-9]+]]:sreg_64 = PHI [[SI_IF_BREAK]], %bb.1
+  ; CHECK-NEXT:   SI_END_CF [[PHI2]], implicit-def dead $exec, implicit-def dead $scc, implicit $exec
+  ; CHECK-NEXT:   SI_RETURN
+  %C = call <32 x half> @f()
+  %old.2 = atomicrmw add ptr %ptr, i8 0 release, align 1, !mmra !0
+  ret void
+}
+
 attributes #0 = { memory(read) }
 attributes #1 = { memory(write) }
 
diff --git a/llvm/test/CodeGen/AMDGPU/mode-register-fptrunc.gfx11plus-fake16.mir b/llvm/test/CodeGen/AMDGPU/mode-register-fptrunc.gfx11plus-fake16.mir
new file mode 100644
index 0000000000000..8667934d70ff0
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/mode-register-fptrunc.gfx11plus-fake16.mir
@@ -0,0 +1,73 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -mtriple=amdgcn -mcpu=gfx1100 -mattr=-real-true16 -run-pass si-mode-register  %s -o - | FileCheck %s --check-prefixes=GFX11
+
+---
+name: ftrunc_tonearest
+
+body: |
+  bb.0:
+    liveins: $sgpr0
+    ; GFX11-LABEL: name: ftrunc_tonearest
+    ; GFX11: liveins: $sgpr0
+    ; GFX11-NEXT: {{  $}}
+    ; GFX11-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
+    ; GFX11-NEXT: $vgpr1 = V_CVT_F16_F32_fake16_e32 $vgpr0, implicit $mode, implicit $exec
+    ; GFX11-NEXT: S_ENDPGM 0
+    $vgpr0 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
+    $vgpr1 = FPTRUNC_ROUND_F16_F32_PSEUDO_fake16_e32 $vgpr0, 0, implicit $mode, implicit $exec
+    S_ENDPGM 0
+...
+
+---
+name: ftrunc_upward
+
+body: |
+  bb.0:
+    liveins: $sgpr0
+    ; GFX11-LABEL: name: ftrunc_upward
+    ; GFX11: liveins: $sgpr0
+    ; GFX11-NEXT: {{  $}}
+    ; GFX11-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
+    ; GFX11-NEXT: S_SETREG_IMM32_B32 1, 129, implicit-def $mode, implicit $mode
+    ; GFX11-NEXT: $vgpr1 = V_CVT_F16_F32_fake16_e32 $vgpr0, implicit $mode, implicit $exec
+    ; GFX11-NEXT: S_ENDPGM 0
+    $vgpr0 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
+    $vgpr1 = FPTRUNC_ROUND_F16_F32_PSEUDO_fake16_e32 $vgpr0, 1, implicit $mode, implicit $exec
+    S_ENDPGM 0
+...
+
+---
+name: ftrunc_downward
+
+body: |
+  bb.0:
+    liveins: $sgpr0
+    ; GFX11-LABEL: name: ftrunc_downward
+    ; GFX11: liveins: $sgpr0
+    ; GFX11-NEXT: {{  $}}
+    ; GFX11-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
+    ; GFX11-NEXT: S_SETREG_IMM32_B32 1, 193, implicit-def $mode, implicit $mode
+    ; GFX11-NEXT: $vgpr0 = V_CVT_F16_F32_fake16_e32 $vgpr1, implicit $mode, implicit $exec
+    ; GFX11-NEXT: S_ENDPGM 0
+    $vgpr1 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
+    $vgpr0 = FPTRUNC_ROUND_F16_F32_PSEUDO_fake16_e32 $vgpr1, 2, implicit $mode, implicit $exec
+    S_ENDPGM 0
+...
+
+---
+name: ftrunc_towardzero
+
+body: |
+  bb.0:
+    liveins: $sgpr0
+    ; GFX11-LABEL: name: ftrunc_towardzero
+    ; GFX11: liveins: $sgpr0
+    ; GFX11-NEXT: {{  $}}
+    ; GFX11-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
+    ; GFX11-NEXT: S_SETREG_IMM32_B32 3, 2177, implicit-def $mode, implicit $mode
+    ; GFX11-NEXT: $vgpr1 = V_CVT_F16_F32_fake16_e32 $vgpr0, implicit $mode, implicit $exec
+    ; GFX11-NEXT: S_ENDPGM 0
+    $vgpr0 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
+    $vgpr1 = FPTRUNC_ROUND_F16_F32_PSEUDO_fake16_e32 $vgpr0, 3, implicit $mode, implicit $exec
+    S_ENDPGM 0
+...
diff --git a/llvm/test/CodeGen/AMDGPU/mode-register-fptrunc.gfx11plus.mir b/llvm/test/CodeGen/AMDGPU/mode-register-fptrunc.gfx11plus.mir
new file mode 100644
index 0000000000000..9b439af4bf7b4
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/mode-register-fptrunc.gfx11plus.mir
@@ -0,0 +1,73 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -mtriple=amdgcn -mcpu=gfx1100 -mattr=+real-true16 -run-pass si-mode-register  %s -o - | FileCheck %s --check-prefixes=GFX11
+
+---
+name: ftrunc_tonearest
+
+body: |
+  bb.0:
+    liveins: $sgpr0
+    ; GFX11-LABEL: name: ftrunc_tonearest
+    ; GFX11: liveins: $sgpr0
+    ; GFX11-NEXT: {{  $}}
+    ; GFX11-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
+    ; GFX11-NEXT: $vgpr1_hi16 = V_CVT_F16_F32_t16_e64 0, $vgpr0, 0, 0, 0, implicit $mode, implicit $exec
+    ; GFX11-NEXT: S_ENDPGM 0
+    $vgpr0 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
+    $vgpr1_hi16 = FPTRUNC_ROUND_F16_F32_PSEUDO_t16_e64 0, $vgpr0, 0, 0, 0, 0, implicit $mode, implicit $exec
+    S_ENDPGM 0
+...
+
+---
+name: ftrunc_upward
+
+body: |
+  bb.0:
+    liveins: $sgpr0
+    ; GFX11-LABEL: name: ftrunc_upward
+    ; GFX11: liveins: $sgpr0
+    ; GFX11-NEXT: {{  $}}
+    ; GFX11-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
+    ; GFX11-NEXT: S_SETREG_IMM32_B32 1, 129, implicit-def $mode, implicit $mode
+    ; GFX11-NEXT: $vgpr1_hi16 = V_CVT_F16_F32_t16_e64 0, $vgpr0, 0, 0, 0, implicit $mode, implicit $exec
+    ; GFX11-NEXT: S_ENDPGM 0
+    $vgpr0 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
+    $vgpr1_hi16 = FPTRUNC_ROUND_F16_F32_PSEUDO_t16_e64 0, $vgpr0, 0, 0, 0, 1, implicit $mode, implicit $exec
+    S_ENDPGM 0
+...
+
+---
+name: ftrunc_downward
+
+body: |
+  bb.0:
+    liveins: $sgpr0
+    ; GFX11-LABEL: name: ftrunc_downward
+    ; GFX11: liveins: $sgpr0
+    ; GFX11-NEXT: {{  $}}
+    ; GFX11-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
+    ; GFX11-NEXT: S_SETREG_IMM32_B32 1, 193, implicit-def $mode, implicit $mode
+    ; GFX11-NEXT: $vgpr0_hi16 = V_CVT_F16_F32_t16_e64 0, $vgpr1, 0, 0, 0, implicit $mode, implicit $exec
+    ; GFX11-NEXT: S_ENDPGM 0
+    $vgpr1 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
+    $vgpr0_hi16 = FPTRUNC_ROUND_F16_F32_PSEUDO_t16_e64 0, $vgpr1, 0, 0, 0, 2, implicit $mode, implicit $exec
+    S_ENDPGM 0
+...
+
+---
+name: ftrunc_towardzero
+
+body: |
+  bb.0:
+    liveins: $sgpr0
+    ; GFX11-LABEL: name: ftrunc_towardzero
+    ; GFX11: liveins: $sgpr0
+    ; GFX11-NEXT: {{  $}}
+    ; GFX11-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
+    ; GFX11-NEXT: S_SETREG_IMM32_B32 3, 2177, implicit-def $mode, implicit $mode
+    ; GFX11-NEXT: $vgpr1_hi16 = V_CVT_F16_F32_t16_e64 0, $vgpr0, 0, 0, 0, implicit $mode, implicit $exec
+    ; GFX11-NEXT: S_ENDPGM 0
+    $vgpr0 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
+    $vgpr1_hi16 = FPTRUNC_ROUND_F16_F32_PSEUDO_t16_e64 0, $vgpr0, 0, 0, 0, 3, implicit $mode, implicit $exec
+    S_ENDPGM 0
+...
diff --git a/llvm/test/CodeGen/AMDGPU/mode-register-fptrunc.mir b/llvm/test/CodeGen/AMDGPU/mode-register-fptrunc.mir
index 67eb719fd2c0d..a28f347603ab7 100644
--- a/llvm/test/CodeGen/AMDGPU/mode-register-fptrunc.mir
+++ b/llvm/test/CodeGen/AMDGPU/mode-register-fptrunc.mir
@@ -1,7 +1,6 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
 # RUN: llc -mtriple=amdgcn -mcpu=gfx900 -run-pass si-mode-register  %s -o - | FileCheck %s --check-prefixes=CHECK
 # RUN: llc -mtriple=amdgcn -mcpu=gfx1010 -run-pass si-mode-register  %s -o - | FileCheck %s --check-prefixes=CHECK
-# RUN: llc -mtriple=amdgcn -mcpu=gfx1100 -run-pass si-mode-register  %s -o - | FileCheck %s --check-prefixes=GFX11
 
 ---
 name: ftrunc_tonearest
@@ -15,13 +14,6 @@ body: |
     ; CHECK-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
     ; CHECK-NEXT: $vgpr1 = V_CVT_F16_F32_e32 $vgpr0, implicit $mode, implicit $exec
     ; CHECK-NEXT: S_ENDPGM 0
-    ;
-    ; GFX11-LABEL: name: ftrunc_tonearest
-    ; GFX11: liveins: $sgpr0
-    ; GFX11-NEXT: {{  $}}
-    ; GFX11-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
-    ; GFX11-NEXT: $vgpr1 = V_CVT_F16_F32_fake16_e64 0, $vgpr0, 0, 0, implicit $mode, implicit $exec
-    ; GFX11-NEXT: S_ENDPGM 0
     $vgpr0 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
     $vgpr1 = FPTRUNC_ROUND_F16_F32_PSEUDO $vgpr0, 0, implicit $mode, implicit $exec
     S_ENDPGM 0
@@ -39,14 +31,6 @@ body: |
     ; CHECK-NEXT: S_SETREG_IMM32_B32 1, 129, implicit-def $mode, implicit $mode
     ; CHECK-NEXT: $vgpr1 = V_CVT_F16_F32_e32 $vgpr0, implicit $mode, implicit $exec
     ; CHECK-NEXT: S_ENDPGM 0
-    ;
-    ; GFX11-LABEL: name: ftrunc_upward
-    ; GFX11: liveins: $sgpr0
-    ; GFX11-NEXT: {{  $}}
-    ; GFX11-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
-    ; GFX11-NEXT: S_SETREG_IMM32_B32 1, 129, implicit-def $mode, implicit $mode
-    ; GFX11-NEXT: $vgpr1 = V_CVT_F16_F32_fake16_e64 0, $vgpr0, 0, 0, implicit $mode, implicit $exec
-    ; GFX11-NEXT: S_ENDPGM 0
     $vgpr0 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
     $vgpr1 = FPTRUNC_ROUND_F16_F32_PSEUDO $vgpr0, 1, implicit $mode, implicit $exec
     S_ENDPGM 0
@@ -64,14 +48,6 @@ body: |
     ; CHECK-NEXT: S_SETREG_IMM32_B32 1, 193, implicit-def $mode, implicit $mode
     ; CHECK-NEXT: $vgpr0 = V_CVT_F16_F32_e32 $vgpr1, implicit $mode, implicit $exec
     ; CHECK-NEXT: S_ENDPGM 0
-    ;
-    ; GFX11-LABEL: name: ftrunc_downward
-    ; GFX11: liveins: $sgpr0
-    ; GFX11-NEXT: {{  $}}
-    ; GFX11-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
-    ; GFX11-NEXT: S_SETREG_IMM32_B32 1, 193, implicit-def $mode, implicit $mode
-    ; GFX11-NEXT: $vgpr0 = V_CVT_F16_F32_fake16_e64 0, $vgpr1, 0, 0, implicit $mode, implicit $exec
-    ; GFX11-NEXT: S_ENDPGM 0
     $vgpr1 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
     $vgpr0 = FPTRUNC_ROUND_F16_F32_PSEUDO $vgpr1, 2, implicit $mode, implicit $exec
     S_ENDPGM 0
@@ -89,14 +65,6 @@ body: |
     ; CHECK-NEXT: S_SETREG_IMM32_B32 3, 2177, implicit-def $mode, implicit $mode
     ; CHECK-NEXT: $vgpr1 = V_CVT_F16_F32_e32 $vgpr0, implicit $mode, implicit $exec
     ; CHECK-NEXT: S_ENDPGM 0
-    ;
-    ; GFX11-LABEL: name: ftrunc_towardzero
-    ; GFX11: liveins: $sgpr0
-    ; GFX11-NEXT: {{  $}}
-    ; GFX11-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
-    ; GFX11-NEXT: S_SETREG_IMM32_B32 3, 2177, implicit-def $mode, implicit $mode
-    ; GFX11-NEXT: $vgpr1 = V_CVT_F16_F32_fake16_e64 0, $vgpr0, 0, 0, implicit $mode, implicit $exec
-    ; GFX11-NEXT: S_ENDPGM 0
     $vgpr0 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
     $vgpr1 = FPTRUNC_ROUND_F16_F32_PSEUDO $vgpr0, 3, implicit $mode, implicit $exec
     S_ENDPGM 0
diff --git a/llvm/test/CodeGen/AMDGPU/move-to-valu-atomicrmw-system.ll b/llvm/test/CodeGen/AMDGPU/move-to-valu-atomicrmw-system.ll
index fab5d386446d3..6f21df3a06ce7 100644
--- a/llvm/test/CodeGen/AMDGPU/move-to-valu-atomicrmw-system.ll
+++ b/llvm/test/CodeGen/AMDGPU/move-to-valu-atomicrmw-system.ll
@@ -28,29 +28,28 @@ define amdgpu_kernel void @atomic_max_i32(ptr addrspace(1) %out, ptr addrspace(1
 ; GCN-NEXT:  ; %bb.1: ; %atomic
 ; GCN-NEXT:    s_mov_b32 s8, s10
 ; GCN-NEXT:    s_mov_b32 s9, s10
-; GCN-NEXT:    buffer_load_dword v4, v[1:2], s[8:11], 0 addr64 offset:400
+; GCN-NEXT:    buffer_load_dword v5, v[1:2], s[8:11], 0 addr64 offset:400
 ; GCN-NEXT:    s_load_dword s4, s[4:5], 0xf
 ; GCN-NEXT:    s_mov_b64 s[2:3], 0
 ; GCN-NEXT:  .LBB0_2: ; %atomicrmw.start
 ; GCN-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GCN-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN-NEXT:    v_max_i32_e32 v3, s4, v4
-; GCN-NEXT:    s_waitcnt expcnt(0)
-; GCN-NEXT:    v_mov_b32_e32 v6, v4
-; GCN-NEXT:    v_mov_b32_e32 v5, v3
-; GCN-NEXT:    buffer_atomic_cmpswap v[5:6], v[1:2], s[8:11], 0 addr64 offset:400 glc
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_max_i32_e32 v4, s4, v5
+; GCN-NEXT:    v_mov_b32_e32 v3, v4
+; GCN-NEXT:    v_mov_b32_e32 v4, v5
+; GCN-NEXT:    buffer_atomic_cmpswap v[3:4], v[1:2], s[8:11], 0 addr64 offset:400 glc
 ; GCN-NEXT:    s_waitcnt vmcnt(0)
 ; GCN-NEXT:    buffer_wbinvl1
-; GCN-NEXT:    v_cmp_eq_u32_e32 vcc, v5, v4
+; GCN-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v5
 ; GCN-NEXT:    s_or_b64 s[2:3], vcc, s[2:3]
-; GCN-NEXT:    v_mov_b32_e32 v4, v5
+; GCN-NEXT:    v_mov_b32_e32 v5, v3
 ; GCN-NEXT:    s_andn2_b64 exec, exec, s[2:3]
 ; GCN-NEXT:    s_cbranch_execnz .LBB0_2
 ; GCN-NEXT:  ; %bb.3: ; %atomicrmw.end
 ; GCN-NEXT:    s_or_b64 exec, exec, s[2:3]
 ; GCN-NEXT:    s_mov_b32 s3, 0xf000
 ; GCN-NEXT:    s_mov_b32 s2, -1
-; GCN-NEXT:    buffer_store_dword v5, off, s[0:3], 0
+; GCN-NEXT:    buffer_store_dword v3, off, s[0:3], 0
 ; GCN-NEXT:  .LBB0_4: ; %exit
 ; GCN-NEXT:    s_endpgm
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
diff --git a/llvm/test/CodeGen/AMDGPU/movrels-bug.mir b/llvm/test/CodeGen/AMDGPU/movrels-bug.mir
index 9c8627399f853..e20de8c745388 100644
--- a/llvm/test/CodeGen/AMDGPU/movrels-bug.mir
+++ b/llvm/test/CodeGen/AMDGPU/movrels-bug.mir
@@ -1,4 +1,5 @@
 # RUN: llc -mtriple=amdgcn -verify-machineinstrs -run-pass post-RA-sched  %s -o - | FileCheck %s
+# RUN: llc -mtriple=amdgcn -passes=post-RA-sched  %s -o - | FileCheck %s
 
 # This tests a situation where a sub-register of a killed super-register operand
 # of V_MOVRELS happens to have an undef use later on. This leads to the post RA
diff --git a/llvm/test/CodeGen/AMDGPU/mul.ll b/llvm/test/CodeGen/AMDGPU/mul.ll
index 01eb1b1a353d1..9b4693f61147a 100644
--- a/llvm/test/CodeGen/AMDGPU/mul.ll
+++ b/llvm/test/CodeGen/AMDGPU/mul.ll
@@ -1459,8 +1459,8 @@ define amdgpu_kernel void @s_mul_i1(ptr addrspace(1) %out, [8 x i32], i1 %a, [8
 ; SI-NEXT:    s_mov_b32 s3, 0xf000
 ; SI-NEXT:    s_mov_b32 s2, -1
 ; SI-NEXT:    s_waitcnt lgkmcnt(0)
-; SI-NEXT:    s_mul_i32 s6, s6, s7
-; SI-NEXT:    s_and_b32 s4, s6, 1
+; SI-NEXT:    s_and_b32 s4, s6, s7
+; SI-NEXT:    s_and_b32 s4, s4, 1
 ; SI-NEXT:    v_mov_b32_e32 v0, s4
 ; SI-NEXT:    buffer_store_byte v0, off, s[0:3], 0
 ; SI-NEXT:    s_endpgm
@@ -1473,8 +1473,8 @@ define amdgpu_kernel void @s_mul_i1(ptr addrspace(1) %out, [8 x i32], i1 %a, [8
 ; VI-NEXT:    s_mov_b32 s3, 0xf000
 ; VI-NEXT:    s_mov_b32 s2, -1
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
-; VI-NEXT:    s_mul_i32 s6, s6, s7
-; VI-NEXT:    s_and_b32 s4, s6, 1
+; VI-NEXT:    s_and_b32 s4, s6, s7
+; VI-NEXT:    s_and_b32 s4, s4, 1
 ; VI-NEXT:    v_mov_b32_e32 v0, s4
 ; VI-NEXT:    buffer_store_byte v0, off, s[0:3], 0
 ; VI-NEXT:    s_endpgm
@@ -1487,8 +1487,8 @@ define amdgpu_kernel void @s_mul_i1(ptr addrspace(1) %out, [8 x i32], i1 %a, [8
 ; GFX9-NEXT:    s_mov_b32 s3, 0xf000
 ; GFX9-NEXT:    s_mov_b32 s2, -1
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NEXT:    s_mul_i32 s6, s6, s7
-; GFX9-NEXT:    s_and_b32 s4, s6, 1
+; GFX9-NEXT:    s_and_b32 s4, s6, s7
+; GFX9-NEXT:    s_and_b32 s4, s4, 1
 ; GFX9-NEXT:    v_mov_b32_e32 v0, s4
 ; GFX9-NEXT:    buffer_store_byte v0, off, s[0:3], 0
 ; GFX9-NEXT:    s_endpgm
@@ -1500,7 +1500,7 @@ define amdgpu_kernel void @s_mul_i1(ptr addrspace(1) %out, [8 x i32], i1 %a, [8
 ; GFX10-NEXT:    s_load_dword s3, s[4:5], 0x70
 ; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX10-NEXT:    s_mul_i32 s2, s2, s3
+; GFX10-NEXT:    s_and_b32 s2, s2, s3
 ; GFX10-NEXT:    s_mov_b32 s3, 0x31016000
 ; GFX10-NEXT:    s_and_b32 s2, s2, 1
 ; GFX10-NEXT:    v_mov_b32_e32 v0, s2
@@ -1515,7 +1515,7 @@ define amdgpu_kernel void @s_mul_i1(ptr addrspace(1) %out, [8 x i32], i1 %a, [8
 ; GFX11-NEXT:    s_load_b32 s3, s[4:5], 0x70
 ; GFX11-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
 ; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX11-NEXT:    s_mul_i32 s2, s2, s3
+; GFX11-NEXT:    s_and_b32 s2, s2, s3
 ; GFX11-NEXT:    s_mov_b32 s3, 0x31016000
 ; GFX11-NEXT:    s_and_b32 s2, s2, 1
 ; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
@@ -1531,7 +1531,7 @@ define amdgpu_kernel void @s_mul_i1(ptr addrspace(1) %out, [8 x i32], i1 %a, [8
 ; GFX12-NEXT:    s_load_b32 s3, s[4:5], 0x70
 ; GFX12-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
 ; GFX12-NEXT:    s_wait_kmcnt 0x0
-; GFX12-NEXT:    s_mul_i32 s2, s2, s3
+; GFX12-NEXT:    s_and_b32 s2, s2, s3
 ; GFX12-NEXT:    s_mov_b32 s3, 0x31016000
 ; GFX12-NEXT:    s_and_b32 s2, s2, 1
 ; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
@@ -1555,7 +1555,7 @@ define amdgpu_kernel void @s_mul_i1(ptr addrspace(1) %out, [8 x i32], i1 %a, [8
 ; EG-NEXT:     MOV * T0.X, 0.0,
 ; EG-NEXT:    ALU clause starting at 11:
 ; EG-NEXT:     AND_INT T0.W, KC0[2].Y, literal.x,
-; EG-NEXT:     MULLO_INT * T0.X, T1.X, T0.X,
+; EG-NEXT:     AND_INT * T1.W, T1.X, T0.X,
 ; EG-NEXT:    3(4.203895e-45), 0(0.000000e+00)
 ; EG-NEXT:     AND_INT T1.W, PS, 1,
 ; EG-NEXT:     LSHL * T0.W, PV.W, literal.x,
@@ -1589,7 +1589,7 @@ define amdgpu_kernel void @v_mul_i1(ptr addrspace(1) %out, ptr addrspace(1) %in)
 ; SI-NEXT:    s_mov_b32 s4, s0
 ; SI-NEXT:    s_mov_b32 s5, s1
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_mul_lo_u32 v0, v0, v1
+; SI-NEXT:    v_and_b32_e32 v0, v0, v1
 ; SI-NEXT:    v_and_b32_e32 v0, 1, v0
 ; SI-NEXT:    buffer_store_byte v0, off, s[4:7], 0
 ; SI-NEXT:    s_endpgm
@@ -1609,7 +1609,7 @@ define amdgpu_kernel void @v_mul_i1(ptr addrspace(1) %out, ptr addrspace(1) %in)
 ; VI-NEXT:    s_mov_b32 s4, s0
 ; VI-NEXT:    s_mov_b32 s5, s1
 ; VI-NEXT:    s_waitcnt vmcnt(0)
-; VI-NEXT:    v_mul_lo_u32 v0, v0, v1
+; VI-NEXT:    v_and_b32_e32 v0, v0, v1
 ; VI-NEXT:    v_and_b32_e32 v0, 1, v0
 ; VI-NEXT:    buffer_store_byte v0, off, s[4:7], 0
 ; VI-NEXT:    s_endpgm
@@ -1629,7 +1629,7 @@ define amdgpu_kernel void @v_mul_i1(ptr addrspace(1) %out, ptr addrspace(1) %in)
 ; GFX9-NEXT:    s_mov_b32 s4, s0
 ; GFX9-NEXT:    s_mov_b32 s5, s1
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    v_mul_lo_u32 v0, v0, v1
+; GFX9-NEXT:    v_and_b32_e32 v0, v0, v1
 ; GFX9-NEXT:    v_and_b32_e32 v0, 1, v0
 ; GFX9-NEXT:    buffer_store_byte v0, off, s[4:7], 0
 ; GFX9-NEXT:    s_endpgm
@@ -1650,7 +1650,7 @@ define amdgpu_kernel void @v_mul_i1(ptr addrspace(1) %out, ptr addrspace(1) %in)
 ; GFX10-NEXT:    s_mov_b32 s4, s0
 ; GFX10-NEXT:    s_mov_b32 s5, s1
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
-; GFX10-NEXT:    v_mul_lo_u32 v0, v0, v1
+; GFX10-NEXT:    v_and_b32_e32 v0, v0, v1
 ; GFX10-NEXT:    v_and_b32_e32 v0, 1, v0
 ; GFX10-NEXT:    buffer_store_byte v0, off, s[4:7], 0
 ; GFX10-NEXT:    s_endpgm
@@ -1671,7 +1671,7 @@ define amdgpu_kernel void @v_mul_i1(ptr addrspace(1) %out, ptr addrspace(1) %in)
 ; GFX11-NEXT:    s_mov_b32 s4, s0
 ; GFX11-NEXT:    s_mov_b32 s5, s1
 ; GFX11-NEXT:    s_waitcnt vmcnt(0)
-; GFX11-NEXT:    v_mul_lo_u32 v0, v0, v1
+; GFX11-NEXT:    v_and_b32_e32 v0, v0, v1
 ; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX11-NEXT:    v_and_b32_e32 v0, 1, v0
 ; GFX11-NEXT:    buffer_store_b8 v0, off, s[4:7], 0
@@ -1693,7 +1693,7 @@ define amdgpu_kernel void @v_mul_i1(ptr addrspace(1) %out, ptr addrspace(1) %in)
 ; GFX12-NEXT:    s_mov_b32 s4, s0
 ; GFX12-NEXT:    s_mov_b32 s5, s1
 ; GFX12-NEXT:    s_wait_loadcnt 0x0
-; GFX12-NEXT:    v_mul_lo_u32 v0, v0, v1
+; GFX12-NEXT:    v_and_b32_e32 v0, v0, v1
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-NEXT:    v_and_b32_e32 v0, 1, v0
 ; GFX12-NEXT:    buffer_store_b8 v0, off, s[4:7], null
@@ -1714,7 +1714,7 @@ define amdgpu_kernel void @v_mul_i1(ptr addrspace(1) %out, ptr addrspace(1) %in)
 ; EG-NEXT:     MOV * T0.X, KC0[2].Z,
 ; EG-NEXT:    ALU clause starting at 11:
 ; EG-NEXT:     AND_INT T0.W, KC0[2].Y, literal.x,
-; EG-NEXT:     MULLO_INT * T0.X, T0.X, T1.X,
+; EG-NEXT:     AND_INT * T1.W, T0.X, T1.X,
 ; EG-NEXT:    3(4.203895e-45), 0(0.000000e+00)
 ; EG-NEXT:     AND_INT T1.W, PS, 1,
 ; EG-NEXT:     LSHL * T0.W, PV.W, literal.x,
@@ -2653,41 +2653,38 @@ define amdgpu_kernel void @s_mul_i128(ptr addrspace(1) %out, [8 x i32], i128 %a,
 ;
 ; VI-LABEL: s_mul_i128:
 ; VI:       ; %bb.0: ; %entry
-; VI-NEXT:    s_load_dwordx4 s[8:11], s[4:5], 0x4c
-; VI-NEXT:    s_load_dwordx4 s[12:15], s[4:5], 0x7c
-; VI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
+; VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x4c
+; VI-NEXT:    s_load_dwordx4 s[8:11], s[4:5], 0x7c
+; VI-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x24
 ; VI-NEXT:    v_mov_b32_e32 v5, 0
-; VI-NEXT:    s_mov_b32 s3, 0xf000
+; VI-NEXT:    s_mov_b32 s7, 0xf000
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
-; VI-NEXT:    v_mov_b32_e32 v0, s10
-; VI-NEXT:    v_mad_u64_u32 v[2:3], s[4:5], s12, v0, 0
-; VI-NEXT:    s_mul_i32 s4, s12, s11
-; VI-NEXT:    v_mov_b32_e32 v6, s12
-; VI-NEXT:    v_add_u32_e32 v3, vcc, s4, v3
-; VI-NEXT:    v_mad_u64_u32 v[0:1], s[4:5], s8, v6, 0
-; VI-NEXT:    s_mul_i32 s6, s13, s10
-; VI-NEXT:    v_add_u32_e32 v3, vcc, s6, v3
+; VI-NEXT:    v_mov_b32_e32 v0, s2
+; VI-NEXT:    v_mad_u64_u32 v[2:3], s[12:13], s8, v0, 0
+; VI-NEXT:    s_mul_i32 s3, s8, s3
+; VI-NEXT:    v_mov_b32_e32 v6, s8
+; VI-NEXT:    v_add_u32_e32 v3, vcc, s3, v3
+; VI-NEXT:    s_mul_i32 s12, s9, s2
+; VI-NEXT:    v_mad_u64_u32 v[0:1], s[2:3], s0, v6, 0
+; VI-NEXT:    v_add_u32_e32 v3, vcc, s12, v3
 ; VI-NEXT:    v_mov_b32_e32 v4, v1
-; VI-NEXT:    v_mad_u64_u32 v[6:7], s[4:5], s9, v6, v[4:5]
-; VI-NEXT:    v_mov_b32_e32 v8, s8
-; VI-NEXT:    v_mad_u64_u32 v[1:2], s[4:5], s14, v8, v[2:3]
-; VI-NEXT:    v_mov_b32_e32 v3, v7
-; VI-NEXT:    v_mov_b32_e32 v7, v5
-; VI-NEXT:    v_mov_b32_e32 v8, s13
-; VI-NEXT:    v_mad_u64_u32 v[4:5], s[4:5], s8, v8, v[6:7]
-; VI-NEXT:    s_mul_i32 s6, s15, s8
-; VI-NEXT:    v_add_u32_e32 v6, vcc, s6, v2
-; VI-NEXT:    v_mov_b32_e32 v2, v5
-; VI-NEXT:    v_add_u32_e32 v2, vcc, v3, v2
-; VI-NEXT:    v_addc_u32_e64 v3, s[4:5], 0, 0, vcc
-; VI-NEXT:    v_mad_u64_u32 v[2:3], s[4:5], s9, v8, v[2:3]
-; VI-NEXT:    s_mul_i32 s6, s14, s9
-; VI-NEXT:    v_add_u32_e32 v5, vcc, s6, v6
-; VI-NEXT:    v_add_u32_e32 v2, vcc, v2, v1
-; VI-NEXT:    s_mov_b32 s2, -1
-; VI-NEXT:    v_addc_u32_e32 v3, vcc, v3, v5, vcc
-; VI-NEXT:    v_mov_b32_e32 v1, v4
-; VI-NEXT:    buffer_store_dwordx4 v[0:3], off, s[0:3], 0
+; VI-NEXT:    v_mad_u64_u32 v[6:7], s[2:3], s1, v6, v[4:5]
+; VI-NEXT:    v_mov_b32_e32 v8, s0
+; VI-NEXT:    v_mad_u64_u32 v[8:9], s[2:3], s10, v8, v[2:3]
+; VI-NEXT:    v_mov_b32_e32 v4, v6
+; VI-NEXT:    v_mov_b32_e32 v6, s9
+; VI-NEXT:    v_mad_u64_u32 v[1:2], s[2:3], s0, v6, v[4:5]
+; VI-NEXT:    s_mul_i32 s8, s11, s0
+; VI-NEXT:    v_add_u32_e32 v4, vcc, s8, v9
+; VI-NEXT:    v_add_u32_e32 v2, vcc, v7, v2
+; VI-NEXT:    v_addc_u32_e64 v3, s[2:3], 0, 0, vcc
+; VI-NEXT:    s_mul_i32 s8, s10, s1
+; VI-NEXT:    v_mad_u64_u32 v[2:3], s[0:1], s1, v6, v[2:3]
+; VI-NEXT:    v_add_u32_e32 v4, vcc, s8, v4
+; VI-NEXT:    v_add_u32_e32 v2, vcc, v2, v8
+; VI-NEXT:    s_mov_b32 s6, -1
+; VI-NEXT:    v_addc_u32_e32 v3, vcc, v3, v4, vcc
+; VI-NEXT:    buffer_store_dwordx4 v[0:3], off, s[4:7], 0
 ; VI-NEXT:    s_endpgm
 ;
 ; GFX9-LABEL: s_mul_i128:
@@ -3012,52 +3009,49 @@ define amdgpu_kernel void @v_mul_i128(ptr addrspace(1) %out, ptr addrspace(1) %a
 ; VI-NEXT:    v_add_u32_e32 v3, vcc, v15, v3
 ; VI-NEXT:    v_add_u32_e32 v15, vcc, v3, v2
 ; VI-NEXT:    v_mad_u64_u32 v[2:3], s[0:1], v1, v4, v[9:10]
-; VI-NEXT:    v_mov_b32_e32 v4, v3
-; VI-NEXT:    v_mov_b32_e32 v3, v10
-; VI-NEXT:    v_mad_u64_u32 v[2:3], s[0:1], v0, v5, v[2:3]
-; VI-NEXT:    v_mad_u64_u32 v[9:10], s[0:1], v6, v0, v[14:15]
-; VI-NEXT:    v_add_u32_e32 v3, vcc, v4, v3
-; VI-NEXT:    v_addc_u32_e64 v4, s[0:1], 0, 0, vcc
-; VI-NEXT:    v_mul_lo_u32 v0, v7, v0
-; VI-NEXT:    v_mad_u64_u32 v[3:4], s[0:1], v1, v5, v[3:4]
-; VI-NEXT:    v_mul_lo_u32 v1, v6, v1
-; VI-NEXT:    v_add_u32_e32 v0, vcc, v0, v10
-; VI-NEXT:    v_add_u32_e32 v0, vcc, v1, v0
-; VI-NEXT:    v_add_u32_e32 v10, vcc, v3, v9
-; VI-NEXT:    v_addc_u32_e32 v11, vcc, v4, v0, vcc
+; VI-NEXT:    v_mad_u64_u32 v[14:15], s[0:1], v6, v0, v[14:15]
 ; VI-NEXT:    v_mov_b32_e32 v9, v2
+; VI-NEXT:    v_mul_lo_u32 v2, v7, v0
+; VI-NEXT:    v_mad_u64_u32 v[9:10], s[0:1], v0, v5, v[9:10]
+; VI-NEXT:    v_mul_lo_u32 v4, v6, v1
+; VI-NEXT:    v_add_u32_e32 v6, vcc, v2, v15
+; VI-NEXT:    v_add_u32_e32 v2, vcc, v3, v10
+; VI-NEXT:    v_addc_u32_e64 v3, s[0:1], 0, 0, vcc
+; VI-NEXT:    v_mad_u64_u32 v[0:1], s[0:1], v1, v5, v[2:3]
+; VI-NEXT:    v_add_u32_e32 v2, vcc, v4, v6
+; VI-NEXT:    v_add_u32_e32 v10, vcc, v0, v14
+; VI-NEXT:    v_addc_u32_e32 v11, vcc, v1, v2, vcc
 ; VI-NEXT:    flat_store_dwordx4 v[12:13], v[8:11]
 ; VI-NEXT:    s_endpgm
 ;
 ; GFX9-LABEL: v_mul_i128:
 ; GFX9:       ; %bb.0: ; %entry
 ; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x2c
-; GFX9-NEXT:    v_lshlrev_b32_e32 v12, 4, v0
+; GFX9-NEXT:    v_lshlrev_b32_e32 v14, 4, v0
 ; GFX9-NEXT:    v_mov_b32_e32 v11, 0
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NEXT:    global_load_dwordx4 v[0:3], v12, s[0:1]
-; GFX9-NEXT:    global_load_dwordx4 v[4:7], v12, s[2:3]
+; GFX9-NEXT:    global_load_dwordx4 v[0:3], v14, s[0:1]
+; GFX9-NEXT:    global_load_dwordx4 v[4:7], v14, s[2:3]
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_mul_lo_u32 v10, v5, v2
-; GFX9-NEXT:    v_mul_lo_u32 v13, v4, v3
+; GFX9-NEXT:    v_mul_lo_u32 v12, v4, v3
 ; GFX9-NEXT:    v_mad_u64_u32 v[8:9], s[0:1], v4, v2, 0
 ; GFX9-NEXT:    v_mad_u64_u32 v[2:3], s[0:1], v0, v4, 0
-; GFX9-NEXT:    v_add3_u32 v9, v9, v13, v10
-; GFX9-NEXT:    v_mul_lo_u32 v13, v6, v1
+; GFX9-NEXT:    v_add3_u32 v9, v9, v12, v10
+; GFX9-NEXT:    v_mul_lo_u32 v15, v6, v1
 ; GFX9-NEXT:    v_mov_b32_e32 v10, v3
-; GFX9-NEXT:    v_mad_u64_u32 v[3:4], s[0:1], v1, v4, v[10:11]
-; GFX9-NEXT:    v_mad_u64_u32 v[8:9], s[0:1], v6, v0, v[8:9]
-; GFX9-NEXT:    v_mov_b32_e32 v10, v4
-; GFX9-NEXT:    v_mov_b32_e32 v4, v11
-; GFX9-NEXT:    v_mad_u64_u32 v[3:4], s[0:1], v0, v5, v[3:4]
-; GFX9-NEXT:    v_mul_lo_u32 v0, v7, v0
-; GFX9-NEXT:    v_add_co_u32_e32 v10, vcc, v10, v4
-; GFX9-NEXT:    v_addc_co_u32_e64 v11, s[0:1], 0, 0, vcc
-; GFX9-NEXT:    v_mad_u64_u32 v[4:5], s[0:1], v1, v5, v[10:11]
-; GFX9-NEXT:    v_add3_u32 v0, v0, v9, v13
-; GFX9-NEXT:    v_add_co_u32_e32 v4, vcc, v4, v8
-; GFX9-NEXT:    v_addc_co_u32_e32 v5, vcc, v5, v0, vcc
-; GFX9-NEXT:    global_store_dwordx4 v12, v[2:5], s[2:3]
+; GFX9-NEXT:    v_mad_u64_u32 v[12:13], s[0:1], v1, v4, v[10:11]
+; GFX9-NEXT:    v_mov_b32_e32 v10, v12
+; GFX9-NEXT:    v_mad_u64_u32 v[3:4], s[0:1], v0, v5, v[10:11]
+; GFX9-NEXT:    v_mul_lo_u32 v10, v7, v0
+; GFX9-NEXT:    v_mad_u64_u32 v[6:7], s[0:1], v6, v0, v[8:9]
+; GFX9-NEXT:    v_add_co_u32_e32 v8, vcc, v13, v4
+; GFX9-NEXT:    v_addc_co_u32_e64 v9, s[0:1], 0, 0, vcc
+; GFX9-NEXT:    v_mad_u64_u32 v[0:1], s[0:1], v1, v5, v[8:9]
+; GFX9-NEXT:    v_add3_u32 v5, v10, v7, v15
+; GFX9-NEXT:    v_add_co_u32_e32 v4, vcc, v0, v6
+; GFX9-NEXT:    v_addc_co_u32_e32 v5, vcc, v1, v5, vcc
+; GFX9-NEXT:    global_store_dwordx4 v14, v[2:5], s[2:3]
 ; GFX9-NEXT:    s_endpgm
 ;
 ; GFX10-LABEL: v_mul_i128:
@@ -3071,22 +3065,20 @@ define amdgpu_kernel void @v_mul_i128(ptr addrspace(1) %out, ptr addrspace(1) %a
 ; GFX10-NEXT:    global_load_dwordx4 v[4:7], v13, s[2:3]
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    v_mad_u64_u32 v[8:9], s0, v0, v4, 0
-; GFX10-NEXT:    v_mul_lo_u32 v15, v5, v2
+; GFX10-NEXT:    v_mul_lo_u32 v14, v5, v2
 ; GFX10-NEXT:    v_mul_lo_u32 v7, v7, v0
 ; GFX10-NEXT:    v_mad_u64_u32 v[11:12], s0, v1, v4, v[9:10]
-; GFX10-NEXT:    v_mov_b32_e32 v14, v12
-; GFX10-NEXT:    v_mov_b32_e32 v12, v10
-; GFX10-NEXT:    v_mad_u64_u32 v[9:10], s0, v0, v5, v[11:12]
+; GFX10-NEXT:    v_mov_b32_e32 v9, v11
 ; GFX10-NEXT:    v_mul_lo_u32 v11, v4, v3
 ; GFX10-NEXT:    v_mad_u64_u32 v[2:3], s0, v4, v2, 0
-; GFX10-NEXT:    v_mul_lo_u32 v12, v6, v1
-; GFX10-NEXT:    v_mov_b32_e32 v4, v10
-; GFX10-NEXT:    v_add3_u32 v3, v3, v11, v15
-; GFX10-NEXT:    v_add_co_u32 v10, s0, v14, v4
+; GFX10-NEXT:    v_mul_lo_u32 v4, v6, v1
+; GFX10-NEXT:    v_mad_u64_u32 v[9:10], s0, v0, v5, v[9:10]
+; GFX10-NEXT:    v_add3_u32 v3, v3, v11, v14
+; GFX10-NEXT:    v_add_co_u32 v10, s0, v12, v10
 ; GFX10-NEXT:    v_add_co_ci_u32_e64 v11, s0, 0, 0, s0
 ; GFX10-NEXT:    v_mad_u64_u32 v[2:3], s0, v6, v0, v[2:3]
 ; GFX10-NEXT:    v_mad_u64_u32 v[0:1], s0, v1, v5, v[10:11]
-; GFX10-NEXT:    v_add3_u32 v3, v7, v3, v12
+; GFX10-NEXT:    v_add3_u32 v3, v7, v3, v4
 ; GFX10-NEXT:    v_add_co_u32 v10, vcc_lo, v0, v2
 ; GFX10-NEXT:    v_add_co_ci_u32_e32 v11, vcc_lo, v1, v3, vcc_lo
 ; GFX10-NEXT:    global_store_dwordx4 v13, v[8:11], s[2:3]
@@ -3097,37 +3089,37 @@ define amdgpu_kernel void @v_mul_i128(ptr addrspace(1) %out, ptr addrspace(1) %a
 ; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x2c
 ; GFX11-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
 ; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT:    v_dual_mov_b32 v10, 0 :: v_dual_lshlrev_b32 v15, 4, v0
+; GFX11-NEXT:    v_dual_mov_b32 v12, 0 :: v_dual_lshlrev_b32 v17, 4, v0
 ; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX11-NEXT:    s_clause 0x1
-; GFX11-NEXT:    global_load_b128 v[0:3], v15, s[0:1]
-; GFX11-NEXT:    global_load_b128 v[4:7], v15, s[2:3]
+; GFX11-NEXT:    global_load_b128 v[0:3], v17, s[0:1]
+; GFX11-NEXT:    global_load_b128 v[4:7], v17, s[2:3]
 ; GFX11-NEXT:    s_waitcnt vmcnt(0)
 ; GFX11-NEXT:    v_mad_u64_u32 v[8:9], null, v0, v4, 0
-; GFX11-NEXT:    v_mul_lo_u32 v14, v5, v2
+; GFX11-NEXT:    v_mul_lo_u32 v18, v5, v2
 ; GFX11-NEXT:    v_mul_lo_u32 v3, v4, v3
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT:    v_mad_u64_u32 v[11:12], null, v1, v4, v[9:10]
-; GFX11-NEXT:    v_dual_mov_b32 v13, v12 :: v_dual_mov_b32 v12, v10
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-NEXT:    v_mad_u64_u32 v[9:10], null, v0, v5, v[11:12]
-; GFX11-NEXT:    v_mad_u64_u32 v[11:12], null, v4, v2, 0
+; GFX11-NEXT:    v_mad_u64_u32 v[15:16], null, v4, v2, 0
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-NEXT:    v_mov_b32_e32 v11, v9
+; GFX11-NEXT:    v_add3_u32 v16, v16, v3, v18
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-NEXT:    v_mad_u64_u32 v[13:14], null, v1, v4, v[11:12]
 ; GFX11-NEXT:    v_mul_lo_u32 v4, v6, v1
-; GFX11-NEXT:    v_mov_b32_e32 v2, v10
-; GFX11-NEXT:    v_mul_lo_u32 v10, v7, v0
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-NEXT:    v_add3_u32 v12, v12, v3, v14
-; GFX11-NEXT:    v_add_co_u32 v2, s0, v13, v2
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-NEXT:    v_mov_b32_e32 v11, v13
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-NEXT:    v_mad_u64_u32 v[9:10], null, v0, v5, v[11:12]
+; GFX11-NEXT:    v_mul_lo_u32 v12, v7, v0
+; GFX11-NEXT:    v_add_co_u32 v2, s0, v14, v10
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX11-NEXT:    v_add_co_ci_u32_e64 v3, null, 0, 0, s0
-; GFX11-NEXT:    v_mad_u64_u32 v[13:14], null, v6, v0, v[11:12]
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-NEXT:    v_mad_u64_u32 v[10:11], null, v6, v0, v[15:16]
 ; GFX11-NEXT:    v_mad_u64_u32 v[6:7], null, v1, v5, v[2:3]
-; GFX11-NEXT:    v_add3_u32 v0, v10, v14, v4
 ; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT:    v_add_co_u32 v10, vcc_lo, v6, v13
+; GFX11-NEXT:    v_add3_u32 v0, v12, v11, v4
+; GFX11-NEXT:    v_add_co_u32 v10, vcc_lo, v6, v10
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2)
 ; GFX11-NEXT:    v_add_co_ci_u32_e32 v11, vcc_lo, v7, v0, vcc_lo
-; GFX11-NEXT:    global_store_b128 v15, v[8:11], s[2:3]
+; GFX11-NEXT:    global_store_b128 v17, v[8:11], s[2:3]
 ; GFX11-NEXT:    s_endpgm
 ;
 ; GFX12-LABEL: v_mul_i128:
@@ -3142,29 +3134,27 @@ define amdgpu_kernel void @v_mul_i128(ptr addrspace(1) %out, ptr addrspace(1) %a
 ; GFX12-NEXT:    global_load_b128 v[4:7], v13, s[2:3]
 ; GFX12-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-NEXT:    v_mad_co_u64_u32 v[8:9], null, v0, v4, 0
-; GFX12-NEXT:    v_mul_lo_u32 v15, v5, v2
+; GFX12-NEXT:    v_mul_lo_u32 v14, v5, v2
 ; GFX12-NEXT:    v_mul_lo_u32 v7, v7, v0
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_mad_co_u64_u32 v[11:12], null, v1, v4, v[9:10]
-; GFX12-NEXT:    v_mov_b32_e32 v14, v12
-; GFX12-NEXT:    v_mov_b32_e32 v12, v10
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX12-NEXT:    v_mad_co_u64_u32 v[9:10], null, v0, v5, v[11:12]
+; GFX12-NEXT:    v_mov_b32_e32 v9, v11
 ; GFX12-NEXT:    v_mul_lo_u32 v11, v4, v3
 ; GFX12-NEXT:    v_mad_co_u64_u32 v[2:3], null, v4, v2, 0
-; GFX12-NEXT:    v_mul_lo_u32 v12, v6, v1
-; GFX12-NEXT:    v_mov_b32_e32 v4, v10
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-NEXT:    v_add3_u32 v3, v3, v11, v15
-; GFX12-NEXT:    v_add_co_u32 v10, s0, v14, v4
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-NEXT:    v_mul_lo_u32 v4, v6, v1
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-NEXT:    v_mad_co_u64_u32 v[9:10], null, v0, v5, v[9:10]
+; GFX12-NEXT:    v_add3_u32 v3, v3, v11, v14
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT:    v_add_co_u32 v10, s0, v12, v10
 ; GFX12-NEXT:    v_add_co_ci_u32_e64 v11, null, 0, 0, s0
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
 ; GFX12-NEXT:    v_mad_co_u64_u32 v[2:3], null, v6, v0, v[2:3]
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
 ; GFX12-NEXT:    v_mad_co_u64_u32 v[0:1], null, v1, v5, v[10:11]
-; GFX12-NEXT:    v_add3_u32 v3, v7, v3, v12
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-NEXT:    v_add3_u32 v3, v7, v3, v4
 ; GFX12-NEXT:    v_add_co_u32 v10, vcc_lo, v0, v2
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2)
 ; GFX12-NEXT:    v_add_co_ci_u32_e32 v11, vcc_lo, v1, v3, vcc_lo
 ; GFX12-NEXT:    global_store_b128 v13, v[8:11], s[2:3]
 ; GFX12-NEXT:    s_endpgm
diff --git a/llvm/test/CodeGen/AMDGPU/mul_int24.ll b/llvm/test/CodeGen/AMDGPU/mul_int24.ll
index 4302810089f0c..5452c80a373e6 100644
--- a/llvm/test/CodeGen/AMDGPU/mul_int24.ll
+++ b/llvm/test/CodeGen/AMDGPU/mul_int24.ll
@@ -740,7 +740,7 @@ define amdgpu_kernel void @simplify_i24_crash(ptr addrspace(1) %out, i32 %arg0,
 ; EG:       ; %bb.0: ; %bb
 ; EG-NEXT:    ALU_PUSH_BEFORE 1, @6, KC0[CB0:0-32], KC1[]
 ; EG-NEXT:    JUMP @5 POP:1
-; EG-NEXT:    ALU 14, @8, KC0[CB0:0-32], KC1[]
+; EG-NEXT:    ALU 12, @8, KC0[CB0:0-32], KC1[]
 ; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T0.XY, T1.X, 0
 ; EG-NEXT:    POP @5 POP:1
 ; EG-NEXT:    CF_END
@@ -748,27 +748,25 @@ define amdgpu_kernel void @simplify_i24_crash(ptr addrspace(1) %out, i32 %arg0,
 ; EG-NEXT:     SETNE_INT * T0.W, KC0[2].Z, 0.0,
 ; EG-NEXT:     PRED_SETE_INT * ExecMask,PredicateBit (MASKED), PV.W, 0.0,
 ; EG-NEXT:    ALU clause starting at 8:
-; EG-NEXT:     MOV T0.X, KC0[3].Y,
-; EG-NEXT:     MOV * T1.X, KC0[2].W,
-; EG-NEXT:     LSHL T0.W, PS, literal.x,
-; EG-NEXT:     LSHL * T1.W, PV.X, literal.x,
+; EG-NEXT:     MOV T0.W, KC0[3].Y,
+; EG-NEXT:     MOV * T1.W, KC0[2].W,
+; EG-NEXT:     LSHL T1.W, PS, literal.x,
+; EG-NEXT:     LSHL * T0.W, PV.W, literal.x,
 ; EG-NEXT:    8(1.121039e-44), 0(0.000000e+00)
-; EG-NEXT:     ASHR T1.W, PS, literal.x,
-; EG-NEXT:     ASHR * T0.W, PV.W, literal.x,
+; EG-NEXT:     ASHR T0.W, PS, literal.x,
+; EG-NEXT:     ASHR * T1.W, PV.W, literal.x,
 ; EG-NEXT:    8(1.121039e-44), 0(0.000000e+00)
 ; EG-NEXT:     MOV T2.W, KC0[2].Y,
 ; EG-NEXT:     MULLO_INT * T0.X, PS, PV.W,
 ; EG-NEXT:     LSHR T1.X, PV.W, literal.x,
-; EG-NEXT:     MOV T0.Y, PS,
-; EG-NEXT:     MOV T0.W, KC0[3].X,
-; EG-NEXT:     MOV * T0.W, KC0[3].Z,
+; EG-NEXT:     MOV * T0.Y, PS,
 ; EG-NEXT:    2(2.802597e-45), 0(0.000000e+00)
 ;
 ; CM-LABEL: simplify_i24_crash:
 ; CM:       ; %bb.0: ; %bb
 ; CM-NEXT:    ALU_PUSH_BEFORE 1, @6, KC0[CB0:0-32], KC1[]
 ; CM-NEXT:    JUMP @5 POP:1
-; CM-NEXT:    ALU 17, @8, KC0[CB0:0-32], KC1[]
+; CM-NEXT:    ALU 15, @8, KC0[CB0:0-32], KC1[]
 ; CM-NEXT:    MEM_RAT_CACHELESS STORE_DWORD T0, T1.X
 ; CM-NEXT:    POP @5 POP:1
 ; CM-NEXT:    CF_END
@@ -776,23 +774,21 @@ define amdgpu_kernel void @simplify_i24_crash(ptr addrspace(1) %out, i32 %arg0,
 ; CM-NEXT:     SETNE_INT * T0.W, KC0[2].Z, 0.0,
 ; CM-NEXT:     PRED_SETE_INT * ExecMask,PredicateBit (MASKED), PV.W, 0.0,
 ; CM-NEXT:    ALU clause starting at 8:
-; CM-NEXT:     MOV * T0.X, KC0[3].Y,
-; CM-NEXT:     MOV * T1.X, KC0[2].W,
-; CM-NEXT:     LSHL T0.Z, PV.X, literal.x,
-; CM-NEXT:     LSHL * T0.W, T0.X, literal.x,
+; CM-NEXT:     MOV T0.Z, KC0[3].Y,
+; CM-NEXT:     MOV * T0.W, KC0[2].W,
+; CM-NEXT:     LSHL T1.Z, PV.W, literal.x,
+; CM-NEXT:     LSHL * T0.W, PV.Z, literal.x,
 ; CM-NEXT:    8(1.121039e-44), 0(0.000000e+00)
 ; CM-NEXT:     MOV T0.Y, KC0[2].Y,
-; CM-NEXT:     ASHR T1.Z, PV.W, literal.x,
+; CM-NEXT:     ASHR T0.Z, PV.W, literal.x,
 ; CM-NEXT:     ASHR * T0.W, PV.Z, literal.x,
 ; CM-NEXT:    8(1.121039e-44), 0(0.000000e+00)
-; CM-NEXT:     MULLO_INT T0.X, T0.W, T1.Z,
-; CM-NEXT:     MULLO_INT T0.Y (MASKED), T0.W, T1.Z,
-; CM-NEXT:     MULLO_INT T0.Z (MASKED), T0.W, T1.Z,
-; CM-NEXT:     MULLO_INT * T0.W (MASKED), T0.W, T1.Z,
+; CM-NEXT:     MULLO_INT T0.X, T0.W, T0.Z,
+; CM-NEXT:     MULLO_INT T0.Y (MASKED), T0.W, T0.Z,
+; CM-NEXT:     MULLO_INT T0.Z (MASKED), T0.W, T0.Z,
+; CM-NEXT:     MULLO_INT * T0.W (MASKED), T0.W, T0.Z,
 ; CM-NEXT:     LSHR T1.X, T0.Y, literal.x,
-; CM-NEXT:     MOV T0.Y, PV.X,
-; CM-NEXT:     MOV T0.Z, KC0[3].X,
-; CM-NEXT:     MOV * T0.W, KC0[3].Z,
+; CM-NEXT:     MOV * T0.Y, PV.X,
 ; CM-NEXT:    2(2.802597e-45), 0(0.000000e+00)
 bb:
   %cmp = icmp eq i32 %arg0, 0
diff --git a/llvm/test/CodeGen/AMDGPU/nested-calls.ll b/llvm/test/CodeGen/AMDGPU/nested-calls.ll
index 593f40fd1b25e..1821872b82c0a 100644
--- a/llvm/test/CodeGen/AMDGPU/nested-calls.ll
+++ b/llvm/test/CodeGen/AMDGPU/nested-calls.ll
@@ -1,51 +1,83 @@
-; RUN: llc -mtriple=amdgcn -mcpu=fiji -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefix=GCN %s
-; RUN: llc -mtriple=amdgcn -mcpu=hawaii -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefix=GCN %s
-; RUN: llc -mtriple=amdgcn -mcpu=gfx900 -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefix=GCN %s
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc -mtriple=amdgcn -mcpu=fiji -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,FIJI %s
+; RUN: llc -mtriple=amdgcn -mcpu=hawaii -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,HAWAII %s
+; RUN: llc -mtriple=amdgcn -mcpu=gfx900 -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GFX9 %s
 
 ; Test calls when called by other callable functions rather than
 ; kernels.
 
 declare void @external_void_func_i32(i32) #0
 
-; GCN-LABEL: {{^}}test_func_call_external_void_func_i32_imm:
-; GCN: s_waitcnt
-
 ; Spill CSR VGPR used for SGPR spilling
-; GCN: s_mov_b32 [[FP_SCRATCH_COPY:s[0-9]+]], s33
-; GCN-NEXT: s_mov_b32 s33, s32
-; GCN-NEXT: s_or_saveexec_b64 [[COPY_EXEC0:s\[[0-9]+:[0-9]+\]]], -1{{$}}
-; GCN-NEXT: buffer_store_dword v40, off, s[0:3], s33 ; 4-byte Folded Spill
-; GCN-NEXT: s_mov_b64 exec, [[COPY_EXEC0]]
-; GCN-DAG: v_writelane_b32 v40, [[FP_SCRATCH_COPY]], 2
-; GCN-DAG: v_writelane_b32 v40, s30, 0
-; GCN-DAG: v_writelane_b32 v40, s31, 1
-
-; GCN: s_swappc_b64
-
-; GCN: v_readlane_b32 s31, v40, 1
-; GCN: v_readlane_b32 s30, v40, 0
-; GCN: s_mov_b32 s32, s33
-
-; GCN-NEXT: v_readlane_b32 [[FP_SCRATCH_COPY:s[0-9]+]], v40, 2
-; GCN: s_or_saveexec_b64 [[COPY_EXEC1:s\[[0-9]+:[0-9]+\]]], -1{{$}}
-; GCN-NEXT: buffer_load_dword v40, off, s[0:3], s33 ; 4-byte Folded Reload
-; GCN-NEXT: s_mov_b64 exec, [[COPY_EXEC1]]
-; GCN-NEXT: s_mov_b32 s33, [[FP_SCRATCH_COPY]]
-; GCN-NEXT: s_waitcnt vmcnt(0)
-; GCN-NEXT: s_setpc_b64 s[30:31]
 define void @test_func_call_external_void_func_i32_imm() #0 {
+; GCN-LABEL: test_func_call_external_void_func_i32_imm:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    s_mov_b32 s16, s33
+; GCN-NEXT:    s_mov_b32 s33, s32
+; GCN-NEXT:    s_or_saveexec_b64 s[18:19], -1
+; GCN-NEXT:    buffer_store_dword v40, off, s[0:3], s33 ; 4-byte Folded Spill
+; GCN-NEXT:    s_mov_b64 exec, s[18:19]
+; GCN-NEXT:    s_addk_i32 s32, 0x400
+; GCN-NEXT:    v_writelane_b32 v40, s16, 2
+; GCN-NEXT:    s_getpc_b64 s[16:17]
+; GCN-NEXT:    s_add_u32 s16, s16, external_void_func_i32@gotpcrel32@lo+4
+; GCN-NEXT:    s_addc_u32 s17, s17, external_void_func_i32@gotpcrel32@hi+12
+; GCN-NEXT:    s_load_dwordx2 s[16:17], s[16:17], 0x0
+; GCN-NEXT:    v_writelane_b32 v40, s30, 0
+; GCN-NEXT:    v_mov_b32_e32 v0, 42
+; GCN-NEXT:    v_writelane_b32 v40, s31, 1
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    s_swappc_b64 s[30:31], s[16:17]
+; GCN-NEXT:    v_readlane_b32 s31, v40, 1
+; GCN-NEXT:    v_readlane_b32 s30, v40, 0
+; GCN-NEXT:    s_mov_b32 s32, s33
+; GCN-NEXT:    v_readlane_b32 s4, v40, 2
+; GCN-NEXT:    s_or_saveexec_b64 s[6:7], -1
+; GCN-NEXT:    buffer_load_dword v40, off, s[0:3], s33 ; 4-byte Folded Reload
+; GCN-NEXT:    s_mov_b64 exec, s[6:7]
+; GCN-NEXT:    s_mov_b32 s33, s4
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    s_setpc_b64 s[30:31]
   call void @external_void_func_i32(i32 42)
   ret void
 }
 
-; GCN-LABEL: {{^}}test_func_call_external_void_func_i32_imm_stack_use:
-; GCN: s_waitcnt
-; GCN: s_mov_b32 s33, s32
-; GCN-DAG: s_addk_i32 s32, 0x1400{{$}}
-; GCN-DAG: buffer_store_dword v{{[0-9]+}}, off, s[0:3], s33 offset:
-; GCN: s_swappc_b64
-; GCN: s_setpc_b64
 define void @test_func_call_external_void_func_i32_imm_stack_use() #0 {
+; GCN-LABEL: test_func_call_external_void_func_i32_imm_stack_use:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    s_mov_b32 s16, s33
+; GCN-NEXT:    s_mov_b32 s33, s32
+; GCN-NEXT:    s_or_saveexec_b64 s[18:19], -1
+; GCN-NEXT:    buffer_store_dword v40, off, s[0:3], s33 offset:64 ; 4-byte Folded Spill
+; GCN-NEXT:    s_mov_b64 exec, s[18:19]
+; GCN-NEXT:    s_addk_i32 s32, 0x1400
+; GCN-NEXT:    v_writelane_b32 v40, s16, 2
+; GCN-NEXT:    s_getpc_b64 s[16:17]
+; GCN-NEXT:    s_add_u32 s16, s16, external_void_func_i32@gotpcrel32@lo+4
+; GCN-NEXT:    s_addc_u32 s17, s17, external_void_func_i32@gotpcrel32@hi+12
+; GCN-NEXT:    s_load_dwordx2 s[16:17], s[16:17], 0x0
+; GCN-NEXT:    v_mov_b32_e32 v0, 0
+; GCN-NEXT:    v_writelane_b32 v40, s30, 0
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s33
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s33 offset:64
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_mov_b32_e32 v0, 42
+; GCN-NEXT:    v_writelane_b32 v40, s31, 1
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    s_swappc_b64 s[30:31], s[16:17]
+; GCN-NEXT:    v_readlane_b32 s31, v40, 1
+; GCN-NEXT:    v_readlane_b32 s30, v40, 0
+; GCN-NEXT:    s_mov_b32 s32, s33
+; GCN-NEXT:    v_readlane_b32 s4, v40, 2
+; GCN-NEXT:    s_or_saveexec_b64 s[6:7], -1
+; GCN-NEXT:    buffer_load_dword v40, off, s[0:3], s33 offset:64 ; 4-byte Folded Reload
+; GCN-NEXT:    s_mov_b64 exec, s[6:7]
+; GCN-NEXT:    s_mov_b32 s33, s4
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    s_setpc_b64 s[30:31]
   %alloca = alloca [16 x i32], align 4, addrspace(5)
   %gep15 = getelementptr inbounds [16 x i32], ptr addrspace(5) %alloca, i32 0, i32 16
   store volatile i32 0, ptr addrspace(5) %alloca
@@ -57,3 +89,7 @@ define void @test_func_call_external_void_func_i32_imm_stack_use() #0 {
 attributes #0 = { nounwind }
 attributes #1 = { nounwind readnone }
 attributes #2 = { nounwind noinline }
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; FIJI: {{.*}}
+; GFX9: {{.*}}
+; HAWAII: {{.*}}
diff --git a/llvm/test/CodeGen/AMDGPU/nested-loop-conditions.ll b/llvm/test/CodeGen/AMDGPU/nested-loop-conditions.ll
index bd6ef9e088b12..b8e451bfb74e7 100644
--- a/llvm/test/CodeGen/AMDGPU/nested-loop-conditions.ll
+++ b/llvm/test/CodeGen/AMDGPU/nested-loop-conditions.ll
@@ -7,7 +7,7 @@
 ; the condition that appears to have no uses until the loop is
 ; completely processed.
 
-define amdgpu_kernel void @reduced_nested_loop_conditions(ptr addrspace(3) nocapture %arg) #0 {
+define amdgpu_kernel void @reduced_nested_loop_conditions(ptr addrspace(3) captures(none) %arg) #0 {
 ; GCN-LABEL: reduced_nested_loop_conditions:
 ; GCN:       ; %bb.0: ; %bb
 ; GCN-NEXT:    s_load_dword s0, s[4:5], 0x9
@@ -47,7 +47,7 @@ define amdgpu_kernel void @reduced_nested_loop_conditions(ptr addrspace(3) nocap
 ; GCN-NEXT:  .LBB0_7: ; %DummyReturnBlock
 ; GCN-NEXT:    s_endpgm
 ; IR-LABEL: define amdgpu_kernel void @reduced_nested_loop_conditions(
-; IR-SAME: ptr addrspace(3) nocapture [[ARG:%.*]]) #[[ATTR0:[0-9]+]] {
+; IR-SAME: ptr addrspace(3) captures(none) [[ARG:%.*]]) #[[ATTR0:[0-9]+]] {
 ; IR-NEXT:  [[BB:.*]]:
 ; IR-NEXT:    [[MY_TMP:%.*]] = tail call i32 @llvm.amdgcn.workitem.id.x() #[[ATTR4:[0-9]+]]
 ; IR-NEXT:    [[MY_TMP1:%.*]] = getelementptr inbounds i64, ptr addrspace(3) [[ARG]], i32 [[MY_TMP]]
@@ -144,7 +144,7 @@ bb23:                                             ; preds = %bb10
 
 ; Earlier version of above, before a run of the structurizer.
 
-define amdgpu_kernel void @nested_loop_conditions(ptr addrspace(1) nocapture %arg) #0 {
+define amdgpu_kernel void @nested_loop_conditions(ptr addrspace(1) captures(none) %arg) #0 {
 ; GCN-LABEL: nested_loop_conditions:
 ; GCN:       ; %bb.0: ; %bb
 ; GCN-NEXT:    s_mov_b32 s3, 0xf000
@@ -190,7 +190,7 @@ define amdgpu_kernel void @nested_loop_conditions(ptr addrspace(1) nocapture %ar
 ; GCN-NEXT:    s_waitcnt vmcnt(0)
 ; GCN-NEXT:    s_endpgm
 ; IR-LABEL: define amdgpu_kernel void @nested_loop_conditions(
-; IR-SAME: ptr addrspace(1) nocapture [[ARG:%.*]]) #[[ATTR0]] {
+; IR-SAME: ptr addrspace(1) captures(none) [[ARG:%.*]]) #[[ATTR0]] {
 ; IR-NEXT:  [[BB:.*]]:
 ; IR-NEXT:    [[MY_TMP1134:%.*]] = load volatile i32, ptr addrspace(1) undef, align 4
 ; IR-NEXT:    [[MY_TMP1235:%.*]] = icmp slt i32 [[MY_TMP1134]], 9
diff --git a/llvm/test/CodeGen/AMDGPU/offset-split-flat.ll b/llvm/test/CodeGen/AMDGPU/offset-split-flat.ll
index b8be5b300bb7b..2c2058473e235 100644
--- a/llvm/test/CodeGen/AMDGPU/offset-split-flat.ll
+++ b/llvm/test/CodeGen/AMDGPU/offset-split-flat.ll
@@ -620,9 +620,11 @@ define i8 @flat_inst_valu_offset_2x_24bit_max(ptr %p) {
 ; GFX12-SDAG-NEXT:    s_wait_bvhcnt 0x0
 ; GFX12-SDAG-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-SDAG-NEXT:    v_add_co_u32 v0, vcc_lo, 0x800000, v0
+; GFX12-SDAG-NEXT:    s_wait_alu 0xfffd
 ; GFX12-SDAG-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
 ; GFX12-SDAG-NEXT:    flat_load_u8 v0, v[0:1] offset:8388606
 ; GFX12-SDAG-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX12-SDAG-NEXT:    s_wait_alu 0xfffd
 ; GFX12-SDAG-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-GISEL-LABEL: flat_inst_valu_offset_2x_24bit_max:
@@ -651,9 +653,11 @@ define i8 @flat_inst_valu_offset_2x_24bit_max(ptr %p) {
 ; GFX12-GISEL-NEXT:    s_wait_bvhcnt 0x0
 ; GFX12-GISEL-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-GISEL-NEXT:    v_add_co_u32 v0, vcc_lo, 0xfffffe, v0
+; GFX12-GISEL-NEXT:    s_wait_alu 0xfffd
 ; GFX12-GISEL-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
 ; GFX12-GISEL-NEXT:    flat_load_u8 v0, v[0:1]
 ; GFX12-GISEL-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX12-GISEL-NEXT:    s_wait_alu 0xfffd
 ; GFX12-GISEL-NEXT:    s_setpc_b64 s[30:31]
   %gep = getelementptr i8, ptr %p, i64 16777214
   %load = load i8, ptr %gep, align 4
@@ -825,9 +829,11 @@ define i8 @flat_inst_valu_offset_2x_neg_24bit_max(ptr %p) {
 ; GFX12-SDAG-NEXT:    s_wait_bvhcnt 0x0
 ; GFX12-SDAG-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-SDAG-NEXT:    v_add_co_u32 v0, vcc_lo, 0xff800000, v0
+; GFX12-SDAG-NEXT:    s_wait_alu 0xfffd
 ; GFX12-SDAG-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, -1, v1, vcc_lo
 ; GFX12-SDAG-NEXT:    flat_load_u8 v0, v[0:1] offset:-8388607
 ; GFX12-SDAG-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX12-SDAG-NEXT:    s_wait_alu 0xfffd
 ; GFX12-SDAG-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX12-GISEL-LABEL: flat_inst_valu_offset_2x_neg_24bit_max:
@@ -838,9 +844,11 @@ define i8 @flat_inst_valu_offset_2x_neg_24bit_max(ptr %p) {
 ; GFX12-GISEL-NEXT:    s_wait_bvhcnt 0x0
 ; GFX12-GISEL-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-GISEL-NEXT:    v_add_co_u32 v0, vcc_lo, 0xff000001, v0
+; GFX12-GISEL-NEXT:    s_wait_alu 0xfffd
 ; GFX12-GISEL-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, -1, v1, vcc_lo
 ; GFX12-GISEL-NEXT:    flat_load_u8 v0, v[0:1]
 ; GFX12-GISEL-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX12-GISEL-NEXT:    s_wait_alu 0xfffd
 ; GFX12-GISEL-NEXT:    s_setpc_b64 s[30:31]
   %gep = getelementptr i8, ptr %p, i64 -16777215
   %load = load i8, ptr %gep, align 4
@@ -884,9 +892,11 @@ define i8 @flat_inst_valu_offset_64bit_11bit_split0(ptr %p) {
 ; GFX12-SDAG-NEXT:    s_wait_bvhcnt 0x0
 ; GFX12-SDAG-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-SDAG-NEXT:    v_add_co_u32 v0, vcc_lo, 0, v0
+; GFX12-SDAG-NEXT:    s_wait_alu 0xfffd
 ; GFX12-SDAG-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, 2, v1, vcc_lo
 ; GFX12-SDAG-NEXT:    flat_load_u8 v0, v[0:1] offset:2047
 ; GFX12-SDAG-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX12-SDAG-NEXT:    s_wait_alu 0xfffd
 ; GFX12-SDAG-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-GISEL-LABEL: flat_inst_valu_offset_64bit_11bit_split0:
@@ -915,9 +925,11 @@ define i8 @flat_inst_valu_offset_64bit_11bit_split0(ptr %p) {
 ; GFX12-GISEL-NEXT:    s_wait_bvhcnt 0x0
 ; GFX12-GISEL-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-GISEL-NEXT:    v_add_co_u32 v0, vcc_lo, 0x7ff, v0
+; GFX12-GISEL-NEXT:    s_wait_alu 0xfffd
 ; GFX12-GISEL-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, 2, v1, vcc_lo
 ; GFX12-GISEL-NEXT:    flat_load_u8 v0, v[0:1]
 ; GFX12-GISEL-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX12-GISEL-NEXT:    s_wait_alu 0xfffd
 ; GFX12-GISEL-NEXT:    s_setpc_b64 s[30:31]
   %gep = getelementptr i8, ptr %p, i64 8589936639
   %load = load i8, ptr %gep, align 4
@@ -961,9 +973,11 @@ define i8 @flat_inst_valu_offset_64bit_11bit_split1(ptr %p) {
 ; GFX12-SDAG-NEXT:    s_wait_bvhcnt 0x0
 ; GFX12-SDAG-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-SDAG-NEXT:    v_add_co_u32 v0, vcc_lo, 0, v0
+; GFX12-SDAG-NEXT:    s_wait_alu 0xfffd
 ; GFX12-SDAG-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, 2, v1, vcc_lo
 ; GFX12-SDAG-NEXT:    flat_load_u8 v0, v[0:1] offset:2048
 ; GFX12-SDAG-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX12-SDAG-NEXT:    s_wait_alu 0xfffd
 ; GFX12-SDAG-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-GISEL-LABEL: flat_inst_valu_offset_64bit_11bit_split1:
@@ -992,9 +1006,11 @@ define i8 @flat_inst_valu_offset_64bit_11bit_split1(ptr %p) {
 ; GFX12-GISEL-NEXT:    s_wait_bvhcnt 0x0
 ; GFX12-GISEL-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-GISEL-NEXT:    v_add_co_u32 v0, vcc_lo, 0x800, v0
+; GFX12-GISEL-NEXT:    s_wait_alu 0xfffd
 ; GFX12-GISEL-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, 2, v1, vcc_lo
 ; GFX12-GISEL-NEXT:    flat_load_u8 v0, v[0:1]
 ; GFX12-GISEL-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX12-GISEL-NEXT:    s_wait_alu 0xfffd
 ; GFX12-GISEL-NEXT:    s_setpc_b64 s[30:31]
   %gep = getelementptr i8, ptr %p, i64 8589936640
   %load = load i8, ptr %gep, align 4
@@ -1038,9 +1054,11 @@ define i8 @flat_inst_valu_offset_64bit_12bit_split0(ptr %p) {
 ; GFX12-SDAG-NEXT:    s_wait_bvhcnt 0x0
 ; GFX12-SDAG-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-SDAG-NEXT:    v_add_co_u32 v0, vcc_lo, 0, v0
+; GFX12-SDAG-NEXT:    s_wait_alu 0xfffd
 ; GFX12-SDAG-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, 2, v1, vcc_lo
 ; GFX12-SDAG-NEXT:    flat_load_u8 v0, v[0:1] offset:4095
 ; GFX12-SDAG-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX12-SDAG-NEXT:    s_wait_alu 0xfffd
 ; GFX12-SDAG-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-GISEL-LABEL: flat_inst_valu_offset_64bit_12bit_split0:
@@ -1069,9 +1087,11 @@ define i8 @flat_inst_valu_offset_64bit_12bit_split0(ptr %p) {
 ; GFX12-GISEL-NEXT:    s_wait_bvhcnt 0x0
 ; GFX12-GISEL-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-GISEL-NEXT:    v_add_co_u32 v0, vcc_lo, 0xfff, v0
+; GFX12-GISEL-NEXT:    s_wait_alu 0xfffd
 ; GFX12-GISEL-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, 2, v1, vcc_lo
 ; GFX12-GISEL-NEXT:    flat_load_u8 v0, v[0:1]
 ; GFX12-GISEL-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX12-GISEL-NEXT:    s_wait_alu 0xfffd
 ; GFX12-GISEL-NEXT:    s_setpc_b64 s[30:31]
   %gep = getelementptr i8, ptr %p, i64 8589938687
   %load = load i8, ptr %gep, align 4
@@ -1115,9 +1135,11 @@ define i8 @flat_inst_valu_offset_64bit_12bit_split1(ptr %p) {
 ; GFX12-SDAG-NEXT:    s_wait_bvhcnt 0x0
 ; GFX12-SDAG-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-SDAG-NEXT:    v_add_co_u32 v0, vcc_lo, 0, v0
+; GFX12-SDAG-NEXT:    s_wait_alu 0xfffd
 ; GFX12-SDAG-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, 2, v1, vcc_lo
 ; GFX12-SDAG-NEXT:    flat_load_u8 v0, v[0:1] offset:4096
 ; GFX12-SDAG-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX12-SDAG-NEXT:    s_wait_alu 0xfffd
 ; GFX12-SDAG-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX12-GISEL-LABEL: flat_inst_valu_offset_64bit_12bit_split1:
@@ -1128,9 +1150,11 @@ define i8 @flat_inst_valu_offset_64bit_12bit_split1(ptr %p) {
 ; GFX12-GISEL-NEXT:    s_wait_bvhcnt 0x0
 ; GFX12-GISEL-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-GISEL-NEXT:    v_add_co_u32 v0, vcc_lo, 0x1000, v0
+; GFX12-GISEL-NEXT:    s_wait_alu 0xfffd
 ; GFX12-GISEL-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, 2, v1, vcc_lo
 ; GFX12-GISEL-NEXT:    flat_load_u8 v0, v[0:1]
 ; GFX12-GISEL-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX12-GISEL-NEXT:    s_wait_alu 0xfffd
 ; GFX12-GISEL-NEXT:    s_setpc_b64 s[30:31]
   %gep = getelementptr i8, ptr %p, i64 8589938688
   %load = load i8, ptr %gep, align 4
@@ -1174,9 +1198,11 @@ define i8 @flat_inst_valu_offset_64bit_13bit_split0(ptr %p) {
 ; GFX12-SDAG-NEXT:    s_wait_bvhcnt 0x0
 ; GFX12-SDAG-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-SDAG-NEXT:    v_add_co_u32 v0, vcc_lo, 0, v0
+; GFX12-SDAG-NEXT:    s_wait_alu 0xfffd
 ; GFX12-SDAG-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, 2, v1, vcc_lo
 ; GFX12-SDAG-NEXT:    flat_load_u8 v0, v[0:1] offset:8191
 ; GFX12-SDAG-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX12-SDAG-NEXT:    s_wait_alu 0xfffd
 ; GFX12-SDAG-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-GISEL-LABEL: flat_inst_valu_offset_64bit_13bit_split0:
@@ -1205,9 +1231,11 @@ define i8 @flat_inst_valu_offset_64bit_13bit_split0(ptr %p) {
 ; GFX12-GISEL-NEXT:    s_wait_bvhcnt 0x0
 ; GFX12-GISEL-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-GISEL-NEXT:    v_add_co_u32 v0, vcc_lo, 0x1fff, v0
+; GFX12-GISEL-NEXT:    s_wait_alu 0xfffd
 ; GFX12-GISEL-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, 2, v1, vcc_lo
 ; GFX12-GISEL-NEXT:    flat_load_u8 v0, v[0:1]
 ; GFX12-GISEL-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX12-GISEL-NEXT:    s_wait_alu 0xfffd
 ; GFX12-GISEL-NEXT:    s_setpc_b64 s[30:31]
   %gep = getelementptr i8, ptr %p, i64 8589942783
   %load = load i8, ptr %gep, align 4
@@ -1251,9 +1279,11 @@ define i8 @flat_inst_valu_offset_64bit_13bit_split1(ptr %p) {
 ; GFX12-SDAG-NEXT:    s_wait_bvhcnt 0x0
 ; GFX12-SDAG-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-SDAG-NEXT:    v_add_co_u32 v0, vcc_lo, 0, v0
+; GFX12-SDAG-NEXT:    s_wait_alu 0xfffd
 ; GFX12-SDAG-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, 2, v1, vcc_lo
 ; GFX12-SDAG-NEXT:    flat_load_u8 v0, v[0:1] offset:8192
 ; GFX12-SDAG-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX12-SDAG-NEXT:    s_wait_alu 0xfffd
 ; GFX12-SDAG-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX12-GISEL-LABEL: flat_inst_valu_offset_64bit_13bit_split1:
@@ -1264,9 +1294,11 @@ define i8 @flat_inst_valu_offset_64bit_13bit_split1(ptr %p) {
 ; GFX12-GISEL-NEXT:    s_wait_bvhcnt 0x0
 ; GFX12-GISEL-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-GISEL-NEXT:    v_add_co_u32 v0, vcc_lo, 0x2000, v0
+; GFX12-GISEL-NEXT:    s_wait_alu 0xfffd
 ; GFX12-GISEL-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, 2, v1, vcc_lo
 ; GFX12-GISEL-NEXT:    flat_load_u8 v0, v[0:1]
 ; GFX12-GISEL-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX12-GISEL-NEXT:    s_wait_alu 0xfffd
 ; GFX12-GISEL-NEXT:    s_setpc_b64 s[30:31]
   %gep = getelementptr i8, ptr %p, i64 8589942784
   %load = load i8, ptr %gep, align 4
@@ -1311,9 +1343,11 @@ define i8 @flat_inst_valu_offset_64bit_11bit_neg_high_split0(ptr %p) {
 ; GFX12-SDAG-NEXT:    s_wait_bvhcnt 0x0
 ; GFX12-SDAG-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-SDAG-NEXT:    v_add_co_u32 v0, vcc_lo, 0x800000, v0
+; GFX12-SDAG-NEXT:    s_wait_alu 0xfffd
 ; GFX12-SDAG-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, 0x80000000, v1, vcc_lo
 ; GFX12-SDAG-NEXT:    flat_load_u8 v0, v[0:1] offset:-8386561
 ; GFX12-SDAG-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX12-SDAG-NEXT:    s_wait_alu 0xfffd
 ; GFX12-SDAG-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-GISEL-LABEL: flat_inst_valu_offset_64bit_11bit_neg_high_split0:
@@ -1334,9 +1368,11 @@ define i8 @flat_inst_valu_offset_64bit_11bit_neg_high_split0(ptr %p) {
 ; GFX12-GISEL-NEXT:    s_wait_bvhcnt 0x0
 ; GFX12-GISEL-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-GISEL-NEXT:    v_add_co_u32 v0, vcc_lo, 0x7ff, v0
+; GFX12-GISEL-NEXT:    s_wait_alu 0xfffd
 ; GFX12-GISEL-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, 0x80000000, v1, vcc_lo
 ; GFX12-GISEL-NEXT:    flat_load_u8 v0, v[0:1]
 ; GFX12-GISEL-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX12-GISEL-NEXT:    s_wait_alu 0xfffd
 ; GFX12-GISEL-NEXT:    s_setpc_b64 s[30:31]
   %gep = getelementptr i8, ptr %p, i64 -9223372036854773761
   %load = load i8, ptr %gep, align 4
@@ -1381,9 +1417,11 @@ define i8 @flat_inst_valu_offset_64bit_11bit_neg_high_split1(ptr %p) {
 ; GFX12-SDAG-NEXT:    s_wait_bvhcnt 0x0
 ; GFX12-SDAG-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-SDAG-NEXT:    v_add_co_u32 v0, vcc_lo, 0x800000, v0
+; GFX12-SDAG-NEXT:    s_wait_alu 0xfffd
 ; GFX12-SDAG-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, 0x80000000, v1, vcc_lo
 ; GFX12-SDAG-NEXT:    flat_load_u8 v0, v[0:1] offset:-8386560
 ; GFX12-SDAG-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX12-SDAG-NEXT:    s_wait_alu 0xfffd
 ; GFX12-SDAG-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-GISEL-LABEL: flat_inst_valu_offset_64bit_11bit_neg_high_split1:
@@ -1404,9 +1442,11 @@ define i8 @flat_inst_valu_offset_64bit_11bit_neg_high_split1(ptr %p) {
 ; GFX12-GISEL-NEXT:    s_wait_bvhcnt 0x0
 ; GFX12-GISEL-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-GISEL-NEXT:    v_add_co_u32 v0, vcc_lo, 0x800, v0
+; GFX12-GISEL-NEXT:    s_wait_alu 0xfffd
 ; GFX12-GISEL-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, 0x80000000, v1, vcc_lo
 ; GFX12-GISEL-NEXT:    flat_load_u8 v0, v[0:1]
 ; GFX12-GISEL-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX12-GISEL-NEXT:    s_wait_alu 0xfffd
 ; GFX12-GISEL-NEXT:    s_setpc_b64 s[30:31]
   %gep = getelementptr i8, ptr %p, i64 -9223372036854773760
   %load = load i8, ptr %gep, align 4
@@ -1451,9 +1491,11 @@ define i8 @flat_inst_valu_offset_64bit_12bit_neg_high_split0(ptr %p) {
 ; GFX12-SDAG-NEXT:    s_wait_bvhcnt 0x0
 ; GFX12-SDAG-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-SDAG-NEXT:    v_add_co_u32 v0, vcc_lo, 0x800000, v0
+; GFX12-SDAG-NEXT:    s_wait_alu 0xfffd
 ; GFX12-SDAG-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, 0x80000000, v1, vcc_lo
 ; GFX12-SDAG-NEXT:    flat_load_u8 v0, v[0:1] offset:-8384513
 ; GFX12-SDAG-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX12-SDAG-NEXT:    s_wait_alu 0xfffd
 ; GFX12-SDAG-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-GISEL-LABEL: flat_inst_valu_offset_64bit_12bit_neg_high_split0:
@@ -1474,9 +1516,11 @@ define i8 @flat_inst_valu_offset_64bit_12bit_neg_high_split0(ptr %p) {
 ; GFX12-GISEL-NEXT:    s_wait_bvhcnt 0x0
 ; GFX12-GISEL-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-GISEL-NEXT:    v_add_co_u32 v0, vcc_lo, 0xfff, v0
+; GFX12-GISEL-NEXT:    s_wait_alu 0xfffd
 ; GFX12-GISEL-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, 0x80000000, v1, vcc_lo
 ; GFX12-GISEL-NEXT:    flat_load_u8 v0, v[0:1]
 ; GFX12-GISEL-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX12-GISEL-NEXT:    s_wait_alu 0xfffd
 ; GFX12-GISEL-NEXT:    s_setpc_b64 s[30:31]
   %gep = getelementptr i8, ptr %p, i64 -9223372036854771713
   %load = load i8, ptr %gep, align 4
@@ -1521,9 +1565,11 @@ define i8 @flat_inst_valu_offset_64bit_12bit_neg_high_split1(ptr %p) {
 ; GFX12-SDAG-NEXT:    s_wait_bvhcnt 0x0
 ; GFX12-SDAG-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-SDAG-NEXT:    v_add_co_u32 v0, vcc_lo, 0x800000, v0
+; GFX12-SDAG-NEXT:    s_wait_alu 0xfffd
 ; GFX12-SDAG-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, 0x80000000, v1, vcc_lo
 ; GFX12-SDAG-NEXT:    flat_load_u8 v0, v[0:1] offset:-8384512
 ; GFX12-SDAG-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX12-SDAG-NEXT:    s_wait_alu 0xfffd
 ; GFX12-SDAG-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-GISEL-LABEL: flat_inst_valu_offset_64bit_12bit_neg_high_split1:
@@ -1544,9 +1590,11 @@ define i8 @flat_inst_valu_offset_64bit_12bit_neg_high_split1(ptr %p) {
 ; GFX12-GISEL-NEXT:    s_wait_bvhcnt 0x0
 ; GFX12-GISEL-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-GISEL-NEXT:    v_add_co_u32 v0, vcc_lo, 0x1000, v0
+; GFX12-GISEL-NEXT:    s_wait_alu 0xfffd
 ; GFX12-GISEL-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, 0x80000000, v1, vcc_lo
 ; GFX12-GISEL-NEXT:    flat_load_u8 v0, v[0:1]
 ; GFX12-GISEL-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX12-GISEL-NEXT:    s_wait_alu 0xfffd
 ; GFX12-GISEL-NEXT:    s_setpc_b64 s[30:31]
   %gep = getelementptr i8, ptr %p, i64 -9223372036854771712
   %load = load i8, ptr %gep, align 4
@@ -1591,9 +1639,11 @@ define i8 @flat_inst_valu_offset_64bit_13bit_neg_high_split0(ptr %p) {
 ; GFX12-SDAG-NEXT:    s_wait_bvhcnt 0x0
 ; GFX12-SDAG-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-SDAG-NEXT:    v_add_co_u32 v0, vcc_lo, 0x800000, v0
+; GFX12-SDAG-NEXT:    s_wait_alu 0xfffd
 ; GFX12-SDAG-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, 0x80000000, v1, vcc_lo
 ; GFX12-SDAG-NEXT:    flat_load_u8 v0, v[0:1] offset:-8380417
 ; GFX12-SDAG-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX12-SDAG-NEXT:    s_wait_alu 0xfffd
 ; GFX12-SDAG-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-GISEL-LABEL: flat_inst_valu_offset_64bit_13bit_neg_high_split0:
@@ -1614,9 +1664,11 @@ define i8 @flat_inst_valu_offset_64bit_13bit_neg_high_split0(ptr %p) {
 ; GFX12-GISEL-NEXT:    s_wait_bvhcnt 0x0
 ; GFX12-GISEL-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-GISEL-NEXT:    v_add_co_u32 v0, vcc_lo, 0x1fff, v0
+; GFX12-GISEL-NEXT:    s_wait_alu 0xfffd
 ; GFX12-GISEL-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, 0x80000000, v1, vcc_lo
 ; GFX12-GISEL-NEXT:    flat_load_u8 v0, v[0:1]
 ; GFX12-GISEL-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX12-GISEL-NEXT:    s_wait_alu 0xfffd
 ; GFX12-GISEL-NEXT:    s_setpc_b64 s[30:31]
   %gep = getelementptr i8, ptr %p, i64 -9223372036854767617
   %load = load i8, ptr %gep, align 4
@@ -1661,9 +1713,11 @@ define i8 @flat_inst_valu_offset_64bit_13bit_neg_high_split1(ptr %p) {
 ; GFX12-SDAG-NEXT:    s_wait_bvhcnt 0x0
 ; GFX12-SDAG-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-SDAG-NEXT:    v_add_co_u32 v0, vcc_lo, 0x800000, v0
+; GFX12-SDAG-NEXT:    s_wait_alu 0xfffd
 ; GFX12-SDAG-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, 0x80000000, v1, vcc_lo
 ; GFX12-SDAG-NEXT:    flat_load_u8 v0, v[0:1] offset:-8380416
 ; GFX12-SDAG-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX12-SDAG-NEXT:    s_wait_alu 0xfffd
 ; GFX12-SDAG-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-GISEL-LABEL: flat_inst_valu_offset_64bit_13bit_neg_high_split1:
@@ -1684,9 +1738,11 @@ define i8 @flat_inst_valu_offset_64bit_13bit_neg_high_split1(ptr %p) {
 ; GFX12-GISEL-NEXT:    s_wait_bvhcnt 0x0
 ; GFX12-GISEL-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-GISEL-NEXT:    v_add_co_u32 v0, vcc_lo, 0x2000, v0
+; GFX12-GISEL-NEXT:    s_wait_alu 0xfffd
 ; GFX12-GISEL-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, 0x80000000, v1, vcc_lo
 ; GFX12-GISEL-NEXT:    flat_load_u8 v0, v[0:1]
 ; GFX12-GISEL-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX12-GISEL-NEXT:    s_wait_alu 0xfffd
 ; GFX12-GISEL-NEXT:    s_setpc_b64 s[30:31]
   %gep = getelementptr i8, ptr %p, i64 -9223372036854767616
   %load = load i8, ptr %gep, align 4
@@ -2657,7 +2713,7 @@ define amdgpu_kernel void @flat_inst_salu_offset_64bit_11bit_split0(ptr %p) {
 ; GFX12-SDAG-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
 ; GFX12-SDAG-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-SDAG-NEXT:    v_add_co_u32 v0, s0, 0, s0
-; GFX12-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX12-SDAG-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-SDAG-NEXT:    v_add_co_ci_u32_e64 v1, null, 2, s1, s0
 ; GFX12-SDAG-NEXT:    flat_load_u8 v0, v[0:1] offset:2047 scope:SCOPE_SYS
 ; GFX12-SDAG-NEXT:    s_wait_loadcnt_dscnt 0x0
@@ -2752,7 +2808,7 @@ define amdgpu_kernel void @flat_inst_salu_offset_64bit_11bit_split1(ptr %p) {
 ; GFX12-SDAG-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
 ; GFX12-SDAG-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-SDAG-NEXT:    v_add_co_u32 v0, s0, 0, s0
-; GFX12-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX12-SDAG-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-SDAG-NEXT:    v_add_co_ci_u32_e64 v1, null, 2, s1, s0
 ; GFX12-SDAG-NEXT:    flat_load_u8 v0, v[0:1] offset:2048 scope:SCOPE_SYS
 ; GFX12-SDAG-NEXT:    s_wait_loadcnt_dscnt 0x0
@@ -2847,7 +2903,7 @@ define amdgpu_kernel void @flat_inst_salu_offset_64bit_12bit_split0(ptr %p) {
 ; GFX12-SDAG-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
 ; GFX12-SDAG-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-SDAG-NEXT:    v_add_co_u32 v0, s0, 0, s0
-; GFX12-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX12-SDAG-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-SDAG-NEXT:    v_add_co_ci_u32_e64 v1, null, 2, s1, s0
 ; GFX12-SDAG-NEXT:    flat_load_u8 v0, v[0:1] offset:4095 scope:SCOPE_SYS
 ; GFX12-SDAG-NEXT:    s_wait_loadcnt_dscnt 0x0
@@ -2943,7 +2999,7 @@ define amdgpu_kernel void @flat_inst_salu_offset_64bit_12bit_split1(ptr %p) {
 ; GFX12-SDAG-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
 ; GFX12-SDAG-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-SDAG-NEXT:    v_add_co_u32 v0, s0, 0, s0
-; GFX12-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX12-SDAG-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-SDAG-NEXT:    v_add_co_ci_u32_e64 v1, null, 2, s1, s0
 ; GFX12-SDAG-NEXT:    flat_load_u8 v0, v[0:1] offset:4096 scope:SCOPE_SYS
 ; GFX12-SDAG-NEXT:    s_wait_loadcnt_dscnt 0x0
@@ -3039,7 +3095,7 @@ define amdgpu_kernel void @flat_inst_salu_offset_64bit_13bit_split0(ptr %p) {
 ; GFX12-SDAG-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
 ; GFX12-SDAG-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-SDAG-NEXT:    v_add_co_u32 v0, s0, 0, s0
-; GFX12-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX12-SDAG-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-SDAG-NEXT:    v_add_co_ci_u32_e64 v1, null, 2, s1, s0
 ; GFX12-SDAG-NEXT:    flat_load_u8 v0, v[0:1] offset:8191 scope:SCOPE_SYS
 ; GFX12-SDAG-NEXT:    s_wait_loadcnt_dscnt 0x0
@@ -3135,7 +3191,7 @@ define amdgpu_kernel void @flat_inst_salu_offset_64bit_13bit_split1(ptr %p) {
 ; GFX12-SDAG-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
 ; GFX12-SDAG-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-SDAG-NEXT:    v_add_co_u32 v0, s0, 0, s0
-; GFX12-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX12-SDAG-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-SDAG-NEXT:    v_add_co_ci_u32_e64 v1, null, 2, s1, s0
 ; GFX12-SDAG-NEXT:    flat_load_u8 v0, v[0:1] offset:8192 scope:SCOPE_SYS
 ; GFX12-SDAG-NEXT:    s_wait_loadcnt_dscnt 0x0
diff --git a/llvm/test/CodeGen/AMDGPU/offset-split-global.ll b/llvm/test/CodeGen/AMDGPU/offset-split-global.ll
index fd62ba3f9da1f..d16d731c34384 100644
--- a/llvm/test/CodeGen/AMDGPU/offset-split-global.ll
+++ b/llvm/test/CodeGen/AMDGPU/offset-split-global.ll
@@ -659,9 +659,11 @@ define i8 @global_inst_valu_offset_2x_24bit_max(ptr addrspace(1) %p) {
 ; GFX12-GISEL-NEXT:    s_wait_bvhcnt 0x0
 ; GFX12-GISEL-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-GISEL-NEXT:    v_add_co_u32 v0, vcc_lo, 0xfffffe, v0
+; GFX12-GISEL-NEXT:    s_wait_alu 0xfffd
 ; GFX12-GISEL-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
 ; GFX12-GISEL-NEXT:    global_load_u8 v0, v[0:1], off
 ; GFX12-GISEL-NEXT:    s_wait_loadcnt 0x0
+; GFX12-GISEL-NEXT:    s_wait_alu 0xfffd
 ; GFX12-GISEL-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-SDAG-LABEL: global_inst_valu_offset_2x_24bit_max:
@@ -699,9 +701,11 @@ define i8 @global_inst_valu_offset_2x_24bit_max(ptr addrspace(1) %p) {
 ; GFX12-SDAG-NEXT:    s_wait_bvhcnt 0x0
 ; GFX12-SDAG-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-SDAG-NEXT:    v_add_co_u32 v0, vcc_lo, 0x800000, v0
+; GFX12-SDAG-NEXT:    s_wait_alu 0xfffd
 ; GFX12-SDAG-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
 ; GFX12-SDAG-NEXT:    global_load_u8 v0, v[0:1], off offset:8388606
 ; GFX12-SDAG-NEXT:    s_wait_loadcnt 0x0
+; GFX12-SDAG-NEXT:    s_wait_alu 0xfffd
 ; GFX12-SDAG-NEXT:    s_setpc_b64 s[30:31]
   %gep = getelementptr i8, ptr addrspace(1) %p, i64 16777214
   %load = load i8, ptr addrspace(1) %gep, align 4
@@ -869,9 +873,11 @@ define i8 @global_inst_valu_offset_2x_neg_24bit_max(ptr addrspace(1) %p) {
 ; GFX12-GISEL-NEXT:    s_wait_bvhcnt 0x0
 ; GFX12-GISEL-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-GISEL-NEXT:    v_add_co_u32 v0, vcc_lo, 0xff000001, v0
+; GFX12-GISEL-NEXT:    s_wait_alu 0xfffd
 ; GFX12-GISEL-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, -1, v1, vcc_lo
 ; GFX12-GISEL-NEXT:    global_load_u8 v0, v[0:1], off
 ; GFX12-GISEL-NEXT:    s_wait_loadcnt 0x0
+; GFX12-GISEL-NEXT:    s_wait_alu 0xfffd
 ; GFX12-GISEL-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-SDAG-LABEL: global_inst_valu_offset_2x_neg_24bit_max:
@@ -909,9 +915,11 @@ define i8 @global_inst_valu_offset_2x_neg_24bit_max(ptr addrspace(1) %p) {
 ; GFX12-SDAG-NEXT:    s_wait_bvhcnt 0x0
 ; GFX12-SDAG-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-SDAG-NEXT:    v_add_co_u32 v0, vcc_lo, 0xff800000, v0
+; GFX12-SDAG-NEXT:    s_wait_alu 0xfffd
 ; GFX12-SDAG-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, -1, v1, vcc_lo
 ; GFX12-SDAG-NEXT:    global_load_u8 v0, v[0:1], off offset:-8388607
 ; GFX12-SDAG-NEXT:    s_wait_loadcnt 0x0
+; GFX12-SDAG-NEXT:    s_wait_alu 0xfffd
 ; GFX12-SDAG-NEXT:    s_setpc_b64 s[30:31]
   %gep = getelementptr i8, ptr addrspace(1) %p, i64 -16777215
   %load = load i8, ptr addrspace(1) %gep, align 4
@@ -956,9 +964,11 @@ define i8 @global_inst_valu_offset_64bit_11bit_split0(ptr addrspace(1) %p) {
 ; GFX12-GISEL-NEXT:    s_wait_bvhcnt 0x0
 ; GFX12-GISEL-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-GISEL-NEXT:    v_add_co_u32 v0, vcc_lo, 0x7ff, v0
+; GFX12-GISEL-NEXT:    s_wait_alu 0xfffd
 ; GFX12-GISEL-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, 2, v1, vcc_lo
 ; GFX12-GISEL-NEXT:    global_load_u8 v0, v[0:1], off
 ; GFX12-GISEL-NEXT:    s_wait_loadcnt 0x0
+; GFX12-GISEL-NEXT:    s_wait_alu 0xfffd
 ; GFX12-GISEL-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-SDAG-LABEL: global_inst_valu_offset_64bit_11bit_split0:
@@ -996,9 +1006,11 @@ define i8 @global_inst_valu_offset_64bit_11bit_split0(ptr addrspace(1) %p) {
 ; GFX12-SDAG-NEXT:    s_wait_bvhcnt 0x0
 ; GFX12-SDAG-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-SDAG-NEXT:    v_add_co_u32 v0, vcc_lo, 0, v0
+; GFX12-SDAG-NEXT:    s_wait_alu 0xfffd
 ; GFX12-SDAG-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, 2, v1, vcc_lo
 ; GFX12-SDAG-NEXT:    global_load_u8 v0, v[0:1], off offset:2047
 ; GFX12-SDAG-NEXT:    s_wait_loadcnt 0x0
+; GFX12-SDAG-NEXT:    s_wait_alu 0xfffd
 ; GFX12-SDAG-NEXT:    s_setpc_b64 s[30:31]
   %gep = getelementptr i8, ptr addrspace(1) %p, i64 8589936639
   %load = load i8, ptr addrspace(1) %gep, align 4
@@ -1042,9 +1054,11 @@ define i8 @global_inst_valu_offset_64bit_11bit_split1(ptr addrspace(1) %p) {
 ; GFX12-GISEL-NEXT:    s_wait_bvhcnt 0x0
 ; GFX12-GISEL-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-GISEL-NEXT:    v_add_co_u32 v0, vcc_lo, 0x800, v0
+; GFX12-GISEL-NEXT:    s_wait_alu 0xfffd
 ; GFX12-GISEL-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, 2, v1, vcc_lo
 ; GFX12-GISEL-NEXT:    global_load_u8 v0, v[0:1], off
 ; GFX12-GISEL-NEXT:    s_wait_loadcnt 0x0
+; GFX12-GISEL-NEXT:    s_wait_alu 0xfffd
 ; GFX12-GISEL-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-SDAG-LABEL: global_inst_valu_offset_64bit_11bit_split1:
@@ -1073,9 +1087,11 @@ define i8 @global_inst_valu_offset_64bit_11bit_split1(ptr addrspace(1) %p) {
 ; GFX12-SDAG-NEXT:    s_wait_bvhcnt 0x0
 ; GFX12-SDAG-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-SDAG-NEXT:    v_add_co_u32 v0, vcc_lo, 0, v0
+; GFX12-SDAG-NEXT:    s_wait_alu 0xfffd
 ; GFX12-SDAG-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, 2, v1, vcc_lo
 ; GFX12-SDAG-NEXT:    global_load_u8 v0, v[0:1], off offset:2048
 ; GFX12-SDAG-NEXT:    s_wait_loadcnt 0x0
+; GFX12-SDAG-NEXT:    s_wait_alu 0xfffd
 ; GFX12-SDAG-NEXT:    s_setpc_b64 s[30:31]
   %gep = getelementptr i8, ptr addrspace(1) %p, i64 8589936640
   %load = load i8, ptr addrspace(1) %gep, align 4
@@ -1119,9 +1135,11 @@ define i8 @global_inst_valu_offset_64bit_12bit_split0(ptr addrspace(1) %p) {
 ; GFX12-GISEL-NEXT:    s_wait_bvhcnt 0x0
 ; GFX12-GISEL-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-GISEL-NEXT:    v_add_co_u32 v0, vcc_lo, 0xfff, v0
+; GFX12-GISEL-NEXT:    s_wait_alu 0xfffd
 ; GFX12-GISEL-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, 2, v1, vcc_lo
 ; GFX12-GISEL-NEXT:    global_load_u8 v0, v[0:1], off
 ; GFX12-GISEL-NEXT:    s_wait_loadcnt 0x0
+; GFX12-GISEL-NEXT:    s_wait_alu 0xfffd
 ; GFX12-GISEL-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-SDAG-LABEL: global_inst_valu_offset_64bit_12bit_split0:
@@ -1159,9 +1177,11 @@ define i8 @global_inst_valu_offset_64bit_12bit_split0(ptr addrspace(1) %p) {
 ; GFX12-SDAG-NEXT:    s_wait_bvhcnt 0x0
 ; GFX12-SDAG-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-SDAG-NEXT:    v_add_co_u32 v0, vcc_lo, 0, v0
+; GFX12-SDAG-NEXT:    s_wait_alu 0xfffd
 ; GFX12-SDAG-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, 2, v1, vcc_lo
 ; GFX12-SDAG-NEXT:    global_load_u8 v0, v[0:1], off offset:4095
 ; GFX12-SDAG-NEXT:    s_wait_loadcnt 0x0
+; GFX12-SDAG-NEXT:    s_wait_alu 0xfffd
 ; GFX12-SDAG-NEXT:    s_setpc_b64 s[30:31]
   %gep = getelementptr i8, ptr addrspace(1) %p, i64 8589938687
   %load = load i8, ptr addrspace(1) %gep, align 4
@@ -1205,9 +1225,11 @@ define i8 @global_inst_valu_offset_64bit_12bit_split1(ptr addrspace(1) %p) {
 ; GFX12-GISEL-NEXT:    s_wait_bvhcnt 0x0
 ; GFX12-GISEL-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-GISEL-NEXT:    v_add_co_u32 v0, vcc_lo, 0x1000, v0
+; GFX12-GISEL-NEXT:    s_wait_alu 0xfffd
 ; GFX12-GISEL-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, 2, v1, vcc_lo
 ; GFX12-GISEL-NEXT:    global_load_u8 v0, v[0:1], off
 ; GFX12-GISEL-NEXT:    s_wait_loadcnt 0x0
+; GFX12-GISEL-NEXT:    s_wait_alu 0xfffd
 ; GFX12-GISEL-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX12-SDAG-LABEL: global_inst_valu_offset_64bit_12bit_split1:
@@ -1218,9 +1240,11 @@ define i8 @global_inst_valu_offset_64bit_12bit_split1(ptr addrspace(1) %p) {
 ; GFX12-SDAG-NEXT:    s_wait_bvhcnt 0x0
 ; GFX12-SDAG-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-SDAG-NEXT:    v_add_co_u32 v0, vcc_lo, 0, v0
+; GFX12-SDAG-NEXT:    s_wait_alu 0xfffd
 ; GFX12-SDAG-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, 2, v1, vcc_lo
 ; GFX12-SDAG-NEXT:    global_load_u8 v0, v[0:1], off offset:4096
 ; GFX12-SDAG-NEXT:    s_wait_loadcnt 0x0
+; GFX12-SDAG-NEXT:    s_wait_alu 0xfffd
 ; GFX12-SDAG-NEXT:    s_setpc_b64 s[30:31]
   %gep = getelementptr i8, ptr addrspace(1) %p, i64 8589938688
   %load = load i8, ptr addrspace(1) %gep, align 4
@@ -1264,9 +1288,11 @@ define i8 @global_inst_valu_offset_64bit_13bit_split0(ptr addrspace(1) %p) {
 ; GFX12-GISEL-NEXT:    s_wait_bvhcnt 0x0
 ; GFX12-GISEL-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-GISEL-NEXT:    v_add_co_u32 v0, vcc_lo, 0x1fff, v0
+; GFX12-GISEL-NEXT:    s_wait_alu 0xfffd
 ; GFX12-GISEL-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, 2, v1, vcc_lo
 ; GFX12-GISEL-NEXT:    global_load_u8 v0, v[0:1], off
 ; GFX12-GISEL-NEXT:    s_wait_loadcnt 0x0
+; GFX12-GISEL-NEXT:    s_wait_alu 0xfffd
 ; GFX12-GISEL-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-SDAG-LABEL: global_inst_valu_offset_64bit_13bit_split0:
@@ -1304,9 +1330,11 @@ define i8 @global_inst_valu_offset_64bit_13bit_split0(ptr addrspace(1) %p) {
 ; GFX12-SDAG-NEXT:    s_wait_bvhcnt 0x0
 ; GFX12-SDAG-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-SDAG-NEXT:    v_add_co_u32 v0, vcc_lo, 0, v0
+; GFX12-SDAG-NEXT:    s_wait_alu 0xfffd
 ; GFX12-SDAG-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, 2, v1, vcc_lo
 ; GFX12-SDAG-NEXT:    global_load_u8 v0, v[0:1], off offset:8191
 ; GFX12-SDAG-NEXT:    s_wait_loadcnt 0x0
+; GFX12-SDAG-NEXT:    s_wait_alu 0xfffd
 ; GFX12-SDAG-NEXT:    s_setpc_b64 s[30:31]
   %gep = getelementptr i8, ptr addrspace(1) %p, i64 8589942783
   %load = load i8, ptr addrspace(1) %gep, align 4
@@ -1350,9 +1378,11 @@ define i8 @global_inst_valu_offset_64bit_13bit_split1(ptr addrspace(1) %p) {
 ; GFX12-GISEL-NEXT:    s_wait_bvhcnt 0x0
 ; GFX12-GISEL-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-GISEL-NEXT:    v_add_co_u32 v0, vcc_lo, 0x2000, v0
+; GFX12-GISEL-NEXT:    s_wait_alu 0xfffd
 ; GFX12-GISEL-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, 2, v1, vcc_lo
 ; GFX12-GISEL-NEXT:    global_load_u8 v0, v[0:1], off
 ; GFX12-GISEL-NEXT:    s_wait_loadcnt 0x0
+; GFX12-GISEL-NEXT:    s_wait_alu 0xfffd
 ; GFX12-GISEL-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX12-SDAG-LABEL: global_inst_valu_offset_64bit_13bit_split1:
@@ -1363,9 +1393,11 @@ define i8 @global_inst_valu_offset_64bit_13bit_split1(ptr addrspace(1) %p) {
 ; GFX12-SDAG-NEXT:    s_wait_bvhcnt 0x0
 ; GFX12-SDAG-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-SDAG-NEXT:    v_add_co_u32 v0, vcc_lo, 0, v0
+; GFX12-SDAG-NEXT:    s_wait_alu 0xfffd
 ; GFX12-SDAG-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, 2, v1, vcc_lo
 ; GFX12-SDAG-NEXT:    global_load_u8 v0, v[0:1], off offset:8192
 ; GFX12-SDAG-NEXT:    s_wait_loadcnt 0x0
+; GFX12-SDAG-NEXT:    s_wait_alu 0xfffd
 ; GFX12-SDAG-NEXT:    s_setpc_b64 s[30:31]
   %gep = getelementptr i8, ptr addrspace(1) %p, i64 8589942784
   %load = load i8, ptr addrspace(1) %gep, align 4
@@ -1410,9 +1442,11 @@ define i8 @global_inst_valu_offset_64bit_11bit_neg_high_split0(ptr addrspace(1)
 ; GFX12-GISEL-NEXT:    s_wait_bvhcnt 0x0
 ; GFX12-GISEL-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-GISEL-NEXT:    v_add_co_u32 v0, vcc_lo, 0x7ff, v0
+; GFX12-GISEL-NEXT:    s_wait_alu 0xfffd
 ; GFX12-GISEL-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, 0x80000000, v1, vcc_lo
 ; GFX12-GISEL-NEXT:    global_load_u8 v0, v[0:1], off
 ; GFX12-GISEL-NEXT:    s_wait_loadcnt 0x0
+; GFX12-GISEL-NEXT:    s_wait_alu 0xfffd
 ; GFX12-GISEL-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-SDAG-LABEL: global_inst_valu_offset_64bit_11bit_neg_high_split0:
@@ -1451,9 +1485,11 @@ define i8 @global_inst_valu_offset_64bit_11bit_neg_high_split0(ptr addrspace(1)
 ; GFX12-SDAG-NEXT:    s_wait_bvhcnt 0x0
 ; GFX12-SDAG-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-SDAG-NEXT:    v_add_co_u32 v0, vcc_lo, 0x800000, v0
+; GFX12-SDAG-NEXT:    s_wait_alu 0xfffd
 ; GFX12-SDAG-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, 0x80000000, v1, vcc_lo
 ; GFX12-SDAG-NEXT:    global_load_u8 v0, v[0:1], off offset:-8386561
 ; GFX12-SDAG-NEXT:    s_wait_loadcnt 0x0
+; GFX12-SDAG-NEXT:    s_wait_alu 0xfffd
 ; GFX12-SDAG-NEXT:    s_setpc_b64 s[30:31]
   %gep = getelementptr i8, ptr addrspace(1) %p, i64 -9223372036854773761
   %load = load i8, ptr addrspace(1) %gep, align 4
@@ -1498,9 +1534,11 @@ define i8 @global_inst_valu_offset_64bit_11bit_neg_high_split1(ptr addrspace(1)
 ; GFX12-GISEL-NEXT:    s_wait_bvhcnt 0x0
 ; GFX12-GISEL-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-GISEL-NEXT:    v_add_co_u32 v0, vcc_lo, 0x800, v0
+; GFX12-GISEL-NEXT:    s_wait_alu 0xfffd
 ; GFX12-GISEL-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, 0x80000000, v1, vcc_lo
 ; GFX12-GISEL-NEXT:    global_load_u8 v0, v[0:1], off
 ; GFX12-GISEL-NEXT:    s_wait_loadcnt 0x0
+; GFX12-GISEL-NEXT:    s_wait_alu 0xfffd
 ; GFX12-GISEL-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-SDAG-LABEL: global_inst_valu_offset_64bit_11bit_neg_high_split1:
@@ -1530,9 +1568,11 @@ define i8 @global_inst_valu_offset_64bit_11bit_neg_high_split1(ptr addrspace(1)
 ; GFX12-SDAG-NEXT:    s_wait_bvhcnt 0x0
 ; GFX12-SDAG-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-SDAG-NEXT:    v_add_co_u32 v0, vcc_lo, 0x800000, v0
+; GFX12-SDAG-NEXT:    s_wait_alu 0xfffd
 ; GFX12-SDAG-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, 0x80000000, v1, vcc_lo
 ; GFX12-SDAG-NEXT:    global_load_u8 v0, v[0:1], off offset:-8386560
 ; GFX12-SDAG-NEXT:    s_wait_loadcnt 0x0
+; GFX12-SDAG-NEXT:    s_wait_alu 0xfffd
 ; GFX12-SDAG-NEXT:    s_setpc_b64 s[30:31]
   %gep = getelementptr i8, ptr addrspace(1) %p, i64 -9223372036854773760
   %load = load i8, ptr addrspace(1) %gep, align 4
@@ -1577,9 +1617,11 @@ define i8 @global_inst_valu_offset_64bit_12bit_neg_high_split0(ptr addrspace(1)
 ; GFX12-GISEL-NEXT:    s_wait_bvhcnt 0x0
 ; GFX12-GISEL-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-GISEL-NEXT:    v_add_co_u32 v0, vcc_lo, 0xfff, v0
+; GFX12-GISEL-NEXT:    s_wait_alu 0xfffd
 ; GFX12-GISEL-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, 0x80000000, v1, vcc_lo
 ; GFX12-GISEL-NEXT:    global_load_u8 v0, v[0:1], off
 ; GFX12-GISEL-NEXT:    s_wait_loadcnt 0x0
+; GFX12-GISEL-NEXT:    s_wait_alu 0xfffd
 ; GFX12-GISEL-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-SDAG-LABEL: global_inst_valu_offset_64bit_12bit_neg_high_split0:
@@ -1618,9 +1660,11 @@ define i8 @global_inst_valu_offset_64bit_12bit_neg_high_split0(ptr addrspace(1)
 ; GFX12-SDAG-NEXT:    s_wait_bvhcnt 0x0
 ; GFX12-SDAG-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-SDAG-NEXT:    v_add_co_u32 v0, vcc_lo, 0x800000, v0
+; GFX12-SDAG-NEXT:    s_wait_alu 0xfffd
 ; GFX12-SDAG-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, 0x80000000, v1, vcc_lo
 ; GFX12-SDAG-NEXT:    global_load_u8 v0, v[0:1], off offset:-8384513
 ; GFX12-SDAG-NEXT:    s_wait_loadcnt 0x0
+; GFX12-SDAG-NEXT:    s_wait_alu 0xfffd
 ; GFX12-SDAG-NEXT:    s_setpc_b64 s[30:31]
   %gep = getelementptr i8, ptr addrspace(1) %p, i64 -9223372036854771713
   %load = load i8, ptr addrspace(1) %gep, align 4
@@ -1665,9 +1709,11 @@ define i8 @global_inst_valu_offset_64bit_12bit_neg_high_split1(ptr addrspace(1)
 ; GFX12-GISEL-NEXT:    s_wait_bvhcnt 0x0
 ; GFX12-GISEL-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-GISEL-NEXT:    v_add_co_u32 v0, vcc_lo, 0x1000, v0
+; GFX12-GISEL-NEXT:    s_wait_alu 0xfffd
 ; GFX12-GISEL-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, 0x80000000, v1, vcc_lo
 ; GFX12-GISEL-NEXT:    global_load_u8 v0, v[0:1], off
 ; GFX12-GISEL-NEXT:    s_wait_loadcnt 0x0
+; GFX12-GISEL-NEXT:    s_wait_alu 0xfffd
 ; GFX12-GISEL-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-SDAG-LABEL: global_inst_valu_offset_64bit_12bit_neg_high_split1:
@@ -1688,9 +1734,11 @@ define i8 @global_inst_valu_offset_64bit_12bit_neg_high_split1(ptr addrspace(1)
 ; GFX12-SDAG-NEXT:    s_wait_bvhcnt 0x0
 ; GFX12-SDAG-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-SDAG-NEXT:    v_add_co_u32 v0, vcc_lo, 0x800000, v0
+; GFX12-SDAG-NEXT:    s_wait_alu 0xfffd
 ; GFX12-SDAG-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, 0x80000000, v1, vcc_lo
 ; GFX12-SDAG-NEXT:    global_load_u8 v0, v[0:1], off offset:-8384512
 ; GFX12-SDAG-NEXT:    s_wait_loadcnt 0x0
+; GFX12-SDAG-NEXT:    s_wait_alu 0xfffd
 ; GFX12-SDAG-NEXT:    s_setpc_b64 s[30:31]
   %gep = getelementptr i8, ptr addrspace(1) %p, i64 -9223372036854771712
   %load = load i8, ptr addrspace(1) %gep, align 4
@@ -1735,9 +1783,11 @@ define i8 @global_inst_valu_offset_64bit_13bit_neg_high_split0(ptr addrspace(1)
 ; GFX12-GISEL-NEXT:    s_wait_bvhcnt 0x0
 ; GFX12-GISEL-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-GISEL-NEXT:    v_add_co_u32 v0, vcc_lo, 0x1fff, v0
+; GFX12-GISEL-NEXT:    s_wait_alu 0xfffd
 ; GFX12-GISEL-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, 0x80000000, v1, vcc_lo
 ; GFX12-GISEL-NEXT:    global_load_u8 v0, v[0:1], off
 ; GFX12-GISEL-NEXT:    s_wait_loadcnt 0x0
+; GFX12-GISEL-NEXT:    s_wait_alu 0xfffd
 ; GFX12-GISEL-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-SDAG-LABEL: global_inst_valu_offset_64bit_13bit_neg_high_split0:
@@ -1776,9 +1826,11 @@ define i8 @global_inst_valu_offset_64bit_13bit_neg_high_split0(ptr addrspace(1)
 ; GFX12-SDAG-NEXT:    s_wait_bvhcnt 0x0
 ; GFX12-SDAG-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-SDAG-NEXT:    v_add_co_u32 v0, vcc_lo, 0x800000, v0
+; GFX12-SDAG-NEXT:    s_wait_alu 0xfffd
 ; GFX12-SDAG-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, 0x80000000, v1, vcc_lo
 ; GFX12-SDAG-NEXT:    global_load_u8 v0, v[0:1], off offset:-8380417
 ; GFX12-SDAG-NEXT:    s_wait_loadcnt 0x0
+; GFX12-SDAG-NEXT:    s_wait_alu 0xfffd
 ; GFX12-SDAG-NEXT:    s_setpc_b64 s[30:31]
   %gep = getelementptr i8, ptr addrspace(1) %p, i64 -9223372036854767617
   %load = load i8, ptr addrspace(1) %gep, align 4
@@ -1823,9 +1875,11 @@ define i8 @global_inst_valu_offset_64bit_13bit_neg_high_split1(ptr addrspace(1)
 ; GFX12-GISEL-NEXT:    s_wait_bvhcnt 0x0
 ; GFX12-GISEL-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-GISEL-NEXT:    v_add_co_u32 v0, vcc_lo, 0x2000, v0
+; GFX12-GISEL-NEXT:    s_wait_alu 0xfffd
 ; GFX12-GISEL-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, 0x80000000, v1, vcc_lo
 ; GFX12-GISEL-NEXT:    global_load_u8 v0, v[0:1], off
 ; GFX12-GISEL-NEXT:    s_wait_loadcnt 0x0
+; GFX12-GISEL-NEXT:    s_wait_alu 0xfffd
 ; GFX12-GISEL-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-SDAG-LABEL: global_inst_valu_offset_64bit_13bit_neg_high_split1:
@@ -1846,9 +1900,11 @@ define i8 @global_inst_valu_offset_64bit_13bit_neg_high_split1(ptr addrspace(1)
 ; GFX12-SDAG-NEXT:    s_wait_bvhcnt 0x0
 ; GFX12-SDAG-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-SDAG-NEXT:    v_add_co_u32 v0, vcc_lo, 0x800000, v0
+; GFX12-SDAG-NEXT:    s_wait_alu 0xfffd
 ; GFX12-SDAG-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, 0x80000000, v1, vcc_lo
 ; GFX12-SDAG-NEXT:    global_load_u8 v0, v[0:1], off offset:-8380416
 ; GFX12-SDAG-NEXT:    s_wait_loadcnt 0x0
+; GFX12-SDAG-NEXT:    s_wait_alu 0xfffd
 ; GFX12-SDAG-NEXT:    s_setpc_b64 s[30:31]
   %gep = getelementptr i8, ptr addrspace(1) %p, i64 -9223372036854767616
   %load = load i8, ptr addrspace(1) %gep, align 4
@@ -2655,7 +2711,7 @@ define amdgpu_kernel void @global_inst_salu_offset_64bit_11bit_split0(ptr addrsp
 ; GFX12-SDAG-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
 ; GFX12-SDAG-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-SDAG-NEXT:    v_add_co_u32 v0, s0, 0, s0
-; GFX12-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX12-SDAG-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-SDAG-NEXT:    v_add_co_ci_u32_e64 v1, null, 2, s1, s0
 ; GFX12-SDAG-NEXT:    global_load_u8 v0, v[0:1], off offset:2047 scope:SCOPE_SYS
 ; GFX12-SDAG-NEXT:    s_wait_loadcnt 0x0
@@ -2748,7 +2804,7 @@ define amdgpu_kernel void @global_inst_salu_offset_64bit_11bit_split1(ptr addrsp
 ; GFX12-SDAG-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
 ; GFX12-SDAG-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-SDAG-NEXT:    v_add_co_u32 v0, s0, 0, s0
-; GFX12-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX12-SDAG-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-SDAG-NEXT:    v_add_co_ci_u32_e64 v1, null, 2, s1, s0
 ; GFX12-SDAG-NEXT:    global_load_u8 v0, v[0:1], off offset:2048 scope:SCOPE_SYS
 ; GFX12-SDAG-NEXT:    s_wait_loadcnt 0x0
@@ -2841,7 +2897,7 @@ define amdgpu_kernel void @global_inst_salu_offset_64bit_12bit_split0(ptr addrsp
 ; GFX12-SDAG-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
 ; GFX12-SDAG-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-SDAG-NEXT:    v_add_co_u32 v0, s0, 0, s0
-; GFX12-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX12-SDAG-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-SDAG-NEXT:    v_add_co_ci_u32_e64 v1, null, 2, s1, s0
 ; GFX12-SDAG-NEXT:    global_load_u8 v0, v[0:1], off offset:4095 scope:SCOPE_SYS
 ; GFX12-SDAG-NEXT:    s_wait_loadcnt 0x0
@@ -2934,7 +2990,7 @@ define amdgpu_kernel void @global_inst_salu_offset_64bit_12bit_split1(ptr addrsp
 ; GFX12-SDAG-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
 ; GFX12-SDAG-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-SDAG-NEXT:    v_add_co_u32 v0, s0, 0, s0
-; GFX12-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX12-SDAG-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-SDAG-NEXT:    v_add_co_ci_u32_e64 v1, null, 2, s1, s0
 ; GFX12-SDAG-NEXT:    global_load_u8 v0, v[0:1], off offset:4096 scope:SCOPE_SYS
 ; GFX12-SDAG-NEXT:    s_wait_loadcnt 0x0
@@ -3027,7 +3083,7 @@ define amdgpu_kernel void @global_inst_salu_offset_64bit_13bit_split0(ptr addrsp
 ; GFX12-SDAG-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
 ; GFX12-SDAG-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-SDAG-NEXT:    v_add_co_u32 v0, s0, 0, s0
-; GFX12-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX12-SDAG-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-SDAG-NEXT:    v_add_co_ci_u32_e64 v1, null, 2, s1, s0
 ; GFX12-SDAG-NEXT:    global_load_u8 v0, v[0:1], off offset:8191 scope:SCOPE_SYS
 ; GFX12-SDAG-NEXT:    s_wait_loadcnt 0x0
@@ -3120,7 +3176,7 @@ define amdgpu_kernel void @global_inst_salu_offset_64bit_13bit_split1(ptr addrsp
 ; GFX12-SDAG-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
 ; GFX12-SDAG-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-SDAG-NEXT:    v_add_co_u32 v0, s0, 0, s0
-; GFX12-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX12-SDAG-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-SDAG-NEXT:    v_add_co_ci_u32_e64 v1, null, 2, s1, s0
 ; GFX12-SDAG-NEXT:    global_load_u8 v0, v[0:1], off offset:8192 scope:SCOPE_SYS
 ; GFX12-SDAG-NEXT:    s_wait_loadcnt 0x0
diff --git a/llvm/test/CodeGen/AMDGPU/pal-metadata-3.0.ll b/llvm/test/CodeGen/AMDGPU/pal-metadata-3.0.ll
index f89341d539a0f..7536e83a9da6b 100644
--- a/llvm/test/CodeGen/AMDGPU/pal-metadata-3.0.ll
+++ b/llvm/test/CodeGen/AMDGPU/pal-metadata-3.0.ll
@@ -53,6 +53,7 @@
 ; CHECK-NEXT:      .cs:
 ; CHECK-NEXT:        .checksum_value: 0x9444d7d0
 ; CHECK-NEXT:        .debug_mode:     false
+; CHECK-NEXT:        .entry_point:    _amdgpu_cs
 ; CHECK-NEXT:        .entry_point_symbol:    _amdgpu_cs_main
 ; CHECK-NEXT:        .excp_en:        0
 ; CHECK-NEXT:        .float_mode:     0xc0
@@ -109,6 +110,7 @@
 ; CHECK-NEXT:        .wgp_mode:       false
 ; CHECK-NEXT:      .gs:
 ; CHECK-NEXT:        .debug_mode:     false
+; CHECK-NEXT:        .entry_point:    _amdgpu_gs
 ; CHECK-NEXT:        .entry_point_symbol:    gs_shader
 ; CHECK-NEXT:        .ieee_mode:      false
 ; CHECK-NEXT:        .lds_size:       0x200
@@ -120,6 +122,7 @@
 ; CHECK-NEXT:        .wgp_mode:       true
 ; CHECK-NEXT:      .hs:
 ; CHECK-NEXT:        .debug_mode:     false
+; CHECK-NEXT:        .entry_point:    _amdgpu_hs
 ; CHECK-NEXT:        .entry_point_symbol:    hs_shader
 ; CHECK-NEXT:        .ieee_mode:      false
 ; CHECK-NEXT:        .lds_size:       0x1000
@@ -131,6 +134,7 @@
 ; CHECK-NEXT:        .wgp_mode:       true
 ; CHECK-NEXT:      .ps:
 ; CHECK-NEXT:        .debug_mode:     false
+; CHECK-NEXT:        .entry_point:    _amdgpu_ps
 ; CHECK-NEXT:        .entry_point_symbol:    ps_shader
 ; CHECK-NEXT:        .ieee_mode:      false
 ; CHECK-NEXT:        .lds_size:       0
diff --git a/llvm/test/CodeGen/AMDGPU/peephole-opt-fold-reg-sequence-subreg.mir b/llvm/test/CodeGen/AMDGPU/peephole-opt-fold-reg-sequence-subreg.mir
new file mode 100644
index 0000000000000..057769372c041
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/peephole-opt-fold-reg-sequence-subreg.mir
@@ -0,0 +1,189 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 5
+# RUN: llc -mtriple=amdgcn--amdhsa -mcpu=gfx940 -run-pass=peephole-opt -o - %s | FileCheck %s
+
+---
+name:            reg_sequence_extract_subreg_sub0_from_regsequence_sub0_sub1
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    liveins: $vgpr0_vgpr1
+
+    ; CHECK-LABEL: name: reg_sequence_extract_subreg_sub0_from_regsequence_sub0_sub1
+    ; CHECK: liveins: $vgpr0_vgpr1
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:vreg_64_align2 = COPY $vgpr0_vgpr1
+    ; CHECK-NEXT: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+    ; CHECK-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_64_align2 = REG_SEQUENCE [[COPY]].sub0, %subreg.sub0, [[V_MOV_B32_e32_]], %subreg.sub1
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY [[COPY]].sub0
+    ; CHECK-NEXT: S_ENDPGM 0, implicit [[COPY1]]
+    %0:vreg_64_align2 = COPY $vgpr0_vgpr1
+    %1:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+    %2:vreg_64_align2 = REG_SEQUENCE %0.sub0, %subreg.sub0, %1, %subreg.sub1
+    %3:vgpr_32 = COPY %2.sub0
+    S_ENDPGM 0, implicit %3
+
+...
+
+---
+name:            reg_sequence_extract_subreg_sub1_from_regsequence_sub0_sub1
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    liveins: $vgpr0_vgpr1
+
+    ; CHECK-LABEL: name: reg_sequence_extract_subreg_sub1_from_regsequence_sub0_sub1
+    ; CHECK: liveins: $vgpr0_vgpr1
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:vreg_64_align2 = COPY $vgpr0_vgpr1
+    ; CHECK-NEXT: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+    ; CHECK-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_64_align2 = REG_SEQUENCE [[COPY]].sub0, %subreg.sub0, [[V_MOV_B32_e32_]], %subreg.sub1
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY [[V_MOV_B32_e32_]]
+    ; CHECK-NEXT: S_ENDPGM 0, implicit [[COPY1]]
+    %0:vreg_64_align2 = COPY $vgpr0_vgpr1
+    %1:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+    %2:vreg_64_align2 = REG_SEQUENCE %0.sub0, %subreg.sub0, %1, %subreg.sub1
+    %3:vgpr_32 = COPY %2.sub1
+    S_ENDPGM 0, implicit %3
+
+...
+
+---
+name:            reg_sequence_extract_subreg_sub0_from_regsequence_sub1_sub0
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    liveins: $vgpr0_vgpr1
+
+    ; CHECK-LABEL: name: reg_sequence_extract_subreg_sub0_from_regsequence_sub1_sub0
+    ; CHECK: liveins: $vgpr0_vgpr1
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:vreg_64_align2 = COPY $vgpr0_vgpr1
+    ; CHECK-NEXT: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+    ; CHECK-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_64_align2 = REG_SEQUENCE [[V_MOV_B32_e32_]], %subreg.sub1, [[COPY]].sub0, %subreg.sub0
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY [[COPY]].sub0
+    ; CHECK-NEXT: S_ENDPGM 0, implicit [[COPY1]]
+    %0:vreg_64_align2 = COPY $vgpr0_vgpr1
+    %1:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+    %2:vreg_64_align2 = REG_SEQUENCE %1, %subreg.sub1, %0.sub0, %subreg.sub0
+    %3:vgpr_32 = COPY %2.sub0
+    S_ENDPGM 0, implicit %3
+
+...
+
+---
+name:            reg_sequence_extract_subreg_sub1_from_regsequence_sub1_sub0
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    liveins: $vgpr0_vgpr1
+
+    ; CHECK-LABEL: name: reg_sequence_extract_subreg_sub1_from_regsequence_sub1_sub0
+    ; CHECK: liveins: $vgpr0_vgpr1
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:vreg_64_align2 = COPY $vgpr0_vgpr1
+    ; CHECK-NEXT: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+    ; CHECK-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_64_align2 = REG_SEQUENCE [[V_MOV_B32_e32_]], %subreg.sub1, [[COPY]].sub0, %subreg.sub0
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY [[V_MOV_B32_e32_]]
+    ; CHECK-NEXT: S_ENDPGM 0, implicit [[COPY1]]
+    %0:vreg_64_align2 = COPY $vgpr0_vgpr1
+    %1:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+    %2:vreg_64_align2 = REG_SEQUENCE %1, %subreg.sub1, %0.sub0, %subreg.sub0
+    %3:vgpr_32 = COPY %2.sub1
+    S_ENDPGM 0, implicit %3
+
+...
+
+---
+name:            reg_sequence_extract_subreg_sub0_from_vreg96
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    liveins: $vgpr1_vgpr2
+
+    ; CHECK-LABEL: name: reg_sequence_extract_subreg_sub0_from_vreg96
+    ; CHECK: liveins: $vgpr1_vgpr2
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr1_vgpr2
+    ; CHECK-NEXT: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+    ; CHECK-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_96 = REG_SEQUENCE [[COPY]].sub0, %subreg.sub0, [[V_MOV_B32_e32_]], %subreg.sub1, [[COPY]].sub1, %subreg.sub2
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY [[COPY]].sub0
+    ; CHECK-NEXT: S_ENDPGM 0, implicit [[COPY1]]
+    %0:vreg_64 = COPY $vgpr1_vgpr2
+    %1:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+    %2:vreg_96 = REG_SEQUENCE %0.sub0, %subreg.sub0, %1, %subreg.sub1, %0.sub1, %subreg.sub2
+    %3:vgpr_32 = COPY %2.sub0
+    S_ENDPGM 0, implicit %3
+
+...
+
+---
+name:            reg_sequence_extract_subreg_sub1_from_vreg96
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    liveins: $vgpr1_vgpr2
+
+    ; CHECK-LABEL: name: reg_sequence_extract_subreg_sub1_from_vreg96
+    ; CHECK: liveins: $vgpr1_vgpr2
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr1_vgpr2
+    ; CHECK-NEXT: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+    ; CHECK-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_96 = REG_SEQUENCE [[COPY]].sub0, %subreg.sub0, [[V_MOV_B32_e32_]], %subreg.sub1, [[COPY]].sub1, %subreg.sub2
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY [[COPY]].sub0
+    ; CHECK-NEXT: S_ENDPGM 0, implicit [[COPY1]]
+    %0:vreg_64 = COPY $vgpr1_vgpr2
+    %1:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+    %2:vreg_96 = REG_SEQUENCE %0.sub0, %subreg.sub0, %1, %subreg.sub1, %0.sub1, %subreg.sub2
+    %3:vgpr_32 = COPY %2.sub0
+    S_ENDPGM 0, implicit %3
+
+...
+
+---
+name:            reg_sequence_compose_0
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    liveins: $vgpr1_vgpr2
+
+    ; CHECK-LABEL: name: reg_sequence_compose_0
+    ; CHECK: liveins: $vgpr1_vgpr2
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr1_vgpr2
+    ; CHECK-NEXT: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+    ; CHECK-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_96 = REG_SEQUENCE [[COPY]], %subreg.sub0_sub1, [[V_MOV_B32_e32_]], %subreg.sub2
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY [[REG_SEQUENCE]].sub1
+    ; CHECK-NEXT: S_ENDPGM 0, implicit [[COPY1]]
+    %0:vreg_64 = COPY $vgpr1_vgpr2
+    %1:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+    %2:vreg_96 = REG_SEQUENCE %0, %subreg.sub0_sub1, %1, %subreg.sub2
+    %3:vgpr_32 = COPY %2.sub1
+    S_ENDPGM 0, implicit %3
+
+...
+
+---
+name:            reg_sequence_compose_1
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    liveins: $vgpr1_vgpr2_vgpr3_vgpr4
+
+    ; CHECK-LABEL: name: reg_sequence_compose_1
+    ; CHECK: liveins: $vgpr1_vgpr2_vgpr3_vgpr4
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:vreg_128 = COPY $vgpr1_vgpr2_vgpr3_vgpr4
+    ; CHECK-NEXT: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+    ; CHECK-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_96 = REG_SEQUENCE [[COPY]].sub2_sub3, %subreg.sub0_sub1, [[COPY]].sub0, %subreg.sub2
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY [[REG_SEQUENCE]].sub1
+    ; CHECK-NEXT: S_ENDPGM 0, implicit [[COPY1]]
+    %0:vreg_128 = COPY $vgpr1_vgpr2_vgpr3_vgpr4
+    %1:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+    %2:vreg_96 = REG_SEQUENCE %0.sub2_sub3, %subreg.sub0_sub1, %0.sub0, %subreg.sub2
+    %3:vgpr_32 = COPY %2.sub1
+    S_ENDPGM 0, implicit %3
+
+...
+
+
+
diff --git a/llvm/test/CodeGen/AMDGPU/peephole-opt-regseq-removal.mir b/llvm/test/CodeGen/AMDGPU/peephole-opt-regseq-removal.mir
index c5e51aa1b4fe8..e1ff42125ce9a 100644
--- a/llvm/test/CodeGen/AMDGPU/peephole-opt-regseq-removal.mir
+++ b/llvm/test/CodeGen/AMDGPU/peephole-opt-regseq-removal.mir
@@ -23,8 +23,8 @@ body:             |
     ; GCN-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
     ; GCN-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1
     ; GCN-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[REG_SEQUENCE]].sub1, %subreg.sub0, [[REG_SEQUENCE]].sub0, %subreg.sub1
-    ; GCN-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY [[COPY1]]
-    ; GCN-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY [[COPY]]
+    ; GCN-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY [[REG_SEQUENCE]].sub1
+    ; GCN-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY [[REG_SEQUENCE]].sub0
     ; GCN-NEXT: KILL [[COPY3]], implicit [[COPY2]]
     %0:vgpr_32 = COPY $vgpr0
     %1:vgpr_32 = COPY $vgpr1
diff --git a/llvm/test/CodeGen/AMDGPU/perfhint.ll b/llvm/test/CodeGen/AMDGPU/perfhint.ll
index f4ee4fb82e7a3..3ff9e6a3b3da4 100644
--- a/llvm/test/CodeGen/AMDGPU/perfhint.ll
+++ b/llvm/test/CodeGen/AMDGPU/perfhint.ll
@@ -8,7 +8,7 @@
 ; GCN: WaveLimiterHint : 1
 define amdgpu_kernel void @test_membound(ptr addrspace(1) nocapture readonly %arg, ptr addrspace(1) nocapture %arg1) {
 ; CHECK-LABEL: define amdgpu_kernel void @test_membound(
-; CHECK-SAME: ptr addrspace(1) nocapture readonly [[ARG:%.*]], ptr addrspace(1) nocapture [[ARG1:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-SAME: ptr addrspace(1) readonly captures(none) [[ARG:%.*]], ptr addrspace(1) captures(none) [[ARG1:%.*]]) #[[ATTR0:[0-9]+]] {
 ; CHECK-NEXT:  [[BB:.*:]]
 ; CHECK-NEXT:    [[TMP:%.*]] = tail call i32 @llvm.amdgcn.workitem.id.x()
 ; CHECK-NEXT:    [[TMP2:%.*]] = zext i32 [[TMP]] to i64
@@ -42,7 +42,7 @@ bb:
 ; GCN: MemoryBound: 1
 define amdgpu_kernel void @test_membound_1(ptr addrspace(1) nocapture readonly %ptr.0,
 ; CHECK-LABEL: define amdgpu_kernel void @test_membound_1(
-; CHECK-SAME: ptr addrspace(1) nocapture readonly [[PTR_0:%.*]], ptr addrspace(1) nocapture [[PTR_1:%.*]], <2 x double> [[ARG_0:%.*]], i32 [[ARG_1:%.*]], <4 x double> [[ARG_2:%.*]]) #[[ATTR1:[0-9]+]] {
+; CHECK-SAME: ptr addrspace(1) readonly captures(none) [[PTR_0:%.*]], ptr addrspace(1) captures(none) [[PTR_1:%.*]], <2 x double> [[ARG_0:%.*]], i32 [[ARG_1:%.*]], <4 x double> [[ARG_2:%.*]]) #[[ATTR1:[0-9]+]] {
 ; CHECK-NEXT:  [[BB_ENTRY:.*:]]
 ; CHECK-NEXT:    [[ID_32:%.*]] = tail call i32 @llvm.amdgcn.workitem.id.x()
 ; CHECK-NEXT:    [[ID_0:%.*]] = zext i32 [[ID_32]] to i64
@@ -167,7 +167,7 @@ bb.ret:
 ; GCN: WaveLimiterHint : 1
 define amdgpu_kernel void @test_large_stride(ptr addrspace(1) nocapture %arg) {
 ; CHECK-LABEL: define amdgpu_kernel void @test_large_stride(
-; CHECK-SAME: ptr addrspace(1) nocapture [[ARG:%.*]]) #[[ATTR2:[0-9]+]] {
+; CHECK-SAME: ptr addrspace(1) captures(none) [[ARG:%.*]]) #[[ATTR2:[0-9]+]] {
 ; CHECK-NEXT:  [[BB:.*:]]
 ; CHECK-NEXT:    [[TMP:%.*]] = getelementptr inbounds i32, ptr addrspace(1) [[ARG]], i64 4096
 ; CHECK-NEXT:    [[TMP1:%.*]] = load i32, ptr addrspace(1) [[TMP]], align 4
@@ -210,7 +210,7 @@ bb:
 ; GCN: WaveLimiterHint : 1
 define amdgpu_kernel void @test_indirect(ptr addrspace(1) nocapture %arg) {
 ; CHECK-LABEL: define amdgpu_kernel void @test_indirect(
-; CHECK-SAME: ptr addrspace(1) nocapture [[ARG:%.*]]) #[[ATTR2]] {
+; CHECK-SAME: ptr addrspace(1) captures(none) [[ARG:%.*]]) #[[ATTR2]] {
 ; CHECK-NEXT:  [[BB:.*:]]
 ; CHECK-NEXT:    [[TMP:%.*]] = getelementptr inbounds i32, ptr addrspace(1) [[ARG]], i64 1
 ; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i32, ptr addrspace(1) [[ARG]], i64 2
@@ -315,7 +315,7 @@ bb2:                                              ; preds = %bb1
 
 define void @test_membound_func(ptr addrspace(1) nocapture readonly %arg, ptr addrspace(1) nocapture %arg1) {
 ; CHECK-LABEL: define void @test_membound_func(
-; CHECK-SAME: ptr addrspace(1) nocapture readonly [[ARG:%.*]], ptr addrspace(1) nocapture [[ARG1:%.*]]) #[[ATTR1]] {
+; CHECK-SAME: ptr addrspace(1) readonly captures(none) [[ARG:%.*]], ptr addrspace(1) captures(none) [[ARG1:%.*]]) #[[ATTR1]] {
 ; CHECK-NEXT:  [[BB:.*:]]
 ; CHECK-NEXT:    [[TMP:%.*]] = tail call i32 @llvm.amdgcn.workitem.id.x()
 ; CHECK-NEXT:    [[TMP2:%.*]] = zext i32 [[TMP]] to i64
@@ -350,8 +350,8 @@ bb:
 ; GCN: WaveLimiterHint : 1
 define amdgpu_kernel void @kernel_call_test_membound_func(ptr addrspace(1) nocapture readonly %arg, ptr addrspace(1) nocapture %arg1) {
 ; CHECK-LABEL: define amdgpu_kernel void @kernel_call_test_membound_func(
-; CHECK-SAME: ptr addrspace(1) nocapture readonly [[ARG:%.*]], ptr addrspace(1) nocapture [[ARG1:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT:    call void @test_membound_func(ptr addrspace(1) nocapture readonly [[ARG]], ptr addrspace(1) nocapture [[ARG1]])
+; CHECK-SAME: ptr addrspace(1) readonly captures(none) [[ARG:%.*]], ptr addrspace(1) captures(none) [[ARG1:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    call void @test_membound_func(ptr addrspace(1) readonly captures(none) [[ARG]], ptr addrspace(1) captures(none) [[ARG1]])
 ; CHECK-NEXT:    ret void
 ;
   call void @test_membound_func(ptr addrspace(1) nocapture readonly %arg, ptr addrspace(1) nocapture %arg1)
@@ -364,8 +364,8 @@ define amdgpu_kernel void @kernel_call_test_membound_func(ptr addrspace(1) nocap
 ; GCN: WaveLimiterHint : 0
 define amdgpu_kernel void @kernel_indirect_call(ptr addrspace(1) nocapture readonly %arg, ptr addrspace(1) nocapture %arg1, ptr %fptr) {
 ; CHECK-LABEL: define amdgpu_kernel void @kernel_indirect_call(
-; CHECK-SAME: ptr addrspace(1) nocapture readonly [[ARG:%.*]], ptr addrspace(1) nocapture [[ARG1:%.*]], ptr [[FPTR:%.*]]) {
-; CHECK-NEXT:    call void [[FPTR]](ptr addrspace(1) nocapture readonly [[ARG]], ptr addrspace(1) nocapture [[ARG1]])
+; CHECK-SAME: ptr addrspace(1) readonly captures(none) [[ARG:%.*]], ptr addrspace(1) captures(none) [[ARG1:%.*]], ptr [[FPTR:%.*]]) {
+; CHECK-NEXT:    call void [[FPTR]](ptr addrspace(1) readonly captures(none) [[ARG]], ptr addrspace(1) captures(none) [[ARG1]])
 ; CHECK-NEXT:    ret void
 ;
   call void %fptr(ptr addrspace(1) nocapture readonly %arg, ptr addrspace(1) nocapture %arg1)
@@ -376,7 +376,7 @@ declare void @extern()
 
 define void @maybe_recursive_test_membound_func(ptr addrspace(1) nocapture readonly %arg, ptr addrspace(1) nocapture %arg1) {
 ; CHECK-LABEL: define void @maybe_recursive_test_membound_func(
-; CHECK-SAME: ptr addrspace(1) nocapture readonly [[ARG:%.*]], ptr addrspace(1) nocapture [[ARG1:%.*]]) #[[ATTR1]] {
+; CHECK-SAME: ptr addrspace(1) readonly captures(none) [[ARG:%.*]], ptr addrspace(1) captures(none) [[ARG1:%.*]]) #[[ATTR1]] {
 ; CHECK-NEXT:  [[BB:.*:]]
 ; CHECK-NEXT:    [[TMP:%.*]] = tail call i32 @llvm.amdgcn.workitem.id.x()
 ; CHECK-NEXT:    [[TMP2:%.*]] = zext i32 [[TMP]] to i64
@@ -413,8 +413,8 @@ bb:
 ; GCN: WaveLimiterHint : 1
 define amdgpu_kernel void @kernel_call_maybe_recursive_test_membound_func(ptr addrspace(1) nocapture readonly %arg, ptr addrspace(1) nocapture %arg1, ptr %fptr) {
 ; CHECK-LABEL: define amdgpu_kernel void @kernel_call_maybe_recursive_test_membound_func(
-; CHECK-SAME: ptr addrspace(1) nocapture readonly [[ARG:%.*]], ptr addrspace(1) nocapture [[ARG1:%.*]], ptr [[FPTR:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT:    call void @maybe_recursive_test_membound_func(ptr addrspace(1) nocapture readonly [[ARG]], ptr addrspace(1) nocapture [[ARG1]])
+; CHECK-SAME: ptr addrspace(1) readonly captures(none) [[ARG:%.*]], ptr addrspace(1) captures(none) [[ARG1:%.*]], ptr [[FPTR:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    call void @maybe_recursive_test_membound_func(ptr addrspace(1) readonly captures(none) [[ARG]], ptr addrspace(1) captures(none) [[ARG1]])
 ; CHECK-NEXT:    ret void
 ;
   call void @maybe_recursive_test_membound_func(ptr addrspace(1) nocapture readonly %arg, ptr addrspace(1) nocapture %arg1)
@@ -423,14 +423,14 @@ define amdgpu_kernel void @kernel_call_maybe_recursive_test_membound_func(ptr ad
 
 define void @mutually_recursive_test_membound_func_0(ptr addrspace(1) nocapture readonly %arg, ptr addrspace(1) nocapture %arg1) {
 ; CHECK-LABEL: define void @mutually_recursive_test_membound_func_0(
-; CHECK-SAME: ptr addrspace(1) nocapture readonly [[ARG:%.*]], ptr addrspace(1) nocapture [[ARG1:%.*]]) #[[ATTR1]] {
+; CHECK-SAME: ptr addrspace(1) readonly captures(none) [[ARG:%.*]], ptr addrspace(1) captures(none) [[ARG1:%.*]]) #[[ATTR1]] {
 ; CHECK-NEXT:    [[TMP:%.*]] = tail call i32 @llvm.amdgcn.workitem.id.x()
 ; CHECK-NEXT:    [[TMP2:%.*]] = zext i32 [[TMP]] to i64
 ; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds <4 x i32>, ptr addrspace(1) [[ARG]], i64 [[TMP2]]
 ; CHECK-NEXT:    [[TMP4:%.*]] = load <4 x i32>, ptr addrspace(1) [[TMP3]], align 16
 ; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr inbounds <4 x i32>, ptr addrspace(1) [[ARG1]], i64 [[TMP2]]
 ; CHECK-NEXT:    store <4 x i32> [[TMP4]], ptr addrspace(1) [[TMP5]], align 16
-; CHECK-NEXT:    call void @mutually_recursive_test_membound_func_0(ptr addrspace(1) nocapture readonly [[ARG]], ptr addrspace(1) nocapture [[ARG1]])
+; CHECK-NEXT:    call void @mutually_recursive_test_membound_func_0(ptr addrspace(1) readonly captures(none) [[ARG]], ptr addrspace(1) captures(none) [[ARG1]])
 ; CHECK-NEXT:    ret void
 ;
   %tmp = tail call i32 @llvm.amdgcn.workitem.id.x()
@@ -445,7 +445,7 @@ define void @mutually_recursive_test_membound_func_0(ptr addrspace(1) nocapture
 
 define void @mutually_recursive_test_membound_func_1(ptr addrspace(1) nocapture readonly %arg, ptr addrspace(1) nocapture %arg1) {
 ; CHECK-LABEL: define void @mutually_recursive_test_membound_func_1(
-; CHECK-SAME: ptr addrspace(1) nocapture readonly [[ARG:%.*]], ptr addrspace(1) nocapture [[ARG1:%.*]]) #[[ATTR1]] {
+; CHECK-SAME: ptr addrspace(1) readonly captures(none) [[ARG:%.*]], ptr addrspace(1) captures(none) [[ARG1:%.*]]) #[[ATTR1]] {
 ; CHECK-NEXT:    [[TMP:%.*]] = tail call i32 @llvm.amdgcn.workitem.id.x()
 ; CHECK-NEXT:    [[TMP2:%.*]] = zext i32 [[TMP]] to i64
 ; CHECK-NEXT:    [[TMP6:%.*]] = add nuw nsw i64 [[TMP2]], 1
@@ -453,7 +453,7 @@ define void @mutually_recursive_test_membound_func_1(ptr addrspace(1) nocapture
 ; CHECK-NEXT:    [[TMP8:%.*]] = load <4 x i32>, ptr addrspace(1) [[TMP7]], align 16
 ; CHECK-NEXT:    [[TMP9:%.*]] = getelementptr inbounds <4 x i32>, ptr addrspace(1) [[ARG1]], i64 [[TMP6]]
 ; CHECK-NEXT:    store <4 x i32> [[TMP8]], ptr addrspace(1) [[TMP9]], align 16
-; CHECK-NEXT:    call void @mutually_recursive_test_membound_func_1(ptr addrspace(1) nocapture readonly [[ARG]], ptr addrspace(1) nocapture [[ARG1]])
+; CHECK-NEXT:    call void @mutually_recursive_test_membound_func_1(ptr addrspace(1) readonly captures(none) [[ARG]], ptr addrspace(1) captures(none) [[ARG1]])
 ; CHECK-NEXT:    ret void
 ;
   %tmp = tail call i32 @llvm.amdgcn.workitem.id.x()
@@ -472,8 +472,8 @@ define void @mutually_recursive_test_membound_func_1(ptr addrspace(1) nocapture
 ; GCN: WaveLimiterHint : 1
 define amdgpu_kernel void @kernel_call_mutually_recursive_test_membound_func_0(ptr addrspace(1) nocapture readonly %arg, ptr addrspace(1) nocapture %arg1, ptr %fptr) {
 ; CHECK-LABEL: define amdgpu_kernel void @kernel_call_mutually_recursive_test_membound_func_0(
-; CHECK-SAME: ptr addrspace(1) nocapture readonly [[ARG:%.*]], ptr addrspace(1) nocapture [[ARG1:%.*]], ptr [[FPTR:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT:    call void @mutually_recursive_test_membound_func_0(ptr addrspace(1) nocapture readonly [[ARG]], ptr addrspace(1) nocapture [[ARG1]])
+; CHECK-SAME: ptr addrspace(1) readonly captures(none) [[ARG:%.*]], ptr addrspace(1) captures(none) [[ARG1:%.*]], ptr [[FPTR:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    call void @mutually_recursive_test_membound_func_0(ptr addrspace(1) readonly captures(none) [[ARG]], ptr addrspace(1) captures(none) [[ARG1]])
 ; CHECK-NEXT:    ret void
 ;
   call void @mutually_recursive_test_membound_func_0(ptr addrspace(1) nocapture readonly %arg, ptr addrspace(1) nocapture %arg1)
diff --git a/llvm/test/CodeGen/AMDGPU/post-ra-sched-kill-bundle-use-inst.mir b/llvm/test/CodeGen/AMDGPU/post-ra-sched-kill-bundle-use-inst.mir
index d707291d0df20..a74f1ade44981 100644
--- a/llvm/test/CodeGen/AMDGPU/post-ra-sched-kill-bundle-use-inst.mir
+++ b/llvm/test/CodeGen/AMDGPU/post-ra-sched-kill-bundle-use-inst.mir
@@ -1,5 +1,6 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
 # RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx900 -run-pass=post-RA-sched -verify-machineinstrs -o -  %s | FileCheck %s
+# RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx900 -passes=post-RA-sched -o -  %s | FileCheck %s
 
 # The scheduler was not inspecting the first instruction in the bundle
 # when adding kill flags, so it would incorrectly mark the first use
diff --git a/llvm/test/CodeGen/AMDGPU/preserve-hi16.ll b/llvm/test/CodeGen/AMDGPU/preserve-hi16.ll
index 4a1287c56ea8e..0ad1c30b5b5a4 100644
--- a/llvm/test/CodeGen/AMDGPU/preserve-hi16.ll
+++ b/llvm/test/CodeGen/AMDGPU/preserve-hi16.ll
@@ -3,7 +3,8 @@
 ; RUN: llc -mtriple=amdgcn -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck --check-prefixes=GFX9ALL,GFX900 %s
 ; RUN: llc -mtriple=amdgcn -mcpu=gfx906 -verify-machineinstrs < %s | FileCheck --check-prefixes=GFX9ALL,GFX906 %s
 ; RUN: llc -mtriple=amdgcn -mcpu=gfx1010 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX10 %s
-; RUN: llc -mtriple=amdgcn -mcpu=gfx1100 -amdgpu-enable-delay-alu=0 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX11 %s
+; RUN: llc -mtriple=amdgcn -mcpu=gfx1100 -amdgpu-enable-delay-alu=0 -mattr=+real-true16 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX11,GFX11-TRUE16 %s
+; RUN: llc -mtriple=amdgcn -mcpu=gfx1100 -amdgpu-enable-delay-alu=0 -mattr=-real-true16 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX11,GFX11-FAKE16 %s
 
 define i16 @shl_i16(i16 %x, i16 %y) {
 ; GFX8-LABEL: shl_i16:
@@ -24,11 +25,17 @@ define i16 @shl_i16(i16 %x, i16 %y) {
 ; GFX10-NEXT:    v_lshlrev_b16 v0, v1, v0
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
 ;
-; GFX11-LABEL: shl_i16:
-; GFX11:       ; %bb.0:
-; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT:    v_lshlrev_b16 v0, v1, v0
-; GFX11-NEXT:    s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: shl_i16:
+; GFX11-TRUE16:       ; %bb.0:
+; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v0.l, v1.l, v0.l
+; GFX11-TRUE16-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: shl_i16:
+; GFX11-FAKE16:       ; %bb.0:
+; GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT:    v_lshlrev_b16 v0, v1, v0
+; GFX11-FAKE16-NEXT:    s_setpc_b64 s[30:31]
   %res = shl i16 %x, %y
   ret i16 %res
 }
@@ -52,11 +59,17 @@ define i16 @lshr_i16(i16 %x, i16 %y) {
 ; GFX10-NEXT:    v_lshrrev_b16 v0, v1, v0
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
 ;
-; GFX11-LABEL: lshr_i16:
-; GFX11:       ; %bb.0:
-; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT:    v_lshrrev_b16 v0, v1, v0
-; GFX11-NEXT:    s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: lshr_i16:
+; GFX11-TRUE16:       ; %bb.0:
+; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT:    v_lshrrev_b16 v0.l, v1.l, v0.l
+; GFX11-TRUE16-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: lshr_i16:
+; GFX11-FAKE16:       ; %bb.0:
+; GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT:    v_lshrrev_b16 v0, v1, v0
+; GFX11-FAKE16-NEXT:    s_setpc_b64 s[30:31]
   %res = lshr i16 %x, %y
   ret i16 %res
 }
@@ -80,11 +93,17 @@ define i16 @ashr_i16(i16 %x, i16 %y) {
 ; GFX10-NEXT:    v_ashrrev_i16 v0, v1, v0
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
 ;
-; GFX11-LABEL: ashr_i16:
-; GFX11:       ; %bb.0:
-; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT:    v_ashrrev_i16 v0, v1, v0
-; GFX11-NEXT:    s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: ashr_i16:
+; GFX11-TRUE16:       ; %bb.0:
+; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT:    v_ashrrev_i16 v0.l, v1.l, v0.l
+; GFX11-TRUE16-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: ashr_i16:
+; GFX11-FAKE16:       ; %bb.0:
+; GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT:    v_ashrrev_i16 v0, v1, v0
+; GFX11-FAKE16-NEXT:    s_setpc_b64 s[30:31]
   %res = ashr i16 %x, %y
   ret i16 %res
 }
@@ -108,11 +127,18 @@ define i16 @add_u16(i16 %x, i16 %y) {
 ; GFX10-NEXT:    v_add_nc_u16 v0, v0, v1
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
 ;
-; GFX11-LABEL: add_u16:
-; GFX11:       ; %bb.0:
-; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT:    v_add_nc_u16 v0, v0, v1
-; GFX11-NEXT:    s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: add_u16:
+; GFX11-TRUE16:       ; %bb.0:
+; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v0.h, v1.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v0.l, v0.l, v0.h
+; GFX11-TRUE16-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: add_u16:
+; GFX11-FAKE16:       ; %bb.0:
+; GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT:    v_add_nc_u16 v0, v0, v1
+; GFX11-FAKE16-NEXT:    s_setpc_b64 s[30:31]
   %res = add i16 %x, %y
   ret i16 %res
 }
@@ -136,11 +162,18 @@ define i16 @sub_u16(i16 %x, i16 %y) {
 ; GFX10-NEXT:    v_sub_nc_u16 v0, v0, v1
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
 ;
-; GFX11-LABEL: sub_u16:
-; GFX11:       ; %bb.0:
-; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT:    v_sub_nc_u16 v0, v0, v1
-; GFX11-NEXT:    s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: sub_u16:
+; GFX11-TRUE16:       ; %bb.0:
+; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v0.h, v1.l
+; GFX11-TRUE16-NEXT:    v_sub_nc_u16 v0.l, v0.l, v0.h
+; GFX11-TRUE16-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: sub_u16:
+; GFX11-FAKE16:       ; %bb.0:
+; GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT:    v_sub_nc_u16 v0, v0, v1
+; GFX11-FAKE16-NEXT:    s_setpc_b64 s[30:31]
   %res = sub i16 %x, %y
   ret i16 %res
 }
@@ -164,11 +197,18 @@ define i16 @mul_lo_u16(i16 %x, i16 %y) {
 ; GFX10-NEXT:    v_mul_lo_u16 v0, v0, v1
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
 ;
-; GFX11-LABEL: mul_lo_u16:
-; GFX11:       ; %bb.0:
-; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT:    v_mul_lo_u16 v0, v0, v1
-; GFX11-NEXT:    s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: mul_lo_u16:
+; GFX11-TRUE16:       ; %bb.0:
+; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v0.h, v1.l
+; GFX11-TRUE16-NEXT:    v_mul_lo_u16 v0.l, v0.l, v0.h
+; GFX11-TRUE16-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: mul_lo_u16:
+; GFX11-FAKE16:       ; %bb.0:
+; GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT:    v_mul_lo_u16 v0, v0, v1
+; GFX11-FAKE16-NEXT:    s_setpc_b64 s[30:31]
   %res = mul i16 %x, %y
   ret i16 %res
 }
@@ -192,11 +232,18 @@ define i16 @min_u16(i16 %x, i16 %y) {
 ; GFX10-NEXT:    v_min_u16 v0, v0, v1
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
 ;
-; GFX11-LABEL: min_u16:
-; GFX11:       ; %bb.0:
-; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT:    v_min_u16 v0, v0, v1
-; GFX11-NEXT:    s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: min_u16:
+; GFX11-TRUE16:       ; %bb.0:
+; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v0.h, v1.l
+; GFX11-TRUE16-NEXT:    v_min_u16 v0.l, v0.l, v0.h
+; GFX11-TRUE16-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: min_u16:
+; GFX11-FAKE16:       ; %bb.0:
+; GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT:    v_min_u16 v0, v0, v1
+; GFX11-FAKE16-NEXT:    s_setpc_b64 s[30:31]
   %cmp = icmp ule i16 %x, %y
   %res = select i1 %cmp, i16 %x, i16 %y
   ret i16 %res
@@ -221,11 +268,18 @@ define i16 @min_i16(i16 %x, i16 %y) {
 ; GFX10-NEXT:    v_min_i16 v0, v0, v1
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
 ;
-; GFX11-LABEL: min_i16:
-; GFX11:       ; %bb.0:
-; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT:    v_min_i16 v0, v0, v1
-; GFX11-NEXT:    s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: min_i16:
+; GFX11-TRUE16:       ; %bb.0:
+; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v0.h, v1.l
+; GFX11-TRUE16-NEXT:    v_min_i16 v0.l, v0.l, v0.h
+; GFX11-TRUE16-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: min_i16:
+; GFX11-FAKE16:       ; %bb.0:
+; GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT:    v_min_i16 v0, v0, v1
+; GFX11-FAKE16-NEXT:    s_setpc_b64 s[30:31]
   %cmp = icmp sle i16 %x, %y
   %res = select i1 %cmp, i16 %x, i16 %y
   ret i16 %res
@@ -250,11 +304,18 @@ define i16 @max_u16(i16 %x, i16 %y) {
 ; GFX10-NEXT:    v_max_u16 v0, v0, v1
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
 ;
-; GFX11-LABEL: max_u16:
-; GFX11:       ; %bb.0:
-; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT:    v_max_u16 v0, v0, v1
-; GFX11-NEXT:    s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: max_u16:
+; GFX11-TRUE16:       ; %bb.0:
+; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v0.h, v1.l
+; GFX11-TRUE16-NEXT:    v_max_u16 v0.l, v0.l, v0.h
+; GFX11-TRUE16-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: max_u16:
+; GFX11-FAKE16:       ; %bb.0:
+; GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT:    v_max_u16 v0, v0, v1
+; GFX11-FAKE16-NEXT:    s_setpc_b64 s[30:31]
   %cmp = icmp uge i16 %x, %y
   %res = select i1 %cmp, i16 %x, i16 %y
   ret i16 %res
@@ -279,11 +340,18 @@ define i16 @max_i16(i16 %x, i16 %y) {
 ; GFX10-NEXT:    v_max_i16 v0, v0, v1
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
 ;
-; GFX11-LABEL: max_i16:
-; GFX11:       ; %bb.0:
-; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT:    v_max_i16 v0, v0, v1
-; GFX11-NEXT:    s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: max_i16:
+; GFX11-TRUE16:       ; %bb.0:
+; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v0.h, v1.l
+; GFX11-TRUE16-NEXT:    v_max_i16 v0.l, v0.l, v0.h
+; GFX11-TRUE16-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: max_i16:
+; GFX11-FAKE16:       ; %bb.0:
+; GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT:    v_max_i16 v0, v0, v1
+; GFX11-FAKE16-NEXT:    s_setpc_b64 s[30:31]
   %cmp = icmp sge i16 %x, %y
   %res = select i1 %cmp, i16 %x, i16 %y
   ret i16 %res
@@ -309,12 +377,19 @@ define i32 @shl_i16_zext_i32(i16 %x, i16 %y) {
 ; GFX10-NEXT:    v_and_b32_e32 v0, 0xffff, v0
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
 ;
-; GFX11-LABEL: shl_i16_zext_i32:
-; GFX11:       ; %bb.0:
-; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT:    v_lshlrev_b16 v0, v1, v0
-; GFX11-NEXT:    v_and_b32_e32 v0, 0xffff, v0
-; GFX11-NEXT:    s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: shl_i16_zext_i32:
+; GFX11-TRUE16:       ; %bb.0:
+; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v0.l, v1.l, v0.l
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xffff, v0
+; GFX11-TRUE16-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: shl_i16_zext_i32:
+; GFX11-FAKE16:       ; %bb.0:
+; GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT:    v_lshlrev_b16 v0, v1, v0
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v0, 0xffff, v0
+; GFX11-FAKE16-NEXT:    s_setpc_b64 s[30:31]
   %res = shl i16 %x, %y
   %zext = zext i16 %res to i32
   ret i32 %zext
@@ -340,12 +415,19 @@ define i32 @lshr_i16_zext_i32(i16 %x, i16 %y) {
 ; GFX10-NEXT:    v_and_b32_e32 v0, 0xffff, v0
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
 ;
-; GFX11-LABEL: lshr_i16_zext_i32:
-; GFX11:       ; %bb.0:
-; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT:    v_lshrrev_b16 v0, v1, v0
-; GFX11-NEXT:    v_and_b32_e32 v0, 0xffff, v0
-; GFX11-NEXT:    s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: lshr_i16_zext_i32:
+; GFX11-TRUE16:       ; %bb.0:
+; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT:    v_lshrrev_b16 v0.l, v1.l, v0.l
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xffff, v0
+; GFX11-TRUE16-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: lshr_i16_zext_i32:
+; GFX11-FAKE16:       ; %bb.0:
+; GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT:    v_lshrrev_b16 v0, v1, v0
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v0, 0xffff, v0
+; GFX11-FAKE16-NEXT:    s_setpc_b64 s[30:31]
   %res = lshr i16 %x, %y
   %zext = zext i16 %res to i32
   ret i32 %zext
@@ -371,12 +453,19 @@ define i32 @ashr_i16_zext_i32(i16 %x, i16 %y) {
 ; GFX10-NEXT:    v_and_b32_e32 v0, 0xffff, v0
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
 ;
-; GFX11-LABEL: ashr_i16_zext_i32:
-; GFX11:       ; %bb.0:
-; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT:    v_ashrrev_i16 v0, v1, v0
-; GFX11-NEXT:    v_and_b32_e32 v0, 0xffff, v0
-; GFX11-NEXT:    s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: ashr_i16_zext_i32:
+; GFX11-TRUE16:       ; %bb.0:
+; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT:    v_ashrrev_i16 v0.l, v1.l, v0.l
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xffff, v0
+; GFX11-TRUE16-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: ashr_i16_zext_i32:
+; GFX11-FAKE16:       ; %bb.0:
+; GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT:    v_ashrrev_i16 v0, v1, v0
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v0, 0xffff, v0
+; GFX11-FAKE16-NEXT:    s_setpc_b64 s[30:31]
   %res = ashr i16 %x, %y
   %zext = zext i16 %res to i32
   ret i32 %zext
@@ -402,12 +491,20 @@ define i32 @add_u16_zext_i32(i16 %x, i16 %y) {
 ; GFX10-NEXT:    v_and_b32_e32 v0, 0xffff, v0
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
 ;
-; GFX11-LABEL: add_u16_zext_i32:
-; GFX11:       ; %bb.0:
-; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT:    v_add_nc_u16 v0, v0, v1
-; GFX11-NEXT:    v_and_b32_e32 v0, 0xffff, v0
-; GFX11-NEXT:    s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: add_u16_zext_i32:
+; GFX11-TRUE16:       ; %bb.0:
+; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v0.h, v1.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v0.l, v0.l, v0.h
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xffff, v0
+; GFX11-TRUE16-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: add_u16_zext_i32:
+; GFX11-FAKE16:       ; %bb.0:
+; GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT:    v_add_nc_u16 v0, v0, v1
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v0, 0xffff, v0
+; GFX11-FAKE16-NEXT:    s_setpc_b64 s[30:31]
   %res = add i16 %x, %y
   %zext = zext i16 %res to i32
   ret i32 %zext
@@ -433,12 +530,20 @@ define i32 @sub_u16_zext_i32(i16 %x, i16 %y) {
 ; GFX10-NEXT:    v_and_b32_e32 v0, 0xffff, v0
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
 ;
-; GFX11-LABEL: sub_u16_zext_i32:
-; GFX11:       ; %bb.0:
-; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT:    v_sub_nc_u16 v0, v0, v1
-; GFX11-NEXT:    v_and_b32_e32 v0, 0xffff, v0
-; GFX11-NEXT:    s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: sub_u16_zext_i32:
+; GFX11-TRUE16:       ; %bb.0:
+; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v0.h, v1.l
+; GFX11-TRUE16-NEXT:    v_sub_nc_u16 v0.l, v0.l, v0.h
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xffff, v0
+; GFX11-TRUE16-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: sub_u16_zext_i32:
+; GFX11-FAKE16:       ; %bb.0:
+; GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT:    v_sub_nc_u16 v0, v0, v1
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v0, 0xffff, v0
+; GFX11-FAKE16-NEXT:    s_setpc_b64 s[30:31]
   %res = sub i16 %x, %y
   %zext = zext i16 %res to i32
   ret i32 %zext
@@ -464,12 +569,20 @@ define i32 @mul_lo_u16_zext_i32(i16 %x, i16 %y) {
 ; GFX10-NEXT:    v_and_b32_e32 v0, 0xffff, v0
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
 ;
-; GFX11-LABEL: mul_lo_u16_zext_i32:
-; GFX11:       ; %bb.0:
-; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT:    v_mul_lo_u16 v0, v0, v1
-; GFX11-NEXT:    v_and_b32_e32 v0, 0xffff, v0
-; GFX11-NEXT:    s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: mul_lo_u16_zext_i32:
+; GFX11-TRUE16:       ; %bb.0:
+; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v0.h, v1.l
+; GFX11-TRUE16-NEXT:    v_mul_lo_u16 v0.l, v0.l, v0.h
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xffff, v0
+; GFX11-TRUE16-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: mul_lo_u16_zext_i32:
+; GFX11-FAKE16:       ; %bb.0:
+; GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT:    v_mul_lo_u16 v0, v0, v1
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v0, 0xffff, v0
+; GFX11-FAKE16-NEXT:    s_setpc_b64 s[30:31]
   %res = mul i16 %x, %y
   %zext = zext i16 %res to i32
   ret i32 %zext
@@ -495,12 +608,20 @@ define i32 @min_u16_zext_i32(i16 %x, i16 %y) {
 ; GFX10-NEXT:    v_and_b32_e32 v0, 0xffff, v0
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
 ;
-; GFX11-LABEL: min_u16_zext_i32:
-; GFX11:       ; %bb.0:
-; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT:    v_min_u16 v0, v0, v1
-; GFX11-NEXT:    v_and_b32_e32 v0, 0xffff, v0
-; GFX11-NEXT:    s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: min_u16_zext_i32:
+; GFX11-TRUE16:       ; %bb.0:
+; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v0.h, v1.l
+; GFX11-TRUE16-NEXT:    v_min_u16 v0.l, v0.l, v0.h
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xffff, v0
+; GFX11-TRUE16-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: min_u16_zext_i32:
+; GFX11-FAKE16:       ; %bb.0:
+; GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT:    v_min_u16 v0, v0, v1
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v0, 0xffff, v0
+; GFX11-FAKE16-NEXT:    s_setpc_b64 s[30:31]
   %cmp = icmp ule i16 %x, %y
   %res = select i1 %cmp, i16 %x, i16 %y
   %zext = zext i16 %res to i32
@@ -527,12 +648,20 @@ define i32 @min_i16_zext_i32(i16 %x, i16 %y) {
 ; GFX10-NEXT:    v_and_b32_e32 v0, 0xffff, v0
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
 ;
-; GFX11-LABEL: min_i16_zext_i32:
-; GFX11:       ; %bb.0:
-; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT:    v_min_i16 v0, v0, v1
-; GFX11-NEXT:    v_and_b32_e32 v0, 0xffff, v0
-; GFX11-NEXT:    s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: min_i16_zext_i32:
+; GFX11-TRUE16:       ; %bb.0:
+; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v0.h, v1.l
+; GFX11-TRUE16-NEXT:    v_min_i16 v0.l, v0.l, v0.h
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xffff, v0
+; GFX11-TRUE16-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: min_i16_zext_i32:
+; GFX11-FAKE16:       ; %bb.0:
+; GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT:    v_min_i16 v0, v0, v1
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v0, 0xffff, v0
+; GFX11-FAKE16-NEXT:    s_setpc_b64 s[30:31]
   %cmp = icmp sle i16 %x, %y
   %res = select i1 %cmp, i16 %x, i16 %y
   %zext = zext i16 %res to i32
@@ -559,12 +688,20 @@ define i32 @max_u16_zext_i32(i16 %x, i16 %y) {
 ; GFX10-NEXT:    v_and_b32_e32 v0, 0xffff, v0
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
 ;
-; GFX11-LABEL: max_u16_zext_i32:
-; GFX11:       ; %bb.0:
-; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT:    v_max_u16 v0, v0, v1
-; GFX11-NEXT:    v_and_b32_e32 v0, 0xffff, v0
-; GFX11-NEXT:    s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: max_u16_zext_i32:
+; GFX11-TRUE16:       ; %bb.0:
+; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v0.h, v1.l
+; GFX11-TRUE16-NEXT:    v_max_u16 v0.l, v0.l, v0.h
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xffff, v0
+; GFX11-TRUE16-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: max_u16_zext_i32:
+; GFX11-FAKE16:       ; %bb.0:
+; GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT:    v_max_u16 v0, v0, v1
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v0, 0xffff, v0
+; GFX11-FAKE16-NEXT:    s_setpc_b64 s[30:31]
   %cmp = icmp uge i16 %x, %y
   %res = select i1 %cmp, i16 %x, i16 %y
   %zext = zext i16 %res to i32
@@ -591,12 +728,20 @@ define i32 @max_i16_zext_i32(i16 %x, i16 %y) {
 ; GFX10-NEXT:    v_and_b32_e32 v0, 0xffff, v0
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
 ;
-; GFX11-LABEL: max_i16_zext_i32:
-; GFX11:       ; %bb.0:
-; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT:    v_max_i16 v0, v0, v1
-; GFX11-NEXT:    v_and_b32_e32 v0, 0xffff, v0
-; GFX11-NEXT:    s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: max_i16_zext_i32:
+; GFX11-TRUE16:       ; %bb.0:
+; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v0.h, v1.l
+; GFX11-TRUE16-NEXT:    v_max_i16 v0.l, v0.l, v0.h
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xffff, v0
+; GFX11-TRUE16-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: max_i16_zext_i32:
+; GFX11-FAKE16:       ; %bb.0:
+; GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT:    v_max_i16 v0, v0, v1
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v0, 0xffff, v0
+; GFX11-FAKE16-NEXT:    s_setpc_b64 s[30:31]
   %cmp = icmp sge i16 %x, %y
   %res = select i1 %cmp, i16 %x, i16 %y
   %zext = zext i16 %res to i32
@@ -623,12 +768,20 @@ define i32 @zext_fadd_f16(half %x, half %y) {
 ; GFX10-NEXT:    v_and_b32_e32 v0, 0xffff, v0
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
 ;
-; GFX11-LABEL: zext_fadd_f16:
-; GFX11:       ; %bb.0:
-; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT:    v_add_f16_e32 v0, v0, v1
-; GFX11-NEXT:    v_and_b32_e32 v0, 0xffff, v0
-; GFX11-NEXT:    s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: zext_fadd_f16:
+; GFX11-TRUE16:       ; %bb.0:
+; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v0.h, v1.l
+; GFX11-TRUE16-NEXT:    v_add_f16_e32 v0.l, v0.l, v0.h
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xffff, v0
+; GFX11-TRUE16-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: zext_fadd_f16:
+; GFX11-FAKE16:       ; %bb.0:
+; GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT:    v_add_f16_e32 v0, v0, v1
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v0, 0xffff, v0
+; GFX11-FAKE16-NEXT:    s_setpc_b64 s[30:31]
   %add = fadd half %x, %y
   %cast = bitcast half %add to i16
   %zext = zext i16 %cast to i32
@@ -656,12 +809,21 @@ define i32 @zext_fma_f16(half %x, half %y, half %z) {
 ; GFX10-NEXT:    v_and_b32_e32 v0, 0xffff, v2
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
 ;
-; GFX11-LABEL: zext_fma_f16:
-; GFX11:       ; %bb.0:
-; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT:    v_fmac_f16_e32 v2, v0, v1
-; GFX11-NEXT:    v_and_b32_e32 v0, 0xffff, v2
-; GFX11-NEXT:    s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: zext_fma_f16:
+; GFX11-TRUE16:       ; %bb.0:
+; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v0.h, v1.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v1.l, v2.l
+; GFX11-TRUE16-NEXT:    v_fma_f16 v0.l, v0.l, v0.h, v1.l
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xffff, v0
+; GFX11-TRUE16-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: zext_fma_f16:
+; GFX11-FAKE16:       ; %bb.0:
+; GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT:    v_fmac_f16_e32 v2, v0, v1
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v0, 0xffff, v2
+; GFX11-FAKE16-NEXT:    s_setpc_b64 s[30:31]
   %fma = call half @llvm.fma.f16(half %x, half %y, half %z)
   %cast = bitcast half %fma to i16
   %zext = zext i16 %cast to i32
@@ -689,12 +851,21 @@ define i32 @zext_div_fixup_f16(half %x, half %y, half %z) {
 ; GFX10-NEXT:    v_and_b32_e32 v0, 0xffff, v0
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
 ;
-; GFX11-LABEL: zext_div_fixup_f16:
-; GFX11:       ; %bb.0:
-; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT:    v_div_fixup_f16 v0, v0, v1, v2
-; GFX11-NEXT:    v_and_b32_e32 v0, 0xffff, v0
-; GFX11-NEXT:    s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: zext_div_fixup_f16:
+; GFX11-TRUE16:       ; %bb.0:
+; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v0.h, v1.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v1.l, v2.l
+; GFX11-TRUE16-NEXT:    v_div_fixup_f16 v0.l, v0.l, v0.h, v1.l
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xffff, v0
+; GFX11-TRUE16-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: zext_div_fixup_f16:
+; GFX11-FAKE16:       ; %bb.0:
+; GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT:    v_div_fixup_f16 v0, v0, v1, v2
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v0, 0xffff, v0
+; GFX11-FAKE16-NEXT:    s_setpc_b64 s[30:31]
   %div.fixup = call half @llvm.amdgcn.div.fixup.f16(half %x, half %y, half %z)
   %cast = bitcast half %div.fixup to i16
   %zext = zext i16 %cast to i32
@@ -724,12 +895,19 @@ define i32 @zext_fptrunc_f16(float %x) {
 ; GFX10-NEXT:    v_and_b32_e32 v0, 0xffff, v0
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
 ;
-; GFX11-LABEL: zext_fptrunc_f16:
-; GFX11:       ; %bb.0:
-; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT:    v_cvt_f16_f32_e32 v0, v0
-; GFX11-NEXT:    v_and_b32_e32 v0, 0xffff, v0
-; GFX11-NEXT:    s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: zext_fptrunc_f16:
+; GFX11-TRUE16:       ; %bb.0:
+; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT:    v_cvt_f16_f32_e32 v0.l, v0
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xffff, v0
+; GFX11-TRUE16-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: zext_fptrunc_f16:
+; GFX11-FAKE16:       ; %bb.0:
+; GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT:    v_cvt_f16_f32_e32 v0, v0
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v0, 0xffff, v0
+; GFX11-FAKE16-NEXT:    s_setpc_b64 s[30:31]
   %fptrunc = fptrunc float %x to half
   %cast = bitcast half %fptrunc to i16
   %zext = zext i16 %cast to i32
diff --git a/llvm/test/CodeGen/AMDGPU/print-pipeline-passes.ll b/llvm/test/CodeGen/AMDGPU/print-pipeline-passes.ll
index c68143f44866f..b1fc76f457ece 100644
--- a/llvm/test/CodeGen/AMDGPU/print-pipeline-passes.ll
+++ b/llvm/test/CodeGen/AMDGPU/print-pipeline-passes.ll
@@ -14,6 +14,7 @@
 
 ; PRE-NOT: internalize
 ; PRE-NOT: amdgpu-attributor
+; PRE-NOT: printfToRuntime
 
 define amdgpu_kernel void @kernel() {
 entry:
diff --git a/llvm/test/CodeGen/AMDGPU/promote-alloca-globals.ll b/llvm/test/CodeGen/AMDGPU/promote-alloca-globals.ll
index 01b65c67da024..86036fbf672b2 100644
--- a/llvm/test/CodeGen/AMDGPU/promote-alloca-globals.ll
+++ b/llvm/test/CodeGen/AMDGPU/promote-alloca-globals.ll
@@ -5,13 +5,13 @@
 @global_array0 = internal unnamed_addr addrspace(3) global [750 x [10 x i32]] undef, align 4
 @global_array1 = internal unnamed_addr addrspace(3) global [750 x [10 x i32]] undef, align 4
 
-; IR-LABEL: define amdgpu_kernel void @promote_alloca_size_256(ptr addrspace(1) nocapture %out, ptr addrspace(1) nocapture %in) {
+; IR-LABEL: define amdgpu_kernel void @promote_alloca_size_256(ptr addrspace(1) captures(none) %out, ptr addrspace(1) captures(none) %in) {
 ; IR-NOT: alloca [10 x i32]
 ; ASM-LABEL: {{^}}promote_alloca_size_256:
 ; ASM: .amdgpu_lds llvm.amdgcn.kernel.promote_alloca_size_256.lds, 60000, 16
 ; ASM-NOT: .amdgpu_lds
 
-define amdgpu_kernel void @promote_alloca_size_256(ptr addrspace(1) nocapture %out, ptr addrspace(1) nocapture %in) {
+define amdgpu_kernel void @promote_alloca_size_256(ptr addrspace(1) captures(none) %out, ptr addrspace(1) captures(none) %in) {
 entry:
   %stack = alloca [10 x i32], align 4, addrspace(5)
   %tmp = load i32, ptr addrspace(1) %in, align 4
diff --git a/llvm/test/CodeGen/AMDGPU/promote-alloca-invariant-markers.ll b/llvm/test/CodeGen/AMDGPU/promote-alloca-invariant-markers.ll
index a585901fc377c..e100e7a41472c 100644
--- a/llvm/test/CodeGen/AMDGPU/promote-alloca-invariant-markers.ll
+++ b/llvm/test/CodeGen/AMDGPU/promote-alloca-invariant-markers.ll
@@ -1,25 +1,75 @@
-; RUN:  llc -amdgpu-scalarize-global-loads=false  -mtriple=amdgcn-- -mattr=+promote-alloca -verify-machineinstrs < %s | FileCheck -check-prefix=GCN %s
-target datalayout = "A5"
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -S -mtriple=amdgcn-amd-amdhsa -passes=amdgpu-promote-alloca %s -o - | FileCheck %s
 
-declare ptr @llvm.invariant.start.p5(i64, ptr addrspace(5) nocapture) #0
-declare void @llvm.invariant.end.p5(ptr, i64, ptr addrspace(5) nocapture) #0
-declare ptr addrspace(5) @llvm.launder.invariant.group.p5(ptr addrspace(5)) #1
+declare ptr @llvm.invariant.start.p5(i64, ptr addrspace(5) nocapture)
+declare void @llvm.invariant.end.p5(ptr, i64, ptr addrspace(5) nocapture)
+declare ptr addrspace(5) @llvm.launder.invariant.group.p5(ptr addrspace(5))
+declare ptr addrspace(5) @llvm.strip.invariant.group.p5(ptr addrspace(5))
 
-; GCN-LABEL: {{^}}use_invariant_promotable_lds:
-; GCN: buffer_load_dword
-; GCN: ds_write_b32
-define amdgpu_kernel void @use_invariant_promotable_lds(ptr addrspace(1) %arg) #2 {
+define amdgpu_kernel void @use_invariant_start_and_end() {
+; CHECK-LABEL: define amdgpu_kernel void @use_invariant_start_and_end() {
+; CHECK-NEXT:  [[BB:.*:]]
+; CHECK-NEXT:    [[TMP0:%.*]] = call noalias nonnull dereferenceable(64) ptr addrspace(4) @llvm.amdgcn.dispatch.ptr()
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i32, ptr addrspace(4) [[TMP0]], i64 1
+; CHECK-NEXT:    [[TMP2:%.*]] = load i32, ptr addrspace(4) [[TMP1]], align 4, !invariant.load [[META0:![0-9]+]]
+; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i32, ptr addrspace(4) [[TMP0]], i64 2
+; CHECK-NEXT:    [[TMP4:%.*]] = load i32, ptr addrspace(4) [[TMP3]], align 4, !range [[RNG1:![0-9]+]], !invariant.load [[META0]]
+; CHECK-NEXT:    [[TMP5:%.*]] = lshr i32 [[TMP2]], 16
+; CHECK-NEXT:    [[TMP6:%.*]] = call range(i32 0, 1024) i32 @llvm.amdgcn.workitem.id.x()
+; CHECK-NEXT:    [[TMP7:%.*]] = call range(i32 0, 1024) i32 @llvm.amdgcn.workitem.id.y()
+; CHECK-NEXT:    [[TMP8:%.*]] = call range(i32 0, 1024) i32 @llvm.amdgcn.workitem.id.z()
+; CHECK-NEXT:    [[TMP9:%.*]] = mul nuw nsw i32 [[TMP5]], [[TMP4]]
+; CHECK-NEXT:    [[TMP10:%.*]] = mul i32 [[TMP9]], [[TMP6]]
+; CHECK-NEXT:    [[TMP11:%.*]] = mul nuw nsw i32 [[TMP7]], [[TMP4]]
+; CHECK-NEXT:    [[TMP12:%.*]] = add i32 [[TMP10]], [[TMP11]]
+; CHECK-NEXT:    [[TMP13:%.*]] = add i32 [[TMP12]], [[TMP8]]
+; CHECK-NEXT:    [[TMP14:%.*]] = getelementptr inbounds [1024 x i32], ptr addrspace(3) @use_invariant_start_and_end.alloca, i32 0, i32 [[TMP13]]
+; CHECK-NEXT:    [[INVARIANT1:%.*]] = call ptr @llvm.invariant.start.p3(i64 0, ptr addrspace(3) [[TMP14]])
+; CHECK-NEXT:    store <2 x i1> zeroinitializer, ptr [[INVARIANT1]], align 1
+; CHECK-NEXT:    call void @llvm.invariant.end.p3(ptr [[INVARIANT1]], i64 0, ptr addrspace(3) [[TMP14]])
+; CHECK-NEXT:    ret void
+;
 bb:
-  %tmp = alloca i32, align 4, addrspace(5)
-  %tmp2 = getelementptr inbounds i32, ptr addrspace(1) %arg, i64 1
-  %tmp3 = load i32, ptr addrspace(1) %tmp2
-  store i32 %tmp3, ptr addrspace(5) %tmp
-  %tmp4 = call ptr @llvm.invariant.start.p5(i64 4, ptr addrspace(5) %tmp) #0
-  call void @llvm.invariant.end.p5(ptr %tmp4, i64 4, ptr addrspace(5) %tmp) #0
-  %tmp5 = call ptr addrspace(5) @llvm.launder.invariant.group.p5(ptr addrspace(5) %tmp) #1
+  %alloca = alloca i32, align 4, addrspace(5)
+  %invariant = call ptr @llvm.invariant.start.p5(i64 0, ptr addrspace(5) %alloca)
+  store <2 x i1> zeroinitializer, ptr %invariant, align 1
+  call void @llvm.invariant.end.p5(ptr %invariant, i64 0, ptr addrspace(5) %alloca)
   ret void
 }
 
-attributes #0 = { argmemonly nounwind }
-attributes #1 = { nounwind readnone }
-attributes #2 = { nounwind }
+define amdgpu_kernel void @use_invariant_group_and_strip() {
+; CHECK-LABEL: define amdgpu_kernel void @use_invariant_group_and_strip() {
+; CHECK-NEXT:  [[BB:.*:]]
+; CHECK-NEXT:    [[TMP0:%.*]] = call noalias nonnull dereferenceable(64) ptr addrspace(4) @llvm.amdgcn.dispatch.ptr()
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i32, ptr addrspace(4) [[TMP0]], i64 1
+; CHECK-NEXT:    [[TMP2:%.*]] = load i32, ptr addrspace(4) [[TMP1]], align 4, !invariant.load [[META0]]
+; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i32, ptr addrspace(4) [[TMP0]], i64 2
+; CHECK-NEXT:    [[TMP4:%.*]] = load i32, ptr addrspace(4) [[TMP3]], align 4, !range [[RNG1]], !invariant.load [[META0]]
+; CHECK-NEXT:    [[TMP5:%.*]] = lshr i32 [[TMP2]], 16
+; CHECK-NEXT:    [[TMP6:%.*]] = call range(i32 0, 1024) i32 @llvm.amdgcn.workitem.id.x()
+; CHECK-NEXT:    [[TMP7:%.*]] = call range(i32 0, 1024) i32 @llvm.amdgcn.workitem.id.y()
+; CHECK-NEXT:    [[TMP8:%.*]] = call range(i32 0, 1024) i32 @llvm.amdgcn.workitem.id.z()
+; CHECK-NEXT:    [[TMP9:%.*]] = mul nuw nsw i32 [[TMP5]], [[TMP4]]
+; CHECK-NEXT:    [[TMP10:%.*]] = mul i32 [[TMP9]], [[TMP6]]
+; CHECK-NEXT:    [[TMP11:%.*]] = mul nuw nsw i32 [[TMP7]], [[TMP4]]
+; CHECK-NEXT:    [[TMP12:%.*]] = add i32 [[TMP10]], [[TMP11]]
+; CHECK-NEXT:    [[TMP13:%.*]] = add i32 [[TMP12]], [[TMP8]]
+; CHECK-NEXT:    [[TMP14:%.*]] = getelementptr inbounds [1024 x i32], ptr addrspace(3) @use_invariant_group_and_strip.alloca, i32 0, i32 [[TMP13]]
+; CHECK-NEXT:    [[INVARIANT2:%.*]] = call ptr addrspace(3) @llvm.launder.invariant.group.p3(ptr addrspace(3) [[TMP14]])
+; CHECK-NEXT:    store <2 x i1> zeroinitializer, ptr addrspace(3) [[INVARIANT2]], align 1
+; CHECK-NEXT:    [[STRIP1:%.*]] = call ptr addrspace(3) @llvm.strip.invariant.group.p3(ptr addrspace(3) [[TMP14]])
+; CHECK-NEXT:    store <2 x i1> zeroinitializer, ptr addrspace(3) [[STRIP1]], align 1
+; CHECK-NEXT:    ret void
+;
+bb:
+  %alloca = alloca i32, align 4, addrspace(5)
+  %invariant = call ptr addrspace(5) @llvm.launder.invariant.group.p5(ptr addrspace(5) %alloca)
+  store <2 x i1> zeroinitializer, ptr addrspace(5) %invariant, align 1
+  %strip = call ptr addrspace(5) @llvm.strip.invariant.group.p5(ptr addrspace(5) %alloca)
+  store <2 x i1> zeroinitializer, ptr addrspace(5) %strip, align 1
+  ret void
+}
+;.
+; CHECK: [[META0]] = !{}
+; CHECK: [[RNG1]] = !{i32 0, i32 1025}
+;.
diff --git a/llvm/test/CodeGen/AMDGPU/pseudo-scalar-transcendental.ll b/llvm/test/CodeGen/AMDGPU/pseudo-scalar-transcendental.ll
index a439f8df10a26..1e6106896e0a4 100644
--- a/llvm/test/CodeGen/AMDGPU/pseudo-scalar-transcendental.ll
+++ b/llvm/test/CodeGen/AMDGPU/pseudo-scalar-transcendental.ll
@@ -11,6 +11,7 @@ define amdgpu_cs float @v_s_exp_f32(float inreg %src) {
 ; GFX12-NEXT:    s_add_f32 s0, s0, s1
 ; GFX12-NEXT:    s_cselect_b32 s1, 0xffffffc0, 0
 ; GFX12-NEXT:    v_s_exp_f32 s0, s0
+; GFX12-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-NEXT:    s_delay_alu instid0(TRANS32_DEP_1)
 ; GFX12-NEXT:    v_ldexp_f32 v0, s0, s1
 ; GFX12-NEXT:    ; return to shader part epilog
@@ -22,6 +23,7 @@ define amdgpu_cs half @v_s_exp_f16(half inreg %src) {
 ; GFX12-LABEL: v_s_exp_f16:
 ; GFX12:       ; %bb.0:
 ; GFX12-NEXT:    v_s_exp_f16 s0, s0
+; GFX12-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-NEXT:    s_delay_alu instid0(TRANS32_DEP_1)
 ; GFX12-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX12-NEXT:    ; return to shader part epilog
@@ -33,6 +35,7 @@ define amdgpu_cs float @v_s_amdgcn_exp_f32(float inreg %src) {
 ; GFX12-LABEL: v_s_amdgcn_exp_f32:
 ; GFX12:       ; %bb.0:
 ; GFX12-NEXT:    v_s_exp_f32 s0, s0
+; GFX12-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-NEXT:    s_delay_alu instid0(TRANS32_DEP_1)
 ; GFX12-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX12-NEXT:    ; return to shader part epilog
@@ -44,6 +47,7 @@ define amdgpu_cs half @v_s_amdgcn_exp_f16(half inreg %src) {
 ; GFX12-LABEL: v_s_amdgcn_exp_f16:
 ; GFX12:       ; %bb.0:
 ; GFX12-NEXT:    v_s_exp_f16 s0, s0
+; GFX12-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-NEXT:    s_delay_alu instid0(TRANS32_DEP_1)
 ; GFX12-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX12-NEXT:    ; return to shader part epilog
@@ -59,9 +63,8 @@ define amdgpu_cs float @v_s_log_f32(float inreg %src) {
 ; GFX12-SDAG-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-SDAG-NEXT:    v_cndmask_b32_e64 v0, 0, 1, s1
 ; GFX12-SDAG-NEXT:    v_lshlrev_b32_e32 v0, 5, v0
-; GFX12-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX12-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
 ; GFX12-SDAG-NEXT:    v_ldexp_f32 v0, s0, v0
-; GFX12-SDAG-NEXT:    s_wait_alu 0xfffe
 ; GFX12-SDAG-NEXT:    s_and_b32 s0, s1, exec_lo
 ; GFX12-SDAG-NEXT:    s_cselect_b32 s0, 0x42000000, 0
 ; GFX12-SDAG-NEXT:    v_log_f32_e32 v0, v0
@@ -92,6 +95,7 @@ define amdgpu_cs half @v_s_log_f16(half inreg %src) {
 ; GFX12-LABEL: v_s_log_f16:
 ; GFX12:       ; %bb.0:
 ; GFX12-NEXT:    v_s_log_f16 s0, s0
+; GFX12-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-NEXT:    s_delay_alu instid0(TRANS32_DEP_1)
 ; GFX12-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX12-NEXT:    ; return to shader part epilog
@@ -103,6 +107,7 @@ define amdgpu_cs float @v_s_amdgcn_log_f32(float inreg %src) {
 ; GFX12-LABEL: v_s_amdgcn_log_f32:
 ; GFX12:       ; %bb.0:
 ; GFX12-NEXT:    v_s_log_f32 s0, s0
+; GFX12-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-NEXT:    s_delay_alu instid0(TRANS32_DEP_1)
 ; GFX12-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX12-NEXT:    ; return to shader part epilog
@@ -114,6 +119,7 @@ define amdgpu_cs half @v_s_amdgcn_log_f16(half inreg %src) {
 ; GFX12-LABEL: v_s_amdgcn_log_f16:
 ; GFX12:       ; %bb.0:
 ; GFX12-NEXT:    v_s_log_f16 s0, s0
+; GFX12-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-NEXT:    s_delay_alu instid0(TRANS32_DEP_1)
 ; GFX12-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX12-NEXT:    ; return to shader part epilog
@@ -125,6 +131,7 @@ define amdgpu_cs float @v_s_rcp_f32(float inreg %src) {
 ; GFX12-LABEL: v_s_rcp_f32:
 ; GFX12:       ; %bb.0:
 ; GFX12-NEXT:    v_s_rcp_f32 s0, s0
+; GFX12-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-NEXT:    s_delay_alu instid0(TRANS32_DEP_1)
 ; GFX12-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX12-NEXT:    ; return to shader part epilog
@@ -136,6 +143,7 @@ define amdgpu_cs half @v_s_rcp_f16(half inreg %src) {
 ; GFX12-LABEL: v_s_rcp_f16:
 ; GFX12:       ; %bb.0:
 ; GFX12-NEXT:    v_s_rcp_f16 s0, s0
+; GFX12-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-NEXT:    s_delay_alu instid0(TRANS32_DEP_1)
 ; GFX12-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX12-NEXT:    ; return to shader part epilog
@@ -148,6 +156,7 @@ define amdgpu_cs float @v_s_rsq_f32(float inreg %src) {
 ; GFX12-SDAG-LABEL: v_s_rsq_f32:
 ; GFX12-SDAG:       ; %bb.0:
 ; GFX12-SDAG-NEXT:    v_s_rsq_f32 s0, s0
+; GFX12-SDAG-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-SDAG-NEXT:    s_delay_alu instid0(TRANS32_DEP_1)
 ; GFX12-SDAG-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX12-SDAG-NEXT:    ; return to shader part epilog
@@ -155,8 +164,10 @@ define amdgpu_cs float @v_s_rsq_f32(float inreg %src) {
 ; GFX12-GISEL-LABEL: v_s_rsq_f32:
 ; GFX12-GISEL:       ; %bb.0:
 ; GFX12-GISEL-NEXT:    v_s_sqrt_f32 s0, s0
-; GFX12-GISEL-NEXT:    s_delay_alu instid0(TRANS32_DEP_1) | instskip(NEXT) | instid1(TRANS32_DEP_1)
+; GFX12-GISEL-NEXT:    s_wait_alu 0xf1ff
+; GFX12-GISEL-NEXT:    s_delay_alu instid0(TRANS32_DEP_1) | instskip(SKIP_1) | instid1(TRANS32_DEP_1)
 ; GFX12-GISEL-NEXT:    v_s_rcp_f32 s0, s0
+; GFX12-GISEL-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-GISEL-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX12-GISEL-NEXT:    ; return to shader part epilog
   %sqrt = call fast float @llvm.sqrt.f32(float %src)
@@ -168,6 +179,7 @@ define amdgpu_cs half @v_s_rsq_f16(half inreg %src) {
 ; GFX12-LABEL: v_s_rsq_f16:
 ; GFX12:       ; %bb.0:
 ; GFX12-NEXT:    v_s_rsq_f16 s0, s0
+; GFX12-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-NEXT:    s_delay_alu instid0(TRANS32_DEP_1)
 ; GFX12-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX12-NEXT:    ; return to shader part epilog
@@ -185,7 +197,6 @@ define amdgpu_cs float @v_s_sqrt_f32(float inreg %src) {
 ; GFX12-SDAG-NEXT:    s_delay_alu instid0(SALU_CYCLE_2) | instskip(NEXT) | instid1(SALU_CYCLE_1)
 ; GFX12-SDAG-NEXT:    s_cselect_b32 s1, s1, s0
 ; GFX12-SDAG-NEXT:    v_s_sqrt_f32 s2, s1
-; GFX12-SDAG-NEXT:    s_wait_alu 0xfffe
 ; GFX12-SDAG-NEXT:    s_mov_b32 s4, s1
 ; GFX12-SDAG-NEXT:    s_delay_alu instid0(TRANS32_DEP_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
 ; GFX12-SDAG-NEXT:    s_add_co_i32 s3, s2, -1
@@ -223,7 +234,6 @@ define amdgpu_cs float @v_s_sqrt_f32(float inreg %src) {
 ; GFX12-GISEL-NEXT:    s_delay_alu instid0(SALU_CYCLE_2) | instskip(NEXT) | instid1(SALU_CYCLE_1)
 ; GFX12-GISEL-NEXT:    s_cselect_b32 s0, s2, s0
 ; GFX12-GISEL-NEXT:    v_s_sqrt_f32 s2, s0
-; GFX12-GISEL-NEXT:    s_wait_alu 0xfffe
 ; GFX12-GISEL-NEXT:    s_mov_b32 s4, s0
 ; GFX12-GISEL-NEXT:    s_mov_b32 s6, s0
 ; GFX12-GISEL-NEXT:    s_delay_alu instid0(TRANS32_DEP_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
@@ -241,11 +251,12 @@ define amdgpu_cs float @v_s_sqrt_f32(float inreg %src) {
 ; GFX12-GISEL-NEXT:    s_cselect_b32 s2, s5, s2
 ; GFX12-GISEL-NEXT:    s_cmp_lg_u32 s1, 0
 ; GFX12-GISEL-NEXT:    s_mul_f32 s3, s2, 0x37800000
-; GFX12-GISEL-NEXT:    s_delay_alu instid0(SALU_CYCLE_3) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX12-GISEL-NEXT:    s_delay_alu instid0(SALU_CYCLE_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
 ; GFX12-GISEL-NEXT:    s_cselect_b32 s1, s3, s2
 ; GFX12-GISEL-NEXT:    s_wait_alu 0xfffe
 ; GFX12-GISEL-NEXT:    v_mov_b32_e32 v0, s1
 ; GFX12-GISEL-NEXT:    v_cmp_class_f32_e64 s1, s0, 0x260
+; GFX12-GISEL-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-GISEL-NEXT:    v_cndmask_b32_e64 v0, v0, s0, s1
 ; GFX12-GISEL-NEXT:    ; return to shader part epilog
   %result = call float @llvm.sqrt.f32(float %src)
@@ -256,6 +267,7 @@ define amdgpu_cs half @v_s_sqrt_f16(half inreg %src) {
 ; GFX12-LABEL: v_s_sqrt_f16:
 ; GFX12:       ; %bb.0:
 ; GFX12-NEXT:    v_s_sqrt_f16 s0, s0
+; GFX12-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-NEXT:    s_delay_alu instid0(TRANS32_DEP_1)
 ; GFX12-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX12-NEXT:    ; return to shader part epilog
@@ -267,6 +279,7 @@ define amdgpu_cs float @v_amdgcn_sqrt_f32(float inreg %src)  {
 ; GFX12-LABEL: v_amdgcn_sqrt_f32:
 ; GFX12:       ; %bb.0:
 ; GFX12-NEXT:    v_s_sqrt_f32 s0, s0
+; GFX12-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-NEXT:    s_delay_alu instid0(TRANS32_DEP_1)
 ; GFX12-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX12-NEXT:    ; return to shader part epilog
@@ -278,6 +291,7 @@ define amdgpu_cs half @v_amdgcn_sqrt_f16(half inreg %src)  {
 ; GFX12-LABEL: v_amdgcn_sqrt_f16:
 ; GFX12:       ; %bb.0:
 ; GFX12-NEXT:    v_s_sqrt_f16 s0, s0
+; GFX12-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-NEXT:    s_delay_alu instid0(TRANS32_DEP_1)
 ; GFX12-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX12-NEXT:    ; return to shader part epilog
@@ -296,7 +310,6 @@ define amdgpu_cs float @srcmods_abs_f32(float inreg %src) {
 ; GFX12-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-SDAG-NEXT:    v_lshlrev_b32_e32 v0, 5, v0
 ; GFX12-SDAG-NEXT:    v_ldexp_f32 v0, |s0|, v0
-; GFX12-SDAG-NEXT:    s_wait_alu 0xfffe
 ; GFX12-SDAG-NEXT:    s_and_b32 s0, s1, exec_lo
 ; GFX12-SDAG-NEXT:    s_cselect_b32 s0, 0x42000000, 0
 ; GFX12-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(TRANS32_DEP_1)
@@ -333,9 +346,8 @@ define amdgpu_cs float @srcmods_neg_f32(float inreg %src) {
 ; GFX12-SDAG-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-SDAG-NEXT:    v_cndmask_b32_e64 v0, 0, 1, s1
 ; GFX12-SDAG-NEXT:    v_lshlrev_b32_e32 v0, 5, v0
-; GFX12-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX12-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
 ; GFX12-SDAG-NEXT:    v_ldexp_f32 v0, -s0, v0
-; GFX12-SDAG-NEXT:    s_wait_alu 0xfffe
 ; GFX12-SDAG-NEXT:    s_and_b32 s0, s1, exec_lo
 ; GFX12-SDAG-NEXT:    s_cselect_b32 s0, 0x42000000, 0
 ; GFX12-SDAG-NEXT:    v_log_f32_e32 v0, v0
@@ -368,6 +380,7 @@ define amdgpu_cs half @srcmods_abs_f16(half inreg %src) {
 ; GFX12-LABEL: srcmods_abs_f16:
 ; GFX12:       ; %bb.0:
 ; GFX12-NEXT:    v_s_log_f16 s0, |s0|
+; GFX12-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-NEXT:    s_delay_alu instid0(TRANS32_DEP_1)
 ; GFX12-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX12-NEXT:    ; return to shader part epilog
@@ -380,6 +393,7 @@ define amdgpu_cs half @srcmods_neg_f16(half inreg %src) {
 ; GFX12-LABEL: srcmods_neg_f16:
 ; GFX12:       ; %bb.0:
 ; GFX12-NEXT:    v_s_log_f16 s0, -s0
+; GFX12-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-NEXT:    s_delay_alu instid0(TRANS32_DEP_1)
 ; GFX12-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX12-NEXT:    ; return to shader part epilog
diff --git a/llvm/test/CodeGen/AMDGPU/rem_i128.ll b/llvm/test/CodeGen/AMDGPU/rem_i128.ll
index 704947523f677..afe1f33d15e42 100644
--- a/llvm/test/CodeGen/AMDGPU/rem_i128.ll
+++ b/llvm/test/CodeGen/AMDGPU/rem_i128.ll
@@ -203,30 +203,27 @@ define i128 @v_srem_i128_vv(i128 %lhs, i128 %rhs) {
 ; GFX9-NEXT:    v_or_b32_e32 v13, v8, v14
 ; GFX9-NEXT:  .LBB0_6: ; %Flow3
 ; GFX9-NEXT:    s_or_b64 exec, exec, s[8:9]
-; GFX9-NEXT:    v_mul_lo_u32 v16, v13, v5
+; GFX9-NEXT:    v_mul_lo_u32 v18, v13, v5
 ; GFX9-NEXT:    v_mad_u64_u32 v[5:6], s[4:5], v23, v13, 0
 ; GFX9-NEXT:    v_mov_b32_e32 v15, 0
-; GFX9-NEXT:    v_mad_u64_u32 v[7:8], s[4:5], v13, v4, 0
-; GFX9-NEXT:    v_mov_b32_e32 v14, v6
-; GFX9-NEXT:    v_mad_u64_u32 v[13:14], s[4:5], v22, v13, v[14:15]
 ; GFX9-NEXT:    v_mul_lo_u32 v9, v11, v4
-; GFX9-NEXT:    v_mul_lo_u32 v10, v10, v23
-; GFX9-NEXT:    v_mov_b32_e32 v4, v14
-; GFX9-NEXT:    v_mov_b32_e32 v14, v15
-; GFX9-NEXT:    v_mad_u64_u32 v[13:14], s[4:5], v23, v11, v[13:14]
-; GFX9-NEXT:    v_add3_u32 v8, v8, v16, v9
+; GFX9-NEXT:    v_mov_b32_e32 v14, v6
+; GFX9-NEXT:    v_mad_u64_u32 v[7:8], s[4:5], v13, v4, 0
+; GFX9-NEXT:    v_mad_u64_u32 v[16:17], s[4:5], v22, v13, v[14:15]
+; GFX9-NEXT:    v_add3_u32 v8, v8, v18, v9
 ; GFX9-NEXT:    v_mad_u64_u32 v[6:7], s[4:5], v12, v23, v[7:8]
-; GFX9-NEXT:    v_mov_b32_e32 v8, v14
-; GFX9-NEXT:    v_add_co_u32_e32 v8, vcc, v4, v8
-; GFX9-NEXT:    v_addc_co_u32_e64 v9, s[4:5], 0, 0, vcc
-; GFX9-NEXT:    v_mul_lo_u32 v12, v12, v22
-; GFX9-NEXT:    v_mad_u64_u32 v[8:9], s[4:5], v22, v11, v[8:9]
-; GFX9-NEXT:    v_add3_u32 v4, v10, v7, v12
-; GFX9-NEXT:    v_add_co_u32_e32 v6, vcc, v8, v6
-; GFX9-NEXT:    v_addc_co_u32_e32 v4, vcc, v9, v4, vcc
-; GFX9-NEXT:    v_mov_b32_e32 v7, v13
+; GFX9-NEXT:    v_mov_b32_e32 v14, v16
+; GFX9-NEXT:    v_mad_u64_u32 v[8:9], s[4:5], v23, v11, v[14:15]
+; GFX9-NEXT:    v_mul_lo_u32 v4, v12, v22
+; GFX9-NEXT:    v_mul_lo_u32 v12, v10, v23
+; GFX9-NEXT:    v_add_co_u32_e32 v9, vcc, v17, v9
+; GFX9-NEXT:    v_addc_co_u32_e64 v10, s[4:5], 0, 0, vcc
+; GFX9-NEXT:    v_mad_u64_u32 v[9:10], s[4:5], v22, v11, v[9:10]
+; GFX9-NEXT:    v_add3_u32 v4, v12, v7, v4
+; GFX9-NEXT:    v_add_co_u32_e32 v6, vcc, v9, v6
+; GFX9-NEXT:    v_addc_co_u32_e32 v4, vcc, v10, v4, vcc
 ; GFX9-NEXT:    v_sub_co_u32_e32 v0, vcc, v0, v5
-; GFX9-NEXT:    v_subb_co_u32_e32 v1, vcc, v1, v7, vcc
+; GFX9-NEXT:    v_subb_co_u32_e32 v1, vcc, v1, v8, vcc
 ; GFX9-NEXT:    v_subb_co_u32_e32 v2, vcc, v2, v6, vcc
 ; GFX9-NEXT:    v_subb_co_u32_e32 v3, vcc, v3, v4, vcc
 ; GFX9-NEXT:    v_xor_b32_e32 v0, v0, v20
@@ -1683,27 +1680,24 @@ define i128 @v_urem_i128_vv(i128 %lhs, i128 %rhs) {
 ; GFX9-NEXT:    v_mul_lo_u32 v19, v12, v7
 ; GFX9-NEXT:    v_mad_u64_u32 v[7:8], s[4:5], v4, v12, 0
 ; GFX9-NEXT:    v_mov_b32_e32 v17, 0
-; GFX9-NEXT:    v_mad_u64_u32 v[9:10], s[4:5], v12, v6, 0
+; GFX9-NEXT:    v_mul_lo_u32 v18, v13, v6
 ; GFX9-NEXT:    v_mov_b32_e32 v16, v8
+; GFX9-NEXT:    v_mad_u64_u32 v[9:10], s[4:5], v12, v6, 0
 ; GFX9-NEXT:    v_mad_u64_u32 v[11:12], s[4:5], v5, v12, v[16:17]
-; GFX9-NEXT:    v_mul_lo_u32 v18, v13, v6
-; GFX9-NEXT:    v_mul_lo_u32 v16, v15, v4
-; GFX9-NEXT:    v_mov_b32_e32 v6, v12
-; GFX9-NEXT:    v_mov_b32_e32 v12, v17
-; GFX9-NEXT:    v_mad_u64_u32 v[11:12], s[4:5], v4, v13, v[11:12]
 ; GFX9-NEXT:    v_add3_u32 v10, v10, v19, v18
 ; GFX9-NEXT:    v_mad_u64_u32 v[8:9], s[4:5], v14, v4, v[9:10]
-; GFX9-NEXT:    v_mov_b32_e32 v4, v12
-; GFX9-NEXT:    v_mul_lo_u32 v10, v14, v5
-; GFX9-NEXT:    v_add_co_u32_e32 v14, vcc, v6, v4
-; GFX9-NEXT:    v_addc_co_u32_e64 v15, s[4:5], 0, 0, vcc
-; GFX9-NEXT:    v_mad_u64_u32 v[4:5], s[4:5], v5, v13, v[14:15]
-; GFX9-NEXT:    v_add3_u32 v6, v16, v9, v10
+; GFX9-NEXT:    v_mov_b32_e32 v16, v11
+; GFX9-NEXT:    v_mad_u64_u32 v[10:11], s[4:5], v4, v13, v[16:17]
+; GFX9-NEXT:    v_mul_lo_u32 v6, v14, v5
+; GFX9-NEXT:    v_mul_lo_u32 v14, v15, v4
+; GFX9-NEXT:    v_add_co_u32_e32 v11, vcc, v12, v11
+; GFX9-NEXT:    v_addc_co_u32_e64 v12, s[4:5], 0, 0, vcc
+; GFX9-NEXT:    v_mad_u64_u32 v[4:5], s[4:5], v5, v13, v[11:12]
+; GFX9-NEXT:    v_add3_u32 v6, v14, v9, v6
 ; GFX9-NEXT:    v_add_co_u32_e32 v4, vcc, v4, v8
 ; GFX9-NEXT:    v_addc_co_u32_e32 v5, vcc, v5, v6, vcc
-; GFX9-NEXT:    v_mov_b32_e32 v6, v11
 ; GFX9-NEXT:    v_sub_co_u32_e32 v0, vcc, v0, v7
-; GFX9-NEXT:    v_subb_co_u32_e32 v1, vcc, v1, v6, vcc
+; GFX9-NEXT:    v_subb_co_u32_e32 v1, vcc, v1, v10, vcc
 ; GFX9-NEXT:    v_subb_co_u32_e32 v2, vcc, v2, v4, vcc
 ; GFX9-NEXT:    v_subb_co_u32_e32 v3, vcc, v3, v5, vcc
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
diff --git a/llvm/test/CodeGen/AMDGPU/remove-incompatible-s-time.ll b/llvm/test/CodeGen/AMDGPU/remove-incompatible-s-time.ll
index 676ba1480e6d2..efb8d836c7b3b 100644
--- a/llvm/test/CodeGen/AMDGPU/remove-incompatible-s-time.ll
+++ b/llvm/test/CodeGen/AMDGPU/remove-incompatible-s-time.ll
@@ -7,7 +7,6 @@
 ; RUN: llc -enable-new-pm -mtriple=amdgcn -mcpu=gfx1030 -stop-after=amdgpu-remove-incompatible-functions\
 ; RUN:   -pass-remarks=amdgpu-remove-incompatible-functions %s -o - 2>%t | FileCheck -check-prefixes=COMPATIBLE,REALTIME,MEMTIME %s
 ; RUN: FileCheck -allow-empty --check-prefixes=WARN-REALTIME,WARN-MEMTIME %s < %t
-; RUN: llc -enable-new-pm -mtriple=amdgcn -mcpu=gfx1030 -verify-machineinstrs < %s
 
 ; RUN: llc -mtriple=amdgcn -mcpu=gfx1102 -stop-after=amdgpu-remove-incompatible-functions\
 ; RUN:   -pass-remarks=amdgpu-remove-incompatible-functions %s -o - 2>%t | FileCheck -check-prefixes=INCOMPATIBLE,NOREALTIME,NOMEMTIME %s
@@ -17,7 +16,6 @@
 ; RUN: llc -enable-new-pm -mtriple=amdgcn -mcpu=gfx1102 -stop-after=amdgpu-remove-incompatible-functions\
 ; RUN:   -pass-remarks=amdgpu-remove-incompatible-functions %s -o - 2>%t | FileCheck -check-prefixes=INCOMPATIBLE,NOREALTIME,NOMEMTIME %s
 ; RUN: FileCheck --check-prefixes=WARN-NOREALTIME,WARN-NOMEMTIME %s < %t
-; RUN: llc -enable-new-pm -mtriple=amdgcn -mcpu=gfx1102 -verify-machineinstrs < %s
 
 ; Note: This test checks the IR, but also has a run line to codegen the file just to check we
 ; do not crash when trying to select those functions.
diff --git a/llvm/test/CodeGen/AMDGPU/remove-incompatible-wave32-feature.ll b/llvm/test/CodeGen/AMDGPU/remove-incompatible-wave32-feature.ll
index 75a388eb1229b..038f49f30649a 100644
--- a/llvm/test/CodeGen/AMDGPU/remove-incompatible-wave32-feature.ll
+++ b/llvm/test/CodeGen/AMDGPU/remove-incompatible-wave32-feature.ll
@@ -14,7 +14,6 @@
 
 ; RUN: llc -enable-new-pm -mtriple=amdgcn -mcpu=gfx1011 -mattr=-wavefrontsize32,+wavefrontsize64 -stop-after=amdgpu-remove-incompatible-functions\
 ; RUN:   -pass-remarks=amdgpu-remove-incompatible-functions < %s 2>%t | FileCheck -check-prefixes=GFX10 %s
-; RUN: llc -enable-new-pm -mtriple=amdgcn -mcpu=gfx1011 -mattr=-wavefrontsize32,+wavefrontsize64 -verify-machineinstrs < %s
 
 ; RUN: llc -mtriple=amdgcn -mcpu=gfx1100 -mattr=-wavefrontsize32,+wavefrontsize64 -stop-after=amdgpu-remove-incompatible-functions\
 ; RUN:   -pass-remarks=amdgpu-remove-incompatible-functions < %s 2>%t | FileCheck -check-prefixes=GFX11 %s
@@ -22,7 +21,6 @@
 
 ; RUN: llc -enable-new-pm -mtriple=amdgcn -mcpu=gfx1100 -mattr=-wavefrontsize32,+wavefrontsize64 -stop-after=amdgpu-remove-incompatible-functions\
 ; RUN:   -pass-remarks=amdgpu-remove-incompatible-functions < %s 2>%t | FileCheck -check-prefixes=GFX11 %s
-; RUN: llc -enable-new-pm -mtriple=amdgcn -mcpu=gfx1100 -mattr=-wavefrontsize32,+wavefrontsize64 -verify-machineinstrs < %s
 
 ; WARN-GFX906: removing function 'needs_wavefrontsize32': +wavefrontsize32 is not supported on the current target
 ; WARN-GFX906-NOT: not supported
diff --git a/llvm/test/CodeGen/AMDGPU/rename-independent-subregs-mac-operands.mir b/llvm/test/CodeGen/AMDGPU/rename-independent-subregs-mac-operands.mir
index bb86f65f850db..daa9997f69f6c 100644
--- a/llvm/test/CodeGen/AMDGPU/rename-independent-subregs-mac-operands.mir
+++ b/llvm/test/CodeGen/AMDGPU/rename-independent-subregs-mac-operands.mir
@@ -1,4 +1,5 @@
 # RUN: llc -mtriple=amdgcn -verify-machineinstrs -run-pass=register-coalescer,rename-independent-subregs -o - %s | FileCheck -check-prefix=GCN %s
+# RUN: llc -mtriple=amdgcn -passes=register-coalescer,rename-independent-subregs -o - %s | FileCheck -check-prefix=GCN %s
 ---
 
 # GCN-LABEL: name: mac_invalid_operands
diff --git a/llvm/test/CodeGen/AMDGPU/rewrite-out-arguments.ll b/llvm/test/CodeGen/AMDGPU/rewrite-out-arguments.ll
index d946d594fde2f..5b48005aaaeb6 100644
--- a/llvm/test/CodeGen/AMDGPU/rewrite-out-arguments.ll
+++ b/llvm/test/CodeGen/AMDGPU/rewrite-out-arguments.ll
@@ -489,7 +489,7 @@ attributes #2 = { alwaysinline nounwind }
 ;
 ; CHECK-LABEL: define {{[^@]+}}@void_one_out_arg_i32_1_use
 ; CHECK-SAME: (ptr addrspace(5) [[TMP0:%.*]]) #[[ATTR2:[0-9]+]] {
-; CHECK-NEXT:    [[TMP2:%.*]] = call [[VOID_ONE_OUT_ARG_I32_1_USE:%.*]] @void_one_out_arg_i32_1_use.body(ptr addrspace(5) poison)
+; CHECK-NEXT:    [[TMP2:%.*]] = call [[VOID_ONE_OUT_ARG_I32_1_USE:%.*]] @[[VOID_ONE_OUT_ARG_I32_1_USE_BODY:[a-zA-Z0-9_$\"\\.-]*[a-zA-Z_$\"\\.-][a-zA-Z0-9_$\"\\.-]*]](ptr addrspace(5) poison)
 ; CHECK-NEXT:    [[TMP3:%.*]] = extractvalue [[VOID_ONE_OUT_ARG_I32_1_USE]] [[TMP2]], 0
 ; CHECK-NEXT:    store i32 [[TMP3]], ptr addrspace(5) [[TMP0]], align 4
 ; CHECK-NEXT:    ret void
@@ -502,7 +502,7 @@ attributes #2 = { alwaysinline nounwind }
 ;
 ; CHECK-LABEL: define {{[^@]+}}@void_one_out_arg_i32_1_use_align
 ; CHECK-SAME: (ptr addrspace(5) align 8 [[TMP0:%.*]]) #[[ATTR2]] {
-; CHECK-NEXT:    [[TMP2:%.*]] = call [[VOID_ONE_OUT_ARG_I32_1_USE_ALIGN:%.*]] @void_one_out_arg_i32_1_use_align.body(ptr addrspace(5) poison)
+; CHECK-NEXT:    [[TMP2:%.*]] = call [[VOID_ONE_OUT_ARG_I32_1_USE_ALIGN:%.*]] @[[VOID_ONE_OUT_ARG_I32_1_USE_ALIGN_BODY:[a-zA-Z0-9_$\"\\.-]*[a-zA-Z_$\"\\.-][a-zA-Z0-9_$\"\\.-]*]](ptr addrspace(5) poison)
 ; CHECK-NEXT:    [[TMP3:%.*]] = extractvalue [[VOID_ONE_OUT_ARG_I32_1_USE_ALIGN]] [[TMP2]], 0
 ; CHECK-NEXT:    store i32 [[TMP3]], ptr addrspace(5) [[TMP0]], align 8
 ; CHECK-NEXT:    ret void
@@ -519,7 +519,7 @@ attributes #2 = { alwaysinline nounwind }
 ;
 ; CHECK-LABEL: define {{[^@]+}}@void_one_out_arg_i32_2_use
 ; CHECK-SAME: (i1 [[TMP0:%.*]], ptr addrspace(5) [[TMP1:%.*]]) #[[ATTR2]] {
-; CHECK-NEXT:    [[TMP3:%.*]] = call [[VOID_ONE_OUT_ARG_I32_2_USE:%.*]] @void_one_out_arg_i32_2_use.body(i1 [[TMP0]], ptr addrspace(5) poison)
+; CHECK-NEXT:    [[TMP3:%.*]] = call [[VOID_ONE_OUT_ARG_I32_2_USE:%.*]] @[[VOID_ONE_OUT_ARG_I32_2_USE_BODY:[a-zA-Z0-9_$\"\\.-]*[a-zA-Z_$\"\\.-][a-zA-Z0-9_$\"\\.-]*]](i1 [[TMP0]], ptr addrspace(5) poison)
 ; CHECK-NEXT:    [[TMP4:%.*]] = extractvalue [[VOID_ONE_OUT_ARG_I32_2_USE]] [[TMP3]], 0
 ; CHECK-NEXT:    store i32 [[TMP4]], ptr addrspace(5) [[TMP1]], align 4
 ; CHECK-NEXT:    ret void
@@ -533,7 +533,7 @@ attributes #2 = { alwaysinline nounwind }
 ;
 ; CHECK-LABEL: define {{[^@]+}}@void_one_out_arg_i32_2_stores
 ; CHECK-SAME: (ptr addrspace(5) [[TMP0:%.*]]) #[[ATTR2]] {
-; CHECK-NEXT:    [[TMP2:%.*]] = call [[VOID_ONE_OUT_ARG_I32_2_STORES:%.*]] @void_one_out_arg_i32_2_stores.body(ptr addrspace(5) poison)
+; CHECK-NEXT:    [[TMP2:%.*]] = call [[VOID_ONE_OUT_ARG_I32_2_STORES:%.*]] @[[VOID_ONE_OUT_ARG_I32_2_STORES_BODY:[a-zA-Z0-9_$\"\\.-]*[a-zA-Z_$\"\\.-][a-zA-Z0-9_$\"\\.-]*]](ptr addrspace(5) poison)
 ; CHECK-NEXT:    [[TMP3:%.*]] = extractvalue [[VOID_ONE_OUT_ARG_I32_2_STORES]] [[TMP2]], 0
 ; CHECK-NEXT:    store i32 [[TMP3]], ptr addrspace(5) [[TMP0]], align 4
 ; CHECK-NEXT:    ret void
@@ -548,7 +548,7 @@ attributes #2 = { alwaysinline nounwind }
 ;
 ; CHECK-LABEL: define {{[^@]+}}@void_one_out_arg_i32_2_stores_clobber
 ; CHECK-SAME: (ptr addrspace(5) [[TMP0:%.*]]) #[[ATTR2]] {
-; CHECK-NEXT:    [[TMP2:%.*]] = call [[VOID_ONE_OUT_ARG_I32_2_STORES_CLOBBER:%.*]] @void_one_out_arg_i32_2_stores_clobber.body(ptr addrspace(5) poison)
+; CHECK-NEXT:    [[TMP2:%.*]] = call [[VOID_ONE_OUT_ARG_I32_2_STORES_CLOBBER:%.*]] @[[VOID_ONE_OUT_ARG_I32_2_STORES_CLOBBER_BODY:[a-zA-Z0-9_$\"\\.-]*[a-zA-Z_$\"\\.-][a-zA-Z0-9_$\"\\.-]*]](ptr addrspace(5) poison)
 ; CHECK-NEXT:    [[TMP3:%.*]] = extractvalue [[VOID_ONE_OUT_ARG_I32_2_STORES_CLOBBER]] [[TMP2]], 0
 ; CHECK-NEXT:    store i32 [[TMP3]], ptr addrspace(5) [[TMP0]], align 4
 ; CHECK-NEXT:    ret void
@@ -569,7 +569,7 @@ attributes #2 = { alwaysinline nounwind }
 ;
 ; CHECK-LABEL: define {{[^@]+}}@void_one_out_arg_i32_pre_call_may_clobber
 ; CHECK-SAME: (ptr addrspace(5) [[TMP0:%.*]]) #[[ATTR2]] {
-; CHECK-NEXT:    [[TMP2:%.*]] = call [[VOID_ONE_OUT_ARG_I32_PRE_CALL_MAY_CLOBBER:%.*]] @void_one_out_arg_i32_pre_call_may_clobber.body(ptr addrspace(5) poison)
+; CHECK-NEXT:    [[TMP2:%.*]] = call [[VOID_ONE_OUT_ARG_I32_PRE_CALL_MAY_CLOBBER:%.*]] @[[VOID_ONE_OUT_ARG_I32_PRE_CALL_MAY_CLOBBER_BODY:[a-zA-Z0-9_$\"\\.-]*[a-zA-Z_$\"\\.-][a-zA-Z0-9_$\"\\.-]*]](ptr addrspace(5) poison)
 ; CHECK-NEXT:    [[TMP3:%.*]] = extractvalue [[VOID_ONE_OUT_ARG_I32_PRE_CALL_MAY_CLOBBER]] [[TMP2]], 0
 ; CHECK-NEXT:    store i32 [[TMP3]], ptr addrspace(5) [[TMP0]], align 4
 ; CHECK-NEXT:    ret void
@@ -608,7 +608,7 @@ attributes #2 = { alwaysinline nounwind }
 ;
 ; CHECK-LABEL: define {{[^@]+}}@void_one_out_arg_v2i32_1_use
 ; CHECK-SAME: (ptr addrspace(5) [[TMP0:%.*]]) #[[ATTR2]] {
-; CHECK-NEXT:    [[TMP2:%.*]] = call [[VOID_ONE_OUT_ARG_V2I32_1_USE:%.*]] @void_one_out_arg_v2i32_1_use.body(ptr addrspace(5) poison)
+; CHECK-NEXT:    [[TMP2:%.*]] = call [[VOID_ONE_OUT_ARG_V2I32_1_USE:%.*]] @[[VOID_ONE_OUT_ARG_V2I32_1_USE_BODY:[a-zA-Z0-9_$\"\\.-]*[a-zA-Z_$\"\\.-][a-zA-Z0-9_$\"\\.-]*]](ptr addrspace(5) poison)
 ; CHECK-NEXT:    [[TMP3:%.*]] = extractvalue [[VOID_ONE_OUT_ARG_V2I32_1_USE]] [[TMP2]], 0
 ; CHECK-NEXT:    store <2 x i32> [[TMP3]], ptr addrspace(5) [[TMP0]], align 8
 ; CHECK-NEXT:    ret void
@@ -621,7 +621,7 @@ attributes #2 = { alwaysinline nounwind }
 ;
 ; CHECK-LABEL: define {{[^@]+}}@void_one_out_arg_struct_1_use
 ; CHECK-SAME: (ptr addrspace(5) [[TMP0:%.*]]) #[[ATTR2]] {
-; CHECK-NEXT:    [[TMP2:%.*]] = call [[VOID_ONE_OUT_ARG_STRUCT_1_USE:%.*]] @void_one_out_arg_struct_1_use.body(ptr addrspace(5) poison)
+; CHECK-NEXT:    [[TMP2:%.*]] = call [[VOID_ONE_OUT_ARG_STRUCT_1_USE:%.*]] @[[VOID_ONE_OUT_ARG_STRUCT_1_USE_BODY:[a-zA-Z0-9_$\"\\.-]*[a-zA-Z_$\"\\.-][a-zA-Z0-9_$\"\\.-]*]](ptr addrspace(5) poison)
 ; CHECK-NEXT:    [[TMP3:%.*]] = extractvalue [[VOID_ONE_OUT_ARG_STRUCT_1_USE]] [[TMP2]], 0
 ; CHECK-NEXT:    store [[STRUCT:%.*]] [[TMP3]], ptr addrspace(5) [[TMP0]], align 4
 ; CHECK-NEXT:    ret void
@@ -634,7 +634,7 @@ attributes #2 = { alwaysinline nounwind }
 ;
 ; CHECK-LABEL: define {{[^@]+}}@i32_one_out_arg_i32_1_use
 ; CHECK-SAME: (ptr addrspace(5) [[TMP0:%.*]]) #[[ATTR2]] {
-; CHECK-NEXT:    [[TMP2:%.*]] = call [[I32_ONE_OUT_ARG_I32_1_USE:%.*]] @i32_one_out_arg_i32_1_use.body(ptr addrspace(5) poison)
+; CHECK-NEXT:    [[TMP2:%.*]] = call [[I32_ONE_OUT_ARG_I32_1_USE:%.*]] @[[I32_ONE_OUT_ARG_I32_1_USE_BODY:[a-zA-Z0-9_$\"\\.-]*[a-zA-Z_$\"\\.-][a-zA-Z0-9_$\"\\.-]*]](ptr addrspace(5) poison)
 ; CHECK-NEXT:    [[TMP3:%.*]] = extractvalue [[I32_ONE_OUT_ARG_I32_1_USE]] [[TMP2]], 1
 ; CHECK-NEXT:    store i32 [[TMP3]], ptr addrspace(5) [[TMP0]], align 4
 ; CHECK-NEXT:    [[TMP4:%.*]] = extractvalue [[I32_ONE_OUT_ARG_I32_1_USE]] [[TMP2]], 0
@@ -642,13 +642,13 @@ attributes #2 = { alwaysinline nounwind }
 ;
 ;
 ; CHECK-LABEL: define {{[^@]+}}@unused_different_type.body
-; CHECK-SAME: (ptr addrspace(5) [[ARG0:%.*]], ptr addrspace(5) nocapture [[ARG1:%.*]]) #[[ATTR0]] {
+; CHECK-SAME: (ptr addrspace(5) [[ARG0:%.*]], ptr addrspace(5) captures(none) [[ARG1:%.*]]) #[[ATTR0]] {
 ; CHECK-NEXT:    ret [[UNUSED_DIFFERENT_TYPE:%.*]] { float 4.000000e+00 }
 ;
 ;
 ; CHECK-LABEL: define {{[^@]+}}@unused_different_type
-; CHECK-SAME: (ptr addrspace(5) [[TMP0:%.*]], ptr addrspace(5) nocapture [[TMP1:%.*]]) #[[ATTR2]] {
-; CHECK-NEXT:    [[TMP3:%.*]] = call [[UNUSED_DIFFERENT_TYPE:%.*]] @unused_different_type.body(ptr addrspace(5) [[TMP0]], ptr addrspace(5) poison)
+; CHECK-SAME: (ptr addrspace(5) [[TMP0:%.*]], ptr addrspace(5) captures(none) [[TMP1:%.*]]) #[[ATTR2]] {
+; CHECK-NEXT:    [[TMP3:%.*]] = call [[UNUSED_DIFFERENT_TYPE:%.*]] @[[UNUSED_DIFFERENT_TYPE_BODY:[a-zA-Z0-9_$\"\\.-]*[a-zA-Z_$\"\\.-][a-zA-Z0-9_$\"\\.-]*]](ptr addrspace(5) [[TMP0]], ptr addrspace(5) poison)
 ; CHECK-NEXT:    [[TMP4:%.*]] = extractvalue [[UNUSED_DIFFERENT_TYPE]] [[TMP3]], 0
 ; CHECK-NEXT:    store float [[TMP4]], ptr addrspace(5) [[TMP1]], align 4
 ; CHECK-NEXT:    ret void
@@ -661,7 +661,7 @@ attributes #2 = { alwaysinline nounwind }
 ;
 ; CHECK-LABEL: define {{[^@]+}}@multiple_same_return_noalias
 ; CHECK-SAME: (ptr addrspace(5) noalias [[TMP0:%.*]], ptr addrspace(5) noalias [[TMP1:%.*]]) #[[ATTR2]] {
-; CHECK-NEXT:    [[TMP3:%.*]] = call [[MULTIPLE_SAME_RETURN_NOALIAS:%.*]] @multiple_same_return_noalias.body(ptr addrspace(5) poison, ptr addrspace(5) poison)
+; CHECK-NEXT:    [[TMP3:%.*]] = call [[MULTIPLE_SAME_RETURN_NOALIAS:%.*]] @[[MULTIPLE_SAME_RETURN_NOALIAS_BODY:[a-zA-Z0-9_$\"\\.-]*[a-zA-Z_$\"\\.-][a-zA-Z0-9_$\"\\.-]*]](ptr addrspace(5) poison, ptr addrspace(5) poison)
 ; CHECK-NEXT:    [[TMP4:%.*]] = extractvalue [[MULTIPLE_SAME_RETURN_NOALIAS]] [[TMP3]], 0
 ; CHECK-NEXT:    store i32 [[TMP4]], ptr addrspace(5) [[TMP0]], align 4
 ; CHECK-NEXT:    [[TMP5:%.*]] = extractvalue [[MULTIPLE_SAME_RETURN_NOALIAS]] [[TMP3]], 1
@@ -676,7 +676,7 @@ attributes #2 = { alwaysinline nounwind }
 ;
 ; CHECK-LABEL: define {{[^@]+}}@multiple_same_return_mayalias
 ; CHECK-SAME: (ptr addrspace(5) [[TMP0:%.*]], ptr addrspace(5) [[TMP1:%.*]]) #[[ATTR2]] {
-; CHECK-NEXT:    [[TMP3:%.*]] = call [[MULTIPLE_SAME_RETURN_MAYALIAS:%.*]] @multiple_same_return_mayalias.body(ptr addrspace(5) poison, ptr addrspace(5) poison)
+; CHECK-NEXT:    [[TMP3:%.*]] = call [[MULTIPLE_SAME_RETURN_MAYALIAS:%.*]] @[[MULTIPLE_SAME_RETURN_MAYALIAS_BODY:[a-zA-Z0-9_$\"\\.-]*[a-zA-Z_$\"\\.-][a-zA-Z0-9_$\"\\.-]*]](ptr addrspace(5) poison, ptr addrspace(5) poison)
 ; CHECK-NEXT:    [[TMP4:%.*]] = extractvalue [[MULTIPLE_SAME_RETURN_MAYALIAS]] [[TMP3]], 0
 ; CHECK-NEXT:    store i32 [[TMP4]], ptr addrspace(5) [[TMP0]], align 4
 ; CHECK-NEXT:    [[TMP5:%.*]] = extractvalue [[MULTIPLE_SAME_RETURN_MAYALIAS]] [[TMP3]], 1
@@ -691,7 +691,7 @@ attributes #2 = { alwaysinline nounwind }
 ;
 ; CHECK-LABEL: define {{[^@]+}}@multiple_same_return_mayalias_order
 ; CHECK-SAME: (ptr addrspace(5) [[TMP0:%.*]], ptr addrspace(5) [[TMP1:%.*]]) #[[ATTR2]] {
-; CHECK-NEXT:    [[TMP3:%.*]] = call [[MULTIPLE_SAME_RETURN_MAYALIAS_ORDER:%.*]] @multiple_same_return_mayalias_order.body(ptr addrspace(5) poison, ptr addrspace(5) poison)
+; CHECK-NEXT:    [[TMP3:%.*]] = call [[MULTIPLE_SAME_RETURN_MAYALIAS_ORDER:%.*]] @[[MULTIPLE_SAME_RETURN_MAYALIAS_ORDER_BODY:[a-zA-Z0-9_$\"\\.-]*[a-zA-Z_$\"\\.-][a-zA-Z0-9_$\"\\.-]*]](ptr addrspace(5) poison, ptr addrspace(5) poison)
 ; CHECK-NEXT:    [[TMP4:%.*]] = extractvalue [[MULTIPLE_SAME_RETURN_MAYALIAS_ORDER]] [[TMP3]], 0
 ; CHECK-NEXT:    store i32 [[TMP4]], ptr addrspace(5) [[TMP0]], align 4
 ; CHECK-NEXT:    [[TMP5:%.*]] = extractvalue [[MULTIPLE_SAME_RETURN_MAYALIAS_ORDER]] [[TMP3]], 1
@@ -720,7 +720,7 @@ attributes #2 = { alwaysinline nounwind }
 ;
 ; CHECK-LABEL: define {{[^@]+}}@i1_one_out_arg_i32_1_use
 ; CHECK-SAME: (ptr addrspace(5) [[TMP0:%.*]]) #[[ATTR2]] {
-; CHECK-NEXT:    [[TMP2:%.*]] = call [[I1_ONE_OUT_ARG_I32_1_USE:%.*]] @i1_one_out_arg_i32_1_use.body(ptr addrspace(5) poison)
+; CHECK-NEXT:    [[TMP2:%.*]] = call [[I1_ONE_OUT_ARG_I32_1_USE:%.*]] @[[I1_ONE_OUT_ARG_I32_1_USE_BODY:[a-zA-Z0-9_$\"\\.-]*[a-zA-Z_$\"\\.-][a-zA-Z0-9_$\"\\.-]*]](ptr addrspace(5) poison)
 ; CHECK-NEXT:    [[TMP3:%.*]] = extractvalue [[I1_ONE_OUT_ARG_I32_1_USE]] [[TMP2]], 1
 ; CHECK-NEXT:    store i32 [[TMP3]], ptr addrspace(5) [[TMP0]], align 4
 ; CHECK-NEXT:    [[TMP4:%.*]] = extractvalue [[I1_ONE_OUT_ARG_I32_1_USE]] [[TMP2]], 0
@@ -734,7 +734,7 @@ attributes #2 = { alwaysinline nounwind }
 ;
 ; CHECK-LABEL: define {{[^@]+}}@i1_zeroext_one_out_arg_i32_1_use
 ; CHECK-SAME: (ptr addrspace(5) [[TMP0:%.*]]) #[[ATTR2]] {
-; CHECK-NEXT:    [[TMP2:%.*]] = call [[I1_ZEROEXT_ONE_OUT_ARG_I32_1_USE:%.*]] @i1_zeroext_one_out_arg_i32_1_use.body(ptr addrspace(5) poison)
+; CHECK-NEXT:    [[TMP2:%.*]] = call [[I1_ZEROEXT_ONE_OUT_ARG_I32_1_USE:%.*]] @[[I1_ZEROEXT_ONE_OUT_ARG_I32_1_USE_BODY:[a-zA-Z0-9_$\"\\.-]*[a-zA-Z_$\"\\.-][a-zA-Z0-9_$\"\\.-]*]](ptr addrspace(5) poison)
 ; CHECK-NEXT:    [[TMP3:%.*]] = extractvalue [[I1_ZEROEXT_ONE_OUT_ARG_I32_1_USE]] [[TMP2]], 1
 ; CHECK-NEXT:    store i32 [[TMP3]], ptr addrspace(5) [[TMP0]], align 4
 ; CHECK-NEXT:    [[TMP4:%.*]] = extractvalue [[I1_ZEROEXT_ONE_OUT_ARG_I32_1_USE]] [[TMP2]], 0
@@ -748,7 +748,7 @@ attributes #2 = { alwaysinline nounwind }
 ;
 ; CHECK-LABEL: define {{[^@]+}}@i1_signext_one_out_arg_i32_1_use
 ; CHECK-SAME: (ptr addrspace(5) [[TMP0:%.*]]) #[[ATTR2]] {
-; CHECK-NEXT:    [[TMP2:%.*]] = call [[I1_SIGNEXT_ONE_OUT_ARG_I32_1_USE:%.*]] @i1_signext_one_out_arg_i32_1_use.body(ptr addrspace(5) poison)
+; CHECK-NEXT:    [[TMP2:%.*]] = call [[I1_SIGNEXT_ONE_OUT_ARG_I32_1_USE:%.*]] @[[I1_SIGNEXT_ONE_OUT_ARG_I32_1_USE_BODY:[a-zA-Z0-9_$\"\\.-]*[a-zA-Z_$\"\\.-][a-zA-Z0-9_$\"\\.-]*]](ptr addrspace(5) poison)
 ; CHECK-NEXT:    [[TMP3:%.*]] = extractvalue [[I1_SIGNEXT_ONE_OUT_ARG_I32_1_USE]] [[TMP2]], 1
 ; CHECK-NEXT:    store i32 [[TMP3]], ptr addrspace(5) [[TMP0]], align 4
 ; CHECK-NEXT:    [[TMP4:%.*]] = extractvalue [[I1_SIGNEXT_ONE_OUT_ARG_I32_1_USE]] [[TMP2]], 0
@@ -762,7 +762,7 @@ attributes #2 = { alwaysinline nounwind }
 ;
 ; CHECK-LABEL: define {{[^@]+}}@p1i32_noalias_one_out_arg_i32_1_use
 ; CHECK-SAME: (ptr addrspace(5) [[TMP0:%.*]]) #[[ATTR2]] {
-; CHECK-NEXT:    [[TMP2:%.*]] = call [[P1I32_NOALIAS_ONE_OUT_ARG_I32_1_USE:%.*]] @p1i32_noalias_one_out_arg_i32_1_use.body(ptr addrspace(5) poison)
+; CHECK-NEXT:    [[TMP2:%.*]] = call [[P1I32_NOALIAS_ONE_OUT_ARG_I32_1_USE:%.*]] @[[P1I32_NOALIAS_ONE_OUT_ARG_I32_1_USE_BODY:[a-zA-Z0-9_$\"\\.-]*[a-zA-Z_$\"\\.-][a-zA-Z0-9_$\"\\.-]*]](ptr addrspace(5) poison)
 ; CHECK-NEXT:    [[TMP3:%.*]] = extractvalue [[P1I32_NOALIAS_ONE_OUT_ARG_I32_1_USE]] [[TMP2]], 1
 ; CHECK-NEXT:    store i32 [[TMP3]], ptr addrspace(5) [[TMP0]], align 4
 ; CHECK-NEXT:    [[TMP4:%.*]] = extractvalue [[P1I32_NOALIAS_ONE_OUT_ARG_I32_1_USE]] [[TMP2]], 0
@@ -784,7 +784,7 @@ attributes #2 = { alwaysinline nounwind }
 ;
 ; CHECK-LABEL: define {{[^@]+}}@func_ptr_type
 ; CHECK-SAME: (ptr addrspace(5) [[TMP0:%.*]]) #[[ATTR2]] {
-; CHECK-NEXT:    [[TMP2:%.*]] = call [[FUNC_PTR_TYPE:%.*]] @func_ptr_type.body(ptr addrspace(5) poison)
+; CHECK-NEXT:    [[TMP2:%.*]] = call [[FUNC_PTR_TYPE:%.*]] @[[FUNC_PTR_TYPE_BODY:[a-zA-Z0-9_$\"\\.-]*[a-zA-Z_$\"\\.-][a-zA-Z0-9_$\"\\.-]*]](ptr addrspace(5) poison)
 ; CHECK-NEXT:    [[TMP3:%.*]] = extractvalue [[FUNC_PTR_TYPE]] [[TMP2]], 0
 ; CHECK-NEXT:    store ptr [[TMP3]], ptr addrspace(5) [[TMP0]], align 8
 ; CHECK-NEXT:    ret void
@@ -799,7 +799,7 @@ attributes #2 = { alwaysinline nounwind }
 ;
 ; CHECK-LABEL: define {{[^@]+}}@bitcast_func_ptr_type
 ; CHECK-SAME: (ptr addrspace(5) [[TMP0:%.*]]) #[[ATTR2]] {
-; CHECK-NEXT:    [[TMP2:%.*]] = call [[BITCAST_FUNC_PTR_TYPE:%.*]] @bitcast_func_ptr_type.body(ptr addrspace(5) poison)
+; CHECK-NEXT:    [[TMP2:%.*]] = call [[BITCAST_FUNC_PTR_TYPE:%.*]] @[[BITCAST_FUNC_PTR_TYPE_BODY:[a-zA-Z0-9_$\"\\.-]*[a-zA-Z_$\"\\.-][a-zA-Z0-9_$\"\\.-]*]](ptr addrspace(5) poison)
 ; CHECK-NEXT:    [[TMP3:%.*]] = extractvalue [[BITCAST_FUNC_PTR_TYPE]] [[TMP2]], 0
 ; CHECK-NEXT:    store ptr [[TMP3]], ptr addrspace(5) [[TMP0]], align 8
 ; CHECK-NEXT:    ret void
@@ -812,7 +812,7 @@ attributes #2 = { alwaysinline nounwind }
 ;
 ; CHECK-LABEL: define {{[^@]+}}@out_arg_small_array
 ; CHECK-SAME: (ptr addrspace(5) [[TMP0:%.*]]) #[[ATTR2]] {
-; CHECK-NEXT:    [[TMP2:%.*]] = call [[OUT_ARG_SMALL_ARRAY:%.*]] @out_arg_small_array.body(ptr addrspace(5) poison)
+; CHECK-NEXT:    [[TMP2:%.*]] = call [[OUT_ARG_SMALL_ARRAY:%.*]] @[[OUT_ARG_SMALL_ARRAY_BODY:[a-zA-Z0-9_$\"\\.-]*[a-zA-Z_$\"\\.-][a-zA-Z0-9_$\"\\.-]*]](ptr addrspace(5) poison)
 ; CHECK-NEXT:    [[TMP3:%.*]] = extractvalue [[OUT_ARG_SMALL_ARRAY]] [[TMP2]], 0
 ; CHECK-NEXT:    store [4 x i32] [[TMP3]], ptr addrspace(5) [[TMP0]], align 4
 ; CHECK-NEXT:    ret void
@@ -841,7 +841,7 @@ attributes #2 = { alwaysinline nounwind }
 ;
 ; CHECK-LABEL: define {{[^@]+}}@num_regs_reach_limit
 ; CHECK-SAME: (ptr addrspace(5) [[TMP0:%.*]], i32 [[TMP1:%.*]]) #[[ATTR2]] {
-; CHECK-NEXT:    [[TMP3:%.*]] = call [[NUM_REGS_REACH_LIMIT:%.*]] @num_regs_reach_limit.body(ptr addrspace(5) poison, i32 [[TMP1]])
+; CHECK-NEXT:    [[TMP3:%.*]] = call [[NUM_REGS_REACH_LIMIT:%.*]] @[[NUM_REGS_REACH_LIMIT_BODY:[a-zA-Z0-9_$\"\\.-]*[a-zA-Z_$\"\\.-][a-zA-Z0-9_$\"\\.-]*]](ptr addrspace(5) poison, i32 [[TMP1]])
 ; CHECK-NEXT:    [[TMP4:%.*]] = extractvalue [[NUM_REGS_REACH_LIMIT]] [[TMP3]], 1
 ; CHECK-NEXT:    store i32 [[TMP4]], ptr addrspace(5) [[TMP0]], align 4
 ; CHECK-NEXT:    [[TMP5:%.*]] = extractvalue [[NUM_REGS_REACH_LIMIT]] [[TMP3]], 0
@@ -860,7 +860,7 @@ attributes #2 = { alwaysinline nounwind }
 ;
 ; CHECK-LABEL: define {{[^@]+}}@num_regs_reach_limit_leftover
 ; CHECK-SAME: (ptr addrspace(5) [[TMP0:%.*]], ptr addrspace(5) [[TMP1:%.*]], i32 [[TMP2:%.*]]) #[[ATTR2]] {
-; CHECK-NEXT:    [[TMP4:%.*]] = call [[NUM_REGS_REACH_LIMIT_LEFTOVER:%.*]] @num_regs_reach_limit_leftover.body(ptr addrspace(5) poison, ptr addrspace(5) poison, i32 [[TMP2]])
+; CHECK-NEXT:    [[TMP4:%.*]] = call [[NUM_REGS_REACH_LIMIT_LEFTOVER:%.*]] @[[NUM_REGS_REACH_LIMIT_LEFTOVER_BODY:[a-zA-Z0-9_$\"\\.-]*[a-zA-Z_$\"\\.-][a-zA-Z0-9_$\"\\.-]*]](ptr addrspace(5) poison, ptr addrspace(5) poison, i32 [[TMP2]])
 ; CHECK-NEXT:    [[TMP5:%.*]] = extractvalue [[NUM_REGS_REACH_LIMIT_LEFTOVER]] [[TMP4]], 1
 ; CHECK-NEXT:    store i32 [[TMP5]], ptr addrspace(5) [[TMP0]], align 4
 ; CHECK-NEXT:    [[TMP6:%.*]] = extractvalue [[NUM_REGS_REACH_LIMIT_LEFTOVER]] [[TMP4]], 2
@@ -878,7 +878,7 @@ attributes #2 = { alwaysinline nounwind }
 ;
 ; CHECK-LABEL: define {{[^@]+}}@preserve_debug_info
 ; CHECK-SAME: (i32 [[TMP0:%.*]], ptr addrspace(5) [[TMP1:%.*]]) #[[ATTR2]] !dbg [[DBG6:![0-9]+]] {
-; CHECK-NEXT:    [[TMP3:%.*]] = call [[PRESERVE_DEBUG_INFO:%.*]] @preserve_debug_info.body(i32 [[TMP0]], ptr addrspace(5) poison)
+; CHECK-NEXT:    [[TMP3:%.*]] = call [[PRESERVE_DEBUG_INFO:%.*]] @[[PRESERVE_DEBUG_INFO_BODY:[a-zA-Z0-9_$\"\\.-]*[a-zA-Z_$\"\\.-][a-zA-Z0-9_$\"\\.-]*]](i32 [[TMP0]], ptr addrspace(5) poison)
 ; CHECK-NEXT:    [[TMP4:%.*]] = extractvalue [[PRESERVE_DEBUG_INFO]] [[TMP3]], 0
 ; CHECK-NEXT:    store i32 [[TMP4]], ptr addrspace(5) [[TMP1]], align 4
 ; CHECK-NEXT:    ret void
@@ -892,8 +892,8 @@ attributes #2 = { alwaysinline nounwind }
 ;
 ;
 ; CHECK-LABEL: define {{[^@]+}}@preserve_metadata
-; CHECK-SAME: (i32 [[TMP0:%.*]], ptr addrspace(5) [[TMP1:%.*]]) #[[ATTR2]] !kernel_arg_access_qual !12 {
-; CHECK-NEXT:    [[TMP3:%.*]] = call [[PRESERVE_METADATA:%.*]] @preserve_metadata.body(i32 [[TMP0]], ptr addrspace(5) poison)
+; CHECK-SAME: (i32 [[TMP0:%.*]], ptr addrspace(5) [[TMP1:%.*]]) #[[ATTR2]] !kernel_arg_access_qual [[META12:![0-9]+]] {
+; CHECK-NEXT:    [[TMP3:%.*]] = call [[PRESERVE_METADATA:%.*]] @[[PRESERVE_METADATA_BODY:[a-zA-Z0-9_$\"\\.-]*[a-zA-Z_$\"\\.-][a-zA-Z0-9_$\"\\.-]*]](i32 [[TMP0]], ptr addrspace(5) poison)
 ; CHECK-NEXT:    [[TMP4:%.*]] = extractvalue [[PRESERVE_METADATA]] [[TMP3]], 0
 ; CHECK-NEXT:    store i32 [[TMP4]], ptr addrspace(5) [[TMP1]], align 4
 ; CHECK-NEXT:    ret void
@@ -908,7 +908,7 @@ attributes #2 = { alwaysinline nounwind }
 ;
 ; CHECK-LABEL: define {{[^@]+}}@bitcast_pointer_v4i32_v3i32
 ; CHECK-SAME: (ptr addrspace(5) [[TMP0:%.*]]) #[[ATTR2]] {
-; CHECK-NEXT:    [[TMP2:%.*]] = call [[BITCAST_POINTER_V4I32_V3I32:%.*]] @bitcast_pointer_v4i32_v3i32.body(ptr addrspace(5) poison)
+; CHECK-NEXT:    [[TMP2:%.*]] = call [[BITCAST_POINTER_V4I32_V3I32:%.*]] @[[BITCAST_POINTER_V4I32_V3I32_BODY:[a-zA-Z0-9_$\"\\.-]*[a-zA-Z_$\"\\.-][a-zA-Z0-9_$\"\\.-]*]](ptr addrspace(5) poison)
 ; CHECK-NEXT:    [[TMP3:%.*]] = extractvalue [[BITCAST_POINTER_V4I32_V3I32]] [[TMP2]], 0
 ; CHECK-NEXT:    store <4 x i32> [[TMP3]], ptr addrspace(5) [[TMP0]], align 16
 ; CHECK-NEXT:    ret void
@@ -923,7 +923,7 @@ attributes #2 = { alwaysinline nounwind }
 ;
 ; CHECK-LABEL: define {{[^@]+}}@bitcast_pointer_v4i32_v3f32
 ; CHECK-SAME: (ptr addrspace(5) [[TMP0:%.*]]) #[[ATTR2]] {
-; CHECK-NEXT:    [[TMP2:%.*]] = call [[BITCAST_POINTER_V4I32_V3F32:%.*]] @bitcast_pointer_v4i32_v3f32.body(ptr addrspace(5) poison)
+; CHECK-NEXT:    [[TMP2:%.*]] = call [[BITCAST_POINTER_V4I32_V3F32:%.*]] @[[BITCAST_POINTER_V4I32_V3F32_BODY:[a-zA-Z0-9_$\"\\.-]*[a-zA-Z_$\"\\.-][a-zA-Z0-9_$\"\\.-]*]](ptr addrspace(5) poison)
 ; CHECK-NEXT:    [[TMP3:%.*]] = extractvalue [[BITCAST_POINTER_V4I32_V3F32]] [[TMP2]], 0
 ; CHECK-NEXT:    store <4 x i32> [[TMP3]], ptr addrspace(5) [[TMP0]], align 16
 ; CHECK-NEXT:    ret void
@@ -938,7 +938,7 @@ attributes #2 = { alwaysinline nounwind }
 ;
 ; CHECK-LABEL: define {{[^@]+}}@bitcast_pointer_i32_f32
 ; CHECK-SAME: (ptr addrspace(5) [[TMP0:%.*]]) #[[ATTR2]] {
-; CHECK-NEXT:    [[TMP2:%.*]] = call [[BITCAST_POINTER_I32_F32:%.*]] @bitcast_pointer_i32_f32.body(ptr addrspace(5) poison)
+; CHECK-NEXT:    [[TMP2:%.*]] = call [[BITCAST_POINTER_I32_F32:%.*]] @[[BITCAST_POINTER_I32_F32_BODY:[a-zA-Z0-9_$\"\\.-]*[a-zA-Z_$\"\\.-][a-zA-Z0-9_$\"\\.-]*]](ptr addrspace(5) poison)
 ; CHECK-NEXT:    [[TMP3:%.*]] = extractvalue [[BITCAST_POINTER_I32_F32]] [[TMP2]], 0
 ; CHECK-NEXT:    store i32 [[TMP3]], ptr addrspace(5) [[TMP0]], align 4
 ; CHECK-NEXT:    ret void
@@ -953,7 +953,7 @@ attributes #2 = { alwaysinline nounwind }
 ;
 ; CHECK-LABEL: define {{[^@]+}}@bitcast_pointer_i32_f16
 ; CHECK-SAME: (ptr addrspace(5) [[TMP0:%.*]]) #[[ATTR2]] {
-; CHECK-NEXT:    [[TMP2:%.*]] = call [[BITCAST_POINTER_I32_F16:%.*]] @bitcast_pointer_i32_f16.body(ptr addrspace(5) poison)
+; CHECK-NEXT:    [[TMP2:%.*]] = call [[BITCAST_POINTER_I32_F16:%.*]] @[[BITCAST_POINTER_I32_F16_BODY:[a-zA-Z0-9_$\"\\.-]*[a-zA-Z_$\"\\.-][a-zA-Z0-9_$\"\\.-]*]](ptr addrspace(5) poison)
 ; CHECK-NEXT:    [[TMP3:%.*]] = extractvalue [[BITCAST_POINTER_I32_F16]] [[TMP2]], 0
 ; CHECK-NEXT:    store i32 [[TMP3]], ptr addrspace(5) [[TMP0]], align 4
 ; CHECK-NEXT:    ret void
@@ -968,7 +968,7 @@ attributes #2 = { alwaysinline nounwind }
 ;
 ; CHECK-LABEL: define {{[^@]+}}@bitcast_pointer_f16_i32
 ; CHECK-SAME: (ptr addrspace(5) [[TMP0:%.*]]) #[[ATTR2]] {
-; CHECK-NEXT:    [[TMP2:%.*]] = call [[BITCAST_POINTER_F16_I32:%.*]] @bitcast_pointer_f16_i32.body(ptr addrspace(5) poison)
+; CHECK-NEXT:    [[TMP2:%.*]] = call [[BITCAST_POINTER_F16_I32:%.*]] @[[BITCAST_POINTER_F16_I32_BODY:[a-zA-Z0-9_$\"\\.-]*[a-zA-Z_$\"\\.-][a-zA-Z0-9_$\"\\.-]*]](ptr addrspace(5) poison)
 ; CHECK-NEXT:    [[TMP3:%.*]] = extractvalue [[BITCAST_POINTER_F16_I32]] [[TMP2]], 0
 ; CHECK-NEXT:    store half [[TMP3]], ptr addrspace(5) [[TMP0]], align 2
 ; CHECK-NEXT:    ret void
@@ -983,7 +983,7 @@ attributes #2 = { alwaysinline nounwind }
 ;
 ; CHECK-LABEL: define {{[^@]+}}@bitcast_struct_v3f32_v3f32
 ; CHECK-SAME: (ptr addrspace(5) [[TMP0:%.*]], <3 x float> [[TMP1:%.*]]) #[[ATTR2]] {
-; CHECK-NEXT:    [[TMP3:%.*]] = call [[BITCAST_STRUCT_V3F32_V3F32:%.*]] @bitcast_struct_v3f32_v3f32.body(ptr addrspace(5) poison, <3 x float> [[TMP1]])
+; CHECK-NEXT:    [[TMP3:%.*]] = call [[BITCAST_STRUCT_V3F32_V3F32:%.*]] @[[BITCAST_STRUCT_V3F32_V3F32_BODY:[a-zA-Z0-9_$\"\\.-]*[a-zA-Z_$\"\\.-][a-zA-Z0-9_$\"\\.-]*]](ptr addrspace(5) poison, <3 x float> [[TMP1]])
 ; CHECK-NEXT:    [[TMP4:%.*]] = extractvalue [[BITCAST_STRUCT_V3F32_V3F32]] [[TMP3]], 0
 ; CHECK-NEXT:    store <4 x float> [[TMP4]], ptr addrspace(5) [[TMP0]], align 16
 ; CHECK-NEXT:    ret void
@@ -998,7 +998,7 @@ attributes #2 = { alwaysinline nounwind }
 ;
 ; CHECK-LABEL: define {{[^@]+}}@bitcast_struct_v3f32_v3i32
 ; CHECK-SAME: (ptr addrspace(5) [[TMP0:%.*]], <3 x i32> [[TMP1:%.*]]) #[[ATTR2]] {
-; CHECK-NEXT:    [[TMP3:%.*]] = call [[BITCAST_STRUCT_V3F32_V3I32:%.*]] @bitcast_struct_v3f32_v3i32.body(ptr addrspace(5) poison, <3 x i32> [[TMP1]])
+; CHECK-NEXT:    [[TMP3:%.*]] = call [[BITCAST_STRUCT_V3F32_V3I32:%.*]] @[[BITCAST_STRUCT_V3F32_V3I32_BODY:[a-zA-Z0-9_$\"\\.-]*[a-zA-Z_$\"\\.-][a-zA-Z0-9_$\"\\.-]*]](ptr addrspace(5) poison, <3 x i32> [[TMP1]])
 ; CHECK-NEXT:    [[TMP4:%.*]] = extractvalue [[BITCAST_STRUCT_V3F32_V3I32]] [[TMP3]], 0
 ; CHECK-NEXT:    store <4 x i32> [[TMP4]], ptr addrspace(5) [[TMP0]], align 16
 ; CHECK-NEXT:    ret void
@@ -1012,7 +1012,7 @@ attributes #2 = { alwaysinline nounwind }
 ;
 ; CHECK-LABEL: define {{[^@]+}}@bitcast_struct_v4f32_v4f32
 ; CHECK-SAME: (ptr addrspace(5) [[TMP0:%.*]], <4 x float> [[TMP1:%.*]]) #[[ATTR2]] {
-; CHECK-NEXT:    [[TMP3:%.*]] = call [[BITCAST_STRUCT_V4F32_V4F32:%.*]] @bitcast_struct_v4f32_v4f32.body(ptr addrspace(5) poison, <4 x float> [[TMP1]])
+; CHECK-NEXT:    [[TMP3:%.*]] = call [[BITCAST_STRUCT_V4F32_V4F32:%.*]] @[[BITCAST_STRUCT_V4F32_V4F32_BODY:[a-zA-Z0-9_$\"\\.-]*[a-zA-Z_$\"\\.-][a-zA-Z0-9_$\"\\.-]*]](ptr addrspace(5) poison, <4 x float> [[TMP1]])
 ; CHECK-NEXT:    [[TMP4:%.*]] = extractvalue [[BITCAST_STRUCT_V4F32_V4F32]] [[TMP3]], 0
 ; CHECK-NEXT:    store <4 x float> [[TMP4]], ptr addrspace(5) [[TMP0]], align 16
 ; CHECK-NEXT:    ret void
@@ -1026,7 +1026,7 @@ attributes #2 = { alwaysinline nounwind }
 ;
 ; CHECK-LABEL: define {{[^@]+}}@bitcast_struct_v3f32_v4i32
 ; CHECK-SAME: (ptr addrspace(5) [[TMP0:%.*]], <4 x i32> [[TMP1:%.*]]) #[[ATTR2]] {
-; CHECK-NEXT:    [[TMP3:%.*]] = call [[BITCAST_STRUCT_V3F32_V4I32:%.*]] @bitcast_struct_v3f32_v4i32.body(ptr addrspace(5) poison, <4 x i32> [[TMP1]])
+; CHECK-NEXT:    [[TMP3:%.*]] = call [[BITCAST_STRUCT_V3F32_V4I32:%.*]] @[[BITCAST_STRUCT_V3F32_V4I32_BODY:[a-zA-Z0-9_$\"\\.-]*[a-zA-Z_$\"\\.-][a-zA-Z0-9_$\"\\.-]*]](ptr addrspace(5) poison, <4 x i32> [[TMP1]])
 ; CHECK-NEXT:    [[TMP4:%.*]] = extractvalue [[BITCAST_STRUCT_V3F32_V4I32]] [[TMP3]], 0
 ; CHECK-NEXT:    store <4 x i32> [[TMP4]], ptr addrspace(5) [[TMP0]], align 16
 ; CHECK-NEXT:    ret void
@@ -1041,7 +1041,7 @@ attributes #2 = { alwaysinline nounwind }
 ;
 ; CHECK-LABEL: define {{[^@]+}}@bitcast_struct_v4f32_v3f32
 ; CHECK-SAME: (ptr addrspace(5) [[TMP0:%.*]], <3 x float> [[TMP1:%.*]]) #[[ATTR2]] {
-; CHECK-NEXT:    [[TMP3:%.*]] = call [[BITCAST_STRUCT_V4F32_V3F32:%.*]] @bitcast_struct_v4f32_v3f32.body(ptr addrspace(5) poison, <3 x float> [[TMP1]])
+; CHECK-NEXT:    [[TMP3:%.*]] = call [[BITCAST_STRUCT_V4F32_V3F32:%.*]] @[[BITCAST_STRUCT_V4F32_V3F32_BODY:[a-zA-Z0-9_$\"\\.-]*[a-zA-Z_$\"\\.-][a-zA-Z0-9_$\"\\.-]*]](ptr addrspace(5) poison, <3 x float> [[TMP1]])
 ; CHECK-NEXT:    [[TMP4:%.*]] = extractvalue [[BITCAST_STRUCT_V4F32_V3F32]] [[TMP3]], 0
 ; CHECK-NEXT:    store <4 x float> [[TMP4]], ptr addrspace(5) [[TMP0]], align 16
 ; CHECK-NEXT:    ret void
@@ -1055,7 +1055,7 @@ attributes #2 = { alwaysinline nounwind }
 ;
 ; CHECK-LABEL: define {{[^@]+}}@bitcast_struct_v3f32_v2f32
 ; CHECK-SAME: (ptr addrspace(5) [[TMP0:%.*]], <2 x float> [[TMP1:%.*]]) #[[ATTR2]] {
-; CHECK-NEXT:    [[TMP3:%.*]] = call [[BITCAST_STRUCT_V3F32_V2F32:%.*]] @bitcast_struct_v3f32_v2f32.body(ptr addrspace(5) poison, <2 x float> [[TMP1]])
+; CHECK-NEXT:    [[TMP3:%.*]] = call [[BITCAST_STRUCT_V3F32_V2F32:%.*]] @[[BITCAST_STRUCT_V3F32_V2F32_BODY:[a-zA-Z0-9_$\"\\.-]*[a-zA-Z_$\"\\.-][a-zA-Z0-9_$\"\\.-]*]](ptr addrspace(5) poison, <2 x float> [[TMP1]])
 ; CHECK-NEXT:    [[TMP4:%.*]] = extractvalue [[BITCAST_STRUCT_V3F32_V2F32]] [[TMP3]], 0
 ; CHECK-NEXT:    store <2 x float> [[TMP4]], ptr addrspace(5) [[TMP0]], align 8
 ; CHECK-NEXT:    ret void
@@ -1070,7 +1070,7 @@ attributes #2 = { alwaysinline nounwind }
 ;
 ; CHECK-LABEL: define {{[^@]+}}@bitcast_struct_v3f32_f32_v3f32
 ; CHECK-SAME: (ptr addrspace(5) [[TMP0:%.*]], <3 x float> [[TMP1:%.*]]) #[[ATTR2]] {
-; CHECK-NEXT:    [[TMP3:%.*]] = call [[BITCAST_STRUCT_V3F32_F32_V3F32:%.*]] @bitcast_struct_v3f32_f32_v3f32.body(ptr addrspace(5) poison, <3 x float> [[TMP1]])
+; CHECK-NEXT:    [[TMP3:%.*]] = call [[BITCAST_STRUCT_V3F32_F32_V3F32:%.*]] @[[BITCAST_STRUCT_V3F32_F32_V3F32_BODY:[a-zA-Z0-9_$\"\\.-]*[a-zA-Z_$\"\\.-][a-zA-Z0-9_$\"\\.-]*]](ptr addrspace(5) poison, <3 x float> [[TMP1]])
 ; CHECK-NEXT:    [[TMP4:%.*]] = extractvalue [[BITCAST_STRUCT_V3F32_F32_V3F32]] [[TMP3]], 0
 ; CHECK-NEXT:    store <4 x float> [[TMP4]], ptr addrspace(5) [[TMP0]], align 16
 ; CHECK-NEXT:    ret void
@@ -1084,7 +1084,7 @@ attributes #2 = { alwaysinline nounwind }
 ;
 ; CHECK-LABEL: define {{[^@]+}}@bitcast_struct_v3f32_f32_v4f32
 ; CHECK-SAME: (ptr addrspace(5) [[TMP0:%.*]], <4 x float> [[TMP1:%.*]]) #[[ATTR2]] {
-; CHECK-NEXT:    [[TMP3:%.*]] = call [[BITCAST_STRUCT_V3F32_F32_V4F32:%.*]] @bitcast_struct_v3f32_f32_v4f32.body(ptr addrspace(5) poison, <4 x float> [[TMP1]])
+; CHECK-NEXT:    [[TMP3:%.*]] = call [[BITCAST_STRUCT_V3F32_F32_V4F32:%.*]] @[[BITCAST_STRUCT_V3F32_F32_V4F32_BODY:[a-zA-Z0-9_$\"\\.-]*[a-zA-Z_$\"\\.-][a-zA-Z0-9_$\"\\.-]*]](ptr addrspace(5) poison, <4 x float> [[TMP1]])
 ; CHECK-NEXT:    [[TMP4:%.*]] = extractvalue [[BITCAST_STRUCT_V3F32_F32_V4F32]] [[TMP3]], 0
 ; CHECK-NEXT:    store <4 x float> [[TMP4]], ptr addrspace(5) [[TMP0]], align 16
 ; CHECK-NEXT:    ret void
@@ -1098,7 +1098,7 @@ attributes #2 = { alwaysinline nounwind }
 ;
 ; CHECK-LABEL: define {{[^@]+}}@bitcast_struct_i128_v4f32
 ; CHECK-SAME: (ptr addrspace(5) [[TMP0:%.*]], <4 x float> [[TMP1:%.*]]) #[[ATTR2]] {
-; CHECK-NEXT:    [[TMP3:%.*]] = call [[BITCAST_STRUCT_I128_V4F32:%.*]] @bitcast_struct_i128_v4f32.body(ptr addrspace(5) poison, <4 x float> [[TMP1]])
+; CHECK-NEXT:    [[TMP3:%.*]] = call [[BITCAST_STRUCT_I128_V4F32:%.*]] @[[BITCAST_STRUCT_I128_V4F32_BODY:[a-zA-Z0-9_$\"\\.-]*[a-zA-Z_$\"\\.-][a-zA-Z0-9_$\"\\.-]*]](ptr addrspace(5) poison, <4 x float> [[TMP1]])
 ; CHECK-NEXT:    [[TMP4:%.*]] = extractvalue [[BITCAST_STRUCT_I128_V4F32]] [[TMP3]], 0
 ; CHECK-NEXT:    store <4 x float> [[TMP4]], ptr addrspace(5) [[TMP0]], align 16
 ; CHECK-NEXT:    ret void
@@ -1112,7 +1112,7 @@ attributes #2 = { alwaysinline nounwind }
 ;
 ; CHECK-LABEL: define {{[^@]+}}@bitcast_array_v4i32_v4f32
 ; CHECK-SAME: (ptr addrspace(5) [[TMP0:%.*]], [4 x float] [[TMP1:%.*]]) #[[ATTR2]] {
-; CHECK-NEXT:    [[TMP3:%.*]] = call [[BITCAST_ARRAY_V4I32_V4F32:%.*]] @bitcast_array_v4i32_v4f32.body(ptr addrspace(5) poison, [4 x float] [[TMP1]])
+; CHECK-NEXT:    [[TMP3:%.*]] = call [[BITCAST_ARRAY_V4I32_V4F32:%.*]] @[[BITCAST_ARRAY_V4I32_V4F32_BODY:[a-zA-Z0-9_$\"\\.-]*[a-zA-Z_$\"\\.-][a-zA-Z0-9_$\"\\.-]*]](ptr addrspace(5) poison, [4 x float] [[TMP1]])
 ; CHECK-NEXT:    [[TMP4:%.*]] = extractvalue [[BITCAST_ARRAY_V4I32_V4F32]] [[TMP3]], 0
 ; CHECK-NEXT:    store [4 x float] [[TMP4]], ptr addrspace(5) [[TMP0]], align 4
 ; CHECK-NEXT:    ret void
@@ -1134,7 +1134,7 @@ attributes #2 = { alwaysinline nounwind }
 ;
 ; CHECK-LABEL: define {{[^@]+}}@multi_return_bitcast_struct_v3f32_v3f32
 ; CHECK-SAME: (i1 [[TMP0:%.*]], ptr addrspace(5) [[TMP1:%.*]], <3 x float> [[TMP2:%.*]]) #[[ATTR2]] {
-; CHECK-NEXT:    [[TMP4:%.*]] = call [[MULTI_RETURN_BITCAST_STRUCT_V3F32_V3F32:%.*]] @multi_return_bitcast_struct_v3f32_v3f32.body(i1 [[TMP0]], ptr addrspace(5) poison, <3 x float> [[TMP2]])
+; CHECK-NEXT:    [[TMP4:%.*]] = call [[MULTI_RETURN_BITCAST_STRUCT_V3F32_V3F32:%.*]] @[[MULTI_RETURN_BITCAST_STRUCT_V3F32_V3F32_BODY:[a-zA-Z0-9_$\"\\.-]*[a-zA-Z_$\"\\.-][a-zA-Z0-9_$\"\\.-]*]](i1 [[TMP0]], ptr addrspace(5) poison, <3 x float> [[TMP2]])
 ; CHECK-NEXT:    [[TMP5:%.*]] = extractvalue [[MULTI_RETURN_BITCAST_STRUCT_V3F32_V3F32]] [[TMP4]], 0
 ; CHECK-NEXT:    store <4 x float> [[TMP5]], ptr addrspace(5) [[TMP1]], align 16
 ; CHECK-NEXT:    ret void
@@ -1148,7 +1148,7 @@ attributes #2 = { alwaysinline nounwind }
 ;
 ; CHECK-LABEL: define {{[^@]+}}@bitcast_v3f32_struct_v3f32
 ; CHECK-SAME: (ptr addrspace(5) [[TMP0:%.*]], [[STRUCT_V3F32:%.*]] [[TMP1:%.*]]) #[[ATTR2]] {
-; CHECK-NEXT:    [[TMP3:%.*]] = call [[BITCAST_V3F32_STRUCT_V3F32:%.*]] @bitcast_v3f32_struct_v3f32.body(ptr addrspace(5) poison, [[STRUCT_V3F32]] [[TMP1]])
+; CHECK-NEXT:    [[TMP3:%.*]] = call [[BITCAST_V3F32_STRUCT_V3F32:%.*]] @[[BITCAST_V3F32_STRUCT_V3F32_BODY:[a-zA-Z0-9_$\"\\.-]*[a-zA-Z_$\"\\.-][a-zA-Z0-9_$\"\\.-]*]](ptr addrspace(5) poison, [[STRUCT_V3F32]] [[TMP1]])
 ; CHECK-NEXT:    [[TMP4:%.*]] = extractvalue [[BITCAST_V3F32_STRUCT_V3F32]] [[TMP3]], 0
 ; CHECK-NEXT:    store [[STRUCT_V3F32]] [[TMP4]], ptr addrspace(5) [[TMP0]], align 16
 ; CHECK-NEXT:    ret void
diff --git a/llvm/test/CodeGen/AMDGPU/s-barrier.ll b/llvm/test/CodeGen/AMDGPU/s-barrier.ll
index 83a077f7f74db..1dcc6a19c29d7 100644
--- a/llvm/test/CodeGen/AMDGPU/s-barrier.ll
+++ b/llvm/test/CodeGen/AMDGPU/s-barrier.ll
@@ -20,7 +20,6 @@ define void @func1() {
 ; GFX12-SDAG-NEXT:    s_mov_b32 m0, 3
 ; GFX12-SDAG-NEXT:    s_barrier_join m0
 ; GFX12-SDAG-NEXT:    s_barrier_wait 1
-; GFX12-SDAG-NEXT:    s_wait_alu 0xfffe
 ; GFX12-SDAG-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX12-GISEL-LABEL: func1:
@@ -35,7 +34,6 @@ define void @func1() {
 ; GFX12-GISEL-NEXT:    s_barrier_signal m0
 ; GFX12-GISEL-NEXT:    s_barrier_join 3
 ; GFX12-GISEL-NEXT:    s_barrier_wait 1
-; GFX12-GISEL-NEXT:    s_wait_alu 0xfffe
 ; GFX12-GISEL-NEXT:    s_setpc_b64 s[30:31]
     call void @llvm.amdgcn.s.barrier.signal.var(ptr addrspace(3) @bar3, i32 7)
     call void @llvm.amdgcn.s.barrier.join(ptr addrspace(3) @bar3)
@@ -57,7 +55,6 @@ define void @func2() {
 ; GFX12-SDAG-NEXT:    s_mov_b32 m0, 1
 ; GFX12-SDAG-NEXT:    s_barrier_join m0
 ; GFX12-SDAG-NEXT:    s_barrier_wait 1
-; GFX12-SDAG-NEXT:    s_wait_alu 0xfffe
 ; GFX12-SDAG-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX12-GISEL-LABEL: func2:
@@ -72,7 +69,6 @@ define void @func2() {
 ; GFX12-GISEL-NEXT:    s_barrier_signal m0
 ; GFX12-GISEL-NEXT:    s_barrier_join 1
 ; GFX12-GISEL-NEXT:    s_barrier_wait 1
-; GFX12-GISEL-NEXT:    s_wait_alu 0xfffe
 ; GFX12-GISEL-NEXT:    s_setpc_b64 s[30:31]
     call void @llvm.amdgcn.s.barrier.signal.var(ptr addrspace(3) @bar2, i32 7)
     call void @llvm.amdgcn.s.barrier.join(ptr addrspace(3) @bar2)
@@ -94,11 +90,10 @@ define amdgpu_kernel void @kernel1(ptr addrspace(1) %out, ptr addrspace(3) %in)
 ; GFX12-SDAG-NEXT:    s_mov_b32 s32, 0
 ; GFX12-SDAG-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-SDAG-NEXT:    s_lshr_b32 s2, s2, 4
-; GFX12-SDAG-NEXT:    s_wait_alu 0xfffe
+; GFX12-SDAG-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
 ; GFX12-SDAG-NEXT:    s_and_b32 s2, s2, 63
-; GFX12-SDAG-NEXT:    s_wait_alu 0xfffe
 ; GFX12-SDAG-NEXT:    s_or_b32 s3, 0x90000, s2
-; GFX12-SDAG-NEXT:    s_wait_alu 0xfffe
+; GFX12-SDAG-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
 ; GFX12-SDAG-NEXT:    s_mov_b32 m0, s3
 ; GFX12-SDAG-NEXT:    s_barrier_init m0
 ; GFX12-SDAG-NEXT:    s_mov_b32 m0, 0xc0002
@@ -117,16 +112,13 @@ define amdgpu_kernel void @kernel1(ptr addrspace(1) %out, ptr addrspace(3) %in)
 ; GFX12-SDAG-NEXT:    s_get_barrier_state s2, m0
 ; GFX12-SDAG-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-SDAG-NEXT:    s_getpc_b64 s[2:3]
-; GFX12-SDAG-NEXT:    s_wait_alu 0xfffe
 ; GFX12-SDAG-NEXT:    s_sext_i32_i16 s3, s3
-; GFX12-SDAG-NEXT:    s_add_co_u32 s2, s2, func1@gotpcrel32@lo+12
-; GFX12-SDAG-NEXT:    s_wait_alu 0xfffe
-; GFX12-SDAG-NEXT:    s_add_co_ci_u32 s3, s3, func1@gotpcrel32@hi+24
+; GFX12-SDAG-NEXT:    s_add_co_u32 s2, s2, func1@gotpcrel32@lo+8
+; GFX12-SDAG-NEXT:    s_add_co_ci_u32 s3, s3, func1@gotpcrel32@hi+16
 ; GFX12-SDAG-NEXT:    s_barrier_signal -1
 ; GFX12-SDAG-NEXT:    s_load_b64 s[2:3], s[2:3], 0x0
 ; GFX12-SDAG-NEXT:    s_barrier_wait -1
 ; GFX12-SDAG-NEXT:    s_wait_kmcnt 0x0
-; GFX12-SDAG-NEXT:    s_wait_alu 0xfffe
 ; GFX12-SDAG-NEXT:    s_swappc_b64 s[30:31], s[2:3]
 ; GFX12-SDAG-NEXT:    s_getpc_b64 s[2:3]
 ; GFX12-SDAG-NEXT:    s_wait_alu 0xfffe
@@ -136,7 +128,6 @@ define amdgpu_kernel void @kernel1(ptr addrspace(1) %out, ptr addrspace(3) %in)
 ; GFX12-SDAG-NEXT:    s_add_co_ci_u32 s3, s3, func2@gotpcrel32@hi+24
 ; GFX12-SDAG-NEXT:    s_load_b64 s[2:3], s[2:3], 0x0
 ; GFX12-SDAG-NEXT:    s_wait_kmcnt 0x0
-; GFX12-SDAG-NEXT:    s_wait_alu 0xfffe
 ; GFX12-SDAG-NEXT:    s_swappc_b64 s[30:31], s[2:3]
 ; GFX12-SDAG-NEXT:    s_get_barrier_state s0, -1
 ; GFX12-SDAG-NEXT:    s_endpgm
@@ -154,11 +145,10 @@ define amdgpu_kernel void @kernel1(ptr addrspace(1) %out, ptr addrspace(3) %in)
 ; GFX12-GISEL-NEXT:    s_mov_b32 s32, 0
 ; GFX12-GISEL-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-GISEL-NEXT:    s_lshr_b32 s0, s0, 4
-; GFX12-GISEL-NEXT:    s_wait_alu 0xfffe
+; GFX12-GISEL-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
 ; GFX12-GISEL-NEXT:    s_and_b32 s0, s0, 63
-; GFX12-GISEL-NEXT:    s_wait_alu 0xfffe
 ; GFX12-GISEL-NEXT:    s_or_b32 s1, s0, 0x90000
-; GFX12-GISEL-NEXT:    s_wait_alu 0xfffe
+; GFX12-GISEL-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
 ; GFX12-GISEL-NEXT:    s_mov_b32 m0, s1
 ; GFX12-GISEL-NEXT:    s_barrier_init m0
 ; GFX12-GISEL-NEXT:    s_mov_b32 m0, 0xc0002
@@ -178,16 +168,13 @@ define amdgpu_kernel void @kernel1(ptr addrspace(1) %out, ptr addrspace(3) %in)
 ; GFX12-GISEL-NEXT:    s_add_co_ci_u32 s9, s13, 0
 ; GFX12-GISEL-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-GISEL-NEXT:    s_getpc_b64 s[0:1]
-; GFX12-GISEL-NEXT:    s_wait_alu 0xfffe
 ; GFX12-GISEL-NEXT:    s_sext_i32_i16 s1, s1
-; GFX12-GISEL-NEXT:    s_add_co_u32 s0, s0, func1@gotpcrel32@lo+12
-; GFX12-GISEL-NEXT:    s_wait_alu 0xfffe
-; GFX12-GISEL-NEXT:    s_add_co_ci_u32 s1, s1, func1@gotpcrel32@hi+24
+; GFX12-GISEL-NEXT:    s_add_co_u32 s0, s0, func1@gotpcrel32@lo+8
+; GFX12-GISEL-NEXT:    s_add_co_ci_u32 s1, s1, func1@gotpcrel32@hi+16
 ; GFX12-GISEL-NEXT:    s_barrier_signal -1
 ; GFX12-GISEL-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
 ; GFX12-GISEL-NEXT:    s_barrier_wait -1
 ; GFX12-GISEL-NEXT:    s_wait_kmcnt 0x0
-; GFX12-GISEL-NEXT:    s_wait_alu 0xfffe
 ; GFX12-GISEL-NEXT:    s_swappc_b64 s[30:31], s[0:1]
 ; GFX12-GISEL-NEXT:    s_add_co_u32 s8, s12, 48
 ; GFX12-GISEL-NEXT:    s_add_co_ci_u32 s9, s13, 0
@@ -199,7 +186,6 @@ define amdgpu_kernel void @kernel1(ptr addrspace(1) %out, ptr addrspace(3) %in)
 ; GFX12-GISEL-NEXT:    s_add_co_ci_u32 s1, s1, func2@gotpcrel32@hi+24
 ; GFX12-GISEL-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
 ; GFX12-GISEL-NEXT:    s_wait_kmcnt 0x0
-; GFX12-GISEL-NEXT:    s_wait_alu 0xfffe
 ; GFX12-GISEL-NEXT:    s_swappc_b64 s[30:31], s[0:1]
 ; GFX12-GISEL-NEXT:    s_get_barrier_state s0, -1
 ; GFX12-GISEL-NEXT:    s_endpgm
@@ -226,11 +212,9 @@ define amdgpu_kernel void @kernel2(ptr addrspace(1) %out, ptr addrspace(3) %in)
 ; GFX12-SDAG:       ; %bb.0:
 ; GFX12-SDAG-NEXT:    s_mov_b64 s[10:11], s[6:7]
 ; GFX12-SDAG-NEXT:    s_getpc_b64 s[6:7]
-; GFX12-SDAG-NEXT:    s_wait_alu 0xfffe
 ; GFX12-SDAG-NEXT:    s_sext_i32_i16 s7, s7
-; GFX12-SDAG-NEXT:    s_add_co_u32 s6, s6, func2@gotpcrel32@lo+12
-; GFX12-SDAG-NEXT:    s_wait_alu 0xfffe
-; GFX12-SDAG-NEXT:    s_add_co_ci_u32 s7, s7, func2@gotpcrel32@hi+24
+; GFX12-SDAG-NEXT:    s_add_co_u32 s6, s6, func2@gotpcrel32@lo+8
+; GFX12-SDAG-NEXT:    s_add_co_ci_u32 s7, s7, func2@gotpcrel32@hi+16
 ; GFX12-SDAG-NEXT:    v_mov_b32_e32 v31, v0
 ; GFX12-SDAG-NEXT:    s_load_b64 s[12:13], s[6:7], 0x0
 ; GFX12-SDAG-NEXT:    s_mov_b32 m0, 0x70002
@@ -243,7 +227,6 @@ define amdgpu_kernel void @kernel2(ptr addrspace(1) %out, ptr addrspace(3) %in)
 ; GFX12-SDAG-NEXT:    s_mov_b32 s32, 0
 ; GFX12-SDAG-NEXT:    s_barrier_join m0
 ; GFX12-SDAG-NEXT:    s_barrier_wait 1
-; GFX12-SDAG-NEXT:    s_wait_alu 0xfffe
 ; GFX12-SDAG-NEXT:    s_swappc_b64 s[30:31], s[12:13]
 ; GFX12-SDAG-NEXT:    s_endpgm
 ;
@@ -252,11 +235,9 @@ define amdgpu_kernel void @kernel2(ptr addrspace(1) %out, ptr addrspace(3) %in)
 ; GFX12-GISEL-NEXT:    s_add_co_u32 s8, s4, 48
 ; GFX12-GISEL-NEXT:    s_add_co_ci_u32 s9, s5, 0
 ; GFX12-GISEL-NEXT:    s_getpc_b64 s[4:5]
-; GFX12-GISEL-NEXT:    s_wait_alu 0xfffe
 ; GFX12-GISEL-NEXT:    s_sext_i32_i16 s5, s5
-; GFX12-GISEL-NEXT:    s_add_co_u32 s4, s4, func2@gotpcrel32@lo+12
-; GFX12-GISEL-NEXT:    s_wait_alu 0xfffe
-; GFX12-GISEL-NEXT:    s_add_co_ci_u32 s5, s5, func2@gotpcrel32@hi+24
+; GFX12-GISEL-NEXT:    s_add_co_u32 s4, s4, func2@gotpcrel32@lo+8
+; GFX12-GISEL-NEXT:    s_add_co_ci_u32 s5, s5, func2@gotpcrel32@hi+16
 ; GFX12-GISEL-NEXT:    v_mov_b32_e32 v31, v0
 ; GFX12-GISEL-NEXT:    s_load_b64 s[12:13], s[4:5], 0x0
 ; GFX12-GISEL-NEXT:    s_mov_b64 s[10:11], s[6:7]
@@ -268,7 +249,6 @@ define amdgpu_kernel void @kernel2(ptr addrspace(1) %out, ptr addrspace(3) %in)
 ; GFX12-GISEL-NEXT:    s_barrier_signal m0
 ; GFX12-GISEL-NEXT:    s_barrier_join 2
 ; GFX12-GISEL-NEXT:    s_barrier_wait 1
-; GFX12-GISEL-NEXT:    s_wait_alu 0xfffe
 ; GFX12-GISEL-NEXT:    s_swappc_b64 s[30:31], s[12:13]
 ; GFX12-GISEL-NEXT:    s_endpgm
     call void @llvm.amdgcn.s.barrier.signal.var(ptr addrspace(3) @bar, i32 7)
diff --git a/llvm/test/CodeGen/AMDGPU/s-getpc-b64-remat.ll b/llvm/test/CodeGen/AMDGPU/s-getpc-b64-remat.ll
index b7aecca45def5..dc91a55e8f3c9 100644
--- a/llvm/test/CodeGen/AMDGPU/s-getpc-b64-remat.ll
+++ b/llvm/test/CodeGen/AMDGPU/s-getpc-b64-remat.ll
@@ -87,7 +87,6 @@ define void @test_remat_s_getpc_b64() {
 ; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_mov_b32 exec_lo, s0
 ; GFX12-NEXT:    s_wait_loadcnt 0x0
-; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 entry:
   %0 = tail call i64 @llvm.amdgcn.s.getpc()
diff --git a/llvm/test/CodeGen/AMDGPU/sched-assert-dead-def-subreg-use-other-subreg.mir b/llvm/test/CodeGen/AMDGPU/sched-assert-dead-def-subreg-use-other-subreg.mir
index 3f88f98e34371..c933fb0de5864 100644
--- a/llvm/test/CodeGen/AMDGPU/sched-assert-dead-def-subreg-use-other-subreg.mir
+++ b/llvm/test/CodeGen/AMDGPU/sched-assert-dead-def-subreg-use-other-subreg.mir
@@ -23,7 +23,7 @@ body:             |
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT:   undef [[V_MOV_B32_e32_:%[0-9]+]].sub3:vreg_512 = V_MOV_B32_e32 0, implicit $exec
   ; CHECK-NEXT:   [[V_MOV_B32_e32_1:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
-  ; CHECK-NEXT:   [[V_ADD_U32_e32_:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 0, [[V_MOV_B32_e32_1]], implicit $exec
+  ; CHECK-NEXT:   [[V_MOV_B32_e32_2:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
   ; CHECK-NEXT:   [[COPY:%[0-9]+]]:vreg_512 = COPY [[V_MOV_B32_e32_]]
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT: bb.1:
@@ -32,17 +32,17 @@ body:             |
   ; CHECK-NEXT:   BUFFER_STORE_DWORD_OFFEN [[V_MOV_B32_e32_]].sub3, undef %5:vgpr_32, $sgpr24_sgpr25_sgpr26_sgpr27, $sgpr32, 0, 0, 0, implicit $exec :: (store (s32), align 8, addrspace 5)
   ; CHECK-NEXT:   dead [[DS_READ_B32_gfx9_:%[0-9]+]]:vgpr_32 = DS_READ_B32_gfx9 undef %7:vgpr_32, 0, 0, implicit $exec
   ; CHECK-NEXT:   dead [[DS_READ_B64_gfx9_:%[0-9]+]]:vreg_64 = DS_READ_B64_gfx9 [[V_MOV_B32_e32_1]], 0, 0, implicit $exec
+  ; CHECK-NEXT:   [[V_ADD_U32_e32_:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 0, [[V_MOV_B32_e32_1]], implicit $exec
   ; CHECK-NEXT:   dead [[DS_READ_B128_gfx9_:%[0-9]+]]:vreg_128 = DS_READ_B128_gfx9 [[V_ADD_U32_e32_]], 0, 0, implicit $exec
   ; CHECK-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY [[COPY]].sub0
   ; CHECK-NEXT:   undef [[COPY2:%[0-9]+]].sub1:vreg_512 = COPY [[COPY]].sub1
   ; CHECK-NEXT:   INLINEASM &"", 1 /* sideeffect attdialect */, 851978 /* regdef:VGPR_16 */, def dead [[COPY1]], 851978 /* regdef:VGPR_16 */, def dead [[COPY]].sub1, 2147483657 /* reguse tiedto:$0 */, [[COPY1]], 2147549193 /* reguse tiedto:$1 */, [[COPY]].sub1
   ; CHECK-NEXT:   [[COPY2:%[0-9]+]].sub0:vreg_512 = COPY [[COPY]].sub0
   ; CHECK-NEXT:   [[COPY2:%[0-9]+]].sub3:vreg_512 = COPY [[COPY]].sub3
-  ; CHECK-NEXT:   [[V_MOV_B32_e32_2:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
-  ; CHECK-NEXT:   dead [[V_ADD_CO_U32_e32_:%[0-9]+]]:vgpr_32 = V_ADD_CO_U32_e32 4, [[V_MOV_B32_e32_2]], implicit-def dead $vcc, implicit $exec
   ; CHECK-NEXT:   [[COPY2:%[0-9]+]].sub2:vreg_512 = COPY undef [[V_MOV_B32_e32_1]]
   ; CHECK-NEXT:   [[COPY2:%[0-9]+]].sub5:vreg_512 = COPY undef [[V_MOV_B32_e32_1]]
   ; CHECK-NEXT:   [[COPY:%[0-9]+]]:vreg_512 = COPY [[COPY2]]
+  ; CHECK-NEXT:   dead [[V_ADD_CO_U32_e32_:%[0-9]+]]:vgpr_32 = V_ADD_CO_U32_e32 4, [[V_MOV_B32_e32_2]], implicit-def dead $vcc, implicit $exec
   ; CHECK-NEXT:   S_BRANCH %bb.1
   bb.0:
     liveins: $sgpr6_sgpr7
diff --git a/llvm/test/CodeGen/AMDGPU/sdiv.ll b/llvm/test/CodeGen/AMDGPU/sdiv.ll
index 57c54c4de7102..d06d9f97db71c 100644
--- a/llvm/test/CodeGen/AMDGPU/sdiv.ll
+++ b/llvm/test/CodeGen/AMDGPU/sdiv.ll
@@ -1594,20 +1594,20 @@ define amdgpu_kernel void @v_sdiv_i8(ptr addrspace(1) %out, ptr addrspace(1) %in
 define amdgpu_kernel void @v_sdiv_i23(ptr addrspace(1) %out, ptr addrspace(1) %in) {
 ; GCN-LABEL: v_sdiv_i23:
 ; GCN:       ; %bb.0:
-; GCN-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
-; GCN-NEXT:    s_mov_b32 s7, 0xf000
-; GCN-NEXT:    s_mov_b32 s6, -1
-; GCN-NEXT:    s_mov_b32 s10, s6
-; GCN-NEXT:    s_mov_b32 s11, s7
+; GCN-NEXT:    s_load_dwordx4 s[4:7], s[4:5], 0x9
+; GCN-NEXT:    s_mov_b32 s3, 0xf000
+; GCN-NEXT:    s_mov_b32 s2, -1
+; GCN-NEXT:    s_mov_b32 s10, s2
+; GCN-NEXT:    s_mov_b32 s11, s3
 ; GCN-NEXT:    s_waitcnt lgkmcnt(0)
-; GCN-NEXT:    s_mov_b32 s8, s2
-; GCN-NEXT:    s_mov_b32 s9, s3
+; GCN-NEXT:    s_mov_b32 s8, s6
+; GCN-NEXT:    s_mov_b32 s9, s7
 ; GCN-NEXT:    buffer_load_ubyte v0, off, s[8:11], 0 offset:2
 ; GCN-NEXT:    buffer_load_ubyte v1, off, s[8:11], 0 offset:6
 ; GCN-NEXT:    buffer_load_ushort v2, off, s[8:11], 0 offset:4
 ; GCN-NEXT:    buffer_load_ushort v3, off, s[8:11], 0
-; GCN-NEXT:    s_mov_b32 s4, s0
-; GCN-NEXT:    s_mov_b32 s5, s1
+; GCN-NEXT:    s_mov_b32 s0, s4
+; GCN-NEXT:    s_mov_b32 s1, s5
 ; GCN-NEXT:    s_waitcnt vmcnt(3)
 ; GCN-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
 ; GCN-NEXT:    s_waitcnt vmcnt(2)
@@ -1632,7 +1632,7 @@ define amdgpu_kernel void @v_sdiv_i23(ptr addrspace(1) %out, ptr addrspace(1) %i
 ; GCN-NEXT:    v_cndmask_b32_e32 v0, 0, v0, vcc
 ; GCN-NEXT:    v_add_i32_e32 v0, vcc, v0, v1
 ; GCN-NEXT:    v_bfe_i32 v0, v0, 0, 23
-; GCN-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], 0
 ; GCN-NEXT:    s_endpgm
 ;
 ; TONGA-LABEL: v_sdiv_i23:
@@ -1783,20 +1783,20 @@ define amdgpu_kernel void @v_sdiv_i23(ptr addrspace(1) %out, ptr addrspace(1) %i
 define amdgpu_kernel void @v_sdiv_i24(ptr addrspace(1) %out, ptr addrspace(1) %in) {
 ; GCN-LABEL: v_sdiv_i24:
 ; GCN:       ; %bb.0:
-; GCN-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
-; GCN-NEXT:    s_mov_b32 s7, 0xf000
-; GCN-NEXT:    s_mov_b32 s6, -1
-; GCN-NEXT:    s_mov_b32 s10, s6
-; GCN-NEXT:    s_mov_b32 s11, s7
+; GCN-NEXT:    s_load_dwordx4 s[4:7], s[4:5], 0x9
+; GCN-NEXT:    s_mov_b32 s3, 0xf000
+; GCN-NEXT:    s_mov_b32 s2, -1
+; GCN-NEXT:    s_mov_b32 s10, s2
+; GCN-NEXT:    s_mov_b32 s11, s3
 ; GCN-NEXT:    s_waitcnt lgkmcnt(0)
-; GCN-NEXT:    s_mov_b32 s8, s2
-; GCN-NEXT:    s_mov_b32 s9, s3
+; GCN-NEXT:    s_mov_b32 s8, s6
+; GCN-NEXT:    s_mov_b32 s9, s7
 ; GCN-NEXT:    buffer_load_sbyte v0, off, s[8:11], 0 offset:6
 ; GCN-NEXT:    buffer_load_ushort v1, off, s[8:11], 0 offset:4
 ; GCN-NEXT:    buffer_load_sbyte v2, off, s[8:11], 0 offset:2
 ; GCN-NEXT:    buffer_load_ushort v3, off, s[8:11], 0
-; GCN-NEXT:    s_mov_b32 s4, s0
-; GCN-NEXT:    s_mov_b32 s5, s1
+; GCN-NEXT:    s_mov_b32 s0, s4
+; GCN-NEXT:    s_mov_b32 s1, s5
 ; GCN-NEXT:    s_waitcnt vmcnt(3)
 ; GCN-NEXT:    v_lshlrev_b32_e32 v4, 16, v0
 ; GCN-NEXT:    s_waitcnt vmcnt(2)
@@ -1819,7 +1819,7 @@ define amdgpu_kernel void @v_sdiv_i24(ptr addrspace(1) %out, ptr addrspace(1) %i
 ; GCN-NEXT:    v_cndmask_b32_e32 v0, 0, v0, vcc
 ; GCN-NEXT:    v_add_i32_e32 v0, vcc, v0, v2
 ; GCN-NEXT:    v_bfe_i32 v0, v0, 0, 24
-; GCN-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], 0
 ; GCN-NEXT:    s_endpgm
 ;
 ; TONGA-LABEL: v_sdiv_i24:
diff --git a/llvm/test/CodeGen/AMDGPU/select-flags-to-fmin-fmax.ll b/llvm/test/CodeGen/AMDGPU/select-flags-to-fmin-fmax.ll
index 50a3336a7483c..9a168c133c552 100644
--- a/llvm/test/CodeGen/AMDGPU/select-flags-to-fmin-fmax.ll
+++ b/llvm/test/CodeGen/AMDGPU/select-flags-to-fmin-fmax.ll
@@ -30,6 +30,7 @@ define float @v_test_fmin_legacy_ule_f32_safe(float %a, float %b) {
 ; GFX12-NEXT:    s_wait_bvhcnt 0x0
 ; GFX12-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-NEXT:    v_cmp_ngt_f32_e32 vcc_lo, v0, v1
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_cndmask_b32_e32 v0, v1, v0, vcc_lo
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
   %cmp = fcmp ule float %a, %b
@@ -59,6 +60,7 @@ define float @v_test_fmin_legacy_ule_f32_nnan_flag(float %a, float %b) {
 ; GFX12-NEXT:    s_wait_bvhcnt 0x0
 ; GFX12-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-NEXT:    v_cmp_ngt_f32_e32 vcc_lo, v0, v1
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_cndmask_b32_e32 v0, v1, v0, vcc_lo
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
   %cmp = fcmp ule float %a, %b
@@ -88,6 +90,7 @@ define float @v_test_fmin_legacy_ule_f32_nsz_flag(float %a, float %b) {
 ; GFX12-NEXT:    s_wait_bvhcnt 0x0
 ; GFX12-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-NEXT:    v_cmp_ngt_f32_e32 vcc_lo, v0, v1
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_cndmask_b32_e32 v0, v1, v0, vcc_lo
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
   %cmp = fcmp ule float %a, %b
@@ -144,6 +147,7 @@ define float @v_test_fmax_legacy_uge_f32_safe(float %a, float %b) {
 ; GFX12-NEXT:    s_wait_bvhcnt 0x0
 ; GFX12-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-NEXT:    v_cmp_nlt_f32_e32 vcc_lo, v0, v1
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_cndmask_b32_e32 v0, v1, v0, vcc_lo
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
   %cmp = fcmp uge float %a, %b
@@ -173,6 +177,7 @@ define float @v_test_fmax_legacy_uge_f32_nnan_flag(float %a, float %b) {
 ; GFX12-NEXT:    s_wait_bvhcnt 0x0
 ; GFX12-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-NEXT:    v_cmp_nlt_f32_e32 vcc_lo, v0, v1
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_cndmask_b32_e32 v0, v1, v0, vcc_lo
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
   %cmp = fcmp uge float %a, %b
@@ -202,6 +207,7 @@ define float @v_test_fmax_legacy_uge_f32_nsz_flag(float %a, float %b) {
 ; GFX12-NEXT:    s_wait_bvhcnt 0x0
 ; GFX12-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-NEXT:    v_cmp_nlt_f32_e32 vcc_lo, v0, v1
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_cndmask_b32_e32 v0, v1, v0, vcc_lo
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
   %cmp = fcmp uge float %a, %b
@@ -261,8 +267,10 @@ define <2 x float> @v_test_fmin_legacy_ule_v2f32_safe(<2 x float> %a, <2 x float
 ; GFX12-NEXT:    s_wait_bvhcnt 0x0
 ; GFX12-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-NEXT:    v_cmp_ngt_f32_e32 vcc_lo, v0, v2
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc_lo
 ; GFX12-NEXT:    v_cmp_ngt_f32_e32 vcc_lo, v1, v3
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc_lo
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
   %cmp = fcmp ule <2 x float> %a, %b
@@ -295,8 +303,10 @@ define <2 x float> @v_test_fmin_legacy_ule_v2f32_nnan_flag(<2 x float> %a, <2 x
 ; GFX12-NEXT:    s_wait_bvhcnt 0x0
 ; GFX12-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-NEXT:    v_cmp_ngt_f32_e32 vcc_lo, v0, v2
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc_lo
 ; GFX12-NEXT:    v_cmp_ngt_f32_e32 vcc_lo, v1, v3
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc_lo
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
   %cmp = fcmp ule <2 x float> %a, %b
@@ -329,8 +339,10 @@ define <2 x float> @v_test_fmin_legacy_ule_v2f32_nsz_flag(<2 x float> %a, <2 x f
 ; GFX12-NEXT:    s_wait_bvhcnt 0x0
 ; GFX12-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-NEXT:    v_cmp_ngt_f32_e32 vcc_lo, v0, v2
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc_lo
 ; GFX12-NEXT:    v_cmp_ngt_f32_e32 vcc_lo, v1, v3
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc_lo
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
   %cmp = fcmp ule <2 x float> %a, %b
@@ -363,8 +375,10 @@ define <2 x float> @v_test_fmin_legacy_ule_v2f32_nnan_nsz_flag(<2 x float> %a, <
 ; GFX12-NEXT:    s_wait_bvhcnt 0x0
 ; GFX12-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-NEXT:    v_cmp_ngt_f32_e32 vcc_lo, v0, v2
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc_lo
 ; GFX12-NEXT:    v_cmp_ngt_f32_e32 vcc_lo, v1, v3
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc_lo
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
   %cmp = fcmp ule <2 x float> %a, %b
@@ -397,8 +411,10 @@ define <2 x float> @v_test_fmax_legacy_uge_v2f32_safe(<2 x float> %a, <2 x float
 ; GFX12-NEXT:    s_wait_bvhcnt 0x0
 ; GFX12-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-NEXT:    v_cmp_nlt_f32_e32 vcc_lo, v0, v2
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc_lo
 ; GFX12-NEXT:    v_cmp_nlt_f32_e32 vcc_lo, v1, v3
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc_lo
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
   %cmp = fcmp uge <2 x float> %a, %b
@@ -431,8 +447,10 @@ define <2 x float> @v_test_fmax_legacy_uge_v2f32_nnan_flag(<2 x float> %a, <2 x
 ; GFX12-NEXT:    s_wait_bvhcnt 0x0
 ; GFX12-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-NEXT:    v_cmp_nlt_f32_e32 vcc_lo, v0, v2
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc_lo
 ; GFX12-NEXT:    v_cmp_nlt_f32_e32 vcc_lo, v1, v3
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc_lo
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
   %cmp = fcmp uge <2 x float> %a, %b
@@ -465,8 +483,10 @@ define <2 x float> @v_test_fmax_legacy_uge_v2f32_nsz_flag(<2 x float> %a, <2 x f
 ; GFX12-NEXT:    s_wait_bvhcnt 0x0
 ; GFX12-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-NEXT:    v_cmp_nlt_f32_e32 vcc_lo, v0, v2
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc_lo
 ; GFX12-NEXT:    v_cmp_nlt_f32_e32 vcc_lo, v1, v3
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc_lo
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
   %cmp = fcmp uge <2 x float> %a, %b
@@ -499,8 +519,10 @@ define <2 x float> @v_test_fmax_legacy_uge_v2f32_nnan_nsz_flag(<2 x float> %a, <
 ; GFX12-NEXT:    s_wait_bvhcnt 0x0
 ; GFX12-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-NEXT:    v_cmp_nlt_f32_e32 vcc_lo, v0, v2
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc_lo
 ; GFX12-NEXT:    v_cmp_nlt_f32_e32 vcc_lo, v1, v3
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc_lo
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
   %cmp = fcmp uge <2 x float> %a, %b
@@ -534,6 +556,7 @@ define half @v_test_fmin_legacy_ule_f16_safe(half %a, half %b) {
 ; GFX12-NEXT:    s_wait_bvhcnt 0x0
 ; GFX12-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-NEXT:    v_cmp_ngt_f16_e32 vcc_lo, v0, v1
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_cndmask_b32_e32 v0, v1, v0, vcc_lo
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
   %cmp = fcmp ule half %a, %b
@@ -567,6 +590,7 @@ define half @v_test_fmin_legacy_ule_f16_nnan_flag(half %a, half %b) {
 ; GFX12-NEXT:    s_wait_bvhcnt 0x0
 ; GFX12-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-NEXT:    v_cmp_ngt_f16_e32 vcc_lo, v0, v1
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_cndmask_b32_e32 v0, v1, v0, vcc_lo
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
   %cmp = fcmp ule half %a, %b
@@ -600,6 +624,7 @@ define half @v_test_fmin_legacy_ule_f16_nsz_flag(half %a, half %b) {
 ; GFX12-NEXT:    s_wait_bvhcnt 0x0
 ; GFX12-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-NEXT:    v_cmp_ngt_f16_e32 vcc_lo, v0, v1
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_cndmask_b32_e32 v0, v1, v0, vcc_lo
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
   %cmp = fcmp ule half %a, %b
@@ -664,6 +689,7 @@ define half @v_test_fmax_legacy_uge_f16_safe(half %a, half %b) {
 ; GFX12-NEXT:    s_wait_bvhcnt 0x0
 ; GFX12-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-NEXT:    v_cmp_nlt_f16_e32 vcc_lo, v0, v1
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_cndmask_b32_e32 v0, v1, v0, vcc_lo
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
   %cmp = fcmp uge half %a, %b
@@ -697,6 +723,7 @@ define half @v_test_fmax_legacy_uge_f16_nnan_flag(half %a, half %b) {
 ; GFX12-NEXT:    s_wait_bvhcnt 0x0
 ; GFX12-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-NEXT:    v_cmp_nlt_f16_e32 vcc_lo, v0, v1
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_cndmask_b32_e32 v0, v1, v0, vcc_lo
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
   %cmp = fcmp uge half %a, %b
@@ -730,6 +757,7 @@ define half @v_test_fmax_legacy_uge_f16_nsz_flag(half %a, half %b) {
 ; GFX12-NEXT:    s_wait_bvhcnt 0x0
 ; GFX12-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-NEXT:    v_cmp_nlt_f16_e32 vcc_lo, v0, v1
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_cndmask_b32_e32 v0, v1, v0, vcc_lo
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
   %cmp = fcmp uge half %a, %b
@@ -806,11 +834,14 @@ define <2 x half> @v_test_fmin_legacy_ule_v2f16_safe(<2 x half> %a, <2 x half> %
 ; GFX12-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-NEXT:    v_lshrrev_b32_e32 v2, 16, v1
 ; GFX12-NEXT:    v_lshrrev_b32_e32 v3, 16, v0
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-NEXT:    v_cmp_ngt_f16_e32 vcc_lo, v3, v2
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_cndmask_b32_e32 v2, v2, v3, vcc_lo
 ; GFX12-NEXT:    v_cmp_ngt_f16_e32 vcc_lo, v0, v1
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_cndmask_b32_e32 v0, v1, v0, vcc_lo
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-NEXT:    v_perm_b32 v0, v2, v0, 0x5040100
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
   %cmp = fcmp ule <2 x half> %a, %b
@@ -856,11 +887,14 @@ define <2 x half> @v_test_fmin_legacy_ule_v2f16_nnan_flag(<2 x half> %a, <2 x ha
 ; GFX12-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-NEXT:    v_lshrrev_b32_e32 v2, 16, v1
 ; GFX12-NEXT:    v_lshrrev_b32_e32 v3, 16, v0
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-NEXT:    v_cmp_ngt_f16_e32 vcc_lo, v3, v2
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_cndmask_b32_e32 v2, v2, v3, vcc_lo
 ; GFX12-NEXT:    v_cmp_ngt_f16_e32 vcc_lo, v0, v1
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_cndmask_b32_e32 v0, v1, v0, vcc_lo
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-NEXT:    v_perm_b32 v0, v2, v0, 0x5040100
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
   %cmp = fcmp ule <2 x half> %a, %b
@@ -906,11 +940,14 @@ define <2 x half> @v_test_fmin_legacy_ule_v2f16_nsz_flag(<2 x half> %a, <2 x hal
 ; GFX12-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-NEXT:    v_lshrrev_b32_e32 v2, 16, v1
 ; GFX12-NEXT:    v_lshrrev_b32_e32 v3, 16, v0
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-NEXT:    v_cmp_ngt_f16_e32 vcc_lo, v3, v2
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_cndmask_b32_e32 v2, v2, v3, vcc_lo
 ; GFX12-NEXT:    v_cmp_ngt_f16_e32 vcc_lo, v0, v1
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_cndmask_b32_e32 v0, v1, v0, vcc_lo
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-NEXT:    v_perm_b32 v0, v2, v0, 0x5040100
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
   %cmp = fcmp ule <2 x half> %a, %b
@@ -992,11 +1029,14 @@ define <2 x half> @v_test_fmax_legacy_uge_v2f16_safe(<2 x half> %a, <2 x half> %
 ; GFX12-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-NEXT:    v_lshrrev_b32_e32 v2, 16, v1
 ; GFX12-NEXT:    v_lshrrev_b32_e32 v3, 16, v0
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-NEXT:    v_cmp_nlt_f16_e32 vcc_lo, v3, v2
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_cndmask_b32_e32 v2, v2, v3, vcc_lo
 ; GFX12-NEXT:    v_cmp_nlt_f16_e32 vcc_lo, v0, v1
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_cndmask_b32_e32 v0, v1, v0, vcc_lo
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-NEXT:    v_perm_b32 v0, v2, v0, 0x5040100
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
   %cmp = fcmp uge <2 x half> %a, %b
@@ -1042,11 +1082,14 @@ define <2 x half> @v_test_fmax_legacy_uge_v2f16_nnan_flag(<2 x half> %a, <2 x ha
 ; GFX12-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-NEXT:    v_lshrrev_b32_e32 v2, 16, v1
 ; GFX12-NEXT:    v_lshrrev_b32_e32 v3, 16, v0
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-NEXT:    v_cmp_nlt_f16_e32 vcc_lo, v3, v2
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_cndmask_b32_e32 v2, v2, v3, vcc_lo
 ; GFX12-NEXT:    v_cmp_nlt_f16_e32 vcc_lo, v0, v1
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_cndmask_b32_e32 v0, v1, v0, vcc_lo
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-NEXT:    v_perm_b32 v0, v2, v0, 0x5040100
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
   %cmp = fcmp uge <2 x half> %a, %b
@@ -1092,11 +1135,14 @@ define <2 x half> @v_test_fmax_legacy_uge_v2f16_nsz_flag(<2 x half> %a, <2 x hal
 ; GFX12-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-NEXT:    v_lshrrev_b32_e32 v2, 16, v1
 ; GFX12-NEXT:    v_lshrrev_b32_e32 v3, 16, v0
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-NEXT:    v_cmp_nlt_f16_e32 vcc_lo, v3, v2
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_cndmask_b32_e32 v2, v2, v3, vcc_lo
 ; GFX12-NEXT:    v_cmp_nlt_f16_e32 vcc_lo, v0, v1
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_cndmask_b32_e32 v0, v1, v0, vcc_lo
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-NEXT:    v_perm_b32 v0, v2, v0, 0x5040100
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
   %cmp = fcmp uge <2 x half> %a, %b
@@ -1197,14 +1243,18 @@ define <4 x half> @v_test_fmin_legacy_ule_v4f16_safe(<4 x half> %a, <4 x half> %
 ; GFX12-NEXT:    v_lshrrev_b32_e32 v5, 16, v1
 ; GFX12-NEXT:    v_lshrrev_b32_e32 v6, 16, v2
 ; GFX12-NEXT:    v_lshrrev_b32_e32 v7, 16, v0
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_3)
 ; GFX12-NEXT:    v_cmp_ngt_f16_e32 vcc_lo, v5, v4
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_cndmask_b32_e32 v4, v4, v5, vcc_lo
 ; GFX12-NEXT:    v_cmp_ngt_f16_e32 vcc_lo, v7, v6
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_cndmask_b32_e32 v5, v6, v7, vcc_lo
 ; GFX12-NEXT:    v_cmp_ngt_f16_e32 vcc_lo, v0, v2
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc_lo
 ; GFX12-NEXT:    v_cmp_ngt_f16_e32 vcc_lo, v1, v3
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc_lo
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
 ; GFX12-NEXT:    v_perm_b32 v0, v5, v0, 0x5040100
@@ -1272,14 +1322,18 @@ define <4 x half> @v_test_fmin_legacy_ule_v4f16_nnan_flag(<4 x half> %a, <4 x ha
 ; GFX12-NEXT:    v_lshrrev_b32_e32 v5, 16, v1
 ; GFX12-NEXT:    v_lshrrev_b32_e32 v6, 16, v2
 ; GFX12-NEXT:    v_lshrrev_b32_e32 v7, 16, v0
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_3)
 ; GFX12-NEXT:    v_cmp_ngt_f16_e32 vcc_lo, v5, v4
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_cndmask_b32_e32 v4, v4, v5, vcc_lo
 ; GFX12-NEXT:    v_cmp_ngt_f16_e32 vcc_lo, v7, v6
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_cndmask_b32_e32 v5, v6, v7, vcc_lo
 ; GFX12-NEXT:    v_cmp_ngt_f16_e32 vcc_lo, v0, v2
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc_lo
 ; GFX12-NEXT:    v_cmp_ngt_f16_e32 vcc_lo, v1, v3
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc_lo
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
 ; GFX12-NEXT:    v_perm_b32 v0, v5, v0, 0x5040100
@@ -1347,14 +1401,18 @@ define <4 x half> @v_test_fmin_legacy_ule_v4f16_nsz_flag(<4 x half> %a, <4 x hal
 ; GFX12-NEXT:    v_lshrrev_b32_e32 v5, 16, v1
 ; GFX12-NEXT:    v_lshrrev_b32_e32 v6, 16, v2
 ; GFX12-NEXT:    v_lshrrev_b32_e32 v7, 16, v0
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_3)
 ; GFX12-NEXT:    v_cmp_ngt_f16_e32 vcc_lo, v5, v4
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_cndmask_b32_e32 v4, v4, v5, vcc_lo
 ; GFX12-NEXT:    v_cmp_ngt_f16_e32 vcc_lo, v7, v6
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_cndmask_b32_e32 v5, v6, v7, vcc_lo
 ; GFX12-NEXT:    v_cmp_ngt_f16_e32 vcc_lo, v0, v2
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc_lo
 ; GFX12-NEXT:    v_cmp_ngt_f16_e32 vcc_lo, v1, v3
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc_lo
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
 ; GFX12-NEXT:    v_perm_b32 v0, v5, v0, 0x5040100
@@ -1470,14 +1528,18 @@ define <4 x half> @v_test_fmax_legacy_uge_v4f16_safe(<4 x half> %a, <4 x half> %
 ; GFX12-NEXT:    v_lshrrev_b32_e32 v5, 16, v1
 ; GFX12-NEXT:    v_lshrrev_b32_e32 v6, 16, v2
 ; GFX12-NEXT:    v_lshrrev_b32_e32 v7, 16, v0
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_3)
 ; GFX12-NEXT:    v_cmp_nlt_f16_e32 vcc_lo, v5, v4
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_cndmask_b32_e32 v4, v4, v5, vcc_lo
 ; GFX12-NEXT:    v_cmp_nlt_f16_e32 vcc_lo, v7, v6
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_cndmask_b32_e32 v5, v6, v7, vcc_lo
 ; GFX12-NEXT:    v_cmp_nlt_f16_e32 vcc_lo, v0, v2
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc_lo
 ; GFX12-NEXT:    v_cmp_nlt_f16_e32 vcc_lo, v1, v3
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc_lo
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
 ; GFX12-NEXT:    v_perm_b32 v0, v5, v0, 0x5040100
@@ -1545,14 +1607,18 @@ define <4 x half> @v_test_fmax_legacy_uge_v4f16_nnan_flag(<4 x half> %a, <4 x ha
 ; GFX12-NEXT:    v_lshrrev_b32_e32 v5, 16, v1
 ; GFX12-NEXT:    v_lshrrev_b32_e32 v6, 16, v2
 ; GFX12-NEXT:    v_lshrrev_b32_e32 v7, 16, v0
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_3)
 ; GFX12-NEXT:    v_cmp_nlt_f16_e32 vcc_lo, v5, v4
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_cndmask_b32_e32 v4, v4, v5, vcc_lo
 ; GFX12-NEXT:    v_cmp_nlt_f16_e32 vcc_lo, v7, v6
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_cndmask_b32_e32 v5, v6, v7, vcc_lo
 ; GFX12-NEXT:    v_cmp_nlt_f16_e32 vcc_lo, v0, v2
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc_lo
 ; GFX12-NEXT:    v_cmp_nlt_f16_e32 vcc_lo, v1, v3
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc_lo
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
 ; GFX12-NEXT:    v_perm_b32 v0, v5, v0, 0x5040100
@@ -1620,14 +1686,18 @@ define <4 x half> @v_test_fmax_legacy_uge_v4f16_nsz_flag(<4 x half> %a, <4 x hal
 ; GFX12-NEXT:    v_lshrrev_b32_e32 v5, 16, v1
 ; GFX12-NEXT:    v_lshrrev_b32_e32 v6, 16, v2
 ; GFX12-NEXT:    v_lshrrev_b32_e32 v7, 16, v0
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_3)
 ; GFX12-NEXT:    v_cmp_nlt_f16_e32 vcc_lo, v5, v4
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_cndmask_b32_e32 v4, v4, v5, vcc_lo
 ; GFX12-NEXT:    v_cmp_nlt_f16_e32 vcc_lo, v7, v6
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_cndmask_b32_e32 v5, v6, v7, vcc_lo
 ; GFX12-NEXT:    v_cmp_nlt_f16_e32 vcc_lo, v0, v2
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc_lo
 ; GFX12-NEXT:    v_cmp_nlt_f16_e32 vcc_lo, v1, v3
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc_lo
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
 ; GFX12-NEXT:    v_perm_b32 v0, v5, v0, 0x5040100
diff --git a/llvm/test/CodeGen/AMDGPU/set-inactive-wwm-overwrite.ll b/llvm/test/CodeGen/AMDGPU/set-inactive-wwm-overwrite.ll
index 6f841c88a6d8b..f60786c1bacbf 100644
--- a/llvm/test/CodeGen/AMDGPU/set-inactive-wwm-overwrite.ll
+++ b/llvm/test/CodeGen/AMDGPU/set-inactive-wwm-overwrite.ll
@@ -4,8 +4,8 @@
 define amdgpu_cs void @if_then(ptr addrspace(8) inreg %input, ptr addrspace(8) inreg %output, <3 x i32> %LocalInvocationId) {
 ; GCN-LABEL: if_then:
 ; GCN:       ; %bb.0: ; %.entry
-; GCN-NEXT:    v_mov_b32_e32 v3, 0
 ; GCN-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v0
+; GCN-NEXT:    v_mov_b32_e32 v3, 0
 ; GCN-NEXT:    s_and_saveexec_b32 s0, vcc_lo
 ; GCN-NEXT:  ; %bb.1: ; %.bb0
 ; GCN-NEXT:    v_mov_b32_e32 v3, 1
@@ -60,8 +60,8 @@ define amdgpu_cs void @if_then(ptr addrspace(8) inreg %input, ptr addrspace(8) i
 define amdgpu_cs void @if_else_vgpr_opt(ptr addrspace(8) inreg %input, ptr addrspace(8) inreg %output, <3 x i32> %LocalInvocationId) {
 ; GCN-LABEL: if_else_vgpr_opt:
 ; GCN:       ; %bb.0: ; %.entry
-; GCN-NEXT:    v_mov_b32_e32 v3, 0
 ; GCN-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v0
+; GCN-NEXT:    v_mov_b32_e32 v3, 0
 ; GCN-NEXT:    s_and_saveexec_b32 s0, vcc_lo
 ; GCN-NEXT:  ; %bb.1: ; %.bb0
 ; GCN-NEXT:    v_mov_b32_e32 v3, 1
diff --git a/llvm/test/CodeGen/AMDGPU/set_kill_i1_for_floation_point_comparison.ll b/llvm/test/CodeGen/AMDGPU/set_kill_i1_for_floation_point_comparison.ll
new file mode 100644
index 0000000000000..5f101c360f148
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/set_kill_i1_for_floation_point_comparison.ll
@@ -0,0 +1,41 @@
+; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 5
+; RUN: llc -mtriple=amdgcn -mcpu=gfx1200 -verify-machineinstrs -stop-after=amdgpu-isel < %s 2>&1 | FileCheck %s
+
+define amdgpu_ps void @_amdgpu_ps_main() {
+  ; CHECK-LABEL: name: _amdgpu_ps_main
+  ; CHECK: bb.0.entry:
+  ; CHECK-NEXT:   successors: %bb.1(0x40000000), %bb.2(0x40000000)
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
+  ; CHECK-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[S_MOV_B32_]], %subreg.sub0, [[S_MOV_B32_]], %subreg.sub1, [[S_MOV_B32_]], %subreg.sub2, [[S_MOV_B32_]], %subreg.sub3
+  ; CHECK-NEXT:   [[S_BUFFER_LOAD_DWORD_IMM:%[0-9]+]]:sreg_32_xm0_xexec = S_BUFFER_LOAD_DWORD_IMM killed [[REG_SEQUENCE]], 0, 0 :: (dereferenceable invariant load (s32))
+  ; CHECK-NEXT:   [[S_MOV_B32_1:%[0-9]+]]:sgpr_32 = S_MOV_B32 0
+  ; CHECK-NEXT:   nofpexcept S_CMP_NLT_F32 [[S_BUFFER_LOAD_DWORD_IMM]], [[S_MOV_B32_1]], implicit-def $scc, implicit $mode
+  ; CHECK-NEXT:   [[COPY:%[0-9]+]]:sreg_32_xm0_xexec = COPY $scc
+  ; CHECK-NEXT:   SI_KILL_I1_PSEUDO killed [[COPY]], 0, implicit-def dead $exec, implicit-def dead $scc, implicit $exec
+  ; CHECK-NEXT:   nofpexcept S_CMP_LT_F32 [[S_BUFFER_LOAD_DWORD_IMM]], [[S_MOV_B32_1]], implicit-def $scc, implicit $mode
+  ; CHECK-NEXT:   S_CBRANCH_SCC1 %bb.2, implicit $scc
+  ; CHECK-NEXT:   S_BRANCH %bb.1
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.1.bb1:
+  ; CHECK-NEXT:   successors: %bb.2(0x80000000)
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.2.bb2:
+  ; CHECK-NEXT:   S_ENDPGM 0
+entry:
+  %i = call i32 @llvm.amdgcn.s.buffer.load.i32(<4 x i32> zeroinitializer, i32 0, i32 0)
+  %i1 = bitcast i32 %i to float
+  %i2 = fcmp uge float %i1, 0.000000e+00
+  call void @llvm.amdgcn.kill(i1 %i2)
+  br i1 %i2, label %bb1, label %bb2
+
+bb1:                                               ; preds = %entry
+  %i3 = call i64 @llvm.amdgcn.s.getpc()
+  %i4 = and i64 %i3, 1
+  %i5 = inttoptr i64 %i4 to ptr addrspace(4)
+  %i6 = getelementptr i8, ptr addrspace(4) %i5, i64 32
+  br label %bb2
+
+bb2:                                              ; preds = %bb, %entry
+  ret void
+}
diff --git a/llvm/test/CodeGen/AMDGPU/shl.ll b/llvm/test/CodeGen/AMDGPU/shl.ll
index 7e7f4f5d19914..c9efeeefdf2d8 100644
--- a/llvm/test/CodeGen/AMDGPU/shl.ll
+++ b/llvm/test/CodeGen/AMDGPU/shl.ll
@@ -681,63 +681,30 @@ define amdgpu_kernel void @shl_v4i16(ptr addrspace(1) %out, ptr addrspace(1) %in
 ;
 ; EG-LABEL: shl_v4i16:
 ; EG:       ; %bb.0:
-; EG-NEXT:    ALU 3, @8, KC0[CB0:0-32], KC1[]
+; EG-NEXT:    ALU 2, @8, KC0[CB0:0-32], KC1[]
 ; EG-NEXT:    TEX 0 @6
-; EG-NEXT:    ALU 42, @12, KC0[CB0:0-32], KC1[]
-; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T10.XY, T0.X, 1
+; EG-NEXT:    ALU 10, @11, KC0[CB0:0-32], KC1[]
+; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T0.XY, T8.X, 1
 ; EG-NEXT:    CF_END
 ; EG-NEXT:    PAD
 ; EG-NEXT:    Fetch clause starting at 6:
-; EG-NEXT:     VTX_READ_128 T10.XYZW, T0.X, 0, #1
+; EG-NEXT:     VTX_READ_128 T8.XYZW, T0.X, 0, #1
 ; EG-NEXT:    ALU clause starting at 8:
-; EG-NEXT:     MOV T0.Y, T6.X,
-; EG-NEXT:     LSHL * T0.W, T0.X, literal.x, BS:VEC_120/SCL_212
+; EG-NEXT:     LSHL * T0.W, T0.X, literal.x,
 ; EG-NEXT:    3(4.203895e-45), 0(0.000000e+00)
 ; EG-NEXT:     ADD_INT * T0.X, KC0[2].Z, PV.W,
-; EG-NEXT:    ALU clause starting at 12:
-; EG-NEXT:     AND_INT * T1.W, T10.Z, literal.x,
-; EG-NEXT:    65535(9.183409e-41), 0(0.000000e+00)
-; EG-NEXT:     LSHL * T1.W, T10.X, PV.W,
-; EG-NEXT:     AND_INT T1.W, PV.W, literal.x,
-; EG-NEXT:     AND_INT * T2.W, T0.Y, literal.y,
-; EG-NEXT:    65535(9.183409e-41), -65536(nan)
-; EG-NEXT:     OR_INT * T1.W, PS, PV.W,
-; EG-NEXT:     MOV * T6.X, PV.W,
-; EG-NEXT:     MOV T0.X, PV.X,
-; EG-NEXT:     LSHR T1.W, T10.Z, literal.x,
-; EG-NEXT:     LSHR * T2.W, T10.X, literal.x,
-; EG-NEXT:    16(2.242078e-44), 0(0.000000e+00)
-; EG-NEXT:     LSHL T1.W, PS, PV.W,
-; EG-NEXT:     AND_INT * T2.W, PV.X, literal.x,
-; EG-NEXT:    65535(9.183409e-41), 0(0.000000e+00)
-; EG-NEXT:     LSHL * T1.W, PV.W, literal.x,
+; EG-NEXT:    ALU clause starting at 11:
+; EG-NEXT:     LSHR T1.W, T8.Z, literal.x,
+; EG-NEXT:     LSHR * T2.W, T8.X, literal.x,
 ; EG-NEXT:    16(2.242078e-44), 0(0.000000e+00)
-; EG-NEXT:     OR_INT * T1.W, T2.W, PV.W,
-; EG-NEXT:     MOV T6.X, PV.W,
-; EG-NEXT:     MOV * T0.X, T7.X,
-; EG-NEXT:     AND_INT * T1.W, T10.W, literal.x,
-; EG-NEXT:    65535(9.183409e-41), 0(0.000000e+00)
-; EG-NEXT:     LSHL T1.W, T10.Y, PV.W,
-; EG-NEXT:     AND_INT * T2.W, T0.X, literal.x,
-; EG-NEXT:    -65536(nan), 0(0.000000e+00)
-; EG-NEXT:     AND_INT * T1.W, PV.W, literal.x,
+; EG-NEXT:     LSHL T0.Y, PS, PV.W,
+; EG-NEXT:     AND_INT T1.W, T8.Z, literal.x,
+; EG-NEXT:     AND_INT * T2.W, T8.X, literal.x,
 ; EG-NEXT:    65535(9.183409e-41), 0(0.000000e+00)
-; EG-NEXT:     OR_INT * T1.W, T2.W, PV.W,
-; EG-NEXT:     MOV * T7.X, PV.W,
-; EG-NEXT:     MOV T0.X, PV.X,
-; EG-NEXT:     LSHR T1.W, T10.W, literal.x,
-; EG-NEXT:     LSHR * T2.W, T10.Y, literal.x,
-; EG-NEXT:    16(2.242078e-44), 0(0.000000e+00)
-; EG-NEXT:     LSHL * T1.W, PS, PV.W,
-; EG-NEXT:     AND_INT T0.Z, T0.X, literal.x,
-; EG-NEXT:     LSHL T1.W, PV.W, literal.y,
+; EG-NEXT:     LSHL T0.X, PS, PV.W,
 ; EG-NEXT:     ADD_INT * T0.W, KC0[2].Y, T0.W,
-; EG-NEXT:    65535(9.183409e-41), 16(2.242078e-44)
-; EG-NEXT:     LSHR T0.X, PS, literal.x,
-; EG-NEXT:     OR_INT * T10.Y, PV.Z, PV.W,
+; EG-NEXT:     LSHR * T8.X, PV.W, literal.x,
 ; EG-NEXT:    2(2.802597e-45), 0(0.000000e+00)
-; EG-NEXT:     MOV T7.X, PV.Y,
-; EG-NEXT:     MOV * T10.X, T6.X,
   %tid = call i32 @llvm.amdgcn.workitem.id.x() #0
   %gep = getelementptr inbounds <4 x i16>, ptr addrspace(1) %in, i32 %tid
   %gep.out = getelementptr inbounds <4 x i16>, ptr addrspace(1) %out, i32 %tid
diff --git a/llvm/test/CodeGen/AMDGPU/shufflevector.v4f32.v2f32.ll b/llvm/test/CodeGen/AMDGPU/shufflevector.v4f32.v2f32.ll
index fafdac7ffac62..9e3c044a76295 100644
--- a/llvm/test/CodeGen/AMDGPU/shufflevector.v4f32.v2f32.ll
+++ b/llvm/test/CodeGen/AMDGPU/shufflevector.v4f32.v2f32.ll
@@ -176,8 +176,7 @@ define void @v_shuffle_v4f32_v2f32__3_0_u_u(ptr addrspace(1) inreg %ptr) {
 ; GFX90A-NEXT:    ;;#ASMSTART
 ; GFX90A-NEXT:    ; def v[2:3]
 ; GFX90A-NEXT:    ;;#ASMEND
-; GFX90A-NEXT:    v_mov_b32_e32 v0, v1
-; GFX90A-NEXT:    v_mov_b32_e32 v1, v2
+; GFX90A-NEXT:    v_pk_mov_b32 v[0:1], v[2:3], v[0:1] op_sel:[1,0]
 ; GFX90A-NEXT:    global_store_dwordx4 v4, v[0:3], s[16:17]
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
 ; GFX90A-NEXT:    s_setpc_b64 s[30:31]
@@ -192,8 +191,8 @@ define void @v_shuffle_v4f32_v2f32__3_0_u_u(ptr addrspace(1) inreg %ptr) {
 ; GFX940-NEXT:    ;;#ASMSTART
 ; GFX940-NEXT:    ; def v[2:3]
 ; GFX940-NEXT:    ;;#ASMEND
-; GFX940-NEXT:    v_mov_b32_e32 v0, v1
-; GFX940-NEXT:    v_mov_b32_e32 v1, v2
+; GFX940-NEXT:    s_nop 0
+; GFX940-NEXT:    v_pk_mov_b32 v[0:1], v[2:3], v[0:1] op_sel:[1,0]
 ; GFX940-NEXT:    global_store_dwordx4 v4, v[0:3], s[0:1] sc0 sc1
 ; GFX940-NEXT:    s_waitcnt vmcnt(0)
 ; GFX940-NEXT:    s_setpc_b64 s[30:31]
@@ -566,16 +565,15 @@ define void @v_shuffle_v4f32_v2f32__3_3_3_0(ptr addrspace(1) inreg %ptr) {
 ; GFX90A:       ; %bb.0:
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX90A-NEXT:    ;;#ASMSTART
-; GFX90A-NEXT:    ; def v[0:1]
+; GFX90A-NEXT:    ; def v[2:3]
 ; GFX90A-NEXT:    ;;#ASMEND
-; GFX90A-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX90A-NEXT:    ;;#ASMSTART
-; GFX90A-NEXT:    ; def v[4:5]
+; GFX90A-NEXT:    ; def v[0:1]
 ; GFX90A-NEXT:    ;;#ASMEND
+; GFX90A-NEXT:    v_mov_b32_e32 v4, 0
+; GFX90A-NEXT:    v_pk_mov_b32 v[2:3], v[0:1], v[2:3] op_sel:[1,0]
 ; GFX90A-NEXT:    v_mov_b32_e32 v0, v1
-; GFX90A-NEXT:    v_mov_b32_e32 v2, v1
-; GFX90A-NEXT:    v_mov_b32_e32 v3, v4
-; GFX90A-NEXT:    global_store_dwordx4 v6, v[0:3], s[16:17]
+; GFX90A-NEXT:    global_store_dwordx4 v4, v[0:3], s[16:17]
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
 ; GFX90A-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -583,16 +581,15 @@ define void @v_shuffle_v4f32_v2f32__3_3_3_0(ptr addrspace(1) inreg %ptr) {
 ; GFX940:       ; %bb.0:
 ; GFX940-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX940-NEXT:    ;;#ASMSTART
-; GFX940-NEXT:    ; def v[0:1]
+; GFX940-NEXT:    ; def v[2:3]
 ; GFX940-NEXT:    ;;#ASMEND
-; GFX940-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX940-NEXT:    ;;#ASMSTART
-; GFX940-NEXT:    ; def v[4:5]
+; GFX940-NEXT:    ; def v[0:1]
 ; GFX940-NEXT:    ;;#ASMEND
+; GFX940-NEXT:    v_mov_b32_e32 v4, 0
+; GFX940-NEXT:    v_pk_mov_b32 v[2:3], v[0:1], v[2:3] op_sel:[1,0]
 ; GFX940-NEXT:    v_mov_b32_e32 v0, v1
-; GFX940-NEXT:    v_mov_b32_e32 v2, v1
-; GFX940-NEXT:    v_mov_b32_e32 v3, v4
-; GFX940-NEXT:    global_store_dwordx4 v6, v[0:3], s[0:1] sc0 sc1
+; GFX940-NEXT:    global_store_dwordx4 v4, v[0:3], s[0:1] sc0 sc1
 ; GFX940-NEXT:    s_waitcnt vmcnt(0)
 ; GFX940-NEXT:    s_setpc_b64 s[30:31]
   %vec0 = call <2 x float> asm "; def $0", "=v"()
@@ -675,31 +672,26 @@ define void @v_shuffle_v4f32_v2f32__3_3_3_2(ptr addrspace(1) inreg %ptr) {
 ; GFX90A-LABEL: v_shuffle_v4f32_v2f32__3_3_3_2:
 ; GFX90A:       ; %bb.0:
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX90A-NEXT:    ;;#ASMSTART
-; GFX90A-NEXT:    ; def v[4:5]
+; GFX90A-NEXT:    ; def v[0:1]
 ; GFX90A-NEXT:    ;;#ASMEND
-; GFX90A-NEXT:    v_mov_b32_e32 v0, v5
-; GFX90A-NEXT:    v_mov_b32_e32 v1, v5
-; GFX90A-NEXT:    v_mov_b32_e32 v2, v5
-; GFX90A-NEXT:    v_mov_b32_e32 v3, v4
-; GFX90A-NEXT:    global_store_dwordx4 v6, v[0:3], s[16:17]
+; GFX90A-NEXT:    v_mov_b32_e32 v4, 0
+; GFX90A-NEXT:    v_pk_mov_b32 v[2:3], v[0:1], v[0:1] op_sel:[1,0]
+; GFX90A-NEXT:    v_mov_b32_e32 v0, v1
+; GFX90A-NEXT:    global_store_dwordx4 v4, v[0:3], s[16:17]
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
 ; GFX90A-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: v_shuffle_v4f32_v2f32__3_3_3_2:
 ; GFX940:       ; %bb.0:
 ; GFX940-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX940-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX940-NEXT:    ;;#ASMSTART
-; GFX940-NEXT:    ; def v[4:5]
+; GFX940-NEXT:    ; def v[0:1]
 ; GFX940-NEXT:    ;;#ASMEND
-; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_mov_b32_e32 v0, v5
-; GFX940-NEXT:    v_mov_b32_e32 v1, v5
-; GFX940-NEXT:    v_mov_b32_e32 v2, v5
-; GFX940-NEXT:    v_mov_b32_e32 v3, v4
-; GFX940-NEXT:    global_store_dwordx4 v6, v[0:3], s[0:1] sc0 sc1
+; GFX940-NEXT:    v_mov_b32_e32 v4, 0
+; GFX940-NEXT:    v_pk_mov_b32 v[2:3], v[0:1], v[0:1] op_sel:[1,0]
+; GFX940-NEXT:    v_mov_b32_e32 v0, v1
+; GFX940-NEXT:    global_store_dwordx4 v4, v[0:3], s[0:1] sc0 sc1
 ; GFX940-NEXT:    s_waitcnt vmcnt(0)
 ; GFX940-NEXT:    s_setpc_b64 s[30:31]
   %vec0 = call <2 x float> asm "; def $0", "=v"()
@@ -873,8 +865,7 @@ define void @v_shuffle_v4f32_v2f32__1_0_0_0(ptr addrspace(1) inreg %ptr) {
 ; GFX90A-NEXT:    ; def v[2:3]
 ; GFX90A-NEXT:    ;;#ASMEND
 ; GFX90A-NEXT:    v_mov_b32_e32 v4, 0
-; GFX90A-NEXT:    v_mov_b32_e32 v0, v3
-; GFX90A-NEXT:    v_mov_b32_e32 v1, v2
+; GFX90A-NEXT:    v_pk_mov_b32 v[0:1], v[2:3], v[2:3] op_sel:[1,0]
 ; GFX90A-NEXT:    v_mov_b32_e32 v3, v2
 ; GFX90A-NEXT:    global_store_dwordx4 v4, v[0:3], s[16:17]
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
@@ -887,8 +878,7 @@ define void @v_shuffle_v4f32_v2f32__1_0_0_0(ptr addrspace(1) inreg %ptr) {
 ; GFX940-NEXT:    ; def v[2:3]
 ; GFX940-NEXT:    ;;#ASMEND
 ; GFX940-NEXT:    v_mov_b32_e32 v4, 0
-; GFX940-NEXT:    v_mov_b32_e32 v0, v3
-; GFX940-NEXT:    v_mov_b32_e32 v1, v2
+; GFX940-NEXT:    v_pk_mov_b32 v[0:1], v[2:3], v[2:3] op_sel:[1,0]
 ; GFX940-NEXT:    v_mov_b32_e32 v3, v2
 ; GFX940-NEXT:    global_store_dwordx4 v4, v[0:3], s[0:1] sc0 sc1
 ; GFX940-NEXT:    s_waitcnt vmcnt(0)
@@ -972,8 +962,7 @@ define void @v_shuffle_v4f32_v2f32__3_0_0_0(ptr addrspace(1) inreg %ptr) {
 ; GFX90A-NEXT:    ; def v[0:1]
 ; GFX90A-NEXT:    ;;#ASMEND
 ; GFX90A-NEXT:    v_mov_b32_e32 v4, 0
-; GFX90A-NEXT:    v_mov_b32_e32 v0, v1
-; GFX90A-NEXT:    v_mov_b32_e32 v1, v2
+; GFX90A-NEXT:    v_pk_mov_b32 v[0:1], v[0:1], v[2:3] op_sel:[1,0]
 ; GFX90A-NEXT:    v_mov_b32_e32 v3, v2
 ; GFX90A-NEXT:    global_store_dwordx4 v4, v[0:3], s[16:17]
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
@@ -989,8 +978,7 @@ define void @v_shuffle_v4f32_v2f32__3_0_0_0(ptr addrspace(1) inreg %ptr) {
 ; GFX940-NEXT:    ; def v[0:1]
 ; GFX940-NEXT:    ;;#ASMEND
 ; GFX940-NEXT:    v_mov_b32_e32 v4, 0
-; GFX940-NEXT:    v_mov_b32_e32 v0, v1
-; GFX940-NEXT:    v_mov_b32_e32 v1, v2
+; GFX940-NEXT:    v_pk_mov_b32 v[0:1], v[0:1], v[2:3] op_sel:[1,0]
 ; GFX940-NEXT:    v_mov_b32_e32 v3, v2
 ; GFX940-NEXT:    global_store_dwordx4 v4, v[0:3], s[0:1] sc0 sc1
 ; GFX940-NEXT:    s_waitcnt vmcnt(0)
@@ -1138,14 +1126,13 @@ define void @v_shuffle_v4f32_v2f32__3_2_0_0(ptr addrspace(1) inreg %ptr) {
 ; GFX90A-NEXT:    ;;#ASMSTART
 ; GFX90A-NEXT:    ; def v[2:3]
 ; GFX90A-NEXT:    ;;#ASMEND
-; GFX90A-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX90A-NEXT:    ;;#ASMSTART
-; GFX90A-NEXT:    ; def v[4:5]
+; GFX90A-NEXT:    ; def v[0:1]
 ; GFX90A-NEXT:    ;;#ASMEND
-; GFX90A-NEXT:    v_mov_b32_e32 v0, v5
-; GFX90A-NEXT:    v_mov_b32_e32 v1, v4
+; GFX90A-NEXT:    v_mov_b32_e32 v4, 0
+; GFX90A-NEXT:    v_pk_mov_b32 v[0:1], v[0:1], v[0:1] op_sel:[1,0]
 ; GFX90A-NEXT:    v_mov_b32_e32 v3, v2
-; GFX90A-NEXT:    global_store_dwordx4 v6, v[0:3], s[16:17]
+; GFX90A-NEXT:    global_store_dwordx4 v4, v[0:3], s[16:17]
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
 ; GFX90A-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -1155,14 +1142,13 @@ define void @v_shuffle_v4f32_v2f32__3_2_0_0(ptr addrspace(1) inreg %ptr) {
 ; GFX940-NEXT:    ;;#ASMSTART
 ; GFX940-NEXT:    ; def v[2:3]
 ; GFX940-NEXT:    ;;#ASMEND
-; GFX940-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX940-NEXT:    ;;#ASMSTART
-; GFX940-NEXT:    ; def v[4:5]
+; GFX940-NEXT:    ; def v[0:1]
 ; GFX940-NEXT:    ;;#ASMEND
+; GFX940-NEXT:    v_mov_b32_e32 v4, 0
+; GFX940-NEXT:    v_pk_mov_b32 v[0:1], v[0:1], v[0:1] op_sel:[1,0]
 ; GFX940-NEXT:    v_mov_b32_e32 v3, v2
-; GFX940-NEXT:    v_mov_b32_e32 v0, v5
-; GFX940-NEXT:    v_mov_b32_e32 v1, v4
-; GFX940-NEXT:    global_store_dwordx4 v6, v[0:3], s[0:1] sc0 sc1
+; GFX940-NEXT:    global_store_dwordx4 v4, v[0:3], s[0:1] sc0 sc1
 ; GFX940-NEXT:    s_waitcnt vmcnt(0)
 ; GFX940-NEXT:    s_setpc_b64 s[30:31]
   %vec0 = call <2 x float> asm "; def $0", "=v"()
@@ -1303,16 +1289,15 @@ define void @v_shuffle_v4f32_v2f32__3_3_1_0(ptr addrspace(1) inreg %ptr) {
 ; GFX90A:       ; %bb.0:
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX90A-NEXT:    ;;#ASMSTART
-; GFX90A-NEXT:    ; def v[0:1]
+; GFX90A-NEXT:    ; def v[2:3]
 ; GFX90A-NEXT:    ;;#ASMEND
-; GFX90A-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX90A-NEXT:    ;;#ASMSTART
-; GFX90A-NEXT:    ; def v[4:5]
+; GFX90A-NEXT:    ; def v[0:1]
 ; GFX90A-NEXT:    ;;#ASMEND
+; GFX90A-NEXT:    v_mov_b32_e32 v4, 0
+; GFX90A-NEXT:    v_pk_mov_b32 v[2:3], v[2:3], v[2:3] op_sel:[1,0]
 ; GFX90A-NEXT:    v_mov_b32_e32 v0, v1
-; GFX90A-NEXT:    v_mov_b32_e32 v2, v5
-; GFX90A-NEXT:    v_mov_b32_e32 v3, v4
-; GFX90A-NEXT:    global_store_dwordx4 v6, v[0:3], s[16:17]
+; GFX90A-NEXT:    global_store_dwordx4 v4, v[0:3], s[16:17]
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
 ; GFX90A-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -1320,16 +1305,15 @@ define void @v_shuffle_v4f32_v2f32__3_3_1_0(ptr addrspace(1) inreg %ptr) {
 ; GFX940:       ; %bb.0:
 ; GFX940-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX940-NEXT:    ;;#ASMSTART
-; GFX940-NEXT:    ; def v[0:1]
+; GFX940-NEXT:    ; def v[2:3]
 ; GFX940-NEXT:    ;;#ASMEND
-; GFX940-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX940-NEXT:    ;;#ASMSTART
-; GFX940-NEXT:    ; def v[4:5]
+; GFX940-NEXT:    ; def v[0:1]
 ; GFX940-NEXT:    ;;#ASMEND
+; GFX940-NEXT:    v_mov_b32_e32 v4, 0
+; GFX940-NEXT:    v_pk_mov_b32 v[2:3], v[2:3], v[2:3] op_sel:[1,0]
 ; GFX940-NEXT:    v_mov_b32_e32 v0, v1
-; GFX940-NEXT:    v_mov_b32_e32 v2, v5
-; GFX940-NEXT:    v_mov_b32_e32 v3, v4
-; GFX940-NEXT:    global_store_dwordx4 v6, v[0:3], s[0:1] sc0 sc1
+; GFX940-NEXT:    global_store_dwordx4 v4, v[0:3], s[0:1] sc0 sc1
 ; GFX940-NEXT:    s_waitcnt vmcnt(0)
 ; GFX940-NEXT:    s_setpc_b64 s[30:31]
   %vec0 = call <2 x float> asm "; def $0", "=v"()
@@ -1723,8 +1707,7 @@ define void @v_shuffle_v4f32_v2f32__3_0_1_1(ptr addrspace(1) inreg %ptr) {
 ; GFX90A-NEXT:    ; def v[0:1]
 ; GFX90A-NEXT:    ;;#ASMEND
 ; GFX90A-NEXT:    v_mov_b32_e32 v4, 0
-; GFX90A-NEXT:    v_mov_b32_e32 v0, v1
-; GFX90A-NEXT:    v_mov_b32_e32 v1, v2
+; GFX90A-NEXT:    v_pk_mov_b32 v[0:1], v[0:1], v[2:3] op_sel:[1,0]
 ; GFX90A-NEXT:    v_mov_b32_e32 v2, v3
 ; GFX90A-NEXT:    global_store_dwordx4 v4, v[0:3], s[16:17]
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
@@ -1740,8 +1723,7 @@ define void @v_shuffle_v4f32_v2f32__3_0_1_1(ptr addrspace(1) inreg %ptr) {
 ; GFX940-NEXT:    ; def v[0:1]
 ; GFX940-NEXT:    ;;#ASMEND
 ; GFX940-NEXT:    v_mov_b32_e32 v4, 0
-; GFX940-NEXT:    v_mov_b32_e32 v0, v1
-; GFX940-NEXT:    v_mov_b32_e32 v1, v2
+; GFX940-NEXT:    v_pk_mov_b32 v[0:1], v[0:1], v[2:3] op_sel:[1,0]
 ; GFX940-NEXT:    v_mov_b32_e32 v2, v3
 ; GFX940-NEXT:    global_store_dwordx4 v4, v[0:3], s[0:1] sc0 sc1
 ; GFX940-NEXT:    s_waitcnt vmcnt(0)
@@ -1776,14 +1758,13 @@ define void @v_shuffle_v4f32_v2f32__3_2_1_1(ptr addrspace(1) inreg %ptr) {
 ; GFX90A-NEXT:    ;;#ASMSTART
 ; GFX90A-NEXT:    ; def v[2:3]
 ; GFX90A-NEXT:    ;;#ASMEND
-; GFX90A-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX90A-NEXT:    ;;#ASMSTART
-; GFX90A-NEXT:    ; def v[4:5]
+; GFX90A-NEXT:    ; def v[0:1]
 ; GFX90A-NEXT:    ;;#ASMEND
-; GFX90A-NEXT:    v_mov_b32_e32 v0, v5
-; GFX90A-NEXT:    v_mov_b32_e32 v1, v4
+; GFX90A-NEXT:    v_mov_b32_e32 v4, 0
+; GFX90A-NEXT:    v_pk_mov_b32 v[0:1], v[0:1], v[0:1] op_sel:[1,0]
 ; GFX90A-NEXT:    v_mov_b32_e32 v2, v3
-; GFX90A-NEXT:    global_store_dwordx4 v6, v[0:3], s[16:17]
+; GFX90A-NEXT:    global_store_dwordx4 v4, v[0:3], s[16:17]
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
 ; GFX90A-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -1793,14 +1774,13 @@ define void @v_shuffle_v4f32_v2f32__3_2_1_1(ptr addrspace(1) inreg %ptr) {
 ; GFX940-NEXT:    ;;#ASMSTART
 ; GFX940-NEXT:    ; def v[2:3]
 ; GFX940-NEXT:    ;;#ASMEND
-; GFX940-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX940-NEXT:    ;;#ASMSTART
-; GFX940-NEXT:    ; def v[4:5]
+; GFX940-NEXT:    ; def v[0:1]
 ; GFX940-NEXT:    ;;#ASMEND
+; GFX940-NEXT:    v_mov_b32_e32 v4, 0
+; GFX940-NEXT:    v_pk_mov_b32 v[0:1], v[0:1], v[0:1] op_sel:[1,0]
 ; GFX940-NEXT:    v_mov_b32_e32 v2, v3
-; GFX940-NEXT:    v_mov_b32_e32 v0, v5
-; GFX940-NEXT:    v_mov_b32_e32 v1, v4
-; GFX940-NEXT:    global_store_dwordx4 v6, v[0:3], s[0:1] sc0 sc1
+; GFX940-NEXT:    global_store_dwordx4 v4, v[0:3], s[0:1] sc0 sc1
 ; GFX940-NEXT:    s_waitcnt vmcnt(0)
 ; GFX940-NEXT:    s_setpc_b64 s[30:31]
   %vec0 = call <2 x float> asm "; def $0", "=v"()
@@ -2152,8 +2132,7 @@ define void @v_shuffle_v4f32_v2f32__3_2_2_2(ptr addrspace(1) inreg %ptr) {
 ; GFX90A-NEXT:    ; def v[2:3]
 ; GFX90A-NEXT:    ;;#ASMEND
 ; GFX90A-NEXT:    v_mov_b32_e32 v4, 0
-; GFX90A-NEXT:    v_mov_b32_e32 v0, v3
-; GFX90A-NEXT:    v_mov_b32_e32 v1, v2
+; GFX90A-NEXT:    v_pk_mov_b32 v[0:1], v[2:3], v[2:3] op_sel:[1,0]
 ; GFX90A-NEXT:    v_mov_b32_e32 v3, v2
 ; GFX90A-NEXT:    global_store_dwordx4 v4, v[0:3], s[16:17]
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
@@ -2166,8 +2145,7 @@ define void @v_shuffle_v4f32_v2f32__3_2_2_2(ptr addrspace(1) inreg %ptr) {
 ; GFX940-NEXT:    ; def v[2:3]
 ; GFX940-NEXT:    ;;#ASMEND
 ; GFX940-NEXT:    v_mov_b32_e32 v4, 0
-; GFX940-NEXT:    v_mov_b32_e32 v0, v3
-; GFX940-NEXT:    v_mov_b32_e32 v1, v2
+; GFX940-NEXT:    v_pk_mov_b32 v[0:1], v[2:3], v[2:3] op_sel:[1,0]
 ; GFX940-NEXT:    v_mov_b32_e32 v3, v2
 ; GFX940-NEXT:    global_store_dwordx4 v4, v[0:3], s[0:1] sc0 sc1
 ; GFX940-NEXT:    s_waitcnt vmcnt(0)
@@ -2246,16 +2224,15 @@ define void @v_shuffle_v4f32_v2f32__3_0_2_2(ptr addrspace(1) inreg %ptr) {
 ; GFX90A:       ; %bb.0:
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX90A-NEXT:    ;;#ASMSTART
-; GFX90A-NEXT:    ; def v[2:3]
+; GFX90A-NEXT:    ; def v[0:1]
 ; GFX90A-NEXT:    ;;#ASMEND
-; GFX90A-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX90A-NEXT:    ;;#ASMSTART
-; GFX90A-NEXT:    ; def v[4:5]
+; GFX90A-NEXT:    ; def v[2:3]
 ; GFX90A-NEXT:    ;;#ASMEND
-; GFX90A-NEXT:    v_mov_b32_e32 v0, v3
-; GFX90A-NEXT:    v_mov_b32_e32 v1, v4
+; GFX90A-NEXT:    v_mov_b32_e32 v4, 0
+; GFX90A-NEXT:    v_pk_mov_b32 v[0:1], v[2:3], v[0:1] op_sel:[1,0]
 ; GFX90A-NEXT:    v_mov_b32_e32 v3, v2
-; GFX90A-NEXT:    global_store_dwordx4 v6, v[0:3], s[16:17]
+; GFX90A-NEXT:    global_store_dwordx4 v4, v[0:3], s[16:17]
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
 ; GFX90A-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -2263,16 +2240,15 @@ define void @v_shuffle_v4f32_v2f32__3_0_2_2(ptr addrspace(1) inreg %ptr) {
 ; GFX940:       ; %bb.0:
 ; GFX940-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX940-NEXT:    ;;#ASMSTART
-; GFX940-NEXT:    ; def v[2:3]
+; GFX940-NEXT:    ; def v[0:1]
 ; GFX940-NEXT:    ;;#ASMEND
-; GFX940-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX940-NEXT:    ;;#ASMSTART
-; GFX940-NEXT:    ; def v[4:5]
+; GFX940-NEXT:    ; def v[2:3]
 ; GFX940-NEXT:    ;;#ASMEND
-; GFX940-NEXT:    v_mov_b32_e32 v0, v3
-; GFX940-NEXT:    v_mov_b32_e32 v1, v4
+; GFX940-NEXT:    v_mov_b32_e32 v4, 0
+; GFX940-NEXT:    v_pk_mov_b32 v[0:1], v[2:3], v[0:1] op_sel:[1,0]
 ; GFX940-NEXT:    v_mov_b32_e32 v3, v2
-; GFX940-NEXT:    global_store_dwordx4 v6, v[0:3], s[0:1] sc0 sc1
+; GFX940-NEXT:    global_store_dwordx4 v4, v[0:3], s[0:1] sc0 sc1
 ; GFX940-NEXT:    s_waitcnt vmcnt(0)
 ; GFX940-NEXT:    s_setpc_b64 s[30:31]
   %vec0 = call <2 x float> asm "; def $0", "=v"()
@@ -2516,15 +2492,13 @@ define void @v_shuffle_v4f32_v2f32__3_3_1_2(ptr addrspace(1) inreg %ptr) {
 ; GFX90A-NEXT:    ;;#ASMSTART
 ; GFX90A-NEXT:    ; def v[2:3]
 ; GFX90A-NEXT:    ;;#ASMEND
-; GFX90A-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX90A-NEXT:    ;;#ASMSTART
-; GFX90A-NEXT:    ; def v[4:5]
+; GFX90A-NEXT:    ; def v[0:1]
 ; GFX90A-NEXT:    ;;#ASMEND
-; GFX90A-NEXT:    v_mov_b32_e32 v0, v5
-; GFX90A-NEXT:    v_mov_b32_e32 v1, v5
-; GFX90A-NEXT:    v_mov_b32_e32 v2, v3
-; GFX90A-NEXT:    v_mov_b32_e32 v3, v4
-; GFX90A-NEXT:    global_store_dwordx4 v6, v[0:3], s[16:17]
+; GFX90A-NEXT:    v_mov_b32_e32 v4, 0
+; GFX90A-NEXT:    v_pk_mov_b32 v[2:3], v[2:3], v[0:1] op_sel:[1,0]
+; GFX90A-NEXT:    v_mov_b32_e32 v0, v1
+; GFX90A-NEXT:    global_store_dwordx4 v4, v[0:3], s[16:17]
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
 ; GFX90A-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -2534,15 +2508,13 @@ define void @v_shuffle_v4f32_v2f32__3_3_1_2(ptr addrspace(1) inreg %ptr) {
 ; GFX940-NEXT:    ;;#ASMSTART
 ; GFX940-NEXT:    ; def v[2:3]
 ; GFX940-NEXT:    ;;#ASMEND
-; GFX940-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX940-NEXT:    ;;#ASMSTART
-; GFX940-NEXT:    ; def v[4:5]
+; GFX940-NEXT:    ; def v[0:1]
 ; GFX940-NEXT:    ;;#ASMEND
-; GFX940-NEXT:    v_mov_b32_e32 v2, v3
-; GFX940-NEXT:    v_mov_b32_e32 v0, v5
-; GFX940-NEXT:    v_mov_b32_e32 v1, v5
-; GFX940-NEXT:    v_mov_b32_e32 v3, v4
-; GFX940-NEXT:    global_store_dwordx4 v6, v[0:3], s[0:1] sc0 sc1
+; GFX940-NEXT:    v_mov_b32_e32 v4, 0
+; GFX940-NEXT:    v_pk_mov_b32 v[2:3], v[2:3], v[0:1] op_sel:[1,0]
+; GFX940-NEXT:    v_mov_b32_e32 v0, v1
+; GFX940-NEXT:    global_store_dwordx4 v4, v[0:3], s[0:1] sc0 sc1
 ; GFX940-NEXT:    s_waitcnt vmcnt(0)
 ; GFX940-NEXT:    s_setpc_b64 s[30:31]
   %vec0 = call <2 x float> asm "; def $0", "=v"()
@@ -2826,16 +2798,15 @@ define void @v_shuffle_v4f32_v2f32__3_0_3_3(ptr addrspace(1) inreg %ptr) {
 ; GFX90A:       ; %bb.0:
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX90A-NEXT:    ;;#ASMSTART
-; GFX90A-NEXT:    ; def v[2:3]
+; GFX90A-NEXT:    ; def v[0:1]
 ; GFX90A-NEXT:    ;;#ASMEND
-; GFX90A-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX90A-NEXT:    ;;#ASMSTART
-; GFX90A-NEXT:    ; def v[4:5]
+; GFX90A-NEXT:    ; def v[2:3]
 ; GFX90A-NEXT:    ;;#ASMEND
-; GFX90A-NEXT:    v_mov_b32_e32 v0, v3
-; GFX90A-NEXT:    v_mov_b32_e32 v1, v4
+; GFX90A-NEXT:    v_mov_b32_e32 v4, 0
+; GFX90A-NEXT:    v_pk_mov_b32 v[0:1], v[2:3], v[0:1] op_sel:[1,0]
 ; GFX90A-NEXT:    v_mov_b32_e32 v2, v3
-; GFX90A-NEXT:    global_store_dwordx4 v6, v[0:3], s[16:17]
+; GFX90A-NEXT:    global_store_dwordx4 v4, v[0:3], s[16:17]
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
 ; GFX90A-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -2843,16 +2814,15 @@ define void @v_shuffle_v4f32_v2f32__3_0_3_3(ptr addrspace(1) inreg %ptr) {
 ; GFX940:       ; %bb.0:
 ; GFX940-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX940-NEXT:    ;;#ASMSTART
-; GFX940-NEXT:    ; def v[2:3]
+; GFX940-NEXT:    ; def v[0:1]
 ; GFX940-NEXT:    ;;#ASMEND
-; GFX940-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX940-NEXT:    ;;#ASMSTART
-; GFX940-NEXT:    ; def v[4:5]
+; GFX940-NEXT:    ; def v[2:3]
 ; GFX940-NEXT:    ;;#ASMEND
-; GFX940-NEXT:    v_mov_b32_e32 v0, v3
-; GFX940-NEXT:    v_mov_b32_e32 v1, v4
+; GFX940-NEXT:    v_mov_b32_e32 v4, 0
+; GFX940-NEXT:    v_pk_mov_b32 v[0:1], v[2:3], v[0:1] op_sel:[1,0]
 ; GFX940-NEXT:    v_mov_b32_e32 v2, v3
-; GFX940-NEXT:    global_store_dwordx4 v6, v[0:3], s[0:1] sc0 sc1
+; GFX940-NEXT:    global_store_dwordx4 v4, v[0:3], s[0:1] sc0 sc1
 ; GFX940-NEXT:    s_waitcnt vmcnt(0)
 ; GFX940-NEXT:    s_setpc_b64 s[30:31]
   %vec0 = call <2 x float> asm "; def $0", "=v"()
@@ -2938,8 +2908,7 @@ define void @v_shuffle_v4f32_v2f32__3_2_3_3(ptr addrspace(1) inreg %ptr) {
 ; GFX90A-NEXT:    ; def v[2:3]
 ; GFX90A-NEXT:    ;;#ASMEND
 ; GFX90A-NEXT:    v_mov_b32_e32 v4, 0
-; GFX90A-NEXT:    v_mov_b32_e32 v0, v3
-; GFX90A-NEXT:    v_mov_b32_e32 v1, v2
+; GFX90A-NEXT:    v_pk_mov_b32 v[0:1], v[2:3], v[2:3] op_sel:[1,0]
 ; GFX90A-NEXT:    v_mov_b32_e32 v2, v3
 ; GFX90A-NEXT:    global_store_dwordx4 v4, v[0:3], s[16:17]
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
@@ -2952,8 +2921,7 @@ define void @v_shuffle_v4f32_v2f32__3_2_3_3(ptr addrspace(1) inreg %ptr) {
 ; GFX940-NEXT:    ; def v[2:3]
 ; GFX940-NEXT:    ;;#ASMEND
 ; GFX940-NEXT:    v_mov_b32_e32 v4, 0
-; GFX940-NEXT:    v_mov_b32_e32 v0, v3
-; GFX940-NEXT:    v_mov_b32_e32 v1, v2
+; GFX940-NEXT:    v_pk_mov_b32 v[0:1], v[2:3], v[2:3] op_sel:[1,0]
 ; GFX940-NEXT:    v_mov_b32_e32 v2, v3
 ; GFX940-NEXT:    global_store_dwordx4 v4, v[0:3], s[0:1] sc0 sc1
 ; GFX940-NEXT:    s_waitcnt vmcnt(0)
diff --git a/llvm/test/CodeGen/AMDGPU/shufflevector.v4f32.v3f32.ll b/llvm/test/CodeGen/AMDGPU/shufflevector.v4f32.v3f32.ll
index c797ef338dee9..31c458f5338cb 100644
--- a/llvm/test/CodeGen/AMDGPU/shufflevector.v4f32.v3f32.ll
+++ b/llvm/test/CodeGen/AMDGPU/shufflevector.v4f32.v3f32.ll
@@ -1280,8 +1280,7 @@ define void @v_shuffle_v4f32_v3f32__1_0_0_0(ptr addrspace(1) inreg %ptr) {
 ; GFX90A-NEXT:    ;;#ASMSTART
 ; GFX90A-NEXT:    ; def v[4:6]
 ; GFX90A-NEXT:    ;;#ASMEND
-; GFX90A-NEXT:    v_mov_b32_e32 v0, v5
-; GFX90A-NEXT:    v_mov_b32_e32 v1, v4
+; GFX90A-NEXT:    v_pk_mov_b32 v[0:1], v[4:5], v[4:5] op_sel:[1,0]
 ; GFX90A-NEXT:    v_mov_b32_e32 v2, v4
 ; GFX90A-NEXT:    v_mov_b32_e32 v3, v4
 ; GFX90A-NEXT:    global_store_dwordx4 v7, v[0:3], s[16:17]
@@ -1296,8 +1295,7 @@ define void @v_shuffle_v4f32_v3f32__1_0_0_0(ptr addrspace(1) inreg %ptr) {
 ; GFX940-NEXT:    ; def v[4:6]
 ; GFX940-NEXT:    ;;#ASMEND
 ; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_mov_b32_e32 v0, v5
-; GFX940-NEXT:    v_mov_b32_e32 v1, v4
+; GFX940-NEXT:    v_pk_mov_b32 v[0:1], v[4:5], v[4:5] op_sel:[1,0]
 ; GFX940-NEXT:    v_mov_b32_e32 v2, v4
 ; GFX940-NEXT:    v_mov_b32_e32 v3, v4
 ; GFX940-NEXT:    global_store_dwordx4 v7, v[0:3], s[0:1] sc0 sc1
@@ -1435,8 +1433,7 @@ define void @v_shuffle_v4f32_v3f32__4_0_0_0(ptr addrspace(1) inreg %ptr) {
 ; GFX90A-NEXT:    ;;#ASMSTART
 ; GFX90A-NEXT:    ; def v[4:6]
 ; GFX90A-NEXT:    ;;#ASMEND
-; GFX90A-NEXT:    v_mov_b32_e32 v0, v1
-; GFX90A-NEXT:    v_mov_b32_e32 v1, v4
+; GFX90A-NEXT:    v_pk_mov_b32 v[0:1], v[0:1], v[4:5] op_sel:[1,0]
 ; GFX90A-NEXT:    v_mov_b32_e32 v2, v4
 ; GFX90A-NEXT:    v_mov_b32_e32 v3, v4
 ; GFX90A-NEXT:    global_store_dwordx4 v7, v[0:3], s[16:17]
@@ -1453,8 +1450,8 @@ define void @v_shuffle_v4f32_v3f32__4_0_0_0(ptr addrspace(1) inreg %ptr) {
 ; GFX940-NEXT:    ;;#ASMSTART
 ; GFX940-NEXT:    ; def v[4:6]
 ; GFX940-NEXT:    ;;#ASMEND
-; GFX940-NEXT:    v_mov_b32_e32 v0, v1
-; GFX940-NEXT:    v_mov_b32_e32 v1, v4
+; GFX940-NEXT:    s_nop 0
+; GFX940-NEXT:    v_pk_mov_b32 v[0:1], v[0:1], v[4:5] op_sel:[1,0]
 ; GFX940-NEXT:    v_mov_b32_e32 v2, v4
 ; GFX940-NEXT:    v_mov_b32_e32 v3, v4
 ; GFX940-NEXT:    global_store_dwordx4 v7, v[0:3], s[0:1] sc0 sc1
@@ -1964,15 +1961,14 @@ define void @v_shuffle_v4f32_v3f32__5_5_1_0(ptr addrspace(1) inreg %ptr) {
 ; GFX90A-NEXT:    ;;#ASMSTART
 ; GFX90A-NEXT:    ; def v[0:2]
 ; GFX90A-NEXT:    ;;#ASMEND
-; GFX90A-NEXT:    v_mov_b32_e32 v7, 0
 ; GFX90A-NEXT:    ;;#ASMSTART
-; GFX90A-NEXT:    ; def v[4:6]
+; GFX90A-NEXT:    ; def v[2:4]
 ; GFX90A-NEXT:    ;;#ASMEND
-; GFX90A-NEXT:    v_mov_b32_e32 v0, v2
-; GFX90A-NEXT:    v_mov_b32_e32 v1, v2
-; GFX90A-NEXT:    v_mov_b32_e32 v2, v5
-; GFX90A-NEXT:    v_mov_b32_e32 v3, v4
-; GFX90A-NEXT:    global_store_dwordx4 v7, v[0:3], s[16:17]
+; GFX90A-NEXT:    v_mov_b32_e32 v5, 0
+; GFX90A-NEXT:    v_pk_mov_b32 v[2:3], v[0:1], v[0:1] op_sel:[1,0]
+; GFX90A-NEXT:    v_mov_b32_e32 v0, v4
+; GFX90A-NEXT:    v_mov_b32_e32 v1, v4
+; GFX90A-NEXT:    global_store_dwordx4 v5, v[0:3], s[16:17]
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
 ; GFX90A-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -1982,15 +1978,15 @@ define void @v_shuffle_v4f32_v3f32__5_5_1_0(ptr addrspace(1) inreg %ptr) {
 ; GFX940-NEXT:    ;;#ASMSTART
 ; GFX940-NEXT:    ; def v[0:2]
 ; GFX940-NEXT:    ;;#ASMEND
-; GFX940-NEXT:    v_mov_b32_e32 v7, 0
+; GFX940-NEXT:    v_mov_b32_e32 v5, 0
 ; GFX940-NEXT:    ;;#ASMSTART
-; GFX940-NEXT:    ; def v[4:6]
+; GFX940-NEXT:    ; def v[2:4]
 ; GFX940-NEXT:    ;;#ASMEND
-; GFX940-NEXT:    v_mov_b32_e32 v0, v2
-; GFX940-NEXT:    v_mov_b32_e32 v1, v2
-; GFX940-NEXT:    v_mov_b32_e32 v2, v5
-; GFX940-NEXT:    v_mov_b32_e32 v3, v4
-; GFX940-NEXT:    global_store_dwordx4 v7, v[0:3], s[0:1] sc0 sc1
+; GFX940-NEXT:    s_nop 0
+; GFX940-NEXT:    v_pk_mov_b32 v[2:3], v[0:1], v[0:1] op_sel:[1,0]
+; GFX940-NEXT:    v_mov_b32_e32 v0, v4
+; GFX940-NEXT:    v_mov_b32_e32 v1, v4
+; GFX940-NEXT:    global_store_dwordx4 v5, v[0:3], s[0:1] sc0 sc1
 ; GFX940-NEXT:    s_waitcnt vmcnt(0)
 ; GFX940-NEXT:    s_setpc_b64 s[30:31]
   %vec0 = call <3 x float> asm "; def $0", "=v"()
@@ -2142,17 +2138,16 @@ define void @v_shuffle_v4f32_v3f32__5_5_4_0(ptr addrspace(1) inreg %ptr) {
 ; GFX90A:       ; %bb.0:
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX90A-NEXT:    ;;#ASMSTART
-; GFX90A-NEXT:    ; def v[4:6]
+; GFX90A-NEXT:    ; def v[0:2]
 ; GFX90A-NEXT:    ;;#ASMEND
-; GFX90A-NEXT:    v_mov_b32_e32 v9, 0
 ; GFX90A-NEXT:    ;;#ASMSTART
-; GFX90A-NEXT:    ; def v[6:8]
+; GFX90A-NEXT:    ; def v[2:4]
 ; GFX90A-NEXT:    ;;#ASMEND
-; GFX90A-NEXT:    v_mov_b32_e32 v0, v8
-; GFX90A-NEXT:    v_mov_b32_e32 v1, v8
-; GFX90A-NEXT:    v_mov_b32_e32 v2, v7
-; GFX90A-NEXT:    v_mov_b32_e32 v3, v4
-; GFX90A-NEXT:    global_store_dwordx4 v9, v[0:3], s[16:17]
+; GFX90A-NEXT:    v_mov_b32_e32 v5, 0
+; GFX90A-NEXT:    v_pk_mov_b32 v[2:3], v[2:3], v[0:1] op_sel:[1,0]
+; GFX90A-NEXT:    v_mov_b32_e32 v0, v4
+; GFX90A-NEXT:    v_mov_b32_e32 v1, v4
+; GFX90A-NEXT:    global_store_dwordx4 v5, v[0:3], s[16:17]
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
 ; GFX90A-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -2160,17 +2155,17 @@ define void @v_shuffle_v4f32_v3f32__5_5_4_0(ptr addrspace(1) inreg %ptr) {
 ; GFX940:       ; %bb.0:
 ; GFX940-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX940-NEXT:    ;;#ASMSTART
-; GFX940-NEXT:    ; def v[4:6]
+; GFX940-NEXT:    ; def v[0:2]
 ; GFX940-NEXT:    ;;#ASMEND
-; GFX940-NEXT:    v_mov_b32_e32 v9, 0
+; GFX940-NEXT:    v_mov_b32_e32 v5, 0
 ; GFX940-NEXT:    ;;#ASMSTART
-; GFX940-NEXT:    ; def v[6:8]
+; GFX940-NEXT:    ; def v[2:4]
 ; GFX940-NEXT:    ;;#ASMEND
-; GFX940-NEXT:    v_mov_b32_e32 v3, v4
-; GFX940-NEXT:    v_mov_b32_e32 v0, v8
-; GFX940-NEXT:    v_mov_b32_e32 v1, v8
-; GFX940-NEXT:    v_mov_b32_e32 v2, v7
-; GFX940-NEXT:    global_store_dwordx4 v9, v[0:3], s[0:1] sc0 sc1
+; GFX940-NEXT:    s_nop 0
+; GFX940-NEXT:    v_pk_mov_b32 v[2:3], v[2:3], v[0:1] op_sel:[1,0]
+; GFX940-NEXT:    v_mov_b32_e32 v0, v4
+; GFX940-NEXT:    v_mov_b32_e32 v1, v4
+; GFX940-NEXT:    global_store_dwordx4 v5, v[0:3], s[0:1] sc0 sc1
 ; GFX940-NEXT:    s_waitcnt vmcnt(0)
 ; GFX940-NEXT:    s_setpc_b64 s[30:31]
   %vec0 = call <3 x float> asm "; def $0", "=v"()
@@ -3277,9 +3272,9 @@ define void @v_shuffle_v4f32_v3f32__1_2_2_2(ptr addrspace(1) inreg %ptr) {
 ; GFX90A-NEXT:    ;;#ASMSTART
 ; GFX90A-NEXT:    ; def v[0:2]
 ; GFX90A-NEXT:    ;;#ASMEND
+; GFX90A-NEXT:    v_mov_b32_e32 v0, v2
 ; GFX90A-NEXT:    v_mov_b32_e32 v4, 0
-; GFX90A-NEXT:    v_mov_b32_e32 v0, v1
-; GFX90A-NEXT:    v_mov_b32_e32 v1, v2
+; GFX90A-NEXT:    v_pk_mov_b32 v[0:1], v[0:1], v[0:1] op_sel:[1,0]
 ; GFX90A-NEXT:    v_mov_b32_e32 v3, v2
 ; GFX90A-NEXT:    global_store_dwordx4 v4, v[0:3], s[16:17]
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
@@ -3292,8 +3287,8 @@ define void @v_shuffle_v4f32_v3f32__1_2_2_2(ptr addrspace(1) inreg %ptr) {
 ; GFX940-NEXT:    ; def v[0:2]
 ; GFX940-NEXT:    ;;#ASMEND
 ; GFX940-NEXT:    v_mov_b32_e32 v4, 0
-; GFX940-NEXT:    v_mov_b32_e32 v0, v1
-; GFX940-NEXT:    v_mov_b32_e32 v1, v2
+; GFX940-NEXT:    v_mov_b32_e32 v0, v2
+; GFX940-NEXT:    v_pk_mov_b32 v[0:1], v[0:1], v[0:1] op_sel:[1,0]
 ; GFX940-NEXT:    v_mov_b32_e32 v3, v2
 ; GFX940-NEXT:    global_store_dwordx4 v4, v[0:3], s[0:1] sc0 sc1
 ; GFX940-NEXT:    s_waitcnt vmcnt(0)
@@ -3421,12 +3416,12 @@ define void @v_shuffle_v4f32_v3f32__4_2_2_2(ptr addrspace(1) inreg %ptr) {
 ; GFX90A-NEXT:    ;;#ASMSTART
 ; GFX90A-NEXT:    ; def v[0:2]
 ; GFX90A-NEXT:    ;;#ASMEND
+; GFX90A-NEXT:    v_mov_b32_e32 v0, v2
 ; GFX90A-NEXT:    v_mov_b32_e32 v7, 0
 ; GFX90A-NEXT:    ;;#ASMSTART
 ; GFX90A-NEXT:    ; def v[4:6]
 ; GFX90A-NEXT:    ;;#ASMEND
-; GFX90A-NEXT:    v_mov_b32_e32 v0, v5
-; GFX90A-NEXT:    v_mov_b32_e32 v1, v2
+; GFX90A-NEXT:    v_pk_mov_b32 v[0:1], v[4:5], v[0:1] op_sel:[1,0]
 ; GFX90A-NEXT:    v_mov_b32_e32 v3, v2
 ; GFX90A-NEXT:    global_store_dwordx4 v7, v[0:3], s[16:17]
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
@@ -3439,12 +3434,12 @@ define void @v_shuffle_v4f32_v3f32__4_2_2_2(ptr addrspace(1) inreg %ptr) {
 ; GFX940-NEXT:    ; def v[0:2]
 ; GFX940-NEXT:    ;;#ASMEND
 ; GFX940-NEXT:    v_mov_b32_e32 v7, 0
+; GFX940-NEXT:    v_mov_b32_e32 v0, v2
 ; GFX940-NEXT:    ;;#ASMSTART
 ; GFX940-NEXT:    ; def v[4:6]
 ; GFX940-NEXT:    ;;#ASMEND
-; GFX940-NEXT:    v_mov_b32_e32 v1, v2
-; GFX940-NEXT:    v_mov_b32_e32 v0, v5
 ; GFX940-NEXT:    v_mov_b32_e32 v3, v2
+; GFX940-NEXT:    v_pk_mov_b32 v[0:1], v[4:5], v[0:1] op_sel:[1,0]
 ; GFX940-NEXT:    global_store_dwordx4 v7, v[0:3], s[0:1] sc0 sc1
 ; GFX940-NEXT:    s_waitcnt vmcnt(0)
 ; GFX940-NEXT:    s_setpc_b64 s[30:31]
@@ -3991,17 +3986,17 @@ define void @v_shuffle_v4f32_v3f32__5_5_1_2(ptr addrspace(1) inreg %ptr) {
 ; GFX90A:       ; %bb.0:
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX90A-NEXT:    ;;#ASMSTART
-; GFX90A-NEXT:    ; def v[2:4]
-; GFX90A-NEXT:    ;;#ASMEND
-; GFX90A-NEXT:    ;;#ASMSTART
 ; GFX90A-NEXT:    ; def v[0:2]
 ; GFX90A-NEXT:    ;;#ASMEND
-; GFX90A-NEXT:    v_mov_b32_e32 v5, 0
 ; GFX90A-NEXT:    v_mov_b32_e32 v0, v2
-; GFX90A-NEXT:    v_mov_b32_e32 v1, v2
-; GFX90A-NEXT:    v_mov_b32_e32 v2, v3
-; GFX90A-NEXT:    v_mov_b32_e32 v3, v4
-; GFX90A-NEXT:    global_store_dwordx4 v5, v[0:3], s[16:17]
+; GFX90A-NEXT:    v_mov_b32_e32 v7, 0
+; GFX90A-NEXT:    ;;#ASMSTART
+; GFX90A-NEXT:    ; def v[4:6]
+; GFX90A-NEXT:    ;;#ASMEND
+; GFX90A-NEXT:    v_pk_mov_b32 v[2:3], v[0:1], v[0:1] op_sel:[1,0]
+; GFX90A-NEXT:    v_mov_b32_e32 v0, v6
+; GFX90A-NEXT:    v_mov_b32_e32 v1, v6
+; GFX90A-NEXT:    global_store_dwordx4 v7, v[0:3], s[16:17]
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
 ; GFX90A-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -4009,18 +4004,17 @@ define void @v_shuffle_v4f32_v3f32__5_5_1_2(ptr addrspace(1) inreg %ptr) {
 ; GFX940:       ; %bb.0:
 ; GFX940-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX940-NEXT:    ;;#ASMSTART
-; GFX940-NEXT:    ; def v[2:4]
-; GFX940-NEXT:    ;;#ASMEND
-; GFX940-NEXT:    v_mov_b32_e32 v5, 0
-; GFX940-NEXT:    ;;#ASMSTART
 ; GFX940-NEXT:    ; def v[0:2]
 ; GFX940-NEXT:    ;;#ASMEND
-; GFX940-NEXT:    s_nop 0
+; GFX940-NEXT:    v_mov_b32_e32 v7, 0
 ; GFX940-NEXT:    v_mov_b32_e32 v0, v2
-; GFX940-NEXT:    v_mov_b32_e32 v1, v2
-; GFX940-NEXT:    v_mov_b32_e32 v2, v3
-; GFX940-NEXT:    v_mov_b32_e32 v3, v4
-; GFX940-NEXT:    global_store_dwordx4 v5, v[0:3], s[0:1] sc0 sc1
+; GFX940-NEXT:    ;;#ASMSTART
+; GFX940-NEXT:    ; def v[4:6]
+; GFX940-NEXT:    ;;#ASMEND
+; GFX940-NEXT:    v_pk_mov_b32 v[2:3], v[0:1], v[0:1] op_sel:[1,0]
+; GFX940-NEXT:    v_mov_b32_e32 v0, v6
+; GFX940-NEXT:    v_mov_b32_e32 v1, v6
+; GFX940-NEXT:    global_store_dwordx4 v7, v[0:3], s[0:1] sc0 sc1
 ; GFX940-NEXT:    s_waitcnt vmcnt(0)
 ; GFX940-NEXT:    s_setpc_b64 s[30:31]
   %vec0 = call <3 x float> asm "; def $0", "=v"()
@@ -4112,17 +4106,17 @@ define void @v_shuffle_v4f32_v3f32__5_5_4_2(ptr addrspace(1) inreg %ptr) {
 ; GFX90A:       ; %bb.0:
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX90A-NEXT:    ;;#ASMSTART
-; GFX90A-NEXT:    ; def v[2:4]
+; GFX90A-NEXT:    ; def v[0:2]
 ; GFX90A-NEXT:    ;;#ASMEND
-; GFX90A-NEXT:    v_mov_b32_e32 v5, 0
+; GFX90A-NEXT:    v_mov_b32_e32 v0, v2
+; GFX90A-NEXT:    v_mov_b32_e32 v7, 0
 ; GFX90A-NEXT:    ;;#ASMSTART
-; GFX90A-NEXT:    ; def v[6:8]
+; GFX90A-NEXT:    ; def v[4:6]
 ; GFX90A-NEXT:    ;;#ASMEND
-; GFX90A-NEXT:    v_mov_b32_e32 v0, v8
-; GFX90A-NEXT:    v_mov_b32_e32 v1, v8
-; GFX90A-NEXT:    v_mov_b32_e32 v2, v7
-; GFX90A-NEXT:    v_mov_b32_e32 v3, v4
-; GFX90A-NEXT:    global_store_dwordx4 v5, v[0:3], s[16:17]
+; GFX90A-NEXT:    v_pk_mov_b32 v[2:3], v[4:5], v[0:1] op_sel:[1,0]
+; GFX90A-NEXT:    v_mov_b32_e32 v0, v6
+; GFX90A-NEXT:    v_mov_b32_e32 v1, v6
+; GFX90A-NEXT:    global_store_dwordx4 v7, v[0:3], s[16:17]
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
 ; GFX90A-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -4130,17 +4124,18 @@ define void @v_shuffle_v4f32_v3f32__5_5_4_2(ptr addrspace(1) inreg %ptr) {
 ; GFX940:       ; %bb.0:
 ; GFX940-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX940-NEXT:    ;;#ASMSTART
-; GFX940-NEXT:    ; def v[2:4]
+; GFX940-NEXT:    ; def v[0:2]
 ; GFX940-NEXT:    ;;#ASMEND
-; GFX940-NEXT:    v_mov_b32_e32 v5, 0
+; GFX940-NEXT:    v_mov_b32_e32 v7, 0
+; GFX940-NEXT:    v_mov_b32_e32 v0, v2
 ; GFX940-NEXT:    ;;#ASMSTART
-; GFX940-NEXT:    ; def v[6:8]
+; GFX940-NEXT:    ; def v[4:6]
 ; GFX940-NEXT:    ;;#ASMEND
-; GFX940-NEXT:    v_mov_b32_e32 v3, v4
-; GFX940-NEXT:    v_mov_b32_e32 v0, v8
-; GFX940-NEXT:    v_mov_b32_e32 v1, v8
-; GFX940-NEXT:    v_mov_b32_e32 v2, v7
-; GFX940-NEXT:    global_store_dwordx4 v5, v[0:3], s[0:1] sc0 sc1
+; GFX940-NEXT:    s_nop 0
+; GFX940-NEXT:    v_pk_mov_b32 v[2:3], v[4:5], v[0:1] op_sel:[1,0]
+; GFX940-NEXT:    v_mov_b32_e32 v0, v6
+; GFX940-NEXT:    v_mov_b32_e32 v1, v6
+; GFX940-NEXT:    global_store_dwordx4 v7, v[0:3], s[0:1] sc0 sc1
 ; GFX940-NEXT:    s_waitcnt vmcnt(0)
 ; GFX940-NEXT:    s_setpc_b64 s[30:31]
   %vec0 = call <3 x float> asm "; def $0", "=v"()
@@ -4317,8 +4312,7 @@ define void @v_shuffle_v4f32_v3f32__4_3_3_3(ptr addrspace(1) inreg %ptr) {
 ; GFX90A-NEXT:    ;;#ASMSTART
 ; GFX90A-NEXT:    ; def v[4:6]
 ; GFX90A-NEXT:    ;;#ASMEND
-; GFX90A-NEXT:    v_mov_b32_e32 v0, v5
-; GFX90A-NEXT:    v_mov_b32_e32 v1, v4
+; GFX90A-NEXT:    v_pk_mov_b32 v[0:1], v[4:5], v[4:5] op_sel:[1,0]
 ; GFX90A-NEXT:    v_mov_b32_e32 v2, v4
 ; GFX90A-NEXT:    v_mov_b32_e32 v3, v4
 ; GFX90A-NEXT:    global_store_dwordx4 v7, v[0:3], s[16:17]
@@ -4333,8 +4327,7 @@ define void @v_shuffle_v4f32_v3f32__4_3_3_3(ptr addrspace(1) inreg %ptr) {
 ; GFX940-NEXT:    ; def v[4:6]
 ; GFX940-NEXT:    ;;#ASMEND
 ; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_mov_b32_e32 v0, v5
-; GFX940-NEXT:    v_mov_b32_e32 v1, v4
+; GFX940-NEXT:    v_pk_mov_b32 v[0:1], v[4:5], v[4:5] op_sel:[1,0]
 ; GFX940-NEXT:    v_mov_b32_e32 v2, v4
 ; GFX940-NEXT:    v_mov_b32_e32 v3, v4
 ; GFX940-NEXT:    global_store_dwordx4 v7, v[0:3], s[0:1] sc0 sc1
@@ -4858,17 +4851,16 @@ define void @v_shuffle_v4f32_v3f32__5_5_1_3(ptr addrspace(1) inreg %ptr) {
 ; GFX90A:       ; %bb.0:
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX90A-NEXT:    ;;#ASMSTART
-; GFX90A-NEXT:    ; def v[2:4]
+; GFX90A-NEXT:    ; def v[0:2]
 ; GFX90A-NEXT:    ;;#ASMEND
-; GFX90A-NEXT:    v_mov_b32_e32 v7, 0
 ; GFX90A-NEXT:    ;;#ASMSTART
-; GFX90A-NEXT:    ; def v[4:6]
+; GFX90A-NEXT:    ; def v[2:4]
 ; GFX90A-NEXT:    ;;#ASMEND
-; GFX90A-NEXT:    v_mov_b32_e32 v0, v6
-; GFX90A-NEXT:    v_mov_b32_e32 v1, v6
-; GFX90A-NEXT:    v_mov_b32_e32 v2, v3
-; GFX90A-NEXT:    v_mov_b32_e32 v3, v4
-; GFX90A-NEXT:    global_store_dwordx4 v7, v[0:3], s[16:17]
+; GFX90A-NEXT:    v_mov_b32_e32 v5, 0
+; GFX90A-NEXT:    v_pk_mov_b32 v[2:3], v[0:1], v[2:3] op_sel:[1,0]
+; GFX90A-NEXT:    v_mov_b32_e32 v0, v4
+; GFX90A-NEXT:    v_mov_b32_e32 v1, v4
+; GFX90A-NEXT:    global_store_dwordx4 v5, v[0:3], s[16:17]
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
 ; GFX90A-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -4876,17 +4868,17 @@ define void @v_shuffle_v4f32_v3f32__5_5_1_3(ptr addrspace(1) inreg %ptr) {
 ; GFX940:       ; %bb.0:
 ; GFX940-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX940-NEXT:    ;;#ASMSTART
-; GFX940-NEXT:    ; def v[2:4]
+; GFX940-NEXT:    ; def v[0:2]
 ; GFX940-NEXT:    ;;#ASMEND
-; GFX940-NEXT:    v_mov_b32_e32 v7, 0
+; GFX940-NEXT:    v_mov_b32_e32 v5, 0
 ; GFX940-NEXT:    ;;#ASMSTART
-; GFX940-NEXT:    ; def v[4:6]
+; GFX940-NEXT:    ; def v[2:4]
 ; GFX940-NEXT:    ;;#ASMEND
-; GFX940-NEXT:    v_mov_b32_e32 v2, v3
-; GFX940-NEXT:    v_mov_b32_e32 v0, v6
-; GFX940-NEXT:    v_mov_b32_e32 v1, v6
-; GFX940-NEXT:    v_mov_b32_e32 v3, v4
-; GFX940-NEXT:    global_store_dwordx4 v7, v[0:3], s[0:1] sc0 sc1
+; GFX940-NEXT:    s_nop 0
+; GFX940-NEXT:    v_pk_mov_b32 v[2:3], v[0:1], v[2:3] op_sel:[1,0]
+; GFX940-NEXT:    v_mov_b32_e32 v0, v4
+; GFX940-NEXT:    v_mov_b32_e32 v1, v4
+; GFX940-NEXT:    global_store_dwordx4 v5, v[0:3], s[0:1] sc0 sc1
 ; GFX940-NEXT:    s_waitcnt vmcnt(0)
 ; GFX940-NEXT:    s_setpc_b64 s[30:31]
   %vec0 = call <3 x float> asm "; def $0", "=v"()
@@ -4972,31 +4964,28 @@ define void @v_shuffle_v4f32_v3f32__5_5_4_3(ptr addrspace(1) inreg %ptr) {
 ; GFX90A-LABEL: v_shuffle_v4f32_v3f32__5_5_4_3:
 ; GFX90A:       ; %bb.0:
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT:    v_mov_b32_e32 v7, 0
 ; GFX90A-NEXT:    ;;#ASMSTART
-; GFX90A-NEXT:    ; def v[4:6]
+; GFX90A-NEXT:    ; def v[2:4]
 ; GFX90A-NEXT:    ;;#ASMEND
-; GFX90A-NEXT:    v_mov_b32_e32 v0, v6
-; GFX90A-NEXT:    v_mov_b32_e32 v1, v6
-; GFX90A-NEXT:    v_mov_b32_e32 v2, v5
-; GFX90A-NEXT:    v_mov_b32_e32 v3, v4
-; GFX90A-NEXT:    global_store_dwordx4 v7, v[0:3], s[16:17]
+; GFX90A-NEXT:    v_mov_b32_e32 v5, 0
+; GFX90A-NEXT:    v_pk_mov_b32 v[2:3], v[2:3], v[2:3] op_sel:[1,0]
+; GFX90A-NEXT:    v_mov_b32_e32 v0, v4
+; GFX90A-NEXT:    v_mov_b32_e32 v1, v4
+; GFX90A-NEXT:    global_store_dwordx4 v5, v[0:3], s[16:17]
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
 ; GFX90A-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: v_shuffle_v4f32_v3f32__5_5_4_3:
 ; GFX940:       ; %bb.0:
 ; GFX940-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX940-NEXT:    v_mov_b32_e32 v7, 0
 ; GFX940-NEXT:    ;;#ASMSTART
-; GFX940-NEXT:    ; def v[4:6]
+; GFX940-NEXT:    ; def v[2:4]
 ; GFX940-NEXT:    ;;#ASMEND
-; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_mov_b32_e32 v0, v6
-; GFX940-NEXT:    v_mov_b32_e32 v1, v6
-; GFX940-NEXT:    v_mov_b32_e32 v2, v5
-; GFX940-NEXT:    v_mov_b32_e32 v3, v4
-; GFX940-NEXT:    global_store_dwordx4 v7, v[0:3], s[0:1] sc0 sc1
+; GFX940-NEXT:    v_mov_b32_e32 v5, 0
+; GFX940-NEXT:    v_pk_mov_b32 v[2:3], v[2:3], v[2:3] op_sel:[1,0]
+; GFX940-NEXT:    v_mov_b32_e32 v0, v4
+; GFX940-NEXT:    v_mov_b32_e32 v1, v4
+; GFX940-NEXT:    global_store_dwordx4 v5, v[0:3], s[0:1] sc0 sc1
 ; GFX940-NEXT:    s_waitcnt vmcnt(0)
 ; GFX940-NEXT:    s_setpc_b64 s[30:31]
   %vec0 = call <3 x float> asm "; def $0", "=v"()
@@ -6094,9 +6083,9 @@ define void @v_shuffle_v4f32_v3f32__1_5_5_5(ptr addrspace(1) inreg %ptr) {
 ; GFX90A-NEXT:    ;;#ASMSTART
 ; GFX90A-NEXT:    ; def v[0:2]
 ; GFX90A-NEXT:    ;;#ASMEND
+; GFX90A-NEXT:    v_mov_b32_e32 v0, v2
 ; GFX90A-NEXT:    v_mov_b32_e32 v5, 0
-; GFX90A-NEXT:    v_mov_b32_e32 v0, v3
-; GFX90A-NEXT:    v_mov_b32_e32 v1, v2
+; GFX90A-NEXT:    v_pk_mov_b32 v[0:1], v[2:3], v[0:1] op_sel:[1,0]
 ; GFX90A-NEXT:    v_mov_b32_e32 v3, v2
 ; GFX90A-NEXT:    global_store_dwordx4 v5, v[0:3], s[16:17]
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
@@ -6113,8 +6102,8 @@ define void @v_shuffle_v4f32_v3f32__1_5_5_5(ptr addrspace(1) inreg %ptr) {
 ; GFX940-NEXT:    ; def v[0:2]
 ; GFX940-NEXT:    ;;#ASMEND
 ; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_mov_b32_e32 v0, v3
-; GFX940-NEXT:    v_mov_b32_e32 v1, v2
+; GFX940-NEXT:    v_mov_b32_e32 v0, v2
+; GFX940-NEXT:    v_pk_mov_b32 v[0:1], v[2:3], v[0:1] op_sel:[1,0]
 ; GFX940-NEXT:    v_mov_b32_e32 v3, v2
 ; GFX940-NEXT:    global_store_dwordx4 v5, v[0:3], s[0:1] sc0 sc1
 ; GFX940-NEXT:    s_waitcnt vmcnt(0)
@@ -6252,9 +6241,9 @@ define void @v_shuffle_v4f32_v3f32__4_5_5_5(ptr addrspace(1) inreg %ptr) {
 ; GFX90A-NEXT:    ;;#ASMSTART
 ; GFX90A-NEXT:    ; def v[0:2]
 ; GFX90A-NEXT:    ;;#ASMEND
+; GFX90A-NEXT:    v_mov_b32_e32 v0, v2
 ; GFX90A-NEXT:    v_mov_b32_e32 v4, 0
-; GFX90A-NEXT:    v_mov_b32_e32 v0, v1
-; GFX90A-NEXT:    v_mov_b32_e32 v1, v2
+; GFX90A-NEXT:    v_pk_mov_b32 v[0:1], v[0:1], v[0:1] op_sel:[1,0]
 ; GFX90A-NEXT:    v_mov_b32_e32 v3, v2
 ; GFX90A-NEXT:    global_store_dwordx4 v4, v[0:3], s[16:17]
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
@@ -6267,8 +6256,8 @@ define void @v_shuffle_v4f32_v3f32__4_5_5_5(ptr addrspace(1) inreg %ptr) {
 ; GFX940-NEXT:    ; def v[0:2]
 ; GFX940-NEXT:    ;;#ASMEND
 ; GFX940-NEXT:    v_mov_b32_e32 v4, 0
-; GFX940-NEXT:    v_mov_b32_e32 v0, v1
-; GFX940-NEXT:    v_mov_b32_e32 v1, v2
+; GFX940-NEXT:    v_mov_b32_e32 v0, v2
+; GFX940-NEXT:    v_pk_mov_b32 v[0:1], v[0:1], v[0:1] op_sel:[1,0]
 ; GFX940-NEXT:    v_mov_b32_e32 v3, v2
 ; GFX940-NEXT:    global_store_dwordx4 v4, v[0:3], s[0:1] sc0 sc1
 ; GFX940-NEXT:    s_waitcnt vmcnt(0)
@@ -6727,17 +6716,16 @@ define void @v_shuffle_v4f32_v3f32__5_5_1_5(ptr addrspace(1) inreg %ptr) {
 ; GFX90A:       ; %bb.0:
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX90A-NEXT:    ;;#ASMSTART
-; GFX90A-NEXT:    ; def v[2:4]
+; GFX90A-NEXT:    ; def v[0:2]
 ; GFX90A-NEXT:    ;;#ASMEND
-; GFX90A-NEXT:    v_mov_b32_e32 v7, 0
 ; GFX90A-NEXT:    ;;#ASMSTART
-; GFX90A-NEXT:    ; def v[4:6]
+; GFX90A-NEXT:    ; def v[2:4]
 ; GFX90A-NEXT:    ;;#ASMEND
-; GFX90A-NEXT:    v_mov_b32_e32 v0, v6
-; GFX90A-NEXT:    v_mov_b32_e32 v1, v6
-; GFX90A-NEXT:    v_mov_b32_e32 v2, v3
-; GFX90A-NEXT:    v_mov_b32_e32 v3, v6
-; GFX90A-NEXT:    global_store_dwordx4 v7, v[0:3], s[16:17]
+; GFX90A-NEXT:    v_mov_b32_e32 v0, v4
+; GFX90A-NEXT:    v_mov_b32_e32 v5, 0
+; GFX90A-NEXT:    v_pk_mov_b32 v[2:3], v[0:1], v[0:1] op_sel:[1,0]
+; GFX90A-NEXT:    v_mov_b32_e32 v1, v4
+; GFX90A-NEXT:    global_store_dwordx4 v5, v[0:3], s[16:17]
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
 ; GFX90A-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -6745,17 +6733,17 @@ define void @v_shuffle_v4f32_v3f32__5_5_1_5(ptr addrspace(1) inreg %ptr) {
 ; GFX940:       ; %bb.0:
 ; GFX940-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX940-NEXT:    ;;#ASMSTART
-; GFX940-NEXT:    ; def v[2:4]
+; GFX940-NEXT:    ; def v[0:2]
 ; GFX940-NEXT:    ;;#ASMEND
-; GFX940-NEXT:    v_mov_b32_e32 v7, 0
+; GFX940-NEXT:    v_mov_b32_e32 v5, 0
 ; GFX940-NEXT:    ;;#ASMSTART
-; GFX940-NEXT:    ; def v[4:6]
+; GFX940-NEXT:    ; def v[2:4]
 ; GFX940-NEXT:    ;;#ASMEND
-; GFX940-NEXT:    v_mov_b32_e32 v2, v3
-; GFX940-NEXT:    v_mov_b32_e32 v0, v6
-; GFX940-NEXT:    v_mov_b32_e32 v1, v6
-; GFX940-NEXT:    v_mov_b32_e32 v3, v6
-; GFX940-NEXT:    global_store_dwordx4 v7, v[0:3], s[0:1] sc0 sc1
+; GFX940-NEXT:    s_nop 0
+; GFX940-NEXT:    v_mov_b32_e32 v0, v4
+; GFX940-NEXT:    v_pk_mov_b32 v[2:3], v[0:1], v[0:1] op_sel:[1,0]
+; GFX940-NEXT:    v_mov_b32_e32 v1, v4
+; GFX940-NEXT:    global_store_dwordx4 v5, v[0:3], s[0:1] sc0 sc1
 ; GFX940-NEXT:    s_waitcnt vmcnt(0)
 ; GFX940-NEXT:    s_setpc_b64 s[30:31]
   %vec0 = call <3 x float> asm "; def $0", "=v"()
@@ -6893,11 +6881,10 @@ define void @v_shuffle_v4f32_v3f32__5_5_4_5(ptr addrspace(1) inreg %ptr) {
 ; GFX90A-NEXT:    ;;#ASMSTART
 ; GFX90A-NEXT:    ; def v[2:4]
 ; GFX90A-NEXT:    ;;#ASMEND
-; GFX90A-NEXT:    v_mov_b32_e32 v5, 0
 ; GFX90A-NEXT:    v_mov_b32_e32 v0, v4
+; GFX90A-NEXT:    v_mov_b32_e32 v5, 0
+; GFX90A-NEXT:    v_pk_mov_b32 v[2:3], v[2:3], v[0:1] op_sel:[1,0]
 ; GFX90A-NEXT:    v_mov_b32_e32 v1, v4
-; GFX90A-NEXT:    v_mov_b32_e32 v2, v3
-; GFX90A-NEXT:    v_mov_b32_e32 v3, v4
 ; GFX90A-NEXT:    global_store_dwordx4 v5, v[0:3], s[16:17]
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
 ; GFX90A-NEXT:    s_setpc_b64 s[30:31]
@@ -6910,9 +6897,8 @@ define void @v_shuffle_v4f32_v3f32__5_5_4_5(ptr addrspace(1) inreg %ptr) {
 ; GFX940-NEXT:    ;;#ASMEND
 ; GFX940-NEXT:    v_mov_b32_e32 v5, 0
 ; GFX940-NEXT:    v_mov_b32_e32 v0, v4
+; GFX940-NEXT:    v_pk_mov_b32 v[2:3], v[2:3], v[0:1] op_sel:[1,0]
 ; GFX940-NEXT:    v_mov_b32_e32 v1, v4
-; GFX940-NEXT:    v_mov_b32_e32 v2, v3
-; GFX940-NEXT:    v_mov_b32_e32 v3, v4
 ; GFX940-NEXT:    global_store_dwordx4 v5, v[0:3], s[0:1] sc0 sc1
 ; GFX940-NEXT:    s_waitcnt vmcnt(0)
 ; GFX940-NEXT:    s_setpc_b64 s[30:31]
diff --git a/llvm/test/CodeGen/AMDGPU/shufflevector.v4f32.v4f32.ll b/llvm/test/CodeGen/AMDGPU/shufflevector.v4f32.v4f32.ll
index 40d6ad90ab34b..e3427cd35c683 100644
--- a/llvm/test/CodeGen/AMDGPU/shufflevector.v4f32.v4f32.ll
+++ b/llvm/test/CodeGen/AMDGPU/shufflevector.v4f32.v4f32.ll
@@ -343,13 +343,12 @@ define void @v_shuffle_v4f32_v4f32__7_0_u_u(ptr addrspace(1) inreg %ptr) {
 ; GFX90A-NEXT:    ;;#ASMSTART
 ; GFX90A-NEXT:    ; def v[0:3]
 ; GFX90A-NEXT:    ;;#ASMEND
+; GFX90A-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX90A-NEXT:    ;;#ASMSTART
 ; GFX90A-NEXT:    ; def v[2:5]
 ; GFX90A-NEXT:    ;;#ASMEND
-; GFX90A-NEXT:    v_mov_b32_e32 v6, 0
-; GFX90A-NEXT:    v_mov_b32_e32 v2, v5
-; GFX90A-NEXT:    v_mov_b32_e32 v3, v0
-; GFX90A-NEXT:    global_store_dwordx4 v6, v[2:5], s[16:17]
+; GFX90A-NEXT:    v_pk_mov_b32 v[0:1], v[4:5], v[0:1] op_sel:[1,0]
+; GFX90A-NEXT:    global_store_dwordx4 v6, v[0:3], s[16:17]
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
 ; GFX90A-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -364,9 +363,8 @@ define void @v_shuffle_v4f32_v4f32__7_0_u_u(ptr addrspace(1) inreg %ptr) {
 ; GFX940-NEXT:    ; def v[2:5]
 ; GFX940-NEXT:    ;;#ASMEND
 ; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_mov_b32_e32 v2, v5
-; GFX940-NEXT:    v_mov_b32_e32 v3, v0
-; GFX940-NEXT:    global_store_dwordx4 v6, v[2:5], s[0:1] sc0 sc1
+; GFX940-NEXT:    v_pk_mov_b32 v[0:1], v[4:5], v[0:1] op_sel:[1,0]
+; GFX940-NEXT:    global_store_dwordx4 v6, v[0:3], s[0:1] sc0 sc1
 ; GFX940-NEXT:    s_waitcnt vmcnt(0)
 ; GFX940-NEXT:    s_setpc_b64 s[30:31]
   %vec0 = call <4 x float> asm "; def $0", "=v"()
@@ -456,8 +454,7 @@ define void @v_shuffle_v4f32_v4f32__7_2_u_u(ptr addrspace(1) inreg %ptr) {
 ; GFX90A-NEXT:    ;;#ASMSTART
 ; GFX90A-NEXT:    ; def v[4:7]
 ; GFX90A-NEXT:    ;;#ASMEND
-; GFX90A-NEXT:    v_mov_b32_e32 v0, v7
-; GFX90A-NEXT:    v_mov_b32_e32 v1, v2
+; GFX90A-NEXT:    v_pk_mov_b32 v[0:1], v[6:7], v[2:3] op_sel:[1,0]
 ; GFX90A-NEXT:    global_store_dwordx4 v8, v[0:3], s[16:17]
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
 ; GFX90A-NEXT:    s_setpc_b64 s[30:31]
@@ -472,8 +469,8 @@ define void @v_shuffle_v4f32_v4f32__7_2_u_u(ptr addrspace(1) inreg %ptr) {
 ; GFX940-NEXT:    ;;#ASMSTART
 ; GFX940-NEXT:    ; def v[4:7]
 ; GFX940-NEXT:    ;;#ASMEND
-; GFX940-NEXT:    v_mov_b32_e32 v1, v2
-; GFX940-NEXT:    v_mov_b32_e32 v0, v7
+; GFX940-NEXT:    s_nop 0
+; GFX940-NEXT:    v_pk_mov_b32 v[0:1], v[6:7], v[2:3] op_sel:[1,0]
 ; GFX940-NEXT:    global_store_dwordx4 v8, v[0:3], s[0:1] sc0 sc1
 ; GFX940-NEXT:    s_waitcnt vmcnt(0)
 ; GFX940-NEXT:    s_setpc_b64 s[30:31]
@@ -560,9 +557,8 @@ define void @v_shuffle_v4f32_v4f32__7_4_u_u(ptr addrspace(1) inreg %ptr) {
 ; GFX90A-NEXT:    ; def v[0:3]
 ; GFX90A-NEXT:    ;;#ASMEND
 ; GFX90A-NEXT:    v_mov_b32_e32 v4, 0
-; GFX90A-NEXT:    v_mov_b32_e32 v2, v3
-; GFX90A-NEXT:    v_mov_b32_e32 v3, v0
-; GFX90A-NEXT:    global_store_dwordx4 v4, v[2:5], s[16:17]
+; GFX90A-NEXT:    v_pk_mov_b32 v[0:1], v[2:3], v[0:1] op_sel:[1,0]
+; GFX90A-NEXT:    global_store_dwordx4 v4, v[0:3], s[16:17]
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
 ; GFX90A-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -573,9 +569,8 @@ define void @v_shuffle_v4f32_v4f32__7_4_u_u(ptr addrspace(1) inreg %ptr) {
 ; GFX940-NEXT:    ; def v[0:3]
 ; GFX940-NEXT:    ;;#ASMEND
 ; GFX940-NEXT:    v_mov_b32_e32 v4, 0
-; GFX940-NEXT:    v_mov_b32_e32 v2, v3
-; GFX940-NEXT:    v_mov_b32_e32 v3, v0
-; GFX940-NEXT:    global_store_dwordx4 v4, v[2:5], s[0:1] sc0 sc1
+; GFX940-NEXT:    v_pk_mov_b32 v[0:1], v[2:3], v[0:1] op_sel:[1,0]
+; GFX940-NEXT:    global_store_dwordx4 v4, v[0:3], s[0:1] sc0 sc1
 ; GFX940-NEXT:    s_waitcnt vmcnt(0)
 ; GFX940-NEXT:    s_setpc_b64 s[30:31]
   %vec0 = call <4 x float> asm "; def $0", "=v"()
@@ -649,8 +644,7 @@ define void @v_shuffle_v4f32_v4f32__7_6_u_u(ptr addrspace(1) inreg %ptr) {
 ; GFX90A-NEXT:    ; def v[0:3]
 ; GFX90A-NEXT:    ;;#ASMEND
 ; GFX90A-NEXT:    v_mov_b32_e32 v4, 0
-; GFX90A-NEXT:    v_mov_b32_e32 v0, v3
-; GFX90A-NEXT:    v_mov_b32_e32 v1, v2
+; GFX90A-NEXT:    v_pk_mov_b32 v[0:1], v[2:3], v[2:3] op_sel:[1,0]
 ; GFX90A-NEXT:    global_store_dwordx4 v4, v[0:3], s[16:17]
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
 ; GFX90A-NEXT:    s_setpc_b64 s[30:31]
@@ -662,8 +656,7 @@ define void @v_shuffle_v4f32_v4f32__7_6_u_u(ptr addrspace(1) inreg %ptr) {
 ; GFX940-NEXT:    ; def v[0:3]
 ; GFX940-NEXT:    ;;#ASMEND
 ; GFX940-NEXT:    v_mov_b32_e32 v4, 0
-; GFX940-NEXT:    v_mov_b32_e32 v0, v3
-; GFX940-NEXT:    v_mov_b32_e32 v1, v2
+; GFX940-NEXT:    v_pk_mov_b32 v[0:1], v[2:3], v[2:3] op_sel:[1,0]
 ; GFX940-NEXT:    global_store_dwordx4 v4, v[0:3], s[0:1] sc0 sc1
 ; GFX940-NEXT:    s_waitcnt vmcnt(0)
 ; GFX940-NEXT:    s_setpc_b64 s[30:31]
@@ -1169,11 +1162,10 @@ define void @v_shuffle_v4f32_v4f32__7_7_7_0(ptr addrspace(1) inreg %ptr) {
 ; GFX90A-NEXT:    ; def v[2:5]
 ; GFX90A-NEXT:    ;;#ASMEND
 ; GFX90A-NEXT:    v_mov_b32_e32 v6, 0
-; GFX90A-NEXT:    v_mov_b32_e32 v2, v5
-; GFX90A-NEXT:    v_mov_b32_e32 v3, v5
-; GFX90A-NEXT:    v_mov_b32_e32 v4, v5
-; GFX90A-NEXT:    v_mov_b32_e32 v5, v0
-; GFX90A-NEXT:    global_store_dwordx4 v6, v[2:5], s[16:17]
+; GFX90A-NEXT:    v_pk_mov_b32 v[2:3], v[4:5], v[0:1] op_sel:[1,0]
+; GFX90A-NEXT:    v_mov_b32_e32 v0, v5
+; GFX90A-NEXT:    v_mov_b32_e32 v1, v5
+; GFX90A-NEXT:    global_store_dwordx4 v6, v[0:3], s[16:17]
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
 ; GFX90A-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -1188,11 +1180,10 @@ define void @v_shuffle_v4f32_v4f32__7_7_7_0(ptr addrspace(1) inreg %ptr) {
 ; GFX940-NEXT:    ; def v[2:5]
 ; GFX940-NEXT:    ;;#ASMEND
 ; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_mov_b32_e32 v2, v5
-; GFX940-NEXT:    v_mov_b32_e32 v3, v5
-; GFX940-NEXT:    v_mov_b32_e32 v4, v5
-; GFX940-NEXT:    v_mov_b32_e32 v5, v0
-; GFX940-NEXT:    global_store_dwordx4 v6, v[2:5], s[0:1] sc0 sc1
+; GFX940-NEXT:    v_pk_mov_b32 v[2:3], v[4:5], v[0:1] op_sel:[1,0]
+; GFX940-NEXT:    v_mov_b32_e32 v0, v5
+; GFX940-NEXT:    v_mov_b32_e32 v1, v5
+; GFX940-NEXT:    global_store_dwordx4 v6, v[0:3], s[0:1] sc0 sc1
 ; GFX940-NEXT:    s_waitcnt vmcnt(0)
 ; GFX940-NEXT:    s_setpc_b64 s[30:31]
   %vec0 = call <4 x float> asm "; def $0", "=v"()
@@ -1287,17 +1278,16 @@ define void @v_shuffle_v4f32_v4f32__7_7_7_2(ptr addrspace(1) inreg %ptr) {
 ; GFX90A:       ; %bb.0:
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX90A-NEXT:    ;;#ASMSTART
-; GFX90A-NEXT:    ; def v[4:7]
+; GFX90A-NEXT:    ; def v[0:3]
 ; GFX90A-NEXT:    ;;#ASMEND
 ; GFX90A-NEXT:    v_mov_b32_e32 v8, 0
 ; GFX90A-NEXT:    ;;#ASMSTART
-; GFX90A-NEXT:    ; def v[0:3]
+; GFX90A-NEXT:    ; def v[4:7]
 ; GFX90A-NEXT:    ;;#ASMEND
-; GFX90A-NEXT:    v_mov_b32_e32 v4, v7
-; GFX90A-NEXT:    v_mov_b32_e32 v5, v7
-; GFX90A-NEXT:    v_mov_b32_e32 v6, v7
-; GFX90A-NEXT:    v_mov_b32_e32 v7, v2
-; GFX90A-NEXT:    global_store_dwordx4 v8, v[4:7], s[16:17]
+; GFX90A-NEXT:    v_pk_mov_b32 v[2:3], v[6:7], v[2:3] op_sel:[1,0]
+; GFX90A-NEXT:    v_mov_b32_e32 v0, v7
+; GFX90A-NEXT:    v_mov_b32_e32 v1, v7
+; GFX90A-NEXT:    global_store_dwordx4 v8, v[0:3], s[16:17]
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
 ; GFX90A-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -1305,17 +1295,17 @@ define void @v_shuffle_v4f32_v4f32__7_7_7_2(ptr addrspace(1) inreg %ptr) {
 ; GFX940:       ; %bb.0:
 ; GFX940-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX940-NEXT:    ;;#ASMSTART
-; GFX940-NEXT:    ; def v[4:7]
+; GFX940-NEXT:    ; def v[0:3]
 ; GFX940-NEXT:    ;;#ASMEND
 ; GFX940-NEXT:    v_mov_b32_e32 v8, 0
 ; GFX940-NEXT:    ;;#ASMSTART
-; GFX940-NEXT:    ; def v[0:3]
+; GFX940-NEXT:    ; def v[4:7]
 ; GFX940-NEXT:    ;;#ASMEND
-; GFX940-NEXT:    v_mov_b32_e32 v4, v7
-; GFX940-NEXT:    v_mov_b32_e32 v5, v7
-; GFX940-NEXT:    v_mov_b32_e32 v6, v7
-; GFX940-NEXT:    v_mov_b32_e32 v7, v2
-; GFX940-NEXT:    global_store_dwordx4 v8, v[4:7], s[0:1] sc0 sc1
+; GFX940-NEXT:    s_nop 0
+; GFX940-NEXT:    v_pk_mov_b32 v[2:3], v[6:7], v[2:3] op_sel:[1,0]
+; GFX940-NEXT:    v_mov_b32_e32 v0, v7
+; GFX940-NEXT:    v_mov_b32_e32 v1, v7
+; GFX940-NEXT:    global_store_dwordx4 v8, v[0:3], s[0:1] sc0 sc1
 ; GFX940-NEXT:    s_waitcnt vmcnt(0)
 ; GFX940-NEXT:    s_setpc_b64 s[30:31]
   %vec0 = call <4 x float> asm "; def $0", "=v"()
@@ -1402,31 +1392,26 @@ define void @v_shuffle_v4f32_v4f32__7_7_7_4(ptr addrspace(1) inreg %ptr) {
 ; GFX90A-LABEL: v_shuffle_v4f32_v4f32__7_7_7_4:
 ; GFX90A:       ; %bb.0:
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT:    v_mov_b32_e32 v8, 0
 ; GFX90A-NEXT:    ;;#ASMSTART
 ; GFX90A-NEXT:    ; def v[0:3]
 ; GFX90A-NEXT:    ;;#ASMEND
-; GFX90A-NEXT:    v_mov_b32_e32 v4, v3
-; GFX90A-NEXT:    v_mov_b32_e32 v5, v3
-; GFX90A-NEXT:    v_mov_b32_e32 v6, v3
-; GFX90A-NEXT:    v_mov_b32_e32 v7, v0
-; GFX90A-NEXT:    global_store_dwordx4 v8, v[4:7], s[16:17]
+; GFX90A-NEXT:    v_mov_b32_e32 v6, 0
+; GFX90A-NEXT:    v_pk_mov_b32 v[4:5], v[2:3], v[0:1] op_sel:[1,0]
+; GFX90A-NEXT:    v_mov_b32_e32 v2, v3
+; GFX90A-NEXT:    global_store_dwordx4 v6, v[2:5], s[16:17]
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
 ; GFX90A-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: v_shuffle_v4f32_v4f32__7_7_7_4:
 ; GFX940:       ; %bb.0:
 ; GFX940-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX940-NEXT:    v_mov_b32_e32 v8, 0
 ; GFX940-NEXT:    ;;#ASMSTART
 ; GFX940-NEXT:    ; def v[0:3]
 ; GFX940-NEXT:    ;;#ASMEND
-; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_mov_b32_e32 v4, v3
-; GFX940-NEXT:    v_mov_b32_e32 v5, v3
-; GFX940-NEXT:    v_mov_b32_e32 v6, v3
-; GFX940-NEXT:    v_mov_b32_e32 v7, v0
-; GFX940-NEXT:    global_store_dwordx4 v8, v[4:7], s[0:1] sc0 sc1
+; GFX940-NEXT:    v_mov_b32_e32 v6, 0
+; GFX940-NEXT:    v_pk_mov_b32 v[4:5], v[2:3], v[0:1] op_sel:[1,0]
+; GFX940-NEXT:    v_mov_b32_e32 v2, v3
+; GFX940-NEXT:    global_store_dwordx4 v6, v[2:5], s[0:1] sc0 sc1
 ; GFX940-NEXT:    s_waitcnt vmcnt(0)
 ; GFX940-NEXT:    s_setpc_b64 s[30:31]
   %vec0 = call <4 x float> asm "; def $0", "=v"()
@@ -1508,31 +1493,26 @@ define void @v_shuffle_v4f32_v4f32__7_7_7_6(ptr addrspace(1) inreg %ptr) {
 ; GFX90A-LABEL: v_shuffle_v4f32_v4f32__7_7_7_6:
 ; GFX90A:       ; %bb.0:
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT:    v_mov_b32_e32 v8, 0
 ; GFX90A-NEXT:    ;;#ASMSTART
 ; GFX90A-NEXT:    ; def v[0:3]
 ; GFX90A-NEXT:    ;;#ASMEND
-; GFX90A-NEXT:    v_mov_b32_e32 v4, v3
-; GFX90A-NEXT:    v_mov_b32_e32 v5, v3
-; GFX90A-NEXT:    v_mov_b32_e32 v6, v3
-; GFX90A-NEXT:    v_mov_b32_e32 v7, v2
-; GFX90A-NEXT:    global_store_dwordx4 v8, v[4:7], s[16:17]
+; GFX90A-NEXT:    v_mov_b32_e32 v6, 0
+; GFX90A-NEXT:    v_pk_mov_b32 v[4:5], v[2:3], v[2:3] op_sel:[1,0]
+; GFX90A-NEXT:    v_mov_b32_e32 v2, v3
+; GFX90A-NEXT:    global_store_dwordx4 v6, v[2:5], s[16:17]
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
 ; GFX90A-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: v_shuffle_v4f32_v4f32__7_7_7_6:
 ; GFX940:       ; %bb.0:
 ; GFX940-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX940-NEXT:    v_mov_b32_e32 v8, 0
 ; GFX940-NEXT:    ;;#ASMSTART
 ; GFX940-NEXT:    ; def v[0:3]
 ; GFX940-NEXT:    ;;#ASMEND
-; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_mov_b32_e32 v4, v3
-; GFX940-NEXT:    v_mov_b32_e32 v5, v3
-; GFX940-NEXT:    v_mov_b32_e32 v6, v3
-; GFX940-NEXT:    v_mov_b32_e32 v7, v2
-; GFX940-NEXT:    global_store_dwordx4 v8, v[4:7], s[0:1] sc0 sc1
+; GFX940-NEXT:    v_mov_b32_e32 v6, 0
+; GFX940-NEXT:    v_pk_mov_b32 v[4:5], v[2:3], v[2:3] op_sel:[1,0]
+; GFX940-NEXT:    v_mov_b32_e32 v2, v3
+; GFX940-NEXT:    global_store_dwordx4 v6, v[2:5], s[0:1] sc0 sc1
 ; GFX940-NEXT:    s_waitcnt vmcnt(0)
 ; GFX940-NEXT:    s_setpc_b64 s[30:31]
   %vec0 = call <4 x float> asm "; def $0", "=v"()
@@ -1709,8 +1689,7 @@ define void @v_shuffle_v4f32_v4f32__1_0_0_0(ptr addrspace(1) inreg %ptr) {
 ; GFX90A-NEXT:    ; def v[0:3]
 ; GFX90A-NEXT:    ;;#ASMEND
 ; GFX90A-NEXT:    v_mov_b32_e32 v6, 0
-; GFX90A-NEXT:    v_mov_b32_e32 v2, v1
-; GFX90A-NEXT:    v_mov_b32_e32 v3, v0
+; GFX90A-NEXT:    v_pk_mov_b32 v[2:3], v[0:1], v[0:1] op_sel:[1,0]
 ; GFX90A-NEXT:    v_mov_b32_e32 v4, v0
 ; GFX90A-NEXT:    v_mov_b32_e32 v5, v0
 ; GFX90A-NEXT:    global_store_dwordx4 v6, v[2:5], s[16:17]
@@ -1724,8 +1703,7 @@ define void @v_shuffle_v4f32_v4f32__1_0_0_0(ptr addrspace(1) inreg %ptr) {
 ; GFX940-NEXT:    ; def v[0:3]
 ; GFX940-NEXT:    ;;#ASMEND
 ; GFX940-NEXT:    v_mov_b32_e32 v6, 0
-; GFX940-NEXT:    v_mov_b32_e32 v2, v1
-; GFX940-NEXT:    v_mov_b32_e32 v3, v0
+; GFX940-NEXT:    v_pk_mov_b32 v[2:3], v[0:1], v[0:1] op_sel:[1,0]
 ; GFX940-NEXT:    v_mov_b32_e32 v4, v0
 ; GFX940-NEXT:    v_mov_b32_e32 v5, v0
 ; GFX940-NEXT:    global_store_dwordx4 v6, v[2:5], s[0:1] sc0 sc1
@@ -1809,8 +1787,7 @@ define void @v_shuffle_v4f32_v4f32__3_0_0_0(ptr addrspace(1) inreg %ptr) {
 ; GFX90A-NEXT:    ; def v[0:3]
 ; GFX90A-NEXT:    ;;#ASMEND
 ; GFX90A-NEXT:    v_mov_b32_e32 v6, 0
-; GFX90A-NEXT:    v_mov_b32_e32 v2, v3
-; GFX90A-NEXT:    v_mov_b32_e32 v3, v0
+; GFX90A-NEXT:    v_pk_mov_b32 v[2:3], v[2:3], v[0:1] op_sel:[1,0]
 ; GFX90A-NEXT:    v_mov_b32_e32 v4, v0
 ; GFX90A-NEXT:    v_mov_b32_e32 v5, v0
 ; GFX90A-NEXT:    global_store_dwordx4 v6, v[2:5], s[16:17]
@@ -1824,8 +1801,7 @@ define void @v_shuffle_v4f32_v4f32__3_0_0_0(ptr addrspace(1) inreg %ptr) {
 ; GFX940-NEXT:    ; def v[0:3]
 ; GFX940-NEXT:    ;;#ASMEND
 ; GFX940-NEXT:    v_mov_b32_e32 v6, 0
-; GFX940-NEXT:    v_mov_b32_e32 v2, v3
-; GFX940-NEXT:    v_mov_b32_e32 v3, v0
+; GFX940-NEXT:    v_pk_mov_b32 v[2:3], v[2:3], v[0:1] op_sel:[1,0]
 ; GFX940-NEXT:    v_mov_b32_e32 v4, v0
 ; GFX940-NEXT:    v_mov_b32_e32 v5, v0
 ; GFX940-NEXT:    global_store_dwordx4 v6, v[2:5], s[0:1] sc0 sc1
@@ -1914,8 +1890,7 @@ define void @v_shuffle_v4f32_v4f32__5_0_0_0(ptr addrspace(1) inreg %ptr) {
 ; GFX90A-NEXT:    ; def v[2:5]
 ; GFX90A-NEXT:    ;;#ASMEND
 ; GFX90A-NEXT:    v_mov_b32_e32 v6, 0
-; GFX90A-NEXT:    v_mov_b32_e32 v2, v3
-; GFX90A-NEXT:    v_mov_b32_e32 v3, v0
+; GFX90A-NEXT:    v_pk_mov_b32 v[2:3], v[2:3], v[0:1] op_sel:[1,0]
 ; GFX90A-NEXT:    v_mov_b32_e32 v4, v0
 ; GFX90A-NEXT:    v_mov_b32_e32 v5, v0
 ; GFX90A-NEXT:    global_store_dwordx4 v6, v[2:5], s[16:17]
@@ -1933,8 +1908,7 @@ define void @v_shuffle_v4f32_v4f32__5_0_0_0(ptr addrspace(1) inreg %ptr) {
 ; GFX940-NEXT:    ; def v[2:5]
 ; GFX940-NEXT:    ;;#ASMEND
 ; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_mov_b32_e32 v2, v3
-; GFX940-NEXT:    v_mov_b32_e32 v3, v0
+; GFX940-NEXT:    v_pk_mov_b32 v[2:3], v[2:3], v[0:1] op_sel:[1,0]
 ; GFX940-NEXT:    v_mov_b32_e32 v4, v0
 ; GFX940-NEXT:    v_mov_b32_e32 v5, v0
 ; GFX940-NEXT:    global_store_dwordx4 v6, v[2:5], s[0:1] sc0 sc1
@@ -2038,8 +2012,7 @@ define void @v_shuffle_v4f32_v4f32__7_0_0_0(ptr addrspace(1) inreg %ptr) {
 ; GFX90A-NEXT:    ; def v[2:5]
 ; GFX90A-NEXT:    ;;#ASMEND
 ; GFX90A-NEXT:    v_mov_b32_e32 v6, 0
-; GFX90A-NEXT:    v_mov_b32_e32 v2, v5
-; GFX90A-NEXT:    v_mov_b32_e32 v3, v0
+; GFX90A-NEXT:    v_pk_mov_b32 v[2:3], v[4:5], v[0:1] op_sel:[1,0]
 ; GFX90A-NEXT:    v_mov_b32_e32 v4, v0
 ; GFX90A-NEXT:    v_mov_b32_e32 v5, v0
 ; GFX90A-NEXT:    global_store_dwordx4 v6, v[2:5], s[16:17]
@@ -2057,8 +2030,7 @@ define void @v_shuffle_v4f32_v4f32__7_0_0_0(ptr addrspace(1) inreg %ptr) {
 ; GFX940-NEXT:    ; def v[2:5]
 ; GFX940-NEXT:    ;;#ASMEND
 ; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_mov_b32_e32 v2, v5
-; GFX940-NEXT:    v_mov_b32_e32 v3, v0
+; GFX940-NEXT:    v_pk_mov_b32 v[2:3], v[4:5], v[0:1] op_sel:[1,0]
 ; GFX940-NEXT:    v_mov_b32_e32 v4, v0
 ; GFX940-NEXT:    v_mov_b32_e32 v5, v0
 ; GFX940-NEXT:    global_store_dwordx4 v6, v[2:5], s[0:1] sc0 sc1
@@ -2214,17 +2186,16 @@ define void @v_shuffle_v4f32_v4f32__7_2_0_0(ptr addrspace(1) inreg %ptr) {
 ; GFX90A:       ; %bb.0:
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX90A-NEXT:    ;;#ASMSTART
-; GFX90A-NEXT:    ; def v[4:7]
+; GFX90A-NEXT:    ; def v[0:3]
 ; GFX90A-NEXT:    ;;#ASMEND
-; GFX90A-NEXT:    v_mov_b32_e32 v8, 0
 ; GFX90A-NEXT:    ;;#ASMSTART
-; GFX90A-NEXT:    ; def v[0:3]
+; GFX90A-NEXT:    ; def v[4:7]
 ; GFX90A-NEXT:    ;;#ASMEND
-; GFX90A-NEXT:    v_mov_b32_e32 v4, v7
-; GFX90A-NEXT:    v_mov_b32_e32 v5, v2
-; GFX90A-NEXT:    v_mov_b32_e32 v6, v0
-; GFX90A-NEXT:    v_mov_b32_e32 v7, v0
-; GFX90A-NEXT:    global_store_dwordx4 v8, v[4:7], s[16:17]
+; GFX90A-NEXT:    v_mov_b32_e32 v8, 0
+; GFX90A-NEXT:    v_pk_mov_b32 v[2:3], v[6:7], v[2:3] op_sel:[1,0]
+; GFX90A-NEXT:    v_mov_b32_e32 v4, v0
+; GFX90A-NEXT:    v_mov_b32_e32 v5, v0
+; GFX90A-NEXT:    global_store_dwordx4 v8, v[2:5], s[16:17]
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
 ; GFX90A-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -2232,17 +2203,16 @@ define void @v_shuffle_v4f32_v4f32__7_2_0_0(ptr addrspace(1) inreg %ptr) {
 ; GFX940:       ; %bb.0:
 ; GFX940-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX940-NEXT:    ;;#ASMSTART
-; GFX940-NEXT:    ; def v[4:7]
+; GFX940-NEXT:    ; def v[0:3]
 ; GFX940-NEXT:    ;;#ASMEND
-; GFX940-NEXT:    v_mov_b32_e32 v8, 0
 ; GFX940-NEXT:    ;;#ASMSTART
-; GFX940-NEXT:    ; def v[0:3]
+; GFX940-NEXT:    ; def v[4:7]
 ; GFX940-NEXT:    ;;#ASMEND
-; GFX940-NEXT:    v_mov_b32_e32 v4, v7
-; GFX940-NEXT:    v_mov_b32_e32 v5, v2
-; GFX940-NEXT:    v_mov_b32_e32 v6, v0
-; GFX940-NEXT:    v_mov_b32_e32 v7, v0
-; GFX940-NEXT:    global_store_dwordx4 v8, v[4:7], s[0:1] sc0 sc1
+; GFX940-NEXT:    v_mov_b32_e32 v8, 0
+; GFX940-NEXT:    v_pk_mov_b32 v[2:3], v[6:7], v[2:3] op_sel:[1,0]
+; GFX940-NEXT:    v_mov_b32_e32 v4, v0
+; GFX940-NEXT:    v_mov_b32_e32 v5, v0
+; GFX940-NEXT:    global_store_dwordx4 v8, v[2:5], s[0:1] sc0 sc1
 ; GFX940-NEXT:    s_waitcnt vmcnt(0)
 ; GFX940-NEXT:    s_setpc_b64 s[30:31]
   %vec0 = call <4 x float> asm "; def $0", "=v"()
@@ -2339,12 +2309,11 @@ define void @v_shuffle_v4f32_v4f32__7_4_0_0(ptr addrspace(1) inreg %ptr) {
 ; GFX90A-NEXT:    ;;#ASMSTART
 ; GFX90A-NEXT:    ; def v[2:5]
 ; GFX90A-NEXT:    ;;#ASMEND
-; GFX90A-NEXT:    v_mov_b32_e32 v8, 0
-; GFX90A-NEXT:    v_mov_b32_e32 v4, v5
-; GFX90A-NEXT:    v_mov_b32_e32 v5, v2
-; GFX90A-NEXT:    v_mov_b32_e32 v6, v0
-; GFX90A-NEXT:    v_mov_b32_e32 v7, v0
-; GFX90A-NEXT:    global_store_dwordx4 v8, v[4:7], s[16:17]
+; GFX90A-NEXT:    v_mov_b32_e32 v6, 0
+; GFX90A-NEXT:    v_pk_mov_b32 v[2:3], v[4:5], v[2:3] op_sel:[1,0]
+; GFX90A-NEXT:    v_mov_b32_e32 v4, v0
+; GFX90A-NEXT:    v_mov_b32_e32 v5, v0
+; GFX90A-NEXT:    global_store_dwordx4 v6, v[2:5], s[16:17]
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
 ; GFX90A-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -2354,15 +2323,15 @@ define void @v_shuffle_v4f32_v4f32__7_4_0_0(ptr addrspace(1) inreg %ptr) {
 ; GFX940-NEXT:    ;;#ASMSTART
 ; GFX940-NEXT:    ; def v[0:3]
 ; GFX940-NEXT:    ;;#ASMEND
-; GFX940-NEXT:    v_mov_b32_e32 v8, 0
+; GFX940-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX940-NEXT:    ;;#ASMSTART
 ; GFX940-NEXT:    ; def v[2:5]
 ; GFX940-NEXT:    ;;#ASMEND
-; GFX940-NEXT:    v_mov_b32_e32 v6, v0
-; GFX940-NEXT:    v_mov_b32_e32 v4, v5
-; GFX940-NEXT:    v_mov_b32_e32 v5, v2
-; GFX940-NEXT:    v_mov_b32_e32 v7, v0
-; GFX940-NEXT:    global_store_dwordx4 v8, v[4:7], s[0:1] sc0 sc1
+; GFX940-NEXT:    s_nop 0
+; GFX940-NEXT:    v_pk_mov_b32 v[2:3], v[4:5], v[2:3] op_sel:[1,0]
+; GFX940-NEXT:    v_mov_b32_e32 v4, v0
+; GFX940-NEXT:    v_mov_b32_e32 v5, v0
+; GFX940-NEXT:    global_store_dwordx4 v6, v[2:5], s[0:1] sc0 sc1
 ; GFX940-NEXT:    s_waitcnt vmcnt(0)
 ; GFX940-NEXT:    s_setpc_b64 s[30:31]
   %vec0 = call <4 x float> asm "; def $0", "=v"()
@@ -2460,8 +2429,7 @@ define void @v_shuffle_v4f32_v4f32__7_6_0_0(ptr addrspace(1) inreg %ptr) {
 ; GFX90A-NEXT:    ; def v[2:5]
 ; GFX90A-NEXT:    ;;#ASMEND
 ; GFX90A-NEXT:    v_mov_b32_e32 v6, 0
-; GFX90A-NEXT:    v_mov_b32_e32 v2, v5
-; GFX90A-NEXT:    v_mov_b32_e32 v3, v4
+; GFX90A-NEXT:    v_pk_mov_b32 v[2:3], v[4:5], v[4:5] op_sel:[1,0]
 ; GFX90A-NEXT:    v_mov_b32_e32 v4, v0
 ; GFX90A-NEXT:    v_mov_b32_e32 v5, v0
 ; GFX90A-NEXT:    global_store_dwordx4 v6, v[2:5], s[16:17]
@@ -2479,8 +2447,7 @@ define void @v_shuffle_v4f32_v4f32__7_6_0_0(ptr addrspace(1) inreg %ptr) {
 ; GFX940-NEXT:    ; def v[2:5]
 ; GFX940-NEXT:    ;;#ASMEND
 ; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_mov_b32_e32 v2, v5
-; GFX940-NEXT:    v_mov_b32_e32 v3, v4
+; GFX940-NEXT:    v_pk_mov_b32 v[2:3], v[4:5], v[4:5] op_sel:[1,0]
 ; GFX940-NEXT:    v_mov_b32_e32 v4, v0
 ; GFX940-NEXT:    v_mov_b32_e32 v5, v0
 ; GFX940-NEXT:    global_store_dwordx4 v6, v[2:5], s[0:1] sc0 sc1
@@ -2643,11 +2610,10 @@ define void @v_shuffle_v4f32_v4f32__7_7_1_0(ptr addrspace(1) inreg %ptr) {
 ; GFX90A-NEXT:    ; def v[2:5]
 ; GFX90A-NEXT:    ;;#ASMEND
 ; GFX90A-NEXT:    v_mov_b32_e32 v6, 0
-; GFX90A-NEXT:    v_mov_b32_e32 v2, v5
-; GFX90A-NEXT:    v_mov_b32_e32 v3, v5
-; GFX90A-NEXT:    v_mov_b32_e32 v4, v1
-; GFX90A-NEXT:    v_mov_b32_e32 v5, v0
-; GFX90A-NEXT:    global_store_dwordx4 v6, v[2:5], s[16:17]
+; GFX90A-NEXT:    v_pk_mov_b32 v[2:3], v[0:1], v[0:1] op_sel:[1,0]
+; GFX90A-NEXT:    v_mov_b32_e32 v0, v5
+; GFX90A-NEXT:    v_mov_b32_e32 v1, v5
+; GFX90A-NEXT:    global_store_dwordx4 v6, v[0:3], s[16:17]
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
 ; GFX90A-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -2662,11 +2628,10 @@ define void @v_shuffle_v4f32_v4f32__7_7_1_0(ptr addrspace(1) inreg %ptr) {
 ; GFX940-NEXT:    ; def v[2:5]
 ; GFX940-NEXT:    ;;#ASMEND
 ; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_mov_b32_e32 v2, v5
-; GFX940-NEXT:    v_mov_b32_e32 v3, v5
-; GFX940-NEXT:    v_mov_b32_e32 v4, v1
-; GFX940-NEXT:    v_mov_b32_e32 v5, v0
-; GFX940-NEXT:    global_store_dwordx4 v6, v[2:5], s[0:1] sc0 sc1
+; GFX940-NEXT:    v_pk_mov_b32 v[2:3], v[0:1], v[0:1] op_sel:[1,0]
+; GFX940-NEXT:    v_mov_b32_e32 v0, v5
+; GFX940-NEXT:    v_mov_b32_e32 v1, v5
+; GFX940-NEXT:    global_store_dwordx4 v6, v[0:3], s[0:1] sc0 sc1
 ; GFX940-NEXT:    s_waitcnt vmcnt(0)
 ; GFX940-NEXT:    s_setpc_b64 s[30:31]
   %vec0 = call <4 x float> asm "; def $0", "=v"()
@@ -2759,17 +2724,16 @@ define void @v_shuffle_v4f32_v4f32__7_7_3_0(ptr addrspace(1) inreg %ptr) {
 ; GFX90A:       ; %bb.0:
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX90A-NEXT:    ;;#ASMSTART
-; GFX90A-NEXT:    ; def v[4:7]
+; GFX90A-NEXT:    ; def v[0:3]
 ; GFX90A-NEXT:    ;;#ASMEND
 ; GFX90A-NEXT:    v_mov_b32_e32 v8, 0
 ; GFX90A-NEXT:    ;;#ASMSTART
-; GFX90A-NEXT:    ; def v[0:3]
+; GFX90A-NEXT:    ; def v[4:7]
 ; GFX90A-NEXT:    ;;#ASMEND
-; GFX90A-NEXT:    v_mov_b32_e32 v4, v7
-; GFX90A-NEXT:    v_mov_b32_e32 v5, v7
-; GFX90A-NEXT:    v_mov_b32_e32 v6, v3
-; GFX90A-NEXT:    v_mov_b32_e32 v7, v0
-; GFX90A-NEXT:    global_store_dwordx4 v8, v[4:7], s[16:17]
+; GFX90A-NEXT:    v_pk_mov_b32 v[2:3], v[2:3], v[0:1] op_sel:[1,0]
+; GFX90A-NEXT:    v_mov_b32_e32 v0, v7
+; GFX90A-NEXT:    v_mov_b32_e32 v1, v7
+; GFX90A-NEXT:    global_store_dwordx4 v8, v[0:3], s[16:17]
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
 ; GFX90A-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -2777,17 +2741,16 @@ define void @v_shuffle_v4f32_v4f32__7_7_3_0(ptr addrspace(1) inreg %ptr) {
 ; GFX940:       ; %bb.0:
 ; GFX940-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX940-NEXT:    ;;#ASMSTART
-; GFX940-NEXT:    ; def v[4:7]
+; GFX940-NEXT:    ; def v[0:3]
 ; GFX940-NEXT:    ;;#ASMEND
 ; GFX940-NEXT:    v_mov_b32_e32 v8, 0
 ; GFX940-NEXT:    ;;#ASMSTART
-; GFX940-NEXT:    ; def v[0:3]
+; GFX940-NEXT:    ; def v[4:7]
 ; GFX940-NEXT:    ;;#ASMEND
-; GFX940-NEXT:    v_mov_b32_e32 v4, v7
-; GFX940-NEXT:    v_mov_b32_e32 v5, v7
-; GFX940-NEXT:    v_mov_b32_e32 v6, v3
-; GFX940-NEXT:    v_mov_b32_e32 v7, v0
-; GFX940-NEXT:    global_store_dwordx4 v8, v[4:7], s[0:1] sc0 sc1
+; GFX940-NEXT:    v_pk_mov_b32 v[2:3], v[2:3], v[0:1] op_sel:[1,0]
+; GFX940-NEXT:    v_mov_b32_e32 v0, v7
+; GFX940-NEXT:    v_mov_b32_e32 v1, v7
+; GFX940-NEXT:    global_store_dwordx4 v8, v[0:3], s[0:1] sc0 sc1
 ; GFX940-NEXT:    s_waitcnt vmcnt(0)
 ; GFX940-NEXT:    s_setpc_b64 s[30:31]
   %vec0 = call <4 x float> asm "; def $0", "=v"()
@@ -2883,11 +2846,11 @@ define void @v_shuffle_v4f32_v4f32__7_7_5_0(ptr addrspace(1) inreg %ptr) {
 ; GFX90A-NEXT:    ;;#ASMSTART
 ; GFX90A-NEXT:    ; def v[2:5]
 ; GFX90A-NEXT:    ;;#ASMEND
-; GFX90A-NEXT:    v_mov_b32_e32 v8, 0
-; GFX90A-NEXT:    v_mov_b32_e32 v4, v5
-; GFX90A-NEXT:    v_mov_b32_e32 v6, v3
-; GFX90A-NEXT:    v_mov_b32_e32 v7, v0
-; GFX90A-NEXT:    global_store_dwordx4 v8, v[4:7], s[16:17]
+; GFX90A-NEXT:    v_mov_b32_e32 v6, 0
+; GFX90A-NEXT:    v_pk_mov_b32 v[2:3], v[2:3], v[0:1] op_sel:[1,0]
+; GFX90A-NEXT:    v_mov_b32_e32 v0, v5
+; GFX90A-NEXT:    v_mov_b32_e32 v1, v5
+; GFX90A-NEXT:    global_store_dwordx4 v6, v[0:3], s[16:17]
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
 ; GFX90A-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -2897,14 +2860,15 @@ define void @v_shuffle_v4f32_v4f32__7_7_5_0(ptr addrspace(1) inreg %ptr) {
 ; GFX940-NEXT:    ;;#ASMSTART
 ; GFX940-NEXT:    ; def v[0:3]
 ; GFX940-NEXT:    ;;#ASMEND
-; GFX940-NEXT:    v_mov_b32_e32 v8, 0
+; GFX940-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX940-NEXT:    ;;#ASMSTART
 ; GFX940-NEXT:    ; def v[2:5]
 ; GFX940-NEXT:    ;;#ASMEND
-; GFX940-NEXT:    v_mov_b32_e32 v7, v0
-; GFX940-NEXT:    v_mov_b32_e32 v4, v5
-; GFX940-NEXT:    v_mov_b32_e32 v6, v3
-; GFX940-NEXT:    global_store_dwordx4 v8, v[4:7], s[0:1] sc0 sc1
+; GFX940-NEXT:    s_nop 0
+; GFX940-NEXT:    v_pk_mov_b32 v[2:3], v[2:3], v[0:1] op_sel:[1,0]
+; GFX940-NEXT:    v_mov_b32_e32 v0, v5
+; GFX940-NEXT:    v_mov_b32_e32 v1, v5
+; GFX940-NEXT:    global_store_dwordx4 v6, v[0:3], s[0:1] sc0 sc1
 ; GFX940-NEXT:    s_waitcnt vmcnt(0)
 ; GFX940-NEXT:    s_setpc_b64 s[30:31]
   %vec0 = call <4 x float> asm "; def $0", "=v"()
@@ -3517,8 +3481,7 @@ define void @v_shuffle_v4f32_v4f32__7_0_1_1(ptr addrspace(1) inreg %ptr) {
 ; GFX90A-NEXT:    ; def v[2:5]
 ; GFX90A-NEXT:    ;;#ASMEND
 ; GFX90A-NEXT:    v_mov_b32_e32 v6, 0
-; GFX90A-NEXT:    v_mov_b32_e32 v2, v5
-; GFX90A-NEXT:    v_mov_b32_e32 v3, v0
+; GFX90A-NEXT:    v_pk_mov_b32 v[2:3], v[4:5], v[0:1] op_sel:[1,0]
 ; GFX90A-NEXT:    v_mov_b32_e32 v4, v1
 ; GFX90A-NEXT:    v_mov_b32_e32 v5, v1
 ; GFX90A-NEXT:    global_store_dwordx4 v6, v[2:5], s[16:17]
@@ -3536,8 +3499,7 @@ define void @v_shuffle_v4f32_v4f32__7_0_1_1(ptr addrspace(1) inreg %ptr) {
 ; GFX940-NEXT:    ; def v[2:5]
 ; GFX940-NEXT:    ;;#ASMEND
 ; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_mov_b32_e32 v2, v5
-; GFX940-NEXT:    v_mov_b32_e32 v3, v0
+; GFX940-NEXT:    v_pk_mov_b32 v[2:3], v[4:5], v[0:1] op_sel:[1,0]
 ; GFX940-NEXT:    v_mov_b32_e32 v4, v1
 ; GFX940-NEXT:    v_mov_b32_e32 v5, v1
 ; GFX940-NEXT:    global_store_dwordx4 v6, v[2:5], s[0:1] sc0 sc1
@@ -3573,17 +3535,16 @@ define void @v_shuffle_v4f32_v4f32__7_2_1_1(ptr addrspace(1) inreg %ptr) {
 ; GFX90A:       ; %bb.0:
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX90A-NEXT:    ;;#ASMSTART
-; GFX90A-NEXT:    ; def v[4:7]
+; GFX90A-NEXT:    ; def v[0:3]
 ; GFX90A-NEXT:    ;;#ASMEND
-; GFX90A-NEXT:    v_mov_b32_e32 v8, 0
 ; GFX90A-NEXT:    ;;#ASMSTART
-; GFX90A-NEXT:    ; def v[0:3]
+; GFX90A-NEXT:    ; def v[4:7]
 ; GFX90A-NEXT:    ;;#ASMEND
-; GFX90A-NEXT:    v_mov_b32_e32 v4, v7
-; GFX90A-NEXT:    v_mov_b32_e32 v5, v2
-; GFX90A-NEXT:    v_mov_b32_e32 v6, v1
-; GFX90A-NEXT:    v_mov_b32_e32 v7, v1
-; GFX90A-NEXT:    global_store_dwordx4 v8, v[4:7], s[16:17]
+; GFX90A-NEXT:    v_mov_b32_e32 v8, 0
+; GFX90A-NEXT:    v_pk_mov_b32 v[2:3], v[6:7], v[2:3] op_sel:[1,0]
+; GFX90A-NEXT:    v_mov_b32_e32 v4, v1
+; GFX90A-NEXT:    v_mov_b32_e32 v5, v1
+; GFX90A-NEXT:    global_store_dwordx4 v8, v[2:5], s[16:17]
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
 ; GFX90A-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -3591,17 +3552,16 @@ define void @v_shuffle_v4f32_v4f32__7_2_1_1(ptr addrspace(1) inreg %ptr) {
 ; GFX940:       ; %bb.0:
 ; GFX940-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX940-NEXT:    ;;#ASMSTART
-; GFX940-NEXT:    ; def v[4:7]
+; GFX940-NEXT:    ; def v[0:3]
 ; GFX940-NEXT:    ;;#ASMEND
-; GFX940-NEXT:    v_mov_b32_e32 v8, 0
 ; GFX940-NEXT:    ;;#ASMSTART
-; GFX940-NEXT:    ; def v[0:3]
+; GFX940-NEXT:    ; def v[4:7]
 ; GFX940-NEXT:    ;;#ASMEND
-; GFX940-NEXT:    v_mov_b32_e32 v4, v7
-; GFX940-NEXT:    v_mov_b32_e32 v5, v2
-; GFX940-NEXT:    v_mov_b32_e32 v6, v1
-; GFX940-NEXT:    v_mov_b32_e32 v7, v1
-; GFX940-NEXT:    global_store_dwordx4 v8, v[4:7], s[0:1] sc0 sc1
+; GFX940-NEXT:    v_mov_b32_e32 v8, 0
+; GFX940-NEXT:    v_pk_mov_b32 v[2:3], v[6:7], v[2:3] op_sel:[1,0]
+; GFX940-NEXT:    v_mov_b32_e32 v4, v1
+; GFX940-NEXT:    v_mov_b32_e32 v5, v1
+; GFX940-NEXT:    global_store_dwordx4 v8, v[2:5], s[0:1] sc0 sc1
 ; GFX940-NEXT:    s_waitcnt vmcnt(0)
 ; GFX940-NEXT:    s_setpc_b64 s[30:31]
   %vec0 = call <4 x float> asm "; def $0", "=v"()
@@ -3697,12 +3657,11 @@ define void @v_shuffle_v4f32_v4f32__7_4_1_1(ptr addrspace(1) inreg %ptr) {
 ; GFX90A-NEXT:    ;;#ASMSTART
 ; GFX90A-NEXT:    ; def v[2:5]
 ; GFX90A-NEXT:    ;;#ASMEND
-; GFX90A-NEXT:    v_mov_b32_e32 v8, 0
-; GFX90A-NEXT:    v_mov_b32_e32 v4, v5
-; GFX90A-NEXT:    v_mov_b32_e32 v5, v2
-; GFX90A-NEXT:    v_mov_b32_e32 v6, v1
-; GFX90A-NEXT:    v_mov_b32_e32 v7, v1
-; GFX90A-NEXT:    global_store_dwordx4 v8, v[4:7], s[16:17]
+; GFX90A-NEXT:    v_mov_b32_e32 v6, 0
+; GFX90A-NEXT:    v_pk_mov_b32 v[2:3], v[4:5], v[2:3] op_sel:[1,0]
+; GFX90A-NEXT:    v_mov_b32_e32 v4, v1
+; GFX90A-NEXT:    v_mov_b32_e32 v5, v1
+; GFX90A-NEXT:    global_store_dwordx4 v6, v[2:5], s[16:17]
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
 ; GFX90A-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -3712,15 +3671,15 @@ define void @v_shuffle_v4f32_v4f32__7_4_1_1(ptr addrspace(1) inreg %ptr) {
 ; GFX940-NEXT:    ;;#ASMSTART
 ; GFX940-NEXT:    ; def v[0:3]
 ; GFX940-NEXT:    ;;#ASMEND
-; GFX940-NEXT:    v_mov_b32_e32 v8, 0
+; GFX940-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX940-NEXT:    ;;#ASMSTART
 ; GFX940-NEXT:    ; def v[2:5]
 ; GFX940-NEXT:    ;;#ASMEND
-; GFX940-NEXT:    v_mov_b32_e32 v6, v1
-; GFX940-NEXT:    v_mov_b32_e32 v4, v5
-; GFX940-NEXT:    v_mov_b32_e32 v5, v2
-; GFX940-NEXT:    v_mov_b32_e32 v7, v1
-; GFX940-NEXT:    global_store_dwordx4 v8, v[4:7], s[0:1] sc0 sc1
+; GFX940-NEXT:    s_nop 0
+; GFX940-NEXT:    v_pk_mov_b32 v[2:3], v[4:5], v[2:3] op_sel:[1,0]
+; GFX940-NEXT:    v_mov_b32_e32 v4, v1
+; GFX940-NEXT:    v_mov_b32_e32 v5, v1
+; GFX940-NEXT:    global_store_dwordx4 v6, v[2:5], s[0:1] sc0 sc1
 ; GFX940-NEXT:    s_waitcnt vmcnt(0)
 ; GFX940-NEXT:    s_setpc_b64 s[30:31]
   %vec0 = call <4 x float> asm "; def $0", "=v"()
@@ -3818,8 +3777,7 @@ define void @v_shuffle_v4f32_v4f32__7_6_1_1(ptr addrspace(1) inreg %ptr) {
 ; GFX90A-NEXT:    ; def v[2:5]
 ; GFX90A-NEXT:    ;;#ASMEND
 ; GFX90A-NEXT:    v_mov_b32_e32 v6, 0
-; GFX90A-NEXT:    v_mov_b32_e32 v2, v5
-; GFX90A-NEXT:    v_mov_b32_e32 v3, v4
+; GFX90A-NEXT:    v_pk_mov_b32 v[2:3], v[4:5], v[4:5] op_sel:[1,0]
 ; GFX90A-NEXT:    v_mov_b32_e32 v4, v1
 ; GFX90A-NEXT:    v_mov_b32_e32 v5, v1
 ; GFX90A-NEXT:    global_store_dwordx4 v6, v[2:5], s[16:17]
@@ -3837,8 +3795,7 @@ define void @v_shuffle_v4f32_v4f32__7_6_1_1(ptr addrspace(1) inreg %ptr) {
 ; GFX940-NEXT:    ; def v[2:5]
 ; GFX940-NEXT:    ;;#ASMEND
 ; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_mov_b32_e32 v2, v5
-; GFX940-NEXT:    v_mov_b32_e32 v3, v4
+; GFX940-NEXT:    v_pk_mov_b32 v[2:3], v[4:5], v[4:5] op_sel:[1,0]
 ; GFX940-NEXT:    v_mov_b32_e32 v4, v1
 ; GFX940-NEXT:    v_mov_b32_e32 v5, v1
 ; GFX940-NEXT:    global_store_dwordx4 v6, v[2:5], s[0:1] sc0 sc1
@@ -4444,8 +4401,7 @@ define void @v_shuffle_v4f32_v4f32__1_2_2_2(ptr addrspace(1) inreg %ptr) {
 ; GFX90A-NEXT:    ; def v[0:3]
 ; GFX90A-NEXT:    ;;#ASMEND
 ; GFX90A-NEXT:    v_mov_b32_e32 v4, 0
-; GFX90A-NEXT:    v_mov_b32_e32 v0, v1
-; GFX90A-NEXT:    v_mov_b32_e32 v1, v2
+; GFX90A-NEXT:    v_pk_mov_b32 v[0:1], v[0:1], v[2:3] op_sel:[1,0]
 ; GFX90A-NEXT:    v_mov_b32_e32 v3, v2
 ; GFX90A-NEXT:    global_store_dwordx4 v4, v[0:3], s[16:17]
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
@@ -4458,8 +4414,7 @@ define void @v_shuffle_v4f32_v4f32__1_2_2_2(ptr addrspace(1) inreg %ptr) {
 ; GFX940-NEXT:    ; def v[0:3]
 ; GFX940-NEXT:    ;;#ASMEND
 ; GFX940-NEXT:    v_mov_b32_e32 v4, 0
-; GFX940-NEXT:    v_mov_b32_e32 v0, v1
-; GFX940-NEXT:    v_mov_b32_e32 v1, v2
+; GFX940-NEXT:    v_pk_mov_b32 v[0:1], v[0:1], v[2:3] op_sel:[1,0]
 ; GFX940-NEXT:    v_mov_b32_e32 v3, v2
 ; GFX940-NEXT:    global_store_dwordx4 v4, v[0:3], s[0:1] sc0 sc1
 ; GFX940-NEXT:    s_waitcnt vmcnt(0)
@@ -4540,8 +4495,7 @@ define void @v_shuffle_v4f32_v4f32__3_2_2_2(ptr addrspace(1) inreg %ptr) {
 ; GFX90A-NEXT:    ; def v[0:3]
 ; GFX90A-NEXT:    ;;#ASMEND
 ; GFX90A-NEXT:    v_mov_b32_e32 v4, 0
-; GFX90A-NEXT:    v_mov_b32_e32 v0, v3
-; GFX90A-NEXT:    v_mov_b32_e32 v1, v2
+; GFX90A-NEXT:    v_pk_mov_b32 v[0:1], v[2:3], v[2:3] op_sel:[1,0]
 ; GFX90A-NEXT:    v_mov_b32_e32 v3, v2
 ; GFX90A-NEXT:    global_store_dwordx4 v4, v[0:3], s[16:17]
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
@@ -4554,8 +4508,7 @@ define void @v_shuffle_v4f32_v4f32__3_2_2_2(ptr addrspace(1) inreg %ptr) {
 ; GFX940-NEXT:    ; def v[0:3]
 ; GFX940-NEXT:    ;;#ASMEND
 ; GFX940-NEXT:    v_mov_b32_e32 v4, 0
-; GFX940-NEXT:    v_mov_b32_e32 v0, v3
-; GFX940-NEXT:    v_mov_b32_e32 v1, v2
+; GFX940-NEXT:    v_pk_mov_b32 v[0:1], v[2:3], v[2:3] op_sel:[1,0]
 ; GFX940-NEXT:    v_mov_b32_e32 v3, v2
 ; GFX940-NEXT:    global_store_dwordx4 v4, v[0:3], s[0:1] sc0 sc1
 ; GFX940-NEXT:    s_waitcnt vmcnt(0)
@@ -4639,8 +4592,7 @@ define void @v_shuffle_v4f32_v4f32__5_2_2_2(ptr addrspace(1) inreg %ptr) {
 ; GFX90A-NEXT:    ;;#ASMSTART
 ; GFX90A-NEXT:    ; def v[4:7]
 ; GFX90A-NEXT:    ;;#ASMEND
-; GFX90A-NEXT:    v_mov_b32_e32 v0, v5
-; GFX90A-NEXT:    v_mov_b32_e32 v1, v2
+; GFX90A-NEXT:    v_pk_mov_b32 v[0:1], v[4:5], v[2:3] op_sel:[1,0]
 ; GFX90A-NEXT:    v_mov_b32_e32 v3, v2
 ; GFX90A-NEXT:    global_store_dwordx4 v8, v[0:3], s[16:17]
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
@@ -4656,8 +4608,8 @@ define void @v_shuffle_v4f32_v4f32__5_2_2_2(ptr addrspace(1) inreg %ptr) {
 ; GFX940-NEXT:    ;;#ASMSTART
 ; GFX940-NEXT:    ; def v[4:7]
 ; GFX940-NEXT:    ;;#ASMEND
-; GFX940-NEXT:    v_mov_b32_e32 v1, v2
-; GFX940-NEXT:    v_mov_b32_e32 v0, v5
+; GFX940-NEXT:    s_nop 0
+; GFX940-NEXT:    v_pk_mov_b32 v[0:1], v[4:5], v[2:3] op_sel:[1,0]
 ; GFX940-NEXT:    v_mov_b32_e32 v3, v2
 ; GFX940-NEXT:    global_store_dwordx4 v8, v[0:3], s[0:1] sc0 sc1
 ; GFX940-NEXT:    s_waitcnt vmcnt(0)
@@ -4755,8 +4707,7 @@ define void @v_shuffle_v4f32_v4f32__7_2_2_2(ptr addrspace(1) inreg %ptr) {
 ; GFX90A-NEXT:    ;;#ASMSTART
 ; GFX90A-NEXT:    ; def v[4:7]
 ; GFX90A-NEXT:    ;;#ASMEND
-; GFX90A-NEXT:    v_mov_b32_e32 v0, v7
-; GFX90A-NEXT:    v_mov_b32_e32 v1, v2
+; GFX90A-NEXT:    v_pk_mov_b32 v[0:1], v[6:7], v[2:3] op_sel:[1,0]
 ; GFX90A-NEXT:    v_mov_b32_e32 v3, v2
 ; GFX90A-NEXT:    global_store_dwordx4 v8, v[0:3], s[16:17]
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
@@ -4772,8 +4723,8 @@ define void @v_shuffle_v4f32_v4f32__7_2_2_2(ptr addrspace(1) inreg %ptr) {
 ; GFX940-NEXT:    ;;#ASMSTART
 ; GFX940-NEXT:    ; def v[4:7]
 ; GFX940-NEXT:    ;;#ASMEND
-; GFX940-NEXT:    v_mov_b32_e32 v1, v2
-; GFX940-NEXT:    v_mov_b32_e32 v0, v7
+; GFX940-NEXT:    s_nop 0
+; GFX940-NEXT:    v_pk_mov_b32 v[0:1], v[6:7], v[2:3] op_sel:[1,0]
 ; GFX940-NEXT:    v_mov_b32_e32 v3, v2
 ; GFX940-NEXT:    global_store_dwordx4 v8, v[0:3], s[0:1] sc0 sc1
 ; GFX940-NEXT:    s_waitcnt vmcnt(0)
@@ -4863,17 +4814,15 @@ define void @v_shuffle_v4f32_v4f32__7_0_2_2(ptr addrspace(1) inreg %ptr) {
 ; GFX90A:       ; %bb.0:
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX90A-NEXT:    ;;#ASMSTART
-; GFX90A-NEXT:    ; def v[4:7]
+; GFX90A-NEXT:    ; def v[0:3]
 ; GFX90A-NEXT:    ;;#ASMEND
 ; GFX90A-NEXT:    v_mov_b32_e32 v8, 0
 ; GFX90A-NEXT:    ;;#ASMSTART
-; GFX90A-NEXT:    ; def v[0:3]
+; GFX90A-NEXT:    ; def v[4:7]
 ; GFX90A-NEXT:    ;;#ASMEND
-; GFX90A-NEXT:    v_mov_b32_e32 v4, v7
-; GFX90A-NEXT:    v_mov_b32_e32 v5, v0
-; GFX90A-NEXT:    v_mov_b32_e32 v6, v2
-; GFX90A-NEXT:    v_mov_b32_e32 v7, v2
-; GFX90A-NEXT:    global_store_dwordx4 v8, v[4:7], s[16:17]
+; GFX90A-NEXT:    v_pk_mov_b32 v[0:1], v[6:7], v[0:1] op_sel:[1,0]
+; GFX90A-NEXT:    v_mov_b32_e32 v3, v2
+; GFX90A-NEXT:    global_store_dwordx4 v8, v[0:3], s[16:17]
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
 ; GFX90A-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -4881,17 +4830,15 @@ define void @v_shuffle_v4f32_v4f32__7_0_2_2(ptr addrspace(1) inreg %ptr) {
 ; GFX940:       ; %bb.0:
 ; GFX940-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX940-NEXT:    ;;#ASMSTART
-; GFX940-NEXT:    ; def v[4:7]
+; GFX940-NEXT:    ; def v[0:3]
 ; GFX940-NEXT:    ;;#ASMEND
 ; GFX940-NEXT:    v_mov_b32_e32 v8, 0
 ; GFX940-NEXT:    ;;#ASMSTART
-; GFX940-NEXT:    ; def v[0:3]
+; GFX940-NEXT:    ; def v[4:7]
 ; GFX940-NEXT:    ;;#ASMEND
-; GFX940-NEXT:    v_mov_b32_e32 v4, v7
-; GFX940-NEXT:    v_mov_b32_e32 v5, v0
-; GFX940-NEXT:    v_mov_b32_e32 v6, v2
-; GFX940-NEXT:    v_mov_b32_e32 v7, v2
-; GFX940-NEXT:    global_store_dwordx4 v8, v[4:7], s[0:1] sc0 sc1
+; GFX940-NEXT:    v_mov_b32_e32 v3, v2
+; GFX940-NEXT:    v_pk_mov_b32 v[0:1], v[6:7], v[0:1] op_sel:[1,0]
+; GFX940-NEXT:    global_store_dwordx4 v8, v[0:3], s[0:1] sc0 sc1
 ; GFX940-NEXT:    s_waitcnt vmcnt(0)
 ; GFX940-NEXT:    s_setpc_b64 s[30:31]
   %vec0 = call <4 x float> asm "; def $0", "=v"()
@@ -5042,8 +4989,7 @@ define void @v_shuffle_v4f32_v4f32__7_4_2_2(ptr addrspace(1) inreg %ptr) {
 ; GFX90A-NEXT:    ;;#ASMSTART
 ; GFX90A-NEXT:    ; def v[4:7]
 ; GFX90A-NEXT:    ;;#ASMEND
-; GFX90A-NEXT:    v_mov_b32_e32 v0, v7
-; GFX90A-NEXT:    v_mov_b32_e32 v1, v4
+; GFX90A-NEXT:    v_pk_mov_b32 v[0:1], v[6:7], v[4:5] op_sel:[1,0]
 ; GFX90A-NEXT:    v_mov_b32_e32 v3, v2
 ; GFX90A-NEXT:    global_store_dwordx4 v8, v[0:3], s[16:17]
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
@@ -5060,8 +5006,7 @@ define void @v_shuffle_v4f32_v4f32__7_4_2_2(ptr addrspace(1) inreg %ptr) {
 ; GFX940-NEXT:    ; def v[4:7]
 ; GFX940-NEXT:    ;;#ASMEND
 ; GFX940-NEXT:    v_mov_b32_e32 v3, v2
-; GFX940-NEXT:    v_mov_b32_e32 v0, v7
-; GFX940-NEXT:    v_mov_b32_e32 v1, v4
+; GFX940-NEXT:    v_pk_mov_b32 v[0:1], v[6:7], v[4:5] op_sel:[1,0]
 ; GFX940-NEXT:    global_store_dwordx4 v8, v[0:3], s[0:1] sc0 sc1
 ; GFX940-NEXT:    s_waitcnt vmcnt(0)
 ; GFX940-NEXT:    s_setpc_b64 s[30:31]
@@ -5158,8 +5103,7 @@ define void @v_shuffle_v4f32_v4f32__7_6_2_2(ptr addrspace(1) inreg %ptr) {
 ; GFX90A-NEXT:    ;;#ASMSTART
 ; GFX90A-NEXT:    ; def v[4:7]
 ; GFX90A-NEXT:    ;;#ASMEND
-; GFX90A-NEXT:    v_mov_b32_e32 v0, v7
-; GFX90A-NEXT:    v_mov_b32_e32 v1, v6
+; GFX90A-NEXT:    v_pk_mov_b32 v[0:1], v[6:7], v[6:7] op_sel:[1,0]
 ; GFX90A-NEXT:    v_mov_b32_e32 v3, v2
 ; GFX90A-NEXT:    global_store_dwordx4 v8, v[0:3], s[16:17]
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
@@ -5176,8 +5120,7 @@ define void @v_shuffle_v4f32_v4f32__7_6_2_2(ptr addrspace(1) inreg %ptr) {
 ; GFX940-NEXT:    ; def v[4:7]
 ; GFX940-NEXT:    ;;#ASMEND
 ; GFX940-NEXT:    v_mov_b32_e32 v3, v2
-; GFX940-NEXT:    v_mov_b32_e32 v0, v7
-; GFX940-NEXT:    v_mov_b32_e32 v1, v6
+; GFX940-NEXT:    v_pk_mov_b32 v[0:1], v[6:7], v[6:7] op_sel:[1,0]
 ; GFX940-NEXT:    global_store_dwordx4 v8, v[0:3], s[0:1] sc0 sc1
 ; GFX940-NEXT:    s_waitcnt vmcnt(0)
 ; GFX940-NEXT:    s_setpc_b64 s[30:31]
@@ -5388,17 +5331,16 @@ define void @v_shuffle_v4f32_v4f32__7_7_1_2(ptr addrspace(1) inreg %ptr) {
 ; GFX90A:       ; %bb.0:
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX90A-NEXT:    ;;#ASMSTART
-; GFX90A-NEXT:    ; def v[4:7]
+; GFX90A-NEXT:    ; def v[0:3]
 ; GFX90A-NEXT:    ;;#ASMEND
 ; GFX90A-NEXT:    v_mov_b32_e32 v8, 0
 ; GFX90A-NEXT:    ;;#ASMSTART
-; GFX90A-NEXT:    ; def v[0:3]
+; GFX90A-NEXT:    ; def v[4:7]
 ; GFX90A-NEXT:    ;;#ASMEND
-; GFX90A-NEXT:    v_mov_b32_e32 v4, v7
-; GFX90A-NEXT:    v_mov_b32_e32 v5, v7
-; GFX90A-NEXT:    v_mov_b32_e32 v6, v1
-; GFX90A-NEXT:    v_mov_b32_e32 v7, v2
-; GFX90A-NEXT:    global_store_dwordx4 v8, v[4:7], s[16:17]
+; GFX90A-NEXT:    v_pk_mov_b32 v[2:3], v[0:1], v[2:3] op_sel:[1,0]
+; GFX90A-NEXT:    v_mov_b32_e32 v0, v7
+; GFX90A-NEXT:    v_mov_b32_e32 v1, v7
+; GFX90A-NEXT:    global_store_dwordx4 v8, v[0:3], s[16:17]
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
 ; GFX90A-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -5406,17 +5348,16 @@ define void @v_shuffle_v4f32_v4f32__7_7_1_2(ptr addrspace(1) inreg %ptr) {
 ; GFX940:       ; %bb.0:
 ; GFX940-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX940-NEXT:    ;;#ASMSTART
-; GFX940-NEXT:    ; def v[4:7]
+; GFX940-NEXT:    ; def v[0:3]
 ; GFX940-NEXT:    ;;#ASMEND
 ; GFX940-NEXT:    v_mov_b32_e32 v8, 0
 ; GFX940-NEXT:    ;;#ASMSTART
-; GFX940-NEXT:    ; def v[0:3]
+; GFX940-NEXT:    ; def v[4:7]
 ; GFX940-NEXT:    ;;#ASMEND
-; GFX940-NEXT:    v_mov_b32_e32 v4, v7
-; GFX940-NEXT:    v_mov_b32_e32 v5, v7
-; GFX940-NEXT:    v_mov_b32_e32 v6, v1
-; GFX940-NEXT:    v_mov_b32_e32 v7, v2
-; GFX940-NEXT:    global_store_dwordx4 v8, v[4:7], s[0:1] sc0 sc1
+; GFX940-NEXT:    v_pk_mov_b32 v[2:3], v[0:1], v[2:3] op_sel:[1,0]
+; GFX940-NEXT:    v_mov_b32_e32 v0, v7
+; GFX940-NEXT:    v_mov_b32_e32 v1, v7
+; GFX940-NEXT:    global_store_dwordx4 v8, v[0:3], s[0:1] sc0 sc1
 ; GFX940-NEXT:    s_waitcnt vmcnt(0)
 ; GFX940-NEXT:    s_setpc_b64 s[30:31]
   %vec0 = call <4 x float> asm "; def $0", "=v"()
@@ -5449,17 +5390,16 @@ define void @v_shuffle_v4f32_v4f32__7_7_3_2(ptr addrspace(1) inreg %ptr) {
 ; GFX90A:       ; %bb.0:
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX90A-NEXT:    ;;#ASMSTART
-; GFX90A-NEXT:    ; def v[4:7]
+; GFX90A-NEXT:    ; def v[0:3]
 ; GFX90A-NEXT:    ;;#ASMEND
 ; GFX90A-NEXT:    v_mov_b32_e32 v8, 0
 ; GFX90A-NEXT:    ;;#ASMSTART
-; GFX90A-NEXT:    ; def v[0:3]
+; GFX90A-NEXT:    ; def v[4:7]
 ; GFX90A-NEXT:    ;;#ASMEND
-; GFX90A-NEXT:    v_mov_b32_e32 v4, v7
-; GFX90A-NEXT:    v_mov_b32_e32 v5, v7
-; GFX90A-NEXT:    v_mov_b32_e32 v6, v3
-; GFX90A-NEXT:    v_mov_b32_e32 v7, v2
-; GFX90A-NEXT:    global_store_dwordx4 v8, v[4:7], s[16:17]
+; GFX90A-NEXT:    v_pk_mov_b32 v[2:3], v[2:3], v[2:3] op_sel:[1,0]
+; GFX90A-NEXT:    v_mov_b32_e32 v0, v7
+; GFX90A-NEXT:    v_mov_b32_e32 v1, v7
+; GFX90A-NEXT:    global_store_dwordx4 v8, v[0:3], s[16:17]
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
 ; GFX90A-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -5467,17 +5407,16 @@ define void @v_shuffle_v4f32_v4f32__7_7_3_2(ptr addrspace(1) inreg %ptr) {
 ; GFX940:       ; %bb.0:
 ; GFX940-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX940-NEXT:    ;;#ASMSTART
-; GFX940-NEXT:    ; def v[4:7]
+; GFX940-NEXT:    ; def v[0:3]
 ; GFX940-NEXT:    ;;#ASMEND
 ; GFX940-NEXT:    v_mov_b32_e32 v8, 0
 ; GFX940-NEXT:    ;;#ASMSTART
-; GFX940-NEXT:    ; def v[0:3]
+; GFX940-NEXT:    ; def v[4:7]
 ; GFX940-NEXT:    ;;#ASMEND
-; GFX940-NEXT:    v_mov_b32_e32 v4, v7
-; GFX940-NEXT:    v_mov_b32_e32 v5, v7
-; GFX940-NEXT:    v_mov_b32_e32 v6, v3
-; GFX940-NEXT:    v_mov_b32_e32 v7, v2
-; GFX940-NEXT:    global_store_dwordx4 v8, v[4:7], s[0:1] sc0 sc1
+; GFX940-NEXT:    v_pk_mov_b32 v[2:3], v[2:3], v[2:3] op_sel:[1,0]
+; GFX940-NEXT:    v_mov_b32_e32 v0, v7
+; GFX940-NEXT:    v_mov_b32_e32 v1, v7
+; GFX940-NEXT:    global_store_dwordx4 v8, v[0:3], s[0:1] sc0 sc1
 ; GFX940-NEXT:    s_waitcnt vmcnt(0)
 ; GFX940-NEXT:    s_setpc_b64 s[30:31]
   %vec0 = call <4 x float> asm "; def $0", "=v"()
@@ -5568,16 +5507,16 @@ define void @v_shuffle_v4f32_v4f32__7_7_5_2(ptr addrspace(1) inreg %ptr) {
 ; GFX90A:       ; %bb.0:
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX90A-NEXT:    ;;#ASMSTART
-; GFX90A-NEXT:    ; def v[4:7]
+; GFX90A-NEXT:    ; def v[0:3]
 ; GFX90A-NEXT:    ;;#ASMEND
-; GFX90A-NEXT:    v_mov_b32_e32 v10, 0
+; GFX90A-NEXT:    v_mov_b32_e32 v8, 0
 ; GFX90A-NEXT:    ;;#ASMSTART
-; GFX90A-NEXT:    ; def v[0:3]
+; GFX90A-NEXT:    ; def v[4:7]
 ; GFX90A-NEXT:    ;;#ASMEND
-; GFX90A-NEXT:    v_mov_b32_e32 v6, v7
-; GFX90A-NEXT:    v_mov_b32_e32 v8, v5
-; GFX90A-NEXT:    v_mov_b32_e32 v9, v2
-; GFX90A-NEXT:    global_store_dwordx4 v10, v[6:9], s[16:17]
+; GFX90A-NEXT:    v_pk_mov_b32 v[2:3], v[4:5], v[2:3] op_sel:[1,0]
+; GFX90A-NEXT:    v_mov_b32_e32 v0, v7
+; GFX90A-NEXT:    v_mov_b32_e32 v1, v7
+; GFX90A-NEXT:    global_store_dwordx4 v8, v[0:3], s[16:17]
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
 ; GFX90A-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -5585,16 +5524,17 @@ define void @v_shuffle_v4f32_v4f32__7_7_5_2(ptr addrspace(1) inreg %ptr) {
 ; GFX940:       ; %bb.0:
 ; GFX940-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX940-NEXT:    ;;#ASMSTART
-; GFX940-NEXT:    ; def v[4:7]
+; GFX940-NEXT:    ; def v[0:3]
 ; GFX940-NEXT:    ;;#ASMEND
-; GFX940-NEXT:    v_mov_b32_e32 v10, 0
+; GFX940-NEXT:    v_mov_b32_e32 v8, 0
 ; GFX940-NEXT:    ;;#ASMSTART
-; GFX940-NEXT:    ; def v[0:3]
+; GFX940-NEXT:    ; def v[4:7]
 ; GFX940-NEXT:    ;;#ASMEND
-; GFX940-NEXT:    v_mov_b32_e32 v6, v7
-; GFX940-NEXT:    v_mov_b32_e32 v8, v5
-; GFX940-NEXT:    v_mov_b32_e32 v9, v2
-; GFX940-NEXT:    global_store_dwordx4 v10, v[6:9], s[0:1] sc0 sc1
+; GFX940-NEXT:    s_nop 0
+; GFX940-NEXT:    v_pk_mov_b32 v[2:3], v[4:5], v[2:3] op_sel:[1,0]
+; GFX940-NEXT:    v_mov_b32_e32 v0, v7
+; GFX940-NEXT:    v_mov_b32_e32 v1, v7
+; GFX940-NEXT:    global_store_dwordx4 v8, v[0:3], s[0:1] sc0 sc1
 ; GFX940-NEXT:    s_waitcnt vmcnt(0)
 ; GFX940-NEXT:    s_setpc_b64 s[30:31]
   %vec0 = call <4 x float> asm "; def $0", "=v"()
@@ -6193,17 +6133,15 @@ define void @v_shuffle_v4f32_v4f32__7_0_3_3(ptr addrspace(1) inreg %ptr) {
 ; GFX90A:       ; %bb.0:
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX90A-NEXT:    ;;#ASMSTART
-; GFX90A-NEXT:    ; def v[4:7]
+; GFX90A-NEXT:    ; def v[0:3]
 ; GFX90A-NEXT:    ;;#ASMEND
 ; GFX90A-NEXT:    v_mov_b32_e32 v8, 0
 ; GFX90A-NEXT:    ;;#ASMSTART
-; GFX90A-NEXT:    ; def v[0:3]
+; GFX90A-NEXT:    ; def v[4:7]
 ; GFX90A-NEXT:    ;;#ASMEND
-; GFX90A-NEXT:    v_mov_b32_e32 v4, v7
-; GFX90A-NEXT:    v_mov_b32_e32 v5, v0
-; GFX90A-NEXT:    v_mov_b32_e32 v6, v3
-; GFX90A-NEXT:    v_mov_b32_e32 v7, v3
-; GFX90A-NEXT:    global_store_dwordx4 v8, v[4:7], s[16:17]
+; GFX90A-NEXT:    v_pk_mov_b32 v[0:1], v[6:7], v[0:1] op_sel:[1,0]
+; GFX90A-NEXT:    v_mov_b32_e32 v2, v3
+; GFX90A-NEXT:    global_store_dwordx4 v8, v[0:3], s[16:17]
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
 ; GFX90A-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -6211,17 +6149,15 @@ define void @v_shuffle_v4f32_v4f32__7_0_3_3(ptr addrspace(1) inreg %ptr) {
 ; GFX940:       ; %bb.0:
 ; GFX940-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX940-NEXT:    ;;#ASMSTART
-; GFX940-NEXT:    ; def v[4:7]
+; GFX940-NEXT:    ; def v[0:3]
 ; GFX940-NEXT:    ;;#ASMEND
 ; GFX940-NEXT:    v_mov_b32_e32 v8, 0
 ; GFX940-NEXT:    ;;#ASMSTART
-; GFX940-NEXT:    ; def v[0:3]
+; GFX940-NEXT:    ; def v[4:7]
 ; GFX940-NEXT:    ;;#ASMEND
-; GFX940-NEXT:    v_mov_b32_e32 v4, v7
-; GFX940-NEXT:    v_mov_b32_e32 v5, v0
-; GFX940-NEXT:    v_mov_b32_e32 v6, v3
-; GFX940-NEXT:    v_mov_b32_e32 v7, v3
-; GFX940-NEXT:    global_store_dwordx4 v8, v[4:7], s[0:1] sc0 sc1
+; GFX940-NEXT:    v_mov_b32_e32 v2, v3
+; GFX940-NEXT:    v_pk_mov_b32 v[0:1], v[6:7], v[0:1] op_sel:[1,0]
+; GFX940-NEXT:    global_store_dwordx4 v8, v[0:3], s[0:1] sc0 sc1
 ; GFX940-NEXT:    s_waitcnt vmcnt(0)
 ; GFX940-NEXT:    s_setpc_b64 s[30:31]
   %vec0 = call <4 x float> asm "; def $0", "=v"()
@@ -6314,8 +6250,7 @@ define void @v_shuffle_v4f32_v4f32__7_2_3_3(ptr addrspace(1) inreg %ptr) {
 ; GFX90A-NEXT:    ;;#ASMSTART
 ; GFX90A-NEXT:    ; def v[4:7]
 ; GFX90A-NEXT:    ;;#ASMEND
-; GFX90A-NEXT:    v_mov_b32_e32 v0, v7
-; GFX90A-NEXT:    v_mov_b32_e32 v1, v2
+; GFX90A-NEXT:    v_pk_mov_b32 v[0:1], v[6:7], v[2:3] op_sel:[1,0]
 ; GFX90A-NEXT:    v_mov_b32_e32 v2, v3
 ; GFX90A-NEXT:    global_store_dwordx4 v8, v[0:3], s[16:17]
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
@@ -6331,8 +6266,8 @@ define void @v_shuffle_v4f32_v4f32__7_2_3_3(ptr addrspace(1) inreg %ptr) {
 ; GFX940-NEXT:    ;;#ASMSTART
 ; GFX940-NEXT:    ; def v[4:7]
 ; GFX940-NEXT:    ;;#ASMEND
-; GFX940-NEXT:    v_mov_b32_e32 v1, v2
-; GFX940-NEXT:    v_mov_b32_e32 v0, v7
+; GFX940-NEXT:    s_nop 0
+; GFX940-NEXT:    v_pk_mov_b32 v[0:1], v[6:7], v[2:3] op_sel:[1,0]
 ; GFX940-NEXT:    v_mov_b32_e32 v2, v3
 ; GFX940-NEXT:    global_store_dwordx4 v8, v[0:3], s[0:1] sc0 sc1
 ; GFX940-NEXT:    s_waitcnt vmcnt(0)
@@ -6372,8 +6307,7 @@ define void @v_shuffle_v4f32_v4f32__7_4_3_3(ptr addrspace(1) inreg %ptr) {
 ; GFX90A-NEXT:    ;;#ASMSTART
 ; GFX90A-NEXT:    ; def v[4:7]
 ; GFX90A-NEXT:    ;;#ASMEND
-; GFX90A-NEXT:    v_mov_b32_e32 v0, v7
-; GFX90A-NEXT:    v_mov_b32_e32 v1, v4
+; GFX90A-NEXT:    v_pk_mov_b32 v[0:1], v[6:7], v[4:5] op_sel:[1,0]
 ; GFX90A-NEXT:    v_mov_b32_e32 v2, v3
 ; GFX90A-NEXT:    global_store_dwordx4 v8, v[0:3], s[16:17]
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
@@ -6390,8 +6324,7 @@ define void @v_shuffle_v4f32_v4f32__7_4_3_3(ptr addrspace(1) inreg %ptr) {
 ; GFX940-NEXT:    ; def v[4:7]
 ; GFX940-NEXT:    ;;#ASMEND
 ; GFX940-NEXT:    v_mov_b32_e32 v2, v3
-; GFX940-NEXT:    v_mov_b32_e32 v0, v7
-; GFX940-NEXT:    v_mov_b32_e32 v1, v4
+; GFX940-NEXT:    v_pk_mov_b32 v[0:1], v[6:7], v[4:5] op_sel:[1,0]
 ; GFX940-NEXT:    global_store_dwordx4 v8, v[0:3], s[0:1] sc0 sc1
 ; GFX940-NEXT:    s_waitcnt vmcnt(0)
 ; GFX940-NEXT:    s_setpc_b64 s[30:31]
@@ -6488,8 +6421,7 @@ define void @v_shuffle_v4f32_v4f32__7_6_3_3(ptr addrspace(1) inreg %ptr) {
 ; GFX90A-NEXT:    ;;#ASMSTART
 ; GFX90A-NEXT:    ; def v[4:7]
 ; GFX90A-NEXT:    ;;#ASMEND
-; GFX90A-NEXT:    v_mov_b32_e32 v0, v7
-; GFX90A-NEXT:    v_mov_b32_e32 v1, v6
+; GFX90A-NEXT:    v_pk_mov_b32 v[0:1], v[6:7], v[6:7] op_sel:[1,0]
 ; GFX90A-NEXT:    v_mov_b32_e32 v2, v3
 ; GFX90A-NEXT:    global_store_dwordx4 v8, v[0:3], s[16:17]
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
@@ -6506,8 +6438,7 @@ define void @v_shuffle_v4f32_v4f32__7_6_3_3(ptr addrspace(1) inreg %ptr) {
 ; GFX940-NEXT:    ; def v[4:7]
 ; GFX940-NEXT:    ;;#ASMEND
 ; GFX940-NEXT:    v_mov_b32_e32 v2, v3
-; GFX940-NEXT:    v_mov_b32_e32 v0, v7
-; GFX940-NEXT:    v_mov_b32_e32 v1, v6
+; GFX940-NEXT:    v_pk_mov_b32 v[0:1], v[6:7], v[6:7] op_sel:[1,0]
 ; GFX940-NEXT:    global_store_dwordx4 v8, v[0:3], s[0:1] sc0 sc1
 ; GFX940-NEXT:    s_waitcnt vmcnt(0)
 ; GFX940-NEXT:    s_setpc_b64 s[30:31]
@@ -7195,8 +7126,7 @@ define void @v_shuffle_v4f32_v4f32__5_4_4_4(ptr addrspace(1) inreg %ptr) {
 ; GFX90A-NEXT:    ; def v[0:3]
 ; GFX90A-NEXT:    ;;#ASMEND
 ; GFX90A-NEXT:    v_mov_b32_e32 v6, 0
-; GFX90A-NEXT:    v_mov_b32_e32 v2, v1
-; GFX90A-NEXT:    v_mov_b32_e32 v3, v0
+; GFX90A-NEXT:    v_pk_mov_b32 v[2:3], v[0:1], v[0:1] op_sel:[1,0]
 ; GFX90A-NEXT:    v_mov_b32_e32 v4, v0
 ; GFX90A-NEXT:    v_mov_b32_e32 v5, v0
 ; GFX90A-NEXT:    global_store_dwordx4 v6, v[2:5], s[16:17]
@@ -7210,8 +7140,7 @@ define void @v_shuffle_v4f32_v4f32__5_4_4_4(ptr addrspace(1) inreg %ptr) {
 ; GFX940-NEXT:    ; def v[0:3]
 ; GFX940-NEXT:    ;;#ASMEND
 ; GFX940-NEXT:    v_mov_b32_e32 v6, 0
-; GFX940-NEXT:    v_mov_b32_e32 v2, v1
-; GFX940-NEXT:    v_mov_b32_e32 v3, v0
+; GFX940-NEXT:    v_pk_mov_b32 v[2:3], v[0:1], v[0:1] op_sel:[1,0]
 ; GFX940-NEXT:    v_mov_b32_e32 v4, v0
 ; GFX940-NEXT:    v_mov_b32_e32 v5, v0
 ; GFX940-NEXT:    global_store_dwordx4 v6, v[2:5], s[0:1] sc0 sc1
@@ -7297,8 +7226,7 @@ define void @v_shuffle_v4f32_v4f32__7_4_4_4(ptr addrspace(1) inreg %ptr) {
 ; GFX90A-NEXT:    ; def v[0:3]
 ; GFX90A-NEXT:    ;;#ASMEND
 ; GFX90A-NEXT:    v_mov_b32_e32 v6, 0
-; GFX90A-NEXT:    v_mov_b32_e32 v2, v3
-; GFX90A-NEXT:    v_mov_b32_e32 v3, v0
+; GFX90A-NEXT:    v_pk_mov_b32 v[2:3], v[2:3], v[0:1] op_sel:[1,0]
 ; GFX90A-NEXT:    v_mov_b32_e32 v4, v0
 ; GFX90A-NEXT:    v_mov_b32_e32 v5, v0
 ; GFX90A-NEXT:    global_store_dwordx4 v6, v[2:5], s[16:17]
@@ -7312,8 +7240,7 @@ define void @v_shuffle_v4f32_v4f32__7_4_4_4(ptr addrspace(1) inreg %ptr) {
 ; GFX940-NEXT:    ; def v[0:3]
 ; GFX940-NEXT:    ;;#ASMEND
 ; GFX940-NEXT:    v_mov_b32_e32 v6, 0
-; GFX940-NEXT:    v_mov_b32_e32 v2, v3
-; GFX940-NEXT:    v_mov_b32_e32 v3, v0
+; GFX940-NEXT:    v_pk_mov_b32 v[2:3], v[2:3], v[0:1] op_sel:[1,0]
 ; GFX940-NEXT:    v_mov_b32_e32 v4, v0
 ; GFX940-NEXT:    v_mov_b32_e32 v5, v0
 ; GFX940-NEXT:    global_store_dwordx4 v6, v[2:5], s[0:1] sc0 sc1
@@ -7404,8 +7331,7 @@ define void @v_shuffle_v4f32_v4f32__7_0_4_4(ptr addrspace(1) inreg %ptr) {
 ; GFX90A-NEXT:    ; def v[2:5]
 ; GFX90A-NEXT:    ;;#ASMEND
 ; GFX90A-NEXT:    v_mov_b32_e32 v8, 0
-; GFX90A-NEXT:    v_mov_b32_e32 v4, v5
-; GFX90A-NEXT:    v_mov_b32_e32 v5, v0
+; GFX90A-NEXT:    v_pk_mov_b32 v[4:5], v[4:5], v[0:1] op_sel:[1,0]
 ; GFX90A-NEXT:    v_mov_b32_e32 v6, v2
 ; GFX90A-NEXT:    v_mov_b32_e32 v7, v2
 ; GFX90A-NEXT:    global_store_dwordx4 v8, v[4:7], s[16:17]
@@ -7423,8 +7349,7 @@ define void @v_shuffle_v4f32_v4f32__7_0_4_4(ptr addrspace(1) inreg %ptr) {
 ; GFX940-NEXT:    ; def v[2:5]
 ; GFX940-NEXT:    ;;#ASMEND
 ; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_mov_b32_e32 v4, v5
-; GFX940-NEXT:    v_mov_b32_e32 v5, v0
+; GFX940-NEXT:    v_pk_mov_b32 v[4:5], v[4:5], v[0:1] op_sel:[1,0]
 ; GFX940-NEXT:    v_mov_b32_e32 v6, v2
 ; GFX940-NEXT:    v_mov_b32_e32 v7, v2
 ; GFX940-NEXT:    global_store_dwordx4 v8, v[4:7], s[0:1] sc0 sc1
@@ -7523,8 +7448,7 @@ define void @v_shuffle_v4f32_v4f32__7_2_4_4(ptr addrspace(1) inreg %ptr) {
 ; GFX90A-NEXT:    ;;#ASMSTART
 ; GFX90A-NEXT:    ; def v[4:7]
 ; GFX90A-NEXT:    ;;#ASMEND
-; GFX90A-NEXT:    v_mov_b32_e32 v0, v7
-; GFX90A-NEXT:    v_mov_b32_e32 v1, v2
+; GFX90A-NEXT:    v_pk_mov_b32 v[0:1], v[6:7], v[2:3] op_sel:[1,0]
 ; GFX90A-NEXT:    v_mov_b32_e32 v2, v4
 ; GFX90A-NEXT:    v_mov_b32_e32 v3, v4
 ; GFX90A-NEXT:    global_store_dwordx4 v8, v[0:3], s[16:17]
@@ -7541,8 +7465,8 @@ define void @v_shuffle_v4f32_v4f32__7_2_4_4(ptr addrspace(1) inreg %ptr) {
 ; GFX940-NEXT:    ;;#ASMSTART
 ; GFX940-NEXT:    ; def v[4:7]
 ; GFX940-NEXT:    ;;#ASMEND
-; GFX940-NEXT:    v_mov_b32_e32 v1, v2
-; GFX940-NEXT:    v_mov_b32_e32 v0, v7
+; GFX940-NEXT:    s_nop 0
+; GFX940-NEXT:    v_pk_mov_b32 v[0:1], v[6:7], v[2:3] op_sel:[1,0]
 ; GFX940-NEXT:    v_mov_b32_e32 v2, v4
 ; GFX940-NEXT:    v_mov_b32_e32 v3, v4
 ; GFX940-NEXT:    global_store_dwordx4 v8, v[0:3], s[0:1] sc0 sc1
@@ -7686,31 +7610,28 @@ define void @v_shuffle_v4f32_v4f32__7_6_4_4(ptr addrspace(1) inreg %ptr) {
 ; GFX90A-LABEL: v_shuffle_v4f32_v4f32__7_6_4_4:
 ; GFX90A:       ; %bb.0:
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT:    v_mov_b32_e32 v8, 0
 ; GFX90A-NEXT:    ;;#ASMSTART
 ; GFX90A-NEXT:    ; def v[0:3]
 ; GFX90A-NEXT:    ;;#ASMEND
-; GFX90A-NEXT:    v_mov_b32_e32 v4, v3
-; GFX90A-NEXT:    v_mov_b32_e32 v5, v2
-; GFX90A-NEXT:    v_mov_b32_e32 v6, v0
-; GFX90A-NEXT:    v_mov_b32_e32 v7, v0
-; GFX90A-NEXT:    global_store_dwordx4 v8, v[4:7], s[16:17]
+; GFX90A-NEXT:    v_mov_b32_e32 v6, 0
+; GFX90A-NEXT:    v_pk_mov_b32 v[2:3], v[2:3], v[2:3] op_sel:[1,0]
+; GFX90A-NEXT:    v_mov_b32_e32 v4, v0
+; GFX90A-NEXT:    v_mov_b32_e32 v5, v0
+; GFX90A-NEXT:    global_store_dwordx4 v6, v[2:5], s[16:17]
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
 ; GFX90A-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: v_shuffle_v4f32_v4f32__7_6_4_4:
 ; GFX940:       ; %bb.0:
 ; GFX940-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX940-NEXT:    v_mov_b32_e32 v8, 0
 ; GFX940-NEXT:    ;;#ASMSTART
 ; GFX940-NEXT:    ; def v[0:3]
 ; GFX940-NEXT:    ;;#ASMEND
-; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_mov_b32_e32 v4, v3
-; GFX940-NEXT:    v_mov_b32_e32 v5, v2
-; GFX940-NEXT:    v_mov_b32_e32 v6, v0
-; GFX940-NEXT:    v_mov_b32_e32 v7, v0
-; GFX940-NEXT:    global_store_dwordx4 v8, v[4:7], s[0:1] sc0 sc1
+; GFX940-NEXT:    v_mov_b32_e32 v6, 0
+; GFX940-NEXT:    v_pk_mov_b32 v[2:3], v[2:3], v[2:3] op_sel:[1,0]
+; GFX940-NEXT:    v_mov_b32_e32 v4, v0
+; GFX940-NEXT:    v_mov_b32_e32 v5, v0
+; GFX940-NEXT:    global_store_dwordx4 v6, v[2:5], s[0:1] sc0 sc1
 ; GFX940-NEXT:    s_waitcnt vmcnt(0)
 ; GFX940-NEXT:    s_setpc_b64 s[30:31]
   %vec0 = call <4 x float> asm "; def $0", "=v"()
@@ -7904,11 +7825,11 @@ define void @v_shuffle_v4f32_v4f32__7_7_1_4(ptr addrspace(1) inreg %ptr) {
 ; GFX90A-NEXT:    ;;#ASMSTART
 ; GFX90A-NEXT:    ; def v[2:5]
 ; GFX90A-NEXT:    ;;#ASMEND
-; GFX90A-NEXT:    v_mov_b32_e32 v8, 0
-; GFX90A-NEXT:    v_mov_b32_e32 v4, v5
-; GFX90A-NEXT:    v_mov_b32_e32 v6, v1
-; GFX90A-NEXT:    v_mov_b32_e32 v7, v2
-; GFX90A-NEXT:    global_store_dwordx4 v8, v[4:7], s[16:17]
+; GFX90A-NEXT:    v_mov_b32_e32 v6, 0
+; GFX90A-NEXT:    v_pk_mov_b32 v[2:3], v[0:1], v[2:3] op_sel:[1,0]
+; GFX90A-NEXT:    v_mov_b32_e32 v0, v5
+; GFX90A-NEXT:    v_mov_b32_e32 v1, v5
+; GFX90A-NEXT:    global_store_dwordx4 v6, v[0:3], s[16:17]
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
 ; GFX90A-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -7918,14 +7839,15 @@ define void @v_shuffle_v4f32_v4f32__7_7_1_4(ptr addrspace(1) inreg %ptr) {
 ; GFX940-NEXT:    ;;#ASMSTART
 ; GFX940-NEXT:    ; def v[0:3]
 ; GFX940-NEXT:    ;;#ASMEND
-; GFX940-NEXT:    v_mov_b32_e32 v8, 0
+; GFX940-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX940-NEXT:    ;;#ASMSTART
 ; GFX940-NEXT:    ; def v[2:5]
 ; GFX940-NEXT:    ;;#ASMEND
-; GFX940-NEXT:    v_mov_b32_e32 v6, v1
-; GFX940-NEXT:    v_mov_b32_e32 v4, v5
-; GFX940-NEXT:    v_mov_b32_e32 v7, v2
-; GFX940-NEXT:    global_store_dwordx4 v8, v[4:7], s[0:1] sc0 sc1
+; GFX940-NEXT:    s_nop 0
+; GFX940-NEXT:    v_pk_mov_b32 v[2:3], v[0:1], v[2:3] op_sel:[1,0]
+; GFX940-NEXT:    v_mov_b32_e32 v0, v5
+; GFX940-NEXT:    v_mov_b32_e32 v1, v5
+; GFX940-NEXT:    global_store_dwordx4 v6, v[0:3], s[0:1] sc0 sc1
 ; GFX940-NEXT:    s_waitcnt vmcnt(0)
 ; GFX940-NEXT:    s_setpc_b64 s[30:31]
   %vec0 = call <4 x float> asm "; def $0", "=v"()
@@ -8022,10 +7944,9 @@ define void @v_shuffle_v4f32_v4f32__7_7_3_4(ptr addrspace(1) inreg %ptr) {
 ; GFX90A-NEXT:    ;;#ASMSTART
 ; GFX90A-NEXT:    ; def v[4:7]
 ; GFX90A-NEXT:    ;;#ASMEND
+; GFX90A-NEXT:    v_pk_mov_b32 v[2:3], v[2:3], v[4:5] op_sel:[1,0]
 ; GFX90A-NEXT:    v_mov_b32_e32 v0, v7
 ; GFX90A-NEXT:    v_mov_b32_e32 v1, v7
-; GFX90A-NEXT:    v_mov_b32_e32 v2, v3
-; GFX90A-NEXT:    v_mov_b32_e32 v3, v4
 ; GFX90A-NEXT:    global_store_dwordx4 v8, v[0:3], s[16:17]
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
 ; GFX90A-NEXT:    s_setpc_b64 s[30:31]
@@ -8040,10 +7961,10 @@ define void @v_shuffle_v4f32_v4f32__7_7_3_4(ptr addrspace(1) inreg %ptr) {
 ; GFX940-NEXT:    ;;#ASMSTART
 ; GFX940-NEXT:    ; def v[4:7]
 ; GFX940-NEXT:    ;;#ASMEND
-; GFX940-NEXT:    v_mov_b32_e32 v2, v3
+; GFX940-NEXT:    s_nop 0
+; GFX940-NEXT:    v_pk_mov_b32 v[2:3], v[2:3], v[4:5] op_sel:[1,0]
 ; GFX940-NEXT:    v_mov_b32_e32 v0, v7
 ; GFX940-NEXT:    v_mov_b32_e32 v1, v7
-; GFX940-NEXT:    v_mov_b32_e32 v3, v4
 ; GFX940-NEXT:    global_store_dwordx4 v8, v[0:3], s[0:1] sc0 sc1
 ; GFX940-NEXT:    s_waitcnt vmcnt(0)
 ; GFX940-NEXT:    s_setpc_b64 s[30:31]
@@ -8076,9 +7997,8 @@ define void @v_shuffle_v4f32_v4f32__7_7_5_4(ptr addrspace(1) inreg %ptr) {
 ; GFX90A-NEXT:    ; def v[0:3]
 ; GFX90A-NEXT:    ;;#ASMEND
 ; GFX90A-NEXT:    v_mov_b32_e32 v6, 0
+; GFX90A-NEXT:    v_pk_mov_b32 v[4:5], v[0:1], v[0:1] op_sel:[1,0]
 ; GFX90A-NEXT:    v_mov_b32_e32 v2, v3
-; GFX90A-NEXT:    v_mov_b32_e32 v4, v1
-; GFX90A-NEXT:    v_mov_b32_e32 v5, v0
 ; GFX90A-NEXT:    global_store_dwordx4 v6, v[2:5], s[16:17]
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
 ; GFX90A-NEXT:    s_setpc_b64 s[30:31]
@@ -8090,9 +8010,8 @@ define void @v_shuffle_v4f32_v4f32__7_7_5_4(ptr addrspace(1) inreg %ptr) {
 ; GFX940-NEXT:    ; def v[0:3]
 ; GFX940-NEXT:    ;;#ASMEND
 ; GFX940-NEXT:    v_mov_b32_e32 v6, 0
+; GFX940-NEXT:    v_pk_mov_b32 v[4:5], v[0:1], v[0:1] op_sel:[1,0]
 ; GFX940-NEXT:    v_mov_b32_e32 v2, v3
-; GFX940-NEXT:    v_mov_b32_e32 v4, v1
-; GFX940-NEXT:    v_mov_b32_e32 v5, v0
 ; GFX940-NEXT:    global_store_dwordx4 v6, v[2:5], s[0:1] sc0 sc1
 ; GFX940-NEXT:    s_waitcnt vmcnt(0)
 ; GFX940-NEXT:    s_setpc_b64 s[30:31]
@@ -8703,12 +8622,10 @@ define void @v_shuffle_v4f32_v4f32__7_0_5_5(ptr addrspace(1) inreg %ptr) {
 ; GFX90A-NEXT:    ;;#ASMSTART
 ; GFX90A-NEXT:    ; def v[2:5]
 ; GFX90A-NEXT:    ;;#ASMEND
-; GFX90A-NEXT:    v_mov_b32_e32 v8, 0
-; GFX90A-NEXT:    v_mov_b32_e32 v4, v5
-; GFX90A-NEXT:    v_mov_b32_e32 v5, v0
-; GFX90A-NEXT:    v_mov_b32_e32 v6, v3
-; GFX90A-NEXT:    v_mov_b32_e32 v7, v3
-; GFX90A-NEXT:    global_store_dwordx4 v8, v[4:7], s[16:17]
+; GFX90A-NEXT:    v_mov_b32_e32 v6, 0
+; GFX90A-NEXT:    v_pk_mov_b32 v[0:1], v[4:5], v[0:1] op_sel:[1,0]
+; GFX90A-NEXT:    v_mov_b32_e32 v2, v3
+; GFX90A-NEXT:    global_store_dwordx4 v6, v[0:3], s[16:17]
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
 ; GFX90A-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -8718,16 +8635,14 @@ define void @v_shuffle_v4f32_v4f32__7_0_5_5(ptr addrspace(1) inreg %ptr) {
 ; GFX940-NEXT:    ;;#ASMSTART
 ; GFX940-NEXT:    ; def v[0:3]
 ; GFX940-NEXT:    ;;#ASMEND
-; GFX940-NEXT:    v_mov_b32_e32 v8, 0
+; GFX940-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX940-NEXT:    ;;#ASMSTART
 ; GFX940-NEXT:    ; def v[2:5]
 ; GFX940-NEXT:    ;;#ASMEND
 ; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_mov_b32_e32 v4, v5
-; GFX940-NEXT:    v_mov_b32_e32 v5, v0
-; GFX940-NEXT:    v_mov_b32_e32 v6, v3
-; GFX940-NEXT:    v_mov_b32_e32 v7, v3
-; GFX940-NEXT:    global_store_dwordx4 v8, v[4:7], s[0:1] sc0 sc1
+; GFX940-NEXT:    v_pk_mov_b32 v[0:1], v[4:5], v[0:1] op_sel:[1,0]
+; GFX940-NEXT:    v_mov_b32_e32 v2, v3
+; GFX940-NEXT:    global_store_dwordx4 v6, v[0:3], s[0:1] sc0 sc1
 ; GFX940-NEXT:    s_waitcnt vmcnt(0)
 ; GFX940-NEXT:    s_setpc_b64 s[30:31]
   %vec0 = call <4 x float> asm "; def $0", "=v"()
@@ -8822,8 +8737,7 @@ define void @v_shuffle_v4f32_v4f32__7_2_5_5(ptr addrspace(1) inreg %ptr) {
 ; GFX90A-NEXT:    ;;#ASMSTART
 ; GFX90A-NEXT:    ; def v[4:7]
 ; GFX90A-NEXT:    ;;#ASMEND
-; GFX90A-NEXT:    v_mov_b32_e32 v0, v7
-; GFX90A-NEXT:    v_mov_b32_e32 v1, v2
+; GFX90A-NEXT:    v_pk_mov_b32 v[0:1], v[6:7], v[2:3] op_sel:[1,0]
 ; GFX90A-NEXT:    v_mov_b32_e32 v2, v5
 ; GFX90A-NEXT:    v_mov_b32_e32 v3, v5
 ; GFX90A-NEXT:    global_store_dwordx4 v8, v[0:3], s[16:17]
@@ -8840,8 +8754,8 @@ define void @v_shuffle_v4f32_v4f32__7_2_5_5(ptr addrspace(1) inreg %ptr) {
 ; GFX940-NEXT:    ;;#ASMSTART
 ; GFX940-NEXT:    ; def v[4:7]
 ; GFX940-NEXT:    ;;#ASMEND
-; GFX940-NEXT:    v_mov_b32_e32 v1, v2
-; GFX940-NEXT:    v_mov_b32_e32 v0, v7
+; GFX940-NEXT:    s_nop 0
+; GFX940-NEXT:    v_pk_mov_b32 v[0:1], v[6:7], v[2:3] op_sel:[1,0]
 ; GFX940-NEXT:    v_mov_b32_e32 v2, v5
 ; GFX940-NEXT:    v_mov_b32_e32 v3, v5
 ; GFX940-NEXT:    global_store_dwordx4 v8, v[0:3], s[0:1] sc0 sc1
@@ -8938,8 +8852,7 @@ define void @v_shuffle_v4f32_v4f32__7_4_5_5(ptr addrspace(1) inreg %ptr) {
 ; GFX90A-NEXT:    ; def v[0:3]
 ; GFX90A-NEXT:    ;;#ASMEND
 ; GFX90A-NEXT:    v_mov_b32_e32 v6, 0
-; GFX90A-NEXT:    v_mov_b32_e32 v2, v3
-; GFX90A-NEXT:    v_mov_b32_e32 v3, v0
+; GFX90A-NEXT:    v_pk_mov_b32 v[2:3], v[2:3], v[0:1] op_sel:[1,0]
 ; GFX90A-NEXT:    v_mov_b32_e32 v4, v1
 ; GFX90A-NEXT:    v_mov_b32_e32 v5, v1
 ; GFX90A-NEXT:    global_store_dwordx4 v6, v[2:5], s[16:17]
@@ -8953,8 +8866,7 @@ define void @v_shuffle_v4f32_v4f32__7_4_5_5(ptr addrspace(1) inreg %ptr) {
 ; GFX940-NEXT:    ; def v[0:3]
 ; GFX940-NEXT:    ;;#ASMEND
 ; GFX940-NEXT:    v_mov_b32_e32 v6, 0
-; GFX940-NEXT:    v_mov_b32_e32 v2, v3
-; GFX940-NEXT:    v_mov_b32_e32 v3, v0
+; GFX940-NEXT:    v_pk_mov_b32 v[2:3], v[2:3], v[0:1] op_sel:[1,0]
 ; GFX940-NEXT:    v_mov_b32_e32 v4, v1
 ; GFX940-NEXT:    v_mov_b32_e32 v5, v1
 ; GFX940-NEXT:    global_store_dwordx4 v6, v[2:5], s[0:1] sc0 sc1
@@ -8985,31 +8897,28 @@ define void @v_shuffle_v4f32_v4f32__7_6_5_5(ptr addrspace(1) inreg %ptr) {
 ; GFX90A-LABEL: v_shuffle_v4f32_v4f32__7_6_5_5:
 ; GFX90A:       ; %bb.0:
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT:    v_mov_b32_e32 v8, 0
 ; GFX90A-NEXT:    ;;#ASMSTART
 ; GFX90A-NEXT:    ; def v[0:3]
 ; GFX90A-NEXT:    ;;#ASMEND
-; GFX90A-NEXT:    v_mov_b32_e32 v4, v3
-; GFX90A-NEXT:    v_mov_b32_e32 v5, v2
-; GFX90A-NEXT:    v_mov_b32_e32 v6, v1
-; GFX90A-NEXT:    v_mov_b32_e32 v7, v1
-; GFX90A-NEXT:    global_store_dwordx4 v8, v[4:7], s[16:17]
+; GFX90A-NEXT:    v_mov_b32_e32 v6, 0
+; GFX90A-NEXT:    v_pk_mov_b32 v[2:3], v[2:3], v[2:3] op_sel:[1,0]
+; GFX90A-NEXT:    v_mov_b32_e32 v4, v1
+; GFX90A-NEXT:    v_mov_b32_e32 v5, v1
+; GFX90A-NEXT:    global_store_dwordx4 v6, v[2:5], s[16:17]
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
 ; GFX90A-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: v_shuffle_v4f32_v4f32__7_6_5_5:
 ; GFX940:       ; %bb.0:
 ; GFX940-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX940-NEXT:    v_mov_b32_e32 v8, 0
 ; GFX940-NEXT:    ;;#ASMSTART
 ; GFX940-NEXT:    ; def v[0:3]
 ; GFX940-NEXT:    ;;#ASMEND
-; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_mov_b32_e32 v4, v3
-; GFX940-NEXT:    v_mov_b32_e32 v5, v2
-; GFX940-NEXT:    v_mov_b32_e32 v6, v1
-; GFX940-NEXT:    v_mov_b32_e32 v7, v1
-; GFX940-NEXT:    global_store_dwordx4 v8, v[4:7], s[0:1] sc0 sc1
+; GFX940-NEXT:    v_mov_b32_e32 v6, 0
+; GFX940-NEXT:    v_pk_mov_b32 v[2:3], v[2:3], v[2:3] op_sel:[1,0]
+; GFX940-NEXT:    v_mov_b32_e32 v4, v1
+; GFX940-NEXT:    v_mov_b32_e32 v5, v1
+; GFX940-NEXT:    global_store_dwordx4 v6, v[2:5], s[0:1] sc0 sc1
 ; GFX940-NEXT:    s_waitcnt vmcnt(0)
 ; GFX940-NEXT:    s_setpc_b64 s[30:31]
   %vec0 = call <4 x float> asm "; def $0", "=v"()
@@ -9584,8 +9493,7 @@ define void @v_shuffle_v4f32_v4f32__1_6_6_6(ptr addrspace(1) inreg %ptr) {
 ; GFX90A-NEXT:    ; def v[2:5]
 ; GFX90A-NEXT:    ;;#ASMEND
 ; GFX90A-NEXT:    v_mov_b32_e32 v6, 0
-; GFX90A-NEXT:    v_mov_b32_e32 v2, v1
-; GFX90A-NEXT:    v_mov_b32_e32 v3, v4
+; GFX90A-NEXT:    v_pk_mov_b32 v[2:3], v[0:1], v[4:5] op_sel:[1,0]
 ; GFX90A-NEXT:    v_mov_b32_e32 v5, v4
 ; GFX90A-NEXT:    global_store_dwordx4 v6, v[2:5], s[16:17]
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
@@ -9602,8 +9510,7 @@ define void @v_shuffle_v4f32_v4f32__1_6_6_6(ptr addrspace(1) inreg %ptr) {
 ; GFX940-NEXT:    ; def v[2:5]
 ; GFX940-NEXT:    ;;#ASMEND
 ; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_mov_b32_e32 v2, v1
-; GFX940-NEXT:    v_mov_b32_e32 v3, v4
+; GFX940-NEXT:    v_pk_mov_b32 v[2:3], v[0:1], v[4:5] op_sel:[1,0]
 ; GFX940-NEXT:    v_mov_b32_e32 v5, v4
 ; GFX940-NEXT:    global_store_dwordx4 v6, v[2:5], s[0:1] sc0 sc1
 ; GFX940-NEXT:    s_waitcnt vmcnt(0)
@@ -9701,8 +9608,7 @@ define void @v_shuffle_v4f32_v4f32__3_6_6_6(ptr addrspace(1) inreg %ptr) {
 ; GFX90A-NEXT:    ;;#ASMSTART
 ; GFX90A-NEXT:    ; def v[0:3]
 ; GFX90A-NEXT:    ;;#ASMEND
-; GFX90A-NEXT:    v_mov_b32_e32 v4, v3
-; GFX90A-NEXT:    v_mov_b32_e32 v5, v6
+; GFX90A-NEXT:    v_pk_mov_b32 v[4:5], v[2:3], v[6:7] op_sel:[1,0]
 ; GFX90A-NEXT:    v_mov_b32_e32 v7, v6
 ; GFX90A-NEXT:    global_store_dwordx4 v8, v[4:7], s[16:17]
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
@@ -9718,8 +9624,8 @@ define void @v_shuffle_v4f32_v4f32__3_6_6_6(ptr addrspace(1) inreg %ptr) {
 ; GFX940-NEXT:    ;;#ASMSTART
 ; GFX940-NEXT:    ; def v[0:3]
 ; GFX940-NEXT:    ;;#ASMEND
-; GFX940-NEXT:    v_mov_b32_e32 v5, v6
-; GFX940-NEXT:    v_mov_b32_e32 v4, v3
+; GFX940-NEXT:    s_nop 0
+; GFX940-NEXT:    v_pk_mov_b32 v[4:5], v[2:3], v[6:7] op_sel:[1,0]
 ; GFX940-NEXT:    v_mov_b32_e32 v7, v6
 ; GFX940-NEXT:    global_store_dwordx4 v8, v[4:7], s[0:1] sc0 sc1
 ; GFX940-NEXT:    s_waitcnt vmcnt(0)
@@ -9799,8 +9705,7 @@ define void @v_shuffle_v4f32_v4f32__5_6_6_6(ptr addrspace(1) inreg %ptr) {
 ; GFX90A-NEXT:    ; def v[0:3]
 ; GFX90A-NEXT:    ;;#ASMEND
 ; GFX90A-NEXT:    v_mov_b32_e32 v4, 0
-; GFX90A-NEXT:    v_mov_b32_e32 v0, v1
-; GFX90A-NEXT:    v_mov_b32_e32 v1, v2
+; GFX90A-NEXT:    v_pk_mov_b32 v[0:1], v[0:1], v[2:3] op_sel:[1,0]
 ; GFX90A-NEXT:    v_mov_b32_e32 v3, v2
 ; GFX90A-NEXT:    global_store_dwordx4 v4, v[0:3], s[16:17]
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
@@ -9813,8 +9718,7 @@ define void @v_shuffle_v4f32_v4f32__5_6_6_6(ptr addrspace(1) inreg %ptr) {
 ; GFX940-NEXT:    ; def v[0:3]
 ; GFX940-NEXT:    ;;#ASMEND
 ; GFX940-NEXT:    v_mov_b32_e32 v4, 0
-; GFX940-NEXT:    v_mov_b32_e32 v0, v1
-; GFX940-NEXT:    v_mov_b32_e32 v1, v2
+; GFX940-NEXT:    v_pk_mov_b32 v[0:1], v[0:1], v[2:3] op_sel:[1,0]
 ; GFX940-NEXT:    v_mov_b32_e32 v3, v2
 ; GFX940-NEXT:    global_store_dwordx4 v4, v[0:3], s[0:1] sc0 sc1
 ; GFX940-NEXT:    s_waitcnt vmcnt(0)
@@ -9897,8 +9801,7 @@ define void @v_shuffle_v4f32_v4f32__7_6_6_6(ptr addrspace(1) inreg %ptr) {
 ; GFX90A-NEXT:    ; def v[0:3]
 ; GFX90A-NEXT:    ;;#ASMEND
 ; GFX90A-NEXT:    v_mov_b32_e32 v4, 0
-; GFX90A-NEXT:    v_mov_b32_e32 v0, v3
-; GFX90A-NEXT:    v_mov_b32_e32 v1, v2
+; GFX90A-NEXT:    v_pk_mov_b32 v[0:1], v[2:3], v[2:3] op_sel:[1,0]
 ; GFX90A-NEXT:    v_mov_b32_e32 v3, v2
 ; GFX90A-NEXT:    global_store_dwordx4 v4, v[0:3], s[16:17]
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
@@ -9911,8 +9814,7 @@ define void @v_shuffle_v4f32_v4f32__7_6_6_6(ptr addrspace(1) inreg %ptr) {
 ; GFX940-NEXT:    ; def v[0:3]
 ; GFX940-NEXT:    ;;#ASMEND
 ; GFX940-NEXT:    v_mov_b32_e32 v4, 0
-; GFX940-NEXT:    v_mov_b32_e32 v0, v3
-; GFX940-NEXT:    v_mov_b32_e32 v1, v2
+; GFX940-NEXT:    v_pk_mov_b32 v[0:1], v[2:3], v[2:3] op_sel:[1,0]
 ; GFX940-NEXT:    v_mov_b32_e32 v3, v2
 ; GFX940-NEXT:    global_store_dwordx4 v4, v[0:3], s[0:1] sc0 sc1
 ; GFX940-NEXT:    s_waitcnt vmcnt(0)
@@ -9998,8 +9900,7 @@ define void @v_shuffle_v4f32_v4f32__7_0_6_6(ptr addrspace(1) inreg %ptr) {
 ; GFX90A-NEXT:    ; def v[2:5]
 ; GFX90A-NEXT:    ;;#ASMEND
 ; GFX90A-NEXT:    v_mov_b32_e32 v6, 0
-; GFX90A-NEXT:    v_mov_b32_e32 v2, v5
-; GFX90A-NEXT:    v_mov_b32_e32 v3, v0
+; GFX90A-NEXT:    v_pk_mov_b32 v[2:3], v[4:5], v[0:1] op_sel:[1,0]
 ; GFX90A-NEXT:    v_mov_b32_e32 v5, v4
 ; GFX90A-NEXT:    global_store_dwordx4 v6, v[2:5], s[16:17]
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
@@ -10016,8 +9917,7 @@ define void @v_shuffle_v4f32_v4f32__7_0_6_6(ptr addrspace(1) inreg %ptr) {
 ; GFX940-NEXT:    ; def v[2:5]
 ; GFX940-NEXT:    ;;#ASMEND
 ; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_mov_b32_e32 v2, v5
-; GFX940-NEXT:    v_mov_b32_e32 v3, v0
+; GFX940-NEXT:    v_pk_mov_b32 v[2:3], v[4:5], v[0:1] op_sel:[1,0]
 ; GFX940-NEXT:    v_mov_b32_e32 v5, v4
 ; GFX940-NEXT:    global_store_dwordx4 v6, v[2:5], s[0:1] sc0 sc1
 ; GFX940-NEXT:    s_waitcnt vmcnt(0)
@@ -10116,8 +10016,7 @@ define void @v_shuffle_v4f32_v4f32__7_2_6_6(ptr addrspace(1) inreg %ptr) {
 ; GFX90A-NEXT:    ;;#ASMSTART
 ; GFX90A-NEXT:    ; def v[0:3]
 ; GFX90A-NEXT:    ;;#ASMEND
-; GFX90A-NEXT:    v_mov_b32_e32 v4, v7
-; GFX90A-NEXT:    v_mov_b32_e32 v5, v2
+; GFX90A-NEXT:    v_pk_mov_b32 v[4:5], v[6:7], v[2:3] op_sel:[1,0]
 ; GFX90A-NEXT:    v_mov_b32_e32 v7, v6
 ; GFX90A-NEXT:    global_store_dwordx4 v8, v[4:7], s[16:17]
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
@@ -10133,8 +10032,8 @@ define void @v_shuffle_v4f32_v4f32__7_2_6_6(ptr addrspace(1) inreg %ptr) {
 ; GFX940-NEXT:    ;;#ASMSTART
 ; GFX940-NEXT:    ; def v[0:3]
 ; GFX940-NEXT:    ;;#ASMEND
-; GFX940-NEXT:    v_mov_b32_e32 v4, v7
-; GFX940-NEXT:    v_mov_b32_e32 v5, v2
+; GFX940-NEXT:    s_nop 0
+; GFX940-NEXT:    v_pk_mov_b32 v[4:5], v[6:7], v[2:3] op_sel:[1,0]
 ; GFX940-NEXT:    v_mov_b32_e32 v7, v6
 ; GFX940-NEXT:    global_store_dwordx4 v8, v[4:7], s[0:1] sc0 sc1
 ; GFX940-NEXT:    s_waitcnt vmcnt(0)
@@ -10222,31 +10121,26 @@ define void @v_shuffle_v4f32_v4f32__7_4_6_6(ptr addrspace(1) inreg %ptr) {
 ; GFX90A-LABEL: v_shuffle_v4f32_v4f32__7_4_6_6:
 ; GFX90A:       ; %bb.0:
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT:    v_mov_b32_e32 v8, 0
 ; GFX90A-NEXT:    ;;#ASMSTART
 ; GFX90A-NEXT:    ; def v[0:3]
 ; GFX90A-NEXT:    ;;#ASMEND
-; GFX90A-NEXT:    v_mov_b32_e32 v4, v3
-; GFX90A-NEXT:    v_mov_b32_e32 v5, v0
-; GFX90A-NEXT:    v_mov_b32_e32 v6, v2
-; GFX90A-NEXT:    v_mov_b32_e32 v7, v2
-; GFX90A-NEXT:    global_store_dwordx4 v8, v[4:7], s[16:17]
+; GFX90A-NEXT:    v_mov_b32_e32 v4, 0
+; GFX90A-NEXT:    v_pk_mov_b32 v[0:1], v[2:3], v[0:1] op_sel:[1,0]
+; GFX90A-NEXT:    v_mov_b32_e32 v3, v2
+; GFX90A-NEXT:    global_store_dwordx4 v4, v[0:3], s[16:17]
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
 ; GFX90A-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: v_shuffle_v4f32_v4f32__7_4_6_6:
 ; GFX940:       ; %bb.0:
 ; GFX940-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX940-NEXT:    v_mov_b32_e32 v8, 0
 ; GFX940-NEXT:    ;;#ASMSTART
 ; GFX940-NEXT:    ; def v[0:3]
 ; GFX940-NEXT:    ;;#ASMEND
-; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_mov_b32_e32 v4, v3
-; GFX940-NEXT:    v_mov_b32_e32 v5, v0
-; GFX940-NEXT:    v_mov_b32_e32 v6, v2
-; GFX940-NEXT:    v_mov_b32_e32 v7, v2
-; GFX940-NEXT:    global_store_dwordx4 v8, v[4:7], s[0:1] sc0 sc1
+; GFX940-NEXT:    v_mov_b32_e32 v4, 0
+; GFX940-NEXT:    v_pk_mov_b32 v[0:1], v[2:3], v[0:1] op_sel:[1,0]
+; GFX940-NEXT:    v_mov_b32_e32 v3, v2
+; GFX940-NEXT:    global_store_dwordx4 v4, v[0:3], s[0:1] sc0 sc1
 ; GFX940-NEXT:    s_waitcnt vmcnt(0)
 ; GFX940-NEXT:    s_setpc_b64 s[30:31]
   %vec0 = call <4 x float> asm "; def $0", "=v"()
@@ -10486,15 +10380,14 @@ define void @v_shuffle_v4f32_v4f32__7_7_1_6(ptr addrspace(1) inreg %ptr) {
 ; GFX90A-NEXT:    ;;#ASMSTART
 ; GFX90A-NEXT:    ; def v[0:3]
 ; GFX90A-NEXT:    ;;#ASMEND
-; GFX90A-NEXT:    v_mov_b32_e32 v10, 0
 ; GFX90A-NEXT:    ;;#ASMSTART
 ; GFX90A-NEXT:    ; def v[2:5]
 ; GFX90A-NEXT:    ;;#ASMEND
-; GFX90A-NEXT:    v_mov_b32_e32 v6, v5
-; GFX90A-NEXT:    v_mov_b32_e32 v7, v5
-; GFX90A-NEXT:    v_mov_b32_e32 v8, v1
-; GFX90A-NEXT:    v_mov_b32_e32 v9, v4
-; GFX90A-NEXT:    global_store_dwordx4 v10, v[6:9], s[16:17]
+; GFX90A-NEXT:    v_mov_b32_e32 v6, 0
+; GFX90A-NEXT:    v_pk_mov_b32 v[2:3], v[0:1], v[4:5] op_sel:[1,0]
+; GFX90A-NEXT:    v_mov_b32_e32 v0, v5
+; GFX90A-NEXT:    v_mov_b32_e32 v1, v5
+; GFX90A-NEXT:    global_store_dwordx4 v6, v[0:3], s[16:17]
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
 ; GFX90A-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -10504,15 +10397,15 @@ define void @v_shuffle_v4f32_v4f32__7_7_1_6(ptr addrspace(1) inreg %ptr) {
 ; GFX940-NEXT:    ;;#ASMSTART
 ; GFX940-NEXT:    ; def v[0:3]
 ; GFX940-NEXT:    ;;#ASMEND
-; GFX940-NEXT:    v_mov_b32_e32 v10, 0
+; GFX940-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX940-NEXT:    ;;#ASMSTART
 ; GFX940-NEXT:    ; def v[2:5]
 ; GFX940-NEXT:    ;;#ASMEND
-; GFX940-NEXT:    v_mov_b32_e32 v8, v1
-; GFX940-NEXT:    v_mov_b32_e32 v6, v5
-; GFX940-NEXT:    v_mov_b32_e32 v7, v5
-; GFX940-NEXT:    v_mov_b32_e32 v9, v4
-; GFX940-NEXT:    global_store_dwordx4 v10, v[6:9], s[0:1] sc0 sc1
+; GFX940-NEXT:    s_nop 0
+; GFX940-NEXT:    v_pk_mov_b32 v[2:3], v[0:1], v[4:5] op_sel:[1,0]
+; GFX940-NEXT:    v_mov_b32_e32 v0, v5
+; GFX940-NEXT:    v_mov_b32_e32 v1, v5
+; GFX940-NEXT:    global_store_dwordx4 v6, v[0:3], s[0:1] sc0 sc1
 ; GFX940-NEXT:    s_waitcnt vmcnt(0)
 ; GFX940-NEXT:    s_setpc_b64 s[30:31]
   %vec0 = call <4 x float> asm "; def $0", "=v"()
@@ -10610,10 +10503,9 @@ define void @v_shuffle_v4f32_v4f32__7_7_3_6(ptr addrspace(1) inreg %ptr) {
 ; GFX90A-NEXT:    ;;#ASMSTART
 ; GFX90A-NEXT:    ; def v[4:7]
 ; GFX90A-NEXT:    ;;#ASMEND
+; GFX90A-NEXT:    v_pk_mov_b32 v[2:3], v[2:3], v[6:7] op_sel:[1,0]
 ; GFX90A-NEXT:    v_mov_b32_e32 v0, v7
 ; GFX90A-NEXT:    v_mov_b32_e32 v1, v7
-; GFX90A-NEXT:    v_mov_b32_e32 v2, v3
-; GFX90A-NEXT:    v_mov_b32_e32 v3, v6
 ; GFX90A-NEXT:    global_store_dwordx4 v8, v[0:3], s[16:17]
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
 ; GFX90A-NEXT:    s_setpc_b64 s[30:31]
@@ -10628,10 +10520,10 @@ define void @v_shuffle_v4f32_v4f32__7_7_3_6(ptr addrspace(1) inreg %ptr) {
 ; GFX940-NEXT:    ;;#ASMSTART
 ; GFX940-NEXT:    ; def v[4:7]
 ; GFX940-NEXT:    ;;#ASMEND
-; GFX940-NEXT:    v_mov_b32_e32 v2, v3
+; GFX940-NEXT:    s_nop 0
+; GFX940-NEXT:    v_pk_mov_b32 v[2:3], v[2:3], v[6:7] op_sel:[1,0]
 ; GFX940-NEXT:    v_mov_b32_e32 v0, v7
 ; GFX940-NEXT:    v_mov_b32_e32 v1, v7
-; GFX940-NEXT:    v_mov_b32_e32 v3, v6
 ; GFX940-NEXT:    global_store_dwordx4 v8, v[0:3], s[0:1] sc0 sc1
 ; GFX940-NEXT:    s_waitcnt vmcnt(0)
 ; GFX940-NEXT:    s_setpc_b64 s[30:31]
@@ -10714,31 +10606,26 @@ define void @v_shuffle_v4f32_v4f32__7_7_5_6(ptr addrspace(1) inreg %ptr) {
 ; GFX90A-LABEL: v_shuffle_v4f32_v4f32__7_7_5_6:
 ; GFX90A:       ; %bb.0:
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT:    v_mov_b32_e32 v8, 0
 ; GFX90A-NEXT:    ;;#ASMSTART
 ; GFX90A-NEXT:    ; def v[0:3]
 ; GFX90A-NEXT:    ;;#ASMEND
-; GFX90A-NEXT:    v_mov_b32_e32 v4, v3
-; GFX90A-NEXT:    v_mov_b32_e32 v5, v3
-; GFX90A-NEXT:    v_mov_b32_e32 v6, v1
-; GFX90A-NEXT:    v_mov_b32_e32 v7, v2
-; GFX90A-NEXT:    global_store_dwordx4 v8, v[4:7], s[16:17]
+; GFX90A-NEXT:    v_mov_b32_e32 v6, 0
+; GFX90A-NEXT:    v_pk_mov_b32 v[4:5], v[0:1], v[2:3] op_sel:[1,0]
+; GFX90A-NEXT:    v_mov_b32_e32 v2, v3
+; GFX90A-NEXT:    global_store_dwordx4 v6, v[2:5], s[16:17]
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
 ; GFX90A-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: v_shuffle_v4f32_v4f32__7_7_5_6:
 ; GFX940:       ; %bb.0:
 ; GFX940-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX940-NEXT:    v_mov_b32_e32 v8, 0
 ; GFX940-NEXT:    ;;#ASMSTART
 ; GFX940-NEXT:    ; def v[0:3]
 ; GFX940-NEXT:    ;;#ASMEND
-; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_mov_b32_e32 v4, v3
-; GFX940-NEXT:    v_mov_b32_e32 v5, v3
-; GFX940-NEXT:    v_mov_b32_e32 v6, v1
-; GFX940-NEXT:    v_mov_b32_e32 v7, v2
-; GFX940-NEXT:    global_store_dwordx4 v8, v[4:7], s[0:1] sc0 sc1
+; GFX940-NEXT:    v_mov_b32_e32 v6, 0
+; GFX940-NEXT:    v_pk_mov_b32 v[4:5], v[0:1], v[2:3] op_sel:[1,0]
+; GFX940-NEXT:    v_mov_b32_e32 v2, v3
+; GFX940-NEXT:    global_store_dwordx4 v6, v[2:5], s[0:1] sc0 sc1
 ; GFX940-NEXT:    s_waitcnt vmcnt(0)
 ; GFX940-NEXT:    s_setpc_b64 s[30:31]
   %vec0 = call <4 x float> asm "; def $0", "=v"()
@@ -11246,8 +11133,7 @@ define void @v_shuffle_v4f32_v4f32__7_0_7_7(ptr addrspace(1) inreg %ptr) {
 ; GFX90A-NEXT:    ; def v[2:5]
 ; GFX90A-NEXT:    ;;#ASMEND
 ; GFX90A-NEXT:    v_mov_b32_e32 v6, 0
-; GFX90A-NEXT:    v_mov_b32_e32 v2, v5
-; GFX90A-NEXT:    v_mov_b32_e32 v3, v0
+; GFX90A-NEXT:    v_pk_mov_b32 v[2:3], v[4:5], v[0:1] op_sel:[1,0]
 ; GFX90A-NEXT:    v_mov_b32_e32 v4, v5
 ; GFX90A-NEXT:    global_store_dwordx4 v6, v[2:5], s[16:17]
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
@@ -11264,8 +11150,7 @@ define void @v_shuffle_v4f32_v4f32__7_0_7_7(ptr addrspace(1) inreg %ptr) {
 ; GFX940-NEXT:    ; def v[2:5]
 ; GFX940-NEXT:    ;;#ASMEND
 ; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_mov_b32_e32 v2, v5
-; GFX940-NEXT:    v_mov_b32_e32 v3, v0
+; GFX940-NEXT:    v_pk_mov_b32 v[2:3], v[4:5], v[0:1] op_sel:[1,0]
 ; GFX940-NEXT:    v_mov_b32_e32 v4, v5
 ; GFX940-NEXT:    global_store_dwordx4 v6, v[2:5], s[0:1] sc0 sc1
 ; GFX940-NEXT:    s_waitcnt vmcnt(0)
@@ -11364,8 +11249,7 @@ define void @v_shuffle_v4f32_v4f32__7_2_7_7(ptr addrspace(1) inreg %ptr) {
 ; GFX90A-NEXT:    ;;#ASMSTART
 ; GFX90A-NEXT:    ; def v[0:3]
 ; GFX90A-NEXT:    ;;#ASMEND
-; GFX90A-NEXT:    v_mov_b32_e32 v4, v7
-; GFX90A-NEXT:    v_mov_b32_e32 v5, v2
+; GFX90A-NEXT:    v_pk_mov_b32 v[4:5], v[6:7], v[2:3] op_sel:[1,0]
 ; GFX90A-NEXT:    v_mov_b32_e32 v6, v7
 ; GFX90A-NEXT:    global_store_dwordx4 v8, v[4:7], s[16:17]
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
@@ -11381,8 +11265,8 @@ define void @v_shuffle_v4f32_v4f32__7_2_7_7(ptr addrspace(1) inreg %ptr) {
 ; GFX940-NEXT:    ;;#ASMSTART
 ; GFX940-NEXT:    ; def v[0:3]
 ; GFX940-NEXT:    ;;#ASMEND
-; GFX940-NEXT:    v_mov_b32_e32 v4, v7
-; GFX940-NEXT:    v_mov_b32_e32 v5, v2
+; GFX940-NEXT:    s_nop 0
+; GFX940-NEXT:    v_pk_mov_b32 v[4:5], v[6:7], v[2:3] op_sel:[1,0]
 ; GFX940-NEXT:    v_mov_b32_e32 v6, v7
 ; GFX940-NEXT:    global_store_dwordx4 v8, v[4:7], s[0:1] sc0 sc1
 ; GFX940-NEXT:    s_waitcnt vmcnt(0)
@@ -11471,31 +11355,26 @@ define void @v_shuffle_v4f32_v4f32__7_4_7_7(ptr addrspace(1) inreg %ptr) {
 ; GFX90A-LABEL: v_shuffle_v4f32_v4f32__7_4_7_7:
 ; GFX90A:       ; %bb.0:
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT:    v_mov_b32_e32 v8, 0
 ; GFX90A-NEXT:    ;;#ASMSTART
 ; GFX90A-NEXT:    ; def v[0:3]
 ; GFX90A-NEXT:    ;;#ASMEND
-; GFX90A-NEXT:    v_mov_b32_e32 v4, v3
-; GFX90A-NEXT:    v_mov_b32_e32 v5, v0
-; GFX90A-NEXT:    v_mov_b32_e32 v6, v3
-; GFX90A-NEXT:    v_mov_b32_e32 v7, v3
-; GFX90A-NEXT:    global_store_dwordx4 v8, v[4:7], s[16:17]
+; GFX90A-NEXT:    v_mov_b32_e32 v4, 0
+; GFX90A-NEXT:    v_pk_mov_b32 v[0:1], v[2:3], v[0:1] op_sel:[1,0]
+; GFX90A-NEXT:    v_mov_b32_e32 v2, v3
+; GFX90A-NEXT:    global_store_dwordx4 v4, v[0:3], s[16:17]
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
 ; GFX90A-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: v_shuffle_v4f32_v4f32__7_4_7_7:
 ; GFX940:       ; %bb.0:
 ; GFX940-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX940-NEXT:    v_mov_b32_e32 v8, 0
 ; GFX940-NEXT:    ;;#ASMSTART
 ; GFX940-NEXT:    ; def v[0:3]
 ; GFX940-NEXT:    ;;#ASMEND
-; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_mov_b32_e32 v4, v3
-; GFX940-NEXT:    v_mov_b32_e32 v5, v0
-; GFX940-NEXT:    v_mov_b32_e32 v6, v3
-; GFX940-NEXT:    v_mov_b32_e32 v7, v3
-; GFX940-NEXT:    global_store_dwordx4 v8, v[4:7], s[0:1] sc0 sc1
+; GFX940-NEXT:    v_mov_b32_e32 v4, 0
+; GFX940-NEXT:    v_pk_mov_b32 v[0:1], v[2:3], v[0:1] op_sel:[1,0]
+; GFX940-NEXT:    v_mov_b32_e32 v2, v3
+; GFX940-NEXT:    global_store_dwordx4 v4, v[0:3], s[0:1] sc0 sc1
 ; GFX940-NEXT:    s_waitcnt vmcnt(0)
 ; GFX940-NEXT:    s_setpc_b64 s[30:31]
   %vec0 = call <4 x float> asm "; def $0", "=v"()
@@ -11573,8 +11452,7 @@ define void @v_shuffle_v4f32_v4f32__7_6_7_7(ptr addrspace(1) inreg %ptr) {
 ; GFX90A-NEXT:    ; def v[0:3]
 ; GFX90A-NEXT:    ;;#ASMEND
 ; GFX90A-NEXT:    v_mov_b32_e32 v4, 0
-; GFX90A-NEXT:    v_mov_b32_e32 v0, v3
-; GFX90A-NEXT:    v_mov_b32_e32 v1, v2
+; GFX90A-NEXT:    v_pk_mov_b32 v[0:1], v[2:3], v[2:3] op_sel:[1,0]
 ; GFX90A-NEXT:    v_mov_b32_e32 v2, v3
 ; GFX90A-NEXT:    global_store_dwordx4 v4, v[0:3], s[16:17]
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
@@ -11587,8 +11465,7 @@ define void @v_shuffle_v4f32_v4f32__7_6_7_7(ptr addrspace(1) inreg %ptr) {
 ; GFX940-NEXT:    ; def v[0:3]
 ; GFX940-NEXT:    ;;#ASMEND
 ; GFX940-NEXT:    v_mov_b32_e32 v4, 0
-; GFX940-NEXT:    v_mov_b32_e32 v0, v3
-; GFX940-NEXT:    v_mov_b32_e32 v1, v2
+; GFX940-NEXT:    v_pk_mov_b32 v[0:1], v[2:3], v[2:3] op_sel:[1,0]
 ; GFX940-NEXT:    v_mov_b32_e32 v2, v3
 ; GFX940-NEXT:    global_store_dwordx4 v4, v[0:3], s[0:1] sc0 sc1
 ; GFX940-NEXT:    s_waitcnt vmcnt(0)
diff --git a/llvm/test/CodeGen/AMDGPU/shufflevector.v4i32.v2i32.ll b/llvm/test/CodeGen/AMDGPU/shufflevector.v4i32.v2i32.ll
index 86e8e2ed267dc..94c61f5ad0e86 100644
--- a/llvm/test/CodeGen/AMDGPU/shufflevector.v4i32.v2i32.ll
+++ b/llvm/test/CodeGen/AMDGPU/shufflevector.v4i32.v2i32.ll
@@ -176,8 +176,7 @@ define void @v_shuffle_v4i32_v2i32__3_0_u_u(ptr addrspace(1) inreg %ptr) {
 ; GFX90A-NEXT:    ;;#ASMSTART
 ; GFX90A-NEXT:    ; def v[2:3]
 ; GFX90A-NEXT:    ;;#ASMEND
-; GFX90A-NEXT:    v_mov_b32_e32 v0, v1
-; GFX90A-NEXT:    v_mov_b32_e32 v1, v2
+; GFX90A-NEXT:    v_pk_mov_b32 v[0:1], v[2:3], v[0:1] op_sel:[1,0]
 ; GFX90A-NEXT:    global_store_dwordx4 v4, v[0:3], s[16:17]
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
 ; GFX90A-NEXT:    s_setpc_b64 s[30:31]
@@ -192,8 +191,8 @@ define void @v_shuffle_v4i32_v2i32__3_0_u_u(ptr addrspace(1) inreg %ptr) {
 ; GFX940-NEXT:    ;;#ASMSTART
 ; GFX940-NEXT:    ; def v[2:3]
 ; GFX940-NEXT:    ;;#ASMEND
-; GFX940-NEXT:    v_mov_b32_e32 v0, v1
-; GFX940-NEXT:    v_mov_b32_e32 v1, v2
+; GFX940-NEXT:    s_nop 0
+; GFX940-NEXT:    v_pk_mov_b32 v[0:1], v[2:3], v[0:1] op_sel:[1,0]
 ; GFX940-NEXT:    global_store_dwordx4 v4, v[0:3], s[0:1] sc0 sc1
 ; GFX940-NEXT:    s_waitcnt vmcnt(0)
 ; GFX940-NEXT:    s_setpc_b64 s[30:31]
@@ -566,16 +565,15 @@ define void @v_shuffle_v4i32_v2i32__3_3_3_0(ptr addrspace(1) inreg %ptr) {
 ; GFX90A:       ; %bb.0:
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX90A-NEXT:    ;;#ASMSTART
-; GFX90A-NEXT:    ; def v[0:1]
+; GFX90A-NEXT:    ; def v[2:3]
 ; GFX90A-NEXT:    ;;#ASMEND
-; GFX90A-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX90A-NEXT:    ;;#ASMSTART
-; GFX90A-NEXT:    ; def v[4:5]
+; GFX90A-NEXT:    ; def v[0:1]
 ; GFX90A-NEXT:    ;;#ASMEND
+; GFX90A-NEXT:    v_mov_b32_e32 v4, 0
+; GFX90A-NEXT:    v_pk_mov_b32 v[2:3], v[0:1], v[2:3] op_sel:[1,0]
 ; GFX90A-NEXT:    v_mov_b32_e32 v0, v1
-; GFX90A-NEXT:    v_mov_b32_e32 v2, v1
-; GFX90A-NEXT:    v_mov_b32_e32 v3, v4
-; GFX90A-NEXT:    global_store_dwordx4 v6, v[0:3], s[16:17]
+; GFX90A-NEXT:    global_store_dwordx4 v4, v[0:3], s[16:17]
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
 ; GFX90A-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -583,16 +581,15 @@ define void @v_shuffle_v4i32_v2i32__3_3_3_0(ptr addrspace(1) inreg %ptr) {
 ; GFX940:       ; %bb.0:
 ; GFX940-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX940-NEXT:    ;;#ASMSTART
-; GFX940-NEXT:    ; def v[0:1]
+; GFX940-NEXT:    ; def v[2:3]
 ; GFX940-NEXT:    ;;#ASMEND
-; GFX940-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX940-NEXT:    ;;#ASMSTART
-; GFX940-NEXT:    ; def v[4:5]
+; GFX940-NEXT:    ; def v[0:1]
 ; GFX940-NEXT:    ;;#ASMEND
+; GFX940-NEXT:    v_mov_b32_e32 v4, 0
+; GFX940-NEXT:    v_pk_mov_b32 v[2:3], v[0:1], v[2:3] op_sel:[1,0]
 ; GFX940-NEXT:    v_mov_b32_e32 v0, v1
-; GFX940-NEXT:    v_mov_b32_e32 v2, v1
-; GFX940-NEXT:    v_mov_b32_e32 v3, v4
-; GFX940-NEXT:    global_store_dwordx4 v6, v[0:3], s[0:1] sc0 sc1
+; GFX940-NEXT:    global_store_dwordx4 v4, v[0:3], s[0:1] sc0 sc1
 ; GFX940-NEXT:    s_waitcnt vmcnt(0)
 ; GFX940-NEXT:    s_setpc_b64 s[30:31]
   %vec0 = call <2 x i32> asm "; def $0", "=v"()
@@ -675,31 +672,26 @@ define void @v_shuffle_v4i32_v2i32__3_3_3_2(ptr addrspace(1) inreg %ptr) {
 ; GFX90A-LABEL: v_shuffle_v4i32_v2i32__3_3_3_2:
 ; GFX90A:       ; %bb.0:
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX90A-NEXT:    ;;#ASMSTART
-; GFX90A-NEXT:    ; def v[4:5]
+; GFX90A-NEXT:    ; def v[0:1]
 ; GFX90A-NEXT:    ;;#ASMEND
-; GFX90A-NEXT:    v_mov_b32_e32 v0, v5
-; GFX90A-NEXT:    v_mov_b32_e32 v1, v5
-; GFX90A-NEXT:    v_mov_b32_e32 v2, v5
-; GFX90A-NEXT:    v_mov_b32_e32 v3, v4
-; GFX90A-NEXT:    global_store_dwordx4 v6, v[0:3], s[16:17]
+; GFX90A-NEXT:    v_mov_b32_e32 v4, 0
+; GFX90A-NEXT:    v_pk_mov_b32 v[2:3], v[0:1], v[0:1] op_sel:[1,0]
+; GFX90A-NEXT:    v_mov_b32_e32 v0, v1
+; GFX90A-NEXT:    global_store_dwordx4 v4, v[0:3], s[16:17]
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
 ; GFX90A-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: v_shuffle_v4i32_v2i32__3_3_3_2:
 ; GFX940:       ; %bb.0:
 ; GFX940-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX940-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX940-NEXT:    ;;#ASMSTART
-; GFX940-NEXT:    ; def v[4:5]
+; GFX940-NEXT:    ; def v[0:1]
 ; GFX940-NEXT:    ;;#ASMEND
-; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_mov_b32_e32 v0, v5
-; GFX940-NEXT:    v_mov_b32_e32 v1, v5
-; GFX940-NEXT:    v_mov_b32_e32 v2, v5
-; GFX940-NEXT:    v_mov_b32_e32 v3, v4
-; GFX940-NEXT:    global_store_dwordx4 v6, v[0:3], s[0:1] sc0 sc1
+; GFX940-NEXT:    v_mov_b32_e32 v4, 0
+; GFX940-NEXT:    v_pk_mov_b32 v[2:3], v[0:1], v[0:1] op_sel:[1,0]
+; GFX940-NEXT:    v_mov_b32_e32 v0, v1
+; GFX940-NEXT:    global_store_dwordx4 v4, v[0:3], s[0:1] sc0 sc1
 ; GFX940-NEXT:    s_waitcnt vmcnt(0)
 ; GFX940-NEXT:    s_setpc_b64 s[30:31]
   %vec0 = call <2 x i32> asm "; def $0", "=v"()
@@ -876,8 +868,7 @@ define void @v_shuffle_v4i32_v2i32__1_0_0_0(ptr addrspace(1) inreg %ptr) {
 ; GFX90A-NEXT:    ; def v[2:3]
 ; GFX90A-NEXT:    ;;#ASMEND
 ; GFX90A-NEXT:    v_mov_b32_e32 v4, 0
-; GFX90A-NEXT:    v_mov_b32_e32 v0, v3
-; GFX90A-NEXT:    v_mov_b32_e32 v1, v2
+; GFX90A-NEXT:    v_pk_mov_b32 v[0:1], v[2:3], v[2:3] op_sel:[1,0]
 ; GFX90A-NEXT:    v_mov_b32_e32 v3, v2
 ; GFX90A-NEXT:    global_store_dwordx4 v4, v[0:3], s[16:17]
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
@@ -890,8 +881,7 @@ define void @v_shuffle_v4i32_v2i32__1_0_0_0(ptr addrspace(1) inreg %ptr) {
 ; GFX940-NEXT:    ; def v[2:3]
 ; GFX940-NEXT:    ;;#ASMEND
 ; GFX940-NEXT:    v_mov_b32_e32 v4, 0
-; GFX940-NEXT:    v_mov_b32_e32 v0, v3
-; GFX940-NEXT:    v_mov_b32_e32 v1, v2
+; GFX940-NEXT:    v_pk_mov_b32 v[0:1], v[2:3], v[2:3] op_sel:[1,0]
 ; GFX940-NEXT:    v_mov_b32_e32 v3, v2
 ; GFX940-NEXT:    global_store_dwordx4 v4, v[0:3], s[0:1] sc0 sc1
 ; GFX940-NEXT:    s_waitcnt vmcnt(0)
@@ -978,8 +968,7 @@ define void @v_shuffle_v4i32_v2i32__3_0_0_0(ptr addrspace(1) inreg %ptr) {
 ; GFX90A-NEXT:    ; def v[0:1]
 ; GFX90A-NEXT:    ;;#ASMEND
 ; GFX90A-NEXT:    v_mov_b32_e32 v4, 0
-; GFX90A-NEXT:    v_mov_b32_e32 v0, v1
-; GFX90A-NEXT:    v_mov_b32_e32 v1, v2
+; GFX90A-NEXT:    v_pk_mov_b32 v[0:1], v[0:1], v[2:3] op_sel:[1,0]
 ; GFX90A-NEXT:    v_mov_b32_e32 v3, v2
 ; GFX90A-NEXT:    global_store_dwordx4 v4, v[0:3], s[16:17]
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
@@ -995,8 +984,7 @@ define void @v_shuffle_v4i32_v2i32__3_0_0_0(ptr addrspace(1) inreg %ptr) {
 ; GFX940-NEXT:    ; def v[0:1]
 ; GFX940-NEXT:    ;;#ASMEND
 ; GFX940-NEXT:    v_mov_b32_e32 v4, 0
-; GFX940-NEXT:    v_mov_b32_e32 v0, v1
-; GFX940-NEXT:    v_mov_b32_e32 v1, v2
+; GFX940-NEXT:    v_pk_mov_b32 v[0:1], v[0:1], v[2:3] op_sel:[1,0]
 ; GFX940-NEXT:    v_mov_b32_e32 v3, v2
 ; GFX940-NEXT:    global_store_dwordx4 v4, v[0:3], s[0:1] sc0 sc1
 ; GFX940-NEXT:    s_waitcnt vmcnt(0)
@@ -1144,14 +1132,13 @@ define void @v_shuffle_v4i32_v2i32__3_2_0_0(ptr addrspace(1) inreg %ptr) {
 ; GFX90A-NEXT:    ;;#ASMSTART
 ; GFX90A-NEXT:    ; def v[2:3]
 ; GFX90A-NEXT:    ;;#ASMEND
-; GFX90A-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX90A-NEXT:    ;;#ASMSTART
-; GFX90A-NEXT:    ; def v[4:5]
+; GFX90A-NEXT:    ; def v[0:1]
 ; GFX90A-NEXT:    ;;#ASMEND
-; GFX90A-NEXT:    v_mov_b32_e32 v0, v5
-; GFX90A-NEXT:    v_mov_b32_e32 v1, v4
+; GFX90A-NEXT:    v_mov_b32_e32 v4, 0
+; GFX90A-NEXT:    v_pk_mov_b32 v[0:1], v[0:1], v[0:1] op_sel:[1,0]
 ; GFX90A-NEXT:    v_mov_b32_e32 v3, v2
-; GFX90A-NEXT:    global_store_dwordx4 v6, v[0:3], s[16:17]
+; GFX90A-NEXT:    global_store_dwordx4 v4, v[0:3], s[16:17]
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
 ; GFX90A-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -1161,14 +1148,13 @@ define void @v_shuffle_v4i32_v2i32__3_2_0_0(ptr addrspace(1) inreg %ptr) {
 ; GFX940-NEXT:    ;;#ASMSTART
 ; GFX940-NEXT:    ; def v[2:3]
 ; GFX940-NEXT:    ;;#ASMEND
-; GFX940-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX940-NEXT:    ;;#ASMSTART
-; GFX940-NEXT:    ; def v[4:5]
+; GFX940-NEXT:    ; def v[0:1]
 ; GFX940-NEXT:    ;;#ASMEND
+; GFX940-NEXT:    v_mov_b32_e32 v4, 0
+; GFX940-NEXT:    v_pk_mov_b32 v[0:1], v[0:1], v[0:1] op_sel:[1,0]
 ; GFX940-NEXT:    v_mov_b32_e32 v3, v2
-; GFX940-NEXT:    v_mov_b32_e32 v0, v5
-; GFX940-NEXT:    v_mov_b32_e32 v1, v4
-; GFX940-NEXT:    global_store_dwordx4 v6, v[0:3], s[0:1] sc0 sc1
+; GFX940-NEXT:    global_store_dwordx4 v4, v[0:3], s[0:1] sc0 sc1
 ; GFX940-NEXT:    s_waitcnt vmcnt(0)
 ; GFX940-NEXT:    s_setpc_b64 s[30:31]
   %vec0 = call <2 x i32> asm "; def $0", "=v"()
@@ -1309,16 +1295,15 @@ define void @v_shuffle_v4i32_v2i32__3_3_1_0(ptr addrspace(1) inreg %ptr) {
 ; GFX90A:       ; %bb.0:
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX90A-NEXT:    ;;#ASMSTART
-; GFX90A-NEXT:    ; def v[0:1]
+; GFX90A-NEXT:    ; def v[2:3]
 ; GFX90A-NEXT:    ;;#ASMEND
-; GFX90A-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX90A-NEXT:    ;;#ASMSTART
-; GFX90A-NEXT:    ; def v[4:5]
+; GFX90A-NEXT:    ; def v[0:1]
 ; GFX90A-NEXT:    ;;#ASMEND
+; GFX90A-NEXT:    v_mov_b32_e32 v4, 0
+; GFX90A-NEXT:    v_pk_mov_b32 v[2:3], v[2:3], v[2:3] op_sel:[1,0]
 ; GFX90A-NEXT:    v_mov_b32_e32 v0, v1
-; GFX90A-NEXT:    v_mov_b32_e32 v2, v5
-; GFX90A-NEXT:    v_mov_b32_e32 v3, v4
-; GFX90A-NEXT:    global_store_dwordx4 v6, v[0:3], s[16:17]
+; GFX90A-NEXT:    global_store_dwordx4 v4, v[0:3], s[16:17]
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
 ; GFX90A-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -1326,16 +1311,15 @@ define void @v_shuffle_v4i32_v2i32__3_3_1_0(ptr addrspace(1) inreg %ptr) {
 ; GFX940:       ; %bb.0:
 ; GFX940-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX940-NEXT:    ;;#ASMSTART
-; GFX940-NEXT:    ; def v[0:1]
+; GFX940-NEXT:    ; def v[2:3]
 ; GFX940-NEXT:    ;;#ASMEND
-; GFX940-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX940-NEXT:    ;;#ASMSTART
-; GFX940-NEXT:    ; def v[4:5]
+; GFX940-NEXT:    ; def v[0:1]
 ; GFX940-NEXT:    ;;#ASMEND
+; GFX940-NEXT:    v_mov_b32_e32 v4, 0
+; GFX940-NEXT:    v_pk_mov_b32 v[2:3], v[2:3], v[2:3] op_sel:[1,0]
 ; GFX940-NEXT:    v_mov_b32_e32 v0, v1
-; GFX940-NEXT:    v_mov_b32_e32 v2, v5
-; GFX940-NEXT:    v_mov_b32_e32 v3, v4
-; GFX940-NEXT:    global_store_dwordx4 v6, v[0:3], s[0:1] sc0 sc1
+; GFX940-NEXT:    global_store_dwordx4 v4, v[0:3], s[0:1] sc0 sc1
 ; GFX940-NEXT:    s_waitcnt vmcnt(0)
 ; GFX940-NEXT:    s_setpc_b64 s[30:31]
   %vec0 = call <2 x i32> asm "; def $0", "=v"()
@@ -1729,8 +1713,7 @@ define void @v_shuffle_v4i32_v2i32__3_0_1_1(ptr addrspace(1) inreg %ptr) {
 ; GFX90A-NEXT:    ; def v[0:1]
 ; GFX90A-NEXT:    ;;#ASMEND
 ; GFX90A-NEXT:    v_mov_b32_e32 v4, 0
-; GFX90A-NEXT:    v_mov_b32_e32 v0, v1
-; GFX90A-NEXT:    v_mov_b32_e32 v1, v2
+; GFX90A-NEXT:    v_pk_mov_b32 v[0:1], v[0:1], v[2:3] op_sel:[1,0]
 ; GFX90A-NEXT:    v_mov_b32_e32 v2, v3
 ; GFX90A-NEXT:    global_store_dwordx4 v4, v[0:3], s[16:17]
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
@@ -1746,8 +1729,7 @@ define void @v_shuffle_v4i32_v2i32__3_0_1_1(ptr addrspace(1) inreg %ptr) {
 ; GFX940-NEXT:    ; def v[0:1]
 ; GFX940-NEXT:    ;;#ASMEND
 ; GFX940-NEXT:    v_mov_b32_e32 v4, 0
-; GFX940-NEXT:    v_mov_b32_e32 v0, v1
-; GFX940-NEXT:    v_mov_b32_e32 v1, v2
+; GFX940-NEXT:    v_pk_mov_b32 v[0:1], v[0:1], v[2:3] op_sel:[1,0]
 ; GFX940-NEXT:    v_mov_b32_e32 v2, v3
 ; GFX940-NEXT:    global_store_dwordx4 v4, v[0:3], s[0:1] sc0 sc1
 ; GFX940-NEXT:    s_waitcnt vmcnt(0)
@@ -1782,14 +1764,13 @@ define void @v_shuffle_v4i32_v2i32__3_2_1_1(ptr addrspace(1) inreg %ptr) {
 ; GFX90A-NEXT:    ;;#ASMSTART
 ; GFX90A-NEXT:    ; def v[2:3]
 ; GFX90A-NEXT:    ;;#ASMEND
-; GFX90A-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX90A-NEXT:    ;;#ASMSTART
-; GFX90A-NEXT:    ; def v[4:5]
+; GFX90A-NEXT:    ; def v[0:1]
 ; GFX90A-NEXT:    ;;#ASMEND
-; GFX90A-NEXT:    v_mov_b32_e32 v0, v5
-; GFX90A-NEXT:    v_mov_b32_e32 v1, v4
+; GFX90A-NEXT:    v_mov_b32_e32 v4, 0
+; GFX90A-NEXT:    v_pk_mov_b32 v[0:1], v[0:1], v[0:1] op_sel:[1,0]
 ; GFX90A-NEXT:    v_mov_b32_e32 v2, v3
-; GFX90A-NEXT:    global_store_dwordx4 v6, v[0:3], s[16:17]
+; GFX90A-NEXT:    global_store_dwordx4 v4, v[0:3], s[16:17]
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
 ; GFX90A-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -1799,14 +1780,13 @@ define void @v_shuffle_v4i32_v2i32__3_2_1_1(ptr addrspace(1) inreg %ptr) {
 ; GFX940-NEXT:    ;;#ASMSTART
 ; GFX940-NEXT:    ; def v[2:3]
 ; GFX940-NEXT:    ;;#ASMEND
-; GFX940-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX940-NEXT:    ;;#ASMSTART
-; GFX940-NEXT:    ; def v[4:5]
+; GFX940-NEXT:    ; def v[0:1]
 ; GFX940-NEXT:    ;;#ASMEND
+; GFX940-NEXT:    v_mov_b32_e32 v4, 0
+; GFX940-NEXT:    v_pk_mov_b32 v[0:1], v[0:1], v[0:1] op_sel:[1,0]
 ; GFX940-NEXT:    v_mov_b32_e32 v2, v3
-; GFX940-NEXT:    v_mov_b32_e32 v0, v5
-; GFX940-NEXT:    v_mov_b32_e32 v1, v4
-; GFX940-NEXT:    global_store_dwordx4 v6, v[0:3], s[0:1] sc0 sc1
+; GFX940-NEXT:    global_store_dwordx4 v4, v[0:3], s[0:1] sc0 sc1
 ; GFX940-NEXT:    s_waitcnt vmcnt(0)
 ; GFX940-NEXT:    s_setpc_b64 s[30:31]
   %vec0 = call <2 x i32> asm "; def $0", "=v"()
@@ -2158,8 +2138,7 @@ define void @v_shuffle_v4i32_v2i32__3_2_2_2(ptr addrspace(1) inreg %ptr) {
 ; GFX90A-NEXT:    ; def v[2:3]
 ; GFX90A-NEXT:    ;;#ASMEND
 ; GFX90A-NEXT:    v_mov_b32_e32 v4, 0
-; GFX90A-NEXT:    v_mov_b32_e32 v0, v3
-; GFX90A-NEXT:    v_mov_b32_e32 v1, v2
+; GFX90A-NEXT:    v_pk_mov_b32 v[0:1], v[2:3], v[2:3] op_sel:[1,0]
 ; GFX90A-NEXT:    v_mov_b32_e32 v3, v2
 ; GFX90A-NEXT:    global_store_dwordx4 v4, v[0:3], s[16:17]
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
@@ -2172,8 +2151,7 @@ define void @v_shuffle_v4i32_v2i32__3_2_2_2(ptr addrspace(1) inreg %ptr) {
 ; GFX940-NEXT:    ; def v[2:3]
 ; GFX940-NEXT:    ;;#ASMEND
 ; GFX940-NEXT:    v_mov_b32_e32 v4, 0
-; GFX940-NEXT:    v_mov_b32_e32 v0, v3
-; GFX940-NEXT:    v_mov_b32_e32 v1, v2
+; GFX940-NEXT:    v_pk_mov_b32 v[0:1], v[2:3], v[2:3] op_sel:[1,0]
 ; GFX940-NEXT:    v_mov_b32_e32 v3, v2
 ; GFX940-NEXT:    global_store_dwordx4 v4, v[0:3], s[0:1] sc0 sc1
 ; GFX940-NEXT:    s_waitcnt vmcnt(0)
@@ -2252,16 +2230,15 @@ define void @v_shuffle_v4i32_v2i32__3_0_2_2(ptr addrspace(1) inreg %ptr) {
 ; GFX90A:       ; %bb.0:
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX90A-NEXT:    ;;#ASMSTART
-; GFX90A-NEXT:    ; def v[2:3]
+; GFX90A-NEXT:    ; def v[0:1]
 ; GFX90A-NEXT:    ;;#ASMEND
-; GFX90A-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX90A-NEXT:    ;;#ASMSTART
-; GFX90A-NEXT:    ; def v[4:5]
+; GFX90A-NEXT:    ; def v[2:3]
 ; GFX90A-NEXT:    ;;#ASMEND
-; GFX90A-NEXT:    v_mov_b32_e32 v0, v3
-; GFX90A-NEXT:    v_mov_b32_e32 v1, v4
+; GFX90A-NEXT:    v_mov_b32_e32 v4, 0
+; GFX90A-NEXT:    v_pk_mov_b32 v[0:1], v[2:3], v[0:1] op_sel:[1,0]
 ; GFX90A-NEXT:    v_mov_b32_e32 v3, v2
-; GFX90A-NEXT:    global_store_dwordx4 v6, v[0:3], s[16:17]
+; GFX90A-NEXT:    global_store_dwordx4 v4, v[0:3], s[16:17]
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
 ; GFX90A-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -2269,16 +2246,15 @@ define void @v_shuffle_v4i32_v2i32__3_0_2_2(ptr addrspace(1) inreg %ptr) {
 ; GFX940:       ; %bb.0:
 ; GFX940-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX940-NEXT:    ;;#ASMSTART
-; GFX940-NEXT:    ; def v[2:3]
+; GFX940-NEXT:    ; def v[0:1]
 ; GFX940-NEXT:    ;;#ASMEND
-; GFX940-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX940-NEXT:    ;;#ASMSTART
-; GFX940-NEXT:    ; def v[4:5]
+; GFX940-NEXT:    ; def v[2:3]
 ; GFX940-NEXT:    ;;#ASMEND
-; GFX940-NEXT:    v_mov_b32_e32 v0, v3
-; GFX940-NEXT:    v_mov_b32_e32 v1, v4
+; GFX940-NEXT:    v_mov_b32_e32 v4, 0
+; GFX940-NEXT:    v_pk_mov_b32 v[0:1], v[2:3], v[0:1] op_sel:[1,0]
 ; GFX940-NEXT:    v_mov_b32_e32 v3, v2
-; GFX940-NEXT:    global_store_dwordx4 v6, v[0:3], s[0:1] sc0 sc1
+; GFX940-NEXT:    global_store_dwordx4 v4, v[0:3], s[0:1] sc0 sc1
 ; GFX940-NEXT:    s_waitcnt vmcnt(0)
 ; GFX940-NEXT:    s_setpc_b64 s[30:31]
   %vec0 = call <2 x i32> asm "; def $0", "=v"()
@@ -2522,15 +2498,13 @@ define void @v_shuffle_v4i32_v2i32__3_3_1_2(ptr addrspace(1) inreg %ptr) {
 ; GFX90A-NEXT:    ;;#ASMSTART
 ; GFX90A-NEXT:    ; def v[2:3]
 ; GFX90A-NEXT:    ;;#ASMEND
-; GFX90A-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX90A-NEXT:    ;;#ASMSTART
-; GFX90A-NEXT:    ; def v[4:5]
+; GFX90A-NEXT:    ; def v[0:1]
 ; GFX90A-NEXT:    ;;#ASMEND
-; GFX90A-NEXT:    v_mov_b32_e32 v0, v5
-; GFX90A-NEXT:    v_mov_b32_e32 v1, v5
-; GFX90A-NEXT:    v_mov_b32_e32 v2, v3
-; GFX90A-NEXT:    v_mov_b32_e32 v3, v4
-; GFX90A-NEXT:    global_store_dwordx4 v6, v[0:3], s[16:17]
+; GFX90A-NEXT:    v_mov_b32_e32 v4, 0
+; GFX90A-NEXT:    v_pk_mov_b32 v[2:3], v[2:3], v[0:1] op_sel:[1,0]
+; GFX90A-NEXT:    v_mov_b32_e32 v0, v1
+; GFX90A-NEXT:    global_store_dwordx4 v4, v[0:3], s[16:17]
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
 ; GFX90A-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -2540,15 +2514,13 @@ define void @v_shuffle_v4i32_v2i32__3_3_1_2(ptr addrspace(1) inreg %ptr) {
 ; GFX940-NEXT:    ;;#ASMSTART
 ; GFX940-NEXT:    ; def v[2:3]
 ; GFX940-NEXT:    ;;#ASMEND
-; GFX940-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX940-NEXT:    ;;#ASMSTART
-; GFX940-NEXT:    ; def v[4:5]
+; GFX940-NEXT:    ; def v[0:1]
 ; GFX940-NEXT:    ;;#ASMEND
-; GFX940-NEXT:    v_mov_b32_e32 v2, v3
-; GFX940-NEXT:    v_mov_b32_e32 v0, v5
-; GFX940-NEXT:    v_mov_b32_e32 v1, v5
-; GFX940-NEXT:    v_mov_b32_e32 v3, v4
-; GFX940-NEXT:    global_store_dwordx4 v6, v[0:3], s[0:1] sc0 sc1
+; GFX940-NEXT:    v_mov_b32_e32 v4, 0
+; GFX940-NEXT:    v_pk_mov_b32 v[2:3], v[2:3], v[0:1] op_sel:[1,0]
+; GFX940-NEXT:    v_mov_b32_e32 v0, v1
+; GFX940-NEXT:    global_store_dwordx4 v4, v[0:3], s[0:1] sc0 sc1
 ; GFX940-NEXT:    s_waitcnt vmcnt(0)
 ; GFX940-NEXT:    s_setpc_b64 s[30:31]
   %vec0 = call <2 x i32> asm "; def $0", "=v"()
@@ -2832,16 +2804,15 @@ define void @v_shuffle_v4i32_v2i32__3_0_3_3(ptr addrspace(1) inreg %ptr) {
 ; GFX90A:       ; %bb.0:
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX90A-NEXT:    ;;#ASMSTART
-; GFX90A-NEXT:    ; def v[2:3]
+; GFX90A-NEXT:    ; def v[0:1]
 ; GFX90A-NEXT:    ;;#ASMEND
-; GFX90A-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX90A-NEXT:    ;;#ASMSTART
-; GFX90A-NEXT:    ; def v[4:5]
+; GFX90A-NEXT:    ; def v[2:3]
 ; GFX90A-NEXT:    ;;#ASMEND
-; GFX90A-NEXT:    v_mov_b32_e32 v0, v3
-; GFX90A-NEXT:    v_mov_b32_e32 v1, v4
+; GFX90A-NEXT:    v_mov_b32_e32 v4, 0
+; GFX90A-NEXT:    v_pk_mov_b32 v[0:1], v[2:3], v[0:1] op_sel:[1,0]
 ; GFX90A-NEXT:    v_mov_b32_e32 v2, v3
-; GFX90A-NEXT:    global_store_dwordx4 v6, v[0:3], s[16:17]
+; GFX90A-NEXT:    global_store_dwordx4 v4, v[0:3], s[16:17]
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
 ; GFX90A-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -2849,16 +2820,15 @@ define void @v_shuffle_v4i32_v2i32__3_0_3_3(ptr addrspace(1) inreg %ptr) {
 ; GFX940:       ; %bb.0:
 ; GFX940-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX940-NEXT:    ;;#ASMSTART
-; GFX940-NEXT:    ; def v[2:3]
+; GFX940-NEXT:    ; def v[0:1]
 ; GFX940-NEXT:    ;;#ASMEND
-; GFX940-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX940-NEXT:    ;;#ASMSTART
-; GFX940-NEXT:    ; def v[4:5]
+; GFX940-NEXT:    ; def v[2:3]
 ; GFX940-NEXT:    ;;#ASMEND
-; GFX940-NEXT:    v_mov_b32_e32 v0, v3
-; GFX940-NEXT:    v_mov_b32_e32 v1, v4
+; GFX940-NEXT:    v_mov_b32_e32 v4, 0
+; GFX940-NEXT:    v_pk_mov_b32 v[0:1], v[2:3], v[0:1] op_sel:[1,0]
 ; GFX940-NEXT:    v_mov_b32_e32 v2, v3
-; GFX940-NEXT:    global_store_dwordx4 v6, v[0:3], s[0:1] sc0 sc1
+; GFX940-NEXT:    global_store_dwordx4 v4, v[0:3], s[0:1] sc0 sc1
 ; GFX940-NEXT:    s_waitcnt vmcnt(0)
 ; GFX940-NEXT:    s_setpc_b64 s[30:31]
   %vec0 = call <2 x i32> asm "; def $0", "=v"()
@@ -2944,8 +2914,7 @@ define void @v_shuffle_v4i32_v2i32__3_2_3_3(ptr addrspace(1) inreg %ptr) {
 ; GFX90A-NEXT:    ; def v[2:3]
 ; GFX90A-NEXT:    ;;#ASMEND
 ; GFX90A-NEXT:    v_mov_b32_e32 v4, 0
-; GFX90A-NEXT:    v_mov_b32_e32 v0, v3
-; GFX90A-NEXT:    v_mov_b32_e32 v1, v2
+; GFX90A-NEXT:    v_pk_mov_b32 v[0:1], v[2:3], v[2:3] op_sel:[1,0]
 ; GFX90A-NEXT:    v_mov_b32_e32 v2, v3
 ; GFX90A-NEXT:    global_store_dwordx4 v4, v[0:3], s[16:17]
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
@@ -2958,8 +2927,7 @@ define void @v_shuffle_v4i32_v2i32__3_2_3_3(ptr addrspace(1) inreg %ptr) {
 ; GFX940-NEXT:    ; def v[2:3]
 ; GFX940-NEXT:    ;;#ASMEND
 ; GFX940-NEXT:    v_mov_b32_e32 v4, 0
-; GFX940-NEXT:    v_mov_b32_e32 v0, v3
-; GFX940-NEXT:    v_mov_b32_e32 v1, v2
+; GFX940-NEXT:    v_pk_mov_b32 v[0:1], v[2:3], v[2:3] op_sel:[1,0]
 ; GFX940-NEXT:    v_mov_b32_e32 v2, v3
 ; GFX940-NEXT:    global_store_dwordx4 v4, v[0:3], s[0:1] sc0 sc1
 ; GFX940-NEXT:    s_waitcnt vmcnt(0)
diff --git a/llvm/test/CodeGen/AMDGPU/shufflevector.v4i32.v3i32.ll b/llvm/test/CodeGen/AMDGPU/shufflevector.v4i32.v3i32.ll
index 770c1c19087b4..1b003a7c5d9bc 100644
--- a/llvm/test/CodeGen/AMDGPU/shufflevector.v4i32.v3i32.ll
+++ b/llvm/test/CodeGen/AMDGPU/shufflevector.v4i32.v3i32.ll
@@ -1280,8 +1280,7 @@ define void @v_shuffle_v4i32_v3i32__1_0_0_0(ptr addrspace(1) inreg %ptr) {
 ; GFX90A-NEXT:    ;;#ASMSTART
 ; GFX90A-NEXT:    ; def v[4:6]
 ; GFX90A-NEXT:    ;;#ASMEND
-; GFX90A-NEXT:    v_mov_b32_e32 v0, v5
-; GFX90A-NEXT:    v_mov_b32_e32 v1, v4
+; GFX90A-NEXT:    v_pk_mov_b32 v[0:1], v[4:5], v[4:5] op_sel:[1,0]
 ; GFX90A-NEXT:    v_mov_b32_e32 v2, v4
 ; GFX90A-NEXT:    v_mov_b32_e32 v3, v4
 ; GFX90A-NEXT:    global_store_dwordx4 v7, v[0:3], s[16:17]
@@ -1296,8 +1295,7 @@ define void @v_shuffle_v4i32_v3i32__1_0_0_0(ptr addrspace(1) inreg %ptr) {
 ; GFX940-NEXT:    ; def v[4:6]
 ; GFX940-NEXT:    ;;#ASMEND
 ; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_mov_b32_e32 v0, v5
-; GFX940-NEXT:    v_mov_b32_e32 v1, v4
+; GFX940-NEXT:    v_pk_mov_b32 v[0:1], v[4:5], v[4:5] op_sel:[1,0]
 ; GFX940-NEXT:    v_mov_b32_e32 v2, v4
 ; GFX940-NEXT:    v_mov_b32_e32 v3, v4
 ; GFX940-NEXT:    global_store_dwordx4 v7, v[0:3], s[0:1] sc0 sc1
@@ -1435,8 +1433,7 @@ define void @v_shuffle_v4i32_v3i32__4_0_0_0(ptr addrspace(1) inreg %ptr) {
 ; GFX90A-NEXT:    ;;#ASMSTART
 ; GFX90A-NEXT:    ; def v[4:6]
 ; GFX90A-NEXT:    ;;#ASMEND
-; GFX90A-NEXT:    v_mov_b32_e32 v0, v1
-; GFX90A-NEXT:    v_mov_b32_e32 v1, v4
+; GFX90A-NEXT:    v_pk_mov_b32 v[0:1], v[0:1], v[4:5] op_sel:[1,0]
 ; GFX90A-NEXT:    v_mov_b32_e32 v2, v4
 ; GFX90A-NEXT:    v_mov_b32_e32 v3, v4
 ; GFX90A-NEXT:    global_store_dwordx4 v7, v[0:3], s[16:17]
@@ -1453,8 +1450,8 @@ define void @v_shuffle_v4i32_v3i32__4_0_0_0(ptr addrspace(1) inreg %ptr) {
 ; GFX940-NEXT:    ;;#ASMSTART
 ; GFX940-NEXT:    ; def v[4:6]
 ; GFX940-NEXT:    ;;#ASMEND
-; GFX940-NEXT:    v_mov_b32_e32 v0, v1
-; GFX940-NEXT:    v_mov_b32_e32 v1, v4
+; GFX940-NEXT:    s_nop 0
+; GFX940-NEXT:    v_pk_mov_b32 v[0:1], v[0:1], v[4:5] op_sel:[1,0]
 ; GFX940-NEXT:    v_mov_b32_e32 v2, v4
 ; GFX940-NEXT:    v_mov_b32_e32 v3, v4
 ; GFX940-NEXT:    global_store_dwordx4 v7, v[0:3], s[0:1] sc0 sc1
@@ -1964,15 +1961,14 @@ define void @v_shuffle_v4i32_v3i32__5_5_1_0(ptr addrspace(1) inreg %ptr) {
 ; GFX90A-NEXT:    ;;#ASMSTART
 ; GFX90A-NEXT:    ; def v[0:2]
 ; GFX90A-NEXT:    ;;#ASMEND
-; GFX90A-NEXT:    v_mov_b32_e32 v7, 0
 ; GFX90A-NEXT:    ;;#ASMSTART
-; GFX90A-NEXT:    ; def v[4:6]
+; GFX90A-NEXT:    ; def v[2:4]
 ; GFX90A-NEXT:    ;;#ASMEND
-; GFX90A-NEXT:    v_mov_b32_e32 v0, v2
-; GFX90A-NEXT:    v_mov_b32_e32 v1, v2
-; GFX90A-NEXT:    v_mov_b32_e32 v2, v5
-; GFX90A-NEXT:    v_mov_b32_e32 v3, v4
-; GFX90A-NEXT:    global_store_dwordx4 v7, v[0:3], s[16:17]
+; GFX90A-NEXT:    v_mov_b32_e32 v5, 0
+; GFX90A-NEXT:    v_pk_mov_b32 v[2:3], v[0:1], v[0:1] op_sel:[1,0]
+; GFX90A-NEXT:    v_mov_b32_e32 v0, v4
+; GFX90A-NEXT:    v_mov_b32_e32 v1, v4
+; GFX90A-NEXT:    global_store_dwordx4 v5, v[0:3], s[16:17]
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
 ; GFX90A-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -1982,15 +1978,15 @@ define void @v_shuffle_v4i32_v3i32__5_5_1_0(ptr addrspace(1) inreg %ptr) {
 ; GFX940-NEXT:    ;;#ASMSTART
 ; GFX940-NEXT:    ; def v[0:2]
 ; GFX940-NEXT:    ;;#ASMEND
-; GFX940-NEXT:    v_mov_b32_e32 v7, 0
+; GFX940-NEXT:    v_mov_b32_e32 v5, 0
 ; GFX940-NEXT:    ;;#ASMSTART
-; GFX940-NEXT:    ; def v[4:6]
+; GFX940-NEXT:    ; def v[2:4]
 ; GFX940-NEXT:    ;;#ASMEND
-; GFX940-NEXT:    v_mov_b32_e32 v0, v2
-; GFX940-NEXT:    v_mov_b32_e32 v1, v2
-; GFX940-NEXT:    v_mov_b32_e32 v2, v5
-; GFX940-NEXT:    v_mov_b32_e32 v3, v4
-; GFX940-NEXT:    global_store_dwordx4 v7, v[0:3], s[0:1] sc0 sc1
+; GFX940-NEXT:    s_nop 0
+; GFX940-NEXT:    v_pk_mov_b32 v[2:3], v[0:1], v[0:1] op_sel:[1,0]
+; GFX940-NEXT:    v_mov_b32_e32 v0, v4
+; GFX940-NEXT:    v_mov_b32_e32 v1, v4
+; GFX940-NEXT:    global_store_dwordx4 v5, v[0:3], s[0:1] sc0 sc1
 ; GFX940-NEXT:    s_waitcnt vmcnt(0)
 ; GFX940-NEXT:    s_setpc_b64 s[30:31]
   %vec0 = call <3 x i32> asm "; def $0", "=v"()
@@ -2142,17 +2138,16 @@ define void @v_shuffle_v4i32_v3i32__5_5_4_0(ptr addrspace(1) inreg %ptr) {
 ; GFX90A:       ; %bb.0:
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX90A-NEXT:    ;;#ASMSTART
-; GFX90A-NEXT:    ; def v[4:6]
+; GFX90A-NEXT:    ; def v[0:2]
 ; GFX90A-NEXT:    ;;#ASMEND
-; GFX90A-NEXT:    v_mov_b32_e32 v9, 0
 ; GFX90A-NEXT:    ;;#ASMSTART
-; GFX90A-NEXT:    ; def v[6:8]
+; GFX90A-NEXT:    ; def v[2:4]
 ; GFX90A-NEXT:    ;;#ASMEND
-; GFX90A-NEXT:    v_mov_b32_e32 v0, v8
-; GFX90A-NEXT:    v_mov_b32_e32 v1, v8
-; GFX90A-NEXT:    v_mov_b32_e32 v2, v7
-; GFX90A-NEXT:    v_mov_b32_e32 v3, v4
-; GFX90A-NEXT:    global_store_dwordx4 v9, v[0:3], s[16:17]
+; GFX90A-NEXT:    v_mov_b32_e32 v5, 0
+; GFX90A-NEXT:    v_pk_mov_b32 v[2:3], v[2:3], v[0:1] op_sel:[1,0]
+; GFX90A-NEXT:    v_mov_b32_e32 v0, v4
+; GFX90A-NEXT:    v_mov_b32_e32 v1, v4
+; GFX90A-NEXT:    global_store_dwordx4 v5, v[0:3], s[16:17]
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
 ; GFX90A-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -2160,17 +2155,17 @@ define void @v_shuffle_v4i32_v3i32__5_5_4_0(ptr addrspace(1) inreg %ptr) {
 ; GFX940:       ; %bb.0:
 ; GFX940-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX940-NEXT:    ;;#ASMSTART
-; GFX940-NEXT:    ; def v[4:6]
+; GFX940-NEXT:    ; def v[0:2]
 ; GFX940-NEXT:    ;;#ASMEND
-; GFX940-NEXT:    v_mov_b32_e32 v9, 0
+; GFX940-NEXT:    v_mov_b32_e32 v5, 0
 ; GFX940-NEXT:    ;;#ASMSTART
-; GFX940-NEXT:    ; def v[6:8]
+; GFX940-NEXT:    ; def v[2:4]
 ; GFX940-NEXT:    ;;#ASMEND
-; GFX940-NEXT:    v_mov_b32_e32 v3, v4
-; GFX940-NEXT:    v_mov_b32_e32 v0, v8
-; GFX940-NEXT:    v_mov_b32_e32 v1, v8
-; GFX940-NEXT:    v_mov_b32_e32 v2, v7
-; GFX940-NEXT:    global_store_dwordx4 v9, v[0:3], s[0:1] sc0 sc1
+; GFX940-NEXT:    s_nop 0
+; GFX940-NEXT:    v_pk_mov_b32 v[2:3], v[2:3], v[0:1] op_sel:[1,0]
+; GFX940-NEXT:    v_mov_b32_e32 v0, v4
+; GFX940-NEXT:    v_mov_b32_e32 v1, v4
+; GFX940-NEXT:    global_store_dwordx4 v5, v[0:3], s[0:1] sc0 sc1
 ; GFX940-NEXT:    s_waitcnt vmcnt(0)
 ; GFX940-NEXT:    s_setpc_b64 s[30:31]
   %vec0 = call <3 x i32> asm "; def $0", "=v"()
@@ -3277,9 +3272,9 @@ define void @v_shuffle_v4i32_v3i32__1_2_2_2(ptr addrspace(1) inreg %ptr) {
 ; GFX90A-NEXT:    ;;#ASMSTART
 ; GFX90A-NEXT:    ; def v[0:2]
 ; GFX90A-NEXT:    ;;#ASMEND
+; GFX90A-NEXT:    v_mov_b32_e32 v0, v2
 ; GFX90A-NEXT:    v_mov_b32_e32 v4, 0
-; GFX90A-NEXT:    v_mov_b32_e32 v0, v1
-; GFX90A-NEXT:    v_mov_b32_e32 v1, v2
+; GFX90A-NEXT:    v_pk_mov_b32 v[0:1], v[0:1], v[0:1] op_sel:[1,0]
 ; GFX90A-NEXT:    v_mov_b32_e32 v3, v2
 ; GFX90A-NEXT:    global_store_dwordx4 v4, v[0:3], s[16:17]
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
@@ -3292,8 +3287,8 @@ define void @v_shuffle_v4i32_v3i32__1_2_2_2(ptr addrspace(1) inreg %ptr) {
 ; GFX940-NEXT:    ; def v[0:2]
 ; GFX940-NEXT:    ;;#ASMEND
 ; GFX940-NEXT:    v_mov_b32_e32 v4, 0
-; GFX940-NEXT:    v_mov_b32_e32 v0, v1
-; GFX940-NEXT:    v_mov_b32_e32 v1, v2
+; GFX940-NEXT:    v_mov_b32_e32 v0, v2
+; GFX940-NEXT:    v_pk_mov_b32 v[0:1], v[0:1], v[0:1] op_sel:[1,0]
 ; GFX940-NEXT:    v_mov_b32_e32 v3, v2
 ; GFX940-NEXT:    global_store_dwordx4 v4, v[0:3], s[0:1] sc0 sc1
 ; GFX940-NEXT:    s_waitcnt vmcnt(0)
@@ -3421,12 +3416,12 @@ define void @v_shuffle_v4i32_v3i32__4_2_2_2(ptr addrspace(1) inreg %ptr) {
 ; GFX90A-NEXT:    ;;#ASMSTART
 ; GFX90A-NEXT:    ; def v[0:2]
 ; GFX90A-NEXT:    ;;#ASMEND
+; GFX90A-NEXT:    v_mov_b32_e32 v0, v2
 ; GFX90A-NEXT:    v_mov_b32_e32 v7, 0
 ; GFX90A-NEXT:    ;;#ASMSTART
 ; GFX90A-NEXT:    ; def v[4:6]
 ; GFX90A-NEXT:    ;;#ASMEND
-; GFX90A-NEXT:    v_mov_b32_e32 v0, v5
-; GFX90A-NEXT:    v_mov_b32_e32 v1, v2
+; GFX90A-NEXT:    v_pk_mov_b32 v[0:1], v[4:5], v[0:1] op_sel:[1,0]
 ; GFX90A-NEXT:    v_mov_b32_e32 v3, v2
 ; GFX90A-NEXT:    global_store_dwordx4 v7, v[0:3], s[16:17]
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
@@ -3439,12 +3434,12 @@ define void @v_shuffle_v4i32_v3i32__4_2_2_2(ptr addrspace(1) inreg %ptr) {
 ; GFX940-NEXT:    ; def v[0:2]
 ; GFX940-NEXT:    ;;#ASMEND
 ; GFX940-NEXT:    v_mov_b32_e32 v7, 0
+; GFX940-NEXT:    v_mov_b32_e32 v0, v2
 ; GFX940-NEXT:    ;;#ASMSTART
 ; GFX940-NEXT:    ; def v[4:6]
 ; GFX940-NEXT:    ;;#ASMEND
-; GFX940-NEXT:    v_mov_b32_e32 v1, v2
-; GFX940-NEXT:    v_mov_b32_e32 v0, v5
 ; GFX940-NEXT:    v_mov_b32_e32 v3, v2
+; GFX940-NEXT:    v_pk_mov_b32 v[0:1], v[4:5], v[0:1] op_sel:[1,0]
 ; GFX940-NEXT:    global_store_dwordx4 v7, v[0:3], s[0:1] sc0 sc1
 ; GFX940-NEXT:    s_waitcnt vmcnt(0)
 ; GFX940-NEXT:    s_setpc_b64 s[30:31]
@@ -3991,17 +3986,17 @@ define void @v_shuffle_v4i32_v3i32__5_5_1_2(ptr addrspace(1) inreg %ptr) {
 ; GFX90A:       ; %bb.0:
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX90A-NEXT:    ;;#ASMSTART
-; GFX90A-NEXT:    ; def v[2:4]
-; GFX90A-NEXT:    ;;#ASMEND
-; GFX90A-NEXT:    ;;#ASMSTART
 ; GFX90A-NEXT:    ; def v[0:2]
 ; GFX90A-NEXT:    ;;#ASMEND
-; GFX90A-NEXT:    v_mov_b32_e32 v5, 0
 ; GFX90A-NEXT:    v_mov_b32_e32 v0, v2
-; GFX90A-NEXT:    v_mov_b32_e32 v1, v2
-; GFX90A-NEXT:    v_mov_b32_e32 v2, v3
-; GFX90A-NEXT:    v_mov_b32_e32 v3, v4
-; GFX90A-NEXT:    global_store_dwordx4 v5, v[0:3], s[16:17]
+; GFX90A-NEXT:    v_mov_b32_e32 v7, 0
+; GFX90A-NEXT:    ;;#ASMSTART
+; GFX90A-NEXT:    ; def v[4:6]
+; GFX90A-NEXT:    ;;#ASMEND
+; GFX90A-NEXT:    v_pk_mov_b32 v[2:3], v[0:1], v[0:1] op_sel:[1,0]
+; GFX90A-NEXT:    v_mov_b32_e32 v0, v6
+; GFX90A-NEXT:    v_mov_b32_e32 v1, v6
+; GFX90A-NEXT:    global_store_dwordx4 v7, v[0:3], s[16:17]
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
 ; GFX90A-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -4009,18 +4004,17 @@ define void @v_shuffle_v4i32_v3i32__5_5_1_2(ptr addrspace(1) inreg %ptr) {
 ; GFX940:       ; %bb.0:
 ; GFX940-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX940-NEXT:    ;;#ASMSTART
-; GFX940-NEXT:    ; def v[2:4]
-; GFX940-NEXT:    ;;#ASMEND
-; GFX940-NEXT:    v_mov_b32_e32 v5, 0
-; GFX940-NEXT:    ;;#ASMSTART
 ; GFX940-NEXT:    ; def v[0:2]
 ; GFX940-NEXT:    ;;#ASMEND
-; GFX940-NEXT:    s_nop 0
+; GFX940-NEXT:    v_mov_b32_e32 v7, 0
 ; GFX940-NEXT:    v_mov_b32_e32 v0, v2
-; GFX940-NEXT:    v_mov_b32_e32 v1, v2
-; GFX940-NEXT:    v_mov_b32_e32 v2, v3
-; GFX940-NEXT:    v_mov_b32_e32 v3, v4
-; GFX940-NEXT:    global_store_dwordx4 v5, v[0:3], s[0:1] sc0 sc1
+; GFX940-NEXT:    ;;#ASMSTART
+; GFX940-NEXT:    ; def v[4:6]
+; GFX940-NEXT:    ;;#ASMEND
+; GFX940-NEXT:    v_pk_mov_b32 v[2:3], v[0:1], v[0:1] op_sel:[1,0]
+; GFX940-NEXT:    v_mov_b32_e32 v0, v6
+; GFX940-NEXT:    v_mov_b32_e32 v1, v6
+; GFX940-NEXT:    global_store_dwordx4 v7, v[0:3], s[0:1] sc0 sc1
 ; GFX940-NEXT:    s_waitcnt vmcnt(0)
 ; GFX940-NEXT:    s_setpc_b64 s[30:31]
   %vec0 = call <3 x i32> asm "; def $0", "=v"()
@@ -4112,17 +4106,17 @@ define void @v_shuffle_v4i32_v3i32__5_5_4_2(ptr addrspace(1) inreg %ptr) {
 ; GFX90A:       ; %bb.0:
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX90A-NEXT:    ;;#ASMSTART
-; GFX90A-NEXT:    ; def v[2:4]
+; GFX90A-NEXT:    ; def v[0:2]
 ; GFX90A-NEXT:    ;;#ASMEND
-; GFX90A-NEXT:    v_mov_b32_e32 v5, 0
+; GFX90A-NEXT:    v_mov_b32_e32 v0, v2
+; GFX90A-NEXT:    v_mov_b32_e32 v7, 0
 ; GFX90A-NEXT:    ;;#ASMSTART
-; GFX90A-NEXT:    ; def v[6:8]
+; GFX90A-NEXT:    ; def v[4:6]
 ; GFX90A-NEXT:    ;;#ASMEND
-; GFX90A-NEXT:    v_mov_b32_e32 v0, v8
-; GFX90A-NEXT:    v_mov_b32_e32 v1, v8
-; GFX90A-NEXT:    v_mov_b32_e32 v2, v7
-; GFX90A-NEXT:    v_mov_b32_e32 v3, v4
-; GFX90A-NEXT:    global_store_dwordx4 v5, v[0:3], s[16:17]
+; GFX90A-NEXT:    v_pk_mov_b32 v[2:3], v[4:5], v[0:1] op_sel:[1,0]
+; GFX90A-NEXT:    v_mov_b32_e32 v0, v6
+; GFX90A-NEXT:    v_mov_b32_e32 v1, v6
+; GFX90A-NEXT:    global_store_dwordx4 v7, v[0:3], s[16:17]
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
 ; GFX90A-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -4130,17 +4124,18 @@ define void @v_shuffle_v4i32_v3i32__5_5_4_2(ptr addrspace(1) inreg %ptr) {
 ; GFX940:       ; %bb.0:
 ; GFX940-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX940-NEXT:    ;;#ASMSTART
-; GFX940-NEXT:    ; def v[2:4]
+; GFX940-NEXT:    ; def v[0:2]
 ; GFX940-NEXT:    ;;#ASMEND
-; GFX940-NEXT:    v_mov_b32_e32 v5, 0
+; GFX940-NEXT:    v_mov_b32_e32 v7, 0
+; GFX940-NEXT:    v_mov_b32_e32 v0, v2
 ; GFX940-NEXT:    ;;#ASMSTART
-; GFX940-NEXT:    ; def v[6:8]
+; GFX940-NEXT:    ; def v[4:6]
 ; GFX940-NEXT:    ;;#ASMEND
-; GFX940-NEXT:    v_mov_b32_e32 v3, v4
-; GFX940-NEXT:    v_mov_b32_e32 v0, v8
-; GFX940-NEXT:    v_mov_b32_e32 v1, v8
-; GFX940-NEXT:    v_mov_b32_e32 v2, v7
-; GFX940-NEXT:    global_store_dwordx4 v5, v[0:3], s[0:1] sc0 sc1
+; GFX940-NEXT:    s_nop 0
+; GFX940-NEXT:    v_pk_mov_b32 v[2:3], v[4:5], v[0:1] op_sel:[1,0]
+; GFX940-NEXT:    v_mov_b32_e32 v0, v6
+; GFX940-NEXT:    v_mov_b32_e32 v1, v6
+; GFX940-NEXT:    global_store_dwordx4 v7, v[0:3], s[0:1] sc0 sc1
 ; GFX940-NEXT:    s_waitcnt vmcnt(0)
 ; GFX940-NEXT:    s_setpc_b64 s[30:31]
   %vec0 = call <3 x i32> asm "; def $0", "=v"()
@@ -4317,8 +4312,7 @@ define void @v_shuffle_v4i32_v3i32__4_3_3_3(ptr addrspace(1) inreg %ptr) {
 ; GFX90A-NEXT:    ;;#ASMSTART
 ; GFX90A-NEXT:    ; def v[4:6]
 ; GFX90A-NEXT:    ;;#ASMEND
-; GFX90A-NEXT:    v_mov_b32_e32 v0, v5
-; GFX90A-NEXT:    v_mov_b32_e32 v1, v4
+; GFX90A-NEXT:    v_pk_mov_b32 v[0:1], v[4:5], v[4:5] op_sel:[1,0]
 ; GFX90A-NEXT:    v_mov_b32_e32 v2, v4
 ; GFX90A-NEXT:    v_mov_b32_e32 v3, v4
 ; GFX90A-NEXT:    global_store_dwordx4 v7, v[0:3], s[16:17]
@@ -4333,8 +4327,7 @@ define void @v_shuffle_v4i32_v3i32__4_3_3_3(ptr addrspace(1) inreg %ptr) {
 ; GFX940-NEXT:    ; def v[4:6]
 ; GFX940-NEXT:    ;;#ASMEND
 ; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_mov_b32_e32 v0, v5
-; GFX940-NEXT:    v_mov_b32_e32 v1, v4
+; GFX940-NEXT:    v_pk_mov_b32 v[0:1], v[4:5], v[4:5] op_sel:[1,0]
 ; GFX940-NEXT:    v_mov_b32_e32 v2, v4
 ; GFX940-NEXT:    v_mov_b32_e32 v3, v4
 ; GFX940-NEXT:    global_store_dwordx4 v7, v[0:3], s[0:1] sc0 sc1
@@ -4858,17 +4851,16 @@ define void @v_shuffle_v4i32_v3i32__5_5_1_3(ptr addrspace(1) inreg %ptr) {
 ; GFX90A:       ; %bb.0:
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX90A-NEXT:    ;;#ASMSTART
-; GFX90A-NEXT:    ; def v[2:4]
+; GFX90A-NEXT:    ; def v[0:2]
 ; GFX90A-NEXT:    ;;#ASMEND
-; GFX90A-NEXT:    v_mov_b32_e32 v7, 0
 ; GFX90A-NEXT:    ;;#ASMSTART
-; GFX90A-NEXT:    ; def v[4:6]
+; GFX90A-NEXT:    ; def v[2:4]
 ; GFX90A-NEXT:    ;;#ASMEND
-; GFX90A-NEXT:    v_mov_b32_e32 v0, v6
-; GFX90A-NEXT:    v_mov_b32_e32 v1, v6
-; GFX90A-NEXT:    v_mov_b32_e32 v2, v3
-; GFX90A-NEXT:    v_mov_b32_e32 v3, v4
-; GFX90A-NEXT:    global_store_dwordx4 v7, v[0:3], s[16:17]
+; GFX90A-NEXT:    v_mov_b32_e32 v5, 0
+; GFX90A-NEXT:    v_pk_mov_b32 v[2:3], v[0:1], v[2:3] op_sel:[1,0]
+; GFX90A-NEXT:    v_mov_b32_e32 v0, v4
+; GFX90A-NEXT:    v_mov_b32_e32 v1, v4
+; GFX90A-NEXT:    global_store_dwordx4 v5, v[0:3], s[16:17]
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
 ; GFX90A-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -4876,17 +4868,17 @@ define void @v_shuffle_v4i32_v3i32__5_5_1_3(ptr addrspace(1) inreg %ptr) {
 ; GFX940:       ; %bb.0:
 ; GFX940-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX940-NEXT:    ;;#ASMSTART
-; GFX940-NEXT:    ; def v[2:4]
+; GFX940-NEXT:    ; def v[0:2]
 ; GFX940-NEXT:    ;;#ASMEND
-; GFX940-NEXT:    v_mov_b32_e32 v7, 0
+; GFX940-NEXT:    v_mov_b32_e32 v5, 0
 ; GFX940-NEXT:    ;;#ASMSTART
-; GFX940-NEXT:    ; def v[4:6]
+; GFX940-NEXT:    ; def v[2:4]
 ; GFX940-NEXT:    ;;#ASMEND
-; GFX940-NEXT:    v_mov_b32_e32 v2, v3
-; GFX940-NEXT:    v_mov_b32_e32 v0, v6
-; GFX940-NEXT:    v_mov_b32_e32 v1, v6
-; GFX940-NEXT:    v_mov_b32_e32 v3, v4
-; GFX940-NEXT:    global_store_dwordx4 v7, v[0:3], s[0:1] sc0 sc1
+; GFX940-NEXT:    s_nop 0
+; GFX940-NEXT:    v_pk_mov_b32 v[2:3], v[0:1], v[2:3] op_sel:[1,0]
+; GFX940-NEXT:    v_mov_b32_e32 v0, v4
+; GFX940-NEXT:    v_mov_b32_e32 v1, v4
+; GFX940-NEXT:    global_store_dwordx4 v5, v[0:3], s[0:1] sc0 sc1
 ; GFX940-NEXT:    s_waitcnt vmcnt(0)
 ; GFX940-NEXT:    s_setpc_b64 s[30:31]
   %vec0 = call <3 x i32> asm "; def $0", "=v"()
@@ -4972,31 +4964,28 @@ define void @v_shuffle_v4i32_v3i32__5_5_4_3(ptr addrspace(1) inreg %ptr) {
 ; GFX90A-LABEL: v_shuffle_v4i32_v3i32__5_5_4_3:
 ; GFX90A:       ; %bb.0:
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT:    v_mov_b32_e32 v7, 0
 ; GFX90A-NEXT:    ;;#ASMSTART
-; GFX90A-NEXT:    ; def v[4:6]
+; GFX90A-NEXT:    ; def v[2:4]
 ; GFX90A-NEXT:    ;;#ASMEND
-; GFX90A-NEXT:    v_mov_b32_e32 v0, v6
-; GFX90A-NEXT:    v_mov_b32_e32 v1, v6
-; GFX90A-NEXT:    v_mov_b32_e32 v2, v5
-; GFX90A-NEXT:    v_mov_b32_e32 v3, v4
-; GFX90A-NEXT:    global_store_dwordx4 v7, v[0:3], s[16:17]
+; GFX90A-NEXT:    v_mov_b32_e32 v5, 0
+; GFX90A-NEXT:    v_pk_mov_b32 v[2:3], v[2:3], v[2:3] op_sel:[1,0]
+; GFX90A-NEXT:    v_mov_b32_e32 v0, v4
+; GFX90A-NEXT:    v_mov_b32_e32 v1, v4
+; GFX90A-NEXT:    global_store_dwordx4 v5, v[0:3], s[16:17]
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
 ; GFX90A-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: v_shuffle_v4i32_v3i32__5_5_4_3:
 ; GFX940:       ; %bb.0:
 ; GFX940-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX940-NEXT:    v_mov_b32_e32 v7, 0
 ; GFX940-NEXT:    ;;#ASMSTART
-; GFX940-NEXT:    ; def v[4:6]
+; GFX940-NEXT:    ; def v[2:4]
 ; GFX940-NEXT:    ;;#ASMEND
-; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_mov_b32_e32 v0, v6
-; GFX940-NEXT:    v_mov_b32_e32 v1, v6
-; GFX940-NEXT:    v_mov_b32_e32 v2, v5
-; GFX940-NEXT:    v_mov_b32_e32 v3, v4
-; GFX940-NEXT:    global_store_dwordx4 v7, v[0:3], s[0:1] sc0 sc1
+; GFX940-NEXT:    v_mov_b32_e32 v5, 0
+; GFX940-NEXT:    v_pk_mov_b32 v[2:3], v[2:3], v[2:3] op_sel:[1,0]
+; GFX940-NEXT:    v_mov_b32_e32 v0, v4
+; GFX940-NEXT:    v_mov_b32_e32 v1, v4
+; GFX940-NEXT:    global_store_dwordx4 v5, v[0:3], s[0:1] sc0 sc1
 ; GFX940-NEXT:    s_waitcnt vmcnt(0)
 ; GFX940-NEXT:    s_setpc_b64 s[30:31]
   %vec0 = call <3 x i32> asm "; def $0", "=v"()
@@ -6094,9 +6083,9 @@ define void @v_shuffle_v4i32_v3i32__1_5_5_5(ptr addrspace(1) inreg %ptr) {
 ; GFX90A-NEXT:    ;;#ASMSTART
 ; GFX90A-NEXT:    ; def v[0:2]
 ; GFX90A-NEXT:    ;;#ASMEND
+; GFX90A-NEXT:    v_mov_b32_e32 v0, v2
 ; GFX90A-NEXT:    v_mov_b32_e32 v5, 0
-; GFX90A-NEXT:    v_mov_b32_e32 v0, v3
-; GFX90A-NEXT:    v_mov_b32_e32 v1, v2
+; GFX90A-NEXT:    v_pk_mov_b32 v[0:1], v[2:3], v[0:1] op_sel:[1,0]
 ; GFX90A-NEXT:    v_mov_b32_e32 v3, v2
 ; GFX90A-NEXT:    global_store_dwordx4 v5, v[0:3], s[16:17]
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
@@ -6113,8 +6102,8 @@ define void @v_shuffle_v4i32_v3i32__1_5_5_5(ptr addrspace(1) inreg %ptr) {
 ; GFX940-NEXT:    ; def v[0:2]
 ; GFX940-NEXT:    ;;#ASMEND
 ; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_mov_b32_e32 v0, v3
-; GFX940-NEXT:    v_mov_b32_e32 v1, v2
+; GFX940-NEXT:    v_mov_b32_e32 v0, v2
+; GFX940-NEXT:    v_pk_mov_b32 v[0:1], v[2:3], v[0:1] op_sel:[1,0]
 ; GFX940-NEXT:    v_mov_b32_e32 v3, v2
 ; GFX940-NEXT:    global_store_dwordx4 v5, v[0:3], s[0:1] sc0 sc1
 ; GFX940-NEXT:    s_waitcnt vmcnt(0)
@@ -6252,9 +6241,9 @@ define void @v_shuffle_v4i32_v3i32__4_5_5_5(ptr addrspace(1) inreg %ptr) {
 ; GFX90A-NEXT:    ;;#ASMSTART
 ; GFX90A-NEXT:    ; def v[0:2]
 ; GFX90A-NEXT:    ;;#ASMEND
+; GFX90A-NEXT:    v_mov_b32_e32 v0, v2
 ; GFX90A-NEXT:    v_mov_b32_e32 v4, 0
-; GFX90A-NEXT:    v_mov_b32_e32 v0, v1
-; GFX90A-NEXT:    v_mov_b32_e32 v1, v2
+; GFX90A-NEXT:    v_pk_mov_b32 v[0:1], v[0:1], v[0:1] op_sel:[1,0]
 ; GFX90A-NEXT:    v_mov_b32_e32 v3, v2
 ; GFX90A-NEXT:    global_store_dwordx4 v4, v[0:3], s[16:17]
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
@@ -6267,8 +6256,8 @@ define void @v_shuffle_v4i32_v3i32__4_5_5_5(ptr addrspace(1) inreg %ptr) {
 ; GFX940-NEXT:    ; def v[0:2]
 ; GFX940-NEXT:    ;;#ASMEND
 ; GFX940-NEXT:    v_mov_b32_e32 v4, 0
-; GFX940-NEXT:    v_mov_b32_e32 v0, v1
-; GFX940-NEXT:    v_mov_b32_e32 v1, v2
+; GFX940-NEXT:    v_mov_b32_e32 v0, v2
+; GFX940-NEXT:    v_pk_mov_b32 v[0:1], v[0:1], v[0:1] op_sel:[1,0]
 ; GFX940-NEXT:    v_mov_b32_e32 v3, v2
 ; GFX940-NEXT:    global_store_dwordx4 v4, v[0:3], s[0:1] sc0 sc1
 ; GFX940-NEXT:    s_waitcnt vmcnt(0)
@@ -6727,17 +6716,16 @@ define void @v_shuffle_v4i32_v3i32__5_5_1_5(ptr addrspace(1) inreg %ptr) {
 ; GFX90A:       ; %bb.0:
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX90A-NEXT:    ;;#ASMSTART
-; GFX90A-NEXT:    ; def v[2:4]
+; GFX90A-NEXT:    ; def v[0:2]
 ; GFX90A-NEXT:    ;;#ASMEND
-; GFX90A-NEXT:    v_mov_b32_e32 v7, 0
 ; GFX90A-NEXT:    ;;#ASMSTART
-; GFX90A-NEXT:    ; def v[4:6]
+; GFX90A-NEXT:    ; def v[2:4]
 ; GFX90A-NEXT:    ;;#ASMEND
-; GFX90A-NEXT:    v_mov_b32_e32 v0, v6
-; GFX90A-NEXT:    v_mov_b32_e32 v1, v6
-; GFX90A-NEXT:    v_mov_b32_e32 v2, v3
-; GFX90A-NEXT:    v_mov_b32_e32 v3, v6
-; GFX90A-NEXT:    global_store_dwordx4 v7, v[0:3], s[16:17]
+; GFX90A-NEXT:    v_mov_b32_e32 v0, v4
+; GFX90A-NEXT:    v_mov_b32_e32 v5, 0
+; GFX90A-NEXT:    v_pk_mov_b32 v[2:3], v[0:1], v[0:1] op_sel:[1,0]
+; GFX90A-NEXT:    v_mov_b32_e32 v1, v4
+; GFX90A-NEXT:    global_store_dwordx4 v5, v[0:3], s[16:17]
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
 ; GFX90A-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -6745,17 +6733,17 @@ define void @v_shuffle_v4i32_v3i32__5_5_1_5(ptr addrspace(1) inreg %ptr) {
 ; GFX940:       ; %bb.0:
 ; GFX940-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX940-NEXT:    ;;#ASMSTART
-; GFX940-NEXT:    ; def v[2:4]
+; GFX940-NEXT:    ; def v[0:2]
 ; GFX940-NEXT:    ;;#ASMEND
-; GFX940-NEXT:    v_mov_b32_e32 v7, 0
+; GFX940-NEXT:    v_mov_b32_e32 v5, 0
 ; GFX940-NEXT:    ;;#ASMSTART
-; GFX940-NEXT:    ; def v[4:6]
+; GFX940-NEXT:    ; def v[2:4]
 ; GFX940-NEXT:    ;;#ASMEND
-; GFX940-NEXT:    v_mov_b32_e32 v2, v3
-; GFX940-NEXT:    v_mov_b32_e32 v0, v6
-; GFX940-NEXT:    v_mov_b32_e32 v1, v6
-; GFX940-NEXT:    v_mov_b32_e32 v3, v6
-; GFX940-NEXT:    global_store_dwordx4 v7, v[0:3], s[0:1] sc0 sc1
+; GFX940-NEXT:    s_nop 0
+; GFX940-NEXT:    v_mov_b32_e32 v0, v4
+; GFX940-NEXT:    v_pk_mov_b32 v[2:3], v[0:1], v[0:1] op_sel:[1,0]
+; GFX940-NEXT:    v_mov_b32_e32 v1, v4
+; GFX940-NEXT:    global_store_dwordx4 v5, v[0:3], s[0:1] sc0 sc1
 ; GFX940-NEXT:    s_waitcnt vmcnt(0)
 ; GFX940-NEXT:    s_setpc_b64 s[30:31]
   %vec0 = call <3 x i32> asm "; def $0", "=v"()
@@ -6893,11 +6881,10 @@ define void @v_shuffle_v4i32_v3i32__5_5_4_5(ptr addrspace(1) inreg %ptr) {
 ; GFX90A-NEXT:    ;;#ASMSTART
 ; GFX90A-NEXT:    ; def v[2:4]
 ; GFX90A-NEXT:    ;;#ASMEND
-; GFX90A-NEXT:    v_mov_b32_e32 v5, 0
 ; GFX90A-NEXT:    v_mov_b32_e32 v0, v4
+; GFX90A-NEXT:    v_mov_b32_e32 v5, 0
+; GFX90A-NEXT:    v_pk_mov_b32 v[2:3], v[2:3], v[0:1] op_sel:[1,0]
 ; GFX90A-NEXT:    v_mov_b32_e32 v1, v4
-; GFX90A-NEXT:    v_mov_b32_e32 v2, v3
-; GFX90A-NEXT:    v_mov_b32_e32 v3, v4
 ; GFX90A-NEXT:    global_store_dwordx4 v5, v[0:3], s[16:17]
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
 ; GFX90A-NEXT:    s_setpc_b64 s[30:31]
@@ -6910,9 +6897,8 @@ define void @v_shuffle_v4i32_v3i32__5_5_4_5(ptr addrspace(1) inreg %ptr) {
 ; GFX940-NEXT:    ;;#ASMEND
 ; GFX940-NEXT:    v_mov_b32_e32 v5, 0
 ; GFX940-NEXT:    v_mov_b32_e32 v0, v4
+; GFX940-NEXT:    v_pk_mov_b32 v[2:3], v[2:3], v[0:1] op_sel:[1,0]
 ; GFX940-NEXT:    v_mov_b32_e32 v1, v4
-; GFX940-NEXT:    v_mov_b32_e32 v2, v3
-; GFX940-NEXT:    v_mov_b32_e32 v3, v4
 ; GFX940-NEXT:    global_store_dwordx4 v5, v[0:3], s[0:1] sc0 sc1
 ; GFX940-NEXT:    s_waitcnt vmcnt(0)
 ; GFX940-NEXT:    s_setpc_b64 s[30:31]
diff --git a/llvm/test/CodeGen/AMDGPU/shufflevector.v4i32.v4i32.ll b/llvm/test/CodeGen/AMDGPU/shufflevector.v4i32.v4i32.ll
index 3d1bb1da92e09..47ad1c4bedb8b 100644
--- a/llvm/test/CodeGen/AMDGPU/shufflevector.v4i32.v4i32.ll
+++ b/llvm/test/CodeGen/AMDGPU/shufflevector.v4i32.v4i32.ll
@@ -343,13 +343,12 @@ define void @v_shuffle_v4i32_v4i32__7_0_u_u(ptr addrspace(1) inreg %ptr) {
 ; GFX90A-NEXT:    ;;#ASMSTART
 ; GFX90A-NEXT:    ; def v[0:3]
 ; GFX90A-NEXT:    ;;#ASMEND
+; GFX90A-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX90A-NEXT:    ;;#ASMSTART
 ; GFX90A-NEXT:    ; def v[2:5]
 ; GFX90A-NEXT:    ;;#ASMEND
-; GFX90A-NEXT:    v_mov_b32_e32 v6, 0
-; GFX90A-NEXT:    v_mov_b32_e32 v2, v5
-; GFX90A-NEXT:    v_mov_b32_e32 v3, v0
-; GFX90A-NEXT:    global_store_dwordx4 v6, v[2:5], s[16:17]
+; GFX90A-NEXT:    v_pk_mov_b32 v[0:1], v[4:5], v[0:1] op_sel:[1,0]
+; GFX90A-NEXT:    global_store_dwordx4 v6, v[0:3], s[16:17]
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
 ; GFX90A-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -364,9 +363,8 @@ define void @v_shuffle_v4i32_v4i32__7_0_u_u(ptr addrspace(1) inreg %ptr) {
 ; GFX940-NEXT:    ; def v[2:5]
 ; GFX940-NEXT:    ;;#ASMEND
 ; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_mov_b32_e32 v2, v5
-; GFX940-NEXT:    v_mov_b32_e32 v3, v0
-; GFX940-NEXT:    global_store_dwordx4 v6, v[2:5], s[0:1] sc0 sc1
+; GFX940-NEXT:    v_pk_mov_b32 v[0:1], v[4:5], v[0:1] op_sel:[1,0]
+; GFX940-NEXT:    global_store_dwordx4 v6, v[0:3], s[0:1] sc0 sc1
 ; GFX940-NEXT:    s_waitcnt vmcnt(0)
 ; GFX940-NEXT:    s_setpc_b64 s[30:31]
   %vec0 = call <4 x i32> asm "; def $0", "=v"()
@@ -456,8 +454,7 @@ define void @v_shuffle_v4i32_v4i32__7_2_u_u(ptr addrspace(1) inreg %ptr) {
 ; GFX90A-NEXT:    ;;#ASMSTART
 ; GFX90A-NEXT:    ; def v[4:7]
 ; GFX90A-NEXT:    ;;#ASMEND
-; GFX90A-NEXT:    v_mov_b32_e32 v0, v7
-; GFX90A-NEXT:    v_mov_b32_e32 v1, v2
+; GFX90A-NEXT:    v_pk_mov_b32 v[0:1], v[6:7], v[2:3] op_sel:[1,0]
 ; GFX90A-NEXT:    global_store_dwordx4 v8, v[0:3], s[16:17]
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
 ; GFX90A-NEXT:    s_setpc_b64 s[30:31]
@@ -472,8 +469,8 @@ define void @v_shuffle_v4i32_v4i32__7_2_u_u(ptr addrspace(1) inreg %ptr) {
 ; GFX940-NEXT:    ;;#ASMSTART
 ; GFX940-NEXT:    ; def v[4:7]
 ; GFX940-NEXT:    ;;#ASMEND
-; GFX940-NEXT:    v_mov_b32_e32 v1, v2
-; GFX940-NEXT:    v_mov_b32_e32 v0, v7
+; GFX940-NEXT:    s_nop 0
+; GFX940-NEXT:    v_pk_mov_b32 v[0:1], v[6:7], v[2:3] op_sel:[1,0]
 ; GFX940-NEXT:    global_store_dwordx4 v8, v[0:3], s[0:1] sc0 sc1
 ; GFX940-NEXT:    s_waitcnt vmcnt(0)
 ; GFX940-NEXT:    s_setpc_b64 s[30:31]
@@ -560,9 +557,8 @@ define void @v_shuffle_v4i32_v4i32__7_4_u_u(ptr addrspace(1) inreg %ptr) {
 ; GFX90A-NEXT:    ; def v[0:3]
 ; GFX90A-NEXT:    ;;#ASMEND
 ; GFX90A-NEXT:    v_mov_b32_e32 v4, 0
-; GFX90A-NEXT:    v_mov_b32_e32 v2, v3
-; GFX90A-NEXT:    v_mov_b32_e32 v3, v0
-; GFX90A-NEXT:    global_store_dwordx4 v4, v[2:5], s[16:17]
+; GFX90A-NEXT:    v_pk_mov_b32 v[0:1], v[2:3], v[0:1] op_sel:[1,0]
+; GFX90A-NEXT:    global_store_dwordx4 v4, v[0:3], s[16:17]
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
 ; GFX90A-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -573,9 +569,8 @@ define void @v_shuffle_v4i32_v4i32__7_4_u_u(ptr addrspace(1) inreg %ptr) {
 ; GFX940-NEXT:    ; def v[0:3]
 ; GFX940-NEXT:    ;;#ASMEND
 ; GFX940-NEXT:    v_mov_b32_e32 v4, 0
-; GFX940-NEXT:    v_mov_b32_e32 v2, v3
-; GFX940-NEXT:    v_mov_b32_e32 v3, v0
-; GFX940-NEXT:    global_store_dwordx4 v4, v[2:5], s[0:1] sc0 sc1
+; GFX940-NEXT:    v_pk_mov_b32 v[0:1], v[2:3], v[0:1] op_sel:[1,0]
+; GFX940-NEXT:    global_store_dwordx4 v4, v[0:3], s[0:1] sc0 sc1
 ; GFX940-NEXT:    s_waitcnt vmcnt(0)
 ; GFX940-NEXT:    s_setpc_b64 s[30:31]
   %vec0 = call <4 x i32> asm "; def $0", "=v"()
@@ -649,8 +644,7 @@ define void @v_shuffle_v4i32_v4i32__7_6_u_u(ptr addrspace(1) inreg %ptr) {
 ; GFX90A-NEXT:    ; def v[0:3]
 ; GFX90A-NEXT:    ;;#ASMEND
 ; GFX90A-NEXT:    v_mov_b32_e32 v4, 0
-; GFX90A-NEXT:    v_mov_b32_e32 v0, v3
-; GFX90A-NEXT:    v_mov_b32_e32 v1, v2
+; GFX90A-NEXT:    v_pk_mov_b32 v[0:1], v[2:3], v[2:3] op_sel:[1,0]
 ; GFX90A-NEXT:    global_store_dwordx4 v4, v[0:3], s[16:17]
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
 ; GFX90A-NEXT:    s_setpc_b64 s[30:31]
@@ -662,8 +656,7 @@ define void @v_shuffle_v4i32_v4i32__7_6_u_u(ptr addrspace(1) inreg %ptr) {
 ; GFX940-NEXT:    ; def v[0:3]
 ; GFX940-NEXT:    ;;#ASMEND
 ; GFX940-NEXT:    v_mov_b32_e32 v4, 0
-; GFX940-NEXT:    v_mov_b32_e32 v0, v3
-; GFX940-NEXT:    v_mov_b32_e32 v1, v2
+; GFX940-NEXT:    v_pk_mov_b32 v[0:1], v[2:3], v[2:3] op_sel:[1,0]
 ; GFX940-NEXT:    global_store_dwordx4 v4, v[0:3], s[0:1] sc0 sc1
 ; GFX940-NEXT:    s_waitcnt vmcnt(0)
 ; GFX940-NEXT:    s_setpc_b64 s[30:31]
@@ -1169,11 +1162,10 @@ define void @v_shuffle_v4i32_v4i32__7_7_7_0(ptr addrspace(1) inreg %ptr) {
 ; GFX90A-NEXT:    ; def v[2:5]
 ; GFX90A-NEXT:    ;;#ASMEND
 ; GFX90A-NEXT:    v_mov_b32_e32 v6, 0
-; GFX90A-NEXT:    v_mov_b32_e32 v2, v5
-; GFX90A-NEXT:    v_mov_b32_e32 v3, v5
-; GFX90A-NEXT:    v_mov_b32_e32 v4, v5
-; GFX90A-NEXT:    v_mov_b32_e32 v5, v0
-; GFX90A-NEXT:    global_store_dwordx4 v6, v[2:5], s[16:17]
+; GFX90A-NEXT:    v_pk_mov_b32 v[2:3], v[4:5], v[0:1] op_sel:[1,0]
+; GFX90A-NEXT:    v_mov_b32_e32 v0, v5
+; GFX90A-NEXT:    v_mov_b32_e32 v1, v5
+; GFX90A-NEXT:    global_store_dwordx4 v6, v[0:3], s[16:17]
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
 ; GFX90A-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -1188,11 +1180,10 @@ define void @v_shuffle_v4i32_v4i32__7_7_7_0(ptr addrspace(1) inreg %ptr) {
 ; GFX940-NEXT:    ; def v[2:5]
 ; GFX940-NEXT:    ;;#ASMEND
 ; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_mov_b32_e32 v2, v5
-; GFX940-NEXT:    v_mov_b32_e32 v3, v5
-; GFX940-NEXT:    v_mov_b32_e32 v4, v5
-; GFX940-NEXT:    v_mov_b32_e32 v5, v0
-; GFX940-NEXT:    global_store_dwordx4 v6, v[2:5], s[0:1] sc0 sc1
+; GFX940-NEXT:    v_pk_mov_b32 v[2:3], v[4:5], v[0:1] op_sel:[1,0]
+; GFX940-NEXT:    v_mov_b32_e32 v0, v5
+; GFX940-NEXT:    v_mov_b32_e32 v1, v5
+; GFX940-NEXT:    global_store_dwordx4 v6, v[0:3], s[0:1] sc0 sc1
 ; GFX940-NEXT:    s_waitcnt vmcnt(0)
 ; GFX940-NEXT:    s_setpc_b64 s[30:31]
   %vec0 = call <4 x i32> asm "; def $0", "=v"()
@@ -1287,17 +1278,16 @@ define void @v_shuffle_v4i32_v4i32__7_7_7_2(ptr addrspace(1) inreg %ptr) {
 ; GFX90A:       ; %bb.0:
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX90A-NEXT:    ;;#ASMSTART
-; GFX90A-NEXT:    ; def v[4:7]
+; GFX90A-NEXT:    ; def v[0:3]
 ; GFX90A-NEXT:    ;;#ASMEND
 ; GFX90A-NEXT:    v_mov_b32_e32 v8, 0
 ; GFX90A-NEXT:    ;;#ASMSTART
-; GFX90A-NEXT:    ; def v[0:3]
+; GFX90A-NEXT:    ; def v[4:7]
 ; GFX90A-NEXT:    ;;#ASMEND
-; GFX90A-NEXT:    v_mov_b32_e32 v4, v7
-; GFX90A-NEXT:    v_mov_b32_e32 v5, v7
-; GFX90A-NEXT:    v_mov_b32_e32 v6, v7
-; GFX90A-NEXT:    v_mov_b32_e32 v7, v2
-; GFX90A-NEXT:    global_store_dwordx4 v8, v[4:7], s[16:17]
+; GFX90A-NEXT:    v_pk_mov_b32 v[2:3], v[6:7], v[2:3] op_sel:[1,0]
+; GFX90A-NEXT:    v_mov_b32_e32 v0, v7
+; GFX90A-NEXT:    v_mov_b32_e32 v1, v7
+; GFX90A-NEXT:    global_store_dwordx4 v8, v[0:3], s[16:17]
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
 ; GFX90A-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -1305,17 +1295,17 @@ define void @v_shuffle_v4i32_v4i32__7_7_7_2(ptr addrspace(1) inreg %ptr) {
 ; GFX940:       ; %bb.0:
 ; GFX940-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX940-NEXT:    ;;#ASMSTART
-; GFX940-NEXT:    ; def v[4:7]
+; GFX940-NEXT:    ; def v[0:3]
 ; GFX940-NEXT:    ;;#ASMEND
 ; GFX940-NEXT:    v_mov_b32_e32 v8, 0
 ; GFX940-NEXT:    ;;#ASMSTART
-; GFX940-NEXT:    ; def v[0:3]
+; GFX940-NEXT:    ; def v[4:7]
 ; GFX940-NEXT:    ;;#ASMEND
-; GFX940-NEXT:    v_mov_b32_e32 v4, v7
-; GFX940-NEXT:    v_mov_b32_e32 v5, v7
-; GFX940-NEXT:    v_mov_b32_e32 v6, v7
-; GFX940-NEXT:    v_mov_b32_e32 v7, v2
-; GFX940-NEXT:    global_store_dwordx4 v8, v[4:7], s[0:1] sc0 sc1
+; GFX940-NEXT:    s_nop 0
+; GFX940-NEXT:    v_pk_mov_b32 v[2:3], v[6:7], v[2:3] op_sel:[1,0]
+; GFX940-NEXT:    v_mov_b32_e32 v0, v7
+; GFX940-NEXT:    v_mov_b32_e32 v1, v7
+; GFX940-NEXT:    global_store_dwordx4 v8, v[0:3], s[0:1] sc0 sc1
 ; GFX940-NEXT:    s_waitcnt vmcnt(0)
 ; GFX940-NEXT:    s_setpc_b64 s[30:31]
   %vec0 = call <4 x i32> asm "; def $0", "=v"()
@@ -1402,31 +1392,26 @@ define void @v_shuffle_v4i32_v4i32__7_7_7_4(ptr addrspace(1) inreg %ptr) {
 ; GFX90A-LABEL: v_shuffle_v4i32_v4i32__7_7_7_4:
 ; GFX90A:       ; %bb.0:
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT:    v_mov_b32_e32 v8, 0
 ; GFX90A-NEXT:    ;;#ASMSTART
 ; GFX90A-NEXT:    ; def v[0:3]
 ; GFX90A-NEXT:    ;;#ASMEND
-; GFX90A-NEXT:    v_mov_b32_e32 v4, v3
-; GFX90A-NEXT:    v_mov_b32_e32 v5, v3
-; GFX90A-NEXT:    v_mov_b32_e32 v6, v3
-; GFX90A-NEXT:    v_mov_b32_e32 v7, v0
-; GFX90A-NEXT:    global_store_dwordx4 v8, v[4:7], s[16:17]
+; GFX90A-NEXT:    v_mov_b32_e32 v6, 0
+; GFX90A-NEXT:    v_pk_mov_b32 v[4:5], v[2:3], v[0:1] op_sel:[1,0]
+; GFX90A-NEXT:    v_mov_b32_e32 v2, v3
+; GFX90A-NEXT:    global_store_dwordx4 v6, v[2:5], s[16:17]
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
 ; GFX90A-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: v_shuffle_v4i32_v4i32__7_7_7_4:
 ; GFX940:       ; %bb.0:
 ; GFX940-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX940-NEXT:    v_mov_b32_e32 v8, 0
 ; GFX940-NEXT:    ;;#ASMSTART
 ; GFX940-NEXT:    ; def v[0:3]
 ; GFX940-NEXT:    ;;#ASMEND
-; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_mov_b32_e32 v4, v3
-; GFX940-NEXT:    v_mov_b32_e32 v5, v3
-; GFX940-NEXT:    v_mov_b32_e32 v6, v3
-; GFX940-NEXT:    v_mov_b32_e32 v7, v0
-; GFX940-NEXT:    global_store_dwordx4 v8, v[4:7], s[0:1] sc0 sc1
+; GFX940-NEXT:    v_mov_b32_e32 v6, 0
+; GFX940-NEXT:    v_pk_mov_b32 v[4:5], v[2:3], v[0:1] op_sel:[1,0]
+; GFX940-NEXT:    v_mov_b32_e32 v2, v3
+; GFX940-NEXT:    global_store_dwordx4 v6, v[2:5], s[0:1] sc0 sc1
 ; GFX940-NEXT:    s_waitcnt vmcnt(0)
 ; GFX940-NEXT:    s_setpc_b64 s[30:31]
   %vec0 = call <4 x i32> asm "; def $0", "=v"()
@@ -1508,31 +1493,26 @@ define void @v_shuffle_v4i32_v4i32__7_7_7_6(ptr addrspace(1) inreg %ptr) {
 ; GFX90A-LABEL: v_shuffle_v4i32_v4i32__7_7_7_6:
 ; GFX90A:       ; %bb.0:
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT:    v_mov_b32_e32 v8, 0
 ; GFX90A-NEXT:    ;;#ASMSTART
 ; GFX90A-NEXT:    ; def v[0:3]
 ; GFX90A-NEXT:    ;;#ASMEND
-; GFX90A-NEXT:    v_mov_b32_e32 v4, v3
-; GFX90A-NEXT:    v_mov_b32_e32 v5, v3
-; GFX90A-NEXT:    v_mov_b32_e32 v6, v3
-; GFX90A-NEXT:    v_mov_b32_e32 v7, v2
-; GFX90A-NEXT:    global_store_dwordx4 v8, v[4:7], s[16:17]
+; GFX90A-NEXT:    v_mov_b32_e32 v6, 0
+; GFX90A-NEXT:    v_pk_mov_b32 v[4:5], v[2:3], v[2:3] op_sel:[1,0]
+; GFX90A-NEXT:    v_mov_b32_e32 v2, v3
+; GFX90A-NEXT:    global_store_dwordx4 v6, v[2:5], s[16:17]
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
 ; GFX90A-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: v_shuffle_v4i32_v4i32__7_7_7_6:
 ; GFX940:       ; %bb.0:
 ; GFX940-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX940-NEXT:    v_mov_b32_e32 v8, 0
 ; GFX940-NEXT:    ;;#ASMSTART
 ; GFX940-NEXT:    ; def v[0:3]
 ; GFX940-NEXT:    ;;#ASMEND
-; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_mov_b32_e32 v4, v3
-; GFX940-NEXT:    v_mov_b32_e32 v5, v3
-; GFX940-NEXT:    v_mov_b32_e32 v6, v3
-; GFX940-NEXT:    v_mov_b32_e32 v7, v2
-; GFX940-NEXT:    global_store_dwordx4 v8, v[4:7], s[0:1] sc0 sc1
+; GFX940-NEXT:    v_mov_b32_e32 v6, 0
+; GFX940-NEXT:    v_pk_mov_b32 v[4:5], v[2:3], v[2:3] op_sel:[1,0]
+; GFX940-NEXT:    v_mov_b32_e32 v2, v3
+; GFX940-NEXT:    global_store_dwordx4 v6, v[2:5], s[0:1] sc0 sc1
 ; GFX940-NEXT:    s_waitcnt vmcnt(0)
 ; GFX940-NEXT:    s_setpc_b64 s[30:31]
   %vec0 = call <4 x i32> asm "; def $0", "=v"()
@@ -1709,8 +1689,7 @@ define void @v_shuffle_v4i32_v4i32__1_0_0_0(ptr addrspace(1) inreg %ptr) {
 ; GFX90A-NEXT:    ; def v[0:3]
 ; GFX90A-NEXT:    ;;#ASMEND
 ; GFX90A-NEXT:    v_mov_b32_e32 v6, 0
-; GFX90A-NEXT:    v_mov_b32_e32 v2, v1
-; GFX90A-NEXT:    v_mov_b32_e32 v3, v0
+; GFX90A-NEXT:    v_pk_mov_b32 v[2:3], v[0:1], v[0:1] op_sel:[1,0]
 ; GFX90A-NEXT:    v_mov_b32_e32 v4, v0
 ; GFX90A-NEXT:    v_mov_b32_e32 v5, v0
 ; GFX90A-NEXT:    global_store_dwordx4 v6, v[2:5], s[16:17]
@@ -1724,8 +1703,7 @@ define void @v_shuffle_v4i32_v4i32__1_0_0_0(ptr addrspace(1) inreg %ptr) {
 ; GFX940-NEXT:    ; def v[0:3]
 ; GFX940-NEXT:    ;;#ASMEND
 ; GFX940-NEXT:    v_mov_b32_e32 v6, 0
-; GFX940-NEXT:    v_mov_b32_e32 v2, v1
-; GFX940-NEXT:    v_mov_b32_e32 v3, v0
+; GFX940-NEXT:    v_pk_mov_b32 v[2:3], v[0:1], v[0:1] op_sel:[1,0]
 ; GFX940-NEXT:    v_mov_b32_e32 v4, v0
 ; GFX940-NEXT:    v_mov_b32_e32 v5, v0
 ; GFX940-NEXT:    global_store_dwordx4 v6, v[2:5], s[0:1] sc0 sc1
@@ -1809,8 +1787,7 @@ define void @v_shuffle_v4i32_v4i32__3_0_0_0(ptr addrspace(1) inreg %ptr) {
 ; GFX90A-NEXT:    ; def v[0:3]
 ; GFX90A-NEXT:    ;;#ASMEND
 ; GFX90A-NEXT:    v_mov_b32_e32 v6, 0
-; GFX90A-NEXT:    v_mov_b32_e32 v2, v3
-; GFX90A-NEXT:    v_mov_b32_e32 v3, v0
+; GFX90A-NEXT:    v_pk_mov_b32 v[2:3], v[2:3], v[0:1] op_sel:[1,0]
 ; GFX90A-NEXT:    v_mov_b32_e32 v4, v0
 ; GFX90A-NEXT:    v_mov_b32_e32 v5, v0
 ; GFX90A-NEXT:    global_store_dwordx4 v6, v[2:5], s[16:17]
@@ -1824,8 +1801,7 @@ define void @v_shuffle_v4i32_v4i32__3_0_0_0(ptr addrspace(1) inreg %ptr) {
 ; GFX940-NEXT:    ; def v[0:3]
 ; GFX940-NEXT:    ;;#ASMEND
 ; GFX940-NEXT:    v_mov_b32_e32 v6, 0
-; GFX940-NEXT:    v_mov_b32_e32 v2, v3
-; GFX940-NEXT:    v_mov_b32_e32 v3, v0
+; GFX940-NEXT:    v_pk_mov_b32 v[2:3], v[2:3], v[0:1] op_sel:[1,0]
 ; GFX940-NEXT:    v_mov_b32_e32 v4, v0
 ; GFX940-NEXT:    v_mov_b32_e32 v5, v0
 ; GFX940-NEXT:    global_store_dwordx4 v6, v[2:5], s[0:1] sc0 sc1
@@ -1914,8 +1890,7 @@ define void @v_shuffle_v4i32_v4i32__5_0_0_0(ptr addrspace(1) inreg %ptr) {
 ; GFX90A-NEXT:    ; def v[2:5]
 ; GFX90A-NEXT:    ;;#ASMEND
 ; GFX90A-NEXT:    v_mov_b32_e32 v6, 0
-; GFX90A-NEXT:    v_mov_b32_e32 v2, v3
-; GFX90A-NEXT:    v_mov_b32_e32 v3, v0
+; GFX90A-NEXT:    v_pk_mov_b32 v[2:3], v[2:3], v[0:1] op_sel:[1,0]
 ; GFX90A-NEXT:    v_mov_b32_e32 v4, v0
 ; GFX90A-NEXT:    v_mov_b32_e32 v5, v0
 ; GFX90A-NEXT:    global_store_dwordx4 v6, v[2:5], s[16:17]
@@ -1933,8 +1908,7 @@ define void @v_shuffle_v4i32_v4i32__5_0_0_0(ptr addrspace(1) inreg %ptr) {
 ; GFX940-NEXT:    ; def v[2:5]
 ; GFX940-NEXT:    ;;#ASMEND
 ; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_mov_b32_e32 v2, v3
-; GFX940-NEXT:    v_mov_b32_e32 v3, v0
+; GFX940-NEXT:    v_pk_mov_b32 v[2:3], v[2:3], v[0:1] op_sel:[1,0]
 ; GFX940-NEXT:    v_mov_b32_e32 v4, v0
 ; GFX940-NEXT:    v_mov_b32_e32 v5, v0
 ; GFX940-NEXT:    global_store_dwordx4 v6, v[2:5], s[0:1] sc0 sc1
@@ -2038,8 +2012,7 @@ define void @v_shuffle_v4i32_v4i32__7_0_0_0(ptr addrspace(1) inreg %ptr) {
 ; GFX90A-NEXT:    ; def v[2:5]
 ; GFX90A-NEXT:    ;;#ASMEND
 ; GFX90A-NEXT:    v_mov_b32_e32 v6, 0
-; GFX90A-NEXT:    v_mov_b32_e32 v2, v5
-; GFX90A-NEXT:    v_mov_b32_e32 v3, v0
+; GFX90A-NEXT:    v_pk_mov_b32 v[2:3], v[4:5], v[0:1] op_sel:[1,0]
 ; GFX90A-NEXT:    v_mov_b32_e32 v4, v0
 ; GFX90A-NEXT:    v_mov_b32_e32 v5, v0
 ; GFX90A-NEXT:    global_store_dwordx4 v6, v[2:5], s[16:17]
@@ -2057,8 +2030,7 @@ define void @v_shuffle_v4i32_v4i32__7_0_0_0(ptr addrspace(1) inreg %ptr) {
 ; GFX940-NEXT:    ; def v[2:5]
 ; GFX940-NEXT:    ;;#ASMEND
 ; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_mov_b32_e32 v2, v5
-; GFX940-NEXT:    v_mov_b32_e32 v3, v0
+; GFX940-NEXT:    v_pk_mov_b32 v[2:3], v[4:5], v[0:1] op_sel:[1,0]
 ; GFX940-NEXT:    v_mov_b32_e32 v4, v0
 ; GFX940-NEXT:    v_mov_b32_e32 v5, v0
 ; GFX940-NEXT:    global_store_dwordx4 v6, v[2:5], s[0:1] sc0 sc1
@@ -2214,17 +2186,16 @@ define void @v_shuffle_v4i32_v4i32__7_2_0_0(ptr addrspace(1) inreg %ptr) {
 ; GFX90A:       ; %bb.0:
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX90A-NEXT:    ;;#ASMSTART
-; GFX90A-NEXT:    ; def v[4:7]
+; GFX90A-NEXT:    ; def v[0:3]
 ; GFX90A-NEXT:    ;;#ASMEND
-; GFX90A-NEXT:    v_mov_b32_e32 v8, 0
 ; GFX90A-NEXT:    ;;#ASMSTART
-; GFX90A-NEXT:    ; def v[0:3]
+; GFX90A-NEXT:    ; def v[4:7]
 ; GFX90A-NEXT:    ;;#ASMEND
-; GFX90A-NEXT:    v_mov_b32_e32 v4, v7
-; GFX90A-NEXT:    v_mov_b32_e32 v5, v2
-; GFX90A-NEXT:    v_mov_b32_e32 v6, v0
-; GFX90A-NEXT:    v_mov_b32_e32 v7, v0
-; GFX90A-NEXT:    global_store_dwordx4 v8, v[4:7], s[16:17]
+; GFX90A-NEXT:    v_mov_b32_e32 v8, 0
+; GFX90A-NEXT:    v_pk_mov_b32 v[2:3], v[6:7], v[2:3] op_sel:[1,0]
+; GFX90A-NEXT:    v_mov_b32_e32 v4, v0
+; GFX90A-NEXT:    v_mov_b32_e32 v5, v0
+; GFX90A-NEXT:    global_store_dwordx4 v8, v[2:5], s[16:17]
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
 ; GFX90A-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -2232,17 +2203,16 @@ define void @v_shuffle_v4i32_v4i32__7_2_0_0(ptr addrspace(1) inreg %ptr) {
 ; GFX940:       ; %bb.0:
 ; GFX940-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX940-NEXT:    ;;#ASMSTART
-; GFX940-NEXT:    ; def v[4:7]
+; GFX940-NEXT:    ; def v[0:3]
 ; GFX940-NEXT:    ;;#ASMEND
-; GFX940-NEXT:    v_mov_b32_e32 v8, 0
 ; GFX940-NEXT:    ;;#ASMSTART
-; GFX940-NEXT:    ; def v[0:3]
+; GFX940-NEXT:    ; def v[4:7]
 ; GFX940-NEXT:    ;;#ASMEND
-; GFX940-NEXT:    v_mov_b32_e32 v4, v7
-; GFX940-NEXT:    v_mov_b32_e32 v5, v2
-; GFX940-NEXT:    v_mov_b32_e32 v6, v0
-; GFX940-NEXT:    v_mov_b32_e32 v7, v0
-; GFX940-NEXT:    global_store_dwordx4 v8, v[4:7], s[0:1] sc0 sc1
+; GFX940-NEXT:    v_mov_b32_e32 v8, 0
+; GFX940-NEXT:    v_pk_mov_b32 v[2:3], v[6:7], v[2:3] op_sel:[1,0]
+; GFX940-NEXT:    v_mov_b32_e32 v4, v0
+; GFX940-NEXT:    v_mov_b32_e32 v5, v0
+; GFX940-NEXT:    global_store_dwordx4 v8, v[2:5], s[0:1] sc0 sc1
 ; GFX940-NEXT:    s_waitcnt vmcnt(0)
 ; GFX940-NEXT:    s_setpc_b64 s[30:31]
   %vec0 = call <4 x i32> asm "; def $0", "=v"()
@@ -2339,12 +2309,11 @@ define void @v_shuffle_v4i32_v4i32__7_4_0_0(ptr addrspace(1) inreg %ptr) {
 ; GFX90A-NEXT:    ;;#ASMSTART
 ; GFX90A-NEXT:    ; def v[2:5]
 ; GFX90A-NEXT:    ;;#ASMEND
-; GFX90A-NEXT:    v_mov_b32_e32 v8, 0
-; GFX90A-NEXT:    v_mov_b32_e32 v4, v5
-; GFX90A-NEXT:    v_mov_b32_e32 v5, v2
-; GFX90A-NEXT:    v_mov_b32_e32 v6, v0
-; GFX90A-NEXT:    v_mov_b32_e32 v7, v0
-; GFX90A-NEXT:    global_store_dwordx4 v8, v[4:7], s[16:17]
+; GFX90A-NEXT:    v_mov_b32_e32 v6, 0
+; GFX90A-NEXT:    v_pk_mov_b32 v[2:3], v[4:5], v[2:3] op_sel:[1,0]
+; GFX90A-NEXT:    v_mov_b32_e32 v4, v0
+; GFX90A-NEXT:    v_mov_b32_e32 v5, v0
+; GFX90A-NEXT:    global_store_dwordx4 v6, v[2:5], s[16:17]
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
 ; GFX90A-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -2354,15 +2323,15 @@ define void @v_shuffle_v4i32_v4i32__7_4_0_0(ptr addrspace(1) inreg %ptr) {
 ; GFX940-NEXT:    ;;#ASMSTART
 ; GFX940-NEXT:    ; def v[0:3]
 ; GFX940-NEXT:    ;;#ASMEND
-; GFX940-NEXT:    v_mov_b32_e32 v8, 0
+; GFX940-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX940-NEXT:    ;;#ASMSTART
 ; GFX940-NEXT:    ; def v[2:5]
 ; GFX940-NEXT:    ;;#ASMEND
-; GFX940-NEXT:    v_mov_b32_e32 v6, v0
-; GFX940-NEXT:    v_mov_b32_e32 v4, v5
-; GFX940-NEXT:    v_mov_b32_e32 v5, v2
-; GFX940-NEXT:    v_mov_b32_e32 v7, v0
-; GFX940-NEXT:    global_store_dwordx4 v8, v[4:7], s[0:1] sc0 sc1
+; GFX940-NEXT:    s_nop 0
+; GFX940-NEXT:    v_pk_mov_b32 v[2:3], v[4:5], v[2:3] op_sel:[1,0]
+; GFX940-NEXT:    v_mov_b32_e32 v4, v0
+; GFX940-NEXT:    v_mov_b32_e32 v5, v0
+; GFX940-NEXT:    global_store_dwordx4 v6, v[2:5], s[0:1] sc0 sc1
 ; GFX940-NEXT:    s_waitcnt vmcnt(0)
 ; GFX940-NEXT:    s_setpc_b64 s[30:31]
   %vec0 = call <4 x i32> asm "; def $0", "=v"()
@@ -2460,8 +2429,7 @@ define void @v_shuffle_v4i32_v4i32__7_6_0_0(ptr addrspace(1) inreg %ptr) {
 ; GFX90A-NEXT:    ; def v[2:5]
 ; GFX90A-NEXT:    ;;#ASMEND
 ; GFX90A-NEXT:    v_mov_b32_e32 v6, 0
-; GFX90A-NEXT:    v_mov_b32_e32 v2, v5
-; GFX90A-NEXT:    v_mov_b32_e32 v3, v4
+; GFX90A-NEXT:    v_pk_mov_b32 v[2:3], v[4:5], v[4:5] op_sel:[1,0]
 ; GFX90A-NEXT:    v_mov_b32_e32 v4, v0
 ; GFX90A-NEXT:    v_mov_b32_e32 v5, v0
 ; GFX90A-NEXT:    global_store_dwordx4 v6, v[2:5], s[16:17]
@@ -2479,8 +2447,7 @@ define void @v_shuffle_v4i32_v4i32__7_6_0_0(ptr addrspace(1) inreg %ptr) {
 ; GFX940-NEXT:    ; def v[2:5]
 ; GFX940-NEXT:    ;;#ASMEND
 ; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_mov_b32_e32 v2, v5
-; GFX940-NEXT:    v_mov_b32_e32 v3, v4
+; GFX940-NEXT:    v_pk_mov_b32 v[2:3], v[4:5], v[4:5] op_sel:[1,0]
 ; GFX940-NEXT:    v_mov_b32_e32 v4, v0
 ; GFX940-NEXT:    v_mov_b32_e32 v5, v0
 ; GFX940-NEXT:    global_store_dwordx4 v6, v[2:5], s[0:1] sc0 sc1
@@ -2643,11 +2610,10 @@ define void @v_shuffle_v4i32_v4i32__7_7_1_0(ptr addrspace(1) inreg %ptr) {
 ; GFX90A-NEXT:    ; def v[2:5]
 ; GFX90A-NEXT:    ;;#ASMEND
 ; GFX90A-NEXT:    v_mov_b32_e32 v6, 0
-; GFX90A-NEXT:    v_mov_b32_e32 v2, v5
-; GFX90A-NEXT:    v_mov_b32_e32 v3, v5
-; GFX90A-NEXT:    v_mov_b32_e32 v4, v1
-; GFX90A-NEXT:    v_mov_b32_e32 v5, v0
-; GFX90A-NEXT:    global_store_dwordx4 v6, v[2:5], s[16:17]
+; GFX90A-NEXT:    v_pk_mov_b32 v[2:3], v[0:1], v[0:1] op_sel:[1,0]
+; GFX90A-NEXT:    v_mov_b32_e32 v0, v5
+; GFX90A-NEXT:    v_mov_b32_e32 v1, v5
+; GFX90A-NEXT:    global_store_dwordx4 v6, v[0:3], s[16:17]
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
 ; GFX90A-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -2662,11 +2628,10 @@ define void @v_shuffle_v4i32_v4i32__7_7_1_0(ptr addrspace(1) inreg %ptr) {
 ; GFX940-NEXT:    ; def v[2:5]
 ; GFX940-NEXT:    ;;#ASMEND
 ; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_mov_b32_e32 v2, v5
-; GFX940-NEXT:    v_mov_b32_e32 v3, v5
-; GFX940-NEXT:    v_mov_b32_e32 v4, v1
-; GFX940-NEXT:    v_mov_b32_e32 v5, v0
-; GFX940-NEXT:    global_store_dwordx4 v6, v[2:5], s[0:1] sc0 sc1
+; GFX940-NEXT:    v_pk_mov_b32 v[2:3], v[0:1], v[0:1] op_sel:[1,0]
+; GFX940-NEXT:    v_mov_b32_e32 v0, v5
+; GFX940-NEXT:    v_mov_b32_e32 v1, v5
+; GFX940-NEXT:    global_store_dwordx4 v6, v[0:3], s[0:1] sc0 sc1
 ; GFX940-NEXT:    s_waitcnt vmcnt(0)
 ; GFX940-NEXT:    s_setpc_b64 s[30:31]
   %vec0 = call <4 x i32> asm "; def $0", "=v"()
@@ -2759,17 +2724,16 @@ define void @v_shuffle_v4i32_v4i32__7_7_3_0(ptr addrspace(1) inreg %ptr) {
 ; GFX90A:       ; %bb.0:
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX90A-NEXT:    ;;#ASMSTART
-; GFX90A-NEXT:    ; def v[4:7]
+; GFX90A-NEXT:    ; def v[0:3]
 ; GFX90A-NEXT:    ;;#ASMEND
 ; GFX90A-NEXT:    v_mov_b32_e32 v8, 0
 ; GFX90A-NEXT:    ;;#ASMSTART
-; GFX90A-NEXT:    ; def v[0:3]
+; GFX90A-NEXT:    ; def v[4:7]
 ; GFX90A-NEXT:    ;;#ASMEND
-; GFX90A-NEXT:    v_mov_b32_e32 v4, v7
-; GFX90A-NEXT:    v_mov_b32_e32 v5, v7
-; GFX90A-NEXT:    v_mov_b32_e32 v6, v3
-; GFX90A-NEXT:    v_mov_b32_e32 v7, v0
-; GFX90A-NEXT:    global_store_dwordx4 v8, v[4:7], s[16:17]
+; GFX90A-NEXT:    v_pk_mov_b32 v[2:3], v[2:3], v[0:1] op_sel:[1,0]
+; GFX90A-NEXT:    v_mov_b32_e32 v0, v7
+; GFX90A-NEXT:    v_mov_b32_e32 v1, v7
+; GFX90A-NEXT:    global_store_dwordx4 v8, v[0:3], s[16:17]
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
 ; GFX90A-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -2777,17 +2741,16 @@ define void @v_shuffle_v4i32_v4i32__7_7_3_0(ptr addrspace(1) inreg %ptr) {
 ; GFX940:       ; %bb.0:
 ; GFX940-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX940-NEXT:    ;;#ASMSTART
-; GFX940-NEXT:    ; def v[4:7]
+; GFX940-NEXT:    ; def v[0:3]
 ; GFX940-NEXT:    ;;#ASMEND
 ; GFX940-NEXT:    v_mov_b32_e32 v8, 0
 ; GFX940-NEXT:    ;;#ASMSTART
-; GFX940-NEXT:    ; def v[0:3]
+; GFX940-NEXT:    ; def v[4:7]
 ; GFX940-NEXT:    ;;#ASMEND
-; GFX940-NEXT:    v_mov_b32_e32 v4, v7
-; GFX940-NEXT:    v_mov_b32_e32 v5, v7
-; GFX940-NEXT:    v_mov_b32_e32 v6, v3
-; GFX940-NEXT:    v_mov_b32_e32 v7, v0
-; GFX940-NEXT:    global_store_dwordx4 v8, v[4:7], s[0:1] sc0 sc1
+; GFX940-NEXT:    v_pk_mov_b32 v[2:3], v[2:3], v[0:1] op_sel:[1,0]
+; GFX940-NEXT:    v_mov_b32_e32 v0, v7
+; GFX940-NEXT:    v_mov_b32_e32 v1, v7
+; GFX940-NEXT:    global_store_dwordx4 v8, v[0:3], s[0:1] sc0 sc1
 ; GFX940-NEXT:    s_waitcnt vmcnt(0)
 ; GFX940-NEXT:    s_setpc_b64 s[30:31]
   %vec0 = call <4 x i32> asm "; def $0", "=v"()
@@ -2883,11 +2846,11 @@ define void @v_shuffle_v4i32_v4i32__7_7_5_0(ptr addrspace(1) inreg %ptr) {
 ; GFX90A-NEXT:    ;;#ASMSTART
 ; GFX90A-NEXT:    ; def v[2:5]
 ; GFX90A-NEXT:    ;;#ASMEND
-; GFX90A-NEXT:    v_mov_b32_e32 v8, 0
-; GFX90A-NEXT:    v_mov_b32_e32 v4, v5
-; GFX90A-NEXT:    v_mov_b32_e32 v6, v3
-; GFX90A-NEXT:    v_mov_b32_e32 v7, v0
-; GFX90A-NEXT:    global_store_dwordx4 v8, v[4:7], s[16:17]
+; GFX90A-NEXT:    v_mov_b32_e32 v6, 0
+; GFX90A-NEXT:    v_pk_mov_b32 v[2:3], v[2:3], v[0:1] op_sel:[1,0]
+; GFX90A-NEXT:    v_mov_b32_e32 v0, v5
+; GFX90A-NEXT:    v_mov_b32_e32 v1, v5
+; GFX90A-NEXT:    global_store_dwordx4 v6, v[0:3], s[16:17]
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
 ; GFX90A-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -2897,14 +2860,15 @@ define void @v_shuffle_v4i32_v4i32__7_7_5_0(ptr addrspace(1) inreg %ptr) {
 ; GFX940-NEXT:    ;;#ASMSTART
 ; GFX940-NEXT:    ; def v[0:3]
 ; GFX940-NEXT:    ;;#ASMEND
-; GFX940-NEXT:    v_mov_b32_e32 v8, 0
+; GFX940-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX940-NEXT:    ;;#ASMSTART
 ; GFX940-NEXT:    ; def v[2:5]
 ; GFX940-NEXT:    ;;#ASMEND
-; GFX940-NEXT:    v_mov_b32_e32 v7, v0
-; GFX940-NEXT:    v_mov_b32_e32 v4, v5
-; GFX940-NEXT:    v_mov_b32_e32 v6, v3
-; GFX940-NEXT:    global_store_dwordx4 v8, v[4:7], s[0:1] sc0 sc1
+; GFX940-NEXT:    s_nop 0
+; GFX940-NEXT:    v_pk_mov_b32 v[2:3], v[2:3], v[0:1] op_sel:[1,0]
+; GFX940-NEXT:    v_mov_b32_e32 v0, v5
+; GFX940-NEXT:    v_mov_b32_e32 v1, v5
+; GFX940-NEXT:    global_store_dwordx4 v6, v[0:3], s[0:1] sc0 sc1
 ; GFX940-NEXT:    s_waitcnt vmcnt(0)
 ; GFX940-NEXT:    s_setpc_b64 s[30:31]
   %vec0 = call <4 x i32> asm "; def $0", "=v"()
@@ -3517,8 +3481,7 @@ define void @v_shuffle_v4i32_v4i32__7_0_1_1(ptr addrspace(1) inreg %ptr) {
 ; GFX90A-NEXT:    ; def v[2:5]
 ; GFX90A-NEXT:    ;;#ASMEND
 ; GFX90A-NEXT:    v_mov_b32_e32 v6, 0
-; GFX90A-NEXT:    v_mov_b32_e32 v2, v5
-; GFX90A-NEXT:    v_mov_b32_e32 v3, v0
+; GFX90A-NEXT:    v_pk_mov_b32 v[2:3], v[4:5], v[0:1] op_sel:[1,0]
 ; GFX90A-NEXT:    v_mov_b32_e32 v4, v1
 ; GFX90A-NEXT:    v_mov_b32_e32 v5, v1
 ; GFX90A-NEXT:    global_store_dwordx4 v6, v[2:5], s[16:17]
@@ -3536,8 +3499,7 @@ define void @v_shuffle_v4i32_v4i32__7_0_1_1(ptr addrspace(1) inreg %ptr) {
 ; GFX940-NEXT:    ; def v[2:5]
 ; GFX940-NEXT:    ;;#ASMEND
 ; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_mov_b32_e32 v2, v5
-; GFX940-NEXT:    v_mov_b32_e32 v3, v0
+; GFX940-NEXT:    v_pk_mov_b32 v[2:3], v[4:5], v[0:1] op_sel:[1,0]
 ; GFX940-NEXT:    v_mov_b32_e32 v4, v1
 ; GFX940-NEXT:    v_mov_b32_e32 v5, v1
 ; GFX940-NEXT:    global_store_dwordx4 v6, v[2:5], s[0:1] sc0 sc1
@@ -3573,17 +3535,16 @@ define void @v_shuffle_v4i32_v4i32__7_2_1_1(ptr addrspace(1) inreg %ptr) {
 ; GFX90A:       ; %bb.0:
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX90A-NEXT:    ;;#ASMSTART
-; GFX90A-NEXT:    ; def v[4:7]
+; GFX90A-NEXT:    ; def v[0:3]
 ; GFX90A-NEXT:    ;;#ASMEND
-; GFX90A-NEXT:    v_mov_b32_e32 v8, 0
 ; GFX90A-NEXT:    ;;#ASMSTART
-; GFX90A-NEXT:    ; def v[0:3]
+; GFX90A-NEXT:    ; def v[4:7]
 ; GFX90A-NEXT:    ;;#ASMEND
-; GFX90A-NEXT:    v_mov_b32_e32 v4, v7
-; GFX90A-NEXT:    v_mov_b32_e32 v5, v2
-; GFX90A-NEXT:    v_mov_b32_e32 v6, v1
-; GFX90A-NEXT:    v_mov_b32_e32 v7, v1
-; GFX90A-NEXT:    global_store_dwordx4 v8, v[4:7], s[16:17]
+; GFX90A-NEXT:    v_mov_b32_e32 v8, 0
+; GFX90A-NEXT:    v_pk_mov_b32 v[2:3], v[6:7], v[2:3] op_sel:[1,0]
+; GFX90A-NEXT:    v_mov_b32_e32 v4, v1
+; GFX90A-NEXT:    v_mov_b32_e32 v5, v1
+; GFX90A-NEXT:    global_store_dwordx4 v8, v[2:5], s[16:17]
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
 ; GFX90A-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -3591,17 +3552,16 @@ define void @v_shuffle_v4i32_v4i32__7_2_1_1(ptr addrspace(1) inreg %ptr) {
 ; GFX940:       ; %bb.0:
 ; GFX940-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX940-NEXT:    ;;#ASMSTART
-; GFX940-NEXT:    ; def v[4:7]
+; GFX940-NEXT:    ; def v[0:3]
 ; GFX940-NEXT:    ;;#ASMEND
-; GFX940-NEXT:    v_mov_b32_e32 v8, 0
 ; GFX940-NEXT:    ;;#ASMSTART
-; GFX940-NEXT:    ; def v[0:3]
+; GFX940-NEXT:    ; def v[4:7]
 ; GFX940-NEXT:    ;;#ASMEND
-; GFX940-NEXT:    v_mov_b32_e32 v4, v7
-; GFX940-NEXT:    v_mov_b32_e32 v5, v2
-; GFX940-NEXT:    v_mov_b32_e32 v6, v1
-; GFX940-NEXT:    v_mov_b32_e32 v7, v1
-; GFX940-NEXT:    global_store_dwordx4 v8, v[4:7], s[0:1] sc0 sc1
+; GFX940-NEXT:    v_mov_b32_e32 v8, 0
+; GFX940-NEXT:    v_pk_mov_b32 v[2:3], v[6:7], v[2:3] op_sel:[1,0]
+; GFX940-NEXT:    v_mov_b32_e32 v4, v1
+; GFX940-NEXT:    v_mov_b32_e32 v5, v1
+; GFX940-NEXT:    global_store_dwordx4 v8, v[2:5], s[0:1] sc0 sc1
 ; GFX940-NEXT:    s_waitcnt vmcnt(0)
 ; GFX940-NEXT:    s_setpc_b64 s[30:31]
   %vec0 = call <4 x i32> asm "; def $0", "=v"()
@@ -3697,12 +3657,11 @@ define void @v_shuffle_v4i32_v4i32__7_4_1_1(ptr addrspace(1) inreg %ptr) {
 ; GFX90A-NEXT:    ;;#ASMSTART
 ; GFX90A-NEXT:    ; def v[2:5]
 ; GFX90A-NEXT:    ;;#ASMEND
-; GFX90A-NEXT:    v_mov_b32_e32 v8, 0
-; GFX90A-NEXT:    v_mov_b32_e32 v4, v5
-; GFX90A-NEXT:    v_mov_b32_e32 v5, v2
-; GFX90A-NEXT:    v_mov_b32_e32 v6, v1
-; GFX90A-NEXT:    v_mov_b32_e32 v7, v1
-; GFX90A-NEXT:    global_store_dwordx4 v8, v[4:7], s[16:17]
+; GFX90A-NEXT:    v_mov_b32_e32 v6, 0
+; GFX90A-NEXT:    v_pk_mov_b32 v[2:3], v[4:5], v[2:3] op_sel:[1,0]
+; GFX90A-NEXT:    v_mov_b32_e32 v4, v1
+; GFX90A-NEXT:    v_mov_b32_e32 v5, v1
+; GFX90A-NEXT:    global_store_dwordx4 v6, v[2:5], s[16:17]
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
 ; GFX90A-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -3712,15 +3671,15 @@ define void @v_shuffle_v4i32_v4i32__7_4_1_1(ptr addrspace(1) inreg %ptr) {
 ; GFX940-NEXT:    ;;#ASMSTART
 ; GFX940-NEXT:    ; def v[0:3]
 ; GFX940-NEXT:    ;;#ASMEND
-; GFX940-NEXT:    v_mov_b32_e32 v8, 0
+; GFX940-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX940-NEXT:    ;;#ASMSTART
 ; GFX940-NEXT:    ; def v[2:5]
 ; GFX940-NEXT:    ;;#ASMEND
-; GFX940-NEXT:    v_mov_b32_e32 v6, v1
-; GFX940-NEXT:    v_mov_b32_e32 v4, v5
-; GFX940-NEXT:    v_mov_b32_e32 v5, v2
-; GFX940-NEXT:    v_mov_b32_e32 v7, v1
-; GFX940-NEXT:    global_store_dwordx4 v8, v[4:7], s[0:1] sc0 sc1
+; GFX940-NEXT:    s_nop 0
+; GFX940-NEXT:    v_pk_mov_b32 v[2:3], v[4:5], v[2:3] op_sel:[1,0]
+; GFX940-NEXT:    v_mov_b32_e32 v4, v1
+; GFX940-NEXT:    v_mov_b32_e32 v5, v1
+; GFX940-NEXT:    global_store_dwordx4 v6, v[2:5], s[0:1] sc0 sc1
 ; GFX940-NEXT:    s_waitcnt vmcnt(0)
 ; GFX940-NEXT:    s_setpc_b64 s[30:31]
   %vec0 = call <4 x i32> asm "; def $0", "=v"()
@@ -3818,8 +3777,7 @@ define void @v_shuffle_v4i32_v4i32__7_6_1_1(ptr addrspace(1) inreg %ptr) {
 ; GFX90A-NEXT:    ; def v[2:5]
 ; GFX90A-NEXT:    ;;#ASMEND
 ; GFX90A-NEXT:    v_mov_b32_e32 v6, 0
-; GFX90A-NEXT:    v_mov_b32_e32 v2, v5
-; GFX90A-NEXT:    v_mov_b32_e32 v3, v4
+; GFX90A-NEXT:    v_pk_mov_b32 v[2:3], v[4:5], v[4:5] op_sel:[1,0]
 ; GFX90A-NEXT:    v_mov_b32_e32 v4, v1
 ; GFX90A-NEXT:    v_mov_b32_e32 v5, v1
 ; GFX90A-NEXT:    global_store_dwordx4 v6, v[2:5], s[16:17]
@@ -3837,8 +3795,7 @@ define void @v_shuffle_v4i32_v4i32__7_6_1_1(ptr addrspace(1) inreg %ptr) {
 ; GFX940-NEXT:    ; def v[2:5]
 ; GFX940-NEXT:    ;;#ASMEND
 ; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_mov_b32_e32 v2, v5
-; GFX940-NEXT:    v_mov_b32_e32 v3, v4
+; GFX940-NEXT:    v_pk_mov_b32 v[2:3], v[4:5], v[4:5] op_sel:[1,0]
 ; GFX940-NEXT:    v_mov_b32_e32 v4, v1
 ; GFX940-NEXT:    v_mov_b32_e32 v5, v1
 ; GFX940-NEXT:    global_store_dwordx4 v6, v[2:5], s[0:1] sc0 sc1
@@ -4444,8 +4401,7 @@ define void @v_shuffle_v4i32_v4i32__1_2_2_2(ptr addrspace(1) inreg %ptr) {
 ; GFX90A-NEXT:    ; def v[0:3]
 ; GFX90A-NEXT:    ;;#ASMEND
 ; GFX90A-NEXT:    v_mov_b32_e32 v4, 0
-; GFX90A-NEXT:    v_mov_b32_e32 v0, v1
-; GFX90A-NEXT:    v_mov_b32_e32 v1, v2
+; GFX90A-NEXT:    v_pk_mov_b32 v[0:1], v[0:1], v[2:3] op_sel:[1,0]
 ; GFX90A-NEXT:    v_mov_b32_e32 v3, v2
 ; GFX90A-NEXT:    global_store_dwordx4 v4, v[0:3], s[16:17]
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
@@ -4458,8 +4414,7 @@ define void @v_shuffle_v4i32_v4i32__1_2_2_2(ptr addrspace(1) inreg %ptr) {
 ; GFX940-NEXT:    ; def v[0:3]
 ; GFX940-NEXT:    ;;#ASMEND
 ; GFX940-NEXT:    v_mov_b32_e32 v4, 0
-; GFX940-NEXT:    v_mov_b32_e32 v0, v1
-; GFX940-NEXT:    v_mov_b32_e32 v1, v2
+; GFX940-NEXT:    v_pk_mov_b32 v[0:1], v[0:1], v[2:3] op_sel:[1,0]
 ; GFX940-NEXT:    v_mov_b32_e32 v3, v2
 ; GFX940-NEXT:    global_store_dwordx4 v4, v[0:3], s[0:1] sc0 sc1
 ; GFX940-NEXT:    s_waitcnt vmcnt(0)
@@ -4540,8 +4495,7 @@ define void @v_shuffle_v4i32_v4i32__3_2_2_2(ptr addrspace(1) inreg %ptr) {
 ; GFX90A-NEXT:    ; def v[0:3]
 ; GFX90A-NEXT:    ;;#ASMEND
 ; GFX90A-NEXT:    v_mov_b32_e32 v4, 0
-; GFX90A-NEXT:    v_mov_b32_e32 v0, v3
-; GFX90A-NEXT:    v_mov_b32_e32 v1, v2
+; GFX90A-NEXT:    v_pk_mov_b32 v[0:1], v[2:3], v[2:3] op_sel:[1,0]
 ; GFX90A-NEXT:    v_mov_b32_e32 v3, v2
 ; GFX90A-NEXT:    global_store_dwordx4 v4, v[0:3], s[16:17]
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
@@ -4554,8 +4508,7 @@ define void @v_shuffle_v4i32_v4i32__3_2_2_2(ptr addrspace(1) inreg %ptr) {
 ; GFX940-NEXT:    ; def v[0:3]
 ; GFX940-NEXT:    ;;#ASMEND
 ; GFX940-NEXT:    v_mov_b32_e32 v4, 0
-; GFX940-NEXT:    v_mov_b32_e32 v0, v3
-; GFX940-NEXT:    v_mov_b32_e32 v1, v2
+; GFX940-NEXT:    v_pk_mov_b32 v[0:1], v[2:3], v[2:3] op_sel:[1,0]
 ; GFX940-NEXT:    v_mov_b32_e32 v3, v2
 ; GFX940-NEXT:    global_store_dwordx4 v4, v[0:3], s[0:1] sc0 sc1
 ; GFX940-NEXT:    s_waitcnt vmcnt(0)
@@ -4639,8 +4592,7 @@ define void @v_shuffle_v4i32_v4i32__5_2_2_2(ptr addrspace(1) inreg %ptr) {
 ; GFX90A-NEXT:    ;;#ASMSTART
 ; GFX90A-NEXT:    ; def v[4:7]
 ; GFX90A-NEXT:    ;;#ASMEND
-; GFX90A-NEXT:    v_mov_b32_e32 v0, v5
-; GFX90A-NEXT:    v_mov_b32_e32 v1, v2
+; GFX90A-NEXT:    v_pk_mov_b32 v[0:1], v[4:5], v[2:3] op_sel:[1,0]
 ; GFX90A-NEXT:    v_mov_b32_e32 v3, v2
 ; GFX90A-NEXT:    global_store_dwordx4 v8, v[0:3], s[16:17]
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
@@ -4656,8 +4608,8 @@ define void @v_shuffle_v4i32_v4i32__5_2_2_2(ptr addrspace(1) inreg %ptr) {
 ; GFX940-NEXT:    ;;#ASMSTART
 ; GFX940-NEXT:    ; def v[4:7]
 ; GFX940-NEXT:    ;;#ASMEND
-; GFX940-NEXT:    v_mov_b32_e32 v1, v2
-; GFX940-NEXT:    v_mov_b32_e32 v0, v5
+; GFX940-NEXT:    s_nop 0
+; GFX940-NEXT:    v_pk_mov_b32 v[0:1], v[4:5], v[2:3] op_sel:[1,0]
 ; GFX940-NEXT:    v_mov_b32_e32 v3, v2
 ; GFX940-NEXT:    global_store_dwordx4 v8, v[0:3], s[0:1] sc0 sc1
 ; GFX940-NEXT:    s_waitcnt vmcnt(0)
@@ -4755,8 +4707,7 @@ define void @v_shuffle_v4i32_v4i32__7_2_2_2(ptr addrspace(1) inreg %ptr) {
 ; GFX90A-NEXT:    ;;#ASMSTART
 ; GFX90A-NEXT:    ; def v[4:7]
 ; GFX90A-NEXT:    ;;#ASMEND
-; GFX90A-NEXT:    v_mov_b32_e32 v0, v7
-; GFX90A-NEXT:    v_mov_b32_e32 v1, v2
+; GFX90A-NEXT:    v_pk_mov_b32 v[0:1], v[6:7], v[2:3] op_sel:[1,0]
 ; GFX90A-NEXT:    v_mov_b32_e32 v3, v2
 ; GFX90A-NEXT:    global_store_dwordx4 v8, v[0:3], s[16:17]
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
@@ -4772,8 +4723,8 @@ define void @v_shuffle_v4i32_v4i32__7_2_2_2(ptr addrspace(1) inreg %ptr) {
 ; GFX940-NEXT:    ;;#ASMSTART
 ; GFX940-NEXT:    ; def v[4:7]
 ; GFX940-NEXT:    ;;#ASMEND
-; GFX940-NEXT:    v_mov_b32_e32 v1, v2
-; GFX940-NEXT:    v_mov_b32_e32 v0, v7
+; GFX940-NEXT:    s_nop 0
+; GFX940-NEXT:    v_pk_mov_b32 v[0:1], v[6:7], v[2:3] op_sel:[1,0]
 ; GFX940-NEXT:    v_mov_b32_e32 v3, v2
 ; GFX940-NEXT:    global_store_dwordx4 v8, v[0:3], s[0:1] sc0 sc1
 ; GFX940-NEXT:    s_waitcnt vmcnt(0)
@@ -4863,17 +4814,15 @@ define void @v_shuffle_v4i32_v4i32__7_0_2_2(ptr addrspace(1) inreg %ptr) {
 ; GFX90A:       ; %bb.0:
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX90A-NEXT:    ;;#ASMSTART
-; GFX90A-NEXT:    ; def v[4:7]
+; GFX90A-NEXT:    ; def v[0:3]
 ; GFX90A-NEXT:    ;;#ASMEND
 ; GFX90A-NEXT:    v_mov_b32_e32 v8, 0
 ; GFX90A-NEXT:    ;;#ASMSTART
-; GFX90A-NEXT:    ; def v[0:3]
+; GFX90A-NEXT:    ; def v[4:7]
 ; GFX90A-NEXT:    ;;#ASMEND
-; GFX90A-NEXT:    v_mov_b32_e32 v4, v7
-; GFX90A-NEXT:    v_mov_b32_e32 v5, v0
-; GFX90A-NEXT:    v_mov_b32_e32 v6, v2
-; GFX90A-NEXT:    v_mov_b32_e32 v7, v2
-; GFX90A-NEXT:    global_store_dwordx4 v8, v[4:7], s[16:17]
+; GFX90A-NEXT:    v_pk_mov_b32 v[0:1], v[6:7], v[0:1] op_sel:[1,0]
+; GFX90A-NEXT:    v_mov_b32_e32 v3, v2
+; GFX90A-NEXT:    global_store_dwordx4 v8, v[0:3], s[16:17]
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
 ; GFX90A-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -4881,17 +4830,15 @@ define void @v_shuffle_v4i32_v4i32__7_0_2_2(ptr addrspace(1) inreg %ptr) {
 ; GFX940:       ; %bb.0:
 ; GFX940-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX940-NEXT:    ;;#ASMSTART
-; GFX940-NEXT:    ; def v[4:7]
+; GFX940-NEXT:    ; def v[0:3]
 ; GFX940-NEXT:    ;;#ASMEND
 ; GFX940-NEXT:    v_mov_b32_e32 v8, 0
 ; GFX940-NEXT:    ;;#ASMSTART
-; GFX940-NEXT:    ; def v[0:3]
+; GFX940-NEXT:    ; def v[4:7]
 ; GFX940-NEXT:    ;;#ASMEND
-; GFX940-NEXT:    v_mov_b32_e32 v4, v7
-; GFX940-NEXT:    v_mov_b32_e32 v5, v0
-; GFX940-NEXT:    v_mov_b32_e32 v6, v2
-; GFX940-NEXT:    v_mov_b32_e32 v7, v2
-; GFX940-NEXT:    global_store_dwordx4 v8, v[4:7], s[0:1] sc0 sc1
+; GFX940-NEXT:    v_mov_b32_e32 v3, v2
+; GFX940-NEXT:    v_pk_mov_b32 v[0:1], v[6:7], v[0:1] op_sel:[1,0]
+; GFX940-NEXT:    global_store_dwordx4 v8, v[0:3], s[0:1] sc0 sc1
 ; GFX940-NEXT:    s_waitcnt vmcnt(0)
 ; GFX940-NEXT:    s_setpc_b64 s[30:31]
   %vec0 = call <4 x i32> asm "; def $0", "=v"()
@@ -5042,8 +4989,7 @@ define void @v_shuffle_v4i32_v4i32__7_4_2_2(ptr addrspace(1) inreg %ptr) {
 ; GFX90A-NEXT:    ;;#ASMSTART
 ; GFX90A-NEXT:    ; def v[4:7]
 ; GFX90A-NEXT:    ;;#ASMEND
-; GFX90A-NEXT:    v_mov_b32_e32 v0, v7
-; GFX90A-NEXT:    v_mov_b32_e32 v1, v4
+; GFX90A-NEXT:    v_pk_mov_b32 v[0:1], v[6:7], v[4:5] op_sel:[1,0]
 ; GFX90A-NEXT:    v_mov_b32_e32 v3, v2
 ; GFX90A-NEXT:    global_store_dwordx4 v8, v[0:3], s[16:17]
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
@@ -5060,8 +5006,7 @@ define void @v_shuffle_v4i32_v4i32__7_4_2_2(ptr addrspace(1) inreg %ptr) {
 ; GFX940-NEXT:    ; def v[4:7]
 ; GFX940-NEXT:    ;;#ASMEND
 ; GFX940-NEXT:    v_mov_b32_e32 v3, v2
-; GFX940-NEXT:    v_mov_b32_e32 v0, v7
-; GFX940-NEXT:    v_mov_b32_e32 v1, v4
+; GFX940-NEXT:    v_pk_mov_b32 v[0:1], v[6:7], v[4:5] op_sel:[1,0]
 ; GFX940-NEXT:    global_store_dwordx4 v8, v[0:3], s[0:1] sc0 sc1
 ; GFX940-NEXT:    s_waitcnt vmcnt(0)
 ; GFX940-NEXT:    s_setpc_b64 s[30:31]
@@ -5158,8 +5103,7 @@ define void @v_shuffle_v4i32_v4i32__7_6_2_2(ptr addrspace(1) inreg %ptr) {
 ; GFX90A-NEXT:    ;;#ASMSTART
 ; GFX90A-NEXT:    ; def v[4:7]
 ; GFX90A-NEXT:    ;;#ASMEND
-; GFX90A-NEXT:    v_mov_b32_e32 v0, v7
-; GFX90A-NEXT:    v_mov_b32_e32 v1, v6
+; GFX90A-NEXT:    v_pk_mov_b32 v[0:1], v[6:7], v[6:7] op_sel:[1,0]
 ; GFX90A-NEXT:    v_mov_b32_e32 v3, v2
 ; GFX90A-NEXT:    global_store_dwordx4 v8, v[0:3], s[16:17]
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
@@ -5176,8 +5120,7 @@ define void @v_shuffle_v4i32_v4i32__7_6_2_2(ptr addrspace(1) inreg %ptr) {
 ; GFX940-NEXT:    ; def v[4:7]
 ; GFX940-NEXT:    ;;#ASMEND
 ; GFX940-NEXT:    v_mov_b32_e32 v3, v2
-; GFX940-NEXT:    v_mov_b32_e32 v0, v7
-; GFX940-NEXT:    v_mov_b32_e32 v1, v6
+; GFX940-NEXT:    v_pk_mov_b32 v[0:1], v[6:7], v[6:7] op_sel:[1,0]
 ; GFX940-NEXT:    global_store_dwordx4 v8, v[0:3], s[0:1] sc0 sc1
 ; GFX940-NEXT:    s_waitcnt vmcnt(0)
 ; GFX940-NEXT:    s_setpc_b64 s[30:31]
@@ -5388,17 +5331,16 @@ define void @v_shuffle_v4i32_v4i32__7_7_1_2(ptr addrspace(1) inreg %ptr) {
 ; GFX90A:       ; %bb.0:
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX90A-NEXT:    ;;#ASMSTART
-; GFX90A-NEXT:    ; def v[4:7]
+; GFX90A-NEXT:    ; def v[0:3]
 ; GFX90A-NEXT:    ;;#ASMEND
 ; GFX90A-NEXT:    v_mov_b32_e32 v8, 0
 ; GFX90A-NEXT:    ;;#ASMSTART
-; GFX90A-NEXT:    ; def v[0:3]
+; GFX90A-NEXT:    ; def v[4:7]
 ; GFX90A-NEXT:    ;;#ASMEND
-; GFX90A-NEXT:    v_mov_b32_e32 v4, v7
-; GFX90A-NEXT:    v_mov_b32_e32 v5, v7
-; GFX90A-NEXT:    v_mov_b32_e32 v6, v1
-; GFX90A-NEXT:    v_mov_b32_e32 v7, v2
-; GFX90A-NEXT:    global_store_dwordx4 v8, v[4:7], s[16:17]
+; GFX90A-NEXT:    v_pk_mov_b32 v[2:3], v[0:1], v[2:3] op_sel:[1,0]
+; GFX90A-NEXT:    v_mov_b32_e32 v0, v7
+; GFX90A-NEXT:    v_mov_b32_e32 v1, v7
+; GFX90A-NEXT:    global_store_dwordx4 v8, v[0:3], s[16:17]
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
 ; GFX90A-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -5406,17 +5348,16 @@ define void @v_shuffle_v4i32_v4i32__7_7_1_2(ptr addrspace(1) inreg %ptr) {
 ; GFX940:       ; %bb.0:
 ; GFX940-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX940-NEXT:    ;;#ASMSTART
-; GFX940-NEXT:    ; def v[4:7]
+; GFX940-NEXT:    ; def v[0:3]
 ; GFX940-NEXT:    ;;#ASMEND
 ; GFX940-NEXT:    v_mov_b32_e32 v8, 0
 ; GFX940-NEXT:    ;;#ASMSTART
-; GFX940-NEXT:    ; def v[0:3]
+; GFX940-NEXT:    ; def v[4:7]
 ; GFX940-NEXT:    ;;#ASMEND
-; GFX940-NEXT:    v_mov_b32_e32 v4, v7
-; GFX940-NEXT:    v_mov_b32_e32 v5, v7
-; GFX940-NEXT:    v_mov_b32_e32 v6, v1
-; GFX940-NEXT:    v_mov_b32_e32 v7, v2
-; GFX940-NEXT:    global_store_dwordx4 v8, v[4:7], s[0:1] sc0 sc1
+; GFX940-NEXT:    v_pk_mov_b32 v[2:3], v[0:1], v[2:3] op_sel:[1,0]
+; GFX940-NEXT:    v_mov_b32_e32 v0, v7
+; GFX940-NEXT:    v_mov_b32_e32 v1, v7
+; GFX940-NEXT:    global_store_dwordx4 v8, v[0:3], s[0:1] sc0 sc1
 ; GFX940-NEXT:    s_waitcnt vmcnt(0)
 ; GFX940-NEXT:    s_setpc_b64 s[30:31]
   %vec0 = call <4 x i32> asm "; def $0", "=v"()
@@ -5449,17 +5390,16 @@ define void @v_shuffle_v4i32_v4i32__7_7_3_2(ptr addrspace(1) inreg %ptr) {
 ; GFX90A:       ; %bb.0:
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX90A-NEXT:    ;;#ASMSTART
-; GFX90A-NEXT:    ; def v[4:7]
+; GFX90A-NEXT:    ; def v[0:3]
 ; GFX90A-NEXT:    ;;#ASMEND
 ; GFX90A-NEXT:    v_mov_b32_e32 v8, 0
 ; GFX90A-NEXT:    ;;#ASMSTART
-; GFX90A-NEXT:    ; def v[0:3]
+; GFX90A-NEXT:    ; def v[4:7]
 ; GFX90A-NEXT:    ;;#ASMEND
-; GFX90A-NEXT:    v_mov_b32_e32 v4, v7
-; GFX90A-NEXT:    v_mov_b32_e32 v5, v7
-; GFX90A-NEXT:    v_mov_b32_e32 v6, v3
-; GFX90A-NEXT:    v_mov_b32_e32 v7, v2
-; GFX90A-NEXT:    global_store_dwordx4 v8, v[4:7], s[16:17]
+; GFX90A-NEXT:    v_pk_mov_b32 v[2:3], v[2:3], v[2:3] op_sel:[1,0]
+; GFX90A-NEXT:    v_mov_b32_e32 v0, v7
+; GFX90A-NEXT:    v_mov_b32_e32 v1, v7
+; GFX90A-NEXT:    global_store_dwordx4 v8, v[0:3], s[16:17]
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
 ; GFX90A-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -5467,17 +5407,16 @@ define void @v_shuffle_v4i32_v4i32__7_7_3_2(ptr addrspace(1) inreg %ptr) {
 ; GFX940:       ; %bb.0:
 ; GFX940-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX940-NEXT:    ;;#ASMSTART
-; GFX940-NEXT:    ; def v[4:7]
+; GFX940-NEXT:    ; def v[0:3]
 ; GFX940-NEXT:    ;;#ASMEND
 ; GFX940-NEXT:    v_mov_b32_e32 v8, 0
 ; GFX940-NEXT:    ;;#ASMSTART
-; GFX940-NEXT:    ; def v[0:3]
+; GFX940-NEXT:    ; def v[4:7]
 ; GFX940-NEXT:    ;;#ASMEND
-; GFX940-NEXT:    v_mov_b32_e32 v4, v7
-; GFX940-NEXT:    v_mov_b32_e32 v5, v7
-; GFX940-NEXT:    v_mov_b32_e32 v6, v3
-; GFX940-NEXT:    v_mov_b32_e32 v7, v2
-; GFX940-NEXT:    global_store_dwordx4 v8, v[4:7], s[0:1] sc0 sc1
+; GFX940-NEXT:    v_pk_mov_b32 v[2:3], v[2:3], v[2:3] op_sel:[1,0]
+; GFX940-NEXT:    v_mov_b32_e32 v0, v7
+; GFX940-NEXT:    v_mov_b32_e32 v1, v7
+; GFX940-NEXT:    global_store_dwordx4 v8, v[0:3], s[0:1] sc0 sc1
 ; GFX940-NEXT:    s_waitcnt vmcnt(0)
 ; GFX940-NEXT:    s_setpc_b64 s[30:31]
   %vec0 = call <4 x i32> asm "; def $0", "=v"()
@@ -5568,16 +5507,16 @@ define void @v_shuffle_v4i32_v4i32__7_7_5_2(ptr addrspace(1) inreg %ptr) {
 ; GFX90A:       ; %bb.0:
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX90A-NEXT:    ;;#ASMSTART
-; GFX90A-NEXT:    ; def v[4:7]
+; GFX90A-NEXT:    ; def v[0:3]
 ; GFX90A-NEXT:    ;;#ASMEND
-; GFX90A-NEXT:    v_mov_b32_e32 v10, 0
+; GFX90A-NEXT:    v_mov_b32_e32 v8, 0
 ; GFX90A-NEXT:    ;;#ASMSTART
-; GFX90A-NEXT:    ; def v[0:3]
+; GFX90A-NEXT:    ; def v[4:7]
 ; GFX90A-NEXT:    ;;#ASMEND
-; GFX90A-NEXT:    v_mov_b32_e32 v6, v7
-; GFX90A-NEXT:    v_mov_b32_e32 v8, v5
-; GFX90A-NEXT:    v_mov_b32_e32 v9, v2
-; GFX90A-NEXT:    global_store_dwordx4 v10, v[6:9], s[16:17]
+; GFX90A-NEXT:    v_pk_mov_b32 v[2:3], v[4:5], v[2:3] op_sel:[1,0]
+; GFX90A-NEXT:    v_mov_b32_e32 v0, v7
+; GFX90A-NEXT:    v_mov_b32_e32 v1, v7
+; GFX90A-NEXT:    global_store_dwordx4 v8, v[0:3], s[16:17]
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
 ; GFX90A-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -5585,16 +5524,17 @@ define void @v_shuffle_v4i32_v4i32__7_7_5_2(ptr addrspace(1) inreg %ptr) {
 ; GFX940:       ; %bb.0:
 ; GFX940-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX940-NEXT:    ;;#ASMSTART
-; GFX940-NEXT:    ; def v[4:7]
+; GFX940-NEXT:    ; def v[0:3]
 ; GFX940-NEXT:    ;;#ASMEND
-; GFX940-NEXT:    v_mov_b32_e32 v10, 0
+; GFX940-NEXT:    v_mov_b32_e32 v8, 0
 ; GFX940-NEXT:    ;;#ASMSTART
-; GFX940-NEXT:    ; def v[0:3]
+; GFX940-NEXT:    ; def v[4:7]
 ; GFX940-NEXT:    ;;#ASMEND
-; GFX940-NEXT:    v_mov_b32_e32 v6, v7
-; GFX940-NEXT:    v_mov_b32_e32 v8, v5
-; GFX940-NEXT:    v_mov_b32_e32 v9, v2
-; GFX940-NEXT:    global_store_dwordx4 v10, v[6:9], s[0:1] sc0 sc1
+; GFX940-NEXT:    s_nop 0
+; GFX940-NEXT:    v_pk_mov_b32 v[2:3], v[4:5], v[2:3] op_sel:[1,0]
+; GFX940-NEXT:    v_mov_b32_e32 v0, v7
+; GFX940-NEXT:    v_mov_b32_e32 v1, v7
+; GFX940-NEXT:    global_store_dwordx4 v8, v[0:3], s[0:1] sc0 sc1
 ; GFX940-NEXT:    s_waitcnt vmcnt(0)
 ; GFX940-NEXT:    s_setpc_b64 s[30:31]
   %vec0 = call <4 x i32> asm "; def $0", "=v"()
@@ -6193,17 +6133,15 @@ define void @v_shuffle_v4i32_v4i32__7_0_3_3(ptr addrspace(1) inreg %ptr) {
 ; GFX90A:       ; %bb.0:
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX90A-NEXT:    ;;#ASMSTART
-; GFX90A-NEXT:    ; def v[4:7]
+; GFX90A-NEXT:    ; def v[0:3]
 ; GFX90A-NEXT:    ;;#ASMEND
 ; GFX90A-NEXT:    v_mov_b32_e32 v8, 0
 ; GFX90A-NEXT:    ;;#ASMSTART
-; GFX90A-NEXT:    ; def v[0:3]
+; GFX90A-NEXT:    ; def v[4:7]
 ; GFX90A-NEXT:    ;;#ASMEND
-; GFX90A-NEXT:    v_mov_b32_e32 v4, v7
-; GFX90A-NEXT:    v_mov_b32_e32 v5, v0
-; GFX90A-NEXT:    v_mov_b32_e32 v6, v3
-; GFX90A-NEXT:    v_mov_b32_e32 v7, v3
-; GFX90A-NEXT:    global_store_dwordx4 v8, v[4:7], s[16:17]
+; GFX90A-NEXT:    v_pk_mov_b32 v[0:1], v[6:7], v[0:1] op_sel:[1,0]
+; GFX90A-NEXT:    v_mov_b32_e32 v2, v3
+; GFX90A-NEXT:    global_store_dwordx4 v8, v[0:3], s[16:17]
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
 ; GFX90A-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -6211,17 +6149,15 @@ define void @v_shuffle_v4i32_v4i32__7_0_3_3(ptr addrspace(1) inreg %ptr) {
 ; GFX940:       ; %bb.0:
 ; GFX940-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX940-NEXT:    ;;#ASMSTART
-; GFX940-NEXT:    ; def v[4:7]
+; GFX940-NEXT:    ; def v[0:3]
 ; GFX940-NEXT:    ;;#ASMEND
 ; GFX940-NEXT:    v_mov_b32_e32 v8, 0
 ; GFX940-NEXT:    ;;#ASMSTART
-; GFX940-NEXT:    ; def v[0:3]
+; GFX940-NEXT:    ; def v[4:7]
 ; GFX940-NEXT:    ;;#ASMEND
-; GFX940-NEXT:    v_mov_b32_e32 v4, v7
-; GFX940-NEXT:    v_mov_b32_e32 v5, v0
-; GFX940-NEXT:    v_mov_b32_e32 v6, v3
-; GFX940-NEXT:    v_mov_b32_e32 v7, v3
-; GFX940-NEXT:    global_store_dwordx4 v8, v[4:7], s[0:1] sc0 sc1
+; GFX940-NEXT:    v_mov_b32_e32 v2, v3
+; GFX940-NEXT:    v_pk_mov_b32 v[0:1], v[6:7], v[0:1] op_sel:[1,0]
+; GFX940-NEXT:    global_store_dwordx4 v8, v[0:3], s[0:1] sc0 sc1
 ; GFX940-NEXT:    s_waitcnt vmcnt(0)
 ; GFX940-NEXT:    s_setpc_b64 s[30:31]
   %vec0 = call <4 x i32> asm "; def $0", "=v"()
@@ -6314,8 +6250,7 @@ define void @v_shuffle_v4i32_v4i32__7_2_3_3(ptr addrspace(1) inreg %ptr) {
 ; GFX90A-NEXT:    ;;#ASMSTART
 ; GFX90A-NEXT:    ; def v[4:7]
 ; GFX90A-NEXT:    ;;#ASMEND
-; GFX90A-NEXT:    v_mov_b32_e32 v0, v7
-; GFX90A-NEXT:    v_mov_b32_e32 v1, v2
+; GFX90A-NEXT:    v_pk_mov_b32 v[0:1], v[6:7], v[2:3] op_sel:[1,0]
 ; GFX90A-NEXT:    v_mov_b32_e32 v2, v3
 ; GFX90A-NEXT:    global_store_dwordx4 v8, v[0:3], s[16:17]
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
@@ -6331,8 +6266,8 @@ define void @v_shuffle_v4i32_v4i32__7_2_3_3(ptr addrspace(1) inreg %ptr) {
 ; GFX940-NEXT:    ;;#ASMSTART
 ; GFX940-NEXT:    ; def v[4:7]
 ; GFX940-NEXT:    ;;#ASMEND
-; GFX940-NEXT:    v_mov_b32_e32 v1, v2
-; GFX940-NEXT:    v_mov_b32_e32 v0, v7
+; GFX940-NEXT:    s_nop 0
+; GFX940-NEXT:    v_pk_mov_b32 v[0:1], v[6:7], v[2:3] op_sel:[1,0]
 ; GFX940-NEXT:    v_mov_b32_e32 v2, v3
 ; GFX940-NEXT:    global_store_dwordx4 v8, v[0:3], s[0:1] sc0 sc1
 ; GFX940-NEXT:    s_waitcnt vmcnt(0)
@@ -6372,8 +6307,7 @@ define void @v_shuffle_v4i32_v4i32__7_4_3_3(ptr addrspace(1) inreg %ptr) {
 ; GFX90A-NEXT:    ;;#ASMSTART
 ; GFX90A-NEXT:    ; def v[4:7]
 ; GFX90A-NEXT:    ;;#ASMEND
-; GFX90A-NEXT:    v_mov_b32_e32 v0, v7
-; GFX90A-NEXT:    v_mov_b32_e32 v1, v4
+; GFX90A-NEXT:    v_pk_mov_b32 v[0:1], v[6:7], v[4:5] op_sel:[1,0]
 ; GFX90A-NEXT:    v_mov_b32_e32 v2, v3
 ; GFX90A-NEXT:    global_store_dwordx4 v8, v[0:3], s[16:17]
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
@@ -6390,8 +6324,7 @@ define void @v_shuffle_v4i32_v4i32__7_4_3_3(ptr addrspace(1) inreg %ptr) {
 ; GFX940-NEXT:    ; def v[4:7]
 ; GFX940-NEXT:    ;;#ASMEND
 ; GFX940-NEXT:    v_mov_b32_e32 v2, v3
-; GFX940-NEXT:    v_mov_b32_e32 v0, v7
-; GFX940-NEXT:    v_mov_b32_e32 v1, v4
+; GFX940-NEXT:    v_pk_mov_b32 v[0:1], v[6:7], v[4:5] op_sel:[1,0]
 ; GFX940-NEXT:    global_store_dwordx4 v8, v[0:3], s[0:1] sc0 sc1
 ; GFX940-NEXT:    s_waitcnt vmcnt(0)
 ; GFX940-NEXT:    s_setpc_b64 s[30:31]
@@ -6488,8 +6421,7 @@ define void @v_shuffle_v4i32_v4i32__7_6_3_3(ptr addrspace(1) inreg %ptr) {
 ; GFX90A-NEXT:    ;;#ASMSTART
 ; GFX90A-NEXT:    ; def v[4:7]
 ; GFX90A-NEXT:    ;;#ASMEND
-; GFX90A-NEXT:    v_mov_b32_e32 v0, v7
-; GFX90A-NEXT:    v_mov_b32_e32 v1, v6
+; GFX90A-NEXT:    v_pk_mov_b32 v[0:1], v[6:7], v[6:7] op_sel:[1,0]
 ; GFX90A-NEXT:    v_mov_b32_e32 v2, v3
 ; GFX90A-NEXT:    global_store_dwordx4 v8, v[0:3], s[16:17]
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
@@ -6506,8 +6438,7 @@ define void @v_shuffle_v4i32_v4i32__7_6_3_3(ptr addrspace(1) inreg %ptr) {
 ; GFX940-NEXT:    ; def v[4:7]
 ; GFX940-NEXT:    ;;#ASMEND
 ; GFX940-NEXT:    v_mov_b32_e32 v2, v3
-; GFX940-NEXT:    v_mov_b32_e32 v0, v7
-; GFX940-NEXT:    v_mov_b32_e32 v1, v6
+; GFX940-NEXT:    v_pk_mov_b32 v[0:1], v[6:7], v[6:7] op_sel:[1,0]
 ; GFX940-NEXT:    global_store_dwordx4 v8, v[0:3], s[0:1] sc0 sc1
 ; GFX940-NEXT:    s_waitcnt vmcnt(0)
 ; GFX940-NEXT:    s_setpc_b64 s[30:31]
@@ -7195,8 +7126,7 @@ define void @v_shuffle_v4i32_v4i32__5_4_4_4(ptr addrspace(1) inreg %ptr) {
 ; GFX90A-NEXT:    ; def v[0:3]
 ; GFX90A-NEXT:    ;;#ASMEND
 ; GFX90A-NEXT:    v_mov_b32_e32 v6, 0
-; GFX90A-NEXT:    v_mov_b32_e32 v2, v1
-; GFX90A-NEXT:    v_mov_b32_e32 v3, v0
+; GFX90A-NEXT:    v_pk_mov_b32 v[2:3], v[0:1], v[0:1] op_sel:[1,0]
 ; GFX90A-NEXT:    v_mov_b32_e32 v4, v0
 ; GFX90A-NEXT:    v_mov_b32_e32 v5, v0
 ; GFX90A-NEXT:    global_store_dwordx4 v6, v[2:5], s[16:17]
@@ -7210,8 +7140,7 @@ define void @v_shuffle_v4i32_v4i32__5_4_4_4(ptr addrspace(1) inreg %ptr) {
 ; GFX940-NEXT:    ; def v[0:3]
 ; GFX940-NEXT:    ;;#ASMEND
 ; GFX940-NEXT:    v_mov_b32_e32 v6, 0
-; GFX940-NEXT:    v_mov_b32_e32 v2, v1
-; GFX940-NEXT:    v_mov_b32_e32 v3, v0
+; GFX940-NEXT:    v_pk_mov_b32 v[2:3], v[0:1], v[0:1] op_sel:[1,0]
 ; GFX940-NEXT:    v_mov_b32_e32 v4, v0
 ; GFX940-NEXT:    v_mov_b32_e32 v5, v0
 ; GFX940-NEXT:    global_store_dwordx4 v6, v[2:5], s[0:1] sc0 sc1
@@ -7297,8 +7226,7 @@ define void @v_shuffle_v4i32_v4i32__7_4_4_4(ptr addrspace(1) inreg %ptr) {
 ; GFX90A-NEXT:    ; def v[0:3]
 ; GFX90A-NEXT:    ;;#ASMEND
 ; GFX90A-NEXT:    v_mov_b32_e32 v6, 0
-; GFX90A-NEXT:    v_mov_b32_e32 v2, v3
-; GFX90A-NEXT:    v_mov_b32_e32 v3, v0
+; GFX90A-NEXT:    v_pk_mov_b32 v[2:3], v[2:3], v[0:1] op_sel:[1,0]
 ; GFX90A-NEXT:    v_mov_b32_e32 v4, v0
 ; GFX90A-NEXT:    v_mov_b32_e32 v5, v0
 ; GFX90A-NEXT:    global_store_dwordx4 v6, v[2:5], s[16:17]
@@ -7312,8 +7240,7 @@ define void @v_shuffle_v4i32_v4i32__7_4_4_4(ptr addrspace(1) inreg %ptr) {
 ; GFX940-NEXT:    ; def v[0:3]
 ; GFX940-NEXT:    ;;#ASMEND
 ; GFX940-NEXT:    v_mov_b32_e32 v6, 0
-; GFX940-NEXT:    v_mov_b32_e32 v2, v3
-; GFX940-NEXT:    v_mov_b32_e32 v3, v0
+; GFX940-NEXT:    v_pk_mov_b32 v[2:3], v[2:3], v[0:1] op_sel:[1,0]
 ; GFX940-NEXT:    v_mov_b32_e32 v4, v0
 ; GFX940-NEXT:    v_mov_b32_e32 v5, v0
 ; GFX940-NEXT:    global_store_dwordx4 v6, v[2:5], s[0:1] sc0 sc1
@@ -7404,8 +7331,7 @@ define void @v_shuffle_v4i32_v4i32__7_0_4_4(ptr addrspace(1) inreg %ptr) {
 ; GFX90A-NEXT:    ; def v[2:5]
 ; GFX90A-NEXT:    ;;#ASMEND
 ; GFX90A-NEXT:    v_mov_b32_e32 v8, 0
-; GFX90A-NEXT:    v_mov_b32_e32 v4, v5
-; GFX90A-NEXT:    v_mov_b32_e32 v5, v0
+; GFX90A-NEXT:    v_pk_mov_b32 v[4:5], v[4:5], v[0:1] op_sel:[1,0]
 ; GFX90A-NEXT:    v_mov_b32_e32 v6, v2
 ; GFX90A-NEXT:    v_mov_b32_e32 v7, v2
 ; GFX90A-NEXT:    global_store_dwordx4 v8, v[4:7], s[16:17]
@@ -7423,8 +7349,7 @@ define void @v_shuffle_v4i32_v4i32__7_0_4_4(ptr addrspace(1) inreg %ptr) {
 ; GFX940-NEXT:    ; def v[2:5]
 ; GFX940-NEXT:    ;;#ASMEND
 ; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_mov_b32_e32 v4, v5
-; GFX940-NEXT:    v_mov_b32_e32 v5, v0
+; GFX940-NEXT:    v_pk_mov_b32 v[4:5], v[4:5], v[0:1] op_sel:[1,0]
 ; GFX940-NEXT:    v_mov_b32_e32 v6, v2
 ; GFX940-NEXT:    v_mov_b32_e32 v7, v2
 ; GFX940-NEXT:    global_store_dwordx4 v8, v[4:7], s[0:1] sc0 sc1
@@ -7523,8 +7448,7 @@ define void @v_shuffle_v4i32_v4i32__7_2_4_4(ptr addrspace(1) inreg %ptr) {
 ; GFX90A-NEXT:    ;;#ASMSTART
 ; GFX90A-NEXT:    ; def v[4:7]
 ; GFX90A-NEXT:    ;;#ASMEND
-; GFX90A-NEXT:    v_mov_b32_e32 v0, v7
-; GFX90A-NEXT:    v_mov_b32_e32 v1, v2
+; GFX90A-NEXT:    v_pk_mov_b32 v[0:1], v[6:7], v[2:3] op_sel:[1,0]
 ; GFX90A-NEXT:    v_mov_b32_e32 v2, v4
 ; GFX90A-NEXT:    v_mov_b32_e32 v3, v4
 ; GFX90A-NEXT:    global_store_dwordx4 v8, v[0:3], s[16:17]
@@ -7541,8 +7465,8 @@ define void @v_shuffle_v4i32_v4i32__7_2_4_4(ptr addrspace(1) inreg %ptr) {
 ; GFX940-NEXT:    ;;#ASMSTART
 ; GFX940-NEXT:    ; def v[4:7]
 ; GFX940-NEXT:    ;;#ASMEND
-; GFX940-NEXT:    v_mov_b32_e32 v1, v2
-; GFX940-NEXT:    v_mov_b32_e32 v0, v7
+; GFX940-NEXT:    s_nop 0
+; GFX940-NEXT:    v_pk_mov_b32 v[0:1], v[6:7], v[2:3] op_sel:[1,0]
 ; GFX940-NEXT:    v_mov_b32_e32 v2, v4
 ; GFX940-NEXT:    v_mov_b32_e32 v3, v4
 ; GFX940-NEXT:    global_store_dwordx4 v8, v[0:3], s[0:1] sc0 sc1
@@ -7686,31 +7610,28 @@ define void @v_shuffle_v4i32_v4i32__7_6_4_4(ptr addrspace(1) inreg %ptr) {
 ; GFX90A-LABEL: v_shuffle_v4i32_v4i32__7_6_4_4:
 ; GFX90A:       ; %bb.0:
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT:    v_mov_b32_e32 v8, 0
 ; GFX90A-NEXT:    ;;#ASMSTART
 ; GFX90A-NEXT:    ; def v[0:3]
 ; GFX90A-NEXT:    ;;#ASMEND
-; GFX90A-NEXT:    v_mov_b32_e32 v4, v3
-; GFX90A-NEXT:    v_mov_b32_e32 v5, v2
-; GFX90A-NEXT:    v_mov_b32_e32 v6, v0
-; GFX90A-NEXT:    v_mov_b32_e32 v7, v0
-; GFX90A-NEXT:    global_store_dwordx4 v8, v[4:7], s[16:17]
+; GFX90A-NEXT:    v_mov_b32_e32 v6, 0
+; GFX90A-NEXT:    v_pk_mov_b32 v[2:3], v[2:3], v[2:3] op_sel:[1,0]
+; GFX90A-NEXT:    v_mov_b32_e32 v4, v0
+; GFX90A-NEXT:    v_mov_b32_e32 v5, v0
+; GFX90A-NEXT:    global_store_dwordx4 v6, v[2:5], s[16:17]
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
 ; GFX90A-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: v_shuffle_v4i32_v4i32__7_6_4_4:
 ; GFX940:       ; %bb.0:
 ; GFX940-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX940-NEXT:    v_mov_b32_e32 v8, 0
 ; GFX940-NEXT:    ;;#ASMSTART
 ; GFX940-NEXT:    ; def v[0:3]
 ; GFX940-NEXT:    ;;#ASMEND
-; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_mov_b32_e32 v4, v3
-; GFX940-NEXT:    v_mov_b32_e32 v5, v2
-; GFX940-NEXT:    v_mov_b32_e32 v6, v0
-; GFX940-NEXT:    v_mov_b32_e32 v7, v0
-; GFX940-NEXT:    global_store_dwordx4 v8, v[4:7], s[0:1] sc0 sc1
+; GFX940-NEXT:    v_mov_b32_e32 v6, 0
+; GFX940-NEXT:    v_pk_mov_b32 v[2:3], v[2:3], v[2:3] op_sel:[1,0]
+; GFX940-NEXT:    v_mov_b32_e32 v4, v0
+; GFX940-NEXT:    v_mov_b32_e32 v5, v0
+; GFX940-NEXT:    global_store_dwordx4 v6, v[2:5], s[0:1] sc0 sc1
 ; GFX940-NEXT:    s_waitcnt vmcnt(0)
 ; GFX940-NEXT:    s_setpc_b64 s[30:31]
   %vec0 = call <4 x i32> asm "; def $0", "=v"()
@@ -7904,11 +7825,11 @@ define void @v_shuffle_v4i32_v4i32__7_7_1_4(ptr addrspace(1) inreg %ptr) {
 ; GFX90A-NEXT:    ;;#ASMSTART
 ; GFX90A-NEXT:    ; def v[2:5]
 ; GFX90A-NEXT:    ;;#ASMEND
-; GFX90A-NEXT:    v_mov_b32_e32 v8, 0
-; GFX90A-NEXT:    v_mov_b32_e32 v4, v5
-; GFX90A-NEXT:    v_mov_b32_e32 v6, v1
-; GFX90A-NEXT:    v_mov_b32_e32 v7, v2
-; GFX90A-NEXT:    global_store_dwordx4 v8, v[4:7], s[16:17]
+; GFX90A-NEXT:    v_mov_b32_e32 v6, 0
+; GFX90A-NEXT:    v_pk_mov_b32 v[2:3], v[0:1], v[2:3] op_sel:[1,0]
+; GFX90A-NEXT:    v_mov_b32_e32 v0, v5
+; GFX90A-NEXT:    v_mov_b32_e32 v1, v5
+; GFX90A-NEXT:    global_store_dwordx4 v6, v[0:3], s[16:17]
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
 ; GFX90A-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -7918,14 +7839,15 @@ define void @v_shuffle_v4i32_v4i32__7_7_1_4(ptr addrspace(1) inreg %ptr) {
 ; GFX940-NEXT:    ;;#ASMSTART
 ; GFX940-NEXT:    ; def v[0:3]
 ; GFX940-NEXT:    ;;#ASMEND
-; GFX940-NEXT:    v_mov_b32_e32 v8, 0
+; GFX940-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX940-NEXT:    ;;#ASMSTART
 ; GFX940-NEXT:    ; def v[2:5]
 ; GFX940-NEXT:    ;;#ASMEND
-; GFX940-NEXT:    v_mov_b32_e32 v6, v1
-; GFX940-NEXT:    v_mov_b32_e32 v4, v5
-; GFX940-NEXT:    v_mov_b32_e32 v7, v2
-; GFX940-NEXT:    global_store_dwordx4 v8, v[4:7], s[0:1] sc0 sc1
+; GFX940-NEXT:    s_nop 0
+; GFX940-NEXT:    v_pk_mov_b32 v[2:3], v[0:1], v[2:3] op_sel:[1,0]
+; GFX940-NEXT:    v_mov_b32_e32 v0, v5
+; GFX940-NEXT:    v_mov_b32_e32 v1, v5
+; GFX940-NEXT:    global_store_dwordx4 v6, v[0:3], s[0:1] sc0 sc1
 ; GFX940-NEXT:    s_waitcnt vmcnt(0)
 ; GFX940-NEXT:    s_setpc_b64 s[30:31]
   %vec0 = call <4 x i32> asm "; def $0", "=v"()
@@ -8022,10 +7944,9 @@ define void @v_shuffle_v4i32_v4i32__7_7_3_4(ptr addrspace(1) inreg %ptr) {
 ; GFX90A-NEXT:    ;;#ASMSTART
 ; GFX90A-NEXT:    ; def v[4:7]
 ; GFX90A-NEXT:    ;;#ASMEND
+; GFX90A-NEXT:    v_pk_mov_b32 v[2:3], v[2:3], v[4:5] op_sel:[1,0]
 ; GFX90A-NEXT:    v_mov_b32_e32 v0, v7
 ; GFX90A-NEXT:    v_mov_b32_e32 v1, v7
-; GFX90A-NEXT:    v_mov_b32_e32 v2, v3
-; GFX90A-NEXT:    v_mov_b32_e32 v3, v4
 ; GFX90A-NEXT:    global_store_dwordx4 v8, v[0:3], s[16:17]
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
 ; GFX90A-NEXT:    s_setpc_b64 s[30:31]
@@ -8040,10 +7961,10 @@ define void @v_shuffle_v4i32_v4i32__7_7_3_4(ptr addrspace(1) inreg %ptr) {
 ; GFX940-NEXT:    ;;#ASMSTART
 ; GFX940-NEXT:    ; def v[4:7]
 ; GFX940-NEXT:    ;;#ASMEND
-; GFX940-NEXT:    v_mov_b32_e32 v2, v3
+; GFX940-NEXT:    s_nop 0
+; GFX940-NEXT:    v_pk_mov_b32 v[2:3], v[2:3], v[4:5] op_sel:[1,0]
 ; GFX940-NEXT:    v_mov_b32_e32 v0, v7
 ; GFX940-NEXT:    v_mov_b32_e32 v1, v7
-; GFX940-NEXT:    v_mov_b32_e32 v3, v4
 ; GFX940-NEXT:    global_store_dwordx4 v8, v[0:3], s[0:1] sc0 sc1
 ; GFX940-NEXT:    s_waitcnt vmcnt(0)
 ; GFX940-NEXT:    s_setpc_b64 s[30:31]
@@ -8076,9 +7997,8 @@ define void @v_shuffle_v4i32_v4i32__7_7_5_4(ptr addrspace(1) inreg %ptr) {
 ; GFX90A-NEXT:    ; def v[0:3]
 ; GFX90A-NEXT:    ;;#ASMEND
 ; GFX90A-NEXT:    v_mov_b32_e32 v6, 0
+; GFX90A-NEXT:    v_pk_mov_b32 v[4:5], v[0:1], v[0:1] op_sel:[1,0]
 ; GFX90A-NEXT:    v_mov_b32_e32 v2, v3
-; GFX90A-NEXT:    v_mov_b32_e32 v4, v1
-; GFX90A-NEXT:    v_mov_b32_e32 v5, v0
 ; GFX90A-NEXT:    global_store_dwordx4 v6, v[2:5], s[16:17]
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
 ; GFX90A-NEXT:    s_setpc_b64 s[30:31]
@@ -8090,9 +8010,8 @@ define void @v_shuffle_v4i32_v4i32__7_7_5_4(ptr addrspace(1) inreg %ptr) {
 ; GFX940-NEXT:    ; def v[0:3]
 ; GFX940-NEXT:    ;;#ASMEND
 ; GFX940-NEXT:    v_mov_b32_e32 v6, 0
+; GFX940-NEXT:    v_pk_mov_b32 v[4:5], v[0:1], v[0:1] op_sel:[1,0]
 ; GFX940-NEXT:    v_mov_b32_e32 v2, v3
-; GFX940-NEXT:    v_mov_b32_e32 v4, v1
-; GFX940-NEXT:    v_mov_b32_e32 v5, v0
 ; GFX940-NEXT:    global_store_dwordx4 v6, v[2:5], s[0:1] sc0 sc1
 ; GFX940-NEXT:    s_waitcnt vmcnt(0)
 ; GFX940-NEXT:    s_setpc_b64 s[30:31]
@@ -8703,12 +8622,10 @@ define void @v_shuffle_v4i32_v4i32__7_0_5_5(ptr addrspace(1) inreg %ptr) {
 ; GFX90A-NEXT:    ;;#ASMSTART
 ; GFX90A-NEXT:    ; def v[2:5]
 ; GFX90A-NEXT:    ;;#ASMEND
-; GFX90A-NEXT:    v_mov_b32_e32 v8, 0
-; GFX90A-NEXT:    v_mov_b32_e32 v4, v5
-; GFX90A-NEXT:    v_mov_b32_e32 v5, v0
-; GFX90A-NEXT:    v_mov_b32_e32 v6, v3
-; GFX90A-NEXT:    v_mov_b32_e32 v7, v3
-; GFX90A-NEXT:    global_store_dwordx4 v8, v[4:7], s[16:17]
+; GFX90A-NEXT:    v_mov_b32_e32 v6, 0
+; GFX90A-NEXT:    v_pk_mov_b32 v[0:1], v[4:5], v[0:1] op_sel:[1,0]
+; GFX90A-NEXT:    v_mov_b32_e32 v2, v3
+; GFX90A-NEXT:    global_store_dwordx4 v6, v[0:3], s[16:17]
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
 ; GFX90A-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -8718,16 +8635,14 @@ define void @v_shuffle_v4i32_v4i32__7_0_5_5(ptr addrspace(1) inreg %ptr) {
 ; GFX940-NEXT:    ;;#ASMSTART
 ; GFX940-NEXT:    ; def v[0:3]
 ; GFX940-NEXT:    ;;#ASMEND
-; GFX940-NEXT:    v_mov_b32_e32 v8, 0
+; GFX940-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX940-NEXT:    ;;#ASMSTART
 ; GFX940-NEXT:    ; def v[2:5]
 ; GFX940-NEXT:    ;;#ASMEND
 ; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_mov_b32_e32 v4, v5
-; GFX940-NEXT:    v_mov_b32_e32 v5, v0
-; GFX940-NEXT:    v_mov_b32_e32 v6, v3
-; GFX940-NEXT:    v_mov_b32_e32 v7, v3
-; GFX940-NEXT:    global_store_dwordx4 v8, v[4:7], s[0:1] sc0 sc1
+; GFX940-NEXT:    v_pk_mov_b32 v[0:1], v[4:5], v[0:1] op_sel:[1,0]
+; GFX940-NEXT:    v_mov_b32_e32 v2, v3
+; GFX940-NEXT:    global_store_dwordx4 v6, v[0:3], s[0:1] sc0 sc1
 ; GFX940-NEXT:    s_waitcnt vmcnt(0)
 ; GFX940-NEXT:    s_setpc_b64 s[30:31]
   %vec0 = call <4 x i32> asm "; def $0", "=v"()
@@ -8822,8 +8737,7 @@ define void @v_shuffle_v4i32_v4i32__7_2_5_5(ptr addrspace(1) inreg %ptr) {
 ; GFX90A-NEXT:    ;;#ASMSTART
 ; GFX90A-NEXT:    ; def v[4:7]
 ; GFX90A-NEXT:    ;;#ASMEND
-; GFX90A-NEXT:    v_mov_b32_e32 v0, v7
-; GFX90A-NEXT:    v_mov_b32_e32 v1, v2
+; GFX90A-NEXT:    v_pk_mov_b32 v[0:1], v[6:7], v[2:3] op_sel:[1,0]
 ; GFX90A-NEXT:    v_mov_b32_e32 v2, v5
 ; GFX90A-NEXT:    v_mov_b32_e32 v3, v5
 ; GFX90A-NEXT:    global_store_dwordx4 v8, v[0:3], s[16:17]
@@ -8840,8 +8754,8 @@ define void @v_shuffle_v4i32_v4i32__7_2_5_5(ptr addrspace(1) inreg %ptr) {
 ; GFX940-NEXT:    ;;#ASMSTART
 ; GFX940-NEXT:    ; def v[4:7]
 ; GFX940-NEXT:    ;;#ASMEND
-; GFX940-NEXT:    v_mov_b32_e32 v1, v2
-; GFX940-NEXT:    v_mov_b32_e32 v0, v7
+; GFX940-NEXT:    s_nop 0
+; GFX940-NEXT:    v_pk_mov_b32 v[0:1], v[6:7], v[2:3] op_sel:[1,0]
 ; GFX940-NEXT:    v_mov_b32_e32 v2, v5
 ; GFX940-NEXT:    v_mov_b32_e32 v3, v5
 ; GFX940-NEXT:    global_store_dwordx4 v8, v[0:3], s[0:1] sc0 sc1
@@ -8938,8 +8852,7 @@ define void @v_shuffle_v4i32_v4i32__7_4_5_5(ptr addrspace(1) inreg %ptr) {
 ; GFX90A-NEXT:    ; def v[0:3]
 ; GFX90A-NEXT:    ;;#ASMEND
 ; GFX90A-NEXT:    v_mov_b32_e32 v6, 0
-; GFX90A-NEXT:    v_mov_b32_e32 v2, v3
-; GFX90A-NEXT:    v_mov_b32_e32 v3, v0
+; GFX90A-NEXT:    v_pk_mov_b32 v[2:3], v[2:3], v[0:1] op_sel:[1,0]
 ; GFX90A-NEXT:    v_mov_b32_e32 v4, v1
 ; GFX90A-NEXT:    v_mov_b32_e32 v5, v1
 ; GFX90A-NEXT:    global_store_dwordx4 v6, v[2:5], s[16:17]
@@ -8953,8 +8866,7 @@ define void @v_shuffle_v4i32_v4i32__7_4_5_5(ptr addrspace(1) inreg %ptr) {
 ; GFX940-NEXT:    ; def v[0:3]
 ; GFX940-NEXT:    ;;#ASMEND
 ; GFX940-NEXT:    v_mov_b32_e32 v6, 0
-; GFX940-NEXT:    v_mov_b32_e32 v2, v3
-; GFX940-NEXT:    v_mov_b32_e32 v3, v0
+; GFX940-NEXT:    v_pk_mov_b32 v[2:3], v[2:3], v[0:1] op_sel:[1,0]
 ; GFX940-NEXT:    v_mov_b32_e32 v4, v1
 ; GFX940-NEXT:    v_mov_b32_e32 v5, v1
 ; GFX940-NEXT:    global_store_dwordx4 v6, v[2:5], s[0:1] sc0 sc1
@@ -8985,31 +8897,28 @@ define void @v_shuffle_v4i32_v4i32__7_6_5_5(ptr addrspace(1) inreg %ptr) {
 ; GFX90A-LABEL: v_shuffle_v4i32_v4i32__7_6_5_5:
 ; GFX90A:       ; %bb.0:
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT:    v_mov_b32_e32 v8, 0
 ; GFX90A-NEXT:    ;;#ASMSTART
 ; GFX90A-NEXT:    ; def v[0:3]
 ; GFX90A-NEXT:    ;;#ASMEND
-; GFX90A-NEXT:    v_mov_b32_e32 v4, v3
-; GFX90A-NEXT:    v_mov_b32_e32 v5, v2
-; GFX90A-NEXT:    v_mov_b32_e32 v6, v1
-; GFX90A-NEXT:    v_mov_b32_e32 v7, v1
-; GFX90A-NEXT:    global_store_dwordx4 v8, v[4:7], s[16:17]
+; GFX90A-NEXT:    v_mov_b32_e32 v6, 0
+; GFX90A-NEXT:    v_pk_mov_b32 v[2:3], v[2:3], v[2:3] op_sel:[1,0]
+; GFX90A-NEXT:    v_mov_b32_e32 v4, v1
+; GFX90A-NEXT:    v_mov_b32_e32 v5, v1
+; GFX90A-NEXT:    global_store_dwordx4 v6, v[2:5], s[16:17]
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
 ; GFX90A-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: v_shuffle_v4i32_v4i32__7_6_5_5:
 ; GFX940:       ; %bb.0:
 ; GFX940-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX940-NEXT:    v_mov_b32_e32 v8, 0
 ; GFX940-NEXT:    ;;#ASMSTART
 ; GFX940-NEXT:    ; def v[0:3]
 ; GFX940-NEXT:    ;;#ASMEND
-; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_mov_b32_e32 v4, v3
-; GFX940-NEXT:    v_mov_b32_e32 v5, v2
-; GFX940-NEXT:    v_mov_b32_e32 v6, v1
-; GFX940-NEXT:    v_mov_b32_e32 v7, v1
-; GFX940-NEXT:    global_store_dwordx4 v8, v[4:7], s[0:1] sc0 sc1
+; GFX940-NEXT:    v_mov_b32_e32 v6, 0
+; GFX940-NEXT:    v_pk_mov_b32 v[2:3], v[2:3], v[2:3] op_sel:[1,0]
+; GFX940-NEXT:    v_mov_b32_e32 v4, v1
+; GFX940-NEXT:    v_mov_b32_e32 v5, v1
+; GFX940-NEXT:    global_store_dwordx4 v6, v[2:5], s[0:1] sc0 sc1
 ; GFX940-NEXT:    s_waitcnt vmcnt(0)
 ; GFX940-NEXT:    s_setpc_b64 s[30:31]
   %vec0 = call <4 x i32> asm "; def $0", "=v"()
@@ -9584,8 +9493,7 @@ define void @v_shuffle_v4i32_v4i32__1_6_6_6(ptr addrspace(1) inreg %ptr) {
 ; GFX90A-NEXT:    ; def v[2:5]
 ; GFX90A-NEXT:    ;;#ASMEND
 ; GFX90A-NEXT:    v_mov_b32_e32 v6, 0
-; GFX90A-NEXT:    v_mov_b32_e32 v2, v1
-; GFX90A-NEXT:    v_mov_b32_e32 v3, v4
+; GFX90A-NEXT:    v_pk_mov_b32 v[2:3], v[0:1], v[4:5] op_sel:[1,0]
 ; GFX90A-NEXT:    v_mov_b32_e32 v5, v4
 ; GFX90A-NEXT:    global_store_dwordx4 v6, v[2:5], s[16:17]
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
@@ -9602,8 +9510,7 @@ define void @v_shuffle_v4i32_v4i32__1_6_6_6(ptr addrspace(1) inreg %ptr) {
 ; GFX940-NEXT:    ; def v[2:5]
 ; GFX940-NEXT:    ;;#ASMEND
 ; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_mov_b32_e32 v2, v1
-; GFX940-NEXT:    v_mov_b32_e32 v3, v4
+; GFX940-NEXT:    v_pk_mov_b32 v[2:3], v[0:1], v[4:5] op_sel:[1,0]
 ; GFX940-NEXT:    v_mov_b32_e32 v5, v4
 ; GFX940-NEXT:    global_store_dwordx4 v6, v[2:5], s[0:1] sc0 sc1
 ; GFX940-NEXT:    s_waitcnt vmcnt(0)
@@ -9701,8 +9608,7 @@ define void @v_shuffle_v4i32_v4i32__3_6_6_6(ptr addrspace(1) inreg %ptr) {
 ; GFX90A-NEXT:    ;;#ASMSTART
 ; GFX90A-NEXT:    ; def v[0:3]
 ; GFX90A-NEXT:    ;;#ASMEND
-; GFX90A-NEXT:    v_mov_b32_e32 v4, v3
-; GFX90A-NEXT:    v_mov_b32_e32 v5, v6
+; GFX90A-NEXT:    v_pk_mov_b32 v[4:5], v[2:3], v[6:7] op_sel:[1,0]
 ; GFX90A-NEXT:    v_mov_b32_e32 v7, v6
 ; GFX90A-NEXT:    global_store_dwordx4 v8, v[4:7], s[16:17]
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
@@ -9718,8 +9624,8 @@ define void @v_shuffle_v4i32_v4i32__3_6_6_6(ptr addrspace(1) inreg %ptr) {
 ; GFX940-NEXT:    ;;#ASMSTART
 ; GFX940-NEXT:    ; def v[0:3]
 ; GFX940-NEXT:    ;;#ASMEND
-; GFX940-NEXT:    v_mov_b32_e32 v5, v6
-; GFX940-NEXT:    v_mov_b32_e32 v4, v3
+; GFX940-NEXT:    s_nop 0
+; GFX940-NEXT:    v_pk_mov_b32 v[4:5], v[2:3], v[6:7] op_sel:[1,0]
 ; GFX940-NEXT:    v_mov_b32_e32 v7, v6
 ; GFX940-NEXT:    global_store_dwordx4 v8, v[4:7], s[0:1] sc0 sc1
 ; GFX940-NEXT:    s_waitcnt vmcnt(0)
@@ -9799,8 +9705,7 @@ define void @v_shuffle_v4i32_v4i32__5_6_6_6(ptr addrspace(1) inreg %ptr) {
 ; GFX90A-NEXT:    ; def v[0:3]
 ; GFX90A-NEXT:    ;;#ASMEND
 ; GFX90A-NEXT:    v_mov_b32_e32 v4, 0
-; GFX90A-NEXT:    v_mov_b32_e32 v0, v1
-; GFX90A-NEXT:    v_mov_b32_e32 v1, v2
+; GFX90A-NEXT:    v_pk_mov_b32 v[0:1], v[0:1], v[2:3] op_sel:[1,0]
 ; GFX90A-NEXT:    v_mov_b32_e32 v3, v2
 ; GFX90A-NEXT:    global_store_dwordx4 v4, v[0:3], s[16:17]
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
@@ -9813,8 +9718,7 @@ define void @v_shuffle_v4i32_v4i32__5_6_6_6(ptr addrspace(1) inreg %ptr) {
 ; GFX940-NEXT:    ; def v[0:3]
 ; GFX940-NEXT:    ;;#ASMEND
 ; GFX940-NEXT:    v_mov_b32_e32 v4, 0
-; GFX940-NEXT:    v_mov_b32_e32 v0, v1
-; GFX940-NEXT:    v_mov_b32_e32 v1, v2
+; GFX940-NEXT:    v_pk_mov_b32 v[0:1], v[0:1], v[2:3] op_sel:[1,0]
 ; GFX940-NEXT:    v_mov_b32_e32 v3, v2
 ; GFX940-NEXT:    global_store_dwordx4 v4, v[0:3], s[0:1] sc0 sc1
 ; GFX940-NEXT:    s_waitcnt vmcnt(0)
@@ -9897,8 +9801,7 @@ define void @v_shuffle_v4i32_v4i32__7_6_6_6(ptr addrspace(1) inreg %ptr) {
 ; GFX90A-NEXT:    ; def v[0:3]
 ; GFX90A-NEXT:    ;;#ASMEND
 ; GFX90A-NEXT:    v_mov_b32_e32 v4, 0
-; GFX90A-NEXT:    v_mov_b32_e32 v0, v3
-; GFX90A-NEXT:    v_mov_b32_e32 v1, v2
+; GFX90A-NEXT:    v_pk_mov_b32 v[0:1], v[2:3], v[2:3] op_sel:[1,0]
 ; GFX90A-NEXT:    v_mov_b32_e32 v3, v2
 ; GFX90A-NEXT:    global_store_dwordx4 v4, v[0:3], s[16:17]
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
@@ -9911,8 +9814,7 @@ define void @v_shuffle_v4i32_v4i32__7_6_6_6(ptr addrspace(1) inreg %ptr) {
 ; GFX940-NEXT:    ; def v[0:3]
 ; GFX940-NEXT:    ;;#ASMEND
 ; GFX940-NEXT:    v_mov_b32_e32 v4, 0
-; GFX940-NEXT:    v_mov_b32_e32 v0, v3
-; GFX940-NEXT:    v_mov_b32_e32 v1, v2
+; GFX940-NEXT:    v_pk_mov_b32 v[0:1], v[2:3], v[2:3] op_sel:[1,0]
 ; GFX940-NEXT:    v_mov_b32_e32 v3, v2
 ; GFX940-NEXT:    global_store_dwordx4 v4, v[0:3], s[0:1] sc0 sc1
 ; GFX940-NEXT:    s_waitcnt vmcnt(0)
@@ -9998,8 +9900,7 @@ define void @v_shuffle_v4i32_v4i32__7_0_6_6(ptr addrspace(1) inreg %ptr) {
 ; GFX90A-NEXT:    ; def v[2:5]
 ; GFX90A-NEXT:    ;;#ASMEND
 ; GFX90A-NEXT:    v_mov_b32_e32 v6, 0
-; GFX90A-NEXT:    v_mov_b32_e32 v2, v5
-; GFX90A-NEXT:    v_mov_b32_e32 v3, v0
+; GFX90A-NEXT:    v_pk_mov_b32 v[2:3], v[4:5], v[0:1] op_sel:[1,0]
 ; GFX90A-NEXT:    v_mov_b32_e32 v5, v4
 ; GFX90A-NEXT:    global_store_dwordx4 v6, v[2:5], s[16:17]
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
@@ -10016,8 +9917,7 @@ define void @v_shuffle_v4i32_v4i32__7_0_6_6(ptr addrspace(1) inreg %ptr) {
 ; GFX940-NEXT:    ; def v[2:5]
 ; GFX940-NEXT:    ;;#ASMEND
 ; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_mov_b32_e32 v2, v5
-; GFX940-NEXT:    v_mov_b32_e32 v3, v0
+; GFX940-NEXT:    v_pk_mov_b32 v[2:3], v[4:5], v[0:1] op_sel:[1,0]
 ; GFX940-NEXT:    v_mov_b32_e32 v5, v4
 ; GFX940-NEXT:    global_store_dwordx4 v6, v[2:5], s[0:1] sc0 sc1
 ; GFX940-NEXT:    s_waitcnt vmcnt(0)
@@ -10116,8 +10016,7 @@ define void @v_shuffle_v4i32_v4i32__7_2_6_6(ptr addrspace(1) inreg %ptr) {
 ; GFX90A-NEXT:    ;;#ASMSTART
 ; GFX90A-NEXT:    ; def v[0:3]
 ; GFX90A-NEXT:    ;;#ASMEND
-; GFX90A-NEXT:    v_mov_b32_e32 v4, v7
-; GFX90A-NEXT:    v_mov_b32_e32 v5, v2
+; GFX90A-NEXT:    v_pk_mov_b32 v[4:5], v[6:7], v[2:3] op_sel:[1,0]
 ; GFX90A-NEXT:    v_mov_b32_e32 v7, v6
 ; GFX90A-NEXT:    global_store_dwordx4 v8, v[4:7], s[16:17]
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
@@ -10133,8 +10032,8 @@ define void @v_shuffle_v4i32_v4i32__7_2_6_6(ptr addrspace(1) inreg %ptr) {
 ; GFX940-NEXT:    ;;#ASMSTART
 ; GFX940-NEXT:    ; def v[0:3]
 ; GFX940-NEXT:    ;;#ASMEND
-; GFX940-NEXT:    v_mov_b32_e32 v4, v7
-; GFX940-NEXT:    v_mov_b32_e32 v5, v2
+; GFX940-NEXT:    s_nop 0
+; GFX940-NEXT:    v_pk_mov_b32 v[4:5], v[6:7], v[2:3] op_sel:[1,0]
 ; GFX940-NEXT:    v_mov_b32_e32 v7, v6
 ; GFX940-NEXT:    global_store_dwordx4 v8, v[4:7], s[0:1] sc0 sc1
 ; GFX940-NEXT:    s_waitcnt vmcnt(0)
@@ -10222,31 +10121,26 @@ define void @v_shuffle_v4i32_v4i32__7_4_6_6(ptr addrspace(1) inreg %ptr) {
 ; GFX90A-LABEL: v_shuffle_v4i32_v4i32__7_4_6_6:
 ; GFX90A:       ; %bb.0:
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT:    v_mov_b32_e32 v8, 0
 ; GFX90A-NEXT:    ;;#ASMSTART
 ; GFX90A-NEXT:    ; def v[0:3]
 ; GFX90A-NEXT:    ;;#ASMEND
-; GFX90A-NEXT:    v_mov_b32_e32 v4, v3
-; GFX90A-NEXT:    v_mov_b32_e32 v5, v0
-; GFX90A-NEXT:    v_mov_b32_e32 v6, v2
-; GFX90A-NEXT:    v_mov_b32_e32 v7, v2
-; GFX90A-NEXT:    global_store_dwordx4 v8, v[4:7], s[16:17]
+; GFX90A-NEXT:    v_mov_b32_e32 v4, 0
+; GFX90A-NEXT:    v_pk_mov_b32 v[0:1], v[2:3], v[0:1] op_sel:[1,0]
+; GFX90A-NEXT:    v_mov_b32_e32 v3, v2
+; GFX90A-NEXT:    global_store_dwordx4 v4, v[0:3], s[16:17]
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
 ; GFX90A-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: v_shuffle_v4i32_v4i32__7_4_6_6:
 ; GFX940:       ; %bb.0:
 ; GFX940-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX940-NEXT:    v_mov_b32_e32 v8, 0
 ; GFX940-NEXT:    ;;#ASMSTART
 ; GFX940-NEXT:    ; def v[0:3]
 ; GFX940-NEXT:    ;;#ASMEND
-; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_mov_b32_e32 v4, v3
-; GFX940-NEXT:    v_mov_b32_e32 v5, v0
-; GFX940-NEXT:    v_mov_b32_e32 v6, v2
-; GFX940-NEXT:    v_mov_b32_e32 v7, v2
-; GFX940-NEXT:    global_store_dwordx4 v8, v[4:7], s[0:1] sc0 sc1
+; GFX940-NEXT:    v_mov_b32_e32 v4, 0
+; GFX940-NEXT:    v_pk_mov_b32 v[0:1], v[2:3], v[0:1] op_sel:[1,0]
+; GFX940-NEXT:    v_mov_b32_e32 v3, v2
+; GFX940-NEXT:    global_store_dwordx4 v4, v[0:3], s[0:1] sc0 sc1
 ; GFX940-NEXT:    s_waitcnt vmcnt(0)
 ; GFX940-NEXT:    s_setpc_b64 s[30:31]
   %vec0 = call <4 x i32> asm "; def $0", "=v"()
@@ -10486,15 +10380,14 @@ define void @v_shuffle_v4i32_v4i32__7_7_1_6(ptr addrspace(1) inreg %ptr) {
 ; GFX90A-NEXT:    ;;#ASMSTART
 ; GFX90A-NEXT:    ; def v[0:3]
 ; GFX90A-NEXT:    ;;#ASMEND
-; GFX90A-NEXT:    v_mov_b32_e32 v10, 0
 ; GFX90A-NEXT:    ;;#ASMSTART
 ; GFX90A-NEXT:    ; def v[2:5]
 ; GFX90A-NEXT:    ;;#ASMEND
-; GFX90A-NEXT:    v_mov_b32_e32 v6, v5
-; GFX90A-NEXT:    v_mov_b32_e32 v7, v5
-; GFX90A-NEXT:    v_mov_b32_e32 v8, v1
-; GFX90A-NEXT:    v_mov_b32_e32 v9, v4
-; GFX90A-NEXT:    global_store_dwordx4 v10, v[6:9], s[16:17]
+; GFX90A-NEXT:    v_mov_b32_e32 v6, 0
+; GFX90A-NEXT:    v_pk_mov_b32 v[2:3], v[0:1], v[4:5] op_sel:[1,0]
+; GFX90A-NEXT:    v_mov_b32_e32 v0, v5
+; GFX90A-NEXT:    v_mov_b32_e32 v1, v5
+; GFX90A-NEXT:    global_store_dwordx4 v6, v[0:3], s[16:17]
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
 ; GFX90A-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -10504,15 +10397,15 @@ define void @v_shuffle_v4i32_v4i32__7_7_1_6(ptr addrspace(1) inreg %ptr) {
 ; GFX940-NEXT:    ;;#ASMSTART
 ; GFX940-NEXT:    ; def v[0:3]
 ; GFX940-NEXT:    ;;#ASMEND
-; GFX940-NEXT:    v_mov_b32_e32 v10, 0
+; GFX940-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX940-NEXT:    ;;#ASMSTART
 ; GFX940-NEXT:    ; def v[2:5]
 ; GFX940-NEXT:    ;;#ASMEND
-; GFX940-NEXT:    v_mov_b32_e32 v8, v1
-; GFX940-NEXT:    v_mov_b32_e32 v6, v5
-; GFX940-NEXT:    v_mov_b32_e32 v7, v5
-; GFX940-NEXT:    v_mov_b32_e32 v9, v4
-; GFX940-NEXT:    global_store_dwordx4 v10, v[6:9], s[0:1] sc0 sc1
+; GFX940-NEXT:    s_nop 0
+; GFX940-NEXT:    v_pk_mov_b32 v[2:3], v[0:1], v[4:5] op_sel:[1,0]
+; GFX940-NEXT:    v_mov_b32_e32 v0, v5
+; GFX940-NEXT:    v_mov_b32_e32 v1, v5
+; GFX940-NEXT:    global_store_dwordx4 v6, v[0:3], s[0:1] sc0 sc1
 ; GFX940-NEXT:    s_waitcnt vmcnt(0)
 ; GFX940-NEXT:    s_setpc_b64 s[30:31]
   %vec0 = call <4 x i32> asm "; def $0", "=v"()
@@ -10610,10 +10503,9 @@ define void @v_shuffle_v4i32_v4i32__7_7_3_6(ptr addrspace(1) inreg %ptr) {
 ; GFX90A-NEXT:    ;;#ASMSTART
 ; GFX90A-NEXT:    ; def v[4:7]
 ; GFX90A-NEXT:    ;;#ASMEND
+; GFX90A-NEXT:    v_pk_mov_b32 v[2:3], v[2:3], v[6:7] op_sel:[1,0]
 ; GFX90A-NEXT:    v_mov_b32_e32 v0, v7
 ; GFX90A-NEXT:    v_mov_b32_e32 v1, v7
-; GFX90A-NEXT:    v_mov_b32_e32 v2, v3
-; GFX90A-NEXT:    v_mov_b32_e32 v3, v6
 ; GFX90A-NEXT:    global_store_dwordx4 v8, v[0:3], s[16:17]
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
 ; GFX90A-NEXT:    s_setpc_b64 s[30:31]
@@ -10628,10 +10520,10 @@ define void @v_shuffle_v4i32_v4i32__7_7_3_6(ptr addrspace(1) inreg %ptr) {
 ; GFX940-NEXT:    ;;#ASMSTART
 ; GFX940-NEXT:    ; def v[4:7]
 ; GFX940-NEXT:    ;;#ASMEND
-; GFX940-NEXT:    v_mov_b32_e32 v2, v3
+; GFX940-NEXT:    s_nop 0
+; GFX940-NEXT:    v_pk_mov_b32 v[2:3], v[2:3], v[6:7] op_sel:[1,0]
 ; GFX940-NEXT:    v_mov_b32_e32 v0, v7
 ; GFX940-NEXT:    v_mov_b32_e32 v1, v7
-; GFX940-NEXT:    v_mov_b32_e32 v3, v6
 ; GFX940-NEXT:    global_store_dwordx4 v8, v[0:3], s[0:1] sc0 sc1
 ; GFX940-NEXT:    s_waitcnt vmcnt(0)
 ; GFX940-NEXT:    s_setpc_b64 s[30:31]
@@ -10714,31 +10606,26 @@ define void @v_shuffle_v4i32_v4i32__7_7_5_6(ptr addrspace(1) inreg %ptr) {
 ; GFX90A-LABEL: v_shuffle_v4i32_v4i32__7_7_5_6:
 ; GFX90A:       ; %bb.0:
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT:    v_mov_b32_e32 v8, 0
 ; GFX90A-NEXT:    ;;#ASMSTART
 ; GFX90A-NEXT:    ; def v[0:3]
 ; GFX90A-NEXT:    ;;#ASMEND
-; GFX90A-NEXT:    v_mov_b32_e32 v4, v3
-; GFX90A-NEXT:    v_mov_b32_e32 v5, v3
-; GFX90A-NEXT:    v_mov_b32_e32 v6, v1
-; GFX90A-NEXT:    v_mov_b32_e32 v7, v2
-; GFX90A-NEXT:    global_store_dwordx4 v8, v[4:7], s[16:17]
+; GFX90A-NEXT:    v_mov_b32_e32 v6, 0
+; GFX90A-NEXT:    v_pk_mov_b32 v[4:5], v[0:1], v[2:3] op_sel:[1,0]
+; GFX90A-NEXT:    v_mov_b32_e32 v2, v3
+; GFX90A-NEXT:    global_store_dwordx4 v6, v[2:5], s[16:17]
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
 ; GFX90A-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: v_shuffle_v4i32_v4i32__7_7_5_6:
 ; GFX940:       ; %bb.0:
 ; GFX940-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX940-NEXT:    v_mov_b32_e32 v8, 0
 ; GFX940-NEXT:    ;;#ASMSTART
 ; GFX940-NEXT:    ; def v[0:3]
 ; GFX940-NEXT:    ;;#ASMEND
-; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_mov_b32_e32 v4, v3
-; GFX940-NEXT:    v_mov_b32_e32 v5, v3
-; GFX940-NEXT:    v_mov_b32_e32 v6, v1
-; GFX940-NEXT:    v_mov_b32_e32 v7, v2
-; GFX940-NEXT:    global_store_dwordx4 v8, v[4:7], s[0:1] sc0 sc1
+; GFX940-NEXT:    v_mov_b32_e32 v6, 0
+; GFX940-NEXT:    v_pk_mov_b32 v[4:5], v[0:1], v[2:3] op_sel:[1,0]
+; GFX940-NEXT:    v_mov_b32_e32 v2, v3
+; GFX940-NEXT:    global_store_dwordx4 v6, v[2:5], s[0:1] sc0 sc1
 ; GFX940-NEXT:    s_waitcnt vmcnt(0)
 ; GFX940-NEXT:    s_setpc_b64 s[30:31]
   %vec0 = call <4 x i32> asm "; def $0", "=v"()
@@ -11246,8 +11133,7 @@ define void @v_shuffle_v4i32_v4i32__7_0_7_7(ptr addrspace(1) inreg %ptr) {
 ; GFX90A-NEXT:    ; def v[2:5]
 ; GFX90A-NEXT:    ;;#ASMEND
 ; GFX90A-NEXT:    v_mov_b32_e32 v6, 0
-; GFX90A-NEXT:    v_mov_b32_e32 v2, v5
-; GFX90A-NEXT:    v_mov_b32_e32 v3, v0
+; GFX90A-NEXT:    v_pk_mov_b32 v[2:3], v[4:5], v[0:1] op_sel:[1,0]
 ; GFX90A-NEXT:    v_mov_b32_e32 v4, v5
 ; GFX90A-NEXT:    global_store_dwordx4 v6, v[2:5], s[16:17]
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
@@ -11264,8 +11150,7 @@ define void @v_shuffle_v4i32_v4i32__7_0_7_7(ptr addrspace(1) inreg %ptr) {
 ; GFX940-NEXT:    ; def v[2:5]
 ; GFX940-NEXT:    ;;#ASMEND
 ; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_mov_b32_e32 v2, v5
-; GFX940-NEXT:    v_mov_b32_e32 v3, v0
+; GFX940-NEXT:    v_pk_mov_b32 v[2:3], v[4:5], v[0:1] op_sel:[1,0]
 ; GFX940-NEXT:    v_mov_b32_e32 v4, v5
 ; GFX940-NEXT:    global_store_dwordx4 v6, v[2:5], s[0:1] sc0 sc1
 ; GFX940-NEXT:    s_waitcnt vmcnt(0)
@@ -11364,8 +11249,7 @@ define void @v_shuffle_v4i32_v4i32__7_2_7_7(ptr addrspace(1) inreg %ptr) {
 ; GFX90A-NEXT:    ;;#ASMSTART
 ; GFX90A-NEXT:    ; def v[0:3]
 ; GFX90A-NEXT:    ;;#ASMEND
-; GFX90A-NEXT:    v_mov_b32_e32 v4, v7
-; GFX90A-NEXT:    v_mov_b32_e32 v5, v2
+; GFX90A-NEXT:    v_pk_mov_b32 v[4:5], v[6:7], v[2:3] op_sel:[1,0]
 ; GFX90A-NEXT:    v_mov_b32_e32 v6, v7
 ; GFX90A-NEXT:    global_store_dwordx4 v8, v[4:7], s[16:17]
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
@@ -11381,8 +11265,8 @@ define void @v_shuffle_v4i32_v4i32__7_2_7_7(ptr addrspace(1) inreg %ptr) {
 ; GFX940-NEXT:    ;;#ASMSTART
 ; GFX940-NEXT:    ; def v[0:3]
 ; GFX940-NEXT:    ;;#ASMEND
-; GFX940-NEXT:    v_mov_b32_e32 v4, v7
-; GFX940-NEXT:    v_mov_b32_e32 v5, v2
+; GFX940-NEXT:    s_nop 0
+; GFX940-NEXT:    v_pk_mov_b32 v[4:5], v[6:7], v[2:3] op_sel:[1,0]
 ; GFX940-NEXT:    v_mov_b32_e32 v6, v7
 ; GFX940-NEXT:    global_store_dwordx4 v8, v[4:7], s[0:1] sc0 sc1
 ; GFX940-NEXT:    s_waitcnt vmcnt(0)
@@ -11471,31 +11355,26 @@ define void @v_shuffle_v4i32_v4i32__7_4_7_7(ptr addrspace(1) inreg %ptr) {
 ; GFX90A-LABEL: v_shuffle_v4i32_v4i32__7_4_7_7:
 ; GFX90A:       ; %bb.0:
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT:    v_mov_b32_e32 v8, 0
 ; GFX90A-NEXT:    ;;#ASMSTART
 ; GFX90A-NEXT:    ; def v[0:3]
 ; GFX90A-NEXT:    ;;#ASMEND
-; GFX90A-NEXT:    v_mov_b32_e32 v4, v3
-; GFX90A-NEXT:    v_mov_b32_e32 v5, v0
-; GFX90A-NEXT:    v_mov_b32_e32 v6, v3
-; GFX90A-NEXT:    v_mov_b32_e32 v7, v3
-; GFX90A-NEXT:    global_store_dwordx4 v8, v[4:7], s[16:17]
+; GFX90A-NEXT:    v_mov_b32_e32 v4, 0
+; GFX90A-NEXT:    v_pk_mov_b32 v[0:1], v[2:3], v[0:1] op_sel:[1,0]
+; GFX90A-NEXT:    v_mov_b32_e32 v2, v3
+; GFX90A-NEXT:    global_store_dwordx4 v4, v[0:3], s[16:17]
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
 ; GFX90A-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: v_shuffle_v4i32_v4i32__7_4_7_7:
 ; GFX940:       ; %bb.0:
 ; GFX940-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX940-NEXT:    v_mov_b32_e32 v8, 0
 ; GFX940-NEXT:    ;;#ASMSTART
 ; GFX940-NEXT:    ; def v[0:3]
 ; GFX940-NEXT:    ;;#ASMEND
-; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_mov_b32_e32 v4, v3
-; GFX940-NEXT:    v_mov_b32_e32 v5, v0
-; GFX940-NEXT:    v_mov_b32_e32 v6, v3
-; GFX940-NEXT:    v_mov_b32_e32 v7, v3
-; GFX940-NEXT:    global_store_dwordx4 v8, v[4:7], s[0:1] sc0 sc1
+; GFX940-NEXT:    v_mov_b32_e32 v4, 0
+; GFX940-NEXT:    v_pk_mov_b32 v[0:1], v[2:3], v[0:1] op_sel:[1,0]
+; GFX940-NEXT:    v_mov_b32_e32 v2, v3
+; GFX940-NEXT:    global_store_dwordx4 v4, v[0:3], s[0:1] sc0 sc1
 ; GFX940-NEXT:    s_waitcnt vmcnt(0)
 ; GFX940-NEXT:    s_setpc_b64 s[30:31]
   %vec0 = call <4 x i32> asm "; def $0", "=v"()
@@ -11573,8 +11452,7 @@ define void @v_shuffle_v4i32_v4i32__7_6_7_7(ptr addrspace(1) inreg %ptr) {
 ; GFX90A-NEXT:    ; def v[0:3]
 ; GFX90A-NEXT:    ;;#ASMEND
 ; GFX90A-NEXT:    v_mov_b32_e32 v4, 0
-; GFX90A-NEXT:    v_mov_b32_e32 v0, v3
-; GFX90A-NEXT:    v_mov_b32_e32 v1, v2
+; GFX90A-NEXT:    v_pk_mov_b32 v[0:1], v[2:3], v[2:3] op_sel:[1,0]
 ; GFX90A-NEXT:    v_mov_b32_e32 v2, v3
 ; GFX90A-NEXT:    global_store_dwordx4 v4, v[0:3], s[16:17]
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
@@ -11587,8 +11465,7 @@ define void @v_shuffle_v4i32_v4i32__7_6_7_7(ptr addrspace(1) inreg %ptr) {
 ; GFX940-NEXT:    ; def v[0:3]
 ; GFX940-NEXT:    ;;#ASMEND
 ; GFX940-NEXT:    v_mov_b32_e32 v4, 0
-; GFX940-NEXT:    v_mov_b32_e32 v0, v3
-; GFX940-NEXT:    v_mov_b32_e32 v1, v2
+; GFX940-NEXT:    v_pk_mov_b32 v[0:1], v[2:3], v[2:3] op_sel:[1,0]
 ; GFX940-NEXT:    v_mov_b32_e32 v2, v3
 ; GFX940-NEXT:    global_store_dwordx4 v4, v[0:3], s[0:1] sc0 sc1
 ; GFX940-NEXT:    s_waitcnt vmcnt(0)
diff --git a/llvm/test/CodeGen/AMDGPU/shufflevector.v4p3.v2p3.ll b/llvm/test/CodeGen/AMDGPU/shufflevector.v4p3.v2p3.ll
index 2dd794404158c..f9253ca1b1ea4 100644
--- a/llvm/test/CodeGen/AMDGPU/shufflevector.v4p3.v2p3.ll
+++ b/llvm/test/CodeGen/AMDGPU/shufflevector.v4p3.v2p3.ll
@@ -176,8 +176,7 @@ define void @v_shuffle_v4p3_v2p3__3_0_u_u(ptr addrspace(1) inreg %ptr) {
 ; GFX90A-NEXT:    ;;#ASMSTART
 ; GFX90A-NEXT:    ; def v[2:3]
 ; GFX90A-NEXT:    ;;#ASMEND
-; GFX90A-NEXT:    v_mov_b32_e32 v0, v1
-; GFX90A-NEXT:    v_mov_b32_e32 v1, v2
+; GFX90A-NEXT:    v_pk_mov_b32 v[0:1], v[2:3], v[0:1] op_sel:[1,0]
 ; GFX90A-NEXT:    global_store_dwordx4 v4, v[0:3], s[16:17]
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
 ; GFX90A-NEXT:    s_setpc_b64 s[30:31]
@@ -192,8 +191,8 @@ define void @v_shuffle_v4p3_v2p3__3_0_u_u(ptr addrspace(1) inreg %ptr) {
 ; GFX940-NEXT:    ;;#ASMSTART
 ; GFX940-NEXT:    ; def v[2:3]
 ; GFX940-NEXT:    ;;#ASMEND
-; GFX940-NEXT:    v_mov_b32_e32 v0, v1
-; GFX940-NEXT:    v_mov_b32_e32 v1, v2
+; GFX940-NEXT:    s_nop 0
+; GFX940-NEXT:    v_pk_mov_b32 v[0:1], v[2:3], v[0:1] op_sel:[1,0]
 ; GFX940-NEXT:    global_store_dwordx4 v4, v[0:3], s[0:1] sc0 sc1
 ; GFX940-NEXT:    s_waitcnt vmcnt(0)
 ; GFX940-NEXT:    s_setpc_b64 s[30:31]
@@ -566,16 +565,15 @@ define void @v_shuffle_v4p3_v2p3__3_3_3_0(ptr addrspace(1) inreg %ptr) {
 ; GFX90A:       ; %bb.0:
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX90A-NEXT:    ;;#ASMSTART
-; GFX90A-NEXT:    ; def v[0:1]
+; GFX90A-NEXT:    ; def v[2:3]
 ; GFX90A-NEXT:    ;;#ASMEND
-; GFX90A-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX90A-NEXT:    ;;#ASMSTART
-; GFX90A-NEXT:    ; def v[4:5]
+; GFX90A-NEXT:    ; def v[0:1]
 ; GFX90A-NEXT:    ;;#ASMEND
+; GFX90A-NEXT:    v_mov_b32_e32 v4, 0
+; GFX90A-NEXT:    v_pk_mov_b32 v[2:3], v[0:1], v[2:3] op_sel:[1,0]
 ; GFX90A-NEXT:    v_mov_b32_e32 v0, v1
-; GFX90A-NEXT:    v_mov_b32_e32 v2, v1
-; GFX90A-NEXT:    v_mov_b32_e32 v3, v4
-; GFX90A-NEXT:    global_store_dwordx4 v6, v[0:3], s[16:17]
+; GFX90A-NEXT:    global_store_dwordx4 v4, v[0:3], s[16:17]
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
 ; GFX90A-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -583,16 +581,15 @@ define void @v_shuffle_v4p3_v2p3__3_3_3_0(ptr addrspace(1) inreg %ptr) {
 ; GFX940:       ; %bb.0:
 ; GFX940-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX940-NEXT:    ;;#ASMSTART
-; GFX940-NEXT:    ; def v[0:1]
+; GFX940-NEXT:    ; def v[2:3]
 ; GFX940-NEXT:    ;;#ASMEND
-; GFX940-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX940-NEXT:    ;;#ASMSTART
-; GFX940-NEXT:    ; def v[4:5]
+; GFX940-NEXT:    ; def v[0:1]
 ; GFX940-NEXT:    ;;#ASMEND
+; GFX940-NEXT:    v_mov_b32_e32 v4, 0
+; GFX940-NEXT:    v_pk_mov_b32 v[2:3], v[0:1], v[2:3] op_sel:[1,0]
 ; GFX940-NEXT:    v_mov_b32_e32 v0, v1
-; GFX940-NEXT:    v_mov_b32_e32 v2, v1
-; GFX940-NEXT:    v_mov_b32_e32 v3, v4
-; GFX940-NEXT:    global_store_dwordx4 v6, v[0:3], s[0:1] sc0 sc1
+; GFX940-NEXT:    global_store_dwordx4 v4, v[0:3], s[0:1] sc0 sc1
 ; GFX940-NEXT:    s_waitcnt vmcnt(0)
 ; GFX940-NEXT:    s_setpc_b64 s[30:31]
   %vec0 = call <2 x ptr addrspace(3)> asm "; def $0", "=v"()
@@ -675,31 +672,26 @@ define void @v_shuffle_v4p3_v2p3__3_3_3_2(ptr addrspace(1) inreg %ptr) {
 ; GFX90A-LABEL: v_shuffle_v4p3_v2p3__3_3_3_2:
 ; GFX90A:       ; %bb.0:
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX90A-NEXT:    ;;#ASMSTART
-; GFX90A-NEXT:    ; def v[4:5]
+; GFX90A-NEXT:    ; def v[0:1]
 ; GFX90A-NEXT:    ;;#ASMEND
-; GFX90A-NEXT:    v_mov_b32_e32 v0, v5
-; GFX90A-NEXT:    v_mov_b32_e32 v1, v5
-; GFX90A-NEXT:    v_mov_b32_e32 v2, v5
-; GFX90A-NEXT:    v_mov_b32_e32 v3, v4
-; GFX90A-NEXT:    global_store_dwordx4 v6, v[0:3], s[16:17]
+; GFX90A-NEXT:    v_mov_b32_e32 v4, 0
+; GFX90A-NEXT:    v_pk_mov_b32 v[2:3], v[0:1], v[0:1] op_sel:[1,0]
+; GFX90A-NEXT:    v_mov_b32_e32 v0, v1
+; GFX90A-NEXT:    global_store_dwordx4 v4, v[0:3], s[16:17]
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
 ; GFX90A-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: v_shuffle_v4p3_v2p3__3_3_3_2:
 ; GFX940:       ; %bb.0:
 ; GFX940-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX940-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX940-NEXT:    ;;#ASMSTART
-; GFX940-NEXT:    ; def v[4:5]
+; GFX940-NEXT:    ; def v[0:1]
 ; GFX940-NEXT:    ;;#ASMEND
-; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_mov_b32_e32 v0, v5
-; GFX940-NEXT:    v_mov_b32_e32 v1, v5
-; GFX940-NEXT:    v_mov_b32_e32 v2, v5
-; GFX940-NEXT:    v_mov_b32_e32 v3, v4
-; GFX940-NEXT:    global_store_dwordx4 v6, v[0:3], s[0:1] sc0 sc1
+; GFX940-NEXT:    v_mov_b32_e32 v4, 0
+; GFX940-NEXT:    v_pk_mov_b32 v[2:3], v[0:1], v[0:1] op_sel:[1,0]
+; GFX940-NEXT:    v_mov_b32_e32 v0, v1
+; GFX940-NEXT:    global_store_dwordx4 v4, v[0:3], s[0:1] sc0 sc1
 ; GFX940-NEXT:    s_waitcnt vmcnt(0)
 ; GFX940-NEXT:    s_setpc_b64 s[30:31]
   %vec0 = call <2 x ptr addrspace(3)> asm "; def $0", "=v"()
@@ -876,8 +868,7 @@ define void @v_shuffle_v4p3_v2p3__1_0_0_0(ptr addrspace(1) inreg %ptr) {
 ; GFX90A-NEXT:    ; def v[2:3]
 ; GFX90A-NEXT:    ;;#ASMEND
 ; GFX90A-NEXT:    v_mov_b32_e32 v4, 0
-; GFX90A-NEXT:    v_mov_b32_e32 v0, v3
-; GFX90A-NEXT:    v_mov_b32_e32 v1, v2
+; GFX90A-NEXT:    v_pk_mov_b32 v[0:1], v[2:3], v[2:3] op_sel:[1,0]
 ; GFX90A-NEXT:    v_mov_b32_e32 v3, v2
 ; GFX90A-NEXT:    global_store_dwordx4 v4, v[0:3], s[16:17]
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
@@ -890,8 +881,7 @@ define void @v_shuffle_v4p3_v2p3__1_0_0_0(ptr addrspace(1) inreg %ptr) {
 ; GFX940-NEXT:    ; def v[2:3]
 ; GFX940-NEXT:    ;;#ASMEND
 ; GFX940-NEXT:    v_mov_b32_e32 v4, 0
-; GFX940-NEXT:    v_mov_b32_e32 v0, v3
-; GFX940-NEXT:    v_mov_b32_e32 v1, v2
+; GFX940-NEXT:    v_pk_mov_b32 v[0:1], v[2:3], v[2:3] op_sel:[1,0]
 ; GFX940-NEXT:    v_mov_b32_e32 v3, v2
 ; GFX940-NEXT:    global_store_dwordx4 v4, v[0:3], s[0:1] sc0 sc1
 ; GFX940-NEXT:    s_waitcnt vmcnt(0)
@@ -978,8 +968,7 @@ define void @v_shuffle_v4p3_v2p3__3_0_0_0(ptr addrspace(1) inreg %ptr) {
 ; GFX90A-NEXT:    ; def v[0:1]
 ; GFX90A-NEXT:    ;;#ASMEND
 ; GFX90A-NEXT:    v_mov_b32_e32 v4, 0
-; GFX90A-NEXT:    v_mov_b32_e32 v0, v1
-; GFX90A-NEXT:    v_mov_b32_e32 v1, v2
+; GFX90A-NEXT:    v_pk_mov_b32 v[0:1], v[0:1], v[2:3] op_sel:[1,0]
 ; GFX90A-NEXT:    v_mov_b32_e32 v3, v2
 ; GFX90A-NEXT:    global_store_dwordx4 v4, v[0:3], s[16:17]
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
@@ -995,8 +984,7 @@ define void @v_shuffle_v4p3_v2p3__3_0_0_0(ptr addrspace(1) inreg %ptr) {
 ; GFX940-NEXT:    ; def v[0:1]
 ; GFX940-NEXT:    ;;#ASMEND
 ; GFX940-NEXT:    v_mov_b32_e32 v4, 0
-; GFX940-NEXT:    v_mov_b32_e32 v0, v1
-; GFX940-NEXT:    v_mov_b32_e32 v1, v2
+; GFX940-NEXT:    v_pk_mov_b32 v[0:1], v[0:1], v[2:3] op_sel:[1,0]
 ; GFX940-NEXT:    v_mov_b32_e32 v3, v2
 ; GFX940-NEXT:    global_store_dwordx4 v4, v[0:3], s[0:1] sc0 sc1
 ; GFX940-NEXT:    s_waitcnt vmcnt(0)
@@ -1144,14 +1132,13 @@ define void @v_shuffle_v4p3_v2p3__3_2_0_0(ptr addrspace(1) inreg %ptr) {
 ; GFX90A-NEXT:    ;;#ASMSTART
 ; GFX90A-NEXT:    ; def v[2:3]
 ; GFX90A-NEXT:    ;;#ASMEND
-; GFX90A-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX90A-NEXT:    ;;#ASMSTART
-; GFX90A-NEXT:    ; def v[4:5]
+; GFX90A-NEXT:    ; def v[0:1]
 ; GFX90A-NEXT:    ;;#ASMEND
-; GFX90A-NEXT:    v_mov_b32_e32 v0, v5
-; GFX90A-NEXT:    v_mov_b32_e32 v1, v4
+; GFX90A-NEXT:    v_mov_b32_e32 v4, 0
+; GFX90A-NEXT:    v_pk_mov_b32 v[0:1], v[0:1], v[0:1] op_sel:[1,0]
 ; GFX90A-NEXT:    v_mov_b32_e32 v3, v2
-; GFX90A-NEXT:    global_store_dwordx4 v6, v[0:3], s[16:17]
+; GFX90A-NEXT:    global_store_dwordx4 v4, v[0:3], s[16:17]
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
 ; GFX90A-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -1161,14 +1148,13 @@ define void @v_shuffle_v4p3_v2p3__3_2_0_0(ptr addrspace(1) inreg %ptr) {
 ; GFX940-NEXT:    ;;#ASMSTART
 ; GFX940-NEXT:    ; def v[2:3]
 ; GFX940-NEXT:    ;;#ASMEND
-; GFX940-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX940-NEXT:    ;;#ASMSTART
-; GFX940-NEXT:    ; def v[4:5]
+; GFX940-NEXT:    ; def v[0:1]
 ; GFX940-NEXT:    ;;#ASMEND
+; GFX940-NEXT:    v_mov_b32_e32 v4, 0
+; GFX940-NEXT:    v_pk_mov_b32 v[0:1], v[0:1], v[0:1] op_sel:[1,0]
 ; GFX940-NEXT:    v_mov_b32_e32 v3, v2
-; GFX940-NEXT:    v_mov_b32_e32 v0, v5
-; GFX940-NEXT:    v_mov_b32_e32 v1, v4
-; GFX940-NEXT:    global_store_dwordx4 v6, v[0:3], s[0:1] sc0 sc1
+; GFX940-NEXT:    global_store_dwordx4 v4, v[0:3], s[0:1] sc0 sc1
 ; GFX940-NEXT:    s_waitcnt vmcnt(0)
 ; GFX940-NEXT:    s_setpc_b64 s[30:31]
   %vec0 = call <2 x ptr addrspace(3)> asm "; def $0", "=v"()
@@ -1309,16 +1295,15 @@ define void @v_shuffle_v4p3_v2p3__3_3_1_0(ptr addrspace(1) inreg %ptr) {
 ; GFX90A:       ; %bb.0:
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX90A-NEXT:    ;;#ASMSTART
-; GFX90A-NEXT:    ; def v[0:1]
+; GFX90A-NEXT:    ; def v[2:3]
 ; GFX90A-NEXT:    ;;#ASMEND
-; GFX90A-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX90A-NEXT:    ;;#ASMSTART
-; GFX90A-NEXT:    ; def v[4:5]
+; GFX90A-NEXT:    ; def v[0:1]
 ; GFX90A-NEXT:    ;;#ASMEND
+; GFX90A-NEXT:    v_mov_b32_e32 v4, 0
+; GFX90A-NEXT:    v_pk_mov_b32 v[2:3], v[2:3], v[2:3] op_sel:[1,0]
 ; GFX90A-NEXT:    v_mov_b32_e32 v0, v1
-; GFX90A-NEXT:    v_mov_b32_e32 v2, v5
-; GFX90A-NEXT:    v_mov_b32_e32 v3, v4
-; GFX90A-NEXT:    global_store_dwordx4 v6, v[0:3], s[16:17]
+; GFX90A-NEXT:    global_store_dwordx4 v4, v[0:3], s[16:17]
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
 ; GFX90A-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -1326,16 +1311,15 @@ define void @v_shuffle_v4p3_v2p3__3_3_1_0(ptr addrspace(1) inreg %ptr) {
 ; GFX940:       ; %bb.0:
 ; GFX940-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX940-NEXT:    ;;#ASMSTART
-; GFX940-NEXT:    ; def v[0:1]
+; GFX940-NEXT:    ; def v[2:3]
 ; GFX940-NEXT:    ;;#ASMEND
-; GFX940-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX940-NEXT:    ;;#ASMSTART
-; GFX940-NEXT:    ; def v[4:5]
+; GFX940-NEXT:    ; def v[0:1]
 ; GFX940-NEXT:    ;;#ASMEND
+; GFX940-NEXT:    v_mov_b32_e32 v4, 0
+; GFX940-NEXT:    v_pk_mov_b32 v[2:3], v[2:3], v[2:3] op_sel:[1,0]
 ; GFX940-NEXT:    v_mov_b32_e32 v0, v1
-; GFX940-NEXT:    v_mov_b32_e32 v2, v5
-; GFX940-NEXT:    v_mov_b32_e32 v3, v4
-; GFX940-NEXT:    global_store_dwordx4 v6, v[0:3], s[0:1] sc0 sc1
+; GFX940-NEXT:    global_store_dwordx4 v4, v[0:3], s[0:1] sc0 sc1
 ; GFX940-NEXT:    s_waitcnt vmcnt(0)
 ; GFX940-NEXT:    s_setpc_b64 s[30:31]
   %vec0 = call <2 x ptr addrspace(3)> asm "; def $0", "=v"()
@@ -1729,8 +1713,7 @@ define void @v_shuffle_v4p3_v2p3__3_0_1_1(ptr addrspace(1) inreg %ptr) {
 ; GFX90A-NEXT:    ; def v[0:1]
 ; GFX90A-NEXT:    ;;#ASMEND
 ; GFX90A-NEXT:    v_mov_b32_e32 v4, 0
-; GFX90A-NEXT:    v_mov_b32_e32 v0, v1
-; GFX90A-NEXT:    v_mov_b32_e32 v1, v2
+; GFX90A-NEXT:    v_pk_mov_b32 v[0:1], v[0:1], v[2:3] op_sel:[1,0]
 ; GFX90A-NEXT:    v_mov_b32_e32 v2, v3
 ; GFX90A-NEXT:    global_store_dwordx4 v4, v[0:3], s[16:17]
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
@@ -1746,8 +1729,7 @@ define void @v_shuffle_v4p3_v2p3__3_0_1_1(ptr addrspace(1) inreg %ptr) {
 ; GFX940-NEXT:    ; def v[0:1]
 ; GFX940-NEXT:    ;;#ASMEND
 ; GFX940-NEXT:    v_mov_b32_e32 v4, 0
-; GFX940-NEXT:    v_mov_b32_e32 v0, v1
-; GFX940-NEXT:    v_mov_b32_e32 v1, v2
+; GFX940-NEXT:    v_pk_mov_b32 v[0:1], v[0:1], v[2:3] op_sel:[1,0]
 ; GFX940-NEXT:    v_mov_b32_e32 v2, v3
 ; GFX940-NEXT:    global_store_dwordx4 v4, v[0:3], s[0:1] sc0 sc1
 ; GFX940-NEXT:    s_waitcnt vmcnt(0)
@@ -1782,14 +1764,13 @@ define void @v_shuffle_v4p3_v2p3__3_2_1_1(ptr addrspace(1) inreg %ptr) {
 ; GFX90A-NEXT:    ;;#ASMSTART
 ; GFX90A-NEXT:    ; def v[2:3]
 ; GFX90A-NEXT:    ;;#ASMEND
-; GFX90A-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX90A-NEXT:    ;;#ASMSTART
-; GFX90A-NEXT:    ; def v[4:5]
+; GFX90A-NEXT:    ; def v[0:1]
 ; GFX90A-NEXT:    ;;#ASMEND
-; GFX90A-NEXT:    v_mov_b32_e32 v0, v5
-; GFX90A-NEXT:    v_mov_b32_e32 v1, v4
+; GFX90A-NEXT:    v_mov_b32_e32 v4, 0
+; GFX90A-NEXT:    v_pk_mov_b32 v[0:1], v[0:1], v[0:1] op_sel:[1,0]
 ; GFX90A-NEXT:    v_mov_b32_e32 v2, v3
-; GFX90A-NEXT:    global_store_dwordx4 v6, v[0:3], s[16:17]
+; GFX90A-NEXT:    global_store_dwordx4 v4, v[0:3], s[16:17]
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
 ; GFX90A-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -1799,14 +1780,13 @@ define void @v_shuffle_v4p3_v2p3__3_2_1_1(ptr addrspace(1) inreg %ptr) {
 ; GFX940-NEXT:    ;;#ASMSTART
 ; GFX940-NEXT:    ; def v[2:3]
 ; GFX940-NEXT:    ;;#ASMEND
-; GFX940-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX940-NEXT:    ;;#ASMSTART
-; GFX940-NEXT:    ; def v[4:5]
+; GFX940-NEXT:    ; def v[0:1]
 ; GFX940-NEXT:    ;;#ASMEND
+; GFX940-NEXT:    v_mov_b32_e32 v4, 0
+; GFX940-NEXT:    v_pk_mov_b32 v[0:1], v[0:1], v[0:1] op_sel:[1,0]
 ; GFX940-NEXT:    v_mov_b32_e32 v2, v3
-; GFX940-NEXT:    v_mov_b32_e32 v0, v5
-; GFX940-NEXT:    v_mov_b32_e32 v1, v4
-; GFX940-NEXT:    global_store_dwordx4 v6, v[0:3], s[0:1] sc0 sc1
+; GFX940-NEXT:    global_store_dwordx4 v4, v[0:3], s[0:1] sc0 sc1
 ; GFX940-NEXT:    s_waitcnt vmcnt(0)
 ; GFX940-NEXT:    s_setpc_b64 s[30:31]
   %vec0 = call <2 x ptr addrspace(3)> asm "; def $0", "=v"()
@@ -2158,8 +2138,7 @@ define void @v_shuffle_v4p3_v2p3__3_2_2_2(ptr addrspace(1) inreg %ptr) {
 ; GFX90A-NEXT:    ; def v[2:3]
 ; GFX90A-NEXT:    ;;#ASMEND
 ; GFX90A-NEXT:    v_mov_b32_e32 v4, 0
-; GFX90A-NEXT:    v_mov_b32_e32 v0, v3
-; GFX90A-NEXT:    v_mov_b32_e32 v1, v2
+; GFX90A-NEXT:    v_pk_mov_b32 v[0:1], v[2:3], v[2:3] op_sel:[1,0]
 ; GFX90A-NEXT:    v_mov_b32_e32 v3, v2
 ; GFX90A-NEXT:    global_store_dwordx4 v4, v[0:3], s[16:17]
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
@@ -2172,8 +2151,7 @@ define void @v_shuffle_v4p3_v2p3__3_2_2_2(ptr addrspace(1) inreg %ptr) {
 ; GFX940-NEXT:    ; def v[2:3]
 ; GFX940-NEXT:    ;;#ASMEND
 ; GFX940-NEXT:    v_mov_b32_e32 v4, 0
-; GFX940-NEXT:    v_mov_b32_e32 v0, v3
-; GFX940-NEXT:    v_mov_b32_e32 v1, v2
+; GFX940-NEXT:    v_pk_mov_b32 v[0:1], v[2:3], v[2:3] op_sel:[1,0]
 ; GFX940-NEXT:    v_mov_b32_e32 v3, v2
 ; GFX940-NEXT:    global_store_dwordx4 v4, v[0:3], s[0:1] sc0 sc1
 ; GFX940-NEXT:    s_waitcnt vmcnt(0)
@@ -2252,16 +2230,15 @@ define void @v_shuffle_v4p3_v2p3__3_0_2_2(ptr addrspace(1) inreg %ptr) {
 ; GFX90A:       ; %bb.0:
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX90A-NEXT:    ;;#ASMSTART
-; GFX90A-NEXT:    ; def v[2:3]
+; GFX90A-NEXT:    ; def v[0:1]
 ; GFX90A-NEXT:    ;;#ASMEND
-; GFX90A-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX90A-NEXT:    ;;#ASMSTART
-; GFX90A-NEXT:    ; def v[4:5]
+; GFX90A-NEXT:    ; def v[2:3]
 ; GFX90A-NEXT:    ;;#ASMEND
-; GFX90A-NEXT:    v_mov_b32_e32 v0, v3
-; GFX90A-NEXT:    v_mov_b32_e32 v1, v4
+; GFX90A-NEXT:    v_mov_b32_e32 v4, 0
+; GFX90A-NEXT:    v_pk_mov_b32 v[0:1], v[2:3], v[0:1] op_sel:[1,0]
 ; GFX90A-NEXT:    v_mov_b32_e32 v3, v2
-; GFX90A-NEXT:    global_store_dwordx4 v6, v[0:3], s[16:17]
+; GFX90A-NEXT:    global_store_dwordx4 v4, v[0:3], s[16:17]
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
 ; GFX90A-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -2269,16 +2246,15 @@ define void @v_shuffle_v4p3_v2p3__3_0_2_2(ptr addrspace(1) inreg %ptr) {
 ; GFX940:       ; %bb.0:
 ; GFX940-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX940-NEXT:    ;;#ASMSTART
-; GFX940-NEXT:    ; def v[2:3]
+; GFX940-NEXT:    ; def v[0:1]
 ; GFX940-NEXT:    ;;#ASMEND
-; GFX940-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX940-NEXT:    ;;#ASMSTART
-; GFX940-NEXT:    ; def v[4:5]
+; GFX940-NEXT:    ; def v[2:3]
 ; GFX940-NEXT:    ;;#ASMEND
-; GFX940-NEXT:    v_mov_b32_e32 v0, v3
-; GFX940-NEXT:    v_mov_b32_e32 v1, v4
+; GFX940-NEXT:    v_mov_b32_e32 v4, 0
+; GFX940-NEXT:    v_pk_mov_b32 v[0:1], v[2:3], v[0:1] op_sel:[1,0]
 ; GFX940-NEXT:    v_mov_b32_e32 v3, v2
-; GFX940-NEXT:    global_store_dwordx4 v6, v[0:3], s[0:1] sc0 sc1
+; GFX940-NEXT:    global_store_dwordx4 v4, v[0:3], s[0:1] sc0 sc1
 ; GFX940-NEXT:    s_waitcnt vmcnt(0)
 ; GFX940-NEXT:    s_setpc_b64 s[30:31]
   %vec0 = call <2 x ptr addrspace(3)> asm "; def $0", "=v"()
@@ -2522,15 +2498,13 @@ define void @v_shuffle_v4p3_v2p3__3_3_1_2(ptr addrspace(1) inreg %ptr) {
 ; GFX90A-NEXT:    ;;#ASMSTART
 ; GFX90A-NEXT:    ; def v[2:3]
 ; GFX90A-NEXT:    ;;#ASMEND
-; GFX90A-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX90A-NEXT:    ;;#ASMSTART
-; GFX90A-NEXT:    ; def v[4:5]
+; GFX90A-NEXT:    ; def v[0:1]
 ; GFX90A-NEXT:    ;;#ASMEND
-; GFX90A-NEXT:    v_mov_b32_e32 v0, v5
-; GFX90A-NEXT:    v_mov_b32_e32 v1, v5
-; GFX90A-NEXT:    v_mov_b32_e32 v2, v3
-; GFX90A-NEXT:    v_mov_b32_e32 v3, v4
-; GFX90A-NEXT:    global_store_dwordx4 v6, v[0:3], s[16:17]
+; GFX90A-NEXT:    v_mov_b32_e32 v4, 0
+; GFX90A-NEXT:    v_pk_mov_b32 v[2:3], v[2:3], v[0:1] op_sel:[1,0]
+; GFX90A-NEXT:    v_mov_b32_e32 v0, v1
+; GFX90A-NEXT:    global_store_dwordx4 v4, v[0:3], s[16:17]
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
 ; GFX90A-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -2540,15 +2514,13 @@ define void @v_shuffle_v4p3_v2p3__3_3_1_2(ptr addrspace(1) inreg %ptr) {
 ; GFX940-NEXT:    ;;#ASMSTART
 ; GFX940-NEXT:    ; def v[2:3]
 ; GFX940-NEXT:    ;;#ASMEND
-; GFX940-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX940-NEXT:    ;;#ASMSTART
-; GFX940-NEXT:    ; def v[4:5]
+; GFX940-NEXT:    ; def v[0:1]
 ; GFX940-NEXT:    ;;#ASMEND
-; GFX940-NEXT:    v_mov_b32_e32 v2, v3
-; GFX940-NEXT:    v_mov_b32_e32 v0, v5
-; GFX940-NEXT:    v_mov_b32_e32 v1, v5
-; GFX940-NEXT:    v_mov_b32_e32 v3, v4
-; GFX940-NEXT:    global_store_dwordx4 v6, v[0:3], s[0:1] sc0 sc1
+; GFX940-NEXT:    v_mov_b32_e32 v4, 0
+; GFX940-NEXT:    v_pk_mov_b32 v[2:3], v[2:3], v[0:1] op_sel:[1,0]
+; GFX940-NEXT:    v_mov_b32_e32 v0, v1
+; GFX940-NEXT:    global_store_dwordx4 v4, v[0:3], s[0:1] sc0 sc1
 ; GFX940-NEXT:    s_waitcnt vmcnt(0)
 ; GFX940-NEXT:    s_setpc_b64 s[30:31]
   %vec0 = call <2 x ptr addrspace(3)> asm "; def $0", "=v"()
@@ -2832,16 +2804,15 @@ define void @v_shuffle_v4p3_v2p3__3_0_3_3(ptr addrspace(1) inreg %ptr) {
 ; GFX90A:       ; %bb.0:
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX90A-NEXT:    ;;#ASMSTART
-; GFX90A-NEXT:    ; def v[2:3]
+; GFX90A-NEXT:    ; def v[0:1]
 ; GFX90A-NEXT:    ;;#ASMEND
-; GFX90A-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX90A-NEXT:    ;;#ASMSTART
-; GFX90A-NEXT:    ; def v[4:5]
+; GFX90A-NEXT:    ; def v[2:3]
 ; GFX90A-NEXT:    ;;#ASMEND
-; GFX90A-NEXT:    v_mov_b32_e32 v0, v3
-; GFX90A-NEXT:    v_mov_b32_e32 v1, v4
+; GFX90A-NEXT:    v_mov_b32_e32 v4, 0
+; GFX90A-NEXT:    v_pk_mov_b32 v[0:1], v[2:3], v[0:1] op_sel:[1,0]
 ; GFX90A-NEXT:    v_mov_b32_e32 v2, v3
-; GFX90A-NEXT:    global_store_dwordx4 v6, v[0:3], s[16:17]
+; GFX90A-NEXT:    global_store_dwordx4 v4, v[0:3], s[16:17]
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
 ; GFX90A-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -2849,16 +2820,15 @@ define void @v_shuffle_v4p3_v2p3__3_0_3_3(ptr addrspace(1) inreg %ptr) {
 ; GFX940:       ; %bb.0:
 ; GFX940-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX940-NEXT:    ;;#ASMSTART
-; GFX940-NEXT:    ; def v[2:3]
+; GFX940-NEXT:    ; def v[0:1]
 ; GFX940-NEXT:    ;;#ASMEND
-; GFX940-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX940-NEXT:    ;;#ASMSTART
-; GFX940-NEXT:    ; def v[4:5]
+; GFX940-NEXT:    ; def v[2:3]
 ; GFX940-NEXT:    ;;#ASMEND
-; GFX940-NEXT:    v_mov_b32_e32 v0, v3
-; GFX940-NEXT:    v_mov_b32_e32 v1, v4
+; GFX940-NEXT:    v_mov_b32_e32 v4, 0
+; GFX940-NEXT:    v_pk_mov_b32 v[0:1], v[2:3], v[0:1] op_sel:[1,0]
 ; GFX940-NEXT:    v_mov_b32_e32 v2, v3
-; GFX940-NEXT:    global_store_dwordx4 v6, v[0:3], s[0:1] sc0 sc1
+; GFX940-NEXT:    global_store_dwordx4 v4, v[0:3], s[0:1] sc0 sc1
 ; GFX940-NEXT:    s_waitcnt vmcnt(0)
 ; GFX940-NEXT:    s_setpc_b64 s[30:31]
   %vec0 = call <2 x ptr addrspace(3)> asm "; def $0", "=v"()
@@ -2944,8 +2914,7 @@ define void @v_shuffle_v4p3_v2p3__3_2_3_3(ptr addrspace(1) inreg %ptr) {
 ; GFX90A-NEXT:    ; def v[2:3]
 ; GFX90A-NEXT:    ;;#ASMEND
 ; GFX90A-NEXT:    v_mov_b32_e32 v4, 0
-; GFX90A-NEXT:    v_mov_b32_e32 v0, v3
-; GFX90A-NEXT:    v_mov_b32_e32 v1, v2
+; GFX90A-NEXT:    v_pk_mov_b32 v[0:1], v[2:3], v[2:3] op_sel:[1,0]
 ; GFX90A-NEXT:    v_mov_b32_e32 v2, v3
 ; GFX90A-NEXT:    global_store_dwordx4 v4, v[0:3], s[16:17]
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
@@ -2958,8 +2927,7 @@ define void @v_shuffle_v4p3_v2p3__3_2_3_3(ptr addrspace(1) inreg %ptr) {
 ; GFX940-NEXT:    ; def v[2:3]
 ; GFX940-NEXT:    ;;#ASMEND
 ; GFX940-NEXT:    v_mov_b32_e32 v4, 0
-; GFX940-NEXT:    v_mov_b32_e32 v0, v3
-; GFX940-NEXT:    v_mov_b32_e32 v1, v2
+; GFX940-NEXT:    v_pk_mov_b32 v[0:1], v[2:3], v[2:3] op_sel:[1,0]
 ; GFX940-NEXT:    v_mov_b32_e32 v2, v3
 ; GFX940-NEXT:    global_store_dwordx4 v4, v[0:3], s[0:1] sc0 sc1
 ; GFX940-NEXT:    s_waitcnt vmcnt(0)
diff --git a/llvm/test/CodeGen/AMDGPU/shufflevector.v4p3.v3p3.ll b/llvm/test/CodeGen/AMDGPU/shufflevector.v4p3.v3p3.ll
index d6bd854283180..28bc61ce57815 100644
--- a/llvm/test/CodeGen/AMDGPU/shufflevector.v4p3.v3p3.ll
+++ b/llvm/test/CodeGen/AMDGPU/shufflevector.v4p3.v3p3.ll
@@ -1280,8 +1280,7 @@ define void @v_shuffle_v4p3_v3p3__1_0_0_0(ptr addrspace(1) inreg %ptr) {
 ; GFX90A-NEXT:    ;;#ASMSTART
 ; GFX90A-NEXT:    ; def v[4:6]
 ; GFX90A-NEXT:    ;;#ASMEND
-; GFX90A-NEXT:    v_mov_b32_e32 v0, v5
-; GFX90A-NEXT:    v_mov_b32_e32 v1, v4
+; GFX90A-NEXT:    v_pk_mov_b32 v[0:1], v[4:5], v[4:5] op_sel:[1,0]
 ; GFX90A-NEXT:    v_mov_b32_e32 v2, v4
 ; GFX90A-NEXT:    v_mov_b32_e32 v3, v4
 ; GFX90A-NEXT:    global_store_dwordx4 v7, v[0:3], s[16:17]
@@ -1296,8 +1295,7 @@ define void @v_shuffle_v4p3_v3p3__1_0_0_0(ptr addrspace(1) inreg %ptr) {
 ; GFX940-NEXT:    ; def v[4:6]
 ; GFX940-NEXT:    ;;#ASMEND
 ; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_mov_b32_e32 v0, v5
-; GFX940-NEXT:    v_mov_b32_e32 v1, v4
+; GFX940-NEXT:    v_pk_mov_b32 v[0:1], v[4:5], v[4:5] op_sel:[1,0]
 ; GFX940-NEXT:    v_mov_b32_e32 v2, v4
 ; GFX940-NEXT:    v_mov_b32_e32 v3, v4
 ; GFX940-NEXT:    global_store_dwordx4 v7, v[0:3], s[0:1] sc0 sc1
@@ -1435,8 +1433,7 @@ define void @v_shuffle_v4p3_v3p3__4_0_0_0(ptr addrspace(1) inreg %ptr) {
 ; GFX90A-NEXT:    ;;#ASMSTART
 ; GFX90A-NEXT:    ; def v[4:6]
 ; GFX90A-NEXT:    ;;#ASMEND
-; GFX90A-NEXT:    v_mov_b32_e32 v0, v1
-; GFX90A-NEXT:    v_mov_b32_e32 v1, v4
+; GFX90A-NEXT:    v_pk_mov_b32 v[0:1], v[0:1], v[4:5] op_sel:[1,0]
 ; GFX90A-NEXT:    v_mov_b32_e32 v2, v4
 ; GFX90A-NEXT:    v_mov_b32_e32 v3, v4
 ; GFX90A-NEXT:    global_store_dwordx4 v7, v[0:3], s[16:17]
@@ -1453,8 +1450,8 @@ define void @v_shuffle_v4p3_v3p3__4_0_0_0(ptr addrspace(1) inreg %ptr) {
 ; GFX940-NEXT:    ;;#ASMSTART
 ; GFX940-NEXT:    ; def v[4:6]
 ; GFX940-NEXT:    ;;#ASMEND
-; GFX940-NEXT:    v_mov_b32_e32 v0, v1
-; GFX940-NEXT:    v_mov_b32_e32 v1, v4
+; GFX940-NEXT:    s_nop 0
+; GFX940-NEXT:    v_pk_mov_b32 v[0:1], v[0:1], v[4:5] op_sel:[1,0]
 ; GFX940-NEXT:    v_mov_b32_e32 v2, v4
 ; GFX940-NEXT:    v_mov_b32_e32 v3, v4
 ; GFX940-NEXT:    global_store_dwordx4 v7, v[0:3], s[0:1] sc0 sc1
@@ -1964,15 +1961,14 @@ define void @v_shuffle_v4p3_v3p3__5_5_1_0(ptr addrspace(1) inreg %ptr) {
 ; GFX90A-NEXT:    ;;#ASMSTART
 ; GFX90A-NEXT:    ; def v[0:2]
 ; GFX90A-NEXT:    ;;#ASMEND
-; GFX90A-NEXT:    v_mov_b32_e32 v7, 0
 ; GFX90A-NEXT:    ;;#ASMSTART
-; GFX90A-NEXT:    ; def v[4:6]
+; GFX90A-NEXT:    ; def v[2:4]
 ; GFX90A-NEXT:    ;;#ASMEND
-; GFX90A-NEXT:    v_mov_b32_e32 v0, v2
-; GFX90A-NEXT:    v_mov_b32_e32 v1, v2
-; GFX90A-NEXT:    v_mov_b32_e32 v2, v5
-; GFX90A-NEXT:    v_mov_b32_e32 v3, v4
-; GFX90A-NEXT:    global_store_dwordx4 v7, v[0:3], s[16:17]
+; GFX90A-NEXT:    v_mov_b32_e32 v5, 0
+; GFX90A-NEXT:    v_pk_mov_b32 v[2:3], v[0:1], v[0:1] op_sel:[1,0]
+; GFX90A-NEXT:    v_mov_b32_e32 v0, v4
+; GFX90A-NEXT:    v_mov_b32_e32 v1, v4
+; GFX90A-NEXT:    global_store_dwordx4 v5, v[0:3], s[16:17]
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
 ; GFX90A-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -1982,15 +1978,15 @@ define void @v_shuffle_v4p3_v3p3__5_5_1_0(ptr addrspace(1) inreg %ptr) {
 ; GFX940-NEXT:    ;;#ASMSTART
 ; GFX940-NEXT:    ; def v[0:2]
 ; GFX940-NEXT:    ;;#ASMEND
-; GFX940-NEXT:    v_mov_b32_e32 v7, 0
+; GFX940-NEXT:    v_mov_b32_e32 v5, 0
 ; GFX940-NEXT:    ;;#ASMSTART
-; GFX940-NEXT:    ; def v[4:6]
+; GFX940-NEXT:    ; def v[2:4]
 ; GFX940-NEXT:    ;;#ASMEND
-; GFX940-NEXT:    v_mov_b32_e32 v0, v2
-; GFX940-NEXT:    v_mov_b32_e32 v1, v2
-; GFX940-NEXT:    v_mov_b32_e32 v2, v5
-; GFX940-NEXT:    v_mov_b32_e32 v3, v4
-; GFX940-NEXT:    global_store_dwordx4 v7, v[0:3], s[0:1] sc0 sc1
+; GFX940-NEXT:    s_nop 0
+; GFX940-NEXT:    v_pk_mov_b32 v[2:3], v[0:1], v[0:1] op_sel:[1,0]
+; GFX940-NEXT:    v_mov_b32_e32 v0, v4
+; GFX940-NEXT:    v_mov_b32_e32 v1, v4
+; GFX940-NEXT:    global_store_dwordx4 v5, v[0:3], s[0:1] sc0 sc1
 ; GFX940-NEXT:    s_waitcnt vmcnt(0)
 ; GFX940-NEXT:    s_setpc_b64 s[30:31]
   %vec0 = call <3 x ptr addrspace(3)> asm "; def $0", "=v"()
@@ -2142,17 +2138,16 @@ define void @v_shuffle_v4p3_v3p3__5_5_4_0(ptr addrspace(1) inreg %ptr) {
 ; GFX90A:       ; %bb.0:
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX90A-NEXT:    ;;#ASMSTART
-; GFX90A-NEXT:    ; def v[4:6]
+; GFX90A-NEXT:    ; def v[0:2]
 ; GFX90A-NEXT:    ;;#ASMEND
-; GFX90A-NEXT:    v_mov_b32_e32 v9, 0
 ; GFX90A-NEXT:    ;;#ASMSTART
-; GFX90A-NEXT:    ; def v[6:8]
+; GFX90A-NEXT:    ; def v[2:4]
 ; GFX90A-NEXT:    ;;#ASMEND
-; GFX90A-NEXT:    v_mov_b32_e32 v0, v8
-; GFX90A-NEXT:    v_mov_b32_e32 v1, v8
-; GFX90A-NEXT:    v_mov_b32_e32 v2, v7
-; GFX90A-NEXT:    v_mov_b32_e32 v3, v4
-; GFX90A-NEXT:    global_store_dwordx4 v9, v[0:3], s[16:17]
+; GFX90A-NEXT:    v_mov_b32_e32 v5, 0
+; GFX90A-NEXT:    v_pk_mov_b32 v[2:3], v[2:3], v[0:1] op_sel:[1,0]
+; GFX90A-NEXT:    v_mov_b32_e32 v0, v4
+; GFX90A-NEXT:    v_mov_b32_e32 v1, v4
+; GFX90A-NEXT:    global_store_dwordx4 v5, v[0:3], s[16:17]
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
 ; GFX90A-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -2160,17 +2155,17 @@ define void @v_shuffle_v4p3_v3p3__5_5_4_0(ptr addrspace(1) inreg %ptr) {
 ; GFX940:       ; %bb.0:
 ; GFX940-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX940-NEXT:    ;;#ASMSTART
-; GFX940-NEXT:    ; def v[4:6]
+; GFX940-NEXT:    ; def v[0:2]
 ; GFX940-NEXT:    ;;#ASMEND
-; GFX940-NEXT:    v_mov_b32_e32 v9, 0
+; GFX940-NEXT:    v_mov_b32_e32 v5, 0
 ; GFX940-NEXT:    ;;#ASMSTART
-; GFX940-NEXT:    ; def v[6:8]
+; GFX940-NEXT:    ; def v[2:4]
 ; GFX940-NEXT:    ;;#ASMEND
-; GFX940-NEXT:    v_mov_b32_e32 v3, v4
-; GFX940-NEXT:    v_mov_b32_e32 v0, v8
-; GFX940-NEXT:    v_mov_b32_e32 v1, v8
-; GFX940-NEXT:    v_mov_b32_e32 v2, v7
-; GFX940-NEXT:    global_store_dwordx4 v9, v[0:3], s[0:1] sc0 sc1
+; GFX940-NEXT:    s_nop 0
+; GFX940-NEXT:    v_pk_mov_b32 v[2:3], v[2:3], v[0:1] op_sel:[1,0]
+; GFX940-NEXT:    v_mov_b32_e32 v0, v4
+; GFX940-NEXT:    v_mov_b32_e32 v1, v4
+; GFX940-NEXT:    global_store_dwordx4 v5, v[0:3], s[0:1] sc0 sc1
 ; GFX940-NEXT:    s_waitcnt vmcnt(0)
 ; GFX940-NEXT:    s_setpc_b64 s[30:31]
   %vec0 = call <3 x ptr addrspace(3)> asm "; def $0", "=v"()
@@ -3277,9 +3272,9 @@ define void @v_shuffle_v4p3_v3p3__1_2_2_2(ptr addrspace(1) inreg %ptr) {
 ; GFX90A-NEXT:    ;;#ASMSTART
 ; GFX90A-NEXT:    ; def v[0:2]
 ; GFX90A-NEXT:    ;;#ASMEND
+; GFX90A-NEXT:    v_mov_b32_e32 v0, v2
 ; GFX90A-NEXT:    v_mov_b32_e32 v4, 0
-; GFX90A-NEXT:    v_mov_b32_e32 v0, v1
-; GFX90A-NEXT:    v_mov_b32_e32 v1, v2
+; GFX90A-NEXT:    v_pk_mov_b32 v[0:1], v[0:1], v[0:1] op_sel:[1,0]
 ; GFX90A-NEXT:    v_mov_b32_e32 v3, v2
 ; GFX90A-NEXT:    global_store_dwordx4 v4, v[0:3], s[16:17]
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
@@ -3292,8 +3287,8 @@ define void @v_shuffle_v4p3_v3p3__1_2_2_2(ptr addrspace(1) inreg %ptr) {
 ; GFX940-NEXT:    ; def v[0:2]
 ; GFX940-NEXT:    ;;#ASMEND
 ; GFX940-NEXT:    v_mov_b32_e32 v4, 0
-; GFX940-NEXT:    v_mov_b32_e32 v0, v1
-; GFX940-NEXT:    v_mov_b32_e32 v1, v2
+; GFX940-NEXT:    v_mov_b32_e32 v0, v2
+; GFX940-NEXT:    v_pk_mov_b32 v[0:1], v[0:1], v[0:1] op_sel:[1,0]
 ; GFX940-NEXT:    v_mov_b32_e32 v3, v2
 ; GFX940-NEXT:    global_store_dwordx4 v4, v[0:3], s[0:1] sc0 sc1
 ; GFX940-NEXT:    s_waitcnt vmcnt(0)
@@ -3421,12 +3416,12 @@ define void @v_shuffle_v4p3_v3p3__4_2_2_2(ptr addrspace(1) inreg %ptr) {
 ; GFX90A-NEXT:    ;;#ASMSTART
 ; GFX90A-NEXT:    ; def v[0:2]
 ; GFX90A-NEXT:    ;;#ASMEND
+; GFX90A-NEXT:    v_mov_b32_e32 v0, v2
 ; GFX90A-NEXT:    v_mov_b32_e32 v7, 0
 ; GFX90A-NEXT:    ;;#ASMSTART
 ; GFX90A-NEXT:    ; def v[4:6]
 ; GFX90A-NEXT:    ;;#ASMEND
-; GFX90A-NEXT:    v_mov_b32_e32 v0, v5
-; GFX90A-NEXT:    v_mov_b32_e32 v1, v2
+; GFX90A-NEXT:    v_pk_mov_b32 v[0:1], v[4:5], v[0:1] op_sel:[1,0]
 ; GFX90A-NEXT:    v_mov_b32_e32 v3, v2
 ; GFX90A-NEXT:    global_store_dwordx4 v7, v[0:3], s[16:17]
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
@@ -3439,12 +3434,12 @@ define void @v_shuffle_v4p3_v3p3__4_2_2_2(ptr addrspace(1) inreg %ptr) {
 ; GFX940-NEXT:    ; def v[0:2]
 ; GFX940-NEXT:    ;;#ASMEND
 ; GFX940-NEXT:    v_mov_b32_e32 v7, 0
+; GFX940-NEXT:    v_mov_b32_e32 v0, v2
 ; GFX940-NEXT:    ;;#ASMSTART
 ; GFX940-NEXT:    ; def v[4:6]
 ; GFX940-NEXT:    ;;#ASMEND
-; GFX940-NEXT:    v_mov_b32_e32 v1, v2
-; GFX940-NEXT:    v_mov_b32_e32 v0, v5
 ; GFX940-NEXT:    v_mov_b32_e32 v3, v2
+; GFX940-NEXT:    v_pk_mov_b32 v[0:1], v[4:5], v[0:1] op_sel:[1,0]
 ; GFX940-NEXT:    global_store_dwordx4 v7, v[0:3], s[0:1] sc0 sc1
 ; GFX940-NEXT:    s_waitcnt vmcnt(0)
 ; GFX940-NEXT:    s_setpc_b64 s[30:31]
@@ -3991,17 +3986,17 @@ define void @v_shuffle_v4p3_v3p3__5_5_1_2(ptr addrspace(1) inreg %ptr) {
 ; GFX90A:       ; %bb.0:
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX90A-NEXT:    ;;#ASMSTART
-; GFX90A-NEXT:    ; def v[2:4]
-; GFX90A-NEXT:    ;;#ASMEND
-; GFX90A-NEXT:    ;;#ASMSTART
 ; GFX90A-NEXT:    ; def v[0:2]
 ; GFX90A-NEXT:    ;;#ASMEND
-; GFX90A-NEXT:    v_mov_b32_e32 v5, 0
 ; GFX90A-NEXT:    v_mov_b32_e32 v0, v2
-; GFX90A-NEXT:    v_mov_b32_e32 v1, v2
-; GFX90A-NEXT:    v_mov_b32_e32 v2, v3
-; GFX90A-NEXT:    v_mov_b32_e32 v3, v4
-; GFX90A-NEXT:    global_store_dwordx4 v5, v[0:3], s[16:17]
+; GFX90A-NEXT:    v_mov_b32_e32 v7, 0
+; GFX90A-NEXT:    ;;#ASMSTART
+; GFX90A-NEXT:    ; def v[4:6]
+; GFX90A-NEXT:    ;;#ASMEND
+; GFX90A-NEXT:    v_pk_mov_b32 v[2:3], v[0:1], v[0:1] op_sel:[1,0]
+; GFX90A-NEXT:    v_mov_b32_e32 v0, v6
+; GFX90A-NEXT:    v_mov_b32_e32 v1, v6
+; GFX90A-NEXT:    global_store_dwordx4 v7, v[0:3], s[16:17]
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
 ; GFX90A-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -4009,18 +4004,17 @@ define void @v_shuffle_v4p3_v3p3__5_5_1_2(ptr addrspace(1) inreg %ptr) {
 ; GFX940:       ; %bb.0:
 ; GFX940-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX940-NEXT:    ;;#ASMSTART
-; GFX940-NEXT:    ; def v[2:4]
-; GFX940-NEXT:    ;;#ASMEND
-; GFX940-NEXT:    v_mov_b32_e32 v5, 0
-; GFX940-NEXT:    ;;#ASMSTART
 ; GFX940-NEXT:    ; def v[0:2]
 ; GFX940-NEXT:    ;;#ASMEND
-; GFX940-NEXT:    s_nop 0
+; GFX940-NEXT:    v_mov_b32_e32 v7, 0
 ; GFX940-NEXT:    v_mov_b32_e32 v0, v2
-; GFX940-NEXT:    v_mov_b32_e32 v1, v2
-; GFX940-NEXT:    v_mov_b32_e32 v2, v3
-; GFX940-NEXT:    v_mov_b32_e32 v3, v4
-; GFX940-NEXT:    global_store_dwordx4 v5, v[0:3], s[0:1] sc0 sc1
+; GFX940-NEXT:    ;;#ASMSTART
+; GFX940-NEXT:    ; def v[4:6]
+; GFX940-NEXT:    ;;#ASMEND
+; GFX940-NEXT:    v_pk_mov_b32 v[2:3], v[0:1], v[0:1] op_sel:[1,0]
+; GFX940-NEXT:    v_mov_b32_e32 v0, v6
+; GFX940-NEXT:    v_mov_b32_e32 v1, v6
+; GFX940-NEXT:    global_store_dwordx4 v7, v[0:3], s[0:1] sc0 sc1
 ; GFX940-NEXT:    s_waitcnt vmcnt(0)
 ; GFX940-NEXT:    s_setpc_b64 s[30:31]
   %vec0 = call <3 x ptr addrspace(3)> asm "; def $0", "=v"()
@@ -4112,17 +4106,17 @@ define void @v_shuffle_v4p3_v3p3__5_5_4_2(ptr addrspace(1) inreg %ptr) {
 ; GFX90A:       ; %bb.0:
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX90A-NEXT:    ;;#ASMSTART
-; GFX90A-NEXT:    ; def v[2:4]
+; GFX90A-NEXT:    ; def v[0:2]
 ; GFX90A-NEXT:    ;;#ASMEND
-; GFX90A-NEXT:    v_mov_b32_e32 v5, 0
+; GFX90A-NEXT:    v_mov_b32_e32 v0, v2
+; GFX90A-NEXT:    v_mov_b32_e32 v7, 0
 ; GFX90A-NEXT:    ;;#ASMSTART
-; GFX90A-NEXT:    ; def v[6:8]
+; GFX90A-NEXT:    ; def v[4:6]
 ; GFX90A-NEXT:    ;;#ASMEND
-; GFX90A-NEXT:    v_mov_b32_e32 v0, v8
-; GFX90A-NEXT:    v_mov_b32_e32 v1, v8
-; GFX90A-NEXT:    v_mov_b32_e32 v2, v7
-; GFX90A-NEXT:    v_mov_b32_e32 v3, v4
-; GFX90A-NEXT:    global_store_dwordx4 v5, v[0:3], s[16:17]
+; GFX90A-NEXT:    v_pk_mov_b32 v[2:3], v[4:5], v[0:1] op_sel:[1,0]
+; GFX90A-NEXT:    v_mov_b32_e32 v0, v6
+; GFX90A-NEXT:    v_mov_b32_e32 v1, v6
+; GFX90A-NEXT:    global_store_dwordx4 v7, v[0:3], s[16:17]
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
 ; GFX90A-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -4130,17 +4124,18 @@ define void @v_shuffle_v4p3_v3p3__5_5_4_2(ptr addrspace(1) inreg %ptr) {
 ; GFX940:       ; %bb.0:
 ; GFX940-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX940-NEXT:    ;;#ASMSTART
-; GFX940-NEXT:    ; def v[2:4]
+; GFX940-NEXT:    ; def v[0:2]
 ; GFX940-NEXT:    ;;#ASMEND
-; GFX940-NEXT:    v_mov_b32_e32 v5, 0
+; GFX940-NEXT:    v_mov_b32_e32 v7, 0
+; GFX940-NEXT:    v_mov_b32_e32 v0, v2
 ; GFX940-NEXT:    ;;#ASMSTART
-; GFX940-NEXT:    ; def v[6:8]
+; GFX940-NEXT:    ; def v[4:6]
 ; GFX940-NEXT:    ;;#ASMEND
-; GFX940-NEXT:    v_mov_b32_e32 v3, v4
-; GFX940-NEXT:    v_mov_b32_e32 v0, v8
-; GFX940-NEXT:    v_mov_b32_e32 v1, v8
-; GFX940-NEXT:    v_mov_b32_e32 v2, v7
-; GFX940-NEXT:    global_store_dwordx4 v5, v[0:3], s[0:1] sc0 sc1
+; GFX940-NEXT:    s_nop 0
+; GFX940-NEXT:    v_pk_mov_b32 v[2:3], v[4:5], v[0:1] op_sel:[1,0]
+; GFX940-NEXT:    v_mov_b32_e32 v0, v6
+; GFX940-NEXT:    v_mov_b32_e32 v1, v6
+; GFX940-NEXT:    global_store_dwordx4 v7, v[0:3], s[0:1] sc0 sc1
 ; GFX940-NEXT:    s_waitcnt vmcnt(0)
 ; GFX940-NEXT:    s_setpc_b64 s[30:31]
   %vec0 = call <3 x ptr addrspace(3)> asm "; def $0", "=v"()
@@ -4317,8 +4312,7 @@ define void @v_shuffle_v4p3_v3p3__4_3_3_3(ptr addrspace(1) inreg %ptr) {
 ; GFX90A-NEXT:    ;;#ASMSTART
 ; GFX90A-NEXT:    ; def v[4:6]
 ; GFX90A-NEXT:    ;;#ASMEND
-; GFX90A-NEXT:    v_mov_b32_e32 v0, v5
-; GFX90A-NEXT:    v_mov_b32_e32 v1, v4
+; GFX90A-NEXT:    v_pk_mov_b32 v[0:1], v[4:5], v[4:5] op_sel:[1,0]
 ; GFX90A-NEXT:    v_mov_b32_e32 v2, v4
 ; GFX90A-NEXT:    v_mov_b32_e32 v3, v4
 ; GFX90A-NEXT:    global_store_dwordx4 v7, v[0:3], s[16:17]
@@ -4333,8 +4327,7 @@ define void @v_shuffle_v4p3_v3p3__4_3_3_3(ptr addrspace(1) inreg %ptr) {
 ; GFX940-NEXT:    ; def v[4:6]
 ; GFX940-NEXT:    ;;#ASMEND
 ; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_mov_b32_e32 v0, v5
-; GFX940-NEXT:    v_mov_b32_e32 v1, v4
+; GFX940-NEXT:    v_pk_mov_b32 v[0:1], v[4:5], v[4:5] op_sel:[1,0]
 ; GFX940-NEXT:    v_mov_b32_e32 v2, v4
 ; GFX940-NEXT:    v_mov_b32_e32 v3, v4
 ; GFX940-NEXT:    global_store_dwordx4 v7, v[0:3], s[0:1] sc0 sc1
@@ -4858,17 +4851,16 @@ define void @v_shuffle_v4p3_v3p3__5_5_1_3(ptr addrspace(1) inreg %ptr) {
 ; GFX90A:       ; %bb.0:
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX90A-NEXT:    ;;#ASMSTART
-; GFX90A-NEXT:    ; def v[2:4]
+; GFX90A-NEXT:    ; def v[0:2]
 ; GFX90A-NEXT:    ;;#ASMEND
-; GFX90A-NEXT:    v_mov_b32_e32 v7, 0
 ; GFX90A-NEXT:    ;;#ASMSTART
-; GFX90A-NEXT:    ; def v[4:6]
+; GFX90A-NEXT:    ; def v[2:4]
 ; GFX90A-NEXT:    ;;#ASMEND
-; GFX90A-NEXT:    v_mov_b32_e32 v0, v6
-; GFX90A-NEXT:    v_mov_b32_e32 v1, v6
-; GFX90A-NEXT:    v_mov_b32_e32 v2, v3
-; GFX90A-NEXT:    v_mov_b32_e32 v3, v4
-; GFX90A-NEXT:    global_store_dwordx4 v7, v[0:3], s[16:17]
+; GFX90A-NEXT:    v_mov_b32_e32 v5, 0
+; GFX90A-NEXT:    v_pk_mov_b32 v[2:3], v[0:1], v[2:3] op_sel:[1,0]
+; GFX90A-NEXT:    v_mov_b32_e32 v0, v4
+; GFX90A-NEXT:    v_mov_b32_e32 v1, v4
+; GFX90A-NEXT:    global_store_dwordx4 v5, v[0:3], s[16:17]
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
 ; GFX90A-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -4876,17 +4868,17 @@ define void @v_shuffle_v4p3_v3p3__5_5_1_3(ptr addrspace(1) inreg %ptr) {
 ; GFX940:       ; %bb.0:
 ; GFX940-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX940-NEXT:    ;;#ASMSTART
-; GFX940-NEXT:    ; def v[2:4]
+; GFX940-NEXT:    ; def v[0:2]
 ; GFX940-NEXT:    ;;#ASMEND
-; GFX940-NEXT:    v_mov_b32_e32 v7, 0
+; GFX940-NEXT:    v_mov_b32_e32 v5, 0
 ; GFX940-NEXT:    ;;#ASMSTART
-; GFX940-NEXT:    ; def v[4:6]
+; GFX940-NEXT:    ; def v[2:4]
 ; GFX940-NEXT:    ;;#ASMEND
-; GFX940-NEXT:    v_mov_b32_e32 v2, v3
-; GFX940-NEXT:    v_mov_b32_e32 v0, v6
-; GFX940-NEXT:    v_mov_b32_e32 v1, v6
-; GFX940-NEXT:    v_mov_b32_e32 v3, v4
-; GFX940-NEXT:    global_store_dwordx4 v7, v[0:3], s[0:1] sc0 sc1
+; GFX940-NEXT:    s_nop 0
+; GFX940-NEXT:    v_pk_mov_b32 v[2:3], v[0:1], v[2:3] op_sel:[1,0]
+; GFX940-NEXT:    v_mov_b32_e32 v0, v4
+; GFX940-NEXT:    v_mov_b32_e32 v1, v4
+; GFX940-NEXT:    global_store_dwordx4 v5, v[0:3], s[0:1] sc0 sc1
 ; GFX940-NEXT:    s_waitcnt vmcnt(0)
 ; GFX940-NEXT:    s_setpc_b64 s[30:31]
   %vec0 = call <3 x ptr addrspace(3)> asm "; def $0", "=v"()
@@ -4972,31 +4964,28 @@ define void @v_shuffle_v4p3_v3p3__5_5_4_3(ptr addrspace(1) inreg %ptr) {
 ; GFX90A-LABEL: v_shuffle_v4p3_v3p3__5_5_4_3:
 ; GFX90A:       ; %bb.0:
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT:    v_mov_b32_e32 v7, 0
 ; GFX90A-NEXT:    ;;#ASMSTART
-; GFX90A-NEXT:    ; def v[4:6]
+; GFX90A-NEXT:    ; def v[2:4]
 ; GFX90A-NEXT:    ;;#ASMEND
-; GFX90A-NEXT:    v_mov_b32_e32 v0, v6
-; GFX90A-NEXT:    v_mov_b32_e32 v1, v6
-; GFX90A-NEXT:    v_mov_b32_e32 v2, v5
-; GFX90A-NEXT:    v_mov_b32_e32 v3, v4
-; GFX90A-NEXT:    global_store_dwordx4 v7, v[0:3], s[16:17]
+; GFX90A-NEXT:    v_mov_b32_e32 v5, 0
+; GFX90A-NEXT:    v_pk_mov_b32 v[2:3], v[2:3], v[2:3] op_sel:[1,0]
+; GFX90A-NEXT:    v_mov_b32_e32 v0, v4
+; GFX90A-NEXT:    v_mov_b32_e32 v1, v4
+; GFX90A-NEXT:    global_store_dwordx4 v5, v[0:3], s[16:17]
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
 ; GFX90A-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: v_shuffle_v4p3_v3p3__5_5_4_3:
 ; GFX940:       ; %bb.0:
 ; GFX940-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX940-NEXT:    v_mov_b32_e32 v7, 0
 ; GFX940-NEXT:    ;;#ASMSTART
-; GFX940-NEXT:    ; def v[4:6]
+; GFX940-NEXT:    ; def v[2:4]
 ; GFX940-NEXT:    ;;#ASMEND
-; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_mov_b32_e32 v0, v6
-; GFX940-NEXT:    v_mov_b32_e32 v1, v6
-; GFX940-NEXT:    v_mov_b32_e32 v2, v5
-; GFX940-NEXT:    v_mov_b32_e32 v3, v4
-; GFX940-NEXT:    global_store_dwordx4 v7, v[0:3], s[0:1] sc0 sc1
+; GFX940-NEXT:    v_mov_b32_e32 v5, 0
+; GFX940-NEXT:    v_pk_mov_b32 v[2:3], v[2:3], v[2:3] op_sel:[1,0]
+; GFX940-NEXT:    v_mov_b32_e32 v0, v4
+; GFX940-NEXT:    v_mov_b32_e32 v1, v4
+; GFX940-NEXT:    global_store_dwordx4 v5, v[0:3], s[0:1] sc0 sc1
 ; GFX940-NEXT:    s_waitcnt vmcnt(0)
 ; GFX940-NEXT:    s_setpc_b64 s[30:31]
   %vec0 = call <3 x ptr addrspace(3)> asm "; def $0", "=v"()
@@ -6094,9 +6083,9 @@ define void @v_shuffle_v4p3_v3p3__1_5_5_5(ptr addrspace(1) inreg %ptr) {
 ; GFX90A-NEXT:    ;;#ASMSTART
 ; GFX90A-NEXT:    ; def v[0:2]
 ; GFX90A-NEXT:    ;;#ASMEND
+; GFX90A-NEXT:    v_mov_b32_e32 v0, v2
 ; GFX90A-NEXT:    v_mov_b32_e32 v5, 0
-; GFX90A-NEXT:    v_mov_b32_e32 v0, v3
-; GFX90A-NEXT:    v_mov_b32_e32 v1, v2
+; GFX90A-NEXT:    v_pk_mov_b32 v[0:1], v[2:3], v[0:1] op_sel:[1,0]
 ; GFX90A-NEXT:    v_mov_b32_e32 v3, v2
 ; GFX90A-NEXT:    global_store_dwordx4 v5, v[0:3], s[16:17]
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
@@ -6113,8 +6102,8 @@ define void @v_shuffle_v4p3_v3p3__1_5_5_5(ptr addrspace(1) inreg %ptr) {
 ; GFX940-NEXT:    ; def v[0:2]
 ; GFX940-NEXT:    ;;#ASMEND
 ; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_mov_b32_e32 v0, v3
-; GFX940-NEXT:    v_mov_b32_e32 v1, v2
+; GFX940-NEXT:    v_mov_b32_e32 v0, v2
+; GFX940-NEXT:    v_pk_mov_b32 v[0:1], v[2:3], v[0:1] op_sel:[1,0]
 ; GFX940-NEXT:    v_mov_b32_e32 v3, v2
 ; GFX940-NEXT:    global_store_dwordx4 v5, v[0:3], s[0:1] sc0 sc1
 ; GFX940-NEXT:    s_waitcnt vmcnt(0)
@@ -6252,9 +6241,9 @@ define void @v_shuffle_v4p3_v3p3__4_5_5_5(ptr addrspace(1) inreg %ptr) {
 ; GFX90A-NEXT:    ;;#ASMSTART
 ; GFX90A-NEXT:    ; def v[0:2]
 ; GFX90A-NEXT:    ;;#ASMEND
+; GFX90A-NEXT:    v_mov_b32_e32 v0, v2
 ; GFX90A-NEXT:    v_mov_b32_e32 v4, 0
-; GFX90A-NEXT:    v_mov_b32_e32 v0, v1
-; GFX90A-NEXT:    v_mov_b32_e32 v1, v2
+; GFX90A-NEXT:    v_pk_mov_b32 v[0:1], v[0:1], v[0:1] op_sel:[1,0]
 ; GFX90A-NEXT:    v_mov_b32_e32 v3, v2
 ; GFX90A-NEXT:    global_store_dwordx4 v4, v[0:3], s[16:17]
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
@@ -6267,8 +6256,8 @@ define void @v_shuffle_v4p3_v3p3__4_5_5_5(ptr addrspace(1) inreg %ptr) {
 ; GFX940-NEXT:    ; def v[0:2]
 ; GFX940-NEXT:    ;;#ASMEND
 ; GFX940-NEXT:    v_mov_b32_e32 v4, 0
-; GFX940-NEXT:    v_mov_b32_e32 v0, v1
-; GFX940-NEXT:    v_mov_b32_e32 v1, v2
+; GFX940-NEXT:    v_mov_b32_e32 v0, v2
+; GFX940-NEXT:    v_pk_mov_b32 v[0:1], v[0:1], v[0:1] op_sel:[1,0]
 ; GFX940-NEXT:    v_mov_b32_e32 v3, v2
 ; GFX940-NEXT:    global_store_dwordx4 v4, v[0:3], s[0:1] sc0 sc1
 ; GFX940-NEXT:    s_waitcnt vmcnt(0)
@@ -6727,17 +6716,16 @@ define void @v_shuffle_v4p3_v3p3__5_5_1_5(ptr addrspace(1) inreg %ptr) {
 ; GFX90A:       ; %bb.0:
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX90A-NEXT:    ;;#ASMSTART
-; GFX90A-NEXT:    ; def v[2:4]
+; GFX90A-NEXT:    ; def v[0:2]
 ; GFX90A-NEXT:    ;;#ASMEND
-; GFX90A-NEXT:    v_mov_b32_e32 v7, 0
 ; GFX90A-NEXT:    ;;#ASMSTART
-; GFX90A-NEXT:    ; def v[4:6]
+; GFX90A-NEXT:    ; def v[2:4]
 ; GFX90A-NEXT:    ;;#ASMEND
-; GFX90A-NEXT:    v_mov_b32_e32 v0, v6
-; GFX90A-NEXT:    v_mov_b32_e32 v1, v6
-; GFX90A-NEXT:    v_mov_b32_e32 v2, v3
-; GFX90A-NEXT:    v_mov_b32_e32 v3, v6
-; GFX90A-NEXT:    global_store_dwordx4 v7, v[0:3], s[16:17]
+; GFX90A-NEXT:    v_mov_b32_e32 v0, v4
+; GFX90A-NEXT:    v_mov_b32_e32 v5, 0
+; GFX90A-NEXT:    v_pk_mov_b32 v[2:3], v[0:1], v[0:1] op_sel:[1,0]
+; GFX90A-NEXT:    v_mov_b32_e32 v1, v4
+; GFX90A-NEXT:    global_store_dwordx4 v5, v[0:3], s[16:17]
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
 ; GFX90A-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -6745,17 +6733,17 @@ define void @v_shuffle_v4p3_v3p3__5_5_1_5(ptr addrspace(1) inreg %ptr) {
 ; GFX940:       ; %bb.0:
 ; GFX940-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX940-NEXT:    ;;#ASMSTART
-; GFX940-NEXT:    ; def v[2:4]
+; GFX940-NEXT:    ; def v[0:2]
 ; GFX940-NEXT:    ;;#ASMEND
-; GFX940-NEXT:    v_mov_b32_e32 v7, 0
+; GFX940-NEXT:    v_mov_b32_e32 v5, 0
 ; GFX940-NEXT:    ;;#ASMSTART
-; GFX940-NEXT:    ; def v[4:6]
+; GFX940-NEXT:    ; def v[2:4]
 ; GFX940-NEXT:    ;;#ASMEND
-; GFX940-NEXT:    v_mov_b32_e32 v2, v3
-; GFX940-NEXT:    v_mov_b32_e32 v0, v6
-; GFX940-NEXT:    v_mov_b32_e32 v1, v6
-; GFX940-NEXT:    v_mov_b32_e32 v3, v6
-; GFX940-NEXT:    global_store_dwordx4 v7, v[0:3], s[0:1] sc0 sc1
+; GFX940-NEXT:    s_nop 0
+; GFX940-NEXT:    v_mov_b32_e32 v0, v4
+; GFX940-NEXT:    v_pk_mov_b32 v[2:3], v[0:1], v[0:1] op_sel:[1,0]
+; GFX940-NEXT:    v_mov_b32_e32 v1, v4
+; GFX940-NEXT:    global_store_dwordx4 v5, v[0:3], s[0:1] sc0 sc1
 ; GFX940-NEXT:    s_waitcnt vmcnt(0)
 ; GFX940-NEXT:    s_setpc_b64 s[30:31]
   %vec0 = call <3 x ptr addrspace(3)> asm "; def $0", "=v"()
@@ -6893,11 +6881,10 @@ define void @v_shuffle_v4p3_v3p3__5_5_4_5(ptr addrspace(1) inreg %ptr) {
 ; GFX90A-NEXT:    ;;#ASMSTART
 ; GFX90A-NEXT:    ; def v[2:4]
 ; GFX90A-NEXT:    ;;#ASMEND
-; GFX90A-NEXT:    v_mov_b32_e32 v5, 0
 ; GFX90A-NEXT:    v_mov_b32_e32 v0, v4
+; GFX90A-NEXT:    v_mov_b32_e32 v5, 0
+; GFX90A-NEXT:    v_pk_mov_b32 v[2:3], v[2:3], v[0:1] op_sel:[1,0]
 ; GFX90A-NEXT:    v_mov_b32_e32 v1, v4
-; GFX90A-NEXT:    v_mov_b32_e32 v2, v3
-; GFX90A-NEXT:    v_mov_b32_e32 v3, v4
 ; GFX90A-NEXT:    global_store_dwordx4 v5, v[0:3], s[16:17]
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
 ; GFX90A-NEXT:    s_setpc_b64 s[30:31]
@@ -6910,9 +6897,8 @@ define void @v_shuffle_v4p3_v3p3__5_5_4_5(ptr addrspace(1) inreg %ptr) {
 ; GFX940-NEXT:    ;;#ASMEND
 ; GFX940-NEXT:    v_mov_b32_e32 v5, 0
 ; GFX940-NEXT:    v_mov_b32_e32 v0, v4
+; GFX940-NEXT:    v_pk_mov_b32 v[2:3], v[2:3], v[0:1] op_sel:[1,0]
 ; GFX940-NEXT:    v_mov_b32_e32 v1, v4
-; GFX940-NEXT:    v_mov_b32_e32 v2, v3
-; GFX940-NEXT:    v_mov_b32_e32 v3, v4
 ; GFX940-NEXT:    global_store_dwordx4 v5, v[0:3], s[0:1] sc0 sc1
 ; GFX940-NEXT:    s_waitcnt vmcnt(0)
 ; GFX940-NEXT:    s_setpc_b64 s[30:31]
diff --git a/llvm/test/CodeGen/AMDGPU/shufflevector.v4p3.v4p3.ll b/llvm/test/CodeGen/AMDGPU/shufflevector.v4p3.v4p3.ll
index e4c7cebb231bd..9cc1b9fe6cf0e 100644
--- a/llvm/test/CodeGen/AMDGPU/shufflevector.v4p3.v4p3.ll
+++ b/llvm/test/CodeGen/AMDGPU/shufflevector.v4p3.v4p3.ll
@@ -343,13 +343,12 @@ define void @v_shuffle_v4p3_v4p3__7_0_u_u(ptr addrspace(1) inreg %ptr) {
 ; GFX90A-NEXT:    ;;#ASMSTART
 ; GFX90A-NEXT:    ; def v[0:3]
 ; GFX90A-NEXT:    ;;#ASMEND
+; GFX90A-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX90A-NEXT:    ;;#ASMSTART
 ; GFX90A-NEXT:    ; def v[2:5]
 ; GFX90A-NEXT:    ;;#ASMEND
-; GFX90A-NEXT:    v_mov_b32_e32 v6, 0
-; GFX90A-NEXT:    v_mov_b32_e32 v2, v5
-; GFX90A-NEXT:    v_mov_b32_e32 v3, v0
-; GFX90A-NEXT:    global_store_dwordx4 v6, v[2:5], s[16:17]
+; GFX90A-NEXT:    v_pk_mov_b32 v[0:1], v[4:5], v[0:1] op_sel:[1,0]
+; GFX90A-NEXT:    global_store_dwordx4 v6, v[0:3], s[16:17]
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
 ; GFX90A-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -364,9 +363,8 @@ define void @v_shuffle_v4p3_v4p3__7_0_u_u(ptr addrspace(1) inreg %ptr) {
 ; GFX940-NEXT:    ; def v[2:5]
 ; GFX940-NEXT:    ;;#ASMEND
 ; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_mov_b32_e32 v2, v5
-; GFX940-NEXT:    v_mov_b32_e32 v3, v0
-; GFX940-NEXT:    global_store_dwordx4 v6, v[2:5], s[0:1] sc0 sc1
+; GFX940-NEXT:    v_pk_mov_b32 v[0:1], v[4:5], v[0:1] op_sel:[1,0]
+; GFX940-NEXT:    global_store_dwordx4 v6, v[0:3], s[0:1] sc0 sc1
 ; GFX940-NEXT:    s_waitcnt vmcnt(0)
 ; GFX940-NEXT:    s_setpc_b64 s[30:31]
   %vec0 = call <4 x ptr addrspace(3)> asm "; def $0", "=v"()
@@ -456,8 +454,7 @@ define void @v_shuffle_v4p3_v4p3__7_2_u_u(ptr addrspace(1) inreg %ptr) {
 ; GFX90A-NEXT:    ;;#ASMSTART
 ; GFX90A-NEXT:    ; def v[4:7]
 ; GFX90A-NEXT:    ;;#ASMEND
-; GFX90A-NEXT:    v_mov_b32_e32 v0, v7
-; GFX90A-NEXT:    v_mov_b32_e32 v1, v2
+; GFX90A-NEXT:    v_pk_mov_b32 v[0:1], v[6:7], v[2:3] op_sel:[1,0]
 ; GFX90A-NEXT:    global_store_dwordx4 v8, v[0:3], s[16:17]
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
 ; GFX90A-NEXT:    s_setpc_b64 s[30:31]
@@ -472,8 +469,8 @@ define void @v_shuffle_v4p3_v4p3__7_2_u_u(ptr addrspace(1) inreg %ptr) {
 ; GFX940-NEXT:    ;;#ASMSTART
 ; GFX940-NEXT:    ; def v[4:7]
 ; GFX940-NEXT:    ;;#ASMEND
-; GFX940-NEXT:    v_mov_b32_e32 v1, v2
-; GFX940-NEXT:    v_mov_b32_e32 v0, v7
+; GFX940-NEXT:    s_nop 0
+; GFX940-NEXT:    v_pk_mov_b32 v[0:1], v[6:7], v[2:3] op_sel:[1,0]
 ; GFX940-NEXT:    global_store_dwordx4 v8, v[0:3], s[0:1] sc0 sc1
 ; GFX940-NEXT:    s_waitcnt vmcnt(0)
 ; GFX940-NEXT:    s_setpc_b64 s[30:31]
@@ -560,9 +557,8 @@ define void @v_shuffle_v4p3_v4p3__7_4_u_u(ptr addrspace(1) inreg %ptr) {
 ; GFX90A-NEXT:    ; def v[0:3]
 ; GFX90A-NEXT:    ;;#ASMEND
 ; GFX90A-NEXT:    v_mov_b32_e32 v4, 0
-; GFX90A-NEXT:    v_mov_b32_e32 v2, v3
-; GFX90A-NEXT:    v_mov_b32_e32 v3, v0
-; GFX90A-NEXT:    global_store_dwordx4 v4, v[2:5], s[16:17]
+; GFX90A-NEXT:    v_pk_mov_b32 v[0:1], v[2:3], v[0:1] op_sel:[1,0]
+; GFX90A-NEXT:    global_store_dwordx4 v4, v[0:3], s[16:17]
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
 ; GFX90A-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -573,9 +569,8 @@ define void @v_shuffle_v4p3_v4p3__7_4_u_u(ptr addrspace(1) inreg %ptr) {
 ; GFX940-NEXT:    ; def v[0:3]
 ; GFX940-NEXT:    ;;#ASMEND
 ; GFX940-NEXT:    v_mov_b32_e32 v4, 0
-; GFX940-NEXT:    v_mov_b32_e32 v2, v3
-; GFX940-NEXT:    v_mov_b32_e32 v3, v0
-; GFX940-NEXT:    global_store_dwordx4 v4, v[2:5], s[0:1] sc0 sc1
+; GFX940-NEXT:    v_pk_mov_b32 v[0:1], v[2:3], v[0:1] op_sel:[1,0]
+; GFX940-NEXT:    global_store_dwordx4 v4, v[0:3], s[0:1] sc0 sc1
 ; GFX940-NEXT:    s_waitcnt vmcnt(0)
 ; GFX940-NEXT:    s_setpc_b64 s[30:31]
   %vec0 = call <4 x ptr addrspace(3)> asm "; def $0", "=v"()
@@ -649,8 +644,7 @@ define void @v_shuffle_v4p3_v4p3__7_6_u_u(ptr addrspace(1) inreg %ptr) {
 ; GFX90A-NEXT:    ; def v[0:3]
 ; GFX90A-NEXT:    ;;#ASMEND
 ; GFX90A-NEXT:    v_mov_b32_e32 v4, 0
-; GFX90A-NEXT:    v_mov_b32_e32 v0, v3
-; GFX90A-NEXT:    v_mov_b32_e32 v1, v2
+; GFX90A-NEXT:    v_pk_mov_b32 v[0:1], v[2:3], v[2:3] op_sel:[1,0]
 ; GFX90A-NEXT:    global_store_dwordx4 v4, v[0:3], s[16:17]
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
 ; GFX90A-NEXT:    s_setpc_b64 s[30:31]
@@ -662,8 +656,7 @@ define void @v_shuffle_v4p3_v4p3__7_6_u_u(ptr addrspace(1) inreg %ptr) {
 ; GFX940-NEXT:    ; def v[0:3]
 ; GFX940-NEXT:    ;;#ASMEND
 ; GFX940-NEXT:    v_mov_b32_e32 v4, 0
-; GFX940-NEXT:    v_mov_b32_e32 v0, v3
-; GFX940-NEXT:    v_mov_b32_e32 v1, v2
+; GFX940-NEXT:    v_pk_mov_b32 v[0:1], v[2:3], v[2:3] op_sel:[1,0]
 ; GFX940-NEXT:    global_store_dwordx4 v4, v[0:3], s[0:1] sc0 sc1
 ; GFX940-NEXT:    s_waitcnt vmcnt(0)
 ; GFX940-NEXT:    s_setpc_b64 s[30:31]
@@ -1169,11 +1162,10 @@ define void @v_shuffle_v4p3_v4p3__7_7_7_0(ptr addrspace(1) inreg %ptr) {
 ; GFX90A-NEXT:    ; def v[2:5]
 ; GFX90A-NEXT:    ;;#ASMEND
 ; GFX90A-NEXT:    v_mov_b32_e32 v6, 0
-; GFX90A-NEXT:    v_mov_b32_e32 v2, v5
-; GFX90A-NEXT:    v_mov_b32_e32 v3, v5
-; GFX90A-NEXT:    v_mov_b32_e32 v4, v5
-; GFX90A-NEXT:    v_mov_b32_e32 v5, v0
-; GFX90A-NEXT:    global_store_dwordx4 v6, v[2:5], s[16:17]
+; GFX90A-NEXT:    v_pk_mov_b32 v[2:3], v[4:5], v[0:1] op_sel:[1,0]
+; GFX90A-NEXT:    v_mov_b32_e32 v0, v5
+; GFX90A-NEXT:    v_mov_b32_e32 v1, v5
+; GFX90A-NEXT:    global_store_dwordx4 v6, v[0:3], s[16:17]
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
 ; GFX90A-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -1188,11 +1180,10 @@ define void @v_shuffle_v4p3_v4p3__7_7_7_0(ptr addrspace(1) inreg %ptr) {
 ; GFX940-NEXT:    ; def v[2:5]
 ; GFX940-NEXT:    ;;#ASMEND
 ; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_mov_b32_e32 v2, v5
-; GFX940-NEXT:    v_mov_b32_e32 v3, v5
-; GFX940-NEXT:    v_mov_b32_e32 v4, v5
-; GFX940-NEXT:    v_mov_b32_e32 v5, v0
-; GFX940-NEXT:    global_store_dwordx4 v6, v[2:5], s[0:1] sc0 sc1
+; GFX940-NEXT:    v_pk_mov_b32 v[2:3], v[4:5], v[0:1] op_sel:[1,0]
+; GFX940-NEXT:    v_mov_b32_e32 v0, v5
+; GFX940-NEXT:    v_mov_b32_e32 v1, v5
+; GFX940-NEXT:    global_store_dwordx4 v6, v[0:3], s[0:1] sc0 sc1
 ; GFX940-NEXT:    s_waitcnt vmcnt(0)
 ; GFX940-NEXT:    s_setpc_b64 s[30:31]
   %vec0 = call <4 x ptr addrspace(3)> asm "; def $0", "=v"()
@@ -1287,17 +1278,16 @@ define void @v_shuffle_v4p3_v4p3__7_7_7_2(ptr addrspace(1) inreg %ptr) {
 ; GFX90A:       ; %bb.0:
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX90A-NEXT:    ;;#ASMSTART
-; GFX90A-NEXT:    ; def v[4:7]
+; GFX90A-NEXT:    ; def v[0:3]
 ; GFX90A-NEXT:    ;;#ASMEND
 ; GFX90A-NEXT:    v_mov_b32_e32 v8, 0
 ; GFX90A-NEXT:    ;;#ASMSTART
-; GFX90A-NEXT:    ; def v[0:3]
+; GFX90A-NEXT:    ; def v[4:7]
 ; GFX90A-NEXT:    ;;#ASMEND
-; GFX90A-NEXT:    v_mov_b32_e32 v4, v7
-; GFX90A-NEXT:    v_mov_b32_e32 v5, v7
-; GFX90A-NEXT:    v_mov_b32_e32 v6, v7
-; GFX90A-NEXT:    v_mov_b32_e32 v7, v2
-; GFX90A-NEXT:    global_store_dwordx4 v8, v[4:7], s[16:17]
+; GFX90A-NEXT:    v_pk_mov_b32 v[2:3], v[6:7], v[2:3] op_sel:[1,0]
+; GFX90A-NEXT:    v_mov_b32_e32 v0, v7
+; GFX90A-NEXT:    v_mov_b32_e32 v1, v7
+; GFX90A-NEXT:    global_store_dwordx4 v8, v[0:3], s[16:17]
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
 ; GFX90A-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -1305,17 +1295,17 @@ define void @v_shuffle_v4p3_v4p3__7_7_7_2(ptr addrspace(1) inreg %ptr) {
 ; GFX940:       ; %bb.0:
 ; GFX940-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX940-NEXT:    ;;#ASMSTART
-; GFX940-NEXT:    ; def v[4:7]
+; GFX940-NEXT:    ; def v[0:3]
 ; GFX940-NEXT:    ;;#ASMEND
 ; GFX940-NEXT:    v_mov_b32_e32 v8, 0
 ; GFX940-NEXT:    ;;#ASMSTART
-; GFX940-NEXT:    ; def v[0:3]
+; GFX940-NEXT:    ; def v[4:7]
 ; GFX940-NEXT:    ;;#ASMEND
-; GFX940-NEXT:    v_mov_b32_e32 v4, v7
-; GFX940-NEXT:    v_mov_b32_e32 v5, v7
-; GFX940-NEXT:    v_mov_b32_e32 v6, v7
-; GFX940-NEXT:    v_mov_b32_e32 v7, v2
-; GFX940-NEXT:    global_store_dwordx4 v8, v[4:7], s[0:1] sc0 sc1
+; GFX940-NEXT:    s_nop 0
+; GFX940-NEXT:    v_pk_mov_b32 v[2:3], v[6:7], v[2:3] op_sel:[1,0]
+; GFX940-NEXT:    v_mov_b32_e32 v0, v7
+; GFX940-NEXT:    v_mov_b32_e32 v1, v7
+; GFX940-NEXT:    global_store_dwordx4 v8, v[0:3], s[0:1] sc0 sc1
 ; GFX940-NEXT:    s_waitcnt vmcnt(0)
 ; GFX940-NEXT:    s_setpc_b64 s[30:31]
   %vec0 = call <4 x ptr addrspace(3)> asm "; def $0", "=v"()
@@ -1402,31 +1392,26 @@ define void @v_shuffle_v4p3_v4p3__7_7_7_4(ptr addrspace(1) inreg %ptr) {
 ; GFX90A-LABEL: v_shuffle_v4p3_v4p3__7_7_7_4:
 ; GFX90A:       ; %bb.0:
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT:    v_mov_b32_e32 v8, 0
 ; GFX90A-NEXT:    ;;#ASMSTART
 ; GFX90A-NEXT:    ; def v[0:3]
 ; GFX90A-NEXT:    ;;#ASMEND
-; GFX90A-NEXT:    v_mov_b32_e32 v4, v3
-; GFX90A-NEXT:    v_mov_b32_e32 v5, v3
-; GFX90A-NEXT:    v_mov_b32_e32 v6, v3
-; GFX90A-NEXT:    v_mov_b32_e32 v7, v0
-; GFX90A-NEXT:    global_store_dwordx4 v8, v[4:7], s[16:17]
+; GFX90A-NEXT:    v_mov_b32_e32 v6, 0
+; GFX90A-NEXT:    v_pk_mov_b32 v[4:5], v[2:3], v[0:1] op_sel:[1,0]
+; GFX90A-NEXT:    v_mov_b32_e32 v2, v3
+; GFX90A-NEXT:    global_store_dwordx4 v6, v[2:5], s[16:17]
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
 ; GFX90A-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: v_shuffle_v4p3_v4p3__7_7_7_4:
 ; GFX940:       ; %bb.0:
 ; GFX940-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX940-NEXT:    v_mov_b32_e32 v8, 0
 ; GFX940-NEXT:    ;;#ASMSTART
 ; GFX940-NEXT:    ; def v[0:3]
 ; GFX940-NEXT:    ;;#ASMEND
-; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_mov_b32_e32 v4, v3
-; GFX940-NEXT:    v_mov_b32_e32 v5, v3
-; GFX940-NEXT:    v_mov_b32_e32 v6, v3
-; GFX940-NEXT:    v_mov_b32_e32 v7, v0
-; GFX940-NEXT:    global_store_dwordx4 v8, v[4:7], s[0:1] sc0 sc1
+; GFX940-NEXT:    v_mov_b32_e32 v6, 0
+; GFX940-NEXT:    v_pk_mov_b32 v[4:5], v[2:3], v[0:1] op_sel:[1,0]
+; GFX940-NEXT:    v_mov_b32_e32 v2, v3
+; GFX940-NEXT:    global_store_dwordx4 v6, v[2:5], s[0:1] sc0 sc1
 ; GFX940-NEXT:    s_waitcnt vmcnt(0)
 ; GFX940-NEXT:    s_setpc_b64 s[30:31]
   %vec0 = call <4 x ptr addrspace(3)> asm "; def $0", "=v"()
@@ -1508,31 +1493,26 @@ define void @v_shuffle_v4p3_v4p3__7_7_7_6(ptr addrspace(1) inreg %ptr) {
 ; GFX90A-LABEL: v_shuffle_v4p3_v4p3__7_7_7_6:
 ; GFX90A:       ; %bb.0:
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT:    v_mov_b32_e32 v8, 0
 ; GFX90A-NEXT:    ;;#ASMSTART
 ; GFX90A-NEXT:    ; def v[0:3]
 ; GFX90A-NEXT:    ;;#ASMEND
-; GFX90A-NEXT:    v_mov_b32_e32 v4, v3
-; GFX90A-NEXT:    v_mov_b32_e32 v5, v3
-; GFX90A-NEXT:    v_mov_b32_e32 v6, v3
-; GFX90A-NEXT:    v_mov_b32_e32 v7, v2
-; GFX90A-NEXT:    global_store_dwordx4 v8, v[4:7], s[16:17]
+; GFX90A-NEXT:    v_mov_b32_e32 v6, 0
+; GFX90A-NEXT:    v_pk_mov_b32 v[4:5], v[2:3], v[2:3] op_sel:[1,0]
+; GFX90A-NEXT:    v_mov_b32_e32 v2, v3
+; GFX90A-NEXT:    global_store_dwordx4 v6, v[2:5], s[16:17]
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
 ; GFX90A-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: v_shuffle_v4p3_v4p3__7_7_7_6:
 ; GFX940:       ; %bb.0:
 ; GFX940-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX940-NEXT:    v_mov_b32_e32 v8, 0
 ; GFX940-NEXT:    ;;#ASMSTART
 ; GFX940-NEXT:    ; def v[0:3]
 ; GFX940-NEXT:    ;;#ASMEND
-; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_mov_b32_e32 v4, v3
-; GFX940-NEXT:    v_mov_b32_e32 v5, v3
-; GFX940-NEXT:    v_mov_b32_e32 v6, v3
-; GFX940-NEXT:    v_mov_b32_e32 v7, v2
-; GFX940-NEXT:    global_store_dwordx4 v8, v[4:7], s[0:1] sc0 sc1
+; GFX940-NEXT:    v_mov_b32_e32 v6, 0
+; GFX940-NEXT:    v_pk_mov_b32 v[4:5], v[2:3], v[2:3] op_sel:[1,0]
+; GFX940-NEXT:    v_mov_b32_e32 v2, v3
+; GFX940-NEXT:    global_store_dwordx4 v6, v[2:5], s[0:1] sc0 sc1
 ; GFX940-NEXT:    s_waitcnt vmcnt(0)
 ; GFX940-NEXT:    s_setpc_b64 s[30:31]
   %vec0 = call <4 x ptr addrspace(3)> asm "; def $0", "=v"()
@@ -1709,8 +1689,7 @@ define void @v_shuffle_v4p3_v4p3__1_0_0_0(ptr addrspace(1) inreg %ptr) {
 ; GFX90A-NEXT:    ; def v[0:3]
 ; GFX90A-NEXT:    ;;#ASMEND
 ; GFX90A-NEXT:    v_mov_b32_e32 v6, 0
-; GFX90A-NEXT:    v_mov_b32_e32 v2, v1
-; GFX90A-NEXT:    v_mov_b32_e32 v3, v0
+; GFX90A-NEXT:    v_pk_mov_b32 v[2:3], v[0:1], v[0:1] op_sel:[1,0]
 ; GFX90A-NEXT:    v_mov_b32_e32 v4, v0
 ; GFX90A-NEXT:    v_mov_b32_e32 v5, v0
 ; GFX90A-NEXT:    global_store_dwordx4 v6, v[2:5], s[16:17]
@@ -1724,8 +1703,7 @@ define void @v_shuffle_v4p3_v4p3__1_0_0_0(ptr addrspace(1) inreg %ptr) {
 ; GFX940-NEXT:    ; def v[0:3]
 ; GFX940-NEXT:    ;;#ASMEND
 ; GFX940-NEXT:    v_mov_b32_e32 v6, 0
-; GFX940-NEXT:    v_mov_b32_e32 v2, v1
-; GFX940-NEXT:    v_mov_b32_e32 v3, v0
+; GFX940-NEXT:    v_pk_mov_b32 v[2:3], v[0:1], v[0:1] op_sel:[1,0]
 ; GFX940-NEXT:    v_mov_b32_e32 v4, v0
 ; GFX940-NEXT:    v_mov_b32_e32 v5, v0
 ; GFX940-NEXT:    global_store_dwordx4 v6, v[2:5], s[0:1] sc0 sc1
@@ -1809,8 +1787,7 @@ define void @v_shuffle_v4p3_v4p3__3_0_0_0(ptr addrspace(1) inreg %ptr) {
 ; GFX90A-NEXT:    ; def v[0:3]
 ; GFX90A-NEXT:    ;;#ASMEND
 ; GFX90A-NEXT:    v_mov_b32_e32 v6, 0
-; GFX90A-NEXT:    v_mov_b32_e32 v2, v3
-; GFX90A-NEXT:    v_mov_b32_e32 v3, v0
+; GFX90A-NEXT:    v_pk_mov_b32 v[2:3], v[2:3], v[0:1] op_sel:[1,0]
 ; GFX90A-NEXT:    v_mov_b32_e32 v4, v0
 ; GFX90A-NEXT:    v_mov_b32_e32 v5, v0
 ; GFX90A-NEXT:    global_store_dwordx4 v6, v[2:5], s[16:17]
@@ -1824,8 +1801,7 @@ define void @v_shuffle_v4p3_v4p3__3_0_0_0(ptr addrspace(1) inreg %ptr) {
 ; GFX940-NEXT:    ; def v[0:3]
 ; GFX940-NEXT:    ;;#ASMEND
 ; GFX940-NEXT:    v_mov_b32_e32 v6, 0
-; GFX940-NEXT:    v_mov_b32_e32 v2, v3
-; GFX940-NEXT:    v_mov_b32_e32 v3, v0
+; GFX940-NEXT:    v_pk_mov_b32 v[2:3], v[2:3], v[0:1] op_sel:[1,0]
 ; GFX940-NEXT:    v_mov_b32_e32 v4, v0
 ; GFX940-NEXT:    v_mov_b32_e32 v5, v0
 ; GFX940-NEXT:    global_store_dwordx4 v6, v[2:5], s[0:1] sc0 sc1
@@ -1914,8 +1890,7 @@ define void @v_shuffle_v4p3_v4p3__5_0_0_0(ptr addrspace(1) inreg %ptr) {
 ; GFX90A-NEXT:    ; def v[2:5]
 ; GFX90A-NEXT:    ;;#ASMEND
 ; GFX90A-NEXT:    v_mov_b32_e32 v6, 0
-; GFX90A-NEXT:    v_mov_b32_e32 v2, v3
-; GFX90A-NEXT:    v_mov_b32_e32 v3, v0
+; GFX90A-NEXT:    v_pk_mov_b32 v[2:3], v[2:3], v[0:1] op_sel:[1,0]
 ; GFX90A-NEXT:    v_mov_b32_e32 v4, v0
 ; GFX90A-NEXT:    v_mov_b32_e32 v5, v0
 ; GFX90A-NEXT:    global_store_dwordx4 v6, v[2:5], s[16:17]
@@ -1933,8 +1908,7 @@ define void @v_shuffle_v4p3_v4p3__5_0_0_0(ptr addrspace(1) inreg %ptr) {
 ; GFX940-NEXT:    ; def v[2:5]
 ; GFX940-NEXT:    ;;#ASMEND
 ; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_mov_b32_e32 v2, v3
-; GFX940-NEXT:    v_mov_b32_e32 v3, v0
+; GFX940-NEXT:    v_pk_mov_b32 v[2:3], v[2:3], v[0:1] op_sel:[1,0]
 ; GFX940-NEXT:    v_mov_b32_e32 v4, v0
 ; GFX940-NEXT:    v_mov_b32_e32 v5, v0
 ; GFX940-NEXT:    global_store_dwordx4 v6, v[2:5], s[0:1] sc0 sc1
@@ -2038,8 +2012,7 @@ define void @v_shuffle_v4p3_v4p3__7_0_0_0(ptr addrspace(1) inreg %ptr) {
 ; GFX90A-NEXT:    ; def v[2:5]
 ; GFX90A-NEXT:    ;;#ASMEND
 ; GFX90A-NEXT:    v_mov_b32_e32 v6, 0
-; GFX90A-NEXT:    v_mov_b32_e32 v2, v5
-; GFX90A-NEXT:    v_mov_b32_e32 v3, v0
+; GFX90A-NEXT:    v_pk_mov_b32 v[2:3], v[4:5], v[0:1] op_sel:[1,0]
 ; GFX90A-NEXT:    v_mov_b32_e32 v4, v0
 ; GFX90A-NEXT:    v_mov_b32_e32 v5, v0
 ; GFX90A-NEXT:    global_store_dwordx4 v6, v[2:5], s[16:17]
@@ -2057,8 +2030,7 @@ define void @v_shuffle_v4p3_v4p3__7_0_0_0(ptr addrspace(1) inreg %ptr) {
 ; GFX940-NEXT:    ; def v[2:5]
 ; GFX940-NEXT:    ;;#ASMEND
 ; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_mov_b32_e32 v2, v5
-; GFX940-NEXT:    v_mov_b32_e32 v3, v0
+; GFX940-NEXT:    v_pk_mov_b32 v[2:3], v[4:5], v[0:1] op_sel:[1,0]
 ; GFX940-NEXT:    v_mov_b32_e32 v4, v0
 ; GFX940-NEXT:    v_mov_b32_e32 v5, v0
 ; GFX940-NEXT:    global_store_dwordx4 v6, v[2:5], s[0:1] sc0 sc1
@@ -2214,17 +2186,16 @@ define void @v_shuffle_v4p3_v4p3__7_2_0_0(ptr addrspace(1) inreg %ptr) {
 ; GFX90A:       ; %bb.0:
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX90A-NEXT:    ;;#ASMSTART
-; GFX90A-NEXT:    ; def v[4:7]
+; GFX90A-NEXT:    ; def v[0:3]
 ; GFX90A-NEXT:    ;;#ASMEND
-; GFX90A-NEXT:    v_mov_b32_e32 v8, 0
 ; GFX90A-NEXT:    ;;#ASMSTART
-; GFX90A-NEXT:    ; def v[0:3]
+; GFX90A-NEXT:    ; def v[4:7]
 ; GFX90A-NEXT:    ;;#ASMEND
-; GFX90A-NEXT:    v_mov_b32_e32 v4, v7
-; GFX90A-NEXT:    v_mov_b32_e32 v5, v2
-; GFX90A-NEXT:    v_mov_b32_e32 v6, v0
-; GFX90A-NEXT:    v_mov_b32_e32 v7, v0
-; GFX90A-NEXT:    global_store_dwordx4 v8, v[4:7], s[16:17]
+; GFX90A-NEXT:    v_mov_b32_e32 v8, 0
+; GFX90A-NEXT:    v_pk_mov_b32 v[2:3], v[6:7], v[2:3] op_sel:[1,0]
+; GFX90A-NEXT:    v_mov_b32_e32 v4, v0
+; GFX90A-NEXT:    v_mov_b32_e32 v5, v0
+; GFX90A-NEXT:    global_store_dwordx4 v8, v[2:5], s[16:17]
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
 ; GFX90A-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -2232,17 +2203,16 @@ define void @v_shuffle_v4p3_v4p3__7_2_0_0(ptr addrspace(1) inreg %ptr) {
 ; GFX940:       ; %bb.0:
 ; GFX940-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX940-NEXT:    ;;#ASMSTART
-; GFX940-NEXT:    ; def v[4:7]
+; GFX940-NEXT:    ; def v[0:3]
 ; GFX940-NEXT:    ;;#ASMEND
-; GFX940-NEXT:    v_mov_b32_e32 v8, 0
 ; GFX940-NEXT:    ;;#ASMSTART
-; GFX940-NEXT:    ; def v[0:3]
+; GFX940-NEXT:    ; def v[4:7]
 ; GFX940-NEXT:    ;;#ASMEND
-; GFX940-NEXT:    v_mov_b32_e32 v4, v7
-; GFX940-NEXT:    v_mov_b32_e32 v5, v2
-; GFX940-NEXT:    v_mov_b32_e32 v6, v0
-; GFX940-NEXT:    v_mov_b32_e32 v7, v0
-; GFX940-NEXT:    global_store_dwordx4 v8, v[4:7], s[0:1] sc0 sc1
+; GFX940-NEXT:    v_mov_b32_e32 v8, 0
+; GFX940-NEXT:    v_pk_mov_b32 v[2:3], v[6:7], v[2:3] op_sel:[1,0]
+; GFX940-NEXT:    v_mov_b32_e32 v4, v0
+; GFX940-NEXT:    v_mov_b32_e32 v5, v0
+; GFX940-NEXT:    global_store_dwordx4 v8, v[2:5], s[0:1] sc0 sc1
 ; GFX940-NEXT:    s_waitcnt vmcnt(0)
 ; GFX940-NEXT:    s_setpc_b64 s[30:31]
   %vec0 = call <4 x ptr addrspace(3)> asm "; def $0", "=v"()
@@ -2339,12 +2309,11 @@ define void @v_shuffle_v4p3_v4p3__7_4_0_0(ptr addrspace(1) inreg %ptr) {
 ; GFX90A-NEXT:    ;;#ASMSTART
 ; GFX90A-NEXT:    ; def v[2:5]
 ; GFX90A-NEXT:    ;;#ASMEND
-; GFX90A-NEXT:    v_mov_b32_e32 v8, 0
-; GFX90A-NEXT:    v_mov_b32_e32 v4, v5
-; GFX90A-NEXT:    v_mov_b32_e32 v5, v2
-; GFX90A-NEXT:    v_mov_b32_e32 v6, v0
-; GFX90A-NEXT:    v_mov_b32_e32 v7, v0
-; GFX90A-NEXT:    global_store_dwordx4 v8, v[4:7], s[16:17]
+; GFX90A-NEXT:    v_mov_b32_e32 v6, 0
+; GFX90A-NEXT:    v_pk_mov_b32 v[2:3], v[4:5], v[2:3] op_sel:[1,0]
+; GFX90A-NEXT:    v_mov_b32_e32 v4, v0
+; GFX90A-NEXT:    v_mov_b32_e32 v5, v0
+; GFX90A-NEXT:    global_store_dwordx4 v6, v[2:5], s[16:17]
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
 ; GFX90A-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -2354,15 +2323,15 @@ define void @v_shuffle_v4p3_v4p3__7_4_0_0(ptr addrspace(1) inreg %ptr) {
 ; GFX940-NEXT:    ;;#ASMSTART
 ; GFX940-NEXT:    ; def v[0:3]
 ; GFX940-NEXT:    ;;#ASMEND
-; GFX940-NEXT:    v_mov_b32_e32 v8, 0
+; GFX940-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX940-NEXT:    ;;#ASMSTART
 ; GFX940-NEXT:    ; def v[2:5]
 ; GFX940-NEXT:    ;;#ASMEND
-; GFX940-NEXT:    v_mov_b32_e32 v6, v0
-; GFX940-NEXT:    v_mov_b32_e32 v4, v5
-; GFX940-NEXT:    v_mov_b32_e32 v5, v2
-; GFX940-NEXT:    v_mov_b32_e32 v7, v0
-; GFX940-NEXT:    global_store_dwordx4 v8, v[4:7], s[0:1] sc0 sc1
+; GFX940-NEXT:    s_nop 0
+; GFX940-NEXT:    v_pk_mov_b32 v[2:3], v[4:5], v[2:3] op_sel:[1,0]
+; GFX940-NEXT:    v_mov_b32_e32 v4, v0
+; GFX940-NEXT:    v_mov_b32_e32 v5, v0
+; GFX940-NEXT:    global_store_dwordx4 v6, v[2:5], s[0:1] sc0 sc1
 ; GFX940-NEXT:    s_waitcnt vmcnt(0)
 ; GFX940-NEXT:    s_setpc_b64 s[30:31]
   %vec0 = call <4 x ptr addrspace(3)> asm "; def $0", "=v"()
@@ -2460,8 +2429,7 @@ define void @v_shuffle_v4p3_v4p3__7_6_0_0(ptr addrspace(1) inreg %ptr) {
 ; GFX90A-NEXT:    ; def v[2:5]
 ; GFX90A-NEXT:    ;;#ASMEND
 ; GFX90A-NEXT:    v_mov_b32_e32 v6, 0
-; GFX90A-NEXT:    v_mov_b32_e32 v2, v5
-; GFX90A-NEXT:    v_mov_b32_e32 v3, v4
+; GFX90A-NEXT:    v_pk_mov_b32 v[2:3], v[4:5], v[4:5] op_sel:[1,0]
 ; GFX90A-NEXT:    v_mov_b32_e32 v4, v0
 ; GFX90A-NEXT:    v_mov_b32_e32 v5, v0
 ; GFX90A-NEXT:    global_store_dwordx4 v6, v[2:5], s[16:17]
@@ -2479,8 +2447,7 @@ define void @v_shuffle_v4p3_v4p3__7_6_0_0(ptr addrspace(1) inreg %ptr) {
 ; GFX940-NEXT:    ; def v[2:5]
 ; GFX940-NEXT:    ;;#ASMEND
 ; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_mov_b32_e32 v2, v5
-; GFX940-NEXT:    v_mov_b32_e32 v3, v4
+; GFX940-NEXT:    v_pk_mov_b32 v[2:3], v[4:5], v[4:5] op_sel:[1,0]
 ; GFX940-NEXT:    v_mov_b32_e32 v4, v0
 ; GFX940-NEXT:    v_mov_b32_e32 v5, v0
 ; GFX940-NEXT:    global_store_dwordx4 v6, v[2:5], s[0:1] sc0 sc1
@@ -2643,11 +2610,10 @@ define void @v_shuffle_v4p3_v4p3__7_7_1_0(ptr addrspace(1) inreg %ptr) {
 ; GFX90A-NEXT:    ; def v[2:5]
 ; GFX90A-NEXT:    ;;#ASMEND
 ; GFX90A-NEXT:    v_mov_b32_e32 v6, 0
-; GFX90A-NEXT:    v_mov_b32_e32 v2, v5
-; GFX90A-NEXT:    v_mov_b32_e32 v3, v5
-; GFX90A-NEXT:    v_mov_b32_e32 v4, v1
-; GFX90A-NEXT:    v_mov_b32_e32 v5, v0
-; GFX90A-NEXT:    global_store_dwordx4 v6, v[2:5], s[16:17]
+; GFX90A-NEXT:    v_pk_mov_b32 v[2:3], v[0:1], v[0:1] op_sel:[1,0]
+; GFX90A-NEXT:    v_mov_b32_e32 v0, v5
+; GFX90A-NEXT:    v_mov_b32_e32 v1, v5
+; GFX90A-NEXT:    global_store_dwordx4 v6, v[0:3], s[16:17]
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
 ; GFX90A-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -2662,11 +2628,10 @@ define void @v_shuffle_v4p3_v4p3__7_7_1_0(ptr addrspace(1) inreg %ptr) {
 ; GFX940-NEXT:    ; def v[2:5]
 ; GFX940-NEXT:    ;;#ASMEND
 ; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_mov_b32_e32 v2, v5
-; GFX940-NEXT:    v_mov_b32_e32 v3, v5
-; GFX940-NEXT:    v_mov_b32_e32 v4, v1
-; GFX940-NEXT:    v_mov_b32_e32 v5, v0
-; GFX940-NEXT:    global_store_dwordx4 v6, v[2:5], s[0:1] sc0 sc1
+; GFX940-NEXT:    v_pk_mov_b32 v[2:3], v[0:1], v[0:1] op_sel:[1,0]
+; GFX940-NEXT:    v_mov_b32_e32 v0, v5
+; GFX940-NEXT:    v_mov_b32_e32 v1, v5
+; GFX940-NEXT:    global_store_dwordx4 v6, v[0:3], s[0:1] sc0 sc1
 ; GFX940-NEXT:    s_waitcnt vmcnt(0)
 ; GFX940-NEXT:    s_setpc_b64 s[30:31]
   %vec0 = call <4 x ptr addrspace(3)> asm "; def $0", "=v"()
@@ -2759,17 +2724,16 @@ define void @v_shuffle_v4p3_v4p3__7_7_3_0(ptr addrspace(1) inreg %ptr) {
 ; GFX90A:       ; %bb.0:
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX90A-NEXT:    ;;#ASMSTART
-; GFX90A-NEXT:    ; def v[4:7]
+; GFX90A-NEXT:    ; def v[0:3]
 ; GFX90A-NEXT:    ;;#ASMEND
 ; GFX90A-NEXT:    v_mov_b32_e32 v8, 0
 ; GFX90A-NEXT:    ;;#ASMSTART
-; GFX90A-NEXT:    ; def v[0:3]
+; GFX90A-NEXT:    ; def v[4:7]
 ; GFX90A-NEXT:    ;;#ASMEND
-; GFX90A-NEXT:    v_mov_b32_e32 v4, v7
-; GFX90A-NEXT:    v_mov_b32_e32 v5, v7
-; GFX90A-NEXT:    v_mov_b32_e32 v6, v3
-; GFX90A-NEXT:    v_mov_b32_e32 v7, v0
-; GFX90A-NEXT:    global_store_dwordx4 v8, v[4:7], s[16:17]
+; GFX90A-NEXT:    v_pk_mov_b32 v[2:3], v[2:3], v[0:1] op_sel:[1,0]
+; GFX90A-NEXT:    v_mov_b32_e32 v0, v7
+; GFX90A-NEXT:    v_mov_b32_e32 v1, v7
+; GFX90A-NEXT:    global_store_dwordx4 v8, v[0:3], s[16:17]
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
 ; GFX90A-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -2777,17 +2741,16 @@ define void @v_shuffle_v4p3_v4p3__7_7_3_0(ptr addrspace(1) inreg %ptr) {
 ; GFX940:       ; %bb.0:
 ; GFX940-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX940-NEXT:    ;;#ASMSTART
-; GFX940-NEXT:    ; def v[4:7]
+; GFX940-NEXT:    ; def v[0:3]
 ; GFX940-NEXT:    ;;#ASMEND
 ; GFX940-NEXT:    v_mov_b32_e32 v8, 0
 ; GFX940-NEXT:    ;;#ASMSTART
-; GFX940-NEXT:    ; def v[0:3]
+; GFX940-NEXT:    ; def v[4:7]
 ; GFX940-NEXT:    ;;#ASMEND
-; GFX940-NEXT:    v_mov_b32_e32 v4, v7
-; GFX940-NEXT:    v_mov_b32_e32 v5, v7
-; GFX940-NEXT:    v_mov_b32_e32 v6, v3
-; GFX940-NEXT:    v_mov_b32_e32 v7, v0
-; GFX940-NEXT:    global_store_dwordx4 v8, v[4:7], s[0:1] sc0 sc1
+; GFX940-NEXT:    v_pk_mov_b32 v[2:3], v[2:3], v[0:1] op_sel:[1,0]
+; GFX940-NEXT:    v_mov_b32_e32 v0, v7
+; GFX940-NEXT:    v_mov_b32_e32 v1, v7
+; GFX940-NEXT:    global_store_dwordx4 v8, v[0:3], s[0:1] sc0 sc1
 ; GFX940-NEXT:    s_waitcnt vmcnt(0)
 ; GFX940-NEXT:    s_setpc_b64 s[30:31]
   %vec0 = call <4 x ptr addrspace(3)> asm "; def $0", "=v"()
@@ -2883,11 +2846,11 @@ define void @v_shuffle_v4p3_v4p3__7_7_5_0(ptr addrspace(1) inreg %ptr) {
 ; GFX90A-NEXT:    ;;#ASMSTART
 ; GFX90A-NEXT:    ; def v[2:5]
 ; GFX90A-NEXT:    ;;#ASMEND
-; GFX90A-NEXT:    v_mov_b32_e32 v8, 0
-; GFX90A-NEXT:    v_mov_b32_e32 v4, v5
-; GFX90A-NEXT:    v_mov_b32_e32 v6, v3
-; GFX90A-NEXT:    v_mov_b32_e32 v7, v0
-; GFX90A-NEXT:    global_store_dwordx4 v8, v[4:7], s[16:17]
+; GFX90A-NEXT:    v_mov_b32_e32 v6, 0
+; GFX90A-NEXT:    v_pk_mov_b32 v[2:3], v[2:3], v[0:1] op_sel:[1,0]
+; GFX90A-NEXT:    v_mov_b32_e32 v0, v5
+; GFX90A-NEXT:    v_mov_b32_e32 v1, v5
+; GFX90A-NEXT:    global_store_dwordx4 v6, v[0:3], s[16:17]
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
 ; GFX90A-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -2897,14 +2860,15 @@ define void @v_shuffle_v4p3_v4p3__7_7_5_0(ptr addrspace(1) inreg %ptr) {
 ; GFX940-NEXT:    ;;#ASMSTART
 ; GFX940-NEXT:    ; def v[0:3]
 ; GFX940-NEXT:    ;;#ASMEND
-; GFX940-NEXT:    v_mov_b32_e32 v8, 0
+; GFX940-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX940-NEXT:    ;;#ASMSTART
 ; GFX940-NEXT:    ; def v[2:5]
 ; GFX940-NEXT:    ;;#ASMEND
-; GFX940-NEXT:    v_mov_b32_e32 v7, v0
-; GFX940-NEXT:    v_mov_b32_e32 v4, v5
-; GFX940-NEXT:    v_mov_b32_e32 v6, v3
-; GFX940-NEXT:    global_store_dwordx4 v8, v[4:7], s[0:1] sc0 sc1
+; GFX940-NEXT:    s_nop 0
+; GFX940-NEXT:    v_pk_mov_b32 v[2:3], v[2:3], v[0:1] op_sel:[1,0]
+; GFX940-NEXT:    v_mov_b32_e32 v0, v5
+; GFX940-NEXT:    v_mov_b32_e32 v1, v5
+; GFX940-NEXT:    global_store_dwordx4 v6, v[0:3], s[0:1] sc0 sc1
 ; GFX940-NEXT:    s_waitcnt vmcnt(0)
 ; GFX940-NEXT:    s_setpc_b64 s[30:31]
   %vec0 = call <4 x ptr addrspace(3)> asm "; def $0", "=v"()
@@ -3517,8 +3481,7 @@ define void @v_shuffle_v4p3_v4p3__7_0_1_1(ptr addrspace(1) inreg %ptr) {
 ; GFX90A-NEXT:    ; def v[2:5]
 ; GFX90A-NEXT:    ;;#ASMEND
 ; GFX90A-NEXT:    v_mov_b32_e32 v6, 0
-; GFX90A-NEXT:    v_mov_b32_e32 v2, v5
-; GFX90A-NEXT:    v_mov_b32_e32 v3, v0
+; GFX90A-NEXT:    v_pk_mov_b32 v[2:3], v[4:5], v[0:1] op_sel:[1,0]
 ; GFX90A-NEXT:    v_mov_b32_e32 v4, v1
 ; GFX90A-NEXT:    v_mov_b32_e32 v5, v1
 ; GFX90A-NEXT:    global_store_dwordx4 v6, v[2:5], s[16:17]
@@ -3536,8 +3499,7 @@ define void @v_shuffle_v4p3_v4p3__7_0_1_1(ptr addrspace(1) inreg %ptr) {
 ; GFX940-NEXT:    ; def v[2:5]
 ; GFX940-NEXT:    ;;#ASMEND
 ; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_mov_b32_e32 v2, v5
-; GFX940-NEXT:    v_mov_b32_e32 v3, v0
+; GFX940-NEXT:    v_pk_mov_b32 v[2:3], v[4:5], v[0:1] op_sel:[1,0]
 ; GFX940-NEXT:    v_mov_b32_e32 v4, v1
 ; GFX940-NEXT:    v_mov_b32_e32 v5, v1
 ; GFX940-NEXT:    global_store_dwordx4 v6, v[2:5], s[0:1] sc0 sc1
@@ -3573,17 +3535,16 @@ define void @v_shuffle_v4p3_v4p3__7_2_1_1(ptr addrspace(1) inreg %ptr) {
 ; GFX90A:       ; %bb.0:
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX90A-NEXT:    ;;#ASMSTART
-; GFX90A-NEXT:    ; def v[4:7]
+; GFX90A-NEXT:    ; def v[0:3]
 ; GFX90A-NEXT:    ;;#ASMEND
-; GFX90A-NEXT:    v_mov_b32_e32 v8, 0
 ; GFX90A-NEXT:    ;;#ASMSTART
-; GFX90A-NEXT:    ; def v[0:3]
+; GFX90A-NEXT:    ; def v[4:7]
 ; GFX90A-NEXT:    ;;#ASMEND
-; GFX90A-NEXT:    v_mov_b32_e32 v4, v7
-; GFX90A-NEXT:    v_mov_b32_e32 v5, v2
-; GFX90A-NEXT:    v_mov_b32_e32 v6, v1
-; GFX90A-NEXT:    v_mov_b32_e32 v7, v1
-; GFX90A-NEXT:    global_store_dwordx4 v8, v[4:7], s[16:17]
+; GFX90A-NEXT:    v_mov_b32_e32 v8, 0
+; GFX90A-NEXT:    v_pk_mov_b32 v[2:3], v[6:7], v[2:3] op_sel:[1,0]
+; GFX90A-NEXT:    v_mov_b32_e32 v4, v1
+; GFX90A-NEXT:    v_mov_b32_e32 v5, v1
+; GFX90A-NEXT:    global_store_dwordx4 v8, v[2:5], s[16:17]
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
 ; GFX90A-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -3591,17 +3552,16 @@ define void @v_shuffle_v4p3_v4p3__7_2_1_1(ptr addrspace(1) inreg %ptr) {
 ; GFX940:       ; %bb.0:
 ; GFX940-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX940-NEXT:    ;;#ASMSTART
-; GFX940-NEXT:    ; def v[4:7]
+; GFX940-NEXT:    ; def v[0:3]
 ; GFX940-NEXT:    ;;#ASMEND
-; GFX940-NEXT:    v_mov_b32_e32 v8, 0
 ; GFX940-NEXT:    ;;#ASMSTART
-; GFX940-NEXT:    ; def v[0:3]
+; GFX940-NEXT:    ; def v[4:7]
 ; GFX940-NEXT:    ;;#ASMEND
-; GFX940-NEXT:    v_mov_b32_e32 v4, v7
-; GFX940-NEXT:    v_mov_b32_e32 v5, v2
-; GFX940-NEXT:    v_mov_b32_e32 v6, v1
-; GFX940-NEXT:    v_mov_b32_e32 v7, v1
-; GFX940-NEXT:    global_store_dwordx4 v8, v[4:7], s[0:1] sc0 sc1
+; GFX940-NEXT:    v_mov_b32_e32 v8, 0
+; GFX940-NEXT:    v_pk_mov_b32 v[2:3], v[6:7], v[2:3] op_sel:[1,0]
+; GFX940-NEXT:    v_mov_b32_e32 v4, v1
+; GFX940-NEXT:    v_mov_b32_e32 v5, v1
+; GFX940-NEXT:    global_store_dwordx4 v8, v[2:5], s[0:1] sc0 sc1
 ; GFX940-NEXT:    s_waitcnt vmcnt(0)
 ; GFX940-NEXT:    s_setpc_b64 s[30:31]
   %vec0 = call <4 x ptr addrspace(3)> asm "; def $0", "=v"()
@@ -3697,12 +3657,11 @@ define void @v_shuffle_v4p3_v4p3__7_4_1_1(ptr addrspace(1) inreg %ptr) {
 ; GFX90A-NEXT:    ;;#ASMSTART
 ; GFX90A-NEXT:    ; def v[2:5]
 ; GFX90A-NEXT:    ;;#ASMEND
-; GFX90A-NEXT:    v_mov_b32_e32 v8, 0
-; GFX90A-NEXT:    v_mov_b32_e32 v4, v5
-; GFX90A-NEXT:    v_mov_b32_e32 v5, v2
-; GFX90A-NEXT:    v_mov_b32_e32 v6, v1
-; GFX90A-NEXT:    v_mov_b32_e32 v7, v1
-; GFX90A-NEXT:    global_store_dwordx4 v8, v[4:7], s[16:17]
+; GFX90A-NEXT:    v_mov_b32_e32 v6, 0
+; GFX90A-NEXT:    v_pk_mov_b32 v[2:3], v[4:5], v[2:3] op_sel:[1,0]
+; GFX90A-NEXT:    v_mov_b32_e32 v4, v1
+; GFX90A-NEXT:    v_mov_b32_e32 v5, v1
+; GFX90A-NEXT:    global_store_dwordx4 v6, v[2:5], s[16:17]
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
 ; GFX90A-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -3712,15 +3671,15 @@ define void @v_shuffle_v4p3_v4p3__7_4_1_1(ptr addrspace(1) inreg %ptr) {
 ; GFX940-NEXT:    ;;#ASMSTART
 ; GFX940-NEXT:    ; def v[0:3]
 ; GFX940-NEXT:    ;;#ASMEND
-; GFX940-NEXT:    v_mov_b32_e32 v8, 0
+; GFX940-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX940-NEXT:    ;;#ASMSTART
 ; GFX940-NEXT:    ; def v[2:5]
 ; GFX940-NEXT:    ;;#ASMEND
-; GFX940-NEXT:    v_mov_b32_e32 v6, v1
-; GFX940-NEXT:    v_mov_b32_e32 v4, v5
-; GFX940-NEXT:    v_mov_b32_e32 v5, v2
-; GFX940-NEXT:    v_mov_b32_e32 v7, v1
-; GFX940-NEXT:    global_store_dwordx4 v8, v[4:7], s[0:1] sc0 sc1
+; GFX940-NEXT:    s_nop 0
+; GFX940-NEXT:    v_pk_mov_b32 v[2:3], v[4:5], v[2:3] op_sel:[1,0]
+; GFX940-NEXT:    v_mov_b32_e32 v4, v1
+; GFX940-NEXT:    v_mov_b32_e32 v5, v1
+; GFX940-NEXT:    global_store_dwordx4 v6, v[2:5], s[0:1] sc0 sc1
 ; GFX940-NEXT:    s_waitcnt vmcnt(0)
 ; GFX940-NEXT:    s_setpc_b64 s[30:31]
   %vec0 = call <4 x ptr addrspace(3)> asm "; def $0", "=v"()
@@ -3818,8 +3777,7 @@ define void @v_shuffle_v4p3_v4p3__7_6_1_1(ptr addrspace(1) inreg %ptr) {
 ; GFX90A-NEXT:    ; def v[2:5]
 ; GFX90A-NEXT:    ;;#ASMEND
 ; GFX90A-NEXT:    v_mov_b32_e32 v6, 0
-; GFX90A-NEXT:    v_mov_b32_e32 v2, v5
-; GFX90A-NEXT:    v_mov_b32_e32 v3, v4
+; GFX90A-NEXT:    v_pk_mov_b32 v[2:3], v[4:5], v[4:5] op_sel:[1,0]
 ; GFX90A-NEXT:    v_mov_b32_e32 v4, v1
 ; GFX90A-NEXT:    v_mov_b32_e32 v5, v1
 ; GFX90A-NEXT:    global_store_dwordx4 v6, v[2:5], s[16:17]
@@ -3837,8 +3795,7 @@ define void @v_shuffle_v4p3_v4p3__7_6_1_1(ptr addrspace(1) inreg %ptr) {
 ; GFX940-NEXT:    ; def v[2:5]
 ; GFX940-NEXT:    ;;#ASMEND
 ; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_mov_b32_e32 v2, v5
-; GFX940-NEXT:    v_mov_b32_e32 v3, v4
+; GFX940-NEXT:    v_pk_mov_b32 v[2:3], v[4:5], v[4:5] op_sel:[1,0]
 ; GFX940-NEXT:    v_mov_b32_e32 v4, v1
 ; GFX940-NEXT:    v_mov_b32_e32 v5, v1
 ; GFX940-NEXT:    global_store_dwordx4 v6, v[2:5], s[0:1] sc0 sc1
@@ -4444,8 +4401,7 @@ define void @v_shuffle_v4p3_v4p3__1_2_2_2(ptr addrspace(1) inreg %ptr) {
 ; GFX90A-NEXT:    ; def v[0:3]
 ; GFX90A-NEXT:    ;;#ASMEND
 ; GFX90A-NEXT:    v_mov_b32_e32 v4, 0
-; GFX90A-NEXT:    v_mov_b32_e32 v0, v1
-; GFX90A-NEXT:    v_mov_b32_e32 v1, v2
+; GFX90A-NEXT:    v_pk_mov_b32 v[0:1], v[0:1], v[2:3] op_sel:[1,0]
 ; GFX90A-NEXT:    v_mov_b32_e32 v3, v2
 ; GFX90A-NEXT:    global_store_dwordx4 v4, v[0:3], s[16:17]
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
@@ -4458,8 +4414,7 @@ define void @v_shuffle_v4p3_v4p3__1_2_2_2(ptr addrspace(1) inreg %ptr) {
 ; GFX940-NEXT:    ; def v[0:3]
 ; GFX940-NEXT:    ;;#ASMEND
 ; GFX940-NEXT:    v_mov_b32_e32 v4, 0
-; GFX940-NEXT:    v_mov_b32_e32 v0, v1
-; GFX940-NEXT:    v_mov_b32_e32 v1, v2
+; GFX940-NEXT:    v_pk_mov_b32 v[0:1], v[0:1], v[2:3] op_sel:[1,0]
 ; GFX940-NEXT:    v_mov_b32_e32 v3, v2
 ; GFX940-NEXT:    global_store_dwordx4 v4, v[0:3], s[0:1] sc0 sc1
 ; GFX940-NEXT:    s_waitcnt vmcnt(0)
@@ -4540,8 +4495,7 @@ define void @v_shuffle_v4p3_v4p3__3_2_2_2(ptr addrspace(1) inreg %ptr) {
 ; GFX90A-NEXT:    ; def v[0:3]
 ; GFX90A-NEXT:    ;;#ASMEND
 ; GFX90A-NEXT:    v_mov_b32_e32 v4, 0
-; GFX90A-NEXT:    v_mov_b32_e32 v0, v3
-; GFX90A-NEXT:    v_mov_b32_e32 v1, v2
+; GFX90A-NEXT:    v_pk_mov_b32 v[0:1], v[2:3], v[2:3] op_sel:[1,0]
 ; GFX90A-NEXT:    v_mov_b32_e32 v3, v2
 ; GFX90A-NEXT:    global_store_dwordx4 v4, v[0:3], s[16:17]
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
@@ -4554,8 +4508,7 @@ define void @v_shuffle_v4p3_v4p3__3_2_2_2(ptr addrspace(1) inreg %ptr) {
 ; GFX940-NEXT:    ; def v[0:3]
 ; GFX940-NEXT:    ;;#ASMEND
 ; GFX940-NEXT:    v_mov_b32_e32 v4, 0
-; GFX940-NEXT:    v_mov_b32_e32 v0, v3
-; GFX940-NEXT:    v_mov_b32_e32 v1, v2
+; GFX940-NEXT:    v_pk_mov_b32 v[0:1], v[2:3], v[2:3] op_sel:[1,0]
 ; GFX940-NEXT:    v_mov_b32_e32 v3, v2
 ; GFX940-NEXT:    global_store_dwordx4 v4, v[0:3], s[0:1] sc0 sc1
 ; GFX940-NEXT:    s_waitcnt vmcnt(0)
@@ -4639,8 +4592,7 @@ define void @v_shuffle_v4p3_v4p3__5_2_2_2(ptr addrspace(1) inreg %ptr) {
 ; GFX90A-NEXT:    ;;#ASMSTART
 ; GFX90A-NEXT:    ; def v[4:7]
 ; GFX90A-NEXT:    ;;#ASMEND
-; GFX90A-NEXT:    v_mov_b32_e32 v0, v5
-; GFX90A-NEXT:    v_mov_b32_e32 v1, v2
+; GFX90A-NEXT:    v_pk_mov_b32 v[0:1], v[4:5], v[2:3] op_sel:[1,0]
 ; GFX90A-NEXT:    v_mov_b32_e32 v3, v2
 ; GFX90A-NEXT:    global_store_dwordx4 v8, v[0:3], s[16:17]
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
@@ -4656,8 +4608,8 @@ define void @v_shuffle_v4p3_v4p3__5_2_2_2(ptr addrspace(1) inreg %ptr) {
 ; GFX940-NEXT:    ;;#ASMSTART
 ; GFX940-NEXT:    ; def v[4:7]
 ; GFX940-NEXT:    ;;#ASMEND
-; GFX940-NEXT:    v_mov_b32_e32 v1, v2
-; GFX940-NEXT:    v_mov_b32_e32 v0, v5
+; GFX940-NEXT:    s_nop 0
+; GFX940-NEXT:    v_pk_mov_b32 v[0:1], v[4:5], v[2:3] op_sel:[1,0]
 ; GFX940-NEXT:    v_mov_b32_e32 v3, v2
 ; GFX940-NEXT:    global_store_dwordx4 v8, v[0:3], s[0:1] sc0 sc1
 ; GFX940-NEXT:    s_waitcnt vmcnt(0)
@@ -4755,8 +4707,7 @@ define void @v_shuffle_v4p3_v4p3__7_2_2_2(ptr addrspace(1) inreg %ptr) {
 ; GFX90A-NEXT:    ;;#ASMSTART
 ; GFX90A-NEXT:    ; def v[4:7]
 ; GFX90A-NEXT:    ;;#ASMEND
-; GFX90A-NEXT:    v_mov_b32_e32 v0, v7
-; GFX90A-NEXT:    v_mov_b32_e32 v1, v2
+; GFX90A-NEXT:    v_pk_mov_b32 v[0:1], v[6:7], v[2:3] op_sel:[1,0]
 ; GFX90A-NEXT:    v_mov_b32_e32 v3, v2
 ; GFX90A-NEXT:    global_store_dwordx4 v8, v[0:3], s[16:17]
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
@@ -4772,8 +4723,8 @@ define void @v_shuffle_v4p3_v4p3__7_2_2_2(ptr addrspace(1) inreg %ptr) {
 ; GFX940-NEXT:    ;;#ASMSTART
 ; GFX940-NEXT:    ; def v[4:7]
 ; GFX940-NEXT:    ;;#ASMEND
-; GFX940-NEXT:    v_mov_b32_e32 v1, v2
-; GFX940-NEXT:    v_mov_b32_e32 v0, v7
+; GFX940-NEXT:    s_nop 0
+; GFX940-NEXT:    v_pk_mov_b32 v[0:1], v[6:7], v[2:3] op_sel:[1,0]
 ; GFX940-NEXT:    v_mov_b32_e32 v3, v2
 ; GFX940-NEXT:    global_store_dwordx4 v8, v[0:3], s[0:1] sc0 sc1
 ; GFX940-NEXT:    s_waitcnt vmcnt(0)
@@ -4863,17 +4814,15 @@ define void @v_shuffle_v4p3_v4p3__7_0_2_2(ptr addrspace(1) inreg %ptr) {
 ; GFX90A:       ; %bb.0:
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX90A-NEXT:    ;;#ASMSTART
-; GFX90A-NEXT:    ; def v[4:7]
+; GFX90A-NEXT:    ; def v[0:3]
 ; GFX90A-NEXT:    ;;#ASMEND
 ; GFX90A-NEXT:    v_mov_b32_e32 v8, 0
 ; GFX90A-NEXT:    ;;#ASMSTART
-; GFX90A-NEXT:    ; def v[0:3]
+; GFX90A-NEXT:    ; def v[4:7]
 ; GFX90A-NEXT:    ;;#ASMEND
-; GFX90A-NEXT:    v_mov_b32_e32 v4, v7
-; GFX90A-NEXT:    v_mov_b32_e32 v5, v0
-; GFX90A-NEXT:    v_mov_b32_e32 v6, v2
-; GFX90A-NEXT:    v_mov_b32_e32 v7, v2
-; GFX90A-NEXT:    global_store_dwordx4 v8, v[4:7], s[16:17]
+; GFX90A-NEXT:    v_pk_mov_b32 v[0:1], v[6:7], v[0:1] op_sel:[1,0]
+; GFX90A-NEXT:    v_mov_b32_e32 v3, v2
+; GFX90A-NEXT:    global_store_dwordx4 v8, v[0:3], s[16:17]
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
 ; GFX90A-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -4881,17 +4830,15 @@ define void @v_shuffle_v4p3_v4p3__7_0_2_2(ptr addrspace(1) inreg %ptr) {
 ; GFX940:       ; %bb.0:
 ; GFX940-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX940-NEXT:    ;;#ASMSTART
-; GFX940-NEXT:    ; def v[4:7]
+; GFX940-NEXT:    ; def v[0:3]
 ; GFX940-NEXT:    ;;#ASMEND
 ; GFX940-NEXT:    v_mov_b32_e32 v8, 0
 ; GFX940-NEXT:    ;;#ASMSTART
-; GFX940-NEXT:    ; def v[0:3]
+; GFX940-NEXT:    ; def v[4:7]
 ; GFX940-NEXT:    ;;#ASMEND
-; GFX940-NEXT:    v_mov_b32_e32 v4, v7
-; GFX940-NEXT:    v_mov_b32_e32 v5, v0
-; GFX940-NEXT:    v_mov_b32_e32 v6, v2
-; GFX940-NEXT:    v_mov_b32_e32 v7, v2
-; GFX940-NEXT:    global_store_dwordx4 v8, v[4:7], s[0:1] sc0 sc1
+; GFX940-NEXT:    v_mov_b32_e32 v3, v2
+; GFX940-NEXT:    v_pk_mov_b32 v[0:1], v[6:7], v[0:1] op_sel:[1,0]
+; GFX940-NEXT:    global_store_dwordx4 v8, v[0:3], s[0:1] sc0 sc1
 ; GFX940-NEXT:    s_waitcnt vmcnt(0)
 ; GFX940-NEXT:    s_setpc_b64 s[30:31]
   %vec0 = call <4 x ptr addrspace(3)> asm "; def $0", "=v"()
@@ -5042,8 +4989,7 @@ define void @v_shuffle_v4p3_v4p3__7_4_2_2(ptr addrspace(1) inreg %ptr) {
 ; GFX90A-NEXT:    ;;#ASMSTART
 ; GFX90A-NEXT:    ; def v[4:7]
 ; GFX90A-NEXT:    ;;#ASMEND
-; GFX90A-NEXT:    v_mov_b32_e32 v0, v7
-; GFX90A-NEXT:    v_mov_b32_e32 v1, v4
+; GFX90A-NEXT:    v_pk_mov_b32 v[0:1], v[6:7], v[4:5] op_sel:[1,0]
 ; GFX90A-NEXT:    v_mov_b32_e32 v3, v2
 ; GFX90A-NEXT:    global_store_dwordx4 v8, v[0:3], s[16:17]
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
@@ -5060,8 +5006,7 @@ define void @v_shuffle_v4p3_v4p3__7_4_2_2(ptr addrspace(1) inreg %ptr) {
 ; GFX940-NEXT:    ; def v[4:7]
 ; GFX940-NEXT:    ;;#ASMEND
 ; GFX940-NEXT:    v_mov_b32_e32 v3, v2
-; GFX940-NEXT:    v_mov_b32_e32 v0, v7
-; GFX940-NEXT:    v_mov_b32_e32 v1, v4
+; GFX940-NEXT:    v_pk_mov_b32 v[0:1], v[6:7], v[4:5] op_sel:[1,0]
 ; GFX940-NEXT:    global_store_dwordx4 v8, v[0:3], s[0:1] sc0 sc1
 ; GFX940-NEXT:    s_waitcnt vmcnt(0)
 ; GFX940-NEXT:    s_setpc_b64 s[30:31]
@@ -5158,8 +5103,7 @@ define void @v_shuffle_v4p3_v4p3__7_6_2_2(ptr addrspace(1) inreg %ptr) {
 ; GFX90A-NEXT:    ;;#ASMSTART
 ; GFX90A-NEXT:    ; def v[4:7]
 ; GFX90A-NEXT:    ;;#ASMEND
-; GFX90A-NEXT:    v_mov_b32_e32 v0, v7
-; GFX90A-NEXT:    v_mov_b32_e32 v1, v6
+; GFX90A-NEXT:    v_pk_mov_b32 v[0:1], v[6:7], v[6:7] op_sel:[1,0]
 ; GFX90A-NEXT:    v_mov_b32_e32 v3, v2
 ; GFX90A-NEXT:    global_store_dwordx4 v8, v[0:3], s[16:17]
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
@@ -5176,8 +5120,7 @@ define void @v_shuffle_v4p3_v4p3__7_6_2_2(ptr addrspace(1) inreg %ptr) {
 ; GFX940-NEXT:    ; def v[4:7]
 ; GFX940-NEXT:    ;;#ASMEND
 ; GFX940-NEXT:    v_mov_b32_e32 v3, v2
-; GFX940-NEXT:    v_mov_b32_e32 v0, v7
-; GFX940-NEXT:    v_mov_b32_e32 v1, v6
+; GFX940-NEXT:    v_pk_mov_b32 v[0:1], v[6:7], v[6:7] op_sel:[1,0]
 ; GFX940-NEXT:    global_store_dwordx4 v8, v[0:3], s[0:1] sc0 sc1
 ; GFX940-NEXT:    s_waitcnt vmcnt(0)
 ; GFX940-NEXT:    s_setpc_b64 s[30:31]
@@ -5388,17 +5331,16 @@ define void @v_shuffle_v4p3_v4p3__7_7_1_2(ptr addrspace(1) inreg %ptr) {
 ; GFX90A:       ; %bb.0:
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX90A-NEXT:    ;;#ASMSTART
-; GFX90A-NEXT:    ; def v[4:7]
+; GFX90A-NEXT:    ; def v[0:3]
 ; GFX90A-NEXT:    ;;#ASMEND
 ; GFX90A-NEXT:    v_mov_b32_e32 v8, 0
 ; GFX90A-NEXT:    ;;#ASMSTART
-; GFX90A-NEXT:    ; def v[0:3]
+; GFX90A-NEXT:    ; def v[4:7]
 ; GFX90A-NEXT:    ;;#ASMEND
-; GFX90A-NEXT:    v_mov_b32_e32 v4, v7
-; GFX90A-NEXT:    v_mov_b32_e32 v5, v7
-; GFX90A-NEXT:    v_mov_b32_e32 v6, v1
-; GFX90A-NEXT:    v_mov_b32_e32 v7, v2
-; GFX90A-NEXT:    global_store_dwordx4 v8, v[4:7], s[16:17]
+; GFX90A-NEXT:    v_pk_mov_b32 v[2:3], v[0:1], v[2:3] op_sel:[1,0]
+; GFX90A-NEXT:    v_mov_b32_e32 v0, v7
+; GFX90A-NEXT:    v_mov_b32_e32 v1, v7
+; GFX90A-NEXT:    global_store_dwordx4 v8, v[0:3], s[16:17]
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
 ; GFX90A-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -5406,17 +5348,16 @@ define void @v_shuffle_v4p3_v4p3__7_7_1_2(ptr addrspace(1) inreg %ptr) {
 ; GFX940:       ; %bb.0:
 ; GFX940-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX940-NEXT:    ;;#ASMSTART
-; GFX940-NEXT:    ; def v[4:7]
+; GFX940-NEXT:    ; def v[0:3]
 ; GFX940-NEXT:    ;;#ASMEND
 ; GFX940-NEXT:    v_mov_b32_e32 v8, 0
 ; GFX940-NEXT:    ;;#ASMSTART
-; GFX940-NEXT:    ; def v[0:3]
+; GFX940-NEXT:    ; def v[4:7]
 ; GFX940-NEXT:    ;;#ASMEND
-; GFX940-NEXT:    v_mov_b32_e32 v4, v7
-; GFX940-NEXT:    v_mov_b32_e32 v5, v7
-; GFX940-NEXT:    v_mov_b32_e32 v6, v1
-; GFX940-NEXT:    v_mov_b32_e32 v7, v2
-; GFX940-NEXT:    global_store_dwordx4 v8, v[4:7], s[0:1] sc0 sc1
+; GFX940-NEXT:    v_pk_mov_b32 v[2:3], v[0:1], v[2:3] op_sel:[1,0]
+; GFX940-NEXT:    v_mov_b32_e32 v0, v7
+; GFX940-NEXT:    v_mov_b32_e32 v1, v7
+; GFX940-NEXT:    global_store_dwordx4 v8, v[0:3], s[0:1] sc0 sc1
 ; GFX940-NEXT:    s_waitcnt vmcnt(0)
 ; GFX940-NEXT:    s_setpc_b64 s[30:31]
   %vec0 = call <4 x ptr addrspace(3)> asm "; def $0", "=v"()
@@ -5449,17 +5390,16 @@ define void @v_shuffle_v4p3_v4p3__7_7_3_2(ptr addrspace(1) inreg %ptr) {
 ; GFX90A:       ; %bb.0:
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX90A-NEXT:    ;;#ASMSTART
-; GFX90A-NEXT:    ; def v[4:7]
+; GFX90A-NEXT:    ; def v[0:3]
 ; GFX90A-NEXT:    ;;#ASMEND
 ; GFX90A-NEXT:    v_mov_b32_e32 v8, 0
 ; GFX90A-NEXT:    ;;#ASMSTART
-; GFX90A-NEXT:    ; def v[0:3]
+; GFX90A-NEXT:    ; def v[4:7]
 ; GFX90A-NEXT:    ;;#ASMEND
-; GFX90A-NEXT:    v_mov_b32_e32 v4, v7
-; GFX90A-NEXT:    v_mov_b32_e32 v5, v7
-; GFX90A-NEXT:    v_mov_b32_e32 v6, v3
-; GFX90A-NEXT:    v_mov_b32_e32 v7, v2
-; GFX90A-NEXT:    global_store_dwordx4 v8, v[4:7], s[16:17]
+; GFX90A-NEXT:    v_pk_mov_b32 v[2:3], v[2:3], v[2:3] op_sel:[1,0]
+; GFX90A-NEXT:    v_mov_b32_e32 v0, v7
+; GFX90A-NEXT:    v_mov_b32_e32 v1, v7
+; GFX90A-NEXT:    global_store_dwordx4 v8, v[0:3], s[16:17]
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
 ; GFX90A-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -5467,17 +5407,16 @@ define void @v_shuffle_v4p3_v4p3__7_7_3_2(ptr addrspace(1) inreg %ptr) {
 ; GFX940:       ; %bb.0:
 ; GFX940-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX940-NEXT:    ;;#ASMSTART
-; GFX940-NEXT:    ; def v[4:7]
+; GFX940-NEXT:    ; def v[0:3]
 ; GFX940-NEXT:    ;;#ASMEND
 ; GFX940-NEXT:    v_mov_b32_e32 v8, 0
 ; GFX940-NEXT:    ;;#ASMSTART
-; GFX940-NEXT:    ; def v[0:3]
+; GFX940-NEXT:    ; def v[4:7]
 ; GFX940-NEXT:    ;;#ASMEND
-; GFX940-NEXT:    v_mov_b32_e32 v4, v7
-; GFX940-NEXT:    v_mov_b32_e32 v5, v7
-; GFX940-NEXT:    v_mov_b32_e32 v6, v3
-; GFX940-NEXT:    v_mov_b32_e32 v7, v2
-; GFX940-NEXT:    global_store_dwordx4 v8, v[4:7], s[0:1] sc0 sc1
+; GFX940-NEXT:    v_pk_mov_b32 v[2:3], v[2:3], v[2:3] op_sel:[1,0]
+; GFX940-NEXT:    v_mov_b32_e32 v0, v7
+; GFX940-NEXT:    v_mov_b32_e32 v1, v7
+; GFX940-NEXT:    global_store_dwordx4 v8, v[0:3], s[0:1] sc0 sc1
 ; GFX940-NEXT:    s_waitcnt vmcnt(0)
 ; GFX940-NEXT:    s_setpc_b64 s[30:31]
   %vec0 = call <4 x ptr addrspace(3)> asm "; def $0", "=v"()
@@ -5568,16 +5507,16 @@ define void @v_shuffle_v4p3_v4p3__7_7_5_2(ptr addrspace(1) inreg %ptr) {
 ; GFX90A:       ; %bb.0:
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX90A-NEXT:    ;;#ASMSTART
-; GFX90A-NEXT:    ; def v[4:7]
+; GFX90A-NEXT:    ; def v[0:3]
 ; GFX90A-NEXT:    ;;#ASMEND
-; GFX90A-NEXT:    v_mov_b32_e32 v10, 0
+; GFX90A-NEXT:    v_mov_b32_e32 v8, 0
 ; GFX90A-NEXT:    ;;#ASMSTART
-; GFX90A-NEXT:    ; def v[0:3]
+; GFX90A-NEXT:    ; def v[4:7]
 ; GFX90A-NEXT:    ;;#ASMEND
-; GFX90A-NEXT:    v_mov_b32_e32 v6, v7
-; GFX90A-NEXT:    v_mov_b32_e32 v8, v5
-; GFX90A-NEXT:    v_mov_b32_e32 v9, v2
-; GFX90A-NEXT:    global_store_dwordx4 v10, v[6:9], s[16:17]
+; GFX90A-NEXT:    v_pk_mov_b32 v[2:3], v[4:5], v[2:3] op_sel:[1,0]
+; GFX90A-NEXT:    v_mov_b32_e32 v0, v7
+; GFX90A-NEXT:    v_mov_b32_e32 v1, v7
+; GFX90A-NEXT:    global_store_dwordx4 v8, v[0:3], s[16:17]
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
 ; GFX90A-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -5585,16 +5524,17 @@ define void @v_shuffle_v4p3_v4p3__7_7_5_2(ptr addrspace(1) inreg %ptr) {
 ; GFX940:       ; %bb.0:
 ; GFX940-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX940-NEXT:    ;;#ASMSTART
-; GFX940-NEXT:    ; def v[4:7]
+; GFX940-NEXT:    ; def v[0:3]
 ; GFX940-NEXT:    ;;#ASMEND
-; GFX940-NEXT:    v_mov_b32_e32 v10, 0
+; GFX940-NEXT:    v_mov_b32_e32 v8, 0
 ; GFX940-NEXT:    ;;#ASMSTART
-; GFX940-NEXT:    ; def v[0:3]
+; GFX940-NEXT:    ; def v[4:7]
 ; GFX940-NEXT:    ;;#ASMEND
-; GFX940-NEXT:    v_mov_b32_e32 v6, v7
-; GFX940-NEXT:    v_mov_b32_e32 v8, v5
-; GFX940-NEXT:    v_mov_b32_e32 v9, v2
-; GFX940-NEXT:    global_store_dwordx4 v10, v[6:9], s[0:1] sc0 sc1
+; GFX940-NEXT:    s_nop 0
+; GFX940-NEXT:    v_pk_mov_b32 v[2:3], v[4:5], v[2:3] op_sel:[1,0]
+; GFX940-NEXT:    v_mov_b32_e32 v0, v7
+; GFX940-NEXT:    v_mov_b32_e32 v1, v7
+; GFX940-NEXT:    global_store_dwordx4 v8, v[0:3], s[0:1] sc0 sc1
 ; GFX940-NEXT:    s_waitcnt vmcnt(0)
 ; GFX940-NEXT:    s_setpc_b64 s[30:31]
   %vec0 = call <4 x ptr addrspace(3)> asm "; def $0", "=v"()
@@ -6193,17 +6133,15 @@ define void @v_shuffle_v4p3_v4p3__7_0_3_3(ptr addrspace(1) inreg %ptr) {
 ; GFX90A:       ; %bb.0:
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX90A-NEXT:    ;;#ASMSTART
-; GFX90A-NEXT:    ; def v[4:7]
+; GFX90A-NEXT:    ; def v[0:3]
 ; GFX90A-NEXT:    ;;#ASMEND
 ; GFX90A-NEXT:    v_mov_b32_e32 v8, 0
 ; GFX90A-NEXT:    ;;#ASMSTART
-; GFX90A-NEXT:    ; def v[0:3]
+; GFX90A-NEXT:    ; def v[4:7]
 ; GFX90A-NEXT:    ;;#ASMEND
-; GFX90A-NEXT:    v_mov_b32_e32 v4, v7
-; GFX90A-NEXT:    v_mov_b32_e32 v5, v0
-; GFX90A-NEXT:    v_mov_b32_e32 v6, v3
-; GFX90A-NEXT:    v_mov_b32_e32 v7, v3
-; GFX90A-NEXT:    global_store_dwordx4 v8, v[4:7], s[16:17]
+; GFX90A-NEXT:    v_pk_mov_b32 v[0:1], v[6:7], v[0:1] op_sel:[1,0]
+; GFX90A-NEXT:    v_mov_b32_e32 v2, v3
+; GFX90A-NEXT:    global_store_dwordx4 v8, v[0:3], s[16:17]
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
 ; GFX90A-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -6211,17 +6149,15 @@ define void @v_shuffle_v4p3_v4p3__7_0_3_3(ptr addrspace(1) inreg %ptr) {
 ; GFX940:       ; %bb.0:
 ; GFX940-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX940-NEXT:    ;;#ASMSTART
-; GFX940-NEXT:    ; def v[4:7]
+; GFX940-NEXT:    ; def v[0:3]
 ; GFX940-NEXT:    ;;#ASMEND
 ; GFX940-NEXT:    v_mov_b32_e32 v8, 0
 ; GFX940-NEXT:    ;;#ASMSTART
-; GFX940-NEXT:    ; def v[0:3]
+; GFX940-NEXT:    ; def v[4:7]
 ; GFX940-NEXT:    ;;#ASMEND
-; GFX940-NEXT:    v_mov_b32_e32 v4, v7
-; GFX940-NEXT:    v_mov_b32_e32 v5, v0
-; GFX940-NEXT:    v_mov_b32_e32 v6, v3
-; GFX940-NEXT:    v_mov_b32_e32 v7, v3
-; GFX940-NEXT:    global_store_dwordx4 v8, v[4:7], s[0:1] sc0 sc1
+; GFX940-NEXT:    v_mov_b32_e32 v2, v3
+; GFX940-NEXT:    v_pk_mov_b32 v[0:1], v[6:7], v[0:1] op_sel:[1,0]
+; GFX940-NEXT:    global_store_dwordx4 v8, v[0:3], s[0:1] sc0 sc1
 ; GFX940-NEXT:    s_waitcnt vmcnt(0)
 ; GFX940-NEXT:    s_setpc_b64 s[30:31]
   %vec0 = call <4 x ptr addrspace(3)> asm "; def $0", "=v"()
@@ -6314,8 +6250,7 @@ define void @v_shuffle_v4p3_v4p3__7_2_3_3(ptr addrspace(1) inreg %ptr) {
 ; GFX90A-NEXT:    ;;#ASMSTART
 ; GFX90A-NEXT:    ; def v[4:7]
 ; GFX90A-NEXT:    ;;#ASMEND
-; GFX90A-NEXT:    v_mov_b32_e32 v0, v7
-; GFX90A-NEXT:    v_mov_b32_e32 v1, v2
+; GFX90A-NEXT:    v_pk_mov_b32 v[0:1], v[6:7], v[2:3] op_sel:[1,0]
 ; GFX90A-NEXT:    v_mov_b32_e32 v2, v3
 ; GFX90A-NEXT:    global_store_dwordx4 v8, v[0:3], s[16:17]
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
@@ -6331,8 +6266,8 @@ define void @v_shuffle_v4p3_v4p3__7_2_3_3(ptr addrspace(1) inreg %ptr) {
 ; GFX940-NEXT:    ;;#ASMSTART
 ; GFX940-NEXT:    ; def v[4:7]
 ; GFX940-NEXT:    ;;#ASMEND
-; GFX940-NEXT:    v_mov_b32_e32 v1, v2
-; GFX940-NEXT:    v_mov_b32_e32 v0, v7
+; GFX940-NEXT:    s_nop 0
+; GFX940-NEXT:    v_pk_mov_b32 v[0:1], v[6:7], v[2:3] op_sel:[1,0]
 ; GFX940-NEXT:    v_mov_b32_e32 v2, v3
 ; GFX940-NEXT:    global_store_dwordx4 v8, v[0:3], s[0:1] sc0 sc1
 ; GFX940-NEXT:    s_waitcnt vmcnt(0)
@@ -6372,8 +6307,7 @@ define void @v_shuffle_v4p3_v4p3__7_4_3_3(ptr addrspace(1) inreg %ptr) {
 ; GFX90A-NEXT:    ;;#ASMSTART
 ; GFX90A-NEXT:    ; def v[4:7]
 ; GFX90A-NEXT:    ;;#ASMEND
-; GFX90A-NEXT:    v_mov_b32_e32 v0, v7
-; GFX90A-NEXT:    v_mov_b32_e32 v1, v4
+; GFX90A-NEXT:    v_pk_mov_b32 v[0:1], v[6:7], v[4:5] op_sel:[1,0]
 ; GFX90A-NEXT:    v_mov_b32_e32 v2, v3
 ; GFX90A-NEXT:    global_store_dwordx4 v8, v[0:3], s[16:17]
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
@@ -6390,8 +6324,7 @@ define void @v_shuffle_v4p3_v4p3__7_4_3_3(ptr addrspace(1) inreg %ptr) {
 ; GFX940-NEXT:    ; def v[4:7]
 ; GFX940-NEXT:    ;;#ASMEND
 ; GFX940-NEXT:    v_mov_b32_e32 v2, v3
-; GFX940-NEXT:    v_mov_b32_e32 v0, v7
-; GFX940-NEXT:    v_mov_b32_e32 v1, v4
+; GFX940-NEXT:    v_pk_mov_b32 v[0:1], v[6:7], v[4:5] op_sel:[1,0]
 ; GFX940-NEXT:    global_store_dwordx4 v8, v[0:3], s[0:1] sc0 sc1
 ; GFX940-NEXT:    s_waitcnt vmcnt(0)
 ; GFX940-NEXT:    s_setpc_b64 s[30:31]
@@ -6488,8 +6421,7 @@ define void @v_shuffle_v4p3_v4p3__7_6_3_3(ptr addrspace(1) inreg %ptr) {
 ; GFX90A-NEXT:    ;;#ASMSTART
 ; GFX90A-NEXT:    ; def v[4:7]
 ; GFX90A-NEXT:    ;;#ASMEND
-; GFX90A-NEXT:    v_mov_b32_e32 v0, v7
-; GFX90A-NEXT:    v_mov_b32_e32 v1, v6
+; GFX90A-NEXT:    v_pk_mov_b32 v[0:1], v[6:7], v[6:7] op_sel:[1,0]
 ; GFX90A-NEXT:    v_mov_b32_e32 v2, v3
 ; GFX90A-NEXT:    global_store_dwordx4 v8, v[0:3], s[16:17]
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
@@ -6506,8 +6438,7 @@ define void @v_shuffle_v4p3_v4p3__7_6_3_3(ptr addrspace(1) inreg %ptr) {
 ; GFX940-NEXT:    ; def v[4:7]
 ; GFX940-NEXT:    ;;#ASMEND
 ; GFX940-NEXT:    v_mov_b32_e32 v2, v3
-; GFX940-NEXT:    v_mov_b32_e32 v0, v7
-; GFX940-NEXT:    v_mov_b32_e32 v1, v6
+; GFX940-NEXT:    v_pk_mov_b32 v[0:1], v[6:7], v[6:7] op_sel:[1,0]
 ; GFX940-NEXT:    global_store_dwordx4 v8, v[0:3], s[0:1] sc0 sc1
 ; GFX940-NEXT:    s_waitcnt vmcnt(0)
 ; GFX940-NEXT:    s_setpc_b64 s[30:31]
@@ -7195,8 +7126,7 @@ define void @v_shuffle_v4p3_v4p3__5_4_4_4(ptr addrspace(1) inreg %ptr) {
 ; GFX90A-NEXT:    ; def v[0:3]
 ; GFX90A-NEXT:    ;;#ASMEND
 ; GFX90A-NEXT:    v_mov_b32_e32 v6, 0
-; GFX90A-NEXT:    v_mov_b32_e32 v2, v1
-; GFX90A-NEXT:    v_mov_b32_e32 v3, v0
+; GFX90A-NEXT:    v_pk_mov_b32 v[2:3], v[0:1], v[0:1] op_sel:[1,0]
 ; GFX90A-NEXT:    v_mov_b32_e32 v4, v0
 ; GFX90A-NEXT:    v_mov_b32_e32 v5, v0
 ; GFX90A-NEXT:    global_store_dwordx4 v6, v[2:5], s[16:17]
@@ -7210,8 +7140,7 @@ define void @v_shuffle_v4p3_v4p3__5_4_4_4(ptr addrspace(1) inreg %ptr) {
 ; GFX940-NEXT:    ; def v[0:3]
 ; GFX940-NEXT:    ;;#ASMEND
 ; GFX940-NEXT:    v_mov_b32_e32 v6, 0
-; GFX940-NEXT:    v_mov_b32_e32 v2, v1
-; GFX940-NEXT:    v_mov_b32_e32 v3, v0
+; GFX940-NEXT:    v_pk_mov_b32 v[2:3], v[0:1], v[0:1] op_sel:[1,0]
 ; GFX940-NEXT:    v_mov_b32_e32 v4, v0
 ; GFX940-NEXT:    v_mov_b32_e32 v5, v0
 ; GFX940-NEXT:    global_store_dwordx4 v6, v[2:5], s[0:1] sc0 sc1
@@ -7297,8 +7226,7 @@ define void @v_shuffle_v4p3_v4p3__7_4_4_4(ptr addrspace(1) inreg %ptr) {
 ; GFX90A-NEXT:    ; def v[0:3]
 ; GFX90A-NEXT:    ;;#ASMEND
 ; GFX90A-NEXT:    v_mov_b32_e32 v6, 0
-; GFX90A-NEXT:    v_mov_b32_e32 v2, v3
-; GFX90A-NEXT:    v_mov_b32_e32 v3, v0
+; GFX90A-NEXT:    v_pk_mov_b32 v[2:3], v[2:3], v[0:1] op_sel:[1,0]
 ; GFX90A-NEXT:    v_mov_b32_e32 v4, v0
 ; GFX90A-NEXT:    v_mov_b32_e32 v5, v0
 ; GFX90A-NEXT:    global_store_dwordx4 v6, v[2:5], s[16:17]
@@ -7312,8 +7240,7 @@ define void @v_shuffle_v4p3_v4p3__7_4_4_4(ptr addrspace(1) inreg %ptr) {
 ; GFX940-NEXT:    ; def v[0:3]
 ; GFX940-NEXT:    ;;#ASMEND
 ; GFX940-NEXT:    v_mov_b32_e32 v6, 0
-; GFX940-NEXT:    v_mov_b32_e32 v2, v3
-; GFX940-NEXT:    v_mov_b32_e32 v3, v0
+; GFX940-NEXT:    v_pk_mov_b32 v[2:3], v[2:3], v[0:1] op_sel:[1,0]
 ; GFX940-NEXT:    v_mov_b32_e32 v4, v0
 ; GFX940-NEXT:    v_mov_b32_e32 v5, v0
 ; GFX940-NEXT:    global_store_dwordx4 v6, v[2:5], s[0:1] sc0 sc1
@@ -7404,8 +7331,7 @@ define void @v_shuffle_v4p3_v4p3__7_0_4_4(ptr addrspace(1) inreg %ptr) {
 ; GFX90A-NEXT:    ; def v[2:5]
 ; GFX90A-NEXT:    ;;#ASMEND
 ; GFX90A-NEXT:    v_mov_b32_e32 v8, 0
-; GFX90A-NEXT:    v_mov_b32_e32 v4, v5
-; GFX90A-NEXT:    v_mov_b32_e32 v5, v0
+; GFX90A-NEXT:    v_pk_mov_b32 v[4:5], v[4:5], v[0:1] op_sel:[1,0]
 ; GFX90A-NEXT:    v_mov_b32_e32 v6, v2
 ; GFX90A-NEXT:    v_mov_b32_e32 v7, v2
 ; GFX90A-NEXT:    global_store_dwordx4 v8, v[4:7], s[16:17]
@@ -7423,8 +7349,7 @@ define void @v_shuffle_v4p3_v4p3__7_0_4_4(ptr addrspace(1) inreg %ptr) {
 ; GFX940-NEXT:    ; def v[2:5]
 ; GFX940-NEXT:    ;;#ASMEND
 ; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_mov_b32_e32 v4, v5
-; GFX940-NEXT:    v_mov_b32_e32 v5, v0
+; GFX940-NEXT:    v_pk_mov_b32 v[4:5], v[4:5], v[0:1] op_sel:[1,0]
 ; GFX940-NEXT:    v_mov_b32_e32 v6, v2
 ; GFX940-NEXT:    v_mov_b32_e32 v7, v2
 ; GFX940-NEXT:    global_store_dwordx4 v8, v[4:7], s[0:1] sc0 sc1
@@ -7523,8 +7448,7 @@ define void @v_shuffle_v4p3_v4p3__7_2_4_4(ptr addrspace(1) inreg %ptr) {
 ; GFX90A-NEXT:    ;;#ASMSTART
 ; GFX90A-NEXT:    ; def v[4:7]
 ; GFX90A-NEXT:    ;;#ASMEND
-; GFX90A-NEXT:    v_mov_b32_e32 v0, v7
-; GFX90A-NEXT:    v_mov_b32_e32 v1, v2
+; GFX90A-NEXT:    v_pk_mov_b32 v[0:1], v[6:7], v[2:3] op_sel:[1,0]
 ; GFX90A-NEXT:    v_mov_b32_e32 v2, v4
 ; GFX90A-NEXT:    v_mov_b32_e32 v3, v4
 ; GFX90A-NEXT:    global_store_dwordx4 v8, v[0:3], s[16:17]
@@ -7541,8 +7465,8 @@ define void @v_shuffle_v4p3_v4p3__7_2_4_4(ptr addrspace(1) inreg %ptr) {
 ; GFX940-NEXT:    ;;#ASMSTART
 ; GFX940-NEXT:    ; def v[4:7]
 ; GFX940-NEXT:    ;;#ASMEND
-; GFX940-NEXT:    v_mov_b32_e32 v1, v2
-; GFX940-NEXT:    v_mov_b32_e32 v0, v7
+; GFX940-NEXT:    s_nop 0
+; GFX940-NEXT:    v_pk_mov_b32 v[0:1], v[6:7], v[2:3] op_sel:[1,0]
 ; GFX940-NEXT:    v_mov_b32_e32 v2, v4
 ; GFX940-NEXT:    v_mov_b32_e32 v3, v4
 ; GFX940-NEXT:    global_store_dwordx4 v8, v[0:3], s[0:1] sc0 sc1
@@ -7686,31 +7610,28 @@ define void @v_shuffle_v4p3_v4p3__7_6_4_4(ptr addrspace(1) inreg %ptr) {
 ; GFX90A-LABEL: v_shuffle_v4p3_v4p3__7_6_4_4:
 ; GFX90A:       ; %bb.0:
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT:    v_mov_b32_e32 v8, 0
 ; GFX90A-NEXT:    ;;#ASMSTART
 ; GFX90A-NEXT:    ; def v[0:3]
 ; GFX90A-NEXT:    ;;#ASMEND
-; GFX90A-NEXT:    v_mov_b32_e32 v4, v3
-; GFX90A-NEXT:    v_mov_b32_e32 v5, v2
-; GFX90A-NEXT:    v_mov_b32_e32 v6, v0
-; GFX90A-NEXT:    v_mov_b32_e32 v7, v0
-; GFX90A-NEXT:    global_store_dwordx4 v8, v[4:7], s[16:17]
+; GFX90A-NEXT:    v_mov_b32_e32 v6, 0
+; GFX90A-NEXT:    v_pk_mov_b32 v[2:3], v[2:3], v[2:3] op_sel:[1,0]
+; GFX90A-NEXT:    v_mov_b32_e32 v4, v0
+; GFX90A-NEXT:    v_mov_b32_e32 v5, v0
+; GFX90A-NEXT:    global_store_dwordx4 v6, v[2:5], s[16:17]
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
 ; GFX90A-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: v_shuffle_v4p3_v4p3__7_6_4_4:
 ; GFX940:       ; %bb.0:
 ; GFX940-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX940-NEXT:    v_mov_b32_e32 v8, 0
 ; GFX940-NEXT:    ;;#ASMSTART
 ; GFX940-NEXT:    ; def v[0:3]
 ; GFX940-NEXT:    ;;#ASMEND
-; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_mov_b32_e32 v4, v3
-; GFX940-NEXT:    v_mov_b32_e32 v5, v2
-; GFX940-NEXT:    v_mov_b32_e32 v6, v0
-; GFX940-NEXT:    v_mov_b32_e32 v7, v0
-; GFX940-NEXT:    global_store_dwordx4 v8, v[4:7], s[0:1] sc0 sc1
+; GFX940-NEXT:    v_mov_b32_e32 v6, 0
+; GFX940-NEXT:    v_pk_mov_b32 v[2:3], v[2:3], v[2:3] op_sel:[1,0]
+; GFX940-NEXT:    v_mov_b32_e32 v4, v0
+; GFX940-NEXT:    v_mov_b32_e32 v5, v0
+; GFX940-NEXT:    global_store_dwordx4 v6, v[2:5], s[0:1] sc0 sc1
 ; GFX940-NEXT:    s_waitcnt vmcnt(0)
 ; GFX940-NEXT:    s_setpc_b64 s[30:31]
   %vec0 = call <4 x ptr addrspace(3)> asm "; def $0", "=v"()
@@ -7904,11 +7825,11 @@ define void @v_shuffle_v4p3_v4p3__7_7_1_4(ptr addrspace(1) inreg %ptr) {
 ; GFX90A-NEXT:    ;;#ASMSTART
 ; GFX90A-NEXT:    ; def v[2:5]
 ; GFX90A-NEXT:    ;;#ASMEND
-; GFX90A-NEXT:    v_mov_b32_e32 v8, 0
-; GFX90A-NEXT:    v_mov_b32_e32 v4, v5
-; GFX90A-NEXT:    v_mov_b32_e32 v6, v1
-; GFX90A-NEXT:    v_mov_b32_e32 v7, v2
-; GFX90A-NEXT:    global_store_dwordx4 v8, v[4:7], s[16:17]
+; GFX90A-NEXT:    v_mov_b32_e32 v6, 0
+; GFX90A-NEXT:    v_pk_mov_b32 v[2:3], v[0:1], v[2:3] op_sel:[1,0]
+; GFX90A-NEXT:    v_mov_b32_e32 v0, v5
+; GFX90A-NEXT:    v_mov_b32_e32 v1, v5
+; GFX90A-NEXT:    global_store_dwordx4 v6, v[0:3], s[16:17]
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
 ; GFX90A-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -7918,14 +7839,15 @@ define void @v_shuffle_v4p3_v4p3__7_7_1_4(ptr addrspace(1) inreg %ptr) {
 ; GFX940-NEXT:    ;;#ASMSTART
 ; GFX940-NEXT:    ; def v[0:3]
 ; GFX940-NEXT:    ;;#ASMEND
-; GFX940-NEXT:    v_mov_b32_e32 v8, 0
+; GFX940-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX940-NEXT:    ;;#ASMSTART
 ; GFX940-NEXT:    ; def v[2:5]
 ; GFX940-NEXT:    ;;#ASMEND
-; GFX940-NEXT:    v_mov_b32_e32 v6, v1
-; GFX940-NEXT:    v_mov_b32_e32 v4, v5
-; GFX940-NEXT:    v_mov_b32_e32 v7, v2
-; GFX940-NEXT:    global_store_dwordx4 v8, v[4:7], s[0:1] sc0 sc1
+; GFX940-NEXT:    s_nop 0
+; GFX940-NEXT:    v_pk_mov_b32 v[2:3], v[0:1], v[2:3] op_sel:[1,0]
+; GFX940-NEXT:    v_mov_b32_e32 v0, v5
+; GFX940-NEXT:    v_mov_b32_e32 v1, v5
+; GFX940-NEXT:    global_store_dwordx4 v6, v[0:3], s[0:1] sc0 sc1
 ; GFX940-NEXT:    s_waitcnt vmcnt(0)
 ; GFX940-NEXT:    s_setpc_b64 s[30:31]
   %vec0 = call <4 x ptr addrspace(3)> asm "; def $0", "=v"()
@@ -8022,10 +7944,9 @@ define void @v_shuffle_v4p3_v4p3__7_7_3_4(ptr addrspace(1) inreg %ptr) {
 ; GFX90A-NEXT:    ;;#ASMSTART
 ; GFX90A-NEXT:    ; def v[4:7]
 ; GFX90A-NEXT:    ;;#ASMEND
+; GFX90A-NEXT:    v_pk_mov_b32 v[2:3], v[2:3], v[4:5] op_sel:[1,0]
 ; GFX90A-NEXT:    v_mov_b32_e32 v0, v7
 ; GFX90A-NEXT:    v_mov_b32_e32 v1, v7
-; GFX90A-NEXT:    v_mov_b32_e32 v2, v3
-; GFX90A-NEXT:    v_mov_b32_e32 v3, v4
 ; GFX90A-NEXT:    global_store_dwordx4 v8, v[0:3], s[16:17]
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
 ; GFX90A-NEXT:    s_setpc_b64 s[30:31]
@@ -8040,10 +7961,10 @@ define void @v_shuffle_v4p3_v4p3__7_7_3_4(ptr addrspace(1) inreg %ptr) {
 ; GFX940-NEXT:    ;;#ASMSTART
 ; GFX940-NEXT:    ; def v[4:7]
 ; GFX940-NEXT:    ;;#ASMEND
-; GFX940-NEXT:    v_mov_b32_e32 v2, v3
+; GFX940-NEXT:    s_nop 0
+; GFX940-NEXT:    v_pk_mov_b32 v[2:3], v[2:3], v[4:5] op_sel:[1,0]
 ; GFX940-NEXT:    v_mov_b32_e32 v0, v7
 ; GFX940-NEXT:    v_mov_b32_e32 v1, v7
-; GFX940-NEXT:    v_mov_b32_e32 v3, v4
 ; GFX940-NEXT:    global_store_dwordx4 v8, v[0:3], s[0:1] sc0 sc1
 ; GFX940-NEXT:    s_waitcnt vmcnt(0)
 ; GFX940-NEXT:    s_setpc_b64 s[30:31]
@@ -8076,9 +7997,8 @@ define void @v_shuffle_v4p3_v4p3__7_7_5_4(ptr addrspace(1) inreg %ptr) {
 ; GFX90A-NEXT:    ; def v[0:3]
 ; GFX90A-NEXT:    ;;#ASMEND
 ; GFX90A-NEXT:    v_mov_b32_e32 v6, 0
+; GFX90A-NEXT:    v_pk_mov_b32 v[4:5], v[0:1], v[0:1] op_sel:[1,0]
 ; GFX90A-NEXT:    v_mov_b32_e32 v2, v3
-; GFX90A-NEXT:    v_mov_b32_e32 v4, v1
-; GFX90A-NEXT:    v_mov_b32_e32 v5, v0
 ; GFX90A-NEXT:    global_store_dwordx4 v6, v[2:5], s[16:17]
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
 ; GFX90A-NEXT:    s_setpc_b64 s[30:31]
@@ -8090,9 +8010,8 @@ define void @v_shuffle_v4p3_v4p3__7_7_5_4(ptr addrspace(1) inreg %ptr) {
 ; GFX940-NEXT:    ; def v[0:3]
 ; GFX940-NEXT:    ;;#ASMEND
 ; GFX940-NEXT:    v_mov_b32_e32 v6, 0
+; GFX940-NEXT:    v_pk_mov_b32 v[4:5], v[0:1], v[0:1] op_sel:[1,0]
 ; GFX940-NEXT:    v_mov_b32_e32 v2, v3
-; GFX940-NEXT:    v_mov_b32_e32 v4, v1
-; GFX940-NEXT:    v_mov_b32_e32 v5, v0
 ; GFX940-NEXT:    global_store_dwordx4 v6, v[2:5], s[0:1] sc0 sc1
 ; GFX940-NEXT:    s_waitcnt vmcnt(0)
 ; GFX940-NEXT:    s_setpc_b64 s[30:31]
@@ -8703,12 +8622,10 @@ define void @v_shuffle_v4p3_v4p3__7_0_5_5(ptr addrspace(1) inreg %ptr) {
 ; GFX90A-NEXT:    ;;#ASMSTART
 ; GFX90A-NEXT:    ; def v[2:5]
 ; GFX90A-NEXT:    ;;#ASMEND
-; GFX90A-NEXT:    v_mov_b32_e32 v8, 0
-; GFX90A-NEXT:    v_mov_b32_e32 v4, v5
-; GFX90A-NEXT:    v_mov_b32_e32 v5, v0
-; GFX90A-NEXT:    v_mov_b32_e32 v6, v3
-; GFX90A-NEXT:    v_mov_b32_e32 v7, v3
-; GFX90A-NEXT:    global_store_dwordx4 v8, v[4:7], s[16:17]
+; GFX90A-NEXT:    v_mov_b32_e32 v6, 0
+; GFX90A-NEXT:    v_pk_mov_b32 v[0:1], v[4:5], v[0:1] op_sel:[1,0]
+; GFX90A-NEXT:    v_mov_b32_e32 v2, v3
+; GFX90A-NEXT:    global_store_dwordx4 v6, v[0:3], s[16:17]
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
 ; GFX90A-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -8718,16 +8635,14 @@ define void @v_shuffle_v4p3_v4p3__7_0_5_5(ptr addrspace(1) inreg %ptr) {
 ; GFX940-NEXT:    ;;#ASMSTART
 ; GFX940-NEXT:    ; def v[0:3]
 ; GFX940-NEXT:    ;;#ASMEND
-; GFX940-NEXT:    v_mov_b32_e32 v8, 0
+; GFX940-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX940-NEXT:    ;;#ASMSTART
 ; GFX940-NEXT:    ; def v[2:5]
 ; GFX940-NEXT:    ;;#ASMEND
 ; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_mov_b32_e32 v4, v5
-; GFX940-NEXT:    v_mov_b32_e32 v5, v0
-; GFX940-NEXT:    v_mov_b32_e32 v6, v3
-; GFX940-NEXT:    v_mov_b32_e32 v7, v3
-; GFX940-NEXT:    global_store_dwordx4 v8, v[4:7], s[0:1] sc0 sc1
+; GFX940-NEXT:    v_pk_mov_b32 v[0:1], v[4:5], v[0:1] op_sel:[1,0]
+; GFX940-NEXT:    v_mov_b32_e32 v2, v3
+; GFX940-NEXT:    global_store_dwordx4 v6, v[0:3], s[0:1] sc0 sc1
 ; GFX940-NEXT:    s_waitcnt vmcnt(0)
 ; GFX940-NEXT:    s_setpc_b64 s[30:31]
   %vec0 = call <4 x ptr addrspace(3)> asm "; def $0", "=v"()
@@ -8822,8 +8737,7 @@ define void @v_shuffle_v4p3_v4p3__7_2_5_5(ptr addrspace(1) inreg %ptr) {
 ; GFX90A-NEXT:    ;;#ASMSTART
 ; GFX90A-NEXT:    ; def v[4:7]
 ; GFX90A-NEXT:    ;;#ASMEND
-; GFX90A-NEXT:    v_mov_b32_e32 v0, v7
-; GFX90A-NEXT:    v_mov_b32_e32 v1, v2
+; GFX90A-NEXT:    v_pk_mov_b32 v[0:1], v[6:7], v[2:3] op_sel:[1,0]
 ; GFX90A-NEXT:    v_mov_b32_e32 v2, v5
 ; GFX90A-NEXT:    v_mov_b32_e32 v3, v5
 ; GFX90A-NEXT:    global_store_dwordx4 v8, v[0:3], s[16:17]
@@ -8840,8 +8754,8 @@ define void @v_shuffle_v4p3_v4p3__7_2_5_5(ptr addrspace(1) inreg %ptr) {
 ; GFX940-NEXT:    ;;#ASMSTART
 ; GFX940-NEXT:    ; def v[4:7]
 ; GFX940-NEXT:    ;;#ASMEND
-; GFX940-NEXT:    v_mov_b32_e32 v1, v2
-; GFX940-NEXT:    v_mov_b32_e32 v0, v7
+; GFX940-NEXT:    s_nop 0
+; GFX940-NEXT:    v_pk_mov_b32 v[0:1], v[6:7], v[2:3] op_sel:[1,0]
 ; GFX940-NEXT:    v_mov_b32_e32 v2, v5
 ; GFX940-NEXT:    v_mov_b32_e32 v3, v5
 ; GFX940-NEXT:    global_store_dwordx4 v8, v[0:3], s[0:1] sc0 sc1
@@ -8938,8 +8852,7 @@ define void @v_shuffle_v4p3_v4p3__7_4_5_5(ptr addrspace(1) inreg %ptr) {
 ; GFX90A-NEXT:    ; def v[0:3]
 ; GFX90A-NEXT:    ;;#ASMEND
 ; GFX90A-NEXT:    v_mov_b32_e32 v6, 0
-; GFX90A-NEXT:    v_mov_b32_e32 v2, v3
-; GFX90A-NEXT:    v_mov_b32_e32 v3, v0
+; GFX90A-NEXT:    v_pk_mov_b32 v[2:3], v[2:3], v[0:1] op_sel:[1,0]
 ; GFX90A-NEXT:    v_mov_b32_e32 v4, v1
 ; GFX90A-NEXT:    v_mov_b32_e32 v5, v1
 ; GFX90A-NEXT:    global_store_dwordx4 v6, v[2:5], s[16:17]
@@ -8953,8 +8866,7 @@ define void @v_shuffle_v4p3_v4p3__7_4_5_5(ptr addrspace(1) inreg %ptr) {
 ; GFX940-NEXT:    ; def v[0:3]
 ; GFX940-NEXT:    ;;#ASMEND
 ; GFX940-NEXT:    v_mov_b32_e32 v6, 0
-; GFX940-NEXT:    v_mov_b32_e32 v2, v3
-; GFX940-NEXT:    v_mov_b32_e32 v3, v0
+; GFX940-NEXT:    v_pk_mov_b32 v[2:3], v[2:3], v[0:1] op_sel:[1,0]
 ; GFX940-NEXT:    v_mov_b32_e32 v4, v1
 ; GFX940-NEXT:    v_mov_b32_e32 v5, v1
 ; GFX940-NEXT:    global_store_dwordx4 v6, v[2:5], s[0:1] sc0 sc1
@@ -8985,31 +8897,28 @@ define void @v_shuffle_v4p3_v4p3__7_6_5_5(ptr addrspace(1) inreg %ptr) {
 ; GFX90A-LABEL: v_shuffle_v4p3_v4p3__7_6_5_5:
 ; GFX90A:       ; %bb.0:
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT:    v_mov_b32_e32 v8, 0
 ; GFX90A-NEXT:    ;;#ASMSTART
 ; GFX90A-NEXT:    ; def v[0:3]
 ; GFX90A-NEXT:    ;;#ASMEND
-; GFX90A-NEXT:    v_mov_b32_e32 v4, v3
-; GFX90A-NEXT:    v_mov_b32_e32 v5, v2
-; GFX90A-NEXT:    v_mov_b32_e32 v6, v1
-; GFX90A-NEXT:    v_mov_b32_e32 v7, v1
-; GFX90A-NEXT:    global_store_dwordx4 v8, v[4:7], s[16:17]
+; GFX90A-NEXT:    v_mov_b32_e32 v6, 0
+; GFX90A-NEXT:    v_pk_mov_b32 v[2:3], v[2:3], v[2:3] op_sel:[1,0]
+; GFX90A-NEXT:    v_mov_b32_e32 v4, v1
+; GFX90A-NEXT:    v_mov_b32_e32 v5, v1
+; GFX90A-NEXT:    global_store_dwordx4 v6, v[2:5], s[16:17]
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
 ; GFX90A-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: v_shuffle_v4p3_v4p3__7_6_5_5:
 ; GFX940:       ; %bb.0:
 ; GFX940-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX940-NEXT:    v_mov_b32_e32 v8, 0
 ; GFX940-NEXT:    ;;#ASMSTART
 ; GFX940-NEXT:    ; def v[0:3]
 ; GFX940-NEXT:    ;;#ASMEND
-; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_mov_b32_e32 v4, v3
-; GFX940-NEXT:    v_mov_b32_e32 v5, v2
-; GFX940-NEXT:    v_mov_b32_e32 v6, v1
-; GFX940-NEXT:    v_mov_b32_e32 v7, v1
-; GFX940-NEXT:    global_store_dwordx4 v8, v[4:7], s[0:1] sc0 sc1
+; GFX940-NEXT:    v_mov_b32_e32 v6, 0
+; GFX940-NEXT:    v_pk_mov_b32 v[2:3], v[2:3], v[2:3] op_sel:[1,0]
+; GFX940-NEXT:    v_mov_b32_e32 v4, v1
+; GFX940-NEXT:    v_mov_b32_e32 v5, v1
+; GFX940-NEXT:    global_store_dwordx4 v6, v[2:5], s[0:1] sc0 sc1
 ; GFX940-NEXT:    s_waitcnt vmcnt(0)
 ; GFX940-NEXT:    s_setpc_b64 s[30:31]
   %vec0 = call <4 x ptr addrspace(3)> asm "; def $0", "=v"()
@@ -9584,8 +9493,7 @@ define void @v_shuffle_v4p3_v4p3__1_6_6_6(ptr addrspace(1) inreg %ptr) {
 ; GFX90A-NEXT:    ; def v[2:5]
 ; GFX90A-NEXT:    ;;#ASMEND
 ; GFX90A-NEXT:    v_mov_b32_e32 v6, 0
-; GFX90A-NEXT:    v_mov_b32_e32 v2, v1
-; GFX90A-NEXT:    v_mov_b32_e32 v3, v4
+; GFX90A-NEXT:    v_pk_mov_b32 v[2:3], v[0:1], v[4:5] op_sel:[1,0]
 ; GFX90A-NEXT:    v_mov_b32_e32 v5, v4
 ; GFX90A-NEXT:    global_store_dwordx4 v6, v[2:5], s[16:17]
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
@@ -9602,8 +9510,7 @@ define void @v_shuffle_v4p3_v4p3__1_6_6_6(ptr addrspace(1) inreg %ptr) {
 ; GFX940-NEXT:    ; def v[2:5]
 ; GFX940-NEXT:    ;;#ASMEND
 ; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_mov_b32_e32 v2, v1
-; GFX940-NEXT:    v_mov_b32_e32 v3, v4
+; GFX940-NEXT:    v_pk_mov_b32 v[2:3], v[0:1], v[4:5] op_sel:[1,0]
 ; GFX940-NEXT:    v_mov_b32_e32 v5, v4
 ; GFX940-NEXT:    global_store_dwordx4 v6, v[2:5], s[0:1] sc0 sc1
 ; GFX940-NEXT:    s_waitcnt vmcnt(0)
@@ -9701,8 +9608,7 @@ define void @v_shuffle_v4p3_v4p3__3_6_6_6(ptr addrspace(1) inreg %ptr) {
 ; GFX90A-NEXT:    ;;#ASMSTART
 ; GFX90A-NEXT:    ; def v[0:3]
 ; GFX90A-NEXT:    ;;#ASMEND
-; GFX90A-NEXT:    v_mov_b32_e32 v4, v3
-; GFX90A-NEXT:    v_mov_b32_e32 v5, v6
+; GFX90A-NEXT:    v_pk_mov_b32 v[4:5], v[2:3], v[6:7] op_sel:[1,0]
 ; GFX90A-NEXT:    v_mov_b32_e32 v7, v6
 ; GFX90A-NEXT:    global_store_dwordx4 v8, v[4:7], s[16:17]
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
@@ -9718,8 +9624,8 @@ define void @v_shuffle_v4p3_v4p3__3_6_6_6(ptr addrspace(1) inreg %ptr) {
 ; GFX940-NEXT:    ;;#ASMSTART
 ; GFX940-NEXT:    ; def v[0:3]
 ; GFX940-NEXT:    ;;#ASMEND
-; GFX940-NEXT:    v_mov_b32_e32 v5, v6
-; GFX940-NEXT:    v_mov_b32_e32 v4, v3
+; GFX940-NEXT:    s_nop 0
+; GFX940-NEXT:    v_pk_mov_b32 v[4:5], v[2:3], v[6:7] op_sel:[1,0]
 ; GFX940-NEXT:    v_mov_b32_e32 v7, v6
 ; GFX940-NEXT:    global_store_dwordx4 v8, v[4:7], s[0:1] sc0 sc1
 ; GFX940-NEXT:    s_waitcnt vmcnt(0)
@@ -9799,8 +9705,7 @@ define void @v_shuffle_v4p3_v4p3__5_6_6_6(ptr addrspace(1) inreg %ptr) {
 ; GFX90A-NEXT:    ; def v[0:3]
 ; GFX90A-NEXT:    ;;#ASMEND
 ; GFX90A-NEXT:    v_mov_b32_e32 v4, 0
-; GFX90A-NEXT:    v_mov_b32_e32 v0, v1
-; GFX90A-NEXT:    v_mov_b32_e32 v1, v2
+; GFX90A-NEXT:    v_pk_mov_b32 v[0:1], v[0:1], v[2:3] op_sel:[1,0]
 ; GFX90A-NEXT:    v_mov_b32_e32 v3, v2
 ; GFX90A-NEXT:    global_store_dwordx4 v4, v[0:3], s[16:17]
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
@@ -9813,8 +9718,7 @@ define void @v_shuffle_v4p3_v4p3__5_6_6_6(ptr addrspace(1) inreg %ptr) {
 ; GFX940-NEXT:    ; def v[0:3]
 ; GFX940-NEXT:    ;;#ASMEND
 ; GFX940-NEXT:    v_mov_b32_e32 v4, 0
-; GFX940-NEXT:    v_mov_b32_e32 v0, v1
-; GFX940-NEXT:    v_mov_b32_e32 v1, v2
+; GFX940-NEXT:    v_pk_mov_b32 v[0:1], v[0:1], v[2:3] op_sel:[1,0]
 ; GFX940-NEXT:    v_mov_b32_e32 v3, v2
 ; GFX940-NEXT:    global_store_dwordx4 v4, v[0:3], s[0:1] sc0 sc1
 ; GFX940-NEXT:    s_waitcnt vmcnt(0)
@@ -9897,8 +9801,7 @@ define void @v_shuffle_v4p3_v4p3__7_6_6_6(ptr addrspace(1) inreg %ptr) {
 ; GFX90A-NEXT:    ; def v[0:3]
 ; GFX90A-NEXT:    ;;#ASMEND
 ; GFX90A-NEXT:    v_mov_b32_e32 v4, 0
-; GFX90A-NEXT:    v_mov_b32_e32 v0, v3
-; GFX90A-NEXT:    v_mov_b32_e32 v1, v2
+; GFX90A-NEXT:    v_pk_mov_b32 v[0:1], v[2:3], v[2:3] op_sel:[1,0]
 ; GFX90A-NEXT:    v_mov_b32_e32 v3, v2
 ; GFX90A-NEXT:    global_store_dwordx4 v4, v[0:3], s[16:17]
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
@@ -9911,8 +9814,7 @@ define void @v_shuffle_v4p3_v4p3__7_6_6_6(ptr addrspace(1) inreg %ptr) {
 ; GFX940-NEXT:    ; def v[0:3]
 ; GFX940-NEXT:    ;;#ASMEND
 ; GFX940-NEXT:    v_mov_b32_e32 v4, 0
-; GFX940-NEXT:    v_mov_b32_e32 v0, v3
-; GFX940-NEXT:    v_mov_b32_e32 v1, v2
+; GFX940-NEXT:    v_pk_mov_b32 v[0:1], v[2:3], v[2:3] op_sel:[1,0]
 ; GFX940-NEXT:    v_mov_b32_e32 v3, v2
 ; GFX940-NEXT:    global_store_dwordx4 v4, v[0:3], s[0:1] sc0 sc1
 ; GFX940-NEXT:    s_waitcnt vmcnt(0)
@@ -9998,8 +9900,7 @@ define void @v_shuffle_v4p3_v4p3__7_0_6_6(ptr addrspace(1) inreg %ptr) {
 ; GFX90A-NEXT:    ; def v[2:5]
 ; GFX90A-NEXT:    ;;#ASMEND
 ; GFX90A-NEXT:    v_mov_b32_e32 v6, 0
-; GFX90A-NEXT:    v_mov_b32_e32 v2, v5
-; GFX90A-NEXT:    v_mov_b32_e32 v3, v0
+; GFX90A-NEXT:    v_pk_mov_b32 v[2:3], v[4:5], v[0:1] op_sel:[1,0]
 ; GFX90A-NEXT:    v_mov_b32_e32 v5, v4
 ; GFX90A-NEXT:    global_store_dwordx4 v6, v[2:5], s[16:17]
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
@@ -10016,8 +9917,7 @@ define void @v_shuffle_v4p3_v4p3__7_0_6_6(ptr addrspace(1) inreg %ptr) {
 ; GFX940-NEXT:    ; def v[2:5]
 ; GFX940-NEXT:    ;;#ASMEND
 ; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_mov_b32_e32 v2, v5
-; GFX940-NEXT:    v_mov_b32_e32 v3, v0
+; GFX940-NEXT:    v_pk_mov_b32 v[2:3], v[4:5], v[0:1] op_sel:[1,0]
 ; GFX940-NEXT:    v_mov_b32_e32 v5, v4
 ; GFX940-NEXT:    global_store_dwordx4 v6, v[2:5], s[0:1] sc0 sc1
 ; GFX940-NEXT:    s_waitcnt vmcnt(0)
@@ -10116,8 +10016,7 @@ define void @v_shuffle_v4p3_v4p3__7_2_6_6(ptr addrspace(1) inreg %ptr) {
 ; GFX90A-NEXT:    ;;#ASMSTART
 ; GFX90A-NEXT:    ; def v[0:3]
 ; GFX90A-NEXT:    ;;#ASMEND
-; GFX90A-NEXT:    v_mov_b32_e32 v4, v7
-; GFX90A-NEXT:    v_mov_b32_e32 v5, v2
+; GFX90A-NEXT:    v_pk_mov_b32 v[4:5], v[6:7], v[2:3] op_sel:[1,0]
 ; GFX90A-NEXT:    v_mov_b32_e32 v7, v6
 ; GFX90A-NEXT:    global_store_dwordx4 v8, v[4:7], s[16:17]
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
@@ -10133,8 +10032,8 @@ define void @v_shuffle_v4p3_v4p3__7_2_6_6(ptr addrspace(1) inreg %ptr) {
 ; GFX940-NEXT:    ;;#ASMSTART
 ; GFX940-NEXT:    ; def v[0:3]
 ; GFX940-NEXT:    ;;#ASMEND
-; GFX940-NEXT:    v_mov_b32_e32 v4, v7
-; GFX940-NEXT:    v_mov_b32_e32 v5, v2
+; GFX940-NEXT:    s_nop 0
+; GFX940-NEXT:    v_pk_mov_b32 v[4:5], v[6:7], v[2:3] op_sel:[1,0]
 ; GFX940-NEXT:    v_mov_b32_e32 v7, v6
 ; GFX940-NEXT:    global_store_dwordx4 v8, v[4:7], s[0:1] sc0 sc1
 ; GFX940-NEXT:    s_waitcnt vmcnt(0)
@@ -10222,31 +10121,26 @@ define void @v_shuffle_v4p3_v4p3__7_4_6_6(ptr addrspace(1) inreg %ptr) {
 ; GFX90A-LABEL: v_shuffle_v4p3_v4p3__7_4_6_6:
 ; GFX90A:       ; %bb.0:
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT:    v_mov_b32_e32 v8, 0
 ; GFX90A-NEXT:    ;;#ASMSTART
 ; GFX90A-NEXT:    ; def v[0:3]
 ; GFX90A-NEXT:    ;;#ASMEND
-; GFX90A-NEXT:    v_mov_b32_e32 v4, v3
-; GFX90A-NEXT:    v_mov_b32_e32 v5, v0
-; GFX90A-NEXT:    v_mov_b32_e32 v6, v2
-; GFX90A-NEXT:    v_mov_b32_e32 v7, v2
-; GFX90A-NEXT:    global_store_dwordx4 v8, v[4:7], s[16:17]
+; GFX90A-NEXT:    v_mov_b32_e32 v4, 0
+; GFX90A-NEXT:    v_pk_mov_b32 v[0:1], v[2:3], v[0:1] op_sel:[1,0]
+; GFX90A-NEXT:    v_mov_b32_e32 v3, v2
+; GFX90A-NEXT:    global_store_dwordx4 v4, v[0:3], s[16:17]
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
 ; GFX90A-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: v_shuffle_v4p3_v4p3__7_4_6_6:
 ; GFX940:       ; %bb.0:
 ; GFX940-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX940-NEXT:    v_mov_b32_e32 v8, 0
 ; GFX940-NEXT:    ;;#ASMSTART
 ; GFX940-NEXT:    ; def v[0:3]
 ; GFX940-NEXT:    ;;#ASMEND
-; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_mov_b32_e32 v4, v3
-; GFX940-NEXT:    v_mov_b32_e32 v5, v0
-; GFX940-NEXT:    v_mov_b32_e32 v6, v2
-; GFX940-NEXT:    v_mov_b32_e32 v7, v2
-; GFX940-NEXT:    global_store_dwordx4 v8, v[4:7], s[0:1] sc0 sc1
+; GFX940-NEXT:    v_mov_b32_e32 v4, 0
+; GFX940-NEXT:    v_pk_mov_b32 v[0:1], v[2:3], v[0:1] op_sel:[1,0]
+; GFX940-NEXT:    v_mov_b32_e32 v3, v2
+; GFX940-NEXT:    global_store_dwordx4 v4, v[0:3], s[0:1] sc0 sc1
 ; GFX940-NEXT:    s_waitcnt vmcnt(0)
 ; GFX940-NEXT:    s_setpc_b64 s[30:31]
   %vec0 = call <4 x ptr addrspace(3)> asm "; def $0", "=v"()
@@ -10486,15 +10380,14 @@ define void @v_shuffle_v4p3_v4p3__7_7_1_6(ptr addrspace(1) inreg %ptr) {
 ; GFX90A-NEXT:    ;;#ASMSTART
 ; GFX90A-NEXT:    ; def v[0:3]
 ; GFX90A-NEXT:    ;;#ASMEND
-; GFX90A-NEXT:    v_mov_b32_e32 v10, 0
 ; GFX90A-NEXT:    ;;#ASMSTART
 ; GFX90A-NEXT:    ; def v[2:5]
 ; GFX90A-NEXT:    ;;#ASMEND
-; GFX90A-NEXT:    v_mov_b32_e32 v6, v5
-; GFX90A-NEXT:    v_mov_b32_e32 v7, v5
-; GFX90A-NEXT:    v_mov_b32_e32 v8, v1
-; GFX90A-NEXT:    v_mov_b32_e32 v9, v4
-; GFX90A-NEXT:    global_store_dwordx4 v10, v[6:9], s[16:17]
+; GFX90A-NEXT:    v_mov_b32_e32 v6, 0
+; GFX90A-NEXT:    v_pk_mov_b32 v[2:3], v[0:1], v[4:5] op_sel:[1,0]
+; GFX90A-NEXT:    v_mov_b32_e32 v0, v5
+; GFX90A-NEXT:    v_mov_b32_e32 v1, v5
+; GFX90A-NEXT:    global_store_dwordx4 v6, v[0:3], s[16:17]
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
 ; GFX90A-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -10504,15 +10397,15 @@ define void @v_shuffle_v4p3_v4p3__7_7_1_6(ptr addrspace(1) inreg %ptr) {
 ; GFX940-NEXT:    ;;#ASMSTART
 ; GFX940-NEXT:    ; def v[0:3]
 ; GFX940-NEXT:    ;;#ASMEND
-; GFX940-NEXT:    v_mov_b32_e32 v10, 0
+; GFX940-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX940-NEXT:    ;;#ASMSTART
 ; GFX940-NEXT:    ; def v[2:5]
 ; GFX940-NEXT:    ;;#ASMEND
-; GFX940-NEXT:    v_mov_b32_e32 v8, v1
-; GFX940-NEXT:    v_mov_b32_e32 v6, v5
-; GFX940-NEXT:    v_mov_b32_e32 v7, v5
-; GFX940-NEXT:    v_mov_b32_e32 v9, v4
-; GFX940-NEXT:    global_store_dwordx4 v10, v[6:9], s[0:1] sc0 sc1
+; GFX940-NEXT:    s_nop 0
+; GFX940-NEXT:    v_pk_mov_b32 v[2:3], v[0:1], v[4:5] op_sel:[1,0]
+; GFX940-NEXT:    v_mov_b32_e32 v0, v5
+; GFX940-NEXT:    v_mov_b32_e32 v1, v5
+; GFX940-NEXT:    global_store_dwordx4 v6, v[0:3], s[0:1] sc0 sc1
 ; GFX940-NEXT:    s_waitcnt vmcnt(0)
 ; GFX940-NEXT:    s_setpc_b64 s[30:31]
   %vec0 = call <4 x ptr addrspace(3)> asm "; def $0", "=v"()
@@ -10610,10 +10503,9 @@ define void @v_shuffle_v4p3_v4p3__7_7_3_6(ptr addrspace(1) inreg %ptr) {
 ; GFX90A-NEXT:    ;;#ASMSTART
 ; GFX90A-NEXT:    ; def v[4:7]
 ; GFX90A-NEXT:    ;;#ASMEND
+; GFX90A-NEXT:    v_pk_mov_b32 v[2:3], v[2:3], v[6:7] op_sel:[1,0]
 ; GFX90A-NEXT:    v_mov_b32_e32 v0, v7
 ; GFX90A-NEXT:    v_mov_b32_e32 v1, v7
-; GFX90A-NEXT:    v_mov_b32_e32 v2, v3
-; GFX90A-NEXT:    v_mov_b32_e32 v3, v6
 ; GFX90A-NEXT:    global_store_dwordx4 v8, v[0:3], s[16:17]
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
 ; GFX90A-NEXT:    s_setpc_b64 s[30:31]
@@ -10628,10 +10520,10 @@ define void @v_shuffle_v4p3_v4p3__7_7_3_6(ptr addrspace(1) inreg %ptr) {
 ; GFX940-NEXT:    ;;#ASMSTART
 ; GFX940-NEXT:    ; def v[4:7]
 ; GFX940-NEXT:    ;;#ASMEND
-; GFX940-NEXT:    v_mov_b32_e32 v2, v3
+; GFX940-NEXT:    s_nop 0
+; GFX940-NEXT:    v_pk_mov_b32 v[2:3], v[2:3], v[6:7] op_sel:[1,0]
 ; GFX940-NEXT:    v_mov_b32_e32 v0, v7
 ; GFX940-NEXT:    v_mov_b32_e32 v1, v7
-; GFX940-NEXT:    v_mov_b32_e32 v3, v6
 ; GFX940-NEXT:    global_store_dwordx4 v8, v[0:3], s[0:1] sc0 sc1
 ; GFX940-NEXT:    s_waitcnt vmcnt(0)
 ; GFX940-NEXT:    s_setpc_b64 s[30:31]
@@ -10714,31 +10606,26 @@ define void @v_shuffle_v4p3_v4p3__7_7_5_6(ptr addrspace(1) inreg %ptr) {
 ; GFX90A-LABEL: v_shuffle_v4p3_v4p3__7_7_5_6:
 ; GFX90A:       ; %bb.0:
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT:    v_mov_b32_e32 v8, 0
 ; GFX90A-NEXT:    ;;#ASMSTART
 ; GFX90A-NEXT:    ; def v[0:3]
 ; GFX90A-NEXT:    ;;#ASMEND
-; GFX90A-NEXT:    v_mov_b32_e32 v4, v3
-; GFX90A-NEXT:    v_mov_b32_e32 v5, v3
-; GFX90A-NEXT:    v_mov_b32_e32 v6, v1
-; GFX90A-NEXT:    v_mov_b32_e32 v7, v2
-; GFX90A-NEXT:    global_store_dwordx4 v8, v[4:7], s[16:17]
+; GFX90A-NEXT:    v_mov_b32_e32 v6, 0
+; GFX90A-NEXT:    v_pk_mov_b32 v[4:5], v[0:1], v[2:3] op_sel:[1,0]
+; GFX90A-NEXT:    v_mov_b32_e32 v2, v3
+; GFX90A-NEXT:    global_store_dwordx4 v6, v[2:5], s[16:17]
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
 ; GFX90A-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: v_shuffle_v4p3_v4p3__7_7_5_6:
 ; GFX940:       ; %bb.0:
 ; GFX940-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX940-NEXT:    v_mov_b32_e32 v8, 0
 ; GFX940-NEXT:    ;;#ASMSTART
 ; GFX940-NEXT:    ; def v[0:3]
 ; GFX940-NEXT:    ;;#ASMEND
-; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_mov_b32_e32 v4, v3
-; GFX940-NEXT:    v_mov_b32_e32 v5, v3
-; GFX940-NEXT:    v_mov_b32_e32 v6, v1
-; GFX940-NEXT:    v_mov_b32_e32 v7, v2
-; GFX940-NEXT:    global_store_dwordx4 v8, v[4:7], s[0:1] sc0 sc1
+; GFX940-NEXT:    v_mov_b32_e32 v6, 0
+; GFX940-NEXT:    v_pk_mov_b32 v[4:5], v[0:1], v[2:3] op_sel:[1,0]
+; GFX940-NEXT:    v_mov_b32_e32 v2, v3
+; GFX940-NEXT:    global_store_dwordx4 v6, v[2:5], s[0:1] sc0 sc1
 ; GFX940-NEXT:    s_waitcnt vmcnt(0)
 ; GFX940-NEXT:    s_setpc_b64 s[30:31]
   %vec0 = call <4 x ptr addrspace(3)> asm "; def $0", "=v"()
@@ -11246,8 +11133,7 @@ define void @v_shuffle_v4p3_v4p3__7_0_7_7(ptr addrspace(1) inreg %ptr) {
 ; GFX90A-NEXT:    ; def v[2:5]
 ; GFX90A-NEXT:    ;;#ASMEND
 ; GFX90A-NEXT:    v_mov_b32_e32 v6, 0
-; GFX90A-NEXT:    v_mov_b32_e32 v2, v5
-; GFX90A-NEXT:    v_mov_b32_e32 v3, v0
+; GFX90A-NEXT:    v_pk_mov_b32 v[2:3], v[4:5], v[0:1] op_sel:[1,0]
 ; GFX90A-NEXT:    v_mov_b32_e32 v4, v5
 ; GFX90A-NEXT:    global_store_dwordx4 v6, v[2:5], s[16:17]
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
@@ -11264,8 +11150,7 @@ define void @v_shuffle_v4p3_v4p3__7_0_7_7(ptr addrspace(1) inreg %ptr) {
 ; GFX940-NEXT:    ; def v[2:5]
 ; GFX940-NEXT:    ;;#ASMEND
 ; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_mov_b32_e32 v2, v5
-; GFX940-NEXT:    v_mov_b32_e32 v3, v0
+; GFX940-NEXT:    v_pk_mov_b32 v[2:3], v[4:5], v[0:1] op_sel:[1,0]
 ; GFX940-NEXT:    v_mov_b32_e32 v4, v5
 ; GFX940-NEXT:    global_store_dwordx4 v6, v[2:5], s[0:1] sc0 sc1
 ; GFX940-NEXT:    s_waitcnt vmcnt(0)
@@ -11364,8 +11249,7 @@ define void @v_shuffle_v4p3_v4p3__7_2_7_7(ptr addrspace(1) inreg %ptr) {
 ; GFX90A-NEXT:    ;;#ASMSTART
 ; GFX90A-NEXT:    ; def v[0:3]
 ; GFX90A-NEXT:    ;;#ASMEND
-; GFX90A-NEXT:    v_mov_b32_e32 v4, v7
-; GFX90A-NEXT:    v_mov_b32_e32 v5, v2
+; GFX90A-NEXT:    v_pk_mov_b32 v[4:5], v[6:7], v[2:3] op_sel:[1,0]
 ; GFX90A-NEXT:    v_mov_b32_e32 v6, v7
 ; GFX90A-NEXT:    global_store_dwordx4 v8, v[4:7], s[16:17]
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
@@ -11381,8 +11265,8 @@ define void @v_shuffle_v4p3_v4p3__7_2_7_7(ptr addrspace(1) inreg %ptr) {
 ; GFX940-NEXT:    ;;#ASMSTART
 ; GFX940-NEXT:    ; def v[0:3]
 ; GFX940-NEXT:    ;;#ASMEND
-; GFX940-NEXT:    v_mov_b32_e32 v4, v7
-; GFX940-NEXT:    v_mov_b32_e32 v5, v2
+; GFX940-NEXT:    s_nop 0
+; GFX940-NEXT:    v_pk_mov_b32 v[4:5], v[6:7], v[2:3] op_sel:[1,0]
 ; GFX940-NEXT:    v_mov_b32_e32 v6, v7
 ; GFX940-NEXT:    global_store_dwordx4 v8, v[4:7], s[0:1] sc0 sc1
 ; GFX940-NEXT:    s_waitcnt vmcnt(0)
@@ -11471,31 +11355,26 @@ define void @v_shuffle_v4p3_v4p3__7_4_7_7(ptr addrspace(1) inreg %ptr) {
 ; GFX90A-LABEL: v_shuffle_v4p3_v4p3__7_4_7_7:
 ; GFX90A:       ; %bb.0:
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT:    v_mov_b32_e32 v8, 0
 ; GFX90A-NEXT:    ;;#ASMSTART
 ; GFX90A-NEXT:    ; def v[0:3]
 ; GFX90A-NEXT:    ;;#ASMEND
-; GFX90A-NEXT:    v_mov_b32_e32 v4, v3
-; GFX90A-NEXT:    v_mov_b32_e32 v5, v0
-; GFX90A-NEXT:    v_mov_b32_e32 v6, v3
-; GFX90A-NEXT:    v_mov_b32_e32 v7, v3
-; GFX90A-NEXT:    global_store_dwordx4 v8, v[4:7], s[16:17]
+; GFX90A-NEXT:    v_mov_b32_e32 v4, 0
+; GFX90A-NEXT:    v_pk_mov_b32 v[0:1], v[2:3], v[0:1] op_sel:[1,0]
+; GFX90A-NEXT:    v_mov_b32_e32 v2, v3
+; GFX90A-NEXT:    global_store_dwordx4 v4, v[0:3], s[16:17]
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
 ; GFX90A-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: v_shuffle_v4p3_v4p3__7_4_7_7:
 ; GFX940:       ; %bb.0:
 ; GFX940-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX940-NEXT:    v_mov_b32_e32 v8, 0
 ; GFX940-NEXT:    ;;#ASMSTART
 ; GFX940-NEXT:    ; def v[0:3]
 ; GFX940-NEXT:    ;;#ASMEND
-; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_mov_b32_e32 v4, v3
-; GFX940-NEXT:    v_mov_b32_e32 v5, v0
-; GFX940-NEXT:    v_mov_b32_e32 v6, v3
-; GFX940-NEXT:    v_mov_b32_e32 v7, v3
-; GFX940-NEXT:    global_store_dwordx4 v8, v[4:7], s[0:1] sc0 sc1
+; GFX940-NEXT:    v_mov_b32_e32 v4, 0
+; GFX940-NEXT:    v_pk_mov_b32 v[0:1], v[2:3], v[0:1] op_sel:[1,0]
+; GFX940-NEXT:    v_mov_b32_e32 v2, v3
+; GFX940-NEXT:    global_store_dwordx4 v4, v[0:3], s[0:1] sc0 sc1
 ; GFX940-NEXT:    s_waitcnt vmcnt(0)
 ; GFX940-NEXT:    s_setpc_b64 s[30:31]
   %vec0 = call <4 x ptr addrspace(3)> asm "; def $0", "=v"()
@@ -11573,8 +11452,7 @@ define void @v_shuffle_v4p3_v4p3__7_6_7_7(ptr addrspace(1) inreg %ptr) {
 ; GFX90A-NEXT:    ; def v[0:3]
 ; GFX90A-NEXT:    ;;#ASMEND
 ; GFX90A-NEXT:    v_mov_b32_e32 v4, 0
-; GFX90A-NEXT:    v_mov_b32_e32 v0, v3
-; GFX90A-NEXT:    v_mov_b32_e32 v1, v2
+; GFX90A-NEXT:    v_pk_mov_b32 v[0:1], v[2:3], v[2:3] op_sel:[1,0]
 ; GFX90A-NEXT:    v_mov_b32_e32 v2, v3
 ; GFX90A-NEXT:    global_store_dwordx4 v4, v[0:3], s[16:17]
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
@@ -11587,8 +11465,7 @@ define void @v_shuffle_v4p3_v4p3__7_6_7_7(ptr addrspace(1) inreg %ptr) {
 ; GFX940-NEXT:    ; def v[0:3]
 ; GFX940-NEXT:    ;;#ASMEND
 ; GFX940-NEXT:    v_mov_b32_e32 v4, 0
-; GFX940-NEXT:    v_mov_b32_e32 v0, v3
-; GFX940-NEXT:    v_mov_b32_e32 v1, v2
+; GFX940-NEXT:    v_pk_mov_b32 v[0:1], v[2:3], v[2:3] op_sel:[1,0]
 ; GFX940-NEXT:    v_mov_b32_e32 v2, v3
 ; GFX940-NEXT:    global_store_dwordx4 v4, v[0:3], s[0:1] sc0 sc1
 ; GFX940-NEXT:    s_waitcnt vmcnt(0)
diff --git a/llvm/test/CodeGen/AMDGPU/si-init-whole-wave.mir b/llvm/test/CodeGen/AMDGPU/si-init-whole-wave.mir
index a4a9c04bb0c6a..c02301446861d 100644
--- a/llvm/test/CodeGen/AMDGPU/si-init-whole-wave.mir
+++ b/llvm/test/CodeGen/AMDGPU/si-init-whole-wave.mir
@@ -1,5 +1,6 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 5
 # RUN: llc -mtriple=amdgcn -mcpu=gfx1200 -verify-machineinstrs -run-pass si-wqm -o -  %s | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=gfx1200 -passes=si-wqm -o -  %s | FileCheck %s
 
 ---
 # Test that we don't do silly things when there is no whole wave mode in the
diff --git a/llvm/test/CodeGen/AMDGPU/sibling-call.ll b/llvm/test/CodeGen/AMDGPU/sibling-call.ll
index 0676bc79a46f5..b57adfe7d9306 100644
--- a/llvm/test/CodeGen/AMDGPU/sibling-call.ll
+++ b/llvm/test/CodeGen/AMDGPU/sibling-call.ll
@@ -1,29 +1,59 @@
-; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=fiji -mattr=-flat-for-global -enable-ipra=0 -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,CIVI %s
-; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=hawaii -enable-ipra=0 -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,CIVI %s
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=fiji -mattr=-flat-for-global -enable-ipra=0 -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,FIJI %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=hawaii -enable-ipra=0 -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,HAWAII %s
 ; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -mattr=-flat-for-global -enable-ipra=0 -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GFX9 %s
 target datalayout = "A5"
 
 ; FIXME: Why is this commuted only sometimes?
-; GCN-LABEL: {{^}}i32_fastcc_i32_i32:
-; GCN: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CIVI-NEXT: v_add_{{i|u}}32_e32 v0, vcc, v0, v1
-; GFX9-NEXT: v_add_u32_e32 v0, v0, v1
-; GCN-NEXT: s_setpc_b64
 define fastcc i32 @i32_fastcc_i32_i32(i32 %arg0, i32 %arg1) #1 {
+; FIJI-LABEL: i32_fastcc_i32_i32:
+; FIJI:       ; %bb.0:
+; FIJI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; FIJI-NEXT:    v_add_u32_e32 v0, vcc, v0, v1
+; FIJI-NEXT:    s_setpc_b64 s[30:31]
+;
+; HAWAII-LABEL: i32_fastcc_i32_i32:
+; HAWAII:       ; %bb.0:
+; HAWAII-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; HAWAII-NEXT:    v_add_i32_e32 v0, vcc, v0, v1
+; HAWAII-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: i32_fastcc_i32_i32:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_add_u32_e32 v0, v0, v1
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
   %add0 = add i32 %arg0, %arg1
   ret i32 %add0
 }
 
-; GCN-LABEL: {{^}}i32_fastcc_i32_i32_stack_object:
-; GCN: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-NEXT: v_mov_b32_e32 [[K:v[0-9]+]], 9
-; CIVI-NEXT: v_add_{{i|u}}32_e32 v0, vcc, v0, v1
-; GFX9-NEXT: v_add_u32_e32 v0, v0, v1
-; GCN: buffer_store_dword [[K]], off, s[0:3], s32 offset:20
-; GCN: s_waitcnt vmcnt(0)
-; GCN: s_setpc_b64
-; GCN: ; ScratchSize: 68
 define fastcc i32 @i32_fastcc_i32_i32_stack_object(i32 %arg0, i32 %arg1) #1 {
+; FIJI-LABEL: i32_fastcc_i32_i32_stack_object:
+; FIJI:       ; %bb.0:
+; FIJI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; FIJI-NEXT:    v_mov_b32_e32 v2, 9
+; FIJI-NEXT:    v_add_u32_e32 v0, vcc, v0, v1
+; FIJI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:20
+; FIJI-NEXT:    s_waitcnt vmcnt(0)
+; FIJI-NEXT:    s_setpc_b64 s[30:31]
+;
+; HAWAII-LABEL: i32_fastcc_i32_i32_stack_object:
+; HAWAII:       ; %bb.0:
+; HAWAII-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; HAWAII-NEXT:    v_mov_b32_e32 v2, 9
+; HAWAII-NEXT:    v_add_i32_e32 v0, vcc, v0, v1
+; HAWAII-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:20
+; HAWAII-NEXT:    s_waitcnt vmcnt(0)
+; HAWAII-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: i32_fastcc_i32_i32_stack_object:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v2, 9
+; GFX9-NEXT:    v_add_u32_e32 v0, v0, v1
+; GFX9-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:20
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
   %alloca = alloca [16 x i32], align 4, addrspace(5)
   %gep = getelementptr inbounds [16 x i32], ptr addrspace(5) %alloca, i32 0, i32 5
   store volatile i32 9, ptr addrspace(5) %gep
@@ -31,19 +61,33 @@ define fastcc i32 @i32_fastcc_i32_i32_stack_object(i32 %arg0, i32 %arg1) #1 {
   ret i32 %add0
 }
 
-; GCN-LABEL: {{^}}sibling_call_i32_fastcc_i32_i32:
 define hidden fastcc i32 @sibling_call_i32_fastcc_i32_i32(i32 %a, i32 %b, i32 %c) #1 {
+; GCN-LABEL: sibling_call_i32_fastcc_i32_i32:
+; GCN:       ; %bb.0: ; %entry
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    s_getpc_b64 s[4:5]
+; GCN-NEXT:    s_add_u32 s4, s4, i32_fastcc_i32_i32@gotpcrel32@lo+4
+; GCN-NEXT:    s_addc_u32 s5, s5, i32_fastcc_i32_i32@gotpcrel32@hi+12
+; GCN-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x0
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    s_setpc_b64 s[4:5]
 entry:
   %ret = tail call fastcc i32 @i32_fastcc_i32_i32(i32 %a, i32 %b)
   ret i32 %ret
 }
 
-; GCN-LABEL: {{^}}sibling_call_i32_fastcc_i32_i32_stack_object:
-; GCN: v_mov_b32_e32 [[NINE:v[0-9]+]], 9
-; GCN: buffer_store_dword [[NINE]], off, s[0:3], s32 offset:20
-; GCN: s_setpc_b64
-; GCN: ; ScratchSize: 68
 define fastcc i32 @sibling_call_i32_fastcc_i32_i32_stack_object(i32 %a, i32 %b, i32 %c) #1 {
+; GCN-LABEL: sibling_call_i32_fastcc_i32_i32_stack_object:
+; GCN:       ; %bb.0: ; %entry
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    s_getpc_b64 s[4:5]
+; GCN-NEXT:    s_add_u32 s4, s4, i32_fastcc_i32_i32@gotpcrel32@lo+4
+; GCN-NEXT:    s_addc_u32 s5, s5, i32_fastcc_i32_i32@gotpcrel32@hi+12
+; GCN-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x0
+; GCN-NEXT:    v_mov_b32_e32 v2, 9
+; GCN-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:20
+; GCN-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN-NEXT:    s_setpc_b64 s[4:5]
 entry:
   %alloca = alloca [16 x i32], align 4, addrspace(5)
   %gep = getelementptr inbounds [16 x i32], ptr addrspace(5) %alloca, i32 0, i32 5
@@ -52,12 +96,18 @@ entry:
   ret i32 %ret
 }
 
-; GCN-LABEL: {{^}}sibling_call_i32_fastcc_i32_i32_callee_stack_object:
-; GCN: v_mov_b32_e32 [[NINE:v[0-9]+]], 9
-; GCN: buffer_store_dword [[NINE]], off, s[0:3], s32 offset:20
-; GCN: s_setpc_b64
-; GCN: ; ScratchSize: 136
 define fastcc i32 @sibling_call_i32_fastcc_i32_i32_callee_stack_object(i32 %a, i32 %b, i32 %c) #1 {
+; GCN-LABEL: sibling_call_i32_fastcc_i32_i32_callee_stack_object:
+; GCN:       ; %bb.0: ; %entry
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    s_getpc_b64 s[4:5]
+; GCN-NEXT:    s_add_u32 s4, s4, i32_fastcc_i32_i32_stack_object@gotpcrel32@lo+4
+; GCN-NEXT:    s_addc_u32 s5, s5, i32_fastcc_i32_i32_stack_object@gotpcrel32@hi+12
+; GCN-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x0
+; GCN-NEXT:    v_mov_b32_e32 v2, 9
+; GCN-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:20
+; GCN-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN-NEXT:    s_setpc_b64 s[4:5]
 entry:
   %alloca = alloca [16 x i32], align 4, addrspace(5)
   %gep = getelementptr inbounds [16 x i32], ptr addrspace(5) %alloca, i32 0, i32 5
@@ -66,45 +116,143 @@ entry:
   ret i32 %ret
 }
 
-; GCN-LABEL: {{^}}sibling_call_i32_fastcc_i32_i32_unused_result:
 define fastcc void @sibling_call_i32_fastcc_i32_i32_unused_result(i32 %a, i32 %b, i32 %c) #1 {
+; GCN-LABEL: sibling_call_i32_fastcc_i32_i32_unused_result:
+; GCN:       ; %bb.0: ; %entry
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    s_getpc_b64 s[4:5]
+; GCN-NEXT:    s_add_u32 s4, s4, i32_fastcc_i32_i32@gotpcrel32@lo+4
+; GCN-NEXT:    s_addc_u32 s5, s5, i32_fastcc_i32_i32@gotpcrel32@hi+12
+; GCN-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x0
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    s_setpc_b64 s[4:5]
 entry:
   %ret = tail call fastcc i32 @i32_fastcc_i32_i32(i32 %a, i32 %b)
   ret void
 }
 
 ; It doesn't make sense to do a tail from a kernel
-; GCN-LABEL: {{^}}kernel_call_i32_fastcc_i32_i32_unused_result:
-;define amdgpu_kernel void @kernel_call_i32_fastcc_i32_i32_unused_result(i32 %a, i32 %b, i32 %c) #1 {
 define amdgpu_kernel void @kernel_call_i32_fastcc_i32_i32_unused_result(i32 %a, i32 %b, i32 %c) #1 {
+; FIJI-LABEL: kernel_call_i32_fastcc_i32_i32_unused_result:
+; FIJI:       ; %bb.0: ; %entry
+; FIJI-NEXT:    s_add_i32 s6, s6, s9
+; FIJI-NEXT:    s_lshr_b32 flat_scratch_hi, s6, 8
+; FIJI-NEXT:    s_add_u32 s0, s0, s9
+; FIJI-NEXT:    s_addc_u32 s1, s1, 0
+; FIJI-NEXT:    s_mov_b32 flat_scratch_lo, s7
+; FIJI-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x0
+; FIJI-NEXT:    s_getpc_b64 s[6:7]
+; FIJI-NEXT:    s_add_u32 s6, s6, i32_fastcc_i32_i32@gotpcrel32@lo+4
+; FIJI-NEXT:    s_addc_u32 s7, s7, i32_fastcc_i32_i32@gotpcrel32@hi+12
+; FIJI-NEXT:    s_load_dwordx2 s[6:7], s[6:7], 0x0
+; FIJI-NEXT:    s_mov_b32 s32, 0
+; FIJI-NEXT:    s_waitcnt lgkmcnt(0)
+; FIJI-NEXT:    v_mov_b32_e32 v0, s4
+; FIJI-NEXT:    v_mov_b32_e32 v1, s5
+; FIJI-NEXT:    s_swappc_b64 s[30:31], s[6:7]
+; FIJI-NEXT:    s_endpgm
+;
+; HAWAII-LABEL: kernel_call_i32_fastcc_i32_i32_unused_result:
+; HAWAII:       ; %bb.0: ; %entry
+; HAWAII-NEXT:    s_add_i32 s6, s6, s9
+; HAWAII-NEXT:    s_lshr_b32 flat_scratch_hi, s6, 8
+; HAWAII-NEXT:    s_add_u32 s0, s0, s9
+; HAWAII-NEXT:    s_addc_u32 s1, s1, 0
+; HAWAII-NEXT:    s_mov_b32 flat_scratch_lo, s7
+; HAWAII-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x0
+; HAWAII-NEXT:    s_getpc_b64 s[6:7]
+; HAWAII-NEXT:    s_add_u32 s6, s6, i32_fastcc_i32_i32@gotpcrel32@lo+4
+; HAWAII-NEXT:    s_addc_u32 s7, s7, i32_fastcc_i32_i32@gotpcrel32@hi+12
+; HAWAII-NEXT:    s_load_dwordx2 s[6:7], s[6:7], 0x0
+; HAWAII-NEXT:    s_mov_b32 s32, 0
+; HAWAII-NEXT:    s_waitcnt lgkmcnt(0)
+; HAWAII-NEXT:    v_mov_b32_e32 v0, s4
+; HAWAII-NEXT:    v_mov_b32_e32 v1, s5
+; HAWAII-NEXT:    s_swappc_b64 s[30:31], s[6:7]
+; HAWAII-NEXT:    s_endpgm
+;
+; GFX9-LABEL: kernel_call_i32_fastcc_i32_i32_unused_result:
+; GFX9:       ; %bb.0: ; %entry
+; GFX9-NEXT:    s_add_u32 flat_scratch_lo, s6, s9
+; GFX9-NEXT:    s_addc_u32 flat_scratch_hi, s7, 0
+; GFX9-NEXT:    s_add_u32 s0, s0, s9
+; GFX9-NEXT:    s_addc_u32 s1, s1, 0
+; GFX9-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x0
+; GFX9-NEXT:    s_getpc_b64 s[6:7]
+; GFX9-NEXT:    s_add_u32 s6, s6, i32_fastcc_i32_i32@gotpcrel32@lo+4
+; GFX9-NEXT:    s_addc_u32 s7, s7, i32_fastcc_i32_i32@gotpcrel32@hi+12
+; GFX9-NEXT:    s_load_dwordx2 s[6:7], s[6:7], 0x0
+; GFX9-NEXT:    s_mov_b32 s32, 0
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v0, s4
+; GFX9-NEXT:    v_mov_b32_e32 v1, s5
+; GFX9-NEXT:    s_swappc_b64 s[30:31], s[6:7]
+; GFX9-NEXT:    s_endpgm
 entry:
   %ret = tail call fastcc i32 @i32_fastcc_i32_i32(i32 %a, i32 %b)
   ret void
 }
 
-; GCN-LABEL: {{^}}i32_fastcc_i32_byval_i32:
-; GCN: s_waitcnt
-; GCN-NEXT: buffer_load_dword v1, off, s[0:3], s32{{$}}
-; GCN-NEXT: s_waitcnt vmcnt(0)
-
-; CIVI-NEXT: v_add_{{i|u}}32_e32 v0, vcc, v0, v1
-; GFX9-NEXT: v_add_u32_e32 v0, v0, v1
-
-; GCN-NEXT: s_setpc_b64 s[30:31]
 define hidden fastcc i32 @i32_fastcc_i32_byval_i32(i32 %arg0, ptr addrspace(5) byval(i32) align 4 %arg1) #1 {
+; FIJI-LABEL: i32_fastcc_i32_byval_i32:
+; FIJI:       ; %bb.0:
+; FIJI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; FIJI-NEXT:    buffer_load_dword v1, off, s[0:3], s32
+; FIJI-NEXT:    s_waitcnt vmcnt(0)
+; FIJI-NEXT:    v_add_u32_e32 v0, vcc, v0, v1
+; FIJI-NEXT:    s_setpc_b64 s[30:31]
+;
+; HAWAII-LABEL: i32_fastcc_i32_byval_i32:
+; HAWAII:       ; %bb.0:
+; HAWAII-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; HAWAII-NEXT:    buffer_load_dword v1, off, s[0:3], s32
+; HAWAII-NEXT:    s_waitcnt vmcnt(0)
+; HAWAII-NEXT:    v_add_i32_e32 v0, vcc, v0, v1
+; HAWAII-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: i32_fastcc_i32_byval_i32:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    buffer_load_dword v1, off, s[0:3], s32
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_add_u32_e32 v0, v0, v1
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
   %arg1.load = load i32, ptr addrspace(5) %arg1, align 4
   %add0 = add i32 %arg0, %arg1.load
   ret i32 %add0
 }
 
 ; Tail call disallowed with byval in parent.
-; GCN-LABEL: {{^}}sibling_call_i32_fastcc_i32_byval_i32_byval_parent:
-; GCN-NOT: v_writelane_b32 v{{[0-9]+}}, s32
-; GCN: buffer_store_dword v{{[0-9]+}}, off, s[0:3], s32{{$}}
-; GCN: s_swappc_b64
-; GCN-NOT: v_readlane_b32 s32
-; GCN: s_setpc_b64
 define fastcc i32 @sibling_call_i32_fastcc_i32_byval_i32_byval_parent(i32 %a, ptr addrspace(5) byval(i32) %b.byval, i32 %c) #1 {
+; GCN-LABEL: sibling_call_i32_fastcc_i32_byval_i32_byval_parent:
+; GCN:       ; %bb.0: ; %entry
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    s_mov_b32 s4, s33
+; GCN-NEXT:    s_mov_b32 s33, s32
+; GCN-NEXT:    s_or_saveexec_b64 s[6:7], -1
+; GCN-NEXT:    buffer_store_dword v40, off, s[0:3], s33 offset:4 ; 4-byte Folded Spill
+; GCN-NEXT:    s_mov_b64 exec, s[6:7]
+; GCN-NEXT:    buffer_load_dword v1, off, s[0:3], s33
+; GCN-NEXT:    v_writelane_b32 v40, s4, 2
+; GCN-NEXT:    s_addk_i32 s32, 0x400
+; GCN-NEXT:    v_writelane_b32 v40, s30, 0
+; GCN-NEXT:    s_getpc_b64 s[4:5]
+; GCN-NEXT:    s_add_u32 s4, s4, i32_fastcc_i32_byval_i32@rel32@lo+4
+; GCN-NEXT:    s_addc_u32 s5, s5, i32_fastcc_i32_byval_i32@rel32@hi+12
+; GCN-NEXT:    v_writelane_b32 v40, s31, 1
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32
+; GCN-NEXT:    s_swappc_b64 s[30:31], s[4:5]
+; GCN-NEXT:    v_readlane_b32 s31, v40, 1
+; GCN-NEXT:    v_readlane_b32 s30, v40, 0
+; GCN-NEXT:    s_mov_b32 s32, s33
+; GCN-NEXT:    v_readlane_b32 s4, v40, 2
+; GCN-NEXT:    s_or_saveexec_b64 s[6:7], -1
+; GCN-NEXT:    buffer_load_dword v40, off, s[0:3], s33 offset:4 ; 4-byte Folded Reload
+; GCN-NEXT:    s_mov_b64 exec, s[6:7]
+; GCN-NEXT:    s_mov_b32 s33, s4
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    s_setpc_b64 s[30:31]
 entry:
   %ret = tail call fastcc i32 @i32_fastcc_i32_byval_i32(i32 %a, ptr addrspace(5) byval(i32) %b.byval)
   ret i32 %ret
@@ -113,34 +261,56 @@ entry:
 ; Tail call disallowed with byval in parent, not callee. The stack
 ; usage of incoming arguments must be <= the outgoing stack
 ; arguments.
-
-; GCN-LABEL: {{^}}sibling_call_i32_fastcc_i32_byval_i32:
-; GCN-NOT: v0
-; GCN-NOT: s32
-; GCN: buffer_load_dword v1, off, s[0:3], 0 offset:16
-; GCN: buffer_store_dword v1, off, s[0:3], s32{{$}}
-; GCN-NEXT: s_setpc_b64
 define fastcc i32 @sibling_call_i32_fastcc_i32_byval_i32(i32 %a, [32 x i32] %large) #1 {
+; GCN-LABEL: sibling_call_i32_fastcc_i32_byval_i32:
+; GCN:       ; %bb.0: ; %entry
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    buffer_load_dword v1, off, s[0:3], 0 offset:16
+; GCN-NEXT:    s_getpc_b64 s[4:5]
+; GCN-NEXT:    s_add_u32 s4, s4, i32_fastcc_i32_byval_i32@rel32@lo+4
+; GCN-NEXT:    s_addc_u32 s5, s5, i32_fastcc_i32_byval_i32@rel32@hi+12
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32
+; GCN-NEXT:    s_setpc_b64 s[4:5]
 entry:
   %ret = tail call fastcc i32 @i32_fastcc_i32_byval_i32(i32 %a, ptr addrspace(5) byval(i32) inttoptr (i32 16 to ptr addrspace(5)))
   ret i32 %ret
 }
 
-; GCN-LABEL: {{^}}i32_fastcc_i32_i32_a32i32:
-; GCN: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-DAG: buffer_load_dword [[LOAD_0:v[0-9]+]], off, s[0:3], s32 offset:4{{$}}
-; GCN-DAG: buffer_load_dword [[LOAD_1:v[0-9]+]], off, s[0:3], s32 offset:8{{$}}
-
-; CIVI-NEXT: v_add_{{i|u}}32_e32 v0, vcc, v0, v1
-; CIVI: v_add_{{i|u}}32_e32 v0, vcc, v0, [[LOAD_0]]
-; CIVI: v_add_{{i|u}}32_e32 v0, vcc, v0, [[LOAD_1]]
-
-
-; GFX9-NEXT: v_add_u32_e32 v0, v0, v1
-; GFX9: v_add3_u32 v0, v0, v3, v2
-
-; GCN-NEXT: s_setpc_b64
 define fastcc i32 @i32_fastcc_i32_i32_a32i32(i32 %arg0, i32 %arg1, [32 x i32] %large) #1 {
+; FIJI-LABEL: i32_fastcc_i32_i32_a32i32:
+; FIJI:       ; %bb.0:
+; FIJI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; FIJI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:4
+; FIJI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:8
+; FIJI-NEXT:    v_add_u32_e32 v0, vcc, v0, v1
+; FIJI-NEXT:    s_waitcnt vmcnt(1)
+; FIJI-NEXT:    v_add_u32_e32 v0, vcc, v0, v2
+; FIJI-NEXT:    s_waitcnt vmcnt(0)
+; FIJI-NEXT:    v_add_u32_e32 v0, vcc, v0, v3
+; FIJI-NEXT:    s_setpc_b64 s[30:31]
+;
+; HAWAII-LABEL: i32_fastcc_i32_i32_a32i32:
+; HAWAII:       ; %bb.0:
+; HAWAII-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; HAWAII-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:4
+; HAWAII-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:8
+; HAWAII-NEXT:    v_add_i32_e32 v0, vcc, v0, v1
+; HAWAII-NEXT:    s_waitcnt vmcnt(1)
+; HAWAII-NEXT:    v_add_i32_e32 v0, vcc, v0, v2
+; HAWAII-NEXT:    s_waitcnt vmcnt(0)
+; HAWAII-NEXT:    v_add_i32_e32 v0, vcc, v0, v3
+; HAWAII-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: i32_fastcc_i32_i32_a32i32:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:8
+; GFX9-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:4
+; GFX9-NEXT:    v_add_u32_e32 v0, v0, v1
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_add3_u32 v0, v0, v3, v2
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
   %val_firststack = extractvalue [32 x i32] %large, 30
   %val_laststack = extractvalue [32 x i32] %large, 31
   %add0 = add i32 %arg0, %arg1
@@ -150,31 +320,49 @@ define fastcc i32 @i32_fastcc_i32_i32_a32i32(i32 %arg0, i32 %arg1, [32 x i32] %l
 }
 
 ; FIXME: Why load and store same location for stack args?
-; GCN-LABEL: {{^}}sibling_call_i32_fastcc_i32_i32_a32i32:
-
-; GCN-DAG: buffer_load_dword [[LOAD_0:v[0-9]+]], off, s[0:3], s32{{$}}
-; GCN-DAG: buffer_load_dword [[LOAD_1:v[0-9]+]], off, s[0:3], s32 offset:4
-; GCN-DAG: buffer_load_dword [[LOAD_2:v[0-9]+]], off, s[0:3], s32 offset:8
-
-; GCN-NOT: s32
-
-; GCN-DAG: buffer_store_dword [[LOAD_0]], off, s[0:3], s32{{$}}
-; GCN-DAG: buffer_store_dword [[LOAD_1]], off, s[0:3], s32 offset:4
-; GCN-DAG: buffer_store_dword [[LOAD_2]], off, s[0:3], s32 offset:8
-
-; GCN-NOT: s32
-; GCN: s_setpc_b64
 define fastcc i32 @sibling_call_i32_fastcc_i32_i32_a32i32(i32 %a, i32 %b, [32 x i32] %c) #1 {
+; GCN-LABEL: sibling_call_i32_fastcc_i32_i32_a32i32:
+; GCN:       ; %bb.0: ; %entry
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    buffer_load_dword v31, off, s[0:3], s32
+; GCN-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:4
+; GCN-NEXT:    buffer_load_dword v33, off, s[0:3], s32 offset:8
+; GCN-NEXT:    s_getpc_b64 s[4:5]
+; GCN-NEXT:    s_add_u32 s4, s4, i32_fastcc_i32_i32_a32i32@gotpcrel32@lo+4
+; GCN-NEXT:    s_addc_u32 s5, s5, i32_fastcc_i32_i32_a32i32@gotpcrel32@hi+12
+; GCN-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x0
+; GCN-NEXT:    s_waitcnt vmcnt(2)
+; GCN-NEXT:    buffer_store_dword v31, off, s[0:3], s32
+; GCN-NEXT:    s_waitcnt vmcnt(2)
+; GCN-NEXT:    buffer_store_dword v32, off, s[0:3], s32 offset:4
+; GCN-NEXT:    s_waitcnt vmcnt(2)
+; GCN-NEXT:    buffer_store_dword v33, off, s[0:3], s32 offset:8
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    s_setpc_b64 s[4:5]
 entry:
   %ret = tail call fastcc i32 @i32_fastcc_i32_i32_a32i32(i32 %a, i32 %b, [32 x i32] %c)
   ret i32 %ret
 }
 
-; GCN-LABEL: {{^}}sibling_call_i32_fastcc_i32_i32_a32i32_stack_object:
-; GCN-DAG: v_mov_b32_e32 [[NINE:v[0-9]+]], 9
-; GCN: buffer_store_dword [[NINE]], off, s[0:3], s32 offset:32
-; GCN: s_setpc_b64
 define fastcc i32 @sibling_call_i32_fastcc_i32_i32_a32i32_stack_object(i32 %a, i32 %b, [32 x i32] %c) #1 {
+; GCN-LABEL: sibling_call_i32_fastcc_i32_i32_a32i32_stack_object:
+; GCN:       ; %bb.0: ; %entry
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    buffer_load_dword v31, off, s[0:3], s32
+; GCN-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:4
+; GCN-NEXT:    buffer_load_dword v33, off, s[0:3], s32 offset:8
+; GCN-NEXT:    s_getpc_b64 s[4:5]
+; GCN-NEXT:    s_add_u32 s4, s4, i32_fastcc_i32_i32_a32i32@gotpcrel32@lo+4
+; GCN-NEXT:    s_addc_u32 s5, s5, i32_fastcc_i32_i32_a32i32@gotpcrel32@hi+12
+; GCN-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x0
+; GCN-NEXT:    v_mov_b32_e32 v34, 9
+; GCN-NEXT:    buffer_store_dword v34, off, s[0:3], s32 offset:32
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    buffer_store_dword v31, off, s[0:3], s32
+; GCN-NEXT:    buffer_store_dword v32, off, s[0:3], s32 offset:4
+; GCN-NEXT:    buffer_store_dword v33, off, s[0:3], s32 offset:8
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    s_setpc_b64 s[4:5]
 entry:
   %alloca = alloca [16 x i32], align 4, addrspace(5)
   %gep = getelementptr inbounds [16 x i32], ptr addrspace(5) %alloca, i32 0, i32 5
@@ -186,54 +374,114 @@ entry:
 ; If the callee requires more stack argument space than the caller,
 ; don't do a tail call.
 ; TODO: Do we really need this restriction?
-
-; GCN-LABEL: {{^}}no_sibling_call_callee_more_stack_space:
-; GCN: s_swappc_b64
-; GCN: s_setpc_b64
 define fastcc i32 @no_sibling_call_callee_more_stack_space(i32 %a, i32 %b) #1 {
+; GCN-LABEL: no_sibling_call_callee_more_stack_space:
+; GCN:       ; %bb.0: ; %entry
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    s_mov_b32 s4, s33
+; GCN-NEXT:    s_mov_b32 s33, s32
+; GCN-NEXT:    s_or_saveexec_b64 s[6:7], -1
+; GCN-NEXT:    buffer_store_dword v40, off, s[0:3], s33 ; 4-byte Folded Spill
+; GCN-NEXT:    s_mov_b64 exec, s[6:7]
+; GCN-NEXT:    s_addk_i32 s32, 0x400
+; GCN-NEXT:    v_writelane_b32 v40, s4, 2
+; GCN-NEXT:    s_getpc_b64 s[4:5]
+; GCN-NEXT:    s_add_u32 s4, s4, i32_fastcc_i32_i32_a32i32@gotpcrel32@lo+4
+; GCN-NEXT:    s_addc_u32 s5, s5, i32_fastcc_i32_i32_a32i32@gotpcrel32@hi+12
+; GCN-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x0
+; GCN-NEXT:    v_mov_b32_e32 v2, 0
+; GCN-NEXT:    v_writelane_b32 v40, s30, 0
+; GCN-NEXT:    buffer_store_dword v2, off, s[0:3], s32
+; GCN-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:4
+; GCN-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:8
+; GCN-NEXT:    v_mov_b32_e32 v2, 0
+; GCN-NEXT:    v_mov_b32_e32 v3, 0
+; GCN-NEXT:    v_mov_b32_e32 v4, 0
+; GCN-NEXT:    v_mov_b32_e32 v5, 0
+; GCN-NEXT:    v_mov_b32_e32 v6, 0
+; GCN-NEXT:    v_mov_b32_e32 v7, 0
+; GCN-NEXT:    v_mov_b32_e32 v8, 0
+; GCN-NEXT:    v_mov_b32_e32 v9, 0
+; GCN-NEXT:    v_mov_b32_e32 v10, 0
+; GCN-NEXT:    v_mov_b32_e32 v11, 0
+; GCN-NEXT:    v_mov_b32_e32 v12, 0
+; GCN-NEXT:    v_mov_b32_e32 v13, 0
+; GCN-NEXT:    v_mov_b32_e32 v14, 0
+; GCN-NEXT:    v_mov_b32_e32 v15, 0
+; GCN-NEXT:    v_mov_b32_e32 v16, 0
+; GCN-NEXT:    v_mov_b32_e32 v17, 0
+; GCN-NEXT:    v_mov_b32_e32 v18, 0
+; GCN-NEXT:    v_mov_b32_e32 v19, 0
+; GCN-NEXT:    v_mov_b32_e32 v20, 0
+; GCN-NEXT:    v_mov_b32_e32 v21, 0
+; GCN-NEXT:    v_mov_b32_e32 v22, 0
+; GCN-NEXT:    v_mov_b32_e32 v23, 0
+; GCN-NEXT:    v_mov_b32_e32 v24, 0
+; GCN-NEXT:    v_mov_b32_e32 v25, 0
+; GCN-NEXT:    v_mov_b32_e32 v26, 0
+; GCN-NEXT:    v_mov_b32_e32 v27, 0
+; GCN-NEXT:    v_mov_b32_e32 v28, 0
+; GCN-NEXT:    v_mov_b32_e32 v29, 0
+; GCN-NEXT:    v_mov_b32_e32 v30, 0
+; GCN-NEXT:    v_writelane_b32 v40, s31, 1
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    s_swappc_b64 s[30:31], s[4:5]
+; GCN-NEXT:    v_readlane_b32 s31, v40, 1
+; GCN-NEXT:    v_readlane_b32 s30, v40, 0
+; GCN-NEXT:    s_mov_b32 s32, s33
+; GCN-NEXT:    v_readlane_b32 s4, v40, 2
+; GCN-NEXT:    s_or_saveexec_b64 s[6:7], -1
+; GCN-NEXT:    buffer_load_dword v40, off, s[0:3], s33 ; 4-byte Folded Reload
+; GCN-NEXT:    s_mov_b64 exec, s[6:7]
+; GCN-NEXT:    s_mov_b32 s33, s4
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    s_setpc_b64 s[30:31]
 entry:
   %ret = tail call fastcc i32 @i32_fastcc_i32_i32_a32i32(i32 %a, i32 %b, [32 x i32] zeroinitializer)
   ret i32 %ret
 }
 
 ; Have another non-tail in the function
-; GCN-LABEL: {{^}}sibling_call_i32_fastcc_i32_i32_other_call:
-; GCN: s_mov_b32 [[FP_SCRATCH_COPY:s[0-9]+]], s33
-; GCN-NEXT: s_mov_b32 s33, s32
-; GCN-NEXT: s_or_saveexec_b64 s{{\[[0-9]+:[0-9]+\]}}, -1
-; GCN-NEXT: buffer_store_dword [[CSRV:v[0-9]+]], off, s[0:3], s33 offset:8 ; 4-byte Folded Spill
-; GCN-NEXT: s_mov_b64 exec
-; GCN-DAG: s_addk_i32 s32, 0x400
-; GCN: v_writelane_b32 [[CSRV]], [[FP_SCRATCH_COPY]], 2
-
-; GCN-DAG: s_getpc_b64 s[4:5]
-; GCN-DAG: s_add_u32 s4, s4, i32_fastcc_i32_i32@gotpcrel32@lo+4
-; GCN-DAG: s_addc_u32 s5, s5, i32_fastcc_i32_i32@gotpcrel32@hi+12
-
-; GCN-DAG: v_writelane_b32 [[CSRV]], s30, 0
-; GCN-DAG: buffer_store_dword v40, off, s[0:3], s33 offset:4 ; 4-byte Folded Spill
-; GCN-DAG: buffer_store_dword v41, off, s[0:3], s33 ; 4-byte Folded Spill
-; GCN-DAG: v_writelane_b32 [[CSRV]], s31, 1
-
-
-; GCN: s_swappc_b64
-
-; GCN-DAG: buffer_load_dword v41, off, s[0:3], s33 ; 4-byte Folded Reload
-; GCN-DAG: buffer_load_dword v40, off, s[0:3], s33 offset:4 ; 4-byte Folded Reload
-
-; GCN: s_getpc_b64 s[4:5]
-; GCN-NEXT: s_add_u32 s4, s4, sibling_call_i32_fastcc_i32_i32@rel32@lo+4
-; GCN-NEXT: s_addc_u32 s5, s5, sibling_call_i32_fastcc_i32_i32@rel32@hi+12
-; GCN-NEXT: v_readlane_b32 s31, [[CSRV]], 1
-; GCN-NEXT: v_readlane_b32 s30, [[CSRV]], 0
-; GCN-NEXT: s_mov_b32 s32, s33
-; GCN-NEXT: v_readlane_b32 [[FP_SCRATCH_COPY:s[0-9]+]], [[CSRV]], 2
-; GCN-NEXT: s_or_saveexec_b64 s[8:9], -1
-; GCN-NEXT: buffer_load_dword [[CSRV]], off, s[0:3], s33 offset:8 ; 4-byte Folded Reload
-; GCN-NEXT: s_mov_b64 exec, s[8:9]
-; GCN-NEXT: s_mov_b32 s33, [[FP_SCRATCH_COPY]]
-; GCN-NEXT: s_setpc_b64 s[4:5]
 define fastcc i32 @sibling_call_i32_fastcc_i32_i32_other_call(i32 %a, i32 %b, i32 %c) #1 {
+; GCN-LABEL: sibling_call_i32_fastcc_i32_i32_other_call:
+; GCN:       ; %bb.0: ; %entry
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    s_mov_b32 s4, s33
+; GCN-NEXT:    s_mov_b32 s33, s32
+; GCN-NEXT:    s_or_saveexec_b64 s[6:7], -1
+; GCN-NEXT:    buffer_store_dword v42, off, s[0:3], s33 offset:8 ; 4-byte Folded Spill
+; GCN-NEXT:    s_mov_b64 exec, s[6:7]
+; GCN-NEXT:    s_addk_i32 s32, 0x400
+; GCN-NEXT:    v_writelane_b32 v42, s4, 2
+; GCN-NEXT:    s_getpc_b64 s[4:5]
+; GCN-NEXT:    s_add_u32 s4, s4, i32_fastcc_i32_i32@gotpcrel32@lo+4
+; GCN-NEXT:    s_addc_u32 s5, s5, i32_fastcc_i32_i32@gotpcrel32@hi+12
+; GCN-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x0
+; GCN-NEXT:    v_writelane_b32 v42, s30, 0
+; GCN-NEXT:    buffer_store_dword v40, off, s[0:3], s33 offset:4 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v41, off, s[0:3], s33 ; 4-byte Folded Spill
+; GCN-NEXT:    v_writelane_b32 v42, s31, 1
+; GCN-NEXT:    v_mov_b32_e32 v40, v1
+; GCN-NEXT:    v_mov_b32_e32 v41, v0
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    s_swappc_b64 s[30:31], s[4:5]
+; GCN-NEXT:    v_mov_b32_e32 v2, v0
+; GCN-NEXT:    v_mov_b32_e32 v0, v41
+; GCN-NEXT:    v_mov_b32_e32 v1, v40
+; GCN-NEXT:    buffer_load_dword v41, off, s[0:3], s33 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v40, off, s[0:3], s33 offset:4 ; 4-byte Folded Reload
+; GCN-NEXT:    s_getpc_b64 s[4:5]
+; GCN-NEXT:    s_add_u32 s4, s4, sibling_call_i32_fastcc_i32_i32@rel32@lo+4
+; GCN-NEXT:    s_addc_u32 s5, s5, sibling_call_i32_fastcc_i32_i32@rel32@hi+12
+; GCN-NEXT:    v_readlane_b32 s31, v42, 1
+; GCN-NEXT:    v_readlane_b32 s30, v42, 0
+; GCN-NEXT:    s_mov_b32 s32, s33
+; GCN-NEXT:    v_readlane_b32 s6, v42, 2
+; GCN-NEXT:    s_or_saveexec_b64 s[8:9], -1
+; GCN-NEXT:    buffer_load_dword v42, off, s[0:3], s33 offset:8 ; 4-byte Folded Reload
+; GCN-NEXT:    s_mov_b64 exec, s[8:9]
+; GCN-NEXT:    s_mov_b32 s33, s6
+; GCN-NEXT:    s_setpc_b64 s[4:5]
 entry:
   %other.call = tail call fastcc i32 @i32_fastcc_i32_i32(i32 %a, i32 %b)
   %ret = tail call fastcc i32 @sibling_call_i32_fastcc_i32_i32(i32 %a, i32 %b, i32 %other.call)
@@ -242,16 +490,25 @@ entry:
 
 ; Have stack object in caller and stack passed arguments. SP should be
 ; in same place at function exit.
-
-; GCN-LABEL: {{^}}sibling_call_stack_objecti32_fastcc_i32_i32_a32i32:
-; GCN-NOT: s33
-; GCN: buffer_load_dword v{{[0-9]+}}, off, s[0:3], s32 offset:
-
-; GCN-NOT: s33
-
-; GCN: buffer_store_dword v{{[0-9]+}}, off, s[0:3], s32 offset:
-; GCN: s_setpc_b64 s[4:5]
 define fastcc i32 @sibling_call_stack_objecti32_fastcc_i32_i32_a32i32(i32 %a, i32 %b, [32 x i32] %c) #1 {
+; GCN-LABEL: sibling_call_stack_objecti32_fastcc_i32_i32_a32i32:
+; GCN:       ; %bb.0: ; %entry
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    buffer_load_dword v31, off, s[0:3], s32
+; GCN-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:4
+; GCN-NEXT:    buffer_load_dword v33, off, s[0:3], s32 offset:8
+; GCN-NEXT:    s_getpc_b64 s[4:5]
+; GCN-NEXT:    s_add_u32 s4, s4, i32_fastcc_i32_i32_a32i32@gotpcrel32@lo+4
+; GCN-NEXT:    s_addc_u32 s5, s5, i32_fastcc_i32_i32_a32i32@gotpcrel32@hi+12
+; GCN-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x0
+; GCN-NEXT:    v_mov_b32_e32 v34, 9
+; GCN-NEXT:    buffer_store_dword v34, off, s[0:3], s32 offset:32
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    buffer_store_dword v31, off, s[0:3], s32
+; GCN-NEXT:    buffer_store_dword v32, off, s[0:3], s32 offset:4
+; GCN-NEXT:    buffer_store_dword v33, off, s[0:3], s32 offset:8
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    s_setpc_b64 s[4:5]
 entry:
   %alloca = alloca [16 x i32], align 4, addrspace(5)
   %gep = getelementptr inbounds [16 x i32], ptr addrspace(5) %alloca, i32 0, i32 5
@@ -260,13 +517,52 @@ entry:
   ret i32 %ret
 }
 
-; GCN-LABEL: {{^}}sibling_call_stack_objecti32_fastcc_i32_i32_a32i32_larger_arg_area:
-; GCN-NOT: s33
-; GCN: buffer_store_dword v{{[0-9]+}}, off, s[0:3], s32 offset:48
-
-; GCN-NOT: s33
-; GCN: s_setpc_b64 s[4:5]
 define fastcc i32 @sibling_call_stack_objecti32_fastcc_i32_i32_a32i32_larger_arg_area(i32 %a, i32 %b, [36 x i32] %c) #1 {
+; GCN-LABEL: sibling_call_stack_objecti32_fastcc_i32_i32_a32i32_larger_arg_area:
+; GCN:       ; %bb.0: ; %entry
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    s_getpc_b64 s[4:5]
+; GCN-NEXT:    s_add_u32 s4, s4, i32_fastcc_i32_i32_a32i32@gotpcrel32@lo+4
+; GCN-NEXT:    s_addc_u32 s5, s5, i32_fastcc_i32_i32_a32i32@gotpcrel32@hi+12
+; GCN-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x0
+; GCN-NEXT:    v_mov_b32_e32 v2, 9
+; GCN-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:48
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_mov_b32_e32 v2, 0
+; GCN-NEXT:    buffer_store_dword v2, off, s[0:3], s32
+; GCN-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:4
+; GCN-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:8
+; GCN-NEXT:    v_mov_b32_e32 v2, 0
+; GCN-NEXT:    v_mov_b32_e32 v3, 0
+; GCN-NEXT:    v_mov_b32_e32 v4, 0
+; GCN-NEXT:    v_mov_b32_e32 v5, 0
+; GCN-NEXT:    v_mov_b32_e32 v6, 0
+; GCN-NEXT:    v_mov_b32_e32 v7, 0
+; GCN-NEXT:    v_mov_b32_e32 v8, 0
+; GCN-NEXT:    v_mov_b32_e32 v9, 0
+; GCN-NEXT:    v_mov_b32_e32 v10, 0
+; GCN-NEXT:    v_mov_b32_e32 v11, 0
+; GCN-NEXT:    v_mov_b32_e32 v12, 0
+; GCN-NEXT:    v_mov_b32_e32 v13, 0
+; GCN-NEXT:    v_mov_b32_e32 v14, 0
+; GCN-NEXT:    v_mov_b32_e32 v15, 0
+; GCN-NEXT:    v_mov_b32_e32 v16, 0
+; GCN-NEXT:    v_mov_b32_e32 v17, 0
+; GCN-NEXT:    v_mov_b32_e32 v18, 0
+; GCN-NEXT:    v_mov_b32_e32 v19, 0
+; GCN-NEXT:    v_mov_b32_e32 v20, 0
+; GCN-NEXT:    v_mov_b32_e32 v21, 0
+; GCN-NEXT:    v_mov_b32_e32 v22, 0
+; GCN-NEXT:    v_mov_b32_e32 v23, 0
+; GCN-NEXT:    v_mov_b32_e32 v24, 0
+; GCN-NEXT:    v_mov_b32_e32 v25, 0
+; GCN-NEXT:    v_mov_b32_e32 v26, 0
+; GCN-NEXT:    v_mov_b32_e32 v27, 0
+; GCN-NEXT:    v_mov_b32_e32 v28, 0
+; GCN-NEXT:    v_mov_b32_e32 v29, 0
+; GCN-NEXT:    v_mov_b32_e32 v30, 0
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    s_setpc_b64 s[4:5]
 entry:
   %alloca = alloca [16 x i32], align 4, addrspace(5)
   %gep = getelementptr inbounds [16 x i32], ptr addrspace(5) %alloca, i32 0, i32 5
@@ -278,11 +574,18 @@ entry:
 @func_ptr_gv = external unnamed_addr addrspace(4) constant ptr, align 4
 
 ; Do support tail calls with a uniform, but unknown, callee.
-; GCN-LABEL: {{^}}indirect_uniform_sibling_call_i32_fastcc_i32_i32:
-; GCN: s_load_dwordx2 [[GV_ADDR:s\[[0-9]+:[0-9]+\]]]
-; GCN: s_load_dwordx2 [[FUNC_PTR:s\[[0-9]+:[0-9]+\]]], [[GV_ADDR]]
-; GCN: s_setpc_b64 [[FUNC_PTR]]
 define hidden fastcc i32 @indirect_uniform_sibling_call_i32_fastcc_i32_i32(i32 %a, i32 %b, i32 %c) #1 {
+; GCN-LABEL: indirect_uniform_sibling_call_i32_fastcc_i32_i32:
+; GCN:       ; %bb.0: ; %entry
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    s_getpc_b64 s[16:17]
+; GCN-NEXT:    s_add_u32 s16, s16, func_ptr_gv@gotpcrel32@lo+4
+; GCN-NEXT:    s_addc_u32 s17, s17, func_ptr_gv@gotpcrel32@hi+12
+; GCN-NEXT:    s_load_dwordx2 s[16:17], s[16:17], 0x0
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    s_load_dwordx2 s[16:17], s[16:17], 0x0
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    s_setpc_b64 s[16:17]
 entry:
   %func.ptr.load = load ptr, ptr addrspace(4) @func_ptr_gv
   %ret = tail call fastcc i32 %func.ptr.load(i32 %a, i32 %b)
@@ -291,14 +594,279 @@ entry:
 
 ; We can't support a tail call to a divergent target. Use a waterfall
 ; loop around a regular call
-; GCN-LABEL: {{^}}indirect_divergent_sibling_call_i32_fastcc_i32_i32:
-; GCN: v_readfirstlane_b32
-; GCN: v_readfirstlane_b32
-; GCN: s_and_saveexec_b64
-; GCN: s_swappc_b64
-; GCN: s_cbranch_execnz
-; GCN: s_setpc_b64
 define hidden fastcc i32 @indirect_divergent_sibling_call_i32_fastcc_i32_i32(ptr %func.ptr, i32 %a, i32 %b, i32 %c) #1 {
+; FIJI-LABEL: indirect_divergent_sibling_call_i32_fastcc_i32_i32:
+; FIJI:       ; %bb.0: ; %entry
+; FIJI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; FIJI-NEXT:    s_mov_b32 s16, s33
+; FIJI-NEXT:    s_mov_b32 s33, s32
+; FIJI-NEXT:    s_or_saveexec_b64 s[18:19], -1
+; FIJI-NEXT:    buffer_store_dword v40, off, s[0:3], s33 ; 4-byte Folded Spill
+; FIJI-NEXT:    s_mov_b64 exec, s[18:19]
+; FIJI-NEXT:    v_writelane_b32 v40, s16, 18
+; FIJI-NEXT:    v_writelane_b32 v40, s30, 0
+; FIJI-NEXT:    v_writelane_b32 v40, s31, 1
+; FIJI-NEXT:    v_writelane_b32 v40, s34, 2
+; FIJI-NEXT:    v_writelane_b32 v40, s35, 3
+; FIJI-NEXT:    v_writelane_b32 v40, s36, 4
+; FIJI-NEXT:    v_writelane_b32 v40, s37, 5
+; FIJI-NEXT:    v_writelane_b32 v40, s38, 6
+; FIJI-NEXT:    v_writelane_b32 v40, s39, 7
+; FIJI-NEXT:    v_writelane_b32 v40, s40, 8
+; FIJI-NEXT:    v_writelane_b32 v40, s41, 9
+; FIJI-NEXT:    v_writelane_b32 v40, s42, 10
+; FIJI-NEXT:    v_writelane_b32 v40, s43, 11
+; FIJI-NEXT:    v_writelane_b32 v40, s44, 12
+; FIJI-NEXT:    v_writelane_b32 v40, s45, 13
+; FIJI-NEXT:    v_writelane_b32 v40, s46, 14
+; FIJI-NEXT:    v_writelane_b32 v40, s47, 15
+; FIJI-NEXT:    v_writelane_b32 v40, s48, 16
+; FIJI-NEXT:    s_mov_b32 s42, s15
+; FIJI-NEXT:    s_mov_b32 s43, s14
+; FIJI-NEXT:    s_mov_b32 s44, s13
+; FIJI-NEXT:    s_mov_b32 s45, s12
+; FIJI-NEXT:    s_mov_b64 s[34:35], s[10:11]
+; FIJI-NEXT:    s_mov_b64 s[36:37], s[8:9]
+; FIJI-NEXT:    s_mov_b64 s[38:39], s[6:7]
+; FIJI-NEXT:    s_mov_b64 s[40:41], s[4:5]
+; FIJI-NEXT:    v_add_u32_e32 v3, vcc, v3, v4
+; FIJI-NEXT:    s_mov_b64 s[46:47], exec
+; FIJI-NEXT:    s_addk_i32 s32, 0x400
+; FIJI-NEXT:    v_writelane_b32 v40, s49, 17
+; FIJI-NEXT:  .LBB18_1: ; =>This Inner Loop Header: Depth=1
+; FIJI-NEXT:    v_readfirstlane_b32 s16, v0
+; FIJI-NEXT:    v_readfirstlane_b32 s17, v1
+; FIJI-NEXT:    v_cmp_eq_u64_e32 vcc, s[16:17], v[0:1]
+; FIJI-NEXT:    s_and_saveexec_b64 s[48:49], vcc
+; FIJI-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; FIJI-NEXT:    s_mov_b64 s[6:7], s[38:39]
+; FIJI-NEXT:    s_mov_b64 s[8:9], s[36:37]
+; FIJI-NEXT:    s_mov_b64 s[10:11], s[34:35]
+; FIJI-NEXT:    s_mov_b32 s12, s45
+; FIJI-NEXT:    s_mov_b32 s13, s44
+; FIJI-NEXT:    s_mov_b32 s14, s43
+; FIJI-NEXT:    s_mov_b32 s15, s42
+; FIJI-NEXT:    v_mov_b32_e32 v0, v2
+; FIJI-NEXT:    v_mov_b32_e32 v1, v3
+; FIJI-NEXT:    s_swappc_b64 s[30:31], s[16:17]
+; FIJI-NEXT:    v_mov_b32_e32 v4, v0
+; FIJI-NEXT:    ; implicit-def: $vgpr0_vgpr1
+; FIJI-NEXT:    ; implicit-def: $vgpr31
+; FIJI-NEXT:    ; implicit-def: $vgpr2
+; FIJI-NEXT:    ; implicit-def: $vgpr3
+; FIJI-NEXT:    s_xor_b64 exec, exec, s[48:49]
+; FIJI-NEXT:    s_cbranch_execnz .LBB18_1
+; FIJI-NEXT:  ; %bb.2:
+; FIJI-NEXT:    s_mov_b64 exec, s[46:47]
+; FIJI-NEXT:    v_mov_b32_e32 v0, v4
+; FIJI-NEXT:    v_readlane_b32 s49, v40, 17
+; FIJI-NEXT:    v_readlane_b32 s48, v40, 16
+; FIJI-NEXT:    v_readlane_b32 s47, v40, 15
+; FIJI-NEXT:    v_readlane_b32 s46, v40, 14
+; FIJI-NEXT:    v_readlane_b32 s45, v40, 13
+; FIJI-NEXT:    v_readlane_b32 s44, v40, 12
+; FIJI-NEXT:    v_readlane_b32 s43, v40, 11
+; FIJI-NEXT:    v_readlane_b32 s42, v40, 10
+; FIJI-NEXT:    v_readlane_b32 s41, v40, 9
+; FIJI-NEXT:    v_readlane_b32 s40, v40, 8
+; FIJI-NEXT:    v_readlane_b32 s39, v40, 7
+; FIJI-NEXT:    v_readlane_b32 s38, v40, 6
+; FIJI-NEXT:    v_readlane_b32 s37, v40, 5
+; FIJI-NEXT:    v_readlane_b32 s36, v40, 4
+; FIJI-NEXT:    v_readlane_b32 s35, v40, 3
+; FIJI-NEXT:    v_readlane_b32 s34, v40, 2
+; FIJI-NEXT:    v_readlane_b32 s31, v40, 1
+; FIJI-NEXT:    v_readlane_b32 s30, v40, 0
+; FIJI-NEXT:    s_mov_b32 s32, s33
+; FIJI-NEXT:    v_readlane_b32 s4, v40, 18
+; FIJI-NEXT:    s_or_saveexec_b64 s[6:7], -1
+; FIJI-NEXT:    buffer_load_dword v40, off, s[0:3], s33 ; 4-byte Folded Reload
+; FIJI-NEXT:    s_mov_b64 exec, s[6:7]
+; FIJI-NEXT:    s_mov_b32 s33, s4
+; FIJI-NEXT:    s_waitcnt vmcnt(0)
+; FIJI-NEXT:    s_setpc_b64 s[30:31]
+;
+; HAWAII-LABEL: indirect_divergent_sibling_call_i32_fastcc_i32_i32:
+; HAWAII:       ; %bb.0: ; %entry
+; HAWAII-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; HAWAII-NEXT:    s_mov_b32 s16, s33
+; HAWAII-NEXT:    s_mov_b32 s33, s32
+; HAWAII-NEXT:    s_or_saveexec_b64 s[18:19], -1
+; HAWAII-NEXT:    buffer_store_dword v40, off, s[0:3], s33 ; 4-byte Folded Spill
+; HAWAII-NEXT:    s_mov_b64 exec, s[18:19]
+; HAWAII-NEXT:    v_writelane_b32 v40, s16, 18
+; HAWAII-NEXT:    v_writelane_b32 v40, s30, 0
+; HAWAII-NEXT:    v_writelane_b32 v40, s31, 1
+; HAWAII-NEXT:    v_writelane_b32 v40, s34, 2
+; HAWAII-NEXT:    v_writelane_b32 v40, s35, 3
+; HAWAII-NEXT:    v_writelane_b32 v40, s36, 4
+; HAWAII-NEXT:    v_writelane_b32 v40, s37, 5
+; HAWAII-NEXT:    v_writelane_b32 v40, s38, 6
+; HAWAII-NEXT:    v_writelane_b32 v40, s39, 7
+; HAWAII-NEXT:    v_writelane_b32 v40, s40, 8
+; HAWAII-NEXT:    v_writelane_b32 v40, s41, 9
+; HAWAII-NEXT:    v_writelane_b32 v40, s42, 10
+; HAWAII-NEXT:    v_writelane_b32 v40, s43, 11
+; HAWAII-NEXT:    v_writelane_b32 v40, s44, 12
+; HAWAII-NEXT:    v_writelane_b32 v40, s45, 13
+; HAWAII-NEXT:    v_writelane_b32 v40, s46, 14
+; HAWAII-NEXT:    v_writelane_b32 v40, s47, 15
+; HAWAII-NEXT:    v_writelane_b32 v40, s48, 16
+; HAWAII-NEXT:    s_mov_b32 s42, s15
+; HAWAII-NEXT:    s_mov_b32 s43, s14
+; HAWAII-NEXT:    s_mov_b32 s44, s13
+; HAWAII-NEXT:    s_mov_b32 s45, s12
+; HAWAII-NEXT:    s_mov_b64 s[34:35], s[10:11]
+; HAWAII-NEXT:    s_mov_b64 s[36:37], s[8:9]
+; HAWAII-NEXT:    s_mov_b64 s[38:39], s[6:7]
+; HAWAII-NEXT:    s_mov_b64 s[40:41], s[4:5]
+; HAWAII-NEXT:    v_add_i32_e32 v3, vcc, v3, v4
+; HAWAII-NEXT:    s_mov_b64 s[46:47], exec
+; HAWAII-NEXT:    s_addk_i32 s32, 0x400
+; HAWAII-NEXT:    v_writelane_b32 v40, s49, 17
+; HAWAII-NEXT:  .LBB18_1: ; =>This Inner Loop Header: Depth=1
+; HAWAII-NEXT:    v_readfirstlane_b32 s16, v0
+; HAWAII-NEXT:    v_readfirstlane_b32 s17, v1
+; HAWAII-NEXT:    v_cmp_eq_u64_e32 vcc, s[16:17], v[0:1]
+; HAWAII-NEXT:    s_and_saveexec_b64 s[48:49], vcc
+; HAWAII-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; HAWAII-NEXT:    s_mov_b64 s[6:7], s[38:39]
+; HAWAII-NEXT:    s_mov_b64 s[8:9], s[36:37]
+; HAWAII-NEXT:    s_mov_b64 s[10:11], s[34:35]
+; HAWAII-NEXT:    s_mov_b32 s12, s45
+; HAWAII-NEXT:    s_mov_b32 s13, s44
+; HAWAII-NEXT:    s_mov_b32 s14, s43
+; HAWAII-NEXT:    s_mov_b32 s15, s42
+; HAWAII-NEXT:    v_mov_b32_e32 v0, v2
+; HAWAII-NEXT:    v_mov_b32_e32 v1, v3
+; HAWAII-NEXT:    s_swappc_b64 s[30:31], s[16:17]
+; HAWAII-NEXT:    v_mov_b32_e32 v4, v0
+; HAWAII-NEXT:    ; implicit-def: $vgpr0_vgpr1
+; HAWAII-NEXT:    ; implicit-def: $vgpr31
+; HAWAII-NEXT:    ; implicit-def: $vgpr2
+; HAWAII-NEXT:    ; implicit-def: $vgpr3
+; HAWAII-NEXT:    s_xor_b64 exec, exec, s[48:49]
+; HAWAII-NEXT:    s_cbranch_execnz .LBB18_1
+; HAWAII-NEXT:  ; %bb.2:
+; HAWAII-NEXT:    s_mov_b64 exec, s[46:47]
+; HAWAII-NEXT:    v_mov_b32_e32 v0, v4
+; HAWAII-NEXT:    v_readlane_b32 s49, v40, 17
+; HAWAII-NEXT:    v_readlane_b32 s48, v40, 16
+; HAWAII-NEXT:    v_readlane_b32 s47, v40, 15
+; HAWAII-NEXT:    v_readlane_b32 s46, v40, 14
+; HAWAII-NEXT:    v_readlane_b32 s45, v40, 13
+; HAWAII-NEXT:    v_readlane_b32 s44, v40, 12
+; HAWAII-NEXT:    v_readlane_b32 s43, v40, 11
+; HAWAII-NEXT:    v_readlane_b32 s42, v40, 10
+; HAWAII-NEXT:    v_readlane_b32 s41, v40, 9
+; HAWAII-NEXT:    v_readlane_b32 s40, v40, 8
+; HAWAII-NEXT:    v_readlane_b32 s39, v40, 7
+; HAWAII-NEXT:    v_readlane_b32 s38, v40, 6
+; HAWAII-NEXT:    v_readlane_b32 s37, v40, 5
+; HAWAII-NEXT:    v_readlane_b32 s36, v40, 4
+; HAWAII-NEXT:    v_readlane_b32 s35, v40, 3
+; HAWAII-NEXT:    v_readlane_b32 s34, v40, 2
+; HAWAII-NEXT:    v_readlane_b32 s31, v40, 1
+; HAWAII-NEXT:    v_readlane_b32 s30, v40, 0
+; HAWAII-NEXT:    s_mov_b32 s32, s33
+; HAWAII-NEXT:    v_readlane_b32 s4, v40, 18
+; HAWAII-NEXT:    s_or_saveexec_b64 s[6:7], -1
+; HAWAII-NEXT:    buffer_load_dword v40, off, s[0:3], s33 ; 4-byte Folded Reload
+; HAWAII-NEXT:    s_mov_b64 exec, s[6:7]
+; HAWAII-NEXT:    s_mov_b32 s33, s4
+; HAWAII-NEXT:    s_waitcnt vmcnt(0)
+; HAWAII-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: indirect_divergent_sibling_call_i32_fastcc_i32_i32:
+; GFX9:       ; %bb.0: ; %entry
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    s_mov_b32 s16, s33
+; GFX9-NEXT:    s_mov_b32 s33, s32
+; GFX9-NEXT:    s_or_saveexec_b64 s[18:19], -1
+; GFX9-NEXT:    buffer_store_dword v40, off, s[0:3], s33 ; 4-byte Folded Spill
+; GFX9-NEXT:    s_mov_b64 exec, s[18:19]
+; GFX9-NEXT:    v_writelane_b32 v40, s16, 18
+; GFX9-NEXT:    v_writelane_b32 v40, s30, 0
+; GFX9-NEXT:    v_writelane_b32 v40, s31, 1
+; GFX9-NEXT:    v_writelane_b32 v40, s34, 2
+; GFX9-NEXT:    v_writelane_b32 v40, s35, 3
+; GFX9-NEXT:    v_writelane_b32 v40, s36, 4
+; GFX9-NEXT:    v_writelane_b32 v40, s37, 5
+; GFX9-NEXT:    v_writelane_b32 v40, s38, 6
+; GFX9-NEXT:    v_writelane_b32 v40, s39, 7
+; GFX9-NEXT:    v_writelane_b32 v40, s40, 8
+; GFX9-NEXT:    v_writelane_b32 v40, s41, 9
+; GFX9-NEXT:    v_writelane_b32 v40, s42, 10
+; GFX9-NEXT:    v_writelane_b32 v40, s43, 11
+; GFX9-NEXT:    v_writelane_b32 v40, s44, 12
+; GFX9-NEXT:    v_writelane_b32 v40, s45, 13
+; GFX9-NEXT:    v_writelane_b32 v40, s46, 14
+; GFX9-NEXT:    v_writelane_b32 v40, s47, 15
+; GFX9-NEXT:    v_writelane_b32 v40, s48, 16
+; GFX9-NEXT:    s_mov_b32 s42, s15
+; GFX9-NEXT:    s_mov_b32 s43, s14
+; GFX9-NEXT:    s_mov_b32 s44, s13
+; GFX9-NEXT:    s_mov_b32 s45, s12
+; GFX9-NEXT:    s_mov_b64 s[34:35], s[10:11]
+; GFX9-NEXT:    s_mov_b64 s[36:37], s[8:9]
+; GFX9-NEXT:    s_mov_b64 s[38:39], s[6:7]
+; GFX9-NEXT:    s_mov_b64 s[40:41], s[4:5]
+; GFX9-NEXT:    v_add_u32_e32 v3, v3, v4
+; GFX9-NEXT:    s_mov_b64 s[46:47], exec
+; GFX9-NEXT:    s_addk_i32 s32, 0x400
+; GFX9-NEXT:    v_writelane_b32 v40, s49, 17
+; GFX9-NEXT:  .LBB18_1: ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT:    v_readfirstlane_b32 s16, v0
+; GFX9-NEXT:    v_readfirstlane_b32 s17, v1
+; GFX9-NEXT:    v_cmp_eq_u64_e32 vcc, s[16:17], v[0:1]
+; GFX9-NEXT:    s_and_saveexec_b64 s[48:49], vcc
+; GFX9-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX9-NEXT:    s_mov_b64 s[6:7], s[38:39]
+; GFX9-NEXT:    s_mov_b64 s[8:9], s[36:37]
+; GFX9-NEXT:    s_mov_b64 s[10:11], s[34:35]
+; GFX9-NEXT:    s_mov_b32 s12, s45
+; GFX9-NEXT:    s_mov_b32 s13, s44
+; GFX9-NEXT:    s_mov_b32 s14, s43
+; GFX9-NEXT:    s_mov_b32 s15, s42
+; GFX9-NEXT:    v_mov_b32_e32 v0, v2
+; GFX9-NEXT:    v_mov_b32_e32 v1, v3
+; GFX9-NEXT:    s_swappc_b64 s[30:31], s[16:17]
+; GFX9-NEXT:    v_mov_b32_e32 v4, v0
+; GFX9-NEXT:    ; implicit-def: $vgpr0_vgpr1
+; GFX9-NEXT:    ; implicit-def: $vgpr31
+; GFX9-NEXT:    ; implicit-def: $vgpr2
+; GFX9-NEXT:    ; implicit-def: $vgpr3
+; GFX9-NEXT:    s_xor_b64 exec, exec, s[48:49]
+; GFX9-NEXT:    s_cbranch_execnz .LBB18_1
+; GFX9-NEXT:  ; %bb.2:
+; GFX9-NEXT:    s_mov_b64 exec, s[46:47]
+; GFX9-NEXT:    v_mov_b32_e32 v0, v4
+; GFX9-NEXT:    v_readlane_b32 s49, v40, 17
+; GFX9-NEXT:    v_readlane_b32 s48, v40, 16
+; GFX9-NEXT:    v_readlane_b32 s47, v40, 15
+; GFX9-NEXT:    v_readlane_b32 s46, v40, 14
+; GFX9-NEXT:    v_readlane_b32 s45, v40, 13
+; GFX9-NEXT:    v_readlane_b32 s44, v40, 12
+; GFX9-NEXT:    v_readlane_b32 s43, v40, 11
+; GFX9-NEXT:    v_readlane_b32 s42, v40, 10
+; GFX9-NEXT:    v_readlane_b32 s41, v40, 9
+; GFX9-NEXT:    v_readlane_b32 s40, v40, 8
+; GFX9-NEXT:    v_readlane_b32 s39, v40, 7
+; GFX9-NEXT:    v_readlane_b32 s38, v40, 6
+; GFX9-NEXT:    v_readlane_b32 s37, v40, 5
+; GFX9-NEXT:    v_readlane_b32 s36, v40, 4
+; GFX9-NEXT:    v_readlane_b32 s35, v40, 3
+; GFX9-NEXT:    v_readlane_b32 s34, v40, 2
+; GFX9-NEXT:    v_readlane_b32 s31, v40, 1
+; GFX9-NEXT:    v_readlane_b32 s30, v40, 0
+; GFX9-NEXT:    s_mov_b32 s32, s33
+; GFX9-NEXT:    v_readlane_b32 s4, v40, 18
+; GFX9-NEXT:    s_or_saveexec_b64 s[6:7], -1
+; GFX9-NEXT:    buffer_load_dword v40, off, s[0:3], s33 ; 4-byte Folded Reload
+; GFX9-NEXT:    s_mov_b64 exec, s[6:7]
+; GFX9-NEXT:    s_mov_b32 s33, s4
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
 entry:
   %add = add i32 %b, %c
   %ret = tail call fastcc i32 %func.ptr(i32 %a, i32 %add)
@@ -307,30 +875,30 @@ entry:
 
 declare hidden void @void_fastcc_multi_byval(i32 %a, ptr addrspace(5) byval([3 x i32]) align 16, ptr addrspace(5) byval([2 x i64]))
 
-; GCN-LABEL: {{^}}sibling_call_fastcc_multi_byval:
-; GCN-DAG: s_getpc_b64 [[TARGET_ADDR:s\[[0-9]+:[0-9]+\]]]
-; GCN-DAG: v_mov_b32_e32 [[ZERO:v[0-9]+]], 0
-; GCN-DAG: v_mov_b32_e32 [[NINE:v[0-9]+]], 9
-
-; GCN-DAG: buffer_store_dword [[NINE]], off, s[0:3], s32 offset:144
-; GCN-DAG: buffer_store_dword [[NINE]], off, s[0:3], s32 offset:148
-; GCN-DAG: buffer_store_dword [[NINE]], off, s[0:3], s32 offset:152
-
-; GCN-DAG: buffer_store_dword [[NINE]], off, s[0:3], s32{{$}}
-; GCN-DAG: buffer_store_dword [[NINE]], off, s[0:3], s32 offset:4{{$}}
-; GCN-DAG: buffer_store_dword [[NINE]], off, s[0:3], s32 offset:8{{$}}
-
-; GCN-DAG: buffer_store_dword [[ZERO]], off, s[0:3], s32 offset:160
-; GCN-DAG: buffer_store_dword [[ZERO]], off, s[0:3], s32 offset:164
-; GCN-DAG: buffer_store_dword [[ZERO]], off, s[0:3], s32 offset:168
-; GCN-DAG: buffer_store_dword [[ZERO]], off, s[0:3], s32 offset:172
-; GCN-DAG: buffer_store_dword [[ZERO]], off, s[0:3], s32 offset:16{{$}}
-; GCN-DAG: buffer_store_dword [[ZERO]], off, s[0:3], s32 offset:20{{$}}
-; GCN-DAG: buffer_store_dword [[ZERO]], off, s[0:3], s32 offset:24{{$}}
-; GCN-DAG: buffer_store_dword [[ZERO]], off, s[0:3], s32 offset:28{{$}}
-
-; GCN: s_setpc_b64 [[TARGET_ADDR]]
 define fastcc void @sibling_call_fastcc_multi_byval(i32 %a, [64 x i32]) #1 {
+; GCN-LABEL: sibling_call_fastcc_multi_byval:
+; GCN:       ; %bb.0: ; %entry
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_mov_b32_e32 v1, 9
+; GCN-NEXT:    v_mov_b32_e32 v2, 0
+; GCN-NEXT:    s_getpc_b64 s[16:17]
+; GCN-NEXT:    s_add_u32 s16, s16, void_fastcc_multi_byval@rel32@lo+4
+; GCN-NEXT:    s_addc_u32 s17, s17, void_fastcc_multi_byval@rel32@hi+12
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:148
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:144
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:152
+; GCN-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:172
+; GCN-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:168
+; GCN-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:164
+; GCN-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:160
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:4
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32
+; GCN-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:28
+; GCN-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:24
+; GCN-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:20
+; GCN-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:16
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:8
+; GCN-NEXT:    s_setpc_b64 s[16:17]
 entry:
   %alloca0 = alloca [3 x i32], align 16, addrspace(5)
   %alloca1 = alloca [2 x i64], align 8, addrspace(5)
@@ -343,26 +911,55 @@ entry:
 declare hidden void @void_fastcc_byval_and_stack_passed(ptr addrspace(5) byval([3 x i32]) align 16, [32 x i32], i32)
 
 ; Callee has a byval and non-byval stack passed argument
-; GCN-LABEL: {{^}}sibling_call_byval_and_stack_passed:
-; GCN: v_mov_b32_e32 [[NINE:v[0-9]+]], 9
-
-; GCN-DAG: buffer_store_dword [[NINE]], off, s[0:3], s32 offset:144
-; GCN-DAG: buffer_store_dword [[NINE]], off, s[0:3], s32 offset:148
-; GCN-DAG: buffer_store_dword [[NINE]], off, s[0:3], s32 offset:152
-; GCN-DAG: buffer_store_dword [[NINE]], off, s[0:3], s32{{$}}
-; GCN-DAG: buffer_store_dword [[NINE]], off, s[0:3], s32 offset:4{{$}}
-; GCN-DAG: buffer_store_dword [[NINE]], off, s[0:3], s32 offset:8{{$}}
-; GCN-DAG: buffer_store_dword [[NINE]], off, s[0:3], s32 offset:12{{$}}
-; GCN-DAG: buffer_store_dword v0, off, s[0:3], s32 offset:16
-
-; GCN: v_mov_b32_e32 v0, 0
-; GCN: v_mov_b32_e32 v30, 0
-
-; GCN: s_getpc_b64 [[TARGET_ADDR:s\[[0-9]+:[0-9]+\]]]
-; GCN-NEXT: s_add_u32
-; GCN-NEXT: s_addc_u32
-; GCN-NEXT: s_setpc_b64 [[TARGET_ADDR]]
 define fastcc void @sibling_call_byval_and_stack_passed(i32 %stack.out.arg, [64 x i32]) #1 {
+; GCN-LABEL: sibling_call_byval_and_stack_passed:
+; GCN:       ; %bb.0: ; %entry
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_mov_b32_e32 v1, 9
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:148
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:144
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:152
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:4
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:8
+; GCN-NEXT:    v_mov_b32_e32 v1, 0
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:12
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:16
+; GCN-NEXT:    s_getpc_b64 s[16:17]
+; GCN-NEXT:    s_add_u32 s16, s16, void_fastcc_byval_and_stack_passed@rel32@lo+4
+; GCN-NEXT:    s_addc_u32 s17, s17, void_fastcc_byval_and_stack_passed@rel32@hi+12
+; GCN-NEXT:    v_mov_b32_e32 v0, 0
+; GCN-NEXT:    v_mov_b32_e32 v1, 0
+; GCN-NEXT:    v_mov_b32_e32 v2, 0
+; GCN-NEXT:    v_mov_b32_e32 v3, 0
+; GCN-NEXT:    v_mov_b32_e32 v4, 0
+; GCN-NEXT:    v_mov_b32_e32 v5, 0
+; GCN-NEXT:    v_mov_b32_e32 v6, 0
+; GCN-NEXT:    v_mov_b32_e32 v7, 0
+; GCN-NEXT:    v_mov_b32_e32 v8, 0
+; GCN-NEXT:    v_mov_b32_e32 v9, 0
+; GCN-NEXT:    v_mov_b32_e32 v10, 0
+; GCN-NEXT:    v_mov_b32_e32 v11, 0
+; GCN-NEXT:    v_mov_b32_e32 v12, 0
+; GCN-NEXT:    v_mov_b32_e32 v13, 0
+; GCN-NEXT:    v_mov_b32_e32 v14, 0
+; GCN-NEXT:    v_mov_b32_e32 v15, 0
+; GCN-NEXT:    v_mov_b32_e32 v16, 0
+; GCN-NEXT:    v_mov_b32_e32 v17, 0
+; GCN-NEXT:    v_mov_b32_e32 v18, 0
+; GCN-NEXT:    v_mov_b32_e32 v19, 0
+; GCN-NEXT:    v_mov_b32_e32 v20, 0
+; GCN-NEXT:    v_mov_b32_e32 v21, 0
+; GCN-NEXT:    v_mov_b32_e32 v22, 0
+; GCN-NEXT:    v_mov_b32_e32 v23, 0
+; GCN-NEXT:    v_mov_b32_e32 v24, 0
+; GCN-NEXT:    v_mov_b32_e32 v25, 0
+; GCN-NEXT:    v_mov_b32_e32 v26, 0
+; GCN-NEXT:    v_mov_b32_e32 v27, 0
+; GCN-NEXT:    v_mov_b32_e32 v28, 0
+; GCN-NEXT:    v_mov_b32_e32 v29, 0
+; GCN-NEXT:    v_mov_b32_e32 v30, 0
+; GCN-NEXT:    s_setpc_b64 s[16:17]
 entry:
   %alloca = alloca [3 x i32], align 16, addrspace(5)
   store [3 x i32] [i32 9, i32 9, i32 9], ptr addrspace(5) %alloca
@@ -372,13 +969,14 @@ entry:
 
 declare hidden fastcc i64 @i64_fastcc_i64(i64 %arg0)
 
-; GCN-LABEL: {{^}}sibling_call_i64_fastcc_i64:
-; GCN: s_waitcnt
-; GCN-NEXT: s_getpc_b64
-; GCN-NEXT: s_add_u32
-; GCN-NEXT: s_addc_u32
-; GCN-NEXT: s_setpc_b64
 define hidden fastcc i64 @sibling_call_i64_fastcc_i64(i64 %a) #1 {
+; GCN-LABEL: sibling_call_i64_fastcc_i64:
+; GCN:       ; %bb.0: ; %entry
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    s_getpc_b64 s[16:17]
+; GCN-NEXT:    s_add_u32 s16, s16, i64_fastcc_i64@rel32@lo+4
+; GCN-NEXT:    s_addc_u32 s17, s17, i64_fastcc_i64@rel32@hi+12
+; GCN-NEXT:    s_setpc_b64 s[16:17]
 entry:
   %ret = tail call fastcc i64 @i64_fastcc_i64(i64 %a)
   ret i64 %ret
@@ -386,13 +984,14 @@ entry:
 
 declare hidden fastcc ptr addrspace(1) @p1i8_fastcc_p1i8(ptr addrspace(1) %arg0)
 
-; GCN-LABEL: {{^}}sibling_call_p1i8_fastcc_p1i8:
-; GCN: s_waitcnt
-; GCN-NEXT: s_getpc_b64
-; GCN-NEXT: s_add_u32
-; GCN-NEXT: s_addc_u32
-; GCN-NEXT: s_setpc_b64
 define hidden fastcc ptr addrspace(1) @sibling_call_p1i8_fastcc_p1i8(ptr addrspace(1) %a) #1 {
+; GCN-LABEL: sibling_call_p1i8_fastcc_p1i8:
+; GCN:       ; %bb.0: ; %entry
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    s_getpc_b64 s[16:17]
+; GCN-NEXT:    s_add_u32 s16, s16, p1i8_fastcc_p1i8@rel32@lo+4
+; GCN-NEXT:    s_addc_u32 s17, s17, p1i8_fastcc_p1i8@rel32@hi+12
+; GCN-NEXT:    s_setpc_b64 s[16:17]
 entry:
   %ret = tail call fastcc ptr addrspace(1) @p1i8_fastcc_p1i8(ptr addrspace(1) %a)
   ret ptr addrspace(1) %ret
@@ -400,13 +999,14 @@ entry:
 
 declare hidden fastcc i16 @i16_fastcc_i16(i16 %arg0)
 
-; GCN-LABEL: {{^}}sibling_call_i16_fastcc_i16:
-; GCN: s_waitcnt
-; GCN-NEXT: s_getpc_b64
-; GCN-NEXT: s_add_u32
-; GCN-NEXT: s_addc_u32
-; GCN-NEXT: s_setpc_b64
 define hidden fastcc i16 @sibling_call_i16_fastcc_i16(i16 %a) #1 {
+; GCN-LABEL: sibling_call_i16_fastcc_i16:
+; GCN:       ; %bb.0: ; %entry
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    s_getpc_b64 s[16:17]
+; GCN-NEXT:    s_add_u32 s16, s16, i16_fastcc_i16@rel32@lo+4
+; GCN-NEXT:    s_addc_u32 s17, s17, i16_fastcc_i16@rel32@hi+12
+; GCN-NEXT:    s_setpc_b64 s[16:17]
 entry:
   %ret = tail call fastcc i16 @i16_fastcc_i16(i16 %a)
   ret i16 %ret
@@ -414,13 +1014,14 @@ entry:
 
 declare hidden fastcc half @f16_fastcc_f16(half %arg0)
 
-; GCN-LABEL: {{^}}sibling_call_f16_fastcc_f16:
-; GCN: s_waitcnt
-; GCN-NEXT: s_getpc_b64
-; GCN-NEXT: s_add_u32
-; GCN-NEXT: s_addc_u32
-; GCN-NEXT: s_setpc_b64
 define hidden fastcc half @sibling_call_f16_fastcc_f16(half %a) #1 {
+; GCN-LABEL: sibling_call_f16_fastcc_f16:
+; GCN:       ; %bb.0: ; %entry
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    s_getpc_b64 s[16:17]
+; GCN-NEXT:    s_add_u32 s16, s16, f16_fastcc_f16@rel32@lo+4
+; GCN-NEXT:    s_addc_u32 s17, s17, f16_fastcc_f16@rel32@hi+12
+; GCN-NEXT:    s_setpc_b64 s[16:17]
 entry:
   %ret = tail call fastcc half @f16_fastcc_f16(half %a)
   ret half %ret
@@ -428,13 +1029,14 @@ entry:
 
 declare hidden fastcc <3 x i16> @v3i16_fastcc_v3i16(<3 x i16> %arg0)
 
-; GCN-LABEL: {{^}}sibling_call_v3i16_fastcc_v3i16:
-; GCN: s_waitcnt
-; GCN-NEXT: s_getpc_b64
-; GCN-NEXT: s_add_u32
-; GCN-NEXT: s_addc_u32
-; GCN-NEXT: s_setpc_b64
 define hidden fastcc <3 x i16> @sibling_call_v3i16_fastcc_v3i16(<3 x i16> %a) #1 {
+; GCN-LABEL: sibling_call_v3i16_fastcc_v3i16:
+; GCN:       ; %bb.0: ; %entry
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    s_getpc_b64 s[16:17]
+; GCN-NEXT:    s_add_u32 s16, s16, v3i16_fastcc_v3i16@rel32@lo+4
+; GCN-NEXT:    s_addc_u32 s17, s17, v3i16_fastcc_v3i16@rel32@hi+12
+; GCN-NEXT:    s_setpc_b64 s[16:17]
 entry:
   %ret = tail call fastcc <3 x i16> @v3i16_fastcc_v3i16(<3 x i16> %a)
   ret <3 x i16> %ret
@@ -442,13 +1044,14 @@ entry:
 
 declare hidden fastcc <4 x i16> @v4i16_fastcc_v4i16(<4 x i16> %arg0)
 
-; GCN-LABEL: {{^}}sibling_call_v4i16_fastcc_v4i16:
-; GCN: s_waitcnt
-; GCN-NEXT: s_getpc_b64
-; GCN-NEXT: s_add_u32
-; GCN-NEXT: s_addc_u32
-; GCN-NEXT: s_setpc_b64
 define hidden fastcc <4 x i16> @sibling_call_v4i16_fastcc_v4i16(<4 x i16> %a) #1 {
+; GCN-LABEL: sibling_call_v4i16_fastcc_v4i16:
+; GCN:       ; %bb.0: ; %entry
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    s_getpc_b64 s[16:17]
+; GCN-NEXT:    s_add_u32 s16, s16, v4i16_fastcc_v4i16@rel32@lo+4
+; GCN-NEXT:    s_addc_u32 s17, s17, v4i16_fastcc_v4i16@rel32@hi+12
+; GCN-NEXT:    s_setpc_b64 s[16:17]
 entry:
   %ret = tail call fastcc <4 x i16> @v4i16_fastcc_v4i16(<4 x i16> %a)
   ret <4 x i16> %ret
@@ -456,13 +1059,14 @@ entry:
 
 declare hidden fastcc <2 x i64> @v2i64_fastcc_v2i64(<2 x i64> %arg0)
 
-; GCN-LABEL: {{^}}sibling_call_v2i64_fastcc_v2i64:
-; GCN: s_waitcnt
-; GCN-NEXT: s_getpc_b64
-; GCN-NEXT: s_add_u32
-; GCN-NEXT: s_addc_u32
-; GCN-NEXT: s_setpc_b64
 define hidden fastcc <2 x i64> @sibling_call_v2i64_fastcc_v2i64(<2 x i64> %a) #1 {
+; GCN-LABEL: sibling_call_v2i64_fastcc_v2i64:
+; GCN:       ; %bb.0: ; %entry
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    s_getpc_b64 s[16:17]
+; GCN-NEXT:    s_add_u32 s16, s16, v2i64_fastcc_v2i64@rel32@lo+4
+; GCN-NEXT:    s_addc_u32 s17, s17, v2i64_fastcc_v2i64@rel32@hi+12
+; GCN-NEXT:    s_setpc_b64 s[16:17]
 entry:
   %ret = tail call fastcc <2 x i64> @v2i64_fastcc_v2i64(<2 x i64> %a)
   ret <2 x i64> %ret
diff --git a/llvm/test/CodeGen/AMDGPU/skip-if-dead.ll b/llvm/test/CodeGen/AMDGPU/skip-if-dead.ll
index 715ea57d473f5..0501602bbd8f4 100644
--- a/llvm/test/CodeGen/AMDGPU/skip-if-dead.ll
+++ b/llvm/test/CodeGen/AMDGPU/skip-if-dead.ll
@@ -1023,8 +1023,8 @@ define amdgpu_ps void @test_kill_divergent_loop(i32 %arg) #0 {
 ;
 ; GFX10-WAVE32-LABEL: test_kill_divergent_loop:
 ; GFX10-WAVE32:       ; %bb.0: ; %entry
-; GFX10-WAVE32-NEXT:    s_mov_b32 s0, exec_lo
 ; GFX10-WAVE32-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
+; GFX10-WAVE32-NEXT:    s_mov_b32 s0, exec_lo
 ; GFX10-WAVE32-NEXT:    s_and_saveexec_b32 s1, vcc_lo
 ; GFX10-WAVE32-NEXT:    s_xor_b32 s1, exec_lo, s1
 ; GFX10-WAVE32-NEXT:    s_cbranch_execz .LBB10_3
diff --git a/llvm/test/CodeGen/AMDGPU/spill-scavenge-offset.ll b/llvm/test/CodeGen/AMDGPU/spill-scavenge-offset.ll
index ae70abc7317c3..3a872a6080952 100644
--- a/llvm/test/CodeGen/AMDGPU/spill-scavenge-offset.ll
+++ b/llvm/test/CodeGen/AMDGPU/spill-scavenge-offset.ll
@@ -9779,7 +9779,7 @@ define amdgpu_kernel void @test_limited_sgpr(ptr addrspace(1) %out, ptr addrspac
 ; GFX6-NEXT:    s_mov_b64 s[4:5], s[2:3]
 ; GFX6-NEXT:    v_lshlrev_b32_e32 v5, 8, v0
 ; GFX6-NEXT:    buffer_load_dwordx4 v[0:3], v[5:6], s[4:7], 0 addr64 offset:240
-; GFX6-NEXT:    s_mov_b32 s2, 0x86a00
+; GFX6-NEXT:    s_mov_b32 s2, 0x86600
 ; GFX6-NEXT:    s_mov_b64 s[8:9], exec
 ; GFX6-NEXT:    s_waitcnt vmcnt(0)
 ; GFX6-NEXT:    buffer_store_dword v0, off, s[40:43], s2 ; 4-byte Folded Spill
@@ -9788,7 +9788,7 @@ define amdgpu_kernel void @test_limited_sgpr(ptr addrspace(1) %out, ptr addrspac
 ; GFX6-NEXT:    buffer_store_dword v3, off, s[40:43], s2 offset:12 ; 4-byte Folded Spill
 ; GFX6-NEXT:    s_waitcnt expcnt(0)
 ; GFX6-NEXT:    buffer_load_dwordx4 v[0:3], v[5:6], s[4:7], 0 addr64 offset:224
-; GFX6-NEXT:    s_mov_b32 s2, 0x86600
+; GFX6-NEXT:    s_mov_b32 s2, 0x86200
 ; GFX6-NEXT:    s_waitcnt vmcnt(0)
 ; GFX6-NEXT:    buffer_store_dword v0, off, s[40:43], s2 ; 4-byte Folded Spill
 ; GFX6-NEXT:    buffer_store_dword v1, off, s[40:43], s2 offset:4 ; 4-byte Folded Spill
@@ -9796,7 +9796,7 @@ define amdgpu_kernel void @test_limited_sgpr(ptr addrspace(1) %out, ptr addrspac
 ; GFX6-NEXT:    buffer_store_dword v3, off, s[40:43], s2 offset:12 ; 4-byte Folded Spill
 ; GFX6-NEXT:    s_waitcnt expcnt(0)
 ; GFX6-NEXT:    buffer_load_dwordx4 v[0:3], v[5:6], s[4:7], 0 addr64 offset:208
-; GFX6-NEXT:    s_mov_b32 s2, 0x86200
+; GFX6-NEXT:    s_mov_b32 s2, 0x85e00
 ; GFX6-NEXT:    s_waitcnt vmcnt(0)
 ; GFX6-NEXT:    buffer_store_dword v0, off, s[40:43], s2 ; 4-byte Folded Spill
 ; GFX6-NEXT:    buffer_store_dword v1, off, s[40:43], s2 offset:4 ; 4-byte Folded Spill
@@ -9804,7 +9804,7 @@ define amdgpu_kernel void @test_limited_sgpr(ptr addrspace(1) %out, ptr addrspac
 ; GFX6-NEXT:    buffer_store_dword v3, off, s[40:43], s2 offset:12 ; 4-byte Folded Spill
 ; GFX6-NEXT:    s_waitcnt expcnt(0)
 ; GFX6-NEXT:    buffer_load_dwordx4 v[0:3], v[5:6], s[4:7], 0 addr64 offset:192
-; GFX6-NEXT:    s_mov_b32 s2, 0x85e00
+; GFX6-NEXT:    s_mov_b32 s2, 0x85a00
 ; GFX6-NEXT:    s_waitcnt vmcnt(0)
 ; GFX6-NEXT:    buffer_store_dword v0, off, s[40:43], s2 ; 4-byte Folded Spill
 ; GFX6-NEXT:    buffer_store_dword v1, off, s[40:43], s2 offset:4 ; 4-byte Folded Spill
@@ -9812,7 +9812,7 @@ define amdgpu_kernel void @test_limited_sgpr(ptr addrspace(1) %out, ptr addrspac
 ; GFX6-NEXT:    buffer_store_dword v3, off, s[40:43], s2 offset:12 ; 4-byte Folded Spill
 ; GFX6-NEXT:    s_waitcnt expcnt(0)
 ; GFX6-NEXT:    buffer_load_dwordx4 v[0:3], v[5:6], s[4:7], 0 addr64 offset:176
-; GFX6-NEXT:    s_mov_b32 s2, 0x85a00
+; GFX6-NEXT:    s_mov_b32 s2, 0x85600
 ; GFX6-NEXT:    s_waitcnt vmcnt(0)
 ; GFX6-NEXT:    buffer_store_dword v0, off, s[40:43], s2 ; 4-byte Folded Spill
 ; GFX6-NEXT:    buffer_store_dword v1, off, s[40:43], s2 offset:4 ; 4-byte Folded Spill
@@ -9820,7 +9820,7 @@ define amdgpu_kernel void @test_limited_sgpr(ptr addrspace(1) %out, ptr addrspac
 ; GFX6-NEXT:    buffer_store_dword v3, off, s[40:43], s2 offset:12 ; 4-byte Folded Spill
 ; GFX6-NEXT:    s_waitcnt expcnt(0)
 ; GFX6-NEXT:    buffer_load_dwordx4 v[0:3], v[5:6], s[4:7], 0 addr64 offset:160
-; GFX6-NEXT:    s_mov_b32 s2, 0x85600
+; GFX6-NEXT:    s_mov_b32 s2, 0x85200
 ; GFX6-NEXT:    s_waitcnt vmcnt(0)
 ; GFX6-NEXT:    buffer_store_dword v0, off, s[40:43], s2 ; 4-byte Folded Spill
 ; GFX6-NEXT:    buffer_store_dword v1, off, s[40:43], s2 offset:4 ; 4-byte Folded Spill
@@ -9828,7 +9828,7 @@ define amdgpu_kernel void @test_limited_sgpr(ptr addrspace(1) %out, ptr addrspac
 ; GFX6-NEXT:    buffer_store_dword v3, off, s[40:43], s2 offset:12 ; 4-byte Folded Spill
 ; GFX6-NEXT:    s_waitcnt expcnt(0)
 ; GFX6-NEXT:    buffer_load_dwordx4 v[0:3], v[5:6], s[4:7], 0 addr64 offset:144
-; GFX6-NEXT:    s_mov_b32 s2, 0x85200
+; GFX6-NEXT:    s_mov_b32 s2, 0x84e00
 ; GFX6-NEXT:    s_waitcnt vmcnt(0)
 ; GFX6-NEXT:    buffer_store_dword v0, off, s[40:43], s2 ; 4-byte Folded Spill
 ; GFX6-NEXT:    buffer_store_dword v1, off, s[40:43], s2 offset:4 ; 4-byte Folded Spill
@@ -9836,7 +9836,7 @@ define amdgpu_kernel void @test_limited_sgpr(ptr addrspace(1) %out, ptr addrspac
 ; GFX6-NEXT:    buffer_store_dword v3, off, s[40:43], s2 offset:12 ; 4-byte Folded Spill
 ; GFX6-NEXT:    s_waitcnt expcnt(0)
 ; GFX6-NEXT:    buffer_load_dwordx4 v[0:3], v[5:6], s[4:7], 0 addr64 offset:128
-; GFX6-NEXT:    s_mov_b32 s2, 0x84e00
+; GFX6-NEXT:    s_mov_b32 s2, 0x84a00
 ; GFX6-NEXT:    s_waitcnt vmcnt(0)
 ; GFX6-NEXT:    buffer_store_dword v0, off, s[40:43], s2 ; 4-byte Folded Spill
 ; GFX6-NEXT:    buffer_store_dword v1, off, s[40:43], s2 offset:4 ; 4-byte Folded Spill
@@ -9844,7 +9844,7 @@ define amdgpu_kernel void @test_limited_sgpr(ptr addrspace(1) %out, ptr addrspac
 ; GFX6-NEXT:    buffer_store_dword v3, off, s[40:43], s2 offset:12 ; 4-byte Folded Spill
 ; GFX6-NEXT:    s_waitcnt expcnt(0)
 ; GFX6-NEXT:    buffer_load_dwordx4 v[0:3], v[5:6], s[4:7], 0 addr64 offset:112
-; GFX6-NEXT:    s_mov_b32 s2, 0x84a00
+; GFX6-NEXT:    s_mov_b32 s2, 0x84600
 ; GFX6-NEXT:    s_waitcnt vmcnt(0)
 ; GFX6-NEXT:    buffer_store_dword v0, off, s[40:43], s2 ; 4-byte Folded Spill
 ; GFX6-NEXT:    buffer_store_dword v1, off, s[40:43], s2 offset:4 ; 4-byte Folded Spill
@@ -9852,7 +9852,7 @@ define amdgpu_kernel void @test_limited_sgpr(ptr addrspace(1) %out, ptr addrspac
 ; GFX6-NEXT:    buffer_store_dword v3, off, s[40:43], s2 offset:12 ; 4-byte Folded Spill
 ; GFX6-NEXT:    s_waitcnt expcnt(0)
 ; GFX6-NEXT:    buffer_load_dwordx4 v[0:3], v[5:6], s[4:7], 0 addr64 offset:96
-; GFX6-NEXT:    s_mov_b32 s2, 0x84600
+; GFX6-NEXT:    s_mov_b32 s2, 0x84200
 ; GFX6-NEXT:    s_waitcnt vmcnt(0)
 ; GFX6-NEXT:    buffer_store_dword v0, off, s[40:43], s2 ; 4-byte Folded Spill
 ; GFX6-NEXT:    buffer_store_dword v1, off, s[40:43], s2 offset:4 ; 4-byte Folded Spill
@@ -9860,7 +9860,7 @@ define amdgpu_kernel void @test_limited_sgpr(ptr addrspace(1) %out, ptr addrspac
 ; GFX6-NEXT:    buffer_store_dword v3, off, s[40:43], s2 offset:12 ; 4-byte Folded Spill
 ; GFX6-NEXT:    s_waitcnt expcnt(0)
 ; GFX6-NEXT:    buffer_load_dwordx4 v[0:3], v[5:6], s[4:7], 0 addr64 offset:80
-; GFX6-NEXT:    s_mov_b32 s2, 0x84200
+; GFX6-NEXT:    s_mov_b32 s2, 0x83e00
 ; GFX6-NEXT:    s_waitcnt vmcnt(0)
 ; GFX6-NEXT:    buffer_store_dword v0, off, s[40:43], s2 ; 4-byte Folded Spill
 ; GFX6-NEXT:    buffer_store_dword v1, off, s[40:43], s2 offset:4 ; 4-byte Folded Spill
@@ -9906,7 +9906,7 @@ define amdgpu_kernel void @test_limited_sgpr(ptr addrspace(1) %out, ptr addrspac
 ; GFX6-NEXT:    s_waitcnt vmcnt(0)
 ; GFX6-NEXT:    s_mov_b64 exec, s[8:9]
 ; GFX6-NEXT:    buffer_load_dwordx4 v[7:10], v[5:6], s[4:7], 0 addr64 offset:48
-; GFX6-NEXT:    s_mov_b32 s0, 0x83e00
+; GFX6-NEXT:    s_mov_b32 s0, 0x86a00
 ; GFX6-NEXT:    v_lshlrev_b32_e32 v4, 13, v0
 ; GFX6-NEXT:    v_add_i32_e32 v4, vcc, 16, v4
 ; GFX6-NEXT:    s_waitcnt vmcnt(0)
@@ -10187,13 +10187,13 @@ define amdgpu_kernel void @test_limited_sgpr(ptr addrspace(1) %out, ptr addrspac
 ; GFX6-NEXT:    buffer_load_dword v4, off, s[40:43], 0
 ; GFX6-NEXT:    s_waitcnt vmcnt(0)
 ; GFX6-NEXT:    s_mov_b64 exec, s[4:5]
-; GFX6-NEXT:    s_mov_b32 s0, 0x86a00
+; GFX6-NEXT:    s_mov_b32 s0, 0x86600
 ; GFX6-NEXT:    buffer_load_dword v7, off, s[40:43], s0 ; 4-byte Folded Reload
 ; GFX6-NEXT:    buffer_load_dword v8, off, s[40:43], s0 offset:4 ; 4-byte Folded Reload
 ; GFX6-NEXT:    buffer_load_dword v9, off, s[40:43], s0 offset:8 ; 4-byte Folded Reload
 ; GFX6-NEXT:    buffer_load_dword v10, off, s[40:43], s0 offset:12 ; 4-byte Folded Reload
 ; GFX6-NEXT:    s_mov_b64 s[38:39], s[2:3]
-; GFX6-NEXT:    s_mov_b32 s0, 0x86600
+; GFX6-NEXT:    s_mov_b32 s0, 0x86200
 ; GFX6-NEXT:    s_waitcnt vmcnt(0)
 ; GFX6-NEXT:    buffer_store_dwordx4 v[7:10], v[5:6], s[36:39], 0 addr64 offset:240
 ; GFX6-NEXT:    s_waitcnt expcnt(0)
@@ -10201,7 +10201,7 @@ define amdgpu_kernel void @test_limited_sgpr(ptr addrspace(1) %out, ptr addrspac
 ; GFX6-NEXT:    buffer_load_dword v8, off, s[40:43], s0 offset:4 ; 4-byte Folded Reload
 ; GFX6-NEXT:    buffer_load_dword v9, off, s[40:43], s0 offset:8 ; 4-byte Folded Reload
 ; GFX6-NEXT:    buffer_load_dword v10, off, s[40:43], s0 offset:12 ; 4-byte Folded Reload
-; GFX6-NEXT:    s_mov_b32 s0, 0x86200
+; GFX6-NEXT:    s_mov_b32 s0, 0x85e00
 ; GFX6-NEXT:    s_waitcnt vmcnt(0)
 ; GFX6-NEXT:    buffer_store_dwordx4 v[7:10], v[5:6], s[36:39], 0 addr64 offset:224
 ; GFX6-NEXT:    s_waitcnt expcnt(0)
@@ -10209,7 +10209,7 @@ define amdgpu_kernel void @test_limited_sgpr(ptr addrspace(1) %out, ptr addrspac
 ; GFX6-NEXT:    buffer_load_dword v8, off, s[40:43], s0 offset:4 ; 4-byte Folded Reload
 ; GFX6-NEXT:    buffer_load_dword v9, off, s[40:43], s0 offset:8 ; 4-byte Folded Reload
 ; GFX6-NEXT:    buffer_load_dword v10, off, s[40:43], s0 offset:12 ; 4-byte Folded Reload
-; GFX6-NEXT:    s_mov_b32 s0, 0x85e00
+; GFX6-NEXT:    s_mov_b32 s0, 0x85a00
 ; GFX6-NEXT:    s_waitcnt vmcnt(0)
 ; GFX6-NEXT:    buffer_store_dwordx4 v[7:10], v[5:6], s[36:39], 0 addr64 offset:208
 ; GFX6-NEXT:    s_waitcnt expcnt(0)
@@ -10217,7 +10217,7 @@ define amdgpu_kernel void @test_limited_sgpr(ptr addrspace(1) %out, ptr addrspac
 ; GFX6-NEXT:    buffer_load_dword v8, off, s[40:43], s0 offset:4 ; 4-byte Folded Reload
 ; GFX6-NEXT:    buffer_load_dword v9, off, s[40:43], s0 offset:8 ; 4-byte Folded Reload
 ; GFX6-NEXT:    buffer_load_dword v10, off, s[40:43], s0 offset:12 ; 4-byte Folded Reload
-; GFX6-NEXT:    s_mov_b32 s0, 0x85a00
+; GFX6-NEXT:    s_mov_b32 s0, 0x85600
 ; GFX6-NEXT:    s_waitcnt vmcnt(0)
 ; GFX6-NEXT:    buffer_store_dwordx4 v[7:10], v[5:6], s[36:39], 0 addr64 offset:192
 ; GFX6-NEXT:    s_waitcnt expcnt(0)
@@ -10225,7 +10225,7 @@ define amdgpu_kernel void @test_limited_sgpr(ptr addrspace(1) %out, ptr addrspac
 ; GFX6-NEXT:    buffer_load_dword v8, off, s[40:43], s0 offset:4 ; 4-byte Folded Reload
 ; GFX6-NEXT:    buffer_load_dword v9, off, s[40:43], s0 offset:8 ; 4-byte Folded Reload
 ; GFX6-NEXT:    buffer_load_dword v10, off, s[40:43], s0 offset:12 ; 4-byte Folded Reload
-; GFX6-NEXT:    s_mov_b32 s0, 0x85600
+; GFX6-NEXT:    s_mov_b32 s0, 0x85200
 ; GFX6-NEXT:    s_waitcnt vmcnt(0)
 ; GFX6-NEXT:    buffer_store_dwordx4 v[7:10], v[5:6], s[36:39], 0 addr64 offset:176
 ; GFX6-NEXT:    s_waitcnt expcnt(0)
@@ -10233,7 +10233,7 @@ define amdgpu_kernel void @test_limited_sgpr(ptr addrspace(1) %out, ptr addrspac
 ; GFX6-NEXT:    buffer_load_dword v8, off, s[40:43], s0 offset:4 ; 4-byte Folded Reload
 ; GFX6-NEXT:    buffer_load_dword v9, off, s[40:43], s0 offset:8 ; 4-byte Folded Reload
 ; GFX6-NEXT:    buffer_load_dword v10, off, s[40:43], s0 offset:12 ; 4-byte Folded Reload
-; GFX6-NEXT:    s_mov_b32 s0, 0x85200
+; GFX6-NEXT:    s_mov_b32 s0, 0x84e00
 ; GFX6-NEXT:    s_waitcnt vmcnt(0)
 ; GFX6-NEXT:    buffer_store_dwordx4 v[7:10], v[5:6], s[36:39], 0 addr64 offset:160
 ; GFX6-NEXT:    s_waitcnt expcnt(0)
@@ -10241,7 +10241,7 @@ define amdgpu_kernel void @test_limited_sgpr(ptr addrspace(1) %out, ptr addrspac
 ; GFX6-NEXT:    buffer_load_dword v8, off, s[40:43], s0 offset:4 ; 4-byte Folded Reload
 ; GFX6-NEXT:    buffer_load_dword v9, off, s[40:43], s0 offset:8 ; 4-byte Folded Reload
 ; GFX6-NEXT:    buffer_load_dword v10, off, s[40:43], s0 offset:12 ; 4-byte Folded Reload
-; GFX6-NEXT:    s_mov_b32 s0, 0x84e00
+; GFX6-NEXT:    s_mov_b32 s0, 0x84a00
 ; GFX6-NEXT:    s_waitcnt vmcnt(0)
 ; GFX6-NEXT:    buffer_store_dwordx4 v[7:10], v[5:6], s[36:39], 0 addr64 offset:144
 ; GFX6-NEXT:    s_waitcnt expcnt(0)
@@ -10249,7 +10249,7 @@ define amdgpu_kernel void @test_limited_sgpr(ptr addrspace(1) %out, ptr addrspac
 ; GFX6-NEXT:    buffer_load_dword v8, off, s[40:43], s0 offset:4 ; 4-byte Folded Reload
 ; GFX6-NEXT:    buffer_load_dword v9, off, s[40:43], s0 offset:8 ; 4-byte Folded Reload
 ; GFX6-NEXT:    buffer_load_dword v10, off, s[40:43], s0 offset:12 ; 4-byte Folded Reload
-; GFX6-NEXT:    s_mov_b32 s0, 0x84a00
+; GFX6-NEXT:    s_mov_b32 s0, 0x84600
 ; GFX6-NEXT:    s_waitcnt vmcnt(0)
 ; GFX6-NEXT:    buffer_store_dwordx4 v[7:10], v[5:6], s[36:39], 0 addr64 offset:128
 ; GFX6-NEXT:    s_waitcnt expcnt(0)
@@ -10257,7 +10257,7 @@ define amdgpu_kernel void @test_limited_sgpr(ptr addrspace(1) %out, ptr addrspac
 ; GFX6-NEXT:    buffer_load_dword v8, off, s[40:43], s0 offset:4 ; 4-byte Folded Reload
 ; GFX6-NEXT:    buffer_load_dword v9, off, s[40:43], s0 offset:8 ; 4-byte Folded Reload
 ; GFX6-NEXT:    buffer_load_dword v10, off, s[40:43], s0 offset:12 ; 4-byte Folded Reload
-; GFX6-NEXT:    s_mov_b32 s0, 0x84600
+; GFX6-NEXT:    s_mov_b32 s0, 0x84200
 ; GFX6-NEXT:    s_waitcnt vmcnt(0)
 ; GFX6-NEXT:    buffer_store_dwordx4 v[7:10], v[5:6], s[36:39], 0 addr64 offset:112
 ; GFX6-NEXT:    s_waitcnt expcnt(0)
@@ -10265,7 +10265,7 @@ define amdgpu_kernel void @test_limited_sgpr(ptr addrspace(1) %out, ptr addrspac
 ; GFX6-NEXT:    buffer_load_dword v8, off, s[40:43], s0 offset:4 ; 4-byte Folded Reload
 ; GFX6-NEXT:    buffer_load_dword v9, off, s[40:43], s0 offset:8 ; 4-byte Folded Reload
 ; GFX6-NEXT:    buffer_load_dword v10, off, s[40:43], s0 offset:12 ; 4-byte Folded Reload
-; GFX6-NEXT:    s_mov_b32 s0, 0x84200
+; GFX6-NEXT:    s_mov_b32 s0, 0x83e00
 ; GFX6-NEXT:    s_waitcnt vmcnt(0)
 ; GFX6-NEXT:    buffer_store_dwordx4 v[7:10], v[5:6], s[36:39], 0 addr64 offset:96
 ; GFX6-NEXT:    s_waitcnt expcnt(0)
@@ -10281,7 +10281,7 @@ define amdgpu_kernel void @test_limited_sgpr(ptr addrspace(1) %out, ptr addrspac
 ; GFX6-NEXT:    buffer_load_dword v8, off, s[40:43], s0 offset:4 ; 4-byte Folded Reload
 ; GFX6-NEXT:    buffer_load_dword v9, off, s[40:43], s0 offset:8 ; 4-byte Folded Reload
 ; GFX6-NEXT:    buffer_load_dword v10, off, s[40:43], s0 offset:12 ; 4-byte Folded Reload
-; GFX6-NEXT:    s_mov_b32 s0, 0x83e00
+; GFX6-NEXT:    s_mov_b32 s0, 0x86a00
 ; GFX6-NEXT:    s_waitcnt vmcnt(0)
 ; GFX6-NEXT:    buffer_store_dwordx4 v[7:10], v[5:6], s[36:39], 0 addr64 offset:64
 ; GFX6-NEXT:    s_waitcnt expcnt(0)
@@ -10324,59 +10324,60 @@ define amdgpu_kernel void @test_limited_sgpr(ptr addrspace(1) %out, ptr addrspac
 ; GFX9-FLATSCR-NEXT:    v_mov_b32_e32 v4, 16
 ; GFX9-FLATSCR-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-FLATSCR-NEXT:    scratch_store_dwordx4 off, v[0:3], s0 ; 16-byte Folded Spill
-; GFX9-FLATSCR-NEXT:    global_load_dwordx4 v[0:3], v5, s[38:39] offset:224
-; GFX9-FLATSCR-NEXT:    s_movk_i32 s0, 0x2040
-; GFX9-FLATSCR-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT:    scratch_store_dwordx4 off, v[0:3], s0 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT:    global_load_dwordx4 v[11:14], v5, s[38:39] offset:224
 ; GFX9-FLATSCR-NEXT:    global_load_dwordx4 v[0:3], v5, s[38:39] offset:208
 ; GFX9-FLATSCR-NEXT:    s_movk_i32 s0, 0x2030
 ; GFX9-FLATSCR-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-FLATSCR-NEXT:    scratch_store_dwordx4 off, v[0:3], s0 ; 16-byte Folded Spill
-; GFX9-FLATSCR-NEXT:    global_load_dwordx4 v[19:22], v5, s[38:39] offset:192
+; GFX9-FLATSCR-NEXT:    global_load_dwordx4 v[0:3], v5, s[38:39] offset:192
+; GFX9-FLATSCR-NEXT:    s_movk_i32 s0, 0x2020
+; GFX9-FLATSCR-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-FLATSCR-NEXT:    scratch_store_dwordx4 off, v[0:3], s0 ; 16-byte Folded Spill
 ; GFX9-FLATSCR-NEXT:    global_load_dwordx4 v[15:18], v5, s[38:39] offset:176
-; GFX9-FLATSCR-NEXT:    global_load_dwordx4 v[11:14], v5, s[38:39] offset:160
+; GFX9-FLATSCR-NEXT:    global_load_dwordx4 v[0:3], v5, s[38:39] offset:160
+; GFX9-FLATSCR-NEXT:    s_movk_i32 s0, 0x2040
+; GFX9-FLATSCR-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-FLATSCR-NEXT:    scratch_store_dwordx4 off, v[0:3], s0 ; 16-byte Folded Spill
 ; GFX9-FLATSCR-NEXT:    global_load_dwordx4 v[0:3], v5, s[38:39] offset:144
-; GFX9-FLATSCR-NEXT:    s_movk_i32 s0, 0x2010
 ; GFX9-FLATSCR-NEXT:    global_load_dwordx4 v[6:9], v5, s[38:39] offset:128
+; GFX9-FLATSCR-NEXT:    s_movk_i32 s0, 0x2010
+; GFX9-FLATSCR-NEXT:    global_load_dwordx4 v[19:22], v5, s[38:39]
 ; GFX9-FLATSCR-NEXT:    s_waitcnt vmcnt(1)
-; GFX9-FLATSCR-NEXT:    scratch_store_dwordx4 off, v[0:3], s0 ; 16-byte Folded Spill
-; GFX9-FLATSCR-NEXT:    global_load_dwordx4 v[0:3], v5, s[38:39] offset:112
-; GFX9-FLATSCR-NEXT:    s_movk_i32 s0, 0x2020
-; GFX9-FLATSCR-NEXT:    s_waitcnt vmcnt(2)
 ; GFX9-FLATSCR-NEXT:    scratch_store_dwordx4 off, v[6:9], s0 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT:    global_load_dwordx4 v[6:9], v5, s[38:39] offset:112
 ; GFX9-FLATSCR-NEXT:    s_movk_i32 s0, 0x20c0
-; GFX9-FLATSCR-NEXT:    v_mov_b32_e32 v6, 1
-; GFX9-FLATSCR-NEXT:    global_load_dwordx4 v[7:10], v5, s[38:39]
 ; GFX9-FLATSCR-NEXT:    s_waitcnt vmcnt(2)
-; GFX9-FLATSCR-NEXT:    scratch_store_dwordx4 off, v[0:3], s0 ; 16-byte Folded Spill
-; GFX9-FLATSCR-NEXT:    global_load_dwordx4 v[0:3], v5, s[38:39] offset:96
+; GFX9-FLATSCR-NEXT:    v_lshl_add_u32 v4, v19, 13, v4
+; GFX9-FLATSCR-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v19
+; GFX9-FLATSCR-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-FLATSCR-NEXT:    scratch_store_dwordx4 off, v[6:9], s0 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT:    global_load_dwordx4 v[6:9], v5, s[38:39] offset:96
 ; GFX9-FLATSCR-NEXT:    s_movk_i32 s0, 0x20b0
-; GFX9-FLATSCR-NEXT:    s_waitcnt vmcnt(2)
-; GFX9-FLATSCR-NEXT:    v_lshl_add_u32 v4, v7, 13, v4
-; GFX9-FLATSCR-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v7
-; GFX9-FLATSCR-NEXT:    scratch_store_dword v4, v6, off
-; GFX9-FLATSCR-NEXT:    s_waitcnt vmcnt(1)
-; GFX9-FLATSCR-NEXT:    scratch_store_dwordx4 off, v[0:3], s0 ; 16-byte Folded Spill
-; GFX9-FLATSCR-NEXT:    global_load_dwordx4 v[0:3], v5, s[38:39] offset:80
+; GFX9-FLATSCR-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-FLATSCR-NEXT:    scratch_store_dwordx4 off, v[6:9], s0 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT:    global_load_dwordx4 v[6:9], v5, s[38:39] offset:80
 ; GFX9-FLATSCR-NEXT:    s_movk_i32 s0, 0x20a0
 ; GFX9-FLATSCR-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT:    scratch_store_dwordx4 off, v[0:3], s0 ; 16-byte Folded Spill
-; GFX9-FLATSCR-NEXT:    global_load_dwordx4 v[0:3], v5, s[38:39] offset:64
+; GFX9-FLATSCR-NEXT:    scratch_store_dwordx4 off, v[6:9], s0 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT:    global_load_dwordx4 v[6:9], v5, s[38:39] offset:64
 ; GFX9-FLATSCR-NEXT:    s_movk_i32 s0, 0x2090
 ; GFX9-FLATSCR-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT:    scratch_store_dwordx4 off, v[0:3], s0 ; 16-byte Folded Spill
-; GFX9-FLATSCR-NEXT:    global_load_dwordx4 v[0:3], v5, s[38:39] offset:48
+; GFX9-FLATSCR-NEXT:    scratch_store_dwordx4 off, v[6:9], s0 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT:    global_load_dwordx4 v[6:9], v5, s[38:39] offset:48
 ; GFX9-FLATSCR-NEXT:    s_movk_i32 s0, 0x2080
 ; GFX9-FLATSCR-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT:    scratch_store_dwordx4 off, v[0:3], s0 ; 16-byte Folded Spill
-; GFX9-FLATSCR-NEXT:    global_load_dwordx4 v[0:3], v5, s[38:39] offset:32
+; GFX9-FLATSCR-NEXT:    scratch_store_dwordx4 off, v[6:9], s0 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT:    global_load_dwordx4 v[6:9], v5, s[38:39] offset:32
 ; GFX9-FLATSCR-NEXT:    s_movk_i32 s0, 0x2070
 ; GFX9-FLATSCR-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT:    scratch_store_dwordx4 off, v[0:3], s0 ; 16-byte Folded Spill
-; GFX9-FLATSCR-NEXT:    global_load_dwordx4 v[0:3], v5, s[38:39] offset:16
+; GFX9-FLATSCR-NEXT:    scratch_store_dwordx4 off, v[6:9], s0 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT:    global_load_dwordx4 v[6:9], v5, s[38:39] offset:16
 ; GFX9-FLATSCR-NEXT:    s_movk_i32 s0, 0x2060
 ; GFX9-FLATSCR-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT:    scratch_store_dwordx4 off, v[0:3], s0 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT:    scratch_store_dwordx4 off, v[6:9], s0 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT:    s_nop 0
+; GFX9-FLATSCR-NEXT:    v_mov_b32_e32 v6, 1
+; GFX9-FLATSCR-NEXT:    scratch_store_dword v4, v6, off
 ; GFX9-FLATSCR-NEXT:    ;;#ASMSTART
 ; GFX9-FLATSCR-NEXT:    ; def s[0:7]
 ; GFX9-FLATSCR-NEXT:    ;;#ASMEND
@@ -10405,27 +10406,26 @@ define amdgpu_kernel void @test_limited_sgpr(ptr addrspace(1) %out, ptr addrspac
 ; GFX9-FLATSCR-NEXT:    ; use s[0:7],s[8:15],s[16:23],s[24:31],s[40:43],s[38:39]
 ; GFX9-FLATSCR-NEXT:    ;;#ASMEND
 ; GFX9-FLATSCR-NEXT:    s_movk_i32 s0, 0x20d0
-; GFX9-FLATSCR-NEXT:    scratch_store_dwordx4 off, v[15:18], s0 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT:    scratch_store_dwordx4 off, v[0:3], s0 ; 16-byte Folded Spill
 ; GFX9-FLATSCR-NEXT:    s_movk_i32 s0, 0x20e0
-; GFX9-FLATSCR-NEXT:    v_mov_b32_e32 v0, v11
-; GFX9-FLATSCR-NEXT:    scratch_store_dwordx4 off, v[19:22], s0 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT:    scratch_store_dwordx4 off, v[15:18], s0 ; 16-byte Folded Spill
 ; GFX9-FLATSCR-NEXT:    s_movk_i32 s0, 0x20f0
-; GFX9-FLATSCR-NEXT:    v_mov_b32_e32 v1, v12
-; GFX9-FLATSCR-NEXT:    v_mov_b32_e32 v2, v13
-; GFX9-FLATSCR-NEXT:    v_mov_b32_e32 v3, v14
-; GFX9-FLATSCR-NEXT:    scratch_store_dwordx4 off, v[7:10], s0 ; 16-byte Folded Spill
-; GFX9-FLATSCR-NEXT:    s_nop 0
+; GFX9-FLATSCR-NEXT:    v_mov_b32_e32 v0, v19
+; GFX9-FLATSCR-NEXT:    scratch_store_dwordx4 off, v[11:14], s0 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT:    v_mov_b32_e32 v1, v20
+; GFX9-FLATSCR-NEXT:    v_mov_b32_e32 v2, v21
+; GFX9-FLATSCR-NEXT:    v_mov_b32_e32 v3, v22
 ; GFX9-FLATSCR-NEXT:    ;;#ASMSTART
 ; GFX9-FLATSCR-NEXT:    ;;#ASMEND
-; GFX9-FLATSCR-NEXT:    scratch_load_dwordx4 v[7:10], off, s0 ; 16-byte Folded Reload
+; GFX9-FLATSCR-NEXT:    scratch_load_dwordx4 v[11:14], off, s0 ; 16-byte Folded Reload
 ; GFX9-FLATSCR-NEXT:    s_movk_i32 s0, 0x20e0
-; GFX9-FLATSCR-NEXT:    scratch_load_dwordx4 v[19:22], off, s0 ; 16-byte Folded Reload
-; GFX9-FLATSCR-NEXT:    s_movk_i32 s0, 0x20d0
+; GFX9-FLATSCR-NEXT:    v_mov_b32_e32 v22, v3
 ; GFX9-FLATSCR-NEXT:    scratch_load_dwordx4 v[15:18], off, s0 ; 16-byte Folded Reload
-; GFX9-FLATSCR-NEXT:    v_mov_b32_e32 v14, v3
-; GFX9-FLATSCR-NEXT:    v_mov_b32_e32 v13, v2
-; GFX9-FLATSCR-NEXT:    v_mov_b32_e32 v12, v1
-; GFX9-FLATSCR-NEXT:    v_mov_b32_e32 v11, v0
+; GFX9-FLATSCR-NEXT:    s_movk_i32 s0, 0x20d0
+; GFX9-FLATSCR-NEXT:    v_mov_b32_e32 v21, v2
+; GFX9-FLATSCR-NEXT:    v_mov_b32_e32 v20, v1
+; GFX9-FLATSCR-NEXT:    v_mov_b32_e32 v19, v0
+; GFX9-FLATSCR-NEXT:    scratch_load_dwordx4 v[0:3], off, s0 ; 16-byte Folded Reload
 ; GFX9-FLATSCR-NEXT:    ;;#ASMSTART
 ; GFX9-FLATSCR-NEXT:    ;;#ASMEND
 ; GFX9-FLATSCR-NEXT:    ;;#ASMSTART
@@ -10441,53 +10441,53 @@ define amdgpu_kernel void @test_limited_sgpr(ptr addrspace(1) %out, ptr addrspac
 ; GFX9-FLATSCR-NEXT:  .LBB1_2: ; %ret
 ; GFX9-FLATSCR-NEXT:    s_or_b64 exec, exec, s[34:35]
 ; GFX9-FLATSCR-NEXT:    s_movk_i32 s0, 0x20c0
-; GFX9-FLATSCR-NEXT:    scratch_load_dwordx4 v[0:3], off, s0 ; 16-byte Folded Reload
+; GFX9-FLATSCR-NEXT:    scratch_load_dwordx4 v[6:9], off, s0 ; 16-byte Folded Reload
 ; GFX9-FLATSCR-NEXT:    s_movk_i32 s0, 0x20b0
 ; GFX9-FLATSCR-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT:    global_store_dwordx4 v5, v[0:3], s[36:37] offset:112
-; GFX9-FLATSCR-NEXT:    scratch_load_dwordx4 v[0:3], off, s0 ; 16-byte Folded Reload
+; GFX9-FLATSCR-NEXT:    global_store_dwordx4 v5, v[6:9], s[36:37] offset:112
+; GFX9-FLATSCR-NEXT:    scratch_load_dwordx4 v[6:9], off, s0 ; 16-byte Folded Reload
 ; GFX9-FLATSCR-NEXT:    s_movk_i32 s0, 0x20a0
 ; GFX9-FLATSCR-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT:    global_store_dwordx4 v5, v[0:3], s[36:37] offset:96
-; GFX9-FLATSCR-NEXT:    scratch_load_dwordx4 v[0:3], off, s0 ; 16-byte Folded Reload
+; GFX9-FLATSCR-NEXT:    global_store_dwordx4 v5, v[6:9], s[36:37] offset:96
+; GFX9-FLATSCR-NEXT:    scratch_load_dwordx4 v[6:9], off, s0 ; 16-byte Folded Reload
 ; GFX9-FLATSCR-NEXT:    s_movk_i32 s0, 0x2090
 ; GFX9-FLATSCR-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT:    global_store_dwordx4 v5, v[0:3], s[36:37] offset:80
-; GFX9-FLATSCR-NEXT:    scratch_load_dwordx4 v[0:3], off, s0 ; 16-byte Folded Reload
+; GFX9-FLATSCR-NEXT:    global_store_dwordx4 v5, v[6:9], s[36:37] offset:80
+; GFX9-FLATSCR-NEXT:    scratch_load_dwordx4 v[6:9], off, s0 ; 16-byte Folded Reload
 ; GFX9-FLATSCR-NEXT:    s_movk_i32 s0, 0x2080
 ; GFX9-FLATSCR-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT:    global_store_dwordx4 v5, v[0:3], s[36:37] offset:64
-; GFX9-FLATSCR-NEXT:    scratch_load_dwordx4 v[0:3], off, s0 ; 16-byte Folded Reload
+; GFX9-FLATSCR-NEXT:    global_store_dwordx4 v5, v[6:9], s[36:37] offset:64
+; GFX9-FLATSCR-NEXT:    scratch_load_dwordx4 v[6:9], off, s0 ; 16-byte Folded Reload
 ; GFX9-FLATSCR-NEXT:    s_movk_i32 s0, 0x2070
 ; GFX9-FLATSCR-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT:    global_store_dwordx4 v5, v[0:3], s[36:37] offset:48
-; GFX9-FLATSCR-NEXT:    scratch_load_dwordx4 v[0:3], off, s0 ; 16-byte Folded Reload
+; GFX9-FLATSCR-NEXT:    global_store_dwordx4 v5, v[6:9], s[36:37] offset:48
+; GFX9-FLATSCR-NEXT:    scratch_load_dwordx4 v[6:9], off, s0 ; 16-byte Folded Reload
 ; GFX9-FLATSCR-NEXT:    s_movk_i32 s0, 0x2060
 ; GFX9-FLATSCR-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT:    global_store_dwordx4 v5, v[0:3], s[36:37] offset:32
-; GFX9-FLATSCR-NEXT:    scratch_load_dwordx4 v[0:3], off, s0 ; 16-byte Folded Reload
-; GFX9-FLATSCR-NEXT:    s_movk_i32 s0, 0x2050
-; GFX9-FLATSCR-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT:    global_store_dwordx4 v5, v[0:3], s[36:37] offset:16
-; GFX9-FLATSCR-NEXT:    global_store_dwordx4 v5, v[7:10], s[36:37]
+; GFX9-FLATSCR-NEXT:    global_store_dwordx4 v5, v[6:9], s[36:37] offset:32
 ; GFX9-FLATSCR-NEXT:    scratch_load_dwordx4 v[6:9], off, s0 ; 16-byte Folded Reload
-; GFX9-FLATSCR-NEXT:    s_movk_i32 s0, 0x2040
+; GFX9-FLATSCR-NEXT:    s_movk_i32 s0, 0x2050
 ; GFX9-FLATSCR-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT:    global_store_dwordx4 v5, v[6:9], s[36:37] offset:240
-; GFX9-FLATSCR-NEXT:    scratch_load_dwordx4 v[6:9], off, s0 ; 16-byte Folded Reload
+; GFX9-FLATSCR-NEXT:    global_store_dwordx4 v5, v[6:9], s[36:37] offset:16
+; GFX9-FLATSCR-NEXT:    global_store_dwordx4 v5, v[19:22], s[36:37]
+; GFX9-FLATSCR-NEXT:    scratch_load_dwordx4 v[19:22], off, s0 ; 16-byte Folded Reload
 ; GFX9-FLATSCR-NEXT:    s_movk_i32 s0, 0x2030
 ; GFX9-FLATSCR-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT:    global_store_dwordx4 v5, v[6:9], s[36:37] offset:224
-; GFX9-FLATSCR-NEXT:    scratch_load_dwordx4 v[6:9], off, s0 ; 16-byte Folded Reload
-; GFX9-FLATSCR-NEXT:    s_movk_i32 s0, 0x2010
+; GFX9-FLATSCR-NEXT:    global_store_dwordx4 v5, v[19:22], s[36:37] offset:240
+; GFX9-FLATSCR-NEXT:    global_store_dwordx4 v5, v[11:14], s[36:37] offset:224
+; GFX9-FLATSCR-NEXT:    scratch_load_dwordx4 v[11:14], off, s0 ; 16-byte Folded Reload
+; GFX9-FLATSCR-NEXT:    s_movk_i32 s0, 0x2020
 ; GFX9-FLATSCR-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT:    global_store_dwordx4 v5, v[6:9], s[36:37] offset:208
-; GFX9-FLATSCR-NEXT:    global_store_dwordx4 v5, v[19:22], s[36:37] offset:192
+; GFX9-FLATSCR-NEXT:    global_store_dwordx4 v5, v[11:14], s[36:37] offset:208
+; GFX9-FLATSCR-NEXT:    scratch_load_dwordx4 v[11:14], off, s0 ; 16-byte Folded Reload
+; GFX9-FLATSCR-NEXT:    s_movk_i32 s0, 0x2040
+; GFX9-FLATSCR-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-FLATSCR-NEXT:    global_store_dwordx4 v5, v[11:14], s[36:37] offset:192
 ; GFX9-FLATSCR-NEXT:    global_store_dwordx4 v5, v[15:18], s[36:37] offset:176
-; GFX9-FLATSCR-NEXT:    global_store_dwordx4 v5, v[11:14], s[36:37] offset:160
-; GFX9-FLATSCR-NEXT:    scratch_load_dwordx4 v[0:3], off, s0 ; 16-byte Folded Reload
-; GFX9-FLATSCR-NEXT:    s_movk_i32 s0, 0x2020
+; GFX9-FLATSCR-NEXT:    scratch_load_dwordx4 v[11:14], off, s0 ; 16-byte Folded Reload
+; GFX9-FLATSCR-NEXT:    s_movk_i32 s0, 0x2010
 ; GFX9-FLATSCR-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-FLATSCR-NEXT:    global_store_dwordx4 v5, v[11:14], s[36:37] offset:160
 ; GFX9-FLATSCR-NEXT:    global_store_dwordx4 v5, v[0:3], s[36:37] offset:144
 ; GFX9-FLATSCR-NEXT:    scratch_load_dwordx4 v[0:3], off, s0 ; 16-byte Folded Reload
 ; GFX9-FLATSCR-NEXT:    s_waitcnt vmcnt(0)
diff --git a/llvm/test/CodeGen/AMDGPU/spill-vgpr.ll b/llvm/test/CodeGen/AMDGPU/spill-vgpr.ll
index 1cc5b7f7d14ee..57496c2be54be 100644
--- a/llvm/test/CodeGen/AMDGPU/spill-vgpr.ll
+++ b/llvm/test/CodeGen/AMDGPU/spill-vgpr.ll
@@ -85,13 +85,13 @@ define amdgpu_kernel void @max_10_vgprs_spill_v32(ptr addrspace(1) %p) #0 {
 ; GFX908-DAG: v_accvgpr_read_b32
 
 ; GFX900: NumVgprs: 256
-; GFX900: ScratchSize: 132
-; GFX908: NumVgprs: 252
+; GFX900: ScratchSize: 148
+; GFX908: NumVgprs: 254
 ; GFX908: ScratchSize: 0
 ; GFX900:    VGPRBlocks: 63
-; GFX908:    VGPRBlocks: 62
+; GFX908:    VGPRBlocks: 63
 ; GFX900:    NumVGPRsForWavesPerEU: 256
-; GFX908:    NumVGPRsForWavesPerEU: 252
+; GFX908:    NumVGPRsForWavesPerEU: 254
 define amdgpu_kernel void @max_256_vgprs_spill_9x32(ptr addrspace(1) %p) #1 {
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
   %p1 = getelementptr inbounds <32 x float>, ptr addrspace(1) %p, i32 %tid
diff --git a/llvm/test/CodeGen/AMDGPU/sra.ll b/llvm/test/CodeGen/AMDGPU/sra.ll
index 8150328dd24f0..386a046113964 100644
--- a/llvm/test/CodeGen/AMDGPU/sra.ll
+++ b/llvm/test/CodeGen/AMDGPU/sra.ll
@@ -282,108 +282,69 @@ define amdgpu_kernel void @ashr_v4i16(ptr addrspace(1) %out, ptr addrspace(1) %i
 ;
 ; VI-LABEL: ashr_v4i16:
 ; VI:       ; %bb.0:
-; VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
-; VI-NEXT:    s_mov_b32 s7, 0xf000
-; VI-NEXT:    s_mov_b32 s6, -1
-; VI-NEXT:    s_mov_b32 s10, s6
-; VI-NEXT:    s_mov_b32 s11, s7
+; VI-NEXT:    s_load_dwordx4 s[4:7], s[4:5], 0x24
+; VI-NEXT:    s_mov_b32 s3, 0xf000
+; VI-NEXT:    s_mov_b32 s2, -1
+; VI-NEXT:    s_mov_b32 s10, s2
+; VI-NEXT:    s_mov_b32 s11, s3
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
-; VI-NEXT:    s_mov_b32 s8, s2
-; VI-NEXT:    s_mov_b32 s9, s3
+; VI-NEXT:    s_mov_b32 s8, s6
+; VI-NEXT:    s_mov_b32 s9, s7
 ; VI-NEXT:    buffer_load_dwordx4 v[0:3], off, s[8:11], 0
-; VI-NEXT:    s_mov_b32 s4, s0
-; VI-NEXT:    s_mov_b32 s5, s1
+; VI-NEXT:    s_mov_b32 s0, s4
+; VI-NEXT:    s_mov_b32 s1, s5
 ; VI-NEXT:    s_waitcnt vmcnt(0)
-; VI-NEXT:    v_readfirstlane_b32 s0, v2
-; VI-NEXT:    v_readfirstlane_b32 s1, v3
-; VI-NEXT:    v_readfirstlane_b32 s2, v0
-; VI-NEXT:    v_readfirstlane_b32 s3, v1
-; VI-NEXT:    s_ashr_i32 s8, s3, 16
-; VI-NEXT:    s_sext_i32_i16 s3, s3
-; VI-NEXT:    s_ashr_i32 s9, s2, 16
-; VI-NEXT:    s_sext_i32_i16 s2, s2
-; VI-NEXT:    s_ashr_i32 s10, s1, 16
-; VI-NEXT:    s_sext_i32_i16 s1, s1
-; VI-NEXT:    s_ashr_i32 s11, s0, 16
-; VI-NEXT:    s_sext_i32_i16 s0, s0
-; VI-NEXT:    s_ashr_i32 s0, s2, s0
-; VI-NEXT:    s_ashr_i32 s2, s9, s11
-; VI-NEXT:    s_ashr_i32 s1, s3, s1
-; VI-NEXT:    s_ashr_i32 s3, s8, s10
-; VI-NEXT:    s_lshl_b32 s3, s3, 16
-; VI-NEXT:    s_and_b32 s1, s1, 0xffff
-; VI-NEXT:    s_lshl_b32 s2, s2, 16
-; VI-NEXT:    s_and_b32 s0, s0, 0xffff
-; VI-NEXT:    s_or_b32 s1, s1, s3
-; VI-NEXT:    s_or_b32 s0, s0, s2
-; VI-NEXT:    v_mov_b32_e32 v0, s0
-; VI-NEXT:    v_mov_b32_e32 v1, s1
-; VI-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
+; VI-NEXT:    v_readfirstlane_b32 s4, v2
+; VI-NEXT:    v_readfirstlane_b32 s5, v3
+; VI-NEXT:    v_readfirstlane_b32 s6, v0
+; VI-NEXT:    v_readfirstlane_b32 s7, v1
+; VI-NEXT:    s_ashr_i32 s8, s7, 16
+; VI-NEXT:    s_sext_i32_i16 s7, s7
+; VI-NEXT:    s_ashr_i32 s9, s6, 16
+; VI-NEXT:    s_sext_i32_i16 s6, s6
+; VI-NEXT:    s_ashr_i32 s10, s5, 16
+; VI-NEXT:    s_sext_i32_i16 s5, s5
+; VI-NEXT:    s_ashr_i32 s11, s4, 16
+; VI-NEXT:    s_sext_i32_i16 s4, s4
+; VI-NEXT:    s_ashr_i32 s4, s6, s4
+; VI-NEXT:    s_ashr_i32 s6, s9, s11
+; VI-NEXT:    s_ashr_i32 s5, s7, s5
+; VI-NEXT:    s_ashr_i32 s7, s8, s10
+; VI-NEXT:    s_lshl_b32 s7, s7, 16
+; VI-NEXT:    s_and_b32 s5, s5, 0xffff
+; VI-NEXT:    s_lshl_b32 s6, s6, 16
+; VI-NEXT:    s_and_b32 s4, s4, 0xffff
+; VI-NEXT:    s_or_b32 s5, s5, s7
+; VI-NEXT:    s_or_b32 s4, s4, s6
+; VI-NEXT:    v_mov_b32_e32 v0, s4
+; VI-NEXT:    v_mov_b32_e32 v1, s5
+; VI-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
 ; VI-NEXT:    s_endpgm
 ;
 ; EG-LABEL: ashr_v4i16:
 ; EG:       ; %bb.0:
-; EG-NEXT:    ALU 1, @8, KC0[CB0:0-32], KC1[]
+; EG-NEXT:    ALU 0, @8, KC0[CB0:0-32], KC1[]
 ; EG-NEXT:    TEX 0 @6
-; EG-NEXT:    ALU 48, @10, KC0[CB0:0-32], KC1[]
-; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T10.XY, T9.X, 1
+; EG-NEXT:    ALU 10, @9, KC0[CB0:0-32], KC1[]
+; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T7.XY, T8.X, 1
 ; EG-NEXT:    CF_END
 ; EG-NEXT:    PAD
 ; EG-NEXT:    Fetch clause starting at 6:
-; EG-NEXT:     VTX_READ_128 T9.XYZW, T9.X, 0, #1
+; EG-NEXT:     VTX_READ_128 T7.XYZW, T7.X, 0, #1
 ; EG-NEXT:    ALU clause starting at 8:
-; EG-NEXT:     MOV * T0.Y, T6.X,
-; EG-NEXT:     MOV * T9.X, KC0[2].Z,
-; EG-NEXT:    ALU clause starting at 10:
-; EG-NEXT:     BFE_INT T0.W, T9.X, 0.0, literal.x,
-; EG-NEXT:     AND_INT * T1.W, T9.Z, literal.y,
-; EG-NEXT:    16(2.242078e-44), 65535(9.183409e-41)
-; EG-NEXT:     ASHR * T0.W, PV.W, PS,
-; EG-NEXT:     AND_INT T0.W, PV.W, literal.x,
-; EG-NEXT:     AND_INT * T1.W, T0.Y, literal.y,
-; EG-NEXT:    65535(9.183409e-41), -65536(nan)
-; EG-NEXT:     OR_INT * T0.W, PS, PV.W,
-; EG-NEXT:     MOV * T6.X, PV.W,
-; EG-NEXT:     MOV T0.Y, PV.X,
-; EG-NEXT:     LSHR * T0.W, T9.X, literal.x,
-; EG-NEXT:    16(2.242078e-44), 0(0.000000e+00)
-; EG-NEXT:     BFE_INT T0.W, PV.W, 0.0, literal.x,
-; EG-NEXT:     LSHR * T1.W, T9.Z, literal.x,
-; EG-NEXT:    16(2.242078e-44), 0(0.000000e+00)
-; EG-NEXT:     ASHR T0.W, PV.W, PS,
-; EG-NEXT:     AND_INT * T1.W, T0.Y, literal.x,
-; EG-NEXT:    65535(9.183409e-41), 0(0.000000e+00)
-; EG-NEXT:     LSHL * T0.W, PV.W, literal.x,
-; EG-NEXT:    16(2.242078e-44), 0(0.000000e+00)
-; EG-NEXT:     OR_INT * T0.W, T1.W, PV.W,
-; EG-NEXT:     MOV T6.X, PV.W,
-; EG-NEXT:     MOV T0.Y, T7.X,
-; EG-NEXT:     BFE_INT T0.W, T9.Y, 0.0, literal.x,
-; EG-NEXT:     AND_INT * T1.W, T9.W, literal.y,
+; EG-NEXT:     MOV * T7.X, KC0[2].Z,
+; EG-NEXT:    ALU clause starting at 9:
+; EG-NEXT:     LSHR T0.Z, T7.X, literal.x,
+; EG-NEXT:     BFE_INT T0.W, T7.X, 0.0, literal.x,
+; EG-NEXT:     AND_INT * T1.W, T7.Z, literal.y,
 ; EG-NEXT:    16(2.242078e-44), 65535(9.183409e-41)
-; EG-NEXT:     ASHR T0.W, PV.W, PS,
-; EG-NEXT:     AND_INT * T1.W, PV.Y, literal.x,
-; EG-NEXT:    -65536(nan), 0(0.000000e+00)
-; EG-NEXT:     AND_INT * T0.W, PV.W, literal.x,
-; EG-NEXT:    65535(9.183409e-41), 0(0.000000e+00)
-; EG-NEXT:     OR_INT * T0.W, T1.W, PV.W,
-; EG-NEXT:     MOV * T7.X, PV.W,
-; EG-NEXT:     MOV T0.Y, PV.X,
-; EG-NEXT:     LSHR * T0.W, T9.Y, literal.x,
-; EG-NEXT:    16(2.242078e-44), 0(0.000000e+00)
-; EG-NEXT:     BFE_INT T0.W, PV.W, 0.0, literal.x,
-; EG-NEXT:     LSHR * T1.W, T9.W, literal.x,
-; EG-NEXT:    16(2.242078e-44), 0(0.000000e+00)
-; EG-NEXT:     ASHR T0.W, PV.W, PS,
-; EG-NEXT:     AND_INT * T1.W, T0.Y, literal.x,
-; EG-NEXT:    65535(9.183409e-41), 0(0.000000e+00)
-; EG-NEXT:     LSHL * T0.W, PV.W, literal.x,
+; EG-NEXT:     ASHR T7.X, PV.W, PS,
+; EG-NEXT:     BFE_INT T0.W, PV.Z, 0.0, literal.x,
+; EG-NEXT:     LSHR * T1.W, T7.Z, literal.x,
 ; EG-NEXT:    16(2.242078e-44), 0(0.000000e+00)
-; EG-NEXT:     LSHR T9.X, KC0[2].Y, literal.x,
-; EG-NEXT:     OR_INT * T10.Y, T1.W, PV.W,
+; EG-NEXT:     LSHR T8.X, KC0[2].Y, literal.x,
+; EG-NEXT:     ASHR * T7.Y, PV.W, PS,
 ; EG-NEXT:    2(2.802597e-45), 0(0.000000e+00)
-; EG-NEXT:     MOV T7.X, PV.Y,
-; EG-NEXT:     MOV * T10.X, T6.X,
   %b_ptr = getelementptr <4 x i16>, ptr addrspace(1) %in, i16 1
   %a = load <4 x i16>, ptr addrspace(1) %in
   %b = load <4 x i16>, ptr addrspace(1) %b_ptr
diff --git a/llvm/test/CodeGen/AMDGPU/stack-pointer-offset-relative-frameindex.ll b/llvm/test/CodeGen/AMDGPU/stack-pointer-offset-relative-frameindex.ll
index 5641c43c40084..d4d3b37a0ed1e 100644
--- a/llvm/test/CodeGen/AMDGPU/stack-pointer-offset-relative-frameindex.ll
+++ b/llvm/test/CodeGen/AMDGPU/stack-pointer-offset-relative-frameindex.ll
@@ -21,10 +21,10 @@ define amdgpu_kernel void @kernel_background_evaluate(ptr addrspace(5) %kg, ptr
 ; MUBUF-NEXT:    v_mov_b32_e32 v2, 0x4000
 ; MUBUF-NEXT:    v_mov_b32_e32 v3, 0
 ; MUBUF-NEXT:    v_mov_b32_e32 v4, 0x400000
-; MUBUF-NEXT:    s_mov_b32 s32, 0xc0000
 ; MUBUF-NEXT:    s_getpc_b64 s[4:5]
 ; MUBUF-NEXT:    s_add_u32 s4, s4, svm_eval_nodes@rel32@lo+4
 ; MUBUF-NEXT:    s_addc_u32 s5, s5, svm_eval_nodes@rel32@hi+12
+; MUBUF-NEXT:    s_mov_b32 s32, 0xc0000
 ; MUBUF-NEXT:    s_waitcnt lgkmcnt(0)
 ; MUBUF-NEXT:    v_mov_b32_e32 v0, s0
 ; MUBUF-NEXT:    s_mov_b64 s[0:1], s[36:37]
@@ -85,10 +85,10 @@ define amdgpu_kernel void @kernel_background_evaluate(ptr addrspace(5) %kg, ptr
 ; MUBUF11-NEXT:    v_mov_b32_e32 v1, 0x2000
 ; MUBUF11-NEXT:    v_dual_mov_b32 v2, 0x4000 :: v_dual_mov_b32 v3, 0
 ; MUBUF11-NEXT:    v_mov_b32_e32 v4, 0x400000
-; MUBUF11-NEXT:    s_movk_i32 s32, 0x6000
 ; MUBUF11-NEXT:    s_getpc_b64 s[0:1]
 ; MUBUF11-NEXT:    s_add_u32 s0, s0, svm_eval_nodes@rel32@lo+4
 ; MUBUF11-NEXT:    s_addc_u32 s1, s1, svm_eval_nodes@rel32@hi+12
+; MUBUF11-NEXT:    s_movk_i32 s32, 0x6000
 ; MUBUF11-NEXT:    s_waitcnt lgkmcnt(0)
 ; MUBUF11-NEXT:    v_mov_b32_e32 v0, s2
 ; MUBUF11-NEXT:    s_swappc_b64 s[30:31], s[0:1]
@@ -112,10 +112,10 @@ define amdgpu_kernel void @kernel_background_evaluate(ptr addrspace(5) %kg, ptr
 ; FLATSCR11-NEXT:    v_mov_b32_e32 v1, 0x2000
 ; FLATSCR11-NEXT:    v_dual_mov_b32 v2, 0x4000 :: v_dual_mov_b32 v3, 0
 ; FLATSCR11-NEXT:    v_mov_b32_e32 v4, 0x400000
-; FLATSCR11-NEXT:    s_movk_i32 s32, 0x6000
 ; FLATSCR11-NEXT:    s_getpc_b64 s[0:1]
 ; FLATSCR11-NEXT:    s_add_u32 s0, s0, svm_eval_nodes@rel32@lo+4
 ; FLATSCR11-NEXT:    s_addc_u32 s1, s1, svm_eval_nodes@rel32@hi+12
+; FLATSCR11-NEXT:    s_movk_i32 s32, 0x6000
 ; FLATSCR11-NEXT:    s_waitcnt lgkmcnt(0)
 ; FLATSCR11-NEXT:    v_mov_b32_e32 v0, s2
 ; FLATSCR11-NEXT:    s_swappc_b64 s[30:31], s[0:1]
diff --git a/llvm/test/CodeGen/AMDGPU/strict_fptrunc.ll b/llvm/test/CodeGen/AMDGPU/strict_fptrunc.ll
index 0f80327638a9c..52d882590cbce 100644
--- a/llvm/test/CodeGen/AMDGPU/strict_fptrunc.ll
+++ b/llvm/test/CodeGen/AMDGPU/strict_fptrunc.ll
@@ -2,8 +2,9 @@
 ; RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=tahiti < %s | FileCheck -check-prefixes=GCN,SI %s
 ; RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx803 < %s | FileCheck -check-prefixes=GCN,GFX89,GFX8 %s
 ; RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx900 < %s | FileCheck -check-prefixes=GCN,GFX89,GFX9 %s
-; RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx1010 < %s | FileCheck -check-prefixes=GCN,GFX1011,GFX10 %s
-; RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx1100 -amdgpu-enable-delay-alu=0 < %s | FileCheck -check-prefixes=GCN,GFX1011,GFX11 %s
+; RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx1010 < %s | FileCheck -check-prefixes=GCN,GFX10 %s
+; RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx1100 -mattr=+real-true16 -amdgpu-enable-delay-alu=0 < %s | FileCheck -check-prefixes=GCN,GFX11-TRUE16 %s
+; RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx1100 -mattr=-real-true16 -amdgpu-enable-delay-alu=0 < %s | FileCheck -check-prefixes=GCN,GFX11-FAKE16 %s
 
 define half @v_constrained_fptrunc_f32_to_f16_fpexcept_strict(float %arg) #0 {
 ; SI-LABEL: v_constrained_fptrunc_f32_to_f16_fpexcept_strict:
@@ -20,11 +21,23 @@ define half @v_constrained_fptrunc_f32_to_f16_fpexcept_strict(float %arg) #0 {
 ; GFX89-NEXT:    v_cvt_f16_f32_e32 v0, v0
 ; GFX89-NEXT:    s_setpc_b64 s[30:31]
 ;
-; GFX1011-LABEL: v_constrained_fptrunc_f32_to_f16_fpexcept_strict:
-; GFX1011:       ; %bb.0:
-; GFX1011-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX1011-NEXT:    v_cvt_f16_f32_e32 v0, v0
-; GFX1011-NEXT:    s_setpc_b64 s[30:31]
+; GFX10-LABEL: v_constrained_fptrunc_f32_to_f16_fpexcept_strict:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    v_cvt_f16_f32_e32 v0, v0
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-TRUE16-LABEL: v_constrained_fptrunc_f32_to_f16_fpexcept_strict:
+; GFX11-TRUE16:       ; %bb.0:
+; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT:    v_cvt_f16_f32_e32 v0.l, v0
+; GFX11-TRUE16-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: v_constrained_fptrunc_f32_to_f16_fpexcept_strict:
+; GFX11-FAKE16:       ; %bb.0:
+; GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT:    v_cvt_f16_f32_e32 v0, v0
+; GFX11-FAKE16-NEXT:    s_setpc_b64 s[30:31]
   %val = call half @llvm.experimental.constrained.fptrunc.f16.f32(float %arg, metadata !"round.tonearest", metadata !"fpexcept.strict")
   ret half %val
 }
@@ -58,13 +71,29 @@ define <2 x half> @v_constrained_fptrunc_v2f32_to_v2f16_fpexcept_strict(<2 x flo
 ; GFX9-NEXT:    v_perm_b32 v0, v1, v0, s4
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
-; GFX1011-LABEL: v_constrained_fptrunc_v2f32_to_v2f16_fpexcept_strict:
-; GFX1011:       ; %bb.0:
-; GFX1011-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX1011-NEXT:    v_cvt_f16_f32_e32 v0, v0
-; GFX1011-NEXT:    v_cvt_f16_f32_e32 v1, v1
-; GFX1011-NEXT:    v_perm_b32 v0, v1, v0, 0x5040100
-; GFX1011-NEXT:    s_setpc_b64 s[30:31]
+; GFX10-LABEL: v_constrained_fptrunc_v2f32_to_v2f16_fpexcept_strict:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    v_cvt_f16_f32_e32 v0, v0
+; GFX10-NEXT:    v_cvt_f16_f32_e32 v1, v1
+; GFX10-NEXT:    v_perm_b32 v0, v1, v0, 0x5040100
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-TRUE16-LABEL: v_constrained_fptrunc_v2f32_to_v2f16_fpexcept_strict:
+; GFX11-TRUE16:       ; %bb.0:
+; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT:    v_cvt_f16_f32_e32 v1.l, v1
+; GFX11-TRUE16-NEXT:    v_cvt_f16_f32_e32 v0.l, v0
+; GFX11-TRUE16-NEXT:    v_perm_b32 v0, v1, v0, 0x5040100
+; GFX11-TRUE16-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: v_constrained_fptrunc_v2f32_to_v2f16_fpexcept_strict:
+; GFX11-FAKE16:       ; %bb.0:
+; GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT:    v_cvt_f16_f32_e32 v0, v0
+; GFX11-FAKE16-NEXT:    v_cvt_f16_f32_e32 v1, v1
+; GFX11-FAKE16-NEXT:    v_perm_b32 v0, v1, v0, 0x5040100
+; GFX11-FAKE16-NEXT:    s_setpc_b64 s[30:31]
   %val = call <2 x half> @llvm.experimental.constrained.fptrunc.v2f16.v2f32(<2 x float> %arg, metadata !"round.tonearest", metadata !"fpexcept.strict")
   ret <2 x half> %val
 }
@@ -103,14 +132,33 @@ define <3 x half> @v_constrained_fptrunc_v3f32_to_v3f16_fpexcept_strict(<3 x flo
 ; GFX9-NEXT:    v_perm_b32 v0, v3, v0, s4
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
-; GFX1011-LABEL: v_constrained_fptrunc_v3f32_to_v3f16_fpexcept_strict:
-; GFX1011:       ; %bb.0:
-; GFX1011-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX1011-NEXT:    v_cvt_f16_f32_e32 v0, v0
-; GFX1011-NEXT:    v_cvt_f16_f32_e32 v1, v1
-; GFX1011-NEXT:    v_perm_b32 v0, v1, v0, 0x5040100
-; GFX1011-NEXT:    v_cvt_f16_f32_e32 v1, v2
-; GFX1011-NEXT:    s_setpc_b64 s[30:31]
+; GFX10-LABEL: v_constrained_fptrunc_v3f32_to_v3f16_fpexcept_strict:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    v_cvt_f16_f32_e32 v0, v0
+; GFX10-NEXT:    v_cvt_f16_f32_e32 v1, v1
+; GFX10-NEXT:    v_perm_b32 v0, v1, v0, 0x5040100
+; GFX10-NEXT:    v_cvt_f16_f32_e32 v1, v2
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-TRUE16-LABEL: v_constrained_fptrunc_v3f32_to_v3f16_fpexcept_strict:
+; GFX11-TRUE16:       ; %bb.0:
+; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT:    v_cvt_f16_f32_e32 v1.l, v1
+; GFX11-TRUE16-NEXT:    v_cvt_f16_f32_e32 v0.l, v0
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v3.l, v1.l
+; GFX11-TRUE16-NEXT:    v_cvt_f16_f32_e32 v1.l, v2
+; GFX11-TRUE16-NEXT:    v_perm_b32 v0, v3, v0, 0x5040100
+; GFX11-TRUE16-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: v_constrained_fptrunc_v3f32_to_v3f16_fpexcept_strict:
+; GFX11-FAKE16:       ; %bb.0:
+; GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT:    v_cvt_f16_f32_e32 v0, v0
+; GFX11-FAKE16-NEXT:    v_cvt_f16_f32_e32 v1, v1
+; GFX11-FAKE16-NEXT:    v_perm_b32 v0, v1, v0, 0x5040100
+; GFX11-FAKE16-NEXT:    v_cvt_f16_f32_e32 v1, v2
+; GFX11-FAKE16-NEXT:    s_setpc_b64 s[30:31]
   %val = call <3 x half> @llvm.experimental.constrained.fptrunc.v3f16.v3f32(<3 x float> %arg, metadata !"round.tonearest", metadata !"fpexcept.strict")
   ret <3 x half> %val
 }
@@ -181,12 +229,26 @@ define half @v_constrained_fneg_fptrunc_f32_to_f16_fpexcept_strict(float %arg) #
 ; GFX89-NEXT:    v_xor_b32_e32 v0, 0x8000, v0
 ; GFX89-NEXT:    s_setpc_b64 s[30:31]
 ;
-; GFX1011-LABEL: v_constrained_fneg_fptrunc_f32_to_f16_fpexcept_strict:
-; GFX1011:       ; %bb.0:
-; GFX1011-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX1011-NEXT:    v_cvt_f16_f32_e32 v0, v0
-; GFX1011-NEXT:    v_xor_b32_e32 v0, 0x8000, v0
-; GFX1011-NEXT:    s_setpc_b64 s[30:31]
+; GFX10-LABEL: v_constrained_fneg_fptrunc_f32_to_f16_fpexcept_strict:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    v_cvt_f16_f32_e32 v0, v0
+; GFX10-NEXT:    v_xor_b32_e32 v0, 0x8000, v0
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-TRUE16-LABEL: v_constrained_fneg_fptrunc_f32_to_f16_fpexcept_strict:
+; GFX11-TRUE16:       ; %bb.0:
+; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT:    v_cvt_f16_f32_e32 v0.l, v0
+; GFX11-TRUE16-NEXT:    v_xor_b16 v0.l, 0x8000, v0.l
+; GFX11-TRUE16-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: v_constrained_fneg_fptrunc_f32_to_f16_fpexcept_strict:
+; GFX11-FAKE16:       ; %bb.0:
+; GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT:    v_cvt_f16_f32_e32 v0, v0
+; GFX11-FAKE16-NEXT:    v_xor_b32_e32 v0, 0x8000, v0
+; GFX11-FAKE16-NEXT:    s_setpc_b64 s[30:31]
   %val = call half @llvm.experimental.constrained.fptrunc.f16.f32(float %arg, metadata !"round.tonearest", metadata !"fpexcept.strict")
   %neg.val = fneg half %val
   ret half %neg.val
@@ -207,11 +269,23 @@ define half @v_constrained_fptrunc_fneg_f32_to_f16_fpexcept_strict(float %arg) #
 ; GFX89-NEXT:    v_cvt_f16_f32_e64 v0, -v0
 ; GFX89-NEXT:    s_setpc_b64 s[30:31]
 ;
-; GFX1011-LABEL: v_constrained_fptrunc_fneg_f32_to_f16_fpexcept_strict:
-; GFX1011:       ; %bb.0:
-; GFX1011-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX1011-NEXT:    v_cvt_f16_f32_e64 v0, -v0
-; GFX1011-NEXT:    s_setpc_b64 s[30:31]
+; GFX10-LABEL: v_constrained_fptrunc_fneg_f32_to_f16_fpexcept_strict:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    v_cvt_f16_f32_e64 v0, -v0
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-TRUE16-LABEL: v_constrained_fptrunc_fneg_f32_to_f16_fpexcept_strict:
+; GFX11-TRUE16:       ; %bb.0:
+; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT:    v_cvt_f16_f32_e64 v0.l, -v0
+; GFX11-TRUE16-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: v_constrained_fptrunc_fneg_f32_to_f16_fpexcept_strict:
+; GFX11-FAKE16:       ; %bb.0:
+; GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT:    v_cvt_f16_f32_e64 v0, -v0
+; GFX11-FAKE16-NEXT:    s_setpc_b64 s[30:31]
   %neg.arg = fneg float %arg
   %val = call half @llvm.experimental.constrained.fptrunc.f16.f32(float %neg.arg, metadata !"round.tonearest", metadata !"fpexcept.strict")
   ret half %val
@@ -255,11 +329,23 @@ define void @v_constrained_fptrunc_f32_to_f16_fpexcept_strict_noabi(float %arg,
 ; GFX89-NEXT:    v_cvt_f16_f32_e32 v0, v0
 ; GFX89-NEXT:    s_setpc_b64 s[30:31]
 ;
-; GFX1011-LABEL: v_constrained_fptrunc_f32_to_f16_fpexcept_strict_noabi:
-; GFX1011:       ; %bb.0:
-; GFX1011-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX1011-NEXT:    v_cvt_f16_f32_e32 v0, v0
-; GFX1011-NEXT:    s_setpc_b64 s[30:31]
+; GFX10-LABEL: v_constrained_fptrunc_f32_to_f16_fpexcept_strict_noabi:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    v_cvt_f16_f32_e32 v0, v0
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-TRUE16-LABEL: v_constrained_fptrunc_f32_to_f16_fpexcept_strict_noabi:
+; GFX11-TRUE16:       ; %bb.0:
+; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT:    v_cvt_f16_f32_e32 v0.l, v0
+; GFX11-TRUE16-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: v_constrained_fptrunc_f32_to_f16_fpexcept_strict_noabi:
+; GFX11-FAKE16:       ; %bb.0:
+; GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT:    v_cvt_f16_f32_e32 v0, v0
+; GFX11-FAKE16-NEXT:    s_setpc_b64 s[30:31]
   %result = call half @llvm.experimental.constrained.fptrunc.f16.f32(float %arg, metadata !"round.tonearest", metadata !"fpexcept.strict")
   ret void
 }
@@ -316,14 +402,23 @@ define void @v_constrained_fptrunc_v2f32_to_v2f16_fpexcept_strict_noabi(<2 x flo
 ; GFX10-NEXT:    global_store_dword v[2:3], v0, off
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
 ;
-; GFX11-LABEL: v_constrained_fptrunc_v2f32_to_v2f16_fpexcept_strict_noabi:
-; GFX11:       ; %bb.0:
-; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT:    v_cvt_f16_f32_e32 v0, v0
-; GFX11-NEXT:    v_cvt_f16_f32_e32 v1, v1
-; GFX11-NEXT:    v_perm_b32 v0, v1, v0, 0x5040100
-; GFX11-NEXT:    global_store_b32 v[2:3], v0, off
-; GFX11-NEXT:    s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: v_constrained_fptrunc_v2f32_to_v2f16_fpexcept_strict_noabi:
+; GFX11-TRUE16:       ; %bb.0:
+; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT:    v_cvt_f16_f32_e32 v1.l, v1
+; GFX11-TRUE16-NEXT:    v_cvt_f16_f32_e32 v0.l, v0
+; GFX11-TRUE16-NEXT:    v_perm_b32 v0, v1, v0, 0x5040100
+; GFX11-TRUE16-NEXT:    global_store_b32 v[2:3], v0, off
+; GFX11-TRUE16-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: v_constrained_fptrunc_v2f32_to_v2f16_fpexcept_strict_noabi:
+; GFX11-FAKE16:       ; %bb.0:
+; GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT:    v_cvt_f16_f32_e32 v0, v0
+; GFX11-FAKE16-NEXT:    v_cvt_f16_f32_e32 v1, v1
+; GFX11-FAKE16-NEXT:    v_perm_b32 v0, v1, v0, 0x5040100
+; GFX11-FAKE16-NEXT:    global_store_b32 v[2:3], v0, off
+; GFX11-FAKE16-NEXT:    s_setpc_b64 s[30:31]
   %result = call <2 x half> @llvm.experimental.constrained.fptrunc.v2f16.v2f32(<2 x float> %arg, metadata !"round.tonearest", metadata !"fpexcept.strict")
   store <2 x half> %result, ptr addrspace(1) %ptr
   ret void
@@ -344,11 +439,23 @@ define void @v_constrained_fptrunc_f32_to_f16_fpexcept_strict_noabi_fneg(float %
 ; GFX89-NEXT:    v_cvt_f16_f32_e64 v0, -v0
 ; GFX89-NEXT:    s_setpc_b64 s[30:31]
 ;
-; GFX1011-LABEL: v_constrained_fptrunc_f32_to_f16_fpexcept_strict_noabi_fneg:
-; GFX1011:       ; %bb.0:
-; GFX1011-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX1011-NEXT:    v_cvt_f16_f32_e64 v0, -v0
-; GFX1011-NEXT:    s_setpc_b64 s[30:31]
+; GFX10-LABEL: v_constrained_fptrunc_f32_to_f16_fpexcept_strict_noabi_fneg:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    v_cvt_f16_f32_e64 v0, -v0
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-TRUE16-LABEL: v_constrained_fptrunc_f32_to_f16_fpexcept_strict_noabi_fneg:
+; GFX11-TRUE16:       ; %bb.0:
+; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT:    v_cvt_f16_f32_e64 v0.l, -v0
+; GFX11-TRUE16-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: v_constrained_fptrunc_f32_to_f16_fpexcept_strict_noabi_fneg:
+; GFX11-FAKE16:       ; %bb.0:
+; GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT:    v_cvt_f16_f32_e64 v0, -v0
+; GFX11-FAKE16-NEXT:    s_setpc_b64 s[30:31]
   %neg.arg = fneg float %arg
   %result = call half @llvm.experimental.constrained.fptrunc.f16.f32(float %neg.arg, metadata !"round.tonearest", metadata !"fpexcept.strict")
   ret void
@@ -369,11 +476,23 @@ define void @v_constrained_fptrunc_f32_to_f16_fpexcept_strict_noabi_fabs(float %
 ; GFX89-NEXT:    v_cvt_f16_f32_e64 v0, |v0|
 ; GFX89-NEXT:    s_setpc_b64 s[30:31]
 ;
-; GFX1011-LABEL: v_constrained_fptrunc_f32_to_f16_fpexcept_strict_noabi_fabs:
-; GFX1011:       ; %bb.0:
-; GFX1011-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX1011-NEXT:    v_cvt_f16_f32_e64 v0, |v0|
-; GFX1011-NEXT:    s_setpc_b64 s[30:31]
+; GFX10-LABEL: v_constrained_fptrunc_f32_to_f16_fpexcept_strict_noabi_fabs:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    v_cvt_f16_f32_e64 v0, |v0|
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-TRUE16-LABEL: v_constrained_fptrunc_f32_to_f16_fpexcept_strict_noabi_fabs:
+; GFX11-TRUE16:       ; %bb.0:
+; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT:    v_cvt_f16_f32_e64 v0.l, |v0|
+; GFX11-TRUE16-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: v_constrained_fptrunc_f32_to_f16_fpexcept_strict_noabi_fabs:
+; GFX11-FAKE16:       ; %bb.0:
+; GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT:    v_cvt_f16_f32_e64 v0, |v0|
+; GFX11-FAKE16-NEXT:    s_setpc_b64 s[30:31]
   %abs.arg = call float @llvm.fabs.f32(float %arg) #0
   %result = call half @llvm.experimental.constrained.fptrunc.f16.f32(float %abs.arg, metadata !"round.tonearest", metadata !"fpexcept.strict")
   ret void
diff --git a/llvm/test/CodeGen/AMDGPU/sub.ll b/llvm/test/CodeGen/AMDGPU/sub.ll
index 80ccd1ffe0294..001c35ef30cc6 100644
--- a/llvm/test/CodeGen/AMDGPU/sub.ll
+++ b/llvm/test/CodeGen/AMDGPU/sub.ll
@@ -858,6 +858,7 @@ define amdgpu_kernel void @v_test_sub_v2i64(ptr addrspace(1) %out, ptr addrspace
 ; GFX12-NEXT:    v_sub_co_u32 v2, vcc_lo, v2, v6
 ; GFX12-NEXT:    v_sub_co_ci_u32_e32 v3, vcc_lo, v3, v7, vcc_lo
 ; GFX12-NEXT:    v_sub_co_u32 v0, vcc_lo, v0, v4
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_sub_co_ci_u32_e32 v1, vcc_lo, v1, v5, vcc_lo
 ; GFX12-NEXT:    global_store_b128 v8, v[0:3], s[0:1]
 ; GFX12-NEXT:    s_endpgm
@@ -990,10 +991,13 @@ define amdgpu_kernel void @v_test_sub_v4i64(ptr addrspace(1) %out, ptr addrspace
 ; GFX12-NEXT:    v_sub_co_ci_u32_e32 v3, vcc_lo, v3, v7, vcc_lo
 ; GFX12-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-NEXT:    v_sub_co_u32 v10, vcc_lo, v10, v14
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_sub_co_ci_u32_e32 v11, vcc_lo, v11, v15, vcc_lo
 ; GFX12-NEXT:    v_sub_co_u32 v8, vcc_lo, v8, v12
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_sub_co_ci_u32_e32 v9, vcc_lo, v9, v13, vcc_lo
 ; GFX12-NEXT:    v_sub_co_u32 v0, vcc_lo, v0, v4
+; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_sub_co_ci_u32_e32 v1, vcc_lo, v1, v5, vcc_lo
 ; GFX12-NEXT:    s_clause 0x1
 ; GFX12-NEXT:    global_store_b128 v16, v[8:11], s[0:1] offset:16
diff --git a/llvm/test/CodeGen/AMDGPU/sub_i1.ll b/llvm/test/CodeGen/AMDGPU/sub_i1.ll
index a6ab1bd9e19f1..19d012fc074f8 100644
--- a/llvm/test/CodeGen/AMDGPU/sub_i1.ll
+++ b/llvm/test/CodeGen/AMDGPU/sub_i1.ll
@@ -4,8 +4,8 @@
 
 
 ; GCN-LABEL: {{^}}sub_var_var_i1:
-; WAVE32: s_xor_b32
-; WAVE64: s_xor_b64
+; WAVE32: v_xor_b32_e32
+; WAVE64: v_xor_b32_e32
 define amdgpu_kernel void @sub_var_var_i1(ptr addrspace(1) %out, ptr addrspace(1) %in0, ptr addrspace(1) %in1) {
   %a = load volatile i1, ptr addrspace(1) %in0
   %b = load volatile i1, ptr addrspace(1) %in1
@@ -15,8 +15,8 @@ define amdgpu_kernel void @sub_var_var_i1(ptr addrspace(1) %out, ptr addrspace(1
 }
 
 ; GCN-LABEL: {{^}}sub_var_imm_i1:
-; WAVE32: s_not_b32
-; WAVE64: s_not_b64
+; WAVE32: s_xor_b32
+; WAVE64: s_xor_b64
 define amdgpu_kernel void @sub_var_imm_i1(ptr addrspace(1) %out, ptr addrspace(1) %in) {
   %a = load volatile i1, ptr addrspace(1) %in
   %sub = sub i1 %a, 1
@@ -26,8 +26,8 @@ define amdgpu_kernel void @sub_var_imm_i1(ptr addrspace(1) %out, ptr addrspace(1
 
 ; GCN-LABEL: {{^}}sub_i1_cf:
 ; GCN: ; %endif
-; WAVE32: s_not_b32
-; WAVE64: s_not_b64
+; WAVE32: s_xor_b32
+; WAVE64: s_xor_b64
 define amdgpu_kernel void @sub_i1_cf(ptr addrspace(1) %out, ptr addrspace(1) %a, ptr addrspace(1) %b) {
 entry:
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
diff --git a/llvm/test/CodeGen/AMDGPU/swdev373493.ll b/llvm/test/CodeGen/AMDGPU/swdev373493.ll
index 4f33e19835172..5917522f2bfa0 100644
--- a/llvm/test/CodeGen/AMDGPU/swdev373493.ll
+++ b/llvm/test/CodeGen/AMDGPU/swdev373493.ll
@@ -25,17 +25,20 @@ define hidden fastcc void @bar(i32 %arg, ptr %arg1, ptr %arg2, ptr %arg3, ptr %a
 ; CHECK-NEXT:    s_getpc_b64 s[18:19]
 ; CHECK-NEXT:    s_add_u32 s18, s18, global@rel32@lo+1948
 ; CHECK-NEXT:    s_addc_u32 s19, s19, global@rel32@hi+1956
-; CHECK-NEXT:    v_mov_b32_e32 v5, 0
-; CHECK-NEXT:    v_mov_b32_e32 v0, s18
-; CHECK-NEXT:    v_mov_b32_e32 v1, s19
 ; CHECK-NEXT:    s_getpc_b64 s[16:17]
 ; CHECK-NEXT:    s_add_u32 s16, s16, eggs@rel32@lo+4
 ; CHECK-NEXT:    s_addc_u32 s17, s17, eggs@rel32@hi+12
+; CHECK-NEXT:    v_mov_b32_e32 v5, 0
+; CHECK-NEXT:    v_mov_b32_e32 v0, s18
+; CHECK-NEXT:    v_mov_b32_e32 v1, s19
 ; CHECK-NEXT:    s_setpc_b64 s[16:17]
 ; CHECK-NEXT:  .LBB0_3: ; %LeafBlock1
 ; CHECK-NEXT:    s_cbranch_scc0 .LBB0_5
 ; CHECK-NEXT:  ; %bb.4: ; %bb8
 ; CHECK-NEXT:    v_mov_b32_e32 v0, v1
+; CHECK-NEXT:    s_getpc_b64 s[16:17]
+; CHECK-NEXT:    s_add_u32 s16, s16, quux@rel32@lo+4
+; CHECK-NEXT:    s_addc_u32 s17, s17, quux@rel32@hi+12
 ; CHECK-NEXT:    v_mov_b32_e32 v1, v2
 ; CHECK-NEXT:    v_mov_b32_e32 v2, v6
 ; CHECK-NEXT:    v_mov_b32_e32 v3, v7
@@ -47,9 +50,6 @@ define hidden fastcc void @bar(i32 %arg, ptr %arg1, ptr %arg2, ptr %arg3, ptr %a
 ; CHECK-NEXT:    v_mov_b32_e32 v9, v13
 ; CHECK-NEXT:    v_mov_b32_e32 v10, v14
 ; CHECK-NEXT:    v_mov_b32_e32 v11, v15
-; CHECK-NEXT:    s_getpc_b64 s[16:17]
-; CHECK-NEXT:    s_add_u32 s16, s16, quux@rel32@lo+4
-; CHECK-NEXT:    s_addc_u32 s17, s17, quux@rel32@hi+12
 ; CHECK-NEXT:    s_setpc_b64 s[16:17]
 ; CHECK-NEXT:  .LBB0_5: ; %bb9
 ; CHECK-NEXT:    s_setpc_b64 s[30:31]
diff --git a/llvm/test/CodeGen/AMDGPU/tail-call-inreg-arguments.ll b/llvm/test/CodeGen/AMDGPU/tail-call-inreg-arguments.ll
index 0689c0585d8a6..80dae9142870a 100644
--- a/llvm/test/CodeGen/AMDGPU/tail-call-inreg-arguments.ll
+++ b/llvm/test/CodeGen/AMDGPU/tail-call-inreg-arguments.ll
@@ -7,10 +7,10 @@ define void @tail_call_i32_inreg_uniform(i32 inreg %sgpr) {
 ; CHECK-LABEL: tail_call_i32_inreg_uniform:
 ; CHECK:       ; %bb.0:
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    s_mov_b32 s0, s16
 ; CHECK-NEXT:    s_getpc_b64 s[18:19]
 ; CHECK-NEXT:    s_add_u32 s18, s18, void_func_i32_inreg@rel32@lo+4
 ; CHECK-NEXT:    s_addc_u32 s19, s19, void_func_i32_inreg@rel32@hi+12
+; CHECK-NEXT:    s_mov_b32 s0, s16
 ; CHECK-NEXT:    s_setpc_b64 s[18:19]
   tail call void @void_func_i32_inreg(i32 inreg %sgpr)
   ret void
diff --git a/llvm/test/CodeGen/AMDGPU/truncate-lshr-cast-build-vector-combine.ll b/llvm/test/CodeGen/AMDGPU/truncate-lshr-cast-build-vector-combine.ll
new file mode 100644
index 0000000000000..1c3091f6b8d3b
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/truncate-lshr-cast-build-vector-combine.ll
@@ -0,0 +1,140 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 < %s | FileCheck %s
+
+; extract element 0 as shift
+define i32 @cast_v4i32_to_i128_trunc_i32(<4 x i32> %arg) {
+; CHECK-LABEL: cast_v4i32_to_i128_trunc_i32:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    s_setpc_b64 s[30:31]
+  %bigint = bitcast <4 x i32> %arg to i128
+  %trunc = trunc i128 %bigint to i32
+  ret i32 %trunc
+}
+
+; extract element 1 as shift
+define i32 @cast_v4i32_to_i128_lshr_32_trunc_i32(<4 x i32> %arg) {
+; CHECK-LABEL: cast_v4i32_to_i128_lshr_32_trunc_i32:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    v_mov_b32_e32 v0, v1
+; CHECK-NEXT:    s_setpc_b64 s[30:31]
+  %bigint = bitcast <4 x i32> %arg to i128
+  %srl = lshr i128 %bigint, 32
+  %trunc = trunc i128 %srl to i32
+  ret i32 %trunc
+}
+
+; extract element 2 as shift
+define i32 @cast_v4i32_to_i128_lshr_64_trunc_i32(<4 x i32> %arg) {
+; CHECK-LABEL: cast_v4i32_to_i128_lshr_64_trunc_i32:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    v_mov_b32_e32 v0, v2
+; CHECK-NEXT:    s_setpc_b64 s[30:31]
+  %bigint = bitcast <4 x i32> %arg to i128
+  %srl = lshr i128 %bigint, 64
+  %trunc = trunc i128 %srl to i32
+  ret i32 %trunc
+}
+
+; extract element 3 as shift
+define i32 @cast_v4i32_to_i128_lshr_96_trunc_i32(<4 x i32> %arg) {
+; CHECK-LABEL: cast_v4i32_to_i128_lshr_96_trunc_i32:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    v_mov_b32_e32 v0, v3
+; CHECK-NEXT:    s_setpc_b64 s[30:31]
+  %bigint = bitcast <4 x i32> %arg to i128
+  %srl = lshr i128 %bigint, 96
+  %trunc = trunc i128 %srl to i32
+  ret i32 %trunc
+}
+
+; Shift not aligned to element, not a simple extract
+define i32 @cast_v4i32_to_i128_lshr_33_trunc_i32(<4 x i32> %arg) {
+; CHECK-LABEL: cast_v4i32_to_i128_lshr_33_trunc_i32:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    v_alignbit_b32 v0, v2, v1, 1
+; CHECK-NEXT:    s_setpc_b64 s[30:31]
+  %bigint = bitcast <4 x i32> %arg to i128
+  %srl = lshr i128 %bigint, 33
+  %trunc = trunc i128 %srl to i32
+  ret i32 %trunc
+}
+
+; extract misaligned element
+define i32 @cast_v4i32_to_i128_lshr_31_trunc_i32(<4 x i32> %arg) {
+; CHECK-LABEL: cast_v4i32_to_i128_lshr_31_trunc_i32:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    v_alignbit_b32 v0, v1, v0, 31
+; CHECK-NEXT:    s_setpc_b64 s[30:31]
+  %bigint = bitcast <4 x i32> %arg to i128
+  %srl = lshr i128 %bigint, 31
+  %trunc = trunc i128 %srl to i32
+  ret i32 %trunc
+}
+
+; extract misaligned element
+define i32 @cast_v4i32_to_i128_lshr_48_trunc_i32(<4 x i32> %arg) {
+; CHECK-LABEL: cast_v4i32_to_i128_lshr_48_trunc_i32:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    s_mov_b32 s4, 0x1000706
+; CHECK-NEXT:    v_perm_b32 v0, v1, v2, s4
+; CHECK-NEXT:    s_setpc_b64 s[30:31]
+  %bigint = bitcast <4 x i32> %arg to i128
+  %srl = lshr i128 %bigint, 48
+  %trunc = trunc i128 %srl to i32
+  ret i32 %trunc
+}
+
+; extract elements 1 and 2 with shift
+define i64 @cast_v4i32_to_i128_lshr_32_trunc_i64(<4 x i32> %arg) {
+; CHECK-LABEL: cast_v4i32_to_i128_lshr_32_trunc_i64:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    v_mov_b32_e32 v0, v1
+; CHECK-NEXT:    v_mov_b32_e32 v1, v2
+; CHECK-NEXT:    s_setpc_b64 s[30:31]
+  %bigint = bitcast <4 x i32> %arg to i128
+  %srl = lshr i128 %bigint, 32
+  %trunc = trunc i128 %srl to i64
+  ret i64 %trunc
+}
+
+; extract elements 2 and 3 with shift
+define i64 @cast_v4i32_to_i128_lshr_64_trunc_i64(<4 x i32> %arg) {
+; CHECK-LABEL: cast_v4i32_to_i128_lshr_64_trunc_i64:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    v_mov_b32_e32 v1, v3
+; CHECK-NEXT:    v_mov_b32_e32 v0, v2
+; CHECK-NEXT:    s_setpc_b64 s[30:31]
+  %bigint = bitcast <4 x i32> %arg to i128
+  %srl = lshr i128 %bigint, 64
+  %trunc = trunc i128 %srl to i64
+  ret i64 %trunc
+}
+
+; FIXME: We don't process this case because we see multiple bitcasts
+; before a 32-bit build_vector
+define i32 @build_vector_i16_to_shift(i16 %arg0, i16 %arg1, i16 %arg2, i16 %arg3) {
+; CHECK-LABEL: build_vector_i16_to_shift:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    s_mov_b32 s4, 0x5040100
+; CHECK-NEXT:    v_perm_b32 v0, v3, v2, s4
+; CHECK-NEXT:    s_setpc_b64 s[30:31]
+  %ins.0 = insertelement <4 x i16> poison, i16 %arg0, i32 0
+  %ins.1 = insertelement <4 x i16> %ins.0, i16 %arg1, i32 1
+  %ins.2 = insertelement <4 x i16> %ins.1, i16 %arg2, i32 2
+  %ins.3 = insertelement <4 x i16> %ins.2, i16 %arg3, i32 3
+
+  %cast = bitcast <4 x i16> %ins.3 to i64
+  %srl = lshr i64 %cast, 32
+  %trunc = trunc i64 %srl to i32
+  ret i32 %trunc
+}
diff --git a/llvm/test/CodeGen/AMDGPU/tuple-allocation-failure.ll b/llvm/test/CodeGen/AMDGPU/tuple-allocation-failure.ll
index 12eec4fa3bd59..dd78c2f46dde8 100644
--- a/llvm/test/CodeGen/AMDGPU/tuple-allocation-failure.ll
+++ b/llvm/test/CodeGen/AMDGPU/tuple-allocation-failure.ll
@@ -290,6 +290,9 @@ define amdgpu_kernel void @kernel(ptr addrspace(1) %arg1.global, i1 %tmp3.i.i, i
 ; GLOBALNESS1-NEXT:  ; %bb.31: ; %bb7.i.i
 ; GLOBALNESS1-NEXT:    s_add_u32 s8, s38, 40
 ; GLOBALNESS1-NEXT:    s_addc_u32 s9, s39, 0
+; GLOBALNESS1-NEXT:    s_getpc_b64 s[16:17]
+; GLOBALNESS1-NEXT:    s_add_u32 s16, s16, widget@rel32@lo+4
+; GLOBALNESS1-NEXT:    s_addc_u32 s17, s17, widget@rel32@hi+12
 ; GLOBALNESS1-NEXT:    s_mov_b64 s[4:5], s[40:41]
 ; GLOBALNESS1-NEXT:    s_mov_b64 s[6:7], s[36:37]
 ; GLOBALNESS1-NEXT:    s_mov_b64 s[10:11], s[34:35]
@@ -297,9 +300,6 @@ define amdgpu_kernel void @kernel(ptr addrspace(1) %arg1.global, i1 %tmp3.i.i, i
 ; GLOBALNESS1-NEXT:    s_mov_b32 s13, s71
 ; GLOBALNESS1-NEXT:    s_mov_b32 s14, s70
 ; GLOBALNESS1-NEXT:    v_mov_b32_e32 v31, v41
-; GLOBALNESS1-NEXT:    s_getpc_b64 s[16:17]
-; GLOBALNESS1-NEXT:    s_add_u32 s16, s16, widget@rel32@lo+4
-; GLOBALNESS1-NEXT:    s_addc_u32 s17, s17, widget@rel32@hi+12
 ; GLOBALNESS1-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GLOBALNESS1-NEXT:    s_mov_b64 s[4:5], 0
 ; GLOBALNESS1-NEXT:  .LBB1_32: ; %Flow
@@ -308,6 +308,9 @@ define amdgpu_kernel void @kernel(ptr addrspace(1) %arg1.global, i1 %tmp3.i.i, i
 ; GLOBALNESS1-NEXT:  ; %bb.33: ; %bb11.i.i
 ; GLOBALNESS1-NEXT:    s_add_u32 s8, s38, 40
 ; GLOBALNESS1-NEXT:    s_addc_u32 s9, s39, 0
+; GLOBALNESS1-NEXT:    s_getpc_b64 s[16:17]
+; GLOBALNESS1-NEXT:    s_add_u32 s16, s16, widget@rel32@lo+4
+; GLOBALNESS1-NEXT:    s_addc_u32 s17, s17, widget@rel32@hi+12
 ; GLOBALNESS1-NEXT:    s_mov_b64 s[4:5], s[40:41]
 ; GLOBALNESS1-NEXT:    s_mov_b64 s[6:7], s[36:37]
 ; GLOBALNESS1-NEXT:    s_mov_b64 s[10:11], s[34:35]
@@ -315,9 +318,6 @@ define amdgpu_kernel void @kernel(ptr addrspace(1) %arg1.global, i1 %tmp3.i.i, i
 ; GLOBALNESS1-NEXT:    s_mov_b32 s13, s71
 ; GLOBALNESS1-NEXT:    s_mov_b32 s14, s70
 ; GLOBALNESS1-NEXT:    v_mov_b32_e32 v31, v41
-; GLOBALNESS1-NEXT:    s_getpc_b64 s[16:17]
-; GLOBALNESS1-NEXT:    s_add_u32 s16, s16, widget@rel32@lo+4
-; GLOBALNESS1-NEXT:    s_addc_u32 s17, s17, widget@rel32@hi+12
 ; GLOBALNESS1-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GLOBALNESS1-NEXT:  .LBB1_34: ; %UnifiedUnreachableBlock
 ;
@@ -582,6 +582,9 @@ define amdgpu_kernel void @kernel(ptr addrspace(1) %arg1.global, i1 %tmp3.i.i, i
 ; GLOBALNESS0-NEXT:  ; %bb.31: ; %bb7.i.i
 ; GLOBALNESS0-NEXT:    s_add_u32 s8, s38, 40
 ; GLOBALNESS0-NEXT:    s_addc_u32 s9, s39, 0
+; GLOBALNESS0-NEXT:    s_getpc_b64 s[16:17]
+; GLOBALNESS0-NEXT:    s_add_u32 s16, s16, widget@rel32@lo+4
+; GLOBALNESS0-NEXT:    s_addc_u32 s17, s17, widget@rel32@hi+12
 ; GLOBALNESS0-NEXT:    s_mov_b64 s[4:5], s[40:41]
 ; GLOBALNESS0-NEXT:    s_mov_b64 s[6:7], s[36:37]
 ; GLOBALNESS0-NEXT:    s_mov_b64 s[10:11], s[34:35]
@@ -589,9 +592,6 @@ define amdgpu_kernel void @kernel(ptr addrspace(1) %arg1.global, i1 %tmp3.i.i, i
 ; GLOBALNESS0-NEXT:    s_mov_b32 s13, s69
 ; GLOBALNESS0-NEXT:    s_mov_b32 s14, s68
 ; GLOBALNESS0-NEXT:    v_mov_b32_e32 v31, v41
-; GLOBALNESS0-NEXT:    s_getpc_b64 s[16:17]
-; GLOBALNESS0-NEXT:    s_add_u32 s16, s16, widget@rel32@lo+4
-; GLOBALNESS0-NEXT:    s_addc_u32 s17, s17, widget@rel32@hi+12
 ; GLOBALNESS0-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GLOBALNESS0-NEXT:    s_mov_b64 s[4:5], 0
 ; GLOBALNESS0-NEXT:  .LBB1_32: ; %Flow
@@ -600,6 +600,9 @@ define amdgpu_kernel void @kernel(ptr addrspace(1) %arg1.global, i1 %tmp3.i.i, i
 ; GLOBALNESS0-NEXT:  ; %bb.33: ; %bb11.i.i
 ; GLOBALNESS0-NEXT:    s_add_u32 s8, s38, 40
 ; GLOBALNESS0-NEXT:    s_addc_u32 s9, s39, 0
+; GLOBALNESS0-NEXT:    s_getpc_b64 s[16:17]
+; GLOBALNESS0-NEXT:    s_add_u32 s16, s16, widget@rel32@lo+4
+; GLOBALNESS0-NEXT:    s_addc_u32 s17, s17, widget@rel32@hi+12
 ; GLOBALNESS0-NEXT:    s_mov_b64 s[4:5], s[40:41]
 ; GLOBALNESS0-NEXT:    s_mov_b64 s[6:7], s[36:37]
 ; GLOBALNESS0-NEXT:    s_mov_b64 s[10:11], s[34:35]
@@ -607,9 +610,6 @@ define amdgpu_kernel void @kernel(ptr addrspace(1) %arg1.global, i1 %tmp3.i.i, i
 ; GLOBALNESS0-NEXT:    s_mov_b32 s13, s69
 ; GLOBALNESS0-NEXT:    s_mov_b32 s14, s68
 ; GLOBALNESS0-NEXT:    v_mov_b32_e32 v31, v41
-; GLOBALNESS0-NEXT:    s_getpc_b64 s[16:17]
-; GLOBALNESS0-NEXT:    s_add_u32 s16, s16, widget@rel32@lo+4
-; GLOBALNESS0-NEXT:    s_addc_u32 s17, s17, widget@rel32@hi+12
 ; GLOBALNESS0-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GLOBALNESS0-NEXT:  .LBB1_34: ; %UnifiedUnreachableBlock
 bb:
diff --git a/llvm/test/CodeGen/AMDGPU/udiv.ll b/llvm/test/CodeGen/AMDGPU/udiv.ll
index 530226baa775e..0d682a6627a1a 100644
--- a/llvm/test/CodeGen/AMDGPU/udiv.ll
+++ b/llvm/test/CodeGen/AMDGPU/udiv.ll
@@ -2509,9 +2509,7 @@ define i64 @v_test_udiv64_mulhi_fold(i64 %arg) {
 ; VI-NEXT:    v_mad_u64_u32 v[0:1], s[4:5], v5, s4, v[2:3]
 ; VI-NEXT:    v_mov_b32_e32 v2, v0
 ; VI-NEXT:    v_mad_u64_u32 v[2:3], s[4:5], v4, s6, v[2:3]
-; VI-NEXT:    v_mov_b32_e32 v0, v1
-; VI-NEXT:    v_mov_b32_e32 v1, v3
-; VI-NEXT:    v_add_u32_e32 v0, vcc, v0, v1
+; VI-NEXT:    v_add_u32_e32 v0, vcc, v1, v3
 ; VI-NEXT:    v_addc_u32_e64 v1, s[4:5], 0, 0, vcc
 ; VI-NEXT:    v_mad_u64_u32 v[0:1], s[4:5], v5, s6, v[0:1]
 ; VI-NEXT:    v_alignbit_b32 v0, v1, v0, 7
@@ -2530,9 +2528,7 @@ define i64 @v_test_udiv64_mulhi_fold(i64 %arg) {
 ; GCN-NEXT:    v_mad_u64_u32 v[0:1], s[4:5], v5, s4, v[2:3]
 ; GCN-NEXT:    v_mov_b32_e32 v2, v0
 ; GCN-NEXT:    v_mad_u64_u32 v[2:3], s[4:5], v4, s6, v[2:3]
-; GCN-NEXT:    v_mov_b32_e32 v0, v1
-; GCN-NEXT:    v_mov_b32_e32 v1, v3
-; GCN-NEXT:    v_add_u32_e32 v0, vcc, v0, v1
+; GCN-NEXT:    v_add_u32_e32 v0, vcc, v1, v3
 ; GCN-NEXT:    v_addc_u32_e64 v1, s[4:5], 0, 0, vcc
 ; GCN-NEXT:    v_mad_u64_u32 v[0:1], s[4:5], v5, s6, v[0:1]
 ; GCN-NEXT:    v_alignbit_b32 v0, v1, v0, 7
@@ -2548,10 +2544,8 @@ define i64 @v_test_udiv64_mulhi_fold(i64 %arg) {
 ; GFX1030-NEXT:    v_mul_hi_u32 v2, 0x71b47843, v4
 ; GFX1030-NEXT:    v_mad_u64_u32 v[0:1], null, 0x71b47843, v5, v[2:3]
 ; GFX1030-NEXT:    v_mov_b32_e32 v2, v0
-; GFX1030-NEXT:    v_mov_b32_e32 v0, v1
 ; GFX1030-NEXT:    v_mad_u64_u32 v[2:3], null, 0xa7c5ac4, v4, v[2:3]
-; GFX1030-NEXT:    v_mov_b32_e32 v1, v3
-; GFX1030-NEXT:    v_add_co_u32 v0, s4, v0, v1
+; GFX1030-NEXT:    v_add_co_u32 v0, s4, v1, v3
 ; GFX1030-NEXT:    v_add_co_ci_u32_e64 v1, null, 0, 0, s4
 ; GFX1030-NEXT:    v_mad_u64_u32 v[0:1], null, 0xa7c5ac4, v5, v[0:1]
 ; GFX1030-NEXT:    v_alignbit_b32 v0, v1, v0, 7
diff --git a/llvm/test/CodeGen/AMDGPU/v_cndmask.ll b/llvm/test/CodeGen/AMDGPU/v_cndmask.ll
index c6cc479b5deb1..5360ff2fa402f 100644
--- a/llvm/test/CodeGen/AMDGPU/v_cndmask.ll
+++ b/llvm/test/CodeGen/AMDGPU/v_cndmask.ll
@@ -2411,7 +2411,6 @@ define amdgpu_kernel void @v_cndmask_abs_neg_f64(ptr addrspace(1) %out, i32 %c,
 ; SI-NEXT:    v_xor_b32_e32 v1, 0x80000000, v1
 ; SI-NEXT:    s_cselect_b64 vcc, -1, 0
 ; SI-NEXT:    v_cndmask_b32_e32 v1, v1, v2, vcc
-; SI-NEXT:    v_cndmask_b32_e32 v0, v0, v0, vcc
 ; SI-NEXT:    s_waitcnt lgkmcnt(0)
 ; SI-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
 ; SI-NEXT:    s_endpgm
@@ -2435,7 +2434,6 @@ define amdgpu_kernel void @v_cndmask_abs_neg_f64(ptr addrspace(1) %out, i32 %c,
 ; VI-NEXT:    v_xor_b32_e32 v1, 0x80000000, v1
 ; VI-NEXT:    v_cndmask_b32_e32 v1, v1, v2, vcc
 ; VI-NEXT:    v_mov_b32_e32 v3, s1
-; VI-NEXT:    v_cndmask_b32_e32 v0, v0, v0, vcc
 ; VI-NEXT:    v_mov_b32_e32 v2, s0
 ; VI-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
 ; VI-NEXT:    s_endpgm
@@ -2457,7 +2455,6 @@ define amdgpu_kernel void @v_cndmask_abs_neg_f64(ptr addrspace(1) %out, i32 %c,
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    v_and_b32_e32 v2, 0x7fffffff, v1
 ; GFX10-NEXT:    v_xor_b32_e32 v1, 0x80000000, v1
-; GFX10-NEXT:    v_cndmask_b32_e32 v0, v0, v0, vcc
 ; GFX10-NEXT:    v_cndmask_b32_e32 v1, v1, v2, vcc
 ; GFX10-NEXT:    global_store_dwordx2 v3, v[0:1], s[0:1]
 ; GFX10-NEXT:    s_endpgm
@@ -2480,8 +2477,7 @@ define amdgpu_kernel void @v_cndmask_abs_neg_f64(ptr addrspace(1) %out, i32 %c,
 ; GFX11-NEXT:    s_waitcnt vmcnt(0)
 ; GFX11-NEXT:    v_and_b32_e32 v2, 0x7fffffff, v1
 ; GFX11-NEXT:    v_xor_b32_e32 v1, 0x80000000, v1
-; GFX11-NEXT:    v_cndmask_b32_e32 v0, v0, v0, vcc
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2)
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX11-NEXT:    v_cndmask_b32_e32 v1, v1, v2, vcc
 ; GFX11-NEXT:    global_store_b64 v3, v[0:1], s[0:1]
 ; GFX11-NEXT:    s_endpgm
@@ -2502,8 +2498,7 @@ define amdgpu_kernel void @v_cndmask_abs_neg_f64(ptr addrspace(1) %out, i32 %c,
 ; GFX12-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-NEXT:    v_and_b32_e32 v2, 0x7fffffff, v1
 ; GFX12-NEXT:    v_xor_b32_e32 v1, 0x80000000, v1
-; GFX12-NEXT:    v_cndmask_b32_e32 v0, v0, v0, vcc
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2)
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-NEXT:    v_cndmask_b32_e32 v1, v1, v2, vcc
 ; GFX12-NEXT:    global_store_b64 v3, v[0:1], s[0:1]
 ; GFX12-NEXT:    s_endpgm
diff --git a/llvm/test/CodeGen/AMDGPU/v_swap_b16.ll b/llvm/test/CodeGen/AMDGPU/v_swap_b16.ll
index bad0be16e75cc..79ec4b8831679 100644
--- a/llvm/test/CodeGen/AMDGPU/v_swap_b16.ll
+++ b/llvm/test/CodeGen/AMDGPU/v_swap_b16.ll
@@ -69,7 +69,6 @@ define half @swap(half %a, half %b, i32 %i) {
 ; GFX12-TRUE16-NEXT:    s_cbranch_execnz .LBB0_1
 ; GFX12-TRUE16-NEXT:  ; %bb.2: ; %ret
 ; GFX12-TRUE16-NEXT:    s_or_b32 exec_lo, exec_lo, s0
-; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffe
 ; GFX12-TRUE16-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX12-FAKE16-LABEL: swap:
@@ -94,7 +93,6 @@ define half @swap(half %a, half %b, i32 %i) {
 ; GFX12-FAKE16-NEXT:  ; %bb.2: ; %ret
 ; GFX12-FAKE16-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX12-FAKE16-NEXT:    v_mov_b32_e32 v0, v1
-; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffe
 ; GFX12-FAKE16-NEXT:    s_setpc_b64 s[30:31]
 entry:
   br label %loop
diff --git a/llvm/test/CodeGen/AMDGPU/valu-mask-write-hazard.mir b/llvm/test/CodeGen/AMDGPU/valu-mask-write-hazard.mir
index d91ee54215924..1eabe62e7710e 100644
--- a/llvm/test/CodeGen/AMDGPU/valu-mask-write-hazard.mir
+++ b/llvm/test/CodeGen/AMDGPU/valu-mask-write-hazard.mir
@@ -1,6 +1,6 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
 # RUN: llc -mtriple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize64 -verify-machineinstrs -run-pass post-RA-hazard-rec -o - %s | FileCheck -check-prefixes=GCN,GFX11 %s
-# RUN: llc -mtriple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize64 -verify-machineinstrs -run-pass post-RA-hazard-rec -o - %s | FileCheck -check-prefixes=GCN,GFX12 %s
+# RUN: llc -mtriple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize64 -verify-machineinstrs -run-pass post-RA-hazard-rec,amdgpu-wait-sgpr-hazards -o - %s | FileCheck -check-prefixes=GCN,GFX12 %s
 
 --- |
   @mem = internal unnamed_addr addrspace(4) constant [4 x <4 x i32>] [<4 x i32> <i32 0, i32 0, i32 0, i32 0>, <4 x i32> <i32 0, i32 0, i32 0, i32 0>, <4 x i32> <i32 0, i32 0, i32 0, i32 0>, <4 x i32> <i32 0, i32 0, i32 0, i32 0>]
diff --git a/llvm/test/CodeGen/AMDGPU/valu-read-sgpr-hazard-attrs.mir b/llvm/test/CodeGen/AMDGPU/valu-read-sgpr-hazard-attrs.mir
new file mode 100644
index 0000000000000..fe3cb30793390
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/valu-read-sgpr-hazard-attrs.mir
@@ -0,0 +1,347 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -mtriple=amdgcn -mcpu=gfx1200 -run-pass post-RA-hazard-rec,amdgpu-wait-sgpr-hazards -o - %s | FileCheck -check-prefix=GCN %s
+
+--- |
+  define amdgpu_gs void @hazard_disable() #0 { ret void }
+  define amdgpu_gs void @hazard_enable() #1 { ret void }
+  define amdgpu_cs void @hazard_calls() #2 { ret void }
+  define void @hazard_callee1() #2 { ret void }
+  define void @hazard_callee2() #2 { ret void }
+  define amdgpu_cs void @hazard_cull_vmem() #3 { ret void }
+  define amdgpu_cs void @hazard_cull_vmem2() #4 { ret void }
+  define amdgpu_cs void @hazard_cull_sample() #3 { ret void }
+  define amdgpu_cs void @hazard_cull_bvh() #3 { ret void }
+  define amdgpu_cs void @hazard_nocull_scratch() #3 { ret void }
+  define amdgpu_cs void @hazard_cull_global() #3 { ret void }
+  define amdgpu_cs void @hazard_nocull_flat() #3 { ret void }
+
+  attributes #0 = { "amdgpu-sgpr-hazard-wait"="0" }
+  attributes #1 = { "amdgpu-sgpr-hazard-wait"="1" }
+  attributes #2 = { "amdgpu-sgpr-hazard-boundary-cull" }
+  attributes #3 = { "amdgpu-sgpr-hazard-mem-wait-cull" "amdgpu-sgpr-hazard-mem-wait-cull-threshold"="1" }
+  attributes #4 = { "amdgpu-sgpr-hazard-mem-wait-cull" "amdgpu-sgpr-hazard-mem-wait-cull-threshold"="2" }
+...
+
+---
+name:            hazard_disable
+body:            |
+  bb.0:
+    ; GCN-LABEL: name: hazard_disable
+    ; GCN: $vgpr1 = V_CNDMASK_B32_e64 0, $vgpr1, 0, $vgpr2, $sgpr0, implicit $exec
+    ; GCN-NEXT: $sgpr0_sgpr1 = S_GETPC_B64
+    ; GCN-NEXT: $sgpr3 = S_ADD_U32 $sgpr0, 0, implicit-def $scc
+    ; GCN-NEXT: S_ENDPGM 0
+    $vgpr1 = V_CNDMASK_B32_e64 0, $vgpr1, 0, $vgpr2, $sgpr0, implicit $exec
+    $sgpr0_sgpr1 = S_GETPC_B64
+    $sgpr3 = S_ADD_U32 $sgpr0, 0, implicit-def $scc
+    S_ENDPGM 0
+...
+
+---
+name:            hazard_enable
+body:            |
+  bb.0:
+    ; GCN-LABEL: name: hazard_enable
+    ; GCN: $vgpr1 = V_CNDMASK_B32_e64 0, $vgpr1, 0, $vgpr2, $sgpr0, implicit $exec
+    ; GCN-NEXT: $sgpr0_sgpr1 = S_GETPC_B64
+    ; GCN-NEXT: S_WAITCNT_DEPCTR 65534
+    ; GCN-NEXT: $sgpr3 = S_ADD_U32 $sgpr0, 0, implicit-def $scc
+    ; GCN-NEXT: S_ENDPGM 0
+    $vgpr1 = V_CNDMASK_B32_e64 0, $vgpr1, 0, $vgpr2, $sgpr0, implicit $exec
+    $sgpr0_sgpr1 = S_GETPC_B64
+    $sgpr3 = S_ADD_U32 $sgpr0, 0, implicit-def $scc
+    S_ENDPGM 0
+...
+
+---
+name:            hazard_calls
+frameInfo:
+  hasCalls: true
+body:            |
+  ; GCN-LABEL: name: hazard_calls
+  ; GCN: bb.0:
+  ; GCN-NEXT:   successors: %bb.2(0x40000000), %bb.1(0x40000000)
+  ; GCN-NEXT: {{  $}}
+  ; GCN-NEXT:   $vgpr0 = V_WRITELANE_B32 0, $sgpr4, $vgpr0
+  ; GCN-NEXT:   $vgpr0 = V_WRITELANE_B32 1, $sgpr8, $vgpr0
+  ; GCN-NEXT:   $vgpr0 = V_WRITELANE_B32 2, $sgpr16, $vgpr0
+  ; GCN-NEXT:   $vgpr0 = V_WRITELANE_B32 3, $sgpr18, $vgpr0
+  ; GCN-NEXT:   $vgpr0 = V_WRITELANE_B32 4, $sgpr20, $vgpr0
+  ; GCN-NEXT:   $vgpr0 = V_WRITELANE_B32 5, $sgpr22, $vgpr0
+  ; GCN-NEXT:   S_CBRANCH_SCC0 %bb.2, implicit $scc
+  ; GCN-NEXT:   S_BRANCH %bb.1
+  ; GCN-NEXT: {{  $}}
+  ; GCN-NEXT: bb.1:
+  ; GCN-NEXT:   successors: %bb.3(0x40000000), %bb.4(0x40000000)
+  ; GCN-NEXT: {{  $}}
+  ; GCN-NEXT:   S_CBRANCH_SCC0 %bb.3, implicit $scc
+  ; GCN-NEXT:   S_BRANCH %bb.4
+  ; GCN-NEXT: {{  $}}
+  ; GCN-NEXT: bb.2:
+  ; GCN-NEXT:   $sgpr16 = S_MOV_B32 0
+  ; GCN-NEXT:   DS_NOP implicit $m0, implicit $exec
+  ; GCN-NEXT:   DS_NOP implicit $m0, implicit $exec
+  ; GCN-NEXT:   DS_NOP implicit $m0, implicit $exec
+  ; GCN-NEXT:   DS_NOP implicit $m0, implicit $exec
+  ; GCN-NEXT:   S_WAITCNT_DEPCTR 65534
+  ; GCN-NEXT:   S_SETPC_B64 $sgpr0_sgpr1
+  ; GCN-NEXT: {{  $}}
+  ; GCN-NEXT: bb.3:
+  ; GCN-NEXT:   $sgpr18 = S_MOV_B32 0
+  ; GCN-NEXT:   DS_NOP implicit $m0, implicit $exec
+  ; GCN-NEXT:   DS_NOP implicit $m0, implicit $exec
+  ; GCN-NEXT:   DS_NOP implicit $m0, implicit $exec
+  ; GCN-NEXT:   DS_NOP implicit $m0, implicit $exec
+  ; GCN-NEXT:   S_WAITCNT_DEPCTR 65534
+  ; GCN-NEXT:   S_SETPC_B64_return $sgpr0_sgpr1
+  ; GCN-NEXT: {{  $}}
+  ; GCN-NEXT: bb.4:
+  ; GCN-NEXT:   successors: %bb.5(0x80000000)
+  ; GCN-NEXT: {{  $}}
+  ; GCN-NEXT:   $vcc_lo = S_MOV_B32 0
+  ; GCN-NEXT:   $sgpr20 = S_MOV_B32 0
+  ; GCN-NEXT:   DS_NOP implicit $m0, implicit $exec
+  ; GCN-NEXT:   DS_NOP implicit $m0, implicit $exec
+  ; GCN-NEXT:   DS_NOP implicit $m0, implicit $exec
+  ; GCN-NEXT:   DS_NOP implicit $m0, implicit $exec
+  ; GCN-NEXT:   S_WAITCNT_DEPCTR 65534
+  ; GCN-NEXT:   $sgpr4_sgpr5 = S_SWAPPC_B64 $sgpr2_sgpr3
+  ; GCN-NEXT:   $sgpr4 = S_ADD_U32 $sgpr4, 0, implicit-def $scc
+  ; GCN-NEXT: {{  $}}
+  ; GCN-NEXT: bb.5:
+  ; GCN-NEXT:   successors: %bb.6(0x80000000)
+  ; GCN-NEXT: {{  $}}
+  ; GCN-NEXT:   $sgpr8_sgpr9 = S_CALL_B64 0
+  ; GCN-NEXT: {{  $}}
+  ; GCN-NEXT: bb.6:
+  ; GCN-NEXT:   $sgpr22 = S_MOV_B32 $sgpr8
+  ; GCN-NEXT:   S_ENDPGM 0
+  bb.0:
+    $vgpr0 = V_WRITELANE_B32 0, $sgpr4, $vgpr0
+    $vgpr0 = V_WRITELANE_B32 1, $sgpr8, $vgpr0
+    $vgpr0 = V_WRITELANE_B32 2, $sgpr16, $vgpr0
+    $vgpr0 = V_WRITELANE_B32 3, $sgpr18, $vgpr0
+    $vgpr0 = V_WRITELANE_B32 4, $sgpr20, $vgpr0
+    $vgpr0 = V_WRITELANE_B32 5, $sgpr22, $vgpr0
+    S_CBRANCH_SCC0 %bb.2, implicit $scc
+    S_BRANCH %bb.1
+
+  bb.1:
+    S_CBRANCH_SCC0 %bb.3, implicit $scc
+    S_BRANCH %bb.4
+
+  bb.2:
+    $sgpr16 = S_MOV_B32 0
+    S_SETPC_B64 $sgpr0_sgpr1
+
+  bb.3:
+    $sgpr18 = S_MOV_B32 0
+    S_SETPC_B64_return $sgpr0_sgpr1
+
+  bb.4:
+    $vcc_lo = S_MOV_B32 0
+    $sgpr20 = S_MOV_B32 0
+    $sgpr4_sgpr5 = S_SWAPPC_B64 $sgpr2_sgpr3
+    $sgpr4 = S_ADD_U32 $sgpr4, 0, implicit-def $scc
+
+  bb.5:
+    $sgpr8_sgpr9 = S_CALL_B64 0
+
+  bb.6:
+    $sgpr22 = S_MOV_B32 $sgpr8
+    S_ENDPGM 0
+...
+
+---
+name:            hazard_callee1
+body:            |
+  bb.0:
+    ; GCN-LABEL: name: hazard_callee1
+    ; GCN: $sgpr1 = S_CSELECT_B32 -1, 0, implicit $scc
+    ; GCN-NEXT: $sgpr2 = S_ADD_U32 $sgpr1, 0, implicit-def $scc
+    ; GCN-NEXT: S_SETPC_B64_return $sgpr30_sgpr31
+    $sgpr1 = S_CSELECT_B32 -1, 0, implicit $scc
+    $sgpr2 = S_ADD_U32 $sgpr1, 0, implicit-def $scc
+    S_SETPC_B64_return $sgpr30_sgpr31
+...
+
+---
+name:            hazard_callee2
+body:            |
+  bb.0:
+    ; GCN-LABEL: name: hazard_callee2
+    ; GCN: $vgpr1, $sgpr0 = V_ADDC_U32_e64 0, $vgpr1, $sgpr0, 0, implicit $exec
+    ; GCN-NEXT: $sgpr1 = S_CSELECT_B32 -1, 0, implicit $scc
+    ; GCN-NEXT: S_WAITCNT_DEPCTR 65534
+    ; GCN-NEXT: $sgpr2 = S_ADD_U32 $sgpr1, 0, implicit-def $scc
+    ; GCN-NEXT: DS_NOP implicit $m0, implicit $exec
+    ; GCN-NEXT: DS_NOP implicit $m0, implicit $exec
+    ; GCN-NEXT: DS_NOP implicit $m0, implicit $exec
+    ; GCN-NEXT: DS_NOP implicit $m0, implicit $exec
+    ; GCN-NEXT: S_SETPC_B64_return $sgpr30_sgpr31
+    $vgpr1, $sgpr0 = V_ADDC_U32_e64 0, $vgpr1, $sgpr0, 0, implicit $exec
+    $sgpr1 = S_CSELECT_B32 -1, 0, implicit $scc
+    $sgpr2 = S_ADD_U32 $sgpr1, 0, implicit-def $scc
+    S_SETPC_B64_return $sgpr30_sgpr31
+...
+
+---
+name:            hazard_cull_vmem
+body:            |
+  bb.0:
+    ; GCN-LABEL: name: hazard_cull_vmem
+    ; GCN: $vgpr1, $sgpr0 = V_ADDC_U32_e64 0, $vgpr1, $sgpr0, 0, implicit $exec
+    ; GCN-NEXT: $vgpr3 = BUFFER_LOAD_DWORD_OFFSET $sgpr4_sgpr5_sgpr6_sgpr7, $sgpr8, 0, 0, 0, implicit $exec
+    ; GCN-NEXT: DS_NOP implicit $m0, implicit $exec
+    ; GCN-NEXT: DS_NOP implicit $m0, implicit $exec
+    ; GCN-NEXT: DS_NOP implicit $m0, implicit $exec
+    ; GCN-NEXT: DS_NOP implicit $m0, implicit $exec
+    ; GCN-NEXT: S_WAIT_LOADCNT 0
+    ; GCN-NEXT: $sgpr0_sgpr1 = S_GETPC_B64
+    ; GCN-NEXT: $sgpr3 = S_ADD_U32 $sgpr0, 0, implicit-def $scc
+    ; GCN-NEXT: S_ENDPGM 0
+    $vgpr1, $sgpr0 = V_ADDC_U32_e64 0, $vgpr1, $sgpr0, 0, implicit $exec
+    $vgpr3 = BUFFER_LOAD_DWORD_OFFSET $sgpr4_sgpr5_sgpr6_sgpr7, $sgpr8, 0, 0, 0, implicit $exec
+    S_WAIT_LOADCNT 0
+    $sgpr0_sgpr1 = S_GETPC_B64
+    $sgpr3 = S_ADD_U32 $sgpr0, 0, implicit-def $scc
+    S_ENDPGM 0
+...
+
+---
+name:            hazard_cull_vmem2
+body:            |
+  bb.0:
+    ; GCN-LABEL: name: hazard_cull_vmem2
+    ; GCN: $vgpr1, $sgpr0 = V_ADDC_U32_e64 0, $vgpr1, $sgpr0, 0, implicit $exec
+    ; GCN-NEXT: $vgpr3 = BUFFER_LOAD_DWORD_OFFSET $sgpr4_sgpr5_sgpr6_sgpr7, $sgpr8, 0, 0, 0, implicit $exec
+    ; GCN-NEXT: S_WAIT_LOADCNT 0
+    ; GCN-NEXT: $vgpr1, $sgpr0 = V_ADDC_U32_e64 0, $vgpr1, $sgpr2, 0, implicit $exec
+    ; GCN-NEXT: $vgpr4 = BUFFER_LOAD_DWORD_OFFSET $sgpr4_sgpr5_sgpr6_sgpr7, $sgpr8, 0, 0, 0, implicit $exec
+    ; GCN-NEXT: DS_NOP implicit $m0, implicit $exec
+    ; GCN-NEXT: DS_NOP implicit $m0, implicit $exec
+    ; GCN-NEXT: DS_NOP implicit $m0, implicit $exec
+    ; GCN-NEXT: DS_NOP implicit $m0, implicit $exec
+    ; GCN-NEXT: S_WAIT_LOADCNT 0
+    ; GCN-NEXT: $sgpr0_sgpr1 = S_GETPC_B64
+    ; GCN-NEXT: $sgpr3 = S_ADD_U32 $sgpr0, 0, implicit-def $scc
+    ; GCN-NEXT: S_ENDPGM 0
+    $vgpr1, $sgpr0 = V_ADDC_U32_e64 0, $vgpr1, $sgpr0, 0, implicit $exec
+    $vgpr3 = BUFFER_LOAD_DWORD_OFFSET $sgpr4_sgpr5_sgpr6_sgpr7, $sgpr8, 0, 0, 0, implicit $exec
+    S_WAIT_LOADCNT 0
+    $vgpr1, $sgpr0 = V_ADDC_U32_e64 0, $vgpr1, $sgpr2, 0, implicit $exec
+    $vgpr4 = BUFFER_LOAD_DWORD_OFFSET $sgpr4_sgpr5_sgpr6_sgpr7, $sgpr8, 0, 0, 0, implicit $exec
+    S_WAIT_LOADCNT 0
+    $sgpr0_sgpr1 = S_GETPC_B64
+    $sgpr3 = S_ADD_U32 $sgpr0, 0, implicit-def $scc
+    S_ENDPGM 0
+...
+
+---
+name:            hazard_cull_sample
+body:            |
+  bb.0:
+    ; GCN-LABEL: name: hazard_cull_sample
+    ; GCN: $vgpr1, $sgpr0 = V_ADDC_U32_e64 0, $vgpr1, $sgpr0, 0, implicit $exec
+    ; GCN-NEXT: $vgpr10 = IMAGE_SAMPLE_LZ_V1_V2_gfx12 $vgpr3, $vgpr4, $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4_sgpr5_sgpr6_sgpr7, $sgpr8_sgpr9_sgpr10_sgpr11, 1, 1, -1, 0, 0, 0, 0, 0, 0, implicit $exec :: (load (s128))
+    ; GCN-NEXT: DS_NOP implicit $m0, implicit $exec
+    ; GCN-NEXT: DS_NOP implicit $m0, implicit $exec
+    ; GCN-NEXT: DS_NOP implicit $m0, implicit $exec
+    ; GCN-NEXT: DS_NOP implicit $m0, implicit $exec
+    ; GCN-NEXT: S_WAIT_SAMPLECNT 0
+    ; GCN-NEXT: $sgpr0_sgpr1 = S_GETPC_B64
+    ; GCN-NEXT: $sgpr3 = S_ADD_U32 $sgpr0, 0, implicit-def $scc
+    ; GCN-NEXT: S_ENDPGM 0
+    $vgpr1, $sgpr0 = V_ADDC_U32_e64 0, $vgpr1, $sgpr0, 0, implicit $exec
+    $vgpr10 = IMAGE_SAMPLE_LZ_V1_V2_gfx12 $vgpr3, $vgpr4, $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4_sgpr5_sgpr6_sgpr7, $sgpr8_sgpr9_sgpr10_sgpr11, 1, 1, -1, 0, 0, 0, 0, 0, 0, implicit $exec :: (load (s128))
+    S_WAIT_SAMPLECNT 0
+    $sgpr0_sgpr1 = S_GETPC_B64
+    $sgpr3 = S_ADD_U32 $sgpr0, 0, implicit-def $scc
+    S_ENDPGM 0
+...
+
+---
+name:            hazard_cull_bvh
+body:            |
+  bb.0:
+    ; GCN-LABEL: name: hazard_cull_bvh
+    ; GCN: $vgpr1, $sgpr0 = V_ADDC_U32_e64 0, $vgpr1, $sgpr0, 0, implicit $exec
+    ; GCN-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = IMAGE_BVH_INTERSECT_RAY_sa_gfx11 $vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14, renamable $sgpr0_sgpr1_sgpr2_sgpr3, 0, implicit $exec :: (dereferenceable load (s128), addrspace 7)
+    ; GCN-NEXT: DS_NOP implicit $m0, implicit $exec
+    ; GCN-NEXT: DS_NOP implicit $m0, implicit $exec
+    ; GCN-NEXT: DS_NOP implicit $m0, implicit $exec
+    ; GCN-NEXT: DS_NOP implicit $m0, implicit $exec
+    ; GCN-NEXT: S_WAIT_BVHCNT 0
+    ; GCN-NEXT: $sgpr0_sgpr1 = S_GETPC_B64
+    ; GCN-NEXT: $sgpr3 = S_ADD_U32 $sgpr0, 0, implicit-def $scc
+    ; GCN-NEXT: S_ENDPGM 0
+    $vgpr1, $sgpr0 = V_ADDC_U32_e64 0, $vgpr1, $sgpr0, 0, implicit $exec
+    $vgpr0_vgpr1_vgpr2_vgpr3 = IMAGE_BVH_INTERSECT_RAY_sa_gfx11 $vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14, renamable $sgpr0_sgpr1_sgpr2_sgpr3, 0, implicit $exec :: (dereferenceable load (s128), addrspace 7)
+    S_WAIT_BVHCNT 0
+    $sgpr0_sgpr1 = S_GETPC_B64
+    $sgpr3 = S_ADD_U32 $sgpr0, 0, implicit-def $scc
+    S_ENDPGM 0
+...
+
+---
+name:            hazard_nocull_scratch
+body:            |
+  bb.0:
+    ; GCN-LABEL: name: hazard_nocull_scratch
+    ; GCN: $vgpr1, $sgpr0 = V_ADDC_U32_e64 0, $vgpr1, $sgpr0, 0, implicit $exec
+    ; GCN-NEXT: $vgpr0 = SCRATCH_LOAD_DWORD $vgpr0, 0, 0, implicit $exec, implicit $flat_scr
+    ; GCN-NEXT: S_WAIT_LOADCNT 0
+    ; GCN-NEXT: $sgpr0_sgpr1 = S_GETPC_B64
+    ; GCN-NEXT: S_WAITCNT_DEPCTR 65534
+    ; GCN-NEXT: $sgpr3 = S_ADD_U32 $sgpr0, 0, implicit-def $scc
+    ; GCN-NEXT: S_ENDPGM 0
+    $vgpr1, $sgpr0 = V_ADDC_U32_e64 0, $vgpr1, $sgpr0, 0, implicit $exec
+    $vgpr0 = SCRATCH_LOAD_DWORD $vgpr0, 0, 0, implicit $exec, implicit $flat_scr
+    S_WAIT_LOADCNT 0
+    $sgpr0_sgpr1 = S_GETPC_B64
+    $sgpr3 = S_ADD_U32 $sgpr0, 0, implicit-def $scc
+    S_ENDPGM 0
+...
+
+---
+name:            hazard_cull_global
+body:            |
+  bb.0:
+    ; GCN-LABEL: name: hazard_cull_global
+    ; GCN: $vgpr1, $sgpr0 = V_ADDC_U32_e64 0, $vgpr1, $sgpr0, 0, implicit $exec
+    ; GCN-NEXT: $vgpr0 = GLOBAL_LOAD_DWORD $vgpr0_vgpr1, 0, 0, implicit $exec
+    ; GCN-NEXT: DS_NOP implicit $m0, implicit $exec
+    ; GCN-NEXT: DS_NOP implicit $m0, implicit $exec
+    ; GCN-NEXT: DS_NOP implicit $m0, implicit $exec
+    ; GCN-NEXT: DS_NOP implicit $m0, implicit $exec
+    ; GCN-NEXT: S_WAIT_LOADCNT 0
+    ; GCN-NEXT: $sgpr0_sgpr1 = S_GETPC_B64
+    ; GCN-NEXT: $sgpr3 = S_ADD_U32 $sgpr0, 0, implicit-def $scc
+    ; GCN-NEXT: S_ENDPGM 0
+    $vgpr1, $sgpr0 = V_ADDC_U32_e64 0, $vgpr1, $sgpr0, 0, implicit $exec
+    $vgpr0 = GLOBAL_LOAD_DWORD $vgpr0_vgpr1, 0, 0, implicit $exec
+    S_WAIT_LOADCNT 0
+    $sgpr0_sgpr1 = S_GETPC_B64
+    $sgpr3 = S_ADD_U32 $sgpr0, 0, implicit-def $scc
+    S_ENDPGM 0
+...
+
+---
+name:            hazard_nocull_flat
+body:            |
+  bb.0:
+    ; GCN-LABEL: name: hazard_nocull_flat
+    ; GCN: $vgpr1, $sgpr0 = V_ADDC_U32_e64 0, $vgpr1, $sgpr0, 0, implicit $exec
+    ; GCN-NEXT: $vgpr0 = FLAT_LOAD_DWORD $vgpr0_vgpr1, 0, 0, implicit $exec, implicit $flat_scr
+    ; GCN-NEXT: S_WAIT_LOADCNT 0
+    ; GCN-NEXT: $sgpr0_sgpr1 = S_GETPC_B64
+    ; GCN-NEXT: S_WAITCNT_DEPCTR 65534
+    ; GCN-NEXT: $sgpr3 = S_ADD_U32 $sgpr0, 0, implicit-def $scc
+    ; GCN-NEXT: S_ENDPGM 0
+    $vgpr1, $sgpr0 = V_ADDC_U32_e64 0, $vgpr1, $sgpr0, 0, implicit $exec
+    $vgpr0 = FLAT_LOAD_DWORD $vgpr0_vgpr1, 0, 0, implicit $exec, implicit $flat_scr
+    S_WAIT_LOADCNT 0
+    $sgpr0_sgpr1 = S_GETPC_B64
+    $sgpr3 = S_ADD_U32 $sgpr0, 0, implicit-def $scc
+    S_ENDPGM 0
+...
diff --git a/llvm/test/CodeGen/AMDGPU/valu-read-sgpr-hazard.mir b/llvm/test/CodeGen/AMDGPU/valu-read-sgpr-hazard.mir
index 2aa16dd904766..04f7e480764e6 100644
--- a/llvm/test/CodeGen/AMDGPU/valu-read-sgpr-hazard.mir
+++ b/llvm/test/CodeGen/AMDGPU/valu-read-sgpr-hazard.mir
@@ -1,6 +1,7 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -O0 -mtriple=amdgcn -mcpu=gfx1200 -verify-machineinstrs -run-pass post-RA-hazard-rec -o - %s | FileCheck -check-prefixes=GCN-O0 %s
-# RUN: llc -O2 -mtriple=amdgcn -mcpu=gfx1200 -verify-machineinstrs -run-pass post-RA-hazard-rec -o - %s | FileCheck -check-prefixes=GCN-O2 %s
+# RUN: llc -mtriple=amdgcn -mcpu=gfx1200 -verify-machineinstrs -run-pass post-RA-hazard-rec,amdgpu-wait-sgpr-hazards -amdgpu-sgpr-hazard-boundary-cull=0 -o - %s | FileCheck -check-prefixes=GCN,NOBC,NOMEMC %s
+# RUN: llc -mtriple=amdgcn -mcpu=gfx1200 -verify-machineinstrs -run-pass post-RA-hazard-rec,amdgpu-wait-sgpr-hazards -amdgpu-sgpr-hazard-boundary-cull=1 -o - %s | FileCheck -check-prefixes=GCN,BC,NOMEMC %s
+# RUN: llc -mtriple=amdgcn -mcpu=gfx1200 -verify-machineinstrs -run-pass post-RA-hazard-rec,amdgpu-wait-sgpr-hazards -amdgpu-sgpr-hazard-boundary-cull=0 -amdgpu-sgpr-hazard-mem-wait-cull=1 -amdgpu-sgpr-hazard-mem-wait-cull-threshold=1 -o - %s | FileCheck -check-prefixes=GCN,NOBC,MEMC %s
 
 --- |
   @mem = internal unnamed_addr addrspace(4) constant [4 x <4 x i32>] [<4 x i32> <i32 0, i32 0, i32 0, i32 0>, <4 x i32> <i32 0, i32 0, i32 0, i32 0>, <4 x i32> <i32 0, i32 0, i32 0, i32 0>, <4 x i32> <i32 0, i32 0, i32 0, i32 0>]
@@ -12,6 +13,7 @@
   define amdgpu_gs void @hazard_vcc1() { ret void }
   define amdgpu_gs void @hazard_vcc2() { ret void }
   define amdgpu_gs void @hazard_vcc3() { ret void }
+  define amdgpu_gs void @hazard_merge_vcc() { ret void }
   define amdgpu_gs void @hazard_addc1() { ret void }
   define amdgpu_gs void @hazard_addc2() { ret void }
   define amdgpu_gs void @hazard_addc3() { ret void }
@@ -28,26 +30,33 @@
   define amdgpu_gs void @hazard_post_order2() { ret void }
   define amdgpu_gs void @hazard_post_order_cycle() { ret void }
   define amdgpu_cs void @hazard_calls() { ret void }
+  define void @hazard_callee1() { ret void }
+  define void @hazard_callee2() { ret void }
+  define amdgpu_cs void @hazard_carry_vcc() { ret void }
+  define amdgpu_cs void @hazard_carry_vcc_no_hazard() { ret void }
+  define amdgpu_cs void @hazard_carry_sgpr() { ret void }
+  define amdgpu_cs void @hazard_carry_sgpr_no_hazard1() { ret void }
+  define amdgpu_cs void @hazard_carry_sgpr_no_hazard2() { ret void }
+  define amdgpu_cs void @hazard_carry_sgpr_no_hazard3() { ret void }
+  define amdgpu_cs void @hazard_cull_vmem() { ret void }
+  define amdgpu_cs void @hazard_cull_sample() { ret void }
+  define amdgpu_cs void @hazard_cull_bvh() { ret void }
+  define amdgpu_cs void @hazard_nocull_scratch() { ret void }
+  define amdgpu_cs void @hazard_cull_global() { ret void }
+  define amdgpu_cs void @hazard_nocull_flat() { ret void }
+  define amdgpu_cs void @hazard_existing_cull() { ret void }
 ...
 
 ---
 name:            hazard_getpc1
 body:            |
   bb.0:
-    ; GCN-O0-LABEL: name: hazard_getpc1
-    ; GCN-O0: $vgpr1 = V_CNDMASK_B32_e64 0, $vgpr1, 0, $vgpr2, $sgpr0, implicit $exec
-    ; GCN-O0-NEXT: $sgpr0_sgpr1 = S_GETPC_B64
-    ; GCN-O0-NEXT: S_WAITCNT_DEPCTR 65534
-    ; GCN-O0-NEXT: $sgpr3 = S_ADD_U32 $sgpr0, 0, implicit-def $scc
-    ; GCN-O0-NEXT: S_WAITCNT_DEPCTR 65534
-    ; GCN-O0-NEXT: S_ENDPGM 0
-    ;
-    ; GCN-O2-LABEL: name: hazard_getpc1
-    ; GCN-O2: $vgpr1 = V_CNDMASK_B32_e64 0, $vgpr1, 0, $vgpr2, $sgpr0, implicit $exec
-    ; GCN-O2-NEXT: $sgpr0_sgpr1 = S_GETPC_B64
-    ; GCN-O2-NEXT: S_WAITCNT_DEPCTR 65534
-    ; GCN-O2-NEXT: $sgpr3 = S_ADD_U32 $sgpr0, 0, implicit-def $scc
-    ; GCN-O2-NEXT: S_ENDPGM 0
+    ; GCN-LABEL: name: hazard_getpc1
+    ; GCN: $vgpr1 = V_CNDMASK_B32_e64 0, $vgpr1, 0, $vgpr2, $sgpr0, implicit $exec
+    ; GCN-NEXT: $sgpr0_sgpr1 = S_GETPC_B64
+    ; GCN-NEXT: S_WAITCNT_DEPCTR 65534
+    ; GCN-NEXT: $sgpr3 = S_ADD_U32 $sgpr0, 0, implicit-def $scc
+    ; GCN-NEXT: S_ENDPGM 0
     $vgpr1 = V_CNDMASK_B32_e64 0, $vgpr1, 0, $vgpr2, $sgpr0, implicit $exec
     $sgpr0_sgpr1 = S_GETPC_B64
     $sgpr3 = S_ADD_U32 $sgpr0, 0, implicit-def $scc
@@ -58,20 +67,12 @@ body:            |
 name:            hazard_getpc2
 body:            |
   bb.0:
-    ; GCN-O0-LABEL: name: hazard_getpc2
-    ; GCN-O0: $vgpr1 = V_CNDMASK_B32_e64 0, $vgpr1, 0, $vgpr2, $sgpr1, implicit $exec
-    ; GCN-O0-NEXT: $sgpr0_sgpr1 = S_GETPC_B64
-    ; GCN-O0-NEXT: S_WAITCNT_DEPCTR 65534
-    ; GCN-O0-NEXT: $sgpr3 = S_ADD_U32 $sgpr0, 0, implicit-def $scc
-    ; GCN-O0-NEXT: S_WAITCNT_DEPCTR 65534
-    ; GCN-O0-NEXT: S_ENDPGM 0
-    ;
-    ; GCN-O2-LABEL: name: hazard_getpc2
-    ; GCN-O2: $vgpr1 = V_CNDMASK_B32_e64 0, $vgpr1, 0, $vgpr2, $sgpr1, implicit $exec
-    ; GCN-O2-NEXT: $sgpr0_sgpr1 = S_GETPC_B64
-    ; GCN-O2-NEXT: S_WAITCNT_DEPCTR 65534
-    ; GCN-O2-NEXT: $sgpr3 = S_ADD_U32 $sgpr0, 0, implicit-def $scc
-    ; GCN-O2-NEXT: S_ENDPGM 0
+    ; GCN-LABEL: name: hazard_getpc2
+    ; GCN: $vgpr1 = V_CNDMASK_B32_e64 0, $vgpr1, 0, $vgpr2, $sgpr1, implicit $exec
+    ; GCN-NEXT: $sgpr0_sgpr1 = S_GETPC_B64
+    ; GCN-NEXT: S_WAITCNT_DEPCTR 65534
+    ; GCN-NEXT: $sgpr3 = S_ADD_U32 $sgpr0, 0, implicit-def $scc
+    ; GCN-NEXT: S_ENDPGM 0
     $vgpr1 = V_CNDMASK_B32_e64 0, $vgpr1, 0, $vgpr2, $sgpr1, implicit $exec
     $sgpr0_sgpr1 = S_GETPC_B64
     $sgpr3 = S_ADD_U32 $sgpr0, 0, implicit-def $scc
@@ -82,27 +83,15 @@ body:            |
 name:            hazard_getpc3
 body:            |
   bb.0:
-    ; GCN-O0-LABEL: name: hazard_getpc3
-    ; GCN-O0: $vgpr1 = V_CNDMASK_B32_e64 0, $vgpr1, 0, $vgpr2, $sgpr0, implicit $exec
-    ; GCN-O0-NEXT: BUNDLE implicit-def $sgpr0_sgpr1 {
-    ; GCN-O0-NEXT:   $sgpr0_sgpr1 = S_GETPC_B64
-    ; GCN-O0-NEXT:   S_WAITCNT_DEPCTR 65534
-    ; GCN-O0-NEXT:   $sgpr0 = S_ADD_U32 $sgpr0, target-flags(amdgpu-rel32-lo) @mem + 8, implicit-def $scc
-    ; GCN-O0-NEXT:   S_WAITCNT_DEPCTR 65534
-    ; GCN-O0-NEXT:   $sgpr1 = S_ADDC_U32 $sgpr1, target-flags(amdgpu-rel32-hi) @mem + 20, implicit-def $scc, implicit $scc
-    ; GCN-O0-NEXT: }
-    ; GCN-O0-NEXT: S_WAITCNT_DEPCTR 65534
-    ; GCN-O0-NEXT: S_ENDPGM 0
-    ;
-    ; GCN-O2-LABEL: name: hazard_getpc3
-    ; GCN-O2: $vgpr1 = V_CNDMASK_B32_e64 0, $vgpr1, 0, $vgpr2, $sgpr0, implicit $exec
-    ; GCN-O2-NEXT: BUNDLE implicit-def $sgpr0_sgpr1 {
-    ; GCN-O2-NEXT:   $sgpr0_sgpr1 = S_GETPC_B64
-    ; GCN-O2-NEXT:   S_WAITCNT_DEPCTR 65534
-    ; GCN-O2-NEXT:   $sgpr0 = S_ADD_U32 $sgpr0, target-flags(amdgpu-rel32-lo) @mem + 8, implicit-def $scc
-    ; GCN-O2-NEXT:   $sgpr1 = S_ADDC_U32 $sgpr1, target-flags(amdgpu-rel32-hi) @mem + 16, implicit-def $scc, implicit $scc
-    ; GCN-O2-NEXT: }
-    ; GCN-O2-NEXT: S_ENDPGM 0
+    ; GCN-LABEL: name: hazard_getpc3
+    ; GCN: $vgpr1 = V_CNDMASK_B32_e64 0, $vgpr1, 0, $vgpr2, $sgpr0, implicit $exec
+    ; GCN-NEXT: BUNDLE implicit-def $sgpr0_sgpr1 {
+    ; GCN-NEXT:   $sgpr0_sgpr1 = S_GETPC_B64
+    ; GCN-NEXT:   S_WAITCNT_DEPCTR 65534
+    ; GCN-NEXT:   $sgpr0 = S_ADD_U32 $sgpr0, target-flags(amdgpu-rel32-lo) @mem + 8, implicit-def $scc
+    ; GCN-NEXT:   $sgpr1 = S_ADDC_U32 $sgpr1, target-flags(amdgpu-rel32-hi) @mem + 16, implicit-def $scc, implicit $scc
+    ; GCN-NEXT: }
+    ; GCN-NEXT: S_ENDPGM 0
     $vgpr1 = V_CNDMASK_B32_e64 0, $vgpr1, 0, $vgpr2, $sgpr0, implicit $exec
     BUNDLE implicit-def $sgpr0_sgpr1 {
       $sgpr0_sgpr1 = S_GETPC_B64
@@ -116,31 +105,17 @@ body:            |
 name:            hazard_getpc4
 body:            |
   bb.0:
-    ; GCN-O0-LABEL: name: hazard_getpc4
-    ; GCN-O0: $vgpr1 = V_CNDMASK_B32_e64 0, $vgpr1, 0, $vgpr2, $sgpr0, implicit $exec
-    ; GCN-O0-NEXT: BUNDLE implicit-def $sgpr0_sgpr1 {
-    ; GCN-O0-NEXT:   $sgpr0_sgpr1 = S_GETPC_B64
-    ; GCN-O0-NEXT:   S_WAITCNT_DEPCTR 65534
-    ; GCN-O0-NEXT:   $sgpr1 = S_SEXT_I32_I16 $sgpr1
-    ; GCN-O0-NEXT:   S_WAITCNT_DEPCTR 65534
-    ; GCN-O0-NEXT:   $sgpr0 = S_ADD_U32 $sgpr0, target-flags(amdgpu-rel32-lo) @mem + 16, implicit-def $scc
-    ; GCN-O0-NEXT:   S_WAITCNT_DEPCTR 65534
-    ; GCN-O0-NEXT:   $sgpr1 = S_ADDC_U32 $sgpr1, target-flags(amdgpu-rel32-hi) @mem + 28, implicit-def $scc, implicit $scc
-    ; GCN-O0-NEXT: }
-    ; GCN-O0-NEXT: S_WAITCNT_DEPCTR 65534
-    ; GCN-O0-NEXT: S_ENDPGM 0
-    ;
-    ; GCN-O2-LABEL: name: hazard_getpc4
-    ; GCN-O2: $vgpr1 = V_CNDMASK_B32_e64 0, $vgpr1, 0, $vgpr2, $sgpr0, implicit $exec
-    ; GCN-O2-NEXT: BUNDLE implicit-def $sgpr0_sgpr1 {
-    ; GCN-O2-NEXT:   $sgpr0_sgpr1 = S_GETPC_B64
-    ; GCN-O2-NEXT:   S_WAITCNT_DEPCTR 65534
-    ; GCN-O2-NEXT:   $sgpr1 = S_SEXT_I32_I16 $sgpr1
-    ; GCN-O2-NEXT:   $sgpr0 = S_ADD_U32 $sgpr0, target-flags(amdgpu-rel32-lo) @mem + 12, implicit-def $scc
-    ; GCN-O2-NEXT:   S_WAITCNT_DEPCTR 65534
-    ; GCN-O2-NEXT:   $sgpr1 = S_ADDC_U32 $sgpr1, target-flags(amdgpu-rel32-hi) @mem + 24, implicit-def $scc, implicit $scc
-    ; GCN-O2-NEXT: }
-    ; GCN-O2-NEXT: S_ENDPGM 0
+    ; GCN-LABEL: name: hazard_getpc4
+    ; GCN: $vgpr1 = V_CNDMASK_B32_e64 0, $vgpr1, 0, $vgpr2, $sgpr0, implicit $exec
+    ; GCN-NEXT: BUNDLE implicit-def $sgpr0_sgpr1 {
+    ; GCN-NEXT:   $sgpr0_sgpr1 = S_GETPC_B64
+    ; GCN-NEXT:   S_WAITCNT_DEPCTR 65534
+    ; GCN-NEXT:   $sgpr1 = S_SEXT_I32_I16 $sgpr1
+    ; GCN-NEXT:   $sgpr0 = S_ADD_U32 $sgpr0, target-flags(amdgpu-rel32-lo) @mem + 12, implicit-def $scc
+    ; GCN-NEXT:   S_WAITCNT_DEPCTR 65534
+    ; GCN-NEXT:   $sgpr1 = S_ADDC_U32 $sgpr1, target-flags(amdgpu-rel32-hi) @mem + 24, implicit-def $scc, implicit $scc
+    ; GCN-NEXT: }
+    ; GCN-NEXT: S_ENDPGM 0
     $vgpr1 = V_CNDMASK_B32_e64 0, $vgpr1, 0, $vgpr2, $sgpr0, implicit $exec
     BUNDLE implicit-def $sgpr0_sgpr1 {
       $sgpr0_sgpr1 = S_GETPC_B64
@@ -155,20 +130,12 @@ body:            |
 name:            hazard_vcc1
 body:            |
   bb.0:
-    ; GCN-O0-LABEL: name: hazard_vcc1
-    ; GCN-O0: $vgpr1 = V_CNDMASK_B32_e64 0, $vgpr1, 0, $vgpr2, $sgpr2, implicit $exec
-    ; GCN-O0-NEXT: $sgpr3 = S_CSELECT_B32 -1, 0, implicit $scc
-    ; GCN-O0-NEXT: S_WAITCNT_DEPCTR 65534
-    ; GCN-O0-NEXT: $sgpr4 = S_ADD_U32 $sgpr3, 0, implicit-def $scc
-    ; GCN-O0-NEXT: S_WAITCNT_DEPCTR 65534
-    ; GCN-O0-NEXT: S_ENDPGM 0
-    ;
-    ; GCN-O2-LABEL: name: hazard_vcc1
-    ; GCN-O2: $vgpr1 = V_CNDMASK_B32_e64 0, $vgpr1, 0, $vgpr2, $sgpr2, implicit $exec
-    ; GCN-O2-NEXT: $sgpr3 = S_CSELECT_B32 -1, 0, implicit $scc
-    ; GCN-O2-NEXT: S_WAITCNT_DEPCTR 65534
-    ; GCN-O2-NEXT: $sgpr4 = S_ADD_U32 $sgpr3, 0, implicit-def $scc
-    ; GCN-O2-NEXT: S_ENDPGM 0
+    ; GCN-LABEL: name: hazard_vcc1
+    ; GCN: $vgpr1 = V_CNDMASK_B32_e64 0, $vgpr1, 0, $vgpr2, $sgpr2, implicit $exec
+    ; GCN-NEXT: $sgpr3 = S_CSELECT_B32 -1, 0, implicit $scc
+    ; GCN-NEXT: S_WAITCNT_DEPCTR 65534
+    ; GCN-NEXT: $sgpr4 = S_ADD_U32 $sgpr3, 0, implicit-def $scc
+    ; GCN-NEXT: S_ENDPGM 0
     $vgpr1 = V_CNDMASK_B32_e64 0, $vgpr1, 0, $vgpr2, $sgpr2, implicit $exec
     $sgpr3 = S_CSELECT_B32 -1, 0, implicit $scc
     $sgpr4 = S_ADD_U32 $sgpr3, 0, implicit-def $scc
@@ -179,20 +146,12 @@ body:            |
 name:            hazard_vcc2
 body:            |
   bb.0:
-    ; GCN-O0-LABEL: name: hazard_vcc2
-    ; GCN-O0: $vgpr1 = V_CNDMASK_B32_e32 $vgpr1, $vgpr2, implicit $vcc_lo, implicit $exec
-    ; GCN-O0-NEXT: $vcc_lo = S_CSELECT_B32 -1, 0, implicit $scc
-    ; GCN-O0-NEXT: S_WAITCNT_DEPCTR 65534
-    ; GCN-O0-NEXT: $sgpr4 = S_ADD_U32 $vcc_lo, 0, implicit-def $scc
-    ; GCN-O0-NEXT: S_WAITCNT_DEPCTR 65534
-    ; GCN-O0-NEXT: S_ENDPGM 0
-    ;
-    ; GCN-O2-LABEL: name: hazard_vcc2
-    ; GCN-O2: $vgpr1 = V_CNDMASK_B32_e32 $vgpr1, $vgpr2, implicit $vcc_lo, implicit $exec
-    ; GCN-O2-NEXT: $vcc_lo = S_CSELECT_B32 -1, 0, implicit $scc
-    ; GCN-O2-NEXT: S_WAITCNT_DEPCTR 65534
-    ; GCN-O2-NEXT: $sgpr4 = S_ADD_U32 $vcc_lo, 0, implicit-def $scc
-    ; GCN-O2-NEXT: S_ENDPGM 0
+    ; GCN-LABEL: name: hazard_vcc2
+    ; GCN: $vgpr1 = V_CNDMASK_B32_e32 $vgpr1, $vgpr2, implicit $vcc_lo, implicit $exec
+    ; GCN-NEXT: $vcc_lo = S_CSELECT_B32 -1, 0, implicit $scc
+    ; GCN-NEXT: S_WAITCNT_DEPCTR 65534
+    ; GCN-NEXT: $sgpr4 = S_ADD_U32 $vcc_lo, 0, implicit-def $scc
+    ; GCN-NEXT: S_ENDPGM 0
     $vgpr1 = V_CNDMASK_B32_e32 $vgpr1, $vgpr2, implicit $vcc, implicit $exec
     $vcc_lo = S_CSELECT_B32 -1, 0, implicit $scc
     $sgpr4 = S_ADD_U32 $vcc_lo, 0, implicit-def $scc
@@ -203,21 +162,56 @@ body:            |
 name:            hazard_vcc3
 body:            |
   bb.0:
-    ; GCN-O0-LABEL: name: hazard_vcc3
-    ; GCN-O0: $vgpr1 = V_CNDMASK_B32_e32 $vgpr1, $vgpr2, implicit $vcc_lo, implicit $exec
-    ; GCN-O0-NEXT: $vcc_lo = S_CSELECT_B32 -1, 0, implicit $scc
-    ; GCN-O0-NEXT: S_WAITCNT_DEPCTR 65534
-    ; GCN-O0-NEXT: $vgpr3 = V_CNDMASK_B32_e32 $vgpr4, $vgpr5, implicit $vcc_lo, implicit $exec
-    ; GCN-O0-NEXT: S_ENDPGM 0
-    ;
-    ; GCN-O2-LABEL: name: hazard_vcc3
-    ; GCN-O2: $vgpr1 = V_CNDMASK_B32_e32 $vgpr1, $vgpr2, implicit $vcc_lo, implicit $exec
-    ; GCN-O2-NEXT: $vcc_lo = S_CSELECT_B32 -1, 0, implicit $scc
-    ; GCN-O2-NEXT: S_WAITCNT_DEPCTR 65534
-    ; GCN-O2-NEXT: $vgpr3 = V_CNDMASK_B32_e32 $vgpr4, $vgpr5, implicit $vcc_lo, implicit $exec
-    ; GCN-O2-NEXT: S_ENDPGM 0
+    ; GCN-LABEL: name: hazard_vcc3
+    ; GCN: $vgpr1 = V_CNDMASK_B32_e32 $vgpr1, $vgpr2, implicit $vcc_lo, implicit $exec
+    ; GCN-NEXT: $vcc_lo = S_CSELECT_B32 -1, 0, implicit $scc
+    ; GCN-NEXT: S_WAITCNT_DEPCTR 65534
+    ; GCN-NEXT: $vgpr3 = V_CNDMASK_B32_e32 $vgpr4, $vgpr5, implicit $vcc_lo, implicit $exec
+    ; GCN-NEXT: S_ENDPGM 0
+    $vgpr1 = V_CNDMASK_B32_e32 $vgpr1, $vgpr2, implicit $vcc, implicit $exec
+    $vcc_lo = S_CSELECT_B32 -1, 0, implicit $scc
+    $vgpr3 = V_CNDMASK_B32_e32 $vgpr4, $vgpr5, implicit $vcc, implicit $exec
+    S_ENDPGM 0
+...
+
+---
+name:            hazard_merge_vcc
+body:            |
+  ; GCN-LABEL: name: hazard_merge_vcc
+  ; GCN: bb.0:
+  ; GCN-NEXT:   successors: %bb.1(0x40000000), %bb.2(0x40000000)
+  ; GCN-NEXT: {{  $}}
+  ; GCN-NEXT:   $vgpr1 = V_CNDMASK_B32_e32 $vgpr1, $vgpr2, implicit $vcc_lo, implicit $exec
+  ; GCN-NEXT:   S_CBRANCH_SCC0 %bb.1, implicit $scc
+  ; GCN-NEXT:   S_BRANCH %bb.2
+  ; GCN-NEXT: {{  $}}
+  ; GCN-NEXT: bb.1:
+  ; GCN-NEXT:   successors: %bb.3(0x80000000)
+  ; GCN-NEXT: {{  $}}
+  ; GCN-NEXT:   $vcc_lo = S_CSELECT_B32 -1, 0, implicit $scc
+  ; GCN-NEXT:   S_BRANCH %bb.3
+  ; GCN-NEXT: {{  $}}
+  ; GCN-NEXT: bb.2:
+  ; GCN-NEXT:   successors: %bb.3(0x80000000)
+  ; GCN-NEXT: {{  $}}
+  ; GCN-NEXT:   $vcc_lo = V_CMP_EQ_F32_e64 0, $vgpr0, 0, $vgpr1, 1, implicit $mode, implicit $exec
+  ; GCN-NEXT:   S_BRANCH %bb.3
+  ; GCN-NEXT: {{  $}}
+  ; GCN-NEXT: bb.3:
+  ; GCN-NEXT:   S_WAITCNT_DEPCTR 65532
+  ; GCN-NEXT:   $vgpr3 = V_CNDMASK_B32_e32 $vgpr4, $vgpr5, implicit $vcc_lo, implicit $exec
+  ; GCN-NEXT:   S_ENDPGM 0
+  bb.0:
     $vgpr1 = V_CNDMASK_B32_e32 $vgpr1, $vgpr2, implicit $vcc, implicit $exec
+    S_CBRANCH_SCC0 %bb.1, implicit $scc
+    S_BRANCH %bb.2
+  bb.1:
     $vcc_lo = S_CSELECT_B32 -1, 0, implicit $scc
+    S_BRANCH %bb.3
+  bb.2:
+    $vcc_lo = V_CMP_EQ_F32_e64 0, $vgpr0, 0, $vgpr1, 1, implicit $mode, implicit $exec
+    S_BRANCH %bb.3
+  bb.3:
     $vgpr3 = V_CNDMASK_B32_e32 $vgpr4, $vgpr5, implicit $vcc, implicit $exec
     S_ENDPGM 0
 ...
@@ -226,20 +220,12 @@ body:            |
 name:            hazard_addc1
 body:            |
   bb.0:
-    ; GCN-O0-LABEL: name: hazard_addc1
-    ; GCN-O0: $vgpr1, $vcc_lo = V_ADDC_U32_e64 0, $vgpr1, $sgpr0, 0, implicit $exec
-    ; GCN-O0-NEXT: $sgpr0 = S_CSELECT_B32 -1, 0, implicit $scc
-    ; GCN-O0-NEXT: S_WAITCNT_DEPCTR 65534
-    ; GCN-O0-NEXT: $sgpr2 = S_ADD_U32 $sgpr0, 0, implicit-def $scc
-    ; GCN-O0-NEXT: S_WAITCNT_DEPCTR 65534
-    ; GCN-O0-NEXT: S_ENDPGM 0
-    ;
-    ; GCN-O2-LABEL: name: hazard_addc1
-    ; GCN-O2: $vgpr1, $vcc_lo = V_ADDC_U32_e64 0, $vgpr1, $sgpr0, 0, implicit $exec
-    ; GCN-O2-NEXT: $sgpr0 = S_CSELECT_B32 -1, 0, implicit $scc
-    ; GCN-O2-NEXT: S_WAITCNT_DEPCTR 65534
-    ; GCN-O2-NEXT: $sgpr2 = S_ADD_U32 $sgpr0, 0, implicit-def $scc
-    ; GCN-O2-NEXT: S_ENDPGM 0
+    ; GCN-LABEL: name: hazard_addc1
+    ; GCN: $vgpr1, $vcc_lo = V_ADDC_U32_e64 0, $vgpr1, $sgpr0, 0, implicit $exec
+    ; GCN-NEXT: $sgpr0 = S_CSELECT_B32 -1, 0, implicit $scc
+    ; GCN-NEXT: S_WAITCNT_DEPCTR 65534
+    ; GCN-NEXT: $sgpr2 = S_ADD_U32 $sgpr0, 0, implicit-def $scc
+    ; GCN-NEXT: S_ENDPGM 0
     $vgpr1, $vcc_lo = V_ADDC_U32_e64 0, $vgpr1, $sgpr0, 0, implicit $exec
     $sgpr0 = S_CSELECT_B32 -1, 0, implicit $scc
     $sgpr2 = S_ADD_U32 $sgpr0, 0, implicit-def $scc
@@ -250,20 +236,12 @@ body:            |
 name:            hazard_addc2
 body:            |
   bb.0:
-    ; GCN-O0-LABEL: name: hazard_addc2
-    ; GCN-O0: $vgpr1, $vcc_lo = V_ADDC_U32_e64 0, $vgpr1, $sgpr1, 0, implicit $exec
-    ; GCN-O0-NEXT: $sgpr0 = S_CSELECT_B32 -1, 0, implicit $scc
-    ; GCN-O0-NEXT: S_WAITCNT_DEPCTR 65534
-    ; GCN-O0-NEXT: $sgpr2 = S_ADD_U32 $sgpr0, 0, implicit-def $scc
-    ; GCN-O0-NEXT: S_WAITCNT_DEPCTR 65534
-    ; GCN-O0-NEXT: S_ENDPGM 0
-    ;
-    ; GCN-O2-LABEL: name: hazard_addc2
-    ; GCN-O2: $vgpr1, $vcc_lo = V_ADDC_U32_e64 0, $vgpr1, $sgpr1, 0, implicit $exec
-    ; GCN-O2-NEXT: $sgpr0 = S_CSELECT_B32 -1, 0, implicit $scc
-    ; GCN-O2-NEXT: S_WAITCNT_DEPCTR 65534
-    ; GCN-O2-NEXT: $sgpr2 = S_ADD_U32 $sgpr0, 0, implicit-def $scc
-    ; GCN-O2-NEXT: S_ENDPGM 0
+    ; GCN-LABEL: name: hazard_addc2
+    ; GCN: $vgpr1, $vcc_lo = V_ADDC_U32_e64 0, $vgpr1, $sgpr1, 0, implicit $exec
+    ; GCN-NEXT: $sgpr0 = S_CSELECT_B32 -1, 0, implicit $scc
+    ; GCN-NEXT: S_WAITCNT_DEPCTR 65534
+    ; GCN-NEXT: $sgpr2 = S_ADD_U32 $sgpr0, 0, implicit-def $scc
+    ; GCN-NEXT: S_ENDPGM 0
     $vgpr1, $vcc_lo = V_ADDC_U32_e64 0, $vgpr1, $sgpr1, 0, implicit $exec
     $sgpr0 = S_CSELECT_B32 -1, 0, implicit $scc
     $sgpr2 = S_ADD_U32 $sgpr0, 0, implicit-def $scc
@@ -274,20 +252,12 @@ body:            |
 name:            hazard_addc3
 body:            |
   bb.0:
-    ; GCN-O0-LABEL: name: hazard_addc3
-    ; GCN-O0: $vgpr1, $vcc_lo = V_ADDC_U32_e64 0, $vgpr1, $sgpr0, 0, implicit $exec
-    ; GCN-O0-NEXT: $sgpr1 = S_CSELECT_B32 -1, 0, implicit $scc
-    ; GCN-O0-NEXT: S_WAITCNT_DEPCTR 65534
-    ; GCN-O0-NEXT: $sgpr2 = S_ADD_U32 $sgpr1, 0, implicit-def $scc
-    ; GCN-O0-NEXT: S_WAITCNT_DEPCTR 65534
-    ; GCN-O0-NEXT: S_ENDPGM 0
-    ;
-    ; GCN-O2-LABEL: name: hazard_addc3
-    ; GCN-O2: $vgpr1, $vcc_lo = V_ADDC_U32_e64 0, $vgpr1, $sgpr0, 0, implicit $exec
-    ; GCN-O2-NEXT: $sgpr1 = S_CSELECT_B32 -1, 0, implicit $scc
-    ; GCN-O2-NEXT: S_WAITCNT_DEPCTR 65534
-    ; GCN-O2-NEXT: $sgpr2 = S_ADD_U32 $sgpr1, 0, implicit-def $scc
-    ; GCN-O2-NEXT: S_ENDPGM 0
+    ; GCN-LABEL: name: hazard_addc3
+    ; GCN: $vgpr1, $vcc_lo = V_ADDC_U32_e64 0, $vgpr1, $sgpr0, 0, implicit $exec
+    ; GCN-NEXT: $sgpr1 = S_CSELECT_B32 -1, 0, implicit $scc
+    ; GCN-NEXT: S_WAITCNT_DEPCTR 65534
+    ; GCN-NEXT: $sgpr2 = S_ADD_U32 $sgpr1, 0, implicit-def $scc
+    ; GCN-NEXT: S_ENDPGM 0
     $vgpr1, $vcc_lo = V_ADDC_U32_e64 0, $vgpr1, $sgpr0, 0, implicit $exec
     $sgpr1 = S_CSELECT_B32 -1, 0, implicit $scc
     $sgpr2 = S_ADD_U32 $sgpr1, 0, implicit-def $scc
@@ -298,20 +268,12 @@ body:            |
 name:            hazard_addc4
 body:            |
   bb.0:
-    ; GCN-O0-LABEL: name: hazard_addc4
-    ; GCN-O0: $vgpr1, $vcc_lo = V_ADDC_U32_e64 0, $vgpr1, $sgpr3, 0, implicit $exec
-    ; GCN-O0-NEXT: $sgpr3 = S_CSELECT_B32 -1, 0, implicit $scc
-    ; GCN-O0-NEXT: S_WAITCNT_DEPCTR 65534
-    ; GCN-O0-NEXT: $sgpr2 = S_ADD_U32 $sgpr3, 0, implicit-def $scc
-    ; GCN-O0-NEXT: S_WAITCNT_DEPCTR 65534
-    ; GCN-O0-NEXT: S_ENDPGM 0
-    ;
-    ; GCN-O2-LABEL: name: hazard_addc4
-    ; GCN-O2: $vgpr1, $vcc_lo = V_ADDC_U32_e64 0, $vgpr1, $sgpr3, 0, implicit $exec
-    ; GCN-O2-NEXT: $sgpr3 = S_CSELECT_B32 -1, 0, implicit $scc
-    ; GCN-O2-NEXT: S_WAITCNT_DEPCTR 65534
-    ; GCN-O2-NEXT: $sgpr2 = S_ADD_U32 $sgpr3, 0, implicit-def $scc
-    ; GCN-O2-NEXT: S_ENDPGM 0
+    ; GCN-LABEL: name: hazard_addc4
+    ; GCN: $vgpr1, $vcc_lo = V_ADDC_U32_e64 0, $vgpr1, $sgpr3, 0, implicit $exec
+    ; GCN-NEXT: $sgpr3 = S_CSELECT_B32 -1, 0, implicit $scc
+    ; GCN-NEXT: S_WAITCNT_DEPCTR 65534
+    ; GCN-NEXT: $sgpr2 = S_ADD_U32 $sgpr3, 0, implicit-def $scc
+    ; GCN-NEXT: S_ENDPGM 0
     $vgpr1, $vcc_lo = V_ADDC_U32_e64 0, $vgpr1, $sgpr3, 0, implicit $exec
     $sgpr3 = S_CSELECT_B32 -1, 0, implicit $scc
     $sgpr2 = S_ADD_U32 $sgpr3, 0, implicit-def $scc
@@ -322,26 +284,14 @@ body:            |
 name:            hazard_addc5
 body:            |
   bb.0:
-    ; GCN-O0-LABEL: name: hazard_addc5
-    ; GCN-O0: $vgpr1, $vcc_lo = V_ADDC_U32_e64 0, $vgpr1, $sgpr0, 0, implicit $exec
-    ; GCN-O0-NEXT: $sgpr16 = S_MOV_B32 0
-    ; GCN-O0-NEXT: S_WAITCNT_DEPCTR 65534
-    ; GCN-O0-NEXT: $sgpr32 = S_MOV_B32 0
-    ; GCN-O0-NEXT: S_WAITCNT_DEPCTR 65534
-    ; GCN-O0-NEXT: $sgpr1 = S_CSELECT_B32 -1, 0, implicit $scc
-    ; GCN-O0-NEXT: S_WAITCNT_DEPCTR 65534
-    ; GCN-O0-NEXT: $sgpr2 = S_ADD_U32 $sgpr1, 0, implicit-def $scc
-    ; GCN-O0-NEXT: S_WAITCNT_DEPCTR 65534
-    ; GCN-O0-NEXT: S_ENDPGM 0
-    ;
-    ; GCN-O2-LABEL: name: hazard_addc5
-    ; GCN-O2: $vgpr1, $vcc_lo = V_ADDC_U32_e64 0, $vgpr1, $sgpr0, 0, implicit $exec
-    ; GCN-O2-NEXT: $sgpr16 = S_MOV_B32 0
-    ; GCN-O2-NEXT: $sgpr32 = S_MOV_B32 0
-    ; GCN-O2-NEXT: $sgpr1 = S_CSELECT_B32 -1, 0, implicit $scc
-    ; GCN-O2-NEXT: S_WAITCNT_DEPCTR 65534
-    ; GCN-O2-NEXT: $sgpr2 = S_ADD_U32 $sgpr1, 0, implicit-def $scc
-    ; GCN-O2-NEXT: S_ENDPGM 0
+    ; GCN-LABEL: name: hazard_addc5
+    ; GCN: $vgpr1, $vcc_lo = V_ADDC_U32_e64 0, $vgpr1, $sgpr0, 0, implicit $exec
+    ; GCN-NEXT: $sgpr16 = S_MOV_B32 0
+    ; GCN-NEXT: $sgpr32 = S_MOV_B32 0
+    ; GCN-NEXT: $sgpr1 = S_CSELECT_B32 -1, 0, implicit $scc
+    ; GCN-NEXT: S_WAITCNT_DEPCTR 65534
+    ; GCN-NEXT: $sgpr2 = S_ADD_U32 $sgpr1, 0, implicit-def $scc
+    ; GCN-NEXT: S_ENDPGM 0
     $vgpr1, $vcc_lo = V_ADDC_U32_e64 0, $vgpr1, $sgpr0, 0, implicit $exec
     $sgpr16 = S_MOV_B32 0
     $sgpr32 = S_MOV_B32 0
@@ -354,35 +304,17 @@ body:            |
 name:            hazard_addc6
 body:            |
   bb.0:
-    ; GCN-O0-LABEL: name: hazard_addc6
-    ; GCN-O0: $vgpr1, $vcc_lo = V_ADDC_U32_e64 0, $vgpr1, $sgpr0, 0, implicit $exec
-    ; GCN-O0-NEXT: $sgpr16 = S_MOV_B32 0
-    ; GCN-O0-NEXT: S_WAITCNT_DEPCTR 65534
-    ; GCN-O0-NEXT: $sgpr32 = S_MOV_B32 0
-    ; GCN-O0-NEXT: S_WAITCNT_DEPCTR 65534
-    ; GCN-O0-NEXT: $sgpr48 = S_MOV_B32 0
-    ; GCN-O0-NEXT: S_WAITCNT_DEPCTR 65534
-    ; GCN-O0-NEXT: $sgpr80 = S_MOV_B32 0
-    ; GCN-O0-NEXT: S_WAITCNT_DEPCTR 65534
-    ; GCN-O0-NEXT: $sgpr96 = S_MOV_B32 0
-    ; GCN-O0-NEXT: S_WAITCNT_DEPCTR 65534
-    ; GCN-O0-NEXT: $sgpr1 = S_CSELECT_B32 -1, 0, implicit $scc
-    ; GCN-O0-NEXT: S_WAITCNT_DEPCTR 65534
-    ; GCN-O0-NEXT: $sgpr2 = S_ADD_U32 $sgpr1, 0, implicit-def $scc
-    ; GCN-O0-NEXT: S_WAITCNT_DEPCTR 65534
-    ; GCN-O0-NEXT: S_ENDPGM 0
-    ;
-    ; GCN-O2-LABEL: name: hazard_addc6
-    ; GCN-O2: $vgpr1, $vcc_lo = V_ADDC_U32_e64 0, $vgpr1, $sgpr0, 0, implicit $exec
-    ; GCN-O2-NEXT: $sgpr16 = S_MOV_B32 0
-    ; GCN-O2-NEXT: $sgpr32 = S_MOV_B32 0
-    ; GCN-O2-NEXT: $sgpr48 = S_MOV_B32 0
-    ; GCN-O2-NEXT: $sgpr80 = S_MOV_B32 0
-    ; GCN-O2-NEXT: $sgpr96 = S_MOV_B32 0
-    ; GCN-O2-NEXT: $sgpr1 = S_CSELECT_B32 -1, 0, implicit $scc
-    ; GCN-O2-NEXT: S_WAITCNT_DEPCTR 65534
-    ; GCN-O2-NEXT: $sgpr2 = S_ADD_U32 $sgpr1, 0, implicit-def $scc
-    ; GCN-O2-NEXT: S_ENDPGM 0
+    ; GCN-LABEL: name: hazard_addc6
+    ; GCN: $vgpr1, $vcc_lo = V_ADDC_U32_e64 0, $vgpr1, $sgpr0, 0, implicit $exec
+    ; GCN-NEXT: $sgpr16 = S_MOV_B32 0
+    ; GCN-NEXT: $sgpr32 = S_MOV_B32 0
+    ; GCN-NEXT: $sgpr48 = S_MOV_B32 0
+    ; GCN-NEXT: $sgpr80 = S_MOV_B32 0
+    ; GCN-NEXT: $sgpr96 = S_MOV_B32 0
+    ; GCN-NEXT: $sgpr1 = S_CSELECT_B32 -1, 0, implicit $scc
+    ; GCN-NEXT: S_WAITCNT_DEPCTR 65534
+    ; GCN-NEXT: $sgpr2 = S_ADD_U32 $sgpr1, 0, implicit-def $scc
+    ; GCN-NEXT: S_ENDPGM 0
     $vgpr1, $vcc_lo = V_ADDC_U32_e64 0, $vgpr1, $sgpr0, 0, implicit $exec
     $sgpr16 = S_MOV_B32 0
     $sgpr32 = S_MOV_B32 0
@@ -398,19 +330,12 @@ body:            |
 name:            hazard_vaddc1
 body:            |
   bb.0:
-    ; GCN-O0-LABEL: name: hazard_vaddc1
-    ; GCN-O0: $vgpr1, $vcc_lo = V_ADDC_U32_e64 0, $vgpr1, $sgpr0, 0, implicit $exec
-    ; GCN-O0-NEXT: $sgpr1 = S_CSELECT_B32 -1, 0, implicit $scc
-    ; GCN-O0-NEXT: S_WAITCNT_DEPCTR 65534
-    ; GCN-O0-NEXT: $vgpr2, $vcc_lo = V_ADDC_U32_e64 0, $vgpr1, $sgpr1, 0, implicit $exec
-    ; GCN-O0-NEXT: S_ENDPGM 0
-    ;
-    ; GCN-O2-LABEL: name: hazard_vaddc1
-    ; GCN-O2: $vgpr1, $vcc_lo = V_ADDC_U32_e64 0, $vgpr1, $sgpr0, 0, implicit $exec
-    ; GCN-O2-NEXT: $sgpr1 = S_CSELECT_B32 -1, 0, implicit $scc
-    ; GCN-O2-NEXT: S_WAITCNT_DEPCTR 65534
-    ; GCN-O2-NEXT: $vgpr2, $vcc_lo = V_ADDC_U32_e64 0, $vgpr1, $sgpr1, 0, implicit $exec
-    ; GCN-O2-NEXT: S_ENDPGM 0
+    ; GCN-LABEL: name: hazard_vaddc1
+    ; GCN: $vgpr1, $vcc_lo = V_ADDC_U32_e64 0, $vgpr1, $sgpr0, 0, implicit $exec
+    ; GCN-NEXT: $sgpr1 = S_CSELECT_B32 -1, 0, implicit $scc
+    ; GCN-NEXT: S_WAITCNT_DEPCTR 65534
+    ; GCN-NEXT: $vgpr2, $vcc_lo = V_ADDC_U32_e64 0, $vgpr1, $sgpr1, 0, implicit $exec
+    ; GCN-NEXT: S_ENDPGM 0
     $vgpr1, $vcc_lo = V_ADDC_U32_e64 0, $vgpr1, $sgpr0, 0, implicit $exec
     $sgpr1 = S_CSELECT_B32 -1, 0, implicit $scc
     $vgpr2, $vcc_lo = V_ADDC_U32_e64 0, $vgpr1, $sgpr1, 0, implicit $exec
@@ -421,36 +346,20 @@ body:            |
 name:            hazard_gap1
 body:            |
   bb.0:
-    ; GCN-O0-LABEL: name: hazard_gap1
-    ; GCN-O0: $vgpr1, $vcc_lo = V_ADDC_U32_e64 0, $vgpr1, $sgpr0, 0, implicit $exec
-    ; GCN-O0-NEXT: S_NOP 0
-    ; GCN-O0-NEXT: S_NOP 0
-    ; GCN-O0-NEXT: S_NOP 0
-    ; GCN-O0-NEXT: S_NOP 0
-    ; GCN-O0-NEXT: S_NOP 0
-    ; GCN-O0-NEXT: S_NOP 0
-    ; GCN-O0-NEXT: S_NOP 0
-    ; GCN-O0-NEXT: S_NOP 0
-    ; GCN-O0-NEXT: $sgpr1 = S_CSELECT_B32 -1, 0, implicit $scc
-    ; GCN-O0-NEXT: S_WAITCNT_DEPCTR 65534
-    ; GCN-O0-NEXT: $sgpr2 = S_ADD_U32 $sgpr1, 0, implicit-def $scc
-    ; GCN-O0-NEXT: S_WAITCNT_DEPCTR 65534
-    ; GCN-O0-NEXT: S_ENDPGM 0
-    ;
-    ; GCN-O2-LABEL: name: hazard_gap1
-    ; GCN-O2: $vgpr1, $vcc_lo = V_ADDC_U32_e64 0, $vgpr1, $sgpr0, 0, implicit $exec
-    ; GCN-O2-NEXT: S_NOP 0
-    ; GCN-O2-NEXT: S_NOP 0
-    ; GCN-O2-NEXT: S_NOP 0
-    ; GCN-O2-NEXT: S_NOP 0
-    ; GCN-O2-NEXT: S_NOP 0
-    ; GCN-O2-NEXT: S_NOP 0
-    ; GCN-O2-NEXT: S_NOP 0
-    ; GCN-O2-NEXT: S_NOP 0
-    ; GCN-O2-NEXT: $sgpr1 = S_CSELECT_B32 -1, 0, implicit $scc
-    ; GCN-O2-NEXT: S_WAITCNT_DEPCTR 65534
-    ; GCN-O2-NEXT: $sgpr2 = S_ADD_U32 $sgpr1, 0, implicit-def $scc
-    ; GCN-O2-NEXT: S_ENDPGM 0
+    ; GCN-LABEL: name: hazard_gap1
+    ; GCN: $vgpr1, $vcc_lo = V_ADDC_U32_e64 0, $vgpr1, $sgpr0, 0, implicit $exec
+    ; GCN-NEXT: S_NOP 0
+    ; GCN-NEXT: S_NOP 0
+    ; GCN-NEXT: S_NOP 0
+    ; GCN-NEXT: S_NOP 0
+    ; GCN-NEXT: S_NOP 0
+    ; GCN-NEXT: S_NOP 0
+    ; GCN-NEXT: S_NOP 0
+    ; GCN-NEXT: S_NOP 0
+    ; GCN-NEXT: $sgpr1 = S_CSELECT_B32 -1, 0, implicit $scc
+    ; GCN-NEXT: S_WAITCNT_DEPCTR 65534
+    ; GCN-NEXT: $sgpr2 = S_ADD_U32 $sgpr1, 0, implicit-def $scc
+    ; GCN-NEXT: S_ENDPGM 0
     $vgpr1, $vcc_lo = V_ADDC_U32_e64 0, $vgpr1, $sgpr0, 0, implicit $exec
     S_NOP 0
     S_NOP 0
@@ -469,36 +378,20 @@ body:            |
 name:            hazard_gap2
 body:            |
   bb.0:
-    ; GCN-O0-LABEL: name: hazard_gap2
-    ; GCN-O0: $vgpr1, $vcc_lo = V_ADDC_U32_e64 0, $vgpr1, $sgpr0, 0, implicit $exec
-    ; GCN-O0-NEXT: $sgpr1 = S_CSELECT_B32 -1, 0, implicit $scc
-    ; GCN-O0-NEXT: S_WAITCNT_DEPCTR 65534
-    ; GCN-O0-NEXT: S_NOP 0
-    ; GCN-O0-NEXT: S_NOP 0
-    ; GCN-O0-NEXT: S_NOP 0
-    ; GCN-O0-NEXT: S_NOP 0
-    ; GCN-O0-NEXT: S_NOP 0
-    ; GCN-O0-NEXT: S_NOP 0
-    ; GCN-O0-NEXT: S_NOP 0
-    ; GCN-O0-NEXT: S_NOP 0
-    ; GCN-O0-NEXT: $sgpr2 = S_ADD_U32 $sgpr1, 0, implicit-def $scc
-    ; GCN-O0-NEXT: S_WAITCNT_DEPCTR 65534
-    ; GCN-O0-NEXT: S_ENDPGM 0
-    ;
-    ; GCN-O2-LABEL: name: hazard_gap2
-    ; GCN-O2: $vgpr1, $vcc_lo = V_ADDC_U32_e64 0, $vgpr1, $sgpr0, 0, implicit $exec
-    ; GCN-O2-NEXT: $sgpr1 = S_CSELECT_B32 -1, 0, implicit $scc
-    ; GCN-O2-NEXT: S_NOP 0
-    ; GCN-O2-NEXT: S_NOP 0
-    ; GCN-O2-NEXT: S_NOP 0
-    ; GCN-O2-NEXT: S_NOP 0
-    ; GCN-O2-NEXT: S_NOP 0
-    ; GCN-O2-NEXT: S_NOP 0
-    ; GCN-O2-NEXT: S_NOP 0
-    ; GCN-O2-NEXT: S_NOP 0
-    ; GCN-O2-NEXT: S_WAITCNT_DEPCTR 65534
-    ; GCN-O2-NEXT: $sgpr2 = S_ADD_U32 $sgpr1, 0, implicit-def $scc
-    ; GCN-O2-NEXT: S_ENDPGM 0
+    ; GCN-LABEL: name: hazard_gap2
+    ; GCN: $vgpr1, $vcc_lo = V_ADDC_U32_e64 0, $vgpr1, $sgpr0, 0, implicit $exec
+    ; GCN-NEXT: $sgpr1 = S_CSELECT_B32 -1, 0, implicit $scc
+    ; GCN-NEXT: S_NOP 0
+    ; GCN-NEXT: S_NOP 0
+    ; GCN-NEXT: S_NOP 0
+    ; GCN-NEXT: S_NOP 0
+    ; GCN-NEXT: S_NOP 0
+    ; GCN-NEXT: S_NOP 0
+    ; GCN-NEXT: S_NOP 0
+    ; GCN-NEXT: S_NOP 0
+    ; GCN-NEXT: S_WAITCNT_DEPCTR 65534
+    ; GCN-NEXT: $sgpr2 = S_ADD_U32 $sgpr1, 0, implicit-def $scc
+    ; GCN-NEXT: S_ENDPGM 0
     $vgpr1, $vcc_lo = V_ADDC_U32_e64 0, $vgpr1, $sgpr0, 0, implicit $exec
     $sgpr1 = S_CSELECT_B32 -1, 0, implicit $scc
     S_NOP 0
@@ -517,41 +410,19 @@ body:            |
 name:            hazard_gap3
 body:            |
   bb.0:
-    ; GCN-O0-LABEL: name: hazard_gap3
-    ; GCN-O0: $vgpr1, $vcc_lo = V_ADDC_U32_e64 0, $vgpr1, $sgpr0, 0, implicit $exec
-    ; GCN-O0-NEXT: $sgpr1 = S_CSELECT_B32 -1, 0, implicit $scc
-    ; GCN-O0-NEXT: S_WAITCNT_DEPCTR 65534
-    ; GCN-O0-NEXT: $sgpr4 = S_ADD_U32 $sgpr3, 0, implicit-def $scc
-    ; GCN-O0-NEXT: S_WAITCNT_DEPCTR 65534
-    ; GCN-O0-NEXT: $sgpr6 = S_ADD_U32 $sgpr5, 0, implicit-def $scc
-    ; GCN-O0-NEXT: S_WAITCNT_DEPCTR 65534
-    ; GCN-O0-NEXT: $sgpr8 = S_ADD_U32 $sgpr7, 0, implicit-def $scc
-    ; GCN-O0-NEXT: S_WAITCNT_DEPCTR 65534
-    ; GCN-O0-NEXT: $sgpr10 = S_ADD_U32 $sgpr9, 0, implicit-def $scc
-    ; GCN-O0-NEXT: S_WAITCNT_DEPCTR 65534
-    ; GCN-O0-NEXT: $sgpr12 = S_ADD_U32 $sgpr11, 0, implicit-def $scc
-    ; GCN-O0-NEXT: S_WAITCNT_DEPCTR 65534
-    ; GCN-O0-NEXT: $sgpr14 = S_ADD_U32 $sgpr13, 0, implicit-def $scc
-    ; GCN-O0-NEXT: S_WAITCNT_DEPCTR 65534
-    ; GCN-O0-NEXT: $sgpr16 = S_ADD_U32 $sgpr15, 0, implicit-def $scc
-    ; GCN-O0-NEXT: S_WAITCNT_DEPCTR 65534
-    ; GCN-O0-NEXT: $sgpr2 = S_ADD_U32 $sgpr1, 0, implicit-def $scc
-    ; GCN-O0-NEXT: S_WAITCNT_DEPCTR 65534
-    ; GCN-O0-NEXT: S_ENDPGM 0
-    ;
-    ; GCN-O2-LABEL: name: hazard_gap3
-    ; GCN-O2: $vgpr1, $vcc_lo = V_ADDC_U32_e64 0, $vgpr1, $sgpr0, 0, implicit $exec
-    ; GCN-O2-NEXT: $sgpr1 = S_CSELECT_B32 -1, 0, implicit $scc
-    ; GCN-O2-NEXT: $sgpr4 = S_ADD_U32 $sgpr3, 0, implicit-def $scc
-    ; GCN-O2-NEXT: $sgpr6 = S_ADD_U32 $sgpr5, 0, implicit-def $scc
-    ; GCN-O2-NEXT: $sgpr8 = S_ADD_U32 $sgpr7, 0, implicit-def $scc
-    ; GCN-O2-NEXT: $sgpr10 = S_ADD_U32 $sgpr9, 0, implicit-def $scc
-    ; GCN-O2-NEXT: $sgpr12 = S_ADD_U32 $sgpr11, 0, implicit-def $scc
-    ; GCN-O2-NEXT: $sgpr14 = S_ADD_U32 $sgpr13, 0, implicit-def $scc
-    ; GCN-O2-NEXT: $sgpr16 = S_ADD_U32 $sgpr15, 0, implicit-def $scc
-    ; GCN-O2-NEXT: S_WAITCNT_DEPCTR 65534
-    ; GCN-O2-NEXT: $sgpr2 = S_ADD_U32 $sgpr1, 0, implicit-def $scc
-    ; GCN-O2-NEXT: S_ENDPGM 0
+    ; GCN-LABEL: name: hazard_gap3
+    ; GCN: $vgpr1, $vcc_lo = V_ADDC_U32_e64 0, $vgpr1, $sgpr0, 0, implicit $exec
+    ; GCN-NEXT: $sgpr1 = S_CSELECT_B32 -1, 0, implicit $scc
+    ; GCN-NEXT: $sgpr4 = S_ADD_U32 $sgpr3, 0, implicit-def $scc
+    ; GCN-NEXT: $sgpr6 = S_ADD_U32 $sgpr5, 0, implicit-def $scc
+    ; GCN-NEXT: $sgpr8 = S_ADD_U32 $sgpr7, 0, implicit-def $scc
+    ; GCN-NEXT: $sgpr10 = S_ADD_U32 $sgpr9, 0, implicit-def $scc
+    ; GCN-NEXT: $sgpr12 = S_ADD_U32 $sgpr11, 0, implicit-def $scc
+    ; GCN-NEXT: $sgpr14 = S_ADD_U32 $sgpr13, 0, implicit-def $scc
+    ; GCN-NEXT: $sgpr16 = S_ADD_U32 $sgpr15, 0, implicit-def $scc
+    ; GCN-NEXT: S_WAITCNT_DEPCTR 65534
+    ; GCN-NEXT: $sgpr2 = S_ADD_U32 $sgpr1, 0, implicit-def $scc
+    ; GCN-NEXT: S_ENDPGM 0
     $vgpr1, $vcc_lo = V_ADDC_U32_e64 0, $vgpr1, $sgpr0, 0, implicit $exec
     $sgpr1 = S_CSELECT_B32 -1, 0, implicit $scc
     $sgpr4 = S_ADD_U32 $sgpr3, 0, implicit-def $scc
@@ -569,49 +440,22 @@ body:            |
 name:            hazard_gap4_no_hazard
 body:            |
   bb.0:
-    ; GCN-O0-LABEL: name: hazard_gap4_no_hazard
-    ; GCN-O0: $vgpr1, $vcc_lo = V_ADDC_U32_e64 0, $vgpr1, $sgpr0, 0, implicit $exec
-    ; GCN-O0-NEXT: $sgpr1 = S_CSELECT_B32 -1, 0, implicit $scc
-    ; GCN-O0-NEXT: S_WAITCNT_DEPCTR 65534
-    ; GCN-O0-NEXT: $sgpr4 = S_ADD_U32 $sgpr3, 0, implicit-def $scc
-    ; GCN-O0-NEXT: S_WAITCNT_DEPCTR 65534
-    ; GCN-O0-NEXT: $sgpr6 = S_ADD_U32 $sgpr5, 0, implicit-def $scc
-    ; GCN-O0-NEXT: S_WAITCNT_DEPCTR 65534
-    ; GCN-O0-NEXT: $sgpr8 = S_ADD_U32 $sgpr7, 0, implicit-def $scc
-    ; GCN-O0-NEXT: S_WAITCNT_DEPCTR 65534
-    ; GCN-O0-NEXT: $sgpr10 = S_ADD_U32 $sgpr9, 0, implicit-def $scc
-    ; GCN-O0-NEXT: S_WAITCNT_DEPCTR 65534
-    ; GCN-O0-NEXT: $sgpr12 = S_ADD_U32 $sgpr11, 0, implicit-def $scc
-    ; GCN-O0-NEXT: S_WAITCNT_DEPCTR 65534
-    ; GCN-O0-NEXT: $sgpr14 = S_ADD_U32 $sgpr13, 0, implicit-def $scc
-    ; GCN-O0-NEXT: S_WAITCNT_DEPCTR 65534
-    ; GCN-O0-NEXT: $sgpr16 = S_ADD_U32 $sgpr15, 0, implicit-def $scc
-    ; GCN-O0-NEXT: S_WAITCNT_DEPCTR 65534
-    ; GCN-O0-NEXT: $sgpr18 = S_ADD_U32 $sgpr17, 0, implicit-def $scc
-    ; GCN-O0-NEXT: S_WAITCNT_DEPCTR 65534
-    ; GCN-O0-NEXT: $sgpr20 = S_ADD_U32 $sgpr19, 0, implicit-def $scc
-    ; GCN-O0-NEXT: S_WAITCNT_DEPCTR 65534
-    ; GCN-O0-NEXT: $sgpr22 = S_ADD_U32 $sgpr21, 0, implicit-def $scc
-    ; GCN-O0-NEXT: S_WAITCNT_DEPCTR 65534
-    ; GCN-O0-NEXT: $sgpr2 = S_ADD_U32 $sgpr1, 0, implicit-def $scc
-    ; GCN-O0-NEXT: S_WAITCNT_DEPCTR 65534
-    ; GCN-O0-NEXT: S_ENDPGM 0
-    ;
-    ; GCN-O2-LABEL: name: hazard_gap4_no_hazard
-    ; GCN-O2: $vgpr1, $vcc_lo = V_ADDC_U32_e64 0, $vgpr1, $sgpr0, 0, implicit $exec
-    ; GCN-O2-NEXT: $sgpr1 = S_CSELECT_B32 -1, 0, implicit $scc
-    ; GCN-O2-NEXT: $sgpr4 = S_ADD_U32 $sgpr3, 0, implicit-def $scc
-    ; GCN-O2-NEXT: $sgpr6 = S_ADD_U32 $sgpr5, 0, implicit-def $scc
-    ; GCN-O2-NEXT: $sgpr8 = S_ADD_U32 $sgpr7, 0, implicit-def $scc
-    ; GCN-O2-NEXT: $sgpr10 = S_ADD_U32 $sgpr9, 0, implicit-def $scc
-    ; GCN-O2-NEXT: $sgpr12 = S_ADD_U32 $sgpr11, 0, implicit-def $scc
-    ; GCN-O2-NEXT: $sgpr14 = S_ADD_U32 $sgpr13, 0, implicit-def $scc
-    ; GCN-O2-NEXT: $sgpr16 = S_ADD_U32 $sgpr15, 0, implicit-def $scc
-    ; GCN-O2-NEXT: $sgpr18 = S_ADD_U32 $sgpr17, 0, implicit-def $scc
-    ; GCN-O2-NEXT: $sgpr20 = S_ADD_U32 $sgpr19, 0, implicit-def $scc
-    ; GCN-O2-NEXT: $sgpr22 = S_ADD_U32 $sgpr21, 0, implicit-def $scc
-    ; GCN-O2-NEXT: $sgpr2 = S_ADD_U32 $sgpr1, 0, implicit-def $scc
-    ; GCN-O2-NEXT: S_ENDPGM 0
+    ; GCN-LABEL: name: hazard_gap4_no_hazard
+    ; GCN: $vgpr1, $vcc_lo = V_ADDC_U32_e64 0, $vgpr1, $sgpr0, 0, implicit $exec
+    ; GCN-NEXT: $sgpr1 = S_CSELECT_B32 -1, 0, implicit $scc
+    ; GCN-NEXT: $sgpr4 = S_ADD_U32 $sgpr3, 0, implicit-def $scc
+    ; GCN-NEXT: $sgpr6 = S_ADD_U32 $sgpr5, 0, implicit-def $scc
+    ; GCN-NEXT: $sgpr8 = S_ADD_U32 $sgpr7, 0, implicit-def $scc
+    ; GCN-NEXT: $sgpr10 = S_ADD_U32 $sgpr9, 0, implicit-def $scc
+    ; GCN-NEXT: $sgpr12 = S_ADD_U32 $sgpr11, 0, implicit-def $scc
+    ; GCN-NEXT: $sgpr14 = S_ADD_U32 $sgpr13, 0, implicit-def $scc
+    ; GCN-NEXT: $sgpr16 = S_ADD_U32 $sgpr15, 0, implicit-def $scc
+    ; GCN-NEXT: $sgpr18 = S_ADD_U32 $sgpr17, 0, implicit-def $scc
+    ; GCN-NEXT: $sgpr20 = S_ADD_U32 $sgpr19, 0, implicit-def $scc
+    ; GCN-NEXT: $sgpr22 = S_ADD_U32 $sgpr21, 0, implicit-def $scc
+    ; GCN-NEXT: S_WAITCNT_DEPCTR 65534
+    ; GCN-NEXT: $sgpr2 = S_ADD_U32 $sgpr1, 0, implicit-def $scc
+    ; GCN-NEXT: S_ENDPGM 0
     $vgpr1, $vcc_lo = V_ADDC_U32_e64 0, $vgpr1, $sgpr0, 0, implicit $exec
     $sgpr1 = S_CSELECT_B32 -1, 0, implicit $scc
     $sgpr4 = S_ADD_U32 $sgpr3, 0, implicit-def $scc
@@ -632,19 +476,11 @@ body:            |
 name:            hazard_valu_write1_no_hazard
 body:            |
   bb.0:
-    ; GCN-O0-LABEL: name: hazard_valu_write1_no_hazard
-    ; GCN-O0: $sgpr1 = V_CMP_EQ_F32_e64 0, $vgpr0, 0, $vgpr1, 1, implicit $mode, implicit $exec
-    ; GCN-O0-NEXT: $sgpr1 = S_CSELECT_B32 -1, 0, implicit $scc
-    ; GCN-O0-NEXT: S_WAITCNT_DEPCTR 65534
-    ; GCN-O0-NEXT: $sgpr2 = S_ADD_U32 $sgpr1, 0, implicit-def $scc
-    ; GCN-O0-NEXT: S_WAITCNT_DEPCTR 65534
-    ; GCN-O0-NEXT: S_ENDPGM 0
-    ;
-    ; GCN-O2-LABEL: name: hazard_valu_write1_no_hazard
-    ; GCN-O2: $sgpr1 = V_CMP_EQ_F32_e64 0, $vgpr0, 0, $vgpr1, 1, implicit $mode, implicit $exec
-    ; GCN-O2-NEXT: $sgpr1 = S_CSELECT_B32 -1, 0, implicit $scc
-    ; GCN-O2-NEXT: $sgpr2 = S_ADD_U32 $sgpr1, 0, implicit-def $scc
-    ; GCN-O2-NEXT: S_ENDPGM 0
+    ; GCN-LABEL: name: hazard_valu_write1_no_hazard
+    ; GCN: $sgpr1 = V_CMP_EQ_F32_e64 0, $vgpr0, 0, $vgpr1, 1, implicit $mode, implicit $exec
+    ; GCN-NEXT: $sgpr1 = S_CSELECT_B32 -1, 0, implicit $scc
+    ; GCN-NEXT: $sgpr2 = S_ADD_U32 $sgpr1, 0, implicit-def $scc
+    ; GCN-NEXT: S_ENDPGM 0
     $sgpr1 = V_CMP_EQ_F32_e64 0, $vgpr0, 0, $vgpr1, 1, implicit $mode, implicit $exec
     $sgpr1 = S_CSELECT_B32 -1, 0, implicit $scc
     $sgpr2 = S_ADD_U32 $sgpr1, 0, implicit-def $scc
@@ -655,19 +491,11 @@ body:            |
 name:            hazard_post_order1
 body:            |
   bb.0:
-    ; GCN-O0-LABEL: name: hazard_post_order1
-    ; GCN-O0: $sgpr0_sgpr1 = S_GETPC_B64
-    ; GCN-O0-NEXT: S_WAITCNT_DEPCTR 65534
-    ; GCN-O0-NEXT: $sgpr3 = S_ADD_U32 $sgpr0, 0, implicit-def $scc
-    ; GCN-O0-NEXT: S_WAITCNT_DEPCTR 65534
-    ; GCN-O0-NEXT: $vgpr1 = V_CNDMASK_B32_e64 0, $vgpr1, 0, $vgpr2, $sgpr0, implicit $exec
-    ; GCN-O0-NEXT: S_ENDPGM 0
-    ;
-    ; GCN-O2-LABEL: name: hazard_post_order1
-    ; GCN-O2: $sgpr0_sgpr1 = S_GETPC_B64
-    ; GCN-O2-NEXT: $sgpr3 = S_ADD_U32 $sgpr0, 0, implicit-def $scc
-    ; GCN-O2-NEXT: $vgpr1 = V_CNDMASK_B32_e64 0, $vgpr1, 0, $vgpr2, $sgpr0, implicit $exec
-    ; GCN-O2-NEXT: S_ENDPGM 0
+    ; GCN-LABEL: name: hazard_post_order1
+    ; GCN: $sgpr0_sgpr1 = S_GETPC_B64
+    ; GCN-NEXT: $sgpr3 = S_ADD_U32 $sgpr0, 0, implicit-def $scc
+    ; GCN-NEXT: $vgpr1 = V_CNDMASK_B32_e64 0, $vgpr1, 0, $vgpr2, $sgpr0, implicit $exec
+    ; GCN-NEXT: S_ENDPGM 0
     $sgpr0_sgpr1 = S_GETPC_B64
     $sgpr3 = S_ADD_U32 $sgpr0, 0, implicit-def $scc
     $vgpr1 = V_CNDMASK_B32_e64 0, $vgpr1, 0, $vgpr2, $sgpr0, implicit $exec
@@ -677,31 +505,17 @@ body:            |
 ---
 name:            hazard_post_order2
 body:            |
-  ; GCN-O0-LABEL: name: hazard_post_order2
-  ; GCN-O0: bb.0:
-  ; GCN-O0-NEXT:   successors: %bb.1(0x80000000)
-  ; GCN-O0-NEXT: {{  $}}
-  ; GCN-O0-NEXT:   $sgpr0_sgpr1 = S_GETPC_B64
-  ; GCN-O0-NEXT:   S_WAITCNT_DEPCTR 65534
-  ; GCN-O0-NEXT:   $sgpr3 = S_ADD_U32 $sgpr0, 0, implicit-def $scc
-  ; GCN-O0-NEXT:   S_WAITCNT_DEPCTR 65534
-  ; GCN-O0-NEXT:   S_BRANCH %bb.1
-  ; GCN-O0-NEXT: {{  $}}
-  ; GCN-O0-NEXT: bb.1:
-  ; GCN-O0-NEXT:   $vgpr1 = V_CNDMASK_B32_e64 0, $vgpr1, 0, $vgpr2, $sgpr0, implicit $exec
-  ; GCN-O0-NEXT:   S_ENDPGM 0
-  ;
-  ; GCN-O2-LABEL: name: hazard_post_order2
-  ; GCN-O2: bb.0:
-  ; GCN-O2-NEXT:   successors: %bb.1(0x80000000)
-  ; GCN-O2-NEXT: {{  $}}
-  ; GCN-O2-NEXT:   $sgpr0_sgpr1 = S_GETPC_B64
-  ; GCN-O2-NEXT:   $sgpr3 = S_ADD_U32 $sgpr0, 0, implicit-def $scc
-  ; GCN-O2-NEXT:   S_BRANCH %bb.1
-  ; GCN-O2-NEXT: {{  $}}
-  ; GCN-O2-NEXT: bb.1:
-  ; GCN-O2-NEXT:   $vgpr1 = V_CNDMASK_B32_e64 0, $vgpr1, 0, $vgpr2, $sgpr0, implicit $exec
-  ; GCN-O2-NEXT:   S_ENDPGM 0
+  ; GCN-LABEL: name: hazard_post_order2
+  ; GCN: bb.0:
+  ; GCN-NEXT:   successors: %bb.1(0x80000000)
+  ; GCN-NEXT: {{  $}}
+  ; GCN-NEXT:   $sgpr0_sgpr1 = S_GETPC_B64
+  ; GCN-NEXT:   $sgpr3 = S_ADD_U32 $sgpr0, 0, implicit-def $scc
+  ; GCN-NEXT:   S_BRANCH %bb.1
+  ; GCN-NEXT: {{  $}}
+  ; GCN-NEXT: bb.1:
+  ; GCN-NEXT:   $vgpr1 = V_CNDMASK_B32_e64 0, $vgpr1, 0, $vgpr2, $sgpr0, implicit $exec
+  ; GCN-NEXT:   S_ENDPGM 0
   bb.0:
     $sgpr0_sgpr1 = S_GETPC_B64
     $sgpr3 = S_ADD_U32 $sgpr0, 0, implicit-def $scc
@@ -715,50 +529,27 @@ body:            |
 ---
 name:            hazard_post_order_cycle
 body:            |
-  ; GCN-O0-LABEL: name: hazard_post_order_cycle
-  ; GCN-O0: bb.0:
-  ; GCN-O0-NEXT:   successors: %bb.1(0x80000000)
-  ; GCN-O0-NEXT: {{  $}}
-  ; GCN-O0-NEXT:   S_NOP 0
-  ; GCN-O0-NEXT: {{  $}}
-  ; GCN-O0-NEXT: bb.1:
-  ; GCN-O0-NEXT:   successors: %bb.2(0x80000000)
-  ; GCN-O0-NEXT: {{  $}}
-  ; GCN-O0-NEXT:   $sgpr0_sgpr1 = S_GETPC_B64
-  ; GCN-O0-NEXT:   S_WAITCNT_DEPCTR 65534
-  ; GCN-O0-NEXT:   $sgpr3 = S_ADD_U32 $sgpr0, 0, implicit-def $scc
-  ; GCN-O0-NEXT:   S_WAITCNT_DEPCTR 65534
-  ; GCN-O0-NEXT: {{  $}}
-  ; GCN-O0-NEXT: bb.2:
-  ; GCN-O0-NEXT:   successors: %bb.1(0x40000000), %bb.3(0x40000000)
-  ; GCN-O0-NEXT: {{  $}}
-  ; GCN-O0-NEXT:   $vgpr1 = V_CNDMASK_B32_e64 0, $vgpr1, 0, $vgpr2, $sgpr0, implicit $exec
-  ; GCN-O0-NEXT:   S_CBRANCH_SCC0 %bb.1, implicit $scc
-  ; GCN-O0-NEXT: {{  $}}
-  ; GCN-O0-NEXT: bb.3:
-  ; GCN-O0-NEXT:   S_ENDPGM 0
-  ;
-  ; GCN-O2-LABEL: name: hazard_post_order_cycle
-  ; GCN-O2: bb.0:
-  ; GCN-O2-NEXT:   successors: %bb.1(0x80000000)
-  ; GCN-O2-NEXT: {{  $}}
-  ; GCN-O2-NEXT:   S_NOP 0
-  ; GCN-O2-NEXT: {{  $}}
-  ; GCN-O2-NEXT: bb.1:
-  ; GCN-O2-NEXT:   successors: %bb.2(0x80000000)
-  ; GCN-O2-NEXT: {{  $}}
-  ; GCN-O2-NEXT:   $sgpr0_sgpr1 = S_GETPC_B64
-  ; GCN-O2-NEXT:   S_WAITCNT_DEPCTR 65534
-  ; GCN-O2-NEXT:   $sgpr3 = S_ADD_U32 $sgpr0, 0, implicit-def $scc
-  ; GCN-O2-NEXT: {{  $}}
-  ; GCN-O2-NEXT: bb.2:
-  ; GCN-O2-NEXT:   successors: %bb.1(0x40000000), %bb.3(0x40000000)
-  ; GCN-O2-NEXT: {{  $}}
-  ; GCN-O2-NEXT:   $vgpr1 = V_CNDMASK_B32_e64 0, $vgpr1, 0, $vgpr2, $sgpr0, implicit $exec
-  ; GCN-O2-NEXT:   S_CBRANCH_SCC0 %bb.1, implicit $scc
-  ; GCN-O2-NEXT: {{  $}}
-  ; GCN-O2-NEXT: bb.3:
-  ; GCN-O2-NEXT:   S_ENDPGM 0
+  ; GCN-LABEL: name: hazard_post_order_cycle
+  ; GCN: bb.0:
+  ; GCN-NEXT:   successors: %bb.1(0x80000000)
+  ; GCN-NEXT: {{  $}}
+  ; GCN-NEXT:   S_NOP 0
+  ; GCN-NEXT: {{  $}}
+  ; GCN-NEXT: bb.1:
+  ; GCN-NEXT:   successors: %bb.2(0x80000000)
+  ; GCN-NEXT: {{  $}}
+  ; GCN-NEXT:   $sgpr0_sgpr1 = S_GETPC_B64
+  ; GCN-NEXT:   S_WAITCNT_DEPCTR 65534
+  ; GCN-NEXT:   $sgpr3 = S_ADD_U32 $sgpr0, 0, implicit-def $scc
+  ; GCN-NEXT: {{  $}}
+  ; GCN-NEXT: bb.2:
+  ; GCN-NEXT:   successors: %bb.1(0x40000000), %bb.3(0x40000000)
+  ; GCN-NEXT: {{  $}}
+  ; GCN-NEXT:   $vgpr1 = V_CNDMASK_B32_e64 0, $vgpr1, 0, $vgpr2, $sgpr0, implicit $exec
+  ; GCN-NEXT:   S_CBRANCH_SCC0 %bb.1, implicit $scc
+  ; GCN-NEXT: {{  $}}
+  ; GCN-NEXT: bb.3:
+  ; GCN-NEXT:   S_ENDPGM 0
   bb.0:
     S_NOP 0
 
@@ -779,84 +570,485 @@ name:            hazard_calls
 frameInfo:
   hasCalls: true
 body:            |
-  ; GCN-O0-LABEL: name: hazard_calls
-  ; GCN-O0: bb.0:
-  ; GCN-O0-NEXT:   $sgpr16 = S_MOV_B32 0
-  ; GCN-O0-NEXT:   S_WAITCNT_DEPCTR 65534
-  ; GCN-O0-NEXT:   S_SETPC_B64 $sgpr0_sgpr1
-  ; GCN-O0-NEXT: {{  $}}
-  ; GCN-O0-NEXT: bb.1:
-  ; GCN-O0-NEXT:   $sgpr18 = S_MOV_B32 0
-  ; GCN-O0-NEXT:   S_WAITCNT_DEPCTR 65534
-  ; GCN-O0-NEXT:   S_SETPC_B64_return $sgpr0_sgpr1
-  ; GCN-O0-NEXT: {{  $}}
-  ; GCN-O0-NEXT: bb.2:
-  ; GCN-O0-NEXT:   successors: %bb.3(0x80000000)
-  ; GCN-O0-NEXT: {{  $}}
-  ; GCN-O0-NEXT:   $sgpr20 = S_MOV_B32 0
-  ; GCN-O0-NEXT:   S_WAITCNT_DEPCTR 65534
-  ; GCN-O0-NEXT:   $sgpr4_sgpr5 = S_SWAPPC_B64 $sgpr2_sgpr3
-  ; GCN-O0-NEXT:   S_WAITCNT_DEPCTR 65534
-  ; GCN-O0-NEXT:   $sgpr4 = S_ADD_U32 $sgpr4, 0, implicit-def $scc
-  ; GCN-O0-NEXT:   S_WAITCNT_DEPCTR 65534
-  ; GCN-O0-NEXT: {{  $}}
-  ; GCN-O0-NEXT: bb.3:
-  ; GCN-O0-NEXT:   successors: %bb.4(0x80000000)
-  ; GCN-O0-NEXT: {{  $}}
-  ; GCN-O0-NEXT:   $sgpr8_sgpr9 = S_CALL_B64 0
-  ; GCN-O0-NEXT:   S_WAITCNT_DEPCTR 65534
-  ; GCN-O0-NEXT: {{  $}}
-  ; GCN-O0-NEXT: bb.4:
-  ; GCN-O0-NEXT:   $sgpr22 = S_MOV_B32 $sgpr8
-  ; GCN-O0-NEXT:   S_WAITCNT_DEPCTR 65534
-  ; GCN-O0-NEXT:   S_ENDPGM 0
+  ; NOBC-LABEL: name: hazard_calls
+  ; NOBC: bb.0:
+  ; NOBC-NEXT:   successors: %bb.2(0x40000000), %bb.1(0x40000000)
+  ; NOBC-NEXT: {{  $}}
+  ; NOBC-NEXT:   $vgpr0 = V_WRITELANE_B32 0, $sgpr4, $vgpr0
+  ; NOBC-NEXT:   $vgpr0 = V_WRITELANE_B32 1, $sgpr8, $vgpr0
+  ; NOBC-NEXT:   $vgpr0 = V_WRITELANE_B32 2, $sgpr16, $vgpr0
+  ; NOBC-NEXT:   $vgpr0 = V_WRITELANE_B32 3, $sgpr18, $vgpr0
+  ; NOBC-NEXT:   $vgpr0 = V_WRITELANE_B32 4, $sgpr20, $vgpr0
+  ; NOBC-NEXT:   $vgpr0 = V_WRITELANE_B32 5, $sgpr22, $vgpr0
+  ; NOBC-NEXT:   S_CBRANCH_SCC0 %bb.2, implicit $scc
+  ; NOBC-NEXT:   S_BRANCH %bb.1
+  ; NOBC-NEXT: {{  $}}
+  ; NOBC-NEXT: bb.1:
+  ; NOBC-NEXT:   successors: %bb.3(0x40000000), %bb.4(0x40000000)
+  ; NOBC-NEXT: {{  $}}
+  ; NOBC-NEXT:   S_CBRANCH_SCC0 %bb.3, implicit $scc
+  ; NOBC-NEXT:   S_BRANCH %bb.4
+  ; NOBC-NEXT: {{  $}}
+  ; NOBC-NEXT: bb.2:
+  ; NOBC-NEXT:   $sgpr16 = S_MOV_B32 0
+  ; NOBC-NEXT:   S_WAITCNT_DEPCTR 65534
+  ; NOBC-NEXT:   S_SETPC_B64 $sgpr0_sgpr1
+  ; NOBC-NEXT: {{  $}}
+  ; NOBC-NEXT: bb.3:
+  ; NOBC-NEXT:   $sgpr18 = S_MOV_B32 0
+  ; NOBC-NEXT:   S_WAITCNT_DEPCTR 65534
+  ; NOBC-NEXT:   S_SETPC_B64_return $sgpr0_sgpr1
+  ; NOBC-NEXT: {{  $}}
+  ; NOBC-NEXT: bb.4:
+  ; NOBC-NEXT:   successors: %bb.5(0x80000000)
+  ; NOBC-NEXT: {{  $}}
+  ; NOBC-NEXT:   $vcc_lo = S_MOV_B32 0
+  ; NOBC-NEXT:   $sgpr20 = S_MOV_B32 0
+  ; NOBC-NEXT:   S_WAITCNT_DEPCTR 65534
+  ; NOBC-NEXT:   $sgpr4_sgpr5 = S_SWAPPC_B64 $sgpr2_sgpr3
+  ; NOBC-NEXT:   S_WAITCNT_DEPCTR 65534
+  ; NOBC-NEXT:   $sgpr4 = S_ADD_U32 $sgpr4, 0, implicit-def $scc
+  ; NOBC-NEXT: {{  $}}
+  ; NOBC-NEXT: bb.5:
+  ; NOBC-NEXT:   successors: %bb.6(0x80000000)
+  ; NOBC-NEXT: {{  $}}
+  ; NOBC-NEXT:   S_WAITCNT_DEPCTR 65534
+  ; NOBC-NEXT:   $sgpr8_sgpr9 = S_CALL_B64 0
+  ; NOBC-NEXT: {{  $}}
+  ; NOBC-NEXT: bb.6:
+  ; NOBC-NEXT:   S_WAITCNT_DEPCTR 65534
+  ; NOBC-NEXT:   $sgpr22 = S_MOV_B32 $sgpr8
+  ; NOBC-NEXT:   S_ENDPGM 0
   ;
-  ; GCN-O2-LABEL: name: hazard_calls
-  ; GCN-O2: bb.0:
-  ; GCN-O2-NEXT:   $sgpr16 = S_MOV_B32 0
-  ; GCN-O2-NEXT:   S_SETPC_B64 $sgpr0_sgpr1
-  ; GCN-O2-NEXT: {{  $}}
-  ; GCN-O2-NEXT: bb.1:
-  ; GCN-O2-NEXT:   $sgpr18 = S_MOV_B32 0
-  ; GCN-O2-NEXT:   S_WAITCNT_DEPCTR 65534
-  ; GCN-O2-NEXT:   S_SETPC_B64_return $sgpr0_sgpr1
-  ; GCN-O2-NEXT: {{  $}}
-  ; GCN-O2-NEXT: bb.2:
-  ; GCN-O2-NEXT:   successors: %bb.3(0x80000000)
-  ; GCN-O2-NEXT: {{  $}}
-  ; GCN-O2-NEXT:   $sgpr20 = S_MOV_B32 0
-  ; GCN-O2-NEXT:   S_WAITCNT_DEPCTR 65534
-  ; GCN-O2-NEXT:   $sgpr4_sgpr5 = S_SWAPPC_B64 $sgpr2_sgpr3
-  ; GCN-O2-NEXT:   S_WAITCNT_DEPCTR 65534
-  ; GCN-O2-NEXT:   $sgpr4 = S_ADD_U32 $sgpr4, 0, implicit-def $scc
-  ; GCN-O2-NEXT: {{  $}}
-  ; GCN-O2-NEXT: bb.3:
-  ; GCN-O2-NEXT:   successors: %bb.4(0x80000000)
-  ; GCN-O2-NEXT: {{  $}}
-  ; GCN-O2-NEXT:   S_WAITCNT_DEPCTR 65534
-  ; GCN-O2-NEXT:   $sgpr8_sgpr9 = S_CALL_B64 0
-  ; GCN-O2-NEXT: {{  $}}
-  ; GCN-O2-NEXT: bb.4:
-  ; GCN-O2-NEXT:   S_WAITCNT_DEPCTR 65534
-  ; GCN-O2-NEXT:   $sgpr22 = S_MOV_B32 $sgpr8
-  ; GCN-O2-NEXT:   S_ENDPGM 0
+  ; BC-LABEL: name: hazard_calls
+  ; BC: bb.0:
+  ; BC-NEXT:   successors: %bb.2(0x40000000), %bb.1(0x40000000)
+  ; BC-NEXT: {{  $}}
+  ; BC-NEXT:   $vgpr0 = V_WRITELANE_B32 0, $sgpr4, $vgpr0
+  ; BC-NEXT:   $vgpr0 = V_WRITELANE_B32 1, $sgpr8, $vgpr0
+  ; BC-NEXT:   $vgpr0 = V_WRITELANE_B32 2, $sgpr16, $vgpr0
+  ; BC-NEXT:   $vgpr0 = V_WRITELANE_B32 3, $sgpr18, $vgpr0
+  ; BC-NEXT:   $vgpr0 = V_WRITELANE_B32 4, $sgpr20, $vgpr0
+  ; BC-NEXT:   $vgpr0 = V_WRITELANE_B32 5, $sgpr22, $vgpr0
+  ; BC-NEXT:   S_CBRANCH_SCC0 %bb.2, implicit $scc
+  ; BC-NEXT:   S_BRANCH %bb.1
+  ; BC-NEXT: {{  $}}
+  ; BC-NEXT: bb.1:
+  ; BC-NEXT:   successors: %bb.3(0x40000000), %bb.4(0x40000000)
+  ; BC-NEXT: {{  $}}
+  ; BC-NEXT:   S_CBRANCH_SCC0 %bb.3, implicit $scc
+  ; BC-NEXT:   S_BRANCH %bb.4
+  ; BC-NEXT: {{  $}}
+  ; BC-NEXT: bb.2:
+  ; BC-NEXT:   $sgpr16 = S_MOV_B32 0
+  ; BC-NEXT:   DS_NOP implicit $m0, implicit $exec
+  ; BC-NEXT:   DS_NOP implicit $m0, implicit $exec
+  ; BC-NEXT:   DS_NOP implicit $m0, implicit $exec
+  ; BC-NEXT:   DS_NOP implicit $m0, implicit $exec
+  ; BC-NEXT:   S_WAITCNT_DEPCTR 65534
+  ; BC-NEXT:   S_SETPC_B64 $sgpr0_sgpr1
+  ; BC-NEXT: {{  $}}
+  ; BC-NEXT: bb.3:
+  ; BC-NEXT:   $sgpr18 = S_MOV_B32 0
+  ; BC-NEXT:   DS_NOP implicit $m0, implicit $exec
+  ; BC-NEXT:   DS_NOP implicit $m0, implicit $exec
+  ; BC-NEXT:   DS_NOP implicit $m0, implicit $exec
+  ; BC-NEXT:   DS_NOP implicit $m0, implicit $exec
+  ; BC-NEXT:   S_WAITCNT_DEPCTR 65534
+  ; BC-NEXT:   S_SETPC_B64_return $sgpr0_sgpr1
+  ; BC-NEXT: {{  $}}
+  ; BC-NEXT: bb.4:
+  ; BC-NEXT:   successors: %bb.5(0x80000000)
+  ; BC-NEXT: {{  $}}
+  ; BC-NEXT:   $vcc_lo = S_MOV_B32 0
+  ; BC-NEXT:   $sgpr20 = S_MOV_B32 0
+  ; BC-NEXT:   DS_NOP implicit $m0, implicit $exec
+  ; BC-NEXT:   DS_NOP implicit $m0, implicit $exec
+  ; BC-NEXT:   DS_NOP implicit $m0, implicit $exec
+  ; BC-NEXT:   DS_NOP implicit $m0, implicit $exec
+  ; BC-NEXT:   S_WAITCNT_DEPCTR 65534
+  ; BC-NEXT:   $sgpr4_sgpr5 = S_SWAPPC_B64 $sgpr2_sgpr3
+  ; BC-NEXT:   $sgpr4 = S_ADD_U32 $sgpr4, 0, implicit-def $scc
+  ; BC-NEXT: {{  $}}
+  ; BC-NEXT: bb.5:
+  ; BC-NEXT:   successors: %bb.6(0x80000000)
+  ; BC-NEXT: {{  $}}
+  ; BC-NEXT:   $sgpr8_sgpr9 = S_CALL_B64 0
+  ; BC-NEXT: {{  $}}
+  ; BC-NEXT: bb.6:
+  ; BC-NEXT:   $sgpr22 = S_MOV_B32 $sgpr8
+  ; BC-NEXT:   S_ENDPGM 0
   bb.0:
+    $vgpr0 = V_WRITELANE_B32 0, $sgpr4, $vgpr0
+    $vgpr0 = V_WRITELANE_B32 1, $sgpr8, $vgpr0
+    $vgpr0 = V_WRITELANE_B32 2, $sgpr16, $vgpr0
+    $vgpr0 = V_WRITELANE_B32 3, $sgpr18, $vgpr0
+    $vgpr0 = V_WRITELANE_B32 4, $sgpr20, $vgpr0
+    $vgpr0 = V_WRITELANE_B32 5, $sgpr22, $vgpr0
+    S_CBRANCH_SCC0 %bb.2, implicit $scc
+    S_BRANCH %bb.1
+
+  bb.1:
+    S_CBRANCH_SCC0 %bb.3, implicit $scc
+    S_BRANCH %bb.4
+
+  bb.2:
     $sgpr16 = S_MOV_B32 0
     S_SETPC_B64 $sgpr0_sgpr1
 
-  bb.1:
+  bb.3:
     $sgpr18 = S_MOV_B32 0
     S_SETPC_B64_return $sgpr0_sgpr1
 
-  bb.2:
+  bb.4:
+    $vcc_lo = S_MOV_B32 0
     $sgpr20 = S_MOV_B32 0
     $sgpr4_sgpr5 = S_SWAPPC_B64 $sgpr2_sgpr3
     $sgpr4 = S_ADD_U32 $sgpr4, 0, implicit-def $scc
 
-  bb.3:
+  bb.5:
     $sgpr8_sgpr9 = S_CALL_B64 0
 
-  bb.4:
+  bb.6:
     $sgpr22 = S_MOV_B32 $sgpr8
     S_ENDPGM 0
 ...
+
+---
+name:            hazard_callee1
+body:            |
+  bb.0:
+    ; NOBC-LABEL: name: hazard_callee1
+    ; NOBC: $sgpr1 = S_CSELECT_B32 -1, 0, implicit $scc
+    ; NOBC-NEXT: S_WAITCNT_DEPCTR 65534
+    ; NOBC-NEXT: $sgpr2 = S_ADD_U32 $sgpr1, 0, implicit-def $scc
+    ; NOBC-NEXT: S_WAITCNT_DEPCTR 65534
+    ; NOBC-NEXT: S_SETPC_B64_return $sgpr30_sgpr31
+    ;
+    ; BC-LABEL: name: hazard_callee1
+    ; BC: $sgpr1 = S_CSELECT_B32 -1, 0, implicit $scc
+    ; BC-NEXT: $sgpr2 = S_ADD_U32 $sgpr1, 0, implicit-def $scc
+    ; BC-NEXT: S_SETPC_B64_return $sgpr30_sgpr31
+    $sgpr1 = S_CSELECT_B32 -1, 0, implicit $scc
+    $sgpr2 = S_ADD_U32 $sgpr1, 0, implicit-def $scc
+    S_SETPC_B64_return $sgpr30_sgpr31
+...
+
+---
+name:            hazard_callee2
+body:            |
+  bb.0:
+    ; NOBC-LABEL: name: hazard_callee2
+    ; NOBC: $vgpr1, $sgpr0 = V_ADDC_U32_e64 0, $vgpr1, $sgpr0, 0, implicit $exec
+    ; NOBC-NEXT: $sgpr1 = S_CSELECT_B32 -1, 0, implicit $scc
+    ; NOBC-NEXT: S_WAITCNT_DEPCTR 65534
+    ; NOBC-NEXT: $sgpr2 = S_ADD_U32 $sgpr1, 0, implicit-def $scc
+    ; NOBC-NEXT: S_WAITCNT_DEPCTR 65534
+    ; NOBC-NEXT: S_SETPC_B64_return $sgpr30_sgpr31
+    ;
+    ; BC-LABEL: name: hazard_callee2
+    ; BC: $vgpr1, $sgpr0 = V_ADDC_U32_e64 0, $vgpr1, $sgpr0, 0, implicit $exec
+    ; BC-NEXT: $sgpr1 = S_CSELECT_B32 -1, 0, implicit $scc
+    ; BC-NEXT: S_WAITCNT_DEPCTR 65534
+    ; BC-NEXT: $sgpr2 = S_ADD_U32 $sgpr1, 0, implicit-def $scc
+    ; BC-NEXT: DS_NOP implicit $m0, implicit $exec
+    ; BC-NEXT: DS_NOP implicit $m0, implicit $exec
+    ; BC-NEXT: DS_NOP implicit $m0, implicit $exec
+    ; BC-NEXT: DS_NOP implicit $m0, implicit $exec
+    ; BC-NEXT: S_SETPC_B64_return $sgpr30_sgpr31
+    $vgpr1, $sgpr0 = V_ADDC_U32_e64 0, $vgpr1, $sgpr0, 0, implicit $exec
+    $sgpr1 = S_CSELECT_B32 -1, 0, implicit $scc
+    $sgpr2 = S_ADD_U32 $sgpr1, 0, implicit-def $scc
+    S_SETPC_B64_return $sgpr30_sgpr31
+...
+
+---
+name:            hazard_carry_vcc
+body:            |
+  bb.0:
+    ; GCN-LABEL: name: hazard_carry_vcc
+    ; GCN: $vgpr1, $sgpr0 = V_ADDC_U32_e64 0, $vgpr1, $vcc_lo, 0, implicit $exec
+    ; GCN-NEXT: $vgpr0 = V_ADD_CO_U32_e32 $vgpr0, $vgpr1, implicit-def $vcc_lo, implicit $exec
+    ; GCN-NEXT: S_WAITCNT_DEPCTR 65533
+    ; GCN-NEXT: $vgpr1 = V_ADDC_U32_e32 $vgpr2, $vgpr3, implicit-def $vcc_lo, implicit $vcc_lo, implicit $exec
+    ; GCN-NEXT: S_ENDPGM 0
+    $vgpr1, $sgpr0 = V_ADDC_U32_e64 0, $vgpr1, $vcc_lo, 0, implicit $exec
+    $vgpr0 = V_ADD_CO_U32_e32 $vgpr0, $vgpr1, implicit-def $vcc, implicit $exec
+    $vgpr1 = V_ADDC_U32_e32 $vgpr2, $vgpr3, implicit-def $vcc, implicit $vcc, implicit $exec
+    S_ENDPGM 0
+...
+
+---
+name:            hazard_carry_vcc_no_hazard
+body:            |
+  bb.0:
+    ; GCN-LABEL: name: hazard_carry_vcc_no_hazard
+    ; GCN: $vgpr1, $sgpr0 = V_ADDC_U32_e64 0, $vgpr1, $vcc_lo, 0, implicit $exec
+    ; GCN-NEXT: $vgpr0 = V_ADD_CO_U32_e32 $vgpr0, $vgpr1, implicit-def $vcc_lo, implicit $exec
+    ; GCN-NEXT: $sgpr8 = S_MOV_B32 $vcc_lo
+    ; GCN-NEXT: $vgpr1 = V_ADDC_U32_e32 $vgpr2, $vgpr3, implicit-def $vcc_lo, implicit $vcc_lo, implicit $exec
+    ; GCN-NEXT: S_ENDPGM 0
+    $vgpr1, $sgpr0 = V_ADDC_U32_e64 0, $vgpr1, $vcc_lo, 0, implicit $exec
+    $vgpr0 = V_ADD_CO_U32_e32 $vgpr0, $vgpr1, implicit-def $vcc, implicit $exec
+    $sgpr8 = S_MOV_B32 $vcc_lo
+    $vgpr1 = V_ADDC_U32_e32 $vgpr2, $vgpr3, implicit-def $vcc, implicit $vcc, implicit $exec
+    S_ENDPGM 0
+...
+
+---
+name:            hazard_carry_sgpr
+body:            |
+  bb.0:
+    ; GCN-LABEL: name: hazard_carry_sgpr
+    ; GCN: $vgpr1, $sgpr0 = V_ADDC_U32_e64 0, $vgpr1, $sgpr0, 0, implicit $exec
+    ; GCN-NEXT: $vgpr0, $sgpr0 = V_ADD_CO_U32_e64 $vgpr0, $vgpr1, 0, implicit $exec
+    ; GCN-NEXT: S_WAITCNT_DEPCTR 61951
+    ; GCN-NEXT: $vgpr1, $sgpr1 = V_ADDC_U32_e64 $vgpr2, $vgpr3, $sgpr0, 0, implicit $exec
+    ; GCN-NEXT: S_ENDPGM 0
+    $vgpr1, $sgpr0 = V_ADDC_U32_e64 0, $vgpr1, $sgpr0, 0, implicit $exec
+    $vgpr0, $sgpr0 = V_ADD_CO_U32_e64 $vgpr0, $vgpr1, 0, implicit $exec
+    $vgpr1, $sgpr1 = V_ADDC_U32_e64 $vgpr2, $vgpr3, $sgpr0, 0, implicit $exec
+    S_ENDPGM 0
+...
+
+---
+name:            hazard_carry_sgpr_no_hazard1
+body:            |
+  bb.0:
+    ; GCN-LABEL: name: hazard_carry_sgpr_no_hazard1
+    ; GCN: $vgpr1, $sgpr0 = V_ADDC_U32_e64 0, $vgpr1, $sgpr0, 0, implicit $exec
+    ; GCN-NEXT: $vgpr0, $sgpr0 = V_ADD_CO_U32_e64 $vgpr0, $vgpr1, 0, implicit $exec
+    ; GCN-NEXT: $sgpr8 = S_MOV_B32 $sgpr0
+    ; GCN-NEXT: $vgpr1, $sgpr1 = V_ADDC_U32_e64 $vgpr2, $vgpr3, $sgpr0, 0, implicit $exec
+    ; GCN-NEXT: S_ENDPGM 0
+    $vgpr1, $sgpr0 = V_ADDC_U32_e64 0, $vgpr1, $sgpr0, 0, implicit $exec
+    $vgpr0, $sgpr0 = V_ADD_CO_U32_e64 $vgpr0, $vgpr1, 0, implicit $exec
+    $sgpr8 = S_MOV_B32 $sgpr0
+    $vgpr1, $sgpr1 = V_ADDC_U32_e64 $vgpr2, $vgpr3, $sgpr0, 0, implicit $exec
+    S_ENDPGM 0
+...
+
+---
+name:            hazard_carry_sgpr_no_hazard2
+body:            |
+  bb.0:
+    ; GCN-LABEL: name: hazard_carry_sgpr_no_hazard2
+    ; GCN: $vgpr1, $sgpr0 = V_ADDC_U32_e64 0, $vgpr1, $sgpr0, 0, implicit $exec
+    ; GCN-NEXT: $vgpr0, $sgpr0 = V_ADD_CO_U32_e64 $vgpr0, $vgpr1, 0, implicit $exec
+    ; GCN-NEXT: $vgpr3 = BUFFER_LOAD_DWORD_OFFSET $sgpr4_sgpr5_sgpr6_sgpr7, $sgpr8, 0, 0, 0, implicit $exec
+    ; GCN-NEXT: $vgpr1, $sgpr1 = V_ADDC_U32_e64 $vgpr2, $vgpr3, $sgpr0, 0, implicit $exec
+    ; GCN-NEXT: S_ENDPGM 0
+    $vgpr1, $sgpr0 = V_ADDC_U32_e64 0, $vgpr1, $sgpr0, 0, implicit $exec
+    $vgpr0, $sgpr0 = V_ADD_CO_U32_e64 $vgpr0, $vgpr1, 0, implicit $exec
+    $vgpr3 = BUFFER_LOAD_DWORD_OFFSET $sgpr4_sgpr5_sgpr6_sgpr7, $sgpr8, 0, 0, 0, implicit $exec
+    $vgpr1, $sgpr1 = V_ADDC_U32_e64 $vgpr2, $vgpr3, $sgpr0, 0, implicit $exec
+    S_ENDPGM 0
+...
+
+---
+name:            hazard_carry_sgpr_no_hazard3
+body:            |
+  bb.0:
+    ; GCN-LABEL: name: hazard_carry_sgpr_no_hazard3
+    ; GCN: $vgpr1, $sgpr0 = V_ADDC_U32_e64 0, $vgpr1, $sgpr0, 0, implicit $exec
+    ; GCN-NEXT: $vgpr0, $sgpr0 = V_ADD_CO_U32_e64 $vgpr0, $vgpr1, 0, implicit $exec
+    ; GCN-NEXT: $sgpr8 = S_LOAD_DWORD_IMM $sgpr6_sgpr7, 0, 0
+    ; GCN-NEXT: $vgpr1, $sgpr1 = V_ADDC_U32_e64 $vgpr2, $vgpr3, $sgpr0, 0, implicit $exec
+    ; GCN-NEXT: S_ENDPGM 0
+    $vgpr1, $sgpr0 = V_ADDC_U32_e64 0, $vgpr1, $sgpr0, 0, implicit $exec
+    $vgpr0, $sgpr0 = V_ADD_CO_U32_e64 $vgpr0, $vgpr1, 0, implicit $exec
+    $sgpr8 = S_LOAD_DWORD_IMM $sgpr6_sgpr7, 0, 0
+    $vgpr1, $sgpr1 = V_ADDC_U32_e64 $vgpr2, $vgpr3, $sgpr0, 0, implicit $exec
+    S_ENDPGM 0
+...
+
+---
+name:            hazard_cull_vmem
+body:            |
+  bb.0:
+    ; NOMEMC-LABEL: name: hazard_cull_vmem
+    ; NOMEMC: $vgpr1, $sgpr0 = V_ADDC_U32_e64 0, $vgpr1, $sgpr0, 0, implicit $exec
+    ; NOMEMC-NEXT: $vgpr3 = BUFFER_LOAD_DWORD_OFFSET $sgpr4_sgpr5_sgpr6_sgpr7, $sgpr8, 0, 0, 0, implicit $exec
+    ; NOMEMC-NEXT: S_WAIT_LOADCNT 0
+    ; NOMEMC-NEXT: $sgpr0_sgpr1 = S_GETPC_B64
+    ; NOMEMC-NEXT: S_WAITCNT_DEPCTR 65534
+    ; NOMEMC-NEXT: $sgpr3 = S_ADD_U32 $sgpr0, 0, implicit-def $scc
+    ; NOMEMC-NEXT: S_ENDPGM 0
+    ;
+    ; MEMC-LABEL: name: hazard_cull_vmem
+    ; MEMC: $vgpr1, $sgpr0 = V_ADDC_U32_e64 0, $vgpr1, $sgpr0, 0, implicit $exec
+    ; MEMC-NEXT: $vgpr3 = BUFFER_LOAD_DWORD_OFFSET $sgpr4_sgpr5_sgpr6_sgpr7, $sgpr8, 0, 0, 0, implicit $exec
+    ; MEMC-NEXT: DS_NOP implicit $m0, implicit $exec
+    ; MEMC-NEXT: DS_NOP implicit $m0, implicit $exec
+    ; MEMC-NEXT: DS_NOP implicit $m0, implicit $exec
+    ; MEMC-NEXT: DS_NOP implicit $m0, implicit $exec
+    ; MEMC-NEXT: S_WAIT_LOADCNT 0
+    ; MEMC-NEXT: $sgpr0_sgpr1 = S_GETPC_B64
+    ; MEMC-NEXT: $sgpr3 = S_ADD_U32 $sgpr0, 0, implicit-def $scc
+    ; MEMC-NEXT: S_ENDPGM 0
+    $vgpr1, $sgpr0 = V_ADDC_U32_e64 0, $vgpr1, $sgpr0, 0, implicit $exec
+    $vgpr3 = BUFFER_LOAD_DWORD_OFFSET $sgpr4_sgpr5_sgpr6_sgpr7, $sgpr8, 0, 0, 0, implicit $exec
+    S_WAIT_LOADCNT 0
+    $sgpr0_sgpr1 = S_GETPC_B64
+    $sgpr3 = S_ADD_U32 $sgpr0, 0, implicit-def $scc
+    S_ENDPGM 0
+...
+
+---
+name:            hazard_cull_sample
+body:            |
+  bb.0:
+    ; NOMEMC-LABEL: name: hazard_cull_sample
+    ; NOMEMC: $vgpr1, $sgpr0 = V_ADDC_U32_e64 0, $vgpr1, $sgpr0, 0, implicit $exec
+    ; NOMEMC-NEXT: $vgpr10 = IMAGE_SAMPLE_LZ_V1_V2_gfx12 $vgpr3, $vgpr4, $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4_sgpr5_sgpr6_sgpr7, $sgpr8_sgpr9_sgpr10_sgpr11, 1, 1, -1, 0, 0, 0, 0, 0, 0, implicit $exec :: (load (s128))
+    ; NOMEMC-NEXT: S_WAIT_SAMPLECNT 0
+    ; NOMEMC-NEXT: $sgpr0_sgpr1 = S_GETPC_B64
+    ; NOMEMC-NEXT: S_WAITCNT_DEPCTR 65534
+    ; NOMEMC-NEXT: $sgpr3 = S_ADD_U32 $sgpr0, 0, implicit-def $scc
+    ; NOMEMC-NEXT: S_ENDPGM 0
+    ;
+    ; MEMC-LABEL: name: hazard_cull_sample
+    ; MEMC: $vgpr1, $sgpr0 = V_ADDC_U32_e64 0, $vgpr1, $sgpr0, 0, implicit $exec
+    ; MEMC-NEXT: $vgpr10 = IMAGE_SAMPLE_LZ_V1_V2_gfx12 $vgpr3, $vgpr4, $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4_sgpr5_sgpr6_sgpr7, $sgpr8_sgpr9_sgpr10_sgpr11, 1, 1, -1, 0, 0, 0, 0, 0, 0, implicit $exec :: (load (s128))
+    ; MEMC-NEXT: DS_NOP implicit $m0, implicit $exec
+    ; MEMC-NEXT: DS_NOP implicit $m0, implicit $exec
+    ; MEMC-NEXT: DS_NOP implicit $m0, implicit $exec
+    ; MEMC-NEXT: DS_NOP implicit $m0, implicit $exec
+    ; MEMC-NEXT: S_WAIT_SAMPLECNT 0
+    ; MEMC-NEXT: $sgpr0_sgpr1 = S_GETPC_B64
+    ; MEMC-NEXT: $sgpr3 = S_ADD_U32 $sgpr0, 0, implicit-def $scc
+    ; MEMC-NEXT: S_ENDPGM 0
+    $vgpr1, $sgpr0 = V_ADDC_U32_e64 0, $vgpr1, $sgpr0, 0, implicit $exec
+    $vgpr10 = IMAGE_SAMPLE_LZ_V1_V2_gfx12 $vgpr3, $vgpr4, $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4_sgpr5_sgpr6_sgpr7, $sgpr8_sgpr9_sgpr10_sgpr11, 1, 1, -1, 0, 0, 0, 0, 0, 0, implicit $exec :: (load (s128))
+    S_WAIT_SAMPLECNT 0
+    $sgpr0_sgpr1 = S_GETPC_B64
+    $sgpr3 = S_ADD_U32 $sgpr0, 0, implicit-def $scc
+    S_ENDPGM 0
+...
+
+---
+name:            hazard_cull_bvh
+body:            |
+  bb.0:
+    ; NOMEMC-LABEL: name: hazard_cull_bvh
+    ; NOMEMC: $vgpr1, $sgpr0 = V_ADDC_U32_e64 0, $vgpr1, $sgpr0, 0, implicit $exec
+    ; NOMEMC-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = IMAGE_BVH_INTERSECT_RAY_sa_gfx11 $vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14, renamable $sgpr0_sgpr1_sgpr2_sgpr3, 0, implicit $exec :: (dereferenceable load (s128), addrspace 7)
+    ; NOMEMC-NEXT: S_WAIT_BVHCNT 0
+    ; NOMEMC-NEXT: $sgpr0_sgpr1 = S_GETPC_B64
+    ; NOMEMC-NEXT: S_WAITCNT_DEPCTR 65534
+    ; NOMEMC-NEXT: $sgpr3 = S_ADD_U32 $sgpr0, 0, implicit-def $scc
+    ; NOMEMC-NEXT: S_ENDPGM 0
+    ;
+    ; MEMC-LABEL: name: hazard_cull_bvh
+    ; MEMC: $vgpr1, $sgpr0 = V_ADDC_U32_e64 0, $vgpr1, $sgpr0, 0, implicit $exec
+    ; MEMC-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = IMAGE_BVH_INTERSECT_RAY_sa_gfx11 $vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14, renamable $sgpr0_sgpr1_sgpr2_sgpr3, 0, implicit $exec :: (dereferenceable load (s128), addrspace 7)
+    ; MEMC-NEXT: DS_NOP implicit $m0, implicit $exec
+    ; MEMC-NEXT: DS_NOP implicit $m0, implicit $exec
+    ; MEMC-NEXT: DS_NOP implicit $m0, implicit $exec
+    ; MEMC-NEXT: DS_NOP implicit $m0, implicit $exec
+    ; MEMC-NEXT: S_WAIT_BVHCNT 0
+    ; MEMC-NEXT: $sgpr0_sgpr1 = S_GETPC_B64
+    ; MEMC-NEXT: $sgpr3 = S_ADD_U32 $sgpr0, 0, implicit-def $scc
+    ; MEMC-NEXT: S_ENDPGM 0
+    $vgpr1, $sgpr0 = V_ADDC_U32_e64 0, $vgpr1, $sgpr0, 0, implicit $exec
+    $vgpr0_vgpr1_vgpr2_vgpr3 = IMAGE_BVH_INTERSECT_RAY_sa_gfx11 $vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14, renamable $sgpr0_sgpr1_sgpr2_sgpr3, 0, implicit $exec :: (dereferenceable load (s128), addrspace 7)
+    S_WAIT_BVHCNT 0
+    $sgpr0_sgpr1 = S_GETPC_B64
+    $sgpr3 = S_ADD_U32 $sgpr0, 0, implicit-def $scc
+    S_ENDPGM 0
+...
+
+---
+name:            hazard_nocull_scratch
+body:            |
+  bb.0:
+    ; GCN-LABEL: name: hazard_nocull_scratch
+    ; GCN: $vgpr1, $sgpr0 = V_ADDC_U32_e64 0, $vgpr1, $sgpr0, 0, implicit $exec
+    ; GCN-NEXT: $vgpr0 = SCRATCH_LOAD_DWORD $vgpr0, 0, 0, implicit $exec, implicit $flat_scr
+    ; GCN-NEXT: S_WAIT_LOADCNT 0
+    ; GCN-NEXT: $sgpr0_sgpr1 = S_GETPC_B64
+    ; GCN-NEXT: S_WAITCNT_DEPCTR 65534
+    ; GCN-NEXT: $sgpr3 = S_ADD_U32 $sgpr0, 0, implicit-def $scc
+    ; GCN-NEXT: S_ENDPGM 0
+    $vgpr1, $sgpr0 = V_ADDC_U32_e64 0, $vgpr1, $sgpr0, 0, implicit $exec
+    $vgpr0 = SCRATCH_LOAD_DWORD $vgpr0, 0, 0, implicit $exec, implicit $flat_scr
+    S_WAIT_LOADCNT 0
+    $sgpr0_sgpr1 = S_GETPC_B64
+    $sgpr3 = S_ADD_U32 $sgpr0, 0, implicit-def $scc
+    S_ENDPGM 0
+...
+
+---
+name:            hazard_cull_global
+body:            |
+  bb.0:
+    ; NOMEMC-LABEL: name: hazard_cull_global
+    ; NOMEMC: $vgpr1, $sgpr0 = V_ADDC_U32_e64 0, $vgpr1, $sgpr0, 0, implicit $exec
+    ; NOMEMC-NEXT: $vgpr0 = GLOBAL_LOAD_DWORD $vgpr0_vgpr1, 0, 0, implicit $exec
+    ; NOMEMC-NEXT: S_WAIT_LOADCNT 0
+    ; NOMEMC-NEXT: $sgpr0_sgpr1 = S_GETPC_B64
+    ; NOMEMC-NEXT: S_WAITCNT_DEPCTR 65534
+    ; NOMEMC-NEXT: $sgpr3 = S_ADD_U32 $sgpr0, 0, implicit-def $scc
+    ; NOMEMC-NEXT: S_ENDPGM 0
+    ;
+    ; MEMC-LABEL: name: hazard_cull_global
+    ; MEMC: $vgpr1, $sgpr0 = V_ADDC_U32_e64 0, $vgpr1, $sgpr0, 0, implicit $exec
+    ; MEMC-NEXT: $vgpr0 = GLOBAL_LOAD_DWORD $vgpr0_vgpr1, 0, 0, implicit $exec
+    ; MEMC-NEXT: DS_NOP implicit $m0, implicit $exec
+    ; MEMC-NEXT: DS_NOP implicit $m0, implicit $exec
+    ; MEMC-NEXT: DS_NOP implicit $m0, implicit $exec
+    ; MEMC-NEXT: DS_NOP implicit $m0, implicit $exec
+    ; MEMC-NEXT: S_WAIT_LOADCNT 0
+    ; MEMC-NEXT: $sgpr0_sgpr1 = S_GETPC_B64
+    ; MEMC-NEXT: $sgpr3 = S_ADD_U32 $sgpr0, 0, implicit-def $scc
+    ; MEMC-NEXT: S_ENDPGM 0
+    $vgpr1, $sgpr0 = V_ADDC_U32_e64 0, $vgpr1, $sgpr0, 0, implicit $exec
+    $vgpr0 = GLOBAL_LOAD_DWORD $vgpr0_vgpr1, 0, 0, implicit $exec
+    S_WAIT_LOADCNT 0
+    $sgpr0_sgpr1 = S_GETPC_B64
+    $sgpr3 = S_ADD_U32 $sgpr0, 0, implicit-def $scc
+    S_ENDPGM 0
+...
+
+---
+name:            hazard_nocull_flat
+body:            |
+  bb.0:
+    ; GCN-LABEL: name: hazard_nocull_flat
+    ; GCN: $vgpr1, $sgpr0 = V_ADDC_U32_e64 0, $vgpr1, $sgpr0, 0, implicit $exec
+    ; GCN-NEXT: $vgpr0 = FLAT_LOAD_DWORD $vgpr0_vgpr1, 0, 0, implicit $exec, implicit $flat_scr
+    ; GCN-NEXT: S_WAIT_LOADCNT 0
+    ; GCN-NEXT: $sgpr0_sgpr1 = S_GETPC_B64
+    ; GCN-NEXT: S_WAITCNT_DEPCTR 65534
+    ; GCN-NEXT: $sgpr3 = S_ADD_U32 $sgpr0, 0, implicit-def $scc
+    ; GCN-NEXT: S_ENDPGM 0
+    $vgpr1, $sgpr0 = V_ADDC_U32_e64 0, $vgpr1, $sgpr0, 0, implicit $exec
+    $vgpr0 = FLAT_LOAD_DWORD $vgpr0_vgpr1, 0, 0, implicit $exec, implicit $flat_scr
+    S_WAIT_LOADCNT 0
+    $sgpr0_sgpr1 = S_GETPC_B64
+    $sgpr3 = S_ADD_U32 $sgpr0, 0, implicit-def $scc
+    S_ENDPGM 0
+...
+
+---
+name:            hazard_existing_cull
+body:            |
+  bb.0:
+    ; GCN-LABEL: name: hazard_existing_cull
+    ; GCN: $vgpr1, $sgpr0 = V_ADDC_U32_e64 0, $vgpr1, $sgpr0, 0, implicit $exec
+    ; GCN-NEXT: DS_NOP implicit $m0, implicit $exec
+    ; GCN-NEXT: DS_NOP implicit $m0, implicit $exec
+    ; GCN-NEXT: DS_NOP implicit $m0, implicit $exec
+    ; GCN-NEXT: DS_NOP implicit $m0, implicit $exec
+    ; GCN-NEXT: $sgpr0_sgpr1 = S_GETPC_B64
+    ; GCN-NEXT: $sgpr3 = S_ADD_U32 $sgpr0, 0, implicit-def $scc
+    ; GCN-NEXT: S_ENDPGM 0
+    $vgpr1, $sgpr0 = V_ADDC_U32_e64 0, $vgpr1, $sgpr0, 0, implicit $exec
+    DS_NOP implicit $m0, implicit $exec
+    DS_NOP implicit $m0, implicit $exec
+    DS_NOP implicit $m0, implicit $exec
+    DS_NOP implicit $m0, implicit $exec
+    $sgpr0_sgpr1 = S_GETPC_B64
+    $sgpr3 = S_ADD_U32 $sgpr0, 0, implicit-def $scc
+    S_ENDPGM 0
+...
diff --git a/llvm/test/CodeGen/AMDGPU/vcmpx-permlane-hazard.mir b/llvm/test/CodeGen/AMDGPU/vcmpx-permlane-hazard.mir
index e3b96c08348fc..d49381c9b8aff 100644
--- a/llvm/test/CodeGen/AMDGPU/vcmpx-permlane-hazard.mir
+++ b/llvm/test/CodeGen/AMDGPU/vcmpx-permlane-hazard.mir
@@ -1,12 +1,11 @@
 # RUN: llc -mtriple=amdgcn -mcpu=gfx1010 -verify-machineinstrs -run-pass post-RA-hazard-rec -o - %s | FileCheck -check-prefix=GCN %s
 # RUN: llc -mtriple=amdgcn -mcpu=gfx1100 -verify-machineinstrs -run-pass post-RA-hazard-rec -o - %s | FileCheck -check-prefix=GCN %s
-# RUN: llc -mtriple=amdgcn -mcpu=gfx1200 -verify-machineinstrs -run-pass post-RA-hazard-rec -o - %s | FileCheck -check-prefixes=GCN,GFX12 %s
+# RUN: llc -mtriple=amdgcn -mcpu=gfx1200 -verify-machineinstrs -run-pass post-RA-hazard-rec -o - %s | FileCheck -check-prefixes=GCN %s
 
 # GCN-LABEL: name: hazard_vcmpx_permlane16
 # GCN:      V_CMPX_LE_F32_nosdst_e32
 # GCN:      S_ADD_U32
 # GCN-NEXT: $vgpr1 = V_MOV_B32_e32 killed $vgpr1, implicit $exec
-# GFX12-NEXT: S_WAITCNT_DEPCTR
 # GCN-NEXT: V_PERMLANE16_B32_e64
 ---
 name:            hazard_vcmpx_permlane16
@@ -52,6 +51,7 @@ body:            |
 # GCN:      V_CMPX_LE_F32_nosdst_e32
 # GCN:      V_NOP
 # GCN-NEXT: $vgpr1 = V_MOV_B32_e32 killed $vgpr1, implicit $exec
+# GFX12-NEXT: S_WAITCNT_DEPCTR
 # GCN-NEXT: V_PERMLANE16_B32_e64
 ---
 name:            hazard_vcmpx_permlane16_v_nop
@@ -129,7 +129,6 @@ body:            |
 # GCN:      V_CMPX_LE_F32_nosdst_e32
 # GCN:      S_ADD_U32
 # GCN-NEXT: dead $vgpr1 = V_MOV_B32_e32 undef $vgpr1, implicit $exec
-# GFX12-NEXT: S_WAITCNT_DEPCTR
 # GCN-NEXT: V_PERMLANE16_B32_e64
 ---
 name:            hazard_vcmpx_permlane16_undef_src
@@ -152,7 +151,6 @@ body:            |
 # GCN:      V_CMPX_LE_F32_nosdst_e64
 # GCN:      S_ADD_U32
 # GCN-NEXT: $vgpr1 = V_MOV_B32_e32 killed $vgpr1, implicit $exec
-# GFX12-NEXT: S_WAITCNT_DEPCTR
 # GCN-NEXT: V_PERMLANE16_B32_e64
 ---
 name:            hazard_vcmpx_e64_permlane16
diff --git a/llvm/test/CodeGen/AMDGPU/vgpr-liverange.ll b/llvm/test/CodeGen/AMDGPU/vgpr-liverange.ll
index a0bce3432a4bd..c0b56d05f72aa 100644
--- a/llvm/test/CodeGen/AMDGPU/vgpr-liverange.ll
+++ b/llvm/test/CodeGen/AMDGPU/vgpr-liverange.ll
@@ -161,16 +161,16 @@ for.end:
 define amdgpu_ps float @loop(i32 %z, float %v, i32 inreg %bound, ptr %extern_func, ptr %extern_func2) #0 {
 ; SI-LABEL: loop:
 ; SI:       ; %bb.0: ; %main_body
+; SI-NEXT:    v_mov_b32_e32 v6, v0
 ; SI-NEXT:    s_mov_b32 s12, SCRATCH_RSRC_DWORD0
 ; SI-NEXT:    s_mov_b32 s13, SCRATCH_RSRC_DWORD1
 ; SI-NEXT:    s_mov_b32 s14, -1
-; SI-NEXT:    v_mov_b32_e32 v6, v0
 ; SI-NEXT:    v_mov_b32_e32 v0, v1
+; SI-NEXT:    v_cmp_gt_i32_e32 vcc_lo, 6, v6
 ; SI-NEXT:    s_mov_b32 s15, 0x31c16000
 ; SI-NEXT:    s_add_u32 s12, s12, s1
 ; SI-NEXT:    s_addc_u32 s13, s13, 0
 ; SI-NEXT:    s_mov_b32 s32, 0
-; SI-NEXT:    v_cmp_gt_i32_e32 vcc_lo, 6, v6
 ; SI-NEXT:    ; implicit-def: $vgpr1
 ; SI-NEXT:    s_and_saveexec_b32 s0, vcc_lo
 ; SI-NEXT:    s_xor_b32 s6, exec_lo, s0
@@ -243,11 +243,11 @@ define amdgpu_ps float @loop_with_use(i32 %z, float %v, i32 inreg %bound, ptr %e
 ; SI-NEXT:    s_mov_b32 s13, SCRATCH_RSRC_DWORD1
 ; SI-NEXT:    s_mov_b32 s14, -1
 ; SI-NEXT:    v_mov_b32_e32 v40, v1
+; SI-NEXT:    v_cmp_gt_i32_e32 vcc_lo, 6, v0
 ; SI-NEXT:    s_mov_b32 s15, 0x31c16000
 ; SI-NEXT:    s_add_u32 s12, s12, s1
 ; SI-NEXT:    s_addc_u32 s13, s13, 0
 ; SI-NEXT:    s_mov_b32 s32, 0
-; SI-NEXT:    v_cmp_gt_i32_e32 vcc_lo, 6, v0
 ; SI-NEXT:    ; implicit-def: $vgpr0
 ; SI-NEXT:    s_and_saveexec_b32 s0, vcc_lo
 ; SI-NEXT:    s_xor_b32 s6, exec_lo, s0
diff --git a/llvm/test/CodeGen/AMDGPU/vni8-live-reg-opt.ll b/llvm/test/CodeGen/AMDGPU/vni8-live-reg-opt.ll
index 29996d68040e7..a7f9a4c51aa75 100644
--- a/llvm/test/CodeGen/AMDGPU/vni8-live-reg-opt.ll
+++ b/llvm/test/CodeGen/AMDGPU/vni8-live-reg-opt.ll
@@ -3,7 +3,7 @@
 
 define amdgpu_kernel void @v3i8_liveout(ptr addrspace(1) %src1, ptr addrspace(1) %src2, ptr addrspace(1) nocapture %dst) {
 ; GFX906-LABEL: define amdgpu_kernel void @v3i8_liveout(
-; GFX906-SAME: ptr addrspace(1) [[SRC1:%.*]], ptr addrspace(1) [[SRC2:%.*]], ptr addrspace(1) nocapture [[DST:%.*]]) #[[ATTR0:[0-9]+]] {
+; GFX906-SAME: ptr addrspace(1) [[SRC1:%.*]], ptr addrspace(1) [[SRC2:%.*]], ptr addrspace(1) captures(none) [[DST:%.*]]) #[[ATTR0:[0-9]+]] {
 ; GFX906-NEXT:  entry:
 ; GFX906-NEXT:    [[IDX:%.*]] = call i32 @llvm.amdgcn.workitem.id.x()
 ; GFX906-NEXT:    [[GEP1:%.*]] = getelementptr <3 x i8>, ptr addrspace(1) [[SRC1]], i32 [[IDX]]
@@ -44,7 +44,7 @@ bb.2:
 
 define amdgpu_kernel void @v4i8_liveout(ptr addrspace(1) %src1, ptr addrspace(1) %src2, ptr addrspace(1) nocapture %dst) {
 ; GFX906-LABEL: define amdgpu_kernel void @v4i8_liveout(
-; GFX906-SAME: ptr addrspace(1) [[SRC1:%.*]], ptr addrspace(1) [[SRC2:%.*]], ptr addrspace(1) nocapture [[DST:%.*]]) #[[ATTR0]] {
+; GFX906-SAME: ptr addrspace(1) [[SRC1:%.*]], ptr addrspace(1) [[SRC2:%.*]], ptr addrspace(1) captures(none) [[DST:%.*]]) #[[ATTR0]] {
 ; GFX906-NEXT:  entry:
 ; GFX906-NEXT:    [[IDX:%.*]] = call i32 @llvm.amdgcn.workitem.id.x()
 ; GFX906-NEXT:    [[GEP1:%.*]] = getelementptr <4 x i8>, ptr addrspace(1) [[SRC1]], i32 [[IDX]]
@@ -82,7 +82,7 @@ bb.2:
 
 define amdgpu_kernel void @v5i8_liveout(ptr addrspace(1) %src1, ptr addrspace(1) %src2, ptr addrspace(1) nocapture %dst) {
 ; GFX906-LABEL: define amdgpu_kernel void @v5i8_liveout(
-; GFX906-SAME: ptr addrspace(1) [[SRC1:%.*]], ptr addrspace(1) [[SRC2:%.*]], ptr addrspace(1) nocapture [[DST:%.*]]) #[[ATTR0]] {
+; GFX906-SAME: ptr addrspace(1) [[SRC1:%.*]], ptr addrspace(1) [[SRC2:%.*]], ptr addrspace(1) captures(none) [[DST:%.*]]) #[[ATTR0]] {
 ; GFX906-NEXT:  entry:
 ; GFX906-NEXT:    [[IDX:%.*]] = call i32 @llvm.amdgcn.workitem.id.x()
 ; GFX906-NEXT:    [[GEP1:%.*]] = getelementptr <5 x i8>, ptr addrspace(1) [[SRC1]], i32 [[IDX]]
@@ -123,7 +123,7 @@ bb.2:
 
 define amdgpu_kernel void @v8i8_liveout(ptr addrspace(1) %src1, ptr addrspace(1) %src2, ptr addrspace(1) nocapture %dst) {
 ; GFX906-LABEL: define amdgpu_kernel void @v8i8_liveout(
-; GFX906-SAME: ptr addrspace(1) [[SRC1:%.*]], ptr addrspace(1) [[SRC2:%.*]], ptr addrspace(1) nocapture [[DST:%.*]]) #[[ATTR0]] {
+; GFX906-SAME: ptr addrspace(1) [[SRC1:%.*]], ptr addrspace(1) [[SRC2:%.*]], ptr addrspace(1) captures(none) [[DST:%.*]]) #[[ATTR0]] {
 ; GFX906-NEXT:  entry:
 ; GFX906-NEXT:    [[IDX:%.*]] = call i32 @llvm.amdgcn.workitem.id.x()
 ; GFX906-NEXT:    [[GEP1:%.*]] = getelementptr <8 x i8>, ptr addrspace(1) [[SRC1]], i32 [[IDX]]
@@ -161,7 +161,7 @@ bb.2:
 
 define amdgpu_kernel void @repeat_successor(i32 %in, ptr addrspace(1) %src1, ptr addrspace(1) %src2, ptr addrspace(1) nocapture %dst) {
 ; GFX906-LABEL: define amdgpu_kernel void @repeat_successor(
-; GFX906-SAME: i32 [[IN:%.*]], ptr addrspace(1) [[SRC1:%.*]], ptr addrspace(1) [[SRC2:%.*]], ptr addrspace(1) nocapture [[DST:%.*]]) #[[ATTR0]] {
+; GFX906-SAME: i32 [[IN:%.*]], ptr addrspace(1) [[SRC1:%.*]], ptr addrspace(1) [[SRC2:%.*]], ptr addrspace(1) captures(none) [[DST:%.*]]) #[[ATTR0]] {
 ; GFX906-NEXT:  entry:
 ; GFX906-NEXT:    [[IDX:%.*]] = call i32 @llvm.amdgcn.workitem.id.x()
 ; GFX906-NEXT:    [[GEP1:%.*]] = getelementptr <4 x i8>, ptr addrspace(1) [[SRC1]], i32 [[IDX]]
@@ -211,7 +211,7 @@ return:
 
 define amdgpu_kernel void @v8i8_phi_chain(ptr addrspace(1) %src1, ptr addrspace(1) %src2, ptr addrspace(1) nocapture %dst0, ptr addrspace(1) nocapture %dst1) {
 ; GFX906-LABEL: define amdgpu_kernel void @v8i8_phi_chain(
-; GFX906-SAME: ptr addrspace(1) [[SRC1:%.*]], ptr addrspace(1) [[SRC2:%.*]], ptr addrspace(1) nocapture [[DST0:%.*]], ptr addrspace(1) nocapture [[DST1:%.*]]) #[[ATTR0]] {
+; GFX906-SAME: ptr addrspace(1) [[SRC1:%.*]], ptr addrspace(1) [[SRC2:%.*]], ptr addrspace(1) captures(none) [[DST0:%.*]], ptr addrspace(1) captures(none) [[DST1:%.*]]) #[[ATTR0]] {
 ; GFX906-NEXT:  entry:
 ; GFX906-NEXT:    [[IDX:%.*]] = call i32 @llvm.amdgcn.workitem.id.x()
 ; GFX906-NEXT:    [[GEP1:%.*]] = getelementptr <8 x i8>, ptr addrspace(1) [[SRC1]], i32 [[IDX]]
@@ -261,7 +261,7 @@ bb.3:
 
 define amdgpu_kernel void @v8i8_multi_block(ptr addrspace(1) %src1, ptr addrspace(1) %src2, ptr addrspace(1) nocapture %dst0, ptr addrspace(1) nocapture %dst1) {
 ; GFX906-LABEL: define amdgpu_kernel void @v8i8_multi_block(
-; GFX906-SAME: ptr addrspace(1) [[SRC1:%.*]], ptr addrspace(1) [[SRC2:%.*]], ptr addrspace(1) nocapture [[DST0:%.*]], ptr addrspace(1) nocapture [[DST1:%.*]]) #[[ATTR0]] {
+; GFX906-SAME: ptr addrspace(1) [[SRC1:%.*]], ptr addrspace(1) [[SRC2:%.*]], ptr addrspace(1) captures(none) [[DST0:%.*]], ptr addrspace(1) captures(none) [[DST1:%.*]]) #[[ATTR0]] {
 ; GFX906-NEXT:  entry:
 ; GFX906-NEXT:    [[IDX:%.*]] = call i32 @llvm.amdgcn.workitem.id.x()
 ; GFX906-NEXT:    [[GEP1:%.*]] = getelementptr <8 x i8>, ptr addrspace(1) [[SRC1]], i32 [[IDX]]
@@ -309,7 +309,7 @@ bb.3:
 
 define amdgpu_kernel void @v32i8_loop_carried(ptr addrspace(1) %src1, ptr addrspace(1) %src2, ptr addrspace(1) nocapture %dst) {
 ; GFX906-LABEL: define amdgpu_kernel void @v32i8_loop_carried(
-; GFX906-SAME: ptr addrspace(1) [[SRC1:%.*]], ptr addrspace(1) [[SRC2:%.*]], ptr addrspace(1) nocapture [[DST:%.*]]) #[[ATTR0]] {
+; GFX906-SAME: ptr addrspace(1) [[SRC1:%.*]], ptr addrspace(1) [[SRC2:%.*]], ptr addrspace(1) captures(none) [[DST:%.*]]) #[[ATTR0]] {
 ; GFX906-NEXT:  entry:
 ; GFX906-NEXT:    [[IDX:%.*]] = call i32 @llvm.amdgcn.workitem.id.x()
 ; GFX906-NEXT:    [[GEP1:%.*]] = getelementptr <32 x i8>, ptr addrspace(1) [[SRC1]], i32 [[IDX]]
diff --git a/llvm/test/CodeGen/AMDGPU/wave32.ll b/llvm/test/CodeGen/AMDGPU/wave32.ll
index 0307472fce732..e49dd9eab30f3 100644
--- a/llvm/test/CodeGen/AMDGPU/wave32.ll
+++ b/llvm/test/CodeGen/AMDGPU/wave32.ll
@@ -372,8 +372,8 @@ define amdgpu_kernel void @test_loop_with_if(ptr addrspace(1) %arg) #0 {
 ; GFX1032-NEXT:  .LBB10_2: ; %bb2
 ; GFX1032-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX1032-NEXT:    v_cmp_ge_i32_e64 s4, v1, v0
-; GFX1032-NEXT:    s_mov_b32 s3, 0
 ; GFX1032-NEXT:    v_cmp_lt_i32_e32 vcc_lo, v1, v0
+; GFX1032-NEXT:    s_mov_b32 s3, 0
 ; GFX1032-NEXT:    s_and_saveexec_b32 s5, vcc_lo
 ; GFX1032-NEXT:    s_cbranch_execz .LBB10_4
 ; GFX1032-NEXT:  ; %bb.3: ; %bb5
@@ -515,8 +515,8 @@ bb13:
 define amdgpu_kernel void @test_loop_with_if_else_break(ptr addrspace(1) %arg) #0 {
 ; GFX1032-LABEL: test_loop_with_if_else_break:
 ; GFX1032:       ; %bb.0: ; %bb
-; GFX1032-NEXT:    s_mov_b32 s2, 0
 ; GFX1032-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v0
+; GFX1032-NEXT:    s_mov_b32 s2, 0
 ; GFX1032-NEXT:    s_and_saveexec_b32 s0, vcc_lo
 ; GFX1032-NEXT:    s_cbranch_execz .LBB11_6
 ; GFX1032-NEXT:  ; %bb.1: ; %.preheader
diff --git a/llvm/test/CodeGen/AMDGPU/wave_dispatch_regs.ll b/llvm/test/CodeGen/AMDGPU/wave_dispatch_regs.ll
index e3a6240aac005..fdc1e6abb051f 100644
--- a/llvm/test/CodeGen/AMDGPU/wave_dispatch_regs.ll
+++ b/llvm/test/CodeGen/AMDGPU/wave_dispatch_regs.ll
@@ -12,6 +12,7 @@
 ; GCN-NEXT: amdpal.pipelines:
 ; GCN-NEXT:   - .hardware_stages:
 ; GCN-NEXT:       .cs:
+; GCN-NEXT:         .entry_point:    _amdgpu_cs
 ; GCN-NEXT:         .entry_point_symbol:    _amdgpu_cs_main
 ; GCN-NEXT:         .scratch_memory_size: 0
 ; SI-NEXT:          .sgpr_count:     0x11
diff --git a/llvm/test/CodeGen/AMDGPU/wqm-terminators.mir b/llvm/test/CodeGen/AMDGPU/wqm-terminators.mir
index 8d75bb3b1280f..7656629a7b009 100644
--- a/llvm/test/CodeGen/AMDGPU/wqm-terminators.mir
+++ b/llvm/test/CodeGen/AMDGPU/wqm-terminators.mir
@@ -1,5 +1,6 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 2
 # RUN: llc -mtriple=amdgcn -mcpu=gfx1010 -verify-machineinstrs -run-pass si-wqm -o -  %s | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=gfx1010 -passes=si-wqm -o -  %s | FileCheck %s
 
 --- |
   define amdgpu_ps void @exit_to_exact() {
diff --git a/llvm/test/CodeGen/AMDGPU/wqm.mir b/llvm/test/CodeGen/AMDGPU/wqm.mir
index 4762760c4ba24..99327e1d3c498 100644
--- a/llvm/test/CodeGen/AMDGPU/wqm.mir
+++ b/llvm/test/CodeGen/AMDGPU/wqm.mir
@@ -1,4 +1,5 @@
 # RUN: llc -mtriple=amdgcn -mcpu=fiji -verify-machineinstrs -run-pass si-wqm -o -  %s | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=fiji -passes=si-wqm -o -  %s | FileCheck %s
 
 --- |
   define amdgpu_ps void @test_strict_wwm_scc() {
diff --git a/llvm/test/CodeGen/AMDGPU/wwm-reserved-spill.ll b/llvm/test/CodeGen/AMDGPU/wwm-reserved-spill.ll
index 2e59a36adb7e4..af7d16968488a 100644
--- a/llvm/test/CodeGen/AMDGPU/wwm-reserved-spill.ll
+++ b/llvm/test/CodeGen/AMDGPU/wwm-reserved-spill.ll
@@ -416,10 +416,10 @@ define amdgpu_gfx void @strict_wwm_call(ptr addrspace(8) inreg %tmp14, i32 inreg
 ; GFX9-O3-NEXT:    v_mov_b32_e32 v0, s8
 ; GFX9-O3-NEXT:    s_or_saveexec_b64 s[34:35], -1
 ; GFX9-O3-NEXT:    v_cndmask_b32_e64 v2, 0, v0, s[34:35]
-; GFX9-O3-NEXT:    v_mov_b32_e32 v0, v2
 ; GFX9-O3-NEXT:    s_getpc_b64 s[36:37]
 ; GFX9-O3-NEXT:    s_add_u32 s36, s36, strict_wwm_called@rel32@lo+4
 ; GFX9-O3-NEXT:    s_addc_u32 s37, s37, strict_wwm_called@rel32@hi+12
+; GFX9-O3-NEXT:    v_mov_b32_e32 v0, v2
 ; GFX9-O3-NEXT:    s_swappc_b64 s[30:31], s[36:37]
 ; GFX9-O3-NEXT:    v_mov_b32_e32 v1, v0
 ; GFX9-O3-NEXT:    v_add_u32_e32 v1, v1, v2
diff --git a/llvm/test/CodeGen/AMDGPU/wwm-reserved.ll b/llvm/test/CodeGen/AMDGPU/wwm-reserved.ll
index 08cc2e4ec7d79..ddc50b7d49504 100644
--- a/llvm/test/CodeGen/AMDGPU/wwm-reserved.ll
+++ b/llvm/test/CodeGen/AMDGPU/wwm-reserved.ll
@@ -426,12 +426,12 @@ define amdgpu_kernel void @call(ptr addrspace(8) inreg %tmp14, i32 inreg %arg) {
 ; GFX9-O3-NEXT:    s_mov_b64 s[4:5], s[0:1]
 ; GFX9-O3-NEXT:    s_mov_b64 s[6:7], s[2:3]
 ; GFX9-O3-NEXT:    s_mov_b64 s[0:1], s[24:25]
-; GFX9-O3-NEXT:    v_mov_b32_e32 v31, v3
-; GFX9-O3-NEXT:    s_mov_b64 s[2:3], s[26:27]
-; GFX9-O3-NEXT:    v_mov_b32_e32 v0, v6
 ; GFX9-O3-NEXT:    s_getpc_b64 s[22:23]
 ; GFX9-O3-NEXT:    s_add_u32 s22, s22, called@rel32@lo+4
 ; GFX9-O3-NEXT:    s_addc_u32 s23, s23, called@rel32@hi+12
+; GFX9-O3-NEXT:    v_mov_b32_e32 v31, v3
+; GFX9-O3-NEXT:    s_mov_b64 s[2:3], s[26:27]
+; GFX9-O3-NEXT:    v_mov_b32_e32 v0, v6
 ; GFX9-O3-NEXT:    s_swappc_b64 s[30:31], s[22:23]
 ; GFX9-O3-NEXT:    v_mov_b32_e32 v3, v0
 ; GFX9-O3-NEXT:    v_add_u32_e32 v3, v3, v6
@@ -1278,12 +1278,12 @@ define amdgpu_kernel void @strict_wwm_call(ptr addrspace(8) inreg %tmp14, i32 in
 ; GFX9-O3-NEXT:    s_mov_b64 s[4:5], s[0:1]
 ; GFX9-O3-NEXT:    s_mov_b64 s[6:7], s[2:3]
 ; GFX9-O3-NEXT:    s_mov_b64 s[0:1], s[24:25]
-; GFX9-O3-NEXT:    v_mov_b32_e32 v31, v3
-; GFX9-O3-NEXT:    s_mov_b64 s[2:3], s[26:27]
-; GFX9-O3-NEXT:    v_mov_b32_e32 v0, v6
 ; GFX9-O3-NEXT:    s_getpc_b64 s[22:23]
 ; GFX9-O3-NEXT:    s_add_u32 s22, s22, strict_wwm_called@rel32@lo+4
 ; GFX9-O3-NEXT:    s_addc_u32 s23, s23, strict_wwm_called@rel32@hi+12
+; GFX9-O3-NEXT:    v_mov_b32_e32 v31, v3
+; GFX9-O3-NEXT:    s_mov_b64 s[2:3], s[26:27]
+; GFX9-O3-NEXT:    v_mov_b32_e32 v0, v6
 ; GFX9-O3-NEXT:    s_swappc_b64 s[30:31], s[22:23]
 ; GFX9-O3-NEXT:    v_mov_b32_e32 v3, v0
 ; GFX9-O3-NEXT:    v_add_u32_e32 v3, v3, v6
diff --git a/llvm/test/CodeGen/ARM/aes-erratum-fix.ll b/llvm/test/CodeGen/ARM/aes-erratum-fix.ll
index 43c403fe6d64d..82f5bfd02a56e 100644
--- a/llvm/test/CodeGen/ARM/aes-erratum-fix.ll
+++ b/llvm/test/CodeGen/ARM/aes-erratum-fix.ll
@@ -1447,12 +1447,13 @@ define arm_aapcs_vfpcc void @aese_setf16_cond_via_ptr(i1 zeroext %0, ptr %1, <16
 ; CHECK-CORTEX-FIX-NEXT:    .pad #24
 ; CHECK-CORTEX-FIX-NEXT:    sub sp, sp, #24
 ; CHECK-CORTEX-FIX-NEXT:    cmp r0, #0
-; CHECK-CORTEX-FIX-NEXT:    beq .LBB36_2
+; CHECK-CORTEX-FIX-NEXT:    beq .LBB36_3
 ; CHECK-CORTEX-FIX-NEXT:  @ %bb.1:
 ; CHECK-CORTEX-FIX-NEXT:    vld1.64 {d16, d17}, [r2]
 ; CHECK-CORTEX-FIX-NEXT:    vmov.u16 r3, d16[1]
 ; CHECK-CORTEX-FIX-NEXT:    vmov.u16 r6, d17[0]
 ; CHECK-CORTEX-FIX-NEXT:    vmov.u16 r7, d17[2]
+; CHECK-CORTEX-FIX-NEXT:    vmov.u16 r4, d17[3]
 ; CHECK-CORTEX-FIX-NEXT:    str r3, [sp, #20] @ 4-byte Spill
 ; CHECK-CORTEX-FIX-NEXT:    vmov.u16 r3, d16[2]
 ; CHECK-CORTEX-FIX-NEXT:    str r3, [sp, #8] @ 4-byte Spill
@@ -1463,37 +1464,37 @@ define arm_aapcs_vfpcc void @aese_setf16_cond_via_ptr(i1 zeroext %0, ptr %1, <16
 ; CHECK-CORTEX-FIX-NEXT:    ldrh r3, [r1]
 ; CHECK-CORTEX-FIX-NEXT:    str r3, [sp, #16] @ 4-byte Spill
 ; CHECK-CORTEX-FIX-NEXT:    mov r3, r6
-; CHECK-CORTEX-FIX-NEXT:    b .LBB36_3
+; CHECK-CORTEX-FIX-NEXT:    cmp r0, #0
+; CHECK-CORTEX-FIX-NEXT:    bne .LBB36_4
 ; CHECK-CORTEX-FIX-NEXT:  .LBB36_2:
-; CHECK-CORTEX-FIX-NEXT:    add r3, r2, #8
+; CHECK-CORTEX-FIX-NEXT:    vmov.u16 r0, d0[0]
+; CHECK-CORTEX-FIX-NEXT:    b .LBB36_5
+; CHECK-CORTEX-FIX-NEXT:  .LBB36_3:
 ; CHECK-CORTEX-FIX-NEXT:    vld1.32 {d16[0]}, [r2:32]
+; CHECK-CORTEX-FIX-NEXT:    add r3, r2, #8
 ; CHECK-CORTEX-FIX-NEXT:    add r7, r2, #4
 ; CHECK-CORTEX-FIX-NEXT:    vld1.32 {d17[0]}, [r3:32]
 ; CHECK-CORTEX-FIX-NEXT:    add r3, r2, #12
 ; CHECK-CORTEX-FIX-NEXT:    vld1.32 {d16[1]}, [r7:32]
 ; CHECK-CORTEX-FIX-NEXT:    vld1.32 {d17[1]}, [r3:32]
 ; CHECK-CORTEX-FIX-NEXT:    vmov.u16 r3, d16[0]
+; CHECK-CORTEX-FIX-NEXT:    vmov.u16 r7, d17[1]
+; CHECK-CORTEX-FIX-NEXT:    vmov.u16 r4, d17[3]
 ; CHECK-CORTEX-FIX-NEXT:    str r3, [sp, #16] @ 4-byte Spill
 ; CHECK-CORTEX-FIX-NEXT:    vmov.u16 r3, d16[1]
-; CHECK-CORTEX-FIX-NEXT:    vmov.u16 r7, d17[1]
-; CHECK-CORTEX-FIX-NEXT:    str r3, [sp, #20] @ 4-byte Spill
-; CHECK-CORTEX-FIX-NEXT:    vmov.u16 r3, d16[2]
 ; CHECK-CORTEX-FIX-NEXT:    str r7, [sp, #12] @ 4-byte Spill
 ; CHECK-CORTEX-FIX-NEXT:    vmov.u16 r7, d17[2]
+; CHECK-CORTEX-FIX-NEXT:    str r3, [sp, #20] @ 4-byte Spill
+; CHECK-CORTEX-FIX-NEXT:    vmov.u16 r3, d16[2]
 ; CHECK-CORTEX-FIX-NEXT:    str r3, [sp, #8] @ 4-byte Spill
 ; CHECK-CORTEX-FIX-NEXT:    vmov.u16 r3, d16[3]
 ; CHECK-CORTEX-FIX-NEXT:    str r3, [sp, #4] @ 4-byte Spill
 ; CHECK-CORTEX-FIX-NEXT:    vmov.u16 r3, d17[0]
-; CHECK-CORTEX-FIX-NEXT:  .LBB36_3:
-; CHECK-CORTEX-FIX-NEXT:    vmov.u16 r4, d17[3]
 ; CHECK-CORTEX-FIX-NEXT:    cmp r0, #0
-; CHECK-CORTEX-FIX-NEXT:    beq .LBB36_5
-; CHECK-CORTEX-FIX-NEXT:  @ %bb.4:
+; CHECK-CORTEX-FIX-NEXT:    beq .LBB36_2
+; CHECK-CORTEX-FIX-NEXT:  .LBB36_4:
 ; CHECK-CORTEX-FIX-NEXT:    ldrh r0, [r1]
-; CHECK-CORTEX-FIX-NEXT:    b .LBB36_6
 ; CHECK-CORTEX-FIX-NEXT:  .LBB36_5:
-; CHECK-CORTEX-FIX-NEXT:    vmov.u16 r0, d0[0]
-; CHECK-CORTEX-FIX-NEXT:  .LBB36_6:
 ; CHECK-CORTEX-FIX-NEXT:    str r0, [sp] @ 4-byte Spill
 ; CHECK-CORTEX-FIX-NEXT:    ldr r0, [sp, #8] @ 4-byte Reload
 ; CHECK-CORTEX-FIX-NEXT:    ldr r1, [sp, #4] @ 4-byte Reload
@@ -1695,8 +1696,8 @@ define arm_aapcs_vfpcc void @aese_setf16_cond_via_val(i1 zeroext %0, half %1, <1
 ; CHECK-CORTEX-FIX-NEXT:    vmov s0, lr
 ; CHECK-CORTEX-FIX-NEXT:    b .LBB37_5
 ; CHECK-CORTEX-FIX-NEXT:  .LBB37_3:
-; CHECK-CORTEX-FIX-NEXT:    add r2, r1, #8
 ; CHECK-CORTEX-FIX-NEXT:    vld1.32 {d16[0]}, [r1:32]
+; CHECK-CORTEX-FIX-NEXT:    add r2, r1, #8
 ; CHECK-CORTEX-FIX-NEXT:    add r3, r1, #4
 ; CHECK-CORTEX-FIX-NEXT:    vld1.32 {d17[0]}, [r2:32]
 ; CHECK-CORTEX-FIX-NEXT:    add r2, r1, #12
@@ -1706,11 +1707,11 @@ define arm_aapcs_vfpcc void @aese_setf16_cond_via_val(i1 zeroext %0, half %1, <1
 ; CHECK-CORTEX-FIX-NEXT:    vmov.u16 r2, d16[0]
 ; CHECK-CORTEX-FIX-NEXT:    vmov.u16 r7, d16[2]
 ; CHECK-CORTEX-FIX-NEXT:    vmov.u16 r10, d16[3]
-; CHECK-CORTEX-FIX-NEXT:    str r3, [sp, #8] @ 4-byte Spill
-; CHECK-CORTEX-FIX-NEXT:    vmov s2, r2
-; CHECK-CORTEX-FIX-NEXT:    vmov.u16 r3, d17[0]
 ; CHECK-CORTEX-FIX-NEXT:    vmov.u16 r11, d17[2]
 ; CHECK-CORTEX-FIX-NEXT:    vmov.u16 r6, d17[3]
+; CHECK-CORTEX-FIX-NEXT:    str r3, [sp, #8] @ 4-byte Spill
+; CHECK-CORTEX-FIX-NEXT:    vmov.u16 r3, d17[0]
+; CHECK-CORTEX-FIX-NEXT:    vmov s2, r2
 ; CHECK-CORTEX-FIX-NEXT:    str r3, [sp, #4] @ 4-byte Spill
 ; CHECK-CORTEX-FIX-NEXT:    vmov.u16 r3, d17[1]
 ; CHECK-CORTEX-FIX-NEXT:    str r3, [sp] @ 4-byte Spill
@@ -3600,12 +3601,13 @@ define arm_aapcs_vfpcc void @aesd_setf16_cond_via_ptr(i1 zeroext %0, ptr %1, <16
 ; CHECK-CORTEX-FIX-NEXT:    .pad #24
 ; CHECK-CORTEX-FIX-NEXT:    sub sp, sp, #24
 ; CHECK-CORTEX-FIX-NEXT:    cmp r0, #0
-; CHECK-CORTEX-FIX-NEXT:    beq .LBB82_2
+; CHECK-CORTEX-FIX-NEXT:    beq .LBB82_3
 ; CHECK-CORTEX-FIX-NEXT:  @ %bb.1:
 ; CHECK-CORTEX-FIX-NEXT:    vld1.64 {d16, d17}, [r2]
 ; CHECK-CORTEX-FIX-NEXT:    vmov.u16 r3, d16[1]
 ; CHECK-CORTEX-FIX-NEXT:    vmov.u16 r6, d17[0]
 ; CHECK-CORTEX-FIX-NEXT:    vmov.u16 r7, d17[2]
+; CHECK-CORTEX-FIX-NEXT:    vmov.u16 r4, d17[3]
 ; CHECK-CORTEX-FIX-NEXT:    str r3, [sp, #20] @ 4-byte Spill
 ; CHECK-CORTEX-FIX-NEXT:    vmov.u16 r3, d16[2]
 ; CHECK-CORTEX-FIX-NEXT:    str r3, [sp, #8] @ 4-byte Spill
@@ -3616,37 +3618,37 @@ define arm_aapcs_vfpcc void @aesd_setf16_cond_via_ptr(i1 zeroext %0, ptr %1, <16
 ; CHECK-CORTEX-FIX-NEXT:    ldrh r3, [r1]
 ; CHECK-CORTEX-FIX-NEXT:    str r3, [sp, #16] @ 4-byte Spill
 ; CHECK-CORTEX-FIX-NEXT:    mov r3, r6
-; CHECK-CORTEX-FIX-NEXT:    b .LBB82_3
+; CHECK-CORTEX-FIX-NEXT:    cmp r0, #0
+; CHECK-CORTEX-FIX-NEXT:    bne .LBB82_4
 ; CHECK-CORTEX-FIX-NEXT:  .LBB82_2:
-; CHECK-CORTEX-FIX-NEXT:    add r3, r2, #8
+; CHECK-CORTEX-FIX-NEXT:    vmov.u16 r0, d0[0]
+; CHECK-CORTEX-FIX-NEXT:    b .LBB82_5
+; CHECK-CORTEX-FIX-NEXT:  .LBB82_3:
 ; CHECK-CORTEX-FIX-NEXT:    vld1.32 {d16[0]}, [r2:32]
+; CHECK-CORTEX-FIX-NEXT:    add r3, r2, #8
 ; CHECK-CORTEX-FIX-NEXT:    add r7, r2, #4
 ; CHECK-CORTEX-FIX-NEXT:    vld1.32 {d17[0]}, [r3:32]
 ; CHECK-CORTEX-FIX-NEXT:    add r3, r2, #12
 ; CHECK-CORTEX-FIX-NEXT:    vld1.32 {d16[1]}, [r7:32]
 ; CHECK-CORTEX-FIX-NEXT:    vld1.32 {d17[1]}, [r3:32]
 ; CHECK-CORTEX-FIX-NEXT:    vmov.u16 r3, d16[0]
+; CHECK-CORTEX-FIX-NEXT:    vmov.u16 r7, d17[1]
+; CHECK-CORTEX-FIX-NEXT:    vmov.u16 r4, d17[3]
 ; CHECK-CORTEX-FIX-NEXT:    str r3, [sp, #16] @ 4-byte Spill
 ; CHECK-CORTEX-FIX-NEXT:    vmov.u16 r3, d16[1]
-; CHECK-CORTEX-FIX-NEXT:    vmov.u16 r7, d17[1]
-; CHECK-CORTEX-FIX-NEXT:    str r3, [sp, #20] @ 4-byte Spill
-; CHECK-CORTEX-FIX-NEXT:    vmov.u16 r3, d16[2]
 ; CHECK-CORTEX-FIX-NEXT:    str r7, [sp, #12] @ 4-byte Spill
 ; CHECK-CORTEX-FIX-NEXT:    vmov.u16 r7, d17[2]
+; CHECK-CORTEX-FIX-NEXT:    str r3, [sp, #20] @ 4-byte Spill
+; CHECK-CORTEX-FIX-NEXT:    vmov.u16 r3, d16[2]
 ; CHECK-CORTEX-FIX-NEXT:    str r3, [sp, #8] @ 4-byte Spill
 ; CHECK-CORTEX-FIX-NEXT:    vmov.u16 r3, d16[3]
 ; CHECK-CORTEX-FIX-NEXT:    str r3, [sp, #4] @ 4-byte Spill
 ; CHECK-CORTEX-FIX-NEXT:    vmov.u16 r3, d17[0]
-; CHECK-CORTEX-FIX-NEXT:  .LBB82_3:
-; CHECK-CORTEX-FIX-NEXT:    vmov.u16 r4, d17[3]
 ; CHECK-CORTEX-FIX-NEXT:    cmp r0, #0
-; CHECK-CORTEX-FIX-NEXT:    beq .LBB82_5
-; CHECK-CORTEX-FIX-NEXT:  @ %bb.4:
+; CHECK-CORTEX-FIX-NEXT:    beq .LBB82_2
+; CHECK-CORTEX-FIX-NEXT:  .LBB82_4:
 ; CHECK-CORTEX-FIX-NEXT:    ldrh r0, [r1]
-; CHECK-CORTEX-FIX-NEXT:    b .LBB82_6
 ; CHECK-CORTEX-FIX-NEXT:  .LBB82_5:
-; CHECK-CORTEX-FIX-NEXT:    vmov.u16 r0, d0[0]
-; CHECK-CORTEX-FIX-NEXT:  .LBB82_6:
 ; CHECK-CORTEX-FIX-NEXT:    str r0, [sp] @ 4-byte Spill
 ; CHECK-CORTEX-FIX-NEXT:    ldr r0, [sp, #8] @ 4-byte Reload
 ; CHECK-CORTEX-FIX-NEXT:    ldr r1, [sp, #4] @ 4-byte Reload
@@ -3848,8 +3850,8 @@ define arm_aapcs_vfpcc void @aesd_setf16_cond_via_val(i1 zeroext %0, half %1, <1
 ; CHECK-CORTEX-FIX-NEXT:    vmov s0, lr
 ; CHECK-CORTEX-FIX-NEXT:    b .LBB83_5
 ; CHECK-CORTEX-FIX-NEXT:  .LBB83_3:
-; CHECK-CORTEX-FIX-NEXT:    add r2, r1, #8
 ; CHECK-CORTEX-FIX-NEXT:    vld1.32 {d16[0]}, [r1:32]
+; CHECK-CORTEX-FIX-NEXT:    add r2, r1, #8
 ; CHECK-CORTEX-FIX-NEXT:    add r3, r1, #4
 ; CHECK-CORTEX-FIX-NEXT:    vld1.32 {d17[0]}, [r2:32]
 ; CHECK-CORTEX-FIX-NEXT:    add r2, r1, #12
@@ -3859,11 +3861,11 @@ define arm_aapcs_vfpcc void @aesd_setf16_cond_via_val(i1 zeroext %0, half %1, <1
 ; CHECK-CORTEX-FIX-NEXT:    vmov.u16 r2, d16[0]
 ; CHECK-CORTEX-FIX-NEXT:    vmov.u16 r7, d16[2]
 ; CHECK-CORTEX-FIX-NEXT:    vmov.u16 r10, d16[3]
-; CHECK-CORTEX-FIX-NEXT:    str r3, [sp, #8] @ 4-byte Spill
-; CHECK-CORTEX-FIX-NEXT:    vmov s2, r2
-; CHECK-CORTEX-FIX-NEXT:    vmov.u16 r3, d17[0]
 ; CHECK-CORTEX-FIX-NEXT:    vmov.u16 r11, d17[2]
 ; CHECK-CORTEX-FIX-NEXT:    vmov.u16 r6, d17[3]
+; CHECK-CORTEX-FIX-NEXT:    str r3, [sp, #8] @ 4-byte Spill
+; CHECK-CORTEX-FIX-NEXT:    vmov.u16 r3, d17[0]
+; CHECK-CORTEX-FIX-NEXT:    vmov s2, r2
 ; CHECK-CORTEX-FIX-NEXT:    str r3, [sp, #4] @ 4-byte Spill
 ; CHECK-CORTEX-FIX-NEXT:    vmov.u16 r3, d17[1]
 ; CHECK-CORTEX-FIX-NEXT:    str r3, [sp] @ 4-byte Spill
diff --git a/llvm/test/CodeGen/ARM/arm-bf16-dotprod-intrinsics.ll b/llvm/test/CodeGen/ARM/arm-bf16-dotprod-intrinsics.ll
index 522726c4e8890..f325fef03f936 100644
--- a/llvm/test/CodeGen/ARM/arm-bf16-dotprod-intrinsics.ll
+++ b/llvm/test/CodeGen/ARM/arm-bf16-dotprod-intrinsics.ll
@@ -64,7 +64,6 @@ entry:
 define <4 x float> @test_vbfdotq_lane_f32(<4 x float> %r, <8 x bfloat> %a, <4 x bfloat> %b) {
 ; CHECK-LABEL: test_vbfdotq_lane_f32:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    @ kill: def $d4 killed $d4 def $q2
 ; CHECK-NEXT:    vdot.bf16 q0, q1, d4[0]
 ; CHECK-NEXT:    bx lr
 entry:
@@ -108,7 +107,6 @@ entry:
 define <4 x float> @test_vbfmlalbq_lane_f32(<4 x float> %r, <8 x bfloat> %a, <4 x bfloat> %b) {
 ; CHECK-LABEL: test_vbfmlalbq_lane_f32:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    @ kill: def $d4 killed $d4 def $q2
 ; CHECK-NEXT:    vfmab.bf16 q0, q1, d4[0]
 ; CHECK-NEXT:    bx lr
 entry:
@@ -131,7 +129,6 @@ entry:
 define <4 x float> @test_vbfmlaltq_lane_f32(<4 x float> %r, <8 x bfloat> %a, <4 x bfloat> %b) {
 ; CHECK-LABEL: test_vbfmlaltq_lane_f32:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    @ kill: def $d4 killed $d4 def $q2
 ; CHECK-NEXT:    vfmat.bf16 q0, q1, d4[0]
 ; CHECK-NEXT:    bx lr
 entry:
diff --git a/llvm/test/CodeGen/ARM/arm-shrink-wrapping-linux.ll b/llvm/test/CodeGen/ARM/arm-shrink-wrapping-linux.ll
index f367a5626c8d3..b92f03d43bb4c 100644
--- a/llvm/test/CodeGen/ARM/arm-shrink-wrapping-linux.ll
+++ b/llvm/test/CodeGen/ARM/arm-shrink-wrapping-linux.ll
@@ -41,8 +41,8 @@ define fastcc ptr @wrongUseOfPostDominate(ptr readonly %s, i32 %off, ptr readnon
 ; ENABLE-NEXT:    bhs .LBB0_6
 ; ENABLE-NEXT:  @ %bb.5: @ %while.body
 ; ENABLE-NEXT:    @ in Loop: Header=BB0_4 Depth=1
-; ENABLE-NEXT:    cmp r0, r2
 ; ENABLE-NEXT:    mov r1, r3
+; ENABLE-NEXT:    cmp r0, r2
 ; ENABLE-NEXT:    blo .LBB0_4
 ; ENABLE-NEXT:  .LBB0_6: @ %if.end29
 ; ENABLE-NEXT:    pop {r11, pc}
@@ -131,8 +131,8 @@ define fastcc ptr @wrongUseOfPostDominate(ptr readonly %s, i32 %off, ptr readnon
 ; DISABLE-NEXT:    bhs .LBB0_6
 ; DISABLE-NEXT:  @ %bb.5: @ %while.body
 ; DISABLE-NEXT:    @ in Loop: Header=BB0_4 Depth=1
-; DISABLE-NEXT:    cmp r0, r2
 ; DISABLE-NEXT:    mov r1, r3
+; DISABLE-NEXT:    cmp r0, r2
 ; DISABLE-NEXT:    blo .LBB0_4
 ; DISABLE-NEXT:  .LBB0_6: @ %if.end29
 ; DISABLE-NEXT:    pop {r11, pc}
diff --git a/llvm/test/CodeGen/ARM/arm-shrink-wrapping.ll b/llvm/test/CodeGen/ARM/arm-shrink-wrapping.ll
index 75416475289f3..5b2d0a8a7c059 100644
--- a/llvm/test/CodeGen/ARM/arm-shrink-wrapping.ll
+++ b/llvm/test/CodeGen/ARM/arm-shrink-wrapping.ll
@@ -2017,8 +2017,8 @@ define float @debug_info(float %gamma, float %slopeLimit, i1 %or.cond, double %t
 ; ARM-DISABLE-NEXT:    sub r4, sp, #24
 ; ARM-DISABLE-NEXT:    bfc r4, #0, #4
 ; ARM-DISABLE-NEXT:    mov sp, r4
-; ARM-DISABLE-NEXT:    tst r2, #1
 ; ARM-DISABLE-NEXT:    vst1.64 {d8, d9}, [r4:128]
+; ARM-DISABLE-NEXT:    tst r2, #1
 ; ARM-DISABLE-NEXT:    vstr d10, [r4, #16]
 ; ARM-DISABLE-NEXT:    beq LBB12_2
 ; ARM-DISABLE-NEXT:  @ %bb.1: @ %bb3
@@ -2123,8 +2123,8 @@ define float @debug_info(float %gamma, float %slopeLimit, i1 %or.cond, double %t
 ; THUMB-DISABLE-NEXT:    sub.w r4, sp, #24
 ; THUMB-DISABLE-NEXT:    bfc r4, #0, #4
 ; THUMB-DISABLE-NEXT:    mov sp, r4
-; THUMB-DISABLE-NEXT:    lsls r1, r2, #31
 ; THUMB-DISABLE-NEXT:    vst1.64 {d8, d9}, [r4:128]
+; THUMB-DISABLE-NEXT:    lsls r1, r2, #31
 ; THUMB-DISABLE-NEXT:    vstr d10, [r4, #16]
 ; THUMB-DISABLE-NEXT:    beq LBB12_2
 ; THUMB-DISABLE-NEXT:  @ %bb.1: @ %bb3
diff --git a/llvm/test/CodeGen/ARM/armv8.2a-fp16-vector-intrinsics.ll b/llvm/test/CodeGen/ARM/armv8.2a-fp16-vector-intrinsics.ll
index 9570c70676dbb..7b130f0007809 100644
--- a/llvm/test/CodeGen/ARM/armv8.2a-fp16-vector-intrinsics.ll
+++ b/llvm/test/CodeGen/ARM/armv8.2a-fp16-vector-intrinsics.ll
@@ -1167,7 +1167,6 @@ entry:
 define dso_local <8 x half> @test_vmulq_lane_f16(<8 x half> %a, <4 x half> %b) {
 ; CHECK-LABEL: test_vmulq_lane_f16:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    @ kill: def $d2 killed $d2 def $q1
 ; CHECK-NEXT:    vmul.f16 q0, q0, d2[3]
 ; CHECK-NEXT:    bx lr
 entry:
@@ -1389,7 +1388,6 @@ entry:
 define dso_local <8 x half> @test_vdupq_lane_f16(<4 x half> %a) {
 ; CHECK-LABEL: test_vdupq_lane_f16:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    @ kill: def $d0 killed $d0 def $q0
 ; CHECK-NEXT:    vdup.16 q0, d0[3]
 ; CHECK-NEXT:    bx lr
 entry:
diff --git a/llvm/test/CodeGen/ARM/bf16-create-get-set-dup.ll b/llvm/test/CodeGen/ARM/bf16-create-get-set-dup.ll
index b16993d9875ba..39416ccdd7765 100644
--- a/llvm/test/CodeGen/ARM/bf16-create-get-set-dup.ll
+++ b/llvm/test/CodeGen/ARM/bf16-create-get-set-dup.ll
@@ -53,7 +53,6 @@ entry:
 define arm_aapcs_vfpcc <8 x bfloat> @test_vdupq_lane_bf16(<4 x bfloat> %v) {
 ; CHECK-LABEL: test_vdupq_lane_bf16:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    @ kill: def $d0 killed $d0 def $q0
 ; CHECK-NEXT:    vdup.16 q0, d0[1]
 ; CHECK-NEXT:    bx lr
 entry:
diff --git a/llvm/test/CodeGen/ARM/bf16-shuffle.ll b/llvm/test/CodeGen/ARM/bf16-shuffle.ll
index 9968e7887f4b3..a45ad8f698b23 100644
--- a/llvm/test/CodeGen/ARM/bf16-shuffle.ll
+++ b/llvm/test/CodeGen/ARM/bf16-shuffle.ll
@@ -229,7 +229,6 @@ entry:
 define dso_local <8 x bfloat> @test_vdupq_lane_bf16(<4 x bfloat> %a) {
 ; CHECK-LABEL: test_vdupq_lane_bf16:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    @ kill: def $d0 killed $d0 def $q0
 ; CHECK-NEXT:    vdup.16 q0, d0[3]
 ; CHECK-NEXT:    bx lr
 entry:
diff --git a/llvm/test/CodeGen/ARM/machine-copyprop.mir b/llvm/test/CodeGen/ARM/machine-copyprop.mir
index f43c38837cec6..73f8830d139de 100644
--- a/llvm/test/CodeGen/ARM/machine-copyprop.mir
+++ b/llvm/test/CodeGen/ARM/machine-copyprop.mir
@@ -1,5 +1,7 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
 # RUN: llc -o - %s -mtriple=armv7s-- -run-pass=machine-cp | FileCheck %s
+
+# RUN: llc -o - %s -mtriple=armv7s-- -passes=machine-cp | FileCheck %s
 ---
 # Test that machine copy prop recognizes the implicit-def operands on a COPY
 # as clobbering the register.
diff --git a/llvm/test/CodeGen/ARM/memcpy-inline.ll b/llvm/test/CodeGen/ARM/memcpy-inline.ll
index 596a58afe46e5..89db22d1f0ed5 100644
--- a/llvm/test/CodeGen/ARM/memcpy-inline.ll
+++ b/llvm/test/CodeGen/ARM/memcpy-inline.ll
@@ -12,6 +12,7 @@
 @.str4 = private unnamed_addr constant [18 x i8] c"DHRYSTONE PROGR  \00", align 1
 @.str5 = private unnamed_addr constant [7 x i8] c"DHRYST\00", align 1
 @.str6 = private unnamed_addr constant [14 x i8] c"/tmp/rmXXXXXX\00", align 1
+@empty = private unnamed_addr constant [31 x i8] zeroinitializer, align 1
 @spool.splbuf = internal global [512 x i8] zeroinitializer, align 16
 
 define i32 @t0() {
@@ -282,5 +283,31 @@ entry:
   ret void
 }
 
+define void @copy_from_zero_constant(ptr nocapture %C) nounwind {
+; CHECK-LABEL: copy_from_zero_constant:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmov.i32 q8, #0x0
+; CHECK-NEXT:    movs r1, #15
+; CHECK-NEXT:    vst1.8 {d16, d17}, [r0], r1
+; CHECK-NEXT:    vst1.8 {d16, d17}, [r0]
+; CHECK-NEXT:    bx lr
+;
+; CHECK-T1-LABEL: copy_from_zero_constant:
+; CHECK-T1:       @ %bb.0: @ %entry
+; CHECK-T1-NEXT:    .save {r7, lr}
+; CHECK-T1-NEXT:    push {r7, lr}
+; CHECK-T1-NEXT:    ldr r1, .LCPI8_0
+; CHECK-T1-NEXT:    movs r2, #31
+; CHECK-T1-NEXT:    bl __aeabi_memcpy
+; CHECK-T1-NEXT:    pop {r7, pc}
+; CHECK-T1-NEXT:    .p2align 2
+; CHECK-T1-NEXT:  @ %bb.1:
+; CHECK-T1-NEXT:  .LCPI8_0:
+; CHECK-T1-NEXT:    .long .Lempty
+entry:
+  tail call void @llvm.memcpy.p0.p0.i64(ptr %C, ptr @empty, i64 31, i1 false)
+  ret void
+}
+
 declare void @llvm.memcpy.p0.p0.i32(ptr nocapture, ptr nocapture, i32, i1) nounwind
 declare void @llvm.memcpy.p0.p0.i64(ptr nocapture, ptr nocapture, i64, i1) nounwind
diff --git a/llvm/test/CodeGen/ARM/minnum-maxnum-intrinsics.ll b/llvm/test/CodeGen/ARM/minnum-maxnum-intrinsics.ll
index dd33b09fe8300..ec7516524ee67 100644
--- a/llvm/test/CodeGen/ARM/minnum-maxnum-intrinsics.ll
+++ b/llvm/test/CodeGen/ARM/minnum-maxnum-intrinsics.ll
@@ -610,20 +610,20 @@ define <4 x float> @fminnumv432_one_zero_intrinsic(<4 x float> %x) {
 ; ARMV7-LABEL: fminnumv432_one_zero_intrinsic:
 ; ARMV7:       @ %bb.0:
 ; ARMV7-NEXT:    vmov d1, r2, r3
-; ARMV7-NEXT:    vldr s8, .LCPI18_0
+; ARMV7-NEXT:    vldr s4, .LCPI18_0
 ; ARMV7-NEXT:    vmov d0, r0, r1
-; ARMV7-NEXT:    vmov.f32 s10, #-1.000000e+00
+; ARMV7-NEXT:    vmov.f32 s6, #-1.000000e+00
 ; ARMV7-NEXT:    vcmp.f32 s1, #0
 ; ARMV7-NEXT:    vmrs APSR_nzcv, fpscr
-; ARMV7-NEXT:    vmov.f32 s4, s3
-; ARMV7-NEXT:    vmin.f32 d6, d2, d5
-; ARMV7-NEXT:    vmin.f32 d3, d1, d5
-; ARMV7-NEXT:    vmin.f32 d2, d0, d5
-; ARMV7-NEXT:    vmovlt.f32 s8, s1
-; ARMV7-NEXT:    vmov.f32 s5, s8
-; ARMV7-NEXT:    vmov.f32 s7, s12
-; ARMV7-NEXT:    vmov r0, r1, d2
-; ARMV7-NEXT:    vmov r2, r3, d3
+; ARMV7-NEXT:    vmov.f32 s8, s3
+; ARMV7-NEXT:    vmin.f32 d7, d1, d3
+; ARMV7-NEXT:    vmin.f32 d6, d0, d3
+; ARMV7-NEXT:    vmin.f32 d4, d4, d3
+; ARMV7-NEXT:    vmovlt.f32 s4, s1
+; ARMV7-NEXT:    vmov.f32 s13, s4
+; ARMV7-NEXT:    vmov.f32 s15, s8
+; ARMV7-NEXT:    vmov r0, r1, d6
+; ARMV7-NEXT:    vmov r2, r3, d7
 ; ARMV7-NEXT:    bx lr
 ; ARMV7-NEXT:    .p2align 2
 ; ARMV7-NEXT:  @ %bb.1:
diff --git a/llvm/test/CodeGen/ARM/neon-copy.ll b/llvm/test/CodeGen/ARM/neon-copy.ll
index e356b7e2181da..9fdadab39060f 100644
--- a/llvm/test/CodeGen/ARM/neon-copy.ll
+++ b/llvm/test/CodeGen/ARM/neon-copy.ll
@@ -775,7 +775,6 @@ define <2 x i32> @test_vdup_lane_s32(<2 x i32> %v1) #0 {
 define <16 x i8> @test_vdupq_lane_s8(<8 x i8> %v1) #0 {
 ; CHECK-LABEL: test_vdupq_lane_s8:
 ; CHECK:       @ %bb.0:
-; CHECK-NEXT:    @ kill: def $d0 killed $d0 def $q0
 ; CHECK-NEXT:    vdup.8 q0, d0[5]
 ; CHECK-NEXT:    bx lr
   %shuffle = shufflevector <8 x i8> %v1, <8 x i8> undef, <16 x i32> <i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5>
@@ -785,7 +784,6 @@ define <16 x i8> @test_vdupq_lane_s8(<8 x i8> %v1) #0 {
 define <8 x i16> @test_vdupq_lane_s16(<4 x i16> %v1) #0 {
 ; CHECK-LABEL: test_vdupq_lane_s16:
 ; CHECK:       @ %bb.0:
-; CHECK-NEXT:    @ kill: def $d0 killed $d0 def $q0
 ; CHECK-NEXT:    vdup.16 q0, d0[2]
 ; CHECK-NEXT:    bx lr
   %shuffle = shufflevector <4 x i16> %v1, <4 x i16> undef, <8 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
@@ -795,7 +793,6 @@ define <8 x i16> @test_vdupq_lane_s16(<4 x i16> %v1) #0 {
 define <4 x i32> @test_vdupq_lane_s32(<2 x i32> %v1) #0 {
 ; CHECK-LABEL: test_vdupq_lane_s32:
 ; CHECK:       @ %bb.0:
-; CHECK-NEXT:    @ kill: def $d0 killed $d0 def $q0
 ; CHECK-NEXT:    vdup.32 q0, d0[1]
 ; CHECK-NEXT:    bx lr
   %shuffle = shufflevector <2 x i32> %v1, <2 x i32> undef, <4 x i32> <i32 1, i32 1, i32 1, i32 1>
diff --git a/llvm/test/CodeGen/ARM/neon-v8.1a.ll b/llvm/test/CodeGen/ARM/neon-v8.1a.ll
index 2b37d042e3b99..6f4ceba1d667b 100644
--- a/llvm/test/CodeGen/ARM/neon-v8.1a.ll
+++ b/llvm/test/CodeGen/ARM/neon-v8.1a.ll
@@ -138,7 +138,6 @@ entry:
 define arm_aapcs_vfpcc <8 x i16> @test_vqrdmulahq_lane_s16(<8 x i16> %acc, <8 x i16> %x, <4 x i16> %v) {
 ; CHECK-LABEL: test_vqrdmulahq_lane_s16:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    @ kill: def $d4 killed $d4 def $q2
 ; CHECK-NEXT:    vqrdmulh.s16 q8, q1, d4[2]
 ; CHECK-NEXT:    vqadd.s16 q0, q0, q8
 ; CHECK-NEXT:    bx lr
@@ -165,7 +164,6 @@ entry:
 define arm_aapcs_vfpcc <4 x i32> @test_vqrdmulahq_lane_s32(<4 x i32> %acc,<4 x i32> %x, <2 x i32> %v) {
 ; CHECK-LABEL: test_vqrdmulahq_lane_s32:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    @ kill: def $d4 killed $d4 def $q2
 ; CHECK-NEXT:    vqrdmulh.s32 q8, q1, d4[0]
 ; CHECK-NEXT:    vqadd.s32 q0, q0, q8
 ; CHECK-NEXT:    bx lr
@@ -192,7 +190,6 @@ entry:
 define arm_aapcs_vfpcc <8 x i16> @test_vqrdmulshq_lane_s16(<8 x i16> %acc, <8 x i16> %x, <4 x i16> %v) {
 ; CHECK-LABEL: test_vqrdmulshq_lane_s16:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    @ kill: def $d4 killed $d4 def $q2
 ; CHECK-NEXT:    vqrdmulh.s16 q8, q1, d4[2]
 ; CHECK-NEXT:    vqsub.s16 q0, q0, q8
 ; CHECK-NEXT:    bx lr
@@ -219,7 +216,6 @@ entry:
 define arm_aapcs_vfpcc <4 x i32> @test_vqrdmulshq_lane_s32(<4 x i32> %acc,<4 x i32> %x, <2 x i32> %v) {
 ; CHECK-LABEL: test_vqrdmulshq_lane_s32:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    @ kill: def $d4 killed $d4 def $q2
 ; CHECK-NEXT:    vqrdmulh.s32 q8, q1, d4[0]
 ; CHECK-NEXT:    vqsub.s32 q0, q0, q8
 ; CHECK-NEXT:    bx lr
@@ -297,7 +293,6 @@ entry:
 define arm_aapcs_vfpcc <8 x i16> @test_vqrdmlahq_lane_s16(<8 x i16> %a, <8 x i16> %b, <4 x i16> %c) {
 ; CHECK-LABEL: test_vqrdmlahq_lane_s16:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    @ kill: def $d4 killed $d4 def $q2
 ; CHECK-NEXT:    vqrdmlah.s16 q0, q1, d4[3]
 ; CHECK-NEXT:    bx lr
 entry:
@@ -309,7 +304,6 @@ entry:
 define arm_aapcs_vfpcc <4 x i32> @test_vqrdmlahq_lane_s32(<4 x i32> %a, <4 x i32> %b, <2 x i32> %c) {
 ; CHECK-LABEL: test_vqrdmlahq_lane_s32:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    @ kill: def $d4 killed $d4 def $q2
 ; CHECK-NEXT:    vqrdmlah.s32 q0, q1, d4[1]
 ; CHECK-NEXT:    bx lr
 entry:
@@ -383,7 +377,6 @@ entry:
 define arm_aapcs_vfpcc <8 x i16> @test_vqrdmlshq_lane_s16(<8 x i16> %a, <8 x i16> %b, <4 x i16> %c) {
 ; CHECK-LABEL: test_vqrdmlshq_lane_s16:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    @ kill: def $d4 killed $d4 def $q2
 ; CHECK-NEXT:    vqrdmlsh.s16 q0, q1, d4[3]
 ; CHECK-NEXT:    bx lr
 entry:
@@ -395,7 +388,6 @@ entry:
 define arm_aapcs_vfpcc <4 x i32> @test_vqrdmlshq_lane_s32(<4 x i32> %a, <4 x i32> %b, <2 x i32> %c) {
 ; CHECK-LABEL: test_vqrdmlshq_lane_s32:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    @ kill: def $d4 killed $d4 def $q2
 ; CHECK-NEXT:    vqrdmlsh.s32 q0, q1, d4[1]
 ; CHECK-NEXT:    bx lr
 entry:
diff --git a/llvm/test/CodeGen/ARM/pr122670-regression.ll b/llvm/test/CodeGen/ARM/pr122670-regression.ll
new file mode 100644
index 0000000000000..396024150e222
--- /dev/null
+++ b/llvm/test/CodeGen/ARM/pr122670-regression.ll
@@ -0,0 +1,20 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc -mtriple=armv7-unknown-linux-gnueabihf < %s | FileCheck %s
+
+; Check for assertion when extractelement of load optimization
+; involves a result element type different from the input vector
+; element type.
+
+define i8 @baz(ptr %ptr, i32 %arg) {
+; CHECK-LABEL: baz:
+; CHECK:       @ %bb.0: @ %bb
+; CHECK-NEXT:    add r1, r1, r1
+; CHECK-NEXT:    and r1, r1, #3
+; CHECK-NEXT:    ldr r0, [r0, r1, lsl #2]
+; CHECK-NEXT:    bx lr
+bb:
+  %load = load <2 x i64>, ptr %ptr, align 8
+  %extractelement = extractelement <2 x i64> %load, i32 %arg
+  %trunc = trunc i64 %extractelement to i8
+  ret i8 %trunc
+}
diff --git a/llvm/test/CodeGen/ARM/twoaddress-asserts-liveints-undef-use.mir b/llvm/test/CodeGen/ARM/twoaddress-asserts-liveints-undef-use.mir
new file mode 100644
index 0000000000000..6e33d4be9345d
--- /dev/null
+++ b/llvm/test/CodeGen/ARM/twoaddress-asserts-liveints-undef-use.mir
@@ -0,0 +1,24 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 5
+# RUN: llc -mtriple=thumbv8.1m.main-none-none-eabi -mattr=+mve -verify-machineinstrs -run-pass=liveintervals,twoaddressinstruction -o - %s | FileCheck %s
+
+---
+name:      undef_use_asserts_lr
+tracksRegLiveness: true
+noPhis:          true
+body:             |
+  bb.0:
+    liveins: $q0
+
+    ; CHECK-LABEL: name: undef_use_asserts_lr
+    ; CHECK: liveins: $q0
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:mqpr = COPY $q0
+    ; CHECK-NEXT: [[DEF:%[0-9]+]]:spr = IMPLICIT_DEF
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:mqpr = COPY [[COPY]]
+    ; CHECK-NEXT: dead [[COPY1:%[0-9]+]].ssub_2:mqpr = COPY [[DEF]]
+    ; CHECK-NEXT: dead [[COPY2:%[0-9]+]]:spr = COPY undef [[COPY1]].ssub_3
+    %0:mqpr = COPY $q0
+    %1:spr = IMPLICIT_DEF
+    %2:mqpr = INSERT_SUBREG %0, %1, %subreg.ssub_2
+    dead %4:spr = COPY undef %2.ssub_3
+...
diff --git a/llvm/test/CodeGen/ARM/vdup.ll b/llvm/test/CodeGen/ARM/vdup.ll
index 3f6d38c929b98..7d0680184623f 100644
--- a/llvm/test/CodeGen/ARM/vdup.ll
+++ b/llvm/test/CodeGen/ARM/vdup.ll
@@ -258,7 +258,6 @@ define arm_aapcs_vfpcc <2 x float> @vduplanefloat(<2 x float> %A) nounwind {
 define arm_aapcs_vfpcc <16 x i8> @vduplaneQ8(<8 x i8> %A) nounwind {
 ; CHECK-LABEL: vduplaneQ8:
 ; CHECK:       @ %bb.0:
-; CHECK-NEXT:    @ kill: def $d0 killed $d0 def $q0
 ; CHECK-NEXT:    vdup.8 q0, d0[1]
 ; CHECK-NEXT:    mov pc, lr
 	%tmp2 = shufflevector <8 x i8> %A, <8 x i8> undef, <16 x i32> < i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1 >
@@ -268,7 +267,6 @@ define arm_aapcs_vfpcc <16 x i8> @vduplaneQ8(<8 x i8> %A) nounwind {
 define arm_aapcs_vfpcc <8 x i16> @vduplaneQ16(<4 x i16> %A) nounwind {
 ; CHECK-LABEL: vduplaneQ16:
 ; CHECK:       @ %bb.0:
-; CHECK-NEXT:    @ kill: def $d0 killed $d0 def $q0
 ; CHECK-NEXT:    vdup.16 q0, d0[1]
 ; CHECK-NEXT:    mov pc, lr
 	%tmp2 = shufflevector <4 x i16> %A, <4 x i16> undef, <8 x i32> < i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1 >
@@ -278,7 +276,6 @@ define arm_aapcs_vfpcc <8 x i16> @vduplaneQ16(<4 x i16> %A) nounwind {
 define arm_aapcs_vfpcc <4 x i32> @vduplaneQ32(<2 x i32> %A) nounwind {
 ; CHECK-LABEL: vduplaneQ32:
 ; CHECK:       @ %bb.0:
-; CHECK-NEXT:    @ kill: def $d0 killed $d0 def $q0
 ; CHECK-NEXT:    vdup.32 q0, d0[1]
 ; CHECK-NEXT:    mov pc, lr
 	%tmp2 = shufflevector <2 x i32> %A, <2 x i32> undef, <4 x i32> < i32 1, i32 1, i32 1, i32 1 >
@@ -288,7 +285,6 @@ define arm_aapcs_vfpcc <4 x i32> @vduplaneQ32(<2 x i32> %A) nounwind {
 define arm_aapcs_vfpcc <4 x float> @vduplaneQfloat(<2 x float> %A) nounwind {
 ; CHECK-LABEL: vduplaneQfloat:
 ; CHECK:       @ %bb.0:
-; CHECK-NEXT:    @ kill: def $d0 killed $d0 def $q0
 ; CHECK-NEXT:    vdup.32 q0, d0[1]
 ; CHECK-NEXT:    mov pc, lr
 	%tmp2 = shufflevector <2 x float> %A, <2 x float> undef, <4 x i32> < i32 1, i32 1, i32 1, i32 1 >
diff --git a/llvm/test/CodeGen/ARM/vector-DAGCombine.ll b/llvm/test/CodeGen/ARM/vector-DAGCombine.ll
index 7bc7b84439627..920742fffcf1e 100644
--- a/llvm/test/CodeGen/ARM/vector-DAGCombine.ll
+++ b/llvm/test/CodeGen/ARM/vector-DAGCombine.ll
@@ -50,9 +50,9 @@ define void @test_pr22678() {
 define <4 x i32> @test_vmovrrd_combine() nounwind {
 ; CHECK-LABEL: test_vmovrrd_combine:
 ; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    @ implicit-def: $q8
 ; CHECK-NEXT:    mov r0, #0
 ; CHECK-NEXT:    cmp r0, #0
-; CHECK-NEXT:    @ implicit-def: $q8
 ; CHECK-NEXT:    bne .LBB3_2
 ; CHECK-NEXT:  @ %bb.1: @ %bb1.preheader
 ; CHECK-NEXT:    vmov.i32 q8, #0x0
diff --git a/llvm/test/CodeGen/ARM/vext.ll b/llvm/test/CodeGen/ARM/vext.ll
index 7ddf1d02834c3..46f778d3c2c91 100644
--- a/llvm/test/CodeGen/ARM/vext.ll
+++ b/llvm/test/CodeGen/ARM/vext.ll
@@ -257,22 +257,22 @@ define <8 x i16> @test_illegal(ptr %A, ptr %B) nounwind {
 ; CHECK-LABEL: test_illegal:
 ; CHECK:       @ %bb.0:
 ; CHECK-NEXT:    vld1.64 {d16, d17}, [r0]
-; CHECK-NEXT:    vorr d22, d16, d16
+; CHECK-NEXT:    vorr d21, d16, d16
 ; CHECK-NEXT:    vmov.u16 r0, d16[0]
-; CHECK-NEXT:    vorr d23, d16, d16
+; CHECK-NEXT:    vorr d22, d16, d16
 ; CHECK-NEXT:    vmov.u16 r2, d17[3]
 ; CHECK-NEXT:    vmov.u16 r3, d17[1]
 ; CHECK-NEXT:    vld1.64 {d18, d19}, [r1]
 ; CHECK-NEXT:    vmov.u16 r1, d19[1]
-; CHECK-NEXT:    vuzp.16 d22, d23
-; CHECK-NEXT:    vuzp.16 d22, d18
+; CHECK-NEXT:    vuzp.16 d21, d22
+; CHECK-NEXT:    vuzp.16 d21, d18
+; CHECK-NEXT:    vext.16 d16, d16, d18, #3
 ; CHECK-NEXT:    vmov.16 d20[0], r0
 ; CHECK-NEXT:    vmov.16 d20[1], r2
 ; CHECK-NEXT:    vmov.16 d20[2], r3
+; CHECK-NEXT:    vmov r2, r3, d16
 ; CHECK-NEXT:    vmov.16 d20[3], r1
-; CHECK-NEXT:    vext.16 d21, d16, d18, #3
 ; CHECK-NEXT:    vmov r0, r1, d20
-; CHECK-NEXT:    vmov r2, r3, d21
 ; CHECK-NEXT:    mov pc, lr
        %tmp1 = load <8 x i16>, ptr %A
        %tmp2 = load <8 x i16>, ptr %B
diff --git a/llvm/test/CodeGen/ARM/vldmia-sched.mir b/llvm/test/CodeGen/ARM/vldmia-sched.mir
index 9a38d28d6b60c..e0cc088b0396a 100644
--- a/llvm/test/CodeGen/ARM/vldmia-sched.mir
+++ b/llvm/test/CodeGen/ARM/vldmia-sched.mir
@@ -1,4 +1,5 @@
 # RUN: llc -run-pass=post-RA-sched %s -o - | FileCheck %s
+# RUN: llc -passes=post-RA-sched %s -o - | FileCheck %s
 # CHECK: VLDMDIA
 --- |
   target triple = "thumbv7-w64-windows-gnu"
diff --git a/llvm/test/CodeGen/ARM/vmul.ll b/llvm/test/CodeGen/ARM/vmul.ll
index 9915e050de2a6..6f6386a81f9ab 100644
--- a/llvm/test/CodeGen/ARM/vmul.ll
+++ b/llvm/test/CodeGen/ARM/vmul.ll
@@ -130,7 +130,6 @@ entry:
 define arm_aapcs_vfpcc <4 x float> @test_vmulQ_lanef32(<4 x float> %arg0_float32x4_t, <2 x float> %arg1_float32x2_t) nounwind readnone {
 ; CHECK-LABEL: test_vmulQ_lanef32:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    @ kill: def $d2 killed $d2 def $q1
 ; CHECK-NEXT:    vmul.f32 q0, q0, d2[1]
 ; CHECK-NEXT:    bx lr
 entry:
@@ -142,7 +141,6 @@ entry:
 define arm_aapcs_vfpcc <8 x i16> @test_vmulQ_lanes16(<8 x i16> %arg0_int16x8_t, <4 x i16> %arg1_int16x4_t) nounwind readnone {
 ; CHECK-LABEL: test_vmulQ_lanes16:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    @ kill: def $d2 killed $d2 def $q1
 ; CHECK-NEXT:    vmul.i16 q0, q0, d2[1]
 ; CHECK-NEXT:    bx lr
 entry:
@@ -154,7 +152,6 @@ entry:
 define arm_aapcs_vfpcc <4 x i32> @test_vmulQ_lanes32(<4 x i32> %arg0_int32x4_t, <2 x i32> %arg1_int32x2_t) nounwind readnone {
 ; CHECK-LABEL: test_vmulQ_lanes32:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    @ kill: def $d2 killed $d2 def $q1
 ; CHECK-NEXT:    vmul.i32 q0, q0, d2[1]
 ; CHECK-NEXT:    bx lr
 entry:
diff --git a/llvm/test/CodeGen/ARM/vpadd.ll b/llvm/test/CodeGen/ARM/vpadd.ll
index cc1d1e6a15fe7..0597d44bc7cbf 100644
--- a/llvm/test/CodeGen/ARM/vpadd.ll
+++ b/llvm/test/CodeGen/ARM/vpadd.ll
@@ -508,14 +508,14 @@ define <2 x i16> @fromExtendingExtractVectorElt_2i16(<8 x i16> %in) {
 ; CHECK-LABEL: fromExtendingExtractVectorElt_2i16:
 ; CHECK:       @ %bb.0:
 ; CHECK-NEXT:    vmov d16, r0, r1
-; CHECK-NEXT:    vmov.u16 r0, d16[0]
 ; CHECK-NEXT:    vmov.u16 r1, d16[1]
-; CHECK-NEXT:    vmov.u16 r3, d16[3]
+; CHECK-NEXT:    vmov.u16 r0, d16[0]
 ; CHECK-NEXT:    vmov.u16 r2, d16[2]
-; CHECK-NEXT:    vmov.32 d16[0], r0
+; CHECK-NEXT:    vmov.u16 r3, d16[3]
 ; CHECK-NEXT:    vmov.32 d17[0], r1
-; CHECK-NEXT:    vmov.32 d16[1], r2
+; CHECK-NEXT:    vmov.32 d16[0], r0
 ; CHECK-NEXT:    vmov.32 d17[1], r3
+; CHECK-NEXT:    vmov.32 d16[1], r2
 ; CHECK-NEXT:    vadd.i32 d16, d17, d16
 ; CHECK-NEXT:    vmov r0, r1, d16
 ; CHECK-NEXT:    mov pc, lr
diff --git a/llvm/test/CodeGen/ARM/vuzp.ll b/llvm/test/CodeGen/ARM/vuzp.ll
index d54446a431ee9..7e1dfba34db2e 100644
--- a/llvm/test/CodeGen/ARM/vuzp.ll
+++ b/llvm/test/CodeGen/ARM/vuzp.ll
@@ -21,11 +21,11 @@ define <8 x i8> @vuzpi8(ptr %A, ptr %B) nounwind {
 define <16 x i8> @vuzpi8_Qres(ptr %A, ptr %B) nounwind {
 ; CHECK-LABEL: vuzpi8_Qres:
 ; CHECK:       @ %bb.0:
-; CHECK-NEXT:    vldr d17, [r1]
-; CHECK-NEXT:    vldr d16, [r0]
-; CHECK-NEXT:    vuzp.8 d16, d17
-; CHECK-NEXT:    vmov r0, r1, d16
-; CHECK-NEXT:    vmov r2, r3, d17
+; CHECK-NEXT:    vldr d16, [r1]
+; CHECK-NEXT:    vldr d17, [r0]
+; CHECK-NEXT:    vuzp.8 d17, d16
+; CHECK-NEXT:    vmov r0, r1, d17
+; CHECK-NEXT:    vmov r2, r3, d16
 ; CHECK-NEXT:    mov pc, lr
 	%tmp1 = load <8 x i8>, ptr %A
 	%tmp2 = load <8 x i8>, ptr %B
@@ -53,11 +53,11 @@ define <4 x i16> @vuzpi16(ptr %A, ptr %B) nounwind {
 define <8 x i16> @vuzpi16_Qres(ptr %A, ptr %B) nounwind {
 ; CHECK-LABEL: vuzpi16_Qres:
 ; CHECK:       @ %bb.0:
-; CHECK-NEXT:    vldr d17, [r1]
-; CHECK-NEXT:    vldr d16, [r0]
-; CHECK-NEXT:    vuzp.16 d16, d17
-; CHECK-NEXT:    vmov r0, r1, d16
-; CHECK-NEXT:    vmov r2, r3, d17
+; CHECK-NEXT:    vldr d16, [r1]
+; CHECK-NEXT:    vldr d17, [r0]
+; CHECK-NEXT:    vuzp.16 d17, d16
+; CHECK-NEXT:    vmov r0, r1, d17
+; CHECK-NEXT:    vmov r2, r3, d16
 ; CHECK-NEXT:    mov pc, lr
 	%tmp1 = load <4 x i16>, ptr %A
 	%tmp2 = load <4 x i16>, ptr %B
@@ -221,11 +221,11 @@ define <8 x i8> @vuzpi8_undef(ptr %A, ptr %B) nounwind {
 define <16 x i8> @vuzpi8_undef_Qres(ptr %A, ptr %B) nounwind {
 ; CHECK-LABEL: vuzpi8_undef_Qres:
 ; CHECK:       @ %bb.0:
-; CHECK-NEXT:    vldr d17, [r1]
-; CHECK-NEXT:    vldr d16, [r0]
-; CHECK-NEXT:    vuzp.8 d16, d17
-; CHECK-NEXT:    vmov r0, r1, d16
-; CHECK-NEXT:    vmov r2, r3, d17
+; CHECK-NEXT:    vldr d16, [r1]
+; CHECK-NEXT:    vldr d17, [r0]
+; CHECK-NEXT:    vuzp.8 d17, d16
+; CHECK-NEXT:    vmov r0, r1, d17
+; CHECK-NEXT:    vmov r2, r3, d16
 ; CHECK-NEXT:    mov pc, lr
 	%tmp1 = load <8 x i8>, ptr %A
 	%tmp2 = load <8 x i8>, ptr %B
@@ -269,11 +269,11 @@ define <16 x i16> @vuzpQi16_undef_QQres(ptr %A, ptr %B) nounwind {
 define <8 x i16> @vuzp_lower_shufflemask_undef(ptr %A, ptr %B) {
 ; CHECK-LABEL: vuzp_lower_shufflemask_undef:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vldr d17, [r1]
-; CHECK-NEXT:    vldr d18, [r0]
-; CHECK-NEXT:    vuzp.16 d18, d17
+; CHECK-NEXT:    vldr d16, [r1]
+; CHECK-NEXT:    vldr d17, [r0]
+; CHECK-NEXT:    vuzp.16 d17, d16
 ; CHECK-NEXT:    vmov r0, r1, d16
-; CHECK-NEXT:    vmov r2, r3, d17
+; CHECK-NEXT:    vmov r2, r3, d16
 ; CHECK-NEXT:    mov pc, lr
 entry:
 	%tmp1 = load <4 x i16>, ptr %A
@@ -285,13 +285,13 @@ entry:
 define <4 x i32> @vuzp_lower_shufflemask_zeroed(ptr %A, ptr %B) {
 ; CHECK-LABEL: vuzp_lower_shufflemask_zeroed:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vldr d18, [r0]
-; CHECK-NEXT:    vorr d19, d18, d18
-; CHECK-NEXT:    vldr d17, [r1]
-; CHECK-NEXT:    vtrn.32 d19, d17
-; CHECK-NEXT:    vdup.32 d16, d18[0]
-; CHECK-NEXT:    vmov r2, r3, d17
-; CHECK-NEXT:    vmov r0, r1, d16
+; CHECK-NEXT:    vldr d17, [r0]
+; CHECK-NEXT:    vorr d18, d17, d17
+; CHECK-NEXT:    vldr d16, [r1]
+; CHECK-NEXT:    vdup.32 d17, d17[0]
+; CHECK-NEXT:    vtrn.32 d18, d16
+; CHECK-NEXT:    vmov r0, r1, d17
+; CHECK-NEXT:    vmov r2, r3, d16
 ; CHECK-NEXT:    mov pc, lr
 entry:
   %tmp1 = load <2 x i32>, ptr %A
diff --git a/llvm/test/CodeGen/ARM/vzip.ll b/llvm/test/CodeGen/ARM/vzip.ll
index 68f00a23da177..dda774abd8516 100644
--- a/llvm/test/CodeGen/ARM/vzip.ll
+++ b/llvm/test/CodeGen/ARM/vzip.ll
@@ -21,11 +21,11 @@ define <8 x i8> @vzipi8(ptr %A, ptr %B) nounwind {
 define <16 x i8> @vzipi8_Qres(ptr %A, ptr %B) nounwind {
 ; CHECK-LABEL: vzipi8_Qres:
 ; CHECK:       @ %bb.0:
-; CHECK-NEXT:    vldr d17, [r1]
-; CHECK-NEXT:    vldr d16, [r0]
-; CHECK-NEXT:    vzip.8 d16, d17
-; CHECK-NEXT:    vmov r0, r1, d16
-; CHECK-NEXT:    vmov r2, r3, d17
+; CHECK-NEXT:    vldr d16, [r1]
+; CHECK-NEXT:    vldr d17, [r0]
+; CHECK-NEXT:    vzip.8 d17, d16
+; CHECK-NEXT:    vmov r0, r1, d17
+; CHECK-NEXT:    vmov r2, r3, d16
 ; CHECK-NEXT:    mov pc, lr
 	%tmp1 = load <8 x i8>, ptr %A
 	%tmp2 = load <8 x i8>, ptr %B
@@ -53,11 +53,11 @@ define <4 x i16> @vzipi16(ptr %A, ptr %B) nounwind {
 define <8 x i16> @vzipi16_Qres(ptr %A, ptr %B) nounwind {
 ; CHECK-LABEL: vzipi16_Qres:
 ; CHECK:       @ %bb.0:
-; CHECK-NEXT:    vldr d17, [r1]
-; CHECK-NEXT:    vldr d16, [r0]
-; CHECK-NEXT:    vzip.16 d16, d17
-; CHECK-NEXT:    vmov r0, r1, d16
-; CHECK-NEXT:    vmov r2, r3, d17
+; CHECK-NEXT:    vldr d16, [r1]
+; CHECK-NEXT:    vldr d17, [r0]
+; CHECK-NEXT:    vzip.16 d17, d16
+; CHECK-NEXT:    vmov r0, r1, d17
+; CHECK-NEXT:    vmov r2, r3, d16
 ; CHECK-NEXT:    mov pc, lr
 	%tmp1 = load <4 x i16>, ptr %A
 	%tmp2 = load <4 x i16>, ptr %B
@@ -221,11 +221,11 @@ define <8 x i8> @vzipi8_undef(ptr %A, ptr %B) nounwind {
 define <16 x i8> @vzipi8_undef_Qres(ptr %A, ptr %B) nounwind {
 ; CHECK-LABEL: vzipi8_undef_Qres:
 ; CHECK:       @ %bb.0:
-; CHECK-NEXT:    vldr d17, [r1]
-; CHECK-NEXT:    vldr d16, [r0]
-; CHECK-NEXT:    vzip.8 d16, d17
-; CHECK-NEXT:    vmov r0, r1, d16
-; CHECK-NEXT:    vmov r2, r3, d17
+; CHECK-NEXT:    vldr d16, [r1]
+; CHECK-NEXT:    vldr d17, [r0]
+; CHECK-NEXT:    vzip.8 d17, d16
+; CHECK-NEXT:    vmov r0, r1, d17
+; CHECK-NEXT:    vmov r2, r3, d16
 ; CHECK-NEXT:    mov pc, lr
 	%tmp1 = load <8 x i8>, ptr %A
 	%tmp2 = load <8 x i8>, ptr %B
@@ -269,11 +269,11 @@ define <32 x i8> @vzipQi8_undef_QQres(ptr %A, ptr %B) nounwind {
 define <8 x i16> @vzip_lower_shufflemask_undef(ptr %A, ptr %B) {
 ; CHECK-LABEL: vzip_lower_shufflemask_undef:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vldr d17, [r1]
-; CHECK-NEXT:    vldr d18, [r0]
-; CHECK-NEXT:    vzip.16 d18, d17
+; CHECK-NEXT:    vldr d16, [r1]
+; CHECK-NEXT:    vldr d17, [r0]
+; CHECK-NEXT:    vzip.16 d17, d16
 ; CHECK-NEXT:    vmov r0, r1, d16
-; CHECK-NEXT:    vmov r2, r3, d17
+; CHECK-NEXT:    vmov r2, r3, d16
 ; CHECK-NEXT:    mov pc, lr
 entry:
 	%tmp1 = load <4 x i16>, ptr %A
@@ -289,10 +289,10 @@ define <8 x i16> @vzip_lower_shufflemask_undef_rev(ptr %A, ptr %B) {
 ; CHECK-LABEL: vzip_lower_shufflemask_undef_rev:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    vldr d16, [r1]
-; CHECK-NEXT:    vldr d19, [r0]
-; CHECK-NEXT:    vtrn.16 d19, d16
-; CHECK-NEXT:    vmov r0, r1, d18
-; CHECK-NEXT:    vmov r2, r3, d19
+; CHECK-NEXT:    vldr d17, [r0]
+; CHECK-NEXT:    vtrn.16 d17, d16
+; CHECK-NEXT:    vmov r0, r1, d16
+; CHECK-NEXT:    vmov r2, r3, d17
 ; CHECK-NEXT:    mov pc, lr
 entry:
   %tmp1 = load <4 x i16>, ptr %A
diff --git a/llvm/test/CodeGen/AVR/return.ll b/llvm/test/CodeGen/AVR/return.ll
index 8cb9f1427c77a..207ad2f23a737 100644
--- a/llvm/test/CodeGen/AVR/return.ll
+++ b/llvm/test/CodeGen/AVR/return.ll
@@ -187,26 +187,26 @@ define i64 @return64_imm() {
 ; TINY-NEXT:    st Z+, r21
 ; TINY-NEXT:    subi r30, 8
 ; TINY-NEXT:    sbci r31, 0
-; TINY-NEXT:    ldi r24, 25
-; TINY-NEXT:    ldi r25, 22
+; TINY-NEXT:    ldi r20, 25
+; TINY-NEXT:    ldi r21, 22
 ; TINY-NEXT:    subi r30, 252
 ; TINY-NEXT:    sbci r31, 255
-; TINY-NEXT:    st Z+, r24
-; TINY-NEXT:    st Z+, r25
+; TINY-NEXT:    st Z+, r20
+; TINY-NEXT:    st Z+, r21
 ; TINY-NEXT:    subi r30, 6
 ; TINY-NEXT:    sbci r31, 0
-; TINY-NEXT:    ldi r24, 104
-; TINY-NEXT:    ldi r25, 37
+; TINY-NEXT:    ldi r20, 104
+; TINY-NEXT:    ldi r21, 37
 ; TINY-NEXT:    subi r30, 254
 ; TINY-NEXT:    sbci r31, 255
-; TINY-NEXT:    st Z+, r24
-; TINY-NEXT:    st Z+, r25
+; TINY-NEXT:    st Z+, r20
+; TINY-NEXT:    st Z+, r21
 ; TINY-NEXT:    subi r30, 4
 ; TINY-NEXT:    sbci r31, 0
-; TINY-NEXT:    ldi r24, 204
-; TINY-NEXT:    ldi r25, 204
-; TINY-NEXT:    st Z+, r24
-; TINY-NEXT:    st Z+, r25
+; TINY-NEXT:    ldi r20, 204
+; TINY-NEXT:    ldi r21, 204
+; TINY-NEXT:    st Z+, r20
+; TINY-NEXT:    st Z+, r21
 ; TINY-NEXT:    ret
     ret i64 13757395258967641292
 }
@@ -243,8 +243,8 @@ define i64 @return64_arg(i64 %x) {
 ; TINY-NEXT:    in r16, 63
 ; TINY-NEXT:    subi r28, 247
 ; TINY-NEXT:    sbci r29, 255
-; TINY-NEXT:    ld r24, Y+
-; TINY-NEXT:    ld r25, Y+
+; TINY-NEXT:    ld r20, Y+
+; TINY-NEXT:    ld r21, Y+
 ; TINY-NEXT:    subi r28, 2
 ; TINY-NEXT:    sbci r29, 0
 ; TINY-NEXT:    subi r28, 9
@@ -252,15 +252,15 @@ define i64 @return64_arg(i64 %x) {
 ; TINY-NEXT:    out 63, r16
 ; TINY-NEXT:    subi r30, 252
 ; TINY-NEXT:    sbci r31, 255
-; TINY-NEXT:    st Z+, r24
-; TINY-NEXT:    st Z+, r25
+; TINY-NEXT:    st Z+, r20
+; TINY-NEXT:    st Z+, r21
 ; TINY-NEXT:    subi r30, 6
 ; TINY-NEXT:    sbci r31, 0
 ; TINY-NEXT:    in r16, 63
 ; TINY-NEXT:    subi r28, 249
 ; TINY-NEXT:    sbci r29, 255
-; TINY-NEXT:    ld r24, Y+
-; TINY-NEXT:    ld r25, Y+
+; TINY-NEXT:    ld r20, Y+
+; TINY-NEXT:    ld r21, Y+
 ; TINY-NEXT:    subi r28, 2
 ; TINY-NEXT:    sbci r29, 0
 ; TINY-NEXT:    subi r28, 7
@@ -268,22 +268,22 @@ define i64 @return64_arg(i64 %x) {
 ; TINY-NEXT:    out 63, r16
 ; TINY-NEXT:    subi r30, 254
 ; TINY-NEXT:    sbci r31, 255
-; TINY-NEXT:    st Z+, r24
-; TINY-NEXT:    st Z+, r25
+; TINY-NEXT:    st Z+, r20
+; TINY-NEXT:    st Z+, r21
 ; TINY-NEXT:    subi r30, 4
 ; TINY-NEXT:    sbci r31, 0
 ; TINY-NEXT:    in r16, 63
 ; TINY-NEXT:    subi r28, 251
 ; TINY-NEXT:    sbci r29, 255
-; TINY-NEXT:    ld r24, Y+
-; TINY-NEXT:    ld r25, Y+
+; TINY-NEXT:    ld r20, Y+
+; TINY-NEXT:    ld r21, Y+
 ; TINY-NEXT:    subi r28, 2
 ; TINY-NEXT:    sbci r29, 0
 ; TINY-NEXT:    subi r28, 5
 ; TINY-NEXT:    sbci r29, 0
 ; TINY-NEXT:    out 63, r16
-; TINY-NEXT:    st Z+, r24
-; TINY-NEXT:    st Z+, r25
+; TINY-NEXT:    st Z+, r20
+; TINY-NEXT:    st Z+, r21
 ; TINY-NEXT:    pop r29
 ; TINY-NEXT:    pop r28
 ; TINY-NEXT:    ret
@@ -336,8 +336,8 @@ define i64 @return64_arg2(i64 %x, i64 %y, i64 %z) {
 ; TINY-NEXT:    in r16, 63
 ; TINY-NEXT:    subi r28, 231
 ; TINY-NEXT:    sbci r29, 255
-; TINY-NEXT:    ld r24, Y+
-; TINY-NEXT:    ld r25, Y+
+; TINY-NEXT:    ld r20, Y+
+; TINY-NEXT:    ld r21, Y+
 ; TINY-NEXT:    subi r28, 2
 ; TINY-NEXT:    sbci r29, 0
 ; TINY-NEXT:    subi r28, 25
@@ -345,15 +345,15 @@ define i64 @return64_arg2(i64 %x, i64 %y, i64 %z) {
 ; TINY-NEXT:    out 63, r16
 ; TINY-NEXT:    subi r30, 252
 ; TINY-NEXT:    sbci r31, 255
-; TINY-NEXT:    st Z+, r24
-; TINY-NEXT:    st Z+, r25
+; TINY-NEXT:    st Z+, r20
+; TINY-NEXT:    st Z+, r21
 ; TINY-NEXT:    subi r30, 6
 ; TINY-NEXT:    sbci r31, 0
 ; TINY-NEXT:    in r16, 63
 ; TINY-NEXT:    subi r28, 233
 ; TINY-NEXT:    sbci r29, 255
-; TINY-NEXT:    ld r24, Y+
-; TINY-NEXT:    ld r25, Y+
+; TINY-NEXT:    ld r20, Y+
+; TINY-NEXT:    ld r21, Y+
 ; TINY-NEXT:    subi r28, 2
 ; TINY-NEXT:    sbci r29, 0
 ; TINY-NEXT:    subi r28, 23
@@ -361,22 +361,22 @@ define i64 @return64_arg2(i64 %x, i64 %y, i64 %z) {
 ; TINY-NEXT:    out 63, r16
 ; TINY-NEXT:    subi r30, 254
 ; TINY-NEXT:    sbci r31, 255
-; TINY-NEXT:    st Z+, r24
-; TINY-NEXT:    st Z+, r25
+; TINY-NEXT:    st Z+, r20
+; TINY-NEXT:    st Z+, r21
 ; TINY-NEXT:    subi r30, 4
 ; TINY-NEXT:    sbci r31, 0
 ; TINY-NEXT:    in r16, 63
 ; TINY-NEXT:    subi r28, 235
 ; TINY-NEXT:    sbci r29, 255
-; TINY-NEXT:    ld r24, Y+
-; TINY-NEXT:    ld r25, Y+
+; TINY-NEXT:    ld r20, Y+
+; TINY-NEXT:    ld r21, Y+
 ; TINY-NEXT:    subi r28, 2
 ; TINY-NEXT:    sbci r29, 0
 ; TINY-NEXT:    subi r28, 21
 ; TINY-NEXT:    sbci r29, 0
 ; TINY-NEXT:    out 63, r16
-; TINY-NEXT:    st Z+, r24
-; TINY-NEXT:    st Z+, r25
+; TINY-NEXT:    st Z+, r20
+; TINY-NEXT:    st Z+, r21
 ; TINY-NEXT:    pop r29
 ; TINY-NEXT:    pop r28
 ; TINY-NEXT:    ret
diff --git a/llvm/test/CodeGen/BPF/is_trunc_free.ll b/llvm/test/CodeGen/BPF/is_trunc_free.ll
index 21c8a2aebeda1..fe00731928625 100644
--- a/llvm/test/CodeGen/BPF/is_trunc_free.ll
+++ b/llvm/test/CodeGen/BPF/is_trunc_free.ll
@@ -58,7 +58,6 @@ cleanup:                                          ; preds = %entry, %if.end10
 }
 
 ; CHECK: w{{[0-9]+}} = *(u32 *)(r{{[0-9]+}} + 0)
-; CHECK-NOT: w{{[0-9]+}} = w{{[0-9]+}}
 
 declare dso_local i32 @work(ptr, i32) local_unnamed_addr #1
 
diff --git a/llvm/test/CodeGen/BPF/loop-exit-cond.ll b/llvm/test/CodeGen/BPF/loop-exit-cond.ll
index df6a2489a432c..69fe7148daa74 100644
--- a/llvm/test/CodeGen/BPF/loop-exit-cond.ll
+++ b/llvm/test/CodeGen/BPF/loop-exit-cond.ll
@@ -27,7 +27,7 @@ target triple = "bpf"
 ; Function Attrs: nounwind
 define dso_local i32 @test(i32 %len, ptr %data) #0 {
 ; CHECK-LABEL: define dso_local noundef i32 @test(
-; CHECK-SAME: i32 [[LEN:%.*]], ptr nocapture readonly [[DATA:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
+; CHECK-SAME: i32 [[LEN:%.*]], ptr readonly captures(none) [[DATA:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[D:%.*]] = alloca [1 x i64], align 8
 ; CHECK-NEXT:    [[TMP0:%.*]] = add i32 [[LEN]], -2
diff --git a/llvm/test/CodeGen/BPF/may_goto_1.ll b/llvm/test/CodeGen/BPF/may_goto_1.ll
new file mode 100644
index 0000000000000..a47716509c7ab
--- /dev/null
+++ b/llvm/test/CodeGen/BPF/may_goto_1.ll
@@ -0,0 +1,27 @@
+; RUN: llc -mtriple=bpfel -mcpu=v3 -filetype=obj -o - %s | llvm-objdump --no-show-raw-insn -d - | FileCheck %s
+
+@j = dso_local local_unnamed_addr global i32 0, align 4
+
+define dso_local noundef i32 @foo() local_unnamed_addr {
+entry:
+  callbr void asm sideeffect "may_goto ${0:l}", "!i"()
+          to label %for.body [label %for.cond.cleanup]
+
+for.cond.cleanup:                                 ; preds = %for.body.2, %for.body.2, %for.body.1, %for.body, %entry
+  ret i32 0
+
+for.body:                                         ; preds = %entry
+  callbr void asm sideeffect "may_goto ${0:l}", "!i"()
+          to label %for.body.1 [label %for.cond.cleanup]
+
+for.body.1:                                       ; preds = %for.body
+  callbr void asm sideeffect "may_goto ${0:l}", "!i"()
+          to label %for.body.2 [label %for.cond.cleanup]
+
+for.body.2:                                       ; preds = %for.body.1
+  callbr void asm sideeffect "may_goto ${0:l}", "!i"()
+          to label %for.cond.cleanup [label %for.cond.cleanup]
+}
+
+; CHECK:       0:       w0 = 0x0
+; CHECK-NEXT:  1:       exit
diff --git a/llvm/test/CodeGen/BPF/may_goto_2.ll b/llvm/test/CodeGen/BPF/may_goto_2.ll
new file mode 100644
index 0000000000000..8f6fea5913d2a
--- /dev/null
+++ b/llvm/test/CodeGen/BPF/may_goto_2.ll
@@ -0,0 +1,27 @@
+; RUN: llc -mtriple=bpfel -mcpu=v3 -filetype=obj -o - %s | llvm-objdump --no-show-raw-insn -d - | FileCheck %s
+
+@j = dso_local local_unnamed_addr global i32 0, align 4
+
+define dso_local noundef i32 @foo() local_unnamed_addr {
+entry:
+  callbr void asm sideeffect "may_goto $0", "!i"()
+          to label %for.body [label %for.cond.cleanup]
+
+for.cond.cleanup:                                 ; preds = %for.body.2, %for.body.2, %for.body.1, %for.body, %entry
+  ret i32 0
+
+for.body:                                         ; preds = %entry
+  callbr void asm sideeffect "may_goto $0", "!i"()
+          to label %for.body.1 [label %for.cond.cleanup]
+
+for.body.1:                                       ; preds = %for.body
+  callbr void asm sideeffect "may_goto $0", "!i"()
+          to label %for.body.2 [label %for.cond.cleanup]
+
+for.body.2:                                       ; preds = %for.body.1
+  callbr void asm sideeffect "may_goto $0", "!i"()
+          to label %for.cond.cleanup [label %for.cond.cleanup]
+}
+
+; CHECK:       0:       w0 = 0x0
+; CHECK-NEXT:  1:       exit
diff --git a/llvm/test/CodeGen/BPF/preserve-static-offset/load-inline.ll b/llvm/test/CodeGen/BPF/preserve-static-offset/load-inline.ll
index b43c73035b332..48fe1f5512eb0 100644
--- a/llvm/test/CodeGen/BPF/preserve-static-offset/load-inline.ll
+++ b/llvm/test/CodeGen/BPF/preserve-static-offset/load-inline.ll
@@ -54,7 +54,7 @@ entry:
   ret void
 }
 
-; CHECK:      define dso_local void @quux(ptr nocapture noundef readonly %[[p:.*]])
+; CHECK:      define dso_local void @quux(ptr noundef readonly captures(none) %[[p:.*]])
 ; CHECK:        %[[bb_i1:.*]] = tail call i32 (ptr, i1, i8, i8, i8, i1, ...)
 ; CHECK-SAME:     @llvm.bpf.getelementptr.and.load.i32
 ; CHECK-SAME:       (ptr readonly elementtype(i8) %[[p]],
diff --git a/llvm/test/CodeGen/BPF/preserve-static-offset/load-simple.ll b/llvm/test/CodeGen/BPF/preserve-static-offset/load-simple.ll
index 03ae7f3272dcf..42885f7de96cf 100644
--- a/llvm/test/CodeGen/BPF/preserve-static-offset/load-simple.ll
+++ b/llvm/test/CodeGen/BPF/preserve-static-offset/load-simple.ll
@@ -45,7 +45,7 @@ entry:
 ; CHECK-NEXT: call void @consume(i32 noundef %[[a1]])
 
 ; CHECK:      declare i32
-; CHECK-SAME:    @llvm.bpf.getelementptr.and.load.i32(ptr nocapture, {{.*}}) #[[v2:.*]]
+; CHECK-SAME:    @llvm.bpf.getelementptr.and.load.i32(ptr captures(none), {{.*}}) #[[v2:.*]]
 
 ; CHECK:      attributes #[[v2]] = { nocallback nofree nounwind willreturn }
 ; CHECK:      attributes #[[v1]] = { memory(argmem: read) }
diff --git a/llvm/test/CodeGen/BPF/preserve-static-offset/load-unroll-inline.ll b/llvm/test/CodeGen/BPF/preserve-static-offset/load-unroll-inline.ll
index 1f0be06755b44..c05727f7b3056 100644
--- a/llvm/test/CodeGen/BPF/preserve-static-offset/load-unroll-inline.ll
+++ b/llvm/test/CodeGen/BPF/preserve-static-offset/load-unroll-inline.ll
@@ -72,7 +72,7 @@ entry:
   ret void
 }
 
-; CHECK:      define dso_local void @quux(ptr nocapture noundef readonly %[[p:.*]])
+; CHECK:      define dso_local void @quux(ptr noundef readonly captures(none) %[[p:.*]])
 ; CHECK:        %[[v1:.*]] = tail call i32 (ptr, i1, i8, i8, i8, i1, ...)
 ; CHECK-SAME:     @llvm.bpf.getelementptr.and.load.i32
 ; CHECK-SAME:       (ptr readonly elementtype(i8) %[[p]],
diff --git a/llvm/test/CodeGen/BPF/preserve-static-offset/load-unroll.ll b/llvm/test/CodeGen/BPF/preserve-static-offset/load-unroll.ll
index 2409fbc8b7e85..6bdb55783f828 100644
--- a/llvm/test/CodeGen/BPF/preserve-static-offset/load-unroll.ll
+++ b/llvm/test/CodeGen/BPF/preserve-static-offset/load-unroll.ll
@@ -50,7 +50,7 @@ while.end:                                        ; preds = %while.cond
   ret void
 }
 
-; CHECK:      define dso_local void @bar(ptr nocapture noundef readonly %[[p:.*]])
+; CHECK:      define dso_local void @bar(ptr noundef readonly captures(none) %[[p:.*]])
 ; CHECK:        %[[v1:.*]] = tail call i32 (ptr, i1, i8, i8, i8, i1, ...)
 ; CHECK-SAME:     @llvm.bpf.getelementptr.and.load.i32
 ; CHECK-SAME:       (ptr readonly elementtype(i8) %[[p]],
diff --git a/llvm/test/CodeGen/BPF/preserve-static-offset/store-align.ll b/llvm/test/CodeGen/BPF/preserve-static-offset/store-align.ll
index 667f8f5a8d8b4..8ebaf0b82603a 100644
--- a/llvm/test/CodeGen/BPF/preserve-static-offset/store-align.ll
+++ b/llvm/test/CodeGen/BPF/preserve-static-offset/store-align.ll
@@ -32,7 +32,7 @@ entry:
   ret void
 }
 
-; CHECK:      define dso_local void @bar(ptr nocapture noundef writeonly %[[p:.*]])
+; CHECK:      define dso_local void @bar(ptr noundef writeonly captures(none) %[[p:.*]])
 ; CHECK:        tail call void (i32, ptr, i1, i8, i8, i8, i1, ...)
 ; CHECK-SAME:     @llvm.bpf.getelementptr.and.store.i32
 ; CHECK-SAME:       (i32 7,
diff --git a/llvm/test/CodeGen/BPF/preserve-static-offset/store-atomic.ll b/llvm/test/CodeGen/BPF/preserve-static-offset/store-atomic.ll
index 443966337b9da..c3ec8159be1e6 100644
--- a/llvm/test/CodeGen/BPF/preserve-static-offset/store-atomic.ll
+++ b/llvm/test/CodeGen/BPF/preserve-static-offset/store-atomic.ll
@@ -31,7 +31,7 @@ entry:
   ret void
 }
 
-; CHECK:      define dso_local void @bar(ptr nocapture noundef %[[p:.*]])
+; CHECK:      define dso_local void @bar(ptr noundef captures(none) %[[p:.*]])
 ; CHECK:        tail call void (i32, ptr, i1, i8, i8, i8, i1, ...)
 ; CHECK-SAME:     @llvm.bpf.getelementptr.and.store.i32
 ; CHECK-SAME:       (i32 7,
diff --git a/llvm/test/CodeGen/BPF/preserve-static-offset/store-chain-oob.ll b/llvm/test/CodeGen/BPF/preserve-static-offset/store-chain-oob.ll
index e2878f0913035..a5301240b7c8e 100644
--- a/llvm/test/CodeGen/BPF/preserve-static-offset/store-chain-oob.ll
+++ b/llvm/test/CodeGen/BPF/preserve-static-offset/store-chain-oob.ll
@@ -39,7 +39,7 @@ entry:
   ret void
 }
 
-; CHECK:      define dso_local void @buz(ptr nocapture noundef writeonly %[[p:.*]])
+; CHECK:      define dso_local void @buz(ptr noundef writeonly captures(none) %[[p:.*]])
 ; CHECK:        tail call void (i32, ptr, i1, i8, i8, i8, i1, ...)
 ; CHECK-SAME:     @llvm.bpf.getelementptr.and.store.i32
 ; CHECK-SAME:       (i32 42,
diff --git a/llvm/test/CodeGen/BPF/preserve-static-offset/store-chain-u8-oob.ll b/llvm/test/CodeGen/BPF/preserve-static-offset/store-chain-u8-oob.ll
index a337325466776..59a65bfc9ed5f 100644
--- a/llvm/test/CodeGen/BPF/preserve-static-offset/store-chain-u8-oob.ll
+++ b/llvm/test/CodeGen/BPF/preserve-static-offset/store-chain-u8-oob.ll
@@ -41,7 +41,7 @@ entry:
   ret void
 }
 
-; CHECK:      define dso_local void @buz(ptr nocapture noundef writeonly %[[p:.*]])
+; CHECK:      define dso_local void @buz(ptr noundef writeonly captures(none) %[[p:.*]])
 ; CHECK:        tail call void (i8, ptr, i1, i8, i8, i8, i1, ...)
 ; CHECK-SAME:     @llvm.bpf.getelementptr.and.store.i8
 ; CHECK-SAME:       (i8 42,
diff --git a/llvm/test/CodeGen/BPF/preserve-static-offset/store-chain-u8.ll b/llvm/test/CodeGen/BPF/preserve-static-offset/store-chain-u8.ll
index 92740603ae69b..b3f460f71404f 100644
--- a/llvm/test/CodeGen/BPF/preserve-static-offset/store-chain-u8.ll
+++ b/llvm/test/CodeGen/BPF/preserve-static-offset/store-chain-u8.ll
@@ -42,7 +42,7 @@ entry:
   ret void
 }
 
-; CHECK:      define dso_local void @buz(ptr nocapture noundef writeonly %[[p:.*]])
+; CHECK:      define dso_local void @buz(ptr noundef writeonly captures(none) %[[p:.*]])
 ; CHECK:        tail call void (i8, ptr, i1, i8, i8, i8, i1, ...)
 ; CHECK-SAME:     @llvm.bpf.getelementptr.and.store.i8
 ; CHECK-SAME:       (i8 42,
diff --git a/llvm/test/CodeGen/BPF/preserve-static-offset/store-chain.ll b/llvm/test/CodeGen/BPF/preserve-static-offset/store-chain.ll
index d4c90616bf5cb..437ba8f719d71 100644
--- a/llvm/test/CodeGen/BPF/preserve-static-offset/store-chain.ll
+++ b/llvm/test/CodeGen/BPF/preserve-static-offset/store-chain.ll
@@ -36,7 +36,7 @@ entry:
   ret void
 }
 
-; CHECK:      define dso_local void @buz(ptr nocapture noundef writeonly %[[p:.*]])
+; CHECK:      define dso_local void @buz(ptr noundef writeonly captures(none) %[[p:.*]])
 ; CHECK:        tail call void (i32, ptr, i1, i8, i8, i8, i1, ...)
 ; CHECK-SAME:     @llvm.bpf.getelementptr.and.store.i32
 ; CHECK-SAME:       (i32 42,
diff --git a/llvm/test/CodeGen/BPF/preserve-static-offset/store-simple.ll b/llvm/test/CodeGen/BPF/preserve-static-offset/store-simple.ll
index a603ad8667394..9c5821c53b119 100644
--- a/llvm/test/CodeGen/BPF/preserve-static-offset/store-simple.ll
+++ b/llvm/test/CodeGen/BPF/preserve-static-offset/store-simple.ll
@@ -33,7 +33,7 @@ entry:
   ret void
 }
 
-; CHECK:      define dso_local void @bar(ptr nocapture noundef writeonly %[[p:.*]])
+; CHECK:      define dso_local void @bar(ptr noundef writeonly captures(none) %[[p:.*]])
 ; CHECK:        tail call void (i32, ptr, i1, i8, i8, i8, i1, ...)
 ; CHECK-SAME:     @llvm.bpf.getelementptr.and.store.i32
 ; CHECK-SAME:       (i32 7,
diff --git a/llvm/test/CodeGen/BPF/preserve-static-offset/store-unroll-inline.ll b/llvm/test/CodeGen/BPF/preserve-static-offset/store-unroll-inline.ll
index 3c6280ca75b64..9774ca48d5b38 100644
--- a/llvm/test/CodeGen/BPF/preserve-static-offset/store-unroll-inline.ll
+++ b/llvm/test/CodeGen/BPF/preserve-static-offset/store-unroll-inline.ll
@@ -70,7 +70,7 @@ entry:
   ret void
 }
 
-; CHECK:      define dso_local void @quux(ptr nocapture noundef writeonly %[[p:.*]])
+; CHECK:      define dso_local void @quux(ptr noundef writeonly captures(none) %[[p:.*]])
 ; CHECK-NEXT: entry:
 ; CHECK-NEXT:   tail call void (i32, ptr, i1, i8, i8, i8, i1, ...)
 ; CHECK-SAME:     @llvm.bpf.getelementptr.and.store.i32
diff --git a/llvm/test/CodeGen/BPF/preserve-static-offset/store-volatile.ll b/llvm/test/CodeGen/BPF/preserve-static-offset/store-volatile.ll
index 8b0493a38efa6..3e4f53001dc93 100644
--- a/llvm/test/CodeGen/BPF/preserve-static-offset/store-volatile.ll
+++ b/llvm/test/CodeGen/BPF/preserve-static-offset/store-volatile.ll
@@ -29,7 +29,7 @@ entry:
   ret void
 }
 
-; CHECK:      define dso_local void @bar(ptr nocapture noundef %[[p:.*]])
+; CHECK:      define dso_local void @bar(ptr noundef captures(none) %[[p:.*]])
 ; CHECK:        tail call void (i32, ptr, i1, i8, i8, i8, i1, ...)
 ; CHECK-SAME:     @llvm.bpf.getelementptr.and.store.i32
 ; CHECK-SAME:       (i32 42,
diff --git a/llvm/test/CodeGen/BPF/preserve-static-offset/store-zero.ll b/llvm/test/CodeGen/BPF/preserve-static-offset/store-zero.ll
index d3929a3706ba8..8bac2a3f393f5 100644
--- a/llvm/test/CodeGen/BPF/preserve-static-offset/store-zero.ll
+++ b/llvm/test/CodeGen/BPF/preserve-static-offset/store-zero.ll
@@ -28,7 +28,7 @@ entry:
   ret void
 }
 
-; CHECK:      define dso_local void @bar(ptr nocapture noundef writeonly initializes((0, 4)) %[[p:.*]])
+; CHECK:      define dso_local void @bar(ptr noundef writeonly captures(none) initializes((0, 4)) %[[p:.*]])
 ; CHECK-NEXT: entry:
 ; CHECK-NEXT:   store i32 0, ptr %[[p]], align 4, !tbaa
 ; CHECK-NEXT:   ret void
diff --git a/llvm/test/CodeGen/DirectX/UAddc.ll b/llvm/test/CodeGen/DirectX/UAddc.ll
new file mode 100644
index 0000000000000..c16a3f6a5b5fe
--- /dev/null
+++ b/llvm/test/CodeGen/DirectX/UAddc.ll
@@ -0,0 +1,72 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -S -scalarizer -dxil-op-lower -mtriple=dxil-pc-shadermodel6.3-library %s | FileCheck %s
+
+; CHECK: %dx.types.i32c = type { i32, i1 }
+
+define noundef i32 @test_UAddc(i32 noundef %a, i32 noundef %b) {
+; CHECK-LABEL: define noundef i32 @test_UAddc(
+; CHECK-SAME: i32 noundef [[A:%.*]], i32 noundef [[B:%.*]]) {
+; CHECK-NEXT:    [[UADDC1:%.*]] = call [[DX_TYPES_I32C:%.*]] @[[DX_OP_BINARYWITHCARRYORBORROW_I32:[a-zA-Z0-9_$\"\\.-]*[a-zA-Z_$\"\\.-][a-zA-Z0-9_$\"\\.-]*]](i32 44, i32 [[A]], i32 [[B]]) #[[ATTR0:[0-9]+]]
+; CHECK-NEXT:    [[CARRY:%.*]] = extractvalue [[DX_TYPES_I32C]] [[UADDC1]], 1
+; CHECK-NEXT:    [[SUM:%.*]] = extractvalue [[DX_TYPES_I32C]] [[UADDC1]], 0
+; CHECK-NEXT:    [[CARRY_ZEXT:%.*]] = zext i1 [[CARRY]] to i32
+; CHECK-NEXT:    [[RESULT:%.*]] = add i32 [[SUM]], [[CARRY_ZEXT]]
+; CHECK-NEXT:    ret i32 [[RESULT]]
+;
+  %uaddc = call { i32, i1 } @llvm.uadd.with.overflow.i32(i32 %a, i32 %b)
+  %carry = extractvalue { i32, i1 } %uaddc, 1
+  %sum = extractvalue { i32, i1 } %uaddc, 0
+  %carry_zext = zext i1 %carry to i32
+  %result = add i32 %sum, %carry_zext
+  ret i32 %result
+}
+
+define noundef <2 x i32> @test_UAddc_vec2(<2 x i32> noundef %a, <2 x i32> noundef %b) {
+; CHECK-LABEL: define noundef <2 x i32> @test_UAddc_vec2(
+; CHECK-SAME: <2 x i32> noundef [[A:%.*]], <2 x i32> noundef [[B:%.*]]) {
+; CHECK-NEXT:    [[A_I0:%.*]] = extractelement <2 x i32> [[A]], i64 0
+; CHECK-NEXT:    [[B_I0:%.*]] = extractelement <2 x i32> [[B]], i64 0
+; CHECK-NEXT:    [[UADDC_I09:%.*]] = call [[DX_TYPES_I32C:%.*]] @[[DX_OP_BINARYWITHCARRYORBORROW_I32]](i32 44, i32 [[A_I0]], i32 [[B_I0]]) #[[ATTR0]]
+; CHECK-NEXT:    [[A_I1:%.*]] = extractelement <2 x i32> [[A]], i64 1
+; CHECK-NEXT:    [[B_I1:%.*]] = extractelement <2 x i32> [[B]], i64 1
+; CHECK-NEXT:    [[UADDC_I18:%.*]] = call [[DX_TYPES_I32C]] @[[DX_OP_BINARYWITHCARRYORBORROW_I32]](i32 44, i32 [[A_I1]], i32 [[B_I1]]) #[[ATTR0]]
+; CHECK-NEXT:    [[CARRY_ELEM1:%.*]] = extractvalue [[DX_TYPES_I32C]] [[UADDC_I09]], 1
+; CHECK-NEXT:    [[CARRY_ELEM11:%.*]] = extractvalue [[DX_TYPES_I32C]] [[UADDC_I18]], 1
+; CHECK-NEXT:    [[CARRY_UPTO0:%.*]] = insertelement <2 x i1> poison, i1 [[CARRY_ELEM1]], i64 0
+; CHECK-NEXT:    [[CARRY:%.*]] = insertelement <2 x i1> [[CARRY_UPTO0]], i1 [[CARRY_ELEM11]], i64 1
+; CHECK-NEXT:    [[CARRY_I0:%.*]] = extractelement <2 x i1> [[CARRY]], i64 0
+; CHECK-NEXT:    [[CARRY_I1:%.*]] = extractelement <2 x i1> [[CARRY]], i64 1
+; CHECK-NEXT:    [[SUM_ELEM0:%.*]] = extractvalue [[DX_TYPES_I32C]] [[UADDC_I09]], 0
+; CHECK-NEXT:    [[SUM_ELEM02:%.*]] = extractvalue [[DX_TYPES_I32C]] [[UADDC_I18]], 0
+; CHECK-NEXT:    [[CARRY_ZEXT_I0:%.*]] = zext i1 [[CARRY_I0]] to i32
+; CHECK-NEXT:    [[CARRY_ZEXT_I1:%.*]] = zext i1 [[CARRY_I1]] to i32
+; CHECK-NEXT:    [[RESULT_I0:%.*]] = add i32 [[SUM_ELEM0]], [[CARRY_ZEXT_I0]]
+; CHECK-NEXT:    [[RESULT_I1:%.*]] = add i32 [[SUM_ELEM02]], [[CARRY_ZEXT_I1]]
+; CHECK-NEXT:    [[RESULT_UPTO0:%.*]] = insertelement <2 x i32> poison, i32 [[RESULT_I0]], i64 0
+; CHECK-NEXT:    [[RESULT:%.*]] = insertelement <2 x i32> [[RESULT_UPTO0]], i32 [[RESULT_I1]], i64 1
+; CHECK-NEXT:    ret <2 x i32> [[RESULT]]
+;
+  %uaddc = call { <2 x i32>, <2 x i1> } @llvm.uadd.with.overflow.v2i32(<2 x i32> %a, <2 x i32> %b)
+  %carry = extractvalue { <2 x i32>, <2 x i1> } %uaddc, 1
+  %sum = extractvalue { <2 x i32>, <2 x i1> } %uaddc, 0
+  %carry_zext = zext <2 x i1> %carry to <2 x i32>
+  %result = add <2 x i32> %sum, %carry_zext
+  ret <2 x i32> %result
+}
+
+define noundef i32 @test_UAddc_insert(i32 noundef %a, i32 noundef %b) {
+; CHECK-LABEL: define noundef i32 @test_UAddc_insert(
+; CHECK-SAME: i32 noundef [[A:%.*]], i32 noundef [[B:%.*]]) {
+; CHECK-NEXT:    [[UADDC1:%.*]] = call [[DX_TYPES_I32C:%.*]] @[[DX_OP_BINARYWITHCARRYORBORROW_I32]](i32 44, i32 [[A]], i32 [[B]]) #[[ATTR0]]
+; CHECK-NEXT:    [[TMP1:%.*]] = insertvalue [[DX_TYPES_I32C]] [[UADDC1]], i32 [[A]], 0
+; CHECK-NEXT:    [[RESULT:%.*]] = extractvalue [[DX_TYPES_I32C]] [[UADDC1]], 0
+; CHECK-NEXT:    ret i32 [[RESULT]]
+;
+  %uaddc = call { i32, i1 } @llvm.uadd.with.overflow.i32(i32 %a, i32 %b)
+  insertvalue { i32, i1 } %uaddc, i32 %a, 0
+  %result = extractvalue { i32, i1 } %uaddc, 0
+  ret i32 %result
+}
+
+declare { i32, i1 } @llvm.uadd.with.overflow.i32(i32, i32)
+
diff --git a/llvm/test/CodeGen/DirectX/UAddc_errors.ll b/llvm/test/CodeGen/DirectX/UAddc_errors.ll
new file mode 100644
index 0000000000000..72c676db9bba4
--- /dev/null
+++ b/llvm/test/CodeGen/DirectX/UAddc_errors.ll
@@ -0,0 +1,30 @@
+; We use llc for this test so that we don't abort after the first error.
+; RUN: not llc %s -o /dev/null 2>&1 | FileCheck %s
+
+target triple = "dxil-pc-shadermodel6.3-library"
+
+; DXIL operation UAddc only supports i32. Other integer types are unsupported.
+; CHECK: error:
+; CHECK-SAME: in function uaddc_i16
+; CHECK-SAME: Cannot create UAddc operation: Invalid overload type
+
+define noundef i16 @uaddc_i16(i16 noundef %a, i16 noundef %b) "hlsl.export" {
+  %uaddc = call { i16, i1 } @llvm.uadd.with.overflow.i16(i16 %a, i16 %b)
+  %carry = extractvalue { i16, i1 } %uaddc, 1
+  %sum = extractvalue { i16, i1 } %uaddc, 0
+  %carry_zext = zext i1 %carry to i16
+  %result = add i16 %sum, %carry_zext
+  ret i16 %result
+}
+
+; CHECK: error:
+; CHECK-SAME: in function uaddc_return
+; CHECK-SAME: llvm.uadd.with.overflow.i32 use is not a ExtractValueInst or InsertValueInst
+
+define noundef { i32, i1 } @uaddc_return(i32 noundef %a, i32 noundef %b) "hlsl.export" {
+  %uaddc = call { i32, i1 } @llvm.uadd.with.overflow.i32(i32 %a, i32 %b)
+  ret { i32, i1 } %uaddc
+}
+
+declare { i16, i1 } @llvm.uadd.with.overflow.i16(i16, i16)
+
diff --git a/llvm/test/CodeGen/DirectX/WaveActiveMax.ll b/llvm/test/CodeGen/DirectX/WaveActiveMax.ll
new file mode 100644
index 0000000000000..d402e67e6c1a6
--- /dev/null
+++ b/llvm/test/CodeGen/DirectX/WaveActiveMax.ll
@@ -0,0 +1,143 @@
+; RUN: opt -S -scalarizer -dxil-op-lower -mtriple=dxil-pc-shadermodel6.3-library < %s | FileCheck %s
+
+; Test that for scalar values, WaveActiveMax maps down to the DirectX op
+
+define noundef half @wave_active_max_half(half noundef %expr) {
+entry:
+; CHECK: call half @dx.op.waveActiveOp.f16(i32 119, half %expr, i8 3, i8 0){{$}}
+  %ret = call half @llvm.dx.wave.reduce.max.f16(half %expr)
+  ret half %ret
+}
+
+define noundef float @wave_active_max_float(float noundef %expr) {
+entry:
+; CHECK: call float @dx.op.waveActiveOp.f32(i32 119, float %expr, i8 3, i8 0){{$}}
+  %ret = call float @llvm.dx.wave.reduce.max.f32(float %expr)
+  ret float %ret
+}
+
+define noundef double @wave_active_max_double(double noundef %expr) {
+entry:
+; CHECK: call double @dx.op.waveActiveOp.f64(i32 119, double %expr, i8 3, i8 0){{$}}
+  %ret = call double @llvm.dx.wave.reduce.max.f64(double %expr)
+  ret double %ret
+}
+
+define noundef i16 @wave_active_max_i16(i16 noundef %expr) {
+entry:
+; CHECK: call i16 @dx.op.waveActiveOp.i16(i32 119, i16 %expr, i8 3, i8 0){{$}}
+  %ret = call i16 @llvm.dx.wave.reduce.max.i16(i16 %expr)
+  ret i16 %ret
+}
+
+define noundef i32 @wave_active_max_i32(i32 noundef %expr) {
+entry:
+; CHECK: call i32 @dx.op.waveActiveOp.i32(i32 119, i32 %expr, i8 3, i8 0){{$}}
+  %ret = call i32 @llvm.dx.wave.reduce.max.i32(i32 %expr)
+  ret i32 %ret
+}
+
+define noundef i64 @wave_active_max_i64(i64 noundef %expr) {
+entry:
+; CHECK: call i64 @dx.op.waveActiveOp.i64(i32 119, i64 %expr, i8 3, i8 0){{$}}
+  %ret = call i64 @llvm.dx.wave.reduce.max.i64(i64 %expr)
+  ret i64 %ret
+}
+
+define noundef i16 @wave_active_umax_i16(i16 noundef %expr) {
+entry:
+; CHECK: call i16 @dx.op.waveActiveOp.i16(i32 119, i16 %expr, i8 3, i8 1){{$}}
+  %ret = call i16 @llvm.dx.wave.reduce.umax.i16(i16 %expr)
+  ret i16 %ret
+}
+
+define noundef i32 @wave_active_umax_i32(i32 noundef %expr) {
+entry:
+; CHECK: call i32 @dx.op.waveActiveOp.i32(i32 119, i32 %expr, i8 3, i8 1){{$}}
+  %ret = call i32 @llvm.dx.wave.reduce.umax.i32(i32 %expr)
+  ret i32 %ret
+}
+
+define noundef i64 @wave_active_umax_i64(i64 noundef %expr) {
+entry:
+; CHECK: call i64 @dx.op.waveActiveOp.i64(i32 119, i64 %expr, i8 3, i8 1){{$}}
+  %ret = call i64 @llvm.dx.wave.reduce.umax.i64(i64 %expr)
+  ret i64 %ret
+}
+
+declare half @llvm.dx.wave.reduce.max.f16(half)
+declare float @llvm.dx.wave.reduce.max.f32(float)
+declare double @llvm.dx.wave.reduce.max.f64(double)
+
+declare i16 @llvm.dx.wave.reduce.max.i16(i16)
+declare i32 @llvm.dx.wave.reduce.max.i32(i32)
+declare i64 @llvm.dx.wave.reduce.max.i64(i64)
+
+declare i16 @llvm.dx.wave.reduce.umax.i16(i16)
+declare i32 @llvm.dx.wave.reduce.umax.i32(i32)
+declare i64 @llvm.dx.wave.reduce.umax.i64(i64)
+
+; Test that for vector values, WaveActiveMax scalarizes and maps down to the
+; DirectX op
+
+define noundef <2 x half> @wave_active_max_v2half(<2 x half> noundef %expr) {
+entry:
+; CHECK: call half @dx.op.waveActiveOp.f16(i32 119, half %expr.i0, i8 3, i8 0){{$}}
+; CHECK: call half @dx.op.waveActiveOp.f16(i32 119, half %expr.i1, i8 3, i8 0){{$}}
+  %ret = call <2 x half> @llvm.dx.wave.reduce.max.v2f16(<2 x half> %expr)
+  ret <2 x half> %ret
+}
+
+define noundef <3 x i32> @wave_active_max_v3i32(<3 x i32> noundef %expr) {
+entry:
+; CHECK: call i32 @dx.op.waveActiveOp.i32(i32 119, i32 %expr.i0, i8 3, i8 0){{$}}
+; CHECK: call i32 @dx.op.waveActiveOp.i32(i32 119, i32 %expr.i1, i8 3, i8 0){{$}}
+; CHECK: call i32 @dx.op.waveActiveOp.i32(i32 119, i32 %expr.i2, i8 3, i8 0){{$}}
+  %ret = call <3 x i32> @llvm.dx.wave.reduce.max.v3i32(<3 x i32> %expr)
+  ret <3 x i32> %ret
+}
+
+define noundef <4 x double> @wave_active_max_v4f64(<4 x double> noundef %expr) {
+entry:
+; CHECK: call double @dx.op.waveActiveOp.f64(i32 119, double %expr.i0, i8 3, i8 0){{$}}
+; CHECK: call double @dx.op.waveActiveOp.f64(i32 119, double %expr.i1, i8 3, i8 0){{$}}
+; CHECK: call double @dx.op.waveActiveOp.f64(i32 119, double %expr.i2, i8 3, i8 0){{$}}
+; CHECK: call double @dx.op.waveActiveOp.f64(i32 119, double %expr.i3, i8 3, i8 0){{$}}
+  %ret = call <4 x double> @llvm.dx.wave.reduce.max.v4f64(<4 x double> %expr)
+  ret <4 x double> %ret
+}
+
+declare <2 x half> @llvm.dx.wave.reduce.max.v2f16(<2 x half>)
+declare <3 x i32> @llvm.dx.wave.reduce.max.v3i32(<3 x i32>)
+declare <4 x double> @llvm.dx.wave.reduce.max.v4f64(<4 x double>)
+
+define noundef <2 x i16> @wave_active_umax_v2i16(<2 x i16> noundef %expr) {
+entry:
+; CHECK: call i16 @dx.op.waveActiveOp.i16(i32 119, i16 %expr.i0, i8 3, i8 1){{$}}
+; CHECK: call i16 @dx.op.waveActiveOp.i16(i32 119, i16 %expr.i1, i8 3, i8 1){{$}}
+  %ret = call <2 x i16> @llvm.dx.wave.reduce.umax.v2f16(<2 x i16> %expr)
+  ret <2 x i16> %ret
+}
+
+define noundef <3 x i32> @wave_active_umax_v3i32(<3 x i32> noundef %expr) {
+entry:
+; CHECK: call i32 @dx.op.waveActiveOp.i32(i32 119, i32 %expr.i0, i8 3, i8 1){{$}}
+; CHECK: call i32 @dx.op.waveActiveOp.i32(i32 119, i32 %expr.i1, i8 3, i8 1){{$}}
+; CHECK: call i32 @dx.op.waveActiveOp.i32(i32 119, i32 %expr.i2, i8 3, i8 1){{$}}
+  %ret = call <3 x i32> @llvm.dx.wave.reduce.umax.v3i32(<3 x i32> %expr)
+  ret <3 x i32> %ret
+}
+
+define noundef <4 x i64> @wave_active_umax_v4f64(<4 x i64> noundef %expr) {
+entry:
+; CHECK: call i64 @dx.op.waveActiveOp.i64(i32 119, i64 %expr.i0, i8 3, i8 1){{$}}
+; CHECK: call i64 @dx.op.waveActiveOp.i64(i32 119, i64 %expr.i1, i8 3, i8 1){{$}}
+; CHECK: call i64 @dx.op.waveActiveOp.i64(i32 119, i64 %expr.i2, i8 3, i8 1){{$}}
+; CHECK: call i64 @dx.op.waveActiveOp.i64(i32 119, i64 %expr.i3, i8 3, i8 1){{$}}
+  %ret = call <4 x i64> @llvm.dx.wave.reduce.umax.v4f64(<4 x i64> %expr)
+  ret <4 x i64> %ret
+}
+
+declare <2 x i16> @llvm.dx.wave.reduce.umax.v2f16(<2 x i16>)
+declare <3 x i32> @llvm.dx.wave.reduce.umax.v3i32(<3 x i32>)
+declare <4 x i64> @llvm.dx.wave.reduce.umax.v4f64(<4 x i64>)
diff --git a/llvm/test/CodeGen/Hexagon/autohvx/vector-align-bad-move2.ll b/llvm/test/CodeGen/Hexagon/autohvx/vector-align-bad-move2.ll
index ce1a99b713322..544769af515c6 100644
--- a/llvm/test/CodeGen/Hexagon/autohvx/vector-align-bad-move2.ll
+++ b/llvm/test/CodeGen/Hexagon/autohvx/vector-align-bad-move2.ll
@@ -9,7 +9,7 @@
 
 define internal fastcc i32 @f0(ptr noalias nocapture align 64 %a0, ptr noalias nocapture readonly align 64 %a1, ptr noalias nocapture readonly align 64 %a2) unnamed_addr #0 {
 ; CHECK-LABEL: define internal fastcc i32 @f0
-; CHECK-SAME: (ptr noalias nocapture align 64 [[A0:%.*]], ptr noalias nocapture readonly align 64 [[A1:%.*]], ptr noalias nocapture readonly align 64 [[A2:%.*]]) unnamed_addr #[[ATTR0:[0-9]+]] {
+; CHECK-SAME: (ptr noalias align 64 captures(none) [[A0:%.*]], ptr noalias readonly align 64 captures(none) [[A1:%.*]], ptr noalias readonly align 64 captures(none) [[A2:%.*]]) unnamed_addr #[[ATTR0:[0-9]+]] {
 ; CHECK-NEXT:  b0:
 ; CHECK-NEXT:    [[V0:%.*]] = load <32 x i32>, ptr [[A2]], align 64, !tbaa [[TBAA0:![0-9]+]]
 ; CHECK-NEXT:    br label [[B1:%.*]]
diff --git a/llvm/test/CodeGen/Hexagon/autohvx/vector-align-overapping-stores.ll b/llvm/test/CodeGen/Hexagon/autohvx/vector-align-overapping-stores.ll
index fc49b8ae391f0..cb43f28526c7b 100644
--- a/llvm/test/CodeGen/Hexagon/autohvx/vector-align-overapping-stores.ll
+++ b/llvm/test/CodeGen/Hexagon/autohvx/vector-align-overapping-stores.ll
@@ -3,7 +3,7 @@
 ; Function Attrs: nofree noinline nosync nounwind memory(argmem: readwrite)
 define fastcc void @fred(ptr noalias nocapture align 128 %a0, ptr noalias nocapture readonly align 128 %a1) #0 {
 ; CHECK-LABEL: define fastcc void @fred
-; CHECK-SAME: (ptr noalias nocapture align 128 [[A0:%.*]], ptr noalias nocapture readonly align 128 [[A1:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-SAME: (ptr noalias align 128 captures(none) [[A0:%.*]], ptr noalias readonly align 128 captures(none) [[A1:%.*]]) #[[ATTR0:[0-9]+]] {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[V0:%.*]] = load <128 x i8>, ptr [[A1]], align 128
 ; CHECK-NEXT:    store <128 x i8> [[V0]], ptr [[A0]], align 128
diff --git a/llvm/test/CodeGen/Hexagon/bank-conflict-load.mir b/llvm/test/CodeGen/Hexagon/bank-conflict-load.mir
index 01cc0ee8696ff..7c741aaae4480 100644
--- a/llvm/test/CodeGen/Hexagon/bank-conflict-load.mir
+++ b/llvm/test/CodeGen/Hexagon/bank-conflict-load.mir
@@ -1,4 +1,5 @@
 # RUN: llc -mtriple=hexagon -run-pass post-RA-sched %s -o - | FileCheck %s
+# RUN: llc -mtriple=hexagon -passes=post-RA-sched %s -o - | FileCheck %s
 
 # The two loads from %a ($r0) can cause a bank conflict. Check that they
 # are not scheduled next to each other.
diff --git a/llvm/test/CodeGen/Hexagon/bank-conflict.mir b/llvm/test/CodeGen/Hexagon/bank-conflict.mir
index 12d7838b8372d..0add375786c30 100644
--- a/llvm/test/CodeGen/Hexagon/bank-conflict.mir
+++ b/llvm/test/CodeGen/Hexagon/bank-conflict.mir
@@ -1,4 +1,5 @@
 # RUN: llc -mtriple=hexagon -run-pass post-RA-sched %s -o - | FileCheck %s
+# RUN: llc -mtriple=hexagon -passes=post-RA-sched %s -o - | FileCheck %s
 
 # Test that the Post RA scheduler does not schedule back-to-back loads
 # when there is another instruction to schedule. The scheduler avoids
diff --git a/llvm/test/CodeGen/Hexagon/rdf-copy-clobber.mir b/llvm/test/CodeGen/Hexagon/rdf-copy-clobber.mir
new file mode 100644
index 0000000000000..e0676a143eefe
--- /dev/null
+++ b/llvm/test/CodeGen/Hexagon/rdf-copy-clobber.mir
@@ -0,0 +1,143 @@
+# RUN: llc -march=hexagon -run-pass=hexagon-rdf-opt -hexagon-rdf-dump -verify-machineinstrs -o /dev/null %s 2>&1 | FileCheck %s
+
+# Check that RDF graph has a phi node for R28 register in bb.3 and bb.4
+# R28 is clobbered by memcpy call. The clobbering def must be present in bb.4's IDF
+# This phi node should prevent $r27 from being replaced by $r28 by RDF copy propagation
+
+#CHECK-LABEL: Starting copy propagation on: foo
+
+#CHECK-LABEL: --- %bb.3 ---
+#CHECK: p{{[0-9]+}}: phi [+d{{[0-9]+}}<R28>
+
+#CHECK-LABEL: --- %bb.4 ---
+#CHECK: p{{[0-9]+}}: phi [+d{{[0-9]+}}<R28>
+
+#CHECK-LABEL: After Hexagon RDF optimizations
+#CHECK-LABEL: bb.3:
+#CHECK: renamable $r0 = A2_add renamable $r27
+
+--- |
+  define internal fastcc void @foo() unnamed_addr {
+  entry:
+    ret void
+  }
+
+  declare void @llvm.memcpy.p0.p0.i32(ptr noalias nocapture writeonly, ptr noalias nocapture readonly, i32, i1 immarg)
+
+---
+name:            foo
+alignment:       16
+exposesReturnsTwice: false
+legalized:       false
+regBankSelected: false
+selected:        false
+failedISel:      false
+tracksRegLiveness: true
+hasWinCFI:       false
+callsEHReturn:   false
+callsUnwindInit: false
+hasEHCatchret:   false
+hasEHScopes:     false
+hasEHFunclets:   false
+isOutlined:      false
+debugInstrRef:   false
+failsVerification: false
+tracksDebugUserValues: true
+registers:       []
+liveins:
+  - { reg: '$d0', virtual-reg: '' }
+  - { reg: '$d3', virtual-reg: '' }
+  - { reg: '$r23', virtual-reg: '' }
+frameInfo:
+  isFrameAddressTaken: false
+  isReturnAddressTaken: false
+  hasStackMap:     false
+  hasPatchPoint:   false
+  stackSize:       0
+  offsetAdjustment: 0
+  maxAlignment:    8
+  adjustsStack:    true
+  hasCalls:        true
+  stackProtector:  ''
+  functionContext: ''
+  maxCallFrameSize: 4294967295
+  cvBytesOfCalleeSavedRegisters: 0
+  hasOpaqueSPAdjustment: false
+  hasVAStart:      false
+  hasMustTailInVarArgFunc: false
+  hasTailCall:     false
+  isCalleeSavedInfoValid: false
+  localFrameSize:  0
+  savePoint:       ''
+  restorePoint:    ''
+fixedStack:
+  - { id: 0, type: default, offset: 40, size: 8, alignment: 8, stack-id: default,
+      isImmutable: true, isAliased: false, callee-saved-register: '', callee-saved-restored: true,
+      debug-info-variable: '', debug-info-expression: '', debug-info-location: '' }
+stack:
+  - { id: 0, name: '', type: spill-slot, offset: 0, size: 8, alignment: 8,
+      stack-id: default, callee-saved-register: '', callee-saved-restored: true,
+      debug-info-variable: '', debug-info-expression: '', debug-info-location: '' }
+  - { id: 1, name: '', type: spill-slot, offset: 0, size: 8, alignment: 8,
+      stack-id: default, callee-saved-register: '', callee-saved-restored: true,
+      debug-info-variable: '', debug-info-expression: '', debug-info-location: '' }
+  - { id: 2, name: '', type: spill-slot, offset: 0, size: 8, alignment: 8,
+      stack-id: default, callee-saved-register: '', callee-saved-restored: true,
+      debug-info-variable: '', debug-info-expression: '', debug-info-location: '' }
+  - { id: 3, name: '', type: spill-slot, offset: 0, size: 8, alignment: 8,
+      stack-id: default, callee-saved-register: '', callee-saved-restored: true,
+      debug-info-variable: '', debug-info-expression: '', debug-info-location: '' }
+entry_values:    []
+callSites:       []
+debugValueSubstitutions: []
+constants:       []
+machineFunctionInfo: {}
+body:             |
+  bb.0.entry:
+    successors: %bb.1
+    liveins: $d0, $d3, $r23
+
+    J2_jump %bb.1, implicit-def dead $pc
+
+  bb.1:
+    successors: %bb.2
+    liveins: $d0:0x0000000000000003, $d3:0x0000000000000003, $r23
+
+    renamable $r28 = L2_loadri_io %fixed-stack.0, 0 :: (load (s32) from %fixed-stack.0)
+    renamable $r27 = COPY killed renamable $r28
+
+  bb.2:
+    successors: %bb.3
+    liveins: $d0:0x0000000000000003, $d3:0x0000000000000003, $r23, $r27
+
+    renamable $d10 = L2_loadrd_io %stack.0, 0 :: (load (s64) from %stack.0)
+    renamable $d11 = L2_loadrd_io %stack.1, 0 :: (load (s64) from %stack.1)
+
+  bb.3:
+    successors: %bb.4, %bb.3
+    liveins: $d0:0x0000000000000003, $d3:0x0000000000000003, $d10:0x0000000000000003, $d11:0x0000000000000002, $r23, $r27
+
+    ADJCALLSTACKDOWN 0, 0, implicit-def $r29, implicit-def dead $r30, implicit $r31, implicit $r30, implicit $r29
+    renamable $r1 = A2_add renamable $r23, killed renamable $r0
+    $r2 = COPY renamable $r22
+    renamable $r0 = A2_add renamable $r27, killed renamable $r6
+    J2_call &memcpy, hexagoncsr, implicit-def dead $pc, implicit-def dead $r31, implicit $r29, implicit $r0, implicit $r1, implicit $r2, implicit-def $r29, implicit-def dead $r0
+    renamable $p0 = C2_cmpgtp renamable $d11, renamable $d10
+    ADJCALLSTACKUP 0, 0, implicit-def dead $r29, implicit-def dead $r30, implicit-def dead $r31, implicit $r29
+    J2_jumpt killed renamable $p0, %bb.3, implicit-def dead $pc
+    J2_jump %bb.4, implicit-def dead $pc
+
+  bb.4:
+    successors: %bb.5, %bb.2
+    liveins: $d10:0x0000000000000003, $d11:0x0000000000000002, $r23, $r27
+
+    renamable $d0 = L2_loadrd_io %stack.2, 0 :: (load (s64) from %stack.2)
+    renamable $d3 = L2_loadrd_io %stack.3, 0 :: (load (s64) from %stack.3)
+    renamable $p0 = C2_cmpgtp killed renamable $d0, killed renamable $d3
+    J2_jumpt killed renamable $p0, %bb.2, implicit-def dead $pc
+    J2_jump %bb.5, implicit-def dead $pc
+
+  bb.5:
+    PS_jmpret $r31, implicit-def dead $pc
+
+...
diff --git a/llvm/test/CodeGen/Hexagon/rdf-phi-clobber.mir b/llvm/test/CodeGen/Hexagon/rdf-phi-clobber.mir
new file mode 100644
index 0000000000000..d49cc3403d644
--- /dev/null
+++ b/llvm/test/CodeGen/Hexagon/rdf-phi-clobber.mir
@@ -0,0 +1,102 @@
+# RUN: llc -march=hexagon -run-pass=hexagon-rdf-opt \
+# RUN: -hexagon-rdf-dump -verify-machineinstrs -o /dev/null %s 2>&1 \
+# RUN: | FileCheck %s
+
+# Check that phi nodes that only have clobbering reaching defs are not created
+# during graph construction. Check that there are no phi nodes for HVX registers
+
+#CHECK-LABEL: --- %bb.1 ---
+#CHECK-NOT: p{{[0-9]+}}: phi [+d{{[0-9]+}}<V{{[0-9]+}}>
+
+--- |
+  @.str.3 = private unnamed_addr constant [2 x i8] c"%d", align 8
+  @.str.4 = private unnamed_addr constant [2 x i8] c"%d", align 8
+
+  define internal fastcc void @foo() unnamed_addr {
+  entry:
+    ret void
+  }
+
+  declare dso_local noundef i32 @printf(ptr nocapture noundef readonly, ...) local_unnamed_addr
+
+---
+name:            foo
+alignment:       16
+exposesReturnsTwice: false
+legalized:       false
+regBankSelected: false
+selected:        false
+failedISel:      false
+tracksRegLiveness: true
+hasWinCFI:       false
+callsEHReturn:   false
+callsUnwindInit: false
+hasEHCatchret:   false
+hasEHScopes:     false
+hasEHFunclets:   false
+isOutlined:      false
+debugInstrRef:   false
+failsVerification: false
+tracksDebugUserValues: true
+registers:       []
+liveins:
+  - { reg: '$d0', virtual-reg: '' }
+  - { reg: '$d3', virtual-reg: '' }
+  - { reg: '$r23', virtual-reg: '' }
+frameInfo:
+  isFrameAddressTaken: false
+  isReturnAddressTaken: false
+  hasStackMap:     false
+  hasPatchPoint:   false
+  stackSize:       0
+  offsetAdjustment: 0
+  maxAlignment:    8
+  adjustsStack:    true
+  hasCalls:        true
+  stackProtector:  ''
+  functionContext: ''
+  maxCallFrameSize: 4294967295
+  cvBytesOfCalleeSavedRegisters: 0
+  hasOpaqueSPAdjustment: false
+  hasVAStart:      false
+  hasMustTailInVarArgFunc: false
+  hasTailCall:     false
+  isCalleeSavedInfoValid: false
+  localFrameSize:  0
+  savePoint:       ''
+  restorePoint:    ''
+entry_values:    []
+callSites:       []
+debugValueSubstitutions: []
+constants:       []
+machineFunctionInfo: {}
+body:             |
+  bb.0.entry:
+    successors: %bb.1
+    liveins: $r25, $r26, $d11
+
+    renamable $r16 = A2_tfrsi 0
+    S2_storerd_io $r29, 0, renamable $d11 :: (store (s64) into stack)
+    $r0 = A2_tfrsi @.str.3
+    J2_call @printf, hexagoncsr, implicit-def dead $pc, implicit-def dead $r31, implicit $r29, implicit $r0, implicit-def $r29, implicit-def dead $r0
+    J2_jump %bb.1, implicit-def dead $pc
+
+  bb.1:
+    successors: %bb.2, %bb.1
+    liveins: $r16, $r25, $r26
+
+    S2_storeri_io $r29, 0, killed renamable $r25 :: (store (s32) into stack)
+    $r0 = A2_tfrsi @.str.4
+    S2_storeri_io $r29, 8, killed renamable $r26 :: (store (s64) into stack + 8)
+    J2_call @printf, hexagoncsr, implicit-def dead $pc, implicit-def dead $r31, implicit $r29, implicit $r0, implicit-def $r29, implicit-def dead $r0
+    renamable $p0 = C2_cmpgti renamable $r16, 4
+    renamable $r16 = nsw A2_addi killed renamable $r16, 1
+    J2_jumpf killed renamable $p0, %bb.2, implicit-def dead $pc
+    J2_jump %bb.1, implicit-def dead $pc
+
+  bb.2:
+    liveins: $r16, $r25, $r26
+
+    PS_jmpret $r31, implicit-def dead $pc
+
+...
diff --git a/llvm/test/CodeGen/Hexagon/swp-carried-dep1.mir b/llvm/test/CodeGen/Hexagon/swp-carried-dep1.mir
index c333f1b7f31df..afc989cbc6921 100644
--- a/llvm/test/CodeGen/Hexagon/swp-carried-dep1.mir
+++ b/llvm/test/CodeGen/Hexagon/swp-carried-dep1.mir
@@ -3,6 +3,13 @@
 
 # Test that the loop carried dependence check correctly identifies a recurrence.
 
+# CHECK: Overlap check:
+# CHECK-NEXT:   BaseMI:   S2_storerh_io %{{[0-9]+}}:intregs, 0, %{{[0-9]+}}:intregs :: (store (s16) into %ir.lsr.iv24)
+# CHECK-NEXT:     Base + 0 + I * 4, Len: 2
+# CHECK-NEXT:   OtherMI:   %{{[0-9]+}}:intregs = L2_loadrh_io %{{[0-9]+}}:intregs, -8 :: (load (s16) from %ir.cgep10)
+# CHECK-NEXT:     Base + -8 + I * 4, Len: 2
+# CHECK-NEXT:   Result: Overlap
+
 # CHECK: Rec NodeSet
 # CHECK: Rec NodeSet
 # CHECK: Rec NodeSet
diff --git a/llvm/test/CodeGen/Hexagon/swp-carried-dep2.mir b/llvm/test/CodeGen/Hexagon/swp-carried-dep2.mir
index 16ff5999e29ca..e16334ba7978f 100644
--- a/llvm/test/CodeGen/Hexagon/swp-carried-dep2.mir
+++ b/llvm/test/CodeGen/Hexagon/swp-carried-dep2.mir
@@ -1,14 +1,26 @@
 # RUN: llc -mtriple=hexagon -run-pass pipeliner -debug-only=pipeliner %s -o /dev/null 2>&1 -pipeliner-experimental-cg=true | FileCheck %s
 # REQUIRES: asserts
 
-# Test that the loop carried dependence check correctly identifies a recurrence
+# Test that the loop carried dependence check correctly identifies dependences
 # when the loop variable decreases and the array index offset is negative.
 
-# CHECK: Rec NodeSet
-# CHECK: Rec NodeSet
-# CHECK: SU(3)
-# CHECK: SU(4)
-# CHECK: SU(5)
+# No dependence from the store to the load.
+# CHECK: Overlap check:
+# CHECK-NEXT:   BaseMI:   S2_storeri_io %{{[0-9]+}}:intregs, 0, %{{[0-9]+}}:intregs :: (store (s32) into %ir.lsr.iv1)
+# CHECK-NEXT:     Base + 0 + I * -4, Len: 4
+# CHECK-NEXT:   OtherMI:   %{{[0-9]+}}:intregs = L2_loadri_io %{{[0-9]+}}:intregs, -8 :: (load (s32) from %ir.cgep)
+# CHECK-NEXT:     Base + -8 + I * -4, Len: 4
+# CHECK-NEXT:   Result: No overlap
+
+# TODO: There is a loop carried dependence from the load to the store but it
+#   is not recognised. addLoopCarriedDependences() should be modified to
+#   recognise the dependence and enable the following checks.
+# CHECK-AFTER-FIX: Overlap check:
+# CHECK-AFTER-FIX-NEXT:   BaseMI:   %{{[0-9]+}}:intregs = L2_loadri_io %{{[0-9]+}}:intregs, -8 :: (load (s32) from %ir.cgep)
+# CHECK-AFTER-FIX-NEXT:     Base + -8 + I * -4, Len: 4
+# CHECK-AFTER-FIX-NEXT:   OtherMI:   S2_storeri_io %{{[0-9]+}}:intregs, 0, %{{[0-9]+}}:intregs :: (store (s32) into %ir.lsr.iv1)
+# CHECK-AFTER-FIX-NEXT:     Base + 0 + I * -4, Len: 4
+# CHECK-AFTER-FIX-NEXT:   Result: Overlap!
 
 --- |
 
diff --git a/llvm/test/CodeGen/Hexagon/swp-carried-dep3.mir b/llvm/test/CodeGen/Hexagon/swp-carried-dep3.mir
index a1b0aec4cf81c..91eb225580910 100644
--- a/llvm/test/CodeGen/Hexagon/swp-carried-dep3.mir
+++ b/llvm/test/CodeGen/Hexagon/swp-carried-dep3.mir
@@ -7,6 +7,13 @@
 # requires to use a single CHECK-NOT to match such a Rec NodeSet. Fortunately
 # the atom '.' does not match a newline but anything else on a line.
 
+# CHECK: Overlap check:
+# CHECK-NEXT:   BaseMI:   %13:intregs = S2_storerh_pi %12:intregs(tied-def 0), 2, %20:intregs :: (store (s16))
+# CHECK-NEXT:     Base + 0 + I * 2, Len: 2
+# CHECK-NEXT:   OtherMI:   %19:intregs, %15:intregs = L2_loadrh_pi %14:intregs(tied-def 1), 2 :: (load (s16))
+# CHECK-NEXT:     Base + 0 + I * 2, Len: 2
+# CHECK-NEXT:   Result: No overlap
+
 # CHECK-NOT: Rec NodeSet{{.+[[:space:]]}} SU(5){{.+[[:space:]]}} SU(7)
 
 ...
diff --git a/llvm/test/CodeGen/Hexagon/swp-carried-dep4.mir b/llvm/test/CodeGen/Hexagon/swp-carried-dep4.mir
new file mode 100644
index 0000000000000..ff04886be561a
--- /dev/null
+++ b/llvm/test/CodeGen/Hexagon/swp-carried-dep4.mir
@@ -0,0 +1,37 @@
+# RUN: llc -mtriple=hexagon -run-pass pipeliner -debug-only=pipeliner %s -o /dev/null 2>&1 | FileCheck %s
+# REQUIRES: asserts
+
+# Loop carried dependence is assumed in cases where increment value cannot be recognized
+# (Not supported for multiple increment instruction)
+
+# CHECK:  Rec NodeSet
+# CHECK:  Rec NodeSet
+# CHECK-NEXT:    SU(1)
+# CHECK-NEXT:    SU(2)
+
+---
+name:            test
+tracksRegLiveness: true
+
+body:             |
+  bb.0:
+    successors: %bb.1
+
+    %10:intregs = IMPLICIT_DEF
+    %11:intregs = IMPLICIT_DEF
+    J2_loop0i %bb.1, 6, implicit-def $lc0, implicit-def $sa0, implicit-def $usr
+
+  bb.1 (machine-block-address-taken):
+    successors: %bb.1, %bb.2
+
+    %0:intregs = PHI %11, %bb.0, %6, %bb.1
+    %4:intregs = L2_loadri_io %0, 0 :: (load (s32))
+    S2_storeri_io %0, 0, %10 :: (store (s32))
+    %7:intregs = A2_addi %0, -8
+    %6:intregs = A2_addi %7, 4
+    ENDLOOP0 %bb.1, implicit-def $pc, implicit-def $lc0, implicit $sa0, implicit $lc0
+    J2_jump %bb.2, implicit-def dead $pc
+
+  bb.2:
+
+...
diff --git a/llvm/test/CodeGen/Hexagon/swp-carried-dep5.mir b/llvm/test/CodeGen/Hexagon/swp-carried-dep5.mir
new file mode 100644
index 0000000000000..bf3cd95a84a72
--- /dev/null
+++ b/llvm/test/CodeGen/Hexagon/swp-carried-dep5.mir
@@ -0,0 +1,146 @@
+# RUN: llc -mtriple=hexagon -run-pass pipeliner -debug-only=pipeliner %s -o /dev/null 2>&1 | FileCheck %s
+# REQUIRES: asserts
+
+# Test boundary cases for loop carried dependence analysis
+
+#CHECK: Overlap check:
+#CHECK:   BaseMI:   S2_storeri_io %10:intregs, 0, %0:intregs :: (store (s32))
+#CHECK:     Base + 0 + I * 3, Len: 4
+#CHECK:   OtherMI:   dead %30:intregs = L2_loadri_io %10:intregs, 0 :: (load (s32))
+#CHECK:     Base + 0 + I * 3, Len: 4
+#CHECK:   Result: Overlap
+#CHECK: Overlap check:
+#CHECK:   BaseMI:   S2_storeri_io %12:intregs, 0, %1:intregs :: (store (s32))
+#CHECK:     Base + 0 + I * 4, Len: 4
+#CHECK:   OtherMI:   dead %31:intregs = L2_loadri_io %12:intregs, 0 :: (load (s32))
+#CHECK:     Base + 0 + I * 4, Len: 4
+#CHECK:   Result: No overlap
+#CHECK: Overlap check:
+#CHECK:   BaseMI:   S2_storeri_io %14:intregs, 0, %2:intregs :: (store (s64))
+#CHECK:     Base + 0 + I * 4, Len: 8
+#CHECK:   OtherMI:   dead %32:intregs = L2_loadri_io %14:intregs, 0 :: (load (s32))
+#CHECK:     Base + 0 + I * 4, Len: 4
+#CHECK:   Result: Overlap
+#CHECK: Overlap check:
+#CHECK:   BaseMI:   S2_storeri_io %16:intregs, 0, %3:intregs :: (store (s32))
+#CHECK:     Base + 0 + I * 4, Len: 4
+#CHECK:   OtherMI:   dead %33:intregs = L2_loadri_io %16:intregs, -1 :: (load (s32))
+#CHECK:     Base + -1 + I * 4, Len: 4
+#CHECK:   Result: Overlap
+#CHECK: Overlap check:
+#CHECK:   BaseMI:   S2_storeri_io %18:intregs, 1, %4:intregs :: (store (s32))
+#CHECK:     Base + 1 + I * 4, Len: 4
+#CHECK:   OtherMI:   dead %34:intregs = L2_loadri_io %18:intregs, 0 :: (load (s32))
+#CHECK:     Base + 0 + I * 4, Len: 4
+#CHECK:   Result: Overlap
+#CHECK: Overlap check:
+#CHECK:   BaseMI:   S2_storeri_io %20:intregs, 0, %5:intregs :: (store (s32))
+#CHECK:     Base + 0 + I * -3, Len: 4
+#CHECK:   OtherMI:   dead %35:intregs = L2_loadri_io %20:intregs, 0 :: (load (s32))
+#CHECK:     Base + 0 + I * -3, Len: 4
+#CHECK:   Result: Overlap
+#CHECK: Overlap check:
+#CHECK:   BaseMI:   S2_storeri_io %22:intregs, 0, %6:intregs :: (store (s32))
+#CHECK:     Base + 0 + I * -4, Len: 4
+#CHECK:   OtherMI:   dead %36:intregs = L2_loadri_io %22:intregs, 0 :: (load (s32))
+#CHECK:     Base + 0 + I * -4, Len: 4
+#CHECK:   Result: No overlap
+#CHECK: Overlap check:
+#CHECK:   BaseMI:   S2_storeri_io %24:intregs, 0, %7:intregs :: (store (s32))
+#CHECK:     Base + 0 + I * -4, Len: 4
+#CHECK:   OtherMI:   dead %37:intregs = L2_loadri_io %24:intregs, 0 :: (load (s64))
+#CHECK:     Base + 0 + I * -4, Len: 8
+#CHECK:   Result: Overlap
+#CHECK: Overlap check:
+#CHECK:   BaseMI:   S2_storeri_io %26:intregs, 0, %8:intregs :: (store (s32))
+#CHECK:     Base + 0 + I * -4, Len: 4
+#CHECK:   OtherMI:   dead %38:intregs = L2_loadri_io %26:intregs, 1 :: (load (s32))
+#CHECK:     Base + 1 + I * -4, Len: 4
+#CHECK:   Result: Overlap
+#CHECK: Overlap check:
+#CHECK:   BaseMI:   S2_storeri_io %28:intregs, -1, %9:intregs :: (store (s32))
+#CHECK:     Base + -1 + I * -4, Len: 4
+#CHECK:   OtherMI:   dead %39:intregs = L2_loadri_io %28:intregs, 0 :: (load (s32))
+#CHECK:     Base + 0 + I * -4, Len: 4
+#CHECK:   Result: Overlap
+
+---
+name:            test
+tracksRegLiveness: true
+
+body:             |
+  bb.0:
+    successors: %bb.1
+
+    %100:intregs = IMPLICIT_DEF
+    %200:intregs = IMPLICIT_DEF
+    %300:intregs = IMPLICIT_DEF
+    %400:intregs = IMPLICIT_DEF
+    %500:intregs = IMPLICIT_DEF
+    %600:intregs = IMPLICIT_DEF
+    %700:intregs = IMPLICIT_DEF
+    %800:intregs = IMPLICIT_DEF
+    %900:intregs = IMPLICIT_DEF
+    %1000:intregs = IMPLICIT_DEF
+    J2_loop0i %bb.1, 6, implicit-def $lc0, implicit-def $sa0, implicit-def $usr
+
+  bb.1 (machine-block-address-taken):
+    successors: %bb.1, %bb.2
+
+    %0:intregs = PHI %100, %bb.0, %1, %bb.1
+    %2:intregs = PHI %200, %bb.0, %3, %bb.1
+    %4:intregs = PHI %300, %bb.0, %5, %bb.1
+    %6:intregs = PHI %400, %bb.0, %7, %bb.1
+    %8:intregs = PHI %500, %bb.0, %9, %bb.1
+    %10:intregs = PHI %600, %bb.0, %11, %bb.1
+    %12:intregs = PHI %700, %bb.0, %13, %bb.1
+    %14:intregs = PHI %800, %bb.0, %15, %bb.1
+    %16:intregs = PHI %900, %bb.0, %17, %bb.1
+    %18:intregs = PHI %1000, %bb.0, %19, %bb.1
+
+    %1:intregs = A2_addi %0, 3
+    %101:intregs = L2_loadri_io %0, 0 :: (load (s32))
+    S2_storeri_io %0, 0, %100 :: (store (s32))
+
+    %3:intregs = A2_addi %2, 4
+    %201:intregs = L2_loadri_io %2, 0 :: (load (s32))
+    S2_storeri_io %2, 0, %200 :: (store (s32))
+
+    %5:intregs = A2_addi %4, 4
+    %301:intregs = L2_loadri_io %4, 0 :: (load (s32))
+    S2_storeri_io %4, 0, %300 :: (store (s64))
+
+    %7:intregs = A2_addi %6, 4
+    %401:intregs = L2_loadri_io %6, -1 :: (load (s32))
+    S2_storeri_io %6, 0, %400 :: (store (s32))
+
+    %9:intregs = A2_addi %8, 4
+    %501:intregs = L2_loadri_io %8, 0 :: (load (s32))
+    S2_storeri_io %8, 1, %500 :: (store (s32))
+
+    %11:intregs = A2_addi %10, -3
+    %601:intregs = L2_loadri_io %10, 0 :: (load (s32))
+    S2_storeri_io %10, 0, %600 :: (store (s32))
+
+    %13:intregs = A2_addi %12, -4
+    %701:intregs = L2_loadri_io %12, 0 :: (load (s32))
+    S2_storeri_io %12, 0, %700 :: (store (s32))
+
+    %15:intregs = A2_addi %14, -4
+    %801:intregs = L2_loadri_io %14, 0 :: (load (s64))
+    S2_storeri_io %14, 0, %800 :: (store (s32))
+
+    %17:intregs = A2_addi %16, -4
+    %901:intregs = L2_loadri_io %16, 1 :: (load (s32))
+    S2_storeri_io %16, 0, %900 :: (store (s32))
+
+    %19:intregs = A2_addi %18, -4
+    %1001:intregs = L2_loadri_io %18, 0 :: (load (s32))
+    S2_storeri_io %18, -1, %1000 :: (store (s32))
+
+    ENDLOOP0 %bb.1, implicit-def $pc, implicit-def $lc0, implicit $sa0, implicit $lc0
+    J2_jump %bb.2, implicit-def dead $pc
+
+  bb.2:
+
+...
diff --git a/llvm/test/CodeGen/Hexagon/swp-epilog-phi8.ll b/llvm/test/CodeGen/Hexagon/swp-epilog-phi8.ll
index 76434e79ad994..e9384e8710211 100644
--- a/llvm/test/CodeGen/Hexagon/swp-epilog-phi8.ll
+++ b/llvm/test/CodeGen/Hexagon/swp-epilog-phi8.ll
@@ -31,7 +31,7 @@ b3:                                               ; preds = %b3, %b2
   %v6 = load i32, ptr %v5, align 4
   %v7 = tail call i32 @llvm.hexagon.A2.subsat(i32 %v2, i32 %v6)
   store i32 %v7, ptr %v3, align 4
-  %v8 = add i32 %v0, -1
+  %v8 = add i32 %v0, 1
   %v9 = icmp sgt i32 %v8, 1
   br i1 %v9, label %b3, label %b4
 
diff --git a/llvm/test/CodeGen/LoongArch/annotate-tablejump.ll b/llvm/test/CodeGen/LoongArch/annotate-tablejump.ll
index a8c660609d990..2bba7f1dc0e96 100644
--- a/llvm/test/CodeGen/LoongArch/annotate-tablejump.ll
+++ b/llvm/test/CodeGen/LoongArch/annotate-tablejump.ll
@@ -57,7 +57,7 @@ define void @switch_4_arms(i32 %in, ptr %out) nounwind {
 ; LA64-JT-NEXT:    slli.d $a3, $a3, 3
 ; LA64-JT-NEXT:    pcalau12i $a4, %pc_hi20(.LJTI0_0)
 ; LA64-JT-NEXT:    addi.d $a4, $a4, %pc_lo12(.LJTI0_0)
-; LA64-JT-NEXT:    ldx.d $a3, $a3, $a4
+; LA64-JT-NEXT:    ldx.d $a3, $a4, $a3
 ; LA64-JT-NEXT:  .Ljrtb_0:
 ; LA64-JT-NEXT:    jr $a3
 ; LA64-JT-NEXT:  .LBB0_2: # %bb1
@@ -80,7 +80,7 @@ define void @switch_4_arms(i32 %in, ptr %out) nounwind {
 ; LA64-JT-NEXT:    slli.d $a3, $a3, 3
 ; LA64-JT-NEXT:    pcalau12i $a4, %pc_hi20(.LJTI0_1)
 ; LA64-JT-NEXT:    addi.d $a4, $a4, %pc_lo12(.LJTI0_1)
-; LA64-JT-NEXT:    ldx.d $a3, $a3, $a4
+; LA64-JT-NEXT:    ldx.d $a3, $a4, $a3
 ; LA64-JT-NEXT:  .Ljrtb_1:
 ; LA64-JT-NEXT:    jr $a3
 ; LA64-JT-NEXT:  .LBB0_9: # %exit2
diff --git a/llvm/test/CodeGen/LoongArch/ir-instruction/add.ll b/llvm/test/CodeGen/LoongArch/ir-instruction/add.ll
index f156f8d6afce5..69db3790fc1b3 100644
--- a/llvm/test/CodeGen/LoongArch/ir-instruction/add.ll
+++ b/llvm/test/CodeGen/LoongArch/ir-instruction/add.ll
@@ -7,12 +7,12 @@
 define i1 @add_i1(i1 %x, i1 %y) {
 ; LA32-LABEL: add_i1:
 ; LA32:       # %bb.0:
-; LA32-NEXT:    add.w $a0, $a0, $a1
+; LA32-NEXT:    xor $a0, $a0, $a1
 ; LA32-NEXT:    ret
 ;
 ; LA64-LABEL: add_i1:
 ; LA64:       # %bb.0:
-; LA64-NEXT:    add.d $a0, $a0, $a1
+; LA64-NEXT:    xor $a0, $a0, $a1
 ; LA64-NEXT:    ret
   %add = add i1 %x, %y
   ret i1 %add
@@ -97,12 +97,12 @@ define i64 @add_i64(i64 %x, i64 %y) {
 define i1 @add_i1_3(i1 %x) {
 ; LA32-LABEL: add_i1_3:
 ; LA32:       # %bb.0:
-; LA32-NEXT:    addi.w $a0, $a0, 1
+; LA32-NEXT:    xori $a0, $a0, 1
 ; LA32-NEXT:    ret
 ;
 ; LA64-LABEL: add_i1_3:
 ; LA64:       # %bb.0:
-; LA64-NEXT:    addi.d $a0, $a0, 1
+; LA64-NEXT:    xori $a0, $a0, 1
 ; LA64-NEXT:    ret
   %add = add i1 %x, 3
   ret i1 %add
diff --git a/llvm/test/CodeGen/LoongArch/ir-instruction/mul.ll b/llvm/test/CodeGen/LoongArch/ir-instruction/mul.ll
index 58cc0e7d6484a..3a0cfd00940c5 100644
--- a/llvm/test/CodeGen/LoongArch/ir-instruction/mul.ll
+++ b/llvm/test/CodeGen/LoongArch/ir-instruction/mul.ll
@@ -7,12 +7,12 @@
 define i1 @mul_i1(i1 %a, i1 %b) {
 ; LA32-LABEL: mul_i1:
 ; LA32:       # %bb.0: # %entry
-; LA32-NEXT:    mul.w $a0, $a0, $a1
+; LA32-NEXT:    and $a0, $a0, $a1
 ; LA32-NEXT:    ret
 ;
 ; LA64-LABEL: mul_i1:
 ; LA64:       # %bb.0: # %entry
-; LA64-NEXT:    mul.d $a0, $a0, $a1
+; LA64-NEXT:    and $a0, $a0, $a1
 ; LA64-NEXT:    ret
 entry:
   %r = mul i1 %a, %b
diff --git a/llvm/test/CodeGen/LoongArch/ir-instruction/sub.ll b/llvm/test/CodeGen/LoongArch/ir-instruction/sub.ll
index 12543f857a198..ce4a199b57c3d 100644
--- a/llvm/test/CodeGen/LoongArch/ir-instruction/sub.ll
+++ b/llvm/test/CodeGen/LoongArch/ir-instruction/sub.ll
@@ -7,12 +7,12 @@
 define i1 @sub_i1(i1 %x, i1 %y) {
 ; LA32-LABEL: sub_i1:
 ; LA32:       # %bb.0:
-; LA32-NEXT:    sub.w $a0, $a0, $a1
+; LA32-NEXT:    xor $a0, $a0, $a1
 ; LA32-NEXT:    ret
 ;
 ; LA64-LABEL: sub_i1:
 ; LA64:       # %bb.0:
-; LA64-NEXT:    sub.d $a0, $a0, $a1
+; LA64-NEXT:    xor $a0, $a0, $a1
 ; LA64-NEXT:    ret
   %sub = sub i1 %x, %y
   ret i1 %sub
diff --git a/llvm/test/CodeGen/LoongArch/jr-without-ra.ll b/llvm/test/CodeGen/LoongArch/jr-without-ra.ll
index d1c4459aaa6ee..1a1fe0e2b19e2 100644
--- a/llvm/test/CodeGen/LoongArch/jr-without-ra.ll
+++ b/llvm/test/CodeGen/LoongArch/jr-without-ra.ll
@@ -78,7 +78,7 @@ define void @jr_without_ra(ptr %rtwdev, ptr %chan, ptr %h2c, i8 %.pre, i1 %cmp.i
 ; CHECK-NEXT:  # %bb.6: # %calc_6g.i
 ; CHECK-NEXT:    # in Loop: Header=BB0_4 Depth=1
 ; CHECK-NEXT:    slli.d $s8, $zero, 3
-; CHECK-NEXT:    ldx.d $s8, $s8, $s1
+; CHECK-NEXT:    ldx.d $s8, $s1, $s8
 ; CHECK-NEXT:    jr $s8
 ; CHECK-NEXT:  .LBB0_7: # %sw.bb12.i.i
 ; CHECK-NEXT:    # in Loop: Header=BB0_4 Depth=1
@@ -113,7 +113,7 @@ define void @jr_without_ra(ptr %rtwdev, ptr %chan, ptr %h2c, i8 %.pre, i1 %cmp.i
 ; CHECK-NEXT:    # in Loop: Header=BB0_4 Depth=1
 ; CHECK-NEXT:    pcalau12i $ra, %pc_hi20(.LJTI0_1)
 ; CHECK-NEXT:    addi.d $ra, $ra, %pc_lo12(.LJTI0_1)
-; CHECK-NEXT:    ldx.d $s5, $s4, $ra
+; CHECK-NEXT:    ldx.d $s5, $ra, $s4
 ; CHECK-NEXT:    jr $s5
 ; CHECK-NEXT:  .LBB0_13: # %phy_tssi_get_ofdm_trim_de.exit
 ; CHECK-NEXT:    # in Loop: Header=BB0_4 Depth=1
diff --git a/llvm/test/CodeGen/LoongArch/jump-table.ll b/llvm/test/CodeGen/LoongArch/jump-table.ll
index 51f1ee6da0b85..e80915c1e8e0f 100644
--- a/llvm/test/CodeGen/LoongArch/jump-table.ll
+++ b/llvm/test/CodeGen/LoongArch/jump-table.ll
@@ -116,7 +116,7 @@ define void @switch_4_arms(i32 %in, ptr %out) nounwind {
 ; LA64-JT-NEXT:    slli.d $a0, $a0, 3
 ; LA64-JT-NEXT:    pcalau12i $a2, %pc_hi20(.LJTI0_0)
 ; LA64-JT-NEXT:    addi.d $a2, $a2, %pc_lo12(.LJTI0_0)
-; LA64-JT-NEXT:    ldx.d $a0, $a0, $a2
+; LA64-JT-NEXT:    ldx.d $a0, $a2, $a0
 ; LA64-JT-NEXT:    jr $a0
 ; LA64-JT-NEXT:  .LBB0_2: # %bb1
 ; LA64-JT-NEXT:    ori $a0, $zero, 4
diff --git a/llvm/test/CodeGen/LoongArch/machinelicm-address-pseudos.ll b/llvm/test/CodeGen/LoongArch/machinelicm-address-pseudos.ll
index e0a93e3051bf8..92d079ab3a8d8 100644
--- a/llvm/test/CodeGen/LoongArch/machinelicm-address-pseudos.ll
+++ b/llvm/test/CodeGen/LoongArch/machinelicm-address-pseudos.ll
@@ -317,11 +317,10 @@ define void @test_la_tls_le(i32 signext %n) {
 ; LA32-NEXT:    move $a1, $zero
 ; LA32-NEXT:    lu12i.w $a2, %le_hi20_r(le)
 ; LA32-NEXT:    add.w $a2, $a2, $tp, %le_add_r(le)
-; LA32-NEXT:    addi.w $a2, $a2, %le_lo12_r(le)
 ; LA32-NEXT:    .p2align 4, , 16
 ; LA32-NEXT:  .LBB4_1: # %loop
 ; LA32-NEXT:    # =>This Inner Loop Header: Depth=1
-; LA32-NEXT:    ld.w $zero, $a2, 0
+; LA32-NEXT:    ld.w $zero, $a2, %le_lo12_r(le)
 ; LA32-NEXT:    addi.w $a1, $a1, 1
 ; LA32-NEXT:    blt $a1, $a0, .LBB4_1
 ; LA32-NEXT:  # %bb.2: # %ret
@@ -332,11 +331,10 @@ define void @test_la_tls_le(i32 signext %n) {
 ; LA64-NEXT:    move $a1, $zero
 ; LA64-NEXT:    lu12i.w $a2, %le_hi20_r(le)
 ; LA64-NEXT:    add.d $a2, $a2, $tp, %le_add_r(le)
-; LA64-NEXT:    addi.d $a2, $a2, %le_lo12_r(le)
 ; LA64-NEXT:    .p2align 4, , 16
 ; LA64-NEXT:  .LBB4_1: # %loop
 ; LA64-NEXT:    # =>This Inner Loop Header: Depth=1
-; LA64-NEXT:    ld.w $zero, $a2, 0
+; LA64-NEXT:    ld.w $zero, $a2, %le_lo12_r(le)
 ; LA64-NEXT:    addi.w $a1, $a1, 1
 ; LA64-NEXT:    blt $a1, $a0, .LBB4_1
 ; LA64-NEXT:  # %bb.2: # %ret
diff --git a/llvm/test/CodeGen/LoongArch/merge-base-offset-tlsle.ll b/llvm/test/CodeGen/LoongArch/merge-base-offset-tlsle.ll
new file mode 100644
index 0000000000000..9ed9a865ce55d
--- /dev/null
+++ b/llvm/test/CodeGen/LoongArch/merge-base-offset-tlsle.ll
@@ -0,0 +1,883 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc --mtriple=loongarch32 --mattr=+lasx --verify-machineinstrs < %s \
+; RUN:   | FileCheck --check-prefix=LA32 %s
+; RUN: llc --mtriple=loongarch64 --mattr=+lasx --verify-machineinstrs < %s \
+; RUN:   | FileCheck --check-prefix=LA64 %s
+
+@g_i8 = dso_local thread_local(localexec) global i8 0
+
+define dso_local signext i8 @tlsle_load_s8() nounwind {
+; LA32-LABEL: tlsle_load_s8:
+; LA32:       # %bb.0: # %entry
+; LA32-NEXT:    lu12i.w $a0, %le_hi20_r(g_i8)
+; LA32-NEXT:    add.w $a0, $a0, $tp, %le_add_r(g_i8)
+; LA32-NEXT:    ld.b $a0, $a0, %le_lo12_r(g_i8)
+; LA32-NEXT:    ret
+;
+; LA64-LABEL: tlsle_load_s8:
+; LA64:       # %bb.0: # %entry
+; LA64-NEXT:    lu12i.w $a0, %le_hi20_r(g_i8)
+; LA64-NEXT:    add.d $a0, $a0, $tp, %le_add_r(g_i8)
+; LA64-NEXT:    ld.b $a0, $a0, %le_lo12_r(g_i8)
+; LA64-NEXT:    ret
+entry:
+  %0 = call ptr @llvm.threadlocal.address.p0(ptr @g_i8)
+  %1 = load i8, ptr %0
+  ret i8 %1
+}
+
+define dso_local zeroext i8 @tlsle_load_u8() nounwind {
+; LA32-LABEL: tlsle_load_u8:
+; LA32:       # %bb.0: # %entry
+; LA32-NEXT:    lu12i.w $a0, %le_hi20_r(g_i8)
+; LA32-NEXT:    add.w $a0, $a0, $tp, %le_add_r(g_i8)
+; LA32-NEXT:    ld.bu $a0, $a0, %le_lo12_r(g_i8)
+; LA32-NEXT:    ret
+;
+; LA64-LABEL: tlsle_load_u8:
+; LA64:       # %bb.0: # %entry
+; LA64-NEXT:    lu12i.w $a0, %le_hi20_r(g_i8)
+; LA64-NEXT:    add.d $a0, $a0, $tp, %le_add_r(g_i8)
+; LA64-NEXT:    ld.bu $a0, $a0, %le_lo12_r(g_i8)
+; LA64-NEXT:    ret
+entry:
+  %0 = call ptr @llvm.threadlocal.address.p0(ptr @g_i8)
+  %1 = load i8, ptr %0
+  ret i8 %1
+}
+
+define dso_local void @tlsle_store_i8() nounwind {
+; LA32-LABEL: tlsle_store_i8:
+; LA32:       # %bb.0: # %entry
+; LA32-NEXT:    lu12i.w $a0, %le_hi20_r(g_i8)
+; LA32-NEXT:    add.w $a0, $a0, $tp, %le_add_r(g_i8)
+; LA32-NEXT:    ori $a1, $zero, 1
+; LA32-NEXT:    st.b $a1, $a0, %le_lo12_r(g_i8)
+; LA32-NEXT:    ret
+;
+; LA64-LABEL: tlsle_store_i8:
+; LA64:       # %bb.0: # %entry
+; LA64-NEXT:    lu12i.w $a0, %le_hi20_r(g_i8)
+; LA64-NEXT:    add.d $a0, $a0, $tp, %le_add_r(g_i8)
+; LA64-NEXT:    ori $a1, $zero, 1
+; LA64-NEXT:    st.b $a1, $a0, %le_lo12_r(g_i8)
+; LA64-NEXT:    ret
+entry:
+  %0 = call ptr @llvm.threadlocal.address.p0(ptr @g_i8)
+  store i8 1, ptr %0
+  ret void
+}
+
+@g_i16 = dso_local thread_local(localexec) global i16 0
+
+define dso_local signext i16 @tlsle_load_s16() nounwind {
+; LA32-LABEL: tlsle_load_s16:
+; LA32:       # %bb.0: # %entry
+; LA32-NEXT:    lu12i.w $a0, %le_hi20_r(g_i16)
+; LA32-NEXT:    add.w $a0, $a0, $tp, %le_add_r(g_i16)
+; LA32-NEXT:    ld.h $a0, $a0, %le_lo12_r(g_i16)
+; LA32-NEXT:    ret
+;
+; LA64-LABEL: tlsle_load_s16:
+; LA64:       # %bb.0: # %entry
+; LA64-NEXT:    lu12i.w $a0, %le_hi20_r(g_i16)
+; LA64-NEXT:    add.d $a0, $a0, $tp, %le_add_r(g_i16)
+; LA64-NEXT:    ld.h $a0, $a0, %le_lo12_r(g_i16)
+; LA64-NEXT:    ret
+entry:
+  %0 = call ptr @llvm.threadlocal.address.p0(ptr @g_i16)
+  %1 = load i16, ptr %0
+  ret i16 %1
+}
+
+define dso_local zeroext i16 @tlsle_load_u16() nounwind {
+; LA32-LABEL: tlsle_load_u16:
+; LA32:       # %bb.0: # %entry
+; LA32-NEXT:    lu12i.w $a0, %le_hi20_r(g_i16)
+; LA32-NEXT:    add.w $a0, $a0, $tp, %le_add_r(g_i16)
+; LA32-NEXT:    ld.hu $a0, $a0, %le_lo12_r(g_i16)
+; LA32-NEXT:    ret
+;
+; LA64-LABEL: tlsle_load_u16:
+; LA64:       # %bb.0: # %entry
+; LA64-NEXT:    lu12i.w $a0, %le_hi20_r(g_i16)
+; LA64-NEXT:    add.d $a0, $a0, $tp, %le_add_r(g_i16)
+; LA64-NEXT:    ld.hu $a0, $a0, %le_lo12_r(g_i16)
+; LA64-NEXT:    ret
+entry:
+  %0 = call ptr @llvm.threadlocal.address.p0(ptr @g_i16)
+  %1 = load i16, ptr %0
+  ret i16 %1
+}
+
+define dso_local void @tlsle_store_i16() nounwind {
+; LA32-LABEL: tlsle_store_i16:
+; LA32:       # %bb.0: # %entry
+; LA32-NEXT:    lu12i.w $a0, %le_hi20_r(g_i16)
+; LA32-NEXT:    add.w $a0, $a0, $tp, %le_add_r(g_i16)
+; LA32-NEXT:    ori $a1, $zero, 1
+; LA32-NEXT:    st.h $a1, $a0, %le_lo12_r(g_i16)
+; LA32-NEXT:    ret
+;
+; LA64-LABEL: tlsle_store_i16:
+; LA64:       # %bb.0: # %entry
+; LA64-NEXT:    lu12i.w $a0, %le_hi20_r(g_i16)
+; LA64-NEXT:    add.d $a0, $a0, $tp, %le_add_r(g_i16)
+; LA64-NEXT:    ori $a1, $zero, 1
+; LA64-NEXT:    st.h $a1, $a0, %le_lo12_r(g_i16)
+; LA64-NEXT:    ret
+entry:
+  %0 = call ptr @llvm.threadlocal.address.p0(ptr @g_i16)
+  store i16 1, ptr %0
+  ret void
+}
+
+@g_i32 = dso_local thread_local(localexec) global i32 0
+
+define dso_local signext i32 @tlsle_load_s32() nounwind {
+; LA32-LABEL: tlsle_load_s32:
+; LA32:       # %bb.0: # %entry
+; LA32-NEXT:    lu12i.w $a0, %le_hi20_r(g_i32)
+; LA32-NEXT:    add.w $a0, $a0, $tp, %le_add_r(g_i32)
+; LA32-NEXT:    ld.w $a0, $a0, %le_lo12_r(g_i32)
+; LA32-NEXT:    ret
+;
+; LA64-LABEL: tlsle_load_s32:
+; LA64:       # %bb.0: # %entry
+; LA64-NEXT:    lu12i.w $a0, %le_hi20_r(g_i32)
+; LA64-NEXT:    add.d $a0, $a0, $tp, %le_add_r(g_i32)
+; LA64-NEXT:    ld.w $a0, $a0, %le_lo12_r(g_i32)
+; LA64-NEXT:    ret
+entry:
+  %0 = call ptr @llvm.threadlocal.address.p0(ptr @g_i32)
+  %1 = load i32, ptr %0
+  ret i32 %1
+}
+
+define dso_local zeroext i32 @tlsle_load_u32() nounwind {
+; LA32-LABEL: tlsle_load_u32:
+; LA32:       # %bb.0: # %entry
+; LA32-NEXT:    lu12i.w $a0, %le_hi20_r(g_i32)
+; LA32-NEXT:    add.w $a0, $a0, $tp, %le_add_r(g_i32)
+; LA32-NEXT:    ld.w $a0, $a0, %le_lo12_r(g_i32)
+; LA32-NEXT:    ret
+;
+; LA64-LABEL: tlsle_load_u32:
+; LA64:       # %bb.0: # %entry
+; LA64-NEXT:    lu12i.w $a0, %le_hi20_r(g_i32)
+; LA64-NEXT:    add.d $a0, $a0, $tp, %le_add_r(g_i32)
+; LA64-NEXT:    ld.wu $a0, $a0, %le_lo12_r(g_i32)
+; LA64-NEXT:    ret
+entry:
+  %0 = call ptr @llvm.threadlocal.address.p0(ptr @g_i32)
+  %1 = load i32, ptr %0
+  ret i32 %1
+}
+
+define dso_local void @tlsle_store_i32() nounwind {
+; LA32-LABEL: tlsle_store_i32:
+; LA32:       # %bb.0: # %entry
+; LA32-NEXT:    lu12i.w $a0, %le_hi20_r(g_i32)
+; LA32-NEXT:    add.w $a0, $a0, $tp, %le_add_r(g_i32)
+; LA32-NEXT:    ori $a1, $zero, 1
+; LA32-NEXT:    st.w $a1, $a0, %le_lo12_r(g_i32)
+; LA32-NEXT:    ret
+;
+; LA64-LABEL: tlsle_store_i32:
+; LA64:       # %bb.0: # %entry
+; LA64-NEXT:    lu12i.w $a0, %le_hi20_r(g_i32)
+; LA64-NEXT:    add.d $a0, $a0, $tp, %le_add_r(g_i32)
+; LA64-NEXT:    ori $a1, $zero, 1
+; LA64-NEXT:    st.w $a1, $a0, %le_lo12_r(g_i32)
+; LA64-NEXT:    ret
+entry:
+  %0 = call ptr @llvm.threadlocal.address.p0(ptr @g_i32)
+  store i32 1, ptr %0
+  ret void
+}
+
+@g_i64 = dso_local thread_local(localexec) global i64 0
+
+define dso_local i64 @tlsle_load_i64() nounwind {
+; LA32-LABEL: tlsle_load_i64:
+; LA32:       # %bb.0: # %entry
+; LA32-NEXT:    lu12i.w $a0, %le_hi20_r(g_i64)
+; LA32-NEXT:    add.w $a0, $a0, $tp, %le_add_r(g_i64)
+; LA32-NEXT:    addi.w $a1, $a0, %le_lo12_r(g_i64)
+; LA32-NEXT:    ld.w $a0, $a1, 0
+; LA32-NEXT:    ld.w $a1, $a1, 4
+; LA32-NEXT:    ret
+;
+; LA64-LABEL: tlsle_load_i64:
+; LA64:       # %bb.0: # %entry
+; LA64-NEXT:    lu12i.w $a0, %le_hi20_r(g_i64)
+; LA64-NEXT:    add.d $a0, $a0, $tp, %le_add_r(g_i64)
+; LA64-NEXT:    ld.d $a0, $a0, %le_lo12_r(g_i64)
+; LA64-NEXT:    ret
+entry:
+  %0 = call ptr @llvm.threadlocal.address.p0(ptr @g_i64)
+  %1 = load i64, ptr %0
+  ret i64 %1
+}
+
+define dso_local void @tlsle_store_i64() nounwind {
+; LA32-LABEL: tlsle_store_i64:
+; LA32:       # %bb.0: # %entry
+; LA32-NEXT:    lu12i.w $a0, %le_hi20_r(g_i64)
+; LA32-NEXT:    add.w $a0, $a0, $tp, %le_add_r(g_i64)
+; LA32-NEXT:    addi.w $a0, $a0, %le_lo12_r(g_i64)
+; LA32-NEXT:    st.w $zero, $a0, 4
+; LA32-NEXT:    ori $a1, $zero, 1
+; LA32-NEXT:    st.w $a1, $a0, 0
+; LA32-NEXT:    ret
+;
+; LA64-LABEL: tlsle_store_i64:
+; LA64:       # %bb.0: # %entry
+; LA64-NEXT:    lu12i.w $a0, %le_hi20_r(g_i64)
+; LA64-NEXT:    add.d $a0, $a0, $tp, %le_add_r(g_i64)
+; LA64-NEXT:    ori $a1, $zero, 1
+; LA64-NEXT:    st.d $a1, $a0, %le_lo12_r(g_i64)
+; LA64-NEXT:    ret
+entry:
+  %0 = call ptr @llvm.threadlocal.address.p0(ptr @g_i64)
+  store i64 1, ptr %0
+  ret void
+}
+
+@g_f32 = dso_local thread_local(localexec) global float 0.0
+
+define dso_local float @tlsle_load_f32() nounwind {
+; LA32-LABEL: tlsle_load_f32:
+; LA32:       # %bb.0: # %entry
+; LA32-NEXT:    lu12i.w $a0, %le_hi20_r(g_f32)
+; LA32-NEXT:    add.w $a0, $a0, $tp, %le_add_r(g_f32)
+; LA32-NEXT:    fld.s $fa0, $a0, %le_lo12_r(g_f32)
+; LA32-NEXT:    ret
+;
+; LA64-LABEL: tlsle_load_f32:
+; LA64:       # %bb.0: # %entry
+; LA64-NEXT:    lu12i.w $a0, %le_hi20_r(g_f32)
+; LA64-NEXT:    add.d $a0, $a0, $tp, %le_add_r(g_f32)
+; LA64-NEXT:    fld.s $fa0, $a0, %le_lo12_r(g_f32)
+; LA64-NEXT:    ret
+entry:
+  %0 = call ptr @llvm.threadlocal.address.p0(ptr @g_f32)
+  %1 = load float, ptr %0
+  ret float %1
+}
+
+define dso_local void @tlsle_store_f32() nounwind {
+; LA32-LABEL: tlsle_store_f32:
+; LA32:       # %bb.0: # %entry
+; LA32-NEXT:    lu12i.w $a0, %le_hi20_r(g_f32)
+; LA32-NEXT:    add.w $a0, $a0, $tp, %le_add_r(g_f32)
+; LA32-NEXT:    lu12i.w $a1, 260096
+; LA32-NEXT:    st.w $a1, $a0, %le_lo12_r(g_f32)
+; LA32-NEXT:    ret
+;
+; LA64-LABEL: tlsle_store_f32:
+; LA64:       # %bb.0: # %entry
+; LA64-NEXT:    lu12i.w $a0, %le_hi20_r(g_f32)
+; LA64-NEXT:    add.d $a0, $a0, $tp, %le_add_r(g_f32)
+; LA64-NEXT:    lu12i.w $a1, 260096
+; LA64-NEXT:    st.w $a1, $a0, %le_lo12_r(g_f32)
+; LA64-NEXT:    ret
+entry:
+  %0 = call ptr @llvm.threadlocal.address.p0(ptr @g_f32)
+  store float 1.0, ptr %0
+  ret void
+}
+
+@g_f64 = dso_local thread_local(localexec) global double 0.0
+
+define dso_local double @tlsle_load_f64() nounwind {
+; LA32-LABEL: tlsle_load_f64:
+; LA32:       # %bb.0: # %entry
+; LA32-NEXT:    lu12i.w $a0, %le_hi20_r(g_f64)
+; LA32-NEXT:    add.w $a0, $a0, $tp, %le_add_r(g_f64)
+; LA32-NEXT:    fld.d $fa0, $a0, %le_lo12_r(g_f64)
+; LA32-NEXT:    ret
+;
+; LA64-LABEL: tlsle_load_f64:
+; LA64:       # %bb.0: # %entry
+; LA64-NEXT:    lu12i.w $a0, %le_hi20_r(g_f64)
+; LA64-NEXT:    add.d $a0, $a0, $tp, %le_add_r(g_f64)
+; LA64-NEXT:    fld.d $fa0, $a0, %le_lo12_r(g_f64)
+; LA64-NEXT:    ret
+entry:
+  %0 = call ptr @llvm.threadlocal.address.p0(ptr @g_f64)
+  %1 = load double, ptr %0
+  ret double %1
+}
+
+define dso_local void @tlsle_store_f64() nounwind {
+; LA32-LABEL: tlsle_store_f64:
+; LA32:       # %bb.0: # %entry
+; LA32-NEXT:    lu12i.w $a0, %le_hi20_r(g_f64)
+; LA32-NEXT:    add.w $a0, $a0, $tp, %le_add_r(g_f64)
+; LA32-NEXT:    vldi $vr0, -912
+; LA32-NEXT:    fst.d $fa0, $a0, %le_lo12_r(g_f64)
+; LA32-NEXT:    ret
+;
+; LA64-LABEL: tlsle_store_f64:
+; LA64:       # %bb.0: # %entry
+; LA64-NEXT:    lu12i.w $a0, %le_hi20_r(g_f64)
+; LA64-NEXT:    add.d $a0, $a0, $tp, %le_add_r(g_f64)
+; LA64-NEXT:    lu52i.d $a1, $zero, 1023
+; LA64-NEXT:    st.d $a1, $a0, %le_lo12_r(g_f64)
+; LA64-NEXT:    ret
+entry:
+  %0 = call ptr @llvm.threadlocal.address.p0(ptr @g_f64)
+  store double 1.0, ptr %0
+  ret void
+}
+
+@g_m64 = dso_local thread_local(localexec) global i64 0
+
+define dso_local void @tlsle_store_multi() nounwind {
+; LA32-LABEL: tlsle_store_multi:
+; LA32:       # %bb.0: # %entry
+; LA32-NEXT:    lu12i.w $a0, %le_hi20_r(g_m64)
+; LA32-NEXT:    add.w $a0, $a0, $tp, %le_add_r(g_m64)
+; LA32-NEXT:    addi.w $a0, $a0, %le_lo12_r(g_m64)
+; LA32-NEXT:    st.w $zero, $a0, 4
+; LA32-NEXT:    ori $a1, $zero, 1
+; LA32-NEXT:    st.w $a1, $a0, 0
+; LA32-NEXT:    st.w $zero, $a0, 4
+; LA32-NEXT:    ori $a1, $zero, 2
+; LA32-NEXT:    st.w $a1, $a0, 0
+; LA32-NEXT:    ret
+;
+; LA64-LABEL: tlsle_store_multi:
+; LA64:       # %bb.0: # %entry
+; LA64-NEXT:    lu12i.w $a0, %le_hi20_r(g_m64)
+; LA64-NEXT:    add.d $a0, $a0, $tp, %le_add_r(g_m64)
+; LA64-NEXT:    ori $a1, $zero, 1
+; LA64-NEXT:    st.d $a1, $a0, %le_lo12_r(g_m64)
+; LA64-NEXT:    ori $a1, $zero, 2
+; LA64-NEXT:    st.d $a1, $a0, %le_lo12_r(g_m64)
+; LA64-NEXT:    ret
+entry:
+  %0 = call ptr @llvm.threadlocal.address.p0(ptr @g_m64)
+  store volatile i64 1, ptr %0
+  store volatile i64 2, ptr %0
+  ret void
+}
+
+@g_sf32 = dso_local thread_local(localexec) global float 0.0
+
+define dso_local void @tlsle_store_sf32() nounwind {
+; LA32-LABEL: tlsle_store_sf32:
+; LA32:       # %bb.0: # %entry
+; LA32-NEXT:    lu12i.w $a0, %le_hi20_r(g_sf32)
+; LA32-NEXT:    add.w $a0, $a0, $tp, %le_add_r(g_sf32)
+; LA32-NEXT:    fld.s $fa0, $a0, %le_lo12_r(g_sf32)
+; LA32-NEXT:    fst.s $fa0, $a0, %le_lo12_r(g_sf32)
+; LA32-NEXT:    ret
+;
+; LA64-LABEL: tlsle_store_sf32:
+; LA64:       # %bb.0: # %entry
+; LA64-NEXT:    lu12i.w $a0, %le_hi20_r(g_sf32)
+; LA64-NEXT:    add.d $a0, $a0, $tp, %le_add_r(g_sf32)
+; LA64-NEXT:    fld.s $fa0, $a0, %le_lo12_r(g_sf32)
+; LA64-NEXT:    fst.s $fa0, $a0, %le_lo12_r(g_sf32)
+; LA64-NEXT:    ret
+entry:
+  %0 = call ptr @llvm.threadlocal.address.p0(ptr @g_sf32)
+  %1 = load float, ptr %0
+  store volatile float %1, ptr %0
+  ret void
+}
+
+@g_sf64 = dso_local thread_local(localexec) global double 0.0
+
+define dso_local void @tlsle_store_sf64() nounwind {
+; LA32-LABEL: tlsle_store_sf64:
+; LA32:       # %bb.0: # %entry
+; LA32-NEXT:    lu12i.w $a0, %le_hi20_r(g_sf64)
+; LA32-NEXT:    add.w $a0, $a0, $tp, %le_add_r(g_sf64)
+; LA32-NEXT:    fld.d $fa0, $a0, %le_lo12_r(g_sf64)
+; LA32-NEXT:    fst.d $fa0, $a0, %le_lo12_r(g_sf64)
+; LA32-NEXT:    ret
+;
+; LA64-LABEL: tlsle_store_sf64:
+; LA64:       # %bb.0: # %entry
+; LA64-NEXT:    lu12i.w $a0, %le_hi20_r(g_sf64)
+; LA64-NEXT:    add.d $a0, $a0, $tp, %le_add_r(g_sf64)
+; LA64-NEXT:    fld.d $fa0, $a0, %le_lo12_r(g_sf64)
+; LA64-NEXT:    fst.d $fa0, $a0, %le_lo12_r(g_sf64)
+; LA64-NEXT:    ret
+entry:
+  %0 = call ptr @llvm.threadlocal.address.p0(ptr @g_sf64)
+  %1 = load double, ptr %0
+  store volatile double %1, ptr %0
+  ret void
+}
+
+@g_i32x4_src = dso_local thread_local(localexec) global [4 x i32] zeroinitializer, align 16
+@g_i32x4_dst = dso_local thread_local(localexec) global [4 x i32] zeroinitializer, align 16
+
+define dso_local void @tlsle_copy_i32x4() nounwind {
+; LA32-LABEL: tlsle_copy_i32x4:
+; LA32:       # %bb.0: # %entry
+; LA32-NEXT:    lu12i.w $a0, %le_hi20_r(g_i32x4_src)
+; LA32-NEXT:    add.w $a0, $a0, $tp, %le_add_r(g_i32x4_src)
+; LA32-NEXT:    vld $vr0, $a0, %le_lo12_r(g_i32x4_src)
+; LA32-NEXT:    lu12i.w $a0, %le_hi20_r(g_i32x4_dst)
+; LA32-NEXT:    add.w $a0, $a0, $tp, %le_add_r(g_i32x4_dst)
+; LA32-NEXT:    vst $vr0, $a0, %le_lo12_r(g_i32x4_dst)
+; LA32-NEXT:    ret
+;
+; LA64-LABEL: tlsle_copy_i32x4:
+; LA64:       # %bb.0: # %entry
+; LA64-NEXT:    lu12i.w $a0, %le_hi20_r(g_i32x4_src)
+; LA64-NEXT:    add.d $a0, $a0, $tp, %le_add_r(g_i32x4_src)
+; LA64-NEXT:    vld $vr0, $a0, %le_lo12_r(g_i32x4_src)
+; LA64-NEXT:    lu12i.w $a0, %le_hi20_r(g_i32x4_dst)
+; LA64-NEXT:    add.d $a0, $a0, $tp, %le_add_r(g_i32x4_dst)
+; LA64-NEXT:    vst $vr0, $a0, %le_lo12_r(g_i32x4_dst)
+; LA64-NEXT:    ret
+entry:
+  %0 = call ptr @llvm.threadlocal.address.p0(ptr @g_i32x4_src)
+  %1 = load <4 x i32>, ptr %0, align 16
+  %2 = call ptr @llvm.threadlocal.address.p0(ptr @g_i32x4_dst)
+  store <4 x i32> %1, ptr %2, align 16
+  ret void
+}
+
+@g_i32x8_src = dso_local thread_local(localexec) global [8 x i32] zeroinitializer, align 32
+@g_i32x8_dst = dso_local thread_local(localexec) global [8 x i32] zeroinitializer, align 32
+
+define dso_local void @tlsle_copy_i32x8() nounwind {
+; LA32-LABEL: tlsle_copy_i32x8:
+; LA32:       # %bb.0: # %entry
+; LA32-NEXT:    lu12i.w $a0, %le_hi20_r(g_i32x8_src)
+; LA32-NEXT:    add.w $a0, $a0, $tp, %le_add_r(g_i32x8_src)
+; LA32-NEXT:    xvld $xr0, $a0, %le_lo12_r(g_i32x8_src)
+; LA32-NEXT:    lu12i.w $a0, %le_hi20_r(g_i32x8_dst)
+; LA32-NEXT:    add.w $a0, $a0, $tp, %le_add_r(g_i32x8_dst)
+; LA32-NEXT:    xvst $xr0, $a0, %le_lo12_r(g_i32x8_dst)
+; LA32-NEXT:    ret
+;
+; LA64-LABEL: tlsle_copy_i32x8:
+; LA64:       # %bb.0: # %entry
+; LA64-NEXT:    lu12i.w $a0, %le_hi20_r(g_i32x8_src)
+; LA64-NEXT:    add.d $a0, $a0, $tp, %le_add_r(g_i32x8_src)
+; LA64-NEXT:    xvld $xr0, $a0, %le_lo12_r(g_i32x8_src)
+; LA64-NEXT:    lu12i.w $a0, %le_hi20_r(g_i32x8_dst)
+; LA64-NEXT:    add.d $a0, $a0, $tp, %le_add_r(g_i32x8_dst)
+; LA64-NEXT:    xvst $xr0, $a0, %le_lo12_r(g_i32x8_dst)
+; LA64-NEXT:    ret
+entry:
+  %0 = call ptr @llvm.threadlocal.address.p0(ptr @g_i32x8_src)
+  %1 = load <8 x i32>, ptr %0, align 32
+  %2 = call ptr @llvm.threadlocal.address.p0(ptr @g_i32x8_dst)
+  store <8 x i32> %1, ptr %2, align 32
+  ret void
+}
+
+@g_i8x16 = dso_local thread_local(localexec) global <16 x i8> zeroinitializer, align 16
+
+define dso_local void @tlsle_copy_i8_to_i8x16() nounwind {
+; LA32-LABEL: tlsle_copy_i8_to_i8x16:
+; LA32:       # %bb.0: # %entry
+; LA32-NEXT:    lu12i.w $a0, %le_hi20_r(g_i8)
+; LA32-NEXT:    add.w $a0, $a0, $tp, %le_add_r(g_i8)
+; LA32-NEXT:    vldrepl.b $vr0, $a0, %le_lo12_r(g_i8)
+; LA32-NEXT:    lu12i.w $a0, %le_hi20_r(g_i8x16)
+; LA32-NEXT:    add.w $a0, $a0, $tp, %le_add_r(g_i8x16)
+; LA32-NEXT:    vst $vr0, $a0, %le_lo12_r(g_i8x16)
+; LA32-NEXT:    ret
+;
+; LA64-LABEL: tlsle_copy_i8_to_i8x16:
+; LA64:       # %bb.0: # %entry
+; LA64-NEXT:    lu12i.w $a0, %le_hi20_r(g_i8)
+; LA64-NEXT:    add.d $a0, $a0, $tp, %le_add_r(g_i8)
+; LA64-NEXT:    vldrepl.b $vr0, $a0, %le_lo12_r(g_i8)
+; LA64-NEXT:    lu12i.w $a0, %le_hi20_r(g_i8x16)
+; LA64-NEXT:    add.d $a0, $a0, $tp, %le_add_r(g_i8x16)
+; LA64-NEXT:    vst $vr0, $a0, %le_lo12_r(g_i8x16)
+; LA64-NEXT:    ret
+entry:
+  %0 = call ptr @llvm.threadlocal.address.p0(ptr @g_i8)
+  %1 = call <16 x i8> @llvm.loongarch.lsx.vldrepl.b(ptr %0, i32 0)
+  %2 = call ptr @llvm.threadlocal.address.p0(ptr @g_i8x16)
+  store <16 x i8> %1, ptr %2, align 16
+  ret void
+}
+
+@g_i8x32 = dso_local thread_local(localexec) global <32 x i8> zeroinitializer, align 32
+
+define dso_local void @tlsle_copy_i8_to_i8x32() nounwind {
+; LA32-LABEL: tlsle_copy_i8_to_i8x32:
+; LA32:       # %bb.0: # %entry
+; LA32-NEXT:    lu12i.w $a0, %le_hi20_r(g_i8)
+; LA32-NEXT:    add.w $a0, $a0, $tp, %le_add_r(g_i8)
+; LA32-NEXT:    xvldrepl.b $xr0, $a0, %le_lo12_r(g_i8)
+; LA32-NEXT:    lu12i.w $a0, %le_hi20_r(g_i8x32)
+; LA32-NEXT:    add.w $a0, $a0, $tp, %le_add_r(g_i8x32)
+; LA32-NEXT:    xvst $xr0, $a0, %le_lo12_r(g_i8x32)
+; LA32-NEXT:    ret
+;
+; LA64-LABEL: tlsle_copy_i8_to_i8x32:
+; LA64:       # %bb.0: # %entry
+; LA64-NEXT:    lu12i.w $a0, %le_hi20_r(g_i8)
+; LA64-NEXT:    add.d $a0, $a0, $tp, %le_add_r(g_i8)
+; LA64-NEXT:    xvldrepl.b $xr0, $a0, %le_lo12_r(g_i8)
+; LA64-NEXT:    lu12i.w $a0, %le_hi20_r(g_i8x32)
+; LA64-NEXT:    add.d $a0, $a0, $tp, %le_add_r(g_i8x32)
+; LA64-NEXT:    xvst $xr0, $a0, %le_lo12_r(g_i8x32)
+; LA64-NEXT:    ret
+entry:
+  %0 = call ptr @llvm.threadlocal.address.p0(ptr @g_i8)
+  %1 = call <32 x i8> @llvm.loongarch.lasx.xvldrepl.b(ptr %0, i32 0)
+  %2 = call ptr @llvm.threadlocal.address.p0(ptr @g_i8x32)
+  store <32 x i8> %1, ptr %2, align 32
+  ret void
+}
+
+@g_rmw = dso_local thread_local(localexec) global i64 0
+
+define dso_local void @tlsle_rmw() nounwind {
+; LA32-LABEL: tlsle_rmw:
+; LA32:       # %bb.0: # %entry
+; LA32-NEXT:    lu12i.w $a0, %le_hi20_r(g_rmw)
+; LA32-NEXT:    add.w $a0, $a0, $tp, %le_add_r(g_rmw)
+; LA32-NEXT:    addi.w $a0, $a0, %le_lo12_r(g_rmw)
+; LA32-NEXT:    ld.w $a1, $a0, 0
+; LA32-NEXT:    ld.w $a2, $a0, 4
+; LA32-NEXT:    addi.w $a1, $a1, 1
+; LA32-NEXT:    sltui $a3, $a1, 1
+; LA32-NEXT:    add.w $a2, $a2, $a3
+; LA32-NEXT:    st.w $a1, $a0, 0
+; LA32-NEXT:    st.w $a2, $a0, 4
+; LA32-NEXT:    ret
+;
+; LA64-LABEL: tlsle_rmw:
+; LA64:       # %bb.0: # %entry
+; LA64-NEXT:    lu12i.w $a0, %le_hi20_r(g_rmw)
+; LA64-NEXT:    add.d $a0, $a0, $tp, %le_add_r(g_rmw)
+; LA64-NEXT:    ld.d $a1, $a0, %le_lo12_r(g_rmw)
+; LA64-NEXT:    addi.d $a1, $a1, 1
+; LA64-NEXT:    st.d $a1, $a0, %le_lo12_r(g_rmw)
+; LA64-NEXT:    ret
+entry:
+  %0 = call ptr @llvm.threadlocal.address.p0(ptr @g_rmw)
+  %1 = load i64, ptr %0
+  %2 = add i64 %1, 1
+  store i64 %2, ptr %0
+  ret void
+}
+
+@g_a32 = dso_local thread_local(localexec) global [2048 x i32] zeroinitializer, align 4
+
+define dso_local void @tlsle_store_a32() nounwind {
+; LA32-LABEL: tlsle_store_a32:
+; LA32:       # %bb.0: # %entry
+; LA32-NEXT:    lu12i.w $a0, %le_hi20_r(g_a32+4096)
+; LA32-NEXT:    add.w $a0, $a0, $tp, %le_add_r(g_a32+4096)
+; LA32-NEXT:    ori $a1, $zero, 1
+; LA32-NEXT:    st.w $a1, $a0, %le_lo12_r(g_a32+4096)
+; LA32-NEXT:    ret
+;
+; LA64-LABEL: tlsle_store_a32:
+; LA64:       # %bb.0: # %entry
+; LA64-NEXT:    lu12i.w $a0, %le_hi20_r(g_a32+4096)
+; LA64-NEXT:    add.d $a0, $a0, $tp, %le_add_r(g_a32+4096)
+; LA64-NEXT:    ori $a1, $zero, 1
+; LA64-NEXT:    st.w $a1, $a0, %le_lo12_r(g_a32+4096)
+; LA64-NEXT:    ret
+entry:
+  store i32 1, ptr getelementptr inbounds ([1 x i32], ptr @g_a32, i32 1024), align 4
+  ret void
+}
+
+define dso_local void @tlsle_store_a32_2() nounwind {
+; LA32-LABEL: tlsle_store_a32_2:
+; LA32:       # %bb.0: # %entry
+; LA32-NEXT:    lu12i.w $a0, %le_hi20_r(g_a32)
+; LA32-NEXT:    add.w $a0, $a0, $tp, %le_add_r(g_a32)
+; LA32-NEXT:    addi.w $a0, $a0, %le_lo12_r(g_a32)
+; LA32-NEXT:    lu12i.w $a1, 1
+; LA32-NEXT:    add.w $a2, $a0, $a1
+; LA32-NEXT:    ori $a3, $zero, 1
+; LA32-NEXT:    st.w $a3, $a2, 0
+; LA32-NEXT:    ori $a1, $a1, 8
+; LA32-NEXT:    add.w $a0, $a0, $a1
+; LA32-NEXT:    ori $a1, $zero, 2
+; LA32-NEXT:    st.w $a1, $a0, 0
+; LA32-NEXT:    ret
+;
+; LA64-LABEL: tlsle_store_a32_2:
+; LA64:       # %bb.0: # %entry
+; LA64-NEXT:    lu12i.w $a0, %le_hi20_r(g_a32)
+; LA64-NEXT:    add.d $a0, $a0, $tp, %le_add_r(g_a32)
+; LA64-NEXT:    addi.d $a0, $a0, %le_lo12_r(g_a32)
+; LA64-NEXT:    ori $a1, $zero, 1
+; LA64-NEXT:    stptr.w $a1, $a0, 4096
+; LA64-NEXT:    ori $a1, $zero, 2
+; LA64-NEXT:    stptr.w $a1, $a0, 4104
+; LA64-NEXT:    ret
+entry:
+  store i32 1, ptr getelementptr inbounds ([1 x i32], ptr @g_a32, i32 1024), align 4
+  store i32 2, ptr getelementptr inbounds ([1 x i32], ptr @g_a32, i32 1026), align 4
+  ret void
+}
+
+define dso_local void @tlsle_control_flow_with_mem_access() nounwind {
+; LA32-LABEL: tlsle_control_flow_with_mem_access:
+; LA32:       # %bb.0: # %entry
+; LA32-NEXT:    lu12i.w $a0, %le_hi20_r(g_a32+4)
+; LA32-NEXT:    add.w $a0, $a0, $tp, %le_add_r(g_a32+4)
+; LA32-NEXT:    ld.w $a1, $a0, %le_lo12_r(g_a32+4)
+; LA32-NEXT:    ori $a2, $zero, 1
+; LA32-NEXT:    blt $a1, $a2, .LBB25_2
+; LA32-NEXT:  # %bb.1: # %if.then
+; LA32-NEXT:    ori $a1, $zero, 10
+; LA32-NEXT:    st.w $a1, $a0, %le_lo12_r(g_a32+4)
+; LA32-NEXT:  .LBB25_2: # %if.end
+; LA32-NEXT:    ret
+;
+; LA64-LABEL: tlsle_control_flow_with_mem_access:
+; LA64:       # %bb.0: # %entry
+; LA64-NEXT:    lu12i.w $a0, %le_hi20_r(g_a32+4)
+; LA64-NEXT:    add.d $a0, $a0, $tp, %le_add_r(g_a32+4)
+; LA64-NEXT:    ld.w $a1, $a0, %le_lo12_r(g_a32+4)
+; LA64-NEXT:    ori $a2, $zero, 1
+; LA64-NEXT:    blt $a1, $a2, .LBB25_2
+; LA64-NEXT:  # %bb.1: # %if.then
+; LA64-NEXT:    ori $a1, $zero, 10
+; LA64-NEXT:    st.w $a1, $a0, %le_lo12_r(g_a32+4)
+; LA64-NEXT:  .LBB25_2: # %if.end
+; LA64-NEXT:    ret
+entry:
+  %0 = load i32, ptr getelementptr inbounds ([1 x i32], ptr @g_a32, i32 1), align 4
+  %cmp = icmp sgt i32 %0, 0
+  br i1 %cmp, label %if.then, label %if.end
+
+if.then:
+  store i32 10, ptr getelementptr inbounds ([1 x i32], ptr @g_a32, i32 1), align 4
+  br label %if.end
+
+if.end:
+  ret void
+}
+
+@g_a64 = dso_local thread_local(localexec) global [614750729487779976 x i64] zeroinitializer, align 8
+
+define dso_local ptr @tlsle_load_addr_offset_1() nounwind {
+; LA32-LABEL: tlsle_load_addr_offset_1:
+; LA32:       # %bb.0: # %entry
+; LA32-NEXT:    lu12i.w $a0, %le_hi20_r(g_a64+8)
+; LA32-NEXT:    add.w $a0, $a0, $tp, %le_add_r(g_a64+8)
+; LA32-NEXT:    addi.w $a0, $a0, %le_lo12_r(g_a64+8)
+; LA32-NEXT:    ret
+;
+; LA64-LABEL: tlsle_load_addr_offset_1:
+; LA64:       # %bb.0: # %entry
+; LA64-NEXT:    lu12i.w $a0, %le_hi20_r(g_a64+8)
+; LA64-NEXT:    add.d $a0, $a0, $tp, %le_add_r(g_a64+8)
+; LA64-NEXT:    addi.d $a0, $a0, %le_lo12_r(g_a64+8)
+; LA64-NEXT:    ret
+entry:
+  ret ptr getelementptr inbounds ([1 x i64], ptr @g_a64, i64 1)
+}
+
+define dso_local ptr @tlsle_load_addr_offset_257() nounwind {
+; LA32-LABEL: tlsle_load_addr_offset_257:
+; LA32:       # %bb.0: # %entry
+; LA32-NEXT:    lu12i.w $a0, %le_hi20_r(g_a64+2056)
+; LA32-NEXT:    add.w $a0, $a0, $tp, %le_add_r(g_a64+2056)
+; LA32-NEXT:    addi.w $a0, $a0, %le_lo12_r(g_a64+2056)
+; LA32-NEXT:    ret
+;
+; LA64-LABEL: tlsle_load_addr_offset_257:
+; LA64:       # %bb.0: # %entry
+; LA64-NEXT:    lu12i.w $a0, %le_hi20_r(g_a64+2056)
+; LA64-NEXT:    add.d $a0, $a0, $tp, %le_add_r(g_a64+2056)
+; LA64-NEXT:    addi.d $a0, $a0, %le_lo12_r(g_a64+2056)
+; LA64-NEXT:    ret
+entry:
+  ret ptr getelementptr inbounds ([1 x i64], ptr @g_a64, i64 257)
+}
+
+define dso_local ptr @tlsle_load_addr_offset_1048576() nounwind {
+; LA32-LABEL: tlsle_load_addr_offset_1048576:
+; LA32:       # %bb.0: # %entry
+; LA32-NEXT:    lu12i.w $a0, %le_hi20_r(g_a64+8388608)
+; LA32-NEXT:    add.w $a0, $a0, $tp, %le_add_r(g_a64+8388608)
+; LA32-NEXT:    addi.w $a0, $a0, %le_lo12_r(g_a64+8388608)
+; LA32-NEXT:    ret
+;
+; LA64-LABEL: tlsle_load_addr_offset_1048576:
+; LA64:       # %bb.0: # %entry
+; LA64-NEXT:    lu12i.w $a0, %le_hi20_r(g_a64+8388608)
+; LA64-NEXT:    add.d $a0, $a0, $tp, %le_add_r(g_a64+8388608)
+; LA64-NEXT:    addi.d $a0, $a0, %le_lo12_r(g_a64+8388608)
+; LA64-NEXT:    ret
+entry:
+  ret ptr getelementptr inbounds ([1 x i64], ptr @g_a64, i64 1048576)
+}
+
+define dso_local ptr @tlsle_load_addr_offset_1048577() nounwind {
+; LA32-LABEL: tlsle_load_addr_offset_1048577:
+; LA32:       # %bb.0: # %entry
+; LA32-NEXT:    lu12i.w $a0, %le_hi20_r(g_a64+8388616)
+; LA32-NEXT:    add.w $a0, $a0, $tp, %le_add_r(g_a64+8388616)
+; LA32-NEXT:    addi.w $a0, $a0, %le_lo12_r(g_a64+8388616)
+; LA32-NEXT:    ret
+;
+; LA64-LABEL: tlsle_load_addr_offset_1048577:
+; LA64:       # %bb.0: # %entry
+; LA64-NEXT:    lu12i.w $a0, %le_hi20_r(g_a64+8388616)
+; LA64-NEXT:    add.d $a0, $a0, $tp, %le_add_r(g_a64+8388616)
+; LA64-NEXT:    addi.d $a0, $a0, %le_lo12_r(g_a64+8388616)
+; LA64-NEXT:    ret
+entry:
+  ret ptr getelementptr inbounds ([1 x i64], ptr @g_a64, i64 1048577)
+}
+
+define dso_local ptr @tlsle_load_addr_offset_268432896() nounwind {
+; LA32-LABEL: tlsle_load_addr_offset_268432896:
+; LA32:       # %bb.0: # %entry
+; LA32-NEXT:    lu12i.w $a0, %le_hi20_r(g_a64+2147463168)
+; LA32-NEXT:    add.w $a0, $a0, $tp, %le_add_r(g_a64+2147463168)
+; LA32-NEXT:    addi.w $a0, $a0, %le_lo12_r(g_a64+2147463168)
+; LA32-NEXT:    ret
+;
+; LA64-LABEL: tlsle_load_addr_offset_268432896:
+; LA64:       # %bb.0: # %entry
+; LA64-NEXT:    lu12i.w $a0, %le_hi20_r(g_a64+2147463168)
+; LA64-NEXT:    add.d $a0, $a0, $tp, %le_add_r(g_a64+2147463168)
+; LA64-NEXT:    addi.d $a0, $a0, %le_lo12_r(g_a64+2147463168)
+; LA64-NEXT:    ret
+entry:
+  ret ptr getelementptr inbounds ([1 x i64], ptr @g_a64, i64 268432896)
+}
+
+define dso_local ptr @tlsle_load_addr_offset_268432897() nounwind {
+; LA32-LABEL: tlsle_load_addr_offset_268432897:
+; LA32:       # %bb.0: # %entry
+; LA32-NEXT:    lu12i.w $a0, %le_hi20_r(g_a64+2147463176)
+; LA32-NEXT:    add.w $a0, $a0, $tp, %le_add_r(g_a64+2147463176)
+; LA32-NEXT:    addi.w $a0, $a0, %le_lo12_r(g_a64+2147463176)
+; LA32-NEXT:    ret
+;
+; LA64-LABEL: tlsle_load_addr_offset_268432897:
+; LA64:       # %bb.0: # %entry
+; LA64-NEXT:    lu12i.w $a0, %le_hi20_r(g_a64+2147463176)
+; LA64-NEXT:    add.d $a0, $a0, $tp, %le_add_r(g_a64+2147463176)
+; LA64-NEXT:    addi.d $a0, $a0, %le_lo12_r(g_a64+2147463176)
+; LA64-NEXT:    ret
+entry:
+  ret ptr getelementptr inbounds ([1 x i64], ptr @g_a64, i64 268432897)
+}
+
+define dso_local ptr @tlsle_load_addr_offset_281474439839744() nounwind {
+; LA32-LABEL: tlsle_load_addr_offset_281474439839744:
+; LA32:       # %bb.0: # %entry
+; LA32-NEXT:    lu12i.w $a0, %le_hi20_r(g_a64)
+; LA32-NEXT:    add.w $a0, $a0, $tp, %le_add_r(g_a64)
+; LA32-NEXT:    addi.w $a0, $a0, %le_lo12_r(g_a64)
+; LA32-NEXT:    ret
+;
+; LA64-LABEL: tlsle_load_addr_offset_281474439839744:
+; LA64:       # %bb.0: # %entry
+; LA64-NEXT:    lu12i.w $a0, %le_hi20_r(g_a64)
+; LA64-NEXT:    add.d $a0, $a0, $tp, %le_add_r(g_a64)
+; LA64-NEXT:    addi.d $a0, $a0, %le_lo12_r(g_a64)
+; LA64-NEXT:    ori $a1, $zero, 0
+; LA64-NEXT:    lu32i.d $a1, 524287
+; LA64-NEXT:    add.d $a0, $a0, $a1
+; LA64-NEXT:    ret
+entry:
+  ret ptr getelementptr inbounds ([1 x i64], ptr @g_a64, i64 281474439839744)
+}
+
+define dso_local ptr @tlsle_load_addr_offset_248792680471040() nounwind {
+; LA32-LABEL: tlsle_load_addr_offset_248792680471040:
+; LA32:       # %bb.0: # %entry
+; LA32-NEXT:    lu12i.w $a0, %le_hi20_r(g_a64+2059194368)
+; LA32-NEXT:    add.w $a0, $a0, $tp, %le_add_r(g_a64+2059194368)
+; LA32-NEXT:    addi.w $a0, $a0, %le_lo12_r(g_a64+2059194368)
+; LA32-NEXT:    ret
+;
+; LA64-LABEL: tlsle_load_addr_offset_248792680471040:
+; LA64:       # %bb.0: # %entry
+; LA64-NEXT:    lu12i.w $a0, %le_hi20_r(g_a64)
+; LA64-NEXT:    add.d $a0, $a0, $tp, %le_add_r(g_a64)
+; LA64-NEXT:    addi.d $a0, $a0, %le_lo12_r(g_a64)
+; LA64-NEXT:    lu12i.w $a1, 502733
+; LA64-NEXT:    lu32i.d $a1, 463412
+; LA64-NEXT:    add.d $a0, $a0, $a1
+; LA64-NEXT:    ret
+entry:
+  ret ptr getelementptr inbounds ([1 x i64], ptr @g_a64, i64 248792680471040)
+}
+
+define dso_local ptr @tlsle_load_addr_offset_9380351707272() nounwind {
+; LA32-LABEL: tlsle_load_addr_offset_9380351707272:
+; LA32:       # %bb.0: # %entry
+; LA32-NEXT:    lu12i.w $a0, %le_hi20_r(g_a64+1145062464)
+; LA32-NEXT:    add.w $a0, $a0, $tp, %le_add_r(g_a64+1145062464)
+; LA32-NEXT:    addi.w $a0, $a0, %le_lo12_r(g_a64+1145062464)
+; LA32-NEXT:    ret
+;
+; LA64-LABEL: tlsle_load_addr_offset_9380351707272:
+; LA64:       # %bb.0: # %entry
+; LA64-NEXT:    lu12i.w $a0, %le_hi20_r(g_a64)
+; LA64-NEXT:    add.d $a0, $a0, $tp, %le_add_r(g_a64)
+; LA64-NEXT:    addi.d $a0, $a0, %le_lo12_r(g_a64)
+; LA64-NEXT:    lu12i.w $a1, 279556
+; LA64-NEXT:    ori $a1, $a1, 1088
+; LA64-NEXT:    lu32i.d $a1, 17472
+; LA64-NEXT:    add.d $a0, $a0, $a1
+; LA64-NEXT:    ret
+entry:
+  ret ptr getelementptr inbounds ([1 x i64], ptr @g_a64, i64 9380351707272)
+}
+
+define dso_local ptr @tlsle_load_addr_offset_562949953421312() nounwind {
+; LA32-LABEL: tlsle_load_addr_offset_562949953421312:
+; LA32:       # %bb.0: # %entry
+; LA32-NEXT:    lu12i.w $a0, %le_hi20_r(g_a64)
+; LA32-NEXT:    add.w $a0, $a0, $tp, %le_add_r(g_a64)
+; LA32-NEXT:    addi.w $a0, $a0, %le_lo12_r(g_a64)
+; LA32-NEXT:    ret
+;
+; LA64-LABEL: tlsle_load_addr_offset_562949953421312:
+; LA64:       # %bb.0: # %entry
+; LA64-NEXT:    lu12i.w $a0, %le_hi20_r(g_a64)
+; LA64-NEXT:    add.d $a0, $a0, $tp, %le_add_r(g_a64)
+; LA64-NEXT:    addi.d $a0, $a0, %le_lo12_r(g_a64)
+; LA64-NEXT:    lu52i.d $a1, $zero, 1
+; LA64-NEXT:    add.d $a0, $a0, $a1
+; LA64-NEXT:    ret
+entry:
+  ret ptr getelementptr inbounds ([1 x i64], ptr @g_a64, i64 562949953421312)
+}
+
+define dso_local ptr @tlsle_load_addr_offset_614749556925924693() nounwind {
+; LA32-LABEL: tlsle_load_addr_offset_614749556925924693:
+; LA32:       # %bb.0: # %entry
+; LA32-NEXT:    lu12i.w $a0, %le_hi20_r(g_a64+858794664)
+; LA32-NEXT:    add.w $a0, $a0, $tp, %le_add_r(g_a64+858794664)
+; LA32-NEXT:    addi.w $a0, $a0, %le_lo12_r(g_a64+858794664)
+; LA32-NEXT:    ret
+;
+; LA64-LABEL: tlsle_load_addr_offset_614749556925924693:
+; LA64:       # %bb.0: # %entry
+; LA64-NEXT:    lu12i.w $a0, %le_hi20_r(g_a64)
+; LA64-NEXT:    add.d $a0, $a0, $tp, %le_add_r(g_a64)
+; LA64-NEXT:    addi.d $a0, $a0, %le_lo12_r(g_a64)
+; LA64-NEXT:    lu12i.w $a1, 209666
+; LA64-NEXT:    ori $a1, $a1, 2728
+; LA64-NEXT:    lu32i.d $a1, 15288
+; LA64-NEXT:    lu52i.d $a1, $a1, 1092
+; LA64-NEXT:    add.d $a0, $a0, $a1
+; LA64-NEXT:    ret
+entry:
+  ret ptr getelementptr inbounds ([1 x i64], ptr @g_a64, i64 614749556925924693)
+}
+
+declare nonnull ptr @llvm.threadlocal.address.p0(ptr nonnull)
diff --git a/llvm/test/CodeGen/LoongArch/merge-base-offset.ll b/llvm/test/CodeGen/LoongArch/merge-base-offset.ll
index 9df5532d51179..2af206699d4ad 100644
--- a/llvm/test/CodeGen/LoongArch/merge-base-offset.ll
+++ b/llvm/test/CodeGen/LoongArch/merge-base-offset.ll
@@ -257,8 +257,8 @@ entry:
 
 @g_i64 = dso_local global i64 0
 
-define dso_local i64 @load_64() nounwind {
-; LA32-LABEL: load_64:
+define dso_local i64 @load_i64() nounwind {
+; LA32-LABEL: load_i64:
 ; LA32:       # %bb.0: # %entry
 ; LA32-NEXT:    pcalau12i $a0, %pc_hi20(g_i64)
 ; LA32-NEXT:    addi.w $a1, $a0, %pc_lo12(g_i64)
@@ -266,13 +266,13 @@ define dso_local i64 @load_64() nounwind {
 ; LA32-NEXT:    ld.w $a1, $a1, 4
 ; LA32-NEXT:    ret
 ;
-; LA64-LABEL: load_64:
+; LA64-LABEL: load_i64:
 ; LA64:       # %bb.0: # %entry
 ; LA64-NEXT:    pcalau12i $a0, %pc_hi20(g_i64)
 ; LA64-NEXT:    ld.d $a0, $a0, %pc_lo12(g_i64)
 ; LA64-NEXT:    ret
 ;
-; LA64-LARGE-LABEL: load_64:
+; LA64-LARGE-LABEL: load_i64:
 ; LA64-LARGE:       # %bb.0: # %entry
 ; LA64-LARGE-NEXT:    pcalau12i $a0, %pc_hi20(g_i64)
 ; LA64-LARGE-NEXT:    addi.d $a1, $zero, %pc_lo12(g_i64)
@@ -580,36 +580,36 @@ entry:
 define dso_local void @copy_i32x8() nounwind {
 ; LA32-LABEL: copy_i32x8:
 ; LA32:       # %bb.0: # %entry
-; LA32-NEXT:    pcalau12i $a0, %pc_hi20(g_i32x4_src)
-; LA32-NEXT:    xvld $xr0, $a0, %pc_lo12(g_i32x4_src)
-; LA32-NEXT:    pcalau12i $a0, %pc_hi20(g_i32x4_dst)
-; LA32-NEXT:    xvst $xr0, $a0, %pc_lo12(g_i32x4_dst)
+; LA32-NEXT:    pcalau12i $a0, %pc_hi20(g_i32x8_src)
+; LA32-NEXT:    xvld $xr0, $a0, %pc_lo12(g_i32x8_src)
+; LA32-NEXT:    pcalau12i $a0, %pc_hi20(g_i32x8_dst)
+; LA32-NEXT:    xvst $xr0, $a0, %pc_lo12(g_i32x8_dst)
 ; LA32-NEXT:    ret
 ;
 ; LA64-LABEL: copy_i32x8:
 ; LA64:       # %bb.0: # %entry
-; LA64-NEXT:    pcalau12i $a0, %pc_hi20(g_i32x4_src)
-; LA64-NEXT:    xvld $xr0, $a0, %pc_lo12(g_i32x4_src)
-; LA64-NEXT:    pcalau12i $a0, %pc_hi20(g_i32x4_dst)
-; LA64-NEXT:    xvst $xr0, $a0, %pc_lo12(g_i32x4_dst)
+; LA64-NEXT:    pcalau12i $a0, %pc_hi20(g_i32x8_src)
+; LA64-NEXT:    xvld $xr0, $a0, %pc_lo12(g_i32x8_src)
+; LA64-NEXT:    pcalau12i $a0, %pc_hi20(g_i32x8_dst)
+; LA64-NEXT:    xvst $xr0, $a0, %pc_lo12(g_i32x8_dst)
 ; LA64-NEXT:    ret
 ;
 ; LA64-LARGE-LABEL: copy_i32x8:
 ; LA64-LARGE:       # %bb.0: # %entry
-; LA64-LARGE-NEXT:    pcalau12i $a0, %pc_hi20(g_i32x4_src)
-; LA64-LARGE-NEXT:    addi.d $a1, $zero, %pc_lo12(g_i32x4_src)
-; LA64-LARGE-NEXT:    lu32i.d $a1, %pc64_lo20(g_i32x4_src)
-; LA64-LARGE-NEXT:    lu52i.d $a1, $a1, %pc64_hi12(g_i32x4_src)
+; LA64-LARGE-NEXT:    pcalau12i $a0, %pc_hi20(g_i32x8_src)
+; LA64-LARGE-NEXT:    addi.d $a1, $zero, %pc_lo12(g_i32x8_src)
+; LA64-LARGE-NEXT:    lu32i.d $a1, %pc64_lo20(g_i32x8_src)
+; LA64-LARGE-NEXT:    lu52i.d $a1, $a1, %pc64_hi12(g_i32x8_src)
 ; LA64-LARGE-NEXT:    xvldx $xr0, $a1, $a0
-; LA64-LARGE-NEXT:    pcalau12i $a0, %pc_hi20(g_i32x4_dst)
-; LA64-LARGE-NEXT:    addi.d $a1, $zero, %pc_lo12(g_i32x4_dst)
-; LA64-LARGE-NEXT:    lu32i.d $a1, %pc64_lo20(g_i32x4_dst)
-; LA64-LARGE-NEXT:    lu52i.d $a1, $a1, %pc64_hi12(g_i32x4_dst)
+; LA64-LARGE-NEXT:    pcalau12i $a0, %pc_hi20(g_i32x8_dst)
+; LA64-LARGE-NEXT:    addi.d $a1, $zero, %pc_lo12(g_i32x8_dst)
+; LA64-LARGE-NEXT:    lu32i.d $a1, %pc64_lo20(g_i32x8_dst)
+; LA64-LARGE-NEXT:    lu52i.d $a1, $a1, %pc64_hi12(g_i32x8_dst)
 ; LA64-LARGE-NEXT:    xvstx $xr0, $a1, $a0
 ; LA64-LARGE-NEXT:    ret
 entry:
-  %0 = load <8 x i32>, ptr @g_i32x4_src, align 32
-  store <8 x i32> %0, ptr @g_i32x4_dst, align 32
+  %0 = load <8 x i32>, ptr @g_i32x8_src, align 32
+  store <8 x i32> %0, ptr @g_i32x8_dst, align 32
   ret void
 }
 
diff --git a/llvm/test/CodeGen/M68k/PR57660.ll b/llvm/test/CodeGen/M68k/PR57660.ll
index bad949b08cafa..359f0c2496356 100644
--- a/llvm/test/CodeGen/M68k/PR57660.ll
+++ b/llvm/test/CodeGen/M68k/PR57660.ll
@@ -8,10 +8,10 @@ define dso_local void @foo1() {
 ; CHECK-NEXT:    suba.l #2, %sp
 ; CHECK-NEXT:    .cfi_def_cfa_offset -6
 ; CHECK-NEXT:    moveq #0, %d0
-; CHECK-NEXT:    move.b %d0, (0,%sp) ; 1-byte Folded Spill
+; CHECK-NEXT:    movem.w %d0, (0,%sp)
 ; CHECK-NEXT:  .LBB0_1: ; %do.body
 ; CHECK-NEXT:    ; =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    move.b (0,%sp), %d0 ; 1-byte Folded Reload
+; CHECK-NEXT:    movem.w (0,%sp), %d0
 ; CHECK-NEXT:    cmpi.b #0, %d0
 ; CHECK-NEXT:    bne .LBB0_1
 ; CHECK-NEXT:  ; %bb.2: ; %do.end
@@ -39,24 +39,24 @@ define i32 @foo2(ptr noundef %0) {
 ; CHECK-NEXT:    .cfi_def_cfa_offset -8
 ; CHECK-NEXT:    move.l (8,%sp), %a0
 ; CHECK-NEXT:    move.b (%a0), %d0
-; CHECK-NEXT:    move.b %d0, (0,%sp) ; 1-byte Folded Spill
+; CHECK-NEXT:    movem.w %d0, (0,%sp)
 ; CHECK-NEXT:    and.b #1, %d0
-; CHECK-NEXT:    move.b %d0, (2,%sp) ; 1-byte Folded Spill
+; CHECK-NEXT:    movem.w %d0, (2,%sp)
 ; CHECK-NEXT:    sub.b #1, %d0
 ; CHECK-NEXT:    bgt .LBB1_2
 ; CHECK-NEXT:  ; %bb.1: ; %if
-; CHECK-NEXT:    move.b (2,%sp), %d0 ; 1-byte Folded Reload
-; CHECK-NEXT:    move.b (0,%sp), %d1 ; 1-byte Folded Reload
+; CHECK-NEXT:    movem.w (2,%sp), %d0
+; CHECK-NEXT:    movem.w (0,%sp), %d1
 ; CHECK-NEXT:    add.b %d1, %d0
 ; CHECK-NEXT:    bra .LBB1_3
 ; CHECK-NEXT:  .LBB1_2: ; %else
-; CHECK-NEXT:    move.b (2,%sp), %d1 ; 1-byte Folded Reload
-; CHECK-NEXT:    move.b (0,%sp), %d0 ; 1-byte Folded Reload
+; CHECK-NEXT:    movem.w (2,%sp), %d1
+; CHECK-NEXT:    movem.w (0,%sp), %d0
 ; CHECK-NEXT:    sub.b %d1, %d0
-; CHECK-NEXT:    move.b %d0, (0,%sp) ; 1-byte Folded Spill
+; CHECK-NEXT:    movem.w %d0, (0,%sp)
 ; CHECK-NEXT:  .LBB1_3: ; %cont
-; CHECK-NEXT:    move.b %d0, (2,%sp) ; 1-byte Folded Spill
-; CHECK-NEXT:    move.b (2,%sp), %d0 ; 1-byte Folded Reload
+; CHECK-NEXT:    movem.w %d0, (2,%sp)
+; CHECK-NEXT:    movem.w (2,%sp), %d0
 ; CHECK-NEXT:    ext.w %d0
 ; CHECK-NEXT:    ext.l %d0
 ; CHECK-NEXT:    adda.l #4, %sp
diff --git a/llvm/test/CodeGen/M68k/register-spills.ll b/llvm/test/CodeGen/M68k/register-spills.ll
new file mode 100644
index 0000000000000..9104a59f5d6b3
--- /dev/null
+++ b/llvm/test/CodeGen/M68k/register-spills.ll
@@ -0,0 +1,464 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=m68k -O0 %s -o - | FileCheck %s
+
+declare zeroext i1 @get1()
+declare i8 @get8()
+declare i16 @get16()
+declare i32 @get32()
+
+define void @test_edge_detection_conditional_branch() {
+; CHECK-LABEL: test_edge_detection_conditional_branch:
+; CHECK:         .cfi_startproc
+; CHECK-NEXT:  ; %bb.0: ; %start
+; CHECK-NEXT:    suba.l #12, %sp
+; CHECK-NEXT:    .cfi_def_cfa_offset -16
+; CHECK-NEXT:    movem.l %d2, (8,%sp) ; 8-byte Folded Spill
+; CHECK-NEXT:    bra .LBB0_1
+; CHECK-NEXT:  .LBB0_1: ; %condition_check
+; CHECK-NEXT:    ; =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    jsr get1
+; CHECK-NEXT:    move.b (7,%sp), %d2
+; CHECK-NEXT:    and.b #1, %d2
+; CHECK-NEXT:    move.b %d0, %d1
+; CHECK-NEXT:    sub.b %d2, %d1
+; CHECK-NEXT:    movem.w %d0, (4,%sp)
+; CHECK-NEXT:    bne .LBB0_1
+; CHECK-NEXT:    bra .LBB0_2
+; CHECK-NEXT:  .LBB0_2: ; %do_something
+; CHECK-NEXT:    movem.w (4,%sp), %d0
+; CHECK-NEXT:    move.b %d0, (7,%sp)
+; CHECK-NEXT:    movem.l (8,%sp), %d2 ; 8-byte Folded Reload
+; CHECK-NEXT:    adda.l #12, %sp
+; CHECK-NEXT:    rts
+start:
+  %prev_state = alloca [1 x i8], align 1
+  br label %condition_check
+
+condition_check:
+  %state = call zeroext i1 @get1()
+  %local_prev_state = load i8, ptr %prev_state, align 1
+  %local_prev_state_trunc = trunc i8 %local_prev_state to i1
+  %result = icmp ne i1 %state, %local_prev_state_trunc
+  br i1 %result, label %condition_check, label %do_something
+
+do_something:
+  %state_ext = zext i1 %state to i8
+  store i8 %state_ext, ptr %prev_state, align 1
+  ret void
+}
+
+define void @test_force_spill_8() {
+; CHECK-LABEL: test_force_spill_8:
+; CHECK:         .cfi_startproc
+; CHECK-NEXT:  ; %bb.0: ; %start
+; CHECK-NEXT:    suba.l #108, %sp
+; CHECK-NEXT:    .cfi_def_cfa_offset -112
+; CHECK-NEXT:    movem.l %d2-%d7, (84,%sp) ; 28-byte Folded Spill
+; CHECK-NEXT:    jsr get8
+; CHECK-NEXT:    movem.w %d0, (82,%sp)
+; CHECK-NEXT:    jsr get8
+; CHECK-NEXT:    movem.w %d0, (66,%sp)
+; CHECK-NEXT:    jsr get8
+; CHECK-NEXT:    move.b %d0, %d2
+; CHECK-NEXT:    jsr get8
+; CHECK-NEXT:    move.b %d0, %d3
+; CHECK-NEXT:    jsr get8
+; CHECK-NEXT:    move.b %d0, %d4
+; CHECK-NEXT:    jsr get8
+; CHECK-NEXT:    move.b %d0, %d5
+; CHECK-NEXT:    jsr get8
+; CHECK-NEXT:    move.b %d0, %d6
+; CHECK-NEXT:    jsr get8
+; CHECK-NEXT:    move.b %d0, %d7
+; CHECK-NEXT:    jsr get8
+; CHECK-NEXT:    movem.w %d0, (80,%sp)
+; CHECK-NEXT:    jsr get8
+; CHECK-NEXT:    movem.w %d0, (78,%sp)
+; CHECK-NEXT:    jsr get8
+; CHECK-NEXT:    movem.w %d0, (76,%sp)
+; CHECK-NEXT:    jsr get8
+; CHECK-NEXT:    movem.w %d0, (74,%sp)
+; CHECK-NEXT:    jsr get8
+; CHECK-NEXT:    movem.w %d0, (72,%sp)
+; CHECK-NEXT:    jsr get8
+; CHECK-NEXT:    movem.w %d0, (70,%sp)
+; CHECK-NEXT:    jsr get8
+; CHECK-NEXT:    movem.w %d0, (68,%sp)
+; CHECK-NEXT:    jsr get8
+; CHECK-NEXT:    movem.w (66,%sp), %d1
+; CHECK-NEXT:    and.l #255, %d0
+; CHECK-NEXT:    move.l %sp, %a0
+; CHECK-NEXT:    move.l %d0, (60,%a0)
+; CHECK-NEXT:    movem.w (68,%sp), %d0
+; CHECK-NEXT:    and.l #255, %d0
+; CHECK-NEXT:    move.l %d0, (56,%a0)
+; CHECK-NEXT:    movem.w (70,%sp), %d0
+; CHECK-NEXT:    and.l #255, %d0
+; CHECK-NEXT:    move.l %d0, (52,%a0)
+; CHECK-NEXT:    movem.w (72,%sp), %d0
+; CHECK-NEXT:    and.l #255, %d0
+; CHECK-NEXT:    move.l %d0, (48,%a0)
+; CHECK-NEXT:    movem.w (74,%sp), %d0
+; CHECK-NEXT:    and.l #255, %d0
+; CHECK-NEXT:    move.l %d0, (44,%a0)
+; CHECK-NEXT:    movem.w (76,%sp), %d0
+; CHECK-NEXT:    and.l #255, %d0
+; CHECK-NEXT:    move.l %d0, (40,%a0)
+; CHECK-NEXT:    movem.w (78,%sp), %d0
+; CHECK-NEXT:    and.l #255, %d0
+; CHECK-NEXT:    move.l %d0, (36,%a0)
+; CHECK-NEXT:    movem.w (80,%sp), %d0
+; CHECK-NEXT:    and.l #255, %d0
+; CHECK-NEXT:    move.l %d0, (32,%a0)
+; CHECK-NEXT:    movem.w (82,%sp), %d0
+; CHECK-NEXT:    and.l #255, %d7
+; CHECK-NEXT:    move.l %d7, (28,%a0)
+; CHECK-NEXT:    and.l #255, %d6
+; CHECK-NEXT:    move.l %d6, (24,%a0)
+; CHECK-NEXT:    and.l #255, %d5
+; CHECK-NEXT:    move.l %d5, (20,%a0)
+; CHECK-NEXT:    and.l #255, %d4
+; CHECK-NEXT:    move.l %d4, (16,%a0)
+; CHECK-NEXT:    and.l #255, %d3
+; CHECK-NEXT:    move.l %d3, (12,%a0)
+; CHECK-NEXT:    and.l #255, %d2
+; CHECK-NEXT:    move.l %d2, (8,%a0)
+; CHECK-NEXT:    and.l #255, %d1
+; CHECK-NEXT:    move.l %d1, (4,%a0)
+; CHECK-NEXT:    and.l #255, %d0
+; CHECK-NEXT:    move.l %d0, (%a0)
+; CHECK-NEXT:    jsr test_force_spill_8_consumer
+; CHECK-NEXT:    movem.l (84,%sp), %d2-%d7 ; 28-byte Folded Reload
+; CHECK-NEXT:    adda.l #108, %sp
+; CHECK-NEXT:    rts
+  start:
+    %r0 = call i8 @get8()
+    %r1 = call i8 @get8()
+    %r2 = call i8 @get8()
+    %r3 = call i8 @get8()
+    %r4 = call i8 @get8()
+    %r5 = call i8 @get8()
+    %r6 = call i8 @get8()
+    %r7 = call i8 @get8()
+    %r8 = call i8 @get8()
+    %r9 = call i8 @get8()
+    %ra = call i8 @get8()
+    %rb = call i8 @get8()
+    %rc = call i8 @get8()
+    %rd = call i8 @get8()
+    %re = call i8 @get8()
+    %rf = call i8 @get8()
+    call void @test_force_spill_8_consumer(i8 %r0, i8 %r1, i8 %r2, i8 %r3, i8 %r4, i8 %r5, i8 %r6, i8 %r7, i8 %r8, i8 %r9, i8 %ra, i8 %rb, i8 %rc, i8 %rd, i8 %re, i8 %rf)
+    ret void
+}
+
+declare void @test_force_spill_8_consumer(i8, i8, i8, i8, i8, i8, i8, i8, i8, i8, i8, i8, i8, i8, i8, i8)
+
+define void @test_force_spill_16() {
+; CHECK-LABEL: test_force_spill_16:
+; CHECK:         .cfi_startproc
+; CHECK-NEXT:  ; %bb.0: ; %start
+; CHECK-NEXT:    suba.l #116, %sp
+; CHECK-NEXT:    .cfi_def_cfa_offset -120
+; CHECK-NEXT:    movem.l %d2-%d7/%a2-%a6, (72,%sp) ; 48-byte Folded Spill
+; CHECK-NEXT:    jsr get16
+; CHECK-NEXT:    movem.w %d0, (70,%sp)
+; CHECK-NEXT:    jsr get16
+; CHECK-NEXT:    movem.w %d0, (66,%sp)
+; CHECK-NEXT:    jsr get16
+; CHECK-NEXT:    movem.w %d0, (64,%sp)
+; CHECK-NEXT:    jsr get16
+; CHECK-NEXT:    move.w %d0, %d2
+; CHECK-NEXT:    jsr get16
+; CHECK-NEXT:    move.w %d0, %d3
+; CHECK-NEXT:    jsr get16
+; CHECK-NEXT:    move.w %d0, %d4
+; CHECK-NEXT:    jsr get16
+; CHECK-NEXT:    move.w %d0, %d5
+; CHECK-NEXT:    jsr get16
+; CHECK-NEXT:    move.w %d0, %d6
+; CHECK-NEXT:    jsr get16
+; CHECK-NEXT:    move.w %d0, %d7
+; CHECK-NEXT:    jsr get16
+; CHECK-NEXT:    move.w %d0, %a2
+; CHECK-NEXT:    jsr get16
+; CHECK-NEXT:    move.w %d0, %a3
+; CHECK-NEXT:    jsr get16
+; CHECK-NEXT:    move.w %d0, %a4
+; CHECK-NEXT:    jsr get16
+; CHECK-NEXT:    move.w %d0, %a5
+; CHECK-NEXT:    jsr get16
+; CHECK-NEXT:    move.w %d0, %a6
+; CHECK-NEXT:    jsr get16
+; CHECK-NEXT:    movem.w %d0, (68,%sp)
+; CHECK-NEXT:    jsr get16
+; CHECK-NEXT:    movem.w (64,%sp), %a1
+; CHECK-NEXT:    movem.w (66,%sp), %d1
+; CHECK-NEXT:    and.l #65535, %d0
+; CHECK-NEXT:    move.l %sp, %a0
+; CHECK-NEXT:    move.l %d0, (60,%a0)
+; CHECK-NEXT:    movem.w (68,%sp), %d0
+; CHECK-NEXT:    and.l #65535, %d0
+; CHECK-NEXT:    move.l %d0, (56,%a0)
+; CHECK-NEXT:    movem.w (70,%sp), %d0
+; CHECK-NEXT:    and.l #65535, %a6
+; CHECK-NEXT:    move.l %a6, (52,%a0)
+; CHECK-NEXT:    and.l #65535, %a5
+; CHECK-NEXT:    move.l %a5, (48,%a0)
+; CHECK-NEXT:    and.l #65535, %a4
+; CHECK-NEXT:    move.l %a4, (44,%a0)
+; CHECK-NEXT:    and.l #65535, %a3
+; CHECK-NEXT:    move.l %a3, (40,%a0)
+; CHECK-NEXT:    and.l #65535, %a2
+; CHECK-NEXT:    move.l %a2, (36,%a0)
+; CHECK-NEXT:    and.l #65535, %d7
+; CHECK-NEXT:    move.l %d7, (32,%a0)
+; CHECK-NEXT:    and.l #65535, %d6
+; CHECK-NEXT:    move.l %d6, (28,%a0)
+; CHECK-NEXT:    and.l #65535, %d5
+; CHECK-NEXT:    move.l %d5, (24,%a0)
+; CHECK-NEXT:    and.l #65535, %d4
+; CHECK-NEXT:    move.l %d4, (20,%a0)
+; CHECK-NEXT:    and.l #65535, %d3
+; CHECK-NEXT:    move.l %d3, (16,%a0)
+; CHECK-NEXT:    and.l #65535, %d2
+; CHECK-NEXT:    move.l %d2, (12,%a0)
+; CHECK-NEXT:    and.l #65535, %a1
+; CHECK-NEXT:    move.l %a1, (8,%a0)
+; CHECK-NEXT:    and.l #65535, %d1
+; CHECK-NEXT:    move.l %d1, (4,%a0)
+; CHECK-NEXT:    and.l #65535, %d0
+; CHECK-NEXT:    move.l %d0, (%a0)
+; CHECK-NEXT:    jsr test_force_spill_16_consumer
+; CHECK-NEXT:    movem.l (72,%sp), %d2-%d7/%a2-%a6 ; 48-byte Folded Reload
+; CHECK-NEXT:    adda.l #116, %sp
+; CHECK-NEXT:    rts
+  start:
+    %r0 = call i16 @get16()
+    %r1 = call i16 @get16()
+    %r2 = call i16 @get16()
+    %r3 = call i16 @get16()
+    %r4 = call i16 @get16()
+    %r5 = call i16 @get16()
+    %r6 = call i16 @get16()
+    %r7 = call i16 @get16()
+    %r8 = call i16 @get16()
+    %r9 = call i16 @get16()
+    %ra = call i16 @get16()
+    %rb = call i16 @get16()
+    %rc = call i16 @get16()
+    %rd = call i16 @get16()
+    %re = call i16 @get16()
+    %rf = call i16 @get16()
+    call void @test_force_spill_16_consumer(i16 %r0, i16 %r1, i16 %r2, i16 %r3, i16 %r4, i16 %r5, i16 %r6, i16 %r7, i16 %r8, i16 %r9, i16 %ra, i16 %rb, i16 %rc, i16 %rd, i16 %re, i16 %rf)
+    ret void
+}
+
+declare void @test_force_spill_16_consumer(i16, i16, i16, i16, i16, i16, i16, i16, i16, i16, i16, i16, i16, i16, i16, i16)
+
+define void @test_force_spill_32() {
+; CHECK-LABEL: test_force_spill_32:
+; CHECK:         .cfi_startproc
+; CHECK-NEXT:  ; %bb.0: ; %start
+; CHECK-NEXT:    suba.l #124, %sp
+; CHECK-NEXT:    .cfi_def_cfa_offset -128
+; CHECK-NEXT:    movem.l %d2-%d7/%a2-%a6, (80,%sp) ; 48-byte Folded Spill
+; CHECK-NEXT:    jsr get32
+; CHECK-NEXT:    movem.l %d0, (76,%sp)
+; CHECK-NEXT:    jsr get32
+; CHECK-NEXT:    movem.l %d0, (68,%sp)
+; CHECK-NEXT:    jsr get32
+; CHECK-NEXT:    movem.l %d0, (64,%sp)
+; CHECK-NEXT:    jsr get32
+; CHECK-NEXT:    move.l %d0, %d2
+; CHECK-NEXT:    jsr get32
+; CHECK-NEXT:    move.l %d0, %d3
+; CHECK-NEXT:    jsr get32
+; CHECK-NEXT:    move.l %d0, %d4
+; CHECK-NEXT:    jsr get32
+; CHECK-NEXT:    move.l %d0, %d5
+; CHECK-NEXT:    jsr get32
+; CHECK-NEXT:    move.l %d0, %d6
+; CHECK-NEXT:    jsr get32
+; CHECK-NEXT:    move.l %d0, %d7
+; CHECK-NEXT:    jsr get32
+; CHECK-NEXT:    move.l %d0, %a2
+; CHECK-NEXT:    jsr get32
+; CHECK-NEXT:    move.l %d0, %a3
+; CHECK-NEXT:    jsr get32
+; CHECK-NEXT:    move.l %d0, %a4
+; CHECK-NEXT:    jsr get32
+; CHECK-NEXT:    move.l %d0, %a5
+; CHECK-NEXT:    jsr get32
+; CHECK-NEXT:    move.l %d0, %a6
+; CHECK-NEXT:    jsr get32
+; CHECK-NEXT:    movem.l %d0, (72,%sp)
+; CHECK-NEXT:    jsr get32
+; CHECK-NEXT:    movem.l (64,%sp), %a1
+; CHECK-NEXT:    movem.l (68,%sp), %d1
+; CHECK-NEXT:    move.l %sp, %a0
+; CHECK-NEXT:    move.l %d0, (60,%a0)
+; CHECK-NEXT:    movem.l (72,%sp), %d0
+; CHECK-NEXT:    move.l %d0, (56,%a0)
+; CHECK-NEXT:    movem.l (76,%sp), %d0
+; CHECK-NEXT:    move.l %a6, (52,%a0)
+; CHECK-NEXT:    move.l %a5, (48,%a0)
+; CHECK-NEXT:    move.l %a4, (44,%a0)
+; CHECK-NEXT:    move.l %a3, (40,%a0)
+; CHECK-NEXT:    move.l %a2, (36,%a0)
+; CHECK-NEXT:    move.l %d7, (32,%a0)
+; CHECK-NEXT:    move.l %d6, (28,%a0)
+; CHECK-NEXT:    move.l %d5, (24,%a0)
+; CHECK-NEXT:    move.l %d4, (20,%a0)
+; CHECK-NEXT:    move.l %d3, (16,%a0)
+; CHECK-NEXT:    move.l %d2, (12,%a0)
+; CHECK-NEXT:    move.l %a1, (8,%a0)
+; CHECK-NEXT:    move.l %d1, (4,%a0)
+; CHECK-NEXT:    move.l %d0, (%a0)
+; CHECK-NEXT:    jsr test_force_spill_32_consumer
+; CHECK-NEXT:    movem.l (80,%sp), %d2-%d7/%a2-%a6 ; 48-byte Folded Reload
+; CHECK-NEXT:    adda.l #124, %sp
+; CHECK-NEXT:    rts
+  start:
+    %r0 = call i32 @get32()
+    %r1 = call i32 @get32()
+    %r2 = call i32 @get32()
+    %r3 = call i32 @get32()
+    %r4 = call i32 @get32()
+    %r5 = call i32 @get32()
+    %r6 = call i32 @get32()
+    %r7 = call i32 @get32()
+    %r8 = call i32 @get32()
+    %r9 = call i32 @get32()
+    %ra = call i32 @get32()
+    %rb = call i32 @get32()
+    %rc = call i32 @get32()
+    %rd = call i32 @get32()
+    %re = call i32 @get32()
+    %rf = call i32 @get32()
+    call void @test_force_spill_32_consumer(i32 %r0, i32 %r1, i32 %r2, i32 %r3, i32 %r4, i32 %r5, i32 %r6, i32 %r7, i32 %r8, i32 %r9, i32 %ra, i32 %rb, i32 %rc, i32 %rd, i32 %re, i32 %rf)
+    ret void
+}
+
+declare void @test_force_spill_32_consumer(i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32)
+
+define void @test_force_spill_mixed() {
+; CHECK-LABEL: test_force_spill_mixed:
+; CHECK:         .cfi_startproc
+; CHECK-NEXT:  ; %bb.0: ; %start
+; CHECK-NEXT:    suba.l #148, %sp
+; CHECK-NEXT:    .cfi_def_cfa_offset -152
+; CHECK-NEXT:    movem.l %d2-%d7/%a2-%a6, (104,%sp) ; 48-byte Folded Spill
+; CHECK-NEXT:    jsr get8
+; CHECK-NEXT:    movem.w %d0, (102,%sp)
+; CHECK-NEXT:    jsr get16
+; CHECK-NEXT:    movem.w %d0, (86,%sp)
+; CHECK-NEXT:    jsr get32
+; CHECK-NEXT:    movem.l %d0, (80,%sp)
+; CHECK-NEXT:    jsr get16
+; CHECK-NEXT:    move.w %d0, %d2
+; CHECK-NEXT:    jsr get8
+; CHECK-NEXT:    move.b %d0, %d3
+; CHECK-NEXT:    jsr get16
+; CHECK-NEXT:    move.w %d0, %d4
+; CHECK-NEXT:    jsr get32
+; CHECK-NEXT:    move.l %d0, %d5
+; CHECK-NEXT:    jsr get16
+; CHECK-NEXT:    move.w %d0, %d6
+; CHECK-NEXT:    jsr get8
+; CHECK-NEXT:    move.b %d0, %d7
+; CHECK-NEXT:    jsr get16
+; CHECK-NEXT:    move.w %d0, %a2
+; CHECK-NEXT:    jsr get32
+; CHECK-NEXT:    move.l %d0, %a3
+; CHECK-NEXT:    jsr get16
+; CHECK-NEXT:    move.w %d0, %a4
+; CHECK-NEXT:    jsr get8
+; CHECK-NEXT:    movem.w %d0, (100,%sp)
+; CHECK-NEXT:    jsr get16
+; CHECK-NEXT:    move.w %d0, %a5
+; CHECK-NEXT:    jsr get32
+; CHECK-NEXT:    move.l %d0, %a6
+; CHECK-NEXT:    jsr get16
+; CHECK-NEXT:    movem.w %d0, (98,%sp)
+; CHECK-NEXT:    jsr get8
+; CHECK-NEXT:    movem.w %d0, (96,%sp)
+; CHECK-NEXT:    jsr get16
+; CHECK-NEXT:    movem.w %d0, (94,%sp)
+; CHECK-NEXT:    jsr get32
+; CHECK-NEXT:    movem.l %d0, (88,%sp)
+; CHECK-NEXT:    jsr get16
+; CHECK-NEXT:    movem.l (80,%sp), %a1
+; CHECK-NEXT:    movem.w (86,%sp), %d1
+; CHECK-NEXT:    and.l #65535, %d0
+; CHECK-NEXT:    move.l %sp, %a0
+; CHECK-NEXT:    move.l %d0, (76,%a0)
+; CHECK-NEXT:    movem.l (88,%sp), %d0
+; CHECK-NEXT:    move.l %d0, (72,%a0)
+; CHECK-NEXT:    movem.w (94,%sp), %d0
+; CHECK-NEXT:    and.l #65535, %d0
+; CHECK-NEXT:    move.l %d0, (68,%a0)
+; CHECK-NEXT:    movem.w (96,%sp), %d0
+; CHECK-NEXT:    and.l #255, %d0
+; CHECK-NEXT:    move.l %d0, (64,%a0)
+; CHECK-NEXT:    movem.w (98,%sp), %d0
+; CHECK-NEXT:    and.l #65535, %d0
+; CHECK-NEXT:    move.l %d0, (60,%a0)
+; CHECK-NEXT:    movem.w (100,%sp), %d0
+; CHECK-NEXT:    move.l %a6, (56,%a0)
+; CHECK-NEXT:    and.l #65535, %a5
+; CHECK-NEXT:    move.l %a5, (52,%a0)
+; CHECK-NEXT:    and.l #255, %d0
+; CHECK-NEXT:    move.l %d0, (48,%a0)
+; CHECK-NEXT:    movem.w (102,%sp), %d0
+; CHECK-NEXT:    and.l #65535, %a4
+; CHECK-NEXT:    move.l %a4, (44,%a0)
+; CHECK-NEXT:    move.l %a3, (40,%a0)
+; CHECK-NEXT:    and.l #65535, %a2
+; CHECK-NEXT:    move.l %a2, (36,%a0)
+; CHECK-NEXT:    and.l #255, %d7
+; CHECK-NEXT:    move.l %d7, (32,%a0)
+; CHECK-NEXT:    and.l #65535, %d6
+; CHECK-NEXT:    move.l %d6, (28,%a0)
+; CHECK-NEXT:    move.l %d5, (24,%a0)
+; CHECK-NEXT:    and.l #65535, %d4
+; CHECK-NEXT:    move.l %d4, (20,%a0)
+; CHECK-NEXT:    and.l #255, %d3
+; CHECK-NEXT:    move.l %d3, (16,%a0)
+; CHECK-NEXT:    and.l #65535, %d2
+; CHECK-NEXT:    move.l %d2, (12,%a0)
+; CHECK-NEXT:    move.l %a1, (8,%a0)
+; CHECK-NEXT:    and.l #65535, %d1
+; CHECK-NEXT:    move.l %d1, (4,%a0)
+; CHECK-NEXT:    and.l #255, %d0
+; CHECK-NEXT:    move.l %d0, (%a0)
+; CHECK-NEXT:    jsr test_force_spill_mixed_consumer
+; CHECK-NEXT:    movem.l (104,%sp), %d2-%d7/%a2-%a6 ; 48-byte Folded Reload
+; CHECK-NEXT:    adda.l #148, %sp
+; CHECK-NEXT:    rts
+  start:
+    %r0 = call i8 @get8()
+    %r1 = call i16 @get16()
+    %r2 = call i32 @get32()
+    %r3 = call i16 @get16()
+    %r4 = call i8 @get8()
+    %r5 = call i16 @get16()
+    %r6 = call i32 @get32()
+    %r7 = call i16 @get16()
+    %r8 = call i8 @get8()
+    %r9 = call i16 @get16()
+    %ra = call i32 @get32()
+    %rb = call i16 @get16()
+    %rc = call i8 @get8()
+    %rd = call i16 @get16()
+    %re = call i32 @get32()
+    %rf = call i16 @get16()
+    %rg = call i8 @get8()
+    %rh = call i16 @get16()
+    %ri = call i32 @get32()
+    %rj = call i16 @get16()
+    call void @test_force_spill_mixed_consumer(i8 %r0, i16 %r1, i32 %r2, i16 %r3, i8 %r4, i16 %r5, i32 %r6, i16 %r7, i8 %r8, i16 %r9, i32 %ra, i16 %rb, i8 %rc, i16 %rd, i32 %re, i16 %rf, i8 %rg, i16 %rh, i32 %ri, i16 %rj)
+    ret void
+}
+
+declare void @test_force_spill_mixed_consumer(i8, i16, i32, i16, i8, i16, i32, i16, i8, i16, i32, i16, i8, i16, i32, i16, i8, i16, i32, i16)
diff --git a/llvm/test/CodeGen/MIR/X86/fake-use-tailcall.mir b/llvm/test/CodeGen/MIR/X86/fake-use-tailcall.mir
index 6c2cb0e55222b..3b833ae73551a 100644
--- a/llvm/test/CodeGen/MIR/X86/fake-use-tailcall.mir
+++ b/llvm/test/CodeGen/MIR/X86/fake-use-tailcall.mir
@@ -8,7 +8,7 @@
 # in the return block itself into the predecessor blocks. This test enures that we do so.
 #
 # Generated from the following source with
-# clang -fextend-lifetimes -S -emit-llvm -O2 -mllvm -stop-before=codegenprepare -o test.mir test.c
+# clang -fextend-variable-liveness -S -emit-llvm -O2 -mllvm -stop-before=codegenprepare -o test.mir test.c
 #
 # extern int f0();
 # extern int f1();
diff --git a/llvm/test/CodeGen/MIR/X86/pr126107.mir b/llvm/test/CodeGen/MIR/X86/pr126107.mir
new file mode 100644
index 0000000000000..e8b3e47f6ff74
--- /dev/null
+++ b/llvm/test/CodeGen/MIR/X86/pr126107.mir
@@ -0,0 +1,17 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 5
+# RUN: llc -o - %s -mtriple=x86_64-- -run-pass=machine-cp | FileCheck %s
+
+---
+name:            main
+body:             |
+ bb.0.entry:
+    liveins: $ymm7
+    ; CHECK-LABEL: name: main
+    ; CHECK: liveins: $ymm7
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: renamable $ymm6 = COPY killed renamable $ymm7
+    ; CHECK-NEXT: CALL64r killed renamable $rax, csr_64_mostregs
+    ; CHECK-NEXT: renamable $ymm6 = VPADDWZ256rr $ymm6, $ymm6
+       renamable $ymm6 = COPY killed renamable $ymm7
+       CALL64r killed renamable $rax, csr_64_mostregs
+       renamable $ymm6 = VPADDWZ256rr $ymm6, $ymm6
diff --git a/llvm/test/CodeGen/Mips/2010-07-20-Switch.ll b/llvm/test/CodeGen/Mips/2010-07-20-Switch.ll
index 08e45155e8af1..bd6c56bc8f6f0 100644
--- a/llvm/test/CodeGen/Mips/2010-07-20-Switch.ll
+++ b/llvm/test/CodeGen/Mips/2010-07-20-Switch.ll
@@ -1,7 +1,8 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
 ; RUN: llc < %s -mtriple=mips -relocation-model=static | \
-; RUN: FileCheck %s -check-prefix=STATIC-O32 
+; RUN: FileCheck %s -check-prefix=STATIC-O32
 ; RUN: llc < %s -mtriple=mips -relocation-model=pic | \
-; RUN: FileCheck %s -check-prefix=PIC-O32 
+; RUN: FileCheck %s -check-prefix=PIC-O32
 ; RUN: llc < %s -mtriple=mips64 -relocation-model=pic -mcpu=mips4 | \
 ; RUN:     FileCheck %s -check-prefix=PIC-N64
 ; RUN: llc < %s -mtriple=mips64 -relocation-model=static -mcpu=mips4 | \
@@ -12,38 +13,175 @@
 ; RUN:     FileCheck %s -check-prefix=STATIC-N64
 
 define i32 @main() nounwind readnone {
+; STATIC-O32-LABEL: main:
+; STATIC-O32:       # %bb.0: # %entry
+; STATIC-O32-NEXT:    addiu $sp, $sp, -8
+; STATIC-O32-NEXT:    addiu $1, $zero, 2
+; STATIC-O32-NEXT:    sw $1, 4($sp)
+; STATIC-O32-NEXT:    lw $2, 4($sp)
+; STATIC-O32-NEXT:    sltiu $1, $2, 4
+; STATIC-O32-NEXT:    beqz $1, $BB0_6
+; STATIC-O32-NEXT:    nop
+; STATIC-O32-NEXT:  # %bb.1: # %entry
+; STATIC-O32-NEXT:    sll $1, $2, 2
+; STATIC-O32-NEXT:    lui $2, %hi($JTI0_0)
+; STATIC-O32-NEXT:    addu $1, $1, $2
+; STATIC-O32-NEXT:    lw $1, %lo($JTI0_0)($1)
+; STATIC-O32-NEXT:    jr $1
+; STATIC-O32-NEXT:    nop
+; STATIC-O32-NEXT:  $BB0_2: # %bb5
+; STATIC-O32-NEXT:    addiu $2, $zero, 1
+; STATIC-O32-NEXT:    jr $ra
+; STATIC-O32-NEXT:    addiu $sp, $sp, 8
+; STATIC-O32-NEXT:  $BB0_3: # %bb2
+; STATIC-O32-NEXT:    addiu $2, $zero, 0
+; STATIC-O32-NEXT:    jr $ra
+; STATIC-O32-NEXT:    addiu $sp, $sp, 8
+; STATIC-O32-NEXT:  $BB0_4: # %bb3
+; STATIC-O32-NEXT:    addiu $2, $zero, 3
+; STATIC-O32-NEXT:    jr $ra
+; STATIC-O32-NEXT:    addiu $sp, $sp, 8
+; STATIC-O32-NEXT:  $BB0_5: # %bb1
+; STATIC-O32-NEXT:    addiu $2, $zero, 2
+; STATIC-O32-NEXT:    jr $ra
+; STATIC-O32-NEXT:    addiu $sp, $sp, 8
+; STATIC-O32-NEXT:  $BB0_6: # %bb4
+; STATIC-O32-NEXT:    addiu $2, $zero, 4
+; STATIC-O32-NEXT:    jr $ra
+; STATIC-O32-NEXT:    addiu $sp, $sp, 8
+;
+; PIC-O32-LABEL: main:
+; PIC-O32:       # %bb.0: # %entry
+; PIC-O32-NEXT:    lui $2, %hi(_gp_disp)
+; PIC-O32-NEXT:    addiu $2, $2, %lo(_gp_disp)
+; PIC-O32-NEXT:    addiu $sp, $sp, -8
+; PIC-O32-NEXT:    addiu $1, $zero, 2
+; PIC-O32-NEXT:    sw $1, 4($sp)
+; PIC-O32-NEXT:    lw $3, 4($sp)
+; PIC-O32-NEXT:    sltiu $1, $3, 4
+; PIC-O32-NEXT:    beqz $1, $BB0_6
+; PIC-O32-NEXT:    addu $2, $2, $25
+; PIC-O32-NEXT:  # %bb.1: # %entry
+; PIC-O32-NEXT:    sll $1, $3, 2
+; PIC-O32-NEXT:    lw $3, %got($JTI0_0)($2)
+; PIC-O32-NEXT:    addu $1, $1, $3
+; PIC-O32-NEXT:    lw $1, %lo($JTI0_0)($1)
+; PIC-O32-NEXT:    addu $1, $2, $1
+; PIC-O32-NEXT:    jr $1
+; PIC-O32-NEXT:    nop
+; PIC-O32-NEXT:  $BB0_2: # %bb5
+; PIC-O32-NEXT:    addiu $2, $zero, 1
+; PIC-O32-NEXT:    jr $ra
+; PIC-O32-NEXT:    addiu $sp, $sp, 8
+; PIC-O32-NEXT:  $BB0_3: # %bb2
+; PIC-O32-NEXT:    addiu $2, $zero, 0
+; PIC-O32-NEXT:    jr $ra
+; PIC-O32-NEXT:    addiu $sp, $sp, 8
+; PIC-O32-NEXT:  $BB0_4: # %bb3
+; PIC-O32-NEXT:    addiu $2, $zero, 3
+; PIC-O32-NEXT:    jr $ra
+; PIC-O32-NEXT:    addiu $sp, $sp, 8
+; PIC-O32-NEXT:  $BB0_5: # %bb1
+; PIC-O32-NEXT:    addiu $2, $zero, 2
+; PIC-O32-NEXT:    jr $ra
+; PIC-O32-NEXT:    addiu $sp, $sp, 8
+; PIC-O32-NEXT:  $BB0_6: # %bb4
+; PIC-O32-NEXT:    addiu $2, $zero, 4
+; PIC-O32-NEXT:    jr $ra
+; PIC-O32-NEXT:    addiu $sp, $sp, 8
+;
+; PIC-N64-LABEL: main:
+; PIC-N64:       # %bb.0: # %entry
+; PIC-N64-NEXT:    daddiu $sp, $sp, -16
+; PIC-N64-NEXT:    lui $1, %hi(%neg(%gp_rel(main)))
+; PIC-N64-NEXT:    daddu $2, $1, $25
+; PIC-N64-NEXT:    addiu $1, $zero, 2
+; PIC-N64-NEXT:    sw $1, 12($sp)
+; PIC-N64-NEXT:    lw $1, 12($sp)
+; PIC-N64-NEXT:    sltiu $4, $1, 4
+; PIC-N64-NEXT:    dsll $3, $1, 32
+; PIC-N64-NEXT:    beqz $4, .LBB0_6
+; PIC-N64-NEXT:    nop
+; PIC-N64-NEXT:  # %bb.1: # %entry
+; PIC-N64-NEXT:    daddiu $1, $2, %lo(%neg(%gp_rel(main)))
+; PIC-N64-NEXT:    dsrl $2, $3, 32
+; PIC-N64-NEXT:    dsll $2, $2, 3
+; PIC-N64-NEXT:    ld $3, %got_page(.LJTI0_0)($1)
+; PIC-N64-NEXT:    daddu $2, $2, $3
+; PIC-N64-NEXT:    ld $2, %got_ofst(.LJTI0_0)($2)
+; PIC-N64-NEXT:    daddu $1, $1, $2
+; PIC-N64-NEXT:    jr $1
+; PIC-N64-NEXT:    nop
+; PIC-N64-NEXT:  .LBB0_2: # %bb5
+; PIC-N64-NEXT:    addiu $2, $zero, 1
+; PIC-N64-NEXT:    jr $ra
+; PIC-N64-NEXT:    daddiu $sp, $sp, 16
+; PIC-N64-NEXT:  .LBB0_3: # %bb2
+; PIC-N64-NEXT:    addiu $2, $zero, 0
+; PIC-N64-NEXT:    jr $ra
+; PIC-N64-NEXT:    daddiu $sp, $sp, 16
+; PIC-N64-NEXT:  .LBB0_4: # %bb3
+; PIC-N64-NEXT:    addiu $2, $zero, 3
+; PIC-N64-NEXT:    jr $ra
+; PIC-N64-NEXT:    daddiu $sp, $sp, 16
+; PIC-N64-NEXT:  .LBB0_5: # %bb1
+; PIC-N64-NEXT:    addiu $2, $zero, 2
+; PIC-N64-NEXT:    jr $ra
+; PIC-N64-NEXT:    daddiu $sp, $sp, 16
+; PIC-N64-NEXT:  .LBB0_6: # %bb4
+; PIC-N64-NEXT:    addiu $2, $zero, 4
+; PIC-N64-NEXT:    jr $ra
+; PIC-N64-NEXT:    daddiu $sp, $sp, 16
+;
+; STATIC-N64-LABEL: main:
+; STATIC-N64:       # %bb.0: # %entry
+; STATIC-N64-NEXT:    daddiu $sp, $sp, -16
+; STATIC-N64-NEXT:    addiu $1, $zero, 2
+; STATIC-N64-NEXT:    sw $1, 12($sp)
+; STATIC-N64-NEXT:    lw $1, 12($sp)
+; STATIC-N64-NEXT:    sltiu $3, $1, 4
+; STATIC-N64-NEXT:    dsll $2, $1, 32
+; STATIC-N64-NEXT:    beqz $3, .LBB0_6
+; STATIC-N64-NEXT:    nop
+; STATIC-N64-NEXT:  # %bb.1: # %entry
+; STATIC-N64-NEXT:    dsrl $1, $2, 32
+; STATIC-N64-NEXT:    dsll $1, $1, 3
+; STATIC-N64-NEXT:    lui $2, %highest(.LJTI0_0)
+; STATIC-N64-NEXT:    daddiu $2, $2, %higher(.LJTI0_0)
+; STATIC-N64-NEXT:    dsll $2, $2, 16
+; STATIC-N64-NEXT:    daddiu $2, $2, %hi(.LJTI0_0)
+; STATIC-N64-NEXT:    dsll $2, $2, 16
+; STATIC-N64-NEXT:    daddu $1, $1, $2
+; STATIC-N64-NEXT:    ld $1, %lo(.LJTI0_0)($1)
+; STATIC-N64-NEXT:    jr $1
+; STATIC-N64-NEXT:    nop
+; STATIC-N64-NEXT:  .LBB0_2: # %bb5
+; STATIC-N64-NEXT:    addiu $2, $zero, 1
+; STATIC-N64-NEXT:    jr $ra
+; STATIC-N64-NEXT:    daddiu $sp, $sp, 16
+; STATIC-N64-NEXT:  .LBB0_3: # %bb2
+; STATIC-N64-NEXT:    addiu $2, $zero, 0
+; STATIC-N64-NEXT:    jr $ra
+; STATIC-N64-NEXT:    daddiu $sp, $sp, 16
+; STATIC-N64-NEXT:  .LBB0_4: # %bb3
+; STATIC-N64-NEXT:    addiu $2, $zero, 3
+; STATIC-N64-NEXT:    jr $ra
+; STATIC-N64-NEXT:    daddiu $sp, $sp, 16
+; STATIC-N64-NEXT:  .LBB0_5: # %bb1
+; STATIC-N64-NEXT:    addiu $2, $zero, 2
+; STATIC-N64-NEXT:    jr $ra
+; STATIC-N64-NEXT:    daddiu $sp, $sp, 16
+; STATIC-N64-NEXT:  .LBB0_6: # %bb4
+; STATIC-N64-NEXT:    addiu $2, $zero, 4
+; STATIC-N64-NEXT:    jr $ra
+; STATIC-N64-NEXT:    daddiu $sp, $sp, 16
 entry:
   %x = alloca i32, align 4                        ; <ptr> [#uses=2]
   store volatile i32 2, ptr %x, align 4
   %0 = load volatile i32, ptr %x, align 4             ; <i32> [#uses=1]
-; STATIC-O32: sll $[[R0:[0-9]+]], ${{[0-9]+}}, 2
-; STATIC-O32: lui $[[R1:[0-9]+]], %hi($JTI0_0)
-; STATIC-O32: addu $[[R2:[0-9]+]], $[[R0]], $[[R1]]
-; STATIC-O32: lw $[[R3:[0-9]+]], %lo($JTI0_0)($[[R2]])
 
-; PIC-O32: sll $[[R0:[0-9]+]], ${{[0-9]+}}, 2
-; PIC-O32: lw $[[R1:[0-9]+]], %got($JTI0_0)
-; PIC-O32: addu $[[R2:[0-9]+]], $[[R0]], $[[R1]]
-; PIC-O32: lw $[[R4:[0-9]+]], %lo($JTI0_0)($[[R2]])
-; PIC-O32: addu $[[R5:[0-9]+]], $[[R4:[0-9]+]]
-; PIC-O32: jr  $[[R5]]
 
-; STATIC-N64: dsrl $[[I32:[0-9]]], ${{[0-9]+}}, 32
-; STATIC-N64: dsll $[[R0:[0-9]]], $[[I32]], 3
-; STATIC-N64: lui $[[R1:[0-9]]], %highest(.LJTI0_0)
-; STATIC-N64: daddiu $[[R2:[0-9]]], $[[R1]], %higher(.LJTI0_0)
-; STATIC-N64: dsll $[[R3:[0-9]]], $[[R2]], 16
-; STATIC-N64: daddiu $[[R4:[0-9]]], $[[R3]], %hi(.LJTI0_0)
-; STATIC-N64: dsll $[[R5:[0-9]]], $[[R4]], 16
-; STATIC-N64: daddu $[[R6:[0-9]]], $[[R0]], $[[R4]]
-; STATIC-N64: ld ${{[0-9]+}}, %lo(.LJTI0_0)($[[R6]])
 
-; PIC-N64: dsll $[[R0:[0-9]+]], ${{[0-9]+}}, 32
-; PIC-N64: ld $[[R1:[0-9]+]], %got_page(.LJTI0_0)
-; PIC-N64: daddu $[[R2:[0-9]+]], $[[R0:[0-9]+]], $[[R1]]
-; PIC-N64: ld $[[R4:[0-9]+]], %got_ofst(.LJTI0_0)($[[R2]])
-; PIC-N64: daddu $[[R5:[0-9]+]], $[[R4:[0-9]+]]
-; PIC-N64: jr  $[[R5]]
   switch i32 %0, label %bb4 [
     i32 0, label %bb5
     i32 1, label %bb1
@@ -66,6 +204,7 @@ bb4:                                              ; preds = %entry
 bb5:                                              ; preds = %entry
   ret i32 1
 }
+; UTC_ARGS: --disable
 
 ; STATIC-O32: .p2align  2
 ; STATIC-O32: $JTI0_0:
@@ -85,7 +224,7 @@ bb5:                                              ; preds = %entry
 ; STATIC-N64: .8byte
 ; STATIC-N64: .8byte
 ; STATIC-N64: .8byte
-;; PIC-N64: .p2align  3
+; PIC-N64: .p2align  3
 ; PIC-N64: .LJTI0_0:
 ; PIC-N64: .gpdword
 ; PIC-N64: .gpdword
diff --git a/llvm/test/CodeGen/Mips/dins.ll b/llvm/test/CodeGen/Mips/dins.ll
index cdb8f419eb2be..2e9816e7d3642 100644
--- a/llvm/test/CodeGen/Mips/dins.ll
+++ b/llvm/test/CodeGen/Mips/dins.ll
@@ -304,3 +304,56 @@ entry:
   store volatile i32 %or, ptr %x.addr, align 4
   ret i32 %and
 }
+
+define i32 @pr125954(i32 %arg, i1 %c) {
+; MIPS64R2-LABEL: pr125954:
+; MIPS64R2:       # %bb.0:
+; MIPS64R2-NEXT:    sll $1, $4, 0
+; MIPS64R2-NEXT:    addiu $2, $zero, -1
+; MIPS64R2-NEXT:    move $3, $1
+; MIPS64R2-NEXT:    ins $3, $2, 8, 24
+; MIPS64R2-NEXT:    andi $2, $1, 255
+; MIPS64R2-NEXT:    sll $1, $5, 0
+; MIPS64R2-NEXT:    andi $1, $1, 1
+; MIPS64R2-NEXT:    jr $ra
+; MIPS64R2-NEXT:    movn $2, $3, $1
+;
+; MIPS32R2-LABEL: pr125954:
+; MIPS32R2:       # %bb.0:
+; MIPS32R2-NEXT:    andi $2, $4, 255
+; MIPS32R2-NEXT:    addiu $1, $zero, -256
+; MIPS32R2-NEXT:    or $1, $2, $1
+; MIPS32R2-NEXT:    andi $3, $5, 1
+; MIPS32R2-NEXT:    jr $ra
+; MIPS32R2-NEXT:    movn $2, $1, $3
+;
+; MIPS16-LABEL: pr125954:
+; MIPS16:       # %bb.0:
+; MIPS16-NEXT:    li $6, 1
+; MIPS16-NEXT:    and $6, $5
+; MIPS16-NEXT:    li $2, 255
+; MIPS16-NEXT:    and $2, $4
+; MIPS16-NEXT:    move $3, $zero
+; MIPS16-NEXT:    beqz $6, $BB2_2 # 16 bit inst
+; MIPS16-NEXT:  # %bb.1:
+; MIPS16-NEXT:    addiu $3, -256
+; MIPS16-NEXT:    or $2, $3
+; MIPS16-NEXT:  $BB2_2:
+; MIPS16-NEXT:    jrc $ra
+;
+; MIPS64R2N32-LABEL: pr125954:
+; MIPS64R2N32:       # %bb.0:
+; MIPS64R2N32-NEXT:    sll $1, $4, 0
+; MIPS64R2N32-NEXT:    addiu $2, $zero, -1
+; MIPS64R2N32-NEXT:    move $3, $1
+; MIPS64R2N32-NEXT:    ins $3, $2, 8, 24
+; MIPS64R2N32-NEXT:    andi $2, $1, 255
+; MIPS64R2N32-NEXT:    sll $1, $5, 0
+; MIPS64R2N32-NEXT:    andi $1, $1, 1
+; MIPS64R2N32-NEXT:    jr $ra
+; MIPS64R2N32-NEXT:    movn $2, $3, $1
+  %and = and i32 %arg, 255
+  %or = or i32 %and, -256
+  %sel = select i1 %c, i32 %or, i32 %and
+  ret i32 %sel
+}
diff --git a/llvm/test/CodeGen/Mips/indirect-jump-hazard/jumptables.ll b/llvm/test/CodeGen/Mips/indirect-jump-hazard/jumptables.ll
index 634216027ef6b..d03514f3d9e03 100644
--- a/llvm/test/CodeGen/Mips/indirect-jump-hazard/jumptables.ll
+++ b/llvm/test/CodeGen/Mips/indirect-jump-hazard/jumptables.ll
@@ -356,7 +356,7 @@ define ptr @_Z3fooi(i32 signext %Letter) {
 ; PIC-MIPS32R2-NEXT:    lw $3, %got($JTI0_0)($2)
 ; PIC-MIPS32R2-NEXT:    addu $1, $1, $3
 ; PIC-MIPS32R2-NEXT:    lw $1, %lo($JTI0_0)($1)
-; PIC-MIPS32R2-NEXT:    addu $1, $1, $2
+; PIC-MIPS32R2-NEXT:    addu $1, $2, $1
 ; PIC-MIPS32R2-NEXT:    jr.hb $1
 ; PIC-MIPS32R2-NEXT:    nop
 ; PIC-MIPS32R2-NEXT:  $BB0_2: # %sw.bb
@@ -418,7 +418,7 @@ define ptr @_Z3fooi(i32 signext %Letter) {
 ; PIC-MIPS32R6-NEXT:    lw $3, %got($JTI0_0)($2)
 ; PIC-MIPS32R6-NEXT:    addu $1, $1, $3
 ; PIC-MIPS32R6-NEXT:    lw $1, %lo($JTI0_0)($1)
-; PIC-MIPS32R6-NEXT:    addu $1, $1, $2
+; PIC-MIPS32R6-NEXT:    addu $1, $2, $1
 ; PIC-MIPS32R6-NEXT:    jr.hb $1
 ; PIC-MIPS32R6-NEXT:    nop
 ; PIC-MIPS32R6-NEXT:  $BB0_2: # %sw.bb
@@ -481,7 +481,7 @@ define ptr @_Z3fooi(i32 signext %Letter) {
 ; PIC-MIPS64R2-NEXT:    ld $3, %got_page(.LJTI0_0)($2)
 ; PIC-MIPS64R2-NEXT:    daddu $1, $1, $3
 ; PIC-MIPS64R2-NEXT:    ld $1, %got_ofst(.LJTI0_0)($1)
-; PIC-MIPS64R2-NEXT:    daddu $1, $1, $2
+; PIC-MIPS64R2-NEXT:    daddu $1, $2, $1
 ; PIC-MIPS64R2-NEXT:    jr.hb $1
 ; PIC-MIPS64R2-NEXT:    nop
 ; PIC-MIPS64R2-NEXT:  .LBB0_2: # %sw.bb
@@ -544,7 +544,7 @@ define ptr @_Z3fooi(i32 signext %Letter) {
 ; PIC-MIPS64R6-NEXT:    ld $3, %got_page(.LJTI0_0)($2)
 ; PIC-MIPS64R6-NEXT:    daddu $1, $1, $3
 ; PIC-MIPS64R6-NEXT:    ld $1, %got_ofst(.LJTI0_0)($1)
-; PIC-MIPS64R6-NEXT:    daddu $1, $1, $2
+; PIC-MIPS64R6-NEXT:    daddu $1, $2, $1
 ; PIC-MIPS64R6-NEXT:    jr.hb $1
 ; PIC-MIPS64R6-NEXT:    nop
 ; PIC-MIPS64R6-NEXT:  .LBB0_2: # %sw.bb
diff --git a/llvm/test/CodeGen/Mips/jump-table-mul.ll b/llvm/test/CodeGen/Mips/jump-table-mul.ll
index cca6080a07544..2211775dc0523 100644
--- a/llvm/test/CodeGen/Mips/jump-table-mul.ll
+++ b/llvm/test/CodeGen/Mips/jump-table-mul.ll
@@ -16,7 +16,7 @@ define i64 @test(i64 %arg) {
 ; CHECK-NEXT:    ld $3, %got_page(.LJTI0_0)($1)
 ; CHECK-NEXT:    daddu $2, $2, $3
 ; CHECK-NEXT:    ld $2, %got_ofst(.LJTI0_0)($2)
-; CHECK-NEXT:    daddu $1, $2, $1
+; CHECK-NEXT:    daddu $1, $1, $2
 ; CHECK-NEXT:    jr $1
 ; CHECK-NEXT:    nop
 ; CHECK-NEXT:  .LBB0_2: # %sw.bb
diff --git a/llvm/test/CodeGen/Mips/llvm-ir/add.ll b/llvm/test/CodeGen/Mips/llvm-ir/add.ll
index f6b3b96aaa0ce..a21477acd5341 100644
--- a/llvm/test/CodeGen/Mips/llvm-ir/add.ll
+++ b/llvm/test/CodeGen/Mips/llvm-ir/add.ll
@@ -38,18 +38,11 @@ define signext i1 @add_i1(i1 signext %a, i1 signext %b) {
 entry:
 ; ALL-LABEL: add_i1:
 
-  ; NOT-R2-R6:  addu   $[[T0:[0-9]+]], $4, $5
-  ; NOT-R2-R6:  andi   $[[T0]], $[[T0]], 1
-  ; NOT-R2-R6:  negu   $2, $[[T0]]
+  ; NOT-R2-R6:  xor    $[[T0:[0-9]+]], $4, $5
 
-  ; R2-R6:      addu   $[[T0:[0-9]+]], $4, $5
-  ; R2-R6:      andi   $[[T0]], $[[T0]], 1
-  ; R2-R6:      negu   $2, $[[T0]]
+  ; R2-R6:      xor    $[[T0:[0-9]+]], $4, $5
 
-  ; MMR6:       addu16  $[[T0:[0-9]+]], $4, $5
-  ; MMR6:       andi16  $[[T0]], $[[T0]], 1
-  ; MMR6:       li16    $[[T1:[0-9]+]], 0
-  ; MMR6:       subu16  $[[T0]], $[[T1]], $[[T0]]
+  ; MMR6:       xor    $[[T0:[0-9]+]], $4, $5
 
   %r = add i1 %a, %b
   ret i1 %r
@@ -368,18 +361,11 @@ define signext i128 @add_i128_4(i128 signext %a) {
 
 define signext i1 @add_i1_3(i1 signext %a) {
 ; ALL-LABEL: add_i1_3:
-  ; GP32:        addiu  $[[T0:[0-9]+]], $4, 1
-  ; GP32:        andi   $[[T0]], $[[T0]], 1
-  ; GP32:        negu   $2, $[[T0]]
-
-  ; GP64:        addiu  $[[T0:[0-9]+]], $4, 1
-  ; GP64:        andi   $[[T0]], $[[T0]], 1
-  ; GP64:        negu   $2, $[[T0]]
-
-  ; MMR6:        addiur2 $[[T0:[0-9]+]], $4, 1
-  ; MMR6:        andi16  $[[T0]], $[[T0]], 1
-  ; MMR6:        li16    $[[T1:[0-9]+]], 0
-  ; MMR6:        subu16  $2, $[[T1]], $[[T0]]
+  ; GP32:        not    $[[T0:[0-9]+]], $4
+
+  ; GP64:        not    $[[T0:[0-9]+]], $4
+
+  ; MMR6:        not16  $[[T0:[0-9]+]], $4
 
   %r = add i1 3, %a
   ret i1 %r
diff --git a/llvm/test/CodeGen/Mips/llvm-ir/mul.ll b/llvm/test/CodeGen/Mips/llvm-ir/mul.ll
index 00b91d1413cfe..2735d53f5fe06 100644
--- a/llvm/test/CodeGen/Mips/llvm-ir/mul.ll
+++ b/llvm/test/CodeGen/Mips/llvm-ir/mul.ll
@@ -31,36 +31,22 @@ define signext i1 @mul_i1(i1 signext %a, i1 signext %b) {
 entry:
 ; ALL-LABEL: mul_i1:
 
-  ; M2:         mult    $4, $5
-  ; M2:         mflo    $[[T0:[0-9]+]]
-  ; M2:         andi    $[[T0]], $[[T0]], 1
-  ; M2:         negu    $2, $[[T0]]
+  ; M2:         and     $[[T0:[0-9]+]], $4, $5
 
-  ; 32R1-R5:    mul     $[[T0:[0-9]+]], $4, $5
-  ; 32R1-R5:    andi    $[[T0]], $[[T0]], 1
-  ; 32R1-R5:    negu    $2, $[[T0]]
+  ; 32R1-R5:    and     $[[T0:[0-9]+]], $4, $5
 
-  ; 32R6:       mul     $[[T0:[0-9]+]], $4, $5
-  ; 32R6:       andi    $[[T0]], $[[T0]], 1
-  ; 32R6:       negu    $2, $[[T0]]
+  ; 32R6:       and     $[[T0:[0-9]+]], $4, $5
 
-  ; M4:         mult    $4, $5
-  ; M4:         mflo    $[[T0:[0-9]+]]
-  ; M4:         andi    $[[T0]], $[[T0]], 1
-  ; M4:         negu    $2, $[[T0]]
+  ; M4:         and     $[[T0:[0-9]+]], $4, $5
 
-  ; 64R1-R5:    mul     $[[T0:[0-9]+]], $4, $5
-  ; 64R1-R5:    andi    $[[T0]], $[[T0]], 1
-  ; 64R1-R5:    negu    $2, $[[T0]]
+  ; 64R1-R5:    and     $[[T0:[0-9]+]], $4, $5
 
-  ; 64R6:       mul     $[[T0:[0-9]+]], $4, $5
-  ; 64R6:       andi    $[[T0]], $[[T0]], 1
-  ; 64R6:       negu    $2, $[[T0]]
+  ; 64R6:       and     $[[T0:[0-9]+]], $4, $5
 
-  ; MM32:       mul     $[[T0:[0-9]+]], $4, $5
-  ; MM32:       andi16  $[[T0]], $[[T0]], 1
-  ; MM32:       li16    $[[T1:[0-9]+]], 0
-  ; MM32:       subu16  $2, $[[T1]], $[[T0]]
+  ; MM32R3:     and16 $4, $5
+  ; MM32R3:     move $2, $4
+
+  ; MM32R6:     and     $[[T0:[0-9]+]], $4, $5
 
   %r = mul i1 %a, %b
   ret i1 %r
diff --git a/llvm/test/CodeGen/Mips/llvm-ir/sub.ll b/llvm/test/CodeGen/Mips/llvm-ir/sub.ll
index b465e24d47a05..dd5e6e957245d 100644
--- a/llvm/test/CodeGen/Mips/llvm-ir/sub.ll
+++ b/llvm/test/CodeGen/Mips/llvm-ir/sub.ll
@@ -33,14 +33,9 @@ define signext i1 @sub_i1(i1 signext %a, i1 signext %b) {
 entry:
 ; ALL-LABEL: sub_i1:
 
-  ; NOT-MM:         subu    $[[T0:[0-9]+]], $4, $5
-  ; NOT-MM:         andi    $[[T0]], $[[T0]], 1
-  ; NOT-MM:         negu    $2, $[[T0]]
+  ; NOT-MM:         xor     $[[T0:[0-9]+]], $4, $5
 
-  ; MM:             subu16  $[[T0:[0-9]+]], $4, $5
-  ; MM:             andi16  $[[T0]], $[[T0]], 1
-  ; MM:             li16    $[[T1:[0-9]+]], 0
-  ; MM:             subu16  $2, $[[T1]], $[[T0]]
+  ; MM:             xor16   $[[T0:[0-9]+]], $4, $5
 
   %r = sub i1 %a, %b
   ret i1 %r
diff --git a/llvm/test/CodeGen/Mips/pseudo-jump-fill.ll b/llvm/test/CodeGen/Mips/pseudo-jump-fill.ll
index 5d7cdbc0b69ed..16186c9be9e26 100644
--- a/llvm/test/CodeGen/Mips/pseudo-jump-fill.ll
+++ b/llvm/test/CodeGen/Mips/pseudo-jump-fill.ll
@@ -20,7 +20,7 @@ define i32 @test(i32 signext %x, i32 signext %c) {
 ; CHECK-NEXT:    lw $6, %got($JTI0_0)($3)
 ; CHECK-NEXT:    addu16 $5, $5, $6
 ; CHECK-NEXT:    lw $5, %lo($JTI0_0)($5)
-; CHECK-NEXT:    addu16 $3, $5, $3
+; CHECK-NEXT:    addu16 $3, $3, $5
 ; CHECK-NEXT:    jr $3
 ; CHECK-NEXT:    nop
 ; CHECK-NEXT:  $BB0_2: # %sw.bb2
diff --git a/llvm/test/CodeGen/NVPTX/boolean-patterns.ll b/llvm/test/CodeGen/NVPTX/boolean-patterns.ll
index 6ed9890610826..fd4d325ae9374 100644
--- a/llvm/test/CodeGen/NVPTX/boolean-patterns.ll
+++ b/llvm/test/CodeGen/NVPTX/boolean-patterns.ll
@@ -3,7 +3,7 @@
 
 ; CHECK-LABEL: m2and_rr
 define i1 @m2and_rr(i1 %a, i1 %b) {
-; CHECK: and.pred %p{{[0-9]+}}, %p{{[0-9]+}}, %p{{[0-9]+}}
+; CHECK: and.b16 %rs{{[0-9]+}}, %rs{{[0-9]+}}, %rs{{[0-9]+}}
 ; CHECK-NOT: mul
   %r = mul i1 %a, %b
   ret i1 %r
diff --git a/llvm/test/CodeGen/NVPTX/convert-sm100.ll b/llvm/test/CodeGen/NVPTX/convert-sm100.ll
new file mode 100644
index 0000000000000..f92822f7e0c16
--- /dev/null
+++ b/llvm/test/CodeGen/NVPTX/convert-sm100.ll
@@ -0,0 +1,68 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc < %s -mtriple=nvptx64 -mcpu=sm_100 -mattr=+ptx86| FileCheck --check-prefixes=CHECK %s
+; RUN: %if ptxas-12.8 %{ llc < %s -mtriple=nvptx64 -mcpu=sm_100 -mattr=+ptx86| %ptxas-verify -arch=sm_100 %}
+
+declare i32 @llvm.nvvm.f2tf32.rn.satfinite(float %f1)
+declare i32 @llvm.nvvm.f2tf32.rn.relu.satfinite(float %f1)
+declare i32 @llvm.nvvm.f2tf32.rz.satfinite(float %f1)
+declare i32 @llvm.nvvm.f2tf32.rz.relu.satfinite(float %f1)
+
+define i32 @cvt_rn_satf_tf32_f32(float %f1) {
+; CHECK-LABEL: cvt_rn_satf_tf32_f32(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b32 %r<2>;
+; CHECK-NEXT:    .reg .f32 %f<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.f32 %f1, [cvt_rn_satf_tf32_f32_param_0];
+; CHECK-NEXT:    cvt.rn.satfinite.tf32.f32 %r1, %f1;
+; CHECK-NEXT:    st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT:    ret;
+  %val = call i32 @llvm.nvvm.f2tf32.rn.satfinite(float %f1)
+  ret i32 %val
+}
+
+define i32 @cvt_rn_relu_satf_tf32_f32(float %f1) {
+; CHECK-LABEL: cvt_rn_relu_satf_tf32_f32(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b32 %r<2>;
+; CHECK-NEXT:    .reg .f32 %f<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.f32 %f1, [cvt_rn_relu_satf_tf32_f32_param_0];
+; CHECK-NEXT:    cvt.rn.relu.satfinite.tf32.f32 %r1, %f1;
+; CHECK-NEXT:    st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT:    ret;
+  %val = call i32 @llvm.nvvm.f2tf32.rn.relu.satfinite(float %f1)
+  ret i32 %val
+}
+
+define i32 @cvt_rz_satf_tf32_f32(float %f1) {
+; CHECK-LABEL: cvt_rz_satf_tf32_f32(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b32 %r<2>;
+; CHECK-NEXT:    .reg .f32 %f<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.f32 %f1, [cvt_rz_satf_tf32_f32_param_0];
+; CHECK-NEXT:    cvt.rz.satfinite.tf32.f32 %r1, %f1;
+; CHECK-NEXT:    st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT:    ret;
+  %val = call i32 @llvm.nvvm.f2tf32.rz.satfinite(float %f1)
+  ret i32 %val
+}
+
+define i32 @cvt_rz_relu_satf_tf32_f32(float %f1) {
+; CHECK-LABEL: cvt_rz_relu_satf_tf32_f32(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b32 %r<2>;
+; CHECK-NEXT:    .reg .f32 %f<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.f32 %f1, [cvt_rz_relu_satf_tf32_f32_param_0];
+; CHECK-NEXT:    cvt.rz.relu.satfinite.tf32.f32 %r1, %f1;
+; CHECK-NEXT:    st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT:    ret;
+  %val = call i32 @llvm.nvvm.f2tf32.rz.relu.satfinite(float %f1)
+  ret i32 %val
+}
diff --git a/llvm/test/CodeGen/NVPTX/cp-async-bulk.ll b/llvm/test/CodeGen/NVPTX/cp-async-bulk.ll
index cbb53df4a49b0..aa8f553351b46 100644
--- a/llvm/test/CodeGen/NVPTX/cp-async-bulk.ll
+++ b/llvm/test/CodeGen/NVPTX/cp-async-bulk.ll
@@ -1,6 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
-; RUN: llc < %s -mtriple=nvptx64 -mcpu=sm_90 -mattr=+ptx80| FileCheck --check-prefixes=CHECK-PTX64 %s
-; RUN: llc < %s -mtriple=nvptx64 -mcpu=sm_90 -mattr=+ptx80 --nvptx-short-ptr| FileCheck --check-prefixes=CHECK-PTX-SHARED32 %s
+; RUN: llc < %s -mtriple=nvptx64 -mcpu=sm_90 -mattr=+ptx80| FileCheck --check-prefixes=CHECK,CHECK-PTX64 %s
+; RUN: llc < %s -mtriple=nvptx64 -mcpu=sm_90 -mattr=+ptx80 --nvptx-short-ptr| FileCheck --check-prefixes=CHECK,CHECK-PTX-SHARED32 %s
 ; RUN: %if ptxas-12.3 %{ llc < %s -mtriple=nvptx64 -mcpu=sm_90 -mattr=+ptx80| %ptxas-verify -arch=sm_90 %}
 ; RUN: %if ptxas-12.3 %{ llc < %s -mtriple=nvptx64 -mcpu=sm_90 -mattr=+ptx80 --nvptx-short-ptr| %ptxas-verify -arch=sm_90 %}
 
@@ -119,18 +119,18 @@ define void @cp_async_bulk_cta_to_cluster(ptr addrspace(3) %src, ptr addrspace(3
 }
 
 define void @cp_async_bulk_prefetch(ptr addrspace(1) %src, i32 %size, i64 %ch) {
-; CHECK-PTX64-LABEL: cp_async_bulk_prefetch(
-; CHECK-PTX64:       {
-; CHECK-PTX64-NEXT:    .reg .b32 %r<2>;
-; CHECK-PTX64-NEXT:    .reg .b64 %rd<3>;
-; CHECK-PTX64-EMPTY:
-; CHECK-PTX64-NEXT:  // %bb.0:
-; CHECK-PTX64-NEXT:    ld.param.u64 %rd1, [cp_async_bulk_prefetch_param_0];
-; CHECK-PTX64-NEXT:    ld.param.u32 %r1, [cp_async_bulk_prefetch_param_1];
-; CHECK-PTX64-NEXT:    ld.param.u64 %rd2, [cp_async_bulk_prefetch_param_2];
-; CHECK-PTX64-NEXT:    cp.async.bulk.prefetch.L2.global.L2::cache_hint [%rd1], %r1, %rd2;
-; CHECK-PTX64-NEXT:    cp.async.bulk.prefetch.L2.global [%rd1], %r1;
-; CHECK-PTX64-NEXT:    ret;
+; CHECK-LABEL: cp_async_bulk_prefetch(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b32 %r<2>;
+; CHECK-NEXT:    .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.u64 %rd1, [cp_async_bulk_prefetch_param_0];
+; CHECK-NEXT:    ld.param.u32 %r1, [cp_async_bulk_prefetch_param_1];
+; CHECK-NEXT:    ld.param.u64 %rd2, [cp_async_bulk_prefetch_param_2];
+; CHECK-NEXT:    cp.async.bulk.prefetch.L2.global.L2::cache_hint [%rd1], %r1, %rd2;
+; CHECK-NEXT:    cp.async.bulk.prefetch.L2.global [%rd1], %r1;
+; CHECK-NEXT:    ret;
   tail call void @llvm.nvvm.cp.async.bulk.prefetch.L2(ptr addrspace(1) %src, i32 %size, i64 %ch, i1 1)
   tail call void @llvm.nvvm.cp.async.bulk.prefetch.L2(ptr addrspace(1) %src, i32 %size, i64 0, i1 0)
   ret void
diff --git a/llvm/test/CodeGen/NVPTX/fence-cluster.ll b/llvm/test/CodeGen/NVPTX/fence-cluster.ll
new file mode 100644
index 0000000000000..1683ec1388188
--- /dev/null
+++ b/llvm/test/CodeGen/NVPTX/fence-cluster.ll
@@ -0,0 +1,55 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc < %s -march=nvptx64 -mcpu=sm_90 -mattr=+ptx87 | FileCheck %s --check-prefix=SM90
+; RUN: %if ptxas-12.8 %{ llc < %s -march=nvptx64 -mcpu=sm_90 -mattr=+ptx87 | %ptxas-verify -arch=sm_90 %}
+
+define void @fence_acquire_cluster() {
+; SM90-LABEL: fence_acquire_cluster(
+; SM90:       {
+; SM90-EMPTY:
+; SM90-EMPTY:
+; SM90-NEXT:  // %bb.0:
+; SM90-NEXT:    fence.acquire.cluster;
+; SM90-NEXT:    ret;
+    fence syncscope("cluster") acquire
+    ret void
+}
+
+
+define void @fence_release_cluster() {
+; SM90-LABEL: fence_release_cluster(
+; SM90:       {
+; SM90-EMPTY:
+; SM90-EMPTY:
+; SM90-NEXT:  // %bb.0:
+; SM90-NEXT:    fence.release.cluster;
+; SM90-NEXT:    ret;
+    fence syncscope("cluster") release
+    ret void
+}
+
+
+define void @fence_acq_rel_cluster() {
+; SM90-LABEL: fence_acq_rel_cluster(
+; SM90:       {
+; SM90-EMPTY:
+; SM90-EMPTY:
+; SM90-NEXT:  // %bb.0:
+; SM90-NEXT:    fence.acq_rel.cluster;
+; SM90-NEXT:    ret;
+    fence syncscope("cluster") acq_rel
+    ret void
+}
+
+
+define void @fence_seq_cst_cluster() {
+; SM90-LABEL: fence_seq_cst_cluster(
+; SM90:       {
+; SM90-EMPTY:
+; SM90-EMPTY:
+; SM90-NEXT:  // %bb.0:
+; SM90-NEXT:    fence.sc.cluster;
+; SM90-NEXT:    ret;
+    fence syncscope("cluster") seq_cst
+    ret void
+}
+
diff --git a/llvm/test/CodeGen/NVPTX/fence-nocluster.ll b/llvm/test/CodeGen/NVPTX/fence-nocluster.ll
new file mode 100644
index 0000000000000..1c6c1744b5375
--- /dev/null
+++ b/llvm/test/CodeGen/NVPTX/fence-nocluster.ll
@@ -0,0 +1,355 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc < %s -march=nvptx64 -mcpu=sm_35 -mattr=+ptx50 | FileCheck %s --check-prefix=SM30
+; RUN: %if ptxas && !ptxas-12.0 %{ llc < %s -march=nvptx64 -mcpu=sm_35 -mattr=+ptx50 | %ptxas-verify -arch=sm_35 %}
+; RUN: llc < %s -march=nvptx64 -mcpu=sm_70 -mattr=+ptx60 | FileCheck %s --check-prefix=SM70
+; RUN: %if ptxas %{ llc < %s -march=nvptx64 -mcpu=sm_70 -mattr=+ptx60 | %ptxas-verify -arch=sm_70 %}
+; RUN: llc < %s -march=nvptx64 -mcpu=sm_90 -mattr=+ptx87 | FileCheck %s --check-prefix=SM90
+; RUN: %if ptxas-12.8 %{ llc < %s -march=nvptx64 -mcpu=sm_90 -mattr=+ptx87 | %ptxas-verify -arch=sm_90 %}
+
+define void @fence_acquire_sys() {
+; SM30-LABEL: fence_acquire_sys(
+; SM30:       {
+; SM30-EMPTY:
+; SM30-EMPTY:
+; SM30-NEXT:  // %bb.0:
+; SM30-NEXT:    membar.sys;
+; SM30-NEXT:    ret;
+;
+; SM70-LABEL: fence_acquire_sys(
+; SM70:       {
+; SM70-EMPTY:
+; SM70-EMPTY:
+; SM70-NEXT:  // %bb.0:
+; SM70-NEXT:    fence.acq_rel.sys;
+; SM70-NEXT:    ret;
+;
+; SM90-LABEL: fence_acquire_sys(
+; SM90:       {
+; SM90-EMPTY:
+; SM90-EMPTY:
+; SM90-NEXT:  // %bb.0:
+; SM90-NEXT:    fence.acquire.sys;
+; SM90-NEXT:    ret;
+    fence syncscope("") acquire
+    ret void
+}
+
+
+define void @fence_acquire_cta() {
+; SM30-LABEL: fence_acquire_cta(
+; SM30:       {
+; SM30-EMPTY:
+; SM30-EMPTY:
+; SM30-NEXT:  // %bb.0:
+; SM30-NEXT:    membar.cta;
+; SM30-NEXT:    ret;
+;
+; SM70-LABEL: fence_acquire_cta(
+; SM70:       {
+; SM70-EMPTY:
+; SM70-EMPTY:
+; SM70-NEXT:  // %bb.0:
+; SM70-NEXT:    fence.acq_rel.cta;
+; SM70-NEXT:    ret;
+;
+; SM90-LABEL: fence_acquire_cta(
+; SM90:       {
+; SM90-EMPTY:
+; SM90-EMPTY:
+; SM90-NEXT:  // %bb.0:
+; SM90-NEXT:    fence.acquire.cta;
+; SM90-NEXT:    ret;
+    fence syncscope("block") acquire
+    ret void
+}
+
+
+define void @fence_acquire_gpu() {
+; SM30-LABEL: fence_acquire_gpu(
+; SM30:       {
+; SM30-EMPTY:
+; SM30-EMPTY:
+; SM30-NEXT:  // %bb.0:
+; SM30-NEXT:    membar.gl;
+; SM30-NEXT:    ret;
+;
+; SM70-LABEL: fence_acquire_gpu(
+; SM70:       {
+; SM70-EMPTY:
+; SM70-EMPTY:
+; SM70-NEXT:  // %bb.0:
+; SM70-NEXT:    fence.acq_rel.gpu;
+; SM70-NEXT:    ret;
+;
+; SM90-LABEL: fence_acquire_gpu(
+; SM90:       {
+; SM90-EMPTY:
+; SM90-EMPTY:
+; SM90-NEXT:  // %bb.0:
+; SM90-NEXT:    fence.acquire.gpu;
+; SM90-NEXT:    ret;
+    fence syncscope("device") acquire
+    ret void
+}
+
+
+define void @fence_release_sys() {
+; SM30-LABEL: fence_release_sys(
+; SM30:       {
+; SM30-EMPTY:
+; SM30-EMPTY:
+; SM30-NEXT:  // %bb.0:
+; SM30-NEXT:    membar.sys;
+; SM30-NEXT:    ret;
+;
+; SM70-LABEL: fence_release_sys(
+; SM70:       {
+; SM70-EMPTY:
+; SM70-EMPTY:
+; SM70-NEXT:  // %bb.0:
+; SM70-NEXT:    fence.acq_rel.sys;
+; SM70-NEXT:    ret;
+;
+; SM90-LABEL: fence_release_sys(
+; SM90:       {
+; SM90-EMPTY:
+; SM90-EMPTY:
+; SM90-NEXT:  // %bb.0:
+; SM90-NEXT:    fence.release.sys;
+; SM90-NEXT:    ret;
+    fence syncscope("") release
+    ret void
+}
+
+
+define void @fence_release_cta() {
+; SM30-LABEL: fence_release_cta(
+; SM30:       {
+; SM30-EMPTY:
+; SM30-EMPTY:
+; SM30-NEXT:  // %bb.0:
+; SM30-NEXT:    membar.cta;
+; SM30-NEXT:    ret;
+;
+; SM70-LABEL: fence_release_cta(
+; SM70:       {
+; SM70-EMPTY:
+; SM70-EMPTY:
+; SM70-NEXT:  // %bb.0:
+; SM70-NEXT:    fence.acq_rel.cta;
+; SM70-NEXT:    ret;
+;
+; SM90-LABEL: fence_release_cta(
+; SM90:       {
+; SM90-EMPTY:
+; SM90-EMPTY:
+; SM90-NEXT:  // %bb.0:
+; SM90-NEXT:    fence.release.cta;
+; SM90-NEXT:    ret;
+    fence syncscope("block") release
+    ret void
+}
+
+
+define void @fence_release_gpu() {
+; SM30-LABEL: fence_release_gpu(
+; SM30:       {
+; SM30-EMPTY:
+; SM30-EMPTY:
+; SM30-NEXT:  // %bb.0:
+; SM30-NEXT:    membar.gl;
+; SM30-NEXT:    ret;
+;
+; SM70-LABEL: fence_release_gpu(
+; SM70:       {
+; SM70-EMPTY:
+; SM70-EMPTY:
+; SM70-NEXT:  // %bb.0:
+; SM70-NEXT:    fence.acq_rel.gpu;
+; SM70-NEXT:    ret;
+;
+; SM90-LABEL: fence_release_gpu(
+; SM90:       {
+; SM90-EMPTY:
+; SM90-EMPTY:
+; SM90-NEXT:  // %bb.0:
+; SM90-NEXT:    fence.release.gpu;
+; SM90-NEXT:    ret;
+    fence syncscope("device") release
+    ret void
+}
+
+
+define void @fence_acq_rel_sys() {
+; SM30-LABEL: fence_acq_rel_sys(
+; SM30:       {
+; SM30-EMPTY:
+; SM30-EMPTY:
+; SM30-NEXT:  // %bb.0:
+; SM30-NEXT:    membar.sys;
+; SM30-NEXT:    ret;
+;
+; SM70-LABEL: fence_acq_rel_sys(
+; SM70:       {
+; SM70-EMPTY:
+; SM70-EMPTY:
+; SM70-NEXT:  // %bb.0:
+; SM70-NEXT:    fence.acq_rel.sys;
+; SM70-NEXT:    ret;
+;
+; SM90-LABEL: fence_acq_rel_sys(
+; SM90:       {
+; SM90-EMPTY:
+; SM90-EMPTY:
+; SM90-NEXT:  // %bb.0:
+; SM90-NEXT:    fence.acq_rel.sys;
+; SM90-NEXT:    ret;
+    fence syncscope("") acq_rel
+    ret void
+}
+
+
+define void @fence_acq_rel_cta() {
+; SM30-LABEL: fence_acq_rel_cta(
+; SM30:       {
+; SM30-EMPTY:
+; SM30-EMPTY:
+; SM30-NEXT:  // %bb.0:
+; SM30-NEXT:    membar.cta;
+; SM30-NEXT:    ret;
+;
+; SM70-LABEL: fence_acq_rel_cta(
+; SM70:       {
+; SM70-EMPTY:
+; SM70-EMPTY:
+; SM70-NEXT:  // %bb.0:
+; SM70-NEXT:    fence.acq_rel.cta;
+; SM70-NEXT:    ret;
+;
+; SM90-LABEL: fence_acq_rel_cta(
+; SM90:       {
+; SM90-EMPTY:
+; SM90-EMPTY:
+; SM90-NEXT:  // %bb.0:
+; SM90-NEXT:    fence.acq_rel.cta;
+; SM90-NEXT:    ret;
+    fence syncscope("block") acq_rel
+    ret void
+}
+
+
+define void @fence_acq_rel_gpu() {
+; SM30-LABEL: fence_acq_rel_gpu(
+; SM30:       {
+; SM30-EMPTY:
+; SM30-EMPTY:
+; SM30-NEXT:  // %bb.0:
+; SM30-NEXT:    membar.gl;
+; SM30-NEXT:    ret;
+;
+; SM70-LABEL: fence_acq_rel_gpu(
+; SM70:       {
+; SM70-EMPTY:
+; SM70-EMPTY:
+; SM70-NEXT:  // %bb.0:
+; SM70-NEXT:    fence.acq_rel.gpu;
+; SM70-NEXT:    ret;
+;
+; SM90-LABEL: fence_acq_rel_gpu(
+; SM90:       {
+; SM90-EMPTY:
+; SM90-EMPTY:
+; SM90-NEXT:  // %bb.0:
+; SM90-NEXT:    fence.acq_rel.gpu;
+; SM90-NEXT:    ret;
+    fence syncscope("device") acq_rel
+    ret void
+}
+
+
+define void @fence_seq_cst_sys() {
+; SM30-LABEL: fence_seq_cst_sys(
+; SM30:       {
+; SM30-EMPTY:
+; SM30-EMPTY:
+; SM30-NEXT:  // %bb.0:
+; SM30-NEXT:    membar.sys;
+; SM30-NEXT:    ret;
+;
+; SM70-LABEL: fence_seq_cst_sys(
+; SM70:       {
+; SM70-EMPTY:
+; SM70-EMPTY:
+; SM70-NEXT:  // %bb.0:
+; SM70-NEXT:    fence.sc.sys;
+; SM70-NEXT:    ret;
+;
+; SM90-LABEL: fence_seq_cst_sys(
+; SM90:       {
+; SM90-EMPTY:
+; SM90-EMPTY:
+; SM90-NEXT:  // %bb.0:
+; SM90-NEXT:    fence.sc.sys;
+; SM90-NEXT:    ret;
+    fence syncscope("") seq_cst
+    ret void
+}
+
+
+define void @fence_seq_cst_cta() {
+; SM30-LABEL: fence_seq_cst_cta(
+; SM30:       {
+; SM30-EMPTY:
+; SM30-EMPTY:
+; SM30-NEXT:  // %bb.0:
+; SM30-NEXT:    membar.cta;
+; SM30-NEXT:    ret;
+;
+; SM70-LABEL: fence_seq_cst_cta(
+; SM70:       {
+; SM70-EMPTY:
+; SM70-EMPTY:
+; SM70-NEXT:  // %bb.0:
+; SM70-NEXT:    fence.sc.cta;
+; SM70-NEXT:    ret;
+;
+; SM90-LABEL: fence_seq_cst_cta(
+; SM90:       {
+; SM90-EMPTY:
+; SM90-EMPTY:
+; SM90-NEXT:  // %bb.0:
+; SM90-NEXT:    fence.sc.cta;
+; SM90-NEXT:    ret;
+    fence syncscope("block") seq_cst
+    ret void
+}
+
+
+define void @fence_seq_cst_gpu() {
+; SM30-LABEL: fence_seq_cst_gpu(
+; SM30:       {
+; SM30-EMPTY:
+; SM30-EMPTY:
+; SM30-NEXT:  // %bb.0:
+; SM30-NEXT:    membar.gl;
+; SM30-NEXT:    ret;
+;
+; SM70-LABEL: fence_seq_cst_gpu(
+; SM70:       {
+; SM70-EMPTY:
+; SM70-EMPTY:
+; SM70-NEXT:  // %bb.0:
+; SM70-NEXT:    fence.sc.gpu;
+; SM70-NEXT:    ret;
+;
+; SM90-LABEL: fence_seq_cst_gpu(
+; SM90:       {
+; SM90-EMPTY:
+; SM90-EMPTY:
+; SM90-NEXT:  // %bb.0:
+; SM90-NEXT:    fence.sc.gpu;
+; SM90-NEXT:    ret;
+    fence syncscope("device") seq_cst
+    ret void
+}
+
diff --git a/llvm/test/CodeGen/NVPTX/fence-sm-90.ll b/llvm/test/CodeGen/NVPTX/fence-sm-90.ll
deleted file mode 100644
index dce39bf3e1e3e..0000000000000
--- a/llvm/test/CodeGen/NVPTX/fence-sm-90.ll
+++ /dev/null
@@ -1,30 +0,0 @@
-; RUN: llc < %s -mtriple=nvptx64 -mcpu=sm_90 -mattr=+ptx78 | FileCheck %s
-; RUN: %if ptxas-12.2 %{ llc < %s -mtriple=nvptx64 -mcpu=sm_90 -mattr=+ptx78 | %ptxas-verify -arch=sm_90 %}
-
-; CHECK-LABEL: fence_sc_cluster
-define void @fence_sc_cluster() local_unnamed_addr {
-  ; CHECK: fence.sc.cluster
-  fence syncscope("cluster") seq_cst
-  ret void
-}
-
-; CHECK-LABEL: fence_acq_rel_cluster
-define void @fence_acq_rel_cluster() local_unnamed_addr {
-  ; CHECK: fence.acq_rel.cluster
-  fence syncscope("cluster") acq_rel
-  ret void
-}
-
-; CHECK-LABEL: fence_release_cluster
-define void @fence_release_cluster() local_unnamed_addr {
-  ; CHECK: fence.acq_rel.cluster
-  fence syncscope("cluster") release
-  ret void
-}
-
-; CHECK-LABEL: fence_acquire_cluster
-define void @fence_acquire_cluster() local_unnamed_addr {
-  ; CHECK: fence.acq_rel.cluster
-  fence syncscope("cluster") acquire
-  ret void
-}
diff --git a/llvm/test/CodeGen/NVPTX/fence.ll b/llvm/test/CodeGen/NVPTX/fence.ll
deleted file mode 100644
index e094ddf5775a6..0000000000000
--- a/llvm/test/CodeGen/NVPTX/fence.ll
+++ /dev/null
@@ -1,102 +0,0 @@
-; RUN: llc < %s -mtriple=nvptx64 -mcpu=sm_20 | FileCheck %s --check-prefix=SM60
-; RUN: %if ptxas %{ llc < %s -mtriple=nvptx64 -mcpu=sm_20 | %ptxas-verify %}
-; RUN: llc < %s -mtriple=nvptx64 -mcpu=sm_70 -mattr=+ptx60 | FileCheck %s --check-prefix=SM70
-; RUN: %if ptxas-12.2 %{ llc < %s -mtriple=nvptx64 -mcpu=sm_70 -mattr=+ptx60 | %ptxas-verify -arch=sm_70 %}
-
-; TODO: implement and test thread scope.
-
-; CHECK-LABEL: fence_sc_sys
-define void @fence_sc_sys() local_unnamed_addr {
-  ; SM60: membar.sys
-  ; SM70: fence.sc.sys
-  fence seq_cst
-  ret void
-}
-
-; CHECK-LABEL: fence_acq_rel_sys
-define void @fence_acq_rel_sys() local_unnamed_addr {
-  ; SM60: membar.sys
-  ; SM70: fence.acq_rel.sys
-  fence acq_rel
-  ret void
-}
-
-; CHECK-LABEL: fence_release_sys
-define void @fence_release_sys() local_unnamed_addr {
-  ; SM60: membar.sys
-  ; SM70: fence.acq_rel.sys
-  fence release
-  ret void
-}
-
-; CHECK-LABEL: fence_acquire_sys
-define void @fence_acquire_sys() local_unnamed_addr {
-  ; SM60: membar.sys
-  ; SM70: fence.acq_rel.sys
-  fence acquire
-  ret void
-}
-
-; CHECK-LABEL: fence_sc_gpu
-define void @fence_sc_gpu() local_unnamed_addr {
-  ; SM60: membar.gl
-  ; SM70: fence.sc.gpu
-  fence syncscope("device") seq_cst
-  ret void
-}
-
-; CHECK-LABEL: fence_acq_rel_gpu
-define void @fence_acq_rel_gpu() local_unnamed_addr {
-  ; SM60: membar.gl
-  ; SM70: fence.acq_rel.gpu
-  fence syncscope("device") acq_rel
-  ret void
-}
-
-; CHECK-LABEL: fence_release_gpu
-define void @fence_release_gpu() local_unnamed_addr {
-  ; SM60: membar.gl
-  ; SM70: fence.acq_rel.gpu
-  fence syncscope("device") release
-  ret void
-}
-
-; CHECK-LABEL: fence_acquire_gpu
-define void @fence_acquire_gpu() local_unnamed_addr {
-  ; SM60: membar.gl
-  ; SM70: fence.acq_rel.gpu
-  fence syncscope("device") acquire
-  ret void
-}
-
-; CHECK-LABEL: fence_sc_cta
-define void @fence_sc_cta() local_unnamed_addr {
-  ; SM60: membar.cta
-  ; SM70: fence.sc.cta
-  fence syncscope("block") seq_cst
-  ret void
-}
-
-; CHECK-LABEL: fence_acq_rel_cta
-define void @fence_acq_rel_cta() local_unnamed_addr {
-  ; SM60: membar.cta
-  ; SM70: fence.acq_rel.cta
-  fence syncscope("block") acq_rel
-  ret void
-}
-
-; CHECK-LABEL: fence_release_cta
-define void @fence_release_cta() local_unnamed_addr {
-  ; SM60: membar.cta
-  ; SM70: fence.acq_rel.cta
-  fence syncscope("block") release
-  ret void
-}
-
-; CHECK-LABEL: fence_acquire_cta
-define void @fence_acquire_cta() local_unnamed_addr {
-  ; SM60: membar.cta
-  ; SM70: fence.acq_rel.cta
-  fence syncscope("block") acquire
-  ret void
-}
\ No newline at end of file
diff --git a/llvm/test/CodeGen/NVPTX/fence.py b/llvm/test/CodeGen/NVPTX/fence.py
new file mode 100644
index 0000000000000..b9f9d294e6fe8
--- /dev/null
+++ b/llvm/test/CodeGen/NVPTX/fence.py
@@ -0,0 +1,56 @@
+# For manual usage, not as a part of lit tests. Used for generating the following tests:
+# fence-sm30.ll, fence-sm70.ll, fence-sm90.ll
+
+from string import Template
+from itertools import product
+
+fence_func = Template(
+    """
+define void @fence_${ordering}_${ptx_scope}() {
+    fence syncscope(\"${llvm_scope}\") ${ordering}
+    ret void
+}
+"""
+)
+
+run_statement = Template(
+    """; RUN: llc < %s -march=nvptx64 -mcpu=sm_${sm} -mattr=+ptx${ptx} | FileCheck %s --check-prefix=SM${sm}
+; RUN: %if ptxas %{ llc < %s -march=nvptx -mcpu=sm_${sm} -mattr=+ptx${ptx} | %ptxas-verify %}"""
+)
+
+# (sm, ptx)
+TESTS = [(30, 50), (70, 60), (90, 87)]
+
+LLVM_SCOPES_NO_CLUSTER = ["", "block", "device"]
+
+SCOPE_LLVM_TO_PTX = {"": "sys", "block": "cta", "cluster": "cluster", "device": "gpu"}
+
+ORDERINGS = ["acquire", "release", "acq_rel", "seq_cst"]
+
+if __name__ == "__main__":
+    # non-cluster orderings are supported on SM30, SM70 and SM90
+    with open("fence-nocluster.ll", "w") as fp:
+        for sm, ptx in TESTS:
+            print(run_statement.substitute(sm=sm, ptx=ptx), file=fp)
+        for ordering, llvm_scope in product(ORDERINGS, LLVM_SCOPES_NO_CLUSTER):
+            print(
+                fence_func.substitute(
+                    llvm_scope=llvm_scope,
+                    ptx_scope=SCOPE_LLVM_TO_PTX[llvm_scope],
+                    ordering=ordering,
+                ),
+                file=fp,
+            )
+
+    # cluster ordering only supported on SM90
+    with open("fence-cluster.ll", "w") as fp:
+        print(run_statement.substitute(sm=90, ptx=87), file=fp)
+        for ordering in ORDERINGS:
+            print(
+                fence_func.substitute(
+                    llvm_scope="cluster",
+                    ptx_scope=SCOPE_LLVM_TO_PTX["cluster"],
+                    ordering=ordering,
+                ),
+                file=fp,
+            )
diff --git a/llvm/test/CodeGen/NVPTX/global-ctor.ll b/llvm/test/CodeGen/NVPTX/global-ctor.ll
deleted file mode 100644
index 6a833128206ce..0000000000000
--- a/llvm/test/CodeGen/NVPTX/global-ctor.ll
+++ /dev/null
@@ -1,9 +0,0 @@
-; RUN: not --crash llc < %s -mtriple=nvptx -mcpu=sm_20 2>&1 | FileCheck %s
-
-; Check that llc dies when given a nonempty global ctor.
-@llvm.global_ctors = appending global [1 x { i32, ptr, ptr }] [{ i32, ptr, ptr } { i32 65535, ptr @foo, ptr null }]
-
-; CHECK: ERROR: Module has a nontrivial global ctor
-define internal void @foo() {
-  ret void
-}
diff --git a/llvm/test/CodeGen/NVPTX/global-dtor.ll b/llvm/test/CodeGen/NVPTX/global-dtor.ll
deleted file mode 100644
index f385d620bba36..0000000000000
--- a/llvm/test/CodeGen/NVPTX/global-dtor.ll
+++ /dev/null
@@ -1,9 +0,0 @@
-; RUN: not --crash llc < %s -mtriple=nvptx -mcpu=sm_20 2>&1 | FileCheck %s
-
-; Check that llc dies when given a nonempty global dtor.
-@llvm.global_dtors = appending global [1 x { i32, ptr, ptr }] [{ i32, ptr, ptr } { i32 65535, ptr @foo, ptr null }]
-
-; CHECK: ERROR: Module has a nontrivial global dtor
-define internal void @foo() {
-  ret void
-}
diff --git a/llvm/test/CodeGen/NVPTX/griddepcontrol.ll b/llvm/test/CodeGen/NVPTX/griddepcontrol.ll
index fe15b3fe4afbd..0bf9196aa2902 100644
--- a/llvm/test/CodeGen/NVPTX/griddepcontrol.ll
+++ b/llvm/test/CodeGen/NVPTX/griddepcontrol.ll
@@ -1,6 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
 ; RUN: llc < %s -mcpu=sm_90 -march=nvptx64 | FileCheck %s
-; RUN: %if ptxas-11.8 %{ llc < %s -mcpu=sm_90 -march=nvptx64 | %ptxas-verify %}
+; RUN: %if ptxas-11.8 %{ llc < %s -mcpu=sm_90 -march=nvptx64 | %ptxas-verify -arch=sm_90 %}
 
 define void @griddepcontrol() {
 ; CHECK-LABEL: griddepcontrol(
diff --git a/llvm/test/CodeGen/NVPTX/inline-asm-b128-test1.ll b/llvm/test/CodeGen/NVPTX/inline-asm-b128-test1.ll
index 4ebeba06032c6..eaf6cf59dd066 100644
--- a/llvm/test/CodeGen/NVPTX/inline-asm-b128-test1.ll
+++ b/llvm/test/CodeGen/NVPTX/inline-asm-b128-test1.ll
@@ -1,6 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --extra_scrub --version 5
 ; RUN: llc < %s -mtriple=nvptx64 -mcpu=sm_70 -mattr=+ptx83 | FileCheck %s
-; RUN: %if ptxas %{ llc < %s -mtriple=nvptx64 -mcpu=sm_70 -mattr=+ptx83 | %ptxas-verify -arch=sm_70 %}
+; RUN: %if ptxas-12.3 %{ llc < %s -mtriple=nvptx64 -mcpu=sm_70 -mattr=+ptx83 | %ptxas-verify -arch=sm_70 %}
 
 target triple = "nvptx64-nvidia-cuda"
 
diff --git a/llvm/test/CodeGen/NVPTX/inline-asm-b128-test2.ll b/llvm/test/CodeGen/NVPTX/inline-asm-b128-test2.ll
index d5f3d83573e29..8441c30e4c4d4 100644
--- a/llvm/test/CodeGen/NVPTX/inline-asm-b128-test2.ll
+++ b/llvm/test/CodeGen/NVPTX/inline-asm-b128-test2.ll
@@ -1,6 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
 ; RUN: llc < %s -mtriple=nvptx64 -mcpu=sm_70 -mattr=+ptx83 | FileCheck %s
-; RUN: %if ptxas %{ llc < %s -mtriple=nvptx64 -mcpu=sm_70 -mattr=+ptx83 | %ptxas-verify -arch=sm_70 %}
+; RUN: %if ptxas-12.3 %{ llc < %s -mtriple=nvptx64 -mcpu=sm_70 -mattr=+ptx83 | %ptxas-verify -arch=sm_70 %}
 
 target triple = "nvptx64-nvidia-cuda"
 
diff --git a/llvm/test/CodeGen/NVPTX/inline-asm-b128-test3.ll b/llvm/test/CodeGen/NVPTX/inline-asm-b128-test3.ll
index a3d40169c4695..39b8bf87e9fc5 100644
--- a/llvm/test/CodeGen/NVPTX/inline-asm-b128-test3.ll
+++ b/llvm/test/CodeGen/NVPTX/inline-asm-b128-test3.ll
@@ -1,6 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --extra_scrub --version 5
 ; RUN: llc < %s -mtriple=nvptx64 -mcpu=sm_70 -mattr=+ptx83 | FileCheck %s
-; RUN: %if ptxas %{ llc < %s -mtriple=nvptx64 -mcpu=sm_70 -mattr=+ptx83 | %ptxas-verify -arch=sm_70 %}
+; RUN: %if ptxas-12.3 %{ llc < %s -mtriple=nvptx64 -mcpu=sm_70 -mattr=+ptx83 | %ptxas-verify -arch=sm_70 %}
 
 target triple = "nvptx64-nvidia-cuda"
 
diff --git a/llvm/test/CodeGen/NVPTX/kernel-param-align.ll b/llvm/test/CodeGen/NVPTX/kernel-param-align.ll
index 2889d2d89a857..a56b85de80143 100644
--- a/llvm/test/CodeGen/NVPTX/kernel-param-align.ll
+++ b/llvm/test/CodeGen/NVPTX/kernel-param-align.ll
@@ -1,5 +1,5 @@
 ; RUN: llc < %s -mtriple=nvptx64 -mcpu=sm_60 | FileCheck %s
-; RUN: %if ptxas %{ llc < %s -mtriple=nvptx64 -mcpu=sm_60 | %ptxas-verify %}
+; RUN: %if ptxas %{ llc < %s -mtriple=nvptx64 -mcpu=sm_60 | %ptxas -arch=sm_60 - %}
 
 %struct.Large = type { [16 x double] }
 
diff --git a/llvm/test/CodeGen/NVPTX/lit.local.cfg b/llvm/test/CodeGen/NVPTX/lit.local.cfg
index e3f06d1a720e3..54a6c338bdf85 100644
--- a/llvm/test/CodeGen/NVPTX/lit.local.cfg
+++ b/llvm/test/CodeGen/NVPTX/lit.local.cfg
@@ -1,3 +1,4 @@
 if not "NVPTX" in config.root.targets:
     config.unsupported = True
 config.suffixes.add(".py")
+config.excludes = ["fence.py"]
diff --git a/llvm/test/CodeGen/NVPTX/lower-args-gridconstant.ll b/llvm/test/CodeGen/NVPTX/lower-args-gridconstant.ll
index 208d4f0ef32ae..28be5d7adbf8a 100644
--- a/llvm/test/CodeGen/NVPTX/lower-args-gridconstant.ll
+++ b/llvm/test/CodeGen/NVPTX/lower-args-gridconstant.ll
@@ -10,7 +10,7 @@
 ; Regular functions mus still make a copy. `cvta.param` does not always work there.
 define dso_local noundef i32 @non_kernel_function(ptr nocapture noundef readonly byval(%struct.uint4) align 16 %a, i1 noundef zeroext %b, i32 noundef %c) local_unnamed_addr #0 {
 ; OPT-LABEL: define dso_local noundef i32 @non_kernel_function(
-; OPT-SAME: ptr nocapture noundef readonly byval([[STRUCT_UINT4:%.*]]) align 16 [[A:%.*]], i1 noundef zeroext [[B:%.*]], i32 noundef [[C:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
+; OPT-SAME: ptr noundef readonly byval([[STRUCT_UINT4:%.*]]) align 16 captures(none) [[A:%.*]], i1 noundef zeroext [[B:%.*]], i32 noundef [[C:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
 ; OPT-NEXT:  [[ENTRY:.*:]]
 ; OPT-NEXT:    [[A1:%.*]] = alloca [[STRUCT_UINT4]], align 16
 ; OPT-NEXT:    [[A2:%.*]] = addrspacecast ptr [[A]] to ptr addrspace(101)
@@ -90,7 +90,6 @@ define ptx_kernel void @grid_const_int(ptr byval(i32) align 4 %input1, i32 %inpu
 ; OPT-NEXT:    [[ADD:%.*]] = add i32 [[TMP]], [[INPUT2]]
 ; OPT-NEXT:    store i32 [[ADD]], ptr [[OUT3]], align 4
 ; OPT-NEXT:    ret void
-;
   %tmp = load i32, ptr %input1, align 4
   %add = add i32 %tmp, %input2
   store i32 %add, ptr %out
@@ -125,7 +124,6 @@ define ptx_kernel void @grid_const_struct(ptr byval(%struct.s) align 4 %input, p
 ; OPT-NEXT:    [[ADD:%.*]] = add i32 [[TMP1]], [[TMP2]]
 ; OPT-NEXT:    store i32 [[ADD]], ptr [[OUT5]], align 4
 ; OPT-NEXT:    ret void
-;
   %gep1 = getelementptr inbounds %struct.s, ptr %input, i32 0, i32 0
   %gep2 = getelementptr inbounds %struct.s, ptr %input, i32 0, i32 1
   %int1 = load i32, ptr %gep1
@@ -166,7 +164,6 @@ define ptx_kernel void @grid_const_escape(ptr byval(%struct.s) align 4 %input) {
 ; OPT-NEXT:    [[INPUT_PARAM_GEN:%.*]] = call ptr @llvm.nvvm.ptr.param.to.gen.p0.p101(ptr addrspace(101) [[INPUT_PARAM]])
 ; OPT-NEXT:    [[CALL:%.*]] = call i32 @escape(ptr [[INPUT_PARAM_GEN]])
 ; OPT-NEXT:    ret void
-;
   %call = call i32 @escape(ptr %input)
   ret void
 }
@@ -224,7 +221,6 @@ define ptx_kernel void @multiple_grid_const_escape(ptr byval(%struct.s) align 4
 ; OPT-NEXT:    store i32 [[A]], ptr [[A_ADDR]], align 4
 ; OPT-NEXT:    [[CALL:%.*]] = call i32 @escape3(ptr [[INPUT_PARAM_GEN]], ptr [[A_ADDR]], ptr [[B_PARAM_GEN]])
 ; OPT-NEXT:    ret void
-;
   %a.addr = alloca i32, align 4
   store i32 %a, ptr %a.addr, align 4
   %call = call i32 @escape3(ptr %input, ptr %a.addr, ptr %b)
@@ -252,7 +248,6 @@ define ptx_kernel void @grid_const_memory_escape(ptr byval(%struct.s) align 4 %i
 ; OPT-NEXT:    [[INPUT1:%.*]] = call ptr @llvm.nvvm.ptr.param.to.gen.p0.p101(ptr addrspace(101) [[INPUT_PARAM]])
 ; OPT-NEXT:    store ptr [[INPUT1]], ptr [[ADDR5]], align 8
 ; OPT-NEXT:    ret void
-;
   store ptr %input, ptr %addr, align 8
   ret void
 }
@@ -286,7 +281,6 @@ define ptx_kernel void @grid_const_inlineasm_escape(ptr byval(%struct.s) align 4
 ; OPT-NEXT:    [[TMP2:%.*]] = call i64 asm "add.s64 $0, $1, $2
 ; OPT-NEXT:    store i64 [[TMP2]], ptr [[RESULT5]], align 8
 ; OPT-NEXT:    ret void
-;
   %tmpptr1 = getelementptr inbounds %struct.s, ptr %input, i32 0, i32 0
   %tmpptr2 = getelementptr inbounds %struct.s, ptr %input, i32 0, i32 1
   %1 = call i64 asm "add.s64 $0, $1, $2;", "=l,l,l"(ptr %tmpptr1, ptr %tmpptr2) #1
@@ -335,7 +329,6 @@ define ptx_kernel void @grid_const_partial_escape(ptr byval(i32) %input, ptr %ou
 ; OPT-NEXT:    store i32 [[TWICE]], ptr [[OUTPUT5]], align 4
 ; OPT-NEXT:    [[CALL:%.*]] = call i32 @escape(ptr [[INPUT1_GEN]])
 ; OPT-NEXT:    ret void
-;
   %val = load i32, ptr %input
   %twice = add i32 %val, %val
   store i32 %twice, ptr %output
@@ -389,7 +382,6 @@ define ptx_kernel i32 @grid_const_partial_escapemem(ptr byval(%struct.s) %input,
 ; OPT-NEXT:    [[ADD:%.*]] = add i32 [[VAL1]], [[VAL2]]
 ; OPT-NEXT:    [[CALL2:%.*]] = call i32 @escape(ptr [[PTR1]])
 ; OPT-NEXT:    ret i32 [[ADD]]
-;
   %ptr1 = getelementptr inbounds %struct.s, ptr %input, i32 0, i32 0
   %val1 = load i32, ptr %ptr1
   %ptr2 = getelementptr inbounds %struct.s, ptr %input, i32 0, i32 1
@@ -442,7 +434,6 @@ define ptx_kernel void @grid_const_phi(ptr byval(%struct.s) align 4 %input1, ptr
 ; OPT-NEXT:    [[VALLOADED:%.*]] = load i32, ptr [[PTRNEW]], align 4
 ; OPT-NEXT:    store i32 [[VALLOADED]], ptr [[INOUT2]], align 4
 ; OPT-NEXT:    ret void
-;
 
   %val = load i32, ptr %inout
   %less = icmp slt i32 %val, 0
@@ -508,7 +499,6 @@ define ptx_kernel void @grid_const_phi_ngc(ptr byval(%struct.s) align 4 %input1,
 ; OPT-NEXT:    [[VALLOADED:%.*]] = load i32, ptr [[PTRNEW]], align 4
 ; OPT-NEXT:    store i32 [[VALLOADED]], ptr [[INOUT2]], align 4
 ; OPT-NEXT:    ret void
-;
   %val = load i32, ptr %inout
   %less = icmp slt i32 %val, 0
   br i1 %less, label %first, label %second
@@ -562,7 +552,6 @@ define ptx_kernel void @grid_const_select(ptr byval(i32) align 4 %input1, ptr by
 ; OPT-NEXT:    [[VALLOADED:%.*]] = load i32, ptr [[PTRNEW]], align 4
 ; OPT-NEXT:    store i32 [[VALLOADED]], ptr [[INOUT2]], align 4
 ; OPT-NEXT:    ret void
-;
   %val = load i32, ptr %inout
   %less = icmp slt i32 %val, 0
   %ptrnew = select i1 %less, ptr %input1, ptr %input2
@@ -594,7 +583,6 @@ define ptx_kernel i32 @grid_const_ptrtoint(ptr byval(i32) %input) {
 ; OPT-NEXT:    [[PTRVAL:%.*]] = ptrtoint ptr [[INPUT1]] to i32
 ; OPT-NEXT:    [[KEEPALIVE:%.*]] = add i32 [[INPUT3]], [[PTRVAL]]
 ; OPT-NEXT:    ret i32 [[KEEPALIVE]]
-;
   %val = load i32, ptr %input
   %ptrval = ptrtoint ptr %input to i32
   %keepalive = add i32 %val, %ptrval
diff --git a/llvm/test/CodeGen/NVPTX/lower-args.ll b/llvm/test/CodeGen/NVPTX/lower-args.ll
index 269bba75dc5fb..81b86c86d40de 100644
--- a/llvm/test/CodeGen/NVPTX/lower-args.ll
+++ b/llvm/test/CodeGen/NVPTX/lower-args.ll
@@ -87,12 +87,12 @@ define ptx_kernel void @ptr_generic(ptr %out, ptr %in) {
 }
 
 ; COMMON-LABEL: ptr_nongeneric
-define ptx_kernel void @ptr_nongeneric(ptr addrspace(1) %out, ptr addrspace(4) %in) {
+define ptx_kernel void @ptr_nongeneric(ptr addrspace(1) %out, ptr addrspace(3) %in) {
 ; IR-NOT: addrspacecast
 ; PTX-NOT: cvta.to.global
-; PTX:  ld.const.u32
+; PTX:  ld.shared.u32
 ; PTX   st.global.u32
-  %v = load i32, ptr addrspace(4) %in, align 4
+  %v = load i32, ptr addrspace(3) %in, align 4
   store i32 %v, ptr addrspace(1) %out, align 4
   ret void
 }
diff --git a/llvm/test/CodeGen/NVPTX/lower-byval-args.ll b/llvm/test/CodeGen/NVPTX/lower-byval-args.ll
index 26102722a483b..a09fb35d2d546 100644
--- a/llvm/test/CodeGen/NVPTX/lower-byval-args.ll
+++ b/llvm/test/CodeGen/NVPTX/lower-byval-args.ll
@@ -26,7 +26,7 @@ declare void @llvm.memset.p0.i64(ptr nocapture writeonly, i8, i64, i1 immarg) #2
 ; Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: readwrite)
 define dso_local ptx_kernel void @read_only(ptr nocapture noundef writeonly %out, ptr nocapture noundef readonly byval(%struct.S) align 4 %s) local_unnamed_addr #0 {
 ; SM_60-LABEL: define dso_local ptx_kernel void @read_only(
-; SM_60-SAME: ptr nocapture noundef writeonly [[OUT:%.*]], ptr nocapture noundef readonly byval([[STRUCT_S:%.*]]) align 4 [[S:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
+; SM_60-SAME: ptr noundef writeonly captures(none) [[OUT:%.*]], ptr noundef readonly byval([[STRUCT_S:%.*]]) align 4 captures(none) [[S:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
 ; SM_60-NEXT:  [[ENTRY:.*:]]
 ; SM_60-NEXT:    [[S3:%.*]] = addrspacecast ptr [[S]] to ptr addrspace(101)
 ; SM_60-NEXT:    [[OUT1:%.*]] = addrspacecast ptr [[OUT]] to ptr addrspace(1)
@@ -36,7 +36,7 @@ define dso_local ptx_kernel void @read_only(ptr nocapture noundef writeonly %out
 ; SM_60-NEXT:    ret void
 ;
 ; SM_70-LABEL: define dso_local ptx_kernel void @read_only(
-; SM_70-SAME: ptr nocapture noundef writeonly [[OUT:%.*]], ptr nocapture noundef readonly byval([[STRUCT_S:%.*]]) align 4 [[S:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
+; SM_70-SAME: ptr noundef writeonly captures(none) [[OUT:%.*]], ptr noundef readonly byval([[STRUCT_S:%.*]]) align 4 captures(none) [[S:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
 ; SM_70-NEXT:  [[ENTRY:.*:]]
 ; SM_70-NEXT:    [[S3:%.*]] = addrspacecast ptr [[S]] to ptr addrspace(101)
 ; SM_70-NEXT:    [[OUT1:%.*]] = addrspacecast ptr [[OUT]] to ptr addrspace(1)
@@ -46,7 +46,7 @@ define dso_local ptx_kernel void @read_only(ptr nocapture noundef writeonly %out
 ; SM_70-NEXT:    ret void
 ;
 ; COPY-LABEL: define dso_local ptx_kernel void @read_only(
-; COPY-SAME: ptr nocapture noundef writeonly [[OUT:%.*]], ptr nocapture noundef readonly byval([[STRUCT_S:%.*]]) align 4 [[S:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
+; COPY-SAME: ptr noundef writeonly captures(none) [[OUT:%.*]], ptr noundef readonly byval([[STRUCT_S:%.*]]) align 4 captures(none) [[S:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
 ; COPY-NEXT:  [[ENTRY:.*:]]
 ; COPY-NEXT:    [[S1:%.*]] = alloca [[STRUCT_S]], align 4
 ; COPY-NEXT:    [[S2:%.*]] = addrspacecast ptr [[S]] to ptr addrspace(101)
@@ -64,7 +64,7 @@ entry:
 ; Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: readwrite)
 define dso_local ptx_kernel void @read_only_gep(ptr nocapture noundef writeonly %out, ptr nocapture noundef readonly byval(%struct.S) align 4 %s) local_unnamed_addr #0 {
 ; SM_60-LABEL: define dso_local ptx_kernel void @read_only_gep(
-; SM_60-SAME: ptr nocapture noundef writeonly [[OUT:%.*]], ptr nocapture noundef readonly byval([[STRUCT_S:%.*]]) align 4 [[S:%.*]]) local_unnamed_addr #[[ATTR0]] {
+; SM_60-SAME: ptr noundef writeonly captures(none) [[OUT:%.*]], ptr noundef readonly byval([[STRUCT_S:%.*]]) align 4 captures(none) [[S:%.*]]) local_unnamed_addr #[[ATTR0]] {
 ; SM_60-NEXT:  [[ENTRY:.*:]]
 ; SM_60-NEXT:    [[S3:%.*]] = addrspacecast ptr [[S]] to ptr addrspace(101)
 ; SM_60-NEXT:    [[OUT1:%.*]] = addrspacecast ptr [[OUT]] to ptr addrspace(1)
@@ -75,7 +75,7 @@ define dso_local ptx_kernel void @read_only_gep(ptr nocapture noundef writeonly
 ; SM_60-NEXT:    ret void
 ;
 ; SM_70-LABEL: define dso_local ptx_kernel void @read_only_gep(
-; SM_70-SAME: ptr nocapture noundef writeonly [[OUT:%.*]], ptr nocapture noundef readonly byval([[STRUCT_S:%.*]]) align 4 [[S:%.*]]) local_unnamed_addr #[[ATTR0]] {
+; SM_70-SAME: ptr noundef writeonly captures(none) [[OUT:%.*]], ptr noundef readonly byval([[STRUCT_S:%.*]]) align 4 captures(none) [[S:%.*]]) local_unnamed_addr #[[ATTR0]] {
 ; SM_70-NEXT:  [[ENTRY:.*:]]
 ; SM_70-NEXT:    [[S3:%.*]] = addrspacecast ptr [[S]] to ptr addrspace(101)
 ; SM_70-NEXT:    [[OUT1:%.*]] = addrspacecast ptr [[OUT]] to ptr addrspace(1)
@@ -86,7 +86,7 @@ define dso_local ptx_kernel void @read_only_gep(ptr nocapture noundef writeonly
 ; SM_70-NEXT:    ret void
 ;
 ; COPY-LABEL: define dso_local ptx_kernel void @read_only_gep(
-; COPY-SAME: ptr nocapture noundef writeonly [[OUT:%.*]], ptr nocapture noundef readonly byval([[STRUCT_S:%.*]]) align 4 [[S:%.*]]) local_unnamed_addr #[[ATTR0]] {
+; COPY-SAME: ptr noundef writeonly captures(none) [[OUT:%.*]], ptr noundef readonly byval([[STRUCT_S:%.*]]) align 4 captures(none) [[S:%.*]]) local_unnamed_addr #[[ATTR0]] {
 ; COPY-NEXT:  [[ENTRY:.*:]]
 ; COPY-NEXT:    [[S1:%.*]] = alloca [[STRUCT_S]], align 4
 ; COPY-NEXT:    [[S2:%.*]] = addrspacecast ptr [[S]] to ptr addrspace(101)
@@ -106,7 +106,7 @@ entry:
 ; Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: readwrite)
 define dso_local ptx_kernel void @read_only_gep_asc(ptr nocapture noundef writeonly %out, ptr nocapture noundef readonly byval(%struct.S) align 4 %s) local_unnamed_addr #0 {
 ; SM_60-LABEL: define dso_local ptx_kernel void @read_only_gep_asc(
-; SM_60-SAME: ptr nocapture noundef writeonly [[OUT:%.*]], ptr nocapture noundef readonly byval([[STRUCT_S:%.*]]) align 4 [[S:%.*]]) local_unnamed_addr #[[ATTR0]] {
+; SM_60-SAME: ptr noundef writeonly captures(none) [[OUT:%.*]], ptr noundef readonly byval([[STRUCT_S:%.*]]) align 4 captures(none) [[S:%.*]]) local_unnamed_addr #[[ATTR0]] {
 ; SM_60-NEXT:  [[ENTRY:.*:]]
 ; SM_60-NEXT:    [[S3:%.*]] = addrspacecast ptr [[S]] to ptr addrspace(101)
 ; SM_60-NEXT:    [[OUT1:%.*]] = addrspacecast ptr [[OUT]] to ptr addrspace(1)
@@ -117,7 +117,7 @@ define dso_local ptx_kernel void @read_only_gep_asc(ptr nocapture noundef writeo
 ; SM_60-NEXT:    ret void
 ;
 ; SM_70-LABEL: define dso_local ptx_kernel void @read_only_gep_asc(
-; SM_70-SAME: ptr nocapture noundef writeonly [[OUT:%.*]], ptr nocapture noundef readonly byval([[STRUCT_S:%.*]]) align 4 [[S:%.*]]) local_unnamed_addr #[[ATTR0]] {
+; SM_70-SAME: ptr noundef writeonly captures(none) [[OUT:%.*]], ptr noundef readonly byval([[STRUCT_S:%.*]]) align 4 captures(none) [[S:%.*]]) local_unnamed_addr #[[ATTR0]] {
 ; SM_70-NEXT:  [[ENTRY:.*:]]
 ; SM_70-NEXT:    [[S3:%.*]] = addrspacecast ptr [[S]] to ptr addrspace(101)
 ; SM_70-NEXT:    [[OUT1:%.*]] = addrspacecast ptr [[OUT]] to ptr addrspace(1)
@@ -128,7 +128,7 @@ define dso_local ptx_kernel void @read_only_gep_asc(ptr nocapture noundef writeo
 ; SM_70-NEXT:    ret void
 ;
 ; COPY-LABEL: define dso_local ptx_kernel void @read_only_gep_asc(
-; COPY-SAME: ptr nocapture noundef writeonly [[OUT:%.*]], ptr nocapture noundef readonly byval([[STRUCT_S:%.*]]) align 4 [[S:%.*]]) local_unnamed_addr #[[ATTR0]] {
+; COPY-SAME: ptr noundef writeonly captures(none) [[OUT:%.*]], ptr noundef readonly byval([[STRUCT_S:%.*]]) align 4 captures(none) [[S:%.*]]) local_unnamed_addr #[[ATTR0]] {
 ; COPY-NEXT:  [[ENTRY:.*:]]
 ; COPY-NEXT:    [[S1:%.*]] = alloca [[STRUCT_S]], align 4
 ; COPY-NEXT:    [[S2:%.*]] = addrspacecast ptr [[S]] to ptr addrspace(101)
@@ -150,7 +150,7 @@ entry:
 ; Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: readwrite)
 define dso_local ptx_kernel void @read_only_gep_asc0(ptr nocapture noundef writeonly %out, ptr nocapture noundef readonly byval(%struct.S) align 4 %s) local_unnamed_addr #0 {
 ; SM_60-LABEL: define dso_local ptx_kernel void @read_only_gep_asc0(
-; SM_60-SAME: ptr nocapture noundef writeonly [[OUT:%.*]], ptr nocapture noundef readonly byval([[STRUCT_S:%.*]]) align 4 [[S:%.*]]) local_unnamed_addr #[[ATTR0]] {
+; SM_60-SAME: ptr noundef writeonly captures(none) [[OUT:%.*]], ptr noundef readonly byval([[STRUCT_S:%.*]]) align 4 captures(none) [[S:%.*]]) local_unnamed_addr #[[ATTR0]] {
 ; SM_60-NEXT:  [[ENTRY:.*:]]
 ; SM_60-NEXT:    [[S3:%.*]] = alloca [[STRUCT_S]], align 4
 ; SM_60-NEXT:    [[S4:%.*]] = addrspacecast ptr [[S]] to ptr addrspace(101)
@@ -165,7 +165,7 @@ define dso_local ptx_kernel void @read_only_gep_asc0(ptr nocapture noundef write
 ; SM_60-NEXT:    ret void
 ;
 ; SM_70-LABEL: define dso_local ptx_kernel void @read_only_gep_asc0(
-; SM_70-SAME: ptr nocapture noundef writeonly [[OUT:%.*]], ptr nocapture noundef readonly byval([[STRUCT_S:%.*]]) align 4 [[S:%.*]]) local_unnamed_addr #[[ATTR0]] {
+; SM_70-SAME: ptr noundef writeonly captures(none) [[OUT:%.*]], ptr noundef readonly byval([[STRUCT_S:%.*]]) align 4 captures(none) [[S:%.*]]) local_unnamed_addr #[[ATTR0]] {
 ; SM_70-NEXT:  [[ENTRY:.*:]]
 ; SM_70-NEXT:    [[S3:%.*]] = alloca [[STRUCT_S]], align 4
 ; SM_70-NEXT:    [[S4:%.*]] = addrspacecast ptr [[S]] to ptr addrspace(101)
@@ -180,7 +180,7 @@ define dso_local ptx_kernel void @read_only_gep_asc0(ptr nocapture noundef write
 ; SM_70-NEXT:    ret void
 ;
 ; COPY-LABEL: define dso_local ptx_kernel void @read_only_gep_asc0(
-; COPY-SAME: ptr nocapture noundef writeonly [[OUT:%.*]], ptr nocapture noundef readonly byval([[STRUCT_S:%.*]]) align 4 [[S:%.*]]) local_unnamed_addr #[[ATTR0]] {
+; COPY-SAME: ptr noundef writeonly captures(none) [[OUT:%.*]], ptr noundef readonly byval([[STRUCT_S:%.*]]) align 4 captures(none) [[S:%.*]]) local_unnamed_addr #[[ATTR0]] {
 ; COPY-NEXT:  [[ENTRY:.*:]]
 ; COPY-NEXT:    [[S1:%.*]] = alloca [[STRUCT_S]], align 4
 ; COPY-NEXT:    [[S2:%.*]] = addrspacecast ptr [[S]] to ptr addrspace(101)
@@ -204,7 +204,7 @@ entry:
 ; Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: readwrite)
 define dso_local ptx_kernel void @escape_ptr(ptr nocapture noundef readnone %out, ptr noundef byval(%struct.S) align 4 %s) local_unnamed_addr #0 {
 ; SM_60-LABEL: define dso_local ptx_kernel void @escape_ptr(
-; SM_60-SAME: ptr nocapture noundef readnone [[OUT:%.*]], ptr noundef byval([[STRUCT_S:%.*]]) align 4 [[S:%.*]]) local_unnamed_addr #[[ATTR0]] {
+; SM_60-SAME: ptr noundef readnone captures(none) [[OUT:%.*]], ptr noundef byval([[STRUCT_S:%.*]]) align 4 [[S:%.*]]) local_unnamed_addr #[[ATTR0]] {
 ; SM_60-NEXT:  [[ENTRY:.*:]]
 ; SM_60-NEXT:    [[S3:%.*]] = alloca [[STRUCT_S]], align 4
 ; SM_60-NEXT:    [[S4:%.*]] = addrspacecast ptr [[S]] to ptr addrspace(101)
@@ -215,7 +215,7 @@ define dso_local ptx_kernel void @escape_ptr(ptr nocapture noundef readnone %out
 ; SM_60-NEXT:    ret void
 ;
 ; SM_70-LABEL: define dso_local ptx_kernel void @escape_ptr(
-; SM_70-SAME: ptr nocapture noundef readnone [[OUT:%.*]], ptr noundef byval([[STRUCT_S:%.*]]) align 4 [[S:%.*]]) local_unnamed_addr #[[ATTR0]] {
+; SM_70-SAME: ptr noundef readnone captures(none) [[OUT:%.*]], ptr noundef byval([[STRUCT_S:%.*]]) align 4 [[S:%.*]]) local_unnamed_addr #[[ATTR0]] {
 ; SM_70-NEXT:  [[ENTRY:.*:]]
 ; SM_70-NEXT:    [[S3:%.*]] = alloca [[STRUCT_S]], align 4
 ; SM_70-NEXT:    [[S4:%.*]] = addrspacecast ptr [[S]] to ptr addrspace(101)
@@ -226,7 +226,7 @@ define dso_local ptx_kernel void @escape_ptr(ptr nocapture noundef readnone %out
 ; SM_70-NEXT:    ret void
 ;
 ; COPY-LABEL: define dso_local ptx_kernel void @escape_ptr(
-; COPY-SAME: ptr nocapture noundef readnone [[OUT:%.*]], ptr noundef byval([[STRUCT_S:%.*]]) align 4 [[S:%.*]]) local_unnamed_addr #[[ATTR0]] {
+; COPY-SAME: ptr noundef readnone captures(none) [[OUT:%.*]], ptr noundef byval([[STRUCT_S:%.*]]) align 4 [[S:%.*]]) local_unnamed_addr #[[ATTR0]] {
 ; COPY-NEXT:  [[ENTRY:.*:]]
 ; COPY-NEXT:    [[S1:%.*]] = alloca [[STRUCT_S]], align 4
 ; COPY-NEXT:    [[S2:%.*]] = addrspacecast ptr [[S]] to ptr addrspace(101)
@@ -242,7 +242,7 @@ entry:
 ; Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: readwrite)
 define dso_local ptx_kernel void @escape_ptr_gep(ptr nocapture noundef readnone %out, ptr noundef byval(%struct.S) align 4 %s) local_unnamed_addr #0 {
 ; SM_60-LABEL: define dso_local ptx_kernel void @escape_ptr_gep(
-; SM_60-SAME: ptr nocapture noundef readnone [[OUT:%.*]], ptr noundef byval([[STRUCT_S:%.*]]) align 4 [[S:%.*]]) local_unnamed_addr #[[ATTR0]] {
+; SM_60-SAME: ptr noundef readnone captures(none) [[OUT:%.*]], ptr noundef byval([[STRUCT_S:%.*]]) align 4 [[S:%.*]]) local_unnamed_addr #[[ATTR0]] {
 ; SM_60-NEXT:  [[ENTRY:.*:]]
 ; SM_60-NEXT:    [[S3:%.*]] = alloca [[STRUCT_S]], align 4
 ; SM_60-NEXT:    [[S4:%.*]] = addrspacecast ptr [[S]] to ptr addrspace(101)
@@ -254,7 +254,7 @@ define dso_local ptx_kernel void @escape_ptr_gep(ptr nocapture noundef readnone
 ; SM_60-NEXT:    ret void
 ;
 ; SM_70-LABEL: define dso_local ptx_kernel void @escape_ptr_gep(
-; SM_70-SAME: ptr nocapture noundef readnone [[OUT:%.*]], ptr noundef byval([[STRUCT_S:%.*]]) align 4 [[S:%.*]]) local_unnamed_addr #[[ATTR0]] {
+; SM_70-SAME: ptr noundef readnone captures(none) [[OUT:%.*]], ptr noundef byval([[STRUCT_S:%.*]]) align 4 [[S:%.*]]) local_unnamed_addr #[[ATTR0]] {
 ; SM_70-NEXT:  [[ENTRY:.*:]]
 ; SM_70-NEXT:    [[S3:%.*]] = alloca [[STRUCT_S]], align 4
 ; SM_70-NEXT:    [[S4:%.*]] = addrspacecast ptr [[S]] to ptr addrspace(101)
@@ -266,7 +266,7 @@ define dso_local ptx_kernel void @escape_ptr_gep(ptr nocapture noundef readnone
 ; SM_70-NEXT:    ret void
 ;
 ; COPY-LABEL: define dso_local ptx_kernel void @escape_ptr_gep(
-; COPY-SAME: ptr nocapture noundef readnone [[OUT:%.*]], ptr noundef byval([[STRUCT_S:%.*]]) align 4 [[S:%.*]]) local_unnamed_addr #[[ATTR0]] {
+; COPY-SAME: ptr noundef readnone captures(none) [[OUT:%.*]], ptr noundef byval([[STRUCT_S:%.*]]) align 4 [[S:%.*]]) local_unnamed_addr #[[ATTR0]] {
 ; COPY-NEXT:  [[ENTRY:.*:]]
 ; COPY-NEXT:    [[S1:%.*]] = alloca [[STRUCT_S]], align 4
 ; COPY-NEXT:    [[S2:%.*]] = addrspacecast ptr [[S]] to ptr addrspace(101)
@@ -284,7 +284,7 @@ entry:
 ; Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: readwrite)
 define dso_local ptx_kernel void @escape_ptr_store(ptr nocapture noundef writeonly %out, ptr noundef byval(%struct.S) align 4 %s) local_unnamed_addr #0 {
 ; SM_60-LABEL: define dso_local ptx_kernel void @escape_ptr_store(
-; SM_60-SAME: ptr nocapture noundef writeonly [[OUT:%.*]], ptr noundef byval([[STRUCT_S:%.*]]) align 4 [[S:%.*]]) local_unnamed_addr #[[ATTR0]] {
+; SM_60-SAME: ptr noundef writeonly captures(none) [[OUT:%.*]], ptr noundef byval([[STRUCT_S:%.*]]) align 4 [[S:%.*]]) local_unnamed_addr #[[ATTR0]] {
 ; SM_60-NEXT:  [[ENTRY:.*:]]
 ; SM_60-NEXT:    [[S3:%.*]] = alloca [[STRUCT_S]], align 4
 ; SM_60-NEXT:    [[S4:%.*]] = addrspacecast ptr [[S]] to ptr addrspace(101)
@@ -295,7 +295,7 @@ define dso_local ptx_kernel void @escape_ptr_store(ptr nocapture noundef writeon
 ; SM_60-NEXT:    ret void
 ;
 ; SM_70-LABEL: define dso_local ptx_kernel void @escape_ptr_store(
-; SM_70-SAME: ptr nocapture noundef writeonly [[OUT:%.*]], ptr noundef byval([[STRUCT_S:%.*]]) align 4 [[S:%.*]]) local_unnamed_addr #[[ATTR0]] {
+; SM_70-SAME: ptr noundef writeonly captures(none) [[OUT:%.*]], ptr noundef byval([[STRUCT_S:%.*]]) align 4 [[S:%.*]]) local_unnamed_addr #[[ATTR0]] {
 ; SM_70-NEXT:  [[ENTRY:.*:]]
 ; SM_70-NEXT:    [[S3:%.*]] = alloca [[STRUCT_S]], align 4
 ; SM_70-NEXT:    [[S4:%.*]] = addrspacecast ptr [[S]] to ptr addrspace(101)
@@ -306,7 +306,7 @@ define dso_local ptx_kernel void @escape_ptr_store(ptr nocapture noundef writeon
 ; SM_70-NEXT:    ret void
 ;
 ; COPY-LABEL: define dso_local ptx_kernel void @escape_ptr_store(
-; COPY-SAME: ptr nocapture noundef writeonly [[OUT:%.*]], ptr noundef byval([[STRUCT_S:%.*]]) align 4 [[S:%.*]]) local_unnamed_addr #[[ATTR0]] {
+; COPY-SAME: ptr noundef writeonly captures(none) [[OUT:%.*]], ptr noundef byval([[STRUCT_S:%.*]]) align 4 [[S:%.*]]) local_unnamed_addr #[[ATTR0]] {
 ; COPY-NEXT:  [[ENTRY:.*:]]
 ; COPY-NEXT:    [[S1:%.*]] = alloca [[STRUCT_S]], align 4
 ; COPY-NEXT:    [[S2:%.*]] = addrspacecast ptr [[S]] to ptr addrspace(101)
@@ -322,7 +322,7 @@ entry:
 ; Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: readwrite)
 define dso_local ptx_kernel void @escape_ptr_gep_store(ptr nocapture noundef writeonly %out, ptr noundef byval(%struct.S) align 4 %s) local_unnamed_addr #0 {
 ; SM_60-LABEL: define dso_local ptx_kernel void @escape_ptr_gep_store(
-; SM_60-SAME: ptr nocapture noundef writeonly [[OUT:%.*]], ptr noundef byval([[STRUCT_S:%.*]]) align 4 [[S:%.*]]) local_unnamed_addr #[[ATTR0]] {
+; SM_60-SAME: ptr noundef writeonly captures(none) [[OUT:%.*]], ptr noundef byval([[STRUCT_S:%.*]]) align 4 [[S:%.*]]) local_unnamed_addr #[[ATTR0]] {
 ; SM_60-NEXT:  [[ENTRY:.*:]]
 ; SM_60-NEXT:    [[S3:%.*]] = alloca [[STRUCT_S]], align 4
 ; SM_60-NEXT:    [[S4:%.*]] = addrspacecast ptr [[S]] to ptr addrspace(101)
@@ -334,7 +334,7 @@ define dso_local ptx_kernel void @escape_ptr_gep_store(ptr nocapture noundef wri
 ; SM_60-NEXT:    ret void
 ;
 ; SM_70-LABEL: define dso_local ptx_kernel void @escape_ptr_gep_store(
-; SM_70-SAME: ptr nocapture noundef writeonly [[OUT:%.*]], ptr noundef byval([[STRUCT_S:%.*]]) align 4 [[S:%.*]]) local_unnamed_addr #[[ATTR0]] {
+; SM_70-SAME: ptr noundef writeonly captures(none) [[OUT:%.*]], ptr noundef byval([[STRUCT_S:%.*]]) align 4 [[S:%.*]]) local_unnamed_addr #[[ATTR0]] {
 ; SM_70-NEXT:  [[ENTRY:.*:]]
 ; SM_70-NEXT:    [[S3:%.*]] = alloca [[STRUCT_S]], align 4
 ; SM_70-NEXT:    [[S4:%.*]] = addrspacecast ptr [[S]] to ptr addrspace(101)
@@ -346,7 +346,7 @@ define dso_local ptx_kernel void @escape_ptr_gep_store(ptr nocapture noundef wri
 ; SM_70-NEXT:    ret void
 ;
 ; COPY-LABEL: define dso_local ptx_kernel void @escape_ptr_gep_store(
-; COPY-SAME: ptr nocapture noundef writeonly [[OUT:%.*]], ptr noundef byval([[STRUCT_S:%.*]]) align 4 [[S:%.*]]) local_unnamed_addr #[[ATTR0]] {
+; COPY-SAME: ptr noundef writeonly captures(none) [[OUT:%.*]], ptr noundef byval([[STRUCT_S:%.*]]) align 4 [[S:%.*]]) local_unnamed_addr #[[ATTR0]] {
 ; COPY-NEXT:  [[ENTRY:.*:]]
 ; COPY-NEXT:    [[S1:%.*]] = alloca [[STRUCT_S]], align 4
 ; COPY-NEXT:    [[S2:%.*]] = addrspacecast ptr [[S]] to ptr addrspace(101)
@@ -364,7 +364,7 @@ entry:
 ; Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: readwrite)
 define dso_local ptx_kernel void @escape_ptrtoint(ptr nocapture noundef writeonly %out, ptr noundef byval(%struct.S) align 4 %s) local_unnamed_addr #0 {
 ; SM_60-LABEL: define dso_local ptx_kernel void @escape_ptrtoint(
-; SM_60-SAME: ptr nocapture noundef writeonly [[OUT:%.*]], ptr noundef byval([[STRUCT_S:%.*]]) align 4 [[S:%.*]]) local_unnamed_addr #[[ATTR0]] {
+; SM_60-SAME: ptr noundef writeonly captures(none) [[OUT:%.*]], ptr noundef byval([[STRUCT_S:%.*]]) align 4 [[S:%.*]]) local_unnamed_addr #[[ATTR0]] {
 ; SM_60-NEXT:  [[ENTRY:.*:]]
 ; SM_60-NEXT:    [[S3:%.*]] = alloca [[STRUCT_S]], align 4
 ; SM_60-NEXT:    [[S4:%.*]] = addrspacecast ptr [[S]] to ptr addrspace(101)
@@ -376,7 +376,7 @@ define dso_local ptx_kernel void @escape_ptrtoint(ptr nocapture noundef writeonl
 ; SM_60-NEXT:    ret void
 ;
 ; SM_70-LABEL: define dso_local ptx_kernel void @escape_ptrtoint(
-; SM_70-SAME: ptr nocapture noundef writeonly [[OUT:%.*]], ptr noundef byval([[STRUCT_S:%.*]]) align 4 [[S:%.*]]) local_unnamed_addr #[[ATTR0]] {
+; SM_70-SAME: ptr noundef writeonly captures(none) [[OUT:%.*]], ptr noundef byval([[STRUCT_S:%.*]]) align 4 [[S:%.*]]) local_unnamed_addr #[[ATTR0]] {
 ; SM_70-NEXT:  [[ENTRY:.*:]]
 ; SM_70-NEXT:    [[S3:%.*]] = alloca [[STRUCT_S]], align 4
 ; SM_70-NEXT:    [[S4:%.*]] = addrspacecast ptr [[S]] to ptr addrspace(101)
@@ -388,7 +388,7 @@ define dso_local ptx_kernel void @escape_ptrtoint(ptr nocapture noundef writeonl
 ; SM_70-NEXT:    ret void
 ;
 ; COPY-LABEL: define dso_local ptx_kernel void @escape_ptrtoint(
-; COPY-SAME: ptr nocapture noundef writeonly [[OUT:%.*]], ptr noundef byval([[STRUCT_S:%.*]]) align 4 [[S:%.*]]) local_unnamed_addr #[[ATTR0]] {
+; COPY-SAME: ptr noundef writeonly captures(none) [[OUT:%.*]], ptr noundef byval([[STRUCT_S:%.*]]) align 4 [[S:%.*]]) local_unnamed_addr #[[ATTR0]] {
 ; COPY-NEXT:  [[ENTRY:.*:]]
 ; COPY-NEXT:    [[S1:%.*]] = alloca [[STRUCT_S]], align 4
 ; COPY-NEXT:    [[S2:%.*]] = addrspacecast ptr [[S]] to ptr addrspace(101)
@@ -406,7 +406,7 @@ entry:
 ; Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: readwrite)
 define dso_local ptx_kernel void @memcpy_from_param(ptr nocapture noundef writeonly %out, ptr nocapture noundef readonly byval(%struct.S) align 4 %s) local_unnamed_addr #0 {
 ; SM_60-LABEL: define dso_local ptx_kernel void @memcpy_from_param(
-; SM_60-SAME: ptr nocapture noundef writeonly [[OUT:%.*]], ptr nocapture noundef readonly byval([[STRUCT_S:%.*]]) align 4 [[S:%.*]]) local_unnamed_addr #[[ATTR0]] {
+; SM_60-SAME: ptr noundef writeonly captures(none) [[OUT:%.*]], ptr noundef readonly byval([[STRUCT_S:%.*]]) align 4 captures(none) [[S:%.*]]) local_unnamed_addr #[[ATTR0]] {
 ; SM_60-NEXT:  [[ENTRY:.*:]]
 ; SM_60-NEXT:    [[S3:%.*]] = addrspacecast ptr [[S]] to ptr addrspace(101)
 ; SM_60-NEXT:    [[OUT1:%.*]] = addrspacecast ptr [[OUT]] to ptr addrspace(1)
@@ -415,7 +415,7 @@ define dso_local ptx_kernel void @memcpy_from_param(ptr nocapture noundef writeo
 ; SM_60-NEXT:    ret void
 ;
 ; SM_70-LABEL: define dso_local ptx_kernel void @memcpy_from_param(
-; SM_70-SAME: ptr nocapture noundef writeonly [[OUT:%.*]], ptr nocapture noundef readonly byval([[STRUCT_S:%.*]]) align 4 [[S:%.*]]) local_unnamed_addr #[[ATTR0]] {
+; SM_70-SAME: ptr noundef writeonly captures(none) [[OUT:%.*]], ptr noundef readonly byval([[STRUCT_S:%.*]]) align 4 captures(none) [[S:%.*]]) local_unnamed_addr #[[ATTR0]] {
 ; SM_70-NEXT:  [[ENTRY:.*:]]
 ; SM_70-NEXT:    [[S3:%.*]] = addrspacecast ptr [[S]] to ptr addrspace(101)
 ; SM_70-NEXT:    [[OUT1:%.*]] = addrspacecast ptr [[OUT]] to ptr addrspace(1)
@@ -424,7 +424,7 @@ define dso_local ptx_kernel void @memcpy_from_param(ptr nocapture noundef writeo
 ; SM_70-NEXT:    ret void
 ;
 ; COPY-LABEL: define dso_local ptx_kernel void @memcpy_from_param(
-; COPY-SAME: ptr nocapture noundef writeonly [[OUT:%.*]], ptr nocapture noundef readonly byval([[STRUCT_S:%.*]]) align 4 [[S:%.*]]) local_unnamed_addr #[[ATTR0]] {
+; COPY-SAME: ptr noundef writeonly captures(none) [[OUT:%.*]], ptr noundef readonly byval([[STRUCT_S:%.*]]) align 4 captures(none) [[S:%.*]]) local_unnamed_addr #[[ATTR0]] {
 ; COPY-NEXT:  [[ENTRY:.*:]]
 ; COPY-NEXT:    [[S1:%.*]] = alloca [[STRUCT_S]], align 4
 ; COPY-NEXT:    [[S2:%.*]] = addrspacecast ptr [[S]] to ptr addrspace(101)
@@ -440,7 +440,7 @@ entry:
 ; Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: readwrite)
 define dso_local ptx_kernel void @memcpy_from_param_noalign (ptr nocapture noundef writeonly %out, ptr nocapture noundef readonly byval(%struct.S) %s) local_unnamed_addr #0 {
 ; SM_60-LABEL: define dso_local ptx_kernel void @memcpy_from_param_noalign(
-; SM_60-SAME: ptr nocapture noundef writeonly [[OUT:%.*]], ptr nocapture noundef readonly byval([[STRUCT_S:%.*]]) align 4 [[S:%.*]]) local_unnamed_addr #[[ATTR0]] {
+; SM_60-SAME: ptr noundef writeonly captures(none) [[OUT:%.*]], ptr noundef readonly byval([[STRUCT_S:%.*]]) align 4 captures(none) [[S:%.*]]) local_unnamed_addr #[[ATTR0]] {
 ; SM_60-NEXT:  [[ENTRY:.*:]]
 ; SM_60-NEXT:    [[S3:%.*]] = addrspacecast ptr [[S]] to ptr addrspace(101)
 ; SM_60-NEXT:    [[OUT1:%.*]] = addrspacecast ptr [[OUT]] to ptr addrspace(1)
@@ -449,7 +449,7 @@ define dso_local ptx_kernel void @memcpy_from_param_noalign (ptr nocapture nound
 ; SM_60-NEXT:    ret void
 ;
 ; SM_70-LABEL: define dso_local ptx_kernel void @memcpy_from_param_noalign(
-; SM_70-SAME: ptr nocapture noundef writeonly [[OUT:%.*]], ptr nocapture noundef readonly byval([[STRUCT_S:%.*]]) align 4 [[S:%.*]]) local_unnamed_addr #[[ATTR0]] {
+; SM_70-SAME: ptr noundef writeonly captures(none) [[OUT:%.*]], ptr noundef readonly byval([[STRUCT_S:%.*]]) align 4 captures(none) [[S:%.*]]) local_unnamed_addr #[[ATTR0]] {
 ; SM_70-NEXT:  [[ENTRY:.*:]]
 ; SM_70-NEXT:    [[S3:%.*]] = addrspacecast ptr [[S]] to ptr addrspace(101)
 ; SM_70-NEXT:    [[OUT1:%.*]] = addrspacecast ptr [[OUT]] to ptr addrspace(1)
@@ -458,7 +458,7 @@ define dso_local ptx_kernel void @memcpy_from_param_noalign (ptr nocapture nound
 ; SM_70-NEXT:    ret void
 ;
 ; COPY-LABEL: define dso_local ptx_kernel void @memcpy_from_param_noalign(
-; COPY-SAME: ptr nocapture noundef writeonly [[OUT:%.*]], ptr nocapture noundef readonly byval([[STRUCT_S:%.*]]) [[S:%.*]]) local_unnamed_addr #[[ATTR0]] {
+; COPY-SAME: ptr noundef writeonly captures(none) [[OUT:%.*]], ptr noundef readonly byval([[STRUCT_S:%.*]]) captures(none) [[S:%.*]]) local_unnamed_addr #[[ATTR0]] {
 ; COPY-NEXT:  [[ENTRY:.*:]]
 ; COPY-NEXT:    [[S1:%.*]] = alloca [[STRUCT_S]], align 8
 ; COPY-NEXT:    [[S2:%.*]] = addrspacecast ptr [[S]] to ptr addrspace(101)
@@ -474,7 +474,7 @@ entry:
 ; Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: readwrite)
 define dso_local ptx_kernel void @memcpy_to_param(ptr nocapture noundef readonly %in, ptr nocapture noundef readnone byval(%struct.S) align 4 %s) local_unnamed_addr #0 {
 ; SM_60-LABEL: define dso_local ptx_kernel void @memcpy_to_param(
-; SM_60-SAME: ptr nocapture noundef readonly [[IN:%.*]], ptr nocapture noundef readnone byval([[STRUCT_S:%.*]]) align 4 [[S:%.*]]) local_unnamed_addr #[[ATTR0]] {
+; SM_60-SAME: ptr noundef readonly captures(none) [[IN:%.*]], ptr noundef readnone byval([[STRUCT_S:%.*]]) align 4 captures(none) [[S:%.*]]) local_unnamed_addr #[[ATTR0]] {
 ; SM_60-NEXT:  [[ENTRY:.*:]]
 ; SM_60-NEXT:    [[S3:%.*]] = alloca [[STRUCT_S]], align 4
 ; SM_60-NEXT:    [[S4:%.*]] = addrspacecast ptr [[S]] to ptr addrspace(101)
@@ -485,7 +485,7 @@ define dso_local ptx_kernel void @memcpy_to_param(ptr nocapture noundef readonly
 ; SM_60-NEXT:    ret void
 ;
 ; SM_70-LABEL: define dso_local ptx_kernel void @memcpy_to_param(
-; SM_70-SAME: ptr nocapture noundef readonly [[IN:%.*]], ptr nocapture noundef readnone byval([[STRUCT_S:%.*]]) align 4 [[S:%.*]]) local_unnamed_addr #[[ATTR0]] {
+; SM_70-SAME: ptr noundef readonly captures(none) [[IN:%.*]], ptr noundef readnone byval([[STRUCT_S:%.*]]) align 4 captures(none) [[S:%.*]]) local_unnamed_addr #[[ATTR0]] {
 ; SM_70-NEXT:  [[ENTRY:.*:]]
 ; SM_70-NEXT:    [[S3:%.*]] = alloca [[STRUCT_S]], align 4
 ; SM_70-NEXT:    [[S4:%.*]] = addrspacecast ptr [[S]] to ptr addrspace(101)
@@ -496,7 +496,7 @@ define dso_local ptx_kernel void @memcpy_to_param(ptr nocapture noundef readonly
 ; SM_70-NEXT:    ret void
 ;
 ; COPY-LABEL: define dso_local ptx_kernel void @memcpy_to_param(
-; COPY-SAME: ptr nocapture noundef readonly [[IN:%.*]], ptr nocapture noundef readnone byval([[STRUCT_S:%.*]]) align 4 [[S:%.*]]) local_unnamed_addr #[[ATTR0]] {
+; COPY-SAME: ptr noundef readonly captures(none) [[IN:%.*]], ptr noundef readnone byval([[STRUCT_S:%.*]]) align 4 captures(none) [[S:%.*]]) local_unnamed_addr #[[ATTR0]] {
 ; COPY-NEXT:  [[ENTRY:.*:]]
 ; COPY-NEXT:    [[S1:%.*]] = alloca [[STRUCT_S]], align 4
 ; COPY-NEXT:    [[S2:%.*]] = addrspacecast ptr [[S]] to ptr addrspace(101)
@@ -512,7 +512,7 @@ entry:
 ; Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: readwrite)
 define dso_local ptx_kernel void @copy_on_store(ptr nocapture noundef readonly %in, ptr nocapture noundef byval(%struct.S) align 4 %s, i1 noundef zeroext %b) local_unnamed_addr #0 {
 ; SM_60-LABEL: define dso_local ptx_kernel void @copy_on_store(
-; SM_60-SAME: ptr nocapture noundef readonly [[IN:%.*]], ptr nocapture noundef byval([[STRUCT_S:%.*]]) align 4 [[S:%.*]], i1 noundef zeroext [[B:%.*]]) local_unnamed_addr #[[ATTR0]] {
+; SM_60-SAME: ptr noundef readonly captures(none) [[IN:%.*]], ptr noundef byval([[STRUCT_S:%.*]]) align 4 captures(none) [[S:%.*]], i1 noundef zeroext [[B:%.*]]) local_unnamed_addr #[[ATTR0]] {
 ; SM_60-NEXT:  [[BB:.*:]]
 ; SM_60-NEXT:    [[S3:%.*]] = alloca [[STRUCT_S]], align 4
 ; SM_60-NEXT:    [[S4:%.*]] = addrspacecast ptr [[S]] to ptr addrspace(101)
@@ -524,7 +524,7 @@ define dso_local ptx_kernel void @copy_on_store(ptr nocapture noundef readonly %
 ; SM_60-NEXT:    ret void
 ;
 ; SM_70-LABEL: define dso_local ptx_kernel void @copy_on_store(
-; SM_70-SAME: ptr nocapture noundef readonly [[IN:%.*]], ptr nocapture noundef byval([[STRUCT_S:%.*]]) align 4 [[S:%.*]], i1 noundef zeroext [[B:%.*]]) local_unnamed_addr #[[ATTR0]] {
+; SM_70-SAME: ptr noundef readonly captures(none) [[IN:%.*]], ptr noundef byval([[STRUCT_S:%.*]]) align 4 captures(none) [[S:%.*]], i1 noundef zeroext [[B:%.*]]) local_unnamed_addr #[[ATTR0]] {
 ; SM_70-NEXT:  [[BB:.*:]]
 ; SM_70-NEXT:    [[S3:%.*]] = alloca [[STRUCT_S]], align 4
 ; SM_70-NEXT:    [[S4:%.*]] = addrspacecast ptr [[S]] to ptr addrspace(101)
@@ -536,7 +536,7 @@ define dso_local ptx_kernel void @copy_on_store(ptr nocapture noundef readonly %
 ; SM_70-NEXT:    ret void
 ;
 ; COPY-LABEL: define dso_local ptx_kernel void @copy_on_store(
-; COPY-SAME: ptr nocapture noundef readonly [[IN:%.*]], ptr nocapture noundef byval([[STRUCT_S:%.*]]) align 4 [[S:%.*]], i1 noundef zeroext [[B:%.*]]) local_unnamed_addr #[[ATTR0]] {
+; COPY-SAME: ptr noundef readonly captures(none) [[IN:%.*]], ptr noundef byval([[STRUCT_S:%.*]]) align 4 captures(none) [[S:%.*]], i1 noundef zeroext [[B:%.*]]) local_unnamed_addr #[[ATTR0]] {
 ; COPY-NEXT:  [[BB:.*:]]
 ; COPY-NEXT:    [[S1:%.*]] = alloca [[STRUCT_S]], align 4
 ; COPY-NEXT:    [[S2:%.*]] = addrspacecast ptr [[S]] to ptr addrspace(101)
diff --git a/llvm/test/CodeGen/NVPTX/lower-ctor-dtor.ll b/llvm/test/CodeGen/NVPTX/lower-ctor-dtor.ll
index 4ee1ca3ad4b1f..60b3d70840af5 100644
--- a/llvm/test/CodeGen/NVPTX/lower-ctor-dtor.ll
+++ b/llvm/test/CodeGen/NVPTX/lower-ctor-dtor.ll
@@ -8,7 +8,7 @@
 
 ; Make sure we get the same result if we run multiple times
 ; RUN: opt -S -mtriple=nvptx64-- -passes=nvptx-lower-ctor-dtor,nvptx-lower-ctor-dtor < %s | FileCheck %s
-; RUN: llc -nvptx-lower-global-ctor-dtor -mtriple=nvptx64-amd-amdhsa -mcpu=sm_70 -filetype=asm -o - < %s | FileCheck %s -check-prefix=VISIBILITY
+; RUN: llc -mtriple=nvptx64-amd-amdhsa -mcpu=sm_70 -filetype=asm -o - < %s | FileCheck %s -check-prefix=VISIBILITY
 
 @llvm.global_ctors = appending addrspace(1) global [1 x { i32, ptr, ptr }] [{ i32, ptr, ptr } { i32 1, ptr @foo, ptr null }]
 @llvm.global_dtors = appending addrspace(1) global [1 x { i32, ptr, ptr }] [{ i32, ptr, ptr } { i32 1, ptr @bar, ptr null }]
diff --git a/llvm/test/CodeGen/NVPTX/prefetch.ll b/llvm/test/CodeGen/NVPTX/prefetch.ll
new file mode 100644
index 0000000000000..cf47000ffd9aa
--- /dev/null
+++ b/llvm/test/CodeGen/NVPTX/prefetch.ll
@@ -0,0 +1,81 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc < %s -mtriple=nvptx64 -mcpu=sm_90 -mattr=+ptx80| FileCheck --check-prefixes=CHECK-PTX64 %s
+; RUN: %if ptxas-12.3 %{ llc < %s -mtriple=nvptx64 -mcpu=sm_90 -mattr=+ptx80| %ptxas-verify -arch=sm_90 %}
+
+target triple = "nvptx64-nvidia-cuda"
+
+declare void  @llvm.nvvm.prefetch.local.L1.evictnormal(ptr addrspace(5) %local_ptr)
+declare void  @llvm.nvvm.prefetch.local.L2.evictnormal(ptr addrspace(5) %local_ptr)
+
+declare void  @llvm.nvvm.prefetch.global.L1.evictnormal(ptr addrspace(1) %global_ptr)
+declare void  @llvm.nvvm.prefetch.global.L2.evictnormal(ptr addrspace(1) %global_ptr)
+declare void  @llvm.nvvm.prefetch.global.L1.evictlast(ptr addrspace(1) %global_ptr)
+declare void  @llvm.nvvm.prefetch.global.L2.evictlast(ptr addrspace(1) %global_ptr)
+
+declare void  @llvm.nvvm.prefetch.L1.evictnormal(ptr %ptr)
+declare void  @llvm.nvvm.prefetch.L2.evictnormal(ptr %ptr)
+
+declare void  @llvm.nvvm.prefetchu.L1.evictnormal(ptr %ptr)
+
+define void @prefetch_local(ptr addrspace(5) %local_ptr) {
+; CHECK-PTX64-LABEL: prefetch_local(
+; CHECK-PTX64:       {
+; CHECK-PTX64-NEXT:    .reg .b64 %rd<2>;
+; CHECK-PTX64-EMPTY:
+; CHECK-PTX64-NEXT:  // %bb.0:
+; CHECK-PTX64-NEXT:    ld.param.u64 %rd1, [prefetch_local_param_0];
+; CHECK-PTX64-NEXT:    prefetch.local.L1.evictnormal [%rd1];
+; CHECK-PTX64-NEXT:    prefetch.local.L2.evictnormal [%rd1];
+; CHECK-PTX64-NEXT:    ret;
+  tail call void @llvm.nvvm.prefetch.local.L1.evictnormal(ptr addrspace(5) %local_ptr)
+  tail call void @llvm.nvvm.prefetch.local.L2.evictnormal(ptr addrspace(5) %local_ptr)
+  ret void
+}
+
+define void @prefetch_global(ptr addrspace(1) %global_ptr) {
+; CHECK-PTX64-LABEL: prefetch_global(
+; CHECK-PTX64:       {
+; CHECK-PTX64-NEXT:    .reg .b64 %rd<2>;
+; CHECK-PTX64-EMPTY:
+; CHECK-PTX64-NEXT:  // %bb.0:
+; CHECK-PTX64-NEXT:    ld.param.u64 %rd1, [prefetch_global_param_0];
+; CHECK-PTX64-NEXT:    prefetch.global.L1.evictnormal [%rd1];
+; CHECK-PTX64-NEXT:    prefetch.global.L2.evictnormal [%rd1];
+; CHECK-PTX64-NEXT:    prefetch.global.L1.evictlast [%rd1];
+; CHECK-PTX64-NEXT:    prefetch.global.L2.evictlast [%rd1];
+; CHECK-PTX64-NEXT:    ret;
+  tail call void @llvm.nvvm.prefetch.global.L1.evictnormal(ptr addrspace(1) %global_ptr)
+  tail call void @llvm.nvvm.prefetch.global.L2.evictnormal(ptr addrspace(1) %global_ptr)
+  tail call void @llvm.nvvm.prefetch.global.L1.evictlast(ptr addrspace(1) %global_ptr)
+  tail call void @llvm.nvvm.prefetch.global.L2.evictlast(ptr addrspace(1) %global_ptr)
+  ret void
+}
+
+
+define void @prefetch_(ptr %ptr) {
+; CHECK-PTX64-LABEL: prefetch_(
+; CHECK-PTX64:       {
+; CHECK-PTX64-NEXT:    .reg .b64 %rd<2>;
+; CHECK-PTX64-EMPTY:
+; CHECK-PTX64-NEXT:  // %bb.0:
+; CHECK-PTX64-NEXT:    ld.param.u64 %rd1, [prefetch__param_0];
+; CHECK-PTX64-NEXT:    prefetch.L1.evictnormal [%rd1];
+; CHECK-PTX64-NEXT:    prefetch.L2.evictnormal [%rd1];
+; CHECK-PTX64-NEXT:    ret;
+  tail call void @llvm.nvvm.prefetch.L1.evictnormal(ptr %ptr)
+  tail call void @llvm.nvvm.prefetch.L2.evictnormal(ptr %ptr)
+  ret void
+}
+
+define void @prefetchu_l1(ptr %ptr) {
+; CHECK-PTX64-LABEL: prefetchu_l1(
+; CHECK-PTX64:       {
+; CHECK-PTX64-NEXT:    .reg .b64 %rd<2>;
+; CHECK-PTX64-EMPTY:
+; CHECK-PTX64-NEXT:  // %bb.0:
+; CHECK-PTX64-NEXT:    ld.param.u64 %rd1, [prefetchu_l1_param_0];
+; CHECK-PTX64-NEXT:    prefetchu.L1.evictnormal [%rd1];
+; CHECK-PTX64-NEXT:    ret;
+  tail call void @llvm.nvvm.prefetchu.L1.evictnormal(ptr %ptr)
+  ret void
+}
\ No newline at end of file
diff --git a/llvm/test/CodeGen/NVPTX/setmaxnreg-sm100a.ll b/llvm/test/CodeGen/NVPTX/setmaxnreg-sm100a.ll
new file mode 100644
index 0000000000000..fecc286c7a2fa
--- /dev/null
+++ b/llvm/test/CodeGen/NVPTX/setmaxnreg-sm100a.ll
@@ -0,0 +1,13 @@
+; RUN: llc < %s -march=nvptx64 -mcpu=sm_100a -mattr=+ptx86 | FileCheck --check-prefixes=CHECK %s
+; RUN: %if ptxas-12.8 %{ llc < %s -march=nvptx64 -mcpu=sm_100a -mattr=+ptx86 | %ptxas-verify -arch=sm_100a %}
+
+; CHECK-LABEL: test_set_maxn_reg_sm100a
+define void @test_set_maxn_reg_sm100a() {
+  ; CHECK: setmaxnreg.inc.sync.aligned.u32 96;
+  call void @llvm.nvvm.setmaxnreg.inc.sync.aligned.u32(i32 96)
+
+  ; CHECK: setmaxnreg.dec.sync.aligned.u32 64;
+  call void @llvm.nvvm.setmaxnreg.dec.sync.aligned.u32(i32 64)
+
+  ret void
+}
diff --git a/llvm/test/CodeGen/NVPTX/tcgen05-alloc.ll b/llvm/test/CodeGen/NVPTX/tcgen05-alloc.ll
new file mode 100644
index 0000000000000..f80b5a5e16ea3
--- /dev/null
+++ b/llvm/test/CodeGen/NVPTX/tcgen05-alloc.ll
@@ -0,0 +1,131 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc < %s -march=nvptx64 -mcpu=sm_100a -mattr=+ptx86 | FileCheck --check-prefixes=CHECK_PTX64 %s
+; RUN: llc < %s -march=nvptx64 -mcpu=sm_100a -mattr=+ptx86 --nvptx-short-ptr | FileCheck --check-prefixes=CHECK_PTX64_SHARED32 %s
+; RUN: %if ptxas-12.8 %{ llc < %s -march=nvptx64 -mcpu=sm_100a -mattr=+ptx86 | %ptxas-verify -arch=sm_100a %}
+; RUN: %if ptxas-12.8 %{ llc < %s -march=nvptx64 -mcpu=sm_100a -mattr=+ptx86 --nvptx-short-ptr | %ptxas-verify -arch=sm_100a %}
+
+declare void @llvm.nvvm.tcgen05.alloc.cg1(ptr %addr, i32 %ncols)
+declare void @llvm.nvvm.tcgen05.alloc.cg2(ptr %addr, i32 %ncols)
+declare void @llvm.nvvm.tcgen05.alloc.shared.cg1(ptr addrspace(3) %addr, i32 %ncols)
+declare void @llvm.nvvm.tcgen05.alloc.shared.cg2(ptr addrspace(3) %addr, i32 %ncols)
+
+; CHECK-LABEL: test_tcgen05_alloc
+define void @test_tcgen05_alloc(ptr %addr, i32 %ncols) {
+; CHECK_PTX64-LABEL: test_tcgen05_alloc(
+; CHECK_PTX64:       {
+; CHECK_PTX64-NEXT:    .reg .b32 %r<2>;
+; CHECK_PTX64-NEXT:    .reg .b64 %rd<2>;
+; CHECK_PTX64-EMPTY:
+; CHECK_PTX64-NEXT:  // %bb.0:
+; CHECK_PTX64-NEXT:    ld.param.u64 %rd1, [test_tcgen05_alloc_param_0];
+; CHECK_PTX64-NEXT:    ld.param.u32 %r1, [test_tcgen05_alloc_param_1];
+; CHECK_PTX64-NEXT:    tcgen05.alloc.cta_group::1.sync.aligned.b32 [%rd1], %r1;
+; CHECK_PTX64-NEXT:    tcgen05.alloc.cta_group::2.sync.aligned.b32 [%rd1], %r1;
+; CHECK_PTX64-NEXT:    ret;
+;
+; CHECK_PTX64_SHARED32-LABEL: test_tcgen05_alloc(
+; CHECK_PTX64_SHARED32:       {
+; CHECK_PTX64_SHARED32-NEXT:    .reg .b32 %r<2>;
+; CHECK_PTX64_SHARED32-NEXT:    .reg .b64 %rd<2>;
+; CHECK_PTX64_SHARED32-EMPTY:
+; CHECK_PTX64_SHARED32-NEXT:  // %bb.0:
+; CHECK_PTX64_SHARED32-NEXT:    ld.param.u64 %rd1, [test_tcgen05_alloc_param_0];
+; CHECK_PTX64_SHARED32-NEXT:    ld.param.u32 %r1, [test_tcgen05_alloc_param_1];
+; CHECK_PTX64_SHARED32-NEXT:    tcgen05.alloc.cta_group::1.sync.aligned.b32 [%rd1], %r1;
+; CHECK_PTX64_SHARED32-NEXT:    tcgen05.alloc.cta_group::2.sync.aligned.b32 [%rd1], %r1;
+; CHECK_PTX64_SHARED32-NEXT:    ret;
+  call void @llvm.nvvm.tcgen05.alloc.cg1(ptr %addr, i32 %ncols)
+  call void @llvm.nvvm.tcgen05.alloc.cg2(ptr %addr, i32 %ncols)
+
+  ret void
+}
+
+; CHECK-LABEL: test_tcgen05_alloc_shared
+define void @test_tcgen05_alloc_shared(ptr addrspace(3) %addr, i32 %ncols) {
+; CHECK_PTX64-LABEL: test_tcgen05_alloc_shared(
+; CHECK_PTX64:       {
+; CHECK_PTX64-NEXT:    .reg .b32 %r<2>;
+; CHECK_PTX64-NEXT:    .reg .b64 %rd<2>;
+; CHECK_PTX64-EMPTY:
+; CHECK_PTX64-NEXT:  // %bb.0:
+; CHECK_PTX64-NEXT:    ld.param.u64 %rd1, [test_tcgen05_alloc_shared_param_0];
+; CHECK_PTX64-NEXT:    ld.param.u32 %r1, [test_tcgen05_alloc_shared_param_1];
+; CHECK_PTX64-NEXT:    tcgen05.alloc.cta_group::1.sync.aligned.shared::cta.b32 [%rd1], %r1;
+; CHECK_PTX64-NEXT:    tcgen05.alloc.cta_group::2.sync.aligned.shared::cta.b32 [%rd1], %r1;
+; CHECK_PTX64-NEXT:    ret;
+;
+; CHECK_PTX64_SHARED32-LABEL: test_tcgen05_alloc_shared(
+; CHECK_PTX64_SHARED32:       {
+; CHECK_PTX64_SHARED32-NEXT:    .reg .b32 %r<3>;
+; CHECK_PTX64_SHARED32-EMPTY:
+; CHECK_PTX64_SHARED32-NEXT:  // %bb.0:
+; CHECK_PTX64_SHARED32-NEXT:    ld.param.u32 %r1, [test_tcgen05_alloc_shared_param_0];
+; CHECK_PTX64_SHARED32-NEXT:    ld.param.u32 %r2, [test_tcgen05_alloc_shared_param_1];
+; CHECK_PTX64_SHARED32-NEXT:    tcgen05.alloc.cta_group::1.sync.aligned.shared::cta.b32 [%r1], %r2;
+; CHECK_PTX64_SHARED32-NEXT:    tcgen05.alloc.cta_group::2.sync.aligned.shared::cta.b32 [%r1], %r2;
+; CHECK_PTX64_SHARED32-NEXT:    ret;
+  call void @llvm.nvvm.tcgen05.alloc.shared.cg1(ptr addrspace(3) %addr, i32 %ncols)
+
+  call void @llvm.nvvm.tcgen05.alloc.shared.cg2(ptr addrspace(3) %addr, i32 %ncols)
+  ret void
+}
+
+declare void @llvm.nvvm.tcgen05.dealloc.cg1(ptr addrspace(6) %tmem_addr, i32 %ncols)
+declare void @llvm.nvvm.tcgen05.dealloc.cg2(ptr addrspace(6) %tmem_addr, i32 %ncols)
+
+; CHECK-LABEL: test_tcgen05_dealloc
+define void @test_tcgen05_dealloc(ptr addrspace(6) %tmem_addr, i32 %ncols) {
+; CHECK_PTX64-LABEL: test_tcgen05_dealloc(
+; CHECK_PTX64:       {
+; CHECK_PTX64-NEXT:    .reg .b32 %r<3>;
+; CHECK_PTX64-EMPTY:
+; CHECK_PTX64-NEXT:  // %bb.0:
+; CHECK_PTX64-NEXT:    ld.param.u32 %r1, [test_tcgen05_dealloc_param_0];
+; CHECK_PTX64-NEXT:    ld.param.u32 %r2, [test_tcgen05_dealloc_param_1];
+; CHECK_PTX64-NEXT:    tcgen05.dealloc.cta_group::1.sync.aligned.b32 %r1, %r2;
+; CHECK_PTX64-NEXT:    tcgen05.dealloc.cta_group::2.sync.aligned.b32 %r1, %r2;
+; CHECK_PTX64-NEXT:    ret;
+;
+; CHECK_PTX64_SHARED32-LABEL: test_tcgen05_dealloc(
+; CHECK_PTX64_SHARED32:       {
+; CHECK_PTX64_SHARED32-NEXT:    .reg .b32 %r<3>;
+; CHECK_PTX64_SHARED32-EMPTY:
+; CHECK_PTX64_SHARED32-NEXT:  // %bb.0:
+; CHECK_PTX64_SHARED32-NEXT:    ld.param.u32 %r1, [test_tcgen05_dealloc_param_0];
+; CHECK_PTX64_SHARED32-NEXT:    ld.param.u32 %r2, [test_tcgen05_dealloc_param_1];
+; CHECK_PTX64_SHARED32-NEXT:    tcgen05.dealloc.cta_group::1.sync.aligned.b32 %r1, %r2;
+; CHECK_PTX64_SHARED32-NEXT:    tcgen05.dealloc.cta_group::2.sync.aligned.b32 %r1, %r2;
+; CHECK_PTX64_SHARED32-NEXT:    ret;
+  call void @llvm.nvvm.tcgen05.dealloc.cg1(ptr addrspace(6) %tmem_addr, i32 %ncols)
+
+  call void @llvm.nvvm.tcgen05.dealloc.cg2(ptr addrspace(6) %tmem_addr, i32 %ncols)
+  ret void
+}
+
+declare void @llvm.nvvm.tcgen05.relinq.alloc.permit.cg1()
+declare void @llvm.nvvm.tcgen05.relinq.alloc.permit.cg2()
+
+; CHECK-LABEL: test_tcgen05_relinquish_alloc_permit
+define void @test_tcgen05_relinquish_alloc_permit() {
+; CHECK_PTX64-LABEL: test_tcgen05_relinquish_alloc_permit(
+; CHECK_PTX64:       {
+; CHECK_PTX64-EMPTY:
+; CHECK_PTX64-EMPTY:
+; CHECK_PTX64-NEXT:  // %bb.0:
+; CHECK_PTX64-NEXT:    tcgen05.relinquish_alloc_permit.cta_group::1.sync.aligned;
+; CHECK_PTX64-NEXT:    tcgen05.relinquish_alloc_permit.cta_group::2.sync.aligned;
+; CHECK_PTX64-NEXT:    ret;
+;
+; CHECK_PTX64_SHARED32-LABEL: test_tcgen05_relinquish_alloc_permit(
+; CHECK_PTX64_SHARED32:       {
+; CHECK_PTX64_SHARED32-EMPTY:
+; CHECK_PTX64_SHARED32-EMPTY:
+; CHECK_PTX64_SHARED32-NEXT:  // %bb.0:
+; CHECK_PTX64_SHARED32-NEXT:    tcgen05.relinquish_alloc_permit.cta_group::1.sync.aligned;
+; CHECK_PTX64_SHARED32-NEXT:    tcgen05.relinquish_alloc_permit.cta_group::2.sync.aligned;
+; CHECK_PTX64_SHARED32-NEXT:    ret;
+  call void @llvm.nvvm.tcgen05.relinq.alloc.permit.cg1()
+
+  call void @llvm.nvvm.tcgen05.relinq.alloc.permit.cg2()
+  ret void
+}
diff --git a/llvm/test/CodeGen/NVPTX/tcgen05-commit.ll b/llvm/test/CodeGen/NVPTX/tcgen05-commit.ll
new file mode 100644
index 0000000000000..6e0ec6bcf4465
--- /dev/null
+++ b/llvm/test/CodeGen/NVPTX/tcgen05-commit.ll
@@ -0,0 +1,135 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc < %s -march=nvptx64 -mcpu=sm_100a -mattr=+ptx86 | FileCheck --check-prefixes=CHECK_PTX64 %s
+; RUN: llc < %s -march=nvptx64 -mcpu=sm_100a -mattr=+ptx86 --nvptx-short-ptr | FileCheck --check-prefixes=CHECK_PTX64_SHARED32 %s
+; RUN: %if ptxas-12.8 %{ llc < %s -march=nvptx64 -mcpu=sm_100a -mattr=+ptx86 | %ptxas-verify -arch=sm_100a %}
+; RUN: %if ptxas-12.8 %{ llc < %s -march=nvptx64 -mcpu=sm_100a -mattr=+ptx86 --nvptx-short-ptr | %ptxas-verify -arch=sm_100a %}
+
+declare void @llvm.nvvm.tcgen05.commit.cg1(ptr %bar_addr)
+declare void @llvm.nvvm.tcgen05.commit.cg2(ptr %bar_addr)
+declare void @llvm.nvvm.tcgen05.commit.shared.cg1(ptr addrspace(3) %bar_addr)
+declare void @llvm.nvvm.tcgen05.commit.shared.cg2(ptr addrspace(3) %bar_addr)
+
+; CHECK-LABEL: test_tcgen05_commit
+define void @test_tcgen05_commit(ptr %bar_addr) {
+; CHECK_PTX64-LABEL: test_tcgen05_commit(
+; CHECK_PTX64:       {
+; CHECK_PTX64-NEXT:    .reg .b64 %rd<2>;
+; CHECK_PTX64-EMPTY:
+; CHECK_PTX64-NEXT:  // %bb.0:
+; CHECK_PTX64-NEXT:    ld.param.u64 %rd1, [test_tcgen05_commit_param_0];
+; CHECK_PTX64-NEXT:    tcgen05.commit.cta_group::1.mbarrier::arrive::one.shared::cluster.b64 [%rd1];
+; CHECK_PTX64-NEXT:    tcgen05.commit.cta_group::2.mbarrier::arrive::one.shared::cluster.b64 [%rd1];
+; CHECK_PTX64-NEXT:    ret;
+;
+; CHECK_PTX64_SHARED32-LABEL: test_tcgen05_commit(
+; CHECK_PTX64_SHARED32:       {
+; CHECK_PTX64_SHARED32-NEXT:    .reg .b64 %rd<2>;
+; CHECK_PTX64_SHARED32-EMPTY:
+; CHECK_PTX64_SHARED32-NEXT:  // %bb.0:
+; CHECK_PTX64_SHARED32-NEXT:    ld.param.u64 %rd1, [test_tcgen05_commit_param_0];
+; CHECK_PTX64_SHARED32-NEXT:    tcgen05.commit.cta_group::1.mbarrier::arrive::one.shared::cluster.b64 [%rd1];
+; CHECK_PTX64_SHARED32-NEXT:    tcgen05.commit.cta_group::2.mbarrier::arrive::one.shared::cluster.b64 [%rd1];
+; CHECK_PTX64_SHARED32-NEXT:    ret;
+  call void @llvm.nvvm.tcgen05.commit.cg1(ptr %bar_addr)
+
+  call void @llvm.nvvm.tcgen05.commit.cg2(ptr %bar_addr)
+
+  ret void
+}
+
+; CHECK-LABEL: test_tcgen05_commit_shared
+define void @test_tcgen05_commit_shared(ptr addrspace(3) %bar_addr) {
+; CHECK_PTX64-LABEL: test_tcgen05_commit_shared(
+; CHECK_PTX64:       {
+; CHECK_PTX64-NEXT:    .reg .b64 %rd<2>;
+; CHECK_PTX64-EMPTY:
+; CHECK_PTX64-NEXT:  // %bb.0:
+; CHECK_PTX64-NEXT:    ld.param.u64 %rd1, [test_tcgen05_commit_shared_param_0];
+; CHECK_PTX64-NEXT:    tcgen05.commit.cta_group::1.mbarrier::arrive::one.shared::cluster.b64 [%rd1];
+; CHECK_PTX64-NEXT:    tcgen05.commit.cta_group::2.mbarrier::arrive::one.shared::cluster.b64 [%rd1];
+; CHECK_PTX64-NEXT:    ret;
+;
+; CHECK_PTX64_SHARED32-LABEL: test_tcgen05_commit_shared(
+; CHECK_PTX64_SHARED32:       {
+; CHECK_PTX64_SHARED32-NEXT:    .reg .b32 %r<2>;
+; CHECK_PTX64_SHARED32-EMPTY:
+; CHECK_PTX64_SHARED32-NEXT:  // %bb.0:
+; CHECK_PTX64_SHARED32-NEXT:    ld.param.u32 %r1, [test_tcgen05_commit_shared_param_0];
+; CHECK_PTX64_SHARED32-NEXT:    tcgen05.commit.cta_group::1.mbarrier::arrive::one.shared::cluster.b64 [%r1];
+; CHECK_PTX64_SHARED32-NEXT:    tcgen05.commit.cta_group::2.mbarrier::arrive::one.shared::cluster.b64 [%r1];
+; CHECK_PTX64_SHARED32-NEXT:    ret;
+  call void @llvm.nvvm.tcgen05.commit.shared.cg1(ptr addrspace(3) %bar_addr)
+
+  call void @llvm.nvvm.tcgen05.commit.shared.cg2(ptr addrspace(3) %bar_addr)
+
+  ret void
+}
+
+declare void @llvm.nvvm.tcgen05.commit.mc.cg1(ptr %bar_addr, i16 %cta_mask)
+declare void @llvm.nvvm.tcgen05.commit.mc.cg2(ptr %bar_addr, i16 %cta_mask)
+declare void @llvm.nvvm.tcgen05.commit.mc.shared.cg1(ptr addrspace(3) %bar_addr, i16 %cta_mask)
+declare void @llvm.nvvm.tcgen05.commit.mc.shared.cg2(ptr addrspace(3) %bar_addr, i16 %cta_mask)
+
+; CHECK-LABEL: test_tcgen05_commit_mc
+define void @test_tcgen05_commit_mc(ptr %bar_addr, i16 %cta_mask) {
+; CHECK_PTX64-LABEL: test_tcgen05_commit_mc(
+; CHECK_PTX64:       {
+; CHECK_PTX64-NEXT:    .reg .b16 %rs<2>;
+; CHECK_PTX64-NEXT:    .reg .b64 %rd<2>;
+; CHECK_PTX64-EMPTY:
+; CHECK_PTX64-NEXT:  // %bb.0:
+; CHECK_PTX64-NEXT:    ld.param.u64 %rd1, [test_tcgen05_commit_mc_param_0];
+; CHECK_PTX64-NEXT:    ld.param.u16 %rs1, [test_tcgen05_commit_mc_param_1];
+; CHECK_PTX64-NEXT:    tcgen05.commit.cta_group::1.mbarrier::arrive::one.shared::cluster.multicast::cluster.b64 [%rd1], %rs1;
+; CHECK_PTX64-NEXT:    tcgen05.commit.cta_group::2.mbarrier::arrive::one.shared::cluster.multicast::cluster.b64 [%rd1], %rs1;
+; CHECK_PTX64-NEXT:    ret;
+;
+; CHECK_PTX64_SHARED32-LABEL: test_tcgen05_commit_mc(
+; CHECK_PTX64_SHARED32:       {
+; CHECK_PTX64_SHARED32-NEXT:    .reg .b16 %rs<2>;
+; CHECK_PTX64_SHARED32-NEXT:    .reg .b64 %rd<2>;
+; CHECK_PTX64_SHARED32-EMPTY:
+; CHECK_PTX64_SHARED32-NEXT:  // %bb.0:
+; CHECK_PTX64_SHARED32-NEXT:    ld.param.u64 %rd1, [test_tcgen05_commit_mc_param_0];
+; CHECK_PTX64_SHARED32-NEXT:    ld.param.u16 %rs1, [test_tcgen05_commit_mc_param_1];
+; CHECK_PTX64_SHARED32-NEXT:    tcgen05.commit.cta_group::1.mbarrier::arrive::one.shared::cluster.multicast::cluster.b64 [%rd1], %rs1;
+; CHECK_PTX64_SHARED32-NEXT:    tcgen05.commit.cta_group::2.mbarrier::arrive::one.shared::cluster.multicast::cluster.b64 [%rd1], %rs1;
+; CHECK_PTX64_SHARED32-NEXT:    ret;
+  call void @llvm.nvvm.tcgen05.commit.mc.cg1(ptr %bar_addr, i16 %cta_mask)
+
+  call void @llvm.nvvm.tcgen05.commit.mc.cg2(ptr %bar_addr, i16 %cta_mask)
+
+  ret void
+}
+
+; CHECK-LABEL: test_tcgen05_commit_mc_shared
+define void @test_tcgen05_commit_mc_shared(ptr addrspace(3) %bar_addr, i16 %cta_mask) {
+; CHECK_PTX64-LABEL: test_tcgen05_commit_mc_shared(
+; CHECK_PTX64:       {
+; CHECK_PTX64-NEXT:    .reg .b16 %rs<2>;
+; CHECK_PTX64-NEXT:    .reg .b64 %rd<2>;
+; CHECK_PTX64-EMPTY:
+; CHECK_PTX64-NEXT:  // %bb.0:
+; CHECK_PTX64-NEXT:    ld.param.u64 %rd1, [test_tcgen05_commit_mc_shared_param_0];
+; CHECK_PTX64-NEXT:    ld.param.u16 %rs1, [test_tcgen05_commit_mc_shared_param_1];
+; CHECK_PTX64-NEXT:    tcgen05.commit.cta_group::1.mbarrier::arrive::one.shared::cluster.multicast::cluster.b64 [%rd1], %rs1;
+; CHECK_PTX64-NEXT:    tcgen05.commit.cta_group::2.mbarrier::arrive::one.shared::cluster.multicast::cluster.b64 [%rd1], %rs1;
+; CHECK_PTX64-NEXT:    ret;
+;
+; CHECK_PTX64_SHARED32-LABEL: test_tcgen05_commit_mc_shared(
+; CHECK_PTX64_SHARED32:       {
+; CHECK_PTX64_SHARED32-NEXT:    .reg .b16 %rs<2>;
+; CHECK_PTX64_SHARED32-NEXT:    .reg .b32 %r<2>;
+; CHECK_PTX64_SHARED32-EMPTY:
+; CHECK_PTX64_SHARED32-NEXT:  // %bb.0:
+; CHECK_PTX64_SHARED32-NEXT:    ld.param.u32 %r1, [test_tcgen05_commit_mc_shared_param_0];
+; CHECK_PTX64_SHARED32-NEXT:    ld.param.u16 %rs1, [test_tcgen05_commit_mc_shared_param_1];
+; CHECK_PTX64_SHARED32-NEXT:    tcgen05.commit.cta_group::1.mbarrier::arrive::one.shared::cluster.multicast::cluster.b64 [%r1], %rs1;
+; CHECK_PTX64_SHARED32-NEXT:    tcgen05.commit.cta_group::2.mbarrier::arrive::one.shared::cluster.multicast::cluster.b64 [%r1], %rs1;
+; CHECK_PTX64_SHARED32-NEXT:    ret;
+  call void @llvm.nvvm.tcgen05.commit.mc.shared.cg1(ptr addrspace(3) %bar_addr, i16 %cta_mask)
+
+  call void @llvm.nvvm.tcgen05.commit.mc.shared.cg2(ptr addrspace(3) %bar_addr, i16 %cta_mask)
+
+  ret void
+}
diff --git a/llvm/test/CodeGen/NVPTX/tcgen05-fence.ll b/llvm/test/CodeGen/NVPTX/tcgen05-fence.ll
new file mode 100644
index 0000000000000..07c62671d2fbd
--- /dev/null
+++ b/llvm/test/CodeGen/NVPTX/tcgen05-fence.ll
@@ -0,0 +1,42 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc < %s -march=nvptx64 -mcpu=sm_100a -mattr=+ptx86 | FileCheck --check-prefixes=CHECK %s
+; RUN: %if ptxas-12.8 %{ llc < %s -march=nvptx64 -mcpu=sm_100a -mattr=+ptx86 | %ptxas-verify -arch=sm_100a %}
+
+declare void @llvm.nvvm.tcgen05.fence.before.thread.sync()
+declare void @llvm.nvvm.tcgen05.fence.after.thread.sync()
+declare void @llvm.nvvm.tcgen05.wait.ld()
+declare void @llvm.nvvm.tcgen05.wait.st()
+
+; CHECK-LABEL: test_tcgen05_fence
+define void @test_tcgen05_fence() {
+; CHECK-LABEL: test_tcgen05_fence(
+; CHECK:       {
+; CHECK-EMPTY:
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    tcgen05.fence::before_thread_sync;
+; CHECK-NEXT:    tcgen05.fence::after_thread_sync;
+; CHECK-NEXT:    ret;
+  call void @llvm.nvvm.tcgen05.fence.before.thread.sync()
+
+  call void @llvm.nvvm.tcgen05.fence.after.thread.sync()
+
+  ret void
+}
+
+; CHECK-LABEL: test_tcgen05_wait
+define void @test_tcgen05_wait() {
+; CHECK-LABEL: test_tcgen05_wait(
+; CHECK:       {
+; CHECK-EMPTY:
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    tcgen05.wait::ld.sync.aligned;
+; CHECK-NEXT:    tcgen05.wait::st.sync.aligned;
+; CHECK-NEXT:    ret;
+  call void @llvm.nvvm.tcgen05.wait.ld()
+
+  call void @llvm.nvvm.tcgen05.wait.st()
+
+  ret void
+}
diff --git a/llvm/test/CodeGen/NVPTX/unrecognized-sm1x.ll b/llvm/test/CodeGen/NVPTX/unrecognized-sm1x.ll
new file mode 100644
index 0000000000000..1f6e18e826e86
--- /dev/null
+++ b/llvm/test/CodeGen/NVPTX/unrecognized-sm1x.ll
@@ -0,0 +1,10 @@
+; RUN: llc < %s -mtriple=nvptx -mcpu=sm_10 -debug-only=nvptx-subtarget -o /dev/null 2>&1 | FileCheck %s --check-prefix=SM10
+; RUN: llc < %s -mtriple=nvptx -mcpu=sm_11 -debug-only=nvptx-subtarget -o /dev/null 2>&1 | FileCheck %s --check-prefix=SM11
+; RUN: llc < %s -mtriple=nvptx -mcpu=sm_12 -debug-only=nvptx-subtarget -o /dev/null 2>&1 | FileCheck %s --check-prefix=SM12
+; RUN: llc < %s -mtriple=nvptx -mcpu=sm_13 -debug-only=nvptx-subtarget -o /dev/null 2>&1 | FileCheck %s --check-prefix=SM13
+; REQUIRES: asserts
+
+; SM10: 'sm_10' is not a recognized processor for this target (ignoring processor)
+; SM11: 'sm_11' is not a recognized processor for this target (ignoring processor)
+; SM12: 'sm_12' is not a recognized processor for this target (ignoring processor)
+; SM13: 'sm_13' is not a recognized processor for this target (ignoring processor)
diff --git a/llvm/test/CodeGen/NVPTX/upgrade-nvvm-annotations.ll b/llvm/test/CodeGen/NVPTX/upgrade-nvvm-annotations.ll
new file mode 100644
index 0000000000000..a9f370a12a945
--- /dev/null
+++ b/llvm/test/CodeGen/NVPTX/upgrade-nvvm-annotations.ll
@@ -0,0 +1,28 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-attributes --check-globals all --version 5
+; RUN: opt < %s -mtriple=nvptx64-unknown-unknown -O0 -S | FileCheck %s
+
+define i32 @foo(i32 %a, i32 %b) {
+; CHECK-LABEL: define i32 @foo(
+; CHECK-SAME: i32 alignstack(8) [[A:%.*]], i32 alignstack(16) [[B:%.*]]) {
+; CHECK-NEXT:    ret i32 0
+;
+  ret i32 0
+}
+
+define i32 @bar(i32 %a, i32 %b) {
+; CHECK-LABEL: define ptx_kernel i32 @bar(
+; CHECK-SAME: i32 [[A:%.*]], i32 [[B:%.*]]) {
+; CHECK-NEXT:    ret i32 0
+;
+  ret i32 0
+}
+
+!nvvm.annotations = !{!0, !1, !2}
+
+!0 = !{ptr @foo, !"align", i32 u0x00000008, !"align", i32 u0x00010008, !"align", i32 u0x00020010}
+!1 = !{null, !"align", i32 u0x00000008, !"align", i32 u0x00010008, !"align", i32 u0x00020008}
+!2 = !{ptr @bar, !"kernel", i32 1}
+
+;.
+; CHECK: [[META0:![0-9]+]] = !{ptr @foo, !"align", i32 8}
+;.
diff --git a/llvm/test/CodeGen/PowerPC/absol-jump-table-enabled.ll b/llvm/test/CodeGen/PowerPC/absol-jump-table-enabled.ll
index 91738c1989d91..990468bf608c4 100644
--- a/llvm/test/CodeGen/PowerPC/absol-jump-table-enabled.ll
+++ b/llvm/test/CodeGen/PowerPC/absol-jump-table-enabled.ll
@@ -48,7 +48,7 @@ define zeroext i32 @jumpTableTest(ptr readonly %list) {
 ; CHECK-LE-LABEL: jumpTableTest:
 ; CHECK-LE:       # %bb.0: # %entry
 ; CHECK-LE:       rldic r[[REG:[0-9]+]], r[[REG]], 3, 29
-; CHECK-LE:       ldx r[[REG]], r[[REG]], r[[REG1:[0-9]+]]
+; CHECK-LE:       ldx r[[REG]], r[[REG1:[0-9]+]], r[[REG]]
 ; CHECK-LE:       mtctr r[[REG]]
 ; CHECK-LE:       bctr
 ; CHECK-LE:       blr
@@ -56,7 +56,7 @@ define zeroext i32 @jumpTableTest(ptr readonly %list) {
 ; CHECK-BE-LABEL: jumpTableTest:
 ; CHECK-BE:       # %bb.0: # %entry
 ; CHECK-BE:       rldic r[[REG:[0-9]+]], r[[REG]], 2, 30
-; CHECK-BE:       lwax r[[REG]], r[[REG]], r[[REG1:[0-9]+]]
+; CHECK-BE:       lwax r[[REG]], r[[REG1:[0-9]+]], r[[REG]]
 ; CHECK-BE:       mtctr r[[REG]]
 ; CHECK-BE:       bctr
 ; CHECK-BE:       blr
@@ -64,7 +64,7 @@ define zeroext i32 @jumpTableTest(ptr readonly %list) {
 ; CHECK-AIX-LABEL: jumpTableTest:
 ; CHECK-AIX:       # %bb.0: # %entry
 ; CHECK-AIX:       rldic r[[REG:[0-9]+]], r[[REG]], 2, 30
-; CHECK-AIX:       lwax r[[REG]], r[[REG]], r[[REG1:[0-9]+]]
+; CHECK-AIX:       lwax r[[REG]], r[[REG1:[0-9]+]], r[[REG]]
 ; CHECK-AIX:       mtctr r[[REG]]
 ; CHECK-AIX:       bctr
 ; CHECK-AIX:       blr
diff --git a/llvm/test/CodeGen/PowerPC/aggressive-anti-dep-breaker-subreg.ll b/llvm/test/CodeGen/PowerPC/aggressive-anti-dep-breaker-subreg.ll
index 1dac1b0f68573..0846f7250ed93 100644
--- a/llvm/test/CodeGen/PowerPC/aggressive-anti-dep-breaker-subreg.ll
+++ b/llvm/test/CodeGen/PowerPC/aggressive-anti-dep-breaker-subreg.ll
@@ -9,7 +9,7 @@ entry:
 
 lnext:
   %elementArray = load ptr, ptr %elementArrayPtr, align 8
-; CHECK: lwz [[LDREG:[0-9]+]], 140(1)                   # 4-byte Folded Reload
+; CHECK: ld [[LDREG:[0-9]+]], 120(1)                   # 8-byte Folded Reload
 ; CHECK: # implicit-def: $x[[TEMPREG:[0-9]+]]
   %element = load i32, ptr %elementArray, align 4
 ; CHECK: mr [[TEMPREG]], [[LDREG]]
diff --git a/llvm/test/CodeGen/PowerPC/aix-lower-jump-table-mir.ll b/llvm/test/CodeGen/PowerPC/aix-lower-jump-table-mir.ll
index 8db4f22ba7c00..2a19fc4652301 100644
--- a/llvm/test/CodeGen/PowerPC/aix-lower-jump-table-mir.ll
+++ b/llvm/test/CodeGen/PowerPC/aix-lower-jump-table-mir.ll
@@ -31,8 +31,8 @@ define i32 @jump_table(i32 %a) {
   ; 32SMALL-MIR-NEXT: {{  $}}
   ; 32SMALL-MIR-NEXT:   renamable $r3 = RLWINM killed renamable $r3, 2, 0, 29
   ; 32SMALL-MIR-NEXT:   renamable $r4 = LWZtoc %jump-table.0, $r2 :: (load (s32) from got)
-  ; 32SMALL-MIR-NEXT:   renamable $r3 = LWZX killed renamable $r3, renamable $r4 :: (load (s32) from jump-table)
-  ; 32SMALL-MIR-NEXT:   renamable $r3 = ADD4 killed renamable $r3, killed renamable $r4
+  ; 32SMALL-MIR-NEXT:   renamable $r3 = LWZX renamable $r4, killed renamable $r3 :: (load (s32) from jump-table)
+  ; 32SMALL-MIR-NEXT:   renamable $r3 = ADD4 killed renamable $r4, killed renamable $r3
   ; 32SMALL-MIR-NEXT:   MTCTR killed renamable $r3, implicit-def $ctr
   ; 32SMALL-MIR-NEXT:   BCTR implicit $ctr
   ; 32SMALL-MIR-NEXT: {{  $}}
@@ -79,8 +79,8 @@ define i32 @jump_table(i32 %a) {
   ; 32LARGE-MIR-NEXT:   renamable $r3 = RLWINM killed renamable $r3, 2, 0, 29
   ; 32LARGE-MIR-NEXT:   renamable $r4 = ADDIStocHA $r2, %jump-table.0
   ; 32LARGE-MIR-NEXT:   renamable $r4 = LWZtocL %jump-table.0, killed renamable $r4, implicit $r2 :: (load (s32) from got)
-  ; 32LARGE-MIR-NEXT:   renamable $r3 = LWZX killed renamable $r3, renamable $r4 :: (load (s32) from jump-table)
-  ; 32LARGE-MIR-NEXT:   renamable $r3 = ADD4 killed renamable $r3, killed renamable $r4
+  ; 32LARGE-MIR-NEXT:   renamable $r3 = LWZX renamable $r4, killed renamable $r3 :: (load (s32) from jump-table)
+  ; 32LARGE-MIR-NEXT:   renamable $r3 = ADD4 killed renamable $r4, killed renamable $r3
   ; 32LARGE-MIR-NEXT:   MTCTR killed renamable $r3, implicit-def $ctr
   ; 32LARGE-MIR-NEXT:   BCTR implicit $ctr
   ; 32LARGE-MIR-NEXT: {{  $}}
@@ -126,8 +126,8 @@ define i32 @jump_table(i32 %a) {
   ; 64SMALL-MIR-NEXT: {{  $}}
   ; 64SMALL-MIR-NEXT:   renamable $x3 = RLDIC killed renamable $x3, 2, 30
   ; 64SMALL-MIR-NEXT:   renamable $x4 = LDtocJTI %jump-table.0, $x2 :: (load (s64) from got)
-  ; 64SMALL-MIR-NEXT:   renamable $x3 = LWAX killed renamable $x3, renamable $x4 :: (load (s32) from jump-table)
-  ; 64SMALL-MIR-NEXT:   renamable $x3 = ADD8 killed renamable $x3, killed renamable $x4
+  ; 64SMALL-MIR-NEXT:   renamable $x3 = LWAX renamable $x4, killed renamable $x3 :: (load (s32) from jump-table)
+  ; 64SMALL-MIR-NEXT:   renamable $x3 = ADD8 killed renamable $x4, killed renamable $x3
   ; 64SMALL-MIR-NEXT:   MTCTR8 killed renamable $x3, implicit-def $ctr8
   ; 64SMALL-MIR-NEXT:   BCTR8 implicit $ctr8
   ; 64SMALL-MIR-NEXT: {{  $}}
@@ -174,8 +174,8 @@ define i32 @jump_table(i32 %a) {
   ; 64LARGE-MIR-NEXT:   renamable $x3 = RLDIC killed renamable $x3, 2, 30
   ; 64LARGE-MIR-NEXT:   renamable $x4 = ADDIStocHA8 $x2, %jump-table.0
   ; 64LARGE-MIR-NEXT:   renamable $x4 = LDtocL %jump-table.0, killed renamable $x4, implicit $x2 :: (load (s64) from got)
-  ; 64LARGE-MIR-NEXT:   renamable $x3 = LWAX killed renamable $x3, renamable $x4 :: (load (s32) from jump-table)
-  ; 64LARGE-MIR-NEXT:   renamable $x3 = ADD8 killed renamable $x3, killed renamable $x4
+  ; 64LARGE-MIR-NEXT:   renamable $x3 = LWAX renamable $x4, killed renamable $x3 :: (load (s32) from jump-table)
+  ; 64LARGE-MIR-NEXT:   renamable $x3 = ADD8 killed renamable $x4, killed renamable $x3
   ; 64LARGE-MIR-NEXT:   MTCTR8 killed renamable $x3, implicit-def $ctr8
   ; 64LARGE-MIR-NEXT:   BCTR8 implicit $ctr8
   ; 64LARGE-MIR-NEXT: {{  $}}
diff --git a/llvm/test/CodeGen/PowerPC/aix-lower-jump-table.ll b/llvm/test/CodeGen/PowerPC/aix-lower-jump-table.ll
index 979dfa08beadd..87d6de3ee1432 100644
--- a/llvm/test/CodeGen/PowerPC/aix-lower-jump-table.ll
+++ b/llvm/test/CodeGen/PowerPC/aix-lower-jump-table.ll
@@ -51,8 +51,8 @@ sw.epilog:
 ; 32SMALL-ASM: 	    bgt	0, L..BB0_3
 ; 32SMALL-ASM: 	    lwz 4, L..C0(2)
 ; 32SMALL-ASM: 	    slwi 3, 3, 2
-; 32SMALL-ASM: 	    lwzx 3, 3, 4
-; 32SMALL-ASM: 	    add 3, 3, 4
+; 32SMALL-ASM: 	    lwzx 3, 4, 3
+; 32SMALL-ASM: 	    add 3, 4, 3
 ; 32SMALL-ASM: 	    mtctr 3
 ; 32SMALL-ASM: 	    bctr
 ; 32SMALL-ASM: L..BB0_2:
@@ -78,8 +78,8 @@ sw.epilog:
 ; 32LARGE-ASM: 	    addis 4, L..C0@u(2)
 ; 32LARGE-ASM: 	    slwi 3, 3, 2
 ; 32LARGE-ASM:      lwz 4, L..C0@l(4)
-; 32LARGE-ASM:      lwzx 3, 3, 4
-; 32LARGE-ASM:      add 3, 3, 4
+; 32LARGE-ASM:      lwzx 3, 4, 3
+; 32LARGE-ASM:      add 3, 4, 3
 ; 32LARGE-ASM:      mtctr 3
 ; 32LARGE-ASM:      bctr
 ; 32LARGE-ASM: L..BB0_2:
@@ -104,8 +104,8 @@ sw.epilog:
 ; 64SMALL-ASM:      bgt     0, L..BB0_3
 ; 64SMALL-ASM:      ld 4, L..C0(2)
 ; 64SMALL-ASM:      rldic 3, 3, 2, 30
-; 64SMALL-ASM:      lwax 3, 3, 4
-; 64SMALL-ASM:      add 3, 3, 4
+; 64SMALL-ASM:      lwax 3, 4, 3
+; 64SMALL-ASM:      add 3, 4, 3
 ; 64SMALL-ASM:      mtctr 3
 ; 64SMALL-ASM:      bctr
 ; 64SMALL-ASM: L..BB0_2:
@@ -131,8 +131,8 @@ sw.epilog:
 ; 64LARGE-ASM:      addis 4, L..C0@u(2)
 ; 64LARGE-ASM:      rldic 3, 3, 2, 30
 ; 64LARGE-ASM:      ld 4, L..C0@l(4)
-; 64LARGE-ASM:      lwax 3, 3, 4
-; 64LARGE-ASM:      add 3, 3, 4
+; 64LARGE-ASM:      lwax 3, 4, 3
+; 64LARGE-ASM:      add 3, 4, 3
 ; 64LARGE-ASM:      mtctr 3
 ; 64LARGE-ASM:      bctr
 ; 64LARGE-ASM: L..BB0_2:
diff --git a/llvm/test/CodeGen/PowerPC/jump-tables-collapse-rotate.ll b/llvm/test/CodeGen/PowerPC/jump-tables-collapse-rotate.ll
index dceb895cc1aac..eabb45b3e5533 100644
--- a/llvm/test/CodeGen/PowerPC/jump-tables-collapse-rotate.ll
+++ b/llvm/test/CodeGen/PowerPC/jump-tables-collapse-rotate.ll
@@ -16,8 +16,8 @@ define dso_local zeroext i32 @test(i32 signext %l) nounwind {
 ; CHECK-NEXT:    addis r4, r2, .LC0@toc@ha
 ; CHECK-NEXT:    rldic r3, r3, 2, 30
 ; CHECK-NEXT:    ld r4, .LC0@toc@l(r4)
-; CHECK-NEXT:    lwax r3, r3, r4
-; CHECK-NEXT:    add r3, r3, r4
+; CHECK-NEXT:    lwax r3, r4, r3
+; CHECK-NEXT:    add r3, r4, r3
 ; CHECK-NEXT:    mtctr r3
 ; CHECK-NEXT:    bctr
 ; CHECK-NEXT:  .LBB0_2: # %sw.bb
diff --git a/llvm/test/CodeGen/PowerPC/mcm-5.ll b/llvm/test/CodeGen/PowerPC/mcm-5.ll
index 5e85c62c29a40..b88f405211b05 100644
--- a/llvm/test/CodeGen/PowerPC/mcm-5.ll
+++ b/llvm/test/CodeGen/PowerPC/mcm-5.ll
@@ -55,8 +55,8 @@ sw.epilog:                                        ; preds = %sw.bb3, %sw.default
 
 ; CHECK:       addis [[REG1:[0-9]+]], 2, .LC[[TOCNUM:[0-9]+]]@toc@ha
 ; CHECK:       ld [[REG2:[0-9]+]], .LC[[TOCNUM]]@toc@l([[REG1]])
-; CHECK:       lwax [[REG3:[0-9]+]], {{[0-9]+}}, [[REG2]]
-; CHECK-NEXT:  add  [[REG4:[0-9]+]], [[REG3]], [[REG2]]
+; CHECK:       lwax [[REG3:[0-9]+]], [[REG2]], {{[0-9]+}}
+; CHECK-NEXT:  add  [[REG4:[0-9]+]], [[REG2]], [[REG3]]
 ; CHECK-NEXT:  mtctr [[REG4]]
 ; CHECK-NEXT:  bctr
 
@@ -70,8 +70,8 @@ sw.epilog:                                        ; preds = %sw.bb3, %sw.default
 
 ; LARGE:       addis [[REG1:[0-9]+]], 2, .LC[[TOCNUM:[0-9]+]]@toc@ha
 ; LARGE:       ld [[REG2:[0-9]+]], .LC[[TOCNUM]]@toc@l([[REG1]])
-; LARGE:       lwax [[REG3:[0-9]+]], {{[0-9]+}}, [[REG2]]
-; LARGE-NEXT:  add  [[REG4:[0-9]+]], [[REG3]], [[REGBASE]]
+; LARGE:       lwax [[REG3:[0-9]+]], [[REG2]], {{[0-9]+}}
+; LARGE-NEXT:  add  [[REG4:[0-9]+]], [[REGBASE]], [[REG3]]
 ; LARGE-NEXT:  mtctr [[REG4]]
 ; LARGE-NEXT:  bctr
 
diff --git a/llvm/test/CodeGen/PowerPC/mma-acc-copy-hints.ll b/llvm/test/CodeGen/PowerPC/mma-acc-copy-hints.ll
index 6b188d7c210e3..5decd9a639af8 100644
--- a/llvm/test/CodeGen/PowerPC/mma-acc-copy-hints.ll
+++ b/llvm/test/CodeGen/PowerPC/mma-acc-copy-hints.ll
@@ -26,14 +26,18 @@ define void @testMultiply(ptr nocapture noundef readonly %a, ptr nocapture nound
 ; CHECK-NEXT:    std r29, -24(r30) # 8-byte Folded Spill
 ; CHECK-NEXT:    mr r29, r5
 ; CHECK-NEXT:    bl _Z15buildVectorPairPu13__vector_pairDv16_hS0_@notoc
-; CHECK-NEXT:    xxsetaccz acc0
-; CHECK-NEXT:    xvf32gerpp acc0, v31, v30
+; CHECK-NEXT:    xxsetaccz acc1
+; CHECK-NEXT:    xvf32gerpp acc1, v31, v30
 ; CHECK-NEXT:    lxv v3, 32(r1)
-; CHECK-NEXT:    lxv v2, 48(r1)
-; CHECK-NEXT:    xvf32gerpp acc0, v3, v2
+; CHECK-NEXT:    lxv vs0, 48(r1)
+; CHECK-NEXT:    xvf32gerpp acc1, v3, vs0
 ; CHECK-NEXT:    lxv v31, -48(r30) # 16-byte Folded Reload
 ; CHECK-NEXT:    lxv v30, -64(r30) # 16-byte Folded Reload
-; CHECK-NEXT:    xxmfacc acc0
+; CHECK-NEXT:    xxmfacc acc1
+; CHECK-NEXT:    xxlor vs0, vs4, vs4
+; CHECK-NEXT:    xxlor vs1, vs5, vs5
+; CHECK-NEXT:    xxlor vs2, vs6, vs6
+; CHECK-NEXT:    xxlor vs3, vs7, vs7
 ; CHECK-NEXT:    stxv vs3, 0(r29)
 ; CHECK-NEXT:    pstxv vs2, 8(r29), 0
 ; CHECK-NEXT:    stxv vs1, 16(r29)
@@ -68,8 +72,8 @@ define void @testMultiply(ptr nocapture noundef readonly %a, ptr nocapture nound
 ; CHECK-BE-NEXT:    xxsetaccz acc1
 ; CHECK-BE-NEXT:    xvf32gerpp acc1, v31, v30
 ; CHECK-BE-NEXT:    lxv v3, 144(r1)
-; CHECK-BE-NEXT:    lxv v2, 128(r1)
-; CHECK-BE-NEXT:    xvf32gerpp acc1, v2, v3
+; CHECK-BE-NEXT:    lxv vs0, 128(r1)
+; CHECK-BE-NEXT:    xvf32gerpp acc1, vs0, v3
 ; CHECK-BE-NEXT:    lxv v31, -48(r30) # 16-byte Folded Reload
 ; CHECK-BE-NEXT:    lxv v30, -64(r30) # 16-byte Folded Reload
 ; CHECK-BE-NEXT:    xxmfacc acc1
diff --git a/llvm/test/CodeGen/PowerPC/mma-acc-memops.ll b/llvm/test/CodeGen/PowerPC/mma-acc-memops.ll
index 7c197449201fa..31ddc619d9762 100644
--- a/llvm/test/CodeGen/PowerPC/mma-acc-memops.ll
+++ b/llvm/test/CodeGen/PowerPC/mma-acc-memops.ll
@@ -376,8 +376,8 @@ define dso_local void @testLdStPair(i64 %SrcIdx, i64 %DstIdx) {
 ; LE-PAIRED-LABEL: testLdStPair:
 ; LE-PAIRED:       # %bb.0: # %entry
 ; LE-PAIRED-NEXT:    plxv v3, g@PCREL+32(0), 1
-; LE-PAIRED-NEXT:    plxv v2, g@PCREL+48(0), 1
-; LE-PAIRED-NEXT:    pstxv v2, g@PCREL+80(0), 1
+; LE-PAIRED-NEXT:    plxv vs0, g@PCREL+48(0), 1
+; LE-PAIRED-NEXT:    pstxv vs0, g@PCREL+80(0), 1
 ; LE-PAIRED-NEXT:    pstxv v3, g@PCREL+64(0), 1
 ; LE-PAIRED-NEXT:    blr
 ;
@@ -386,9 +386,9 @@ define dso_local void @testLdStPair(i64 %SrcIdx, i64 %DstIdx) {
 ; BE-PAIRED-NEXT:    addis r3, r2, g@toc@ha
 ; BE-PAIRED-NEXT:    addi r3, r3, g@toc@l
 ; BE-PAIRED-NEXT:    lxv v3, 48(r3)
-; BE-PAIRED-NEXT:    lxv v2, 32(r3)
+; BE-PAIRED-NEXT:    lxv vs0, 32(r3)
 ; BE-PAIRED-NEXT:    stxv v3, 80(r3)
-; BE-PAIRED-NEXT:    stxv v2, 64(r3)
+; BE-PAIRED-NEXT:    stxv vs0, 64(r3)
 ; BE-PAIRED-NEXT:    blr
 ;
 ; LE-PWR9-LABEL: testLdStPair:
@@ -455,9 +455,9 @@ define dso_local void @testXLdStPair(i64 %SrcIdx, i64 %DstIdx) {
 ; LE-PAIRED-NEXT:    lxvx v3, r5, r3
 ; LE-PAIRED-NEXT:    sldi r3, r4, 5
 ; LE-PAIRED-NEXT:    add r4, r5, r3
-; LE-PAIRED-NEXT:    lxv v2, 16(r6)
+; LE-PAIRED-NEXT:    lxv vs0, 16(r6)
 ; LE-PAIRED-NEXT:    stxvx v3, r5, r3
-; LE-PAIRED-NEXT:    stxv v2, 16(r4)
+; LE-PAIRED-NEXT:    stxv vs0, 16(r4)
 ; LE-PAIRED-NEXT:    blr
 ;
 ; BE-PAIRED-LABEL: testXLdStPair:
@@ -466,11 +466,11 @@ define dso_local void @testXLdStPair(i64 %SrcIdx, i64 %DstIdx) {
 ; BE-PAIRED-NEXT:    sldi r3, r3, 5
 ; BE-PAIRED-NEXT:    addi r5, r5, g@toc@l
 ; BE-PAIRED-NEXT:    add r6, r5, r3
-; BE-PAIRED-NEXT:    lxvx v2, r5, r3
+; BE-PAIRED-NEXT:    lxvx vs0, r5, r3
 ; BE-PAIRED-NEXT:    sldi r3, r4, 5
 ; BE-PAIRED-NEXT:    add r4, r5, r3
 ; BE-PAIRED-NEXT:    lxv v3, 16(r6)
-; BE-PAIRED-NEXT:    stxvx v2, r5, r3
+; BE-PAIRED-NEXT:    stxvx vs0, r5, r3
 ; BE-PAIRED-NEXT:    stxv v3, 16(r4)
 ; BE-PAIRED-NEXT:    blr
 ;
@@ -543,8 +543,8 @@ define dso_local void @testUnalignedLdStPair() {
 ; LE-PAIRED-LABEL: testUnalignedLdStPair:
 ; LE-PAIRED:       # %bb.0: # %entry
 ; LE-PAIRED-NEXT:    plxv v3, g@PCREL+11(0), 1
-; LE-PAIRED-NEXT:    plxv v2, g@PCREL+27(0), 1
-; LE-PAIRED-NEXT:    pstxv v2, g@PCREL+35(0), 1
+; LE-PAIRED-NEXT:    plxv vs0, g@PCREL+27(0), 1
+; LE-PAIRED-NEXT:    pstxv vs0, g@PCREL+35(0), 1
 ; LE-PAIRED-NEXT:    pstxv v3, g@PCREL+19(0), 1
 ; LE-PAIRED-NEXT:    blr
 ;
@@ -553,9 +553,9 @@ define dso_local void @testUnalignedLdStPair() {
 ; BE-PAIRED-NEXT:    addis r3, r2, g@toc@ha
 ; BE-PAIRED-NEXT:    addi r3, r3, g@toc@l
 ; BE-PAIRED-NEXT:    plxv v3, 27(r3), 0
-; BE-PAIRED-NEXT:    plxv v2, 11(r3), 0
+; BE-PAIRED-NEXT:    plxv vs0, 11(r3), 0
 ; BE-PAIRED-NEXT:    pstxv v3, 35(r3), 0
-; BE-PAIRED-NEXT:    pstxv v2, 19(r3), 0
+; BE-PAIRED-NEXT:    pstxv vs0, 19(r3), 0
 ; BE-PAIRED-NEXT:    blr
 ;
 ; LE-PWR9-LABEL: testUnalignedLdStPair:
diff --git a/llvm/test/CodeGen/PowerPC/p10-spill-creq.ll b/llvm/test/CodeGen/PowerPC/p10-spill-creq.ll
index ac9641ff35b0c..c36b3bfd50228 100644
--- a/llvm/test/CodeGen/PowerPC/p10-spill-creq.ll
+++ b/llvm/test/CodeGen/PowerPC/p10-spill-creq.ll
@@ -54,13 +54,13 @@ define dso_local double @P10_Spill_CR_EQ(ptr %arg) local_unnamed_addr #0 {
 ; CHECK-NEXT:    # implicit-def: $r4
 ; CHECK-NEXT:  .LBB0_8: # %bb20
 ; CHECK-NEXT:    mfcr r12
-; CHECK-NEXT:    cmpwi cr2, r3, -1
 ; CHECK-NEXT:    cmpwi cr3, r4, -1
+; CHECK-NEXT:    cmpwi cr2, r3, -1
 ; CHECK-NEXT:    stw r12, 8(r1)
 ; CHECK-NEXT:    cmpwi cr7, r3, 0
 ; CHECK-NEXT:    cmpwi cr6, r4, 0
-; CHECK-NEXT:    crand 4*cr5+gt, 4*cr2+gt, 4*cr1+lt
 ; CHECK-NEXT:    crand 4*cr5+lt, 4*cr3+gt, 4*cr5+un
+; CHECK-NEXT:    crand 4*cr5+gt, 4*cr2+gt, 4*cr1+lt
 ; CHECK-NEXT:    # implicit-def: $x3
 ; CHECK-NEXT:    bc 4, 4*cr5+gt, .LBB0_10
 ; CHECK-NEXT:  # %bb.9: # %bb34
@@ -95,15 +95,15 @@ define dso_local double @P10_Spill_CR_EQ(ptr %arg) local_unnamed_addr #0 {
 ; CHECK-NEXT:    lwz r7, 0(r3)
 ; CHECK-NEXT:  .LBB0_18: # %bb58
 ; CHECK-NEXT:    lwz r6, 92(r6)
+; CHECK-NEXT:    cmpwi cr4, r7, 1
 ; CHECK-NEXT:    crand 4*cr7+un, 4*cr3+gt, 4*cr6+un
 ; CHECK-NEXT:    cmpwi cr3, r5, 1
-; CHECK-NEXT:    cmpwi cr4, r7, 1
 ; CHECK-NEXT:    crand 4*cr7+gt, 4*cr7+eq, 4*cr1+lt
 ; CHECK-NEXT:    # implicit-def: $x5
 ; CHECK-NEXT:    crand 4*cr6+un, 4*cr2+eq, 4*cr6+un
 ; CHECK-NEXT:    crand 4*cr5+un, 4*cr6+eq, 4*cr5+un
-; CHECK-NEXT:    crand 4*cr6+gt, 4*cr3+lt, 4*cr6+gt
 ; CHECK-NEXT:    crand 4*cr7+lt, 4*cr4+lt, 4*cr7+lt
+; CHECK-NEXT:    crand 4*cr6+gt, 4*cr3+lt, 4*cr6+gt
 ; CHECK-NEXT:    cmpwi r6, 1
 ; CHECK-NEXT:    crand 4*cr6+lt, lt, 4*cr6+lt
 ; CHECK-NEXT:    bc 4, 4*cr6+gt, .LBB0_20
diff --git a/llvm/test/CodeGen/PowerPC/p10-spill-crgt.ll b/llvm/test/CodeGen/PowerPC/p10-spill-crgt.ll
index f4e49d8b96cf8..f13ba7765e398 100644
--- a/llvm/test/CodeGen/PowerPC/p10-spill-crgt.ll
+++ b/llvm/test/CodeGen/PowerPC/p10-spill-crgt.ll
@@ -65,8 +65,8 @@ define dso_local fastcc void @P10_Spill_CR_GT() unnamed_addr {
 ; CHECK-NEXT:    lwz r5, 0(r3)
 ; CHECK-NEXT:    rlwinm r4, r5, 0, 21, 22
 ; CHECK-NEXT:    cmpwi cr3, r4, 512
-; CHECK-NEXT:    lwax r4, r30, r29
-; CHECK-NEXT:    add r4, r4, r29
+; CHECK-NEXT:    lwax r4, r29, r30
+; CHECK-NEXT:    add r4, r29, r4
 ; CHECK-NEXT:    mtctr r4
 ; CHECK-NEXT:    li r4, 0
 ; CHECK-NEXT:    bctr
@@ -258,8 +258,8 @@ define dso_local fastcc void @P10_Spill_CR_GT() unnamed_addr {
 ; CHECK-BE-NEXT:    lwz r5, 0(r3)
 ; CHECK-BE-NEXT:    rlwinm r4, r5, 0, 21, 22
 ; CHECK-BE-NEXT:    cmpwi cr3, r4, 512
-; CHECK-BE-NEXT:    lwax r4, r30, r29
-; CHECK-BE-NEXT:    add r4, r4, r29
+; CHECK-BE-NEXT:    lwax r4, r29, r30
+; CHECK-BE-NEXT:    add r4, r29, r4
 ; CHECK-BE-NEXT:    mtctr r4
 ; CHECK-BE-NEXT:    li r4, 0
 ; CHECK-BE-NEXT:    bctr
diff --git a/llvm/test/CodeGen/PowerPC/paired-vector-intrinsics.ll b/llvm/test/CodeGen/PowerPC/paired-vector-intrinsics.ll
index cfa9315896af3..dd62b0d54f84b 100644
--- a/llvm/test/CodeGen/PowerPC/paired-vector-intrinsics.ll
+++ b/llvm/test/CodeGen/PowerPC/paired-vector-intrinsics.ll
@@ -20,14 +20,14 @@ declare <256 x i1> @llvm.ppc.vsx.assemble.pair(<16 x i8>, <16 x i8>)
 define void @ass_pair(ptr %ptr, <16 x i8> %vc) {
 ; CHECK-LABEL: ass_pair:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    stxv v2, 0(r3)
 ; CHECK-NEXT:    stxv v2, 16(r3)
+; CHECK-NEXT:    stxv v2, 0(r3)
 ; CHECK-NEXT:    blr
 ;
 ; CHECK-NOMMA-LABEL: ass_pair:
 ; CHECK-NOMMA:       # %bb.0: # %entry
-; CHECK-NOMMA-NEXT:    stxv v2, 0(r3)
 ; CHECK-NOMMA-NEXT:    stxv v2, 16(r3)
+; CHECK-NOMMA-NEXT:    stxv v2, 0(r3)
 ; CHECK-NOMMA-NEXT:    blr
 ;
 ; CHECK-BE-LABEL: ass_pair:
@@ -53,32 +53,32 @@ define void @disass_pair(ptr %ptr1, ptr %ptr2, ptr %ptr3) {
 ; CHECK-LABEL: disass_pair:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    lxv v3, 0(r3)
-; CHECK-NEXT:    lxv v2, 16(r3)
+; CHECK-NEXT:    lxv vs0, 16(r3)
 ; CHECK-NEXT:    stxv v3, 0(r4)
-; CHECK-NEXT:    stxv v2, 0(r5)
+; CHECK-NEXT:    stxv vs0, 0(r5)
 ; CHECK-NEXT:    blr
 ;
 ; CHECK-NOMMA-LABEL: disass_pair:
 ; CHECK-NOMMA:       # %bb.0: # %entry
 ; CHECK-NOMMA-NEXT:    lxv v3, 0(r3)
-; CHECK-NOMMA-NEXT:    lxv v2, 16(r3)
+; CHECK-NOMMA-NEXT:    lxv vs0, 16(r3)
 ; CHECK-NOMMA-NEXT:    stxv v3, 0(r4)
-; CHECK-NOMMA-NEXT:    stxv v2, 0(r5)
+; CHECK-NOMMA-NEXT:    stxv vs0, 0(r5)
 ; CHECK-NOMMA-NEXT:    blr
 ;
 ; CHECK-BE-LABEL: disass_pair:
 ; CHECK-BE:       # %bb.0: # %entry
 ; CHECK-BE-NEXT:    lxv v3, 16(r3)
-; CHECK-BE-NEXT:    lxv v2, 0(r3)
-; CHECK-BE-NEXT:    stxv v2, 0(r4)
+; CHECK-BE-NEXT:    lxv vs0, 0(r3)
+; CHECK-BE-NEXT:    stxv vs0, 0(r4)
 ; CHECK-BE-NEXT:    stxv v3, 0(r5)
 ; CHECK-BE-NEXT:    blr
 ;
 ; CHECK-BE-NOMMA-LABEL: disass_pair:
 ; CHECK-BE-NOMMA:       # %bb.0: # %entry
 ; CHECK-BE-NOMMA-NEXT:    lxv v3, 16(r3)
-; CHECK-BE-NOMMA-NEXT:    lxv v2, 0(r3)
-; CHECK-BE-NOMMA-NEXT:    stxv v2, 0(r4)
+; CHECK-BE-NOMMA-NEXT:    lxv vs0, 0(r3)
+; CHECK-BE-NOMMA-NEXT:    stxv vs0, 0(r4)
 ; CHECK-BE-NOMMA-NEXT:    stxv v3, 0(r5)
 ; CHECK-BE-NOMMA-NEXT:    blr
 entry:
diff --git a/llvm/test/CodeGen/PowerPC/pcrel-jump-table.ll b/llvm/test/CodeGen/PowerPC/pcrel-jump-table.ll
index fb1dbadd48e81..2c032f9717bce 100644
--- a/llvm/test/CodeGen/PowerPC/pcrel-jump-table.ll
+++ b/llvm/test/CodeGen/PowerPC/pcrel-jump-table.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
 ; RUN: llc -verify-machineinstrs -mtriple=powerpc64le-unknown-linux-gnu \
 ; RUN:   -mcpu=pwr10 -ppc-asm-full-reg-names -ppc-min-jump-table-entries=4 < %s | FileCheck %s \
 ; RUN:   --check-prefix=CHECK-R
@@ -19,27 +20,89 @@
 
 define dso_local signext i32 @jumptable(i32 signext %param) {
 ; CHECK-R-LABEL: jumptable:
-; CHECK-R:       # %bb.1: # %entry
-; CHECK-R-NEXT:    rldic r4, r4
+; CHECK-R:       # %bb.0: # %entry
+; CHECK-R-NEXT:    addi r4, r3, -1
+; CHECK-R-NEXT:    cmplwi r4, 19
+; CHECK-R-NEXT:    bgt cr0, .LBB0_3
+; CHECK-R-NEXT:  # %bb.1: # %entry
+; CHECK-R-NEXT:    rldic r4, r4, 2, 30
 ; CHECK-R-NEXT:    paddi r5, 0, .LJTI0_0@PCREL, 1
-; CHECK-R-NEXT:    lwax r4, r4, r5
-; CHECK-R-NEXT:    add r4, r4, r5
+; CHECK-R-NEXT:    lwax r4, r5, r4
+; CHECK-R-NEXT:    add r4, r5, r4
 ; CHECK-R-NEXT:    mtctr r4
 ; CHECK-R-NEXT:    bctr
+; CHECK-R-NEXT:  .LBB0_2: # %sw.bb1
+; CHECK-R-NEXT:    li r3, 4
+; CHECK-R-NEXT:    blr
+; CHECK-R-NEXT:  .LBB0_3: # %sw.default
+; CHECK-R-NEXT:    li r3, -1
+; CHECK-R-NEXT:  .LBB0_4: # %return
+; CHECK-R-NEXT:    blr
+; CHECK-R-NEXT:  .LBB0_5: # %sw.bb2
+; CHECK-R-NEXT:    li r3, 9
+; CHECK-R-NEXT:    blr
+; CHECK-R-NEXT:  .LBB0_6: # %sw.bb3
+; CHECK-R-NEXT:    li r3, 16
+; CHECK-R-NEXT:    blr
+; CHECK-R-NEXT:  .LBB0_7: # %sw.bb4
+; CHECK-R-NEXT:    li r3, 400
+; CHECK-R-NEXT:    blr
+;
 ; CHECK-A-LE-LABEL: jumptable:
-; CHECK-A-LE:       # %bb.1: # %entry
-; CHECK-A-LE-NEXT:    rldic r4, r4
+; CHECK-A-LE:       # %bb.0: # %entry
+; CHECK-A-LE-NEXT:    addi r4, r3, -1
+; CHECK-A-LE-NEXT:    cmplwi r4, 19
+; CHECK-A-LE-NEXT:    bgt cr0, .LBB0_3
+; CHECK-A-LE-NEXT:  # %bb.1: # %entry
+; CHECK-A-LE-NEXT:    rldic r4, r4, 3, 29
 ; CHECK-A-LE-NEXT:    paddi r5, 0, .LJTI0_0@PCREL, 1
-; CHECK-A-LE-NEXT:    ldx r4, r4, r5
+; CHECK-A-LE-NEXT:    ldx r4, r5, r4
 ; CHECK-A-LE-NEXT:    mtctr r4
 ; CHECK-A-LE-NEXT:    bctr
+; CHECK-A-LE-NEXT:  .LBB0_2: # %sw.bb1
+; CHECK-A-LE-NEXT:    li r3, 4
+; CHECK-A-LE-NEXT:    blr
+; CHECK-A-LE-NEXT:  .LBB0_3: # %sw.default
+; CHECK-A-LE-NEXT:    li r3, -1
+; CHECK-A-LE-NEXT:  .LBB0_4: # %return
+; CHECK-A-LE-NEXT:    blr
+; CHECK-A-LE-NEXT:  .LBB0_5: # %sw.bb2
+; CHECK-A-LE-NEXT:    li r3, 9
+; CHECK-A-LE-NEXT:    blr
+; CHECK-A-LE-NEXT:  .LBB0_6: # %sw.bb3
+; CHECK-A-LE-NEXT:    li r3, 16
+; CHECK-A-LE-NEXT:    blr
+; CHECK-A-LE-NEXT:  .LBB0_7: # %sw.bb4
+; CHECK-A-LE-NEXT:    li r3, 400
+; CHECK-A-LE-NEXT:    blr
+;
 ; CHECK-A-BE-LABEL: jumptable:
-; CHECK-A-BE:       # %bb.1: # %entry
-; CHECK-A-BE-NEXT:    rldic r4, r4
+; CHECK-A-BE:       # %bb.0: # %entry
+; CHECK-A-BE-NEXT:    addi r4, r3, -1
+; CHECK-A-BE-NEXT:    cmplwi r4, 19
+; CHECK-A-BE-NEXT:    bgt cr0, .LBB0_3
+; CHECK-A-BE-NEXT:  # %bb.1: # %entry
+; CHECK-A-BE-NEXT:    rldic r4, r4, 2, 30
 ; CHECK-A-BE-NEXT:    paddi r5, 0, .LJTI0_0@PCREL, 1
-; CHECK-A-BE-NEXT:    lwax r4, r4, r5
+; CHECK-A-BE-NEXT:    lwax r4, r5, r4
 ; CHECK-A-BE-NEXT:    mtctr r4
 ; CHECK-A-BE-NEXT:    bctr
+; CHECK-A-BE-NEXT:  .LBB0_2: # %sw.bb1
+; CHECK-A-BE-NEXT:    li r3, 4
+; CHECK-A-BE-NEXT:    blr
+; CHECK-A-BE-NEXT:  .LBB0_3: # %sw.default
+; CHECK-A-BE-NEXT:    li r3, -1
+; CHECK-A-BE-NEXT:  .LBB0_4: # %return
+; CHECK-A-BE-NEXT:    blr
+; CHECK-A-BE-NEXT:  .LBB0_5: # %sw.bb2
+; CHECK-A-BE-NEXT:    li r3, 9
+; CHECK-A-BE-NEXT:    blr
+; CHECK-A-BE-NEXT:  .LBB0_6: # %sw.bb3
+; CHECK-A-BE-NEXT:    li r3, 16
+; CHECK-A-BE-NEXT:    blr
+; CHECK-A-BE-NEXT:  .LBB0_7: # %sw.bb4
+; CHECK-A-BE-NEXT:    li r3, 400
+; CHECK-A-BE-NEXT:    blr
 
 
 entry:
diff --git a/llvm/test/CodeGen/PowerPC/peephole-subreg-def.mir b/llvm/test/CodeGen/PowerPC/peephole-subreg-def.mir
index e89e2068d9278..2a234e83f8e9a 100644
--- a/llvm/test/CodeGen/PowerPC/peephole-subreg-def.mir
+++ b/llvm/test/CodeGen/PowerPC/peephole-subreg-def.mir
@@ -18,16 +18,17 @@ body:             |
 
     ; CHECK-LABEL: name: test_peephole_subreg_def
     ; CHECK: liveins: $x3
-    ; CHECK: [[COPY:%[0-9]+]]:g8rc_and_g8rc_nox0 = COPY $x3
-    ; CHECK: [[ADDI8_:%[0-9]+]]:g8rc = ADDI8 [[COPY]], 1
-    ; CHECK: [[EXTSW:%[0-9]+]]:g8rc_and_g8rc_nox0 = EXTSW [[ADDI8_]]
-    ; CHECK: [[LI8_:%[0-9]+]]:g8rc = LI8 0
-    ; CHECK: STB8 [[LI8_]], 0, [[EXTSW]]
-    ; CHECK: [[COPY1:%[0-9]+]]:gprc_and_gprc_nor0 = COPY [[EXTSW]].sub_32
-    ; CHECK: [[COPY2:%[0-9]+]]:gprc_and_gprc_nor0 = COPY [[COPY1]]
-    ; CHECK: [[ADDI:%[0-9]+]]:gprc = ADDI killed [[COPY2]], 1
-    ; CHECK: [[EXTSW_32_64_:%[0-9]+]]:g8rc_and_g8rc_nox0 = EXTSW_32_64 killed [[ADDI]]
-    ; CHECK: STB8 [[LI8_]], 0, killed [[EXTSW_32_64_]]
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:g8rc_and_g8rc_nox0 = COPY $x3
+    ; CHECK-NEXT: [[ADDI8_:%[0-9]+]]:g8rc = ADDI8 [[COPY]], 1
+    ; CHECK-NEXT: [[EXTSW:%[0-9]+]]:g8rc_and_g8rc_nox0 = EXTSW [[ADDI8_]]
+    ; CHECK-NEXT: [[LI8_:%[0-9]+]]:g8rc = LI8 0
+    ; CHECK-NEXT: STB8 [[LI8_]], 0, [[EXTSW]]
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:gprc_and_gprc_nor0 = COPY [[EXTSW]].sub_32
+    ; CHECK-NEXT: [[COPY2:%[0-9]+]]:gprc_and_gprc_nor0 = COPY [[EXTSW]].sub_32
+    ; CHECK-NEXT: [[ADDI:%[0-9]+]]:gprc = ADDI killed [[COPY2]], 1
+    ; CHECK-NEXT: [[EXTSW_32_64_:%[0-9]+]]:g8rc_and_g8rc_nox0 = EXTSW_32_64 killed [[ADDI]]
+    ; CHECK-NEXT: STB8 [[LI8_]], 0, killed [[EXTSW_32_64_]]
     %0:g8rc_and_g8rc_nox0 = COPY $x3
     %1:g8rc = ADDI8 %0, 1
     %2:g8rc_and_g8rc_nox0 = EXTSW %1
diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/combine.ll b/llvm/test/CodeGen/RISCV/GlobalISel/combine.ll
index 360e84d37ec85..61d1fa5a5b9f4 100644
--- a/llvm/test/CodeGen/RISCV/GlobalISel/combine.ll
+++ b/llvm/test/CodeGen/RISCV/GlobalISel/combine.ll
@@ -20,7 +20,7 @@ define i32 @constant_to_rhs(i32 %x) {
 ; RV64-O0:       # %bb.0:
 ; RV64-O0-NEXT:    mv a1, a0
 ; RV64-O0-NEXT:    li a0, 1
-; RV64-O0-NEXT:    add a0, a0, a1
+; RV64-O0-NEXT:    addw a0, a0, a1
 ; RV64-O0-NEXT:    sext.w a0, a0
 ; RV64-O0-NEXT:    ret
 ;
diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/div-by-constant.ll b/llvm/test/CodeGen/RISCV/GlobalISel/div-by-constant.ll
new file mode 100644
index 0000000000000..f62902cdd14d9
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/GlobalISel/div-by-constant.ll
@@ -0,0 +1,811 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=riscv32 -mattr=+m -verify-machineinstrs -global-isel < %s \
+; RUN:   | FileCheck -check-prefixes=RV32,RV32IM %s
+; RUN: llc -mtriple=riscv32 -mattr=+m,+zba,+zbb \
+; RUN:    -verify-machineinstrs -global-isel < %s \
+; RUN:   | FileCheck -check-prefixes=RV32,RV32IMZB %s
+; RUN: llc -mtriple=riscv64 -mattr=+m -verify-machineinstrs -global-isel < %s \
+; RUN:   | FileCheck -check-prefixes=RV64,RV64IM %s
+; RUN: llc -mtriple=riscv64 -mattr=+m,+zba,+zbb \
+; RUN:   -verify-machineinstrs -global-isel < %s \
+; RUN:   | FileCheck -check-prefixes=RV64,RV64IMZB %s
+
+; Test that there is a single shift after the mul and no addition.
+define i32 @udiv_constant_no_add(i32 %a) nounwind {
+; RV32-LABEL: udiv_constant_no_add:
+; RV32:       # %bb.0:
+; RV32-NEXT:    lui a1, 838861
+; RV32-NEXT:    addi a1, a1, -819
+; RV32-NEXT:    mulhu a0, a0, a1
+; RV32-NEXT:    srli a0, a0, 2
+; RV32-NEXT:    ret
+;
+; RV64IM-LABEL: udiv_constant_no_add:
+; RV64IM:       # %bb.0:
+; RV64IM-NEXT:    slli a0, a0, 32
+; RV64IM-NEXT:    lui a1, 205
+; RV64IM-NEXT:    srli a0, a0, 32
+; RV64IM-NEXT:    addiw a1, a1, -819
+; RV64IM-NEXT:    slli a1, a1, 12
+; RV64IM-NEXT:    addi a1, a1, -819
+; RV64IM-NEXT:    mul a0, a0, a1
+; RV64IM-NEXT:    srli a0, a0, 34
+; RV64IM-NEXT:    ret
+;
+; RV64IMZB-LABEL: udiv_constant_no_add:
+; RV64IMZB:       # %bb.0:
+; RV64IMZB-NEXT:    zext.w a0, a0
+; RV64IMZB-NEXT:    lui a1, 838861
+; RV64IMZB-NEXT:    addi a1, a1, -819
+; RV64IMZB-NEXT:    zext.w a1, a1
+; RV64IMZB-NEXT:    mul a0, a0, a1
+; RV64IMZB-NEXT:    srli a0, a0, 34
+; RV64IMZB-NEXT:    ret
+  %1 = udiv i32 %a, 5
+  ret i32 %1
+}
+
+; This constant requires a sub, shrli, add sequence after the mul.
+define i32 @udiv_constant_add(i32 %a) nounwind {
+; RV32-LABEL: udiv_constant_add:
+; RV32:       # %bb.0:
+; RV32-NEXT:    lui a1, 149797
+; RV32-NEXT:    addi a1, a1, -1755
+; RV32-NEXT:    mulhu a1, a0, a1
+; RV32-NEXT:    sub a0, a0, a1
+; RV32-NEXT:    srli a0, a0, 1
+; RV32-NEXT:    add a0, a0, a1
+; RV32-NEXT:    srli a0, a0, 2
+; RV32-NEXT:    ret
+;
+; RV64IM-LABEL: udiv_constant_add:
+; RV64IM:       # %bb.0:
+; RV64IM-NEXT:    lui a1, 149797
+; RV64IM-NEXT:    slli a2, a0, 32
+; RV64IM-NEXT:    addiw a1, a1, -1755
+; RV64IM-NEXT:    srli a2, a2, 32
+; RV64IM-NEXT:    mul a1, a2, a1
+; RV64IM-NEXT:    srli a1, a1, 32
+; RV64IM-NEXT:    subw a0, a0, a1
+; RV64IM-NEXT:    srliw a0, a0, 1
+; RV64IM-NEXT:    add a0, a0, a1
+; RV64IM-NEXT:    srliw a0, a0, 2
+; RV64IM-NEXT:    ret
+;
+; RV64IMZB-LABEL: udiv_constant_add:
+; RV64IMZB:       # %bb.0:
+; RV64IMZB-NEXT:    lui a1, 149797
+; RV64IMZB-NEXT:    addiw a1, a1, -1755
+; RV64IMZB-NEXT:    zext.w a2, a0
+; RV64IMZB-NEXT:    mul a1, a2, a1
+; RV64IMZB-NEXT:    srli a1, a1, 32
+; RV64IMZB-NEXT:    subw a0, a0, a1
+; RV64IMZB-NEXT:    srliw a0, a0, 1
+; RV64IMZB-NEXT:    add a0, a0, a1
+; RV64IMZB-NEXT:    srliw a0, a0, 2
+; RV64IMZB-NEXT:    ret
+  %1 = udiv i32 %a, 7
+  ret i32 %1
+}
+
+define i64 @udiv64_constant_no_add(i64 %a) nounwind {
+; RV32-LABEL: udiv64_constant_no_add:
+; RV32:       # %bb.0:
+; RV32-NEXT:    lui a2, 838861
+; RV32-NEXT:    mulhu a3, a0, zero
+; RV32-NEXT:    addi a4, a2, -819
+; RV32-NEXT:    addi a2, a2, -820
+; RV32-NEXT:    mul a5, a1, a4
+; RV32-NEXT:    mul a6, a0, a2
+; RV32-NEXT:    mulhu a7, a0, a4
+; RV32-NEXT:    mul t0, zero, a4
+; RV32-NEXT:    mul t1, a1, a2
+; RV32-NEXT:    mulhu t2, a1, a4
+; RV32-NEXT:    mulhu a0, a0, a2
+; RV32-NEXT:    mulhu a1, a1, a2
+; RV32-NEXT:    mul a2, zero, a2
+; RV32-NEXT:    mulhu a4, zero, a4
+; RV32-NEXT:    add a5, a5, a6
+; RV32-NEXT:    add a2, t0, a2
+; RV32-NEXT:    add t0, t0, t1
+; RV32-NEXT:    add a1, a4, a1
+; RV32-NEXT:    sltu a4, a5, a6
+; RV32-NEXT:    add a5, a5, a7
+; RV32-NEXT:    sltu a6, t0, t1
+; RV32-NEXT:    sltiu t1, t0, 0
+; RV32-NEXT:    add t0, t0, t2
+; RV32-NEXT:    add a1, a2, a1
+; RV32-NEXT:    sltu a2, a5, a7
+; RV32-NEXT:    add a6, a6, t1
+; RV32-NEXT:    sltu a5, t0, t2
+; RV32-NEXT:    add t0, t0, a0
+; RV32-NEXT:    add a1, a1, a3
+; RV32-NEXT:    add a2, a4, a2
+; RV32-NEXT:    add a5, a6, a5
+; RV32-NEXT:    sltu a0, t0, a0
+; RV32-NEXT:    add a0, a5, a0
+; RV32-NEXT:    add t0, t0, a2
+; RV32-NEXT:    sltu a2, t0, a2
+; RV32-NEXT:    srli a3, t0, 2
+; RV32-NEXT:    add a0, a0, a2
+; RV32-NEXT:    add a1, a1, a0
+; RV32-NEXT:    slli a0, a1, 30
+; RV32-NEXT:    or a0, a3, a0
+; RV32-NEXT:    srli a1, a1, 2
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: udiv64_constant_no_add:
+; RV64:       # %bb.0:
+; RV64-NEXT:    lui a1, 1035469
+; RV64-NEXT:    addi a1, a1, -819
+; RV64-NEXT:    slli a1, a1, 12
+; RV64-NEXT:    addi a1, a1, -819
+; RV64-NEXT:    slli a1, a1, 12
+; RV64-NEXT:    addi a1, a1, -819
+; RV64-NEXT:    slli a1, a1, 12
+; RV64-NEXT:    addi a1, a1, -819
+; RV64-NEXT:    mulhu a0, a0, a1
+; RV64-NEXT:    srli a0, a0, 2
+; RV64-NEXT:    ret
+  %1 = udiv i64 %a, 5
+  ret i64 %1
+}
+
+define i64 @udiv64_constant_add(i64 %a) nounwind {
+; RV32-LABEL: udiv64_constant_add:
+; RV32:       # %bb.0:
+; RV32-NEXT:    lui a2, 599186
+; RV32-NEXT:    lui a3, 149797
+; RV32-NEXT:    mulhu a4, a0, zero
+; RV32-NEXT:    addi a2, a2, 1171
+; RV32-NEXT:    addi a3, a3, -1756
+; RV32-NEXT:    mul a5, a1, a2
+; RV32-NEXT:    mul a6, a0, a3
+; RV32-NEXT:    mulhu a7, a0, a2
+; RV32-NEXT:    mul t0, zero, a2
+; RV32-NEXT:    mulhu t1, zero, a2
+; RV32-NEXT:    mulhu t2, a1, a3
+; RV32-NEXT:    add t1, t1, t2
+; RV32-NEXT:    mul t2, zero, a3
+; RV32-NEXT:    add t2, t0, t2
+; RV32-NEXT:    add t1, t2, t1
+; RV32-NEXT:    mul t2, a1, a3
+; RV32-NEXT:    mulhu a2, a1, a2
+; RV32-NEXT:    mulhu a3, a0, a3
+; RV32-NEXT:    add a5, a5, a6
+; RV32-NEXT:    add t0, t0, t2
+; RV32-NEXT:    sltu a6, a5, a6
+; RV32-NEXT:    add a5, a5, a7
+; RV32-NEXT:    sltu t2, t0, t2
+; RV32-NEXT:    sltu a5, a5, a7
+; RV32-NEXT:    sltiu a7, t0, 0
+; RV32-NEXT:    add t0, t0, a2
+; RV32-NEXT:    add a7, t2, a7
+; RV32-NEXT:    sltu a2, t0, a2
+; RV32-NEXT:    add t0, t0, a3
+; RV32-NEXT:    add a4, t1, a4
+; RV32-NEXT:    add a5, a6, a5
+; RV32-NEXT:    add a2, a7, a2
+; RV32-NEXT:    sltu a3, t0, a3
+; RV32-NEXT:    add a2, a2, a3
+; RV32-NEXT:    add t0, t0, a5
+; RV32-NEXT:    sltu a3, t0, a5
+; RV32-NEXT:    sub a5, a0, t0
+; RV32-NEXT:    sltu a0, a0, t0
+; RV32-NEXT:    add a2, a2, a3
+; RV32-NEXT:    sub a1, a1, a0
+; RV32-NEXT:    srli a5, a5, 1
+; RV32-NEXT:    add a2, a4, a2
+; RV32-NEXT:    sub a1, a1, a2
+; RV32-NEXT:    slli a0, a1, 31
+; RV32-NEXT:    srli a1, a1, 1
+; RV32-NEXT:    or a0, a5, a0
+; RV32-NEXT:    add a1, a1, a2
+; RV32-NEXT:    add a0, a0, t0
+; RV32-NEXT:    sltu a2, a0, t0
+; RV32-NEXT:    srli a0, a0, 2
+; RV32-NEXT:    add a1, a1, a2
+; RV32-NEXT:    slli a2, a1, 30
+; RV32-NEXT:    or a0, a0, a2
+; RV32-NEXT:    srli a1, a1, 2
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: udiv64_constant_add:
+; RV64:       # %bb.0:
+; RV64-NEXT:    lui a1, %hi(.LCPI3_0)
+; RV64-NEXT:    ld a1, %lo(.LCPI3_0)(a1)
+; RV64-NEXT:    mulhu a1, a0, a1
+; RV64-NEXT:    sub a0, a0, a1
+; RV64-NEXT:    srli a0, a0, 1
+; RV64-NEXT:    add a0, a0, a1
+; RV64-NEXT:    srli a0, a0, 2
+; RV64-NEXT:    ret
+  %1 = udiv i64 %a, 7
+  ret i64 %1
+}
+
+define i8 @udiv8_constant_no_add(i8 %a) nounwind {
+; RV32-LABEL: udiv8_constant_no_add:
+; RV32:       # %bb.0:
+; RV32-NEXT:    andi a0, a0, 255
+; RV32-NEXT:    li a1, 205
+; RV32-NEXT:    mul a0, a0, a1
+; RV32-NEXT:    srli a0, a0, 10
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: udiv8_constant_no_add:
+; RV64:       # %bb.0:
+; RV64-NEXT:    andi a0, a0, 255
+; RV64-NEXT:    li a1, 205
+; RV64-NEXT:    mul a0, a0, a1
+; RV64-NEXT:    srli a0, a0, 10
+; RV64-NEXT:    ret
+  %1 = udiv i8 %a, 5
+  ret i8 %1
+}
+
+define i8 @udiv8_constant_add(i8 %a) nounwind {
+; RV32-LABEL: udiv8_constant_add:
+; RV32:       # %bb.0:
+; RV32-NEXT:    li a1, 37
+; RV32-NEXT:    andi a2, a0, 255
+; RV32-NEXT:    mul a1, a2, a1
+; RV32-NEXT:    srli a1, a1, 8
+; RV32-NEXT:    sub a0, a0, a1
+; RV32-NEXT:    andi a0, a0, 255
+; RV32-NEXT:    srli a0, a0, 1
+; RV32-NEXT:    add a0, a0, a1
+; RV32-NEXT:    andi a0, a0, 255
+; RV32-NEXT:    srli a0, a0, 2
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: udiv8_constant_add:
+; RV64:       # %bb.0:
+; RV64-NEXT:    li a1, 37
+; RV64-NEXT:    andi a2, a0, 255
+; RV64-NEXT:    mul a1, a2, a1
+; RV64-NEXT:    srli a1, a1, 8
+; RV64-NEXT:    subw a0, a0, a1
+; RV64-NEXT:    andi a0, a0, 255
+; RV64-NEXT:    srli a0, a0, 1
+; RV64-NEXT:    add a0, a0, a1
+; RV64-NEXT:    andi a0, a0, 255
+; RV64-NEXT:    srli a0, a0, 2
+; RV64-NEXT:    ret
+  %1 = udiv i8 %a, 7
+  ret i8 %1
+}
+
+define i16 @udiv16_constant_no_add(i16 %a) nounwind {
+; RV32IM-LABEL: udiv16_constant_no_add:
+; RV32IM:       # %bb.0:
+; RV32IM-NEXT:    slli a0, a0, 16
+; RV32IM-NEXT:    lui a1, 13
+; RV32IM-NEXT:    srli a0, a0, 16
+; RV32IM-NEXT:    addi a1, a1, -819
+; RV32IM-NEXT:    mul a0, a0, a1
+; RV32IM-NEXT:    srli a0, a0, 18
+; RV32IM-NEXT:    ret
+;
+; RV32IMZB-LABEL: udiv16_constant_no_add:
+; RV32IMZB:       # %bb.0:
+; RV32IMZB-NEXT:    zext.h a0, a0
+; RV32IMZB-NEXT:    lui a1, 13
+; RV32IMZB-NEXT:    addi a1, a1, -819
+; RV32IMZB-NEXT:    mul a0, a0, a1
+; RV32IMZB-NEXT:    srli a0, a0, 18
+; RV32IMZB-NEXT:    ret
+;
+; RV64IM-LABEL: udiv16_constant_no_add:
+; RV64IM:       # %bb.0:
+; RV64IM-NEXT:    slli a0, a0, 48
+; RV64IM-NEXT:    lui a1, 13
+; RV64IM-NEXT:    srli a0, a0, 48
+; RV64IM-NEXT:    addiw a1, a1, -819
+; RV64IM-NEXT:    mul a0, a0, a1
+; RV64IM-NEXT:    srli a0, a0, 18
+; RV64IM-NEXT:    ret
+;
+; RV64IMZB-LABEL: udiv16_constant_no_add:
+; RV64IMZB:       # %bb.0:
+; RV64IMZB-NEXT:    zext.h a0, a0
+; RV64IMZB-NEXT:    lui a1, 13
+; RV64IMZB-NEXT:    addiw a1, a1, -819
+; RV64IMZB-NEXT:    mul a0, a0, a1
+; RV64IMZB-NEXT:    srli a0, a0, 18
+; RV64IMZB-NEXT:    ret
+  %1 = udiv i16 %a, 5
+  ret i16 %1
+}
+
+define i16 @udiv16_constant_add(i16 %a) nounwind {
+; RV32IM-LABEL: udiv16_constant_add:
+; RV32IM:       # %bb.0:
+; RV32IM-NEXT:    lui a1, 2
+; RV32IM-NEXT:    lui a2, 16
+; RV32IM-NEXT:    addi a1, a1, 1171
+; RV32IM-NEXT:    addi a2, a2, -1
+; RV32IM-NEXT:    and a3, a0, a2
+; RV32IM-NEXT:    mul a1, a3, a1
+; RV32IM-NEXT:    srli a1, a1, 16
+; RV32IM-NEXT:    sub a0, a0, a1
+; RV32IM-NEXT:    and a0, a0, a2
+; RV32IM-NEXT:    srli a0, a0, 1
+; RV32IM-NEXT:    add a0, a0, a1
+; RV32IM-NEXT:    and a0, a0, a2
+; RV32IM-NEXT:    srli a0, a0, 2
+; RV32IM-NEXT:    ret
+;
+; RV32IMZB-LABEL: udiv16_constant_add:
+; RV32IMZB:       # %bb.0:
+; RV32IMZB-NEXT:    lui a1, 2
+; RV32IMZB-NEXT:    addi a1, a1, 1171
+; RV32IMZB-NEXT:    zext.h a2, a0
+; RV32IMZB-NEXT:    mul a1, a2, a1
+; RV32IMZB-NEXT:    srli a1, a1, 16
+; RV32IMZB-NEXT:    sub a0, a0, a1
+; RV32IMZB-NEXT:    zext.h a0, a0
+; RV32IMZB-NEXT:    srli a0, a0, 1
+; RV32IMZB-NEXT:    add a0, a0, a1
+; RV32IMZB-NEXT:    zext.h a0, a0
+; RV32IMZB-NEXT:    srli a0, a0, 2
+; RV32IMZB-NEXT:    ret
+;
+; RV64IM-LABEL: udiv16_constant_add:
+; RV64IM:       # %bb.0:
+; RV64IM-NEXT:    lui a1, 2
+; RV64IM-NEXT:    lui a2, 16
+; RV64IM-NEXT:    addiw a1, a1, 1171
+; RV64IM-NEXT:    addiw a2, a2, -1
+; RV64IM-NEXT:    and a3, a0, a2
+; RV64IM-NEXT:    mul a1, a3, a1
+; RV64IM-NEXT:    srli a1, a1, 16
+; RV64IM-NEXT:    sub a0, a0, a1
+; RV64IM-NEXT:    and a0, a0, a2
+; RV64IM-NEXT:    srli a0, a0, 1
+; RV64IM-NEXT:    add a0, a0, a1
+; RV64IM-NEXT:    and a0, a0, a2
+; RV64IM-NEXT:    srli a0, a0, 2
+; RV64IM-NEXT:    ret
+;
+; RV64IMZB-LABEL: udiv16_constant_add:
+; RV64IMZB:       # %bb.0:
+; RV64IMZB-NEXT:    lui a1, 2
+; RV64IMZB-NEXT:    addi a1, a1, 1171
+; RV64IMZB-NEXT:    zext.h a2, a0
+; RV64IMZB-NEXT:    mul a1, a2, a1
+; RV64IMZB-NEXT:    srli a1, a1, 16
+; RV64IMZB-NEXT:    sub a0, a0, a1
+; RV64IMZB-NEXT:    zext.h a0, a0
+; RV64IMZB-NEXT:    srli a0, a0, 1
+; RV64IMZB-NEXT:    add a0, a0, a1
+; RV64IMZB-NEXT:    zext.h a0, a0
+; RV64IMZB-NEXT:    srli a0, a0, 2
+; RV64IMZB-NEXT:    ret
+  %1 = udiv i16 %a, 7
+  ret i16 %1
+}
+
+; Test the simplest case a srli and an add after the mul. No srai.
+define i32 @sdiv_constant_no_srai(i32 %a) nounwind {
+; RV32-LABEL: sdiv_constant_no_srai:
+; RV32:       # %bb.0:
+; RV32-NEXT:    li a1, 3
+; RV32-NEXT:    div a0, a0, a1
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: sdiv_constant_no_srai:
+; RV64:       # %bb.0:
+; RV64-NEXT:    li a1, 3
+; RV64-NEXT:    divw a0, a0, a1
+; RV64-NEXT:    ret
+  %1 = sdiv i32 %a, 3
+  ret i32 %1
+}
+
+; This constant requires an srai between the mul and the add.
+define i32 @sdiv_constant_srai(i32 %a) nounwind {
+; RV32-LABEL: sdiv_constant_srai:
+; RV32:       # %bb.0:
+; RV32-NEXT:    li a1, 5
+; RV32-NEXT:    div a0, a0, a1
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: sdiv_constant_srai:
+; RV64:       # %bb.0:
+; RV64-NEXT:    li a1, 5
+; RV64-NEXT:    divw a0, a0, a1
+; RV64-NEXT:    ret
+  %1 = sdiv i32 %a, 5
+  ret i32 %1
+}
+
+; This constant requires an add and an srai after the mul.
+define i32 @sdiv_constant_add_srai(i32 %a) nounwind {
+; RV32-LABEL: sdiv_constant_add_srai:
+; RV32:       # %bb.0:
+; RV32-NEXT:    li a1, 7
+; RV32-NEXT:    div a0, a0, a1
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: sdiv_constant_add_srai:
+; RV64:       # %bb.0:
+; RV64-NEXT:    li a1, 7
+; RV64-NEXT:    divw a0, a0, a1
+; RV64-NEXT:    ret
+  %1 = sdiv i32 %a, 7
+  ret i32 %1
+}
+
+; This constant requires a sub and an srai after the mul.
+define i32 @sdiv_constant_sub_srai(i32 %a) nounwind {
+; RV32-LABEL: sdiv_constant_sub_srai:
+; RV32:       # %bb.0:
+; RV32-NEXT:    li a1, -7
+; RV32-NEXT:    div a0, a0, a1
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: sdiv_constant_sub_srai:
+; RV64:       # %bb.0:
+; RV64-NEXT:    li a1, -7
+; RV64-NEXT:    divw a0, a0, a1
+; RV64-NEXT:    ret
+  %1 = sdiv i32 %a, -7
+  ret i32 %1
+}
+
+define i64 @sdiv64_constant_no_srai(i64 %a) nounwind {
+; RV32-LABEL: sdiv64_constant_no_srai:
+; RV32:       # %bb.0:
+; RV32-NEXT:    addi sp, sp, -16
+; RV32-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32-NEXT:    li a2, 3
+; RV32-NEXT:    li a3, 0
+; RV32-NEXT:    call __divdi3
+; RV32-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32-NEXT:    addi sp, sp, 16
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: sdiv64_constant_no_srai:
+; RV64:       # %bb.0:
+; RV64-NEXT:    li a1, 3
+; RV64-NEXT:    div a0, a0, a1
+; RV64-NEXT:    ret
+  %1 = sdiv i64 %a, 3
+  ret i64 %1
+}
+
+define i64 @sdiv64_constant_srai(i64 %a) nounwind {
+; RV32-LABEL: sdiv64_constant_srai:
+; RV32:       # %bb.0:
+; RV32-NEXT:    addi sp, sp, -16
+; RV32-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32-NEXT:    li a2, 5
+; RV32-NEXT:    li a3, 0
+; RV32-NEXT:    call __divdi3
+; RV32-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32-NEXT:    addi sp, sp, 16
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: sdiv64_constant_srai:
+; RV64:       # %bb.0:
+; RV64-NEXT:    li a1, 5
+; RV64-NEXT:    div a0, a0, a1
+; RV64-NEXT:    ret
+  %1 = sdiv i64 %a, 5
+  ret i64 %1
+}
+
+define i64 @sdiv64_constant_add_srai(i64 %a) nounwind {
+; RV32-LABEL: sdiv64_constant_add_srai:
+; RV32:       # %bb.0:
+; RV32-NEXT:    addi sp, sp, -16
+; RV32-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32-NEXT:    li a2, 15
+; RV32-NEXT:    li a3, 0
+; RV32-NEXT:    call __divdi3
+; RV32-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32-NEXT:    addi sp, sp, 16
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: sdiv64_constant_add_srai:
+; RV64:       # %bb.0:
+; RV64-NEXT:    li a1, 15
+; RV64-NEXT:    div a0, a0, a1
+; RV64-NEXT:    ret
+  %1 = sdiv i64 %a, 15
+  ret i64 %1
+}
+
+define i64 @sdiv64_constant_sub_srai(i64 %a) nounwind {
+; RV32-LABEL: sdiv64_constant_sub_srai:
+; RV32:       # %bb.0:
+; RV32-NEXT:    addi sp, sp, -16
+; RV32-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32-NEXT:    li a2, -3
+; RV32-NEXT:    li a3, -1
+; RV32-NEXT:    call __divdi3
+; RV32-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32-NEXT:    addi sp, sp, 16
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: sdiv64_constant_sub_srai:
+; RV64:       # %bb.0:
+; RV64-NEXT:    li a1, -3
+; RV64-NEXT:    div a0, a0, a1
+; RV64-NEXT:    ret
+  %1 = sdiv i64 %a, -3
+  ret i64 %1
+}
+
+define i8 @sdiv8_constant_no_srai(i8 %a) nounwind {
+; RV32IM-LABEL: sdiv8_constant_no_srai:
+; RV32IM:       # %bb.0:
+; RV32IM-NEXT:    li a1, 3
+; RV32IM-NEXT:    slli a0, a0, 24
+; RV32IM-NEXT:    srai a0, a0, 24
+; RV32IM-NEXT:    div a0, a0, a1
+; RV32IM-NEXT:    ret
+;
+; RV32IMZB-LABEL: sdiv8_constant_no_srai:
+; RV32IMZB:       # %bb.0:
+; RV32IMZB-NEXT:    li a1, 3
+; RV32IMZB-NEXT:    sext.b a0, a0
+; RV32IMZB-NEXT:    div a0, a0, a1
+; RV32IMZB-NEXT:    ret
+;
+; RV64IM-LABEL: sdiv8_constant_no_srai:
+; RV64IM:       # %bb.0:
+; RV64IM-NEXT:    li a1, 3
+; RV64IM-NEXT:    slli a0, a0, 56
+; RV64IM-NEXT:    srai a0, a0, 56
+; RV64IM-NEXT:    divw a0, a0, a1
+; RV64IM-NEXT:    ret
+;
+; RV64IMZB-LABEL: sdiv8_constant_no_srai:
+; RV64IMZB:       # %bb.0:
+; RV64IMZB-NEXT:    li a1, 3
+; RV64IMZB-NEXT:    sext.b a0, a0
+; RV64IMZB-NEXT:    divw a0, a0, a1
+; RV64IMZB-NEXT:    ret
+  %1 = sdiv i8 %a, 3
+  ret i8 %1
+}
+
+define i8 @sdiv8_constant_srai(i8 %a) nounwind {
+; RV32IM-LABEL: sdiv8_constant_srai:
+; RV32IM:       # %bb.0:
+; RV32IM-NEXT:    li a1, 5
+; RV32IM-NEXT:    slli a0, a0, 24
+; RV32IM-NEXT:    srai a0, a0, 24
+; RV32IM-NEXT:    div a0, a0, a1
+; RV32IM-NEXT:    ret
+;
+; RV32IMZB-LABEL: sdiv8_constant_srai:
+; RV32IMZB:       # %bb.0:
+; RV32IMZB-NEXT:    li a1, 5
+; RV32IMZB-NEXT:    sext.b a0, a0
+; RV32IMZB-NEXT:    div a0, a0, a1
+; RV32IMZB-NEXT:    ret
+;
+; RV64IM-LABEL: sdiv8_constant_srai:
+; RV64IM:       # %bb.0:
+; RV64IM-NEXT:    li a1, 5
+; RV64IM-NEXT:    slli a0, a0, 56
+; RV64IM-NEXT:    srai a0, a0, 56
+; RV64IM-NEXT:    divw a0, a0, a1
+; RV64IM-NEXT:    ret
+;
+; RV64IMZB-LABEL: sdiv8_constant_srai:
+; RV64IMZB:       # %bb.0:
+; RV64IMZB-NEXT:    li a1, 5
+; RV64IMZB-NEXT:    sext.b a0, a0
+; RV64IMZB-NEXT:    divw a0, a0, a1
+; RV64IMZB-NEXT:    ret
+  %1 = sdiv i8 %a, 5
+  ret i8 %1
+}
+
+define i8 @sdiv8_constant_add_srai(i8 %a) nounwind {
+; RV32IM-LABEL: sdiv8_constant_add_srai:
+; RV32IM:       # %bb.0:
+; RV32IM-NEXT:    li a1, 7
+; RV32IM-NEXT:    slli a0, a0, 24
+; RV32IM-NEXT:    srai a0, a0, 24
+; RV32IM-NEXT:    div a0, a0, a1
+; RV32IM-NEXT:    ret
+;
+; RV32IMZB-LABEL: sdiv8_constant_add_srai:
+; RV32IMZB:       # %bb.0:
+; RV32IMZB-NEXT:    li a1, 7
+; RV32IMZB-NEXT:    sext.b a0, a0
+; RV32IMZB-NEXT:    div a0, a0, a1
+; RV32IMZB-NEXT:    ret
+;
+; RV64IM-LABEL: sdiv8_constant_add_srai:
+; RV64IM:       # %bb.0:
+; RV64IM-NEXT:    li a1, 7
+; RV64IM-NEXT:    slli a0, a0, 56
+; RV64IM-NEXT:    srai a0, a0, 56
+; RV64IM-NEXT:    divw a0, a0, a1
+; RV64IM-NEXT:    ret
+;
+; RV64IMZB-LABEL: sdiv8_constant_add_srai:
+; RV64IMZB:       # %bb.0:
+; RV64IMZB-NEXT:    li a1, 7
+; RV64IMZB-NEXT:    sext.b a0, a0
+; RV64IMZB-NEXT:    divw a0, a0, a1
+; RV64IMZB-NEXT:    ret
+  %1 = sdiv i8 %a, 7
+  ret i8 %1
+}
+
+define i8 @sdiv8_constant_sub_srai(i8 %a) nounwind {
+; RV32IM-LABEL: sdiv8_constant_sub_srai:
+; RV32IM:       # %bb.0:
+; RV32IM-NEXT:    li a1, -7
+; RV32IM-NEXT:    slli a0, a0, 24
+; RV32IM-NEXT:    srai a0, a0, 24
+; RV32IM-NEXT:    div a0, a0, a1
+; RV32IM-NEXT:    ret
+;
+; RV32IMZB-LABEL: sdiv8_constant_sub_srai:
+; RV32IMZB:       # %bb.0:
+; RV32IMZB-NEXT:    li a1, -7
+; RV32IMZB-NEXT:    sext.b a0, a0
+; RV32IMZB-NEXT:    div a0, a0, a1
+; RV32IMZB-NEXT:    ret
+;
+; RV64IM-LABEL: sdiv8_constant_sub_srai:
+; RV64IM:       # %bb.0:
+; RV64IM-NEXT:    li a1, -7
+; RV64IM-NEXT:    slli a0, a0, 56
+; RV64IM-NEXT:    srai a0, a0, 56
+; RV64IM-NEXT:    divw a0, a0, a1
+; RV64IM-NEXT:    ret
+;
+; RV64IMZB-LABEL: sdiv8_constant_sub_srai:
+; RV64IMZB:       # %bb.0:
+; RV64IMZB-NEXT:    li a1, -7
+; RV64IMZB-NEXT:    sext.b a0, a0
+; RV64IMZB-NEXT:    divw a0, a0, a1
+; RV64IMZB-NEXT:    ret
+  %1 = sdiv i8 %a, -7
+  ret i8 %1
+}
+
+define i16 @sdiv16_constant_no_srai(i16 %a) nounwind {
+; RV32IM-LABEL: sdiv16_constant_no_srai:
+; RV32IM:       # %bb.0:
+; RV32IM-NEXT:    li a1, 3
+; RV32IM-NEXT:    slli a0, a0, 16
+; RV32IM-NEXT:    srai a0, a0, 16
+; RV32IM-NEXT:    div a0, a0, a1
+; RV32IM-NEXT:    ret
+;
+; RV32IMZB-LABEL: sdiv16_constant_no_srai:
+; RV32IMZB:       # %bb.0:
+; RV32IMZB-NEXT:    li a1, 3
+; RV32IMZB-NEXT:    sext.h a0, a0
+; RV32IMZB-NEXT:    div a0, a0, a1
+; RV32IMZB-NEXT:    ret
+;
+; RV64IM-LABEL: sdiv16_constant_no_srai:
+; RV64IM:       # %bb.0:
+; RV64IM-NEXT:    li a1, 3
+; RV64IM-NEXT:    slli a0, a0, 48
+; RV64IM-NEXT:    srai a0, a0, 48
+; RV64IM-NEXT:    divw a0, a0, a1
+; RV64IM-NEXT:    ret
+;
+; RV64IMZB-LABEL: sdiv16_constant_no_srai:
+; RV64IMZB:       # %bb.0:
+; RV64IMZB-NEXT:    li a1, 3
+; RV64IMZB-NEXT:    sext.h a0, a0
+; RV64IMZB-NEXT:    divw a0, a0, a1
+; RV64IMZB-NEXT:    ret
+  %1 = sdiv i16 %a, 3
+  ret i16 %1
+}
+
+define i16 @sdiv16_constant_srai(i16 %a) nounwind {
+; RV32IM-LABEL: sdiv16_constant_srai:
+; RV32IM:       # %bb.0:
+; RV32IM-NEXT:    li a1, 5
+; RV32IM-NEXT:    slli a0, a0, 16
+; RV32IM-NEXT:    srai a0, a0, 16
+; RV32IM-NEXT:    div a0, a0, a1
+; RV32IM-NEXT:    ret
+;
+; RV32IMZB-LABEL: sdiv16_constant_srai:
+; RV32IMZB:       # %bb.0:
+; RV32IMZB-NEXT:    li a1, 5
+; RV32IMZB-NEXT:    sext.h a0, a0
+; RV32IMZB-NEXT:    div a0, a0, a1
+; RV32IMZB-NEXT:    ret
+;
+; RV64IM-LABEL: sdiv16_constant_srai:
+; RV64IM:       # %bb.0:
+; RV64IM-NEXT:    li a1, 5
+; RV64IM-NEXT:    slli a0, a0, 48
+; RV64IM-NEXT:    srai a0, a0, 48
+; RV64IM-NEXT:    divw a0, a0, a1
+; RV64IM-NEXT:    ret
+;
+; RV64IMZB-LABEL: sdiv16_constant_srai:
+; RV64IMZB:       # %bb.0:
+; RV64IMZB-NEXT:    li a1, 5
+; RV64IMZB-NEXT:    sext.h a0, a0
+; RV64IMZB-NEXT:    divw a0, a0, a1
+; RV64IMZB-NEXT:    ret
+  %1 = sdiv i16 %a, 5
+  ret i16 %1
+}
+
+define i16 @sdiv16_constant_add_srai(i16 %a) nounwind {
+; RV32IM-LABEL: sdiv16_constant_add_srai:
+; RV32IM:       # %bb.0:
+; RV32IM-NEXT:    li a1, 15
+; RV32IM-NEXT:    slli a0, a0, 16
+; RV32IM-NEXT:    srai a0, a0, 16
+; RV32IM-NEXT:    div a0, a0, a1
+; RV32IM-NEXT:    ret
+;
+; RV32IMZB-LABEL: sdiv16_constant_add_srai:
+; RV32IMZB:       # %bb.0:
+; RV32IMZB-NEXT:    li a1, 15
+; RV32IMZB-NEXT:    sext.h a0, a0
+; RV32IMZB-NEXT:    div a0, a0, a1
+; RV32IMZB-NEXT:    ret
+;
+; RV64IM-LABEL: sdiv16_constant_add_srai:
+; RV64IM:       # %bb.0:
+; RV64IM-NEXT:    li a1, 15
+; RV64IM-NEXT:    slli a0, a0, 48
+; RV64IM-NEXT:    srai a0, a0, 48
+; RV64IM-NEXT:    divw a0, a0, a1
+; RV64IM-NEXT:    ret
+;
+; RV64IMZB-LABEL: sdiv16_constant_add_srai:
+; RV64IMZB:       # %bb.0:
+; RV64IMZB-NEXT:    li a1, 15
+; RV64IMZB-NEXT:    sext.h a0, a0
+; RV64IMZB-NEXT:    divw a0, a0, a1
+; RV64IMZB-NEXT:    ret
+  %1 = sdiv i16 %a, 15
+  ret i16 %1
+}
+
+define i16 @sdiv16_constant_sub_srai(i16 %a) nounwind {
+; RV32IM-LABEL: sdiv16_constant_sub_srai:
+; RV32IM:       # %bb.0:
+; RV32IM-NEXT:    li a1, -15
+; RV32IM-NEXT:    slli a0, a0, 16
+; RV32IM-NEXT:    srai a0, a0, 16
+; RV32IM-NEXT:    div a0, a0, a1
+; RV32IM-NEXT:    ret
+;
+; RV32IMZB-LABEL: sdiv16_constant_sub_srai:
+; RV32IMZB:       # %bb.0:
+; RV32IMZB-NEXT:    li a1, -15
+; RV32IMZB-NEXT:    sext.h a0, a0
+; RV32IMZB-NEXT:    div a0, a0, a1
+; RV32IMZB-NEXT:    ret
+;
+; RV64IM-LABEL: sdiv16_constant_sub_srai:
+; RV64IM:       # %bb.0:
+; RV64IM-NEXT:    li a1, -15
+; RV64IM-NEXT:    slli a0, a0, 48
+; RV64IM-NEXT:    srai a0, a0, 48
+; RV64IM-NEXT:    divw a0, a0, a1
+; RV64IM-NEXT:    ret
+;
+; RV64IMZB-LABEL: sdiv16_constant_sub_srai:
+; RV64IMZB:       # %bb.0:
+; RV64IMZB-NEXT:    li a1, -15
+; RV64IMZB-NEXT:    sext.h a0, a0
+; RV64IMZB-NEXT:    divw a0, a0, a1
+; RV64IMZB-NEXT:    ret
+  %1 = sdiv i16 %a, -15
+  ret i16 %1
+}
diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-fshl-fshr-rv32.mir b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-fshl-fshr-rv32.mir
index 0745d7bb55017..1b7726251e289 100644
--- a/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-fshl-fshr-rv32.mir
+++ b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-fshl-fshr-rv32.mir
@@ -25,7 +25,7 @@ body:             |
     ; CHECK-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C3]]
     ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[AND2]], [[C2]](s32)
     ; CHECK-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[LSHR]], [[AND1]](s32)
-    ; CHECK-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL]], [[LSHR1]]
+    ; CHECK-NEXT: [[OR:%[0-9]+]]:_(s32) = disjoint G_OR [[SHL]], [[LSHR1]]
     ; CHECK-NEXT: $x10 = COPY [[OR]](s32)
     ; CHECK-NEXT: PseudoRET implicit $x10
     %3:_(s32) = COPY $x10
@@ -63,7 +63,7 @@ body:             |
     ; CHECK-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C3]]
     ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[AND2]], [[C2]](s32)
     ; CHECK-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[LSHR]], [[AND1]](s32)
-    ; CHECK-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL]], [[LSHR1]]
+    ; CHECK-NEXT: [[OR:%[0-9]+]]:_(s32) = disjoint G_OR [[SHL]], [[LSHR1]]
     ; CHECK-NEXT: $x10 = COPY [[OR]](s32)
     ; CHECK-NEXT: PseudoRET implicit $x10
     %3:_(s32) = COPY $x10
@@ -99,7 +99,7 @@ body:             |
     ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY]], [[AND]](s32)
     ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[COPY1]], [[C2]](s32)
     ; CHECK-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[LSHR]], [[AND1]](s32)
-    ; CHECK-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL]], [[LSHR1]]
+    ; CHECK-NEXT: [[OR:%[0-9]+]]:_(s32) = disjoint G_OR [[SHL]], [[LSHR1]]
     ; CHECK-NEXT: $x10 = COPY [[OR]](s32)
     ; CHECK-NEXT: PseudoRET implicit $x10
     %0:_(s32) = COPY $x10
@@ -205,7 +205,7 @@ body:             |
     ; CHECK-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
     ; CHECK-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C3]]
     ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[AND2]], [[AND]](s32)
-    ; CHECK-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL1]], [[LSHR]]
+    ; CHECK-NEXT: [[OR:%[0-9]+]]:_(s32) = disjoint G_OR [[SHL1]], [[LSHR]]
     ; CHECK-NEXT: $x10 = COPY [[OR]](s32)
     ; CHECK-NEXT: PseudoRET implicit $x10
     %3:_(s32) = COPY $x10
@@ -243,7 +243,7 @@ body:             |
     ; CHECK-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
     ; CHECK-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C3]]
     ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[AND2]], [[AND]](s32)
-    ; CHECK-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL1]], [[LSHR]]
+    ; CHECK-NEXT: [[OR:%[0-9]+]]:_(s32) = disjoint G_OR [[SHL1]], [[LSHR]]
     ; CHECK-NEXT: $x10 = COPY [[OR]](s32)
     ; CHECK-NEXT: PseudoRET implicit $x10
     %3:_(s32) = COPY $x10
@@ -279,7 +279,7 @@ body:             |
     ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY]], [[C2]](s32)
     ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[SHL]], [[AND1]](s32)
     ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[COPY1]], [[AND]](s32)
-    ; CHECK-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL1]], [[LSHR]]
+    ; CHECK-NEXT: [[OR:%[0-9]+]]:_(s32) = disjoint G_OR [[SHL1]], [[LSHR]]
     ; CHECK-NEXT: $x10 = COPY [[OR]](s32)
     ; CHECK-NEXT: PseudoRET implicit $x10
     %0:_(s32) = COPY $x10
diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-fshl-fshr-rv64.mir b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-fshl-fshr-rv64.mir
index 012342b127608..f3d95d3a9b292 100644
--- a/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-fshl-fshr-rv64.mir
+++ b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-fshl-fshr-rv64.mir
@@ -25,7 +25,7 @@ body:             |
     ; CHECK-NEXT: [[AND2:%[0-9]+]]:_(s64) = G_AND [[COPY1]], [[C3]]
     ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s64) = G_LSHR [[AND2]], [[C2]](s64)
     ; CHECK-NEXT: [[LSHR1:%[0-9]+]]:_(s64) = G_LSHR [[LSHR]], [[AND1]](s64)
-    ; CHECK-NEXT: [[OR:%[0-9]+]]:_(s64) = G_OR [[SHL]], [[LSHR1]]
+    ; CHECK-NEXT: [[OR:%[0-9]+]]:_(s64) = disjoint G_OR [[SHL]], [[LSHR1]]
     ; CHECK-NEXT: $x10 = COPY [[OR]](s64)
     ; CHECK-NEXT: PseudoRET implicit $x10
     %3:_(s64) = COPY $x10
@@ -63,7 +63,7 @@ body:             |
     ; CHECK-NEXT: [[AND2:%[0-9]+]]:_(s64) = G_AND [[COPY1]], [[C3]]
     ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s64) = G_LSHR [[AND2]], [[C2]](s64)
     ; CHECK-NEXT: [[LSHR1:%[0-9]+]]:_(s64) = G_LSHR [[LSHR]], [[AND1]](s64)
-    ; CHECK-NEXT: [[OR:%[0-9]+]]:_(s64) = G_OR [[SHL]], [[LSHR1]]
+    ; CHECK-NEXT: [[OR:%[0-9]+]]:_(s64) = disjoint G_OR [[SHL]], [[LSHR1]]
     ; CHECK-NEXT: $x10 = COPY [[OR]](s64)
     ; CHECK-NEXT: PseudoRET implicit $x10
     %3:_(s64) = COPY $x10
@@ -101,7 +101,7 @@ body:             |
     ; CHECK-NEXT: [[AND2:%[0-9]+]]:_(s64) = G_AND [[COPY1]], [[C3]]
     ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s64) = G_LSHR [[AND2]], [[C2]](s64)
     ; CHECK-NEXT: [[SRLW:%[0-9]+]]:_(s64) = G_SRLW [[LSHR]], [[AND1]]
-    ; CHECK-NEXT: [[OR:%[0-9]+]]:_(s64) = G_OR [[SLLW]], [[SRLW]]
+    ; CHECK-NEXT: [[OR:%[0-9]+]]:_(s64) = disjoint G_OR [[SLLW]], [[SRLW]]
     ; CHECK-NEXT: $x10 = COPY [[OR]](s64)
     ; CHECK-NEXT: PseudoRET implicit $x10
     %3:_(s64) = COPY $x10
@@ -137,7 +137,7 @@ body:             |
     ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s64) = G_SHL [[COPY]], [[AND]](s64)
     ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s64) = G_LSHR [[COPY1]], [[C2]](s64)
     ; CHECK-NEXT: [[LSHR1:%[0-9]+]]:_(s64) = G_LSHR [[LSHR]], [[AND1]](s64)
-    ; CHECK-NEXT: [[OR:%[0-9]+]]:_(s64) = G_OR [[SHL]], [[LSHR1]]
+    ; CHECK-NEXT: [[OR:%[0-9]+]]:_(s64) = disjoint G_OR [[SHL]], [[LSHR1]]
     ; CHECK-NEXT: $x10 = COPY [[OR]](s64)
     ; CHECK-NEXT: PseudoRET implicit $x10
     %0:_(s64) = COPY $x10
@@ -171,7 +171,7 @@ body:             |
     ; CHECK-NEXT: [[C3:%[0-9]+]]:_(s64) = G_CONSTANT i64 255
     ; CHECK-NEXT: [[AND2:%[0-9]+]]:_(s64) = G_AND [[COPY1]], [[C3]]
     ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s64) = G_LSHR [[AND2]], [[AND]](s64)
-    ; CHECK-NEXT: [[OR:%[0-9]+]]:_(s64) = G_OR [[SHL1]], [[LSHR]]
+    ; CHECK-NEXT: [[OR:%[0-9]+]]:_(s64) = disjoint G_OR [[SHL1]], [[LSHR]]
     ; CHECK-NEXT: $x10 = COPY [[OR]](s64)
     ; CHECK-NEXT: PseudoRET implicit $x10
     %3:_(s64) = COPY $x10
@@ -209,7 +209,7 @@ body:             |
     ; CHECK-NEXT: [[C3:%[0-9]+]]:_(s64) = G_CONSTANT i64 65535
     ; CHECK-NEXT: [[AND2:%[0-9]+]]:_(s64) = G_AND [[COPY1]], [[C3]]
     ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s64) = G_LSHR [[AND2]], [[AND]](s64)
-    ; CHECK-NEXT: [[OR:%[0-9]+]]:_(s64) = G_OR [[SHL1]], [[LSHR]]
+    ; CHECK-NEXT: [[OR:%[0-9]+]]:_(s64) = disjoint G_OR [[SHL1]], [[LSHR]]
     ; CHECK-NEXT: $x10 = COPY [[OR]](s64)
     ; CHECK-NEXT: PseudoRET implicit $x10
     %3:_(s64) = COPY $x10
@@ -245,7 +245,7 @@ body:             |
     ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s64) = G_SHL [[COPY]], [[C2]](s64)
     ; CHECK-NEXT: [[SLLW:%[0-9]+]]:_(s64) = G_SLLW [[SHL]], [[AND1]]
     ; CHECK-NEXT: [[SRLW:%[0-9]+]]:_(s64) = G_SRLW [[COPY1]], [[AND]]
-    ; CHECK-NEXT: [[OR:%[0-9]+]]:_(s64) = G_OR [[SLLW]], [[SRLW]]
+    ; CHECK-NEXT: [[OR:%[0-9]+]]:_(s64) = disjoint G_OR [[SLLW]], [[SRLW]]
     ; CHECK-NEXT: $x10 = COPY [[OR]](s64)
     ; CHECK-NEXT: PseudoRET implicit $x10
     %3:_(s64) = COPY $x10
@@ -281,7 +281,7 @@ body:             |
     ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s64) = G_SHL [[COPY]], [[C2]](s64)
     ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(s64) = G_SHL [[SHL]], [[AND1]](s64)
     ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s64) = G_LSHR [[COPY1]], [[AND]](s64)
-    ; CHECK-NEXT: [[OR:%[0-9]+]]:_(s64) = G_OR [[SHL1]], [[LSHR]]
+    ; CHECK-NEXT: [[OR:%[0-9]+]]:_(s64) = disjoint G_OR [[SHL1]], [[LSHR]]
     ; CHECK-NEXT: $x10 = COPY [[OR]](s64)
     ; CHECK-NEXT: PseudoRET implicit $x10
     %0:_(s64) = COPY $x10
diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/rotl-rotr.ll b/llvm/test/CodeGen/RISCV/GlobalISel/rotl-rotr.ll
new file mode 100644
index 0000000000000..8a786fc9993d2
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/GlobalISel/rotl-rotr.ll
@@ -0,0 +1,3412 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=riscv32 -verify-machineinstrs -global-isel < %s \
+; RUN:   | FileCheck %s -check-prefix=RV32I
+; RUN: llc -mtriple=riscv64 -verify-machineinstrs -enable-legalize-types-checking -global-isel < %s \
+; RUN:   | FileCheck %s -check-prefix=RV64I
+; RUN: llc -mtriple=riscv32 -mattr=+zbb -verify-machineinstrs -global-isel < %s \
+; RUN:   | FileCheck %s -check-prefix=RV32ZBB
+; RUN: llc -mtriple=riscv64 -mattr=+zbb -verify-machineinstrs -global-isel < %s \
+; RUN:   | FileCheck %s -check-prefix=RV64ZBB
+; RUN: llc -mtriple=riscv32 -mattr=+xtheadbb -verify-machineinstrs -global-isel < %s \
+; RUN:   | FileCheck %s -check-prefix=RV32XTHEADBB
+; RUN: llc -mtriple=riscv64 -mattr=+xtheadbb -verify-machineinstrs -global-isel < %s \
+; RUN:   | FileCheck %s -check-prefix=RV64XTHEADBB
+
+; NOTE: -enable-legalize-types-checking is on one command line due to a previous
+; assertion failure on an expensive checks build for @rotr_32_mask_multiple.
+
+; These IR sequences are idioms for rotates. If rotate instructions are
+; supported, they will be turned into ISD::ROTL or ISD::ROTR.
+
+define i32 @rotl_32(i32 %x, i32 %y) nounwind {
+; RV32I-LABEL: rotl_32:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    neg a2, a1
+; RV32I-NEXT:    sll a1, a0, a1
+; RV32I-NEXT:    srl a0, a0, a2
+; RV32I-NEXT:    or a0, a1, a0
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: rotl_32:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    negw a2, a1
+; RV64I-NEXT:    sllw a1, a0, a1
+; RV64I-NEXT:    srlw a0, a0, a2
+; RV64I-NEXT:    or a0, a1, a0
+; RV64I-NEXT:    ret
+;
+; RV32ZBB-LABEL: rotl_32:
+; RV32ZBB:       # %bb.0:
+; RV32ZBB-NEXT:    rol a0, a0, a1
+; RV32ZBB-NEXT:    ret
+;
+; RV64ZBB-LABEL: rotl_32:
+; RV64ZBB:       # %bb.0:
+; RV64ZBB-NEXT:    rolw a0, a0, a1
+; RV64ZBB-NEXT:    ret
+;
+; RV32XTHEADBB-LABEL: rotl_32:
+; RV32XTHEADBB:       # %bb.0:
+; RV32XTHEADBB-NEXT:    neg a2, a1
+; RV32XTHEADBB-NEXT:    sll a1, a0, a1
+; RV32XTHEADBB-NEXT:    srl a0, a0, a2
+; RV32XTHEADBB-NEXT:    or a0, a1, a0
+; RV32XTHEADBB-NEXT:    ret
+;
+; RV64XTHEADBB-LABEL: rotl_32:
+; RV64XTHEADBB:       # %bb.0:
+; RV64XTHEADBB-NEXT:    negw a2, a1
+; RV64XTHEADBB-NEXT:    sllw a1, a0, a1
+; RV64XTHEADBB-NEXT:    srlw a0, a0, a2
+; RV64XTHEADBB-NEXT:    or a0, a1, a0
+; RV64XTHEADBB-NEXT:    ret
+  %z = sub i32 32, %y
+  %b = shl i32 %x, %y
+  %c = lshr i32 %x, %z
+  %d = or i32 %b, %c
+  ret i32 %d
+}
+
+define i32 @rotr_32(i32 %x, i32 %y) nounwind {
+; RV32I-LABEL: rotr_32:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    neg a2, a1
+; RV32I-NEXT:    srl a1, a0, a1
+; RV32I-NEXT:    sll a0, a0, a2
+; RV32I-NEXT:    or a0, a1, a0
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: rotr_32:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    negw a2, a1
+; RV64I-NEXT:    srlw a1, a0, a1
+; RV64I-NEXT:    sllw a0, a0, a2
+; RV64I-NEXT:    or a0, a1, a0
+; RV64I-NEXT:    ret
+;
+; RV32ZBB-LABEL: rotr_32:
+; RV32ZBB:       # %bb.0:
+; RV32ZBB-NEXT:    ror a0, a0, a1
+; RV32ZBB-NEXT:    ret
+;
+; RV64ZBB-LABEL: rotr_32:
+; RV64ZBB:       # %bb.0:
+; RV64ZBB-NEXT:    rorw a0, a0, a1
+; RV64ZBB-NEXT:    ret
+;
+; RV32XTHEADBB-LABEL: rotr_32:
+; RV32XTHEADBB:       # %bb.0:
+; RV32XTHEADBB-NEXT:    neg a2, a1
+; RV32XTHEADBB-NEXT:    srl a1, a0, a1
+; RV32XTHEADBB-NEXT:    sll a0, a0, a2
+; RV32XTHEADBB-NEXT:    or a0, a1, a0
+; RV32XTHEADBB-NEXT:    ret
+;
+; RV64XTHEADBB-LABEL: rotr_32:
+; RV64XTHEADBB:       # %bb.0:
+; RV64XTHEADBB-NEXT:    negw a2, a1
+; RV64XTHEADBB-NEXT:    srlw a1, a0, a1
+; RV64XTHEADBB-NEXT:    sllw a0, a0, a2
+; RV64XTHEADBB-NEXT:    or a0, a1, a0
+; RV64XTHEADBB-NEXT:    ret
+  %z = sub i32 32, %y
+  %b = lshr i32 %x, %y
+  %c = shl i32 %x, %z
+  %d = or i32 %b, %c
+  ret i32 %d
+}
+
+define i64 @rotl_64(i64 %x, i64 %y) nounwind {
+; RV32I-LABEL: rotl_64:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    andi a6, a2, 63
+; RV32I-NEXT:    li a4, 32
+; RV32I-NEXT:    bltu a6, a4, .LBB2_2
+; RV32I-NEXT:  # %bb.1:
+; RV32I-NEXT:    li a3, 0
+; RV32I-NEXT:    sll a7, a0, a6
+; RV32I-NEXT:    j .LBB2_3
+; RV32I-NEXT:  .LBB2_2:
+; RV32I-NEXT:    sll a3, a0, a2
+; RV32I-NEXT:    neg a5, a6
+; RV32I-NEXT:    srl a5, a0, a5
+; RV32I-NEXT:    sll a7, a1, a2
+; RV32I-NEXT:    or a7, a5, a7
+; RV32I-NEXT:  .LBB2_3:
+; RV32I-NEXT:    neg a5, a2
+; RV32I-NEXT:    mv a2, a1
+; RV32I-NEXT:    beqz a6, .LBB2_5
+; RV32I-NEXT:  # %bb.4:
+; RV32I-NEXT:    mv a2, a7
+; RV32I-NEXT:  .LBB2_5:
+; RV32I-NEXT:    andi a6, a5, 63
+; RV32I-NEXT:    bltu a6, a4, .LBB2_7
+; RV32I-NEXT:  # %bb.6:
+; RV32I-NEXT:    srl a7, a1, a6
+; RV32I-NEXT:    bnez a6, .LBB2_8
+; RV32I-NEXT:    j .LBB2_9
+; RV32I-NEXT:  .LBB2_7:
+; RV32I-NEXT:    srl a7, a0, a5
+; RV32I-NEXT:    neg t0, a6
+; RV32I-NEXT:    sll t0, a1, t0
+; RV32I-NEXT:    or a7, a7, t0
+; RV32I-NEXT:    beqz a6, .LBB2_9
+; RV32I-NEXT:  .LBB2_8:
+; RV32I-NEXT:    mv a0, a7
+; RV32I-NEXT:  .LBB2_9:
+; RV32I-NEXT:    bltu a6, a4, .LBB2_11
+; RV32I-NEXT:  # %bb.10:
+; RV32I-NEXT:    li a1, 0
+; RV32I-NEXT:    j .LBB2_12
+; RV32I-NEXT:  .LBB2_11:
+; RV32I-NEXT:    srl a1, a1, a5
+; RV32I-NEXT:  .LBB2_12:
+; RV32I-NEXT:    or a0, a3, a0
+; RV32I-NEXT:    or a1, a2, a1
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: rotl_64:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    negw a2, a1
+; RV64I-NEXT:    sll a1, a0, a1
+; RV64I-NEXT:    srl a0, a0, a2
+; RV64I-NEXT:    or a0, a1, a0
+; RV64I-NEXT:    ret
+;
+; RV32ZBB-LABEL: rotl_64:
+; RV32ZBB:       # %bb.0:
+; RV32ZBB-NEXT:    andi a6, a2, 63
+; RV32ZBB-NEXT:    li a4, 32
+; RV32ZBB-NEXT:    bltu a6, a4, .LBB2_2
+; RV32ZBB-NEXT:  # %bb.1:
+; RV32ZBB-NEXT:    li a3, 0
+; RV32ZBB-NEXT:    sll a7, a0, a6
+; RV32ZBB-NEXT:    j .LBB2_3
+; RV32ZBB-NEXT:  .LBB2_2:
+; RV32ZBB-NEXT:    sll a3, a0, a2
+; RV32ZBB-NEXT:    neg a5, a6
+; RV32ZBB-NEXT:    srl a5, a0, a5
+; RV32ZBB-NEXT:    sll a7, a1, a2
+; RV32ZBB-NEXT:    or a7, a5, a7
+; RV32ZBB-NEXT:  .LBB2_3:
+; RV32ZBB-NEXT:    neg a5, a2
+; RV32ZBB-NEXT:    mv a2, a1
+; RV32ZBB-NEXT:    beqz a6, .LBB2_5
+; RV32ZBB-NEXT:  # %bb.4:
+; RV32ZBB-NEXT:    mv a2, a7
+; RV32ZBB-NEXT:  .LBB2_5:
+; RV32ZBB-NEXT:    andi a6, a5, 63
+; RV32ZBB-NEXT:    bltu a6, a4, .LBB2_7
+; RV32ZBB-NEXT:  # %bb.6:
+; RV32ZBB-NEXT:    srl a7, a1, a6
+; RV32ZBB-NEXT:    bnez a6, .LBB2_8
+; RV32ZBB-NEXT:    j .LBB2_9
+; RV32ZBB-NEXT:  .LBB2_7:
+; RV32ZBB-NEXT:    srl a7, a0, a5
+; RV32ZBB-NEXT:    neg t0, a6
+; RV32ZBB-NEXT:    sll t0, a1, t0
+; RV32ZBB-NEXT:    or a7, a7, t0
+; RV32ZBB-NEXT:    beqz a6, .LBB2_9
+; RV32ZBB-NEXT:  .LBB2_8:
+; RV32ZBB-NEXT:    mv a0, a7
+; RV32ZBB-NEXT:  .LBB2_9:
+; RV32ZBB-NEXT:    bltu a6, a4, .LBB2_11
+; RV32ZBB-NEXT:  # %bb.10:
+; RV32ZBB-NEXT:    li a1, 0
+; RV32ZBB-NEXT:    j .LBB2_12
+; RV32ZBB-NEXT:  .LBB2_11:
+; RV32ZBB-NEXT:    srl a1, a1, a5
+; RV32ZBB-NEXT:  .LBB2_12:
+; RV32ZBB-NEXT:    or a0, a3, a0
+; RV32ZBB-NEXT:    or a1, a2, a1
+; RV32ZBB-NEXT:    ret
+;
+; RV64ZBB-LABEL: rotl_64:
+; RV64ZBB:       # %bb.0:
+; RV64ZBB-NEXT:    rol a0, a0, a1
+; RV64ZBB-NEXT:    ret
+;
+; RV32XTHEADBB-LABEL: rotl_64:
+; RV32XTHEADBB:       # %bb.0:
+; RV32XTHEADBB-NEXT:    andi a6, a2, 63
+; RV32XTHEADBB-NEXT:    li a4, 32
+; RV32XTHEADBB-NEXT:    bltu a6, a4, .LBB2_2
+; RV32XTHEADBB-NEXT:  # %bb.1:
+; RV32XTHEADBB-NEXT:    li a3, 0
+; RV32XTHEADBB-NEXT:    sll a7, a0, a6
+; RV32XTHEADBB-NEXT:    j .LBB2_3
+; RV32XTHEADBB-NEXT:  .LBB2_2:
+; RV32XTHEADBB-NEXT:    sll a3, a0, a2
+; RV32XTHEADBB-NEXT:    neg a5, a6
+; RV32XTHEADBB-NEXT:    srl a5, a0, a5
+; RV32XTHEADBB-NEXT:    sll a7, a1, a2
+; RV32XTHEADBB-NEXT:    or a7, a5, a7
+; RV32XTHEADBB-NEXT:  .LBB2_3:
+; RV32XTHEADBB-NEXT:    neg a5, a2
+; RV32XTHEADBB-NEXT:    mv a2, a1
+; RV32XTHEADBB-NEXT:    beqz a6, .LBB2_5
+; RV32XTHEADBB-NEXT:  # %bb.4:
+; RV32XTHEADBB-NEXT:    mv a2, a7
+; RV32XTHEADBB-NEXT:  .LBB2_5:
+; RV32XTHEADBB-NEXT:    andi a6, a5, 63
+; RV32XTHEADBB-NEXT:    bltu a6, a4, .LBB2_7
+; RV32XTHEADBB-NEXT:  # %bb.6:
+; RV32XTHEADBB-NEXT:    srl a7, a1, a6
+; RV32XTHEADBB-NEXT:    bnez a6, .LBB2_8
+; RV32XTHEADBB-NEXT:    j .LBB2_9
+; RV32XTHEADBB-NEXT:  .LBB2_7:
+; RV32XTHEADBB-NEXT:    srl a7, a0, a5
+; RV32XTHEADBB-NEXT:    neg t0, a6
+; RV32XTHEADBB-NEXT:    sll t0, a1, t0
+; RV32XTHEADBB-NEXT:    or a7, a7, t0
+; RV32XTHEADBB-NEXT:    beqz a6, .LBB2_9
+; RV32XTHEADBB-NEXT:  .LBB2_8:
+; RV32XTHEADBB-NEXT:    mv a0, a7
+; RV32XTHEADBB-NEXT:  .LBB2_9:
+; RV32XTHEADBB-NEXT:    bltu a6, a4, .LBB2_11
+; RV32XTHEADBB-NEXT:  # %bb.10:
+; RV32XTHEADBB-NEXT:    li a1, 0
+; RV32XTHEADBB-NEXT:    j .LBB2_12
+; RV32XTHEADBB-NEXT:  .LBB2_11:
+; RV32XTHEADBB-NEXT:    srl a1, a1, a5
+; RV32XTHEADBB-NEXT:  .LBB2_12:
+; RV32XTHEADBB-NEXT:    or a0, a3, a0
+; RV32XTHEADBB-NEXT:    or a1, a2, a1
+; RV32XTHEADBB-NEXT:    ret
+;
+; RV64XTHEADBB-LABEL: rotl_64:
+; RV64XTHEADBB:       # %bb.0:
+; RV64XTHEADBB-NEXT:    negw a2, a1
+; RV64XTHEADBB-NEXT:    sll a1, a0, a1
+; RV64XTHEADBB-NEXT:    srl a0, a0, a2
+; RV64XTHEADBB-NEXT:    or a0, a1, a0
+; RV64XTHEADBB-NEXT:    ret
+  %z = sub i64 64, %y
+  %b = shl i64 %x, %y
+  %c = lshr i64 %x, %z
+  %d = or i64 %b, %c
+  ret i64 %d
+}
+
+define i64 @rotr_64(i64 %x, i64 %y) nounwind {
+; RV32I-LABEL: rotr_64:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    andi a5, a2, 63
+; RV32I-NEXT:    li a4, 32
+; RV32I-NEXT:    bltu a5, a4, .LBB3_2
+; RV32I-NEXT:  # %bb.1:
+; RV32I-NEXT:    srl a6, a1, a5
+; RV32I-NEXT:    mv a3, a0
+; RV32I-NEXT:    bnez a5, .LBB3_3
+; RV32I-NEXT:    j .LBB3_4
+; RV32I-NEXT:  .LBB3_2:
+; RV32I-NEXT:    srl a3, a0, a2
+; RV32I-NEXT:    neg a6, a5
+; RV32I-NEXT:    sll a6, a1, a6
+; RV32I-NEXT:    or a6, a3, a6
+; RV32I-NEXT:    mv a3, a0
+; RV32I-NEXT:    beqz a5, .LBB3_4
+; RV32I-NEXT:  .LBB3_3:
+; RV32I-NEXT:    mv a3, a6
+; RV32I-NEXT:  .LBB3_4:
+; RV32I-NEXT:    neg a6, a2
+; RV32I-NEXT:    bltu a5, a4, .LBB3_7
+; RV32I-NEXT:  # %bb.5:
+; RV32I-NEXT:    li a2, 0
+; RV32I-NEXT:    andi a5, a6, 63
+; RV32I-NEXT:    bgeu a5, a4, .LBB3_8
+; RV32I-NEXT:  .LBB3_6:
+; RV32I-NEXT:    sll a4, a0, a6
+; RV32I-NEXT:    neg a7, a5
+; RV32I-NEXT:    srl a0, a0, a7
+; RV32I-NEXT:    sll a6, a1, a6
+; RV32I-NEXT:    or a0, a0, a6
+; RV32I-NEXT:    bnez a5, .LBB3_9
+; RV32I-NEXT:    j .LBB3_10
+; RV32I-NEXT:  .LBB3_7:
+; RV32I-NEXT:    srl a2, a1, a2
+; RV32I-NEXT:    andi a5, a6, 63
+; RV32I-NEXT:    bltu a5, a4, .LBB3_6
+; RV32I-NEXT:  .LBB3_8:
+; RV32I-NEXT:    li a4, 0
+; RV32I-NEXT:    sll a0, a0, a5
+; RV32I-NEXT:    beqz a5, .LBB3_10
+; RV32I-NEXT:  .LBB3_9:
+; RV32I-NEXT:    mv a1, a0
+; RV32I-NEXT:  .LBB3_10:
+; RV32I-NEXT:    or a0, a3, a4
+; RV32I-NEXT:    or a1, a2, a1
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: rotr_64:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    negw a2, a1
+; RV64I-NEXT:    srl a1, a0, a1
+; RV64I-NEXT:    sll a0, a0, a2
+; RV64I-NEXT:    or a0, a1, a0
+; RV64I-NEXT:    ret
+;
+; RV32ZBB-LABEL: rotr_64:
+; RV32ZBB:       # %bb.0:
+; RV32ZBB-NEXT:    andi a5, a2, 63
+; RV32ZBB-NEXT:    li a4, 32
+; RV32ZBB-NEXT:    bltu a5, a4, .LBB3_2
+; RV32ZBB-NEXT:  # %bb.1:
+; RV32ZBB-NEXT:    srl a6, a1, a5
+; RV32ZBB-NEXT:    mv a3, a0
+; RV32ZBB-NEXT:    bnez a5, .LBB3_3
+; RV32ZBB-NEXT:    j .LBB3_4
+; RV32ZBB-NEXT:  .LBB3_2:
+; RV32ZBB-NEXT:    srl a3, a0, a2
+; RV32ZBB-NEXT:    neg a6, a5
+; RV32ZBB-NEXT:    sll a6, a1, a6
+; RV32ZBB-NEXT:    or a6, a3, a6
+; RV32ZBB-NEXT:    mv a3, a0
+; RV32ZBB-NEXT:    beqz a5, .LBB3_4
+; RV32ZBB-NEXT:  .LBB3_3:
+; RV32ZBB-NEXT:    mv a3, a6
+; RV32ZBB-NEXT:  .LBB3_4:
+; RV32ZBB-NEXT:    neg a6, a2
+; RV32ZBB-NEXT:    bltu a5, a4, .LBB3_7
+; RV32ZBB-NEXT:  # %bb.5:
+; RV32ZBB-NEXT:    li a2, 0
+; RV32ZBB-NEXT:    andi a5, a6, 63
+; RV32ZBB-NEXT:    bgeu a5, a4, .LBB3_8
+; RV32ZBB-NEXT:  .LBB3_6:
+; RV32ZBB-NEXT:    sll a4, a0, a6
+; RV32ZBB-NEXT:    neg a7, a5
+; RV32ZBB-NEXT:    srl a0, a0, a7
+; RV32ZBB-NEXT:    sll a6, a1, a6
+; RV32ZBB-NEXT:    or a0, a0, a6
+; RV32ZBB-NEXT:    bnez a5, .LBB3_9
+; RV32ZBB-NEXT:    j .LBB3_10
+; RV32ZBB-NEXT:  .LBB3_7:
+; RV32ZBB-NEXT:    srl a2, a1, a2
+; RV32ZBB-NEXT:    andi a5, a6, 63
+; RV32ZBB-NEXT:    bltu a5, a4, .LBB3_6
+; RV32ZBB-NEXT:  .LBB3_8:
+; RV32ZBB-NEXT:    li a4, 0
+; RV32ZBB-NEXT:    sll a0, a0, a5
+; RV32ZBB-NEXT:    beqz a5, .LBB3_10
+; RV32ZBB-NEXT:  .LBB3_9:
+; RV32ZBB-NEXT:    mv a1, a0
+; RV32ZBB-NEXT:  .LBB3_10:
+; RV32ZBB-NEXT:    or a0, a3, a4
+; RV32ZBB-NEXT:    or a1, a2, a1
+; RV32ZBB-NEXT:    ret
+;
+; RV64ZBB-LABEL: rotr_64:
+; RV64ZBB:       # %bb.0:
+; RV64ZBB-NEXT:    ror a0, a0, a1
+; RV64ZBB-NEXT:    ret
+;
+; RV32XTHEADBB-LABEL: rotr_64:
+; RV32XTHEADBB:       # %bb.0:
+; RV32XTHEADBB-NEXT:    andi a5, a2, 63
+; RV32XTHEADBB-NEXT:    li a4, 32
+; RV32XTHEADBB-NEXT:    bltu a5, a4, .LBB3_2
+; RV32XTHEADBB-NEXT:  # %bb.1:
+; RV32XTHEADBB-NEXT:    srl a6, a1, a5
+; RV32XTHEADBB-NEXT:    mv a3, a0
+; RV32XTHEADBB-NEXT:    bnez a5, .LBB3_3
+; RV32XTHEADBB-NEXT:    j .LBB3_4
+; RV32XTHEADBB-NEXT:  .LBB3_2:
+; RV32XTHEADBB-NEXT:    srl a3, a0, a2
+; RV32XTHEADBB-NEXT:    neg a6, a5
+; RV32XTHEADBB-NEXT:    sll a6, a1, a6
+; RV32XTHEADBB-NEXT:    or a6, a3, a6
+; RV32XTHEADBB-NEXT:    mv a3, a0
+; RV32XTHEADBB-NEXT:    beqz a5, .LBB3_4
+; RV32XTHEADBB-NEXT:  .LBB3_3:
+; RV32XTHEADBB-NEXT:    mv a3, a6
+; RV32XTHEADBB-NEXT:  .LBB3_4:
+; RV32XTHEADBB-NEXT:    neg a6, a2
+; RV32XTHEADBB-NEXT:    bltu a5, a4, .LBB3_7
+; RV32XTHEADBB-NEXT:  # %bb.5:
+; RV32XTHEADBB-NEXT:    li a2, 0
+; RV32XTHEADBB-NEXT:    andi a5, a6, 63
+; RV32XTHEADBB-NEXT:    bgeu a5, a4, .LBB3_8
+; RV32XTHEADBB-NEXT:  .LBB3_6:
+; RV32XTHEADBB-NEXT:    sll a4, a0, a6
+; RV32XTHEADBB-NEXT:    neg a7, a5
+; RV32XTHEADBB-NEXT:    srl a0, a0, a7
+; RV32XTHEADBB-NEXT:    sll a6, a1, a6
+; RV32XTHEADBB-NEXT:    or a0, a0, a6
+; RV32XTHEADBB-NEXT:    bnez a5, .LBB3_9
+; RV32XTHEADBB-NEXT:    j .LBB3_10
+; RV32XTHEADBB-NEXT:  .LBB3_7:
+; RV32XTHEADBB-NEXT:    srl a2, a1, a2
+; RV32XTHEADBB-NEXT:    andi a5, a6, 63
+; RV32XTHEADBB-NEXT:    bltu a5, a4, .LBB3_6
+; RV32XTHEADBB-NEXT:  .LBB3_8:
+; RV32XTHEADBB-NEXT:    li a4, 0
+; RV32XTHEADBB-NEXT:    sll a0, a0, a5
+; RV32XTHEADBB-NEXT:    beqz a5, .LBB3_10
+; RV32XTHEADBB-NEXT:  .LBB3_9:
+; RV32XTHEADBB-NEXT:    mv a1, a0
+; RV32XTHEADBB-NEXT:  .LBB3_10:
+; RV32XTHEADBB-NEXT:    or a0, a3, a4
+; RV32XTHEADBB-NEXT:    or a1, a2, a1
+; RV32XTHEADBB-NEXT:    ret
+;
+; RV64XTHEADBB-LABEL: rotr_64:
+; RV64XTHEADBB:       # %bb.0:
+; RV64XTHEADBB-NEXT:    negw a2, a1
+; RV64XTHEADBB-NEXT:    srl a1, a0, a1
+; RV64XTHEADBB-NEXT:    sll a0, a0, a2
+; RV64XTHEADBB-NEXT:    or a0, a1, a0
+; RV64XTHEADBB-NEXT:    ret
+  %z = sub i64 64, %y
+  %b = lshr i64 %x, %y
+  %c = shl i64 %x, %z
+  %d = or i64 %b, %c
+  ret i64 %d
+}
+
+define i32 @rotl_32_mask(i32 %x, i32 %y) nounwind {
+; RV32I-LABEL: rotl_32_mask:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    neg a2, a1
+; RV32I-NEXT:    sll a1, a0, a1
+; RV32I-NEXT:    srl a0, a0, a2
+; RV32I-NEXT:    or a0, a1, a0
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: rotl_32_mask:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    negw a2, a1
+; RV64I-NEXT:    sllw a1, a0, a1
+; RV64I-NEXT:    srlw a0, a0, a2
+; RV64I-NEXT:    or a0, a1, a0
+; RV64I-NEXT:    ret
+;
+; RV32ZBB-LABEL: rotl_32_mask:
+; RV32ZBB:       # %bb.0:
+; RV32ZBB-NEXT:    neg a2, a1
+; RV32ZBB-NEXT:    sll a1, a0, a1
+; RV32ZBB-NEXT:    srl a0, a0, a2
+; RV32ZBB-NEXT:    or a0, a1, a0
+; RV32ZBB-NEXT:    ret
+;
+; RV64ZBB-LABEL: rotl_32_mask:
+; RV64ZBB:       # %bb.0:
+; RV64ZBB-NEXT:    negw a2, a1
+; RV64ZBB-NEXT:    sllw a1, a0, a1
+; RV64ZBB-NEXT:    srlw a0, a0, a2
+; RV64ZBB-NEXT:    or a0, a1, a0
+; RV64ZBB-NEXT:    ret
+;
+; RV32XTHEADBB-LABEL: rotl_32_mask:
+; RV32XTHEADBB:       # %bb.0:
+; RV32XTHEADBB-NEXT:    neg a2, a1
+; RV32XTHEADBB-NEXT:    sll a1, a0, a1
+; RV32XTHEADBB-NEXT:    srl a0, a0, a2
+; RV32XTHEADBB-NEXT:    or a0, a1, a0
+; RV32XTHEADBB-NEXT:    ret
+;
+; RV64XTHEADBB-LABEL: rotl_32_mask:
+; RV64XTHEADBB:       # %bb.0:
+; RV64XTHEADBB-NEXT:    negw a2, a1
+; RV64XTHEADBB-NEXT:    sllw a1, a0, a1
+; RV64XTHEADBB-NEXT:    srlw a0, a0, a2
+; RV64XTHEADBB-NEXT:    or a0, a1, a0
+; RV64XTHEADBB-NEXT:    ret
+  %z = sub i32 0, %y
+  %and = and i32 %z, 31
+  %b = shl i32 %x, %y
+  %c = lshr i32 %x, %and
+  %d = or i32 %b, %c
+  ret i32 %d
+}
+
+define i32 @rotl_32_mask_and_63_and_31(i32 %x, i32 %y) nounwind {
+; RV32I-LABEL: rotl_32_mask_and_63_and_31:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    sll a2, a0, a1
+; RV32I-NEXT:    neg a1, a1
+; RV32I-NEXT:    srl a0, a0, a1
+; RV32I-NEXT:    or a0, a2, a0
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: rotl_32_mask_and_63_and_31:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    sllw a2, a0, a1
+; RV64I-NEXT:    negw a1, a1
+; RV64I-NEXT:    srlw a0, a0, a1
+; RV64I-NEXT:    or a0, a2, a0
+; RV64I-NEXT:    ret
+;
+; RV32ZBB-LABEL: rotl_32_mask_and_63_and_31:
+; RV32ZBB:       # %bb.0:
+; RV32ZBB-NEXT:    sll a2, a0, a1
+; RV32ZBB-NEXT:    neg a1, a1
+; RV32ZBB-NEXT:    srl a0, a0, a1
+; RV32ZBB-NEXT:    or a0, a2, a0
+; RV32ZBB-NEXT:    ret
+;
+; RV64ZBB-LABEL: rotl_32_mask_and_63_and_31:
+; RV64ZBB:       # %bb.0:
+; RV64ZBB-NEXT:    sllw a2, a0, a1
+; RV64ZBB-NEXT:    negw a1, a1
+; RV64ZBB-NEXT:    srlw a0, a0, a1
+; RV64ZBB-NEXT:    or a0, a2, a0
+; RV64ZBB-NEXT:    ret
+;
+; RV32XTHEADBB-LABEL: rotl_32_mask_and_63_and_31:
+; RV32XTHEADBB:       # %bb.0:
+; RV32XTHEADBB-NEXT:    sll a2, a0, a1
+; RV32XTHEADBB-NEXT:    neg a1, a1
+; RV32XTHEADBB-NEXT:    srl a0, a0, a1
+; RV32XTHEADBB-NEXT:    or a0, a2, a0
+; RV32XTHEADBB-NEXT:    ret
+;
+; RV64XTHEADBB-LABEL: rotl_32_mask_and_63_and_31:
+; RV64XTHEADBB:       # %bb.0:
+; RV64XTHEADBB-NEXT:    sllw a2, a0, a1
+; RV64XTHEADBB-NEXT:    negw a1, a1
+; RV64XTHEADBB-NEXT:    srlw a0, a0, a1
+; RV64XTHEADBB-NEXT:    or a0, a2, a0
+; RV64XTHEADBB-NEXT:    ret
+  %a = and i32 %y, 63
+  %b = shl i32 %x, %a
+  %c = sub i32 0, %y
+  %d = and i32 %c, 31
+  %e = lshr i32 %x, %d
+  %f = or i32 %b, %e
+  ret i32 %f
+}
+
+define i32 @rotl_32_mask_or_64_or_32(i32 %x, i32 %y) nounwind {
+; RV32I-LABEL: rotl_32_mask_or_64_or_32:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    ori a1, a1, 64
+; RV32I-NEXT:    sll a0, a0, a1
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: rotl_32_mask_or_64_or_32:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    ori a1, a1, 64
+; RV64I-NEXT:    sllw a0, a0, a1
+; RV64I-NEXT:    ret
+;
+; RV32ZBB-LABEL: rotl_32_mask_or_64_or_32:
+; RV32ZBB:       # %bb.0:
+; RV32ZBB-NEXT:    ori a1, a1, 64
+; RV32ZBB-NEXT:    sll a0, a0, a1
+; RV32ZBB-NEXT:    ret
+;
+; RV64ZBB-LABEL: rotl_32_mask_or_64_or_32:
+; RV64ZBB:       # %bb.0:
+; RV64ZBB-NEXT:    ori a1, a1, 64
+; RV64ZBB-NEXT:    sllw a0, a0, a1
+; RV64ZBB-NEXT:    ret
+;
+; RV32XTHEADBB-LABEL: rotl_32_mask_or_64_or_32:
+; RV32XTHEADBB:       # %bb.0:
+; RV32XTHEADBB-NEXT:    ori a1, a1, 64
+; RV32XTHEADBB-NEXT:    sll a0, a0, a1
+; RV32XTHEADBB-NEXT:    ret
+;
+; RV64XTHEADBB-LABEL: rotl_32_mask_or_64_or_32:
+; RV64XTHEADBB:       # %bb.0:
+; RV64XTHEADBB-NEXT:    ori a1, a1, 64
+; RV64XTHEADBB-NEXT:    sllw a0, a0, a1
+; RV64XTHEADBB-NEXT:    ret
+  %a = or i32 %y, 64
+  %b = shl i32 %x, %a
+  %c = sub i32 0, %y
+  %d = or i32 %c, 32
+  %e = lshr i32 %x, %d
+  %f = or i32 %b, %e
+  ret i32 %f
+}
+
+define i32 @rotr_32_mask(i32 %x, i32 %y) nounwind {
+; RV32I-LABEL: rotr_32_mask:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    neg a2, a1
+; RV32I-NEXT:    srl a1, a0, a1
+; RV32I-NEXT:    sll a0, a0, a2
+; RV32I-NEXT:    or a0, a1, a0
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: rotr_32_mask:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    negw a2, a1
+; RV64I-NEXT:    srlw a1, a0, a1
+; RV64I-NEXT:    sllw a0, a0, a2
+; RV64I-NEXT:    or a0, a1, a0
+; RV64I-NEXT:    ret
+;
+; RV32ZBB-LABEL: rotr_32_mask:
+; RV32ZBB:       # %bb.0:
+; RV32ZBB-NEXT:    neg a2, a1
+; RV32ZBB-NEXT:    srl a1, a0, a1
+; RV32ZBB-NEXT:    sll a0, a0, a2
+; RV32ZBB-NEXT:    or a0, a1, a0
+; RV32ZBB-NEXT:    ret
+;
+; RV64ZBB-LABEL: rotr_32_mask:
+; RV64ZBB:       # %bb.0:
+; RV64ZBB-NEXT:    negw a2, a1
+; RV64ZBB-NEXT:    srlw a1, a0, a1
+; RV64ZBB-NEXT:    sllw a0, a0, a2
+; RV64ZBB-NEXT:    or a0, a1, a0
+; RV64ZBB-NEXT:    ret
+;
+; RV32XTHEADBB-LABEL: rotr_32_mask:
+; RV32XTHEADBB:       # %bb.0:
+; RV32XTHEADBB-NEXT:    neg a2, a1
+; RV32XTHEADBB-NEXT:    srl a1, a0, a1
+; RV32XTHEADBB-NEXT:    sll a0, a0, a2
+; RV32XTHEADBB-NEXT:    or a0, a1, a0
+; RV32XTHEADBB-NEXT:    ret
+;
+; RV64XTHEADBB-LABEL: rotr_32_mask:
+; RV64XTHEADBB:       # %bb.0:
+; RV64XTHEADBB-NEXT:    negw a2, a1
+; RV64XTHEADBB-NEXT:    srlw a1, a0, a1
+; RV64XTHEADBB-NEXT:    sllw a0, a0, a2
+; RV64XTHEADBB-NEXT:    or a0, a1, a0
+; RV64XTHEADBB-NEXT:    ret
+  %z = sub i32 0, %y
+  %and = and i32 %z, 31
+  %b = lshr i32 %x, %y
+  %c = shl i32 %x, %and
+  %d = or i32 %b, %c
+  ret i32 %d
+}
+
+define i32 @rotr_32_mask_and_63_and_31(i32 %x, i32 %y) nounwind {
+; RV32I-LABEL: rotr_32_mask_and_63_and_31:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    srl a2, a0, a1
+; RV32I-NEXT:    neg a1, a1
+; RV32I-NEXT:    sll a0, a0, a1
+; RV32I-NEXT:    or a0, a2, a0
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: rotr_32_mask_and_63_and_31:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    srlw a2, a0, a1
+; RV64I-NEXT:    negw a1, a1
+; RV64I-NEXT:    sllw a0, a0, a1
+; RV64I-NEXT:    or a0, a2, a0
+; RV64I-NEXT:    ret
+;
+; RV32ZBB-LABEL: rotr_32_mask_and_63_and_31:
+; RV32ZBB:       # %bb.0:
+; RV32ZBB-NEXT:    srl a2, a0, a1
+; RV32ZBB-NEXT:    neg a1, a1
+; RV32ZBB-NEXT:    sll a0, a0, a1
+; RV32ZBB-NEXT:    or a0, a2, a0
+; RV32ZBB-NEXT:    ret
+;
+; RV64ZBB-LABEL: rotr_32_mask_and_63_and_31:
+; RV64ZBB:       # %bb.0:
+; RV64ZBB-NEXT:    srlw a2, a0, a1
+; RV64ZBB-NEXT:    negw a1, a1
+; RV64ZBB-NEXT:    sllw a0, a0, a1
+; RV64ZBB-NEXT:    or a0, a2, a0
+; RV64ZBB-NEXT:    ret
+;
+; RV32XTHEADBB-LABEL: rotr_32_mask_and_63_and_31:
+; RV32XTHEADBB:       # %bb.0:
+; RV32XTHEADBB-NEXT:    srl a2, a0, a1
+; RV32XTHEADBB-NEXT:    neg a1, a1
+; RV32XTHEADBB-NEXT:    sll a0, a0, a1
+; RV32XTHEADBB-NEXT:    or a0, a2, a0
+; RV32XTHEADBB-NEXT:    ret
+;
+; RV64XTHEADBB-LABEL: rotr_32_mask_and_63_and_31:
+; RV64XTHEADBB:       # %bb.0:
+; RV64XTHEADBB-NEXT:    srlw a2, a0, a1
+; RV64XTHEADBB-NEXT:    negw a1, a1
+; RV64XTHEADBB-NEXT:    sllw a0, a0, a1
+; RV64XTHEADBB-NEXT:    or a0, a2, a0
+; RV64XTHEADBB-NEXT:    ret
+  %a = and i32 %y, 63
+  %b = lshr i32 %x, %a
+  %c = sub i32 0, %y
+  %d = and i32 %c, 31
+  %e = shl i32 %x, %d
+  %f = or i32 %b, %e
+  ret i32 %f
+}
+
+define i32 @rotr_32_mask_or_64_or_32(i32 %x, i32 %y) nounwind {
+; RV32I-LABEL: rotr_32_mask_or_64_or_32:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    ori a1, a1, 64
+; RV32I-NEXT:    srl a0, a0, a1
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: rotr_32_mask_or_64_or_32:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    ori a1, a1, 64
+; RV64I-NEXT:    srlw a0, a0, a1
+; RV64I-NEXT:    ret
+;
+; RV32ZBB-LABEL: rotr_32_mask_or_64_or_32:
+; RV32ZBB:       # %bb.0:
+; RV32ZBB-NEXT:    ori a1, a1, 64
+; RV32ZBB-NEXT:    srl a0, a0, a1
+; RV32ZBB-NEXT:    ret
+;
+; RV64ZBB-LABEL: rotr_32_mask_or_64_or_32:
+; RV64ZBB:       # %bb.0:
+; RV64ZBB-NEXT:    ori a1, a1, 64
+; RV64ZBB-NEXT:    srlw a0, a0, a1
+; RV64ZBB-NEXT:    ret
+;
+; RV32XTHEADBB-LABEL: rotr_32_mask_or_64_or_32:
+; RV32XTHEADBB:       # %bb.0:
+; RV32XTHEADBB-NEXT:    ori a1, a1, 64
+; RV32XTHEADBB-NEXT:    srl a0, a0, a1
+; RV32XTHEADBB-NEXT:    ret
+;
+; RV64XTHEADBB-LABEL: rotr_32_mask_or_64_or_32:
+; RV64XTHEADBB:       # %bb.0:
+; RV64XTHEADBB-NEXT:    ori a1, a1, 64
+; RV64XTHEADBB-NEXT:    srlw a0, a0, a1
+; RV64XTHEADBB-NEXT:    ret
+  %a = or i32 %y, 64
+  %b = lshr i32 %x, %a
+  %c = sub i32 0, %y
+  %d = or i32 %c, 32
+  %e = shl i32 %x, %d
+  %f = or i32 %b, %e
+  ret i32 %f
+}
+
+define i64 @rotl_64_mask(i64 %x, i64 %y) nounwind {
+; RV32I-LABEL: rotl_64_mask:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    li a5, 32
+; RV32I-NEXT:    neg a4, a2
+; RV32I-NEXT:    bltu a2, a5, .LBB10_2
+; RV32I-NEXT:  # %bb.1:
+; RV32I-NEXT:    li a3, 0
+; RV32I-NEXT:    sll t0, a0, a2
+; RV32I-NEXT:    j .LBB10_3
+; RV32I-NEXT:  .LBB10_2:
+; RV32I-NEXT:    sll a3, a0, a2
+; RV32I-NEXT:    neg a6, a2
+; RV32I-NEXT:    srl a6, a0, a6
+; RV32I-NEXT:    sll a7, a1, a2
+; RV32I-NEXT:    or t0, a6, a7
+; RV32I-NEXT:  .LBB10_3:
+; RV32I-NEXT:    andi a7, a4, 63
+; RV32I-NEXT:    mv a6, a1
+; RV32I-NEXT:    beqz a2, .LBB10_5
+; RV32I-NEXT:  # %bb.4:
+; RV32I-NEXT:    mv a6, t0
+; RV32I-NEXT:  .LBB10_5:
+; RV32I-NEXT:    bltu a7, a5, .LBB10_7
+; RV32I-NEXT:  # %bb.6:
+; RV32I-NEXT:    srl a2, a1, a7
+; RV32I-NEXT:    bnez a7, .LBB10_8
+; RV32I-NEXT:    j .LBB10_9
+; RV32I-NEXT:  .LBB10_7:
+; RV32I-NEXT:    srl a2, a0, a4
+; RV32I-NEXT:    neg t0, a7
+; RV32I-NEXT:    sll t0, a1, t0
+; RV32I-NEXT:    or a2, a2, t0
+; RV32I-NEXT:    beqz a7, .LBB10_9
+; RV32I-NEXT:  .LBB10_8:
+; RV32I-NEXT:    mv a0, a2
+; RV32I-NEXT:  .LBB10_9:
+; RV32I-NEXT:    bltu a7, a5, .LBB10_11
+; RV32I-NEXT:  # %bb.10:
+; RV32I-NEXT:    li a1, 0
+; RV32I-NEXT:    j .LBB10_12
+; RV32I-NEXT:  .LBB10_11:
+; RV32I-NEXT:    srl a1, a1, a4
+; RV32I-NEXT:  .LBB10_12:
+; RV32I-NEXT:    or a0, a3, a0
+; RV32I-NEXT:    or a1, a6, a1
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: rotl_64_mask:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    negw a2, a1
+; RV64I-NEXT:    sll a1, a0, a1
+; RV64I-NEXT:    srl a0, a0, a2
+; RV64I-NEXT:    or a0, a1, a0
+; RV64I-NEXT:    ret
+;
+; RV32ZBB-LABEL: rotl_64_mask:
+; RV32ZBB:       # %bb.0:
+; RV32ZBB-NEXT:    li a5, 32
+; RV32ZBB-NEXT:    neg a4, a2
+; RV32ZBB-NEXT:    bltu a2, a5, .LBB10_2
+; RV32ZBB-NEXT:  # %bb.1:
+; RV32ZBB-NEXT:    li a3, 0
+; RV32ZBB-NEXT:    sll t0, a0, a2
+; RV32ZBB-NEXT:    j .LBB10_3
+; RV32ZBB-NEXT:  .LBB10_2:
+; RV32ZBB-NEXT:    sll a3, a0, a2
+; RV32ZBB-NEXT:    neg a6, a2
+; RV32ZBB-NEXT:    srl a6, a0, a6
+; RV32ZBB-NEXT:    sll a7, a1, a2
+; RV32ZBB-NEXT:    or t0, a6, a7
+; RV32ZBB-NEXT:  .LBB10_3:
+; RV32ZBB-NEXT:    andi a7, a4, 63
+; RV32ZBB-NEXT:    mv a6, a1
+; RV32ZBB-NEXT:    beqz a2, .LBB10_5
+; RV32ZBB-NEXT:  # %bb.4:
+; RV32ZBB-NEXT:    mv a6, t0
+; RV32ZBB-NEXT:  .LBB10_5:
+; RV32ZBB-NEXT:    bltu a7, a5, .LBB10_7
+; RV32ZBB-NEXT:  # %bb.6:
+; RV32ZBB-NEXT:    srl a2, a1, a7
+; RV32ZBB-NEXT:    bnez a7, .LBB10_8
+; RV32ZBB-NEXT:    j .LBB10_9
+; RV32ZBB-NEXT:  .LBB10_7:
+; RV32ZBB-NEXT:    srl a2, a0, a4
+; RV32ZBB-NEXT:    neg t0, a7
+; RV32ZBB-NEXT:    sll t0, a1, t0
+; RV32ZBB-NEXT:    or a2, a2, t0
+; RV32ZBB-NEXT:    beqz a7, .LBB10_9
+; RV32ZBB-NEXT:  .LBB10_8:
+; RV32ZBB-NEXT:    mv a0, a2
+; RV32ZBB-NEXT:  .LBB10_9:
+; RV32ZBB-NEXT:    bltu a7, a5, .LBB10_11
+; RV32ZBB-NEXT:  # %bb.10:
+; RV32ZBB-NEXT:    li a1, 0
+; RV32ZBB-NEXT:    j .LBB10_12
+; RV32ZBB-NEXT:  .LBB10_11:
+; RV32ZBB-NEXT:    srl a1, a1, a4
+; RV32ZBB-NEXT:  .LBB10_12:
+; RV32ZBB-NEXT:    or a0, a3, a0
+; RV32ZBB-NEXT:    or a1, a6, a1
+; RV32ZBB-NEXT:    ret
+;
+; RV64ZBB-LABEL: rotl_64_mask:
+; RV64ZBB:       # %bb.0:
+; RV64ZBB-NEXT:    negw a2, a1
+; RV64ZBB-NEXT:    sll a1, a0, a1
+; RV64ZBB-NEXT:    srl a0, a0, a2
+; RV64ZBB-NEXT:    or a0, a1, a0
+; RV64ZBB-NEXT:    ret
+;
+; RV32XTHEADBB-LABEL: rotl_64_mask:
+; RV32XTHEADBB:       # %bb.0:
+; RV32XTHEADBB-NEXT:    li a5, 32
+; RV32XTHEADBB-NEXT:    neg a4, a2
+; RV32XTHEADBB-NEXT:    bltu a2, a5, .LBB10_2
+; RV32XTHEADBB-NEXT:  # %bb.1:
+; RV32XTHEADBB-NEXT:    li a3, 0
+; RV32XTHEADBB-NEXT:    sll t0, a0, a2
+; RV32XTHEADBB-NEXT:    j .LBB10_3
+; RV32XTHEADBB-NEXT:  .LBB10_2:
+; RV32XTHEADBB-NEXT:    sll a3, a0, a2
+; RV32XTHEADBB-NEXT:    neg a6, a2
+; RV32XTHEADBB-NEXT:    srl a6, a0, a6
+; RV32XTHEADBB-NEXT:    sll a7, a1, a2
+; RV32XTHEADBB-NEXT:    or t0, a6, a7
+; RV32XTHEADBB-NEXT:  .LBB10_3:
+; RV32XTHEADBB-NEXT:    andi a7, a4, 63
+; RV32XTHEADBB-NEXT:    mv a6, a1
+; RV32XTHEADBB-NEXT:    beqz a2, .LBB10_5
+; RV32XTHEADBB-NEXT:  # %bb.4:
+; RV32XTHEADBB-NEXT:    mv a6, t0
+; RV32XTHEADBB-NEXT:  .LBB10_5:
+; RV32XTHEADBB-NEXT:    bltu a7, a5, .LBB10_7
+; RV32XTHEADBB-NEXT:  # %bb.6:
+; RV32XTHEADBB-NEXT:    srl a2, a1, a7
+; RV32XTHEADBB-NEXT:    bnez a7, .LBB10_8
+; RV32XTHEADBB-NEXT:    j .LBB10_9
+; RV32XTHEADBB-NEXT:  .LBB10_7:
+; RV32XTHEADBB-NEXT:    srl a2, a0, a4
+; RV32XTHEADBB-NEXT:    neg t0, a7
+; RV32XTHEADBB-NEXT:    sll t0, a1, t0
+; RV32XTHEADBB-NEXT:    or a2, a2, t0
+; RV32XTHEADBB-NEXT:    beqz a7, .LBB10_9
+; RV32XTHEADBB-NEXT:  .LBB10_8:
+; RV32XTHEADBB-NEXT:    mv a0, a2
+; RV32XTHEADBB-NEXT:  .LBB10_9:
+; RV32XTHEADBB-NEXT:    bltu a7, a5, .LBB10_11
+; RV32XTHEADBB-NEXT:  # %bb.10:
+; RV32XTHEADBB-NEXT:    li a1, 0
+; RV32XTHEADBB-NEXT:    j .LBB10_12
+; RV32XTHEADBB-NEXT:  .LBB10_11:
+; RV32XTHEADBB-NEXT:    srl a1, a1, a4
+; RV32XTHEADBB-NEXT:  .LBB10_12:
+; RV32XTHEADBB-NEXT:    or a0, a3, a0
+; RV32XTHEADBB-NEXT:    or a1, a6, a1
+; RV32XTHEADBB-NEXT:    ret
+;
+; RV64XTHEADBB-LABEL: rotl_64_mask:
+; RV64XTHEADBB:       # %bb.0:
+; RV64XTHEADBB-NEXT:    negw a2, a1
+; RV64XTHEADBB-NEXT:    sll a1, a0, a1
+; RV64XTHEADBB-NEXT:    srl a0, a0, a2
+; RV64XTHEADBB-NEXT:    or a0, a1, a0
+; RV64XTHEADBB-NEXT:    ret
+  %z = sub i64 0, %y
+  %and = and i64 %z, 63
+  %b = shl i64 %x, %y
+  %c = lshr i64 %x, %and
+  %d = or i64 %b, %c
+  ret i64 %d
+}
+
+define i64 @rotl_64_mask_and_127_and_63(i64 %x, i64 %y) nounwind {
+; RV32I-LABEL: rotl_64_mask_and_127_and_63:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    andi a6, a2, 127
+; RV32I-NEXT:    li a4, 32
+; RV32I-NEXT:    bltu a6, a4, .LBB11_2
+; RV32I-NEXT:  # %bb.1:
+; RV32I-NEXT:    li a3, 0
+; RV32I-NEXT:    sll a7, a0, a6
+; RV32I-NEXT:    mv a5, a1
+; RV32I-NEXT:    bnez a6, .LBB11_3
+; RV32I-NEXT:    j .LBB11_4
+; RV32I-NEXT:  .LBB11_2:
+; RV32I-NEXT:    sll a3, a0, a2
+; RV32I-NEXT:    neg a5, a6
+; RV32I-NEXT:    srl a5, a0, a5
+; RV32I-NEXT:    sll a7, a1, a2
+; RV32I-NEXT:    or a7, a5, a7
+; RV32I-NEXT:    mv a5, a1
+; RV32I-NEXT:    beqz a6, .LBB11_4
+; RV32I-NEXT:  .LBB11_3:
+; RV32I-NEXT:    mv a5, a7
+; RV32I-NEXT:  .LBB11_4:
+; RV32I-NEXT:    neg a2, a2
+; RV32I-NEXT:    andi a6, a2, 63
+; RV32I-NEXT:    bltu a6, a4, .LBB11_6
+; RV32I-NEXT:  # %bb.5:
+; RV32I-NEXT:    srl a7, a1, a6
+; RV32I-NEXT:    bnez a6, .LBB11_7
+; RV32I-NEXT:    j .LBB11_8
+; RV32I-NEXT:  .LBB11_6:
+; RV32I-NEXT:    srl a7, a0, a2
+; RV32I-NEXT:    neg t0, a6
+; RV32I-NEXT:    sll t0, a1, t0
+; RV32I-NEXT:    or a7, a7, t0
+; RV32I-NEXT:    beqz a6, .LBB11_8
+; RV32I-NEXT:  .LBB11_7:
+; RV32I-NEXT:    mv a0, a7
+; RV32I-NEXT:  .LBB11_8:
+; RV32I-NEXT:    bltu a6, a4, .LBB11_10
+; RV32I-NEXT:  # %bb.9:
+; RV32I-NEXT:    li a1, 0
+; RV32I-NEXT:    j .LBB11_11
+; RV32I-NEXT:  .LBB11_10:
+; RV32I-NEXT:    srl a1, a1, a2
+; RV32I-NEXT:  .LBB11_11:
+; RV32I-NEXT:    or a0, a3, a0
+; RV32I-NEXT:    or a1, a5, a1
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: rotl_64_mask_and_127_and_63:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    sll a2, a0, a1
+; RV64I-NEXT:    negw a1, a1
+; RV64I-NEXT:    srl a0, a0, a1
+; RV64I-NEXT:    or a0, a2, a0
+; RV64I-NEXT:    ret
+;
+; RV32ZBB-LABEL: rotl_64_mask_and_127_and_63:
+; RV32ZBB:       # %bb.0:
+; RV32ZBB-NEXT:    andi a6, a2, 127
+; RV32ZBB-NEXT:    li a4, 32
+; RV32ZBB-NEXT:    bltu a6, a4, .LBB11_2
+; RV32ZBB-NEXT:  # %bb.1:
+; RV32ZBB-NEXT:    li a3, 0
+; RV32ZBB-NEXT:    sll a7, a0, a6
+; RV32ZBB-NEXT:    mv a5, a1
+; RV32ZBB-NEXT:    bnez a6, .LBB11_3
+; RV32ZBB-NEXT:    j .LBB11_4
+; RV32ZBB-NEXT:  .LBB11_2:
+; RV32ZBB-NEXT:    sll a3, a0, a2
+; RV32ZBB-NEXT:    neg a5, a6
+; RV32ZBB-NEXT:    srl a5, a0, a5
+; RV32ZBB-NEXT:    sll a7, a1, a2
+; RV32ZBB-NEXT:    or a7, a5, a7
+; RV32ZBB-NEXT:    mv a5, a1
+; RV32ZBB-NEXT:    beqz a6, .LBB11_4
+; RV32ZBB-NEXT:  .LBB11_3:
+; RV32ZBB-NEXT:    mv a5, a7
+; RV32ZBB-NEXT:  .LBB11_4:
+; RV32ZBB-NEXT:    neg a2, a2
+; RV32ZBB-NEXT:    andi a6, a2, 63
+; RV32ZBB-NEXT:    bltu a6, a4, .LBB11_6
+; RV32ZBB-NEXT:  # %bb.5:
+; RV32ZBB-NEXT:    srl a7, a1, a6
+; RV32ZBB-NEXT:    bnez a6, .LBB11_7
+; RV32ZBB-NEXT:    j .LBB11_8
+; RV32ZBB-NEXT:  .LBB11_6:
+; RV32ZBB-NEXT:    srl a7, a0, a2
+; RV32ZBB-NEXT:    neg t0, a6
+; RV32ZBB-NEXT:    sll t0, a1, t0
+; RV32ZBB-NEXT:    or a7, a7, t0
+; RV32ZBB-NEXT:    beqz a6, .LBB11_8
+; RV32ZBB-NEXT:  .LBB11_7:
+; RV32ZBB-NEXT:    mv a0, a7
+; RV32ZBB-NEXT:  .LBB11_8:
+; RV32ZBB-NEXT:    bltu a6, a4, .LBB11_10
+; RV32ZBB-NEXT:  # %bb.9:
+; RV32ZBB-NEXT:    li a1, 0
+; RV32ZBB-NEXT:    j .LBB11_11
+; RV32ZBB-NEXT:  .LBB11_10:
+; RV32ZBB-NEXT:    srl a1, a1, a2
+; RV32ZBB-NEXT:  .LBB11_11:
+; RV32ZBB-NEXT:    or a0, a3, a0
+; RV32ZBB-NEXT:    or a1, a5, a1
+; RV32ZBB-NEXT:    ret
+;
+; RV64ZBB-LABEL: rotl_64_mask_and_127_and_63:
+; RV64ZBB:       # %bb.0:
+; RV64ZBB-NEXT:    sll a2, a0, a1
+; RV64ZBB-NEXT:    negw a1, a1
+; RV64ZBB-NEXT:    srl a0, a0, a1
+; RV64ZBB-NEXT:    or a0, a2, a0
+; RV64ZBB-NEXT:    ret
+;
+; RV32XTHEADBB-LABEL: rotl_64_mask_and_127_and_63:
+; RV32XTHEADBB:       # %bb.0:
+; RV32XTHEADBB-NEXT:    andi a6, a2, 127
+; RV32XTHEADBB-NEXT:    li a4, 32
+; RV32XTHEADBB-NEXT:    bltu a6, a4, .LBB11_2
+; RV32XTHEADBB-NEXT:  # %bb.1:
+; RV32XTHEADBB-NEXT:    li a3, 0
+; RV32XTHEADBB-NEXT:    sll a7, a0, a6
+; RV32XTHEADBB-NEXT:    mv a5, a1
+; RV32XTHEADBB-NEXT:    bnez a6, .LBB11_3
+; RV32XTHEADBB-NEXT:    j .LBB11_4
+; RV32XTHEADBB-NEXT:  .LBB11_2:
+; RV32XTHEADBB-NEXT:    sll a3, a0, a2
+; RV32XTHEADBB-NEXT:    neg a5, a6
+; RV32XTHEADBB-NEXT:    srl a5, a0, a5
+; RV32XTHEADBB-NEXT:    sll a7, a1, a2
+; RV32XTHEADBB-NEXT:    or a7, a5, a7
+; RV32XTHEADBB-NEXT:    mv a5, a1
+; RV32XTHEADBB-NEXT:    beqz a6, .LBB11_4
+; RV32XTHEADBB-NEXT:  .LBB11_3:
+; RV32XTHEADBB-NEXT:    mv a5, a7
+; RV32XTHEADBB-NEXT:  .LBB11_4:
+; RV32XTHEADBB-NEXT:    neg a2, a2
+; RV32XTHEADBB-NEXT:    andi a6, a2, 63
+; RV32XTHEADBB-NEXT:    bltu a6, a4, .LBB11_6
+; RV32XTHEADBB-NEXT:  # %bb.5:
+; RV32XTHEADBB-NEXT:    srl a7, a1, a6
+; RV32XTHEADBB-NEXT:    bnez a6, .LBB11_7
+; RV32XTHEADBB-NEXT:    j .LBB11_8
+; RV32XTHEADBB-NEXT:  .LBB11_6:
+; RV32XTHEADBB-NEXT:    srl a7, a0, a2
+; RV32XTHEADBB-NEXT:    neg t0, a6
+; RV32XTHEADBB-NEXT:    sll t0, a1, t0
+; RV32XTHEADBB-NEXT:    or a7, a7, t0
+; RV32XTHEADBB-NEXT:    beqz a6, .LBB11_8
+; RV32XTHEADBB-NEXT:  .LBB11_7:
+; RV32XTHEADBB-NEXT:    mv a0, a7
+; RV32XTHEADBB-NEXT:  .LBB11_8:
+; RV32XTHEADBB-NEXT:    bltu a6, a4, .LBB11_10
+; RV32XTHEADBB-NEXT:  # %bb.9:
+; RV32XTHEADBB-NEXT:    li a1, 0
+; RV32XTHEADBB-NEXT:    j .LBB11_11
+; RV32XTHEADBB-NEXT:  .LBB11_10:
+; RV32XTHEADBB-NEXT:    srl a1, a1, a2
+; RV32XTHEADBB-NEXT:  .LBB11_11:
+; RV32XTHEADBB-NEXT:    or a0, a3, a0
+; RV32XTHEADBB-NEXT:    or a1, a5, a1
+; RV32XTHEADBB-NEXT:    ret
+;
+; RV64XTHEADBB-LABEL: rotl_64_mask_and_127_and_63:
+; RV64XTHEADBB:       # %bb.0:
+; RV64XTHEADBB-NEXT:    sll a2, a0, a1
+; RV64XTHEADBB-NEXT:    negw a1, a1
+; RV64XTHEADBB-NEXT:    srl a0, a0, a1
+; RV64XTHEADBB-NEXT:    or a0, a2, a0
+; RV64XTHEADBB-NEXT:    ret
+  %a = and i64 %y, 127
+  %b = shl i64 %x, %a
+  %c = sub i64 0, %y
+  %d = and i64 %c, 63
+  %e = lshr i64 %x, %d
+  %f = or i64 %b, %e
+  ret i64 %f
+}
+
+define i64 @rotl_64_mask_or_128_or_64(i64 %x, i64 %y) nounwind {
+; RV32I-LABEL: rotl_64_mask_or_128_or_64:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    mv a3, a0
+; RV32I-NEXT:    ori a2, a2, 128
+; RV32I-NEXT:    li a0, 32
+; RV32I-NEXT:    bltu a2, a0, .LBB12_2
+; RV32I-NEXT:  # %bb.1:
+; RV32I-NEXT:    li a0, 0
+; RV32I-NEXT:    sll a3, a3, a2
+; RV32I-NEXT:    bnez a2, .LBB12_3
+; RV32I-NEXT:    j .LBB12_4
+; RV32I-NEXT:  .LBB12_2:
+; RV32I-NEXT:    sll a0, a3, a2
+; RV32I-NEXT:    neg a4, a2
+; RV32I-NEXT:    srl a3, a3, a4
+; RV32I-NEXT:    sll a4, a1, a2
+; RV32I-NEXT:    or a3, a3, a4
+; RV32I-NEXT:    beqz a2, .LBB12_4
+; RV32I-NEXT:  .LBB12_3:
+; RV32I-NEXT:    mv a1, a3
+; RV32I-NEXT:  .LBB12_4:
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: rotl_64_mask_or_128_or_64:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    ori a1, a1, 128
+; RV64I-NEXT:    sll a0, a0, a1
+; RV64I-NEXT:    ret
+;
+; RV32ZBB-LABEL: rotl_64_mask_or_128_or_64:
+; RV32ZBB:       # %bb.0:
+; RV32ZBB-NEXT:    mv a3, a0
+; RV32ZBB-NEXT:    ori a2, a2, 128
+; RV32ZBB-NEXT:    li a0, 32
+; RV32ZBB-NEXT:    bltu a2, a0, .LBB12_2
+; RV32ZBB-NEXT:  # %bb.1:
+; RV32ZBB-NEXT:    li a0, 0
+; RV32ZBB-NEXT:    sll a3, a3, a2
+; RV32ZBB-NEXT:    bnez a2, .LBB12_3
+; RV32ZBB-NEXT:    j .LBB12_4
+; RV32ZBB-NEXT:  .LBB12_2:
+; RV32ZBB-NEXT:    sll a0, a3, a2
+; RV32ZBB-NEXT:    neg a4, a2
+; RV32ZBB-NEXT:    srl a3, a3, a4
+; RV32ZBB-NEXT:    sll a4, a1, a2
+; RV32ZBB-NEXT:    or a3, a3, a4
+; RV32ZBB-NEXT:    beqz a2, .LBB12_4
+; RV32ZBB-NEXT:  .LBB12_3:
+; RV32ZBB-NEXT:    mv a1, a3
+; RV32ZBB-NEXT:  .LBB12_4:
+; RV32ZBB-NEXT:    ret
+;
+; RV64ZBB-LABEL: rotl_64_mask_or_128_or_64:
+; RV64ZBB:       # %bb.0:
+; RV64ZBB-NEXT:    ori a1, a1, 128
+; RV64ZBB-NEXT:    sll a0, a0, a1
+; RV64ZBB-NEXT:    ret
+;
+; RV32XTHEADBB-LABEL: rotl_64_mask_or_128_or_64:
+; RV32XTHEADBB:       # %bb.0:
+; RV32XTHEADBB-NEXT:    mv a3, a0
+; RV32XTHEADBB-NEXT:    ori a2, a2, 128
+; RV32XTHEADBB-NEXT:    li a0, 32
+; RV32XTHEADBB-NEXT:    bltu a2, a0, .LBB12_2
+; RV32XTHEADBB-NEXT:  # %bb.1:
+; RV32XTHEADBB-NEXT:    li a0, 0
+; RV32XTHEADBB-NEXT:    sll a3, a3, a2
+; RV32XTHEADBB-NEXT:    bnez a2, .LBB12_3
+; RV32XTHEADBB-NEXT:    j .LBB12_4
+; RV32XTHEADBB-NEXT:  .LBB12_2:
+; RV32XTHEADBB-NEXT:    sll a0, a3, a2
+; RV32XTHEADBB-NEXT:    neg a4, a2
+; RV32XTHEADBB-NEXT:    srl a3, a3, a4
+; RV32XTHEADBB-NEXT:    sll a4, a1, a2
+; RV32XTHEADBB-NEXT:    or a3, a3, a4
+; RV32XTHEADBB-NEXT:    beqz a2, .LBB12_4
+; RV32XTHEADBB-NEXT:  .LBB12_3:
+; RV32XTHEADBB-NEXT:    mv a1, a3
+; RV32XTHEADBB-NEXT:  .LBB12_4:
+; RV32XTHEADBB-NEXT:    ret
+;
+; RV64XTHEADBB-LABEL: rotl_64_mask_or_128_or_64:
+; RV64XTHEADBB:       # %bb.0:
+; RV64XTHEADBB-NEXT:    ori a1, a1, 128
+; RV64XTHEADBB-NEXT:    sll a0, a0, a1
+; RV64XTHEADBB-NEXT:    ret
+  %a = or i64 %y, 128
+  %b = shl i64 %x, %a
+  %c = sub i64 0, %y
+  %d = or i64 %c, 64
+  %e = lshr i64 %x, %d
+  %f = or i64 %b, %e
+  ret i64 %f
+}
+
+define i64 @rotr_64_mask(i64 %x, i64 %y) nounwind {
+; RV32I-LABEL: rotr_64_mask:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    li a4, 32
+; RV32I-NEXT:    bltu a2, a4, .LBB13_2
+; RV32I-NEXT:  # %bb.1:
+; RV32I-NEXT:    srl a5, a1, a2
+; RV32I-NEXT:    j .LBB13_3
+; RV32I-NEXT:  .LBB13_2:
+; RV32I-NEXT:    srl a3, a0, a2
+; RV32I-NEXT:    neg a5, a2
+; RV32I-NEXT:    sll a5, a1, a5
+; RV32I-NEXT:    or a5, a3, a5
+; RV32I-NEXT:  .LBB13_3:
+; RV32I-NEXT:    neg a6, a2
+; RV32I-NEXT:    mv a3, a0
+; RV32I-NEXT:    beqz a2, .LBB13_5
+; RV32I-NEXT:  # %bb.4:
+; RV32I-NEXT:    mv a3, a5
+; RV32I-NEXT:  .LBB13_5:
+; RV32I-NEXT:    andi a5, a6, 63
+; RV32I-NEXT:    bltu a2, a4, .LBB13_8
+; RV32I-NEXT:  # %bb.6:
+; RV32I-NEXT:    li a2, 0
+; RV32I-NEXT:    bgeu a5, a4, .LBB13_9
+; RV32I-NEXT:  .LBB13_7:
+; RV32I-NEXT:    sll a4, a0, a6
+; RV32I-NEXT:    neg a7, a5
+; RV32I-NEXT:    srl a0, a0, a7
+; RV32I-NEXT:    sll a6, a1, a6
+; RV32I-NEXT:    or a0, a0, a6
+; RV32I-NEXT:    bnez a5, .LBB13_10
+; RV32I-NEXT:    j .LBB13_11
+; RV32I-NEXT:  .LBB13_8:
+; RV32I-NEXT:    srl a2, a1, a2
+; RV32I-NEXT:    bltu a5, a4, .LBB13_7
+; RV32I-NEXT:  .LBB13_9:
+; RV32I-NEXT:    li a4, 0
+; RV32I-NEXT:    sll a0, a0, a5
+; RV32I-NEXT:    beqz a5, .LBB13_11
+; RV32I-NEXT:  .LBB13_10:
+; RV32I-NEXT:    mv a1, a0
+; RV32I-NEXT:  .LBB13_11:
+; RV32I-NEXT:    or a0, a3, a4
+; RV32I-NEXT:    or a1, a2, a1
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: rotr_64_mask:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    negw a2, a1
+; RV64I-NEXT:    srl a1, a0, a1
+; RV64I-NEXT:    sll a0, a0, a2
+; RV64I-NEXT:    or a0, a1, a0
+; RV64I-NEXT:    ret
+;
+; RV32ZBB-LABEL: rotr_64_mask:
+; RV32ZBB:       # %bb.0:
+; RV32ZBB-NEXT:    li a4, 32
+; RV32ZBB-NEXT:    bltu a2, a4, .LBB13_2
+; RV32ZBB-NEXT:  # %bb.1:
+; RV32ZBB-NEXT:    srl a5, a1, a2
+; RV32ZBB-NEXT:    j .LBB13_3
+; RV32ZBB-NEXT:  .LBB13_2:
+; RV32ZBB-NEXT:    srl a3, a0, a2
+; RV32ZBB-NEXT:    neg a5, a2
+; RV32ZBB-NEXT:    sll a5, a1, a5
+; RV32ZBB-NEXT:    or a5, a3, a5
+; RV32ZBB-NEXT:  .LBB13_3:
+; RV32ZBB-NEXT:    neg a6, a2
+; RV32ZBB-NEXT:    mv a3, a0
+; RV32ZBB-NEXT:    beqz a2, .LBB13_5
+; RV32ZBB-NEXT:  # %bb.4:
+; RV32ZBB-NEXT:    mv a3, a5
+; RV32ZBB-NEXT:  .LBB13_5:
+; RV32ZBB-NEXT:    andi a5, a6, 63
+; RV32ZBB-NEXT:    bltu a2, a4, .LBB13_8
+; RV32ZBB-NEXT:  # %bb.6:
+; RV32ZBB-NEXT:    li a2, 0
+; RV32ZBB-NEXT:    bgeu a5, a4, .LBB13_9
+; RV32ZBB-NEXT:  .LBB13_7:
+; RV32ZBB-NEXT:    sll a4, a0, a6
+; RV32ZBB-NEXT:    neg a7, a5
+; RV32ZBB-NEXT:    srl a0, a0, a7
+; RV32ZBB-NEXT:    sll a6, a1, a6
+; RV32ZBB-NEXT:    or a0, a0, a6
+; RV32ZBB-NEXT:    bnez a5, .LBB13_10
+; RV32ZBB-NEXT:    j .LBB13_11
+; RV32ZBB-NEXT:  .LBB13_8:
+; RV32ZBB-NEXT:    srl a2, a1, a2
+; RV32ZBB-NEXT:    bltu a5, a4, .LBB13_7
+; RV32ZBB-NEXT:  .LBB13_9:
+; RV32ZBB-NEXT:    li a4, 0
+; RV32ZBB-NEXT:    sll a0, a0, a5
+; RV32ZBB-NEXT:    beqz a5, .LBB13_11
+; RV32ZBB-NEXT:  .LBB13_10:
+; RV32ZBB-NEXT:    mv a1, a0
+; RV32ZBB-NEXT:  .LBB13_11:
+; RV32ZBB-NEXT:    or a0, a3, a4
+; RV32ZBB-NEXT:    or a1, a2, a1
+; RV32ZBB-NEXT:    ret
+;
+; RV64ZBB-LABEL: rotr_64_mask:
+; RV64ZBB:       # %bb.0:
+; RV64ZBB-NEXT:    negw a2, a1
+; RV64ZBB-NEXT:    srl a1, a0, a1
+; RV64ZBB-NEXT:    sll a0, a0, a2
+; RV64ZBB-NEXT:    or a0, a1, a0
+; RV64ZBB-NEXT:    ret
+;
+; RV32XTHEADBB-LABEL: rotr_64_mask:
+; RV32XTHEADBB:       # %bb.0:
+; RV32XTHEADBB-NEXT:    li a4, 32
+; RV32XTHEADBB-NEXT:    bltu a2, a4, .LBB13_2
+; RV32XTHEADBB-NEXT:  # %bb.1:
+; RV32XTHEADBB-NEXT:    srl a5, a1, a2
+; RV32XTHEADBB-NEXT:    j .LBB13_3
+; RV32XTHEADBB-NEXT:  .LBB13_2:
+; RV32XTHEADBB-NEXT:    srl a3, a0, a2
+; RV32XTHEADBB-NEXT:    neg a5, a2
+; RV32XTHEADBB-NEXT:    sll a5, a1, a5
+; RV32XTHEADBB-NEXT:    or a5, a3, a5
+; RV32XTHEADBB-NEXT:  .LBB13_3:
+; RV32XTHEADBB-NEXT:    neg a6, a2
+; RV32XTHEADBB-NEXT:    mv a3, a0
+; RV32XTHEADBB-NEXT:    beqz a2, .LBB13_5
+; RV32XTHEADBB-NEXT:  # %bb.4:
+; RV32XTHEADBB-NEXT:    mv a3, a5
+; RV32XTHEADBB-NEXT:  .LBB13_5:
+; RV32XTHEADBB-NEXT:    andi a5, a6, 63
+; RV32XTHEADBB-NEXT:    bltu a2, a4, .LBB13_8
+; RV32XTHEADBB-NEXT:  # %bb.6:
+; RV32XTHEADBB-NEXT:    li a2, 0
+; RV32XTHEADBB-NEXT:    bgeu a5, a4, .LBB13_9
+; RV32XTHEADBB-NEXT:  .LBB13_7:
+; RV32XTHEADBB-NEXT:    sll a4, a0, a6
+; RV32XTHEADBB-NEXT:    neg a7, a5
+; RV32XTHEADBB-NEXT:    srl a0, a0, a7
+; RV32XTHEADBB-NEXT:    sll a6, a1, a6
+; RV32XTHEADBB-NEXT:    or a0, a0, a6
+; RV32XTHEADBB-NEXT:    bnez a5, .LBB13_10
+; RV32XTHEADBB-NEXT:    j .LBB13_11
+; RV32XTHEADBB-NEXT:  .LBB13_8:
+; RV32XTHEADBB-NEXT:    srl a2, a1, a2
+; RV32XTHEADBB-NEXT:    bltu a5, a4, .LBB13_7
+; RV32XTHEADBB-NEXT:  .LBB13_9:
+; RV32XTHEADBB-NEXT:    li a4, 0
+; RV32XTHEADBB-NEXT:    sll a0, a0, a5
+; RV32XTHEADBB-NEXT:    beqz a5, .LBB13_11
+; RV32XTHEADBB-NEXT:  .LBB13_10:
+; RV32XTHEADBB-NEXT:    mv a1, a0
+; RV32XTHEADBB-NEXT:  .LBB13_11:
+; RV32XTHEADBB-NEXT:    or a0, a3, a4
+; RV32XTHEADBB-NEXT:    or a1, a2, a1
+; RV32XTHEADBB-NEXT:    ret
+;
+; RV64XTHEADBB-LABEL: rotr_64_mask:
+; RV64XTHEADBB:       # %bb.0:
+; RV64XTHEADBB-NEXT:    negw a2, a1
+; RV64XTHEADBB-NEXT:    srl a1, a0, a1
+; RV64XTHEADBB-NEXT:    sll a0, a0, a2
+; RV64XTHEADBB-NEXT:    or a0, a1, a0
+; RV64XTHEADBB-NEXT:    ret
+  %z = sub i64 0, %y
+  %and = and i64 %z, 63
+  %b = lshr i64 %x, %y
+  %c = shl i64 %x, %and
+  %d = or i64 %b, %c
+  ret i64 %d
+}
+
+define i64 @rotr_64_mask_and_127_and_63(i64 %x, i64 %y) nounwind {
+; RV32I-LABEL: rotr_64_mask_and_127_and_63:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    andi a4, a2, 127
+; RV32I-NEXT:    li a5, 32
+; RV32I-NEXT:    bltu a4, a5, .LBB14_2
+; RV32I-NEXT:  # %bb.1:
+; RV32I-NEXT:    srl a6, a1, a4
+; RV32I-NEXT:    mv a3, a0
+; RV32I-NEXT:    bnez a4, .LBB14_3
+; RV32I-NEXT:    j .LBB14_4
+; RV32I-NEXT:  .LBB14_2:
+; RV32I-NEXT:    srl a3, a0, a2
+; RV32I-NEXT:    neg a6, a4
+; RV32I-NEXT:    sll a6, a1, a6
+; RV32I-NEXT:    or a6, a3, a6
+; RV32I-NEXT:    mv a3, a0
+; RV32I-NEXT:    beqz a4, .LBB14_4
+; RV32I-NEXT:  .LBB14_3:
+; RV32I-NEXT:    mv a3, a6
+; RV32I-NEXT:  .LBB14_4:
+; RV32I-NEXT:    bltu a4, a5, .LBB14_6
+; RV32I-NEXT:  # %bb.5:
+; RV32I-NEXT:    li a4, 0
+; RV32I-NEXT:    j .LBB14_7
+; RV32I-NEXT:  .LBB14_6:
+; RV32I-NEXT:    srl a4, a1, a2
+; RV32I-NEXT:  .LBB14_7:
+; RV32I-NEXT:    neg a7, a2
+; RV32I-NEXT:    andi a6, a7, 63
+; RV32I-NEXT:    bltu a6, a5, .LBB14_9
+; RV32I-NEXT:  # %bb.8:
+; RV32I-NEXT:    li a2, 0
+; RV32I-NEXT:    sll a0, a0, a6
+; RV32I-NEXT:    bnez a6, .LBB14_10
+; RV32I-NEXT:    j .LBB14_11
+; RV32I-NEXT:  .LBB14_9:
+; RV32I-NEXT:    sll a2, a0, a7
+; RV32I-NEXT:    neg a5, a6
+; RV32I-NEXT:    srl a0, a0, a5
+; RV32I-NEXT:    sll a5, a1, a7
+; RV32I-NEXT:    or a0, a0, a5
+; RV32I-NEXT:    beqz a6, .LBB14_11
+; RV32I-NEXT:  .LBB14_10:
+; RV32I-NEXT:    mv a1, a0
+; RV32I-NEXT:  .LBB14_11:
+; RV32I-NEXT:    or a0, a3, a2
+; RV32I-NEXT:    or a1, a4, a1
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: rotr_64_mask_and_127_and_63:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    srl a2, a0, a1
+; RV64I-NEXT:    negw a1, a1
+; RV64I-NEXT:    sll a0, a0, a1
+; RV64I-NEXT:    or a0, a2, a0
+; RV64I-NEXT:    ret
+;
+; RV32ZBB-LABEL: rotr_64_mask_and_127_and_63:
+; RV32ZBB:       # %bb.0:
+; RV32ZBB-NEXT:    andi a4, a2, 127
+; RV32ZBB-NEXT:    li a5, 32
+; RV32ZBB-NEXT:    bltu a4, a5, .LBB14_2
+; RV32ZBB-NEXT:  # %bb.1:
+; RV32ZBB-NEXT:    srl a6, a1, a4
+; RV32ZBB-NEXT:    mv a3, a0
+; RV32ZBB-NEXT:    bnez a4, .LBB14_3
+; RV32ZBB-NEXT:    j .LBB14_4
+; RV32ZBB-NEXT:  .LBB14_2:
+; RV32ZBB-NEXT:    srl a3, a0, a2
+; RV32ZBB-NEXT:    neg a6, a4
+; RV32ZBB-NEXT:    sll a6, a1, a6
+; RV32ZBB-NEXT:    or a6, a3, a6
+; RV32ZBB-NEXT:    mv a3, a0
+; RV32ZBB-NEXT:    beqz a4, .LBB14_4
+; RV32ZBB-NEXT:  .LBB14_3:
+; RV32ZBB-NEXT:    mv a3, a6
+; RV32ZBB-NEXT:  .LBB14_4:
+; RV32ZBB-NEXT:    bltu a4, a5, .LBB14_6
+; RV32ZBB-NEXT:  # %bb.5:
+; RV32ZBB-NEXT:    li a4, 0
+; RV32ZBB-NEXT:    j .LBB14_7
+; RV32ZBB-NEXT:  .LBB14_6:
+; RV32ZBB-NEXT:    srl a4, a1, a2
+; RV32ZBB-NEXT:  .LBB14_7:
+; RV32ZBB-NEXT:    neg a7, a2
+; RV32ZBB-NEXT:    andi a6, a7, 63
+; RV32ZBB-NEXT:    bltu a6, a5, .LBB14_9
+; RV32ZBB-NEXT:  # %bb.8:
+; RV32ZBB-NEXT:    li a2, 0
+; RV32ZBB-NEXT:    sll a0, a0, a6
+; RV32ZBB-NEXT:    bnez a6, .LBB14_10
+; RV32ZBB-NEXT:    j .LBB14_11
+; RV32ZBB-NEXT:  .LBB14_9:
+; RV32ZBB-NEXT:    sll a2, a0, a7
+; RV32ZBB-NEXT:    neg a5, a6
+; RV32ZBB-NEXT:    srl a0, a0, a5
+; RV32ZBB-NEXT:    sll a5, a1, a7
+; RV32ZBB-NEXT:    or a0, a0, a5
+; RV32ZBB-NEXT:    beqz a6, .LBB14_11
+; RV32ZBB-NEXT:  .LBB14_10:
+; RV32ZBB-NEXT:    mv a1, a0
+; RV32ZBB-NEXT:  .LBB14_11:
+; RV32ZBB-NEXT:    or a0, a3, a2
+; RV32ZBB-NEXT:    or a1, a4, a1
+; RV32ZBB-NEXT:    ret
+;
+; RV64ZBB-LABEL: rotr_64_mask_and_127_and_63:
+; RV64ZBB:       # %bb.0:
+; RV64ZBB-NEXT:    srl a2, a0, a1
+; RV64ZBB-NEXT:    negw a1, a1
+; RV64ZBB-NEXT:    sll a0, a0, a1
+; RV64ZBB-NEXT:    or a0, a2, a0
+; RV64ZBB-NEXT:    ret
+;
+; RV32XTHEADBB-LABEL: rotr_64_mask_and_127_and_63:
+; RV32XTHEADBB:       # %bb.0:
+; RV32XTHEADBB-NEXT:    andi a4, a2, 127
+; RV32XTHEADBB-NEXT:    li a5, 32
+; RV32XTHEADBB-NEXT:    bltu a4, a5, .LBB14_2
+; RV32XTHEADBB-NEXT:  # %bb.1:
+; RV32XTHEADBB-NEXT:    srl a6, a1, a4
+; RV32XTHEADBB-NEXT:    mv a3, a0
+; RV32XTHEADBB-NEXT:    bnez a4, .LBB14_3
+; RV32XTHEADBB-NEXT:    j .LBB14_4
+; RV32XTHEADBB-NEXT:  .LBB14_2:
+; RV32XTHEADBB-NEXT:    srl a3, a0, a2
+; RV32XTHEADBB-NEXT:    neg a6, a4
+; RV32XTHEADBB-NEXT:    sll a6, a1, a6
+; RV32XTHEADBB-NEXT:    or a6, a3, a6
+; RV32XTHEADBB-NEXT:    mv a3, a0
+; RV32XTHEADBB-NEXT:    beqz a4, .LBB14_4
+; RV32XTHEADBB-NEXT:  .LBB14_3:
+; RV32XTHEADBB-NEXT:    mv a3, a6
+; RV32XTHEADBB-NEXT:  .LBB14_4:
+; RV32XTHEADBB-NEXT:    bltu a4, a5, .LBB14_6
+; RV32XTHEADBB-NEXT:  # %bb.5:
+; RV32XTHEADBB-NEXT:    li a4, 0
+; RV32XTHEADBB-NEXT:    j .LBB14_7
+; RV32XTHEADBB-NEXT:  .LBB14_6:
+; RV32XTHEADBB-NEXT:    srl a4, a1, a2
+; RV32XTHEADBB-NEXT:  .LBB14_7:
+; RV32XTHEADBB-NEXT:    neg a7, a2
+; RV32XTHEADBB-NEXT:    andi a6, a7, 63
+; RV32XTHEADBB-NEXT:    bltu a6, a5, .LBB14_9
+; RV32XTHEADBB-NEXT:  # %bb.8:
+; RV32XTHEADBB-NEXT:    li a2, 0
+; RV32XTHEADBB-NEXT:    sll a0, a0, a6
+; RV32XTHEADBB-NEXT:    bnez a6, .LBB14_10
+; RV32XTHEADBB-NEXT:    j .LBB14_11
+; RV32XTHEADBB-NEXT:  .LBB14_9:
+; RV32XTHEADBB-NEXT:    sll a2, a0, a7
+; RV32XTHEADBB-NEXT:    neg a5, a6
+; RV32XTHEADBB-NEXT:    srl a0, a0, a5
+; RV32XTHEADBB-NEXT:    sll a5, a1, a7
+; RV32XTHEADBB-NEXT:    or a0, a0, a5
+; RV32XTHEADBB-NEXT:    beqz a6, .LBB14_11
+; RV32XTHEADBB-NEXT:  .LBB14_10:
+; RV32XTHEADBB-NEXT:    mv a1, a0
+; RV32XTHEADBB-NEXT:  .LBB14_11:
+; RV32XTHEADBB-NEXT:    or a0, a3, a2
+; RV32XTHEADBB-NEXT:    or a1, a4, a1
+; RV32XTHEADBB-NEXT:    ret
+;
+; RV64XTHEADBB-LABEL: rotr_64_mask_and_127_and_63:
+; RV64XTHEADBB:       # %bb.0:
+; RV64XTHEADBB-NEXT:    srl a2, a0, a1
+; RV64XTHEADBB-NEXT:    negw a1, a1
+; RV64XTHEADBB-NEXT:    sll a0, a0, a1
+; RV64XTHEADBB-NEXT:    or a0, a2, a0
+; RV64XTHEADBB-NEXT:    ret
+  %a = and i64 %y, 127
+  %b = lshr i64 %x, %a
+  %c = sub i64 0, %y
+  %d = and i64 %c, 63
+  %e = shl i64 %x, %d
+  %f = or i64 %b, %e
+  ret i64 %f
+}
+
+define i64 @rotr_64_mask_or_128_or_64(i64 %x, i64 %y) nounwind {
+; RV32I-LABEL: rotr_64_mask_or_128_or_64:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    ori a2, a2, 128
+; RV32I-NEXT:    li a3, 32
+; RV32I-NEXT:    bltu a2, a3, .LBB15_2
+; RV32I-NEXT:  # %bb.1:
+; RV32I-NEXT:    srl a4, a1, a2
+; RV32I-NEXT:    bnez a2, .LBB15_3
+; RV32I-NEXT:    j .LBB15_4
+; RV32I-NEXT:  .LBB15_2:
+; RV32I-NEXT:    srl a4, a0, a2
+; RV32I-NEXT:    neg a5, a2
+; RV32I-NEXT:    sll a5, a1, a5
+; RV32I-NEXT:    or a4, a4, a5
+; RV32I-NEXT:    beqz a2, .LBB15_4
+; RV32I-NEXT:  .LBB15_3:
+; RV32I-NEXT:    mv a0, a4
+; RV32I-NEXT:  .LBB15_4:
+; RV32I-NEXT:    bltu a2, a3, .LBB15_6
+; RV32I-NEXT:  # %bb.5:
+; RV32I-NEXT:    li a1, 0
+; RV32I-NEXT:    ret
+; RV32I-NEXT:  .LBB15_6:
+; RV32I-NEXT:    srl a1, a1, a2
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: rotr_64_mask_or_128_or_64:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    ori a1, a1, 128
+; RV64I-NEXT:    srl a0, a0, a1
+; RV64I-NEXT:    ret
+;
+; RV32ZBB-LABEL: rotr_64_mask_or_128_or_64:
+; RV32ZBB:       # %bb.0:
+; RV32ZBB-NEXT:    ori a2, a2, 128
+; RV32ZBB-NEXT:    li a3, 32
+; RV32ZBB-NEXT:    bltu a2, a3, .LBB15_2
+; RV32ZBB-NEXT:  # %bb.1:
+; RV32ZBB-NEXT:    srl a4, a1, a2
+; RV32ZBB-NEXT:    bnez a2, .LBB15_3
+; RV32ZBB-NEXT:    j .LBB15_4
+; RV32ZBB-NEXT:  .LBB15_2:
+; RV32ZBB-NEXT:    srl a4, a0, a2
+; RV32ZBB-NEXT:    neg a5, a2
+; RV32ZBB-NEXT:    sll a5, a1, a5
+; RV32ZBB-NEXT:    or a4, a4, a5
+; RV32ZBB-NEXT:    beqz a2, .LBB15_4
+; RV32ZBB-NEXT:  .LBB15_3:
+; RV32ZBB-NEXT:    mv a0, a4
+; RV32ZBB-NEXT:  .LBB15_4:
+; RV32ZBB-NEXT:    bltu a2, a3, .LBB15_6
+; RV32ZBB-NEXT:  # %bb.5:
+; RV32ZBB-NEXT:    li a1, 0
+; RV32ZBB-NEXT:    ret
+; RV32ZBB-NEXT:  .LBB15_6:
+; RV32ZBB-NEXT:    srl a1, a1, a2
+; RV32ZBB-NEXT:    ret
+;
+; RV64ZBB-LABEL: rotr_64_mask_or_128_or_64:
+; RV64ZBB:       # %bb.0:
+; RV64ZBB-NEXT:    ori a1, a1, 128
+; RV64ZBB-NEXT:    srl a0, a0, a1
+; RV64ZBB-NEXT:    ret
+;
+; RV32XTHEADBB-LABEL: rotr_64_mask_or_128_or_64:
+; RV32XTHEADBB:       # %bb.0:
+; RV32XTHEADBB-NEXT:    ori a2, a2, 128
+; RV32XTHEADBB-NEXT:    li a3, 32
+; RV32XTHEADBB-NEXT:    bltu a2, a3, .LBB15_2
+; RV32XTHEADBB-NEXT:  # %bb.1:
+; RV32XTHEADBB-NEXT:    srl a4, a1, a2
+; RV32XTHEADBB-NEXT:    bnez a2, .LBB15_3
+; RV32XTHEADBB-NEXT:    j .LBB15_4
+; RV32XTHEADBB-NEXT:  .LBB15_2:
+; RV32XTHEADBB-NEXT:    srl a4, a0, a2
+; RV32XTHEADBB-NEXT:    neg a5, a2
+; RV32XTHEADBB-NEXT:    sll a5, a1, a5
+; RV32XTHEADBB-NEXT:    or a4, a4, a5
+; RV32XTHEADBB-NEXT:    beqz a2, .LBB15_4
+; RV32XTHEADBB-NEXT:  .LBB15_3:
+; RV32XTHEADBB-NEXT:    mv a0, a4
+; RV32XTHEADBB-NEXT:  .LBB15_4:
+; RV32XTHEADBB-NEXT:    bltu a2, a3, .LBB15_6
+; RV32XTHEADBB-NEXT:  # %bb.5:
+; RV32XTHEADBB-NEXT:    li a1, 0
+; RV32XTHEADBB-NEXT:    ret
+; RV32XTHEADBB-NEXT:  .LBB15_6:
+; RV32XTHEADBB-NEXT:    srl a1, a1, a2
+; RV32XTHEADBB-NEXT:    ret
+;
+; RV64XTHEADBB-LABEL: rotr_64_mask_or_128_or_64:
+; RV64XTHEADBB:       # %bb.0:
+; RV64XTHEADBB-NEXT:    ori a1, a1, 128
+; RV64XTHEADBB-NEXT:    srl a0, a0, a1
+; RV64XTHEADBB-NEXT:    ret
+  %a = or i64 %y, 128
+  %b = lshr i64 %x, %a
+  %c = sub i64 0, %y
+  %d = or i64 %c, 64
+  %e = shl i64 %x, %d
+  %f = or i64 %b, %e
+  ret i64 %f
+}
+
+; Test that we're able to remove a mask on the rotate amount that has more than
+; one use.
+define signext i32 @rotl_32_mask_shared(i32 signext %a, i32 signext %b, i32 signext %amt) nounwind {
+; RV32I-LABEL: rotl_32_mask_shared:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    andi a3, a2, 31
+; RV32I-NEXT:    sll a4, a0, a2
+; RV32I-NEXT:    neg a3, a3
+; RV32I-NEXT:    srl a0, a0, a3
+; RV32I-NEXT:    or a0, a4, a0
+; RV32I-NEXT:    sll a1, a1, a2
+; RV32I-NEXT:    add a0, a0, a1
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: rotl_32_mask_shared:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    andi a3, a2, 31
+; RV64I-NEXT:    sllw a4, a0, a2
+; RV64I-NEXT:    negw a3, a3
+; RV64I-NEXT:    srlw a0, a0, a3
+; RV64I-NEXT:    or a0, a4, a0
+; RV64I-NEXT:    sllw a1, a1, a2
+; RV64I-NEXT:    addw a0, a0, a1
+; RV64I-NEXT:    ret
+;
+; RV32ZBB-LABEL: rotl_32_mask_shared:
+; RV32ZBB:       # %bb.0:
+; RV32ZBB-NEXT:    rol a0, a0, a2
+; RV32ZBB-NEXT:    sll a1, a1, a2
+; RV32ZBB-NEXT:    add a0, a0, a1
+; RV32ZBB-NEXT:    ret
+;
+; RV64ZBB-LABEL: rotl_32_mask_shared:
+; RV64ZBB:       # %bb.0:
+; RV64ZBB-NEXT:    rolw a0, a0, a2
+; RV64ZBB-NEXT:    sllw a1, a1, a2
+; RV64ZBB-NEXT:    addw a0, a0, a1
+; RV64ZBB-NEXT:    ret
+;
+; RV32XTHEADBB-LABEL: rotl_32_mask_shared:
+; RV32XTHEADBB:       # %bb.0:
+; RV32XTHEADBB-NEXT:    andi a3, a2, 31
+; RV32XTHEADBB-NEXT:    sll a4, a0, a2
+; RV32XTHEADBB-NEXT:    neg a3, a3
+; RV32XTHEADBB-NEXT:    srl a0, a0, a3
+; RV32XTHEADBB-NEXT:    or a0, a4, a0
+; RV32XTHEADBB-NEXT:    sll a1, a1, a2
+; RV32XTHEADBB-NEXT:    add a0, a0, a1
+; RV32XTHEADBB-NEXT:    ret
+;
+; RV64XTHEADBB-LABEL: rotl_32_mask_shared:
+; RV64XTHEADBB:       # %bb.0:
+; RV64XTHEADBB-NEXT:    andi a3, a2, 31
+; RV64XTHEADBB-NEXT:    sllw a4, a0, a2
+; RV64XTHEADBB-NEXT:    negw a3, a3
+; RV64XTHEADBB-NEXT:    srlw a0, a0, a3
+; RV64XTHEADBB-NEXT:    or a0, a4, a0
+; RV64XTHEADBB-NEXT:    sllw a1, a1, a2
+; RV64XTHEADBB-NEXT:    addw a0, a0, a1
+; RV64XTHEADBB-NEXT:    ret
+  %maskedamt = and i32 %amt, 31
+  %1 = tail call i32 @llvm.fshl.i32(i32 %a, i32 %a, i32 %maskedamt)
+  %2 = shl i32 %b, %maskedamt
+  %3 = add i32 %1, %2
+  ret i32 %3
+}
+declare i32 @llvm.fshl.i32(i32, i32, i32)
+
+define signext i64 @rotl_64_mask_shared(i64 signext %a, i64 signext %b, i64 signext %amt) nounwind {
+; RV32I-LABEL: rotl_64_mask_shared:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    andi a5, a4, 63
+; RV32I-NEXT:    li t0, 32
+; RV32I-NEXT:    neg a7, a5
+; RV32I-NEXT:    bltu a5, t0, .LBB17_2
+; RV32I-NEXT:  # %bb.1:
+; RV32I-NEXT:    li a6, 0
+; RV32I-NEXT:    sll t3, a0, a5
+; RV32I-NEXT:    j .LBB17_3
+; RV32I-NEXT:  .LBB17_2:
+; RV32I-NEXT:    sll a6, a0, a4
+; RV32I-NEXT:    srl t1, a0, a7
+; RV32I-NEXT:    sll t2, a1, a4
+; RV32I-NEXT:    or t3, t1, t2
+; RV32I-NEXT:  .LBB17_3:
+; RV32I-NEXT:    neg t2, a5
+; RV32I-NEXT:    mv t1, a1
+; RV32I-NEXT:    beqz a5, .LBB17_5
+; RV32I-NEXT:  # %bb.4:
+; RV32I-NEXT:    mv t1, t3
+; RV32I-NEXT:  .LBB17_5:
+; RV32I-NEXT:    andi t3, t2, 63
+; RV32I-NEXT:    bltu t3, t0, .LBB17_7
+; RV32I-NEXT:  # %bb.6:
+; RV32I-NEXT:    srl t4, a1, t3
+; RV32I-NEXT:    bnez t3, .LBB17_8
+; RV32I-NEXT:    j .LBB17_9
+; RV32I-NEXT:  .LBB17_7:
+; RV32I-NEXT:    srl t4, a0, t2
+; RV32I-NEXT:    neg t5, t3
+; RV32I-NEXT:    sll t5, a1, t5
+; RV32I-NEXT:    or t4, t4, t5
+; RV32I-NEXT:    beqz t3, .LBB17_9
+; RV32I-NEXT:  .LBB17_8:
+; RV32I-NEXT:    mv a0, t4
+; RV32I-NEXT:  .LBB17_9:
+; RV32I-NEXT:    bltu t3, t0, .LBB17_12
+; RV32I-NEXT:  # %bb.10:
+; RV32I-NEXT:    li a1, 0
+; RV32I-NEXT:    bgeu a5, t0, .LBB17_13
+; RV32I-NEXT:  .LBB17_11:
+; RV32I-NEXT:    sll t0, a2, a4
+; RV32I-NEXT:    srl a2, a2, a7
+; RV32I-NEXT:    sll a4, a3, a4
+; RV32I-NEXT:    or a2, a2, a4
+; RV32I-NEXT:    j .LBB17_14
+; RV32I-NEXT:  .LBB17_12:
+; RV32I-NEXT:    srl a1, a1, t2
+; RV32I-NEXT:    bltu a5, t0, .LBB17_11
+; RV32I-NEXT:  .LBB17_13:
+; RV32I-NEXT:    li t0, 0
+; RV32I-NEXT:    sll a2, a2, a5
+; RV32I-NEXT:  .LBB17_14:
+; RV32I-NEXT:    or a0, a6, a0
+; RV32I-NEXT:    or a1, t1, a1
+; RV32I-NEXT:    beqz a5, .LBB17_16
+; RV32I-NEXT:  # %bb.15:
+; RV32I-NEXT:    mv a3, a2
+; RV32I-NEXT:  .LBB17_16:
+; RV32I-NEXT:    add a0, a0, t0
+; RV32I-NEXT:    sltu a2, a0, t0
+; RV32I-NEXT:    add a1, a1, a3
+; RV32I-NEXT:    add a1, a1, a2
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: rotl_64_mask_shared:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    andi a3, a2, 63
+; RV64I-NEXT:    sll a4, a0, a2
+; RV64I-NEXT:    negw a3, a3
+; RV64I-NEXT:    srl a0, a0, a3
+; RV64I-NEXT:    or a0, a4, a0
+; RV64I-NEXT:    sll a1, a1, a2
+; RV64I-NEXT:    add a0, a0, a1
+; RV64I-NEXT:    ret
+;
+; RV32ZBB-LABEL: rotl_64_mask_shared:
+; RV32ZBB:       # %bb.0:
+; RV32ZBB-NEXT:    andi a5, a4, 63
+; RV32ZBB-NEXT:    li t0, 32
+; RV32ZBB-NEXT:    neg a7, a5
+; RV32ZBB-NEXT:    bltu a5, t0, .LBB17_2
+; RV32ZBB-NEXT:  # %bb.1:
+; RV32ZBB-NEXT:    li a6, 0
+; RV32ZBB-NEXT:    sll t3, a0, a5
+; RV32ZBB-NEXT:    j .LBB17_3
+; RV32ZBB-NEXT:  .LBB17_2:
+; RV32ZBB-NEXT:    sll a6, a0, a4
+; RV32ZBB-NEXT:    srl t1, a0, a7
+; RV32ZBB-NEXT:    sll t2, a1, a4
+; RV32ZBB-NEXT:    or t3, t1, t2
+; RV32ZBB-NEXT:  .LBB17_3:
+; RV32ZBB-NEXT:    neg t2, a5
+; RV32ZBB-NEXT:    mv t1, a1
+; RV32ZBB-NEXT:    beqz a5, .LBB17_5
+; RV32ZBB-NEXT:  # %bb.4:
+; RV32ZBB-NEXT:    mv t1, t3
+; RV32ZBB-NEXT:  .LBB17_5:
+; RV32ZBB-NEXT:    andi t3, t2, 63
+; RV32ZBB-NEXT:    bltu t3, t0, .LBB17_7
+; RV32ZBB-NEXT:  # %bb.6:
+; RV32ZBB-NEXT:    srl t4, a1, t3
+; RV32ZBB-NEXT:    bnez t3, .LBB17_8
+; RV32ZBB-NEXT:    j .LBB17_9
+; RV32ZBB-NEXT:  .LBB17_7:
+; RV32ZBB-NEXT:    srl t4, a0, t2
+; RV32ZBB-NEXT:    neg t5, t3
+; RV32ZBB-NEXT:    sll t5, a1, t5
+; RV32ZBB-NEXT:    or t4, t4, t5
+; RV32ZBB-NEXT:    beqz t3, .LBB17_9
+; RV32ZBB-NEXT:  .LBB17_8:
+; RV32ZBB-NEXT:    mv a0, t4
+; RV32ZBB-NEXT:  .LBB17_9:
+; RV32ZBB-NEXT:    bltu t3, t0, .LBB17_12
+; RV32ZBB-NEXT:  # %bb.10:
+; RV32ZBB-NEXT:    li a1, 0
+; RV32ZBB-NEXT:    bgeu a5, t0, .LBB17_13
+; RV32ZBB-NEXT:  .LBB17_11:
+; RV32ZBB-NEXT:    sll t0, a2, a4
+; RV32ZBB-NEXT:    srl a2, a2, a7
+; RV32ZBB-NEXT:    sll a4, a3, a4
+; RV32ZBB-NEXT:    or a2, a2, a4
+; RV32ZBB-NEXT:    j .LBB17_14
+; RV32ZBB-NEXT:  .LBB17_12:
+; RV32ZBB-NEXT:    srl a1, a1, t2
+; RV32ZBB-NEXT:    bltu a5, t0, .LBB17_11
+; RV32ZBB-NEXT:  .LBB17_13:
+; RV32ZBB-NEXT:    li t0, 0
+; RV32ZBB-NEXT:    sll a2, a2, a5
+; RV32ZBB-NEXT:  .LBB17_14:
+; RV32ZBB-NEXT:    or a0, a6, a0
+; RV32ZBB-NEXT:    or a1, t1, a1
+; RV32ZBB-NEXT:    beqz a5, .LBB17_16
+; RV32ZBB-NEXT:  # %bb.15:
+; RV32ZBB-NEXT:    mv a3, a2
+; RV32ZBB-NEXT:  .LBB17_16:
+; RV32ZBB-NEXT:    add a0, a0, t0
+; RV32ZBB-NEXT:    sltu a2, a0, t0
+; RV32ZBB-NEXT:    add a1, a1, a3
+; RV32ZBB-NEXT:    add a1, a1, a2
+; RV32ZBB-NEXT:    ret
+;
+; RV64ZBB-LABEL: rotl_64_mask_shared:
+; RV64ZBB:       # %bb.0:
+; RV64ZBB-NEXT:    rol a0, a0, a2
+; RV64ZBB-NEXT:    sll a1, a1, a2
+; RV64ZBB-NEXT:    add a0, a0, a1
+; RV64ZBB-NEXT:    ret
+;
+; RV32XTHEADBB-LABEL: rotl_64_mask_shared:
+; RV32XTHEADBB:       # %bb.0:
+; RV32XTHEADBB-NEXT:    andi a5, a4, 63
+; RV32XTHEADBB-NEXT:    li t0, 32
+; RV32XTHEADBB-NEXT:    neg a7, a5
+; RV32XTHEADBB-NEXT:    bltu a5, t0, .LBB17_2
+; RV32XTHEADBB-NEXT:  # %bb.1:
+; RV32XTHEADBB-NEXT:    li a6, 0
+; RV32XTHEADBB-NEXT:    sll t3, a0, a5
+; RV32XTHEADBB-NEXT:    j .LBB17_3
+; RV32XTHEADBB-NEXT:  .LBB17_2:
+; RV32XTHEADBB-NEXT:    sll a6, a0, a4
+; RV32XTHEADBB-NEXT:    srl t1, a0, a7
+; RV32XTHEADBB-NEXT:    sll t2, a1, a4
+; RV32XTHEADBB-NEXT:    or t3, t1, t2
+; RV32XTHEADBB-NEXT:  .LBB17_3:
+; RV32XTHEADBB-NEXT:    neg t2, a5
+; RV32XTHEADBB-NEXT:    mv t1, a1
+; RV32XTHEADBB-NEXT:    beqz a5, .LBB17_5
+; RV32XTHEADBB-NEXT:  # %bb.4:
+; RV32XTHEADBB-NEXT:    mv t1, t3
+; RV32XTHEADBB-NEXT:  .LBB17_5:
+; RV32XTHEADBB-NEXT:    andi t3, t2, 63
+; RV32XTHEADBB-NEXT:    bltu t3, t0, .LBB17_7
+; RV32XTHEADBB-NEXT:  # %bb.6:
+; RV32XTHEADBB-NEXT:    srl t4, a1, t3
+; RV32XTHEADBB-NEXT:    bnez t3, .LBB17_8
+; RV32XTHEADBB-NEXT:    j .LBB17_9
+; RV32XTHEADBB-NEXT:  .LBB17_7:
+; RV32XTHEADBB-NEXT:    srl t4, a0, t2
+; RV32XTHEADBB-NEXT:    neg t5, t3
+; RV32XTHEADBB-NEXT:    sll t5, a1, t5
+; RV32XTHEADBB-NEXT:    or t4, t4, t5
+; RV32XTHEADBB-NEXT:    beqz t3, .LBB17_9
+; RV32XTHEADBB-NEXT:  .LBB17_8:
+; RV32XTHEADBB-NEXT:    mv a0, t4
+; RV32XTHEADBB-NEXT:  .LBB17_9:
+; RV32XTHEADBB-NEXT:    bltu t3, t0, .LBB17_12
+; RV32XTHEADBB-NEXT:  # %bb.10:
+; RV32XTHEADBB-NEXT:    li a1, 0
+; RV32XTHEADBB-NEXT:    bgeu a5, t0, .LBB17_13
+; RV32XTHEADBB-NEXT:  .LBB17_11:
+; RV32XTHEADBB-NEXT:    sll t0, a2, a4
+; RV32XTHEADBB-NEXT:    srl a2, a2, a7
+; RV32XTHEADBB-NEXT:    sll a4, a3, a4
+; RV32XTHEADBB-NEXT:    or a2, a2, a4
+; RV32XTHEADBB-NEXT:    j .LBB17_14
+; RV32XTHEADBB-NEXT:  .LBB17_12:
+; RV32XTHEADBB-NEXT:    srl a1, a1, t2
+; RV32XTHEADBB-NEXT:    bltu a5, t0, .LBB17_11
+; RV32XTHEADBB-NEXT:  .LBB17_13:
+; RV32XTHEADBB-NEXT:    li t0, 0
+; RV32XTHEADBB-NEXT:    sll a2, a2, a5
+; RV32XTHEADBB-NEXT:  .LBB17_14:
+; RV32XTHEADBB-NEXT:    or a0, a6, a0
+; RV32XTHEADBB-NEXT:    or a1, t1, a1
+; RV32XTHEADBB-NEXT:    beqz a5, .LBB17_16
+; RV32XTHEADBB-NEXT:  # %bb.15:
+; RV32XTHEADBB-NEXT:    mv a3, a2
+; RV32XTHEADBB-NEXT:  .LBB17_16:
+; RV32XTHEADBB-NEXT:    add a0, a0, t0
+; RV32XTHEADBB-NEXT:    sltu a2, a0, t0
+; RV32XTHEADBB-NEXT:    add a1, a1, a3
+; RV32XTHEADBB-NEXT:    add a1, a1, a2
+; RV32XTHEADBB-NEXT:    ret
+;
+; RV64XTHEADBB-LABEL: rotl_64_mask_shared:
+; RV64XTHEADBB:       # %bb.0:
+; RV64XTHEADBB-NEXT:    andi a3, a2, 63
+; RV64XTHEADBB-NEXT:    sll a4, a0, a2
+; RV64XTHEADBB-NEXT:    negw a3, a3
+; RV64XTHEADBB-NEXT:    srl a0, a0, a3
+; RV64XTHEADBB-NEXT:    or a0, a4, a0
+; RV64XTHEADBB-NEXT:    sll a1, a1, a2
+; RV64XTHEADBB-NEXT:    add a0, a0, a1
+; RV64XTHEADBB-NEXT:    ret
+  %maskedamt = and i64 %amt, 63
+  %1 = tail call i64 @llvm.fshl.i64(i64 %a, i64 %a, i64 %maskedamt)
+  %2 = shl i64 %b, %maskedamt
+  %3 = add i64 %1, %2
+  ret i64 %3
+}
+declare i64 @llvm.fshl.i64(i64, i64, i64)
+
+define signext i32 @rotr_32_mask_shared(i32 signext %a, i32 signext %b, i32 signext %amt) nounwind {
+; RV32I-LABEL: rotr_32_mask_shared:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    andi a3, a2, 31
+; RV32I-NEXT:    srl a4, a0, a2
+; RV32I-NEXT:    neg a3, a3
+; RV32I-NEXT:    sll a0, a0, a3
+; RV32I-NEXT:    or a0, a4, a0
+; RV32I-NEXT:    sll a1, a1, a2
+; RV32I-NEXT:    add a0, a0, a1
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: rotr_32_mask_shared:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    andi a3, a2, 31
+; RV64I-NEXT:    srlw a4, a0, a2
+; RV64I-NEXT:    negw a3, a3
+; RV64I-NEXT:    sllw a0, a0, a3
+; RV64I-NEXT:    or a0, a4, a0
+; RV64I-NEXT:    sllw a1, a1, a2
+; RV64I-NEXT:    addw a0, a0, a1
+; RV64I-NEXT:    ret
+;
+; RV32ZBB-LABEL: rotr_32_mask_shared:
+; RV32ZBB:       # %bb.0:
+; RV32ZBB-NEXT:    ror a0, a0, a2
+; RV32ZBB-NEXT:    sll a1, a1, a2
+; RV32ZBB-NEXT:    add a0, a0, a1
+; RV32ZBB-NEXT:    ret
+;
+; RV64ZBB-LABEL: rotr_32_mask_shared:
+; RV64ZBB:       # %bb.0:
+; RV64ZBB-NEXT:    rorw a0, a0, a2
+; RV64ZBB-NEXT:    sllw a1, a1, a2
+; RV64ZBB-NEXT:    addw a0, a0, a1
+; RV64ZBB-NEXT:    ret
+;
+; RV32XTHEADBB-LABEL: rotr_32_mask_shared:
+; RV32XTHEADBB:       # %bb.0:
+; RV32XTHEADBB-NEXT:    andi a3, a2, 31
+; RV32XTHEADBB-NEXT:    srl a4, a0, a2
+; RV32XTHEADBB-NEXT:    neg a3, a3
+; RV32XTHEADBB-NEXT:    sll a0, a0, a3
+; RV32XTHEADBB-NEXT:    or a0, a4, a0
+; RV32XTHEADBB-NEXT:    sll a1, a1, a2
+; RV32XTHEADBB-NEXT:    add a0, a0, a1
+; RV32XTHEADBB-NEXT:    ret
+;
+; RV64XTHEADBB-LABEL: rotr_32_mask_shared:
+; RV64XTHEADBB:       # %bb.0:
+; RV64XTHEADBB-NEXT:    andi a3, a2, 31
+; RV64XTHEADBB-NEXT:    srlw a4, a0, a2
+; RV64XTHEADBB-NEXT:    negw a3, a3
+; RV64XTHEADBB-NEXT:    sllw a0, a0, a3
+; RV64XTHEADBB-NEXT:    or a0, a4, a0
+; RV64XTHEADBB-NEXT:    sllw a1, a1, a2
+; RV64XTHEADBB-NEXT:    addw a0, a0, a1
+; RV64XTHEADBB-NEXT:    ret
+  %maskedamt = and i32 %amt, 31
+  %1 = tail call i32 @llvm.fshr.i32(i32 %a, i32 %a, i32 %maskedamt)
+  %2 = shl i32 %b, %maskedamt
+  %3 = add i32 %1, %2
+  ret i32 %3
+}
+declare i32 @llvm.fshr.i32(i32, i32, i32)
+
+define signext i64 @rotr_64_mask_shared(i64 signext %a, i64 signext %b, i64 signext %amt) nounwind {
+; RV32I-LABEL: rotr_64_mask_shared:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    andi a5, a4, 63
+; RV32I-NEXT:    li t0, 32
+; RV32I-NEXT:    neg a6, a5
+; RV32I-NEXT:    bltu a5, t0, .LBB19_2
+; RV32I-NEXT:  # %bb.1:
+; RV32I-NEXT:    srl t1, a1, a5
+; RV32I-NEXT:    mv a7, a0
+; RV32I-NEXT:    bnez a5, .LBB19_3
+; RV32I-NEXT:    j .LBB19_4
+; RV32I-NEXT:  .LBB19_2:
+; RV32I-NEXT:    srl a7, a0, a4
+; RV32I-NEXT:    sll t1, a1, a6
+; RV32I-NEXT:    or t1, a7, t1
+; RV32I-NEXT:    mv a7, a0
+; RV32I-NEXT:    beqz a5, .LBB19_4
+; RV32I-NEXT:  .LBB19_3:
+; RV32I-NEXT:    mv a7, t1
+; RV32I-NEXT:  .LBB19_4:
+; RV32I-NEXT:    neg t4, a5
+; RV32I-NEXT:    bltu a5, t0, .LBB19_7
+; RV32I-NEXT:  # %bb.5:
+; RV32I-NEXT:    li t1, 0
+; RV32I-NEXT:    andi t3, t4, 63
+; RV32I-NEXT:    bgeu t3, t0, .LBB19_8
+; RV32I-NEXT:  .LBB19_6:
+; RV32I-NEXT:    sll t2, a0, t4
+; RV32I-NEXT:    neg t5, t3
+; RV32I-NEXT:    srl a0, a0, t5
+; RV32I-NEXT:    sll t4, a1, t4
+; RV32I-NEXT:    or a0, a0, t4
+; RV32I-NEXT:    bnez t3, .LBB19_9
+; RV32I-NEXT:    j .LBB19_10
+; RV32I-NEXT:  .LBB19_7:
+; RV32I-NEXT:    srl t1, a1, a4
+; RV32I-NEXT:    andi t3, t4, 63
+; RV32I-NEXT:    bltu t3, t0, .LBB19_6
+; RV32I-NEXT:  .LBB19_8:
+; RV32I-NEXT:    li t2, 0
+; RV32I-NEXT:    sll a0, a0, t3
+; RV32I-NEXT:    beqz t3, .LBB19_10
+; RV32I-NEXT:  .LBB19_9:
+; RV32I-NEXT:    mv a1, a0
+; RV32I-NEXT:  .LBB19_10:
+; RV32I-NEXT:    bltu a5, t0, .LBB19_12
+; RV32I-NEXT:  # %bb.11:
+; RV32I-NEXT:    li t0, 0
+; RV32I-NEXT:    sll a0, a2, a5
+; RV32I-NEXT:    j .LBB19_13
+; RV32I-NEXT:  .LBB19_12:
+; RV32I-NEXT:    sll t0, a2, a4
+; RV32I-NEXT:    srl a0, a2, a6
+; RV32I-NEXT:    sll a2, a3, a4
+; RV32I-NEXT:    or a0, a0, a2
+; RV32I-NEXT:  .LBB19_13:
+; RV32I-NEXT:    or a2, a7, t2
+; RV32I-NEXT:    or a1, t1, a1
+; RV32I-NEXT:    beqz a5, .LBB19_15
+; RV32I-NEXT:  # %bb.14:
+; RV32I-NEXT:    mv a3, a0
+; RV32I-NEXT:  .LBB19_15:
+; RV32I-NEXT:    add a0, a2, t0
+; RV32I-NEXT:    sltu a2, a0, t0
+; RV32I-NEXT:    add a1, a1, a3
+; RV32I-NEXT:    add a1, a1, a2
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: rotr_64_mask_shared:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    andi a3, a2, 63
+; RV64I-NEXT:    srl a4, a0, a2
+; RV64I-NEXT:    negw a3, a3
+; RV64I-NEXT:    sll a0, a0, a3
+; RV64I-NEXT:    or a0, a4, a0
+; RV64I-NEXT:    sll a1, a1, a2
+; RV64I-NEXT:    add a0, a0, a1
+; RV64I-NEXT:    ret
+;
+; RV32ZBB-LABEL: rotr_64_mask_shared:
+; RV32ZBB:       # %bb.0:
+; RV32ZBB-NEXT:    andi a5, a4, 63
+; RV32ZBB-NEXT:    li t0, 32
+; RV32ZBB-NEXT:    neg a6, a5
+; RV32ZBB-NEXT:    bltu a5, t0, .LBB19_2
+; RV32ZBB-NEXT:  # %bb.1:
+; RV32ZBB-NEXT:    srl t1, a1, a5
+; RV32ZBB-NEXT:    mv a7, a0
+; RV32ZBB-NEXT:    bnez a5, .LBB19_3
+; RV32ZBB-NEXT:    j .LBB19_4
+; RV32ZBB-NEXT:  .LBB19_2:
+; RV32ZBB-NEXT:    srl a7, a0, a4
+; RV32ZBB-NEXT:    sll t1, a1, a6
+; RV32ZBB-NEXT:    or t1, a7, t1
+; RV32ZBB-NEXT:    mv a7, a0
+; RV32ZBB-NEXT:    beqz a5, .LBB19_4
+; RV32ZBB-NEXT:  .LBB19_3:
+; RV32ZBB-NEXT:    mv a7, t1
+; RV32ZBB-NEXT:  .LBB19_4:
+; RV32ZBB-NEXT:    neg t4, a5
+; RV32ZBB-NEXT:    bltu a5, t0, .LBB19_7
+; RV32ZBB-NEXT:  # %bb.5:
+; RV32ZBB-NEXT:    li t1, 0
+; RV32ZBB-NEXT:    andi t3, t4, 63
+; RV32ZBB-NEXT:    bgeu t3, t0, .LBB19_8
+; RV32ZBB-NEXT:  .LBB19_6:
+; RV32ZBB-NEXT:    sll t2, a0, t4
+; RV32ZBB-NEXT:    neg t5, t3
+; RV32ZBB-NEXT:    srl a0, a0, t5
+; RV32ZBB-NEXT:    sll t4, a1, t4
+; RV32ZBB-NEXT:    or a0, a0, t4
+; RV32ZBB-NEXT:    bnez t3, .LBB19_9
+; RV32ZBB-NEXT:    j .LBB19_10
+; RV32ZBB-NEXT:  .LBB19_7:
+; RV32ZBB-NEXT:    srl t1, a1, a4
+; RV32ZBB-NEXT:    andi t3, t4, 63
+; RV32ZBB-NEXT:    bltu t3, t0, .LBB19_6
+; RV32ZBB-NEXT:  .LBB19_8:
+; RV32ZBB-NEXT:    li t2, 0
+; RV32ZBB-NEXT:    sll a0, a0, t3
+; RV32ZBB-NEXT:    beqz t3, .LBB19_10
+; RV32ZBB-NEXT:  .LBB19_9:
+; RV32ZBB-NEXT:    mv a1, a0
+; RV32ZBB-NEXT:  .LBB19_10:
+; RV32ZBB-NEXT:    bltu a5, t0, .LBB19_12
+; RV32ZBB-NEXT:  # %bb.11:
+; RV32ZBB-NEXT:    li t0, 0
+; RV32ZBB-NEXT:    sll a0, a2, a5
+; RV32ZBB-NEXT:    j .LBB19_13
+; RV32ZBB-NEXT:  .LBB19_12:
+; RV32ZBB-NEXT:    sll t0, a2, a4
+; RV32ZBB-NEXT:    srl a0, a2, a6
+; RV32ZBB-NEXT:    sll a2, a3, a4
+; RV32ZBB-NEXT:    or a0, a0, a2
+; RV32ZBB-NEXT:  .LBB19_13:
+; RV32ZBB-NEXT:    or a2, a7, t2
+; RV32ZBB-NEXT:    or a1, t1, a1
+; RV32ZBB-NEXT:    beqz a5, .LBB19_15
+; RV32ZBB-NEXT:  # %bb.14:
+; RV32ZBB-NEXT:    mv a3, a0
+; RV32ZBB-NEXT:  .LBB19_15:
+; RV32ZBB-NEXT:    add a0, a2, t0
+; RV32ZBB-NEXT:    sltu a2, a0, t0
+; RV32ZBB-NEXT:    add a1, a1, a3
+; RV32ZBB-NEXT:    add a1, a1, a2
+; RV32ZBB-NEXT:    ret
+;
+; RV64ZBB-LABEL: rotr_64_mask_shared:
+; RV64ZBB:       # %bb.0:
+; RV64ZBB-NEXT:    ror a0, a0, a2
+; RV64ZBB-NEXT:    sll a1, a1, a2
+; RV64ZBB-NEXT:    add a0, a0, a1
+; RV64ZBB-NEXT:    ret
+;
+; RV32XTHEADBB-LABEL: rotr_64_mask_shared:
+; RV32XTHEADBB:       # %bb.0:
+; RV32XTHEADBB-NEXT:    andi a5, a4, 63
+; RV32XTHEADBB-NEXT:    li t0, 32
+; RV32XTHEADBB-NEXT:    neg a6, a5
+; RV32XTHEADBB-NEXT:    bltu a5, t0, .LBB19_2
+; RV32XTHEADBB-NEXT:  # %bb.1:
+; RV32XTHEADBB-NEXT:    srl t1, a1, a5
+; RV32XTHEADBB-NEXT:    mv a7, a0
+; RV32XTHEADBB-NEXT:    bnez a5, .LBB19_3
+; RV32XTHEADBB-NEXT:    j .LBB19_4
+; RV32XTHEADBB-NEXT:  .LBB19_2:
+; RV32XTHEADBB-NEXT:    srl a7, a0, a4
+; RV32XTHEADBB-NEXT:    sll t1, a1, a6
+; RV32XTHEADBB-NEXT:    or t1, a7, t1
+; RV32XTHEADBB-NEXT:    mv a7, a0
+; RV32XTHEADBB-NEXT:    beqz a5, .LBB19_4
+; RV32XTHEADBB-NEXT:  .LBB19_3:
+; RV32XTHEADBB-NEXT:    mv a7, t1
+; RV32XTHEADBB-NEXT:  .LBB19_4:
+; RV32XTHEADBB-NEXT:    neg t4, a5
+; RV32XTHEADBB-NEXT:    bltu a5, t0, .LBB19_7
+; RV32XTHEADBB-NEXT:  # %bb.5:
+; RV32XTHEADBB-NEXT:    li t1, 0
+; RV32XTHEADBB-NEXT:    andi t3, t4, 63
+; RV32XTHEADBB-NEXT:    bgeu t3, t0, .LBB19_8
+; RV32XTHEADBB-NEXT:  .LBB19_6:
+; RV32XTHEADBB-NEXT:    sll t2, a0, t4
+; RV32XTHEADBB-NEXT:    neg t5, t3
+; RV32XTHEADBB-NEXT:    srl a0, a0, t5
+; RV32XTHEADBB-NEXT:    sll t4, a1, t4
+; RV32XTHEADBB-NEXT:    or a0, a0, t4
+; RV32XTHEADBB-NEXT:    bnez t3, .LBB19_9
+; RV32XTHEADBB-NEXT:    j .LBB19_10
+; RV32XTHEADBB-NEXT:  .LBB19_7:
+; RV32XTHEADBB-NEXT:    srl t1, a1, a4
+; RV32XTHEADBB-NEXT:    andi t3, t4, 63
+; RV32XTHEADBB-NEXT:    bltu t3, t0, .LBB19_6
+; RV32XTHEADBB-NEXT:  .LBB19_8:
+; RV32XTHEADBB-NEXT:    li t2, 0
+; RV32XTHEADBB-NEXT:    sll a0, a0, t3
+; RV32XTHEADBB-NEXT:    beqz t3, .LBB19_10
+; RV32XTHEADBB-NEXT:  .LBB19_9:
+; RV32XTHEADBB-NEXT:    mv a1, a0
+; RV32XTHEADBB-NEXT:  .LBB19_10:
+; RV32XTHEADBB-NEXT:    bltu a5, t0, .LBB19_12
+; RV32XTHEADBB-NEXT:  # %bb.11:
+; RV32XTHEADBB-NEXT:    li t0, 0
+; RV32XTHEADBB-NEXT:    sll a0, a2, a5
+; RV32XTHEADBB-NEXT:    j .LBB19_13
+; RV32XTHEADBB-NEXT:  .LBB19_12:
+; RV32XTHEADBB-NEXT:    sll t0, a2, a4
+; RV32XTHEADBB-NEXT:    srl a0, a2, a6
+; RV32XTHEADBB-NEXT:    sll a2, a3, a4
+; RV32XTHEADBB-NEXT:    or a0, a0, a2
+; RV32XTHEADBB-NEXT:  .LBB19_13:
+; RV32XTHEADBB-NEXT:    or a2, a7, t2
+; RV32XTHEADBB-NEXT:    or a1, t1, a1
+; RV32XTHEADBB-NEXT:    beqz a5, .LBB19_15
+; RV32XTHEADBB-NEXT:  # %bb.14:
+; RV32XTHEADBB-NEXT:    mv a3, a0
+; RV32XTHEADBB-NEXT:  .LBB19_15:
+; RV32XTHEADBB-NEXT:    add a0, a2, t0
+; RV32XTHEADBB-NEXT:    sltu a2, a0, t0
+; RV32XTHEADBB-NEXT:    add a1, a1, a3
+; RV32XTHEADBB-NEXT:    add a1, a1, a2
+; RV32XTHEADBB-NEXT:    ret
+;
+; RV64XTHEADBB-LABEL: rotr_64_mask_shared:
+; RV64XTHEADBB:       # %bb.0:
+; RV64XTHEADBB-NEXT:    andi a3, a2, 63
+; RV64XTHEADBB-NEXT:    srl a4, a0, a2
+; RV64XTHEADBB-NEXT:    negw a3, a3
+; RV64XTHEADBB-NEXT:    sll a0, a0, a3
+; RV64XTHEADBB-NEXT:    or a0, a4, a0
+; RV64XTHEADBB-NEXT:    sll a1, a1, a2
+; RV64XTHEADBB-NEXT:    add a0, a0, a1
+; RV64XTHEADBB-NEXT:    ret
+  %maskedamt = and i64 %amt, 63
+  %1 = tail call i64 @llvm.fshr.i64(i64 %a, i64 %a, i64 %maskedamt)
+  %2 = shl i64 %b, %maskedamt
+  %3 = add i64 %1, %2
+  ret i64 %3
+}
+declare i64 @llvm.fshr.i64(i64, i64, i64)
+
+define signext i32 @rotl_32_mask_multiple(i32 signext %a, i32 signext %b, i32 signext %amt) nounwind {
+; RV32I-LABEL: rotl_32_mask_multiple:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    andi a3, a2, 31
+; RV32I-NEXT:    sll a4, a0, a2
+; RV32I-NEXT:    sll a2, a1, a2
+; RV32I-NEXT:    neg a3, a3
+; RV32I-NEXT:    srl a0, a0, a3
+; RV32I-NEXT:    srl a1, a1, a3
+; RV32I-NEXT:    or a0, a4, a0
+; RV32I-NEXT:    or a1, a2, a1
+; RV32I-NEXT:    add a0, a0, a1
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: rotl_32_mask_multiple:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    andi a3, a2, 31
+; RV64I-NEXT:    sllw a4, a0, a2
+; RV64I-NEXT:    sllw a2, a1, a2
+; RV64I-NEXT:    negw a5, a3
+; RV64I-NEXT:    negw a3, a3
+; RV64I-NEXT:    srlw a0, a0, a5
+; RV64I-NEXT:    srlw a1, a1, a3
+; RV64I-NEXT:    or a0, a4, a0
+; RV64I-NEXT:    or a1, a2, a1
+; RV64I-NEXT:    addw a0, a0, a1
+; RV64I-NEXT:    ret
+;
+; RV32ZBB-LABEL: rotl_32_mask_multiple:
+; RV32ZBB:       # %bb.0:
+; RV32ZBB-NEXT:    rol a0, a0, a2
+; RV32ZBB-NEXT:    rol a1, a1, a2
+; RV32ZBB-NEXT:    add a0, a0, a1
+; RV32ZBB-NEXT:    ret
+;
+; RV64ZBB-LABEL: rotl_32_mask_multiple:
+; RV64ZBB:       # %bb.0:
+; RV64ZBB-NEXT:    rolw a0, a0, a2
+; RV64ZBB-NEXT:    rolw a1, a1, a2
+; RV64ZBB-NEXT:    addw a0, a0, a1
+; RV64ZBB-NEXT:    ret
+;
+; RV32XTHEADBB-LABEL: rotl_32_mask_multiple:
+; RV32XTHEADBB:       # %bb.0:
+; RV32XTHEADBB-NEXT:    andi a3, a2, 31
+; RV32XTHEADBB-NEXT:    sll a4, a0, a2
+; RV32XTHEADBB-NEXT:    sll a2, a1, a2
+; RV32XTHEADBB-NEXT:    neg a3, a3
+; RV32XTHEADBB-NEXT:    srl a0, a0, a3
+; RV32XTHEADBB-NEXT:    srl a1, a1, a3
+; RV32XTHEADBB-NEXT:    or a0, a4, a0
+; RV32XTHEADBB-NEXT:    or a1, a2, a1
+; RV32XTHEADBB-NEXT:    add a0, a0, a1
+; RV32XTHEADBB-NEXT:    ret
+;
+; RV64XTHEADBB-LABEL: rotl_32_mask_multiple:
+; RV64XTHEADBB:       # %bb.0:
+; RV64XTHEADBB-NEXT:    andi a3, a2, 31
+; RV64XTHEADBB-NEXT:    sllw a4, a0, a2
+; RV64XTHEADBB-NEXT:    sllw a2, a1, a2
+; RV64XTHEADBB-NEXT:    negw a5, a3
+; RV64XTHEADBB-NEXT:    negw a3, a3
+; RV64XTHEADBB-NEXT:    srlw a0, a0, a5
+; RV64XTHEADBB-NEXT:    srlw a1, a1, a3
+; RV64XTHEADBB-NEXT:    or a0, a4, a0
+; RV64XTHEADBB-NEXT:    or a1, a2, a1
+; RV64XTHEADBB-NEXT:    addw a0, a0, a1
+; RV64XTHEADBB-NEXT:    ret
+  %maskedamt = and i32 %amt, 31
+  %1 = tail call i32 @llvm.fshl.i32(i32 %a, i32 %a, i32 %maskedamt)
+  %2 = tail call i32 @llvm.fshl.i32(i32 %b, i32 %b, i32 %maskedamt)
+  %3 = add i32 %1, %2
+  ret i32 %3
+}
+
+define i64 @rotl_64_mask_multiple(i64 %a, i64 %b, i64 %amt) nounwind {
+; RV32I-LABEL: rotl_64_mask_multiple:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    andi t1, a4, 63
+; RV32I-NEXT:    li a5, 32
+; RV32I-NEXT:    neg t3, t1
+; RV32I-NEXT:    bltu t1, a5, .LBB21_2
+; RV32I-NEXT:  # %bb.1:
+; RV32I-NEXT:    li a6, 0
+; RV32I-NEXT:    sll t2, a0, t1
+; RV32I-NEXT:    j .LBB21_3
+; RV32I-NEXT:  .LBB21_2:
+; RV32I-NEXT:    sll a6, a0, a4
+; RV32I-NEXT:    srl a7, a0, t3
+; RV32I-NEXT:    sll t0, a1, a4
+; RV32I-NEXT:    or t2, a7, t0
+; RV32I-NEXT:  .LBB21_3:
+; RV32I-NEXT:    neg a7, t1
+; RV32I-NEXT:    mv t0, a1
+; RV32I-NEXT:    beqz t1, .LBB21_5
+; RV32I-NEXT:  # %bb.4:
+; RV32I-NEXT:    mv t0, t2
+; RV32I-NEXT:  .LBB21_5:
+; RV32I-NEXT:    andi t2, a7, 63
+; RV32I-NEXT:    neg t4, t2
+; RV32I-NEXT:    bltu t2, a5, .LBB21_7
+; RV32I-NEXT:  # %bb.6:
+; RV32I-NEXT:    srl t5, a1, t2
+; RV32I-NEXT:    bnez t2, .LBB21_8
+; RV32I-NEXT:    j .LBB21_9
+; RV32I-NEXT:  .LBB21_7:
+; RV32I-NEXT:    srl t5, a0, a7
+; RV32I-NEXT:    sll t6, a1, t4
+; RV32I-NEXT:    or t5, t5, t6
+; RV32I-NEXT:    beqz t2, .LBB21_9
+; RV32I-NEXT:  .LBB21_8:
+; RV32I-NEXT:    mv a0, t5
+; RV32I-NEXT:  .LBB21_9:
+; RV32I-NEXT:    bltu t2, a5, .LBB21_12
+; RV32I-NEXT:  # %bb.10:
+; RV32I-NEXT:    li t5, 0
+; RV32I-NEXT:    bgeu t1, a5, .LBB21_13
+; RV32I-NEXT:  .LBB21_11:
+; RV32I-NEXT:    sll a1, a2, a4
+; RV32I-NEXT:    srl t3, a2, t3
+; RV32I-NEXT:    sll a4, a3, a4
+; RV32I-NEXT:    or t3, t3, a4
+; RV32I-NEXT:    mv a4, a3
+; RV32I-NEXT:    bnez t1, .LBB21_14
+; RV32I-NEXT:    j .LBB21_15
+; RV32I-NEXT:  .LBB21_12:
+; RV32I-NEXT:    srl t5, a1, a7
+; RV32I-NEXT:    bltu t1, a5, .LBB21_11
+; RV32I-NEXT:  .LBB21_13:
+; RV32I-NEXT:    li a1, 0
+; RV32I-NEXT:    sll t3, a2, t1
+; RV32I-NEXT:    mv a4, a3
+; RV32I-NEXT:    beqz t1, .LBB21_15
+; RV32I-NEXT:  .LBB21_14:
+; RV32I-NEXT:    mv a4, t3
+; RV32I-NEXT:  .LBB21_15:
+; RV32I-NEXT:    bltu t2, a5, .LBB21_17
+; RV32I-NEXT:  # %bb.16:
+; RV32I-NEXT:    srl t1, a3, t2
+; RV32I-NEXT:    bnez t2, .LBB21_18
+; RV32I-NEXT:    j .LBB21_19
+; RV32I-NEXT:  .LBB21_17:
+; RV32I-NEXT:    srl t1, a2, a7
+; RV32I-NEXT:    sll t3, a3, t4
+; RV32I-NEXT:    or t1, t1, t3
+; RV32I-NEXT:    beqz t2, .LBB21_19
+; RV32I-NEXT:  .LBB21_18:
+; RV32I-NEXT:    mv a2, t1
+; RV32I-NEXT:  .LBB21_19:
+; RV32I-NEXT:    or a0, a6, a0
+; RV32I-NEXT:    or a6, t0, t5
+; RV32I-NEXT:    bltu t2, a5, .LBB21_21
+; RV32I-NEXT:  # %bb.20:
+; RV32I-NEXT:    li a3, 0
+; RV32I-NEXT:    j .LBB21_22
+; RV32I-NEXT:  .LBB21_21:
+; RV32I-NEXT:    srl a3, a3, a7
+; RV32I-NEXT:  .LBB21_22:
+; RV32I-NEXT:    or a1, a1, a2
+; RV32I-NEXT:    or a3, a4, a3
+; RV32I-NEXT:    add a0, a0, a1
+; RV32I-NEXT:    sltu a1, a0, a1
+; RV32I-NEXT:    add a3, a6, a3
+; RV32I-NEXT:    add a1, a3, a1
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: rotl_64_mask_multiple:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    andi a3, a2, 63
+; RV64I-NEXT:    sll a4, a0, a2
+; RV64I-NEXT:    sll a2, a1, a2
+; RV64I-NEXT:    negw a3, a3
+; RV64I-NEXT:    srl a0, a0, a3
+; RV64I-NEXT:    srl a1, a1, a3
+; RV64I-NEXT:    or a0, a4, a0
+; RV64I-NEXT:    or a1, a2, a1
+; RV64I-NEXT:    add a0, a0, a1
+; RV64I-NEXT:    ret
+;
+; RV32ZBB-LABEL: rotl_64_mask_multiple:
+; RV32ZBB:       # %bb.0:
+; RV32ZBB-NEXT:    andi t1, a4, 63
+; RV32ZBB-NEXT:    li a5, 32
+; RV32ZBB-NEXT:    neg t3, t1
+; RV32ZBB-NEXT:    bltu t1, a5, .LBB21_2
+; RV32ZBB-NEXT:  # %bb.1:
+; RV32ZBB-NEXT:    li a6, 0
+; RV32ZBB-NEXT:    sll t2, a0, t1
+; RV32ZBB-NEXT:    j .LBB21_3
+; RV32ZBB-NEXT:  .LBB21_2:
+; RV32ZBB-NEXT:    sll a6, a0, a4
+; RV32ZBB-NEXT:    srl a7, a0, t3
+; RV32ZBB-NEXT:    sll t0, a1, a4
+; RV32ZBB-NEXT:    or t2, a7, t0
+; RV32ZBB-NEXT:  .LBB21_3:
+; RV32ZBB-NEXT:    neg a7, t1
+; RV32ZBB-NEXT:    mv t0, a1
+; RV32ZBB-NEXT:    beqz t1, .LBB21_5
+; RV32ZBB-NEXT:  # %bb.4:
+; RV32ZBB-NEXT:    mv t0, t2
+; RV32ZBB-NEXT:  .LBB21_5:
+; RV32ZBB-NEXT:    andi t2, a7, 63
+; RV32ZBB-NEXT:    neg t4, t2
+; RV32ZBB-NEXT:    bltu t2, a5, .LBB21_7
+; RV32ZBB-NEXT:  # %bb.6:
+; RV32ZBB-NEXT:    srl t5, a1, t2
+; RV32ZBB-NEXT:    bnez t2, .LBB21_8
+; RV32ZBB-NEXT:    j .LBB21_9
+; RV32ZBB-NEXT:  .LBB21_7:
+; RV32ZBB-NEXT:    srl t5, a0, a7
+; RV32ZBB-NEXT:    sll t6, a1, t4
+; RV32ZBB-NEXT:    or t5, t5, t6
+; RV32ZBB-NEXT:    beqz t2, .LBB21_9
+; RV32ZBB-NEXT:  .LBB21_8:
+; RV32ZBB-NEXT:    mv a0, t5
+; RV32ZBB-NEXT:  .LBB21_9:
+; RV32ZBB-NEXT:    bltu t2, a5, .LBB21_12
+; RV32ZBB-NEXT:  # %bb.10:
+; RV32ZBB-NEXT:    li t5, 0
+; RV32ZBB-NEXT:    bgeu t1, a5, .LBB21_13
+; RV32ZBB-NEXT:  .LBB21_11:
+; RV32ZBB-NEXT:    sll a1, a2, a4
+; RV32ZBB-NEXT:    srl t3, a2, t3
+; RV32ZBB-NEXT:    sll a4, a3, a4
+; RV32ZBB-NEXT:    or t3, t3, a4
+; RV32ZBB-NEXT:    mv a4, a3
+; RV32ZBB-NEXT:    bnez t1, .LBB21_14
+; RV32ZBB-NEXT:    j .LBB21_15
+; RV32ZBB-NEXT:  .LBB21_12:
+; RV32ZBB-NEXT:    srl t5, a1, a7
+; RV32ZBB-NEXT:    bltu t1, a5, .LBB21_11
+; RV32ZBB-NEXT:  .LBB21_13:
+; RV32ZBB-NEXT:    li a1, 0
+; RV32ZBB-NEXT:    sll t3, a2, t1
+; RV32ZBB-NEXT:    mv a4, a3
+; RV32ZBB-NEXT:    beqz t1, .LBB21_15
+; RV32ZBB-NEXT:  .LBB21_14:
+; RV32ZBB-NEXT:    mv a4, t3
+; RV32ZBB-NEXT:  .LBB21_15:
+; RV32ZBB-NEXT:    bltu t2, a5, .LBB21_17
+; RV32ZBB-NEXT:  # %bb.16:
+; RV32ZBB-NEXT:    srl t1, a3, t2
+; RV32ZBB-NEXT:    bnez t2, .LBB21_18
+; RV32ZBB-NEXT:    j .LBB21_19
+; RV32ZBB-NEXT:  .LBB21_17:
+; RV32ZBB-NEXT:    srl t1, a2, a7
+; RV32ZBB-NEXT:    sll t3, a3, t4
+; RV32ZBB-NEXT:    or t1, t1, t3
+; RV32ZBB-NEXT:    beqz t2, .LBB21_19
+; RV32ZBB-NEXT:  .LBB21_18:
+; RV32ZBB-NEXT:    mv a2, t1
+; RV32ZBB-NEXT:  .LBB21_19:
+; RV32ZBB-NEXT:    or a0, a6, a0
+; RV32ZBB-NEXT:    or a6, t0, t5
+; RV32ZBB-NEXT:    bltu t2, a5, .LBB21_21
+; RV32ZBB-NEXT:  # %bb.20:
+; RV32ZBB-NEXT:    li a3, 0
+; RV32ZBB-NEXT:    j .LBB21_22
+; RV32ZBB-NEXT:  .LBB21_21:
+; RV32ZBB-NEXT:    srl a3, a3, a7
+; RV32ZBB-NEXT:  .LBB21_22:
+; RV32ZBB-NEXT:    or a1, a1, a2
+; RV32ZBB-NEXT:    or a3, a4, a3
+; RV32ZBB-NEXT:    add a0, a0, a1
+; RV32ZBB-NEXT:    sltu a1, a0, a1
+; RV32ZBB-NEXT:    add a3, a6, a3
+; RV32ZBB-NEXT:    add a1, a3, a1
+; RV32ZBB-NEXT:    ret
+;
+; RV64ZBB-LABEL: rotl_64_mask_multiple:
+; RV64ZBB:       # %bb.0:
+; RV64ZBB-NEXT:    rol a0, a0, a2
+; RV64ZBB-NEXT:    rol a1, a1, a2
+; RV64ZBB-NEXT:    add a0, a0, a1
+; RV64ZBB-NEXT:    ret
+;
+; RV32XTHEADBB-LABEL: rotl_64_mask_multiple:
+; RV32XTHEADBB:       # %bb.0:
+; RV32XTHEADBB-NEXT:    andi t1, a4, 63
+; RV32XTHEADBB-NEXT:    li a5, 32
+; RV32XTHEADBB-NEXT:    neg t3, t1
+; RV32XTHEADBB-NEXT:    bltu t1, a5, .LBB21_2
+; RV32XTHEADBB-NEXT:  # %bb.1:
+; RV32XTHEADBB-NEXT:    li a6, 0
+; RV32XTHEADBB-NEXT:    sll t2, a0, t1
+; RV32XTHEADBB-NEXT:    j .LBB21_3
+; RV32XTHEADBB-NEXT:  .LBB21_2:
+; RV32XTHEADBB-NEXT:    sll a6, a0, a4
+; RV32XTHEADBB-NEXT:    srl a7, a0, t3
+; RV32XTHEADBB-NEXT:    sll t0, a1, a4
+; RV32XTHEADBB-NEXT:    or t2, a7, t0
+; RV32XTHEADBB-NEXT:  .LBB21_3:
+; RV32XTHEADBB-NEXT:    neg a7, t1
+; RV32XTHEADBB-NEXT:    mv t0, a1
+; RV32XTHEADBB-NEXT:    beqz t1, .LBB21_5
+; RV32XTHEADBB-NEXT:  # %bb.4:
+; RV32XTHEADBB-NEXT:    mv t0, t2
+; RV32XTHEADBB-NEXT:  .LBB21_5:
+; RV32XTHEADBB-NEXT:    andi t2, a7, 63
+; RV32XTHEADBB-NEXT:    neg t4, t2
+; RV32XTHEADBB-NEXT:    bltu t2, a5, .LBB21_7
+; RV32XTHEADBB-NEXT:  # %bb.6:
+; RV32XTHEADBB-NEXT:    srl t5, a1, t2
+; RV32XTHEADBB-NEXT:    bnez t2, .LBB21_8
+; RV32XTHEADBB-NEXT:    j .LBB21_9
+; RV32XTHEADBB-NEXT:  .LBB21_7:
+; RV32XTHEADBB-NEXT:    srl t5, a0, a7
+; RV32XTHEADBB-NEXT:    sll t6, a1, t4
+; RV32XTHEADBB-NEXT:    or t5, t5, t6
+; RV32XTHEADBB-NEXT:    beqz t2, .LBB21_9
+; RV32XTHEADBB-NEXT:  .LBB21_8:
+; RV32XTHEADBB-NEXT:    mv a0, t5
+; RV32XTHEADBB-NEXT:  .LBB21_9:
+; RV32XTHEADBB-NEXT:    bltu t2, a5, .LBB21_12
+; RV32XTHEADBB-NEXT:  # %bb.10:
+; RV32XTHEADBB-NEXT:    li t5, 0
+; RV32XTHEADBB-NEXT:    bgeu t1, a5, .LBB21_13
+; RV32XTHEADBB-NEXT:  .LBB21_11:
+; RV32XTHEADBB-NEXT:    sll a1, a2, a4
+; RV32XTHEADBB-NEXT:    srl t3, a2, t3
+; RV32XTHEADBB-NEXT:    sll a4, a3, a4
+; RV32XTHEADBB-NEXT:    or t3, t3, a4
+; RV32XTHEADBB-NEXT:    mv a4, a3
+; RV32XTHEADBB-NEXT:    bnez t1, .LBB21_14
+; RV32XTHEADBB-NEXT:    j .LBB21_15
+; RV32XTHEADBB-NEXT:  .LBB21_12:
+; RV32XTHEADBB-NEXT:    srl t5, a1, a7
+; RV32XTHEADBB-NEXT:    bltu t1, a5, .LBB21_11
+; RV32XTHEADBB-NEXT:  .LBB21_13:
+; RV32XTHEADBB-NEXT:    li a1, 0
+; RV32XTHEADBB-NEXT:    sll t3, a2, t1
+; RV32XTHEADBB-NEXT:    mv a4, a3
+; RV32XTHEADBB-NEXT:    beqz t1, .LBB21_15
+; RV32XTHEADBB-NEXT:  .LBB21_14:
+; RV32XTHEADBB-NEXT:    mv a4, t3
+; RV32XTHEADBB-NEXT:  .LBB21_15:
+; RV32XTHEADBB-NEXT:    bltu t2, a5, .LBB21_17
+; RV32XTHEADBB-NEXT:  # %bb.16:
+; RV32XTHEADBB-NEXT:    srl t1, a3, t2
+; RV32XTHEADBB-NEXT:    bnez t2, .LBB21_18
+; RV32XTHEADBB-NEXT:    j .LBB21_19
+; RV32XTHEADBB-NEXT:  .LBB21_17:
+; RV32XTHEADBB-NEXT:    srl t1, a2, a7
+; RV32XTHEADBB-NEXT:    sll t3, a3, t4
+; RV32XTHEADBB-NEXT:    or t1, t1, t3
+; RV32XTHEADBB-NEXT:    beqz t2, .LBB21_19
+; RV32XTHEADBB-NEXT:  .LBB21_18:
+; RV32XTHEADBB-NEXT:    mv a2, t1
+; RV32XTHEADBB-NEXT:  .LBB21_19:
+; RV32XTHEADBB-NEXT:    or a0, a6, a0
+; RV32XTHEADBB-NEXT:    or a6, t0, t5
+; RV32XTHEADBB-NEXT:    bltu t2, a5, .LBB21_21
+; RV32XTHEADBB-NEXT:  # %bb.20:
+; RV32XTHEADBB-NEXT:    li a3, 0
+; RV32XTHEADBB-NEXT:    j .LBB21_22
+; RV32XTHEADBB-NEXT:  .LBB21_21:
+; RV32XTHEADBB-NEXT:    srl a3, a3, a7
+; RV32XTHEADBB-NEXT:  .LBB21_22:
+; RV32XTHEADBB-NEXT:    or a1, a1, a2
+; RV32XTHEADBB-NEXT:    or a3, a4, a3
+; RV32XTHEADBB-NEXT:    add a0, a0, a1
+; RV32XTHEADBB-NEXT:    sltu a1, a0, a1
+; RV32XTHEADBB-NEXT:    add a3, a6, a3
+; RV32XTHEADBB-NEXT:    add a1, a3, a1
+; RV32XTHEADBB-NEXT:    ret
+;
+; RV64XTHEADBB-LABEL: rotl_64_mask_multiple:
+; RV64XTHEADBB:       # %bb.0:
+; RV64XTHEADBB-NEXT:    andi a3, a2, 63
+; RV64XTHEADBB-NEXT:    sll a4, a0, a2
+; RV64XTHEADBB-NEXT:    sll a2, a1, a2
+; RV64XTHEADBB-NEXT:    negw a3, a3
+; RV64XTHEADBB-NEXT:    srl a0, a0, a3
+; RV64XTHEADBB-NEXT:    srl a1, a1, a3
+; RV64XTHEADBB-NEXT:    or a0, a4, a0
+; RV64XTHEADBB-NEXT:    or a1, a2, a1
+; RV64XTHEADBB-NEXT:    add a0, a0, a1
+; RV64XTHEADBB-NEXT:    ret
+  %maskedamt = and i64 %amt, 63
+  %1 = tail call i64 @llvm.fshl.i64(i64 %a, i64 %a, i64 %maskedamt)
+  %2 = tail call i64 @llvm.fshl.i64(i64 %b, i64 %b, i64 %maskedamt)
+  %3 = add i64 %1, %2
+  ret i64 %3
+}
+
+define signext i32 @rotr_32_mask_multiple(i32 signext %a, i32 signext %b, i32 signext %amt) nounwind {
+; RV32I-LABEL: rotr_32_mask_multiple:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    andi a3, a2, 31
+; RV32I-NEXT:    srl a4, a0, a2
+; RV32I-NEXT:    srl a2, a1, a2
+; RV32I-NEXT:    neg a3, a3
+; RV32I-NEXT:    sll a0, a0, a3
+; RV32I-NEXT:    sll a1, a1, a3
+; RV32I-NEXT:    or a0, a4, a0
+; RV32I-NEXT:    or a1, a2, a1
+; RV32I-NEXT:    add a0, a0, a1
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: rotr_32_mask_multiple:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    andi a3, a2, 31
+; RV64I-NEXT:    srlw a4, a0, a2
+; RV64I-NEXT:    srlw a2, a1, a2
+; RV64I-NEXT:    negw a5, a3
+; RV64I-NEXT:    negw a3, a3
+; RV64I-NEXT:    sllw a0, a0, a5
+; RV64I-NEXT:    sllw a1, a1, a3
+; RV64I-NEXT:    or a0, a4, a0
+; RV64I-NEXT:    or a1, a2, a1
+; RV64I-NEXT:    addw a0, a0, a1
+; RV64I-NEXT:    ret
+;
+; RV32ZBB-LABEL: rotr_32_mask_multiple:
+; RV32ZBB:       # %bb.0:
+; RV32ZBB-NEXT:    ror a0, a0, a2
+; RV32ZBB-NEXT:    ror a1, a1, a2
+; RV32ZBB-NEXT:    add a0, a0, a1
+; RV32ZBB-NEXT:    ret
+;
+; RV64ZBB-LABEL: rotr_32_mask_multiple:
+; RV64ZBB:       # %bb.0:
+; RV64ZBB-NEXT:    rorw a0, a0, a2
+; RV64ZBB-NEXT:    rorw a1, a1, a2
+; RV64ZBB-NEXT:    addw a0, a0, a1
+; RV64ZBB-NEXT:    ret
+;
+; RV32XTHEADBB-LABEL: rotr_32_mask_multiple:
+; RV32XTHEADBB:       # %bb.0:
+; RV32XTHEADBB-NEXT:    andi a3, a2, 31
+; RV32XTHEADBB-NEXT:    srl a4, a0, a2
+; RV32XTHEADBB-NEXT:    srl a2, a1, a2
+; RV32XTHEADBB-NEXT:    neg a3, a3
+; RV32XTHEADBB-NEXT:    sll a0, a0, a3
+; RV32XTHEADBB-NEXT:    sll a1, a1, a3
+; RV32XTHEADBB-NEXT:    or a0, a4, a0
+; RV32XTHEADBB-NEXT:    or a1, a2, a1
+; RV32XTHEADBB-NEXT:    add a0, a0, a1
+; RV32XTHEADBB-NEXT:    ret
+;
+; RV64XTHEADBB-LABEL: rotr_32_mask_multiple:
+; RV64XTHEADBB:       # %bb.0:
+; RV64XTHEADBB-NEXT:    andi a3, a2, 31
+; RV64XTHEADBB-NEXT:    srlw a4, a0, a2
+; RV64XTHEADBB-NEXT:    srlw a2, a1, a2
+; RV64XTHEADBB-NEXT:    negw a5, a3
+; RV64XTHEADBB-NEXT:    negw a3, a3
+; RV64XTHEADBB-NEXT:    sllw a0, a0, a5
+; RV64XTHEADBB-NEXT:    sllw a1, a1, a3
+; RV64XTHEADBB-NEXT:    or a0, a4, a0
+; RV64XTHEADBB-NEXT:    or a1, a2, a1
+; RV64XTHEADBB-NEXT:    addw a0, a0, a1
+; RV64XTHEADBB-NEXT:    ret
+  %maskedamt = and i32 %amt, 31
+  %1 = tail call i32 @llvm.fshr.i32(i32 %a, i32 %a, i32 %maskedamt)
+  %2 = tail call i32 @llvm.fshr.i32(i32 %b, i32 %b, i32 %maskedamt)
+  %3 = add i32 %1, %2
+  ret i32 %3
+}
+
+define i64 @rotr_64_mask_multiple(i64 %a, i64 %b, i64 %amt) nounwind {
+; RV32I-LABEL: rotr_64_mask_multiple:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    andi t0, a4, 63
+; RV32I-NEXT:    li a6, 32
+; RV32I-NEXT:    neg t4, t0
+; RV32I-NEXT:    bltu t0, a6, .LBB23_2
+; RV32I-NEXT:  # %bb.1:
+; RV32I-NEXT:    srl a7, a1, t0
+; RV32I-NEXT:    mv a5, a0
+; RV32I-NEXT:    bnez t0, .LBB23_3
+; RV32I-NEXT:    j .LBB23_4
+; RV32I-NEXT:  .LBB23_2:
+; RV32I-NEXT:    srl a5, a0, a4
+; RV32I-NEXT:    sll a7, a1, t4
+; RV32I-NEXT:    or a7, a5, a7
+; RV32I-NEXT:    mv a5, a0
+; RV32I-NEXT:    beqz t0, .LBB23_4
+; RV32I-NEXT:  .LBB23_3:
+; RV32I-NEXT:    mv a5, a7
+; RV32I-NEXT:  .LBB23_4:
+; RV32I-NEXT:    neg t2, t0
+; RV32I-NEXT:    bltu t0, a6, .LBB23_6
+; RV32I-NEXT:  # %bb.5:
+; RV32I-NEXT:    li a7, 0
+; RV32I-NEXT:    j .LBB23_7
+; RV32I-NEXT:  .LBB23_6:
+; RV32I-NEXT:    srl a7, a1, a4
+; RV32I-NEXT:  .LBB23_7:
+; RV32I-NEXT:    andi t1, t2, 63
+; RV32I-NEXT:    neg t5, t1
+; RV32I-NEXT:    bltu t1, a6, .LBB23_9
+; RV32I-NEXT:  # %bb.8:
+; RV32I-NEXT:    li t3, 0
+; RV32I-NEXT:    sll a0, a0, t1
+; RV32I-NEXT:    bnez t1, .LBB23_10
+; RV32I-NEXT:    j .LBB23_11
+; RV32I-NEXT:  .LBB23_9:
+; RV32I-NEXT:    sll t3, a0, t2
+; RV32I-NEXT:    srl a0, a0, t5
+; RV32I-NEXT:    sll t6, a1, t2
+; RV32I-NEXT:    or a0, a0, t6
+; RV32I-NEXT:    beqz t1, .LBB23_11
+; RV32I-NEXT:  .LBB23_10:
+; RV32I-NEXT:    mv a1, a0
+; RV32I-NEXT:  .LBB23_11:
+; RV32I-NEXT:    bltu t0, a6, .LBB23_13
+; RV32I-NEXT:  # %bb.12:
+; RV32I-NEXT:    srl t4, a3, t0
+; RV32I-NEXT:    mv a0, a2
+; RV32I-NEXT:    bnez t0, .LBB23_14
+; RV32I-NEXT:    j .LBB23_15
+; RV32I-NEXT:  .LBB23_13:
+; RV32I-NEXT:    srl a0, a2, a4
+; RV32I-NEXT:    sll t4, a3, t4
+; RV32I-NEXT:    or t4, a0, t4
+; RV32I-NEXT:    mv a0, a2
+; RV32I-NEXT:    beqz t0, .LBB23_15
+; RV32I-NEXT:  .LBB23_14:
+; RV32I-NEXT:    mv a0, t4
+; RV32I-NEXT:  .LBB23_15:
+; RV32I-NEXT:    bltu t0, a6, .LBB23_18
+; RV32I-NEXT:  # %bb.16:
+; RV32I-NEXT:    li a4, 0
+; RV32I-NEXT:    bgeu t1, a6, .LBB23_19
+; RV32I-NEXT:  .LBB23_17:
+; RV32I-NEXT:    sll a6, a2, t2
+; RV32I-NEXT:    srl a2, a2, t5
+; RV32I-NEXT:    sll t0, a3, t2
+; RV32I-NEXT:    or a2, a2, t0
+; RV32I-NEXT:    j .LBB23_20
+; RV32I-NEXT:  .LBB23_18:
+; RV32I-NEXT:    srl a4, a3, a4
+; RV32I-NEXT:    bltu t1, a6, .LBB23_17
+; RV32I-NEXT:  .LBB23_19:
+; RV32I-NEXT:    li a6, 0
+; RV32I-NEXT:    sll a2, a2, t1
+; RV32I-NEXT:  .LBB23_20:
+; RV32I-NEXT:    or a5, a5, t3
+; RV32I-NEXT:    or a1, a7, a1
+; RV32I-NEXT:    beqz t1, .LBB23_22
+; RV32I-NEXT:  # %bb.21:
+; RV32I-NEXT:    mv a3, a2
+; RV32I-NEXT:  .LBB23_22:
+; RV32I-NEXT:    or a2, a0, a6
+; RV32I-NEXT:    or a3, a4, a3
+; RV32I-NEXT:    add a0, a5, a2
+; RV32I-NEXT:    sltu a2, a0, a2
+; RV32I-NEXT:    add a1, a1, a3
+; RV32I-NEXT:    add a1, a1, a2
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: rotr_64_mask_multiple:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    andi a3, a2, 63
+; RV64I-NEXT:    srl a4, a0, a2
+; RV64I-NEXT:    srl a2, a1, a2
+; RV64I-NEXT:    negw a3, a3
+; RV64I-NEXT:    sll a0, a0, a3
+; RV64I-NEXT:    sll a1, a1, a3
+; RV64I-NEXT:    or a0, a4, a0
+; RV64I-NEXT:    or a1, a2, a1
+; RV64I-NEXT:    add a0, a0, a1
+; RV64I-NEXT:    ret
+;
+; RV32ZBB-LABEL: rotr_64_mask_multiple:
+; RV32ZBB:       # %bb.0:
+; RV32ZBB-NEXT:    andi t0, a4, 63
+; RV32ZBB-NEXT:    li a6, 32
+; RV32ZBB-NEXT:    neg t4, t0
+; RV32ZBB-NEXT:    bltu t0, a6, .LBB23_2
+; RV32ZBB-NEXT:  # %bb.1:
+; RV32ZBB-NEXT:    srl a7, a1, t0
+; RV32ZBB-NEXT:    mv a5, a0
+; RV32ZBB-NEXT:    bnez t0, .LBB23_3
+; RV32ZBB-NEXT:    j .LBB23_4
+; RV32ZBB-NEXT:  .LBB23_2:
+; RV32ZBB-NEXT:    srl a5, a0, a4
+; RV32ZBB-NEXT:    sll a7, a1, t4
+; RV32ZBB-NEXT:    or a7, a5, a7
+; RV32ZBB-NEXT:    mv a5, a0
+; RV32ZBB-NEXT:    beqz t0, .LBB23_4
+; RV32ZBB-NEXT:  .LBB23_3:
+; RV32ZBB-NEXT:    mv a5, a7
+; RV32ZBB-NEXT:  .LBB23_4:
+; RV32ZBB-NEXT:    neg t2, t0
+; RV32ZBB-NEXT:    bltu t0, a6, .LBB23_6
+; RV32ZBB-NEXT:  # %bb.5:
+; RV32ZBB-NEXT:    li a7, 0
+; RV32ZBB-NEXT:    j .LBB23_7
+; RV32ZBB-NEXT:  .LBB23_6:
+; RV32ZBB-NEXT:    srl a7, a1, a4
+; RV32ZBB-NEXT:  .LBB23_7:
+; RV32ZBB-NEXT:    andi t1, t2, 63
+; RV32ZBB-NEXT:    neg t5, t1
+; RV32ZBB-NEXT:    bltu t1, a6, .LBB23_9
+; RV32ZBB-NEXT:  # %bb.8:
+; RV32ZBB-NEXT:    li t3, 0
+; RV32ZBB-NEXT:    sll a0, a0, t1
+; RV32ZBB-NEXT:    bnez t1, .LBB23_10
+; RV32ZBB-NEXT:    j .LBB23_11
+; RV32ZBB-NEXT:  .LBB23_9:
+; RV32ZBB-NEXT:    sll t3, a0, t2
+; RV32ZBB-NEXT:    srl a0, a0, t5
+; RV32ZBB-NEXT:    sll t6, a1, t2
+; RV32ZBB-NEXT:    or a0, a0, t6
+; RV32ZBB-NEXT:    beqz t1, .LBB23_11
+; RV32ZBB-NEXT:  .LBB23_10:
+; RV32ZBB-NEXT:    mv a1, a0
+; RV32ZBB-NEXT:  .LBB23_11:
+; RV32ZBB-NEXT:    bltu t0, a6, .LBB23_13
+; RV32ZBB-NEXT:  # %bb.12:
+; RV32ZBB-NEXT:    srl t4, a3, t0
+; RV32ZBB-NEXT:    mv a0, a2
+; RV32ZBB-NEXT:    bnez t0, .LBB23_14
+; RV32ZBB-NEXT:    j .LBB23_15
+; RV32ZBB-NEXT:  .LBB23_13:
+; RV32ZBB-NEXT:    srl a0, a2, a4
+; RV32ZBB-NEXT:    sll t4, a3, t4
+; RV32ZBB-NEXT:    or t4, a0, t4
+; RV32ZBB-NEXT:    mv a0, a2
+; RV32ZBB-NEXT:    beqz t0, .LBB23_15
+; RV32ZBB-NEXT:  .LBB23_14:
+; RV32ZBB-NEXT:    mv a0, t4
+; RV32ZBB-NEXT:  .LBB23_15:
+; RV32ZBB-NEXT:    bltu t0, a6, .LBB23_18
+; RV32ZBB-NEXT:  # %bb.16:
+; RV32ZBB-NEXT:    li a4, 0
+; RV32ZBB-NEXT:    bgeu t1, a6, .LBB23_19
+; RV32ZBB-NEXT:  .LBB23_17:
+; RV32ZBB-NEXT:    sll a6, a2, t2
+; RV32ZBB-NEXT:    srl a2, a2, t5
+; RV32ZBB-NEXT:    sll t0, a3, t2
+; RV32ZBB-NEXT:    or a2, a2, t0
+; RV32ZBB-NEXT:    j .LBB23_20
+; RV32ZBB-NEXT:  .LBB23_18:
+; RV32ZBB-NEXT:    srl a4, a3, a4
+; RV32ZBB-NEXT:    bltu t1, a6, .LBB23_17
+; RV32ZBB-NEXT:  .LBB23_19:
+; RV32ZBB-NEXT:    li a6, 0
+; RV32ZBB-NEXT:    sll a2, a2, t1
+; RV32ZBB-NEXT:  .LBB23_20:
+; RV32ZBB-NEXT:    or a5, a5, t3
+; RV32ZBB-NEXT:    or a1, a7, a1
+; RV32ZBB-NEXT:    beqz t1, .LBB23_22
+; RV32ZBB-NEXT:  # %bb.21:
+; RV32ZBB-NEXT:    mv a3, a2
+; RV32ZBB-NEXT:  .LBB23_22:
+; RV32ZBB-NEXT:    or a2, a0, a6
+; RV32ZBB-NEXT:    or a3, a4, a3
+; RV32ZBB-NEXT:    add a0, a5, a2
+; RV32ZBB-NEXT:    sltu a2, a0, a2
+; RV32ZBB-NEXT:    add a1, a1, a3
+; RV32ZBB-NEXT:    add a1, a1, a2
+; RV32ZBB-NEXT:    ret
+;
+; RV64ZBB-LABEL: rotr_64_mask_multiple:
+; RV64ZBB:       # %bb.0:
+; RV64ZBB-NEXT:    ror a0, a0, a2
+; RV64ZBB-NEXT:    ror a1, a1, a2
+; RV64ZBB-NEXT:    add a0, a0, a1
+; RV64ZBB-NEXT:    ret
+;
+; RV32XTHEADBB-LABEL: rotr_64_mask_multiple:
+; RV32XTHEADBB:       # %bb.0:
+; RV32XTHEADBB-NEXT:    andi t0, a4, 63
+; RV32XTHEADBB-NEXT:    li a6, 32
+; RV32XTHEADBB-NEXT:    neg t4, t0
+; RV32XTHEADBB-NEXT:    bltu t0, a6, .LBB23_2
+; RV32XTHEADBB-NEXT:  # %bb.1:
+; RV32XTHEADBB-NEXT:    srl a7, a1, t0
+; RV32XTHEADBB-NEXT:    mv a5, a0
+; RV32XTHEADBB-NEXT:    bnez t0, .LBB23_3
+; RV32XTHEADBB-NEXT:    j .LBB23_4
+; RV32XTHEADBB-NEXT:  .LBB23_2:
+; RV32XTHEADBB-NEXT:    srl a5, a0, a4
+; RV32XTHEADBB-NEXT:    sll a7, a1, t4
+; RV32XTHEADBB-NEXT:    or a7, a5, a7
+; RV32XTHEADBB-NEXT:    mv a5, a0
+; RV32XTHEADBB-NEXT:    beqz t0, .LBB23_4
+; RV32XTHEADBB-NEXT:  .LBB23_3:
+; RV32XTHEADBB-NEXT:    mv a5, a7
+; RV32XTHEADBB-NEXT:  .LBB23_4:
+; RV32XTHEADBB-NEXT:    neg t2, t0
+; RV32XTHEADBB-NEXT:    bltu t0, a6, .LBB23_6
+; RV32XTHEADBB-NEXT:  # %bb.5:
+; RV32XTHEADBB-NEXT:    li a7, 0
+; RV32XTHEADBB-NEXT:    j .LBB23_7
+; RV32XTHEADBB-NEXT:  .LBB23_6:
+; RV32XTHEADBB-NEXT:    srl a7, a1, a4
+; RV32XTHEADBB-NEXT:  .LBB23_7:
+; RV32XTHEADBB-NEXT:    andi t1, t2, 63
+; RV32XTHEADBB-NEXT:    neg t5, t1
+; RV32XTHEADBB-NEXT:    bltu t1, a6, .LBB23_9
+; RV32XTHEADBB-NEXT:  # %bb.8:
+; RV32XTHEADBB-NEXT:    li t3, 0
+; RV32XTHEADBB-NEXT:    sll a0, a0, t1
+; RV32XTHEADBB-NEXT:    bnez t1, .LBB23_10
+; RV32XTHEADBB-NEXT:    j .LBB23_11
+; RV32XTHEADBB-NEXT:  .LBB23_9:
+; RV32XTHEADBB-NEXT:    sll t3, a0, t2
+; RV32XTHEADBB-NEXT:    srl a0, a0, t5
+; RV32XTHEADBB-NEXT:    sll t6, a1, t2
+; RV32XTHEADBB-NEXT:    or a0, a0, t6
+; RV32XTHEADBB-NEXT:    beqz t1, .LBB23_11
+; RV32XTHEADBB-NEXT:  .LBB23_10:
+; RV32XTHEADBB-NEXT:    mv a1, a0
+; RV32XTHEADBB-NEXT:  .LBB23_11:
+; RV32XTHEADBB-NEXT:    bltu t0, a6, .LBB23_13
+; RV32XTHEADBB-NEXT:  # %bb.12:
+; RV32XTHEADBB-NEXT:    srl t4, a3, t0
+; RV32XTHEADBB-NEXT:    mv a0, a2
+; RV32XTHEADBB-NEXT:    bnez t0, .LBB23_14
+; RV32XTHEADBB-NEXT:    j .LBB23_15
+; RV32XTHEADBB-NEXT:  .LBB23_13:
+; RV32XTHEADBB-NEXT:    srl a0, a2, a4
+; RV32XTHEADBB-NEXT:    sll t4, a3, t4
+; RV32XTHEADBB-NEXT:    or t4, a0, t4
+; RV32XTHEADBB-NEXT:    mv a0, a2
+; RV32XTHEADBB-NEXT:    beqz t0, .LBB23_15
+; RV32XTHEADBB-NEXT:  .LBB23_14:
+; RV32XTHEADBB-NEXT:    mv a0, t4
+; RV32XTHEADBB-NEXT:  .LBB23_15:
+; RV32XTHEADBB-NEXT:    bltu t0, a6, .LBB23_18
+; RV32XTHEADBB-NEXT:  # %bb.16:
+; RV32XTHEADBB-NEXT:    li a4, 0
+; RV32XTHEADBB-NEXT:    bgeu t1, a6, .LBB23_19
+; RV32XTHEADBB-NEXT:  .LBB23_17:
+; RV32XTHEADBB-NEXT:    sll a6, a2, t2
+; RV32XTHEADBB-NEXT:    srl a2, a2, t5
+; RV32XTHEADBB-NEXT:    sll t0, a3, t2
+; RV32XTHEADBB-NEXT:    or a2, a2, t0
+; RV32XTHEADBB-NEXT:    j .LBB23_20
+; RV32XTHEADBB-NEXT:  .LBB23_18:
+; RV32XTHEADBB-NEXT:    srl a4, a3, a4
+; RV32XTHEADBB-NEXT:    bltu t1, a6, .LBB23_17
+; RV32XTHEADBB-NEXT:  .LBB23_19:
+; RV32XTHEADBB-NEXT:    li a6, 0
+; RV32XTHEADBB-NEXT:    sll a2, a2, t1
+; RV32XTHEADBB-NEXT:  .LBB23_20:
+; RV32XTHEADBB-NEXT:    or a5, a5, t3
+; RV32XTHEADBB-NEXT:    or a1, a7, a1
+; RV32XTHEADBB-NEXT:    beqz t1, .LBB23_22
+; RV32XTHEADBB-NEXT:  # %bb.21:
+; RV32XTHEADBB-NEXT:    mv a3, a2
+; RV32XTHEADBB-NEXT:  .LBB23_22:
+; RV32XTHEADBB-NEXT:    or a2, a0, a6
+; RV32XTHEADBB-NEXT:    or a3, a4, a3
+; RV32XTHEADBB-NEXT:    add a0, a5, a2
+; RV32XTHEADBB-NEXT:    sltu a2, a0, a2
+; RV32XTHEADBB-NEXT:    add a1, a1, a3
+; RV32XTHEADBB-NEXT:    add a1, a1, a2
+; RV32XTHEADBB-NEXT:    ret
+;
+; RV64XTHEADBB-LABEL: rotr_64_mask_multiple:
+; RV64XTHEADBB:       # %bb.0:
+; RV64XTHEADBB-NEXT:    andi a3, a2, 63
+; RV64XTHEADBB-NEXT:    srl a4, a0, a2
+; RV64XTHEADBB-NEXT:    srl a2, a1, a2
+; RV64XTHEADBB-NEXT:    negw a3, a3
+; RV64XTHEADBB-NEXT:    sll a0, a0, a3
+; RV64XTHEADBB-NEXT:    sll a1, a1, a3
+; RV64XTHEADBB-NEXT:    or a0, a4, a0
+; RV64XTHEADBB-NEXT:    or a1, a2, a1
+; RV64XTHEADBB-NEXT:    add a0, a0, a1
+; RV64XTHEADBB-NEXT:    ret
+  %maskedamt = and i64 %amt, 63
+  %1 = tail call i64 @llvm.fshr.i64(i64 %a, i64 %a, i64 %maskedamt)
+  %2 = tail call i64 @llvm.fshr.i64(i64 %b, i64 %b, i64 %maskedamt)
+  %3 = add i64 %1, %2
+  ret i64 %3
+}
+
+define i64 @rotl_64_zext(i64 %x, i32 %y) nounwind {
+; RV32I-LABEL: rotl_64_zext:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    li a6, 64
+; RV32I-NEXT:    li a4, 32
+; RV32I-NEXT:    neg a5, a2
+; RV32I-NEXT:    srl a7, a0, a5
+; RV32I-NEXT:    bltu a2, a4, .LBB24_2
+; RV32I-NEXT:  # %bb.1:
+; RV32I-NEXT:    li a3, 0
+; RV32I-NEXT:    sll t1, a0, a2
+; RV32I-NEXT:    j .LBB24_3
+; RV32I-NEXT:  .LBB24_2:
+; RV32I-NEXT:    sll a3, a0, a2
+; RV32I-NEXT:    sll t0, a1, a2
+; RV32I-NEXT:    or t1, a7, t0
+; RV32I-NEXT:  .LBB24_3:
+; RV32I-NEXT:    sub t0, a6, a2
+; RV32I-NEXT:    mv a6, a1
+; RV32I-NEXT:    beqz a2, .LBB24_5
+; RV32I-NEXT:  # %bb.4:
+; RV32I-NEXT:    mv a6, t1
+; RV32I-NEXT:  .LBB24_5:
+; RV32I-NEXT:    bltu t0, a4, .LBB24_7
+; RV32I-NEXT:  # %bb.6:
+; RV32I-NEXT:    srl a2, a1, t0
+; RV32I-NEXT:    bnez t0, .LBB24_8
+; RV32I-NEXT:    j .LBB24_9
+; RV32I-NEXT:  .LBB24_7:
+; RV32I-NEXT:    neg a2, t0
+; RV32I-NEXT:    sll a2, a1, a2
+; RV32I-NEXT:    or a2, a7, a2
+; RV32I-NEXT:    beqz t0, .LBB24_9
+; RV32I-NEXT:  .LBB24_8:
+; RV32I-NEXT:    mv a0, a2
+; RV32I-NEXT:  .LBB24_9:
+; RV32I-NEXT:    bltu t0, a4, .LBB24_11
+; RV32I-NEXT:  # %bb.10:
+; RV32I-NEXT:    li a1, 0
+; RV32I-NEXT:    j .LBB24_12
+; RV32I-NEXT:  .LBB24_11:
+; RV32I-NEXT:    srl a1, a1, a5
+; RV32I-NEXT:  .LBB24_12:
+; RV32I-NEXT:    or a0, a3, a0
+; RV32I-NEXT:    or a1, a6, a1
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: rotl_64_zext:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    li a2, 64
+; RV64I-NEXT:    subw a2, a2, a1
+; RV64I-NEXT:    sll a1, a0, a1
+; RV64I-NEXT:    srl a0, a0, a2
+; RV64I-NEXT:    or a0, a1, a0
+; RV64I-NEXT:    ret
+;
+; RV32ZBB-LABEL: rotl_64_zext:
+; RV32ZBB:       # %bb.0:
+; RV32ZBB-NEXT:    li a6, 64
+; RV32ZBB-NEXT:    li a4, 32
+; RV32ZBB-NEXT:    neg a5, a2
+; RV32ZBB-NEXT:    srl a7, a0, a5
+; RV32ZBB-NEXT:    bltu a2, a4, .LBB24_2
+; RV32ZBB-NEXT:  # %bb.1:
+; RV32ZBB-NEXT:    li a3, 0
+; RV32ZBB-NEXT:    sll t1, a0, a2
+; RV32ZBB-NEXT:    j .LBB24_3
+; RV32ZBB-NEXT:  .LBB24_2:
+; RV32ZBB-NEXT:    sll a3, a0, a2
+; RV32ZBB-NEXT:    sll t0, a1, a2
+; RV32ZBB-NEXT:    or t1, a7, t0
+; RV32ZBB-NEXT:  .LBB24_3:
+; RV32ZBB-NEXT:    sub t0, a6, a2
+; RV32ZBB-NEXT:    mv a6, a1
+; RV32ZBB-NEXT:    beqz a2, .LBB24_5
+; RV32ZBB-NEXT:  # %bb.4:
+; RV32ZBB-NEXT:    mv a6, t1
+; RV32ZBB-NEXT:  .LBB24_5:
+; RV32ZBB-NEXT:    bltu t0, a4, .LBB24_7
+; RV32ZBB-NEXT:  # %bb.6:
+; RV32ZBB-NEXT:    srl a2, a1, t0
+; RV32ZBB-NEXT:    bnez t0, .LBB24_8
+; RV32ZBB-NEXT:    j .LBB24_9
+; RV32ZBB-NEXT:  .LBB24_7:
+; RV32ZBB-NEXT:    neg a2, t0
+; RV32ZBB-NEXT:    sll a2, a1, a2
+; RV32ZBB-NEXT:    or a2, a7, a2
+; RV32ZBB-NEXT:    beqz t0, .LBB24_9
+; RV32ZBB-NEXT:  .LBB24_8:
+; RV32ZBB-NEXT:    mv a0, a2
+; RV32ZBB-NEXT:  .LBB24_9:
+; RV32ZBB-NEXT:    bltu t0, a4, .LBB24_11
+; RV32ZBB-NEXT:  # %bb.10:
+; RV32ZBB-NEXT:    li a1, 0
+; RV32ZBB-NEXT:    j .LBB24_12
+; RV32ZBB-NEXT:  .LBB24_11:
+; RV32ZBB-NEXT:    srl a1, a1, a5
+; RV32ZBB-NEXT:  .LBB24_12:
+; RV32ZBB-NEXT:    or a0, a3, a0
+; RV32ZBB-NEXT:    or a1, a6, a1
+; RV32ZBB-NEXT:    ret
+;
+; RV64ZBB-LABEL: rotl_64_zext:
+; RV64ZBB:       # %bb.0:
+; RV64ZBB-NEXT:    li a2, 64
+; RV64ZBB-NEXT:    subw a2, a2, a1
+; RV64ZBB-NEXT:    sll a1, a0, a1
+; RV64ZBB-NEXT:    srl a0, a0, a2
+; RV64ZBB-NEXT:    or a0, a1, a0
+; RV64ZBB-NEXT:    ret
+;
+; RV32XTHEADBB-LABEL: rotl_64_zext:
+; RV32XTHEADBB:       # %bb.0:
+; RV32XTHEADBB-NEXT:    li a6, 64
+; RV32XTHEADBB-NEXT:    li a4, 32
+; RV32XTHEADBB-NEXT:    neg a5, a2
+; RV32XTHEADBB-NEXT:    srl a7, a0, a5
+; RV32XTHEADBB-NEXT:    bltu a2, a4, .LBB24_2
+; RV32XTHEADBB-NEXT:  # %bb.1:
+; RV32XTHEADBB-NEXT:    li a3, 0
+; RV32XTHEADBB-NEXT:    sll t1, a0, a2
+; RV32XTHEADBB-NEXT:    j .LBB24_3
+; RV32XTHEADBB-NEXT:  .LBB24_2:
+; RV32XTHEADBB-NEXT:    sll a3, a0, a2
+; RV32XTHEADBB-NEXT:    sll t0, a1, a2
+; RV32XTHEADBB-NEXT:    or t1, a7, t0
+; RV32XTHEADBB-NEXT:  .LBB24_3:
+; RV32XTHEADBB-NEXT:    sub t0, a6, a2
+; RV32XTHEADBB-NEXT:    mv a6, a1
+; RV32XTHEADBB-NEXT:    beqz a2, .LBB24_5
+; RV32XTHEADBB-NEXT:  # %bb.4:
+; RV32XTHEADBB-NEXT:    mv a6, t1
+; RV32XTHEADBB-NEXT:  .LBB24_5:
+; RV32XTHEADBB-NEXT:    bltu t0, a4, .LBB24_7
+; RV32XTHEADBB-NEXT:  # %bb.6:
+; RV32XTHEADBB-NEXT:    srl a2, a1, t0
+; RV32XTHEADBB-NEXT:    bnez t0, .LBB24_8
+; RV32XTHEADBB-NEXT:    j .LBB24_9
+; RV32XTHEADBB-NEXT:  .LBB24_7:
+; RV32XTHEADBB-NEXT:    neg a2, t0
+; RV32XTHEADBB-NEXT:    sll a2, a1, a2
+; RV32XTHEADBB-NEXT:    or a2, a7, a2
+; RV32XTHEADBB-NEXT:    beqz t0, .LBB24_9
+; RV32XTHEADBB-NEXT:  .LBB24_8:
+; RV32XTHEADBB-NEXT:    mv a0, a2
+; RV32XTHEADBB-NEXT:  .LBB24_9:
+; RV32XTHEADBB-NEXT:    bltu t0, a4, .LBB24_11
+; RV32XTHEADBB-NEXT:  # %bb.10:
+; RV32XTHEADBB-NEXT:    li a1, 0
+; RV32XTHEADBB-NEXT:    j .LBB24_12
+; RV32XTHEADBB-NEXT:  .LBB24_11:
+; RV32XTHEADBB-NEXT:    srl a1, a1, a5
+; RV32XTHEADBB-NEXT:  .LBB24_12:
+; RV32XTHEADBB-NEXT:    or a0, a3, a0
+; RV32XTHEADBB-NEXT:    or a1, a6, a1
+; RV32XTHEADBB-NEXT:    ret
+;
+; RV64XTHEADBB-LABEL: rotl_64_zext:
+; RV64XTHEADBB:       # %bb.0:
+; RV64XTHEADBB-NEXT:    li a2, 64
+; RV64XTHEADBB-NEXT:    subw a2, a2, a1
+; RV64XTHEADBB-NEXT:    sll a1, a0, a1
+; RV64XTHEADBB-NEXT:    srl a0, a0, a2
+; RV64XTHEADBB-NEXT:    or a0, a1, a0
+; RV64XTHEADBB-NEXT:    ret
+  %z = sub i32 64, %y
+  %zext = zext i32 %z to i64
+  %zexty = zext i32 %y to i64
+  %b = shl i64 %x, %zexty
+  %c = lshr i64 %x, %zext
+  %d = or i64 %b, %c
+  ret i64 %d
+}
+
+define i64 @rotr_64_zext(i64 %x, i32 %y) nounwind {
+; RV32I-LABEL: rotr_64_zext:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    li a5, 32
+; RV32I-NEXT:    neg a6, a2
+; RV32I-NEXT:    sll a4, a1, a6
+; RV32I-NEXT:    bltu a2, a5, .LBB25_2
+; RV32I-NEXT:  # %bb.1:
+; RV32I-NEXT:    srl a7, a1, a2
+; RV32I-NEXT:    j .LBB25_3
+; RV32I-NEXT:  .LBB25_2:
+; RV32I-NEXT:    srl a3, a0, a2
+; RV32I-NEXT:    or a7, a3, a4
+; RV32I-NEXT:  .LBB25_3:
+; RV32I-NEXT:    li t0, 64
+; RV32I-NEXT:    mv a3, a0
+; RV32I-NEXT:    beqz a2, .LBB25_5
+; RV32I-NEXT:  # %bb.4:
+; RV32I-NEXT:    mv a3, a7
+; RV32I-NEXT:  .LBB25_5:
+; RV32I-NEXT:    sub a7, t0, a2
+; RV32I-NEXT:    bltu a2, a5, .LBB25_8
+; RV32I-NEXT:  # %bb.6:
+; RV32I-NEXT:    li a2, 0
+; RV32I-NEXT:    bgeu a7, a5, .LBB25_9
+; RV32I-NEXT:  .LBB25_7:
+; RV32I-NEXT:    sll a5, a0, a6
+; RV32I-NEXT:    neg a6, a7
+; RV32I-NEXT:    srl a0, a0, a6
+; RV32I-NEXT:    or a0, a0, a4
+; RV32I-NEXT:    bnez a7, .LBB25_10
+; RV32I-NEXT:    j .LBB25_11
+; RV32I-NEXT:  .LBB25_8:
+; RV32I-NEXT:    srl a2, a1, a2
+; RV32I-NEXT:    bltu a7, a5, .LBB25_7
+; RV32I-NEXT:  .LBB25_9:
+; RV32I-NEXT:    li a5, 0
+; RV32I-NEXT:    sll a0, a0, a7
+; RV32I-NEXT:    beqz a7, .LBB25_11
+; RV32I-NEXT:  .LBB25_10:
+; RV32I-NEXT:    mv a1, a0
+; RV32I-NEXT:  .LBB25_11:
+; RV32I-NEXT:    or a0, a3, a5
+; RV32I-NEXT:    or a1, a2, a1
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: rotr_64_zext:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    li a2, 64
+; RV64I-NEXT:    subw a2, a2, a1
+; RV64I-NEXT:    srl a1, a0, a1
+; RV64I-NEXT:    sll a0, a0, a2
+; RV64I-NEXT:    or a0, a1, a0
+; RV64I-NEXT:    ret
+;
+; RV32ZBB-LABEL: rotr_64_zext:
+; RV32ZBB:       # %bb.0:
+; RV32ZBB-NEXT:    li a5, 32
+; RV32ZBB-NEXT:    neg a6, a2
+; RV32ZBB-NEXT:    sll a4, a1, a6
+; RV32ZBB-NEXT:    bltu a2, a5, .LBB25_2
+; RV32ZBB-NEXT:  # %bb.1:
+; RV32ZBB-NEXT:    srl a7, a1, a2
+; RV32ZBB-NEXT:    j .LBB25_3
+; RV32ZBB-NEXT:  .LBB25_2:
+; RV32ZBB-NEXT:    srl a3, a0, a2
+; RV32ZBB-NEXT:    or a7, a3, a4
+; RV32ZBB-NEXT:  .LBB25_3:
+; RV32ZBB-NEXT:    li t0, 64
+; RV32ZBB-NEXT:    mv a3, a0
+; RV32ZBB-NEXT:    beqz a2, .LBB25_5
+; RV32ZBB-NEXT:  # %bb.4:
+; RV32ZBB-NEXT:    mv a3, a7
+; RV32ZBB-NEXT:  .LBB25_5:
+; RV32ZBB-NEXT:    sub a7, t0, a2
+; RV32ZBB-NEXT:    bltu a2, a5, .LBB25_8
+; RV32ZBB-NEXT:  # %bb.6:
+; RV32ZBB-NEXT:    li a2, 0
+; RV32ZBB-NEXT:    bgeu a7, a5, .LBB25_9
+; RV32ZBB-NEXT:  .LBB25_7:
+; RV32ZBB-NEXT:    sll a5, a0, a6
+; RV32ZBB-NEXT:    neg a6, a7
+; RV32ZBB-NEXT:    srl a0, a0, a6
+; RV32ZBB-NEXT:    or a0, a0, a4
+; RV32ZBB-NEXT:    bnez a7, .LBB25_10
+; RV32ZBB-NEXT:    j .LBB25_11
+; RV32ZBB-NEXT:  .LBB25_8:
+; RV32ZBB-NEXT:    srl a2, a1, a2
+; RV32ZBB-NEXT:    bltu a7, a5, .LBB25_7
+; RV32ZBB-NEXT:  .LBB25_9:
+; RV32ZBB-NEXT:    li a5, 0
+; RV32ZBB-NEXT:    sll a0, a0, a7
+; RV32ZBB-NEXT:    beqz a7, .LBB25_11
+; RV32ZBB-NEXT:  .LBB25_10:
+; RV32ZBB-NEXT:    mv a1, a0
+; RV32ZBB-NEXT:  .LBB25_11:
+; RV32ZBB-NEXT:    or a0, a3, a5
+; RV32ZBB-NEXT:    or a1, a2, a1
+; RV32ZBB-NEXT:    ret
+;
+; RV64ZBB-LABEL: rotr_64_zext:
+; RV64ZBB:       # %bb.0:
+; RV64ZBB-NEXT:    li a2, 64
+; RV64ZBB-NEXT:    subw a2, a2, a1
+; RV64ZBB-NEXT:    srl a1, a0, a1
+; RV64ZBB-NEXT:    sll a0, a0, a2
+; RV64ZBB-NEXT:    or a0, a1, a0
+; RV64ZBB-NEXT:    ret
+;
+; RV32XTHEADBB-LABEL: rotr_64_zext:
+; RV32XTHEADBB:       # %bb.0:
+; RV32XTHEADBB-NEXT:    li a5, 32
+; RV32XTHEADBB-NEXT:    neg a6, a2
+; RV32XTHEADBB-NEXT:    sll a4, a1, a6
+; RV32XTHEADBB-NEXT:    bltu a2, a5, .LBB25_2
+; RV32XTHEADBB-NEXT:  # %bb.1:
+; RV32XTHEADBB-NEXT:    srl a7, a1, a2
+; RV32XTHEADBB-NEXT:    j .LBB25_3
+; RV32XTHEADBB-NEXT:  .LBB25_2:
+; RV32XTHEADBB-NEXT:    srl a3, a0, a2
+; RV32XTHEADBB-NEXT:    or a7, a3, a4
+; RV32XTHEADBB-NEXT:  .LBB25_3:
+; RV32XTHEADBB-NEXT:    li t0, 64
+; RV32XTHEADBB-NEXT:    mv a3, a0
+; RV32XTHEADBB-NEXT:    beqz a2, .LBB25_5
+; RV32XTHEADBB-NEXT:  # %bb.4:
+; RV32XTHEADBB-NEXT:    mv a3, a7
+; RV32XTHEADBB-NEXT:  .LBB25_5:
+; RV32XTHEADBB-NEXT:    sub a7, t0, a2
+; RV32XTHEADBB-NEXT:    bltu a2, a5, .LBB25_8
+; RV32XTHEADBB-NEXT:  # %bb.6:
+; RV32XTHEADBB-NEXT:    li a2, 0
+; RV32XTHEADBB-NEXT:    bgeu a7, a5, .LBB25_9
+; RV32XTHEADBB-NEXT:  .LBB25_7:
+; RV32XTHEADBB-NEXT:    sll a5, a0, a6
+; RV32XTHEADBB-NEXT:    neg a6, a7
+; RV32XTHEADBB-NEXT:    srl a0, a0, a6
+; RV32XTHEADBB-NEXT:    or a0, a0, a4
+; RV32XTHEADBB-NEXT:    bnez a7, .LBB25_10
+; RV32XTHEADBB-NEXT:    j .LBB25_11
+; RV32XTHEADBB-NEXT:  .LBB25_8:
+; RV32XTHEADBB-NEXT:    srl a2, a1, a2
+; RV32XTHEADBB-NEXT:    bltu a7, a5, .LBB25_7
+; RV32XTHEADBB-NEXT:  .LBB25_9:
+; RV32XTHEADBB-NEXT:    li a5, 0
+; RV32XTHEADBB-NEXT:    sll a0, a0, a7
+; RV32XTHEADBB-NEXT:    beqz a7, .LBB25_11
+; RV32XTHEADBB-NEXT:  .LBB25_10:
+; RV32XTHEADBB-NEXT:    mv a1, a0
+; RV32XTHEADBB-NEXT:  .LBB25_11:
+; RV32XTHEADBB-NEXT:    or a0, a3, a5
+; RV32XTHEADBB-NEXT:    or a1, a2, a1
+; RV32XTHEADBB-NEXT:    ret
+;
+; RV64XTHEADBB-LABEL: rotr_64_zext:
+; RV64XTHEADBB:       # %bb.0:
+; RV64XTHEADBB-NEXT:    li a2, 64
+; RV64XTHEADBB-NEXT:    subw a2, a2, a1
+; RV64XTHEADBB-NEXT:    srl a1, a0, a1
+; RV64XTHEADBB-NEXT:    sll a0, a0, a2
+; RV64XTHEADBB-NEXT:    or a0, a1, a0
+; RV64XTHEADBB-NEXT:    ret
+  %z = sub i32 64, %y
+  %zext = zext i32 %z to i64
+  %zexty = zext i32 %y to i64
+  %b = lshr i64 %x, %zexty
+  %c = shl i64 %x, %zext
+  %d = or i64 %b, %c
+  ret i64 %d
+}
diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/rv64zbb-zbkb.ll b/llvm/test/CodeGen/RISCV/GlobalISel/rv64zbb-zbkb.ll
index a29219bfde06b..79d08772e8853 100644
--- a/llvm/test/CodeGen/RISCV/GlobalISel/rv64zbb-zbkb.ll
+++ b/llvm/test/CodeGen/RISCV/GlobalISel/rv64zbb-zbkb.ll
@@ -107,7 +107,7 @@ declare i32 @llvm.fshl.i32(i32, i32, i32)
 define signext i32 @rol_i32(i32 signext %a, i32 signext %b) nounwind {
 ; RV64I-LABEL: rol_i32:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    neg a2, a1
+; RV64I-NEXT:    negw a2, a1
 ; RV64I-NEXT:    sllw a1, a0, a1
 ; RV64I-NEXT:    srlw a0, a0, a2
 ; RV64I-NEXT:    or a0, a1, a0
@@ -125,7 +125,7 @@ define signext i32 @rol_i32(i32 signext %a, i32 signext %b) nounwind {
 define void @rol_i32_nosext(i32 signext %a, i32 signext %b, ptr %x) nounwind {
 ; RV64I-LABEL: rol_i32_nosext:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    neg a3, a1
+; RV64I-NEXT:    negw a3, a1
 ; RV64I-NEXT:    sllw a1, a0, a1
 ; RV64I-NEXT:    srlw a0, a0, a3
 ; RV64I-NEXT:    or a0, a1, a0
@@ -146,7 +146,7 @@ define signext i32 @rol_i32_neg_constant_rhs(i32 signext %a) nounwind {
 ; RV64I-LABEL: rol_i32_neg_constant_rhs:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    li a1, -2
-; RV64I-NEXT:    neg a2, a0
+; RV64I-NEXT:    negw a2, a0
 ; RV64I-NEXT:    sllw a0, a1, a0
 ; RV64I-NEXT:    srlw a1, a1, a2
 ; RV64I-NEXT:    or a0, a0, a1
@@ -166,7 +166,7 @@ declare i64 @llvm.fshl.i64(i64, i64, i64)
 define i64 @rol_i64(i64 %a, i64 %b) nounwind {
 ; RV64I-LABEL: rol_i64:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    neg a2, a1
+; RV64I-NEXT:    negw a2, a1
 ; RV64I-NEXT:    sll a1, a0, a1
 ; RV64I-NEXT:    srl a0, a0, a2
 ; RV64I-NEXT:    or a0, a1, a0
@@ -185,7 +185,7 @@ declare i32 @llvm.fshr.i32(i32, i32, i32)
 define signext i32 @ror_i32(i32 signext %a, i32 signext %b) nounwind {
 ; RV64I-LABEL: ror_i32:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    neg a2, a1
+; RV64I-NEXT:    negw a2, a1
 ; RV64I-NEXT:    srlw a1, a0, a1
 ; RV64I-NEXT:    sllw a0, a0, a2
 ; RV64I-NEXT:    or a0, a1, a0
@@ -203,7 +203,7 @@ define signext i32 @ror_i32(i32 signext %a, i32 signext %b) nounwind {
 define void @ror_i32_nosext(i32 signext %a, i32 signext %b, ptr %x) nounwind {
 ; RV64I-LABEL: ror_i32_nosext:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    neg a3, a1
+; RV64I-NEXT:    negw a3, a1
 ; RV64I-NEXT:    srlw a1, a0, a1
 ; RV64I-NEXT:    sllw a0, a0, a3
 ; RV64I-NEXT:    or a0, a1, a0
@@ -224,7 +224,7 @@ define signext i32 @ror_i32_neg_constant_rhs(i32 signext %a) nounwind {
 ; RV64I-LABEL: ror_i32_neg_constant_rhs:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    li a1, -2
-; RV64I-NEXT:    neg a2, a0
+; RV64I-NEXT:    negw a2, a0
 ; RV64I-NEXT:    srlw a0, a1, a0
 ; RV64I-NEXT:    sllw a1, a1, a2
 ; RV64I-NEXT:    or a0, a0, a1
@@ -244,7 +244,7 @@ declare i64 @llvm.fshr.i64(i64, i64, i64)
 define i64 @ror_i64(i64 %a, i64 %b) nounwind {
 ; RV64I-LABEL: ror_i64:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    neg a2, a1
+; RV64I-NEXT:    negw a2, a1
 ; RV64I-NEXT:    srl a1, a0, a1
 ; RV64I-NEXT:    sll a0, a0, a2
 ; RV64I-NEXT:    or a0, a1, a0
diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/rv64zbb.ll b/llvm/test/CodeGen/RISCV/GlobalISel/rv64zbb.ll
index 9df319e73a11a..9a6c718703a27 100644
--- a/llvm/test/CodeGen/RISCV/GlobalISel/rv64zbb.ll
+++ b/llvm/test/CodeGen/RISCV/GlobalISel/rv64zbb.ll
@@ -31,13 +31,13 @@ define signext i32 @ctlz_i32(i32 signext %a) nounwind {
 ; RV64I-NEXT:    and a1, a2, a1
 ; RV64I-NEXT:    lui a2, 209715
 ; RV64I-NEXT:    addi a2, a2, 819
-; RV64I-NEXT:    sub a0, a0, a1
+; RV64I-NEXT:    subw a0, a0, a1
 ; RV64I-NEXT:    srliw a1, a0, 2
 ; RV64I-NEXT:    and a0, a0, a2
 ; RV64I-NEXT:    and a1, a1, a2
 ; RV64I-NEXT:    lui a2, 61681
-; RV64I-NEXT:    addw a0, a1, a0
-; RV64I-NEXT:    srli a1, a0, 4
+; RV64I-NEXT:    add a0, a1, a0
+; RV64I-NEXT:    sraiw a1, a0, 4
 ; RV64I-NEXT:    addw a0, a1, a0
 ; RV64I-NEXT:    lui a1, 4112
 ; RV64I-NEXT:    addiw a2, a2, -241
@@ -88,13 +88,13 @@ define signext i32 @log2_i32(i32 signext %a) nounwind {
 ; RV64I-NEXT:    and a1, a2, a1
 ; RV64I-NEXT:    lui a2, 209715
 ; RV64I-NEXT:    addi a2, a2, 819
-; RV64I-NEXT:    sub a0, a0, a1
+; RV64I-NEXT:    subw a0, a0, a1
 ; RV64I-NEXT:    srliw a1, a0, 2
 ; RV64I-NEXT:    and a0, a0, a2
 ; RV64I-NEXT:    and a1, a1, a2
 ; RV64I-NEXT:    lui a2, 61681
-; RV64I-NEXT:    addw a0, a1, a0
-; RV64I-NEXT:    srli a1, a0, 4
+; RV64I-NEXT:    add a0, a1, a0
+; RV64I-NEXT:    sraiw a1, a0, 4
 ; RV64I-NEXT:    addw a0, a1, a0
 ; RV64I-NEXT:    lui a1, 4112
 ; RV64I-NEXT:    addiw a2, a2, -241
@@ -103,7 +103,7 @@ define signext i32 @log2_i32(i32 signext %a) nounwind {
 ; RV64I-NEXT:    call __muldi3
 ; RV64I-NEXT:    srliw a0, a0, 24
 ; RV64I-NEXT:    li a1, 32
-; RV64I-NEXT:    sub a0, a1, a0
+; RV64I-NEXT:    subw a0, a1, a0
 ; RV64I-NEXT:    j .LBB1_3
 ; RV64I-NEXT:  .LBB1_2:
 ; RV64I-NEXT:    li a0, 32
@@ -153,13 +153,13 @@ define signext i32 @log2_ceil_i32(i32 signext %a) nounwind {
 ; RV64I-NEXT:    and a1, a2, a1
 ; RV64I-NEXT:    lui a2, 209715
 ; RV64I-NEXT:    addi a2, a2, 819
-; RV64I-NEXT:    sub a0, a0, a1
+; RV64I-NEXT:    subw a0, a0, a1
 ; RV64I-NEXT:    srliw a1, a0, 2
 ; RV64I-NEXT:    and a0, a0, a2
 ; RV64I-NEXT:    and a1, a1, a2
 ; RV64I-NEXT:    lui a2, 61681
-; RV64I-NEXT:    addw a0, a1, a0
-; RV64I-NEXT:    srli a1, a0, 4
+; RV64I-NEXT:    add a0, a1, a0
+; RV64I-NEXT:    sraiw a1, a0, 4
 ; RV64I-NEXT:    addw a0, a1, a0
 ; RV64I-NEXT:    lui a1, 4112
 ; RV64I-NEXT:    addiw a2, a2, -241
@@ -168,7 +168,7 @@ define signext i32 @log2_ceil_i32(i32 signext %a) nounwind {
 ; RV64I-NEXT:    call __muldi3
 ; RV64I-NEXT:    srliw a0, a0, 24
 ; RV64I-NEXT:    li a1, 32
-; RV64I-NEXT:    sub a1, a1, a0
+; RV64I-NEXT:    subw a1, a1, a0
 ; RV64I-NEXT:  .LBB2_2: # %cond.end
 ; RV64I-NEXT:    subw a0, s0, a1
 ; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
@@ -212,13 +212,13 @@ define signext i32 @findLastSet_i32(i32 signext %a) nounwind {
 ; RV64I-NEXT:    and a1, a2, a1
 ; RV64I-NEXT:    lui a2, 209715
 ; RV64I-NEXT:    addi a2, a2, 819
-; RV64I-NEXT:    sub a0, a0, a1
+; RV64I-NEXT:    subw a0, a0, a1
 ; RV64I-NEXT:    srliw a1, a0, 2
 ; RV64I-NEXT:    and a0, a0, a2
 ; RV64I-NEXT:    and a1, a1, a2
 ; RV64I-NEXT:    lui a2, 61681
-; RV64I-NEXT:    addw a0, a1, a0
-; RV64I-NEXT:    srli a1, a0, 4
+; RV64I-NEXT:    add a0, a1, a0
+; RV64I-NEXT:    sraiw a1, a0, 4
 ; RV64I-NEXT:    addw a0, a1, a0
 ; RV64I-NEXT:    lui a1, 4112
 ; RV64I-NEXT:    addiw a2, a2, -241
@@ -283,13 +283,13 @@ define i32 @ctlz_lshr_i32(i32 signext %a) {
 ; RV64I-NEXT:    and a1, a2, a1
 ; RV64I-NEXT:    lui a2, 209715
 ; RV64I-NEXT:    addi a2, a2, 819
-; RV64I-NEXT:    sub a0, a0, a1
+; RV64I-NEXT:    subw a0, a0, a1
 ; RV64I-NEXT:    srliw a1, a0, 2
 ; RV64I-NEXT:    and a0, a0, a2
 ; RV64I-NEXT:    and a1, a1, a2
 ; RV64I-NEXT:    lui a2, 61681
-; RV64I-NEXT:    addw a0, a1, a0
-; RV64I-NEXT:    srli a1, a0, 4
+; RV64I-NEXT:    add a0, a1, a0
+; RV64I-NEXT:    sraiw a1, a0, 4
 ; RV64I-NEXT:    addw a0, a1, a0
 ; RV64I-NEXT:    lui a1, 4112
 ; RV64I-NEXT:    addiw a2, a2, -241
@@ -412,13 +412,13 @@ define signext i32 @cttz_i32(i32 signext %a) nounwind {
 ; RV64I-NEXT:    and a1, a2, a1
 ; RV64I-NEXT:    lui a2, 209715
 ; RV64I-NEXT:    addi a2, a2, 819
-; RV64I-NEXT:    sub a0, a0, a1
+; RV64I-NEXT:    subw a0, a0, a1
 ; RV64I-NEXT:    srliw a1, a0, 2
 ; RV64I-NEXT:    and a0, a0, a2
 ; RV64I-NEXT:    and a1, a1, a2
 ; RV64I-NEXT:    lui a2, 61681
-; RV64I-NEXT:    addw a0, a1, a0
-; RV64I-NEXT:    srli a1, a0, 4
+; RV64I-NEXT:    add a0, a1, a0
+; RV64I-NEXT:    sraiw a1, a0, 4
 ; RV64I-NEXT:    addw a0, a1, a0
 ; RV64I-NEXT:    lui a1, 4112
 ; RV64I-NEXT:    addiw a2, a2, -241
@@ -455,13 +455,13 @@ define signext i32 @cttz_zero_undef_i32(i32 signext %a) nounwind {
 ; RV64I-NEXT:    and a1, a2, a1
 ; RV64I-NEXT:    lui a2, 209715
 ; RV64I-NEXT:    addi a2, a2, 819
-; RV64I-NEXT:    sub a0, a0, a1
+; RV64I-NEXT:    subw a0, a0, a1
 ; RV64I-NEXT:    srliw a1, a0, 2
 ; RV64I-NEXT:    and a0, a0, a2
 ; RV64I-NEXT:    and a1, a1, a2
 ; RV64I-NEXT:    lui a2, 61681
-; RV64I-NEXT:    addw a0, a1, a0
-; RV64I-NEXT:    srli a1, a0, 4
+; RV64I-NEXT:    add a0, a1, a0
+; RV64I-NEXT:    sraiw a1, a0, 4
 ; RV64I-NEXT:    addw a0, a1, a0
 ; RV64I-NEXT:    lui a1, 4112
 ; RV64I-NEXT:    addiw a2, a2, -241
@@ -497,13 +497,13 @@ define signext i32 @findFirstSet_i32(i32 signext %a) nounwind {
 ; RV64I-NEXT:    and a1, a2, a1
 ; RV64I-NEXT:    lui a2, 209715
 ; RV64I-NEXT:    addi a2, a2, 819
-; RV64I-NEXT:    sub a0, a0, a1
+; RV64I-NEXT:    subw a0, a0, a1
 ; RV64I-NEXT:    srliw a1, a0, 2
 ; RV64I-NEXT:    and a0, a0, a2
 ; RV64I-NEXT:    and a1, a1, a2
 ; RV64I-NEXT:    lui a2, 61681
-; RV64I-NEXT:    addw a0, a1, a0
-; RV64I-NEXT:    srli a1, a0, 4
+; RV64I-NEXT:    add a0, a1, a0
+; RV64I-NEXT:    sraiw a1, a0, 4
 ; RV64I-NEXT:    addw a0, a1, a0
 ; RV64I-NEXT:    lui a1, 4112
 ; RV64I-NEXT:    addiw a2, a2, -241
@@ -553,13 +553,13 @@ define signext i32 @ffs_i32(i32 signext %a) nounwind {
 ; RV64I-NEXT:    and a1, a2, a1
 ; RV64I-NEXT:    lui a2, 209715
 ; RV64I-NEXT:    addi a2, a2, 819
-; RV64I-NEXT:    sub a0, a0, a1
+; RV64I-NEXT:    subw a0, a0, a1
 ; RV64I-NEXT:    srliw a1, a0, 2
 ; RV64I-NEXT:    and a0, a0, a2
 ; RV64I-NEXT:    and a1, a1, a2
 ; RV64I-NEXT:    lui a2, 61681
-; RV64I-NEXT:    addw a0, a1, a0
-; RV64I-NEXT:    srli a1, a0, 4
+; RV64I-NEXT:    add a0, a1, a0
+; RV64I-NEXT:    sraiw a1, a0, 4
 ; RV64I-NEXT:    addw a0, a1, a0
 ; RV64I-NEXT:    lui a1, 4112
 ; RV64I-NEXT:    addiw a2, a2, -241
@@ -672,13 +672,13 @@ define signext i32 @ctpop_i32(i32 signext %a) nounwind {
 ; RV64I-NEXT:    and a1, a1, a2
 ; RV64I-NEXT:    lui a2, 209715
 ; RV64I-NEXT:    addi a2, a2, 819
-; RV64I-NEXT:    sub a0, a0, a1
+; RV64I-NEXT:    subw a0, a0, a1
 ; RV64I-NEXT:    srliw a1, a0, 2
 ; RV64I-NEXT:    and a0, a0, a2
 ; RV64I-NEXT:    and a1, a1, a2
 ; RV64I-NEXT:    lui a2, 61681
-; RV64I-NEXT:    addw a0, a1, a0
-; RV64I-NEXT:    srli a1, a0, 4
+; RV64I-NEXT:    add a0, a1, a0
+; RV64I-NEXT:    sraiw a1, a0, 4
 ; RV64I-NEXT:    addw a0, a1, a0
 ; RV64I-NEXT:    lui a1, 4112
 ; RV64I-NEXT:    addiw a2, a2, -241
@@ -709,13 +709,13 @@ define i1 @ctpop_i32_ult_two(i32 signext %a) nounwind {
 ; RV64I-NEXT:    and a1, a1, a2
 ; RV64I-NEXT:    lui a2, 209715
 ; RV64I-NEXT:    addi a2, a2, 819
-; RV64I-NEXT:    sub a0, a0, a1
+; RV64I-NEXT:    subw a0, a0, a1
 ; RV64I-NEXT:    srliw a1, a0, 2
 ; RV64I-NEXT:    and a0, a0, a2
 ; RV64I-NEXT:    and a1, a1, a2
 ; RV64I-NEXT:    lui a2, 61681
-; RV64I-NEXT:    addw a0, a1, a0
-; RV64I-NEXT:    srli a1, a0, 4
+; RV64I-NEXT:    add a0, a1, a0
+; RV64I-NEXT:    sraiw a1, a0, 4
 ; RV64I-NEXT:    addw a0, a1, a0
 ; RV64I-NEXT:    lui a1, 4112
 ; RV64I-NEXT:    addiw a2, a2, -241
@@ -750,13 +750,13 @@ define signext i32 @ctpop_i32_load(ptr %p) nounwind {
 ; RV64I-NEXT:    and a1, a2, a1
 ; RV64I-NEXT:    lui a2, 209715
 ; RV64I-NEXT:    addi a2, a2, 819
-; RV64I-NEXT:    sub a0, a0, a1
+; RV64I-NEXT:    subw a0, a0, a1
 ; RV64I-NEXT:    srliw a1, a0, 2
 ; RV64I-NEXT:    and a0, a0, a2
 ; RV64I-NEXT:    and a1, a1, a2
 ; RV64I-NEXT:    lui a2, 61681
-; RV64I-NEXT:    addw a0, a1, a0
-; RV64I-NEXT:    srli a1, a0, 4
+; RV64I-NEXT:    add a0, a1, a0
+; RV64I-NEXT:    sraiw a1, a0, 4
 ; RV64I-NEXT:    addw a0, a1, a0
 ; RV64I-NEXT:    lui a1, 4112
 ; RV64I-NEXT:    addiw a2, a2, -241
diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/rv64zbkb.ll b/llvm/test/CodeGen/RISCV/GlobalISel/rv64zbkb.ll
index bf430c618afca..558424b53be95 100644
--- a/llvm/test/CodeGen/RISCV/GlobalISel/rv64zbkb.ll
+++ b/llvm/test/CodeGen/RISCV/GlobalISel/rv64zbkb.ll
@@ -16,9 +16,7 @@ define signext i32 @pack_i32(i32 signext %a, i32 signext %b) nounwind {
 ;
 ; RV64ZBKB-LABEL: pack_i32:
 ; RV64ZBKB:       # %bb.0:
-; RV64ZBKB-NEXT:    zext.h a0, a0
-; RV64ZBKB-NEXT:    slliw a1, a1, 16
-; RV64ZBKB-NEXT:    or a0, a1, a0
+; RV64ZBKB-NEXT:    packw a0, a0, a1
 ; RV64ZBKB-NEXT:    ret
   %shl = and i32 %a, 65535
   %shl1 = shl i32 %b, 16
@@ -37,9 +35,7 @@ define signext i32 @pack_i32_2(i16 zeroext %a, i16 zeroext %b) nounwind {
 ;
 ; RV64ZBKB-LABEL: pack_i32_2:
 ; RV64ZBKB:       # %bb.0:
-; RV64ZBKB-NEXT:    slli a1, a1, 16
-; RV64ZBKB-NEXT:    or a0, a1, a0
-; RV64ZBKB-NEXT:    sext.w a0, a0
+; RV64ZBKB-NEXT:    packw a0, a0, a1
 ; RV64ZBKB-NEXT:    ret
   %zexta = zext i16 %a to i32
   %zextb = zext i16 %b to i32
@@ -60,8 +56,7 @@ define signext i32 @pack_i32_3(i16 zeroext %0, i16 zeroext %1, i32 signext %2) {
 ;
 ; RV64ZBKB-LABEL: pack_i32_3:
 ; RV64ZBKB:       # %bb.0:
-; RV64ZBKB-NEXT:    slli a0, a0, 16
-; RV64ZBKB-NEXT:    or a0, a0, a1
+; RV64ZBKB-NEXT:    packw a0, a1, a0
 ; RV64ZBKB-NEXT:    addw a0, a0, a2
 ; RV64ZBKB-NEXT:    ret
   %4 = zext i16 %0 to i32
@@ -343,9 +338,7 @@ define signext i32 @pack_i32_allWUsers(i16 zeroext %0, i16 zeroext %1, i16 zeroe
 ; RV64ZBKB:       # %bb.0:
 ; RV64ZBKB-NEXT:    add a0, a1, a0
 ; RV64ZBKB-NEXT:    zext.h a0, a0
-; RV64ZBKB-NEXT:    slli a0, a0, 16
-; RV64ZBKB-NEXT:    or a0, a0, a2
-; RV64ZBKB-NEXT:    sext.w a0, a0
+; RV64ZBKB-NEXT:    packw a0, a2, a0
 ; RV64ZBKB-NEXT:    ret
   %4 = add i16 %1, %0
   %5 = zext i16 %4 to i32
diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/shifts.ll b/llvm/test/CodeGen/RISCV/GlobalISel/shifts.ll
new file mode 100644
index 0000000000000..8b262db56ccd2
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/GlobalISel/shifts.ll
@@ -0,0 +1,962 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=riscv32 -verify-machineinstrs -global-isel < %s \
+; RUN:   | FileCheck %s -check-prefix=RV32I
+; RUN: llc -mtriple=riscv64 -verify-machineinstrs -global-isel < %s \
+; RUN:   | FileCheck %s -check-prefix=RV64I
+
+; Basic shift support is tested as part of ALU.ll. This file ensures that
+; shifts which may not be supported natively are lowered properly.
+
+declare i64 @llvm.fshr.i64(i64, i64, i64)
+declare i128 @llvm.fshr.i128(i128, i128, i128)
+
+define i64 @lshr64(i64 %a, i64 %b) nounwind {
+; RV32I-LABEL: lshr64:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    li a3, 32
+; RV32I-NEXT:    bltu a2, a3, .LBB0_2
+; RV32I-NEXT:  # %bb.1:
+; RV32I-NEXT:    srl a4, a1, a2
+; RV32I-NEXT:    bnez a2, .LBB0_3
+; RV32I-NEXT:    j .LBB0_4
+; RV32I-NEXT:  .LBB0_2:
+; RV32I-NEXT:    srl a4, a0, a2
+; RV32I-NEXT:    neg a5, a2
+; RV32I-NEXT:    sll a5, a1, a5
+; RV32I-NEXT:    or a4, a4, a5
+; RV32I-NEXT:    beqz a2, .LBB0_4
+; RV32I-NEXT:  .LBB0_3:
+; RV32I-NEXT:    mv a0, a4
+; RV32I-NEXT:  .LBB0_4:
+; RV32I-NEXT:    bltu a2, a3, .LBB0_6
+; RV32I-NEXT:  # %bb.5:
+; RV32I-NEXT:    li a1, 0
+; RV32I-NEXT:    ret
+; RV32I-NEXT:  .LBB0_6:
+; RV32I-NEXT:    srl a1, a1, a2
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: lshr64:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    srl a0, a0, a1
+; RV64I-NEXT:    ret
+  %1 = lshr i64 %a, %b
+  ret i64 %1
+}
+
+define i64 @lshr64_minsize(i64 %a, i64 %b) minsize nounwind {
+; RV32I-LABEL: lshr64_minsize:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    li a3, 32
+; RV32I-NEXT:    bltu a2, a3, .LBB1_2
+; RV32I-NEXT:  # %bb.1:
+; RV32I-NEXT:    srl a4, a1, a2
+; RV32I-NEXT:    bnez a2, .LBB1_3
+; RV32I-NEXT:    j .LBB1_4
+; RV32I-NEXT:  .LBB1_2:
+; RV32I-NEXT:    srl a4, a0, a2
+; RV32I-NEXT:    neg a5, a2
+; RV32I-NEXT:    sll a5, a1, a5
+; RV32I-NEXT:    or a4, a4, a5
+; RV32I-NEXT:    beqz a2, .LBB1_4
+; RV32I-NEXT:  .LBB1_3:
+; RV32I-NEXT:    mv a0, a4
+; RV32I-NEXT:  .LBB1_4:
+; RV32I-NEXT:    bltu a2, a3, .LBB1_6
+; RV32I-NEXT:  # %bb.5:
+; RV32I-NEXT:    li a1, 0
+; RV32I-NEXT:    ret
+; RV32I-NEXT:  .LBB1_6:
+; RV32I-NEXT:    srl a1, a1, a2
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: lshr64_minsize:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    srl a0, a0, a1
+; RV64I-NEXT:    ret
+  %1 = lshr i64 %a, %b
+  ret i64 %1
+}
+
+define i64 @ashr64(i64 %a, i64 %b) nounwind {
+; RV32I-LABEL: ashr64:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    li a3, 32
+; RV32I-NEXT:    bltu a2, a3, .LBB2_2
+; RV32I-NEXT:  # %bb.1:
+; RV32I-NEXT:    sra a4, a1, a2
+; RV32I-NEXT:    bnez a2, .LBB2_3
+; RV32I-NEXT:    j .LBB2_4
+; RV32I-NEXT:  .LBB2_2:
+; RV32I-NEXT:    srl a4, a0, a2
+; RV32I-NEXT:    neg a5, a2
+; RV32I-NEXT:    sll a5, a1, a5
+; RV32I-NEXT:    or a4, a4, a5
+; RV32I-NEXT:    beqz a2, .LBB2_4
+; RV32I-NEXT:  .LBB2_3:
+; RV32I-NEXT:    mv a0, a4
+; RV32I-NEXT:  .LBB2_4:
+; RV32I-NEXT:    bltu a2, a3, .LBB2_6
+; RV32I-NEXT:  # %bb.5:
+; RV32I-NEXT:    srai a1, a1, 31
+; RV32I-NEXT:    ret
+; RV32I-NEXT:  .LBB2_6:
+; RV32I-NEXT:    sra a1, a1, a2
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: ashr64:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    sra a0, a0, a1
+; RV64I-NEXT:    ret
+  %1 = ashr i64 %a, %b
+  ret i64 %1
+}
+
+define i64 @ashr64_minsize(i64 %a, i64 %b) minsize nounwind {
+; RV32I-LABEL: ashr64_minsize:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    li a3, 32
+; RV32I-NEXT:    bltu a2, a3, .LBB3_2
+; RV32I-NEXT:  # %bb.1:
+; RV32I-NEXT:    sra a4, a1, a2
+; RV32I-NEXT:    bnez a2, .LBB3_3
+; RV32I-NEXT:    j .LBB3_4
+; RV32I-NEXT:  .LBB3_2:
+; RV32I-NEXT:    srl a4, a0, a2
+; RV32I-NEXT:    neg a5, a2
+; RV32I-NEXT:    sll a5, a1, a5
+; RV32I-NEXT:    or a4, a4, a5
+; RV32I-NEXT:    beqz a2, .LBB3_4
+; RV32I-NEXT:  .LBB3_3:
+; RV32I-NEXT:    mv a0, a4
+; RV32I-NEXT:  .LBB3_4:
+; RV32I-NEXT:    bltu a2, a3, .LBB3_6
+; RV32I-NEXT:  # %bb.5:
+; RV32I-NEXT:    srai a1, a1, 31
+; RV32I-NEXT:    ret
+; RV32I-NEXT:  .LBB3_6:
+; RV32I-NEXT:    sra a1, a1, a2
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: ashr64_minsize:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    sra a0, a0, a1
+; RV64I-NEXT:    ret
+  %1 = ashr i64 %a, %b
+  ret i64 %1
+}
+
+define i64 @shl64(i64 %a, i64 %b) nounwind {
+; RV32I-LABEL: shl64:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    mv a3, a0
+; RV32I-NEXT:    li a0, 32
+; RV32I-NEXT:    bltu a2, a0, .LBB4_2
+; RV32I-NEXT:  # %bb.1:
+; RV32I-NEXT:    li a0, 0
+; RV32I-NEXT:    sll a3, a3, a2
+; RV32I-NEXT:    bnez a2, .LBB4_3
+; RV32I-NEXT:    j .LBB4_4
+; RV32I-NEXT:  .LBB4_2:
+; RV32I-NEXT:    sll a0, a3, a2
+; RV32I-NEXT:    neg a4, a2
+; RV32I-NEXT:    srl a3, a3, a4
+; RV32I-NEXT:    sll a4, a1, a2
+; RV32I-NEXT:    or a3, a3, a4
+; RV32I-NEXT:    beqz a2, .LBB4_4
+; RV32I-NEXT:  .LBB4_3:
+; RV32I-NEXT:    mv a1, a3
+; RV32I-NEXT:  .LBB4_4:
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: shl64:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    sll a0, a0, a1
+; RV64I-NEXT:    ret
+  %1 = shl i64 %a, %b
+  ret i64 %1
+}
+
+define i64 @shl64_minsize(i64 %a, i64 %b) minsize nounwind {
+; RV32I-LABEL: shl64_minsize:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    mv a3, a0
+; RV32I-NEXT:    li a0, 32
+; RV32I-NEXT:    bltu a2, a0, .LBB5_2
+; RV32I-NEXT:  # %bb.1:
+; RV32I-NEXT:    li a0, 0
+; RV32I-NEXT:    sll a3, a3, a2
+; RV32I-NEXT:    bnez a2, .LBB5_3
+; RV32I-NEXT:    j .LBB5_4
+; RV32I-NEXT:  .LBB5_2:
+; RV32I-NEXT:    sll a0, a3, a2
+; RV32I-NEXT:    neg a4, a2
+; RV32I-NEXT:    srl a3, a3, a4
+; RV32I-NEXT:    sll a4, a1, a2
+; RV32I-NEXT:    or a3, a3, a4
+; RV32I-NEXT:    beqz a2, .LBB5_4
+; RV32I-NEXT:  .LBB5_3:
+; RV32I-NEXT:    mv a1, a3
+; RV32I-NEXT:  .LBB5_4:
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: shl64_minsize:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    sll a0, a0, a1
+; RV64I-NEXT:    ret
+  %1 = shl i64 %a, %b
+  ret i64 %1
+}
+
+define i128 @lshr128(i128 %a, i128 %b) nounwind {
+; RV32I-LABEL: lshr128:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    lw a2, 0(a2)
+; RV32I-NEXT:    lw a3, 8(a1)
+; RV32I-NEXT:    lw a7, 12(a1)
+; RV32I-NEXT:    li t0, 32
+; RV32I-NEXT:    srl t2, a3, a2
+; RV32I-NEXT:    neg t6, a2
+; RV32I-NEXT:    sll t5, a7, t6
+; RV32I-NEXT:    bltu a2, t0, .LBB6_2
+; RV32I-NEXT:  # %bb.1:
+; RV32I-NEXT:    srl a5, a7, a2
+; RV32I-NEXT:    mv a4, a3
+; RV32I-NEXT:    bnez a2, .LBB6_3
+; RV32I-NEXT:    j .LBB6_4
+; RV32I-NEXT:  .LBB6_2:
+; RV32I-NEXT:    or a5, t2, t5
+; RV32I-NEXT:    mv a4, a3
+; RV32I-NEXT:    beqz a2, .LBB6_4
+; RV32I-NEXT:  .LBB6_3:
+; RV32I-NEXT:    mv a4, a5
+; RV32I-NEXT:  .LBB6_4:
+; RV32I-NEXT:    lw a5, 0(a1)
+; RV32I-NEXT:    lw a1, 4(a1)
+; RV32I-NEXT:    bltu a2, t0, .LBB6_6
+; RV32I-NEXT:  # %bb.5:
+; RV32I-NEXT:    li a6, 0
+; RV32I-NEXT:    srl t4, a1, a2
+; RV32I-NEXT:    j .LBB6_7
+; RV32I-NEXT:  .LBB6_6:
+; RV32I-NEXT:    srl a6, a7, a2
+; RV32I-NEXT:    srl t1, a5, a2
+; RV32I-NEXT:    sll t3, a1, t6
+; RV32I-NEXT:    or t4, t1, t3
+; RV32I-NEXT:  .LBB6_7:
+; RV32I-NEXT:    li t1, 64
+; RV32I-NEXT:    mv t3, a5
+; RV32I-NEXT:    beqz a2, .LBB6_9
+; RV32I-NEXT:  # %bb.8:
+; RV32I-NEXT:    mv t3, t4
+; RV32I-NEXT:  .LBB6_9:
+; RV32I-NEXT:    addi sp, sp, -16
+; RV32I-NEXT:    sw s0, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s1, 8(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s2, 4(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sub s0, t1, a2
+; RV32I-NEXT:    bltu a2, t0, .LBB6_12
+; RV32I-NEXT:  # %bb.10:
+; RV32I-NEXT:    li t4, 0
+; RV32I-NEXT:    bgeu s0, t0, .LBB6_13
+; RV32I-NEXT:  .LBB6_11:
+; RV32I-NEXT:    sll t6, a3, t6
+; RV32I-NEXT:    neg s1, s0
+; RV32I-NEXT:    srl s1, a3, s1
+; RV32I-NEXT:    or s2, s1, t5
+; RV32I-NEXT:    j .LBB6_14
+; RV32I-NEXT:  .LBB6_12:
+; RV32I-NEXT:    srl t4, a1, a2
+; RV32I-NEXT:    bltu s0, t0, .LBB6_11
+; RV32I-NEXT:  .LBB6_13:
+; RV32I-NEXT:    li t6, 0
+; RV32I-NEXT:    sll s2, a3, s0
+; RV32I-NEXT:  .LBB6_14:
+; RV32I-NEXT:    addi s1, a2, -64
+; RV32I-NEXT:    mv t5, a7
+; RV32I-NEXT:    beqz s0, .LBB6_16
+; RV32I-NEXT:  # %bb.15:
+; RV32I-NEXT:    mv t5, s2
+; RV32I-NEXT:  .LBB6_16:
+; RV32I-NEXT:    bltu s1, t0, .LBB6_18
+; RV32I-NEXT:  # %bb.17:
+; RV32I-NEXT:    srl t2, a7, s1
+; RV32I-NEXT:    bnez s1, .LBB6_19
+; RV32I-NEXT:    j .LBB6_20
+; RV32I-NEXT:  .LBB6_18:
+; RV32I-NEXT:    neg s0, s1
+; RV32I-NEXT:    sll s0, a7, s0
+; RV32I-NEXT:    or t2, t2, s0
+; RV32I-NEXT:    beqz s1, .LBB6_20
+; RV32I-NEXT:  .LBB6_19:
+; RV32I-NEXT:    mv a3, t2
+; RV32I-NEXT:  .LBB6_20:
+; RV32I-NEXT:    bltu s1, t0, .LBB6_22
+; RV32I-NEXT:  # %bb.21:
+; RV32I-NEXT:    li a7, 0
+; RV32I-NEXT:    bltu a2, t1, .LBB6_23
+; RV32I-NEXT:    j .LBB6_24
+; RV32I-NEXT:  .LBB6_22:
+; RV32I-NEXT:    srl a7, a7, a2
+; RV32I-NEXT:    bgeu a2, t1, .LBB6_24
+; RV32I-NEXT:  .LBB6_23:
+; RV32I-NEXT:    or a3, t3, t6
+; RV32I-NEXT:    or a7, t4, t5
+; RV32I-NEXT:  .LBB6_24:
+; RV32I-NEXT:    bnez a2, .LBB6_28
+; RV32I-NEXT:  # %bb.25:
+; RV32I-NEXT:    bltu a2, t1, .LBB6_27
+; RV32I-NEXT:  .LBB6_26:
+; RV32I-NEXT:    li a4, 0
+; RV32I-NEXT:    li a6, 0
+; RV32I-NEXT:  .LBB6_27:
+; RV32I-NEXT:    sw a5, 0(a0)
+; RV32I-NEXT:    sw a1, 4(a0)
+; RV32I-NEXT:    sw a4, 8(a0)
+; RV32I-NEXT:    sw a6, 12(a0)
+; RV32I-NEXT:    lw s0, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s1, 8(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s2, 4(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    addi sp, sp, 16
+; RV32I-NEXT:    ret
+; RV32I-NEXT:  .LBB6_28:
+; RV32I-NEXT:    mv a5, a3
+; RV32I-NEXT:    mv a1, a7
+; RV32I-NEXT:    bgeu a2, t1, .LBB6_26
+; RV32I-NEXT:    j .LBB6_27
+;
+; RV64I-LABEL: lshr128:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    li a3, 64
+; RV64I-NEXT:    bltu a2, a3, .LBB6_2
+; RV64I-NEXT:  # %bb.1:
+; RV64I-NEXT:    subw a4, a2, a3
+; RV64I-NEXT:    srl a4, a1, a4
+; RV64I-NEXT:    bnez a2, .LBB6_3
+; RV64I-NEXT:    j .LBB6_4
+; RV64I-NEXT:  .LBB6_2:
+; RV64I-NEXT:    srl a4, a0, a2
+; RV64I-NEXT:    negw a5, a2
+; RV64I-NEXT:    sll a5, a1, a5
+; RV64I-NEXT:    or a4, a4, a5
+; RV64I-NEXT:    beqz a2, .LBB6_4
+; RV64I-NEXT:  .LBB6_3:
+; RV64I-NEXT:    mv a0, a4
+; RV64I-NEXT:  .LBB6_4:
+; RV64I-NEXT:    bltu a2, a3, .LBB6_6
+; RV64I-NEXT:  # %bb.5:
+; RV64I-NEXT:    li a1, 0
+; RV64I-NEXT:    ret
+; RV64I-NEXT:  .LBB6_6:
+; RV64I-NEXT:    srl a1, a1, a2
+; RV64I-NEXT:    ret
+  %1 = lshr i128 %a, %b
+  ret i128 %1
+}
+
+define i128 @ashr128(i128 %a, i128 %b) nounwind {
+; RV32I-LABEL: ashr128:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    lw a2, 0(a2)
+; RV32I-NEXT:    lw a4, 8(a1)
+; RV32I-NEXT:    lw a3, 12(a1)
+; RV32I-NEXT:    li t0, 32
+; RV32I-NEXT:    srl t2, a4, a2
+; RV32I-NEXT:    neg t6, a2
+; RV32I-NEXT:    sll t5, a3, t6
+; RV32I-NEXT:    bltu a2, t0, .LBB7_2
+; RV32I-NEXT:  # %bb.1:
+; RV32I-NEXT:    sra a6, a3, a2
+; RV32I-NEXT:    mv a5, a4
+; RV32I-NEXT:    bnez a2, .LBB7_3
+; RV32I-NEXT:    j .LBB7_4
+; RV32I-NEXT:  .LBB7_2:
+; RV32I-NEXT:    or a6, t2, t5
+; RV32I-NEXT:    mv a5, a4
+; RV32I-NEXT:    beqz a2, .LBB7_4
+; RV32I-NEXT:  .LBB7_3:
+; RV32I-NEXT:    mv a5, a6
+; RV32I-NEXT:  .LBB7_4:
+; RV32I-NEXT:    lw a6, 0(a1)
+; RV32I-NEXT:    lw a1, 4(a1)
+; RV32I-NEXT:    bltu a2, t0, .LBB7_6
+; RV32I-NEXT:  # %bb.5:
+; RV32I-NEXT:    srai a7, a3, 31
+; RV32I-NEXT:    srl t4, a1, a2
+; RV32I-NEXT:    j .LBB7_7
+; RV32I-NEXT:  .LBB7_6:
+; RV32I-NEXT:    sra a7, a3, a2
+; RV32I-NEXT:    srl t1, a6, a2
+; RV32I-NEXT:    sll t3, a1, t6
+; RV32I-NEXT:    or t4, t1, t3
+; RV32I-NEXT:  .LBB7_7:
+; RV32I-NEXT:    li t1, 64
+; RV32I-NEXT:    mv t3, a6
+; RV32I-NEXT:    beqz a2, .LBB7_9
+; RV32I-NEXT:  # %bb.8:
+; RV32I-NEXT:    mv t3, t4
+; RV32I-NEXT:  .LBB7_9:
+; RV32I-NEXT:    addi sp, sp, -16
+; RV32I-NEXT:    sw s0, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s1, 8(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s2, 4(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sub s0, t1, a2
+; RV32I-NEXT:    bltu a2, t0, .LBB7_12
+; RV32I-NEXT:  # %bb.10:
+; RV32I-NEXT:    li t4, 0
+; RV32I-NEXT:    bgeu s0, t0, .LBB7_13
+; RV32I-NEXT:  .LBB7_11:
+; RV32I-NEXT:    sll t6, a4, t6
+; RV32I-NEXT:    neg s1, s0
+; RV32I-NEXT:    srl s1, a4, s1
+; RV32I-NEXT:    or s2, s1, t5
+; RV32I-NEXT:    j .LBB7_14
+; RV32I-NEXT:  .LBB7_12:
+; RV32I-NEXT:    srl t4, a1, a2
+; RV32I-NEXT:    bltu s0, t0, .LBB7_11
+; RV32I-NEXT:  .LBB7_13:
+; RV32I-NEXT:    li t6, 0
+; RV32I-NEXT:    sll s2, a4, s0
+; RV32I-NEXT:  .LBB7_14:
+; RV32I-NEXT:    addi s1, a2, -64
+; RV32I-NEXT:    mv t5, a3
+; RV32I-NEXT:    beqz s0, .LBB7_16
+; RV32I-NEXT:  # %bb.15:
+; RV32I-NEXT:    mv t5, s2
+; RV32I-NEXT:  .LBB7_16:
+; RV32I-NEXT:    bltu s1, t0, .LBB7_18
+; RV32I-NEXT:  # %bb.17:
+; RV32I-NEXT:    sra t2, a3, s1
+; RV32I-NEXT:    bnez s1, .LBB7_19
+; RV32I-NEXT:    j .LBB7_20
+; RV32I-NEXT:  .LBB7_18:
+; RV32I-NEXT:    neg s0, s1
+; RV32I-NEXT:    sll s0, a3, s0
+; RV32I-NEXT:    or t2, t2, s0
+; RV32I-NEXT:    beqz s1, .LBB7_20
+; RV32I-NEXT:  .LBB7_19:
+; RV32I-NEXT:    mv a4, t2
+; RV32I-NEXT:  .LBB7_20:
+; RV32I-NEXT:    bltu s1, t0, .LBB7_22
+; RV32I-NEXT:  # %bb.21:
+; RV32I-NEXT:    srai t0, a3, 31
+; RV32I-NEXT:    bltu a2, t1, .LBB7_23
+; RV32I-NEXT:    j .LBB7_24
+; RV32I-NEXT:  .LBB7_22:
+; RV32I-NEXT:    sra t0, a3, a2
+; RV32I-NEXT:    bgeu a2, t1, .LBB7_24
+; RV32I-NEXT:  .LBB7_23:
+; RV32I-NEXT:    or a4, t3, t6
+; RV32I-NEXT:    or t0, t4, t5
+; RV32I-NEXT:  .LBB7_24:
+; RV32I-NEXT:    bnez a2, .LBB7_28
+; RV32I-NEXT:  # %bb.25:
+; RV32I-NEXT:    bltu a2, t1, .LBB7_27
+; RV32I-NEXT:  .LBB7_26:
+; RV32I-NEXT:    srai a5, a3, 31
+; RV32I-NEXT:    mv a7, a5
+; RV32I-NEXT:  .LBB7_27:
+; RV32I-NEXT:    sw a6, 0(a0)
+; RV32I-NEXT:    sw a1, 4(a0)
+; RV32I-NEXT:    sw a5, 8(a0)
+; RV32I-NEXT:    sw a7, 12(a0)
+; RV32I-NEXT:    lw s0, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s1, 8(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s2, 4(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    addi sp, sp, 16
+; RV32I-NEXT:    ret
+; RV32I-NEXT:  .LBB7_28:
+; RV32I-NEXT:    mv a6, a4
+; RV32I-NEXT:    mv a1, t0
+; RV32I-NEXT:    bgeu a2, t1, .LBB7_26
+; RV32I-NEXT:    j .LBB7_27
+;
+; RV64I-LABEL: ashr128:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    li a3, 64
+; RV64I-NEXT:    bltu a2, a3, .LBB7_2
+; RV64I-NEXT:  # %bb.1:
+; RV64I-NEXT:    subw a4, a2, a3
+; RV64I-NEXT:    sra a4, a1, a4
+; RV64I-NEXT:    bnez a2, .LBB7_3
+; RV64I-NEXT:    j .LBB7_4
+; RV64I-NEXT:  .LBB7_2:
+; RV64I-NEXT:    srl a4, a0, a2
+; RV64I-NEXT:    negw a5, a2
+; RV64I-NEXT:    sll a5, a1, a5
+; RV64I-NEXT:    or a4, a4, a5
+; RV64I-NEXT:    beqz a2, .LBB7_4
+; RV64I-NEXT:  .LBB7_3:
+; RV64I-NEXT:    mv a0, a4
+; RV64I-NEXT:  .LBB7_4:
+; RV64I-NEXT:    bltu a2, a3, .LBB7_6
+; RV64I-NEXT:  # %bb.5:
+; RV64I-NEXT:    srai a1, a1, 63
+; RV64I-NEXT:    ret
+; RV64I-NEXT:  .LBB7_6:
+; RV64I-NEXT:    sra a1, a1, a2
+; RV64I-NEXT:    ret
+  %1 = ashr i128 %a, %b
+  ret i128 %1
+}
+
+define i128 @shl128(i128 %a, i128 %b) nounwind {
+; RV32I-LABEL: shl128:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    lw a2, 0(a2)
+; RV32I-NEXT:    lw a7, 0(a1)
+; RV32I-NEXT:    lw a3, 4(a1)
+; RV32I-NEXT:    li a6, 64
+; RV32I-NEXT:    li t1, 32
+; RV32I-NEXT:    neg t5, a2
+; RV32I-NEXT:    srl t2, a7, t5
+; RV32I-NEXT:    sll t0, a3, a2
+; RV32I-NEXT:    bltu a2, t1, .LBB8_2
+; RV32I-NEXT:  # %bb.1:
+; RV32I-NEXT:    li a4, 0
+; RV32I-NEXT:    sll t3, a7, a2
+; RV32I-NEXT:    j .LBB8_3
+; RV32I-NEXT:  .LBB8_2:
+; RV32I-NEXT:    sll a4, a7, a2
+; RV32I-NEXT:    or t3, t2, t0
+; RV32I-NEXT:  .LBB8_3:
+; RV32I-NEXT:    sub t4, a6, a2
+; RV32I-NEXT:    mv a5, a3
+; RV32I-NEXT:    beqz a2, .LBB8_5
+; RV32I-NEXT:  # %bb.4:
+; RV32I-NEXT:    mv a5, t3
+; RV32I-NEXT:  .LBB8_5:
+; RV32I-NEXT:    bltu t4, t1, .LBB8_7
+; RV32I-NEXT:  # %bb.6:
+; RV32I-NEXT:    srl t2, a3, t4
+; RV32I-NEXT:    mv t3, a7
+; RV32I-NEXT:    bnez t4, .LBB8_8
+; RV32I-NEXT:    j .LBB8_9
+; RV32I-NEXT:  .LBB8_7:
+; RV32I-NEXT:    neg t3, t4
+; RV32I-NEXT:    sll t3, a3, t3
+; RV32I-NEXT:    or t2, t2, t3
+; RV32I-NEXT:    mv t3, a7
+; RV32I-NEXT:    beqz t4, .LBB8_9
+; RV32I-NEXT:  .LBB8_8:
+; RV32I-NEXT:    mv t3, t2
+; RV32I-NEXT:  .LBB8_9:
+; RV32I-NEXT:    bltu t4, t1, .LBB8_11
+; RV32I-NEXT:  # %bb.10:
+; RV32I-NEXT:    li t4, 0
+; RV32I-NEXT:    j .LBB8_12
+; RV32I-NEXT:  .LBB8_11:
+; RV32I-NEXT:    srl t4, a3, t5
+; RV32I-NEXT:  .LBB8_12:
+; RV32I-NEXT:    addi sp, sp, -16
+; RV32I-NEXT:    sw s0, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s1, 8(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    lw t2, 8(a1)
+; RV32I-NEXT:    lw a1, 12(a1)
+; RV32I-NEXT:    bltu a2, t1, .LBB8_14
+; RV32I-NEXT:  # %bb.13:
+; RV32I-NEXT:    li t6, 0
+; RV32I-NEXT:    sll s1, t2, a2
+; RV32I-NEXT:    j .LBB8_15
+; RV32I-NEXT:  .LBB8_14:
+; RV32I-NEXT:    sll t6, t2, a2
+; RV32I-NEXT:    srl t5, t2, t5
+; RV32I-NEXT:    sll s0, a1, a2
+; RV32I-NEXT:    or s1, t5, s0
+; RV32I-NEXT:  .LBB8_15:
+; RV32I-NEXT:    addi s0, a2, -64
+; RV32I-NEXT:    mv t5, a1
+; RV32I-NEXT:    beqz a2, .LBB8_17
+; RV32I-NEXT:  # %bb.16:
+; RV32I-NEXT:    mv t5, s1
+; RV32I-NEXT:  .LBB8_17:
+; RV32I-NEXT:    bltu s0, t1, .LBB8_19
+; RV32I-NEXT:  # %bb.18:
+; RV32I-NEXT:    li t1, 0
+; RV32I-NEXT:    sll a7, a7, s0
+; RV32I-NEXT:    bnez s0, .LBB8_20
+; RV32I-NEXT:    j .LBB8_21
+; RV32I-NEXT:  .LBB8_19:
+; RV32I-NEXT:    sll t1, a7, a2
+; RV32I-NEXT:    neg s1, s0
+; RV32I-NEXT:    srl a7, a7, s1
+; RV32I-NEXT:    or a7, a7, t0
+; RV32I-NEXT:    beqz s0, .LBB8_21
+; RV32I-NEXT:  .LBB8_20:
+; RV32I-NEXT:    mv a3, a7
+; RV32I-NEXT:  .LBB8_21:
+; RV32I-NEXT:    bltu a2, a6, .LBB8_23
+; RV32I-NEXT:  # %bb.22:
+; RV32I-NEXT:    li a4, 0
+; RV32I-NEXT:    li a5, 0
+; RV32I-NEXT:    bnez a2, .LBB8_24
+; RV32I-NEXT:    j .LBB8_25
+; RV32I-NEXT:  .LBB8_23:
+; RV32I-NEXT:    or t1, t3, t6
+; RV32I-NEXT:    or a3, t4, t5
+; RV32I-NEXT:    beqz a2, .LBB8_25
+; RV32I-NEXT:  .LBB8_24:
+; RV32I-NEXT:    mv t2, t1
+; RV32I-NEXT:    mv a1, a3
+; RV32I-NEXT:  .LBB8_25:
+; RV32I-NEXT:    sw a4, 0(a0)
+; RV32I-NEXT:    sw a5, 4(a0)
+; RV32I-NEXT:    sw t2, 8(a0)
+; RV32I-NEXT:    sw a1, 12(a0)
+; RV32I-NEXT:    lw s0, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s1, 8(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    addi sp, sp, 16
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: shl128:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    mv a3, a0
+; RV64I-NEXT:    li a4, 64
+; RV64I-NEXT:    bltu a2, a4, .LBB8_2
+; RV64I-NEXT:  # %bb.1:
+; RV64I-NEXT:    li a0, 0
+; RV64I-NEXT:    subw a4, a2, a4
+; RV64I-NEXT:    sll a3, a3, a4
+; RV64I-NEXT:    bnez a2, .LBB8_3
+; RV64I-NEXT:    j .LBB8_4
+; RV64I-NEXT:  .LBB8_2:
+; RV64I-NEXT:    sll a0, a3, a2
+; RV64I-NEXT:    negw a4, a2
+; RV64I-NEXT:    srl a3, a3, a4
+; RV64I-NEXT:    sll a4, a1, a2
+; RV64I-NEXT:    or a3, a3, a4
+; RV64I-NEXT:    beqz a2, .LBB8_4
+; RV64I-NEXT:  .LBB8_3:
+; RV64I-NEXT:    mv a1, a3
+; RV64I-NEXT:  .LBB8_4:
+; RV64I-NEXT:    ret
+  %1 = shl i128 %a, %b
+  ret i128 %1
+}
+
+define i64 @fshr64_minsize(i64 %a, i64 %b) minsize nounwind {
+; RV32I-LABEL: fshr64_minsize:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    andi a5, a2, 63
+; RV32I-NEXT:    li a4, 32
+; RV32I-NEXT:    bltu a5, a4, .LBB9_2
+; RV32I-NEXT:  # %bb.1:
+; RV32I-NEXT:    srl a6, a1, a5
+; RV32I-NEXT:    j .LBB9_3
+; RV32I-NEXT:  .LBB9_2:
+; RV32I-NEXT:    srl a3, a0, a2
+; RV32I-NEXT:    neg a6, a5
+; RV32I-NEXT:    sll a6, a1, a6
+; RV32I-NEXT:    or a6, a3, a6
+; RV32I-NEXT:  .LBB9_3:
+; RV32I-NEXT:    mv a3, a0
+; RV32I-NEXT:    beqz a5, .LBB9_5
+; RV32I-NEXT:  # %bb.4:
+; RV32I-NEXT:    mv a3, a6
+; RV32I-NEXT:  .LBB9_5:
+; RV32I-NEXT:    neg a6, a2
+; RV32I-NEXT:    bltu a5, a4, .LBB9_7
+; RV32I-NEXT:  # %bb.6:
+; RV32I-NEXT:    li a2, 0
+; RV32I-NEXT:    j .LBB9_8
+; RV32I-NEXT:  .LBB9_7:
+; RV32I-NEXT:    srl a2, a1, a2
+; RV32I-NEXT:  .LBB9_8:
+; RV32I-NEXT:    andi a5, a6, 63
+; RV32I-NEXT:    bltu a5, a4, .LBB9_10
+; RV32I-NEXT:  # %bb.9:
+; RV32I-NEXT:    li a4, 0
+; RV32I-NEXT:    sll a0, a0, a5
+; RV32I-NEXT:    bnez a5, .LBB9_11
+; RV32I-NEXT:    j .LBB9_12
+; RV32I-NEXT:  .LBB9_10:
+; RV32I-NEXT:    sll a4, a0, a6
+; RV32I-NEXT:    neg a7, a5
+; RV32I-NEXT:    srl a0, a0, a7
+; RV32I-NEXT:    sll a6, a1, a6
+; RV32I-NEXT:    or a0, a0, a6
+; RV32I-NEXT:    beqz a5, .LBB9_12
+; RV32I-NEXT:  .LBB9_11:
+; RV32I-NEXT:    mv a1, a0
+; RV32I-NEXT:  .LBB9_12:
+; RV32I-NEXT:    or a0, a3, a4
+; RV32I-NEXT:    or a1, a2, a1
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: fshr64_minsize:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    negw a2, a1
+; RV64I-NEXT:    srl a1, a0, a1
+; RV64I-NEXT:    sll a0, a0, a2
+; RV64I-NEXT:    or a0, a1, a0
+; RV64I-NEXT:    ret
+  %res = tail call i64 @llvm.fshr.i64(i64 %a, i64 %a, i64 %b)
+  ret i64 %res
+}
+
+define i128 @fshr128_minsize(i128 %a, i128 %b) minsize nounwind {
+; RV32I-LABEL: fshr128_minsize:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    lw t3, 0(a2)
+; RV32I-NEXT:    lw a2, 8(a1)
+; RV32I-NEXT:    lw a3, 12(a1)
+; RV32I-NEXT:    andi t4, t3, 127
+; RV32I-NEXT:    li a6, 32
+; RV32I-NEXT:    neg t6, t4
+; RV32I-NEXT:    sll t5, a3, t6
+; RV32I-NEXT:    bltu t4, a6, .LBB10_2
+; RV32I-NEXT:  # %bb.1:
+; RV32I-NEXT:    srl a5, a3, t4
+; RV32I-NEXT:    j .LBB10_3
+; RV32I-NEXT:  .LBB10_2:
+; RV32I-NEXT:    srl a4, a2, t3
+; RV32I-NEXT:    or a5, a4, t5
+; RV32I-NEXT:  .LBB10_3:
+; RV32I-NEXT:    mv a4, a2
+; RV32I-NEXT:    beqz t4, .LBB10_5
+; RV32I-NEXT:  # %bb.4:
+; RV32I-NEXT:    mv a4, a5
+; RV32I-NEXT:  .LBB10_5:
+; RV32I-NEXT:    lw a7, 0(a1)
+; RV32I-NEXT:    lw a5, 4(a1)
+; RV32I-NEXT:    bltu t4, a6, .LBB10_7
+; RV32I-NEXT:  # %bb.6:
+; RV32I-NEXT:    li a1, 0
+; RV32I-NEXT:    srl t2, a5, t4
+; RV32I-NEXT:    j .LBB10_8
+; RV32I-NEXT:  .LBB10_7:
+; RV32I-NEXT:    srl a1, a3, t3
+; RV32I-NEXT:    srl t0, a7, t3
+; RV32I-NEXT:    sll t1, a5, t6
+; RV32I-NEXT:    or t2, t0, t1
+; RV32I-NEXT:  .LBB10_8:
+; RV32I-NEXT:    li t0, 64
+; RV32I-NEXT:    mv t1, a7
+; RV32I-NEXT:    beqz t4, .LBB10_10
+; RV32I-NEXT:  # %bb.9:
+; RV32I-NEXT:    mv t1, t2
+; RV32I-NEXT:  .LBB10_10:
+; RV32I-NEXT:    addi sp, sp, -32
+; RV32I-NEXT:    sw s0, 28(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s1, 24(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s2, 20(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s3, 16(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s4, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sub s0, t0, t4
+; RV32I-NEXT:    bltu t4, a6, .LBB10_13
+; RV32I-NEXT:  # %bb.11:
+; RV32I-NEXT:    li t2, 0
+; RV32I-NEXT:    bgeu s0, a6, .LBB10_14
+; RV32I-NEXT:  .LBB10_12:
+; RV32I-NEXT:    sll t6, a2, t6
+; RV32I-NEXT:    neg s1, s0
+; RV32I-NEXT:    srl s1, a2, s1
+; RV32I-NEXT:    or s2, s1, t5
+; RV32I-NEXT:    j .LBB10_15
+; RV32I-NEXT:  .LBB10_13:
+; RV32I-NEXT:    srl t2, a5, t3
+; RV32I-NEXT:    bltu s0, a6, .LBB10_12
+; RV32I-NEXT:  .LBB10_14:
+; RV32I-NEXT:    li t6, 0
+; RV32I-NEXT:    sll s2, a2, s0
+; RV32I-NEXT:  .LBB10_15:
+; RV32I-NEXT:    addi s1, t4, -64
+; RV32I-NEXT:    mv t5, a3
+; RV32I-NEXT:    beqz s0, .LBB10_17
+; RV32I-NEXT:  # %bb.16:
+; RV32I-NEXT:    mv t5, s2
+; RV32I-NEXT:  .LBB10_17:
+; RV32I-NEXT:    bltu s1, a6, .LBB10_19
+; RV32I-NEXT:  # %bb.18:
+; RV32I-NEXT:    srl s2, a3, s1
+; RV32I-NEXT:    j .LBB10_20
+; RV32I-NEXT:  .LBB10_19:
+; RV32I-NEXT:    srl s0, a2, t4
+; RV32I-NEXT:    neg s2, s1
+; RV32I-NEXT:    sll s2, a3, s2
+; RV32I-NEXT:    or s2, s0, s2
+; RV32I-NEXT:  .LBB10_20:
+; RV32I-NEXT:    mv s0, a2
+; RV32I-NEXT:    beqz s1, .LBB10_22
+; RV32I-NEXT:  # %bb.21:
+; RV32I-NEXT:    mv s0, s2
+; RV32I-NEXT:  .LBB10_22:
+; RV32I-NEXT:    bltu s1, a6, .LBB10_24
+; RV32I-NEXT:  # %bb.23:
+; RV32I-NEXT:    li s1, 0
+; RV32I-NEXT:    bltu t4, t0, .LBB10_25
+; RV32I-NEXT:    j .LBB10_26
+; RV32I-NEXT:  .LBB10_24:
+; RV32I-NEXT:    srl s1, a3, t4
+; RV32I-NEXT:    bgeu t4, t0, .LBB10_26
+; RV32I-NEXT:  .LBB10_25:
+; RV32I-NEXT:    or s0, t1, t6
+; RV32I-NEXT:    or s1, t2, t5
+; RV32I-NEXT:  .LBB10_26:
+; RV32I-NEXT:    mv t1, a7
+; RV32I-NEXT:    mv t2, a5
+; RV32I-NEXT:    beqz t4, .LBB10_28
+; RV32I-NEXT:  # %bb.27:
+; RV32I-NEXT:    mv t1, s0
+; RV32I-NEXT:    mv t2, s1
+; RV32I-NEXT:  .LBB10_28:
+; RV32I-NEXT:    neg t6, t3
+; RV32I-NEXT:    bltu t4, t0, .LBB10_30
+; RV32I-NEXT:  # %bb.29:
+; RV32I-NEXT:    li a4, 0
+; RV32I-NEXT:    li a1, 0
+; RV32I-NEXT:  .LBB10_30:
+; RV32I-NEXT:    andi t3, t6, 127
+; RV32I-NEXT:    neg s2, t3
+; RV32I-NEXT:    srl s0, a7, s2
+; RV32I-NEXT:    bltu t3, a6, .LBB10_32
+; RV32I-NEXT:  # %bb.31:
+; RV32I-NEXT:    li t4, 0
+; RV32I-NEXT:    sll s3, a7, t3
+; RV32I-NEXT:    j .LBB10_33
+; RV32I-NEXT:  .LBB10_32:
+; RV32I-NEXT:    sll t4, a7, t6
+; RV32I-NEXT:    sll t5, a5, t6
+; RV32I-NEXT:    or s3, s0, t5
+; RV32I-NEXT:  .LBB10_33:
+; RV32I-NEXT:    sub s1, t0, t3
+; RV32I-NEXT:    mv t5, a5
+; RV32I-NEXT:    beqz t3, .LBB10_35
+; RV32I-NEXT:  # %bb.34:
+; RV32I-NEXT:    mv t5, s3
+; RV32I-NEXT:  .LBB10_35:
+; RV32I-NEXT:    bltu s1, a6, .LBB10_37
+; RV32I-NEXT:  # %bb.36:
+; RV32I-NEXT:    srl s3, a5, s1
+; RV32I-NEXT:    j .LBB10_38
+; RV32I-NEXT:  .LBB10_37:
+; RV32I-NEXT:    neg s3, s1
+; RV32I-NEXT:    sll s3, a5, s3
+; RV32I-NEXT:    or s3, s0, s3
+; RV32I-NEXT:  .LBB10_38:
+; RV32I-NEXT:    mv s0, a7
+; RV32I-NEXT:    beqz s1, .LBB10_40
+; RV32I-NEXT:  # %bb.39:
+; RV32I-NEXT:    mv s0, s3
+; RV32I-NEXT:  .LBB10_40:
+; RV32I-NEXT:    bltu s1, a6, .LBB10_43
+; RV32I-NEXT:  # %bb.41:
+; RV32I-NEXT:    li s1, 0
+; RV32I-NEXT:    bgeu t3, a6, .LBB10_44
+; RV32I-NEXT:  .LBB10_42:
+; RV32I-NEXT:    sll s3, a2, t6
+; RV32I-NEXT:    srl s2, a2, s2
+; RV32I-NEXT:    sll t6, a3, t6
+; RV32I-NEXT:    or s4, s2, t6
+; RV32I-NEXT:    j .LBB10_45
+; RV32I-NEXT:  .LBB10_43:
+; RV32I-NEXT:    srl s1, a5, s2
+; RV32I-NEXT:    bltu t3, a6, .LBB10_42
+; RV32I-NEXT:  .LBB10_44:
+; RV32I-NEXT:    li s3, 0
+; RV32I-NEXT:    sll s4, a2, t3
+; RV32I-NEXT:  .LBB10_45:
+; RV32I-NEXT:    addi s2, t3, -64
+; RV32I-NEXT:    mv t6, a3
+; RV32I-NEXT:    beqz t3, .LBB10_47
+; RV32I-NEXT:  # %bb.46:
+; RV32I-NEXT:    mv t6, s4
+; RV32I-NEXT:  .LBB10_47:
+; RV32I-NEXT:    bltu s2, a6, .LBB10_49
+; RV32I-NEXT:  # %bb.48:
+; RV32I-NEXT:    li a6, 0
+; RV32I-NEXT:    sll a7, a7, s2
+; RV32I-NEXT:    bnez s2, .LBB10_50
+; RV32I-NEXT:    j .LBB10_51
+; RV32I-NEXT:  .LBB10_49:
+; RV32I-NEXT:    sll a6, a7, t3
+; RV32I-NEXT:    neg s4, s2
+; RV32I-NEXT:    srl a7, a7, s4
+; RV32I-NEXT:    sll s4, a5, t3
+; RV32I-NEXT:    or a7, a7, s4
+; RV32I-NEXT:    beqz s2, .LBB10_51
+; RV32I-NEXT:  .LBB10_50:
+; RV32I-NEXT:    mv a5, a7
+; RV32I-NEXT:  .LBB10_51:
+; RV32I-NEXT:    bltu t3, t0, .LBB10_53
+; RV32I-NEXT:  # %bb.52:
+; RV32I-NEXT:    li t4, 0
+; RV32I-NEXT:    li t5, 0
+; RV32I-NEXT:    bnez t3, .LBB10_54
+; RV32I-NEXT:    j .LBB10_55
+; RV32I-NEXT:  .LBB10_53:
+; RV32I-NEXT:    or a6, s0, s3
+; RV32I-NEXT:    or a5, s1, t6
+; RV32I-NEXT:    beqz t3, .LBB10_55
+; RV32I-NEXT:  .LBB10_54:
+; RV32I-NEXT:    mv a2, a6
+; RV32I-NEXT:    mv a3, a5
+; RV32I-NEXT:  .LBB10_55:
+; RV32I-NEXT:    or a5, t1, t4
+; RV32I-NEXT:    or a6, t2, t5
+; RV32I-NEXT:    or a2, a4, a2
+; RV32I-NEXT:    or a1, a1, a3
+; RV32I-NEXT:    sw a5, 0(a0)
+; RV32I-NEXT:    sw a6, 4(a0)
+; RV32I-NEXT:    sw a2, 8(a0)
+; RV32I-NEXT:    sw a1, 12(a0)
+; RV32I-NEXT:    lw s0, 28(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s1, 24(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s2, 20(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s3, 16(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s4, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    addi sp, sp, 32
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: fshr128_minsize:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    andi a5, a2, 127
+; RV64I-NEXT:    li a4, 64
+; RV64I-NEXT:    bltu a5, a4, .LBB10_2
+; RV64I-NEXT:  # %bb.1:
+; RV64I-NEXT:    subw a3, a5, a4
+; RV64I-NEXT:    srl a6, a1, a3
+; RV64I-NEXT:    j .LBB10_3
+; RV64I-NEXT:  .LBB10_2:
+; RV64I-NEXT:    srl a3, a0, a2
+; RV64I-NEXT:    negw a6, a5
+; RV64I-NEXT:    sll a6, a1, a6
+; RV64I-NEXT:    or a6, a3, a6
+; RV64I-NEXT:  .LBB10_3:
+; RV64I-NEXT:    mv a3, a0
+; RV64I-NEXT:    beqz a5, .LBB10_5
+; RV64I-NEXT:  # %bb.4:
+; RV64I-NEXT:    mv a3, a6
+; RV64I-NEXT:  .LBB10_5:
+; RV64I-NEXT:    negw a7, a2
+; RV64I-NEXT:    bltu a5, a4, .LBB10_7
+; RV64I-NEXT:  # %bb.6:
+; RV64I-NEXT:    li a2, 0
+; RV64I-NEXT:    j .LBB10_8
+; RV64I-NEXT:  .LBB10_7:
+; RV64I-NEXT:    srl a2, a1, a2
+; RV64I-NEXT:  .LBB10_8:
+; RV64I-NEXT:    andi a6, a7, 127
+; RV64I-NEXT:    bltu a6, a4, .LBB10_10
+; RV64I-NEXT:  # %bb.9:
+; RV64I-NEXT:    li a5, 0
+; RV64I-NEXT:    subw a4, a6, a4
+; RV64I-NEXT:    sll a0, a0, a4
+; RV64I-NEXT:    bnez a6, .LBB10_11
+; RV64I-NEXT:    j .LBB10_12
+; RV64I-NEXT:  .LBB10_10:
+; RV64I-NEXT:    sll a5, a0, a7
+; RV64I-NEXT:    negw a4, a6
+; RV64I-NEXT:    srl a0, a0, a4
+; RV64I-NEXT:    sll a4, a1, a7
+; RV64I-NEXT:    or a0, a0, a4
+; RV64I-NEXT:    beqz a6, .LBB10_12
+; RV64I-NEXT:  .LBB10_11:
+; RV64I-NEXT:    mv a1, a0
+; RV64I-NEXT:  .LBB10_12:
+; RV64I-NEXT:    or a0, a3, a5
+; RV64I-NEXT:    or a1, a2, a1
+; RV64I-NEXT:    ret
+  %res = tail call i128 @llvm.fshr.i128(i128 %a, i128 %a, i128 %b)
+  ret i128 %res
+}
diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/wide-scalar-shift-by-byte-multiple-legalization.ll b/llvm/test/CodeGen/RISCV/GlobalISel/wide-scalar-shift-by-byte-multiple-legalization.ll
new file mode 100644
index 0000000000000..bc002fee4417c
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/GlobalISel/wide-scalar-shift-by-byte-multiple-legalization.ll
@@ -0,0 +1,10982 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=riscv64 -verify-machineinstrs -global-isel < %s | FileCheck %s -check-prefixes=RV64I
+; RUN: llc -mtriple=riscv32 -verify-machineinstrs -global-isel < %s | FileCheck %s -check-prefixes=RV32I
+
+define void @lshr_4bytes(ptr %src.ptr, ptr %byteOff.ptr, ptr %dst) nounwind {
+; RV64I-LABEL: lshr_4bytes:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    lbu a3, 1(a0)
+; RV64I-NEXT:    lbu a4, 0(a0)
+; RV64I-NEXT:    lbu a5, 2(a0)
+; RV64I-NEXT:    lbu a0, 3(a0)
+; RV64I-NEXT:    slli a3, a3, 8
+; RV64I-NEXT:    or a3, a3, a4
+; RV64I-NEXT:    lbu a4, 0(a1)
+; RV64I-NEXT:    lbu a6, 1(a1)
+; RV64I-NEXT:    lbu a7, 2(a1)
+; RV64I-NEXT:    lbu a1, 3(a1)
+; RV64I-NEXT:    slli a0, a0, 8
+; RV64I-NEXT:    or a0, a0, a5
+; RV64I-NEXT:    slli a6, a6, 8
+; RV64I-NEXT:    or a4, a6, a4
+; RV64I-NEXT:    slli a1, a1, 8
+; RV64I-NEXT:    or a1, a1, a7
+; RV64I-NEXT:    slli a0, a0, 16
+; RV64I-NEXT:    slli a1, a1, 16
+; RV64I-NEXT:    or a0, a0, a3
+; RV64I-NEXT:    or a1, a1, a4
+; RV64I-NEXT:    slli a1, a1, 3
+; RV64I-NEXT:    srlw a0, a0, a1
+; RV64I-NEXT:    srliw a1, a0, 16
+; RV64I-NEXT:    slli a3, a0, 48
+; RV64I-NEXT:    srliw a4, a0, 24
+; RV64I-NEXT:    srli a3, a3, 48
+; RV64I-NEXT:    srli a3, a3, 8
+; RV64I-NEXT:    sb a0, 0(a2)
+; RV64I-NEXT:    sb a3, 1(a2)
+; RV64I-NEXT:    sb a1, 2(a2)
+; RV64I-NEXT:    sb a4, 3(a2)
+; RV64I-NEXT:    ret
+;
+; RV32I-LABEL: lshr_4bytes:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    lbu a3, 1(a0)
+; RV32I-NEXT:    lbu a4, 0(a0)
+; RV32I-NEXT:    lbu a5, 2(a0)
+; RV32I-NEXT:    lbu a0, 3(a0)
+; RV32I-NEXT:    slli a3, a3, 8
+; RV32I-NEXT:    or a3, a3, a4
+; RV32I-NEXT:    lbu a4, 0(a1)
+; RV32I-NEXT:    lbu a6, 1(a1)
+; RV32I-NEXT:    lbu a7, 2(a1)
+; RV32I-NEXT:    lbu a1, 3(a1)
+; RV32I-NEXT:    slli a0, a0, 8
+; RV32I-NEXT:    or a0, a0, a5
+; RV32I-NEXT:    slli a6, a6, 8
+; RV32I-NEXT:    or a4, a6, a4
+; RV32I-NEXT:    slli a1, a1, 8
+; RV32I-NEXT:    or a1, a1, a7
+; RV32I-NEXT:    slli a0, a0, 16
+; RV32I-NEXT:    slli a1, a1, 16
+; RV32I-NEXT:    or a0, a0, a3
+; RV32I-NEXT:    or a1, a1, a4
+; RV32I-NEXT:    slli a1, a1, 3
+; RV32I-NEXT:    srl a0, a0, a1
+; RV32I-NEXT:    srli a1, a0, 16
+; RV32I-NEXT:    slli a3, a0, 16
+; RV32I-NEXT:    srli a4, a0, 24
+; RV32I-NEXT:    srli a3, a3, 16
+; RV32I-NEXT:    srli a3, a3, 8
+; RV32I-NEXT:    sb a0, 0(a2)
+; RV32I-NEXT:    sb a3, 1(a2)
+; RV32I-NEXT:    sb a1, 2(a2)
+; RV32I-NEXT:    sb a4, 3(a2)
+; RV32I-NEXT:    ret
+  %src = load i32, ptr %src.ptr, align 1
+  %byteOff = load i32, ptr %byteOff.ptr, align 1
+  %bitOff = shl i32 %byteOff, 3
+  %res = lshr i32 %src, %bitOff
+  store i32 %res, ptr %dst, align 1
+  ret void
+}
+define void @shl_4bytes(ptr %src.ptr, ptr %byteOff.ptr, ptr %dst) nounwind {
+; RV64I-LABEL: shl_4bytes:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    lbu a3, 1(a0)
+; RV64I-NEXT:    lbu a4, 0(a0)
+; RV64I-NEXT:    lbu a5, 2(a0)
+; RV64I-NEXT:    lbu a0, 3(a0)
+; RV64I-NEXT:    slli a3, a3, 8
+; RV64I-NEXT:    or a3, a3, a4
+; RV64I-NEXT:    lbu a4, 0(a1)
+; RV64I-NEXT:    lbu a6, 1(a1)
+; RV64I-NEXT:    lbu a7, 2(a1)
+; RV64I-NEXT:    lbu a1, 3(a1)
+; RV64I-NEXT:    slli a0, a0, 8
+; RV64I-NEXT:    or a0, a0, a5
+; RV64I-NEXT:    slli a6, a6, 8
+; RV64I-NEXT:    or a4, a6, a4
+; RV64I-NEXT:    slli a1, a1, 8
+; RV64I-NEXT:    or a1, a1, a7
+; RV64I-NEXT:    slli a0, a0, 16
+; RV64I-NEXT:    slli a1, a1, 16
+; RV64I-NEXT:    or a0, a0, a3
+; RV64I-NEXT:    or a1, a1, a4
+; RV64I-NEXT:    slli a1, a1, 3
+; RV64I-NEXT:    sllw a0, a0, a1
+; RV64I-NEXT:    srliw a1, a0, 16
+; RV64I-NEXT:    slli a3, a0, 48
+; RV64I-NEXT:    srliw a4, a0, 24
+; RV64I-NEXT:    srli a3, a3, 48
+; RV64I-NEXT:    srli a3, a3, 8
+; RV64I-NEXT:    sb a0, 0(a2)
+; RV64I-NEXT:    sb a3, 1(a2)
+; RV64I-NEXT:    sb a1, 2(a2)
+; RV64I-NEXT:    sb a4, 3(a2)
+; RV64I-NEXT:    ret
+;
+; RV32I-LABEL: shl_4bytes:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    lbu a3, 1(a0)
+; RV32I-NEXT:    lbu a4, 0(a0)
+; RV32I-NEXT:    lbu a5, 2(a0)
+; RV32I-NEXT:    lbu a0, 3(a0)
+; RV32I-NEXT:    slli a3, a3, 8
+; RV32I-NEXT:    or a3, a3, a4
+; RV32I-NEXT:    lbu a4, 0(a1)
+; RV32I-NEXT:    lbu a6, 1(a1)
+; RV32I-NEXT:    lbu a7, 2(a1)
+; RV32I-NEXT:    lbu a1, 3(a1)
+; RV32I-NEXT:    slli a0, a0, 8
+; RV32I-NEXT:    or a0, a0, a5
+; RV32I-NEXT:    slli a6, a6, 8
+; RV32I-NEXT:    or a4, a6, a4
+; RV32I-NEXT:    slli a1, a1, 8
+; RV32I-NEXT:    or a1, a1, a7
+; RV32I-NEXT:    slli a0, a0, 16
+; RV32I-NEXT:    slli a1, a1, 16
+; RV32I-NEXT:    or a0, a0, a3
+; RV32I-NEXT:    or a1, a1, a4
+; RV32I-NEXT:    slli a1, a1, 3
+; RV32I-NEXT:    sll a0, a0, a1
+; RV32I-NEXT:    srli a1, a0, 16
+; RV32I-NEXT:    slli a3, a0, 16
+; RV32I-NEXT:    srli a4, a0, 24
+; RV32I-NEXT:    srli a3, a3, 16
+; RV32I-NEXT:    srli a3, a3, 8
+; RV32I-NEXT:    sb a0, 0(a2)
+; RV32I-NEXT:    sb a3, 1(a2)
+; RV32I-NEXT:    sb a1, 2(a2)
+; RV32I-NEXT:    sb a4, 3(a2)
+; RV32I-NEXT:    ret
+  %src = load i32, ptr %src.ptr, align 1
+  %byteOff = load i32, ptr %byteOff.ptr, align 1
+  %bitOff = shl i32 %byteOff, 3
+  %res = shl i32 %src, %bitOff
+  store i32 %res, ptr %dst, align 1
+  ret void
+}
+define void @ashr_4bytes(ptr %src.ptr, ptr %byteOff.ptr, ptr %dst) nounwind {
+; RV64I-LABEL: ashr_4bytes:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    lbu a3, 1(a0)
+; RV64I-NEXT:    lbu a4, 0(a0)
+; RV64I-NEXT:    lbu a5, 2(a0)
+; RV64I-NEXT:    lbu a0, 3(a0)
+; RV64I-NEXT:    slli a3, a3, 8
+; RV64I-NEXT:    or a3, a3, a4
+; RV64I-NEXT:    lbu a4, 0(a1)
+; RV64I-NEXT:    lbu a6, 1(a1)
+; RV64I-NEXT:    lbu a7, 2(a1)
+; RV64I-NEXT:    lbu a1, 3(a1)
+; RV64I-NEXT:    slli a0, a0, 8
+; RV64I-NEXT:    or a0, a0, a5
+; RV64I-NEXT:    slli a6, a6, 8
+; RV64I-NEXT:    or a4, a6, a4
+; RV64I-NEXT:    slli a1, a1, 8
+; RV64I-NEXT:    or a1, a1, a7
+; RV64I-NEXT:    slli a0, a0, 16
+; RV64I-NEXT:    slli a1, a1, 16
+; RV64I-NEXT:    or a0, a0, a3
+; RV64I-NEXT:    or a1, a1, a4
+; RV64I-NEXT:    slli a1, a1, 3
+; RV64I-NEXT:    sraw a0, a0, a1
+; RV64I-NEXT:    srliw a1, a0, 16
+; RV64I-NEXT:    slli a3, a0, 48
+; RV64I-NEXT:    srliw a4, a0, 24
+; RV64I-NEXT:    srli a3, a3, 48
+; RV64I-NEXT:    srli a3, a3, 8
+; RV64I-NEXT:    sb a0, 0(a2)
+; RV64I-NEXT:    sb a3, 1(a2)
+; RV64I-NEXT:    sb a1, 2(a2)
+; RV64I-NEXT:    sb a4, 3(a2)
+; RV64I-NEXT:    ret
+;
+; RV32I-LABEL: ashr_4bytes:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    lbu a3, 1(a0)
+; RV32I-NEXT:    lbu a4, 0(a0)
+; RV32I-NEXT:    lbu a5, 2(a0)
+; RV32I-NEXT:    lbu a0, 3(a0)
+; RV32I-NEXT:    slli a3, a3, 8
+; RV32I-NEXT:    or a3, a3, a4
+; RV32I-NEXT:    lbu a4, 0(a1)
+; RV32I-NEXT:    lbu a6, 1(a1)
+; RV32I-NEXT:    lbu a7, 2(a1)
+; RV32I-NEXT:    lbu a1, 3(a1)
+; RV32I-NEXT:    slli a0, a0, 8
+; RV32I-NEXT:    or a0, a0, a5
+; RV32I-NEXT:    slli a6, a6, 8
+; RV32I-NEXT:    or a4, a6, a4
+; RV32I-NEXT:    slli a1, a1, 8
+; RV32I-NEXT:    or a1, a1, a7
+; RV32I-NEXT:    slli a0, a0, 16
+; RV32I-NEXT:    slli a1, a1, 16
+; RV32I-NEXT:    or a0, a0, a3
+; RV32I-NEXT:    or a1, a1, a4
+; RV32I-NEXT:    slli a1, a1, 3
+; RV32I-NEXT:    sra a0, a0, a1
+; RV32I-NEXT:    srli a1, a0, 16
+; RV32I-NEXT:    slli a3, a0, 16
+; RV32I-NEXT:    srli a4, a0, 24
+; RV32I-NEXT:    srli a3, a3, 16
+; RV32I-NEXT:    srli a3, a3, 8
+; RV32I-NEXT:    sb a0, 0(a2)
+; RV32I-NEXT:    sb a3, 1(a2)
+; RV32I-NEXT:    sb a1, 2(a2)
+; RV32I-NEXT:    sb a4, 3(a2)
+; RV32I-NEXT:    ret
+  %src = load i32, ptr %src.ptr, align 1
+  %byteOff = load i32, ptr %byteOff.ptr, align 1
+  %bitOff = shl i32 %byteOff, 3
+  %res = ashr i32 %src, %bitOff
+  store i32 %res, ptr %dst, align 1
+  ret void
+}
+
+define void @lshr_8bytes(ptr %src.ptr, ptr %byteOff.ptr, ptr %dst) nounwind {
+; RV64I-LABEL: lshr_8bytes:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    lbu a3, 0(a0)
+; RV64I-NEXT:    lbu a4, 1(a0)
+; RV64I-NEXT:    lbu a5, 2(a0)
+; RV64I-NEXT:    lbu a6, 3(a0)
+; RV64I-NEXT:    lbu a7, 4(a0)
+; RV64I-NEXT:    lbu t0, 5(a0)
+; RV64I-NEXT:    lbu t1, 6(a0)
+; RV64I-NEXT:    lbu a0, 7(a0)
+; RV64I-NEXT:    slli a4, a4, 8
+; RV64I-NEXT:    slli a6, a6, 8
+; RV64I-NEXT:    or a3, a4, a3
+; RV64I-NEXT:    or a4, a6, a5
+; RV64I-NEXT:    lbu a5, 0(a1)
+; RV64I-NEXT:    lbu a6, 1(a1)
+; RV64I-NEXT:    lbu t2, 2(a1)
+; RV64I-NEXT:    lbu t3, 3(a1)
+; RV64I-NEXT:    slli t0, t0, 8
+; RV64I-NEXT:    slli a0, a0, 8
+; RV64I-NEXT:    slli a6, a6, 8
+; RV64I-NEXT:    or a7, t0, a7
+; RV64I-NEXT:    or a0, a0, t1
+; RV64I-NEXT:    or a5, a6, a5
+; RV64I-NEXT:    lbu a6, 4(a1)
+; RV64I-NEXT:    lbu t0, 5(a1)
+; RV64I-NEXT:    lbu t1, 6(a1)
+; RV64I-NEXT:    lbu a1, 7(a1)
+; RV64I-NEXT:    slli t3, t3, 8
+; RV64I-NEXT:    or t2, t3, t2
+; RV64I-NEXT:    slli t0, t0, 8
+; RV64I-NEXT:    or a6, t0, a6
+; RV64I-NEXT:    slli a1, a1, 8
+; RV64I-NEXT:    or a1, a1, t1
+; RV64I-NEXT:    slli a4, a4, 16
+; RV64I-NEXT:    or a3, a4, a3
+; RV64I-NEXT:    lui a4, 16
+; RV64I-NEXT:    addi a4, a4, -1
+; RV64I-NEXT:    slli a0, a0, 16
+; RV64I-NEXT:    slli t2, t2, 16
+; RV64I-NEXT:    slli a1, a1, 16
+; RV64I-NEXT:    or a0, a0, a7
+; RV64I-NEXT:    or a5, t2, a5
+; RV64I-NEXT:    or a1, a1, a6
+; RV64I-NEXT:    slli a0, a0, 32
+; RV64I-NEXT:    slli a1, a1, 32
+; RV64I-NEXT:    or a0, a0, a3
+; RV64I-NEXT:    or a1, a1, a5
+; RV64I-NEXT:    slli a1, a1, 3
+; RV64I-NEXT:    srl a0, a0, a1
+; RV64I-NEXT:    srli a1, a0, 32
+; RV64I-NEXT:    srliw a3, a0, 16
+; RV64I-NEXT:    and a5, a0, a4
+; RV64I-NEXT:    srliw a6, a0, 24
+; RV64I-NEXT:    srli a7, a0, 48
+; RV64I-NEXT:    srli t0, a0, 56
+; RV64I-NEXT:    srli a5, a5, 8
+; RV64I-NEXT:    and a4, a1, a4
+; RV64I-NEXT:    sb a0, 0(a2)
+; RV64I-NEXT:    sb a5, 1(a2)
+; RV64I-NEXT:    sb a3, 2(a2)
+; RV64I-NEXT:    sb a6, 3(a2)
+; RV64I-NEXT:    srli a4, a4, 8
+; RV64I-NEXT:    sb a1, 4(a2)
+; RV64I-NEXT:    sb a4, 5(a2)
+; RV64I-NEXT:    sb a7, 6(a2)
+; RV64I-NEXT:    sb t0, 7(a2)
+; RV64I-NEXT:    ret
+;
+; RV32I-LABEL: lshr_8bytes:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    lbu a3, 0(a0)
+; RV32I-NEXT:    lbu a4, 1(a0)
+; RV32I-NEXT:    lbu a5, 2(a0)
+; RV32I-NEXT:    lbu a6, 3(a0)
+; RV32I-NEXT:    lbu a7, 4(a0)
+; RV32I-NEXT:    lbu t0, 5(a0)
+; RV32I-NEXT:    lbu t1, 6(a0)
+; RV32I-NEXT:    lbu a0, 7(a0)
+; RV32I-NEXT:    slli a4, a4, 8
+; RV32I-NEXT:    slli a6, a6, 8
+; RV32I-NEXT:    slli t0, t0, 8
+; RV32I-NEXT:    or a3, a4, a3
+; RV32I-NEXT:    or a4, a6, a5
+; RV32I-NEXT:    or a5, t0, a7
+; RV32I-NEXT:    lbu a6, 0(a1)
+; RV32I-NEXT:    lbu a7, 1(a1)
+; RV32I-NEXT:    lbu t0, 2(a1)
+; RV32I-NEXT:    lbu a1, 3(a1)
+; RV32I-NEXT:    slli a0, a0, 8
+; RV32I-NEXT:    or t1, a0, t1
+; RV32I-NEXT:    slli a7, a7, 8
+; RV32I-NEXT:    or a6, a7, a6
+; RV32I-NEXT:    slli a1, a1, 8
+; RV32I-NEXT:    or a1, a1, t0
+; RV32I-NEXT:    slli a0, a4, 16
+; RV32I-NEXT:    or a0, a0, a3
+; RV32I-NEXT:    slli a3, t1, 16
+; RV32I-NEXT:    slli a1, a1, 16
+; RV32I-NEXT:    or a1, a1, a6
+; RV32I-NEXT:    slli a1, a1, 3
+; RV32I-NEXT:    li a4, 32
+; RV32I-NEXT:    or a3, a3, a5
+; RV32I-NEXT:    bltu a1, a4, .LBB3_2
+; RV32I-NEXT:  # %bb.1:
+; RV32I-NEXT:    srl a5, a3, a1
+; RV32I-NEXT:    bnez a1, .LBB3_3
+; RV32I-NEXT:    j .LBB3_4
+; RV32I-NEXT:  .LBB3_2:
+; RV32I-NEXT:    srl a5, a0, a1
+; RV32I-NEXT:    neg a6, a1
+; RV32I-NEXT:    sll a6, a3, a6
+; RV32I-NEXT:    or a5, a5, a6
+; RV32I-NEXT:    beqz a1, .LBB3_4
+; RV32I-NEXT:  .LBB3_3:
+; RV32I-NEXT:    mv a0, a5
+; RV32I-NEXT:  .LBB3_4:
+; RV32I-NEXT:    bltu a1, a4, .LBB3_6
+; RV32I-NEXT:  # %bb.5:
+; RV32I-NEXT:    li a1, 0
+; RV32I-NEXT:    j .LBB3_7
+; RV32I-NEXT:  .LBB3_6:
+; RV32I-NEXT:    srl a1, a3, a1
+; RV32I-NEXT:  .LBB3_7:
+; RV32I-NEXT:    srli a3, a0, 16
+; RV32I-NEXT:    lui a4, 16
+; RV32I-NEXT:    srli a5, a0, 24
+; RV32I-NEXT:    srli a6, a1, 16
+; RV32I-NEXT:    srli a7, a1, 24
+; RV32I-NEXT:    addi a4, a4, -1
+; RV32I-NEXT:    and t0, a0, a4
+; RV32I-NEXT:    and a4, a1, a4
+; RV32I-NEXT:    srli t0, t0, 8
+; RV32I-NEXT:    srli a4, a4, 8
+; RV32I-NEXT:    sb a0, 0(a2)
+; RV32I-NEXT:    sb t0, 1(a2)
+; RV32I-NEXT:    sb a3, 2(a2)
+; RV32I-NEXT:    sb a5, 3(a2)
+; RV32I-NEXT:    sb a1, 4(a2)
+; RV32I-NEXT:    sb a4, 5(a2)
+; RV32I-NEXT:    sb a6, 6(a2)
+; RV32I-NEXT:    sb a7, 7(a2)
+; RV32I-NEXT:    ret
+  %src = load i64, ptr %src.ptr, align 1
+  %byteOff = load i64, ptr %byteOff.ptr, align 1
+  %bitOff = shl i64 %byteOff, 3
+  %res = lshr i64 %src, %bitOff
+  store i64 %res, ptr %dst, align 1
+  ret void
+}
+define void @shl_8bytes(ptr %src.ptr, ptr %byteOff.ptr, ptr %dst) nounwind {
+; RV64I-LABEL: shl_8bytes:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    lbu a3, 0(a0)
+; RV64I-NEXT:    lbu a4, 1(a0)
+; RV64I-NEXT:    lbu a5, 2(a0)
+; RV64I-NEXT:    lbu a6, 3(a0)
+; RV64I-NEXT:    lbu a7, 4(a0)
+; RV64I-NEXT:    lbu t0, 5(a0)
+; RV64I-NEXT:    lbu t1, 6(a0)
+; RV64I-NEXT:    lbu a0, 7(a0)
+; RV64I-NEXT:    slli a4, a4, 8
+; RV64I-NEXT:    slli a6, a6, 8
+; RV64I-NEXT:    or a3, a4, a3
+; RV64I-NEXT:    or a4, a6, a5
+; RV64I-NEXT:    lbu a5, 0(a1)
+; RV64I-NEXT:    lbu a6, 1(a1)
+; RV64I-NEXT:    lbu t2, 2(a1)
+; RV64I-NEXT:    lbu t3, 3(a1)
+; RV64I-NEXT:    slli t0, t0, 8
+; RV64I-NEXT:    slli a0, a0, 8
+; RV64I-NEXT:    slli a6, a6, 8
+; RV64I-NEXT:    or a7, t0, a7
+; RV64I-NEXT:    or a0, a0, t1
+; RV64I-NEXT:    or a5, a6, a5
+; RV64I-NEXT:    lbu a6, 4(a1)
+; RV64I-NEXT:    lbu t0, 5(a1)
+; RV64I-NEXT:    lbu t1, 6(a1)
+; RV64I-NEXT:    lbu a1, 7(a1)
+; RV64I-NEXT:    slli t3, t3, 8
+; RV64I-NEXT:    or t2, t3, t2
+; RV64I-NEXT:    slli t0, t0, 8
+; RV64I-NEXT:    or a6, t0, a6
+; RV64I-NEXT:    slli a1, a1, 8
+; RV64I-NEXT:    or a1, a1, t1
+; RV64I-NEXT:    slli a4, a4, 16
+; RV64I-NEXT:    or a3, a4, a3
+; RV64I-NEXT:    lui a4, 16
+; RV64I-NEXT:    addi a4, a4, -1
+; RV64I-NEXT:    slli a0, a0, 16
+; RV64I-NEXT:    slli t2, t2, 16
+; RV64I-NEXT:    slli a1, a1, 16
+; RV64I-NEXT:    or a0, a0, a7
+; RV64I-NEXT:    or a5, t2, a5
+; RV64I-NEXT:    or a1, a1, a6
+; RV64I-NEXT:    slli a0, a0, 32
+; RV64I-NEXT:    slli a1, a1, 32
+; RV64I-NEXT:    or a0, a0, a3
+; RV64I-NEXT:    or a1, a1, a5
+; RV64I-NEXT:    slli a1, a1, 3
+; RV64I-NEXT:    sll a0, a0, a1
+; RV64I-NEXT:    srli a1, a0, 32
+; RV64I-NEXT:    srliw a3, a0, 16
+; RV64I-NEXT:    and a5, a0, a4
+; RV64I-NEXT:    srliw a6, a0, 24
+; RV64I-NEXT:    srli a7, a0, 48
+; RV64I-NEXT:    srli t0, a0, 56
+; RV64I-NEXT:    srli a5, a5, 8
+; RV64I-NEXT:    and a4, a1, a4
+; RV64I-NEXT:    sb a0, 0(a2)
+; RV64I-NEXT:    sb a5, 1(a2)
+; RV64I-NEXT:    sb a3, 2(a2)
+; RV64I-NEXT:    sb a6, 3(a2)
+; RV64I-NEXT:    srli a4, a4, 8
+; RV64I-NEXT:    sb a1, 4(a2)
+; RV64I-NEXT:    sb a4, 5(a2)
+; RV64I-NEXT:    sb a7, 6(a2)
+; RV64I-NEXT:    sb t0, 7(a2)
+; RV64I-NEXT:    ret
+;
+; RV32I-LABEL: shl_8bytes:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    lbu a3, 0(a0)
+; RV32I-NEXT:    lbu a4, 1(a0)
+; RV32I-NEXT:    lbu a5, 2(a0)
+; RV32I-NEXT:    lbu a6, 3(a0)
+; RV32I-NEXT:    lbu a7, 4(a0)
+; RV32I-NEXT:    lbu t0, 5(a0)
+; RV32I-NEXT:    lbu t1, 6(a0)
+; RV32I-NEXT:    lbu a0, 7(a0)
+; RV32I-NEXT:    slli a4, a4, 8
+; RV32I-NEXT:    slli a6, a6, 8
+; RV32I-NEXT:    slli t0, t0, 8
+; RV32I-NEXT:    or a3, a4, a3
+; RV32I-NEXT:    or a4, a6, a5
+; RV32I-NEXT:    or a5, t0, a7
+; RV32I-NEXT:    lbu a6, 0(a1)
+; RV32I-NEXT:    lbu a7, 1(a1)
+; RV32I-NEXT:    lbu t0, 2(a1)
+; RV32I-NEXT:    lbu a1, 3(a1)
+; RV32I-NEXT:    slli a0, a0, 8
+; RV32I-NEXT:    or a0, a0, t1
+; RV32I-NEXT:    slli a7, a7, 8
+; RV32I-NEXT:    or a6, a7, a6
+; RV32I-NEXT:    slli a1, a1, 8
+; RV32I-NEXT:    or a1, a1, t0
+; RV32I-NEXT:    slli a4, a4, 16
+; RV32I-NEXT:    or a4, a4, a3
+; RV32I-NEXT:    slli a0, a0, 16
+; RV32I-NEXT:    slli a1, a1, 16
+; RV32I-NEXT:    or a3, a1, a6
+; RV32I-NEXT:    slli a3, a3, 3
+; RV32I-NEXT:    li a1, 32
+; RV32I-NEXT:    or a0, a0, a5
+; RV32I-NEXT:    bltu a3, a1, .LBB4_2
+; RV32I-NEXT:  # %bb.1:
+; RV32I-NEXT:    li a1, 0
+; RV32I-NEXT:    sll a4, a4, a3
+; RV32I-NEXT:    bnez a3, .LBB4_3
+; RV32I-NEXT:    j .LBB4_4
+; RV32I-NEXT:  .LBB4_2:
+; RV32I-NEXT:    sll a1, a4, a3
+; RV32I-NEXT:    neg a5, a3
+; RV32I-NEXT:    srl a4, a4, a5
+; RV32I-NEXT:    sll a5, a0, a3
+; RV32I-NEXT:    or a4, a4, a5
+; RV32I-NEXT:    beqz a3, .LBB4_4
+; RV32I-NEXT:  .LBB4_3:
+; RV32I-NEXT:    mv a0, a4
+; RV32I-NEXT:  .LBB4_4:
+; RV32I-NEXT:    srli a3, a1, 16
+; RV32I-NEXT:    lui a4, 16
+; RV32I-NEXT:    srli a5, a1, 24
+; RV32I-NEXT:    srli a6, a0, 16
+; RV32I-NEXT:    srli a7, a0, 24
+; RV32I-NEXT:    addi a4, a4, -1
+; RV32I-NEXT:    and t0, a1, a4
+; RV32I-NEXT:    and a4, a0, a4
+; RV32I-NEXT:    srli t0, t0, 8
+; RV32I-NEXT:    srli a4, a4, 8
+; RV32I-NEXT:    sb a1, 0(a2)
+; RV32I-NEXT:    sb t0, 1(a2)
+; RV32I-NEXT:    sb a3, 2(a2)
+; RV32I-NEXT:    sb a5, 3(a2)
+; RV32I-NEXT:    sb a0, 4(a2)
+; RV32I-NEXT:    sb a4, 5(a2)
+; RV32I-NEXT:    sb a6, 6(a2)
+; RV32I-NEXT:    sb a7, 7(a2)
+; RV32I-NEXT:    ret
+  %src = load i64, ptr %src.ptr, align 1
+  %byteOff = load i64, ptr %byteOff.ptr, align 1
+  %bitOff = shl i64 %byteOff, 3
+  %res = shl i64 %src, %bitOff
+  store i64 %res, ptr %dst, align 1
+  ret void
+}
+define void @ashr_8bytes(ptr %src.ptr, ptr %byteOff.ptr, ptr %dst) nounwind {
+; RV64I-LABEL: ashr_8bytes:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    lbu a3, 0(a0)
+; RV64I-NEXT:    lbu a4, 1(a0)
+; RV64I-NEXT:    lbu a5, 2(a0)
+; RV64I-NEXT:    lbu a6, 3(a0)
+; RV64I-NEXT:    lbu a7, 4(a0)
+; RV64I-NEXT:    lbu t0, 5(a0)
+; RV64I-NEXT:    lbu t1, 6(a0)
+; RV64I-NEXT:    lbu a0, 7(a0)
+; RV64I-NEXT:    slli a4, a4, 8
+; RV64I-NEXT:    slli a6, a6, 8
+; RV64I-NEXT:    or a3, a4, a3
+; RV64I-NEXT:    or a4, a6, a5
+; RV64I-NEXT:    lbu a5, 0(a1)
+; RV64I-NEXT:    lbu a6, 1(a1)
+; RV64I-NEXT:    lbu t2, 2(a1)
+; RV64I-NEXT:    lbu t3, 3(a1)
+; RV64I-NEXT:    slli t0, t0, 8
+; RV64I-NEXT:    slli a0, a0, 8
+; RV64I-NEXT:    slli a6, a6, 8
+; RV64I-NEXT:    or a7, t0, a7
+; RV64I-NEXT:    or a0, a0, t1
+; RV64I-NEXT:    or a5, a6, a5
+; RV64I-NEXT:    lbu a6, 4(a1)
+; RV64I-NEXT:    lbu t0, 5(a1)
+; RV64I-NEXT:    lbu t1, 6(a1)
+; RV64I-NEXT:    lbu a1, 7(a1)
+; RV64I-NEXT:    slli t3, t3, 8
+; RV64I-NEXT:    or t2, t3, t2
+; RV64I-NEXT:    slli t0, t0, 8
+; RV64I-NEXT:    or a6, t0, a6
+; RV64I-NEXT:    slli a1, a1, 8
+; RV64I-NEXT:    or a1, a1, t1
+; RV64I-NEXT:    slli a4, a4, 16
+; RV64I-NEXT:    or a3, a4, a3
+; RV64I-NEXT:    lui a4, 16
+; RV64I-NEXT:    addi a4, a4, -1
+; RV64I-NEXT:    slli a0, a0, 16
+; RV64I-NEXT:    slli t2, t2, 16
+; RV64I-NEXT:    slli a1, a1, 16
+; RV64I-NEXT:    or a0, a0, a7
+; RV64I-NEXT:    or a5, t2, a5
+; RV64I-NEXT:    or a1, a1, a6
+; RV64I-NEXT:    slli a0, a0, 32
+; RV64I-NEXT:    slli a1, a1, 32
+; RV64I-NEXT:    or a0, a0, a3
+; RV64I-NEXT:    or a1, a1, a5
+; RV64I-NEXT:    slli a1, a1, 3
+; RV64I-NEXT:    sra a0, a0, a1
+; RV64I-NEXT:    srli a1, a0, 32
+; RV64I-NEXT:    srliw a3, a0, 16
+; RV64I-NEXT:    and a5, a0, a4
+; RV64I-NEXT:    srliw a6, a0, 24
+; RV64I-NEXT:    srli a7, a0, 48
+; RV64I-NEXT:    srli t0, a0, 56
+; RV64I-NEXT:    srli a5, a5, 8
+; RV64I-NEXT:    and a4, a1, a4
+; RV64I-NEXT:    sb a0, 0(a2)
+; RV64I-NEXT:    sb a5, 1(a2)
+; RV64I-NEXT:    sb a3, 2(a2)
+; RV64I-NEXT:    sb a6, 3(a2)
+; RV64I-NEXT:    srli a4, a4, 8
+; RV64I-NEXT:    sb a1, 4(a2)
+; RV64I-NEXT:    sb a4, 5(a2)
+; RV64I-NEXT:    sb a7, 6(a2)
+; RV64I-NEXT:    sb t0, 7(a2)
+; RV64I-NEXT:    ret
+;
+; RV32I-LABEL: ashr_8bytes:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    lbu a3, 0(a0)
+; RV32I-NEXT:    lbu a4, 1(a0)
+; RV32I-NEXT:    lbu a5, 2(a0)
+; RV32I-NEXT:    lbu a6, 3(a0)
+; RV32I-NEXT:    lbu a7, 4(a0)
+; RV32I-NEXT:    lbu t0, 5(a0)
+; RV32I-NEXT:    lbu t1, 6(a0)
+; RV32I-NEXT:    lbu a0, 7(a0)
+; RV32I-NEXT:    slli a4, a4, 8
+; RV32I-NEXT:    slli a6, a6, 8
+; RV32I-NEXT:    slli t0, t0, 8
+; RV32I-NEXT:    or a3, a4, a3
+; RV32I-NEXT:    or a4, a6, a5
+; RV32I-NEXT:    or a5, t0, a7
+; RV32I-NEXT:    lbu a6, 0(a1)
+; RV32I-NEXT:    lbu a7, 1(a1)
+; RV32I-NEXT:    lbu t0, 2(a1)
+; RV32I-NEXT:    lbu a1, 3(a1)
+; RV32I-NEXT:    slli a0, a0, 8
+; RV32I-NEXT:    or t1, a0, t1
+; RV32I-NEXT:    slli a7, a7, 8
+; RV32I-NEXT:    or a6, a7, a6
+; RV32I-NEXT:    slli a1, a1, 8
+; RV32I-NEXT:    or a1, a1, t0
+; RV32I-NEXT:    slli a0, a4, 16
+; RV32I-NEXT:    or a0, a0, a3
+; RV32I-NEXT:    slli a3, t1, 16
+; RV32I-NEXT:    slli a1, a1, 16
+; RV32I-NEXT:    or a1, a1, a6
+; RV32I-NEXT:    slli a1, a1, 3
+; RV32I-NEXT:    li a4, 32
+; RV32I-NEXT:    or a3, a3, a5
+; RV32I-NEXT:    bltu a1, a4, .LBB5_2
+; RV32I-NEXT:  # %bb.1:
+; RV32I-NEXT:    sra a5, a3, a1
+; RV32I-NEXT:    bnez a1, .LBB5_3
+; RV32I-NEXT:    j .LBB5_4
+; RV32I-NEXT:  .LBB5_2:
+; RV32I-NEXT:    srl a5, a0, a1
+; RV32I-NEXT:    neg a6, a1
+; RV32I-NEXT:    sll a6, a3, a6
+; RV32I-NEXT:    or a5, a5, a6
+; RV32I-NEXT:    beqz a1, .LBB5_4
+; RV32I-NEXT:  .LBB5_3:
+; RV32I-NEXT:    mv a0, a5
+; RV32I-NEXT:  .LBB5_4:
+; RV32I-NEXT:    bltu a1, a4, .LBB5_6
+; RV32I-NEXT:  # %bb.5:
+; RV32I-NEXT:    srai a1, a3, 31
+; RV32I-NEXT:    j .LBB5_7
+; RV32I-NEXT:  .LBB5_6:
+; RV32I-NEXT:    sra a1, a3, a1
+; RV32I-NEXT:  .LBB5_7:
+; RV32I-NEXT:    srli a3, a0, 16
+; RV32I-NEXT:    lui a4, 16
+; RV32I-NEXT:    srli a5, a0, 24
+; RV32I-NEXT:    srli a6, a1, 16
+; RV32I-NEXT:    srli a7, a1, 24
+; RV32I-NEXT:    addi a4, a4, -1
+; RV32I-NEXT:    and t0, a0, a4
+; RV32I-NEXT:    and a4, a1, a4
+; RV32I-NEXT:    srli t0, t0, 8
+; RV32I-NEXT:    srli a4, a4, 8
+; RV32I-NEXT:    sb a0, 0(a2)
+; RV32I-NEXT:    sb t0, 1(a2)
+; RV32I-NEXT:    sb a3, 2(a2)
+; RV32I-NEXT:    sb a5, 3(a2)
+; RV32I-NEXT:    sb a1, 4(a2)
+; RV32I-NEXT:    sb a4, 5(a2)
+; RV32I-NEXT:    sb a6, 6(a2)
+; RV32I-NEXT:    sb a7, 7(a2)
+; RV32I-NEXT:    ret
+  %src = load i64, ptr %src.ptr, align 1
+  %byteOff = load i64, ptr %byteOff.ptr, align 1
+  %bitOff = shl i64 %byteOff, 3
+  %res = ashr i64 %src, %bitOff
+  store i64 %res, ptr %dst, align 1
+  ret void
+}
+
+define void @lshr_16bytes(ptr %src.ptr, ptr %byteOff.ptr, ptr %dst) nounwind {
+; RV64I-LABEL: lshr_16bytes:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd s0, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    lbu a3, 0(a0)
+; RV64I-NEXT:    lbu a4, 1(a0)
+; RV64I-NEXT:    lbu a5, 2(a0)
+; RV64I-NEXT:    lbu a6, 3(a0)
+; RV64I-NEXT:    lbu a7, 4(a0)
+; RV64I-NEXT:    lbu t0, 5(a0)
+; RV64I-NEXT:    lbu t1, 6(a0)
+; RV64I-NEXT:    lbu t2, 7(a0)
+; RV64I-NEXT:    lbu t3, 8(a0)
+; RV64I-NEXT:    lbu t4, 9(a0)
+; RV64I-NEXT:    lbu t5, 10(a0)
+; RV64I-NEXT:    lbu t6, 11(a0)
+; RV64I-NEXT:    slli a4, a4, 8
+; RV64I-NEXT:    slli a6, a6, 8
+; RV64I-NEXT:    or a3, a4, a3
+; RV64I-NEXT:    or a4, a6, a5
+; RV64I-NEXT:    lbu a5, 12(a0)
+; RV64I-NEXT:    lbu a6, 13(a0)
+; RV64I-NEXT:    lbu s0, 14(a0)
+; RV64I-NEXT:    lbu a0, 15(a0)
+; RV64I-NEXT:    slli t0, t0, 8
+; RV64I-NEXT:    slli t2, t2, 8
+; RV64I-NEXT:    slli t4, t4, 8
+; RV64I-NEXT:    slli t6, t6, 8
+; RV64I-NEXT:    or a7, t0, a7
+; RV64I-NEXT:    or t0, t2, t1
+; RV64I-NEXT:    or t1, t4, t3
+; RV64I-NEXT:    or t2, t6, t5
+; RV64I-NEXT:    lbu t3, 0(a1)
+; RV64I-NEXT:    lbu t4, 1(a1)
+; RV64I-NEXT:    lbu t5, 2(a1)
+; RV64I-NEXT:    lbu t6, 3(a1)
+; RV64I-NEXT:    slli a6, a6, 8
+; RV64I-NEXT:    slli a0, a0, 8
+; RV64I-NEXT:    slli t4, t4, 8
+; RV64I-NEXT:    or a5, a6, a5
+; RV64I-NEXT:    or a0, a0, s0
+; RV64I-NEXT:    or a6, t4, t3
+; RV64I-NEXT:    lbu t3, 4(a1)
+; RV64I-NEXT:    lbu t4, 5(a1)
+; RV64I-NEXT:    lbu s0, 6(a1)
+; RV64I-NEXT:    lbu a1, 7(a1)
+; RV64I-NEXT:    slli t6, t6, 8
+; RV64I-NEXT:    or t5, t6, t5
+; RV64I-NEXT:    slli t4, t4, 8
+; RV64I-NEXT:    or t3, t4, t3
+; RV64I-NEXT:    slli a1, a1, 8
+; RV64I-NEXT:    or a1, a1, s0
+; RV64I-NEXT:    slli a4, a4, 16
+; RV64I-NEXT:    or a3, a4, a3
+; RV64I-NEXT:    slli t0, t0, 16
+; RV64I-NEXT:    or a4, t0, a7
+; RV64I-NEXT:    slli t2, t2, 16
+; RV64I-NEXT:    or a7, t2, t1
+; RV64I-NEXT:    slli a0, a0, 16
+; RV64I-NEXT:    or a0, a0, a5
+; RV64I-NEXT:    slli t5, t5, 16
+; RV64I-NEXT:    or a5, t5, a6
+; RV64I-NEXT:    slli a1, a1, 16
+; RV64I-NEXT:    or a1, a1, t3
+; RV64I-NEXT:    slli a4, a4, 32
+; RV64I-NEXT:    slli a6, a0, 32
+; RV64I-NEXT:    slli a1, a1, 32
+; RV64I-NEXT:    or a0, a4, a3
+; RV64I-NEXT:    or a1, a1, a5
+; RV64I-NEXT:    slli a1, a1, 3
+; RV64I-NEXT:    li a4, 64
+; RV64I-NEXT:    or a3, a6, a7
+; RV64I-NEXT:    bltu a1, a4, .LBB6_2
+; RV64I-NEXT:  # %bb.1:
+; RV64I-NEXT:    subw a5, a1, a4
+; RV64I-NEXT:    srl a5, a3, a5
+; RV64I-NEXT:    bnez a1, .LBB6_3
+; RV64I-NEXT:    j .LBB6_4
+; RV64I-NEXT:  .LBB6_2:
+; RV64I-NEXT:    srl a5, a0, a1
+; RV64I-NEXT:    negw a6, a1
+; RV64I-NEXT:    sll a6, a3, a6
+; RV64I-NEXT:    or a5, a5, a6
+; RV64I-NEXT:    beqz a1, .LBB6_4
+; RV64I-NEXT:  .LBB6_3:
+; RV64I-NEXT:    mv a0, a5
+; RV64I-NEXT:  .LBB6_4:
+; RV64I-NEXT:    bltu a1, a4, .LBB6_6
+; RV64I-NEXT:  # %bb.5:
+; RV64I-NEXT:    li a1, 0
+; RV64I-NEXT:    j .LBB6_7
+; RV64I-NEXT:  .LBB6_6:
+; RV64I-NEXT:    srl a1, a3, a1
+; RV64I-NEXT:  .LBB6_7:
+; RV64I-NEXT:    srli a3, a0, 32
+; RV64I-NEXT:    srliw a4, a0, 16
+; RV64I-NEXT:    lui a5, 16
+; RV64I-NEXT:    srliw a6, a0, 24
+; RV64I-NEXT:    srli a7, a0, 48
+; RV64I-NEXT:    srli t0, a0, 56
+; RV64I-NEXT:    srli t1, a1, 32
+; RV64I-NEXT:    srliw t2, a1, 16
+; RV64I-NEXT:    srliw t3, a1, 24
+; RV64I-NEXT:    srli t4, a1, 48
+; RV64I-NEXT:    srli t5, a1, 56
+; RV64I-NEXT:    addi a5, a5, -1
+; RV64I-NEXT:    and t6, a0, a5
+; RV64I-NEXT:    srli t6, t6, 8
+; RV64I-NEXT:    sb a0, 0(a2)
+; RV64I-NEXT:    sb t6, 1(a2)
+; RV64I-NEXT:    sb a4, 2(a2)
+; RV64I-NEXT:    sb a6, 3(a2)
+; RV64I-NEXT:    and a0, a3, a5
+; RV64I-NEXT:    srli a0, a0, 8
+; RV64I-NEXT:    sb a3, 4(a2)
+; RV64I-NEXT:    sb a0, 5(a2)
+; RV64I-NEXT:    sb a7, 6(a2)
+; RV64I-NEXT:    sb t0, 7(a2)
+; RV64I-NEXT:    and a0, a1, a5
+; RV64I-NEXT:    and a3, t1, a5
+; RV64I-NEXT:    srli a0, a0, 8
+; RV64I-NEXT:    srli a3, a3, 8
+; RV64I-NEXT:    sb a1, 8(a2)
+; RV64I-NEXT:    sb a0, 9(a2)
+; RV64I-NEXT:    sb t2, 10(a2)
+; RV64I-NEXT:    sb t3, 11(a2)
+; RV64I-NEXT:    sb t1, 12(a2)
+; RV64I-NEXT:    sb a3, 13(a2)
+; RV64I-NEXT:    sb t4, 14(a2)
+; RV64I-NEXT:    sb t5, 15(a2)
+; RV64I-NEXT:    ld s0, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV32I-LABEL: lshr_16bytes:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    addi sp, sp, -16
+; RV32I-NEXT:    sw s0, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s1, 8(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s2, 4(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    lbu a3, 0(a0)
+; RV32I-NEXT:    lbu a5, 1(a0)
+; RV32I-NEXT:    lbu a6, 2(a0)
+; RV32I-NEXT:    lbu a7, 3(a0)
+; RV32I-NEXT:    lbu a4, 4(a0)
+; RV32I-NEXT:    lbu t0, 5(a0)
+; RV32I-NEXT:    lbu t1, 6(a0)
+; RV32I-NEXT:    lbu t2, 7(a0)
+; RV32I-NEXT:    lbu t4, 8(a0)
+; RV32I-NEXT:    lbu t5, 9(a0)
+; RV32I-NEXT:    lbu t6, 10(a0)
+; RV32I-NEXT:    lbu s0, 11(a0)
+; RV32I-NEXT:    slli a7, a7, 8
+; RV32I-NEXT:    slli t2, t2, 8
+; RV32I-NEXT:    or t3, a7, a6
+; RV32I-NEXT:    or t1, t2, t1
+; RV32I-NEXT:    lbu a6, 12(a0)
+; RV32I-NEXT:    lbu a7, 13(a0)
+; RV32I-NEXT:    lbu t2, 14(a0)
+; RV32I-NEXT:    lbu a0, 15(a0)
+; RV32I-NEXT:    slli t5, t5, 8
+; RV32I-NEXT:    slli s0, s0, 8
+; RV32I-NEXT:    slli a7, a7, 8
+; RV32I-NEXT:    or t4, t5, t4
+; RV32I-NEXT:    or t5, s0, t6
+; RV32I-NEXT:    or t6, a7, a6
+; RV32I-NEXT:    lbu a6, 0(a1)
+; RV32I-NEXT:    lbu a7, 1(a1)
+; RV32I-NEXT:    lbu s0, 2(a1)
+; RV32I-NEXT:    lbu a1, 3(a1)
+; RV32I-NEXT:    slli a0, a0, 8
+; RV32I-NEXT:    or t2, a0, t2
+; RV32I-NEXT:    slli a7, a7, 8
+; RV32I-NEXT:    or s1, a7, a6
+; RV32I-NEXT:    slli a1, a1, 8
+; RV32I-NEXT:    or s0, a1, s0
+; RV32I-NEXT:    li a7, 32
+; RV32I-NEXT:    slli a1, a5, 8
+; RV32I-NEXT:    slli a0, t0, 8
+; RV32I-NEXT:    slli t5, t5, 16
+; RV32I-NEXT:    slli t2, t2, 16
+; RV32I-NEXT:    slli s0, s0, 16
+; RV32I-NEXT:    or a6, t5, t4
+; RV32I-NEXT:    or t0, t2, t6
+; RV32I-NEXT:    or a5, s0, s1
+; RV32I-NEXT:    slli a5, a5, 3
+; RV32I-NEXT:    srl t2, a6, a5
+; RV32I-NEXT:    neg t5, a5
+; RV32I-NEXT:    sll t4, t0, t5
+; RV32I-NEXT:    bltu a5, a7, .LBB6_2
+; RV32I-NEXT:  # %bb.1:
+; RV32I-NEXT:    srl t6, t0, a5
+; RV32I-NEXT:    j .LBB6_3
+; RV32I-NEXT:  .LBB6_2:
+; RV32I-NEXT:    or t6, t2, t4
+; RV32I-NEXT:  .LBB6_3:
+; RV32I-NEXT:    or a1, a1, a3
+; RV32I-NEXT:    slli t3, t3, 16
+; RV32I-NEXT:    or a3, a0, a4
+; RV32I-NEXT:    slli t1, t1, 16
+; RV32I-NEXT:    mv a0, a6
+; RV32I-NEXT:    beqz a5, .LBB6_5
+; RV32I-NEXT:  # %bb.4:
+; RV32I-NEXT:    mv a0, t6
+; RV32I-NEXT:  .LBB6_5:
+; RV32I-NEXT:    or a4, t3, a1
+; RV32I-NEXT:    or a3, t1, a3
+; RV32I-NEXT:    bltu a5, a7, .LBB6_7
+; RV32I-NEXT:  # %bb.6:
+; RV32I-NEXT:    li a1, 0
+; RV32I-NEXT:    srl t6, a3, a5
+; RV32I-NEXT:    j .LBB6_8
+; RV32I-NEXT:  .LBB6_7:
+; RV32I-NEXT:    srl a1, t0, a5
+; RV32I-NEXT:    srl t1, a4, a5
+; RV32I-NEXT:    sll t3, a3, t5
+; RV32I-NEXT:    or t6, t1, t3
+; RV32I-NEXT:  .LBB6_8:
+; RV32I-NEXT:    li t1, 64
+; RV32I-NEXT:    mv t3, a4
+; RV32I-NEXT:    beqz a5, .LBB6_10
+; RV32I-NEXT:  # %bb.9:
+; RV32I-NEXT:    mv t3, t6
+; RV32I-NEXT:  .LBB6_10:
+; RV32I-NEXT:    sub s0, t1, a5
+; RV32I-NEXT:    bltu a5, a7, .LBB6_13
+; RV32I-NEXT:  # %bb.11:
+; RV32I-NEXT:    li t6, 0
+; RV32I-NEXT:    bgeu s0, a7, .LBB6_14
+; RV32I-NEXT:  .LBB6_12:
+; RV32I-NEXT:    sll t5, a6, t5
+; RV32I-NEXT:    neg s1, s0
+; RV32I-NEXT:    srl s1, a6, s1
+; RV32I-NEXT:    or s2, s1, t4
+; RV32I-NEXT:    j .LBB6_15
+; RV32I-NEXT:  .LBB6_13:
+; RV32I-NEXT:    srl t6, a3, a5
+; RV32I-NEXT:    bltu s0, a7, .LBB6_12
+; RV32I-NEXT:  .LBB6_14:
+; RV32I-NEXT:    li t5, 0
+; RV32I-NEXT:    sll s2, a6, s0
+; RV32I-NEXT:  .LBB6_15:
+; RV32I-NEXT:    addi s1, a5, -64
+; RV32I-NEXT:    mv t4, t0
+; RV32I-NEXT:    beqz s0, .LBB6_17
+; RV32I-NEXT:  # %bb.16:
+; RV32I-NEXT:    mv t4, s2
+; RV32I-NEXT:  .LBB6_17:
+; RV32I-NEXT:    bltu s1, a7, .LBB6_19
+; RV32I-NEXT:  # %bb.18:
+; RV32I-NEXT:    srl t2, t0, s1
+; RV32I-NEXT:    bnez s1, .LBB6_20
+; RV32I-NEXT:    j .LBB6_21
+; RV32I-NEXT:  .LBB6_19:
+; RV32I-NEXT:    neg s0, s1
+; RV32I-NEXT:    sll s0, t0, s0
+; RV32I-NEXT:    or t2, t2, s0
+; RV32I-NEXT:    beqz s1, .LBB6_21
+; RV32I-NEXT:  .LBB6_20:
+; RV32I-NEXT:    mv a6, t2
+; RV32I-NEXT:  .LBB6_21:
+; RV32I-NEXT:    bltu s1, a7, .LBB6_23
+; RV32I-NEXT:  # %bb.22:
+; RV32I-NEXT:    li a7, 0
+; RV32I-NEXT:    bltu a5, t1, .LBB6_24
+; RV32I-NEXT:    j .LBB6_25
+; RV32I-NEXT:  .LBB6_23:
+; RV32I-NEXT:    srl a7, t0, a5
+; RV32I-NEXT:    bgeu a5, t1, .LBB6_25
+; RV32I-NEXT:  .LBB6_24:
+; RV32I-NEXT:    or a6, t3, t5
+; RV32I-NEXT:    or a7, t6, t4
+; RV32I-NEXT:  .LBB6_25:
+; RV32I-NEXT:    bnez a5, .LBB6_29
+; RV32I-NEXT:  # %bb.26:
+; RV32I-NEXT:    bltu a5, t1, .LBB6_28
+; RV32I-NEXT:  .LBB6_27:
+; RV32I-NEXT:    li a0, 0
+; RV32I-NEXT:    li a1, 0
+; RV32I-NEXT:  .LBB6_28:
+; RV32I-NEXT:    srli a5, a4, 16
+; RV32I-NEXT:    lui a6, 16
+; RV32I-NEXT:    srli a7, a4, 24
+; RV32I-NEXT:    srli t0, a3, 16
+; RV32I-NEXT:    srli t1, a3, 24
+; RV32I-NEXT:    srli t2, a0, 16
+; RV32I-NEXT:    srli t3, a0, 24
+; RV32I-NEXT:    srli t4, a1, 16
+; RV32I-NEXT:    srli t5, a1, 24
+; RV32I-NEXT:    addi a6, a6, -1
+; RV32I-NEXT:    and t6, a4, a6
+; RV32I-NEXT:    srli t6, t6, 8
+; RV32I-NEXT:    sb a4, 0(a2)
+; RV32I-NEXT:    sb t6, 1(a2)
+; RV32I-NEXT:    sb a5, 2(a2)
+; RV32I-NEXT:    sb a7, 3(a2)
+; RV32I-NEXT:    and a4, a3, a6
+; RV32I-NEXT:    srli a4, a4, 8
+; RV32I-NEXT:    sb a3, 4(a2)
+; RV32I-NEXT:    sb a4, 5(a2)
+; RV32I-NEXT:    sb t0, 6(a2)
+; RV32I-NEXT:    sb t1, 7(a2)
+; RV32I-NEXT:    and a3, a0, a6
+; RV32I-NEXT:    and a4, a1, a6
+; RV32I-NEXT:    srli a3, a3, 8
+; RV32I-NEXT:    srli a4, a4, 8
+; RV32I-NEXT:    sb a0, 8(a2)
+; RV32I-NEXT:    sb a3, 9(a2)
+; RV32I-NEXT:    sb t2, 10(a2)
+; RV32I-NEXT:    sb t3, 11(a2)
+; RV32I-NEXT:    sb a1, 12(a2)
+; RV32I-NEXT:    sb a4, 13(a2)
+; RV32I-NEXT:    sb t4, 14(a2)
+; RV32I-NEXT:    sb t5, 15(a2)
+; RV32I-NEXT:    lw s0, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s1, 8(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s2, 4(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    addi sp, sp, 16
+; RV32I-NEXT:    ret
+; RV32I-NEXT:  .LBB6_29:
+; RV32I-NEXT:    mv a4, a6
+; RV32I-NEXT:    mv a3, a7
+; RV32I-NEXT:    bgeu a5, t1, .LBB6_27
+; RV32I-NEXT:    j .LBB6_28
+  %src = load i128, ptr %src.ptr, align 1
+  %byteOff = load i128, ptr %byteOff.ptr, align 1
+  %bitOff = shl i128 %byteOff, 3
+  %res = lshr i128 %src, %bitOff
+  store i128 %res, ptr %dst, align 1
+  ret void
+}
+
+define void @lshr_16bytes_wordOff(ptr %src.ptr, ptr %wordOff.ptr, ptr %dst) nounwind {
+; RV64I-LABEL: lshr_16bytes_wordOff:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd s0, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    lbu a3, 0(a0)
+; RV64I-NEXT:    lbu a4, 1(a0)
+; RV64I-NEXT:    lbu a5, 2(a0)
+; RV64I-NEXT:    lbu a6, 3(a0)
+; RV64I-NEXT:    lbu a7, 4(a0)
+; RV64I-NEXT:    lbu t0, 5(a0)
+; RV64I-NEXT:    lbu t1, 6(a0)
+; RV64I-NEXT:    lbu t2, 7(a0)
+; RV64I-NEXT:    lbu t3, 8(a0)
+; RV64I-NEXT:    lbu t4, 9(a0)
+; RV64I-NEXT:    lbu t5, 10(a0)
+; RV64I-NEXT:    lbu t6, 11(a0)
+; RV64I-NEXT:    slli a4, a4, 8
+; RV64I-NEXT:    slli a6, a6, 8
+; RV64I-NEXT:    or a3, a4, a3
+; RV64I-NEXT:    or a4, a6, a5
+; RV64I-NEXT:    lbu a5, 12(a0)
+; RV64I-NEXT:    lbu a6, 13(a0)
+; RV64I-NEXT:    lbu s0, 14(a0)
+; RV64I-NEXT:    lbu a0, 15(a0)
+; RV64I-NEXT:    slli t0, t0, 8
+; RV64I-NEXT:    slli t2, t2, 8
+; RV64I-NEXT:    slli t4, t4, 8
+; RV64I-NEXT:    slli t6, t6, 8
+; RV64I-NEXT:    or a7, t0, a7
+; RV64I-NEXT:    or t0, t2, t1
+; RV64I-NEXT:    or t1, t4, t3
+; RV64I-NEXT:    or t2, t6, t5
+; RV64I-NEXT:    lbu t3, 0(a1)
+; RV64I-NEXT:    lbu t4, 1(a1)
+; RV64I-NEXT:    lbu t5, 2(a1)
+; RV64I-NEXT:    lbu t6, 3(a1)
+; RV64I-NEXT:    slli a6, a6, 8
+; RV64I-NEXT:    slli a0, a0, 8
+; RV64I-NEXT:    slli t4, t4, 8
+; RV64I-NEXT:    or a5, a6, a5
+; RV64I-NEXT:    or a0, a0, s0
+; RV64I-NEXT:    or a6, t4, t3
+; RV64I-NEXT:    lbu t3, 4(a1)
+; RV64I-NEXT:    lbu t4, 5(a1)
+; RV64I-NEXT:    lbu s0, 6(a1)
+; RV64I-NEXT:    lbu a1, 7(a1)
+; RV64I-NEXT:    slli t6, t6, 8
+; RV64I-NEXT:    or t5, t6, t5
+; RV64I-NEXT:    slli t4, t4, 8
+; RV64I-NEXT:    or t3, t4, t3
+; RV64I-NEXT:    slli a1, a1, 8
+; RV64I-NEXT:    or a1, a1, s0
+; RV64I-NEXT:    slli a4, a4, 16
+; RV64I-NEXT:    or a3, a4, a3
+; RV64I-NEXT:    slli t0, t0, 16
+; RV64I-NEXT:    or a4, t0, a7
+; RV64I-NEXT:    slli t2, t2, 16
+; RV64I-NEXT:    or a7, t2, t1
+; RV64I-NEXT:    slli a0, a0, 16
+; RV64I-NEXT:    or a0, a0, a5
+; RV64I-NEXT:    slli t5, t5, 16
+; RV64I-NEXT:    or a5, t5, a6
+; RV64I-NEXT:    slli a1, a1, 16
+; RV64I-NEXT:    or a1, a1, t3
+; RV64I-NEXT:    slli a4, a4, 32
+; RV64I-NEXT:    slli a6, a0, 32
+; RV64I-NEXT:    slli a1, a1, 32
+; RV64I-NEXT:    or a0, a4, a3
+; RV64I-NEXT:    or a1, a1, a5
+; RV64I-NEXT:    slli a1, a1, 5
+; RV64I-NEXT:    li a4, 64
+; RV64I-NEXT:    or a3, a6, a7
+; RV64I-NEXT:    bltu a1, a4, .LBB7_2
+; RV64I-NEXT:  # %bb.1:
+; RV64I-NEXT:    subw a5, a1, a4
+; RV64I-NEXT:    srl a5, a3, a5
+; RV64I-NEXT:    bnez a1, .LBB7_3
+; RV64I-NEXT:    j .LBB7_4
+; RV64I-NEXT:  .LBB7_2:
+; RV64I-NEXT:    srl a5, a0, a1
+; RV64I-NEXT:    negw a6, a1
+; RV64I-NEXT:    sll a6, a3, a6
+; RV64I-NEXT:    or a5, a5, a6
+; RV64I-NEXT:    beqz a1, .LBB7_4
+; RV64I-NEXT:  .LBB7_3:
+; RV64I-NEXT:    mv a0, a5
+; RV64I-NEXT:  .LBB7_4:
+; RV64I-NEXT:    bltu a1, a4, .LBB7_6
+; RV64I-NEXT:  # %bb.5:
+; RV64I-NEXT:    li a1, 0
+; RV64I-NEXT:    j .LBB7_7
+; RV64I-NEXT:  .LBB7_6:
+; RV64I-NEXT:    srl a1, a3, a1
+; RV64I-NEXT:  .LBB7_7:
+; RV64I-NEXT:    srli a3, a0, 32
+; RV64I-NEXT:    srliw a4, a0, 16
+; RV64I-NEXT:    lui a5, 16
+; RV64I-NEXT:    srliw a6, a0, 24
+; RV64I-NEXT:    srli a7, a0, 48
+; RV64I-NEXT:    srli t0, a0, 56
+; RV64I-NEXT:    srli t1, a1, 32
+; RV64I-NEXT:    srliw t2, a1, 16
+; RV64I-NEXT:    srliw t3, a1, 24
+; RV64I-NEXT:    srli t4, a1, 48
+; RV64I-NEXT:    srli t5, a1, 56
+; RV64I-NEXT:    addi a5, a5, -1
+; RV64I-NEXT:    and t6, a0, a5
+; RV64I-NEXT:    srli t6, t6, 8
+; RV64I-NEXT:    sb a0, 0(a2)
+; RV64I-NEXT:    sb t6, 1(a2)
+; RV64I-NEXT:    sb a4, 2(a2)
+; RV64I-NEXT:    sb a6, 3(a2)
+; RV64I-NEXT:    and a0, a3, a5
+; RV64I-NEXT:    srli a0, a0, 8
+; RV64I-NEXT:    sb a3, 4(a2)
+; RV64I-NEXT:    sb a0, 5(a2)
+; RV64I-NEXT:    sb a7, 6(a2)
+; RV64I-NEXT:    sb t0, 7(a2)
+; RV64I-NEXT:    and a0, a1, a5
+; RV64I-NEXT:    and a3, t1, a5
+; RV64I-NEXT:    srli a0, a0, 8
+; RV64I-NEXT:    srli a3, a3, 8
+; RV64I-NEXT:    sb a1, 8(a2)
+; RV64I-NEXT:    sb a0, 9(a2)
+; RV64I-NEXT:    sb t2, 10(a2)
+; RV64I-NEXT:    sb t3, 11(a2)
+; RV64I-NEXT:    sb t1, 12(a2)
+; RV64I-NEXT:    sb a3, 13(a2)
+; RV64I-NEXT:    sb t4, 14(a2)
+; RV64I-NEXT:    sb t5, 15(a2)
+; RV64I-NEXT:    ld s0, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV32I-LABEL: lshr_16bytes_wordOff:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    addi sp, sp, -16
+; RV32I-NEXT:    sw s0, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s1, 8(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s2, 4(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    lbu a3, 0(a0)
+; RV32I-NEXT:    lbu a5, 1(a0)
+; RV32I-NEXT:    lbu a6, 2(a0)
+; RV32I-NEXT:    lbu a7, 3(a0)
+; RV32I-NEXT:    lbu a4, 4(a0)
+; RV32I-NEXT:    lbu t0, 5(a0)
+; RV32I-NEXT:    lbu t1, 6(a0)
+; RV32I-NEXT:    lbu t2, 7(a0)
+; RV32I-NEXT:    lbu t4, 8(a0)
+; RV32I-NEXT:    lbu t5, 9(a0)
+; RV32I-NEXT:    lbu t6, 10(a0)
+; RV32I-NEXT:    lbu s0, 11(a0)
+; RV32I-NEXT:    slli a7, a7, 8
+; RV32I-NEXT:    slli t2, t2, 8
+; RV32I-NEXT:    or t3, a7, a6
+; RV32I-NEXT:    or t1, t2, t1
+; RV32I-NEXT:    lbu a6, 12(a0)
+; RV32I-NEXT:    lbu a7, 13(a0)
+; RV32I-NEXT:    lbu t2, 14(a0)
+; RV32I-NEXT:    lbu a0, 15(a0)
+; RV32I-NEXT:    slli t5, t5, 8
+; RV32I-NEXT:    slli s0, s0, 8
+; RV32I-NEXT:    slli a7, a7, 8
+; RV32I-NEXT:    or t4, t5, t4
+; RV32I-NEXT:    or t5, s0, t6
+; RV32I-NEXT:    or t6, a7, a6
+; RV32I-NEXT:    lbu a6, 0(a1)
+; RV32I-NEXT:    lbu a7, 1(a1)
+; RV32I-NEXT:    lbu s0, 2(a1)
+; RV32I-NEXT:    lbu a1, 3(a1)
+; RV32I-NEXT:    slli a0, a0, 8
+; RV32I-NEXT:    or t2, a0, t2
+; RV32I-NEXT:    slli a7, a7, 8
+; RV32I-NEXT:    or s1, a7, a6
+; RV32I-NEXT:    slli a1, a1, 8
+; RV32I-NEXT:    or s0, a1, s0
+; RV32I-NEXT:    li a7, 32
+; RV32I-NEXT:    slli a1, a5, 8
+; RV32I-NEXT:    slli a0, t0, 8
+; RV32I-NEXT:    slli t5, t5, 16
+; RV32I-NEXT:    slli t2, t2, 16
+; RV32I-NEXT:    slli s0, s0, 16
+; RV32I-NEXT:    or a6, t5, t4
+; RV32I-NEXT:    or t0, t2, t6
+; RV32I-NEXT:    or a5, s0, s1
+; RV32I-NEXT:    slli a5, a5, 5
+; RV32I-NEXT:    srl t2, a6, a5
+; RV32I-NEXT:    neg t5, a5
+; RV32I-NEXT:    sll t4, t0, t5
+; RV32I-NEXT:    bltu a5, a7, .LBB7_2
+; RV32I-NEXT:  # %bb.1:
+; RV32I-NEXT:    srl t6, t0, a5
+; RV32I-NEXT:    j .LBB7_3
+; RV32I-NEXT:  .LBB7_2:
+; RV32I-NEXT:    or t6, t2, t4
+; RV32I-NEXT:  .LBB7_3:
+; RV32I-NEXT:    or a1, a1, a3
+; RV32I-NEXT:    slli t3, t3, 16
+; RV32I-NEXT:    or a3, a0, a4
+; RV32I-NEXT:    slli t1, t1, 16
+; RV32I-NEXT:    mv a0, a6
+; RV32I-NEXT:    beqz a5, .LBB7_5
+; RV32I-NEXT:  # %bb.4:
+; RV32I-NEXT:    mv a0, t6
+; RV32I-NEXT:  .LBB7_5:
+; RV32I-NEXT:    or a4, t3, a1
+; RV32I-NEXT:    or a3, t1, a3
+; RV32I-NEXT:    bltu a5, a7, .LBB7_7
+; RV32I-NEXT:  # %bb.6:
+; RV32I-NEXT:    li a1, 0
+; RV32I-NEXT:    srl t6, a3, a5
+; RV32I-NEXT:    j .LBB7_8
+; RV32I-NEXT:  .LBB7_7:
+; RV32I-NEXT:    srl a1, t0, a5
+; RV32I-NEXT:    srl t1, a4, a5
+; RV32I-NEXT:    sll t3, a3, t5
+; RV32I-NEXT:    or t6, t1, t3
+; RV32I-NEXT:  .LBB7_8:
+; RV32I-NEXT:    li t1, 64
+; RV32I-NEXT:    mv t3, a4
+; RV32I-NEXT:    beqz a5, .LBB7_10
+; RV32I-NEXT:  # %bb.9:
+; RV32I-NEXT:    mv t3, t6
+; RV32I-NEXT:  .LBB7_10:
+; RV32I-NEXT:    sub s0, t1, a5
+; RV32I-NEXT:    bltu a5, a7, .LBB7_13
+; RV32I-NEXT:  # %bb.11:
+; RV32I-NEXT:    li t6, 0
+; RV32I-NEXT:    bgeu s0, a7, .LBB7_14
+; RV32I-NEXT:  .LBB7_12:
+; RV32I-NEXT:    sll t5, a6, t5
+; RV32I-NEXT:    neg s1, s0
+; RV32I-NEXT:    srl s1, a6, s1
+; RV32I-NEXT:    or s2, s1, t4
+; RV32I-NEXT:    j .LBB7_15
+; RV32I-NEXT:  .LBB7_13:
+; RV32I-NEXT:    srl t6, a3, a5
+; RV32I-NEXT:    bltu s0, a7, .LBB7_12
+; RV32I-NEXT:  .LBB7_14:
+; RV32I-NEXT:    li t5, 0
+; RV32I-NEXT:    sll s2, a6, s0
+; RV32I-NEXT:  .LBB7_15:
+; RV32I-NEXT:    addi s1, a5, -64
+; RV32I-NEXT:    mv t4, t0
+; RV32I-NEXT:    beqz s0, .LBB7_17
+; RV32I-NEXT:  # %bb.16:
+; RV32I-NEXT:    mv t4, s2
+; RV32I-NEXT:  .LBB7_17:
+; RV32I-NEXT:    bltu s1, a7, .LBB7_19
+; RV32I-NEXT:  # %bb.18:
+; RV32I-NEXT:    srl t2, t0, s1
+; RV32I-NEXT:    bnez s1, .LBB7_20
+; RV32I-NEXT:    j .LBB7_21
+; RV32I-NEXT:  .LBB7_19:
+; RV32I-NEXT:    neg s0, s1
+; RV32I-NEXT:    sll s0, t0, s0
+; RV32I-NEXT:    or t2, t2, s0
+; RV32I-NEXT:    beqz s1, .LBB7_21
+; RV32I-NEXT:  .LBB7_20:
+; RV32I-NEXT:    mv a6, t2
+; RV32I-NEXT:  .LBB7_21:
+; RV32I-NEXT:    bltu s1, a7, .LBB7_23
+; RV32I-NEXT:  # %bb.22:
+; RV32I-NEXT:    li a7, 0
+; RV32I-NEXT:    bltu a5, t1, .LBB7_24
+; RV32I-NEXT:    j .LBB7_25
+; RV32I-NEXT:  .LBB7_23:
+; RV32I-NEXT:    srl a7, t0, a5
+; RV32I-NEXT:    bgeu a5, t1, .LBB7_25
+; RV32I-NEXT:  .LBB7_24:
+; RV32I-NEXT:    or a6, t3, t5
+; RV32I-NEXT:    or a7, t6, t4
+; RV32I-NEXT:  .LBB7_25:
+; RV32I-NEXT:    bnez a5, .LBB7_29
+; RV32I-NEXT:  # %bb.26:
+; RV32I-NEXT:    bltu a5, t1, .LBB7_28
+; RV32I-NEXT:  .LBB7_27:
+; RV32I-NEXT:    li a0, 0
+; RV32I-NEXT:    li a1, 0
+; RV32I-NEXT:  .LBB7_28:
+; RV32I-NEXT:    srli a5, a4, 16
+; RV32I-NEXT:    lui a6, 16
+; RV32I-NEXT:    srli a7, a4, 24
+; RV32I-NEXT:    srli t0, a3, 16
+; RV32I-NEXT:    srli t1, a3, 24
+; RV32I-NEXT:    srli t2, a0, 16
+; RV32I-NEXT:    srli t3, a0, 24
+; RV32I-NEXT:    srli t4, a1, 16
+; RV32I-NEXT:    srli t5, a1, 24
+; RV32I-NEXT:    addi a6, a6, -1
+; RV32I-NEXT:    and t6, a4, a6
+; RV32I-NEXT:    srli t6, t6, 8
+; RV32I-NEXT:    sb a4, 0(a2)
+; RV32I-NEXT:    sb t6, 1(a2)
+; RV32I-NEXT:    sb a5, 2(a2)
+; RV32I-NEXT:    sb a7, 3(a2)
+; RV32I-NEXT:    and a4, a3, a6
+; RV32I-NEXT:    srli a4, a4, 8
+; RV32I-NEXT:    sb a3, 4(a2)
+; RV32I-NEXT:    sb a4, 5(a2)
+; RV32I-NEXT:    sb t0, 6(a2)
+; RV32I-NEXT:    sb t1, 7(a2)
+; RV32I-NEXT:    and a3, a0, a6
+; RV32I-NEXT:    and a4, a1, a6
+; RV32I-NEXT:    srli a3, a3, 8
+; RV32I-NEXT:    srli a4, a4, 8
+; RV32I-NEXT:    sb a0, 8(a2)
+; RV32I-NEXT:    sb a3, 9(a2)
+; RV32I-NEXT:    sb t2, 10(a2)
+; RV32I-NEXT:    sb t3, 11(a2)
+; RV32I-NEXT:    sb a1, 12(a2)
+; RV32I-NEXT:    sb a4, 13(a2)
+; RV32I-NEXT:    sb t4, 14(a2)
+; RV32I-NEXT:    sb t5, 15(a2)
+; RV32I-NEXT:    lw s0, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s1, 8(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s2, 4(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    addi sp, sp, 16
+; RV32I-NEXT:    ret
+; RV32I-NEXT:  .LBB7_29:
+; RV32I-NEXT:    mv a4, a6
+; RV32I-NEXT:    mv a3, a7
+; RV32I-NEXT:    bgeu a5, t1, .LBB7_27
+; RV32I-NEXT:    j .LBB7_28
+  %src = load i128, ptr %src.ptr, align 1
+  %wordOff = load i128, ptr %wordOff.ptr, align 1
+  %bitOff = shl i128 %wordOff, 5
+  %res = lshr i128 %src, %bitOff
+  store i128 %res, ptr %dst, align 1
+  ret void
+}
+
+define void @shl_16bytes(ptr %src.ptr, ptr %byteOff.ptr, ptr %dst) nounwind {
+; RV64I-LABEL: shl_16bytes:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd s0, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    lbu a3, 0(a0)
+; RV64I-NEXT:    lbu a4, 1(a0)
+; RV64I-NEXT:    lbu a5, 2(a0)
+; RV64I-NEXT:    lbu a6, 3(a0)
+; RV64I-NEXT:    lbu a7, 4(a0)
+; RV64I-NEXT:    lbu t0, 5(a0)
+; RV64I-NEXT:    lbu t1, 6(a0)
+; RV64I-NEXT:    lbu t2, 7(a0)
+; RV64I-NEXT:    lbu t3, 8(a0)
+; RV64I-NEXT:    lbu t4, 9(a0)
+; RV64I-NEXT:    lbu t5, 10(a0)
+; RV64I-NEXT:    lbu t6, 11(a0)
+; RV64I-NEXT:    slli a4, a4, 8
+; RV64I-NEXT:    slli a6, a6, 8
+; RV64I-NEXT:    or a3, a4, a3
+; RV64I-NEXT:    or a4, a6, a5
+; RV64I-NEXT:    lbu a5, 12(a0)
+; RV64I-NEXT:    lbu a6, 13(a0)
+; RV64I-NEXT:    lbu s0, 14(a0)
+; RV64I-NEXT:    lbu a0, 15(a0)
+; RV64I-NEXT:    slli t0, t0, 8
+; RV64I-NEXT:    slli t2, t2, 8
+; RV64I-NEXT:    slli t4, t4, 8
+; RV64I-NEXT:    slli t6, t6, 8
+; RV64I-NEXT:    or a7, t0, a7
+; RV64I-NEXT:    or t0, t2, t1
+; RV64I-NEXT:    or t1, t4, t3
+; RV64I-NEXT:    or t2, t6, t5
+; RV64I-NEXT:    lbu t3, 0(a1)
+; RV64I-NEXT:    lbu t4, 1(a1)
+; RV64I-NEXT:    lbu t5, 2(a1)
+; RV64I-NEXT:    lbu t6, 3(a1)
+; RV64I-NEXT:    slli a6, a6, 8
+; RV64I-NEXT:    slli a0, a0, 8
+; RV64I-NEXT:    slli t4, t4, 8
+; RV64I-NEXT:    or a5, a6, a5
+; RV64I-NEXT:    or a0, a0, s0
+; RV64I-NEXT:    or a6, t4, t3
+; RV64I-NEXT:    lbu t3, 4(a1)
+; RV64I-NEXT:    lbu t4, 5(a1)
+; RV64I-NEXT:    lbu s0, 6(a1)
+; RV64I-NEXT:    lbu a1, 7(a1)
+; RV64I-NEXT:    slli t6, t6, 8
+; RV64I-NEXT:    or t5, t6, t5
+; RV64I-NEXT:    slli t4, t4, 8
+; RV64I-NEXT:    or t3, t4, t3
+; RV64I-NEXT:    slli a1, a1, 8
+; RV64I-NEXT:    or a1, a1, s0
+; RV64I-NEXT:    slli a4, a4, 16
+; RV64I-NEXT:    or a3, a4, a3
+; RV64I-NEXT:    slli t0, t0, 16
+; RV64I-NEXT:    or a4, t0, a7
+; RV64I-NEXT:    slli t2, t2, 16
+; RV64I-NEXT:    or a7, t2, t1
+; RV64I-NEXT:    slli a0, a0, 16
+; RV64I-NEXT:    or a0, a0, a5
+; RV64I-NEXT:    slli t5, t5, 16
+; RV64I-NEXT:    or a5, t5, a6
+; RV64I-NEXT:    slli a1, a1, 16
+; RV64I-NEXT:    or a1, a1, t3
+; RV64I-NEXT:    slli a4, a4, 32
+; RV64I-NEXT:    slli a0, a0, 32
+; RV64I-NEXT:    slli a1, a1, 32
+; RV64I-NEXT:    or a4, a4, a3
+; RV64I-NEXT:    or a1, a1, a5
+; RV64I-NEXT:    slli a3, a1, 3
+; RV64I-NEXT:    li a5, 64
+; RV64I-NEXT:    or a0, a0, a7
+; RV64I-NEXT:    bltu a3, a5, .LBB8_2
+; RV64I-NEXT:  # %bb.1:
+; RV64I-NEXT:    li a1, 0
+; RV64I-NEXT:    subw a5, a3, a5
+; RV64I-NEXT:    sll a4, a4, a5
+; RV64I-NEXT:    bnez a3, .LBB8_3
+; RV64I-NEXT:    j .LBB8_4
+; RV64I-NEXT:  .LBB8_2:
+; RV64I-NEXT:    sll a1, a4, a3
+; RV64I-NEXT:    negw a5, a3
+; RV64I-NEXT:    srl a4, a4, a5
+; RV64I-NEXT:    sll a5, a0, a3
+; RV64I-NEXT:    or a4, a4, a5
+; RV64I-NEXT:    beqz a3, .LBB8_4
+; RV64I-NEXT:  .LBB8_3:
+; RV64I-NEXT:    mv a0, a4
+; RV64I-NEXT:  .LBB8_4:
+; RV64I-NEXT:    srli a3, a1, 32
+; RV64I-NEXT:    srliw a4, a1, 16
+; RV64I-NEXT:    lui a5, 16
+; RV64I-NEXT:    srliw a6, a1, 24
+; RV64I-NEXT:    srli a7, a1, 48
+; RV64I-NEXT:    srli t0, a1, 56
+; RV64I-NEXT:    srli t1, a0, 32
+; RV64I-NEXT:    srliw t2, a0, 16
+; RV64I-NEXT:    srliw t3, a0, 24
+; RV64I-NEXT:    srli t4, a0, 48
+; RV64I-NEXT:    srli t5, a0, 56
+; RV64I-NEXT:    addi a5, a5, -1
+; RV64I-NEXT:    and t6, a1, a5
+; RV64I-NEXT:    srli t6, t6, 8
+; RV64I-NEXT:    sb a1, 0(a2)
+; RV64I-NEXT:    sb t6, 1(a2)
+; RV64I-NEXT:    sb a4, 2(a2)
+; RV64I-NEXT:    sb a6, 3(a2)
+; RV64I-NEXT:    and a1, a3, a5
+; RV64I-NEXT:    srli a1, a1, 8
+; RV64I-NEXT:    sb a3, 4(a2)
+; RV64I-NEXT:    sb a1, 5(a2)
+; RV64I-NEXT:    sb a7, 6(a2)
+; RV64I-NEXT:    sb t0, 7(a2)
+; RV64I-NEXT:    and a1, a0, a5
+; RV64I-NEXT:    and a3, t1, a5
+; RV64I-NEXT:    srli a1, a1, 8
+; RV64I-NEXT:    srli a3, a3, 8
+; RV64I-NEXT:    sb a0, 8(a2)
+; RV64I-NEXT:    sb a1, 9(a2)
+; RV64I-NEXT:    sb t2, 10(a2)
+; RV64I-NEXT:    sb t3, 11(a2)
+; RV64I-NEXT:    sb t1, 12(a2)
+; RV64I-NEXT:    sb a3, 13(a2)
+; RV64I-NEXT:    sb t4, 14(a2)
+; RV64I-NEXT:    sb t5, 15(a2)
+; RV64I-NEXT:    ld s0, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV32I-LABEL: shl_16bytes:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    lbu a3, 0(a0)
+; RV32I-NEXT:    lbu a4, 1(a0)
+; RV32I-NEXT:    lbu a5, 2(a0)
+; RV32I-NEXT:    lbu a6, 3(a0)
+; RV32I-NEXT:    lbu a7, 4(a0)
+; RV32I-NEXT:    lbu t0, 5(a0)
+; RV32I-NEXT:    lbu t1, 6(a0)
+; RV32I-NEXT:    lbu t2, 7(a0)
+; RV32I-NEXT:    slli a4, a4, 8
+; RV32I-NEXT:    or a3, a4, a3
+; RV32I-NEXT:    slli a6, a6, 8
+; RV32I-NEXT:    slli t0, t0, 8
+; RV32I-NEXT:    slli t2, t2, 8
+; RV32I-NEXT:    or a4, a6, a5
+; RV32I-NEXT:    or a5, t0, a7
+; RV32I-NEXT:    lbu a6, 0(a1)
+; RV32I-NEXT:    lbu a7, 1(a1)
+; RV32I-NEXT:    or t0, t2, t1
+; RV32I-NEXT:    lbu t1, 2(a1)
+; RV32I-NEXT:    lbu a1, 3(a1)
+; RV32I-NEXT:    slli a7, a7, 8
+; RV32I-NEXT:    or a7, a7, a6
+; RV32I-NEXT:    li a6, 64
+; RV32I-NEXT:    slli a1, a1, 8
+; RV32I-NEXT:    or a1, a1, t1
+; RV32I-NEXT:    li t1, 32
+; RV32I-NEXT:    slli a4, a4, 16
+; RV32I-NEXT:    slli t2, t0, 16
+; RV32I-NEXT:    slli a1, a1, 16
+; RV32I-NEXT:    or t0, a4, a3
+; RV32I-NEXT:    or a4, t2, a5
+; RV32I-NEXT:    or a5, a1, a7
+; RV32I-NEXT:    slli a5, a5, 3
+; RV32I-NEXT:    neg t3, a5
+; RV32I-NEXT:    srl t4, t0, t3
+; RV32I-NEXT:    sll t2, a4, a5
+; RV32I-NEXT:    bltu a5, t1, .LBB8_2
+; RV32I-NEXT:  # %bb.1:
+; RV32I-NEXT:    li a1, 0
+; RV32I-NEXT:    sll t5, t0, a5
+; RV32I-NEXT:    j .LBB8_3
+; RV32I-NEXT:  .LBB8_2:
+; RV32I-NEXT:    sll a1, t0, a5
+; RV32I-NEXT:    or t5, t4, t2
+; RV32I-NEXT:  .LBB8_3:
+; RV32I-NEXT:    addi sp, sp, -32
+; RV32I-NEXT:    sw s0, 28(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s1, 24(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s2, 20(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s3, 16(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s4, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s5, 8(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s6, 4(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    lbu s3, 11(a0)
+; RV32I-NEXT:    lbu s1, 15(a0)
+; RV32I-NEXT:    sub a7, a6, a5
+; RV32I-NEXT:    mv a3, a4
+; RV32I-NEXT:    beqz a5, .LBB8_5
+; RV32I-NEXT:  # %bb.4:
+; RV32I-NEXT:    mv a3, t5
+; RV32I-NEXT:  .LBB8_5:
+; RV32I-NEXT:    lbu s2, 9(a0)
+; RV32I-NEXT:    lbu t5, 10(a0)
+; RV32I-NEXT:    lbu s0, 13(a0)
+; RV32I-NEXT:    lbu t6, 14(a0)
+; RV32I-NEXT:    slli s3, s3, 8
+; RV32I-NEXT:    slli s1, s1, 8
+; RV32I-NEXT:    bltu a7, t1, .LBB8_7
+; RV32I-NEXT:  # %bb.6:
+; RV32I-NEXT:    srl s4, a4, a7
+; RV32I-NEXT:    j .LBB8_8
+; RV32I-NEXT:  .LBB8_7:
+; RV32I-NEXT:    neg s4, a7
+; RV32I-NEXT:    sll s4, a4, s4
+; RV32I-NEXT:    or s4, t4, s4
+; RV32I-NEXT:  .LBB8_8:
+; RV32I-NEXT:    slli s2, s2, 8
+; RV32I-NEXT:    lbu s6, 8(a0)
+; RV32I-NEXT:    lbu s5, 12(a0)
+; RV32I-NEXT:    or s3, s3, t5
+; RV32I-NEXT:    slli t5, s0, 8
+; RV32I-NEXT:    or s1, s1, t6
+; RV32I-NEXT:    mv t4, t0
+; RV32I-NEXT:    beqz a7, .LBB8_10
+; RV32I-NEXT:  # %bb.9:
+; RV32I-NEXT:    mv t4, s4
+; RV32I-NEXT:  .LBB8_10:
+; RV32I-NEXT:    or a0, s2, s6
+; RV32I-NEXT:    slli s0, s3, 16
+; RV32I-NEXT:    or t6, t5, s5
+; RV32I-NEXT:    slli s1, s1, 16
+; RV32I-NEXT:    bltu a7, t1, .LBB8_12
+; RV32I-NEXT:  # %bb.11:
+; RV32I-NEXT:    li t5, 0
+; RV32I-NEXT:    j .LBB8_13
+; RV32I-NEXT:  .LBB8_12:
+; RV32I-NEXT:    srl t5, a4, t3
+; RV32I-NEXT:  .LBB8_13:
+; RV32I-NEXT:    or a7, s0, a0
+; RV32I-NEXT:    or a0, s1, t6
+; RV32I-NEXT:    bltu a5, t1, .LBB8_15
+; RV32I-NEXT:  # %bb.14:
+; RV32I-NEXT:    li t6, 0
+; RV32I-NEXT:    sll s1, a7, a5
+; RV32I-NEXT:    j .LBB8_16
+; RV32I-NEXT:  .LBB8_15:
+; RV32I-NEXT:    sll t6, a7, a5
+; RV32I-NEXT:    srl t3, a7, t3
+; RV32I-NEXT:    sll s0, a0, a5
+; RV32I-NEXT:    or s1, t3, s0
+; RV32I-NEXT:  .LBB8_16:
+; RV32I-NEXT:    addi s0, a5, -64
+; RV32I-NEXT:    mv t3, a0
+; RV32I-NEXT:    beqz a5, .LBB8_18
+; RV32I-NEXT:  # %bb.17:
+; RV32I-NEXT:    mv t3, s1
+; RV32I-NEXT:  .LBB8_18:
+; RV32I-NEXT:    bltu s0, t1, .LBB8_20
+; RV32I-NEXT:  # %bb.19:
+; RV32I-NEXT:    li t1, 0
+; RV32I-NEXT:    sll t0, t0, s0
+; RV32I-NEXT:    bnez s0, .LBB8_21
+; RV32I-NEXT:    j .LBB8_22
+; RV32I-NEXT:  .LBB8_20:
+; RV32I-NEXT:    sll t1, t0, a5
+; RV32I-NEXT:    neg s1, s0
+; RV32I-NEXT:    srl t0, t0, s1
+; RV32I-NEXT:    or t0, t0, t2
+; RV32I-NEXT:    beqz s0, .LBB8_22
+; RV32I-NEXT:  .LBB8_21:
+; RV32I-NEXT:    mv a4, t0
+; RV32I-NEXT:  .LBB8_22:
+; RV32I-NEXT:    lw s0, 28(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s1, 24(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s2, 20(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s3, 16(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s4, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s5, 8(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s6, 4(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    addi sp, sp, 32
+; RV32I-NEXT:    bltu a5, a6, .LBB8_24
+; RV32I-NEXT:  # %bb.23:
+; RV32I-NEXT:    li a1, 0
+; RV32I-NEXT:    li a3, 0
+; RV32I-NEXT:    bnez a5, .LBB8_25
+; RV32I-NEXT:    j .LBB8_26
+; RV32I-NEXT:  .LBB8_24:
+; RV32I-NEXT:    or t1, t4, t6
+; RV32I-NEXT:    or a4, t5, t3
+; RV32I-NEXT:    beqz a5, .LBB8_26
+; RV32I-NEXT:  .LBB8_25:
+; RV32I-NEXT:    mv a7, t1
+; RV32I-NEXT:    mv a0, a4
+; RV32I-NEXT:  .LBB8_26:
+; RV32I-NEXT:    srli a4, a1, 16
+; RV32I-NEXT:    lui a5, 16
+; RV32I-NEXT:    srli a6, a1, 24
+; RV32I-NEXT:    srli t0, a3, 16
+; RV32I-NEXT:    srli t1, a3, 24
+; RV32I-NEXT:    srli t2, a7, 16
+; RV32I-NEXT:    srli t3, a7, 24
+; RV32I-NEXT:    srli t4, a0, 16
+; RV32I-NEXT:    srli t5, a0, 24
+; RV32I-NEXT:    addi a5, a5, -1
+; RV32I-NEXT:    and t6, a1, a5
+; RV32I-NEXT:    srli t6, t6, 8
+; RV32I-NEXT:    sb a1, 0(a2)
+; RV32I-NEXT:    sb t6, 1(a2)
+; RV32I-NEXT:    sb a4, 2(a2)
+; RV32I-NEXT:    sb a6, 3(a2)
+; RV32I-NEXT:    and a1, a3, a5
+; RV32I-NEXT:    srli a1, a1, 8
+; RV32I-NEXT:    sb a3, 4(a2)
+; RV32I-NEXT:    sb a1, 5(a2)
+; RV32I-NEXT:    sb t0, 6(a2)
+; RV32I-NEXT:    sb t1, 7(a2)
+; RV32I-NEXT:    and a1, a7, a5
+; RV32I-NEXT:    and a5, a0, a5
+; RV32I-NEXT:    srli a1, a1, 8
+; RV32I-NEXT:    srli a5, a5, 8
+; RV32I-NEXT:    sb a7, 8(a2)
+; RV32I-NEXT:    sb a1, 9(a2)
+; RV32I-NEXT:    sb t2, 10(a2)
+; RV32I-NEXT:    sb t3, 11(a2)
+; RV32I-NEXT:    sb a0, 12(a2)
+; RV32I-NEXT:    sb a5, 13(a2)
+; RV32I-NEXT:    sb t4, 14(a2)
+; RV32I-NEXT:    sb t5, 15(a2)
+; RV32I-NEXT:    ret
+  %src = load i128, ptr %src.ptr, align 1
+  %byteOff = load i128, ptr %byteOff.ptr, align 1
+  %bitOff = shl i128 %byteOff, 3
+  %res = shl i128 %src, %bitOff
+  store i128 %res, ptr %dst, align 1
+  ret void
+}
+
+define void @shl_16bytes_wordOff(ptr %src.ptr, ptr %wordOff.ptr, ptr %dst) nounwind {
+; RV64I-LABEL: shl_16bytes_wordOff:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd s0, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    lbu a3, 0(a0)
+; RV64I-NEXT:    lbu a4, 1(a0)
+; RV64I-NEXT:    lbu a5, 2(a0)
+; RV64I-NEXT:    lbu a6, 3(a0)
+; RV64I-NEXT:    lbu a7, 4(a0)
+; RV64I-NEXT:    lbu t0, 5(a0)
+; RV64I-NEXT:    lbu t1, 6(a0)
+; RV64I-NEXT:    lbu t2, 7(a0)
+; RV64I-NEXT:    lbu t3, 8(a0)
+; RV64I-NEXT:    lbu t4, 9(a0)
+; RV64I-NEXT:    lbu t5, 10(a0)
+; RV64I-NEXT:    lbu t6, 11(a0)
+; RV64I-NEXT:    slli a4, a4, 8
+; RV64I-NEXT:    slli a6, a6, 8
+; RV64I-NEXT:    or a3, a4, a3
+; RV64I-NEXT:    or a4, a6, a5
+; RV64I-NEXT:    lbu a5, 12(a0)
+; RV64I-NEXT:    lbu a6, 13(a0)
+; RV64I-NEXT:    lbu s0, 14(a0)
+; RV64I-NEXT:    lbu a0, 15(a0)
+; RV64I-NEXT:    slli t0, t0, 8
+; RV64I-NEXT:    slli t2, t2, 8
+; RV64I-NEXT:    slli t4, t4, 8
+; RV64I-NEXT:    slli t6, t6, 8
+; RV64I-NEXT:    or a7, t0, a7
+; RV64I-NEXT:    or t0, t2, t1
+; RV64I-NEXT:    or t1, t4, t3
+; RV64I-NEXT:    or t2, t6, t5
+; RV64I-NEXT:    lbu t3, 0(a1)
+; RV64I-NEXT:    lbu t4, 1(a1)
+; RV64I-NEXT:    lbu t5, 2(a1)
+; RV64I-NEXT:    lbu t6, 3(a1)
+; RV64I-NEXT:    slli a6, a6, 8
+; RV64I-NEXT:    slli a0, a0, 8
+; RV64I-NEXT:    slli t4, t4, 8
+; RV64I-NEXT:    or a5, a6, a5
+; RV64I-NEXT:    or a0, a0, s0
+; RV64I-NEXT:    or a6, t4, t3
+; RV64I-NEXT:    lbu t3, 4(a1)
+; RV64I-NEXT:    lbu t4, 5(a1)
+; RV64I-NEXT:    lbu s0, 6(a1)
+; RV64I-NEXT:    lbu a1, 7(a1)
+; RV64I-NEXT:    slli t6, t6, 8
+; RV64I-NEXT:    or t5, t6, t5
+; RV64I-NEXT:    slli t4, t4, 8
+; RV64I-NEXT:    or t3, t4, t3
+; RV64I-NEXT:    slli a1, a1, 8
+; RV64I-NEXT:    or a1, a1, s0
+; RV64I-NEXT:    slli a4, a4, 16
+; RV64I-NEXT:    or a3, a4, a3
+; RV64I-NEXT:    slli t0, t0, 16
+; RV64I-NEXT:    or a4, t0, a7
+; RV64I-NEXT:    slli t2, t2, 16
+; RV64I-NEXT:    or a7, t2, t1
+; RV64I-NEXT:    slli a0, a0, 16
+; RV64I-NEXT:    or a0, a0, a5
+; RV64I-NEXT:    slli t5, t5, 16
+; RV64I-NEXT:    or a5, t5, a6
+; RV64I-NEXT:    slli a1, a1, 16
+; RV64I-NEXT:    or a1, a1, t3
+; RV64I-NEXT:    slli a4, a4, 32
+; RV64I-NEXT:    slli a0, a0, 32
+; RV64I-NEXT:    slli a1, a1, 32
+; RV64I-NEXT:    or a4, a4, a3
+; RV64I-NEXT:    or a1, a1, a5
+; RV64I-NEXT:    slli a3, a1, 5
+; RV64I-NEXT:    li a5, 64
+; RV64I-NEXT:    or a0, a0, a7
+; RV64I-NEXT:    bltu a3, a5, .LBB9_2
+; RV64I-NEXT:  # %bb.1:
+; RV64I-NEXT:    li a1, 0
+; RV64I-NEXT:    subw a5, a3, a5
+; RV64I-NEXT:    sll a4, a4, a5
+; RV64I-NEXT:    bnez a3, .LBB9_3
+; RV64I-NEXT:    j .LBB9_4
+; RV64I-NEXT:  .LBB9_2:
+; RV64I-NEXT:    sll a1, a4, a3
+; RV64I-NEXT:    negw a5, a3
+; RV64I-NEXT:    srl a4, a4, a5
+; RV64I-NEXT:    sll a5, a0, a3
+; RV64I-NEXT:    or a4, a4, a5
+; RV64I-NEXT:    beqz a3, .LBB9_4
+; RV64I-NEXT:  .LBB9_3:
+; RV64I-NEXT:    mv a0, a4
+; RV64I-NEXT:  .LBB9_4:
+; RV64I-NEXT:    srli a3, a1, 32
+; RV64I-NEXT:    srliw a4, a1, 16
+; RV64I-NEXT:    lui a5, 16
+; RV64I-NEXT:    srliw a6, a1, 24
+; RV64I-NEXT:    srli a7, a1, 48
+; RV64I-NEXT:    srli t0, a1, 56
+; RV64I-NEXT:    srli t1, a0, 32
+; RV64I-NEXT:    srliw t2, a0, 16
+; RV64I-NEXT:    srliw t3, a0, 24
+; RV64I-NEXT:    srli t4, a0, 48
+; RV64I-NEXT:    srli t5, a0, 56
+; RV64I-NEXT:    addi a5, a5, -1
+; RV64I-NEXT:    and t6, a1, a5
+; RV64I-NEXT:    srli t6, t6, 8
+; RV64I-NEXT:    sb a1, 0(a2)
+; RV64I-NEXT:    sb t6, 1(a2)
+; RV64I-NEXT:    sb a4, 2(a2)
+; RV64I-NEXT:    sb a6, 3(a2)
+; RV64I-NEXT:    and a1, a3, a5
+; RV64I-NEXT:    srli a1, a1, 8
+; RV64I-NEXT:    sb a3, 4(a2)
+; RV64I-NEXT:    sb a1, 5(a2)
+; RV64I-NEXT:    sb a7, 6(a2)
+; RV64I-NEXT:    sb t0, 7(a2)
+; RV64I-NEXT:    and a1, a0, a5
+; RV64I-NEXT:    and a3, t1, a5
+; RV64I-NEXT:    srli a1, a1, 8
+; RV64I-NEXT:    srli a3, a3, 8
+; RV64I-NEXT:    sb a0, 8(a2)
+; RV64I-NEXT:    sb a1, 9(a2)
+; RV64I-NEXT:    sb t2, 10(a2)
+; RV64I-NEXT:    sb t3, 11(a2)
+; RV64I-NEXT:    sb t1, 12(a2)
+; RV64I-NEXT:    sb a3, 13(a2)
+; RV64I-NEXT:    sb t4, 14(a2)
+; RV64I-NEXT:    sb t5, 15(a2)
+; RV64I-NEXT:    ld s0, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV32I-LABEL: shl_16bytes_wordOff:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    lbu a3, 0(a0)
+; RV32I-NEXT:    lbu a4, 1(a0)
+; RV32I-NEXT:    lbu a5, 2(a0)
+; RV32I-NEXT:    lbu a6, 3(a0)
+; RV32I-NEXT:    lbu a7, 4(a0)
+; RV32I-NEXT:    lbu t0, 5(a0)
+; RV32I-NEXT:    lbu t1, 6(a0)
+; RV32I-NEXT:    lbu t2, 7(a0)
+; RV32I-NEXT:    slli a4, a4, 8
+; RV32I-NEXT:    or a3, a4, a3
+; RV32I-NEXT:    slli a6, a6, 8
+; RV32I-NEXT:    slli t0, t0, 8
+; RV32I-NEXT:    slli t2, t2, 8
+; RV32I-NEXT:    or a4, a6, a5
+; RV32I-NEXT:    or a5, t0, a7
+; RV32I-NEXT:    lbu a6, 0(a1)
+; RV32I-NEXT:    lbu a7, 1(a1)
+; RV32I-NEXT:    or t0, t2, t1
+; RV32I-NEXT:    lbu t1, 2(a1)
+; RV32I-NEXT:    lbu a1, 3(a1)
+; RV32I-NEXT:    slli a7, a7, 8
+; RV32I-NEXT:    or a7, a7, a6
+; RV32I-NEXT:    li a6, 64
+; RV32I-NEXT:    slli a1, a1, 8
+; RV32I-NEXT:    or a1, a1, t1
+; RV32I-NEXT:    li t1, 32
+; RV32I-NEXT:    slli a4, a4, 16
+; RV32I-NEXT:    slli t2, t0, 16
+; RV32I-NEXT:    slli a1, a1, 16
+; RV32I-NEXT:    or t0, a4, a3
+; RV32I-NEXT:    or a4, t2, a5
+; RV32I-NEXT:    or a5, a1, a7
+; RV32I-NEXT:    slli a5, a5, 5
+; RV32I-NEXT:    neg t3, a5
+; RV32I-NEXT:    srl t4, t0, t3
+; RV32I-NEXT:    sll t2, a4, a5
+; RV32I-NEXT:    bltu a5, t1, .LBB9_2
+; RV32I-NEXT:  # %bb.1:
+; RV32I-NEXT:    li a1, 0
+; RV32I-NEXT:    sll t5, t0, a5
+; RV32I-NEXT:    j .LBB9_3
+; RV32I-NEXT:  .LBB9_2:
+; RV32I-NEXT:    sll a1, t0, a5
+; RV32I-NEXT:    or t5, t4, t2
+; RV32I-NEXT:  .LBB9_3:
+; RV32I-NEXT:    addi sp, sp, -32
+; RV32I-NEXT:    sw s0, 28(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s1, 24(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s2, 20(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s3, 16(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s4, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s5, 8(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s6, 4(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    lbu s3, 11(a0)
+; RV32I-NEXT:    lbu s1, 15(a0)
+; RV32I-NEXT:    sub a7, a6, a5
+; RV32I-NEXT:    mv a3, a4
+; RV32I-NEXT:    beqz a5, .LBB9_5
+; RV32I-NEXT:  # %bb.4:
+; RV32I-NEXT:    mv a3, t5
+; RV32I-NEXT:  .LBB9_5:
+; RV32I-NEXT:    lbu s2, 9(a0)
+; RV32I-NEXT:    lbu t5, 10(a0)
+; RV32I-NEXT:    lbu s0, 13(a0)
+; RV32I-NEXT:    lbu t6, 14(a0)
+; RV32I-NEXT:    slli s3, s3, 8
+; RV32I-NEXT:    slli s1, s1, 8
+; RV32I-NEXT:    bltu a7, t1, .LBB9_7
+; RV32I-NEXT:  # %bb.6:
+; RV32I-NEXT:    srl s4, a4, a7
+; RV32I-NEXT:    j .LBB9_8
+; RV32I-NEXT:  .LBB9_7:
+; RV32I-NEXT:    neg s4, a7
+; RV32I-NEXT:    sll s4, a4, s4
+; RV32I-NEXT:    or s4, t4, s4
+; RV32I-NEXT:  .LBB9_8:
+; RV32I-NEXT:    slli s2, s2, 8
+; RV32I-NEXT:    lbu s6, 8(a0)
+; RV32I-NEXT:    lbu s5, 12(a0)
+; RV32I-NEXT:    or s3, s3, t5
+; RV32I-NEXT:    slli t5, s0, 8
+; RV32I-NEXT:    or s1, s1, t6
+; RV32I-NEXT:    mv t4, t0
+; RV32I-NEXT:    beqz a7, .LBB9_10
+; RV32I-NEXT:  # %bb.9:
+; RV32I-NEXT:    mv t4, s4
+; RV32I-NEXT:  .LBB9_10:
+; RV32I-NEXT:    or a0, s2, s6
+; RV32I-NEXT:    slli s0, s3, 16
+; RV32I-NEXT:    or t6, t5, s5
+; RV32I-NEXT:    slli s1, s1, 16
+; RV32I-NEXT:    bltu a7, t1, .LBB9_12
+; RV32I-NEXT:  # %bb.11:
+; RV32I-NEXT:    li t5, 0
+; RV32I-NEXT:    j .LBB9_13
+; RV32I-NEXT:  .LBB9_12:
+; RV32I-NEXT:    srl t5, a4, t3
+; RV32I-NEXT:  .LBB9_13:
+; RV32I-NEXT:    or a7, s0, a0
+; RV32I-NEXT:    or a0, s1, t6
+; RV32I-NEXT:    bltu a5, t1, .LBB9_15
+; RV32I-NEXT:  # %bb.14:
+; RV32I-NEXT:    li t6, 0
+; RV32I-NEXT:    sll s1, a7, a5
+; RV32I-NEXT:    j .LBB9_16
+; RV32I-NEXT:  .LBB9_15:
+; RV32I-NEXT:    sll t6, a7, a5
+; RV32I-NEXT:    srl t3, a7, t3
+; RV32I-NEXT:    sll s0, a0, a5
+; RV32I-NEXT:    or s1, t3, s0
+; RV32I-NEXT:  .LBB9_16:
+; RV32I-NEXT:    addi s0, a5, -64
+; RV32I-NEXT:    mv t3, a0
+; RV32I-NEXT:    beqz a5, .LBB9_18
+; RV32I-NEXT:  # %bb.17:
+; RV32I-NEXT:    mv t3, s1
+; RV32I-NEXT:  .LBB9_18:
+; RV32I-NEXT:    bltu s0, t1, .LBB9_20
+; RV32I-NEXT:  # %bb.19:
+; RV32I-NEXT:    li t1, 0
+; RV32I-NEXT:    sll t0, t0, s0
+; RV32I-NEXT:    bnez s0, .LBB9_21
+; RV32I-NEXT:    j .LBB9_22
+; RV32I-NEXT:  .LBB9_20:
+; RV32I-NEXT:    sll t1, t0, a5
+; RV32I-NEXT:    neg s1, s0
+; RV32I-NEXT:    srl t0, t0, s1
+; RV32I-NEXT:    or t0, t0, t2
+; RV32I-NEXT:    beqz s0, .LBB9_22
+; RV32I-NEXT:  .LBB9_21:
+; RV32I-NEXT:    mv a4, t0
+; RV32I-NEXT:  .LBB9_22:
+; RV32I-NEXT:    lw s0, 28(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s1, 24(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s2, 20(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s3, 16(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s4, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s5, 8(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s6, 4(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    addi sp, sp, 32
+; RV32I-NEXT:    bltu a5, a6, .LBB9_24
+; RV32I-NEXT:  # %bb.23:
+; RV32I-NEXT:    li a1, 0
+; RV32I-NEXT:    li a3, 0
+; RV32I-NEXT:    bnez a5, .LBB9_25
+; RV32I-NEXT:    j .LBB9_26
+; RV32I-NEXT:  .LBB9_24:
+; RV32I-NEXT:    or t1, t4, t6
+; RV32I-NEXT:    or a4, t5, t3
+; RV32I-NEXT:    beqz a5, .LBB9_26
+; RV32I-NEXT:  .LBB9_25:
+; RV32I-NEXT:    mv a7, t1
+; RV32I-NEXT:    mv a0, a4
+; RV32I-NEXT:  .LBB9_26:
+; RV32I-NEXT:    srli a4, a1, 16
+; RV32I-NEXT:    lui a5, 16
+; RV32I-NEXT:    srli a6, a1, 24
+; RV32I-NEXT:    srli t0, a3, 16
+; RV32I-NEXT:    srli t1, a3, 24
+; RV32I-NEXT:    srli t2, a7, 16
+; RV32I-NEXT:    srli t3, a7, 24
+; RV32I-NEXT:    srli t4, a0, 16
+; RV32I-NEXT:    srli t5, a0, 24
+; RV32I-NEXT:    addi a5, a5, -1
+; RV32I-NEXT:    and t6, a1, a5
+; RV32I-NEXT:    srli t6, t6, 8
+; RV32I-NEXT:    sb a1, 0(a2)
+; RV32I-NEXT:    sb t6, 1(a2)
+; RV32I-NEXT:    sb a4, 2(a2)
+; RV32I-NEXT:    sb a6, 3(a2)
+; RV32I-NEXT:    and a1, a3, a5
+; RV32I-NEXT:    srli a1, a1, 8
+; RV32I-NEXT:    sb a3, 4(a2)
+; RV32I-NEXT:    sb a1, 5(a2)
+; RV32I-NEXT:    sb t0, 6(a2)
+; RV32I-NEXT:    sb t1, 7(a2)
+; RV32I-NEXT:    and a1, a7, a5
+; RV32I-NEXT:    and a5, a0, a5
+; RV32I-NEXT:    srli a1, a1, 8
+; RV32I-NEXT:    srli a5, a5, 8
+; RV32I-NEXT:    sb a7, 8(a2)
+; RV32I-NEXT:    sb a1, 9(a2)
+; RV32I-NEXT:    sb t2, 10(a2)
+; RV32I-NEXT:    sb t3, 11(a2)
+; RV32I-NEXT:    sb a0, 12(a2)
+; RV32I-NEXT:    sb a5, 13(a2)
+; RV32I-NEXT:    sb t4, 14(a2)
+; RV32I-NEXT:    sb t5, 15(a2)
+; RV32I-NEXT:    ret
+  %src = load i128, ptr %src.ptr, align 1
+  %wordOff = load i128, ptr %wordOff.ptr, align 1
+  %bitOff = shl i128 %wordOff, 5
+  %res = shl i128 %src, %bitOff
+  store i128 %res, ptr %dst, align 1
+  ret void
+}
+
+
+define void @ashr_16bytes(ptr %src.ptr, ptr %byteOff.ptr, ptr %dst) nounwind {
+; RV64I-LABEL: ashr_16bytes:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd s0, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    lbu a3, 0(a0)
+; RV64I-NEXT:    lbu a4, 1(a0)
+; RV64I-NEXT:    lbu a5, 2(a0)
+; RV64I-NEXT:    lbu a6, 3(a0)
+; RV64I-NEXT:    lbu a7, 4(a0)
+; RV64I-NEXT:    lbu t0, 5(a0)
+; RV64I-NEXT:    lbu t1, 6(a0)
+; RV64I-NEXT:    lbu t2, 7(a0)
+; RV64I-NEXT:    lbu t3, 8(a0)
+; RV64I-NEXT:    lbu t4, 9(a0)
+; RV64I-NEXT:    lbu t5, 10(a0)
+; RV64I-NEXT:    lbu t6, 11(a0)
+; RV64I-NEXT:    slli a4, a4, 8
+; RV64I-NEXT:    slli a6, a6, 8
+; RV64I-NEXT:    or a3, a4, a3
+; RV64I-NEXT:    or a4, a6, a5
+; RV64I-NEXT:    lbu a5, 12(a0)
+; RV64I-NEXT:    lbu a6, 13(a0)
+; RV64I-NEXT:    lbu s0, 14(a0)
+; RV64I-NEXT:    lbu a0, 15(a0)
+; RV64I-NEXT:    slli t0, t0, 8
+; RV64I-NEXT:    slli t2, t2, 8
+; RV64I-NEXT:    slli t4, t4, 8
+; RV64I-NEXT:    slli t6, t6, 8
+; RV64I-NEXT:    or a7, t0, a7
+; RV64I-NEXT:    or t0, t2, t1
+; RV64I-NEXT:    or t1, t4, t3
+; RV64I-NEXT:    or t2, t6, t5
+; RV64I-NEXT:    lbu t3, 0(a1)
+; RV64I-NEXT:    lbu t4, 1(a1)
+; RV64I-NEXT:    lbu t5, 2(a1)
+; RV64I-NEXT:    lbu t6, 3(a1)
+; RV64I-NEXT:    slli a6, a6, 8
+; RV64I-NEXT:    slli a0, a0, 8
+; RV64I-NEXT:    slli t4, t4, 8
+; RV64I-NEXT:    or a5, a6, a5
+; RV64I-NEXT:    or a0, a0, s0
+; RV64I-NEXT:    or a6, t4, t3
+; RV64I-NEXT:    lbu t3, 4(a1)
+; RV64I-NEXT:    lbu t4, 5(a1)
+; RV64I-NEXT:    lbu s0, 6(a1)
+; RV64I-NEXT:    lbu a1, 7(a1)
+; RV64I-NEXT:    slli t6, t6, 8
+; RV64I-NEXT:    or t5, t6, t5
+; RV64I-NEXT:    slli t4, t4, 8
+; RV64I-NEXT:    or t3, t4, t3
+; RV64I-NEXT:    slli a1, a1, 8
+; RV64I-NEXT:    or a1, a1, s0
+; RV64I-NEXT:    slli a4, a4, 16
+; RV64I-NEXT:    or a3, a4, a3
+; RV64I-NEXT:    slli t0, t0, 16
+; RV64I-NEXT:    or a4, t0, a7
+; RV64I-NEXT:    slli t2, t2, 16
+; RV64I-NEXT:    or a7, t2, t1
+; RV64I-NEXT:    slli a0, a0, 16
+; RV64I-NEXT:    or a0, a0, a5
+; RV64I-NEXT:    slli t5, t5, 16
+; RV64I-NEXT:    or a5, t5, a6
+; RV64I-NEXT:    slli a1, a1, 16
+; RV64I-NEXT:    or a1, a1, t3
+; RV64I-NEXT:    slli a4, a4, 32
+; RV64I-NEXT:    slli a6, a0, 32
+; RV64I-NEXT:    slli a1, a1, 32
+; RV64I-NEXT:    or a0, a4, a3
+; RV64I-NEXT:    or a1, a1, a5
+; RV64I-NEXT:    slli a1, a1, 3
+; RV64I-NEXT:    li a4, 64
+; RV64I-NEXT:    or a3, a6, a7
+; RV64I-NEXT:    bltu a1, a4, .LBB10_2
+; RV64I-NEXT:  # %bb.1:
+; RV64I-NEXT:    subw a5, a1, a4
+; RV64I-NEXT:    sra a5, a3, a5
+; RV64I-NEXT:    bnez a1, .LBB10_3
+; RV64I-NEXT:    j .LBB10_4
+; RV64I-NEXT:  .LBB10_2:
+; RV64I-NEXT:    srl a5, a0, a1
+; RV64I-NEXT:    negw a6, a1
+; RV64I-NEXT:    sll a6, a3, a6
+; RV64I-NEXT:    or a5, a5, a6
+; RV64I-NEXT:    beqz a1, .LBB10_4
+; RV64I-NEXT:  .LBB10_3:
+; RV64I-NEXT:    mv a0, a5
+; RV64I-NEXT:  .LBB10_4:
+; RV64I-NEXT:    bltu a1, a4, .LBB10_6
+; RV64I-NEXT:  # %bb.5:
+; RV64I-NEXT:    srai a1, a3, 63
+; RV64I-NEXT:    j .LBB10_7
+; RV64I-NEXT:  .LBB10_6:
+; RV64I-NEXT:    sra a1, a3, a1
+; RV64I-NEXT:  .LBB10_7:
+; RV64I-NEXT:    srli a3, a0, 32
+; RV64I-NEXT:    srliw a4, a0, 16
+; RV64I-NEXT:    lui a5, 16
+; RV64I-NEXT:    srliw a6, a0, 24
+; RV64I-NEXT:    srli a7, a0, 48
+; RV64I-NEXT:    srli t0, a0, 56
+; RV64I-NEXT:    srli t1, a1, 32
+; RV64I-NEXT:    srliw t2, a1, 16
+; RV64I-NEXT:    srliw t3, a1, 24
+; RV64I-NEXT:    srli t4, a1, 48
+; RV64I-NEXT:    srli t5, a1, 56
+; RV64I-NEXT:    addi a5, a5, -1
+; RV64I-NEXT:    and t6, a0, a5
+; RV64I-NEXT:    srli t6, t6, 8
+; RV64I-NEXT:    sb a0, 0(a2)
+; RV64I-NEXT:    sb t6, 1(a2)
+; RV64I-NEXT:    sb a4, 2(a2)
+; RV64I-NEXT:    sb a6, 3(a2)
+; RV64I-NEXT:    and a0, a3, a5
+; RV64I-NEXT:    srli a0, a0, 8
+; RV64I-NEXT:    sb a3, 4(a2)
+; RV64I-NEXT:    sb a0, 5(a2)
+; RV64I-NEXT:    sb a7, 6(a2)
+; RV64I-NEXT:    sb t0, 7(a2)
+; RV64I-NEXT:    and a0, a1, a5
+; RV64I-NEXT:    and a3, t1, a5
+; RV64I-NEXT:    srli a0, a0, 8
+; RV64I-NEXT:    srli a3, a3, 8
+; RV64I-NEXT:    sb a1, 8(a2)
+; RV64I-NEXT:    sb a0, 9(a2)
+; RV64I-NEXT:    sb t2, 10(a2)
+; RV64I-NEXT:    sb t3, 11(a2)
+; RV64I-NEXT:    sb t1, 12(a2)
+; RV64I-NEXT:    sb a3, 13(a2)
+; RV64I-NEXT:    sb t4, 14(a2)
+; RV64I-NEXT:    sb t5, 15(a2)
+; RV64I-NEXT:    ld s0, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV32I-LABEL: ashr_16bytes:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    addi sp, sp, -16
+; RV32I-NEXT:    sw s0, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s1, 8(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s2, 4(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    lbu a3, 0(a0)
+; RV32I-NEXT:    lbu a5, 1(a0)
+; RV32I-NEXT:    lbu a6, 2(a0)
+; RV32I-NEXT:    lbu a7, 3(a0)
+; RV32I-NEXT:    lbu a4, 4(a0)
+; RV32I-NEXT:    lbu t2, 5(a0)
+; RV32I-NEXT:    lbu t0, 6(a0)
+; RV32I-NEXT:    lbu t1, 7(a0)
+; RV32I-NEXT:    lbu t4, 8(a0)
+; RV32I-NEXT:    lbu t5, 9(a0)
+; RV32I-NEXT:    lbu t6, 10(a0)
+; RV32I-NEXT:    lbu s0, 11(a0)
+; RV32I-NEXT:    slli a7, a7, 8
+; RV32I-NEXT:    slli t1, t1, 8
+; RV32I-NEXT:    or t3, a7, a6
+; RV32I-NEXT:    or t1, t1, t0
+; RV32I-NEXT:    lbu a6, 12(a0)
+; RV32I-NEXT:    lbu a7, 13(a0)
+; RV32I-NEXT:    lbu t0, 14(a0)
+; RV32I-NEXT:    lbu a0, 15(a0)
+; RV32I-NEXT:    slli t5, t5, 8
+; RV32I-NEXT:    slli s0, s0, 8
+; RV32I-NEXT:    slli a7, a7, 8
+; RV32I-NEXT:    or t4, t5, t4
+; RV32I-NEXT:    or t5, s0, t6
+; RV32I-NEXT:    or a6, a7, a6
+; RV32I-NEXT:    lbu a7, 0(a1)
+; RV32I-NEXT:    lbu t6, 1(a1)
+; RV32I-NEXT:    lbu s0, 2(a1)
+; RV32I-NEXT:    lbu a1, 3(a1)
+; RV32I-NEXT:    slli a0, a0, 8
+; RV32I-NEXT:    or s1, a0, t0
+; RV32I-NEXT:    slli t6, t6, 8
+; RV32I-NEXT:    or t6, t6, a7
+; RV32I-NEXT:    slli a1, a1, 8
+; RV32I-NEXT:    or s0, a1, s0
+; RV32I-NEXT:    li t0, 32
+; RV32I-NEXT:    slli a1, a5, 8
+; RV32I-NEXT:    slli a0, t2, 8
+; RV32I-NEXT:    slli t5, t5, 16
+; RV32I-NEXT:    slli s1, s1, 16
+; RV32I-NEXT:    slli s0, s0, 16
+; RV32I-NEXT:    or a7, t5, t4
+; RV32I-NEXT:    or a5, s1, a6
+; RV32I-NEXT:    or a6, s0, t6
+; RV32I-NEXT:    slli a6, a6, 3
+; RV32I-NEXT:    srl t2, a7, a6
+; RV32I-NEXT:    neg t6, a6
+; RV32I-NEXT:    sll t4, a5, t6
+; RV32I-NEXT:    bltu a6, t0, .LBB10_2
+; RV32I-NEXT:  # %bb.1:
+; RV32I-NEXT:    sra t5, a5, a6
+; RV32I-NEXT:    j .LBB10_3
+; RV32I-NEXT:  .LBB10_2:
+; RV32I-NEXT:    or t5, t2, t4
+; RV32I-NEXT:  .LBB10_3:
+; RV32I-NEXT:    or a1, a1, a3
+; RV32I-NEXT:    slli t3, t3, 16
+; RV32I-NEXT:    or a3, a0, a4
+; RV32I-NEXT:    slli t1, t1, 16
+; RV32I-NEXT:    mv a0, a7
+; RV32I-NEXT:    beqz a6, .LBB10_5
+; RV32I-NEXT:  # %bb.4:
+; RV32I-NEXT:    mv a0, t5
+; RV32I-NEXT:  .LBB10_5:
+; RV32I-NEXT:    or a4, t3, a1
+; RV32I-NEXT:    or a3, t1, a3
+; RV32I-NEXT:    bltu a6, t0, .LBB10_7
+; RV32I-NEXT:  # %bb.6:
+; RV32I-NEXT:    srai a1, a5, 31
+; RV32I-NEXT:    srl t5, a3, a6
+; RV32I-NEXT:    j .LBB10_8
+; RV32I-NEXT:  .LBB10_7:
+; RV32I-NEXT:    sra a1, a5, a6
+; RV32I-NEXT:    srl t1, a4, a6
+; RV32I-NEXT:    sll t3, a3, t6
+; RV32I-NEXT:    or t5, t1, t3
+; RV32I-NEXT:  .LBB10_8:
+; RV32I-NEXT:    li t1, 64
+; RV32I-NEXT:    mv t3, a4
+; RV32I-NEXT:    beqz a6, .LBB10_10
+; RV32I-NEXT:  # %bb.9:
+; RV32I-NEXT:    mv t3, t5
+; RV32I-NEXT:  .LBB10_10:
+; RV32I-NEXT:    sub s0, t1, a6
+; RV32I-NEXT:    bltu a6, t0, .LBB10_13
+; RV32I-NEXT:  # %bb.11:
+; RV32I-NEXT:    li t5, 0
+; RV32I-NEXT:    bgeu s0, t0, .LBB10_14
+; RV32I-NEXT:  .LBB10_12:
+; RV32I-NEXT:    sll t6, a7, t6
+; RV32I-NEXT:    neg s1, s0
+; RV32I-NEXT:    srl s1, a7, s1
+; RV32I-NEXT:    or s2, s1, t4
+; RV32I-NEXT:    j .LBB10_15
+; RV32I-NEXT:  .LBB10_13:
+; RV32I-NEXT:    srl t5, a3, a6
+; RV32I-NEXT:    bltu s0, t0, .LBB10_12
+; RV32I-NEXT:  .LBB10_14:
+; RV32I-NEXT:    li t6, 0
+; RV32I-NEXT:    sll s2, a7, s0
+; RV32I-NEXT:  .LBB10_15:
+; RV32I-NEXT:    addi s1, a6, -64
+; RV32I-NEXT:    mv t4, a5
+; RV32I-NEXT:    beqz s0, .LBB10_17
+; RV32I-NEXT:  # %bb.16:
+; RV32I-NEXT:    mv t4, s2
+; RV32I-NEXT:  .LBB10_17:
+; RV32I-NEXT:    bltu s1, t0, .LBB10_19
+; RV32I-NEXT:  # %bb.18:
+; RV32I-NEXT:    sra t2, a5, s1
+; RV32I-NEXT:    bnez s1, .LBB10_20
+; RV32I-NEXT:    j .LBB10_21
+; RV32I-NEXT:  .LBB10_19:
+; RV32I-NEXT:    neg s0, s1
+; RV32I-NEXT:    sll s0, a5, s0
+; RV32I-NEXT:    or t2, t2, s0
+; RV32I-NEXT:    beqz s1, .LBB10_21
+; RV32I-NEXT:  .LBB10_20:
+; RV32I-NEXT:    mv a7, t2
+; RV32I-NEXT:  .LBB10_21:
+; RV32I-NEXT:    bltu s1, t0, .LBB10_23
+; RV32I-NEXT:  # %bb.22:
+; RV32I-NEXT:    srai t0, a5, 31
+; RV32I-NEXT:    bltu a6, t1, .LBB10_24
+; RV32I-NEXT:    j .LBB10_25
+; RV32I-NEXT:  .LBB10_23:
+; RV32I-NEXT:    sra t0, a5, a6
+; RV32I-NEXT:    bgeu a6, t1, .LBB10_25
+; RV32I-NEXT:  .LBB10_24:
+; RV32I-NEXT:    or a7, t3, t6
+; RV32I-NEXT:    or t0, t5, t4
+; RV32I-NEXT:  .LBB10_25:
+; RV32I-NEXT:    bnez a6, .LBB10_29
+; RV32I-NEXT:  # %bb.26:
+; RV32I-NEXT:    bltu a6, t1, .LBB10_28
+; RV32I-NEXT:  .LBB10_27:
+; RV32I-NEXT:    srai a0, a5, 31
+; RV32I-NEXT:    mv a1, a0
+; RV32I-NEXT:  .LBB10_28:
+; RV32I-NEXT:    srli a5, a4, 16
+; RV32I-NEXT:    lui a6, 16
+; RV32I-NEXT:    srli a7, a4, 24
+; RV32I-NEXT:    srli t0, a3, 16
+; RV32I-NEXT:    srli t1, a3, 24
+; RV32I-NEXT:    srli t2, a0, 16
+; RV32I-NEXT:    srli t3, a0, 24
+; RV32I-NEXT:    srli t4, a1, 16
+; RV32I-NEXT:    srli t5, a1, 24
+; RV32I-NEXT:    addi a6, a6, -1
+; RV32I-NEXT:    and t6, a4, a6
+; RV32I-NEXT:    srli t6, t6, 8
+; RV32I-NEXT:    sb a4, 0(a2)
+; RV32I-NEXT:    sb t6, 1(a2)
+; RV32I-NEXT:    sb a5, 2(a2)
+; RV32I-NEXT:    sb a7, 3(a2)
+; RV32I-NEXT:    and a4, a3, a6
+; RV32I-NEXT:    srli a4, a4, 8
+; RV32I-NEXT:    sb a3, 4(a2)
+; RV32I-NEXT:    sb a4, 5(a2)
+; RV32I-NEXT:    sb t0, 6(a2)
+; RV32I-NEXT:    sb t1, 7(a2)
+; RV32I-NEXT:    and a3, a0, a6
+; RV32I-NEXT:    and a4, a1, a6
+; RV32I-NEXT:    srli a3, a3, 8
+; RV32I-NEXT:    srli a4, a4, 8
+; RV32I-NEXT:    sb a0, 8(a2)
+; RV32I-NEXT:    sb a3, 9(a2)
+; RV32I-NEXT:    sb t2, 10(a2)
+; RV32I-NEXT:    sb t3, 11(a2)
+; RV32I-NEXT:    sb a1, 12(a2)
+; RV32I-NEXT:    sb a4, 13(a2)
+; RV32I-NEXT:    sb t4, 14(a2)
+; RV32I-NEXT:    sb t5, 15(a2)
+; RV32I-NEXT:    lw s0, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s1, 8(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s2, 4(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    addi sp, sp, 16
+; RV32I-NEXT:    ret
+; RV32I-NEXT:  .LBB10_29:
+; RV32I-NEXT:    mv a4, a7
+; RV32I-NEXT:    mv a3, t0
+; RV32I-NEXT:    bgeu a6, t1, .LBB10_27
+; RV32I-NEXT:    j .LBB10_28
+  %src = load i128, ptr %src.ptr, align 1
+  %byteOff = load i128, ptr %byteOff.ptr, align 1
+  %bitOff = shl i128 %byteOff, 3
+  %res = ashr i128 %src, %bitOff
+  store i128 %res, ptr %dst, align 1
+  ret void
+}
+
+define void @ashr_16bytes_wordOff(ptr %src.ptr, ptr %wordOff.ptr, ptr %dst) nounwind {
+; RV64I-LABEL: ashr_16bytes_wordOff:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd s0, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    lbu a3, 0(a0)
+; RV64I-NEXT:    lbu a4, 1(a0)
+; RV64I-NEXT:    lbu a5, 2(a0)
+; RV64I-NEXT:    lbu a6, 3(a0)
+; RV64I-NEXT:    lbu a7, 4(a0)
+; RV64I-NEXT:    lbu t0, 5(a0)
+; RV64I-NEXT:    lbu t1, 6(a0)
+; RV64I-NEXT:    lbu t2, 7(a0)
+; RV64I-NEXT:    lbu t3, 8(a0)
+; RV64I-NEXT:    lbu t4, 9(a0)
+; RV64I-NEXT:    lbu t5, 10(a0)
+; RV64I-NEXT:    lbu t6, 11(a0)
+; RV64I-NEXT:    slli a4, a4, 8
+; RV64I-NEXT:    slli a6, a6, 8
+; RV64I-NEXT:    or a3, a4, a3
+; RV64I-NEXT:    or a4, a6, a5
+; RV64I-NEXT:    lbu a5, 12(a0)
+; RV64I-NEXT:    lbu a6, 13(a0)
+; RV64I-NEXT:    lbu s0, 14(a0)
+; RV64I-NEXT:    lbu a0, 15(a0)
+; RV64I-NEXT:    slli t0, t0, 8
+; RV64I-NEXT:    slli t2, t2, 8
+; RV64I-NEXT:    slli t4, t4, 8
+; RV64I-NEXT:    slli t6, t6, 8
+; RV64I-NEXT:    or a7, t0, a7
+; RV64I-NEXT:    or t0, t2, t1
+; RV64I-NEXT:    or t1, t4, t3
+; RV64I-NEXT:    or t2, t6, t5
+; RV64I-NEXT:    lbu t3, 0(a1)
+; RV64I-NEXT:    lbu t4, 1(a1)
+; RV64I-NEXT:    lbu t5, 2(a1)
+; RV64I-NEXT:    lbu t6, 3(a1)
+; RV64I-NEXT:    slli a6, a6, 8
+; RV64I-NEXT:    slli a0, a0, 8
+; RV64I-NEXT:    slli t4, t4, 8
+; RV64I-NEXT:    or a5, a6, a5
+; RV64I-NEXT:    or a0, a0, s0
+; RV64I-NEXT:    or a6, t4, t3
+; RV64I-NEXT:    lbu t3, 4(a1)
+; RV64I-NEXT:    lbu t4, 5(a1)
+; RV64I-NEXT:    lbu s0, 6(a1)
+; RV64I-NEXT:    lbu a1, 7(a1)
+; RV64I-NEXT:    slli t6, t6, 8
+; RV64I-NEXT:    or t5, t6, t5
+; RV64I-NEXT:    slli t4, t4, 8
+; RV64I-NEXT:    or t3, t4, t3
+; RV64I-NEXT:    slli a1, a1, 8
+; RV64I-NEXT:    or a1, a1, s0
+; RV64I-NEXT:    slli a4, a4, 16
+; RV64I-NEXT:    or a3, a4, a3
+; RV64I-NEXT:    slli t0, t0, 16
+; RV64I-NEXT:    or a4, t0, a7
+; RV64I-NEXT:    slli t2, t2, 16
+; RV64I-NEXT:    or a7, t2, t1
+; RV64I-NEXT:    slli a0, a0, 16
+; RV64I-NEXT:    or a0, a0, a5
+; RV64I-NEXT:    slli t5, t5, 16
+; RV64I-NEXT:    or a5, t5, a6
+; RV64I-NEXT:    slli a1, a1, 16
+; RV64I-NEXT:    or a1, a1, t3
+; RV64I-NEXT:    slli a4, a4, 32
+; RV64I-NEXT:    slli a6, a0, 32
+; RV64I-NEXT:    slli a1, a1, 32
+; RV64I-NEXT:    or a0, a4, a3
+; RV64I-NEXT:    or a1, a1, a5
+; RV64I-NEXT:    slli a1, a1, 5
+; RV64I-NEXT:    li a4, 64
+; RV64I-NEXT:    or a3, a6, a7
+; RV64I-NEXT:    bltu a1, a4, .LBB11_2
+; RV64I-NEXT:  # %bb.1:
+; RV64I-NEXT:    subw a5, a1, a4
+; RV64I-NEXT:    sra a5, a3, a5
+; RV64I-NEXT:    bnez a1, .LBB11_3
+; RV64I-NEXT:    j .LBB11_4
+; RV64I-NEXT:  .LBB11_2:
+; RV64I-NEXT:    srl a5, a0, a1
+; RV64I-NEXT:    negw a6, a1
+; RV64I-NEXT:    sll a6, a3, a6
+; RV64I-NEXT:    or a5, a5, a6
+; RV64I-NEXT:    beqz a1, .LBB11_4
+; RV64I-NEXT:  .LBB11_3:
+; RV64I-NEXT:    mv a0, a5
+; RV64I-NEXT:  .LBB11_4:
+; RV64I-NEXT:    bltu a1, a4, .LBB11_6
+; RV64I-NEXT:  # %bb.5:
+; RV64I-NEXT:    srai a1, a3, 63
+; RV64I-NEXT:    j .LBB11_7
+; RV64I-NEXT:  .LBB11_6:
+; RV64I-NEXT:    sra a1, a3, a1
+; RV64I-NEXT:  .LBB11_7:
+; RV64I-NEXT:    srli a3, a0, 32
+; RV64I-NEXT:    srliw a4, a0, 16
+; RV64I-NEXT:    lui a5, 16
+; RV64I-NEXT:    srliw a6, a0, 24
+; RV64I-NEXT:    srli a7, a0, 48
+; RV64I-NEXT:    srli t0, a0, 56
+; RV64I-NEXT:    srli t1, a1, 32
+; RV64I-NEXT:    srliw t2, a1, 16
+; RV64I-NEXT:    srliw t3, a1, 24
+; RV64I-NEXT:    srli t4, a1, 48
+; RV64I-NEXT:    srli t5, a1, 56
+; RV64I-NEXT:    addi a5, a5, -1
+; RV64I-NEXT:    and t6, a0, a5
+; RV64I-NEXT:    srli t6, t6, 8
+; RV64I-NEXT:    sb a0, 0(a2)
+; RV64I-NEXT:    sb t6, 1(a2)
+; RV64I-NEXT:    sb a4, 2(a2)
+; RV64I-NEXT:    sb a6, 3(a2)
+; RV64I-NEXT:    and a0, a3, a5
+; RV64I-NEXT:    srli a0, a0, 8
+; RV64I-NEXT:    sb a3, 4(a2)
+; RV64I-NEXT:    sb a0, 5(a2)
+; RV64I-NEXT:    sb a7, 6(a2)
+; RV64I-NEXT:    sb t0, 7(a2)
+; RV64I-NEXT:    and a0, a1, a5
+; RV64I-NEXT:    and a3, t1, a5
+; RV64I-NEXT:    srli a0, a0, 8
+; RV64I-NEXT:    srli a3, a3, 8
+; RV64I-NEXT:    sb a1, 8(a2)
+; RV64I-NEXT:    sb a0, 9(a2)
+; RV64I-NEXT:    sb t2, 10(a2)
+; RV64I-NEXT:    sb t3, 11(a2)
+; RV64I-NEXT:    sb t1, 12(a2)
+; RV64I-NEXT:    sb a3, 13(a2)
+; RV64I-NEXT:    sb t4, 14(a2)
+; RV64I-NEXT:    sb t5, 15(a2)
+; RV64I-NEXT:    ld s0, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV32I-LABEL: ashr_16bytes_wordOff:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    addi sp, sp, -16
+; RV32I-NEXT:    sw s0, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s1, 8(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s2, 4(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    lbu a3, 0(a0)
+; RV32I-NEXT:    lbu a5, 1(a0)
+; RV32I-NEXT:    lbu a6, 2(a0)
+; RV32I-NEXT:    lbu a7, 3(a0)
+; RV32I-NEXT:    lbu a4, 4(a0)
+; RV32I-NEXT:    lbu t2, 5(a0)
+; RV32I-NEXT:    lbu t0, 6(a0)
+; RV32I-NEXT:    lbu t1, 7(a0)
+; RV32I-NEXT:    lbu t4, 8(a0)
+; RV32I-NEXT:    lbu t5, 9(a0)
+; RV32I-NEXT:    lbu t6, 10(a0)
+; RV32I-NEXT:    lbu s0, 11(a0)
+; RV32I-NEXT:    slli a7, a7, 8
+; RV32I-NEXT:    slli t1, t1, 8
+; RV32I-NEXT:    or t3, a7, a6
+; RV32I-NEXT:    or t1, t1, t0
+; RV32I-NEXT:    lbu a6, 12(a0)
+; RV32I-NEXT:    lbu a7, 13(a0)
+; RV32I-NEXT:    lbu t0, 14(a0)
+; RV32I-NEXT:    lbu a0, 15(a0)
+; RV32I-NEXT:    slli t5, t5, 8
+; RV32I-NEXT:    slli s0, s0, 8
+; RV32I-NEXT:    slli a7, a7, 8
+; RV32I-NEXT:    or t4, t5, t4
+; RV32I-NEXT:    or t5, s0, t6
+; RV32I-NEXT:    or a6, a7, a6
+; RV32I-NEXT:    lbu a7, 0(a1)
+; RV32I-NEXT:    lbu t6, 1(a1)
+; RV32I-NEXT:    lbu s0, 2(a1)
+; RV32I-NEXT:    lbu a1, 3(a1)
+; RV32I-NEXT:    slli a0, a0, 8
+; RV32I-NEXT:    or s1, a0, t0
+; RV32I-NEXT:    slli t6, t6, 8
+; RV32I-NEXT:    or t6, t6, a7
+; RV32I-NEXT:    slli a1, a1, 8
+; RV32I-NEXT:    or s0, a1, s0
+; RV32I-NEXT:    li t0, 32
+; RV32I-NEXT:    slli a1, a5, 8
+; RV32I-NEXT:    slli a0, t2, 8
+; RV32I-NEXT:    slli t5, t5, 16
+; RV32I-NEXT:    slli s1, s1, 16
+; RV32I-NEXT:    slli s0, s0, 16
+; RV32I-NEXT:    or a7, t5, t4
+; RV32I-NEXT:    or a5, s1, a6
+; RV32I-NEXT:    or a6, s0, t6
+; RV32I-NEXT:    slli a6, a6, 5
+; RV32I-NEXT:    srl t2, a7, a6
+; RV32I-NEXT:    neg t6, a6
+; RV32I-NEXT:    sll t4, a5, t6
+; RV32I-NEXT:    bltu a6, t0, .LBB11_2
+; RV32I-NEXT:  # %bb.1:
+; RV32I-NEXT:    sra t5, a5, a6
+; RV32I-NEXT:    j .LBB11_3
+; RV32I-NEXT:  .LBB11_2:
+; RV32I-NEXT:    or t5, t2, t4
+; RV32I-NEXT:  .LBB11_3:
+; RV32I-NEXT:    or a1, a1, a3
+; RV32I-NEXT:    slli t3, t3, 16
+; RV32I-NEXT:    or a3, a0, a4
+; RV32I-NEXT:    slli t1, t1, 16
+; RV32I-NEXT:    mv a0, a7
+; RV32I-NEXT:    beqz a6, .LBB11_5
+; RV32I-NEXT:  # %bb.4:
+; RV32I-NEXT:    mv a0, t5
+; RV32I-NEXT:  .LBB11_5:
+; RV32I-NEXT:    or a4, t3, a1
+; RV32I-NEXT:    or a3, t1, a3
+; RV32I-NEXT:    bltu a6, t0, .LBB11_7
+; RV32I-NEXT:  # %bb.6:
+; RV32I-NEXT:    srai a1, a5, 31
+; RV32I-NEXT:    srl t5, a3, a6
+; RV32I-NEXT:    j .LBB11_8
+; RV32I-NEXT:  .LBB11_7:
+; RV32I-NEXT:    sra a1, a5, a6
+; RV32I-NEXT:    srl t1, a4, a6
+; RV32I-NEXT:    sll t3, a3, t6
+; RV32I-NEXT:    or t5, t1, t3
+; RV32I-NEXT:  .LBB11_8:
+; RV32I-NEXT:    li t1, 64
+; RV32I-NEXT:    mv t3, a4
+; RV32I-NEXT:    beqz a6, .LBB11_10
+; RV32I-NEXT:  # %bb.9:
+; RV32I-NEXT:    mv t3, t5
+; RV32I-NEXT:  .LBB11_10:
+; RV32I-NEXT:    sub s0, t1, a6
+; RV32I-NEXT:    bltu a6, t0, .LBB11_13
+; RV32I-NEXT:  # %bb.11:
+; RV32I-NEXT:    li t5, 0
+; RV32I-NEXT:    bgeu s0, t0, .LBB11_14
+; RV32I-NEXT:  .LBB11_12:
+; RV32I-NEXT:    sll t6, a7, t6
+; RV32I-NEXT:    neg s1, s0
+; RV32I-NEXT:    srl s1, a7, s1
+; RV32I-NEXT:    or s2, s1, t4
+; RV32I-NEXT:    j .LBB11_15
+; RV32I-NEXT:  .LBB11_13:
+; RV32I-NEXT:    srl t5, a3, a6
+; RV32I-NEXT:    bltu s0, t0, .LBB11_12
+; RV32I-NEXT:  .LBB11_14:
+; RV32I-NEXT:    li t6, 0
+; RV32I-NEXT:    sll s2, a7, s0
+; RV32I-NEXT:  .LBB11_15:
+; RV32I-NEXT:    addi s1, a6, -64
+; RV32I-NEXT:    mv t4, a5
+; RV32I-NEXT:    beqz s0, .LBB11_17
+; RV32I-NEXT:  # %bb.16:
+; RV32I-NEXT:    mv t4, s2
+; RV32I-NEXT:  .LBB11_17:
+; RV32I-NEXT:    bltu s1, t0, .LBB11_19
+; RV32I-NEXT:  # %bb.18:
+; RV32I-NEXT:    sra t2, a5, s1
+; RV32I-NEXT:    bnez s1, .LBB11_20
+; RV32I-NEXT:    j .LBB11_21
+; RV32I-NEXT:  .LBB11_19:
+; RV32I-NEXT:    neg s0, s1
+; RV32I-NEXT:    sll s0, a5, s0
+; RV32I-NEXT:    or t2, t2, s0
+; RV32I-NEXT:    beqz s1, .LBB11_21
+; RV32I-NEXT:  .LBB11_20:
+; RV32I-NEXT:    mv a7, t2
+; RV32I-NEXT:  .LBB11_21:
+; RV32I-NEXT:    bltu s1, t0, .LBB11_23
+; RV32I-NEXT:  # %bb.22:
+; RV32I-NEXT:    srai t0, a5, 31
+; RV32I-NEXT:    bltu a6, t1, .LBB11_24
+; RV32I-NEXT:    j .LBB11_25
+; RV32I-NEXT:  .LBB11_23:
+; RV32I-NEXT:    sra t0, a5, a6
+; RV32I-NEXT:    bgeu a6, t1, .LBB11_25
+; RV32I-NEXT:  .LBB11_24:
+; RV32I-NEXT:    or a7, t3, t6
+; RV32I-NEXT:    or t0, t5, t4
+; RV32I-NEXT:  .LBB11_25:
+; RV32I-NEXT:    bnez a6, .LBB11_29
+; RV32I-NEXT:  # %bb.26:
+; RV32I-NEXT:    bltu a6, t1, .LBB11_28
+; RV32I-NEXT:  .LBB11_27:
+; RV32I-NEXT:    srai a0, a5, 31
+; RV32I-NEXT:    mv a1, a0
+; RV32I-NEXT:  .LBB11_28:
+; RV32I-NEXT:    srli a5, a4, 16
+; RV32I-NEXT:    lui a6, 16
+; RV32I-NEXT:    srli a7, a4, 24
+; RV32I-NEXT:    srli t0, a3, 16
+; RV32I-NEXT:    srli t1, a3, 24
+; RV32I-NEXT:    srli t2, a0, 16
+; RV32I-NEXT:    srli t3, a0, 24
+; RV32I-NEXT:    srli t4, a1, 16
+; RV32I-NEXT:    srli t5, a1, 24
+; RV32I-NEXT:    addi a6, a6, -1
+; RV32I-NEXT:    and t6, a4, a6
+; RV32I-NEXT:    srli t6, t6, 8
+; RV32I-NEXT:    sb a4, 0(a2)
+; RV32I-NEXT:    sb t6, 1(a2)
+; RV32I-NEXT:    sb a5, 2(a2)
+; RV32I-NEXT:    sb a7, 3(a2)
+; RV32I-NEXT:    and a4, a3, a6
+; RV32I-NEXT:    srli a4, a4, 8
+; RV32I-NEXT:    sb a3, 4(a2)
+; RV32I-NEXT:    sb a4, 5(a2)
+; RV32I-NEXT:    sb t0, 6(a2)
+; RV32I-NEXT:    sb t1, 7(a2)
+; RV32I-NEXT:    and a3, a0, a6
+; RV32I-NEXT:    and a4, a1, a6
+; RV32I-NEXT:    srli a3, a3, 8
+; RV32I-NEXT:    srli a4, a4, 8
+; RV32I-NEXT:    sb a0, 8(a2)
+; RV32I-NEXT:    sb a3, 9(a2)
+; RV32I-NEXT:    sb t2, 10(a2)
+; RV32I-NEXT:    sb t3, 11(a2)
+; RV32I-NEXT:    sb a1, 12(a2)
+; RV32I-NEXT:    sb a4, 13(a2)
+; RV32I-NEXT:    sb t4, 14(a2)
+; RV32I-NEXT:    sb t5, 15(a2)
+; RV32I-NEXT:    lw s0, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s1, 8(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s2, 4(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    addi sp, sp, 16
+; RV32I-NEXT:    ret
+; RV32I-NEXT:  .LBB11_29:
+; RV32I-NEXT:    mv a4, a7
+; RV32I-NEXT:    mv a3, t0
+; RV32I-NEXT:    bgeu a6, t1, .LBB11_27
+; RV32I-NEXT:    j .LBB11_28
+  %src = load i128, ptr %src.ptr, align 1
+  %wordOff = load i128, ptr %wordOff.ptr, align 1
+  %bitOff = shl i128 %wordOff, 5
+  %res = ashr i128 %src, %bitOff
+  store i128 %res, ptr %dst, align 1
+  ret void
+}
+
+define void @lshr_32bytes(ptr %src.ptr, ptr %byteOff.ptr, ptr %dst) nounwind {
+; RV64I-LABEL: lshr_32bytes:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -96
+; RV64I-NEXT:    sd s0, 88(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s1, 80(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s2, 72(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s3, 64(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s4, 56(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s5, 48(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s6, 40(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s7, 32(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s8, 24(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s9, 16(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s10, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s11, 0(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    lbu a3, 0(a0)
+; RV64I-NEXT:    lbu a4, 1(a0)
+; RV64I-NEXT:    lbu a5, 2(a0)
+; RV64I-NEXT:    lbu a6, 3(a0)
+; RV64I-NEXT:    lbu a7, 4(a0)
+; RV64I-NEXT:    lbu t0, 5(a0)
+; RV64I-NEXT:    lbu t1, 6(a0)
+; RV64I-NEXT:    lbu t2, 7(a0)
+; RV64I-NEXT:    lbu t3, 8(a0)
+; RV64I-NEXT:    lbu t4, 9(a0)
+; RV64I-NEXT:    lbu t5, 10(a0)
+; RV64I-NEXT:    lbu t6, 11(a0)
+; RV64I-NEXT:    lbu s0, 12(a0)
+; RV64I-NEXT:    lbu s1, 13(a0)
+; RV64I-NEXT:    lbu s2, 14(a0)
+; RV64I-NEXT:    lbu s3, 15(a0)
+; RV64I-NEXT:    lbu s4, 16(a0)
+; RV64I-NEXT:    lbu s5, 17(a0)
+; RV64I-NEXT:    lbu s6, 18(a0)
+; RV64I-NEXT:    lbu s7, 19(a0)
+; RV64I-NEXT:    slli a4, a4, 8
+; RV64I-NEXT:    slli a6, a6, 8
+; RV64I-NEXT:    slli t0, t0, 8
+; RV64I-NEXT:    or a3, a4, a3
+; RV64I-NEXT:    or a5, a6, a5
+; RV64I-NEXT:    or a7, t0, a7
+; RV64I-NEXT:    lbu s8, 20(a0)
+; RV64I-NEXT:    lbu s9, 21(a0)
+; RV64I-NEXT:    lbu s10, 22(a0)
+; RV64I-NEXT:    lbu s11, 23(a0)
+; RV64I-NEXT:    slli t2, t2, 8
+; RV64I-NEXT:    slli t4, t4, 8
+; RV64I-NEXT:    slli t6, t6, 8
+; RV64I-NEXT:    slli s1, s1, 8
+; RV64I-NEXT:    or t1, t2, t1
+; RV64I-NEXT:    or a4, t4, t3
+; RV64I-NEXT:    or a6, t6, t5
+; RV64I-NEXT:    or t0, s1, s0
+; RV64I-NEXT:    lbu t5, 24(a0)
+; RV64I-NEXT:    lbu t6, 25(a0)
+; RV64I-NEXT:    lbu s0, 26(a0)
+; RV64I-NEXT:    lbu s1, 27(a0)
+; RV64I-NEXT:    slli s3, s3, 8
+; RV64I-NEXT:    slli s5, s5, 8
+; RV64I-NEXT:    slli s7, s7, 8
+; RV64I-NEXT:    or t4, s3, s2
+; RV64I-NEXT:    or t2, s5, s4
+; RV64I-NEXT:    or t3, s7, s6
+; RV64I-NEXT:    lbu s2, 28(a0)
+; RV64I-NEXT:    lbu s3, 29(a0)
+; RV64I-NEXT:    lbu s4, 30(a0)
+; RV64I-NEXT:    lbu a0, 31(a0)
+; RV64I-NEXT:    slli s9, s9, 8
+; RV64I-NEXT:    slli s11, s11, 8
+; RV64I-NEXT:    slli t6, t6, 8
+; RV64I-NEXT:    slli s1, s1, 8
+; RV64I-NEXT:    or s5, s9, s8
+; RV64I-NEXT:    or s6, s11, s10
+; RV64I-NEXT:    or t5, t6, t5
+; RV64I-NEXT:    or s0, s1, s0
+; RV64I-NEXT:    lbu t6, 0(a1)
+; RV64I-NEXT:    lbu s1, 1(a1)
+; RV64I-NEXT:    lbu s7, 2(a1)
+; RV64I-NEXT:    lbu s8, 3(a1)
+; RV64I-NEXT:    slli s3, s3, 8
+; RV64I-NEXT:    slli a0, a0, 8
+; RV64I-NEXT:    slli s1, s1, 8
+; RV64I-NEXT:    or s2, s3, s2
+; RV64I-NEXT:    or s3, a0, s4
+; RV64I-NEXT:    or t6, s1, t6
+; RV64I-NEXT:    lbu a0, 4(a1)
+; RV64I-NEXT:    lbu s1, 5(a1)
+; RV64I-NEXT:    lbu s4, 6(a1)
+; RV64I-NEXT:    lbu a1, 7(a1)
+; RV64I-NEXT:    slli s8, s8, 8
+; RV64I-NEXT:    or s7, s8, s7
+; RV64I-NEXT:    slli s1, s1, 8
+; RV64I-NEXT:    or s1, s1, a0
+; RV64I-NEXT:    slli a1, a1, 8
+; RV64I-NEXT:    or s4, a1, s4
+; RV64I-NEXT:    slli t1, t1, 16
+; RV64I-NEXT:    or a1, t1, a7
+; RV64I-NEXT:    slli t4, t4, 16
+; RV64I-NEXT:    or a0, t4, t0
+; RV64I-NEXT:    slli t3, t3, 16
+; RV64I-NEXT:    or t0, t3, t2
+; RV64I-NEXT:    slli s6, s6, 16
+; RV64I-NEXT:    or t1, s6, s5
+; RV64I-NEXT:    slli s0, s0, 16
+; RV64I-NEXT:    or t3, s0, t5
+; RV64I-NEXT:    slli s3, s3, 16
+; RV64I-NEXT:    or t5, s3, s2
+; RV64I-NEXT:    slli s7, s7, 16
+; RV64I-NEXT:    or t6, s7, t6
+; RV64I-NEXT:    slli s4, s4, 16
+; RV64I-NEXT:    or s0, s4, s1
+; RV64I-NEXT:    li a7, 64
+; RV64I-NEXT:    slli t4, a5, 16
+; RV64I-NEXT:    slli t2, a6, 16
+; RV64I-NEXT:    slli t1, t1, 32
+; RV64I-NEXT:    slli t5, t5, 32
+; RV64I-NEXT:    slli s0, s0, 32
+; RV64I-NEXT:    or a6, t1, t0
+; RV64I-NEXT:    or t0, t5, t3
+; RV64I-NEXT:    or a5, s0, t6
+; RV64I-NEXT:    slli a5, a5, 3
+; RV64I-NEXT:    subw t1, a5, a7
+; RV64I-NEXT:    negw t5, a5
+; RV64I-NEXT:    sll t3, t0, t5
+; RV64I-NEXT:    bltu a5, a7, .LBB12_2
+; RV64I-NEXT:  # %bb.1:
+; RV64I-NEXT:    srl t6, t0, t1
+; RV64I-NEXT:    j .LBB12_3
+; RV64I-NEXT:  .LBB12_2:
+; RV64I-NEXT:    srl t6, a6, a5
+; RV64I-NEXT:    or t6, t6, t3
+; RV64I-NEXT:  .LBB12_3:
+; RV64I-NEXT:    or a3, t4, a3
+; RV64I-NEXT:    slli t4, a1, 32
+; RV64I-NEXT:    or t2, t2, a4
+; RV64I-NEXT:    slli a0, a0, 32
+; RV64I-NEXT:    mv a1, a6
+; RV64I-NEXT:    beqz a5, .LBB12_5
+; RV64I-NEXT:  # %bb.4:
+; RV64I-NEXT:    mv a1, t6
+; RV64I-NEXT:  .LBB12_5:
+; RV64I-NEXT:    or a4, t4, a3
+; RV64I-NEXT:    or a3, a0, t2
+; RV64I-NEXT:    bltu a5, a7, .LBB12_7
+; RV64I-NEXT:  # %bb.6:
+; RV64I-NEXT:    li a0, 0
+; RV64I-NEXT:    srl t4, a3, t1
+; RV64I-NEXT:    j .LBB12_8
+; RV64I-NEXT:  .LBB12_7:
+; RV64I-NEXT:    srl a0, t0, a5
+; RV64I-NEXT:    srl t1, a4, a5
+; RV64I-NEXT:    sll t2, a3, t5
+; RV64I-NEXT:    or t4, t1, t2
+; RV64I-NEXT:  .LBB12_8:
+; RV64I-NEXT:    li t1, 128
+; RV64I-NEXT:    mv t2, a4
+; RV64I-NEXT:    beqz a5, .LBB12_10
+; RV64I-NEXT:  # %bb.9:
+; RV64I-NEXT:    mv t2, t4
+; RV64I-NEXT:  .LBB12_10:
+; RV64I-NEXT:    sub t6, t1, a5
+; RV64I-NEXT:    bltu a5, a7, .LBB12_13
+; RV64I-NEXT:  # %bb.11:
+; RV64I-NEXT:    li t4, 0
+; RV64I-NEXT:    bgeu t6, a7, .LBB12_14
+; RV64I-NEXT:  .LBB12_12:
+; RV64I-NEXT:    sll t5, a6, t5
+; RV64I-NEXT:    negw s0, t6
+; RV64I-NEXT:    srl s0, a6, s0
+; RV64I-NEXT:    or s1, s0, t3
+; RV64I-NEXT:    j .LBB12_15
+; RV64I-NEXT:  .LBB12_13:
+; RV64I-NEXT:    srl t4, a3, a5
+; RV64I-NEXT:    bltu t6, a7, .LBB12_12
+; RV64I-NEXT:  .LBB12_14:
+; RV64I-NEXT:    li t5, 0
+; RV64I-NEXT:    subw t3, t6, a7
+; RV64I-NEXT:    sll s1, a6, t3
+; RV64I-NEXT:  .LBB12_15:
+; RV64I-NEXT:    sub s0, a5, t1
+; RV64I-NEXT:    mv t3, t0
+; RV64I-NEXT:    beqz t6, .LBB12_17
+; RV64I-NEXT:  # %bb.16:
+; RV64I-NEXT:    mv t3, s1
+; RV64I-NEXT:  .LBB12_17:
+; RV64I-NEXT:    bltu s0, a7, .LBB12_19
+; RV64I-NEXT:  # %bb.18:
+; RV64I-NEXT:    subw t6, s0, a7
+; RV64I-NEXT:    srl t6, t0, t6
+; RV64I-NEXT:    bnez s0, .LBB12_20
+; RV64I-NEXT:    j .LBB12_21
+; RV64I-NEXT:  .LBB12_19:
+; RV64I-NEXT:    srl t6, a6, s0
+; RV64I-NEXT:    negw s1, s0
+; RV64I-NEXT:    sll s1, t0, s1
+; RV64I-NEXT:    or t6, t6, s1
+; RV64I-NEXT:    beqz s0, .LBB12_21
+; RV64I-NEXT:  .LBB12_20:
+; RV64I-NEXT:    mv a6, t6
+; RV64I-NEXT:  .LBB12_21:
+; RV64I-NEXT:    bltu s0, a7, .LBB12_23
+; RV64I-NEXT:  # %bb.22:
+; RV64I-NEXT:    li a7, 0
+; RV64I-NEXT:    bltu a5, t1, .LBB12_24
+; RV64I-NEXT:    j .LBB12_25
+; RV64I-NEXT:  .LBB12_23:
+; RV64I-NEXT:    srl a7, t0, s0
+; RV64I-NEXT:    bgeu a5, t1, .LBB12_25
+; RV64I-NEXT:  .LBB12_24:
+; RV64I-NEXT:    or a6, t2, t5
+; RV64I-NEXT:    or a7, t4, t3
+; RV64I-NEXT:  .LBB12_25:
+; RV64I-NEXT:    bnez a5, .LBB12_29
+; RV64I-NEXT:  # %bb.26:
+; RV64I-NEXT:    bltu a5, t1, .LBB12_28
+; RV64I-NEXT:  .LBB12_27:
+; RV64I-NEXT:    li a1, 0
+; RV64I-NEXT:    li a0, 0
+; RV64I-NEXT:  .LBB12_28:
+; RV64I-NEXT:    srli a5, a4, 32
+; RV64I-NEXT:    srliw a6, a4, 16
+; RV64I-NEXT:    lui t2, 16
+; RV64I-NEXT:    srliw t1, a4, 24
+; RV64I-NEXT:    srli t0, a4, 48
+; RV64I-NEXT:    srli t5, a4, 56
+; RV64I-NEXT:    srli a7, a3, 32
+; RV64I-NEXT:    srliw t4, a3, 16
+; RV64I-NEXT:    srliw s0, a3, 24
+; RV64I-NEXT:    srli t6, a3, 48
+; RV64I-NEXT:    srli s3, a3, 56
+; RV64I-NEXT:    srli t3, a1, 32
+; RV64I-NEXT:    srliw s2, a1, 16
+; RV64I-NEXT:    srliw s6, a1, 24
+; RV64I-NEXT:    srli s4, a1, 48
+; RV64I-NEXT:    srli s7, a1, 56
+; RV64I-NEXT:    srli s1, a0, 32
+; RV64I-NEXT:    srliw s5, a0, 16
+; RV64I-NEXT:    srliw s8, a0, 24
+; RV64I-NEXT:    srli s9, a0, 48
+; RV64I-NEXT:    srli s10, a0, 56
+; RV64I-NEXT:    addi t2, t2, -1
+; RV64I-NEXT:    and s11, a4, t2
+; RV64I-NEXT:    srli s11, s11, 8
+; RV64I-NEXT:    sb a4, 0(a2)
+; RV64I-NEXT:    sb s11, 1(a2)
+; RV64I-NEXT:    sb a6, 2(a2)
+; RV64I-NEXT:    sb t1, 3(a2)
+; RV64I-NEXT:    and a4, a5, t2
+; RV64I-NEXT:    srli a4, a4, 8
+; RV64I-NEXT:    sb a5, 4(a2)
+; RV64I-NEXT:    sb a4, 5(a2)
+; RV64I-NEXT:    sb t0, 6(a2)
+; RV64I-NEXT:    sb t5, 7(a2)
+; RV64I-NEXT:    and a4, a3, t2
+; RV64I-NEXT:    srli a4, a4, 8
+; RV64I-NEXT:    sb a3, 8(a2)
+; RV64I-NEXT:    sb a4, 9(a2)
+; RV64I-NEXT:    sb t4, 10(a2)
+; RV64I-NEXT:    sb s0, 11(a2)
+; RV64I-NEXT:    and a3, a7, t2
+; RV64I-NEXT:    srli a3, a3, 8
+; RV64I-NEXT:    sb a7, 12(a2)
+; RV64I-NEXT:    sb a3, 13(a2)
+; RV64I-NEXT:    sb t6, 14(a2)
+; RV64I-NEXT:    sb s3, 15(a2)
+; RV64I-NEXT:    and a3, a1, t2
+; RV64I-NEXT:    srli a3, a3, 8
+; RV64I-NEXT:    sb a1, 16(a2)
+; RV64I-NEXT:    sb a3, 17(a2)
+; RV64I-NEXT:    sb s2, 18(a2)
+; RV64I-NEXT:    sb s6, 19(a2)
+; RV64I-NEXT:    and a1, t3, t2
+; RV64I-NEXT:    srli a1, a1, 8
+; RV64I-NEXT:    sb t3, 20(a2)
+; RV64I-NEXT:    sb a1, 21(a2)
+; RV64I-NEXT:    sb s4, 22(a2)
+; RV64I-NEXT:    sb s7, 23(a2)
+; RV64I-NEXT:    and a1, a0, t2
+; RV64I-NEXT:    and a3, s1, t2
+; RV64I-NEXT:    srli a1, a1, 8
+; RV64I-NEXT:    srli a3, a3, 8
+; RV64I-NEXT:    sb a0, 24(a2)
+; RV64I-NEXT:    sb a1, 25(a2)
+; RV64I-NEXT:    sb s5, 26(a2)
+; RV64I-NEXT:    sb s8, 27(a2)
+; RV64I-NEXT:    sb s1, 28(a2)
+; RV64I-NEXT:    sb a3, 29(a2)
+; RV64I-NEXT:    sb s9, 30(a2)
+; RV64I-NEXT:    sb s10, 31(a2)
+; RV64I-NEXT:    ld s0, 88(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s1, 80(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s2, 72(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s3, 64(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s4, 56(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s5, 48(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s6, 40(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s7, 32(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s8, 24(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s9, 16(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s10, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s11, 0(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    addi sp, sp, 96
+; RV64I-NEXT:    ret
+; RV64I-NEXT:  .LBB12_29:
+; RV64I-NEXT:    mv a4, a6
+; RV64I-NEXT:    mv a3, a7
+; RV64I-NEXT:    bgeu a5, t1, .LBB12_27
+; RV64I-NEXT:    j .LBB12_28
+;
+; RV32I-LABEL: lshr_32bytes:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    addi sp, sp, -96
+; RV32I-NEXT:    sw ra, 92(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s0, 88(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s1, 84(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s2, 80(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s3, 76(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s4, 72(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s5, 68(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s6, 64(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s7, 60(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s8, 56(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s9, 52(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s10, 48(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s11, 44(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    lbu a3, 16(a0)
+; RV32I-NEXT:    lbu a4, 17(a0)
+; RV32I-NEXT:    lbu a5, 18(a0)
+; RV32I-NEXT:    lbu a6, 19(a0)
+; RV32I-NEXT:    lbu a7, 20(a0)
+; RV32I-NEXT:    lbu t0, 21(a0)
+; RV32I-NEXT:    lbu t1, 22(a0)
+; RV32I-NEXT:    lbu t2, 23(a0)
+; RV32I-NEXT:    lbu t3, 24(a0)
+; RV32I-NEXT:    lbu t4, 25(a0)
+; RV32I-NEXT:    lbu t5, 26(a0)
+; RV32I-NEXT:    lbu t6, 27(a0)
+; RV32I-NEXT:    slli a4, a4, 8
+; RV32I-NEXT:    slli a6, a6, 8
+; RV32I-NEXT:    slli t0, t0, 8
+; RV32I-NEXT:    slli t2, t2, 8
+; RV32I-NEXT:    or a3, a4, a3
+; RV32I-NEXT:    or a6, a6, a5
+; RV32I-NEXT:    or a5, t0, a7
+; RV32I-NEXT:    or a4, t2, t1
+; RV32I-NEXT:    lbu a7, 28(a0)
+; RV32I-NEXT:    lbu t0, 29(a0)
+; RV32I-NEXT:    lbu t1, 30(a0)
+; RV32I-NEXT:    lbu t2, 31(a0)
+; RV32I-NEXT:    slli t4, t4, 8
+; RV32I-NEXT:    slli t6, t6, 8
+; RV32I-NEXT:    slli t0, t0, 8
+; RV32I-NEXT:    or t4, t4, t3
+; RV32I-NEXT:    or t5, t6, t5
+; RV32I-NEXT:    or a7, t0, a7
+; RV32I-NEXT:    lbu t0, 0(a1)
+; RV32I-NEXT:    lbu t3, 1(a1)
+; RV32I-NEXT:    lbu t6, 2(a1)
+; RV32I-NEXT:    lbu a1, 3(a1)
+; RV32I-NEXT:    slli t2, t2, 8
+; RV32I-NEXT:    or t1, t2, t1
+; RV32I-NEXT:    slli t3, t3, 8
+; RV32I-NEXT:    or t0, t3, t0
+; RV32I-NEXT:    slli a1, a1, 8
+; RV32I-NEXT:    or t2, a1, t6
+; RV32I-NEXT:    li t3, 32
+; RV32I-NEXT:    slli a6, a6, 16
+; RV32I-NEXT:    slli a1, a4, 16
+; RV32I-NEXT:    slli t5, t5, 16
+; RV32I-NEXT:    slli a4, t1, 16
+; RV32I-NEXT:    slli t2, t2, 16
+; RV32I-NEXT:    or t1, t5, t4
+; RV32I-NEXT:    or t5, a4, a7
+; RV32I-NEXT:    or a4, t2, t0
+; RV32I-NEXT:    slli a4, a4, 3
+; RV32I-NEXT:    srl s0, t1, a4
+; RV32I-NEXT:    neg s6, a4
+; RV32I-NEXT:    sll t4, t5, s6
+; RV32I-NEXT:    bltu a4, t3, .LBB12_2
+; RV32I-NEXT:  # %bb.1:
+; RV32I-NEXT:    srl a7, t5, a4
+; RV32I-NEXT:    j .LBB12_3
+; RV32I-NEXT:  .LBB12_2:
+; RV32I-NEXT:    or a7, s0, t4
+; RV32I-NEXT:  .LBB12_3:
+; RV32I-NEXT:    or t0, a6, a3
+; RV32I-NEXT:    or a6, a1, a5
+; RV32I-NEXT:    mv a1, t1
+; RV32I-NEXT:    beqz a4, .LBB12_5
+; RV32I-NEXT:  # %bb.4:
+; RV32I-NEXT:    mv a1, a7
+; RV32I-NEXT:  .LBB12_5:
+; RV32I-NEXT:    srl a3, t0, a4
+; RV32I-NEXT:    sll a5, a6, s6
+; RV32I-NEXT:    sw a3, 32(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    bltu a4, t3, .LBB12_7
+; RV32I-NEXT:  # %bb.6:
+; RV32I-NEXT:    li ra, 0
+; RV32I-NEXT:    srl a3, a6, a4
+; RV32I-NEXT:    j .LBB12_8
+; RV32I-NEXT:  .LBB12_7:
+; RV32I-NEXT:    srl ra, t5, a4
+; RV32I-NEXT:    or a3, a3, a5
+; RV32I-NEXT:  .LBB12_8:
+; RV32I-NEXT:    li t6, 64
+; RV32I-NEXT:    mv a7, t0
+; RV32I-NEXT:    beqz a4, .LBB12_10
+; RV32I-NEXT:  # %bb.9:
+; RV32I-NEXT:    mv a7, a3
+; RV32I-NEXT:  .LBB12_10:
+; RV32I-NEXT:    sw a5, 16(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sub s7, t6, a4
+; RV32I-NEXT:    bltu a4, t3, .LBB12_12
+; RV32I-NEXT:  # %bb.11:
+; RV32I-NEXT:    li a5, 0
+; RV32I-NEXT:    j .LBB12_13
+; RV32I-NEXT:  .LBB12_12:
+; RV32I-NEXT:    srl a5, a6, a4
+; RV32I-NEXT:  .LBB12_13:
+; RV32I-NEXT:    neg s10, s7
+; RV32I-NEXT:    sw t4, 20(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    bltu s7, t3, .LBB12_15
+; RV32I-NEXT:  # %bb.14:
+; RV32I-NEXT:    li t2, 0
+; RV32I-NEXT:    sll a3, t1, s7
+; RV32I-NEXT:    j .LBB12_16
+; RV32I-NEXT:  .LBB12_15:
+; RV32I-NEXT:    sll t2, t1, s6
+; RV32I-NEXT:    srl a3, t1, s10
+; RV32I-NEXT:    or a3, a3, t4
+; RV32I-NEXT:  .LBB12_16:
+; RV32I-NEXT:    addi s9, a4, -64
+; RV32I-NEXT:    mv t4, t5
+; RV32I-NEXT:    beqz s7, .LBB12_18
+; RV32I-NEXT:  # %bb.17:
+; RV32I-NEXT:    mv t4, a3
+; RV32I-NEXT:  .LBB12_18:
+; RV32I-NEXT:    neg s11, s9
+; RV32I-NEXT:    sw s0, 36(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    bltu s9, t3, .LBB12_20
+; RV32I-NEXT:  # %bb.19:
+; RV32I-NEXT:    srl s2, t5, s9
+; RV32I-NEXT:    j .LBB12_21
+; RV32I-NEXT:  .LBB12_20:
+; RV32I-NEXT:    sll a3, t5, s11
+; RV32I-NEXT:    or s2, s0, a3
+; RV32I-NEXT:  .LBB12_21:
+; RV32I-NEXT:    lbu s1, 11(a0)
+; RV32I-NEXT:    lbu a3, 15(a0)
+; RV32I-NEXT:    mv s0, t1
+; RV32I-NEXT:    beqz s9, .LBB12_23
+; RV32I-NEXT:  # %bb.22:
+; RV32I-NEXT:    mv s0, s2
+; RV32I-NEXT:  .LBB12_23:
+; RV32I-NEXT:    lbu s4, 9(a0)
+; RV32I-NEXT:    lbu s2, 10(a0)
+; RV32I-NEXT:    lbu s5, 13(a0)
+; RV32I-NEXT:    lbu s8, 14(a0)
+; RV32I-NEXT:    slli s3, s1, 8
+; RV32I-NEXT:    slli a3, a3, 8
+; RV32I-NEXT:    sw ra, 40(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    bltu s9, t3, .LBB12_25
+; RV32I-NEXT:  # %bb.24:
+; RV32I-NEXT:    li s1, 0
+; RV32I-NEXT:    j .LBB12_26
+; RV32I-NEXT:  .LBB12_25:
+; RV32I-NEXT:    srl s1, t5, a4
+; RV32I-NEXT:  .LBB12_26:
+; RV32I-NEXT:    or s2, s3, s2
+; RV32I-NEXT:    lbu ra, 8(a0)
+; RV32I-NEXT:    lbu s3, 12(a0)
+; RV32I-NEXT:    slli s4, s4, 8
+; RV32I-NEXT:    slli s5, s5, 8
+; RV32I-NEXT:    or s8, a3, s8
+; RV32I-NEXT:    bgeu a4, t6, .LBB12_28
+; RV32I-NEXT:  # %bb.27:
+; RV32I-NEXT:    or s0, a7, t2
+; RV32I-NEXT:    or s1, a5, t4
+; RV32I-NEXT:  .LBB12_28:
+; RV32I-NEXT:    lbu a3, 3(a0)
+; RV32I-NEXT:    lbu t2, 7(a0)
+; RV32I-NEXT:    or a5, s4, ra
+; RV32I-NEXT:    slli t4, s2, 16
+; RV32I-NEXT:    or s2, s5, s3
+; RV32I-NEXT:    slli s3, s8, 16
+; RV32I-NEXT:    mv s4, t0
+; RV32I-NEXT:    mv a7, a6
+; RV32I-NEXT:    beqz a4, .LBB12_30
+; RV32I-NEXT:  # %bb.29:
+; RV32I-NEXT:    mv s4, s0
+; RV32I-NEXT:    mv a7, s1
+; RV32I-NEXT:  .LBB12_30:
+; RV32I-NEXT:    slli s5, a3, 8
+; RV32I-NEXT:    lbu ra, 1(a0)
+; RV32I-NEXT:    lbu a3, 2(a0)
+; RV32I-NEXT:    lbu s1, 5(a0)
+; RV32I-NEXT:    lbu s0, 6(a0)
+; RV32I-NEXT:    slli s8, t2, 8
+; RV32I-NEXT:    or t4, t4, a5
+; RV32I-NEXT:    or t2, s3, s2
+; RV32I-NEXT:    bltu a4, t6, .LBB12_32
+; RV32I-NEXT:  # %bb.31:
+; RV32I-NEXT:    li a1, 0
+; RV32I-NEXT:    sw zero, 40(sp) # 4-byte Folded Spill
+; RV32I-NEXT:  .LBB12_32:
+; RV32I-NEXT:    slli s3, ra, 8
+; RV32I-NEXT:    or a5, s5, a3
+; RV32I-NEXT:    lbu s5, 0(a0)
+; RV32I-NEXT:    lbu a0, 4(a0)
+; RV32I-NEXT:    slli s1, s1, 8
+; RV32I-NEXT:    or a3, s8, s0
+; RV32I-NEXT:    srl s2, t4, a4
+; RV32I-NEXT:    sll ra, t2, s6
+; RV32I-NEXT:    bltu a4, t3, .LBB12_34
+; RV32I-NEXT:  # %bb.33:
+; RV32I-NEXT:    srl s0, t2, a4
+; RV32I-NEXT:    j .LBB12_35
+; RV32I-NEXT:  .LBB12_34:
+; RV32I-NEXT:    or s0, s2, ra
+; RV32I-NEXT:  .LBB12_35:
+; RV32I-NEXT:    or s3, s3, s5
+; RV32I-NEXT:    slli a5, a5, 16
+; RV32I-NEXT:    or a0, s1, a0
+; RV32I-NEXT:    slli a3, a3, 16
+; RV32I-NEXT:    mv s5, t4
+; RV32I-NEXT:    beqz a4, .LBB12_37
+; RV32I-NEXT:  # %bb.36:
+; RV32I-NEXT:    mv s5, s0
+; RV32I-NEXT:  .LBB12_37:
+; RV32I-NEXT:    or s0, a5, s3
+; RV32I-NEXT:    or a0, a3, a0
+; RV32I-NEXT:    bltu a4, t3, .LBB12_39
+; RV32I-NEXT:  # %bb.38:
+; RV32I-NEXT:    li s8, 0
+; RV32I-NEXT:    srl a3, a0, a4
+; RV32I-NEXT:    mv a5, s0
+; RV32I-NEXT:    bnez a4, .LBB12_40
+; RV32I-NEXT:    j .LBB12_41
+; RV32I-NEXT:  .LBB12_39:
+; RV32I-NEXT:    srl s8, t2, a4
+; RV32I-NEXT:    srl a3, s0, a4
+; RV32I-NEXT:    sll a5, a0, s6
+; RV32I-NEXT:    or a3, a3, a5
+; RV32I-NEXT:    mv a5, s0
+; RV32I-NEXT:    beqz a4, .LBB12_41
+; RV32I-NEXT:  .LBB12_40:
+; RV32I-NEXT:    mv a5, a3
+; RV32I-NEXT:  .LBB12_41:
+; RV32I-NEXT:    bltu a4, t3, .LBB12_44
+; RV32I-NEXT:  # %bb.42:
+; RV32I-NEXT:    li s1, 0
+; RV32I-NEXT:    bgeu s7, t3, .LBB12_45
+; RV32I-NEXT:  .LBB12_43:
+; RV32I-NEXT:    sll s3, t4, s6
+; RV32I-NEXT:    srl a3, t4, s10
+; RV32I-NEXT:    or a3, a3, ra
+; RV32I-NEXT:    mv s10, t2
+; RV32I-NEXT:    bnez s7, .LBB12_46
+; RV32I-NEXT:    j .LBB12_47
+; RV32I-NEXT:  .LBB12_44:
+; RV32I-NEXT:    srl s1, a0, a4
+; RV32I-NEXT:    bltu s7, t3, .LBB12_43
+; RV32I-NEXT:  .LBB12_45:
+; RV32I-NEXT:    li s3, 0
+; RV32I-NEXT:    sll a3, t4, s7
+; RV32I-NEXT:    mv s10, t2
+; RV32I-NEXT:    beqz s7, .LBB12_47
+; RV32I-NEXT:  .LBB12_46:
+; RV32I-NEXT:    mv s10, a3
+; RV32I-NEXT:  .LBB12_47:
+; RV32I-NEXT:    bltu s9, t3, .LBB12_49
+; RV32I-NEXT:  # %bb.48:
+; RV32I-NEXT:    srl a3, t2, s9
+; RV32I-NEXT:    mv s2, t4
+; RV32I-NEXT:    bnez s9, .LBB12_50
+; RV32I-NEXT:    j .LBB12_51
+; RV32I-NEXT:  .LBB12_49:
+; RV32I-NEXT:    sll a3, t2, s11
+; RV32I-NEXT:    or a3, s2, a3
+; RV32I-NEXT:    mv s2, t4
+; RV32I-NEXT:    beqz s9, .LBB12_51
+; RV32I-NEXT:  .LBB12_50:
+; RV32I-NEXT:    mv s2, a3
+; RV32I-NEXT:  .LBB12_51:
+; RV32I-NEXT:    bltu s9, t3, .LBB12_53
+; RV32I-NEXT:  # %bb.52:
+; RV32I-NEXT:    li s7, 0
+; RV32I-NEXT:    bltu a4, t6, .LBB12_54
+; RV32I-NEXT:    j .LBB12_55
+; RV32I-NEXT:  .LBB12_53:
+; RV32I-NEXT:    srl s7, t2, a4
+; RV32I-NEXT:    bgeu a4, t6, .LBB12_55
+; RV32I-NEXT:  .LBB12_54:
+; RV32I-NEXT:    or s2, a5, s3
+; RV32I-NEXT:    or s7, s1, s10
+; RV32I-NEXT:  .LBB12_55:
+; RV32I-NEXT:    li a3, 128
+; RV32I-NEXT:    mv a5, s0
+; RV32I-NEXT:    mv s1, a0
+; RV32I-NEXT:    beqz a4, .LBB12_57
+; RV32I-NEXT:  # %bb.56:
+; RV32I-NEXT:    mv a5, s2
+; RV32I-NEXT:    mv s1, s7
+; RV32I-NEXT:  .LBB12_57:
+; RV32I-NEXT:    sw a5, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sub s2, a3, a4
+; RV32I-NEXT:    bltu a4, t6, .LBB12_59
+; RV32I-NEXT:  # %bb.58:
+; RV32I-NEXT:    li s5, 0
+; RV32I-NEXT:    li s8, 0
+; RV32I-NEXT:  .LBB12_59:
+; RV32I-NEXT:    neg s3, s2
+; RV32I-NEXT:    srl a5, t0, s3
+; RV32I-NEXT:    sw s8, 24(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s1, 8(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    bltu s2, t3, .LBB12_61
+; RV32I-NEXT:  # %bb.60:
+; RV32I-NEXT:    li s10, 0
+; RV32I-NEXT:    sll a3, t0, s2
+; RV32I-NEXT:    j .LBB12_62
+; RV32I-NEXT:  .LBB12_61:
+; RV32I-NEXT:    sll s10, t0, s6
+; RV32I-NEXT:    lw a3, 16(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    or a3, a5, a3
+; RV32I-NEXT:  .LBB12_62:
+; RV32I-NEXT:    sub s1, t6, s2
+; RV32I-NEXT:    mv s8, a6
+; RV32I-NEXT:    beqz s2, .LBB12_64
+; RV32I-NEXT:  # %bb.63:
+; RV32I-NEXT:    mv s8, a3
+; RV32I-NEXT:  .LBB12_64:
+; RV32I-NEXT:    bltu s1, t3, .LBB12_66
+; RV32I-NEXT:  # %bb.65:
+; RV32I-NEXT:    srl a3, a6, s1
+; RV32I-NEXT:    mv a5, t0
+; RV32I-NEXT:    bnez s1, .LBB12_67
+; RV32I-NEXT:    j .LBB12_68
+; RV32I-NEXT:  .LBB12_66:
+; RV32I-NEXT:    neg a3, s1
+; RV32I-NEXT:    sll a3, a6, a3
+; RV32I-NEXT:    or a3, a5, a3
+; RV32I-NEXT:    mv a5, t0
+; RV32I-NEXT:    beqz s1, .LBB12_68
+; RV32I-NEXT:  .LBB12_67:
+; RV32I-NEXT:    mv a5, a3
+; RV32I-NEXT:  .LBB12_68:
+; RV32I-NEXT:    bltu s1, t3, .LBB12_71
+; RV32I-NEXT:  # %bb.69:
+; RV32I-NEXT:    li s1, 0
+; RV32I-NEXT:    sw s5, 28(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    bgeu s2, t3, .LBB12_72
+; RV32I-NEXT:  .LBB12_70:
+; RV32I-NEXT:    sll s6, t1, s6
+; RV32I-NEXT:    srl a3, t1, s3
+; RV32I-NEXT:    lw s3, 20(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    or a3, a3, s3
+; RV32I-NEXT:    j .LBB12_73
+; RV32I-NEXT:  .LBB12_71:
+; RV32I-NEXT:    srl s1, a6, s3
+; RV32I-NEXT:    sw s5, 28(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    bltu s2, t3, .LBB12_70
+; RV32I-NEXT:  .LBB12_72:
+; RV32I-NEXT:    li s6, 0
+; RV32I-NEXT:    sll a3, t1, s2
+; RV32I-NEXT:  .LBB12_73:
+; RV32I-NEXT:    addi s9, s2, -64
+; RV32I-NEXT:    mv s5, t5
+; RV32I-NEXT:    beqz s2, .LBB12_75
+; RV32I-NEXT:  # %bb.74:
+; RV32I-NEXT:    mv s5, a3
+; RV32I-NEXT:  .LBB12_75:
+; RV32I-NEXT:    bltu s9, t3, .LBB12_77
+; RV32I-NEXT:  # %bb.76:
+; RV32I-NEXT:    li s3, 0
+; RV32I-NEXT:    sll a3, t0, s9
+; RV32I-NEXT:    mv s7, a6
+; RV32I-NEXT:    bnez s9, .LBB12_78
+; RV32I-NEXT:    j .LBB12_79
+; RV32I-NEXT:  .LBB12_77:
+; RV32I-NEXT:    sll s3, t0, s2
+; RV32I-NEXT:    neg a3, s9
+; RV32I-NEXT:    srl a3, t0, a3
+; RV32I-NEXT:    sll s7, a6, s2
+; RV32I-NEXT:    or a3, a3, s7
+; RV32I-NEXT:    mv s7, a6
+; RV32I-NEXT:    beqz s9, .LBB12_79
+; RV32I-NEXT:  .LBB12_78:
+; RV32I-NEXT:    mv s7, a3
+; RV32I-NEXT:  .LBB12_79:
+; RV32I-NEXT:    bltu s2, t6, .LBB12_81
+; RV32I-NEXT:  # %bb.80:
+; RV32I-NEXT:    sw zero, 20(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    li s8, 0
+; RV32I-NEXT:    j .LBB12_82
+; RV32I-NEXT:  .LBB12_81:
+; RV32I-NEXT:    sw s10, 20(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    or s3, a5, s6
+; RV32I-NEXT:    or s7, s1, s5
+; RV32I-NEXT:  .LBB12_82:
+; RV32I-NEXT:    addi ra, a4, -128
+; RV32I-NEXT:    mv s5, t1
+; RV32I-NEXT:    mv s6, t5
+; RV32I-NEXT:    beqz s2, .LBB12_84
+; RV32I-NEXT:  # %bb.83:
+; RV32I-NEXT:    mv s5, s3
+; RV32I-NEXT:    mv s6, s7
+; RV32I-NEXT:  .LBB12_84:
+; RV32I-NEXT:    neg s9, ra
+; RV32I-NEXT:    sll s3, t5, s9
+; RV32I-NEXT:    bltu ra, t3, .LBB12_86
+; RV32I-NEXT:  # %bb.85:
+; RV32I-NEXT:    srl a3, t5, ra
+; RV32I-NEXT:    mv s1, t1
+; RV32I-NEXT:    bnez ra, .LBB12_87
+; RV32I-NEXT:    j .LBB12_88
+; RV32I-NEXT:  .LBB12_86:
+; RV32I-NEXT:    lw a3, 36(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    or a3, a3, s3
+; RV32I-NEXT:    mv s1, t1
+; RV32I-NEXT:    beqz ra, .LBB12_88
+; RV32I-NEXT:  .LBB12_87:
+; RV32I-NEXT:    mv s1, a3
+; RV32I-NEXT:  .LBB12_88:
+; RV32I-NEXT:    bltu ra, t3, .LBB12_90
+; RV32I-NEXT:  # %bb.89:
+; RV32I-NEXT:    li s2, 0
+; RV32I-NEXT:    srl a3, a6, ra
+; RV32I-NEXT:    mv a5, t0
+; RV32I-NEXT:    bnez ra, .LBB12_91
+; RV32I-NEXT:    j .LBB12_92
+; RV32I-NEXT:  .LBB12_90:
+; RV32I-NEXT:    srl s2, t5, a4
+; RV32I-NEXT:    sll a3, a6, s9
+; RV32I-NEXT:    lw a5, 32(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    or a3, a5, a3
+; RV32I-NEXT:    mv a5, t0
+; RV32I-NEXT:    beqz ra, .LBB12_92
+; RV32I-NEXT:  .LBB12_91:
+; RV32I-NEXT:    mv a5, a3
+; RV32I-NEXT:  .LBB12_92:
+; RV32I-NEXT:    sub s10, t6, ra
+; RV32I-NEXT:    bltu ra, t3, .LBB12_95
+; RV32I-NEXT:  # %bb.93:
+; RV32I-NEXT:    li s7, 0
+; RV32I-NEXT:    bgeu s10, t3, .LBB12_96
+; RV32I-NEXT:  .LBB12_94:
+; RV32I-NEXT:    sll s9, t1, s9
+; RV32I-NEXT:    neg a3, s10
+; RV32I-NEXT:    srl a3, t1, a3
+; RV32I-NEXT:    or a3, a3, s3
+; RV32I-NEXT:    j .LBB12_97
+; RV32I-NEXT:  .LBB12_95:
+; RV32I-NEXT:    srl s7, a6, a4
+; RV32I-NEXT:    bltu s10, t3, .LBB12_94
+; RV32I-NEXT:  .LBB12_96:
+; RV32I-NEXT:    li s9, 0
+; RV32I-NEXT:    sll a3, t1, s10
+; RV32I-NEXT:  .LBB12_97:
+; RV32I-NEXT:    addi s11, ra, -64
+; RV32I-NEXT:    mv s3, t5
+; RV32I-NEXT:    beqz s10, .LBB12_99
+; RV32I-NEXT:  # %bb.98:
+; RV32I-NEXT:    mv s3, a3
+; RV32I-NEXT:  .LBB12_99:
+; RV32I-NEXT:    bltu s11, t3, .LBB12_101
+; RV32I-NEXT:  # %bb.100:
+; RV32I-NEXT:    srl a3, t5, s11
+; RV32I-NEXT:    bnez s11, .LBB12_102
+; RV32I-NEXT:    j .LBB12_103
+; RV32I-NEXT:  .LBB12_101:
+; RV32I-NEXT:    srl a3, t1, ra
+; RV32I-NEXT:    neg s10, s11
+; RV32I-NEXT:    sll s10, t5, s10
+; RV32I-NEXT:    or a3, a3, s10
+; RV32I-NEXT:    beqz s11, .LBB12_103
+; RV32I-NEXT:  .LBB12_102:
+; RV32I-NEXT:    mv t1, a3
+; RV32I-NEXT:  .LBB12_103:
+; RV32I-NEXT:    bltu s11, t3, .LBB12_105
+; RV32I-NEXT:  # %bb.104:
+; RV32I-NEXT:    li t3, 0
+; RV32I-NEXT:    bltu ra, t6, .LBB12_106
+; RV32I-NEXT:    j .LBB12_107
+; RV32I-NEXT:  .LBB12_105:
+; RV32I-NEXT:    srl t3, t5, ra
+; RV32I-NEXT:    bgeu ra, t6, .LBB12_107
+; RV32I-NEXT:  .LBB12_106:
+; RV32I-NEXT:    or t1, a5, s9
+; RV32I-NEXT:    or t3, s7, s3
+; RV32I-NEXT:  .LBB12_107:
+; RV32I-NEXT:    li a5, 128
+; RV32I-NEXT:    bnez ra, .LBB12_114
+; RV32I-NEXT:  # %bb.108:
+; RV32I-NEXT:    bgeu ra, t6, .LBB12_115
+; RV32I-NEXT:  .LBB12_109:
+; RV32I-NEXT:    bltu a4, a5, .LBB12_116
+; RV32I-NEXT:  .LBB12_110:
+; RV32I-NEXT:    lw ra, 40(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    bnez a4, .LBB12_117
+; RV32I-NEXT:  .LBB12_111:
+; RV32I-NEXT:    bltu a4, a5, .LBB12_113
+; RV32I-NEXT:  .LBB12_112:
+; RV32I-NEXT:    li s4, 0
+; RV32I-NEXT:    li a7, 0
+; RV32I-NEXT:    li a1, 0
+; RV32I-NEXT:    li ra, 0
+; RV32I-NEXT:  .LBB12_113:
+; RV32I-NEXT:    srli a4, s0, 16
+; RV32I-NEXT:    lui t1, 16
+; RV32I-NEXT:    srli t0, s0, 24
+; RV32I-NEXT:    srli a5, a0, 16
+; RV32I-NEXT:    srli t5, a0, 24
+; RV32I-NEXT:    srli a6, t4, 16
+; RV32I-NEXT:    srli s2, t4, 24
+; RV32I-NEXT:    srli t3, t2, 16
+; RV32I-NEXT:    srli s3, t2, 24
+; RV32I-NEXT:    srli s1, s4, 16
+; RV32I-NEXT:    srli a3, s4, 24
+; RV32I-NEXT:    srli t6, a7, 16
+; RV32I-NEXT:    srli s6, a7, 24
+; RV32I-NEXT:    srli s5, a1, 16
+; RV32I-NEXT:    srli s7, a1, 24
+; RV32I-NEXT:    srli s8, ra, 16
+; RV32I-NEXT:    srli s9, ra, 24
+; RV32I-NEXT:    addi t1, t1, -1
+; RV32I-NEXT:    and s10, s0, t1
+; RV32I-NEXT:    and s11, a0, t1
+; RV32I-NEXT:    srli s10, s10, 8
+; RV32I-NEXT:    sb s0, 0(a2)
+; RV32I-NEXT:    sb s10, 1(a2)
+; RV32I-NEXT:    sb a4, 2(a2)
+; RV32I-NEXT:    sb t0, 3(a2)
+; RV32I-NEXT:    and a4, t4, t1
+; RV32I-NEXT:    srli t0, s11, 8
+; RV32I-NEXT:    sb a0, 4(a2)
+; RV32I-NEXT:    sb t0, 5(a2)
+; RV32I-NEXT:    sb a5, 6(a2)
+; RV32I-NEXT:    sb t5, 7(a2)
+; RV32I-NEXT:    and a0, t2, t1
+; RV32I-NEXT:    srli a4, a4, 8
+; RV32I-NEXT:    sb t4, 8(a2)
+; RV32I-NEXT:    sb a4, 9(a2)
+; RV32I-NEXT:    sb a6, 10(a2)
+; RV32I-NEXT:    sb s2, 11(a2)
+; RV32I-NEXT:    and a4, s4, t1
+; RV32I-NEXT:    srli a0, a0, 8
+; RV32I-NEXT:    sb t2, 12(a2)
+; RV32I-NEXT:    sb a0, 13(a2)
+; RV32I-NEXT:    sb t3, 14(a2)
+; RV32I-NEXT:    sb s3, 15(a2)
+; RV32I-NEXT:    and a0, a7, t1
+; RV32I-NEXT:    srli a4, a4, 8
+; RV32I-NEXT:    sb s4, 16(a2)
+; RV32I-NEXT:    sb a4, 17(a2)
+; RV32I-NEXT:    sb s1, 18(a2)
+; RV32I-NEXT:    sb a3, 19(a2)
+; RV32I-NEXT:    and a3, a1, t1
+; RV32I-NEXT:    and a4, ra, t1
+; RV32I-NEXT:    srli a0, a0, 8
+; RV32I-NEXT:    srli a3, a3, 8
+; RV32I-NEXT:    srli a4, a4, 8
+; RV32I-NEXT:    sb a7, 20(a2)
+; RV32I-NEXT:    sb a0, 21(a2)
+; RV32I-NEXT:    sb t6, 22(a2)
+; RV32I-NEXT:    sb s6, 23(a2)
+; RV32I-NEXT:    sb a1, 24(a2)
+; RV32I-NEXT:    sb a3, 25(a2)
+; RV32I-NEXT:    sb s5, 26(a2)
+; RV32I-NEXT:    sb s7, 27(a2)
+; RV32I-NEXT:    sb ra, 28(a2)
+; RV32I-NEXT:    sb a4, 29(a2)
+; RV32I-NEXT:    sb s8, 30(a2)
+; RV32I-NEXT:    sb s9, 31(a2)
+; RV32I-NEXT:    lw ra, 92(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s0, 88(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s1, 84(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s2, 80(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s3, 76(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s4, 72(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s5, 68(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s6, 64(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s7, 60(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s8, 56(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s9, 52(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s10, 48(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s11, 44(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    addi sp, sp, 96
+; RV32I-NEXT:    ret
+; RV32I-NEXT:  .LBB12_114:
+; RV32I-NEXT:    mv t0, t1
+; RV32I-NEXT:    mv a6, t3
+; RV32I-NEXT:    bltu ra, t6, .LBB12_109
+; RV32I-NEXT:  .LBB12_115:
+; RV32I-NEXT:    li s1, 0
+; RV32I-NEXT:    li s2, 0
+; RV32I-NEXT:    bgeu a4, a5, .LBB12_110
+; RV32I-NEXT:  .LBB12_116:
+; RV32I-NEXT:    lw a3, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw a6, 20(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    or t0, a3, a6
+; RV32I-NEXT:    lw a3, 8(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    or a6, a3, s8
+; RV32I-NEXT:    lw a3, 28(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    or s1, a3, s5
+; RV32I-NEXT:    lw a3, 24(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    or s2, a3, s6
+; RV32I-NEXT:    lw ra, 40(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    beqz a4, .LBB12_111
+; RV32I-NEXT:  .LBB12_117:
+; RV32I-NEXT:    mv s0, t0
+; RV32I-NEXT:    mv a0, a6
+; RV32I-NEXT:    mv t4, s1
+; RV32I-NEXT:    mv t2, s2
+; RV32I-NEXT:    bgeu a4, a5, .LBB12_112
+; RV32I-NEXT:    j .LBB12_113
+  %src = load i256, ptr %src.ptr, align 1
+  %byteOff = load i256, ptr %byteOff.ptr, align 1
+  %bitOff = shl i256 %byteOff, 3
+  %res = lshr i256 %src, %bitOff
+  store i256 %res, ptr %dst, align 1
+  ret void
+}
+
+define void @lshr_32bytes_wordOff(ptr %src.ptr, ptr %wordOff.ptr, ptr %dst) nounwind {
+; RV64I-LABEL: lshr_32bytes_wordOff:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -96
+; RV64I-NEXT:    sd s0, 88(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s1, 80(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s2, 72(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s3, 64(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s4, 56(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s5, 48(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s6, 40(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s7, 32(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s8, 24(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s9, 16(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s10, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s11, 0(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    lbu a3, 0(a0)
+; RV64I-NEXT:    lbu a4, 1(a0)
+; RV64I-NEXT:    lbu a5, 2(a0)
+; RV64I-NEXT:    lbu a6, 3(a0)
+; RV64I-NEXT:    lbu a7, 4(a0)
+; RV64I-NEXT:    lbu t0, 5(a0)
+; RV64I-NEXT:    lbu t1, 6(a0)
+; RV64I-NEXT:    lbu t2, 7(a0)
+; RV64I-NEXT:    lbu t3, 8(a0)
+; RV64I-NEXT:    lbu t4, 9(a0)
+; RV64I-NEXT:    lbu t5, 10(a0)
+; RV64I-NEXT:    lbu t6, 11(a0)
+; RV64I-NEXT:    lbu s0, 12(a0)
+; RV64I-NEXT:    lbu s1, 13(a0)
+; RV64I-NEXT:    lbu s2, 14(a0)
+; RV64I-NEXT:    lbu s3, 15(a0)
+; RV64I-NEXT:    lbu s4, 16(a0)
+; RV64I-NEXT:    lbu s5, 17(a0)
+; RV64I-NEXT:    lbu s6, 18(a0)
+; RV64I-NEXT:    lbu s7, 19(a0)
+; RV64I-NEXT:    slli a4, a4, 8
+; RV64I-NEXT:    slli a6, a6, 8
+; RV64I-NEXT:    slli t0, t0, 8
+; RV64I-NEXT:    or a3, a4, a3
+; RV64I-NEXT:    or a5, a6, a5
+; RV64I-NEXT:    or a7, t0, a7
+; RV64I-NEXT:    lbu s8, 20(a0)
+; RV64I-NEXT:    lbu s9, 21(a0)
+; RV64I-NEXT:    lbu s10, 22(a0)
+; RV64I-NEXT:    lbu s11, 23(a0)
+; RV64I-NEXT:    slli t2, t2, 8
+; RV64I-NEXT:    slli t4, t4, 8
+; RV64I-NEXT:    slli t6, t6, 8
+; RV64I-NEXT:    slli s1, s1, 8
+; RV64I-NEXT:    or t1, t2, t1
+; RV64I-NEXT:    or a4, t4, t3
+; RV64I-NEXT:    or a6, t6, t5
+; RV64I-NEXT:    or t0, s1, s0
+; RV64I-NEXT:    lbu t5, 24(a0)
+; RV64I-NEXT:    lbu t6, 25(a0)
+; RV64I-NEXT:    lbu s0, 26(a0)
+; RV64I-NEXT:    lbu s1, 27(a0)
+; RV64I-NEXT:    slli s3, s3, 8
+; RV64I-NEXT:    slli s5, s5, 8
+; RV64I-NEXT:    slli s7, s7, 8
+; RV64I-NEXT:    or t4, s3, s2
+; RV64I-NEXT:    or t2, s5, s4
+; RV64I-NEXT:    or t3, s7, s6
+; RV64I-NEXT:    lbu s2, 28(a0)
+; RV64I-NEXT:    lbu s3, 29(a0)
+; RV64I-NEXT:    lbu s4, 30(a0)
+; RV64I-NEXT:    lbu a0, 31(a0)
+; RV64I-NEXT:    slli s9, s9, 8
+; RV64I-NEXT:    slli s11, s11, 8
+; RV64I-NEXT:    slli t6, t6, 8
+; RV64I-NEXT:    slli s1, s1, 8
+; RV64I-NEXT:    or s5, s9, s8
+; RV64I-NEXT:    or s6, s11, s10
+; RV64I-NEXT:    or t5, t6, t5
+; RV64I-NEXT:    or s0, s1, s0
+; RV64I-NEXT:    lbu t6, 0(a1)
+; RV64I-NEXT:    lbu s1, 1(a1)
+; RV64I-NEXT:    lbu s7, 2(a1)
+; RV64I-NEXT:    lbu s8, 3(a1)
+; RV64I-NEXT:    slli s3, s3, 8
+; RV64I-NEXT:    slli a0, a0, 8
+; RV64I-NEXT:    slli s1, s1, 8
+; RV64I-NEXT:    or s2, s3, s2
+; RV64I-NEXT:    or s3, a0, s4
+; RV64I-NEXT:    or t6, s1, t6
+; RV64I-NEXT:    lbu a0, 4(a1)
+; RV64I-NEXT:    lbu s1, 5(a1)
+; RV64I-NEXT:    lbu s4, 6(a1)
+; RV64I-NEXT:    lbu a1, 7(a1)
+; RV64I-NEXT:    slli s8, s8, 8
+; RV64I-NEXT:    or s7, s8, s7
+; RV64I-NEXT:    slli s1, s1, 8
+; RV64I-NEXT:    or s1, s1, a0
+; RV64I-NEXT:    slli a1, a1, 8
+; RV64I-NEXT:    or s4, a1, s4
+; RV64I-NEXT:    slli t1, t1, 16
+; RV64I-NEXT:    or a1, t1, a7
+; RV64I-NEXT:    slli t4, t4, 16
+; RV64I-NEXT:    or a0, t4, t0
+; RV64I-NEXT:    slli t3, t3, 16
+; RV64I-NEXT:    or t0, t3, t2
+; RV64I-NEXT:    slli s6, s6, 16
+; RV64I-NEXT:    or t1, s6, s5
+; RV64I-NEXT:    slli s0, s0, 16
+; RV64I-NEXT:    or t3, s0, t5
+; RV64I-NEXT:    slli s3, s3, 16
+; RV64I-NEXT:    or t5, s3, s2
+; RV64I-NEXT:    slli s7, s7, 16
+; RV64I-NEXT:    or t6, s7, t6
+; RV64I-NEXT:    slli s4, s4, 16
+; RV64I-NEXT:    or s0, s4, s1
+; RV64I-NEXT:    li a7, 64
+; RV64I-NEXT:    slli t4, a5, 16
+; RV64I-NEXT:    slli t2, a6, 16
+; RV64I-NEXT:    slli t1, t1, 32
+; RV64I-NEXT:    slli t5, t5, 32
+; RV64I-NEXT:    slli s0, s0, 32
+; RV64I-NEXT:    or a6, t1, t0
+; RV64I-NEXT:    or t0, t5, t3
+; RV64I-NEXT:    or a5, s0, t6
+; RV64I-NEXT:    slli a5, a5, 5
+; RV64I-NEXT:    subw t1, a5, a7
+; RV64I-NEXT:    negw t5, a5
+; RV64I-NEXT:    sll t3, t0, t5
+; RV64I-NEXT:    bltu a5, a7, .LBB13_2
+; RV64I-NEXT:  # %bb.1:
+; RV64I-NEXT:    srl t6, t0, t1
+; RV64I-NEXT:    j .LBB13_3
+; RV64I-NEXT:  .LBB13_2:
+; RV64I-NEXT:    srl t6, a6, a5
+; RV64I-NEXT:    or t6, t6, t3
+; RV64I-NEXT:  .LBB13_3:
+; RV64I-NEXT:    or a3, t4, a3
+; RV64I-NEXT:    slli t4, a1, 32
+; RV64I-NEXT:    or t2, t2, a4
+; RV64I-NEXT:    slli a0, a0, 32
+; RV64I-NEXT:    mv a1, a6
+; RV64I-NEXT:    beqz a5, .LBB13_5
+; RV64I-NEXT:  # %bb.4:
+; RV64I-NEXT:    mv a1, t6
+; RV64I-NEXT:  .LBB13_5:
+; RV64I-NEXT:    or a4, t4, a3
+; RV64I-NEXT:    or a3, a0, t2
+; RV64I-NEXT:    bltu a5, a7, .LBB13_7
+; RV64I-NEXT:  # %bb.6:
+; RV64I-NEXT:    li a0, 0
+; RV64I-NEXT:    srl t4, a3, t1
+; RV64I-NEXT:    j .LBB13_8
+; RV64I-NEXT:  .LBB13_7:
+; RV64I-NEXT:    srl a0, t0, a5
+; RV64I-NEXT:    srl t1, a4, a5
+; RV64I-NEXT:    sll t2, a3, t5
+; RV64I-NEXT:    or t4, t1, t2
+; RV64I-NEXT:  .LBB13_8:
+; RV64I-NEXT:    li t1, 128
+; RV64I-NEXT:    mv t2, a4
+; RV64I-NEXT:    beqz a5, .LBB13_10
+; RV64I-NEXT:  # %bb.9:
+; RV64I-NEXT:    mv t2, t4
+; RV64I-NEXT:  .LBB13_10:
+; RV64I-NEXT:    sub t6, t1, a5
+; RV64I-NEXT:    bltu a5, a7, .LBB13_13
+; RV64I-NEXT:  # %bb.11:
+; RV64I-NEXT:    li t4, 0
+; RV64I-NEXT:    bgeu t6, a7, .LBB13_14
+; RV64I-NEXT:  .LBB13_12:
+; RV64I-NEXT:    sll t5, a6, t5
+; RV64I-NEXT:    negw s0, t6
+; RV64I-NEXT:    srl s0, a6, s0
+; RV64I-NEXT:    or s1, s0, t3
+; RV64I-NEXT:    j .LBB13_15
+; RV64I-NEXT:  .LBB13_13:
+; RV64I-NEXT:    srl t4, a3, a5
+; RV64I-NEXT:    bltu t6, a7, .LBB13_12
+; RV64I-NEXT:  .LBB13_14:
+; RV64I-NEXT:    li t5, 0
+; RV64I-NEXT:    subw t3, t6, a7
+; RV64I-NEXT:    sll s1, a6, t3
+; RV64I-NEXT:  .LBB13_15:
+; RV64I-NEXT:    sub s0, a5, t1
+; RV64I-NEXT:    mv t3, t0
+; RV64I-NEXT:    beqz t6, .LBB13_17
+; RV64I-NEXT:  # %bb.16:
+; RV64I-NEXT:    mv t3, s1
+; RV64I-NEXT:  .LBB13_17:
+; RV64I-NEXT:    bltu s0, a7, .LBB13_19
+; RV64I-NEXT:  # %bb.18:
+; RV64I-NEXT:    subw t6, s0, a7
+; RV64I-NEXT:    srl t6, t0, t6
+; RV64I-NEXT:    bnez s0, .LBB13_20
+; RV64I-NEXT:    j .LBB13_21
+; RV64I-NEXT:  .LBB13_19:
+; RV64I-NEXT:    srl t6, a6, s0
+; RV64I-NEXT:    negw s1, s0
+; RV64I-NEXT:    sll s1, t0, s1
+; RV64I-NEXT:    or t6, t6, s1
+; RV64I-NEXT:    beqz s0, .LBB13_21
+; RV64I-NEXT:  .LBB13_20:
+; RV64I-NEXT:    mv a6, t6
+; RV64I-NEXT:  .LBB13_21:
+; RV64I-NEXT:    bltu s0, a7, .LBB13_23
+; RV64I-NEXT:  # %bb.22:
+; RV64I-NEXT:    li a7, 0
+; RV64I-NEXT:    bltu a5, t1, .LBB13_24
+; RV64I-NEXT:    j .LBB13_25
+; RV64I-NEXT:  .LBB13_23:
+; RV64I-NEXT:    srl a7, t0, s0
+; RV64I-NEXT:    bgeu a5, t1, .LBB13_25
+; RV64I-NEXT:  .LBB13_24:
+; RV64I-NEXT:    or a6, t2, t5
+; RV64I-NEXT:    or a7, t4, t3
+; RV64I-NEXT:  .LBB13_25:
+; RV64I-NEXT:    bnez a5, .LBB13_29
+; RV64I-NEXT:  # %bb.26:
+; RV64I-NEXT:    bltu a5, t1, .LBB13_28
+; RV64I-NEXT:  .LBB13_27:
+; RV64I-NEXT:    li a1, 0
+; RV64I-NEXT:    li a0, 0
+; RV64I-NEXT:  .LBB13_28:
+; RV64I-NEXT:    srli a5, a4, 32
+; RV64I-NEXT:    srliw a6, a4, 16
+; RV64I-NEXT:    lui t2, 16
+; RV64I-NEXT:    srliw t1, a4, 24
+; RV64I-NEXT:    srli t0, a4, 48
+; RV64I-NEXT:    srli t5, a4, 56
+; RV64I-NEXT:    srli a7, a3, 32
+; RV64I-NEXT:    srliw t4, a3, 16
+; RV64I-NEXT:    srliw s0, a3, 24
+; RV64I-NEXT:    srli t6, a3, 48
+; RV64I-NEXT:    srli s3, a3, 56
+; RV64I-NEXT:    srli t3, a1, 32
+; RV64I-NEXT:    srliw s2, a1, 16
+; RV64I-NEXT:    srliw s6, a1, 24
+; RV64I-NEXT:    srli s4, a1, 48
+; RV64I-NEXT:    srli s7, a1, 56
+; RV64I-NEXT:    srli s1, a0, 32
+; RV64I-NEXT:    srliw s5, a0, 16
+; RV64I-NEXT:    srliw s8, a0, 24
+; RV64I-NEXT:    srli s9, a0, 48
+; RV64I-NEXT:    srli s10, a0, 56
+; RV64I-NEXT:    addi t2, t2, -1
+; RV64I-NEXT:    and s11, a4, t2
+; RV64I-NEXT:    srli s11, s11, 8
+; RV64I-NEXT:    sb a4, 0(a2)
+; RV64I-NEXT:    sb s11, 1(a2)
+; RV64I-NEXT:    sb a6, 2(a2)
+; RV64I-NEXT:    sb t1, 3(a2)
+; RV64I-NEXT:    and a4, a5, t2
+; RV64I-NEXT:    srli a4, a4, 8
+; RV64I-NEXT:    sb a5, 4(a2)
+; RV64I-NEXT:    sb a4, 5(a2)
+; RV64I-NEXT:    sb t0, 6(a2)
+; RV64I-NEXT:    sb t5, 7(a2)
+; RV64I-NEXT:    and a4, a3, t2
+; RV64I-NEXT:    srli a4, a4, 8
+; RV64I-NEXT:    sb a3, 8(a2)
+; RV64I-NEXT:    sb a4, 9(a2)
+; RV64I-NEXT:    sb t4, 10(a2)
+; RV64I-NEXT:    sb s0, 11(a2)
+; RV64I-NEXT:    and a3, a7, t2
+; RV64I-NEXT:    srli a3, a3, 8
+; RV64I-NEXT:    sb a7, 12(a2)
+; RV64I-NEXT:    sb a3, 13(a2)
+; RV64I-NEXT:    sb t6, 14(a2)
+; RV64I-NEXT:    sb s3, 15(a2)
+; RV64I-NEXT:    and a3, a1, t2
+; RV64I-NEXT:    srli a3, a3, 8
+; RV64I-NEXT:    sb a1, 16(a2)
+; RV64I-NEXT:    sb a3, 17(a2)
+; RV64I-NEXT:    sb s2, 18(a2)
+; RV64I-NEXT:    sb s6, 19(a2)
+; RV64I-NEXT:    and a1, t3, t2
+; RV64I-NEXT:    srli a1, a1, 8
+; RV64I-NEXT:    sb t3, 20(a2)
+; RV64I-NEXT:    sb a1, 21(a2)
+; RV64I-NEXT:    sb s4, 22(a2)
+; RV64I-NEXT:    sb s7, 23(a2)
+; RV64I-NEXT:    and a1, a0, t2
+; RV64I-NEXT:    and a3, s1, t2
+; RV64I-NEXT:    srli a1, a1, 8
+; RV64I-NEXT:    srli a3, a3, 8
+; RV64I-NEXT:    sb a0, 24(a2)
+; RV64I-NEXT:    sb a1, 25(a2)
+; RV64I-NEXT:    sb s5, 26(a2)
+; RV64I-NEXT:    sb s8, 27(a2)
+; RV64I-NEXT:    sb s1, 28(a2)
+; RV64I-NEXT:    sb a3, 29(a2)
+; RV64I-NEXT:    sb s9, 30(a2)
+; RV64I-NEXT:    sb s10, 31(a2)
+; RV64I-NEXT:    ld s0, 88(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s1, 80(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s2, 72(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s3, 64(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s4, 56(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s5, 48(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s6, 40(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s7, 32(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s8, 24(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s9, 16(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s10, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s11, 0(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    addi sp, sp, 96
+; RV64I-NEXT:    ret
+; RV64I-NEXT:  .LBB13_29:
+; RV64I-NEXT:    mv a4, a6
+; RV64I-NEXT:    mv a3, a7
+; RV64I-NEXT:    bgeu a5, t1, .LBB13_27
+; RV64I-NEXT:    j .LBB13_28
+;
+; RV32I-LABEL: lshr_32bytes_wordOff:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    addi sp, sp, -96
+; RV32I-NEXT:    sw ra, 92(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s0, 88(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s1, 84(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s2, 80(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s3, 76(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s4, 72(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s5, 68(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s6, 64(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s7, 60(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s8, 56(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s9, 52(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s10, 48(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s11, 44(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    lbu a3, 16(a0)
+; RV32I-NEXT:    lbu a4, 17(a0)
+; RV32I-NEXT:    lbu a5, 18(a0)
+; RV32I-NEXT:    lbu a6, 19(a0)
+; RV32I-NEXT:    lbu a7, 20(a0)
+; RV32I-NEXT:    lbu t0, 21(a0)
+; RV32I-NEXT:    lbu t1, 22(a0)
+; RV32I-NEXT:    lbu t2, 23(a0)
+; RV32I-NEXT:    lbu t3, 24(a0)
+; RV32I-NEXT:    lbu t4, 25(a0)
+; RV32I-NEXT:    lbu t5, 26(a0)
+; RV32I-NEXT:    lbu t6, 27(a0)
+; RV32I-NEXT:    slli a4, a4, 8
+; RV32I-NEXT:    slli a6, a6, 8
+; RV32I-NEXT:    slli t0, t0, 8
+; RV32I-NEXT:    slli t2, t2, 8
+; RV32I-NEXT:    or a3, a4, a3
+; RV32I-NEXT:    or a6, a6, a5
+; RV32I-NEXT:    or a5, t0, a7
+; RV32I-NEXT:    or a4, t2, t1
+; RV32I-NEXT:    lbu a7, 28(a0)
+; RV32I-NEXT:    lbu t0, 29(a0)
+; RV32I-NEXT:    lbu t1, 30(a0)
+; RV32I-NEXT:    lbu t2, 31(a0)
+; RV32I-NEXT:    slli t4, t4, 8
+; RV32I-NEXT:    slli t6, t6, 8
+; RV32I-NEXT:    slli t0, t0, 8
+; RV32I-NEXT:    or t4, t4, t3
+; RV32I-NEXT:    or t5, t6, t5
+; RV32I-NEXT:    or a7, t0, a7
+; RV32I-NEXT:    lbu t0, 0(a1)
+; RV32I-NEXT:    lbu t3, 1(a1)
+; RV32I-NEXT:    lbu t6, 2(a1)
+; RV32I-NEXT:    lbu a1, 3(a1)
+; RV32I-NEXT:    slli t2, t2, 8
+; RV32I-NEXT:    or t1, t2, t1
+; RV32I-NEXT:    slli t3, t3, 8
+; RV32I-NEXT:    or t0, t3, t0
+; RV32I-NEXT:    slli a1, a1, 8
+; RV32I-NEXT:    or t2, a1, t6
+; RV32I-NEXT:    li t3, 32
+; RV32I-NEXT:    slli a6, a6, 16
+; RV32I-NEXT:    slli a1, a4, 16
+; RV32I-NEXT:    slli t5, t5, 16
+; RV32I-NEXT:    slli a4, t1, 16
+; RV32I-NEXT:    slli t2, t2, 16
+; RV32I-NEXT:    or t1, t5, t4
+; RV32I-NEXT:    or t5, a4, a7
+; RV32I-NEXT:    or a4, t2, t0
+; RV32I-NEXT:    slli a4, a4, 5
+; RV32I-NEXT:    srl s0, t1, a4
+; RV32I-NEXT:    neg s6, a4
+; RV32I-NEXT:    sll t4, t5, s6
+; RV32I-NEXT:    bltu a4, t3, .LBB13_2
+; RV32I-NEXT:  # %bb.1:
+; RV32I-NEXT:    srl a7, t5, a4
+; RV32I-NEXT:    j .LBB13_3
+; RV32I-NEXT:  .LBB13_2:
+; RV32I-NEXT:    or a7, s0, t4
+; RV32I-NEXT:  .LBB13_3:
+; RV32I-NEXT:    or t0, a6, a3
+; RV32I-NEXT:    or a6, a1, a5
+; RV32I-NEXT:    mv a1, t1
+; RV32I-NEXT:    beqz a4, .LBB13_5
+; RV32I-NEXT:  # %bb.4:
+; RV32I-NEXT:    mv a1, a7
+; RV32I-NEXT:  .LBB13_5:
+; RV32I-NEXT:    srl a3, t0, a4
+; RV32I-NEXT:    sll a5, a6, s6
+; RV32I-NEXT:    sw a3, 32(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    bltu a4, t3, .LBB13_7
+; RV32I-NEXT:  # %bb.6:
+; RV32I-NEXT:    li ra, 0
+; RV32I-NEXT:    srl a3, a6, a4
+; RV32I-NEXT:    j .LBB13_8
+; RV32I-NEXT:  .LBB13_7:
+; RV32I-NEXT:    srl ra, t5, a4
+; RV32I-NEXT:    or a3, a3, a5
+; RV32I-NEXT:  .LBB13_8:
+; RV32I-NEXT:    li t6, 64
+; RV32I-NEXT:    mv a7, t0
+; RV32I-NEXT:    beqz a4, .LBB13_10
+; RV32I-NEXT:  # %bb.9:
+; RV32I-NEXT:    mv a7, a3
+; RV32I-NEXT:  .LBB13_10:
+; RV32I-NEXT:    sw a5, 16(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sub s7, t6, a4
+; RV32I-NEXT:    bltu a4, t3, .LBB13_12
+; RV32I-NEXT:  # %bb.11:
+; RV32I-NEXT:    li a5, 0
+; RV32I-NEXT:    j .LBB13_13
+; RV32I-NEXT:  .LBB13_12:
+; RV32I-NEXT:    srl a5, a6, a4
+; RV32I-NEXT:  .LBB13_13:
+; RV32I-NEXT:    neg s10, s7
+; RV32I-NEXT:    sw t4, 20(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    bltu s7, t3, .LBB13_15
+; RV32I-NEXT:  # %bb.14:
+; RV32I-NEXT:    li t2, 0
+; RV32I-NEXT:    sll a3, t1, s7
+; RV32I-NEXT:    j .LBB13_16
+; RV32I-NEXT:  .LBB13_15:
+; RV32I-NEXT:    sll t2, t1, s6
+; RV32I-NEXT:    srl a3, t1, s10
+; RV32I-NEXT:    or a3, a3, t4
+; RV32I-NEXT:  .LBB13_16:
+; RV32I-NEXT:    addi s9, a4, -64
+; RV32I-NEXT:    mv t4, t5
+; RV32I-NEXT:    beqz s7, .LBB13_18
+; RV32I-NEXT:  # %bb.17:
+; RV32I-NEXT:    mv t4, a3
+; RV32I-NEXT:  .LBB13_18:
+; RV32I-NEXT:    neg s11, s9
+; RV32I-NEXT:    sw s0, 36(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    bltu s9, t3, .LBB13_20
+; RV32I-NEXT:  # %bb.19:
+; RV32I-NEXT:    srl s2, t5, s9
+; RV32I-NEXT:    j .LBB13_21
+; RV32I-NEXT:  .LBB13_20:
+; RV32I-NEXT:    sll a3, t5, s11
+; RV32I-NEXT:    or s2, s0, a3
+; RV32I-NEXT:  .LBB13_21:
+; RV32I-NEXT:    lbu s1, 11(a0)
+; RV32I-NEXT:    lbu a3, 15(a0)
+; RV32I-NEXT:    mv s0, t1
+; RV32I-NEXT:    beqz s9, .LBB13_23
+; RV32I-NEXT:  # %bb.22:
+; RV32I-NEXT:    mv s0, s2
+; RV32I-NEXT:  .LBB13_23:
+; RV32I-NEXT:    lbu s4, 9(a0)
+; RV32I-NEXT:    lbu s2, 10(a0)
+; RV32I-NEXT:    lbu s5, 13(a0)
+; RV32I-NEXT:    lbu s8, 14(a0)
+; RV32I-NEXT:    slli s3, s1, 8
+; RV32I-NEXT:    slli a3, a3, 8
+; RV32I-NEXT:    sw ra, 40(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    bltu s9, t3, .LBB13_25
+; RV32I-NEXT:  # %bb.24:
+; RV32I-NEXT:    li s1, 0
+; RV32I-NEXT:    j .LBB13_26
+; RV32I-NEXT:  .LBB13_25:
+; RV32I-NEXT:    srl s1, t5, a4
+; RV32I-NEXT:  .LBB13_26:
+; RV32I-NEXT:    or s2, s3, s2
+; RV32I-NEXT:    lbu ra, 8(a0)
+; RV32I-NEXT:    lbu s3, 12(a0)
+; RV32I-NEXT:    slli s4, s4, 8
+; RV32I-NEXT:    slli s5, s5, 8
+; RV32I-NEXT:    or s8, a3, s8
+; RV32I-NEXT:    bgeu a4, t6, .LBB13_28
+; RV32I-NEXT:  # %bb.27:
+; RV32I-NEXT:    or s0, a7, t2
+; RV32I-NEXT:    or s1, a5, t4
+; RV32I-NEXT:  .LBB13_28:
+; RV32I-NEXT:    lbu a3, 3(a0)
+; RV32I-NEXT:    lbu t2, 7(a0)
+; RV32I-NEXT:    or a5, s4, ra
+; RV32I-NEXT:    slli t4, s2, 16
+; RV32I-NEXT:    or s2, s5, s3
+; RV32I-NEXT:    slli s3, s8, 16
+; RV32I-NEXT:    mv s4, t0
+; RV32I-NEXT:    mv a7, a6
+; RV32I-NEXT:    beqz a4, .LBB13_30
+; RV32I-NEXT:  # %bb.29:
+; RV32I-NEXT:    mv s4, s0
+; RV32I-NEXT:    mv a7, s1
+; RV32I-NEXT:  .LBB13_30:
+; RV32I-NEXT:    slli s5, a3, 8
+; RV32I-NEXT:    lbu ra, 1(a0)
+; RV32I-NEXT:    lbu a3, 2(a0)
+; RV32I-NEXT:    lbu s1, 5(a0)
+; RV32I-NEXT:    lbu s0, 6(a0)
+; RV32I-NEXT:    slli s8, t2, 8
+; RV32I-NEXT:    or t4, t4, a5
+; RV32I-NEXT:    or t2, s3, s2
+; RV32I-NEXT:    bltu a4, t6, .LBB13_32
+; RV32I-NEXT:  # %bb.31:
+; RV32I-NEXT:    li a1, 0
+; RV32I-NEXT:    sw zero, 40(sp) # 4-byte Folded Spill
+; RV32I-NEXT:  .LBB13_32:
+; RV32I-NEXT:    slli s3, ra, 8
+; RV32I-NEXT:    or a5, s5, a3
+; RV32I-NEXT:    lbu s5, 0(a0)
+; RV32I-NEXT:    lbu a0, 4(a0)
+; RV32I-NEXT:    slli s1, s1, 8
+; RV32I-NEXT:    or a3, s8, s0
+; RV32I-NEXT:    srl s2, t4, a4
+; RV32I-NEXT:    sll ra, t2, s6
+; RV32I-NEXT:    bltu a4, t3, .LBB13_34
+; RV32I-NEXT:  # %bb.33:
+; RV32I-NEXT:    srl s0, t2, a4
+; RV32I-NEXT:    j .LBB13_35
+; RV32I-NEXT:  .LBB13_34:
+; RV32I-NEXT:    or s0, s2, ra
+; RV32I-NEXT:  .LBB13_35:
+; RV32I-NEXT:    or s3, s3, s5
+; RV32I-NEXT:    slli a5, a5, 16
+; RV32I-NEXT:    or a0, s1, a0
+; RV32I-NEXT:    slli a3, a3, 16
+; RV32I-NEXT:    mv s5, t4
+; RV32I-NEXT:    beqz a4, .LBB13_37
+; RV32I-NEXT:  # %bb.36:
+; RV32I-NEXT:    mv s5, s0
+; RV32I-NEXT:  .LBB13_37:
+; RV32I-NEXT:    or s0, a5, s3
+; RV32I-NEXT:    or a0, a3, a0
+; RV32I-NEXT:    bltu a4, t3, .LBB13_39
+; RV32I-NEXT:  # %bb.38:
+; RV32I-NEXT:    li s8, 0
+; RV32I-NEXT:    srl a3, a0, a4
+; RV32I-NEXT:    mv a5, s0
+; RV32I-NEXT:    bnez a4, .LBB13_40
+; RV32I-NEXT:    j .LBB13_41
+; RV32I-NEXT:  .LBB13_39:
+; RV32I-NEXT:    srl s8, t2, a4
+; RV32I-NEXT:    srl a3, s0, a4
+; RV32I-NEXT:    sll a5, a0, s6
+; RV32I-NEXT:    or a3, a3, a5
+; RV32I-NEXT:    mv a5, s0
+; RV32I-NEXT:    beqz a4, .LBB13_41
+; RV32I-NEXT:  .LBB13_40:
+; RV32I-NEXT:    mv a5, a3
+; RV32I-NEXT:  .LBB13_41:
+; RV32I-NEXT:    bltu a4, t3, .LBB13_44
+; RV32I-NEXT:  # %bb.42:
+; RV32I-NEXT:    li s1, 0
+; RV32I-NEXT:    bgeu s7, t3, .LBB13_45
+; RV32I-NEXT:  .LBB13_43:
+; RV32I-NEXT:    sll s3, t4, s6
+; RV32I-NEXT:    srl a3, t4, s10
+; RV32I-NEXT:    or a3, a3, ra
+; RV32I-NEXT:    mv s10, t2
+; RV32I-NEXT:    bnez s7, .LBB13_46
+; RV32I-NEXT:    j .LBB13_47
+; RV32I-NEXT:  .LBB13_44:
+; RV32I-NEXT:    srl s1, a0, a4
+; RV32I-NEXT:    bltu s7, t3, .LBB13_43
+; RV32I-NEXT:  .LBB13_45:
+; RV32I-NEXT:    li s3, 0
+; RV32I-NEXT:    sll a3, t4, s7
+; RV32I-NEXT:    mv s10, t2
+; RV32I-NEXT:    beqz s7, .LBB13_47
+; RV32I-NEXT:  .LBB13_46:
+; RV32I-NEXT:    mv s10, a3
+; RV32I-NEXT:  .LBB13_47:
+; RV32I-NEXT:    bltu s9, t3, .LBB13_49
+; RV32I-NEXT:  # %bb.48:
+; RV32I-NEXT:    srl a3, t2, s9
+; RV32I-NEXT:    mv s2, t4
+; RV32I-NEXT:    bnez s9, .LBB13_50
+; RV32I-NEXT:    j .LBB13_51
+; RV32I-NEXT:  .LBB13_49:
+; RV32I-NEXT:    sll a3, t2, s11
+; RV32I-NEXT:    or a3, s2, a3
+; RV32I-NEXT:    mv s2, t4
+; RV32I-NEXT:    beqz s9, .LBB13_51
+; RV32I-NEXT:  .LBB13_50:
+; RV32I-NEXT:    mv s2, a3
+; RV32I-NEXT:  .LBB13_51:
+; RV32I-NEXT:    bltu s9, t3, .LBB13_53
+; RV32I-NEXT:  # %bb.52:
+; RV32I-NEXT:    li s7, 0
+; RV32I-NEXT:    bltu a4, t6, .LBB13_54
+; RV32I-NEXT:    j .LBB13_55
+; RV32I-NEXT:  .LBB13_53:
+; RV32I-NEXT:    srl s7, t2, a4
+; RV32I-NEXT:    bgeu a4, t6, .LBB13_55
+; RV32I-NEXT:  .LBB13_54:
+; RV32I-NEXT:    or s2, a5, s3
+; RV32I-NEXT:    or s7, s1, s10
+; RV32I-NEXT:  .LBB13_55:
+; RV32I-NEXT:    li a3, 128
+; RV32I-NEXT:    mv a5, s0
+; RV32I-NEXT:    mv s1, a0
+; RV32I-NEXT:    beqz a4, .LBB13_57
+; RV32I-NEXT:  # %bb.56:
+; RV32I-NEXT:    mv a5, s2
+; RV32I-NEXT:    mv s1, s7
+; RV32I-NEXT:  .LBB13_57:
+; RV32I-NEXT:    sw a5, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sub s2, a3, a4
+; RV32I-NEXT:    bltu a4, t6, .LBB13_59
+; RV32I-NEXT:  # %bb.58:
+; RV32I-NEXT:    li s5, 0
+; RV32I-NEXT:    li s8, 0
+; RV32I-NEXT:  .LBB13_59:
+; RV32I-NEXT:    neg s3, s2
+; RV32I-NEXT:    srl a5, t0, s3
+; RV32I-NEXT:    sw s8, 24(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s1, 8(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    bltu s2, t3, .LBB13_61
+; RV32I-NEXT:  # %bb.60:
+; RV32I-NEXT:    li s10, 0
+; RV32I-NEXT:    sll a3, t0, s2
+; RV32I-NEXT:    j .LBB13_62
+; RV32I-NEXT:  .LBB13_61:
+; RV32I-NEXT:    sll s10, t0, s6
+; RV32I-NEXT:    lw a3, 16(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    or a3, a5, a3
+; RV32I-NEXT:  .LBB13_62:
+; RV32I-NEXT:    sub s1, t6, s2
+; RV32I-NEXT:    mv s8, a6
+; RV32I-NEXT:    beqz s2, .LBB13_64
+; RV32I-NEXT:  # %bb.63:
+; RV32I-NEXT:    mv s8, a3
+; RV32I-NEXT:  .LBB13_64:
+; RV32I-NEXT:    bltu s1, t3, .LBB13_66
+; RV32I-NEXT:  # %bb.65:
+; RV32I-NEXT:    srl a3, a6, s1
+; RV32I-NEXT:    mv a5, t0
+; RV32I-NEXT:    bnez s1, .LBB13_67
+; RV32I-NEXT:    j .LBB13_68
+; RV32I-NEXT:  .LBB13_66:
+; RV32I-NEXT:    neg a3, s1
+; RV32I-NEXT:    sll a3, a6, a3
+; RV32I-NEXT:    or a3, a5, a3
+; RV32I-NEXT:    mv a5, t0
+; RV32I-NEXT:    beqz s1, .LBB13_68
+; RV32I-NEXT:  .LBB13_67:
+; RV32I-NEXT:    mv a5, a3
+; RV32I-NEXT:  .LBB13_68:
+; RV32I-NEXT:    bltu s1, t3, .LBB13_71
+; RV32I-NEXT:  # %bb.69:
+; RV32I-NEXT:    li s1, 0
+; RV32I-NEXT:    sw s5, 28(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    bgeu s2, t3, .LBB13_72
+; RV32I-NEXT:  .LBB13_70:
+; RV32I-NEXT:    sll s6, t1, s6
+; RV32I-NEXT:    srl a3, t1, s3
+; RV32I-NEXT:    lw s3, 20(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    or a3, a3, s3
+; RV32I-NEXT:    j .LBB13_73
+; RV32I-NEXT:  .LBB13_71:
+; RV32I-NEXT:    srl s1, a6, s3
+; RV32I-NEXT:    sw s5, 28(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    bltu s2, t3, .LBB13_70
+; RV32I-NEXT:  .LBB13_72:
+; RV32I-NEXT:    li s6, 0
+; RV32I-NEXT:    sll a3, t1, s2
+; RV32I-NEXT:  .LBB13_73:
+; RV32I-NEXT:    addi s9, s2, -64
+; RV32I-NEXT:    mv s5, t5
+; RV32I-NEXT:    beqz s2, .LBB13_75
+; RV32I-NEXT:  # %bb.74:
+; RV32I-NEXT:    mv s5, a3
+; RV32I-NEXT:  .LBB13_75:
+; RV32I-NEXT:    bltu s9, t3, .LBB13_77
+; RV32I-NEXT:  # %bb.76:
+; RV32I-NEXT:    li s3, 0
+; RV32I-NEXT:    sll a3, t0, s9
+; RV32I-NEXT:    mv s7, a6
+; RV32I-NEXT:    bnez s9, .LBB13_78
+; RV32I-NEXT:    j .LBB13_79
+; RV32I-NEXT:  .LBB13_77:
+; RV32I-NEXT:    sll s3, t0, s2
+; RV32I-NEXT:    neg a3, s9
+; RV32I-NEXT:    srl a3, t0, a3
+; RV32I-NEXT:    sll s7, a6, s2
+; RV32I-NEXT:    or a3, a3, s7
+; RV32I-NEXT:    mv s7, a6
+; RV32I-NEXT:    beqz s9, .LBB13_79
+; RV32I-NEXT:  .LBB13_78:
+; RV32I-NEXT:    mv s7, a3
+; RV32I-NEXT:  .LBB13_79:
+; RV32I-NEXT:    bltu s2, t6, .LBB13_81
+; RV32I-NEXT:  # %bb.80:
+; RV32I-NEXT:    sw zero, 20(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    li s8, 0
+; RV32I-NEXT:    j .LBB13_82
+; RV32I-NEXT:  .LBB13_81:
+; RV32I-NEXT:    sw s10, 20(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    or s3, a5, s6
+; RV32I-NEXT:    or s7, s1, s5
+; RV32I-NEXT:  .LBB13_82:
+; RV32I-NEXT:    addi ra, a4, -128
+; RV32I-NEXT:    mv s5, t1
+; RV32I-NEXT:    mv s6, t5
+; RV32I-NEXT:    beqz s2, .LBB13_84
+; RV32I-NEXT:  # %bb.83:
+; RV32I-NEXT:    mv s5, s3
+; RV32I-NEXT:    mv s6, s7
+; RV32I-NEXT:  .LBB13_84:
+; RV32I-NEXT:    neg s9, ra
+; RV32I-NEXT:    sll s3, t5, s9
+; RV32I-NEXT:    bltu ra, t3, .LBB13_86
+; RV32I-NEXT:  # %bb.85:
+; RV32I-NEXT:    srl a3, t5, ra
+; RV32I-NEXT:    mv s1, t1
+; RV32I-NEXT:    bnez ra, .LBB13_87
+; RV32I-NEXT:    j .LBB13_88
+; RV32I-NEXT:  .LBB13_86:
+; RV32I-NEXT:    lw a3, 36(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    or a3, a3, s3
+; RV32I-NEXT:    mv s1, t1
+; RV32I-NEXT:    beqz ra, .LBB13_88
+; RV32I-NEXT:  .LBB13_87:
+; RV32I-NEXT:    mv s1, a3
+; RV32I-NEXT:  .LBB13_88:
+; RV32I-NEXT:    bltu ra, t3, .LBB13_90
+; RV32I-NEXT:  # %bb.89:
+; RV32I-NEXT:    li s2, 0
+; RV32I-NEXT:    srl a3, a6, ra
+; RV32I-NEXT:    mv a5, t0
+; RV32I-NEXT:    bnez ra, .LBB13_91
+; RV32I-NEXT:    j .LBB13_92
+; RV32I-NEXT:  .LBB13_90:
+; RV32I-NEXT:    srl s2, t5, a4
+; RV32I-NEXT:    sll a3, a6, s9
+; RV32I-NEXT:    lw a5, 32(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    or a3, a5, a3
+; RV32I-NEXT:    mv a5, t0
+; RV32I-NEXT:    beqz ra, .LBB13_92
+; RV32I-NEXT:  .LBB13_91:
+; RV32I-NEXT:    mv a5, a3
+; RV32I-NEXT:  .LBB13_92:
+; RV32I-NEXT:    sub s10, t6, ra
+; RV32I-NEXT:    bltu ra, t3, .LBB13_95
+; RV32I-NEXT:  # %bb.93:
+; RV32I-NEXT:    li s7, 0
+; RV32I-NEXT:    bgeu s10, t3, .LBB13_96
+; RV32I-NEXT:  .LBB13_94:
+; RV32I-NEXT:    sll s9, t1, s9
+; RV32I-NEXT:    neg a3, s10
+; RV32I-NEXT:    srl a3, t1, a3
+; RV32I-NEXT:    or a3, a3, s3
+; RV32I-NEXT:    j .LBB13_97
+; RV32I-NEXT:  .LBB13_95:
+; RV32I-NEXT:    srl s7, a6, a4
+; RV32I-NEXT:    bltu s10, t3, .LBB13_94
+; RV32I-NEXT:  .LBB13_96:
+; RV32I-NEXT:    li s9, 0
+; RV32I-NEXT:    sll a3, t1, s10
+; RV32I-NEXT:  .LBB13_97:
+; RV32I-NEXT:    addi s11, ra, -64
+; RV32I-NEXT:    mv s3, t5
+; RV32I-NEXT:    beqz s10, .LBB13_99
+; RV32I-NEXT:  # %bb.98:
+; RV32I-NEXT:    mv s3, a3
+; RV32I-NEXT:  .LBB13_99:
+; RV32I-NEXT:    bltu s11, t3, .LBB13_101
+; RV32I-NEXT:  # %bb.100:
+; RV32I-NEXT:    srl a3, t5, s11
+; RV32I-NEXT:    bnez s11, .LBB13_102
+; RV32I-NEXT:    j .LBB13_103
+; RV32I-NEXT:  .LBB13_101:
+; RV32I-NEXT:    srl a3, t1, ra
+; RV32I-NEXT:    neg s10, s11
+; RV32I-NEXT:    sll s10, t5, s10
+; RV32I-NEXT:    or a3, a3, s10
+; RV32I-NEXT:    beqz s11, .LBB13_103
+; RV32I-NEXT:  .LBB13_102:
+; RV32I-NEXT:    mv t1, a3
+; RV32I-NEXT:  .LBB13_103:
+; RV32I-NEXT:    bltu s11, t3, .LBB13_105
+; RV32I-NEXT:  # %bb.104:
+; RV32I-NEXT:    li t3, 0
+; RV32I-NEXT:    bltu ra, t6, .LBB13_106
+; RV32I-NEXT:    j .LBB13_107
+; RV32I-NEXT:  .LBB13_105:
+; RV32I-NEXT:    srl t3, t5, ra
+; RV32I-NEXT:    bgeu ra, t6, .LBB13_107
+; RV32I-NEXT:  .LBB13_106:
+; RV32I-NEXT:    or t1, a5, s9
+; RV32I-NEXT:    or t3, s7, s3
+; RV32I-NEXT:  .LBB13_107:
+; RV32I-NEXT:    li a5, 128
+; RV32I-NEXT:    bnez ra, .LBB13_114
+; RV32I-NEXT:  # %bb.108:
+; RV32I-NEXT:    bgeu ra, t6, .LBB13_115
+; RV32I-NEXT:  .LBB13_109:
+; RV32I-NEXT:    bltu a4, a5, .LBB13_116
+; RV32I-NEXT:  .LBB13_110:
+; RV32I-NEXT:    lw ra, 40(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    bnez a4, .LBB13_117
+; RV32I-NEXT:  .LBB13_111:
+; RV32I-NEXT:    bltu a4, a5, .LBB13_113
+; RV32I-NEXT:  .LBB13_112:
+; RV32I-NEXT:    li s4, 0
+; RV32I-NEXT:    li a7, 0
+; RV32I-NEXT:    li a1, 0
+; RV32I-NEXT:    li ra, 0
+; RV32I-NEXT:  .LBB13_113:
+; RV32I-NEXT:    srli a4, s0, 16
+; RV32I-NEXT:    lui t1, 16
+; RV32I-NEXT:    srli t0, s0, 24
+; RV32I-NEXT:    srli a5, a0, 16
+; RV32I-NEXT:    srli t5, a0, 24
+; RV32I-NEXT:    srli a6, t4, 16
+; RV32I-NEXT:    srli s2, t4, 24
+; RV32I-NEXT:    srli t3, t2, 16
+; RV32I-NEXT:    srli s3, t2, 24
+; RV32I-NEXT:    srli s1, s4, 16
+; RV32I-NEXT:    srli a3, s4, 24
+; RV32I-NEXT:    srli t6, a7, 16
+; RV32I-NEXT:    srli s6, a7, 24
+; RV32I-NEXT:    srli s5, a1, 16
+; RV32I-NEXT:    srli s7, a1, 24
+; RV32I-NEXT:    srli s8, ra, 16
+; RV32I-NEXT:    srli s9, ra, 24
+; RV32I-NEXT:    addi t1, t1, -1
+; RV32I-NEXT:    and s10, s0, t1
+; RV32I-NEXT:    and s11, a0, t1
+; RV32I-NEXT:    srli s10, s10, 8
+; RV32I-NEXT:    sb s0, 0(a2)
+; RV32I-NEXT:    sb s10, 1(a2)
+; RV32I-NEXT:    sb a4, 2(a2)
+; RV32I-NEXT:    sb t0, 3(a2)
+; RV32I-NEXT:    and a4, t4, t1
+; RV32I-NEXT:    srli t0, s11, 8
+; RV32I-NEXT:    sb a0, 4(a2)
+; RV32I-NEXT:    sb t0, 5(a2)
+; RV32I-NEXT:    sb a5, 6(a2)
+; RV32I-NEXT:    sb t5, 7(a2)
+; RV32I-NEXT:    and a0, t2, t1
+; RV32I-NEXT:    srli a4, a4, 8
+; RV32I-NEXT:    sb t4, 8(a2)
+; RV32I-NEXT:    sb a4, 9(a2)
+; RV32I-NEXT:    sb a6, 10(a2)
+; RV32I-NEXT:    sb s2, 11(a2)
+; RV32I-NEXT:    and a4, s4, t1
+; RV32I-NEXT:    srli a0, a0, 8
+; RV32I-NEXT:    sb t2, 12(a2)
+; RV32I-NEXT:    sb a0, 13(a2)
+; RV32I-NEXT:    sb t3, 14(a2)
+; RV32I-NEXT:    sb s3, 15(a2)
+; RV32I-NEXT:    and a0, a7, t1
+; RV32I-NEXT:    srli a4, a4, 8
+; RV32I-NEXT:    sb s4, 16(a2)
+; RV32I-NEXT:    sb a4, 17(a2)
+; RV32I-NEXT:    sb s1, 18(a2)
+; RV32I-NEXT:    sb a3, 19(a2)
+; RV32I-NEXT:    and a3, a1, t1
+; RV32I-NEXT:    and a4, ra, t1
+; RV32I-NEXT:    srli a0, a0, 8
+; RV32I-NEXT:    srli a3, a3, 8
+; RV32I-NEXT:    srli a4, a4, 8
+; RV32I-NEXT:    sb a7, 20(a2)
+; RV32I-NEXT:    sb a0, 21(a2)
+; RV32I-NEXT:    sb t6, 22(a2)
+; RV32I-NEXT:    sb s6, 23(a2)
+; RV32I-NEXT:    sb a1, 24(a2)
+; RV32I-NEXT:    sb a3, 25(a2)
+; RV32I-NEXT:    sb s5, 26(a2)
+; RV32I-NEXT:    sb s7, 27(a2)
+; RV32I-NEXT:    sb ra, 28(a2)
+; RV32I-NEXT:    sb a4, 29(a2)
+; RV32I-NEXT:    sb s8, 30(a2)
+; RV32I-NEXT:    sb s9, 31(a2)
+; RV32I-NEXT:    lw ra, 92(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s0, 88(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s1, 84(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s2, 80(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s3, 76(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s4, 72(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s5, 68(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s6, 64(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s7, 60(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s8, 56(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s9, 52(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s10, 48(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s11, 44(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    addi sp, sp, 96
+; RV32I-NEXT:    ret
+; RV32I-NEXT:  .LBB13_114:
+; RV32I-NEXT:    mv t0, t1
+; RV32I-NEXT:    mv a6, t3
+; RV32I-NEXT:    bltu ra, t6, .LBB13_109
+; RV32I-NEXT:  .LBB13_115:
+; RV32I-NEXT:    li s1, 0
+; RV32I-NEXT:    li s2, 0
+; RV32I-NEXT:    bgeu a4, a5, .LBB13_110
+; RV32I-NEXT:  .LBB13_116:
+; RV32I-NEXT:    lw a3, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw a6, 20(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    or t0, a3, a6
+; RV32I-NEXT:    lw a3, 8(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    or a6, a3, s8
+; RV32I-NEXT:    lw a3, 28(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    or s1, a3, s5
+; RV32I-NEXT:    lw a3, 24(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    or s2, a3, s6
+; RV32I-NEXT:    lw ra, 40(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    beqz a4, .LBB13_111
+; RV32I-NEXT:  .LBB13_117:
+; RV32I-NEXT:    mv s0, t0
+; RV32I-NEXT:    mv a0, a6
+; RV32I-NEXT:    mv t4, s1
+; RV32I-NEXT:    mv t2, s2
+; RV32I-NEXT:    bgeu a4, a5, .LBB13_112
+; RV32I-NEXT:    j .LBB13_113
+  %src = load i256, ptr %src.ptr, align 1
+  %wordOff = load i256, ptr %wordOff.ptr, align 1
+  %bitOff = shl i256 %wordOff, 5
+  %res = lshr i256 %src, %bitOff
+  store i256 %res, ptr %dst, align 1
+  ret void
+}
+
+define void @lshr_32bytes_dwordOff(ptr %src.ptr, ptr %dwordOff.ptr, ptr %dst) nounwind {
+; RV64I-LABEL: lshr_32bytes_dwordOff:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -96
+; RV64I-NEXT:    sd s0, 88(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s1, 80(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s2, 72(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s3, 64(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s4, 56(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s5, 48(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s6, 40(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s7, 32(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s8, 24(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s9, 16(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s10, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s11, 0(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    lbu a3, 0(a0)
+; RV64I-NEXT:    lbu a4, 1(a0)
+; RV64I-NEXT:    lbu a5, 2(a0)
+; RV64I-NEXT:    lbu a6, 3(a0)
+; RV64I-NEXT:    lbu a7, 4(a0)
+; RV64I-NEXT:    lbu t0, 5(a0)
+; RV64I-NEXT:    lbu t1, 6(a0)
+; RV64I-NEXT:    lbu t2, 7(a0)
+; RV64I-NEXT:    lbu t3, 8(a0)
+; RV64I-NEXT:    lbu t4, 9(a0)
+; RV64I-NEXT:    lbu t5, 10(a0)
+; RV64I-NEXT:    lbu t6, 11(a0)
+; RV64I-NEXT:    lbu s0, 12(a0)
+; RV64I-NEXT:    lbu s1, 13(a0)
+; RV64I-NEXT:    lbu s2, 14(a0)
+; RV64I-NEXT:    lbu s3, 15(a0)
+; RV64I-NEXT:    lbu s4, 16(a0)
+; RV64I-NEXT:    lbu s5, 17(a0)
+; RV64I-NEXT:    lbu s6, 18(a0)
+; RV64I-NEXT:    lbu s7, 19(a0)
+; RV64I-NEXT:    slli a4, a4, 8
+; RV64I-NEXT:    slli a6, a6, 8
+; RV64I-NEXT:    slli t0, t0, 8
+; RV64I-NEXT:    or a3, a4, a3
+; RV64I-NEXT:    or a5, a6, a5
+; RV64I-NEXT:    or a7, t0, a7
+; RV64I-NEXT:    lbu s8, 20(a0)
+; RV64I-NEXT:    lbu s9, 21(a0)
+; RV64I-NEXT:    lbu s10, 22(a0)
+; RV64I-NEXT:    lbu s11, 23(a0)
+; RV64I-NEXT:    slli t2, t2, 8
+; RV64I-NEXT:    slli t4, t4, 8
+; RV64I-NEXT:    slli t6, t6, 8
+; RV64I-NEXT:    slli s1, s1, 8
+; RV64I-NEXT:    or t1, t2, t1
+; RV64I-NEXT:    or a4, t4, t3
+; RV64I-NEXT:    or a6, t6, t5
+; RV64I-NEXT:    or t0, s1, s0
+; RV64I-NEXT:    lbu t5, 24(a0)
+; RV64I-NEXT:    lbu t6, 25(a0)
+; RV64I-NEXT:    lbu s0, 26(a0)
+; RV64I-NEXT:    lbu s1, 27(a0)
+; RV64I-NEXT:    slli s3, s3, 8
+; RV64I-NEXT:    slli s5, s5, 8
+; RV64I-NEXT:    slli s7, s7, 8
+; RV64I-NEXT:    or t4, s3, s2
+; RV64I-NEXT:    or t2, s5, s4
+; RV64I-NEXT:    or t3, s7, s6
+; RV64I-NEXT:    lbu s2, 28(a0)
+; RV64I-NEXT:    lbu s3, 29(a0)
+; RV64I-NEXT:    lbu s4, 30(a0)
+; RV64I-NEXT:    lbu a0, 31(a0)
+; RV64I-NEXT:    slli s9, s9, 8
+; RV64I-NEXT:    slli s11, s11, 8
+; RV64I-NEXT:    slli t6, t6, 8
+; RV64I-NEXT:    slli s1, s1, 8
+; RV64I-NEXT:    or s5, s9, s8
+; RV64I-NEXT:    or s6, s11, s10
+; RV64I-NEXT:    or t5, t6, t5
+; RV64I-NEXT:    or s0, s1, s0
+; RV64I-NEXT:    lbu t6, 0(a1)
+; RV64I-NEXT:    lbu s1, 1(a1)
+; RV64I-NEXT:    lbu s7, 2(a1)
+; RV64I-NEXT:    lbu s8, 3(a1)
+; RV64I-NEXT:    slli s3, s3, 8
+; RV64I-NEXT:    slli a0, a0, 8
+; RV64I-NEXT:    slli s1, s1, 8
+; RV64I-NEXT:    or s2, s3, s2
+; RV64I-NEXT:    or s3, a0, s4
+; RV64I-NEXT:    or t6, s1, t6
+; RV64I-NEXT:    lbu a0, 4(a1)
+; RV64I-NEXT:    lbu s1, 5(a1)
+; RV64I-NEXT:    lbu s4, 6(a1)
+; RV64I-NEXT:    lbu a1, 7(a1)
+; RV64I-NEXT:    slli s8, s8, 8
+; RV64I-NEXT:    or s7, s8, s7
+; RV64I-NEXT:    slli s1, s1, 8
+; RV64I-NEXT:    or s1, s1, a0
+; RV64I-NEXT:    slli a1, a1, 8
+; RV64I-NEXT:    or s4, a1, s4
+; RV64I-NEXT:    slli t1, t1, 16
+; RV64I-NEXT:    or a1, t1, a7
+; RV64I-NEXT:    slli t4, t4, 16
+; RV64I-NEXT:    or a0, t4, t0
+; RV64I-NEXT:    slli t3, t3, 16
+; RV64I-NEXT:    or t0, t3, t2
+; RV64I-NEXT:    slli s6, s6, 16
+; RV64I-NEXT:    or t1, s6, s5
+; RV64I-NEXT:    slli s0, s0, 16
+; RV64I-NEXT:    or t3, s0, t5
+; RV64I-NEXT:    slli s3, s3, 16
+; RV64I-NEXT:    or t5, s3, s2
+; RV64I-NEXT:    slli s7, s7, 16
+; RV64I-NEXT:    or t6, s7, t6
+; RV64I-NEXT:    slli s4, s4, 16
+; RV64I-NEXT:    or s0, s4, s1
+; RV64I-NEXT:    li a7, 64
+; RV64I-NEXT:    slli t4, a5, 16
+; RV64I-NEXT:    slli t2, a6, 16
+; RV64I-NEXT:    slli t1, t1, 32
+; RV64I-NEXT:    slli t5, t5, 32
+; RV64I-NEXT:    slli s0, s0, 32
+; RV64I-NEXT:    or a6, t1, t0
+; RV64I-NEXT:    or t0, t5, t3
+; RV64I-NEXT:    or a5, s0, t6
+; RV64I-NEXT:    slli a5, a5, 6
+; RV64I-NEXT:    subw t1, a5, a7
+; RV64I-NEXT:    negw t5, a5
+; RV64I-NEXT:    sll t3, t0, t5
+; RV64I-NEXT:    bltu a5, a7, .LBB14_2
+; RV64I-NEXT:  # %bb.1:
+; RV64I-NEXT:    srl t6, t0, t1
+; RV64I-NEXT:    j .LBB14_3
+; RV64I-NEXT:  .LBB14_2:
+; RV64I-NEXT:    srl t6, a6, a5
+; RV64I-NEXT:    or t6, t6, t3
+; RV64I-NEXT:  .LBB14_3:
+; RV64I-NEXT:    or a3, t4, a3
+; RV64I-NEXT:    slli t4, a1, 32
+; RV64I-NEXT:    or t2, t2, a4
+; RV64I-NEXT:    slli a0, a0, 32
+; RV64I-NEXT:    mv a1, a6
+; RV64I-NEXT:    beqz a5, .LBB14_5
+; RV64I-NEXT:  # %bb.4:
+; RV64I-NEXT:    mv a1, t6
+; RV64I-NEXT:  .LBB14_5:
+; RV64I-NEXT:    or a4, t4, a3
+; RV64I-NEXT:    or a3, a0, t2
+; RV64I-NEXT:    bltu a5, a7, .LBB14_7
+; RV64I-NEXT:  # %bb.6:
+; RV64I-NEXT:    li a0, 0
+; RV64I-NEXT:    srl t4, a3, t1
+; RV64I-NEXT:    j .LBB14_8
+; RV64I-NEXT:  .LBB14_7:
+; RV64I-NEXT:    srl a0, t0, a5
+; RV64I-NEXT:    srl t1, a4, a5
+; RV64I-NEXT:    sll t2, a3, t5
+; RV64I-NEXT:    or t4, t1, t2
+; RV64I-NEXT:  .LBB14_8:
+; RV64I-NEXT:    li t1, 128
+; RV64I-NEXT:    mv t2, a4
+; RV64I-NEXT:    beqz a5, .LBB14_10
+; RV64I-NEXT:  # %bb.9:
+; RV64I-NEXT:    mv t2, t4
+; RV64I-NEXT:  .LBB14_10:
+; RV64I-NEXT:    sub t6, t1, a5
+; RV64I-NEXT:    bltu a5, a7, .LBB14_13
+; RV64I-NEXT:  # %bb.11:
+; RV64I-NEXT:    li t4, 0
+; RV64I-NEXT:    bgeu t6, a7, .LBB14_14
+; RV64I-NEXT:  .LBB14_12:
+; RV64I-NEXT:    sll t5, a6, t5
+; RV64I-NEXT:    negw s0, t6
+; RV64I-NEXT:    srl s0, a6, s0
+; RV64I-NEXT:    or s1, s0, t3
+; RV64I-NEXT:    j .LBB14_15
+; RV64I-NEXT:  .LBB14_13:
+; RV64I-NEXT:    srl t4, a3, a5
+; RV64I-NEXT:    bltu t6, a7, .LBB14_12
+; RV64I-NEXT:  .LBB14_14:
+; RV64I-NEXT:    li t5, 0
+; RV64I-NEXT:    subw t3, t6, a7
+; RV64I-NEXT:    sll s1, a6, t3
+; RV64I-NEXT:  .LBB14_15:
+; RV64I-NEXT:    sub s0, a5, t1
+; RV64I-NEXT:    mv t3, t0
+; RV64I-NEXT:    beqz t6, .LBB14_17
+; RV64I-NEXT:  # %bb.16:
+; RV64I-NEXT:    mv t3, s1
+; RV64I-NEXT:  .LBB14_17:
+; RV64I-NEXT:    bltu s0, a7, .LBB14_19
+; RV64I-NEXT:  # %bb.18:
+; RV64I-NEXT:    subw t6, s0, a7
+; RV64I-NEXT:    srl t6, t0, t6
+; RV64I-NEXT:    bnez s0, .LBB14_20
+; RV64I-NEXT:    j .LBB14_21
+; RV64I-NEXT:  .LBB14_19:
+; RV64I-NEXT:    srl t6, a6, s0
+; RV64I-NEXT:    negw s1, s0
+; RV64I-NEXT:    sll s1, t0, s1
+; RV64I-NEXT:    or t6, t6, s1
+; RV64I-NEXT:    beqz s0, .LBB14_21
+; RV64I-NEXT:  .LBB14_20:
+; RV64I-NEXT:    mv a6, t6
+; RV64I-NEXT:  .LBB14_21:
+; RV64I-NEXT:    bltu s0, a7, .LBB14_23
+; RV64I-NEXT:  # %bb.22:
+; RV64I-NEXT:    li a7, 0
+; RV64I-NEXT:    bltu a5, t1, .LBB14_24
+; RV64I-NEXT:    j .LBB14_25
+; RV64I-NEXT:  .LBB14_23:
+; RV64I-NEXT:    srl a7, t0, s0
+; RV64I-NEXT:    bgeu a5, t1, .LBB14_25
+; RV64I-NEXT:  .LBB14_24:
+; RV64I-NEXT:    or a6, t2, t5
+; RV64I-NEXT:    or a7, t4, t3
+; RV64I-NEXT:  .LBB14_25:
+; RV64I-NEXT:    bnez a5, .LBB14_29
+; RV64I-NEXT:  # %bb.26:
+; RV64I-NEXT:    bltu a5, t1, .LBB14_28
+; RV64I-NEXT:  .LBB14_27:
+; RV64I-NEXT:    li a1, 0
+; RV64I-NEXT:    li a0, 0
+; RV64I-NEXT:  .LBB14_28:
+; RV64I-NEXT:    srli a5, a4, 32
+; RV64I-NEXT:    srliw a6, a4, 16
+; RV64I-NEXT:    lui t2, 16
+; RV64I-NEXT:    srliw t1, a4, 24
+; RV64I-NEXT:    srli t0, a4, 48
+; RV64I-NEXT:    srli t5, a4, 56
+; RV64I-NEXT:    srli a7, a3, 32
+; RV64I-NEXT:    srliw t4, a3, 16
+; RV64I-NEXT:    srliw s0, a3, 24
+; RV64I-NEXT:    srli t6, a3, 48
+; RV64I-NEXT:    srli s3, a3, 56
+; RV64I-NEXT:    srli t3, a1, 32
+; RV64I-NEXT:    srliw s2, a1, 16
+; RV64I-NEXT:    srliw s6, a1, 24
+; RV64I-NEXT:    srli s4, a1, 48
+; RV64I-NEXT:    srli s7, a1, 56
+; RV64I-NEXT:    srli s1, a0, 32
+; RV64I-NEXT:    srliw s5, a0, 16
+; RV64I-NEXT:    srliw s8, a0, 24
+; RV64I-NEXT:    srli s9, a0, 48
+; RV64I-NEXT:    srli s10, a0, 56
+; RV64I-NEXT:    addi t2, t2, -1
+; RV64I-NEXT:    and s11, a4, t2
+; RV64I-NEXT:    srli s11, s11, 8
+; RV64I-NEXT:    sb a4, 0(a2)
+; RV64I-NEXT:    sb s11, 1(a2)
+; RV64I-NEXT:    sb a6, 2(a2)
+; RV64I-NEXT:    sb t1, 3(a2)
+; RV64I-NEXT:    and a4, a5, t2
+; RV64I-NEXT:    srli a4, a4, 8
+; RV64I-NEXT:    sb a5, 4(a2)
+; RV64I-NEXT:    sb a4, 5(a2)
+; RV64I-NEXT:    sb t0, 6(a2)
+; RV64I-NEXT:    sb t5, 7(a2)
+; RV64I-NEXT:    and a4, a3, t2
+; RV64I-NEXT:    srli a4, a4, 8
+; RV64I-NEXT:    sb a3, 8(a2)
+; RV64I-NEXT:    sb a4, 9(a2)
+; RV64I-NEXT:    sb t4, 10(a2)
+; RV64I-NEXT:    sb s0, 11(a2)
+; RV64I-NEXT:    and a3, a7, t2
+; RV64I-NEXT:    srli a3, a3, 8
+; RV64I-NEXT:    sb a7, 12(a2)
+; RV64I-NEXT:    sb a3, 13(a2)
+; RV64I-NEXT:    sb t6, 14(a2)
+; RV64I-NEXT:    sb s3, 15(a2)
+; RV64I-NEXT:    and a3, a1, t2
+; RV64I-NEXT:    srli a3, a3, 8
+; RV64I-NEXT:    sb a1, 16(a2)
+; RV64I-NEXT:    sb a3, 17(a2)
+; RV64I-NEXT:    sb s2, 18(a2)
+; RV64I-NEXT:    sb s6, 19(a2)
+; RV64I-NEXT:    and a1, t3, t2
+; RV64I-NEXT:    srli a1, a1, 8
+; RV64I-NEXT:    sb t3, 20(a2)
+; RV64I-NEXT:    sb a1, 21(a2)
+; RV64I-NEXT:    sb s4, 22(a2)
+; RV64I-NEXT:    sb s7, 23(a2)
+; RV64I-NEXT:    and a1, a0, t2
+; RV64I-NEXT:    and a3, s1, t2
+; RV64I-NEXT:    srli a1, a1, 8
+; RV64I-NEXT:    srli a3, a3, 8
+; RV64I-NEXT:    sb a0, 24(a2)
+; RV64I-NEXT:    sb a1, 25(a2)
+; RV64I-NEXT:    sb s5, 26(a2)
+; RV64I-NEXT:    sb s8, 27(a2)
+; RV64I-NEXT:    sb s1, 28(a2)
+; RV64I-NEXT:    sb a3, 29(a2)
+; RV64I-NEXT:    sb s9, 30(a2)
+; RV64I-NEXT:    sb s10, 31(a2)
+; RV64I-NEXT:    ld s0, 88(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s1, 80(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s2, 72(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s3, 64(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s4, 56(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s5, 48(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s6, 40(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s7, 32(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s8, 24(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s9, 16(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s10, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s11, 0(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    addi sp, sp, 96
+; RV64I-NEXT:    ret
+; RV64I-NEXT:  .LBB14_29:
+; RV64I-NEXT:    mv a4, a6
+; RV64I-NEXT:    mv a3, a7
+; RV64I-NEXT:    bgeu a5, t1, .LBB14_27
+; RV64I-NEXT:    j .LBB14_28
+;
+; RV32I-LABEL: lshr_32bytes_dwordOff:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    addi sp, sp, -96
+; RV32I-NEXT:    sw ra, 92(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s0, 88(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s1, 84(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s2, 80(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s3, 76(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s4, 72(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s5, 68(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s6, 64(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s7, 60(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s8, 56(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s9, 52(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s10, 48(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s11, 44(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    lbu a3, 16(a0)
+; RV32I-NEXT:    lbu a4, 17(a0)
+; RV32I-NEXT:    lbu a5, 18(a0)
+; RV32I-NEXT:    lbu a6, 19(a0)
+; RV32I-NEXT:    lbu a7, 20(a0)
+; RV32I-NEXT:    lbu t0, 21(a0)
+; RV32I-NEXT:    lbu t1, 22(a0)
+; RV32I-NEXT:    lbu t2, 23(a0)
+; RV32I-NEXT:    lbu t3, 24(a0)
+; RV32I-NEXT:    lbu t4, 25(a0)
+; RV32I-NEXT:    lbu t5, 26(a0)
+; RV32I-NEXT:    lbu t6, 27(a0)
+; RV32I-NEXT:    slli a4, a4, 8
+; RV32I-NEXT:    slli a6, a6, 8
+; RV32I-NEXT:    slli t0, t0, 8
+; RV32I-NEXT:    slli t2, t2, 8
+; RV32I-NEXT:    or a3, a4, a3
+; RV32I-NEXT:    or a6, a6, a5
+; RV32I-NEXT:    or a5, t0, a7
+; RV32I-NEXT:    or a4, t2, t1
+; RV32I-NEXT:    lbu a7, 28(a0)
+; RV32I-NEXT:    lbu t0, 29(a0)
+; RV32I-NEXT:    lbu t1, 30(a0)
+; RV32I-NEXT:    lbu t2, 31(a0)
+; RV32I-NEXT:    slli t4, t4, 8
+; RV32I-NEXT:    slli t6, t6, 8
+; RV32I-NEXT:    slli t0, t0, 8
+; RV32I-NEXT:    or t4, t4, t3
+; RV32I-NEXT:    or t5, t6, t5
+; RV32I-NEXT:    or a7, t0, a7
+; RV32I-NEXT:    lbu t0, 0(a1)
+; RV32I-NEXT:    lbu t3, 1(a1)
+; RV32I-NEXT:    lbu t6, 2(a1)
+; RV32I-NEXT:    lbu a1, 3(a1)
+; RV32I-NEXT:    slli t2, t2, 8
+; RV32I-NEXT:    or t1, t2, t1
+; RV32I-NEXT:    slli t3, t3, 8
+; RV32I-NEXT:    or t0, t3, t0
+; RV32I-NEXT:    slli a1, a1, 8
+; RV32I-NEXT:    or t2, a1, t6
+; RV32I-NEXT:    li t3, 32
+; RV32I-NEXT:    slli a6, a6, 16
+; RV32I-NEXT:    slli a1, a4, 16
+; RV32I-NEXT:    slli t5, t5, 16
+; RV32I-NEXT:    slli a4, t1, 16
+; RV32I-NEXT:    slli t2, t2, 16
+; RV32I-NEXT:    or t1, t5, t4
+; RV32I-NEXT:    or t5, a4, a7
+; RV32I-NEXT:    or a4, t2, t0
+; RV32I-NEXT:    slli a4, a4, 6
+; RV32I-NEXT:    srl s0, t1, a4
+; RV32I-NEXT:    neg s6, a4
+; RV32I-NEXT:    sll t4, t5, s6
+; RV32I-NEXT:    bltu a4, t3, .LBB14_2
+; RV32I-NEXT:  # %bb.1:
+; RV32I-NEXT:    srl a7, t5, a4
+; RV32I-NEXT:    j .LBB14_3
+; RV32I-NEXT:  .LBB14_2:
+; RV32I-NEXT:    or a7, s0, t4
+; RV32I-NEXT:  .LBB14_3:
+; RV32I-NEXT:    or t0, a6, a3
+; RV32I-NEXT:    or a6, a1, a5
+; RV32I-NEXT:    mv a1, t1
+; RV32I-NEXT:    beqz a4, .LBB14_5
+; RV32I-NEXT:  # %bb.4:
+; RV32I-NEXT:    mv a1, a7
+; RV32I-NEXT:  .LBB14_5:
+; RV32I-NEXT:    srl a3, t0, a4
+; RV32I-NEXT:    sll a5, a6, s6
+; RV32I-NEXT:    sw a3, 32(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    bltu a4, t3, .LBB14_7
+; RV32I-NEXT:  # %bb.6:
+; RV32I-NEXT:    li ra, 0
+; RV32I-NEXT:    srl a3, a6, a4
+; RV32I-NEXT:    j .LBB14_8
+; RV32I-NEXT:  .LBB14_7:
+; RV32I-NEXT:    srl ra, t5, a4
+; RV32I-NEXT:    or a3, a3, a5
+; RV32I-NEXT:  .LBB14_8:
+; RV32I-NEXT:    li t6, 64
+; RV32I-NEXT:    mv a7, t0
+; RV32I-NEXT:    beqz a4, .LBB14_10
+; RV32I-NEXT:  # %bb.9:
+; RV32I-NEXT:    mv a7, a3
+; RV32I-NEXT:  .LBB14_10:
+; RV32I-NEXT:    sw a5, 16(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sub s7, t6, a4
+; RV32I-NEXT:    bltu a4, t3, .LBB14_12
+; RV32I-NEXT:  # %bb.11:
+; RV32I-NEXT:    li a5, 0
+; RV32I-NEXT:    j .LBB14_13
+; RV32I-NEXT:  .LBB14_12:
+; RV32I-NEXT:    srl a5, a6, a4
+; RV32I-NEXT:  .LBB14_13:
+; RV32I-NEXT:    neg s10, s7
+; RV32I-NEXT:    sw t4, 20(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    bltu s7, t3, .LBB14_15
+; RV32I-NEXT:  # %bb.14:
+; RV32I-NEXT:    li t2, 0
+; RV32I-NEXT:    sll a3, t1, s7
+; RV32I-NEXT:    j .LBB14_16
+; RV32I-NEXT:  .LBB14_15:
+; RV32I-NEXT:    sll t2, t1, s6
+; RV32I-NEXT:    srl a3, t1, s10
+; RV32I-NEXT:    or a3, a3, t4
+; RV32I-NEXT:  .LBB14_16:
+; RV32I-NEXT:    addi s9, a4, -64
+; RV32I-NEXT:    mv t4, t5
+; RV32I-NEXT:    beqz s7, .LBB14_18
+; RV32I-NEXT:  # %bb.17:
+; RV32I-NEXT:    mv t4, a3
+; RV32I-NEXT:  .LBB14_18:
+; RV32I-NEXT:    neg s11, s9
+; RV32I-NEXT:    sw s0, 36(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    bltu s9, t3, .LBB14_20
+; RV32I-NEXT:  # %bb.19:
+; RV32I-NEXT:    srl s2, t5, s9
+; RV32I-NEXT:    j .LBB14_21
+; RV32I-NEXT:  .LBB14_20:
+; RV32I-NEXT:    sll a3, t5, s11
+; RV32I-NEXT:    or s2, s0, a3
+; RV32I-NEXT:  .LBB14_21:
+; RV32I-NEXT:    lbu s1, 11(a0)
+; RV32I-NEXT:    lbu a3, 15(a0)
+; RV32I-NEXT:    mv s0, t1
+; RV32I-NEXT:    beqz s9, .LBB14_23
+; RV32I-NEXT:  # %bb.22:
+; RV32I-NEXT:    mv s0, s2
+; RV32I-NEXT:  .LBB14_23:
+; RV32I-NEXT:    lbu s4, 9(a0)
+; RV32I-NEXT:    lbu s2, 10(a0)
+; RV32I-NEXT:    lbu s5, 13(a0)
+; RV32I-NEXT:    lbu s8, 14(a0)
+; RV32I-NEXT:    slli s3, s1, 8
+; RV32I-NEXT:    slli a3, a3, 8
+; RV32I-NEXT:    sw ra, 40(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    bltu s9, t3, .LBB14_25
+; RV32I-NEXT:  # %bb.24:
+; RV32I-NEXT:    li s1, 0
+; RV32I-NEXT:    j .LBB14_26
+; RV32I-NEXT:  .LBB14_25:
+; RV32I-NEXT:    srl s1, t5, a4
+; RV32I-NEXT:  .LBB14_26:
+; RV32I-NEXT:    or s2, s3, s2
+; RV32I-NEXT:    lbu ra, 8(a0)
+; RV32I-NEXT:    lbu s3, 12(a0)
+; RV32I-NEXT:    slli s4, s4, 8
+; RV32I-NEXT:    slli s5, s5, 8
+; RV32I-NEXT:    or s8, a3, s8
+; RV32I-NEXT:    bgeu a4, t6, .LBB14_28
+; RV32I-NEXT:  # %bb.27:
+; RV32I-NEXT:    or s0, a7, t2
+; RV32I-NEXT:    or s1, a5, t4
+; RV32I-NEXT:  .LBB14_28:
+; RV32I-NEXT:    lbu a3, 3(a0)
+; RV32I-NEXT:    lbu t2, 7(a0)
+; RV32I-NEXT:    or a5, s4, ra
+; RV32I-NEXT:    slli t4, s2, 16
+; RV32I-NEXT:    or s2, s5, s3
+; RV32I-NEXT:    slli s3, s8, 16
+; RV32I-NEXT:    mv s4, t0
+; RV32I-NEXT:    mv a7, a6
+; RV32I-NEXT:    beqz a4, .LBB14_30
+; RV32I-NEXT:  # %bb.29:
+; RV32I-NEXT:    mv s4, s0
+; RV32I-NEXT:    mv a7, s1
+; RV32I-NEXT:  .LBB14_30:
+; RV32I-NEXT:    slli s5, a3, 8
+; RV32I-NEXT:    lbu ra, 1(a0)
+; RV32I-NEXT:    lbu a3, 2(a0)
+; RV32I-NEXT:    lbu s1, 5(a0)
+; RV32I-NEXT:    lbu s0, 6(a0)
+; RV32I-NEXT:    slli s8, t2, 8
+; RV32I-NEXT:    or t4, t4, a5
+; RV32I-NEXT:    or t2, s3, s2
+; RV32I-NEXT:    bltu a4, t6, .LBB14_32
+; RV32I-NEXT:  # %bb.31:
+; RV32I-NEXT:    li a1, 0
+; RV32I-NEXT:    sw zero, 40(sp) # 4-byte Folded Spill
+; RV32I-NEXT:  .LBB14_32:
+; RV32I-NEXT:    slli s3, ra, 8
+; RV32I-NEXT:    or a5, s5, a3
+; RV32I-NEXT:    lbu s5, 0(a0)
+; RV32I-NEXT:    lbu a0, 4(a0)
+; RV32I-NEXT:    slli s1, s1, 8
+; RV32I-NEXT:    or a3, s8, s0
+; RV32I-NEXT:    srl s2, t4, a4
+; RV32I-NEXT:    sll ra, t2, s6
+; RV32I-NEXT:    bltu a4, t3, .LBB14_34
+; RV32I-NEXT:  # %bb.33:
+; RV32I-NEXT:    srl s0, t2, a4
+; RV32I-NEXT:    j .LBB14_35
+; RV32I-NEXT:  .LBB14_34:
+; RV32I-NEXT:    or s0, s2, ra
+; RV32I-NEXT:  .LBB14_35:
+; RV32I-NEXT:    or s3, s3, s5
+; RV32I-NEXT:    slli a5, a5, 16
+; RV32I-NEXT:    or a0, s1, a0
+; RV32I-NEXT:    slli a3, a3, 16
+; RV32I-NEXT:    mv s5, t4
+; RV32I-NEXT:    beqz a4, .LBB14_37
+; RV32I-NEXT:  # %bb.36:
+; RV32I-NEXT:    mv s5, s0
+; RV32I-NEXT:  .LBB14_37:
+; RV32I-NEXT:    or s0, a5, s3
+; RV32I-NEXT:    or a0, a3, a0
+; RV32I-NEXT:    bltu a4, t3, .LBB14_39
+; RV32I-NEXT:  # %bb.38:
+; RV32I-NEXT:    li s8, 0
+; RV32I-NEXT:    srl a3, a0, a4
+; RV32I-NEXT:    mv a5, s0
+; RV32I-NEXT:    bnez a4, .LBB14_40
+; RV32I-NEXT:    j .LBB14_41
+; RV32I-NEXT:  .LBB14_39:
+; RV32I-NEXT:    srl s8, t2, a4
+; RV32I-NEXT:    srl a3, s0, a4
+; RV32I-NEXT:    sll a5, a0, s6
+; RV32I-NEXT:    or a3, a3, a5
+; RV32I-NEXT:    mv a5, s0
+; RV32I-NEXT:    beqz a4, .LBB14_41
+; RV32I-NEXT:  .LBB14_40:
+; RV32I-NEXT:    mv a5, a3
+; RV32I-NEXT:  .LBB14_41:
+; RV32I-NEXT:    bltu a4, t3, .LBB14_44
+; RV32I-NEXT:  # %bb.42:
+; RV32I-NEXT:    li s1, 0
+; RV32I-NEXT:    bgeu s7, t3, .LBB14_45
+; RV32I-NEXT:  .LBB14_43:
+; RV32I-NEXT:    sll s3, t4, s6
+; RV32I-NEXT:    srl a3, t4, s10
+; RV32I-NEXT:    or a3, a3, ra
+; RV32I-NEXT:    mv s10, t2
+; RV32I-NEXT:    bnez s7, .LBB14_46
+; RV32I-NEXT:    j .LBB14_47
+; RV32I-NEXT:  .LBB14_44:
+; RV32I-NEXT:    srl s1, a0, a4
+; RV32I-NEXT:    bltu s7, t3, .LBB14_43
+; RV32I-NEXT:  .LBB14_45:
+; RV32I-NEXT:    li s3, 0
+; RV32I-NEXT:    sll a3, t4, s7
+; RV32I-NEXT:    mv s10, t2
+; RV32I-NEXT:    beqz s7, .LBB14_47
+; RV32I-NEXT:  .LBB14_46:
+; RV32I-NEXT:    mv s10, a3
+; RV32I-NEXT:  .LBB14_47:
+; RV32I-NEXT:    bltu s9, t3, .LBB14_49
+; RV32I-NEXT:  # %bb.48:
+; RV32I-NEXT:    srl a3, t2, s9
+; RV32I-NEXT:    mv s2, t4
+; RV32I-NEXT:    bnez s9, .LBB14_50
+; RV32I-NEXT:    j .LBB14_51
+; RV32I-NEXT:  .LBB14_49:
+; RV32I-NEXT:    sll a3, t2, s11
+; RV32I-NEXT:    or a3, s2, a3
+; RV32I-NEXT:    mv s2, t4
+; RV32I-NEXT:    beqz s9, .LBB14_51
+; RV32I-NEXT:  .LBB14_50:
+; RV32I-NEXT:    mv s2, a3
+; RV32I-NEXT:  .LBB14_51:
+; RV32I-NEXT:    bltu s9, t3, .LBB14_53
+; RV32I-NEXT:  # %bb.52:
+; RV32I-NEXT:    li s7, 0
+; RV32I-NEXT:    bltu a4, t6, .LBB14_54
+; RV32I-NEXT:    j .LBB14_55
+; RV32I-NEXT:  .LBB14_53:
+; RV32I-NEXT:    srl s7, t2, a4
+; RV32I-NEXT:    bgeu a4, t6, .LBB14_55
+; RV32I-NEXT:  .LBB14_54:
+; RV32I-NEXT:    or s2, a5, s3
+; RV32I-NEXT:    or s7, s1, s10
+; RV32I-NEXT:  .LBB14_55:
+; RV32I-NEXT:    li a3, 128
+; RV32I-NEXT:    mv a5, s0
+; RV32I-NEXT:    mv s1, a0
+; RV32I-NEXT:    beqz a4, .LBB14_57
+; RV32I-NEXT:  # %bb.56:
+; RV32I-NEXT:    mv a5, s2
+; RV32I-NEXT:    mv s1, s7
+; RV32I-NEXT:  .LBB14_57:
+; RV32I-NEXT:    sw a5, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sub s2, a3, a4
+; RV32I-NEXT:    bltu a4, t6, .LBB14_59
+; RV32I-NEXT:  # %bb.58:
+; RV32I-NEXT:    li s5, 0
+; RV32I-NEXT:    li s8, 0
+; RV32I-NEXT:  .LBB14_59:
+; RV32I-NEXT:    neg s3, s2
+; RV32I-NEXT:    srl a5, t0, s3
+; RV32I-NEXT:    sw s8, 24(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s1, 8(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    bltu s2, t3, .LBB14_61
+; RV32I-NEXT:  # %bb.60:
+; RV32I-NEXT:    li s10, 0
+; RV32I-NEXT:    sll a3, t0, s2
+; RV32I-NEXT:    j .LBB14_62
+; RV32I-NEXT:  .LBB14_61:
+; RV32I-NEXT:    sll s10, t0, s6
+; RV32I-NEXT:    lw a3, 16(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    or a3, a5, a3
+; RV32I-NEXT:  .LBB14_62:
+; RV32I-NEXT:    sub s1, t6, s2
+; RV32I-NEXT:    mv s8, a6
+; RV32I-NEXT:    beqz s2, .LBB14_64
+; RV32I-NEXT:  # %bb.63:
+; RV32I-NEXT:    mv s8, a3
+; RV32I-NEXT:  .LBB14_64:
+; RV32I-NEXT:    bltu s1, t3, .LBB14_66
+; RV32I-NEXT:  # %bb.65:
+; RV32I-NEXT:    srl a3, a6, s1
+; RV32I-NEXT:    mv a5, t0
+; RV32I-NEXT:    bnez s1, .LBB14_67
+; RV32I-NEXT:    j .LBB14_68
+; RV32I-NEXT:  .LBB14_66:
+; RV32I-NEXT:    neg a3, s1
+; RV32I-NEXT:    sll a3, a6, a3
+; RV32I-NEXT:    or a3, a5, a3
+; RV32I-NEXT:    mv a5, t0
+; RV32I-NEXT:    beqz s1, .LBB14_68
+; RV32I-NEXT:  .LBB14_67:
+; RV32I-NEXT:    mv a5, a3
+; RV32I-NEXT:  .LBB14_68:
+; RV32I-NEXT:    bltu s1, t3, .LBB14_71
+; RV32I-NEXT:  # %bb.69:
+; RV32I-NEXT:    li s1, 0
+; RV32I-NEXT:    sw s5, 28(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    bgeu s2, t3, .LBB14_72
+; RV32I-NEXT:  .LBB14_70:
+; RV32I-NEXT:    sll s6, t1, s6
+; RV32I-NEXT:    srl a3, t1, s3
+; RV32I-NEXT:    lw s3, 20(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    or a3, a3, s3
+; RV32I-NEXT:    j .LBB14_73
+; RV32I-NEXT:  .LBB14_71:
+; RV32I-NEXT:    srl s1, a6, s3
+; RV32I-NEXT:    sw s5, 28(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    bltu s2, t3, .LBB14_70
+; RV32I-NEXT:  .LBB14_72:
+; RV32I-NEXT:    li s6, 0
+; RV32I-NEXT:    sll a3, t1, s2
+; RV32I-NEXT:  .LBB14_73:
+; RV32I-NEXT:    addi s9, s2, -64
+; RV32I-NEXT:    mv s5, t5
+; RV32I-NEXT:    beqz s2, .LBB14_75
+; RV32I-NEXT:  # %bb.74:
+; RV32I-NEXT:    mv s5, a3
+; RV32I-NEXT:  .LBB14_75:
+; RV32I-NEXT:    bltu s9, t3, .LBB14_77
+; RV32I-NEXT:  # %bb.76:
+; RV32I-NEXT:    li s3, 0
+; RV32I-NEXT:    sll a3, t0, s9
+; RV32I-NEXT:    mv s7, a6
+; RV32I-NEXT:    bnez s9, .LBB14_78
+; RV32I-NEXT:    j .LBB14_79
+; RV32I-NEXT:  .LBB14_77:
+; RV32I-NEXT:    sll s3, t0, s2
+; RV32I-NEXT:    neg a3, s9
+; RV32I-NEXT:    srl a3, t0, a3
+; RV32I-NEXT:    sll s7, a6, s2
+; RV32I-NEXT:    or a3, a3, s7
+; RV32I-NEXT:    mv s7, a6
+; RV32I-NEXT:    beqz s9, .LBB14_79
+; RV32I-NEXT:  .LBB14_78:
+; RV32I-NEXT:    mv s7, a3
+; RV32I-NEXT:  .LBB14_79:
+; RV32I-NEXT:    bltu s2, t6, .LBB14_81
+; RV32I-NEXT:  # %bb.80:
+; RV32I-NEXT:    sw zero, 20(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    li s8, 0
+; RV32I-NEXT:    j .LBB14_82
+; RV32I-NEXT:  .LBB14_81:
+; RV32I-NEXT:    sw s10, 20(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    or s3, a5, s6
+; RV32I-NEXT:    or s7, s1, s5
+; RV32I-NEXT:  .LBB14_82:
+; RV32I-NEXT:    addi ra, a4, -128
+; RV32I-NEXT:    mv s5, t1
+; RV32I-NEXT:    mv s6, t5
+; RV32I-NEXT:    beqz s2, .LBB14_84
+; RV32I-NEXT:  # %bb.83:
+; RV32I-NEXT:    mv s5, s3
+; RV32I-NEXT:    mv s6, s7
+; RV32I-NEXT:  .LBB14_84:
+; RV32I-NEXT:    neg s9, ra
+; RV32I-NEXT:    sll s3, t5, s9
+; RV32I-NEXT:    bltu ra, t3, .LBB14_86
+; RV32I-NEXT:  # %bb.85:
+; RV32I-NEXT:    srl a3, t5, ra
+; RV32I-NEXT:    mv s1, t1
+; RV32I-NEXT:    bnez ra, .LBB14_87
+; RV32I-NEXT:    j .LBB14_88
+; RV32I-NEXT:  .LBB14_86:
+; RV32I-NEXT:    lw a3, 36(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    or a3, a3, s3
+; RV32I-NEXT:    mv s1, t1
+; RV32I-NEXT:    beqz ra, .LBB14_88
+; RV32I-NEXT:  .LBB14_87:
+; RV32I-NEXT:    mv s1, a3
+; RV32I-NEXT:  .LBB14_88:
+; RV32I-NEXT:    bltu ra, t3, .LBB14_90
+; RV32I-NEXT:  # %bb.89:
+; RV32I-NEXT:    li s2, 0
+; RV32I-NEXT:    srl a3, a6, ra
+; RV32I-NEXT:    mv a5, t0
+; RV32I-NEXT:    bnez ra, .LBB14_91
+; RV32I-NEXT:    j .LBB14_92
+; RV32I-NEXT:  .LBB14_90:
+; RV32I-NEXT:    srl s2, t5, a4
+; RV32I-NEXT:    sll a3, a6, s9
+; RV32I-NEXT:    lw a5, 32(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    or a3, a5, a3
+; RV32I-NEXT:    mv a5, t0
+; RV32I-NEXT:    beqz ra, .LBB14_92
+; RV32I-NEXT:  .LBB14_91:
+; RV32I-NEXT:    mv a5, a3
+; RV32I-NEXT:  .LBB14_92:
+; RV32I-NEXT:    sub s10, t6, ra
+; RV32I-NEXT:    bltu ra, t3, .LBB14_95
+; RV32I-NEXT:  # %bb.93:
+; RV32I-NEXT:    li s7, 0
+; RV32I-NEXT:    bgeu s10, t3, .LBB14_96
+; RV32I-NEXT:  .LBB14_94:
+; RV32I-NEXT:    sll s9, t1, s9
+; RV32I-NEXT:    neg a3, s10
+; RV32I-NEXT:    srl a3, t1, a3
+; RV32I-NEXT:    or a3, a3, s3
+; RV32I-NEXT:    j .LBB14_97
+; RV32I-NEXT:  .LBB14_95:
+; RV32I-NEXT:    srl s7, a6, a4
+; RV32I-NEXT:    bltu s10, t3, .LBB14_94
+; RV32I-NEXT:  .LBB14_96:
+; RV32I-NEXT:    li s9, 0
+; RV32I-NEXT:    sll a3, t1, s10
+; RV32I-NEXT:  .LBB14_97:
+; RV32I-NEXT:    addi s11, ra, -64
+; RV32I-NEXT:    mv s3, t5
+; RV32I-NEXT:    beqz s10, .LBB14_99
+; RV32I-NEXT:  # %bb.98:
+; RV32I-NEXT:    mv s3, a3
+; RV32I-NEXT:  .LBB14_99:
+; RV32I-NEXT:    bltu s11, t3, .LBB14_101
+; RV32I-NEXT:  # %bb.100:
+; RV32I-NEXT:    srl a3, t5, s11
+; RV32I-NEXT:    bnez s11, .LBB14_102
+; RV32I-NEXT:    j .LBB14_103
+; RV32I-NEXT:  .LBB14_101:
+; RV32I-NEXT:    srl a3, t1, ra
+; RV32I-NEXT:    neg s10, s11
+; RV32I-NEXT:    sll s10, t5, s10
+; RV32I-NEXT:    or a3, a3, s10
+; RV32I-NEXT:    beqz s11, .LBB14_103
+; RV32I-NEXT:  .LBB14_102:
+; RV32I-NEXT:    mv t1, a3
+; RV32I-NEXT:  .LBB14_103:
+; RV32I-NEXT:    bltu s11, t3, .LBB14_105
+; RV32I-NEXT:  # %bb.104:
+; RV32I-NEXT:    li t3, 0
+; RV32I-NEXT:    bltu ra, t6, .LBB14_106
+; RV32I-NEXT:    j .LBB14_107
+; RV32I-NEXT:  .LBB14_105:
+; RV32I-NEXT:    srl t3, t5, ra
+; RV32I-NEXT:    bgeu ra, t6, .LBB14_107
+; RV32I-NEXT:  .LBB14_106:
+; RV32I-NEXT:    or t1, a5, s9
+; RV32I-NEXT:    or t3, s7, s3
+; RV32I-NEXT:  .LBB14_107:
+; RV32I-NEXT:    li a5, 128
+; RV32I-NEXT:    bnez ra, .LBB14_114
+; RV32I-NEXT:  # %bb.108:
+; RV32I-NEXT:    bgeu ra, t6, .LBB14_115
+; RV32I-NEXT:  .LBB14_109:
+; RV32I-NEXT:    bltu a4, a5, .LBB14_116
+; RV32I-NEXT:  .LBB14_110:
+; RV32I-NEXT:    lw ra, 40(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    bnez a4, .LBB14_117
+; RV32I-NEXT:  .LBB14_111:
+; RV32I-NEXT:    bltu a4, a5, .LBB14_113
+; RV32I-NEXT:  .LBB14_112:
+; RV32I-NEXT:    li s4, 0
+; RV32I-NEXT:    li a7, 0
+; RV32I-NEXT:    li a1, 0
+; RV32I-NEXT:    li ra, 0
+; RV32I-NEXT:  .LBB14_113:
+; RV32I-NEXT:    srli a4, s0, 16
+; RV32I-NEXT:    lui t1, 16
+; RV32I-NEXT:    srli t0, s0, 24
+; RV32I-NEXT:    srli a5, a0, 16
+; RV32I-NEXT:    srli t5, a0, 24
+; RV32I-NEXT:    srli a6, t4, 16
+; RV32I-NEXT:    srli s2, t4, 24
+; RV32I-NEXT:    srli t3, t2, 16
+; RV32I-NEXT:    srli s3, t2, 24
+; RV32I-NEXT:    srli s1, s4, 16
+; RV32I-NEXT:    srli a3, s4, 24
+; RV32I-NEXT:    srli t6, a7, 16
+; RV32I-NEXT:    srli s6, a7, 24
+; RV32I-NEXT:    srli s5, a1, 16
+; RV32I-NEXT:    srli s7, a1, 24
+; RV32I-NEXT:    srli s8, ra, 16
+; RV32I-NEXT:    srli s9, ra, 24
+; RV32I-NEXT:    addi t1, t1, -1
+; RV32I-NEXT:    and s10, s0, t1
+; RV32I-NEXT:    and s11, a0, t1
+; RV32I-NEXT:    srli s10, s10, 8
+; RV32I-NEXT:    sb s0, 0(a2)
+; RV32I-NEXT:    sb s10, 1(a2)
+; RV32I-NEXT:    sb a4, 2(a2)
+; RV32I-NEXT:    sb t0, 3(a2)
+; RV32I-NEXT:    and a4, t4, t1
+; RV32I-NEXT:    srli t0, s11, 8
+; RV32I-NEXT:    sb a0, 4(a2)
+; RV32I-NEXT:    sb t0, 5(a2)
+; RV32I-NEXT:    sb a5, 6(a2)
+; RV32I-NEXT:    sb t5, 7(a2)
+; RV32I-NEXT:    and a0, t2, t1
+; RV32I-NEXT:    srli a4, a4, 8
+; RV32I-NEXT:    sb t4, 8(a2)
+; RV32I-NEXT:    sb a4, 9(a2)
+; RV32I-NEXT:    sb a6, 10(a2)
+; RV32I-NEXT:    sb s2, 11(a2)
+; RV32I-NEXT:    and a4, s4, t1
+; RV32I-NEXT:    srli a0, a0, 8
+; RV32I-NEXT:    sb t2, 12(a2)
+; RV32I-NEXT:    sb a0, 13(a2)
+; RV32I-NEXT:    sb t3, 14(a2)
+; RV32I-NEXT:    sb s3, 15(a2)
+; RV32I-NEXT:    and a0, a7, t1
+; RV32I-NEXT:    srli a4, a4, 8
+; RV32I-NEXT:    sb s4, 16(a2)
+; RV32I-NEXT:    sb a4, 17(a2)
+; RV32I-NEXT:    sb s1, 18(a2)
+; RV32I-NEXT:    sb a3, 19(a2)
+; RV32I-NEXT:    and a3, a1, t1
+; RV32I-NEXT:    and a4, ra, t1
+; RV32I-NEXT:    srli a0, a0, 8
+; RV32I-NEXT:    srli a3, a3, 8
+; RV32I-NEXT:    srli a4, a4, 8
+; RV32I-NEXT:    sb a7, 20(a2)
+; RV32I-NEXT:    sb a0, 21(a2)
+; RV32I-NEXT:    sb t6, 22(a2)
+; RV32I-NEXT:    sb s6, 23(a2)
+; RV32I-NEXT:    sb a1, 24(a2)
+; RV32I-NEXT:    sb a3, 25(a2)
+; RV32I-NEXT:    sb s5, 26(a2)
+; RV32I-NEXT:    sb s7, 27(a2)
+; RV32I-NEXT:    sb ra, 28(a2)
+; RV32I-NEXT:    sb a4, 29(a2)
+; RV32I-NEXT:    sb s8, 30(a2)
+; RV32I-NEXT:    sb s9, 31(a2)
+; RV32I-NEXT:    lw ra, 92(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s0, 88(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s1, 84(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s2, 80(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s3, 76(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s4, 72(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s5, 68(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s6, 64(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s7, 60(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s8, 56(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s9, 52(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s10, 48(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s11, 44(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    addi sp, sp, 96
+; RV32I-NEXT:    ret
+; RV32I-NEXT:  .LBB14_114:
+; RV32I-NEXT:    mv t0, t1
+; RV32I-NEXT:    mv a6, t3
+; RV32I-NEXT:    bltu ra, t6, .LBB14_109
+; RV32I-NEXT:  .LBB14_115:
+; RV32I-NEXT:    li s1, 0
+; RV32I-NEXT:    li s2, 0
+; RV32I-NEXT:    bgeu a4, a5, .LBB14_110
+; RV32I-NEXT:  .LBB14_116:
+; RV32I-NEXT:    lw a3, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw a6, 20(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    or t0, a3, a6
+; RV32I-NEXT:    lw a3, 8(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    or a6, a3, s8
+; RV32I-NEXT:    lw a3, 28(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    or s1, a3, s5
+; RV32I-NEXT:    lw a3, 24(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    or s2, a3, s6
+; RV32I-NEXT:    lw ra, 40(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    beqz a4, .LBB14_111
+; RV32I-NEXT:  .LBB14_117:
+; RV32I-NEXT:    mv s0, t0
+; RV32I-NEXT:    mv a0, a6
+; RV32I-NEXT:    mv t4, s1
+; RV32I-NEXT:    mv t2, s2
+; RV32I-NEXT:    bgeu a4, a5, .LBB14_112
+; RV32I-NEXT:    j .LBB14_113
+  %src = load i256, ptr %src.ptr, align 1
+  %dwordOff = load i256, ptr %dwordOff.ptr, align 1
+  %bitOff = shl i256 %dwordOff, 6
+  %res = lshr i256 %src, %bitOff
+  store i256 %res, ptr %dst, align 1
+  ret void
+}
+
+define void @shl_32bytes(ptr %src.ptr, ptr %byteOff.ptr, ptr %dst) nounwind {
+; RV64I-LABEL: shl_32bytes:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -112
+; RV64I-NEXT:    sd ra, 104(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s0, 96(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s1, 88(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s2, 80(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s3, 72(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s4, 64(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s5, 56(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s6, 48(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s7, 40(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s8, 32(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s9, 24(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s10, 16(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s11, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    lbu a3, 0(a0)
+; RV64I-NEXT:    lbu a4, 1(a0)
+; RV64I-NEXT:    lbu a5, 2(a0)
+; RV64I-NEXT:    lbu a6, 3(a0)
+; RV64I-NEXT:    lbu a7, 4(a0)
+; RV64I-NEXT:    lbu t0, 5(a0)
+; RV64I-NEXT:    lbu t1, 6(a0)
+; RV64I-NEXT:    lbu t2, 7(a0)
+; RV64I-NEXT:    lbu t3, 8(a0)
+; RV64I-NEXT:    lbu t4, 9(a0)
+; RV64I-NEXT:    lbu t5, 10(a0)
+; RV64I-NEXT:    lbu t6, 11(a0)
+; RV64I-NEXT:    slli a4, a4, 8
+; RV64I-NEXT:    or a3, a4, a3
+; RV64I-NEXT:    lbu a4, 12(a0)
+; RV64I-NEXT:    lbu s0, 13(a0)
+; RV64I-NEXT:    lbu s1, 14(a0)
+; RV64I-NEXT:    lbu s2, 15(a0)
+; RV64I-NEXT:    slli a6, a6, 8
+; RV64I-NEXT:    slli t0, t0, 8
+; RV64I-NEXT:    slli t2, t2, 8
+; RV64I-NEXT:    slli t4, t4, 8
+; RV64I-NEXT:    or s3, a6, a5
+; RV64I-NEXT:    or a5, t0, a7
+; RV64I-NEXT:    or a7, t2, t1
+; RV64I-NEXT:    or a6, t4, t3
+; RV64I-NEXT:    lbu t0, 0(a1)
+; RV64I-NEXT:    lbu t1, 1(a1)
+; RV64I-NEXT:    lbu t2, 2(a1)
+; RV64I-NEXT:    lbu t3, 3(a1)
+; RV64I-NEXT:    slli t6, t6, 8
+; RV64I-NEXT:    slli s0, s0, 8
+; RV64I-NEXT:    slli s2, s2, 8
+; RV64I-NEXT:    or t6, t6, t5
+; RV64I-NEXT:    or s0, s0, a4
+; RV64I-NEXT:    or s1, s2, s1
+; RV64I-NEXT:    lbu a4, 4(a1)
+; RV64I-NEXT:    lbu t4, 5(a1)
+; RV64I-NEXT:    lbu t5, 6(a1)
+; RV64I-NEXT:    lbu a1, 7(a1)
+; RV64I-NEXT:    slli t1, t1, 8
+; RV64I-NEXT:    slli t3, t3, 8
+; RV64I-NEXT:    slli t4, t4, 8
+; RV64I-NEXT:    slli a1, a1, 8
+; RV64I-NEXT:    or t0, t1, t0
+; RV64I-NEXT:    or t1, t3, t2
+; RV64I-NEXT:    or t2, t4, a4
+; RV64I-NEXT:    or a1, a1, t5
+; RV64I-NEXT:    lbu t5, 19(a0)
+; RV64I-NEXT:    lbu t4, 21(a0)
+; RV64I-NEXT:    lbu a4, 22(a0)
+; RV64I-NEXT:    lbu t3, 23(a0)
+; RV64I-NEXT:    slli s3, s3, 16
+; RV64I-NEXT:    slli a7, a7, 16
+; RV64I-NEXT:    slli t6, t6, 16
+; RV64I-NEXT:    slli s1, s1, 16
+; RV64I-NEXT:    or s4, s3, a3
+; RV64I-NEXT:    or a5, a7, a5
+; RV64I-NEXT:    or a6, t6, a6
+; RV64I-NEXT:    or s0, s1, s0
+; RV64I-NEXT:    lbu s1, 27(a0)
+; RV64I-NEXT:    lbu t6, 29(a0)
+; RV64I-NEXT:    lbu a3, 30(a0)
+; RV64I-NEXT:    lbu s2, 31(a0)
+; RV64I-NEXT:    slli t1, t1, 16
+; RV64I-NEXT:    or s5, t1, t0
+; RV64I-NEXT:    li a7, 128
+; RV64I-NEXT:    slli a1, a1, 16
+; RV64I-NEXT:    or a1, a1, t2
+; RV64I-NEXT:    li t0, 64
+; RV64I-NEXT:    slli s3, t3, 8
+; RV64I-NEXT:    slli s2, s2, 8
+; RV64I-NEXT:    slli a5, a5, 32
+; RV64I-NEXT:    slli s0, s0, 32
+; RV64I-NEXT:    slli a1, a1, 32
+; RV64I-NEXT:    or t1, a5, s4
+; RV64I-NEXT:    or a5, s0, a6
+; RV64I-NEXT:    or a6, a1, s5
+; RV64I-NEXT:    slli a6, a6, 3
+; RV64I-NEXT:    subw t2, a6, t0
+; RV64I-NEXT:    negw t3, a6
+; RV64I-NEXT:    srl s0, t1, t3
+; RV64I-NEXT:    bltu a6, t0, .LBB15_2
+; RV64I-NEXT:  # %bb.1:
+; RV64I-NEXT:    li a1, 0
+; RV64I-NEXT:    sll s4, t1, t2
+; RV64I-NEXT:    j .LBB15_3
+; RV64I-NEXT:  .LBB15_2:
+; RV64I-NEXT:    sll a1, t1, a6
+; RV64I-NEXT:    sll s4, a5, a6
+; RV64I-NEXT:    or s4, s0, s4
+; RV64I-NEXT:  .LBB15_3:
+; RV64I-NEXT:    slli t5, t5, 8
+; RV64I-NEXT:    slli t4, t4, 8
+; RV64I-NEXT:    or s3, s3, a4
+; RV64I-NEXT:    lbu ra, 17(a0)
+; RV64I-NEXT:    lbu s11, 18(a0)
+; RV64I-NEXT:    lbu s8, 20(a0)
+; RV64I-NEXT:    lbu s5, 25(a0)
+; RV64I-NEXT:    slli s1, s1, 8
+; RV64I-NEXT:    lbu s7, 26(a0)
+; RV64I-NEXT:    lbu s6, 28(a0)
+; RV64I-NEXT:    slli s10, t6, 8
+; RV64I-NEXT:    or s9, s2, a3
+; RV64I-NEXT:    sub a4, a7, a6
+; RV64I-NEXT:    mv a3, a5
+; RV64I-NEXT:    beqz a6, .LBB15_5
+; RV64I-NEXT:  # %bb.4:
+; RV64I-NEXT:    mv a3, s4
+; RV64I-NEXT:  .LBB15_5:
+; RV64I-NEXT:    slli t6, ra, 8
+; RV64I-NEXT:    or t5, t5, s11
+; RV64I-NEXT:    or t4, t4, s8
+; RV64I-NEXT:    slli s3, s3, 16
+; RV64I-NEXT:    lbu s8, 16(a0)
+; RV64I-NEXT:    lbu a0, 24(a0)
+; RV64I-NEXT:    slli s5, s5, 8
+; RV64I-NEXT:    or s2, s1, s7
+; RV64I-NEXT:    or s1, s10, s6
+; RV64I-NEXT:    slli s4, s9, 16
+; RV64I-NEXT:    bltu a4, t0, .LBB15_7
+; RV64I-NEXT:  # %bb.6:
+; RV64I-NEXT:    subw s0, a4, t0
+; RV64I-NEXT:    srl s0, a5, s0
+; RV64I-NEXT:    j .LBB15_8
+; RV64I-NEXT:  .LBB15_7:
+; RV64I-NEXT:    negw s6, a4
+; RV64I-NEXT:    sll s6, a5, s6
+; RV64I-NEXT:    or s0, s0, s6
+; RV64I-NEXT:  .LBB15_8:
+; RV64I-NEXT:    or t6, t6, s8
+; RV64I-NEXT:    slli s6, t5, 16
+; RV64I-NEXT:    or s3, s3, t4
+; RV64I-NEXT:    or t5, s5, a0
+; RV64I-NEXT:    slli s2, s2, 16
+; RV64I-NEXT:    or s1, s4, s1
+; RV64I-NEXT:    mv t4, t1
+; RV64I-NEXT:    beqz a4, .LBB15_10
+; RV64I-NEXT:  # %bb.9:
+; RV64I-NEXT:    mv t4, s0
+; RV64I-NEXT:  .LBB15_10:
+; RV64I-NEXT:    or a0, s6, t6
+; RV64I-NEXT:    slli s0, s3, 32
+; RV64I-NEXT:    or t6, s2, t5
+; RV64I-NEXT:    slli s1, s1, 32
+; RV64I-NEXT:    bltu a4, t0, .LBB15_12
+; RV64I-NEXT:  # %bb.11:
+; RV64I-NEXT:    li t5, 0
+; RV64I-NEXT:    j .LBB15_13
+; RV64I-NEXT:  .LBB15_12:
+; RV64I-NEXT:    srl t5, a5, t3
+; RV64I-NEXT:  .LBB15_13:
+; RV64I-NEXT:    or a4, s0, a0
+; RV64I-NEXT:    or a0, s1, t6
+; RV64I-NEXT:    bltu a6, t0, .LBB15_15
+; RV64I-NEXT:  # %bb.14:
+; RV64I-NEXT:    li t6, 0
+; RV64I-NEXT:    sll t2, a4, t2
+; RV64I-NEXT:    j .LBB15_16
+; RV64I-NEXT:  .LBB15_15:
+; RV64I-NEXT:    sll t6, a4, a6
+; RV64I-NEXT:    srl t2, a4, t3
+; RV64I-NEXT:    sll t3, a0, a6
+; RV64I-NEXT:    or t2, t2, t3
+; RV64I-NEXT:  .LBB15_16:
+; RV64I-NEXT:    sub s0, a6, a7
+; RV64I-NEXT:    mv t3, a0
+; RV64I-NEXT:    beqz a6, .LBB15_18
+; RV64I-NEXT:  # %bb.17:
+; RV64I-NEXT:    mv t3, t2
+; RV64I-NEXT:  .LBB15_18:
+; RV64I-NEXT:    bltu s0, t0, .LBB15_20
+; RV64I-NEXT:  # %bb.19:
+; RV64I-NEXT:    li t2, 0
+; RV64I-NEXT:    subw t0, s0, t0
+; RV64I-NEXT:    sll t0, t1, t0
+; RV64I-NEXT:    bnez s0, .LBB15_21
+; RV64I-NEXT:    j .LBB15_22
+; RV64I-NEXT:  .LBB15_20:
+; RV64I-NEXT:    sll t2, t1, s0
+; RV64I-NEXT:    negw t0, s0
+; RV64I-NEXT:    srl t0, t1, t0
+; RV64I-NEXT:    sll t1, a5, s0
+; RV64I-NEXT:    or t0, t0, t1
+; RV64I-NEXT:    beqz s0, .LBB15_22
+; RV64I-NEXT:  .LBB15_21:
+; RV64I-NEXT:    mv a5, t0
+; RV64I-NEXT:  .LBB15_22:
+; RV64I-NEXT:    bltu a6, a7, .LBB15_24
+; RV64I-NEXT:  # %bb.23:
+; RV64I-NEXT:    li a1, 0
+; RV64I-NEXT:    li a3, 0
+; RV64I-NEXT:    bnez a6, .LBB15_25
+; RV64I-NEXT:    j .LBB15_26
+; RV64I-NEXT:  .LBB15_24:
+; RV64I-NEXT:    or t2, t4, t6
+; RV64I-NEXT:    or a5, t5, t3
+; RV64I-NEXT:    beqz a6, .LBB15_26
+; RV64I-NEXT:  .LBB15_25:
+; RV64I-NEXT:    mv a4, t2
+; RV64I-NEXT:    mv a0, a5
+; RV64I-NEXT:  .LBB15_26:
+; RV64I-NEXT:    srli a5, a1, 32
+; RV64I-NEXT:    srliw a6, a1, 16
+; RV64I-NEXT:    lui t2, 16
+; RV64I-NEXT:    srliw t1, a1, 24
+; RV64I-NEXT:    srli t0, a1, 48
+; RV64I-NEXT:    srli t5, a1, 56
+; RV64I-NEXT:    srli a7, a3, 32
+; RV64I-NEXT:    srliw t4, a3, 16
+; RV64I-NEXT:    srliw s0, a3, 24
+; RV64I-NEXT:    srli t6, a3, 48
+; RV64I-NEXT:    srli s3, a3, 56
+; RV64I-NEXT:    srli t3, a4, 32
+; RV64I-NEXT:    srliw s2, a4, 16
+; RV64I-NEXT:    srliw s6, a4, 24
+; RV64I-NEXT:    srli s4, a4, 48
+; RV64I-NEXT:    srli s7, a4, 56
+; RV64I-NEXT:    srli s1, a0, 32
+; RV64I-NEXT:    srliw s5, a0, 16
+; RV64I-NEXT:    srliw s8, a0, 24
+; RV64I-NEXT:    srli s9, a0, 48
+; RV64I-NEXT:    srli s10, a0, 56
+; RV64I-NEXT:    addi t2, t2, -1
+; RV64I-NEXT:    and s11, a1, t2
+; RV64I-NEXT:    srli s11, s11, 8
+; RV64I-NEXT:    sb a1, 0(a2)
+; RV64I-NEXT:    sb s11, 1(a2)
+; RV64I-NEXT:    sb a6, 2(a2)
+; RV64I-NEXT:    sb t1, 3(a2)
+; RV64I-NEXT:    and a1, a5, t2
+; RV64I-NEXT:    srli a1, a1, 8
+; RV64I-NEXT:    sb a5, 4(a2)
+; RV64I-NEXT:    sb a1, 5(a2)
+; RV64I-NEXT:    sb t0, 6(a2)
+; RV64I-NEXT:    sb t5, 7(a2)
+; RV64I-NEXT:    and a1, a3, t2
+; RV64I-NEXT:    srli a1, a1, 8
+; RV64I-NEXT:    sb a3, 8(a2)
+; RV64I-NEXT:    sb a1, 9(a2)
+; RV64I-NEXT:    sb t4, 10(a2)
+; RV64I-NEXT:    sb s0, 11(a2)
+; RV64I-NEXT:    and a1, a7, t2
+; RV64I-NEXT:    srli a1, a1, 8
+; RV64I-NEXT:    sb a7, 12(a2)
+; RV64I-NEXT:    sb a1, 13(a2)
+; RV64I-NEXT:    sb t6, 14(a2)
+; RV64I-NEXT:    sb s3, 15(a2)
+; RV64I-NEXT:    and a1, a4, t2
+; RV64I-NEXT:    srli a1, a1, 8
+; RV64I-NEXT:    sb a4, 16(a2)
+; RV64I-NEXT:    sb a1, 17(a2)
+; RV64I-NEXT:    sb s2, 18(a2)
+; RV64I-NEXT:    sb s6, 19(a2)
+; RV64I-NEXT:    and a1, t3, t2
+; RV64I-NEXT:    srli a1, a1, 8
+; RV64I-NEXT:    sb t3, 20(a2)
+; RV64I-NEXT:    sb a1, 21(a2)
+; RV64I-NEXT:    sb s4, 22(a2)
+; RV64I-NEXT:    sb s7, 23(a2)
+; RV64I-NEXT:    and a1, a0, t2
+; RV64I-NEXT:    and a3, s1, t2
+; RV64I-NEXT:    srli a1, a1, 8
+; RV64I-NEXT:    srli a3, a3, 8
+; RV64I-NEXT:    sb a0, 24(a2)
+; RV64I-NEXT:    sb a1, 25(a2)
+; RV64I-NEXT:    sb s5, 26(a2)
+; RV64I-NEXT:    sb s8, 27(a2)
+; RV64I-NEXT:    sb s1, 28(a2)
+; RV64I-NEXT:    sb a3, 29(a2)
+; RV64I-NEXT:    sb s9, 30(a2)
+; RV64I-NEXT:    sb s10, 31(a2)
+; RV64I-NEXT:    ld ra, 104(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s0, 96(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s1, 88(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s2, 80(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s3, 72(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s4, 64(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s5, 56(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s6, 48(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s7, 40(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s8, 32(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s9, 24(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s10, 16(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s11, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    addi sp, sp, 112
+; RV64I-NEXT:    ret
+;
+; RV32I-LABEL: shl_32bytes:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    addi sp, sp, -96
+; RV32I-NEXT:    sw ra, 92(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s0, 88(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s1, 84(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s2, 80(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s3, 76(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s4, 72(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s5, 68(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s6, 64(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s7, 60(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s8, 56(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s9, 52(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s10, 48(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s11, 44(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    lbu a3, 0(a0)
+; RV32I-NEXT:    lbu a4, 1(a0)
+; RV32I-NEXT:    lbu a5, 2(a0)
+; RV32I-NEXT:    lbu a7, 3(a0)
+; RV32I-NEXT:    lbu t0, 4(a0)
+; RV32I-NEXT:    lbu t1, 5(a0)
+; RV32I-NEXT:    lbu t2, 6(a0)
+; RV32I-NEXT:    lbu t3, 7(a0)
+; RV32I-NEXT:    slli a4, a4, 8
+; RV32I-NEXT:    or a4, a4, a3
+; RV32I-NEXT:    lbu a6, 11(a0)
+; RV32I-NEXT:    lbu a3, 15(a0)
+; RV32I-NEXT:    slli a7, a7, 8
+; RV32I-NEXT:    slli t1, t1, 8
+; RV32I-NEXT:    slli t3, t3, 8
+; RV32I-NEXT:    or a5, a7, a5
+; RV32I-NEXT:    or a7, t1, t0
+; RV32I-NEXT:    lbu t0, 0(a1)
+; RV32I-NEXT:    lbu t1, 1(a1)
+; RV32I-NEXT:    or t2, t3, t2
+; RV32I-NEXT:    lbu t3, 2(a1)
+; RV32I-NEXT:    lbu a1, 3(a1)
+; RV32I-NEXT:    slli t1, t1, 8
+; RV32I-NEXT:    or t0, t1, t0
+; RV32I-NEXT:    li s9, 64
+; RV32I-NEXT:    slli a1, a1, 8
+; RV32I-NEXT:    or a1, a1, t3
+; RV32I-NEXT:    li t4, 32
+; RV32I-NEXT:    slli a5, a5, 16
+; RV32I-NEXT:    slli t2, t2, 16
+; RV32I-NEXT:    slli a1, a1, 16
+; RV32I-NEXT:    or t3, a5, a4
+; RV32I-NEXT:    or a5, t2, a7
+; RV32I-NEXT:    or a4, a1, t0
+; RV32I-NEXT:    slli a4, a4, 3
+; RV32I-NEXT:    neg s10, a4
+; RV32I-NEXT:    srl t5, t3, s10
+; RV32I-NEXT:    sll s5, a5, a4
+; RV32I-NEXT:    bltu a4, t4, .LBB15_2
+; RV32I-NEXT:  # %bb.1:
+; RV32I-NEXT:    li s8, 0
+; RV32I-NEXT:    sll a7, t3, a4
+; RV32I-NEXT:    j .LBB15_3
+; RV32I-NEXT:  .LBB15_2:
+; RV32I-NEXT:    sll s8, t3, a4
+; RV32I-NEXT:    or a7, t5, s5
+; RV32I-NEXT:  .LBB15_3:
+; RV32I-NEXT:    lbu t2, 9(a0)
+; RV32I-NEXT:    lbu a1, 10(a0)
+; RV32I-NEXT:    lbu t1, 13(a0)
+; RV32I-NEXT:    lbu t0, 14(a0)
+; RV32I-NEXT:    slli a6, a6, 8
+; RV32I-NEXT:    slli t6, a3, 8
+; RV32I-NEXT:    sub s6, s9, a4
+; RV32I-NEXT:    mv a3, a5
+; RV32I-NEXT:    beqz a4, .LBB15_5
+; RV32I-NEXT:  # %bb.4:
+; RV32I-NEXT:    mv a3, a7
+; RV32I-NEXT:  .LBB15_5:
+; RV32I-NEXT:    slli a7, t2, 8
+; RV32I-NEXT:    or a6, a6, a1
+; RV32I-NEXT:    lbu t2, 8(a0)
+; RV32I-NEXT:    lbu a1, 12(a0)
+; RV32I-NEXT:    slli t1, t1, 8
+; RV32I-NEXT:    or t0, t6, t0
+; RV32I-NEXT:    neg t6, s6
+; RV32I-NEXT:    sw t6, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    bltu s6, t4, .LBB15_7
+; RV32I-NEXT:  # %bb.6:
+; RV32I-NEXT:    srl t6, a5, s6
+; RV32I-NEXT:    j .LBB15_8
+; RV32I-NEXT:  .LBB15_7:
+; RV32I-NEXT:    sll t6, a5, t6
+; RV32I-NEXT:    or t6, t5, t6
+; RV32I-NEXT:  .LBB15_8:
+; RV32I-NEXT:    or a7, a7, t2
+; RV32I-NEXT:    slli t2, a6, 16
+; RV32I-NEXT:    or a1, t1, a1
+; RV32I-NEXT:    slli t0, t0, 16
+; RV32I-NEXT:    mv a6, t3
+; RV32I-NEXT:    beqz s6, .LBB15_10
+; RV32I-NEXT:  # %bb.9:
+; RV32I-NEXT:    mv a6, t6
+; RV32I-NEXT:  .LBB15_10:
+; RV32I-NEXT:    or t1, t2, a7
+; RV32I-NEXT:    or t2, t0, a1
+; RV32I-NEXT:    bltu s6, t4, .LBB15_12
+; RV32I-NEXT:  # %bb.11:
+; RV32I-NEXT:    li a7, 0
+; RV32I-NEXT:    j .LBB15_13
+; RV32I-NEXT:  .LBB15_12:
+; RV32I-NEXT:    srl a7, a5, s10
+; RV32I-NEXT:  .LBB15_13:
+; RV32I-NEXT:    srl s0, t1, s10
+; RV32I-NEXT:    sll a1, t2, a4
+; RV32I-NEXT:    sw a1, 20(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    bltu a4, t4, .LBB15_15
+; RV32I-NEXT:  # %bb.14:
+; RV32I-NEXT:    li s1, 0
+; RV32I-NEXT:    sll a1, t1, a4
+; RV32I-NEXT:    j .LBB15_16
+; RV32I-NEXT:  .LBB15_15:
+; RV32I-NEXT:    sll s1, t1, a4
+; RV32I-NEXT:    or a1, s0, a1
+; RV32I-NEXT:  .LBB15_16:
+; RV32I-NEXT:    addi s7, a4, -64
+; RV32I-NEXT:    mv s3, t2
+; RV32I-NEXT:    beqz a4, .LBB15_18
+; RV32I-NEXT:  # %bb.17:
+; RV32I-NEXT:    mv s3, a1
+; RV32I-NEXT:  .LBB15_18:
+; RV32I-NEXT:    neg a1, s7
+; RV32I-NEXT:    sw a1, 16(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    bltu s7, t4, .LBB15_20
+; RV32I-NEXT:  # %bb.19:
+; RV32I-NEXT:    li s2, 0
+; RV32I-NEXT:    sll a1, t3, s7
+; RV32I-NEXT:    mv s4, a5
+; RV32I-NEXT:    bnez s7, .LBB15_21
+; RV32I-NEXT:    j .LBB15_22
+; RV32I-NEXT:  .LBB15_20:
+; RV32I-NEXT:    sll s2, t3, a4
+; RV32I-NEXT:    srl a1, t3, a1
+; RV32I-NEXT:    or a1, a1, s5
+; RV32I-NEXT:    mv s4, a5
+; RV32I-NEXT:    beqz s7, .LBB15_22
+; RV32I-NEXT:  .LBB15_21:
+; RV32I-NEXT:    mv s4, a1
+; RV32I-NEXT:  .LBB15_22:
+; RV32I-NEXT:    li a1, 128
+; RV32I-NEXT:    bltu a4, s9, .LBB15_24
+; RV32I-NEXT:  # %bb.23:
+; RV32I-NEXT:    sw zero, 28(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    li a3, 0
+; RV32I-NEXT:    j .LBB15_25
+; RV32I-NEXT:  .LBB15_24:
+; RV32I-NEXT:    sw s8, 28(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    or s2, a6, s1
+; RV32I-NEXT:    or s4, a7, s3
+; RV32I-NEXT:  .LBB15_25:
+; RV32I-NEXT:    sub ra, a1, a4
+; RV32I-NEXT:    mv a7, t1
+; RV32I-NEXT:    mv a6, t2
+; RV32I-NEXT:    beqz a4, .LBB15_27
+; RV32I-NEXT:  # %bb.26:
+; RV32I-NEXT:    mv a7, s2
+; RV32I-NEXT:    mv a6, s4
+; RV32I-NEXT:  .LBB15_27:
+; RV32I-NEXT:    neg s1, ra
+; RV32I-NEXT:    sll s2, t2, s1
+; RV32I-NEXT:    bltu ra, t4, .LBB15_29
+; RV32I-NEXT:  # %bb.28:
+; RV32I-NEXT:    srl a1, t2, ra
+; RV32I-NEXT:    sw t1, 40(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    bnez ra, .LBB15_30
+; RV32I-NEXT:    j .LBB15_31
+; RV32I-NEXT:  .LBB15_29:
+; RV32I-NEXT:    or a1, s0, s2
+; RV32I-NEXT:    sw t1, 40(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    beqz ra, .LBB15_31
+; RV32I-NEXT:  .LBB15_30:
+; RV32I-NEXT:    sw a1, 40(sp) # 4-byte Folded Spill
+; RV32I-NEXT:  .LBB15_31:
+; RV32I-NEXT:    bltu ra, t4, .LBB15_33
+; RV32I-NEXT:  # %bb.32:
+; RV32I-NEXT:    sw zero, 36(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    srl a1, a5, ra
+; RV32I-NEXT:    mv t5, t3
+; RV32I-NEXT:    bnez ra, .LBB15_34
+; RV32I-NEXT:    j .LBB15_35
+; RV32I-NEXT:  .LBB15_33:
+; RV32I-NEXT:    srl a1, t2, s10
+; RV32I-NEXT:    sw a1, 36(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sll a1, a5, s1
+; RV32I-NEXT:    or a1, t5, a1
+; RV32I-NEXT:    mv t5, t3
+; RV32I-NEXT:    beqz ra, .LBB15_35
+; RV32I-NEXT:  .LBB15_34:
+; RV32I-NEXT:    mv t5, a1
+; RV32I-NEXT:  .LBB15_35:
+; RV32I-NEXT:    sub s3, s9, ra
+; RV32I-NEXT:    bltu ra, t4, .LBB15_38
+; RV32I-NEXT:  # %bb.36:
+; RV32I-NEXT:    sw zero, 32(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    bgeu s3, t4, .LBB15_39
+; RV32I-NEXT:  .LBB15_37:
+; RV32I-NEXT:    sll s1, t1, s1
+; RV32I-NEXT:    neg a1, s3
+; RV32I-NEXT:    srl a1, t1, a1
+; RV32I-NEXT:    or a1, a1, s2
+; RV32I-NEXT:    j .LBB15_40
+; RV32I-NEXT:  .LBB15_38:
+; RV32I-NEXT:    srl a1, a5, s10
+; RV32I-NEXT:    sw a1, 32(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    bltu s3, t4, .LBB15_37
+; RV32I-NEXT:  .LBB15_39:
+; RV32I-NEXT:    li s1, 0
+; RV32I-NEXT:    sll a1, t1, s3
+; RV32I-NEXT:  .LBB15_40:
+; RV32I-NEXT:    addi s4, ra, -64
+; RV32I-NEXT:    mv s2, t2
+; RV32I-NEXT:    beqz s3, .LBB15_42
+; RV32I-NEXT:  # %bb.41:
+; RV32I-NEXT:    mv s2, a1
+; RV32I-NEXT:  .LBB15_42:
+; RV32I-NEXT:    sw s5, 24(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    mv s5, a7
+; RV32I-NEXT:    bltu s4, t4, .LBB15_44
+; RV32I-NEXT:  # %bb.43:
+; RV32I-NEXT:    srl t0, t2, s4
+; RV32I-NEXT:    j .LBB15_45
+; RV32I-NEXT:  .LBB15_44:
+; RV32I-NEXT:    srl a1, t1, ra
+; RV32I-NEXT:    neg t0, s4
+; RV32I-NEXT:    sll t0, t2, t0
+; RV32I-NEXT:    or t0, a1, t0
+; RV32I-NEXT:  .LBB15_45:
+; RV32I-NEXT:    mv s0, s10
+; RV32I-NEXT:    mv a7, a6
+; RV32I-NEXT:    lbu s8, 19(a0)
+; RV32I-NEXT:    lbu a1, 23(a0)
+; RV32I-NEXT:    mv s3, t1
+; RV32I-NEXT:    beqz s4, .LBB15_47
+; RV32I-NEXT:  # %bb.46:
+; RV32I-NEXT:    mv s3, t0
+; RV32I-NEXT:  .LBB15_47:
+; RV32I-NEXT:    mv a6, a3
+; RV32I-NEXT:    lbu s10, 17(a0)
+; RV32I-NEXT:    lbu t0, 18(a0)
+; RV32I-NEXT:    lbu s9, 21(a0)
+; RV32I-NEXT:    lbu t6, 22(a0)
+; RV32I-NEXT:    slli s8, s8, 8
+; RV32I-NEXT:    slli a1, a1, 8
+; RV32I-NEXT:    li a3, 64
+; RV32I-NEXT:    bltu s4, t4, .LBB15_49
+; RV32I-NEXT:  # %bb.48:
+; RV32I-NEXT:    li s4, 0
+; RV32I-NEXT:    j .LBB15_50
+; RV32I-NEXT:  .LBB15_49:
+; RV32I-NEXT:    srl s4, t2, ra
+; RV32I-NEXT:  .LBB15_50:
+; RV32I-NEXT:    or s11, s8, t0
+; RV32I-NEXT:    lbu t0, 16(a0)
+; RV32I-NEXT:    lbu s8, 20(a0)
+; RV32I-NEXT:    slli s10, s10, 8
+; RV32I-NEXT:    slli s9, s9, 8
+; RV32I-NEXT:    or t6, a1, t6
+; RV32I-NEXT:    bgeu ra, a3, .LBB15_52
+; RV32I-NEXT:  # %bb.51:
+; RV32I-NEXT:    or s3, t5, s1
+; RV32I-NEXT:    lw a1, 32(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    or s4, a1, s2
+; RV32I-NEXT:  .LBB15_52:
+; RV32I-NEXT:    or a1, s10, t0
+; RV32I-NEXT:    slli s11, s11, 16
+; RV32I-NEXT:    or t0, s9, s8
+; RV32I-NEXT:    slli t6, t6, 16
+; RV32I-NEXT:    mv t5, t3
+; RV32I-NEXT:    mv s1, a5
+; RV32I-NEXT:    mv a3, a6
+; RV32I-NEXT:    beqz ra, .LBB15_54
+; RV32I-NEXT:  # %bb.53:
+; RV32I-NEXT:    mv t5, s3
+; RV32I-NEXT:    mv s1, s4
+; RV32I-NEXT:  .LBB15_54:
+; RV32I-NEXT:    sw s1, 4(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    or s2, s11, a1
+; RV32I-NEXT:    or s1, t6, t0
+; RV32I-NEXT:    li a1, 64
+; RV32I-NEXT:    mv a6, a7
+; RV32I-NEXT:    mv a7, s0
+; RV32I-NEXT:    bltu ra, a1, .LBB15_56
+; RV32I-NEXT:  # %bb.55:
+; RV32I-NEXT:    sw zero, 40(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw zero, 36(sp) # 4-byte Folded Spill
+; RV32I-NEXT:  .LBB15_56:
+; RV32I-NEXT:    srl s3, s2, a7
+; RV32I-NEXT:    sll ra, s1, a4
+; RV32I-NEXT:    mv a7, s5
+; RV32I-NEXT:    sw t5, 8(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    bltu a4, t4, .LBB15_58
+; RV32I-NEXT:  # %bb.57:
+; RV32I-NEXT:    sw zero, 32(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sll a1, s2, a4
+; RV32I-NEXT:    j .LBB15_59
+; RV32I-NEXT:  .LBB15_58:
+; RV32I-NEXT:    sll a1, s2, a4
+; RV32I-NEXT:    sw a1, 32(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    or a1, s3, ra
+; RV32I-NEXT:  .LBB15_59:
+; RV32I-NEXT:    lbu s9, 27(a0)
+; RV32I-NEXT:    lbu t6, 31(a0)
+; RV32I-NEXT:    mv t5, s1
+; RV32I-NEXT:    beqz a4, .LBB15_61
+; RV32I-NEXT:  # %bb.60:
+; RV32I-NEXT:    mv t5, a1
+; RV32I-NEXT:  .LBB15_61:
+; RV32I-NEXT:    lbu s8, 25(a0)
+; RV32I-NEXT:    lbu s4, 26(a0)
+; RV32I-NEXT:    lbu s11, 29(a0)
+; RV32I-NEXT:    lbu s10, 30(a0)
+; RV32I-NEXT:    slli s9, s9, 8
+; RV32I-NEXT:    slli t6, t6, 8
+; RV32I-NEXT:    bltu s6, t4, .LBB15_63
+; RV32I-NEXT:  # %bb.62:
+; RV32I-NEXT:    srl t0, s1, s6
+; RV32I-NEXT:    j .LBB15_64
+; RV32I-NEXT:  .LBB15_63:
+; RV32I-NEXT:    lw a1, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    sll a1, s1, a1
+; RV32I-NEXT:    or t0, s3, a1
+; RV32I-NEXT:  .LBB15_64:
+; RV32I-NEXT:    slli s8, s8, 8
+; RV32I-NEXT:    lbu s3, 24(a0)
+; RV32I-NEXT:    lbu a1, 28(a0)
+; RV32I-NEXT:    or s4, s9, s4
+; RV32I-NEXT:    slli s11, s11, 8
+; RV32I-NEXT:    or t6, t6, s10
+; RV32I-NEXT:    mv s9, s2
+; RV32I-NEXT:    beqz s6, .LBB15_66
+; RV32I-NEXT:  # %bb.65:
+; RV32I-NEXT:    mv s9, t0
+; RV32I-NEXT:  .LBB15_66:
+; RV32I-NEXT:    or a0, s8, s3
+; RV32I-NEXT:    slli t0, s4, 16
+; RV32I-NEXT:    or a1, s11, a1
+; RV32I-NEXT:    slli t6, t6, 16
+; RV32I-NEXT:    bltu s6, t4, .LBB15_68
+; RV32I-NEXT:  # %bb.67:
+; RV32I-NEXT:    li s4, 0
+; RV32I-NEXT:    j .LBB15_69
+; RV32I-NEXT:  .LBB15_68:
+; RV32I-NEXT:    srl s4, s1, s0
+; RV32I-NEXT:  .LBB15_69:
+; RV32I-NEXT:    li s11, 64
+; RV32I-NEXT:    or s6, t0, a0
+; RV32I-NEXT:    or a0, t6, a1
+; RV32I-NEXT:    bltu a4, t4, .LBB15_71
+; RV32I-NEXT:  # %bb.70:
+; RV32I-NEXT:    li s3, 0
+; RV32I-NEXT:    sll a1, s6, a4
+; RV32I-NEXT:    mv s10, a0
+; RV32I-NEXT:    bnez a4, .LBB15_72
+; RV32I-NEXT:    j .LBB15_73
+; RV32I-NEXT:  .LBB15_71:
+; RV32I-NEXT:    sll s3, s6, a4
+; RV32I-NEXT:    srl a1, s6, s0
+; RV32I-NEXT:    sll t0, a0, a4
+; RV32I-NEXT:    or a1, a1, t0
+; RV32I-NEXT:    mv s10, a0
+; RV32I-NEXT:    beqz a4, .LBB15_73
+; RV32I-NEXT:  .LBB15_72:
+; RV32I-NEXT:    mv s10, a1
+; RV32I-NEXT:  .LBB15_73:
+; RV32I-NEXT:    bltu s7, t4, .LBB15_75
+; RV32I-NEXT:  # %bb.74:
+; RV32I-NEXT:    li s5, 0
+; RV32I-NEXT:    sll a1, s2, s7
+; RV32I-NEXT:    mv s0, s1
+; RV32I-NEXT:    bnez s7, .LBB15_76
+; RV32I-NEXT:    j .LBB15_77
+; RV32I-NEXT:  .LBB15_75:
+; RV32I-NEXT:    sll s5, s2, a4
+; RV32I-NEXT:    lw a1, 16(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    srl a1, s2, a1
+; RV32I-NEXT:    or a1, a1, ra
+; RV32I-NEXT:    mv s0, s1
+; RV32I-NEXT:    beqz s7, .LBB15_77
+; RV32I-NEXT:  .LBB15_76:
+; RV32I-NEXT:    mv s0, a1
+; RV32I-NEXT:  .LBB15_77:
+; RV32I-NEXT:    bltu a4, s11, .LBB15_79
+; RV32I-NEXT:  # %bb.78:
+; RV32I-NEXT:    sw zero, 32(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    li t5, 0
+; RV32I-NEXT:    j .LBB15_80
+; RV32I-NEXT:  .LBB15_79:
+; RV32I-NEXT:    or s5, s9, s3
+; RV32I-NEXT:    or s0, s4, s10
+; RV32I-NEXT:  .LBB15_80:
+; RV32I-NEXT:    addi s9, a4, -128
+; RV32I-NEXT:    mv s7, s6
+; RV32I-NEXT:    mv s8, a0
+; RV32I-NEXT:    beqz a4, .LBB15_82
+; RV32I-NEXT:  # %bb.81:
+; RV32I-NEXT:    mv s7, s5
+; RV32I-NEXT:    mv s8, s0
+; RV32I-NEXT:  .LBB15_82:
+; RV32I-NEXT:    neg s3, s9
+; RV32I-NEXT:    srl s0, t3, s3
+; RV32I-NEXT:    bltu s9, t4, .LBB15_84
+; RV32I-NEXT:  # %bb.83:
+; RV32I-NEXT:    li s5, 0
+; RV32I-NEXT:    sll a1, t3, s9
+; RV32I-NEXT:    j .LBB15_85
+; RV32I-NEXT:  .LBB15_84:
+; RV32I-NEXT:    sll s5, t3, a4
+; RV32I-NEXT:    lw a1, 24(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    or a1, s0, a1
+; RV32I-NEXT:  .LBB15_85:
+; RV32I-NEXT:    sub s4, s11, s9
+; RV32I-NEXT:    mv t6, a5
+; RV32I-NEXT:    beqz s9, .LBB15_87
+; RV32I-NEXT:  # %bb.86:
+; RV32I-NEXT:    mv t6, a1
+; RV32I-NEXT:  .LBB15_87:
+; RV32I-NEXT:    bltu s4, t4, .LBB15_89
+; RV32I-NEXT:  # %bb.88:
+; RV32I-NEXT:    srl a1, a5, s4
+; RV32I-NEXT:    mv s0, t3
+; RV32I-NEXT:    bnez s4, .LBB15_90
+; RV32I-NEXT:    j .LBB15_91
+; RV32I-NEXT:  .LBB15_89:
+; RV32I-NEXT:    neg a1, s4
+; RV32I-NEXT:    sll a1, a5, a1
+; RV32I-NEXT:    or a1, s0, a1
+; RV32I-NEXT:    mv s0, t3
+; RV32I-NEXT:    beqz s4, .LBB15_91
+; RV32I-NEXT:  .LBB15_90:
+; RV32I-NEXT:    mv s0, a1
+; RV32I-NEXT:  .LBB15_91:
+; RV32I-NEXT:    bltu s4, t4, .LBB15_94
+; RV32I-NEXT:  # %bb.92:
+; RV32I-NEXT:    li s4, 0
+; RV32I-NEXT:    li ra, 64
+; RV32I-NEXT:    bgeu s9, t4, .LBB15_95
+; RV32I-NEXT:  .LBB15_93:
+; RV32I-NEXT:    sll s10, t1, a4
+; RV32I-NEXT:    srl a1, t1, s3
+; RV32I-NEXT:    lw t0, 20(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    or a1, a1, t0
+; RV32I-NEXT:    j .LBB15_96
+; RV32I-NEXT:  .LBB15_94:
+; RV32I-NEXT:    srl s4, a5, s3
+; RV32I-NEXT:    li ra, 64
+; RV32I-NEXT:    bltu s9, t4, .LBB15_93
+; RV32I-NEXT:  .LBB15_95:
+; RV32I-NEXT:    li s10, 0
+; RV32I-NEXT:    sll a1, t1, s9
+; RV32I-NEXT:  .LBB15_96:
+; RV32I-NEXT:    addi s11, s9, -64
+; RV32I-NEXT:    mv s3, t2
+; RV32I-NEXT:    beqz s9, .LBB15_98
+; RV32I-NEXT:  # %bb.97:
+; RV32I-NEXT:    mv s3, a1
+; RV32I-NEXT:  .LBB15_98:
+; RV32I-NEXT:    bltu s11, t4, .LBB15_100
+; RV32I-NEXT:  # %bb.99:
+; RV32I-NEXT:    li t4, 0
+; RV32I-NEXT:    sll a1, t3, s11
+; RV32I-NEXT:    bnez s11, .LBB15_101
+; RV32I-NEXT:    j .LBB15_102
+; RV32I-NEXT:  .LBB15_100:
+; RV32I-NEXT:    sll t4, t3, s9
+; RV32I-NEXT:    neg a1, s11
+; RV32I-NEXT:    srl a1, t3, a1
+; RV32I-NEXT:    sll t0, a5, s9
+; RV32I-NEXT:    or a1, a1, t0
+; RV32I-NEXT:    beqz s11, .LBB15_102
+; RV32I-NEXT:  .LBB15_101:
+; RV32I-NEXT:    mv a5, a1
+; RV32I-NEXT:  .LBB15_102:
+; RV32I-NEXT:    bltu s9, ra, .LBB15_104
+; RV32I-NEXT:  # %bb.103:
+; RV32I-NEXT:    li s5, 0
+; RV32I-NEXT:    li t6, 0
+; RV32I-NEXT:    li a1, 128
+; RV32I-NEXT:    bnez s9, .LBB15_105
+; RV32I-NEXT:    j .LBB15_106
+; RV32I-NEXT:  .LBB15_104:
+; RV32I-NEXT:    or t4, s0, s10
+; RV32I-NEXT:    or a5, s4, s3
+; RV32I-NEXT:    li a1, 128
+; RV32I-NEXT:    beqz s9, .LBB15_106
+; RV32I-NEXT:  .LBB15_105:
+; RV32I-NEXT:    mv t1, t4
+; RV32I-NEXT:    mv t2, a5
+; RV32I-NEXT:  .LBB15_106:
+; RV32I-NEXT:    bltu a4, a1, .LBB15_108
+; RV32I-NEXT:  # %bb.107:
+; RV32I-NEXT:    li ra, 0
+; RV32I-NEXT:    li a3, 0
+; RV32I-NEXT:    li a7, 0
+; RV32I-NEXT:    li a6, 0
+; RV32I-NEXT:    bnez a4, .LBB15_109
+; RV32I-NEXT:    j .LBB15_110
+; RV32I-NEXT:  .LBB15_108:
+; RV32I-NEXT:    lw a1, 8(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw a5, 32(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    or s5, a1, a5
+; RV32I-NEXT:    lw a1, 4(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    or t6, a1, t5
+; RV32I-NEXT:    lw a1, 40(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    or t1, a1, s7
+; RV32I-NEXT:    lw a1, 36(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    or t2, a1, s8
+; RV32I-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    beqz a4, .LBB15_110
+; RV32I-NEXT:  .LBB15_109:
+; RV32I-NEXT:    mv s2, s5
+; RV32I-NEXT:    mv s1, t6
+; RV32I-NEXT:    mv s6, t1
+; RV32I-NEXT:    mv a0, t2
+; RV32I-NEXT:  .LBB15_110:
+; RV32I-NEXT:    srli a4, ra, 16
+; RV32I-NEXT:    lui t2, 16
+; RV32I-NEXT:    srli t1, ra, 24
+; RV32I-NEXT:    srli a5, a3, 16
+; RV32I-NEXT:    srli t4, a3, 24
+; RV32I-NEXT:    srli t0, a7, 16
+; RV32I-NEXT:    srli s0, a7, 24
+; RV32I-NEXT:    srli t3, a6, 16
+; RV32I-NEXT:    srli s3, a6, 24
+; RV32I-NEXT:    srli t6, s2, 16
+; RV32I-NEXT:    srli a1, s2, 24
+; RV32I-NEXT:    srli t5, s1, 16
+; RV32I-NEXT:    srli s5, s1, 24
+; RV32I-NEXT:    srli s4, s6, 16
+; RV32I-NEXT:    srli s7, s6, 24
+; RV32I-NEXT:    srli s8, a0, 16
+; RV32I-NEXT:    srli s9, a0, 24
+; RV32I-NEXT:    addi t2, t2, -1
+; RV32I-NEXT:    and s10, ra, t2
+; RV32I-NEXT:    and s11, a3, t2
+; RV32I-NEXT:    srli s10, s10, 8
+; RV32I-NEXT:    sb ra, 0(a2)
+; RV32I-NEXT:    sb s10, 1(a2)
+; RV32I-NEXT:    sb a4, 2(a2)
+; RV32I-NEXT:    sb t1, 3(a2)
+; RV32I-NEXT:    and a4, a7, t2
+; RV32I-NEXT:    srli t1, s11, 8
+; RV32I-NEXT:    sb a3, 4(a2)
+; RV32I-NEXT:    sb t1, 5(a2)
+; RV32I-NEXT:    sb a5, 6(a2)
+; RV32I-NEXT:    sb t4, 7(a2)
+; RV32I-NEXT:    and a3, a6, t2
+; RV32I-NEXT:    srli a4, a4, 8
+; RV32I-NEXT:    sb a7, 8(a2)
+; RV32I-NEXT:    sb a4, 9(a2)
+; RV32I-NEXT:    sb t0, 10(a2)
+; RV32I-NEXT:    sb s0, 11(a2)
+; RV32I-NEXT:    and a4, s2, t2
+; RV32I-NEXT:    srli a3, a3, 8
+; RV32I-NEXT:    sb a6, 12(a2)
+; RV32I-NEXT:    sb a3, 13(a2)
+; RV32I-NEXT:    sb t3, 14(a2)
+; RV32I-NEXT:    sb s3, 15(a2)
+; RV32I-NEXT:    and a3, s1, t2
+; RV32I-NEXT:    srli a4, a4, 8
+; RV32I-NEXT:    sb s2, 16(a2)
+; RV32I-NEXT:    sb a4, 17(a2)
+; RV32I-NEXT:    sb t6, 18(a2)
+; RV32I-NEXT:    sb a1, 19(a2)
+; RV32I-NEXT:    and a1, s6, t2
+; RV32I-NEXT:    and a4, a0, t2
+; RV32I-NEXT:    srli a3, a3, 8
+; RV32I-NEXT:    srli a1, a1, 8
+; RV32I-NEXT:    srli a4, a4, 8
+; RV32I-NEXT:    sb s1, 20(a2)
+; RV32I-NEXT:    sb a3, 21(a2)
+; RV32I-NEXT:    sb t5, 22(a2)
+; RV32I-NEXT:    sb s5, 23(a2)
+; RV32I-NEXT:    sb s6, 24(a2)
+; RV32I-NEXT:    sb a1, 25(a2)
+; RV32I-NEXT:    sb s4, 26(a2)
+; RV32I-NEXT:    sb s7, 27(a2)
+; RV32I-NEXT:    sb a0, 28(a2)
+; RV32I-NEXT:    sb a4, 29(a2)
+; RV32I-NEXT:    sb s8, 30(a2)
+; RV32I-NEXT:    sb s9, 31(a2)
+; RV32I-NEXT:    lw ra, 92(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s0, 88(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s1, 84(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s2, 80(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s3, 76(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s4, 72(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s5, 68(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s6, 64(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s7, 60(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s8, 56(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s9, 52(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s10, 48(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s11, 44(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    addi sp, sp, 96
+; RV32I-NEXT:    ret
+  %src = load i256, ptr %src.ptr, align 1
+  %byteOff = load i256, ptr %byteOff.ptr, align 1
+  %bitOff = shl i256 %byteOff, 3
+  %res = shl i256 %src, %bitOff
+  store i256 %res, ptr %dst, align 1
+  ret void
+}
+
+define void @shl_32bytes_wordOff(ptr %src.ptr, ptr %wordOff.ptr, ptr %dst) nounwind {
+; RV64I-LABEL: shl_32bytes_wordOff:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -112
+; RV64I-NEXT:    sd ra, 104(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s0, 96(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s1, 88(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s2, 80(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s3, 72(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s4, 64(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s5, 56(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s6, 48(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s7, 40(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s8, 32(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s9, 24(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s10, 16(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s11, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    lbu a3, 0(a0)
+; RV64I-NEXT:    lbu a4, 1(a0)
+; RV64I-NEXT:    lbu a5, 2(a0)
+; RV64I-NEXT:    lbu a6, 3(a0)
+; RV64I-NEXT:    lbu a7, 4(a0)
+; RV64I-NEXT:    lbu t0, 5(a0)
+; RV64I-NEXT:    lbu t1, 6(a0)
+; RV64I-NEXT:    lbu t2, 7(a0)
+; RV64I-NEXT:    lbu t3, 8(a0)
+; RV64I-NEXT:    lbu t4, 9(a0)
+; RV64I-NEXT:    lbu t5, 10(a0)
+; RV64I-NEXT:    lbu t6, 11(a0)
+; RV64I-NEXT:    slli a4, a4, 8
+; RV64I-NEXT:    or a3, a4, a3
+; RV64I-NEXT:    lbu a4, 12(a0)
+; RV64I-NEXT:    lbu s0, 13(a0)
+; RV64I-NEXT:    lbu s1, 14(a0)
+; RV64I-NEXT:    lbu s2, 15(a0)
+; RV64I-NEXT:    slli a6, a6, 8
+; RV64I-NEXT:    slli t0, t0, 8
+; RV64I-NEXT:    slli t2, t2, 8
+; RV64I-NEXT:    slli t4, t4, 8
+; RV64I-NEXT:    or s3, a6, a5
+; RV64I-NEXT:    or a5, t0, a7
+; RV64I-NEXT:    or a7, t2, t1
+; RV64I-NEXT:    or a6, t4, t3
+; RV64I-NEXT:    lbu t0, 0(a1)
+; RV64I-NEXT:    lbu t1, 1(a1)
+; RV64I-NEXT:    lbu t2, 2(a1)
+; RV64I-NEXT:    lbu t3, 3(a1)
+; RV64I-NEXT:    slli t6, t6, 8
+; RV64I-NEXT:    slli s0, s0, 8
+; RV64I-NEXT:    slli s2, s2, 8
+; RV64I-NEXT:    or t6, t6, t5
+; RV64I-NEXT:    or s0, s0, a4
+; RV64I-NEXT:    or s1, s2, s1
+; RV64I-NEXT:    lbu a4, 4(a1)
+; RV64I-NEXT:    lbu t4, 5(a1)
+; RV64I-NEXT:    lbu t5, 6(a1)
+; RV64I-NEXT:    lbu a1, 7(a1)
+; RV64I-NEXT:    slli t1, t1, 8
+; RV64I-NEXT:    slli t3, t3, 8
+; RV64I-NEXT:    slli t4, t4, 8
+; RV64I-NEXT:    slli a1, a1, 8
+; RV64I-NEXT:    or t0, t1, t0
+; RV64I-NEXT:    or t1, t3, t2
+; RV64I-NEXT:    or t2, t4, a4
+; RV64I-NEXT:    or a1, a1, t5
+; RV64I-NEXT:    lbu t5, 19(a0)
+; RV64I-NEXT:    lbu t4, 21(a0)
+; RV64I-NEXT:    lbu a4, 22(a0)
+; RV64I-NEXT:    lbu t3, 23(a0)
+; RV64I-NEXT:    slli s3, s3, 16
+; RV64I-NEXT:    slli a7, a7, 16
+; RV64I-NEXT:    slli t6, t6, 16
+; RV64I-NEXT:    slli s1, s1, 16
+; RV64I-NEXT:    or s4, s3, a3
+; RV64I-NEXT:    or a5, a7, a5
+; RV64I-NEXT:    or a6, t6, a6
+; RV64I-NEXT:    or s0, s1, s0
+; RV64I-NEXT:    lbu s1, 27(a0)
+; RV64I-NEXT:    lbu t6, 29(a0)
+; RV64I-NEXT:    lbu a3, 30(a0)
+; RV64I-NEXT:    lbu s2, 31(a0)
+; RV64I-NEXT:    slli t1, t1, 16
+; RV64I-NEXT:    or s5, t1, t0
+; RV64I-NEXT:    li a7, 128
+; RV64I-NEXT:    slli a1, a1, 16
+; RV64I-NEXT:    or a1, a1, t2
+; RV64I-NEXT:    li t0, 64
+; RV64I-NEXT:    slli s3, t3, 8
+; RV64I-NEXT:    slli s2, s2, 8
+; RV64I-NEXT:    slli a5, a5, 32
+; RV64I-NEXT:    slli s0, s0, 32
+; RV64I-NEXT:    slli a1, a1, 32
+; RV64I-NEXT:    or t1, a5, s4
+; RV64I-NEXT:    or a5, s0, a6
+; RV64I-NEXT:    or a6, a1, s5
+; RV64I-NEXT:    slli a6, a6, 5
+; RV64I-NEXT:    subw t2, a6, t0
+; RV64I-NEXT:    negw t3, a6
+; RV64I-NEXT:    srl s0, t1, t3
+; RV64I-NEXT:    bltu a6, t0, .LBB16_2
+; RV64I-NEXT:  # %bb.1:
+; RV64I-NEXT:    li a1, 0
+; RV64I-NEXT:    sll s4, t1, t2
+; RV64I-NEXT:    j .LBB16_3
+; RV64I-NEXT:  .LBB16_2:
+; RV64I-NEXT:    sll a1, t1, a6
+; RV64I-NEXT:    sll s4, a5, a6
+; RV64I-NEXT:    or s4, s0, s4
+; RV64I-NEXT:  .LBB16_3:
+; RV64I-NEXT:    slli t5, t5, 8
+; RV64I-NEXT:    slli t4, t4, 8
+; RV64I-NEXT:    or s3, s3, a4
+; RV64I-NEXT:    lbu ra, 17(a0)
+; RV64I-NEXT:    lbu s11, 18(a0)
+; RV64I-NEXT:    lbu s8, 20(a0)
+; RV64I-NEXT:    lbu s5, 25(a0)
+; RV64I-NEXT:    slli s1, s1, 8
+; RV64I-NEXT:    lbu s7, 26(a0)
+; RV64I-NEXT:    lbu s6, 28(a0)
+; RV64I-NEXT:    slli s10, t6, 8
+; RV64I-NEXT:    or s9, s2, a3
+; RV64I-NEXT:    sub a4, a7, a6
+; RV64I-NEXT:    mv a3, a5
+; RV64I-NEXT:    beqz a6, .LBB16_5
+; RV64I-NEXT:  # %bb.4:
+; RV64I-NEXT:    mv a3, s4
+; RV64I-NEXT:  .LBB16_5:
+; RV64I-NEXT:    slli t6, ra, 8
+; RV64I-NEXT:    or t5, t5, s11
+; RV64I-NEXT:    or t4, t4, s8
+; RV64I-NEXT:    slli s3, s3, 16
+; RV64I-NEXT:    lbu s8, 16(a0)
+; RV64I-NEXT:    lbu a0, 24(a0)
+; RV64I-NEXT:    slli s5, s5, 8
+; RV64I-NEXT:    or s2, s1, s7
+; RV64I-NEXT:    or s1, s10, s6
+; RV64I-NEXT:    slli s4, s9, 16
+; RV64I-NEXT:    bltu a4, t0, .LBB16_7
+; RV64I-NEXT:  # %bb.6:
+; RV64I-NEXT:    subw s0, a4, t0
+; RV64I-NEXT:    srl s0, a5, s0
+; RV64I-NEXT:    j .LBB16_8
+; RV64I-NEXT:  .LBB16_7:
+; RV64I-NEXT:    negw s6, a4
+; RV64I-NEXT:    sll s6, a5, s6
+; RV64I-NEXT:    or s0, s0, s6
+; RV64I-NEXT:  .LBB16_8:
+; RV64I-NEXT:    or t6, t6, s8
+; RV64I-NEXT:    slli s6, t5, 16
+; RV64I-NEXT:    or s3, s3, t4
+; RV64I-NEXT:    or t5, s5, a0
+; RV64I-NEXT:    slli s2, s2, 16
+; RV64I-NEXT:    or s1, s4, s1
+; RV64I-NEXT:    mv t4, t1
+; RV64I-NEXT:    beqz a4, .LBB16_10
+; RV64I-NEXT:  # %bb.9:
+; RV64I-NEXT:    mv t4, s0
+; RV64I-NEXT:  .LBB16_10:
+; RV64I-NEXT:    or a0, s6, t6
+; RV64I-NEXT:    slli s0, s3, 32
+; RV64I-NEXT:    or t6, s2, t5
+; RV64I-NEXT:    slli s1, s1, 32
+; RV64I-NEXT:    bltu a4, t0, .LBB16_12
+; RV64I-NEXT:  # %bb.11:
+; RV64I-NEXT:    li t5, 0
+; RV64I-NEXT:    j .LBB16_13
+; RV64I-NEXT:  .LBB16_12:
+; RV64I-NEXT:    srl t5, a5, t3
+; RV64I-NEXT:  .LBB16_13:
+; RV64I-NEXT:    or a4, s0, a0
+; RV64I-NEXT:    or a0, s1, t6
+; RV64I-NEXT:    bltu a6, t0, .LBB16_15
+; RV64I-NEXT:  # %bb.14:
+; RV64I-NEXT:    li t6, 0
+; RV64I-NEXT:    sll t2, a4, t2
+; RV64I-NEXT:    j .LBB16_16
+; RV64I-NEXT:  .LBB16_15:
+; RV64I-NEXT:    sll t6, a4, a6
+; RV64I-NEXT:    srl t2, a4, t3
+; RV64I-NEXT:    sll t3, a0, a6
+; RV64I-NEXT:    or t2, t2, t3
+; RV64I-NEXT:  .LBB16_16:
+; RV64I-NEXT:    sub s0, a6, a7
+; RV64I-NEXT:    mv t3, a0
+; RV64I-NEXT:    beqz a6, .LBB16_18
+; RV64I-NEXT:  # %bb.17:
+; RV64I-NEXT:    mv t3, t2
+; RV64I-NEXT:  .LBB16_18:
+; RV64I-NEXT:    bltu s0, t0, .LBB16_20
+; RV64I-NEXT:  # %bb.19:
+; RV64I-NEXT:    li t2, 0
+; RV64I-NEXT:    subw t0, s0, t0
+; RV64I-NEXT:    sll t0, t1, t0
+; RV64I-NEXT:    bnez s0, .LBB16_21
+; RV64I-NEXT:    j .LBB16_22
+; RV64I-NEXT:  .LBB16_20:
+; RV64I-NEXT:    sll t2, t1, s0
+; RV64I-NEXT:    negw t0, s0
+; RV64I-NEXT:    srl t0, t1, t0
+; RV64I-NEXT:    sll t1, a5, s0
+; RV64I-NEXT:    or t0, t0, t1
+; RV64I-NEXT:    beqz s0, .LBB16_22
+; RV64I-NEXT:  .LBB16_21:
+; RV64I-NEXT:    mv a5, t0
+; RV64I-NEXT:  .LBB16_22:
+; RV64I-NEXT:    bltu a6, a7, .LBB16_24
+; RV64I-NEXT:  # %bb.23:
+; RV64I-NEXT:    li a1, 0
+; RV64I-NEXT:    li a3, 0
+; RV64I-NEXT:    bnez a6, .LBB16_25
+; RV64I-NEXT:    j .LBB16_26
+; RV64I-NEXT:  .LBB16_24:
+; RV64I-NEXT:    or t2, t4, t6
+; RV64I-NEXT:    or a5, t5, t3
+; RV64I-NEXT:    beqz a6, .LBB16_26
+; RV64I-NEXT:  .LBB16_25:
+; RV64I-NEXT:    mv a4, t2
+; RV64I-NEXT:    mv a0, a5
+; RV64I-NEXT:  .LBB16_26:
+; RV64I-NEXT:    srli a5, a1, 32
+; RV64I-NEXT:    srliw a6, a1, 16
+; RV64I-NEXT:    lui t2, 16
+; RV64I-NEXT:    srliw t1, a1, 24
+; RV64I-NEXT:    srli t0, a1, 48
+; RV64I-NEXT:    srli t5, a1, 56
+; RV64I-NEXT:    srli a7, a3, 32
+; RV64I-NEXT:    srliw t4, a3, 16
+; RV64I-NEXT:    srliw s0, a3, 24
+; RV64I-NEXT:    srli t6, a3, 48
+; RV64I-NEXT:    srli s3, a3, 56
+; RV64I-NEXT:    srli t3, a4, 32
+; RV64I-NEXT:    srliw s2, a4, 16
+; RV64I-NEXT:    srliw s6, a4, 24
+; RV64I-NEXT:    srli s4, a4, 48
+; RV64I-NEXT:    srli s7, a4, 56
+; RV64I-NEXT:    srli s1, a0, 32
+; RV64I-NEXT:    srliw s5, a0, 16
+; RV64I-NEXT:    srliw s8, a0, 24
+; RV64I-NEXT:    srli s9, a0, 48
+; RV64I-NEXT:    srli s10, a0, 56
+; RV64I-NEXT:    addi t2, t2, -1
+; RV64I-NEXT:    and s11, a1, t2
+; RV64I-NEXT:    srli s11, s11, 8
+; RV64I-NEXT:    sb a1, 0(a2)
+; RV64I-NEXT:    sb s11, 1(a2)
+; RV64I-NEXT:    sb a6, 2(a2)
+; RV64I-NEXT:    sb t1, 3(a2)
+; RV64I-NEXT:    and a1, a5, t2
+; RV64I-NEXT:    srli a1, a1, 8
+; RV64I-NEXT:    sb a5, 4(a2)
+; RV64I-NEXT:    sb a1, 5(a2)
+; RV64I-NEXT:    sb t0, 6(a2)
+; RV64I-NEXT:    sb t5, 7(a2)
+; RV64I-NEXT:    and a1, a3, t2
+; RV64I-NEXT:    srli a1, a1, 8
+; RV64I-NEXT:    sb a3, 8(a2)
+; RV64I-NEXT:    sb a1, 9(a2)
+; RV64I-NEXT:    sb t4, 10(a2)
+; RV64I-NEXT:    sb s0, 11(a2)
+; RV64I-NEXT:    and a1, a7, t2
+; RV64I-NEXT:    srli a1, a1, 8
+; RV64I-NEXT:    sb a7, 12(a2)
+; RV64I-NEXT:    sb a1, 13(a2)
+; RV64I-NEXT:    sb t6, 14(a2)
+; RV64I-NEXT:    sb s3, 15(a2)
+; RV64I-NEXT:    and a1, a4, t2
+; RV64I-NEXT:    srli a1, a1, 8
+; RV64I-NEXT:    sb a4, 16(a2)
+; RV64I-NEXT:    sb a1, 17(a2)
+; RV64I-NEXT:    sb s2, 18(a2)
+; RV64I-NEXT:    sb s6, 19(a2)
+; RV64I-NEXT:    and a1, t3, t2
+; RV64I-NEXT:    srli a1, a1, 8
+; RV64I-NEXT:    sb t3, 20(a2)
+; RV64I-NEXT:    sb a1, 21(a2)
+; RV64I-NEXT:    sb s4, 22(a2)
+; RV64I-NEXT:    sb s7, 23(a2)
+; RV64I-NEXT:    and a1, a0, t2
+; RV64I-NEXT:    and a3, s1, t2
+; RV64I-NEXT:    srli a1, a1, 8
+; RV64I-NEXT:    srli a3, a3, 8
+; RV64I-NEXT:    sb a0, 24(a2)
+; RV64I-NEXT:    sb a1, 25(a2)
+; RV64I-NEXT:    sb s5, 26(a2)
+; RV64I-NEXT:    sb s8, 27(a2)
+; RV64I-NEXT:    sb s1, 28(a2)
+; RV64I-NEXT:    sb a3, 29(a2)
+; RV64I-NEXT:    sb s9, 30(a2)
+; RV64I-NEXT:    sb s10, 31(a2)
+; RV64I-NEXT:    ld ra, 104(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s0, 96(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s1, 88(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s2, 80(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s3, 72(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s4, 64(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s5, 56(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s6, 48(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s7, 40(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s8, 32(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s9, 24(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s10, 16(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s11, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    addi sp, sp, 112
+; RV64I-NEXT:    ret
+;
+; RV32I-LABEL: shl_32bytes_wordOff:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    addi sp, sp, -96
+; RV32I-NEXT:    sw ra, 92(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s0, 88(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s1, 84(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s2, 80(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s3, 76(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s4, 72(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s5, 68(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s6, 64(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s7, 60(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s8, 56(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s9, 52(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s10, 48(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s11, 44(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    lbu a3, 0(a0)
+; RV32I-NEXT:    lbu a4, 1(a0)
+; RV32I-NEXT:    lbu a5, 2(a0)
+; RV32I-NEXT:    lbu a7, 3(a0)
+; RV32I-NEXT:    lbu t0, 4(a0)
+; RV32I-NEXT:    lbu t1, 5(a0)
+; RV32I-NEXT:    lbu t2, 6(a0)
+; RV32I-NEXT:    lbu t3, 7(a0)
+; RV32I-NEXT:    slli a4, a4, 8
+; RV32I-NEXT:    or a4, a4, a3
+; RV32I-NEXT:    lbu a6, 11(a0)
+; RV32I-NEXT:    lbu a3, 15(a0)
+; RV32I-NEXT:    slli a7, a7, 8
+; RV32I-NEXT:    slli t1, t1, 8
+; RV32I-NEXT:    slli t3, t3, 8
+; RV32I-NEXT:    or a5, a7, a5
+; RV32I-NEXT:    or a7, t1, t0
+; RV32I-NEXT:    lbu t0, 0(a1)
+; RV32I-NEXT:    lbu t1, 1(a1)
+; RV32I-NEXT:    or t2, t3, t2
+; RV32I-NEXT:    lbu t3, 2(a1)
+; RV32I-NEXT:    lbu a1, 3(a1)
+; RV32I-NEXT:    slli t1, t1, 8
+; RV32I-NEXT:    or t0, t1, t0
+; RV32I-NEXT:    li s9, 64
+; RV32I-NEXT:    slli a1, a1, 8
+; RV32I-NEXT:    or a1, a1, t3
+; RV32I-NEXT:    li t4, 32
+; RV32I-NEXT:    slli a5, a5, 16
+; RV32I-NEXT:    slli t2, t2, 16
+; RV32I-NEXT:    slli a1, a1, 16
+; RV32I-NEXT:    or t3, a5, a4
+; RV32I-NEXT:    or a5, t2, a7
+; RV32I-NEXT:    or a4, a1, t0
+; RV32I-NEXT:    slli a4, a4, 5
+; RV32I-NEXT:    neg s10, a4
+; RV32I-NEXT:    srl t5, t3, s10
+; RV32I-NEXT:    sll s5, a5, a4
+; RV32I-NEXT:    bltu a4, t4, .LBB16_2
+; RV32I-NEXT:  # %bb.1:
+; RV32I-NEXT:    li s8, 0
+; RV32I-NEXT:    sll a7, t3, a4
+; RV32I-NEXT:    j .LBB16_3
+; RV32I-NEXT:  .LBB16_2:
+; RV32I-NEXT:    sll s8, t3, a4
+; RV32I-NEXT:    or a7, t5, s5
+; RV32I-NEXT:  .LBB16_3:
+; RV32I-NEXT:    lbu t2, 9(a0)
+; RV32I-NEXT:    lbu a1, 10(a0)
+; RV32I-NEXT:    lbu t1, 13(a0)
+; RV32I-NEXT:    lbu t0, 14(a0)
+; RV32I-NEXT:    slli a6, a6, 8
+; RV32I-NEXT:    slli t6, a3, 8
+; RV32I-NEXT:    sub s6, s9, a4
+; RV32I-NEXT:    mv a3, a5
+; RV32I-NEXT:    beqz a4, .LBB16_5
+; RV32I-NEXT:  # %bb.4:
+; RV32I-NEXT:    mv a3, a7
+; RV32I-NEXT:  .LBB16_5:
+; RV32I-NEXT:    slli a7, t2, 8
+; RV32I-NEXT:    or a6, a6, a1
+; RV32I-NEXT:    lbu t2, 8(a0)
+; RV32I-NEXT:    lbu a1, 12(a0)
+; RV32I-NEXT:    slli t1, t1, 8
+; RV32I-NEXT:    or t0, t6, t0
+; RV32I-NEXT:    neg t6, s6
+; RV32I-NEXT:    sw t6, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    bltu s6, t4, .LBB16_7
+; RV32I-NEXT:  # %bb.6:
+; RV32I-NEXT:    srl t6, a5, s6
+; RV32I-NEXT:    j .LBB16_8
+; RV32I-NEXT:  .LBB16_7:
+; RV32I-NEXT:    sll t6, a5, t6
+; RV32I-NEXT:    or t6, t5, t6
+; RV32I-NEXT:  .LBB16_8:
+; RV32I-NEXT:    or a7, a7, t2
+; RV32I-NEXT:    slli t2, a6, 16
+; RV32I-NEXT:    or a1, t1, a1
+; RV32I-NEXT:    slli t0, t0, 16
+; RV32I-NEXT:    mv a6, t3
+; RV32I-NEXT:    beqz s6, .LBB16_10
+; RV32I-NEXT:  # %bb.9:
+; RV32I-NEXT:    mv a6, t6
+; RV32I-NEXT:  .LBB16_10:
+; RV32I-NEXT:    or t1, t2, a7
+; RV32I-NEXT:    or t2, t0, a1
+; RV32I-NEXT:    bltu s6, t4, .LBB16_12
+; RV32I-NEXT:  # %bb.11:
+; RV32I-NEXT:    li a7, 0
+; RV32I-NEXT:    j .LBB16_13
+; RV32I-NEXT:  .LBB16_12:
+; RV32I-NEXT:    srl a7, a5, s10
+; RV32I-NEXT:  .LBB16_13:
+; RV32I-NEXT:    srl s0, t1, s10
+; RV32I-NEXT:    sll a1, t2, a4
+; RV32I-NEXT:    sw a1, 20(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    bltu a4, t4, .LBB16_15
+; RV32I-NEXT:  # %bb.14:
+; RV32I-NEXT:    li s1, 0
+; RV32I-NEXT:    sll a1, t1, a4
+; RV32I-NEXT:    j .LBB16_16
+; RV32I-NEXT:  .LBB16_15:
+; RV32I-NEXT:    sll s1, t1, a4
+; RV32I-NEXT:    or a1, s0, a1
+; RV32I-NEXT:  .LBB16_16:
+; RV32I-NEXT:    addi s7, a4, -64
+; RV32I-NEXT:    mv s3, t2
+; RV32I-NEXT:    beqz a4, .LBB16_18
+; RV32I-NEXT:  # %bb.17:
+; RV32I-NEXT:    mv s3, a1
+; RV32I-NEXT:  .LBB16_18:
+; RV32I-NEXT:    neg a1, s7
+; RV32I-NEXT:    sw a1, 16(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    bltu s7, t4, .LBB16_20
+; RV32I-NEXT:  # %bb.19:
+; RV32I-NEXT:    li s2, 0
+; RV32I-NEXT:    sll a1, t3, s7
+; RV32I-NEXT:    mv s4, a5
+; RV32I-NEXT:    bnez s7, .LBB16_21
+; RV32I-NEXT:    j .LBB16_22
+; RV32I-NEXT:  .LBB16_20:
+; RV32I-NEXT:    sll s2, t3, a4
+; RV32I-NEXT:    srl a1, t3, a1
+; RV32I-NEXT:    or a1, a1, s5
+; RV32I-NEXT:    mv s4, a5
+; RV32I-NEXT:    beqz s7, .LBB16_22
+; RV32I-NEXT:  .LBB16_21:
+; RV32I-NEXT:    mv s4, a1
+; RV32I-NEXT:  .LBB16_22:
+; RV32I-NEXT:    li a1, 128
+; RV32I-NEXT:    bltu a4, s9, .LBB16_24
+; RV32I-NEXT:  # %bb.23:
+; RV32I-NEXT:    sw zero, 28(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    li a3, 0
+; RV32I-NEXT:    j .LBB16_25
+; RV32I-NEXT:  .LBB16_24:
+; RV32I-NEXT:    sw s8, 28(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    or s2, a6, s1
+; RV32I-NEXT:    or s4, a7, s3
+; RV32I-NEXT:  .LBB16_25:
+; RV32I-NEXT:    sub ra, a1, a4
+; RV32I-NEXT:    mv a7, t1
+; RV32I-NEXT:    mv a6, t2
+; RV32I-NEXT:    beqz a4, .LBB16_27
+; RV32I-NEXT:  # %bb.26:
+; RV32I-NEXT:    mv a7, s2
+; RV32I-NEXT:    mv a6, s4
+; RV32I-NEXT:  .LBB16_27:
+; RV32I-NEXT:    neg s1, ra
+; RV32I-NEXT:    sll s2, t2, s1
+; RV32I-NEXT:    bltu ra, t4, .LBB16_29
+; RV32I-NEXT:  # %bb.28:
+; RV32I-NEXT:    srl a1, t2, ra
+; RV32I-NEXT:    sw t1, 40(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    bnez ra, .LBB16_30
+; RV32I-NEXT:    j .LBB16_31
+; RV32I-NEXT:  .LBB16_29:
+; RV32I-NEXT:    or a1, s0, s2
+; RV32I-NEXT:    sw t1, 40(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    beqz ra, .LBB16_31
+; RV32I-NEXT:  .LBB16_30:
+; RV32I-NEXT:    sw a1, 40(sp) # 4-byte Folded Spill
+; RV32I-NEXT:  .LBB16_31:
+; RV32I-NEXT:    bltu ra, t4, .LBB16_33
+; RV32I-NEXT:  # %bb.32:
+; RV32I-NEXT:    sw zero, 36(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    srl a1, a5, ra
+; RV32I-NEXT:    mv t5, t3
+; RV32I-NEXT:    bnez ra, .LBB16_34
+; RV32I-NEXT:    j .LBB16_35
+; RV32I-NEXT:  .LBB16_33:
+; RV32I-NEXT:    srl a1, t2, s10
+; RV32I-NEXT:    sw a1, 36(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sll a1, a5, s1
+; RV32I-NEXT:    or a1, t5, a1
+; RV32I-NEXT:    mv t5, t3
+; RV32I-NEXT:    beqz ra, .LBB16_35
+; RV32I-NEXT:  .LBB16_34:
+; RV32I-NEXT:    mv t5, a1
+; RV32I-NEXT:  .LBB16_35:
+; RV32I-NEXT:    sub s3, s9, ra
+; RV32I-NEXT:    bltu ra, t4, .LBB16_38
+; RV32I-NEXT:  # %bb.36:
+; RV32I-NEXT:    sw zero, 32(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    bgeu s3, t4, .LBB16_39
+; RV32I-NEXT:  .LBB16_37:
+; RV32I-NEXT:    sll s1, t1, s1
+; RV32I-NEXT:    neg a1, s3
+; RV32I-NEXT:    srl a1, t1, a1
+; RV32I-NEXT:    or a1, a1, s2
+; RV32I-NEXT:    j .LBB16_40
+; RV32I-NEXT:  .LBB16_38:
+; RV32I-NEXT:    srl a1, a5, s10
+; RV32I-NEXT:    sw a1, 32(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    bltu s3, t4, .LBB16_37
+; RV32I-NEXT:  .LBB16_39:
+; RV32I-NEXT:    li s1, 0
+; RV32I-NEXT:    sll a1, t1, s3
+; RV32I-NEXT:  .LBB16_40:
+; RV32I-NEXT:    addi s4, ra, -64
+; RV32I-NEXT:    mv s2, t2
+; RV32I-NEXT:    beqz s3, .LBB16_42
+; RV32I-NEXT:  # %bb.41:
+; RV32I-NEXT:    mv s2, a1
+; RV32I-NEXT:  .LBB16_42:
+; RV32I-NEXT:    sw s5, 24(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    mv s5, a7
+; RV32I-NEXT:    bltu s4, t4, .LBB16_44
+; RV32I-NEXT:  # %bb.43:
+; RV32I-NEXT:    srl t0, t2, s4
+; RV32I-NEXT:    j .LBB16_45
+; RV32I-NEXT:  .LBB16_44:
+; RV32I-NEXT:    srl a1, t1, ra
+; RV32I-NEXT:    neg t0, s4
+; RV32I-NEXT:    sll t0, t2, t0
+; RV32I-NEXT:    or t0, a1, t0
+; RV32I-NEXT:  .LBB16_45:
+; RV32I-NEXT:    mv s0, s10
+; RV32I-NEXT:    mv a7, a6
+; RV32I-NEXT:    lbu s8, 19(a0)
+; RV32I-NEXT:    lbu a1, 23(a0)
+; RV32I-NEXT:    mv s3, t1
+; RV32I-NEXT:    beqz s4, .LBB16_47
+; RV32I-NEXT:  # %bb.46:
+; RV32I-NEXT:    mv s3, t0
+; RV32I-NEXT:  .LBB16_47:
+; RV32I-NEXT:    mv a6, a3
+; RV32I-NEXT:    lbu s10, 17(a0)
+; RV32I-NEXT:    lbu t0, 18(a0)
+; RV32I-NEXT:    lbu s9, 21(a0)
+; RV32I-NEXT:    lbu t6, 22(a0)
+; RV32I-NEXT:    slli s8, s8, 8
+; RV32I-NEXT:    slli a1, a1, 8
+; RV32I-NEXT:    li a3, 64
+; RV32I-NEXT:    bltu s4, t4, .LBB16_49
+; RV32I-NEXT:  # %bb.48:
+; RV32I-NEXT:    li s4, 0
+; RV32I-NEXT:    j .LBB16_50
+; RV32I-NEXT:  .LBB16_49:
+; RV32I-NEXT:    srl s4, t2, ra
+; RV32I-NEXT:  .LBB16_50:
+; RV32I-NEXT:    or s11, s8, t0
+; RV32I-NEXT:    lbu t0, 16(a0)
+; RV32I-NEXT:    lbu s8, 20(a0)
+; RV32I-NEXT:    slli s10, s10, 8
+; RV32I-NEXT:    slli s9, s9, 8
+; RV32I-NEXT:    or t6, a1, t6
+; RV32I-NEXT:    bgeu ra, a3, .LBB16_52
+; RV32I-NEXT:  # %bb.51:
+; RV32I-NEXT:    or s3, t5, s1
+; RV32I-NEXT:    lw a1, 32(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    or s4, a1, s2
+; RV32I-NEXT:  .LBB16_52:
+; RV32I-NEXT:    or a1, s10, t0
+; RV32I-NEXT:    slli s11, s11, 16
+; RV32I-NEXT:    or t0, s9, s8
+; RV32I-NEXT:    slli t6, t6, 16
+; RV32I-NEXT:    mv t5, t3
+; RV32I-NEXT:    mv s1, a5
+; RV32I-NEXT:    mv a3, a6
+; RV32I-NEXT:    beqz ra, .LBB16_54
+; RV32I-NEXT:  # %bb.53:
+; RV32I-NEXT:    mv t5, s3
+; RV32I-NEXT:    mv s1, s4
+; RV32I-NEXT:  .LBB16_54:
+; RV32I-NEXT:    sw s1, 4(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    or s2, s11, a1
+; RV32I-NEXT:    or s1, t6, t0
+; RV32I-NEXT:    li a1, 64
+; RV32I-NEXT:    mv a6, a7
+; RV32I-NEXT:    mv a7, s0
+; RV32I-NEXT:    bltu ra, a1, .LBB16_56
+; RV32I-NEXT:  # %bb.55:
+; RV32I-NEXT:    sw zero, 40(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw zero, 36(sp) # 4-byte Folded Spill
+; RV32I-NEXT:  .LBB16_56:
+; RV32I-NEXT:    srl s3, s2, a7
+; RV32I-NEXT:    sll ra, s1, a4
+; RV32I-NEXT:    mv a7, s5
+; RV32I-NEXT:    sw t5, 8(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    bltu a4, t4, .LBB16_58
+; RV32I-NEXT:  # %bb.57:
+; RV32I-NEXT:    sw zero, 32(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sll a1, s2, a4
+; RV32I-NEXT:    j .LBB16_59
+; RV32I-NEXT:  .LBB16_58:
+; RV32I-NEXT:    sll a1, s2, a4
+; RV32I-NEXT:    sw a1, 32(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    or a1, s3, ra
+; RV32I-NEXT:  .LBB16_59:
+; RV32I-NEXT:    lbu s9, 27(a0)
+; RV32I-NEXT:    lbu t6, 31(a0)
+; RV32I-NEXT:    mv t5, s1
+; RV32I-NEXT:    beqz a4, .LBB16_61
+; RV32I-NEXT:  # %bb.60:
+; RV32I-NEXT:    mv t5, a1
+; RV32I-NEXT:  .LBB16_61:
+; RV32I-NEXT:    lbu s8, 25(a0)
+; RV32I-NEXT:    lbu s4, 26(a0)
+; RV32I-NEXT:    lbu s11, 29(a0)
+; RV32I-NEXT:    lbu s10, 30(a0)
+; RV32I-NEXT:    slli s9, s9, 8
+; RV32I-NEXT:    slli t6, t6, 8
+; RV32I-NEXT:    bltu s6, t4, .LBB16_63
+; RV32I-NEXT:  # %bb.62:
+; RV32I-NEXT:    srl t0, s1, s6
+; RV32I-NEXT:    j .LBB16_64
+; RV32I-NEXT:  .LBB16_63:
+; RV32I-NEXT:    lw a1, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    sll a1, s1, a1
+; RV32I-NEXT:    or t0, s3, a1
+; RV32I-NEXT:  .LBB16_64:
+; RV32I-NEXT:    slli s8, s8, 8
+; RV32I-NEXT:    lbu s3, 24(a0)
+; RV32I-NEXT:    lbu a1, 28(a0)
+; RV32I-NEXT:    or s4, s9, s4
+; RV32I-NEXT:    slli s11, s11, 8
+; RV32I-NEXT:    or t6, t6, s10
+; RV32I-NEXT:    mv s9, s2
+; RV32I-NEXT:    beqz s6, .LBB16_66
+; RV32I-NEXT:  # %bb.65:
+; RV32I-NEXT:    mv s9, t0
+; RV32I-NEXT:  .LBB16_66:
+; RV32I-NEXT:    or a0, s8, s3
+; RV32I-NEXT:    slli t0, s4, 16
+; RV32I-NEXT:    or a1, s11, a1
+; RV32I-NEXT:    slli t6, t6, 16
+; RV32I-NEXT:    bltu s6, t4, .LBB16_68
+; RV32I-NEXT:  # %bb.67:
+; RV32I-NEXT:    li s4, 0
+; RV32I-NEXT:    j .LBB16_69
+; RV32I-NEXT:  .LBB16_68:
+; RV32I-NEXT:    srl s4, s1, s0
+; RV32I-NEXT:  .LBB16_69:
+; RV32I-NEXT:    li s11, 64
+; RV32I-NEXT:    or s6, t0, a0
+; RV32I-NEXT:    or a0, t6, a1
+; RV32I-NEXT:    bltu a4, t4, .LBB16_71
+; RV32I-NEXT:  # %bb.70:
+; RV32I-NEXT:    li s3, 0
+; RV32I-NEXT:    sll a1, s6, a4
+; RV32I-NEXT:    mv s10, a0
+; RV32I-NEXT:    bnez a4, .LBB16_72
+; RV32I-NEXT:    j .LBB16_73
+; RV32I-NEXT:  .LBB16_71:
+; RV32I-NEXT:    sll s3, s6, a4
+; RV32I-NEXT:    srl a1, s6, s0
+; RV32I-NEXT:    sll t0, a0, a4
+; RV32I-NEXT:    or a1, a1, t0
+; RV32I-NEXT:    mv s10, a0
+; RV32I-NEXT:    beqz a4, .LBB16_73
+; RV32I-NEXT:  .LBB16_72:
+; RV32I-NEXT:    mv s10, a1
+; RV32I-NEXT:  .LBB16_73:
+; RV32I-NEXT:    bltu s7, t4, .LBB16_75
+; RV32I-NEXT:  # %bb.74:
+; RV32I-NEXT:    li s5, 0
+; RV32I-NEXT:    sll a1, s2, s7
+; RV32I-NEXT:    mv s0, s1
+; RV32I-NEXT:    bnez s7, .LBB16_76
+; RV32I-NEXT:    j .LBB16_77
+; RV32I-NEXT:  .LBB16_75:
+; RV32I-NEXT:    sll s5, s2, a4
+; RV32I-NEXT:    lw a1, 16(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    srl a1, s2, a1
+; RV32I-NEXT:    or a1, a1, ra
+; RV32I-NEXT:    mv s0, s1
+; RV32I-NEXT:    beqz s7, .LBB16_77
+; RV32I-NEXT:  .LBB16_76:
+; RV32I-NEXT:    mv s0, a1
+; RV32I-NEXT:  .LBB16_77:
+; RV32I-NEXT:    bltu a4, s11, .LBB16_79
+; RV32I-NEXT:  # %bb.78:
+; RV32I-NEXT:    sw zero, 32(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    li t5, 0
+; RV32I-NEXT:    j .LBB16_80
+; RV32I-NEXT:  .LBB16_79:
+; RV32I-NEXT:    or s5, s9, s3
+; RV32I-NEXT:    or s0, s4, s10
+; RV32I-NEXT:  .LBB16_80:
+; RV32I-NEXT:    addi s9, a4, -128
+; RV32I-NEXT:    mv s7, s6
+; RV32I-NEXT:    mv s8, a0
+; RV32I-NEXT:    beqz a4, .LBB16_82
+; RV32I-NEXT:  # %bb.81:
+; RV32I-NEXT:    mv s7, s5
+; RV32I-NEXT:    mv s8, s0
+; RV32I-NEXT:  .LBB16_82:
+; RV32I-NEXT:    neg s3, s9
+; RV32I-NEXT:    srl s0, t3, s3
+; RV32I-NEXT:    bltu s9, t4, .LBB16_84
+; RV32I-NEXT:  # %bb.83:
+; RV32I-NEXT:    li s5, 0
+; RV32I-NEXT:    sll a1, t3, s9
+; RV32I-NEXT:    j .LBB16_85
+; RV32I-NEXT:  .LBB16_84:
+; RV32I-NEXT:    sll s5, t3, a4
+; RV32I-NEXT:    lw a1, 24(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    or a1, s0, a1
+; RV32I-NEXT:  .LBB16_85:
+; RV32I-NEXT:    sub s4, s11, s9
+; RV32I-NEXT:    mv t6, a5
+; RV32I-NEXT:    beqz s9, .LBB16_87
+; RV32I-NEXT:  # %bb.86:
+; RV32I-NEXT:    mv t6, a1
+; RV32I-NEXT:  .LBB16_87:
+; RV32I-NEXT:    bltu s4, t4, .LBB16_89
+; RV32I-NEXT:  # %bb.88:
+; RV32I-NEXT:    srl a1, a5, s4
+; RV32I-NEXT:    mv s0, t3
+; RV32I-NEXT:    bnez s4, .LBB16_90
+; RV32I-NEXT:    j .LBB16_91
+; RV32I-NEXT:  .LBB16_89:
+; RV32I-NEXT:    neg a1, s4
+; RV32I-NEXT:    sll a1, a5, a1
+; RV32I-NEXT:    or a1, s0, a1
+; RV32I-NEXT:    mv s0, t3
+; RV32I-NEXT:    beqz s4, .LBB16_91
+; RV32I-NEXT:  .LBB16_90:
+; RV32I-NEXT:    mv s0, a1
+; RV32I-NEXT:  .LBB16_91:
+; RV32I-NEXT:    bltu s4, t4, .LBB16_94
+; RV32I-NEXT:  # %bb.92:
+; RV32I-NEXT:    li s4, 0
+; RV32I-NEXT:    li ra, 64
+; RV32I-NEXT:    bgeu s9, t4, .LBB16_95
+; RV32I-NEXT:  .LBB16_93:
+; RV32I-NEXT:    sll s10, t1, a4
+; RV32I-NEXT:    srl a1, t1, s3
+; RV32I-NEXT:    lw t0, 20(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    or a1, a1, t0
+; RV32I-NEXT:    j .LBB16_96
+; RV32I-NEXT:  .LBB16_94:
+; RV32I-NEXT:    srl s4, a5, s3
+; RV32I-NEXT:    li ra, 64
+; RV32I-NEXT:    bltu s9, t4, .LBB16_93
+; RV32I-NEXT:  .LBB16_95:
+; RV32I-NEXT:    li s10, 0
+; RV32I-NEXT:    sll a1, t1, s9
+; RV32I-NEXT:  .LBB16_96:
+; RV32I-NEXT:    addi s11, s9, -64
+; RV32I-NEXT:    mv s3, t2
+; RV32I-NEXT:    beqz s9, .LBB16_98
+; RV32I-NEXT:  # %bb.97:
+; RV32I-NEXT:    mv s3, a1
+; RV32I-NEXT:  .LBB16_98:
+; RV32I-NEXT:    bltu s11, t4, .LBB16_100
+; RV32I-NEXT:  # %bb.99:
+; RV32I-NEXT:    li t4, 0
+; RV32I-NEXT:    sll a1, t3, s11
+; RV32I-NEXT:    bnez s11, .LBB16_101
+; RV32I-NEXT:    j .LBB16_102
+; RV32I-NEXT:  .LBB16_100:
+; RV32I-NEXT:    sll t4, t3, s9
+; RV32I-NEXT:    neg a1, s11
+; RV32I-NEXT:    srl a1, t3, a1
+; RV32I-NEXT:    sll t0, a5, s9
+; RV32I-NEXT:    or a1, a1, t0
+; RV32I-NEXT:    beqz s11, .LBB16_102
+; RV32I-NEXT:  .LBB16_101:
+; RV32I-NEXT:    mv a5, a1
+; RV32I-NEXT:  .LBB16_102:
+; RV32I-NEXT:    bltu s9, ra, .LBB16_104
+; RV32I-NEXT:  # %bb.103:
+; RV32I-NEXT:    li s5, 0
+; RV32I-NEXT:    li t6, 0
+; RV32I-NEXT:    li a1, 128
+; RV32I-NEXT:    bnez s9, .LBB16_105
+; RV32I-NEXT:    j .LBB16_106
+; RV32I-NEXT:  .LBB16_104:
+; RV32I-NEXT:    or t4, s0, s10
+; RV32I-NEXT:    or a5, s4, s3
+; RV32I-NEXT:    li a1, 128
+; RV32I-NEXT:    beqz s9, .LBB16_106
+; RV32I-NEXT:  .LBB16_105:
+; RV32I-NEXT:    mv t1, t4
+; RV32I-NEXT:    mv t2, a5
+; RV32I-NEXT:  .LBB16_106:
+; RV32I-NEXT:    bltu a4, a1, .LBB16_108
+; RV32I-NEXT:  # %bb.107:
+; RV32I-NEXT:    li ra, 0
+; RV32I-NEXT:    li a3, 0
+; RV32I-NEXT:    li a7, 0
+; RV32I-NEXT:    li a6, 0
+; RV32I-NEXT:    bnez a4, .LBB16_109
+; RV32I-NEXT:    j .LBB16_110
+; RV32I-NEXT:  .LBB16_108:
+; RV32I-NEXT:    lw a1, 8(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw a5, 32(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    or s5, a1, a5
+; RV32I-NEXT:    lw a1, 4(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    or t6, a1, t5
+; RV32I-NEXT:    lw a1, 40(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    or t1, a1, s7
+; RV32I-NEXT:    lw a1, 36(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    or t2, a1, s8
+; RV32I-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    beqz a4, .LBB16_110
+; RV32I-NEXT:  .LBB16_109:
+; RV32I-NEXT:    mv s2, s5
+; RV32I-NEXT:    mv s1, t6
+; RV32I-NEXT:    mv s6, t1
+; RV32I-NEXT:    mv a0, t2
+; RV32I-NEXT:  .LBB16_110:
+; RV32I-NEXT:    srli a4, ra, 16
+; RV32I-NEXT:    lui t2, 16
+; RV32I-NEXT:    srli t1, ra, 24
+; RV32I-NEXT:    srli a5, a3, 16
+; RV32I-NEXT:    srli t4, a3, 24
+; RV32I-NEXT:    srli t0, a7, 16
+; RV32I-NEXT:    srli s0, a7, 24
+; RV32I-NEXT:    srli t3, a6, 16
+; RV32I-NEXT:    srli s3, a6, 24
+; RV32I-NEXT:    srli t6, s2, 16
+; RV32I-NEXT:    srli a1, s2, 24
+; RV32I-NEXT:    srli t5, s1, 16
+; RV32I-NEXT:    srli s5, s1, 24
+; RV32I-NEXT:    srli s4, s6, 16
+; RV32I-NEXT:    srli s7, s6, 24
+; RV32I-NEXT:    srli s8, a0, 16
+; RV32I-NEXT:    srli s9, a0, 24
+; RV32I-NEXT:    addi t2, t2, -1
+; RV32I-NEXT:    and s10, ra, t2
+; RV32I-NEXT:    and s11, a3, t2
+; RV32I-NEXT:    srli s10, s10, 8
+; RV32I-NEXT:    sb ra, 0(a2)
+; RV32I-NEXT:    sb s10, 1(a2)
+; RV32I-NEXT:    sb a4, 2(a2)
+; RV32I-NEXT:    sb t1, 3(a2)
+; RV32I-NEXT:    and a4, a7, t2
+; RV32I-NEXT:    srli t1, s11, 8
+; RV32I-NEXT:    sb a3, 4(a2)
+; RV32I-NEXT:    sb t1, 5(a2)
+; RV32I-NEXT:    sb a5, 6(a2)
+; RV32I-NEXT:    sb t4, 7(a2)
+; RV32I-NEXT:    and a3, a6, t2
+; RV32I-NEXT:    srli a4, a4, 8
+; RV32I-NEXT:    sb a7, 8(a2)
+; RV32I-NEXT:    sb a4, 9(a2)
+; RV32I-NEXT:    sb t0, 10(a2)
+; RV32I-NEXT:    sb s0, 11(a2)
+; RV32I-NEXT:    and a4, s2, t2
+; RV32I-NEXT:    srli a3, a3, 8
+; RV32I-NEXT:    sb a6, 12(a2)
+; RV32I-NEXT:    sb a3, 13(a2)
+; RV32I-NEXT:    sb t3, 14(a2)
+; RV32I-NEXT:    sb s3, 15(a2)
+; RV32I-NEXT:    and a3, s1, t2
+; RV32I-NEXT:    srli a4, a4, 8
+; RV32I-NEXT:    sb s2, 16(a2)
+; RV32I-NEXT:    sb a4, 17(a2)
+; RV32I-NEXT:    sb t6, 18(a2)
+; RV32I-NEXT:    sb a1, 19(a2)
+; RV32I-NEXT:    and a1, s6, t2
+; RV32I-NEXT:    and a4, a0, t2
+; RV32I-NEXT:    srli a3, a3, 8
+; RV32I-NEXT:    srli a1, a1, 8
+; RV32I-NEXT:    srli a4, a4, 8
+; RV32I-NEXT:    sb s1, 20(a2)
+; RV32I-NEXT:    sb a3, 21(a2)
+; RV32I-NEXT:    sb t5, 22(a2)
+; RV32I-NEXT:    sb s5, 23(a2)
+; RV32I-NEXT:    sb s6, 24(a2)
+; RV32I-NEXT:    sb a1, 25(a2)
+; RV32I-NEXT:    sb s4, 26(a2)
+; RV32I-NEXT:    sb s7, 27(a2)
+; RV32I-NEXT:    sb a0, 28(a2)
+; RV32I-NEXT:    sb a4, 29(a2)
+; RV32I-NEXT:    sb s8, 30(a2)
+; RV32I-NEXT:    sb s9, 31(a2)
+; RV32I-NEXT:    lw ra, 92(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s0, 88(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s1, 84(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s2, 80(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s3, 76(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s4, 72(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s5, 68(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s6, 64(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s7, 60(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s8, 56(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s9, 52(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s10, 48(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s11, 44(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    addi sp, sp, 96
+; RV32I-NEXT:    ret
+  %src = load i256, ptr %src.ptr, align 1
+  %wordOff = load i256, ptr %wordOff.ptr, align 1
+  %bitOff = shl i256 %wordOff, 5
+  %res = shl i256 %src, %bitOff
+  store i256 %res, ptr %dst, align 1
+  ret void
+}
+
+define void @shl_32bytes_dwordOff(ptr %src.ptr, ptr %dwordOff.ptr, ptr %dst) nounwind {
+; RV64I-LABEL: shl_32bytes_dwordOff:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -112
+; RV64I-NEXT:    sd ra, 104(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s0, 96(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s1, 88(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s2, 80(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s3, 72(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s4, 64(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s5, 56(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s6, 48(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s7, 40(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s8, 32(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s9, 24(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s10, 16(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s11, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    lbu a3, 0(a0)
+; RV64I-NEXT:    lbu a4, 1(a0)
+; RV64I-NEXT:    lbu a5, 2(a0)
+; RV64I-NEXT:    lbu a6, 3(a0)
+; RV64I-NEXT:    lbu a7, 4(a0)
+; RV64I-NEXT:    lbu t0, 5(a0)
+; RV64I-NEXT:    lbu t1, 6(a0)
+; RV64I-NEXT:    lbu t2, 7(a0)
+; RV64I-NEXT:    lbu t3, 8(a0)
+; RV64I-NEXT:    lbu t4, 9(a0)
+; RV64I-NEXT:    lbu t5, 10(a0)
+; RV64I-NEXT:    lbu t6, 11(a0)
+; RV64I-NEXT:    slli a4, a4, 8
+; RV64I-NEXT:    or a3, a4, a3
+; RV64I-NEXT:    lbu a4, 12(a0)
+; RV64I-NEXT:    lbu s0, 13(a0)
+; RV64I-NEXT:    lbu s1, 14(a0)
+; RV64I-NEXT:    lbu s2, 15(a0)
+; RV64I-NEXT:    slli a6, a6, 8
+; RV64I-NEXT:    slli t0, t0, 8
+; RV64I-NEXT:    slli t2, t2, 8
+; RV64I-NEXT:    slli t4, t4, 8
+; RV64I-NEXT:    or s3, a6, a5
+; RV64I-NEXT:    or a5, t0, a7
+; RV64I-NEXT:    or a7, t2, t1
+; RV64I-NEXT:    or a6, t4, t3
+; RV64I-NEXT:    lbu t0, 0(a1)
+; RV64I-NEXT:    lbu t1, 1(a1)
+; RV64I-NEXT:    lbu t2, 2(a1)
+; RV64I-NEXT:    lbu t3, 3(a1)
+; RV64I-NEXT:    slli t6, t6, 8
+; RV64I-NEXT:    slli s0, s0, 8
+; RV64I-NEXT:    slli s2, s2, 8
+; RV64I-NEXT:    or t6, t6, t5
+; RV64I-NEXT:    or s0, s0, a4
+; RV64I-NEXT:    or s1, s2, s1
+; RV64I-NEXT:    lbu a4, 4(a1)
+; RV64I-NEXT:    lbu t4, 5(a1)
+; RV64I-NEXT:    lbu t5, 6(a1)
+; RV64I-NEXT:    lbu a1, 7(a1)
+; RV64I-NEXT:    slli t1, t1, 8
+; RV64I-NEXT:    slli t3, t3, 8
+; RV64I-NEXT:    slli t4, t4, 8
+; RV64I-NEXT:    slli a1, a1, 8
+; RV64I-NEXT:    or t0, t1, t0
+; RV64I-NEXT:    or t1, t3, t2
+; RV64I-NEXT:    or t2, t4, a4
+; RV64I-NEXT:    or a1, a1, t5
+; RV64I-NEXT:    lbu t5, 19(a0)
+; RV64I-NEXT:    lbu t4, 21(a0)
+; RV64I-NEXT:    lbu a4, 22(a0)
+; RV64I-NEXT:    lbu t3, 23(a0)
+; RV64I-NEXT:    slli s3, s3, 16
+; RV64I-NEXT:    slli a7, a7, 16
+; RV64I-NEXT:    slli t6, t6, 16
+; RV64I-NEXT:    slli s1, s1, 16
+; RV64I-NEXT:    or s4, s3, a3
+; RV64I-NEXT:    or a5, a7, a5
+; RV64I-NEXT:    or a6, t6, a6
+; RV64I-NEXT:    or s0, s1, s0
+; RV64I-NEXT:    lbu s1, 27(a0)
+; RV64I-NEXT:    lbu t6, 29(a0)
+; RV64I-NEXT:    lbu a3, 30(a0)
+; RV64I-NEXT:    lbu s2, 31(a0)
+; RV64I-NEXT:    slli t1, t1, 16
+; RV64I-NEXT:    or s5, t1, t0
+; RV64I-NEXT:    li a7, 128
+; RV64I-NEXT:    slli a1, a1, 16
+; RV64I-NEXT:    or a1, a1, t2
+; RV64I-NEXT:    li t0, 64
+; RV64I-NEXT:    slli s3, t3, 8
+; RV64I-NEXT:    slli s2, s2, 8
+; RV64I-NEXT:    slli a5, a5, 32
+; RV64I-NEXT:    slli s0, s0, 32
+; RV64I-NEXT:    slli a1, a1, 32
+; RV64I-NEXT:    or t1, a5, s4
+; RV64I-NEXT:    or a5, s0, a6
+; RV64I-NEXT:    or a6, a1, s5
+; RV64I-NEXT:    slli a6, a6, 6
+; RV64I-NEXT:    subw t2, a6, t0
+; RV64I-NEXT:    negw t3, a6
+; RV64I-NEXT:    srl s0, t1, t3
+; RV64I-NEXT:    bltu a6, t0, .LBB17_2
+; RV64I-NEXT:  # %bb.1:
+; RV64I-NEXT:    li a1, 0
+; RV64I-NEXT:    sll s4, t1, t2
+; RV64I-NEXT:    j .LBB17_3
+; RV64I-NEXT:  .LBB17_2:
+; RV64I-NEXT:    sll a1, t1, a6
+; RV64I-NEXT:    sll s4, a5, a6
+; RV64I-NEXT:    or s4, s0, s4
+; RV64I-NEXT:  .LBB17_3:
+; RV64I-NEXT:    slli t5, t5, 8
+; RV64I-NEXT:    slli t4, t4, 8
+; RV64I-NEXT:    or s3, s3, a4
+; RV64I-NEXT:    lbu ra, 17(a0)
+; RV64I-NEXT:    lbu s11, 18(a0)
+; RV64I-NEXT:    lbu s8, 20(a0)
+; RV64I-NEXT:    lbu s5, 25(a0)
+; RV64I-NEXT:    slli s1, s1, 8
+; RV64I-NEXT:    lbu s7, 26(a0)
+; RV64I-NEXT:    lbu s6, 28(a0)
+; RV64I-NEXT:    slli s10, t6, 8
+; RV64I-NEXT:    or s9, s2, a3
+; RV64I-NEXT:    sub a4, a7, a6
+; RV64I-NEXT:    mv a3, a5
+; RV64I-NEXT:    beqz a6, .LBB17_5
+; RV64I-NEXT:  # %bb.4:
+; RV64I-NEXT:    mv a3, s4
+; RV64I-NEXT:  .LBB17_5:
+; RV64I-NEXT:    slli t6, ra, 8
+; RV64I-NEXT:    or t5, t5, s11
+; RV64I-NEXT:    or t4, t4, s8
+; RV64I-NEXT:    slli s3, s3, 16
+; RV64I-NEXT:    lbu s8, 16(a0)
+; RV64I-NEXT:    lbu a0, 24(a0)
+; RV64I-NEXT:    slli s5, s5, 8
+; RV64I-NEXT:    or s2, s1, s7
+; RV64I-NEXT:    or s1, s10, s6
+; RV64I-NEXT:    slli s4, s9, 16
+; RV64I-NEXT:    bltu a4, t0, .LBB17_7
+; RV64I-NEXT:  # %bb.6:
+; RV64I-NEXT:    subw s0, a4, t0
+; RV64I-NEXT:    srl s0, a5, s0
+; RV64I-NEXT:    j .LBB17_8
+; RV64I-NEXT:  .LBB17_7:
+; RV64I-NEXT:    negw s6, a4
+; RV64I-NEXT:    sll s6, a5, s6
+; RV64I-NEXT:    or s0, s0, s6
+; RV64I-NEXT:  .LBB17_8:
+; RV64I-NEXT:    or t6, t6, s8
+; RV64I-NEXT:    slli s6, t5, 16
+; RV64I-NEXT:    or s3, s3, t4
+; RV64I-NEXT:    or t5, s5, a0
+; RV64I-NEXT:    slli s2, s2, 16
+; RV64I-NEXT:    or s1, s4, s1
+; RV64I-NEXT:    mv t4, t1
+; RV64I-NEXT:    beqz a4, .LBB17_10
+; RV64I-NEXT:  # %bb.9:
+; RV64I-NEXT:    mv t4, s0
+; RV64I-NEXT:  .LBB17_10:
+; RV64I-NEXT:    or a0, s6, t6
+; RV64I-NEXT:    slli s0, s3, 32
+; RV64I-NEXT:    or t6, s2, t5
+; RV64I-NEXT:    slli s1, s1, 32
+; RV64I-NEXT:    bltu a4, t0, .LBB17_12
+; RV64I-NEXT:  # %bb.11:
+; RV64I-NEXT:    li t5, 0
+; RV64I-NEXT:    j .LBB17_13
+; RV64I-NEXT:  .LBB17_12:
+; RV64I-NEXT:    srl t5, a5, t3
+; RV64I-NEXT:  .LBB17_13:
+; RV64I-NEXT:    or a4, s0, a0
+; RV64I-NEXT:    or a0, s1, t6
+; RV64I-NEXT:    bltu a6, t0, .LBB17_15
+; RV64I-NEXT:  # %bb.14:
+; RV64I-NEXT:    li t6, 0
+; RV64I-NEXT:    sll t2, a4, t2
+; RV64I-NEXT:    j .LBB17_16
+; RV64I-NEXT:  .LBB17_15:
+; RV64I-NEXT:    sll t6, a4, a6
+; RV64I-NEXT:    srl t2, a4, t3
+; RV64I-NEXT:    sll t3, a0, a6
+; RV64I-NEXT:    or t2, t2, t3
+; RV64I-NEXT:  .LBB17_16:
+; RV64I-NEXT:    sub s0, a6, a7
+; RV64I-NEXT:    mv t3, a0
+; RV64I-NEXT:    beqz a6, .LBB17_18
+; RV64I-NEXT:  # %bb.17:
+; RV64I-NEXT:    mv t3, t2
+; RV64I-NEXT:  .LBB17_18:
+; RV64I-NEXT:    bltu s0, t0, .LBB17_20
+; RV64I-NEXT:  # %bb.19:
+; RV64I-NEXT:    li t2, 0
+; RV64I-NEXT:    subw t0, s0, t0
+; RV64I-NEXT:    sll t0, t1, t0
+; RV64I-NEXT:    bnez s0, .LBB17_21
+; RV64I-NEXT:    j .LBB17_22
+; RV64I-NEXT:  .LBB17_20:
+; RV64I-NEXT:    sll t2, t1, s0
+; RV64I-NEXT:    negw t0, s0
+; RV64I-NEXT:    srl t0, t1, t0
+; RV64I-NEXT:    sll t1, a5, s0
+; RV64I-NEXT:    or t0, t0, t1
+; RV64I-NEXT:    beqz s0, .LBB17_22
+; RV64I-NEXT:  .LBB17_21:
+; RV64I-NEXT:    mv a5, t0
+; RV64I-NEXT:  .LBB17_22:
+; RV64I-NEXT:    bltu a6, a7, .LBB17_24
+; RV64I-NEXT:  # %bb.23:
+; RV64I-NEXT:    li a1, 0
+; RV64I-NEXT:    li a3, 0
+; RV64I-NEXT:    bnez a6, .LBB17_25
+; RV64I-NEXT:    j .LBB17_26
+; RV64I-NEXT:  .LBB17_24:
+; RV64I-NEXT:    or t2, t4, t6
+; RV64I-NEXT:    or a5, t5, t3
+; RV64I-NEXT:    beqz a6, .LBB17_26
+; RV64I-NEXT:  .LBB17_25:
+; RV64I-NEXT:    mv a4, t2
+; RV64I-NEXT:    mv a0, a5
+; RV64I-NEXT:  .LBB17_26:
+; RV64I-NEXT:    srli a5, a1, 32
+; RV64I-NEXT:    srliw a6, a1, 16
+; RV64I-NEXT:    lui t2, 16
+; RV64I-NEXT:    srliw t1, a1, 24
+; RV64I-NEXT:    srli t0, a1, 48
+; RV64I-NEXT:    srli t5, a1, 56
+; RV64I-NEXT:    srli a7, a3, 32
+; RV64I-NEXT:    srliw t4, a3, 16
+; RV64I-NEXT:    srliw s0, a3, 24
+; RV64I-NEXT:    srli t6, a3, 48
+; RV64I-NEXT:    srli s3, a3, 56
+; RV64I-NEXT:    srli t3, a4, 32
+; RV64I-NEXT:    srliw s2, a4, 16
+; RV64I-NEXT:    srliw s6, a4, 24
+; RV64I-NEXT:    srli s4, a4, 48
+; RV64I-NEXT:    srli s7, a4, 56
+; RV64I-NEXT:    srli s1, a0, 32
+; RV64I-NEXT:    srliw s5, a0, 16
+; RV64I-NEXT:    srliw s8, a0, 24
+; RV64I-NEXT:    srli s9, a0, 48
+; RV64I-NEXT:    srli s10, a0, 56
+; RV64I-NEXT:    addi t2, t2, -1
+; RV64I-NEXT:    and s11, a1, t2
+; RV64I-NEXT:    srli s11, s11, 8
+; RV64I-NEXT:    sb a1, 0(a2)
+; RV64I-NEXT:    sb s11, 1(a2)
+; RV64I-NEXT:    sb a6, 2(a2)
+; RV64I-NEXT:    sb t1, 3(a2)
+; RV64I-NEXT:    and a1, a5, t2
+; RV64I-NEXT:    srli a1, a1, 8
+; RV64I-NEXT:    sb a5, 4(a2)
+; RV64I-NEXT:    sb a1, 5(a2)
+; RV64I-NEXT:    sb t0, 6(a2)
+; RV64I-NEXT:    sb t5, 7(a2)
+; RV64I-NEXT:    and a1, a3, t2
+; RV64I-NEXT:    srli a1, a1, 8
+; RV64I-NEXT:    sb a3, 8(a2)
+; RV64I-NEXT:    sb a1, 9(a2)
+; RV64I-NEXT:    sb t4, 10(a2)
+; RV64I-NEXT:    sb s0, 11(a2)
+; RV64I-NEXT:    and a1, a7, t2
+; RV64I-NEXT:    srli a1, a1, 8
+; RV64I-NEXT:    sb a7, 12(a2)
+; RV64I-NEXT:    sb a1, 13(a2)
+; RV64I-NEXT:    sb t6, 14(a2)
+; RV64I-NEXT:    sb s3, 15(a2)
+; RV64I-NEXT:    and a1, a4, t2
+; RV64I-NEXT:    srli a1, a1, 8
+; RV64I-NEXT:    sb a4, 16(a2)
+; RV64I-NEXT:    sb a1, 17(a2)
+; RV64I-NEXT:    sb s2, 18(a2)
+; RV64I-NEXT:    sb s6, 19(a2)
+; RV64I-NEXT:    and a1, t3, t2
+; RV64I-NEXT:    srli a1, a1, 8
+; RV64I-NEXT:    sb t3, 20(a2)
+; RV64I-NEXT:    sb a1, 21(a2)
+; RV64I-NEXT:    sb s4, 22(a2)
+; RV64I-NEXT:    sb s7, 23(a2)
+; RV64I-NEXT:    and a1, a0, t2
+; RV64I-NEXT:    and a3, s1, t2
+; RV64I-NEXT:    srli a1, a1, 8
+; RV64I-NEXT:    srli a3, a3, 8
+; RV64I-NEXT:    sb a0, 24(a2)
+; RV64I-NEXT:    sb a1, 25(a2)
+; RV64I-NEXT:    sb s5, 26(a2)
+; RV64I-NEXT:    sb s8, 27(a2)
+; RV64I-NEXT:    sb s1, 28(a2)
+; RV64I-NEXT:    sb a3, 29(a2)
+; RV64I-NEXT:    sb s9, 30(a2)
+; RV64I-NEXT:    sb s10, 31(a2)
+; RV64I-NEXT:    ld ra, 104(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s0, 96(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s1, 88(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s2, 80(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s3, 72(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s4, 64(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s5, 56(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s6, 48(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s7, 40(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s8, 32(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s9, 24(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s10, 16(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s11, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    addi sp, sp, 112
+; RV64I-NEXT:    ret
+;
+; RV32I-LABEL: shl_32bytes_dwordOff:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    addi sp, sp, -96
+; RV32I-NEXT:    sw ra, 92(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s0, 88(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s1, 84(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s2, 80(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s3, 76(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s4, 72(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s5, 68(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s6, 64(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s7, 60(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s8, 56(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s9, 52(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s10, 48(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s11, 44(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    lbu a3, 0(a0)
+; RV32I-NEXT:    lbu a4, 1(a0)
+; RV32I-NEXT:    lbu a5, 2(a0)
+; RV32I-NEXT:    lbu a7, 3(a0)
+; RV32I-NEXT:    lbu t0, 4(a0)
+; RV32I-NEXT:    lbu t1, 5(a0)
+; RV32I-NEXT:    lbu t2, 6(a0)
+; RV32I-NEXT:    lbu t3, 7(a0)
+; RV32I-NEXT:    slli a4, a4, 8
+; RV32I-NEXT:    or a4, a4, a3
+; RV32I-NEXT:    lbu a6, 11(a0)
+; RV32I-NEXT:    lbu a3, 15(a0)
+; RV32I-NEXT:    slli a7, a7, 8
+; RV32I-NEXT:    slli t1, t1, 8
+; RV32I-NEXT:    slli t3, t3, 8
+; RV32I-NEXT:    or a5, a7, a5
+; RV32I-NEXT:    or a7, t1, t0
+; RV32I-NEXT:    lbu t0, 0(a1)
+; RV32I-NEXT:    lbu t1, 1(a1)
+; RV32I-NEXT:    or t2, t3, t2
+; RV32I-NEXT:    lbu t3, 2(a1)
+; RV32I-NEXT:    lbu a1, 3(a1)
+; RV32I-NEXT:    slli t1, t1, 8
+; RV32I-NEXT:    or t0, t1, t0
+; RV32I-NEXT:    li s9, 64
+; RV32I-NEXT:    slli a1, a1, 8
+; RV32I-NEXT:    or a1, a1, t3
+; RV32I-NEXT:    li t4, 32
+; RV32I-NEXT:    slli a5, a5, 16
+; RV32I-NEXT:    slli t2, t2, 16
+; RV32I-NEXT:    slli a1, a1, 16
+; RV32I-NEXT:    or t3, a5, a4
+; RV32I-NEXT:    or a5, t2, a7
+; RV32I-NEXT:    or a4, a1, t0
+; RV32I-NEXT:    slli a4, a4, 6
+; RV32I-NEXT:    neg s10, a4
+; RV32I-NEXT:    srl t5, t3, s10
+; RV32I-NEXT:    sll s5, a5, a4
+; RV32I-NEXT:    bltu a4, t4, .LBB17_2
+; RV32I-NEXT:  # %bb.1:
+; RV32I-NEXT:    li s8, 0
+; RV32I-NEXT:    sll a7, t3, a4
+; RV32I-NEXT:    j .LBB17_3
+; RV32I-NEXT:  .LBB17_2:
+; RV32I-NEXT:    sll s8, t3, a4
+; RV32I-NEXT:    or a7, t5, s5
+; RV32I-NEXT:  .LBB17_3:
+; RV32I-NEXT:    lbu t2, 9(a0)
+; RV32I-NEXT:    lbu a1, 10(a0)
+; RV32I-NEXT:    lbu t1, 13(a0)
+; RV32I-NEXT:    lbu t0, 14(a0)
+; RV32I-NEXT:    slli a6, a6, 8
+; RV32I-NEXT:    slli t6, a3, 8
+; RV32I-NEXT:    sub s6, s9, a4
+; RV32I-NEXT:    mv a3, a5
+; RV32I-NEXT:    beqz a4, .LBB17_5
+; RV32I-NEXT:  # %bb.4:
+; RV32I-NEXT:    mv a3, a7
+; RV32I-NEXT:  .LBB17_5:
+; RV32I-NEXT:    slli a7, t2, 8
+; RV32I-NEXT:    or a6, a6, a1
+; RV32I-NEXT:    lbu t2, 8(a0)
+; RV32I-NEXT:    lbu a1, 12(a0)
+; RV32I-NEXT:    slli t1, t1, 8
+; RV32I-NEXT:    or t0, t6, t0
+; RV32I-NEXT:    neg t6, s6
+; RV32I-NEXT:    sw t6, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    bltu s6, t4, .LBB17_7
+; RV32I-NEXT:  # %bb.6:
+; RV32I-NEXT:    srl t6, a5, s6
+; RV32I-NEXT:    j .LBB17_8
+; RV32I-NEXT:  .LBB17_7:
+; RV32I-NEXT:    sll t6, a5, t6
+; RV32I-NEXT:    or t6, t5, t6
+; RV32I-NEXT:  .LBB17_8:
+; RV32I-NEXT:    or a7, a7, t2
+; RV32I-NEXT:    slli t2, a6, 16
+; RV32I-NEXT:    or a1, t1, a1
+; RV32I-NEXT:    slli t0, t0, 16
+; RV32I-NEXT:    mv a6, t3
+; RV32I-NEXT:    beqz s6, .LBB17_10
+; RV32I-NEXT:  # %bb.9:
+; RV32I-NEXT:    mv a6, t6
+; RV32I-NEXT:  .LBB17_10:
+; RV32I-NEXT:    or t1, t2, a7
+; RV32I-NEXT:    or t2, t0, a1
+; RV32I-NEXT:    bltu s6, t4, .LBB17_12
+; RV32I-NEXT:  # %bb.11:
+; RV32I-NEXT:    li a7, 0
+; RV32I-NEXT:    j .LBB17_13
+; RV32I-NEXT:  .LBB17_12:
+; RV32I-NEXT:    srl a7, a5, s10
+; RV32I-NEXT:  .LBB17_13:
+; RV32I-NEXT:    srl s0, t1, s10
+; RV32I-NEXT:    sll a1, t2, a4
+; RV32I-NEXT:    sw a1, 20(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    bltu a4, t4, .LBB17_15
+; RV32I-NEXT:  # %bb.14:
+; RV32I-NEXT:    li s1, 0
+; RV32I-NEXT:    sll a1, t1, a4
+; RV32I-NEXT:    j .LBB17_16
+; RV32I-NEXT:  .LBB17_15:
+; RV32I-NEXT:    sll s1, t1, a4
+; RV32I-NEXT:    or a1, s0, a1
+; RV32I-NEXT:  .LBB17_16:
+; RV32I-NEXT:    addi s7, a4, -64
+; RV32I-NEXT:    mv s3, t2
+; RV32I-NEXT:    beqz a4, .LBB17_18
+; RV32I-NEXT:  # %bb.17:
+; RV32I-NEXT:    mv s3, a1
+; RV32I-NEXT:  .LBB17_18:
+; RV32I-NEXT:    neg a1, s7
+; RV32I-NEXT:    sw a1, 16(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    bltu s7, t4, .LBB17_20
+; RV32I-NEXT:  # %bb.19:
+; RV32I-NEXT:    li s2, 0
+; RV32I-NEXT:    sll a1, t3, s7
+; RV32I-NEXT:    mv s4, a5
+; RV32I-NEXT:    bnez s7, .LBB17_21
+; RV32I-NEXT:    j .LBB17_22
+; RV32I-NEXT:  .LBB17_20:
+; RV32I-NEXT:    sll s2, t3, a4
+; RV32I-NEXT:    srl a1, t3, a1
+; RV32I-NEXT:    or a1, a1, s5
+; RV32I-NEXT:    mv s4, a5
+; RV32I-NEXT:    beqz s7, .LBB17_22
+; RV32I-NEXT:  .LBB17_21:
+; RV32I-NEXT:    mv s4, a1
+; RV32I-NEXT:  .LBB17_22:
+; RV32I-NEXT:    li a1, 128
+; RV32I-NEXT:    bltu a4, s9, .LBB17_24
+; RV32I-NEXT:  # %bb.23:
+; RV32I-NEXT:    sw zero, 28(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    li a3, 0
+; RV32I-NEXT:    j .LBB17_25
+; RV32I-NEXT:  .LBB17_24:
+; RV32I-NEXT:    sw s8, 28(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    or s2, a6, s1
+; RV32I-NEXT:    or s4, a7, s3
+; RV32I-NEXT:  .LBB17_25:
+; RV32I-NEXT:    sub ra, a1, a4
+; RV32I-NEXT:    mv a7, t1
+; RV32I-NEXT:    mv a6, t2
+; RV32I-NEXT:    beqz a4, .LBB17_27
+; RV32I-NEXT:  # %bb.26:
+; RV32I-NEXT:    mv a7, s2
+; RV32I-NEXT:    mv a6, s4
+; RV32I-NEXT:  .LBB17_27:
+; RV32I-NEXT:    neg s1, ra
+; RV32I-NEXT:    sll s2, t2, s1
+; RV32I-NEXT:    bltu ra, t4, .LBB17_29
+; RV32I-NEXT:  # %bb.28:
+; RV32I-NEXT:    srl a1, t2, ra
+; RV32I-NEXT:    sw t1, 40(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    bnez ra, .LBB17_30
+; RV32I-NEXT:    j .LBB17_31
+; RV32I-NEXT:  .LBB17_29:
+; RV32I-NEXT:    or a1, s0, s2
+; RV32I-NEXT:    sw t1, 40(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    beqz ra, .LBB17_31
+; RV32I-NEXT:  .LBB17_30:
+; RV32I-NEXT:    sw a1, 40(sp) # 4-byte Folded Spill
+; RV32I-NEXT:  .LBB17_31:
+; RV32I-NEXT:    bltu ra, t4, .LBB17_33
+; RV32I-NEXT:  # %bb.32:
+; RV32I-NEXT:    sw zero, 36(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    srl a1, a5, ra
+; RV32I-NEXT:    mv t5, t3
+; RV32I-NEXT:    bnez ra, .LBB17_34
+; RV32I-NEXT:    j .LBB17_35
+; RV32I-NEXT:  .LBB17_33:
+; RV32I-NEXT:    srl a1, t2, s10
+; RV32I-NEXT:    sw a1, 36(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sll a1, a5, s1
+; RV32I-NEXT:    or a1, t5, a1
+; RV32I-NEXT:    mv t5, t3
+; RV32I-NEXT:    beqz ra, .LBB17_35
+; RV32I-NEXT:  .LBB17_34:
+; RV32I-NEXT:    mv t5, a1
+; RV32I-NEXT:  .LBB17_35:
+; RV32I-NEXT:    sub s3, s9, ra
+; RV32I-NEXT:    bltu ra, t4, .LBB17_38
+; RV32I-NEXT:  # %bb.36:
+; RV32I-NEXT:    sw zero, 32(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    bgeu s3, t4, .LBB17_39
+; RV32I-NEXT:  .LBB17_37:
+; RV32I-NEXT:    sll s1, t1, s1
+; RV32I-NEXT:    neg a1, s3
+; RV32I-NEXT:    srl a1, t1, a1
+; RV32I-NEXT:    or a1, a1, s2
+; RV32I-NEXT:    j .LBB17_40
+; RV32I-NEXT:  .LBB17_38:
+; RV32I-NEXT:    srl a1, a5, s10
+; RV32I-NEXT:    sw a1, 32(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    bltu s3, t4, .LBB17_37
+; RV32I-NEXT:  .LBB17_39:
+; RV32I-NEXT:    li s1, 0
+; RV32I-NEXT:    sll a1, t1, s3
+; RV32I-NEXT:  .LBB17_40:
+; RV32I-NEXT:    addi s4, ra, -64
+; RV32I-NEXT:    mv s2, t2
+; RV32I-NEXT:    beqz s3, .LBB17_42
+; RV32I-NEXT:  # %bb.41:
+; RV32I-NEXT:    mv s2, a1
+; RV32I-NEXT:  .LBB17_42:
+; RV32I-NEXT:    sw s5, 24(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    mv s5, a7
+; RV32I-NEXT:    bltu s4, t4, .LBB17_44
+; RV32I-NEXT:  # %bb.43:
+; RV32I-NEXT:    srl t0, t2, s4
+; RV32I-NEXT:    j .LBB17_45
+; RV32I-NEXT:  .LBB17_44:
+; RV32I-NEXT:    srl a1, t1, ra
+; RV32I-NEXT:    neg t0, s4
+; RV32I-NEXT:    sll t0, t2, t0
+; RV32I-NEXT:    or t0, a1, t0
+; RV32I-NEXT:  .LBB17_45:
+; RV32I-NEXT:    mv s0, s10
+; RV32I-NEXT:    mv a7, a6
+; RV32I-NEXT:    lbu s8, 19(a0)
+; RV32I-NEXT:    lbu a1, 23(a0)
+; RV32I-NEXT:    mv s3, t1
+; RV32I-NEXT:    beqz s4, .LBB17_47
+; RV32I-NEXT:  # %bb.46:
+; RV32I-NEXT:    mv s3, t0
+; RV32I-NEXT:  .LBB17_47:
+; RV32I-NEXT:    mv a6, a3
+; RV32I-NEXT:    lbu s10, 17(a0)
+; RV32I-NEXT:    lbu t0, 18(a0)
+; RV32I-NEXT:    lbu s9, 21(a0)
+; RV32I-NEXT:    lbu t6, 22(a0)
+; RV32I-NEXT:    slli s8, s8, 8
+; RV32I-NEXT:    slli a1, a1, 8
+; RV32I-NEXT:    li a3, 64
+; RV32I-NEXT:    bltu s4, t4, .LBB17_49
+; RV32I-NEXT:  # %bb.48:
+; RV32I-NEXT:    li s4, 0
+; RV32I-NEXT:    j .LBB17_50
+; RV32I-NEXT:  .LBB17_49:
+; RV32I-NEXT:    srl s4, t2, ra
+; RV32I-NEXT:  .LBB17_50:
+; RV32I-NEXT:    or s11, s8, t0
+; RV32I-NEXT:    lbu t0, 16(a0)
+; RV32I-NEXT:    lbu s8, 20(a0)
+; RV32I-NEXT:    slli s10, s10, 8
+; RV32I-NEXT:    slli s9, s9, 8
+; RV32I-NEXT:    or t6, a1, t6
+; RV32I-NEXT:    bgeu ra, a3, .LBB17_52
+; RV32I-NEXT:  # %bb.51:
+; RV32I-NEXT:    or s3, t5, s1
+; RV32I-NEXT:    lw a1, 32(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    or s4, a1, s2
+; RV32I-NEXT:  .LBB17_52:
+; RV32I-NEXT:    or a1, s10, t0
+; RV32I-NEXT:    slli s11, s11, 16
+; RV32I-NEXT:    or t0, s9, s8
+; RV32I-NEXT:    slli t6, t6, 16
+; RV32I-NEXT:    mv t5, t3
+; RV32I-NEXT:    mv s1, a5
+; RV32I-NEXT:    mv a3, a6
+; RV32I-NEXT:    beqz ra, .LBB17_54
+; RV32I-NEXT:  # %bb.53:
+; RV32I-NEXT:    mv t5, s3
+; RV32I-NEXT:    mv s1, s4
+; RV32I-NEXT:  .LBB17_54:
+; RV32I-NEXT:    sw s1, 4(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    or s2, s11, a1
+; RV32I-NEXT:    or s1, t6, t0
+; RV32I-NEXT:    li a1, 64
+; RV32I-NEXT:    mv a6, a7
+; RV32I-NEXT:    mv a7, s0
+; RV32I-NEXT:    bltu ra, a1, .LBB17_56
+; RV32I-NEXT:  # %bb.55:
+; RV32I-NEXT:    sw zero, 40(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw zero, 36(sp) # 4-byte Folded Spill
+; RV32I-NEXT:  .LBB17_56:
+; RV32I-NEXT:    srl s3, s2, a7
+; RV32I-NEXT:    sll ra, s1, a4
+; RV32I-NEXT:    mv a7, s5
+; RV32I-NEXT:    sw t5, 8(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    bltu a4, t4, .LBB17_58
+; RV32I-NEXT:  # %bb.57:
+; RV32I-NEXT:    sw zero, 32(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sll a1, s2, a4
+; RV32I-NEXT:    j .LBB17_59
+; RV32I-NEXT:  .LBB17_58:
+; RV32I-NEXT:    sll a1, s2, a4
+; RV32I-NEXT:    sw a1, 32(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    or a1, s3, ra
+; RV32I-NEXT:  .LBB17_59:
+; RV32I-NEXT:    lbu s9, 27(a0)
+; RV32I-NEXT:    lbu t6, 31(a0)
+; RV32I-NEXT:    mv t5, s1
+; RV32I-NEXT:    beqz a4, .LBB17_61
+; RV32I-NEXT:  # %bb.60:
+; RV32I-NEXT:    mv t5, a1
+; RV32I-NEXT:  .LBB17_61:
+; RV32I-NEXT:    lbu s8, 25(a0)
+; RV32I-NEXT:    lbu s4, 26(a0)
+; RV32I-NEXT:    lbu s11, 29(a0)
+; RV32I-NEXT:    lbu s10, 30(a0)
+; RV32I-NEXT:    slli s9, s9, 8
+; RV32I-NEXT:    slli t6, t6, 8
+; RV32I-NEXT:    bltu s6, t4, .LBB17_63
+; RV32I-NEXT:  # %bb.62:
+; RV32I-NEXT:    srl t0, s1, s6
+; RV32I-NEXT:    j .LBB17_64
+; RV32I-NEXT:  .LBB17_63:
+; RV32I-NEXT:    lw a1, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    sll a1, s1, a1
+; RV32I-NEXT:    or t0, s3, a1
+; RV32I-NEXT:  .LBB17_64:
+; RV32I-NEXT:    slli s8, s8, 8
+; RV32I-NEXT:    lbu s3, 24(a0)
+; RV32I-NEXT:    lbu a1, 28(a0)
+; RV32I-NEXT:    or s4, s9, s4
+; RV32I-NEXT:    slli s11, s11, 8
+; RV32I-NEXT:    or t6, t6, s10
+; RV32I-NEXT:    mv s9, s2
+; RV32I-NEXT:    beqz s6, .LBB17_66
+; RV32I-NEXT:  # %bb.65:
+; RV32I-NEXT:    mv s9, t0
+; RV32I-NEXT:  .LBB17_66:
+; RV32I-NEXT:    or a0, s8, s3
+; RV32I-NEXT:    slli t0, s4, 16
+; RV32I-NEXT:    or a1, s11, a1
+; RV32I-NEXT:    slli t6, t6, 16
+; RV32I-NEXT:    bltu s6, t4, .LBB17_68
+; RV32I-NEXT:  # %bb.67:
+; RV32I-NEXT:    li s4, 0
+; RV32I-NEXT:    j .LBB17_69
+; RV32I-NEXT:  .LBB17_68:
+; RV32I-NEXT:    srl s4, s1, s0
+; RV32I-NEXT:  .LBB17_69:
+; RV32I-NEXT:    li s11, 64
+; RV32I-NEXT:    or s6, t0, a0
+; RV32I-NEXT:    or a0, t6, a1
+; RV32I-NEXT:    bltu a4, t4, .LBB17_71
+; RV32I-NEXT:  # %bb.70:
+; RV32I-NEXT:    li s3, 0
+; RV32I-NEXT:    sll a1, s6, a4
+; RV32I-NEXT:    mv s10, a0
+; RV32I-NEXT:    bnez a4, .LBB17_72
+; RV32I-NEXT:    j .LBB17_73
+; RV32I-NEXT:  .LBB17_71:
+; RV32I-NEXT:    sll s3, s6, a4
+; RV32I-NEXT:    srl a1, s6, s0
+; RV32I-NEXT:    sll t0, a0, a4
+; RV32I-NEXT:    or a1, a1, t0
+; RV32I-NEXT:    mv s10, a0
+; RV32I-NEXT:    beqz a4, .LBB17_73
+; RV32I-NEXT:  .LBB17_72:
+; RV32I-NEXT:    mv s10, a1
+; RV32I-NEXT:  .LBB17_73:
+; RV32I-NEXT:    bltu s7, t4, .LBB17_75
+; RV32I-NEXT:  # %bb.74:
+; RV32I-NEXT:    li s5, 0
+; RV32I-NEXT:    sll a1, s2, s7
+; RV32I-NEXT:    mv s0, s1
+; RV32I-NEXT:    bnez s7, .LBB17_76
+; RV32I-NEXT:    j .LBB17_77
+; RV32I-NEXT:  .LBB17_75:
+; RV32I-NEXT:    sll s5, s2, a4
+; RV32I-NEXT:    lw a1, 16(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    srl a1, s2, a1
+; RV32I-NEXT:    or a1, a1, ra
+; RV32I-NEXT:    mv s0, s1
+; RV32I-NEXT:    beqz s7, .LBB17_77
+; RV32I-NEXT:  .LBB17_76:
+; RV32I-NEXT:    mv s0, a1
+; RV32I-NEXT:  .LBB17_77:
+; RV32I-NEXT:    bltu a4, s11, .LBB17_79
+; RV32I-NEXT:  # %bb.78:
+; RV32I-NEXT:    sw zero, 32(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    li t5, 0
+; RV32I-NEXT:    j .LBB17_80
+; RV32I-NEXT:  .LBB17_79:
+; RV32I-NEXT:    or s5, s9, s3
+; RV32I-NEXT:    or s0, s4, s10
+; RV32I-NEXT:  .LBB17_80:
+; RV32I-NEXT:    addi s9, a4, -128
+; RV32I-NEXT:    mv s7, s6
+; RV32I-NEXT:    mv s8, a0
+; RV32I-NEXT:    beqz a4, .LBB17_82
+; RV32I-NEXT:  # %bb.81:
+; RV32I-NEXT:    mv s7, s5
+; RV32I-NEXT:    mv s8, s0
+; RV32I-NEXT:  .LBB17_82:
+; RV32I-NEXT:    neg s3, s9
+; RV32I-NEXT:    srl s0, t3, s3
+; RV32I-NEXT:    bltu s9, t4, .LBB17_84
+; RV32I-NEXT:  # %bb.83:
+; RV32I-NEXT:    li s5, 0
+; RV32I-NEXT:    sll a1, t3, s9
+; RV32I-NEXT:    j .LBB17_85
+; RV32I-NEXT:  .LBB17_84:
+; RV32I-NEXT:    sll s5, t3, a4
+; RV32I-NEXT:    lw a1, 24(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    or a1, s0, a1
+; RV32I-NEXT:  .LBB17_85:
+; RV32I-NEXT:    sub s4, s11, s9
+; RV32I-NEXT:    mv t6, a5
+; RV32I-NEXT:    beqz s9, .LBB17_87
+; RV32I-NEXT:  # %bb.86:
+; RV32I-NEXT:    mv t6, a1
+; RV32I-NEXT:  .LBB17_87:
+; RV32I-NEXT:    bltu s4, t4, .LBB17_89
+; RV32I-NEXT:  # %bb.88:
+; RV32I-NEXT:    srl a1, a5, s4
+; RV32I-NEXT:    mv s0, t3
+; RV32I-NEXT:    bnez s4, .LBB17_90
+; RV32I-NEXT:    j .LBB17_91
+; RV32I-NEXT:  .LBB17_89:
+; RV32I-NEXT:    neg a1, s4
+; RV32I-NEXT:    sll a1, a5, a1
+; RV32I-NEXT:    or a1, s0, a1
+; RV32I-NEXT:    mv s0, t3
+; RV32I-NEXT:    beqz s4, .LBB17_91
+; RV32I-NEXT:  .LBB17_90:
+; RV32I-NEXT:    mv s0, a1
+; RV32I-NEXT:  .LBB17_91:
+; RV32I-NEXT:    bltu s4, t4, .LBB17_94
+; RV32I-NEXT:  # %bb.92:
+; RV32I-NEXT:    li s4, 0
+; RV32I-NEXT:    li ra, 64
+; RV32I-NEXT:    bgeu s9, t4, .LBB17_95
+; RV32I-NEXT:  .LBB17_93:
+; RV32I-NEXT:    sll s10, t1, a4
+; RV32I-NEXT:    srl a1, t1, s3
+; RV32I-NEXT:    lw t0, 20(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    or a1, a1, t0
+; RV32I-NEXT:    j .LBB17_96
+; RV32I-NEXT:  .LBB17_94:
+; RV32I-NEXT:    srl s4, a5, s3
+; RV32I-NEXT:    li ra, 64
+; RV32I-NEXT:    bltu s9, t4, .LBB17_93
+; RV32I-NEXT:  .LBB17_95:
+; RV32I-NEXT:    li s10, 0
+; RV32I-NEXT:    sll a1, t1, s9
+; RV32I-NEXT:  .LBB17_96:
+; RV32I-NEXT:    addi s11, s9, -64
+; RV32I-NEXT:    mv s3, t2
+; RV32I-NEXT:    beqz s9, .LBB17_98
+; RV32I-NEXT:  # %bb.97:
+; RV32I-NEXT:    mv s3, a1
+; RV32I-NEXT:  .LBB17_98:
+; RV32I-NEXT:    bltu s11, t4, .LBB17_100
+; RV32I-NEXT:  # %bb.99:
+; RV32I-NEXT:    li t4, 0
+; RV32I-NEXT:    sll a1, t3, s11
+; RV32I-NEXT:    bnez s11, .LBB17_101
+; RV32I-NEXT:    j .LBB17_102
+; RV32I-NEXT:  .LBB17_100:
+; RV32I-NEXT:    sll t4, t3, s9
+; RV32I-NEXT:    neg a1, s11
+; RV32I-NEXT:    srl a1, t3, a1
+; RV32I-NEXT:    sll t0, a5, s9
+; RV32I-NEXT:    or a1, a1, t0
+; RV32I-NEXT:    beqz s11, .LBB17_102
+; RV32I-NEXT:  .LBB17_101:
+; RV32I-NEXT:    mv a5, a1
+; RV32I-NEXT:  .LBB17_102:
+; RV32I-NEXT:    bltu s9, ra, .LBB17_104
+; RV32I-NEXT:  # %bb.103:
+; RV32I-NEXT:    li s5, 0
+; RV32I-NEXT:    li t6, 0
+; RV32I-NEXT:    li a1, 128
+; RV32I-NEXT:    bnez s9, .LBB17_105
+; RV32I-NEXT:    j .LBB17_106
+; RV32I-NEXT:  .LBB17_104:
+; RV32I-NEXT:    or t4, s0, s10
+; RV32I-NEXT:    or a5, s4, s3
+; RV32I-NEXT:    li a1, 128
+; RV32I-NEXT:    beqz s9, .LBB17_106
+; RV32I-NEXT:  .LBB17_105:
+; RV32I-NEXT:    mv t1, t4
+; RV32I-NEXT:    mv t2, a5
+; RV32I-NEXT:  .LBB17_106:
+; RV32I-NEXT:    bltu a4, a1, .LBB17_108
+; RV32I-NEXT:  # %bb.107:
+; RV32I-NEXT:    li ra, 0
+; RV32I-NEXT:    li a3, 0
+; RV32I-NEXT:    li a7, 0
+; RV32I-NEXT:    li a6, 0
+; RV32I-NEXT:    bnez a4, .LBB17_109
+; RV32I-NEXT:    j .LBB17_110
+; RV32I-NEXT:  .LBB17_108:
+; RV32I-NEXT:    lw a1, 8(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw a5, 32(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    or s5, a1, a5
+; RV32I-NEXT:    lw a1, 4(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    or t6, a1, t5
+; RV32I-NEXT:    lw a1, 40(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    or t1, a1, s7
+; RV32I-NEXT:    lw a1, 36(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    or t2, a1, s8
+; RV32I-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    beqz a4, .LBB17_110
+; RV32I-NEXT:  .LBB17_109:
+; RV32I-NEXT:    mv s2, s5
+; RV32I-NEXT:    mv s1, t6
+; RV32I-NEXT:    mv s6, t1
+; RV32I-NEXT:    mv a0, t2
+; RV32I-NEXT:  .LBB17_110:
+; RV32I-NEXT:    srli a4, ra, 16
+; RV32I-NEXT:    lui t2, 16
+; RV32I-NEXT:    srli t1, ra, 24
+; RV32I-NEXT:    srli a5, a3, 16
+; RV32I-NEXT:    srli t4, a3, 24
+; RV32I-NEXT:    srli t0, a7, 16
+; RV32I-NEXT:    srli s0, a7, 24
+; RV32I-NEXT:    srli t3, a6, 16
+; RV32I-NEXT:    srli s3, a6, 24
+; RV32I-NEXT:    srli t6, s2, 16
+; RV32I-NEXT:    srli a1, s2, 24
+; RV32I-NEXT:    srli t5, s1, 16
+; RV32I-NEXT:    srli s5, s1, 24
+; RV32I-NEXT:    srli s4, s6, 16
+; RV32I-NEXT:    srli s7, s6, 24
+; RV32I-NEXT:    srli s8, a0, 16
+; RV32I-NEXT:    srli s9, a0, 24
+; RV32I-NEXT:    addi t2, t2, -1
+; RV32I-NEXT:    and s10, ra, t2
+; RV32I-NEXT:    and s11, a3, t2
+; RV32I-NEXT:    srli s10, s10, 8
+; RV32I-NEXT:    sb ra, 0(a2)
+; RV32I-NEXT:    sb s10, 1(a2)
+; RV32I-NEXT:    sb a4, 2(a2)
+; RV32I-NEXT:    sb t1, 3(a2)
+; RV32I-NEXT:    and a4, a7, t2
+; RV32I-NEXT:    srli t1, s11, 8
+; RV32I-NEXT:    sb a3, 4(a2)
+; RV32I-NEXT:    sb t1, 5(a2)
+; RV32I-NEXT:    sb a5, 6(a2)
+; RV32I-NEXT:    sb t4, 7(a2)
+; RV32I-NEXT:    and a3, a6, t2
+; RV32I-NEXT:    srli a4, a4, 8
+; RV32I-NEXT:    sb a7, 8(a2)
+; RV32I-NEXT:    sb a4, 9(a2)
+; RV32I-NEXT:    sb t0, 10(a2)
+; RV32I-NEXT:    sb s0, 11(a2)
+; RV32I-NEXT:    and a4, s2, t2
+; RV32I-NEXT:    srli a3, a3, 8
+; RV32I-NEXT:    sb a6, 12(a2)
+; RV32I-NEXT:    sb a3, 13(a2)
+; RV32I-NEXT:    sb t3, 14(a2)
+; RV32I-NEXT:    sb s3, 15(a2)
+; RV32I-NEXT:    and a3, s1, t2
+; RV32I-NEXT:    srli a4, a4, 8
+; RV32I-NEXT:    sb s2, 16(a2)
+; RV32I-NEXT:    sb a4, 17(a2)
+; RV32I-NEXT:    sb t6, 18(a2)
+; RV32I-NEXT:    sb a1, 19(a2)
+; RV32I-NEXT:    and a1, s6, t2
+; RV32I-NEXT:    and a4, a0, t2
+; RV32I-NEXT:    srli a3, a3, 8
+; RV32I-NEXT:    srli a1, a1, 8
+; RV32I-NEXT:    srli a4, a4, 8
+; RV32I-NEXT:    sb s1, 20(a2)
+; RV32I-NEXT:    sb a3, 21(a2)
+; RV32I-NEXT:    sb t5, 22(a2)
+; RV32I-NEXT:    sb s5, 23(a2)
+; RV32I-NEXT:    sb s6, 24(a2)
+; RV32I-NEXT:    sb a1, 25(a2)
+; RV32I-NEXT:    sb s4, 26(a2)
+; RV32I-NEXT:    sb s7, 27(a2)
+; RV32I-NEXT:    sb a0, 28(a2)
+; RV32I-NEXT:    sb a4, 29(a2)
+; RV32I-NEXT:    sb s8, 30(a2)
+; RV32I-NEXT:    sb s9, 31(a2)
+; RV32I-NEXT:    lw ra, 92(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s0, 88(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s1, 84(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s2, 80(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s3, 76(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s4, 72(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s5, 68(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s6, 64(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s7, 60(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s8, 56(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s9, 52(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s10, 48(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s11, 44(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    addi sp, sp, 96
+; RV32I-NEXT:    ret
+  %src = load i256, ptr %src.ptr, align 1
+  %dwordOff = load i256, ptr %dwordOff.ptr, align 1
+  %bitOff = shl i256 %dwordOff, 6
+  %res = shl i256 %src, %bitOff
+  store i256 %res, ptr %dst, align 1
+  ret void
+}
+
+define void @ashr_32bytes(ptr %src.ptr, ptr %byteOff.ptr, ptr %dst) nounwind {
+; RV64I-LABEL: ashr_32bytes:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -96
+; RV64I-NEXT:    sd s0, 88(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s1, 80(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s2, 72(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s3, 64(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s4, 56(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s5, 48(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s6, 40(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s7, 32(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s8, 24(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s9, 16(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s10, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s11, 0(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    lbu a3, 0(a0)
+; RV64I-NEXT:    lbu a4, 1(a0)
+; RV64I-NEXT:    lbu a5, 2(a0)
+; RV64I-NEXT:    lbu a6, 3(a0)
+; RV64I-NEXT:    lbu a7, 4(a0)
+; RV64I-NEXT:    lbu t0, 5(a0)
+; RV64I-NEXT:    lbu t1, 6(a0)
+; RV64I-NEXT:    lbu t2, 7(a0)
+; RV64I-NEXT:    lbu t3, 8(a0)
+; RV64I-NEXT:    lbu t4, 9(a0)
+; RV64I-NEXT:    lbu t5, 10(a0)
+; RV64I-NEXT:    lbu t6, 11(a0)
+; RV64I-NEXT:    lbu s0, 12(a0)
+; RV64I-NEXT:    lbu s1, 13(a0)
+; RV64I-NEXT:    lbu s2, 14(a0)
+; RV64I-NEXT:    lbu s3, 15(a0)
+; RV64I-NEXT:    lbu s4, 16(a0)
+; RV64I-NEXT:    lbu s5, 17(a0)
+; RV64I-NEXT:    lbu s6, 18(a0)
+; RV64I-NEXT:    lbu s7, 19(a0)
+; RV64I-NEXT:    slli a4, a4, 8
+; RV64I-NEXT:    slli a6, a6, 8
+; RV64I-NEXT:    slli t0, t0, 8
+; RV64I-NEXT:    or a3, a4, a3
+; RV64I-NEXT:    or a5, a6, a5
+; RV64I-NEXT:    or a7, t0, a7
+; RV64I-NEXT:    lbu s8, 20(a0)
+; RV64I-NEXT:    lbu s9, 21(a0)
+; RV64I-NEXT:    lbu s10, 22(a0)
+; RV64I-NEXT:    lbu s11, 23(a0)
+; RV64I-NEXT:    slli t2, t2, 8
+; RV64I-NEXT:    slli t4, t4, 8
+; RV64I-NEXT:    slli t6, t6, 8
+; RV64I-NEXT:    slli s1, s1, 8
+; RV64I-NEXT:    or t1, t2, t1
+; RV64I-NEXT:    or a4, t4, t3
+; RV64I-NEXT:    or a6, t6, t5
+; RV64I-NEXT:    or t0, s1, s0
+; RV64I-NEXT:    lbu t5, 24(a0)
+; RV64I-NEXT:    lbu t6, 25(a0)
+; RV64I-NEXT:    lbu s0, 26(a0)
+; RV64I-NEXT:    lbu s1, 27(a0)
+; RV64I-NEXT:    slli s3, s3, 8
+; RV64I-NEXT:    slli s5, s5, 8
+; RV64I-NEXT:    slli s7, s7, 8
+; RV64I-NEXT:    or t4, s3, s2
+; RV64I-NEXT:    or t2, s5, s4
+; RV64I-NEXT:    or t3, s7, s6
+; RV64I-NEXT:    lbu s2, 28(a0)
+; RV64I-NEXT:    lbu s3, 29(a0)
+; RV64I-NEXT:    lbu s4, 30(a0)
+; RV64I-NEXT:    lbu a0, 31(a0)
+; RV64I-NEXT:    slli s9, s9, 8
+; RV64I-NEXT:    slli s11, s11, 8
+; RV64I-NEXT:    slli t6, t6, 8
+; RV64I-NEXT:    slli s1, s1, 8
+; RV64I-NEXT:    or s5, s9, s8
+; RV64I-NEXT:    or s6, s11, s10
+; RV64I-NEXT:    or t5, t6, t5
+; RV64I-NEXT:    or s0, s1, s0
+; RV64I-NEXT:    lbu t6, 0(a1)
+; RV64I-NEXT:    lbu s1, 1(a1)
+; RV64I-NEXT:    lbu s7, 2(a1)
+; RV64I-NEXT:    lbu s8, 3(a1)
+; RV64I-NEXT:    slli s3, s3, 8
+; RV64I-NEXT:    slli a0, a0, 8
+; RV64I-NEXT:    slli s1, s1, 8
+; RV64I-NEXT:    or s2, s3, s2
+; RV64I-NEXT:    or s3, a0, s4
+; RV64I-NEXT:    or t6, s1, t6
+; RV64I-NEXT:    lbu a0, 4(a1)
+; RV64I-NEXT:    lbu s1, 5(a1)
+; RV64I-NEXT:    lbu s4, 6(a1)
+; RV64I-NEXT:    lbu a1, 7(a1)
+; RV64I-NEXT:    slli s8, s8, 8
+; RV64I-NEXT:    or s7, s8, s7
+; RV64I-NEXT:    slli s1, s1, 8
+; RV64I-NEXT:    or s1, s1, a0
+; RV64I-NEXT:    slli a1, a1, 8
+; RV64I-NEXT:    or s4, a1, s4
+; RV64I-NEXT:    slli t1, t1, 16
+; RV64I-NEXT:    or a1, t1, a7
+; RV64I-NEXT:    slli t4, t4, 16
+; RV64I-NEXT:    or a0, t4, t0
+; RV64I-NEXT:    slli t3, t3, 16
+; RV64I-NEXT:    or a7, t3, t2
+; RV64I-NEXT:    slli s6, s6, 16
+; RV64I-NEXT:    or t1, s6, s5
+; RV64I-NEXT:    slli s0, s0, 16
+; RV64I-NEXT:    or t4, s0, t5
+; RV64I-NEXT:    slli s3, s3, 16
+; RV64I-NEXT:    or t5, s3, s2
+; RV64I-NEXT:    slli s7, s7, 16
+; RV64I-NEXT:    or t6, s7, t6
+; RV64I-NEXT:    slli s4, s4, 16
+; RV64I-NEXT:    or s0, s4, s1
+; RV64I-NEXT:    li t0, 64
+; RV64I-NEXT:    slli t3, a5, 16
+; RV64I-NEXT:    slli t2, a6, 16
+; RV64I-NEXT:    slli t1, t1, 32
+; RV64I-NEXT:    slli t5, t5, 32
+; RV64I-NEXT:    slli s0, s0, 32
+; RV64I-NEXT:    or a7, t1, a7
+; RV64I-NEXT:    or a5, t5, t4
+; RV64I-NEXT:    or a6, s0, t6
+; RV64I-NEXT:    slli a6, a6, 3
+; RV64I-NEXT:    subw t1, a6, t0
+; RV64I-NEXT:    negw t5, a6
+; RV64I-NEXT:    sll t4, a5, t5
+; RV64I-NEXT:    bltu a6, t0, .LBB18_2
+; RV64I-NEXT:  # %bb.1:
+; RV64I-NEXT:    sra t6, a5, t1
+; RV64I-NEXT:    j .LBB18_3
+; RV64I-NEXT:  .LBB18_2:
+; RV64I-NEXT:    srl t6, a7, a6
+; RV64I-NEXT:    or t6, t6, t4
+; RV64I-NEXT:  .LBB18_3:
+; RV64I-NEXT:    or a3, t3, a3
+; RV64I-NEXT:    slli t3, a1, 32
+; RV64I-NEXT:    or t2, t2, a4
+; RV64I-NEXT:    slli a0, a0, 32
+; RV64I-NEXT:    mv a1, a7
+; RV64I-NEXT:    beqz a6, .LBB18_5
+; RV64I-NEXT:  # %bb.4:
+; RV64I-NEXT:    mv a1, t6
+; RV64I-NEXT:  .LBB18_5:
+; RV64I-NEXT:    or a4, t3, a3
+; RV64I-NEXT:    or a3, a0, t2
+; RV64I-NEXT:    bltu a6, t0, .LBB18_7
+; RV64I-NEXT:  # %bb.6:
+; RV64I-NEXT:    srai a0, a5, 63
+; RV64I-NEXT:    srl t3, a3, t1
+; RV64I-NEXT:    j .LBB18_8
+; RV64I-NEXT:  .LBB18_7:
+; RV64I-NEXT:    sra a0, a5, a6
+; RV64I-NEXT:    srl t1, a4, a6
+; RV64I-NEXT:    sll t2, a3, t5
+; RV64I-NEXT:    or t3, t1, t2
+; RV64I-NEXT:  .LBB18_8:
+; RV64I-NEXT:    li t1, 128
+; RV64I-NEXT:    mv t2, a4
+; RV64I-NEXT:    beqz a6, .LBB18_10
+; RV64I-NEXT:  # %bb.9:
+; RV64I-NEXT:    mv t2, t3
+; RV64I-NEXT:  .LBB18_10:
+; RV64I-NEXT:    sub t6, t1, a6
+; RV64I-NEXT:    bltu a6, t0, .LBB18_13
+; RV64I-NEXT:  # %bb.11:
+; RV64I-NEXT:    li t3, 0
+; RV64I-NEXT:    bgeu t6, t0, .LBB18_14
+; RV64I-NEXT:  .LBB18_12:
+; RV64I-NEXT:    sll t5, a7, t5
+; RV64I-NEXT:    negw s0, t6
+; RV64I-NEXT:    srl s0, a7, s0
+; RV64I-NEXT:    or s1, s0, t4
+; RV64I-NEXT:    j .LBB18_15
+; RV64I-NEXT:  .LBB18_13:
+; RV64I-NEXT:    srl t3, a3, a6
+; RV64I-NEXT:    bltu t6, t0, .LBB18_12
+; RV64I-NEXT:  .LBB18_14:
+; RV64I-NEXT:    li t5, 0
+; RV64I-NEXT:    subw t4, t6, t0
+; RV64I-NEXT:    sll s1, a7, t4
+; RV64I-NEXT:  .LBB18_15:
+; RV64I-NEXT:    sub s0, a6, t1
+; RV64I-NEXT:    mv t4, a5
+; RV64I-NEXT:    beqz t6, .LBB18_17
+; RV64I-NEXT:  # %bb.16:
+; RV64I-NEXT:    mv t4, s1
+; RV64I-NEXT:  .LBB18_17:
+; RV64I-NEXT:    bltu s0, t0, .LBB18_19
+; RV64I-NEXT:  # %bb.18:
+; RV64I-NEXT:    subw t6, s0, t0
+; RV64I-NEXT:    sra t6, a5, t6
+; RV64I-NEXT:    bnez s0, .LBB18_20
+; RV64I-NEXT:    j .LBB18_21
+; RV64I-NEXT:  .LBB18_19:
+; RV64I-NEXT:    srl t6, a7, s0
+; RV64I-NEXT:    negw s1, s0
+; RV64I-NEXT:    sll s1, a5, s1
+; RV64I-NEXT:    or t6, t6, s1
+; RV64I-NEXT:    beqz s0, .LBB18_21
+; RV64I-NEXT:  .LBB18_20:
+; RV64I-NEXT:    mv a7, t6
+; RV64I-NEXT:  .LBB18_21:
+; RV64I-NEXT:    bltu s0, t0, .LBB18_23
+; RV64I-NEXT:  # %bb.22:
+; RV64I-NEXT:    srai t0, a5, 63
+; RV64I-NEXT:    bltu a6, t1, .LBB18_24
+; RV64I-NEXT:    j .LBB18_25
+; RV64I-NEXT:  .LBB18_23:
+; RV64I-NEXT:    sra t0, a5, s0
+; RV64I-NEXT:    bgeu a6, t1, .LBB18_25
+; RV64I-NEXT:  .LBB18_24:
+; RV64I-NEXT:    or a7, t2, t5
+; RV64I-NEXT:    or t0, t3, t4
+; RV64I-NEXT:  .LBB18_25:
+; RV64I-NEXT:    bnez a6, .LBB18_29
+; RV64I-NEXT:  # %bb.26:
+; RV64I-NEXT:    bltu a6, t1, .LBB18_28
+; RV64I-NEXT:  .LBB18_27:
+; RV64I-NEXT:    srai a1, a5, 63
+; RV64I-NEXT:    mv a0, a1
+; RV64I-NEXT:  .LBB18_28:
+; RV64I-NEXT:    srli a5, a4, 32
+; RV64I-NEXT:    srliw a6, a4, 16
+; RV64I-NEXT:    lui t2, 16
+; RV64I-NEXT:    srliw t1, a4, 24
+; RV64I-NEXT:    srli t0, a4, 48
+; RV64I-NEXT:    srli t5, a4, 56
+; RV64I-NEXT:    srli a7, a3, 32
+; RV64I-NEXT:    srliw t4, a3, 16
+; RV64I-NEXT:    srliw s0, a3, 24
+; RV64I-NEXT:    srli t6, a3, 48
+; RV64I-NEXT:    srli s3, a3, 56
+; RV64I-NEXT:    srli t3, a1, 32
+; RV64I-NEXT:    srliw s2, a1, 16
+; RV64I-NEXT:    srliw s6, a1, 24
+; RV64I-NEXT:    srli s4, a1, 48
+; RV64I-NEXT:    srli s7, a1, 56
+; RV64I-NEXT:    srli s1, a0, 32
+; RV64I-NEXT:    srliw s5, a0, 16
+; RV64I-NEXT:    srliw s8, a0, 24
+; RV64I-NEXT:    srli s9, a0, 48
+; RV64I-NEXT:    srli s10, a0, 56
+; RV64I-NEXT:    addi t2, t2, -1
+; RV64I-NEXT:    and s11, a4, t2
+; RV64I-NEXT:    srli s11, s11, 8
+; RV64I-NEXT:    sb a4, 0(a2)
+; RV64I-NEXT:    sb s11, 1(a2)
+; RV64I-NEXT:    sb a6, 2(a2)
+; RV64I-NEXT:    sb t1, 3(a2)
+; RV64I-NEXT:    and a4, a5, t2
+; RV64I-NEXT:    srli a4, a4, 8
+; RV64I-NEXT:    sb a5, 4(a2)
+; RV64I-NEXT:    sb a4, 5(a2)
+; RV64I-NEXT:    sb t0, 6(a2)
+; RV64I-NEXT:    sb t5, 7(a2)
+; RV64I-NEXT:    and a4, a3, t2
+; RV64I-NEXT:    srli a4, a4, 8
+; RV64I-NEXT:    sb a3, 8(a2)
+; RV64I-NEXT:    sb a4, 9(a2)
+; RV64I-NEXT:    sb t4, 10(a2)
+; RV64I-NEXT:    sb s0, 11(a2)
+; RV64I-NEXT:    and a3, a7, t2
+; RV64I-NEXT:    srli a3, a3, 8
+; RV64I-NEXT:    sb a7, 12(a2)
+; RV64I-NEXT:    sb a3, 13(a2)
+; RV64I-NEXT:    sb t6, 14(a2)
+; RV64I-NEXT:    sb s3, 15(a2)
+; RV64I-NEXT:    and a3, a1, t2
+; RV64I-NEXT:    srli a3, a3, 8
+; RV64I-NEXT:    sb a1, 16(a2)
+; RV64I-NEXT:    sb a3, 17(a2)
+; RV64I-NEXT:    sb s2, 18(a2)
+; RV64I-NEXT:    sb s6, 19(a2)
+; RV64I-NEXT:    and a1, t3, t2
+; RV64I-NEXT:    srli a1, a1, 8
+; RV64I-NEXT:    sb t3, 20(a2)
+; RV64I-NEXT:    sb a1, 21(a2)
+; RV64I-NEXT:    sb s4, 22(a2)
+; RV64I-NEXT:    sb s7, 23(a2)
+; RV64I-NEXT:    and a1, a0, t2
+; RV64I-NEXT:    and a3, s1, t2
+; RV64I-NEXT:    srli a1, a1, 8
+; RV64I-NEXT:    srli a3, a3, 8
+; RV64I-NEXT:    sb a0, 24(a2)
+; RV64I-NEXT:    sb a1, 25(a2)
+; RV64I-NEXT:    sb s5, 26(a2)
+; RV64I-NEXT:    sb s8, 27(a2)
+; RV64I-NEXT:    sb s1, 28(a2)
+; RV64I-NEXT:    sb a3, 29(a2)
+; RV64I-NEXT:    sb s9, 30(a2)
+; RV64I-NEXT:    sb s10, 31(a2)
+; RV64I-NEXT:    ld s0, 88(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s1, 80(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s2, 72(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s3, 64(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s4, 56(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s5, 48(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s6, 40(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s7, 32(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s8, 24(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s9, 16(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s10, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s11, 0(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    addi sp, sp, 96
+; RV64I-NEXT:    ret
+; RV64I-NEXT:  .LBB18_29:
+; RV64I-NEXT:    mv a4, a7
+; RV64I-NEXT:    mv a3, t0
+; RV64I-NEXT:    bgeu a6, t1, .LBB18_27
+; RV64I-NEXT:    j .LBB18_28
+;
+; RV32I-LABEL: ashr_32bytes:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    addi sp, sp, -96
+; RV32I-NEXT:    sw ra, 92(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s0, 88(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s1, 84(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s2, 80(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s3, 76(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s4, 72(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s5, 68(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s6, 64(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s7, 60(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s8, 56(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s9, 52(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s10, 48(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s11, 44(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    lbu a3, 16(a0)
+; RV32I-NEXT:    lbu a4, 17(a0)
+; RV32I-NEXT:    lbu a5, 18(a0)
+; RV32I-NEXT:    lbu a6, 19(a0)
+; RV32I-NEXT:    lbu a7, 20(a0)
+; RV32I-NEXT:    lbu t0, 21(a0)
+; RV32I-NEXT:    lbu t1, 22(a0)
+; RV32I-NEXT:    lbu t2, 23(a0)
+; RV32I-NEXT:    lbu t3, 24(a0)
+; RV32I-NEXT:    lbu t4, 25(a0)
+; RV32I-NEXT:    lbu t5, 26(a0)
+; RV32I-NEXT:    lbu t6, 27(a0)
+; RV32I-NEXT:    slli a4, a4, 8
+; RV32I-NEXT:    slli a6, a6, 8
+; RV32I-NEXT:    slli t0, t0, 8
+; RV32I-NEXT:    slli t2, t2, 8
+; RV32I-NEXT:    or a3, a4, a3
+; RV32I-NEXT:    or a4, a6, a5
+; RV32I-NEXT:    or a6, t0, a7
+; RV32I-NEXT:    or a5, t2, t1
+; RV32I-NEXT:    lbu a7, 28(a0)
+; RV32I-NEXT:    lbu t0, 29(a0)
+; RV32I-NEXT:    lbu t1, 30(a0)
+; RV32I-NEXT:    lbu t2, 31(a0)
+; RV32I-NEXT:    slli t4, t4, 8
+; RV32I-NEXT:    slli t6, t6, 8
+; RV32I-NEXT:    slli t0, t0, 8
+; RV32I-NEXT:    or t3, t4, t3
+; RV32I-NEXT:    or t4, t6, t5
+; RV32I-NEXT:    or t0, t0, a7
+; RV32I-NEXT:    lbu a7, 0(a1)
+; RV32I-NEXT:    lbu t5, 1(a1)
+; RV32I-NEXT:    lbu t6, 2(a1)
+; RV32I-NEXT:    lbu a1, 3(a1)
+; RV32I-NEXT:    slli t2, t2, 8
+; RV32I-NEXT:    or t1, t2, t1
+; RV32I-NEXT:    slli t5, t5, 8
+; RV32I-NEXT:    or s0, t5, a7
+; RV32I-NEXT:    slli a1, a1, 8
+; RV32I-NEXT:    or t2, a1, t6
+; RV32I-NEXT:    li t5, 32
+; RV32I-NEXT:    slli a7, a4, 16
+; RV32I-NEXT:    slli a1, a5, 16
+; RV32I-NEXT:    slli t4, t4, 16
+; RV32I-NEXT:    slli t1, t1, 16
+; RV32I-NEXT:    slli a5, t2, 16
+; RV32I-NEXT:    or t2, t4, t3
+; RV32I-NEXT:    or a4, t1, t0
+; RV32I-NEXT:    or a5, a5, s0
+; RV32I-NEXT:    slli a5, a5, 3
+; RV32I-NEXT:    srl s0, t2, a5
+; RV32I-NEXT:    neg s6, a5
+; RV32I-NEXT:    sll s1, a4, s6
+; RV32I-NEXT:    bltu a5, t5, .LBB18_2
+; RV32I-NEXT:  # %bb.1:
+; RV32I-NEXT:    sra t0, a4, a5
+; RV32I-NEXT:    j .LBB18_3
+; RV32I-NEXT:  .LBB18_2:
+; RV32I-NEXT:    or t0, s0, s1
+; RV32I-NEXT:  .LBB18_3:
+; RV32I-NEXT:    or t1, a7, a3
+; RV32I-NEXT:    or a7, a1, a6
+; RV32I-NEXT:    mv t3, t2
+; RV32I-NEXT:    beqz a5, .LBB18_5
+; RV32I-NEXT:  # %bb.4:
+; RV32I-NEXT:    mv t3, t0
+; RV32I-NEXT:  .LBB18_5:
+; RV32I-NEXT:    srl a3, t1, a5
+; RV32I-NEXT:    sll a1, a7, s6
+; RV32I-NEXT:    sw a1, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    bltu a5, t5, .LBB18_7
+; RV32I-NEXT:  # %bb.6:
+; RV32I-NEXT:    srai t4, a4, 31
+; RV32I-NEXT:    srl a1, a7, a5
+; RV32I-NEXT:    j .LBB18_8
+; RV32I-NEXT:  .LBB18_7:
+; RV32I-NEXT:    sra t4, a4, a5
+; RV32I-NEXT:    or a1, a3, a1
+; RV32I-NEXT:  .LBB18_8:
+; RV32I-NEXT:    li t6, 64
+; RV32I-NEXT:    mv t0, t1
+; RV32I-NEXT:    beqz a5, .LBB18_10
+; RV32I-NEXT:  # %bb.9:
+; RV32I-NEXT:    mv t0, a1
+; RV32I-NEXT:  .LBB18_10:
+; RV32I-NEXT:    sub s7, t6, a5
+; RV32I-NEXT:    bltu a5, t5, .LBB18_12
+; RV32I-NEXT:  # %bb.11:
+; RV32I-NEXT:    li a1, 0
+; RV32I-NEXT:    j .LBB18_13
+; RV32I-NEXT:  .LBB18_12:
+; RV32I-NEXT:    srl a1, a7, a5
+; RV32I-NEXT:  .LBB18_13:
+; RV32I-NEXT:    sw a3, 28(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    neg s10, s7
+; RV32I-NEXT:    bltu s7, t5, .LBB18_15
+; RV32I-NEXT:  # %bb.14:
+; RV32I-NEXT:    li a6, 0
+; RV32I-NEXT:    sll a3, t2, s7
+; RV32I-NEXT:    j .LBB18_16
+; RV32I-NEXT:  .LBB18_15:
+; RV32I-NEXT:    sll a6, t2, s6
+; RV32I-NEXT:    srl a3, t2, s10
+; RV32I-NEXT:    or a3, a3, s1
+; RV32I-NEXT:  .LBB18_16:
+; RV32I-NEXT:    sw t3, 40(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    addi s9, a5, -64
+; RV32I-NEXT:    mv t3, a4
+; RV32I-NEXT:    beqz s7, .LBB18_18
+; RV32I-NEXT:  # %bb.17:
+; RV32I-NEXT:    mv t3, a3
+; RV32I-NEXT:  .LBB18_18:
+; RV32I-NEXT:    neg s11, s9
+; RV32I-NEXT:    sw s0, 32(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    bltu s9, t5, .LBB18_20
+; RV32I-NEXT:  # %bb.19:
+; RV32I-NEXT:    sra s0, a4, s9
+; RV32I-NEXT:    j .LBB18_21
+; RV32I-NEXT:  .LBB18_20:
+; RV32I-NEXT:    sll a3, a4, s11
+; RV32I-NEXT:    or s0, s0, a3
+; RV32I-NEXT:  .LBB18_21:
+; RV32I-NEXT:    sw s1, 16(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw t4, 36(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    lbu s3, 11(a0)
+; RV32I-NEXT:    lbu a3, 15(a0)
+; RV32I-NEXT:    mv t4, t2
+; RV32I-NEXT:    beqz s9, .LBB18_23
+; RV32I-NEXT:  # %bb.22:
+; RV32I-NEXT:    mv t4, s0
+; RV32I-NEXT:  .LBB18_23:
+; RV32I-NEXT:    lbu s2, 9(a0)
+; RV32I-NEXT:    lbu s1, 10(a0)
+; RV32I-NEXT:    lbu s8, 13(a0)
+; RV32I-NEXT:    lbu ra, 14(a0)
+; RV32I-NEXT:    slli s3, s3, 8
+; RV32I-NEXT:    slli a3, a3, 8
+; RV32I-NEXT:    bltu s9, t5, .LBB18_25
+; RV32I-NEXT:  # %bb.24:
+; RV32I-NEXT:    srai s0, a4, 31
+; RV32I-NEXT:    j .LBB18_26
+; RV32I-NEXT:  .LBB18_25:
+; RV32I-NEXT:    sra s0, a4, a5
+; RV32I-NEXT:  .LBB18_26:
+; RV32I-NEXT:    or s1, s3, s1
+; RV32I-NEXT:    lbu s5, 8(a0)
+; RV32I-NEXT:    lbu s3, 12(a0)
+; RV32I-NEXT:    slli s2, s2, 8
+; RV32I-NEXT:    slli s4, s8, 8
+; RV32I-NEXT:    or s8, a3, ra
+; RV32I-NEXT:    bgeu a5, t6, .LBB18_28
+; RV32I-NEXT:  # %bb.27:
+; RV32I-NEXT:    or t4, t0, a6
+; RV32I-NEXT:    or s0, a1, t3
+; RV32I-NEXT:  .LBB18_28:
+; RV32I-NEXT:    lbu a3, 3(a0)
+; RV32I-NEXT:    lbu t3, 7(a0)
+; RV32I-NEXT:    or a6, s2, s5
+; RV32I-NEXT:    slli s2, s1, 16
+; RV32I-NEXT:    or s1, s4, s3
+; RV32I-NEXT:    slli s8, s8, 16
+; RV32I-NEXT:    mv a1, t1
+; RV32I-NEXT:    mv t0, a7
+; RV32I-NEXT:    beqz a5, .LBB18_30
+; RV32I-NEXT:  # %bb.29:
+; RV32I-NEXT:    mv a1, t4
+; RV32I-NEXT:    mv t0, s0
+; RV32I-NEXT:  .LBB18_30:
+; RV32I-NEXT:    slli s5, a3, 8
+; RV32I-NEXT:    lbu ra, 1(a0)
+; RV32I-NEXT:    lbu a3, 2(a0)
+; RV32I-NEXT:    lbu s3, 5(a0)
+; RV32I-NEXT:    lbu s0, 6(a0)
+; RV32I-NEXT:    slli s4, t3, 8
+; RV32I-NEXT:    or t4, s2, a6
+; RV32I-NEXT:    or t3, s8, s1
+; RV32I-NEXT:    bltu a5, t6, .LBB18_32
+; RV32I-NEXT:  # %bb.31:
+; RV32I-NEXT:    srai a6, a4, 31
+; RV32I-NEXT:    sw a6, 40(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw a6, 36(sp) # 4-byte Folded Spill
+; RV32I-NEXT:  .LBB18_32:
+; RV32I-NEXT:    slli a6, ra, 8
+; RV32I-NEXT:    or a3, s5, a3
+; RV32I-NEXT:    lbu s1, 0(a0)
+; RV32I-NEXT:    lbu a0, 4(a0)
+; RV32I-NEXT:    slli s3, s3, 8
+; RV32I-NEXT:    or s0, s4, s0
+; RV32I-NEXT:    srl s2, t4, a5
+; RV32I-NEXT:    sll ra, t3, s6
+; RV32I-NEXT:    bltu a5, t5, .LBB18_34
+; RV32I-NEXT:  # %bb.33:
+; RV32I-NEXT:    srl s4, t3, a5
+; RV32I-NEXT:    j .LBB18_35
+; RV32I-NEXT:  .LBB18_34:
+; RV32I-NEXT:    or s4, s2, ra
+; RV32I-NEXT:  .LBB18_35:
+; RV32I-NEXT:    or a6, a6, s1
+; RV32I-NEXT:    slli a3, a3, 16
+; RV32I-NEXT:    or a0, s3, a0
+; RV32I-NEXT:    slli s1, s0, 16
+; RV32I-NEXT:    mv s5, t4
+; RV32I-NEXT:    beqz a5, .LBB18_37
+; RV32I-NEXT:  # %bb.36:
+; RV32I-NEXT:    mv s5, s4
+; RV32I-NEXT:  .LBB18_37:
+; RV32I-NEXT:    or s0, a3, a6
+; RV32I-NEXT:    or a0, s1, a0
+; RV32I-NEXT:    bltu a5, t5, .LBB18_39
+; RV32I-NEXT:  # %bb.38:
+; RV32I-NEXT:    li s4, 0
+; RV32I-NEXT:    srl a3, a0, a5
+; RV32I-NEXT:    mv a6, s0
+; RV32I-NEXT:    bnez a5, .LBB18_40
+; RV32I-NEXT:    j .LBB18_41
+; RV32I-NEXT:  .LBB18_39:
+; RV32I-NEXT:    srl s4, t3, a5
+; RV32I-NEXT:    srl a3, s0, a5
+; RV32I-NEXT:    sll a6, a0, s6
+; RV32I-NEXT:    or a3, a3, a6
+; RV32I-NEXT:    mv a6, s0
+; RV32I-NEXT:    beqz a5, .LBB18_41
+; RV32I-NEXT:  .LBB18_40:
+; RV32I-NEXT:    mv a6, a3
+; RV32I-NEXT:  .LBB18_41:
+; RV32I-NEXT:    bltu a5, t5, .LBB18_44
+; RV32I-NEXT:  # %bb.42:
+; RV32I-NEXT:    li s1, 0
+; RV32I-NEXT:    bgeu s7, t5, .LBB18_45
+; RV32I-NEXT:  .LBB18_43:
+; RV32I-NEXT:    sll s3, t4, s6
+; RV32I-NEXT:    srl a3, t4, s10
+; RV32I-NEXT:    or a3, a3, ra
+; RV32I-NEXT:    mv s10, t3
+; RV32I-NEXT:    bnez s7, .LBB18_46
+; RV32I-NEXT:    j .LBB18_47
+; RV32I-NEXT:  .LBB18_44:
+; RV32I-NEXT:    srl s1, a0, a5
+; RV32I-NEXT:    bltu s7, t5, .LBB18_43
+; RV32I-NEXT:  .LBB18_45:
+; RV32I-NEXT:    li s3, 0
+; RV32I-NEXT:    sll a3, t4, s7
+; RV32I-NEXT:    mv s10, t3
+; RV32I-NEXT:    beqz s7, .LBB18_47
+; RV32I-NEXT:  .LBB18_46:
+; RV32I-NEXT:    mv s10, a3
+; RV32I-NEXT:  .LBB18_47:
+; RV32I-NEXT:    bltu s9, t5, .LBB18_49
+; RV32I-NEXT:  # %bb.48:
+; RV32I-NEXT:    srl a3, t3, s9
+; RV32I-NEXT:    mv s2, t4
+; RV32I-NEXT:    bnez s9, .LBB18_50
+; RV32I-NEXT:    j .LBB18_51
+; RV32I-NEXT:  .LBB18_49:
+; RV32I-NEXT:    sll a3, t3, s11
+; RV32I-NEXT:    or a3, s2, a3
+; RV32I-NEXT:    mv s2, t4
+; RV32I-NEXT:    beqz s9, .LBB18_51
+; RV32I-NEXT:  .LBB18_50:
+; RV32I-NEXT:    mv s2, a3
+; RV32I-NEXT:  .LBB18_51:
+; RV32I-NEXT:    bltu s9, t5, .LBB18_53
+; RV32I-NEXT:  # %bb.52:
+; RV32I-NEXT:    li s7, 0
+; RV32I-NEXT:    bltu a5, t6, .LBB18_54
+; RV32I-NEXT:    j .LBB18_55
+; RV32I-NEXT:  .LBB18_53:
+; RV32I-NEXT:    srl s7, t3, a5
+; RV32I-NEXT:    bgeu a5, t6, .LBB18_55
+; RV32I-NEXT:  .LBB18_54:
+; RV32I-NEXT:    or s2, a6, s3
+; RV32I-NEXT:    or s7, s1, s10
+; RV32I-NEXT:  .LBB18_55:
+; RV32I-NEXT:    li a3, 128
+; RV32I-NEXT:    mv a6, s0
+; RV32I-NEXT:    mv s1, a0
+; RV32I-NEXT:    beqz a5, .LBB18_57
+; RV32I-NEXT:  # %bb.56:
+; RV32I-NEXT:    mv a6, s2
+; RV32I-NEXT:    mv s1, s7
+; RV32I-NEXT:  .LBB18_57:
+; RV32I-NEXT:    sw a6, 8(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sub s2, a3, a5
+; RV32I-NEXT:    bltu a5, t6, .LBB18_59
+; RV32I-NEXT:  # %bb.58:
+; RV32I-NEXT:    li s5, 0
+; RV32I-NEXT:    li s4, 0
+; RV32I-NEXT:  .LBB18_59:
+; RV32I-NEXT:    neg s3, s2
+; RV32I-NEXT:    srl a6, t1, s3
+; RV32I-NEXT:    sw s4, 20(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s1, 4(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    bltu s2, t5, .LBB18_61
+; RV32I-NEXT:  # %bb.60:
+; RV32I-NEXT:    li s11, 0
+; RV32I-NEXT:    sll a3, t1, s2
+; RV32I-NEXT:    j .LBB18_62
+; RV32I-NEXT:  .LBB18_61:
+; RV32I-NEXT:    sll s11, t1, s6
+; RV32I-NEXT:    lw a3, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    or a3, a6, a3
+; RV32I-NEXT:  .LBB18_62:
+; RV32I-NEXT:    sub s1, t6, s2
+; RV32I-NEXT:    mv s8, a7
+; RV32I-NEXT:    beqz s2, .LBB18_64
+; RV32I-NEXT:  # %bb.63:
+; RV32I-NEXT:    mv s8, a3
+; RV32I-NEXT:  .LBB18_64:
+; RV32I-NEXT:    bltu s1, t5, .LBB18_66
+; RV32I-NEXT:  # %bb.65:
+; RV32I-NEXT:    srl a3, a7, s1
+; RV32I-NEXT:    mv a6, t1
+; RV32I-NEXT:    bnez s1, .LBB18_67
+; RV32I-NEXT:    j .LBB18_68
+; RV32I-NEXT:  .LBB18_66:
+; RV32I-NEXT:    neg a3, s1
+; RV32I-NEXT:    sll a3, a7, a3
+; RV32I-NEXT:    or a3, a6, a3
+; RV32I-NEXT:    mv a6, t1
+; RV32I-NEXT:    beqz s1, .LBB18_68
+; RV32I-NEXT:  .LBB18_67:
+; RV32I-NEXT:    mv a6, a3
+; RV32I-NEXT:  .LBB18_68:
+; RV32I-NEXT:    bltu s1, t5, .LBB18_71
+; RV32I-NEXT:  # %bb.69:
+; RV32I-NEXT:    li s1, 0
+; RV32I-NEXT:    sw s5, 24(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    bgeu s2, t5, .LBB18_72
+; RV32I-NEXT:  .LBB18_70:
+; RV32I-NEXT:    sll s6, t2, s6
+; RV32I-NEXT:    srl a3, t2, s3
+; RV32I-NEXT:    lw s3, 16(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    or a3, a3, s3
+; RV32I-NEXT:    j .LBB18_73
+; RV32I-NEXT:  .LBB18_71:
+; RV32I-NEXT:    srl s1, a7, s3
+; RV32I-NEXT:    sw s5, 24(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    bltu s2, t5, .LBB18_70
+; RV32I-NEXT:  .LBB18_72:
+; RV32I-NEXT:    li s6, 0
+; RV32I-NEXT:    sll a3, t2, s2
+; RV32I-NEXT:  .LBB18_73:
+; RV32I-NEXT:    addi s9, s2, -64
+; RV32I-NEXT:    mv s5, a4
+; RV32I-NEXT:    beqz s2, .LBB18_75
+; RV32I-NEXT:  # %bb.74:
+; RV32I-NEXT:    mv s5, a3
+; RV32I-NEXT:  .LBB18_75:
+; RV32I-NEXT:    bltu s9, t5, .LBB18_77
+; RV32I-NEXT:  # %bb.76:
+; RV32I-NEXT:    li s3, 0
+; RV32I-NEXT:    sll a3, t1, s9
+; RV32I-NEXT:    mv s7, a7
+; RV32I-NEXT:    bnez s9, .LBB18_78
+; RV32I-NEXT:    j .LBB18_79
+; RV32I-NEXT:  .LBB18_77:
+; RV32I-NEXT:    sll s3, t1, s2
+; RV32I-NEXT:    neg a3, s9
+; RV32I-NEXT:    srl a3, t1, a3
+; RV32I-NEXT:    sll s4, a7, s2
+; RV32I-NEXT:    or a3, a3, s4
+; RV32I-NEXT:    mv s7, a7
+; RV32I-NEXT:    beqz s9, .LBB18_79
+; RV32I-NEXT:  .LBB18_78:
+; RV32I-NEXT:    mv s7, a3
+; RV32I-NEXT:  .LBB18_79:
+; RV32I-NEXT:    bltu s2, t6, .LBB18_81
+; RV32I-NEXT:  # %bb.80:
+; RV32I-NEXT:    li s11, 0
+; RV32I-NEXT:    li s8, 0
+; RV32I-NEXT:    j .LBB18_82
+; RV32I-NEXT:  .LBB18_81:
+; RV32I-NEXT:    or s3, a6, s6
+; RV32I-NEXT:    or s7, s1, s5
+; RV32I-NEXT:  .LBB18_82:
+; RV32I-NEXT:    addi ra, a5, -128
+; RV32I-NEXT:    mv s4, t2
+; RV32I-NEXT:    mv s6, a4
+; RV32I-NEXT:    beqz s2, .LBB18_84
+; RV32I-NEXT:  # %bb.83:
+; RV32I-NEXT:    mv s4, s3
+; RV32I-NEXT:    mv s6, s7
+; RV32I-NEXT:  .LBB18_84:
+; RV32I-NEXT:    neg s9, ra
+; RV32I-NEXT:    sll s3, a4, s9
+; RV32I-NEXT:    bltu ra, t5, .LBB18_86
+; RV32I-NEXT:  # %bb.85:
+; RV32I-NEXT:    sra a3, a4, ra
+; RV32I-NEXT:    mv s1, t2
+; RV32I-NEXT:    bnez ra, .LBB18_87
+; RV32I-NEXT:    j .LBB18_88
+; RV32I-NEXT:  .LBB18_86:
+; RV32I-NEXT:    lw a3, 32(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    or a3, a3, s3
+; RV32I-NEXT:    mv s1, t2
+; RV32I-NEXT:    beqz ra, .LBB18_88
+; RV32I-NEXT:  .LBB18_87:
+; RV32I-NEXT:    mv s1, a3
+; RV32I-NEXT:  .LBB18_88:
+; RV32I-NEXT:    bltu ra, t5, .LBB18_90
+; RV32I-NEXT:  # %bb.89:
+; RV32I-NEXT:    srai s2, a4, 31
+; RV32I-NEXT:    srl a3, a7, ra
+; RV32I-NEXT:    mv a6, t1
+; RV32I-NEXT:    bnez ra, .LBB18_91
+; RV32I-NEXT:    j .LBB18_92
+; RV32I-NEXT:  .LBB18_90:
+; RV32I-NEXT:    sra s2, a4, a5
+; RV32I-NEXT:    sll a3, a7, s9
+; RV32I-NEXT:    lw a6, 28(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    or a3, a6, a3
+; RV32I-NEXT:    mv a6, t1
+; RV32I-NEXT:    beqz ra, .LBB18_92
+; RV32I-NEXT:  .LBB18_91:
+; RV32I-NEXT:    mv a6, a3
+; RV32I-NEXT:  .LBB18_92:
+; RV32I-NEXT:    mv s5, t0
+; RV32I-NEXT:    sub s10, t6, ra
+; RV32I-NEXT:    li t0, 64
+; RV32I-NEXT:    bltu ra, t5, .LBB18_94
+; RV32I-NEXT:  # %bb.93:
+; RV32I-NEXT:    li s7, 0
+; RV32I-NEXT:    j .LBB18_95
+; RV32I-NEXT:  .LBB18_94:
+; RV32I-NEXT:    srl s7, a7, a5
+; RV32I-NEXT:  .LBB18_95:
+; RV32I-NEXT:    mv t6, s8
+; RV32I-NEXT:    mv s8, s11
+; RV32I-NEXT:    bltu s10, t5, .LBB18_97
+; RV32I-NEXT:  # %bb.96:
+; RV32I-NEXT:    li s9, 0
+; RV32I-NEXT:    sll a3, t2, s10
+; RV32I-NEXT:    j .LBB18_98
+; RV32I-NEXT:  .LBB18_97:
+; RV32I-NEXT:    sll s9, t2, s9
+; RV32I-NEXT:    neg a3, s10
+; RV32I-NEXT:    srl a3, t2, a3
+; RV32I-NEXT:    or a3, a3, s3
+; RV32I-NEXT:  .LBB18_98:
+; RV32I-NEXT:    addi s11, ra, -64
+; RV32I-NEXT:    mv s3, a4
+; RV32I-NEXT:    beqz s10, .LBB18_100
+; RV32I-NEXT:  # %bb.99:
+; RV32I-NEXT:    mv s3, a3
+; RV32I-NEXT:  .LBB18_100:
+; RV32I-NEXT:    bltu s11, t5, .LBB18_102
+; RV32I-NEXT:  # %bb.101:
+; RV32I-NEXT:    sra a3, a4, s11
+; RV32I-NEXT:    bnez s11, .LBB18_103
+; RV32I-NEXT:    j .LBB18_104
+; RV32I-NEXT:  .LBB18_102:
+; RV32I-NEXT:    srl a3, t2, ra
+; RV32I-NEXT:    mv s10, s4
+; RV32I-NEXT:    neg s4, s11
+; RV32I-NEXT:    sll s4, a4, s4
+; RV32I-NEXT:    or a3, a3, s4
+; RV32I-NEXT:    mv s4, s10
+; RV32I-NEXT:    beqz s11, .LBB18_104
+; RV32I-NEXT:  .LBB18_103:
+; RV32I-NEXT:    mv t2, a3
+; RV32I-NEXT:  .LBB18_104:
+; RV32I-NEXT:    bltu s11, t5, .LBB18_106
+; RV32I-NEXT:  # %bb.105:
+; RV32I-NEXT:    srai t5, a4, 31
+; RV32I-NEXT:    lw s11, 40(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    bltu ra, t0, .LBB18_107
+; RV32I-NEXT:    j .LBB18_108
+; RV32I-NEXT:  .LBB18_106:
+; RV32I-NEXT:    sra t5, a4, ra
+; RV32I-NEXT:    lw s11, 40(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    bgeu ra, t0, .LBB18_108
+; RV32I-NEXT:  .LBB18_107:
+; RV32I-NEXT:    or t2, a6, s9
+; RV32I-NEXT:    or t5, s7, s3
+; RV32I-NEXT:  .LBB18_108:
+; RV32I-NEXT:    li a6, 128
+; RV32I-NEXT:    bnez ra, .LBB18_117
+; RV32I-NEXT:  # %bb.109:
+; RV32I-NEXT:    bgeu ra, t0, .LBB18_118
+; RV32I-NEXT:  .LBB18_110:
+; RV32I-NEXT:    bgeu a5, a6, .LBB18_112
+; RV32I-NEXT:  .LBB18_111:
+; RV32I-NEXT:    lw a3, 8(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    or t1, a3, s8
+; RV32I-NEXT:    lw a3, 4(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    or a7, a3, t6
+; RV32I-NEXT:    lw a3, 24(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    or s1, a3, s4
+; RV32I-NEXT:    lw a3, 20(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    or s2, a3, s6
+; RV32I-NEXT:  .LBB18_112:
+; RV32I-NEXT:    lw ra, 36(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    mv t0, s5
+; RV32I-NEXT:    beqz a5, .LBB18_114
+; RV32I-NEXT:  # %bb.113:
+; RV32I-NEXT:    mv s0, t1
+; RV32I-NEXT:    mv a0, a7
+; RV32I-NEXT:    mv t4, s1
+; RV32I-NEXT:    mv t3, s2
+; RV32I-NEXT:  .LBB18_114:
+; RV32I-NEXT:    bltu a5, a6, .LBB18_116
+; RV32I-NEXT:  # %bb.115:
+; RV32I-NEXT:    srai a1, a4, 31
+; RV32I-NEXT:    mv t0, a1
+; RV32I-NEXT:    mv s11, a1
+; RV32I-NEXT:    mv ra, a1
+; RV32I-NEXT:  .LBB18_116:
+; RV32I-NEXT:    srli a4, s0, 16
+; RV32I-NEXT:    lui t1, 16
+; RV32I-NEXT:    srli a7, s0, 24
+; RV32I-NEXT:    srli a5, a0, 16
+; RV32I-NEXT:    srli t5, a0, 24
+; RV32I-NEXT:    srli a6, t4, 16
+; RV32I-NEXT:    srli s2, t4, 24
+; RV32I-NEXT:    srli t2, t3, 16
+; RV32I-NEXT:    srli s3, t3, 24
+; RV32I-NEXT:    srli s1, a1, 16
+; RV32I-NEXT:    srli a3, a1, 24
+; RV32I-NEXT:    srli t6, t0, 16
+; RV32I-NEXT:    srli s6, t0, 24
+; RV32I-NEXT:    srli s5, s11, 16
+; RV32I-NEXT:    srli s4, s11, 24
+; RV32I-NEXT:    srli s7, ra, 16
+; RV32I-NEXT:    srli s8, ra, 24
+; RV32I-NEXT:    addi t1, t1, -1
+; RV32I-NEXT:    and s9, s0, t1
+; RV32I-NEXT:    and s10, a0, t1
+; RV32I-NEXT:    srli s9, s9, 8
+; RV32I-NEXT:    sb s0, 0(a2)
+; RV32I-NEXT:    sb s9, 1(a2)
+; RV32I-NEXT:    sb a4, 2(a2)
+; RV32I-NEXT:    sb a7, 3(a2)
+; RV32I-NEXT:    and a4, t4, t1
+; RV32I-NEXT:    srli a7, s10, 8
+; RV32I-NEXT:    sb a0, 4(a2)
+; RV32I-NEXT:    sb a7, 5(a2)
+; RV32I-NEXT:    sb a5, 6(a2)
+; RV32I-NEXT:    sb t5, 7(a2)
+; RV32I-NEXT:    and a0, t3, t1
+; RV32I-NEXT:    srli a4, a4, 8
+; RV32I-NEXT:    sb t4, 8(a2)
+; RV32I-NEXT:    sb a4, 9(a2)
+; RV32I-NEXT:    sb a6, 10(a2)
+; RV32I-NEXT:    sb s2, 11(a2)
+; RV32I-NEXT:    and a4, a1, t1
+; RV32I-NEXT:    srli a0, a0, 8
+; RV32I-NEXT:    sb t3, 12(a2)
+; RV32I-NEXT:    sb a0, 13(a2)
+; RV32I-NEXT:    sb t2, 14(a2)
+; RV32I-NEXT:    sb s3, 15(a2)
+; RV32I-NEXT:    and a0, t0, t1
+; RV32I-NEXT:    srli a4, a4, 8
+; RV32I-NEXT:    sb a1, 16(a2)
+; RV32I-NEXT:    sb a4, 17(a2)
+; RV32I-NEXT:    sb s1, 18(a2)
+; RV32I-NEXT:    sb a3, 19(a2)
+; RV32I-NEXT:    and a1, s11, t1
+; RV32I-NEXT:    and a3, ra, t1
+; RV32I-NEXT:    srli a0, a0, 8
+; RV32I-NEXT:    srli a1, a1, 8
+; RV32I-NEXT:    srli a3, a3, 8
+; RV32I-NEXT:    sb t0, 20(a2)
+; RV32I-NEXT:    sb a0, 21(a2)
+; RV32I-NEXT:    sb t6, 22(a2)
+; RV32I-NEXT:    sb s6, 23(a2)
+; RV32I-NEXT:    sb s11, 24(a2)
+; RV32I-NEXT:    sb a1, 25(a2)
+; RV32I-NEXT:    sb s5, 26(a2)
+; RV32I-NEXT:    sb s4, 27(a2)
+; RV32I-NEXT:    sb ra, 28(a2)
+; RV32I-NEXT:    sb a3, 29(a2)
+; RV32I-NEXT:    sb s7, 30(a2)
+; RV32I-NEXT:    sb s8, 31(a2)
+; RV32I-NEXT:    lw ra, 92(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s0, 88(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s1, 84(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s2, 80(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s3, 76(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s4, 72(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s5, 68(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s6, 64(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s7, 60(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s8, 56(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s9, 52(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s10, 48(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s11, 44(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    addi sp, sp, 96
+; RV32I-NEXT:    ret
+; RV32I-NEXT:  .LBB18_117:
+; RV32I-NEXT:    mv t1, t2
+; RV32I-NEXT:    mv a7, t5
+; RV32I-NEXT:    bltu ra, t0, .LBB18_110
+; RV32I-NEXT:  .LBB18_118:
+; RV32I-NEXT:    srai s1, a4, 31
+; RV32I-NEXT:    mv s2, s1
+; RV32I-NEXT:    bltu a5, a6, .LBB18_111
+; RV32I-NEXT:    j .LBB18_112
+  %src = load i256, ptr %src.ptr, align 1
+  %byteOff = load i256, ptr %byteOff.ptr, align 1
+  %bitOff = shl i256 %byteOff, 3
+  %res = ashr i256 %src, %bitOff
+  store i256 %res, ptr %dst, align 1
+  ret void
+}
+
+define void @ashr_32bytes_wordOff(ptr %src.ptr, ptr %wordOff.ptr, ptr %dst) nounwind {
+; RV64I-LABEL: ashr_32bytes_wordOff:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -96
+; RV64I-NEXT:    sd s0, 88(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s1, 80(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s2, 72(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s3, 64(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s4, 56(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s5, 48(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s6, 40(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s7, 32(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s8, 24(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s9, 16(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s10, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s11, 0(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    lbu a3, 0(a0)
+; RV64I-NEXT:    lbu a4, 1(a0)
+; RV64I-NEXT:    lbu a5, 2(a0)
+; RV64I-NEXT:    lbu a6, 3(a0)
+; RV64I-NEXT:    lbu a7, 4(a0)
+; RV64I-NEXT:    lbu t0, 5(a0)
+; RV64I-NEXT:    lbu t1, 6(a0)
+; RV64I-NEXT:    lbu t2, 7(a0)
+; RV64I-NEXT:    lbu t3, 8(a0)
+; RV64I-NEXT:    lbu t4, 9(a0)
+; RV64I-NEXT:    lbu t5, 10(a0)
+; RV64I-NEXT:    lbu t6, 11(a0)
+; RV64I-NEXT:    lbu s0, 12(a0)
+; RV64I-NEXT:    lbu s1, 13(a0)
+; RV64I-NEXT:    lbu s2, 14(a0)
+; RV64I-NEXT:    lbu s3, 15(a0)
+; RV64I-NEXT:    lbu s4, 16(a0)
+; RV64I-NEXT:    lbu s5, 17(a0)
+; RV64I-NEXT:    lbu s6, 18(a0)
+; RV64I-NEXT:    lbu s7, 19(a0)
+; RV64I-NEXT:    slli a4, a4, 8
+; RV64I-NEXT:    slli a6, a6, 8
+; RV64I-NEXT:    slli t0, t0, 8
+; RV64I-NEXT:    or a3, a4, a3
+; RV64I-NEXT:    or a5, a6, a5
+; RV64I-NEXT:    or a7, t0, a7
+; RV64I-NEXT:    lbu s8, 20(a0)
+; RV64I-NEXT:    lbu s9, 21(a0)
+; RV64I-NEXT:    lbu s10, 22(a0)
+; RV64I-NEXT:    lbu s11, 23(a0)
+; RV64I-NEXT:    slli t2, t2, 8
+; RV64I-NEXT:    slli t4, t4, 8
+; RV64I-NEXT:    slli t6, t6, 8
+; RV64I-NEXT:    slli s1, s1, 8
+; RV64I-NEXT:    or t1, t2, t1
+; RV64I-NEXT:    or a4, t4, t3
+; RV64I-NEXT:    or a6, t6, t5
+; RV64I-NEXT:    or t0, s1, s0
+; RV64I-NEXT:    lbu t5, 24(a0)
+; RV64I-NEXT:    lbu t6, 25(a0)
+; RV64I-NEXT:    lbu s0, 26(a0)
+; RV64I-NEXT:    lbu s1, 27(a0)
+; RV64I-NEXT:    slli s3, s3, 8
+; RV64I-NEXT:    slli s5, s5, 8
+; RV64I-NEXT:    slli s7, s7, 8
+; RV64I-NEXT:    or t4, s3, s2
+; RV64I-NEXT:    or t2, s5, s4
+; RV64I-NEXT:    or t3, s7, s6
+; RV64I-NEXT:    lbu s2, 28(a0)
+; RV64I-NEXT:    lbu s3, 29(a0)
+; RV64I-NEXT:    lbu s4, 30(a0)
+; RV64I-NEXT:    lbu a0, 31(a0)
+; RV64I-NEXT:    slli s9, s9, 8
+; RV64I-NEXT:    slli s11, s11, 8
+; RV64I-NEXT:    slli t6, t6, 8
+; RV64I-NEXT:    slli s1, s1, 8
+; RV64I-NEXT:    or s5, s9, s8
+; RV64I-NEXT:    or s6, s11, s10
+; RV64I-NEXT:    or t5, t6, t5
+; RV64I-NEXT:    or s0, s1, s0
+; RV64I-NEXT:    lbu t6, 0(a1)
+; RV64I-NEXT:    lbu s1, 1(a1)
+; RV64I-NEXT:    lbu s7, 2(a1)
+; RV64I-NEXT:    lbu s8, 3(a1)
+; RV64I-NEXT:    slli s3, s3, 8
+; RV64I-NEXT:    slli a0, a0, 8
+; RV64I-NEXT:    slli s1, s1, 8
+; RV64I-NEXT:    or s2, s3, s2
+; RV64I-NEXT:    or s3, a0, s4
+; RV64I-NEXT:    or t6, s1, t6
+; RV64I-NEXT:    lbu a0, 4(a1)
+; RV64I-NEXT:    lbu s1, 5(a1)
+; RV64I-NEXT:    lbu s4, 6(a1)
+; RV64I-NEXT:    lbu a1, 7(a1)
+; RV64I-NEXT:    slli s8, s8, 8
+; RV64I-NEXT:    or s7, s8, s7
+; RV64I-NEXT:    slli s1, s1, 8
+; RV64I-NEXT:    or s1, s1, a0
+; RV64I-NEXT:    slli a1, a1, 8
+; RV64I-NEXT:    or s4, a1, s4
+; RV64I-NEXT:    slli t1, t1, 16
+; RV64I-NEXT:    or a1, t1, a7
+; RV64I-NEXT:    slli t4, t4, 16
+; RV64I-NEXT:    or a0, t4, t0
+; RV64I-NEXT:    slli t3, t3, 16
+; RV64I-NEXT:    or a7, t3, t2
+; RV64I-NEXT:    slli s6, s6, 16
+; RV64I-NEXT:    or t1, s6, s5
+; RV64I-NEXT:    slli s0, s0, 16
+; RV64I-NEXT:    or t4, s0, t5
+; RV64I-NEXT:    slli s3, s3, 16
+; RV64I-NEXT:    or t5, s3, s2
+; RV64I-NEXT:    slli s7, s7, 16
+; RV64I-NEXT:    or t6, s7, t6
+; RV64I-NEXT:    slli s4, s4, 16
+; RV64I-NEXT:    or s0, s4, s1
+; RV64I-NEXT:    li t0, 64
+; RV64I-NEXT:    slli t3, a5, 16
+; RV64I-NEXT:    slli t2, a6, 16
+; RV64I-NEXT:    slli t1, t1, 32
+; RV64I-NEXT:    slli t5, t5, 32
+; RV64I-NEXT:    slli s0, s0, 32
+; RV64I-NEXT:    or a7, t1, a7
+; RV64I-NEXT:    or a5, t5, t4
+; RV64I-NEXT:    or a6, s0, t6
+; RV64I-NEXT:    slli a6, a6, 5
+; RV64I-NEXT:    subw t1, a6, t0
+; RV64I-NEXT:    negw t5, a6
+; RV64I-NEXT:    sll t4, a5, t5
+; RV64I-NEXT:    bltu a6, t0, .LBB19_2
+; RV64I-NEXT:  # %bb.1:
+; RV64I-NEXT:    sra t6, a5, t1
+; RV64I-NEXT:    j .LBB19_3
+; RV64I-NEXT:  .LBB19_2:
+; RV64I-NEXT:    srl t6, a7, a6
+; RV64I-NEXT:    or t6, t6, t4
+; RV64I-NEXT:  .LBB19_3:
+; RV64I-NEXT:    or a3, t3, a3
+; RV64I-NEXT:    slli t3, a1, 32
+; RV64I-NEXT:    or t2, t2, a4
+; RV64I-NEXT:    slli a0, a0, 32
+; RV64I-NEXT:    mv a1, a7
+; RV64I-NEXT:    beqz a6, .LBB19_5
+; RV64I-NEXT:  # %bb.4:
+; RV64I-NEXT:    mv a1, t6
+; RV64I-NEXT:  .LBB19_5:
+; RV64I-NEXT:    or a4, t3, a3
+; RV64I-NEXT:    or a3, a0, t2
+; RV64I-NEXT:    bltu a6, t0, .LBB19_7
+; RV64I-NEXT:  # %bb.6:
+; RV64I-NEXT:    srai a0, a5, 63
+; RV64I-NEXT:    srl t3, a3, t1
+; RV64I-NEXT:    j .LBB19_8
+; RV64I-NEXT:  .LBB19_7:
+; RV64I-NEXT:    sra a0, a5, a6
+; RV64I-NEXT:    srl t1, a4, a6
+; RV64I-NEXT:    sll t2, a3, t5
+; RV64I-NEXT:    or t3, t1, t2
+; RV64I-NEXT:  .LBB19_8:
+; RV64I-NEXT:    li t1, 128
+; RV64I-NEXT:    mv t2, a4
+; RV64I-NEXT:    beqz a6, .LBB19_10
+; RV64I-NEXT:  # %bb.9:
+; RV64I-NEXT:    mv t2, t3
+; RV64I-NEXT:  .LBB19_10:
+; RV64I-NEXT:    sub t6, t1, a6
+; RV64I-NEXT:    bltu a6, t0, .LBB19_13
+; RV64I-NEXT:  # %bb.11:
+; RV64I-NEXT:    li t3, 0
+; RV64I-NEXT:    bgeu t6, t0, .LBB19_14
+; RV64I-NEXT:  .LBB19_12:
+; RV64I-NEXT:    sll t5, a7, t5
+; RV64I-NEXT:    negw s0, t6
+; RV64I-NEXT:    srl s0, a7, s0
+; RV64I-NEXT:    or s1, s0, t4
+; RV64I-NEXT:    j .LBB19_15
+; RV64I-NEXT:  .LBB19_13:
+; RV64I-NEXT:    srl t3, a3, a6
+; RV64I-NEXT:    bltu t6, t0, .LBB19_12
+; RV64I-NEXT:  .LBB19_14:
+; RV64I-NEXT:    li t5, 0
+; RV64I-NEXT:    subw t4, t6, t0
+; RV64I-NEXT:    sll s1, a7, t4
+; RV64I-NEXT:  .LBB19_15:
+; RV64I-NEXT:    sub s0, a6, t1
+; RV64I-NEXT:    mv t4, a5
+; RV64I-NEXT:    beqz t6, .LBB19_17
+; RV64I-NEXT:  # %bb.16:
+; RV64I-NEXT:    mv t4, s1
+; RV64I-NEXT:  .LBB19_17:
+; RV64I-NEXT:    bltu s0, t0, .LBB19_19
+; RV64I-NEXT:  # %bb.18:
+; RV64I-NEXT:    subw t6, s0, t0
+; RV64I-NEXT:    sra t6, a5, t6
+; RV64I-NEXT:    bnez s0, .LBB19_20
+; RV64I-NEXT:    j .LBB19_21
+; RV64I-NEXT:  .LBB19_19:
+; RV64I-NEXT:    srl t6, a7, s0
+; RV64I-NEXT:    negw s1, s0
+; RV64I-NEXT:    sll s1, a5, s1
+; RV64I-NEXT:    or t6, t6, s1
+; RV64I-NEXT:    beqz s0, .LBB19_21
+; RV64I-NEXT:  .LBB19_20:
+; RV64I-NEXT:    mv a7, t6
+; RV64I-NEXT:  .LBB19_21:
+; RV64I-NEXT:    bltu s0, t0, .LBB19_23
+; RV64I-NEXT:  # %bb.22:
+; RV64I-NEXT:    srai t0, a5, 63
+; RV64I-NEXT:    bltu a6, t1, .LBB19_24
+; RV64I-NEXT:    j .LBB19_25
+; RV64I-NEXT:  .LBB19_23:
+; RV64I-NEXT:    sra t0, a5, s0
+; RV64I-NEXT:    bgeu a6, t1, .LBB19_25
+; RV64I-NEXT:  .LBB19_24:
+; RV64I-NEXT:    or a7, t2, t5
+; RV64I-NEXT:    or t0, t3, t4
+; RV64I-NEXT:  .LBB19_25:
+; RV64I-NEXT:    bnez a6, .LBB19_29
+; RV64I-NEXT:  # %bb.26:
+; RV64I-NEXT:    bltu a6, t1, .LBB19_28
+; RV64I-NEXT:  .LBB19_27:
+; RV64I-NEXT:    srai a1, a5, 63
+; RV64I-NEXT:    mv a0, a1
+; RV64I-NEXT:  .LBB19_28:
+; RV64I-NEXT:    srli a5, a4, 32
+; RV64I-NEXT:    srliw a6, a4, 16
+; RV64I-NEXT:    lui t2, 16
+; RV64I-NEXT:    srliw t1, a4, 24
+; RV64I-NEXT:    srli t0, a4, 48
+; RV64I-NEXT:    srli t5, a4, 56
+; RV64I-NEXT:    srli a7, a3, 32
+; RV64I-NEXT:    srliw t4, a3, 16
+; RV64I-NEXT:    srliw s0, a3, 24
+; RV64I-NEXT:    srli t6, a3, 48
+; RV64I-NEXT:    srli s3, a3, 56
+; RV64I-NEXT:    srli t3, a1, 32
+; RV64I-NEXT:    srliw s2, a1, 16
+; RV64I-NEXT:    srliw s6, a1, 24
+; RV64I-NEXT:    srli s4, a1, 48
+; RV64I-NEXT:    srli s7, a1, 56
+; RV64I-NEXT:    srli s1, a0, 32
+; RV64I-NEXT:    srliw s5, a0, 16
+; RV64I-NEXT:    srliw s8, a0, 24
+; RV64I-NEXT:    srli s9, a0, 48
+; RV64I-NEXT:    srli s10, a0, 56
+; RV64I-NEXT:    addi t2, t2, -1
+; RV64I-NEXT:    and s11, a4, t2
+; RV64I-NEXT:    srli s11, s11, 8
+; RV64I-NEXT:    sb a4, 0(a2)
+; RV64I-NEXT:    sb s11, 1(a2)
+; RV64I-NEXT:    sb a6, 2(a2)
+; RV64I-NEXT:    sb t1, 3(a2)
+; RV64I-NEXT:    and a4, a5, t2
+; RV64I-NEXT:    srli a4, a4, 8
+; RV64I-NEXT:    sb a5, 4(a2)
+; RV64I-NEXT:    sb a4, 5(a2)
+; RV64I-NEXT:    sb t0, 6(a2)
+; RV64I-NEXT:    sb t5, 7(a2)
+; RV64I-NEXT:    and a4, a3, t2
+; RV64I-NEXT:    srli a4, a4, 8
+; RV64I-NEXT:    sb a3, 8(a2)
+; RV64I-NEXT:    sb a4, 9(a2)
+; RV64I-NEXT:    sb t4, 10(a2)
+; RV64I-NEXT:    sb s0, 11(a2)
+; RV64I-NEXT:    and a3, a7, t2
+; RV64I-NEXT:    srli a3, a3, 8
+; RV64I-NEXT:    sb a7, 12(a2)
+; RV64I-NEXT:    sb a3, 13(a2)
+; RV64I-NEXT:    sb t6, 14(a2)
+; RV64I-NEXT:    sb s3, 15(a2)
+; RV64I-NEXT:    and a3, a1, t2
+; RV64I-NEXT:    srli a3, a3, 8
+; RV64I-NEXT:    sb a1, 16(a2)
+; RV64I-NEXT:    sb a3, 17(a2)
+; RV64I-NEXT:    sb s2, 18(a2)
+; RV64I-NEXT:    sb s6, 19(a2)
+; RV64I-NEXT:    and a1, t3, t2
+; RV64I-NEXT:    srli a1, a1, 8
+; RV64I-NEXT:    sb t3, 20(a2)
+; RV64I-NEXT:    sb a1, 21(a2)
+; RV64I-NEXT:    sb s4, 22(a2)
+; RV64I-NEXT:    sb s7, 23(a2)
+; RV64I-NEXT:    and a1, a0, t2
+; RV64I-NEXT:    and a3, s1, t2
+; RV64I-NEXT:    srli a1, a1, 8
+; RV64I-NEXT:    srli a3, a3, 8
+; RV64I-NEXT:    sb a0, 24(a2)
+; RV64I-NEXT:    sb a1, 25(a2)
+; RV64I-NEXT:    sb s5, 26(a2)
+; RV64I-NEXT:    sb s8, 27(a2)
+; RV64I-NEXT:    sb s1, 28(a2)
+; RV64I-NEXT:    sb a3, 29(a2)
+; RV64I-NEXT:    sb s9, 30(a2)
+; RV64I-NEXT:    sb s10, 31(a2)
+; RV64I-NEXT:    ld s0, 88(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s1, 80(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s2, 72(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s3, 64(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s4, 56(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s5, 48(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s6, 40(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s7, 32(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s8, 24(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s9, 16(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s10, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s11, 0(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    addi sp, sp, 96
+; RV64I-NEXT:    ret
+; RV64I-NEXT:  .LBB19_29:
+; RV64I-NEXT:    mv a4, a7
+; RV64I-NEXT:    mv a3, t0
+; RV64I-NEXT:    bgeu a6, t1, .LBB19_27
+; RV64I-NEXT:    j .LBB19_28
+;
+; RV32I-LABEL: ashr_32bytes_wordOff:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    addi sp, sp, -96
+; RV32I-NEXT:    sw ra, 92(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s0, 88(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s1, 84(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s2, 80(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s3, 76(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s4, 72(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s5, 68(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s6, 64(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s7, 60(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s8, 56(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s9, 52(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s10, 48(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s11, 44(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    lbu a3, 16(a0)
+; RV32I-NEXT:    lbu a4, 17(a0)
+; RV32I-NEXT:    lbu a5, 18(a0)
+; RV32I-NEXT:    lbu a6, 19(a0)
+; RV32I-NEXT:    lbu a7, 20(a0)
+; RV32I-NEXT:    lbu t0, 21(a0)
+; RV32I-NEXT:    lbu t1, 22(a0)
+; RV32I-NEXT:    lbu t2, 23(a0)
+; RV32I-NEXT:    lbu t3, 24(a0)
+; RV32I-NEXT:    lbu t4, 25(a0)
+; RV32I-NEXT:    lbu t5, 26(a0)
+; RV32I-NEXT:    lbu t6, 27(a0)
+; RV32I-NEXT:    slli a4, a4, 8
+; RV32I-NEXT:    slli a6, a6, 8
+; RV32I-NEXT:    slli t0, t0, 8
+; RV32I-NEXT:    slli t2, t2, 8
+; RV32I-NEXT:    or a3, a4, a3
+; RV32I-NEXT:    or a4, a6, a5
+; RV32I-NEXT:    or a6, t0, a7
+; RV32I-NEXT:    or a5, t2, t1
+; RV32I-NEXT:    lbu a7, 28(a0)
+; RV32I-NEXT:    lbu t0, 29(a0)
+; RV32I-NEXT:    lbu t1, 30(a0)
+; RV32I-NEXT:    lbu t2, 31(a0)
+; RV32I-NEXT:    slli t4, t4, 8
+; RV32I-NEXT:    slli t6, t6, 8
+; RV32I-NEXT:    slli t0, t0, 8
+; RV32I-NEXT:    or t3, t4, t3
+; RV32I-NEXT:    or t4, t6, t5
+; RV32I-NEXT:    or t0, t0, a7
+; RV32I-NEXT:    lbu a7, 0(a1)
+; RV32I-NEXT:    lbu t5, 1(a1)
+; RV32I-NEXT:    lbu t6, 2(a1)
+; RV32I-NEXT:    lbu a1, 3(a1)
+; RV32I-NEXT:    slli t2, t2, 8
+; RV32I-NEXT:    or t1, t2, t1
+; RV32I-NEXT:    slli t5, t5, 8
+; RV32I-NEXT:    or s0, t5, a7
+; RV32I-NEXT:    slli a1, a1, 8
+; RV32I-NEXT:    or t2, a1, t6
+; RV32I-NEXT:    li t5, 32
+; RV32I-NEXT:    slli a7, a4, 16
+; RV32I-NEXT:    slli a1, a5, 16
+; RV32I-NEXT:    slli t4, t4, 16
+; RV32I-NEXT:    slli t1, t1, 16
+; RV32I-NEXT:    slli a5, t2, 16
+; RV32I-NEXT:    or t2, t4, t3
+; RV32I-NEXT:    or a4, t1, t0
+; RV32I-NEXT:    or a5, a5, s0
+; RV32I-NEXT:    slli a5, a5, 5
+; RV32I-NEXT:    srl s0, t2, a5
+; RV32I-NEXT:    neg s6, a5
+; RV32I-NEXT:    sll s1, a4, s6
+; RV32I-NEXT:    bltu a5, t5, .LBB19_2
+; RV32I-NEXT:  # %bb.1:
+; RV32I-NEXT:    sra t0, a4, a5
+; RV32I-NEXT:    j .LBB19_3
+; RV32I-NEXT:  .LBB19_2:
+; RV32I-NEXT:    or t0, s0, s1
+; RV32I-NEXT:  .LBB19_3:
+; RV32I-NEXT:    or t1, a7, a3
+; RV32I-NEXT:    or a7, a1, a6
+; RV32I-NEXT:    mv t3, t2
+; RV32I-NEXT:    beqz a5, .LBB19_5
+; RV32I-NEXT:  # %bb.4:
+; RV32I-NEXT:    mv t3, t0
+; RV32I-NEXT:  .LBB19_5:
+; RV32I-NEXT:    srl a3, t1, a5
+; RV32I-NEXT:    sll a1, a7, s6
+; RV32I-NEXT:    sw a1, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    bltu a5, t5, .LBB19_7
+; RV32I-NEXT:  # %bb.6:
+; RV32I-NEXT:    srai t4, a4, 31
+; RV32I-NEXT:    srl a1, a7, a5
+; RV32I-NEXT:    j .LBB19_8
+; RV32I-NEXT:  .LBB19_7:
+; RV32I-NEXT:    sra t4, a4, a5
+; RV32I-NEXT:    or a1, a3, a1
+; RV32I-NEXT:  .LBB19_8:
+; RV32I-NEXT:    li t6, 64
+; RV32I-NEXT:    mv t0, t1
+; RV32I-NEXT:    beqz a5, .LBB19_10
+; RV32I-NEXT:  # %bb.9:
+; RV32I-NEXT:    mv t0, a1
+; RV32I-NEXT:  .LBB19_10:
+; RV32I-NEXT:    sub s7, t6, a5
+; RV32I-NEXT:    bltu a5, t5, .LBB19_12
+; RV32I-NEXT:  # %bb.11:
+; RV32I-NEXT:    li a1, 0
+; RV32I-NEXT:    j .LBB19_13
+; RV32I-NEXT:  .LBB19_12:
+; RV32I-NEXT:    srl a1, a7, a5
+; RV32I-NEXT:  .LBB19_13:
+; RV32I-NEXT:    sw a3, 28(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    neg s10, s7
+; RV32I-NEXT:    bltu s7, t5, .LBB19_15
+; RV32I-NEXT:  # %bb.14:
+; RV32I-NEXT:    li a6, 0
+; RV32I-NEXT:    sll a3, t2, s7
+; RV32I-NEXT:    j .LBB19_16
+; RV32I-NEXT:  .LBB19_15:
+; RV32I-NEXT:    sll a6, t2, s6
+; RV32I-NEXT:    srl a3, t2, s10
+; RV32I-NEXT:    or a3, a3, s1
+; RV32I-NEXT:  .LBB19_16:
+; RV32I-NEXT:    sw t3, 40(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    addi s9, a5, -64
+; RV32I-NEXT:    mv t3, a4
+; RV32I-NEXT:    beqz s7, .LBB19_18
+; RV32I-NEXT:  # %bb.17:
+; RV32I-NEXT:    mv t3, a3
+; RV32I-NEXT:  .LBB19_18:
+; RV32I-NEXT:    neg s11, s9
+; RV32I-NEXT:    sw s0, 32(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    bltu s9, t5, .LBB19_20
+; RV32I-NEXT:  # %bb.19:
+; RV32I-NEXT:    sra s0, a4, s9
+; RV32I-NEXT:    j .LBB19_21
+; RV32I-NEXT:  .LBB19_20:
+; RV32I-NEXT:    sll a3, a4, s11
+; RV32I-NEXT:    or s0, s0, a3
+; RV32I-NEXT:  .LBB19_21:
+; RV32I-NEXT:    sw s1, 16(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw t4, 36(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    lbu s3, 11(a0)
+; RV32I-NEXT:    lbu a3, 15(a0)
+; RV32I-NEXT:    mv t4, t2
+; RV32I-NEXT:    beqz s9, .LBB19_23
+; RV32I-NEXT:  # %bb.22:
+; RV32I-NEXT:    mv t4, s0
+; RV32I-NEXT:  .LBB19_23:
+; RV32I-NEXT:    lbu s2, 9(a0)
+; RV32I-NEXT:    lbu s1, 10(a0)
+; RV32I-NEXT:    lbu s8, 13(a0)
+; RV32I-NEXT:    lbu ra, 14(a0)
+; RV32I-NEXT:    slli s3, s3, 8
+; RV32I-NEXT:    slli a3, a3, 8
+; RV32I-NEXT:    bltu s9, t5, .LBB19_25
+; RV32I-NEXT:  # %bb.24:
+; RV32I-NEXT:    srai s0, a4, 31
+; RV32I-NEXT:    j .LBB19_26
+; RV32I-NEXT:  .LBB19_25:
+; RV32I-NEXT:    sra s0, a4, a5
+; RV32I-NEXT:  .LBB19_26:
+; RV32I-NEXT:    or s1, s3, s1
+; RV32I-NEXT:    lbu s5, 8(a0)
+; RV32I-NEXT:    lbu s3, 12(a0)
+; RV32I-NEXT:    slli s2, s2, 8
+; RV32I-NEXT:    slli s4, s8, 8
+; RV32I-NEXT:    or s8, a3, ra
+; RV32I-NEXT:    bgeu a5, t6, .LBB19_28
+; RV32I-NEXT:  # %bb.27:
+; RV32I-NEXT:    or t4, t0, a6
+; RV32I-NEXT:    or s0, a1, t3
+; RV32I-NEXT:  .LBB19_28:
+; RV32I-NEXT:    lbu a3, 3(a0)
+; RV32I-NEXT:    lbu t3, 7(a0)
+; RV32I-NEXT:    or a6, s2, s5
+; RV32I-NEXT:    slli s2, s1, 16
+; RV32I-NEXT:    or s1, s4, s3
+; RV32I-NEXT:    slli s8, s8, 16
+; RV32I-NEXT:    mv a1, t1
+; RV32I-NEXT:    mv t0, a7
+; RV32I-NEXT:    beqz a5, .LBB19_30
+; RV32I-NEXT:  # %bb.29:
+; RV32I-NEXT:    mv a1, t4
+; RV32I-NEXT:    mv t0, s0
+; RV32I-NEXT:  .LBB19_30:
+; RV32I-NEXT:    slli s5, a3, 8
+; RV32I-NEXT:    lbu ra, 1(a0)
+; RV32I-NEXT:    lbu a3, 2(a0)
+; RV32I-NEXT:    lbu s3, 5(a0)
+; RV32I-NEXT:    lbu s0, 6(a0)
+; RV32I-NEXT:    slli s4, t3, 8
+; RV32I-NEXT:    or t4, s2, a6
+; RV32I-NEXT:    or t3, s8, s1
+; RV32I-NEXT:    bltu a5, t6, .LBB19_32
+; RV32I-NEXT:  # %bb.31:
+; RV32I-NEXT:    srai a6, a4, 31
+; RV32I-NEXT:    sw a6, 40(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw a6, 36(sp) # 4-byte Folded Spill
+; RV32I-NEXT:  .LBB19_32:
+; RV32I-NEXT:    slli a6, ra, 8
+; RV32I-NEXT:    or a3, s5, a3
+; RV32I-NEXT:    lbu s1, 0(a0)
+; RV32I-NEXT:    lbu a0, 4(a0)
+; RV32I-NEXT:    slli s3, s3, 8
+; RV32I-NEXT:    or s0, s4, s0
+; RV32I-NEXT:    srl s2, t4, a5
+; RV32I-NEXT:    sll ra, t3, s6
+; RV32I-NEXT:    bltu a5, t5, .LBB19_34
+; RV32I-NEXT:  # %bb.33:
+; RV32I-NEXT:    srl s4, t3, a5
+; RV32I-NEXT:    j .LBB19_35
+; RV32I-NEXT:  .LBB19_34:
+; RV32I-NEXT:    or s4, s2, ra
+; RV32I-NEXT:  .LBB19_35:
+; RV32I-NEXT:    or a6, a6, s1
+; RV32I-NEXT:    slli a3, a3, 16
+; RV32I-NEXT:    or a0, s3, a0
+; RV32I-NEXT:    slli s1, s0, 16
+; RV32I-NEXT:    mv s5, t4
+; RV32I-NEXT:    beqz a5, .LBB19_37
+; RV32I-NEXT:  # %bb.36:
+; RV32I-NEXT:    mv s5, s4
+; RV32I-NEXT:  .LBB19_37:
+; RV32I-NEXT:    or s0, a3, a6
+; RV32I-NEXT:    or a0, s1, a0
+; RV32I-NEXT:    bltu a5, t5, .LBB19_39
+; RV32I-NEXT:  # %bb.38:
+; RV32I-NEXT:    li s4, 0
+; RV32I-NEXT:    srl a3, a0, a5
+; RV32I-NEXT:    mv a6, s0
+; RV32I-NEXT:    bnez a5, .LBB19_40
+; RV32I-NEXT:    j .LBB19_41
+; RV32I-NEXT:  .LBB19_39:
+; RV32I-NEXT:    srl s4, t3, a5
+; RV32I-NEXT:    srl a3, s0, a5
+; RV32I-NEXT:    sll a6, a0, s6
+; RV32I-NEXT:    or a3, a3, a6
+; RV32I-NEXT:    mv a6, s0
+; RV32I-NEXT:    beqz a5, .LBB19_41
+; RV32I-NEXT:  .LBB19_40:
+; RV32I-NEXT:    mv a6, a3
+; RV32I-NEXT:  .LBB19_41:
+; RV32I-NEXT:    bltu a5, t5, .LBB19_44
+; RV32I-NEXT:  # %bb.42:
+; RV32I-NEXT:    li s1, 0
+; RV32I-NEXT:    bgeu s7, t5, .LBB19_45
+; RV32I-NEXT:  .LBB19_43:
+; RV32I-NEXT:    sll s3, t4, s6
+; RV32I-NEXT:    srl a3, t4, s10
+; RV32I-NEXT:    or a3, a3, ra
+; RV32I-NEXT:    mv s10, t3
+; RV32I-NEXT:    bnez s7, .LBB19_46
+; RV32I-NEXT:    j .LBB19_47
+; RV32I-NEXT:  .LBB19_44:
+; RV32I-NEXT:    srl s1, a0, a5
+; RV32I-NEXT:    bltu s7, t5, .LBB19_43
+; RV32I-NEXT:  .LBB19_45:
+; RV32I-NEXT:    li s3, 0
+; RV32I-NEXT:    sll a3, t4, s7
+; RV32I-NEXT:    mv s10, t3
+; RV32I-NEXT:    beqz s7, .LBB19_47
+; RV32I-NEXT:  .LBB19_46:
+; RV32I-NEXT:    mv s10, a3
+; RV32I-NEXT:  .LBB19_47:
+; RV32I-NEXT:    bltu s9, t5, .LBB19_49
+; RV32I-NEXT:  # %bb.48:
+; RV32I-NEXT:    srl a3, t3, s9
+; RV32I-NEXT:    mv s2, t4
+; RV32I-NEXT:    bnez s9, .LBB19_50
+; RV32I-NEXT:    j .LBB19_51
+; RV32I-NEXT:  .LBB19_49:
+; RV32I-NEXT:    sll a3, t3, s11
+; RV32I-NEXT:    or a3, s2, a3
+; RV32I-NEXT:    mv s2, t4
+; RV32I-NEXT:    beqz s9, .LBB19_51
+; RV32I-NEXT:  .LBB19_50:
+; RV32I-NEXT:    mv s2, a3
+; RV32I-NEXT:  .LBB19_51:
+; RV32I-NEXT:    bltu s9, t5, .LBB19_53
+; RV32I-NEXT:  # %bb.52:
+; RV32I-NEXT:    li s7, 0
+; RV32I-NEXT:    bltu a5, t6, .LBB19_54
+; RV32I-NEXT:    j .LBB19_55
+; RV32I-NEXT:  .LBB19_53:
+; RV32I-NEXT:    srl s7, t3, a5
+; RV32I-NEXT:    bgeu a5, t6, .LBB19_55
+; RV32I-NEXT:  .LBB19_54:
+; RV32I-NEXT:    or s2, a6, s3
+; RV32I-NEXT:    or s7, s1, s10
+; RV32I-NEXT:  .LBB19_55:
+; RV32I-NEXT:    li a3, 128
+; RV32I-NEXT:    mv a6, s0
+; RV32I-NEXT:    mv s1, a0
+; RV32I-NEXT:    beqz a5, .LBB19_57
+; RV32I-NEXT:  # %bb.56:
+; RV32I-NEXT:    mv a6, s2
+; RV32I-NEXT:    mv s1, s7
+; RV32I-NEXT:  .LBB19_57:
+; RV32I-NEXT:    sw a6, 8(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sub s2, a3, a5
+; RV32I-NEXT:    bltu a5, t6, .LBB19_59
+; RV32I-NEXT:  # %bb.58:
+; RV32I-NEXT:    li s5, 0
+; RV32I-NEXT:    li s4, 0
+; RV32I-NEXT:  .LBB19_59:
+; RV32I-NEXT:    neg s3, s2
+; RV32I-NEXT:    srl a6, t1, s3
+; RV32I-NEXT:    sw s4, 20(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s1, 4(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    bltu s2, t5, .LBB19_61
+; RV32I-NEXT:  # %bb.60:
+; RV32I-NEXT:    li s11, 0
+; RV32I-NEXT:    sll a3, t1, s2
+; RV32I-NEXT:    j .LBB19_62
+; RV32I-NEXT:  .LBB19_61:
+; RV32I-NEXT:    sll s11, t1, s6
+; RV32I-NEXT:    lw a3, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    or a3, a6, a3
+; RV32I-NEXT:  .LBB19_62:
+; RV32I-NEXT:    sub s1, t6, s2
+; RV32I-NEXT:    mv s8, a7
+; RV32I-NEXT:    beqz s2, .LBB19_64
+; RV32I-NEXT:  # %bb.63:
+; RV32I-NEXT:    mv s8, a3
+; RV32I-NEXT:  .LBB19_64:
+; RV32I-NEXT:    bltu s1, t5, .LBB19_66
+; RV32I-NEXT:  # %bb.65:
+; RV32I-NEXT:    srl a3, a7, s1
+; RV32I-NEXT:    mv a6, t1
+; RV32I-NEXT:    bnez s1, .LBB19_67
+; RV32I-NEXT:    j .LBB19_68
+; RV32I-NEXT:  .LBB19_66:
+; RV32I-NEXT:    neg a3, s1
+; RV32I-NEXT:    sll a3, a7, a3
+; RV32I-NEXT:    or a3, a6, a3
+; RV32I-NEXT:    mv a6, t1
+; RV32I-NEXT:    beqz s1, .LBB19_68
+; RV32I-NEXT:  .LBB19_67:
+; RV32I-NEXT:    mv a6, a3
+; RV32I-NEXT:  .LBB19_68:
+; RV32I-NEXT:    bltu s1, t5, .LBB19_71
+; RV32I-NEXT:  # %bb.69:
+; RV32I-NEXT:    li s1, 0
+; RV32I-NEXT:    sw s5, 24(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    bgeu s2, t5, .LBB19_72
+; RV32I-NEXT:  .LBB19_70:
+; RV32I-NEXT:    sll s6, t2, s6
+; RV32I-NEXT:    srl a3, t2, s3
+; RV32I-NEXT:    lw s3, 16(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    or a3, a3, s3
+; RV32I-NEXT:    j .LBB19_73
+; RV32I-NEXT:  .LBB19_71:
+; RV32I-NEXT:    srl s1, a7, s3
+; RV32I-NEXT:    sw s5, 24(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    bltu s2, t5, .LBB19_70
+; RV32I-NEXT:  .LBB19_72:
+; RV32I-NEXT:    li s6, 0
+; RV32I-NEXT:    sll a3, t2, s2
+; RV32I-NEXT:  .LBB19_73:
+; RV32I-NEXT:    addi s9, s2, -64
+; RV32I-NEXT:    mv s5, a4
+; RV32I-NEXT:    beqz s2, .LBB19_75
+; RV32I-NEXT:  # %bb.74:
+; RV32I-NEXT:    mv s5, a3
+; RV32I-NEXT:  .LBB19_75:
+; RV32I-NEXT:    bltu s9, t5, .LBB19_77
+; RV32I-NEXT:  # %bb.76:
+; RV32I-NEXT:    li s3, 0
+; RV32I-NEXT:    sll a3, t1, s9
+; RV32I-NEXT:    mv s7, a7
+; RV32I-NEXT:    bnez s9, .LBB19_78
+; RV32I-NEXT:    j .LBB19_79
+; RV32I-NEXT:  .LBB19_77:
+; RV32I-NEXT:    sll s3, t1, s2
+; RV32I-NEXT:    neg a3, s9
+; RV32I-NEXT:    srl a3, t1, a3
+; RV32I-NEXT:    sll s4, a7, s2
+; RV32I-NEXT:    or a3, a3, s4
+; RV32I-NEXT:    mv s7, a7
+; RV32I-NEXT:    beqz s9, .LBB19_79
+; RV32I-NEXT:  .LBB19_78:
+; RV32I-NEXT:    mv s7, a3
+; RV32I-NEXT:  .LBB19_79:
+; RV32I-NEXT:    bltu s2, t6, .LBB19_81
+; RV32I-NEXT:  # %bb.80:
+; RV32I-NEXT:    li s11, 0
+; RV32I-NEXT:    li s8, 0
+; RV32I-NEXT:    j .LBB19_82
+; RV32I-NEXT:  .LBB19_81:
+; RV32I-NEXT:    or s3, a6, s6
+; RV32I-NEXT:    or s7, s1, s5
+; RV32I-NEXT:  .LBB19_82:
+; RV32I-NEXT:    addi ra, a5, -128
+; RV32I-NEXT:    mv s4, t2
+; RV32I-NEXT:    mv s6, a4
+; RV32I-NEXT:    beqz s2, .LBB19_84
+; RV32I-NEXT:  # %bb.83:
+; RV32I-NEXT:    mv s4, s3
+; RV32I-NEXT:    mv s6, s7
+; RV32I-NEXT:  .LBB19_84:
+; RV32I-NEXT:    neg s9, ra
+; RV32I-NEXT:    sll s3, a4, s9
+; RV32I-NEXT:    bltu ra, t5, .LBB19_86
+; RV32I-NEXT:  # %bb.85:
+; RV32I-NEXT:    sra a3, a4, ra
+; RV32I-NEXT:    mv s1, t2
+; RV32I-NEXT:    bnez ra, .LBB19_87
+; RV32I-NEXT:    j .LBB19_88
+; RV32I-NEXT:  .LBB19_86:
+; RV32I-NEXT:    lw a3, 32(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    or a3, a3, s3
+; RV32I-NEXT:    mv s1, t2
+; RV32I-NEXT:    beqz ra, .LBB19_88
+; RV32I-NEXT:  .LBB19_87:
+; RV32I-NEXT:    mv s1, a3
+; RV32I-NEXT:  .LBB19_88:
+; RV32I-NEXT:    bltu ra, t5, .LBB19_90
+; RV32I-NEXT:  # %bb.89:
+; RV32I-NEXT:    srai s2, a4, 31
+; RV32I-NEXT:    srl a3, a7, ra
+; RV32I-NEXT:    mv a6, t1
+; RV32I-NEXT:    bnez ra, .LBB19_91
+; RV32I-NEXT:    j .LBB19_92
+; RV32I-NEXT:  .LBB19_90:
+; RV32I-NEXT:    sra s2, a4, a5
+; RV32I-NEXT:    sll a3, a7, s9
+; RV32I-NEXT:    lw a6, 28(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    or a3, a6, a3
+; RV32I-NEXT:    mv a6, t1
+; RV32I-NEXT:    beqz ra, .LBB19_92
+; RV32I-NEXT:  .LBB19_91:
+; RV32I-NEXT:    mv a6, a3
+; RV32I-NEXT:  .LBB19_92:
+; RV32I-NEXT:    mv s5, t0
+; RV32I-NEXT:    sub s10, t6, ra
+; RV32I-NEXT:    li t0, 64
+; RV32I-NEXT:    bltu ra, t5, .LBB19_94
+; RV32I-NEXT:  # %bb.93:
+; RV32I-NEXT:    li s7, 0
+; RV32I-NEXT:    j .LBB19_95
+; RV32I-NEXT:  .LBB19_94:
+; RV32I-NEXT:    srl s7, a7, a5
+; RV32I-NEXT:  .LBB19_95:
+; RV32I-NEXT:    mv t6, s8
+; RV32I-NEXT:    mv s8, s11
+; RV32I-NEXT:    bltu s10, t5, .LBB19_97
+; RV32I-NEXT:  # %bb.96:
+; RV32I-NEXT:    li s9, 0
+; RV32I-NEXT:    sll a3, t2, s10
+; RV32I-NEXT:    j .LBB19_98
+; RV32I-NEXT:  .LBB19_97:
+; RV32I-NEXT:    sll s9, t2, s9
+; RV32I-NEXT:    neg a3, s10
+; RV32I-NEXT:    srl a3, t2, a3
+; RV32I-NEXT:    or a3, a3, s3
+; RV32I-NEXT:  .LBB19_98:
+; RV32I-NEXT:    addi s11, ra, -64
+; RV32I-NEXT:    mv s3, a4
+; RV32I-NEXT:    beqz s10, .LBB19_100
+; RV32I-NEXT:  # %bb.99:
+; RV32I-NEXT:    mv s3, a3
+; RV32I-NEXT:  .LBB19_100:
+; RV32I-NEXT:    bltu s11, t5, .LBB19_102
+; RV32I-NEXT:  # %bb.101:
+; RV32I-NEXT:    sra a3, a4, s11
+; RV32I-NEXT:    bnez s11, .LBB19_103
+; RV32I-NEXT:    j .LBB19_104
+; RV32I-NEXT:  .LBB19_102:
+; RV32I-NEXT:    srl a3, t2, ra
+; RV32I-NEXT:    mv s10, s4
+; RV32I-NEXT:    neg s4, s11
+; RV32I-NEXT:    sll s4, a4, s4
+; RV32I-NEXT:    or a3, a3, s4
+; RV32I-NEXT:    mv s4, s10
+; RV32I-NEXT:    beqz s11, .LBB19_104
+; RV32I-NEXT:  .LBB19_103:
+; RV32I-NEXT:    mv t2, a3
+; RV32I-NEXT:  .LBB19_104:
+; RV32I-NEXT:    bltu s11, t5, .LBB19_106
+; RV32I-NEXT:  # %bb.105:
+; RV32I-NEXT:    srai t5, a4, 31
+; RV32I-NEXT:    lw s11, 40(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    bltu ra, t0, .LBB19_107
+; RV32I-NEXT:    j .LBB19_108
+; RV32I-NEXT:  .LBB19_106:
+; RV32I-NEXT:    sra t5, a4, ra
+; RV32I-NEXT:    lw s11, 40(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    bgeu ra, t0, .LBB19_108
+; RV32I-NEXT:  .LBB19_107:
+; RV32I-NEXT:    or t2, a6, s9
+; RV32I-NEXT:    or t5, s7, s3
+; RV32I-NEXT:  .LBB19_108:
+; RV32I-NEXT:    li a6, 128
+; RV32I-NEXT:    bnez ra, .LBB19_117
+; RV32I-NEXT:  # %bb.109:
+; RV32I-NEXT:    bgeu ra, t0, .LBB19_118
+; RV32I-NEXT:  .LBB19_110:
+; RV32I-NEXT:    bgeu a5, a6, .LBB19_112
+; RV32I-NEXT:  .LBB19_111:
+; RV32I-NEXT:    lw a3, 8(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    or t1, a3, s8
+; RV32I-NEXT:    lw a3, 4(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    or a7, a3, t6
+; RV32I-NEXT:    lw a3, 24(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    or s1, a3, s4
+; RV32I-NEXT:    lw a3, 20(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    or s2, a3, s6
+; RV32I-NEXT:  .LBB19_112:
+; RV32I-NEXT:    lw ra, 36(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    mv t0, s5
+; RV32I-NEXT:    beqz a5, .LBB19_114
+; RV32I-NEXT:  # %bb.113:
+; RV32I-NEXT:    mv s0, t1
+; RV32I-NEXT:    mv a0, a7
+; RV32I-NEXT:    mv t4, s1
+; RV32I-NEXT:    mv t3, s2
+; RV32I-NEXT:  .LBB19_114:
+; RV32I-NEXT:    bltu a5, a6, .LBB19_116
+; RV32I-NEXT:  # %bb.115:
+; RV32I-NEXT:    srai a1, a4, 31
+; RV32I-NEXT:    mv t0, a1
+; RV32I-NEXT:    mv s11, a1
+; RV32I-NEXT:    mv ra, a1
+; RV32I-NEXT:  .LBB19_116:
+; RV32I-NEXT:    srli a4, s0, 16
+; RV32I-NEXT:    lui t1, 16
+; RV32I-NEXT:    srli a7, s0, 24
+; RV32I-NEXT:    srli a5, a0, 16
+; RV32I-NEXT:    srli t5, a0, 24
+; RV32I-NEXT:    srli a6, t4, 16
+; RV32I-NEXT:    srli s2, t4, 24
+; RV32I-NEXT:    srli t2, t3, 16
+; RV32I-NEXT:    srli s3, t3, 24
+; RV32I-NEXT:    srli s1, a1, 16
+; RV32I-NEXT:    srli a3, a1, 24
+; RV32I-NEXT:    srli t6, t0, 16
+; RV32I-NEXT:    srli s6, t0, 24
+; RV32I-NEXT:    srli s5, s11, 16
+; RV32I-NEXT:    srli s4, s11, 24
+; RV32I-NEXT:    srli s7, ra, 16
+; RV32I-NEXT:    srli s8, ra, 24
+; RV32I-NEXT:    addi t1, t1, -1
+; RV32I-NEXT:    and s9, s0, t1
+; RV32I-NEXT:    and s10, a0, t1
+; RV32I-NEXT:    srli s9, s9, 8
+; RV32I-NEXT:    sb s0, 0(a2)
+; RV32I-NEXT:    sb s9, 1(a2)
+; RV32I-NEXT:    sb a4, 2(a2)
+; RV32I-NEXT:    sb a7, 3(a2)
+; RV32I-NEXT:    and a4, t4, t1
+; RV32I-NEXT:    srli a7, s10, 8
+; RV32I-NEXT:    sb a0, 4(a2)
+; RV32I-NEXT:    sb a7, 5(a2)
+; RV32I-NEXT:    sb a5, 6(a2)
+; RV32I-NEXT:    sb t5, 7(a2)
+; RV32I-NEXT:    and a0, t3, t1
+; RV32I-NEXT:    srli a4, a4, 8
+; RV32I-NEXT:    sb t4, 8(a2)
+; RV32I-NEXT:    sb a4, 9(a2)
+; RV32I-NEXT:    sb a6, 10(a2)
+; RV32I-NEXT:    sb s2, 11(a2)
+; RV32I-NEXT:    and a4, a1, t1
+; RV32I-NEXT:    srli a0, a0, 8
+; RV32I-NEXT:    sb t3, 12(a2)
+; RV32I-NEXT:    sb a0, 13(a2)
+; RV32I-NEXT:    sb t2, 14(a2)
+; RV32I-NEXT:    sb s3, 15(a2)
+; RV32I-NEXT:    and a0, t0, t1
+; RV32I-NEXT:    srli a4, a4, 8
+; RV32I-NEXT:    sb a1, 16(a2)
+; RV32I-NEXT:    sb a4, 17(a2)
+; RV32I-NEXT:    sb s1, 18(a2)
+; RV32I-NEXT:    sb a3, 19(a2)
+; RV32I-NEXT:    and a1, s11, t1
+; RV32I-NEXT:    and a3, ra, t1
+; RV32I-NEXT:    srli a0, a0, 8
+; RV32I-NEXT:    srli a1, a1, 8
+; RV32I-NEXT:    srli a3, a3, 8
+; RV32I-NEXT:    sb t0, 20(a2)
+; RV32I-NEXT:    sb a0, 21(a2)
+; RV32I-NEXT:    sb t6, 22(a2)
+; RV32I-NEXT:    sb s6, 23(a2)
+; RV32I-NEXT:    sb s11, 24(a2)
+; RV32I-NEXT:    sb a1, 25(a2)
+; RV32I-NEXT:    sb s5, 26(a2)
+; RV32I-NEXT:    sb s4, 27(a2)
+; RV32I-NEXT:    sb ra, 28(a2)
+; RV32I-NEXT:    sb a3, 29(a2)
+; RV32I-NEXT:    sb s7, 30(a2)
+; RV32I-NEXT:    sb s8, 31(a2)
+; RV32I-NEXT:    lw ra, 92(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s0, 88(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s1, 84(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s2, 80(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s3, 76(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s4, 72(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s5, 68(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s6, 64(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s7, 60(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s8, 56(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s9, 52(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s10, 48(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s11, 44(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    addi sp, sp, 96
+; RV32I-NEXT:    ret
+; RV32I-NEXT:  .LBB19_117:
+; RV32I-NEXT:    mv t1, t2
+; RV32I-NEXT:    mv a7, t5
+; RV32I-NEXT:    bltu ra, t0, .LBB19_110
+; RV32I-NEXT:  .LBB19_118:
+; RV32I-NEXT:    srai s1, a4, 31
+; RV32I-NEXT:    mv s2, s1
+; RV32I-NEXT:    bltu a5, a6, .LBB19_111
+; RV32I-NEXT:    j .LBB19_112
+  %src = load i256, ptr %src.ptr, align 1
+  %wordOff = load i256, ptr %wordOff.ptr, align 1
+  %bitOff = shl i256 %wordOff, 5
+  %res = ashr i256 %src, %bitOff
+  store i256 %res, ptr %dst, align 1
+  ret void
+}
+
+define void @ashr_32bytes_dwordOff(ptr %src.ptr, ptr %dwordOff.ptr, ptr %dst) nounwind {
+; RV64I-LABEL: ashr_32bytes_dwordOff:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -96
+; RV64I-NEXT:    sd s0, 88(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s1, 80(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s2, 72(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s3, 64(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s4, 56(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s5, 48(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s6, 40(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s7, 32(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s8, 24(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s9, 16(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s10, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s11, 0(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    lbu a3, 0(a0)
+; RV64I-NEXT:    lbu a4, 1(a0)
+; RV64I-NEXT:    lbu a5, 2(a0)
+; RV64I-NEXT:    lbu a6, 3(a0)
+; RV64I-NEXT:    lbu a7, 4(a0)
+; RV64I-NEXT:    lbu t0, 5(a0)
+; RV64I-NEXT:    lbu t1, 6(a0)
+; RV64I-NEXT:    lbu t2, 7(a0)
+; RV64I-NEXT:    lbu t3, 8(a0)
+; RV64I-NEXT:    lbu t4, 9(a0)
+; RV64I-NEXT:    lbu t5, 10(a0)
+; RV64I-NEXT:    lbu t6, 11(a0)
+; RV64I-NEXT:    lbu s0, 12(a0)
+; RV64I-NEXT:    lbu s1, 13(a0)
+; RV64I-NEXT:    lbu s2, 14(a0)
+; RV64I-NEXT:    lbu s3, 15(a0)
+; RV64I-NEXT:    lbu s4, 16(a0)
+; RV64I-NEXT:    lbu s5, 17(a0)
+; RV64I-NEXT:    lbu s6, 18(a0)
+; RV64I-NEXT:    lbu s7, 19(a0)
+; RV64I-NEXT:    slli a4, a4, 8
+; RV64I-NEXT:    slli a6, a6, 8
+; RV64I-NEXT:    slli t0, t0, 8
+; RV64I-NEXT:    or a3, a4, a3
+; RV64I-NEXT:    or a5, a6, a5
+; RV64I-NEXT:    or a7, t0, a7
+; RV64I-NEXT:    lbu s8, 20(a0)
+; RV64I-NEXT:    lbu s9, 21(a0)
+; RV64I-NEXT:    lbu s10, 22(a0)
+; RV64I-NEXT:    lbu s11, 23(a0)
+; RV64I-NEXT:    slli t2, t2, 8
+; RV64I-NEXT:    slli t4, t4, 8
+; RV64I-NEXT:    slli t6, t6, 8
+; RV64I-NEXT:    slli s1, s1, 8
+; RV64I-NEXT:    or t1, t2, t1
+; RV64I-NEXT:    or a4, t4, t3
+; RV64I-NEXT:    or a6, t6, t5
+; RV64I-NEXT:    or t0, s1, s0
+; RV64I-NEXT:    lbu t5, 24(a0)
+; RV64I-NEXT:    lbu t6, 25(a0)
+; RV64I-NEXT:    lbu s0, 26(a0)
+; RV64I-NEXT:    lbu s1, 27(a0)
+; RV64I-NEXT:    slli s3, s3, 8
+; RV64I-NEXT:    slli s5, s5, 8
+; RV64I-NEXT:    slli s7, s7, 8
+; RV64I-NEXT:    or t4, s3, s2
+; RV64I-NEXT:    or t2, s5, s4
+; RV64I-NEXT:    or t3, s7, s6
+; RV64I-NEXT:    lbu s2, 28(a0)
+; RV64I-NEXT:    lbu s3, 29(a0)
+; RV64I-NEXT:    lbu s4, 30(a0)
+; RV64I-NEXT:    lbu a0, 31(a0)
+; RV64I-NEXT:    slli s9, s9, 8
+; RV64I-NEXT:    slli s11, s11, 8
+; RV64I-NEXT:    slli t6, t6, 8
+; RV64I-NEXT:    slli s1, s1, 8
+; RV64I-NEXT:    or s5, s9, s8
+; RV64I-NEXT:    or s6, s11, s10
+; RV64I-NEXT:    or t5, t6, t5
+; RV64I-NEXT:    or s0, s1, s0
+; RV64I-NEXT:    lbu t6, 0(a1)
+; RV64I-NEXT:    lbu s1, 1(a1)
+; RV64I-NEXT:    lbu s7, 2(a1)
+; RV64I-NEXT:    lbu s8, 3(a1)
+; RV64I-NEXT:    slli s3, s3, 8
+; RV64I-NEXT:    slli a0, a0, 8
+; RV64I-NEXT:    slli s1, s1, 8
+; RV64I-NEXT:    or s2, s3, s2
+; RV64I-NEXT:    or s3, a0, s4
+; RV64I-NEXT:    or t6, s1, t6
+; RV64I-NEXT:    lbu a0, 4(a1)
+; RV64I-NEXT:    lbu s1, 5(a1)
+; RV64I-NEXT:    lbu s4, 6(a1)
+; RV64I-NEXT:    lbu a1, 7(a1)
+; RV64I-NEXT:    slli s8, s8, 8
+; RV64I-NEXT:    or s7, s8, s7
+; RV64I-NEXT:    slli s1, s1, 8
+; RV64I-NEXT:    or s1, s1, a0
+; RV64I-NEXT:    slli a1, a1, 8
+; RV64I-NEXT:    or s4, a1, s4
+; RV64I-NEXT:    slli t1, t1, 16
+; RV64I-NEXT:    or a1, t1, a7
+; RV64I-NEXT:    slli t4, t4, 16
+; RV64I-NEXT:    or a0, t4, t0
+; RV64I-NEXT:    slli t3, t3, 16
+; RV64I-NEXT:    or a7, t3, t2
+; RV64I-NEXT:    slli s6, s6, 16
+; RV64I-NEXT:    or t1, s6, s5
+; RV64I-NEXT:    slli s0, s0, 16
+; RV64I-NEXT:    or t4, s0, t5
+; RV64I-NEXT:    slli s3, s3, 16
+; RV64I-NEXT:    or t5, s3, s2
+; RV64I-NEXT:    slli s7, s7, 16
+; RV64I-NEXT:    or t6, s7, t6
+; RV64I-NEXT:    slli s4, s4, 16
+; RV64I-NEXT:    or s0, s4, s1
+; RV64I-NEXT:    li t0, 64
+; RV64I-NEXT:    slli t3, a5, 16
+; RV64I-NEXT:    slli t2, a6, 16
+; RV64I-NEXT:    slli t1, t1, 32
+; RV64I-NEXT:    slli t5, t5, 32
+; RV64I-NEXT:    slli s0, s0, 32
+; RV64I-NEXT:    or a7, t1, a7
+; RV64I-NEXT:    or a5, t5, t4
+; RV64I-NEXT:    or a6, s0, t6
+; RV64I-NEXT:    slli a6, a6, 6
+; RV64I-NEXT:    subw t1, a6, t0
+; RV64I-NEXT:    negw t5, a6
+; RV64I-NEXT:    sll t4, a5, t5
+; RV64I-NEXT:    bltu a6, t0, .LBB20_2
+; RV64I-NEXT:  # %bb.1:
+; RV64I-NEXT:    sra t6, a5, t1
+; RV64I-NEXT:    j .LBB20_3
+; RV64I-NEXT:  .LBB20_2:
+; RV64I-NEXT:    srl t6, a7, a6
+; RV64I-NEXT:    or t6, t6, t4
+; RV64I-NEXT:  .LBB20_3:
+; RV64I-NEXT:    or a3, t3, a3
+; RV64I-NEXT:    slli t3, a1, 32
+; RV64I-NEXT:    or t2, t2, a4
+; RV64I-NEXT:    slli a0, a0, 32
+; RV64I-NEXT:    mv a1, a7
+; RV64I-NEXT:    beqz a6, .LBB20_5
+; RV64I-NEXT:  # %bb.4:
+; RV64I-NEXT:    mv a1, t6
+; RV64I-NEXT:  .LBB20_5:
+; RV64I-NEXT:    or a4, t3, a3
+; RV64I-NEXT:    or a3, a0, t2
+; RV64I-NEXT:    bltu a6, t0, .LBB20_7
+; RV64I-NEXT:  # %bb.6:
+; RV64I-NEXT:    srai a0, a5, 63
+; RV64I-NEXT:    srl t3, a3, t1
+; RV64I-NEXT:    j .LBB20_8
+; RV64I-NEXT:  .LBB20_7:
+; RV64I-NEXT:    sra a0, a5, a6
+; RV64I-NEXT:    srl t1, a4, a6
+; RV64I-NEXT:    sll t2, a3, t5
+; RV64I-NEXT:    or t3, t1, t2
+; RV64I-NEXT:  .LBB20_8:
+; RV64I-NEXT:    li t1, 128
+; RV64I-NEXT:    mv t2, a4
+; RV64I-NEXT:    beqz a6, .LBB20_10
+; RV64I-NEXT:  # %bb.9:
+; RV64I-NEXT:    mv t2, t3
+; RV64I-NEXT:  .LBB20_10:
+; RV64I-NEXT:    sub t6, t1, a6
+; RV64I-NEXT:    bltu a6, t0, .LBB20_13
+; RV64I-NEXT:  # %bb.11:
+; RV64I-NEXT:    li t3, 0
+; RV64I-NEXT:    bgeu t6, t0, .LBB20_14
+; RV64I-NEXT:  .LBB20_12:
+; RV64I-NEXT:    sll t5, a7, t5
+; RV64I-NEXT:    negw s0, t6
+; RV64I-NEXT:    srl s0, a7, s0
+; RV64I-NEXT:    or s1, s0, t4
+; RV64I-NEXT:    j .LBB20_15
+; RV64I-NEXT:  .LBB20_13:
+; RV64I-NEXT:    srl t3, a3, a6
+; RV64I-NEXT:    bltu t6, t0, .LBB20_12
+; RV64I-NEXT:  .LBB20_14:
+; RV64I-NEXT:    li t5, 0
+; RV64I-NEXT:    subw t4, t6, t0
+; RV64I-NEXT:    sll s1, a7, t4
+; RV64I-NEXT:  .LBB20_15:
+; RV64I-NEXT:    sub s0, a6, t1
+; RV64I-NEXT:    mv t4, a5
+; RV64I-NEXT:    beqz t6, .LBB20_17
+; RV64I-NEXT:  # %bb.16:
+; RV64I-NEXT:    mv t4, s1
+; RV64I-NEXT:  .LBB20_17:
+; RV64I-NEXT:    bltu s0, t0, .LBB20_19
+; RV64I-NEXT:  # %bb.18:
+; RV64I-NEXT:    subw t6, s0, t0
+; RV64I-NEXT:    sra t6, a5, t6
+; RV64I-NEXT:    bnez s0, .LBB20_20
+; RV64I-NEXT:    j .LBB20_21
+; RV64I-NEXT:  .LBB20_19:
+; RV64I-NEXT:    srl t6, a7, s0
+; RV64I-NEXT:    negw s1, s0
+; RV64I-NEXT:    sll s1, a5, s1
+; RV64I-NEXT:    or t6, t6, s1
+; RV64I-NEXT:    beqz s0, .LBB20_21
+; RV64I-NEXT:  .LBB20_20:
+; RV64I-NEXT:    mv a7, t6
+; RV64I-NEXT:  .LBB20_21:
+; RV64I-NEXT:    bltu s0, t0, .LBB20_23
+; RV64I-NEXT:  # %bb.22:
+; RV64I-NEXT:    srai t0, a5, 63
+; RV64I-NEXT:    bltu a6, t1, .LBB20_24
+; RV64I-NEXT:    j .LBB20_25
+; RV64I-NEXT:  .LBB20_23:
+; RV64I-NEXT:    sra t0, a5, s0
+; RV64I-NEXT:    bgeu a6, t1, .LBB20_25
+; RV64I-NEXT:  .LBB20_24:
+; RV64I-NEXT:    or a7, t2, t5
+; RV64I-NEXT:    or t0, t3, t4
+; RV64I-NEXT:  .LBB20_25:
+; RV64I-NEXT:    bnez a6, .LBB20_29
+; RV64I-NEXT:  # %bb.26:
+; RV64I-NEXT:    bltu a6, t1, .LBB20_28
+; RV64I-NEXT:  .LBB20_27:
+; RV64I-NEXT:    srai a1, a5, 63
+; RV64I-NEXT:    mv a0, a1
+; RV64I-NEXT:  .LBB20_28:
+; RV64I-NEXT:    srli a5, a4, 32
+; RV64I-NEXT:    srliw a6, a4, 16
+; RV64I-NEXT:    lui t2, 16
+; RV64I-NEXT:    srliw t1, a4, 24
+; RV64I-NEXT:    srli t0, a4, 48
+; RV64I-NEXT:    srli t5, a4, 56
+; RV64I-NEXT:    srli a7, a3, 32
+; RV64I-NEXT:    srliw t4, a3, 16
+; RV64I-NEXT:    srliw s0, a3, 24
+; RV64I-NEXT:    srli t6, a3, 48
+; RV64I-NEXT:    srli s3, a3, 56
+; RV64I-NEXT:    srli t3, a1, 32
+; RV64I-NEXT:    srliw s2, a1, 16
+; RV64I-NEXT:    srliw s6, a1, 24
+; RV64I-NEXT:    srli s4, a1, 48
+; RV64I-NEXT:    srli s7, a1, 56
+; RV64I-NEXT:    srli s1, a0, 32
+; RV64I-NEXT:    srliw s5, a0, 16
+; RV64I-NEXT:    srliw s8, a0, 24
+; RV64I-NEXT:    srli s9, a0, 48
+; RV64I-NEXT:    srli s10, a0, 56
+; RV64I-NEXT:    addi t2, t2, -1
+; RV64I-NEXT:    and s11, a4, t2
+; RV64I-NEXT:    srli s11, s11, 8
+; RV64I-NEXT:    sb a4, 0(a2)
+; RV64I-NEXT:    sb s11, 1(a2)
+; RV64I-NEXT:    sb a6, 2(a2)
+; RV64I-NEXT:    sb t1, 3(a2)
+; RV64I-NEXT:    and a4, a5, t2
+; RV64I-NEXT:    srli a4, a4, 8
+; RV64I-NEXT:    sb a5, 4(a2)
+; RV64I-NEXT:    sb a4, 5(a2)
+; RV64I-NEXT:    sb t0, 6(a2)
+; RV64I-NEXT:    sb t5, 7(a2)
+; RV64I-NEXT:    and a4, a3, t2
+; RV64I-NEXT:    srli a4, a4, 8
+; RV64I-NEXT:    sb a3, 8(a2)
+; RV64I-NEXT:    sb a4, 9(a2)
+; RV64I-NEXT:    sb t4, 10(a2)
+; RV64I-NEXT:    sb s0, 11(a2)
+; RV64I-NEXT:    and a3, a7, t2
+; RV64I-NEXT:    srli a3, a3, 8
+; RV64I-NEXT:    sb a7, 12(a2)
+; RV64I-NEXT:    sb a3, 13(a2)
+; RV64I-NEXT:    sb t6, 14(a2)
+; RV64I-NEXT:    sb s3, 15(a2)
+; RV64I-NEXT:    and a3, a1, t2
+; RV64I-NEXT:    srli a3, a3, 8
+; RV64I-NEXT:    sb a1, 16(a2)
+; RV64I-NEXT:    sb a3, 17(a2)
+; RV64I-NEXT:    sb s2, 18(a2)
+; RV64I-NEXT:    sb s6, 19(a2)
+; RV64I-NEXT:    and a1, t3, t2
+; RV64I-NEXT:    srli a1, a1, 8
+; RV64I-NEXT:    sb t3, 20(a2)
+; RV64I-NEXT:    sb a1, 21(a2)
+; RV64I-NEXT:    sb s4, 22(a2)
+; RV64I-NEXT:    sb s7, 23(a2)
+; RV64I-NEXT:    and a1, a0, t2
+; RV64I-NEXT:    and a3, s1, t2
+; RV64I-NEXT:    srli a1, a1, 8
+; RV64I-NEXT:    srli a3, a3, 8
+; RV64I-NEXT:    sb a0, 24(a2)
+; RV64I-NEXT:    sb a1, 25(a2)
+; RV64I-NEXT:    sb s5, 26(a2)
+; RV64I-NEXT:    sb s8, 27(a2)
+; RV64I-NEXT:    sb s1, 28(a2)
+; RV64I-NEXT:    sb a3, 29(a2)
+; RV64I-NEXT:    sb s9, 30(a2)
+; RV64I-NEXT:    sb s10, 31(a2)
+; RV64I-NEXT:    ld s0, 88(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s1, 80(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s2, 72(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s3, 64(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s4, 56(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s5, 48(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s6, 40(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s7, 32(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s8, 24(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s9, 16(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s10, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s11, 0(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    addi sp, sp, 96
+; RV64I-NEXT:    ret
+; RV64I-NEXT:  .LBB20_29:
+; RV64I-NEXT:    mv a4, a7
+; RV64I-NEXT:    mv a3, t0
+; RV64I-NEXT:    bgeu a6, t1, .LBB20_27
+; RV64I-NEXT:    j .LBB20_28
+;
+; RV32I-LABEL: ashr_32bytes_dwordOff:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    addi sp, sp, -96
+; RV32I-NEXT:    sw ra, 92(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s0, 88(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s1, 84(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s2, 80(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s3, 76(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s4, 72(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s5, 68(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s6, 64(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s7, 60(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s8, 56(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s9, 52(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s10, 48(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s11, 44(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    lbu a3, 16(a0)
+; RV32I-NEXT:    lbu a4, 17(a0)
+; RV32I-NEXT:    lbu a5, 18(a0)
+; RV32I-NEXT:    lbu a6, 19(a0)
+; RV32I-NEXT:    lbu a7, 20(a0)
+; RV32I-NEXT:    lbu t0, 21(a0)
+; RV32I-NEXT:    lbu t1, 22(a0)
+; RV32I-NEXT:    lbu t2, 23(a0)
+; RV32I-NEXT:    lbu t3, 24(a0)
+; RV32I-NEXT:    lbu t4, 25(a0)
+; RV32I-NEXT:    lbu t5, 26(a0)
+; RV32I-NEXT:    lbu t6, 27(a0)
+; RV32I-NEXT:    slli a4, a4, 8
+; RV32I-NEXT:    slli a6, a6, 8
+; RV32I-NEXT:    slli t0, t0, 8
+; RV32I-NEXT:    slli t2, t2, 8
+; RV32I-NEXT:    or a3, a4, a3
+; RV32I-NEXT:    or a4, a6, a5
+; RV32I-NEXT:    or a6, t0, a7
+; RV32I-NEXT:    or a5, t2, t1
+; RV32I-NEXT:    lbu a7, 28(a0)
+; RV32I-NEXT:    lbu t0, 29(a0)
+; RV32I-NEXT:    lbu t1, 30(a0)
+; RV32I-NEXT:    lbu t2, 31(a0)
+; RV32I-NEXT:    slli t4, t4, 8
+; RV32I-NEXT:    slli t6, t6, 8
+; RV32I-NEXT:    slli t0, t0, 8
+; RV32I-NEXT:    or t3, t4, t3
+; RV32I-NEXT:    or t4, t6, t5
+; RV32I-NEXT:    or t0, t0, a7
+; RV32I-NEXT:    lbu a7, 0(a1)
+; RV32I-NEXT:    lbu t5, 1(a1)
+; RV32I-NEXT:    lbu t6, 2(a1)
+; RV32I-NEXT:    lbu a1, 3(a1)
+; RV32I-NEXT:    slli t2, t2, 8
+; RV32I-NEXT:    or t1, t2, t1
+; RV32I-NEXT:    slli t5, t5, 8
+; RV32I-NEXT:    or s0, t5, a7
+; RV32I-NEXT:    slli a1, a1, 8
+; RV32I-NEXT:    or t2, a1, t6
+; RV32I-NEXT:    li t5, 32
+; RV32I-NEXT:    slli a7, a4, 16
+; RV32I-NEXT:    slli a1, a5, 16
+; RV32I-NEXT:    slli t4, t4, 16
+; RV32I-NEXT:    slli t1, t1, 16
+; RV32I-NEXT:    slli a5, t2, 16
+; RV32I-NEXT:    or t2, t4, t3
+; RV32I-NEXT:    or a4, t1, t0
+; RV32I-NEXT:    or a5, a5, s0
+; RV32I-NEXT:    slli a5, a5, 6
+; RV32I-NEXT:    srl s0, t2, a5
+; RV32I-NEXT:    neg s6, a5
+; RV32I-NEXT:    sll s1, a4, s6
+; RV32I-NEXT:    bltu a5, t5, .LBB20_2
+; RV32I-NEXT:  # %bb.1:
+; RV32I-NEXT:    sra t0, a4, a5
+; RV32I-NEXT:    j .LBB20_3
+; RV32I-NEXT:  .LBB20_2:
+; RV32I-NEXT:    or t0, s0, s1
+; RV32I-NEXT:  .LBB20_3:
+; RV32I-NEXT:    or t1, a7, a3
+; RV32I-NEXT:    or a7, a1, a6
+; RV32I-NEXT:    mv t3, t2
+; RV32I-NEXT:    beqz a5, .LBB20_5
+; RV32I-NEXT:  # %bb.4:
+; RV32I-NEXT:    mv t3, t0
+; RV32I-NEXT:  .LBB20_5:
+; RV32I-NEXT:    srl a3, t1, a5
+; RV32I-NEXT:    sll a1, a7, s6
+; RV32I-NEXT:    sw a1, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    bltu a5, t5, .LBB20_7
+; RV32I-NEXT:  # %bb.6:
+; RV32I-NEXT:    srai t4, a4, 31
+; RV32I-NEXT:    srl a1, a7, a5
+; RV32I-NEXT:    j .LBB20_8
+; RV32I-NEXT:  .LBB20_7:
+; RV32I-NEXT:    sra t4, a4, a5
+; RV32I-NEXT:    or a1, a3, a1
+; RV32I-NEXT:  .LBB20_8:
+; RV32I-NEXT:    li t6, 64
+; RV32I-NEXT:    mv t0, t1
+; RV32I-NEXT:    beqz a5, .LBB20_10
+; RV32I-NEXT:  # %bb.9:
+; RV32I-NEXT:    mv t0, a1
+; RV32I-NEXT:  .LBB20_10:
+; RV32I-NEXT:    sub s7, t6, a5
+; RV32I-NEXT:    bltu a5, t5, .LBB20_12
+; RV32I-NEXT:  # %bb.11:
+; RV32I-NEXT:    li a1, 0
+; RV32I-NEXT:    j .LBB20_13
+; RV32I-NEXT:  .LBB20_12:
+; RV32I-NEXT:    srl a1, a7, a5
+; RV32I-NEXT:  .LBB20_13:
+; RV32I-NEXT:    sw a3, 28(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    neg s10, s7
+; RV32I-NEXT:    bltu s7, t5, .LBB20_15
+; RV32I-NEXT:  # %bb.14:
+; RV32I-NEXT:    li a6, 0
+; RV32I-NEXT:    sll a3, t2, s7
+; RV32I-NEXT:    j .LBB20_16
+; RV32I-NEXT:  .LBB20_15:
+; RV32I-NEXT:    sll a6, t2, s6
+; RV32I-NEXT:    srl a3, t2, s10
+; RV32I-NEXT:    or a3, a3, s1
+; RV32I-NEXT:  .LBB20_16:
+; RV32I-NEXT:    sw t3, 40(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    addi s9, a5, -64
+; RV32I-NEXT:    mv t3, a4
+; RV32I-NEXT:    beqz s7, .LBB20_18
+; RV32I-NEXT:  # %bb.17:
+; RV32I-NEXT:    mv t3, a3
+; RV32I-NEXT:  .LBB20_18:
+; RV32I-NEXT:    neg s11, s9
+; RV32I-NEXT:    sw s0, 32(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    bltu s9, t5, .LBB20_20
+; RV32I-NEXT:  # %bb.19:
+; RV32I-NEXT:    sra s0, a4, s9
+; RV32I-NEXT:    j .LBB20_21
+; RV32I-NEXT:  .LBB20_20:
+; RV32I-NEXT:    sll a3, a4, s11
+; RV32I-NEXT:    or s0, s0, a3
+; RV32I-NEXT:  .LBB20_21:
+; RV32I-NEXT:    sw s1, 16(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw t4, 36(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    lbu s3, 11(a0)
+; RV32I-NEXT:    lbu a3, 15(a0)
+; RV32I-NEXT:    mv t4, t2
+; RV32I-NEXT:    beqz s9, .LBB20_23
+; RV32I-NEXT:  # %bb.22:
+; RV32I-NEXT:    mv t4, s0
+; RV32I-NEXT:  .LBB20_23:
+; RV32I-NEXT:    lbu s2, 9(a0)
+; RV32I-NEXT:    lbu s1, 10(a0)
+; RV32I-NEXT:    lbu s8, 13(a0)
+; RV32I-NEXT:    lbu ra, 14(a0)
+; RV32I-NEXT:    slli s3, s3, 8
+; RV32I-NEXT:    slli a3, a3, 8
+; RV32I-NEXT:    bltu s9, t5, .LBB20_25
+; RV32I-NEXT:  # %bb.24:
+; RV32I-NEXT:    srai s0, a4, 31
+; RV32I-NEXT:    j .LBB20_26
+; RV32I-NEXT:  .LBB20_25:
+; RV32I-NEXT:    sra s0, a4, a5
+; RV32I-NEXT:  .LBB20_26:
+; RV32I-NEXT:    or s1, s3, s1
+; RV32I-NEXT:    lbu s5, 8(a0)
+; RV32I-NEXT:    lbu s3, 12(a0)
+; RV32I-NEXT:    slli s2, s2, 8
+; RV32I-NEXT:    slli s4, s8, 8
+; RV32I-NEXT:    or s8, a3, ra
+; RV32I-NEXT:    bgeu a5, t6, .LBB20_28
+; RV32I-NEXT:  # %bb.27:
+; RV32I-NEXT:    or t4, t0, a6
+; RV32I-NEXT:    or s0, a1, t3
+; RV32I-NEXT:  .LBB20_28:
+; RV32I-NEXT:    lbu a3, 3(a0)
+; RV32I-NEXT:    lbu t3, 7(a0)
+; RV32I-NEXT:    or a6, s2, s5
+; RV32I-NEXT:    slli s2, s1, 16
+; RV32I-NEXT:    or s1, s4, s3
+; RV32I-NEXT:    slli s8, s8, 16
+; RV32I-NEXT:    mv a1, t1
+; RV32I-NEXT:    mv t0, a7
+; RV32I-NEXT:    beqz a5, .LBB20_30
+; RV32I-NEXT:  # %bb.29:
+; RV32I-NEXT:    mv a1, t4
+; RV32I-NEXT:    mv t0, s0
+; RV32I-NEXT:  .LBB20_30:
+; RV32I-NEXT:    slli s5, a3, 8
+; RV32I-NEXT:    lbu ra, 1(a0)
+; RV32I-NEXT:    lbu a3, 2(a0)
+; RV32I-NEXT:    lbu s3, 5(a0)
+; RV32I-NEXT:    lbu s0, 6(a0)
+; RV32I-NEXT:    slli s4, t3, 8
+; RV32I-NEXT:    or t4, s2, a6
+; RV32I-NEXT:    or t3, s8, s1
+; RV32I-NEXT:    bltu a5, t6, .LBB20_32
+; RV32I-NEXT:  # %bb.31:
+; RV32I-NEXT:    srai a6, a4, 31
+; RV32I-NEXT:    sw a6, 40(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw a6, 36(sp) # 4-byte Folded Spill
+; RV32I-NEXT:  .LBB20_32:
+; RV32I-NEXT:    slli a6, ra, 8
+; RV32I-NEXT:    or a3, s5, a3
+; RV32I-NEXT:    lbu s1, 0(a0)
+; RV32I-NEXT:    lbu a0, 4(a0)
+; RV32I-NEXT:    slli s3, s3, 8
+; RV32I-NEXT:    or s0, s4, s0
+; RV32I-NEXT:    srl s2, t4, a5
+; RV32I-NEXT:    sll ra, t3, s6
+; RV32I-NEXT:    bltu a5, t5, .LBB20_34
+; RV32I-NEXT:  # %bb.33:
+; RV32I-NEXT:    srl s4, t3, a5
+; RV32I-NEXT:    j .LBB20_35
+; RV32I-NEXT:  .LBB20_34:
+; RV32I-NEXT:    or s4, s2, ra
+; RV32I-NEXT:  .LBB20_35:
+; RV32I-NEXT:    or a6, a6, s1
+; RV32I-NEXT:    slli a3, a3, 16
+; RV32I-NEXT:    or a0, s3, a0
+; RV32I-NEXT:    slli s1, s0, 16
+; RV32I-NEXT:    mv s5, t4
+; RV32I-NEXT:    beqz a5, .LBB20_37
+; RV32I-NEXT:  # %bb.36:
+; RV32I-NEXT:    mv s5, s4
+; RV32I-NEXT:  .LBB20_37:
+; RV32I-NEXT:    or s0, a3, a6
+; RV32I-NEXT:    or a0, s1, a0
+; RV32I-NEXT:    bltu a5, t5, .LBB20_39
+; RV32I-NEXT:  # %bb.38:
+; RV32I-NEXT:    li s4, 0
+; RV32I-NEXT:    srl a3, a0, a5
+; RV32I-NEXT:    mv a6, s0
+; RV32I-NEXT:    bnez a5, .LBB20_40
+; RV32I-NEXT:    j .LBB20_41
+; RV32I-NEXT:  .LBB20_39:
+; RV32I-NEXT:    srl s4, t3, a5
+; RV32I-NEXT:    srl a3, s0, a5
+; RV32I-NEXT:    sll a6, a0, s6
+; RV32I-NEXT:    or a3, a3, a6
+; RV32I-NEXT:    mv a6, s0
+; RV32I-NEXT:    beqz a5, .LBB20_41
+; RV32I-NEXT:  .LBB20_40:
+; RV32I-NEXT:    mv a6, a3
+; RV32I-NEXT:  .LBB20_41:
+; RV32I-NEXT:    bltu a5, t5, .LBB20_44
+; RV32I-NEXT:  # %bb.42:
+; RV32I-NEXT:    li s1, 0
+; RV32I-NEXT:    bgeu s7, t5, .LBB20_45
+; RV32I-NEXT:  .LBB20_43:
+; RV32I-NEXT:    sll s3, t4, s6
+; RV32I-NEXT:    srl a3, t4, s10
+; RV32I-NEXT:    or a3, a3, ra
+; RV32I-NEXT:    mv s10, t3
+; RV32I-NEXT:    bnez s7, .LBB20_46
+; RV32I-NEXT:    j .LBB20_47
+; RV32I-NEXT:  .LBB20_44:
+; RV32I-NEXT:    srl s1, a0, a5
+; RV32I-NEXT:    bltu s7, t5, .LBB20_43
+; RV32I-NEXT:  .LBB20_45:
+; RV32I-NEXT:    li s3, 0
+; RV32I-NEXT:    sll a3, t4, s7
+; RV32I-NEXT:    mv s10, t3
+; RV32I-NEXT:    beqz s7, .LBB20_47
+; RV32I-NEXT:  .LBB20_46:
+; RV32I-NEXT:    mv s10, a3
+; RV32I-NEXT:  .LBB20_47:
+; RV32I-NEXT:    bltu s9, t5, .LBB20_49
+; RV32I-NEXT:  # %bb.48:
+; RV32I-NEXT:    srl a3, t3, s9
+; RV32I-NEXT:    mv s2, t4
+; RV32I-NEXT:    bnez s9, .LBB20_50
+; RV32I-NEXT:    j .LBB20_51
+; RV32I-NEXT:  .LBB20_49:
+; RV32I-NEXT:    sll a3, t3, s11
+; RV32I-NEXT:    or a3, s2, a3
+; RV32I-NEXT:    mv s2, t4
+; RV32I-NEXT:    beqz s9, .LBB20_51
+; RV32I-NEXT:  .LBB20_50:
+; RV32I-NEXT:    mv s2, a3
+; RV32I-NEXT:  .LBB20_51:
+; RV32I-NEXT:    bltu s9, t5, .LBB20_53
+; RV32I-NEXT:  # %bb.52:
+; RV32I-NEXT:    li s7, 0
+; RV32I-NEXT:    bltu a5, t6, .LBB20_54
+; RV32I-NEXT:    j .LBB20_55
+; RV32I-NEXT:  .LBB20_53:
+; RV32I-NEXT:    srl s7, t3, a5
+; RV32I-NEXT:    bgeu a5, t6, .LBB20_55
+; RV32I-NEXT:  .LBB20_54:
+; RV32I-NEXT:    or s2, a6, s3
+; RV32I-NEXT:    or s7, s1, s10
+; RV32I-NEXT:  .LBB20_55:
+; RV32I-NEXT:    li a3, 128
+; RV32I-NEXT:    mv a6, s0
+; RV32I-NEXT:    mv s1, a0
+; RV32I-NEXT:    beqz a5, .LBB20_57
+; RV32I-NEXT:  # %bb.56:
+; RV32I-NEXT:    mv a6, s2
+; RV32I-NEXT:    mv s1, s7
+; RV32I-NEXT:  .LBB20_57:
+; RV32I-NEXT:    sw a6, 8(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sub s2, a3, a5
+; RV32I-NEXT:    bltu a5, t6, .LBB20_59
+; RV32I-NEXT:  # %bb.58:
+; RV32I-NEXT:    li s5, 0
+; RV32I-NEXT:    li s4, 0
+; RV32I-NEXT:  .LBB20_59:
+; RV32I-NEXT:    neg s3, s2
+; RV32I-NEXT:    srl a6, t1, s3
+; RV32I-NEXT:    sw s4, 20(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s1, 4(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    bltu s2, t5, .LBB20_61
+; RV32I-NEXT:  # %bb.60:
+; RV32I-NEXT:    li s11, 0
+; RV32I-NEXT:    sll a3, t1, s2
+; RV32I-NEXT:    j .LBB20_62
+; RV32I-NEXT:  .LBB20_61:
+; RV32I-NEXT:    sll s11, t1, s6
+; RV32I-NEXT:    lw a3, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    or a3, a6, a3
+; RV32I-NEXT:  .LBB20_62:
+; RV32I-NEXT:    sub s1, t6, s2
+; RV32I-NEXT:    mv s8, a7
+; RV32I-NEXT:    beqz s2, .LBB20_64
+; RV32I-NEXT:  # %bb.63:
+; RV32I-NEXT:    mv s8, a3
+; RV32I-NEXT:  .LBB20_64:
+; RV32I-NEXT:    bltu s1, t5, .LBB20_66
+; RV32I-NEXT:  # %bb.65:
+; RV32I-NEXT:    srl a3, a7, s1
+; RV32I-NEXT:    mv a6, t1
+; RV32I-NEXT:    bnez s1, .LBB20_67
+; RV32I-NEXT:    j .LBB20_68
+; RV32I-NEXT:  .LBB20_66:
+; RV32I-NEXT:    neg a3, s1
+; RV32I-NEXT:    sll a3, a7, a3
+; RV32I-NEXT:    or a3, a6, a3
+; RV32I-NEXT:    mv a6, t1
+; RV32I-NEXT:    beqz s1, .LBB20_68
+; RV32I-NEXT:  .LBB20_67:
+; RV32I-NEXT:    mv a6, a3
+; RV32I-NEXT:  .LBB20_68:
+; RV32I-NEXT:    bltu s1, t5, .LBB20_71
+; RV32I-NEXT:  # %bb.69:
+; RV32I-NEXT:    li s1, 0
+; RV32I-NEXT:    sw s5, 24(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    bgeu s2, t5, .LBB20_72
+; RV32I-NEXT:  .LBB20_70:
+; RV32I-NEXT:    sll s6, t2, s6
+; RV32I-NEXT:    srl a3, t2, s3
+; RV32I-NEXT:    lw s3, 16(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    or a3, a3, s3
+; RV32I-NEXT:    j .LBB20_73
+; RV32I-NEXT:  .LBB20_71:
+; RV32I-NEXT:    srl s1, a7, s3
+; RV32I-NEXT:    sw s5, 24(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    bltu s2, t5, .LBB20_70
+; RV32I-NEXT:  .LBB20_72:
+; RV32I-NEXT:    li s6, 0
+; RV32I-NEXT:    sll a3, t2, s2
+; RV32I-NEXT:  .LBB20_73:
+; RV32I-NEXT:    addi s9, s2, -64
+; RV32I-NEXT:    mv s5, a4
+; RV32I-NEXT:    beqz s2, .LBB20_75
+; RV32I-NEXT:  # %bb.74:
+; RV32I-NEXT:    mv s5, a3
+; RV32I-NEXT:  .LBB20_75:
+; RV32I-NEXT:    bltu s9, t5, .LBB20_77
+; RV32I-NEXT:  # %bb.76:
+; RV32I-NEXT:    li s3, 0
+; RV32I-NEXT:    sll a3, t1, s9
+; RV32I-NEXT:    mv s7, a7
+; RV32I-NEXT:    bnez s9, .LBB20_78
+; RV32I-NEXT:    j .LBB20_79
+; RV32I-NEXT:  .LBB20_77:
+; RV32I-NEXT:    sll s3, t1, s2
+; RV32I-NEXT:    neg a3, s9
+; RV32I-NEXT:    srl a3, t1, a3
+; RV32I-NEXT:    sll s4, a7, s2
+; RV32I-NEXT:    or a3, a3, s4
+; RV32I-NEXT:    mv s7, a7
+; RV32I-NEXT:    beqz s9, .LBB20_79
+; RV32I-NEXT:  .LBB20_78:
+; RV32I-NEXT:    mv s7, a3
+; RV32I-NEXT:  .LBB20_79:
+; RV32I-NEXT:    bltu s2, t6, .LBB20_81
+; RV32I-NEXT:  # %bb.80:
+; RV32I-NEXT:    li s11, 0
+; RV32I-NEXT:    li s8, 0
+; RV32I-NEXT:    j .LBB20_82
+; RV32I-NEXT:  .LBB20_81:
+; RV32I-NEXT:    or s3, a6, s6
+; RV32I-NEXT:    or s7, s1, s5
+; RV32I-NEXT:  .LBB20_82:
+; RV32I-NEXT:    addi ra, a5, -128
+; RV32I-NEXT:    mv s4, t2
+; RV32I-NEXT:    mv s6, a4
+; RV32I-NEXT:    beqz s2, .LBB20_84
+; RV32I-NEXT:  # %bb.83:
+; RV32I-NEXT:    mv s4, s3
+; RV32I-NEXT:    mv s6, s7
+; RV32I-NEXT:  .LBB20_84:
+; RV32I-NEXT:    neg s9, ra
+; RV32I-NEXT:    sll s3, a4, s9
+; RV32I-NEXT:    bltu ra, t5, .LBB20_86
+; RV32I-NEXT:  # %bb.85:
+; RV32I-NEXT:    sra a3, a4, ra
+; RV32I-NEXT:    mv s1, t2
+; RV32I-NEXT:    bnez ra, .LBB20_87
+; RV32I-NEXT:    j .LBB20_88
+; RV32I-NEXT:  .LBB20_86:
+; RV32I-NEXT:    lw a3, 32(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    or a3, a3, s3
+; RV32I-NEXT:    mv s1, t2
+; RV32I-NEXT:    beqz ra, .LBB20_88
+; RV32I-NEXT:  .LBB20_87:
+; RV32I-NEXT:    mv s1, a3
+; RV32I-NEXT:  .LBB20_88:
+; RV32I-NEXT:    bltu ra, t5, .LBB20_90
+; RV32I-NEXT:  # %bb.89:
+; RV32I-NEXT:    srai s2, a4, 31
+; RV32I-NEXT:    srl a3, a7, ra
+; RV32I-NEXT:    mv a6, t1
+; RV32I-NEXT:    bnez ra, .LBB20_91
+; RV32I-NEXT:    j .LBB20_92
+; RV32I-NEXT:  .LBB20_90:
+; RV32I-NEXT:    sra s2, a4, a5
+; RV32I-NEXT:    sll a3, a7, s9
+; RV32I-NEXT:    lw a6, 28(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    or a3, a6, a3
+; RV32I-NEXT:    mv a6, t1
+; RV32I-NEXT:    beqz ra, .LBB20_92
+; RV32I-NEXT:  .LBB20_91:
+; RV32I-NEXT:    mv a6, a3
+; RV32I-NEXT:  .LBB20_92:
+; RV32I-NEXT:    mv s5, t0
+; RV32I-NEXT:    sub s10, t6, ra
+; RV32I-NEXT:    li t0, 64
+; RV32I-NEXT:    bltu ra, t5, .LBB20_94
+; RV32I-NEXT:  # %bb.93:
+; RV32I-NEXT:    li s7, 0
+; RV32I-NEXT:    j .LBB20_95
+; RV32I-NEXT:  .LBB20_94:
+; RV32I-NEXT:    srl s7, a7, a5
+; RV32I-NEXT:  .LBB20_95:
+; RV32I-NEXT:    mv t6, s8
+; RV32I-NEXT:    mv s8, s11
+; RV32I-NEXT:    bltu s10, t5, .LBB20_97
+; RV32I-NEXT:  # %bb.96:
+; RV32I-NEXT:    li s9, 0
+; RV32I-NEXT:    sll a3, t2, s10
+; RV32I-NEXT:    j .LBB20_98
+; RV32I-NEXT:  .LBB20_97:
+; RV32I-NEXT:    sll s9, t2, s9
+; RV32I-NEXT:    neg a3, s10
+; RV32I-NEXT:    srl a3, t2, a3
+; RV32I-NEXT:    or a3, a3, s3
+; RV32I-NEXT:  .LBB20_98:
+; RV32I-NEXT:    addi s11, ra, -64
+; RV32I-NEXT:    mv s3, a4
+; RV32I-NEXT:    beqz s10, .LBB20_100
+; RV32I-NEXT:  # %bb.99:
+; RV32I-NEXT:    mv s3, a3
+; RV32I-NEXT:  .LBB20_100:
+; RV32I-NEXT:    bltu s11, t5, .LBB20_102
+; RV32I-NEXT:  # %bb.101:
+; RV32I-NEXT:    sra a3, a4, s11
+; RV32I-NEXT:    bnez s11, .LBB20_103
+; RV32I-NEXT:    j .LBB20_104
+; RV32I-NEXT:  .LBB20_102:
+; RV32I-NEXT:    srl a3, t2, ra
+; RV32I-NEXT:    mv s10, s4
+; RV32I-NEXT:    neg s4, s11
+; RV32I-NEXT:    sll s4, a4, s4
+; RV32I-NEXT:    or a3, a3, s4
+; RV32I-NEXT:    mv s4, s10
+; RV32I-NEXT:    beqz s11, .LBB20_104
+; RV32I-NEXT:  .LBB20_103:
+; RV32I-NEXT:    mv t2, a3
+; RV32I-NEXT:  .LBB20_104:
+; RV32I-NEXT:    bltu s11, t5, .LBB20_106
+; RV32I-NEXT:  # %bb.105:
+; RV32I-NEXT:    srai t5, a4, 31
+; RV32I-NEXT:    lw s11, 40(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    bltu ra, t0, .LBB20_107
+; RV32I-NEXT:    j .LBB20_108
+; RV32I-NEXT:  .LBB20_106:
+; RV32I-NEXT:    sra t5, a4, ra
+; RV32I-NEXT:    lw s11, 40(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    bgeu ra, t0, .LBB20_108
+; RV32I-NEXT:  .LBB20_107:
+; RV32I-NEXT:    or t2, a6, s9
+; RV32I-NEXT:    or t5, s7, s3
+; RV32I-NEXT:  .LBB20_108:
+; RV32I-NEXT:    li a6, 128
+; RV32I-NEXT:    bnez ra, .LBB20_117
+; RV32I-NEXT:  # %bb.109:
+; RV32I-NEXT:    bgeu ra, t0, .LBB20_118
+; RV32I-NEXT:  .LBB20_110:
+; RV32I-NEXT:    bgeu a5, a6, .LBB20_112
+; RV32I-NEXT:  .LBB20_111:
+; RV32I-NEXT:    lw a3, 8(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    or t1, a3, s8
+; RV32I-NEXT:    lw a3, 4(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    or a7, a3, t6
+; RV32I-NEXT:    lw a3, 24(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    or s1, a3, s4
+; RV32I-NEXT:    lw a3, 20(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    or s2, a3, s6
+; RV32I-NEXT:  .LBB20_112:
+; RV32I-NEXT:    lw ra, 36(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    mv t0, s5
+; RV32I-NEXT:    beqz a5, .LBB20_114
+; RV32I-NEXT:  # %bb.113:
+; RV32I-NEXT:    mv s0, t1
+; RV32I-NEXT:    mv a0, a7
+; RV32I-NEXT:    mv t4, s1
+; RV32I-NEXT:    mv t3, s2
+; RV32I-NEXT:  .LBB20_114:
+; RV32I-NEXT:    bltu a5, a6, .LBB20_116
+; RV32I-NEXT:  # %bb.115:
+; RV32I-NEXT:    srai a1, a4, 31
+; RV32I-NEXT:    mv t0, a1
+; RV32I-NEXT:    mv s11, a1
+; RV32I-NEXT:    mv ra, a1
+; RV32I-NEXT:  .LBB20_116:
+; RV32I-NEXT:    srli a4, s0, 16
+; RV32I-NEXT:    lui t1, 16
+; RV32I-NEXT:    srli a7, s0, 24
+; RV32I-NEXT:    srli a5, a0, 16
+; RV32I-NEXT:    srli t5, a0, 24
+; RV32I-NEXT:    srli a6, t4, 16
+; RV32I-NEXT:    srli s2, t4, 24
+; RV32I-NEXT:    srli t2, t3, 16
+; RV32I-NEXT:    srli s3, t3, 24
+; RV32I-NEXT:    srli s1, a1, 16
+; RV32I-NEXT:    srli a3, a1, 24
+; RV32I-NEXT:    srli t6, t0, 16
+; RV32I-NEXT:    srli s6, t0, 24
+; RV32I-NEXT:    srli s5, s11, 16
+; RV32I-NEXT:    srli s4, s11, 24
+; RV32I-NEXT:    srli s7, ra, 16
+; RV32I-NEXT:    srli s8, ra, 24
+; RV32I-NEXT:    addi t1, t1, -1
+; RV32I-NEXT:    and s9, s0, t1
+; RV32I-NEXT:    and s10, a0, t1
+; RV32I-NEXT:    srli s9, s9, 8
+; RV32I-NEXT:    sb s0, 0(a2)
+; RV32I-NEXT:    sb s9, 1(a2)
+; RV32I-NEXT:    sb a4, 2(a2)
+; RV32I-NEXT:    sb a7, 3(a2)
+; RV32I-NEXT:    and a4, t4, t1
+; RV32I-NEXT:    srli a7, s10, 8
+; RV32I-NEXT:    sb a0, 4(a2)
+; RV32I-NEXT:    sb a7, 5(a2)
+; RV32I-NEXT:    sb a5, 6(a2)
+; RV32I-NEXT:    sb t5, 7(a2)
+; RV32I-NEXT:    and a0, t3, t1
+; RV32I-NEXT:    srli a4, a4, 8
+; RV32I-NEXT:    sb t4, 8(a2)
+; RV32I-NEXT:    sb a4, 9(a2)
+; RV32I-NEXT:    sb a6, 10(a2)
+; RV32I-NEXT:    sb s2, 11(a2)
+; RV32I-NEXT:    and a4, a1, t1
+; RV32I-NEXT:    srli a0, a0, 8
+; RV32I-NEXT:    sb t3, 12(a2)
+; RV32I-NEXT:    sb a0, 13(a2)
+; RV32I-NEXT:    sb t2, 14(a2)
+; RV32I-NEXT:    sb s3, 15(a2)
+; RV32I-NEXT:    and a0, t0, t1
+; RV32I-NEXT:    srli a4, a4, 8
+; RV32I-NEXT:    sb a1, 16(a2)
+; RV32I-NEXT:    sb a4, 17(a2)
+; RV32I-NEXT:    sb s1, 18(a2)
+; RV32I-NEXT:    sb a3, 19(a2)
+; RV32I-NEXT:    and a1, s11, t1
+; RV32I-NEXT:    and a3, ra, t1
+; RV32I-NEXT:    srli a0, a0, 8
+; RV32I-NEXT:    srli a1, a1, 8
+; RV32I-NEXT:    srli a3, a3, 8
+; RV32I-NEXT:    sb t0, 20(a2)
+; RV32I-NEXT:    sb a0, 21(a2)
+; RV32I-NEXT:    sb t6, 22(a2)
+; RV32I-NEXT:    sb s6, 23(a2)
+; RV32I-NEXT:    sb s11, 24(a2)
+; RV32I-NEXT:    sb a1, 25(a2)
+; RV32I-NEXT:    sb s5, 26(a2)
+; RV32I-NEXT:    sb s4, 27(a2)
+; RV32I-NEXT:    sb ra, 28(a2)
+; RV32I-NEXT:    sb a3, 29(a2)
+; RV32I-NEXT:    sb s7, 30(a2)
+; RV32I-NEXT:    sb s8, 31(a2)
+; RV32I-NEXT:    lw ra, 92(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s0, 88(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s1, 84(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s2, 80(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s3, 76(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s4, 72(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s5, 68(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s6, 64(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s7, 60(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s8, 56(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s9, 52(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s10, 48(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s11, 44(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    addi sp, sp, 96
+; RV32I-NEXT:    ret
+; RV32I-NEXT:  .LBB20_117:
+; RV32I-NEXT:    mv t1, t2
+; RV32I-NEXT:    mv a7, t5
+; RV32I-NEXT:    bltu ra, t0, .LBB20_110
+; RV32I-NEXT:  .LBB20_118:
+; RV32I-NEXT:    srai s1, a4, 31
+; RV32I-NEXT:    mv s2, s1
+; RV32I-NEXT:    bltu a5, a6, .LBB20_111
+; RV32I-NEXT:    j .LBB20_112
+  %src = load i256, ptr %src.ptr, align 1
+  %dwordOff = load i256, ptr %dwordOff.ptr, align 1
+  %bitOff = shl i256 %dwordOff, 6
+  %res = ashr i256 %src, %bitOff
+  store i256 %res, ptr %dst, align 1
+  ret void
+}
diff --git a/llvm/test/CodeGen/RISCV/attributes.ll b/llvm/test/CodeGen/RISCV/attributes.ll
index caed0bdfb0498..b9d5bf0a7227c 100644
--- a/llvm/test/CodeGen/RISCV/attributes.ll
+++ b/llvm/test/CodeGen/RISCV/attributes.ll
@@ -398,7 +398,7 @@
 ; RV32XTHEADSYNC: .attribute 5, "rv32i2p1_xtheadsync1p0"
 ; RV32XWCHC: .attribute 5, "rv32i2p1_xwchc2p2"
 ; RV32XQCIA: .attribute 5, "rv32i2p1_xqcia0p2"
-; RV32XQCIAC: .attribute 5, "rv32i2p1_zca1p0_xqciac0p2"
+; RV32XQCIAC: .attribute 5, "rv32i2p1_zca1p0_xqciac0p3"
 ; RV32XQCICLI: .attribute 5, "rv32i2p1_xqcicli0p2"
 ; RV32XQCICM: .attribute 5, "rv32i2p1_zca1p0_xqcicm0p2"
 ; RV32XQCICS: .attribute 5, "rv32i2p1_xqcics0p2"
diff --git a/llvm/test/CodeGen/RISCV/callee-saved-gprs.ll b/llvm/test/CodeGen/RISCV/callee-saved-gprs.ll
index 90a1ebec5abff..f9f1ba60a8ac0 100644
--- a/llvm/test/CodeGen/RISCV/callee-saved-gprs.ll
+++ b/llvm/test/CodeGen/RISCV/callee-saved-gprs.ll
@@ -37,10 +37,11 @@
 ; This function tests that RISCVRegisterInfo::getCalleeSavedRegs returns
 ; something appropriate.
 
-define void @callee() nounwind {
+define void @callee() {
 ; RV32I-LABEL: callee:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -80
+; RV32I-NEXT:    .cfi_def_cfa_offset 80
 ; RV32I-NEXT:    sw ra, 76(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    sw s0, 72(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    sw s1, 68(sp) # 4-byte Folded Spill
@@ -54,6 +55,19 @@ define void @callee() nounwind {
 ; RV32I-NEXT:    sw s9, 36(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    sw s10, 32(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    sw s11, 28(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    .cfi_offset ra, -4
+; RV32I-NEXT:    .cfi_offset s0, -8
+; RV32I-NEXT:    .cfi_offset s1, -12
+; RV32I-NEXT:    .cfi_offset s2, -16
+; RV32I-NEXT:    .cfi_offset s3, -20
+; RV32I-NEXT:    .cfi_offset s4, -24
+; RV32I-NEXT:    .cfi_offset s5, -28
+; RV32I-NEXT:    .cfi_offset s6, -32
+; RV32I-NEXT:    .cfi_offset s7, -36
+; RV32I-NEXT:    .cfi_offset s8, -40
+; RV32I-NEXT:    .cfi_offset s9, -44
+; RV32I-NEXT:    .cfi_offset s10, -48
+; RV32I-NEXT:    .cfi_offset s11, -52
 ; RV32I-NEXT:    lui a7, %hi(var)
 ; RV32I-NEXT:    lw a0, %lo(var)(a7)
 ; RV32I-NEXT:    sw a0, 24(sp) # 4-byte Folded Spill
@@ -145,15 +159,33 @@ define void @callee() nounwind {
 ; RV32I-NEXT:    lw s9, 36(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    lw s10, 32(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    lw s11, 28(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    .cfi_restore ra
+; RV32I-NEXT:    .cfi_restore s0
+; RV32I-NEXT:    .cfi_restore s1
+; RV32I-NEXT:    .cfi_restore s2
+; RV32I-NEXT:    .cfi_restore s3
+; RV32I-NEXT:    .cfi_restore s4
+; RV32I-NEXT:    .cfi_restore s5
+; RV32I-NEXT:    .cfi_restore s6
+; RV32I-NEXT:    .cfi_restore s7
+; RV32I-NEXT:    .cfi_restore s8
+; RV32I-NEXT:    .cfi_restore s9
+; RV32I-NEXT:    .cfi_restore s10
+; RV32I-NEXT:    .cfi_restore s11
 ; RV32I-NEXT:    addi sp, sp, 80
+; RV32I-NEXT:    .cfi_def_cfa_offset 0
 ; RV32I-NEXT:    ret
 ;
 ; RV32I-ILP32E-LABEL: callee:
 ; RV32I-ILP32E:       # %bb.0:
 ; RV32I-ILP32E-NEXT:    addi sp, sp, -36
+; RV32I-ILP32E-NEXT:    .cfi_def_cfa_offset 36
 ; RV32I-ILP32E-NEXT:    sw ra, 32(sp) # 4-byte Folded Spill
 ; RV32I-ILP32E-NEXT:    sw s0, 28(sp) # 4-byte Folded Spill
 ; RV32I-ILP32E-NEXT:    sw s1, 24(sp) # 4-byte Folded Spill
+; RV32I-ILP32E-NEXT:    .cfi_offset ra, -4
+; RV32I-ILP32E-NEXT:    .cfi_offset s0, -8
+; RV32I-ILP32E-NEXT:    .cfi_offset s1, -12
 ; RV32I-ILP32E-NEXT:    lui a7, %hi(var)
 ; RV32I-ILP32E-NEXT:    lw a0, %lo(var)(a7)
 ; RV32I-ILP32E-NEXT:    sw a0, 20(sp) # 4-byte Folded Spill
@@ -235,12 +267,17 @@ define void @callee() nounwind {
 ; RV32I-ILP32E-NEXT:    lw ra, 32(sp) # 4-byte Folded Reload
 ; RV32I-ILP32E-NEXT:    lw s0, 28(sp) # 4-byte Folded Reload
 ; RV32I-ILP32E-NEXT:    lw s1, 24(sp) # 4-byte Folded Reload
+; RV32I-ILP32E-NEXT:    .cfi_restore ra
+; RV32I-ILP32E-NEXT:    .cfi_restore s0
+; RV32I-ILP32E-NEXT:    .cfi_restore s1
 ; RV32I-ILP32E-NEXT:    addi sp, sp, 36
+; RV32I-ILP32E-NEXT:    .cfi_def_cfa_offset 0
 ; RV32I-ILP32E-NEXT:    ret
 ;
 ; RV32I-WITH-FP-LABEL: callee:
 ; RV32I-WITH-FP:       # %bb.0:
 ; RV32I-WITH-FP-NEXT:    addi sp, sp, -80
+; RV32I-WITH-FP-NEXT:    .cfi_def_cfa_offset 80
 ; RV32I-WITH-FP-NEXT:    sw ra, 76(sp) # 4-byte Folded Spill
 ; RV32I-WITH-FP-NEXT:    sw s0, 72(sp) # 4-byte Folded Spill
 ; RV32I-WITH-FP-NEXT:    sw s1, 68(sp) # 4-byte Folded Spill
@@ -254,7 +291,21 @@ define void @callee() nounwind {
 ; RV32I-WITH-FP-NEXT:    sw s9, 36(sp) # 4-byte Folded Spill
 ; RV32I-WITH-FP-NEXT:    sw s10, 32(sp) # 4-byte Folded Spill
 ; RV32I-WITH-FP-NEXT:    sw s11, 28(sp) # 4-byte Folded Spill
+; RV32I-WITH-FP-NEXT:    .cfi_offset ra, -4
+; RV32I-WITH-FP-NEXT:    .cfi_offset s0, -8
+; RV32I-WITH-FP-NEXT:    .cfi_offset s1, -12
+; RV32I-WITH-FP-NEXT:    .cfi_offset s2, -16
+; RV32I-WITH-FP-NEXT:    .cfi_offset s3, -20
+; RV32I-WITH-FP-NEXT:    .cfi_offset s4, -24
+; RV32I-WITH-FP-NEXT:    .cfi_offset s5, -28
+; RV32I-WITH-FP-NEXT:    .cfi_offset s6, -32
+; RV32I-WITH-FP-NEXT:    .cfi_offset s7, -36
+; RV32I-WITH-FP-NEXT:    .cfi_offset s8, -40
+; RV32I-WITH-FP-NEXT:    .cfi_offset s9, -44
+; RV32I-WITH-FP-NEXT:    .cfi_offset s10, -48
+; RV32I-WITH-FP-NEXT:    .cfi_offset s11, -52
 ; RV32I-WITH-FP-NEXT:    addi s0, sp, 80
+; RV32I-WITH-FP-NEXT:    .cfi_def_cfa s0, 0
 ; RV32I-WITH-FP-NEXT:    lui t0, %hi(var)
 ; RV32I-WITH-FP-NEXT:    lw a0, %lo(var)(t0)
 ; RV32I-WITH-FP-NEXT:    sw a0, -56(s0) # 4-byte Folded Spill
@@ -335,6 +386,7 @@ define void @callee() nounwind {
 ; RV32I-WITH-FP-NEXT:    sw a0, %lo(var+4)(t0)
 ; RV32I-WITH-FP-NEXT:    lw a0, -56(s0) # 4-byte Folded Reload
 ; RV32I-WITH-FP-NEXT:    sw a0, %lo(var)(t0)
+; RV32I-WITH-FP-NEXT:    .cfi_def_cfa sp, 80
 ; RV32I-WITH-FP-NEXT:    lw ra, 76(sp) # 4-byte Folded Reload
 ; RV32I-WITH-FP-NEXT:    lw s0, 72(sp) # 4-byte Folded Reload
 ; RV32I-WITH-FP-NEXT:    lw s1, 68(sp) # 4-byte Folded Reload
@@ -348,26 +400,54 @@ define void @callee() nounwind {
 ; RV32I-WITH-FP-NEXT:    lw s9, 36(sp) # 4-byte Folded Reload
 ; RV32I-WITH-FP-NEXT:    lw s10, 32(sp) # 4-byte Folded Reload
 ; RV32I-WITH-FP-NEXT:    lw s11, 28(sp) # 4-byte Folded Reload
+; RV32I-WITH-FP-NEXT:    .cfi_restore ra
+; RV32I-WITH-FP-NEXT:    .cfi_restore s0
+; RV32I-WITH-FP-NEXT:    .cfi_restore s1
+; RV32I-WITH-FP-NEXT:    .cfi_restore s2
+; RV32I-WITH-FP-NEXT:    .cfi_restore s3
+; RV32I-WITH-FP-NEXT:    .cfi_restore s4
+; RV32I-WITH-FP-NEXT:    .cfi_restore s5
+; RV32I-WITH-FP-NEXT:    .cfi_restore s6
+; RV32I-WITH-FP-NEXT:    .cfi_restore s7
+; RV32I-WITH-FP-NEXT:    .cfi_restore s8
+; RV32I-WITH-FP-NEXT:    .cfi_restore s9
+; RV32I-WITH-FP-NEXT:    .cfi_restore s10
+; RV32I-WITH-FP-NEXT:    .cfi_restore s11
 ; RV32I-WITH-FP-NEXT:    addi sp, sp, 80
+; RV32I-WITH-FP-NEXT:    .cfi_def_cfa_offset 0
 ; RV32I-WITH-FP-NEXT:    ret
 ;
 ; RV32IZCMP-LABEL: callee:
 ; RV32IZCMP:       # %bb.0:
-; RV32IZCMP-NEXT:    cm.push {ra, s0-s11}, -96
+; RV32IZCMP-NEXT:    cm.push {ra, s0-s11}, -80
+; RV32IZCMP-NEXT:    .cfi_def_cfa_offset 80
+; RV32IZCMP-NEXT:    .cfi_offset ra, -52
+; RV32IZCMP-NEXT:    .cfi_offset s0, -48
+; RV32IZCMP-NEXT:    .cfi_offset s1, -44
+; RV32IZCMP-NEXT:    .cfi_offset s2, -40
+; RV32IZCMP-NEXT:    .cfi_offset s3, -36
+; RV32IZCMP-NEXT:    .cfi_offset s4, -32
+; RV32IZCMP-NEXT:    .cfi_offset s5, -28
+; RV32IZCMP-NEXT:    .cfi_offset s6, -24
+; RV32IZCMP-NEXT:    .cfi_offset s7, -20
+; RV32IZCMP-NEXT:    .cfi_offset s8, -16
+; RV32IZCMP-NEXT:    .cfi_offset s9, -12
+; RV32IZCMP-NEXT:    .cfi_offset s10, -8
+; RV32IZCMP-NEXT:    .cfi_offset s11, -4
 ; RV32IZCMP-NEXT:    lui t0, %hi(var)
 ; RV32IZCMP-NEXT:    lw a0, %lo(var)(t0)
-; RV32IZCMP-NEXT:    sw a0, 28(sp) # 4-byte Folded Spill
-; RV32IZCMP-NEXT:    lw a0, %lo(var+4)(t0)
 ; RV32IZCMP-NEXT:    sw a0, 24(sp) # 4-byte Folded Spill
-; RV32IZCMP-NEXT:    lw a0, %lo(var+8)(t0)
+; RV32IZCMP-NEXT:    lw a0, %lo(var+4)(t0)
 ; RV32IZCMP-NEXT:    sw a0, 20(sp) # 4-byte Folded Spill
-; RV32IZCMP-NEXT:    lw a0, %lo(var+12)(t0)
+; RV32IZCMP-NEXT:    lw a0, %lo(var+8)(t0)
 ; RV32IZCMP-NEXT:    sw a0, 16(sp) # 4-byte Folded Spill
+; RV32IZCMP-NEXT:    lw a0, %lo(var+12)(t0)
+; RV32IZCMP-NEXT:    sw a0, 12(sp) # 4-byte Folded Spill
 ; RV32IZCMP-NEXT:    addi a5, t0, %lo(var)
 ; RV32IZCMP-NEXT:    lw a0, 16(a5)
-; RV32IZCMP-NEXT:    sw a0, 12(sp) # 4-byte Folded Spill
-; RV32IZCMP-NEXT:    lw a0, 20(a5)
 ; RV32IZCMP-NEXT:    sw a0, 8(sp) # 4-byte Folded Spill
+; RV32IZCMP-NEXT:    lw a0, 20(a5)
+; RV32IZCMP-NEXT:    sw a0, 4(sp) # 4-byte Folded Spill
 ; RV32IZCMP-NEXT:    lw t4, 24(a5)
 ; RV32IZCMP-NEXT:    lw t5, 28(a5)
 ; RV32IZCMP-NEXT:    lw t6, 32(a5)
@@ -420,23 +500,24 @@ define void @callee() nounwind {
 ; RV32IZCMP-NEXT:    sw t6, 32(a5)
 ; RV32IZCMP-NEXT:    sw t5, 28(a5)
 ; RV32IZCMP-NEXT:    sw t4, 24(a5)
-; RV32IZCMP-NEXT:    lw a0, 8(sp) # 4-byte Folded Reload
+; RV32IZCMP-NEXT:    lw a0, 4(sp) # 4-byte Folded Reload
 ; RV32IZCMP-NEXT:    sw a0, 20(a5)
-; RV32IZCMP-NEXT:    lw a0, 12(sp) # 4-byte Folded Reload
+; RV32IZCMP-NEXT:    lw a0, 8(sp) # 4-byte Folded Reload
 ; RV32IZCMP-NEXT:    sw a0, 16(a5)
-; RV32IZCMP-NEXT:    lw a0, 16(sp) # 4-byte Folded Reload
+; RV32IZCMP-NEXT:    lw a0, 12(sp) # 4-byte Folded Reload
 ; RV32IZCMP-NEXT:    sw a0, %lo(var+12)(t0)
-; RV32IZCMP-NEXT:    lw a0, 20(sp) # 4-byte Folded Reload
+; RV32IZCMP-NEXT:    lw a0, 16(sp) # 4-byte Folded Reload
 ; RV32IZCMP-NEXT:    sw a0, %lo(var+8)(t0)
-; RV32IZCMP-NEXT:    lw a0, 24(sp) # 4-byte Folded Reload
+; RV32IZCMP-NEXT:    lw a0, 20(sp) # 4-byte Folded Reload
 ; RV32IZCMP-NEXT:    sw a0, %lo(var+4)(t0)
-; RV32IZCMP-NEXT:    lw a0, 28(sp) # 4-byte Folded Reload
+; RV32IZCMP-NEXT:    lw a0, 24(sp) # 4-byte Folded Reload
 ; RV32IZCMP-NEXT:    sw a0, %lo(var)(t0)
-; RV32IZCMP-NEXT:    cm.popret {ra, s0-s11}, 96
+; RV32IZCMP-NEXT:    cm.popret {ra, s0-s11}, 80
 ;
 ; RV32IZCMP-WITH-FP-LABEL: callee:
 ; RV32IZCMP-WITH-FP:       # %bb.0:
 ; RV32IZCMP-WITH-FP-NEXT:    addi sp, sp, -80
+; RV32IZCMP-WITH-FP-NEXT:    .cfi_def_cfa_offset 80
 ; RV32IZCMP-WITH-FP-NEXT:    sw ra, 76(sp) # 4-byte Folded Spill
 ; RV32IZCMP-WITH-FP-NEXT:    sw s0, 72(sp) # 4-byte Folded Spill
 ; RV32IZCMP-WITH-FP-NEXT:    sw s1, 68(sp) # 4-byte Folded Spill
@@ -450,7 +531,21 @@ define void @callee() nounwind {
 ; RV32IZCMP-WITH-FP-NEXT:    sw s9, 36(sp) # 4-byte Folded Spill
 ; RV32IZCMP-WITH-FP-NEXT:    sw s10, 32(sp) # 4-byte Folded Spill
 ; RV32IZCMP-WITH-FP-NEXT:    sw s11, 28(sp) # 4-byte Folded Spill
+; RV32IZCMP-WITH-FP-NEXT:    .cfi_offset ra, -4
+; RV32IZCMP-WITH-FP-NEXT:    .cfi_offset s0, -8
+; RV32IZCMP-WITH-FP-NEXT:    .cfi_offset s1, -12
+; RV32IZCMP-WITH-FP-NEXT:    .cfi_offset s2, -16
+; RV32IZCMP-WITH-FP-NEXT:    .cfi_offset s3, -20
+; RV32IZCMP-WITH-FP-NEXT:    .cfi_offset s4, -24
+; RV32IZCMP-WITH-FP-NEXT:    .cfi_offset s5, -28
+; RV32IZCMP-WITH-FP-NEXT:    .cfi_offset s6, -32
+; RV32IZCMP-WITH-FP-NEXT:    .cfi_offset s7, -36
+; RV32IZCMP-WITH-FP-NEXT:    .cfi_offset s8, -40
+; RV32IZCMP-WITH-FP-NEXT:    .cfi_offset s9, -44
+; RV32IZCMP-WITH-FP-NEXT:    .cfi_offset s10, -48
+; RV32IZCMP-WITH-FP-NEXT:    .cfi_offset s11, -52
 ; RV32IZCMP-WITH-FP-NEXT:    addi s0, sp, 80
+; RV32IZCMP-WITH-FP-NEXT:    .cfi_def_cfa s0, 0
 ; RV32IZCMP-WITH-FP-NEXT:    lui t1, %hi(var)
 ; RV32IZCMP-WITH-FP-NEXT:    lw a0, %lo(var)(t1)
 ; RV32IZCMP-WITH-FP-NEXT:    sw a0, -56(s0) # 4-byte Folded Spill
@@ -531,6 +626,7 @@ define void @callee() nounwind {
 ; RV32IZCMP-WITH-FP-NEXT:    sw a0, %lo(var+4)(t1)
 ; RV32IZCMP-WITH-FP-NEXT:    lw a0, -56(s0) # 4-byte Folded Reload
 ; RV32IZCMP-WITH-FP-NEXT:    sw a0, %lo(var)(t1)
+; RV32IZCMP-WITH-FP-NEXT:    .cfi_def_cfa sp, 80
 ; RV32IZCMP-WITH-FP-NEXT:    lw ra, 76(sp) # 4-byte Folded Reload
 ; RV32IZCMP-WITH-FP-NEXT:    lw s0, 72(sp) # 4-byte Folded Reload
 ; RV32IZCMP-WITH-FP-NEXT:    lw s1, 68(sp) # 4-byte Folded Reload
@@ -544,12 +640,27 @@ define void @callee() nounwind {
 ; RV32IZCMP-WITH-FP-NEXT:    lw s9, 36(sp) # 4-byte Folded Reload
 ; RV32IZCMP-WITH-FP-NEXT:    lw s10, 32(sp) # 4-byte Folded Reload
 ; RV32IZCMP-WITH-FP-NEXT:    lw s11, 28(sp) # 4-byte Folded Reload
+; RV32IZCMP-WITH-FP-NEXT:    .cfi_restore ra
+; RV32IZCMP-WITH-FP-NEXT:    .cfi_restore s0
+; RV32IZCMP-WITH-FP-NEXT:    .cfi_restore s1
+; RV32IZCMP-WITH-FP-NEXT:    .cfi_restore s2
+; RV32IZCMP-WITH-FP-NEXT:    .cfi_restore s3
+; RV32IZCMP-WITH-FP-NEXT:    .cfi_restore s4
+; RV32IZCMP-WITH-FP-NEXT:    .cfi_restore s5
+; RV32IZCMP-WITH-FP-NEXT:    .cfi_restore s6
+; RV32IZCMP-WITH-FP-NEXT:    .cfi_restore s7
+; RV32IZCMP-WITH-FP-NEXT:    .cfi_restore s8
+; RV32IZCMP-WITH-FP-NEXT:    .cfi_restore s9
+; RV32IZCMP-WITH-FP-NEXT:    .cfi_restore s10
+; RV32IZCMP-WITH-FP-NEXT:    .cfi_restore s11
 ; RV32IZCMP-WITH-FP-NEXT:    addi sp, sp, 80
+; RV32IZCMP-WITH-FP-NEXT:    .cfi_def_cfa_offset 0
 ; RV32IZCMP-WITH-FP-NEXT:    ret
 ;
 ; RV64I-LABEL: callee:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -160
+; RV64I-NEXT:    .cfi_def_cfa_offset 160
 ; RV64I-NEXT:    sd ra, 152(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    sd s0, 144(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    sd s1, 136(sp) # 8-byte Folded Spill
@@ -563,6 +674,19 @@ define void @callee() nounwind {
 ; RV64I-NEXT:    sd s9, 72(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    sd s10, 64(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    sd s11, 56(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    .cfi_offset ra, -8
+; RV64I-NEXT:    .cfi_offset s0, -16
+; RV64I-NEXT:    .cfi_offset s1, -24
+; RV64I-NEXT:    .cfi_offset s2, -32
+; RV64I-NEXT:    .cfi_offset s3, -40
+; RV64I-NEXT:    .cfi_offset s4, -48
+; RV64I-NEXT:    .cfi_offset s5, -56
+; RV64I-NEXT:    .cfi_offset s6, -64
+; RV64I-NEXT:    .cfi_offset s7, -72
+; RV64I-NEXT:    .cfi_offset s8, -80
+; RV64I-NEXT:    .cfi_offset s9, -88
+; RV64I-NEXT:    .cfi_offset s10, -96
+; RV64I-NEXT:    .cfi_offset s11, -104
 ; RV64I-NEXT:    lui a7, %hi(var)
 ; RV64I-NEXT:    lw a0, %lo(var)(a7)
 ; RV64I-NEXT:    sd a0, 48(sp) # 8-byte Folded Spill
@@ -654,15 +778,33 @@ define void @callee() nounwind {
 ; RV64I-NEXT:    ld s9, 72(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    ld s10, 64(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    ld s11, 56(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    .cfi_restore ra
+; RV64I-NEXT:    .cfi_restore s0
+; RV64I-NEXT:    .cfi_restore s1
+; RV64I-NEXT:    .cfi_restore s2
+; RV64I-NEXT:    .cfi_restore s3
+; RV64I-NEXT:    .cfi_restore s4
+; RV64I-NEXT:    .cfi_restore s5
+; RV64I-NEXT:    .cfi_restore s6
+; RV64I-NEXT:    .cfi_restore s7
+; RV64I-NEXT:    .cfi_restore s8
+; RV64I-NEXT:    .cfi_restore s9
+; RV64I-NEXT:    .cfi_restore s10
+; RV64I-NEXT:    .cfi_restore s11
 ; RV64I-NEXT:    addi sp, sp, 160
+; RV64I-NEXT:    .cfi_def_cfa_offset 0
 ; RV64I-NEXT:    ret
 ;
 ; RV64I-LP64E-LABEL: callee:
 ; RV64I-LP64E:       # %bb.0:
 ; RV64I-LP64E-NEXT:    addi sp, sp, -72
+; RV64I-LP64E-NEXT:    .cfi_def_cfa_offset 72
 ; RV64I-LP64E-NEXT:    sd ra, 64(sp) # 8-byte Folded Spill
 ; RV64I-LP64E-NEXT:    sd s0, 56(sp) # 8-byte Folded Spill
 ; RV64I-LP64E-NEXT:    sd s1, 48(sp) # 8-byte Folded Spill
+; RV64I-LP64E-NEXT:    .cfi_offset ra, -8
+; RV64I-LP64E-NEXT:    .cfi_offset s0, -16
+; RV64I-LP64E-NEXT:    .cfi_offset s1, -24
 ; RV64I-LP64E-NEXT:    lui a7, %hi(var)
 ; RV64I-LP64E-NEXT:    lw a0, %lo(var)(a7)
 ; RV64I-LP64E-NEXT:    sd a0, 40(sp) # 8-byte Folded Spill
@@ -744,12 +886,17 @@ define void @callee() nounwind {
 ; RV64I-LP64E-NEXT:    ld ra, 64(sp) # 8-byte Folded Reload
 ; RV64I-LP64E-NEXT:    ld s0, 56(sp) # 8-byte Folded Reload
 ; RV64I-LP64E-NEXT:    ld s1, 48(sp) # 8-byte Folded Reload
+; RV64I-LP64E-NEXT:    .cfi_restore ra
+; RV64I-LP64E-NEXT:    .cfi_restore s0
+; RV64I-LP64E-NEXT:    .cfi_restore s1
 ; RV64I-LP64E-NEXT:    addi sp, sp, 72
+; RV64I-LP64E-NEXT:    .cfi_def_cfa_offset 0
 ; RV64I-LP64E-NEXT:    ret
 ;
 ; RV64I-WITH-FP-LABEL: callee:
 ; RV64I-WITH-FP:       # %bb.0:
 ; RV64I-WITH-FP-NEXT:    addi sp, sp, -160
+; RV64I-WITH-FP-NEXT:    .cfi_def_cfa_offset 160
 ; RV64I-WITH-FP-NEXT:    sd ra, 152(sp) # 8-byte Folded Spill
 ; RV64I-WITH-FP-NEXT:    sd s0, 144(sp) # 8-byte Folded Spill
 ; RV64I-WITH-FP-NEXT:    sd s1, 136(sp) # 8-byte Folded Spill
@@ -763,7 +910,21 @@ define void @callee() nounwind {
 ; RV64I-WITH-FP-NEXT:    sd s9, 72(sp) # 8-byte Folded Spill
 ; RV64I-WITH-FP-NEXT:    sd s10, 64(sp) # 8-byte Folded Spill
 ; RV64I-WITH-FP-NEXT:    sd s11, 56(sp) # 8-byte Folded Spill
+; RV64I-WITH-FP-NEXT:    .cfi_offset ra, -8
+; RV64I-WITH-FP-NEXT:    .cfi_offset s0, -16
+; RV64I-WITH-FP-NEXT:    .cfi_offset s1, -24
+; RV64I-WITH-FP-NEXT:    .cfi_offset s2, -32
+; RV64I-WITH-FP-NEXT:    .cfi_offset s3, -40
+; RV64I-WITH-FP-NEXT:    .cfi_offset s4, -48
+; RV64I-WITH-FP-NEXT:    .cfi_offset s5, -56
+; RV64I-WITH-FP-NEXT:    .cfi_offset s6, -64
+; RV64I-WITH-FP-NEXT:    .cfi_offset s7, -72
+; RV64I-WITH-FP-NEXT:    .cfi_offset s8, -80
+; RV64I-WITH-FP-NEXT:    .cfi_offset s9, -88
+; RV64I-WITH-FP-NEXT:    .cfi_offset s10, -96
+; RV64I-WITH-FP-NEXT:    .cfi_offset s11, -104
 ; RV64I-WITH-FP-NEXT:    addi s0, sp, 160
+; RV64I-WITH-FP-NEXT:    .cfi_def_cfa s0, 0
 ; RV64I-WITH-FP-NEXT:    lui t0, %hi(var)
 ; RV64I-WITH-FP-NEXT:    lw a0, %lo(var)(t0)
 ; RV64I-WITH-FP-NEXT:    sd a0, -112(s0) # 8-byte Folded Spill
@@ -844,6 +1005,7 @@ define void @callee() nounwind {
 ; RV64I-WITH-FP-NEXT:    sw a0, %lo(var+4)(t0)
 ; RV64I-WITH-FP-NEXT:    ld a0, -112(s0) # 8-byte Folded Reload
 ; RV64I-WITH-FP-NEXT:    sw a0, %lo(var)(t0)
+; RV64I-WITH-FP-NEXT:    .cfi_def_cfa sp, 160
 ; RV64I-WITH-FP-NEXT:    ld ra, 152(sp) # 8-byte Folded Reload
 ; RV64I-WITH-FP-NEXT:    ld s0, 144(sp) # 8-byte Folded Reload
 ; RV64I-WITH-FP-NEXT:    ld s1, 136(sp) # 8-byte Folded Reload
@@ -857,26 +1019,54 @@ define void @callee() nounwind {
 ; RV64I-WITH-FP-NEXT:    ld s9, 72(sp) # 8-byte Folded Reload
 ; RV64I-WITH-FP-NEXT:    ld s10, 64(sp) # 8-byte Folded Reload
 ; RV64I-WITH-FP-NEXT:    ld s11, 56(sp) # 8-byte Folded Reload
+; RV64I-WITH-FP-NEXT:    .cfi_restore ra
+; RV64I-WITH-FP-NEXT:    .cfi_restore s0
+; RV64I-WITH-FP-NEXT:    .cfi_restore s1
+; RV64I-WITH-FP-NEXT:    .cfi_restore s2
+; RV64I-WITH-FP-NEXT:    .cfi_restore s3
+; RV64I-WITH-FP-NEXT:    .cfi_restore s4
+; RV64I-WITH-FP-NEXT:    .cfi_restore s5
+; RV64I-WITH-FP-NEXT:    .cfi_restore s6
+; RV64I-WITH-FP-NEXT:    .cfi_restore s7
+; RV64I-WITH-FP-NEXT:    .cfi_restore s8
+; RV64I-WITH-FP-NEXT:    .cfi_restore s9
+; RV64I-WITH-FP-NEXT:    .cfi_restore s10
+; RV64I-WITH-FP-NEXT:    .cfi_restore s11
 ; RV64I-WITH-FP-NEXT:    addi sp, sp, 160
+; RV64I-WITH-FP-NEXT:    .cfi_def_cfa_offset 0
 ; RV64I-WITH-FP-NEXT:    ret
 ;
 ; RV64IZCMP-LABEL: callee:
 ; RV64IZCMP:       # %bb.0:
 ; RV64IZCMP-NEXT:    cm.push {ra, s0-s11}, -160
+; RV64IZCMP-NEXT:    .cfi_def_cfa_offset 160
+; RV64IZCMP-NEXT:    .cfi_offset ra, -104
+; RV64IZCMP-NEXT:    .cfi_offset s0, -96
+; RV64IZCMP-NEXT:    .cfi_offset s1, -88
+; RV64IZCMP-NEXT:    .cfi_offset s2, -80
+; RV64IZCMP-NEXT:    .cfi_offset s3, -72
+; RV64IZCMP-NEXT:    .cfi_offset s4, -64
+; RV64IZCMP-NEXT:    .cfi_offset s5, -56
+; RV64IZCMP-NEXT:    .cfi_offset s6, -48
+; RV64IZCMP-NEXT:    .cfi_offset s7, -40
+; RV64IZCMP-NEXT:    .cfi_offset s8, -32
+; RV64IZCMP-NEXT:    .cfi_offset s9, -24
+; RV64IZCMP-NEXT:    .cfi_offset s10, -16
+; RV64IZCMP-NEXT:    .cfi_offset s11, -8
 ; RV64IZCMP-NEXT:    lui t0, %hi(var)
 ; RV64IZCMP-NEXT:    lw a0, %lo(var)(t0)
-; RV64IZCMP-NEXT:    sd a0, 40(sp) # 8-byte Folded Spill
+; RV64IZCMP-NEXT:    sd a0, 48(sp) # 8-byte Folded Spill
 ; RV64IZCMP-NEXT:    lw a0, %lo(var+4)(t0)
-; RV64IZCMP-NEXT:    sd a0, 32(sp) # 8-byte Folded Spill
+; RV64IZCMP-NEXT:    sd a0, 40(sp) # 8-byte Folded Spill
 ; RV64IZCMP-NEXT:    lw a0, %lo(var+8)(t0)
-; RV64IZCMP-NEXT:    sd a0, 24(sp) # 8-byte Folded Spill
+; RV64IZCMP-NEXT:    sd a0, 32(sp) # 8-byte Folded Spill
 ; RV64IZCMP-NEXT:    lw a0, %lo(var+12)(t0)
-; RV64IZCMP-NEXT:    sd a0, 16(sp) # 8-byte Folded Spill
+; RV64IZCMP-NEXT:    sd a0, 24(sp) # 8-byte Folded Spill
 ; RV64IZCMP-NEXT:    addi a5, t0, %lo(var)
 ; RV64IZCMP-NEXT:    lw a0, 16(a5)
-; RV64IZCMP-NEXT:    sd a0, 8(sp) # 8-byte Folded Spill
+; RV64IZCMP-NEXT:    sd a0, 16(sp) # 8-byte Folded Spill
 ; RV64IZCMP-NEXT:    lw a0, 20(a5)
-; RV64IZCMP-NEXT:    sd a0, 0(sp) # 8-byte Folded Spill
+; RV64IZCMP-NEXT:    sd a0, 8(sp) # 8-byte Folded Spill
 ; RV64IZCMP-NEXT:    lw t4, 24(a5)
 ; RV64IZCMP-NEXT:    lw t5, 28(a5)
 ; RV64IZCMP-NEXT:    lw t6, 32(a5)
@@ -929,23 +1119,24 @@ define void @callee() nounwind {
 ; RV64IZCMP-NEXT:    sw t6, 32(a5)
 ; RV64IZCMP-NEXT:    sw t5, 28(a5)
 ; RV64IZCMP-NEXT:    sw t4, 24(a5)
-; RV64IZCMP-NEXT:    ld a0, 0(sp) # 8-byte Folded Reload
-; RV64IZCMP-NEXT:    sw a0, 20(a5)
 ; RV64IZCMP-NEXT:    ld a0, 8(sp) # 8-byte Folded Reload
-; RV64IZCMP-NEXT:    sw a0, 16(a5)
+; RV64IZCMP-NEXT:    sw a0, 20(a5)
 ; RV64IZCMP-NEXT:    ld a0, 16(sp) # 8-byte Folded Reload
-; RV64IZCMP-NEXT:    sw a0, %lo(var+12)(t0)
+; RV64IZCMP-NEXT:    sw a0, 16(a5)
 ; RV64IZCMP-NEXT:    ld a0, 24(sp) # 8-byte Folded Reload
-; RV64IZCMP-NEXT:    sw a0, %lo(var+8)(t0)
+; RV64IZCMP-NEXT:    sw a0, %lo(var+12)(t0)
 ; RV64IZCMP-NEXT:    ld a0, 32(sp) # 8-byte Folded Reload
-; RV64IZCMP-NEXT:    sw a0, %lo(var+4)(t0)
+; RV64IZCMP-NEXT:    sw a0, %lo(var+8)(t0)
 ; RV64IZCMP-NEXT:    ld a0, 40(sp) # 8-byte Folded Reload
+; RV64IZCMP-NEXT:    sw a0, %lo(var+4)(t0)
+; RV64IZCMP-NEXT:    ld a0, 48(sp) # 8-byte Folded Reload
 ; RV64IZCMP-NEXT:    sw a0, %lo(var)(t0)
 ; RV64IZCMP-NEXT:    cm.popret {ra, s0-s11}, 160
 ;
 ; RV64IZCMP-WITH-FP-LABEL: callee:
 ; RV64IZCMP-WITH-FP:       # %bb.0:
 ; RV64IZCMP-WITH-FP-NEXT:    addi sp, sp, -160
+; RV64IZCMP-WITH-FP-NEXT:    .cfi_def_cfa_offset 160
 ; RV64IZCMP-WITH-FP-NEXT:    sd ra, 152(sp) # 8-byte Folded Spill
 ; RV64IZCMP-WITH-FP-NEXT:    sd s0, 144(sp) # 8-byte Folded Spill
 ; RV64IZCMP-WITH-FP-NEXT:    sd s1, 136(sp) # 8-byte Folded Spill
@@ -959,7 +1150,21 @@ define void @callee() nounwind {
 ; RV64IZCMP-WITH-FP-NEXT:    sd s9, 72(sp) # 8-byte Folded Spill
 ; RV64IZCMP-WITH-FP-NEXT:    sd s10, 64(sp) # 8-byte Folded Spill
 ; RV64IZCMP-WITH-FP-NEXT:    sd s11, 56(sp) # 8-byte Folded Spill
+; RV64IZCMP-WITH-FP-NEXT:    .cfi_offset ra, -8
+; RV64IZCMP-WITH-FP-NEXT:    .cfi_offset s0, -16
+; RV64IZCMP-WITH-FP-NEXT:    .cfi_offset s1, -24
+; RV64IZCMP-WITH-FP-NEXT:    .cfi_offset s2, -32
+; RV64IZCMP-WITH-FP-NEXT:    .cfi_offset s3, -40
+; RV64IZCMP-WITH-FP-NEXT:    .cfi_offset s4, -48
+; RV64IZCMP-WITH-FP-NEXT:    .cfi_offset s5, -56
+; RV64IZCMP-WITH-FP-NEXT:    .cfi_offset s6, -64
+; RV64IZCMP-WITH-FP-NEXT:    .cfi_offset s7, -72
+; RV64IZCMP-WITH-FP-NEXT:    .cfi_offset s8, -80
+; RV64IZCMP-WITH-FP-NEXT:    .cfi_offset s9, -88
+; RV64IZCMP-WITH-FP-NEXT:    .cfi_offset s10, -96
+; RV64IZCMP-WITH-FP-NEXT:    .cfi_offset s11, -104
 ; RV64IZCMP-WITH-FP-NEXT:    addi s0, sp, 160
+; RV64IZCMP-WITH-FP-NEXT:    .cfi_def_cfa s0, 0
 ; RV64IZCMP-WITH-FP-NEXT:    lui t1, %hi(var)
 ; RV64IZCMP-WITH-FP-NEXT:    lw a0, %lo(var)(t1)
 ; RV64IZCMP-WITH-FP-NEXT:    sd a0, -112(s0) # 8-byte Folded Spill
@@ -1040,6 +1245,7 @@ define void @callee() nounwind {
 ; RV64IZCMP-WITH-FP-NEXT:    sw a0, %lo(var+4)(t1)
 ; RV64IZCMP-WITH-FP-NEXT:    ld a0, -112(s0) # 8-byte Folded Reload
 ; RV64IZCMP-WITH-FP-NEXT:    sw a0, %lo(var)(t1)
+; RV64IZCMP-WITH-FP-NEXT:    .cfi_def_cfa sp, 160
 ; RV64IZCMP-WITH-FP-NEXT:    ld ra, 152(sp) # 8-byte Folded Reload
 ; RV64IZCMP-WITH-FP-NEXT:    ld s0, 144(sp) # 8-byte Folded Reload
 ; RV64IZCMP-WITH-FP-NEXT:    ld s1, 136(sp) # 8-byte Folded Reload
@@ -1053,7 +1259,21 @@ define void @callee() nounwind {
 ; RV64IZCMP-WITH-FP-NEXT:    ld s9, 72(sp) # 8-byte Folded Reload
 ; RV64IZCMP-WITH-FP-NEXT:    ld s10, 64(sp) # 8-byte Folded Reload
 ; RV64IZCMP-WITH-FP-NEXT:    ld s11, 56(sp) # 8-byte Folded Reload
+; RV64IZCMP-WITH-FP-NEXT:    .cfi_restore ra
+; RV64IZCMP-WITH-FP-NEXT:    .cfi_restore s0
+; RV64IZCMP-WITH-FP-NEXT:    .cfi_restore s1
+; RV64IZCMP-WITH-FP-NEXT:    .cfi_restore s2
+; RV64IZCMP-WITH-FP-NEXT:    .cfi_restore s3
+; RV64IZCMP-WITH-FP-NEXT:    .cfi_restore s4
+; RV64IZCMP-WITH-FP-NEXT:    .cfi_restore s5
+; RV64IZCMP-WITH-FP-NEXT:    .cfi_restore s6
+; RV64IZCMP-WITH-FP-NEXT:    .cfi_restore s7
+; RV64IZCMP-WITH-FP-NEXT:    .cfi_restore s8
+; RV64IZCMP-WITH-FP-NEXT:    .cfi_restore s9
+; RV64IZCMP-WITH-FP-NEXT:    .cfi_restore s10
+; RV64IZCMP-WITH-FP-NEXT:    .cfi_restore s11
 ; RV64IZCMP-WITH-FP-NEXT:    addi sp, sp, 160
+; RV64IZCMP-WITH-FP-NEXT:    .cfi_def_cfa_offset 0
 ; RV64IZCMP-WITH-FP-NEXT:    ret
   %val = load [32 x i32], ptr @var
   store volatile [32 x i32] %val, ptr @var
@@ -1063,10 +1283,11 @@ define void @callee() nounwind {
 ; This function tests that RISCVRegisterInfo::getCallPreservedMask returns
 ; something appropriate.
 
-define void @caller() nounwind {
+define void @caller() {
 ; RV32I-LABEL: caller:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -144
+; RV32I-NEXT:    .cfi_def_cfa_offset 144
 ; RV32I-NEXT:    sw ra, 140(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    sw s0, 136(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    sw s1, 132(sp) # 4-byte Folded Spill
@@ -1080,6 +1301,19 @@ define void @caller() nounwind {
 ; RV32I-NEXT:    sw s9, 100(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    sw s10, 96(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    sw s11, 92(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    .cfi_offset ra, -4
+; RV32I-NEXT:    .cfi_offset s0, -8
+; RV32I-NEXT:    .cfi_offset s1, -12
+; RV32I-NEXT:    .cfi_offset s2, -16
+; RV32I-NEXT:    .cfi_offset s3, -20
+; RV32I-NEXT:    .cfi_offset s4, -24
+; RV32I-NEXT:    .cfi_offset s5, -28
+; RV32I-NEXT:    .cfi_offset s6, -32
+; RV32I-NEXT:    .cfi_offset s7, -36
+; RV32I-NEXT:    .cfi_offset s8, -40
+; RV32I-NEXT:    .cfi_offset s9, -44
+; RV32I-NEXT:    .cfi_offset s10, -48
+; RV32I-NEXT:    .cfi_offset s11, -52
 ; RV32I-NEXT:    lui s0, %hi(var)
 ; RV32I-NEXT:    lw a0, %lo(var)(s0)
 ; RV32I-NEXT:    sw a0, 88(sp) # 4-byte Folded Spill
@@ -1204,15 +1438,33 @@ define void @caller() nounwind {
 ; RV32I-NEXT:    lw s9, 100(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    lw s10, 96(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    lw s11, 92(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    .cfi_restore ra
+; RV32I-NEXT:    .cfi_restore s0
+; RV32I-NEXT:    .cfi_restore s1
+; RV32I-NEXT:    .cfi_restore s2
+; RV32I-NEXT:    .cfi_restore s3
+; RV32I-NEXT:    .cfi_restore s4
+; RV32I-NEXT:    .cfi_restore s5
+; RV32I-NEXT:    .cfi_restore s6
+; RV32I-NEXT:    .cfi_restore s7
+; RV32I-NEXT:    .cfi_restore s8
+; RV32I-NEXT:    .cfi_restore s9
+; RV32I-NEXT:    .cfi_restore s10
+; RV32I-NEXT:    .cfi_restore s11
 ; RV32I-NEXT:    addi sp, sp, 144
+; RV32I-NEXT:    .cfi_def_cfa_offset 0
 ; RV32I-NEXT:    ret
 ;
 ; RV32I-ILP32E-LABEL: caller:
 ; RV32I-ILP32E:       # %bb.0:
 ; RV32I-ILP32E-NEXT:    addi sp, sp, -136
+; RV32I-ILP32E-NEXT:    .cfi_def_cfa_offset 136
 ; RV32I-ILP32E-NEXT:    sw ra, 132(sp) # 4-byte Folded Spill
 ; RV32I-ILP32E-NEXT:    sw s0, 128(sp) # 4-byte Folded Spill
 ; RV32I-ILP32E-NEXT:    sw s1, 124(sp) # 4-byte Folded Spill
+; RV32I-ILP32E-NEXT:    .cfi_offset ra, -4
+; RV32I-ILP32E-NEXT:    .cfi_offset s0, -8
+; RV32I-ILP32E-NEXT:    .cfi_offset s1, -12
 ; RV32I-ILP32E-NEXT:    lui a0, %hi(var)
 ; RV32I-ILP32E-NEXT:    lw a1, %lo(var)(a0)
 ; RV32I-ILP32E-NEXT:    sw a1, 120(sp) # 4-byte Folded Spill
@@ -1346,12 +1598,17 @@ define void @caller() nounwind {
 ; RV32I-ILP32E-NEXT:    lw ra, 132(sp) # 4-byte Folded Reload
 ; RV32I-ILP32E-NEXT:    lw s0, 128(sp) # 4-byte Folded Reload
 ; RV32I-ILP32E-NEXT:    lw s1, 124(sp) # 4-byte Folded Reload
+; RV32I-ILP32E-NEXT:    .cfi_restore ra
+; RV32I-ILP32E-NEXT:    .cfi_restore s0
+; RV32I-ILP32E-NEXT:    .cfi_restore s1
 ; RV32I-ILP32E-NEXT:    addi sp, sp, 136
+; RV32I-ILP32E-NEXT:    .cfi_def_cfa_offset 0
 ; RV32I-ILP32E-NEXT:    ret
 ;
 ; RV32I-WITH-FP-LABEL: caller:
 ; RV32I-WITH-FP:       # %bb.0:
 ; RV32I-WITH-FP-NEXT:    addi sp, sp, -144
+; RV32I-WITH-FP-NEXT:    .cfi_def_cfa_offset 144
 ; RV32I-WITH-FP-NEXT:    sw ra, 140(sp) # 4-byte Folded Spill
 ; RV32I-WITH-FP-NEXT:    sw s0, 136(sp) # 4-byte Folded Spill
 ; RV32I-WITH-FP-NEXT:    sw s1, 132(sp) # 4-byte Folded Spill
@@ -1365,7 +1622,21 @@ define void @caller() nounwind {
 ; RV32I-WITH-FP-NEXT:    sw s9, 100(sp) # 4-byte Folded Spill
 ; RV32I-WITH-FP-NEXT:    sw s10, 96(sp) # 4-byte Folded Spill
 ; RV32I-WITH-FP-NEXT:    sw s11, 92(sp) # 4-byte Folded Spill
+; RV32I-WITH-FP-NEXT:    .cfi_offset ra, -4
+; RV32I-WITH-FP-NEXT:    .cfi_offset s0, -8
+; RV32I-WITH-FP-NEXT:    .cfi_offset s1, -12
+; RV32I-WITH-FP-NEXT:    .cfi_offset s2, -16
+; RV32I-WITH-FP-NEXT:    .cfi_offset s3, -20
+; RV32I-WITH-FP-NEXT:    .cfi_offset s4, -24
+; RV32I-WITH-FP-NEXT:    .cfi_offset s5, -28
+; RV32I-WITH-FP-NEXT:    .cfi_offset s6, -32
+; RV32I-WITH-FP-NEXT:    .cfi_offset s7, -36
+; RV32I-WITH-FP-NEXT:    .cfi_offset s8, -40
+; RV32I-WITH-FP-NEXT:    .cfi_offset s9, -44
+; RV32I-WITH-FP-NEXT:    .cfi_offset s10, -48
+; RV32I-WITH-FP-NEXT:    .cfi_offset s11, -52
 ; RV32I-WITH-FP-NEXT:    addi s0, sp, 144
+; RV32I-WITH-FP-NEXT:    .cfi_def_cfa s0, 0
 ; RV32I-WITH-FP-NEXT:    lui s1, %hi(var)
 ; RV32I-WITH-FP-NEXT:    lw a0, %lo(var)(s1)
 ; RV32I-WITH-FP-NEXT:    sw a0, -56(s0) # 4-byte Folded Spill
@@ -1479,6 +1750,7 @@ define void @caller() nounwind {
 ; RV32I-WITH-FP-NEXT:    sw a0, %lo(var+4)(s1)
 ; RV32I-WITH-FP-NEXT:    lw a0, -56(s0) # 4-byte Folded Reload
 ; RV32I-WITH-FP-NEXT:    sw a0, %lo(var)(s1)
+; RV32I-WITH-FP-NEXT:    .cfi_def_cfa sp, 144
 ; RV32I-WITH-FP-NEXT:    lw ra, 140(sp) # 4-byte Folded Reload
 ; RV32I-WITH-FP-NEXT:    lw s0, 136(sp) # 4-byte Folded Reload
 ; RV32I-WITH-FP-NEXT:    lw s1, 132(sp) # 4-byte Folded Reload
@@ -1492,59 +1764,88 @@ define void @caller() nounwind {
 ; RV32I-WITH-FP-NEXT:    lw s9, 100(sp) # 4-byte Folded Reload
 ; RV32I-WITH-FP-NEXT:    lw s10, 96(sp) # 4-byte Folded Reload
 ; RV32I-WITH-FP-NEXT:    lw s11, 92(sp) # 4-byte Folded Reload
+; RV32I-WITH-FP-NEXT:    .cfi_restore ra
+; RV32I-WITH-FP-NEXT:    .cfi_restore s0
+; RV32I-WITH-FP-NEXT:    .cfi_restore s1
+; RV32I-WITH-FP-NEXT:    .cfi_restore s2
+; RV32I-WITH-FP-NEXT:    .cfi_restore s3
+; RV32I-WITH-FP-NEXT:    .cfi_restore s4
+; RV32I-WITH-FP-NEXT:    .cfi_restore s5
+; RV32I-WITH-FP-NEXT:    .cfi_restore s6
+; RV32I-WITH-FP-NEXT:    .cfi_restore s7
+; RV32I-WITH-FP-NEXT:    .cfi_restore s8
+; RV32I-WITH-FP-NEXT:    .cfi_restore s9
+; RV32I-WITH-FP-NEXT:    .cfi_restore s10
+; RV32I-WITH-FP-NEXT:    .cfi_restore s11
 ; RV32I-WITH-FP-NEXT:    addi sp, sp, 144
+; RV32I-WITH-FP-NEXT:    .cfi_def_cfa_offset 0
 ; RV32I-WITH-FP-NEXT:    ret
 ;
 ; RV32IZCMP-LABEL: caller:
 ; RV32IZCMP:       # %bb.0:
 ; RV32IZCMP-NEXT:    cm.push {ra, s0-s11}, -112
-; RV32IZCMP-NEXT:    addi sp, sp, -48
+; RV32IZCMP-NEXT:    .cfi_def_cfa_offset 112
+; RV32IZCMP-NEXT:    .cfi_offset ra, -52
+; RV32IZCMP-NEXT:    .cfi_offset s0, -48
+; RV32IZCMP-NEXT:    .cfi_offset s1, -44
+; RV32IZCMP-NEXT:    .cfi_offset s2, -40
+; RV32IZCMP-NEXT:    .cfi_offset s3, -36
+; RV32IZCMP-NEXT:    .cfi_offset s4, -32
+; RV32IZCMP-NEXT:    .cfi_offset s5, -28
+; RV32IZCMP-NEXT:    .cfi_offset s6, -24
+; RV32IZCMP-NEXT:    .cfi_offset s7, -20
+; RV32IZCMP-NEXT:    .cfi_offset s8, -16
+; RV32IZCMP-NEXT:    .cfi_offset s9, -12
+; RV32IZCMP-NEXT:    .cfi_offset s10, -8
+; RV32IZCMP-NEXT:    .cfi_offset s11, -4
+; RV32IZCMP-NEXT:    addi sp, sp, -32
+; RV32IZCMP-NEXT:    .cfi_def_cfa_offset 144
 ; RV32IZCMP-NEXT:    lui s0, %hi(var)
 ; RV32IZCMP-NEXT:    lw a0, %lo(var)(s0)
-; RV32IZCMP-NEXT:    sw a0, 92(sp) # 4-byte Folded Spill
-; RV32IZCMP-NEXT:    lw a0, %lo(var+4)(s0)
 ; RV32IZCMP-NEXT:    sw a0, 88(sp) # 4-byte Folded Spill
-; RV32IZCMP-NEXT:    lw a0, %lo(var+8)(s0)
+; RV32IZCMP-NEXT:    lw a0, %lo(var+4)(s0)
 ; RV32IZCMP-NEXT:    sw a0, 84(sp) # 4-byte Folded Spill
-; RV32IZCMP-NEXT:    lw a0, %lo(var+12)(s0)
+; RV32IZCMP-NEXT:    lw a0, %lo(var+8)(s0)
 ; RV32IZCMP-NEXT:    sw a0, 80(sp) # 4-byte Folded Spill
+; RV32IZCMP-NEXT:    lw a0, %lo(var+12)(s0)
+; RV32IZCMP-NEXT:    sw a0, 76(sp) # 4-byte Folded Spill
 ; RV32IZCMP-NEXT:    addi s1, s0, %lo(var)
 ; RV32IZCMP-NEXT:    lw a0, 16(s1)
-; RV32IZCMP-NEXT:    sw a0, 76(sp) # 4-byte Folded Spill
-; RV32IZCMP-NEXT:    lw a0, 20(s1)
 ; RV32IZCMP-NEXT:    sw a0, 72(sp) # 4-byte Folded Spill
-; RV32IZCMP-NEXT:    lw a0, 24(s1)
+; RV32IZCMP-NEXT:    lw a0, 20(s1)
 ; RV32IZCMP-NEXT:    sw a0, 68(sp) # 4-byte Folded Spill
-; RV32IZCMP-NEXT:    lw a0, 28(s1)
+; RV32IZCMP-NEXT:    lw a0, 24(s1)
 ; RV32IZCMP-NEXT:    sw a0, 64(sp) # 4-byte Folded Spill
-; RV32IZCMP-NEXT:    lw a0, 32(s1)
+; RV32IZCMP-NEXT:    lw a0, 28(s1)
 ; RV32IZCMP-NEXT:    sw a0, 60(sp) # 4-byte Folded Spill
-; RV32IZCMP-NEXT:    lw a0, 36(s1)
+; RV32IZCMP-NEXT:    lw a0, 32(s1)
 ; RV32IZCMP-NEXT:    sw a0, 56(sp) # 4-byte Folded Spill
-; RV32IZCMP-NEXT:    lw a0, 40(s1)
+; RV32IZCMP-NEXT:    lw a0, 36(s1)
 ; RV32IZCMP-NEXT:    sw a0, 52(sp) # 4-byte Folded Spill
-; RV32IZCMP-NEXT:    lw a0, 44(s1)
+; RV32IZCMP-NEXT:    lw a0, 40(s1)
 ; RV32IZCMP-NEXT:    sw a0, 48(sp) # 4-byte Folded Spill
-; RV32IZCMP-NEXT:    lw a0, 48(s1)
+; RV32IZCMP-NEXT:    lw a0, 44(s1)
 ; RV32IZCMP-NEXT:    sw a0, 44(sp) # 4-byte Folded Spill
-; RV32IZCMP-NEXT:    lw a0, 52(s1)
+; RV32IZCMP-NEXT:    lw a0, 48(s1)
 ; RV32IZCMP-NEXT:    sw a0, 40(sp) # 4-byte Folded Spill
-; RV32IZCMP-NEXT:    lw a0, 56(s1)
+; RV32IZCMP-NEXT:    lw a0, 52(s1)
 ; RV32IZCMP-NEXT:    sw a0, 36(sp) # 4-byte Folded Spill
-; RV32IZCMP-NEXT:    lw a0, 60(s1)
+; RV32IZCMP-NEXT:    lw a0, 56(s1)
 ; RV32IZCMP-NEXT:    sw a0, 32(sp) # 4-byte Folded Spill
-; RV32IZCMP-NEXT:    lw a0, 64(s1)
+; RV32IZCMP-NEXT:    lw a0, 60(s1)
 ; RV32IZCMP-NEXT:    sw a0, 28(sp) # 4-byte Folded Spill
-; RV32IZCMP-NEXT:    lw a0, 68(s1)
+; RV32IZCMP-NEXT:    lw a0, 64(s1)
 ; RV32IZCMP-NEXT:    sw a0, 24(sp) # 4-byte Folded Spill
-; RV32IZCMP-NEXT:    lw a0, 72(s1)
+; RV32IZCMP-NEXT:    lw a0, 68(s1)
 ; RV32IZCMP-NEXT:    sw a0, 20(sp) # 4-byte Folded Spill
-; RV32IZCMP-NEXT:    lw a0, 76(s1)
+; RV32IZCMP-NEXT:    lw a0, 72(s1)
 ; RV32IZCMP-NEXT:    sw a0, 16(sp) # 4-byte Folded Spill
-; RV32IZCMP-NEXT:    lw a0, 80(s1)
+; RV32IZCMP-NEXT:    lw a0, 76(s1)
 ; RV32IZCMP-NEXT:    sw a0, 12(sp) # 4-byte Folded Spill
-; RV32IZCMP-NEXT:    lw a0, 84(s1)
+; RV32IZCMP-NEXT:    lw a0, 80(s1)
 ; RV32IZCMP-NEXT:    sw a0, 8(sp) # 4-byte Folded Spill
+; RV32IZCMP-NEXT:    lw a0, 84(s1)
+; RV32IZCMP-NEXT:    sw a0, 4(sp) # 4-byte Folded Spill
 ; RV32IZCMP-NEXT:    lw s4, 88(s1)
 ; RV32IZCMP-NEXT:    lw s5, 92(s1)
 ; RV32IZCMP-NEXT:    lw s6, 96(s1)
@@ -1566,56 +1867,58 @@ define void @caller() nounwind {
 ; RV32IZCMP-NEXT:    sw s6, 96(s1)
 ; RV32IZCMP-NEXT:    sw s5, 92(s1)
 ; RV32IZCMP-NEXT:    sw s4, 88(s1)
-; RV32IZCMP-NEXT:    lw a0, 8(sp) # 4-byte Folded Reload
+; RV32IZCMP-NEXT:    lw a0, 4(sp) # 4-byte Folded Reload
 ; RV32IZCMP-NEXT:    sw a0, 84(s1)
-; RV32IZCMP-NEXT:    lw a0, 12(sp) # 4-byte Folded Reload
+; RV32IZCMP-NEXT:    lw a0, 8(sp) # 4-byte Folded Reload
 ; RV32IZCMP-NEXT:    sw a0, 80(s1)
-; RV32IZCMP-NEXT:    lw a0, 16(sp) # 4-byte Folded Reload
+; RV32IZCMP-NEXT:    lw a0, 12(sp) # 4-byte Folded Reload
 ; RV32IZCMP-NEXT:    sw a0, 76(s1)
-; RV32IZCMP-NEXT:    lw a0, 20(sp) # 4-byte Folded Reload
+; RV32IZCMP-NEXT:    lw a0, 16(sp) # 4-byte Folded Reload
 ; RV32IZCMP-NEXT:    sw a0, 72(s1)
-; RV32IZCMP-NEXT:    lw a0, 24(sp) # 4-byte Folded Reload
+; RV32IZCMP-NEXT:    lw a0, 20(sp) # 4-byte Folded Reload
 ; RV32IZCMP-NEXT:    sw a0, 68(s1)
-; RV32IZCMP-NEXT:    lw a0, 28(sp) # 4-byte Folded Reload
+; RV32IZCMP-NEXT:    lw a0, 24(sp) # 4-byte Folded Reload
 ; RV32IZCMP-NEXT:    sw a0, 64(s1)
-; RV32IZCMP-NEXT:    lw a0, 32(sp) # 4-byte Folded Reload
+; RV32IZCMP-NEXT:    lw a0, 28(sp) # 4-byte Folded Reload
 ; RV32IZCMP-NEXT:    sw a0, 60(s1)
-; RV32IZCMP-NEXT:    lw a0, 36(sp) # 4-byte Folded Reload
+; RV32IZCMP-NEXT:    lw a0, 32(sp) # 4-byte Folded Reload
 ; RV32IZCMP-NEXT:    sw a0, 56(s1)
-; RV32IZCMP-NEXT:    lw a0, 40(sp) # 4-byte Folded Reload
+; RV32IZCMP-NEXT:    lw a0, 36(sp) # 4-byte Folded Reload
 ; RV32IZCMP-NEXT:    sw a0, 52(s1)
-; RV32IZCMP-NEXT:    lw a0, 44(sp) # 4-byte Folded Reload
+; RV32IZCMP-NEXT:    lw a0, 40(sp) # 4-byte Folded Reload
 ; RV32IZCMP-NEXT:    sw a0, 48(s1)
-; RV32IZCMP-NEXT:    lw a0, 48(sp) # 4-byte Folded Reload
+; RV32IZCMP-NEXT:    lw a0, 44(sp) # 4-byte Folded Reload
 ; RV32IZCMP-NEXT:    sw a0, 44(s1)
-; RV32IZCMP-NEXT:    lw a0, 52(sp) # 4-byte Folded Reload
+; RV32IZCMP-NEXT:    lw a0, 48(sp) # 4-byte Folded Reload
 ; RV32IZCMP-NEXT:    sw a0, 40(s1)
-; RV32IZCMP-NEXT:    lw a0, 56(sp) # 4-byte Folded Reload
+; RV32IZCMP-NEXT:    lw a0, 52(sp) # 4-byte Folded Reload
 ; RV32IZCMP-NEXT:    sw a0, 36(s1)
-; RV32IZCMP-NEXT:    lw a0, 60(sp) # 4-byte Folded Reload
+; RV32IZCMP-NEXT:    lw a0, 56(sp) # 4-byte Folded Reload
 ; RV32IZCMP-NEXT:    sw a0, 32(s1)
-; RV32IZCMP-NEXT:    lw a0, 64(sp) # 4-byte Folded Reload
+; RV32IZCMP-NEXT:    lw a0, 60(sp) # 4-byte Folded Reload
 ; RV32IZCMP-NEXT:    sw a0, 28(s1)
-; RV32IZCMP-NEXT:    lw a0, 68(sp) # 4-byte Folded Reload
+; RV32IZCMP-NEXT:    lw a0, 64(sp) # 4-byte Folded Reload
 ; RV32IZCMP-NEXT:    sw a0, 24(s1)
-; RV32IZCMP-NEXT:    lw a0, 72(sp) # 4-byte Folded Reload
+; RV32IZCMP-NEXT:    lw a0, 68(sp) # 4-byte Folded Reload
 ; RV32IZCMP-NEXT:    sw a0, 20(s1)
-; RV32IZCMP-NEXT:    lw a0, 76(sp) # 4-byte Folded Reload
+; RV32IZCMP-NEXT:    lw a0, 72(sp) # 4-byte Folded Reload
 ; RV32IZCMP-NEXT:    sw a0, 16(s1)
-; RV32IZCMP-NEXT:    lw a0, 80(sp) # 4-byte Folded Reload
+; RV32IZCMP-NEXT:    lw a0, 76(sp) # 4-byte Folded Reload
 ; RV32IZCMP-NEXT:    sw a0, %lo(var+12)(s0)
-; RV32IZCMP-NEXT:    lw a0, 84(sp) # 4-byte Folded Reload
+; RV32IZCMP-NEXT:    lw a0, 80(sp) # 4-byte Folded Reload
 ; RV32IZCMP-NEXT:    sw a0, %lo(var+8)(s0)
-; RV32IZCMP-NEXT:    lw a0, 88(sp) # 4-byte Folded Reload
+; RV32IZCMP-NEXT:    lw a0, 84(sp) # 4-byte Folded Reload
 ; RV32IZCMP-NEXT:    sw a0, %lo(var+4)(s0)
-; RV32IZCMP-NEXT:    lw a0, 92(sp) # 4-byte Folded Reload
+; RV32IZCMP-NEXT:    lw a0, 88(sp) # 4-byte Folded Reload
 ; RV32IZCMP-NEXT:    sw a0, %lo(var)(s0)
-; RV32IZCMP-NEXT:    addi sp, sp, 48
+; RV32IZCMP-NEXT:    addi sp, sp, 32
+; RV32IZCMP-NEXT:    .cfi_def_cfa_offset 112
 ; RV32IZCMP-NEXT:    cm.popret {ra, s0-s11}, 112
 ;
 ; RV32IZCMP-WITH-FP-LABEL: caller:
 ; RV32IZCMP-WITH-FP:       # %bb.0:
 ; RV32IZCMP-WITH-FP-NEXT:    addi sp, sp, -144
+; RV32IZCMP-WITH-FP-NEXT:    .cfi_def_cfa_offset 144
 ; RV32IZCMP-WITH-FP-NEXT:    sw ra, 140(sp) # 4-byte Folded Spill
 ; RV32IZCMP-WITH-FP-NEXT:    sw s0, 136(sp) # 4-byte Folded Spill
 ; RV32IZCMP-WITH-FP-NEXT:    sw s1, 132(sp) # 4-byte Folded Spill
@@ -1629,7 +1932,21 @@ define void @caller() nounwind {
 ; RV32IZCMP-WITH-FP-NEXT:    sw s9, 100(sp) # 4-byte Folded Spill
 ; RV32IZCMP-WITH-FP-NEXT:    sw s10, 96(sp) # 4-byte Folded Spill
 ; RV32IZCMP-WITH-FP-NEXT:    sw s11, 92(sp) # 4-byte Folded Spill
+; RV32IZCMP-WITH-FP-NEXT:    .cfi_offset ra, -4
+; RV32IZCMP-WITH-FP-NEXT:    .cfi_offset s0, -8
+; RV32IZCMP-WITH-FP-NEXT:    .cfi_offset s1, -12
+; RV32IZCMP-WITH-FP-NEXT:    .cfi_offset s2, -16
+; RV32IZCMP-WITH-FP-NEXT:    .cfi_offset s3, -20
+; RV32IZCMP-WITH-FP-NEXT:    .cfi_offset s4, -24
+; RV32IZCMP-WITH-FP-NEXT:    .cfi_offset s5, -28
+; RV32IZCMP-WITH-FP-NEXT:    .cfi_offset s6, -32
+; RV32IZCMP-WITH-FP-NEXT:    .cfi_offset s7, -36
+; RV32IZCMP-WITH-FP-NEXT:    .cfi_offset s8, -40
+; RV32IZCMP-WITH-FP-NEXT:    .cfi_offset s9, -44
+; RV32IZCMP-WITH-FP-NEXT:    .cfi_offset s10, -48
+; RV32IZCMP-WITH-FP-NEXT:    .cfi_offset s11, -52
 ; RV32IZCMP-WITH-FP-NEXT:    addi s0, sp, 144
+; RV32IZCMP-WITH-FP-NEXT:    .cfi_def_cfa s0, 0
 ; RV32IZCMP-WITH-FP-NEXT:    lui s6, %hi(var)
 ; RV32IZCMP-WITH-FP-NEXT:    lw a0, %lo(var)(s6)
 ; RV32IZCMP-WITH-FP-NEXT:    sw a0, -56(s0) # 4-byte Folded Spill
@@ -1743,6 +2060,7 @@ define void @caller() nounwind {
 ; RV32IZCMP-WITH-FP-NEXT:    sw a0, %lo(var+4)(s6)
 ; RV32IZCMP-WITH-FP-NEXT:    lw a0, -56(s0) # 4-byte Folded Reload
 ; RV32IZCMP-WITH-FP-NEXT:    sw a0, %lo(var)(s6)
+; RV32IZCMP-WITH-FP-NEXT:    .cfi_def_cfa sp, 144
 ; RV32IZCMP-WITH-FP-NEXT:    lw ra, 140(sp) # 4-byte Folded Reload
 ; RV32IZCMP-WITH-FP-NEXT:    lw s0, 136(sp) # 4-byte Folded Reload
 ; RV32IZCMP-WITH-FP-NEXT:    lw s1, 132(sp) # 4-byte Folded Reload
@@ -1756,12 +2074,27 @@ define void @caller() nounwind {
 ; RV32IZCMP-WITH-FP-NEXT:    lw s9, 100(sp) # 4-byte Folded Reload
 ; RV32IZCMP-WITH-FP-NEXT:    lw s10, 96(sp) # 4-byte Folded Reload
 ; RV32IZCMP-WITH-FP-NEXT:    lw s11, 92(sp) # 4-byte Folded Reload
+; RV32IZCMP-WITH-FP-NEXT:    .cfi_restore ra
+; RV32IZCMP-WITH-FP-NEXT:    .cfi_restore s0
+; RV32IZCMP-WITH-FP-NEXT:    .cfi_restore s1
+; RV32IZCMP-WITH-FP-NEXT:    .cfi_restore s2
+; RV32IZCMP-WITH-FP-NEXT:    .cfi_restore s3
+; RV32IZCMP-WITH-FP-NEXT:    .cfi_restore s4
+; RV32IZCMP-WITH-FP-NEXT:    .cfi_restore s5
+; RV32IZCMP-WITH-FP-NEXT:    .cfi_restore s6
+; RV32IZCMP-WITH-FP-NEXT:    .cfi_restore s7
+; RV32IZCMP-WITH-FP-NEXT:    .cfi_restore s8
+; RV32IZCMP-WITH-FP-NEXT:    .cfi_restore s9
+; RV32IZCMP-WITH-FP-NEXT:    .cfi_restore s10
+; RV32IZCMP-WITH-FP-NEXT:    .cfi_restore s11
 ; RV32IZCMP-WITH-FP-NEXT:    addi sp, sp, 144
+; RV32IZCMP-WITH-FP-NEXT:    .cfi_def_cfa_offset 0
 ; RV32IZCMP-WITH-FP-NEXT:    ret
 ;
 ; RV64I-LABEL: caller:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -288
+; RV64I-NEXT:    .cfi_def_cfa_offset 288
 ; RV64I-NEXT:    sd ra, 280(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    sd s0, 272(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    sd s1, 264(sp) # 8-byte Folded Spill
@@ -1775,6 +2108,19 @@ define void @caller() nounwind {
 ; RV64I-NEXT:    sd s9, 200(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    sd s10, 192(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    sd s11, 184(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    .cfi_offset ra, -8
+; RV64I-NEXT:    .cfi_offset s0, -16
+; RV64I-NEXT:    .cfi_offset s1, -24
+; RV64I-NEXT:    .cfi_offset s2, -32
+; RV64I-NEXT:    .cfi_offset s3, -40
+; RV64I-NEXT:    .cfi_offset s4, -48
+; RV64I-NEXT:    .cfi_offset s5, -56
+; RV64I-NEXT:    .cfi_offset s6, -64
+; RV64I-NEXT:    .cfi_offset s7, -72
+; RV64I-NEXT:    .cfi_offset s8, -80
+; RV64I-NEXT:    .cfi_offset s9, -88
+; RV64I-NEXT:    .cfi_offset s10, -96
+; RV64I-NEXT:    .cfi_offset s11, -104
 ; RV64I-NEXT:    lui s0, %hi(var)
 ; RV64I-NEXT:    lw a0, %lo(var)(s0)
 ; RV64I-NEXT:    sd a0, 176(sp) # 8-byte Folded Spill
@@ -1899,15 +2245,33 @@ define void @caller() nounwind {
 ; RV64I-NEXT:    ld s9, 200(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    ld s10, 192(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    ld s11, 184(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    .cfi_restore ra
+; RV64I-NEXT:    .cfi_restore s0
+; RV64I-NEXT:    .cfi_restore s1
+; RV64I-NEXT:    .cfi_restore s2
+; RV64I-NEXT:    .cfi_restore s3
+; RV64I-NEXT:    .cfi_restore s4
+; RV64I-NEXT:    .cfi_restore s5
+; RV64I-NEXT:    .cfi_restore s6
+; RV64I-NEXT:    .cfi_restore s7
+; RV64I-NEXT:    .cfi_restore s8
+; RV64I-NEXT:    .cfi_restore s9
+; RV64I-NEXT:    .cfi_restore s10
+; RV64I-NEXT:    .cfi_restore s11
 ; RV64I-NEXT:    addi sp, sp, 288
+; RV64I-NEXT:    .cfi_def_cfa_offset 0
 ; RV64I-NEXT:    ret
 ;
 ; RV64I-LP64E-LABEL: caller:
 ; RV64I-LP64E:       # %bb.0:
 ; RV64I-LP64E-NEXT:    addi sp, sp, -272
+; RV64I-LP64E-NEXT:    .cfi_def_cfa_offset 272
 ; RV64I-LP64E-NEXT:    sd ra, 264(sp) # 8-byte Folded Spill
 ; RV64I-LP64E-NEXT:    sd s0, 256(sp) # 8-byte Folded Spill
 ; RV64I-LP64E-NEXT:    sd s1, 248(sp) # 8-byte Folded Spill
+; RV64I-LP64E-NEXT:    .cfi_offset ra, -8
+; RV64I-LP64E-NEXT:    .cfi_offset s0, -16
+; RV64I-LP64E-NEXT:    .cfi_offset s1, -24
 ; RV64I-LP64E-NEXT:    lui a0, %hi(var)
 ; RV64I-LP64E-NEXT:    lw a1, %lo(var)(a0)
 ; RV64I-LP64E-NEXT:    sd a1, 240(sp) # 8-byte Folded Spill
@@ -2041,12 +2405,17 @@ define void @caller() nounwind {
 ; RV64I-LP64E-NEXT:    ld ra, 264(sp) # 8-byte Folded Reload
 ; RV64I-LP64E-NEXT:    ld s0, 256(sp) # 8-byte Folded Reload
 ; RV64I-LP64E-NEXT:    ld s1, 248(sp) # 8-byte Folded Reload
+; RV64I-LP64E-NEXT:    .cfi_restore ra
+; RV64I-LP64E-NEXT:    .cfi_restore s0
+; RV64I-LP64E-NEXT:    .cfi_restore s1
 ; RV64I-LP64E-NEXT:    addi sp, sp, 272
+; RV64I-LP64E-NEXT:    .cfi_def_cfa_offset 0
 ; RV64I-LP64E-NEXT:    ret
 ;
 ; RV64I-WITH-FP-LABEL: caller:
 ; RV64I-WITH-FP:       # %bb.0:
 ; RV64I-WITH-FP-NEXT:    addi sp, sp, -288
+; RV64I-WITH-FP-NEXT:    .cfi_def_cfa_offset 288
 ; RV64I-WITH-FP-NEXT:    sd ra, 280(sp) # 8-byte Folded Spill
 ; RV64I-WITH-FP-NEXT:    sd s0, 272(sp) # 8-byte Folded Spill
 ; RV64I-WITH-FP-NEXT:    sd s1, 264(sp) # 8-byte Folded Spill
@@ -2060,7 +2429,21 @@ define void @caller() nounwind {
 ; RV64I-WITH-FP-NEXT:    sd s9, 200(sp) # 8-byte Folded Spill
 ; RV64I-WITH-FP-NEXT:    sd s10, 192(sp) # 8-byte Folded Spill
 ; RV64I-WITH-FP-NEXT:    sd s11, 184(sp) # 8-byte Folded Spill
+; RV64I-WITH-FP-NEXT:    .cfi_offset ra, -8
+; RV64I-WITH-FP-NEXT:    .cfi_offset s0, -16
+; RV64I-WITH-FP-NEXT:    .cfi_offset s1, -24
+; RV64I-WITH-FP-NEXT:    .cfi_offset s2, -32
+; RV64I-WITH-FP-NEXT:    .cfi_offset s3, -40
+; RV64I-WITH-FP-NEXT:    .cfi_offset s4, -48
+; RV64I-WITH-FP-NEXT:    .cfi_offset s5, -56
+; RV64I-WITH-FP-NEXT:    .cfi_offset s6, -64
+; RV64I-WITH-FP-NEXT:    .cfi_offset s7, -72
+; RV64I-WITH-FP-NEXT:    .cfi_offset s8, -80
+; RV64I-WITH-FP-NEXT:    .cfi_offset s9, -88
+; RV64I-WITH-FP-NEXT:    .cfi_offset s10, -96
+; RV64I-WITH-FP-NEXT:    .cfi_offset s11, -104
 ; RV64I-WITH-FP-NEXT:    addi s0, sp, 288
+; RV64I-WITH-FP-NEXT:    .cfi_def_cfa s0, 0
 ; RV64I-WITH-FP-NEXT:    lui s1, %hi(var)
 ; RV64I-WITH-FP-NEXT:    lw a0, %lo(var)(s1)
 ; RV64I-WITH-FP-NEXT:    sd a0, -112(s0) # 8-byte Folded Spill
@@ -2174,6 +2557,7 @@ define void @caller() nounwind {
 ; RV64I-WITH-FP-NEXT:    sw a0, %lo(var+4)(s1)
 ; RV64I-WITH-FP-NEXT:    ld a0, -112(s0) # 8-byte Folded Reload
 ; RV64I-WITH-FP-NEXT:    sw a0, %lo(var)(s1)
+; RV64I-WITH-FP-NEXT:    .cfi_def_cfa sp, 288
 ; RV64I-WITH-FP-NEXT:    ld ra, 280(sp) # 8-byte Folded Reload
 ; RV64I-WITH-FP-NEXT:    ld s0, 272(sp) # 8-byte Folded Reload
 ; RV64I-WITH-FP-NEXT:    ld s1, 264(sp) # 8-byte Folded Reload
@@ -2187,59 +2571,88 @@ define void @caller() nounwind {
 ; RV64I-WITH-FP-NEXT:    ld s9, 200(sp) # 8-byte Folded Reload
 ; RV64I-WITH-FP-NEXT:    ld s10, 192(sp) # 8-byte Folded Reload
 ; RV64I-WITH-FP-NEXT:    ld s11, 184(sp) # 8-byte Folded Reload
+; RV64I-WITH-FP-NEXT:    .cfi_restore ra
+; RV64I-WITH-FP-NEXT:    .cfi_restore s0
+; RV64I-WITH-FP-NEXT:    .cfi_restore s1
+; RV64I-WITH-FP-NEXT:    .cfi_restore s2
+; RV64I-WITH-FP-NEXT:    .cfi_restore s3
+; RV64I-WITH-FP-NEXT:    .cfi_restore s4
+; RV64I-WITH-FP-NEXT:    .cfi_restore s5
+; RV64I-WITH-FP-NEXT:    .cfi_restore s6
+; RV64I-WITH-FP-NEXT:    .cfi_restore s7
+; RV64I-WITH-FP-NEXT:    .cfi_restore s8
+; RV64I-WITH-FP-NEXT:    .cfi_restore s9
+; RV64I-WITH-FP-NEXT:    .cfi_restore s10
+; RV64I-WITH-FP-NEXT:    .cfi_restore s11
 ; RV64I-WITH-FP-NEXT:    addi sp, sp, 288
+; RV64I-WITH-FP-NEXT:    .cfi_def_cfa_offset 0
 ; RV64I-WITH-FP-NEXT:    ret
 ;
 ; RV64IZCMP-LABEL: caller:
 ; RV64IZCMP:       # %bb.0:
 ; RV64IZCMP-NEXT:    cm.push {ra, s0-s11}, -160
+; RV64IZCMP-NEXT:    .cfi_def_cfa_offset 160
+; RV64IZCMP-NEXT:    .cfi_offset ra, -104
+; RV64IZCMP-NEXT:    .cfi_offset s0, -96
+; RV64IZCMP-NEXT:    .cfi_offset s1, -88
+; RV64IZCMP-NEXT:    .cfi_offset s2, -80
+; RV64IZCMP-NEXT:    .cfi_offset s3, -72
+; RV64IZCMP-NEXT:    .cfi_offset s4, -64
+; RV64IZCMP-NEXT:    .cfi_offset s5, -56
+; RV64IZCMP-NEXT:    .cfi_offset s6, -48
+; RV64IZCMP-NEXT:    .cfi_offset s7, -40
+; RV64IZCMP-NEXT:    .cfi_offset s8, -32
+; RV64IZCMP-NEXT:    .cfi_offset s9, -24
+; RV64IZCMP-NEXT:    .cfi_offset s10, -16
+; RV64IZCMP-NEXT:    .cfi_offset s11, -8
 ; RV64IZCMP-NEXT:    addi sp, sp, -128
+; RV64IZCMP-NEXT:    .cfi_def_cfa_offset 288
 ; RV64IZCMP-NEXT:    lui s0, %hi(var)
 ; RV64IZCMP-NEXT:    lw a0, %lo(var)(s0)
-; RV64IZCMP-NEXT:    sd a0, 168(sp) # 8-byte Folded Spill
+; RV64IZCMP-NEXT:    sd a0, 176(sp) # 8-byte Folded Spill
 ; RV64IZCMP-NEXT:    lw a0, %lo(var+4)(s0)
-; RV64IZCMP-NEXT:    sd a0, 160(sp) # 8-byte Folded Spill
+; RV64IZCMP-NEXT:    sd a0, 168(sp) # 8-byte Folded Spill
 ; RV64IZCMP-NEXT:    lw a0, %lo(var+8)(s0)
-; RV64IZCMP-NEXT:    sd a0, 152(sp) # 8-byte Folded Spill
+; RV64IZCMP-NEXT:    sd a0, 160(sp) # 8-byte Folded Spill
 ; RV64IZCMP-NEXT:    lw a0, %lo(var+12)(s0)
-; RV64IZCMP-NEXT:    sd a0, 144(sp) # 8-byte Folded Spill
+; RV64IZCMP-NEXT:    sd a0, 152(sp) # 8-byte Folded Spill
 ; RV64IZCMP-NEXT:    addi s1, s0, %lo(var)
 ; RV64IZCMP-NEXT:    lw a0, 16(s1)
-; RV64IZCMP-NEXT:    sd a0, 136(sp) # 8-byte Folded Spill
+; RV64IZCMP-NEXT:    sd a0, 144(sp) # 8-byte Folded Spill
 ; RV64IZCMP-NEXT:    lw a0, 20(s1)
-; RV64IZCMP-NEXT:    sd a0, 128(sp) # 8-byte Folded Spill
+; RV64IZCMP-NEXT:    sd a0, 136(sp) # 8-byte Folded Spill
 ; RV64IZCMP-NEXT:    lw a0, 24(s1)
-; RV64IZCMP-NEXT:    sd a0, 120(sp) # 8-byte Folded Spill
+; RV64IZCMP-NEXT:    sd a0, 128(sp) # 8-byte Folded Spill
 ; RV64IZCMP-NEXT:    lw a0, 28(s1)
-; RV64IZCMP-NEXT:    sd a0, 112(sp) # 8-byte Folded Spill
+; RV64IZCMP-NEXT:    sd a0, 120(sp) # 8-byte Folded Spill
 ; RV64IZCMP-NEXT:    lw a0, 32(s1)
-; RV64IZCMP-NEXT:    sd a0, 104(sp) # 8-byte Folded Spill
+; RV64IZCMP-NEXT:    sd a0, 112(sp) # 8-byte Folded Spill
 ; RV64IZCMP-NEXT:    lw a0, 36(s1)
-; RV64IZCMP-NEXT:    sd a0, 96(sp) # 8-byte Folded Spill
+; RV64IZCMP-NEXT:    sd a0, 104(sp) # 8-byte Folded Spill
 ; RV64IZCMP-NEXT:    lw a0, 40(s1)
-; RV64IZCMP-NEXT:    sd a0, 88(sp) # 8-byte Folded Spill
+; RV64IZCMP-NEXT:    sd a0, 96(sp) # 8-byte Folded Spill
 ; RV64IZCMP-NEXT:    lw a0, 44(s1)
-; RV64IZCMP-NEXT:    sd a0, 80(sp) # 8-byte Folded Spill
+; RV64IZCMP-NEXT:    sd a0, 88(sp) # 8-byte Folded Spill
 ; RV64IZCMP-NEXT:    lw a0, 48(s1)
-; RV64IZCMP-NEXT:    sd a0, 72(sp) # 8-byte Folded Spill
+; RV64IZCMP-NEXT:    sd a0, 80(sp) # 8-byte Folded Spill
 ; RV64IZCMP-NEXT:    lw a0, 52(s1)
-; RV64IZCMP-NEXT:    sd a0, 64(sp) # 8-byte Folded Spill
+; RV64IZCMP-NEXT:    sd a0, 72(sp) # 8-byte Folded Spill
 ; RV64IZCMP-NEXT:    lw a0, 56(s1)
-; RV64IZCMP-NEXT:    sd a0, 56(sp) # 8-byte Folded Spill
+; RV64IZCMP-NEXT:    sd a0, 64(sp) # 8-byte Folded Spill
 ; RV64IZCMP-NEXT:    lw a0, 60(s1)
-; RV64IZCMP-NEXT:    sd a0, 48(sp) # 8-byte Folded Spill
+; RV64IZCMP-NEXT:    sd a0, 56(sp) # 8-byte Folded Spill
 ; RV64IZCMP-NEXT:    lw a0, 64(s1)
-; RV64IZCMP-NEXT:    sd a0, 40(sp) # 8-byte Folded Spill
+; RV64IZCMP-NEXT:    sd a0, 48(sp) # 8-byte Folded Spill
 ; RV64IZCMP-NEXT:    lw a0, 68(s1)
-; RV64IZCMP-NEXT:    sd a0, 32(sp) # 8-byte Folded Spill
+; RV64IZCMP-NEXT:    sd a0, 40(sp) # 8-byte Folded Spill
 ; RV64IZCMP-NEXT:    lw a0, 72(s1)
-; RV64IZCMP-NEXT:    sd a0, 24(sp) # 8-byte Folded Spill
+; RV64IZCMP-NEXT:    sd a0, 32(sp) # 8-byte Folded Spill
 ; RV64IZCMP-NEXT:    lw a0, 76(s1)
-; RV64IZCMP-NEXT:    sd a0, 16(sp) # 8-byte Folded Spill
+; RV64IZCMP-NEXT:    sd a0, 24(sp) # 8-byte Folded Spill
 ; RV64IZCMP-NEXT:    lw a0, 80(s1)
-; RV64IZCMP-NEXT:    sd a0, 8(sp) # 8-byte Folded Spill
+; RV64IZCMP-NEXT:    sd a0, 16(sp) # 8-byte Folded Spill
 ; RV64IZCMP-NEXT:    lw a0, 84(s1)
-; RV64IZCMP-NEXT:    sd a0, 0(sp) # 8-byte Folded Spill
+; RV64IZCMP-NEXT:    sd a0, 8(sp) # 8-byte Folded Spill
 ; RV64IZCMP-NEXT:    lw s4, 88(s1)
 ; RV64IZCMP-NEXT:    lw s5, 92(s1)
 ; RV64IZCMP-NEXT:    lw s6, 96(s1)
@@ -2261,56 +2674,58 @@ define void @caller() nounwind {
 ; RV64IZCMP-NEXT:    sw s6, 96(s1)
 ; RV64IZCMP-NEXT:    sw s5, 92(s1)
 ; RV64IZCMP-NEXT:    sw s4, 88(s1)
-; RV64IZCMP-NEXT:    ld a0, 0(sp) # 8-byte Folded Reload
-; RV64IZCMP-NEXT:    sw a0, 84(s1)
 ; RV64IZCMP-NEXT:    ld a0, 8(sp) # 8-byte Folded Reload
-; RV64IZCMP-NEXT:    sw a0, 80(s1)
+; RV64IZCMP-NEXT:    sw a0, 84(s1)
 ; RV64IZCMP-NEXT:    ld a0, 16(sp) # 8-byte Folded Reload
-; RV64IZCMP-NEXT:    sw a0, 76(s1)
+; RV64IZCMP-NEXT:    sw a0, 80(s1)
 ; RV64IZCMP-NEXT:    ld a0, 24(sp) # 8-byte Folded Reload
-; RV64IZCMP-NEXT:    sw a0, 72(s1)
+; RV64IZCMP-NEXT:    sw a0, 76(s1)
 ; RV64IZCMP-NEXT:    ld a0, 32(sp) # 8-byte Folded Reload
-; RV64IZCMP-NEXT:    sw a0, 68(s1)
+; RV64IZCMP-NEXT:    sw a0, 72(s1)
 ; RV64IZCMP-NEXT:    ld a0, 40(sp) # 8-byte Folded Reload
-; RV64IZCMP-NEXT:    sw a0, 64(s1)
+; RV64IZCMP-NEXT:    sw a0, 68(s1)
 ; RV64IZCMP-NEXT:    ld a0, 48(sp) # 8-byte Folded Reload
-; RV64IZCMP-NEXT:    sw a0, 60(s1)
+; RV64IZCMP-NEXT:    sw a0, 64(s1)
 ; RV64IZCMP-NEXT:    ld a0, 56(sp) # 8-byte Folded Reload
-; RV64IZCMP-NEXT:    sw a0, 56(s1)
+; RV64IZCMP-NEXT:    sw a0, 60(s1)
 ; RV64IZCMP-NEXT:    ld a0, 64(sp) # 8-byte Folded Reload
-; RV64IZCMP-NEXT:    sw a0, 52(s1)
+; RV64IZCMP-NEXT:    sw a0, 56(s1)
 ; RV64IZCMP-NEXT:    ld a0, 72(sp) # 8-byte Folded Reload
-; RV64IZCMP-NEXT:    sw a0, 48(s1)
+; RV64IZCMP-NEXT:    sw a0, 52(s1)
 ; RV64IZCMP-NEXT:    ld a0, 80(sp) # 8-byte Folded Reload
-; RV64IZCMP-NEXT:    sw a0, 44(s1)
+; RV64IZCMP-NEXT:    sw a0, 48(s1)
 ; RV64IZCMP-NEXT:    ld a0, 88(sp) # 8-byte Folded Reload
-; RV64IZCMP-NEXT:    sw a0, 40(s1)
+; RV64IZCMP-NEXT:    sw a0, 44(s1)
 ; RV64IZCMP-NEXT:    ld a0, 96(sp) # 8-byte Folded Reload
-; RV64IZCMP-NEXT:    sw a0, 36(s1)
+; RV64IZCMP-NEXT:    sw a0, 40(s1)
 ; RV64IZCMP-NEXT:    ld a0, 104(sp) # 8-byte Folded Reload
-; RV64IZCMP-NEXT:    sw a0, 32(s1)
+; RV64IZCMP-NEXT:    sw a0, 36(s1)
 ; RV64IZCMP-NEXT:    ld a0, 112(sp) # 8-byte Folded Reload
-; RV64IZCMP-NEXT:    sw a0, 28(s1)
+; RV64IZCMP-NEXT:    sw a0, 32(s1)
 ; RV64IZCMP-NEXT:    ld a0, 120(sp) # 8-byte Folded Reload
-; RV64IZCMP-NEXT:    sw a0, 24(s1)
+; RV64IZCMP-NEXT:    sw a0, 28(s1)
 ; RV64IZCMP-NEXT:    ld a0, 128(sp) # 8-byte Folded Reload
-; RV64IZCMP-NEXT:    sw a0, 20(s1)
+; RV64IZCMP-NEXT:    sw a0, 24(s1)
 ; RV64IZCMP-NEXT:    ld a0, 136(sp) # 8-byte Folded Reload
-; RV64IZCMP-NEXT:    sw a0, 16(s1)
+; RV64IZCMP-NEXT:    sw a0, 20(s1)
 ; RV64IZCMP-NEXT:    ld a0, 144(sp) # 8-byte Folded Reload
-; RV64IZCMP-NEXT:    sw a0, %lo(var+12)(s0)
+; RV64IZCMP-NEXT:    sw a0, 16(s1)
 ; RV64IZCMP-NEXT:    ld a0, 152(sp) # 8-byte Folded Reload
-; RV64IZCMP-NEXT:    sw a0, %lo(var+8)(s0)
+; RV64IZCMP-NEXT:    sw a0, %lo(var+12)(s0)
 ; RV64IZCMP-NEXT:    ld a0, 160(sp) # 8-byte Folded Reload
-; RV64IZCMP-NEXT:    sw a0, %lo(var+4)(s0)
+; RV64IZCMP-NEXT:    sw a0, %lo(var+8)(s0)
 ; RV64IZCMP-NEXT:    ld a0, 168(sp) # 8-byte Folded Reload
+; RV64IZCMP-NEXT:    sw a0, %lo(var+4)(s0)
+; RV64IZCMP-NEXT:    ld a0, 176(sp) # 8-byte Folded Reload
 ; RV64IZCMP-NEXT:    sw a0, %lo(var)(s0)
 ; RV64IZCMP-NEXT:    addi sp, sp, 128
+; RV64IZCMP-NEXT:    .cfi_def_cfa_offset 160
 ; RV64IZCMP-NEXT:    cm.popret {ra, s0-s11}, 160
 ;
 ; RV64IZCMP-WITH-FP-LABEL: caller:
 ; RV64IZCMP-WITH-FP:       # %bb.0:
 ; RV64IZCMP-WITH-FP-NEXT:    addi sp, sp, -288
+; RV64IZCMP-WITH-FP-NEXT:    .cfi_def_cfa_offset 288
 ; RV64IZCMP-WITH-FP-NEXT:    sd ra, 280(sp) # 8-byte Folded Spill
 ; RV64IZCMP-WITH-FP-NEXT:    sd s0, 272(sp) # 8-byte Folded Spill
 ; RV64IZCMP-WITH-FP-NEXT:    sd s1, 264(sp) # 8-byte Folded Spill
@@ -2324,7 +2739,21 @@ define void @caller() nounwind {
 ; RV64IZCMP-WITH-FP-NEXT:    sd s9, 200(sp) # 8-byte Folded Spill
 ; RV64IZCMP-WITH-FP-NEXT:    sd s10, 192(sp) # 8-byte Folded Spill
 ; RV64IZCMP-WITH-FP-NEXT:    sd s11, 184(sp) # 8-byte Folded Spill
+; RV64IZCMP-WITH-FP-NEXT:    .cfi_offset ra, -8
+; RV64IZCMP-WITH-FP-NEXT:    .cfi_offset s0, -16
+; RV64IZCMP-WITH-FP-NEXT:    .cfi_offset s1, -24
+; RV64IZCMP-WITH-FP-NEXT:    .cfi_offset s2, -32
+; RV64IZCMP-WITH-FP-NEXT:    .cfi_offset s3, -40
+; RV64IZCMP-WITH-FP-NEXT:    .cfi_offset s4, -48
+; RV64IZCMP-WITH-FP-NEXT:    .cfi_offset s5, -56
+; RV64IZCMP-WITH-FP-NEXT:    .cfi_offset s6, -64
+; RV64IZCMP-WITH-FP-NEXT:    .cfi_offset s7, -72
+; RV64IZCMP-WITH-FP-NEXT:    .cfi_offset s8, -80
+; RV64IZCMP-WITH-FP-NEXT:    .cfi_offset s9, -88
+; RV64IZCMP-WITH-FP-NEXT:    .cfi_offset s10, -96
+; RV64IZCMP-WITH-FP-NEXT:    .cfi_offset s11, -104
 ; RV64IZCMP-WITH-FP-NEXT:    addi s0, sp, 288
+; RV64IZCMP-WITH-FP-NEXT:    .cfi_def_cfa s0, 0
 ; RV64IZCMP-WITH-FP-NEXT:    lui s6, %hi(var)
 ; RV64IZCMP-WITH-FP-NEXT:    lw a0, %lo(var)(s6)
 ; RV64IZCMP-WITH-FP-NEXT:    sd a0, -112(s0) # 8-byte Folded Spill
@@ -2438,6 +2867,7 @@ define void @caller() nounwind {
 ; RV64IZCMP-WITH-FP-NEXT:    sw a0, %lo(var+4)(s6)
 ; RV64IZCMP-WITH-FP-NEXT:    ld a0, -112(s0) # 8-byte Folded Reload
 ; RV64IZCMP-WITH-FP-NEXT:    sw a0, %lo(var)(s6)
+; RV64IZCMP-WITH-FP-NEXT:    .cfi_def_cfa sp, 288
 ; RV64IZCMP-WITH-FP-NEXT:    ld ra, 280(sp) # 8-byte Folded Reload
 ; RV64IZCMP-WITH-FP-NEXT:    ld s0, 272(sp) # 8-byte Folded Reload
 ; RV64IZCMP-WITH-FP-NEXT:    ld s1, 264(sp) # 8-byte Folded Reload
@@ -2451,7 +2881,21 @@ define void @caller() nounwind {
 ; RV64IZCMP-WITH-FP-NEXT:    ld s9, 200(sp) # 8-byte Folded Reload
 ; RV64IZCMP-WITH-FP-NEXT:    ld s10, 192(sp) # 8-byte Folded Reload
 ; RV64IZCMP-WITH-FP-NEXT:    ld s11, 184(sp) # 8-byte Folded Reload
+; RV64IZCMP-WITH-FP-NEXT:    .cfi_restore ra
+; RV64IZCMP-WITH-FP-NEXT:    .cfi_restore s0
+; RV64IZCMP-WITH-FP-NEXT:    .cfi_restore s1
+; RV64IZCMP-WITH-FP-NEXT:    .cfi_restore s2
+; RV64IZCMP-WITH-FP-NEXT:    .cfi_restore s3
+; RV64IZCMP-WITH-FP-NEXT:    .cfi_restore s4
+; RV64IZCMP-WITH-FP-NEXT:    .cfi_restore s5
+; RV64IZCMP-WITH-FP-NEXT:    .cfi_restore s6
+; RV64IZCMP-WITH-FP-NEXT:    .cfi_restore s7
+; RV64IZCMP-WITH-FP-NEXT:    .cfi_restore s8
+; RV64IZCMP-WITH-FP-NEXT:    .cfi_restore s9
+; RV64IZCMP-WITH-FP-NEXT:    .cfi_restore s10
+; RV64IZCMP-WITH-FP-NEXT:    .cfi_restore s11
 ; RV64IZCMP-WITH-FP-NEXT:    addi sp, sp, 288
+; RV64IZCMP-WITH-FP-NEXT:    .cfi_def_cfa_offset 0
 ; RV64IZCMP-WITH-FP-NEXT:    ret
   %val = load [32 x i32], ptr @var
   call void @callee()
diff --git a/llvm/test/CodeGen/RISCV/debug-line.ll b/llvm/test/CodeGen/RISCV/debug-line.ll
new file mode 100644
index 0000000000000..3cb4d6dc74086
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/debug-line.ll
@@ -0,0 +1,47 @@
+; RUN: llc -mtriple=riscv64 < %s | FileCheck %s
+
+define void @foo() #0 !dbg !3 {
+; CHECK-LABEL: foo:
+; CHECK: .Lfunc_begin0:
+; CHECK-NEXT: 	.file	1 "test.c"
+; CHECK-NEXT: 	.loc	1 5 0                           # test.c:5:0
+; CHECK-NEXT: 	.cfi_startproc
+; CHECK-NEXT: # %bb.0:                                # %entry
+; CHECK-NEXT: 	addi	sp, sp, -16
+; CHECK-NEXT: 	.cfi_def_cfa_offset 16
+; CHECK-NEXT: 	sd	ra, 8(sp)                       # 8-byte Folded Spill
+; CHECK-NEXT: 	sd	s0, 0(sp)                       # 8-byte Folded Spill
+; CHECK-NEXT: 	.cfi_offset ra, -8
+; CHECK-NEXT: 	.cfi_offset s0, -16
+; CHECK-NEXT: 	addi	s0, sp, 16
+; CHECK-NEXT: 	.cfi_def_cfa s0, 0
+; CHECK-NEXT: .Ltmp0:
+; CHECK-NEXT: 	.loc	1 6 4 prologue_end              # test.c:6:4
+; CHECK-NEXT: 	sw	zero, 0(zero)
+; CHECK-NEXT:   .cfi_def_cfa sp, 16
+; CHECK-NEXT: 	.loc	1 7 1 epilogue_begin            # test.c:7:1
+; CHECK-NEXT: 	ld	ra, 8(sp)                       # 8-byte Folded Reload
+; CHECK-NEXT: 	ld	s0, 0(sp)                       # 8-byte Folded Reload
+; CHECK-NEXT:   .cfi_restore ra
+; CHECK-NEXT:   .cfi_restore s0
+; CHECK-NEXT: 	addi	sp, sp, 16
+; CHECK-NEXT:   .cfi_def_cfa_offset 0 
+; CHECK-NEXT: 	ret
+entry:
+  store i32 0, ptr null, align 4, !dbg !6
+  ret void, !dbg !7
+}
+
+attributes #0 = { "frame-pointer"="all" }
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!2}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C11, file: !1, emissionKind: FullDebug)
+!1 = !DIFile(filename: "test.c", directory: "")
+!2 = !{i32 2, !"Debug Info Version", i32 3}
+!3 = distinct !DISubprogram(name: "foo", scope: !1, file: !1, line: 5, type: !4, scopeLine: 5, unit: !0)
+!4 = !DISubroutineType(types: !5)
+!5 = !{null}
+!6 = !DILocation(line: 6, column: 4, scope: !3)
+!7 = !DILocation(line: 7, column: 1, scope: !3)
diff --git a/llvm/test/CodeGen/RISCV/ipra.ll b/llvm/test/CodeGen/RISCV/ipra.ll
new file mode 100644
index 0000000000000..0bcded99c5975
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/ipra.ll
@@ -0,0 +1,127 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc -mtriple=riscv64 -verify-machineinstrs -enable-ipra < %s | FileCheck %s -check-prefix=RV64
+; RUN: llc -mtriple=riscv32 -verify-machineinstrs -enable-ipra < %s | FileCheck %s -check-prefix=RV32
+; RUN: llc -mtriple=riscv64 -verify-machineinstrs -frame-pointer=all -enable-ipra < %s \
+; RUN:   | FileCheck %s -check-prefix=RV64-WITHFP
+; RUN: llc -mtriple=riscv32 -verify-machineinstrs -frame-pointer=all -enable-ipra < %s \
+; RUN:   | FileCheck %s -check-prefix=RV32-WITHFP
+
+%struct.bitmap_iterator = type { ptr, ptr, i32, i64 }
+
+define i32 @main() nounwind {
+; RV64-LABEL: main:
+; RV64:       # %bb.0: # %entry
+; RV64-NEXT:    addi sp, sp, -16
+; RV64-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64-NEXT:    li a0, 0
+; RV64-NEXT:    call foobar
+; RV64-NEXT:    li a0, 0
+; RV64-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64-NEXT:    addi sp, sp, 16
+; RV64-NEXT:    ret
+;
+; RV32-LABEL: main:
+; RV32:       # %bb.0: # %entry
+; RV32-NEXT:    addi sp, sp, -16
+; RV32-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32-NEXT:    li a0, 0
+; RV32-NEXT:    call foobar
+; RV32-NEXT:    li a0, 0
+; RV32-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32-NEXT:    addi sp, sp, 16
+; RV32-NEXT:    ret
+;
+; RV64-WITHFP-LABEL: main:
+; RV64-WITHFP:       # %bb.0: # %entry
+; RV64-WITHFP-NEXT:    addi sp, sp, -16
+; RV64-WITHFP-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64-WITHFP-NEXT:    sd s0, 0(sp) # 8-byte Folded Spill
+; RV64-WITHFP-NEXT:    addi s0, sp, 16
+; RV64-WITHFP-NEXT:    li a0, 0
+; RV64-WITHFP-NEXT:    call foobar
+; RV64-WITHFP-NEXT:    li a0, 0
+; RV64-WITHFP-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64-WITHFP-NEXT:    ld s0, 0(sp) # 8-byte Folded Reload
+; RV64-WITHFP-NEXT:    addi sp, sp, 16
+; RV64-WITHFP-NEXT:    ret
+;
+; RV32-WITHFP-LABEL: main:
+; RV32-WITHFP:       # %bb.0: # %entry
+; RV32-WITHFP-NEXT:    addi sp, sp, -16
+; RV32-WITHFP-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32-WITHFP-NEXT:    sw s0, 8(sp) # 4-byte Folded Spill
+; RV32-WITHFP-NEXT:    addi s0, sp, 16
+; RV32-WITHFP-NEXT:    li a0, 0
+; RV32-WITHFP-NEXT:    call foobar
+; RV32-WITHFP-NEXT:    li a0, 0
+; RV32-WITHFP-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32-WITHFP-NEXT:    lw s0, 8(sp) # 4-byte Folded Reload
+; RV32-WITHFP-NEXT:    addi sp, sp, 16
+; RV32-WITHFP-NEXT:    ret
+entry:
+  call void @foobar(ptr null)
+  ret i32 0
+}
+
+define internal void @foobar(ptr %live_throughout.0.val) norecurse nounwind {
+; RV64-LABEL: foobar:
+; RV64:       # %bb.0: # %entry
+; RV64-NEXT:    addi sp, sp, -48
+; RV64-NEXT:    sd ra, 40(sp) # 8-byte Folded Spill
+; RV64-NEXT:    mv a1, a0
+; RV64-NEXT:    addi a0, sp, 8
+; RV64-NEXT:    addi a2, sp, 4
+; RV64-NEXT:    call bmp_iter_set_init
+; RV64-NEXT:    ld ra, 40(sp) # 8-byte Folded Reload
+; RV64-NEXT:    addi sp, sp, 48
+; RV64-NEXT:    ret
+;
+; RV32-LABEL: foobar:
+; RV32:       # %bb.0: # %entry
+; RV32-NEXT:    addi sp, sp, -48
+; RV32-NEXT:    sw ra, 44(sp) # 4-byte Folded Spill
+; RV32-NEXT:    mv a1, a0
+; RV32-NEXT:    addi a0, sp, 16
+; RV32-NEXT:    addi a2, sp, 12
+; RV32-NEXT:    call bmp_iter_set_init
+; RV32-NEXT:    lw ra, 44(sp) # 4-byte Folded Reload
+; RV32-NEXT:    addi sp, sp, 48
+; RV32-NEXT:    ret
+;
+; RV64-WITHFP-LABEL: foobar:
+; RV64-WITHFP:       # %bb.0: # %entry
+; RV64-WITHFP-NEXT:    addi sp, sp, -64
+; RV64-WITHFP-NEXT:    sd ra, 56(sp) # 8-byte Folded Spill
+; RV64-WITHFP-NEXT:    sd s0, 48(sp) # 8-byte Folded Spill
+; RV64-WITHFP-NEXT:    addi s0, sp, 64
+; RV64-WITHFP-NEXT:    mv a1, a0
+; RV64-WITHFP-NEXT:    addi a0, s0, -48
+; RV64-WITHFP-NEXT:    addi a2, s0, -52
+; RV64-WITHFP-NEXT:    call bmp_iter_set_init
+; RV64-WITHFP-NEXT:    ld ra, 56(sp) # 8-byte Folded Reload
+; RV64-WITHFP-NEXT:    ld s0, 48(sp) # 8-byte Folded Reload
+; RV64-WITHFP-NEXT:    addi sp, sp, 64
+; RV64-WITHFP-NEXT:    ret
+;
+; RV32-WITHFP-LABEL: foobar:
+; RV32-WITHFP:       # %bb.0: # %entry
+; RV32-WITHFP-NEXT:    addi sp, sp, -48
+; RV32-WITHFP-NEXT:    sw ra, 44(sp) # 4-byte Folded Spill
+; RV32-WITHFP-NEXT:    sw s0, 40(sp) # 4-byte Folded Spill
+; RV32-WITHFP-NEXT:    addi s0, sp, 48
+; RV32-WITHFP-NEXT:    mv a1, a0
+; RV32-WITHFP-NEXT:    addi a0, s0, -32
+; RV32-WITHFP-NEXT:    addi a2, s0, -36
+; RV32-WITHFP-NEXT:    call bmp_iter_set_init
+; RV32-WITHFP-NEXT:    lw ra, 44(sp) # 4-byte Folded Reload
+; RV32-WITHFP-NEXT:    lw s0, 40(sp) # 4-byte Folded Reload
+; RV32-WITHFP-NEXT:    addi sp, sp, 48
+; RV32-WITHFP-NEXT:    ret
+entry:
+  %rsi = alloca %struct.bitmap_iterator, align 8
+  %regno = alloca i32, i32 0, align 4
+  call void @bmp_iter_set_init(ptr %rsi, ptr %live_throughout.0.val, ptr %regno)
+  ret void
+}
+
+declare void @bmp_iter_set_init(ptr, ptr, ptr)
diff --git a/llvm/test/CodeGen/RISCV/jumptable-swguarded.ll b/llvm/test/CodeGen/RISCV/jumptable-swguarded.ll
index 0e87d8d6f82fe..b4ab0585c0cc9 100644
--- a/llvm/test/CodeGen/RISCV/jumptable-swguarded.ll
+++ b/llvm/test/CodeGen/RISCV/jumptable-swguarded.ll
@@ -16,7 +16,7 @@ define void @above_threshold(i32 signext %in, ptr %out) nounwind {
 ; CHECK-NEXT:    slli a0, a0, 2
 ; CHECK-NEXT:    lui a2, %hi(.LJTI0_0)
 ; CHECK-NEXT:    addi a2, a2, %lo(.LJTI0_0)
-; CHECK-NEXT:    add a0, a0, a2
+; CHECK-NEXT:    add a0, a2, a0
 ; CHECK-NEXT:    lw t2, 0(a0)
 ; CHECK-NEXT:    jr t2
 ; CHECK-NEXT:  .LBB0_2: # %bb1
@@ -50,7 +50,7 @@ define void @above_threshold(i32 signext %in, ptr %out) nounwind {
 ; NO-ZICFILP-NEXT:    slli a0, a0, 2
 ; NO-ZICFILP-NEXT:    lui a2, %hi(.LJTI0_0)
 ; NO-ZICFILP-NEXT:    addi a2, a2, %lo(.LJTI0_0)
-; NO-ZICFILP-NEXT:    add a0, a0, a2
+; NO-ZICFILP-NEXT:    add a0, a2, a0
 ; NO-ZICFILP-NEXT:    lw a0, 0(a0)
 ; NO-ZICFILP-NEXT:    jr a0
 ; NO-ZICFILP-NEXT:  .LBB0_2: # %bb1
diff --git a/llvm/test/CodeGen/RISCV/jumptable.ll b/llvm/test/CodeGen/RISCV/jumptable.ll
index 2b5500920a32a..8584579b81384 100644
--- a/llvm/test/CodeGen/RISCV/jumptable.ll
+++ b/llvm/test/CodeGen/RISCV/jumptable.ll
@@ -89,7 +89,7 @@ define void @above_threshold(i32 signext %in, ptr %out) nounwind {
 ; RV32I-SMALL-NEXT:    slli a0, a0, 2
 ; RV32I-SMALL-NEXT:    lui a2, %hi(.LJTI1_0)
 ; RV32I-SMALL-NEXT:    addi a2, a2, %lo(.LJTI1_0)
-; RV32I-SMALL-NEXT:    add a0, a0, a2
+; RV32I-SMALL-NEXT:    add a0, a2, a0
 ; RV32I-SMALL-NEXT:    lw a0, 0(a0)
 ; RV32I-SMALL-NEXT:    jr a0
 ; RV32I-SMALL-NEXT:  .LBB1_2: # %bb1
@@ -124,7 +124,7 @@ define void @above_threshold(i32 signext %in, ptr %out) nounwind {
 ; RV32I-MEDIUM-NEXT:  .Lpcrel_hi0:
 ; RV32I-MEDIUM-NEXT:    auipc a2, %pcrel_hi(.LJTI1_0)
 ; RV32I-MEDIUM-NEXT:    addi a2, a2, %pcrel_lo(.Lpcrel_hi0)
-; RV32I-MEDIUM-NEXT:    add a0, a0, a2
+; RV32I-MEDIUM-NEXT:    add a0, a2, a0
 ; RV32I-MEDIUM-NEXT:    lw a0, 0(a0)
 ; RV32I-MEDIUM-NEXT:    jr a0
 ; RV32I-MEDIUM-NEXT:  .LBB1_2: # %bb1
@@ -159,9 +159,9 @@ define void @above_threshold(i32 signext %in, ptr %out) nounwind {
 ; RV32I-PIC-NEXT:  .Lpcrel_hi0:
 ; RV32I-PIC-NEXT:    auipc a2, %pcrel_hi(.LJTI1_0)
 ; RV32I-PIC-NEXT:    addi a2, a2, %pcrel_lo(.Lpcrel_hi0)
-; RV32I-PIC-NEXT:    add a0, a0, a2
+; RV32I-PIC-NEXT:    add a0, a2, a0
 ; RV32I-PIC-NEXT:    lw a0, 0(a0)
-; RV32I-PIC-NEXT:    add a0, a0, a2
+; RV32I-PIC-NEXT:    add a0, a2, a0
 ; RV32I-PIC-NEXT:    jr a0
 ; RV32I-PIC-NEXT:  .LBB1_2: # %bb1
 ; RV32I-PIC-NEXT:    li a0, 4
@@ -194,7 +194,7 @@ define void @above_threshold(i32 signext %in, ptr %out) nounwind {
 ; RV64I-SMALL-NEXT:    slli a0, a0, 2
 ; RV64I-SMALL-NEXT:    lui a2, %hi(.LJTI1_0)
 ; RV64I-SMALL-NEXT:    addi a2, a2, %lo(.LJTI1_0)
-; RV64I-SMALL-NEXT:    add a0, a0, a2
+; RV64I-SMALL-NEXT:    add a0, a2, a0
 ; RV64I-SMALL-NEXT:    lw a0, 0(a0)
 ; RV64I-SMALL-NEXT:    jr a0
 ; RV64I-SMALL-NEXT:  .LBB1_2: # %bb1
@@ -229,7 +229,7 @@ define void @above_threshold(i32 signext %in, ptr %out) nounwind {
 ; RV64I-MEDIUM-NEXT:  .Lpcrel_hi0:
 ; RV64I-MEDIUM-NEXT:    auipc a2, %pcrel_hi(.LJTI1_0)
 ; RV64I-MEDIUM-NEXT:    addi a2, a2, %pcrel_lo(.Lpcrel_hi0)
-; RV64I-MEDIUM-NEXT:    add a0, a0, a2
+; RV64I-MEDIUM-NEXT:    add a0, a2, a0
 ; RV64I-MEDIUM-NEXT:    ld a0, 0(a0)
 ; RV64I-MEDIUM-NEXT:    jr a0
 ; RV64I-MEDIUM-NEXT:  .LBB1_2: # %bb1
@@ -264,9 +264,9 @@ define void @above_threshold(i32 signext %in, ptr %out) nounwind {
 ; RV64I-PIC-NEXT:  .Lpcrel_hi0:
 ; RV64I-PIC-NEXT:    auipc a2, %pcrel_hi(.LJTI1_0)
 ; RV64I-PIC-NEXT:    addi a2, a2, %pcrel_lo(.Lpcrel_hi0)
-; RV64I-PIC-NEXT:    add a0, a0, a2
+; RV64I-PIC-NEXT:    add a0, a2, a0
 ; RV64I-PIC-NEXT:    lw a0, 0(a0)
-; RV64I-PIC-NEXT:    add a0, a0, a2
+; RV64I-PIC-NEXT:    add a0, a2, a0
 ; RV64I-PIC-NEXT:    jr a0
 ; RV64I-PIC-NEXT:  .LBB1_2: # %bb1
 ; RV64I-PIC-NEXT:    li a0, 4
diff --git a/llvm/test/CodeGen/RISCV/kcfi-mir.ll b/llvm/test/CodeGen/RISCV/kcfi-mir.ll
index 0c0d39a8bf87d..2efdc129f2621 100644
--- a/llvm/test/CodeGen/RISCV/kcfi-mir.ll
+++ b/llvm/test/CodeGen/RISCV/kcfi-mir.ll
@@ -8,13 +8,13 @@ define void @f1(ptr noundef %x) !kcfi_type !1 {
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT:   $x2 = frame-setup ADDI $x2, -16
   ; CHECK-NEXT:   frame-setup CFI_INSTRUCTION def_cfa_offset 16
-  ; CHECK-NEXT:   SD killed $x1, $x2, 8 :: (store (s64) into %stack.0)
+  ; CHECK-NEXT:   frame-setup SD killed $x1, $x2, 8 :: (store (s64) into %stack.0)
   ; CHECK-NEXT:   frame-setup CFI_INSTRUCTION offset $x1, -8
   ; CHECK-NEXT:   BUNDLE implicit-def $x6, implicit-def $x6_w, implicit-def $x6_h, implicit-def $x7, implicit-def $x7_w, implicit-def $x7_h, implicit-def $x28, implicit-def $x28_w, implicit-def $x28_h, implicit-def $x29, implicit-def $x29_w, implicit-def $x29_h, implicit-def $x30, implicit-def $x30_w, implicit-def $x30_h, implicit-def $x31, implicit-def $x31_w, implicit-def $x31_h, implicit-def dead $x1, implicit-def $x2, implicit-def $x2_w, implicit-def $x2_h, implicit killed $x10 {
   ; CHECK-NEXT:     KCFI_CHECK $x10, 12345678, implicit-def $x6, implicit-def $x7, implicit-def $x28, implicit-def $x29, implicit-def $x30, implicit-def $x31
   ; CHECK-NEXT:     PseudoCALLIndirect killed $x10, csr_ilp32_lp64, implicit-def dead $x1, implicit-def $x2
   ; CHECK-NEXT:   }
-  ; CHECK-NEXT:   $x1 = LD $x2, 8 :: (load (s64) from %stack.0)
+  ; CHECK-NEXT:   $x1 = frame-destroy LD $x2, 8 :: (load (s64) from %stack.0)
   ; CHECK-NEXT:   frame-destroy CFI_INSTRUCTION restore $x1
   ; CHECK-NEXT:   $x2 = frame-destroy ADDI $x2, 16
   ; CHECK-NEXT:   frame-destroy CFI_INSTRUCTION def_cfa_offset 0
diff --git a/llvm/test/CodeGen/RISCV/live-sp.mir b/llvm/test/CodeGen/RISCV/live-sp.mir
index 9f40870feb00f..1c4c6e43a9234 100644
--- a/llvm/test/CodeGen/RISCV/live-sp.mir
+++ b/llvm/test/CodeGen/RISCV/live-sp.mir
@@ -74,13 +74,13 @@ body:             |
     ; CHECK-NEXT: {{  $}}
     ; CHECK-NEXT: $x2 = frame-setup ADDI $x2, -16
     ; CHECK-NEXT: frame-setup CFI_INSTRUCTION def_cfa_offset 16
-    ; CHECK-NEXT: SD $x1, $x2, 8 :: (store (s64) into %stack.1)
+    ; CHECK-NEXT: frame-setup SD $x1, $x2, 8 :: (store (s64) into %stack.1)
     ; CHECK-NEXT: frame-setup CFI_INSTRUCTION offset $x1, -8
     ; CHECK-NEXT: SW renamable $x1, $x2, 4 :: (store (s32) into %ir.a)
     ; CHECK-NEXT: renamable $x11 = ADDIW killed renamable $x1, 0
     ; CHECK-NEXT: $x10 = COPY $x0
     ; CHECK-NEXT: PseudoCALL target-flags(riscv-call) @vararg, csr_ilp32_lp64, implicit-def dead $x1, implicit killed $x10, implicit $x11, implicit-def $x2
-    ; CHECK-NEXT: $x1 = LD $x2, 8 :: (load (s64) from %stack.1)
+    ; CHECK-NEXT: $x1 = frame-destroy LD $x2, 8 :: (load (s64) from %stack.1)
     ; CHECK-NEXT: frame-destroy CFI_INSTRUCTION restore $x1
     ; CHECK-NEXT: $x2 = frame-destroy ADDI $x2, 16
     ; CHECK-NEXT: frame-destroy CFI_INSTRUCTION def_cfa_offset 0
diff --git a/llvm/test/CodeGen/RISCV/machinelicm-address-pseudos.ll b/llvm/test/CodeGen/RISCV/machinelicm-address-pseudos.ll
index af8105644b57d..8deb17582cb11 100644
--- a/llvm/test/CodeGen/RISCV/machinelicm-address-pseudos.ll
+++ b/llvm/test/CodeGen/RISCV/machinelicm-address-pseudos.ll
@@ -1,8 +1,12 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mtriple=riscv32 -relocation-model=pic -verify-machineinstrs < %s \
-; RUN:   | FileCheck -check-prefixes=RV32I %s
+; RUN:   | FileCheck -check-prefixes=RV32I,RV32NOFUSION %s
 ; RUN: llc -mtriple=riscv64 -relocation-model=pic -verify-machineinstrs < %s \
-; RUN:   | FileCheck -check-prefixes=RV64I %s
+; RUN:   | FileCheck -check-prefixes=RV64I,RV64NOFUSION %s
+; RUN: llc -mtriple=riscv32 -relocation-model=pic -verify-machineinstrs < %s \
+; RUN:   -mattr=+auipc-addi-fusion | FileCheck -check-prefixes=RV32I,RV32FUSION %s
+; RUN: llc -mtriple=riscv64 -relocation-model=pic -verify-machineinstrs < %s \
+; RUN:   -mattr=+auipc-addi-fusion | FileCheck -check-prefixes=RV64I,RV64FUSION %s
 
 ; Verifies that MachineLICM can hoist address generation pseudos out of loops.
 
@@ -141,59 +145,113 @@ ret:
 @gd = external thread_local global i32
 
 define void @test_la_tls_gd(i32 signext %n) nounwind {
-; RV32I-LABEL: test_la_tls_gd:
-; RV32I:       # %bb.0: # %entry
-; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
-; RV32I-NEXT:    sw s0, 8(sp) # 4-byte Folded Spill
-; RV32I-NEXT:    sw s1, 4(sp) # 4-byte Folded Spill
-; RV32I-NEXT:    sw s2, 0(sp) # 4-byte Folded Spill
-; RV32I-NEXT:    mv s0, a0
-; RV32I-NEXT:    li s2, 0
-; RV32I-NEXT:  .Lpcrel_hi3:
-; RV32I-NEXT:    auipc a0, %tls_gd_pcrel_hi(gd)
-; RV32I-NEXT:    addi s1, a0, %pcrel_lo(.Lpcrel_hi3)
-; RV32I-NEXT:  .LBB3_1: # %loop
-; RV32I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV32I-NEXT:    mv a0, s1
-; RV32I-NEXT:    call __tls_get_addr
-; RV32I-NEXT:    lw zero, 0(a0)
-; RV32I-NEXT:    addi s2, s2, 1
-; RV32I-NEXT:    blt s2, s0, .LBB3_1
-; RV32I-NEXT:  # %bb.2: # %ret
-; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
-; RV32I-NEXT:    lw s0, 8(sp) # 4-byte Folded Reload
-; RV32I-NEXT:    lw s1, 4(sp) # 4-byte Folded Reload
-; RV32I-NEXT:    lw s2, 0(sp) # 4-byte Folded Reload
-; RV32I-NEXT:    addi sp, sp, 16
-; RV32I-NEXT:    ret
+; RV32NOFUSION-LABEL: test_la_tls_gd:
+; RV32NOFUSION:       # %bb.0: # %entry
+; RV32NOFUSION-NEXT:    addi sp, sp, -16
+; RV32NOFUSION-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32NOFUSION-NEXT:    sw s0, 8(sp) # 4-byte Folded Spill
+; RV32NOFUSION-NEXT:    sw s1, 4(sp) # 4-byte Folded Spill
+; RV32NOFUSION-NEXT:    sw s2, 0(sp) # 4-byte Folded Spill
+; RV32NOFUSION-NEXT:    mv s0, a0
+; RV32NOFUSION-NEXT:    li s2, 0
+; RV32NOFUSION-NEXT:  .Lpcrel_hi3:
+; RV32NOFUSION-NEXT:    auipc a0, %tls_gd_pcrel_hi(gd)
+; RV32NOFUSION-NEXT:    addi s1, a0, %pcrel_lo(.Lpcrel_hi3)
+; RV32NOFUSION-NEXT:  .LBB3_1: # %loop
+; RV32NOFUSION-NEXT:    # =>This Inner Loop Header: Depth=1
+; RV32NOFUSION-NEXT:    mv a0, s1
+; RV32NOFUSION-NEXT:    call __tls_get_addr
+; RV32NOFUSION-NEXT:    lw zero, 0(a0)
+; RV32NOFUSION-NEXT:    addi s2, s2, 1
+; RV32NOFUSION-NEXT:    blt s2, s0, .LBB3_1
+; RV32NOFUSION-NEXT:  # %bb.2: # %ret
+; RV32NOFUSION-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32NOFUSION-NEXT:    lw s0, 8(sp) # 4-byte Folded Reload
+; RV32NOFUSION-NEXT:    lw s1, 4(sp) # 4-byte Folded Reload
+; RV32NOFUSION-NEXT:    lw s2, 0(sp) # 4-byte Folded Reload
+; RV32NOFUSION-NEXT:    addi sp, sp, 16
+; RV32NOFUSION-NEXT:    ret
 ;
-; RV64I-LABEL: test_la_tls_gd:
-; RV64I:       # %bb.0: # %entry
-; RV64I-NEXT:    addi sp, sp, -32
-; RV64I-NEXT:    sd ra, 24(sp) # 8-byte Folded Spill
-; RV64I-NEXT:    sd s0, 16(sp) # 8-byte Folded Spill
-; RV64I-NEXT:    sd s1, 8(sp) # 8-byte Folded Spill
-; RV64I-NEXT:    sd s2, 0(sp) # 8-byte Folded Spill
-; RV64I-NEXT:    mv s0, a0
-; RV64I-NEXT:    li s2, 0
-; RV64I-NEXT:  .Lpcrel_hi3:
-; RV64I-NEXT:    auipc a0, %tls_gd_pcrel_hi(gd)
-; RV64I-NEXT:    addi s1, a0, %pcrel_lo(.Lpcrel_hi3)
-; RV64I-NEXT:  .LBB3_1: # %loop
-; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV64I-NEXT:    mv a0, s1
-; RV64I-NEXT:    call __tls_get_addr
-; RV64I-NEXT:    lw zero, 0(a0)
-; RV64I-NEXT:    addiw s2, s2, 1
-; RV64I-NEXT:    blt s2, s0, .LBB3_1
-; RV64I-NEXT:  # %bb.2: # %ret
-; RV64I-NEXT:    ld ra, 24(sp) # 8-byte Folded Reload
-; RV64I-NEXT:    ld s0, 16(sp) # 8-byte Folded Reload
-; RV64I-NEXT:    ld s1, 8(sp) # 8-byte Folded Reload
-; RV64I-NEXT:    ld s2, 0(sp) # 8-byte Folded Reload
-; RV64I-NEXT:    addi sp, sp, 32
-; RV64I-NEXT:    ret
+; RV64NOFUSION-LABEL: test_la_tls_gd:
+; RV64NOFUSION:       # %bb.0: # %entry
+; RV64NOFUSION-NEXT:    addi sp, sp, -32
+; RV64NOFUSION-NEXT:    sd ra, 24(sp) # 8-byte Folded Spill
+; RV64NOFUSION-NEXT:    sd s0, 16(sp) # 8-byte Folded Spill
+; RV64NOFUSION-NEXT:    sd s1, 8(sp) # 8-byte Folded Spill
+; RV64NOFUSION-NEXT:    sd s2, 0(sp) # 8-byte Folded Spill
+; RV64NOFUSION-NEXT:    mv s0, a0
+; RV64NOFUSION-NEXT:    li s2, 0
+; RV64NOFUSION-NEXT:  .Lpcrel_hi3:
+; RV64NOFUSION-NEXT:    auipc a0, %tls_gd_pcrel_hi(gd)
+; RV64NOFUSION-NEXT:    addi s1, a0, %pcrel_lo(.Lpcrel_hi3)
+; RV64NOFUSION-NEXT:  .LBB3_1: # %loop
+; RV64NOFUSION-NEXT:    # =>This Inner Loop Header: Depth=1
+; RV64NOFUSION-NEXT:    mv a0, s1
+; RV64NOFUSION-NEXT:    call __tls_get_addr
+; RV64NOFUSION-NEXT:    lw zero, 0(a0)
+; RV64NOFUSION-NEXT:    addiw s2, s2, 1
+; RV64NOFUSION-NEXT:    blt s2, s0, .LBB3_1
+; RV64NOFUSION-NEXT:  # %bb.2: # %ret
+; RV64NOFUSION-NEXT:    ld ra, 24(sp) # 8-byte Folded Reload
+; RV64NOFUSION-NEXT:    ld s0, 16(sp) # 8-byte Folded Reload
+; RV64NOFUSION-NEXT:    ld s1, 8(sp) # 8-byte Folded Reload
+; RV64NOFUSION-NEXT:    ld s2, 0(sp) # 8-byte Folded Reload
+; RV64NOFUSION-NEXT:    addi sp, sp, 32
+; RV64NOFUSION-NEXT:    ret
+;
+; RV32FUSION-LABEL: test_la_tls_gd:
+; RV32FUSION:       # %bb.0: # %entry
+; RV32FUSION-NEXT:    addi sp, sp, -16
+; RV32FUSION-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32FUSION-NEXT:    sw s0, 8(sp) # 4-byte Folded Spill
+; RV32FUSION-NEXT:    sw s1, 4(sp) # 4-byte Folded Spill
+; RV32FUSION-NEXT:    sw s2, 0(sp) # 4-byte Folded Spill
+; RV32FUSION-NEXT:    mv s0, a0
+; RV32FUSION-NEXT:    li s2, 0
+; RV32FUSION-NEXT:  .Lpcrel_hi3:
+; RV32FUSION-NEXT:    auipc s1, %tls_gd_pcrel_hi(gd)
+; RV32FUSION-NEXT:    addi s1, s1, %pcrel_lo(.Lpcrel_hi3)
+; RV32FUSION-NEXT:  .LBB3_1: # %loop
+; RV32FUSION-NEXT:    # =>This Inner Loop Header: Depth=1
+; RV32FUSION-NEXT:    mv a0, s1
+; RV32FUSION-NEXT:    call __tls_get_addr
+; RV32FUSION-NEXT:    lw zero, 0(a0)
+; RV32FUSION-NEXT:    addi s2, s2, 1
+; RV32FUSION-NEXT:    blt s2, s0, .LBB3_1
+; RV32FUSION-NEXT:  # %bb.2: # %ret
+; RV32FUSION-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32FUSION-NEXT:    lw s0, 8(sp) # 4-byte Folded Reload
+; RV32FUSION-NEXT:    lw s1, 4(sp) # 4-byte Folded Reload
+; RV32FUSION-NEXT:    lw s2, 0(sp) # 4-byte Folded Reload
+; RV32FUSION-NEXT:    addi sp, sp, 16
+; RV32FUSION-NEXT:    ret
+;
+; RV64FUSION-LABEL: test_la_tls_gd:
+; RV64FUSION:       # %bb.0: # %entry
+; RV64FUSION-NEXT:    addi sp, sp, -32
+; RV64FUSION-NEXT:    sd ra, 24(sp) # 8-byte Folded Spill
+; RV64FUSION-NEXT:    sd s0, 16(sp) # 8-byte Folded Spill
+; RV64FUSION-NEXT:    sd s1, 8(sp) # 8-byte Folded Spill
+; RV64FUSION-NEXT:    sd s2, 0(sp) # 8-byte Folded Spill
+; RV64FUSION-NEXT:    mv s0, a0
+; RV64FUSION-NEXT:    li s2, 0
+; RV64FUSION-NEXT:  .Lpcrel_hi3:
+; RV64FUSION-NEXT:    auipc s1, %tls_gd_pcrel_hi(gd)
+; RV64FUSION-NEXT:    addi s1, s1, %pcrel_lo(.Lpcrel_hi3)
+; RV64FUSION-NEXT:  .LBB3_1: # %loop
+; RV64FUSION-NEXT:    # =>This Inner Loop Header: Depth=1
+; RV64FUSION-NEXT:    mv a0, s1
+; RV64FUSION-NEXT:    call __tls_get_addr
+; RV64FUSION-NEXT:    lw zero, 0(a0)
+; RV64FUSION-NEXT:    addiw s2, s2, 1
+; RV64FUSION-NEXT:    blt s2, s0, .LBB3_1
+; RV64FUSION-NEXT:  # %bb.2: # %ret
+; RV64FUSION-NEXT:    ld ra, 24(sp) # 8-byte Folded Reload
+; RV64FUSION-NEXT:    ld s0, 16(sp) # 8-byte Folded Reload
+; RV64FUSION-NEXT:    ld s1, 8(sp) # 8-byte Folded Reload
+; RV64FUSION-NEXT:    ld s2, 0(sp) # 8-byte Folded Reload
+; RV64FUSION-NEXT:    addi sp, sp, 32
+; RV64FUSION-NEXT:    ret
 entry:
   br label %loop
 
diff --git a/llvm/test/CodeGen/RISCV/macro-fusion-lui-addi.ll b/llvm/test/CodeGen/RISCV/macro-fusion-lui-addi.ll
index 3f758e25c4217..d1b10af16063a 100644
--- a/llvm/test/CodeGen/RISCV/macro-fusion-lui-addi.ll
+++ b/llvm/test/CodeGen/RISCV/macro-fusion-lui-addi.ll
@@ -12,6 +12,8 @@
 ;RUN: llc < %s -mtriple=riscv64 -mattr=+f,+lui-addi-fusion,+use-postra-scheduler -mcpu=sifive-u74 \
 ;RUN:   -misched-postra-direction=bidirectional -target-abi=lp64f \
 ;RUN:   | FileCheck %s --check-prefixes=FUSION-POSTRA,FUSION-POSTRA-BIDIRECTIONAL
+;RUN: llc < %s -mtriple=riscv64 -mattr=+f,+lui-addi-fusion -target-abi=lp64f \
+;RUN:   | FileCheck %s --check-prefix=FUSION-GENERIC
 
 @.str = private constant [4 x i8] c"%f\0A\00", align 1
 
@@ -50,6 +52,13 @@ define void @foo(i32 signext %0, i32 signext %1) {
 ; FUSION-POSTRA-BIDIRECTIONAL-NEXT:    addi a0, a0, %lo(.L.str)
 ; FUSION-POSTRA-BIDIRECTIONAL-NEXT:    fcvt.s.w fa0, a1
 ; FUSION-POSTRA-BIDIRECTIONAL-NEXT:    tail bar
+;
+; FUSION-GENERIC-LABEL: foo:
+; FUSION-GENERIC:       # %bb.0:
+; FUSION-GENERIC-NEXT:    fcvt.s.w fa0, a1
+; FUSION-GENERIC-NEXT:    lui a0, %hi(.L.str)
+; FUSION-GENERIC-NEXT:    addi a0, a0, %lo(.L.str)
+; FUSION-GENERIC-NEXT:    tail bar
   %3 = sitofp i32 %1 to float
   tail call void @bar(ptr @.str, float %3)
   ret void
@@ -76,5 +85,44 @@ define i32 @test_matint() {
 ; FUSION-POSTRA-NEXT:    lui a0, 1
 ; FUSION-POSTRA-NEXT:    addiw a0, a0, -2048
 ; FUSION-POSTRA-NEXT:    ret
+;
+; FUSION-GENERIC-LABEL: test_matint:
+; FUSION-GENERIC:       # %bb.0:
+; FUSION-GENERIC-NEXT:    lui a0, 1
+; FUSION-GENERIC-NEXT:    addiw a0, a0, -2048
+; FUSION-GENERIC-NEXT:    ret
   ret i32 2048
 }
+
+define void @test_regalloc_hint(i32 noundef signext %0, i32 noundef signext %1) {
+; NOFUSION-LABEL: test_regalloc_hint:
+; NOFUSION:       # %bb.0:
+; NOFUSION-NEXT:    mv a0, a1
+; NOFUSION-NEXT:    lui a1, 3014
+; NOFUSION-NEXT:    addiw a1, a1, 334
+; NOFUSION-NEXT:    tail bar
+;
+; FUSION-LABEL: test_regalloc_hint:
+; FUSION:       # %bb.0:
+; FUSION-NEXT:    mv a0, a1
+; FUSION-NEXT:    lui a1, 3014
+; FUSION-NEXT:    addiw a1, a1, 334
+; FUSION-NEXT:    tail bar
+;
+; FUSION-POSTRA-LABEL: test_regalloc_hint:
+; FUSION-POSTRA:       # %bb.0:
+; FUSION-POSTRA-NEXT:    mv a0, a1
+; FUSION-POSTRA-NEXT:    lui a1, 3014
+; FUSION-POSTRA-NEXT:    addiw a1, a1, 334
+; FUSION-POSTRA-NEXT:    tail bar
+;
+; FUSION-GENERIC-LABEL: test_regalloc_hint:
+; FUSION-GENERIC:       # %bb.0:
+; FUSION-GENERIC-NEXT:    lui a2, 3014
+; FUSION-GENERIC-NEXT:    addiw a2, a2, 334
+; FUSION-GENERIC-NEXT:    mv a0, a1
+; FUSION-GENERIC-NEXT:    mv a1, a2
+; FUSION-GENERIC-NEXT:    tail bar
+  tail call void @bar(i32 noundef signext %1, i32 noundef signext 12345678)
+  ret void
+}
diff --git a/llvm/test/CodeGen/RISCV/memmove.ll b/llvm/test/CodeGen/RISCV/memmove.ll
new file mode 100644
index 0000000000000..62915bd4ad99d
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/memmove.ll
@@ -0,0 +1,670 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=riscv32 \
+; RUN:   | FileCheck %s --check-prefixes=RV32-BOTH,RV32
+; RUN: llc < %s -mtriple=riscv64 \
+; RUN:   | FileCheck %s --check-prefixes=RV64-BOTH,RV64
+; RUN: llc < %s -mtriple=riscv32 -mattr=+unaligned-scalar-mem \
+; RUN:   | FileCheck %s --check-prefixes=RV32-BOTH,RV32-FAST
+; RUN: llc < %s -mtriple=riscv64 -mattr=+unaligned-scalar-mem \
+; RUN:   | FileCheck %s --check-prefixes=RV64-BOTH,RV64-FAST
+
+; ----------------------------------------------------------------------
+; Fully unaligned cases
+
+define void @unaligned_memmove0(ptr nocapture %dest, ptr %src) nounwind {
+; RV32-BOTH-LABEL: unaligned_memmove0:
+; RV32-BOTH:       # %bb.0: # %entry
+; RV32-BOTH-NEXT:    ret
+;
+; RV64-BOTH-LABEL: unaligned_memmove0:
+; RV64-BOTH:       # %bb.0: # %entry
+; RV64-BOTH-NEXT:    ret
+entry:
+  tail call void @llvm.memmove.p0.p0.i64(ptr %dest, ptr %src, i64 0, i1 false)
+  ret void
+}
+
+define void @unaligned_memmove1(ptr nocapture %dest, ptr %src) nounwind {
+; RV32-BOTH-LABEL: unaligned_memmove1:
+; RV32-BOTH:       # %bb.0: # %entry
+; RV32-BOTH-NEXT:    lbu a1, 0(a1)
+; RV32-BOTH-NEXT:    sb a1, 0(a0)
+; RV32-BOTH-NEXT:    ret
+;
+; RV64-BOTH-LABEL: unaligned_memmove1:
+; RV64-BOTH:       # %bb.0: # %entry
+; RV64-BOTH-NEXT:    lbu a1, 0(a1)
+; RV64-BOTH-NEXT:    sb a1, 0(a0)
+; RV64-BOTH-NEXT:    ret
+entry:
+  tail call void @llvm.memmove.p0.p0.i64(ptr %dest, ptr %src, i64 1, i1 false)
+  ret void
+}
+
+define void @unaligned_memmove2(ptr nocapture %dest, ptr %src) nounwind {
+; RV32-LABEL: unaligned_memmove2:
+; RV32:       # %bb.0: # %entry
+; RV32-NEXT:    lbu a2, 0(a1)
+; RV32-NEXT:    lbu a1, 1(a1)
+; RV32-NEXT:    sb a2, 0(a0)
+; RV32-NEXT:    sb a1, 1(a0)
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: unaligned_memmove2:
+; RV64:       # %bb.0: # %entry
+; RV64-NEXT:    lbu a2, 0(a1)
+; RV64-NEXT:    lbu a1, 1(a1)
+; RV64-NEXT:    sb a2, 0(a0)
+; RV64-NEXT:    sb a1, 1(a0)
+; RV64-NEXT:    ret
+;
+; RV32-FAST-LABEL: unaligned_memmove2:
+; RV32-FAST:       # %bb.0: # %entry
+; RV32-FAST-NEXT:    lh a1, 0(a1)
+; RV32-FAST-NEXT:    sh a1, 0(a0)
+; RV32-FAST-NEXT:    ret
+;
+; RV64-FAST-LABEL: unaligned_memmove2:
+; RV64-FAST:       # %bb.0: # %entry
+; RV64-FAST-NEXT:    lh a1, 0(a1)
+; RV64-FAST-NEXT:    sh a1, 0(a0)
+; RV64-FAST-NEXT:    ret
+entry:
+  tail call void @llvm.memmove.p0.p0.i64(ptr %dest, ptr %src, i64 2, i1 false)
+  ret void
+}
+
+define void @unaligned_memmove3(ptr nocapture %dest, ptr %src) nounwind {
+; RV32-LABEL: unaligned_memmove3:
+; RV32:       # %bb.0: # %entry
+; RV32-NEXT:    lbu a2, 0(a1)
+; RV32-NEXT:    lbu a3, 1(a1)
+; RV32-NEXT:    lbu a1, 2(a1)
+; RV32-NEXT:    sb a2, 0(a0)
+; RV32-NEXT:    sb a3, 1(a0)
+; RV32-NEXT:    sb a1, 2(a0)
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: unaligned_memmove3:
+; RV64:       # %bb.0: # %entry
+; RV64-NEXT:    lbu a2, 0(a1)
+; RV64-NEXT:    lbu a3, 1(a1)
+; RV64-NEXT:    lbu a1, 2(a1)
+; RV64-NEXT:    sb a2, 0(a0)
+; RV64-NEXT:    sb a3, 1(a0)
+; RV64-NEXT:    sb a1, 2(a0)
+; RV64-NEXT:    ret
+;
+; RV32-FAST-LABEL: unaligned_memmove3:
+; RV32-FAST:       # %bb.0: # %entry
+; RV32-FAST-NEXT:    lh a2, 0(a1)
+; RV32-FAST-NEXT:    lbu a1, 2(a1)
+; RV32-FAST-NEXT:    sh a2, 0(a0)
+; RV32-FAST-NEXT:    sb a1, 2(a0)
+; RV32-FAST-NEXT:    ret
+;
+; RV64-FAST-LABEL: unaligned_memmove3:
+; RV64-FAST:       # %bb.0: # %entry
+; RV64-FAST-NEXT:    lh a2, 0(a1)
+; RV64-FAST-NEXT:    lbu a1, 2(a1)
+; RV64-FAST-NEXT:    sh a2, 0(a0)
+; RV64-FAST-NEXT:    sb a1, 2(a0)
+; RV64-FAST-NEXT:    ret
+entry:
+  tail call void @llvm.memmove.p0.p0.i64(ptr %dest, ptr %src, i64 3, i1 false)
+  ret void
+}
+
+define void @unaligned_memmove4(ptr nocapture %dest, ptr %src) nounwind {
+; RV32-LABEL: unaligned_memmove4:
+; RV32:       # %bb.0: # %entry
+; RV32-NEXT:    lbu a2, 0(a1)
+; RV32-NEXT:    lbu a3, 1(a1)
+; RV32-NEXT:    lbu a4, 2(a1)
+; RV32-NEXT:    lbu a1, 3(a1)
+; RV32-NEXT:    sb a2, 0(a0)
+; RV32-NEXT:    sb a3, 1(a0)
+; RV32-NEXT:    sb a4, 2(a0)
+; RV32-NEXT:    sb a1, 3(a0)
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: unaligned_memmove4:
+; RV64:       # %bb.0: # %entry
+; RV64-NEXT:    lbu a2, 0(a1)
+; RV64-NEXT:    lbu a3, 1(a1)
+; RV64-NEXT:    lbu a4, 2(a1)
+; RV64-NEXT:    lbu a1, 3(a1)
+; RV64-NEXT:    sb a2, 0(a0)
+; RV64-NEXT:    sb a3, 1(a0)
+; RV64-NEXT:    sb a4, 2(a0)
+; RV64-NEXT:    sb a1, 3(a0)
+; RV64-NEXT:    ret
+;
+; RV32-FAST-LABEL: unaligned_memmove4:
+; RV32-FAST:       # %bb.0: # %entry
+; RV32-FAST-NEXT:    lw a1, 0(a1)
+; RV32-FAST-NEXT:    sw a1, 0(a0)
+; RV32-FAST-NEXT:    ret
+;
+; RV64-FAST-LABEL: unaligned_memmove4:
+; RV64-FAST:       # %bb.0: # %entry
+; RV64-FAST-NEXT:    lw a1, 0(a1)
+; RV64-FAST-NEXT:    sw a1, 0(a0)
+; RV64-FAST-NEXT:    ret
+entry:
+  tail call void @llvm.memmove.p0.p0.i64(ptr %dest, ptr %src, i64 4, i1 false)
+  ret void
+}
+
+define void @unaligned_memmove7(ptr nocapture %dest, ptr %src) nounwind {
+; RV32-LABEL: unaligned_memmove7:
+; RV32:       # %bb.0: # %entry
+; RV32-NEXT:    lbu a2, 4(a1)
+; RV32-NEXT:    lbu a3, 5(a1)
+; RV32-NEXT:    lbu a4, 6(a1)
+; RV32-NEXT:    lbu a5, 0(a1)
+; RV32-NEXT:    lbu a6, 1(a1)
+; RV32-NEXT:    lbu a7, 2(a1)
+; RV32-NEXT:    lbu a1, 3(a1)
+; RV32-NEXT:    sb a2, 4(a0)
+; RV32-NEXT:    sb a3, 5(a0)
+; RV32-NEXT:    sb a4, 6(a0)
+; RV32-NEXT:    sb a5, 0(a0)
+; RV32-NEXT:    sb a6, 1(a0)
+; RV32-NEXT:    sb a7, 2(a0)
+; RV32-NEXT:    sb a1, 3(a0)
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: unaligned_memmove7:
+; RV64:       # %bb.0: # %entry
+; RV64-NEXT:    lbu a2, 4(a1)
+; RV64-NEXT:    lbu a3, 5(a1)
+; RV64-NEXT:    lbu a4, 6(a1)
+; RV64-NEXT:    lbu a5, 0(a1)
+; RV64-NEXT:    lbu a6, 1(a1)
+; RV64-NEXT:    lbu a7, 2(a1)
+; RV64-NEXT:    lbu a1, 3(a1)
+; RV64-NEXT:    sb a2, 4(a0)
+; RV64-NEXT:    sb a3, 5(a0)
+; RV64-NEXT:    sb a4, 6(a0)
+; RV64-NEXT:    sb a5, 0(a0)
+; RV64-NEXT:    sb a6, 1(a0)
+; RV64-NEXT:    sb a7, 2(a0)
+; RV64-NEXT:    sb a1, 3(a0)
+; RV64-NEXT:    ret
+;
+; RV32-FAST-LABEL: unaligned_memmove7:
+; RV32-FAST:       # %bb.0: # %entry
+; RV32-FAST-NEXT:    lw a2, 0(a1)
+; RV32-FAST-NEXT:    lh a3, 4(a1)
+; RV32-FAST-NEXT:    lbu a1, 6(a1)
+; RV32-FAST-NEXT:    sw a2, 0(a0)
+; RV32-FAST-NEXT:    sh a3, 4(a0)
+; RV32-FAST-NEXT:    sb a1, 6(a0)
+; RV32-FAST-NEXT:    ret
+;
+; RV64-FAST-LABEL: unaligned_memmove7:
+; RV64-FAST:       # %bb.0: # %entry
+; RV64-FAST-NEXT:    lw a2, 0(a1)
+; RV64-FAST-NEXT:    lh a3, 4(a1)
+; RV64-FAST-NEXT:    lbu a1, 6(a1)
+; RV64-FAST-NEXT:    sw a2, 0(a0)
+; RV64-FAST-NEXT:    sh a3, 4(a0)
+; RV64-FAST-NEXT:    sb a1, 6(a0)
+; RV64-FAST-NEXT:    ret
+entry:
+  tail call void @llvm.memmove.p0.p0.i64(ptr %dest, ptr %src, i64 7, i1 false)
+  ret void
+}
+
+define void @unaligned_memmove8(ptr nocapture %dest, ptr %src) nounwind {
+; RV32-LABEL: unaligned_memmove8:
+; RV32:       # %bb.0: # %entry
+; RV32-NEXT:    lbu a2, 0(a1)
+; RV32-NEXT:    lbu a3, 1(a1)
+; RV32-NEXT:    lbu a4, 2(a1)
+; RV32-NEXT:    lbu a5, 3(a1)
+; RV32-NEXT:    lbu a6, 4(a1)
+; RV32-NEXT:    lbu a7, 5(a1)
+; RV32-NEXT:    lbu t0, 6(a1)
+; RV32-NEXT:    lbu a1, 7(a1)
+; RV32-NEXT:    sb a6, 4(a0)
+; RV32-NEXT:    sb a7, 5(a0)
+; RV32-NEXT:    sb t0, 6(a0)
+; RV32-NEXT:    sb a1, 7(a0)
+; RV32-NEXT:    sb a2, 0(a0)
+; RV32-NEXT:    sb a3, 1(a0)
+; RV32-NEXT:    sb a4, 2(a0)
+; RV32-NEXT:    sb a5, 3(a0)
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: unaligned_memmove8:
+; RV64:       # %bb.0: # %entry
+; RV64-NEXT:    lbu a2, 0(a1)
+; RV64-NEXT:    lbu a3, 1(a1)
+; RV64-NEXT:    lbu a4, 2(a1)
+; RV64-NEXT:    lbu a5, 3(a1)
+; RV64-NEXT:    lbu a6, 4(a1)
+; RV64-NEXT:    lbu a7, 5(a1)
+; RV64-NEXT:    lbu t0, 6(a1)
+; RV64-NEXT:    lbu a1, 7(a1)
+; RV64-NEXT:    sb a6, 4(a0)
+; RV64-NEXT:    sb a7, 5(a0)
+; RV64-NEXT:    sb t0, 6(a0)
+; RV64-NEXT:    sb a1, 7(a0)
+; RV64-NEXT:    sb a2, 0(a0)
+; RV64-NEXT:    sb a3, 1(a0)
+; RV64-NEXT:    sb a4, 2(a0)
+; RV64-NEXT:    sb a5, 3(a0)
+; RV64-NEXT:    ret
+;
+; RV32-FAST-LABEL: unaligned_memmove8:
+; RV32-FAST:       # %bb.0: # %entry
+; RV32-FAST-NEXT:    lw a2, 0(a1)
+; RV32-FAST-NEXT:    lw a1, 4(a1)
+; RV32-FAST-NEXT:    sw a2, 0(a0)
+; RV32-FAST-NEXT:    sw a1, 4(a0)
+; RV32-FAST-NEXT:    ret
+;
+; RV64-FAST-LABEL: unaligned_memmove8:
+; RV64-FAST:       # %bb.0: # %entry
+; RV64-FAST-NEXT:    ld a1, 0(a1)
+; RV64-FAST-NEXT:    sd a1, 0(a0)
+; RV64-FAST-NEXT:    ret
+entry:
+  tail call void @llvm.memmove.p0.p0.i64(ptr %dest, ptr %src, i64 8, i1 false)
+  ret void
+}
+
+define void @unaligned_memmove15(ptr nocapture %dest, ptr %src) nounwind {
+; RV32-LABEL: unaligned_memmove15:
+; RV32:       # %bb.0: # %entry
+; RV32-NEXT:    li a2, 15
+; RV32-NEXT:    tail memmove
+;
+; RV64-LABEL: unaligned_memmove15:
+; RV64:       # %bb.0: # %entry
+; RV64-NEXT:    li a2, 15
+; RV64-NEXT:    tail memmove
+;
+; RV32-FAST-LABEL: unaligned_memmove15:
+; RV32-FAST:       # %bb.0: # %entry
+; RV32-FAST-NEXT:    lbu a2, 14(a1)
+; RV32-FAST-NEXT:    lw a3, 0(a1)
+; RV32-FAST-NEXT:    lw a4, 4(a1)
+; RV32-FAST-NEXT:    lw a5, 8(a1)
+; RV32-FAST-NEXT:    lh a1, 12(a1)
+; RV32-FAST-NEXT:    sb a2, 14(a0)
+; RV32-FAST-NEXT:    sw a3, 0(a0)
+; RV32-FAST-NEXT:    sw a4, 4(a0)
+; RV32-FAST-NEXT:    sw a5, 8(a0)
+; RV32-FAST-NEXT:    sh a1, 12(a0)
+; RV32-FAST-NEXT:    ret
+;
+; RV64-FAST-LABEL: unaligned_memmove15:
+; RV64-FAST:       # %bb.0: # %entry
+; RV64-FAST-NEXT:    ld a2, 0(a1)
+; RV64-FAST-NEXT:    lw a3, 8(a1)
+; RV64-FAST-NEXT:    lh a4, 12(a1)
+; RV64-FAST-NEXT:    lbu a1, 14(a1)
+; RV64-FAST-NEXT:    sd a2, 0(a0)
+; RV64-FAST-NEXT:    sw a3, 8(a0)
+; RV64-FAST-NEXT:    sh a4, 12(a0)
+; RV64-FAST-NEXT:    sb a1, 14(a0)
+; RV64-FAST-NEXT:    ret
+entry:
+  tail call void @llvm.memmove.p0.p0.i64(ptr %dest, ptr %src, i64 15, i1 false)
+  ret void
+}
+
+define void @unaligned_memmove16(ptr nocapture %dest, ptr %src) nounwind {
+; RV32-LABEL: unaligned_memmove16:
+; RV32:       # %bb.0: # %entry
+; RV32-NEXT:    li a2, 16
+; RV32-NEXT:    tail memmove
+;
+; RV64-LABEL: unaligned_memmove16:
+; RV64:       # %bb.0: # %entry
+; RV64-NEXT:    li a2, 16
+; RV64-NEXT:    tail memmove
+;
+; RV32-FAST-LABEL: unaligned_memmove16:
+; RV32-FAST:       # %bb.0: # %entry
+; RV32-FAST-NEXT:    lw a2, 0(a1)
+; RV32-FAST-NEXT:    lw a3, 4(a1)
+; RV32-FAST-NEXT:    lw a4, 8(a1)
+; RV32-FAST-NEXT:    lw a1, 12(a1)
+; RV32-FAST-NEXT:    sw a2, 0(a0)
+; RV32-FAST-NEXT:    sw a3, 4(a0)
+; RV32-FAST-NEXT:    sw a4, 8(a0)
+; RV32-FAST-NEXT:    sw a1, 12(a0)
+; RV32-FAST-NEXT:    ret
+;
+; RV64-FAST-LABEL: unaligned_memmove16:
+; RV64-FAST:       # %bb.0: # %entry
+; RV64-FAST-NEXT:    ld a2, 0(a1)
+; RV64-FAST-NEXT:    ld a1, 8(a1)
+; RV64-FAST-NEXT:    sd a2, 0(a0)
+; RV64-FAST-NEXT:    sd a1, 8(a0)
+; RV64-FAST-NEXT:    ret
+entry:
+  tail call void @llvm.memmove.p0.p0.i64(ptr %dest, ptr %src, i64 16, i1 false)
+  ret void
+}
+
+define void @unaligned_memmove31(ptr nocapture %dest, ptr %src) nounwind {
+; RV32-BOTH-LABEL: unaligned_memmove31:
+; RV32-BOTH:       # %bb.0: # %entry
+; RV32-BOTH-NEXT:    li a2, 31
+; RV32-BOTH-NEXT:    tail memmove
+;
+; RV64-LABEL: unaligned_memmove31:
+; RV64:       # %bb.0: # %entry
+; RV64-NEXT:    li a2, 31
+; RV64-NEXT:    tail memmove
+;
+; RV64-FAST-LABEL: unaligned_memmove31:
+; RV64-FAST:       # %bb.0: # %entry
+; RV64-FAST-NEXT:    lh a2, 28(a1)
+; RV64-FAST-NEXT:    lbu a3, 30(a1)
+; RV64-FAST-NEXT:    ld a4, 0(a1)
+; RV64-FAST-NEXT:    ld a5, 8(a1)
+; RV64-FAST-NEXT:    ld a6, 16(a1)
+; RV64-FAST-NEXT:    lw a1, 24(a1)
+; RV64-FAST-NEXT:    sh a2, 28(a0)
+; RV64-FAST-NEXT:    sb a3, 30(a0)
+; RV64-FAST-NEXT:    sd a4, 0(a0)
+; RV64-FAST-NEXT:    sd a5, 8(a0)
+; RV64-FAST-NEXT:    sd a6, 16(a0)
+; RV64-FAST-NEXT:    sw a1, 24(a0)
+; RV64-FAST-NEXT:    ret
+entry:
+  tail call void @llvm.memmove.p0.p0.i64(ptr %dest, ptr %src, i64 31, i1 false)
+  ret void
+}
+
+; ----------------------------------------------------------------------
+; Fully aligned cases
+
+define void @aligned_memmove0(ptr nocapture %dest, ptr %src) nounwind {
+; RV32-BOTH-LABEL: aligned_memmove0:
+; RV32-BOTH:       # %bb.0: # %entry
+; RV32-BOTH-NEXT:    ret
+;
+; RV64-BOTH-LABEL: aligned_memmove0:
+; RV64-BOTH:       # %bb.0: # %entry
+; RV64-BOTH-NEXT:    ret
+entry:
+  tail call void @llvm.memmove.p0.p0.i64(ptr align 8 %dest, ptr align 8 %src, i64 0, i1 false)
+  ret void
+}
+
+define void @aligned_memmove1(ptr nocapture %dest, ptr %src) nounwind {
+; RV32-BOTH-LABEL: aligned_memmove1:
+; RV32-BOTH:       # %bb.0: # %entry
+; RV32-BOTH-NEXT:    lbu a1, 0(a1)
+; RV32-BOTH-NEXT:    sb a1, 0(a0)
+; RV32-BOTH-NEXT:    ret
+;
+; RV64-BOTH-LABEL: aligned_memmove1:
+; RV64-BOTH:       # %bb.0: # %entry
+; RV64-BOTH-NEXT:    lbu a1, 0(a1)
+; RV64-BOTH-NEXT:    sb a1, 0(a0)
+; RV64-BOTH-NEXT:    ret
+entry:
+  tail call void @llvm.memmove.p0.p0.i64(ptr align 8 %dest, ptr align 8 %src, i64 1, i1 false)
+  ret void
+}
+
+define void @aligned_memmove2(ptr nocapture %dest, ptr %src) nounwind {
+; RV32-BOTH-LABEL: aligned_memmove2:
+; RV32-BOTH:       # %bb.0: # %entry
+; RV32-BOTH-NEXT:    lh a1, 0(a1)
+; RV32-BOTH-NEXT:    sh a1, 0(a0)
+; RV32-BOTH-NEXT:    ret
+;
+; RV64-BOTH-LABEL: aligned_memmove2:
+; RV64-BOTH:       # %bb.0: # %entry
+; RV64-BOTH-NEXT:    lh a1, 0(a1)
+; RV64-BOTH-NEXT:    sh a1, 0(a0)
+; RV64-BOTH-NEXT:    ret
+entry:
+  tail call void @llvm.memmove.p0.p0.i64(ptr align 8 %dest, ptr align 8 %src, i64 2, i1 false)
+  ret void
+}
+
+define void @aligned_memmove3(ptr nocapture %dest, ptr %src) nounwind {
+; RV32-BOTH-LABEL: aligned_memmove3:
+; RV32-BOTH:       # %bb.0: # %entry
+; RV32-BOTH-NEXT:    lh a2, 0(a1)
+; RV32-BOTH-NEXT:    lbu a1, 2(a1)
+; RV32-BOTH-NEXT:    sh a2, 0(a0)
+; RV32-BOTH-NEXT:    sb a1, 2(a0)
+; RV32-BOTH-NEXT:    ret
+;
+; RV64-BOTH-LABEL: aligned_memmove3:
+; RV64-BOTH:       # %bb.0: # %entry
+; RV64-BOTH-NEXT:    lh a2, 0(a1)
+; RV64-BOTH-NEXT:    lbu a1, 2(a1)
+; RV64-BOTH-NEXT:    sh a2, 0(a0)
+; RV64-BOTH-NEXT:    sb a1, 2(a0)
+; RV64-BOTH-NEXT:    ret
+entry:
+  tail call void @llvm.memmove.p0.p0.i64(ptr align 8 %dest, ptr align 8 %src, i64 3, i1 false)
+  ret void
+}
+
+define void @aligned_memmove4(ptr nocapture %dest, ptr %src) nounwind {
+; RV32-BOTH-LABEL: aligned_memmove4:
+; RV32-BOTH:       # %bb.0: # %entry
+; RV32-BOTH-NEXT:    lw a1, 0(a1)
+; RV32-BOTH-NEXT:    sw a1, 0(a0)
+; RV32-BOTH-NEXT:    ret
+;
+; RV64-BOTH-LABEL: aligned_memmove4:
+; RV64-BOTH:       # %bb.0: # %entry
+; RV64-BOTH-NEXT:    lw a1, 0(a1)
+; RV64-BOTH-NEXT:    sw a1, 0(a0)
+; RV64-BOTH-NEXT:    ret
+entry:
+  tail call void @llvm.memmove.p0.p0.i64(ptr align 8 %dest, ptr align 8 %src, i64 4, i1 false)
+  ret void
+}
+
+define void @aligned_memmove7(ptr nocapture %dest, ptr %src) nounwind {
+; RV32-BOTH-LABEL: aligned_memmove7:
+; RV32-BOTH:       # %bb.0: # %entry
+; RV32-BOTH-NEXT:    lw a2, 0(a1)
+; RV32-BOTH-NEXT:    lh a3, 4(a1)
+; RV32-BOTH-NEXT:    lbu a1, 6(a1)
+; RV32-BOTH-NEXT:    sw a2, 0(a0)
+; RV32-BOTH-NEXT:    sh a3, 4(a0)
+; RV32-BOTH-NEXT:    sb a1, 6(a0)
+; RV32-BOTH-NEXT:    ret
+;
+; RV64-BOTH-LABEL: aligned_memmove7:
+; RV64-BOTH:       # %bb.0: # %entry
+; RV64-BOTH-NEXT:    lw a2, 0(a1)
+; RV64-BOTH-NEXT:    lh a3, 4(a1)
+; RV64-BOTH-NEXT:    lbu a1, 6(a1)
+; RV64-BOTH-NEXT:    sw a2, 0(a0)
+; RV64-BOTH-NEXT:    sh a3, 4(a0)
+; RV64-BOTH-NEXT:    sb a1, 6(a0)
+; RV64-BOTH-NEXT:    ret
+entry:
+  tail call void @llvm.memmove.p0.p0.i64(ptr align 8 %dest, ptr align 8 %src, i64 7, i1 false)
+  ret void
+}
+
+define void @aligned_memmove8(ptr nocapture %dest, ptr %src) nounwind {
+; RV32-BOTH-LABEL: aligned_memmove8:
+; RV32-BOTH:       # %bb.0: # %entry
+; RV32-BOTH-NEXT:    lw a2, 0(a1)
+; RV32-BOTH-NEXT:    sw a2, 0(a0)
+; RV32-BOTH-NEXT:    lw a1, 4(a1)
+; RV32-BOTH-NEXT:    sw a1, 4(a0)
+; RV32-BOTH-NEXT:    ret
+;
+; RV64-BOTH-LABEL: aligned_memmove8:
+; RV64-BOTH:       # %bb.0: # %entry
+; RV64-BOTH-NEXT:    ld a1, 0(a1)
+; RV64-BOTH-NEXT:    sd a1, 0(a0)
+; RV64-BOTH-NEXT:    ret
+entry:
+  tail call void @llvm.memmove.p0.p0.i64(ptr align 8 %dest, ptr align 8 %src, i64 8, i1 false)
+  ret void
+}
+
+define void @aligned_memmove15(ptr nocapture %dest, ptr %src) nounwind {
+; RV32-BOTH-LABEL: aligned_memmove15:
+; RV32-BOTH:       # %bb.0: # %entry
+; RV32-BOTH-NEXT:    lw a2, 0(a1)
+; RV32-BOTH-NEXT:    lw a3, 8(a1)
+; RV32-BOTH-NEXT:    lh a4, 12(a1)
+; RV32-BOTH-NEXT:    lbu a5, 14(a1)
+; RV32-BOTH-NEXT:    sw a2, 0(a0)
+; RV32-BOTH-NEXT:    lw a1, 4(a1)
+; RV32-BOTH-NEXT:    sw a1, 4(a0)
+; RV32-BOTH-NEXT:    sw a3, 8(a0)
+; RV32-BOTH-NEXT:    sh a4, 12(a0)
+; RV32-BOTH-NEXT:    sb a5, 14(a0)
+; RV32-BOTH-NEXT:    ret
+;
+; RV64-BOTH-LABEL: aligned_memmove15:
+; RV64-BOTH:       # %bb.0: # %entry
+; RV64-BOTH-NEXT:    ld a2, 0(a1)
+; RV64-BOTH-NEXT:    lw a3, 8(a1)
+; RV64-BOTH-NEXT:    lh a4, 12(a1)
+; RV64-BOTH-NEXT:    lbu a1, 14(a1)
+; RV64-BOTH-NEXT:    sd a2, 0(a0)
+; RV64-BOTH-NEXT:    sw a3, 8(a0)
+; RV64-BOTH-NEXT:    sh a4, 12(a0)
+; RV64-BOTH-NEXT:    sb a1, 14(a0)
+; RV64-BOTH-NEXT:    ret
+entry:
+  tail call void @llvm.memmove.p0.p0.i64(ptr align 8 %dest, ptr align 8 %src, i64 15, i1 false)
+  ret void
+}
+
+define void @aligned_memmove16(ptr nocapture %dest, ptr %src) nounwind {
+; RV32-BOTH-LABEL: aligned_memmove16:
+; RV32-BOTH:       # %bb.0: # %entry
+; RV32-BOTH-NEXT:    lw a2, 0(a1)
+; RV32-BOTH-NEXT:    lw a3, 8(a1)
+; RV32-BOTH-NEXT:    sw a2, 0(a0)
+; RV32-BOTH-NEXT:    lw a2, 4(a1)
+; RV32-BOTH-NEXT:    lw a1, 12(a1)
+; RV32-BOTH-NEXT:    sw a2, 4(a0)
+; RV32-BOTH-NEXT:    sw a3, 8(a0)
+; RV32-BOTH-NEXT:    sw a1, 12(a0)
+; RV32-BOTH-NEXT:    ret
+;
+; RV64-BOTH-LABEL: aligned_memmove16:
+; RV64-BOTH:       # %bb.0: # %entry
+; RV64-BOTH-NEXT:    ld a2, 0(a1)
+; RV64-BOTH-NEXT:    ld a1, 8(a1)
+; RV64-BOTH-NEXT:    sd a2, 0(a0)
+; RV64-BOTH-NEXT:    sd a1, 8(a0)
+; RV64-BOTH-NEXT:    ret
+entry:
+  tail call void @llvm.memmove.p0.p0.i64(ptr align 8 %dest, ptr align 8 %src, i64 16, i1 false)
+  ret void
+}
+
+define void @aligned_memmove31(ptr nocapture %dest, ptr %src) nounwind {
+; RV32-BOTH-LABEL: aligned_memmove31:
+; RV32-BOTH:       # %bb.0: # %entry
+; RV32-BOTH-NEXT:    li a2, 31
+; RV32-BOTH-NEXT:    tail memmove
+;
+; RV64-BOTH-LABEL: aligned_memmove31:
+; RV64-BOTH:       # %bb.0: # %entry
+; RV64-BOTH-NEXT:    lh a2, 28(a1)
+; RV64-BOTH-NEXT:    lbu a3, 30(a1)
+; RV64-BOTH-NEXT:    ld a4, 0(a1)
+; RV64-BOTH-NEXT:    ld a5, 8(a1)
+; RV64-BOTH-NEXT:    ld a6, 16(a1)
+; RV64-BOTH-NEXT:    lw a1, 24(a1)
+; RV64-BOTH-NEXT:    sh a2, 28(a0)
+; RV64-BOTH-NEXT:    sb a3, 30(a0)
+; RV64-BOTH-NEXT:    sd a4, 0(a0)
+; RV64-BOTH-NEXT:    sd a5, 8(a0)
+; RV64-BOTH-NEXT:    sd a6, 16(a0)
+; RV64-BOTH-NEXT:    sw a1, 24(a0)
+; RV64-BOTH-NEXT:    ret
+entry:
+  tail call void @llvm.memmove.p0.p0.i64(ptr align 8 %dest, ptr align 8 %src, i64 31, i1 false)
+  ret void
+}
+
+; ------------------------------------------------------------------------
+; A few partially aligned cases
+
+
+define void @memmove16_align4(ptr nocapture %dest, ptr nocapture %src) nounwind {
+; RV32-BOTH-LABEL: memmove16_align4:
+; RV32-BOTH:       # %bb.0: # %entry
+; RV32-BOTH-NEXT:    lw a2, 0(a1)
+; RV32-BOTH-NEXT:    lw a3, 4(a1)
+; RV32-BOTH-NEXT:    lw a4, 8(a1)
+; RV32-BOTH-NEXT:    lw a1, 12(a1)
+; RV32-BOTH-NEXT:    sw a2, 0(a0)
+; RV32-BOTH-NEXT:    sw a3, 4(a0)
+; RV32-BOTH-NEXT:    sw a4, 8(a0)
+; RV32-BOTH-NEXT:    sw a1, 12(a0)
+; RV32-BOTH-NEXT:    ret
+;
+; RV64-LABEL: memmove16_align4:
+; RV64:       # %bb.0: # %entry
+; RV64-NEXT:    lw a2, 0(a1)
+; RV64-NEXT:    lw a3, 4(a1)
+; RV64-NEXT:    lw a4, 8(a1)
+; RV64-NEXT:    lw a1, 12(a1)
+; RV64-NEXT:    sw a2, 0(a0)
+; RV64-NEXT:    sw a3, 4(a0)
+; RV64-NEXT:    sw a4, 8(a0)
+; RV64-NEXT:    sw a1, 12(a0)
+; RV64-NEXT:    ret
+;
+; RV64-FAST-LABEL: memmove16_align4:
+; RV64-FAST:       # %bb.0: # %entry
+; RV64-FAST-NEXT:    ld a2, 0(a1)
+; RV64-FAST-NEXT:    ld a1, 8(a1)
+; RV64-FAST-NEXT:    sd a2, 0(a0)
+; RV64-FAST-NEXT:    sd a1, 8(a0)
+; RV64-FAST-NEXT:    ret
+entry:
+  tail call void @llvm.memmove.p0.p0.i64(ptr align 4 %dest, ptr align 4 %src, i64 16, i1 false)
+  ret void
+}
+
+define i32 @memmove11_align8(ptr nocapture %dest, ptr %src) {
+; RV32-BOTH-LABEL: memmove11_align8:
+; RV32-BOTH:       # %bb.0: # %entry
+; RV32-BOTH-NEXT:    lw a2, 0(a1)
+; RV32-BOTH-NEXT:    lh a3, 8(a1)
+; RV32-BOTH-NEXT:    lbu a4, 10(a1)
+; RV32-BOTH-NEXT:    sw a2, 0(a0)
+; RV32-BOTH-NEXT:    lw a1, 4(a1)
+; RV32-BOTH-NEXT:    sw a1, 4(a0)
+; RV32-BOTH-NEXT:    sh a3, 8(a0)
+; RV32-BOTH-NEXT:    sb a4, 10(a0)
+; RV32-BOTH-NEXT:    li a0, 0
+; RV32-BOTH-NEXT:    ret
+;
+; RV64-BOTH-LABEL: memmove11_align8:
+; RV64-BOTH:       # %bb.0: # %entry
+; RV64-BOTH-NEXT:    ld a2, 0(a1)
+; RV64-BOTH-NEXT:    lh a3, 8(a1)
+; RV64-BOTH-NEXT:    lbu a1, 10(a1)
+; RV64-BOTH-NEXT:    sd a2, 0(a0)
+; RV64-BOTH-NEXT:    sh a3, 8(a0)
+; RV64-BOTH-NEXT:    sb a1, 10(a0)
+; RV64-BOTH-NEXT:    li a0, 0
+; RV64-BOTH-NEXT:    ret
+entry:
+  call void @llvm.memmove.p0.p0.i64(ptr align 8 %dest, ptr align 8 %src, i64 11, i1 false)
+  ret i32 0
+}
+
+declare void @llvm.memmove.p0.p0.i64(ptr nocapture, ptr nocapture, i64, i1) nounwind
diff --git a/llvm/test/CodeGen/RISCV/opt-w-instrs.mir b/llvm/test/CodeGen/RISCV/opt-w-instrs.mir
index 03b4e330a0968..7d0e483c17aec 100644
--- a/llvm/test/CodeGen/RISCV/opt-w-instrs.mir
+++ b/llvm/test/CodeGen/RISCV/opt-w-instrs.mir
@@ -110,4 +110,62 @@ body:             |
      $x11 = COPY %4
      PseudoRET
 ...
+---
+name:            movgprnox0_1
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    liveins: $x10, $x11, $x12, $x13
+    ; CHECK-LABEL: name: movgprnox0_1
+    ; CHECK: liveins: $x10, $x11, $x12, $x13
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:gpr = COPY $x10
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:gpr = COPY $x11
+    ; CHECK-NEXT: [[COPY2:%[0-9]+]]:gprnox0 = COPY $x12
+    ; CHECK-NEXT: [[COPY3:%[0-9]+]]:gprnox0 = COPY $x13
+    ; CHECK-NEXT: [[PseudoCCMOVGPRNoX0_:%[0-9]+]]:gprnox0 = PseudoCCMOVGPRNoX0 [[COPY]], [[COPY1]], 1, [[COPY2]], [[COPY3]]
+    ; CHECK-NEXT: [[ADDIW:%[0-9]+]]:gpr = ADDIW [[PseudoCCMOVGPRNoX0_]], 0
+    ; CHECK-NEXT: $x10 = COPY [[ADDIW]]
+    ; CHECK-NEXT: PseudoRET implicit $x10
+    %0:gpr = COPY $x10
+    %1:gpr = COPY $x11
+    %2:gpr = COPY $x12
+    %3:gpr = COPY $x13
+    %4:gprnox0 = ADDIW %2, 0
+    %5:gprnox0 = ADDIW %3, 0
+    %6:gprnox0 = PseudoCCMOVGPRNoX0 %0, %1, 1, %4, %5
+    %7:gpr = ADDIW %6, 0
+    $x10 = COPY %7
+    PseudoRET implicit $x10
+
+...
+---
+name:            movgprnox0_2
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    liveins: $x10, $x11, $x12, $x13
+    ; CHECK-LABEL: name: movgprnox0_2
+    ; CHECK: liveins: $x10, $x11, $x12, $x13
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:gpr = COPY $x10
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:gpr = COPY $x11
+    ; CHECK-NEXT: [[COPY2:%[0-9]+]]:gpr = COPY $x12
+    ; CHECK-NEXT: [[COPY3:%[0-9]+]]:gpr = COPY $x13
+    ; CHECK-NEXT: [[SRAIW:%[0-9]+]]:gprnox0 = SRAIW [[COPY2]], 0
+    ; CHECK-NEXT: [[SRAIW1:%[0-9]+]]:gprnox0 = SRAIW [[COPY3]], 0
+    ; CHECK-NEXT: [[PseudoCCMOVGPRNoX0_:%[0-9]+]]:gprnox0 = PseudoCCMOVGPRNoX0 [[COPY]], [[COPY1]], 1, [[SRAIW]], [[SRAIW1]]
+    ; CHECK-NEXT: $x10 = COPY [[PseudoCCMOVGPRNoX0_]]
+    ; CHECK-NEXT: PseudoRET implicit $x10
+    %0:gpr = COPY $x10
+    %1:gpr = COPY $x11
+    %2:gpr = COPY $x12
+    %3:gpr = COPY $x13
+    %4:gprnox0 = SRAIW %2, 0
+    %5:gprnox0 = SRAIW %3, 0
+    %6:gprnox0 = PseudoCCMOVGPRNoX0 %0, %1, 1, %4, %5
+    %7:gpr = ADDIW %6, 0
+    $x10 = COPY %7
+    PseudoRET implicit $x10
 
+...
diff --git a/llvm/test/CodeGen/RISCV/postra-copy-expand.mir b/llvm/test/CodeGen/RISCV/postra-copy-expand.mir
new file mode 100644
index 0000000000000..e5b85659a0340
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/postra-copy-expand.mir
@@ -0,0 +1,24 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 5
+# RUN: llc -mtriple=riscv64 -mattr=+v -run-pass=postrapseudos %s -o - | FileCheck %s
+
+---
+name:            copy
+isSSA:           false
+noVRegs:         true
+liveins:
+  - { reg: '$v0', virtual-reg: '' }
+body:             |
+  bb.0:
+    liveins: $v0
+
+    ; CHECK-LABEL: name: copy
+    ; CHECK: liveins: $v0
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: $v20m2 = VMV2R_V $v14m2, implicit $vtype, implicit $v14_v15_v16_v17_v18
+    ; CHECK-NEXT: $v22m2 = VMV2R_V $v16m2, implicit $vtype, implicit $v14_v15_v16_v17_v18
+    ; CHECK-NEXT: $v24 = VMV1R_V $v18, implicit $vtype, implicit $v14_v15_v16_v17_v18, implicit $vtype
+    ; CHECK-NEXT: PseudoRET implicit $v0
+    renamable $v20_v21_v22_v23_v24 = COPY renamable $v14_v15_v16_v17_v18, implicit $vtype
+    PseudoRET implicit $v0
+
+...
diff --git a/llvm/test/CodeGen/RISCV/push-pop-popret.ll b/llvm/test/CodeGen/RISCV/push-pop-popret.ll
index 865d55a4dcb95..1fbdaa76dfb68 100644
--- a/llvm/test/CodeGen/RISCV/push-pop-popret.ll
+++ b/llvm/test/CodeGen/RISCV/push-pop-popret.ll
@@ -24,7 +24,7 @@ define i32 @foo() {
 ; RV32IZCMP-NEXT:    .cfi_offset ra, -4
 ; RV32IZCMP-NEXT:    addi sp, sp, -464
 ; RV32IZCMP-NEXT:    .cfi_def_cfa_offset 528
-; RV32IZCMP-NEXT:    mv a0, sp
+; RV32IZCMP-NEXT:    addi a0, sp, 12
 ; RV32IZCMP-NEXT:    call test
 ; RV32IZCMP-NEXT:    addi sp, sp, 464
 ; RV32IZCMP-NEXT:    .cfi_def_cfa_offset 64
@@ -37,7 +37,7 @@ define i32 @foo() {
 ; RV64IZCMP-NEXT:    .cfi_offset ra, -8
 ; RV64IZCMP-NEXT:    addi sp, sp, -464
 ; RV64IZCMP-NEXT:    .cfi_def_cfa_offset 528
-; RV64IZCMP-NEXT:    mv a0, sp
+; RV64IZCMP-NEXT:    addi a0, sp, 8
 ; RV64IZCMP-NEXT:    call test
 ; RV64IZCMP-NEXT:    addi sp, sp, 464
 ; RV64IZCMP-NEXT:    .cfi_def_cfa_offset 64
@@ -50,7 +50,7 @@ define i32 @foo() {
 ; RV32IZCMP-SR-NEXT:    .cfi_offset ra, -4
 ; RV32IZCMP-SR-NEXT:    addi sp, sp, -464
 ; RV32IZCMP-SR-NEXT:    .cfi_def_cfa_offset 528
-; RV32IZCMP-SR-NEXT:    mv a0, sp
+; RV32IZCMP-SR-NEXT:    addi a0, sp, 12
 ; RV32IZCMP-SR-NEXT:    call test
 ; RV32IZCMP-SR-NEXT:    addi sp, sp, 464
 ; RV32IZCMP-SR-NEXT:    .cfi_def_cfa_offset 64
@@ -63,7 +63,7 @@ define i32 @foo() {
 ; RV64IZCMP-SR-NEXT:    .cfi_offset ra, -8
 ; RV64IZCMP-SR-NEXT:    addi sp, sp, -464
 ; RV64IZCMP-SR-NEXT:    .cfi_def_cfa_offset 528
-; RV64IZCMP-SR-NEXT:    mv a0, sp
+; RV64IZCMP-SR-NEXT:    addi a0, sp, 8
 ; RV64IZCMP-SR-NEXT:    call test
 ; RV64IZCMP-SR-NEXT:    addi sp, sp, 464
 ; RV64IZCMP-SR-NEXT:    .cfi_def_cfa_offset 64
@@ -104,7 +104,7 @@ define i32 @foo() {
   ret i32 0
 }
 
-define i32 @pushpopret0(i32 signext %size){
+define i32 @pushpopret0(i32 signext %size) {
 ; RV32IZCMP-LABEL: pushpopret0:
 ; RV32IZCMP:       # %bb.0: # %entry
 ; RV32IZCMP-NEXT:    cm.push {ra, s0}, -16
@@ -1151,10 +1151,11 @@ entry:
 declare void @llvm.va_start(ptr)
 declare void @llvm.va_end(ptr)
 
-define i32 @varargs(ptr %fmt, ...) nounwind {
+define i32 @varargs(ptr %fmt, ...) {
 ; RV32IZCMP-LABEL: varargs:
 ; RV32IZCMP:       # %bb.0:
 ; RV32IZCMP-NEXT:    addi sp, sp, -48
+; RV32IZCMP-NEXT:    .cfi_def_cfa_offset 48
 ; RV32IZCMP-NEXT:    mv a0, a1
 ; RV32IZCMP-NEXT:    sw a5, 36(sp)
 ; RV32IZCMP-NEXT:    sw a6, 40(sp)
@@ -1166,11 +1167,13 @@ define i32 @varargs(ptr %fmt, ...) nounwind {
 ; RV32IZCMP-NEXT:    addi a1, sp, 24
 ; RV32IZCMP-NEXT:    sw a1, 12(sp)
 ; RV32IZCMP-NEXT:    addi sp, sp, 48
+; RV32IZCMP-NEXT:    .cfi_def_cfa_offset 0
 ; RV32IZCMP-NEXT:    ret
 ;
 ; RV64IZCMP-LABEL: varargs:
 ; RV64IZCMP:       # %bb.0:
 ; RV64IZCMP-NEXT:    addi sp, sp, -80
+; RV64IZCMP-NEXT:    .cfi_def_cfa_offset 80
 ; RV64IZCMP-NEXT:    sd a1, 24(sp)
 ; RV64IZCMP-NEXT:    addi a0, sp, 28
 ; RV64IZCMP-NEXT:    sd a0, 8(sp)
@@ -1182,11 +1185,13 @@ define i32 @varargs(ptr %fmt, ...) nounwind {
 ; RV64IZCMP-NEXT:    sd a3, 40(sp)
 ; RV64IZCMP-NEXT:    sd a4, 48(sp)
 ; RV64IZCMP-NEXT:    addi sp, sp, 80
+; RV64IZCMP-NEXT:    .cfi_def_cfa_offset 0
 ; RV64IZCMP-NEXT:    ret
 ;
 ; RV32IZCMP-SR-LABEL: varargs:
 ; RV32IZCMP-SR:       # %bb.0:
 ; RV32IZCMP-SR-NEXT:    addi sp, sp, -48
+; RV32IZCMP-SR-NEXT:    .cfi_def_cfa_offset 48
 ; RV32IZCMP-SR-NEXT:    mv a0, a1
 ; RV32IZCMP-SR-NEXT:    sw a5, 36(sp)
 ; RV32IZCMP-SR-NEXT:    sw a6, 40(sp)
@@ -1198,11 +1203,13 @@ define i32 @varargs(ptr %fmt, ...) nounwind {
 ; RV32IZCMP-SR-NEXT:    addi a1, sp, 24
 ; RV32IZCMP-SR-NEXT:    sw a1, 12(sp)
 ; RV32IZCMP-SR-NEXT:    addi sp, sp, 48
+; RV32IZCMP-SR-NEXT:    .cfi_def_cfa_offset 0
 ; RV32IZCMP-SR-NEXT:    ret
 ;
 ; RV64IZCMP-SR-LABEL: varargs:
 ; RV64IZCMP-SR:       # %bb.0:
 ; RV64IZCMP-SR-NEXT:    addi sp, sp, -80
+; RV64IZCMP-SR-NEXT:    .cfi_def_cfa_offset 80
 ; RV64IZCMP-SR-NEXT:    sd a1, 24(sp)
 ; RV64IZCMP-SR-NEXT:    addi a0, sp, 28
 ; RV64IZCMP-SR-NEXT:    sd a0, 8(sp)
@@ -1214,11 +1221,13 @@ define i32 @varargs(ptr %fmt, ...) nounwind {
 ; RV64IZCMP-SR-NEXT:    sd a3, 40(sp)
 ; RV64IZCMP-SR-NEXT:    sd a4, 48(sp)
 ; RV64IZCMP-SR-NEXT:    addi sp, sp, 80
+; RV64IZCMP-SR-NEXT:    .cfi_def_cfa_offset 0
 ; RV64IZCMP-SR-NEXT:    ret
 ;
 ; RV32I-LABEL: varargs:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -48
+; RV32I-NEXT:    .cfi_def_cfa_offset 48
 ; RV32I-NEXT:    mv a0, a1
 ; RV32I-NEXT:    sw a5, 36(sp)
 ; RV32I-NEXT:    sw a6, 40(sp)
@@ -1230,11 +1239,13 @@ define i32 @varargs(ptr %fmt, ...) nounwind {
 ; RV32I-NEXT:    addi a1, sp, 24
 ; RV32I-NEXT:    sw a1, 12(sp)
 ; RV32I-NEXT:    addi sp, sp, 48
+; RV32I-NEXT:    .cfi_def_cfa_offset 0
 ; RV32I-NEXT:    ret
 ;
 ; RV64I-LABEL: varargs:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -80
+; RV64I-NEXT:    .cfi_def_cfa_offset 80
 ; RV64I-NEXT:    sd a1, 24(sp)
 ; RV64I-NEXT:    addi a0, sp, 28
 ; RV64I-NEXT:    sd a0, 8(sp)
@@ -1246,6 +1257,7 @@ define i32 @varargs(ptr %fmt, ...) nounwind {
 ; RV64I-NEXT:    sd a3, 40(sp)
 ; RV64I-NEXT:    sd a4, 48(sp)
 ; RV64I-NEXT:    addi sp, sp, 80
+; RV64I-NEXT:    .cfi_def_cfa_offset 0
 ; RV64I-NEXT:    ret
   %va = alloca ptr
   call void @llvm.va_start(ptr %va)
@@ -1259,10 +1271,16 @@ define i32 @varargs(ptr %fmt, ...) nounwind {
 
 @var0 = global [18 x i32] zeroinitializer
 
-define void @many_args(i32, i32, i32, i32, i32, i32, i32, i32, i32) nounwind {
+define void @many_args(i32, i32, i32, i32, i32, i32, i32, i32, i32) {
 ; RV32IZCMP-LABEL: many_args:
 ; RV32IZCMP:       # %bb.0: # %entry
 ; RV32IZCMP-NEXT:    cm.push {ra, s0-s4}, -32
+; RV32IZCMP-NEXT:    .cfi_def_cfa_offset 32
+; RV32IZCMP-NEXT:    .cfi_offset s0, -20
+; RV32IZCMP-NEXT:    .cfi_offset s1, -16
+; RV32IZCMP-NEXT:    .cfi_offset s2, -12
+; RV32IZCMP-NEXT:    .cfi_offset s3, -8
+; RV32IZCMP-NEXT:    .cfi_offset s4, -4
 ; RV32IZCMP-NEXT:    lui a0, %hi(var0)
 ; RV32IZCMP-NEXT:    lw a6, %lo(var0)(a0)
 ; RV32IZCMP-NEXT:    lw a7, %lo(var0+4)(a0)
@@ -1306,6 +1324,12 @@ define void @many_args(i32, i32, i32, i32, i32, i32, i32, i32, i32) nounwind {
 ; RV64IZCMP-LABEL: many_args:
 ; RV64IZCMP:       # %bb.0: # %entry
 ; RV64IZCMP-NEXT:    cm.push {ra, s0-s4}, -48
+; RV64IZCMP-NEXT:    .cfi_def_cfa_offset 48
+; RV64IZCMP-NEXT:    .cfi_offset s0, -40
+; RV64IZCMP-NEXT:    .cfi_offset s1, -32
+; RV64IZCMP-NEXT:    .cfi_offset s2, -24
+; RV64IZCMP-NEXT:    .cfi_offset s3, -16
+; RV64IZCMP-NEXT:    .cfi_offset s4, -8
 ; RV64IZCMP-NEXT:    lui a0, %hi(var0)
 ; RV64IZCMP-NEXT:    lw a6, %lo(var0)(a0)
 ; RV64IZCMP-NEXT:    lw a7, %lo(var0+4)(a0)
@@ -1349,6 +1373,12 @@ define void @many_args(i32, i32, i32, i32, i32, i32, i32, i32, i32) nounwind {
 ; RV32IZCMP-SR-LABEL: many_args:
 ; RV32IZCMP-SR:       # %bb.0: # %entry
 ; RV32IZCMP-SR-NEXT:    cm.push {ra, s0-s4}, -32
+; RV32IZCMP-SR-NEXT:    .cfi_def_cfa_offset 32
+; RV32IZCMP-SR-NEXT:    .cfi_offset s0, -20
+; RV32IZCMP-SR-NEXT:    .cfi_offset s1, -16
+; RV32IZCMP-SR-NEXT:    .cfi_offset s2, -12
+; RV32IZCMP-SR-NEXT:    .cfi_offset s3, -8
+; RV32IZCMP-SR-NEXT:    .cfi_offset s4, -4
 ; RV32IZCMP-SR-NEXT:    lui a0, %hi(var0)
 ; RV32IZCMP-SR-NEXT:    lw a6, %lo(var0)(a0)
 ; RV32IZCMP-SR-NEXT:    lw a7, %lo(var0+4)(a0)
@@ -1392,6 +1422,12 @@ define void @many_args(i32, i32, i32, i32, i32, i32, i32, i32, i32) nounwind {
 ; RV64IZCMP-SR-LABEL: many_args:
 ; RV64IZCMP-SR:       # %bb.0: # %entry
 ; RV64IZCMP-SR-NEXT:    cm.push {ra, s0-s4}, -48
+; RV64IZCMP-SR-NEXT:    .cfi_def_cfa_offset 48
+; RV64IZCMP-SR-NEXT:    .cfi_offset s0, -40
+; RV64IZCMP-SR-NEXT:    .cfi_offset s1, -32
+; RV64IZCMP-SR-NEXT:    .cfi_offset s2, -24
+; RV64IZCMP-SR-NEXT:    .cfi_offset s3, -16
+; RV64IZCMP-SR-NEXT:    .cfi_offset s4, -8
 ; RV64IZCMP-SR-NEXT:    lui a0, %hi(var0)
 ; RV64IZCMP-SR-NEXT:    lw a6, %lo(var0)(a0)
 ; RV64IZCMP-SR-NEXT:    lw a7, %lo(var0+4)(a0)
@@ -1435,11 +1471,17 @@ define void @many_args(i32, i32, i32, i32, i32, i32, i32, i32, i32) nounwind {
 ; RV32I-LABEL: many_args:
 ; RV32I:       # %bb.0: # %entry
 ; RV32I-NEXT:    addi sp, sp, -32
+; RV32I-NEXT:    .cfi_def_cfa_offset 32
 ; RV32I-NEXT:    sw s0, 28(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    sw s1, 24(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    sw s2, 20(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    sw s3, 16(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    sw s4, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    .cfi_offset s0, -4
+; RV32I-NEXT:    .cfi_offset s1, -8
+; RV32I-NEXT:    .cfi_offset s2, -12
+; RV32I-NEXT:    .cfi_offset s3, -16
+; RV32I-NEXT:    .cfi_offset s4, -20
 ; RV32I-NEXT:    lui a0, %hi(var0)
 ; RV32I-NEXT:    lw a1, %lo(var0)(a0)
 ; RV32I-NEXT:    lw a2, %lo(var0+4)(a0)
@@ -1483,17 +1525,29 @@ define void @many_args(i32, i32, i32, i32, i32, i32, i32, i32, i32) nounwind {
 ; RV32I-NEXT:    lw s2, 20(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    lw s3, 16(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    lw s4, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    .cfi_restore s0
+; RV32I-NEXT:    .cfi_restore s1
+; RV32I-NEXT:    .cfi_restore s2
+; RV32I-NEXT:    .cfi_restore s3
+; RV32I-NEXT:    .cfi_restore s4
 ; RV32I-NEXT:    addi sp, sp, 32
+; RV32I-NEXT:    .cfi_def_cfa_offset 0
 ; RV32I-NEXT:    ret
 ;
 ; RV64I-LABEL: many_args:
 ; RV64I:       # %bb.0: # %entry
 ; RV64I-NEXT:    addi sp, sp, -48
+; RV64I-NEXT:    .cfi_def_cfa_offset 48
 ; RV64I-NEXT:    sd s0, 40(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    sd s1, 32(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    sd s2, 24(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    sd s3, 16(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    sd s4, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    .cfi_offset s0, -8
+; RV64I-NEXT:    .cfi_offset s1, -16
+; RV64I-NEXT:    .cfi_offset s2, -24
+; RV64I-NEXT:    .cfi_offset s3, -32
+; RV64I-NEXT:    .cfi_offset s4, -40
 ; RV64I-NEXT:    lui a0, %hi(var0)
 ; RV64I-NEXT:    lw a1, %lo(var0)(a0)
 ; RV64I-NEXT:    lw a2, %lo(var0+4)(a0)
@@ -1537,7 +1591,13 @@ define void @many_args(i32, i32, i32, i32, i32, i32, i32, i32, i32) nounwind {
 ; RV64I-NEXT:    ld s2, 24(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    ld s3, 16(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    ld s4, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    .cfi_restore s0
+; RV64I-NEXT:    .cfi_restore s1
+; RV64I-NEXT:    .cfi_restore s2
+; RV64I-NEXT:    .cfi_restore s3
+; RV64I-NEXT:    .cfi_restore s4
 ; RV64I-NEXT:    addi sp, sp, 48
+; RV64I-NEXT:    .cfi_def_cfa_offset 0
 ; RV64I-NEXT:    ret
 entry:
   %val = load [18 x i32], ptr @var0
@@ -1551,11 +1611,16 @@ declare ptr @llvm.stacksave()
 declare void @llvm.stackrestore(ptr)
 declare void @notdead(ptr)
 
-define void @alloca(i32 %n) nounwind {
+define void @alloca(i32 %n) {
 ; RV32IZCMP-LABEL: alloca:
 ; RV32IZCMP:       # %bb.0:
 ; RV32IZCMP-NEXT:    cm.push {ra, s0-s1}, -16
+; RV32IZCMP-NEXT:    .cfi_def_cfa_offset 16
+; RV32IZCMP-NEXT:    .cfi_offset ra, -12
+; RV32IZCMP-NEXT:    .cfi_offset s0, -8
+; RV32IZCMP-NEXT:    .cfi_offset s1, -4
 ; RV32IZCMP-NEXT:    addi s0, sp, 16
+; RV32IZCMP-NEXT:    .cfi_def_cfa s0, 0
 ; RV32IZCMP-NEXT:    mv s1, sp
 ; RV32IZCMP-NEXT:    addi a0, a0, 15
 ; RV32IZCMP-NEXT:    andi a0, a0, -16
@@ -1564,12 +1629,18 @@ define void @alloca(i32 %n) nounwind {
 ; RV32IZCMP-NEXT:    call notdead
 ; RV32IZCMP-NEXT:    mv sp, s1
 ; RV32IZCMP-NEXT:    addi sp, s0, -16
+; RV32IZCMP-NEXT:    .cfi_def_cfa sp, 16
 ; RV32IZCMP-NEXT:    cm.popret {ra, s0-s1}, 16
 ;
 ; RV64IZCMP-LABEL: alloca:
 ; RV64IZCMP:       # %bb.0:
 ; RV64IZCMP-NEXT:    cm.push {ra, s0-s1}, -32
+; RV64IZCMP-NEXT:    .cfi_def_cfa_offset 32
+; RV64IZCMP-NEXT:    .cfi_offset ra, -24
+; RV64IZCMP-NEXT:    .cfi_offset s0, -16
+; RV64IZCMP-NEXT:    .cfi_offset s1, -8
 ; RV64IZCMP-NEXT:    addi s0, sp, 32
+; RV64IZCMP-NEXT:    .cfi_def_cfa s0, 0
 ; RV64IZCMP-NEXT:    mv s1, sp
 ; RV64IZCMP-NEXT:    slli a0, a0, 32
 ; RV64IZCMP-NEXT:    srli a0, a0, 32
@@ -1580,12 +1651,18 @@ define void @alloca(i32 %n) nounwind {
 ; RV64IZCMP-NEXT:    call notdead
 ; RV64IZCMP-NEXT:    mv sp, s1
 ; RV64IZCMP-NEXT:    addi sp, s0, -32
+; RV64IZCMP-NEXT:    .cfi_def_cfa sp, 32
 ; RV64IZCMP-NEXT:    cm.popret {ra, s0-s1}, 32
 ;
 ; RV32IZCMP-SR-LABEL: alloca:
 ; RV32IZCMP-SR:       # %bb.0:
 ; RV32IZCMP-SR-NEXT:    cm.push {ra, s0-s1}, -16
+; RV32IZCMP-SR-NEXT:    .cfi_def_cfa_offset 16
+; RV32IZCMP-SR-NEXT:    .cfi_offset ra, -12
+; RV32IZCMP-SR-NEXT:    .cfi_offset s0, -8
+; RV32IZCMP-SR-NEXT:    .cfi_offset s1, -4
 ; RV32IZCMP-SR-NEXT:    addi s0, sp, 16
+; RV32IZCMP-SR-NEXT:    .cfi_def_cfa s0, 0
 ; RV32IZCMP-SR-NEXT:    mv s1, sp
 ; RV32IZCMP-SR-NEXT:    addi a0, a0, 15
 ; RV32IZCMP-SR-NEXT:    andi a0, a0, -16
@@ -1594,12 +1671,18 @@ define void @alloca(i32 %n) nounwind {
 ; RV32IZCMP-SR-NEXT:    call notdead
 ; RV32IZCMP-SR-NEXT:    mv sp, s1
 ; RV32IZCMP-SR-NEXT:    addi sp, s0, -16
+; RV32IZCMP-SR-NEXT:    .cfi_def_cfa sp, 16
 ; RV32IZCMP-SR-NEXT:    cm.popret {ra, s0-s1}, 16
 ;
 ; RV64IZCMP-SR-LABEL: alloca:
 ; RV64IZCMP-SR:       # %bb.0:
 ; RV64IZCMP-SR-NEXT:    cm.push {ra, s0-s1}, -32
+; RV64IZCMP-SR-NEXT:    .cfi_def_cfa_offset 32
+; RV64IZCMP-SR-NEXT:    .cfi_offset ra, -24
+; RV64IZCMP-SR-NEXT:    .cfi_offset s0, -16
+; RV64IZCMP-SR-NEXT:    .cfi_offset s1, -8
 ; RV64IZCMP-SR-NEXT:    addi s0, sp, 32
+; RV64IZCMP-SR-NEXT:    .cfi_def_cfa s0, 0
 ; RV64IZCMP-SR-NEXT:    mv s1, sp
 ; RV64IZCMP-SR-NEXT:    slli a0, a0, 32
 ; RV64IZCMP-SR-NEXT:    srli a0, a0, 32
@@ -1610,15 +1693,21 @@ define void @alloca(i32 %n) nounwind {
 ; RV64IZCMP-SR-NEXT:    call notdead
 ; RV64IZCMP-SR-NEXT:    mv sp, s1
 ; RV64IZCMP-SR-NEXT:    addi sp, s0, -32
+; RV64IZCMP-SR-NEXT:    .cfi_def_cfa sp, 32
 ; RV64IZCMP-SR-NEXT:    cm.popret {ra, s0-s1}, 32
 ;
 ; RV32I-LABEL: alloca:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
+; RV32I-NEXT:    .cfi_def_cfa_offset 16
 ; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    sw s0, 8(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    sw s1, 4(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    .cfi_offset ra, -4
+; RV32I-NEXT:    .cfi_offset s0, -8
+; RV32I-NEXT:    .cfi_offset s1, -12
 ; RV32I-NEXT:    addi s0, sp, 16
+; RV32I-NEXT:    .cfi_def_cfa s0, 0
 ; RV32I-NEXT:    mv s1, sp
 ; RV32I-NEXT:    addi a0, a0, 15
 ; RV32I-NEXT:    andi a0, a0, -16
@@ -1627,19 +1716,29 @@ define void @alloca(i32 %n) nounwind {
 ; RV32I-NEXT:    call notdead
 ; RV32I-NEXT:    mv sp, s1
 ; RV32I-NEXT:    addi sp, s0, -16
+; RV32I-NEXT:    .cfi_def_cfa sp, 16
 ; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    lw s0, 8(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    lw s1, 4(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    .cfi_restore ra
+; RV32I-NEXT:    .cfi_restore s0
+; RV32I-NEXT:    .cfi_restore s1
 ; RV32I-NEXT:    addi sp, sp, 16
+; RV32I-NEXT:    .cfi_def_cfa_offset 0
 ; RV32I-NEXT:    ret
 ;
 ; RV64I-LABEL: alloca:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -32
+; RV64I-NEXT:    .cfi_def_cfa_offset 32
 ; RV64I-NEXT:    sd ra, 24(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    sd s0, 16(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    sd s1, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    .cfi_offset ra, -8
+; RV64I-NEXT:    .cfi_offset s0, -16
+; RV64I-NEXT:    .cfi_offset s1, -24
 ; RV64I-NEXT:    addi s0, sp, 32
+; RV64I-NEXT:    .cfi_def_cfa s0, 0
 ; RV64I-NEXT:    mv s1, sp
 ; RV64I-NEXT:    slli a0, a0, 32
 ; RV64I-NEXT:    srli a0, a0, 32
@@ -1650,10 +1749,15 @@ define void @alloca(i32 %n) nounwind {
 ; RV64I-NEXT:    call notdead
 ; RV64I-NEXT:    mv sp, s1
 ; RV64I-NEXT:    addi sp, s0, -32
+; RV64I-NEXT:    .cfi_def_cfa sp, 32
 ; RV64I-NEXT:    ld ra, 24(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    ld s0, 16(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    ld s1, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    .cfi_restore ra
+; RV64I-NEXT:    .cfi_restore s0
+; RV64I-NEXT:    .cfi_restore s1
 ; RV64I-NEXT:    addi sp, sp, 32
+; RV64I-NEXT:    .cfi_def_cfa_offset 0
 ; RV64I-NEXT:    ret
   %sp = call ptr @llvm.stacksave()
   %addr = alloca i8, i32 %n
@@ -1665,166 +1769,303 @@ define void @alloca(i32 %n) nounwind {
 declare i32 @foo_test_irq(...)
 @var_test_irq = global [32 x i32] zeroinitializer
 
-define void @foo_with_irq() nounwind "interrupt"="user" {
+define void @foo_with_irq() "interrupt"="user" {
 ; RV32IZCMP-LABEL: foo_with_irq:
 ; RV32IZCMP:       # %bb.0:
 ; RV32IZCMP-NEXT:    cm.push {ra}, -64
-; RV32IZCMP-NEXT:    addi sp, sp, -16
-; RV32IZCMP-NEXT:    sw t0, 60(sp) # 4-byte Folded Spill
-; RV32IZCMP-NEXT:    sw t1, 56(sp) # 4-byte Folded Spill
-; RV32IZCMP-NEXT:    sw t2, 52(sp) # 4-byte Folded Spill
-; RV32IZCMP-NEXT:    sw a0, 48(sp) # 4-byte Folded Spill
-; RV32IZCMP-NEXT:    sw a1, 44(sp) # 4-byte Folded Spill
-; RV32IZCMP-NEXT:    sw a2, 40(sp) # 4-byte Folded Spill
-; RV32IZCMP-NEXT:    sw a3, 36(sp) # 4-byte Folded Spill
-; RV32IZCMP-NEXT:    sw a4, 32(sp) # 4-byte Folded Spill
-; RV32IZCMP-NEXT:    sw a5, 28(sp) # 4-byte Folded Spill
-; RV32IZCMP-NEXT:    sw a6, 24(sp) # 4-byte Folded Spill
-; RV32IZCMP-NEXT:    sw a7, 20(sp) # 4-byte Folded Spill
-; RV32IZCMP-NEXT:    sw t3, 16(sp) # 4-byte Folded Spill
-; RV32IZCMP-NEXT:    sw t4, 12(sp) # 4-byte Folded Spill
-; RV32IZCMP-NEXT:    sw t5, 8(sp) # 4-byte Folded Spill
-; RV32IZCMP-NEXT:    sw t6, 4(sp) # 4-byte Folded Spill
+; RV32IZCMP-NEXT:    .cfi_def_cfa_offset 64
+; RV32IZCMP-NEXT:    .cfi_offset ra, -4
+; RV32IZCMP-NEXT:    sw t0, 56(sp) # 4-byte Folded Spill
+; RV32IZCMP-NEXT:    sw t1, 52(sp) # 4-byte Folded Spill
+; RV32IZCMP-NEXT:    sw t2, 48(sp) # 4-byte Folded Spill
+; RV32IZCMP-NEXT:    sw a0, 44(sp) # 4-byte Folded Spill
+; RV32IZCMP-NEXT:    sw a1, 40(sp) # 4-byte Folded Spill
+; RV32IZCMP-NEXT:    sw a2, 36(sp) # 4-byte Folded Spill
+; RV32IZCMP-NEXT:    sw a3, 32(sp) # 4-byte Folded Spill
+; RV32IZCMP-NEXT:    sw a4, 28(sp) # 4-byte Folded Spill
+; RV32IZCMP-NEXT:    sw a5, 24(sp) # 4-byte Folded Spill
+; RV32IZCMP-NEXT:    sw a6, 20(sp) # 4-byte Folded Spill
+; RV32IZCMP-NEXT:    sw a7, 16(sp) # 4-byte Folded Spill
+; RV32IZCMP-NEXT:    sw t3, 12(sp) # 4-byte Folded Spill
+; RV32IZCMP-NEXT:    sw t4, 8(sp) # 4-byte Folded Spill
+; RV32IZCMP-NEXT:    sw t5, 4(sp) # 4-byte Folded Spill
+; RV32IZCMP-NEXT:    sw t6, 0(sp) # 4-byte Folded Spill
+; RV32IZCMP-NEXT:    .cfi_offset t0, -8
+; RV32IZCMP-NEXT:    .cfi_offset t1, -12
+; RV32IZCMP-NEXT:    .cfi_offset t2, -16
+; RV32IZCMP-NEXT:    .cfi_offset a0, -20
+; RV32IZCMP-NEXT:    .cfi_offset a1, -24
+; RV32IZCMP-NEXT:    .cfi_offset a2, -28
+; RV32IZCMP-NEXT:    .cfi_offset a3, -32
+; RV32IZCMP-NEXT:    .cfi_offset a4, -36
+; RV32IZCMP-NEXT:    .cfi_offset a5, -40
+; RV32IZCMP-NEXT:    .cfi_offset a6, -44
+; RV32IZCMP-NEXT:    .cfi_offset a7, -48
+; RV32IZCMP-NEXT:    .cfi_offset t3, -52
+; RV32IZCMP-NEXT:    .cfi_offset t4, -56
+; RV32IZCMP-NEXT:    .cfi_offset t5, -60
+; RV32IZCMP-NEXT:    .cfi_offset t6, -64
 ; RV32IZCMP-NEXT:    call foo_test_irq
-; RV32IZCMP-NEXT:    lw t0, 60(sp) # 4-byte Folded Reload
-; RV32IZCMP-NEXT:    lw t1, 56(sp) # 4-byte Folded Reload
-; RV32IZCMP-NEXT:    lw t2, 52(sp) # 4-byte Folded Reload
-; RV32IZCMP-NEXT:    lw a0, 48(sp) # 4-byte Folded Reload
-; RV32IZCMP-NEXT:    lw a1, 44(sp) # 4-byte Folded Reload
-; RV32IZCMP-NEXT:    lw a2, 40(sp) # 4-byte Folded Reload
-; RV32IZCMP-NEXT:    lw a3, 36(sp) # 4-byte Folded Reload
-; RV32IZCMP-NEXT:    lw a4, 32(sp) # 4-byte Folded Reload
-; RV32IZCMP-NEXT:    lw a5, 28(sp) # 4-byte Folded Reload
-; RV32IZCMP-NEXT:    lw a6, 24(sp) # 4-byte Folded Reload
-; RV32IZCMP-NEXT:    lw a7, 20(sp) # 4-byte Folded Reload
-; RV32IZCMP-NEXT:    lw t3, 16(sp) # 4-byte Folded Reload
-; RV32IZCMP-NEXT:    lw t4, 12(sp) # 4-byte Folded Reload
-; RV32IZCMP-NEXT:    lw t5, 8(sp) # 4-byte Folded Reload
-; RV32IZCMP-NEXT:    lw t6, 4(sp) # 4-byte Folded Reload
-; RV32IZCMP-NEXT:    addi sp, sp, 16
+; RV32IZCMP-NEXT:    lw t0, 56(sp) # 4-byte Folded Reload
+; RV32IZCMP-NEXT:    lw t1, 52(sp) # 4-byte Folded Reload
+; RV32IZCMP-NEXT:    lw t2, 48(sp) # 4-byte Folded Reload
+; RV32IZCMP-NEXT:    lw a0, 44(sp) # 4-byte Folded Reload
+; RV32IZCMP-NEXT:    lw a1, 40(sp) # 4-byte Folded Reload
+; RV32IZCMP-NEXT:    lw a2, 36(sp) # 4-byte Folded Reload
+; RV32IZCMP-NEXT:    lw a3, 32(sp) # 4-byte Folded Reload
+; RV32IZCMP-NEXT:    lw a4, 28(sp) # 4-byte Folded Reload
+; RV32IZCMP-NEXT:    lw a5, 24(sp) # 4-byte Folded Reload
+; RV32IZCMP-NEXT:    lw a6, 20(sp) # 4-byte Folded Reload
+; RV32IZCMP-NEXT:    lw a7, 16(sp) # 4-byte Folded Reload
+; RV32IZCMP-NEXT:    lw t3, 12(sp) # 4-byte Folded Reload
+; RV32IZCMP-NEXT:    lw t4, 8(sp) # 4-byte Folded Reload
+; RV32IZCMP-NEXT:    lw t5, 4(sp) # 4-byte Folded Reload
+; RV32IZCMP-NEXT:    lw t6, 0(sp) # 4-byte Folded Reload
+; RV32IZCMP-NEXT:    .cfi_restore t0
+; RV32IZCMP-NEXT:    .cfi_restore t1
+; RV32IZCMP-NEXT:    .cfi_restore t2
+; RV32IZCMP-NEXT:    .cfi_restore a0
+; RV32IZCMP-NEXT:    .cfi_restore a1
+; RV32IZCMP-NEXT:    .cfi_restore a2
+; RV32IZCMP-NEXT:    .cfi_restore a3
+; RV32IZCMP-NEXT:    .cfi_restore a4
+; RV32IZCMP-NEXT:    .cfi_restore a5
+; RV32IZCMP-NEXT:    .cfi_restore a6
+; RV32IZCMP-NEXT:    .cfi_restore a7
+; RV32IZCMP-NEXT:    .cfi_restore t3
+; RV32IZCMP-NEXT:    .cfi_restore t4
+; RV32IZCMP-NEXT:    .cfi_restore t5
+; RV32IZCMP-NEXT:    .cfi_restore t6
 ; RV32IZCMP-NEXT:    cm.pop {ra}, 64
+; RV32IZCMP-NEXT:    .cfi_restore ra
+; RV32IZCMP-NEXT:    .cfi_def_cfa_offset 0
 ; RV32IZCMP-NEXT:    mret
 ;
 ; RV64IZCMP-LABEL: foo_with_irq:
 ; RV64IZCMP:       # %bb.0:
 ; RV64IZCMP-NEXT:    cm.push {ra}, -64
-; RV64IZCMP-NEXT:    addi sp, sp, -80
-; RV64IZCMP-NEXT:    sd t0, 120(sp) # 8-byte Folded Spill
-; RV64IZCMP-NEXT:    sd t1, 112(sp) # 8-byte Folded Spill
-; RV64IZCMP-NEXT:    sd t2, 104(sp) # 8-byte Folded Spill
-; RV64IZCMP-NEXT:    sd a0, 96(sp) # 8-byte Folded Spill
-; RV64IZCMP-NEXT:    sd a1, 88(sp) # 8-byte Folded Spill
-; RV64IZCMP-NEXT:    sd a2, 80(sp) # 8-byte Folded Spill
-; RV64IZCMP-NEXT:    sd a3, 72(sp) # 8-byte Folded Spill
-; RV64IZCMP-NEXT:    sd a4, 64(sp) # 8-byte Folded Spill
-; RV64IZCMP-NEXT:    sd a5, 56(sp) # 8-byte Folded Spill
-; RV64IZCMP-NEXT:    sd a6, 48(sp) # 8-byte Folded Spill
-; RV64IZCMP-NEXT:    sd a7, 40(sp) # 8-byte Folded Spill
-; RV64IZCMP-NEXT:    sd t3, 32(sp) # 8-byte Folded Spill
-; RV64IZCMP-NEXT:    sd t4, 24(sp) # 8-byte Folded Spill
-; RV64IZCMP-NEXT:    sd t5, 16(sp) # 8-byte Folded Spill
-; RV64IZCMP-NEXT:    sd t6, 8(sp) # 8-byte Folded Spill
+; RV64IZCMP-NEXT:    .cfi_def_cfa_offset 64
+; RV64IZCMP-NEXT:    .cfi_offset ra, -8
+; RV64IZCMP-NEXT:    addi sp, sp, -64
+; RV64IZCMP-NEXT:    .cfi_def_cfa_offset 128
+; RV64IZCMP-NEXT:    sd t0, 112(sp) # 8-byte Folded Spill
+; RV64IZCMP-NEXT:    sd t1, 104(sp) # 8-byte Folded Spill
+; RV64IZCMP-NEXT:    sd t2, 96(sp) # 8-byte Folded Spill
+; RV64IZCMP-NEXT:    sd a0, 88(sp) # 8-byte Folded Spill
+; RV64IZCMP-NEXT:    sd a1, 80(sp) # 8-byte Folded Spill
+; RV64IZCMP-NEXT:    sd a2, 72(sp) # 8-byte Folded Spill
+; RV64IZCMP-NEXT:    sd a3, 64(sp) # 8-byte Folded Spill
+; RV64IZCMP-NEXT:    sd a4, 56(sp) # 8-byte Folded Spill
+; RV64IZCMP-NEXT:    sd a5, 48(sp) # 8-byte Folded Spill
+; RV64IZCMP-NEXT:    sd a6, 40(sp) # 8-byte Folded Spill
+; RV64IZCMP-NEXT:    sd a7, 32(sp) # 8-byte Folded Spill
+; RV64IZCMP-NEXT:    sd t3, 24(sp) # 8-byte Folded Spill
+; RV64IZCMP-NEXT:    sd t4, 16(sp) # 8-byte Folded Spill
+; RV64IZCMP-NEXT:    sd t5, 8(sp) # 8-byte Folded Spill
+; RV64IZCMP-NEXT:    sd t6, 0(sp) # 8-byte Folded Spill
+; RV64IZCMP-NEXT:    .cfi_offset t0, -16
+; RV64IZCMP-NEXT:    .cfi_offset t1, -24
+; RV64IZCMP-NEXT:    .cfi_offset t2, -32
+; RV64IZCMP-NEXT:    .cfi_offset a0, -40
+; RV64IZCMP-NEXT:    .cfi_offset a1, -48
+; RV64IZCMP-NEXT:    .cfi_offset a2, -56
+; RV64IZCMP-NEXT:    .cfi_offset a3, -64
+; RV64IZCMP-NEXT:    .cfi_offset a4, -72
+; RV64IZCMP-NEXT:    .cfi_offset a5, -80
+; RV64IZCMP-NEXT:    .cfi_offset a6, -88
+; RV64IZCMP-NEXT:    .cfi_offset a7, -96
+; RV64IZCMP-NEXT:    .cfi_offset t3, -104
+; RV64IZCMP-NEXT:    .cfi_offset t4, -112
+; RV64IZCMP-NEXT:    .cfi_offset t5, -120
+; RV64IZCMP-NEXT:    .cfi_offset t6, -128
 ; RV64IZCMP-NEXT:    call foo_test_irq
-; RV64IZCMP-NEXT:    ld t0, 120(sp) # 8-byte Folded Reload
-; RV64IZCMP-NEXT:    ld t1, 112(sp) # 8-byte Folded Reload
-; RV64IZCMP-NEXT:    ld t2, 104(sp) # 8-byte Folded Reload
-; RV64IZCMP-NEXT:    ld a0, 96(sp) # 8-byte Folded Reload
-; RV64IZCMP-NEXT:    ld a1, 88(sp) # 8-byte Folded Reload
-; RV64IZCMP-NEXT:    ld a2, 80(sp) # 8-byte Folded Reload
-; RV64IZCMP-NEXT:    ld a3, 72(sp) # 8-byte Folded Reload
-; RV64IZCMP-NEXT:    ld a4, 64(sp) # 8-byte Folded Reload
-; RV64IZCMP-NEXT:    ld a5, 56(sp) # 8-byte Folded Reload
-; RV64IZCMP-NEXT:    ld a6, 48(sp) # 8-byte Folded Reload
-; RV64IZCMP-NEXT:    ld a7, 40(sp) # 8-byte Folded Reload
-; RV64IZCMP-NEXT:    ld t3, 32(sp) # 8-byte Folded Reload
-; RV64IZCMP-NEXT:    ld t4, 24(sp) # 8-byte Folded Reload
-; RV64IZCMP-NEXT:    ld t5, 16(sp) # 8-byte Folded Reload
-; RV64IZCMP-NEXT:    ld t6, 8(sp) # 8-byte Folded Reload
-; RV64IZCMP-NEXT:    addi sp, sp, 80
+; RV64IZCMP-NEXT:    ld t0, 112(sp) # 8-byte Folded Reload
+; RV64IZCMP-NEXT:    ld t1, 104(sp) # 8-byte Folded Reload
+; RV64IZCMP-NEXT:    ld t2, 96(sp) # 8-byte Folded Reload
+; RV64IZCMP-NEXT:    ld a0, 88(sp) # 8-byte Folded Reload
+; RV64IZCMP-NEXT:    ld a1, 80(sp) # 8-byte Folded Reload
+; RV64IZCMP-NEXT:    ld a2, 72(sp) # 8-byte Folded Reload
+; RV64IZCMP-NEXT:    ld a3, 64(sp) # 8-byte Folded Reload
+; RV64IZCMP-NEXT:    ld a4, 56(sp) # 8-byte Folded Reload
+; RV64IZCMP-NEXT:    ld a5, 48(sp) # 8-byte Folded Reload
+; RV64IZCMP-NEXT:    ld a6, 40(sp) # 8-byte Folded Reload
+; RV64IZCMP-NEXT:    ld a7, 32(sp) # 8-byte Folded Reload
+; RV64IZCMP-NEXT:    ld t3, 24(sp) # 8-byte Folded Reload
+; RV64IZCMP-NEXT:    ld t4, 16(sp) # 8-byte Folded Reload
+; RV64IZCMP-NEXT:    ld t5, 8(sp) # 8-byte Folded Reload
+; RV64IZCMP-NEXT:    ld t6, 0(sp) # 8-byte Folded Reload
+; RV64IZCMP-NEXT:    .cfi_restore t0
+; RV64IZCMP-NEXT:    .cfi_restore t1
+; RV64IZCMP-NEXT:    .cfi_restore t2
+; RV64IZCMP-NEXT:    .cfi_restore a0
+; RV64IZCMP-NEXT:    .cfi_restore a1
+; RV64IZCMP-NEXT:    .cfi_restore a2
+; RV64IZCMP-NEXT:    .cfi_restore a3
+; RV64IZCMP-NEXT:    .cfi_restore a4
+; RV64IZCMP-NEXT:    .cfi_restore a5
+; RV64IZCMP-NEXT:    .cfi_restore a6
+; RV64IZCMP-NEXT:    .cfi_restore a7
+; RV64IZCMP-NEXT:    .cfi_restore t3
+; RV64IZCMP-NEXT:    .cfi_restore t4
+; RV64IZCMP-NEXT:    .cfi_restore t5
+; RV64IZCMP-NEXT:    .cfi_restore t6
+; RV64IZCMP-NEXT:    addi sp, sp, 64
+; RV64IZCMP-NEXT:    .cfi_def_cfa_offset 64
 ; RV64IZCMP-NEXT:    cm.pop {ra}, 64
+; RV64IZCMP-NEXT:    .cfi_restore ra
+; RV64IZCMP-NEXT:    .cfi_def_cfa_offset 0
 ; RV64IZCMP-NEXT:    mret
 ;
 ; RV32IZCMP-SR-LABEL: foo_with_irq:
 ; RV32IZCMP-SR:       # %bb.0:
 ; RV32IZCMP-SR-NEXT:    cm.push {ra}, -64
-; RV32IZCMP-SR-NEXT:    addi sp, sp, -16
-; RV32IZCMP-SR-NEXT:    sw t0, 60(sp) # 4-byte Folded Spill
-; RV32IZCMP-SR-NEXT:    sw t1, 56(sp) # 4-byte Folded Spill
-; RV32IZCMP-SR-NEXT:    sw t2, 52(sp) # 4-byte Folded Spill
-; RV32IZCMP-SR-NEXT:    sw a0, 48(sp) # 4-byte Folded Spill
-; RV32IZCMP-SR-NEXT:    sw a1, 44(sp) # 4-byte Folded Spill
-; RV32IZCMP-SR-NEXT:    sw a2, 40(sp) # 4-byte Folded Spill
-; RV32IZCMP-SR-NEXT:    sw a3, 36(sp) # 4-byte Folded Spill
-; RV32IZCMP-SR-NEXT:    sw a4, 32(sp) # 4-byte Folded Spill
-; RV32IZCMP-SR-NEXT:    sw a5, 28(sp) # 4-byte Folded Spill
-; RV32IZCMP-SR-NEXT:    sw a6, 24(sp) # 4-byte Folded Spill
-; RV32IZCMP-SR-NEXT:    sw a7, 20(sp) # 4-byte Folded Spill
-; RV32IZCMP-SR-NEXT:    sw t3, 16(sp) # 4-byte Folded Spill
-; RV32IZCMP-SR-NEXT:    sw t4, 12(sp) # 4-byte Folded Spill
-; RV32IZCMP-SR-NEXT:    sw t5, 8(sp) # 4-byte Folded Spill
-; RV32IZCMP-SR-NEXT:    sw t6, 4(sp) # 4-byte Folded Spill
+; RV32IZCMP-SR-NEXT:    .cfi_def_cfa_offset 64
+; RV32IZCMP-SR-NEXT:    .cfi_offset ra, -4
+; RV32IZCMP-SR-NEXT:    sw t0, 56(sp) # 4-byte Folded Spill
+; RV32IZCMP-SR-NEXT:    sw t1, 52(sp) # 4-byte Folded Spill
+; RV32IZCMP-SR-NEXT:    sw t2, 48(sp) # 4-byte Folded Spill
+; RV32IZCMP-SR-NEXT:    sw a0, 44(sp) # 4-byte Folded Spill
+; RV32IZCMP-SR-NEXT:    sw a1, 40(sp) # 4-byte Folded Spill
+; RV32IZCMP-SR-NEXT:    sw a2, 36(sp) # 4-byte Folded Spill
+; RV32IZCMP-SR-NEXT:    sw a3, 32(sp) # 4-byte Folded Spill
+; RV32IZCMP-SR-NEXT:    sw a4, 28(sp) # 4-byte Folded Spill
+; RV32IZCMP-SR-NEXT:    sw a5, 24(sp) # 4-byte Folded Spill
+; RV32IZCMP-SR-NEXT:    sw a6, 20(sp) # 4-byte Folded Spill
+; RV32IZCMP-SR-NEXT:    sw a7, 16(sp) # 4-byte Folded Spill
+; RV32IZCMP-SR-NEXT:    sw t3, 12(sp) # 4-byte Folded Spill
+; RV32IZCMP-SR-NEXT:    sw t4, 8(sp) # 4-byte Folded Spill
+; RV32IZCMP-SR-NEXT:    sw t5, 4(sp) # 4-byte Folded Spill
+; RV32IZCMP-SR-NEXT:    sw t6, 0(sp) # 4-byte Folded Spill
+; RV32IZCMP-SR-NEXT:    .cfi_offset t0, -8
+; RV32IZCMP-SR-NEXT:    .cfi_offset t1, -12
+; RV32IZCMP-SR-NEXT:    .cfi_offset t2, -16
+; RV32IZCMP-SR-NEXT:    .cfi_offset a0, -20
+; RV32IZCMP-SR-NEXT:    .cfi_offset a1, -24
+; RV32IZCMP-SR-NEXT:    .cfi_offset a2, -28
+; RV32IZCMP-SR-NEXT:    .cfi_offset a3, -32
+; RV32IZCMP-SR-NEXT:    .cfi_offset a4, -36
+; RV32IZCMP-SR-NEXT:    .cfi_offset a5, -40
+; RV32IZCMP-SR-NEXT:    .cfi_offset a6, -44
+; RV32IZCMP-SR-NEXT:    .cfi_offset a7, -48
+; RV32IZCMP-SR-NEXT:    .cfi_offset t3, -52
+; RV32IZCMP-SR-NEXT:    .cfi_offset t4, -56
+; RV32IZCMP-SR-NEXT:    .cfi_offset t5, -60
+; RV32IZCMP-SR-NEXT:    .cfi_offset t6, -64
 ; RV32IZCMP-SR-NEXT:    call foo_test_irq
-; RV32IZCMP-SR-NEXT:    lw t0, 60(sp) # 4-byte Folded Reload
-; RV32IZCMP-SR-NEXT:    lw t1, 56(sp) # 4-byte Folded Reload
-; RV32IZCMP-SR-NEXT:    lw t2, 52(sp) # 4-byte Folded Reload
-; RV32IZCMP-SR-NEXT:    lw a0, 48(sp) # 4-byte Folded Reload
-; RV32IZCMP-SR-NEXT:    lw a1, 44(sp) # 4-byte Folded Reload
-; RV32IZCMP-SR-NEXT:    lw a2, 40(sp) # 4-byte Folded Reload
-; RV32IZCMP-SR-NEXT:    lw a3, 36(sp) # 4-byte Folded Reload
-; RV32IZCMP-SR-NEXT:    lw a4, 32(sp) # 4-byte Folded Reload
-; RV32IZCMP-SR-NEXT:    lw a5, 28(sp) # 4-byte Folded Reload
-; RV32IZCMP-SR-NEXT:    lw a6, 24(sp) # 4-byte Folded Reload
-; RV32IZCMP-SR-NEXT:    lw a7, 20(sp) # 4-byte Folded Reload
-; RV32IZCMP-SR-NEXT:    lw t3, 16(sp) # 4-byte Folded Reload
-; RV32IZCMP-SR-NEXT:    lw t4, 12(sp) # 4-byte Folded Reload
-; RV32IZCMP-SR-NEXT:    lw t5, 8(sp) # 4-byte Folded Reload
-; RV32IZCMP-SR-NEXT:    lw t6, 4(sp) # 4-byte Folded Reload
-; RV32IZCMP-SR-NEXT:    addi sp, sp, 16
+; RV32IZCMP-SR-NEXT:    lw t0, 56(sp) # 4-byte Folded Reload
+; RV32IZCMP-SR-NEXT:    lw t1, 52(sp) # 4-byte Folded Reload
+; RV32IZCMP-SR-NEXT:    lw t2, 48(sp) # 4-byte Folded Reload
+; RV32IZCMP-SR-NEXT:    lw a0, 44(sp) # 4-byte Folded Reload
+; RV32IZCMP-SR-NEXT:    lw a1, 40(sp) # 4-byte Folded Reload
+; RV32IZCMP-SR-NEXT:    lw a2, 36(sp) # 4-byte Folded Reload
+; RV32IZCMP-SR-NEXT:    lw a3, 32(sp) # 4-byte Folded Reload
+; RV32IZCMP-SR-NEXT:    lw a4, 28(sp) # 4-byte Folded Reload
+; RV32IZCMP-SR-NEXT:    lw a5, 24(sp) # 4-byte Folded Reload
+; RV32IZCMP-SR-NEXT:    lw a6, 20(sp) # 4-byte Folded Reload
+; RV32IZCMP-SR-NEXT:    lw a7, 16(sp) # 4-byte Folded Reload
+; RV32IZCMP-SR-NEXT:    lw t3, 12(sp) # 4-byte Folded Reload
+; RV32IZCMP-SR-NEXT:    lw t4, 8(sp) # 4-byte Folded Reload
+; RV32IZCMP-SR-NEXT:    lw t5, 4(sp) # 4-byte Folded Reload
+; RV32IZCMP-SR-NEXT:    lw t6, 0(sp) # 4-byte Folded Reload
+; RV32IZCMP-SR-NEXT:    .cfi_restore t0
+; RV32IZCMP-SR-NEXT:    .cfi_restore t1
+; RV32IZCMP-SR-NEXT:    .cfi_restore t2
+; RV32IZCMP-SR-NEXT:    .cfi_restore a0
+; RV32IZCMP-SR-NEXT:    .cfi_restore a1
+; RV32IZCMP-SR-NEXT:    .cfi_restore a2
+; RV32IZCMP-SR-NEXT:    .cfi_restore a3
+; RV32IZCMP-SR-NEXT:    .cfi_restore a4
+; RV32IZCMP-SR-NEXT:    .cfi_restore a5
+; RV32IZCMP-SR-NEXT:    .cfi_restore a6
+; RV32IZCMP-SR-NEXT:    .cfi_restore a7
+; RV32IZCMP-SR-NEXT:    .cfi_restore t3
+; RV32IZCMP-SR-NEXT:    .cfi_restore t4
+; RV32IZCMP-SR-NEXT:    .cfi_restore t5
+; RV32IZCMP-SR-NEXT:    .cfi_restore t6
 ; RV32IZCMP-SR-NEXT:    cm.pop {ra}, 64
+; RV32IZCMP-SR-NEXT:    .cfi_restore ra
+; RV32IZCMP-SR-NEXT:    .cfi_def_cfa_offset 0
 ; RV32IZCMP-SR-NEXT:    mret
 ;
 ; RV64IZCMP-SR-LABEL: foo_with_irq:
 ; RV64IZCMP-SR:       # %bb.0:
 ; RV64IZCMP-SR-NEXT:    cm.push {ra}, -64
-; RV64IZCMP-SR-NEXT:    addi sp, sp, -80
-; RV64IZCMP-SR-NEXT:    sd t0, 120(sp) # 8-byte Folded Spill
-; RV64IZCMP-SR-NEXT:    sd t1, 112(sp) # 8-byte Folded Spill
-; RV64IZCMP-SR-NEXT:    sd t2, 104(sp) # 8-byte Folded Spill
-; RV64IZCMP-SR-NEXT:    sd a0, 96(sp) # 8-byte Folded Spill
-; RV64IZCMP-SR-NEXT:    sd a1, 88(sp) # 8-byte Folded Spill
-; RV64IZCMP-SR-NEXT:    sd a2, 80(sp) # 8-byte Folded Spill
-; RV64IZCMP-SR-NEXT:    sd a3, 72(sp) # 8-byte Folded Spill
-; RV64IZCMP-SR-NEXT:    sd a4, 64(sp) # 8-byte Folded Spill
-; RV64IZCMP-SR-NEXT:    sd a5, 56(sp) # 8-byte Folded Spill
-; RV64IZCMP-SR-NEXT:    sd a6, 48(sp) # 8-byte Folded Spill
-; RV64IZCMP-SR-NEXT:    sd a7, 40(sp) # 8-byte Folded Spill
-; RV64IZCMP-SR-NEXT:    sd t3, 32(sp) # 8-byte Folded Spill
-; RV64IZCMP-SR-NEXT:    sd t4, 24(sp) # 8-byte Folded Spill
-; RV64IZCMP-SR-NEXT:    sd t5, 16(sp) # 8-byte Folded Spill
-; RV64IZCMP-SR-NEXT:    sd t6, 8(sp) # 8-byte Folded Spill
+; RV64IZCMP-SR-NEXT:    .cfi_def_cfa_offset 64
+; RV64IZCMP-SR-NEXT:    .cfi_offset ra, -8
+; RV64IZCMP-SR-NEXT:    addi sp, sp, -64
+; RV64IZCMP-SR-NEXT:    .cfi_def_cfa_offset 128
+; RV64IZCMP-SR-NEXT:    sd t0, 112(sp) # 8-byte Folded Spill
+; RV64IZCMP-SR-NEXT:    sd t1, 104(sp) # 8-byte Folded Spill
+; RV64IZCMP-SR-NEXT:    sd t2, 96(sp) # 8-byte Folded Spill
+; RV64IZCMP-SR-NEXT:    sd a0, 88(sp) # 8-byte Folded Spill
+; RV64IZCMP-SR-NEXT:    sd a1, 80(sp) # 8-byte Folded Spill
+; RV64IZCMP-SR-NEXT:    sd a2, 72(sp) # 8-byte Folded Spill
+; RV64IZCMP-SR-NEXT:    sd a3, 64(sp) # 8-byte Folded Spill
+; RV64IZCMP-SR-NEXT:    sd a4, 56(sp) # 8-byte Folded Spill
+; RV64IZCMP-SR-NEXT:    sd a5, 48(sp) # 8-byte Folded Spill
+; RV64IZCMP-SR-NEXT:    sd a6, 40(sp) # 8-byte Folded Spill
+; RV64IZCMP-SR-NEXT:    sd a7, 32(sp) # 8-byte Folded Spill
+; RV64IZCMP-SR-NEXT:    sd t3, 24(sp) # 8-byte Folded Spill
+; RV64IZCMP-SR-NEXT:    sd t4, 16(sp) # 8-byte Folded Spill
+; RV64IZCMP-SR-NEXT:    sd t5, 8(sp) # 8-byte Folded Spill
+; RV64IZCMP-SR-NEXT:    sd t6, 0(sp) # 8-byte Folded Spill
+; RV64IZCMP-SR-NEXT:    .cfi_offset t0, -16
+; RV64IZCMP-SR-NEXT:    .cfi_offset t1, -24
+; RV64IZCMP-SR-NEXT:    .cfi_offset t2, -32
+; RV64IZCMP-SR-NEXT:    .cfi_offset a0, -40
+; RV64IZCMP-SR-NEXT:    .cfi_offset a1, -48
+; RV64IZCMP-SR-NEXT:    .cfi_offset a2, -56
+; RV64IZCMP-SR-NEXT:    .cfi_offset a3, -64
+; RV64IZCMP-SR-NEXT:    .cfi_offset a4, -72
+; RV64IZCMP-SR-NEXT:    .cfi_offset a5, -80
+; RV64IZCMP-SR-NEXT:    .cfi_offset a6, -88
+; RV64IZCMP-SR-NEXT:    .cfi_offset a7, -96
+; RV64IZCMP-SR-NEXT:    .cfi_offset t3, -104
+; RV64IZCMP-SR-NEXT:    .cfi_offset t4, -112
+; RV64IZCMP-SR-NEXT:    .cfi_offset t5, -120
+; RV64IZCMP-SR-NEXT:    .cfi_offset t6, -128
 ; RV64IZCMP-SR-NEXT:    call foo_test_irq
-; RV64IZCMP-SR-NEXT:    ld t0, 120(sp) # 8-byte Folded Reload
-; RV64IZCMP-SR-NEXT:    ld t1, 112(sp) # 8-byte Folded Reload
-; RV64IZCMP-SR-NEXT:    ld t2, 104(sp) # 8-byte Folded Reload
-; RV64IZCMP-SR-NEXT:    ld a0, 96(sp) # 8-byte Folded Reload
-; RV64IZCMP-SR-NEXT:    ld a1, 88(sp) # 8-byte Folded Reload
-; RV64IZCMP-SR-NEXT:    ld a2, 80(sp) # 8-byte Folded Reload
-; RV64IZCMP-SR-NEXT:    ld a3, 72(sp) # 8-byte Folded Reload
-; RV64IZCMP-SR-NEXT:    ld a4, 64(sp) # 8-byte Folded Reload
-; RV64IZCMP-SR-NEXT:    ld a5, 56(sp) # 8-byte Folded Reload
-; RV64IZCMP-SR-NEXT:    ld a6, 48(sp) # 8-byte Folded Reload
-; RV64IZCMP-SR-NEXT:    ld a7, 40(sp) # 8-byte Folded Reload
-; RV64IZCMP-SR-NEXT:    ld t3, 32(sp) # 8-byte Folded Reload
-; RV64IZCMP-SR-NEXT:    ld t4, 24(sp) # 8-byte Folded Reload
-; RV64IZCMP-SR-NEXT:    ld t5, 16(sp) # 8-byte Folded Reload
-; RV64IZCMP-SR-NEXT:    ld t6, 8(sp) # 8-byte Folded Reload
-; RV64IZCMP-SR-NEXT:    addi sp, sp, 80
+; RV64IZCMP-SR-NEXT:    ld t0, 112(sp) # 8-byte Folded Reload
+; RV64IZCMP-SR-NEXT:    ld t1, 104(sp) # 8-byte Folded Reload
+; RV64IZCMP-SR-NEXT:    ld t2, 96(sp) # 8-byte Folded Reload
+; RV64IZCMP-SR-NEXT:    ld a0, 88(sp) # 8-byte Folded Reload
+; RV64IZCMP-SR-NEXT:    ld a1, 80(sp) # 8-byte Folded Reload
+; RV64IZCMP-SR-NEXT:    ld a2, 72(sp) # 8-byte Folded Reload
+; RV64IZCMP-SR-NEXT:    ld a3, 64(sp) # 8-byte Folded Reload
+; RV64IZCMP-SR-NEXT:    ld a4, 56(sp) # 8-byte Folded Reload
+; RV64IZCMP-SR-NEXT:    ld a5, 48(sp) # 8-byte Folded Reload
+; RV64IZCMP-SR-NEXT:    ld a6, 40(sp) # 8-byte Folded Reload
+; RV64IZCMP-SR-NEXT:    ld a7, 32(sp) # 8-byte Folded Reload
+; RV64IZCMP-SR-NEXT:    ld t3, 24(sp) # 8-byte Folded Reload
+; RV64IZCMP-SR-NEXT:    ld t4, 16(sp) # 8-byte Folded Reload
+; RV64IZCMP-SR-NEXT:    ld t5, 8(sp) # 8-byte Folded Reload
+; RV64IZCMP-SR-NEXT:    ld t6, 0(sp) # 8-byte Folded Reload
+; RV64IZCMP-SR-NEXT:    .cfi_restore t0
+; RV64IZCMP-SR-NEXT:    .cfi_restore t1
+; RV64IZCMP-SR-NEXT:    .cfi_restore t2
+; RV64IZCMP-SR-NEXT:    .cfi_restore a0
+; RV64IZCMP-SR-NEXT:    .cfi_restore a1
+; RV64IZCMP-SR-NEXT:    .cfi_restore a2
+; RV64IZCMP-SR-NEXT:    .cfi_restore a3
+; RV64IZCMP-SR-NEXT:    .cfi_restore a4
+; RV64IZCMP-SR-NEXT:    .cfi_restore a5
+; RV64IZCMP-SR-NEXT:    .cfi_restore a6
+; RV64IZCMP-SR-NEXT:    .cfi_restore a7
+; RV64IZCMP-SR-NEXT:    .cfi_restore t3
+; RV64IZCMP-SR-NEXT:    .cfi_restore t4
+; RV64IZCMP-SR-NEXT:    .cfi_restore t5
+; RV64IZCMP-SR-NEXT:    .cfi_restore t6
+; RV64IZCMP-SR-NEXT:    addi sp, sp, 64
+; RV64IZCMP-SR-NEXT:    .cfi_def_cfa_offset 64
 ; RV64IZCMP-SR-NEXT:    cm.pop {ra}, 64
+; RV64IZCMP-SR-NEXT:    .cfi_restore ra
+; RV64IZCMP-SR-NEXT:    .cfi_def_cfa_offset 0
 ; RV64IZCMP-SR-NEXT:    mret
 ;
 ; RV32I-LABEL: foo_with_irq:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -64
+; RV32I-NEXT:    .cfi_def_cfa_offset 64
 ; RV32I-NEXT:    sw ra, 60(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    sw t0, 56(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    sw t1, 52(sp) # 4-byte Folded Spill
@@ -1841,6 +2082,22 @@ define void @foo_with_irq() nounwind "interrupt"="user" {
 ; RV32I-NEXT:    sw t4, 8(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    sw t5, 4(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    sw t6, 0(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    .cfi_offset ra, -4
+; RV32I-NEXT:    .cfi_offset t0, -8
+; RV32I-NEXT:    .cfi_offset t1, -12
+; RV32I-NEXT:    .cfi_offset t2, -16
+; RV32I-NEXT:    .cfi_offset a0, -20
+; RV32I-NEXT:    .cfi_offset a1, -24
+; RV32I-NEXT:    .cfi_offset a2, -28
+; RV32I-NEXT:    .cfi_offset a3, -32
+; RV32I-NEXT:    .cfi_offset a4, -36
+; RV32I-NEXT:    .cfi_offset a5, -40
+; RV32I-NEXT:    .cfi_offset a6, -44
+; RV32I-NEXT:    .cfi_offset a7, -48
+; RV32I-NEXT:    .cfi_offset t3, -52
+; RV32I-NEXT:    .cfi_offset t4, -56
+; RV32I-NEXT:    .cfi_offset t5, -60
+; RV32I-NEXT:    .cfi_offset t6, -64
 ; RV32I-NEXT:    call foo_test_irq
 ; RV32I-NEXT:    lw ra, 60(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    lw t0, 56(sp) # 4-byte Folded Reload
@@ -1858,12 +2115,30 @@ define void @foo_with_irq() nounwind "interrupt"="user" {
 ; RV32I-NEXT:    lw t4, 8(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    lw t5, 4(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    lw t6, 0(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    .cfi_restore ra
+; RV32I-NEXT:    .cfi_restore t0
+; RV32I-NEXT:    .cfi_restore t1
+; RV32I-NEXT:    .cfi_restore t2
+; RV32I-NEXT:    .cfi_restore a0
+; RV32I-NEXT:    .cfi_restore a1
+; RV32I-NEXT:    .cfi_restore a2
+; RV32I-NEXT:    .cfi_restore a3
+; RV32I-NEXT:    .cfi_restore a4
+; RV32I-NEXT:    .cfi_restore a5
+; RV32I-NEXT:    .cfi_restore a6
+; RV32I-NEXT:    .cfi_restore a7
+; RV32I-NEXT:    .cfi_restore t3
+; RV32I-NEXT:    .cfi_restore t4
+; RV32I-NEXT:    .cfi_restore t5
+; RV32I-NEXT:    .cfi_restore t6
 ; RV32I-NEXT:    addi sp, sp, 64
+; RV32I-NEXT:    .cfi_def_cfa_offset 0
 ; RV32I-NEXT:    mret
 ;
 ; RV64I-LABEL: foo_with_irq:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -128
+; RV64I-NEXT:    .cfi_def_cfa_offset 128
 ; RV64I-NEXT:    sd ra, 120(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    sd t0, 112(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    sd t1, 104(sp) # 8-byte Folded Spill
@@ -1880,6 +2155,22 @@ define void @foo_with_irq() nounwind "interrupt"="user" {
 ; RV64I-NEXT:    sd t4, 16(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    sd t5, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    sd t6, 0(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    .cfi_offset ra, -8
+; RV64I-NEXT:    .cfi_offset t0, -16
+; RV64I-NEXT:    .cfi_offset t1, -24
+; RV64I-NEXT:    .cfi_offset t2, -32
+; RV64I-NEXT:    .cfi_offset a0, -40
+; RV64I-NEXT:    .cfi_offset a1, -48
+; RV64I-NEXT:    .cfi_offset a2, -56
+; RV64I-NEXT:    .cfi_offset a3, -64
+; RV64I-NEXT:    .cfi_offset a4, -72
+; RV64I-NEXT:    .cfi_offset a5, -80
+; RV64I-NEXT:    .cfi_offset a6, -88
+; RV64I-NEXT:    .cfi_offset a7, -96
+; RV64I-NEXT:    .cfi_offset t3, -104
+; RV64I-NEXT:    .cfi_offset t4, -112
+; RV64I-NEXT:    .cfi_offset t5, -120
+; RV64I-NEXT:    .cfi_offset t6, -128
 ; RV64I-NEXT:    call foo_test_irq
 ; RV64I-NEXT:    ld ra, 120(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    ld t0, 112(sp) # 8-byte Folded Reload
@@ -1897,92 +2188,155 @@ define void @foo_with_irq() nounwind "interrupt"="user" {
 ; RV64I-NEXT:    ld t4, 16(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    ld t5, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    ld t6, 0(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    .cfi_restore ra
+; RV64I-NEXT:    .cfi_restore t0
+; RV64I-NEXT:    .cfi_restore t1
+; RV64I-NEXT:    .cfi_restore t2
+; RV64I-NEXT:    .cfi_restore a0
+; RV64I-NEXT:    .cfi_restore a1
+; RV64I-NEXT:    .cfi_restore a2
+; RV64I-NEXT:    .cfi_restore a3
+; RV64I-NEXT:    .cfi_restore a4
+; RV64I-NEXT:    .cfi_restore a5
+; RV64I-NEXT:    .cfi_restore a6
+; RV64I-NEXT:    .cfi_restore a7
+; RV64I-NEXT:    .cfi_restore t3
+; RV64I-NEXT:    .cfi_restore t4
+; RV64I-NEXT:    .cfi_restore t5
+; RV64I-NEXT:    .cfi_restore t6
 ; RV64I-NEXT:    addi sp, sp, 128
+; RV64I-NEXT:    .cfi_def_cfa_offset 0
 ; RV64I-NEXT:    mret
   %call = call i32 @foo_test_irq()
   ret void
 }
 
-define void @foo_no_irq() nounwind{
+define void @foo_no_irq() {
 ; RV32IZCMP-LABEL: foo_no_irq:
 ; RV32IZCMP:       # %bb.0:
 ; RV32IZCMP-NEXT:    cm.push {ra}, -16
+; RV32IZCMP-NEXT:    .cfi_def_cfa_offset 16
+; RV32IZCMP-NEXT:    .cfi_offset ra, -4
 ; RV32IZCMP-NEXT:    call foo_test_irq
 ; RV32IZCMP-NEXT:    cm.popret {ra}, 16
 ;
 ; RV64IZCMP-LABEL: foo_no_irq:
 ; RV64IZCMP:       # %bb.0:
 ; RV64IZCMP-NEXT:    cm.push {ra}, -16
+; RV64IZCMP-NEXT:    .cfi_def_cfa_offset 16
+; RV64IZCMP-NEXT:    .cfi_offset ra, -8
 ; RV64IZCMP-NEXT:    call foo_test_irq
 ; RV64IZCMP-NEXT:    cm.popret {ra}, 16
 ;
 ; RV32IZCMP-SR-LABEL: foo_no_irq:
 ; RV32IZCMP-SR:       # %bb.0:
 ; RV32IZCMP-SR-NEXT:    cm.push {ra}, -16
+; RV32IZCMP-SR-NEXT:    .cfi_def_cfa_offset 16
+; RV32IZCMP-SR-NEXT:    .cfi_offset ra, -4
 ; RV32IZCMP-SR-NEXT:    call foo_test_irq
 ; RV32IZCMP-SR-NEXT:    cm.popret {ra}, 16
 ;
 ; RV64IZCMP-SR-LABEL: foo_no_irq:
 ; RV64IZCMP-SR:       # %bb.0:
 ; RV64IZCMP-SR-NEXT:    cm.push {ra}, -16
+; RV64IZCMP-SR-NEXT:    .cfi_def_cfa_offset 16
+; RV64IZCMP-SR-NEXT:    .cfi_offset ra, -8
 ; RV64IZCMP-SR-NEXT:    call foo_test_irq
 ; RV64IZCMP-SR-NEXT:    cm.popret {ra}, 16
 ;
 ; RV32I-LABEL: foo_no_irq:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
+; RV32I-NEXT:    .cfi_def_cfa_offset 16
 ; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    .cfi_offset ra, -4
 ; RV32I-NEXT:    call foo_test_irq
 ; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    .cfi_restore ra
 ; RV32I-NEXT:    addi sp, sp, 16
+; RV32I-NEXT:    .cfi_def_cfa_offset 0
 ; RV32I-NEXT:    ret
 ;
 ; RV64I-LABEL: foo_no_irq:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    .cfi_def_cfa_offset 16
 ; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    .cfi_offset ra, -8
 ; RV64I-NEXT:    call foo_test_irq
 ; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    .cfi_restore ra
 ; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    .cfi_def_cfa_offset 0
 ; RV64I-NEXT:    ret
   %call = call i32 @foo_test_irq()
   ret void
 }
 
-define void @callee_with_irq() nounwind "interrupt"="user" {
+define void @callee_with_irq() "interrupt"="user" {
 ; RV32IZCMP-LABEL: callee_with_irq:
 ; RV32IZCMP:       # %bb.0:
 ; RV32IZCMP-NEXT:    cm.push {ra, s0-s11}, -112
-; RV32IZCMP-NEXT:    addi sp, sp, -48
-; RV32IZCMP-NEXT:    sw t0, 92(sp) # 4-byte Folded Spill
-; RV32IZCMP-NEXT:    sw t1, 88(sp) # 4-byte Folded Spill
-; RV32IZCMP-NEXT:    sw t2, 84(sp) # 4-byte Folded Spill
-; RV32IZCMP-NEXT:    sw a0, 80(sp) # 4-byte Folded Spill
-; RV32IZCMP-NEXT:    sw a1, 76(sp) # 4-byte Folded Spill
-; RV32IZCMP-NEXT:    sw a2, 72(sp) # 4-byte Folded Spill
-; RV32IZCMP-NEXT:    sw a3, 68(sp) # 4-byte Folded Spill
-; RV32IZCMP-NEXT:    sw a4, 64(sp) # 4-byte Folded Spill
-; RV32IZCMP-NEXT:    sw a5, 60(sp) # 4-byte Folded Spill
-; RV32IZCMP-NEXT:    sw a6, 56(sp) # 4-byte Folded Spill
-; RV32IZCMP-NEXT:    sw a7, 52(sp) # 4-byte Folded Spill
-; RV32IZCMP-NEXT:    sw t3, 48(sp) # 4-byte Folded Spill
-; RV32IZCMP-NEXT:    sw t4, 44(sp) # 4-byte Folded Spill
-; RV32IZCMP-NEXT:    sw t5, 40(sp) # 4-byte Folded Spill
-; RV32IZCMP-NEXT:    sw t6, 36(sp) # 4-byte Folded Spill
+; RV32IZCMP-NEXT:    .cfi_def_cfa_offset 112
+; RV32IZCMP-NEXT:    .cfi_offset ra, -52
+; RV32IZCMP-NEXT:    .cfi_offset s0, -48
+; RV32IZCMP-NEXT:    .cfi_offset s1, -44
+; RV32IZCMP-NEXT:    .cfi_offset s2, -40
+; RV32IZCMP-NEXT:    .cfi_offset s3, -36
+; RV32IZCMP-NEXT:    .cfi_offset s4, -32
+; RV32IZCMP-NEXT:    .cfi_offset s5, -28
+; RV32IZCMP-NEXT:    .cfi_offset s6, -24
+; RV32IZCMP-NEXT:    .cfi_offset s7, -20
+; RV32IZCMP-NEXT:    .cfi_offset s8, -16
+; RV32IZCMP-NEXT:    .cfi_offset s9, -12
+; RV32IZCMP-NEXT:    .cfi_offset s10, -8
+; RV32IZCMP-NEXT:    .cfi_offset s11, -4
+; RV32IZCMP-NEXT:    addi sp, sp, -32
+; RV32IZCMP-NEXT:    .cfi_def_cfa_offset 144
+; RV32IZCMP-NEXT:    sw t0, 88(sp) # 4-byte Folded Spill
+; RV32IZCMP-NEXT:    sw t1, 84(sp) # 4-byte Folded Spill
+; RV32IZCMP-NEXT:    sw t2, 80(sp) # 4-byte Folded Spill
+; RV32IZCMP-NEXT:    sw a0, 76(sp) # 4-byte Folded Spill
+; RV32IZCMP-NEXT:    sw a1, 72(sp) # 4-byte Folded Spill
+; RV32IZCMP-NEXT:    sw a2, 68(sp) # 4-byte Folded Spill
+; RV32IZCMP-NEXT:    sw a3, 64(sp) # 4-byte Folded Spill
+; RV32IZCMP-NEXT:    sw a4, 60(sp) # 4-byte Folded Spill
+; RV32IZCMP-NEXT:    sw a5, 56(sp) # 4-byte Folded Spill
+; RV32IZCMP-NEXT:    sw a6, 52(sp) # 4-byte Folded Spill
+; RV32IZCMP-NEXT:    sw a7, 48(sp) # 4-byte Folded Spill
+; RV32IZCMP-NEXT:    sw t3, 44(sp) # 4-byte Folded Spill
+; RV32IZCMP-NEXT:    sw t4, 40(sp) # 4-byte Folded Spill
+; RV32IZCMP-NEXT:    sw t5, 36(sp) # 4-byte Folded Spill
+; RV32IZCMP-NEXT:    sw t6, 32(sp) # 4-byte Folded Spill
+; RV32IZCMP-NEXT:    .cfi_offset t0, -56
+; RV32IZCMP-NEXT:    .cfi_offset t1, -60
+; RV32IZCMP-NEXT:    .cfi_offset t2, -64
+; RV32IZCMP-NEXT:    .cfi_offset a0, -68
+; RV32IZCMP-NEXT:    .cfi_offset a1, -72
+; RV32IZCMP-NEXT:    .cfi_offset a2, -76
+; RV32IZCMP-NEXT:    .cfi_offset a3, -80
+; RV32IZCMP-NEXT:    .cfi_offset a4, -84
+; RV32IZCMP-NEXT:    .cfi_offset a5, -88
+; RV32IZCMP-NEXT:    .cfi_offset a6, -92
+; RV32IZCMP-NEXT:    .cfi_offset a7, -96
+; RV32IZCMP-NEXT:    .cfi_offset t3, -100
+; RV32IZCMP-NEXT:    .cfi_offset t4, -104
+; RV32IZCMP-NEXT:    .cfi_offset t5, -108
+; RV32IZCMP-NEXT:    .cfi_offset t6, -112
 ; RV32IZCMP-NEXT:    lui t0, %hi(var_test_irq)
 ; RV32IZCMP-NEXT:    lw a0, %lo(var_test_irq)(t0)
-; RV32IZCMP-NEXT:    sw a0, 32(sp) # 4-byte Folded Spill
-; RV32IZCMP-NEXT:    lw a0, %lo(var_test_irq+4)(t0)
 ; RV32IZCMP-NEXT:    sw a0, 28(sp) # 4-byte Folded Spill
-; RV32IZCMP-NEXT:    lw a0, %lo(var_test_irq+8)(t0)
+; RV32IZCMP-NEXT:    lw a0, %lo(var_test_irq+4)(t0)
 ; RV32IZCMP-NEXT:    sw a0, 24(sp) # 4-byte Folded Spill
-; RV32IZCMP-NEXT:    lw a0, %lo(var_test_irq+12)(t0)
+; RV32IZCMP-NEXT:    lw a0, %lo(var_test_irq+8)(t0)
 ; RV32IZCMP-NEXT:    sw a0, 20(sp) # 4-byte Folded Spill
+; RV32IZCMP-NEXT:    lw a0, %lo(var_test_irq+12)(t0)
+; RV32IZCMP-NEXT:    sw a0, 16(sp) # 4-byte Folded Spill
 ; RV32IZCMP-NEXT:    addi a5, t0, %lo(var_test_irq)
 ; RV32IZCMP-NEXT:    lw a0, 16(a5)
-; RV32IZCMP-NEXT:    sw a0, 16(sp) # 4-byte Folded Spill
-; RV32IZCMP-NEXT:    lw a0, 20(a5)
 ; RV32IZCMP-NEXT:    sw a0, 12(sp) # 4-byte Folded Spill
+; RV32IZCMP-NEXT:    lw a0, 20(a5)
+; RV32IZCMP-NEXT:    sw a0, 8(sp) # 4-byte Folded Spill
 ; RV32IZCMP-NEXT:    lw t4, 24(a5)
 ; RV32IZCMP-NEXT:    lw t5, 28(a5)
 ; RV32IZCMP-NEXT:    lw t6, 32(a5)
@@ -2035,70 +2389,130 @@ define void @callee_with_irq() nounwind "interrupt"="user" {
 ; RV32IZCMP-NEXT:    sw t6, 32(a5)
 ; RV32IZCMP-NEXT:    sw t5, 28(a5)
 ; RV32IZCMP-NEXT:    sw t4, 24(a5)
-; RV32IZCMP-NEXT:    lw a0, 12(sp) # 4-byte Folded Reload
+; RV32IZCMP-NEXT:    lw a0, 8(sp) # 4-byte Folded Reload
 ; RV32IZCMP-NEXT:    sw a0, 20(a5)
-; RV32IZCMP-NEXT:    lw a0, 16(sp) # 4-byte Folded Reload
+; RV32IZCMP-NEXT:    lw a0, 12(sp) # 4-byte Folded Reload
 ; RV32IZCMP-NEXT:    sw a0, 16(a5)
-; RV32IZCMP-NEXT:    lw a0, 20(sp) # 4-byte Folded Reload
+; RV32IZCMP-NEXT:    lw a0, 16(sp) # 4-byte Folded Reload
 ; RV32IZCMP-NEXT:    sw a0, %lo(var_test_irq+12)(t0)
-; RV32IZCMP-NEXT:    lw a0, 24(sp) # 4-byte Folded Reload
+; RV32IZCMP-NEXT:    lw a0, 20(sp) # 4-byte Folded Reload
 ; RV32IZCMP-NEXT:    sw a0, %lo(var_test_irq+8)(t0)
-; RV32IZCMP-NEXT:    lw a0, 28(sp) # 4-byte Folded Reload
+; RV32IZCMP-NEXT:    lw a0, 24(sp) # 4-byte Folded Reload
 ; RV32IZCMP-NEXT:    sw a0, %lo(var_test_irq+4)(t0)
-; RV32IZCMP-NEXT:    lw a0, 32(sp) # 4-byte Folded Reload
+; RV32IZCMP-NEXT:    lw a0, 28(sp) # 4-byte Folded Reload
 ; RV32IZCMP-NEXT:    sw a0, %lo(var_test_irq)(t0)
-; RV32IZCMP-NEXT:    lw t0, 92(sp) # 4-byte Folded Reload
-; RV32IZCMP-NEXT:    lw t1, 88(sp) # 4-byte Folded Reload
-; RV32IZCMP-NEXT:    lw t2, 84(sp) # 4-byte Folded Reload
-; RV32IZCMP-NEXT:    lw a0, 80(sp) # 4-byte Folded Reload
-; RV32IZCMP-NEXT:    lw a1, 76(sp) # 4-byte Folded Reload
-; RV32IZCMP-NEXT:    lw a2, 72(sp) # 4-byte Folded Reload
-; RV32IZCMP-NEXT:    lw a3, 68(sp) # 4-byte Folded Reload
-; RV32IZCMP-NEXT:    lw a4, 64(sp) # 4-byte Folded Reload
-; RV32IZCMP-NEXT:    lw a5, 60(sp) # 4-byte Folded Reload
-; RV32IZCMP-NEXT:    lw a6, 56(sp) # 4-byte Folded Reload
-; RV32IZCMP-NEXT:    lw a7, 52(sp) # 4-byte Folded Reload
-; RV32IZCMP-NEXT:    lw t3, 48(sp) # 4-byte Folded Reload
-; RV32IZCMP-NEXT:    lw t4, 44(sp) # 4-byte Folded Reload
-; RV32IZCMP-NEXT:    lw t5, 40(sp) # 4-byte Folded Reload
-; RV32IZCMP-NEXT:    lw t6, 36(sp) # 4-byte Folded Reload
-; RV32IZCMP-NEXT:    addi sp, sp, 48
+; RV32IZCMP-NEXT:    lw t0, 88(sp) # 4-byte Folded Reload
+; RV32IZCMP-NEXT:    lw t1, 84(sp) # 4-byte Folded Reload
+; RV32IZCMP-NEXT:    lw t2, 80(sp) # 4-byte Folded Reload
+; RV32IZCMP-NEXT:    lw a0, 76(sp) # 4-byte Folded Reload
+; RV32IZCMP-NEXT:    lw a1, 72(sp) # 4-byte Folded Reload
+; RV32IZCMP-NEXT:    lw a2, 68(sp) # 4-byte Folded Reload
+; RV32IZCMP-NEXT:    lw a3, 64(sp) # 4-byte Folded Reload
+; RV32IZCMP-NEXT:    lw a4, 60(sp) # 4-byte Folded Reload
+; RV32IZCMP-NEXT:    lw a5, 56(sp) # 4-byte Folded Reload
+; RV32IZCMP-NEXT:    lw a6, 52(sp) # 4-byte Folded Reload
+; RV32IZCMP-NEXT:    lw a7, 48(sp) # 4-byte Folded Reload
+; RV32IZCMP-NEXT:    lw t3, 44(sp) # 4-byte Folded Reload
+; RV32IZCMP-NEXT:    lw t4, 40(sp) # 4-byte Folded Reload
+; RV32IZCMP-NEXT:    lw t5, 36(sp) # 4-byte Folded Reload
+; RV32IZCMP-NEXT:    lw t6, 32(sp) # 4-byte Folded Reload
+; RV32IZCMP-NEXT:    .cfi_restore t0
+; RV32IZCMP-NEXT:    .cfi_restore t1
+; RV32IZCMP-NEXT:    .cfi_restore t2
+; RV32IZCMP-NEXT:    .cfi_restore a0
+; RV32IZCMP-NEXT:    .cfi_restore a1
+; RV32IZCMP-NEXT:    .cfi_restore a2
+; RV32IZCMP-NEXT:    .cfi_restore a3
+; RV32IZCMP-NEXT:    .cfi_restore a4
+; RV32IZCMP-NEXT:    .cfi_restore a5
+; RV32IZCMP-NEXT:    .cfi_restore a6
+; RV32IZCMP-NEXT:    .cfi_restore a7
+; RV32IZCMP-NEXT:    .cfi_restore t3
+; RV32IZCMP-NEXT:    .cfi_restore t4
+; RV32IZCMP-NEXT:    .cfi_restore t5
+; RV32IZCMP-NEXT:    .cfi_restore t6
+; RV32IZCMP-NEXT:    addi sp, sp, 32
+; RV32IZCMP-NEXT:    .cfi_def_cfa_offset 112
 ; RV32IZCMP-NEXT:    cm.pop {ra, s0-s11}, 112
+; RV32IZCMP-NEXT:    .cfi_restore ra
+; RV32IZCMP-NEXT:    .cfi_restore s0
+; RV32IZCMP-NEXT:    .cfi_restore s1
+; RV32IZCMP-NEXT:    .cfi_restore s2
+; RV32IZCMP-NEXT:    .cfi_restore s3
+; RV32IZCMP-NEXT:    .cfi_restore s4
+; RV32IZCMP-NEXT:    .cfi_restore s5
+; RV32IZCMP-NEXT:    .cfi_restore s6
+; RV32IZCMP-NEXT:    .cfi_restore s7
+; RV32IZCMP-NEXT:    .cfi_restore s8
+; RV32IZCMP-NEXT:    .cfi_restore s9
+; RV32IZCMP-NEXT:    .cfi_restore s10
+; RV32IZCMP-NEXT:    .cfi_restore s11
+; RV32IZCMP-NEXT:    .cfi_def_cfa_offset 0
 ; RV32IZCMP-NEXT:    mret
 ;
 ; RV64IZCMP-LABEL: callee_with_irq:
 ; RV64IZCMP:       # %bb.0:
 ; RV64IZCMP-NEXT:    cm.push {ra, s0-s11}, -160
-; RV64IZCMP-NEXT:    addi sp, sp, -128
-; RV64IZCMP-NEXT:    sd t0, 168(sp) # 8-byte Folded Spill
-; RV64IZCMP-NEXT:    sd t1, 160(sp) # 8-byte Folded Spill
-; RV64IZCMP-NEXT:    sd t2, 152(sp) # 8-byte Folded Spill
-; RV64IZCMP-NEXT:    sd a0, 144(sp) # 8-byte Folded Spill
-; RV64IZCMP-NEXT:    sd a1, 136(sp) # 8-byte Folded Spill
-; RV64IZCMP-NEXT:    sd a2, 128(sp) # 8-byte Folded Spill
-; RV64IZCMP-NEXT:    sd a3, 120(sp) # 8-byte Folded Spill
-; RV64IZCMP-NEXT:    sd a4, 112(sp) # 8-byte Folded Spill
-; RV64IZCMP-NEXT:    sd a5, 104(sp) # 8-byte Folded Spill
-; RV64IZCMP-NEXT:    sd a6, 96(sp) # 8-byte Folded Spill
-; RV64IZCMP-NEXT:    sd a7, 88(sp) # 8-byte Folded Spill
-; RV64IZCMP-NEXT:    sd t3, 80(sp) # 8-byte Folded Spill
-; RV64IZCMP-NEXT:    sd t4, 72(sp) # 8-byte Folded Spill
-; RV64IZCMP-NEXT:    sd t5, 64(sp) # 8-byte Folded Spill
-; RV64IZCMP-NEXT:    sd t6, 56(sp) # 8-byte Folded Spill
+; RV64IZCMP-NEXT:    .cfi_def_cfa_offset 160
+; RV64IZCMP-NEXT:    .cfi_offset ra, -104
+; RV64IZCMP-NEXT:    .cfi_offset s0, -96
+; RV64IZCMP-NEXT:    .cfi_offset s1, -88
+; RV64IZCMP-NEXT:    .cfi_offset s2, -80
+; RV64IZCMP-NEXT:    .cfi_offset s3, -72
+; RV64IZCMP-NEXT:    .cfi_offset s4, -64
+; RV64IZCMP-NEXT:    .cfi_offset s5, -56
+; RV64IZCMP-NEXT:    .cfi_offset s6, -48
+; RV64IZCMP-NEXT:    .cfi_offset s7, -40
+; RV64IZCMP-NEXT:    .cfi_offset s8, -32
+; RV64IZCMP-NEXT:    .cfi_offset s9, -24
+; RV64IZCMP-NEXT:    .cfi_offset s10, -16
+; RV64IZCMP-NEXT:    .cfi_offset s11, -8
+; RV64IZCMP-NEXT:    addi sp, sp, -112
+; RV64IZCMP-NEXT:    .cfi_def_cfa_offset 272
+; RV64IZCMP-NEXT:    sd t0, 160(sp) # 8-byte Folded Spill
+; RV64IZCMP-NEXT:    sd t1, 152(sp) # 8-byte Folded Spill
+; RV64IZCMP-NEXT:    sd t2, 144(sp) # 8-byte Folded Spill
+; RV64IZCMP-NEXT:    sd a0, 136(sp) # 8-byte Folded Spill
+; RV64IZCMP-NEXT:    sd a1, 128(sp) # 8-byte Folded Spill
+; RV64IZCMP-NEXT:    sd a2, 120(sp) # 8-byte Folded Spill
+; RV64IZCMP-NEXT:    sd a3, 112(sp) # 8-byte Folded Spill
+; RV64IZCMP-NEXT:    sd a4, 104(sp) # 8-byte Folded Spill
+; RV64IZCMP-NEXT:    sd a5, 96(sp) # 8-byte Folded Spill
+; RV64IZCMP-NEXT:    sd a6, 88(sp) # 8-byte Folded Spill
+; RV64IZCMP-NEXT:    sd a7, 80(sp) # 8-byte Folded Spill
+; RV64IZCMP-NEXT:    sd t3, 72(sp) # 8-byte Folded Spill
+; RV64IZCMP-NEXT:    sd t4, 64(sp) # 8-byte Folded Spill
+; RV64IZCMP-NEXT:    sd t5, 56(sp) # 8-byte Folded Spill
+; RV64IZCMP-NEXT:    sd t6, 48(sp) # 8-byte Folded Spill
+; RV64IZCMP-NEXT:    .cfi_offset t0, -112
+; RV64IZCMP-NEXT:    .cfi_offset t1, -120
+; RV64IZCMP-NEXT:    .cfi_offset t2, -128
+; RV64IZCMP-NEXT:    .cfi_offset a0, -136
+; RV64IZCMP-NEXT:    .cfi_offset a1, -144
+; RV64IZCMP-NEXT:    .cfi_offset a2, -152
+; RV64IZCMP-NEXT:    .cfi_offset a3, -160
+; RV64IZCMP-NEXT:    .cfi_offset a4, -168
+; RV64IZCMP-NEXT:    .cfi_offset a5, -176
+; RV64IZCMP-NEXT:    .cfi_offset a6, -184
+; RV64IZCMP-NEXT:    .cfi_offset a7, -192
+; RV64IZCMP-NEXT:    .cfi_offset t3, -200
+; RV64IZCMP-NEXT:    .cfi_offset t4, -208
+; RV64IZCMP-NEXT:    .cfi_offset t5, -216
+; RV64IZCMP-NEXT:    .cfi_offset t6, -224
 ; RV64IZCMP-NEXT:    lui t0, %hi(var_test_irq)
 ; RV64IZCMP-NEXT:    lw a0, %lo(var_test_irq)(t0)
-; RV64IZCMP-NEXT:    sd a0, 48(sp) # 8-byte Folded Spill
-; RV64IZCMP-NEXT:    lw a0, %lo(var_test_irq+4)(t0)
 ; RV64IZCMP-NEXT:    sd a0, 40(sp) # 8-byte Folded Spill
-; RV64IZCMP-NEXT:    lw a0, %lo(var_test_irq+8)(t0)
+; RV64IZCMP-NEXT:    lw a0, %lo(var_test_irq+4)(t0)
 ; RV64IZCMP-NEXT:    sd a0, 32(sp) # 8-byte Folded Spill
-; RV64IZCMP-NEXT:    lw a0, %lo(var_test_irq+12)(t0)
+; RV64IZCMP-NEXT:    lw a0, %lo(var_test_irq+8)(t0)
 ; RV64IZCMP-NEXT:    sd a0, 24(sp) # 8-byte Folded Spill
+; RV64IZCMP-NEXT:    lw a0, %lo(var_test_irq+12)(t0)
+; RV64IZCMP-NEXT:    sd a0, 16(sp) # 8-byte Folded Spill
 ; RV64IZCMP-NEXT:    addi a5, t0, %lo(var_test_irq)
 ; RV64IZCMP-NEXT:    lw a0, 16(a5)
-; RV64IZCMP-NEXT:    sd a0, 16(sp) # 8-byte Folded Spill
-; RV64IZCMP-NEXT:    lw a0, 20(a5)
 ; RV64IZCMP-NEXT:    sd a0, 8(sp) # 8-byte Folded Spill
+; RV64IZCMP-NEXT:    lw a0, 20(a5)
+; RV64IZCMP-NEXT:    sd a0, 0(sp) # 8-byte Folded Spill
 ; RV64IZCMP-NEXT:    lw t4, 24(a5)
 ; RV64IZCMP-NEXT:    lw t5, 28(a5)
 ; RV64IZCMP-NEXT:    lw t6, 32(a5)
@@ -2151,70 +2565,130 @@ define void @callee_with_irq() nounwind "interrupt"="user" {
 ; RV64IZCMP-NEXT:    sw t6, 32(a5)
 ; RV64IZCMP-NEXT:    sw t5, 28(a5)
 ; RV64IZCMP-NEXT:    sw t4, 24(a5)
-; RV64IZCMP-NEXT:    ld a0, 8(sp) # 8-byte Folded Reload
+; RV64IZCMP-NEXT:    ld a0, 0(sp) # 8-byte Folded Reload
 ; RV64IZCMP-NEXT:    sw a0, 20(a5)
-; RV64IZCMP-NEXT:    ld a0, 16(sp) # 8-byte Folded Reload
+; RV64IZCMP-NEXT:    ld a0, 8(sp) # 8-byte Folded Reload
 ; RV64IZCMP-NEXT:    sw a0, 16(a5)
-; RV64IZCMP-NEXT:    ld a0, 24(sp) # 8-byte Folded Reload
+; RV64IZCMP-NEXT:    ld a0, 16(sp) # 8-byte Folded Reload
 ; RV64IZCMP-NEXT:    sw a0, %lo(var_test_irq+12)(t0)
-; RV64IZCMP-NEXT:    ld a0, 32(sp) # 8-byte Folded Reload
+; RV64IZCMP-NEXT:    ld a0, 24(sp) # 8-byte Folded Reload
 ; RV64IZCMP-NEXT:    sw a0, %lo(var_test_irq+8)(t0)
-; RV64IZCMP-NEXT:    ld a0, 40(sp) # 8-byte Folded Reload
+; RV64IZCMP-NEXT:    ld a0, 32(sp) # 8-byte Folded Reload
 ; RV64IZCMP-NEXT:    sw a0, %lo(var_test_irq+4)(t0)
-; RV64IZCMP-NEXT:    ld a0, 48(sp) # 8-byte Folded Reload
+; RV64IZCMP-NEXT:    ld a0, 40(sp) # 8-byte Folded Reload
 ; RV64IZCMP-NEXT:    sw a0, %lo(var_test_irq)(t0)
-; RV64IZCMP-NEXT:    ld t0, 168(sp) # 8-byte Folded Reload
-; RV64IZCMP-NEXT:    ld t1, 160(sp) # 8-byte Folded Reload
-; RV64IZCMP-NEXT:    ld t2, 152(sp) # 8-byte Folded Reload
-; RV64IZCMP-NEXT:    ld a0, 144(sp) # 8-byte Folded Reload
-; RV64IZCMP-NEXT:    ld a1, 136(sp) # 8-byte Folded Reload
-; RV64IZCMP-NEXT:    ld a2, 128(sp) # 8-byte Folded Reload
-; RV64IZCMP-NEXT:    ld a3, 120(sp) # 8-byte Folded Reload
-; RV64IZCMP-NEXT:    ld a4, 112(sp) # 8-byte Folded Reload
-; RV64IZCMP-NEXT:    ld a5, 104(sp) # 8-byte Folded Reload
-; RV64IZCMP-NEXT:    ld a6, 96(sp) # 8-byte Folded Reload
-; RV64IZCMP-NEXT:    ld a7, 88(sp) # 8-byte Folded Reload
-; RV64IZCMP-NEXT:    ld t3, 80(sp) # 8-byte Folded Reload
-; RV64IZCMP-NEXT:    ld t4, 72(sp) # 8-byte Folded Reload
-; RV64IZCMP-NEXT:    ld t5, 64(sp) # 8-byte Folded Reload
-; RV64IZCMP-NEXT:    ld t6, 56(sp) # 8-byte Folded Reload
-; RV64IZCMP-NEXT:    addi sp, sp, 128
+; RV64IZCMP-NEXT:    ld t0, 160(sp) # 8-byte Folded Reload
+; RV64IZCMP-NEXT:    ld t1, 152(sp) # 8-byte Folded Reload
+; RV64IZCMP-NEXT:    ld t2, 144(sp) # 8-byte Folded Reload
+; RV64IZCMP-NEXT:    ld a0, 136(sp) # 8-byte Folded Reload
+; RV64IZCMP-NEXT:    ld a1, 128(sp) # 8-byte Folded Reload
+; RV64IZCMP-NEXT:    ld a2, 120(sp) # 8-byte Folded Reload
+; RV64IZCMP-NEXT:    ld a3, 112(sp) # 8-byte Folded Reload
+; RV64IZCMP-NEXT:    ld a4, 104(sp) # 8-byte Folded Reload
+; RV64IZCMP-NEXT:    ld a5, 96(sp) # 8-byte Folded Reload
+; RV64IZCMP-NEXT:    ld a6, 88(sp) # 8-byte Folded Reload
+; RV64IZCMP-NEXT:    ld a7, 80(sp) # 8-byte Folded Reload
+; RV64IZCMP-NEXT:    ld t3, 72(sp) # 8-byte Folded Reload
+; RV64IZCMP-NEXT:    ld t4, 64(sp) # 8-byte Folded Reload
+; RV64IZCMP-NEXT:    ld t5, 56(sp) # 8-byte Folded Reload
+; RV64IZCMP-NEXT:    ld t6, 48(sp) # 8-byte Folded Reload
+; RV64IZCMP-NEXT:    .cfi_restore t0
+; RV64IZCMP-NEXT:    .cfi_restore t1
+; RV64IZCMP-NEXT:    .cfi_restore t2
+; RV64IZCMP-NEXT:    .cfi_restore a0
+; RV64IZCMP-NEXT:    .cfi_restore a1
+; RV64IZCMP-NEXT:    .cfi_restore a2
+; RV64IZCMP-NEXT:    .cfi_restore a3
+; RV64IZCMP-NEXT:    .cfi_restore a4
+; RV64IZCMP-NEXT:    .cfi_restore a5
+; RV64IZCMP-NEXT:    .cfi_restore a6
+; RV64IZCMP-NEXT:    .cfi_restore a7
+; RV64IZCMP-NEXT:    .cfi_restore t3
+; RV64IZCMP-NEXT:    .cfi_restore t4
+; RV64IZCMP-NEXT:    .cfi_restore t5
+; RV64IZCMP-NEXT:    .cfi_restore t6
+; RV64IZCMP-NEXT:    addi sp, sp, 112
+; RV64IZCMP-NEXT:    .cfi_def_cfa_offset 160
 ; RV64IZCMP-NEXT:    cm.pop {ra, s0-s11}, 160
+; RV64IZCMP-NEXT:    .cfi_restore ra
+; RV64IZCMP-NEXT:    .cfi_restore s0
+; RV64IZCMP-NEXT:    .cfi_restore s1
+; RV64IZCMP-NEXT:    .cfi_restore s2
+; RV64IZCMP-NEXT:    .cfi_restore s3
+; RV64IZCMP-NEXT:    .cfi_restore s4
+; RV64IZCMP-NEXT:    .cfi_restore s5
+; RV64IZCMP-NEXT:    .cfi_restore s6
+; RV64IZCMP-NEXT:    .cfi_restore s7
+; RV64IZCMP-NEXT:    .cfi_restore s8
+; RV64IZCMP-NEXT:    .cfi_restore s9
+; RV64IZCMP-NEXT:    .cfi_restore s10
+; RV64IZCMP-NEXT:    .cfi_restore s11
+; RV64IZCMP-NEXT:    .cfi_def_cfa_offset 0
 ; RV64IZCMP-NEXT:    mret
 ;
 ; RV32IZCMP-SR-LABEL: callee_with_irq:
 ; RV32IZCMP-SR:       # %bb.0:
 ; RV32IZCMP-SR-NEXT:    cm.push {ra, s0-s11}, -112
-; RV32IZCMP-SR-NEXT:    addi sp, sp, -48
-; RV32IZCMP-SR-NEXT:    sw t0, 92(sp) # 4-byte Folded Spill
-; RV32IZCMP-SR-NEXT:    sw t1, 88(sp) # 4-byte Folded Spill
-; RV32IZCMP-SR-NEXT:    sw t2, 84(sp) # 4-byte Folded Spill
-; RV32IZCMP-SR-NEXT:    sw a0, 80(sp) # 4-byte Folded Spill
-; RV32IZCMP-SR-NEXT:    sw a1, 76(sp) # 4-byte Folded Spill
-; RV32IZCMP-SR-NEXT:    sw a2, 72(sp) # 4-byte Folded Spill
-; RV32IZCMP-SR-NEXT:    sw a3, 68(sp) # 4-byte Folded Spill
-; RV32IZCMP-SR-NEXT:    sw a4, 64(sp) # 4-byte Folded Spill
-; RV32IZCMP-SR-NEXT:    sw a5, 60(sp) # 4-byte Folded Spill
-; RV32IZCMP-SR-NEXT:    sw a6, 56(sp) # 4-byte Folded Spill
-; RV32IZCMP-SR-NEXT:    sw a7, 52(sp) # 4-byte Folded Spill
-; RV32IZCMP-SR-NEXT:    sw t3, 48(sp) # 4-byte Folded Spill
-; RV32IZCMP-SR-NEXT:    sw t4, 44(sp) # 4-byte Folded Spill
-; RV32IZCMP-SR-NEXT:    sw t5, 40(sp) # 4-byte Folded Spill
-; RV32IZCMP-SR-NEXT:    sw t6, 36(sp) # 4-byte Folded Spill
+; RV32IZCMP-SR-NEXT:    .cfi_def_cfa_offset 112
+; RV32IZCMP-SR-NEXT:    .cfi_offset ra, -52
+; RV32IZCMP-SR-NEXT:    .cfi_offset s0, -48
+; RV32IZCMP-SR-NEXT:    .cfi_offset s1, -44
+; RV32IZCMP-SR-NEXT:    .cfi_offset s2, -40
+; RV32IZCMP-SR-NEXT:    .cfi_offset s3, -36
+; RV32IZCMP-SR-NEXT:    .cfi_offset s4, -32
+; RV32IZCMP-SR-NEXT:    .cfi_offset s5, -28
+; RV32IZCMP-SR-NEXT:    .cfi_offset s6, -24
+; RV32IZCMP-SR-NEXT:    .cfi_offset s7, -20
+; RV32IZCMP-SR-NEXT:    .cfi_offset s8, -16
+; RV32IZCMP-SR-NEXT:    .cfi_offset s9, -12
+; RV32IZCMP-SR-NEXT:    .cfi_offset s10, -8
+; RV32IZCMP-SR-NEXT:    .cfi_offset s11, -4
+; RV32IZCMP-SR-NEXT:    addi sp, sp, -32
+; RV32IZCMP-SR-NEXT:    .cfi_def_cfa_offset 144
+; RV32IZCMP-SR-NEXT:    sw t0, 88(sp) # 4-byte Folded Spill
+; RV32IZCMP-SR-NEXT:    sw t1, 84(sp) # 4-byte Folded Spill
+; RV32IZCMP-SR-NEXT:    sw t2, 80(sp) # 4-byte Folded Spill
+; RV32IZCMP-SR-NEXT:    sw a0, 76(sp) # 4-byte Folded Spill
+; RV32IZCMP-SR-NEXT:    sw a1, 72(sp) # 4-byte Folded Spill
+; RV32IZCMP-SR-NEXT:    sw a2, 68(sp) # 4-byte Folded Spill
+; RV32IZCMP-SR-NEXT:    sw a3, 64(sp) # 4-byte Folded Spill
+; RV32IZCMP-SR-NEXT:    sw a4, 60(sp) # 4-byte Folded Spill
+; RV32IZCMP-SR-NEXT:    sw a5, 56(sp) # 4-byte Folded Spill
+; RV32IZCMP-SR-NEXT:    sw a6, 52(sp) # 4-byte Folded Spill
+; RV32IZCMP-SR-NEXT:    sw a7, 48(sp) # 4-byte Folded Spill
+; RV32IZCMP-SR-NEXT:    sw t3, 44(sp) # 4-byte Folded Spill
+; RV32IZCMP-SR-NEXT:    sw t4, 40(sp) # 4-byte Folded Spill
+; RV32IZCMP-SR-NEXT:    sw t5, 36(sp) # 4-byte Folded Spill
+; RV32IZCMP-SR-NEXT:    sw t6, 32(sp) # 4-byte Folded Spill
+; RV32IZCMP-SR-NEXT:    .cfi_offset t0, -56
+; RV32IZCMP-SR-NEXT:    .cfi_offset t1, -60
+; RV32IZCMP-SR-NEXT:    .cfi_offset t2, -64
+; RV32IZCMP-SR-NEXT:    .cfi_offset a0, -68
+; RV32IZCMP-SR-NEXT:    .cfi_offset a1, -72
+; RV32IZCMP-SR-NEXT:    .cfi_offset a2, -76
+; RV32IZCMP-SR-NEXT:    .cfi_offset a3, -80
+; RV32IZCMP-SR-NEXT:    .cfi_offset a4, -84
+; RV32IZCMP-SR-NEXT:    .cfi_offset a5, -88
+; RV32IZCMP-SR-NEXT:    .cfi_offset a6, -92
+; RV32IZCMP-SR-NEXT:    .cfi_offset a7, -96
+; RV32IZCMP-SR-NEXT:    .cfi_offset t3, -100
+; RV32IZCMP-SR-NEXT:    .cfi_offset t4, -104
+; RV32IZCMP-SR-NEXT:    .cfi_offset t5, -108
+; RV32IZCMP-SR-NEXT:    .cfi_offset t6, -112
 ; RV32IZCMP-SR-NEXT:    lui t0, %hi(var_test_irq)
 ; RV32IZCMP-SR-NEXT:    lw a0, %lo(var_test_irq)(t0)
-; RV32IZCMP-SR-NEXT:    sw a0, 32(sp) # 4-byte Folded Spill
-; RV32IZCMP-SR-NEXT:    lw a0, %lo(var_test_irq+4)(t0)
 ; RV32IZCMP-SR-NEXT:    sw a0, 28(sp) # 4-byte Folded Spill
-; RV32IZCMP-SR-NEXT:    lw a0, %lo(var_test_irq+8)(t0)
+; RV32IZCMP-SR-NEXT:    lw a0, %lo(var_test_irq+4)(t0)
 ; RV32IZCMP-SR-NEXT:    sw a0, 24(sp) # 4-byte Folded Spill
-; RV32IZCMP-SR-NEXT:    lw a0, %lo(var_test_irq+12)(t0)
+; RV32IZCMP-SR-NEXT:    lw a0, %lo(var_test_irq+8)(t0)
 ; RV32IZCMP-SR-NEXT:    sw a0, 20(sp) # 4-byte Folded Spill
+; RV32IZCMP-SR-NEXT:    lw a0, %lo(var_test_irq+12)(t0)
+; RV32IZCMP-SR-NEXT:    sw a0, 16(sp) # 4-byte Folded Spill
 ; RV32IZCMP-SR-NEXT:    addi a5, t0, %lo(var_test_irq)
 ; RV32IZCMP-SR-NEXT:    lw a0, 16(a5)
-; RV32IZCMP-SR-NEXT:    sw a0, 16(sp) # 4-byte Folded Spill
-; RV32IZCMP-SR-NEXT:    lw a0, 20(a5)
 ; RV32IZCMP-SR-NEXT:    sw a0, 12(sp) # 4-byte Folded Spill
+; RV32IZCMP-SR-NEXT:    lw a0, 20(a5)
+; RV32IZCMP-SR-NEXT:    sw a0, 8(sp) # 4-byte Folded Spill
 ; RV32IZCMP-SR-NEXT:    lw t4, 24(a5)
 ; RV32IZCMP-SR-NEXT:    lw t5, 28(a5)
 ; RV32IZCMP-SR-NEXT:    lw t6, 32(a5)
@@ -2267,70 +2741,130 @@ define void @callee_with_irq() nounwind "interrupt"="user" {
 ; RV32IZCMP-SR-NEXT:    sw t6, 32(a5)
 ; RV32IZCMP-SR-NEXT:    sw t5, 28(a5)
 ; RV32IZCMP-SR-NEXT:    sw t4, 24(a5)
-; RV32IZCMP-SR-NEXT:    lw a0, 12(sp) # 4-byte Folded Reload
+; RV32IZCMP-SR-NEXT:    lw a0, 8(sp) # 4-byte Folded Reload
 ; RV32IZCMP-SR-NEXT:    sw a0, 20(a5)
-; RV32IZCMP-SR-NEXT:    lw a0, 16(sp) # 4-byte Folded Reload
+; RV32IZCMP-SR-NEXT:    lw a0, 12(sp) # 4-byte Folded Reload
 ; RV32IZCMP-SR-NEXT:    sw a0, 16(a5)
-; RV32IZCMP-SR-NEXT:    lw a0, 20(sp) # 4-byte Folded Reload
+; RV32IZCMP-SR-NEXT:    lw a0, 16(sp) # 4-byte Folded Reload
 ; RV32IZCMP-SR-NEXT:    sw a0, %lo(var_test_irq+12)(t0)
-; RV32IZCMP-SR-NEXT:    lw a0, 24(sp) # 4-byte Folded Reload
+; RV32IZCMP-SR-NEXT:    lw a0, 20(sp) # 4-byte Folded Reload
 ; RV32IZCMP-SR-NEXT:    sw a0, %lo(var_test_irq+8)(t0)
-; RV32IZCMP-SR-NEXT:    lw a0, 28(sp) # 4-byte Folded Reload
+; RV32IZCMP-SR-NEXT:    lw a0, 24(sp) # 4-byte Folded Reload
 ; RV32IZCMP-SR-NEXT:    sw a0, %lo(var_test_irq+4)(t0)
-; RV32IZCMP-SR-NEXT:    lw a0, 32(sp) # 4-byte Folded Reload
+; RV32IZCMP-SR-NEXT:    lw a0, 28(sp) # 4-byte Folded Reload
 ; RV32IZCMP-SR-NEXT:    sw a0, %lo(var_test_irq)(t0)
-; RV32IZCMP-SR-NEXT:    lw t0, 92(sp) # 4-byte Folded Reload
-; RV32IZCMP-SR-NEXT:    lw t1, 88(sp) # 4-byte Folded Reload
-; RV32IZCMP-SR-NEXT:    lw t2, 84(sp) # 4-byte Folded Reload
-; RV32IZCMP-SR-NEXT:    lw a0, 80(sp) # 4-byte Folded Reload
-; RV32IZCMP-SR-NEXT:    lw a1, 76(sp) # 4-byte Folded Reload
-; RV32IZCMP-SR-NEXT:    lw a2, 72(sp) # 4-byte Folded Reload
-; RV32IZCMP-SR-NEXT:    lw a3, 68(sp) # 4-byte Folded Reload
-; RV32IZCMP-SR-NEXT:    lw a4, 64(sp) # 4-byte Folded Reload
-; RV32IZCMP-SR-NEXT:    lw a5, 60(sp) # 4-byte Folded Reload
-; RV32IZCMP-SR-NEXT:    lw a6, 56(sp) # 4-byte Folded Reload
-; RV32IZCMP-SR-NEXT:    lw a7, 52(sp) # 4-byte Folded Reload
-; RV32IZCMP-SR-NEXT:    lw t3, 48(sp) # 4-byte Folded Reload
-; RV32IZCMP-SR-NEXT:    lw t4, 44(sp) # 4-byte Folded Reload
-; RV32IZCMP-SR-NEXT:    lw t5, 40(sp) # 4-byte Folded Reload
-; RV32IZCMP-SR-NEXT:    lw t6, 36(sp) # 4-byte Folded Reload
-; RV32IZCMP-SR-NEXT:    addi sp, sp, 48
+; RV32IZCMP-SR-NEXT:    lw t0, 88(sp) # 4-byte Folded Reload
+; RV32IZCMP-SR-NEXT:    lw t1, 84(sp) # 4-byte Folded Reload
+; RV32IZCMP-SR-NEXT:    lw t2, 80(sp) # 4-byte Folded Reload
+; RV32IZCMP-SR-NEXT:    lw a0, 76(sp) # 4-byte Folded Reload
+; RV32IZCMP-SR-NEXT:    lw a1, 72(sp) # 4-byte Folded Reload
+; RV32IZCMP-SR-NEXT:    lw a2, 68(sp) # 4-byte Folded Reload
+; RV32IZCMP-SR-NEXT:    lw a3, 64(sp) # 4-byte Folded Reload
+; RV32IZCMP-SR-NEXT:    lw a4, 60(sp) # 4-byte Folded Reload
+; RV32IZCMP-SR-NEXT:    lw a5, 56(sp) # 4-byte Folded Reload
+; RV32IZCMP-SR-NEXT:    lw a6, 52(sp) # 4-byte Folded Reload
+; RV32IZCMP-SR-NEXT:    lw a7, 48(sp) # 4-byte Folded Reload
+; RV32IZCMP-SR-NEXT:    lw t3, 44(sp) # 4-byte Folded Reload
+; RV32IZCMP-SR-NEXT:    lw t4, 40(sp) # 4-byte Folded Reload
+; RV32IZCMP-SR-NEXT:    lw t5, 36(sp) # 4-byte Folded Reload
+; RV32IZCMP-SR-NEXT:    lw t6, 32(sp) # 4-byte Folded Reload
+; RV32IZCMP-SR-NEXT:    .cfi_restore t0
+; RV32IZCMP-SR-NEXT:    .cfi_restore t1
+; RV32IZCMP-SR-NEXT:    .cfi_restore t2
+; RV32IZCMP-SR-NEXT:    .cfi_restore a0
+; RV32IZCMP-SR-NEXT:    .cfi_restore a1
+; RV32IZCMP-SR-NEXT:    .cfi_restore a2
+; RV32IZCMP-SR-NEXT:    .cfi_restore a3
+; RV32IZCMP-SR-NEXT:    .cfi_restore a4
+; RV32IZCMP-SR-NEXT:    .cfi_restore a5
+; RV32IZCMP-SR-NEXT:    .cfi_restore a6
+; RV32IZCMP-SR-NEXT:    .cfi_restore a7
+; RV32IZCMP-SR-NEXT:    .cfi_restore t3
+; RV32IZCMP-SR-NEXT:    .cfi_restore t4
+; RV32IZCMP-SR-NEXT:    .cfi_restore t5
+; RV32IZCMP-SR-NEXT:    .cfi_restore t6
+; RV32IZCMP-SR-NEXT:    addi sp, sp, 32
+; RV32IZCMP-SR-NEXT:    .cfi_def_cfa_offset 112
 ; RV32IZCMP-SR-NEXT:    cm.pop {ra, s0-s11}, 112
+; RV32IZCMP-SR-NEXT:    .cfi_restore ra
+; RV32IZCMP-SR-NEXT:    .cfi_restore s0
+; RV32IZCMP-SR-NEXT:    .cfi_restore s1
+; RV32IZCMP-SR-NEXT:    .cfi_restore s2
+; RV32IZCMP-SR-NEXT:    .cfi_restore s3
+; RV32IZCMP-SR-NEXT:    .cfi_restore s4
+; RV32IZCMP-SR-NEXT:    .cfi_restore s5
+; RV32IZCMP-SR-NEXT:    .cfi_restore s6
+; RV32IZCMP-SR-NEXT:    .cfi_restore s7
+; RV32IZCMP-SR-NEXT:    .cfi_restore s8
+; RV32IZCMP-SR-NEXT:    .cfi_restore s9
+; RV32IZCMP-SR-NEXT:    .cfi_restore s10
+; RV32IZCMP-SR-NEXT:    .cfi_restore s11
+; RV32IZCMP-SR-NEXT:    .cfi_def_cfa_offset 0
 ; RV32IZCMP-SR-NEXT:    mret
 ;
 ; RV64IZCMP-SR-LABEL: callee_with_irq:
 ; RV64IZCMP-SR:       # %bb.0:
 ; RV64IZCMP-SR-NEXT:    cm.push {ra, s0-s11}, -160
-; RV64IZCMP-SR-NEXT:    addi sp, sp, -128
-; RV64IZCMP-SR-NEXT:    sd t0, 168(sp) # 8-byte Folded Spill
-; RV64IZCMP-SR-NEXT:    sd t1, 160(sp) # 8-byte Folded Spill
-; RV64IZCMP-SR-NEXT:    sd t2, 152(sp) # 8-byte Folded Spill
-; RV64IZCMP-SR-NEXT:    sd a0, 144(sp) # 8-byte Folded Spill
-; RV64IZCMP-SR-NEXT:    sd a1, 136(sp) # 8-byte Folded Spill
-; RV64IZCMP-SR-NEXT:    sd a2, 128(sp) # 8-byte Folded Spill
-; RV64IZCMP-SR-NEXT:    sd a3, 120(sp) # 8-byte Folded Spill
-; RV64IZCMP-SR-NEXT:    sd a4, 112(sp) # 8-byte Folded Spill
-; RV64IZCMP-SR-NEXT:    sd a5, 104(sp) # 8-byte Folded Spill
-; RV64IZCMP-SR-NEXT:    sd a6, 96(sp) # 8-byte Folded Spill
-; RV64IZCMP-SR-NEXT:    sd a7, 88(sp) # 8-byte Folded Spill
-; RV64IZCMP-SR-NEXT:    sd t3, 80(sp) # 8-byte Folded Spill
-; RV64IZCMP-SR-NEXT:    sd t4, 72(sp) # 8-byte Folded Spill
-; RV64IZCMP-SR-NEXT:    sd t5, 64(sp) # 8-byte Folded Spill
-; RV64IZCMP-SR-NEXT:    sd t6, 56(sp) # 8-byte Folded Spill
+; RV64IZCMP-SR-NEXT:    .cfi_def_cfa_offset 160
+; RV64IZCMP-SR-NEXT:    .cfi_offset ra, -104
+; RV64IZCMP-SR-NEXT:    .cfi_offset s0, -96
+; RV64IZCMP-SR-NEXT:    .cfi_offset s1, -88
+; RV64IZCMP-SR-NEXT:    .cfi_offset s2, -80
+; RV64IZCMP-SR-NEXT:    .cfi_offset s3, -72
+; RV64IZCMP-SR-NEXT:    .cfi_offset s4, -64
+; RV64IZCMP-SR-NEXT:    .cfi_offset s5, -56
+; RV64IZCMP-SR-NEXT:    .cfi_offset s6, -48
+; RV64IZCMP-SR-NEXT:    .cfi_offset s7, -40
+; RV64IZCMP-SR-NEXT:    .cfi_offset s8, -32
+; RV64IZCMP-SR-NEXT:    .cfi_offset s9, -24
+; RV64IZCMP-SR-NEXT:    .cfi_offset s10, -16
+; RV64IZCMP-SR-NEXT:    .cfi_offset s11, -8
+; RV64IZCMP-SR-NEXT:    addi sp, sp, -112
+; RV64IZCMP-SR-NEXT:    .cfi_def_cfa_offset 272
+; RV64IZCMP-SR-NEXT:    sd t0, 160(sp) # 8-byte Folded Spill
+; RV64IZCMP-SR-NEXT:    sd t1, 152(sp) # 8-byte Folded Spill
+; RV64IZCMP-SR-NEXT:    sd t2, 144(sp) # 8-byte Folded Spill
+; RV64IZCMP-SR-NEXT:    sd a0, 136(sp) # 8-byte Folded Spill
+; RV64IZCMP-SR-NEXT:    sd a1, 128(sp) # 8-byte Folded Spill
+; RV64IZCMP-SR-NEXT:    sd a2, 120(sp) # 8-byte Folded Spill
+; RV64IZCMP-SR-NEXT:    sd a3, 112(sp) # 8-byte Folded Spill
+; RV64IZCMP-SR-NEXT:    sd a4, 104(sp) # 8-byte Folded Spill
+; RV64IZCMP-SR-NEXT:    sd a5, 96(sp) # 8-byte Folded Spill
+; RV64IZCMP-SR-NEXT:    sd a6, 88(sp) # 8-byte Folded Spill
+; RV64IZCMP-SR-NEXT:    sd a7, 80(sp) # 8-byte Folded Spill
+; RV64IZCMP-SR-NEXT:    sd t3, 72(sp) # 8-byte Folded Spill
+; RV64IZCMP-SR-NEXT:    sd t4, 64(sp) # 8-byte Folded Spill
+; RV64IZCMP-SR-NEXT:    sd t5, 56(sp) # 8-byte Folded Spill
+; RV64IZCMP-SR-NEXT:    sd t6, 48(sp) # 8-byte Folded Spill
+; RV64IZCMP-SR-NEXT:    .cfi_offset t0, -112
+; RV64IZCMP-SR-NEXT:    .cfi_offset t1, -120
+; RV64IZCMP-SR-NEXT:    .cfi_offset t2, -128
+; RV64IZCMP-SR-NEXT:    .cfi_offset a0, -136
+; RV64IZCMP-SR-NEXT:    .cfi_offset a1, -144
+; RV64IZCMP-SR-NEXT:    .cfi_offset a2, -152
+; RV64IZCMP-SR-NEXT:    .cfi_offset a3, -160
+; RV64IZCMP-SR-NEXT:    .cfi_offset a4, -168
+; RV64IZCMP-SR-NEXT:    .cfi_offset a5, -176
+; RV64IZCMP-SR-NEXT:    .cfi_offset a6, -184
+; RV64IZCMP-SR-NEXT:    .cfi_offset a7, -192
+; RV64IZCMP-SR-NEXT:    .cfi_offset t3, -200
+; RV64IZCMP-SR-NEXT:    .cfi_offset t4, -208
+; RV64IZCMP-SR-NEXT:    .cfi_offset t5, -216
+; RV64IZCMP-SR-NEXT:    .cfi_offset t6, -224
 ; RV64IZCMP-SR-NEXT:    lui t0, %hi(var_test_irq)
 ; RV64IZCMP-SR-NEXT:    lw a0, %lo(var_test_irq)(t0)
-; RV64IZCMP-SR-NEXT:    sd a0, 48(sp) # 8-byte Folded Spill
-; RV64IZCMP-SR-NEXT:    lw a0, %lo(var_test_irq+4)(t0)
 ; RV64IZCMP-SR-NEXT:    sd a0, 40(sp) # 8-byte Folded Spill
-; RV64IZCMP-SR-NEXT:    lw a0, %lo(var_test_irq+8)(t0)
+; RV64IZCMP-SR-NEXT:    lw a0, %lo(var_test_irq+4)(t0)
 ; RV64IZCMP-SR-NEXT:    sd a0, 32(sp) # 8-byte Folded Spill
-; RV64IZCMP-SR-NEXT:    lw a0, %lo(var_test_irq+12)(t0)
+; RV64IZCMP-SR-NEXT:    lw a0, %lo(var_test_irq+8)(t0)
 ; RV64IZCMP-SR-NEXT:    sd a0, 24(sp) # 8-byte Folded Spill
+; RV64IZCMP-SR-NEXT:    lw a0, %lo(var_test_irq+12)(t0)
+; RV64IZCMP-SR-NEXT:    sd a0, 16(sp) # 8-byte Folded Spill
 ; RV64IZCMP-SR-NEXT:    addi a5, t0, %lo(var_test_irq)
 ; RV64IZCMP-SR-NEXT:    lw a0, 16(a5)
-; RV64IZCMP-SR-NEXT:    sd a0, 16(sp) # 8-byte Folded Spill
-; RV64IZCMP-SR-NEXT:    lw a0, 20(a5)
 ; RV64IZCMP-SR-NEXT:    sd a0, 8(sp) # 8-byte Folded Spill
+; RV64IZCMP-SR-NEXT:    lw a0, 20(a5)
+; RV64IZCMP-SR-NEXT:    sd a0, 0(sp) # 8-byte Folded Spill
 ; RV64IZCMP-SR-NEXT:    lw t4, 24(a5)
 ; RV64IZCMP-SR-NEXT:    lw t5, 28(a5)
 ; RV64IZCMP-SR-NEXT:    lw t6, 32(a5)
@@ -2383,40 +2917,71 @@ define void @callee_with_irq() nounwind "interrupt"="user" {
 ; RV64IZCMP-SR-NEXT:    sw t6, 32(a5)
 ; RV64IZCMP-SR-NEXT:    sw t5, 28(a5)
 ; RV64IZCMP-SR-NEXT:    sw t4, 24(a5)
-; RV64IZCMP-SR-NEXT:    ld a0, 8(sp) # 8-byte Folded Reload
+; RV64IZCMP-SR-NEXT:    ld a0, 0(sp) # 8-byte Folded Reload
 ; RV64IZCMP-SR-NEXT:    sw a0, 20(a5)
-; RV64IZCMP-SR-NEXT:    ld a0, 16(sp) # 8-byte Folded Reload
+; RV64IZCMP-SR-NEXT:    ld a0, 8(sp) # 8-byte Folded Reload
 ; RV64IZCMP-SR-NEXT:    sw a0, 16(a5)
-; RV64IZCMP-SR-NEXT:    ld a0, 24(sp) # 8-byte Folded Reload
+; RV64IZCMP-SR-NEXT:    ld a0, 16(sp) # 8-byte Folded Reload
 ; RV64IZCMP-SR-NEXT:    sw a0, %lo(var_test_irq+12)(t0)
-; RV64IZCMP-SR-NEXT:    ld a0, 32(sp) # 8-byte Folded Reload
+; RV64IZCMP-SR-NEXT:    ld a0, 24(sp) # 8-byte Folded Reload
 ; RV64IZCMP-SR-NEXT:    sw a0, %lo(var_test_irq+8)(t0)
-; RV64IZCMP-SR-NEXT:    ld a0, 40(sp) # 8-byte Folded Reload
+; RV64IZCMP-SR-NEXT:    ld a0, 32(sp) # 8-byte Folded Reload
 ; RV64IZCMP-SR-NEXT:    sw a0, %lo(var_test_irq+4)(t0)
-; RV64IZCMP-SR-NEXT:    ld a0, 48(sp) # 8-byte Folded Reload
+; RV64IZCMP-SR-NEXT:    ld a0, 40(sp) # 8-byte Folded Reload
 ; RV64IZCMP-SR-NEXT:    sw a0, %lo(var_test_irq)(t0)
-; RV64IZCMP-SR-NEXT:    ld t0, 168(sp) # 8-byte Folded Reload
-; RV64IZCMP-SR-NEXT:    ld t1, 160(sp) # 8-byte Folded Reload
-; RV64IZCMP-SR-NEXT:    ld t2, 152(sp) # 8-byte Folded Reload
-; RV64IZCMP-SR-NEXT:    ld a0, 144(sp) # 8-byte Folded Reload
-; RV64IZCMP-SR-NEXT:    ld a1, 136(sp) # 8-byte Folded Reload
-; RV64IZCMP-SR-NEXT:    ld a2, 128(sp) # 8-byte Folded Reload
-; RV64IZCMP-SR-NEXT:    ld a3, 120(sp) # 8-byte Folded Reload
-; RV64IZCMP-SR-NEXT:    ld a4, 112(sp) # 8-byte Folded Reload
-; RV64IZCMP-SR-NEXT:    ld a5, 104(sp) # 8-byte Folded Reload
-; RV64IZCMP-SR-NEXT:    ld a6, 96(sp) # 8-byte Folded Reload
-; RV64IZCMP-SR-NEXT:    ld a7, 88(sp) # 8-byte Folded Reload
-; RV64IZCMP-SR-NEXT:    ld t3, 80(sp) # 8-byte Folded Reload
-; RV64IZCMP-SR-NEXT:    ld t4, 72(sp) # 8-byte Folded Reload
-; RV64IZCMP-SR-NEXT:    ld t5, 64(sp) # 8-byte Folded Reload
-; RV64IZCMP-SR-NEXT:    ld t6, 56(sp) # 8-byte Folded Reload
-; RV64IZCMP-SR-NEXT:    addi sp, sp, 128
+; RV64IZCMP-SR-NEXT:    ld t0, 160(sp) # 8-byte Folded Reload
+; RV64IZCMP-SR-NEXT:    ld t1, 152(sp) # 8-byte Folded Reload
+; RV64IZCMP-SR-NEXT:    ld t2, 144(sp) # 8-byte Folded Reload
+; RV64IZCMP-SR-NEXT:    ld a0, 136(sp) # 8-byte Folded Reload
+; RV64IZCMP-SR-NEXT:    ld a1, 128(sp) # 8-byte Folded Reload
+; RV64IZCMP-SR-NEXT:    ld a2, 120(sp) # 8-byte Folded Reload
+; RV64IZCMP-SR-NEXT:    ld a3, 112(sp) # 8-byte Folded Reload
+; RV64IZCMP-SR-NEXT:    ld a4, 104(sp) # 8-byte Folded Reload
+; RV64IZCMP-SR-NEXT:    ld a5, 96(sp) # 8-byte Folded Reload
+; RV64IZCMP-SR-NEXT:    ld a6, 88(sp) # 8-byte Folded Reload
+; RV64IZCMP-SR-NEXT:    ld a7, 80(sp) # 8-byte Folded Reload
+; RV64IZCMP-SR-NEXT:    ld t3, 72(sp) # 8-byte Folded Reload
+; RV64IZCMP-SR-NEXT:    ld t4, 64(sp) # 8-byte Folded Reload
+; RV64IZCMP-SR-NEXT:    ld t5, 56(sp) # 8-byte Folded Reload
+; RV64IZCMP-SR-NEXT:    ld t6, 48(sp) # 8-byte Folded Reload
+; RV64IZCMP-SR-NEXT:    .cfi_restore t0
+; RV64IZCMP-SR-NEXT:    .cfi_restore t1
+; RV64IZCMP-SR-NEXT:    .cfi_restore t2
+; RV64IZCMP-SR-NEXT:    .cfi_restore a0
+; RV64IZCMP-SR-NEXT:    .cfi_restore a1
+; RV64IZCMP-SR-NEXT:    .cfi_restore a2
+; RV64IZCMP-SR-NEXT:    .cfi_restore a3
+; RV64IZCMP-SR-NEXT:    .cfi_restore a4
+; RV64IZCMP-SR-NEXT:    .cfi_restore a5
+; RV64IZCMP-SR-NEXT:    .cfi_restore a6
+; RV64IZCMP-SR-NEXT:    .cfi_restore a7
+; RV64IZCMP-SR-NEXT:    .cfi_restore t3
+; RV64IZCMP-SR-NEXT:    .cfi_restore t4
+; RV64IZCMP-SR-NEXT:    .cfi_restore t5
+; RV64IZCMP-SR-NEXT:    .cfi_restore t6
+; RV64IZCMP-SR-NEXT:    addi sp, sp, 112
+; RV64IZCMP-SR-NEXT:    .cfi_def_cfa_offset 160
 ; RV64IZCMP-SR-NEXT:    cm.pop {ra, s0-s11}, 160
+; RV64IZCMP-SR-NEXT:    .cfi_restore ra
+; RV64IZCMP-SR-NEXT:    .cfi_restore s0
+; RV64IZCMP-SR-NEXT:    .cfi_restore s1
+; RV64IZCMP-SR-NEXT:    .cfi_restore s2
+; RV64IZCMP-SR-NEXT:    .cfi_restore s3
+; RV64IZCMP-SR-NEXT:    .cfi_restore s4
+; RV64IZCMP-SR-NEXT:    .cfi_restore s5
+; RV64IZCMP-SR-NEXT:    .cfi_restore s6
+; RV64IZCMP-SR-NEXT:    .cfi_restore s7
+; RV64IZCMP-SR-NEXT:    .cfi_restore s8
+; RV64IZCMP-SR-NEXT:    .cfi_restore s9
+; RV64IZCMP-SR-NEXT:    .cfi_restore s10
+; RV64IZCMP-SR-NEXT:    .cfi_restore s11
+; RV64IZCMP-SR-NEXT:    .cfi_def_cfa_offset 0
 ; RV64IZCMP-SR-NEXT:    mret
 ;
 ; RV32I-LABEL: callee_with_irq:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -144
+; RV32I-NEXT:    .cfi_def_cfa_offset 144
 ; RV32I-NEXT:    sw ra, 140(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    sw t0, 136(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    sw t1, 132(sp) # 4-byte Folded Spill
@@ -2445,6 +3010,34 @@ define void @callee_with_irq() nounwind "interrupt"="user" {
 ; RV32I-NEXT:    sw t4, 40(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    sw t5, 36(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    sw t6, 32(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    .cfi_offset ra, -4
+; RV32I-NEXT:    .cfi_offset t0, -8
+; RV32I-NEXT:    .cfi_offset t1, -12
+; RV32I-NEXT:    .cfi_offset t2, -16
+; RV32I-NEXT:    .cfi_offset s0, -20
+; RV32I-NEXT:    .cfi_offset s1, -24
+; RV32I-NEXT:    .cfi_offset a0, -28
+; RV32I-NEXT:    .cfi_offset a1, -32
+; RV32I-NEXT:    .cfi_offset a2, -36
+; RV32I-NEXT:    .cfi_offset a3, -40
+; RV32I-NEXT:    .cfi_offset a4, -44
+; RV32I-NEXT:    .cfi_offset a5, -48
+; RV32I-NEXT:    .cfi_offset a6, -52
+; RV32I-NEXT:    .cfi_offset a7, -56
+; RV32I-NEXT:    .cfi_offset s2, -60
+; RV32I-NEXT:    .cfi_offset s3, -64
+; RV32I-NEXT:    .cfi_offset s4, -68
+; RV32I-NEXT:    .cfi_offset s5, -72
+; RV32I-NEXT:    .cfi_offset s6, -76
+; RV32I-NEXT:    .cfi_offset s7, -80
+; RV32I-NEXT:    .cfi_offset s8, -84
+; RV32I-NEXT:    .cfi_offset s9, -88
+; RV32I-NEXT:    .cfi_offset s10, -92
+; RV32I-NEXT:    .cfi_offset s11, -96
+; RV32I-NEXT:    .cfi_offset t3, -100
+; RV32I-NEXT:    .cfi_offset t4, -104
+; RV32I-NEXT:    .cfi_offset t5, -108
+; RV32I-NEXT:    .cfi_offset t6, -112
 ; RV32I-NEXT:    lui a7, %hi(var_test_irq)
 ; RV32I-NEXT:    lw a0, %lo(var_test_irq)(a7)
 ; RV32I-NEXT:    sw a0, 28(sp) # 4-byte Folded Spill
@@ -2551,12 +3144,42 @@ define void @callee_with_irq() nounwind "interrupt"="user" {
 ; RV32I-NEXT:    lw t4, 40(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    lw t5, 36(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    lw t6, 32(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    .cfi_restore ra
+; RV32I-NEXT:    .cfi_restore t0
+; RV32I-NEXT:    .cfi_restore t1
+; RV32I-NEXT:    .cfi_restore t2
+; RV32I-NEXT:    .cfi_restore s0
+; RV32I-NEXT:    .cfi_restore s1
+; RV32I-NEXT:    .cfi_restore a0
+; RV32I-NEXT:    .cfi_restore a1
+; RV32I-NEXT:    .cfi_restore a2
+; RV32I-NEXT:    .cfi_restore a3
+; RV32I-NEXT:    .cfi_restore a4
+; RV32I-NEXT:    .cfi_restore a5
+; RV32I-NEXT:    .cfi_restore a6
+; RV32I-NEXT:    .cfi_restore a7
+; RV32I-NEXT:    .cfi_restore s2
+; RV32I-NEXT:    .cfi_restore s3
+; RV32I-NEXT:    .cfi_restore s4
+; RV32I-NEXT:    .cfi_restore s5
+; RV32I-NEXT:    .cfi_restore s6
+; RV32I-NEXT:    .cfi_restore s7
+; RV32I-NEXT:    .cfi_restore s8
+; RV32I-NEXT:    .cfi_restore s9
+; RV32I-NEXT:    .cfi_restore s10
+; RV32I-NEXT:    .cfi_restore s11
+; RV32I-NEXT:    .cfi_restore t3
+; RV32I-NEXT:    .cfi_restore t4
+; RV32I-NEXT:    .cfi_restore t5
+; RV32I-NEXT:    .cfi_restore t6
 ; RV32I-NEXT:    addi sp, sp, 144
+; RV32I-NEXT:    .cfi_def_cfa_offset 0
 ; RV32I-NEXT:    mret
 ;
 ; RV64I-LABEL: callee_with_irq:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -272
+; RV64I-NEXT:    .cfi_def_cfa_offset 272
 ; RV64I-NEXT:    sd ra, 264(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    sd t0, 256(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    sd t1, 248(sp) # 8-byte Folded Spill
@@ -2585,6 +3208,34 @@ define void @callee_with_irq() nounwind "interrupt"="user" {
 ; RV64I-NEXT:    sd t4, 64(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    sd t5, 56(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    sd t6, 48(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    .cfi_offset ra, -8
+; RV64I-NEXT:    .cfi_offset t0, -16
+; RV64I-NEXT:    .cfi_offset t1, -24
+; RV64I-NEXT:    .cfi_offset t2, -32
+; RV64I-NEXT:    .cfi_offset s0, -40
+; RV64I-NEXT:    .cfi_offset s1, -48
+; RV64I-NEXT:    .cfi_offset a0, -56
+; RV64I-NEXT:    .cfi_offset a1, -64
+; RV64I-NEXT:    .cfi_offset a2, -72
+; RV64I-NEXT:    .cfi_offset a3, -80
+; RV64I-NEXT:    .cfi_offset a4, -88
+; RV64I-NEXT:    .cfi_offset a5, -96
+; RV64I-NEXT:    .cfi_offset a6, -104
+; RV64I-NEXT:    .cfi_offset a7, -112
+; RV64I-NEXT:    .cfi_offset s2, -120
+; RV64I-NEXT:    .cfi_offset s3, -128
+; RV64I-NEXT:    .cfi_offset s4, -136
+; RV64I-NEXT:    .cfi_offset s5, -144
+; RV64I-NEXT:    .cfi_offset s6, -152
+; RV64I-NEXT:    .cfi_offset s7, -160
+; RV64I-NEXT:    .cfi_offset s8, -168
+; RV64I-NEXT:    .cfi_offset s9, -176
+; RV64I-NEXT:    .cfi_offset s10, -184
+; RV64I-NEXT:    .cfi_offset s11, -192
+; RV64I-NEXT:    .cfi_offset t3, -200
+; RV64I-NEXT:    .cfi_offset t4, -208
+; RV64I-NEXT:    .cfi_offset t5, -216
+; RV64I-NEXT:    .cfi_offset t6, -224
 ; RV64I-NEXT:    lui a7, %hi(var_test_irq)
 ; RV64I-NEXT:    lw a0, %lo(var_test_irq)(a7)
 ; RV64I-NEXT:    sd a0, 40(sp) # 8-byte Folded Spill
@@ -2691,31 +3342,74 @@ define void @callee_with_irq() nounwind "interrupt"="user" {
 ; RV64I-NEXT:    ld t4, 64(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    ld t5, 56(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    ld t6, 48(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    .cfi_restore ra
+; RV64I-NEXT:    .cfi_restore t0
+; RV64I-NEXT:    .cfi_restore t1
+; RV64I-NEXT:    .cfi_restore t2
+; RV64I-NEXT:    .cfi_restore s0
+; RV64I-NEXT:    .cfi_restore s1
+; RV64I-NEXT:    .cfi_restore a0
+; RV64I-NEXT:    .cfi_restore a1
+; RV64I-NEXT:    .cfi_restore a2
+; RV64I-NEXT:    .cfi_restore a3
+; RV64I-NEXT:    .cfi_restore a4
+; RV64I-NEXT:    .cfi_restore a5
+; RV64I-NEXT:    .cfi_restore a6
+; RV64I-NEXT:    .cfi_restore a7
+; RV64I-NEXT:    .cfi_restore s2
+; RV64I-NEXT:    .cfi_restore s3
+; RV64I-NEXT:    .cfi_restore s4
+; RV64I-NEXT:    .cfi_restore s5
+; RV64I-NEXT:    .cfi_restore s6
+; RV64I-NEXT:    .cfi_restore s7
+; RV64I-NEXT:    .cfi_restore s8
+; RV64I-NEXT:    .cfi_restore s9
+; RV64I-NEXT:    .cfi_restore s10
+; RV64I-NEXT:    .cfi_restore s11
+; RV64I-NEXT:    .cfi_restore t3
+; RV64I-NEXT:    .cfi_restore t4
+; RV64I-NEXT:    .cfi_restore t5
+; RV64I-NEXT:    .cfi_restore t6
 ; RV64I-NEXT:    addi sp, sp, 272
+; RV64I-NEXT:    .cfi_def_cfa_offset 0
 ; RV64I-NEXT:    mret
   %val = load [32 x i32], ptr @var_test_irq
   store volatile [32 x i32] %val, ptr @var_test_irq
   ret void
 }
 
-define void @callee_no_irq() nounwind{
+define void @callee_no_irq() {
 ; RV32IZCMP-LABEL: callee_no_irq:
 ; RV32IZCMP:       # %bb.0:
-; RV32IZCMP-NEXT:    cm.push {ra, s0-s11}, -96
+; RV32IZCMP-NEXT:    cm.push {ra, s0-s11}, -80
+; RV32IZCMP-NEXT:    .cfi_def_cfa_offset 80
+; RV32IZCMP-NEXT:    .cfi_offset ra, -52
+; RV32IZCMP-NEXT:    .cfi_offset s0, -48
+; RV32IZCMP-NEXT:    .cfi_offset s1, -44
+; RV32IZCMP-NEXT:    .cfi_offset s2, -40
+; RV32IZCMP-NEXT:    .cfi_offset s3, -36
+; RV32IZCMP-NEXT:    .cfi_offset s4, -32
+; RV32IZCMP-NEXT:    .cfi_offset s5, -28
+; RV32IZCMP-NEXT:    .cfi_offset s6, -24
+; RV32IZCMP-NEXT:    .cfi_offset s7, -20
+; RV32IZCMP-NEXT:    .cfi_offset s8, -16
+; RV32IZCMP-NEXT:    .cfi_offset s9, -12
+; RV32IZCMP-NEXT:    .cfi_offset s10, -8
+; RV32IZCMP-NEXT:    .cfi_offset s11, -4
 ; RV32IZCMP-NEXT:    lui t0, %hi(var_test_irq)
 ; RV32IZCMP-NEXT:    lw a0, %lo(var_test_irq)(t0)
-; RV32IZCMP-NEXT:    sw a0, 28(sp) # 4-byte Folded Spill
-; RV32IZCMP-NEXT:    lw a0, %lo(var_test_irq+4)(t0)
 ; RV32IZCMP-NEXT:    sw a0, 24(sp) # 4-byte Folded Spill
-; RV32IZCMP-NEXT:    lw a0, %lo(var_test_irq+8)(t0)
+; RV32IZCMP-NEXT:    lw a0, %lo(var_test_irq+4)(t0)
 ; RV32IZCMP-NEXT:    sw a0, 20(sp) # 4-byte Folded Spill
-; RV32IZCMP-NEXT:    lw a0, %lo(var_test_irq+12)(t0)
+; RV32IZCMP-NEXT:    lw a0, %lo(var_test_irq+8)(t0)
 ; RV32IZCMP-NEXT:    sw a0, 16(sp) # 4-byte Folded Spill
+; RV32IZCMP-NEXT:    lw a0, %lo(var_test_irq+12)(t0)
+; RV32IZCMP-NEXT:    sw a0, 12(sp) # 4-byte Folded Spill
 ; RV32IZCMP-NEXT:    addi a5, t0, %lo(var_test_irq)
 ; RV32IZCMP-NEXT:    lw a0, 16(a5)
-; RV32IZCMP-NEXT:    sw a0, 12(sp) # 4-byte Folded Spill
-; RV32IZCMP-NEXT:    lw a0, 20(a5)
 ; RV32IZCMP-NEXT:    sw a0, 8(sp) # 4-byte Folded Spill
+; RV32IZCMP-NEXT:    lw a0, 20(a5)
+; RV32IZCMP-NEXT:    sw a0, 4(sp) # 4-byte Folded Spill
 ; RV32IZCMP-NEXT:    lw t4, 24(a5)
 ; RV32IZCMP-NEXT:    lw t5, 28(a5)
 ; RV32IZCMP-NEXT:    lw t6, 32(a5)
@@ -2768,37 +3462,51 @@ define void @callee_no_irq() nounwind{
 ; RV32IZCMP-NEXT:    sw t6, 32(a5)
 ; RV32IZCMP-NEXT:    sw t5, 28(a5)
 ; RV32IZCMP-NEXT:    sw t4, 24(a5)
-; RV32IZCMP-NEXT:    lw a0, 8(sp) # 4-byte Folded Reload
+; RV32IZCMP-NEXT:    lw a0, 4(sp) # 4-byte Folded Reload
 ; RV32IZCMP-NEXT:    sw a0, 20(a5)
-; RV32IZCMP-NEXT:    lw a0, 12(sp) # 4-byte Folded Reload
+; RV32IZCMP-NEXT:    lw a0, 8(sp) # 4-byte Folded Reload
 ; RV32IZCMP-NEXT:    sw a0, 16(a5)
-; RV32IZCMP-NEXT:    lw a0, 16(sp) # 4-byte Folded Reload
+; RV32IZCMP-NEXT:    lw a0, 12(sp) # 4-byte Folded Reload
 ; RV32IZCMP-NEXT:    sw a0, %lo(var_test_irq+12)(t0)
-; RV32IZCMP-NEXT:    lw a0, 20(sp) # 4-byte Folded Reload
+; RV32IZCMP-NEXT:    lw a0, 16(sp) # 4-byte Folded Reload
 ; RV32IZCMP-NEXT:    sw a0, %lo(var_test_irq+8)(t0)
-; RV32IZCMP-NEXT:    lw a0, 24(sp) # 4-byte Folded Reload
+; RV32IZCMP-NEXT:    lw a0, 20(sp) # 4-byte Folded Reload
 ; RV32IZCMP-NEXT:    sw a0, %lo(var_test_irq+4)(t0)
-; RV32IZCMP-NEXT:    lw a0, 28(sp) # 4-byte Folded Reload
+; RV32IZCMP-NEXT:    lw a0, 24(sp) # 4-byte Folded Reload
 ; RV32IZCMP-NEXT:    sw a0, %lo(var_test_irq)(t0)
-; RV32IZCMP-NEXT:    cm.popret {ra, s0-s11}, 96
+; RV32IZCMP-NEXT:    cm.popret {ra, s0-s11}, 80
 ;
 ; RV64IZCMP-LABEL: callee_no_irq:
 ; RV64IZCMP:       # %bb.0:
 ; RV64IZCMP-NEXT:    cm.push {ra, s0-s11}, -160
+; RV64IZCMP-NEXT:    .cfi_def_cfa_offset 160
+; RV64IZCMP-NEXT:    .cfi_offset ra, -104
+; RV64IZCMP-NEXT:    .cfi_offset s0, -96
+; RV64IZCMP-NEXT:    .cfi_offset s1, -88
+; RV64IZCMP-NEXT:    .cfi_offset s2, -80
+; RV64IZCMP-NEXT:    .cfi_offset s3, -72
+; RV64IZCMP-NEXT:    .cfi_offset s4, -64
+; RV64IZCMP-NEXT:    .cfi_offset s5, -56
+; RV64IZCMP-NEXT:    .cfi_offset s6, -48
+; RV64IZCMP-NEXT:    .cfi_offset s7, -40
+; RV64IZCMP-NEXT:    .cfi_offset s8, -32
+; RV64IZCMP-NEXT:    .cfi_offset s9, -24
+; RV64IZCMP-NEXT:    .cfi_offset s10, -16
+; RV64IZCMP-NEXT:    .cfi_offset s11, -8
 ; RV64IZCMP-NEXT:    lui t0, %hi(var_test_irq)
 ; RV64IZCMP-NEXT:    lw a0, %lo(var_test_irq)(t0)
-; RV64IZCMP-NEXT:    sd a0, 40(sp) # 8-byte Folded Spill
+; RV64IZCMP-NEXT:    sd a0, 48(sp) # 8-byte Folded Spill
 ; RV64IZCMP-NEXT:    lw a0, %lo(var_test_irq+4)(t0)
-; RV64IZCMP-NEXT:    sd a0, 32(sp) # 8-byte Folded Spill
+; RV64IZCMP-NEXT:    sd a0, 40(sp) # 8-byte Folded Spill
 ; RV64IZCMP-NEXT:    lw a0, %lo(var_test_irq+8)(t0)
-; RV64IZCMP-NEXT:    sd a0, 24(sp) # 8-byte Folded Spill
+; RV64IZCMP-NEXT:    sd a0, 32(sp) # 8-byte Folded Spill
 ; RV64IZCMP-NEXT:    lw a0, %lo(var_test_irq+12)(t0)
-; RV64IZCMP-NEXT:    sd a0, 16(sp) # 8-byte Folded Spill
+; RV64IZCMP-NEXT:    sd a0, 24(sp) # 8-byte Folded Spill
 ; RV64IZCMP-NEXT:    addi a5, t0, %lo(var_test_irq)
 ; RV64IZCMP-NEXT:    lw a0, 16(a5)
-; RV64IZCMP-NEXT:    sd a0, 8(sp) # 8-byte Folded Spill
+; RV64IZCMP-NEXT:    sd a0, 16(sp) # 8-byte Folded Spill
 ; RV64IZCMP-NEXT:    lw a0, 20(a5)
-; RV64IZCMP-NEXT:    sd a0, 0(sp) # 8-byte Folded Spill
+; RV64IZCMP-NEXT:    sd a0, 8(sp) # 8-byte Folded Spill
 ; RV64IZCMP-NEXT:    lw t4, 24(a5)
 ; RV64IZCMP-NEXT:    lw t5, 28(a5)
 ; RV64IZCMP-NEXT:    lw t6, 32(a5)
@@ -2851,37 +3559,51 @@ define void @callee_no_irq() nounwind{
 ; RV64IZCMP-NEXT:    sw t6, 32(a5)
 ; RV64IZCMP-NEXT:    sw t5, 28(a5)
 ; RV64IZCMP-NEXT:    sw t4, 24(a5)
-; RV64IZCMP-NEXT:    ld a0, 0(sp) # 8-byte Folded Reload
-; RV64IZCMP-NEXT:    sw a0, 20(a5)
 ; RV64IZCMP-NEXT:    ld a0, 8(sp) # 8-byte Folded Reload
-; RV64IZCMP-NEXT:    sw a0, 16(a5)
+; RV64IZCMP-NEXT:    sw a0, 20(a5)
 ; RV64IZCMP-NEXT:    ld a0, 16(sp) # 8-byte Folded Reload
-; RV64IZCMP-NEXT:    sw a0, %lo(var_test_irq+12)(t0)
+; RV64IZCMP-NEXT:    sw a0, 16(a5)
 ; RV64IZCMP-NEXT:    ld a0, 24(sp) # 8-byte Folded Reload
-; RV64IZCMP-NEXT:    sw a0, %lo(var_test_irq+8)(t0)
+; RV64IZCMP-NEXT:    sw a0, %lo(var_test_irq+12)(t0)
 ; RV64IZCMP-NEXT:    ld a0, 32(sp) # 8-byte Folded Reload
-; RV64IZCMP-NEXT:    sw a0, %lo(var_test_irq+4)(t0)
+; RV64IZCMP-NEXT:    sw a0, %lo(var_test_irq+8)(t0)
 ; RV64IZCMP-NEXT:    ld a0, 40(sp) # 8-byte Folded Reload
+; RV64IZCMP-NEXT:    sw a0, %lo(var_test_irq+4)(t0)
+; RV64IZCMP-NEXT:    ld a0, 48(sp) # 8-byte Folded Reload
 ; RV64IZCMP-NEXT:    sw a0, %lo(var_test_irq)(t0)
 ; RV64IZCMP-NEXT:    cm.popret {ra, s0-s11}, 160
 ;
 ; RV32IZCMP-SR-LABEL: callee_no_irq:
 ; RV32IZCMP-SR:       # %bb.0:
-; RV32IZCMP-SR-NEXT:    cm.push {ra, s0-s11}, -96
+; RV32IZCMP-SR-NEXT:    cm.push {ra, s0-s11}, -80
+; RV32IZCMP-SR-NEXT:    .cfi_def_cfa_offset 80
+; RV32IZCMP-SR-NEXT:    .cfi_offset ra, -52
+; RV32IZCMP-SR-NEXT:    .cfi_offset s0, -48
+; RV32IZCMP-SR-NEXT:    .cfi_offset s1, -44
+; RV32IZCMP-SR-NEXT:    .cfi_offset s2, -40
+; RV32IZCMP-SR-NEXT:    .cfi_offset s3, -36
+; RV32IZCMP-SR-NEXT:    .cfi_offset s4, -32
+; RV32IZCMP-SR-NEXT:    .cfi_offset s5, -28
+; RV32IZCMP-SR-NEXT:    .cfi_offset s6, -24
+; RV32IZCMP-SR-NEXT:    .cfi_offset s7, -20
+; RV32IZCMP-SR-NEXT:    .cfi_offset s8, -16
+; RV32IZCMP-SR-NEXT:    .cfi_offset s9, -12
+; RV32IZCMP-SR-NEXT:    .cfi_offset s10, -8
+; RV32IZCMP-SR-NEXT:    .cfi_offset s11, -4
 ; RV32IZCMP-SR-NEXT:    lui t0, %hi(var_test_irq)
 ; RV32IZCMP-SR-NEXT:    lw a0, %lo(var_test_irq)(t0)
-; RV32IZCMP-SR-NEXT:    sw a0, 28(sp) # 4-byte Folded Spill
-; RV32IZCMP-SR-NEXT:    lw a0, %lo(var_test_irq+4)(t0)
 ; RV32IZCMP-SR-NEXT:    sw a0, 24(sp) # 4-byte Folded Spill
-; RV32IZCMP-SR-NEXT:    lw a0, %lo(var_test_irq+8)(t0)
+; RV32IZCMP-SR-NEXT:    lw a0, %lo(var_test_irq+4)(t0)
 ; RV32IZCMP-SR-NEXT:    sw a0, 20(sp) # 4-byte Folded Spill
-; RV32IZCMP-SR-NEXT:    lw a0, %lo(var_test_irq+12)(t0)
+; RV32IZCMP-SR-NEXT:    lw a0, %lo(var_test_irq+8)(t0)
 ; RV32IZCMP-SR-NEXT:    sw a0, 16(sp) # 4-byte Folded Spill
+; RV32IZCMP-SR-NEXT:    lw a0, %lo(var_test_irq+12)(t0)
+; RV32IZCMP-SR-NEXT:    sw a0, 12(sp) # 4-byte Folded Spill
 ; RV32IZCMP-SR-NEXT:    addi a5, t0, %lo(var_test_irq)
 ; RV32IZCMP-SR-NEXT:    lw a0, 16(a5)
-; RV32IZCMP-SR-NEXT:    sw a0, 12(sp) # 4-byte Folded Spill
-; RV32IZCMP-SR-NEXT:    lw a0, 20(a5)
 ; RV32IZCMP-SR-NEXT:    sw a0, 8(sp) # 4-byte Folded Spill
+; RV32IZCMP-SR-NEXT:    lw a0, 20(a5)
+; RV32IZCMP-SR-NEXT:    sw a0, 4(sp) # 4-byte Folded Spill
 ; RV32IZCMP-SR-NEXT:    lw t4, 24(a5)
 ; RV32IZCMP-SR-NEXT:    lw t5, 28(a5)
 ; RV32IZCMP-SR-NEXT:    lw t6, 32(a5)
@@ -2934,37 +3656,51 @@ define void @callee_no_irq() nounwind{
 ; RV32IZCMP-SR-NEXT:    sw t6, 32(a5)
 ; RV32IZCMP-SR-NEXT:    sw t5, 28(a5)
 ; RV32IZCMP-SR-NEXT:    sw t4, 24(a5)
-; RV32IZCMP-SR-NEXT:    lw a0, 8(sp) # 4-byte Folded Reload
+; RV32IZCMP-SR-NEXT:    lw a0, 4(sp) # 4-byte Folded Reload
 ; RV32IZCMP-SR-NEXT:    sw a0, 20(a5)
-; RV32IZCMP-SR-NEXT:    lw a0, 12(sp) # 4-byte Folded Reload
+; RV32IZCMP-SR-NEXT:    lw a0, 8(sp) # 4-byte Folded Reload
 ; RV32IZCMP-SR-NEXT:    sw a0, 16(a5)
-; RV32IZCMP-SR-NEXT:    lw a0, 16(sp) # 4-byte Folded Reload
+; RV32IZCMP-SR-NEXT:    lw a0, 12(sp) # 4-byte Folded Reload
 ; RV32IZCMP-SR-NEXT:    sw a0, %lo(var_test_irq+12)(t0)
-; RV32IZCMP-SR-NEXT:    lw a0, 20(sp) # 4-byte Folded Reload
+; RV32IZCMP-SR-NEXT:    lw a0, 16(sp) # 4-byte Folded Reload
 ; RV32IZCMP-SR-NEXT:    sw a0, %lo(var_test_irq+8)(t0)
-; RV32IZCMP-SR-NEXT:    lw a0, 24(sp) # 4-byte Folded Reload
+; RV32IZCMP-SR-NEXT:    lw a0, 20(sp) # 4-byte Folded Reload
 ; RV32IZCMP-SR-NEXT:    sw a0, %lo(var_test_irq+4)(t0)
-; RV32IZCMP-SR-NEXT:    lw a0, 28(sp) # 4-byte Folded Reload
+; RV32IZCMP-SR-NEXT:    lw a0, 24(sp) # 4-byte Folded Reload
 ; RV32IZCMP-SR-NEXT:    sw a0, %lo(var_test_irq)(t0)
-; RV32IZCMP-SR-NEXT:    cm.popret {ra, s0-s11}, 96
+; RV32IZCMP-SR-NEXT:    cm.popret {ra, s0-s11}, 80
 ;
 ; RV64IZCMP-SR-LABEL: callee_no_irq:
 ; RV64IZCMP-SR:       # %bb.0:
 ; RV64IZCMP-SR-NEXT:    cm.push {ra, s0-s11}, -160
+; RV64IZCMP-SR-NEXT:    .cfi_def_cfa_offset 160
+; RV64IZCMP-SR-NEXT:    .cfi_offset ra, -104
+; RV64IZCMP-SR-NEXT:    .cfi_offset s0, -96
+; RV64IZCMP-SR-NEXT:    .cfi_offset s1, -88
+; RV64IZCMP-SR-NEXT:    .cfi_offset s2, -80
+; RV64IZCMP-SR-NEXT:    .cfi_offset s3, -72
+; RV64IZCMP-SR-NEXT:    .cfi_offset s4, -64
+; RV64IZCMP-SR-NEXT:    .cfi_offset s5, -56
+; RV64IZCMP-SR-NEXT:    .cfi_offset s6, -48
+; RV64IZCMP-SR-NEXT:    .cfi_offset s7, -40
+; RV64IZCMP-SR-NEXT:    .cfi_offset s8, -32
+; RV64IZCMP-SR-NEXT:    .cfi_offset s9, -24
+; RV64IZCMP-SR-NEXT:    .cfi_offset s10, -16
+; RV64IZCMP-SR-NEXT:    .cfi_offset s11, -8
 ; RV64IZCMP-SR-NEXT:    lui t0, %hi(var_test_irq)
 ; RV64IZCMP-SR-NEXT:    lw a0, %lo(var_test_irq)(t0)
-; RV64IZCMP-SR-NEXT:    sd a0, 40(sp) # 8-byte Folded Spill
+; RV64IZCMP-SR-NEXT:    sd a0, 48(sp) # 8-byte Folded Spill
 ; RV64IZCMP-SR-NEXT:    lw a0, %lo(var_test_irq+4)(t0)
-; RV64IZCMP-SR-NEXT:    sd a0, 32(sp) # 8-byte Folded Spill
+; RV64IZCMP-SR-NEXT:    sd a0, 40(sp) # 8-byte Folded Spill
 ; RV64IZCMP-SR-NEXT:    lw a0, %lo(var_test_irq+8)(t0)
-; RV64IZCMP-SR-NEXT:    sd a0, 24(sp) # 8-byte Folded Spill
+; RV64IZCMP-SR-NEXT:    sd a0, 32(sp) # 8-byte Folded Spill
 ; RV64IZCMP-SR-NEXT:    lw a0, %lo(var_test_irq+12)(t0)
-; RV64IZCMP-SR-NEXT:    sd a0, 16(sp) # 8-byte Folded Spill
+; RV64IZCMP-SR-NEXT:    sd a0, 24(sp) # 8-byte Folded Spill
 ; RV64IZCMP-SR-NEXT:    addi a5, t0, %lo(var_test_irq)
 ; RV64IZCMP-SR-NEXT:    lw a0, 16(a5)
-; RV64IZCMP-SR-NEXT:    sd a0, 8(sp) # 8-byte Folded Spill
+; RV64IZCMP-SR-NEXT:    sd a0, 16(sp) # 8-byte Folded Spill
 ; RV64IZCMP-SR-NEXT:    lw a0, 20(a5)
-; RV64IZCMP-SR-NEXT:    sd a0, 0(sp) # 8-byte Folded Spill
+; RV64IZCMP-SR-NEXT:    sd a0, 8(sp) # 8-byte Folded Spill
 ; RV64IZCMP-SR-NEXT:    lw t4, 24(a5)
 ; RV64IZCMP-SR-NEXT:    lw t5, 28(a5)
 ; RV64IZCMP-SR-NEXT:    lw t6, 32(a5)
@@ -3017,23 +3753,24 @@ define void @callee_no_irq() nounwind{
 ; RV64IZCMP-SR-NEXT:    sw t6, 32(a5)
 ; RV64IZCMP-SR-NEXT:    sw t5, 28(a5)
 ; RV64IZCMP-SR-NEXT:    sw t4, 24(a5)
-; RV64IZCMP-SR-NEXT:    ld a0, 0(sp) # 8-byte Folded Reload
-; RV64IZCMP-SR-NEXT:    sw a0, 20(a5)
 ; RV64IZCMP-SR-NEXT:    ld a0, 8(sp) # 8-byte Folded Reload
-; RV64IZCMP-SR-NEXT:    sw a0, 16(a5)
+; RV64IZCMP-SR-NEXT:    sw a0, 20(a5)
 ; RV64IZCMP-SR-NEXT:    ld a0, 16(sp) # 8-byte Folded Reload
-; RV64IZCMP-SR-NEXT:    sw a0, %lo(var_test_irq+12)(t0)
+; RV64IZCMP-SR-NEXT:    sw a0, 16(a5)
 ; RV64IZCMP-SR-NEXT:    ld a0, 24(sp) # 8-byte Folded Reload
-; RV64IZCMP-SR-NEXT:    sw a0, %lo(var_test_irq+8)(t0)
+; RV64IZCMP-SR-NEXT:    sw a0, %lo(var_test_irq+12)(t0)
 ; RV64IZCMP-SR-NEXT:    ld a0, 32(sp) # 8-byte Folded Reload
-; RV64IZCMP-SR-NEXT:    sw a0, %lo(var_test_irq+4)(t0)
+; RV64IZCMP-SR-NEXT:    sw a0, %lo(var_test_irq+8)(t0)
 ; RV64IZCMP-SR-NEXT:    ld a0, 40(sp) # 8-byte Folded Reload
+; RV64IZCMP-SR-NEXT:    sw a0, %lo(var_test_irq+4)(t0)
+; RV64IZCMP-SR-NEXT:    ld a0, 48(sp) # 8-byte Folded Reload
 ; RV64IZCMP-SR-NEXT:    sw a0, %lo(var_test_irq)(t0)
 ; RV64IZCMP-SR-NEXT:    cm.popret {ra, s0-s11}, 160
 ;
 ; RV32I-LABEL: callee_no_irq:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -80
+; RV32I-NEXT:    .cfi_def_cfa_offset 80
 ; RV32I-NEXT:    sw ra, 76(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    sw s0, 72(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    sw s1, 68(sp) # 4-byte Folded Spill
@@ -3047,6 +3784,19 @@ define void @callee_no_irq() nounwind{
 ; RV32I-NEXT:    sw s9, 36(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    sw s10, 32(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    sw s11, 28(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    .cfi_offset ra, -4
+; RV32I-NEXT:    .cfi_offset s0, -8
+; RV32I-NEXT:    .cfi_offset s1, -12
+; RV32I-NEXT:    .cfi_offset s2, -16
+; RV32I-NEXT:    .cfi_offset s3, -20
+; RV32I-NEXT:    .cfi_offset s4, -24
+; RV32I-NEXT:    .cfi_offset s5, -28
+; RV32I-NEXT:    .cfi_offset s6, -32
+; RV32I-NEXT:    .cfi_offset s7, -36
+; RV32I-NEXT:    .cfi_offset s8, -40
+; RV32I-NEXT:    .cfi_offset s9, -44
+; RV32I-NEXT:    .cfi_offset s10, -48
+; RV32I-NEXT:    .cfi_offset s11, -52
 ; RV32I-NEXT:    lui a7, %hi(var_test_irq)
 ; RV32I-NEXT:    lw a0, %lo(var_test_irq)(a7)
 ; RV32I-NEXT:    sw a0, 24(sp) # 4-byte Folded Spill
@@ -3138,12 +3888,27 @@ define void @callee_no_irq() nounwind{
 ; RV32I-NEXT:    lw s9, 36(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    lw s10, 32(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    lw s11, 28(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    .cfi_restore ra
+; RV32I-NEXT:    .cfi_restore s0
+; RV32I-NEXT:    .cfi_restore s1
+; RV32I-NEXT:    .cfi_restore s2
+; RV32I-NEXT:    .cfi_restore s3
+; RV32I-NEXT:    .cfi_restore s4
+; RV32I-NEXT:    .cfi_restore s5
+; RV32I-NEXT:    .cfi_restore s6
+; RV32I-NEXT:    .cfi_restore s7
+; RV32I-NEXT:    .cfi_restore s8
+; RV32I-NEXT:    .cfi_restore s9
+; RV32I-NEXT:    .cfi_restore s10
+; RV32I-NEXT:    .cfi_restore s11
 ; RV32I-NEXT:    addi sp, sp, 80
+; RV32I-NEXT:    .cfi_def_cfa_offset 0
 ; RV32I-NEXT:    ret
 ;
 ; RV64I-LABEL: callee_no_irq:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -160
+; RV64I-NEXT:    .cfi_def_cfa_offset 160
 ; RV64I-NEXT:    sd ra, 152(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    sd s0, 144(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    sd s1, 136(sp) # 8-byte Folded Spill
@@ -3157,6 +3922,19 @@ define void @callee_no_irq() nounwind{
 ; RV64I-NEXT:    sd s9, 72(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    sd s10, 64(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    sd s11, 56(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    .cfi_offset ra, -8
+; RV64I-NEXT:    .cfi_offset s0, -16
+; RV64I-NEXT:    .cfi_offset s1, -24
+; RV64I-NEXT:    .cfi_offset s2, -32
+; RV64I-NEXT:    .cfi_offset s3, -40
+; RV64I-NEXT:    .cfi_offset s4, -48
+; RV64I-NEXT:    .cfi_offset s5, -56
+; RV64I-NEXT:    .cfi_offset s6, -64
+; RV64I-NEXT:    .cfi_offset s7, -72
+; RV64I-NEXT:    .cfi_offset s8, -80
+; RV64I-NEXT:    .cfi_offset s9, -88
+; RV64I-NEXT:    .cfi_offset s10, -96
+; RV64I-NEXT:    .cfi_offset s11, -104
 ; RV64I-NEXT:    lui a7, %hi(var_test_irq)
 ; RV64I-NEXT:    lw a0, %lo(var_test_irq)(a7)
 ; RV64I-NEXT:    sd a0, 48(sp) # 8-byte Folded Spill
@@ -3248,7 +4026,21 @@ define void @callee_no_irq() nounwind{
 ; RV64I-NEXT:    ld s9, 72(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    ld s10, 64(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    ld s11, 56(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    .cfi_restore ra
+; RV64I-NEXT:    .cfi_restore s0
+; RV64I-NEXT:    .cfi_restore s1
+; RV64I-NEXT:    .cfi_restore s2
+; RV64I-NEXT:    .cfi_restore s3
+; RV64I-NEXT:    .cfi_restore s4
+; RV64I-NEXT:    .cfi_restore s5
+; RV64I-NEXT:    .cfi_restore s6
+; RV64I-NEXT:    .cfi_restore s7
+; RV64I-NEXT:    .cfi_restore s8
+; RV64I-NEXT:    .cfi_restore s9
+; RV64I-NEXT:    .cfi_restore s10
+; RV64I-NEXT:    .cfi_restore s11
 ; RV64I-NEXT:    addi sp, sp, 160
+; RV64I-NEXT:    .cfi_def_cfa_offset 0
 ; RV64I-NEXT:    ret
   %val = load [32 x i32], ptr @var_test_irq
   store volatile [32 x i32] %val, ptr @var_test_irq
@@ -3261,71 +4053,71 @@ declare ptr @llvm.frameaddress.p0(i32 immarg)
 define i32 @use_fp(i32 %x) {
 ; RV32IZCMP-LABEL: use_fp:
 ; RV32IZCMP:       # %bb.0: # %entry
-; RV32IZCMP-NEXT:    cm.push {ra, s0-s1}, -32
-; RV32IZCMP-NEXT:    .cfi_def_cfa_offset 32
+; RV32IZCMP-NEXT:    cm.push {ra, s0-s1}, -16
+; RV32IZCMP-NEXT:    .cfi_def_cfa_offset 16
 ; RV32IZCMP-NEXT:    .cfi_offset ra, -12
 ; RV32IZCMP-NEXT:    .cfi_offset s0, -8
 ; RV32IZCMP-NEXT:    .cfi_offset s1, -4
-; RV32IZCMP-NEXT:    addi s0, sp, 32
+; RV32IZCMP-NEXT:    addi s0, sp, 16
 ; RV32IZCMP-NEXT:    .cfi_def_cfa s0, 0
 ; RV32IZCMP-NEXT:    mv s1, a0
-; RV32IZCMP-NEXT:    addi a1, s0, -20
+; RV32IZCMP-NEXT:    addi a1, s0, -16
 ; RV32IZCMP-NEXT:    mv a0, s0
 ; RV32IZCMP-NEXT:    call bar
 ; RV32IZCMP-NEXT:    mv a0, s1
-; RV32IZCMP-NEXT:    .cfi_def_cfa sp, 32
-; RV32IZCMP-NEXT:    cm.popret {ra, s0-s1}, 32
+; RV32IZCMP-NEXT:    .cfi_def_cfa sp, 16
+; RV32IZCMP-NEXT:    cm.popret {ra, s0-s1}, 16
 ;
 ; RV64IZCMP-LABEL: use_fp:
 ; RV64IZCMP:       # %bb.0: # %entry
-; RV64IZCMP-NEXT:    cm.push {ra, s0-s1}, -48
-; RV64IZCMP-NEXT:    .cfi_def_cfa_offset 48
+; RV64IZCMP-NEXT:    cm.push {ra, s0-s1}, -32
+; RV64IZCMP-NEXT:    .cfi_def_cfa_offset 32
 ; RV64IZCMP-NEXT:    .cfi_offset ra, -24
 ; RV64IZCMP-NEXT:    .cfi_offset s0, -16
 ; RV64IZCMP-NEXT:    .cfi_offset s1, -8
-; RV64IZCMP-NEXT:    addi s0, sp, 48
+; RV64IZCMP-NEXT:    addi s0, sp, 32
 ; RV64IZCMP-NEXT:    .cfi_def_cfa s0, 0
 ; RV64IZCMP-NEXT:    mv s1, a0
-; RV64IZCMP-NEXT:    addi a1, s0, -36
+; RV64IZCMP-NEXT:    addi a1, s0, -28
 ; RV64IZCMP-NEXT:    mv a0, s0
 ; RV64IZCMP-NEXT:    call bar
 ; RV64IZCMP-NEXT:    mv a0, s1
-; RV64IZCMP-NEXT:    .cfi_def_cfa sp, 48
-; RV64IZCMP-NEXT:    cm.popret {ra, s0-s1}, 48
+; RV64IZCMP-NEXT:    .cfi_def_cfa sp, 32
+; RV64IZCMP-NEXT:    cm.popret {ra, s0-s1}, 32
 ;
 ; RV32IZCMP-SR-LABEL: use_fp:
 ; RV32IZCMP-SR:       # %bb.0: # %entry
-; RV32IZCMP-SR-NEXT:    cm.push {ra, s0-s1}, -32
-; RV32IZCMP-SR-NEXT:    .cfi_def_cfa_offset 32
+; RV32IZCMP-SR-NEXT:    cm.push {ra, s0-s1}, -16
+; RV32IZCMP-SR-NEXT:    .cfi_def_cfa_offset 16
 ; RV32IZCMP-SR-NEXT:    .cfi_offset ra, -12
 ; RV32IZCMP-SR-NEXT:    .cfi_offset s0, -8
 ; RV32IZCMP-SR-NEXT:    .cfi_offset s1, -4
-; RV32IZCMP-SR-NEXT:    addi s0, sp, 32
+; RV32IZCMP-SR-NEXT:    addi s0, sp, 16
 ; RV32IZCMP-SR-NEXT:    .cfi_def_cfa s0, 0
 ; RV32IZCMP-SR-NEXT:    mv s1, a0
-; RV32IZCMP-SR-NEXT:    addi a1, s0, -20
+; RV32IZCMP-SR-NEXT:    addi a1, s0, -16
 ; RV32IZCMP-SR-NEXT:    mv a0, s0
 ; RV32IZCMP-SR-NEXT:    call bar
 ; RV32IZCMP-SR-NEXT:    mv a0, s1
-; RV32IZCMP-SR-NEXT:    .cfi_def_cfa sp, 32
-; RV32IZCMP-SR-NEXT:    cm.popret {ra, s0-s1}, 32
+; RV32IZCMP-SR-NEXT:    .cfi_def_cfa sp, 16
+; RV32IZCMP-SR-NEXT:    cm.popret {ra, s0-s1}, 16
 ;
 ; RV64IZCMP-SR-LABEL: use_fp:
 ; RV64IZCMP-SR:       # %bb.0: # %entry
-; RV64IZCMP-SR-NEXT:    cm.push {ra, s0-s1}, -48
-; RV64IZCMP-SR-NEXT:    .cfi_def_cfa_offset 48
+; RV64IZCMP-SR-NEXT:    cm.push {ra, s0-s1}, -32
+; RV64IZCMP-SR-NEXT:    .cfi_def_cfa_offset 32
 ; RV64IZCMP-SR-NEXT:    .cfi_offset ra, -24
 ; RV64IZCMP-SR-NEXT:    .cfi_offset s0, -16
 ; RV64IZCMP-SR-NEXT:    .cfi_offset s1, -8
-; RV64IZCMP-SR-NEXT:    addi s0, sp, 48
+; RV64IZCMP-SR-NEXT:    addi s0, sp, 32
 ; RV64IZCMP-SR-NEXT:    .cfi_def_cfa s0, 0
 ; RV64IZCMP-SR-NEXT:    mv s1, a0
-; RV64IZCMP-SR-NEXT:    addi a1, s0, -36
+; RV64IZCMP-SR-NEXT:    addi a1, s0, -28
 ; RV64IZCMP-SR-NEXT:    mv a0, s0
 ; RV64IZCMP-SR-NEXT:    call bar
 ; RV64IZCMP-SR-NEXT:    mv a0, s1
-; RV64IZCMP-SR-NEXT:    .cfi_def_cfa sp, 48
-; RV64IZCMP-SR-NEXT:    cm.popret {ra, s0-s1}, 48
+; RV64IZCMP-SR-NEXT:    .cfi_def_cfa sp, 32
+; RV64IZCMP-SR-NEXT:    cm.popret {ra, s0-s1}, 32
 ;
 ; RV32I-LABEL: use_fp:
 ; RV32I:       # %bb.0: # %entry
diff --git a/llvm/test/CodeGen/RISCV/rda-stack.mir b/llvm/test/CodeGen/RISCV/rda-stack.mir
new file mode 100644
index 0000000000000..b3111e662e2e9
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/rda-stack.mir
@@ -0,0 +1,191 @@
+# RUN: llc %s -mtriple=riscv64 -run-pass=reaching-defs-analysis -print-all-reaching-defs -o - 2>&1 | FileCheck %s
+
+---
+name:            test0
+tracksRegLiveness: true
+stack:
+  - { id: 0, name: '', type: default, offset: 0, size: 4, alignment: 4,
+      stack-id: default, callee-saved-register: '', callee-saved-restored: true,
+      debug-info-variable: '', debug-info-expression: '', debug-info-location: '' }
+body:             |
+  ; CHECK-LABEL: RDA results for test0
+  ; CHECK-NEXT: %stack.0:{ }
+  ; CHECK-NEXT:0: $x10 = LD %stack.0, 0 :: (load (s64))
+  ; CHECK-EMPTY:
+  ; CHECK-NEXT: implicit $x10:{ 0 }
+  ; CHECK-NEXT:1: PseudoRET implicit $x10
+
+  bb.0.entry:
+    $x10 = LD %stack.0, 0 :: (load (s64))
+    PseudoRET implicit $x10
+
+...
+---
+name:            test1
+tracksRegLiveness: true
+stack:
+  - { id: 0, name: '', type: default, offset: 0, size: 4, alignment: 4,
+      stack-id: default, callee-saved-register: '', callee-saved-restored: true,
+      debug-info-variable: '', debug-info-expression: '', debug-info-location: '' }
+  - { id: 1, name: '', type: default, offset: 0, size: 4, alignment: 4,
+      stack-id: default, callee-saved-register: '', callee-saved-restored: true,
+      debug-info-variable: '', debug-info-expression: '', debug-info-location: '' }
+body:             |
+  ; CHECK-LABEL: RDA results for test1
+  ; CHECK-NEXT: %stack.0:{ }
+  ; CHECK-NEXT: 0: $x10 = LD %stack.0, 0 :: (load (s64))
+  ; CHECK-EMPTY: 
+  ; CHECK-NEXT: %stack.1:{ }
+  ; CHECK-NEXT: 1: $x11 = LD %stack.1, 0 :: (load (s64))
+  ; CHECK-EMPTY: 
+  ; CHECK-NEXT: $x10:{ 0 }
+  ; CHECK-NEXT: $x11:{ 1 }
+  ; CHECK-NEXT: 2: $x10 = ADD $x10, $x11
+  ; CHECK-EMPTY: 
+  ; CHECK-NEXT: implicit $x10:{ 2 }
+  ; CHECK-NEXT: 3: PseudoRET implicit $x10
+
+  bb.0.entry:
+    $x10 = LD %stack.0, 0 :: (load (s64))
+    $x11 = LD %stack.1, 0 :: (load (s64))
+    $x10 = ADD $x10, $x11
+    PseudoRET implicit $x10
+
+...
+---
+name:            test2
+tracksRegLiveness: true
+stack:
+  - { id: 0, name: '', type: default, offset: 0, size: 4, alignment: 4,
+      stack-id: default, callee-saved-register: '', callee-saved-restored: true,
+      debug-info-variable: '', debug-info-expression: '', debug-info-location: '' }
+  - { id: 1, name: '', type: default, offset: 0, size: 4, alignment: 4,
+      stack-id: default, callee-saved-register: '', callee-saved-restored: true,
+      debug-info-variable: '', debug-info-expression: '', debug-info-location: '' }
+body:             |
+  ; CHECK-LABEL: RDA results for test2
+  ; CHECK-NEXT: %stack.0:{ }
+  ; CHECK-NEXT: 0: $x10 = LD %stack.0, 0 :: (load (s64))
+  ; CHECK-EMPTY: 
+  ; CHECK-NEXT: %stack.1:{ }
+  ; CHECK-NEXT: 1: $x11 = LD %stack.1, 0 :: (load (s64))
+  ; CHECK-EMPTY: 
+  ; CHECK-NEXT: $x10:{ 0 }
+  ; CHECK-NEXT: $x11:{ 1 }
+  ; CHECK-NEXT: 2: $x10 = ADD $x10, $x11
+  ; CHECK-EMPTY: 
+  ; CHECK-NEXT: $x10:{ 2 }
+  ; CHECK-NEXT: %stack.0:{ }
+  ; CHECK-NEXT: 3: SD $x10, %stack.0, 0 :: (store (s64))
+  ; CHECK-EMPTY: 
+  ; CHECK-NEXT: %stack.0:{ 3 }
+  ; CHECK-NEXT: 4: $x10 = LD %stack.0, 0 :: (load (s64))
+  ; CHECK-EMPTY: 
+  ; CHECK-NEXT: implicit $x10:{ 4 }
+  ; CHECK-NEXT: 5: PseudoRET implicit $x10
+
+  bb.0.entry:
+    $x10 = LD %stack.0, 0 :: (load (s64))
+    $x11 = LD %stack.1, 0 :: (load (s64))
+    $x10 = ADD $x10, $x11
+    SD $x10, %stack.0, 0 :: (store (s64))
+    $x10 = LD %stack.0, 0 :: (load (s64))
+    PseudoRET implicit $x10
+
+...
+---
+name:            test3
+tracksRegLiveness: true
+stack:
+  - { id: 0, name: '', type: default, offset: 0, size: 4, alignment: 4,
+      stack-id: default, callee-saved-register: '', callee-saved-restored: true,
+      debug-info-variable: '', debug-info-expression: '', debug-info-location: '' }
+body:             |
+  ; CHECK-LABEL: RDA results for test3
+  ; CHECK-NEXT: $x10:{ }
+  ; CHECK-NEXT: $x0:{ }
+  ; CHECK-NEXT: 0: BEQ $x10, $x0, %bb.2
+  ; CHECK-EMPTY: 
+  ; CHECK-NEXT: $x10:{ }
+  ; CHECK-NEXT: 1: $x10 = ADDI $x10, 1
+  ; CHECK-EMPTY: 
+  ; CHECK-NEXT: $x10:{ 1 }
+  ; CHECK-NEXT: %stack.0:{ }
+  ; CHECK-NEXT: 2: SD $x10, %stack.0, 0 :: (store (s64))
+  ; CHECK-EMPTY: 
+  ; CHECK-NEXT: $x0:{ }
+  ; CHECK-NEXT: $x0:{ }
+  ; CHECK-NEXT: 3: BEQ $x0, $x0, %bb.3
+  ; CHECK-EMPTY: 
+  ; CHECK-NEXT: $x10:{ 1 }
+  ; CHECK-NEXT: 4: $x10 = ADDI $x10, 2
+  ; CHECK-EMPTY: 
+  ; CHECK-NEXT: $x10:{ 4 }
+  ; CHECK-NEXT: %stack.0:{ 2 }
+  ; CHECK-NEXT: 5: SD $x10, %stack.0, 0 :: (store (s64))
+  ; CHECK-EMPTY: 
+  ; CHECK-NEXT: %stack.0:{ 2 5 }
+  ; CHECK-NEXT: 6: $x10 = LD %stack.0, 0 :: (load (s64))
+  ; CHECK-EMPTY: 
+  ; CHECK-NEXT: implicit $x10:{ 6 }
+  ; CHECK-NEXT: 7: PseudoRET implicit $x10
+
+  bb.0.entry:
+    liveins: $x10
+    BEQ $x10, $x0, %bb.2
+
+  bb.1:
+    liveins: $x10
+    $x10 = ADDI $x10, 1
+    SD $x10, %stack.0, 0 :: (store (s64))
+    BEQ $x0, $x0, %bb.3
+
+  bb.2:
+    liveins: $x10
+    $x10 = ADDI $x10, 2
+    SD $x10, %stack.0, 0 :: (store (s64))
+
+  bb.3:
+    $x10 = LD %stack.0, 0 :: (load (s64))
+    PseudoRET implicit $x10
+...
+---
+name:            test4
+tracksRegLiveness: true
+fixedStack:
+  - { id: 0, type: default, offset: 0, size: 4, alignment: 16,
+      isImmutable: true, isAliased: false }
+stack:
+  - { id: 0, name: '', type: default, offset: 0, size: 4, alignment: 4,
+      stack-id: default, callee-saved-register: '', callee-saved-restored: true,
+      debug-info-variable: '', debug-info-expression: '', debug-info-location: '' }
+  - { id: 1, name: '', type: default, offset: 0, size: 4, alignment: 4,
+      stack-id: default, callee-saved-register: '', callee-saved-restored: true,
+      debug-info-variable: '', debug-info-expression: '', debug-info-location: '' }
+body:             |
+  ; CHECK: RDA results for test4
+  ; CHECK-NEXT: $x10:{ }
+  ; CHECK-NEXT: %stack.0:{ }
+  ; CHECK-NEXT: 0: SD $x10, %stack.0, 0 :: (store (s64))
+  ; CHECK-EMPTY:
+  ; CHECK-NEXT: $x11:{ }
+  ; CHECK-NEXT: %stack.0:{ 0 }
+  ; CHECK-NEXT: 1: SD $x11, %stack.0, 0 :: (store (s64))
+  ; CHECK-EMPTY:
+  ; CHECK-NEXT: $x10:{ }
+  ; CHECK-NEXT: %stack.1:{ }
+  ; CHECK-NEXT: 2: SD $x10, %stack.1, 0 :: (store (s64))
+  ; CHECK-EMPTY:
+  ; CHECK-NEXT: $x11:{ }
+  ; CHECK-NEXT: %stack.1:{ 2 }
+  ; CHECK-NEXT: 3: SD $x11, %stack.1, 0 :: (store (s64))
+  ; CHECK-EMPTY:
+  ; CHECK-NEXT: 4: PseudoRET
+  bb.0.entry:
+    liveins: $x10, $x11
+    SD $x10, %stack.0, 0 :: (store (s64))
+    SD $x11, %stack.0, 0 :: (store (s64))
+    SD $x10, %stack.1, 0 :: (store (s64))
+    SD $x11, %stack.1, 0 :: (store (s64))
+    PseudoRET
+...
diff --git a/llvm/test/CodeGen/RISCV/rvv/abd.ll b/llvm/test/CodeGen/RISCV/rvv/abd.ll
index 5e610c453e1ba..583d872238df7 100644
--- a/llvm/test/CodeGen/RISCV/rvv/abd.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/abd.ll
@@ -103,15 +103,21 @@ define <vscale x 4 x i32> @sabd_s_promoted_ops(<vscale x 4 x i16> %a, <vscale x
   ret <vscale x 4 x i32> %abs
 }
 
-; FIXME: Crashes legalization if enabled
-;; define <vscale x 2 x i64> @sabd_d(<vscale x 2 x i64> %a, <vscale x 2 x i64> %b) {
-;;   %a.sext = sext <vscale x 2 x i64> %a to <vscale x 2 x i128>
-;;   %b.sext = sext <vscale x 2 x i64> %b to <vscale x 2 x i128>
-;;   %sub = sub <vscale x 2 x i128> %a.sext, %b.sext
-;;   %abs = call <vscale x 2 x i128> @llvm.abs.nxv2i128(<vscale x 2 x i128> %sub, i1 true)
-;;   %trunc = trunc <vscale x 2 x i128> %abs to <vscale x 2 x i64>
-;;   ret <vscale x 2 x i64> %trunc
-;; }
+define <vscale x 2 x i64> @sabd_d(<vscale x 2 x i64> %a, <vscale x 2 x i64> %b) {
+; CHECK-LABEL: sabd_d:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e64, m2, ta, ma
+; CHECK-NEXT:    vmin.vv v12, v8, v10
+; CHECK-NEXT:    vmax.vv v8, v8, v10
+; CHECK-NEXT:    vsub.vv v8, v8, v12
+; CHECK-NEXT:    ret
+  %a.sext = sext <vscale x 2 x i64> %a to <vscale x 2 x i128>
+  %b.sext = sext <vscale x 2 x i64> %b to <vscale x 2 x i128>
+  %sub = sub <vscale x 2 x i128> %a.sext, %b.sext
+  %abs = call <vscale x 2 x i128> @llvm.abs.nxv2i128(<vscale x 2 x i128> %sub, i1 true)
+  %trunc = trunc <vscale x 2 x i128> %abs to <vscale x 2 x i64>
+  ret <vscale x 2 x i64> %trunc
+}
 
 define <vscale x 2 x i64> @sabd_d_promoted_ops(<vscale x 2 x i32> %a, <vscale x 2 x i32> %b) {
 ; CHECK-LABEL: sabd_d_promoted_ops:
@@ -231,15 +237,21 @@ define <vscale x 4 x i32> @uabd_s_promoted_ops(<vscale x 4 x i16> %a, <vscale x
   ret <vscale x 4 x i32> %abs
 }
 
-; FIXME: Crashes legalization if enabled
-;; define <vscale x 2 x i64> @uabd_d(<vscale x 2 x i64> %a, <vscale x 2 x i64> %b) {
-;;   %a.zext = zext <vscale x 2 x i64> %a to <vscale x 2 x i128>
-;;   %b.zext = zext <vscale x 2 x i64> %b to <vscale x 2 x i128>
-;;   %sub = sub <vscale x 2 x i128> %a.zext, %b.zext
-;;   %abs = call <vscale x 2 x i128> @llvm.abs.nxv2i128(<vscale x 2 x i128> %sub, i1 true)
-;;   %trunc = trunc <vscale x 2 x i128> %abs to <vscale x 2 x i64>
-;;   ret <vscale x 2 x i64> %trunc
-;; }
+define <vscale x 2 x i64> @uabd_d(<vscale x 2 x i64> %a, <vscale x 2 x i64> %b) {
+; CHECK-LABEL: uabd_d:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e64, m2, ta, ma
+; CHECK-NEXT:    vminu.vv v12, v8, v10
+; CHECK-NEXT:    vmaxu.vv v8, v8, v10
+; CHECK-NEXT:    vsub.vv v8, v8, v12
+; CHECK-NEXT:    ret
+  %a.zext = zext <vscale x 2 x i64> %a to <vscale x 2 x i128>
+  %b.zext = zext <vscale x 2 x i64> %b to <vscale x 2 x i128>
+  %sub = sub <vscale x 2 x i128> %a.zext, %b.zext
+  %abs = call <vscale x 2 x i128> @llvm.abs.nxv2i128(<vscale x 2 x i128> %sub, i1 true)
+  %trunc = trunc <vscale x 2 x i128> %abs to <vscale x 2 x i64>
+  ret <vscale x 2 x i64> %trunc
+}
 
 define <vscale x 2 x i64> @uabd_d_promoted_ops(<vscale x 2 x i32> %a, <vscale x 2 x i32> %b) {
 ; CHECK-LABEL: uabd_d_promoted_ops:
diff --git a/llvm/test/CodeGen/RISCV/rvv/addi-scalable-offset.mir b/llvm/test/CodeGen/RISCV/rvv/addi-scalable-offset.mir
index 2694fe52de8a6..cb1aebf0f95dd 100644
--- a/llvm/test/CodeGen/RISCV/rvv/addi-scalable-offset.mir
+++ b/llvm/test/CodeGen/RISCV/rvv/addi-scalable-offset.mir
@@ -30,8 +30,8 @@ body: |
     ; CHECK-NEXT: {{  $}}
     ; CHECK-NEXT: $x2 = frame-setup ADDI $x2, -2032
     ; CHECK-NEXT: frame-setup CFI_INSTRUCTION def_cfa_offset 2032
-    ; CHECK-NEXT: SD killed $x1, $x2, 2024 :: (store (s64) into %stack.3)
-    ; CHECK-NEXT: SD killed $x8, $x2, 2016 :: (store (s64) into %stack.4)
+    ; CHECK-NEXT: frame-setup SD killed $x1, $x2, 2024 :: (store (s64) into %stack.3)
+    ; CHECK-NEXT: frame-setup SD killed $x8, $x2, 2016 :: (store (s64) into %stack.4)
     ; CHECK-NEXT: frame-setup CFI_INSTRUCTION offset $x1, -8
     ; CHECK-NEXT: frame-setup CFI_INSTRUCTION offset $x8, -16
     ; CHECK-NEXT: $x8 = frame-setup ADDI $x2, 2032
@@ -48,8 +48,8 @@ body: |
     ; CHECK-NEXT: VS1R_V killed renamable $v8, killed renamable $x10
     ; CHECK-NEXT: $x2 = frame-destroy ADDI $x8, -2032
     ; CHECK-NEXT: frame-destroy CFI_INSTRUCTION def_cfa $x2, 2032
-    ; CHECK-NEXT: $x1 = LD $x2, 2024 :: (load (s64) from %stack.3)
-    ; CHECK-NEXT: $x8 = LD $x2, 2016 :: (load (s64) from %stack.4)
+    ; CHECK-NEXT: $x1 = frame-destroy LD $x2, 2024 :: (load (s64) from %stack.3)
+    ; CHECK-NEXT: $x8 = frame-destroy LD $x2, 2016 :: (load (s64) from %stack.4)
     ; CHECK-NEXT: frame-destroy CFI_INSTRUCTION restore $x1
     ; CHECK-NEXT: frame-destroy CFI_INSTRUCTION restore $x8
     ; CHECK-NEXT: $x2 = frame-destroy ADDI $x2, 2032
diff --git a/llvm/test/CodeGen/RISCV/rvv/emergency-slot.mir b/llvm/test/CodeGen/RISCV/rvv/emergency-slot.mir
index 9e6a36d68833d..9e72382e072c3 100644
--- a/llvm/test/CodeGen/RISCV/rvv/emergency-slot.mir
+++ b/llvm/test/CodeGen/RISCV/rvv/emergency-slot.mir
@@ -55,18 +55,18 @@ body:             |
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT:   $x2 = frame-setup ADDI $x2, -2032
   ; CHECK-NEXT:   frame-setup CFI_INSTRUCTION def_cfa_offset 2032
-  ; CHECK-NEXT:   SD killed $x1, $x2, 2024 :: (store (s64) into %stack.3)
-  ; CHECK-NEXT:   SD killed $x8, $x2, 2016 :: (store (s64) into %stack.4)
-  ; CHECK-NEXT:   SD killed $x18, $x2, 2008 :: (store (s64) into %stack.5)
-  ; CHECK-NEXT:   SD killed $x19, $x2, 2000 :: (store (s64) into %stack.6)
-  ; CHECK-NEXT:   SD killed $x20, $x2, 1992 :: (store (s64) into %stack.7)
-  ; CHECK-NEXT:   SD killed $x21, $x2, 1984 :: (store (s64) into %stack.8)
-  ; CHECK-NEXT:   SD killed $x22, $x2, 1976 :: (store (s64) into %stack.9)
-  ; CHECK-NEXT:   SD killed $x23, $x2, 1968 :: (store (s64) into %stack.10)
-  ; CHECK-NEXT:   SD killed $x24, $x2, 1960 :: (store (s64) into %stack.11)
-  ; CHECK-NEXT:   SD killed $x25, $x2, 1952 :: (store (s64) into %stack.12)
-  ; CHECK-NEXT:   SD killed $x26, $x2, 1944 :: (store (s64) into %stack.13)
-  ; CHECK-NEXT:   SD killed $x27, $x2, 1936 :: (store (s64) into %stack.14)
+  ; CHECK-NEXT:   frame-setup SD killed $x1, $x2, 2024 :: (store (s64) into %stack.3)
+  ; CHECK-NEXT:   frame-setup SD killed $x8, $x2, 2016 :: (store (s64) into %stack.4)
+  ; CHECK-NEXT:   frame-setup SD killed $x18, $x2, 2008 :: (store (s64) into %stack.5)
+  ; CHECK-NEXT:   frame-setup SD killed $x19, $x2, 2000 :: (store (s64) into %stack.6)
+  ; CHECK-NEXT:   frame-setup SD killed $x20, $x2, 1992 :: (store (s64) into %stack.7)
+  ; CHECK-NEXT:   frame-setup SD killed $x21, $x2, 1984 :: (store (s64) into %stack.8)
+  ; CHECK-NEXT:   frame-setup SD killed $x22, $x2, 1976 :: (store (s64) into %stack.9)
+  ; CHECK-NEXT:   frame-setup SD killed $x23, $x2, 1968 :: (store (s64) into %stack.10)
+  ; CHECK-NEXT:   frame-setup SD killed $x24, $x2, 1960 :: (store (s64) into %stack.11)
+  ; CHECK-NEXT:   frame-setup SD killed $x25, $x2, 1952 :: (store (s64) into %stack.12)
+  ; CHECK-NEXT:   frame-setup SD killed $x26, $x2, 1944 :: (store (s64) into %stack.13)
+  ; CHECK-NEXT:   frame-setup SD killed $x27, $x2, 1936 :: (store (s64) into %stack.14)
   ; CHECK-NEXT:   frame-setup CFI_INSTRUCTION offset $x1, -8
   ; CHECK-NEXT:   frame-setup CFI_INSTRUCTION offset $x8, -16
   ; CHECK-NEXT:   frame-setup CFI_INSTRUCTION offset $x18, -24
@@ -152,18 +152,18 @@ body:             |
   ; CHECK-NEXT: bb.2:
   ; CHECK-NEXT:   $x2 = frame-destroy ADDI $x8, -2032
   ; CHECK-NEXT:   frame-destroy CFI_INSTRUCTION def_cfa $x2, 2032
-  ; CHECK-NEXT:   $x1 = LD $x2, 2024 :: (load (s64) from %stack.3)
-  ; CHECK-NEXT:   $x8 = LD $x2, 2016 :: (load (s64) from %stack.4)
-  ; CHECK-NEXT:   $x18 = LD $x2, 2008 :: (load (s64) from %stack.5)
-  ; CHECK-NEXT:   $x19 = LD $x2, 2000 :: (load (s64) from %stack.6)
-  ; CHECK-NEXT:   $x20 = LD $x2, 1992 :: (load (s64) from %stack.7)
-  ; CHECK-NEXT:   $x21 = LD $x2, 1984 :: (load (s64) from %stack.8)
-  ; CHECK-NEXT:   $x22 = LD $x2, 1976 :: (load (s64) from %stack.9)
-  ; CHECK-NEXT:   $x23 = LD $x2, 1968 :: (load (s64) from %stack.10)
-  ; CHECK-NEXT:   $x24 = LD $x2, 1960 :: (load (s64) from %stack.11)
-  ; CHECK-NEXT:   $x25 = LD $x2, 1952 :: (load (s64) from %stack.12)
-  ; CHECK-NEXT:   $x26 = LD $x2, 1944 :: (load (s64) from %stack.13)
-  ; CHECK-NEXT:   $x27 = LD $x2, 1936 :: (load (s64) from %stack.14)
+  ; CHECK-NEXT:   $x1 = frame-destroy LD $x2, 2024 :: (load (s64) from %stack.3)
+  ; CHECK-NEXT:   $x8 = frame-destroy LD $x2, 2016 :: (load (s64) from %stack.4)
+  ; CHECK-NEXT:   $x18 = frame-destroy LD $x2, 2008 :: (load (s64) from %stack.5)
+  ; CHECK-NEXT:   $x19 = frame-destroy LD $x2, 2000 :: (load (s64) from %stack.6)
+  ; CHECK-NEXT:   $x20 = frame-destroy LD $x2, 1992 :: (load (s64) from %stack.7)
+  ; CHECK-NEXT:   $x21 = frame-destroy LD $x2, 1984 :: (load (s64) from %stack.8)
+  ; CHECK-NEXT:   $x22 = frame-destroy LD $x2, 1976 :: (load (s64) from %stack.9)
+  ; CHECK-NEXT:   $x23 = frame-destroy LD $x2, 1968 :: (load (s64) from %stack.10)
+  ; CHECK-NEXT:   $x24 = frame-destroy LD $x2, 1960 :: (load (s64) from %stack.11)
+  ; CHECK-NEXT:   $x25 = frame-destroy LD $x2, 1952 :: (load (s64) from %stack.12)
+  ; CHECK-NEXT:   $x26 = frame-destroy LD $x2, 1944 :: (load (s64) from %stack.13)
+  ; CHECK-NEXT:   $x27 = frame-destroy LD $x2, 1936 :: (load (s64) from %stack.14)
   ; CHECK-NEXT:   frame-destroy CFI_INSTRUCTION restore $x1
   ; CHECK-NEXT:   frame-destroy CFI_INSTRUCTION restore $x8
   ; CHECK-NEXT:   frame-destroy CFI_INSTRUCTION restore $x18
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-cttz-vp.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-cttz-vp.ll
index cd4b19f11d160..5f275da1740cb 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-cttz-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-cttz-vp.ll
@@ -1735,7 +1735,8 @@ define <32 x i64> @vp_cttz_v32i64(<32 x i64> %va, <32 x i1> %m, i32 zeroext %evl
 ; RV32-NEXT:    sub sp, sp, a1
 ; RV32-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x30, 0x22, 0x11, 0x30, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 48 + 48 * vlenb
 ; RV32-NEXT:    csrr a1, vlenb
-; RV32-NEXT:    slli a1, a1, 4
+; RV32-NEXT:    li a2, 24
+; RV32-NEXT:    mul a1, a1, a2
 ; RV32-NEXT:    add a1, sp, a1
 ; RV32-NEXT:    addi a1, a1, 48
 ; RV32-NEXT:    vs8r.v v16, (a1) # Unknown-size Folded Spill
@@ -1770,8 +1771,7 @@ define <32 x i64> @vp_cttz_v32i64(<32 x i64> %va, <32 x i1> %m, i32 zeroext %evl
 ; RV32-NEXT:    vnot.v v8, v8, v0.t
 ; RV32-NEXT:    vand.vv v8, v8, v16, v0.t
 ; RV32-NEXT:    csrr a4, vlenb
-; RV32-NEXT:    li a5, 24
-; RV32-NEXT:    mul a4, a4, a5
+; RV32-NEXT:    slli a4, a4, 5
 ; RV32-NEXT:    add a4, sp, a4
 ; RV32-NEXT:    addi a4, a4, 48
 ; RV32-NEXT:    vs8r.v v8, (a4) # Unknown-size Folded Spill
@@ -1786,13 +1786,12 @@ define <32 x i64> @vp_cttz_v32i64(<32 x i64> %va, <32 x i1> %m, i32 zeroext %evl
 ; RV32-NEXT:    addi a3, sp, 32
 ; RV32-NEXT:    vlse64.v v8, (a3), zero
 ; RV32-NEXT:    csrr a3, vlenb
-; RV32-NEXT:    slli a3, a3, 5
+; RV32-NEXT:    slli a3, a3, 4
 ; RV32-NEXT:    add a3, sp, a3
 ; RV32-NEXT:    addi a3, a3, 48
 ; RV32-NEXT:    vs8r.v v8, (a3) # Unknown-size Folded Spill
 ; RV32-NEXT:    csrr a3, vlenb
-; RV32-NEXT:    li a4, 24
-; RV32-NEXT:    mul a3, a3, a4
+; RV32-NEXT:    slli a3, a3, 5
 ; RV32-NEXT:    add a3, sp, a3
 ; RV32-NEXT:    addi a3, a3, 48
 ; RV32-NEXT:    vl8r.v v16, (a3) # Unknown-size Folded Reload
@@ -1806,24 +1805,21 @@ define <32 x i64> @vp_cttz_v32i64(<32 x i64> %va, <32 x i1> %m, i32 zeroext %evl
 ; RV32-NEXT:    vl8r.v v16, (a3) # Unknown-size Folded Reload
 ; RV32-NEXT:    vand.vv v24, v24, v16, v0.t
 ; RV32-NEXT:    csrr a3, vlenb
-; RV32-NEXT:    li a4, 24
-; RV32-NEXT:    mul a3, a3, a4
+; RV32-NEXT:    slli a3, a3, 5
 ; RV32-NEXT:    add a3, sp, a3
 ; RV32-NEXT:    addi a3, a3, 48
 ; RV32-NEXT:    vl8r.v v16, (a3) # Unknown-size Folded Reload
 ; RV32-NEXT:    vsub.vv v24, v16, v24, v0.t
 ; RV32-NEXT:    vand.vv v16, v24, v8, v0.t
 ; RV32-NEXT:    csrr a3, vlenb
-; RV32-NEXT:    li a4, 24
-; RV32-NEXT:    mul a3, a3, a4
+; RV32-NEXT:    slli a3, a3, 5
 ; RV32-NEXT:    add a3, sp, a3
 ; RV32-NEXT:    addi a3, a3, 48
 ; RV32-NEXT:    vs8r.v v16, (a3) # Unknown-size Folded Spill
 ; RV32-NEXT:    vsrl.vi v16, v24, 2, v0.t
 ; RV32-NEXT:    vand.vv v16, v16, v8, v0.t
 ; RV32-NEXT:    csrr a3, vlenb
-; RV32-NEXT:    li a4, 24
-; RV32-NEXT:    mul a3, a3, a4
+; RV32-NEXT:    slli a3, a3, 5
 ; RV32-NEXT:    add a3, sp, a3
 ; RV32-NEXT:    addi a3, a3, 48
 ; RV32-NEXT:    vl8r.v v8, (a3) # Unknown-size Folded Reload
@@ -1841,8 +1837,7 @@ define <32 x i64> @vp_cttz_v32i64(<32 x i64> %va, <32 x i1> %m, i32 zeroext %evl
 ; RV32-NEXT:    vs8r.v v16, (a3) # Unknown-size Folded Spill
 ; RV32-NEXT:    vlse64.v v8, (a4), zero
 ; RV32-NEXT:    csrr a3, vlenb
-; RV32-NEXT:    li a4, 24
-; RV32-NEXT:    mul a3, a3, a4
+; RV32-NEXT:    slli a3, a3, 5
 ; RV32-NEXT:    add a3, sp, a3
 ; RV32-NEXT:    addi a3, a3, 48
 ; RV32-NEXT:    vs8r.v v8, (a3) # Unknown-size Folded Spill
@@ -1856,8 +1851,7 @@ define <32 x i64> @vp_cttz_v32i64(<32 x i64> %va, <32 x i1> %m, i32 zeroext %evl
 ; RV32-NEXT:    vadd.vv v8, v8, v24, v0.t
 ; RV32-NEXT:    vand.vv v16, v8, v16, v0.t
 ; RV32-NEXT:    csrr a2, vlenb
-; RV32-NEXT:    li a3, 24
-; RV32-NEXT:    mul a2, a2, a3
+; RV32-NEXT:    slli a2, a2, 5
 ; RV32-NEXT:    add a2, sp, a2
 ; RV32-NEXT:    addi a2, a2, 48
 ; RV32-NEXT:    vl8r.v v8, (a2) # Unknown-size Folded Reload
@@ -1875,7 +1869,8 @@ define <32 x i64> @vp_cttz_v32i64(<32 x i64> %va, <32 x i1> %m, i32 zeroext %evl
 ; RV32-NEXT:    and a0, a0, a3
 ; RV32-NEXT:    vmv1r.v v0, v7
 ; RV32-NEXT:    csrr a3, vlenb
-; RV32-NEXT:    slli a3, a3, 4
+; RV32-NEXT:    li a4, 24
+; RV32-NEXT:    mul a3, a3, a4
 ; RV32-NEXT:    add a3, sp, a3
 ; RV32-NEXT:    addi a3, a3, 48
 ; RV32-NEXT:    vl8r.v v16, (a3) # Unknown-size Folded Reload
@@ -1891,31 +1886,44 @@ define <32 x i64> @vp_cttz_v32i64(<32 x i64> %va, <32 x i1> %m, i32 zeroext %evl
 ; RV32-NEXT:    addi a0, a0, 48
 ; RV32-NEXT:    vl8r.v v16, (a0) # Unknown-size Folded Reload
 ; RV32-NEXT:    vand.vv v16, v24, v16, v0.t
-; RV32-NEXT:    vsub.vv v24, v8, v16, v0.t
+; RV32-NEXT:    vsub.vv v8, v8, v16, v0.t
 ; RV32-NEXT:    csrr a0, vlenb
-; RV32-NEXT:    slli a0, a0, 5
+; RV32-NEXT:    li a1, 40
+; RV32-NEXT:    mul a0, a0, a1
 ; RV32-NEXT:    add a0, sp, a0
 ; RV32-NEXT:    addi a0, a0, 48
-; RV32-NEXT:    vl8r.v v8, (a0) # Unknown-size Folded Reload
-; RV32-NEXT:    vand.vv v16, v24, v8, v0.t
+; RV32-NEXT:    vs8r.v v8, (a0) # Unknown-size Folded Spill
+; RV32-NEXT:    csrr a0, vlenb
+; RV32-NEXT:    slli a0, a0, 4
+; RV32-NEXT:    add a0, sp, a0
+; RV32-NEXT:    addi a0, a0, 48
+; RV32-NEXT:    vl8r.v v24, (a0) # Unknown-size Folded Reload
 ; RV32-NEXT:    csrr a0, vlenb
 ; RV32-NEXT:    li a1, 40
 ; RV32-NEXT:    mul a0, a0, a1
 ; RV32-NEXT:    add a0, sp, a0
 ; RV32-NEXT:    addi a0, a0, 48
-; RV32-NEXT:    vs8r.v v16, (a0) # Unknown-size Folded Spill
-; RV32-NEXT:    vsrl.vi v8, v24, 2, v0.t
+; RV32-NEXT:    vl8r.v v16, (a0) # Unknown-size Folded Reload
+; RV32-NEXT:    vand.vv v16, v16, v24, v0.t
 ; RV32-NEXT:    csrr a0, vlenb
-; RV32-NEXT:    slli a0, a0, 5
+; RV32-NEXT:    li a1, 24
+; RV32-NEXT:    mul a0, a0, a1
 ; RV32-NEXT:    add a0, sp, a0
 ; RV32-NEXT:    addi a0, a0, 48
-; RV32-NEXT:    vl8r.v v16, (a0) # Unknown-size Folded Reload
-; RV32-NEXT:    vand.vv v8, v8, v16, v0.t
+; RV32-NEXT:    vs8r.v v16, (a0) # Unknown-size Folded Spill
 ; RV32-NEXT:    csrr a0, vlenb
 ; RV32-NEXT:    li a1, 40
 ; RV32-NEXT:    mul a0, a0, a1
 ; RV32-NEXT:    add a0, sp, a0
 ; RV32-NEXT:    addi a0, a0, 48
+; RV32-NEXT:    vl8r.v v8, (a0) # Unknown-size Folded Reload
+; RV32-NEXT:    vsrl.vi v8, v8, 2, v0.t
+; RV32-NEXT:    vand.vv v8, v8, v24, v0.t
+; RV32-NEXT:    csrr a0, vlenb
+; RV32-NEXT:    li a1, 24
+; RV32-NEXT:    mul a0, a0, a1
+; RV32-NEXT:    add a0, sp, a0
+; RV32-NEXT:    addi a0, a0, 48
 ; RV32-NEXT:    vl8r.v v16, (a0) # Unknown-size Folded Reload
 ; RV32-NEXT:    vadd.vv v8, v16, v8, v0.t
 ; RV32-NEXT:    vsrl.vi v16, v8, 4, v0.t
@@ -1924,8 +1932,7 @@ define <32 x i64> @vp_cttz_v32i64(<32 x i64> %va, <32 x i1> %m, i32 zeroext %evl
 ; RV32-NEXT:    vl8r.v v16, (a0) # Unknown-size Folded Reload
 ; RV32-NEXT:    vand.vv v8, v8, v16, v0.t
 ; RV32-NEXT:    csrr a0, vlenb
-; RV32-NEXT:    li a1, 24
-; RV32-NEXT:    mul a0, a0, a1
+; RV32-NEXT:    slli a0, a0, 5
 ; RV32-NEXT:    add a0, sp, a0
 ; RV32-NEXT:    addi a0, a0, 48
 ; RV32-NEXT:    vl8r.v v16, (a0) # Unknown-size Folded Reload
@@ -3921,7 +3928,8 @@ define <32 x i64> @vp_cttz_zero_undef_v32i64(<32 x i64> %va, <32 x i1> %m, i32 z
 ; RV32-NEXT:    sub sp, sp, a1
 ; RV32-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x30, 0x22, 0x11, 0x30, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 48 + 48 * vlenb
 ; RV32-NEXT:    csrr a1, vlenb
-; RV32-NEXT:    slli a1, a1, 4
+; RV32-NEXT:    li a2, 24
+; RV32-NEXT:    mul a1, a1, a2
 ; RV32-NEXT:    add a1, sp, a1
 ; RV32-NEXT:    addi a1, a1, 48
 ; RV32-NEXT:    vs8r.v v16, (a1) # Unknown-size Folded Spill
@@ -3956,8 +3964,7 @@ define <32 x i64> @vp_cttz_zero_undef_v32i64(<32 x i64> %va, <32 x i1> %m, i32 z
 ; RV32-NEXT:    vnot.v v8, v8, v0.t
 ; RV32-NEXT:    vand.vv v8, v8, v16, v0.t
 ; RV32-NEXT:    csrr a4, vlenb
-; RV32-NEXT:    li a5, 24
-; RV32-NEXT:    mul a4, a4, a5
+; RV32-NEXT:    slli a4, a4, 5
 ; RV32-NEXT:    add a4, sp, a4
 ; RV32-NEXT:    addi a4, a4, 48
 ; RV32-NEXT:    vs8r.v v8, (a4) # Unknown-size Folded Spill
@@ -3972,13 +3979,12 @@ define <32 x i64> @vp_cttz_zero_undef_v32i64(<32 x i64> %va, <32 x i1> %m, i32 z
 ; RV32-NEXT:    addi a3, sp, 32
 ; RV32-NEXT:    vlse64.v v8, (a3), zero
 ; RV32-NEXT:    csrr a3, vlenb
-; RV32-NEXT:    slli a3, a3, 5
+; RV32-NEXT:    slli a3, a3, 4
 ; RV32-NEXT:    add a3, sp, a3
 ; RV32-NEXT:    addi a3, a3, 48
 ; RV32-NEXT:    vs8r.v v8, (a3) # Unknown-size Folded Spill
 ; RV32-NEXT:    csrr a3, vlenb
-; RV32-NEXT:    li a4, 24
-; RV32-NEXT:    mul a3, a3, a4
+; RV32-NEXT:    slli a3, a3, 5
 ; RV32-NEXT:    add a3, sp, a3
 ; RV32-NEXT:    addi a3, a3, 48
 ; RV32-NEXT:    vl8r.v v16, (a3) # Unknown-size Folded Reload
@@ -3992,24 +3998,21 @@ define <32 x i64> @vp_cttz_zero_undef_v32i64(<32 x i64> %va, <32 x i1> %m, i32 z
 ; RV32-NEXT:    vl8r.v v16, (a3) # Unknown-size Folded Reload
 ; RV32-NEXT:    vand.vv v24, v24, v16, v0.t
 ; RV32-NEXT:    csrr a3, vlenb
-; RV32-NEXT:    li a4, 24
-; RV32-NEXT:    mul a3, a3, a4
+; RV32-NEXT:    slli a3, a3, 5
 ; RV32-NEXT:    add a3, sp, a3
 ; RV32-NEXT:    addi a3, a3, 48
 ; RV32-NEXT:    vl8r.v v16, (a3) # Unknown-size Folded Reload
 ; RV32-NEXT:    vsub.vv v24, v16, v24, v0.t
 ; RV32-NEXT:    vand.vv v16, v24, v8, v0.t
 ; RV32-NEXT:    csrr a3, vlenb
-; RV32-NEXT:    li a4, 24
-; RV32-NEXT:    mul a3, a3, a4
+; RV32-NEXT:    slli a3, a3, 5
 ; RV32-NEXT:    add a3, sp, a3
 ; RV32-NEXT:    addi a3, a3, 48
 ; RV32-NEXT:    vs8r.v v16, (a3) # Unknown-size Folded Spill
 ; RV32-NEXT:    vsrl.vi v16, v24, 2, v0.t
 ; RV32-NEXT:    vand.vv v16, v16, v8, v0.t
 ; RV32-NEXT:    csrr a3, vlenb
-; RV32-NEXT:    li a4, 24
-; RV32-NEXT:    mul a3, a3, a4
+; RV32-NEXT:    slli a3, a3, 5
 ; RV32-NEXT:    add a3, sp, a3
 ; RV32-NEXT:    addi a3, a3, 48
 ; RV32-NEXT:    vl8r.v v8, (a3) # Unknown-size Folded Reload
@@ -4027,8 +4030,7 @@ define <32 x i64> @vp_cttz_zero_undef_v32i64(<32 x i64> %va, <32 x i1> %m, i32 z
 ; RV32-NEXT:    vs8r.v v16, (a3) # Unknown-size Folded Spill
 ; RV32-NEXT:    vlse64.v v8, (a4), zero
 ; RV32-NEXT:    csrr a3, vlenb
-; RV32-NEXT:    li a4, 24
-; RV32-NEXT:    mul a3, a3, a4
+; RV32-NEXT:    slli a3, a3, 5
 ; RV32-NEXT:    add a3, sp, a3
 ; RV32-NEXT:    addi a3, a3, 48
 ; RV32-NEXT:    vs8r.v v8, (a3) # Unknown-size Folded Spill
@@ -4042,8 +4044,7 @@ define <32 x i64> @vp_cttz_zero_undef_v32i64(<32 x i64> %va, <32 x i1> %m, i32 z
 ; RV32-NEXT:    vadd.vv v8, v8, v24, v0.t
 ; RV32-NEXT:    vand.vv v16, v8, v16, v0.t
 ; RV32-NEXT:    csrr a2, vlenb
-; RV32-NEXT:    li a3, 24
-; RV32-NEXT:    mul a2, a2, a3
+; RV32-NEXT:    slli a2, a2, 5
 ; RV32-NEXT:    add a2, sp, a2
 ; RV32-NEXT:    addi a2, a2, 48
 ; RV32-NEXT:    vl8r.v v8, (a2) # Unknown-size Folded Reload
@@ -4061,7 +4062,8 @@ define <32 x i64> @vp_cttz_zero_undef_v32i64(<32 x i64> %va, <32 x i1> %m, i32 z
 ; RV32-NEXT:    and a0, a0, a3
 ; RV32-NEXT:    vmv1r.v v0, v7
 ; RV32-NEXT:    csrr a3, vlenb
-; RV32-NEXT:    slli a3, a3, 4
+; RV32-NEXT:    li a4, 24
+; RV32-NEXT:    mul a3, a3, a4
 ; RV32-NEXT:    add a3, sp, a3
 ; RV32-NEXT:    addi a3, a3, 48
 ; RV32-NEXT:    vl8r.v v16, (a3) # Unknown-size Folded Reload
@@ -4077,31 +4079,44 @@ define <32 x i64> @vp_cttz_zero_undef_v32i64(<32 x i64> %va, <32 x i1> %m, i32 z
 ; RV32-NEXT:    addi a0, a0, 48
 ; RV32-NEXT:    vl8r.v v16, (a0) # Unknown-size Folded Reload
 ; RV32-NEXT:    vand.vv v16, v24, v16, v0.t
-; RV32-NEXT:    vsub.vv v24, v8, v16, v0.t
+; RV32-NEXT:    vsub.vv v8, v8, v16, v0.t
 ; RV32-NEXT:    csrr a0, vlenb
-; RV32-NEXT:    slli a0, a0, 5
+; RV32-NEXT:    li a1, 40
+; RV32-NEXT:    mul a0, a0, a1
 ; RV32-NEXT:    add a0, sp, a0
 ; RV32-NEXT:    addi a0, a0, 48
-; RV32-NEXT:    vl8r.v v8, (a0) # Unknown-size Folded Reload
-; RV32-NEXT:    vand.vv v16, v24, v8, v0.t
+; RV32-NEXT:    vs8r.v v8, (a0) # Unknown-size Folded Spill
+; RV32-NEXT:    csrr a0, vlenb
+; RV32-NEXT:    slli a0, a0, 4
+; RV32-NEXT:    add a0, sp, a0
+; RV32-NEXT:    addi a0, a0, 48
+; RV32-NEXT:    vl8r.v v24, (a0) # Unknown-size Folded Reload
 ; RV32-NEXT:    csrr a0, vlenb
 ; RV32-NEXT:    li a1, 40
 ; RV32-NEXT:    mul a0, a0, a1
 ; RV32-NEXT:    add a0, sp, a0
 ; RV32-NEXT:    addi a0, a0, 48
-; RV32-NEXT:    vs8r.v v16, (a0) # Unknown-size Folded Spill
-; RV32-NEXT:    vsrl.vi v8, v24, 2, v0.t
+; RV32-NEXT:    vl8r.v v16, (a0) # Unknown-size Folded Reload
+; RV32-NEXT:    vand.vv v16, v16, v24, v0.t
 ; RV32-NEXT:    csrr a0, vlenb
-; RV32-NEXT:    slli a0, a0, 5
+; RV32-NEXT:    li a1, 24
+; RV32-NEXT:    mul a0, a0, a1
 ; RV32-NEXT:    add a0, sp, a0
 ; RV32-NEXT:    addi a0, a0, 48
-; RV32-NEXT:    vl8r.v v16, (a0) # Unknown-size Folded Reload
-; RV32-NEXT:    vand.vv v8, v8, v16, v0.t
+; RV32-NEXT:    vs8r.v v16, (a0) # Unknown-size Folded Spill
 ; RV32-NEXT:    csrr a0, vlenb
 ; RV32-NEXT:    li a1, 40
 ; RV32-NEXT:    mul a0, a0, a1
 ; RV32-NEXT:    add a0, sp, a0
 ; RV32-NEXT:    addi a0, a0, 48
+; RV32-NEXT:    vl8r.v v8, (a0) # Unknown-size Folded Reload
+; RV32-NEXT:    vsrl.vi v8, v8, 2, v0.t
+; RV32-NEXT:    vand.vv v8, v8, v24, v0.t
+; RV32-NEXT:    csrr a0, vlenb
+; RV32-NEXT:    li a1, 24
+; RV32-NEXT:    mul a0, a0, a1
+; RV32-NEXT:    add a0, sp, a0
+; RV32-NEXT:    addi a0, a0, 48
 ; RV32-NEXT:    vl8r.v v16, (a0) # Unknown-size Folded Reload
 ; RV32-NEXT:    vadd.vv v8, v16, v8, v0.t
 ; RV32-NEXT:    vsrl.vi v16, v8, 4, v0.t
@@ -4110,8 +4125,7 @@ define <32 x i64> @vp_cttz_zero_undef_v32i64(<32 x i64> %va, <32 x i1> %m, i32 z
 ; RV32-NEXT:    vl8r.v v16, (a0) # Unknown-size Folded Reload
 ; RV32-NEXT:    vand.vv v8, v8, v16, v0.t
 ; RV32-NEXT:    csrr a0, vlenb
-; RV32-NEXT:    li a1, 24
-; RV32-NEXT:    mul a0, a0, a1
+; RV32-NEXT:    slli a0, a0, 5
 ; RV32-NEXT:    add a0, sp, a0
 ; RV32-NEXT:    addi a0, a0, 48
 ; RV32-NEXT:    vl8r.v v16, (a0) # Unknown-size Folded Reload
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-fp.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-fp.ll
index 49db94e1a02df..585a331e55094 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-fp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-fp.ll
@@ -1444,13 +1444,12 @@ define void @fma_v6bf16(ptr %x, ptr %y, ptr %z) {
 ; CHECK-NEXT:    vle16.v v8, (a2)
 ; CHECK-NEXT:    vle16.v v9, (a0)
 ; CHECK-NEXT:    vle16.v v10, (a1)
-; CHECK-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
 ; CHECK-NEXT:    vfwcvtbf16.f.f.v v12, v8
 ; CHECK-NEXT:    vfwcvtbf16.f.f.v v14, v9
 ; CHECK-NEXT:    vfwcvtbf16.f.f.v v8, v10
 ; CHECK-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
 ; CHECK-NEXT:    vfmadd.vv v8, v14, v12
-; CHECK-NEXT:    vsetivli zero, 6, e16, m1, ta, ma
+; CHECK-NEXT:    vsetvli zero, zero, e16, m1, ta, ma
 ; CHECK-NEXT:    vfncvtbf16.f.f.w v10, v8
 ; CHECK-NEXT:    vse16.v v10, (a0)
 ; CHECK-NEXT:    ret
@@ -1513,13 +1512,12 @@ define void @fma_v6f16(ptr %x, ptr %y, ptr %z) {
 ; ZVFHMIN-NEXT:    vle16.v v8, (a2)
 ; ZVFHMIN-NEXT:    vle16.v v9, (a0)
 ; ZVFHMIN-NEXT:    vle16.v v10, (a1)
-; ZVFHMIN-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
 ; ZVFHMIN-NEXT:    vfwcvt.f.f.v v12, v8
 ; ZVFHMIN-NEXT:    vfwcvt.f.f.v v14, v9
 ; ZVFHMIN-NEXT:    vfwcvt.f.f.v v8, v10
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
 ; ZVFHMIN-NEXT:    vfmadd.vv v8, v14, v12
-; ZVFHMIN-NEXT:    vsetivli zero, 6, e16, m1, ta, ma
+; ZVFHMIN-NEXT:    vsetvli zero, zero, e16, m1, ta, ma
 ; ZVFHMIN-NEXT:    vfncvt.f.f.w v10, v8
 ; ZVFHMIN-NEXT:    vse16.v v10, (a0)
 ; ZVFHMIN-NEXT:    ret
@@ -1602,14 +1600,13 @@ define void @fmsub_v6bf16(ptr %x, ptr %y, ptr %z) {
 ; CHECK-NEXT:    vle16.v v9, (a0)
 ; CHECK-NEXT:    vle16.v v10, (a1)
 ; CHECK-NEXT:    lui a1, 8
-; CHECK-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
 ; CHECK-NEXT:    vxor.vx v8, v8, a1
 ; CHECK-NEXT:    vfwcvtbf16.f.f.v v12, v9
 ; CHECK-NEXT:    vfwcvtbf16.f.f.v v14, v8
 ; CHECK-NEXT:    vfwcvtbf16.f.f.v v8, v10
 ; CHECK-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
 ; CHECK-NEXT:    vfmadd.vv v8, v12, v14
-; CHECK-NEXT:    vsetivli zero, 6, e16, m1, ta, ma
+; CHECK-NEXT:    vsetvli zero, zero, e16, m1, ta, ma
 ; CHECK-NEXT:    vfncvtbf16.f.f.w v10, v8
 ; CHECK-NEXT:    vse16.v v10, (a0)
 ; CHECK-NEXT:    ret
@@ -1677,14 +1674,13 @@ define void @fmsub_v6f16(ptr %x, ptr %y, ptr %z) {
 ; ZVFHMIN-NEXT:    vle16.v v9, (a0)
 ; ZVFHMIN-NEXT:    vle16.v v10, (a1)
 ; ZVFHMIN-NEXT:    lui a1, 8
-; ZVFHMIN-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
 ; ZVFHMIN-NEXT:    vxor.vx v8, v8, a1
 ; ZVFHMIN-NEXT:    vfwcvt.f.f.v v12, v9
 ; ZVFHMIN-NEXT:    vfwcvt.f.f.v v14, v8
 ; ZVFHMIN-NEXT:    vfwcvt.f.f.v v8, v10
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
 ; ZVFHMIN-NEXT:    vfmadd.vv v8, v12, v14
-; ZVFHMIN-NEXT:    vsetivli zero, 6, e16, m1, ta, ma
+; ZVFHMIN-NEXT:    vsetvli zero, zero, e16, m1, ta, ma
 ; ZVFHMIN-NEXT:    vfncvt.f.f.w v10, v8
 ; ZVFHMIN-NEXT:    vse16.v v10, (a0)
 ; ZVFHMIN-NEXT:    ret
@@ -3381,14 +3377,13 @@ define void @fma_vf_v6bf16(ptr %x, ptr %y, bfloat %z) {
 ; CHECK-NEXT:    vle16.v v8, (a1)
 ; CHECK-NEXT:    vle16.v v9, (a0)
 ; CHECK-NEXT:    fmv.x.w a1, fa0
-; CHECK-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
 ; CHECK-NEXT:    vmv.v.x v10, a1
 ; CHECK-NEXT:    vfwcvtbf16.f.f.v v12, v8
 ; CHECK-NEXT:    vfwcvtbf16.f.f.v v14, v9
 ; CHECK-NEXT:    vfwcvtbf16.f.f.v v8, v10
 ; CHECK-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
 ; CHECK-NEXT:    vfmadd.vv v8, v14, v12
-; CHECK-NEXT:    vsetivli zero, 6, e16, m1, ta, ma
+; CHECK-NEXT:    vsetvli zero, zero, e16, m1, ta, ma
 ; CHECK-NEXT:    vfncvtbf16.f.f.w v10, v8
 ; CHECK-NEXT:    vse16.v v10, (a0)
 ; CHECK-NEXT:    ret
@@ -3452,14 +3447,13 @@ define void @fma_vf_v6f16(ptr %x, ptr %y, half %z) {
 ; ZVFHMIN-NEXT:    vle16.v v8, (a1)
 ; ZVFHMIN-NEXT:    vle16.v v9, (a0)
 ; ZVFHMIN-NEXT:    fmv.x.w a1, fa0
-; ZVFHMIN-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
 ; ZVFHMIN-NEXT:    vmv.v.x v10, a1
 ; ZVFHMIN-NEXT:    vfwcvt.f.f.v v12, v8
 ; ZVFHMIN-NEXT:    vfwcvt.f.f.v v14, v9
 ; ZVFHMIN-NEXT:    vfwcvt.f.f.v v8, v10
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
 ; ZVFHMIN-NEXT:    vfmadd.vv v8, v14, v12
-; ZVFHMIN-NEXT:    vsetivli zero, 6, e16, m1, ta, ma
+; ZVFHMIN-NEXT:    vsetvli zero, zero, e16, m1, ta, ma
 ; ZVFHMIN-NEXT:    vfncvt.f.f.w v10, v8
 ; ZVFHMIN-NEXT:    vse16.v v10, (a0)
 ; ZVFHMIN-NEXT:    ret
@@ -3541,14 +3535,13 @@ define void @fma_fv_v6bf16(ptr %x, ptr %y, bfloat %z) {
 ; CHECK-NEXT:    vle16.v v8, (a1)
 ; CHECK-NEXT:    vle16.v v9, (a0)
 ; CHECK-NEXT:    fmv.x.w a1, fa0
-; CHECK-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
 ; CHECK-NEXT:    vmv.v.x v10, a1
 ; CHECK-NEXT:    vfwcvtbf16.f.f.v v12, v8
 ; CHECK-NEXT:    vfwcvtbf16.f.f.v v14, v9
 ; CHECK-NEXT:    vfwcvtbf16.f.f.v v8, v10
 ; CHECK-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
 ; CHECK-NEXT:    vfmadd.vv v8, v14, v12
-; CHECK-NEXT:    vsetivli zero, 6, e16, m1, ta, ma
+; CHECK-NEXT:    vsetvli zero, zero, e16, m1, ta, ma
 ; CHECK-NEXT:    vfncvtbf16.f.f.w v10, v8
 ; CHECK-NEXT:    vse16.v v10, (a0)
 ; CHECK-NEXT:    ret
@@ -3612,14 +3605,13 @@ define void @fma_fv_v6f16(ptr %x, ptr %y, half %z) {
 ; ZVFHMIN-NEXT:    vle16.v v8, (a1)
 ; ZVFHMIN-NEXT:    vle16.v v9, (a0)
 ; ZVFHMIN-NEXT:    fmv.x.w a1, fa0
-; ZVFHMIN-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
 ; ZVFHMIN-NEXT:    vmv.v.x v10, a1
 ; ZVFHMIN-NEXT:    vfwcvt.f.f.v v12, v8
 ; ZVFHMIN-NEXT:    vfwcvt.f.f.v v14, v9
 ; ZVFHMIN-NEXT:    vfwcvt.f.f.v v8, v10
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
 ; ZVFHMIN-NEXT:    vfmadd.vv v8, v14, v12
-; ZVFHMIN-NEXT:    vsetivli zero, 6, e16, m1, ta, ma
+; ZVFHMIN-NEXT:    vsetvli zero, zero, e16, m1, ta, ma
 ; ZVFHMIN-NEXT:    vfncvt.f.f.w v10, v8
 ; ZVFHMIN-NEXT:    vse16.v v10, (a0)
 ; ZVFHMIN-NEXT:    ret
@@ -3705,7 +3697,6 @@ define void @fmsub_vf_v6bf16(ptr %x, ptr %y, bfloat %z) {
 ; CHECK-NEXT:    vle16.v v8, (a1)
 ; CHECK-NEXT:    vle16.v v9, (a0)
 ; CHECK-NEXT:    lui a1, 8
-; CHECK-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
 ; CHECK-NEXT:    vmv.v.x v10, a2
 ; CHECK-NEXT:    vxor.vx v8, v8, a1
 ; CHECK-NEXT:    vfwcvtbf16.f.f.v v12, v9
@@ -3713,7 +3704,7 @@ define void @fmsub_vf_v6bf16(ptr %x, ptr %y, bfloat %z) {
 ; CHECK-NEXT:    vfwcvtbf16.f.f.v v8, v10
 ; CHECK-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
 ; CHECK-NEXT:    vfmadd.vv v8, v12, v14
-; CHECK-NEXT:    vsetivli zero, 6, e16, m1, ta, ma
+; CHECK-NEXT:    vsetvli zero, zero, e16, m1, ta, ma
 ; CHECK-NEXT:    vfncvtbf16.f.f.w v10, v8
 ; CHECK-NEXT:    vse16.v v10, (a0)
 ; CHECK-NEXT:    ret
@@ -3782,7 +3773,6 @@ define void @fmsub_vf_v6f16(ptr %x, ptr %y, half %z) {
 ; ZVFHMIN-NEXT:    vle16.v v8, (a1)
 ; ZVFHMIN-NEXT:    vle16.v v9, (a0)
 ; ZVFHMIN-NEXT:    lui a1, 8
-; ZVFHMIN-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
 ; ZVFHMIN-NEXT:    vmv.v.x v10, a2
 ; ZVFHMIN-NEXT:    vxor.vx v8, v8, a1
 ; ZVFHMIN-NEXT:    vfwcvt.f.f.v v12, v9
@@ -3790,7 +3780,7 @@ define void @fmsub_vf_v6f16(ptr %x, ptr %y, half %z) {
 ; ZVFHMIN-NEXT:    vfwcvt.f.f.v v8, v10
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
 ; ZVFHMIN-NEXT:    vfmadd.vv v8, v12, v14
-; ZVFHMIN-NEXT:    vsetivli zero, 6, e16, m1, ta, ma
+; ZVFHMIN-NEXT:    vsetvli zero, zero, e16, m1, ta, ma
 ; ZVFHMIN-NEXT:    vfncvt.f.f.w v10, v8
 ; ZVFHMIN-NEXT:    vse16.v v10, (a0)
 ; ZVFHMIN-NEXT:    ret
@@ -3919,11 +3909,12 @@ define void @trunc_v6bf16(ptr %x) {
 ; CHECK-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
 ; CHECK-NEXT:    vfabs.v v8, v10
 ; CHECK-NEXT:    vmflt.vf v0, v8, fa5
+; CHECK-NEXT:    vsetivli zero, 6, e32, m2, ta, ma
 ; CHECK-NEXT:    vfcvt.rtz.x.f.v v8, v10, v0.t
 ; CHECK-NEXT:    vfcvt.f.x.v v8, v8, v0.t
 ; CHECK-NEXT:    vsetvli zero, zero, e32, m2, ta, mu
 ; CHECK-NEXT:    vfsgnj.vv v10, v8, v10, v0.t
-; CHECK-NEXT:    vsetivli zero, 6, e16, m1, ta, ma
+; CHECK-NEXT:    vsetvli zero, zero, e16, m1, ta, ma
 ; CHECK-NEXT:    vfncvtbf16.f.f.w v8, v10
 ; CHECK-NEXT:    vse16.v v8, (a0)
 ; CHECK-NEXT:    ret
@@ -4002,11 +3993,12 @@ define void @trunc_v6f16(ptr %x) {
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
 ; ZVFHMIN-NEXT:    vfabs.v v8, v10
 ; ZVFHMIN-NEXT:    vmflt.vf v0, v8, fa5
+; ZVFHMIN-NEXT:    vsetivli zero, 6, e32, m2, ta, ma
 ; ZVFHMIN-NEXT:    vfcvt.rtz.x.f.v v8, v10, v0.t
 ; ZVFHMIN-NEXT:    vfcvt.f.x.v v8, v8, v0.t
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m2, ta, mu
 ; ZVFHMIN-NEXT:    vfsgnj.vv v10, v8, v10, v0.t
-; ZVFHMIN-NEXT:    vsetivli zero, 6, e16, m1, ta, ma
+; ZVFHMIN-NEXT:    vsetvli zero, zero, e16, m1, ta, ma
 ; ZVFHMIN-NEXT:    vfncvt.f.f.w v8, v10
 ; ZVFHMIN-NEXT:    vse16.v v8, (a0)
 ; ZVFHMIN-NEXT:    ret
@@ -4098,12 +4090,13 @@ define void @ceil_v6bf16(ptr %x) {
 ; CHECK-NEXT:    vfabs.v v8, v10
 ; CHECK-NEXT:    vmflt.vf v0, v8, fa5
 ; CHECK-NEXT:    fsrmi a1, 3
+; CHECK-NEXT:    vsetivli zero, 6, e32, m2, ta, ma
 ; CHECK-NEXT:    vfcvt.x.f.v v8, v10, v0.t
 ; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    vfcvt.f.x.v v8, v8, v0.t
 ; CHECK-NEXT:    vsetvli zero, zero, e32, m2, ta, mu
 ; CHECK-NEXT:    vfsgnj.vv v10, v8, v10, v0.t
-; CHECK-NEXT:    vsetivli zero, 6, e16, m1, ta, ma
+; CHECK-NEXT:    vsetvli zero, zero, e16, m1, ta, ma
 ; CHECK-NEXT:    vfncvtbf16.f.f.w v8, v10
 ; CHECK-NEXT:    vse16.v v8, (a0)
 ; CHECK-NEXT:    ret
@@ -4189,12 +4182,13 @@ define void @ceil_v6f16(ptr %x) {
 ; ZVFHMIN-NEXT:    vfabs.v v8, v10
 ; ZVFHMIN-NEXT:    vmflt.vf v0, v8, fa5
 ; ZVFHMIN-NEXT:    fsrmi a1, 3
+; ZVFHMIN-NEXT:    vsetivli zero, 6, e32, m2, ta, ma
 ; ZVFHMIN-NEXT:    vfcvt.x.f.v v8, v10, v0.t
 ; ZVFHMIN-NEXT:    fsrm a1
 ; ZVFHMIN-NEXT:    vfcvt.f.x.v v8, v8, v0.t
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m2, ta, mu
 ; ZVFHMIN-NEXT:    vfsgnj.vv v10, v8, v10, v0.t
-; ZVFHMIN-NEXT:    vsetivli zero, 6, e16, m1, ta, ma
+; ZVFHMIN-NEXT:    vsetvli zero, zero, e16, m1, ta, ma
 ; ZVFHMIN-NEXT:    vfncvt.f.f.w v8, v10
 ; ZVFHMIN-NEXT:    vse16.v v8, (a0)
 ; ZVFHMIN-NEXT:    ret
@@ -4290,12 +4284,13 @@ define void @floor_v6bf16(ptr %x) {
 ; CHECK-NEXT:    vfabs.v v8, v10
 ; CHECK-NEXT:    vmflt.vf v0, v8, fa5
 ; CHECK-NEXT:    fsrmi a1, 2
+; CHECK-NEXT:    vsetivli zero, 6, e32, m2, ta, ma
 ; CHECK-NEXT:    vfcvt.x.f.v v8, v10, v0.t
 ; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    vfcvt.f.x.v v8, v8, v0.t
 ; CHECK-NEXT:    vsetvli zero, zero, e32, m2, ta, mu
 ; CHECK-NEXT:    vfsgnj.vv v10, v8, v10, v0.t
-; CHECK-NEXT:    vsetivli zero, 6, e16, m1, ta, ma
+; CHECK-NEXT:    vsetvli zero, zero, e16, m1, ta, ma
 ; CHECK-NEXT:    vfncvtbf16.f.f.w v8, v10
 ; CHECK-NEXT:    vse16.v v8, (a0)
 ; CHECK-NEXT:    ret
@@ -4381,12 +4376,13 @@ define void @floor_v6f16(ptr %x) {
 ; ZVFHMIN-NEXT:    vfabs.v v8, v10
 ; ZVFHMIN-NEXT:    vmflt.vf v0, v8, fa5
 ; ZVFHMIN-NEXT:    fsrmi a1, 2
+; ZVFHMIN-NEXT:    vsetivli zero, 6, e32, m2, ta, ma
 ; ZVFHMIN-NEXT:    vfcvt.x.f.v v8, v10, v0.t
 ; ZVFHMIN-NEXT:    fsrm a1
 ; ZVFHMIN-NEXT:    vfcvt.f.x.v v8, v8, v0.t
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m2, ta, mu
 ; ZVFHMIN-NEXT:    vfsgnj.vv v10, v8, v10, v0.t
-; ZVFHMIN-NEXT:    vsetivli zero, 6, e16, m1, ta, ma
+; ZVFHMIN-NEXT:    vsetvli zero, zero, e16, m1, ta, ma
 ; ZVFHMIN-NEXT:    vfncvt.f.f.w v8, v10
 ; ZVFHMIN-NEXT:    vse16.v v8, (a0)
 ; ZVFHMIN-NEXT:    ret
@@ -4482,12 +4478,13 @@ define void @round_v6bf16(ptr %x) {
 ; CHECK-NEXT:    vfabs.v v8, v10
 ; CHECK-NEXT:    vmflt.vf v0, v8, fa5
 ; CHECK-NEXT:    fsrmi a1, 4
+; CHECK-NEXT:    vsetivli zero, 6, e32, m2, ta, ma
 ; CHECK-NEXT:    vfcvt.x.f.v v8, v10, v0.t
 ; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    vfcvt.f.x.v v8, v8, v0.t
 ; CHECK-NEXT:    vsetvli zero, zero, e32, m2, ta, mu
 ; CHECK-NEXT:    vfsgnj.vv v10, v8, v10, v0.t
-; CHECK-NEXT:    vsetivli zero, 6, e16, m1, ta, ma
+; CHECK-NEXT:    vsetvli zero, zero, e16, m1, ta, ma
 ; CHECK-NEXT:    vfncvtbf16.f.f.w v8, v10
 ; CHECK-NEXT:    vse16.v v8, (a0)
 ; CHECK-NEXT:    ret
@@ -4573,12 +4570,13 @@ define void @round_v6f16(ptr %x) {
 ; ZVFHMIN-NEXT:    vfabs.v v8, v10
 ; ZVFHMIN-NEXT:    vmflt.vf v0, v8, fa5
 ; ZVFHMIN-NEXT:    fsrmi a1, 4
+; ZVFHMIN-NEXT:    vsetivli zero, 6, e32, m2, ta, ma
 ; ZVFHMIN-NEXT:    vfcvt.x.f.v v8, v10, v0.t
 ; ZVFHMIN-NEXT:    fsrm a1
 ; ZVFHMIN-NEXT:    vfcvt.f.x.v v8, v8, v0.t
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m2, ta, mu
 ; ZVFHMIN-NEXT:    vfsgnj.vv v10, v8, v10, v0.t
-; ZVFHMIN-NEXT:    vsetivli zero, 6, e16, m1, ta, ma
+; ZVFHMIN-NEXT:    vsetvli zero, zero, e16, m1, ta, ma
 ; ZVFHMIN-NEXT:    vfncvt.f.f.w v8, v10
 ; ZVFHMIN-NEXT:    vse16.v v8, (a0)
 ; ZVFHMIN-NEXT:    ret
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-int-shuffles.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-int-shuffles.ll
index 8b26c58d5bee1..6849d7b2d3aaf 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-int-shuffles.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-int-shuffles.ll
@@ -1340,3 +1340,66 @@ define void @shuffle_i256_splat(ptr %p) nounwind {
   ret void
 }
 
+define <16 x i32> @shuffle_m1_prefix(<16 x i32> %a) {
+; CHECK-LABEL: shuffle_m1_prefix:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lui a0, 8208
+; CHECK-NEXT:    addi a0, a0, 770
+; CHECK-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
+; CHECK-NEXT:    vmv.s.x v9, a0
+; CHECK-NEXT:    vsext.vf4 v10, v9
+; CHECK-NEXT:    vrgather.vv v12, v8, v10
+; CHECK-NEXT:    vmv4r.v v8, v12
+; CHECK-NEXT:    ret
+  %out = shufflevector <16 x i32> %a, <16 x i32> poison, <16 x i32> <i32 2, i32 3, i32 1, i32 2, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+  ret <16 x i32> %out
+}
+
+define <16 x i32> @shuffle_m2_prefix(<16 x i32> %a) {
+; CHECK-LABEL: shuffle_m2_prefix:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lui a0, %hi(.LCPI85_0)
+; CHECK-NEXT:    addi a0, a0, %lo(.LCPI85_0)
+; CHECK-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
+; CHECK-NEXT:    vle16.v v14, (a0)
+; CHECK-NEXT:    vrgatherei16.vv v12, v8, v14
+; CHECK-NEXT:    vmv4r.v v8, v12
+; CHECK-NEXT:    ret
+  %out = shufflevector <16 x i32> %a, <16 x i32> poison, <16 x i32> <i32 2, i32 3, i32 5, i32 2, i32 3, i32 5, i32 7, i32 4, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+  ret <16 x i32> %out
+}
+
+define <4 x i16> @vmerge_1(<4 x i16> %x) {
+; CHECK-LABEL: vmerge_1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 4, e16, mf2, ta, ma
+; CHECK-NEXT:    vmv.v.i v0, 6
+; CHECK-NEXT:    vmerge.vim v8, v8, 5, v0
+; CHECK-NEXT:    ret
+   %s = shufflevector <4 x i16> %x, <4 x i16> <i16 5, i16 5, i16 5, i16 5>, <4 x i32> <i32 0, i32 5, i32 6, i32 3>
+   ret <4 x i16> %s
+}
+
+define <4 x i16> @vmerge_2(<4 x i16> %x) {
+; CHECK-LABEL: vmerge_2:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 4, e16, mf2, ta, ma
+; CHECK-NEXT:    vmv.v.i v0, 9
+; CHECK-NEXT:    vmv.v.i v9, 5
+; CHECK-NEXT:    vmerge.vvm v8, v9, v8, v0
+; CHECK-NEXT:    ret
+   %s = shufflevector <4 x i16> %x, <4 x i16> <i16 poison, i16 5, i16 5, i16 poison>, <4 x i32> <i32 0, i32 5, i32 6, i32 3>
+   ret <4 x i16> %s
+}
+
+define <4 x i16> @vmerge_3(<4 x i16> %x) {
+; CHECK-LABEL: vmerge_3:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 4, e16, mf2, ta, mu
+; CHECK-NEXT:    vmv.v.i v0, 6
+; CHECK-NEXT:    vmv.v.i v9, 5
+; CHECK-NEXT:    vrgather.vi v8, v9, 1, v0.t
+; CHECK-NEXT:    ret
+   %s = shufflevector <4 x i16> %x, <4 x i16> <i16 poison, i16 5, i16 poison, i16 poison>, <4 x i32> <i32 0, i32 5, i32 5, i32 3>
+   ret <4 x i16> %s
+}
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-interleaved-access.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-interleaved-access.ll
index 0c7d7925edf39..1516c67bf7ecc 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-interleaved-access.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-interleaved-access.ll
@@ -183,10 +183,10 @@ define {<8 x i64>, <8 x i64>, <8 x i64>, <8 x i64>, <8 x i64>, <8 x i64>} @load_
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
 ; RV32-NEXT:    csrr a2, vlenb
-; RV32-NEXT:    li a3, 92
+; RV32-NEXT:    li a3, 88
 ; RV32-NEXT:    mul a2, a2, a3
 ; RV32-NEXT:    sub sp, sp, a2
-; RV32-NEXT:    .cfi_escape 0x0f, 0x0e, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0xdc, 0x00, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 92 * vlenb
+; RV32-NEXT:    .cfi_escape 0x0f, 0x0e, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0xd8, 0x00, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 88 * vlenb
 ; RV32-NEXT:    addi a3, a1, 256
 ; RV32-NEXT:    addi a4, a1, 128
 ; RV32-NEXT:    li a2, 32
@@ -198,15 +198,14 @@ define {<8 x i64>, <8 x i64>, <8 x i64>, <8 x i64>, <8 x i64>, <8 x i64>} @load_
 ; RV32-NEXT:    vsetvli zero, a2, e32, m8, ta, ma
 ; RV32-NEXT:    vle32.v v16, (a1)
 ; RV32-NEXT:    csrr a1, vlenb
-; RV32-NEXT:    li t1, 76
+; RV32-NEXT:    li t1, 72
 ; RV32-NEXT:    mul a1, a1, t1
 ; RV32-NEXT:    add a1, sp, a1
 ; RV32-NEXT:    addi a1, a1, 16
 ; RV32-NEXT:    vs8r.v v16, (a1) # Unknown-size Folded Spill
 ; RV32-NEXT:    vle32.v v8, (a4)
 ; RV32-NEXT:    csrr a1, vlenb
-; RV32-NEXT:    li a4, 68
-; RV32-NEXT:    mul a1, a1, a4
+; RV32-NEXT:    slli a1, a1, 6
 ; RV32-NEXT:    add a1, sp, a1
 ; RV32-NEXT:    addi a1, a1, 16
 ; RV32-NEXT:    vs8r.v v8, (a1) # Unknown-size Folded Spill
@@ -221,7 +220,7 @@ define {<8 x i64>, <8 x i64>, <8 x i64>, <8 x i64>, <8 x i64>, <8 x i64>} @load_
 ; RV32-NEXT:    vsetivli zero, 16, e64, m8, ta, ma
 ; RV32-NEXT:    vrgatherei16.vv v24, v16, v6
 ; RV32-NEXT:    csrr a4, vlenb
-; RV32-NEXT:    li a5, 52
+; RV32-NEXT:    li a5, 48
 ; RV32-NEXT:    mul a4, a4, a5
 ; RV32-NEXT:    add a4, sp, a4
 ; RV32-NEXT:    addi a4, a4, 16
@@ -236,13 +235,13 @@ define {<8 x i64>, <8 x i64>, <8 x i64>, <8 x i64>, <8 x i64>, <8 x i64>} @load_
 ; RV32-NEXT:    vsetivli zero, 16, e32, m8, ta, ma
 ; RV32-NEXT:    vslidedown.vi v24, v16, 16
 ; RV32-NEXT:    csrr a1, vlenb
-; RV32-NEXT:    li a3, 60
+; RV32-NEXT:    li a3, 56
 ; RV32-NEXT:    mul a1, a1, a3
 ; RV32-NEXT:    add a1, sp, a1
 ; RV32-NEXT:    addi a1, a1, 16
 ; RV32-NEXT:    vs8r.v v24, (a1) # Unknown-size Folded Spill
 ; RV32-NEXT:    csrr a1, vlenb
-; RV32-NEXT:    li a3, 84
+; RV32-NEXT:    li a3, 80
 ; RV32-NEXT:    mul a1, a1, a3
 ; RV32-NEXT:    add a1, sp, a1
 ; RV32-NEXT:    addi a1, a1, 16
@@ -250,14 +249,14 @@ define {<8 x i64>, <8 x i64>, <8 x i64>, <8 x i64>, <8 x i64>, <8 x i64>} @load_
 ; RV32-NEXT:    vsetivli zero, 16, e32, m4, ta, ma
 ; RV32-NEXT:    vmerge.vvm v20, v24, v16, v0
 ; RV32-NEXT:    csrr a1, vlenb
-; RV32-NEXT:    li a3, 40
+; RV32-NEXT:    li a3, 36
 ; RV32-NEXT:    mul a1, a1, a3
 ; RV32-NEXT:    add a1, sp, a1
 ; RV32-NEXT:    addi a1, a1, 16
 ; RV32-NEXT:    vs4r.v v20, (a1) # Unknown-size Folded Spill
 ; RV32-NEXT:    vmv1r.v v0, v7
 ; RV32-NEXT:    csrr a1, vlenb
-; RV32-NEXT:    li a3, 76
+; RV32-NEXT:    li a3, 72
 ; RV32-NEXT:    mul a1, a1, a3
 ; RV32-NEXT:    add a1, sp, a1
 ; RV32-NEXT:    addi a1, a1, 16
@@ -267,7 +266,7 @@ define {<8 x i64>, <8 x i64>, <8 x i64>, <8 x i64>, <8 x i64>, <8 x i64>} @load_
 ; RV32-NEXT:    vsetivli zero, 16, e64, m8, ta, ma
 ; RV32-NEXT:    vrgatherei16.vv v8, v24, v4
 ; RV32-NEXT:    csrr a1, vlenb
-; RV32-NEXT:    li a3, 44
+; RV32-NEXT:    li a3, 40
 ; RV32-NEXT:    mul a1, a1, a3
 ; RV32-NEXT:    add a1, sp, a1
 ; RV32-NEXT:    addi a1, a1, 16
@@ -282,14 +281,14 @@ define {<8 x i64>, <8 x i64>, <8 x i64>, <8 x i64>, <8 x i64>, <8 x i64>} @load_
 ; RV32-NEXT:    vle16.v v14, (a4)
 ; RV32-NEXT:    vmv.s.x v12, a3
 ; RV32-NEXT:    csrr a1, vlenb
-; RV32-NEXT:    li a3, 84
+; RV32-NEXT:    li a3, 80
 ; RV32-NEXT:    mul a1, a1, a3
 ; RV32-NEXT:    add a1, sp, a1
 ; RV32-NEXT:    addi a1, a1, 16
 ; RV32-NEXT:    vl8r.v v24, (a1) # Unknown-size Folded Reload
 ; RV32-NEXT:    vmv4r.v v8, v24
 ; RV32-NEXT:    csrr a1, vlenb
-; RV32-NEXT:    li a3, 60
+; RV32-NEXT:    li a3, 56
 ; RV32-NEXT:    mul a1, a1, a3
 ; RV32-NEXT:    add a1, sp, a1
 ; RV32-NEXT:    addi a1, a1, 16
@@ -304,8 +303,7 @@ define {<8 x i64>, <8 x i64>, <8 x i64>, <8 x i64>, <8 x i64>, <8 x i64>} @load_
 ; RV32-NEXT:    vs4r.v v8, (a1) # Unknown-size Folded Spill
 ; RV32-NEXT:    vmv1r.v v0, v12
 ; RV32-NEXT:    csrr a1, vlenb
-; RV32-NEXT:    li a3, 68
-; RV32-NEXT:    mul a1, a1, a3
+; RV32-NEXT:    slli a1, a1, 6
 ; RV32-NEXT:    add a1, sp, a1
 ; RV32-NEXT:    addi a1, a1, 16
 ; RV32-NEXT:    vl8r.v v24, (a1) # Unknown-size Folded Reload
@@ -314,7 +312,8 @@ define {<8 x i64>, <8 x i64>, <8 x i64>, <8 x i64>, <8 x i64>, <8 x i64>} @load_
 ; RV32-NEXT:    vsetivli zero, 16, e64, m8, ta, ma
 ; RV32-NEXT:    vrgatherei16.vv v16, v24, v14
 ; RV32-NEXT:    csrr a1, vlenb
-; RV32-NEXT:    slli a1, a1, 5
+; RV32-NEXT:    li a3, 28
+; RV32-NEXT:    mul a1, a1, a3
 ; RV32-NEXT:    add a1, sp, a1
 ; RV32-NEXT:    addi a1, a1, 16
 ; RV32-NEXT:    vs8r.v v16, (a1) # Unknown-size Folded Spill
@@ -330,22 +329,22 @@ define {<8 x i64>, <8 x i64>, <8 x i64>, <8 x i64>, <8 x i64>, <8 x i64>} @load_
 ; RV32-NEXT:    addi a4, a4, 12
 ; RV32-NEXT:    addi a5, a5, 768
 ; RV32-NEXT:    addi a7, a7, -1024
-; RV32-NEXT:    vmv.s.x v1, a6
-; RV32-NEXT:    vmv.s.x v12, t0
+; RV32-NEXT:    vmv.s.x v13, a6
+; RV32-NEXT:    vmv.s.x v2, t0
 ; RV32-NEXT:    vmv.s.x v0, a1
-; RV32-NEXT:    vmv.s.x v3, a3
-; RV32-NEXT:    vmv.s.x v2, a4
-; RV32-NEXT:    vmv.s.x v13, a5
-; RV32-NEXT:    vmv.s.x v14, a7
+; RV32-NEXT:    vmv.s.x v12, a3
+; RV32-NEXT:    vmv.s.x v3, a4
+; RV32-NEXT:    vmv.s.x v14, a5
+; RV32-NEXT:    vmv.s.x v1, a7
 ; RV32-NEXT:    csrr a1, vlenb
-; RV32-NEXT:    li a3, 60
+; RV32-NEXT:    li a3, 56
 ; RV32-NEXT:    mul a1, a1, a3
 ; RV32-NEXT:    add a1, sp, a1
 ; RV32-NEXT:    addi a1, a1, 16
 ; RV32-NEXT:    vl8r.v v16, (a1) # Unknown-size Folded Reload
 ; RV32-NEXT:    vmv4r.v v8, v16
 ; RV32-NEXT:    csrr a1, vlenb
-; RV32-NEXT:    li a3, 84
+; RV32-NEXT:    li a3, 80
 ; RV32-NEXT:    mul a1, a1, a3
 ; RV32-NEXT:    add a1, sp, a1
 ; RV32-NEXT:    addi a1, a1, 16
@@ -354,15 +353,14 @@ define {<8 x i64>, <8 x i64>, <8 x i64>, <8 x i64>, <8 x i64>, <8 x i64>} @load_
 ; RV32-NEXT:    vmerge.vvm v20, v8, v16, v0
 ; RV32-NEXT:    addi a1, sp, 16
 ; RV32-NEXT:    vs4r.v v20, (a1) # Unknown-size Folded Spill
-; RV32-NEXT:    vmv1r.v v0, v3
+; RV32-NEXT:    vmv1r.v v0, v12
 ; RV32-NEXT:    csrr a1, vlenb
-; RV32-NEXT:    li a3, 68
-; RV32-NEXT:    mul a1, a1, a3
+; RV32-NEXT:    slli a1, a1, 6
 ; RV32-NEXT:    add a1, sp, a1
 ; RV32-NEXT:    addi a1, a1, 16
 ; RV32-NEXT:    vl8r.v v16, (a1) # Unknown-size Folded Reload
 ; RV32-NEXT:    csrr a1, vlenb
-; RV32-NEXT:    li a3, 76
+; RV32-NEXT:    li a3, 72
 ; RV32-NEXT:    mul a1, a1, a3
 ; RV32-NEXT:    add a1, sp, a1
 ; RV32-NEXT:    addi a1, a1, 16
@@ -370,28 +368,23 @@ define {<8 x i64>, <8 x i64>, <8 x i64>, <8 x i64>, <8 x i64>, <8 x i64>} @load_
 ; RV32-NEXT:    vsetvli zero, a2, e32, m8, ta, ma
 ; RV32-NEXT:    vmerge.vvm v24, v16, v24, v0
 ; RV32-NEXT:    csrr a1, vlenb
-; RV32-NEXT:    slli a1, a1, 4
+; RV32-NEXT:    li a3, 12
+; RV32-NEXT:    mul a1, a1, a3
 ; RV32-NEXT:    add a1, sp, a1
 ; RV32-NEXT:    addi a1, a1, 16
 ; RV32-NEXT:    vs8r.v v24, (a1) # Unknown-size Folded Spill
-; RV32-NEXT:    vmv1r.v v0, v2
+; RV32-NEXT:    vmv1r.v v0, v3
 ; RV32-NEXT:    csrr a1, vlenb
-; RV32-NEXT:    li a3, 84
+; RV32-NEXT:    li a3, 80
 ; RV32-NEXT:    mul a1, a1, a3
 ; RV32-NEXT:    add a1, sp, a1
 ; RV32-NEXT:    addi a1, a1, 16
 ; RV32-NEXT:    vl8r.v v24, (a1) # Unknown-size Folded Reload
 ; RV32-NEXT:    vsetivli zero, 16, e32, m4, ta, ma
-; RV32-NEXT:    vmerge.vvm v24, v8, v24, v0
-; RV32-NEXT:    csrr a1, vlenb
-; RV32-NEXT:    li a3, 12
-; RV32-NEXT:    mul a1, a1, a3
-; RV32-NEXT:    add a1, sp, a1
-; RV32-NEXT:    addi a1, a1, 16
-; RV32-NEXT:    vs4r.v v24, (a1) # Unknown-size Folded Spill
-; RV32-NEXT:    vmv1r.v v0, v13
+; RV32-NEXT:    vmerge.vvm v4, v8, v24, v0
+; RV32-NEXT:    vmv1r.v v0, v14
 ; RV32-NEXT:    csrr a1, vlenb
-; RV32-NEXT:    li a3, 76
+; RV32-NEXT:    li a3, 72
 ; RV32-NEXT:    mul a1, a1, a3
 ; RV32-NEXT:    add a1, sp, a1
 ; RV32-NEXT:    addi a1, a1, 16
@@ -403,45 +396,44 @@ define {<8 x i64>, <8 x i64>, <8 x i64>, <8 x i64>, <8 x i64>, <8 x i64>} @load_
 ; RV32-NEXT:    add a1, sp, a1
 ; RV32-NEXT:    addi a1, a1, 16
 ; RV32-NEXT:    vs8r.v v24, (a1) # Unknown-size Folded Spill
-; RV32-NEXT:    vmv1r.v v0, v1
+; RV32-NEXT:    vmv1r.v v0, v13
 ; RV32-NEXT:    csrr a1, vlenb
-; RV32-NEXT:    li a3, 84
+; RV32-NEXT:    li a3, 80
 ; RV32-NEXT:    mul a1, a1, a3
 ; RV32-NEXT:    add a1, sp, a1
 ; RV32-NEXT:    addi a1, a1, 16
 ; RV32-NEXT:    vl8r.v v24, (a1) # Unknown-size Folded Reload
 ; RV32-NEXT:    vsetivli zero, 16, e32, m4, ta, ma
-; RV32-NEXT:    vmerge.vvm v4, v8, v24, v0
+; RV32-NEXT:    vmerge.vvm v12, v8, v24, v0
 ; RV32-NEXT:    csrr a1, vlenb
-; RV32-NEXT:    li a3, 28
+; RV32-NEXT:    li a3, 20
 ; RV32-NEXT:    mul a1, a1, a3
 ; RV32-NEXT:    add a1, sp, a1
 ; RV32-NEXT:    addi a1, a1, 16
-; RV32-NEXT:    vs4r.v v4, (a1) # Unknown-size Folded Spill
-; RV32-NEXT:    vmv1r.v v0, v14
+; RV32-NEXT:    vs4r.v v12, (a1) # Unknown-size Folded Spill
+; RV32-NEXT:    vmv1r.v v0, v1
 ; RV32-NEXT:    csrr a1, vlenb
-; RV32-NEXT:    li a3, 76
+; RV32-NEXT:    li a3, 72
 ; RV32-NEXT:    mul a1, a1, a3
 ; RV32-NEXT:    add a1, sp, a1
 ; RV32-NEXT:    addi a1, a1, 16
 ; RV32-NEXT:    vl8r.v v16, (a1) # Unknown-size Folded Reload
 ; RV32-NEXT:    csrr a1, vlenb
-; RV32-NEXT:    li a3, 68
-; RV32-NEXT:    mul a1, a1, a3
+; RV32-NEXT:    slli a1, a1, 6
 ; RV32-NEXT:    add a1, sp, a1
 ; RV32-NEXT:    addi a1, a1, 16
 ; RV32-NEXT:    vl8r.v v24, (a1) # Unknown-size Folded Reload
 ; RV32-NEXT:    vsetvli zero, a2, e32, m8, ta, ma
 ; RV32-NEXT:    vmerge.vvm v16, v24, v16, v0
 ; RV32-NEXT:    csrr a1, vlenb
-; RV32-NEXT:    li a2, 76
+; RV32-NEXT:    li a2, 72
 ; RV32-NEXT:    mul a1, a1, a2
 ; RV32-NEXT:    add a1, sp, a1
 ; RV32-NEXT:    addi a1, a1, 16
 ; RV32-NEXT:    vs8r.v v16, (a1) # Unknown-size Folded Spill
-; RV32-NEXT:    vmv1r.v v0, v12
+; RV32-NEXT:    vmv1r.v v0, v2
 ; RV32-NEXT:    csrr a1, vlenb
-; RV32-NEXT:    li a2, 84
+; RV32-NEXT:    li a2, 80
 ; RV32-NEXT:    mul a1, a1, a2
 ; RV32-NEXT:    add a1, sp, a1
 ; RV32-NEXT:    addi a1, a1, 16
@@ -449,190 +441,208 @@ define {<8 x i64>, <8 x i64>, <8 x i64>, <8 x i64>, <8 x i64>, <8 x i64>} @load_
 ; RV32-NEXT:    vsetivli zero, 16, e32, m4, ta, ma
 ; RV32-NEXT:    vmerge.vvm v8, v8, v16, v0
 ; RV32-NEXT:    csrr a1, vlenb
-; RV32-NEXT:    li a2, 68
-; RV32-NEXT:    mul a1, a1, a2
+; RV32-NEXT:    slli a1, a1, 6
 ; RV32-NEXT:    add a1, sp, a1
 ; RV32-NEXT:    addi a1, a1, 16
 ; RV32-NEXT:    vs4r.v v8, (a1) # Unknown-size Folded Spill
 ; RV32-NEXT:    lui a1, 32
 ; RV32-NEXT:    addi a1, a1, 4
 ; RV32-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
-; RV32-NEXT:    vmv.v.x v16, a1
+; RV32-NEXT:    vmv.v.x v12, a1
 ; RV32-NEXT:    csrr a1, vlenb
-; RV32-NEXT:    li a2, 40
+; RV32-NEXT:    li a2, 36
 ; RV32-NEXT:    mul a1, a1, a2
 ; RV32-NEXT:    add a1, sp, a1
 ; RV32-NEXT:    addi a1, a1, 16
 ; RV32-NEXT:    vl4r.v v8, (a1) # Unknown-size Folded Reload
 ; RV32-NEXT:    vsetivli zero, 8, e64, m4, ta, ma
-; RV32-NEXT:    vrgatherei16.vv v20, v8, v16
+; RV32-NEXT:    vrgatherei16.vv v16, v8, v12
 ; RV32-NEXT:    csrr a1, vlenb
-; RV32-NEXT:    li a2, 52
+; RV32-NEXT:    li a2, 48
 ; RV32-NEXT:    mul a1, a1, a2
 ; RV32-NEXT:    add a1, sp, a1
 ; RV32-NEXT:    addi a1, a1, 16
 ; RV32-NEXT:    vl8r.v v8, (a1) # Unknown-size Folded Reload
 ; RV32-NEXT:    vsetivli zero, 12, e32, m4, tu, ma
-; RV32-NEXT:    vmv.v.v v20, v8
+; RV32-NEXT:    vmv.v.v v16, v8
 ; RV32-NEXT:    csrr a1, vlenb
-; RV32-NEXT:    li a2, 84
+; RV32-NEXT:    li a2, 80
 ; RV32-NEXT:    mul a1, a1, a2
 ; RV32-NEXT:    add a1, sp, a1
 ; RV32-NEXT:    addi a1, a1, 16
-; RV32-NEXT:    vs4r.v v20, (a1) # Unknown-size Folded Spill
+; RV32-NEXT:    vs4r.v v16, (a1) # Unknown-size Folded Spill
 ; RV32-NEXT:    lui a1, 48
 ; RV32-NEXT:    lui a2, %hi(.LCPI8_3)
 ; RV32-NEXT:    addi a2, a2, %lo(.LCPI8_3)
 ; RV32-NEXT:    addi a1, a1, 5
 ; RV32-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
-; RV32-NEXT:    vle16.v v28, (a2)
+; RV32-NEXT:    vle16.v v24, (a2)
 ; RV32-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
-; RV32-NEXT:    vmv.v.x v20, a1
+; RV32-NEXT:    vmv.v.x v25, a1
 ; RV32-NEXT:    csrr a1, vlenb
 ; RV32-NEXT:    li a2, 24
 ; RV32-NEXT:    mul a1, a1, a2
 ; RV32-NEXT:    add a1, sp, a1
 ; RV32-NEXT:    addi a1, a1, 16
-; RV32-NEXT:    vl4r.v v12, (a1) # Unknown-size Folded Reload
+; RV32-NEXT:    vl4r.v v8, (a1) # Unknown-size Folded Reload
 ; RV32-NEXT:    vsetivli zero, 8, e64, m4, ta, ma
-; RV32-NEXT:    vrgatherei16.vv v8, v12, v20
+; RV32-NEXT:    vrgatherei16.vv v16, v8, v25
 ; RV32-NEXT:    csrr a1, vlenb
-; RV32-NEXT:    li a2, 44
+; RV32-NEXT:    li a2, 40
 ; RV32-NEXT:    mul a1, a1, a2
 ; RV32-NEXT:    add a1, sp, a1
 ; RV32-NEXT:    addi a1, a1, 16
-; RV32-NEXT:    vl8r.v v16, (a1) # Unknown-size Folded Reload
+; RV32-NEXT:    vl8r.v v8, (a1) # Unknown-size Folded Reload
 ; RV32-NEXT:    vsetivli zero, 12, e32, m4, tu, ma
-; RV32-NEXT:    vmv.v.v v8, v16
+; RV32-NEXT:    vmv.v.v v16, v8
 ; RV32-NEXT:    csrr a1, vlenb
-; RV32-NEXT:    li a2, 52
+; RV32-NEXT:    li a2, 56
 ; RV32-NEXT:    mul a1, a1, a2
 ; RV32-NEXT:    add a1, sp, a1
 ; RV32-NEXT:    addi a1, a1, 16
-; RV32-NEXT:    vs4r.v v8, (a1) # Unknown-size Folded Spill
+; RV32-NEXT:    vs4r.v v16, (a1) # Unknown-size Folded Spill
 ; RV32-NEXT:    addi a1, sp, 16
-; RV32-NEXT:    vl4r.v v12, (a1) # Unknown-size Folded Reload
+; RV32-NEXT:    vl4r.v v8, (a1) # Unknown-size Folded Reload
 ; RV32-NEXT:    vsetivli zero, 8, e64, m4, ta, ma
-; RV32-NEXT:    vrgatherei16.vv v24, v12, v28
+; RV32-NEXT:    vrgatherei16.vv v16, v8, v24
 ; RV32-NEXT:    csrr a1, vlenb
-; RV32-NEXT:    slli a1, a1, 5
+; RV32-NEXT:    li a2, 28
+; RV32-NEXT:    mul a1, a1, a2
 ; RV32-NEXT:    add a1, sp, a1
 ; RV32-NEXT:    addi a1, a1, 16
-; RV32-NEXT:    vl8r.v v16, (a1) # Unknown-size Folded Reload
+; RV32-NEXT:    vl8r.v v8, (a1) # Unknown-size Folded Reload
 ; RV32-NEXT:    vsetivli zero, 10, e32, m4, tu, ma
-; RV32-NEXT:    vmv.v.v v24, v16
+; RV32-NEXT:    vmv.v.v v16, v8
+; RV32-NEXT:    csrr a1, vlenb
+; RV32-NEXT:    li a2, 40
+; RV32-NEXT:    mul a1, a1, a2
+; RV32-NEXT:    add a1, sp, a1
+; RV32-NEXT:    addi a1, a1, 16
+; RV32-NEXT:    vs4r.v v16, (a1) # Unknown-size Folded Spill
 ; RV32-NEXT:    lui a1, %hi(.LCPI8_4)
 ; RV32-NEXT:    addi a1, a1, %lo(.LCPI8_4)
 ; RV32-NEXT:    lui a2, %hi(.LCPI8_5)
 ; RV32-NEXT:    addi a2, a2, %lo(.LCPI8_5)
 ; RV32-NEXT:    vsetivli zero, 16, e16, m2, ta, ma
-; RV32-NEXT:    vle16.v v12, (a1)
+; RV32-NEXT:    vle16.v v26, (a1)
 ; RV32-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
-; RV32-NEXT:    vle16.v v28, (a2)
+; RV32-NEXT:    vle16.v v24, (a2)
 ; RV32-NEXT:    lui a1, %hi(.LCPI8_6)
 ; RV32-NEXT:    addi a1, a1, %lo(.LCPI8_6)
 ; RV32-NEXT:    vsetivli zero, 16, e64, m8, ta, ma
-; RV32-NEXT:    vle16.v v30, (a1)
-; RV32-NEXT:    csrr a1, vlenb
-; RV32-NEXT:    slli a1, a1, 4
-; RV32-NEXT:    add a1, sp, a1
-; RV32-NEXT:    addi a1, a1, 16
-; RV32-NEXT:    vl8r.v v0, (a1) # Unknown-size Folded Reload
-; RV32-NEXT:    vrgatherei16.vv v16, v0, v12
+; RV32-NEXT:    vle16.v v2, (a1)
 ; RV32-NEXT:    csrr a1, vlenb
 ; RV32-NEXT:    li a2, 12
 ; RV32-NEXT:    mul a1, a1, a2
 ; RV32-NEXT:    add a1, sp, a1
 ; RV32-NEXT:    addi a1, a1, 16
-; RV32-NEXT:    vl4r.v v20, (a1) # Unknown-size Folded Reload
+; RV32-NEXT:    vl8r.v v16, (a1) # Unknown-size Folded Reload
+; RV32-NEXT:    vrgatherei16.vv v8, v16, v26
 ; RV32-NEXT:    vsetivli zero, 8, e64, m4, ta, ma
-; RV32-NEXT:    vrgatherei16.vv v12, v20, v28
+; RV32-NEXT:    vrgatherei16.vv v20, v4, v24
 ; RV32-NEXT:    vsetivli zero, 10, e32, m4, tu, ma
-; RV32-NEXT:    vmv.v.v v12, v16
+; RV32-NEXT:    vmv.v.v v20, v8
 ; RV32-NEXT:    csrr a1, vlenb
 ; RV32-NEXT:    slli a1, a1, 2
 ; RV32-NEXT:    add a1, sp, a1
 ; RV32-NEXT:    addi a1, a1, 16
-; RV32-NEXT:    vl8r.v v0, (a1) # Unknown-size Folded Reload
+; RV32-NEXT:    vl8r.v v8, (a1) # Unknown-size Folded Reload
 ; RV32-NEXT:    vsetivli zero, 16, e64, m8, ta, ma
-; RV32-NEXT:    vrgatherei16.vv v16, v0, v30
+; RV32-NEXT:    vrgatherei16.vv v24, v8, v2
+; RV32-NEXT:    csrr a1, vlenb
+; RV32-NEXT:    li a2, 48
+; RV32-NEXT:    mul a1, a1, a2
+; RV32-NEXT:    add a1, sp, a1
+; RV32-NEXT:    addi a1, a1, 16
+; RV32-NEXT:    vs8r.v v24, (a1) # Unknown-size Folded Spill
 ; RV32-NEXT:    lui a1, %hi(.LCPI8_7)
 ; RV32-NEXT:    addi a1, a1, %lo(.LCPI8_7)
 ; RV32-NEXT:    lui a2, %hi(.LCPI8_8)
 ; RV32-NEXT:    addi a2, a2, %lo(.LCPI8_8)
 ; RV32-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
-; RV32-NEXT:    vle16.v v20, (a1)
+; RV32-NEXT:    vle16.v v12, (a1)
 ; RV32-NEXT:    lui a1, %hi(.LCPI8_9)
 ; RV32-NEXT:    addi a1, a1, %lo(.LCPI8_9)
 ; RV32-NEXT:    vsetivli zero, 16, e16, m2, ta, ma
-; RV32-NEXT:    vle16.v v8, (a2)
+; RV32-NEXT:    vle16.v v16, (a2)
 ; RV32-NEXT:    vsetivli zero, 8, e64, m4, ta, ma
-; RV32-NEXT:    vle16.v v10, (a1)
+; RV32-NEXT:    vle16.v v18, (a1)
 ; RV32-NEXT:    csrr a1, vlenb
-; RV32-NEXT:    li a2, 28
+; RV32-NEXT:    li a2, 20
 ; RV32-NEXT:    mul a1, a1, a2
 ; RV32-NEXT:    add a1, sp, a1
 ; RV32-NEXT:    addi a1, a1, 16
 ; RV32-NEXT:    vl4r.v v0, (a1) # Unknown-size Folded Reload
-; RV32-NEXT:    vrgatherei16.vv v28, v0, v20
+; RV32-NEXT:    vrgatherei16.vv v24, v0, v12
+; RV32-NEXT:    csrr a1, vlenb
+; RV32-NEXT:    li a2, 48
+; RV32-NEXT:    mul a1, a1, a2
+; RV32-NEXT:    add a1, sp, a1
+; RV32-NEXT:    addi a1, a1, 16
+; RV32-NEXT:    vl8r.v v0, (a1) # Unknown-size Folded Reload
 ; RV32-NEXT:    vsetivli zero, 10, e32, m4, tu, ma
-; RV32-NEXT:    vmv.v.v v28, v16
+; RV32-NEXT:    vmv.v.v v24, v0
 ; RV32-NEXT:    csrr a1, vlenb
-; RV32-NEXT:    li a2, 76
+; RV32-NEXT:    li a2, 72
 ; RV32-NEXT:    mul a1, a1, a2
 ; RV32-NEXT:    add a1, sp, a1
 ; RV32-NEXT:    addi a1, a1, 16
 ; RV32-NEXT:    vl8r.v v0, (a1) # Unknown-size Folded Reload
 ; RV32-NEXT:    vsetivli zero, 16, e64, m8, ta, ma
-; RV32-NEXT:    vrgatherei16.vv v16, v0, v8
+; RV32-NEXT:    vrgatherei16.vv v8, v0, v16
 ; RV32-NEXT:    csrr a1, vlenb
-; RV32-NEXT:    li a2, 60
+; RV32-NEXT:    li a2, 48
 ; RV32-NEXT:    mul a1, a1, a2
 ; RV32-NEXT:    add a1, sp, a1
 ; RV32-NEXT:    addi a1, a1, 16
-; RV32-NEXT:    vs8r.v v16, (a1) # Unknown-size Folded Spill
+; RV32-NEXT:    vs8r.v v8, (a1) # Unknown-size Folded Spill
 ; RV32-NEXT:    csrr a1, vlenb
-; RV32-NEXT:    li a2, 68
-; RV32-NEXT:    mul a1, a1, a2
+; RV32-NEXT:    slli a1, a1, 6
 ; RV32-NEXT:    add a1, sp, a1
 ; RV32-NEXT:    addi a1, a1, 16
 ; RV32-NEXT:    vl4r.v v4, (a1) # Unknown-size Folded Reload
 ; RV32-NEXT:    vsetivli zero, 8, e64, m4, ta, ma
-; RV32-NEXT:    vrgatherei16.vv v16, v4, v10
+; RV32-NEXT:    vrgatherei16.vv v8, v4, v18
 ; RV32-NEXT:    csrr a1, vlenb
-; RV32-NEXT:    li a2, 60
+; RV32-NEXT:    li a2, 48
 ; RV32-NEXT:    mul a1, a1, a2
 ; RV32-NEXT:    add a1, sp, a1
 ; RV32-NEXT:    addi a1, a1, 16
 ; RV32-NEXT:    vl8r.v v0, (a1) # Unknown-size Folded Reload
 ; RV32-NEXT:    vsetivli zero, 10, e32, m4, tu, ma
-; RV32-NEXT:    vmv.v.v v16, v0
+; RV32-NEXT:    vmv.v.v v8, v0
 ; RV32-NEXT:    addi a1, a0, 320
 ; RV32-NEXT:    vsetivli zero, 16, e32, m4, ta, ma
-; RV32-NEXT:    vse32.v v16, (a1)
+; RV32-NEXT:    vse32.v v8, (a1)
 ; RV32-NEXT:    addi a1, a0, 256
-; RV32-NEXT:    vse32.v v28, (a1)
+; RV32-NEXT:    vse32.v v24, (a1)
 ; RV32-NEXT:    addi a1, a0, 192
-; RV32-NEXT:    vse32.v v12, (a1)
+; RV32-NEXT:    vse32.v v20, (a1)
 ; RV32-NEXT:    addi a1, a0, 128
-; RV32-NEXT:    vse32.v v24, (a1)
+; RV32-NEXT:    csrr a2, vlenb
+; RV32-NEXT:    li a3, 40
+; RV32-NEXT:    mul a2, a2, a3
+; RV32-NEXT:    add a2, sp, a2
+; RV32-NEXT:    addi a2, a2, 16
+; RV32-NEXT:    vl4r.v v8, (a2) # Unknown-size Folded Reload
+; RV32-NEXT:    vse32.v v8, (a1)
 ; RV32-NEXT:    addi a1, a0, 64
 ; RV32-NEXT:    csrr a2, vlenb
-; RV32-NEXT:    li a3, 52
+; RV32-NEXT:    li a3, 56
 ; RV32-NEXT:    mul a2, a2, a3
 ; RV32-NEXT:    add a2, sp, a2
 ; RV32-NEXT:    addi a2, a2, 16
 ; RV32-NEXT:    vl4r.v v8, (a2) # Unknown-size Folded Reload
 ; RV32-NEXT:    vse32.v v8, (a1)
 ; RV32-NEXT:    csrr a1, vlenb
-; RV32-NEXT:    li a2, 84
+; RV32-NEXT:    li a2, 80
 ; RV32-NEXT:    mul a1, a1, a2
 ; RV32-NEXT:    add a1, sp, a1
 ; RV32-NEXT:    addi a1, a1, 16
 ; RV32-NEXT:    vl4r.v v8, (a1) # Unknown-size Folded Reload
 ; RV32-NEXT:    vse32.v v8, (a0)
 ; RV32-NEXT:    csrr a0, vlenb
-; RV32-NEXT:    li a1, 92
+; RV32-NEXT:    li a1, 88
 ; RV32-NEXT:    mul a0, a0, a1
 ; RV32-NEXT:    add sp, sp, a0
 ; RV32-NEXT:    .cfi_def_cfa sp, 16
@@ -665,145 +675,154 @@ define {<8 x i64>, <8 x i64>, <8 x i64>, <8 x i64>, <8 x i64>, <8 x i64>} @load_
 ; RV64-NEXT:    vmv.v.x v17, a6
 ; RV64-NEXT:    addi a6, a2, 65
 ; RV64-NEXT:    vsetivli zero, 8, e64, m4, ta, ma
-; RV64-NEXT:    vrgather.vi v24, v8, 4
+; RV64-NEXT:    vrgather.vi v4, v8, 4
 ; RV64-NEXT:    vrgather.vi v20, v8, 5
 ; RV64-NEXT:    csrr a7, vlenb
-; RV64-NEXT:    li t0, 68
+; RV64-NEXT:    li t0, 84
 ; RV64-NEXT:    mul a7, a7, t0
 ; RV64-NEXT:    add a7, sp, a7
 ; RV64-NEXT:    addi a7, a7, 16
 ; RV64-NEXT:    vs4r.v v20, (a7) # Unknown-size Folded Spill
 ; RV64-NEXT:    vrgatherei16.vv v20, v8, v16
 ; RV64-NEXT:    csrr a7, vlenb
-; RV64-NEXT:    li t0, 84
-; RV64-NEXT:    mul a7, a7, t0
+; RV64-NEXT:    slli a7, a7, 6
 ; RV64-NEXT:    add a7, sp, a7
 ; RV64-NEXT:    addi a7, a7, 16
 ; RV64-NEXT:    vs4r.v v20, (a7) # Unknown-size Folded Spill
 ; RV64-NEXT:    vrgatherei16.vv v20, v8, v17
 ; RV64-NEXT:    csrr a7, vlenb
-; RV64-NEXT:    li t0, 72
+; RV64-NEXT:    li t0, 56
 ; RV64-NEXT:    mul a7, a7, t0
 ; RV64-NEXT:    add a7, sp, a7
 ; RV64-NEXT:    addi a7, a7, 16
 ; RV64-NEXT:    vs4r.v v20, (a7) # Unknown-size Folded Spill
 ; RV64-NEXT:    vrgather.vi v16, v8, 2
 ; RV64-NEXT:    csrr a7, vlenb
-; RV64-NEXT:    slli a7, a7, 6
+; RV64-NEXT:    li t0, 72
+; RV64-NEXT:    mul a7, a7, t0
 ; RV64-NEXT:    add a7, sp, a7
 ; RV64-NEXT:    addi a7, a7, 16
 ; RV64-NEXT:    vs4r.v v16, (a7) # Unknown-size Folded Spill
 ; RV64-NEXT:    vrgather.vi v16, v8, 3
 ; RV64-NEXT:    csrr a7, vlenb
-; RV64-NEXT:    li t0, 56
+; RV64-NEXT:    li t0, 68
 ; RV64-NEXT:    mul a7, a7, t0
 ; RV64-NEXT:    add a7, sp, a7
 ; RV64-NEXT:    addi a7, a7, 16
 ; RV64-NEXT:    vs4r.v v16, (a7) # Unknown-size Folded Spill
 ; RV64-NEXT:    vsetivli zero, 8, e64, m8, ta, ma
-; RV64-NEXT:    vslidedown.vi v16, v8, 8
+; RV64-NEXT:    vslidedown.vi v8, v8, 8
 ; RV64-NEXT:    csrr a7, vlenb
-; RV64-NEXT:    li t0, 48
+; RV64-NEXT:    li t0, 40
 ; RV64-NEXT:    mul a7, a7, t0
 ; RV64-NEXT:    add a7, sp, a7
 ; RV64-NEXT:    addi a7, a7, 16
-; RV64-NEXT:    vs8r.v v16, (a7) # Unknown-size Folded Spill
-; RV64-NEXT:    vmv.s.x v21, a4
+; RV64-NEXT:    vs8r.v v8, (a7) # Unknown-size Folded Spill
+; RV64-NEXT:    vmv.s.x v0, a4
+; RV64-NEXT:    csrr a4, vlenb
+; RV64-NEXT:    slli a4, a4, 5
+; RV64-NEXT:    add a4, sp, a4
+; RV64-NEXT:    addi a4, a4, 16
+; RV64-NEXT:    vs1r.v v0, (a4) # Unknown-size Folded Spill
 ; RV64-NEXT:    vsetivli zero, 16, e64, m8, ta, ma
-; RV64-NEXT:    vle64.v v8, (a1)
-; RV64-NEXT:    vle64.v v0, (a3)
+; RV64-NEXT:    vle64.v v24, (a1)
+; RV64-NEXT:    vle64.v v16, (a3)
 ; RV64-NEXT:    csrr a1, vlenb
-; RV64-NEXT:    li a3, 40
+; RV64-NEXT:    li a3, 76
 ; RV64-NEXT:    mul a1, a1, a3
 ; RV64-NEXT:    add a1, sp, a1
 ; RV64-NEXT:    addi a1, a1, 16
-; RV64-NEXT:    vs8r.v v0, (a1) # Unknown-size Folded Spill
-; RV64-NEXT:    vle16.v v2, (a5)
-; RV64-NEXT:    vmv.s.x v20, a6
-; RV64-NEXT:    vmv1r.v v0, v21
-; RV64-NEXT:    vmv1r.v v7, v21
+; RV64-NEXT:    vs8r.v v16, (a1) # Unknown-size Folded Spill
+; RV64-NEXT:    vle16.v v12, (a5)
+; RV64-NEXT:    csrr a1, vlenb
+; RV64-NEXT:    slli a1, a1, 4
+; RV64-NEXT:    add a1, sp, a1
+; RV64-NEXT:    addi a1, a1, 16
+; RV64-NEXT:    vs2r.v v12, (a1) # Unknown-size Folded Spill
+; RV64-NEXT:    vmv.s.x v2, a6
 ; RV64-NEXT:    vsetivli zero, 8, e64, m4, ta, mu
-; RV64-NEXT:    vrgather.vi v24, v16, 2, v0.t
+; RV64-NEXT:    vrgather.vi v4, v8, 2, v0.t
 ; RV64-NEXT:    csrr a1, vlenb
 ; RV64-NEXT:    li a3, 60
 ; RV64-NEXT:    mul a1, a1, a3
 ; RV64-NEXT:    add a1, sp, a1
 ; RV64-NEXT:    addi a1, a1, 16
-; RV64-NEXT:    vs4r.v v24, (a1) # Unknown-size Folded Spill
-; RV64-NEXT:    vmv1r.v v0, v20
+; RV64-NEXT:    vs4r.v v4, (a1) # Unknown-size Folded Spill
+; RV64-NEXT:    vmv1r.v v0, v2
+; RV64-NEXT:    vmv8r.v v8, v24
 ; RV64-NEXT:    csrr a1, vlenb
-; RV64-NEXT:    li a3, 40
+; RV64-NEXT:    li a3, 48
 ; RV64-NEXT:    mul a1, a1, a3
 ; RV64-NEXT:    add a1, sp, a1
 ; RV64-NEXT:    addi a1, a1, 16
-; RV64-NEXT:    vl8r.v v16, (a1) # Unknown-size Folded Reload
+; RV64-NEXT:    vs8r.v v24, (a1) # Unknown-size Folded Spill
 ; RV64-NEXT:    vsetivli zero, 16, e64, m8, ta, ma
-; RV64-NEXT:    vmerge.vvm v24, v16, v8, v0
-; RV64-NEXT:    vmv8r.v v16, v8
+; RV64-NEXT:    vmerge.vvm v24, v16, v24, v0
 ; RV64-NEXT:    csrr a1, vlenb
-; RV64-NEXT:    li a3, 76
-; RV64-NEXT:    mul a1, a1, a3
+; RV64-NEXT:    slli a1, a1, 4
 ; RV64-NEXT:    add a1, sp, a1
 ; RV64-NEXT:    addi a1, a1, 16
-; RV64-NEXT:    vs8r.v v8, (a1) # Unknown-size Folded Spill
-; RV64-NEXT:    vrgatherei16.vv v8, v24, v2
+; RV64-NEXT:    vl2r.v v16, (a1) # Unknown-size Folded Reload
+; RV64-NEXT:    vrgatherei16.vv v0, v24, v16
 ; RV64-NEXT:    csrr a1, vlenb
-; RV64-NEXT:    slli a1, a1, 5
+; RV64-NEXT:    li a3, 24
+; RV64-NEXT:    mul a1, a1, a3
 ; RV64-NEXT:    add a1, sp, a1
 ; RV64-NEXT:    addi a1, a1, 16
-; RV64-NEXT:    vs8r.v v8, (a1) # Unknown-size Folded Spill
+; RV64-NEXT:    vs8r.v v0, (a1) # Unknown-size Folded Spill
 ; RV64-NEXT:    lui a1, 2
 ; RV64-NEXT:    lui a3, %hi(.LCPI8_1)
 ; RV64-NEXT:    addi a3, a3, %lo(.LCPI8_1)
 ; RV64-NEXT:    addi a1, a1, 130
-; RV64-NEXT:    vle16.v v8, (a3)
+; RV64-NEXT:    vle16.v v16, (a3)
 ; RV64-NEXT:    csrr a3, vlenb
-; RV64-NEXT:    slli a3, a3, 4
+; RV64-NEXT:    slli a3, a3, 3
 ; RV64-NEXT:    add a3, sp, a3
 ; RV64-NEXT:    addi a3, a3, 16
-; RV64-NEXT:    vs2r.v v8, (a3) # Unknown-size Folded Spill
+; RV64-NEXT:    vs2r.v v16, (a3) # Unknown-size Folded Spill
 ; RV64-NEXT:    vmv.s.x v2, a1
-; RV64-NEXT:    vmv1r.v v0, v7
-; RV64-NEXT:    addi a1, sp, 16
-; RV64-NEXT:    vs1r.v v7, (a1) # Unknown-size Folded Spill
 ; RV64-NEXT:    csrr a1, vlenb
-; RV64-NEXT:    li a3, 68
+; RV64-NEXT:    slli a1, a1, 5
+; RV64-NEXT:    add a1, sp, a1
+; RV64-NEXT:    addi a1, a1, 16
+; RV64-NEXT:    vl1r.v v0, (a1) # Unknown-size Folded Reload
+; RV64-NEXT:    csrr a1, vlenb
+; RV64-NEXT:    li a3, 84
 ; RV64-NEXT:    mul a1, a1, a3
 ; RV64-NEXT:    add a1, sp, a1
 ; RV64-NEXT:    addi a1, a1, 16
 ; RV64-NEXT:    vl4r.v v24, (a1) # Unknown-size Folded Reload
 ; RV64-NEXT:    csrr a1, vlenb
-; RV64-NEXT:    li a3, 48
+; RV64-NEXT:    li a3, 40
 ; RV64-NEXT:    mul a1, a1, a3
 ; RV64-NEXT:    add a1, sp, a1
 ; RV64-NEXT:    addi a1, a1, 16
-; RV64-NEXT:    vl8r.v v8, (a1) # Unknown-size Folded Reload
+; RV64-NEXT:    vl8r.v v16, (a1) # Unknown-size Folded Reload
 ; RV64-NEXT:    vsetivli zero, 8, e64, m4, ta, mu
-; RV64-NEXT:    vrgather.vi v24, v8, 3, v0.t
+; RV64-NEXT:    vrgather.vi v24, v16, 3, v0.t
 ; RV64-NEXT:    csrr a1, vlenb
-; RV64-NEXT:    li a3, 68
+; RV64-NEXT:    li a3, 84
 ; RV64-NEXT:    mul a1, a1, a3
 ; RV64-NEXT:    add a1, sp, a1
 ; RV64-NEXT:    addi a1, a1, 16
 ; RV64-NEXT:    vs4r.v v24, (a1) # Unknown-size Folded Spill
 ; RV64-NEXT:    vmv1r.v v0, v2
 ; RV64-NEXT:    csrr a1, vlenb
-; RV64-NEXT:    li a3, 40
+; RV64-NEXT:    li a3, 76
 ; RV64-NEXT:    mul a1, a1, a3
 ; RV64-NEXT:    add a1, sp, a1
 ; RV64-NEXT:    addi a1, a1, 16
-; RV64-NEXT:    vl8r.v v8, (a1) # Unknown-size Folded Reload
+; RV64-NEXT:    vl8r.v v16, (a1) # Unknown-size Folded Reload
 ; RV64-NEXT:    vsetivli zero, 16, e64, m8, ta, ma
-; RV64-NEXT:    vmerge.vvm v24, v8, v16, v0
+; RV64-NEXT:    vmerge.vvm v24, v16, v8, v0
 ; RV64-NEXT:    csrr a1, vlenb
-; RV64-NEXT:    slli a1, a1, 4
+; RV64-NEXT:    slli a1, a1, 3
 ; RV64-NEXT:    add a1, sp, a1
 ; RV64-NEXT:    addi a1, a1, 16
-; RV64-NEXT:    vl2r.v v16, (a1) # Unknown-size Folded Reload
-; RV64-NEXT:    vrgatherei16.vv v0, v24, v16
+; RV64-NEXT:    vl2r.v v8, (a1) # Unknown-size Folded Reload
+; RV64-NEXT:    vrgatherei16.vv v0, v24, v8
 ; RV64-NEXT:    csrr a1, vlenb
-; RV64-NEXT:    li a3, 24
-; RV64-NEXT:    mul a1, a1, a3
+; RV64-NEXT:    slli a1, a1, 4
 ; RV64-NEXT:    add a1, sp, a1
 ; RV64-NEXT:    addi a1, a1, 16
 ; RV64-NEXT:    vs8r.v v0, (a1) # Unknown-size Folded Spill
@@ -814,40 +833,39 @@ define {<8 x i64>, <8 x i64>, <8 x i64>, <8 x i64>, <8 x i64>, <8 x i64>} @load_
 ; RV64-NEXT:    vmv.s.x v0, a1
 ; RV64-NEXT:    vmv.s.x v2, a3
 ; RV64-NEXT:    csrr a1, vlenb
-; RV64-NEXT:    li a3, 76
+; RV64-NEXT:    li a3, 48
 ; RV64-NEXT:    mul a1, a1, a3
 ; RV64-NEXT:    add a1, sp, a1
 ; RV64-NEXT:    addi a1, a1, 16
-; RV64-NEXT:    vl8r.v v16, (a1) # Unknown-size Folded Reload
-; RV64-NEXT:    vmerge.vvm v24, v8, v16, v0
+; RV64-NEXT:    vl8r.v v24, (a1) # Unknown-size Folded Reload
+; RV64-NEXT:    vmerge.vvm v8, v16, v24, v0
+; RV64-NEXT:    addi a1, sp, 16
+; RV64-NEXT:    vs8r.v v8, (a1) # Unknown-size Folded Spill
 ; RV64-NEXT:    csrr a1, vlenb
-; RV64-NEXT:    slli a1, a1, 3
+; RV64-NEXT:    slli a1, a1, 5
 ; RV64-NEXT:    add a1, sp, a1
 ; RV64-NEXT:    addi a1, a1, 16
-; RV64-NEXT:    vs8r.v v24, (a1) # Unknown-size Folded Spill
-; RV64-NEXT:    addi a1, sp, 16
 ; RV64-NEXT:    vl1r.v v7, (a1) # Unknown-size Folded Reload
 ; RV64-NEXT:    vmv1r.v v0, v7
 ; RV64-NEXT:    csrr a1, vlenb
-; RV64-NEXT:    li a3, 84
-; RV64-NEXT:    mul a1, a1, a3
+; RV64-NEXT:    slli a1, a1, 6
 ; RV64-NEXT:    add a1, sp, a1
 ; RV64-NEXT:    addi a1, a1, 16
-; RV64-NEXT:    vl4r.v v24, (a1) # Unknown-size Folded Reload
+; RV64-NEXT:    vl4r.v v12, (a1) # Unknown-size Folded Reload
 ; RV64-NEXT:    csrr a1, vlenb
-; RV64-NEXT:    li a3, 48
+; RV64-NEXT:    li a3, 40
 ; RV64-NEXT:    mul a1, a1, a3
 ; RV64-NEXT:    add a1, sp, a1
 ; RV64-NEXT:    addi a1, a1, 16
 ; RV64-NEXT:    vl8r.v v16, (a1) # Unknown-size Folded Reload
+; RV64-NEXT:    vmv4r.v v8, v16
 ; RV64-NEXT:    vsetivli zero, 8, e64, m4, ta, mu
-; RV64-NEXT:    vrgather.vi v24, v16, 4, v0.t
+; RV64-NEXT:    vrgather.vi v12, v16, 4, v0.t
 ; RV64-NEXT:    csrr a1, vlenb
-; RV64-NEXT:    li a3, 84
-; RV64-NEXT:    mul a1, a1, a3
+; RV64-NEXT:    slli a1, a1, 6
 ; RV64-NEXT:    add a1, sp, a1
 ; RV64-NEXT:    addi a1, a1, 16
-; RV64-NEXT:    vs4r.v v24, (a1) # Unknown-size Folded Spill
+; RV64-NEXT:    vs4r.v v12, (a1) # Unknown-size Folded Spill
 ; RV64-NEXT:    vmv1r.v v0, v2
 ; RV64-NEXT:    csrr a1, vlenb
 ; RV64-NEXT:    li a3, 76
@@ -856,53 +874,48 @@ define {<8 x i64>, <8 x i64>, <8 x i64>, <8 x i64>, <8 x i64>, <8 x i64>} @load_
 ; RV64-NEXT:    addi a1, a1, 16
 ; RV64-NEXT:    vl8r.v v16, (a1) # Unknown-size Folded Reload
 ; RV64-NEXT:    vsetivli zero, 16, e64, m8, ta, ma
-; RV64-NEXT:    vmerge.vvm v24, v8, v16, v0
+; RV64-NEXT:    vmerge.vvm v16, v16, v24, v0
 ; RV64-NEXT:    csrr a1, vlenb
-; RV64-NEXT:    slli a1, a1, 4
+; RV64-NEXT:    slli a1, a1, 3
 ; RV64-NEXT:    add a1, sp, a1
 ; RV64-NEXT:    addi a1, a1, 16
-; RV64-NEXT:    vs8r.v v24, (a1) # Unknown-size Folded Spill
-; RV64-NEXT:    vmv8r.v v16, v8
+; RV64-NEXT:    vs8r.v v16, (a1) # Unknown-size Folded Spill
 ; RV64-NEXT:    vmv1r.v v0, v7
 ; RV64-NEXT:    csrr a1, vlenb
-; RV64-NEXT:    li a3, 72
-; RV64-NEXT:    mul a1, a1, a3
-; RV64-NEXT:    add a1, sp, a1
-; RV64-NEXT:    addi a1, a1, 16
-; RV64-NEXT:    vl4r.v v12, (a1) # Unknown-size Folded Reload
-; RV64-NEXT:    csrr a1, vlenb
-; RV64-NEXT:    li a3, 48
+; RV64-NEXT:    li a3, 56
 ; RV64-NEXT:    mul a1, a1, a3
 ; RV64-NEXT:    add a1, sp, a1
 ; RV64-NEXT:    addi a1, a1, 16
-; RV64-NEXT:    vl8r.v v24, (a1) # Unknown-size Folded Reload
-; RV64-NEXT:    vmv4r.v v8, v24
+; RV64-NEXT:    vl4r.v v24, (a1) # Unknown-size Folded Reload
 ; RV64-NEXT:    vsetivli zero, 8, e64, m4, ta, mu
-; RV64-NEXT:    vrgather.vi v12, v24, 5, v0.t
+; RV64-NEXT:    vrgather.vi v24, v8, 5, v0.t
 ; RV64-NEXT:    csrr a1, vlenb
-; RV64-NEXT:    li a3, 72
+; RV64-NEXT:    li a3, 56
 ; RV64-NEXT:    mul a1, a1, a3
 ; RV64-NEXT:    add a1, sp, a1
 ; RV64-NEXT:    addi a1, a1, 16
-; RV64-NEXT:    vs4r.v v12, (a1) # Unknown-size Folded Spill
+; RV64-NEXT:    vs4r.v v24, (a1) # Unknown-size Folded Spill
 ; RV64-NEXT:    lui a1, 96
 ; RV64-NEXT:    li a3, 192
 ; RV64-NEXT:    vmv.s.x v3, a3
 ; RV64-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
-; RV64-NEXT:    vmv.v.x v12, a1
+; RV64-NEXT:    vmv.v.x v24, a1
 ; RV64-NEXT:    vmv1r.v v0, v3
 ; RV64-NEXT:    csrr a1, vlenb
-; RV64-NEXT:    slli a1, a1, 6
+; RV64-NEXT:    li a3, 72
+; RV64-NEXT:    mul a1, a1, a3
 ; RV64-NEXT:    add a1, sp, a1
 ; RV64-NEXT:    addi a1, a1, 16
-; RV64-NEXT:    vl4r.v v24, (a1) # Unknown-size Folded Reload
+; RV64-NEXT:    vl4r.v v28, (a1) # Unknown-size Folded Reload
 ; RV64-NEXT:    vsetivli zero, 8, e64, m4, ta, mu
-; RV64-NEXT:    vrgatherei16.vv v24, v8, v12, v0.t
+; RV64-NEXT:    vrgatherei16.vv v28, v8, v24, v0.t
+; RV64-NEXT:    vmv4r.v v16, v8
 ; RV64-NEXT:    csrr a1, vlenb
-; RV64-NEXT:    slli a1, a1, 6
+; RV64-NEXT:    li a3, 72
+; RV64-NEXT:    mul a1, a1, a3
 ; RV64-NEXT:    add a1, sp, a1
 ; RV64-NEXT:    addi a1, a1, 16
-; RV64-NEXT:    vs4r.v v24, (a1) # Unknown-size Folded Spill
+; RV64-NEXT:    vs4r.v v28, (a1) # Unknown-size Folded Spill
 ; RV64-NEXT:    lui a1, %hi(.LCPI8_2)
 ; RV64-NEXT:    addi a1, a1, %lo(.LCPI8_2)
 ; RV64-NEXT:    li a3, 1040
@@ -910,50 +923,67 @@ define {<8 x i64>, <8 x i64>, <8 x i64>, <8 x i64>, <8 x i64>, <8 x i64>} @load_
 ; RV64-NEXT:    addi a4, a4, 1
 ; RV64-NEXT:    vmv.s.x v0, a3
 ; RV64-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
-; RV64-NEXT:    vmv.v.x v12, a4
+; RV64-NEXT:    vmv.v.x v5, a4
 ; RV64-NEXT:    vsetivli zero, 16, e64, m8, ta, ma
 ; RV64-NEXT:    vle16.v v6, (a1)
-; RV64-NEXT:    vmv8r.v v24, v16
 ; RV64-NEXT:    csrr a1, vlenb
 ; RV64-NEXT:    li a3, 76
 ; RV64-NEXT:    mul a1, a1, a3
 ; RV64-NEXT:    add a1, sp, a1
 ; RV64-NEXT:    addi a1, a1, 16
-; RV64-NEXT:    vl8r.v v16, (a1) # Unknown-size Folded Reload
-; RV64-NEXT:    vmerge.vvm v16, v24, v16, v0
-; RV64-NEXT:    addi a1, sp, 16
-; RV64-NEXT:    vs8r.v v16, (a1) # Unknown-size Folded Spill
+; RV64-NEXT:    vl8r.v v8, (a1) # Unknown-size Folded Reload
+; RV64-NEXT:    csrr a1, vlenb
+; RV64-NEXT:    li a3, 48
+; RV64-NEXT:    mul a1, a1, a3
+; RV64-NEXT:    add a1, sp, a1
+; RV64-NEXT:    addi a1, a1, 16
+; RV64-NEXT:    vl8r.v v24, (a1) # Unknown-size Folded Reload
+; RV64-NEXT:    vmerge.vvm v24, v8, v24, v0
+; RV64-NEXT:    csrr a1, vlenb
+; RV64-NEXT:    slli a1, a1, 5
+; RV64-NEXT:    add a1, sp, a1
+; RV64-NEXT:    addi a1, a1, 16
+; RV64-NEXT:    vs8r.v v24, (a1) # Unknown-size Folded Spill
 ; RV64-NEXT:    vmv1r.v v0, v3
 ; RV64-NEXT:    csrr a1, vlenb
-; RV64-NEXT:    li a3, 56
+; RV64-NEXT:    li a3, 68
 ; RV64-NEXT:    mul a1, a1, a3
 ; RV64-NEXT:    add a1, sp, a1
 ; RV64-NEXT:    addi a1, a1, 16
-; RV64-NEXT:    vl4r.v v16, (a1) # Unknown-size Folded Reload
+; RV64-NEXT:    vl4r.v v28, (a1) # Unknown-size Folded Reload
 ; RV64-NEXT:    vsetivli zero, 8, e64, m4, ta, mu
-; RV64-NEXT:    vrgatherei16.vv v16, v8, v12, v0.t
+; RV64-NEXT:    vrgatherei16.vv v28, v16, v5, v0.t
 ; RV64-NEXT:    csrr a1, vlenb
-; RV64-NEXT:    li a3, 56
+; RV64-NEXT:    li a3, 68
 ; RV64-NEXT:    mul a1, a1, a3
 ; RV64-NEXT:    add a1, sp, a1
 ; RV64-NEXT:    addi a1, a1, 16
-; RV64-NEXT:    vs4r.v v16, (a1) # Unknown-size Folded Spill
+; RV64-NEXT:    vs4r.v v28, (a1) # Unknown-size Folded Spill
 ; RV64-NEXT:    addi a1, a2, -2016
 ; RV64-NEXT:    vmv.s.x v0, a1
+; RV64-NEXT:    addi a1, sp, 16
+; RV64-NEXT:    vl8r.v v24, (a1) # Unknown-size Folded Reload
+; RV64-NEXT:    vsetivli zero, 16, e64, m8, ta, ma
+; RV64-NEXT:    vrgatherei16.vv v16, v24, v6
 ; RV64-NEXT:    csrr a1, vlenb
-; RV64-NEXT:    slli a1, a1, 3
+; RV64-NEXT:    li a2, 40
+; RV64-NEXT:    mul a1, a1, a2
+; RV64-NEXT:    add a1, sp, a1
+; RV64-NEXT:    addi a1, a1, 16
+; RV64-NEXT:    vs8r.v v16, (a1) # Unknown-size Folded Spill
+; RV64-NEXT:    csrr a1, vlenb
+; RV64-NEXT:    li a2, 48
+; RV64-NEXT:    mul a1, a1, a2
 ; RV64-NEXT:    add a1, sp, a1
 ; RV64-NEXT:    addi a1, a1, 16
 ; RV64-NEXT:    vl8r.v v8, (a1) # Unknown-size Folded Reload
-; RV64-NEXT:    vsetivli zero, 16, e64, m8, ta, ma
-; RV64-NEXT:    vrgatherei16.vv v16, v8, v6
 ; RV64-NEXT:    csrr a1, vlenb
 ; RV64-NEXT:    li a2, 76
 ; RV64-NEXT:    mul a1, a1, a2
 ; RV64-NEXT:    add a1, sp, a1
 ; RV64-NEXT:    addi a1, a1, 16
-; RV64-NEXT:    vl8r.v v8, (a1) # Unknown-size Folded Reload
-; RV64-NEXT:    vmerge.vvm v8, v24, v8, v0
+; RV64-NEXT:    vl8r.v v16, (a1) # Unknown-size Folded Reload
+; RV64-NEXT:    vmerge.vvm v8, v16, v8, v0
 ; RV64-NEXT:    csrr a1, vlenb
 ; RV64-NEXT:    li a2, 76
 ; RV64-NEXT:    mul a1, a1, a2
@@ -962,80 +992,94 @@ define {<8 x i64>, <8 x i64>, <8 x i64>, <8 x i64>, <8 x i64>, <8 x i64>} @load_
 ; RV64-NEXT:    vs8r.v v8, (a1) # Unknown-size Folded Spill
 ; RV64-NEXT:    lui a1, %hi(.LCPI8_3)
 ; RV64-NEXT:    addi a1, a1, %lo(.LCPI8_3)
-; RV64-NEXT:    vle16.v v24, (a1)
+; RV64-NEXT:    vle16.v v8, (a1)
 ; RV64-NEXT:    csrr a1, vlenb
-; RV64-NEXT:    slli a1, a1, 5
+; RV64-NEXT:    li a2, 60
+; RV64-NEXT:    mul a1, a1, a2
 ; RV64-NEXT:    add a1, sp, a1
 ; RV64-NEXT:    addi a1, a1, 16
-; RV64-NEXT:    vl8r.v v0, (a1) # Unknown-size Folded Reload
+; RV64-NEXT:    vl4r.v v12, (a1) # Unknown-size Folded Reload
 ; RV64-NEXT:    csrr a1, vlenb
-; RV64-NEXT:    li a2, 60
+; RV64-NEXT:    li a2, 24
 ; RV64-NEXT:    mul a1, a1, a2
 ; RV64-NEXT:    add a1, sp, a1
 ; RV64-NEXT:    addi a1, a1, 16
-; RV64-NEXT:    vl4r.v v8, (a1) # Unknown-size Folded Reload
+; RV64-NEXT:    vl8r.v v0, (a1) # Unknown-size Folded Reload
 ; RV64-NEXT:    vsetivli zero, 6, e64, m4, tu, ma
-; RV64-NEXT:    vmv.v.v v8, v0
+; RV64-NEXT:    vmv.v.v v12, v0
 ; RV64-NEXT:    csrr a1, vlenb
 ; RV64-NEXT:    li a2, 60
 ; RV64-NEXT:    mul a1, a1, a2
 ; RV64-NEXT:    add a1, sp, a1
 ; RV64-NEXT:    addi a1, a1, 16
-; RV64-NEXT:    vs4r.v v8, (a1) # Unknown-size Folded Spill
+; RV64-NEXT:    vs4r.v v12, (a1) # Unknown-size Folded Spill
 ; RV64-NEXT:    csrr a1, vlenb
-; RV64-NEXT:    li a2, 68
+; RV64-NEXT:    li a2, 84
 ; RV64-NEXT:    mul a1, a1, a2
 ; RV64-NEXT:    add a1, sp, a1
 ; RV64-NEXT:    addi a1, a1, 16
-; RV64-NEXT:    vl4r.v v0, (a1) # Unknown-size Folded Reload
+; RV64-NEXT:    vl4r.v v12, (a1) # Unknown-size Folded Reload
 ; RV64-NEXT:    csrr a1, vlenb
-; RV64-NEXT:    li a2, 24
-; RV64-NEXT:    mul a1, a1, a2
+; RV64-NEXT:    slli a1, a1, 4
 ; RV64-NEXT:    add a1, sp, a1
 ; RV64-NEXT:    addi a1, a1, 16
-; RV64-NEXT:    vl8r.v v8, (a1) # Unknown-size Folded Reload
-; RV64-NEXT:    vmv.v.v v0, v8
+; RV64-NEXT:    vl8r.v v24, (a1) # Unknown-size Folded Reload
+; RV64-NEXT:    vmv.v.v v12, v24
 ; RV64-NEXT:    csrr a1, vlenb
 ; RV64-NEXT:    li a2, 84
 ; RV64-NEXT:    mul a1, a1, a2
 ; RV64-NEXT:    add a1, sp, a1
 ; RV64-NEXT:    addi a1, a1, 16
-; RV64-NEXT:    vl4r.v v8, (a1) # Unknown-size Folded Reload
-; RV64-NEXT:    vsetivli zero, 5, e64, m4, tu, ma
-; RV64-NEXT:    vmv.v.v v8, v16
+; RV64-NEXT:    vs4r.v v12, (a1) # Unknown-size Folded Spill
 ; RV64-NEXT:    csrr a1, vlenb
-; RV64-NEXT:    li a2, 84
+; RV64-NEXT:    slli a1, a1, 6
+; RV64-NEXT:    add a1, sp, a1
+; RV64-NEXT:    addi a1, a1, 16
+; RV64-NEXT:    vl4r.v v16, (a1) # Unknown-size Folded Reload
+; RV64-NEXT:    csrr a1, vlenb
+; RV64-NEXT:    li a2, 40
 ; RV64-NEXT:    mul a1, a1, a2
 ; RV64-NEXT:    add a1, sp, a1
 ; RV64-NEXT:    addi a1, a1, 16
-; RV64-NEXT:    vs4r.v v8, (a1) # Unknown-size Folded Spill
+; RV64-NEXT:    vl8r.v v24, (a1) # Unknown-size Folded Reload
+; RV64-NEXT:    vsetivli zero, 5, e64, m4, tu, ma
+; RV64-NEXT:    vmv.v.v v16, v24
 ; RV64-NEXT:    csrr a1, vlenb
-; RV64-NEXT:    slli a1, a1, 4
+; RV64-NEXT:    slli a1, a1, 3
 ; RV64-NEXT:    add a1, sp, a1
 ; RV64-NEXT:    addi a1, a1, 16
-; RV64-NEXT:    vl8r.v v8, (a1) # Unknown-size Folded Reload
+; RV64-NEXT:    vl8r.v v24, (a1) # Unknown-size Folded Reload
 ; RV64-NEXT:    vsetivli zero, 16, e64, m8, ta, ma
-; RV64-NEXT:    vrgatherei16.vv v16, v8, v24
+; RV64-NEXT:    vrgatherei16.vv v0, v24, v8
 ; RV64-NEXT:    lui a1, %hi(.LCPI8_4)
 ; RV64-NEXT:    addi a1, a1, %lo(.LCPI8_4)
 ; RV64-NEXT:    vle16.v v8, (a1)
 ; RV64-NEXT:    lui a1, %hi(.LCPI8_5)
 ; RV64-NEXT:    addi a1, a1, %lo(.LCPI8_5)
-; RV64-NEXT:    vle16.v v6, (a1)
+; RV64-NEXT:    vle16.v v10, (a1)
 ; RV64-NEXT:    csrr a1, vlenb
-; RV64-NEXT:    li a2, 72
+; RV64-NEXT:    slli a1, a1, 6
+; RV64-NEXT:    add a1, sp, a1
+; RV64-NEXT:    addi a1, a1, 16
+; RV64-NEXT:    vs2r.v v10, (a1) # Unknown-size Folded Spill
+; RV64-NEXT:    csrr a1, vlenb
+; RV64-NEXT:    li a2, 56
 ; RV64-NEXT:    mul a1, a1, a2
 ; RV64-NEXT:    add a1, sp, a1
 ; RV64-NEXT:    addi a1, a1, 16
 ; RV64-NEXT:    vl4r.v v12, (a1) # Unknown-size Folded Reload
 ; RV64-NEXT:    vsetivli zero, 5, e64, m4, tu, ma
-; RV64-NEXT:    vmv.v.v v12, v16
-; RV64-NEXT:    addi a1, sp, 16
-; RV64-NEXT:    vl8r.v v16, (a1) # Unknown-size Folded Reload
+; RV64-NEXT:    vmv.v.v v12, v0
+; RV64-NEXT:    csrr a1, vlenb
+; RV64-NEXT:    slli a1, a1, 5
+; RV64-NEXT:    add a1, sp, a1
+; RV64-NEXT:    addi a1, a1, 16
+; RV64-NEXT:    vl8r.v v0, (a1) # Unknown-size Folded Reload
 ; RV64-NEXT:    vsetivli zero, 16, e64, m8, ta, ma
-; RV64-NEXT:    vrgatherei16.vv v24, v16, v8
+; RV64-NEXT:    vrgatherei16.vv v24, v0, v8
 ; RV64-NEXT:    csrr a1, vlenb
-; RV64-NEXT:    slli a1, a1, 6
+; RV64-NEXT:    li a2, 72
+; RV64-NEXT:    mul a1, a1, a2
 ; RV64-NEXT:    add a1, sp, a1
 ; RV64-NEXT:    addi a1, a1, 16
 ; RV64-NEXT:    vl4r.v v8, (a1) # Unknown-size Folded Reload
@@ -1046,25 +1090,32 @@ define {<8 x i64>, <8 x i64>, <8 x i64>, <8 x i64>, <8 x i64>, <8 x i64>} @load_
 ; RV64-NEXT:    mul a1, a1, a2
 ; RV64-NEXT:    add a1, sp, a1
 ; RV64-NEXT:    addi a1, a1, 16
-; RV64-NEXT:    vl8r.v v16, (a1) # Unknown-size Folded Reload
+; RV64-NEXT:    vl8r.v v0, (a1) # Unknown-size Folded Reload
+; RV64-NEXT:    csrr a1, vlenb
+; RV64-NEXT:    slli a1, a1, 6
+; RV64-NEXT:    add a1, sp, a1
+; RV64-NEXT:    addi a1, a1, 16
+; RV64-NEXT:    vl2r.v v20, (a1) # Unknown-size Folded Reload
 ; RV64-NEXT:    vsetivli zero, 16, e64, m8, ta, ma
-; RV64-NEXT:    vrgatherei16.vv v24, v16, v6
+; RV64-NEXT:    vrgatherei16.vv v24, v0, v20
 ; RV64-NEXT:    csrr a1, vlenb
-; RV64-NEXT:    li a2, 56
+; RV64-NEXT:    li a2, 68
 ; RV64-NEXT:    mul a1, a1, a2
 ; RV64-NEXT:    add a1, sp, a1
 ; RV64-NEXT:    addi a1, a1, 16
-; RV64-NEXT:    vl4r.v v16, (a1) # Unknown-size Folded Reload
+; RV64-NEXT:    vl4r.v v28, (a1) # Unknown-size Folded Reload
 ; RV64-NEXT:    vsetivli zero, 5, e64, m4, tu, ma
-; RV64-NEXT:    vmv.v.v v16, v24
+; RV64-NEXT:    vmv.v.v v28, v24
 ; RV64-NEXT:    addi a1, a0, 256
 ; RV64-NEXT:    vsetivli zero, 8, e64, m4, ta, ma
 ; RV64-NEXT:    vse64.v v8, (a1)
 ; RV64-NEXT:    addi a1, a0, 320
-; RV64-NEXT:    vse64.v v16, (a1)
+; RV64-NEXT:    vse64.v v28, (a1)
 ; RV64-NEXT:    addi a1, a0, 192
 ; RV64-NEXT:    vse64.v v12, (a1)
 ; RV64-NEXT:    addi a1, a0, 128
+; RV64-NEXT:    vse64.v v16, (a1)
+; RV64-NEXT:    addi a1, a0, 64
 ; RV64-NEXT:    csrr a2, vlenb
 ; RV64-NEXT:    li a3, 84
 ; RV64-NEXT:    mul a2, a2, a3
@@ -1072,8 +1123,6 @@ define {<8 x i64>, <8 x i64>, <8 x i64>, <8 x i64>, <8 x i64>, <8 x i64>} @load_
 ; RV64-NEXT:    addi a2, a2, 16
 ; RV64-NEXT:    vl4r.v v8, (a2) # Unknown-size Folded Reload
 ; RV64-NEXT:    vse64.v v8, (a1)
-; RV64-NEXT:    addi a1, a0, 64
-; RV64-NEXT:    vse64.v v0, (a1)
 ; RV64-NEXT:    csrr a1, vlenb
 ; RV64-NEXT:    li a2, 60
 ; RV64-NEXT:    mul a1, a1, a2
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-masked-gather.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-masked-gather.ll
index c6e12c52122d2..232a364e87f0e 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-masked-gather.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-masked-gather.ll
@@ -2730,21 +2730,21 @@ define <8 x i32> @mgather_baseidx_sext_v8i8_v8i32(ptr %base, <8 x i8> %idxs, <8
 define <8 x i32> @mgather_baseidx_zext_v8i8_v8i32(ptr %base, <8 x i8> %idxs, <8 x i1> %m, <8 x i32> %passthru) {
 ; RV32-LABEL: mgather_baseidx_zext_v8i8_v8i32:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
-; RV32-NEXT:    vzext.vf2 v9, v8
-; RV32-NEXT:    vsll.vi v8, v9, 2
+; RV32-NEXT:    li a1, 4
+; RV32-NEXT:    vsetivli zero, 8, e8, mf2, ta, ma
+; RV32-NEXT:    vwmulu.vx v9, v8, a1
 ; RV32-NEXT:    vsetvli zero, zero, e32, m2, ta, mu
-; RV32-NEXT:    vluxei16.v v10, (a0), v8, v0.t
+; RV32-NEXT:    vluxei16.v v10, (a0), v9, v0.t
 ; RV32-NEXT:    vmv.v.v v8, v10
 ; RV32-NEXT:    ret
 ;
 ; RV64V-LABEL: mgather_baseidx_zext_v8i8_v8i32:
 ; RV64V:       # %bb.0:
-; RV64V-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
-; RV64V-NEXT:    vzext.vf2 v9, v8
-; RV64V-NEXT:    vsll.vi v8, v9, 2
+; RV64V-NEXT:    li a1, 4
+; RV64V-NEXT:    vsetivli zero, 8, e8, mf2, ta, ma
+; RV64V-NEXT:    vwmulu.vx v9, v8, a1
 ; RV64V-NEXT:    vsetvli zero, zero, e32, m2, ta, mu
-; RV64V-NEXT:    vluxei16.v v10, (a0), v8, v0.t
+; RV64V-NEXT:    vluxei16.v v10, (a0), v9, v0.t
 ; RV64V-NEXT:    vmv.v.v v8, v10
 ; RV64V-NEXT:    ret
 ;
@@ -2888,10 +2888,11 @@ define <8 x i32> @mgather_baseidx_zext_v8i8_v8i32(ptr %base, <8 x i8> %idxs, <8
 define <8 x i32> @mgather_baseidx_v8i16_v8i32(ptr %base, <8 x i16> %idxs, <8 x i1> %m, <8 x i32> %passthru) {
 ; RV32-LABEL: mgather_baseidx_v8i16_v8i32:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    vsetivli zero, 8, e32, m2, ta, mu
-; RV32-NEXT:    vsext.vf2 v12, v8
-; RV32-NEXT:    vsll.vi v8, v12, 2
-; RV32-NEXT:    vluxei32.v v10, (a0), v8, v0.t
+; RV32-NEXT:    li a1, 4
+; RV32-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; RV32-NEXT:    vwmulsu.vx v12, v8, a1
+; RV32-NEXT:    vsetvli zero, zero, e32, m2, ta, mu
+; RV32-NEXT:    vluxei32.v v10, (a0), v12, v0.t
 ; RV32-NEXT:    vmv.v.v v8, v10
 ; RV32-NEXT:    ret
 ;
@@ -3037,10 +3038,11 @@ define <8 x i32> @mgather_baseidx_v8i16_v8i32(ptr %base, <8 x i16> %idxs, <8 x i
 define <8 x i32> @mgather_baseidx_sext_v8i16_v8i32(ptr %base, <8 x i16> %idxs, <8 x i1> %m, <8 x i32> %passthru) {
 ; RV32-LABEL: mgather_baseidx_sext_v8i16_v8i32:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    vsetivli zero, 8, e32, m2, ta, mu
-; RV32-NEXT:    vsext.vf2 v12, v8
-; RV32-NEXT:    vsll.vi v8, v12, 2
-; RV32-NEXT:    vluxei32.v v10, (a0), v8, v0.t
+; RV32-NEXT:    li a1, 4
+; RV32-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; RV32-NEXT:    vwmulsu.vx v12, v8, a1
+; RV32-NEXT:    vsetvli zero, zero, e32, m2, ta, mu
+; RV32-NEXT:    vluxei32.v v10, (a0), v12, v0.t
 ; RV32-NEXT:    vmv.v.v v8, v10
 ; RV32-NEXT:    ret
 ;
@@ -3187,19 +3189,21 @@ define <8 x i32> @mgather_baseidx_sext_v8i16_v8i32(ptr %base, <8 x i16> %idxs, <
 define <8 x i32> @mgather_baseidx_zext_v8i16_v8i32(ptr %base, <8 x i16> %idxs, <8 x i1> %m, <8 x i32> %passthru) {
 ; RV32-LABEL: mgather_baseidx_zext_v8i16_v8i32:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    vsetivli zero, 8, e32, m2, ta, mu
-; RV32-NEXT:    vzext.vf2 v12, v8
-; RV32-NEXT:    vsll.vi v8, v12, 2
-; RV32-NEXT:    vluxei32.v v10, (a0), v8, v0.t
+; RV32-NEXT:    li a1, 4
+; RV32-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; RV32-NEXT:    vwmulu.vx v12, v8, a1
+; RV32-NEXT:    vsetvli zero, zero, e32, m2, ta, mu
+; RV32-NEXT:    vluxei32.v v10, (a0), v12, v0.t
 ; RV32-NEXT:    vmv.v.v v8, v10
 ; RV32-NEXT:    ret
 ;
 ; RV64V-LABEL: mgather_baseidx_zext_v8i16_v8i32:
 ; RV64V:       # %bb.0:
-; RV64V-NEXT:    vsetivli zero, 8, e32, m2, ta, mu
-; RV64V-NEXT:    vzext.vf2 v12, v8
-; RV64V-NEXT:    vsll.vi v8, v12, 2
-; RV64V-NEXT:    vluxei32.v v10, (a0), v8, v0.t
+; RV64V-NEXT:    li a1, 4
+; RV64V-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; RV64V-NEXT:    vwmulu.vx v12, v8, a1
+; RV64V-NEXT:    vsetvli zero, zero, e32, m2, ta, mu
+; RV64V-NEXT:    vluxei32.v v10, (a0), v12, v0.t
 ; RV64V-NEXT:    vmv.v.v v8, v10
 ; RV64V-NEXT:    ret
 ;
@@ -3352,10 +3356,9 @@ define <8 x i32> @mgather_baseidx_v8i32(ptr %base, <8 x i32> %idxs, <8 x i1> %m,
 ;
 ; RV64V-LABEL: mgather_baseidx_v8i32:
 ; RV64V:       # %bb.0:
-; RV64V-NEXT:    vsetivli zero, 8, e64, m4, ta, ma
-; RV64V-NEXT:    vsext.vf2 v12, v8
-; RV64V-NEXT:    vsll.vi v12, v12, 2
-; RV64V-NEXT:    vsetvli zero, zero, e32, m2, ta, mu
+; RV64V-NEXT:    li a1, 4
+; RV64V-NEXT:    vsetivli zero, 8, e32, m2, ta, mu
+; RV64V-NEXT:    vwmulsu.vx v12, v8, a1
 ; RV64V-NEXT:    vluxei64.v v10, (a0), v12, v0.t
 ; RV64V-NEXT:    vmv.v.v v8, v10
 ; RV64V-NEXT:    ret
@@ -4636,21 +4639,21 @@ define <8 x i64> @mgather_baseidx_sext_v8i8_v8i64(ptr %base, <8 x i8> %idxs, <8
 define <8 x i64> @mgather_baseidx_zext_v8i8_v8i64(ptr %base, <8 x i8> %idxs, <8 x i1> %m, <8 x i64> %passthru) {
 ; RV32V-LABEL: mgather_baseidx_zext_v8i8_v8i64:
 ; RV32V:       # %bb.0:
-; RV32V-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
-; RV32V-NEXT:    vzext.vf2 v9, v8
-; RV32V-NEXT:    vsll.vi v8, v9, 3
+; RV32V-NEXT:    li a1, 8
+; RV32V-NEXT:    vsetivli zero, 8, e8, mf2, ta, ma
+; RV32V-NEXT:    vwmulu.vx v9, v8, a1
 ; RV32V-NEXT:    vsetvli zero, zero, e64, m4, ta, mu
-; RV32V-NEXT:    vluxei16.v v12, (a0), v8, v0.t
+; RV32V-NEXT:    vluxei16.v v12, (a0), v9, v0.t
 ; RV32V-NEXT:    vmv.v.v v8, v12
 ; RV32V-NEXT:    ret
 ;
 ; RV64V-LABEL: mgather_baseidx_zext_v8i8_v8i64:
 ; RV64V:       # %bb.0:
-; RV64V-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
-; RV64V-NEXT:    vzext.vf2 v9, v8
-; RV64V-NEXT:    vsll.vi v8, v9, 3
+; RV64V-NEXT:    li a1, 8
+; RV64V-NEXT:    vsetivli zero, 8, e8, mf2, ta, ma
+; RV64V-NEXT:    vwmulu.vx v9, v8, a1
 ; RV64V-NEXT:    vsetvli zero, zero, e64, m4, ta, mu
-; RV64V-NEXT:    vluxei16.v v12, (a0), v8, v0.t
+; RV64V-NEXT:    vluxei16.v v12, (a0), v9, v0.t
 ; RV64V-NEXT:    vmv.v.v v8, v12
 ; RV64V-NEXT:    ret
 ;
@@ -4923,11 +4926,11 @@ define <8 x i64> @mgather_baseidx_zext_v8i8_v8i64(ptr %base, <8 x i8> %idxs, <8
 define <8 x i64> @mgather_baseidx_v8i16_v8i64(ptr %base, <8 x i16> %idxs, <8 x i1> %m, <8 x i64> %passthru) {
 ; RV32V-LABEL: mgather_baseidx_v8i16_v8i64:
 ; RV32V:       # %bb.0:
-; RV32V-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
-; RV32V-NEXT:    vsext.vf2 v10, v8
-; RV32V-NEXT:    vsll.vi v8, v10, 3
+; RV32V-NEXT:    li a1, 8
+; RV32V-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; RV32V-NEXT:    vwmulsu.vx v10, v8, a1
 ; RV32V-NEXT:    vsetvli zero, zero, e64, m4, ta, mu
-; RV32V-NEXT:    vluxei32.v v12, (a0), v8, v0.t
+; RV32V-NEXT:    vluxei32.v v12, (a0), v10, v0.t
 ; RV32V-NEXT:    vmv.v.v v8, v12
 ; RV32V-NEXT:    ret
 ;
@@ -4943,16 +4946,17 @@ define <8 x i64> @mgather_baseidx_v8i16_v8i64(ptr %base, <8 x i16> %idxs, <8 x i
 ; RV32ZVE32F-LABEL: mgather_baseidx_v8i16_v8i64:
 ; RV32ZVE32F:       # %bb.0:
 ; RV32ZVE32F-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
-; RV32ZVE32F-NEXT:    vsext.vf2 v10, v8
+; RV32ZVE32F-NEXT:    vmv.v.x v10, a1
+; RV32ZVE32F-NEXT:    li a1, 8
 ; RV32ZVE32F-NEXT:    vsetvli zero, zero, e8, mf2, ta, ma
 ; RV32ZVE32F-NEXT:    vmv.x.s t0, v0
-; RV32ZVE32F-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
-; RV32ZVE32F-NEXT:    vsll.vi v8, v10, 3
 ; RV32ZVE32F-NEXT:    andi a3, t0, 1
-; RV32ZVE32F-NEXT:    vadd.vx v8, v8, a1
+; RV32ZVE32F-NEXT:    vsetvli zero, zero, e16, m1, ta, ma
+; RV32ZVE32F-NEXT:    vwmaccus.vx v10, a1, v8
 ; RV32ZVE32F-NEXT:    beqz a3, .LBB51_7
 ; RV32ZVE32F-NEXT:  # %bb.1: # %cond.load
-; RV32ZVE32F-NEXT:    vmv.x.s a3, v8
+; RV32ZVE32F-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
+; RV32ZVE32F-NEXT:    vmv.x.s a3, v10
 ; RV32ZVE32F-NEXT:    lw a1, 0(a3)
 ; RV32ZVE32F-NEXT:    lw a3, 4(a3)
 ; RV32ZVE32F-NEXT:    andi a4, t0, 2
@@ -4988,40 +4992,40 @@ define <8 x i64> @mgather_baseidx_v8i16_v8i64(ptr %base, <8 x i16> %idxs, <8 x i
 ; RV32ZVE32F-NEXT:    beqz a4, .LBB51_2
 ; RV32ZVE32F-NEXT:  .LBB51_8: # %cond.load1
 ; RV32ZVE32F-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
-; RV32ZVE32F-NEXT:    vslidedown.vi v10, v8, 1
-; RV32ZVE32F-NEXT:    vmv.x.s a5, v10
+; RV32ZVE32F-NEXT:    vslidedown.vi v8, v10, 1
+; RV32ZVE32F-NEXT:    vmv.x.s a5, v8
 ; RV32ZVE32F-NEXT:    lw a4, 0(a5)
 ; RV32ZVE32F-NEXT:    lw a5, 4(a5)
 ; RV32ZVE32F-NEXT:    andi a6, t0, 4
 ; RV32ZVE32F-NEXT:    beqz a6, .LBB51_3
 ; RV32ZVE32F-NEXT:  .LBB51_9: # %cond.load4
 ; RV32ZVE32F-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
-; RV32ZVE32F-NEXT:    vslidedown.vi v10, v8, 2
-; RV32ZVE32F-NEXT:    vmv.x.s a7, v10
+; RV32ZVE32F-NEXT:    vslidedown.vi v8, v10, 2
+; RV32ZVE32F-NEXT:    vmv.x.s a7, v8
 ; RV32ZVE32F-NEXT:    lw a6, 0(a7)
 ; RV32ZVE32F-NEXT:    lw a7, 4(a7)
 ; RV32ZVE32F-NEXT:    andi t1, t0, 8
 ; RV32ZVE32F-NEXT:    beqz t1, .LBB51_4
 ; RV32ZVE32F-NEXT:  .LBB51_10: # %cond.load7
 ; RV32ZVE32F-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
-; RV32ZVE32F-NEXT:    vslidedown.vi v10, v8, 3
-; RV32ZVE32F-NEXT:    vmv.x.s t2, v10
+; RV32ZVE32F-NEXT:    vslidedown.vi v8, v10, 3
+; RV32ZVE32F-NEXT:    vmv.x.s t2, v8
 ; RV32ZVE32F-NEXT:    lw t1, 0(t2)
 ; RV32ZVE32F-NEXT:    lw t2, 4(t2)
 ; RV32ZVE32F-NEXT:    andi t3, t0, 16
 ; RV32ZVE32F-NEXT:    beqz t3, .LBB51_5
 ; RV32ZVE32F-NEXT:  .LBB51_11: # %cond.load10
 ; RV32ZVE32F-NEXT:    vsetivli zero, 1, e32, m2, ta, ma
-; RV32ZVE32F-NEXT:    vslidedown.vi v10, v8, 4
-; RV32ZVE32F-NEXT:    vmv.x.s t4, v10
+; RV32ZVE32F-NEXT:    vslidedown.vi v8, v10, 4
+; RV32ZVE32F-NEXT:    vmv.x.s t4, v8
 ; RV32ZVE32F-NEXT:    lw t3, 0(t4)
 ; RV32ZVE32F-NEXT:    lw t4, 4(t4)
 ; RV32ZVE32F-NEXT:    andi t5, t0, 32
 ; RV32ZVE32F-NEXT:    beqz t5, .LBB51_6
 ; RV32ZVE32F-NEXT:  .LBB51_12: # %cond.load13
 ; RV32ZVE32F-NEXT:    vsetivli zero, 1, e32, m2, ta, ma
-; RV32ZVE32F-NEXT:    vslidedown.vi v10, v8, 5
-; RV32ZVE32F-NEXT:    vmv.x.s t6, v10
+; RV32ZVE32F-NEXT:    vslidedown.vi v8, v10, 5
+; RV32ZVE32F-NEXT:    vmv.x.s t6, v8
 ; RV32ZVE32F-NEXT:    lw t5, 0(t6)
 ; RV32ZVE32F-NEXT:    lw t6, 4(t6)
 ; RV32ZVE32F-NEXT:  .LBB51_13: # %else14
@@ -5035,8 +5039,8 @@ define <8 x i64> @mgather_baseidx_v8i16_v8i64(ptr %base, <8 x i16> %idxs, <8 x i
 ; RV32ZVE32F-NEXT:    beqz s0, .LBB51_16
 ; RV32ZVE32F-NEXT:  # %bb.14: # %cond.load16
 ; RV32ZVE32F-NEXT:    vsetivli zero, 1, e32, m2, ta, ma
-; RV32ZVE32F-NEXT:    vslidedown.vi v10, v8, 6
-; RV32ZVE32F-NEXT:    vmv.x.s s1, v10
+; RV32ZVE32F-NEXT:    vslidedown.vi v8, v10, 6
+; RV32ZVE32F-NEXT:    vmv.x.s s1, v8
 ; RV32ZVE32F-NEXT:    lw s0, 0(s1)
 ; RV32ZVE32F-NEXT:    lw s1, 4(s1)
 ; RV32ZVE32F-NEXT:    andi t0, t0, -128
@@ -5052,7 +5056,7 @@ define <8 x i64> @mgather_baseidx_v8i16_v8i64(ptr %base, <8 x i16> %idxs, <8 x i
 ; RV32ZVE32F-NEXT:    beqz t0, .LBB51_15
 ; RV32ZVE32F-NEXT:  .LBB51_17: # %cond.load19
 ; RV32ZVE32F-NEXT:    vsetivli zero, 1, e32, m2, ta, ma
-; RV32ZVE32F-NEXT:    vslidedown.vi v8, v8, 7
+; RV32ZVE32F-NEXT:    vslidedown.vi v8, v10, 7
 ; RV32ZVE32F-NEXT:    vmv.x.s a2, v8
 ; RV32ZVE32F-NEXT:    lw t0, 0(a2)
 ; RV32ZVE32F-NEXT:    lw a2, 4(a2)
@@ -5201,11 +5205,11 @@ define <8 x i64> @mgather_baseidx_v8i16_v8i64(ptr %base, <8 x i16> %idxs, <8 x i
 define <8 x i64> @mgather_baseidx_sext_v8i16_v8i64(ptr %base, <8 x i16> %idxs, <8 x i1> %m, <8 x i64> %passthru) {
 ; RV32V-LABEL: mgather_baseidx_sext_v8i16_v8i64:
 ; RV32V:       # %bb.0:
-; RV32V-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
-; RV32V-NEXT:    vsext.vf2 v10, v8
-; RV32V-NEXT:    vsll.vi v8, v10, 3
+; RV32V-NEXT:    li a1, 8
+; RV32V-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; RV32V-NEXT:    vwmulsu.vx v10, v8, a1
 ; RV32V-NEXT:    vsetvli zero, zero, e64, m4, ta, mu
-; RV32V-NEXT:    vluxei32.v v12, (a0), v8, v0.t
+; RV32V-NEXT:    vluxei32.v v12, (a0), v10, v0.t
 ; RV32V-NEXT:    vmv.v.v v8, v12
 ; RV32V-NEXT:    ret
 ;
@@ -5221,16 +5225,17 @@ define <8 x i64> @mgather_baseidx_sext_v8i16_v8i64(ptr %base, <8 x i16> %idxs, <
 ; RV32ZVE32F-LABEL: mgather_baseidx_sext_v8i16_v8i64:
 ; RV32ZVE32F:       # %bb.0:
 ; RV32ZVE32F-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
-; RV32ZVE32F-NEXT:    vsext.vf2 v10, v8
+; RV32ZVE32F-NEXT:    vmv.v.x v10, a1
+; RV32ZVE32F-NEXT:    li a1, 8
 ; RV32ZVE32F-NEXT:    vsetvli zero, zero, e8, mf2, ta, ma
 ; RV32ZVE32F-NEXT:    vmv.x.s t0, v0
-; RV32ZVE32F-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
-; RV32ZVE32F-NEXT:    vsll.vi v8, v10, 3
 ; RV32ZVE32F-NEXT:    andi a3, t0, 1
-; RV32ZVE32F-NEXT:    vadd.vx v8, v8, a1
+; RV32ZVE32F-NEXT:    vsetvli zero, zero, e16, m1, ta, ma
+; RV32ZVE32F-NEXT:    vwmaccus.vx v10, a1, v8
 ; RV32ZVE32F-NEXT:    beqz a3, .LBB52_7
 ; RV32ZVE32F-NEXT:  # %bb.1: # %cond.load
-; RV32ZVE32F-NEXT:    vmv.x.s a3, v8
+; RV32ZVE32F-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
+; RV32ZVE32F-NEXT:    vmv.x.s a3, v10
 ; RV32ZVE32F-NEXT:    lw a1, 0(a3)
 ; RV32ZVE32F-NEXT:    lw a3, 4(a3)
 ; RV32ZVE32F-NEXT:    andi a4, t0, 2
@@ -5266,40 +5271,40 @@ define <8 x i64> @mgather_baseidx_sext_v8i16_v8i64(ptr %base, <8 x i16> %idxs, <
 ; RV32ZVE32F-NEXT:    beqz a4, .LBB52_2
 ; RV32ZVE32F-NEXT:  .LBB52_8: # %cond.load1
 ; RV32ZVE32F-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
-; RV32ZVE32F-NEXT:    vslidedown.vi v10, v8, 1
-; RV32ZVE32F-NEXT:    vmv.x.s a5, v10
+; RV32ZVE32F-NEXT:    vslidedown.vi v8, v10, 1
+; RV32ZVE32F-NEXT:    vmv.x.s a5, v8
 ; RV32ZVE32F-NEXT:    lw a4, 0(a5)
 ; RV32ZVE32F-NEXT:    lw a5, 4(a5)
 ; RV32ZVE32F-NEXT:    andi a6, t0, 4
 ; RV32ZVE32F-NEXT:    beqz a6, .LBB52_3
 ; RV32ZVE32F-NEXT:  .LBB52_9: # %cond.load4
 ; RV32ZVE32F-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
-; RV32ZVE32F-NEXT:    vslidedown.vi v10, v8, 2
-; RV32ZVE32F-NEXT:    vmv.x.s a7, v10
+; RV32ZVE32F-NEXT:    vslidedown.vi v8, v10, 2
+; RV32ZVE32F-NEXT:    vmv.x.s a7, v8
 ; RV32ZVE32F-NEXT:    lw a6, 0(a7)
 ; RV32ZVE32F-NEXT:    lw a7, 4(a7)
 ; RV32ZVE32F-NEXT:    andi t1, t0, 8
 ; RV32ZVE32F-NEXT:    beqz t1, .LBB52_4
 ; RV32ZVE32F-NEXT:  .LBB52_10: # %cond.load7
 ; RV32ZVE32F-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
-; RV32ZVE32F-NEXT:    vslidedown.vi v10, v8, 3
-; RV32ZVE32F-NEXT:    vmv.x.s t2, v10
+; RV32ZVE32F-NEXT:    vslidedown.vi v8, v10, 3
+; RV32ZVE32F-NEXT:    vmv.x.s t2, v8
 ; RV32ZVE32F-NEXT:    lw t1, 0(t2)
 ; RV32ZVE32F-NEXT:    lw t2, 4(t2)
 ; RV32ZVE32F-NEXT:    andi t3, t0, 16
 ; RV32ZVE32F-NEXT:    beqz t3, .LBB52_5
 ; RV32ZVE32F-NEXT:  .LBB52_11: # %cond.load10
 ; RV32ZVE32F-NEXT:    vsetivli zero, 1, e32, m2, ta, ma
-; RV32ZVE32F-NEXT:    vslidedown.vi v10, v8, 4
-; RV32ZVE32F-NEXT:    vmv.x.s t4, v10
+; RV32ZVE32F-NEXT:    vslidedown.vi v8, v10, 4
+; RV32ZVE32F-NEXT:    vmv.x.s t4, v8
 ; RV32ZVE32F-NEXT:    lw t3, 0(t4)
 ; RV32ZVE32F-NEXT:    lw t4, 4(t4)
 ; RV32ZVE32F-NEXT:    andi t5, t0, 32
 ; RV32ZVE32F-NEXT:    beqz t5, .LBB52_6
 ; RV32ZVE32F-NEXT:  .LBB52_12: # %cond.load13
 ; RV32ZVE32F-NEXT:    vsetivli zero, 1, e32, m2, ta, ma
-; RV32ZVE32F-NEXT:    vslidedown.vi v10, v8, 5
-; RV32ZVE32F-NEXT:    vmv.x.s t6, v10
+; RV32ZVE32F-NEXT:    vslidedown.vi v8, v10, 5
+; RV32ZVE32F-NEXT:    vmv.x.s t6, v8
 ; RV32ZVE32F-NEXT:    lw t5, 0(t6)
 ; RV32ZVE32F-NEXT:    lw t6, 4(t6)
 ; RV32ZVE32F-NEXT:  .LBB52_13: # %else14
@@ -5313,8 +5318,8 @@ define <8 x i64> @mgather_baseidx_sext_v8i16_v8i64(ptr %base, <8 x i16> %idxs, <
 ; RV32ZVE32F-NEXT:    beqz s0, .LBB52_16
 ; RV32ZVE32F-NEXT:  # %bb.14: # %cond.load16
 ; RV32ZVE32F-NEXT:    vsetivli zero, 1, e32, m2, ta, ma
-; RV32ZVE32F-NEXT:    vslidedown.vi v10, v8, 6
-; RV32ZVE32F-NEXT:    vmv.x.s s1, v10
+; RV32ZVE32F-NEXT:    vslidedown.vi v8, v10, 6
+; RV32ZVE32F-NEXT:    vmv.x.s s1, v8
 ; RV32ZVE32F-NEXT:    lw s0, 0(s1)
 ; RV32ZVE32F-NEXT:    lw s1, 4(s1)
 ; RV32ZVE32F-NEXT:    andi t0, t0, -128
@@ -5330,7 +5335,7 @@ define <8 x i64> @mgather_baseidx_sext_v8i16_v8i64(ptr %base, <8 x i16> %idxs, <
 ; RV32ZVE32F-NEXT:    beqz t0, .LBB52_15
 ; RV32ZVE32F-NEXT:  .LBB52_17: # %cond.load19
 ; RV32ZVE32F-NEXT:    vsetivli zero, 1, e32, m2, ta, ma
-; RV32ZVE32F-NEXT:    vslidedown.vi v8, v8, 7
+; RV32ZVE32F-NEXT:    vslidedown.vi v8, v10, 7
 ; RV32ZVE32F-NEXT:    vmv.x.s a2, v8
 ; RV32ZVE32F-NEXT:    lw t0, 0(a2)
 ; RV32ZVE32F-NEXT:    lw a2, 4(a2)
@@ -5480,37 +5485,38 @@ define <8 x i64> @mgather_baseidx_sext_v8i16_v8i64(ptr %base, <8 x i16> %idxs, <
 define <8 x i64> @mgather_baseidx_zext_v8i16_v8i64(ptr %base, <8 x i16> %idxs, <8 x i1> %m, <8 x i64> %passthru) {
 ; RV32V-LABEL: mgather_baseidx_zext_v8i16_v8i64:
 ; RV32V:       # %bb.0:
-; RV32V-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
-; RV32V-NEXT:    vzext.vf2 v10, v8
-; RV32V-NEXT:    vsll.vi v8, v10, 3
+; RV32V-NEXT:    li a1, 8
+; RV32V-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; RV32V-NEXT:    vwmulu.vx v10, v8, a1
 ; RV32V-NEXT:    vsetvli zero, zero, e64, m4, ta, mu
-; RV32V-NEXT:    vluxei32.v v12, (a0), v8, v0.t
+; RV32V-NEXT:    vluxei32.v v12, (a0), v10, v0.t
 ; RV32V-NEXT:    vmv.v.v v8, v12
 ; RV32V-NEXT:    ret
 ;
 ; RV64V-LABEL: mgather_baseidx_zext_v8i16_v8i64:
 ; RV64V:       # %bb.0:
-; RV64V-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
-; RV64V-NEXT:    vzext.vf2 v10, v8
-; RV64V-NEXT:    vsll.vi v8, v10, 3
+; RV64V-NEXT:    li a1, 8
+; RV64V-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; RV64V-NEXT:    vwmulu.vx v10, v8, a1
 ; RV64V-NEXT:    vsetvli zero, zero, e64, m4, ta, mu
-; RV64V-NEXT:    vluxei32.v v12, (a0), v8, v0.t
+; RV64V-NEXT:    vluxei32.v v12, (a0), v10, v0.t
 ; RV64V-NEXT:    vmv.v.v v8, v12
 ; RV64V-NEXT:    ret
 ;
 ; RV32ZVE32F-LABEL: mgather_baseidx_zext_v8i16_v8i64:
 ; RV32ZVE32F:       # %bb.0:
 ; RV32ZVE32F-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
-; RV32ZVE32F-NEXT:    vzext.vf2 v10, v8
+; RV32ZVE32F-NEXT:    vmv.v.x v10, a1
+; RV32ZVE32F-NEXT:    li a1, 8
 ; RV32ZVE32F-NEXT:    vsetvli zero, zero, e8, mf2, ta, ma
 ; RV32ZVE32F-NEXT:    vmv.x.s t0, v0
-; RV32ZVE32F-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
-; RV32ZVE32F-NEXT:    vsll.vi v8, v10, 3
 ; RV32ZVE32F-NEXT:    andi a3, t0, 1
-; RV32ZVE32F-NEXT:    vadd.vx v8, v8, a1
+; RV32ZVE32F-NEXT:    vsetvli zero, zero, e16, m1, ta, ma
+; RV32ZVE32F-NEXT:    vwmaccu.vx v10, a1, v8
 ; RV32ZVE32F-NEXT:    beqz a3, .LBB53_7
 ; RV32ZVE32F-NEXT:  # %bb.1: # %cond.load
-; RV32ZVE32F-NEXT:    vmv.x.s a3, v8
+; RV32ZVE32F-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
+; RV32ZVE32F-NEXT:    vmv.x.s a3, v10
 ; RV32ZVE32F-NEXT:    lw a1, 0(a3)
 ; RV32ZVE32F-NEXT:    lw a3, 4(a3)
 ; RV32ZVE32F-NEXT:    andi a4, t0, 2
@@ -5546,40 +5552,40 @@ define <8 x i64> @mgather_baseidx_zext_v8i16_v8i64(ptr %base, <8 x i16> %idxs, <
 ; RV32ZVE32F-NEXT:    beqz a4, .LBB53_2
 ; RV32ZVE32F-NEXT:  .LBB53_8: # %cond.load1
 ; RV32ZVE32F-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
-; RV32ZVE32F-NEXT:    vslidedown.vi v10, v8, 1
-; RV32ZVE32F-NEXT:    vmv.x.s a5, v10
+; RV32ZVE32F-NEXT:    vslidedown.vi v8, v10, 1
+; RV32ZVE32F-NEXT:    vmv.x.s a5, v8
 ; RV32ZVE32F-NEXT:    lw a4, 0(a5)
 ; RV32ZVE32F-NEXT:    lw a5, 4(a5)
 ; RV32ZVE32F-NEXT:    andi a6, t0, 4
 ; RV32ZVE32F-NEXT:    beqz a6, .LBB53_3
 ; RV32ZVE32F-NEXT:  .LBB53_9: # %cond.load4
 ; RV32ZVE32F-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
-; RV32ZVE32F-NEXT:    vslidedown.vi v10, v8, 2
-; RV32ZVE32F-NEXT:    vmv.x.s a7, v10
+; RV32ZVE32F-NEXT:    vslidedown.vi v8, v10, 2
+; RV32ZVE32F-NEXT:    vmv.x.s a7, v8
 ; RV32ZVE32F-NEXT:    lw a6, 0(a7)
 ; RV32ZVE32F-NEXT:    lw a7, 4(a7)
 ; RV32ZVE32F-NEXT:    andi t1, t0, 8
 ; RV32ZVE32F-NEXT:    beqz t1, .LBB53_4
 ; RV32ZVE32F-NEXT:  .LBB53_10: # %cond.load7
 ; RV32ZVE32F-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
-; RV32ZVE32F-NEXT:    vslidedown.vi v10, v8, 3
-; RV32ZVE32F-NEXT:    vmv.x.s t2, v10
+; RV32ZVE32F-NEXT:    vslidedown.vi v8, v10, 3
+; RV32ZVE32F-NEXT:    vmv.x.s t2, v8
 ; RV32ZVE32F-NEXT:    lw t1, 0(t2)
 ; RV32ZVE32F-NEXT:    lw t2, 4(t2)
 ; RV32ZVE32F-NEXT:    andi t3, t0, 16
 ; RV32ZVE32F-NEXT:    beqz t3, .LBB53_5
 ; RV32ZVE32F-NEXT:  .LBB53_11: # %cond.load10
 ; RV32ZVE32F-NEXT:    vsetivli zero, 1, e32, m2, ta, ma
-; RV32ZVE32F-NEXT:    vslidedown.vi v10, v8, 4
-; RV32ZVE32F-NEXT:    vmv.x.s t4, v10
+; RV32ZVE32F-NEXT:    vslidedown.vi v8, v10, 4
+; RV32ZVE32F-NEXT:    vmv.x.s t4, v8
 ; RV32ZVE32F-NEXT:    lw t3, 0(t4)
 ; RV32ZVE32F-NEXT:    lw t4, 4(t4)
 ; RV32ZVE32F-NEXT:    andi t5, t0, 32
 ; RV32ZVE32F-NEXT:    beqz t5, .LBB53_6
 ; RV32ZVE32F-NEXT:  .LBB53_12: # %cond.load13
 ; RV32ZVE32F-NEXT:    vsetivli zero, 1, e32, m2, ta, ma
-; RV32ZVE32F-NEXT:    vslidedown.vi v10, v8, 5
-; RV32ZVE32F-NEXT:    vmv.x.s t6, v10
+; RV32ZVE32F-NEXT:    vslidedown.vi v8, v10, 5
+; RV32ZVE32F-NEXT:    vmv.x.s t6, v8
 ; RV32ZVE32F-NEXT:    lw t5, 0(t6)
 ; RV32ZVE32F-NEXT:    lw t6, 4(t6)
 ; RV32ZVE32F-NEXT:  .LBB53_13: # %else14
@@ -5593,8 +5599,8 @@ define <8 x i64> @mgather_baseidx_zext_v8i16_v8i64(ptr %base, <8 x i16> %idxs, <
 ; RV32ZVE32F-NEXT:    beqz s0, .LBB53_16
 ; RV32ZVE32F-NEXT:  # %bb.14: # %cond.load16
 ; RV32ZVE32F-NEXT:    vsetivli zero, 1, e32, m2, ta, ma
-; RV32ZVE32F-NEXT:    vslidedown.vi v10, v8, 6
-; RV32ZVE32F-NEXT:    vmv.x.s s1, v10
+; RV32ZVE32F-NEXT:    vslidedown.vi v8, v10, 6
+; RV32ZVE32F-NEXT:    vmv.x.s s1, v8
 ; RV32ZVE32F-NEXT:    lw s0, 0(s1)
 ; RV32ZVE32F-NEXT:    lw s1, 4(s1)
 ; RV32ZVE32F-NEXT:    andi t0, t0, -128
@@ -5610,7 +5616,7 @@ define <8 x i64> @mgather_baseidx_zext_v8i16_v8i64(ptr %base, <8 x i16> %idxs, <
 ; RV32ZVE32F-NEXT:    beqz t0, .LBB53_15
 ; RV32ZVE32F-NEXT:  .LBB53_17: # %cond.load19
 ; RV32ZVE32F-NEXT:    vsetivli zero, 1, e32, m2, ta, ma
-; RV32ZVE32F-NEXT:    vslidedown.vi v8, v8, 7
+; RV32ZVE32F-NEXT:    vslidedown.vi v8, v10, 7
 ; RV32ZVE32F-NEXT:    vmv.x.s a2, v8
 ; RV32ZVE32F-NEXT:    lw t0, 0(a2)
 ; RV32ZVE32F-NEXT:    lw a2, 4(a2)
@@ -5777,10 +5783,11 @@ define <8 x i64> @mgather_baseidx_v8i32_v8i64(ptr %base, <8 x i32> %idxs, <8 x i
 ;
 ; RV64V-LABEL: mgather_baseidx_v8i32_v8i64:
 ; RV64V:       # %bb.0:
-; RV64V-NEXT:    vsetivli zero, 8, e64, m4, ta, mu
-; RV64V-NEXT:    vsext.vf2 v16, v8
-; RV64V-NEXT:    vsll.vi v8, v16, 3
-; RV64V-NEXT:    vluxei64.v v12, (a0), v8, v0.t
+; RV64V-NEXT:    li a1, 8
+; RV64V-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
+; RV64V-NEXT:    vwmulsu.vx v16, v8, a1
+; RV64V-NEXT:    vsetvli zero, zero, e64, m4, ta, mu
+; RV64V-NEXT:    vluxei64.v v12, (a0), v16, v0.t
 ; RV64V-NEXT:    vmv.v.v v8, v12
 ; RV64V-NEXT:    ret
 ;
@@ -6053,10 +6060,11 @@ define <8 x i64> @mgather_baseidx_sext_v8i32_v8i64(ptr %base, <8 x i32> %idxs, <
 ;
 ; RV64V-LABEL: mgather_baseidx_sext_v8i32_v8i64:
 ; RV64V:       # %bb.0:
-; RV64V-NEXT:    vsetivli zero, 8, e64, m4, ta, mu
-; RV64V-NEXT:    vsext.vf2 v16, v8
-; RV64V-NEXT:    vsll.vi v8, v16, 3
-; RV64V-NEXT:    vluxei64.v v12, (a0), v8, v0.t
+; RV64V-NEXT:    li a1, 8
+; RV64V-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
+; RV64V-NEXT:    vwmulsu.vx v16, v8, a1
+; RV64V-NEXT:    vsetvli zero, zero, e64, m4, ta, mu
+; RV64V-NEXT:    vluxei64.v v12, (a0), v16, v0.t
 ; RV64V-NEXT:    vmv.v.v v8, v12
 ; RV64V-NEXT:    ret
 ;
@@ -6330,10 +6338,11 @@ define <8 x i64> @mgather_baseidx_zext_v8i32_v8i64(ptr %base, <8 x i32> %idxs, <
 ;
 ; RV64V-LABEL: mgather_baseidx_zext_v8i32_v8i64:
 ; RV64V:       # %bb.0:
-; RV64V-NEXT:    vsetivli zero, 8, e64, m4, ta, mu
-; RV64V-NEXT:    vzext.vf2 v16, v8
-; RV64V-NEXT:    vsll.vi v8, v16, 3
-; RV64V-NEXT:    vluxei64.v v12, (a0), v8, v0.t
+; RV64V-NEXT:    li a1, 8
+; RV64V-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
+; RV64V-NEXT:    vwmulu.vx v16, v8, a1
+; RV64V-NEXT:    vsetvli zero, zero, e64, m4, ta, mu
+; RV64V-NEXT:    vluxei64.v v12, (a0), v16, v0.t
 ; RV64V-NEXT:    vmv.v.v v8, v12
 ; RV64V-NEXT:    ret
 ;
@@ -10032,21 +10041,21 @@ define <8 x float> @mgather_baseidx_sext_v8i8_v8f32(ptr %base, <8 x i8> %idxs, <
 define <8 x float> @mgather_baseidx_zext_v8i8_v8f32(ptr %base, <8 x i8> %idxs, <8 x i1> %m, <8 x float> %passthru) {
 ; RV32-LABEL: mgather_baseidx_zext_v8i8_v8f32:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
-; RV32-NEXT:    vzext.vf2 v9, v8
-; RV32-NEXT:    vsll.vi v8, v9, 2
+; RV32-NEXT:    li a1, 4
+; RV32-NEXT:    vsetivli zero, 8, e8, mf2, ta, ma
+; RV32-NEXT:    vwmulu.vx v9, v8, a1
 ; RV32-NEXT:    vsetvli zero, zero, e32, m2, ta, mu
-; RV32-NEXT:    vluxei16.v v10, (a0), v8, v0.t
+; RV32-NEXT:    vluxei16.v v10, (a0), v9, v0.t
 ; RV32-NEXT:    vmv.v.v v8, v10
 ; RV32-NEXT:    ret
 ;
 ; RV64V-LABEL: mgather_baseidx_zext_v8i8_v8f32:
 ; RV64V:       # %bb.0:
-; RV64V-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
-; RV64V-NEXT:    vzext.vf2 v9, v8
-; RV64V-NEXT:    vsll.vi v8, v9, 2
+; RV64V-NEXT:    li a1, 4
+; RV64V-NEXT:    vsetivli zero, 8, e8, mf2, ta, ma
+; RV64V-NEXT:    vwmulu.vx v9, v8, a1
 ; RV64V-NEXT:    vsetvli zero, zero, e32, m2, ta, mu
-; RV64V-NEXT:    vluxei16.v v10, (a0), v8, v0.t
+; RV64V-NEXT:    vluxei16.v v10, (a0), v9, v0.t
 ; RV64V-NEXT:    vmv.v.v v8, v10
 ; RV64V-NEXT:    ret
 ;
@@ -10190,10 +10199,11 @@ define <8 x float> @mgather_baseidx_zext_v8i8_v8f32(ptr %base, <8 x i8> %idxs, <
 define <8 x float> @mgather_baseidx_v8i16_v8f32(ptr %base, <8 x i16> %idxs, <8 x i1> %m, <8 x float> %passthru) {
 ; RV32-LABEL: mgather_baseidx_v8i16_v8f32:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    vsetivli zero, 8, e32, m2, ta, mu
-; RV32-NEXT:    vsext.vf2 v12, v8
-; RV32-NEXT:    vsll.vi v8, v12, 2
-; RV32-NEXT:    vluxei32.v v10, (a0), v8, v0.t
+; RV32-NEXT:    li a1, 4
+; RV32-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; RV32-NEXT:    vwmulsu.vx v12, v8, a1
+; RV32-NEXT:    vsetvli zero, zero, e32, m2, ta, mu
+; RV32-NEXT:    vluxei32.v v10, (a0), v12, v0.t
 ; RV32-NEXT:    vmv.v.v v8, v10
 ; RV32-NEXT:    ret
 ;
@@ -10339,10 +10349,11 @@ define <8 x float> @mgather_baseidx_v8i16_v8f32(ptr %base, <8 x i16> %idxs, <8 x
 define <8 x float> @mgather_baseidx_sext_v8i16_v8f32(ptr %base, <8 x i16> %idxs, <8 x i1> %m, <8 x float> %passthru) {
 ; RV32-LABEL: mgather_baseidx_sext_v8i16_v8f32:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    vsetivli zero, 8, e32, m2, ta, mu
-; RV32-NEXT:    vsext.vf2 v12, v8
-; RV32-NEXT:    vsll.vi v8, v12, 2
-; RV32-NEXT:    vluxei32.v v10, (a0), v8, v0.t
+; RV32-NEXT:    li a1, 4
+; RV32-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; RV32-NEXT:    vwmulsu.vx v12, v8, a1
+; RV32-NEXT:    vsetvli zero, zero, e32, m2, ta, mu
+; RV32-NEXT:    vluxei32.v v10, (a0), v12, v0.t
 ; RV32-NEXT:    vmv.v.v v8, v10
 ; RV32-NEXT:    ret
 ;
@@ -10489,19 +10500,21 @@ define <8 x float> @mgather_baseidx_sext_v8i16_v8f32(ptr %base, <8 x i16> %idxs,
 define <8 x float> @mgather_baseidx_zext_v8i16_v8f32(ptr %base, <8 x i16> %idxs, <8 x i1> %m, <8 x float> %passthru) {
 ; RV32-LABEL: mgather_baseidx_zext_v8i16_v8f32:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    vsetivli zero, 8, e32, m2, ta, mu
-; RV32-NEXT:    vzext.vf2 v12, v8
-; RV32-NEXT:    vsll.vi v8, v12, 2
-; RV32-NEXT:    vluxei32.v v10, (a0), v8, v0.t
+; RV32-NEXT:    li a1, 4
+; RV32-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; RV32-NEXT:    vwmulu.vx v12, v8, a1
+; RV32-NEXT:    vsetvli zero, zero, e32, m2, ta, mu
+; RV32-NEXT:    vluxei32.v v10, (a0), v12, v0.t
 ; RV32-NEXT:    vmv.v.v v8, v10
 ; RV32-NEXT:    ret
 ;
 ; RV64V-LABEL: mgather_baseidx_zext_v8i16_v8f32:
 ; RV64V:       # %bb.0:
-; RV64V-NEXT:    vsetivli zero, 8, e32, m2, ta, mu
-; RV64V-NEXT:    vzext.vf2 v12, v8
-; RV64V-NEXT:    vsll.vi v8, v12, 2
-; RV64V-NEXT:    vluxei32.v v10, (a0), v8, v0.t
+; RV64V-NEXT:    li a1, 4
+; RV64V-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; RV64V-NEXT:    vwmulu.vx v12, v8, a1
+; RV64V-NEXT:    vsetvli zero, zero, e32, m2, ta, mu
+; RV64V-NEXT:    vluxei32.v v10, (a0), v12, v0.t
 ; RV64V-NEXT:    vmv.v.v v8, v10
 ; RV64V-NEXT:    ret
 ;
@@ -10654,10 +10667,9 @@ define <8 x float> @mgather_baseidx_v8f32(ptr %base, <8 x i32> %idxs, <8 x i1> %
 ;
 ; RV64V-LABEL: mgather_baseidx_v8f32:
 ; RV64V:       # %bb.0:
-; RV64V-NEXT:    vsetivli zero, 8, e64, m4, ta, ma
-; RV64V-NEXT:    vsext.vf2 v12, v8
-; RV64V-NEXT:    vsll.vi v12, v12, 2
-; RV64V-NEXT:    vsetvli zero, zero, e32, m2, ta, mu
+; RV64V-NEXT:    li a1, 4
+; RV64V-NEXT:    vsetivli zero, 8, e32, m2, ta, mu
+; RV64V-NEXT:    vwmulsu.vx v12, v8, a1
 ; RV64V-NEXT:    vluxei64.v v10, (a0), v12, v0.t
 ; RV64V-NEXT:    vmv.v.v v8, v10
 ; RV64V-NEXT:    ret
@@ -11702,21 +11714,21 @@ define <8 x double> @mgather_baseidx_sext_v8i8_v8f64(ptr %base, <8 x i8> %idxs,
 define <8 x double> @mgather_baseidx_zext_v8i8_v8f64(ptr %base, <8 x i8> %idxs, <8 x i1> %m, <8 x double> %passthru) {
 ; RV32V-LABEL: mgather_baseidx_zext_v8i8_v8f64:
 ; RV32V:       # %bb.0:
-; RV32V-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
-; RV32V-NEXT:    vzext.vf2 v9, v8
-; RV32V-NEXT:    vsll.vi v8, v9, 3
+; RV32V-NEXT:    li a1, 8
+; RV32V-NEXT:    vsetivli zero, 8, e8, mf2, ta, ma
+; RV32V-NEXT:    vwmulu.vx v9, v8, a1
 ; RV32V-NEXT:    vsetvli zero, zero, e64, m4, ta, mu
-; RV32V-NEXT:    vluxei16.v v12, (a0), v8, v0.t
+; RV32V-NEXT:    vluxei16.v v12, (a0), v9, v0.t
 ; RV32V-NEXT:    vmv.v.v v8, v12
 ; RV32V-NEXT:    ret
 ;
 ; RV64V-LABEL: mgather_baseidx_zext_v8i8_v8f64:
 ; RV64V:       # %bb.0:
-; RV64V-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
-; RV64V-NEXT:    vzext.vf2 v9, v8
-; RV64V-NEXT:    vsll.vi v8, v9, 3
+; RV64V-NEXT:    li a1, 8
+; RV64V-NEXT:    vsetivli zero, 8, e8, mf2, ta, ma
+; RV64V-NEXT:    vwmulu.vx v9, v8, a1
 ; RV64V-NEXT:    vsetvli zero, zero, e64, m4, ta, mu
-; RV64V-NEXT:    vluxei16.v v12, (a0), v8, v0.t
+; RV64V-NEXT:    vluxei16.v v12, (a0), v9, v0.t
 ; RV64V-NEXT:    vmv.v.v v8, v12
 ; RV64V-NEXT:    ret
 ;
@@ -11927,11 +11939,11 @@ define <8 x double> @mgather_baseidx_zext_v8i8_v8f64(ptr %base, <8 x i8> %idxs,
 define <8 x double> @mgather_baseidx_v8i16_v8f64(ptr %base, <8 x i16> %idxs, <8 x i1> %m, <8 x double> %passthru) {
 ; RV32V-LABEL: mgather_baseidx_v8i16_v8f64:
 ; RV32V:       # %bb.0:
-; RV32V-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
-; RV32V-NEXT:    vsext.vf2 v10, v8
-; RV32V-NEXT:    vsll.vi v8, v10, 3
+; RV32V-NEXT:    li a1, 8
+; RV32V-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; RV32V-NEXT:    vwmulsu.vx v10, v8, a1
 ; RV32V-NEXT:    vsetvli zero, zero, e64, m4, ta, mu
-; RV32V-NEXT:    vluxei32.v v12, (a0), v8, v0.t
+; RV32V-NEXT:    vluxei32.v v12, (a0), v10, v0.t
 ; RV32V-NEXT:    vmv.v.v v8, v12
 ; RV32V-NEXT:    ret
 ;
@@ -11947,38 +11959,38 @@ define <8 x double> @mgather_baseidx_v8i16_v8f64(ptr %base, <8 x i16> %idxs, <8
 ; RV32ZVE32F-LABEL: mgather_baseidx_v8i16_v8f64:
 ; RV32ZVE32F:       # %bb.0:
 ; RV32ZVE32F-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
-; RV32ZVE32F-NEXT:    vsext.vf2 v10, v8
+; RV32ZVE32F-NEXT:    vmv.v.x v10, a1
+; RV32ZVE32F-NEXT:    li a2, 8
 ; RV32ZVE32F-NEXT:    vsetvli zero, zero, e8, mf2, ta, ma
-; RV32ZVE32F-NEXT:    vmv.x.s a2, v0
-; RV32ZVE32F-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
-; RV32ZVE32F-NEXT:    vsll.vi v8, v10, 3
-; RV32ZVE32F-NEXT:    andi a3, a2, 1
-; RV32ZVE32F-NEXT:    vadd.vx v8, v8, a1
+; RV32ZVE32F-NEXT:    vmv.x.s a1, v0
+; RV32ZVE32F-NEXT:    andi a3, a1, 1
+; RV32ZVE32F-NEXT:    vsetvli zero, zero, e16, m1, ta, ma
+; RV32ZVE32F-NEXT:    vwmaccus.vx v10, a2, v8
 ; RV32ZVE32F-NEXT:    bnez a3, .LBB100_10
 ; RV32ZVE32F-NEXT:  # %bb.1: # %else
-; RV32ZVE32F-NEXT:    andi a1, a2, 2
-; RV32ZVE32F-NEXT:    bnez a1, .LBB100_11
+; RV32ZVE32F-NEXT:    andi a2, a1, 2
+; RV32ZVE32F-NEXT:    bnez a2, .LBB100_11
 ; RV32ZVE32F-NEXT:  .LBB100_2: # %else2
-; RV32ZVE32F-NEXT:    andi a1, a2, 4
-; RV32ZVE32F-NEXT:    bnez a1, .LBB100_12
+; RV32ZVE32F-NEXT:    andi a2, a1, 4
+; RV32ZVE32F-NEXT:    bnez a2, .LBB100_12
 ; RV32ZVE32F-NEXT:  .LBB100_3: # %else5
-; RV32ZVE32F-NEXT:    andi a1, a2, 8
-; RV32ZVE32F-NEXT:    bnez a1, .LBB100_13
+; RV32ZVE32F-NEXT:    andi a2, a1, 8
+; RV32ZVE32F-NEXT:    bnez a2, .LBB100_13
 ; RV32ZVE32F-NEXT:  .LBB100_4: # %else8
-; RV32ZVE32F-NEXT:    andi a1, a2, 16
-; RV32ZVE32F-NEXT:    bnez a1, .LBB100_14
+; RV32ZVE32F-NEXT:    andi a2, a1, 16
+; RV32ZVE32F-NEXT:    bnez a2, .LBB100_14
 ; RV32ZVE32F-NEXT:  .LBB100_5: # %else11
-; RV32ZVE32F-NEXT:    andi a1, a2, 32
-; RV32ZVE32F-NEXT:    bnez a1, .LBB100_15
+; RV32ZVE32F-NEXT:    andi a2, a1, 32
+; RV32ZVE32F-NEXT:    bnez a2, .LBB100_15
 ; RV32ZVE32F-NEXT:  .LBB100_6: # %else14
-; RV32ZVE32F-NEXT:    andi a1, a2, 64
-; RV32ZVE32F-NEXT:    bnez a1, .LBB100_16
+; RV32ZVE32F-NEXT:    andi a2, a1, 64
+; RV32ZVE32F-NEXT:    bnez a2, .LBB100_16
 ; RV32ZVE32F-NEXT:  .LBB100_7: # %else17
-; RV32ZVE32F-NEXT:    andi a1, a2, -128
+; RV32ZVE32F-NEXT:    andi a1, a1, -128
 ; RV32ZVE32F-NEXT:    beqz a1, .LBB100_9
 ; RV32ZVE32F-NEXT:  .LBB100_8: # %cond.load19
 ; RV32ZVE32F-NEXT:    vsetivli zero, 1, e32, m2, ta, ma
-; RV32ZVE32F-NEXT:    vslidedown.vi v8, v8, 7
+; RV32ZVE32F-NEXT:    vslidedown.vi v8, v10, 7
 ; RV32ZVE32F-NEXT:    vmv.x.s a1, v8
 ; RV32ZVE32F-NEXT:    fld fa7, 0(a1)
 ; RV32ZVE32F-NEXT:  .LBB100_9: # %else20
@@ -11992,51 +12004,52 @@ define <8 x double> @mgather_baseidx_v8i16_v8f64(ptr %base, <8 x i16> %idxs, <8
 ; RV32ZVE32F-NEXT:    fsd fa7, 56(a0)
 ; RV32ZVE32F-NEXT:    ret
 ; RV32ZVE32F-NEXT:  .LBB100_10: # %cond.load
-; RV32ZVE32F-NEXT:    vmv.x.s a1, v8
-; RV32ZVE32F-NEXT:    fld fa0, 0(a1)
-; RV32ZVE32F-NEXT:    andi a1, a2, 2
-; RV32ZVE32F-NEXT:    beqz a1, .LBB100_2
+; RV32ZVE32F-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
+; RV32ZVE32F-NEXT:    vmv.x.s a2, v10
+; RV32ZVE32F-NEXT:    fld fa0, 0(a2)
+; RV32ZVE32F-NEXT:    andi a2, a1, 2
+; RV32ZVE32F-NEXT:    beqz a2, .LBB100_2
 ; RV32ZVE32F-NEXT:  .LBB100_11: # %cond.load1
 ; RV32ZVE32F-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
-; RV32ZVE32F-NEXT:    vslidedown.vi v10, v8, 1
-; RV32ZVE32F-NEXT:    vmv.x.s a1, v10
-; RV32ZVE32F-NEXT:    fld fa1, 0(a1)
-; RV32ZVE32F-NEXT:    andi a1, a2, 4
-; RV32ZVE32F-NEXT:    beqz a1, .LBB100_3
+; RV32ZVE32F-NEXT:    vslidedown.vi v8, v10, 1
+; RV32ZVE32F-NEXT:    vmv.x.s a2, v8
+; RV32ZVE32F-NEXT:    fld fa1, 0(a2)
+; RV32ZVE32F-NEXT:    andi a2, a1, 4
+; RV32ZVE32F-NEXT:    beqz a2, .LBB100_3
 ; RV32ZVE32F-NEXT:  .LBB100_12: # %cond.load4
 ; RV32ZVE32F-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
-; RV32ZVE32F-NEXT:    vslidedown.vi v10, v8, 2
-; RV32ZVE32F-NEXT:    vmv.x.s a1, v10
-; RV32ZVE32F-NEXT:    fld fa2, 0(a1)
-; RV32ZVE32F-NEXT:    andi a1, a2, 8
-; RV32ZVE32F-NEXT:    beqz a1, .LBB100_4
+; RV32ZVE32F-NEXT:    vslidedown.vi v8, v10, 2
+; RV32ZVE32F-NEXT:    vmv.x.s a2, v8
+; RV32ZVE32F-NEXT:    fld fa2, 0(a2)
+; RV32ZVE32F-NEXT:    andi a2, a1, 8
+; RV32ZVE32F-NEXT:    beqz a2, .LBB100_4
 ; RV32ZVE32F-NEXT:  .LBB100_13: # %cond.load7
 ; RV32ZVE32F-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
-; RV32ZVE32F-NEXT:    vslidedown.vi v10, v8, 3
-; RV32ZVE32F-NEXT:    vmv.x.s a1, v10
-; RV32ZVE32F-NEXT:    fld fa3, 0(a1)
-; RV32ZVE32F-NEXT:    andi a1, a2, 16
-; RV32ZVE32F-NEXT:    beqz a1, .LBB100_5
+; RV32ZVE32F-NEXT:    vslidedown.vi v8, v10, 3
+; RV32ZVE32F-NEXT:    vmv.x.s a2, v8
+; RV32ZVE32F-NEXT:    fld fa3, 0(a2)
+; RV32ZVE32F-NEXT:    andi a2, a1, 16
+; RV32ZVE32F-NEXT:    beqz a2, .LBB100_5
 ; RV32ZVE32F-NEXT:  .LBB100_14: # %cond.load10
 ; RV32ZVE32F-NEXT:    vsetivli zero, 1, e32, m2, ta, ma
-; RV32ZVE32F-NEXT:    vslidedown.vi v10, v8, 4
-; RV32ZVE32F-NEXT:    vmv.x.s a1, v10
-; RV32ZVE32F-NEXT:    fld fa4, 0(a1)
-; RV32ZVE32F-NEXT:    andi a1, a2, 32
-; RV32ZVE32F-NEXT:    beqz a1, .LBB100_6
+; RV32ZVE32F-NEXT:    vslidedown.vi v8, v10, 4
+; RV32ZVE32F-NEXT:    vmv.x.s a2, v8
+; RV32ZVE32F-NEXT:    fld fa4, 0(a2)
+; RV32ZVE32F-NEXT:    andi a2, a1, 32
+; RV32ZVE32F-NEXT:    beqz a2, .LBB100_6
 ; RV32ZVE32F-NEXT:  .LBB100_15: # %cond.load13
 ; RV32ZVE32F-NEXT:    vsetivli zero, 1, e32, m2, ta, ma
-; RV32ZVE32F-NEXT:    vslidedown.vi v10, v8, 5
-; RV32ZVE32F-NEXT:    vmv.x.s a1, v10
-; RV32ZVE32F-NEXT:    fld fa5, 0(a1)
-; RV32ZVE32F-NEXT:    andi a1, a2, 64
-; RV32ZVE32F-NEXT:    beqz a1, .LBB100_7
+; RV32ZVE32F-NEXT:    vslidedown.vi v8, v10, 5
+; RV32ZVE32F-NEXT:    vmv.x.s a2, v8
+; RV32ZVE32F-NEXT:    fld fa5, 0(a2)
+; RV32ZVE32F-NEXT:    andi a2, a1, 64
+; RV32ZVE32F-NEXT:    beqz a2, .LBB100_7
 ; RV32ZVE32F-NEXT:  .LBB100_16: # %cond.load16
 ; RV32ZVE32F-NEXT:    vsetivli zero, 1, e32, m2, ta, ma
-; RV32ZVE32F-NEXT:    vslidedown.vi v10, v8, 6
-; RV32ZVE32F-NEXT:    vmv.x.s a1, v10
-; RV32ZVE32F-NEXT:    fld fa6, 0(a1)
-; RV32ZVE32F-NEXT:    andi a1, a2, -128
+; RV32ZVE32F-NEXT:    vslidedown.vi v8, v10, 6
+; RV32ZVE32F-NEXT:    vmv.x.s a2, v8
+; RV32ZVE32F-NEXT:    fld fa6, 0(a2)
+; RV32ZVE32F-NEXT:    andi a1, a1, -128
 ; RV32ZVE32F-NEXT:    bnez a1, .LBB100_8
 ; RV32ZVE32F-NEXT:    j .LBB100_9
 ;
@@ -12143,11 +12156,11 @@ define <8 x double> @mgather_baseidx_v8i16_v8f64(ptr %base, <8 x i16> %idxs, <8
 define <8 x double> @mgather_baseidx_sext_v8i16_v8f64(ptr %base, <8 x i16> %idxs, <8 x i1> %m, <8 x double> %passthru) {
 ; RV32V-LABEL: mgather_baseidx_sext_v8i16_v8f64:
 ; RV32V:       # %bb.0:
-; RV32V-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
-; RV32V-NEXT:    vsext.vf2 v10, v8
-; RV32V-NEXT:    vsll.vi v8, v10, 3
+; RV32V-NEXT:    li a1, 8
+; RV32V-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; RV32V-NEXT:    vwmulsu.vx v10, v8, a1
 ; RV32V-NEXT:    vsetvli zero, zero, e64, m4, ta, mu
-; RV32V-NEXT:    vluxei32.v v12, (a0), v8, v0.t
+; RV32V-NEXT:    vluxei32.v v12, (a0), v10, v0.t
 ; RV32V-NEXT:    vmv.v.v v8, v12
 ; RV32V-NEXT:    ret
 ;
@@ -12163,38 +12176,38 @@ define <8 x double> @mgather_baseidx_sext_v8i16_v8f64(ptr %base, <8 x i16> %idxs
 ; RV32ZVE32F-LABEL: mgather_baseidx_sext_v8i16_v8f64:
 ; RV32ZVE32F:       # %bb.0:
 ; RV32ZVE32F-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
-; RV32ZVE32F-NEXT:    vsext.vf2 v10, v8
+; RV32ZVE32F-NEXT:    vmv.v.x v10, a1
+; RV32ZVE32F-NEXT:    li a2, 8
 ; RV32ZVE32F-NEXT:    vsetvli zero, zero, e8, mf2, ta, ma
-; RV32ZVE32F-NEXT:    vmv.x.s a2, v0
-; RV32ZVE32F-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
-; RV32ZVE32F-NEXT:    vsll.vi v8, v10, 3
-; RV32ZVE32F-NEXT:    andi a3, a2, 1
-; RV32ZVE32F-NEXT:    vadd.vx v8, v8, a1
+; RV32ZVE32F-NEXT:    vmv.x.s a1, v0
+; RV32ZVE32F-NEXT:    andi a3, a1, 1
+; RV32ZVE32F-NEXT:    vsetvli zero, zero, e16, m1, ta, ma
+; RV32ZVE32F-NEXT:    vwmaccus.vx v10, a2, v8
 ; RV32ZVE32F-NEXT:    bnez a3, .LBB101_10
 ; RV32ZVE32F-NEXT:  # %bb.1: # %else
-; RV32ZVE32F-NEXT:    andi a1, a2, 2
-; RV32ZVE32F-NEXT:    bnez a1, .LBB101_11
+; RV32ZVE32F-NEXT:    andi a2, a1, 2
+; RV32ZVE32F-NEXT:    bnez a2, .LBB101_11
 ; RV32ZVE32F-NEXT:  .LBB101_2: # %else2
-; RV32ZVE32F-NEXT:    andi a1, a2, 4
-; RV32ZVE32F-NEXT:    bnez a1, .LBB101_12
+; RV32ZVE32F-NEXT:    andi a2, a1, 4
+; RV32ZVE32F-NEXT:    bnez a2, .LBB101_12
 ; RV32ZVE32F-NEXT:  .LBB101_3: # %else5
-; RV32ZVE32F-NEXT:    andi a1, a2, 8
-; RV32ZVE32F-NEXT:    bnez a1, .LBB101_13
+; RV32ZVE32F-NEXT:    andi a2, a1, 8
+; RV32ZVE32F-NEXT:    bnez a2, .LBB101_13
 ; RV32ZVE32F-NEXT:  .LBB101_4: # %else8
-; RV32ZVE32F-NEXT:    andi a1, a2, 16
-; RV32ZVE32F-NEXT:    bnez a1, .LBB101_14
+; RV32ZVE32F-NEXT:    andi a2, a1, 16
+; RV32ZVE32F-NEXT:    bnez a2, .LBB101_14
 ; RV32ZVE32F-NEXT:  .LBB101_5: # %else11
-; RV32ZVE32F-NEXT:    andi a1, a2, 32
-; RV32ZVE32F-NEXT:    bnez a1, .LBB101_15
+; RV32ZVE32F-NEXT:    andi a2, a1, 32
+; RV32ZVE32F-NEXT:    bnez a2, .LBB101_15
 ; RV32ZVE32F-NEXT:  .LBB101_6: # %else14
-; RV32ZVE32F-NEXT:    andi a1, a2, 64
-; RV32ZVE32F-NEXT:    bnez a1, .LBB101_16
+; RV32ZVE32F-NEXT:    andi a2, a1, 64
+; RV32ZVE32F-NEXT:    bnez a2, .LBB101_16
 ; RV32ZVE32F-NEXT:  .LBB101_7: # %else17
-; RV32ZVE32F-NEXT:    andi a1, a2, -128
+; RV32ZVE32F-NEXT:    andi a1, a1, -128
 ; RV32ZVE32F-NEXT:    beqz a1, .LBB101_9
 ; RV32ZVE32F-NEXT:  .LBB101_8: # %cond.load19
 ; RV32ZVE32F-NEXT:    vsetivli zero, 1, e32, m2, ta, ma
-; RV32ZVE32F-NEXT:    vslidedown.vi v8, v8, 7
+; RV32ZVE32F-NEXT:    vslidedown.vi v8, v10, 7
 ; RV32ZVE32F-NEXT:    vmv.x.s a1, v8
 ; RV32ZVE32F-NEXT:    fld fa7, 0(a1)
 ; RV32ZVE32F-NEXT:  .LBB101_9: # %else20
@@ -12208,51 +12221,52 @@ define <8 x double> @mgather_baseidx_sext_v8i16_v8f64(ptr %base, <8 x i16> %idxs
 ; RV32ZVE32F-NEXT:    fsd fa7, 56(a0)
 ; RV32ZVE32F-NEXT:    ret
 ; RV32ZVE32F-NEXT:  .LBB101_10: # %cond.load
-; RV32ZVE32F-NEXT:    vmv.x.s a1, v8
-; RV32ZVE32F-NEXT:    fld fa0, 0(a1)
-; RV32ZVE32F-NEXT:    andi a1, a2, 2
-; RV32ZVE32F-NEXT:    beqz a1, .LBB101_2
+; RV32ZVE32F-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
+; RV32ZVE32F-NEXT:    vmv.x.s a2, v10
+; RV32ZVE32F-NEXT:    fld fa0, 0(a2)
+; RV32ZVE32F-NEXT:    andi a2, a1, 2
+; RV32ZVE32F-NEXT:    beqz a2, .LBB101_2
 ; RV32ZVE32F-NEXT:  .LBB101_11: # %cond.load1
 ; RV32ZVE32F-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
-; RV32ZVE32F-NEXT:    vslidedown.vi v10, v8, 1
-; RV32ZVE32F-NEXT:    vmv.x.s a1, v10
-; RV32ZVE32F-NEXT:    fld fa1, 0(a1)
-; RV32ZVE32F-NEXT:    andi a1, a2, 4
-; RV32ZVE32F-NEXT:    beqz a1, .LBB101_3
+; RV32ZVE32F-NEXT:    vslidedown.vi v8, v10, 1
+; RV32ZVE32F-NEXT:    vmv.x.s a2, v8
+; RV32ZVE32F-NEXT:    fld fa1, 0(a2)
+; RV32ZVE32F-NEXT:    andi a2, a1, 4
+; RV32ZVE32F-NEXT:    beqz a2, .LBB101_3
 ; RV32ZVE32F-NEXT:  .LBB101_12: # %cond.load4
 ; RV32ZVE32F-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
-; RV32ZVE32F-NEXT:    vslidedown.vi v10, v8, 2
-; RV32ZVE32F-NEXT:    vmv.x.s a1, v10
-; RV32ZVE32F-NEXT:    fld fa2, 0(a1)
-; RV32ZVE32F-NEXT:    andi a1, a2, 8
-; RV32ZVE32F-NEXT:    beqz a1, .LBB101_4
+; RV32ZVE32F-NEXT:    vslidedown.vi v8, v10, 2
+; RV32ZVE32F-NEXT:    vmv.x.s a2, v8
+; RV32ZVE32F-NEXT:    fld fa2, 0(a2)
+; RV32ZVE32F-NEXT:    andi a2, a1, 8
+; RV32ZVE32F-NEXT:    beqz a2, .LBB101_4
 ; RV32ZVE32F-NEXT:  .LBB101_13: # %cond.load7
 ; RV32ZVE32F-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
-; RV32ZVE32F-NEXT:    vslidedown.vi v10, v8, 3
-; RV32ZVE32F-NEXT:    vmv.x.s a1, v10
-; RV32ZVE32F-NEXT:    fld fa3, 0(a1)
-; RV32ZVE32F-NEXT:    andi a1, a2, 16
-; RV32ZVE32F-NEXT:    beqz a1, .LBB101_5
+; RV32ZVE32F-NEXT:    vslidedown.vi v8, v10, 3
+; RV32ZVE32F-NEXT:    vmv.x.s a2, v8
+; RV32ZVE32F-NEXT:    fld fa3, 0(a2)
+; RV32ZVE32F-NEXT:    andi a2, a1, 16
+; RV32ZVE32F-NEXT:    beqz a2, .LBB101_5
 ; RV32ZVE32F-NEXT:  .LBB101_14: # %cond.load10
 ; RV32ZVE32F-NEXT:    vsetivli zero, 1, e32, m2, ta, ma
-; RV32ZVE32F-NEXT:    vslidedown.vi v10, v8, 4
-; RV32ZVE32F-NEXT:    vmv.x.s a1, v10
-; RV32ZVE32F-NEXT:    fld fa4, 0(a1)
-; RV32ZVE32F-NEXT:    andi a1, a2, 32
-; RV32ZVE32F-NEXT:    beqz a1, .LBB101_6
+; RV32ZVE32F-NEXT:    vslidedown.vi v8, v10, 4
+; RV32ZVE32F-NEXT:    vmv.x.s a2, v8
+; RV32ZVE32F-NEXT:    fld fa4, 0(a2)
+; RV32ZVE32F-NEXT:    andi a2, a1, 32
+; RV32ZVE32F-NEXT:    beqz a2, .LBB101_6
 ; RV32ZVE32F-NEXT:  .LBB101_15: # %cond.load13
 ; RV32ZVE32F-NEXT:    vsetivli zero, 1, e32, m2, ta, ma
-; RV32ZVE32F-NEXT:    vslidedown.vi v10, v8, 5
-; RV32ZVE32F-NEXT:    vmv.x.s a1, v10
-; RV32ZVE32F-NEXT:    fld fa5, 0(a1)
-; RV32ZVE32F-NEXT:    andi a1, a2, 64
-; RV32ZVE32F-NEXT:    beqz a1, .LBB101_7
+; RV32ZVE32F-NEXT:    vslidedown.vi v8, v10, 5
+; RV32ZVE32F-NEXT:    vmv.x.s a2, v8
+; RV32ZVE32F-NEXT:    fld fa5, 0(a2)
+; RV32ZVE32F-NEXT:    andi a2, a1, 64
+; RV32ZVE32F-NEXT:    beqz a2, .LBB101_7
 ; RV32ZVE32F-NEXT:  .LBB101_16: # %cond.load16
 ; RV32ZVE32F-NEXT:    vsetivli zero, 1, e32, m2, ta, ma
-; RV32ZVE32F-NEXT:    vslidedown.vi v10, v8, 6
-; RV32ZVE32F-NEXT:    vmv.x.s a1, v10
-; RV32ZVE32F-NEXT:    fld fa6, 0(a1)
-; RV32ZVE32F-NEXT:    andi a1, a2, -128
+; RV32ZVE32F-NEXT:    vslidedown.vi v8, v10, 6
+; RV32ZVE32F-NEXT:    vmv.x.s a2, v8
+; RV32ZVE32F-NEXT:    fld fa6, 0(a2)
+; RV32ZVE32F-NEXT:    andi a1, a1, -128
 ; RV32ZVE32F-NEXT:    bnez a1, .LBB101_8
 ; RV32ZVE32F-NEXT:    j .LBB101_9
 ;
@@ -12360,59 +12374,59 @@ define <8 x double> @mgather_baseidx_sext_v8i16_v8f64(ptr %base, <8 x i16> %idxs
 define <8 x double> @mgather_baseidx_zext_v8i16_v8f64(ptr %base, <8 x i16> %idxs, <8 x i1> %m, <8 x double> %passthru) {
 ; RV32V-LABEL: mgather_baseidx_zext_v8i16_v8f64:
 ; RV32V:       # %bb.0:
-; RV32V-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
-; RV32V-NEXT:    vzext.vf2 v10, v8
-; RV32V-NEXT:    vsll.vi v8, v10, 3
+; RV32V-NEXT:    li a1, 8
+; RV32V-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; RV32V-NEXT:    vwmulu.vx v10, v8, a1
 ; RV32V-NEXT:    vsetvli zero, zero, e64, m4, ta, mu
-; RV32V-NEXT:    vluxei32.v v12, (a0), v8, v0.t
+; RV32V-NEXT:    vluxei32.v v12, (a0), v10, v0.t
 ; RV32V-NEXT:    vmv.v.v v8, v12
 ; RV32V-NEXT:    ret
 ;
 ; RV64V-LABEL: mgather_baseidx_zext_v8i16_v8f64:
 ; RV64V:       # %bb.0:
-; RV64V-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
-; RV64V-NEXT:    vzext.vf2 v10, v8
-; RV64V-NEXT:    vsll.vi v8, v10, 3
+; RV64V-NEXT:    li a1, 8
+; RV64V-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; RV64V-NEXT:    vwmulu.vx v10, v8, a1
 ; RV64V-NEXT:    vsetvli zero, zero, e64, m4, ta, mu
-; RV64V-NEXT:    vluxei32.v v12, (a0), v8, v0.t
+; RV64V-NEXT:    vluxei32.v v12, (a0), v10, v0.t
 ; RV64V-NEXT:    vmv.v.v v8, v12
 ; RV64V-NEXT:    ret
 ;
 ; RV32ZVE32F-LABEL: mgather_baseidx_zext_v8i16_v8f64:
 ; RV32ZVE32F:       # %bb.0:
 ; RV32ZVE32F-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
-; RV32ZVE32F-NEXT:    vzext.vf2 v10, v8
+; RV32ZVE32F-NEXT:    vmv.v.x v10, a1
+; RV32ZVE32F-NEXT:    li a2, 8
 ; RV32ZVE32F-NEXT:    vsetvli zero, zero, e8, mf2, ta, ma
-; RV32ZVE32F-NEXT:    vmv.x.s a2, v0
-; RV32ZVE32F-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
-; RV32ZVE32F-NEXT:    vsll.vi v8, v10, 3
-; RV32ZVE32F-NEXT:    andi a3, a2, 1
-; RV32ZVE32F-NEXT:    vadd.vx v8, v8, a1
+; RV32ZVE32F-NEXT:    vmv.x.s a1, v0
+; RV32ZVE32F-NEXT:    andi a3, a1, 1
+; RV32ZVE32F-NEXT:    vsetvli zero, zero, e16, m1, ta, ma
+; RV32ZVE32F-NEXT:    vwmaccu.vx v10, a2, v8
 ; RV32ZVE32F-NEXT:    bnez a3, .LBB102_10
 ; RV32ZVE32F-NEXT:  # %bb.1: # %else
-; RV32ZVE32F-NEXT:    andi a1, a2, 2
-; RV32ZVE32F-NEXT:    bnez a1, .LBB102_11
+; RV32ZVE32F-NEXT:    andi a2, a1, 2
+; RV32ZVE32F-NEXT:    bnez a2, .LBB102_11
 ; RV32ZVE32F-NEXT:  .LBB102_2: # %else2
-; RV32ZVE32F-NEXT:    andi a1, a2, 4
-; RV32ZVE32F-NEXT:    bnez a1, .LBB102_12
+; RV32ZVE32F-NEXT:    andi a2, a1, 4
+; RV32ZVE32F-NEXT:    bnez a2, .LBB102_12
 ; RV32ZVE32F-NEXT:  .LBB102_3: # %else5
-; RV32ZVE32F-NEXT:    andi a1, a2, 8
-; RV32ZVE32F-NEXT:    bnez a1, .LBB102_13
+; RV32ZVE32F-NEXT:    andi a2, a1, 8
+; RV32ZVE32F-NEXT:    bnez a2, .LBB102_13
 ; RV32ZVE32F-NEXT:  .LBB102_4: # %else8
-; RV32ZVE32F-NEXT:    andi a1, a2, 16
-; RV32ZVE32F-NEXT:    bnez a1, .LBB102_14
+; RV32ZVE32F-NEXT:    andi a2, a1, 16
+; RV32ZVE32F-NEXT:    bnez a2, .LBB102_14
 ; RV32ZVE32F-NEXT:  .LBB102_5: # %else11
-; RV32ZVE32F-NEXT:    andi a1, a2, 32
-; RV32ZVE32F-NEXT:    bnez a1, .LBB102_15
+; RV32ZVE32F-NEXT:    andi a2, a1, 32
+; RV32ZVE32F-NEXT:    bnez a2, .LBB102_15
 ; RV32ZVE32F-NEXT:  .LBB102_6: # %else14
-; RV32ZVE32F-NEXT:    andi a1, a2, 64
-; RV32ZVE32F-NEXT:    bnez a1, .LBB102_16
+; RV32ZVE32F-NEXT:    andi a2, a1, 64
+; RV32ZVE32F-NEXT:    bnez a2, .LBB102_16
 ; RV32ZVE32F-NEXT:  .LBB102_7: # %else17
-; RV32ZVE32F-NEXT:    andi a1, a2, -128
+; RV32ZVE32F-NEXT:    andi a1, a1, -128
 ; RV32ZVE32F-NEXT:    beqz a1, .LBB102_9
 ; RV32ZVE32F-NEXT:  .LBB102_8: # %cond.load19
 ; RV32ZVE32F-NEXT:    vsetivli zero, 1, e32, m2, ta, ma
-; RV32ZVE32F-NEXT:    vslidedown.vi v8, v8, 7
+; RV32ZVE32F-NEXT:    vslidedown.vi v8, v10, 7
 ; RV32ZVE32F-NEXT:    vmv.x.s a1, v8
 ; RV32ZVE32F-NEXT:    fld fa7, 0(a1)
 ; RV32ZVE32F-NEXT:  .LBB102_9: # %else20
@@ -12426,51 +12440,52 @@ define <8 x double> @mgather_baseidx_zext_v8i16_v8f64(ptr %base, <8 x i16> %idxs
 ; RV32ZVE32F-NEXT:    fsd fa7, 56(a0)
 ; RV32ZVE32F-NEXT:    ret
 ; RV32ZVE32F-NEXT:  .LBB102_10: # %cond.load
-; RV32ZVE32F-NEXT:    vmv.x.s a1, v8
-; RV32ZVE32F-NEXT:    fld fa0, 0(a1)
-; RV32ZVE32F-NEXT:    andi a1, a2, 2
-; RV32ZVE32F-NEXT:    beqz a1, .LBB102_2
+; RV32ZVE32F-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
+; RV32ZVE32F-NEXT:    vmv.x.s a2, v10
+; RV32ZVE32F-NEXT:    fld fa0, 0(a2)
+; RV32ZVE32F-NEXT:    andi a2, a1, 2
+; RV32ZVE32F-NEXT:    beqz a2, .LBB102_2
 ; RV32ZVE32F-NEXT:  .LBB102_11: # %cond.load1
 ; RV32ZVE32F-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
-; RV32ZVE32F-NEXT:    vslidedown.vi v10, v8, 1
-; RV32ZVE32F-NEXT:    vmv.x.s a1, v10
-; RV32ZVE32F-NEXT:    fld fa1, 0(a1)
-; RV32ZVE32F-NEXT:    andi a1, a2, 4
-; RV32ZVE32F-NEXT:    beqz a1, .LBB102_3
+; RV32ZVE32F-NEXT:    vslidedown.vi v8, v10, 1
+; RV32ZVE32F-NEXT:    vmv.x.s a2, v8
+; RV32ZVE32F-NEXT:    fld fa1, 0(a2)
+; RV32ZVE32F-NEXT:    andi a2, a1, 4
+; RV32ZVE32F-NEXT:    beqz a2, .LBB102_3
 ; RV32ZVE32F-NEXT:  .LBB102_12: # %cond.load4
 ; RV32ZVE32F-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
-; RV32ZVE32F-NEXT:    vslidedown.vi v10, v8, 2
-; RV32ZVE32F-NEXT:    vmv.x.s a1, v10
-; RV32ZVE32F-NEXT:    fld fa2, 0(a1)
-; RV32ZVE32F-NEXT:    andi a1, a2, 8
-; RV32ZVE32F-NEXT:    beqz a1, .LBB102_4
+; RV32ZVE32F-NEXT:    vslidedown.vi v8, v10, 2
+; RV32ZVE32F-NEXT:    vmv.x.s a2, v8
+; RV32ZVE32F-NEXT:    fld fa2, 0(a2)
+; RV32ZVE32F-NEXT:    andi a2, a1, 8
+; RV32ZVE32F-NEXT:    beqz a2, .LBB102_4
 ; RV32ZVE32F-NEXT:  .LBB102_13: # %cond.load7
 ; RV32ZVE32F-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
-; RV32ZVE32F-NEXT:    vslidedown.vi v10, v8, 3
-; RV32ZVE32F-NEXT:    vmv.x.s a1, v10
-; RV32ZVE32F-NEXT:    fld fa3, 0(a1)
-; RV32ZVE32F-NEXT:    andi a1, a2, 16
-; RV32ZVE32F-NEXT:    beqz a1, .LBB102_5
+; RV32ZVE32F-NEXT:    vslidedown.vi v8, v10, 3
+; RV32ZVE32F-NEXT:    vmv.x.s a2, v8
+; RV32ZVE32F-NEXT:    fld fa3, 0(a2)
+; RV32ZVE32F-NEXT:    andi a2, a1, 16
+; RV32ZVE32F-NEXT:    beqz a2, .LBB102_5
 ; RV32ZVE32F-NEXT:  .LBB102_14: # %cond.load10
 ; RV32ZVE32F-NEXT:    vsetivli zero, 1, e32, m2, ta, ma
-; RV32ZVE32F-NEXT:    vslidedown.vi v10, v8, 4
-; RV32ZVE32F-NEXT:    vmv.x.s a1, v10
-; RV32ZVE32F-NEXT:    fld fa4, 0(a1)
-; RV32ZVE32F-NEXT:    andi a1, a2, 32
-; RV32ZVE32F-NEXT:    beqz a1, .LBB102_6
+; RV32ZVE32F-NEXT:    vslidedown.vi v8, v10, 4
+; RV32ZVE32F-NEXT:    vmv.x.s a2, v8
+; RV32ZVE32F-NEXT:    fld fa4, 0(a2)
+; RV32ZVE32F-NEXT:    andi a2, a1, 32
+; RV32ZVE32F-NEXT:    beqz a2, .LBB102_6
 ; RV32ZVE32F-NEXT:  .LBB102_15: # %cond.load13
 ; RV32ZVE32F-NEXT:    vsetivli zero, 1, e32, m2, ta, ma
-; RV32ZVE32F-NEXT:    vslidedown.vi v10, v8, 5
-; RV32ZVE32F-NEXT:    vmv.x.s a1, v10
-; RV32ZVE32F-NEXT:    fld fa5, 0(a1)
-; RV32ZVE32F-NEXT:    andi a1, a2, 64
-; RV32ZVE32F-NEXT:    beqz a1, .LBB102_7
+; RV32ZVE32F-NEXT:    vslidedown.vi v8, v10, 5
+; RV32ZVE32F-NEXT:    vmv.x.s a2, v8
+; RV32ZVE32F-NEXT:    fld fa5, 0(a2)
+; RV32ZVE32F-NEXT:    andi a2, a1, 64
+; RV32ZVE32F-NEXT:    beqz a2, .LBB102_7
 ; RV32ZVE32F-NEXT:  .LBB102_16: # %cond.load16
 ; RV32ZVE32F-NEXT:    vsetivli zero, 1, e32, m2, ta, ma
-; RV32ZVE32F-NEXT:    vslidedown.vi v10, v8, 6
-; RV32ZVE32F-NEXT:    vmv.x.s a1, v10
-; RV32ZVE32F-NEXT:    fld fa6, 0(a1)
-; RV32ZVE32F-NEXT:    andi a1, a2, -128
+; RV32ZVE32F-NEXT:    vslidedown.vi v8, v10, 6
+; RV32ZVE32F-NEXT:    vmv.x.s a2, v8
+; RV32ZVE32F-NEXT:    fld fa6, 0(a2)
+; RV32ZVE32F-NEXT:    andi a1, a1, -128
 ; RV32ZVE32F-NEXT:    bnez a1, .LBB102_8
 ; RV32ZVE32F-NEXT:    j .LBB102_9
 ;
@@ -12595,10 +12610,11 @@ define <8 x double> @mgather_baseidx_v8i32_v8f64(ptr %base, <8 x i32> %idxs, <8
 ;
 ; RV64V-LABEL: mgather_baseidx_v8i32_v8f64:
 ; RV64V:       # %bb.0:
-; RV64V-NEXT:    vsetivli zero, 8, e64, m4, ta, mu
-; RV64V-NEXT:    vsext.vf2 v16, v8
-; RV64V-NEXT:    vsll.vi v8, v16, 3
-; RV64V-NEXT:    vluxei64.v v12, (a0), v8, v0.t
+; RV64V-NEXT:    li a1, 8
+; RV64V-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
+; RV64V-NEXT:    vwmulsu.vx v16, v8, a1
+; RV64V-NEXT:    vsetvli zero, zero, e64, m4, ta, mu
+; RV64V-NEXT:    vluxei64.v v12, (a0), v16, v0.t
 ; RV64V-NEXT:    vmv.v.v v8, v12
 ; RV64V-NEXT:    ret
 ;
@@ -12809,10 +12825,11 @@ define <8 x double> @mgather_baseidx_sext_v8i32_v8f64(ptr %base, <8 x i32> %idxs
 ;
 ; RV64V-LABEL: mgather_baseidx_sext_v8i32_v8f64:
 ; RV64V:       # %bb.0:
-; RV64V-NEXT:    vsetivli zero, 8, e64, m4, ta, mu
-; RV64V-NEXT:    vsext.vf2 v16, v8
-; RV64V-NEXT:    vsll.vi v8, v16, 3
-; RV64V-NEXT:    vluxei64.v v12, (a0), v8, v0.t
+; RV64V-NEXT:    li a1, 8
+; RV64V-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
+; RV64V-NEXT:    vwmulsu.vx v16, v8, a1
+; RV64V-NEXT:    vsetvli zero, zero, e64, m4, ta, mu
+; RV64V-NEXT:    vluxei64.v v12, (a0), v16, v0.t
 ; RV64V-NEXT:    vmv.v.v v8, v12
 ; RV64V-NEXT:    ret
 ;
@@ -13024,10 +13041,11 @@ define <8 x double> @mgather_baseidx_zext_v8i32_v8f64(ptr %base, <8 x i32> %idxs
 ;
 ; RV64V-LABEL: mgather_baseidx_zext_v8i32_v8f64:
 ; RV64V:       # %bb.0:
-; RV64V-NEXT:    vsetivli zero, 8, e64, m4, ta, mu
-; RV64V-NEXT:    vzext.vf2 v16, v8
-; RV64V-NEXT:    vsll.vi v8, v16, 3
-; RV64V-NEXT:    vluxei64.v v12, (a0), v8, v0.t
+; RV64V-NEXT:    li a1, 8
+; RV64V-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
+; RV64V-NEXT:    vwmulu.vx v16, v8, a1
+; RV64V-NEXT:    vsetvli zero, zero, e64, m4, ta, mu
+; RV64V-NEXT:    vluxei64.v v12, (a0), v16, v0.t
 ; RV64V-NEXT:    vmv.v.v v8, v12
 ; RV64V-NEXT:    ret
 ;
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-masked-scatter.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-masked-scatter.ll
index 7ec4726925704..7354f9afa9a71 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-masked-scatter.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-masked-scatter.ll
@@ -2148,20 +2148,20 @@ define void @mscatter_baseidx_sext_v8i8_v8i32(<8 x i32> %val, ptr %base, <8 x i8
 define void @mscatter_baseidx_zext_v8i8_v8i32(<8 x i32> %val, ptr %base, <8 x i8> %idxs, <8 x i1> %m) {
 ; RV32-LABEL: mscatter_baseidx_zext_v8i8_v8i32:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
-; RV32-NEXT:    vzext.vf2 v11, v10
-; RV32-NEXT:    vsll.vi v10, v11, 2
+; RV32-NEXT:    li a1, 4
+; RV32-NEXT:    vsetivli zero, 8, e8, mf2, ta, ma
+; RV32-NEXT:    vwmulu.vx v11, v10, a1
 ; RV32-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
-; RV32-NEXT:    vsoxei16.v v8, (a0), v10, v0.t
+; RV32-NEXT:    vsoxei16.v v8, (a0), v11, v0.t
 ; RV32-NEXT:    ret
 ;
 ; RV64V-LABEL: mscatter_baseidx_zext_v8i8_v8i32:
 ; RV64V:       # %bb.0:
-; RV64V-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
-; RV64V-NEXT:    vzext.vf2 v11, v10
-; RV64V-NEXT:    vsll.vi v10, v11, 2
+; RV64V-NEXT:    li a1, 4
+; RV64V-NEXT:    vsetivli zero, 8, e8, mf2, ta, ma
+; RV64V-NEXT:    vwmulu.vx v11, v10, a1
 ; RV64V-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
-; RV64V-NEXT:    vsoxei16.v v8, (a0), v10, v0.t
+; RV64V-NEXT:    vsoxei16.v v8, (a0), v11, v0.t
 ; RV64V-NEXT:    ret
 ;
 ; RV64ZVE32F-LABEL: mscatter_baseidx_zext_v8i8_v8i32:
@@ -2295,10 +2295,11 @@ define void @mscatter_baseidx_zext_v8i8_v8i32(<8 x i32> %val, ptr %base, <8 x i8
 define void @mscatter_baseidx_v8i16_v8i32(<8 x i32> %val, ptr %base, <8 x i16> %idxs, <8 x i1> %m) {
 ; RV32-LABEL: mscatter_baseidx_v8i16_v8i32:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
-; RV32-NEXT:    vsext.vf2 v12, v10
-; RV32-NEXT:    vsll.vi v10, v12, 2
-; RV32-NEXT:    vsoxei32.v v8, (a0), v10, v0.t
+; RV32-NEXT:    li a1, 4
+; RV32-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; RV32-NEXT:    vwmulsu.vx v12, v10, a1
+; RV32-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
+; RV32-NEXT:    vsoxei32.v v8, (a0), v12, v0.t
 ; RV32-NEXT:    ret
 ;
 ; RV64V-LABEL: mscatter_baseidx_v8i16_v8i32:
@@ -2433,10 +2434,11 @@ define void @mscatter_baseidx_v8i16_v8i32(<8 x i32> %val, ptr %base, <8 x i16> %
 define void @mscatter_baseidx_sext_v8i16_v8i32(<8 x i32> %val, ptr %base, <8 x i16> %idxs, <8 x i1> %m) {
 ; RV32-LABEL: mscatter_baseidx_sext_v8i16_v8i32:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
-; RV32-NEXT:    vsext.vf2 v12, v10
-; RV32-NEXT:    vsll.vi v10, v12, 2
-; RV32-NEXT:    vsoxei32.v v8, (a0), v10, v0.t
+; RV32-NEXT:    li a1, 4
+; RV32-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; RV32-NEXT:    vwmulsu.vx v12, v10, a1
+; RV32-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
+; RV32-NEXT:    vsoxei32.v v8, (a0), v12, v0.t
 ; RV32-NEXT:    ret
 ;
 ; RV64V-LABEL: mscatter_baseidx_sext_v8i16_v8i32:
@@ -2572,18 +2574,20 @@ define void @mscatter_baseidx_sext_v8i16_v8i32(<8 x i32> %val, ptr %base, <8 x i
 define void @mscatter_baseidx_zext_v8i16_v8i32(<8 x i32> %val, ptr %base, <8 x i16> %idxs, <8 x i1> %m) {
 ; RV32-LABEL: mscatter_baseidx_zext_v8i16_v8i32:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
-; RV32-NEXT:    vzext.vf2 v12, v10
-; RV32-NEXT:    vsll.vi v10, v12, 2
-; RV32-NEXT:    vsoxei32.v v8, (a0), v10, v0.t
+; RV32-NEXT:    li a1, 4
+; RV32-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; RV32-NEXT:    vwmulu.vx v12, v10, a1
+; RV32-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
+; RV32-NEXT:    vsoxei32.v v8, (a0), v12, v0.t
 ; RV32-NEXT:    ret
 ;
 ; RV64V-LABEL: mscatter_baseidx_zext_v8i16_v8i32:
 ; RV64V:       # %bb.0:
-; RV64V-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
-; RV64V-NEXT:    vzext.vf2 v12, v10
-; RV64V-NEXT:    vsll.vi v10, v12, 2
-; RV64V-NEXT:    vsoxei32.v v8, (a0), v10, v0.t
+; RV64V-NEXT:    li a1, 4
+; RV64V-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; RV64V-NEXT:    vwmulu.vx v12, v10, a1
+; RV64V-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
+; RV64V-NEXT:    vsoxei32.v v8, (a0), v12, v0.t
 ; RV64V-NEXT:    ret
 ;
 ; RV64ZVE32F-LABEL: mscatter_baseidx_zext_v8i16_v8i32:
@@ -2725,10 +2729,9 @@ define void @mscatter_baseidx_v8i32(<8 x i32> %val, ptr %base, <8 x i32> %idxs,
 ;
 ; RV64V-LABEL: mscatter_baseidx_v8i32:
 ; RV64V:       # %bb.0:
-; RV64V-NEXT:    vsetivli zero, 8, e64, m4, ta, ma
-; RV64V-NEXT:    vsext.vf2 v12, v10
-; RV64V-NEXT:    vsll.vi v12, v12, 2
-; RV64V-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
+; RV64V-NEXT:    li a1, 4
+; RV64V-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
+; RV64V-NEXT:    vwmulsu.vx v12, v10, a1
 ; RV64V-NEXT:    vsoxei64.v v8, (a0), v12, v0.t
 ; RV64V-NEXT:    ret
 ;
@@ -3881,20 +3884,20 @@ define void @mscatter_baseidx_sext_v8i8_v8i64(<8 x i64> %val, ptr %base, <8 x i8
 define void @mscatter_baseidx_zext_v8i8_v8i64(<8 x i64> %val, ptr %base, <8 x i8> %idxs, <8 x i1> %m) {
 ; RV32V-LABEL: mscatter_baseidx_zext_v8i8_v8i64:
 ; RV32V:       # %bb.0:
-; RV32V-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
-; RV32V-NEXT:    vzext.vf2 v13, v12
-; RV32V-NEXT:    vsll.vi v12, v13, 3
+; RV32V-NEXT:    li a1, 8
+; RV32V-NEXT:    vsetivli zero, 8, e8, mf2, ta, ma
+; RV32V-NEXT:    vwmulu.vx v13, v12, a1
 ; RV32V-NEXT:    vsetvli zero, zero, e64, m4, ta, ma
-; RV32V-NEXT:    vsoxei16.v v8, (a0), v12, v0.t
+; RV32V-NEXT:    vsoxei16.v v8, (a0), v13, v0.t
 ; RV32V-NEXT:    ret
 ;
 ; RV64V-LABEL: mscatter_baseidx_zext_v8i8_v8i64:
 ; RV64V:       # %bb.0:
-; RV64V-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
-; RV64V-NEXT:    vzext.vf2 v13, v12
-; RV64V-NEXT:    vsll.vi v12, v13, 3
+; RV64V-NEXT:    li a1, 8
+; RV64V-NEXT:    vsetivli zero, 8, e8, mf2, ta, ma
+; RV64V-NEXT:    vwmulu.vx v13, v12, a1
 ; RV64V-NEXT:    vsetvli zero, zero, e64, m4, ta, ma
-; RV64V-NEXT:    vsoxei16.v v8, (a0), v12, v0.t
+; RV64V-NEXT:    vsoxei16.v v8, (a0), v13, v0.t
 ; RV64V-NEXT:    ret
 ;
 ; RV32ZVE32F-LABEL: mscatter_baseidx_zext_v8i8_v8i64:
@@ -4029,7 +4032,7 @@ define void @mscatter_baseidx_zext_v8i8_v8i64(<8 x i64> %val, ptr %base, <8 x i8
 ;
 ; RV64ZVE32F-LABEL: mscatter_baseidx_zext_v8i8_v8i64:
 ; RV64ZVE32F:       # %bb.0:
-; RV64ZVE32F-NEXT:    ld a4, 40(a0)
+; RV64ZVE32F-NEXT:    ld a5, 40(a0)
 ; RV64ZVE32F-NEXT:    ld a3, 48(a0)
 ; RV64ZVE32F-NEXT:    ld a2, 56(a0)
 ; RV64ZVE32F-NEXT:    ld t1, 8(a0)
@@ -4037,8 +4040,8 @@ define void @mscatter_baseidx_zext_v8i8_v8i64(<8 x i64> %val, ptr %base, <8 x i8
 ; RV64ZVE32F-NEXT:    ld a7, 24(a0)
 ; RV64ZVE32F-NEXT:    ld a6, 32(a0)
 ; RV64ZVE32F-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
-; RV64ZVE32F-NEXT:    vmv.x.s a5, v0
-; RV64ZVE32F-NEXT:    andi t2, a5, 1
+; RV64ZVE32F-NEXT:    vmv.x.s a4, v0
+; RV64ZVE32F-NEXT:    andi t2, a4, 1
 ; RV64ZVE32F-NEXT:    beqz t2, .LBB44_2
 ; RV64ZVE32F-NEXT:  # %bb.1: # %cond.store
 ; RV64ZVE32F-NEXT:    ld a0, 0(a0)
@@ -4048,7 +4051,7 @@ define void @mscatter_baseidx_zext_v8i8_v8i64(<8 x i64> %val, ptr %base, <8 x i8
 ; RV64ZVE32F-NEXT:    add t2, a1, t2
 ; RV64ZVE32F-NEXT:    sd a0, 0(t2)
 ; RV64ZVE32F-NEXT:  .LBB44_2: # %else
-; RV64ZVE32F-NEXT:    andi a0, a5, 2
+; RV64ZVE32F-NEXT:    andi a0, a4, 2
 ; RV64ZVE32F-NEXT:    beqz a0, .LBB44_4
 ; RV64ZVE32F-NEXT:  # %bb.3: # %cond.store1
 ; RV64ZVE32F-NEXT:    vsetivli zero, 1, e8, mf4, ta, ma
@@ -4061,18 +4064,18 @@ define void @mscatter_baseidx_zext_v8i8_v8i64(<8 x i64> %val, ptr %base, <8 x i8
 ; RV64ZVE32F-NEXT:  .LBB44_4: # %else2
 ; RV64ZVE32F-NEXT:    vsetivli zero, 4, e8, mf2, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v9, v8, 4
-; RV64ZVE32F-NEXT:    andi a0, a5, 4
+; RV64ZVE32F-NEXT:    andi a0, a4, 4
 ; RV64ZVE32F-NEXT:    vsetivli zero, 2, e8, mf4, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v8, v8, 2
 ; RV64ZVE32F-NEXT:    bnez a0, .LBB44_12
 ; RV64ZVE32F-NEXT:  # %bb.5: # %else4
-; RV64ZVE32F-NEXT:    andi a0, a5, 8
+; RV64ZVE32F-NEXT:    andi a0, a4, 8
 ; RV64ZVE32F-NEXT:    bnez a0, .LBB44_13
 ; RV64ZVE32F-NEXT:  .LBB44_6: # %else6
-; RV64ZVE32F-NEXT:    andi a0, a5, 16
+; RV64ZVE32F-NEXT:    andi a0, a4, 16
 ; RV64ZVE32F-NEXT:    bnez a0, .LBB44_14
 ; RV64ZVE32F-NEXT:  .LBB44_7: # %else8
-; RV64ZVE32F-NEXT:    andi a0, a5, 32
+; RV64ZVE32F-NEXT:    andi a0, a4, 32
 ; RV64ZVE32F-NEXT:    beqz a0, .LBB44_9
 ; RV64ZVE32F-NEXT:  .LBB44_8: # %cond.store9
 ; RV64ZVE32F-NEXT:    vslidedown.vi v8, v9, 1
@@ -4080,13 +4083,13 @@ define void @mscatter_baseidx_zext_v8i8_v8i64(<8 x i64> %val, ptr %base, <8 x i8
 ; RV64ZVE32F-NEXT:    andi a0, a0, 255
 ; RV64ZVE32F-NEXT:    slli a0, a0, 3
 ; RV64ZVE32F-NEXT:    add a0, a1, a0
-; RV64ZVE32F-NEXT:    sd a4, 0(a0)
+; RV64ZVE32F-NEXT:    sd a5, 0(a0)
 ; RV64ZVE32F-NEXT:  .LBB44_9: # %else10
-; RV64ZVE32F-NEXT:    andi a0, a5, 64
+; RV64ZVE32F-NEXT:    andi a0, a4, 64
 ; RV64ZVE32F-NEXT:    vslidedown.vi v8, v9, 2
 ; RV64ZVE32F-NEXT:    bnez a0, .LBB44_15
 ; RV64ZVE32F-NEXT:  # %bb.10: # %else12
-; RV64ZVE32F-NEXT:    andi a0, a5, -128
+; RV64ZVE32F-NEXT:    andi a0, a4, -128
 ; RV64ZVE32F-NEXT:    bnez a0, .LBB44_16
 ; RV64ZVE32F-NEXT:  .LBB44_11: # %else14
 ; RV64ZVE32F-NEXT:    ret
@@ -4096,7 +4099,7 @@ define void @mscatter_baseidx_zext_v8i8_v8i64(<8 x i64> %val, ptr %base, <8 x i8
 ; RV64ZVE32F-NEXT:    slli a0, a0, 3
 ; RV64ZVE32F-NEXT:    add a0, a1, a0
 ; RV64ZVE32F-NEXT:    sd t0, 0(a0)
-; RV64ZVE32F-NEXT:    andi a0, a5, 8
+; RV64ZVE32F-NEXT:    andi a0, a4, 8
 ; RV64ZVE32F-NEXT:    beqz a0, .LBB44_6
 ; RV64ZVE32F-NEXT:  .LBB44_13: # %cond.store5
 ; RV64ZVE32F-NEXT:    vslidedown.vi v8, v8, 1
@@ -4105,7 +4108,7 @@ define void @mscatter_baseidx_zext_v8i8_v8i64(<8 x i64> %val, ptr %base, <8 x i8
 ; RV64ZVE32F-NEXT:    slli a0, a0, 3
 ; RV64ZVE32F-NEXT:    add a0, a1, a0
 ; RV64ZVE32F-NEXT:    sd a7, 0(a0)
-; RV64ZVE32F-NEXT:    andi a0, a5, 16
+; RV64ZVE32F-NEXT:    andi a0, a4, 16
 ; RV64ZVE32F-NEXT:    beqz a0, .LBB44_7
 ; RV64ZVE32F-NEXT:  .LBB44_14: # %cond.store7
 ; RV64ZVE32F-NEXT:    vmv.x.s a0, v9
@@ -4113,7 +4116,7 @@ define void @mscatter_baseidx_zext_v8i8_v8i64(<8 x i64> %val, ptr %base, <8 x i8
 ; RV64ZVE32F-NEXT:    slli a0, a0, 3
 ; RV64ZVE32F-NEXT:    add a0, a1, a0
 ; RV64ZVE32F-NEXT:    sd a6, 0(a0)
-; RV64ZVE32F-NEXT:    andi a0, a5, 32
+; RV64ZVE32F-NEXT:    andi a0, a4, 32
 ; RV64ZVE32F-NEXT:    bnez a0, .LBB44_8
 ; RV64ZVE32F-NEXT:    j .LBB44_9
 ; RV64ZVE32F-NEXT:  .LBB44_15: # %cond.store11
@@ -4122,7 +4125,7 @@ define void @mscatter_baseidx_zext_v8i8_v8i64(<8 x i64> %val, ptr %base, <8 x i8
 ; RV64ZVE32F-NEXT:    slli a0, a0, 3
 ; RV64ZVE32F-NEXT:    add a0, a1, a0
 ; RV64ZVE32F-NEXT:    sd a3, 0(a0)
-; RV64ZVE32F-NEXT:    andi a0, a5, -128
+; RV64ZVE32F-NEXT:    andi a0, a4, -128
 ; RV64ZVE32F-NEXT:    beqz a0, .LBB44_11
 ; RV64ZVE32F-NEXT:  .LBB44_16: # %cond.store13
 ; RV64ZVE32F-NEXT:    vslidedown.vi v8, v8, 1
@@ -4141,11 +4144,11 @@ define void @mscatter_baseidx_zext_v8i8_v8i64(<8 x i64> %val, ptr %base, <8 x i8
 define void @mscatter_baseidx_v8i16_v8i64(<8 x i64> %val, ptr %base, <8 x i16> %idxs, <8 x i1> %m) {
 ; RV32V-LABEL: mscatter_baseidx_v8i16_v8i64:
 ; RV32V:       # %bb.0:
-; RV32V-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
-; RV32V-NEXT:    vsext.vf2 v14, v12
-; RV32V-NEXT:    vsll.vi v12, v14, 3
+; RV32V-NEXT:    li a1, 8
+; RV32V-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; RV32V-NEXT:    vwmulsu.vx v14, v12, a1
 ; RV32V-NEXT:    vsetvli zero, zero, e64, m4, ta, ma
-; RV32V-NEXT:    vsoxei32.v v8, (a0), v12, v0.t
+; RV32V-NEXT:    vsoxei32.v v8, (a0), v14, v0.t
 ; RV32V-NEXT:    ret
 ;
 ; RV64V-LABEL: mscatter_baseidx_v8i16_v8i64:
@@ -4173,47 +4176,47 @@ define void @mscatter_baseidx_v8i16_v8i64(<8 x i64> %val, ptr %base, <8 x i16> %
 ; RV32ZVE32F-NEXT:    lw a7, 44(a0)
 ; RV32ZVE32F-NEXT:    lw a4, 48(a0)
 ; RV32ZVE32F-NEXT:    lw a5, 52(a0)
-; RV32ZVE32F-NEXT:    lw t3, 24(a0)
-; RV32ZVE32F-NEXT:    lw t4, 28(a0)
-; RV32ZVE32F-NEXT:    lw t1, 32(a0)
-; RV32ZVE32F-NEXT:    lw t2, 36(a0)
-; RV32ZVE32F-NEXT:    lw s0, 8(a0)
-; RV32ZVE32F-NEXT:    lw s1, 12(a0)
-; RV32ZVE32F-NEXT:    lw t5, 16(a0)
-; RV32ZVE32F-NEXT:    lw t6, 20(a0)
+; RV32ZVE32F-NEXT:    lw t2, 24(a0)
+; RV32ZVE32F-NEXT:    lw t3, 28(a0)
+; RV32ZVE32F-NEXT:    lw t0, 32(a0)
+; RV32ZVE32F-NEXT:    lw t1, 36(a0)
+; RV32ZVE32F-NEXT:    lw t6, 8(a0)
+; RV32ZVE32F-NEXT:    lw s0, 12(a0)
+; RV32ZVE32F-NEXT:    lw t4, 16(a0)
+; RV32ZVE32F-NEXT:    lw t5, 20(a0)
 ; RV32ZVE32F-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
-; RV32ZVE32F-NEXT:    vsext.vf2 v10, v8
-; RV32ZVE32F-NEXT:    vsll.vi v8, v10, 3
+; RV32ZVE32F-NEXT:    vmv.v.x v10, a1
+; RV32ZVE32F-NEXT:    li s1, 8
 ; RV32ZVE32F-NEXT:    vsetvli zero, zero, e8, mf2, ta, ma
-; RV32ZVE32F-NEXT:    vmv.x.s t0, v0
-; RV32ZVE32F-NEXT:    andi s2, t0, 1
-; RV32ZVE32F-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
-; RV32ZVE32F-NEXT:    vadd.vx v8, v8, a1
+; RV32ZVE32F-NEXT:    vmv.x.s a1, v0
+; RV32ZVE32F-NEXT:    andi s2, a1, 1
+; RV32ZVE32F-NEXT:    vsetvli zero, zero, e16, m1, ta, ma
+; RV32ZVE32F-NEXT:    vwmaccus.vx v10, s1, v8
 ; RV32ZVE32F-NEXT:    bnez s2, .LBB45_10
 ; RV32ZVE32F-NEXT:  # %bb.1: # %else
-; RV32ZVE32F-NEXT:    andi a0, t0, 2
+; RV32ZVE32F-NEXT:    andi a0, a1, 2
 ; RV32ZVE32F-NEXT:    bnez a0, .LBB45_11
 ; RV32ZVE32F-NEXT:  .LBB45_2: # %else2
-; RV32ZVE32F-NEXT:    andi a0, t0, 4
+; RV32ZVE32F-NEXT:    andi a0, a1, 4
 ; RV32ZVE32F-NEXT:    bnez a0, .LBB45_12
 ; RV32ZVE32F-NEXT:  .LBB45_3: # %else4
-; RV32ZVE32F-NEXT:    andi a0, t0, 8
+; RV32ZVE32F-NEXT:    andi a0, a1, 8
 ; RV32ZVE32F-NEXT:    bnez a0, .LBB45_13
 ; RV32ZVE32F-NEXT:  .LBB45_4: # %else6
-; RV32ZVE32F-NEXT:    andi a0, t0, 16
+; RV32ZVE32F-NEXT:    andi a0, a1, 16
 ; RV32ZVE32F-NEXT:    bnez a0, .LBB45_14
 ; RV32ZVE32F-NEXT:  .LBB45_5: # %else8
-; RV32ZVE32F-NEXT:    andi a0, t0, 32
+; RV32ZVE32F-NEXT:    andi a0, a1, 32
 ; RV32ZVE32F-NEXT:    bnez a0, .LBB45_15
 ; RV32ZVE32F-NEXT:  .LBB45_6: # %else10
-; RV32ZVE32F-NEXT:    andi a0, t0, 64
+; RV32ZVE32F-NEXT:    andi a0, a1, 64
 ; RV32ZVE32F-NEXT:    bnez a0, .LBB45_16
 ; RV32ZVE32F-NEXT:  .LBB45_7: # %else12
-; RV32ZVE32F-NEXT:    andi a0, t0, -128
+; RV32ZVE32F-NEXT:    andi a0, a1, -128
 ; RV32ZVE32F-NEXT:    beqz a0, .LBB45_9
 ; RV32ZVE32F-NEXT:  .LBB45_8: # %cond.store13
 ; RV32ZVE32F-NEXT:    vsetivli zero, 1, e32, m2, ta, ma
-; RV32ZVE32F-NEXT:    vslidedown.vi v8, v8, 7
+; RV32ZVE32F-NEXT:    vslidedown.vi v8, v10, 7
 ; RV32ZVE32F-NEXT:    vmv.x.s a0, v8
 ; RV32ZVE32F-NEXT:    sw a2, 0(a0)
 ; RV32ZVE32F-NEXT:    sw a3, 4(a0)
@@ -4229,60 +4232,61 @@ define void @mscatter_baseidx_v8i16_v8i64(<8 x i64> %val, ptr %base, <8 x i16> %
 ; RV32ZVE32F-NEXT:    ret
 ; RV32ZVE32F-NEXT:  .LBB45_10: # %cond.store
 ; RV32ZVE32F-NEXT:    .cfi_restore_state
-; RV32ZVE32F-NEXT:    lw a1, 0(a0)
+; RV32ZVE32F-NEXT:    lw s1, 0(a0)
 ; RV32ZVE32F-NEXT:    lw a0, 4(a0)
-; RV32ZVE32F-NEXT:    vmv.x.s s2, v8
-; RV32ZVE32F-NEXT:    sw a1, 0(s2)
+; RV32ZVE32F-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
+; RV32ZVE32F-NEXT:    vmv.x.s s2, v10
+; RV32ZVE32F-NEXT:    sw s1, 0(s2)
 ; RV32ZVE32F-NEXT:    sw a0, 4(s2)
-; RV32ZVE32F-NEXT:    andi a0, t0, 2
+; RV32ZVE32F-NEXT:    andi a0, a1, 2
 ; RV32ZVE32F-NEXT:    beqz a0, .LBB45_2
 ; RV32ZVE32F-NEXT:  .LBB45_11: # %cond.store1
 ; RV32ZVE32F-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
-; RV32ZVE32F-NEXT:    vslidedown.vi v10, v8, 1
-; RV32ZVE32F-NEXT:    vmv.x.s a0, v10
-; RV32ZVE32F-NEXT:    sw s0, 0(a0)
-; RV32ZVE32F-NEXT:    sw s1, 4(a0)
-; RV32ZVE32F-NEXT:    andi a0, t0, 4
+; RV32ZVE32F-NEXT:    vslidedown.vi v8, v10, 1
+; RV32ZVE32F-NEXT:    vmv.x.s a0, v8
+; RV32ZVE32F-NEXT:    sw t6, 0(a0)
+; RV32ZVE32F-NEXT:    sw s0, 4(a0)
+; RV32ZVE32F-NEXT:    andi a0, a1, 4
 ; RV32ZVE32F-NEXT:    beqz a0, .LBB45_3
 ; RV32ZVE32F-NEXT:  .LBB45_12: # %cond.store3
 ; RV32ZVE32F-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
-; RV32ZVE32F-NEXT:    vslidedown.vi v10, v8, 2
-; RV32ZVE32F-NEXT:    vmv.x.s a0, v10
-; RV32ZVE32F-NEXT:    sw t5, 0(a0)
-; RV32ZVE32F-NEXT:    sw t6, 4(a0)
-; RV32ZVE32F-NEXT:    andi a0, t0, 8
+; RV32ZVE32F-NEXT:    vslidedown.vi v8, v10, 2
+; RV32ZVE32F-NEXT:    vmv.x.s a0, v8
+; RV32ZVE32F-NEXT:    sw t4, 0(a0)
+; RV32ZVE32F-NEXT:    sw t5, 4(a0)
+; RV32ZVE32F-NEXT:    andi a0, a1, 8
 ; RV32ZVE32F-NEXT:    beqz a0, .LBB45_4
 ; RV32ZVE32F-NEXT:  .LBB45_13: # %cond.store5
 ; RV32ZVE32F-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
-; RV32ZVE32F-NEXT:    vslidedown.vi v10, v8, 3
-; RV32ZVE32F-NEXT:    vmv.x.s a0, v10
-; RV32ZVE32F-NEXT:    sw t3, 0(a0)
-; RV32ZVE32F-NEXT:    sw t4, 4(a0)
-; RV32ZVE32F-NEXT:    andi a0, t0, 16
+; RV32ZVE32F-NEXT:    vslidedown.vi v8, v10, 3
+; RV32ZVE32F-NEXT:    vmv.x.s a0, v8
+; RV32ZVE32F-NEXT:    sw t2, 0(a0)
+; RV32ZVE32F-NEXT:    sw t3, 4(a0)
+; RV32ZVE32F-NEXT:    andi a0, a1, 16
 ; RV32ZVE32F-NEXT:    beqz a0, .LBB45_5
 ; RV32ZVE32F-NEXT:  .LBB45_14: # %cond.store7
 ; RV32ZVE32F-NEXT:    vsetivli zero, 1, e32, m2, ta, ma
-; RV32ZVE32F-NEXT:    vslidedown.vi v10, v8, 4
-; RV32ZVE32F-NEXT:    vmv.x.s a0, v10
-; RV32ZVE32F-NEXT:    sw t1, 0(a0)
-; RV32ZVE32F-NEXT:    sw t2, 4(a0)
-; RV32ZVE32F-NEXT:    andi a0, t0, 32
+; RV32ZVE32F-NEXT:    vslidedown.vi v8, v10, 4
+; RV32ZVE32F-NEXT:    vmv.x.s a0, v8
+; RV32ZVE32F-NEXT:    sw t0, 0(a0)
+; RV32ZVE32F-NEXT:    sw t1, 4(a0)
+; RV32ZVE32F-NEXT:    andi a0, a1, 32
 ; RV32ZVE32F-NEXT:    beqz a0, .LBB45_6
 ; RV32ZVE32F-NEXT:  .LBB45_15: # %cond.store9
 ; RV32ZVE32F-NEXT:    vsetivli zero, 1, e32, m2, ta, ma
-; RV32ZVE32F-NEXT:    vslidedown.vi v10, v8, 5
-; RV32ZVE32F-NEXT:    vmv.x.s a0, v10
+; RV32ZVE32F-NEXT:    vslidedown.vi v8, v10, 5
+; RV32ZVE32F-NEXT:    vmv.x.s a0, v8
 ; RV32ZVE32F-NEXT:    sw a6, 0(a0)
 ; RV32ZVE32F-NEXT:    sw a7, 4(a0)
-; RV32ZVE32F-NEXT:    andi a0, t0, 64
+; RV32ZVE32F-NEXT:    andi a0, a1, 64
 ; RV32ZVE32F-NEXT:    beqz a0, .LBB45_7
 ; RV32ZVE32F-NEXT:  .LBB45_16: # %cond.store11
 ; RV32ZVE32F-NEXT:    vsetivli zero, 1, e32, m2, ta, ma
-; RV32ZVE32F-NEXT:    vslidedown.vi v10, v8, 6
-; RV32ZVE32F-NEXT:    vmv.x.s a0, v10
+; RV32ZVE32F-NEXT:    vslidedown.vi v8, v10, 6
+; RV32ZVE32F-NEXT:    vmv.x.s a0, v8
 ; RV32ZVE32F-NEXT:    sw a4, 0(a0)
 ; RV32ZVE32F-NEXT:    sw a5, 4(a0)
-; RV32ZVE32F-NEXT:    andi a0, t0, -128
+; RV32ZVE32F-NEXT:    andi a0, a1, -128
 ; RV32ZVE32F-NEXT:    bnez a0, .LBB45_8
 ; RV32ZVE32F-NEXT:    j .LBB45_9
 ;
@@ -4392,11 +4396,11 @@ define void @mscatter_baseidx_v8i16_v8i64(<8 x i64> %val, ptr %base, <8 x i16> %
 define void @mscatter_baseidx_sext_v8i16_v8i64(<8 x i64> %val, ptr %base, <8 x i16> %idxs, <8 x i1> %m) {
 ; RV32V-LABEL: mscatter_baseidx_sext_v8i16_v8i64:
 ; RV32V:       # %bb.0:
-; RV32V-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
-; RV32V-NEXT:    vsext.vf2 v14, v12
-; RV32V-NEXT:    vsll.vi v12, v14, 3
+; RV32V-NEXT:    li a1, 8
+; RV32V-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; RV32V-NEXT:    vwmulsu.vx v14, v12, a1
 ; RV32V-NEXT:    vsetvli zero, zero, e64, m4, ta, ma
-; RV32V-NEXT:    vsoxei32.v v8, (a0), v12, v0.t
+; RV32V-NEXT:    vsoxei32.v v8, (a0), v14, v0.t
 ; RV32V-NEXT:    ret
 ;
 ; RV64V-LABEL: mscatter_baseidx_sext_v8i16_v8i64:
@@ -4424,47 +4428,47 @@ define void @mscatter_baseidx_sext_v8i16_v8i64(<8 x i64> %val, ptr %base, <8 x i
 ; RV32ZVE32F-NEXT:    lw a7, 44(a0)
 ; RV32ZVE32F-NEXT:    lw a4, 48(a0)
 ; RV32ZVE32F-NEXT:    lw a5, 52(a0)
-; RV32ZVE32F-NEXT:    lw t3, 24(a0)
-; RV32ZVE32F-NEXT:    lw t4, 28(a0)
-; RV32ZVE32F-NEXT:    lw t1, 32(a0)
-; RV32ZVE32F-NEXT:    lw t2, 36(a0)
-; RV32ZVE32F-NEXT:    lw s0, 8(a0)
-; RV32ZVE32F-NEXT:    lw s1, 12(a0)
-; RV32ZVE32F-NEXT:    lw t5, 16(a0)
-; RV32ZVE32F-NEXT:    lw t6, 20(a0)
+; RV32ZVE32F-NEXT:    lw t2, 24(a0)
+; RV32ZVE32F-NEXT:    lw t3, 28(a0)
+; RV32ZVE32F-NEXT:    lw t0, 32(a0)
+; RV32ZVE32F-NEXT:    lw t1, 36(a0)
+; RV32ZVE32F-NEXT:    lw t6, 8(a0)
+; RV32ZVE32F-NEXT:    lw s0, 12(a0)
+; RV32ZVE32F-NEXT:    lw t4, 16(a0)
+; RV32ZVE32F-NEXT:    lw t5, 20(a0)
 ; RV32ZVE32F-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
-; RV32ZVE32F-NEXT:    vsext.vf2 v10, v8
-; RV32ZVE32F-NEXT:    vsll.vi v8, v10, 3
+; RV32ZVE32F-NEXT:    vmv.v.x v10, a1
+; RV32ZVE32F-NEXT:    li s1, 8
 ; RV32ZVE32F-NEXT:    vsetvli zero, zero, e8, mf2, ta, ma
-; RV32ZVE32F-NEXT:    vmv.x.s t0, v0
-; RV32ZVE32F-NEXT:    andi s2, t0, 1
-; RV32ZVE32F-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
-; RV32ZVE32F-NEXT:    vadd.vx v8, v8, a1
+; RV32ZVE32F-NEXT:    vmv.x.s a1, v0
+; RV32ZVE32F-NEXT:    andi s2, a1, 1
+; RV32ZVE32F-NEXT:    vsetvli zero, zero, e16, m1, ta, ma
+; RV32ZVE32F-NEXT:    vwmaccus.vx v10, s1, v8
 ; RV32ZVE32F-NEXT:    bnez s2, .LBB46_10
 ; RV32ZVE32F-NEXT:  # %bb.1: # %else
-; RV32ZVE32F-NEXT:    andi a0, t0, 2
+; RV32ZVE32F-NEXT:    andi a0, a1, 2
 ; RV32ZVE32F-NEXT:    bnez a0, .LBB46_11
 ; RV32ZVE32F-NEXT:  .LBB46_2: # %else2
-; RV32ZVE32F-NEXT:    andi a0, t0, 4
+; RV32ZVE32F-NEXT:    andi a0, a1, 4
 ; RV32ZVE32F-NEXT:    bnez a0, .LBB46_12
 ; RV32ZVE32F-NEXT:  .LBB46_3: # %else4
-; RV32ZVE32F-NEXT:    andi a0, t0, 8
+; RV32ZVE32F-NEXT:    andi a0, a1, 8
 ; RV32ZVE32F-NEXT:    bnez a0, .LBB46_13
 ; RV32ZVE32F-NEXT:  .LBB46_4: # %else6
-; RV32ZVE32F-NEXT:    andi a0, t0, 16
+; RV32ZVE32F-NEXT:    andi a0, a1, 16
 ; RV32ZVE32F-NEXT:    bnez a0, .LBB46_14
 ; RV32ZVE32F-NEXT:  .LBB46_5: # %else8
-; RV32ZVE32F-NEXT:    andi a0, t0, 32
+; RV32ZVE32F-NEXT:    andi a0, a1, 32
 ; RV32ZVE32F-NEXT:    bnez a0, .LBB46_15
 ; RV32ZVE32F-NEXT:  .LBB46_6: # %else10
-; RV32ZVE32F-NEXT:    andi a0, t0, 64
+; RV32ZVE32F-NEXT:    andi a0, a1, 64
 ; RV32ZVE32F-NEXT:    bnez a0, .LBB46_16
 ; RV32ZVE32F-NEXT:  .LBB46_7: # %else12
-; RV32ZVE32F-NEXT:    andi a0, t0, -128
+; RV32ZVE32F-NEXT:    andi a0, a1, -128
 ; RV32ZVE32F-NEXT:    beqz a0, .LBB46_9
 ; RV32ZVE32F-NEXT:  .LBB46_8: # %cond.store13
 ; RV32ZVE32F-NEXT:    vsetivli zero, 1, e32, m2, ta, ma
-; RV32ZVE32F-NEXT:    vslidedown.vi v8, v8, 7
+; RV32ZVE32F-NEXT:    vslidedown.vi v8, v10, 7
 ; RV32ZVE32F-NEXT:    vmv.x.s a0, v8
 ; RV32ZVE32F-NEXT:    sw a2, 0(a0)
 ; RV32ZVE32F-NEXT:    sw a3, 4(a0)
@@ -4480,60 +4484,61 @@ define void @mscatter_baseidx_sext_v8i16_v8i64(<8 x i64> %val, ptr %base, <8 x i
 ; RV32ZVE32F-NEXT:    ret
 ; RV32ZVE32F-NEXT:  .LBB46_10: # %cond.store
 ; RV32ZVE32F-NEXT:    .cfi_restore_state
-; RV32ZVE32F-NEXT:    lw a1, 0(a0)
+; RV32ZVE32F-NEXT:    lw s1, 0(a0)
 ; RV32ZVE32F-NEXT:    lw a0, 4(a0)
-; RV32ZVE32F-NEXT:    vmv.x.s s2, v8
-; RV32ZVE32F-NEXT:    sw a1, 0(s2)
+; RV32ZVE32F-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
+; RV32ZVE32F-NEXT:    vmv.x.s s2, v10
+; RV32ZVE32F-NEXT:    sw s1, 0(s2)
 ; RV32ZVE32F-NEXT:    sw a0, 4(s2)
-; RV32ZVE32F-NEXT:    andi a0, t0, 2
+; RV32ZVE32F-NEXT:    andi a0, a1, 2
 ; RV32ZVE32F-NEXT:    beqz a0, .LBB46_2
 ; RV32ZVE32F-NEXT:  .LBB46_11: # %cond.store1
 ; RV32ZVE32F-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
-; RV32ZVE32F-NEXT:    vslidedown.vi v10, v8, 1
-; RV32ZVE32F-NEXT:    vmv.x.s a0, v10
-; RV32ZVE32F-NEXT:    sw s0, 0(a0)
-; RV32ZVE32F-NEXT:    sw s1, 4(a0)
-; RV32ZVE32F-NEXT:    andi a0, t0, 4
+; RV32ZVE32F-NEXT:    vslidedown.vi v8, v10, 1
+; RV32ZVE32F-NEXT:    vmv.x.s a0, v8
+; RV32ZVE32F-NEXT:    sw t6, 0(a0)
+; RV32ZVE32F-NEXT:    sw s0, 4(a0)
+; RV32ZVE32F-NEXT:    andi a0, a1, 4
 ; RV32ZVE32F-NEXT:    beqz a0, .LBB46_3
 ; RV32ZVE32F-NEXT:  .LBB46_12: # %cond.store3
 ; RV32ZVE32F-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
-; RV32ZVE32F-NEXT:    vslidedown.vi v10, v8, 2
-; RV32ZVE32F-NEXT:    vmv.x.s a0, v10
-; RV32ZVE32F-NEXT:    sw t5, 0(a0)
-; RV32ZVE32F-NEXT:    sw t6, 4(a0)
-; RV32ZVE32F-NEXT:    andi a0, t0, 8
+; RV32ZVE32F-NEXT:    vslidedown.vi v8, v10, 2
+; RV32ZVE32F-NEXT:    vmv.x.s a0, v8
+; RV32ZVE32F-NEXT:    sw t4, 0(a0)
+; RV32ZVE32F-NEXT:    sw t5, 4(a0)
+; RV32ZVE32F-NEXT:    andi a0, a1, 8
 ; RV32ZVE32F-NEXT:    beqz a0, .LBB46_4
 ; RV32ZVE32F-NEXT:  .LBB46_13: # %cond.store5
 ; RV32ZVE32F-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
-; RV32ZVE32F-NEXT:    vslidedown.vi v10, v8, 3
-; RV32ZVE32F-NEXT:    vmv.x.s a0, v10
-; RV32ZVE32F-NEXT:    sw t3, 0(a0)
-; RV32ZVE32F-NEXT:    sw t4, 4(a0)
-; RV32ZVE32F-NEXT:    andi a0, t0, 16
+; RV32ZVE32F-NEXT:    vslidedown.vi v8, v10, 3
+; RV32ZVE32F-NEXT:    vmv.x.s a0, v8
+; RV32ZVE32F-NEXT:    sw t2, 0(a0)
+; RV32ZVE32F-NEXT:    sw t3, 4(a0)
+; RV32ZVE32F-NEXT:    andi a0, a1, 16
 ; RV32ZVE32F-NEXT:    beqz a0, .LBB46_5
 ; RV32ZVE32F-NEXT:  .LBB46_14: # %cond.store7
 ; RV32ZVE32F-NEXT:    vsetivli zero, 1, e32, m2, ta, ma
-; RV32ZVE32F-NEXT:    vslidedown.vi v10, v8, 4
-; RV32ZVE32F-NEXT:    vmv.x.s a0, v10
-; RV32ZVE32F-NEXT:    sw t1, 0(a0)
-; RV32ZVE32F-NEXT:    sw t2, 4(a0)
-; RV32ZVE32F-NEXT:    andi a0, t0, 32
+; RV32ZVE32F-NEXT:    vslidedown.vi v8, v10, 4
+; RV32ZVE32F-NEXT:    vmv.x.s a0, v8
+; RV32ZVE32F-NEXT:    sw t0, 0(a0)
+; RV32ZVE32F-NEXT:    sw t1, 4(a0)
+; RV32ZVE32F-NEXT:    andi a0, a1, 32
 ; RV32ZVE32F-NEXT:    beqz a0, .LBB46_6
 ; RV32ZVE32F-NEXT:  .LBB46_15: # %cond.store9
 ; RV32ZVE32F-NEXT:    vsetivli zero, 1, e32, m2, ta, ma
-; RV32ZVE32F-NEXT:    vslidedown.vi v10, v8, 5
-; RV32ZVE32F-NEXT:    vmv.x.s a0, v10
+; RV32ZVE32F-NEXT:    vslidedown.vi v8, v10, 5
+; RV32ZVE32F-NEXT:    vmv.x.s a0, v8
 ; RV32ZVE32F-NEXT:    sw a6, 0(a0)
 ; RV32ZVE32F-NEXT:    sw a7, 4(a0)
-; RV32ZVE32F-NEXT:    andi a0, t0, 64
+; RV32ZVE32F-NEXT:    andi a0, a1, 64
 ; RV32ZVE32F-NEXT:    beqz a0, .LBB46_7
 ; RV32ZVE32F-NEXT:  .LBB46_16: # %cond.store11
 ; RV32ZVE32F-NEXT:    vsetivli zero, 1, e32, m2, ta, ma
-; RV32ZVE32F-NEXT:    vslidedown.vi v10, v8, 6
-; RV32ZVE32F-NEXT:    vmv.x.s a0, v10
+; RV32ZVE32F-NEXT:    vslidedown.vi v8, v10, 6
+; RV32ZVE32F-NEXT:    vmv.x.s a0, v8
 ; RV32ZVE32F-NEXT:    sw a4, 0(a0)
 ; RV32ZVE32F-NEXT:    sw a5, 4(a0)
-; RV32ZVE32F-NEXT:    andi a0, t0, -128
+; RV32ZVE32F-NEXT:    andi a0, a1, -128
 ; RV32ZVE32F-NEXT:    bnez a0, .LBB46_8
 ; RV32ZVE32F-NEXT:    j .LBB46_9
 ;
@@ -4644,20 +4649,20 @@ define void @mscatter_baseidx_sext_v8i16_v8i64(<8 x i64> %val, ptr %base, <8 x i
 define void @mscatter_baseidx_zext_v8i16_v8i64(<8 x i64> %val, ptr %base, <8 x i16> %idxs, <8 x i1> %m) {
 ; RV32V-LABEL: mscatter_baseidx_zext_v8i16_v8i64:
 ; RV32V:       # %bb.0:
-; RV32V-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
-; RV32V-NEXT:    vzext.vf2 v14, v12
-; RV32V-NEXT:    vsll.vi v12, v14, 3
+; RV32V-NEXT:    li a1, 8
+; RV32V-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; RV32V-NEXT:    vwmulu.vx v14, v12, a1
 ; RV32V-NEXT:    vsetvli zero, zero, e64, m4, ta, ma
-; RV32V-NEXT:    vsoxei32.v v8, (a0), v12, v0.t
+; RV32V-NEXT:    vsoxei32.v v8, (a0), v14, v0.t
 ; RV32V-NEXT:    ret
 ;
 ; RV64V-LABEL: mscatter_baseidx_zext_v8i16_v8i64:
 ; RV64V:       # %bb.0:
-; RV64V-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
-; RV64V-NEXT:    vzext.vf2 v14, v12
-; RV64V-NEXT:    vsll.vi v12, v14, 3
+; RV64V-NEXT:    li a1, 8
+; RV64V-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; RV64V-NEXT:    vwmulu.vx v14, v12, a1
 ; RV64V-NEXT:    vsetvli zero, zero, e64, m4, ta, ma
-; RV64V-NEXT:    vsoxei32.v v8, (a0), v12, v0.t
+; RV64V-NEXT:    vsoxei32.v v8, (a0), v14, v0.t
 ; RV64V-NEXT:    ret
 ;
 ; RV32ZVE32F-LABEL: mscatter_baseidx_zext_v8i16_v8i64:
@@ -4677,47 +4682,47 @@ define void @mscatter_baseidx_zext_v8i16_v8i64(<8 x i64> %val, ptr %base, <8 x i
 ; RV32ZVE32F-NEXT:    lw a7, 44(a0)
 ; RV32ZVE32F-NEXT:    lw a4, 48(a0)
 ; RV32ZVE32F-NEXT:    lw a5, 52(a0)
-; RV32ZVE32F-NEXT:    lw t3, 24(a0)
-; RV32ZVE32F-NEXT:    lw t4, 28(a0)
-; RV32ZVE32F-NEXT:    lw t1, 32(a0)
-; RV32ZVE32F-NEXT:    lw t2, 36(a0)
-; RV32ZVE32F-NEXT:    lw s0, 8(a0)
-; RV32ZVE32F-NEXT:    lw s1, 12(a0)
-; RV32ZVE32F-NEXT:    lw t5, 16(a0)
-; RV32ZVE32F-NEXT:    lw t6, 20(a0)
+; RV32ZVE32F-NEXT:    lw t2, 24(a0)
+; RV32ZVE32F-NEXT:    lw t3, 28(a0)
+; RV32ZVE32F-NEXT:    lw t0, 32(a0)
+; RV32ZVE32F-NEXT:    lw t1, 36(a0)
+; RV32ZVE32F-NEXT:    lw t6, 8(a0)
+; RV32ZVE32F-NEXT:    lw s0, 12(a0)
+; RV32ZVE32F-NEXT:    lw t4, 16(a0)
+; RV32ZVE32F-NEXT:    lw t5, 20(a0)
 ; RV32ZVE32F-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
-; RV32ZVE32F-NEXT:    vzext.vf2 v10, v8
-; RV32ZVE32F-NEXT:    vsll.vi v8, v10, 3
+; RV32ZVE32F-NEXT:    vmv.v.x v10, a1
+; RV32ZVE32F-NEXT:    li s1, 8
 ; RV32ZVE32F-NEXT:    vsetvli zero, zero, e8, mf2, ta, ma
-; RV32ZVE32F-NEXT:    vmv.x.s t0, v0
-; RV32ZVE32F-NEXT:    andi s2, t0, 1
-; RV32ZVE32F-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
-; RV32ZVE32F-NEXT:    vadd.vx v8, v8, a1
+; RV32ZVE32F-NEXT:    vmv.x.s a1, v0
+; RV32ZVE32F-NEXT:    andi s2, a1, 1
+; RV32ZVE32F-NEXT:    vsetvli zero, zero, e16, m1, ta, ma
+; RV32ZVE32F-NEXT:    vwmaccu.vx v10, s1, v8
 ; RV32ZVE32F-NEXT:    bnez s2, .LBB47_10
 ; RV32ZVE32F-NEXT:  # %bb.1: # %else
-; RV32ZVE32F-NEXT:    andi a0, t0, 2
+; RV32ZVE32F-NEXT:    andi a0, a1, 2
 ; RV32ZVE32F-NEXT:    bnez a0, .LBB47_11
 ; RV32ZVE32F-NEXT:  .LBB47_2: # %else2
-; RV32ZVE32F-NEXT:    andi a0, t0, 4
+; RV32ZVE32F-NEXT:    andi a0, a1, 4
 ; RV32ZVE32F-NEXT:    bnez a0, .LBB47_12
 ; RV32ZVE32F-NEXT:  .LBB47_3: # %else4
-; RV32ZVE32F-NEXT:    andi a0, t0, 8
+; RV32ZVE32F-NEXT:    andi a0, a1, 8
 ; RV32ZVE32F-NEXT:    bnez a0, .LBB47_13
 ; RV32ZVE32F-NEXT:  .LBB47_4: # %else6
-; RV32ZVE32F-NEXT:    andi a0, t0, 16
+; RV32ZVE32F-NEXT:    andi a0, a1, 16
 ; RV32ZVE32F-NEXT:    bnez a0, .LBB47_14
 ; RV32ZVE32F-NEXT:  .LBB47_5: # %else8
-; RV32ZVE32F-NEXT:    andi a0, t0, 32
+; RV32ZVE32F-NEXT:    andi a0, a1, 32
 ; RV32ZVE32F-NEXT:    bnez a0, .LBB47_15
 ; RV32ZVE32F-NEXT:  .LBB47_6: # %else10
-; RV32ZVE32F-NEXT:    andi a0, t0, 64
+; RV32ZVE32F-NEXT:    andi a0, a1, 64
 ; RV32ZVE32F-NEXT:    bnez a0, .LBB47_16
 ; RV32ZVE32F-NEXT:  .LBB47_7: # %else12
-; RV32ZVE32F-NEXT:    andi a0, t0, -128
+; RV32ZVE32F-NEXT:    andi a0, a1, -128
 ; RV32ZVE32F-NEXT:    beqz a0, .LBB47_9
 ; RV32ZVE32F-NEXT:  .LBB47_8: # %cond.store13
 ; RV32ZVE32F-NEXT:    vsetivli zero, 1, e32, m2, ta, ma
-; RV32ZVE32F-NEXT:    vslidedown.vi v8, v8, 7
+; RV32ZVE32F-NEXT:    vslidedown.vi v8, v10, 7
 ; RV32ZVE32F-NEXT:    vmv.x.s a0, v8
 ; RV32ZVE32F-NEXT:    sw a2, 0(a0)
 ; RV32ZVE32F-NEXT:    sw a3, 4(a0)
@@ -4733,66 +4738,67 @@ define void @mscatter_baseidx_zext_v8i16_v8i64(<8 x i64> %val, ptr %base, <8 x i
 ; RV32ZVE32F-NEXT:    ret
 ; RV32ZVE32F-NEXT:  .LBB47_10: # %cond.store
 ; RV32ZVE32F-NEXT:    .cfi_restore_state
-; RV32ZVE32F-NEXT:    lw a1, 0(a0)
+; RV32ZVE32F-NEXT:    lw s1, 0(a0)
 ; RV32ZVE32F-NEXT:    lw a0, 4(a0)
-; RV32ZVE32F-NEXT:    vmv.x.s s2, v8
-; RV32ZVE32F-NEXT:    sw a1, 0(s2)
+; RV32ZVE32F-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
+; RV32ZVE32F-NEXT:    vmv.x.s s2, v10
+; RV32ZVE32F-NEXT:    sw s1, 0(s2)
 ; RV32ZVE32F-NEXT:    sw a0, 4(s2)
-; RV32ZVE32F-NEXT:    andi a0, t0, 2
+; RV32ZVE32F-NEXT:    andi a0, a1, 2
 ; RV32ZVE32F-NEXT:    beqz a0, .LBB47_2
 ; RV32ZVE32F-NEXT:  .LBB47_11: # %cond.store1
 ; RV32ZVE32F-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
-; RV32ZVE32F-NEXT:    vslidedown.vi v10, v8, 1
-; RV32ZVE32F-NEXT:    vmv.x.s a0, v10
-; RV32ZVE32F-NEXT:    sw s0, 0(a0)
-; RV32ZVE32F-NEXT:    sw s1, 4(a0)
-; RV32ZVE32F-NEXT:    andi a0, t0, 4
+; RV32ZVE32F-NEXT:    vslidedown.vi v8, v10, 1
+; RV32ZVE32F-NEXT:    vmv.x.s a0, v8
+; RV32ZVE32F-NEXT:    sw t6, 0(a0)
+; RV32ZVE32F-NEXT:    sw s0, 4(a0)
+; RV32ZVE32F-NEXT:    andi a0, a1, 4
 ; RV32ZVE32F-NEXT:    beqz a0, .LBB47_3
 ; RV32ZVE32F-NEXT:  .LBB47_12: # %cond.store3
 ; RV32ZVE32F-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
-; RV32ZVE32F-NEXT:    vslidedown.vi v10, v8, 2
-; RV32ZVE32F-NEXT:    vmv.x.s a0, v10
-; RV32ZVE32F-NEXT:    sw t5, 0(a0)
-; RV32ZVE32F-NEXT:    sw t6, 4(a0)
-; RV32ZVE32F-NEXT:    andi a0, t0, 8
+; RV32ZVE32F-NEXT:    vslidedown.vi v8, v10, 2
+; RV32ZVE32F-NEXT:    vmv.x.s a0, v8
+; RV32ZVE32F-NEXT:    sw t4, 0(a0)
+; RV32ZVE32F-NEXT:    sw t5, 4(a0)
+; RV32ZVE32F-NEXT:    andi a0, a1, 8
 ; RV32ZVE32F-NEXT:    beqz a0, .LBB47_4
 ; RV32ZVE32F-NEXT:  .LBB47_13: # %cond.store5
 ; RV32ZVE32F-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
-; RV32ZVE32F-NEXT:    vslidedown.vi v10, v8, 3
-; RV32ZVE32F-NEXT:    vmv.x.s a0, v10
-; RV32ZVE32F-NEXT:    sw t3, 0(a0)
-; RV32ZVE32F-NEXT:    sw t4, 4(a0)
-; RV32ZVE32F-NEXT:    andi a0, t0, 16
+; RV32ZVE32F-NEXT:    vslidedown.vi v8, v10, 3
+; RV32ZVE32F-NEXT:    vmv.x.s a0, v8
+; RV32ZVE32F-NEXT:    sw t2, 0(a0)
+; RV32ZVE32F-NEXT:    sw t3, 4(a0)
+; RV32ZVE32F-NEXT:    andi a0, a1, 16
 ; RV32ZVE32F-NEXT:    beqz a0, .LBB47_5
 ; RV32ZVE32F-NEXT:  .LBB47_14: # %cond.store7
 ; RV32ZVE32F-NEXT:    vsetivli zero, 1, e32, m2, ta, ma
-; RV32ZVE32F-NEXT:    vslidedown.vi v10, v8, 4
-; RV32ZVE32F-NEXT:    vmv.x.s a0, v10
-; RV32ZVE32F-NEXT:    sw t1, 0(a0)
-; RV32ZVE32F-NEXT:    sw t2, 4(a0)
-; RV32ZVE32F-NEXT:    andi a0, t0, 32
+; RV32ZVE32F-NEXT:    vslidedown.vi v8, v10, 4
+; RV32ZVE32F-NEXT:    vmv.x.s a0, v8
+; RV32ZVE32F-NEXT:    sw t0, 0(a0)
+; RV32ZVE32F-NEXT:    sw t1, 4(a0)
+; RV32ZVE32F-NEXT:    andi a0, a1, 32
 ; RV32ZVE32F-NEXT:    beqz a0, .LBB47_6
 ; RV32ZVE32F-NEXT:  .LBB47_15: # %cond.store9
 ; RV32ZVE32F-NEXT:    vsetivli zero, 1, e32, m2, ta, ma
-; RV32ZVE32F-NEXT:    vslidedown.vi v10, v8, 5
-; RV32ZVE32F-NEXT:    vmv.x.s a0, v10
+; RV32ZVE32F-NEXT:    vslidedown.vi v8, v10, 5
+; RV32ZVE32F-NEXT:    vmv.x.s a0, v8
 ; RV32ZVE32F-NEXT:    sw a6, 0(a0)
 ; RV32ZVE32F-NEXT:    sw a7, 4(a0)
-; RV32ZVE32F-NEXT:    andi a0, t0, 64
+; RV32ZVE32F-NEXT:    andi a0, a1, 64
 ; RV32ZVE32F-NEXT:    beqz a0, .LBB47_7
 ; RV32ZVE32F-NEXT:  .LBB47_16: # %cond.store11
 ; RV32ZVE32F-NEXT:    vsetivli zero, 1, e32, m2, ta, ma
-; RV32ZVE32F-NEXT:    vslidedown.vi v10, v8, 6
-; RV32ZVE32F-NEXT:    vmv.x.s a0, v10
+; RV32ZVE32F-NEXT:    vslidedown.vi v8, v10, 6
+; RV32ZVE32F-NEXT:    vmv.x.s a0, v8
 ; RV32ZVE32F-NEXT:    sw a4, 0(a0)
 ; RV32ZVE32F-NEXT:    sw a5, 4(a0)
-; RV32ZVE32F-NEXT:    andi a0, t0, -128
+; RV32ZVE32F-NEXT:    andi a0, a1, -128
 ; RV32ZVE32F-NEXT:    bnez a0, .LBB47_8
 ; RV32ZVE32F-NEXT:    j .LBB47_9
 ;
 ; RV64ZVE32F-LABEL: mscatter_baseidx_zext_v8i16_v8i64:
 ; RV64ZVE32F:       # %bb.0:
-; RV64ZVE32F-NEXT:    ld a4, 40(a0)
+; RV64ZVE32F-NEXT:    ld a5, 40(a0)
 ; RV64ZVE32F-NEXT:    ld a3, 48(a0)
 ; RV64ZVE32F-NEXT:    ld a2, 56(a0)
 ; RV64ZVE32F-NEXT:    ld t1, 8(a0)
@@ -4800,8 +4806,8 @@ define void @mscatter_baseidx_zext_v8i16_v8i64(<8 x i64> %val, ptr %base, <8 x i
 ; RV64ZVE32F-NEXT:    ld a7, 24(a0)
 ; RV64ZVE32F-NEXT:    ld a6, 32(a0)
 ; RV64ZVE32F-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
-; RV64ZVE32F-NEXT:    vmv.x.s a5, v0
-; RV64ZVE32F-NEXT:    andi t2, a5, 1
+; RV64ZVE32F-NEXT:    vmv.x.s a4, v0
+; RV64ZVE32F-NEXT:    andi t2, a4, 1
 ; RV64ZVE32F-NEXT:    beqz t2, .LBB47_2
 ; RV64ZVE32F-NEXT:  # %bb.1: # %cond.store
 ; RV64ZVE32F-NEXT:    ld a0, 0(a0)
@@ -4812,7 +4818,7 @@ define void @mscatter_baseidx_zext_v8i16_v8i64(<8 x i64> %val, ptr %base, <8 x i
 ; RV64ZVE32F-NEXT:    add t2, a1, t2
 ; RV64ZVE32F-NEXT:    sd a0, 0(t2)
 ; RV64ZVE32F-NEXT:  .LBB47_2: # %else
-; RV64ZVE32F-NEXT:    andi a0, a5, 2
+; RV64ZVE32F-NEXT:    andi a0, a4, 2
 ; RV64ZVE32F-NEXT:    beqz a0, .LBB47_4
 ; RV64ZVE32F-NEXT:  # %bb.3: # %cond.store1
 ; RV64ZVE32F-NEXT:    vsetivli zero, 1, e16, mf2, ta, ma
@@ -4825,18 +4831,18 @@ define void @mscatter_baseidx_zext_v8i16_v8i64(<8 x i64> %val, ptr %base, <8 x i
 ; RV64ZVE32F-NEXT:  .LBB47_4: # %else2
 ; RV64ZVE32F-NEXT:    vsetivli zero, 4, e16, m1, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v9, v8, 4
-; RV64ZVE32F-NEXT:    andi a0, a5, 4
+; RV64ZVE32F-NEXT:    andi a0, a4, 4
 ; RV64ZVE32F-NEXT:    vsetivli zero, 2, e16, mf2, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v8, v8, 2
 ; RV64ZVE32F-NEXT:    bnez a0, .LBB47_12
 ; RV64ZVE32F-NEXT:  # %bb.5: # %else4
-; RV64ZVE32F-NEXT:    andi a0, a5, 8
+; RV64ZVE32F-NEXT:    andi a0, a4, 8
 ; RV64ZVE32F-NEXT:    bnez a0, .LBB47_13
 ; RV64ZVE32F-NEXT:  .LBB47_6: # %else6
-; RV64ZVE32F-NEXT:    andi a0, a5, 16
+; RV64ZVE32F-NEXT:    andi a0, a4, 16
 ; RV64ZVE32F-NEXT:    bnez a0, .LBB47_14
 ; RV64ZVE32F-NEXT:  .LBB47_7: # %else8
-; RV64ZVE32F-NEXT:    andi a0, a5, 32
+; RV64ZVE32F-NEXT:    andi a0, a4, 32
 ; RV64ZVE32F-NEXT:    beqz a0, .LBB47_9
 ; RV64ZVE32F-NEXT:  .LBB47_8: # %cond.store9
 ; RV64ZVE32F-NEXT:    vslidedown.vi v8, v9, 1
@@ -4844,13 +4850,13 @@ define void @mscatter_baseidx_zext_v8i16_v8i64(<8 x i64> %val, ptr %base, <8 x i
 ; RV64ZVE32F-NEXT:    slli a0, a0, 48
 ; RV64ZVE32F-NEXT:    srli a0, a0, 45
 ; RV64ZVE32F-NEXT:    add a0, a1, a0
-; RV64ZVE32F-NEXT:    sd a4, 0(a0)
+; RV64ZVE32F-NEXT:    sd a5, 0(a0)
 ; RV64ZVE32F-NEXT:  .LBB47_9: # %else10
-; RV64ZVE32F-NEXT:    andi a0, a5, 64
+; RV64ZVE32F-NEXT:    andi a0, a4, 64
 ; RV64ZVE32F-NEXT:    vslidedown.vi v8, v9, 2
 ; RV64ZVE32F-NEXT:    bnez a0, .LBB47_15
 ; RV64ZVE32F-NEXT:  # %bb.10: # %else12
-; RV64ZVE32F-NEXT:    andi a0, a5, -128
+; RV64ZVE32F-NEXT:    andi a0, a4, -128
 ; RV64ZVE32F-NEXT:    bnez a0, .LBB47_16
 ; RV64ZVE32F-NEXT:  .LBB47_11: # %else14
 ; RV64ZVE32F-NEXT:    ret
@@ -4860,7 +4866,7 @@ define void @mscatter_baseidx_zext_v8i16_v8i64(<8 x i64> %val, ptr %base, <8 x i
 ; RV64ZVE32F-NEXT:    srli a0, a0, 45
 ; RV64ZVE32F-NEXT:    add a0, a1, a0
 ; RV64ZVE32F-NEXT:    sd t0, 0(a0)
-; RV64ZVE32F-NEXT:    andi a0, a5, 8
+; RV64ZVE32F-NEXT:    andi a0, a4, 8
 ; RV64ZVE32F-NEXT:    beqz a0, .LBB47_6
 ; RV64ZVE32F-NEXT:  .LBB47_13: # %cond.store5
 ; RV64ZVE32F-NEXT:    vslidedown.vi v8, v8, 1
@@ -4869,7 +4875,7 @@ define void @mscatter_baseidx_zext_v8i16_v8i64(<8 x i64> %val, ptr %base, <8 x i
 ; RV64ZVE32F-NEXT:    srli a0, a0, 45
 ; RV64ZVE32F-NEXT:    add a0, a1, a0
 ; RV64ZVE32F-NEXT:    sd a7, 0(a0)
-; RV64ZVE32F-NEXT:    andi a0, a5, 16
+; RV64ZVE32F-NEXT:    andi a0, a4, 16
 ; RV64ZVE32F-NEXT:    beqz a0, .LBB47_7
 ; RV64ZVE32F-NEXT:  .LBB47_14: # %cond.store7
 ; RV64ZVE32F-NEXT:    vmv.x.s a0, v9
@@ -4877,7 +4883,7 @@ define void @mscatter_baseidx_zext_v8i16_v8i64(<8 x i64> %val, ptr %base, <8 x i
 ; RV64ZVE32F-NEXT:    srli a0, a0, 45
 ; RV64ZVE32F-NEXT:    add a0, a1, a0
 ; RV64ZVE32F-NEXT:    sd a6, 0(a0)
-; RV64ZVE32F-NEXT:    andi a0, a5, 32
+; RV64ZVE32F-NEXT:    andi a0, a4, 32
 ; RV64ZVE32F-NEXT:    bnez a0, .LBB47_8
 ; RV64ZVE32F-NEXT:    j .LBB47_9
 ; RV64ZVE32F-NEXT:  .LBB47_15: # %cond.store11
@@ -4886,7 +4892,7 @@ define void @mscatter_baseidx_zext_v8i16_v8i64(<8 x i64> %val, ptr %base, <8 x i
 ; RV64ZVE32F-NEXT:    srli a0, a0, 45
 ; RV64ZVE32F-NEXT:    add a0, a1, a0
 ; RV64ZVE32F-NEXT:    sd a3, 0(a0)
-; RV64ZVE32F-NEXT:    andi a0, a5, -128
+; RV64ZVE32F-NEXT:    andi a0, a4, -128
 ; RV64ZVE32F-NEXT:    beqz a0, .LBB47_11
 ; RV64ZVE32F-NEXT:  .LBB47_16: # %cond.store13
 ; RV64ZVE32F-NEXT:    vslidedown.vi v8, v8, 1
@@ -4913,10 +4919,11 @@ define void @mscatter_baseidx_v8i32_v8i64(<8 x i64> %val, ptr %base, <8 x i32> %
 ;
 ; RV64V-LABEL: mscatter_baseidx_v8i32_v8i64:
 ; RV64V:       # %bb.0:
-; RV64V-NEXT:    vsetivli zero, 8, e64, m4, ta, ma
-; RV64V-NEXT:    vsext.vf2 v16, v12
-; RV64V-NEXT:    vsll.vi v12, v16, 3
-; RV64V-NEXT:    vsoxei64.v v8, (a0), v12, v0.t
+; RV64V-NEXT:    li a1, 8
+; RV64V-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
+; RV64V-NEXT:    vwmulsu.vx v16, v12, a1
+; RV64V-NEXT:    vsetvli zero, zero, e64, m4, ta, ma
+; RV64V-NEXT:    vsoxei64.v v8, (a0), v16, v0.t
 ; RV64V-NEXT:    ret
 ;
 ; RV32ZVE32F-LABEL: mscatter_baseidx_v8i32_v8i64:
@@ -5162,10 +5169,11 @@ define void @mscatter_baseidx_sext_v8i32_v8i64(<8 x i64> %val, ptr %base, <8 x i
 ;
 ; RV64V-LABEL: mscatter_baseidx_sext_v8i32_v8i64:
 ; RV64V:       # %bb.0:
-; RV64V-NEXT:    vsetivli zero, 8, e64, m4, ta, ma
-; RV64V-NEXT:    vsext.vf2 v16, v12
-; RV64V-NEXT:    vsll.vi v12, v16, 3
-; RV64V-NEXT:    vsoxei64.v v8, (a0), v12, v0.t
+; RV64V-NEXT:    li a1, 8
+; RV64V-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
+; RV64V-NEXT:    vwmulsu.vx v16, v12, a1
+; RV64V-NEXT:    vsetvli zero, zero, e64, m4, ta, ma
+; RV64V-NEXT:    vsoxei64.v v8, (a0), v16, v0.t
 ; RV64V-NEXT:    ret
 ;
 ; RV32ZVE32F-LABEL: mscatter_baseidx_sext_v8i32_v8i64:
@@ -5412,10 +5420,11 @@ define void @mscatter_baseidx_zext_v8i32_v8i64(<8 x i64> %val, ptr %base, <8 x i
 ;
 ; RV64V-LABEL: mscatter_baseidx_zext_v8i32_v8i64:
 ; RV64V:       # %bb.0:
-; RV64V-NEXT:    vsetivli zero, 8, e64, m4, ta, ma
-; RV64V-NEXT:    vzext.vf2 v16, v12
-; RV64V-NEXT:    vsll.vi v12, v16, 3
-; RV64V-NEXT:    vsoxei64.v v8, (a0), v12, v0.t
+; RV64V-NEXT:    li a1, 8
+; RV64V-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
+; RV64V-NEXT:    vwmulu.vx v16, v12, a1
+; RV64V-NEXT:    vsetvli zero, zero, e64, m4, ta, ma
+; RV64V-NEXT:    vsoxei64.v v8, (a0), v16, v0.t
 ; RV64V-NEXT:    ret
 ;
 ; RV32ZVE32F-LABEL: mscatter_baseidx_zext_v8i32_v8i64:
@@ -9019,20 +9028,20 @@ define void @mscatter_baseidx_sext_v8i8_v8f32(<8 x float> %val, ptr %base, <8 x
 define void @mscatter_baseidx_zext_v8i8_v8f32(<8 x float> %val, ptr %base, <8 x i8> %idxs, <8 x i1> %m) {
 ; RV32-LABEL: mscatter_baseidx_zext_v8i8_v8f32:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
-; RV32-NEXT:    vzext.vf2 v11, v10
-; RV32-NEXT:    vsll.vi v10, v11, 2
+; RV32-NEXT:    li a1, 4
+; RV32-NEXT:    vsetivli zero, 8, e8, mf2, ta, ma
+; RV32-NEXT:    vwmulu.vx v11, v10, a1
 ; RV32-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
-; RV32-NEXT:    vsoxei16.v v8, (a0), v10, v0.t
+; RV32-NEXT:    vsoxei16.v v8, (a0), v11, v0.t
 ; RV32-NEXT:    ret
 ;
 ; RV64V-LABEL: mscatter_baseidx_zext_v8i8_v8f32:
 ; RV64V:       # %bb.0:
-; RV64V-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
-; RV64V-NEXT:    vzext.vf2 v11, v10
-; RV64V-NEXT:    vsll.vi v10, v11, 2
+; RV64V-NEXT:    li a1, 4
+; RV64V-NEXT:    vsetivli zero, 8, e8, mf2, ta, ma
+; RV64V-NEXT:    vwmulu.vx v11, v10, a1
 ; RV64V-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
-; RV64V-NEXT:    vsoxei16.v v8, (a0), v10, v0.t
+; RV64V-NEXT:    vsoxei16.v v8, (a0), v11, v0.t
 ; RV64V-NEXT:    ret
 ;
 ; RV64ZVE32F-LABEL: mscatter_baseidx_zext_v8i8_v8f32:
@@ -9166,10 +9175,11 @@ define void @mscatter_baseidx_zext_v8i8_v8f32(<8 x float> %val, ptr %base, <8 x
 define void @mscatter_baseidx_v8i16_v8f32(<8 x float> %val, ptr %base, <8 x i16> %idxs, <8 x i1> %m) {
 ; RV32-LABEL: mscatter_baseidx_v8i16_v8f32:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
-; RV32-NEXT:    vsext.vf2 v12, v10
-; RV32-NEXT:    vsll.vi v10, v12, 2
-; RV32-NEXT:    vsoxei32.v v8, (a0), v10, v0.t
+; RV32-NEXT:    li a1, 4
+; RV32-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; RV32-NEXT:    vwmulsu.vx v12, v10, a1
+; RV32-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
+; RV32-NEXT:    vsoxei32.v v8, (a0), v12, v0.t
 ; RV32-NEXT:    ret
 ;
 ; RV64V-LABEL: mscatter_baseidx_v8i16_v8f32:
@@ -9304,10 +9314,11 @@ define void @mscatter_baseidx_v8i16_v8f32(<8 x float> %val, ptr %base, <8 x i16>
 define void @mscatter_baseidx_sext_v8i16_v8f32(<8 x float> %val, ptr %base, <8 x i16> %idxs, <8 x i1> %m) {
 ; RV32-LABEL: mscatter_baseidx_sext_v8i16_v8f32:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
-; RV32-NEXT:    vsext.vf2 v12, v10
-; RV32-NEXT:    vsll.vi v10, v12, 2
-; RV32-NEXT:    vsoxei32.v v8, (a0), v10, v0.t
+; RV32-NEXT:    li a1, 4
+; RV32-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; RV32-NEXT:    vwmulsu.vx v12, v10, a1
+; RV32-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
+; RV32-NEXT:    vsoxei32.v v8, (a0), v12, v0.t
 ; RV32-NEXT:    ret
 ;
 ; RV64V-LABEL: mscatter_baseidx_sext_v8i16_v8f32:
@@ -9443,18 +9454,20 @@ define void @mscatter_baseidx_sext_v8i16_v8f32(<8 x float> %val, ptr %base, <8 x
 define void @mscatter_baseidx_zext_v8i16_v8f32(<8 x float> %val, ptr %base, <8 x i16> %idxs, <8 x i1> %m) {
 ; RV32-LABEL: mscatter_baseidx_zext_v8i16_v8f32:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
-; RV32-NEXT:    vzext.vf2 v12, v10
-; RV32-NEXT:    vsll.vi v10, v12, 2
-; RV32-NEXT:    vsoxei32.v v8, (a0), v10, v0.t
+; RV32-NEXT:    li a1, 4
+; RV32-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; RV32-NEXT:    vwmulu.vx v12, v10, a1
+; RV32-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
+; RV32-NEXT:    vsoxei32.v v8, (a0), v12, v0.t
 ; RV32-NEXT:    ret
 ;
 ; RV64V-LABEL: mscatter_baseidx_zext_v8i16_v8f32:
 ; RV64V:       # %bb.0:
-; RV64V-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
-; RV64V-NEXT:    vzext.vf2 v12, v10
-; RV64V-NEXT:    vsll.vi v10, v12, 2
-; RV64V-NEXT:    vsoxei32.v v8, (a0), v10, v0.t
+; RV64V-NEXT:    li a1, 4
+; RV64V-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; RV64V-NEXT:    vwmulu.vx v12, v10, a1
+; RV64V-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
+; RV64V-NEXT:    vsoxei32.v v8, (a0), v12, v0.t
 ; RV64V-NEXT:    ret
 ;
 ; RV64ZVE32F-LABEL: mscatter_baseidx_zext_v8i16_v8f32:
@@ -9596,10 +9609,9 @@ define void @mscatter_baseidx_v8f32(<8 x float> %val, ptr %base, <8 x i32> %idxs
 ;
 ; RV64V-LABEL: mscatter_baseidx_v8f32:
 ; RV64V:       # %bb.0:
-; RV64V-NEXT:    vsetivli zero, 8, e64, m4, ta, ma
-; RV64V-NEXT:    vsext.vf2 v12, v10
-; RV64V-NEXT:    vsll.vi v12, v12, 2
-; RV64V-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
+; RV64V-NEXT:    li a1, 4
+; RV64V-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
+; RV64V-NEXT:    vwmulsu.vx v12, v10, a1
 ; RV64V-NEXT:    vsoxei64.v v8, (a0), v12, v0.t
 ; RV64V-NEXT:    ret
 ;
@@ -10545,20 +10557,20 @@ define void @mscatter_baseidx_sext_v8i8_v8f64(<8 x double> %val, ptr %base, <8 x
 define void @mscatter_baseidx_zext_v8i8_v8f64(<8 x double> %val, ptr %base, <8 x i8> %idxs, <8 x i1> %m) {
 ; RV32V-LABEL: mscatter_baseidx_zext_v8i8_v8f64:
 ; RV32V:       # %bb.0:
-; RV32V-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
-; RV32V-NEXT:    vzext.vf2 v13, v12
-; RV32V-NEXT:    vsll.vi v12, v13, 3
+; RV32V-NEXT:    li a1, 8
+; RV32V-NEXT:    vsetivli zero, 8, e8, mf2, ta, ma
+; RV32V-NEXT:    vwmulu.vx v13, v12, a1
 ; RV32V-NEXT:    vsetvli zero, zero, e64, m4, ta, ma
-; RV32V-NEXT:    vsoxei16.v v8, (a0), v12, v0.t
+; RV32V-NEXT:    vsoxei16.v v8, (a0), v13, v0.t
 ; RV32V-NEXT:    ret
 ;
 ; RV64V-LABEL: mscatter_baseidx_zext_v8i8_v8f64:
 ; RV64V:       # %bb.0:
-; RV64V-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
-; RV64V-NEXT:    vzext.vf2 v13, v12
-; RV64V-NEXT:    vsll.vi v12, v13, 3
+; RV64V-NEXT:    li a1, 8
+; RV64V-NEXT:    vsetivli zero, 8, e8, mf2, ta, ma
+; RV64V-NEXT:    vwmulu.vx v13, v12, a1
 ; RV64V-NEXT:    vsetvli zero, zero, e64, m4, ta, ma
-; RV64V-NEXT:    vsoxei16.v v8, (a0), v12, v0.t
+; RV64V-NEXT:    vsoxei16.v v8, (a0), v13, v0.t
 ; RV64V-NEXT:    ret
 ;
 ; RV32ZVE32F-LABEL: mscatter_baseidx_zext_v8i8_v8f64:
@@ -10755,11 +10767,11 @@ define void @mscatter_baseidx_zext_v8i8_v8f64(<8 x double> %val, ptr %base, <8 x
 define void @mscatter_baseidx_v8i16_v8f64(<8 x double> %val, ptr %base, <8 x i16> %idxs, <8 x i1> %m) {
 ; RV32V-LABEL: mscatter_baseidx_v8i16_v8f64:
 ; RV32V:       # %bb.0:
-; RV32V-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
-; RV32V-NEXT:    vsext.vf2 v14, v12
-; RV32V-NEXT:    vsll.vi v12, v14, 3
+; RV32V-NEXT:    li a1, 8
+; RV32V-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; RV32V-NEXT:    vwmulsu.vx v14, v12, a1
 ; RV32V-NEXT:    vsetvli zero, zero, e64, m4, ta, ma
-; RV32V-NEXT:    vsoxei32.v v8, (a0), v12, v0.t
+; RV32V-NEXT:    vsoxei32.v v8, (a0), v14, v0.t
 ; RV32V-NEXT:    ret
 ;
 ; RV64V-LABEL: mscatter_baseidx_v8i16_v8f64:
@@ -10773,87 +10785,88 @@ define void @mscatter_baseidx_v8i16_v8f64(<8 x double> %val, ptr %base, <8 x i16
 ; RV32ZVE32F-LABEL: mscatter_baseidx_v8i16_v8f64:
 ; RV32ZVE32F:       # %bb.0:
 ; RV32ZVE32F-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
-; RV32ZVE32F-NEXT:    vsext.vf2 v10, v8
+; RV32ZVE32F-NEXT:    vmv.v.x v10, a0
+; RV32ZVE32F-NEXT:    li a1, 8
 ; RV32ZVE32F-NEXT:    vsetvli zero, zero, e8, mf2, ta, ma
-; RV32ZVE32F-NEXT:    vmv.x.s a1, v0
-; RV32ZVE32F-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
-; RV32ZVE32F-NEXT:    vsll.vi v8, v10, 3
-; RV32ZVE32F-NEXT:    andi a2, a1, 1
-; RV32ZVE32F-NEXT:    vadd.vx v8, v8, a0
+; RV32ZVE32F-NEXT:    vmv.x.s a0, v0
+; RV32ZVE32F-NEXT:    andi a2, a0, 1
+; RV32ZVE32F-NEXT:    vsetvli zero, zero, e16, m1, ta, ma
+; RV32ZVE32F-NEXT:    vwmaccus.vx v10, a1, v8
 ; RV32ZVE32F-NEXT:    bnez a2, .LBB94_9
 ; RV32ZVE32F-NEXT:  # %bb.1: # %else
-; RV32ZVE32F-NEXT:    andi a0, a1, 2
-; RV32ZVE32F-NEXT:    bnez a0, .LBB94_10
+; RV32ZVE32F-NEXT:    andi a1, a0, 2
+; RV32ZVE32F-NEXT:    bnez a1, .LBB94_10
 ; RV32ZVE32F-NEXT:  .LBB94_2: # %else2
-; RV32ZVE32F-NEXT:    andi a0, a1, 4
-; RV32ZVE32F-NEXT:    bnez a0, .LBB94_11
+; RV32ZVE32F-NEXT:    andi a1, a0, 4
+; RV32ZVE32F-NEXT:    bnez a1, .LBB94_11
 ; RV32ZVE32F-NEXT:  .LBB94_3: # %else4
-; RV32ZVE32F-NEXT:    andi a0, a1, 8
-; RV32ZVE32F-NEXT:    bnez a0, .LBB94_12
+; RV32ZVE32F-NEXT:    andi a1, a0, 8
+; RV32ZVE32F-NEXT:    bnez a1, .LBB94_12
 ; RV32ZVE32F-NEXT:  .LBB94_4: # %else6
-; RV32ZVE32F-NEXT:    andi a0, a1, 16
-; RV32ZVE32F-NEXT:    bnez a0, .LBB94_13
+; RV32ZVE32F-NEXT:    andi a1, a0, 16
+; RV32ZVE32F-NEXT:    bnez a1, .LBB94_13
 ; RV32ZVE32F-NEXT:  .LBB94_5: # %else8
-; RV32ZVE32F-NEXT:    andi a0, a1, 32
-; RV32ZVE32F-NEXT:    bnez a0, .LBB94_14
+; RV32ZVE32F-NEXT:    andi a1, a0, 32
+; RV32ZVE32F-NEXT:    bnez a1, .LBB94_14
 ; RV32ZVE32F-NEXT:  .LBB94_6: # %else10
-; RV32ZVE32F-NEXT:    andi a0, a1, 64
-; RV32ZVE32F-NEXT:    bnez a0, .LBB94_15
+; RV32ZVE32F-NEXT:    andi a1, a0, 64
+; RV32ZVE32F-NEXT:    bnez a1, .LBB94_15
 ; RV32ZVE32F-NEXT:  .LBB94_7: # %else12
-; RV32ZVE32F-NEXT:    andi a0, a1, -128
+; RV32ZVE32F-NEXT:    andi a0, a0, -128
 ; RV32ZVE32F-NEXT:    bnez a0, .LBB94_16
 ; RV32ZVE32F-NEXT:  .LBB94_8: # %else14
 ; RV32ZVE32F-NEXT:    ret
 ; RV32ZVE32F-NEXT:  .LBB94_9: # %cond.store
-; RV32ZVE32F-NEXT:    vmv.x.s a0, v8
-; RV32ZVE32F-NEXT:    fsd fa0, 0(a0)
-; RV32ZVE32F-NEXT:    andi a0, a1, 2
-; RV32ZVE32F-NEXT:    beqz a0, .LBB94_2
+; RV32ZVE32F-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
+; RV32ZVE32F-NEXT:    vmv.x.s a1, v10
+; RV32ZVE32F-NEXT:    fsd fa0, 0(a1)
+; RV32ZVE32F-NEXT:    andi a1, a0, 2
+; RV32ZVE32F-NEXT:    beqz a1, .LBB94_2
 ; RV32ZVE32F-NEXT:  .LBB94_10: # %cond.store1
 ; RV32ZVE32F-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
-; RV32ZVE32F-NEXT:    vslidedown.vi v10, v8, 1
-; RV32ZVE32F-NEXT:    vmv.x.s a0, v10
-; RV32ZVE32F-NEXT:    fsd fa1, 0(a0)
-; RV32ZVE32F-NEXT:    andi a0, a1, 4
-; RV32ZVE32F-NEXT:    beqz a0, .LBB94_3
+; RV32ZVE32F-NEXT:    vslidedown.vi v8, v10, 1
+; RV32ZVE32F-NEXT:    vmv.x.s a1, v8
+; RV32ZVE32F-NEXT:    fsd fa1, 0(a1)
+; RV32ZVE32F-NEXT:    andi a1, a0, 4
+; RV32ZVE32F-NEXT:    beqz a1, .LBB94_3
 ; RV32ZVE32F-NEXT:  .LBB94_11: # %cond.store3
 ; RV32ZVE32F-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
-; RV32ZVE32F-NEXT:    vslidedown.vi v10, v8, 2
-; RV32ZVE32F-NEXT:    vmv.x.s a0, v10
-; RV32ZVE32F-NEXT:    fsd fa2, 0(a0)
-; RV32ZVE32F-NEXT:    andi a0, a1, 8
-; RV32ZVE32F-NEXT:    beqz a0, .LBB94_4
+; RV32ZVE32F-NEXT:    vslidedown.vi v8, v10, 2
+; RV32ZVE32F-NEXT:    vmv.x.s a1, v8
+; RV32ZVE32F-NEXT:    fsd fa2, 0(a1)
+; RV32ZVE32F-NEXT:    andi a1, a0, 8
+; RV32ZVE32F-NEXT:    beqz a1, .LBB94_4
 ; RV32ZVE32F-NEXT:  .LBB94_12: # %cond.store5
 ; RV32ZVE32F-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
-; RV32ZVE32F-NEXT:    vslidedown.vi v10, v8, 3
-; RV32ZVE32F-NEXT:    vmv.x.s a0, v10
-; RV32ZVE32F-NEXT:    fsd fa3, 0(a0)
-; RV32ZVE32F-NEXT:    andi a0, a1, 16
-; RV32ZVE32F-NEXT:    beqz a0, .LBB94_5
+; RV32ZVE32F-NEXT:    vslidedown.vi v8, v10, 3
+; RV32ZVE32F-NEXT:    vmv.x.s a1, v8
+; RV32ZVE32F-NEXT:    fsd fa3, 0(a1)
+; RV32ZVE32F-NEXT:    andi a1, a0, 16
+; RV32ZVE32F-NEXT:    beqz a1, .LBB94_5
 ; RV32ZVE32F-NEXT:  .LBB94_13: # %cond.store7
 ; RV32ZVE32F-NEXT:    vsetivli zero, 1, e32, m2, ta, ma
-; RV32ZVE32F-NEXT:    vslidedown.vi v10, v8, 4
-; RV32ZVE32F-NEXT:    vmv.x.s a0, v10
-; RV32ZVE32F-NEXT:    fsd fa4, 0(a0)
-; RV32ZVE32F-NEXT:    andi a0, a1, 32
-; RV32ZVE32F-NEXT:    beqz a0, .LBB94_6
+; RV32ZVE32F-NEXT:    vslidedown.vi v8, v10, 4
+; RV32ZVE32F-NEXT:    vmv.x.s a1, v8
+; RV32ZVE32F-NEXT:    fsd fa4, 0(a1)
+; RV32ZVE32F-NEXT:    andi a1, a0, 32
+; RV32ZVE32F-NEXT:    beqz a1, .LBB94_6
 ; RV32ZVE32F-NEXT:  .LBB94_14: # %cond.store9
 ; RV32ZVE32F-NEXT:    vsetivli zero, 1, e32, m2, ta, ma
-; RV32ZVE32F-NEXT:    vslidedown.vi v10, v8, 5
-; RV32ZVE32F-NEXT:    vmv.x.s a0, v10
-; RV32ZVE32F-NEXT:    fsd fa5, 0(a0)
-; RV32ZVE32F-NEXT:    andi a0, a1, 64
-; RV32ZVE32F-NEXT:    beqz a0, .LBB94_7
+; RV32ZVE32F-NEXT:    vslidedown.vi v8, v10, 5
+; RV32ZVE32F-NEXT:    vmv.x.s a1, v8
+; RV32ZVE32F-NEXT:    fsd fa5, 0(a1)
+; RV32ZVE32F-NEXT:    andi a1, a0, 64
+; RV32ZVE32F-NEXT:    beqz a1, .LBB94_7
 ; RV32ZVE32F-NEXT:  .LBB94_15: # %cond.store11
 ; RV32ZVE32F-NEXT:    vsetivli zero, 1, e32, m2, ta, ma
-; RV32ZVE32F-NEXT:    vslidedown.vi v10, v8, 6
-; RV32ZVE32F-NEXT:    vmv.x.s a0, v10
-; RV32ZVE32F-NEXT:    fsd fa6, 0(a0)
-; RV32ZVE32F-NEXT:    andi a0, a1, -128
+; RV32ZVE32F-NEXT:    vslidedown.vi v8, v10, 6
+; RV32ZVE32F-NEXT:    vmv.x.s a1, v8
+; RV32ZVE32F-NEXT:    fsd fa6, 0(a1)
+; RV32ZVE32F-NEXT:    andi a0, a0, -128
 ; RV32ZVE32F-NEXT:    beqz a0, .LBB94_8
 ; RV32ZVE32F-NEXT:  .LBB94_16: # %cond.store13
 ; RV32ZVE32F-NEXT:    vsetivli zero, 1, e32, m2, ta, ma
-; RV32ZVE32F-NEXT:    vslidedown.vi v8, v8, 7
+; RV32ZVE32F-NEXT:    vslidedown.vi v8, v10, 7
 ; RV32ZVE32F-NEXT:    vmv.x.s a0, v8
 ; RV32ZVE32F-NEXT:    fsd fa7, 0(a0)
 ; RV32ZVE32F-NEXT:    ret
@@ -10956,11 +10969,11 @@ define void @mscatter_baseidx_v8i16_v8f64(<8 x double> %val, ptr %base, <8 x i16
 define void @mscatter_baseidx_sext_v8i16_v8f64(<8 x double> %val, ptr %base, <8 x i16> %idxs, <8 x i1> %m) {
 ; RV32V-LABEL: mscatter_baseidx_sext_v8i16_v8f64:
 ; RV32V:       # %bb.0:
-; RV32V-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
-; RV32V-NEXT:    vsext.vf2 v14, v12
-; RV32V-NEXT:    vsll.vi v12, v14, 3
+; RV32V-NEXT:    li a1, 8
+; RV32V-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; RV32V-NEXT:    vwmulsu.vx v14, v12, a1
 ; RV32V-NEXT:    vsetvli zero, zero, e64, m4, ta, ma
-; RV32V-NEXT:    vsoxei32.v v8, (a0), v12, v0.t
+; RV32V-NEXT:    vsoxei32.v v8, (a0), v14, v0.t
 ; RV32V-NEXT:    ret
 ;
 ; RV64V-LABEL: mscatter_baseidx_sext_v8i16_v8f64:
@@ -10974,87 +10987,88 @@ define void @mscatter_baseidx_sext_v8i16_v8f64(<8 x double> %val, ptr %base, <8
 ; RV32ZVE32F-LABEL: mscatter_baseidx_sext_v8i16_v8f64:
 ; RV32ZVE32F:       # %bb.0:
 ; RV32ZVE32F-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
-; RV32ZVE32F-NEXT:    vsext.vf2 v10, v8
+; RV32ZVE32F-NEXT:    vmv.v.x v10, a0
+; RV32ZVE32F-NEXT:    li a1, 8
 ; RV32ZVE32F-NEXT:    vsetvli zero, zero, e8, mf2, ta, ma
-; RV32ZVE32F-NEXT:    vmv.x.s a1, v0
-; RV32ZVE32F-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
-; RV32ZVE32F-NEXT:    vsll.vi v8, v10, 3
-; RV32ZVE32F-NEXT:    andi a2, a1, 1
-; RV32ZVE32F-NEXT:    vadd.vx v8, v8, a0
+; RV32ZVE32F-NEXT:    vmv.x.s a0, v0
+; RV32ZVE32F-NEXT:    andi a2, a0, 1
+; RV32ZVE32F-NEXT:    vsetvli zero, zero, e16, m1, ta, ma
+; RV32ZVE32F-NEXT:    vwmaccus.vx v10, a1, v8
 ; RV32ZVE32F-NEXT:    bnez a2, .LBB95_9
 ; RV32ZVE32F-NEXT:  # %bb.1: # %else
-; RV32ZVE32F-NEXT:    andi a0, a1, 2
-; RV32ZVE32F-NEXT:    bnez a0, .LBB95_10
+; RV32ZVE32F-NEXT:    andi a1, a0, 2
+; RV32ZVE32F-NEXT:    bnez a1, .LBB95_10
 ; RV32ZVE32F-NEXT:  .LBB95_2: # %else2
-; RV32ZVE32F-NEXT:    andi a0, a1, 4
-; RV32ZVE32F-NEXT:    bnez a0, .LBB95_11
+; RV32ZVE32F-NEXT:    andi a1, a0, 4
+; RV32ZVE32F-NEXT:    bnez a1, .LBB95_11
 ; RV32ZVE32F-NEXT:  .LBB95_3: # %else4
-; RV32ZVE32F-NEXT:    andi a0, a1, 8
-; RV32ZVE32F-NEXT:    bnez a0, .LBB95_12
+; RV32ZVE32F-NEXT:    andi a1, a0, 8
+; RV32ZVE32F-NEXT:    bnez a1, .LBB95_12
 ; RV32ZVE32F-NEXT:  .LBB95_4: # %else6
-; RV32ZVE32F-NEXT:    andi a0, a1, 16
-; RV32ZVE32F-NEXT:    bnez a0, .LBB95_13
+; RV32ZVE32F-NEXT:    andi a1, a0, 16
+; RV32ZVE32F-NEXT:    bnez a1, .LBB95_13
 ; RV32ZVE32F-NEXT:  .LBB95_5: # %else8
-; RV32ZVE32F-NEXT:    andi a0, a1, 32
-; RV32ZVE32F-NEXT:    bnez a0, .LBB95_14
+; RV32ZVE32F-NEXT:    andi a1, a0, 32
+; RV32ZVE32F-NEXT:    bnez a1, .LBB95_14
 ; RV32ZVE32F-NEXT:  .LBB95_6: # %else10
-; RV32ZVE32F-NEXT:    andi a0, a1, 64
-; RV32ZVE32F-NEXT:    bnez a0, .LBB95_15
+; RV32ZVE32F-NEXT:    andi a1, a0, 64
+; RV32ZVE32F-NEXT:    bnez a1, .LBB95_15
 ; RV32ZVE32F-NEXT:  .LBB95_7: # %else12
-; RV32ZVE32F-NEXT:    andi a0, a1, -128
+; RV32ZVE32F-NEXT:    andi a0, a0, -128
 ; RV32ZVE32F-NEXT:    bnez a0, .LBB95_16
 ; RV32ZVE32F-NEXT:  .LBB95_8: # %else14
 ; RV32ZVE32F-NEXT:    ret
 ; RV32ZVE32F-NEXT:  .LBB95_9: # %cond.store
-; RV32ZVE32F-NEXT:    vmv.x.s a0, v8
-; RV32ZVE32F-NEXT:    fsd fa0, 0(a0)
-; RV32ZVE32F-NEXT:    andi a0, a1, 2
-; RV32ZVE32F-NEXT:    beqz a0, .LBB95_2
+; RV32ZVE32F-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
+; RV32ZVE32F-NEXT:    vmv.x.s a1, v10
+; RV32ZVE32F-NEXT:    fsd fa0, 0(a1)
+; RV32ZVE32F-NEXT:    andi a1, a0, 2
+; RV32ZVE32F-NEXT:    beqz a1, .LBB95_2
 ; RV32ZVE32F-NEXT:  .LBB95_10: # %cond.store1
 ; RV32ZVE32F-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
-; RV32ZVE32F-NEXT:    vslidedown.vi v10, v8, 1
-; RV32ZVE32F-NEXT:    vmv.x.s a0, v10
-; RV32ZVE32F-NEXT:    fsd fa1, 0(a0)
-; RV32ZVE32F-NEXT:    andi a0, a1, 4
-; RV32ZVE32F-NEXT:    beqz a0, .LBB95_3
+; RV32ZVE32F-NEXT:    vslidedown.vi v8, v10, 1
+; RV32ZVE32F-NEXT:    vmv.x.s a1, v8
+; RV32ZVE32F-NEXT:    fsd fa1, 0(a1)
+; RV32ZVE32F-NEXT:    andi a1, a0, 4
+; RV32ZVE32F-NEXT:    beqz a1, .LBB95_3
 ; RV32ZVE32F-NEXT:  .LBB95_11: # %cond.store3
 ; RV32ZVE32F-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
-; RV32ZVE32F-NEXT:    vslidedown.vi v10, v8, 2
-; RV32ZVE32F-NEXT:    vmv.x.s a0, v10
-; RV32ZVE32F-NEXT:    fsd fa2, 0(a0)
-; RV32ZVE32F-NEXT:    andi a0, a1, 8
-; RV32ZVE32F-NEXT:    beqz a0, .LBB95_4
+; RV32ZVE32F-NEXT:    vslidedown.vi v8, v10, 2
+; RV32ZVE32F-NEXT:    vmv.x.s a1, v8
+; RV32ZVE32F-NEXT:    fsd fa2, 0(a1)
+; RV32ZVE32F-NEXT:    andi a1, a0, 8
+; RV32ZVE32F-NEXT:    beqz a1, .LBB95_4
 ; RV32ZVE32F-NEXT:  .LBB95_12: # %cond.store5
 ; RV32ZVE32F-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
-; RV32ZVE32F-NEXT:    vslidedown.vi v10, v8, 3
-; RV32ZVE32F-NEXT:    vmv.x.s a0, v10
-; RV32ZVE32F-NEXT:    fsd fa3, 0(a0)
-; RV32ZVE32F-NEXT:    andi a0, a1, 16
-; RV32ZVE32F-NEXT:    beqz a0, .LBB95_5
+; RV32ZVE32F-NEXT:    vslidedown.vi v8, v10, 3
+; RV32ZVE32F-NEXT:    vmv.x.s a1, v8
+; RV32ZVE32F-NEXT:    fsd fa3, 0(a1)
+; RV32ZVE32F-NEXT:    andi a1, a0, 16
+; RV32ZVE32F-NEXT:    beqz a1, .LBB95_5
 ; RV32ZVE32F-NEXT:  .LBB95_13: # %cond.store7
 ; RV32ZVE32F-NEXT:    vsetivli zero, 1, e32, m2, ta, ma
-; RV32ZVE32F-NEXT:    vslidedown.vi v10, v8, 4
-; RV32ZVE32F-NEXT:    vmv.x.s a0, v10
-; RV32ZVE32F-NEXT:    fsd fa4, 0(a0)
-; RV32ZVE32F-NEXT:    andi a0, a1, 32
-; RV32ZVE32F-NEXT:    beqz a0, .LBB95_6
+; RV32ZVE32F-NEXT:    vslidedown.vi v8, v10, 4
+; RV32ZVE32F-NEXT:    vmv.x.s a1, v8
+; RV32ZVE32F-NEXT:    fsd fa4, 0(a1)
+; RV32ZVE32F-NEXT:    andi a1, a0, 32
+; RV32ZVE32F-NEXT:    beqz a1, .LBB95_6
 ; RV32ZVE32F-NEXT:  .LBB95_14: # %cond.store9
 ; RV32ZVE32F-NEXT:    vsetivli zero, 1, e32, m2, ta, ma
-; RV32ZVE32F-NEXT:    vslidedown.vi v10, v8, 5
-; RV32ZVE32F-NEXT:    vmv.x.s a0, v10
-; RV32ZVE32F-NEXT:    fsd fa5, 0(a0)
-; RV32ZVE32F-NEXT:    andi a0, a1, 64
-; RV32ZVE32F-NEXT:    beqz a0, .LBB95_7
+; RV32ZVE32F-NEXT:    vslidedown.vi v8, v10, 5
+; RV32ZVE32F-NEXT:    vmv.x.s a1, v8
+; RV32ZVE32F-NEXT:    fsd fa5, 0(a1)
+; RV32ZVE32F-NEXT:    andi a1, a0, 64
+; RV32ZVE32F-NEXT:    beqz a1, .LBB95_7
 ; RV32ZVE32F-NEXT:  .LBB95_15: # %cond.store11
 ; RV32ZVE32F-NEXT:    vsetivli zero, 1, e32, m2, ta, ma
-; RV32ZVE32F-NEXT:    vslidedown.vi v10, v8, 6
-; RV32ZVE32F-NEXT:    vmv.x.s a0, v10
-; RV32ZVE32F-NEXT:    fsd fa6, 0(a0)
-; RV32ZVE32F-NEXT:    andi a0, a1, -128
+; RV32ZVE32F-NEXT:    vslidedown.vi v8, v10, 6
+; RV32ZVE32F-NEXT:    vmv.x.s a1, v8
+; RV32ZVE32F-NEXT:    fsd fa6, 0(a1)
+; RV32ZVE32F-NEXT:    andi a0, a0, -128
 ; RV32ZVE32F-NEXT:    beqz a0, .LBB95_8
 ; RV32ZVE32F-NEXT:  .LBB95_16: # %cond.store13
 ; RV32ZVE32F-NEXT:    vsetivli zero, 1, e32, m2, ta, ma
-; RV32ZVE32F-NEXT:    vslidedown.vi v8, v8, 7
+; RV32ZVE32F-NEXT:    vslidedown.vi v8, v10, 7
 ; RV32ZVE32F-NEXT:    vmv.x.s a0, v8
 ; RV32ZVE32F-NEXT:    fsd fa7, 0(a0)
 ; RV32ZVE32F-NEXT:    ret
@@ -11158,106 +11172,107 @@ define void @mscatter_baseidx_sext_v8i16_v8f64(<8 x double> %val, ptr %base, <8
 define void @mscatter_baseidx_zext_v8i16_v8f64(<8 x double> %val, ptr %base, <8 x i16> %idxs, <8 x i1> %m) {
 ; RV32V-LABEL: mscatter_baseidx_zext_v8i16_v8f64:
 ; RV32V:       # %bb.0:
-; RV32V-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
-; RV32V-NEXT:    vzext.vf2 v14, v12
-; RV32V-NEXT:    vsll.vi v12, v14, 3
+; RV32V-NEXT:    li a1, 8
+; RV32V-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; RV32V-NEXT:    vwmulu.vx v14, v12, a1
 ; RV32V-NEXT:    vsetvli zero, zero, e64, m4, ta, ma
-; RV32V-NEXT:    vsoxei32.v v8, (a0), v12, v0.t
+; RV32V-NEXT:    vsoxei32.v v8, (a0), v14, v0.t
 ; RV32V-NEXT:    ret
 ;
 ; RV64V-LABEL: mscatter_baseidx_zext_v8i16_v8f64:
 ; RV64V:       # %bb.0:
-; RV64V-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
-; RV64V-NEXT:    vzext.vf2 v14, v12
-; RV64V-NEXT:    vsll.vi v12, v14, 3
+; RV64V-NEXT:    li a1, 8
+; RV64V-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; RV64V-NEXT:    vwmulu.vx v14, v12, a1
 ; RV64V-NEXT:    vsetvli zero, zero, e64, m4, ta, ma
-; RV64V-NEXT:    vsoxei32.v v8, (a0), v12, v0.t
+; RV64V-NEXT:    vsoxei32.v v8, (a0), v14, v0.t
 ; RV64V-NEXT:    ret
 ;
 ; RV32ZVE32F-LABEL: mscatter_baseidx_zext_v8i16_v8f64:
 ; RV32ZVE32F:       # %bb.0:
 ; RV32ZVE32F-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
-; RV32ZVE32F-NEXT:    vzext.vf2 v10, v8
+; RV32ZVE32F-NEXT:    vmv.v.x v10, a0
+; RV32ZVE32F-NEXT:    li a1, 8
 ; RV32ZVE32F-NEXT:    vsetvli zero, zero, e8, mf2, ta, ma
-; RV32ZVE32F-NEXT:    vmv.x.s a1, v0
-; RV32ZVE32F-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
-; RV32ZVE32F-NEXT:    vsll.vi v8, v10, 3
-; RV32ZVE32F-NEXT:    andi a2, a1, 1
-; RV32ZVE32F-NEXT:    vadd.vx v8, v8, a0
+; RV32ZVE32F-NEXT:    vmv.x.s a0, v0
+; RV32ZVE32F-NEXT:    andi a2, a0, 1
+; RV32ZVE32F-NEXT:    vsetvli zero, zero, e16, m1, ta, ma
+; RV32ZVE32F-NEXT:    vwmaccu.vx v10, a1, v8
 ; RV32ZVE32F-NEXT:    bnez a2, .LBB96_9
 ; RV32ZVE32F-NEXT:  # %bb.1: # %else
-; RV32ZVE32F-NEXT:    andi a0, a1, 2
-; RV32ZVE32F-NEXT:    bnez a0, .LBB96_10
+; RV32ZVE32F-NEXT:    andi a1, a0, 2
+; RV32ZVE32F-NEXT:    bnez a1, .LBB96_10
 ; RV32ZVE32F-NEXT:  .LBB96_2: # %else2
-; RV32ZVE32F-NEXT:    andi a0, a1, 4
-; RV32ZVE32F-NEXT:    bnez a0, .LBB96_11
+; RV32ZVE32F-NEXT:    andi a1, a0, 4
+; RV32ZVE32F-NEXT:    bnez a1, .LBB96_11
 ; RV32ZVE32F-NEXT:  .LBB96_3: # %else4
-; RV32ZVE32F-NEXT:    andi a0, a1, 8
-; RV32ZVE32F-NEXT:    bnez a0, .LBB96_12
+; RV32ZVE32F-NEXT:    andi a1, a0, 8
+; RV32ZVE32F-NEXT:    bnez a1, .LBB96_12
 ; RV32ZVE32F-NEXT:  .LBB96_4: # %else6
-; RV32ZVE32F-NEXT:    andi a0, a1, 16
-; RV32ZVE32F-NEXT:    bnez a0, .LBB96_13
+; RV32ZVE32F-NEXT:    andi a1, a0, 16
+; RV32ZVE32F-NEXT:    bnez a1, .LBB96_13
 ; RV32ZVE32F-NEXT:  .LBB96_5: # %else8
-; RV32ZVE32F-NEXT:    andi a0, a1, 32
-; RV32ZVE32F-NEXT:    bnez a0, .LBB96_14
+; RV32ZVE32F-NEXT:    andi a1, a0, 32
+; RV32ZVE32F-NEXT:    bnez a1, .LBB96_14
 ; RV32ZVE32F-NEXT:  .LBB96_6: # %else10
-; RV32ZVE32F-NEXT:    andi a0, a1, 64
-; RV32ZVE32F-NEXT:    bnez a0, .LBB96_15
+; RV32ZVE32F-NEXT:    andi a1, a0, 64
+; RV32ZVE32F-NEXT:    bnez a1, .LBB96_15
 ; RV32ZVE32F-NEXT:  .LBB96_7: # %else12
-; RV32ZVE32F-NEXT:    andi a0, a1, -128
+; RV32ZVE32F-NEXT:    andi a0, a0, -128
 ; RV32ZVE32F-NEXT:    bnez a0, .LBB96_16
 ; RV32ZVE32F-NEXT:  .LBB96_8: # %else14
 ; RV32ZVE32F-NEXT:    ret
 ; RV32ZVE32F-NEXT:  .LBB96_9: # %cond.store
-; RV32ZVE32F-NEXT:    vmv.x.s a0, v8
-; RV32ZVE32F-NEXT:    fsd fa0, 0(a0)
-; RV32ZVE32F-NEXT:    andi a0, a1, 2
-; RV32ZVE32F-NEXT:    beqz a0, .LBB96_2
+; RV32ZVE32F-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
+; RV32ZVE32F-NEXT:    vmv.x.s a1, v10
+; RV32ZVE32F-NEXT:    fsd fa0, 0(a1)
+; RV32ZVE32F-NEXT:    andi a1, a0, 2
+; RV32ZVE32F-NEXT:    beqz a1, .LBB96_2
 ; RV32ZVE32F-NEXT:  .LBB96_10: # %cond.store1
 ; RV32ZVE32F-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
-; RV32ZVE32F-NEXT:    vslidedown.vi v10, v8, 1
-; RV32ZVE32F-NEXT:    vmv.x.s a0, v10
-; RV32ZVE32F-NEXT:    fsd fa1, 0(a0)
-; RV32ZVE32F-NEXT:    andi a0, a1, 4
-; RV32ZVE32F-NEXT:    beqz a0, .LBB96_3
+; RV32ZVE32F-NEXT:    vslidedown.vi v8, v10, 1
+; RV32ZVE32F-NEXT:    vmv.x.s a1, v8
+; RV32ZVE32F-NEXT:    fsd fa1, 0(a1)
+; RV32ZVE32F-NEXT:    andi a1, a0, 4
+; RV32ZVE32F-NEXT:    beqz a1, .LBB96_3
 ; RV32ZVE32F-NEXT:  .LBB96_11: # %cond.store3
 ; RV32ZVE32F-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
-; RV32ZVE32F-NEXT:    vslidedown.vi v10, v8, 2
-; RV32ZVE32F-NEXT:    vmv.x.s a0, v10
-; RV32ZVE32F-NEXT:    fsd fa2, 0(a0)
-; RV32ZVE32F-NEXT:    andi a0, a1, 8
-; RV32ZVE32F-NEXT:    beqz a0, .LBB96_4
+; RV32ZVE32F-NEXT:    vslidedown.vi v8, v10, 2
+; RV32ZVE32F-NEXT:    vmv.x.s a1, v8
+; RV32ZVE32F-NEXT:    fsd fa2, 0(a1)
+; RV32ZVE32F-NEXT:    andi a1, a0, 8
+; RV32ZVE32F-NEXT:    beqz a1, .LBB96_4
 ; RV32ZVE32F-NEXT:  .LBB96_12: # %cond.store5
 ; RV32ZVE32F-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
-; RV32ZVE32F-NEXT:    vslidedown.vi v10, v8, 3
-; RV32ZVE32F-NEXT:    vmv.x.s a0, v10
-; RV32ZVE32F-NEXT:    fsd fa3, 0(a0)
-; RV32ZVE32F-NEXT:    andi a0, a1, 16
-; RV32ZVE32F-NEXT:    beqz a0, .LBB96_5
+; RV32ZVE32F-NEXT:    vslidedown.vi v8, v10, 3
+; RV32ZVE32F-NEXT:    vmv.x.s a1, v8
+; RV32ZVE32F-NEXT:    fsd fa3, 0(a1)
+; RV32ZVE32F-NEXT:    andi a1, a0, 16
+; RV32ZVE32F-NEXT:    beqz a1, .LBB96_5
 ; RV32ZVE32F-NEXT:  .LBB96_13: # %cond.store7
 ; RV32ZVE32F-NEXT:    vsetivli zero, 1, e32, m2, ta, ma
-; RV32ZVE32F-NEXT:    vslidedown.vi v10, v8, 4
-; RV32ZVE32F-NEXT:    vmv.x.s a0, v10
-; RV32ZVE32F-NEXT:    fsd fa4, 0(a0)
-; RV32ZVE32F-NEXT:    andi a0, a1, 32
-; RV32ZVE32F-NEXT:    beqz a0, .LBB96_6
+; RV32ZVE32F-NEXT:    vslidedown.vi v8, v10, 4
+; RV32ZVE32F-NEXT:    vmv.x.s a1, v8
+; RV32ZVE32F-NEXT:    fsd fa4, 0(a1)
+; RV32ZVE32F-NEXT:    andi a1, a0, 32
+; RV32ZVE32F-NEXT:    beqz a1, .LBB96_6
 ; RV32ZVE32F-NEXT:  .LBB96_14: # %cond.store9
 ; RV32ZVE32F-NEXT:    vsetivli zero, 1, e32, m2, ta, ma
-; RV32ZVE32F-NEXT:    vslidedown.vi v10, v8, 5
-; RV32ZVE32F-NEXT:    vmv.x.s a0, v10
-; RV32ZVE32F-NEXT:    fsd fa5, 0(a0)
-; RV32ZVE32F-NEXT:    andi a0, a1, 64
-; RV32ZVE32F-NEXT:    beqz a0, .LBB96_7
+; RV32ZVE32F-NEXT:    vslidedown.vi v8, v10, 5
+; RV32ZVE32F-NEXT:    vmv.x.s a1, v8
+; RV32ZVE32F-NEXT:    fsd fa5, 0(a1)
+; RV32ZVE32F-NEXT:    andi a1, a0, 64
+; RV32ZVE32F-NEXT:    beqz a1, .LBB96_7
 ; RV32ZVE32F-NEXT:  .LBB96_15: # %cond.store11
 ; RV32ZVE32F-NEXT:    vsetivli zero, 1, e32, m2, ta, ma
-; RV32ZVE32F-NEXT:    vslidedown.vi v10, v8, 6
-; RV32ZVE32F-NEXT:    vmv.x.s a0, v10
-; RV32ZVE32F-NEXT:    fsd fa6, 0(a0)
-; RV32ZVE32F-NEXT:    andi a0, a1, -128
+; RV32ZVE32F-NEXT:    vslidedown.vi v8, v10, 6
+; RV32ZVE32F-NEXT:    vmv.x.s a1, v8
+; RV32ZVE32F-NEXT:    fsd fa6, 0(a1)
+; RV32ZVE32F-NEXT:    andi a0, a0, -128
 ; RV32ZVE32F-NEXT:    beqz a0, .LBB96_8
 ; RV32ZVE32F-NEXT:  .LBB96_16: # %cond.store13
 ; RV32ZVE32F-NEXT:    vsetivli zero, 1, e32, m2, ta, ma
-; RV32ZVE32F-NEXT:    vslidedown.vi v8, v8, 7
+; RV32ZVE32F-NEXT:    vslidedown.vi v8, v10, 7
 ; RV32ZVE32F-NEXT:    vmv.x.s a0, v8
 ; RV32ZVE32F-NEXT:    fsd fa7, 0(a0)
 ; RV32ZVE32F-NEXT:    ret
@@ -11377,10 +11392,11 @@ define void @mscatter_baseidx_v8i32_v8f64(<8 x double> %val, ptr %base, <8 x i32
 ;
 ; RV64V-LABEL: mscatter_baseidx_v8i32_v8f64:
 ; RV64V:       # %bb.0:
-; RV64V-NEXT:    vsetivli zero, 8, e64, m4, ta, ma
-; RV64V-NEXT:    vsext.vf2 v16, v12
-; RV64V-NEXT:    vsll.vi v12, v16, 3
-; RV64V-NEXT:    vsoxei64.v v8, (a0), v12, v0.t
+; RV64V-NEXT:    li a1, 8
+; RV64V-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
+; RV64V-NEXT:    vwmulsu.vx v16, v12, a1
+; RV64V-NEXT:    vsetvli zero, zero, e64, m4, ta, ma
+; RV64V-NEXT:    vsoxei64.v v8, (a0), v16, v0.t
 ; RV64V-NEXT:    ret
 ;
 ; RV32ZVE32F-LABEL: mscatter_baseidx_v8i32_v8f64:
@@ -11576,10 +11592,11 @@ define void @mscatter_baseidx_sext_v8i32_v8f64(<8 x double> %val, ptr %base, <8
 ;
 ; RV64V-LABEL: mscatter_baseidx_sext_v8i32_v8f64:
 ; RV64V:       # %bb.0:
-; RV64V-NEXT:    vsetivli zero, 8, e64, m4, ta, ma
-; RV64V-NEXT:    vsext.vf2 v16, v12
-; RV64V-NEXT:    vsll.vi v12, v16, 3
-; RV64V-NEXT:    vsoxei64.v v8, (a0), v12, v0.t
+; RV64V-NEXT:    li a1, 8
+; RV64V-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
+; RV64V-NEXT:    vwmulsu.vx v16, v12, a1
+; RV64V-NEXT:    vsetvli zero, zero, e64, m4, ta, ma
+; RV64V-NEXT:    vsoxei64.v v8, (a0), v16, v0.t
 ; RV64V-NEXT:    ret
 ;
 ; RV32ZVE32F-LABEL: mscatter_baseidx_sext_v8i32_v8f64:
@@ -11776,10 +11793,11 @@ define void @mscatter_baseidx_zext_v8i32_v8f64(<8 x double> %val, ptr %base, <8
 ;
 ; RV64V-LABEL: mscatter_baseidx_zext_v8i32_v8f64:
 ; RV64V:       # %bb.0:
-; RV64V-NEXT:    vsetivli zero, 8, e64, m4, ta, ma
-; RV64V-NEXT:    vzext.vf2 v16, v12
-; RV64V-NEXT:    vsll.vi v12, v16, 3
-; RV64V-NEXT:    vsoxei64.v v8, (a0), v12, v0.t
+; RV64V-NEXT:    li a1, 8
+; RV64V-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
+; RV64V-NEXT:    vwmulu.vx v16, v12, a1
+; RV64V-NEXT:    vsetvli zero, zero, e64, m4, ta, ma
+; RV64V-NEXT:    vsoxei64.v v8, (a0), v16, v0.t
 ; RV64V-NEXT:    ret
 ;
 ; RV32ZVE32F-LABEL: mscatter_baseidx_zext_v8i32_v8f64:
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-shuffle-exact-vlen.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-shuffle-exact-vlen.ll
index afd560fd74d16..c0c17d4e0623e 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-shuffle-exact-vlen.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-shuffle-exact-vlen.ll
@@ -431,3 +431,31 @@ define void @shuffle_i256_ldst(ptr %p) vscale_range(2,2) {
   store <4 x i256> %res, ptr %p
   ret void
 }
+
+define void @shuffle_3_input_vectors() vscale_range(4,4) {
+; CHECK-LABEL: shuffle_3_input_vectors:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e64, m8, ta, ma
+; CHECK-NEXT:    vmv.v.i v8, 1
+; CHECK-NEXT:    vsetivli zero, 1, e8, mf8, ta, ma
+; CHECK-NEXT:    vmv.v.i v0, 6
+; CHECK-NEXT:    vsetvli a0, zero, e64, m8, ta, ma
+; CHECK-NEXT:    vmv.v.i v16, 0
+; CHECK-NEXT:    vsetivli zero, 4, e64, m1, ta, mu
+; CHECK-NEXT:    vslidedown.vi v20, v8, 1, v0.t
+; CHECK-NEXT:    vslideup.vi v20, v9, 3
+; CHECK-NEXT:    vslidedown.vi v21, v9, 1
+; CHECK-NEXT:    vmv1r.v v22, v8
+; CHECK-NEXT:    vsetvli a0, zero, e64, m8, ta, ma
+; CHECK-NEXT:    vmsgt.vi v8, v16, 0
+; CHECK-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
+; CHECK-NEXT:    vmv.x.s a0, v8
+; CHECK-NEXT:    sb a0, 0(zero)
+; CHECK-NEXT:    ret
+  %1 = shufflevector <32 x i64> zeroinitializer, <32 x i64> splat (i64 1), <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 poison, i32 poison, i32 33, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+  %2 = icmp slt <32 x i64> zeroinitializer, %1
+  %3 = bitcast <32 x i1> %2 to i32
+  %4 = trunc i32 %3 to i8
+  store i8 %4, ptr null, align 1
+  ret void
+}
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-shuffle-reverse.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-shuffle-reverse.ll
index 5fd7e47507f71..c222626a166fe 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-shuffle-reverse.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-shuffle-reverse.ll
@@ -839,14 +839,13 @@ define <4 x i8> @reverse_v4i8_2(<2 x i8> %a, <2 x i8> %b) {
 define <8 x i8> @reverse_v8i8_2(<4 x i8> %a, <4 x i8> %b) {
 ; CHECK-LABEL: reverse_v8i8_2:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetivli zero, 8, e8, mf2, ta, mu
-; CHECK-NEXT:    vid.v v11
-; CHECK-NEXT:    vrsub.vi v12, v11, 7
-; CHECK-NEXT:    vrgather.vv v10, v8, v12
-; CHECK-NEXT:    vmv.v.i v0, 15
-; CHECK-NEXT:    vrsub.vi v8, v11, 3
-; CHECK-NEXT:    vrgather.vv v10, v9, v8, v0.t
-; CHECK-NEXT:    vmv1r.v v8, v10
+; CHECK-NEXT:    vsetivli zero, 4, e8, mf4, ta, ma
+; CHECK-NEXT:    vid.v v10
+; CHECK-NEXT:    vrsub.vi v10, v10, 3
+; CHECK-NEXT:    vrgather.vv v11, v8, v10
+; CHECK-NEXT:    vrgather.vv v8, v9, v10
+; CHECK-NEXT:    vsetivli zero, 8, e8, mf2, ta, ma
+; CHECK-NEXT:    vslideup.vi v8, v11, 4
 ; CHECK-NEXT:    ret
   %res = shufflevector <4 x i8> %a, <4 x i8> %b, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
   ret <8 x i8> %res
@@ -855,17 +854,13 @@ define <8 x i8> @reverse_v8i8_2(<4 x i8> %a, <4 x i8> %b) {
 define <16 x i8> @reverse_v16i8_2(<8 x i8> %a, <8 x i8> %b) {
 ; CHECK-LABEL: reverse_v16i8_2:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 8, e8, mf2, ta, ma
+; CHECK-NEXT:    vid.v v10
+; CHECK-NEXT:    vrsub.vi v10, v10, 7
+; CHECK-NEXT:    vrgather.vv v11, v8, v10
+; CHECK-NEXT:    vrgather.vv v8, v9, v10
 ; CHECK-NEXT:    vsetivli zero, 16, e8, m1, ta, ma
-; CHECK-NEXT:    vid.v v11
-; CHECK-NEXT:    li a0, 255
-; CHECK-NEXT:    vrsub.vi v12, v11, 15
-; CHECK-NEXT:    vrgather.vv v10, v8, v12
-; CHECK-NEXT:    vsetvli zero, zero, e16, m2, ta, ma
-; CHECK-NEXT:    vmv.s.x v0, a0
-; CHECK-NEXT:    vsetvli zero, zero, e8, m1, ta, mu
-; CHECK-NEXT:    vrsub.vi v8, v11, 7
-; CHECK-NEXT:    vrgather.vv v10, v9, v8, v0.t
-; CHECK-NEXT:    vmv.v.v v8, v10
+; CHECK-NEXT:    vslideup.vi v8, v11, 8
 ; CHECK-NEXT:    ret
   %res = shufflevector <8 x i8> %a, <8 x i8> %b, <16 x i32> <i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
   ret <16 x i8> %res
@@ -874,27 +869,14 @@ define <16 x i8> @reverse_v16i8_2(<8 x i8> %a, <8 x i8> %b) {
 define <32 x i8> @reverse_v32i8_2(<16 x i8> %a, <16 x i8> %b) {
 ; CHECK-LABEL: reverse_v32i8_2:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a0, zero, e16, m2, ta, ma
-; CHECK-NEXT:    vmv1r.v v10, v9
-; CHECK-NEXT:    csrr a0, vlenb
-; CHECK-NEXT:    vid.v v12
-; CHECK-NEXT:    addi a1, a0, -1
-; CHECK-NEXT:    vrsub.vx v12, v12, a1
-; CHECK-NEXT:    lui a1, 16
-; CHECK-NEXT:    addi a1, a1, -1
-; CHECK-NEXT:    vsetvli zero, zero, e8, m1, ta, ma
-; CHECK-NEXT:    vrgatherei16.vv v15, v8, v12
-; CHECK-NEXT:    vrgatherei16.vv v14, v9, v12
-; CHECK-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
-; CHECK-NEXT:    vmv.s.x v0, a1
-; CHECK-NEXT:    li a1, 32
-; CHECK-NEXT:    slli a0, a0, 1
-; CHECK-NEXT:    vsetvli zero, a1, e8, m2, ta, mu
-; CHECK-NEXT:    vid.v v8
-; CHECK-NEXT:    addi a0, a0, -32
-; CHECK-NEXT:    vrsub.vi v12, v8, 15
-; CHECK-NEXT:    vslidedown.vx v8, v14, a0
-; CHECK-NEXT:    vrgather.vv v8, v10, v12, v0.t
+; CHECK-NEXT:    vsetivli zero, 16, e8, m1, ta, ma
+; CHECK-NEXT:    vid.v v10
+; CHECK-NEXT:    vrsub.vi v10, v10, 15
+; CHECK-NEXT:    vrgather.vv v12, v8, v10
+; CHECK-NEXT:    vrgather.vv v8, v9, v10
+; CHECK-NEXT:    li a0, 32
+; CHECK-NEXT:    vsetvli zero, a0, e8, m2, ta, ma
+; CHECK-NEXT:    vslideup.vi v8, v12, 16
 ; CHECK-NEXT:    ret
   %res = shufflevector <16 x i8> %a, <16 x i8> %b,  <32 x i32> <i32 31, i32 30, i32 29, i32 28, i32 27, i32 26, i32 25, i32 24, i32 23, i32 22, i32 21, i32 20, i32 19, i32 18, i32 17, i32 16, i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
   ret <32 x i8> %res
@@ -927,14 +909,13 @@ define <4 x i16> @reverse_v4i16_2(<2 x i16> %a, <2 x i16> %b) {
 define <8 x i16> @reverse_v8i16_2(<4 x i16> %a, <4 x i16> %b) {
 ; CHECK-LABEL: reverse_v8i16_2:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetivli zero, 8, e16, m1, ta, mu
-; CHECK-NEXT:    vid.v v11
-; CHECK-NEXT:    vrsub.vi v12, v11, 7
-; CHECK-NEXT:    vrgather.vv v10, v8, v12
-; CHECK-NEXT:    vmv.v.i v0, 15
-; CHECK-NEXT:    vrsub.vi v8, v11, 3
-; CHECK-NEXT:    vrgather.vv v10, v9, v8, v0.t
-; CHECK-NEXT:    vmv.v.v v8, v10
+; CHECK-NEXT:    vsetivli zero, 4, e16, mf2, ta, ma
+; CHECK-NEXT:    vid.v v10
+; CHECK-NEXT:    vrsub.vi v10, v10, 3
+; CHECK-NEXT:    vrgather.vv v11, v8, v10
+; CHECK-NEXT:    vrgather.vv v8, v9, v10
+; CHECK-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; CHECK-NEXT:    vslideup.vi v8, v11, 4
 ; CHECK-NEXT:    ret
   %res = shufflevector <4 x i16> %a, <4 x i16> %b, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
   ret <8 x i16> %res
@@ -943,23 +924,13 @@ define <8 x i16> @reverse_v8i16_2(<4 x i16> %a, <4 x i16> %b) {
 define <16 x i16> @reverse_v16i16_2(<8 x i16> %a, <8 x i16> %b) {
 ; CHECK-LABEL: reverse_v16i16_2:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a0, zero, e16, m1, ta, ma
-; CHECK-NEXT:    vmv1r.v v10, v9
-; CHECK-NEXT:    csrr a0, vlenb
-; CHECK-NEXT:    vid.v v9
-; CHECK-NEXT:    srli a1, a0, 1
-; CHECK-NEXT:    addi a1, a1, -1
-; CHECK-NEXT:    vrsub.vx v9, v9, a1
-; CHECK-NEXT:    vrgather.vv v13, v8, v9
-; CHECK-NEXT:    vrgather.vv v12, v11, v9
-; CHECK-NEXT:    vsetivli zero, 16, e16, m2, ta, mu
-; CHECK-NEXT:    vid.v v8
-; CHECK-NEXT:    li a1, 255
-; CHECK-NEXT:    addi a0, a0, -16
-; CHECK-NEXT:    vrsub.vi v14, v8, 7
-; CHECK-NEXT:    vmv.s.x v0, a1
-; CHECK-NEXT:    vslidedown.vx v8, v12, a0
-; CHECK-NEXT:    vrgather.vv v8, v10, v14, v0.t
+; CHECK-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; CHECK-NEXT:    vid.v v10
+; CHECK-NEXT:    vrsub.vi v10, v10, 7
+; CHECK-NEXT:    vrgather.vv v12, v8, v10
+; CHECK-NEXT:    vrgather.vv v8, v9, v10
+; CHECK-NEXT:    vsetivli zero, 16, e16, m2, ta, ma
+; CHECK-NEXT:    vslideup.vi v8, v12, 8
 ; CHECK-NEXT:    ret
   %res = shufflevector <8 x i16> %a, <8 x i16> %b,  <16 x i32> <i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
   ret <16 x i16> %res
@@ -968,30 +939,23 @@ define <16 x i16> @reverse_v16i16_2(<8 x i16> %a, <8 x i16> %b) {
 define <32 x i16> @reverse_v32i16_2(<16 x i16> %a, <16 x i16> %b) {
 ; CHECK-LABEL: reverse_v32i16_2:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a0, zero, e16, m1, ta, ma
-; CHECK-NEXT:    vmv2r.v v12, v10
 ; CHECK-NEXT:    csrr a0, vlenb
-; CHECK-NEXT:    vid.v v10
-; CHECK-NEXT:    lui a1, 16
-; CHECK-NEXT:    addi a1, a1, -1
-; CHECK-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
-; CHECK-NEXT:    vmv.s.x v0, a1
+; CHECK-NEXT:    vsetvli a1, zero, e16, m1, ta, ma
+; CHECK-NEXT:    vid.v v12
 ; CHECK-NEXT:    srli a1, a0, 1
+; CHECK-NEXT:    addi a0, a0, -16
 ; CHECK-NEXT:    addi a1, a1, -1
-; CHECK-NEXT:    vsetvli zero, zero, e16, m1, ta, ma
-; CHECK-NEXT:    vrsub.vx v10, v10, a1
-; CHECK-NEXT:    li a1, 32
-; CHECK-NEXT:    slli a0, a0, 1
-; CHECK-NEXT:    vrgather.vv v19, v8, v10
-; CHECK-NEXT:    vrgather.vv v18, v9, v10
-; CHECK-NEXT:    vrgather.vv v16, v11, v10
-; CHECK-NEXT:    vsetvli zero, a1, e16, m4, ta, mu
-; CHECK-NEXT:    vid.v v8
-; CHECK-NEXT:    addi a0, a0, -32
-; CHECK-NEXT:    vrsub.vi v20, v8, 15
-; CHECK-NEXT:    vmv1r.v v17, v16
+; CHECK-NEXT:    vrsub.vx v12, v12, a1
+; CHECK-NEXT:    vrgather.vv v15, v8, v12
+; CHECK-NEXT:    vrgather.vv v17, v10, v12
+; CHECK-NEXT:    vrgather.vv v14, v9, v12
+; CHECK-NEXT:    vrgather.vv v16, v11, v12
+; CHECK-NEXT:    vsetivli zero, 16, e16, m2, ta, ma
+; CHECK-NEXT:    vslidedown.vx v12, v14, a0
 ; CHECK-NEXT:    vslidedown.vx v8, v16, a0
-; CHECK-NEXT:    vrgather.vv v8, v12, v20, v0.t
+; CHECK-NEXT:    li a0, 32
+; CHECK-NEXT:    vsetvli zero, a0, e16, m4, ta, ma
+; CHECK-NEXT:    vslideup.vi v8, v12, 16
 ; CHECK-NEXT:    ret
   %res = shufflevector <16 x i16> %a, <16 x i16> %b,  <32 x i32> <i32 31, i32 30, i32 29, i32 28, i32 27, i32 26, i32 25, i32 24, i32 23, i32 22, i32 21, i32 20, i32 19, i32 18, i32 17, i32 16, i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
   ret <32 x i16> %res
@@ -1024,24 +988,13 @@ define <4 x i32> @reverse_v4i32_2(<2 x i32> %a, < 2 x i32> %b) {
 define <8 x i32> @reverse_v8i32_2(<4 x i32> %a, <4 x i32> %b) {
 ; CHECK-LABEL: reverse_v8i32_2:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a0, zero, e32, m1, ta, ma
-; CHECK-NEXT:    vmv1r.v v10, v9
-; CHECK-NEXT:    csrr a0, vlenb
-; CHECK-NEXT:    vid.v v9
-; CHECK-NEXT:    srli a1, a0, 2
-; CHECK-NEXT:    addi a1, a1, -1
-; CHECK-NEXT:    vrsub.vx v9, v9, a1
-; CHECK-NEXT:    vrgather.vv v13, v8, v9
-; CHECK-NEXT:    vrgather.vv v12, v11, v9
-; CHECK-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
-; CHECK-NEXT:    vid.v v8
-; CHECK-NEXT:    vmv.v.i v0, 15
-; CHECK-NEXT:    srli a0, a0, 1
-; CHECK-NEXT:    vrsub.vi v14, v8, 3
-; CHECK-NEXT:    addi a0, a0, -8
-; CHECK-NEXT:    vsetvli zero, zero, e32, m2, ta, mu
-; CHECK-NEXT:    vslidedown.vx v8, v12, a0
-; CHECK-NEXT:    vrgatherei16.vv v8, v10, v14, v0.t
+; CHECK-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
+; CHECK-NEXT:    vid.v v10
+; CHECK-NEXT:    vrsub.vi v10, v10, 3
+; CHECK-NEXT:    vrgather.vv v12, v8, v10
+; CHECK-NEXT:    vrgather.vv v8, v9, v10
+; CHECK-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
+; CHECK-NEXT:    vslideup.vi v8, v12, 4
 ; CHECK-NEXT:    ret
   %res = shufflevector <4 x i32> %a, <4 x i32> %b, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
   ret <8 x i32> %res
@@ -1050,26 +1003,23 @@ define <8 x i32> @reverse_v8i32_2(<4 x i32> %a, <4 x i32> %b) {
 define <16 x i32> @reverse_v16i32_2(<8 x i32> %a, <8 x i32> %b) {
 ; CHECK-LABEL: reverse_v16i32_2:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a0, zero, e32, m1, ta, ma
-; CHECK-NEXT:    vmv2r.v v12, v10
 ; CHECK-NEXT:    csrr a0, vlenb
-; CHECK-NEXT:    vid.v v10
+; CHECK-NEXT:    vsetvli a1, zero, e32, m1, ta, ma
+; CHECK-NEXT:    vid.v v12
 ; CHECK-NEXT:    srli a1, a0, 2
+; CHECK-NEXT:    srli a0, a0, 1
 ; CHECK-NEXT:    addi a1, a1, -1
-; CHECK-NEXT:    vrsub.vx v14, v10, a1
-; CHECK-NEXT:    vrgather.vv v11, v8, v14
-; CHECK-NEXT:    vrgather.vv v10, v9, v14
-; CHECK-NEXT:    vrgather.vv v8, v9, v14
-; CHECK-NEXT:    vsetivli zero, 16, e16, m2, ta, ma
-; CHECK-NEXT:    vid.v v14
-; CHECK-NEXT:    li a1, 255
-; CHECK-NEXT:    addi a0, a0, -16
-; CHECK-NEXT:    vrsub.vi v16, v14, 7
-; CHECK-NEXT:    vmv.s.x v0, a1
-; CHECK-NEXT:    vmv1r.v v9, v8
-; CHECK-NEXT:    vsetvli zero, zero, e32, m4, ta, mu
-; CHECK-NEXT:    vslidedown.vx v8, v8, a0
-; CHECK-NEXT:    vrgatherei16.vv v8, v12, v16, v0.t
+; CHECK-NEXT:    addi a0, a0, -8
+; CHECK-NEXT:    vrsub.vx v12, v12, a1
+; CHECK-NEXT:    vrgather.vv v15, v8, v12
+; CHECK-NEXT:    vrgather.vv v17, v10, v12
+; CHECK-NEXT:    vrgather.vv v14, v9, v12
+; CHECK-NEXT:    vrgather.vv v16, v11, v12
+; CHECK-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
+; CHECK-NEXT:    vslidedown.vx v12, v14, a0
+; CHECK-NEXT:    vslidedown.vx v8, v16, a0
+; CHECK-NEXT:    vsetivli zero, 16, e32, m4, ta, ma
+; CHECK-NEXT:    vslideup.vi v8, v12, 8
 ; CHECK-NEXT:    ret
   %res = shufflevector <8 x i32> %a, <8 x i32> %b,  <16 x i32> <i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
   ret <16 x i32> %res
@@ -1078,32 +1028,27 @@ define <16 x i32> @reverse_v16i32_2(<8 x i32> %a, <8 x i32> %b) {
 define <32 x i32> @reverse_v32i32_2(<16 x i32> %a, <16 x i32> %b) {
 ; CHECK-LABEL: reverse_v32i32_2:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a0, zero, e32, m1, ta, ma
-; CHECK-NEXT:    vmv4r.v v16, v12
 ; CHECK-NEXT:    csrr a0, vlenb
-; CHECK-NEXT:    vid.v v12
+; CHECK-NEXT:    vsetvli a1, zero, e32, m1, ta, ma
+; CHECK-NEXT:    vid.v v16
 ; CHECK-NEXT:    srli a1, a0, 2
+; CHECK-NEXT:    addi a0, a0, -16
 ; CHECK-NEXT:    addi a1, a1, -1
-; CHECK-NEXT:    vrsub.vx v20, v12, a1
-; CHECK-NEXT:    vrgather.vv v15, v8, v20
-; CHECK-NEXT:    vrgather.vv v14, v9, v20
-; CHECK-NEXT:    vrgather.vv v13, v10, v20
-; CHECK-NEXT:    vrgather.vv v12, v11, v20
-; CHECK-NEXT:    lui a1, 16
-; CHECK-NEXT:    addi a1, a1, -1
-; CHECK-NEXT:    vmv.s.x v0, a1
-; CHECK-NEXT:    li a1, 32
-; CHECK-NEXT:    slli a0, a0, 1
-; CHECK-NEXT:    vrgather.vv v8, v9, v20
-; CHECK-NEXT:    vsetvli zero, a1, e16, m4, ta, ma
-; CHECK-NEXT:    vid.v v20
-; CHECK-NEXT:    addi a0, a0, -32
-; CHECK-NEXT:    vmv1r.v v9, v8
-; CHECK-NEXT:    vrsub.vi v24, v20, 15
-; CHECK-NEXT:    vmv2r.v v10, v8
-; CHECK-NEXT:    vsetvli zero, zero, e32, m8, ta, mu
-; CHECK-NEXT:    vslidedown.vx v8, v8, a0
-; CHECK-NEXT:    vrgatherei16.vv v8, v16, v24, v0.t
+; CHECK-NEXT:    vrsub.vx v24, v16, a1
+; CHECK-NEXT:    vrgather.vv v23, v8, v24
+; CHECK-NEXT:    vrgather.vv v19, v12, v24
+; CHECK-NEXT:    vrgather.vv v22, v9, v24
+; CHECK-NEXT:    vrgather.vv v18, v13, v24
+; CHECK-NEXT:    vrgather.vv v21, v10, v24
+; CHECK-NEXT:    vrgather.vv v17, v14, v24
+; CHECK-NEXT:    vrgather.vv v20, v11, v24
+; CHECK-NEXT:    vrgather.vv v16, v15, v24
+; CHECK-NEXT:    vsetivli zero, 16, e32, m4, ta, ma
+; CHECK-NEXT:    vslidedown.vx v24, v20, a0
+; CHECK-NEXT:    vslidedown.vx v8, v16, a0
+; CHECK-NEXT:    li a0, 32
+; CHECK-NEXT:    vsetvli zero, a0, e32, m8, ta, ma
+; CHECK-NEXT:    vslideup.vi v8, v24, 16
 ; CHECK-NEXT:    ret
   %res = shufflevector <16 x i32> %a, <16 x i32> %b,  <32 x i32> <i32 31, i32 30, i32 29, i32 28, i32 27, i32 26, i32 25, i32 24, i32 23, i32 22, i32 21, i32 20, i32 19, i32 18, i32 17, i32 16, i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
   ret <32 x i32> %res
@@ -1127,28 +1072,23 @@ define <4 x i64> @reverse_v4i64_2(<2 x i64> %a, < 2 x i64> %b) {
 define <8 x i64> @reverse_v8i64_2(<4 x i64> %a, <4 x i64> %b) {
 ; CHECK-LABEL: reverse_v8i64_2:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a0, zero, e64, m1, ta, ma
-; CHECK-NEXT:    vmv2r.v v12, v10
 ; CHECK-NEXT:    csrr a0, vlenb
-; CHECK-NEXT:    vid.v v10
+; CHECK-NEXT:    vsetvli a1, zero, e64, m1, ta, ma
+; CHECK-NEXT:    vid.v v12
 ; CHECK-NEXT:    srli a1, a0, 3
+; CHECK-NEXT:    srli a0, a0, 2
 ; CHECK-NEXT:    addi a1, a1, -1
-; CHECK-NEXT:    vrsub.vx v14, v10, a1
-; CHECK-NEXT:    vrgather.vv v11, v8, v14
-; CHECK-NEXT:    vrgather.vv v10, v9, v14
-; CHECK-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
-; CHECK-NEXT:    vid.v v15
-; CHECK-NEXT:    vsetvli a1, zero, e64, m1, ta, ma
-; CHECK-NEXT:    vrgather.vv v8, v9, v14
-; CHECK-NEXT:    vmv.v.i v0, 15
-; CHECK-NEXT:    srli a0, a0, 1
-; CHECK-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
-; CHECK-NEXT:    vrsub.vi v16, v15, 3
-; CHECK-NEXT:    addi a0, a0, -8
-; CHECK-NEXT:    vmv1r.v v9, v8
-; CHECK-NEXT:    vsetvli zero, zero, e64, m4, ta, mu
-; CHECK-NEXT:    vslidedown.vx v8, v8, a0
-; CHECK-NEXT:    vrgatherei16.vv v8, v12, v16, v0.t
+; CHECK-NEXT:    addi a0, a0, -4
+; CHECK-NEXT:    vrsub.vx v12, v12, a1
+; CHECK-NEXT:    vrgather.vv v15, v8, v12
+; CHECK-NEXT:    vrgather.vv v17, v10, v12
+; CHECK-NEXT:    vrgather.vv v14, v9, v12
+; CHECK-NEXT:    vrgather.vv v16, v11, v12
+; CHECK-NEXT:    vsetivli zero, 4, e64, m2, ta, ma
+; CHECK-NEXT:    vslidedown.vx v12, v14, a0
+; CHECK-NEXT:    vslidedown.vx v8, v16, a0
+; CHECK-NEXT:    vsetivli zero, 8, e64, m4, ta, ma
+; CHECK-NEXT:    vslideup.vi v8, v12, 4
 ; CHECK-NEXT:    ret
   %res = shufflevector <4 x i64> %a, <4 x i64> %b, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
   ret <8 x i64> %res
@@ -1181,14 +1121,13 @@ define <4 x half> @reverse_v4f16_2(<2 x half> %a, <2 x half> %b) {
 define <8 x half> @reverse_v8f16_2(<4 x half> %a, <4 x half> %b) {
 ; CHECK-LABEL: reverse_v8f16_2:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetivli zero, 8, e16, m1, ta, mu
-; CHECK-NEXT:    vid.v v11
-; CHECK-NEXT:    vrsub.vi v12, v11, 7
-; CHECK-NEXT:    vrgather.vv v10, v8, v12
-; CHECK-NEXT:    vmv.v.i v0, 15
-; CHECK-NEXT:    vrsub.vi v8, v11, 3
-; CHECK-NEXT:    vrgather.vv v10, v9, v8, v0.t
-; CHECK-NEXT:    vmv.v.v v8, v10
+; CHECK-NEXT:    vsetivli zero, 4, e16, mf2, ta, ma
+; CHECK-NEXT:    vid.v v10
+; CHECK-NEXT:    vrsub.vi v10, v10, 3
+; CHECK-NEXT:    vrgather.vv v11, v8, v10
+; CHECK-NEXT:    vrgather.vv v8, v9, v10
+; CHECK-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; CHECK-NEXT:    vslideup.vi v8, v11, 4
 ; CHECK-NEXT:    ret
   %res = shufflevector <4 x half> %a, <4 x half> %b, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
   ret <8 x half> %res
@@ -1197,23 +1136,13 @@ define <8 x half> @reverse_v8f16_2(<4 x half> %a, <4 x half> %b) {
 define <16 x half> @reverse_v16f16_2(<8 x half> %a, <8 x half> %b) {
 ; CHECK-LABEL: reverse_v16f16_2:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a0, zero, e16, m1, ta, ma
-; CHECK-NEXT:    vmv1r.v v10, v9
-; CHECK-NEXT:    csrr a0, vlenb
-; CHECK-NEXT:    vid.v v9
-; CHECK-NEXT:    srli a1, a0, 1
-; CHECK-NEXT:    addi a1, a1, -1
-; CHECK-NEXT:    vrsub.vx v9, v9, a1
-; CHECK-NEXT:    vrgather.vv v13, v8, v9
-; CHECK-NEXT:    vrgather.vv v12, v11, v9
-; CHECK-NEXT:    vsetivli zero, 16, e16, m2, ta, mu
-; CHECK-NEXT:    vid.v v8
-; CHECK-NEXT:    li a1, 255
-; CHECK-NEXT:    addi a0, a0, -16
-; CHECK-NEXT:    vrsub.vi v14, v8, 7
-; CHECK-NEXT:    vmv.s.x v0, a1
-; CHECK-NEXT:    vslidedown.vx v8, v12, a0
-; CHECK-NEXT:    vrgather.vv v8, v10, v14, v0.t
+; CHECK-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; CHECK-NEXT:    vid.v v10
+; CHECK-NEXT:    vrsub.vi v10, v10, 7
+; CHECK-NEXT:    vrgather.vv v12, v8, v10
+; CHECK-NEXT:    vrgather.vv v8, v9, v10
+; CHECK-NEXT:    vsetivli zero, 16, e16, m2, ta, ma
+; CHECK-NEXT:    vslideup.vi v8, v12, 8
 ; CHECK-NEXT:    ret
   %res = shufflevector <8 x half> %a, <8 x half> %b,  <16 x i32> <i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
   ret <16 x half> %res
@@ -1269,24 +1198,13 @@ define <4 x float> @reverse_v4f32_2(<2 x float> %a, <2 x float> %b) {
 define <8 x float> @reverse_v8f32_2(<4 x float> %a, <4 x float> %b) {
 ; CHECK-LABEL: reverse_v8f32_2:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a0, zero, e32, m1, ta, ma
-; CHECK-NEXT:    vmv1r.v v10, v9
-; CHECK-NEXT:    csrr a0, vlenb
-; CHECK-NEXT:    vid.v v9
-; CHECK-NEXT:    srli a1, a0, 2
-; CHECK-NEXT:    addi a1, a1, -1
-; CHECK-NEXT:    vrsub.vx v9, v9, a1
-; CHECK-NEXT:    vrgather.vv v13, v8, v9
-; CHECK-NEXT:    vrgather.vv v12, v11, v9
-; CHECK-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
-; CHECK-NEXT:    vid.v v8
-; CHECK-NEXT:    vmv.v.i v0, 15
-; CHECK-NEXT:    srli a0, a0, 1
-; CHECK-NEXT:    vrsub.vi v14, v8, 3
-; CHECK-NEXT:    addi a0, a0, -8
-; CHECK-NEXT:    vsetvli zero, zero, e32, m2, ta, mu
-; CHECK-NEXT:    vslidedown.vx v8, v12, a0
-; CHECK-NEXT:    vrgatherei16.vv v8, v10, v14, v0.t
+; CHECK-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
+; CHECK-NEXT:    vid.v v10
+; CHECK-NEXT:    vrsub.vi v10, v10, 3
+; CHECK-NEXT:    vrgather.vv v12, v8, v10
+; CHECK-NEXT:    vrgather.vv v8, v9, v10
+; CHECK-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
+; CHECK-NEXT:    vslideup.vi v8, v12, 4
 ; CHECK-NEXT:    ret
   %res = shufflevector <4 x float> %a, <4 x float> %b, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
   ret <8 x float> %res
@@ -1295,26 +1213,23 @@ define <8 x float> @reverse_v8f32_2(<4 x float> %a, <4 x float> %b) {
 define <16 x float> @reverse_v16f32_2(<8 x float> %a, <8 x float> %b) {
 ; CHECK-LABEL: reverse_v16f32_2:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a0, zero, e32, m1, ta, ma
-; CHECK-NEXT:    vmv2r.v v12, v10
 ; CHECK-NEXT:    csrr a0, vlenb
-; CHECK-NEXT:    vid.v v10
+; CHECK-NEXT:    vsetvli a1, zero, e32, m1, ta, ma
+; CHECK-NEXT:    vid.v v12
 ; CHECK-NEXT:    srli a1, a0, 2
+; CHECK-NEXT:    srli a0, a0, 1
 ; CHECK-NEXT:    addi a1, a1, -1
-; CHECK-NEXT:    vrsub.vx v14, v10, a1
-; CHECK-NEXT:    vrgather.vv v11, v8, v14
-; CHECK-NEXT:    vrgather.vv v10, v9, v14
-; CHECK-NEXT:    vrgather.vv v8, v9, v14
-; CHECK-NEXT:    vsetivli zero, 16, e16, m2, ta, ma
-; CHECK-NEXT:    vid.v v14
-; CHECK-NEXT:    li a1, 255
-; CHECK-NEXT:    addi a0, a0, -16
-; CHECK-NEXT:    vrsub.vi v16, v14, 7
-; CHECK-NEXT:    vmv.s.x v0, a1
-; CHECK-NEXT:    vmv1r.v v9, v8
-; CHECK-NEXT:    vsetvli zero, zero, e32, m4, ta, mu
-; CHECK-NEXT:    vslidedown.vx v8, v8, a0
-; CHECK-NEXT:    vrgatherei16.vv v8, v12, v16, v0.t
+; CHECK-NEXT:    addi a0, a0, -8
+; CHECK-NEXT:    vrsub.vx v12, v12, a1
+; CHECK-NEXT:    vrgather.vv v15, v8, v12
+; CHECK-NEXT:    vrgather.vv v17, v10, v12
+; CHECK-NEXT:    vrgather.vv v14, v9, v12
+; CHECK-NEXT:    vrgather.vv v16, v11, v12
+; CHECK-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
+; CHECK-NEXT:    vslidedown.vx v12, v14, a0
+; CHECK-NEXT:    vslidedown.vx v8, v16, a0
+; CHECK-NEXT:    vsetivli zero, 16, e32, m4, ta, ma
+; CHECK-NEXT:    vslideup.vi v8, v12, 8
 ; CHECK-NEXT:    ret
   %res = shufflevector <8 x float> %a, <8 x float> %b, <16 x i32> <i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
   ret <16 x float> %res
@@ -1338,28 +1253,23 @@ define <4 x double> @reverse_v4f64_2(<2 x double> %a, < 2 x double> %b) {
 define <8 x double> @reverse_v8f64_2(<4 x double> %a, <4 x double> %b) {
 ; CHECK-LABEL: reverse_v8f64_2:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a0, zero, e64, m1, ta, ma
-; CHECK-NEXT:    vmv2r.v v12, v10
 ; CHECK-NEXT:    csrr a0, vlenb
-; CHECK-NEXT:    vid.v v10
+; CHECK-NEXT:    vsetvli a1, zero, e64, m1, ta, ma
+; CHECK-NEXT:    vid.v v12
 ; CHECK-NEXT:    srli a1, a0, 3
+; CHECK-NEXT:    srli a0, a0, 2
 ; CHECK-NEXT:    addi a1, a1, -1
-; CHECK-NEXT:    vrsub.vx v14, v10, a1
-; CHECK-NEXT:    vrgather.vv v11, v8, v14
-; CHECK-NEXT:    vrgather.vv v10, v9, v14
-; CHECK-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
-; CHECK-NEXT:    vid.v v15
-; CHECK-NEXT:    vsetvli a1, zero, e64, m1, ta, ma
-; CHECK-NEXT:    vrgather.vv v8, v9, v14
-; CHECK-NEXT:    vmv.v.i v0, 15
-; CHECK-NEXT:    srli a0, a0, 1
-; CHECK-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
-; CHECK-NEXT:    vrsub.vi v16, v15, 3
-; CHECK-NEXT:    addi a0, a0, -8
-; CHECK-NEXT:    vmv1r.v v9, v8
-; CHECK-NEXT:    vsetvli zero, zero, e64, m4, ta, mu
-; CHECK-NEXT:    vslidedown.vx v8, v8, a0
-; CHECK-NEXT:    vrgatherei16.vv v8, v12, v16, v0.t
+; CHECK-NEXT:    addi a0, a0, -4
+; CHECK-NEXT:    vrsub.vx v12, v12, a1
+; CHECK-NEXT:    vrgather.vv v15, v8, v12
+; CHECK-NEXT:    vrgather.vv v17, v10, v12
+; CHECK-NEXT:    vrgather.vv v14, v9, v12
+; CHECK-NEXT:    vrgather.vv v16, v11, v12
+; CHECK-NEXT:    vsetivli zero, 4, e64, m2, ta, ma
+; CHECK-NEXT:    vslidedown.vx v12, v14, a0
+; CHECK-NEXT:    vslidedown.vx v8, v16, a0
+; CHECK-NEXT:    vsetivli zero, 8, e64, m4, ta, ma
+; CHECK-NEXT:    vslideup.vi v8, v12, 4
 ; CHECK-NEXT:    ret
   %res = shufflevector <4 x double> %a, <4 x double> %b, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
   ret <8 x double> %res
@@ -1420,6 +1330,48 @@ define <16 x i32> @reverse_v16i32_exact_vlen_256(<16 x i32> %a) vscale_range(4,
   %res = shufflevector <16 x i32> %a, <16 x i32> poison, <16 x i32> <i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
   ret <16 x i32> %res
 }
+
+define <8 x i32> @reverse_v8i32_undef_suffix(<8 x i32> %a) {
+; CHECK-LABEL: reverse_v8i32_undef_suffix:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    csrr a0, vlenb
+; CHECK-NEXT:    vsetvli a1, zero, e32, m1, ta, ma
+; CHECK-NEXT:    vid.v v10
+; CHECK-NEXT:    srli a1, a0, 2
+; CHECK-NEXT:    srli a0, a0, 1
+; CHECK-NEXT:    addi a1, a1, -1
+; CHECK-NEXT:    vrsub.vx v10, v10, a1
+; CHECK-NEXT:    vrgather.vv v13, v8, v10
+; CHECK-NEXT:    vrgather.vv v12, v9, v10
+; CHECK-NEXT:    addi a0, a0, -8
+; CHECK-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
+; CHECK-NEXT:    vslidedown.vx v8, v12, a0
+; CHECK-NEXT:    ret
+  %res = shufflevector <8 x i32> %a, <8 x i32> poison, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 undef, i32 undef, i32 undef, i32 undef>
+  ret <8 x i32> %res
+}
+
+define <8 x i32> @reverse_v8i32_undef_prefix(<8 x i32> %a) {
+; CHECK-LABEL: reverse_v8i32_undef_prefix:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    csrr a0, vlenb
+; CHECK-NEXT:    vsetvli a1, zero, e32, m1, ta, ma
+; CHECK-NEXT:    vid.v v10
+; CHECK-NEXT:    srli a1, a0, 2
+; CHECK-NEXT:    srli a0, a0, 1
+; CHECK-NEXT:    addi a1, a1, -1
+; CHECK-NEXT:    vrsub.vx v10, v10, a1
+; CHECK-NEXT:    vrgather.vv v13, v8, v10
+; CHECK-NEXT:    vrgather.vv v12, v9, v10
+; CHECK-NEXT:    addi a0, a0, -8
+; CHECK-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
+; CHECK-NEXT:    vslidedown.vx v8, v12, a0
+; CHECK-NEXT:    ret
+  %res = shufflevector <8 x i32> %a, <8 x i32> poison, <8 x i32> <i32 undef, i32 undef, i32 undef, i32 undef, i32 3, i32 2, i32 1, i32 0>
+  ret <8 x i32> %res
+}
+
+
 ;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
 ; RV32: {{.*}}
 ; RV32-ZVBB: {{.*}}
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-shuffle-rotate.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-shuffle-rotate.ll
index 464b4eca35aba..86d8a275a9055 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-shuffle-rotate.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-shuffle-rotate.ll
@@ -515,8 +515,10 @@ define <8 x i16> @shuffle_v8i16_as_i64_16(<8 x i16> %v) {
 ; ZVKB-ZVE32X-NEXT:    vsetivli zero, 8, e16, m2, ta, ma
 ; ZVKB-ZVE32X-NEXT:    vle8.v v10, (a0)
 ; ZVKB-ZVE32X-NEXT:    vsext.vf2 v12, v10
+; ZVKB-ZVE32X-NEXT:    vsetvli a0, zero, e16, m1, ta, ma
+; ZVKB-ZVE32X-NEXT:    vrgather.vv v11, v9, v12
 ; ZVKB-ZVE32X-NEXT:    vrgather.vv v10, v8, v12
-; ZVKB-ZVE32X-NEXT:    vmv.v.v v8, v10
+; ZVKB-ZVE32X-NEXT:    vmv2r.v v8, v10
 ; ZVKB-ZVE32X-NEXT:    ret
   %shuffle = shufflevector <8 x i16> %v, <8 x i16> poison, <8 x i32> <i32 1, i32 2, i32 3, i32 0, i32 5, i32 6, i32 7, i32 4>
   ret <8 x i16> %shuffle
@@ -562,9 +564,10 @@ define <8 x i16> @shuffle_v8i16_as_i64_32(<8 x i16> %v) {
 ; ZVKB-ZVE32X-NEXT:    vmv.s.x v10, a0
 ; ZVKB-ZVE32X-NEXT:    vsetivli zero, 4, e16, m1, ta, ma
 ; ZVKB-ZVE32X-NEXT:    vsext.vf2 v12, v10
-; ZVKB-ZVE32X-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
+; ZVKB-ZVE32X-NEXT:    vsetvli a0, zero, e32, m1, ta, ma
+; ZVKB-ZVE32X-NEXT:    vrgatherei16.vv v11, v9, v12
 ; ZVKB-ZVE32X-NEXT:    vrgatherei16.vv v10, v8, v12
-; ZVKB-ZVE32X-NEXT:    vmv.v.v v8, v10
+; ZVKB-ZVE32X-NEXT:    vmv2r.v v8, v10
 ; ZVKB-ZVE32X-NEXT:    ret
   %shuffle = shufflevector <8 x i16> %v, <8 x i16> poison, <8 x i32> <i32 2, i32 3, i32 0, i32 1, i32 6, i32 7, i32 4, i32 5>
   ret <8 x i16> %shuffle
@@ -609,8 +612,10 @@ define <8 x i16> @shuffle_v8i16_as_i64_48(<8 x i16> %v) {
 ; ZVKB-ZVE32X-NEXT:    vsetivli zero, 8, e16, m2, ta, ma
 ; ZVKB-ZVE32X-NEXT:    vle8.v v10, (a0)
 ; ZVKB-ZVE32X-NEXT:    vsext.vf2 v12, v10
+; ZVKB-ZVE32X-NEXT:    vsetvli a0, zero, e16, m1, ta, ma
+; ZVKB-ZVE32X-NEXT:    vrgather.vv v11, v9, v12
 ; ZVKB-ZVE32X-NEXT:    vrgather.vv v10, v8, v12
-; ZVKB-ZVE32X-NEXT:    vmv.v.v v8, v10
+; ZVKB-ZVE32X-NEXT:    vmv2r.v v8, v10
 ; ZVKB-ZVE32X-NEXT:    ret
   %shuffle = shufflevector <8 x i16> %v, <8 x i16> poison, <8 x i32> <i32 3, i32 0, i32 1, i32 2, i32 7, i32 4, i32 5, i32 6>
   ret <8 x i16> %shuffle
@@ -655,9 +660,12 @@ define <8 x i32> @shuffle_v8i32_as_i64(<8 x i32> %v) {
 ; ZVKB-ZVE32X-NEXT:    vsetivli zero, 8, e16, m2, ta, ma
 ; ZVKB-ZVE32X-NEXT:    vle8.v v12, (a0)
 ; ZVKB-ZVE32X-NEXT:    vsext.vf2 v16, v12
-; ZVKB-ZVE32X-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
+; ZVKB-ZVE32X-NEXT:    vsetvli a0, zero, e32, m1, ta, ma
+; ZVKB-ZVE32X-NEXT:    vrgatherei16.vv v13, v9, v16
 ; ZVKB-ZVE32X-NEXT:    vrgatherei16.vv v12, v8, v16
-; ZVKB-ZVE32X-NEXT:    vmv.v.v v8, v12
+; ZVKB-ZVE32X-NEXT:    vrgatherei16.vv v14, v10, v16
+; ZVKB-ZVE32X-NEXT:    vrgatherei16.vv v15, v11, v16
+; ZVKB-ZVE32X-NEXT:    vmv4r.v v8, v12
 ; ZVKB-ZVE32X-NEXT:    ret
   %shuffle = shufflevector <8 x i32> %v, <8 x i32> poison, <8 x i32> <i32 1, i32 0, i32 3, i32 2, i32 5, i32 4, i32 7, i32 6>
   ret <8 x i32> %shuffle
@@ -726,8 +734,10 @@ define <8 x half> @shuffle_v8f16_as_i64_16(<8 x half> %v) {
 ; ZVKB-ZVE32X-NEXT:    vsetivli zero, 8, e16, m2, ta, ma
 ; ZVKB-ZVE32X-NEXT:    vle8.v v10, (a0)
 ; ZVKB-ZVE32X-NEXT:    vsext.vf2 v12, v10
+; ZVKB-ZVE32X-NEXT:    vsetvli a0, zero, e16, m1, ta, ma
+; ZVKB-ZVE32X-NEXT:    vrgather.vv v11, v9, v12
 ; ZVKB-ZVE32X-NEXT:    vrgather.vv v10, v8, v12
-; ZVKB-ZVE32X-NEXT:    vmv.v.v v8, v10
+; ZVKB-ZVE32X-NEXT:    vmv2r.v v8, v10
 ; ZVKB-ZVE32X-NEXT:    ret
   %shuffle = shufflevector <8 x half> %v, <8 x half> poison, <8 x i32> <i32 1, i32 2, i32 3, i32 0, i32 5, i32 6, i32 7, i32 4>
   ret <8 x half> %shuffle
@@ -773,9 +783,10 @@ define <8 x half> @shuffle_v8f16_as_i64_32(<8 x half> %v) {
 ; ZVKB-ZVE32X-NEXT:    vmv.s.x v10, a0
 ; ZVKB-ZVE32X-NEXT:    vsetivli zero, 4, e16, m1, ta, ma
 ; ZVKB-ZVE32X-NEXT:    vsext.vf2 v12, v10
-; ZVKB-ZVE32X-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
+; ZVKB-ZVE32X-NEXT:    vsetvli a0, zero, e32, m1, ta, ma
+; ZVKB-ZVE32X-NEXT:    vrgatherei16.vv v11, v9, v12
 ; ZVKB-ZVE32X-NEXT:    vrgatherei16.vv v10, v8, v12
-; ZVKB-ZVE32X-NEXT:    vmv.v.v v8, v10
+; ZVKB-ZVE32X-NEXT:    vmv2r.v v8, v10
 ; ZVKB-ZVE32X-NEXT:    ret
   %shuffle = shufflevector <8 x half> %v, <8 x half> poison, <8 x i32> <i32 2, i32 3, i32 0, i32 1, i32 6, i32 7, i32 4, i32 5>
   ret <8 x half> %shuffle
@@ -820,8 +831,10 @@ define <8 x half> @shuffle_v8f16_as_i64_48(<8 x half> %v) {
 ; ZVKB-ZVE32X-NEXT:    vsetivli zero, 8, e16, m2, ta, ma
 ; ZVKB-ZVE32X-NEXT:    vle8.v v10, (a0)
 ; ZVKB-ZVE32X-NEXT:    vsext.vf2 v12, v10
+; ZVKB-ZVE32X-NEXT:    vsetvli a0, zero, e16, m1, ta, ma
+; ZVKB-ZVE32X-NEXT:    vrgather.vv v11, v9, v12
 ; ZVKB-ZVE32X-NEXT:    vrgather.vv v10, v8, v12
-; ZVKB-ZVE32X-NEXT:    vmv.v.v v8, v10
+; ZVKB-ZVE32X-NEXT:    vmv2r.v v8, v10
 ; ZVKB-ZVE32X-NEXT:    ret
   %shuffle = shufflevector <8 x half> %v, <8 x half> poison, <8 x i32> <i32 3, i32 0, i32 1, i32 2, i32 7, i32 4, i32 5, i32 6>
   ret <8 x half> %shuffle
@@ -866,9 +879,12 @@ define <8 x float> @shuffle_v8f32_as_i64(<8 x float> %v) {
 ; ZVKB-ZVE32X-NEXT:    vsetivli zero, 8, e16, m2, ta, ma
 ; ZVKB-ZVE32X-NEXT:    vle8.v v12, (a0)
 ; ZVKB-ZVE32X-NEXT:    vsext.vf2 v16, v12
-; ZVKB-ZVE32X-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
+; ZVKB-ZVE32X-NEXT:    vsetvli a0, zero, e32, m1, ta, ma
+; ZVKB-ZVE32X-NEXT:    vrgatherei16.vv v13, v9, v16
 ; ZVKB-ZVE32X-NEXT:    vrgatherei16.vv v12, v8, v16
-; ZVKB-ZVE32X-NEXT:    vmv.v.v v8, v12
+; ZVKB-ZVE32X-NEXT:    vrgatherei16.vv v14, v10, v16
+; ZVKB-ZVE32X-NEXT:    vrgatherei16.vv v15, v11, v16
+; ZVKB-ZVE32X-NEXT:    vmv4r.v v8, v12
 ; ZVKB-ZVE32X-NEXT:    ret
   %shuffle = shufflevector <8 x float> %v, <8 x float> poison, <8 x i32> <i32 1, i32 0, i32 3, i32 2, i32 5, i32 4, i32 7, i32 6>
   ret <8 x float> %shuffle
@@ -920,3 +936,87 @@ define <8 x float> @shuffle_v8f32_as_i64_exact(<8 x float> %v) vscale_range(2,2)
   %shuffle = shufflevector <8 x float> %v, <8 x float> poison, <8 x i32> <i32 1, i32 0, i32 3, i32 2, i32 5, i32 4, i32 7, i32 6>
   ret <8 x float> %shuffle
 }
+
+define <8 x i64> @shuffle_v8i64_as_i128(<8 x i64> %v) {
+; CHECK-LABEL: shuffle_v8i64_as_i128:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lui a0, %hi(.LCPI29_0)
+; CHECK-NEXT:    addi a0, a0, %lo(.LCPI29_0)
+; CHECK-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; CHECK-NEXT:    vle16.v v16, (a0)
+; CHECK-NEXT:    vsetvli a0, zero, e64, m1, ta, ma
+; CHECK-NEXT:    vrgatherei16.vv v13, v9, v16
+; CHECK-NEXT:    vrgatherei16.vv v12, v8, v16
+; CHECK-NEXT:    vrgatherei16.vv v14, v10, v16
+; CHECK-NEXT:    vrgatherei16.vv v15, v11, v16
+; CHECK-NEXT:    vmv4r.v v8, v12
+; CHECK-NEXT:    ret
+;
+; ZVKB-V-LABEL: shuffle_v8i64_as_i128:
+; ZVKB-V:       # %bb.0:
+; ZVKB-V-NEXT:    lui a0, %hi(.LCPI29_0)
+; ZVKB-V-NEXT:    addi a0, a0, %lo(.LCPI29_0)
+; ZVKB-V-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; ZVKB-V-NEXT:    vle16.v v16, (a0)
+; ZVKB-V-NEXT:    vsetvli a0, zero, e64, m1, ta, ma
+; ZVKB-V-NEXT:    vrgatherei16.vv v13, v9, v16
+; ZVKB-V-NEXT:    vrgatherei16.vv v12, v8, v16
+; ZVKB-V-NEXT:    vrgatherei16.vv v14, v10, v16
+; ZVKB-V-NEXT:    vrgatherei16.vv v15, v11, v16
+; ZVKB-V-NEXT:    vmv4r.v v8, v12
+; ZVKB-V-NEXT:    ret
+  %shuffle = shufflevector <8 x i64> %v, <8 x i64> poison, <8 x i32> <i32 1, i32 0, i32 3, i32 2, i32 5, i32 4, i32 7, i32 6>
+  ret <8 x i64> %shuffle
+}
+
+define <8 x i64> @shuffle_v8i64_as_i256(<8 x i64> %v) {
+; CHECK-LABEL: shuffle_v8i64_as_i256:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lui a0, %hi(.LCPI30_0)
+; CHECK-NEXT:    addi a0, a0, %lo(.LCPI30_0)
+; CHECK-NEXT:    vsetivli zero, 8, e64, m4, ta, ma
+; CHECK-NEXT:    vle16.v v16, (a0)
+; CHECK-NEXT:    vrgatherei16.vv v12, v8, v16
+; CHECK-NEXT:    vmv.v.v v8, v12
+; CHECK-NEXT:    ret
+;
+; ZVKB-V-LABEL: shuffle_v8i64_as_i256:
+; ZVKB-V:       # %bb.0:
+; ZVKB-V-NEXT:    lui a0, %hi(.LCPI30_0)
+; ZVKB-V-NEXT:    addi a0, a0, %lo(.LCPI30_0)
+; ZVKB-V-NEXT:    vsetivli zero, 8, e64, m4, ta, ma
+; ZVKB-V-NEXT:    vle16.v v16, (a0)
+; ZVKB-V-NEXT:    vrgatherei16.vv v12, v8, v16
+; ZVKB-V-NEXT:    vmv.v.v v8, v12
+; ZVKB-V-NEXT:    ret
+  %shuffle = shufflevector <8 x i64> %v, <8 x i64> poison, <8 x i32> <i32 2, i32 1, i32 0, i32 3, i32 6, i32 5, i32 4, i32 7>
+  ret <8 x i64> %shuffle
+}
+
+define <8 x i64> @shuffle_v8i64_as_i256_zvl256b(<8 x i64> %v) vscale_range(4,0) {
+; CHECK-LABEL: shuffle_v8i64_as_i256_zvl256b:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lui a0, %hi(.LCPI31_0)
+; CHECK-NEXT:    addi a0, a0, %lo(.LCPI31_0)
+; CHECK-NEXT:    vsetivli zero, 8, e16, mf2, ta, ma
+; CHECK-NEXT:    vle16.v v12, (a0)
+; CHECK-NEXT:    vsetvli a0, zero, e64, m1, ta, ma
+; CHECK-NEXT:    vrgatherei16.vv v11, v9, v12
+; CHECK-NEXT:    vrgatherei16.vv v10, v8, v12
+; CHECK-NEXT:    vmv2r.v v8, v10
+; CHECK-NEXT:    ret
+;
+; ZVKB-V-LABEL: shuffle_v8i64_as_i256_zvl256b:
+; ZVKB-V:       # %bb.0:
+; ZVKB-V-NEXT:    lui a0, %hi(.LCPI31_0)
+; ZVKB-V-NEXT:    addi a0, a0, %lo(.LCPI31_0)
+; ZVKB-V-NEXT:    vsetivli zero, 8, e16, mf2, ta, ma
+; ZVKB-V-NEXT:    vle16.v v12, (a0)
+; ZVKB-V-NEXT:    vsetvli a0, zero, e64, m1, ta, ma
+; ZVKB-V-NEXT:    vrgatherei16.vv v11, v9, v12
+; ZVKB-V-NEXT:    vrgatherei16.vv v10, v8, v12
+; ZVKB-V-NEXT:    vmv2r.v v8, v10
+; ZVKB-V-NEXT:    ret
+  %shuffle = shufflevector <8 x i64> %v, <8 x i64> poison, <8 x i32> <i32 2, i32 1, i32 0, i32 3, i32 6, i32 5, i32 4, i32 7>
+  ret <8 x i64> %shuffle
+}
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-shufflevector-vnsrl.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-shufflevector-vnsrl.ll
index 847ef9a7b3601..3c28e978842b9 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-shufflevector-vnsrl.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-shufflevector-vnsrl.ll
@@ -525,13 +525,10 @@ define void @vnsrl_0_i32_single_src_m8(ptr %in, ptr %out) {
 ; V-NEXT:    li a2, 64
 ; V-NEXT:    vsetvli zero, a2, e32, m8, ta, ma
 ; V-NEXT:    vle32.v v8, (a0)
-; V-NEXT:    lui a0, 341
-; V-NEXT:    addiw a0, a0, 1365
-; V-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
-; V-NEXT:    vmv.s.x v16, a0
+; V-NEXT:    vsetivli zero, 16, e32, m2, ta, ma
+; V-NEXT:    vnsrl.wi v16, v8, 0
 ; V-NEXT:    vsetvli zero, a2, e32, m8, ta, ma
-; V-NEXT:    vcompress.vm v24, v8, v16
-; V-NEXT:    vse32.v v24, (a1)
+; V-NEXT:    vse32.v v16, (a1)
 ; V-NEXT:    ret
 ;
 ; ZVE32F-LABEL: vnsrl_0_i32_single_src_m8:
@@ -539,15 +536,14 @@ define void @vnsrl_0_i32_single_src_m8(ptr %in, ptr %out) {
 ; ZVE32F-NEXT:    li a2, 64
 ; ZVE32F-NEXT:    vsetvli zero, a2, e32, m8, ta, ma
 ; ZVE32F-NEXT:    vle32.v v8, (a0)
-; ZVE32F-NEXT:    vsetivli zero, 2, e32, m1, ta, ma
-; ZVE32F-NEXT:    vmv.v.i v16, 0
 ; ZVE32F-NEXT:    lui a0, 341
 ; ZVE32F-NEXT:    addi a0, a0, 1365
-; ZVE32F-NEXT:    vsetvli zero, zero, e32, m1, tu, ma
-; ZVE32F-NEXT:    vmv.s.x v16, a0
+; ZVE32F-NEXT:    vmv.s.x v12, a0
+; ZVE32F-NEXT:    li a0, 32
+; ZVE32F-NEXT:    vsetvli zero, a0, e32, m4, ta, ma
+; ZVE32F-NEXT:    vcompress.vm v16, v8, v12
 ; ZVE32F-NEXT:    vsetvli zero, a2, e32, m8, ta, ma
-; ZVE32F-NEXT:    vcompress.vm v24, v8, v16
-; ZVE32F-NEXT:    vse32.v v24, (a1)
+; ZVE32F-NEXT:    vse32.v v16, (a1)
 ; ZVE32F-NEXT:    ret
 entry:
   %0 = load <64 x i32>, ptr %in, align 4
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vpgather.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vpgather.ll
index df9ff0fc39a7e..1f6513ae09d60 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vpgather.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vpgather.ll
@@ -736,18 +736,18 @@ define <8 x i32> @vpgather_baseidx_sext_v8i8_v8i32(ptr %base, <8 x i8> %idxs, <8
 define <8 x i32> @vpgather_baseidx_zext_v8i8_v8i32(ptr %base, <8 x i8> %idxs, <8 x i1> %m, i32 zeroext %evl) {
 ; RV32-LABEL: vpgather_baseidx_zext_v8i8_v8i32:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
-; RV32-NEXT:    vzext.vf2 v9, v8
-; RV32-NEXT:    vsll.vi v10, v9, 2
+; RV32-NEXT:    li a2, 4
+; RV32-NEXT:    vsetivli zero, 8, e8, mf2, ta, ma
+; RV32-NEXT:    vwmulu.vx v10, v8, a2
 ; RV32-NEXT:    vsetvli zero, a1, e32, m2, ta, ma
 ; RV32-NEXT:    vluxei16.v v8, (a0), v10, v0.t
 ; RV32-NEXT:    ret
 ;
 ; RV64-LABEL: vpgather_baseidx_zext_v8i8_v8i32:
 ; RV64:       # %bb.0:
-; RV64-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
-; RV64-NEXT:    vzext.vf2 v9, v8
-; RV64-NEXT:    vsll.vi v10, v9, 2
+; RV64-NEXT:    li a2, 4
+; RV64-NEXT:    vsetivli zero, 8, e8, mf2, ta, ma
+; RV64-NEXT:    vwmulu.vx v10, v8, a2
 ; RV64-NEXT:    vsetvli zero, a1, e32, m2, ta, ma
 ; RV64-NEXT:    vluxei16.v v8, (a0), v10, v0.t
 ; RV64-NEXT:    ret
@@ -760,11 +760,11 @@ define <8 x i32> @vpgather_baseidx_zext_v8i8_v8i32(ptr %base, <8 x i8> %idxs, <8
 define <8 x i32> @vpgather_baseidx_v8i16_v8i32(ptr %base, <8 x i16> %idxs, <8 x i1> %m, i32 zeroext %evl) {
 ; RV32-LABEL: vpgather_baseidx_v8i16_v8i32:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
-; RV32-NEXT:    vsext.vf2 v10, v8
-; RV32-NEXT:    vsll.vi v8, v10, 2
+; RV32-NEXT:    li a2, 4
+; RV32-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; RV32-NEXT:    vwmulsu.vx v10, v8, a2
 ; RV32-NEXT:    vsetvli zero, a1, e32, m2, ta, ma
-; RV32-NEXT:    vluxei32.v v8, (a0), v8, v0.t
+; RV32-NEXT:    vluxei32.v v8, (a0), v10, v0.t
 ; RV32-NEXT:    ret
 ;
 ; RV64-LABEL: vpgather_baseidx_v8i16_v8i32:
@@ -783,11 +783,11 @@ define <8 x i32> @vpgather_baseidx_v8i16_v8i32(ptr %base, <8 x i16> %idxs, <8 x
 define <8 x i32> @vpgather_baseidx_sext_v8i16_v8i32(ptr %base, <8 x i16> %idxs, <8 x i1> %m, i32 zeroext %evl) {
 ; RV32-LABEL: vpgather_baseidx_sext_v8i16_v8i32:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
-; RV32-NEXT:    vsext.vf2 v10, v8
-; RV32-NEXT:    vsll.vi v8, v10, 2
+; RV32-NEXT:    li a2, 4
+; RV32-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; RV32-NEXT:    vwmulsu.vx v10, v8, a2
 ; RV32-NEXT:    vsetvli zero, a1, e32, m2, ta, ma
-; RV32-NEXT:    vluxei32.v v8, (a0), v8, v0.t
+; RV32-NEXT:    vluxei32.v v8, (a0), v10, v0.t
 ; RV32-NEXT:    ret
 ;
 ; RV64-LABEL: vpgather_baseidx_sext_v8i16_v8i32:
@@ -807,20 +807,20 @@ define <8 x i32> @vpgather_baseidx_sext_v8i16_v8i32(ptr %base, <8 x i16> %idxs,
 define <8 x i32> @vpgather_baseidx_zext_v8i16_v8i32(ptr %base, <8 x i16> %idxs, <8 x i1> %m, i32 zeroext %evl) {
 ; RV32-LABEL: vpgather_baseidx_zext_v8i16_v8i32:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
-; RV32-NEXT:    vzext.vf2 v10, v8
-; RV32-NEXT:    vsll.vi v8, v10, 2
+; RV32-NEXT:    li a2, 4
+; RV32-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; RV32-NEXT:    vwmulu.vx v10, v8, a2
 ; RV32-NEXT:    vsetvli zero, a1, e32, m2, ta, ma
-; RV32-NEXT:    vluxei32.v v8, (a0), v8, v0.t
+; RV32-NEXT:    vluxei32.v v8, (a0), v10, v0.t
 ; RV32-NEXT:    ret
 ;
 ; RV64-LABEL: vpgather_baseidx_zext_v8i16_v8i32:
 ; RV64:       # %bb.0:
-; RV64-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
-; RV64-NEXT:    vzext.vf2 v10, v8
-; RV64-NEXT:    vsll.vi v8, v10, 2
+; RV64-NEXT:    li a2, 4
+; RV64-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; RV64-NEXT:    vwmulu.vx v10, v8, a2
 ; RV64-NEXT:    vsetvli zero, a1, e32, m2, ta, ma
-; RV64-NEXT:    vluxei32.v v8, (a0), v8, v0.t
+; RV64-NEXT:    vluxei32.v v8, (a0), v10, v0.t
 ; RV64-NEXT:    ret
   %eidxs = zext <8 x i16> %idxs to <8 x i32>
   %ptrs = getelementptr inbounds i32, ptr %base, <8 x i32> %eidxs
@@ -839,9 +839,9 @@ define <8 x i32> @vpgather_baseidx_v8i32(ptr %base, <8 x i32> %idxs, <8 x i1> %m
 ;
 ; RV64-LABEL: vpgather_baseidx_v8i32:
 ; RV64:       # %bb.0:
-; RV64-NEXT:    vsetivli zero, 8, e64, m4, ta, ma
-; RV64-NEXT:    vsext.vf2 v12, v8
-; RV64-NEXT:    vsll.vi v12, v12, 2
+; RV64-NEXT:    li a2, 4
+; RV64-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
+; RV64-NEXT:    vwmulsu.vx v12, v8, a2
 ; RV64-NEXT:    vsetvli zero, a1, e32, m2, ta, ma
 ; RV64-NEXT:    vluxei64.v v8, (a0), v12, v0.t
 ; RV64-NEXT:    ret
@@ -974,18 +974,18 @@ define <8 x i64> @vpgather_baseidx_sext_v8i8_v8i64(ptr %base, <8 x i8> %idxs, <8
 define <8 x i64> @vpgather_baseidx_zext_v8i8_v8i64(ptr %base, <8 x i8> %idxs, <8 x i1> %m, i32 zeroext %evl) {
 ; RV32-LABEL: vpgather_baseidx_zext_v8i8_v8i64:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
-; RV32-NEXT:    vzext.vf2 v9, v8
-; RV32-NEXT:    vsll.vi v12, v9, 3
+; RV32-NEXT:    li a2, 8
+; RV32-NEXT:    vsetivli zero, 8, e8, mf2, ta, ma
+; RV32-NEXT:    vwmulu.vx v12, v8, a2
 ; RV32-NEXT:    vsetvli zero, a1, e64, m4, ta, ma
 ; RV32-NEXT:    vluxei16.v v8, (a0), v12, v0.t
 ; RV32-NEXT:    ret
 ;
 ; RV64-LABEL: vpgather_baseidx_zext_v8i8_v8i64:
 ; RV64:       # %bb.0:
-; RV64-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
-; RV64-NEXT:    vzext.vf2 v9, v8
-; RV64-NEXT:    vsll.vi v12, v9, 3
+; RV64-NEXT:    li a2, 8
+; RV64-NEXT:    vsetivli zero, 8, e8, mf2, ta, ma
+; RV64-NEXT:    vwmulu.vx v12, v8, a2
 ; RV64-NEXT:    vsetvli zero, a1, e64, m4, ta, ma
 ; RV64-NEXT:    vluxei16.v v8, (a0), v12, v0.t
 ; RV64-NEXT:    ret
@@ -998,9 +998,9 @@ define <8 x i64> @vpgather_baseidx_zext_v8i8_v8i64(ptr %base, <8 x i8> %idxs, <8
 define <8 x i64> @vpgather_baseidx_v8i16_v8i64(ptr %base, <8 x i16> %idxs, <8 x i1> %m, i32 zeroext %evl) {
 ; RV32-LABEL: vpgather_baseidx_v8i16_v8i64:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
-; RV32-NEXT:    vsext.vf2 v10, v8
-; RV32-NEXT:    vsll.vi v12, v10, 3
+; RV32-NEXT:    li a2, 8
+; RV32-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; RV32-NEXT:    vwmulsu.vx v12, v8, a2
 ; RV32-NEXT:    vsetvli zero, a1, e64, m4, ta, ma
 ; RV32-NEXT:    vluxei32.v v8, (a0), v12, v0.t
 ; RV32-NEXT:    ret
@@ -1021,9 +1021,9 @@ define <8 x i64> @vpgather_baseidx_v8i16_v8i64(ptr %base, <8 x i16> %idxs, <8 x
 define <8 x i64> @vpgather_baseidx_sext_v8i16_v8i64(ptr %base, <8 x i16> %idxs, <8 x i1> %m, i32 zeroext %evl) {
 ; RV32-LABEL: vpgather_baseidx_sext_v8i16_v8i64:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
-; RV32-NEXT:    vsext.vf2 v10, v8
-; RV32-NEXT:    vsll.vi v12, v10, 3
+; RV32-NEXT:    li a2, 8
+; RV32-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; RV32-NEXT:    vwmulsu.vx v12, v8, a2
 ; RV32-NEXT:    vsetvli zero, a1, e64, m4, ta, ma
 ; RV32-NEXT:    vluxei32.v v8, (a0), v12, v0.t
 ; RV32-NEXT:    ret
@@ -1045,18 +1045,18 @@ define <8 x i64> @vpgather_baseidx_sext_v8i16_v8i64(ptr %base, <8 x i16> %idxs,
 define <8 x i64> @vpgather_baseidx_zext_v8i16_v8i64(ptr %base, <8 x i16> %idxs, <8 x i1> %m, i32 zeroext %evl) {
 ; RV32-LABEL: vpgather_baseidx_zext_v8i16_v8i64:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
-; RV32-NEXT:    vzext.vf2 v10, v8
-; RV32-NEXT:    vsll.vi v12, v10, 3
+; RV32-NEXT:    li a2, 8
+; RV32-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; RV32-NEXT:    vwmulu.vx v12, v8, a2
 ; RV32-NEXT:    vsetvli zero, a1, e64, m4, ta, ma
 ; RV32-NEXT:    vluxei32.v v8, (a0), v12, v0.t
 ; RV32-NEXT:    ret
 ;
 ; RV64-LABEL: vpgather_baseidx_zext_v8i16_v8i64:
 ; RV64:       # %bb.0:
-; RV64-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
-; RV64-NEXT:    vzext.vf2 v10, v8
-; RV64-NEXT:    vsll.vi v12, v10, 3
+; RV64-NEXT:    li a2, 8
+; RV64-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; RV64-NEXT:    vwmulu.vx v12, v8, a2
 ; RV64-NEXT:    vsetvli zero, a1, e64, m4, ta, ma
 ; RV64-NEXT:    vluxei32.v v8, (a0), v12, v0.t
 ; RV64-NEXT:    ret
@@ -1077,11 +1077,11 @@ define <8 x i64> @vpgather_baseidx_v8i32_v8i64(ptr %base, <8 x i32> %idxs, <8 x
 ;
 ; RV64-LABEL: vpgather_baseidx_v8i32_v8i64:
 ; RV64:       # %bb.0:
-; RV64-NEXT:    vsetivli zero, 8, e64, m4, ta, ma
-; RV64-NEXT:    vsext.vf2 v12, v8
-; RV64-NEXT:    vsll.vi v8, v12, 3
+; RV64-NEXT:    li a2, 8
+; RV64-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
+; RV64-NEXT:    vwmulsu.vx v12, v8, a2
 ; RV64-NEXT:    vsetvli zero, a1, e64, m4, ta, ma
-; RV64-NEXT:    vluxei64.v v8, (a0), v8, v0.t
+; RV64-NEXT:    vluxei64.v v8, (a0), v12, v0.t
 ; RV64-NEXT:    ret
   %ptrs = getelementptr inbounds i64, ptr %base, <8 x i32> %idxs
   %v = call <8 x i64> @llvm.vp.gather.v8i64.v8p0(<8 x ptr> %ptrs, <8 x i1> %m, i32 %evl)
@@ -1099,11 +1099,11 @@ define <8 x i64> @vpgather_baseidx_sext_v8i32_v8i64(ptr %base, <8 x i32> %idxs,
 ;
 ; RV64-LABEL: vpgather_baseidx_sext_v8i32_v8i64:
 ; RV64:       # %bb.0:
-; RV64-NEXT:    vsetivli zero, 8, e64, m4, ta, ma
-; RV64-NEXT:    vsext.vf2 v12, v8
-; RV64-NEXT:    vsll.vi v8, v12, 3
+; RV64-NEXT:    li a2, 8
+; RV64-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
+; RV64-NEXT:    vwmulsu.vx v12, v8, a2
 ; RV64-NEXT:    vsetvli zero, a1, e64, m4, ta, ma
-; RV64-NEXT:    vluxei64.v v8, (a0), v8, v0.t
+; RV64-NEXT:    vluxei64.v v8, (a0), v12, v0.t
 ; RV64-NEXT:    ret
   %eidxs = sext <8 x i32> %idxs to <8 x i64>
   %ptrs = getelementptr inbounds i64, ptr %base, <8 x i64> %eidxs
@@ -1122,11 +1122,11 @@ define <8 x i64> @vpgather_baseidx_zext_v8i32_v8i64(ptr %base, <8 x i32> %idxs,
 ;
 ; RV64-LABEL: vpgather_baseidx_zext_v8i32_v8i64:
 ; RV64:       # %bb.0:
-; RV64-NEXT:    vsetivli zero, 8, e64, m4, ta, ma
-; RV64-NEXT:    vzext.vf2 v12, v8
-; RV64-NEXT:    vsll.vi v8, v12, 3
+; RV64-NEXT:    li a2, 8
+; RV64-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
+; RV64-NEXT:    vwmulu.vx v12, v8, a2
 ; RV64-NEXT:    vsetvli zero, a1, e64, m4, ta, ma
-; RV64-NEXT:    vluxei64.v v8, (a0), v8, v0.t
+; RV64-NEXT:    vluxei64.v v8, (a0), v12, v0.t
 ; RV64-NEXT:    ret
   %eidxs = zext <8 x i32> %idxs to <8 x i64>
   %ptrs = getelementptr inbounds i64, ptr %base, <8 x i64> %eidxs
@@ -1618,18 +1618,18 @@ define <8 x float> @vpgather_baseidx_sext_v8i8_v8f32(ptr %base, <8 x i8> %idxs,
 define <8 x float> @vpgather_baseidx_zext_v8i8_v8f32(ptr %base, <8 x i8> %idxs, <8 x i1> %m, i32 zeroext %evl) {
 ; RV32-LABEL: vpgather_baseidx_zext_v8i8_v8f32:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
-; RV32-NEXT:    vzext.vf2 v9, v8
-; RV32-NEXT:    vsll.vi v10, v9, 2
+; RV32-NEXT:    li a2, 4
+; RV32-NEXT:    vsetivli zero, 8, e8, mf2, ta, ma
+; RV32-NEXT:    vwmulu.vx v10, v8, a2
 ; RV32-NEXT:    vsetvli zero, a1, e32, m2, ta, ma
 ; RV32-NEXT:    vluxei16.v v8, (a0), v10, v0.t
 ; RV32-NEXT:    ret
 ;
 ; RV64-LABEL: vpgather_baseidx_zext_v8i8_v8f32:
 ; RV64:       # %bb.0:
-; RV64-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
-; RV64-NEXT:    vzext.vf2 v9, v8
-; RV64-NEXT:    vsll.vi v10, v9, 2
+; RV64-NEXT:    li a2, 4
+; RV64-NEXT:    vsetivli zero, 8, e8, mf2, ta, ma
+; RV64-NEXT:    vwmulu.vx v10, v8, a2
 ; RV64-NEXT:    vsetvli zero, a1, e32, m2, ta, ma
 ; RV64-NEXT:    vluxei16.v v8, (a0), v10, v0.t
 ; RV64-NEXT:    ret
@@ -1642,11 +1642,11 @@ define <8 x float> @vpgather_baseidx_zext_v8i8_v8f32(ptr %base, <8 x i8> %idxs,
 define <8 x float> @vpgather_baseidx_v8i16_v8f32(ptr %base, <8 x i16> %idxs, <8 x i1> %m, i32 zeroext %evl) {
 ; RV32-LABEL: vpgather_baseidx_v8i16_v8f32:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
-; RV32-NEXT:    vsext.vf2 v10, v8
-; RV32-NEXT:    vsll.vi v8, v10, 2
+; RV32-NEXT:    li a2, 4
+; RV32-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; RV32-NEXT:    vwmulsu.vx v10, v8, a2
 ; RV32-NEXT:    vsetvli zero, a1, e32, m2, ta, ma
-; RV32-NEXT:    vluxei32.v v8, (a0), v8, v0.t
+; RV32-NEXT:    vluxei32.v v8, (a0), v10, v0.t
 ; RV32-NEXT:    ret
 ;
 ; RV64-LABEL: vpgather_baseidx_v8i16_v8f32:
@@ -1665,11 +1665,11 @@ define <8 x float> @vpgather_baseidx_v8i16_v8f32(ptr %base, <8 x i16> %idxs, <8
 define <8 x float> @vpgather_baseidx_sext_v8i16_v8f32(ptr %base, <8 x i16> %idxs, <8 x i1> %m, i32 zeroext %evl) {
 ; RV32-LABEL: vpgather_baseidx_sext_v8i16_v8f32:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
-; RV32-NEXT:    vsext.vf2 v10, v8
-; RV32-NEXT:    vsll.vi v8, v10, 2
+; RV32-NEXT:    li a2, 4
+; RV32-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; RV32-NEXT:    vwmulsu.vx v10, v8, a2
 ; RV32-NEXT:    vsetvli zero, a1, e32, m2, ta, ma
-; RV32-NEXT:    vluxei32.v v8, (a0), v8, v0.t
+; RV32-NEXT:    vluxei32.v v8, (a0), v10, v0.t
 ; RV32-NEXT:    ret
 ;
 ; RV64-LABEL: vpgather_baseidx_sext_v8i16_v8f32:
@@ -1689,20 +1689,20 @@ define <8 x float> @vpgather_baseidx_sext_v8i16_v8f32(ptr %base, <8 x i16> %idxs
 define <8 x float> @vpgather_baseidx_zext_v8i16_v8f32(ptr %base, <8 x i16> %idxs, <8 x i1> %m, i32 zeroext %evl) {
 ; RV32-LABEL: vpgather_baseidx_zext_v8i16_v8f32:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
-; RV32-NEXT:    vzext.vf2 v10, v8
-; RV32-NEXT:    vsll.vi v8, v10, 2
+; RV32-NEXT:    li a2, 4
+; RV32-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; RV32-NEXT:    vwmulu.vx v10, v8, a2
 ; RV32-NEXT:    vsetvli zero, a1, e32, m2, ta, ma
-; RV32-NEXT:    vluxei32.v v8, (a0), v8, v0.t
+; RV32-NEXT:    vluxei32.v v8, (a0), v10, v0.t
 ; RV32-NEXT:    ret
 ;
 ; RV64-LABEL: vpgather_baseidx_zext_v8i16_v8f32:
 ; RV64:       # %bb.0:
-; RV64-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
-; RV64-NEXT:    vzext.vf2 v10, v8
-; RV64-NEXT:    vsll.vi v8, v10, 2
+; RV64-NEXT:    li a2, 4
+; RV64-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; RV64-NEXT:    vwmulu.vx v10, v8, a2
 ; RV64-NEXT:    vsetvli zero, a1, e32, m2, ta, ma
-; RV64-NEXT:    vluxei32.v v8, (a0), v8, v0.t
+; RV64-NEXT:    vluxei32.v v8, (a0), v10, v0.t
 ; RV64-NEXT:    ret
   %eidxs = zext <8 x i16> %idxs to <8 x i32>
   %ptrs = getelementptr inbounds float, ptr %base, <8 x i32> %eidxs
@@ -1721,9 +1721,9 @@ define <8 x float> @vpgather_baseidx_v8f32(ptr %base, <8 x i32> %idxs, <8 x i1>
 ;
 ; RV64-LABEL: vpgather_baseidx_v8f32:
 ; RV64:       # %bb.0:
-; RV64-NEXT:    vsetivli zero, 8, e64, m4, ta, ma
-; RV64-NEXT:    vsext.vf2 v12, v8
-; RV64-NEXT:    vsll.vi v12, v12, 2
+; RV64-NEXT:    li a2, 4
+; RV64-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
+; RV64-NEXT:    vwmulsu.vx v12, v8, a2
 ; RV64-NEXT:    vsetvli zero, a1, e32, m2, ta, ma
 ; RV64-NEXT:    vluxei64.v v8, (a0), v12, v0.t
 ; RV64-NEXT:    ret
@@ -1856,18 +1856,18 @@ define <8 x double> @vpgather_baseidx_sext_v8i8_v8f64(ptr %base, <8 x i8> %idxs,
 define <8 x double> @vpgather_baseidx_zext_v8i8_v8f64(ptr %base, <8 x i8> %idxs, <8 x i1> %m, i32 zeroext %evl) {
 ; RV32-LABEL: vpgather_baseidx_zext_v8i8_v8f64:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
-; RV32-NEXT:    vzext.vf2 v9, v8
-; RV32-NEXT:    vsll.vi v12, v9, 3
+; RV32-NEXT:    li a2, 8
+; RV32-NEXT:    vsetivli zero, 8, e8, mf2, ta, ma
+; RV32-NEXT:    vwmulu.vx v12, v8, a2
 ; RV32-NEXT:    vsetvli zero, a1, e64, m4, ta, ma
 ; RV32-NEXT:    vluxei16.v v8, (a0), v12, v0.t
 ; RV32-NEXT:    ret
 ;
 ; RV64-LABEL: vpgather_baseidx_zext_v8i8_v8f64:
 ; RV64:       # %bb.0:
-; RV64-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
-; RV64-NEXT:    vzext.vf2 v9, v8
-; RV64-NEXT:    vsll.vi v12, v9, 3
+; RV64-NEXT:    li a2, 8
+; RV64-NEXT:    vsetivli zero, 8, e8, mf2, ta, ma
+; RV64-NEXT:    vwmulu.vx v12, v8, a2
 ; RV64-NEXT:    vsetvli zero, a1, e64, m4, ta, ma
 ; RV64-NEXT:    vluxei16.v v8, (a0), v12, v0.t
 ; RV64-NEXT:    ret
@@ -1880,9 +1880,9 @@ define <8 x double> @vpgather_baseidx_zext_v8i8_v8f64(ptr %base, <8 x i8> %idxs,
 define <8 x double> @vpgather_baseidx_v8i16_v8f64(ptr %base, <8 x i16> %idxs, <8 x i1> %m, i32 zeroext %evl) {
 ; RV32-LABEL: vpgather_baseidx_v8i16_v8f64:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
-; RV32-NEXT:    vsext.vf2 v10, v8
-; RV32-NEXT:    vsll.vi v12, v10, 3
+; RV32-NEXT:    li a2, 8
+; RV32-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; RV32-NEXT:    vwmulsu.vx v12, v8, a2
 ; RV32-NEXT:    vsetvli zero, a1, e64, m4, ta, ma
 ; RV32-NEXT:    vluxei32.v v8, (a0), v12, v0.t
 ; RV32-NEXT:    ret
@@ -1903,9 +1903,9 @@ define <8 x double> @vpgather_baseidx_v8i16_v8f64(ptr %base, <8 x i16> %idxs, <8
 define <8 x double> @vpgather_baseidx_sext_v8i16_v8f64(ptr %base, <8 x i16> %idxs, <8 x i1> %m, i32 zeroext %evl) {
 ; RV32-LABEL: vpgather_baseidx_sext_v8i16_v8f64:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
-; RV32-NEXT:    vsext.vf2 v10, v8
-; RV32-NEXT:    vsll.vi v12, v10, 3
+; RV32-NEXT:    li a2, 8
+; RV32-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; RV32-NEXT:    vwmulsu.vx v12, v8, a2
 ; RV32-NEXT:    vsetvli zero, a1, e64, m4, ta, ma
 ; RV32-NEXT:    vluxei32.v v8, (a0), v12, v0.t
 ; RV32-NEXT:    ret
@@ -1927,18 +1927,18 @@ define <8 x double> @vpgather_baseidx_sext_v8i16_v8f64(ptr %base, <8 x i16> %idx
 define <8 x double> @vpgather_baseidx_zext_v8i16_v8f64(ptr %base, <8 x i16> %idxs, <8 x i1> %m, i32 zeroext %evl) {
 ; RV32-LABEL: vpgather_baseidx_zext_v8i16_v8f64:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
-; RV32-NEXT:    vzext.vf2 v10, v8
-; RV32-NEXT:    vsll.vi v12, v10, 3
+; RV32-NEXT:    li a2, 8
+; RV32-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; RV32-NEXT:    vwmulu.vx v12, v8, a2
 ; RV32-NEXT:    vsetvli zero, a1, e64, m4, ta, ma
 ; RV32-NEXT:    vluxei32.v v8, (a0), v12, v0.t
 ; RV32-NEXT:    ret
 ;
 ; RV64-LABEL: vpgather_baseidx_zext_v8i16_v8f64:
 ; RV64:       # %bb.0:
-; RV64-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
-; RV64-NEXT:    vzext.vf2 v10, v8
-; RV64-NEXT:    vsll.vi v12, v10, 3
+; RV64-NEXT:    li a2, 8
+; RV64-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; RV64-NEXT:    vwmulu.vx v12, v8, a2
 ; RV64-NEXT:    vsetvli zero, a1, e64, m4, ta, ma
 ; RV64-NEXT:    vluxei32.v v8, (a0), v12, v0.t
 ; RV64-NEXT:    ret
@@ -1959,11 +1959,11 @@ define <8 x double> @vpgather_baseidx_v8i32_v8f64(ptr %base, <8 x i32> %idxs, <8
 ;
 ; RV64-LABEL: vpgather_baseidx_v8i32_v8f64:
 ; RV64:       # %bb.0:
-; RV64-NEXT:    vsetivli zero, 8, e64, m4, ta, ma
-; RV64-NEXT:    vsext.vf2 v12, v8
-; RV64-NEXT:    vsll.vi v8, v12, 3
+; RV64-NEXT:    li a2, 8
+; RV64-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
+; RV64-NEXT:    vwmulsu.vx v12, v8, a2
 ; RV64-NEXT:    vsetvli zero, a1, e64, m4, ta, ma
-; RV64-NEXT:    vluxei64.v v8, (a0), v8, v0.t
+; RV64-NEXT:    vluxei64.v v8, (a0), v12, v0.t
 ; RV64-NEXT:    ret
   %ptrs = getelementptr inbounds double, ptr %base, <8 x i32> %idxs
   %v = call <8 x double> @llvm.vp.gather.v8f64.v8p0(<8 x ptr> %ptrs, <8 x i1> %m, i32 %evl)
@@ -1981,11 +1981,11 @@ define <8 x double> @vpgather_baseidx_sext_v8i32_v8f64(ptr %base, <8 x i32> %idx
 ;
 ; RV64-LABEL: vpgather_baseidx_sext_v8i32_v8f64:
 ; RV64:       # %bb.0:
-; RV64-NEXT:    vsetivli zero, 8, e64, m4, ta, ma
-; RV64-NEXT:    vsext.vf2 v12, v8
-; RV64-NEXT:    vsll.vi v8, v12, 3
+; RV64-NEXT:    li a2, 8
+; RV64-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
+; RV64-NEXT:    vwmulsu.vx v12, v8, a2
 ; RV64-NEXT:    vsetvli zero, a1, e64, m4, ta, ma
-; RV64-NEXT:    vluxei64.v v8, (a0), v8, v0.t
+; RV64-NEXT:    vluxei64.v v8, (a0), v12, v0.t
 ; RV64-NEXT:    ret
   %eidxs = sext <8 x i32> %idxs to <8 x i64>
   %ptrs = getelementptr inbounds double, ptr %base, <8 x i64> %eidxs
@@ -2004,11 +2004,11 @@ define <8 x double> @vpgather_baseidx_zext_v8i32_v8f64(ptr %base, <8 x i32> %idx
 ;
 ; RV64-LABEL: vpgather_baseidx_zext_v8i32_v8f64:
 ; RV64:       # %bb.0:
-; RV64-NEXT:    vsetivli zero, 8, e64, m4, ta, ma
-; RV64-NEXT:    vzext.vf2 v12, v8
-; RV64-NEXT:    vsll.vi v8, v12, 3
+; RV64-NEXT:    li a2, 8
+; RV64-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
+; RV64-NEXT:    vwmulu.vx v12, v8, a2
 ; RV64-NEXT:    vsetvli zero, a1, e64, m4, ta, ma
-; RV64-NEXT:    vluxei64.v v8, (a0), v8, v0.t
+; RV64-NEXT:    vluxei64.v v8, (a0), v12, v0.t
 ; RV64-NEXT:    ret
   %eidxs = zext <8 x i32> %idxs to <8 x i64>
   %ptrs = getelementptr inbounds double, ptr %base, <8 x i64> %eidxs
@@ -2209,25 +2209,25 @@ define <32 x double> @vpgather_baseidx_zext_v32i8_v32f64(ptr %base, <32 x i8> %i
 ; RV32-LABEL: vpgather_baseidx_zext_v32i8_v32f64:
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    li a2, 32
-; RV32-NEXT:    vsetvli zero, a2, e16, m4, ta, ma
-; RV32-NEXT:    vzext.vf2 v12, v8
-; RV32-NEXT:    li a3, 16
-; RV32-NEXT:    vsll.vi v16, v12, 3
+; RV32-NEXT:    li a3, 8
+; RV32-NEXT:    li a4, 16
+; RV32-NEXT:    vsetvli zero, a2, e8, m2, ta, ma
+; RV32-NEXT:    vwmulu.vx v16, v8, a3
 ; RV32-NEXT:    mv a2, a1
-; RV32-NEXT:    bltu a1, a3, .LBB97_2
+; RV32-NEXT:    bltu a1, a4, .LBB97_2
 ; RV32-NEXT:  # %bb.1:
 ; RV32-NEXT:    li a2, 16
 ; RV32-NEXT:  .LBB97_2:
 ; RV32-NEXT:    vsetvli zero, a2, e64, m8, ta, ma
 ; RV32-NEXT:    vluxei16.v v8, (a0), v16, v0.t
+; RV32-NEXT:    vsetivli zero, 16, e16, m4, ta, ma
+; RV32-NEXT:    vslidedown.vi v24, v16, 16
 ; RV32-NEXT:    addi a2, a1, -16
-; RV32-NEXT:    vsetivli zero, 2, e8, mf4, ta, ma
-; RV32-NEXT:    vslidedown.vi v0, v0, 2
 ; RV32-NEXT:    sltu a1, a1, a2
 ; RV32-NEXT:    addi a1, a1, -1
+; RV32-NEXT:    vsetivli zero, 2, e8, mf4, ta, ma
+; RV32-NEXT:    vslidedown.vi v0, v0, 2
 ; RV32-NEXT:    and a1, a1, a2
-; RV32-NEXT:    vsetivli zero, 16, e16, m4, ta, ma
-; RV32-NEXT:    vslidedown.vi v24, v16, 16
 ; RV32-NEXT:    vsetvli zero, a1, e64, m8, ta, ma
 ; RV32-NEXT:    vluxei16.v v16, (a0), v24, v0.t
 ; RV32-NEXT:    ret
@@ -2235,25 +2235,25 @@ define <32 x double> @vpgather_baseidx_zext_v32i8_v32f64(ptr %base, <32 x i8> %i
 ; RV64-LABEL: vpgather_baseidx_zext_v32i8_v32f64:
 ; RV64:       # %bb.0:
 ; RV64-NEXT:    li a2, 32
-; RV64-NEXT:    vsetvli zero, a2, e16, m4, ta, ma
-; RV64-NEXT:    vzext.vf2 v12, v8
-; RV64-NEXT:    li a3, 16
-; RV64-NEXT:    vsll.vi v16, v12, 3
+; RV64-NEXT:    li a3, 8
+; RV64-NEXT:    li a4, 16
+; RV64-NEXT:    vsetvli zero, a2, e8, m2, ta, ma
+; RV64-NEXT:    vwmulu.vx v16, v8, a3
 ; RV64-NEXT:    mv a2, a1
-; RV64-NEXT:    bltu a1, a3, .LBB97_2
+; RV64-NEXT:    bltu a1, a4, .LBB97_2
 ; RV64-NEXT:  # %bb.1:
 ; RV64-NEXT:    li a2, 16
 ; RV64-NEXT:  .LBB97_2:
 ; RV64-NEXT:    vsetvli zero, a2, e64, m8, ta, ma
 ; RV64-NEXT:    vluxei16.v v8, (a0), v16, v0.t
+; RV64-NEXT:    vsetivli zero, 16, e16, m4, ta, ma
+; RV64-NEXT:    vslidedown.vi v24, v16, 16
 ; RV64-NEXT:    addi a2, a1, -16
-; RV64-NEXT:    vsetivli zero, 2, e8, mf4, ta, ma
-; RV64-NEXT:    vslidedown.vi v0, v0, 2
 ; RV64-NEXT:    sltu a1, a1, a2
 ; RV64-NEXT:    addi a1, a1, -1
+; RV64-NEXT:    vsetivli zero, 2, e8, mf4, ta, ma
+; RV64-NEXT:    vslidedown.vi v0, v0, 2
 ; RV64-NEXT:    and a1, a1, a2
-; RV64-NEXT:    vsetivli zero, 16, e16, m4, ta, ma
-; RV64-NEXT:    vslidedown.vi v24, v16, 16
 ; RV64-NEXT:    vsetvli zero, a1, e64, m8, ta, ma
 ; RV64-NEXT:    vluxei16.v v16, (a0), v24, v0.t
 ; RV64-NEXT:    ret
@@ -2267,25 +2267,25 @@ define <32 x double> @vpgather_baseidx_v32i16_v32f64(ptr %base, <32 x i16> %idxs
 ; RV32-LABEL: vpgather_baseidx_v32i16_v32f64:
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    li a2, 32
-; RV32-NEXT:    vsetvli zero, a2, e32, m8, ta, ma
-; RV32-NEXT:    vsext.vf2 v16, v8
-; RV32-NEXT:    li a3, 16
-; RV32-NEXT:    vsll.vi v16, v16, 3
+; RV32-NEXT:    li a3, 8
+; RV32-NEXT:    li a4, 16
+; RV32-NEXT:    vsetvli zero, a2, e16, m4, ta, ma
+; RV32-NEXT:    vwmulsu.vx v16, v8, a3
 ; RV32-NEXT:    mv a2, a1
-; RV32-NEXT:    bltu a1, a3, .LBB98_2
+; RV32-NEXT:    bltu a1, a4, .LBB98_2
 ; RV32-NEXT:  # %bb.1:
 ; RV32-NEXT:    li a2, 16
 ; RV32-NEXT:  .LBB98_2:
 ; RV32-NEXT:    vsetvli zero, a2, e64, m8, ta, ma
 ; RV32-NEXT:    vluxei32.v v8, (a0), v16, v0.t
+; RV32-NEXT:    vsetivli zero, 16, e32, m8, ta, ma
+; RV32-NEXT:    vslidedown.vi v24, v16, 16
 ; RV32-NEXT:    addi a2, a1, -16
-; RV32-NEXT:    vsetivli zero, 2, e8, mf4, ta, ma
-; RV32-NEXT:    vslidedown.vi v0, v0, 2
 ; RV32-NEXT:    sltu a1, a1, a2
 ; RV32-NEXT:    addi a1, a1, -1
+; RV32-NEXT:    vsetivli zero, 2, e8, mf4, ta, ma
+; RV32-NEXT:    vslidedown.vi v0, v0, 2
 ; RV32-NEXT:    and a1, a1, a2
-; RV32-NEXT:    vsetivli zero, 16, e32, m8, ta, ma
-; RV32-NEXT:    vslidedown.vi v24, v16, 16
 ; RV32-NEXT:    vsetvli zero, a1, e64, m8, ta, ma
 ; RV32-NEXT:    vluxei32.v v16, (a0), v24, v0.t
 ; RV32-NEXT:    ret
@@ -2325,25 +2325,25 @@ define <32 x double> @vpgather_baseidx_sext_v32i16_v32f64(ptr %base, <32 x i16>
 ; RV32-LABEL: vpgather_baseidx_sext_v32i16_v32f64:
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    li a2, 32
-; RV32-NEXT:    vsetvli zero, a2, e32, m8, ta, ma
-; RV32-NEXT:    vsext.vf2 v16, v8
-; RV32-NEXT:    li a3, 16
-; RV32-NEXT:    vsll.vi v16, v16, 3
+; RV32-NEXT:    li a3, 8
+; RV32-NEXT:    li a4, 16
+; RV32-NEXT:    vsetvli zero, a2, e16, m4, ta, ma
+; RV32-NEXT:    vwmulsu.vx v16, v8, a3
 ; RV32-NEXT:    mv a2, a1
-; RV32-NEXT:    bltu a1, a3, .LBB99_2
+; RV32-NEXT:    bltu a1, a4, .LBB99_2
 ; RV32-NEXT:  # %bb.1:
 ; RV32-NEXT:    li a2, 16
 ; RV32-NEXT:  .LBB99_2:
 ; RV32-NEXT:    vsetvli zero, a2, e64, m8, ta, ma
 ; RV32-NEXT:    vluxei32.v v8, (a0), v16, v0.t
+; RV32-NEXT:    vsetivli zero, 16, e32, m8, ta, ma
+; RV32-NEXT:    vslidedown.vi v24, v16, 16
 ; RV32-NEXT:    addi a2, a1, -16
-; RV32-NEXT:    vsetivli zero, 2, e8, mf4, ta, ma
-; RV32-NEXT:    vslidedown.vi v0, v0, 2
 ; RV32-NEXT:    sltu a1, a1, a2
 ; RV32-NEXT:    addi a1, a1, -1
+; RV32-NEXT:    vsetivli zero, 2, e8, mf4, ta, ma
+; RV32-NEXT:    vslidedown.vi v0, v0, 2
 ; RV32-NEXT:    and a1, a1, a2
-; RV32-NEXT:    vsetivli zero, 16, e32, m8, ta, ma
-; RV32-NEXT:    vslidedown.vi v24, v16, 16
 ; RV32-NEXT:    vsetvli zero, a1, e64, m8, ta, ma
 ; RV32-NEXT:    vluxei32.v v16, (a0), v24, v0.t
 ; RV32-NEXT:    ret
@@ -2385,25 +2385,25 @@ define <32 x double> @vpgather_baseidx_zext_v32i16_v32f64(ptr %base, <32 x i16>
 ; RV32-LABEL: vpgather_baseidx_zext_v32i16_v32f64:
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    li a2, 32
-; RV32-NEXT:    vsetvli zero, a2, e32, m8, ta, ma
-; RV32-NEXT:    vzext.vf2 v16, v8
-; RV32-NEXT:    li a3, 16
-; RV32-NEXT:    vsll.vi v16, v16, 3
+; RV32-NEXT:    li a3, 8
+; RV32-NEXT:    li a4, 16
+; RV32-NEXT:    vsetvli zero, a2, e16, m4, ta, ma
+; RV32-NEXT:    vwmulu.vx v16, v8, a3
 ; RV32-NEXT:    mv a2, a1
-; RV32-NEXT:    bltu a1, a3, .LBB100_2
+; RV32-NEXT:    bltu a1, a4, .LBB100_2
 ; RV32-NEXT:  # %bb.1:
 ; RV32-NEXT:    li a2, 16
 ; RV32-NEXT:  .LBB100_2:
 ; RV32-NEXT:    vsetvli zero, a2, e64, m8, ta, ma
 ; RV32-NEXT:    vluxei32.v v8, (a0), v16, v0.t
+; RV32-NEXT:    vsetivli zero, 16, e32, m8, ta, ma
+; RV32-NEXT:    vslidedown.vi v24, v16, 16
 ; RV32-NEXT:    addi a2, a1, -16
-; RV32-NEXT:    vsetivli zero, 2, e8, mf4, ta, ma
-; RV32-NEXT:    vslidedown.vi v0, v0, 2
 ; RV32-NEXT:    sltu a1, a1, a2
 ; RV32-NEXT:    addi a1, a1, -1
+; RV32-NEXT:    vsetivli zero, 2, e8, mf4, ta, ma
+; RV32-NEXT:    vslidedown.vi v0, v0, 2
 ; RV32-NEXT:    and a1, a1, a2
-; RV32-NEXT:    vsetivli zero, 16, e32, m8, ta, ma
-; RV32-NEXT:    vslidedown.vi v24, v16, 16
 ; RV32-NEXT:    vsetvli zero, a1, e64, m8, ta, ma
 ; RV32-NEXT:    vluxei32.v v16, (a0), v24, v0.t
 ; RV32-NEXT:    ret
@@ -2411,25 +2411,25 @@ define <32 x double> @vpgather_baseidx_zext_v32i16_v32f64(ptr %base, <32 x i16>
 ; RV64-LABEL: vpgather_baseidx_zext_v32i16_v32f64:
 ; RV64:       # %bb.0:
 ; RV64-NEXT:    li a2, 32
-; RV64-NEXT:    vsetvli zero, a2, e32, m8, ta, ma
-; RV64-NEXT:    vzext.vf2 v16, v8
-; RV64-NEXT:    li a3, 16
-; RV64-NEXT:    vsll.vi v16, v16, 3
+; RV64-NEXT:    li a3, 8
+; RV64-NEXT:    li a4, 16
+; RV64-NEXT:    vsetvli zero, a2, e16, m4, ta, ma
+; RV64-NEXT:    vwmulu.vx v16, v8, a3
 ; RV64-NEXT:    mv a2, a1
-; RV64-NEXT:    bltu a1, a3, .LBB100_2
+; RV64-NEXT:    bltu a1, a4, .LBB100_2
 ; RV64-NEXT:  # %bb.1:
 ; RV64-NEXT:    li a2, 16
 ; RV64-NEXT:  .LBB100_2:
 ; RV64-NEXT:    vsetvli zero, a2, e64, m8, ta, ma
 ; RV64-NEXT:    vluxei32.v v8, (a0), v16, v0.t
+; RV64-NEXT:    vsetivli zero, 16, e32, m8, ta, ma
+; RV64-NEXT:    vslidedown.vi v24, v16, 16
 ; RV64-NEXT:    addi a2, a1, -16
-; RV64-NEXT:    vsetivli zero, 2, e8, mf4, ta, ma
-; RV64-NEXT:    vslidedown.vi v0, v0, 2
 ; RV64-NEXT:    sltu a1, a1, a2
 ; RV64-NEXT:    addi a1, a1, -1
+; RV64-NEXT:    vsetivli zero, 2, e8, mf4, ta, ma
+; RV64-NEXT:    vslidedown.vi v0, v0, 2
 ; RV64-NEXT:    and a1, a1, a2
-; RV64-NEXT:    vsetivli zero, 16, e32, m8, ta, ma
-; RV64-NEXT:    vslidedown.vi v24, v16, 16
 ; RV64-NEXT:    vsetvli zero, a1, e64, m8, ta, ma
 ; RV64-NEXT:    vluxei32.v v16, (a0), v24, v0.t
 ; RV64-NEXT:    ret
@@ -2468,20 +2468,19 @@ define <32 x double> @vpgather_baseidx_v32i32_v32f64(ptr %base, <32 x i32> %idxs
 ; RV64-LABEL: vpgather_baseidx_v32i32_v32f64:
 ; RV64:       # %bb.0:
 ; RV64-NEXT:    vsetivli zero, 16, e32, m8, ta, ma
-; RV64-NEXT:    vslidedown.vi v16, v8, 16
-; RV64-NEXT:    vsetivli zero, 16, e64, m8, ta, ma
-; RV64-NEXT:    vsext.vf2 v24, v8
+; RV64-NEXT:    vslidedown.vi v24, v8, 16
+; RV64-NEXT:    li a2, 8
 ; RV64-NEXT:    li a3, 16
-; RV64-NEXT:    vsext.vf2 v8, v16
-; RV64-NEXT:    vsll.vi v16, v8, 3
-; RV64-NEXT:    vsll.vi v8, v24, 3
+; RV64-NEXT:    vsetivli zero, 16, e32, m4, ta, ma
+; RV64-NEXT:    vwmulsu.vx v16, v24, a2
+; RV64-NEXT:    vwmulsu.vx v24, v8, a2
 ; RV64-NEXT:    mv a2, a1
 ; RV64-NEXT:    bltu a1, a3, .LBB101_2
 ; RV64-NEXT:  # %bb.1:
 ; RV64-NEXT:    li a2, 16
 ; RV64-NEXT:  .LBB101_2:
 ; RV64-NEXT:    vsetvli zero, a2, e64, m8, ta, ma
-; RV64-NEXT:    vluxei64.v v8, (a0), v8, v0.t
+; RV64-NEXT:    vluxei64.v v8, (a0), v24, v0.t
 ; RV64-NEXT:    addi a2, a1, -16
 ; RV64-NEXT:    sltu a1, a1, a2
 ; RV64-NEXT:    addi a1, a1, -1
@@ -2524,22 +2523,20 @@ define <32 x double> @vpgather_baseidx_sext_v32i32_v32f64(ptr %base, <32 x i32>
 ;
 ; RV64-LABEL: vpgather_baseidx_sext_v32i32_v32f64:
 ; RV64:       # %bb.0:
-; RV64-NEXT:    vsetivli zero, 16, e64, m8, ta, ma
-; RV64-NEXT:    vsext.vf2 v24, v8
 ; RV64-NEXT:    vsetivli zero, 16, e32, m8, ta, ma
-; RV64-NEXT:    vslidedown.vi v8, v8, 16
+; RV64-NEXT:    vslidedown.vi v24, v8, 16
+; RV64-NEXT:    li a2, 8
 ; RV64-NEXT:    li a3, 16
-; RV64-NEXT:    vsetivli zero, 16, e64, m8, ta, ma
-; RV64-NEXT:    vsext.vf2 v16, v8
-; RV64-NEXT:    vsll.vi v16, v16, 3
-; RV64-NEXT:    vsll.vi v8, v24, 3
+; RV64-NEXT:    vsetivli zero, 16, e32, m4, ta, ma
+; RV64-NEXT:    vwmulsu.vx v16, v24, a2
+; RV64-NEXT:    vwmulsu.vx v24, v8, a2
 ; RV64-NEXT:    mv a2, a1
 ; RV64-NEXT:    bltu a1, a3, .LBB102_2
 ; RV64-NEXT:  # %bb.1:
 ; RV64-NEXT:    li a2, 16
 ; RV64-NEXT:  .LBB102_2:
 ; RV64-NEXT:    vsetvli zero, a2, e64, m8, ta, ma
-; RV64-NEXT:    vluxei64.v v8, (a0), v8, v0.t
+; RV64-NEXT:    vluxei64.v v8, (a0), v24, v0.t
 ; RV64-NEXT:    addi a2, a1, -16
 ; RV64-NEXT:    sltu a1, a1, a2
 ; RV64-NEXT:    addi a1, a1, -1
@@ -2583,22 +2580,20 @@ define <32 x double> @vpgather_baseidx_zext_v32i32_v32f64(ptr %base, <32 x i32>
 ;
 ; RV64-LABEL: vpgather_baseidx_zext_v32i32_v32f64:
 ; RV64:       # %bb.0:
-; RV64-NEXT:    vsetivli zero, 16, e64, m8, ta, ma
-; RV64-NEXT:    vzext.vf2 v24, v8
 ; RV64-NEXT:    vsetivli zero, 16, e32, m8, ta, ma
-; RV64-NEXT:    vslidedown.vi v8, v8, 16
+; RV64-NEXT:    vslidedown.vi v24, v8, 16
+; RV64-NEXT:    li a2, 8
 ; RV64-NEXT:    li a3, 16
-; RV64-NEXT:    vsetivli zero, 16, e64, m8, ta, ma
-; RV64-NEXT:    vzext.vf2 v16, v8
-; RV64-NEXT:    vsll.vi v16, v16, 3
-; RV64-NEXT:    vsll.vi v8, v24, 3
+; RV64-NEXT:    vsetivli zero, 16, e32, m4, ta, ma
+; RV64-NEXT:    vwmulu.vx v16, v24, a2
+; RV64-NEXT:    vwmulu.vx v24, v8, a2
 ; RV64-NEXT:    mv a2, a1
 ; RV64-NEXT:    bltu a1, a3, .LBB103_2
 ; RV64-NEXT:  # %bb.1:
 ; RV64-NEXT:    li a2, 16
 ; RV64-NEXT:  .LBB103_2:
 ; RV64-NEXT:    vsetvli zero, a2, e64, m8, ta, ma
-; RV64-NEXT:    vluxei64.v v8, (a0), v8, v0.t
+; RV64-NEXT:    vluxei64.v v8, (a0), v24, v0.t
 ; RV64-NEXT:    addi a2, a1, -16
 ; RV64-NEXT:    sltu a1, a1, a2
 ; RV64-NEXT:    addi a1, a1, -1
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vpscatter.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vpscatter.ll
index d691dcd5c54b6..f7e4716d2c847 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vpscatter.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vpscatter.ll
@@ -556,20 +556,20 @@ define void @vpscatter_baseidx_sext_v8i8_v8i32(<8 x i32> %val, ptr %base, <8 x i
 define void @vpscatter_baseidx_zext_v8i8_v8i32(<8 x i32> %val, ptr %base, <8 x i8> %idxs, <8 x i1> %m, i32 zeroext %evl) {
 ; RV32-LABEL: vpscatter_baseidx_zext_v8i8_v8i32:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
-; RV32-NEXT:    vzext.vf2 v11, v10
-; RV32-NEXT:    vsll.vi v10, v11, 2
+; RV32-NEXT:    li a2, 4
+; RV32-NEXT:    vsetivli zero, 8, e8, mf2, ta, ma
+; RV32-NEXT:    vwmulu.vx v11, v10, a2
 ; RV32-NEXT:    vsetvli zero, a1, e32, m2, ta, ma
-; RV32-NEXT:    vsoxei16.v v8, (a0), v10, v0.t
+; RV32-NEXT:    vsoxei16.v v8, (a0), v11, v0.t
 ; RV32-NEXT:    ret
 ;
 ; RV64-LABEL: vpscatter_baseidx_zext_v8i8_v8i32:
 ; RV64:       # %bb.0:
-; RV64-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
-; RV64-NEXT:    vzext.vf2 v11, v10
-; RV64-NEXT:    vsll.vi v10, v11, 2
+; RV64-NEXT:    li a2, 4
+; RV64-NEXT:    vsetivli zero, 8, e8, mf2, ta, ma
+; RV64-NEXT:    vwmulu.vx v11, v10, a2
 ; RV64-NEXT:    vsetvli zero, a1, e32, m2, ta, ma
-; RV64-NEXT:    vsoxei16.v v8, (a0), v10, v0.t
+; RV64-NEXT:    vsoxei16.v v8, (a0), v11, v0.t
 ; RV64-NEXT:    ret
   %eidxs = zext <8 x i8> %idxs to <8 x i32>
   %ptrs = getelementptr inbounds i32, ptr %base, <8 x i32> %eidxs
@@ -580,11 +580,11 @@ define void @vpscatter_baseidx_zext_v8i8_v8i32(<8 x i32> %val, ptr %base, <8 x i
 define void @vpscatter_baseidx_v8i16_v8i32(<8 x i32> %val, ptr %base, <8 x i16> %idxs, <8 x i1> %m, i32 zeroext %evl) {
 ; RV32-LABEL: vpscatter_baseidx_v8i16_v8i32:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
-; RV32-NEXT:    vsext.vf2 v12, v10
-; RV32-NEXT:    vsll.vi v10, v12, 2
+; RV32-NEXT:    li a2, 4
+; RV32-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; RV32-NEXT:    vwmulsu.vx v12, v10, a2
 ; RV32-NEXT:    vsetvli zero, a1, e32, m2, ta, ma
-; RV32-NEXT:    vsoxei32.v v8, (a0), v10, v0.t
+; RV32-NEXT:    vsoxei32.v v8, (a0), v12, v0.t
 ; RV32-NEXT:    ret
 ;
 ; RV64-LABEL: vpscatter_baseidx_v8i16_v8i32:
@@ -603,11 +603,11 @@ define void @vpscatter_baseidx_v8i16_v8i32(<8 x i32> %val, ptr %base, <8 x i16>
 define void @vpscatter_baseidx_sext_v8i16_v8i32(<8 x i32> %val, ptr %base, <8 x i16> %idxs, <8 x i1> %m, i32 zeroext %evl) {
 ; RV32-LABEL: vpscatter_baseidx_sext_v8i16_v8i32:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
-; RV32-NEXT:    vsext.vf2 v12, v10
-; RV32-NEXT:    vsll.vi v10, v12, 2
+; RV32-NEXT:    li a2, 4
+; RV32-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; RV32-NEXT:    vwmulsu.vx v12, v10, a2
 ; RV32-NEXT:    vsetvli zero, a1, e32, m2, ta, ma
-; RV32-NEXT:    vsoxei32.v v8, (a0), v10, v0.t
+; RV32-NEXT:    vsoxei32.v v8, (a0), v12, v0.t
 ; RV32-NEXT:    ret
 ;
 ; RV64-LABEL: vpscatter_baseidx_sext_v8i16_v8i32:
@@ -627,20 +627,20 @@ define void @vpscatter_baseidx_sext_v8i16_v8i32(<8 x i32> %val, ptr %base, <8 x
 define void @vpscatter_baseidx_zext_v8i16_v8i32(<8 x i32> %val, ptr %base, <8 x i16> %idxs, <8 x i1> %m, i32 zeroext %evl) {
 ; RV32-LABEL: vpscatter_baseidx_zext_v8i16_v8i32:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
-; RV32-NEXT:    vzext.vf2 v12, v10
-; RV32-NEXT:    vsll.vi v10, v12, 2
+; RV32-NEXT:    li a2, 4
+; RV32-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; RV32-NEXT:    vwmulu.vx v12, v10, a2
 ; RV32-NEXT:    vsetvli zero, a1, e32, m2, ta, ma
-; RV32-NEXT:    vsoxei32.v v8, (a0), v10, v0.t
+; RV32-NEXT:    vsoxei32.v v8, (a0), v12, v0.t
 ; RV32-NEXT:    ret
 ;
 ; RV64-LABEL: vpscatter_baseidx_zext_v8i16_v8i32:
 ; RV64:       # %bb.0:
-; RV64-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
-; RV64-NEXT:    vzext.vf2 v12, v10
-; RV64-NEXT:    vsll.vi v10, v12, 2
+; RV64-NEXT:    li a2, 4
+; RV64-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; RV64-NEXT:    vwmulu.vx v12, v10, a2
 ; RV64-NEXT:    vsetvli zero, a1, e32, m2, ta, ma
-; RV64-NEXT:    vsoxei32.v v8, (a0), v10, v0.t
+; RV64-NEXT:    vsoxei32.v v8, (a0), v12, v0.t
 ; RV64-NEXT:    ret
   %eidxs = zext <8 x i16> %idxs to <8 x i32>
   %ptrs = getelementptr inbounds i32, ptr %base, <8 x i32> %eidxs
@@ -659,9 +659,9 @@ define void @vpscatter_baseidx_v8i32(<8 x i32> %val, ptr %base, <8 x i32> %idxs,
 ;
 ; RV64-LABEL: vpscatter_baseidx_v8i32:
 ; RV64:       # %bb.0:
-; RV64-NEXT:    vsetivli zero, 8, e64, m4, ta, ma
-; RV64-NEXT:    vsext.vf2 v12, v10
-; RV64-NEXT:    vsll.vi v12, v12, 2
+; RV64-NEXT:    li a2, 4
+; RV64-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
+; RV64-NEXT:    vwmulsu.vx v12, v10, a2
 ; RV64-NEXT:    vsetvli zero, a1, e32, m2, ta, ma
 ; RV64-NEXT:    vsoxei64.v v8, (a0), v12, v0.t
 ; RV64-NEXT:    ret
@@ -790,20 +790,20 @@ define void @vpscatter_baseidx_sext_v8i8_v8i64(<8 x i64> %val, ptr %base, <8 x i
 define void @vpscatter_baseidx_zext_v8i8_v8i64(<8 x i64> %val, ptr %base, <8 x i8> %idxs, <8 x i1> %m, i32 zeroext %evl) {
 ; RV32-LABEL: vpscatter_baseidx_zext_v8i8_v8i64:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
-; RV32-NEXT:    vzext.vf2 v13, v12
-; RV32-NEXT:    vsll.vi v12, v13, 3
+; RV32-NEXT:    li a2, 8
+; RV32-NEXT:    vsetivli zero, 8, e8, mf2, ta, ma
+; RV32-NEXT:    vwmulu.vx v13, v12, a2
 ; RV32-NEXT:    vsetvli zero, a1, e64, m4, ta, ma
-; RV32-NEXT:    vsoxei16.v v8, (a0), v12, v0.t
+; RV32-NEXT:    vsoxei16.v v8, (a0), v13, v0.t
 ; RV32-NEXT:    ret
 ;
 ; RV64-LABEL: vpscatter_baseidx_zext_v8i8_v8i64:
 ; RV64:       # %bb.0:
-; RV64-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
-; RV64-NEXT:    vzext.vf2 v13, v12
-; RV64-NEXT:    vsll.vi v12, v13, 3
+; RV64-NEXT:    li a2, 8
+; RV64-NEXT:    vsetivli zero, 8, e8, mf2, ta, ma
+; RV64-NEXT:    vwmulu.vx v13, v12, a2
 ; RV64-NEXT:    vsetvli zero, a1, e64, m4, ta, ma
-; RV64-NEXT:    vsoxei16.v v8, (a0), v12, v0.t
+; RV64-NEXT:    vsoxei16.v v8, (a0), v13, v0.t
 ; RV64-NEXT:    ret
   %eidxs = zext <8 x i8> %idxs to <8 x i64>
   %ptrs = getelementptr inbounds i64, ptr %base, <8 x i64> %eidxs
@@ -814,11 +814,11 @@ define void @vpscatter_baseidx_zext_v8i8_v8i64(<8 x i64> %val, ptr %base, <8 x i
 define void @vpscatter_baseidx_v8i16_v8i64(<8 x i64> %val, ptr %base, <8 x i16> %idxs, <8 x i1> %m, i32 zeroext %evl) {
 ; RV32-LABEL: vpscatter_baseidx_v8i16_v8i64:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
-; RV32-NEXT:    vsext.vf2 v14, v12
-; RV32-NEXT:    vsll.vi v12, v14, 3
+; RV32-NEXT:    li a2, 8
+; RV32-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; RV32-NEXT:    vwmulsu.vx v14, v12, a2
 ; RV32-NEXT:    vsetvli zero, a1, e64, m4, ta, ma
-; RV32-NEXT:    vsoxei32.v v8, (a0), v12, v0.t
+; RV32-NEXT:    vsoxei32.v v8, (a0), v14, v0.t
 ; RV32-NEXT:    ret
 ;
 ; RV64-LABEL: vpscatter_baseidx_v8i16_v8i64:
@@ -837,11 +837,11 @@ define void @vpscatter_baseidx_v8i16_v8i64(<8 x i64> %val, ptr %base, <8 x i16>
 define void @vpscatter_baseidx_sext_v8i16_v8i64(<8 x i64> %val, ptr %base, <8 x i16> %idxs, <8 x i1> %m, i32 zeroext %evl) {
 ; RV32-LABEL: vpscatter_baseidx_sext_v8i16_v8i64:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
-; RV32-NEXT:    vsext.vf2 v14, v12
-; RV32-NEXT:    vsll.vi v12, v14, 3
+; RV32-NEXT:    li a2, 8
+; RV32-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; RV32-NEXT:    vwmulsu.vx v14, v12, a2
 ; RV32-NEXT:    vsetvli zero, a1, e64, m4, ta, ma
-; RV32-NEXT:    vsoxei32.v v8, (a0), v12, v0.t
+; RV32-NEXT:    vsoxei32.v v8, (a0), v14, v0.t
 ; RV32-NEXT:    ret
 ;
 ; RV64-LABEL: vpscatter_baseidx_sext_v8i16_v8i64:
@@ -861,20 +861,20 @@ define void @vpscatter_baseidx_sext_v8i16_v8i64(<8 x i64> %val, ptr %base, <8 x
 define void @vpscatter_baseidx_zext_v8i16_v8i64(<8 x i64> %val, ptr %base, <8 x i16> %idxs, <8 x i1> %m, i32 zeroext %evl) {
 ; RV32-LABEL: vpscatter_baseidx_zext_v8i16_v8i64:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
-; RV32-NEXT:    vzext.vf2 v14, v12
-; RV32-NEXT:    vsll.vi v12, v14, 3
+; RV32-NEXT:    li a2, 8
+; RV32-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; RV32-NEXT:    vwmulu.vx v14, v12, a2
 ; RV32-NEXT:    vsetvli zero, a1, e64, m4, ta, ma
-; RV32-NEXT:    vsoxei32.v v8, (a0), v12, v0.t
+; RV32-NEXT:    vsoxei32.v v8, (a0), v14, v0.t
 ; RV32-NEXT:    ret
 ;
 ; RV64-LABEL: vpscatter_baseidx_zext_v8i16_v8i64:
 ; RV64:       # %bb.0:
-; RV64-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
-; RV64-NEXT:    vzext.vf2 v14, v12
-; RV64-NEXT:    vsll.vi v12, v14, 3
+; RV64-NEXT:    li a2, 8
+; RV64-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; RV64-NEXT:    vwmulu.vx v14, v12, a2
 ; RV64-NEXT:    vsetvli zero, a1, e64, m4, ta, ma
-; RV64-NEXT:    vsoxei32.v v8, (a0), v12, v0.t
+; RV64-NEXT:    vsoxei32.v v8, (a0), v14, v0.t
 ; RV64-NEXT:    ret
   %eidxs = zext <8 x i16> %idxs to <8 x i64>
   %ptrs = getelementptr inbounds i64, ptr %base, <8 x i64> %eidxs
@@ -893,11 +893,11 @@ define void @vpscatter_baseidx_v8i32_v8i64(<8 x i64> %val, ptr %base, <8 x i32>
 ;
 ; RV64-LABEL: vpscatter_baseidx_v8i32_v8i64:
 ; RV64:       # %bb.0:
-; RV64-NEXT:    vsetivli zero, 8, e64, m4, ta, ma
-; RV64-NEXT:    vsext.vf2 v16, v12
-; RV64-NEXT:    vsll.vi v12, v16, 3
+; RV64-NEXT:    li a2, 8
+; RV64-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
+; RV64-NEXT:    vwmulsu.vx v16, v12, a2
 ; RV64-NEXT:    vsetvli zero, a1, e64, m4, ta, ma
-; RV64-NEXT:    vsoxei64.v v8, (a0), v12, v0.t
+; RV64-NEXT:    vsoxei64.v v8, (a0), v16, v0.t
 ; RV64-NEXT:    ret
   %ptrs = getelementptr inbounds i64, ptr %base, <8 x i32> %idxs
   call void @llvm.vp.scatter.v8i64.v8p0(<8 x i64> %val, <8 x ptr> %ptrs, <8 x i1> %m, i32 %evl)
@@ -915,11 +915,11 @@ define void @vpscatter_baseidx_sext_v8i32_v8i64(<8 x i64> %val, ptr %base, <8 x
 ;
 ; RV64-LABEL: vpscatter_baseidx_sext_v8i32_v8i64:
 ; RV64:       # %bb.0:
-; RV64-NEXT:    vsetivli zero, 8, e64, m4, ta, ma
-; RV64-NEXT:    vsext.vf2 v16, v12
-; RV64-NEXT:    vsll.vi v12, v16, 3
+; RV64-NEXT:    li a2, 8
+; RV64-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
+; RV64-NEXT:    vwmulsu.vx v16, v12, a2
 ; RV64-NEXT:    vsetvli zero, a1, e64, m4, ta, ma
-; RV64-NEXT:    vsoxei64.v v8, (a0), v12, v0.t
+; RV64-NEXT:    vsoxei64.v v8, (a0), v16, v0.t
 ; RV64-NEXT:    ret
   %eidxs = sext <8 x i32> %idxs to <8 x i64>
   %ptrs = getelementptr inbounds i64, ptr %base, <8 x i64> %eidxs
@@ -938,11 +938,11 @@ define void @vpscatter_baseidx_zext_v8i32_v8i64(<8 x i64> %val, ptr %base, <8 x
 ;
 ; RV64-LABEL: vpscatter_baseidx_zext_v8i32_v8i64:
 ; RV64:       # %bb.0:
-; RV64-NEXT:    vsetivli zero, 8, e64, m4, ta, ma
-; RV64-NEXT:    vzext.vf2 v16, v12
-; RV64-NEXT:    vsll.vi v12, v16, 3
+; RV64-NEXT:    li a2, 8
+; RV64-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
+; RV64-NEXT:    vwmulu.vx v16, v12, a2
 ; RV64-NEXT:    vsetvli zero, a1, e64, m4, ta, ma
-; RV64-NEXT:    vsoxei64.v v8, (a0), v12, v0.t
+; RV64-NEXT:    vsoxei64.v v8, (a0), v16, v0.t
 ; RV64-NEXT:    ret
   %eidxs = zext <8 x i32> %idxs to <8 x i64>
   %ptrs = getelementptr inbounds i64, ptr %base, <8 x i64> %eidxs
@@ -1323,20 +1323,20 @@ define void @vpscatter_baseidx_sext_v8i8_v8f32(<8 x float> %val, ptr %base, <8 x
 define void @vpscatter_baseidx_zext_v8i8_v8f32(<8 x float> %val, ptr %base, <8 x i8> %idxs, <8 x i1> %m, i32 zeroext %evl) {
 ; RV32-LABEL: vpscatter_baseidx_zext_v8i8_v8f32:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
-; RV32-NEXT:    vzext.vf2 v11, v10
-; RV32-NEXT:    vsll.vi v10, v11, 2
+; RV32-NEXT:    li a2, 4
+; RV32-NEXT:    vsetivli zero, 8, e8, mf2, ta, ma
+; RV32-NEXT:    vwmulu.vx v11, v10, a2
 ; RV32-NEXT:    vsetvli zero, a1, e32, m2, ta, ma
-; RV32-NEXT:    vsoxei16.v v8, (a0), v10, v0.t
+; RV32-NEXT:    vsoxei16.v v8, (a0), v11, v0.t
 ; RV32-NEXT:    ret
 ;
 ; RV64-LABEL: vpscatter_baseidx_zext_v8i8_v8f32:
 ; RV64:       # %bb.0:
-; RV64-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
-; RV64-NEXT:    vzext.vf2 v11, v10
-; RV64-NEXT:    vsll.vi v10, v11, 2
+; RV64-NEXT:    li a2, 4
+; RV64-NEXT:    vsetivli zero, 8, e8, mf2, ta, ma
+; RV64-NEXT:    vwmulu.vx v11, v10, a2
 ; RV64-NEXT:    vsetvli zero, a1, e32, m2, ta, ma
-; RV64-NEXT:    vsoxei16.v v8, (a0), v10, v0.t
+; RV64-NEXT:    vsoxei16.v v8, (a0), v11, v0.t
 ; RV64-NEXT:    ret
   %eidxs = zext <8 x i8> %idxs to <8 x i32>
   %ptrs = getelementptr inbounds float, ptr %base, <8 x i32> %eidxs
@@ -1347,11 +1347,11 @@ define void @vpscatter_baseidx_zext_v8i8_v8f32(<8 x float> %val, ptr %base, <8 x
 define void @vpscatter_baseidx_v8i16_v8f32(<8 x float> %val, ptr %base, <8 x i16> %idxs, <8 x i1> %m, i32 zeroext %evl) {
 ; RV32-LABEL: vpscatter_baseidx_v8i16_v8f32:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
-; RV32-NEXT:    vsext.vf2 v12, v10
-; RV32-NEXT:    vsll.vi v10, v12, 2
+; RV32-NEXT:    li a2, 4
+; RV32-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; RV32-NEXT:    vwmulsu.vx v12, v10, a2
 ; RV32-NEXT:    vsetvli zero, a1, e32, m2, ta, ma
-; RV32-NEXT:    vsoxei32.v v8, (a0), v10, v0.t
+; RV32-NEXT:    vsoxei32.v v8, (a0), v12, v0.t
 ; RV32-NEXT:    ret
 ;
 ; RV64-LABEL: vpscatter_baseidx_v8i16_v8f32:
@@ -1370,11 +1370,11 @@ define void @vpscatter_baseidx_v8i16_v8f32(<8 x float> %val, ptr %base, <8 x i16
 define void @vpscatter_baseidx_sext_v8i16_v8f32(<8 x float> %val, ptr %base, <8 x i16> %idxs, <8 x i1> %m, i32 zeroext %evl) {
 ; RV32-LABEL: vpscatter_baseidx_sext_v8i16_v8f32:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
-; RV32-NEXT:    vsext.vf2 v12, v10
-; RV32-NEXT:    vsll.vi v10, v12, 2
+; RV32-NEXT:    li a2, 4
+; RV32-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; RV32-NEXT:    vwmulsu.vx v12, v10, a2
 ; RV32-NEXT:    vsetvli zero, a1, e32, m2, ta, ma
-; RV32-NEXT:    vsoxei32.v v8, (a0), v10, v0.t
+; RV32-NEXT:    vsoxei32.v v8, (a0), v12, v0.t
 ; RV32-NEXT:    ret
 ;
 ; RV64-LABEL: vpscatter_baseidx_sext_v8i16_v8f32:
@@ -1394,20 +1394,20 @@ define void @vpscatter_baseidx_sext_v8i16_v8f32(<8 x float> %val, ptr %base, <8
 define void @vpscatter_baseidx_zext_v8i16_v8f32(<8 x float> %val, ptr %base, <8 x i16> %idxs, <8 x i1> %m, i32 zeroext %evl) {
 ; RV32-LABEL: vpscatter_baseidx_zext_v8i16_v8f32:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
-; RV32-NEXT:    vzext.vf2 v12, v10
-; RV32-NEXT:    vsll.vi v10, v12, 2
+; RV32-NEXT:    li a2, 4
+; RV32-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; RV32-NEXT:    vwmulu.vx v12, v10, a2
 ; RV32-NEXT:    vsetvli zero, a1, e32, m2, ta, ma
-; RV32-NEXT:    vsoxei32.v v8, (a0), v10, v0.t
+; RV32-NEXT:    vsoxei32.v v8, (a0), v12, v0.t
 ; RV32-NEXT:    ret
 ;
 ; RV64-LABEL: vpscatter_baseidx_zext_v8i16_v8f32:
 ; RV64:       # %bb.0:
-; RV64-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
-; RV64-NEXT:    vzext.vf2 v12, v10
-; RV64-NEXT:    vsll.vi v10, v12, 2
+; RV64-NEXT:    li a2, 4
+; RV64-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; RV64-NEXT:    vwmulu.vx v12, v10, a2
 ; RV64-NEXT:    vsetvli zero, a1, e32, m2, ta, ma
-; RV64-NEXT:    vsoxei32.v v8, (a0), v10, v0.t
+; RV64-NEXT:    vsoxei32.v v8, (a0), v12, v0.t
 ; RV64-NEXT:    ret
   %eidxs = zext <8 x i16> %idxs to <8 x i32>
   %ptrs = getelementptr inbounds float, ptr %base, <8 x i32> %eidxs
@@ -1426,9 +1426,9 @@ define void @vpscatter_baseidx_v8f32(<8 x float> %val, ptr %base, <8 x i32> %idx
 ;
 ; RV64-LABEL: vpscatter_baseidx_v8f32:
 ; RV64:       # %bb.0:
-; RV64-NEXT:    vsetivli zero, 8, e64, m4, ta, ma
-; RV64-NEXT:    vsext.vf2 v12, v10
-; RV64-NEXT:    vsll.vi v12, v12, 2
+; RV64-NEXT:    li a2, 4
+; RV64-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
+; RV64-NEXT:    vwmulsu.vx v12, v10, a2
 ; RV64-NEXT:    vsetvli zero, a1, e32, m2, ta, ma
 ; RV64-NEXT:    vsoxei64.v v8, (a0), v12, v0.t
 ; RV64-NEXT:    ret
@@ -1557,20 +1557,20 @@ define void @vpscatter_baseidx_sext_v8i8_v8f64(<8 x double> %val, ptr %base, <8
 define void @vpscatter_baseidx_zext_v8i8_v8f64(<8 x double> %val, ptr %base, <8 x i8> %idxs, <8 x i1> %m, i32 zeroext %evl) {
 ; RV32-LABEL: vpscatter_baseidx_zext_v8i8_v8f64:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
-; RV32-NEXT:    vzext.vf2 v13, v12
-; RV32-NEXT:    vsll.vi v12, v13, 3
+; RV32-NEXT:    li a2, 8
+; RV32-NEXT:    vsetivli zero, 8, e8, mf2, ta, ma
+; RV32-NEXT:    vwmulu.vx v13, v12, a2
 ; RV32-NEXT:    vsetvli zero, a1, e64, m4, ta, ma
-; RV32-NEXT:    vsoxei16.v v8, (a0), v12, v0.t
+; RV32-NEXT:    vsoxei16.v v8, (a0), v13, v0.t
 ; RV32-NEXT:    ret
 ;
 ; RV64-LABEL: vpscatter_baseidx_zext_v8i8_v8f64:
 ; RV64:       # %bb.0:
-; RV64-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
-; RV64-NEXT:    vzext.vf2 v13, v12
-; RV64-NEXT:    vsll.vi v12, v13, 3
+; RV64-NEXT:    li a2, 8
+; RV64-NEXT:    vsetivli zero, 8, e8, mf2, ta, ma
+; RV64-NEXT:    vwmulu.vx v13, v12, a2
 ; RV64-NEXT:    vsetvli zero, a1, e64, m4, ta, ma
-; RV64-NEXT:    vsoxei16.v v8, (a0), v12, v0.t
+; RV64-NEXT:    vsoxei16.v v8, (a0), v13, v0.t
 ; RV64-NEXT:    ret
   %eidxs = zext <8 x i8> %idxs to <8 x i64>
   %ptrs = getelementptr inbounds double, ptr %base, <8 x i64> %eidxs
@@ -1581,11 +1581,11 @@ define void @vpscatter_baseidx_zext_v8i8_v8f64(<8 x double> %val, ptr %base, <8
 define void @vpscatter_baseidx_v8i16_v8f64(<8 x double> %val, ptr %base, <8 x i16> %idxs, <8 x i1> %m, i32 zeroext %evl) {
 ; RV32-LABEL: vpscatter_baseidx_v8i16_v8f64:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
-; RV32-NEXT:    vsext.vf2 v14, v12
-; RV32-NEXT:    vsll.vi v12, v14, 3
+; RV32-NEXT:    li a2, 8
+; RV32-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; RV32-NEXT:    vwmulsu.vx v14, v12, a2
 ; RV32-NEXT:    vsetvli zero, a1, e64, m4, ta, ma
-; RV32-NEXT:    vsoxei32.v v8, (a0), v12, v0.t
+; RV32-NEXT:    vsoxei32.v v8, (a0), v14, v0.t
 ; RV32-NEXT:    ret
 ;
 ; RV64-LABEL: vpscatter_baseidx_v8i16_v8f64:
@@ -1604,11 +1604,11 @@ define void @vpscatter_baseidx_v8i16_v8f64(<8 x double> %val, ptr %base, <8 x i1
 define void @vpscatter_baseidx_sext_v8i16_v8f64(<8 x double> %val, ptr %base, <8 x i16> %idxs, <8 x i1> %m, i32 zeroext %evl) {
 ; RV32-LABEL: vpscatter_baseidx_sext_v8i16_v8f64:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
-; RV32-NEXT:    vsext.vf2 v14, v12
-; RV32-NEXT:    vsll.vi v12, v14, 3
+; RV32-NEXT:    li a2, 8
+; RV32-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; RV32-NEXT:    vwmulsu.vx v14, v12, a2
 ; RV32-NEXT:    vsetvli zero, a1, e64, m4, ta, ma
-; RV32-NEXT:    vsoxei32.v v8, (a0), v12, v0.t
+; RV32-NEXT:    vsoxei32.v v8, (a0), v14, v0.t
 ; RV32-NEXT:    ret
 ;
 ; RV64-LABEL: vpscatter_baseidx_sext_v8i16_v8f64:
@@ -1628,20 +1628,20 @@ define void @vpscatter_baseidx_sext_v8i16_v8f64(<8 x double> %val, ptr %base, <8
 define void @vpscatter_baseidx_zext_v8i16_v8f64(<8 x double> %val, ptr %base, <8 x i16> %idxs, <8 x i1> %m, i32 zeroext %evl) {
 ; RV32-LABEL: vpscatter_baseidx_zext_v8i16_v8f64:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
-; RV32-NEXT:    vzext.vf2 v14, v12
-; RV32-NEXT:    vsll.vi v12, v14, 3
+; RV32-NEXT:    li a2, 8
+; RV32-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; RV32-NEXT:    vwmulu.vx v14, v12, a2
 ; RV32-NEXT:    vsetvli zero, a1, e64, m4, ta, ma
-; RV32-NEXT:    vsoxei32.v v8, (a0), v12, v0.t
+; RV32-NEXT:    vsoxei32.v v8, (a0), v14, v0.t
 ; RV32-NEXT:    ret
 ;
 ; RV64-LABEL: vpscatter_baseidx_zext_v8i16_v8f64:
 ; RV64:       # %bb.0:
-; RV64-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
-; RV64-NEXT:    vzext.vf2 v14, v12
-; RV64-NEXT:    vsll.vi v12, v14, 3
+; RV64-NEXT:    li a2, 8
+; RV64-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; RV64-NEXT:    vwmulu.vx v14, v12, a2
 ; RV64-NEXT:    vsetvli zero, a1, e64, m4, ta, ma
-; RV64-NEXT:    vsoxei32.v v8, (a0), v12, v0.t
+; RV64-NEXT:    vsoxei32.v v8, (a0), v14, v0.t
 ; RV64-NEXT:    ret
   %eidxs = zext <8 x i16> %idxs to <8 x i64>
   %ptrs = getelementptr inbounds double, ptr %base, <8 x i64> %eidxs
@@ -1660,11 +1660,11 @@ define void @vpscatter_baseidx_v8i32_v8f64(<8 x double> %val, ptr %base, <8 x i3
 ;
 ; RV64-LABEL: vpscatter_baseidx_v8i32_v8f64:
 ; RV64:       # %bb.0:
-; RV64-NEXT:    vsetivli zero, 8, e64, m4, ta, ma
-; RV64-NEXT:    vsext.vf2 v16, v12
-; RV64-NEXT:    vsll.vi v12, v16, 3
+; RV64-NEXT:    li a2, 8
+; RV64-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
+; RV64-NEXT:    vwmulsu.vx v16, v12, a2
 ; RV64-NEXT:    vsetvli zero, a1, e64, m4, ta, ma
-; RV64-NEXT:    vsoxei64.v v8, (a0), v12, v0.t
+; RV64-NEXT:    vsoxei64.v v8, (a0), v16, v0.t
 ; RV64-NEXT:    ret
   %ptrs = getelementptr inbounds double, ptr %base, <8 x i32> %idxs
   call void @llvm.vp.scatter.v8f64.v8p0(<8 x double> %val, <8 x ptr> %ptrs, <8 x i1> %m, i32 %evl)
@@ -1682,11 +1682,11 @@ define void @vpscatter_baseidx_sext_v8i32_v8f64(<8 x double> %val, ptr %base, <8
 ;
 ; RV64-LABEL: vpscatter_baseidx_sext_v8i32_v8f64:
 ; RV64:       # %bb.0:
-; RV64-NEXT:    vsetivli zero, 8, e64, m4, ta, ma
-; RV64-NEXT:    vsext.vf2 v16, v12
-; RV64-NEXT:    vsll.vi v12, v16, 3
+; RV64-NEXT:    li a2, 8
+; RV64-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
+; RV64-NEXT:    vwmulsu.vx v16, v12, a2
 ; RV64-NEXT:    vsetvli zero, a1, e64, m4, ta, ma
-; RV64-NEXT:    vsoxei64.v v8, (a0), v12, v0.t
+; RV64-NEXT:    vsoxei64.v v8, (a0), v16, v0.t
 ; RV64-NEXT:    ret
   %eidxs = sext <8 x i32> %idxs to <8 x i64>
   %ptrs = getelementptr inbounds double, ptr %base, <8 x i64> %eidxs
@@ -1705,11 +1705,11 @@ define void @vpscatter_baseidx_zext_v8i32_v8f64(<8 x double> %val, ptr %base, <8
 ;
 ; RV64-LABEL: vpscatter_baseidx_zext_v8i32_v8f64:
 ; RV64:       # %bb.0:
-; RV64-NEXT:    vsetivli zero, 8, e64, m4, ta, ma
-; RV64-NEXT:    vzext.vf2 v16, v12
-; RV64-NEXT:    vsll.vi v12, v16, 3
+; RV64-NEXT:    li a2, 8
+; RV64-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
+; RV64-NEXT:    vwmulu.vx v16, v12, a2
 ; RV64-NEXT:    vsetvli zero, a1, e64, m4, ta, ma
-; RV64-NEXT:    vsoxei64.v v8, (a0), v12, v0.t
+; RV64-NEXT:    vsoxei64.v v8, (a0), v16, v0.t
 ; RV64-NEXT:    ret
   %eidxs = zext <8 x i32> %idxs to <8 x i64>
   %ptrs = getelementptr inbounds double, ptr %base, <8 x i64> %eidxs
@@ -1842,36 +1842,35 @@ define void @vpscatter_baseidx_v32i32_v32f64(<32 x double> %val, ptr %base, <32
 ; RV64-NEXT:    addi sp, sp, -16
 ; RV64-NEXT:    .cfi_def_cfa_offset 16
 ; RV64-NEXT:    csrr a3, vlenb
-; RV64-NEXT:    slli a4, a3, 3
-; RV64-NEXT:    add a3, a4, a3
+; RV64-NEXT:    slli a3, a3, 4
 ; RV64-NEXT:    sub sp, sp, a3
-; RV64-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x09, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 9 * vlenb
-; RV64-NEXT:    addi a3, sp, 16
-; RV64-NEXT:    vs1r.v v0, (a3) # Unknown-size Folded Spill
+; RV64-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x10, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 16 * vlenb
 ; RV64-NEXT:    csrr a3, vlenb
+; RV64-NEXT:    slli a3, a3, 3
 ; RV64-NEXT:    add a3, sp, a3
 ; RV64-NEXT:    addi a3, a3, 16
 ; RV64-NEXT:    vs8r.v v16, (a3) # Unknown-size Folded Spill
+; RV64-NEXT:    addi a3, sp, 16
+; RV64-NEXT:    vs8r.v v8, (a3) # Unknown-size Folded Spill
 ; RV64-NEXT:    li a3, 32
 ; RV64-NEXT:    vsetvli zero, a3, e32, m8, ta, ma
 ; RV64-NEXT:    vle32.v v24, (a1)
+; RV64-NEXT:    li a1, 8
 ; RV64-NEXT:    li a3, 16
 ; RV64-NEXT:    vsetivli zero, 16, e32, m8, ta, ma
 ; RV64-NEXT:    vslidedown.vi v16, v24, 16
-; RV64-NEXT:    vsetivli zero, 16, e64, m8, ta, ma
-; RV64-NEXT:    vsext.vf2 v0, v24
-; RV64-NEXT:    vsext.vf2 v24, v16
-; RV64-NEXT:    vsll.vi v16, v24, 3
-; RV64-NEXT:    vsll.vi v24, v0, 3
+; RV64-NEXT:    vsetivli zero, 16, e32, m4, ta, ma
+; RV64-NEXT:    vwmulsu.vx v8, v16, a1
+; RV64-NEXT:    vwmulsu.vx v16, v24, a1
 ; RV64-NEXT:    mv a1, a2
 ; RV64-NEXT:    bltu a2, a3, .LBB84_2
 ; RV64-NEXT:  # %bb.1:
 ; RV64-NEXT:    li a1, 16
 ; RV64-NEXT:  .LBB84_2:
 ; RV64-NEXT:    addi a3, sp, 16
-; RV64-NEXT:    vl1r.v v0, (a3) # Unknown-size Folded Reload
+; RV64-NEXT:    vl8r.v v24, (a3) # Unknown-size Folded Reload
 ; RV64-NEXT:    vsetvli zero, a1, e64, m8, ta, ma
-; RV64-NEXT:    vsoxei64.v v8, (a0), v24, v0.t
+; RV64-NEXT:    vsoxei64.v v24, (a0), v16, v0.t
 ; RV64-NEXT:    addi a1, a2, -16
 ; RV64-NEXT:    sltu a2, a2, a1
 ; RV64-NEXT:    addi a2, a2, -1
@@ -1879,14 +1878,14 @@ define void @vpscatter_baseidx_v32i32_v32f64(<32 x double> %val, ptr %base, <32
 ; RV64-NEXT:    vslidedown.vi v0, v0, 2
 ; RV64-NEXT:    and a1, a2, a1
 ; RV64-NEXT:    csrr a2, vlenb
+; RV64-NEXT:    slli a2, a2, 3
 ; RV64-NEXT:    add a2, sp, a2
 ; RV64-NEXT:    addi a2, a2, 16
-; RV64-NEXT:    vl8r.v v8, (a2) # Unknown-size Folded Reload
+; RV64-NEXT:    vl8r.v v16, (a2) # Unknown-size Folded Reload
 ; RV64-NEXT:    vsetvli zero, a1, e64, m8, ta, ma
-; RV64-NEXT:    vsoxei64.v v8, (a0), v16, v0.t
+; RV64-NEXT:    vsoxei64.v v16, (a0), v8, v0.t
 ; RV64-NEXT:    csrr a0, vlenb
-; RV64-NEXT:    slli a1, a0, 3
-; RV64-NEXT:    add a0, a1, a0
+; RV64-NEXT:    slli a0, a0, 4
 ; RV64-NEXT:    add sp, sp, a0
 ; RV64-NEXT:    .cfi_def_cfa sp, 16
 ; RV64-NEXT:    addi sp, sp, 16
@@ -1942,24 +1941,22 @@ define void @vpscatter_baseidx_sext_v32i32_v32f64(<32 x double> %val, ptr %base,
 ; RV64-NEXT:    li a3, 32
 ; RV64-NEXT:    vsetvli zero, a3, e32, m8, ta, ma
 ; RV64-NEXT:    vle32.v v24, (a1)
+; RV64-NEXT:    li a1, 8
 ; RV64-NEXT:    li a3, 16
-; RV64-NEXT:    vsetivli zero, 16, e64, m8, ta, ma
-; RV64-NEXT:    vsext.vf2 v16, v24
 ; RV64-NEXT:    vsetivli zero, 16, e32, m8, ta, ma
-; RV64-NEXT:    vslidedown.vi v8, v24, 16
-; RV64-NEXT:    vsetivli zero, 16, e64, m8, ta, ma
-; RV64-NEXT:    vsext.vf2 v24, v8
-; RV64-NEXT:    vsll.vi v8, v24, 3
-; RV64-NEXT:    vsll.vi v24, v16, 3
+; RV64-NEXT:    vslidedown.vi v16, v24, 16
+; RV64-NEXT:    vsetivli zero, 16, e32, m4, ta, ma
+; RV64-NEXT:    vwmulsu.vx v8, v16, a1
+; RV64-NEXT:    vwmulsu.vx v16, v24, a1
 ; RV64-NEXT:    mv a1, a2
 ; RV64-NEXT:    bltu a2, a3, .LBB85_2
 ; RV64-NEXT:  # %bb.1:
 ; RV64-NEXT:    li a1, 16
 ; RV64-NEXT:  .LBB85_2:
 ; RV64-NEXT:    addi a3, sp, 16
-; RV64-NEXT:    vl8r.v v16, (a3) # Unknown-size Folded Reload
+; RV64-NEXT:    vl8r.v v24, (a3) # Unknown-size Folded Reload
 ; RV64-NEXT:    vsetvli zero, a1, e64, m8, ta, ma
-; RV64-NEXT:    vsoxei64.v v16, (a0), v24, v0.t
+; RV64-NEXT:    vsoxei64.v v24, (a0), v16, v0.t
 ; RV64-NEXT:    addi a1, a2, -16
 ; RV64-NEXT:    sltu a2, a2, a1
 ; RV64-NEXT:    addi a2, a2, -1
@@ -2031,24 +2028,22 @@ define void @vpscatter_baseidx_zext_v32i32_v32f64(<32 x double> %val, ptr %base,
 ; RV64-NEXT:    li a3, 32
 ; RV64-NEXT:    vsetvli zero, a3, e32, m8, ta, ma
 ; RV64-NEXT:    vle32.v v24, (a1)
+; RV64-NEXT:    li a1, 8
 ; RV64-NEXT:    li a3, 16
-; RV64-NEXT:    vsetivli zero, 16, e64, m8, ta, ma
-; RV64-NEXT:    vzext.vf2 v16, v24
 ; RV64-NEXT:    vsetivli zero, 16, e32, m8, ta, ma
-; RV64-NEXT:    vslidedown.vi v8, v24, 16
-; RV64-NEXT:    vsetivli zero, 16, e64, m8, ta, ma
-; RV64-NEXT:    vzext.vf2 v24, v8
-; RV64-NEXT:    vsll.vi v8, v24, 3
-; RV64-NEXT:    vsll.vi v24, v16, 3
+; RV64-NEXT:    vslidedown.vi v16, v24, 16
+; RV64-NEXT:    vsetivli zero, 16, e32, m4, ta, ma
+; RV64-NEXT:    vwmulu.vx v8, v16, a1
+; RV64-NEXT:    vwmulu.vx v16, v24, a1
 ; RV64-NEXT:    mv a1, a2
 ; RV64-NEXT:    bltu a2, a3, .LBB86_2
 ; RV64-NEXT:  # %bb.1:
 ; RV64-NEXT:    li a1, 16
 ; RV64-NEXT:  .LBB86_2:
 ; RV64-NEXT:    addi a3, sp, 16
-; RV64-NEXT:    vl8r.v v16, (a3) # Unknown-size Folded Reload
+; RV64-NEXT:    vl8r.v v24, (a3) # Unknown-size Folded Reload
 ; RV64-NEXT:    vsetvli zero, a1, e64, m8, ta, ma
-; RV64-NEXT:    vsoxei64.v v16, (a0), v24, v0.t
+; RV64-NEXT:    vsoxei64.v v24, (a0), v16, v0.t
 ; RV64-NEXT:    addi a1, a2, -16
 ; RV64-NEXT:    sltu a2, a2, a1
 ; RV64-NEXT:    addi a2, a2, -1
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vwsll.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vwsll.ll
index fce22849a58af..7b12a56b78661 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vwsll.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vwsll.ll
@@ -225,9 +225,10 @@ define <4 x i64> @vwsll_vx_i8_v4i64_zext(<4 x i32> %a, i8 %b) {
 define <4 x i64> @vwsll_vi_v4i64(<4 x i32> %a) {
 ; CHECK-LABEL: vwsll_vi_v4i64:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetivli zero, 4, e64, m2, ta, ma
-; CHECK-NEXT:    vzext.vf2 v10, v8
-; CHECK-NEXT:    vsll.vi v8, v10, 2
+; CHECK-NEXT:    li a0, 4
+; CHECK-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
+; CHECK-NEXT:    vwmulu.vx v10, v8, a0
+; CHECK-NEXT:    vmv2r.v v8, v10
 ; CHECK-NEXT:    ret
 ;
 ; CHECK-ZVBB-LABEL: vwsll_vi_v4i64:
@@ -434,9 +435,10 @@ define <8 x i32> @vwsll_vx_i8_v8i32_zext(<8 x i16> %a, i8 %b) {
 define <8 x i32> @vwsll_vi_v8i32(<8 x i16> %a) {
 ; CHECK-LABEL: vwsll_vi_v8i32:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
-; CHECK-NEXT:    vzext.vf2 v10, v8
-; CHECK-NEXT:    vsll.vi v8, v10, 2
+; CHECK-NEXT:    li a0, 4
+; CHECK-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; CHECK-NEXT:    vwmulu.vx v10, v8, a0
+; CHECK-NEXT:    vmv2r.v v8, v10
 ; CHECK-NEXT:    ret
 ;
 ; CHECK-ZVBB-LABEL: vwsll_vi_v8i32:
@@ -654,9 +656,10 @@ define <16 x i16> @vwsll_vx_i8_v16i16_zext(<16 x i8> %a, i8 %b) {
 define <16 x i16> @vwsll_vi_v16i16(<16 x i8> %a) {
 ; CHECK-LABEL: vwsll_vi_v16i16:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetivli zero, 16, e16, m2, ta, ma
-; CHECK-NEXT:    vzext.vf2 v10, v8
-; CHECK-NEXT:    vsll.vi v8, v10, 2
+; CHECK-NEXT:    li a0, 4
+; CHECK-NEXT:    vsetivli zero, 16, e8, m1, ta, ma
+; CHECK-NEXT:    vwmulu.vx v10, v8, a0
+; CHECK-NEXT:    vmv2r.v v8, v10
 ; CHECK-NEXT:    ret
 ;
 ; CHECK-ZVBB-LABEL: vwsll_vi_v16i16:
diff --git a/llvm/test/CodeGen/RISCV/rvv/fmaximum-sdnode.ll b/llvm/test/CodeGen/RISCV/rvv/fmaximum-sdnode.ll
index 8b527fb152d68..7fad68dbfbbda 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fmaximum-sdnode.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fmaximum-sdnode.ll
@@ -755,14 +755,12 @@ define <vscale x 1 x half> @vfmax_nxv1f16_vv_nnana(<vscale x 1 x half> %a, <vsca
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, mf2, ta, ma
 ; ZVFHMIN-NEXT:    vfadd.vv v9, v10, v10
 ; ZVFHMIN-NEXT:    vmfeq.vv v0, v8, v8
-; ZVFHMIN-NEXT:    vsetvli zero, zero, e16, mf4, ta, ma
+; ZVFHMIN-NEXT:    vsetvli zero, zero, e16, mf4, ta, mu
 ; ZVFHMIN-NEXT:    vfncvt.f.f.w v10, v9
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v9, v10
+; ZVFHMIN-NEXT:    vmv1r.v v9, v8
+; ZVFHMIN-NEXT:    vfwcvt.f.f.v v9, v10, v0.t
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, mf2, ta, ma
-; ZVFHMIN-NEXT:    vmerge.vvm v10, v8, v9, v0
-; ZVFHMIN-NEXT:    vmfeq.vv v0, v9, v9
-; ZVFHMIN-NEXT:    vmerge.vvm v8, v9, v8, v0
-; ZVFHMIN-NEXT:    vfmax.vv v9, v10, v8
+; ZVFHMIN-NEXT:    vfmax.vv v9, v9, v8
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e16, mf4, ta, ma
 ; ZVFHMIN-NEXT:    vfncvt.f.f.w v8, v9
 ; ZVFHMIN-NEXT:    ret
@@ -789,14 +787,12 @@ define <vscale x 1 x half> @vfmax_nxv1f16_vv_nnanb(<vscale x 1 x half> %a, <vsca
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, mf2, ta, ma
 ; ZVFHMIN-NEXT:    vfadd.vv v8, v10, v10
 ; ZVFHMIN-NEXT:    vmfeq.vv v0, v9, v9
-; ZVFHMIN-NEXT:    vsetvli zero, zero, e16, mf4, ta, ma
+; ZVFHMIN-NEXT:    vsetvli zero, zero, e16, mf4, ta, mu
 ; ZVFHMIN-NEXT:    vfncvt.f.f.w v10, v8
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v8, v10
+; ZVFHMIN-NEXT:    vmv1r.v v8, v9
+; ZVFHMIN-NEXT:    vfwcvt.f.f.v v8, v10, v0.t
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, mf2, ta, ma
-; ZVFHMIN-NEXT:    vmerge.vvm v10, v9, v8, v0
-; ZVFHMIN-NEXT:    vmfeq.vv v0, v8, v8
-; ZVFHMIN-NEXT:    vmerge.vvm v8, v8, v9, v0
-; ZVFHMIN-NEXT:    vfmax.vv v9, v8, v10
+; ZVFHMIN-NEXT:    vfmax.vv v9, v9, v8
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e16, mf4, ta, ma
 ; ZVFHMIN-NEXT:    vfncvt.f.f.w v8, v9
 ; ZVFHMIN-NEXT:    ret
diff --git a/llvm/test/CodeGen/RISCV/rvv/fminimum-sdnode.ll b/llvm/test/CodeGen/RISCV/rvv/fminimum-sdnode.ll
index 16c119dd87fe0..8cae0bbc03c8e 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fminimum-sdnode.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fminimum-sdnode.ll
@@ -755,14 +755,12 @@ define <vscale x 1 x half> @vfmin_nxv1f16_vv_nnana(<vscale x 1 x half> %a, <vsca
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, mf2, ta, ma
 ; ZVFHMIN-NEXT:    vfadd.vv v9, v10, v10
 ; ZVFHMIN-NEXT:    vmfeq.vv v0, v8, v8
-; ZVFHMIN-NEXT:    vsetvli zero, zero, e16, mf4, ta, ma
+; ZVFHMIN-NEXT:    vsetvli zero, zero, e16, mf4, ta, mu
 ; ZVFHMIN-NEXT:    vfncvt.f.f.w v10, v9
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v9, v10
+; ZVFHMIN-NEXT:    vmv1r.v v9, v8
+; ZVFHMIN-NEXT:    vfwcvt.f.f.v v9, v10, v0.t
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, mf2, ta, ma
-; ZVFHMIN-NEXT:    vmerge.vvm v10, v8, v9, v0
-; ZVFHMIN-NEXT:    vmfeq.vv v0, v9, v9
-; ZVFHMIN-NEXT:    vmerge.vvm v8, v9, v8, v0
-; ZVFHMIN-NEXT:    vfmin.vv v9, v10, v8
+; ZVFHMIN-NEXT:    vfmin.vv v9, v9, v8
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e16, mf4, ta, ma
 ; ZVFHMIN-NEXT:    vfncvt.f.f.w v8, v9
 ; ZVFHMIN-NEXT:    ret
@@ -789,14 +787,12 @@ define <vscale x 1 x half> @vfmin_nxv1f16_vv_nnanb(<vscale x 1 x half> %a, <vsca
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, mf2, ta, ma
 ; ZVFHMIN-NEXT:    vfadd.vv v8, v10, v10
 ; ZVFHMIN-NEXT:    vmfeq.vv v0, v9, v9
-; ZVFHMIN-NEXT:    vsetvli zero, zero, e16, mf4, ta, ma
+; ZVFHMIN-NEXT:    vsetvli zero, zero, e16, mf4, ta, mu
 ; ZVFHMIN-NEXT:    vfncvt.f.f.w v10, v8
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v8, v10
+; ZVFHMIN-NEXT:    vmv1r.v v8, v9
+; ZVFHMIN-NEXT:    vfwcvt.f.f.v v8, v10, v0.t
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, mf2, ta, ma
-; ZVFHMIN-NEXT:    vmerge.vvm v10, v9, v8, v0
-; ZVFHMIN-NEXT:    vmfeq.vv v0, v8, v8
-; ZVFHMIN-NEXT:    vmerge.vvm v8, v8, v9, v0
-; ZVFHMIN-NEXT:    vfmin.vv v9, v8, v10
+; ZVFHMIN-NEXT:    vfmin.vv v9, v9, v8
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e16, mf4, ta, ma
 ; ZVFHMIN-NEXT:    vfncvt.f.f.w v8, v9
 ; ZVFHMIN-NEXT:    ret
diff --git a/llvm/test/CodeGen/RISCV/rvv/mask-reg-alloc.mir b/llvm/test/CodeGen/RISCV/rvv/mask-reg-alloc.mir
index b891207341b33..6fe228f44a1c8 100644
--- a/llvm/test/CodeGen/RISCV/rvv/mask-reg-alloc.mir
+++ b/llvm/test/CodeGen/RISCV/rvv/mask-reg-alloc.mir
@@ -18,7 +18,7 @@ body:             |
     ; CHECK-NEXT: {{  $}}
     ; CHECK-NEXT: dead $x0 = PseudoVSETIVLI 1, 192 /* e8, m1, ta, ma */, implicit-def $vl, implicit-def $vtype
     ; CHECK-NEXT: renamable $v8 = PseudoVMERGE_VIM_M1 undef renamable $v8, killed renamable $v2, 1, killed renamable $v0, 1, 3 /* e8 */, implicit $vl, implicit $vtype
-    ; CHECK-NEXT: renamable $v0 = COPY killed renamable $v1
+    ; CHECK-NEXT: renamable $v0 = COPY killed renamable $v1, implicit $vtype
     ; CHECK-NEXT: renamable $v9 = PseudoVMERGE_VIM_M1 undef renamable $v9, killed renamable $v3, 1, killed renamable $v0, 1, 3 /* e8 */, implicit $vl, implicit $vtype
     ; CHECK-NEXT: renamable $v0 = PseudoVADD_VV_M1 undef renamable $v0, killed renamable $v8, killed renamable $v9, 1, 3 /* e8 */, 0 /* tu, mu */, implicit $vl, implicit $vtype
     ; CHECK-NEXT: PseudoRET implicit $v0
diff --git a/llvm/test/CodeGen/RISCV/rvv/mgather-sdnode.ll b/llvm/test/CodeGen/RISCV/rvv/mgather-sdnode.ll
index 9ee2324f615dd..0fad09f27007c 100644
--- a/llvm/test/CodeGen/RISCV/rvv/mgather-sdnode.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/mgather-sdnode.ll
@@ -775,11 +775,11 @@ define <vscale x 8 x i32> @mgather_baseidx_sext_nxv8i8_nxv8i32(ptr %base, <vscal
 define <vscale x 8 x i32> @mgather_baseidx_zext_nxv8i8_nxv8i32(ptr %base, <vscale x 8 x i8> %idxs, <vscale x 8 x i1> %m, <vscale x 8 x i32> %passthru) {
 ; CHECK-LABEL: mgather_baseidx_zext_nxv8i8_nxv8i32:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a1, zero, e16, m2, ta, ma
-; CHECK-NEXT:    vzext.vf2 v10, v8
-; CHECK-NEXT:    vsll.vi v8, v10, 2
+; CHECK-NEXT:    li a1, 4
+; CHECK-NEXT:    vsetvli a2, zero, e8, m1, ta, ma
+; CHECK-NEXT:    vwmulu.vx v10, v8, a1
 ; CHECK-NEXT:    vsetvli zero, zero, e32, m4, ta, mu
-; CHECK-NEXT:    vluxei16.v v12, (a0), v8, v0.t
+; CHECK-NEXT:    vluxei16.v v12, (a0), v10, v0.t
 ; CHECK-NEXT:    vmv.v.v v8, v12
 ; CHECK-NEXT:    ret
   %eidxs = zext <vscale x 8 x i8> %idxs to <vscale x 8 x i32>
@@ -791,10 +791,11 @@ define <vscale x 8 x i32> @mgather_baseidx_zext_nxv8i8_nxv8i32(ptr %base, <vscal
 define <vscale x 8 x i32> @mgather_baseidx_nxv8i16_nxv8i32(ptr %base, <vscale x 8 x i16> %idxs, <vscale x 8 x i1> %m, <vscale x 8 x i32> %passthru) {
 ; RV32-LABEL: mgather_baseidx_nxv8i16_nxv8i32:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    vsetvli a1, zero, e32, m4, ta, mu
-; RV32-NEXT:    vsext.vf2 v16, v8
-; RV32-NEXT:    vsll.vi v8, v16, 2
-; RV32-NEXT:    vluxei32.v v12, (a0), v8, v0.t
+; RV32-NEXT:    li a1, 4
+; RV32-NEXT:    vsetvli a2, zero, e16, m2, ta, ma
+; RV32-NEXT:    vwmulsu.vx v16, v8, a1
+; RV32-NEXT:    vsetvli zero, zero, e32, m4, ta, mu
+; RV32-NEXT:    vluxei32.v v12, (a0), v16, v0.t
 ; RV32-NEXT:    vmv.v.v v8, v12
 ; RV32-NEXT:    ret
 ;
@@ -815,10 +816,11 @@ define <vscale x 8 x i32> @mgather_baseidx_nxv8i16_nxv8i32(ptr %base, <vscale x
 define <vscale x 8 x i32> @mgather_baseidx_sext_nxv8i16_nxv8i32(ptr %base, <vscale x 8 x i16> %idxs, <vscale x 8 x i1> %m, <vscale x 8 x i32> %passthru) {
 ; RV32-LABEL: mgather_baseidx_sext_nxv8i16_nxv8i32:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    vsetvli a1, zero, e32, m4, ta, mu
-; RV32-NEXT:    vsext.vf2 v16, v8
-; RV32-NEXT:    vsll.vi v8, v16, 2
-; RV32-NEXT:    vluxei32.v v12, (a0), v8, v0.t
+; RV32-NEXT:    li a1, 4
+; RV32-NEXT:    vsetvli a2, zero, e16, m2, ta, ma
+; RV32-NEXT:    vwmulsu.vx v16, v8, a1
+; RV32-NEXT:    vsetvli zero, zero, e32, m4, ta, mu
+; RV32-NEXT:    vluxei32.v v12, (a0), v16, v0.t
 ; RV32-NEXT:    vmv.v.v v8, v12
 ; RV32-NEXT:    ret
 ;
@@ -840,10 +842,11 @@ define <vscale x 8 x i32> @mgather_baseidx_sext_nxv8i16_nxv8i32(ptr %base, <vsca
 define <vscale x 8 x i32> @mgather_baseidx_zext_nxv8i16_nxv8i32(ptr %base, <vscale x 8 x i16> %idxs, <vscale x 8 x i1> %m, <vscale x 8 x i32> %passthru) {
 ; CHECK-LABEL: mgather_baseidx_zext_nxv8i16_nxv8i32:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a1, zero, e32, m4, ta, mu
-; CHECK-NEXT:    vzext.vf2 v16, v8
-; CHECK-NEXT:    vsll.vi v8, v16, 2
-; CHECK-NEXT:    vluxei32.v v12, (a0), v8, v0.t
+; CHECK-NEXT:    li a1, 4
+; CHECK-NEXT:    vsetvli a2, zero, e16, m2, ta, ma
+; CHECK-NEXT:    vwmulu.vx v16, v8, a1
+; CHECK-NEXT:    vsetvli zero, zero, e32, m4, ta, mu
+; CHECK-NEXT:    vluxei32.v v12, (a0), v16, v0.t
 ; CHECK-NEXT:    vmv.v.v v8, v12
 ; CHECK-NEXT:    ret
   %eidxs = zext <vscale x 8 x i16> %idxs to <vscale x 8 x i32>
@@ -863,10 +866,9 @@ define <vscale x 8 x i32> @mgather_baseidx_nxv8i32(ptr %base, <vscale x 8 x i32>
 ;
 ; RV64-LABEL: mgather_baseidx_nxv8i32:
 ; RV64:       # %bb.0:
-; RV64-NEXT:    vsetvli a1, zero, e64, m8, ta, ma
-; RV64-NEXT:    vsext.vf2 v16, v8
-; RV64-NEXT:    vsll.vi v16, v16, 2
-; RV64-NEXT:    vsetvli zero, zero, e32, m4, ta, mu
+; RV64-NEXT:    li a1, 4
+; RV64-NEXT:    vsetvli a2, zero, e32, m4, ta, mu
+; RV64-NEXT:    vwmulsu.vx v16, v8, a1
 ; RV64-NEXT:    vluxei64.v v12, (a0), v16, v0.t
 ; RV64-NEXT:    vmv.v.v v8, v12
 ; RV64-NEXT:    ret
@@ -1034,11 +1036,11 @@ define <vscale x 8 x i64> @mgather_baseidx_sext_nxv8i8_nxv8i64(ptr %base, <vscal
 define <vscale x 8 x i64> @mgather_baseidx_zext_nxv8i8_nxv8i64(ptr %base, <vscale x 8 x i8> %idxs, <vscale x 8 x i1> %m, <vscale x 8 x i64> %passthru) {
 ; CHECK-LABEL: mgather_baseidx_zext_nxv8i8_nxv8i64:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a1, zero, e16, m2, ta, ma
-; CHECK-NEXT:    vzext.vf2 v10, v8
-; CHECK-NEXT:    vsll.vi v8, v10, 3
+; CHECK-NEXT:    li a1, 8
+; CHECK-NEXT:    vsetvli a2, zero, e8, m1, ta, ma
+; CHECK-NEXT:    vwmulu.vx v10, v8, a1
 ; CHECK-NEXT:    vsetvli zero, zero, e64, m8, ta, mu
-; CHECK-NEXT:    vluxei16.v v16, (a0), v8, v0.t
+; CHECK-NEXT:    vluxei16.v v16, (a0), v10, v0.t
 ; CHECK-NEXT:    vmv.v.v v8, v16
 ; CHECK-NEXT:    ret
   %eidxs = zext <vscale x 8 x i8> %idxs to <vscale x 8 x i64>
@@ -1050,11 +1052,11 @@ define <vscale x 8 x i64> @mgather_baseidx_zext_nxv8i8_nxv8i64(ptr %base, <vscal
 define <vscale x 8 x i64> @mgather_baseidx_nxv8i16_nxv8i64(ptr %base, <vscale x 8 x i16> %idxs, <vscale x 8 x i1> %m, <vscale x 8 x i64> %passthru) {
 ; RV32-LABEL: mgather_baseidx_nxv8i16_nxv8i64:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    vsetvli a1, zero, e32, m4, ta, ma
-; RV32-NEXT:    vsext.vf2 v12, v8
-; RV32-NEXT:    vsll.vi v8, v12, 3
+; RV32-NEXT:    li a1, 8
+; RV32-NEXT:    vsetvli a2, zero, e16, m2, ta, ma
+; RV32-NEXT:    vwmulsu.vx v12, v8, a1
 ; RV32-NEXT:    vsetvli zero, zero, e64, m8, ta, mu
-; RV32-NEXT:    vluxei32.v v16, (a0), v8, v0.t
+; RV32-NEXT:    vluxei32.v v16, (a0), v12, v0.t
 ; RV32-NEXT:    vmv.v.v v8, v16
 ; RV32-NEXT:    ret
 ;
@@ -1074,11 +1076,11 @@ define <vscale x 8 x i64> @mgather_baseidx_nxv8i16_nxv8i64(ptr %base, <vscale x
 define <vscale x 8 x i64> @mgather_baseidx_sext_nxv8i16_nxv8i64(ptr %base, <vscale x 8 x i16> %idxs, <vscale x 8 x i1> %m, <vscale x 8 x i64> %passthru) {
 ; RV32-LABEL: mgather_baseidx_sext_nxv8i16_nxv8i64:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    vsetvli a1, zero, e32, m4, ta, ma
-; RV32-NEXT:    vsext.vf2 v12, v8
-; RV32-NEXT:    vsll.vi v8, v12, 3
+; RV32-NEXT:    li a1, 8
+; RV32-NEXT:    vsetvli a2, zero, e16, m2, ta, ma
+; RV32-NEXT:    vwmulsu.vx v12, v8, a1
 ; RV32-NEXT:    vsetvli zero, zero, e64, m8, ta, mu
-; RV32-NEXT:    vluxei32.v v16, (a0), v8, v0.t
+; RV32-NEXT:    vluxei32.v v16, (a0), v12, v0.t
 ; RV32-NEXT:    vmv.v.v v8, v16
 ; RV32-NEXT:    ret
 ;
@@ -1099,11 +1101,11 @@ define <vscale x 8 x i64> @mgather_baseidx_sext_nxv8i16_nxv8i64(ptr %base, <vsca
 define <vscale x 8 x i64> @mgather_baseidx_zext_nxv8i16_nxv8i64(ptr %base, <vscale x 8 x i16> %idxs, <vscale x 8 x i1> %m, <vscale x 8 x i64> %passthru) {
 ; CHECK-LABEL: mgather_baseidx_zext_nxv8i16_nxv8i64:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a1, zero, e32, m4, ta, ma
-; CHECK-NEXT:    vzext.vf2 v12, v8
-; CHECK-NEXT:    vsll.vi v8, v12, 3
+; CHECK-NEXT:    li a1, 8
+; CHECK-NEXT:    vsetvli a2, zero, e16, m2, ta, ma
+; CHECK-NEXT:    vwmulu.vx v12, v8, a1
 ; CHECK-NEXT:    vsetvli zero, zero, e64, m8, ta, mu
-; CHECK-NEXT:    vluxei32.v v16, (a0), v8, v0.t
+; CHECK-NEXT:    vluxei32.v v16, (a0), v12, v0.t
 ; CHECK-NEXT:    vmv.v.v v8, v16
 ; CHECK-NEXT:    ret
   %eidxs = zext <vscale x 8 x i16> %idxs to <vscale x 8 x i64>
@@ -1124,10 +1126,11 @@ define <vscale x 8 x i64> @mgather_baseidx_nxv8i32_nxv8i64(ptr %base, <vscale x
 ;
 ; RV64-LABEL: mgather_baseidx_nxv8i32_nxv8i64:
 ; RV64:       # %bb.0:
-; RV64-NEXT:    vsetvli a1, zero, e64, m8, ta, mu
-; RV64-NEXT:    vsext.vf2 v24, v8
-; RV64-NEXT:    vsll.vi v8, v24, 3
-; RV64-NEXT:    vluxei64.v v16, (a0), v8, v0.t
+; RV64-NEXT:    li a1, 8
+; RV64-NEXT:    vsetvli a2, zero, e32, m4, ta, ma
+; RV64-NEXT:    vwmulsu.vx v24, v8, a1
+; RV64-NEXT:    vsetvli zero, zero, e64, m8, ta, mu
+; RV64-NEXT:    vluxei64.v v16, (a0), v24, v0.t
 ; RV64-NEXT:    vmv.v.v v8, v16
 ; RV64-NEXT:    ret
   %ptrs = getelementptr inbounds i64, ptr %base, <vscale x 8 x i32> %idxs
@@ -1147,10 +1150,11 @@ define <vscale x 8 x i64> @mgather_baseidx_sext_nxv8i32_nxv8i64(ptr %base, <vsca
 ;
 ; RV64-LABEL: mgather_baseidx_sext_nxv8i32_nxv8i64:
 ; RV64:       # %bb.0:
-; RV64-NEXT:    vsetvli a1, zero, e64, m8, ta, mu
-; RV64-NEXT:    vsext.vf2 v24, v8
-; RV64-NEXT:    vsll.vi v8, v24, 3
-; RV64-NEXT:    vluxei64.v v16, (a0), v8, v0.t
+; RV64-NEXT:    li a1, 8
+; RV64-NEXT:    vsetvli a2, zero, e32, m4, ta, ma
+; RV64-NEXT:    vwmulsu.vx v24, v8, a1
+; RV64-NEXT:    vsetvli zero, zero, e64, m8, ta, mu
+; RV64-NEXT:    vluxei64.v v16, (a0), v24, v0.t
 ; RV64-NEXT:    vmv.v.v v8, v16
 ; RV64-NEXT:    ret
   %eidxs = sext <vscale x 8 x i32> %idxs to <vscale x 8 x i64>
@@ -1171,10 +1175,11 @@ define <vscale x 8 x i64> @mgather_baseidx_zext_nxv8i32_nxv8i64(ptr %base, <vsca
 ;
 ; RV64-LABEL: mgather_baseidx_zext_nxv8i32_nxv8i64:
 ; RV64:       # %bb.0:
-; RV64-NEXT:    vsetvli a1, zero, e64, m8, ta, mu
-; RV64-NEXT:    vzext.vf2 v24, v8
-; RV64-NEXT:    vsll.vi v8, v24, 3
-; RV64-NEXT:    vluxei64.v v16, (a0), v8, v0.t
+; RV64-NEXT:    li a1, 8
+; RV64-NEXT:    vsetvli a2, zero, e32, m4, ta, ma
+; RV64-NEXT:    vwmulu.vx v24, v8, a1
+; RV64-NEXT:    vsetvli zero, zero, e64, m8, ta, mu
+; RV64-NEXT:    vluxei64.v v16, (a0), v24, v0.t
 ; RV64-NEXT:    vmv.v.v v8, v16
 ; RV64-NEXT:    ret
   %eidxs = zext <vscale x 8 x i32> %idxs to <vscale x 8 x i64>
@@ -1845,11 +1850,11 @@ define <vscale x 8 x float> @mgather_baseidx_sext_nxv8i8_nxv8f32(ptr %base, <vsc
 define <vscale x 8 x float> @mgather_baseidx_zext_nxv8i8_nxv8f32(ptr %base, <vscale x 8 x i8> %idxs, <vscale x 8 x i1> %m, <vscale x 8 x float> %passthru) {
 ; CHECK-LABEL: mgather_baseidx_zext_nxv8i8_nxv8f32:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a1, zero, e16, m2, ta, ma
-; CHECK-NEXT:    vzext.vf2 v10, v8
-; CHECK-NEXT:    vsll.vi v8, v10, 2
+; CHECK-NEXT:    li a1, 4
+; CHECK-NEXT:    vsetvli a2, zero, e8, m1, ta, ma
+; CHECK-NEXT:    vwmulu.vx v10, v8, a1
 ; CHECK-NEXT:    vsetvli zero, zero, e32, m4, ta, mu
-; CHECK-NEXT:    vluxei16.v v12, (a0), v8, v0.t
+; CHECK-NEXT:    vluxei16.v v12, (a0), v10, v0.t
 ; CHECK-NEXT:    vmv.v.v v8, v12
 ; CHECK-NEXT:    ret
   %eidxs = zext <vscale x 8 x i8> %idxs to <vscale x 8 x i32>
@@ -1861,10 +1866,11 @@ define <vscale x 8 x float> @mgather_baseidx_zext_nxv8i8_nxv8f32(ptr %base, <vsc
 define <vscale x 8 x float> @mgather_baseidx_nxv8i16_nxv8f32(ptr %base, <vscale x 8 x i16> %idxs, <vscale x 8 x i1> %m, <vscale x 8 x float> %passthru) {
 ; RV32-LABEL: mgather_baseidx_nxv8i16_nxv8f32:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    vsetvli a1, zero, e32, m4, ta, mu
-; RV32-NEXT:    vsext.vf2 v16, v8
-; RV32-NEXT:    vsll.vi v8, v16, 2
-; RV32-NEXT:    vluxei32.v v12, (a0), v8, v0.t
+; RV32-NEXT:    li a1, 4
+; RV32-NEXT:    vsetvli a2, zero, e16, m2, ta, ma
+; RV32-NEXT:    vwmulsu.vx v16, v8, a1
+; RV32-NEXT:    vsetvli zero, zero, e32, m4, ta, mu
+; RV32-NEXT:    vluxei32.v v12, (a0), v16, v0.t
 ; RV32-NEXT:    vmv.v.v v8, v12
 ; RV32-NEXT:    ret
 ;
@@ -1885,10 +1891,11 @@ define <vscale x 8 x float> @mgather_baseidx_nxv8i16_nxv8f32(ptr %base, <vscale
 define <vscale x 8 x float> @mgather_baseidx_sext_nxv8i16_nxv8f32(ptr %base, <vscale x 8 x i16> %idxs, <vscale x 8 x i1> %m, <vscale x 8 x float> %passthru) {
 ; RV32-LABEL: mgather_baseidx_sext_nxv8i16_nxv8f32:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    vsetvli a1, zero, e32, m4, ta, mu
-; RV32-NEXT:    vsext.vf2 v16, v8
-; RV32-NEXT:    vsll.vi v8, v16, 2
-; RV32-NEXT:    vluxei32.v v12, (a0), v8, v0.t
+; RV32-NEXT:    li a1, 4
+; RV32-NEXT:    vsetvli a2, zero, e16, m2, ta, ma
+; RV32-NEXT:    vwmulsu.vx v16, v8, a1
+; RV32-NEXT:    vsetvli zero, zero, e32, m4, ta, mu
+; RV32-NEXT:    vluxei32.v v12, (a0), v16, v0.t
 ; RV32-NEXT:    vmv.v.v v8, v12
 ; RV32-NEXT:    ret
 ;
@@ -1910,10 +1917,11 @@ define <vscale x 8 x float> @mgather_baseidx_sext_nxv8i16_nxv8f32(ptr %base, <vs
 define <vscale x 8 x float> @mgather_baseidx_zext_nxv8i16_nxv8f32(ptr %base, <vscale x 8 x i16> %idxs, <vscale x 8 x i1> %m, <vscale x 8 x float> %passthru) {
 ; CHECK-LABEL: mgather_baseidx_zext_nxv8i16_nxv8f32:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a1, zero, e32, m4, ta, mu
-; CHECK-NEXT:    vzext.vf2 v16, v8
-; CHECK-NEXT:    vsll.vi v8, v16, 2
-; CHECK-NEXT:    vluxei32.v v12, (a0), v8, v0.t
+; CHECK-NEXT:    li a1, 4
+; CHECK-NEXT:    vsetvli a2, zero, e16, m2, ta, ma
+; CHECK-NEXT:    vwmulu.vx v16, v8, a1
+; CHECK-NEXT:    vsetvli zero, zero, e32, m4, ta, mu
+; CHECK-NEXT:    vluxei32.v v12, (a0), v16, v0.t
 ; CHECK-NEXT:    vmv.v.v v8, v12
 ; CHECK-NEXT:    ret
   %eidxs = zext <vscale x 8 x i16> %idxs to <vscale x 8 x i32>
@@ -1933,10 +1941,9 @@ define <vscale x 8 x float> @mgather_baseidx_nxv8f32(ptr %base, <vscale x 8 x i3
 ;
 ; RV64-LABEL: mgather_baseidx_nxv8f32:
 ; RV64:       # %bb.0:
-; RV64-NEXT:    vsetvli a1, zero, e64, m8, ta, ma
-; RV64-NEXT:    vsext.vf2 v16, v8
-; RV64-NEXT:    vsll.vi v16, v16, 2
-; RV64-NEXT:    vsetvli zero, zero, e32, m4, ta, mu
+; RV64-NEXT:    li a1, 4
+; RV64-NEXT:    vsetvli a2, zero, e32, m4, ta, mu
+; RV64-NEXT:    vwmulsu.vx v16, v8, a1
 ; RV64-NEXT:    vluxei64.v v12, (a0), v16, v0.t
 ; RV64-NEXT:    vmv.v.v v8, v12
 ; RV64-NEXT:    ret
@@ -2104,11 +2111,11 @@ define <vscale x 8 x double> @mgather_baseidx_sext_nxv8i8_nxv8f64(ptr %base, <vs
 define <vscale x 8 x double> @mgather_baseidx_zext_nxv8i8_nxv8f64(ptr %base, <vscale x 8 x i8> %idxs, <vscale x 8 x i1> %m, <vscale x 8 x double> %passthru) {
 ; CHECK-LABEL: mgather_baseidx_zext_nxv8i8_nxv8f64:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a1, zero, e16, m2, ta, ma
-; CHECK-NEXT:    vzext.vf2 v10, v8
-; CHECK-NEXT:    vsll.vi v8, v10, 3
+; CHECK-NEXT:    li a1, 8
+; CHECK-NEXT:    vsetvli a2, zero, e8, m1, ta, ma
+; CHECK-NEXT:    vwmulu.vx v10, v8, a1
 ; CHECK-NEXT:    vsetvli zero, zero, e64, m8, ta, mu
-; CHECK-NEXT:    vluxei16.v v16, (a0), v8, v0.t
+; CHECK-NEXT:    vluxei16.v v16, (a0), v10, v0.t
 ; CHECK-NEXT:    vmv.v.v v8, v16
 ; CHECK-NEXT:    ret
   %eidxs = zext <vscale x 8 x i8> %idxs to <vscale x 8 x i64>
@@ -2120,11 +2127,11 @@ define <vscale x 8 x double> @mgather_baseidx_zext_nxv8i8_nxv8f64(ptr %base, <vs
 define <vscale x 8 x double> @mgather_baseidx_nxv8i16_nxv8f64(ptr %base, <vscale x 8 x i16> %idxs, <vscale x 8 x i1> %m, <vscale x 8 x double> %passthru) {
 ; RV32-LABEL: mgather_baseidx_nxv8i16_nxv8f64:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    vsetvli a1, zero, e32, m4, ta, ma
-; RV32-NEXT:    vsext.vf2 v12, v8
-; RV32-NEXT:    vsll.vi v8, v12, 3
+; RV32-NEXT:    li a1, 8
+; RV32-NEXT:    vsetvli a2, zero, e16, m2, ta, ma
+; RV32-NEXT:    vwmulsu.vx v12, v8, a1
 ; RV32-NEXT:    vsetvli zero, zero, e64, m8, ta, mu
-; RV32-NEXT:    vluxei32.v v16, (a0), v8, v0.t
+; RV32-NEXT:    vluxei32.v v16, (a0), v12, v0.t
 ; RV32-NEXT:    vmv.v.v v8, v16
 ; RV32-NEXT:    ret
 ;
@@ -2144,11 +2151,11 @@ define <vscale x 8 x double> @mgather_baseidx_nxv8i16_nxv8f64(ptr %base, <vscale
 define <vscale x 8 x double> @mgather_baseidx_sext_nxv8i16_nxv8f64(ptr %base, <vscale x 8 x i16> %idxs, <vscale x 8 x i1> %m, <vscale x 8 x double> %passthru) {
 ; RV32-LABEL: mgather_baseidx_sext_nxv8i16_nxv8f64:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    vsetvli a1, zero, e32, m4, ta, ma
-; RV32-NEXT:    vsext.vf2 v12, v8
-; RV32-NEXT:    vsll.vi v8, v12, 3
+; RV32-NEXT:    li a1, 8
+; RV32-NEXT:    vsetvli a2, zero, e16, m2, ta, ma
+; RV32-NEXT:    vwmulsu.vx v12, v8, a1
 ; RV32-NEXT:    vsetvli zero, zero, e64, m8, ta, mu
-; RV32-NEXT:    vluxei32.v v16, (a0), v8, v0.t
+; RV32-NEXT:    vluxei32.v v16, (a0), v12, v0.t
 ; RV32-NEXT:    vmv.v.v v8, v16
 ; RV32-NEXT:    ret
 ;
@@ -2169,11 +2176,11 @@ define <vscale x 8 x double> @mgather_baseidx_sext_nxv8i16_nxv8f64(ptr %base, <v
 define <vscale x 8 x double> @mgather_baseidx_zext_nxv8i16_nxv8f64(ptr %base, <vscale x 8 x i16> %idxs, <vscale x 8 x i1> %m, <vscale x 8 x double> %passthru) {
 ; CHECK-LABEL: mgather_baseidx_zext_nxv8i16_nxv8f64:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a1, zero, e32, m4, ta, ma
-; CHECK-NEXT:    vzext.vf2 v12, v8
-; CHECK-NEXT:    vsll.vi v8, v12, 3
+; CHECK-NEXT:    li a1, 8
+; CHECK-NEXT:    vsetvli a2, zero, e16, m2, ta, ma
+; CHECK-NEXT:    vwmulu.vx v12, v8, a1
 ; CHECK-NEXT:    vsetvli zero, zero, e64, m8, ta, mu
-; CHECK-NEXT:    vluxei32.v v16, (a0), v8, v0.t
+; CHECK-NEXT:    vluxei32.v v16, (a0), v12, v0.t
 ; CHECK-NEXT:    vmv.v.v v8, v16
 ; CHECK-NEXT:    ret
   %eidxs = zext <vscale x 8 x i16> %idxs to <vscale x 8 x i64>
@@ -2194,10 +2201,11 @@ define <vscale x 8 x double> @mgather_baseidx_nxv8i32_nxv8f64(ptr %base, <vscale
 ;
 ; RV64-LABEL: mgather_baseidx_nxv8i32_nxv8f64:
 ; RV64:       # %bb.0:
-; RV64-NEXT:    vsetvli a1, zero, e64, m8, ta, mu
-; RV64-NEXT:    vsext.vf2 v24, v8
-; RV64-NEXT:    vsll.vi v8, v24, 3
-; RV64-NEXT:    vluxei64.v v16, (a0), v8, v0.t
+; RV64-NEXT:    li a1, 8
+; RV64-NEXT:    vsetvli a2, zero, e32, m4, ta, ma
+; RV64-NEXT:    vwmulsu.vx v24, v8, a1
+; RV64-NEXT:    vsetvli zero, zero, e64, m8, ta, mu
+; RV64-NEXT:    vluxei64.v v16, (a0), v24, v0.t
 ; RV64-NEXT:    vmv.v.v v8, v16
 ; RV64-NEXT:    ret
   %ptrs = getelementptr inbounds double, ptr %base, <vscale x 8 x i32> %idxs
@@ -2217,10 +2225,11 @@ define <vscale x 8 x double> @mgather_baseidx_sext_nxv8i32_nxv8f64(ptr %base, <v
 ;
 ; RV64-LABEL: mgather_baseidx_sext_nxv8i32_nxv8f64:
 ; RV64:       # %bb.0:
-; RV64-NEXT:    vsetvli a1, zero, e64, m8, ta, mu
-; RV64-NEXT:    vsext.vf2 v24, v8
-; RV64-NEXT:    vsll.vi v8, v24, 3
-; RV64-NEXT:    vluxei64.v v16, (a0), v8, v0.t
+; RV64-NEXT:    li a1, 8
+; RV64-NEXT:    vsetvli a2, zero, e32, m4, ta, ma
+; RV64-NEXT:    vwmulsu.vx v24, v8, a1
+; RV64-NEXT:    vsetvli zero, zero, e64, m8, ta, mu
+; RV64-NEXT:    vluxei64.v v16, (a0), v24, v0.t
 ; RV64-NEXT:    vmv.v.v v8, v16
 ; RV64-NEXT:    ret
   %eidxs = sext <vscale x 8 x i32> %idxs to <vscale x 8 x i64>
@@ -2241,10 +2250,11 @@ define <vscale x 8 x double> @mgather_baseidx_zext_nxv8i32_nxv8f64(ptr %base, <v
 ;
 ; RV64-LABEL: mgather_baseidx_zext_nxv8i32_nxv8f64:
 ; RV64:       # %bb.0:
-; RV64-NEXT:    vsetvli a1, zero, e64, m8, ta, mu
-; RV64-NEXT:    vzext.vf2 v24, v8
-; RV64-NEXT:    vsll.vi v8, v24, 3
-; RV64-NEXT:    vluxei64.v v16, (a0), v8, v0.t
+; RV64-NEXT:    li a1, 8
+; RV64-NEXT:    vsetvli a2, zero, e32, m4, ta, ma
+; RV64-NEXT:    vwmulu.vx v24, v8, a1
+; RV64-NEXT:    vsetvli zero, zero, e64, m8, ta, mu
+; RV64-NEXT:    vluxei64.v v16, (a0), v24, v0.t
 ; RV64-NEXT:    vmv.v.v v8, v16
 ; RV64-NEXT:    ret
   %eidxs = zext <vscale x 8 x i32> %idxs to <vscale x 8 x i64>
diff --git a/llvm/test/CodeGen/RISCV/rvv/mscatter-sdnode.ll b/llvm/test/CodeGen/RISCV/rvv/mscatter-sdnode.ll
index 77a1f508d2218..3cf7cc9cb5152 100644
--- a/llvm/test/CodeGen/RISCV/rvv/mscatter-sdnode.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/mscatter-sdnode.ll
@@ -581,11 +581,11 @@ define void @mscatter_baseidx_sext_nxv8i8_nxv8i32(<vscale x 8 x i32> %val, ptr %
 define void @mscatter_baseidx_zext_nxv8i8_nxv8i32(<vscale x 8 x i32> %val, ptr %base, <vscale x 8 x i8> %idxs, <vscale x 8 x i1> %m) {
 ; CHECK-LABEL: mscatter_baseidx_zext_nxv8i8_nxv8i32:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a1, zero, e16, m2, ta, ma
-; CHECK-NEXT:    vzext.vf2 v14, v12
-; CHECK-NEXT:    vsll.vi v12, v14, 2
+; CHECK-NEXT:    li a1, 4
+; CHECK-NEXT:    vsetvli a2, zero, e8, m1, ta, ma
+; CHECK-NEXT:    vwmulu.vx v14, v12, a1
 ; CHECK-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
-; CHECK-NEXT:    vsoxei16.v v8, (a0), v12, v0.t
+; CHECK-NEXT:    vsoxei16.v v8, (a0), v14, v0.t
 ; CHECK-NEXT:    ret
   %eidxs = zext <vscale x 8 x i8> %idxs to <vscale x 8 x i32>
   %ptrs = getelementptr inbounds i32, ptr %base, <vscale x 8 x i32> %eidxs
@@ -596,10 +596,11 @@ define void @mscatter_baseidx_zext_nxv8i8_nxv8i32(<vscale x 8 x i32> %val, ptr %
 define void @mscatter_baseidx_nxv8i16_nxv8i32(<vscale x 8 x i32> %val, ptr %base, <vscale x 8 x i16> %idxs, <vscale x 8 x i1> %m) {
 ; RV32-LABEL: mscatter_baseidx_nxv8i16_nxv8i32:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    vsetvli a1, zero, e32, m4, ta, ma
-; RV32-NEXT:    vsext.vf2 v16, v12
-; RV32-NEXT:    vsll.vi v12, v16, 2
-; RV32-NEXT:    vsoxei32.v v8, (a0), v12, v0.t
+; RV32-NEXT:    li a1, 4
+; RV32-NEXT:    vsetvli a2, zero, e16, m2, ta, ma
+; RV32-NEXT:    vwmulsu.vx v16, v12, a1
+; RV32-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
+; RV32-NEXT:    vsoxei32.v v8, (a0), v16, v0.t
 ; RV32-NEXT:    ret
 ;
 ; RV64-LABEL: mscatter_baseidx_nxv8i16_nxv8i32:
@@ -618,10 +619,11 @@ define void @mscatter_baseidx_nxv8i16_nxv8i32(<vscale x 8 x i32> %val, ptr %base
 define void @mscatter_baseidx_sext_nxv8i16_nxv8i32(<vscale x 8 x i32> %val, ptr %base, <vscale x 8 x i16> %idxs, <vscale x 8 x i1> %m) {
 ; RV32-LABEL: mscatter_baseidx_sext_nxv8i16_nxv8i32:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    vsetvli a1, zero, e32, m4, ta, ma
-; RV32-NEXT:    vsext.vf2 v16, v12
-; RV32-NEXT:    vsll.vi v12, v16, 2
-; RV32-NEXT:    vsoxei32.v v8, (a0), v12, v0.t
+; RV32-NEXT:    li a1, 4
+; RV32-NEXT:    vsetvli a2, zero, e16, m2, ta, ma
+; RV32-NEXT:    vwmulsu.vx v16, v12, a1
+; RV32-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
+; RV32-NEXT:    vsoxei32.v v8, (a0), v16, v0.t
 ; RV32-NEXT:    ret
 ;
 ; RV64-LABEL: mscatter_baseidx_sext_nxv8i16_nxv8i32:
@@ -641,10 +643,11 @@ define void @mscatter_baseidx_sext_nxv8i16_nxv8i32(<vscale x 8 x i32> %val, ptr
 define void @mscatter_baseidx_zext_nxv8i16_nxv8i32(<vscale x 8 x i32> %val, ptr %base, <vscale x 8 x i16> %idxs, <vscale x 8 x i1> %m) {
 ; CHECK-LABEL: mscatter_baseidx_zext_nxv8i16_nxv8i32:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a1, zero, e32, m4, ta, ma
-; CHECK-NEXT:    vzext.vf2 v16, v12
-; CHECK-NEXT:    vsll.vi v12, v16, 2
-; CHECK-NEXT:    vsoxei32.v v8, (a0), v12, v0.t
+; CHECK-NEXT:    li a1, 4
+; CHECK-NEXT:    vsetvli a2, zero, e16, m2, ta, ma
+; CHECK-NEXT:    vwmulu.vx v16, v12, a1
+; CHECK-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
+; CHECK-NEXT:    vsoxei32.v v8, (a0), v16, v0.t
 ; CHECK-NEXT:    ret
   %eidxs = zext <vscale x 8 x i16> %idxs to <vscale x 8 x i32>
   %ptrs = getelementptr inbounds i32, ptr %base, <vscale x 8 x i32> %eidxs
@@ -662,10 +665,9 @@ define void @mscatter_baseidx_nxv8i32(<vscale x 8 x i32> %val, ptr %base, <vscal
 ;
 ; RV64-LABEL: mscatter_baseidx_nxv8i32:
 ; RV64:       # %bb.0:
-; RV64-NEXT:    vsetvli a1, zero, e64, m8, ta, ma
-; RV64-NEXT:    vsext.vf2 v16, v12
-; RV64-NEXT:    vsll.vi v16, v16, 2
-; RV64-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
+; RV64-NEXT:    li a1, 4
+; RV64-NEXT:    vsetvli a2, zero, e32, m4, ta, ma
+; RV64-NEXT:    vwmulsu.vx v16, v12, a1
 ; RV64-NEXT:    vsoxei64.v v8, (a0), v16, v0.t
 ; RV64-NEXT:    ret
   %ptrs = getelementptr inbounds i32, ptr %base, <vscale x 8 x i32> %idxs
@@ -817,11 +819,11 @@ define void @mscatter_baseidx_sext_nxv8i8_nxv8i64(<vscale x 8 x i64> %val, ptr %
 define void @mscatter_baseidx_zext_nxv8i8_nxv8i64(<vscale x 8 x i64> %val, ptr %base, <vscale x 8 x i8> %idxs, <vscale x 8 x i1> %m) {
 ; CHECK-LABEL: mscatter_baseidx_zext_nxv8i8_nxv8i64:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a1, zero, e16, m2, ta, ma
-; CHECK-NEXT:    vzext.vf2 v18, v16
-; CHECK-NEXT:    vsll.vi v16, v18, 3
+; CHECK-NEXT:    li a1, 8
+; CHECK-NEXT:    vsetvli a2, zero, e8, m1, ta, ma
+; CHECK-NEXT:    vwmulu.vx v18, v16, a1
 ; CHECK-NEXT:    vsetvli zero, zero, e64, m8, ta, ma
-; CHECK-NEXT:    vsoxei16.v v8, (a0), v16, v0.t
+; CHECK-NEXT:    vsoxei16.v v8, (a0), v18, v0.t
 ; CHECK-NEXT:    ret
   %eidxs = zext <vscale x 8 x i8> %idxs to <vscale x 8 x i64>
   %ptrs = getelementptr inbounds i64, ptr %base, <vscale x 8 x i64> %eidxs
@@ -832,11 +834,11 @@ define void @mscatter_baseidx_zext_nxv8i8_nxv8i64(<vscale x 8 x i64> %val, ptr %
 define void @mscatter_baseidx_nxv8i16_nxv8i64(<vscale x 8 x i64> %val, ptr %base, <vscale x 8 x i16> %idxs, <vscale x 8 x i1> %m) {
 ; RV32-LABEL: mscatter_baseidx_nxv8i16_nxv8i64:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    vsetvli a1, zero, e32, m4, ta, ma
-; RV32-NEXT:    vsext.vf2 v20, v16
-; RV32-NEXT:    vsll.vi v16, v20, 3
+; RV32-NEXT:    li a1, 8
+; RV32-NEXT:    vsetvli a2, zero, e16, m2, ta, ma
+; RV32-NEXT:    vwmulsu.vx v20, v16, a1
 ; RV32-NEXT:    vsetvli zero, zero, e64, m8, ta, ma
-; RV32-NEXT:    vsoxei32.v v8, (a0), v16, v0.t
+; RV32-NEXT:    vsoxei32.v v8, (a0), v20, v0.t
 ; RV32-NEXT:    ret
 ;
 ; RV64-LABEL: mscatter_baseidx_nxv8i16_nxv8i64:
@@ -854,11 +856,11 @@ define void @mscatter_baseidx_nxv8i16_nxv8i64(<vscale x 8 x i64> %val, ptr %base
 define void @mscatter_baseidx_sext_nxv8i16_nxv8i64(<vscale x 8 x i64> %val, ptr %base, <vscale x 8 x i16> %idxs, <vscale x 8 x i1> %m) {
 ; RV32-LABEL: mscatter_baseidx_sext_nxv8i16_nxv8i64:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    vsetvli a1, zero, e32, m4, ta, ma
-; RV32-NEXT:    vsext.vf2 v20, v16
-; RV32-NEXT:    vsll.vi v16, v20, 3
+; RV32-NEXT:    li a1, 8
+; RV32-NEXT:    vsetvli a2, zero, e16, m2, ta, ma
+; RV32-NEXT:    vwmulsu.vx v20, v16, a1
 ; RV32-NEXT:    vsetvli zero, zero, e64, m8, ta, ma
-; RV32-NEXT:    vsoxei32.v v8, (a0), v16, v0.t
+; RV32-NEXT:    vsoxei32.v v8, (a0), v20, v0.t
 ; RV32-NEXT:    ret
 ;
 ; RV64-LABEL: mscatter_baseidx_sext_nxv8i16_nxv8i64:
@@ -877,11 +879,11 @@ define void @mscatter_baseidx_sext_nxv8i16_nxv8i64(<vscale x 8 x i64> %val, ptr
 define void @mscatter_baseidx_zext_nxv8i16_nxv8i64(<vscale x 8 x i64> %val, ptr %base, <vscale x 8 x i16> %idxs, <vscale x 8 x i1> %m) {
 ; CHECK-LABEL: mscatter_baseidx_zext_nxv8i16_nxv8i64:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a1, zero, e32, m4, ta, ma
-; CHECK-NEXT:    vzext.vf2 v20, v16
-; CHECK-NEXT:    vsll.vi v16, v20, 3
+; CHECK-NEXT:    li a1, 8
+; CHECK-NEXT:    vsetvli a2, zero, e16, m2, ta, ma
+; CHECK-NEXT:    vwmulu.vx v20, v16, a1
 ; CHECK-NEXT:    vsetvli zero, zero, e64, m8, ta, ma
-; CHECK-NEXT:    vsoxei32.v v8, (a0), v16, v0.t
+; CHECK-NEXT:    vsoxei32.v v8, (a0), v20, v0.t
 ; CHECK-NEXT:    ret
   %eidxs = zext <vscale x 8 x i16> %idxs to <vscale x 8 x i64>
   %ptrs = getelementptr inbounds i64, ptr %base, <vscale x 8 x i64> %eidxs
@@ -900,10 +902,11 @@ define void @mscatter_baseidx_nxv8i32_nxv8i64(<vscale x 8 x i64> %val, ptr %base
 ;
 ; RV64-LABEL: mscatter_baseidx_nxv8i32_nxv8i64:
 ; RV64:       # %bb.0:
-; RV64-NEXT:    vsetvli a1, zero, e64, m8, ta, ma
-; RV64-NEXT:    vsext.vf2 v24, v16
-; RV64-NEXT:    vsll.vi v16, v24, 3
-; RV64-NEXT:    vsoxei64.v v8, (a0), v16, v0.t
+; RV64-NEXT:    li a1, 8
+; RV64-NEXT:    vsetvli a2, zero, e32, m4, ta, ma
+; RV64-NEXT:    vwmulsu.vx v24, v16, a1
+; RV64-NEXT:    vsetvli zero, zero, e64, m8, ta, ma
+; RV64-NEXT:    vsoxei64.v v8, (a0), v24, v0.t
 ; RV64-NEXT:    ret
   %ptrs = getelementptr inbounds i64, ptr %base, <vscale x 8 x i32> %idxs
   call void @llvm.masked.scatter.nxv8i64.nxv8p0(<vscale x 8 x i64> %val, <vscale x 8 x ptr> %ptrs, i32 8, <vscale x 8 x i1> %m)
@@ -921,10 +924,11 @@ define void @mscatter_baseidx_sext_nxv8i32_nxv8i64(<vscale x 8 x i64> %val, ptr
 ;
 ; RV64-LABEL: mscatter_baseidx_sext_nxv8i32_nxv8i64:
 ; RV64:       # %bb.0:
-; RV64-NEXT:    vsetvli a1, zero, e64, m8, ta, ma
-; RV64-NEXT:    vsext.vf2 v24, v16
-; RV64-NEXT:    vsll.vi v16, v24, 3
-; RV64-NEXT:    vsoxei64.v v8, (a0), v16, v0.t
+; RV64-NEXT:    li a1, 8
+; RV64-NEXT:    vsetvli a2, zero, e32, m4, ta, ma
+; RV64-NEXT:    vwmulsu.vx v24, v16, a1
+; RV64-NEXT:    vsetvli zero, zero, e64, m8, ta, ma
+; RV64-NEXT:    vsoxei64.v v8, (a0), v24, v0.t
 ; RV64-NEXT:    ret
   %eidxs = sext <vscale x 8 x i32> %idxs to <vscale x 8 x i64>
   %ptrs = getelementptr inbounds i64, ptr %base, <vscale x 8 x i64> %eidxs
@@ -943,10 +947,11 @@ define void @mscatter_baseidx_zext_nxv8i32_nxv8i64(<vscale x 8 x i64> %val, ptr
 ;
 ; RV64-LABEL: mscatter_baseidx_zext_nxv8i32_nxv8i64:
 ; RV64:       # %bb.0:
-; RV64-NEXT:    vsetvli a1, zero, e64, m8, ta, ma
-; RV64-NEXT:    vzext.vf2 v24, v16
-; RV64-NEXT:    vsll.vi v16, v24, 3
-; RV64-NEXT:    vsoxei64.v v8, (a0), v16, v0.t
+; RV64-NEXT:    li a1, 8
+; RV64-NEXT:    vsetvli a2, zero, e32, m4, ta, ma
+; RV64-NEXT:    vwmulu.vx v24, v16, a1
+; RV64-NEXT:    vsetvli zero, zero, e64, m8, ta, ma
+; RV64-NEXT:    vsoxei64.v v8, (a0), v24, v0.t
 ; RV64-NEXT:    ret
   %eidxs = zext <vscale x 8 x i32> %idxs to <vscale x 8 x i64>
   %ptrs = getelementptr inbounds i64, ptr %base, <vscale x 8 x i64> %eidxs
@@ -1475,11 +1480,11 @@ define void @mscatter_baseidx_sext_nxv8i8_nxv8f32(<vscale x 8 x float> %val, ptr
 define void @mscatter_baseidx_zext_nxv8i8_nxv8f32(<vscale x 8 x float> %val, ptr %base, <vscale x 8 x i8> %idxs, <vscale x 8 x i1> %m) {
 ; CHECK-LABEL: mscatter_baseidx_zext_nxv8i8_nxv8f32:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a1, zero, e16, m2, ta, ma
-; CHECK-NEXT:    vzext.vf2 v14, v12
-; CHECK-NEXT:    vsll.vi v12, v14, 2
+; CHECK-NEXT:    li a1, 4
+; CHECK-NEXT:    vsetvli a2, zero, e8, m1, ta, ma
+; CHECK-NEXT:    vwmulu.vx v14, v12, a1
 ; CHECK-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
-; CHECK-NEXT:    vsoxei16.v v8, (a0), v12, v0.t
+; CHECK-NEXT:    vsoxei16.v v8, (a0), v14, v0.t
 ; CHECK-NEXT:    ret
   %eidxs = zext <vscale x 8 x i8> %idxs to <vscale x 8 x i32>
   %ptrs = getelementptr inbounds float, ptr %base, <vscale x 8 x i32> %eidxs
@@ -1490,10 +1495,11 @@ define void @mscatter_baseidx_zext_nxv8i8_nxv8f32(<vscale x 8 x float> %val, ptr
 define void @mscatter_baseidx_nxv8i16_nxv8f32(<vscale x 8 x float> %val, ptr %base, <vscale x 8 x i16> %idxs, <vscale x 8 x i1> %m) {
 ; RV32-LABEL: mscatter_baseidx_nxv8i16_nxv8f32:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    vsetvli a1, zero, e32, m4, ta, ma
-; RV32-NEXT:    vsext.vf2 v16, v12
-; RV32-NEXT:    vsll.vi v12, v16, 2
-; RV32-NEXT:    vsoxei32.v v8, (a0), v12, v0.t
+; RV32-NEXT:    li a1, 4
+; RV32-NEXT:    vsetvli a2, zero, e16, m2, ta, ma
+; RV32-NEXT:    vwmulsu.vx v16, v12, a1
+; RV32-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
+; RV32-NEXT:    vsoxei32.v v8, (a0), v16, v0.t
 ; RV32-NEXT:    ret
 ;
 ; RV64-LABEL: mscatter_baseidx_nxv8i16_nxv8f32:
@@ -1512,10 +1518,11 @@ define void @mscatter_baseidx_nxv8i16_nxv8f32(<vscale x 8 x float> %val, ptr %ba
 define void @mscatter_baseidx_sext_nxv8i16_nxv8f32(<vscale x 8 x float> %val, ptr %base, <vscale x 8 x i16> %idxs, <vscale x 8 x i1> %m) {
 ; RV32-LABEL: mscatter_baseidx_sext_nxv8i16_nxv8f32:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    vsetvli a1, zero, e32, m4, ta, ma
-; RV32-NEXT:    vsext.vf2 v16, v12
-; RV32-NEXT:    vsll.vi v12, v16, 2
-; RV32-NEXT:    vsoxei32.v v8, (a0), v12, v0.t
+; RV32-NEXT:    li a1, 4
+; RV32-NEXT:    vsetvli a2, zero, e16, m2, ta, ma
+; RV32-NEXT:    vwmulsu.vx v16, v12, a1
+; RV32-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
+; RV32-NEXT:    vsoxei32.v v8, (a0), v16, v0.t
 ; RV32-NEXT:    ret
 ;
 ; RV64-LABEL: mscatter_baseidx_sext_nxv8i16_nxv8f32:
@@ -1535,10 +1542,11 @@ define void @mscatter_baseidx_sext_nxv8i16_nxv8f32(<vscale x 8 x float> %val, pt
 define void @mscatter_baseidx_zext_nxv8i16_nxv8f32(<vscale x 8 x float> %val, ptr %base, <vscale x 8 x i16> %idxs, <vscale x 8 x i1> %m) {
 ; CHECK-LABEL: mscatter_baseidx_zext_nxv8i16_nxv8f32:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a1, zero, e32, m4, ta, ma
-; CHECK-NEXT:    vzext.vf2 v16, v12
-; CHECK-NEXT:    vsll.vi v12, v16, 2
-; CHECK-NEXT:    vsoxei32.v v8, (a0), v12, v0.t
+; CHECK-NEXT:    li a1, 4
+; CHECK-NEXT:    vsetvli a2, zero, e16, m2, ta, ma
+; CHECK-NEXT:    vwmulu.vx v16, v12, a1
+; CHECK-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
+; CHECK-NEXT:    vsoxei32.v v8, (a0), v16, v0.t
 ; CHECK-NEXT:    ret
   %eidxs = zext <vscale x 8 x i16> %idxs to <vscale x 8 x i32>
   %ptrs = getelementptr inbounds float, ptr %base, <vscale x 8 x i32> %eidxs
@@ -1556,10 +1564,9 @@ define void @mscatter_baseidx_nxv8f32(<vscale x 8 x float> %val, ptr %base, <vsc
 ;
 ; RV64-LABEL: mscatter_baseidx_nxv8f32:
 ; RV64:       # %bb.0:
-; RV64-NEXT:    vsetvli a1, zero, e64, m8, ta, ma
-; RV64-NEXT:    vsext.vf2 v16, v12
-; RV64-NEXT:    vsll.vi v16, v16, 2
-; RV64-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
+; RV64-NEXT:    li a1, 4
+; RV64-NEXT:    vsetvli a2, zero, e32, m4, ta, ma
+; RV64-NEXT:    vwmulsu.vx v16, v12, a1
 ; RV64-NEXT:    vsoxei64.v v8, (a0), v16, v0.t
 ; RV64-NEXT:    ret
   %ptrs = getelementptr inbounds float, ptr %base, <vscale x 8 x i32> %idxs
@@ -1711,11 +1718,11 @@ define void @mscatter_baseidx_sext_nxv8i8_nxv8f64(<vscale x 8 x double> %val, pt
 define void @mscatter_baseidx_zext_nxv8i8_nxv8f64(<vscale x 8 x double> %val, ptr %base, <vscale x 8 x i8> %idxs, <vscale x 8 x i1> %m) {
 ; CHECK-LABEL: mscatter_baseidx_zext_nxv8i8_nxv8f64:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a1, zero, e16, m2, ta, ma
-; CHECK-NEXT:    vzext.vf2 v18, v16
-; CHECK-NEXT:    vsll.vi v16, v18, 3
+; CHECK-NEXT:    li a1, 8
+; CHECK-NEXT:    vsetvli a2, zero, e8, m1, ta, ma
+; CHECK-NEXT:    vwmulu.vx v18, v16, a1
 ; CHECK-NEXT:    vsetvli zero, zero, e64, m8, ta, ma
-; CHECK-NEXT:    vsoxei16.v v8, (a0), v16, v0.t
+; CHECK-NEXT:    vsoxei16.v v8, (a0), v18, v0.t
 ; CHECK-NEXT:    ret
   %eidxs = zext <vscale x 8 x i8> %idxs to <vscale x 8 x i64>
   %ptrs = getelementptr inbounds double, ptr %base, <vscale x 8 x i64> %eidxs
@@ -1726,11 +1733,11 @@ define void @mscatter_baseidx_zext_nxv8i8_nxv8f64(<vscale x 8 x double> %val, pt
 define void @mscatter_baseidx_nxv8i16_nxv8f64(<vscale x 8 x double> %val, ptr %base, <vscale x 8 x i16> %idxs, <vscale x 8 x i1> %m) {
 ; RV32-LABEL: mscatter_baseidx_nxv8i16_nxv8f64:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    vsetvli a1, zero, e32, m4, ta, ma
-; RV32-NEXT:    vsext.vf2 v20, v16
-; RV32-NEXT:    vsll.vi v16, v20, 3
+; RV32-NEXT:    li a1, 8
+; RV32-NEXT:    vsetvli a2, zero, e16, m2, ta, ma
+; RV32-NEXT:    vwmulsu.vx v20, v16, a1
 ; RV32-NEXT:    vsetvli zero, zero, e64, m8, ta, ma
-; RV32-NEXT:    vsoxei32.v v8, (a0), v16, v0.t
+; RV32-NEXT:    vsoxei32.v v8, (a0), v20, v0.t
 ; RV32-NEXT:    ret
 ;
 ; RV64-LABEL: mscatter_baseidx_nxv8i16_nxv8f64:
@@ -1748,11 +1755,11 @@ define void @mscatter_baseidx_nxv8i16_nxv8f64(<vscale x 8 x double> %val, ptr %b
 define void @mscatter_baseidx_sext_nxv8i16_nxv8f64(<vscale x 8 x double> %val, ptr %base, <vscale x 8 x i16> %idxs, <vscale x 8 x i1> %m) {
 ; RV32-LABEL: mscatter_baseidx_sext_nxv8i16_nxv8f64:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    vsetvli a1, zero, e32, m4, ta, ma
-; RV32-NEXT:    vsext.vf2 v20, v16
-; RV32-NEXT:    vsll.vi v16, v20, 3
+; RV32-NEXT:    li a1, 8
+; RV32-NEXT:    vsetvli a2, zero, e16, m2, ta, ma
+; RV32-NEXT:    vwmulsu.vx v20, v16, a1
 ; RV32-NEXT:    vsetvli zero, zero, e64, m8, ta, ma
-; RV32-NEXT:    vsoxei32.v v8, (a0), v16, v0.t
+; RV32-NEXT:    vsoxei32.v v8, (a0), v20, v0.t
 ; RV32-NEXT:    ret
 ;
 ; RV64-LABEL: mscatter_baseidx_sext_nxv8i16_nxv8f64:
@@ -1771,11 +1778,11 @@ define void @mscatter_baseidx_sext_nxv8i16_nxv8f64(<vscale x 8 x double> %val, p
 define void @mscatter_baseidx_zext_nxv8i16_nxv8f64(<vscale x 8 x double> %val, ptr %base, <vscale x 8 x i16> %idxs, <vscale x 8 x i1> %m) {
 ; CHECK-LABEL: mscatter_baseidx_zext_nxv8i16_nxv8f64:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a1, zero, e32, m4, ta, ma
-; CHECK-NEXT:    vzext.vf2 v20, v16
-; CHECK-NEXT:    vsll.vi v16, v20, 3
+; CHECK-NEXT:    li a1, 8
+; CHECK-NEXT:    vsetvli a2, zero, e16, m2, ta, ma
+; CHECK-NEXT:    vwmulu.vx v20, v16, a1
 ; CHECK-NEXT:    vsetvli zero, zero, e64, m8, ta, ma
-; CHECK-NEXT:    vsoxei32.v v8, (a0), v16, v0.t
+; CHECK-NEXT:    vsoxei32.v v8, (a0), v20, v0.t
 ; CHECK-NEXT:    ret
   %eidxs = zext <vscale x 8 x i16> %idxs to <vscale x 8 x i64>
   %ptrs = getelementptr inbounds double, ptr %base, <vscale x 8 x i64> %eidxs
@@ -1794,10 +1801,11 @@ define void @mscatter_baseidx_nxv8i32_nxv8f64(<vscale x 8 x double> %val, ptr %b
 ;
 ; RV64-LABEL: mscatter_baseidx_nxv8i32_nxv8f64:
 ; RV64:       # %bb.0:
-; RV64-NEXT:    vsetvli a1, zero, e64, m8, ta, ma
-; RV64-NEXT:    vsext.vf2 v24, v16
-; RV64-NEXT:    vsll.vi v16, v24, 3
-; RV64-NEXT:    vsoxei64.v v8, (a0), v16, v0.t
+; RV64-NEXT:    li a1, 8
+; RV64-NEXT:    vsetvli a2, zero, e32, m4, ta, ma
+; RV64-NEXT:    vwmulsu.vx v24, v16, a1
+; RV64-NEXT:    vsetvli zero, zero, e64, m8, ta, ma
+; RV64-NEXT:    vsoxei64.v v8, (a0), v24, v0.t
 ; RV64-NEXT:    ret
   %ptrs = getelementptr inbounds double, ptr %base, <vscale x 8 x i32> %idxs
   call void @llvm.masked.scatter.nxv8f64.nxv8p0(<vscale x 8 x double> %val, <vscale x 8 x ptr> %ptrs, i32 8, <vscale x 8 x i1> %m)
@@ -1815,10 +1823,11 @@ define void @mscatter_baseidx_sext_nxv8i32_nxv8f64(<vscale x 8 x double> %val, p
 ;
 ; RV64-LABEL: mscatter_baseidx_sext_nxv8i32_nxv8f64:
 ; RV64:       # %bb.0:
-; RV64-NEXT:    vsetvli a1, zero, e64, m8, ta, ma
-; RV64-NEXT:    vsext.vf2 v24, v16
-; RV64-NEXT:    vsll.vi v16, v24, 3
-; RV64-NEXT:    vsoxei64.v v8, (a0), v16, v0.t
+; RV64-NEXT:    li a1, 8
+; RV64-NEXT:    vsetvli a2, zero, e32, m4, ta, ma
+; RV64-NEXT:    vwmulsu.vx v24, v16, a1
+; RV64-NEXT:    vsetvli zero, zero, e64, m8, ta, ma
+; RV64-NEXT:    vsoxei64.v v8, (a0), v24, v0.t
 ; RV64-NEXT:    ret
   %eidxs = sext <vscale x 8 x i32> %idxs to <vscale x 8 x i64>
   %ptrs = getelementptr inbounds double, ptr %base, <vscale x 8 x i64> %eidxs
@@ -1837,10 +1846,11 @@ define void @mscatter_baseidx_zext_nxv8i32_nxv8f64(<vscale x 8 x double> %val, p
 ;
 ; RV64-LABEL: mscatter_baseidx_zext_nxv8i32_nxv8f64:
 ; RV64:       # %bb.0:
-; RV64-NEXT:    vsetvli a1, zero, e64, m8, ta, ma
-; RV64-NEXT:    vzext.vf2 v24, v16
-; RV64-NEXT:    vsll.vi v16, v24, 3
-; RV64-NEXT:    vsoxei64.v v8, (a0), v16, v0.t
+; RV64-NEXT:    li a1, 8
+; RV64-NEXT:    vsetvli a2, zero, e32, m4, ta, ma
+; RV64-NEXT:    vwmulu.vx v24, v16, a1
+; RV64-NEXT:    vsetvli zero, zero, e64, m8, ta, ma
+; RV64-NEXT:    vsoxei64.v v8, (a0), v24, v0.t
 ; RV64-NEXT:    ret
   %eidxs = zext <vscale x 8 x i32> %idxs to <vscale x 8 x i64>
   %ptrs = getelementptr inbounds double, ptr %base, <vscale x 8 x i64> %eidxs
@@ -2015,15 +2025,15 @@ define void @mscatter_baseidx_nxv16i16_nxv16f64(<vscale x 8 x double> %val0, <vs
 ; RV32-NEXT:    csrr a1, vlenb
 ; RV32-NEXT:    srli a1, a1, 3
 ; RV32-NEXT:    vslidedown.vx v7, v0, a1
-; RV32-NEXT:    vsetvli a1, zero, e32, m8, ta, ma
-; RV32-NEXT:    vsext.vf2 v24, v8
-; RV32-NEXT:    vsll.vi v8, v24, 3
+; RV32-NEXT:    li a1, 8
+; RV32-NEXT:    vsetvli a2, zero, e16, m4, ta, ma
+; RV32-NEXT:    vwmulsu.vx v24, v8, a1
 ; RV32-NEXT:    vsetvli a1, zero, e64, m8, ta, ma
-; RV32-NEXT:    vsoxei32.v v16, (a0), v8, v0.t
+; RV32-NEXT:    vsoxei32.v v16, (a0), v24, v0.t
 ; RV32-NEXT:    vmv1r.v v0, v7
 ; RV32-NEXT:    addi a1, sp, 16
-; RV32-NEXT:    vl8r.v v16, (a1) # Unknown-size Folded Reload
-; RV32-NEXT:    vsoxei32.v v16, (a0), v12, v0.t
+; RV32-NEXT:    vl8r.v v8, (a1) # Unknown-size Folded Reload
+; RV32-NEXT:    vsoxei32.v v8, (a0), v28, v0.t
 ; RV32-NEXT:    csrr a0, vlenb
 ; RV32-NEXT:    slli a0, a0, 3
 ; RV32-NEXT:    add sp, sp, a0
diff --git a/llvm/test/CodeGen/RISCV/rvv/pr125306.ll b/llvm/test/CodeGen/RISCV/rvv/pr125306.ll
new file mode 100644
index 0000000000000..111f87de220db
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/rvv/pr125306.ll
@@ -0,0 +1,118 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc < %s -mtriple=riscv64 -mattr=+m,+v | FileCheck %s
+
+; Test for an "Invalid size request on a scalable vector". Attempts to reduce
+; the test faurther were not successful. The failure requires a shuffle with 2
+; scalable->fixed extracts from the same vector. 0 is the only valid index for a
+; scalable->fixed extract so the 2 extract must be the same. Shuffles with the
+; same source are aggressively canonicalized to a unary shuffle so it requires
+; the extracts to become identical through other optimizations without the
+; shuffle being canonicalized before it is lowered.
+
+define <2 x i32> @main(ptr %0) {
+; CHECK-LABEL: main:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetivli zero, 16, e32, m4, ta, ma
+; CHECK-NEXT:    vmv.v.i v8, 0
+; CHECK-NEXT:    vse32.v v8, (zero)
+; CHECK-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
+; CHECK-NEXT:    vmv.v.i v8, 0
+; CHECK-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
+; CHECK-NEXT:    vmv.v.i v10, 0
+; CHECK-NEXT:    li a2, 64
+; CHECK-NEXT:    sw zero, 80(zero)
+; CHECK-NEXT:    lui a1, 7
+; CHECK-NEXT:    lui a3, 1
+; CHECK-NEXT:    vsetivli zero, 2, e32, mf2, ta, ma
+; CHECK-NEXT:    vid.v v11
+; CHECK-NEXT:    li a4, 16
+; CHECK-NEXT:    lui a5, 2
+; CHECK-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
+; CHECK-NEXT:    vse32.v v10, (a2)
+; CHECK-NEXT:    vsetivli zero, 2, e32, mf2, ta, ma
+; CHECK-NEXT:    vmv.v.i v10, 0
+; CHECK-NEXT:    li a2, 24
+; CHECK-NEXT:    sh zero, -392(a3)
+; CHECK-NEXT:    sh zero, 534(a3)
+; CHECK-NEXT:    sh zero, 1460(a3)
+; CHECK-NEXT:    li a3, 32
+; CHECK-NEXT:    vse32.v v10, (a2)
+; CHECK-NEXT:    li a2, 40
+; CHECK-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
+; CHECK-NEXT:    vse32.v v8, (a0)
+; CHECK-NEXT:    sh zero, -1710(a5)
+; CHECK-NEXT:    sh zero, -784(a5)
+; CHECK-NEXT:    sh zero, 142(a5)
+; CHECK-NEXT:    lw a5, -304(a1)
+; CHECK-NEXT:    vsetivli zero, 2, e32, mf2, ta, ma
+; CHECK-NEXT:    vadd.vi v9, v11, -1
+; CHECK-NEXT:    vse32.v v10, (a3)
+; CHECK-NEXT:    sh zero, 0(a0)
+; CHECK-NEXT:    lw a0, -188(a1)
+; CHECK-NEXT:    vse32.v v10, (a2)
+; CHECK-NEXT:    lw a2, -188(a1)
+; CHECK-NEXT:    lw a3, 1244(a1)
+; CHECK-NEXT:    vmv.v.x v8, a0
+; CHECK-NEXT:    lw a0, 1244(a1)
+; CHECK-NEXT:    lw a1, -304(a1)
+; CHECK-NEXT:    vmv.v.x v10, a3
+; CHECK-NEXT:    vmv.v.x v11, a5
+; CHECK-NEXT:    vslide1down.vx v8, v8, zero
+; CHECK-NEXT:    vslide1down.vx v10, v10, zero
+; CHECK-NEXT:    vmin.vv v8, v10, v8
+; CHECK-NEXT:    vmv.v.x v10, a0
+; CHECK-NEXT:    vslide1down.vx v11, v11, zero
+; CHECK-NEXT:    vmin.vx v10, v10, a2
+; CHECK-NEXT:    vmin.vx v10, v10, a1
+; CHECK-NEXT:    vmin.vv v11, v8, v11
+; CHECK-NEXT:    vmv1r.v v8, v10
+; CHECK-NEXT:    vand.vv v9, v11, v9
+; CHECK-NEXT:    vslideup.vi v8, v10, 1
+; CHECK-NEXT:    vse32.v v9, (a4)
+; CHECK-NEXT:    sh zero, 0(zero)
+; CHECK-NEXT:    ret
+entry:
+  store <16 x i32> zeroinitializer, ptr null, align 4
+  store <8 x i32> zeroinitializer, ptr %0, align 4
+  store <4 x i32> zeroinitializer, ptr getelementptr inbounds nuw (i8, ptr null, i64 64), align 4
+  store i32 0, ptr getelementptr inbounds nuw (i8, ptr null, i64 80), align 4
+  %1 = load i32, ptr getelementptr inbounds nuw (i8, ptr null, i64 29916), align 4
+  %broadcast.splatinsert53 = insertelement <4 x i32> zeroinitializer, i32 %1, i64 0
+  %2 = load i32, ptr getelementptr inbounds nuw (i8, ptr null, i64 28484), align 4
+  %broadcast.splatinsert55 = insertelement <4 x i32> zeroinitializer, i32 %2, i64 0
+  %3 = call <4 x i32> @llvm.smin.v4i32(<4 x i32> %broadcast.splatinsert53, <4 x i32> %broadcast.splatinsert55)
+  %4 = load i32, ptr getelementptr inbounds nuw (i8, ptr null, i64 28368), align 4
+  %broadcast.splatinsert57 = insertelement <4 x i32> zeroinitializer, i32 %4, i64 0
+  %5 = call <4 x i32> @llvm.smin.v4i32(<4 x i32> %3, <4 x i32> %broadcast.splatinsert57)
+  store i16 0, ptr getelementptr inbounds nuw (i8, ptr null, i64 3704), align 2
+  store i16 0, ptr getelementptr inbounds nuw (i8, ptr null, i64 4630), align 2
+  %6 = shufflevector <4 x i32> %5, <4 x i32> zeroinitializer, <2 x i32> <i32 0, i32 4>
+  store <2 x i32> %6, ptr getelementptr inbounds nuw (i8, ptr null, i64 16), align 4
+  store i16 0, ptr getelementptr inbounds nuw (i8, ptr null, i64 5556), align 2
+  store i16 0, ptr getelementptr inbounds nuw (i8, ptr null, i64 6482), align 2
+  store <2 x i32> zeroinitializer, ptr getelementptr inbounds nuw (i8, ptr null, i64 24), align 4
+  store i16 0, ptr getelementptr inbounds nuw (i8, ptr null, i64 7408), align 2
+  store i16 0, ptr getelementptr inbounds nuw (i8, ptr null, i64 8334), align 2
+  store <2 x i32> zeroinitializer, ptr getelementptr inbounds nuw (i8, ptr null, i64 32), align 4
+  store i16 0, ptr %0, align 2
+  store <2 x i32> zeroinitializer, ptr getelementptr inbounds nuw (i8, ptr null, i64 40), align 4
+  %7 = load i32, ptr getelementptr inbounds nuw (i8, ptr null, i64 29916), align 4
+  %broadcast.splatinsert165 = insertelement <4 x i32> poison, i32 %7, i64 0
+  %8 = load i32, ptr getelementptr inbounds nuw (i8, ptr null, i64 28484), align 4
+  %broadcast.splatinsert167 = insertelement <4 x i32> poison, i32 %8, i64 0
+  %9 = call <4 x i32> @llvm.smin.v4i32(<4 x i32> %broadcast.splatinsert165, <4 x i32> %broadcast.splatinsert167)
+  %10 = load i32, ptr getelementptr inbounds nuw (i8, ptr null, i64 28368), align 4
+  %broadcast.splatinsert169 = insertelement <4 x i32> poison, i32 %10, i64 0
+  %11 = call <4 x i32> @llvm.smin.v4i32(<4 x i32> %9, <4 x i32> %broadcast.splatinsert169)
+  store i16 0, ptr null, align 2
+  %12 = load i32, ptr getelementptr inbounds nuw (i8, ptr null, i64 29916), align 4
+  %broadcast.splatinsert179 = insertelement <4 x i32> poison, i32 %12, i64 0
+  %13 = load i32, ptr getelementptr inbounds nuw (i8, ptr null, i64 28484), align 4
+  %broadcast.splatinsert181 = insertelement <4 x i32> poison, i32 %13, i64 0
+  %14 = call <4 x i32> @llvm.smin.v4i32(<4 x i32> %broadcast.splatinsert179, <4 x i32> %broadcast.splatinsert181)
+  %15 = load i32, ptr getelementptr inbounds nuw (i8, ptr null, i64 28368), align 4
+  %broadcast.splatinsert183 = insertelement <4 x i32> poison, i32 %15, i64 0
+  %16 = call <4 x i32> @llvm.smin.v4i32(<4 x i32> %14, <4 x i32> %broadcast.splatinsert183)
+  %17 = shufflevector <4 x i32> %11, <4 x i32> %16, <2 x i32> <i32 0, i32 4>
+  ret <2 x i32> %17
+}
diff --git a/llvm/test/CodeGen/RISCV/rvv/rv32-spill-vector-csr.ll b/llvm/test/CodeGen/RISCV/rvv/rv32-spill-vector-csr.ll
index f636ab9ebd0ce..1205ff17d113e 100644
--- a/llvm/test/CodeGen/RISCV/rvv/rv32-spill-vector-csr.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/rv32-spill-vector-csr.ll
@@ -8,14 +8,17 @@
 
 @.str = private unnamed_addr constant [6 x i8] c"hello\00", align 1
 
-define <vscale x 1 x double> @foo(<vscale x 1 x double> %a, <vscale x 1 x double> %b, <vscale x 1 x double> %c, i32 %gvl) nounwind
+define <vscale x 1 x double> @foo(<vscale x 1 x double> %a, <vscale x 1 x double> %b, <vscale x 1 x double> %c, i32 %gvl)
 ; SPILL-O0-LABEL: foo:
 ; SPILL-O0:       # %bb.0:
 ; SPILL-O0-NEXT:    addi sp, sp, -32
+; SPILL-O0-NEXT:    .cfi_def_cfa_offset 32
 ; SPILL-O0-NEXT:    sw ra, 28(sp) # 4-byte Folded Spill
+; SPILL-O0-NEXT:    .cfi_offset ra, -4
 ; SPILL-O0-NEXT:    csrr a1, vlenb
 ; SPILL-O0-NEXT:    slli a1, a1, 1
 ; SPILL-O0-NEXT:    sub sp, sp, a1
+; SPILL-O0-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x20, 0x22, 0x11, 0x02, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 32 + 2 * vlenb
 ; SPILL-O0-NEXT:    sw a0, 8(sp) # 4-byte Folded Spill
 ; SPILL-O0-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; SPILL-O0-NEXT:    vmv1r.v v10, v9
@@ -46,18 +49,25 @@ define <vscale x 1 x double> @foo(<vscale x 1 x double> %a, <vscale x 1 x double
 ; SPILL-O0-NEXT:    csrr a0, vlenb
 ; SPILL-O0-NEXT:    slli a0, a0, 1
 ; SPILL-O0-NEXT:    add sp, sp, a0
+; SPILL-O0-NEXT:    .cfi_def_cfa sp, 32
 ; SPILL-O0-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
+; SPILL-O0-NEXT:    .cfi_restore ra
 ; SPILL-O0-NEXT:    addi sp, sp, 32
+; SPILL-O0-NEXT:    .cfi_def_cfa_offset 0
 ; SPILL-O0-NEXT:    ret
 ;
 ; SPILL-O2-LABEL: foo:
 ; SPILL-O2:       # %bb.0:
 ; SPILL-O2-NEXT:    addi sp, sp, -32
+; SPILL-O2-NEXT:    .cfi_def_cfa_offset 32
 ; SPILL-O2-NEXT:    sw ra, 28(sp) # 4-byte Folded Spill
 ; SPILL-O2-NEXT:    sw s0, 24(sp) # 4-byte Folded Spill
+; SPILL-O2-NEXT:    .cfi_offset ra, -4
+; SPILL-O2-NEXT:    .cfi_offset s0, -8
 ; SPILL-O2-NEXT:    csrr a1, vlenb
 ; SPILL-O2-NEXT:    slli a1, a1, 1
 ; SPILL-O2-NEXT:    sub sp, sp, a1
+; SPILL-O2-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x20, 0x22, 0x11, 0x02, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 32 + 2 * vlenb
 ; SPILL-O2-NEXT:    mv s0, a0
 ; SPILL-O2-NEXT:    addi a1, sp, 16
 ; SPILL-O2-NEXT:    vs1r.v v8, (a1) # Unknown-size Folded Spill
@@ -81,41 +91,46 @@ define <vscale x 1 x double> @foo(<vscale x 1 x double> %a, <vscale x 1 x double
 ; SPILL-O2-NEXT:    csrr a0, vlenb
 ; SPILL-O2-NEXT:    slli a0, a0, 1
 ; SPILL-O2-NEXT:    add sp, sp, a0
+; SPILL-O2-NEXT:    .cfi_def_cfa sp, 32
 ; SPILL-O2-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
 ; SPILL-O2-NEXT:    lw s0, 24(sp) # 4-byte Folded Reload
+; SPILL-O2-NEXT:    .cfi_restore ra
+; SPILL-O2-NEXT:    .cfi_restore s0
 ; SPILL-O2-NEXT:    addi sp, sp, 32
+; SPILL-O2-NEXT:    .cfi_def_cfa_offset 0
 ; SPILL-O2-NEXT:    ret
 ;
 ; SPILL-O2-ZCMP-LABEL: foo:
 ; SPILL-O2-ZCMP:       # %bb.0:
-; SPILL-O2-ZCMP-NEXT:    cm.push {ra, s0}, -32
+; SPILL-O2-ZCMP-NEXT:    cm.push {ra, s0}, -16
+; SPILL-O2-ZCMP-NEXT:    .cfi_def_cfa_offset 16
+; SPILL-O2-ZCMP-NEXT:    .cfi_offset ra, -8
+; SPILL-O2-ZCMP-NEXT:    .cfi_offset s0, -4
 ; SPILL-O2-ZCMP-NEXT:    csrr a1, vlenb
 ; SPILL-O2-ZCMP-NEXT:    slli a1, a1, 1
 ; SPILL-O2-ZCMP-NEXT:    sub sp, sp, a1
+; SPILL-O2-ZCMP-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x02, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 2 * vlenb
 ; SPILL-O2-ZCMP-NEXT:    mv s0, a0
-; SPILL-O2-ZCMP-NEXT:    addi a1, sp, 16
-; SPILL-O2-ZCMP-NEXT:    vs1r.v v8, (a1) # Unknown-size Folded Spill
+; SPILL-O2-ZCMP-NEXT:    vs1r.v v8, (sp) # Unknown-size Folded Spill
 ; SPILL-O2-ZCMP-NEXT:    vsetvli zero, a0, e64, m1, ta, ma
 ; SPILL-O2-ZCMP-NEXT:    vfadd.vv v9, v8, v9
 ; SPILL-O2-ZCMP-NEXT:    csrr a0, vlenb
 ; SPILL-O2-ZCMP-NEXT:    add a0, a0, sp
-; SPILL-O2-ZCMP-NEXT:    addi a0, a0, 16
 ; SPILL-O2-ZCMP-NEXT:    vs1r.v v9, (a0) # Unknown-size Folded Spill
 ; SPILL-O2-ZCMP-NEXT:    lui a0, %hi(.L.str)
 ; SPILL-O2-ZCMP-NEXT:    addi a0, a0, %lo(.L.str)
 ; SPILL-O2-ZCMP-NEXT:    call puts
 ; SPILL-O2-ZCMP-NEXT:    csrr a0, vlenb
 ; SPILL-O2-ZCMP-NEXT:    add a0, a0, sp
-; SPILL-O2-ZCMP-NEXT:    addi a0, a0, 16
 ; SPILL-O2-ZCMP-NEXT:    vl1r.v v8, (a0) # Unknown-size Folded Reload
-; SPILL-O2-ZCMP-NEXT:    addi a0, sp, 16
-; SPILL-O2-ZCMP-NEXT:    vl1r.v v9, (a0) # Unknown-size Folded Reload
+; SPILL-O2-ZCMP-NEXT:    vl1r.v v9, (sp) # Unknown-size Folded Reload
 ; SPILL-O2-ZCMP-NEXT:    vsetvli zero, s0, e64, m1, ta, ma
 ; SPILL-O2-ZCMP-NEXT:    vfadd.vv v8, v9, v8
 ; SPILL-O2-ZCMP-NEXT:    csrr a0, vlenb
 ; SPILL-O2-ZCMP-NEXT:    slli a0, a0, 1
 ; SPILL-O2-ZCMP-NEXT:    add sp, sp, a0
-; SPILL-O2-ZCMP-NEXT:    cm.popret {ra, s0}, 32
+; SPILL-O2-ZCMP-NEXT:    .cfi_def_cfa sp, 16
+; SPILL-O2-ZCMP-NEXT:    cm.popret {ra, s0}, 16
 {
    %x = call <vscale x 1 x double> @llvm.riscv.vfadd.nxv1f64.nxv1f64(<vscale x 1 x double> undef, <vscale x 1 x double> %a, <vscale x 1 x double> %b, i32 7, i32 %gvl)
    %call = call signext i32 @puts(ptr @.str)
diff --git a/llvm/test/CodeGen/RISCV/rvv/rvv-peephole-vmerge-vops.ll b/llvm/test/CodeGen/RISCV/rvv/rvv-peephole-vmerge-vops.ll
index d329979857a6b..403cc0eb9dce1 100644
--- a/llvm/test/CodeGen/RISCV/rvv/rvv-peephole-vmerge-vops.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/rvv-peephole-vmerge-vops.ll
@@ -894,9 +894,10 @@ define void @test_dag_loop() {
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a0, zero, e8, m4, ta, ma
 ; CHECK-NEXT:    vmclr.m v0
+; CHECK-NEXT:    vsetivli zero, 0, e8, m4, ta, ma
 ; CHECK-NEXT:    vmv.v.i v8, 0
 ; CHECK-NEXT:    vmv.v.i v12, 0
-; CHECK-NEXT:    vsetivli zero, 0, e8, m4, tu, mu
+; CHECK-NEXT:    vsetvli zero, zero, e8, m4, tu, mu
 ; CHECK-NEXT:    vssubu.vx v12, v8, zero, v0.t
 ; CHECK-NEXT:    vsetvli zero, zero, e8, m4, ta, ma
 ; CHECK-NEXT:    vmseq.vv v0, v12, v8
diff --git a/llvm/test/CodeGen/RISCV/rvv/setcc-fp-vp.ll b/llvm/test/CodeGen/RISCV/rvv/setcc-fp-vp.ll
index 30ef3dccd426b..e6272701a6033 100644
--- a/llvm/test/CodeGen/RISCV/rvv/setcc-fp-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/setcc-fp-vp.ll
@@ -30,13 +30,11 @@ define <vscale x 1 x i1> @fcmp_oeq_vv_nxv1bf16(<vscale x 1 x bfloat> %va, <vscal
 define <vscale x 1 x i1> @fcmp_oeq_vf_nxv1bf16(<vscale x 1 x bfloat> %va, bfloat %b, <vscale x 1 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: fcmp_oeq_vf_nxv1bf16:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    fmv.x.h a1, fa0
+; CHECK-NEXT:    fcvt.s.bf16 fa5, fa0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, ma
-; CHECK-NEXT:    vmv.v.x v9, a1
-; CHECK-NEXT:    vfwcvtbf16.f.f.v v10, v8
-; CHECK-NEXT:    vfwcvtbf16.f.f.v v8, v9
+; CHECK-NEXT:    vfwcvtbf16.f.f.v v9, v8
 ; CHECK-NEXT:    vsetvli zero, zero, e32, mf2, ta, ma
-; CHECK-NEXT:    vmfeq.vv v0, v10, v8, v0.t
+; CHECK-NEXT:    vmfeq.vf v0, v9, fa5, v0.t
 ; CHECK-NEXT:    ret
   %elt.head = insertelement <vscale x 1 x bfloat> poison, bfloat %b, i32 0
   %vb = shufflevector <vscale x 1 x bfloat> %elt.head, <vscale x 1 x bfloat> poison, <vscale x 1 x i32> zeroinitializer
@@ -47,13 +45,11 @@ define <vscale x 1 x i1> @fcmp_oeq_vf_nxv1bf16(<vscale x 1 x bfloat> %va, bfloat
 define <vscale x 1 x i1> @fcmp_oeq_vf_swap_nxv1bf16(<vscale x 1 x bfloat> %va, bfloat %b, <vscale x 1 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: fcmp_oeq_vf_swap_nxv1bf16:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    fmv.x.h a1, fa0
+; CHECK-NEXT:    fcvt.s.bf16 fa5, fa0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, ma
-; CHECK-NEXT:    vmv.v.x v9, a1
-; CHECK-NEXT:    vfwcvtbf16.f.f.v v10, v8
-; CHECK-NEXT:    vfwcvtbf16.f.f.v v8, v9
+; CHECK-NEXT:    vfwcvtbf16.f.f.v v9, v8
 ; CHECK-NEXT:    vsetvli zero, zero, e32, mf2, ta, ma
-; CHECK-NEXT:    vmfeq.vv v0, v8, v10, v0.t
+; CHECK-NEXT:    vmfeq.vf v0, v9, fa5, v0.t
 ; CHECK-NEXT:    ret
   %elt.head = insertelement <vscale x 1 x bfloat> poison, bfloat %b, i32 0
   %vb = shufflevector <vscale x 1 x bfloat> %elt.head, <vscale x 1 x bfloat> poison, <vscale x 1 x i32> zeroinitializer
@@ -77,13 +73,11 @@ define <vscale x 1 x i1> @fcmp_ogt_vv_nxv1bf16(<vscale x 1 x bfloat> %va, <vscal
 define <vscale x 1 x i1> @fcmp_ogt_vf_nxv1bf16(<vscale x 1 x bfloat> %va, bfloat %b, <vscale x 1 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: fcmp_ogt_vf_nxv1bf16:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    fmv.x.h a1, fa0
+; CHECK-NEXT:    fcvt.s.bf16 fa5, fa0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, ma
-; CHECK-NEXT:    vmv.v.x v9, a1
-; CHECK-NEXT:    vfwcvtbf16.f.f.v v10, v8
-; CHECK-NEXT:    vfwcvtbf16.f.f.v v8, v9
+; CHECK-NEXT:    vfwcvtbf16.f.f.v v9, v8
 ; CHECK-NEXT:    vsetvli zero, zero, e32, mf2, ta, ma
-; CHECK-NEXT:    vmflt.vv v0, v8, v10, v0.t
+; CHECK-NEXT:    vmfgt.vf v0, v9, fa5, v0.t
 ; CHECK-NEXT:    ret
   %elt.head = insertelement <vscale x 1 x bfloat> poison, bfloat %b, i32 0
   %vb = shufflevector <vscale x 1 x bfloat> %elt.head, <vscale x 1 x bfloat> poison, <vscale x 1 x i32> zeroinitializer
@@ -94,13 +88,11 @@ define <vscale x 1 x i1> @fcmp_ogt_vf_nxv1bf16(<vscale x 1 x bfloat> %va, bfloat
 define <vscale x 1 x i1> @fcmp_ogt_vf_swap_nxv1bf16(<vscale x 1 x bfloat> %va, bfloat %b, <vscale x 1 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: fcmp_ogt_vf_swap_nxv1bf16:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    fmv.x.h a1, fa0
+; CHECK-NEXT:    fcvt.s.bf16 fa5, fa0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, ma
-; CHECK-NEXT:    vmv.v.x v9, a1
-; CHECK-NEXT:    vfwcvtbf16.f.f.v v10, v8
-; CHECK-NEXT:    vfwcvtbf16.f.f.v v8, v9
+; CHECK-NEXT:    vfwcvtbf16.f.f.v v9, v8
 ; CHECK-NEXT:    vsetvli zero, zero, e32, mf2, ta, ma
-; CHECK-NEXT:    vmflt.vv v0, v10, v8, v0.t
+; CHECK-NEXT:    vmflt.vf v0, v9, fa5, v0.t
 ; CHECK-NEXT:    ret
   %elt.head = insertelement <vscale x 1 x bfloat> poison, bfloat %b, i32 0
   %vb = shufflevector <vscale x 1 x bfloat> %elt.head, <vscale x 1 x bfloat> poison, <vscale x 1 x i32> zeroinitializer
@@ -124,13 +116,11 @@ define <vscale x 1 x i1> @fcmp_oge_vv_nxv1bf16(<vscale x 1 x bfloat> %va, <vscal
 define <vscale x 1 x i1> @fcmp_oge_vf_nxv1bf16(<vscale x 1 x bfloat> %va, bfloat %b, <vscale x 1 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: fcmp_oge_vf_nxv1bf16:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    fmv.x.h a1, fa0
+; CHECK-NEXT:    fcvt.s.bf16 fa5, fa0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, ma
-; CHECK-NEXT:    vmv.v.x v9, a1
-; CHECK-NEXT:    vfwcvtbf16.f.f.v v10, v8
-; CHECK-NEXT:    vfwcvtbf16.f.f.v v8, v9
+; CHECK-NEXT:    vfwcvtbf16.f.f.v v9, v8
 ; CHECK-NEXT:    vsetvli zero, zero, e32, mf2, ta, ma
-; CHECK-NEXT:    vmfle.vv v0, v8, v10, v0.t
+; CHECK-NEXT:    vmfge.vf v0, v9, fa5, v0.t
 ; CHECK-NEXT:    ret
   %elt.head = insertelement <vscale x 1 x bfloat> poison, bfloat %b, i32 0
   %vb = shufflevector <vscale x 1 x bfloat> %elt.head, <vscale x 1 x bfloat> poison, <vscale x 1 x i32> zeroinitializer
@@ -141,13 +131,11 @@ define <vscale x 1 x i1> @fcmp_oge_vf_nxv1bf16(<vscale x 1 x bfloat> %va, bfloat
 define <vscale x 1 x i1> @fcmp_oge_vf_swap_nxv1bf16(<vscale x 1 x bfloat> %va, bfloat %b, <vscale x 1 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: fcmp_oge_vf_swap_nxv1bf16:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    fmv.x.h a1, fa0
+; CHECK-NEXT:    fcvt.s.bf16 fa5, fa0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, ma
-; CHECK-NEXT:    vmv.v.x v9, a1
-; CHECK-NEXT:    vfwcvtbf16.f.f.v v10, v8
-; CHECK-NEXT:    vfwcvtbf16.f.f.v v8, v9
+; CHECK-NEXT:    vfwcvtbf16.f.f.v v9, v8
 ; CHECK-NEXT:    vsetvli zero, zero, e32, mf2, ta, ma
-; CHECK-NEXT:    vmfle.vv v0, v10, v8, v0.t
+; CHECK-NEXT:    vmfle.vf v0, v9, fa5, v0.t
 ; CHECK-NEXT:    ret
   %elt.head = insertelement <vscale x 1 x bfloat> poison, bfloat %b, i32 0
   %vb = shufflevector <vscale x 1 x bfloat> %elt.head, <vscale x 1 x bfloat> poison, <vscale x 1 x i32> zeroinitializer
@@ -171,13 +159,11 @@ define <vscale x 1 x i1> @fcmp_olt_vv_nxv1bf16(<vscale x 1 x bfloat> %va, <vscal
 define <vscale x 1 x i1> @fcmp_olt_vf_nxv1bf16(<vscale x 1 x bfloat> %va, bfloat %b, <vscale x 1 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: fcmp_olt_vf_nxv1bf16:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    fmv.x.h a1, fa0
+; CHECK-NEXT:    fcvt.s.bf16 fa5, fa0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, ma
-; CHECK-NEXT:    vmv.v.x v9, a1
-; CHECK-NEXT:    vfwcvtbf16.f.f.v v10, v8
-; CHECK-NEXT:    vfwcvtbf16.f.f.v v8, v9
+; CHECK-NEXT:    vfwcvtbf16.f.f.v v9, v8
 ; CHECK-NEXT:    vsetvli zero, zero, e32, mf2, ta, ma
-; CHECK-NEXT:    vmflt.vv v0, v10, v8, v0.t
+; CHECK-NEXT:    vmflt.vf v0, v9, fa5, v0.t
 ; CHECK-NEXT:    ret
   %elt.head = insertelement <vscale x 1 x bfloat> poison, bfloat %b, i32 0
   %vb = shufflevector <vscale x 1 x bfloat> %elt.head, <vscale x 1 x bfloat> poison, <vscale x 1 x i32> zeroinitializer
@@ -188,13 +174,11 @@ define <vscale x 1 x i1> @fcmp_olt_vf_nxv1bf16(<vscale x 1 x bfloat> %va, bfloat
 define <vscale x 1 x i1> @fcmp_olt_vf_swap_nxv1bf16(<vscale x 1 x bfloat> %va, bfloat %b, <vscale x 1 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: fcmp_olt_vf_swap_nxv1bf16:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    fmv.x.h a1, fa0
+; CHECK-NEXT:    fcvt.s.bf16 fa5, fa0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, ma
-; CHECK-NEXT:    vmv.v.x v9, a1
-; CHECK-NEXT:    vfwcvtbf16.f.f.v v10, v8
-; CHECK-NEXT:    vfwcvtbf16.f.f.v v8, v9
+; CHECK-NEXT:    vfwcvtbf16.f.f.v v9, v8
 ; CHECK-NEXT:    vsetvli zero, zero, e32, mf2, ta, ma
-; CHECK-NEXT:    vmflt.vv v0, v8, v10, v0.t
+; CHECK-NEXT:    vmfgt.vf v0, v9, fa5, v0.t
 ; CHECK-NEXT:    ret
   %elt.head = insertelement <vscale x 1 x bfloat> poison, bfloat %b, i32 0
   %vb = shufflevector <vscale x 1 x bfloat> %elt.head, <vscale x 1 x bfloat> poison, <vscale x 1 x i32> zeroinitializer
@@ -218,13 +202,11 @@ define <vscale x 1 x i1> @fcmp_ole_vv_nxv1bf16(<vscale x 1 x bfloat> %va, <vscal
 define <vscale x 1 x i1> @fcmp_ole_vf_nxv1bf16(<vscale x 1 x bfloat> %va, bfloat %b, <vscale x 1 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: fcmp_ole_vf_nxv1bf16:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    fmv.x.h a1, fa0
+; CHECK-NEXT:    fcvt.s.bf16 fa5, fa0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, ma
-; CHECK-NEXT:    vmv.v.x v9, a1
-; CHECK-NEXT:    vfwcvtbf16.f.f.v v10, v8
-; CHECK-NEXT:    vfwcvtbf16.f.f.v v8, v9
+; CHECK-NEXT:    vfwcvtbf16.f.f.v v9, v8
 ; CHECK-NEXT:    vsetvli zero, zero, e32, mf2, ta, ma
-; CHECK-NEXT:    vmfle.vv v0, v10, v8, v0.t
+; CHECK-NEXT:    vmfle.vf v0, v9, fa5, v0.t
 ; CHECK-NEXT:    ret
   %elt.head = insertelement <vscale x 1 x bfloat> poison, bfloat %b, i32 0
   %vb = shufflevector <vscale x 1 x bfloat> %elt.head, <vscale x 1 x bfloat> poison, <vscale x 1 x i32> zeroinitializer
@@ -235,13 +217,11 @@ define <vscale x 1 x i1> @fcmp_ole_vf_nxv1bf16(<vscale x 1 x bfloat> %va, bfloat
 define <vscale x 1 x i1> @fcmp_ole_vf_swap_nxv1bf16(<vscale x 1 x bfloat> %va, bfloat %b, <vscale x 1 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: fcmp_ole_vf_swap_nxv1bf16:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    fmv.x.h a1, fa0
+; CHECK-NEXT:    fcvt.s.bf16 fa5, fa0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, ma
-; CHECK-NEXT:    vmv.v.x v9, a1
-; CHECK-NEXT:    vfwcvtbf16.f.f.v v10, v8
-; CHECK-NEXT:    vfwcvtbf16.f.f.v v8, v9
+; CHECK-NEXT:    vfwcvtbf16.f.f.v v9, v8
 ; CHECK-NEXT:    vsetvli zero, zero, e32, mf2, ta, ma
-; CHECK-NEXT:    vmfle.vv v0, v8, v10, v0.t
+; CHECK-NEXT:    vmfge.vf v0, v9, fa5, v0.t
 ; CHECK-NEXT:    ret
   %elt.head = insertelement <vscale x 1 x bfloat> poison, bfloat %b, i32 0
   %vb = shufflevector <vscale x 1 x bfloat> %elt.head, <vscale x 1 x bfloat> poison, <vscale x 1 x i32> zeroinitializer
@@ -267,14 +247,12 @@ define <vscale x 1 x i1> @fcmp_one_vv_nxv1bf16(<vscale x 1 x bfloat> %va, <vscal
 define <vscale x 1 x i1> @fcmp_one_vf_nxv1bf16(<vscale x 1 x bfloat> %va, bfloat %b, <vscale x 1 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: fcmp_one_vf_nxv1bf16:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    fmv.x.h a1, fa0
+; CHECK-NEXT:    fcvt.s.bf16 fa5, fa0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, ma
 ; CHECK-NEXT:    vfwcvtbf16.f.f.v v9, v8
-; CHECK-NEXT:    vmv.v.x v8, a1
-; CHECK-NEXT:    vfwcvtbf16.f.f.v v10, v8
 ; CHECK-NEXT:    vsetvli zero, zero, e32, mf2, ta, ma
-; CHECK-NEXT:    vmflt.vv v8, v9, v10, v0.t
-; CHECK-NEXT:    vmflt.vv v9, v10, v9, v0.t
+; CHECK-NEXT:    vmflt.vf v8, v9, fa5, v0.t
+; CHECK-NEXT:    vmfgt.vf v9, v9, fa5, v0.t
 ; CHECK-NEXT:    vmor.mm v0, v9, v8
 ; CHECK-NEXT:    ret
   %elt.head = insertelement <vscale x 1 x bfloat> poison, bfloat %b, i32 0
@@ -286,14 +264,12 @@ define <vscale x 1 x i1> @fcmp_one_vf_nxv1bf16(<vscale x 1 x bfloat> %va, bfloat
 define <vscale x 1 x i1> @fcmp_one_vf_swap_nxv1bf16(<vscale x 1 x bfloat> %va, bfloat %b, <vscale x 1 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: fcmp_one_vf_swap_nxv1bf16:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    fmv.x.h a1, fa0
+; CHECK-NEXT:    fcvt.s.bf16 fa5, fa0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, ma
 ; CHECK-NEXT:    vfwcvtbf16.f.f.v v9, v8
-; CHECK-NEXT:    vmv.v.x v8, a1
-; CHECK-NEXT:    vfwcvtbf16.f.f.v v10, v8
 ; CHECK-NEXT:    vsetvli zero, zero, e32, mf2, ta, ma
-; CHECK-NEXT:    vmflt.vv v8, v10, v9, v0.t
-; CHECK-NEXT:    vmflt.vv v9, v9, v10, v0.t
+; CHECK-NEXT:    vmfgt.vf v8, v9, fa5, v0.t
+; CHECK-NEXT:    vmflt.vf v9, v9, fa5, v0.t
 ; CHECK-NEXT:    vmor.mm v0, v9, v8
 ; CHECK-NEXT:    ret
   %elt.head = insertelement <vscale x 1 x bfloat> poison, bfloat %b, i32 0
@@ -320,16 +296,14 @@ define <vscale x 1 x i1> @fcmp_ord_vv_nxv1bf16(<vscale x 1 x bfloat> %va, <vscal
 define <vscale x 1 x i1> @fcmp_ord_vf_nxv1bf16(<vscale x 1 x bfloat> %va, bfloat %b, <vscale x 1 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: fcmp_ord_vf_nxv1bf16:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    fmv.x.h a1, fa0
+; CHECK-NEXT:    fcvt.s.bf16 fa5, fa0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, ma
 ; CHECK-NEXT:    vfwcvtbf16.f.f.v v9, v8
-; CHECK-NEXT:    vmv.v.x v8, a1
-; CHECK-NEXT:    vsetvli zero, zero, e32, mf2, ta, ma
+; CHECK-NEXT:    vsetvli a1, zero, e32, mf2, ta, ma
+; CHECK-NEXT:    vfmv.v.f v8, fa5
+; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, ta, ma
+; CHECK-NEXT:    vmfeq.vf v8, v8, fa5, v0.t
 ; CHECK-NEXT:    vmfeq.vv v9, v9, v9, v0.t
-; CHECK-NEXT:    vsetvli zero, zero, e16, mf4, ta, ma
-; CHECK-NEXT:    vfwcvtbf16.f.f.v v10, v8
-; CHECK-NEXT:    vsetvli zero, zero, e32, mf2, ta, ma
-; CHECK-NEXT:    vmfeq.vv v8, v10, v10, v0.t
 ; CHECK-NEXT:    vmand.mm v0, v9, v8
 ; CHECK-NEXT:    ret
   %elt.head = insertelement <vscale x 1 x bfloat> poison, bfloat %b, i32 0
@@ -341,16 +315,14 @@ define <vscale x 1 x i1> @fcmp_ord_vf_nxv1bf16(<vscale x 1 x bfloat> %va, bfloat
 define <vscale x 1 x i1> @fcmp_ord_vf_swap_nxv1bf16(<vscale x 1 x bfloat> %va, bfloat %b, <vscale x 1 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: fcmp_ord_vf_swap_nxv1bf16:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    fmv.x.h a1, fa0
+; CHECK-NEXT:    fcvt.s.bf16 fa5, fa0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, ma
 ; CHECK-NEXT:    vfwcvtbf16.f.f.v v9, v8
-; CHECK-NEXT:    vmv.v.x v8, a1
-; CHECK-NEXT:    vsetvli zero, zero, e32, mf2, ta, ma
+; CHECK-NEXT:    vsetvli a1, zero, e32, mf2, ta, ma
+; CHECK-NEXT:    vfmv.v.f v8, fa5
+; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, ta, ma
+; CHECK-NEXT:    vmfeq.vf v8, v8, fa5, v0.t
 ; CHECK-NEXT:    vmfeq.vv v9, v9, v9, v0.t
-; CHECK-NEXT:    vsetvli zero, zero, e16, mf4, ta, ma
-; CHECK-NEXT:    vfwcvtbf16.f.f.v v10, v8
-; CHECK-NEXT:    vsetvli zero, zero, e32, mf2, ta, ma
-; CHECK-NEXT:    vmfeq.vv v8, v10, v10, v0.t
 ; CHECK-NEXT:    vmand.mm v0, v8, v9
 ; CHECK-NEXT:    ret
   %elt.head = insertelement <vscale x 1 x bfloat> poison, bfloat %b, i32 0
@@ -377,14 +349,12 @@ define <vscale x 1 x i1> @fcmp_ueq_vv_nxv1bf16(<vscale x 1 x bfloat> %va, <vscal
 define <vscale x 1 x i1> @fcmp_ueq_vf_nxv1bf16(<vscale x 1 x bfloat> %va, bfloat %b, <vscale x 1 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: fcmp_ueq_vf_nxv1bf16:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    fmv.x.h a1, fa0
+; CHECK-NEXT:    fcvt.s.bf16 fa5, fa0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, ma
 ; CHECK-NEXT:    vfwcvtbf16.f.f.v v9, v8
-; CHECK-NEXT:    vmv.v.x v8, a1
-; CHECK-NEXT:    vfwcvtbf16.f.f.v v10, v8
 ; CHECK-NEXT:    vsetvli zero, zero, e32, mf2, ta, ma
-; CHECK-NEXT:    vmflt.vv v8, v9, v10, v0.t
-; CHECK-NEXT:    vmflt.vv v9, v10, v9, v0.t
+; CHECK-NEXT:    vmflt.vf v8, v9, fa5, v0.t
+; CHECK-NEXT:    vmfgt.vf v9, v9, fa5, v0.t
 ; CHECK-NEXT:    vmnor.mm v0, v9, v8
 ; CHECK-NEXT:    ret
   %elt.head = insertelement <vscale x 1 x bfloat> poison, bfloat %b, i32 0
@@ -396,14 +366,12 @@ define <vscale x 1 x i1> @fcmp_ueq_vf_nxv1bf16(<vscale x 1 x bfloat> %va, bfloat
 define <vscale x 1 x i1> @fcmp_ueq_vf_swap_nxv1bf16(<vscale x 1 x bfloat> %va, bfloat %b, <vscale x 1 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: fcmp_ueq_vf_swap_nxv1bf16:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    fmv.x.h a1, fa0
+; CHECK-NEXT:    fcvt.s.bf16 fa5, fa0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, ma
 ; CHECK-NEXT:    vfwcvtbf16.f.f.v v9, v8
-; CHECK-NEXT:    vmv.v.x v8, a1
-; CHECK-NEXT:    vfwcvtbf16.f.f.v v10, v8
 ; CHECK-NEXT:    vsetvli zero, zero, e32, mf2, ta, ma
-; CHECK-NEXT:    vmflt.vv v8, v10, v9, v0.t
-; CHECK-NEXT:    vmflt.vv v9, v9, v10, v0.t
+; CHECK-NEXT:    vmfgt.vf v8, v9, fa5, v0.t
+; CHECK-NEXT:    vmflt.vf v9, v9, fa5, v0.t
 ; CHECK-NEXT:    vmnor.mm v0, v9, v8
 ; CHECK-NEXT:    ret
   %elt.head = insertelement <vscale x 1 x bfloat> poison, bfloat %b, i32 0
@@ -429,13 +397,11 @@ define <vscale x 1 x i1> @fcmp_ugt_vv_nxv1bf16(<vscale x 1 x bfloat> %va, <vscal
 define <vscale x 1 x i1> @fcmp_ugt_vf_nxv1bf16(<vscale x 1 x bfloat> %va, bfloat %b, <vscale x 1 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: fcmp_ugt_vf_nxv1bf16:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    fmv.x.h a1, fa0
+; CHECK-NEXT:    fcvt.s.bf16 fa5, fa0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, ma
-; CHECK-NEXT:    vmv.v.x v9, a1
-; CHECK-NEXT:    vfwcvtbf16.f.f.v v10, v8
-; CHECK-NEXT:    vfwcvtbf16.f.f.v v8, v9
+; CHECK-NEXT:    vfwcvtbf16.f.f.v v9, v8
 ; CHECK-NEXT:    vsetvli zero, zero, e32, mf2, ta, ma
-; CHECK-NEXT:    vmfle.vv v8, v10, v8, v0.t
+; CHECK-NEXT:    vmfle.vf v8, v9, fa5, v0.t
 ; CHECK-NEXT:    vmnot.m v0, v8
 ; CHECK-NEXT:    ret
   %elt.head = insertelement <vscale x 1 x bfloat> poison, bfloat %b, i32 0
@@ -447,13 +413,11 @@ define <vscale x 1 x i1> @fcmp_ugt_vf_nxv1bf16(<vscale x 1 x bfloat> %va, bfloat
 define <vscale x 1 x i1> @fcmp_ugt_vf_swap_nxv1bf16(<vscale x 1 x bfloat> %va, bfloat %b, <vscale x 1 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: fcmp_ugt_vf_swap_nxv1bf16:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    fmv.x.h a1, fa0
+; CHECK-NEXT:    fcvt.s.bf16 fa5, fa0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, ma
-; CHECK-NEXT:    vmv.v.x v9, a1
-; CHECK-NEXT:    vfwcvtbf16.f.f.v v10, v8
-; CHECK-NEXT:    vfwcvtbf16.f.f.v v8, v9
+; CHECK-NEXT:    vfwcvtbf16.f.f.v v9, v8
 ; CHECK-NEXT:    vsetvli zero, zero, e32, mf2, ta, ma
-; CHECK-NEXT:    vmfle.vv v8, v8, v10, v0.t
+; CHECK-NEXT:    vmfge.vf v8, v9, fa5, v0.t
 ; CHECK-NEXT:    vmnot.m v0, v8
 ; CHECK-NEXT:    ret
   %elt.head = insertelement <vscale x 1 x bfloat> poison, bfloat %b, i32 0
@@ -479,13 +443,11 @@ define <vscale x 1 x i1> @fcmp_uge_vv_nxv1bf16(<vscale x 1 x bfloat> %va, <vscal
 define <vscale x 1 x i1> @fcmp_uge_vf_nxv1bf16(<vscale x 1 x bfloat> %va, bfloat %b, <vscale x 1 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: fcmp_uge_vf_nxv1bf16:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    fmv.x.h a1, fa0
+; CHECK-NEXT:    fcvt.s.bf16 fa5, fa0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, ma
-; CHECK-NEXT:    vmv.v.x v9, a1
-; CHECK-NEXT:    vfwcvtbf16.f.f.v v10, v8
-; CHECK-NEXT:    vfwcvtbf16.f.f.v v8, v9
+; CHECK-NEXT:    vfwcvtbf16.f.f.v v9, v8
 ; CHECK-NEXT:    vsetvli zero, zero, e32, mf2, ta, ma
-; CHECK-NEXT:    vmflt.vv v8, v10, v8, v0.t
+; CHECK-NEXT:    vmflt.vf v8, v9, fa5, v0.t
 ; CHECK-NEXT:    vmnot.m v0, v8
 ; CHECK-NEXT:    ret
   %elt.head = insertelement <vscale x 1 x bfloat> poison, bfloat %b, i32 0
@@ -497,13 +459,11 @@ define <vscale x 1 x i1> @fcmp_uge_vf_nxv1bf16(<vscale x 1 x bfloat> %va, bfloat
 define <vscale x 1 x i1> @fcmp_uge_vf_swap_nxv1bf16(<vscale x 1 x bfloat> %va, bfloat %b, <vscale x 1 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: fcmp_uge_vf_swap_nxv1bf16:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    fmv.x.h a1, fa0
+; CHECK-NEXT:    fcvt.s.bf16 fa5, fa0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, ma
-; CHECK-NEXT:    vmv.v.x v9, a1
-; CHECK-NEXT:    vfwcvtbf16.f.f.v v10, v8
-; CHECK-NEXT:    vfwcvtbf16.f.f.v v8, v9
+; CHECK-NEXT:    vfwcvtbf16.f.f.v v9, v8
 ; CHECK-NEXT:    vsetvli zero, zero, e32, mf2, ta, ma
-; CHECK-NEXT:    vmflt.vv v8, v8, v10, v0.t
+; CHECK-NEXT:    vmfgt.vf v8, v9, fa5, v0.t
 ; CHECK-NEXT:    vmnot.m v0, v8
 ; CHECK-NEXT:    ret
   %elt.head = insertelement <vscale x 1 x bfloat> poison, bfloat %b, i32 0
@@ -529,13 +489,11 @@ define <vscale x 1 x i1> @fcmp_ult_vv_nxv1bf16(<vscale x 1 x bfloat> %va, <vscal
 define <vscale x 1 x i1> @fcmp_ult_vf_nxv1bf16(<vscale x 1 x bfloat> %va, bfloat %b, <vscale x 1 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: fcmp_ult_vf_nxv1bf16:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    fmv.x.h a1, fa0
+; CHECK-NEXT:    fcvt.s.bf16 fa5, fa0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, ma
-; CHECK-NEXT:    vmv.v.x v9, a1
-; CHECK-NEXT:    vfwcvtbf16.f.f.v v10, v8
-; CHECK-NEXT:    vfwcvtbf16.f.f.v v8, v9
+; CHECK-NEXT:    vfwcvtbf16.f.f.v v9, v8
 ; CHECK-NEXT:    vsetvli zero, zero, e32, mf2, ta, ma
-; CHECK-NEXT:    vmfle.vv v8, v8, v10, v0.t
+; CHECK-NEXT:    vmfge.vf v8, v9, fa5, v0.t
 ; CHECK-NEXT:    vmnot.m v0, v8
 ; CHECK-NEXT:    ret
   %elt.head = insertelement <vscale x 1 x bfloat> poison, bfloat %b, i32 0
@@ -547,13 +505,11 @@ define <vscale x 1 x i1> @fcmp_ult_vf_nxv1bf16(<vscale x 1 x bfloat> %va, bfloat
 define <vscale x 1 x i1> @fcmp_ult_vf_swap_nxv1bf16(<vscale x 1 x bfloat> %va, bfloat %b, <vscale x 1 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: fcmp_ult_vf_swap_nxv1bf16:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    fmv.x.h a1, fa0
+; CHECK-NEXT:    fcvt.s.bf16 fa5, fa0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, ma
-; CHECK-NEXT:    vmv.v.x v9, a1
-; CHECK-NEXT:    vfwcvtbf16.f.f.v v10, v8
-; CHECK-NEXT:    vfwcvtbf16.f.f.v v8, v9
+; CHECK-NEXT:    vfwcvtbf16.f.f.v v9, v8
 ; CHECK-NEXT:    vsetvli zero, zero, e32, mf2, ta, ma
-; CHECK-NEXT:    vmfle.vv v8, v10, v8, v0.t
+; CHECK-NEXT:    vmfle.vf v8, v9, fa5, v0.t
 ; CHECK-NEXT:    vmnot.m v0, v8
 ; CHECK-NEXT:    ret
   %elt.head = insertelement <vscale x 1 x bfloat> poison, bfloat %b, i32 0
@@ -579,13 +535,11 @@ define <vscale x 1 x i1> @fcmp_ule_vv_nxv1bf16(<vscale x 1 x bfloat> %va, <vscal
 define <vscale x 1 x i1> @fcmp_ule_vf_nxv1bf16(<vscale x 1 x bfloat> %va, bfloat %b, <vscale x 1 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: fcmp_ule_vf_nxv1bf16:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    fmv.x.h a1, fa0
+; CHECK-NEXT:    fcvt.s.bf16 fa5, fa0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, ma
-; CHECK-NEXT:    vmv.v.x v9, a1
-; CHECK-NEXT:    vfwcvtbf16.f.f.v v10, v8
-; CHECK-NEXT:    vfwcvtbf16.f.f.v v8, v9
+; CHECK-NEXT:    vfwcvtbf16.f.f.v v9, v8
 ; CHECK-NEXT:    vsetvli zero, zero, e32, mf2, ta, ma
-; CHECK-NEXT:    vmflt.vv v8, v8, v10, v0.t
+; CHECK-NEXT:    vmfgt.vf v8, v9, fa5, v0.t
 ; CHECK-NEXT:    vmnot.m v0, v8
 ; CHECK-NEXT:    ret
   %elt.head = insertelement <vscale x 1 x bfloat> poison, bfloat %b, i32 0
@@ -597,13 +551,11 @@ define <vscale x 1 x i1> @fcmp_ule_vf_nxv1bf16(<vscale x 1 x bfloat> %va, bfloat
 define <vscale x 1 x i1> @fcmp_ule_vf_swap_nxv1bf16(<vscale x 1 x bfloat> %va, bfloat %b, <vscale x 1 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: fcmp_ule_vf_swap_nxv1bf16:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    fmv.x.h a1, fa0
+; CHECK-NEXT:    fcvt.s.bf16 fa5, fa0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, ma
-; CHECK-NEXT:    vmv.v.x v9, a1
-; CHECK-NEXT:    vfwcvtbf16.f.f.v v10, v8
-; CHECK-NEXT:    vfwcvtbf16.f.f.v v8, v9
+; CHECK-NEXT:    vfwcvtbf16.f.f.v v9, v8
 ; CHECK-NEXT:    vsetvli zero, zero, e32, mf2, ta, ma
-; CHECK-NEXT:    vmflt.vv v8, v10, v8, v0.t
+; CHECK-NEXT:    vmflt.vf v8, v9, fa5, v0.t
 ; CHECK-NEXT:    vmnot.m v0, v8
 ; CHECK-NEXT:    ret
   %elt.head = insertelement <vscale x 1 x bfloat> poison, bfloat %b, i32 0
@@ -628,13 +580,11 @@ define <vscale x 1 x i1> @fcmp_une_vv_nxv1bf16(<vscale x 1 x bfloat> %va, <vscal
 define <vscale x 1 x i1> @fcmp_une_vf_nxv1bf16(<vscale x 1 x bfloat> %va, bfloat %b, <vscale x 1 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: fcmp_une_vf_nxv1bf16:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    fmv.x.h a1, fa0
+; CHECK-NEXT:    fcvt.s.bf16 fa5, fa0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, ma
-; CHECK-NEXT:    vmv.v.x v9, a1
-; CHECK-NEXT:    vfwcvtbf16.f.f.v v10, v8
-; CHECK-NEXT:    vfwcvtbf16.f.f.v v8, v9
+; CHECK-NEXT:    vfwcvtbf16.f.f.v v9, v8
 ; CHECK-NEXT:    vsetvli zero, zero, e32, mf2, ta, ma
-; CHECK-NEXT:    vmfne.vv v0, v10, v8, v0.t
+; CHECK-NEXT:    vmfne.vf v0, v9, fa5, v0.t
 ; CHECK-NEXT:    ret
   %elt.head = insertelement <vscale x 1 x bfloat> poison, bfloat %b, i32 0
   %vb = shufflevector <vscale x 1 x bfloat> %elt.head, <vscale x 1 x bfloat> poison, <vscale x 1 x i32> zeroinitializer
@@ -645,13 +595,11 @@ define <vscale x 1 x i1> @fcmp_une_vf_nxv1bf16(<vscale x 1 x bfloat> %va, bfloat
 define <vscale x 1 x i1> @fcmp_une_vf_swap_nxv1bf16(<vscale x 1 x bfloat> %va, bfloat %b, <vscale x 1 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: fcmp_une_vf_swap_nxv1bf16:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    fmv.x.h a1, fa0
+; CHECK-NEXT:    fcvt.s.bf16 fa5, fa0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, ma
-; CHECK-NEXT:    vmv.v.x v9, a1
-; CHECK-NEXT:    vfwcvtbf16.f.f.v v10, v8
-; CHECK-NEXT:    vfwcvtbf16.f.f.v v8, v9
+; CHECK-NEXT:    vfwcvtbf16.f.f.v v9, v8
 ; CHECK-NEXT:    vsetvli zero, zero, e32, mf2, ta, ma
-; CHECK-NEXT:    vmfne.vv v0, v8, v10, v0.t
+; CHECK-NEXT:    vmfne.vf v0, v9, fa5, v0.t
 ; CHECK-NEXT:    ret
   %elt.head = insertelement <vscale x 1 x bfloat> poison, bfloat %b, i32 0
   %vb = shufflevector <vscale x 1 x bfloat> %elt.head, <vscale x 1 x bfloat> poison, <vscale x 1 x i32> zeroinitializer
@@ -677,16 +625,14 @@ define <vscale x 1 x i1> @fcmp_uno_vv_nxv1bf16(<vscale x 1 x bfloat> %va, <vscal
 define <vscale x 1 x i1> @fcmp_uno_vf_nxv1bf16(<vscale x 1 x bfloat> %va, bfloat %b, <vscale x 1 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: fcmp_uno_vf_nxv1bf16:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    fmv.x.h a1, fa0
+; CHECK-NEXT:    fcvt.s.bf16 fa5, fa0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, ma
 ; CHECK-NEXT:    vfwcvtbf16.f.f.v v9, v8
-; CHECK-NEXT:    vmv.v.x v8, a1
-; CHECK-NEXT:    vsetvli zero, zero, e32, mf2, ta, ma
+; CHECK-NEXT:    vsetvli a1, zero, e32, mf2, ta, ma
+; CHECK-NEXT:    vfmv.v.f v8, fa5
+; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, ta, ma
+; CHECK-NEXT:    vmfne.vf v8, v8, fa5, v0.t
 ; CHECK-NEXT:    vmfne.vv v9, v9, v9, v0.t
-; CHECK-NEXT:    vsetvli zero, zero, e16, mf4, ta, ma
-; CHECK-NEXT:    vfwcvtbf16.f.f.v v10, v8
-; CHECK-NEXT:    vsetvli zero, zero, e32, mf2, ta, ma
-; CHECK-NEXT:    vmfne.vv v8, v10, v10, v0.t
 ; CHECK-NEXT:    vmor.mm v0, v9, v8
 ; CHECK-NEXT:    ret
   %elt.head = insertelement <vscale x 1 x bfloat> poison, bfloat %b, i32 0
@@ -698,16 +644,14 @@ define <vscale x 1 x i1> @fcmp_uno_vf_nxv1bf16(<vscale x 1 x bfloat> %va, bfloat
 define <vscale x 1 x i1> @fcmp_uno_vf_swap_nxv1bf16(<vscale x 1 x bfloat> %va, bfloat %b, <vscale x 1 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: fcmp_uno_vf_swap_nxv1bf16:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    fmv.x.h a1, fa0
+; CHECK-NEXT:    fcvt.s.bf16 fa5, fa0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, ma
 ; CHECK-NEXT:    vfwcvtbf16.f.f.v v9, v8
-; CHECK-NEXT:    vmv.v.x v8, a1
-; CHECK-NEXT:    vsetvli zero, zero, e32, mf2, ta, ma
+; CHECK-NEXT:    vsetvli a1, zero, e32, mf2, ta, ma
+; CHECK-NEXT:    vfmv.v.f v8, fa5
+; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, ta, ma
+; CHECK-NEXT:    vmfne.vf v8, v8, fa5, v0.t
 ; CHECK-NEXT:    vmfne.vv v9, v9, v9, v0.t
-; CHECK-NEXT:    vsetvli zero, zero, e16, mf4, ta, ma
-; CHECK-NEXT:    vfwcvtbf16.f.f.v v10, v8
-; CHECK-NEXT:    vsetvli zero, zero, e32, mf2, ta, ma
-; CHECK-NEXT:    vmfne.vv v8, v10, v10, v0.t
 ; CHECK-NEXT:    vmor.mm v0, v8, v9
 ; CHECK-NEXT:    ret
   %elt.head = insertelement <vscale x 1 x bfloat> poison, bfloat %b, i32 0
@@ -751,13 +695,11 @@ define <vscale x 8 x i1> @fcmp_oeq_vv_nxv8bf16(<vscale x 8 x bfloat> %va, <vscal
 define <vscale x 8 x i1> @fcmp_oeq_vf_nxv8bf16(<vscale x 8 x bfloat> %va, bfloat %b, <vscale x 8 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: fcmp_oeq_vf_nxv8bf16:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    fmv.x.h a1, fa0
+; CHECK-NEXT:    fcvt.s.bf16 fa5, fa0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m2, ta, ma
-; CHECK-NEXT:    vmv.v.x v10, a1
 ; CHECK-NEXT:    vfwcvtbf16.f.f.v v12, v8
-; CHECK-NEXT:    vfwcvtbf16.f.f.v v16, v10
 ; CHECK-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
-; CHECK-NEXT:    vmfeq.vv v8, v12, v16, v0.t
+; CHECK-NEXT:    vmfeq.vf v8, v12, fa5, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v8
 ; CHECK-NEXT:    ret
   %elt.head = insertelement <vscale x 8 x bfloat> poison, bfloat %b, i32 0
@@ -769,13 +711,11 @@ define <vscale x 8 x i1> @fcmp_oeq_vf_nxv8bf16(<vscale x 8 x bfloat> %va, bfloat
 define <vscale x 8 x i1> @fcmp_oeq_vf_swap_nxv8bf16(<vscale x 8 x bfloat> %va, bfloat %b, <vscale x 8 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: fcmp_oeq_vf_swap_nxv8bf16:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    fmv.x.h a1, fa0
+; CHECK-NEXT:    fcvt.s.bf16 fa5, fa0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m2, ta, ma
-; CHECK-NEXT:    vmv.v.x v10, a1
 ; CHECK-NEXT:    vfwcvtbf16.f.f.v v12, v8
-; CHECK-NEXT:    vfwcvtbf16.f.f.v v16, v10
 ; CHECK-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
-; CHECK-NEXT:    vmfeq.vv v8, v16, v12, v0.t
+; CHECK-NEXT:    vmfeq.vf v8, v12, fa5, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v8
 ; CHECK-NEXT:    ret
   %elt.head = insertelement <vscale x 8 x bfloat> poison, bfloat %b, i32 0
@@ -801,13 +741,11 @@ define <vscale x 8 x i1> @fcmp_ogt_vv_nxv8bf16(<vscale x 8 x bfloat> %va, <vscal
 define <vscale x 8 x i1> @fcmp_ogt_vf_nxv8bf16(<vscale x 8 x bfloat> %va, bfloat %b, <vscale x 8 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: fcmp_ogt_vf_nxv8bf16:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    fmv.x.h a1, fa0
+; CHECK-NEXT:    fcvt.s.bf16 fa5, fa0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m2, ta, ma
-; CHECK-NEXT:    vmv.v.x v10, a1
 ; CHECK-NEXT:    vfwcvtbf16.f.f.v v12, v8
-; CHECK-NEXT:    vfwcvtbf16.f.f.v v16, v10
 ; CHECK-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
-; CHECK-NEXT:    vmflt.vv v8, v16, v12, v0.t
+; CHECK-NEXT:    vmfgt.vf v8, v12, fa5, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v8
 ; CHECK-NEXT:    ret
   %elt.head = insertelement <vscale x 8 x bfloat> poison, bfloat %b, i32 0
@@ -819,13 +757,11 @@ define <vscale x 8 x i1> @fcmp_ogt_vf_nxv8bf16(<vscale x 8 x bfloat> %va, bfloat
 define <vscale x 8 x i1> @fcmp_ogt_vf_swap_nxv8bf16(<vscale x 8 x bfloat> %va, bfloat %b, <vscale x 8 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: fcmp_ogt_vf_swap_nxv8bf16:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    fmv.x.h a1, fa0
+; CHECK-NEXT:    fcvt.s.bf16 fa5, fa0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m2, ta, ma
-; CHECK-NEXT:    vmv.v.x v10, a1
 ; CHECK-NEXT:    vfwcvtbf16.f.f.v v12, v8
-; CHECK-NEXT:    vfwcvtbf16.f.f.v v16, v10
 ; CHECK-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
-; CHECK-NEXT:    vmflt.vv v8, v12, v16, v0.t
+; CHECK-NEXT:    vmflt.vf v8, v12, fa5, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v8
 ; CHECK-NEXT:    ret
   %elt.head = insertelement <vscale x 8 x bfloat> poison, bfloat %b, i32 0
@@ -851,13 +787,11 @@ define <vscale x 8 x i1> @fcmp_oge_vv_nxv8bf16(<vscale x 8 x bfloat> %va, <vscal
 define <vscale x 8 x i1> @fcmp_oge_vf_nxv8bf16(<vscale x 8 x bfloat> %va, bfloat %b, <vscale x 8 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: fcmp_oge_vf_nxv8bf16:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    fmv.x.h a1, fa0
+; CHECK-NEXT:    fcvt.s.bf16 fa5, fa0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m2, ta, ma
-; CHECK-NEXT:    vmv.v.x v10, a1
 ; CHECK-NEXT:    vfwcvtbf16.f.f.v v12, v8
-; CHECK-NEXT:    vfwcvtbf16.f.f.v v16, v10
 ; CHECK-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
-; CHECK-NEXT:    vmfle.vv v8, v16, v12, v0.t
+; CHECK-NEXT:    vmfge.vf v8, v12, fa5, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v8
 ; CHECK-NEXT:    ret
   %elt.head = insertelement <vscale x 8 x bfloat> poison, bfloat %b, i32 0
@@ -869,13 +803,11 @@ define <vscale x 8 x i1> @fcmp_oge_vf_nxv8bf16(<vscale x 8 x bfloat> %va, bfloat
 define <vscale x 8 x i1> @fcmp_oge_vf_swap_nxv8bf16(<vscale x 8 x bfloat> %va, bfloat %b, <vscale x 8 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: fcmp_oge_vf_swap_nxv8bf16:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    fmv.x.h a1, fa0
+; CHECK-NEXT:    fcvt.s.bf16 fa5, fa0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m2, ta, ma
-; CHECK-NEXT:    vmv.v.x v10, a1
 ; CHECK-NEXT:    vfwcvtbf16.f.f.v v12, v8
-; CHECK-NEXT:    vfwcvtbf16.f.f.v v16, v10
 ; CHECK-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
-; CHECK-NEXT:    vmfle.vv v8, v12, v16, v0.t
+; CHECK-NEXT:    vmfle.vf v8, v12, fa5, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v8
 ; CHECK-NEXT:    ret
   %elt.head = insertelement <vscale x 8 x bfloat> poison, bfloat %b, i32 0
@@ -901,13 +833,11 @@ define <vscale x 8 x i1> @fcmp_olt_vv_nxv8bf16(<vscale x 8 x bfloat> %va, <vscal
 define <vscale x 8 x i1> @fcmp_olt_vf_nxv8bf16(<vscale x 8 x bfloat> %va, bfloat %b, <vscale x 8 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: fcmp_olt_vf_nxv8bf16:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    fmv.x.h a1, fa0
+; CHECK-NEXT:    fcvt.s.bf16 fa5, fa0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m2, ta, ma
-; CHECK-NEXT:    vmv.v.x v10, a1
 ; CHECK-NEXT:    vfwcvtbf16.f.f.v v12, v8
-; CHECK-NEXT:    vfwcvtbf16.f.f.v v16, v10
 ; CHECK-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
-; CHECK-NEXT:    vmflt.vv v8, v12, v16, v0.t
+; CHECK-NEXT:    vmflt.vf v8, v12, fa5, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v8
 ; CHECK-NEXT:    ret
   %elt.head = insertelement <vscale x 8 x bfloat> poison, bfloat %b, i32 0
@@ -919,13 +849,11 @@ define <vscale x 8 x i1> @fcmp_olt_vf_nxv8bf16(<vscale x 8 x bfloat> %va, bfloat
 define <vscale x 8 x i1> @fcmp_olt_vf_swap_nxv8bf16(<vscale x 8 x bfloat> %va, bfloat %b, <vscale x 8 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: fcmp_olt_vf_swap_nxv8bf16:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    fmv.x.h a1, fa0
+; CHECK-NEXT:    fcvt.s.bf16 fa5, fa0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m2, ta, ma
-; CHECK-NEXT:    vmv.v.x v10, a1
 ; CHECK-NEXT:    vfwcvtbf16.f.f.v v12, v8
-; CHECK-NEXT:    vfwcvtbf16.f.f.v v16, v10
 ; CHECK-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
-; CHECK-NEXT:    vmflt.vv v8, v16, v12, v0.t
+; CHECK-NEXT:    vmfgt.vf v8, v12, fa5, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v8
 ; CHECK-NEXT:    ret
   %elt.head = insertelement <vscale x 8 x bfloat> poison, bfloat %b, i32 0
@@ -951,13 +879,11 @@ define <vscale x 8 x i1> @fcmp_ole_vv_nxv8bf16(<vscale x 8 x bfloat> %va, <vscal
 define <vscale x 8 x i1> @fcmp_ole_vf_nxv8bf16(<vscale x 8 x bfloat> %va, bfloat %b, <vscale x 8 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: fcmp_ole_vf_nxv8bf16:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    fmv.x.h a1, fa0
+; CHECK-NEXT:    fcvt.s.bf16 fa5, fa0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m2, ta, ma
-; CHECK-NEXT:    vmv.v.x v10, a1
 ; CHECK-NEXT:    vfwcvtbf16.f.f.v v12, v8
-; CHECK-NEXT:    vfwcvtbf16.f.f.v v16, v10
 ; CHECK-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
-; CHECK-NEXT:    vmfle.vv v8, v12, v16, v0.t
+; CHECK-NEXT:    vmfle.vf v8, v12, fa5, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v8
 ; CHECK-NEXT:    ret
   %elt.head = insertelement <vscale x 8 x bfloat> poison, bfloat %b, i32 0
@@ -969,13 +895,11 @@ define <vscale x 8 x i1> @fcmp_ole_vf_nxv8bf16(<vscale x 8 x bfloat> %va, bfloat
 define <vscale x 8 x i1> @fcmp_ole_vf_swap_nxv8bf16(<vscale x 8 x bfloat> %va, bfloat %b, <vscale x 8 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: fcmp_ole_vf_swap_nxv8bf16:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    fmv.x.h a1, fa0
+; CHECK-NEXT:    fcvt.s.bf16 fa5, fa0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m2, ta, ma
-; CHECK-NEXT:    vmv.v.x v10, a1
 ; CHECK-NEXT:    vfwcvtbf16.f.f.v v12, v8
-; CHECK-NEXT:    vfwcvtbf16.f.f.v v16, v10
 ; CHECK-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
-; CHECK-NEXT:    vmfle.vv v8, v16, v12, v0.t
+; CHECK-NEXT:    vmfge.vf v8, v12, fa5, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v8
 ; CHECK-NEXT:    ret
   %elt.head = insertelement <vscale x 8 x bfloat> poison, bfloat %b, i32 0
@@ -1002,14 +926,12 @@ define <vscale x 8 x i1> @fcmp_one_vv_nxv8bf16(<vscale x 8 x bfloat> %va, <vscal
 define <vscale x 8 x i1> @fcmp_one_vf_nxv8bf16(<vscale x 8 x bfloat> %va, bfloat %b, <vscale x 8 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: fcmp_one_vf_nxv8bf16:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    fmv.x.h a1, fa0
+; CHECK-NEXT:    fcvt.s.bf16 fa5, fa0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m2, ta, ma
 ; CHECK-NEXT:    vfwcvtbf16.f.f.v v12, v8
-; CHECK-NEXT:    vmv.v.x v8, a1
-; CHECK-NEXT:    vfwcvtbf16.f.f.v v16, v8
 ; CHECK-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
-; CHECK-NEXT:    vmflt.vv v8, v12, v16, v0.t
-; CHECK-NEXT:    vmflt.vv v9, v16, v12, v0.t
+; CHECK-NEXT:    vmflt.vf v8, v12, fa5, v0.t
+; CHECK-NEXT:    vmfgt.vf v9, v12, fa5, v0.t
 ; CHECK-NEXT:    vmor.mm v0, v9, v8
 ; CHECK-NEXT:    ret
   %elt.head = insertelement <vscale x 8 x bfloat> poison, bfloat %b, i32 0
@@ -1021,14 +943,12 @@ define <vscale x 8 x i1> @fcmp_one_vf_nxv8bf16(<vscale x 8 x bfloat> %va, bfloat
 define <vscale x 8 x i1> @fcmp_one_vf_swap_nxv8bf16(<vscale x 8 x bfloat> %va, bfloat %b, <vscale x 8 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: fcmp_one_vf_swap_nxv8bf16:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    fmv.x.h a1, fa0
+; CHECK-NEXT:    fcvt.s.bf16 fa5, fa0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m2, ta, ma
 ; CHECK-NEXT:    vfwcvtbf16.f.f.v v12, v8
-; CHECK-NEXT:    vmv.v.x v8, a1
-; CHECK-NEXT:    vfwcvtbf16.f.f.v v16, v8
 ; CHECK-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
-; CHECK-NEXT:    vmflt.vv v8, v16, v12, v0.t
-; CHECK-NEXT:    vmflt.vv v9, v12, v16, v0.t
+; CHECK-NEXT:    vmfgt.vf v8, v12, fa5, v0.t
+; CHECK-NEXT:    vmflt.vf v9, v12, fa5, v0.t
 ; CHECK-NEXT:    vmor.mm v0, v9, v8
 ; CHECK-NEXT:    ret
   %elt.head = insertelement <vscale x 8 x bfloat> poison, bfloat %b, i32 0
@@ -1057,17 +977,16 @@ define <vscale x 8 x i1> @fcmp_ord_vv_nxv8bf16(<vscale x 8 x bfloat> %va, <vscal
 define <vscale x 8 x i1> @fcmp_ord_vf_nxv8bf16(<vscale x 8 x bfloat> %va, bfloat %b, <vscale x 8 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: fcmp_ord_vf_nxv8bf16:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    fmv.x.h a1, fa0
-; CHECK-NEXT:    vsetvli zero, a0, e16, m2, ta, ma
-; CHECK-NEXT:    vfwcvtbf16.f.f.v v12, v8
-; CHECK-NEXT:    vmv.v.x v8, a1
-; CHECK-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
-; CHECK-NEXT:    vmfeq.vv v10, v12, v12, v0.t
+; CHECK-NEXT:    fcvt.s.bf16 fa5, fa0
+; CHECK-NEXT:    vsetvli a1, zero, e32, m4, ta, ma
+; CHECK-NEXT:    vfmv.v.f v12, fa5
+; CHECK-NEXT:    vsetvli zero, a0, e32, m4, ta, ma
+; CHECK-NEXT:    vmfeq.vf v10, v12, fa5, v0.t
 ; CHECK-NEXT:    vsetvli zero, zero, e16, m2, ta, ma
 ; CHECK-NEXT:    vfwcvtbf16.f.f.v v12, v8
 ; CHECK-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
 ; CHECK-NEXT:    vmfeq.vv v8, v12, v12, v0.t
-; CHECK-NEXT:    vmand.mm v0, v10, v8
+; CHECK-NEXT:    vmand.mm v0, v8, v10
 ; CHECK-NEXT:    ret
   %elt.head = insertelement <vscale x 8 x bfloat> poison, bfloat %b, i32 0
   %vb = shufflevector <vscale x 8 x bfloat> %elt.head, <vscale x 8 x bfloat> poison, <vscale x 8 x i32> zeroinitializer
@@ -1078,17 +997,16 @@ define <vscale x 8 x i1> @fcmp_ord_vf_nxv8bf16(<vscale x 8 x bfloat> %va, bfloat
 define <vscale x 8 x i1> @fcmp_ord_vf_swap_nxv8bf16(<vscale x 8 x bfloat> %va, bfloat %b, <vscale x 8 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: fcmp_ord_vf_swap_nxv8bf16:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    fmv.x.h a1, fa0
-; CHECK-NEXT:    vsetvli zero, a0, e16, m2, ta, ma
-; CHECK-NEXT:    vfwcvtbf16.f.f.v v12, v8
-; CHECK-NEXT:    vmv.v.x v8, a1
-; CHECK-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
-; CHECK-NEXT:    vmfeq.vv v10, v12, v12, v0.t
+; CHECK-NEXT:    fcvt.s.bf16 fa5, fa0
+; CHECK-NEXT:    vsetvli a1, zero, e32, m4, ta, ma
+; CHECK-NEXT:    vfmv.v.f v12, fa5
+; CHECK-NEXT:    vsetvli zero, a0, e32, m4, ta, ma
+; CHECK-NEXT:    vmfeq.vf v10, v12, fa5, v0.t
 ; CHECK-NEXT:    vsetvli zero, zero, e16, m2, ta, ma
 ; CHECK-NEXT:    vfwcvtbf16.f.f.v v12, v8
 ; CHECK-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
 ; CHECK-NEXT:    vmfeq.vv v8, v12, v12, v0.t
-; CHECK-NEXT:    vmand.mm v0, v8, v10
+; CHECK-NEXT:    vmand.mm v0, v10, v8
 ; CHECK-NEXT:    ret
   %elt.head = insertelement <vscale x 8 x bfloat> poison, bfloat %b, i32 0
   %vb = shufflevector <vscale x 8 x bfloat> %elt.head, <vscale x 8 x bfloat> poison, <vscale x 8 x i32> zeroinitializer
@@ -1114,14 +1032,12 @@ define <vscale x 8 x i1> @fcmp_ueq_vv_nxv8bf16(<vscale x 8 x bfloat> %va, <vscal
 define <vscale x 8 x i1> @fcmp_ueq_vf_nxv8bf16(<vscale x 8 x bfloat> %va, bfloat %b, <vscale x 8 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: fcmp_ueq_vf_nxv8bf16:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    fmv.x.h a1, fa0
+; CHECK-NEXT:    fcvt.s.bf16 fa5, fa0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m2, ta, ma
 ; CHECK-NEXT:    vfwcvtbf16.f.f.v v12, v8
-; CHECK-NEXT:    vmv.v.x v8, a1
-; CHECK-NEXT:    vfwcvtbf16.f.f.v v16, v8
 ; CHECK-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
-; CHECK-NEXT:    vmflt.vv v8, v12, v16, v0.t
-; CHECK-NEXT:    vmflt.vv v9, v16, v12, v0.t
+; CHECK-NEXT:    vmflt.vf v8, v12, fa5, v0.t
+; CHECK-NEXT:    vmfgt.vf v9, v12, fa5, v0.t
 ; CHECK-NEXT:    vmnor.mm v0, v9, v8
 ; CHECK-NEXT:    ret
   %elt.head = insertelement <vscale x 8 x bfloat> poison, bfloat %b, i32 0
@@ -1133,14 +1049,12 @@ define <vscale x 8 x i1> @fcmp_ueq_vf_nxv8bf16(<vscale x 8 x bfloat> %va, bfloat
 define <vscale x 8 x i1> @fcmp_ueq_vf_swap_nxv8bf16(<vscale x 8 x bfloat> %va, bfloat %b, <vscale x 8 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: fcmp_ueq_vf_swap_nxv8bf16:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    fmv.x.h a1, fa0
+; CHECK-NEXT:    fcvt.s.bf16 fa5, fa0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m2, ta, ma
 ; CHECK-NEXT:    vfwcvtbf16.f.f.v v12, v8
-; CHECK-NEXT:    vmv.v.x v8, a1
-; CHECK-NEXT:    vfwcvtbf16.f.f.v v16, v8
 ; CHECK-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
-; CHECK-NEXT:    vmflt.vv v8, v16, v12, v0.t
-; CHECK-NEXT:    vmflt.vv v9, v12, v16, v0.t
+; CHECK-NEXT:    vmfgt.vf v8, v12, fa5, v0.t
+; CHECK-NEXT:    vmflt.vf v9, v12, fa5, v0.t
 ; CHECK-NEXT:    vmnor.mm v0, v9, v8
 ; CHECK-NEXT:    ret
   %elt.head = insertelement <vscale x 8 x bfloat> poison, bfloat %b, i32 0
@@ -1166,13 +1080,11 @@ define <vscale x 8 x i1> @fcmp_ugt_vv_nxv8bf16(<vscale x 8 x bfloat> %va, <vscal
 define <vscale x 8 x i1> @fcmp_ugt_vf_nxv8bf16(<vscale x 8 x bfloat> %va, bfloat %b, <vscale x 8 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: fcmp_ugt_vf_nxv8bf16:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    fmv.x.h a1, fa0
+; CHECK-NEXT:    fcvt.s.bf16 fa5, fa0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m2, ta, ma
-; CHECK-NEXT:    vmv.v.x v10, a1
 ; CHECK-NEXT:    vfwcvtbf16.f.f.v v12, v8
-; CHECK-NEXT:    vfwcvtbf16.f.f.v v16, v10
 ; CHECK-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
-; CHECK-NEXT:    vmfle.vv v8, v12, v16, v0.t
+; CHECK-NEXT:    vmfle.vf v8, v12, fa5, v0.t
 ; CHECK-NEXT:    vmnot.m v0, v8
 ; CHECK-NEXT:    ret
   %elt.head = insertelement <vscale x 8 x bfloat> poison, bfloat %b, i32 0
@@ -1184,13 +1096,11 @@ define <vscale x 8 x i1> @fcmp_ugt_vf_nxv8bf16(<vscale x 8 x bfloat> %va, bfloat
 define <vscale x 8 x i1> @fcmp_ugt_vf_swap_nxv8bf16(<vscale x 8 x bfloat> %va, bfloat %b, <vscale x 8 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: fcmp_ugt_vf_swap_nxv8bf16:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    fmv.x.h a1, fa0
+; CHECK-NEXT:    fcvt.s.bf16 fa5, fa0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m2, ta, ma
-; CHECK-NEXT:    vmv.v.x v10, a1
 ; CHECK-NEXT:    vfwcvtbf16.f.f.v v12, v8
-; CHECK-NEXT:    vfwcvtbf16.f.f.v v16, v10
 ; CHECK-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
-; CHECK-NEXT:    vmfle.vv v8, v16, v12, v0.t
+; CHECK-NEXT:    vmfge.vf v8, v12, fa5, v0.t
 ; CHECK-NEXT:    vmnot.m v0, v8
 ; CHECK-NEXT:    ret
   %elt.head = insertelement <vscale x 8 x bfloat> poison, bfloat %b, i32 0
@@ -1216,13 +1126,11 @@ define <vscale x 8 x i1> @fcmp_uge_vv_nxv8bf16(<vscale x 8 x bfloat> %va, <vscal
 define <vscale x 8 x i1> @fcmp_uge_vf_nxv8bf16(<vscale x 8 x bfloat> %va, bfloat %b, <vscale x 8 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: fcmp_uge_vf_nxv8bf16:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    fmv.x.h a1, fa0
+; CHECK-NEXT:    fcvt.s.bf16 fa5, fa0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m2, ta, ma
-; CHECK-NEXT:    vmv.v.x v10, a1
 ; CHECK-NEXT:    vfwcvtbf16.f.f.v v12, v8
-; CHECK-NEXT:    vfwcvtbf16.f.f.v v16, v10
 ; CHECK-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
-; CHECK-NEXT:    vmflt.vv v8, v12, v16, v0.t
+; CHECK-NEXT:    vmflt.vf v8, v12, fa5, v0.t
 ; CHECK-NEXT:    vmnot.m v0, v8
 ; CHECK-NEXT:    ret
   %elt.head = insertelement <vscale x 8 x bfloat> poison, bfloat %b, i32 0
@@ -1234,13 +1142,11 @@ define <vscale x 8 x i1> @fcmp_uge_vf_nxv8bf16(<vscale x 8 x bfloat> %va, bfloat
 define <vscale x 8 x i1> @fcmp_uge_vf_swap_nxv8bf16(<vscale x 8 x bfloat> %va, bfloat %b, <vscale x 8 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: fcmp_uge_vf_swap_nxv8bf16:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    fmv.x.h a1, fa0
+; CHECK-NEXT:    fcvt.s.bf16 fa5, fa0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m2, ta, ma
-; CHECK-NEXT:    vmv.v.x v10, a1
 ; CHECK-NEXT:    vfwcvtbf16.f.f.v v12, v8
-; CHECK-NEXT:    vfwcvtbf16.f.f.v v16, v10
 ; CHECK-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
-; CHECK-NEXT:    vmflt.vv v8, v16, v12, v0.t
+; CHECK-NEXT:    vmfgt.vf v8, v12, fa5, v0.t
 ; CHECK-NEXT:    vmnot.m v0, v8
 ; CHECK-NEXT:    ret
   %elt.head = insertelement <vscale x 8 x bfloat> poison, bfloat %b, i32 0
@@ -1266,13 +1172,11 @@ define <vscale x 8 x i1> @fcmp_ult_vv_nxv8bf16(<vscale x 8 x bfloat> %va, <vscal
 define <vscale x 8 x i1> @fcmp_ult_vf_nxv8bf16(<vscale x 8 x bfloat> %va, bfloat %b, <vscale x 8 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: fcmp_ult_vf_nxv8bf16:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    fmv.x.h a1, fa0
+; CHECK-NEXT:    fcvt.s.bf16 fa5, fa0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m2, ta, ma
-; CHECK-NEXT:    vmv.v.x v10, a1
 ; CHECK-NEXT:    vfwcvtbf16.f.f.v v12, v8
-; CHECK-NEXT:    vfwcvtbf16.f.f.v v16, v10
 ; CHECK-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
-; CHECK-NEXT:    vmfle.vv v8, v16, v12, v0.t
+; CHECK-NEXT:    vmfge.vf v8, v12, fa5, v0.t
 ; CHECK-NEXT:    vmnot.m v0, v8
 ; CHECK-NEXT:    ret
   %elt.head = insertelement <vscale x 8 x bfloat> poison, bfloat %b, i32 0
@@ -1284,13 +1188,11 @@ define <vscale x 8 x i1> @fcmp_ult_vf_nxv8bf16(<vscale x 8 x bfloat> %va, bfloat
 define <vscale x 8 x i1> @fcmp_ult_vf_swap_nxv8bf16(<vscale x 8 x bfloat> %va, bfloat %b, <vscale x 8 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: fcmp_ult_vf_swap_nxv8bf16:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    fmv.x.h a1, fa0
+; CHECK-NEXT:    fcvt.s.bf16 fa5, fa0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m2, ta, ma
-; CHECK-NEXT:    vmv.v.x v10, a1
 ; CHECK-NEXT:    vfwcvtbf16.f.f.v v12, v8
-; CHECK-NEXT:    vfwcvtbf16.f.f.v v16, v10
 ; CHECK-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
-; CHECK-NEXT:    vmfle.vv v8, v12, v16, v0.t
+; CHECK-NEXT:    vmfle.vf v8, v12, fa5, v0.t
 ; CHECK-NEXT:    vmnot.m v0, v8
 ; CHECK-NEXT:    ret
   %elt.head = insertelement <vscale x 8 x bfloat> poison, bfloat %b, i32 0
@@ -1316,13 +1218,11 @@ define <vscale x 8 x i1> @fcmp_ule_vv_nxv8bf16(<vscale x 8 x bfloat> %va, <vscal
 define <vscale x 8 x i1> @fcmp_ule_vf_nxv8bf16(<vscale x 8 x bfloat> %va, bfloat %b, <vscale x 8 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: fcmp_ule_vf_nxv8bf16:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    fmv.x.h a1, fa0
+; CHECK-NEXT:    fcvt.s.bf16 fa5, fa0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m2, ta, ma
-; CHECK-NEXT:    vmv.v.x v10, a1
 ; CHECK-NEXT:    vfwcvtbf16.f.f.v v12, v8
-; CHECK-NEXT:    vfwcvtbf16.f.f.v v16, v10
 ; CHECK-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
-; CHECK-NEXT:    vmflt.vv v8, v16, v12, v0.t
+; CHECK-NEXT:    vmfgt.vf v8, v12, fa5, v0.t
 ; CHECK-NEXT:    vmnot.m v0, v8
 ; CHECK-NEXT:    ret
   %elt.head = insertelement <vscale x 8 x bfloat> poison, bfloat %b, i32 0
@@ -1334,13 +1234,11 @@ define <vscale x 8 x i1> @fcmp_ule_vf_nxv8bf16(<vscale x 8 x bfloat> %va, bfloat
 define <vscale x 8 x i1> @fcmp_ule_vf_swap_nxv8bf16(<vscale x 8 x bfloat> %va, bfloat %b, <vscale x 8 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: fcmp_ule_vf_swap_nxv8bf16:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    fmv.x.h a1, fa0
+; CHECK-NEXT:    fcvt.s.bf16 fa5, fa0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m2, ta, ma
-; CHECK-NEXT:    vmv.v.x v10, a1
 ; CHECK-NEXT:    vfwcvtbf16.f.f.v v12, v8
-; CHECK-NEXT:    vfwcvtbf16.f.f.v v16, v10
 ; CHECK-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
-; CHECK-NEXT:    vmflt.vv v8, v12, v16, v0.t
+; CHECK-NEXT:    vmflt.vf v8, v12, fa5, v0.t
 ; CHECK-NEXT:    vmnot.m v0, v8
 ; CHECK-NEXT:    ret
   %elt.head = insertelement <vscale x 8 x bfloat> poison, bfloat %b, i32 0
@@ -1366,13 +1264,11 @@ define <vscale x 8 x i1> @fcmp_une_vv_nxv8bf16(<vscale x 8 x bfloat> %va, <vscal
 define <vscale x 8 x i1> @fcmp_une_vf_nxv8bf16(<vscale x 8 x bfloat> %va, bfloat %b, <vscale x 8 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: fcmp_une_vf_nxv8bf16:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    fmv.x.h a1, fa0
+; CHECK-NEXT:    fcvt.s.bf16 fa5, fa0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m2, ta, ma
-; CHECK-NEXT:    vmv.v.x v10, a1
 ; CHECK-NEXT:    vfwcvtbf16.f.f.v v12, v8
-; CHECK-NEXT:    vfwcvtbf16.f.f.v v16, v10
 ; CHECK-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
-; CHECK-NEXT:    vmfne.vv v8, v12, v16, v0.t
+; CHECK-NEXT:    vmfne.vf v8, v12, fa5, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v8
 ; CHECK-NEXT:    ret
   %elt.head = insertelement <vscale x 8 x bfloat> poison, bfloat %b, i32 0
@@ -1384,13 +1280,11 @@ define <vscale x 8 x i1> @fcmp_une_vf_nxv8bf16(<vscale x 8 x bfloat> %va, bfloat
 define <vscale x 8 x i1> @fcmp_une_vf_swap_nxv8bf16(<vscale x 8 x bfloat> %va, bfloat %b, <vscale x 8 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: fcmp_une_vf_swap_nxv8bf16:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    fmv.x.h a1, fa0
+; CHECK-NEXT:    fcvt.s.bf16 fa5, fa0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m2, ta, ma
-; CHECK-NEXT:    vmv.v.x v10, a1
 ; CHECK-NEXT:    vfwcvtbf16.f.f.v v12, v8
-; CHECK-NEXT:    vfwcvtbf16.f.f.v v16, v10
 ; CHECK-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
-; CHECK-NEXT:    vmfne.vv v8, v16, v12, v0.t
+; CHECK-NEXT:    vmfne.vf v8, v12, fa5, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v8
 ; CHECK-NEXT:    ret
   %elt.head = insertelement <vscale x 8 x bfloat> poison, bfloat %b, i32 0
@@ -1419,17 +1313,16 @@ define <vscale x 8 x i1> @fcmp_uno_vv_nxv8bf16(<vscale x 8 x bfloat> %va, <vscal
 define <vscale x 8 x i1> @fcmp_uno_vf_nxv8bf16(<vscale x 8 x bfloat> %va, bfloat %b, <vscale x 8 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: fcmp_uno_vf_nxv8bf16:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    fmv.x.h a1, fa0
-; CHECK-NEXT:    vsetvli zero, a0, e16, m2, ta, ma
-; CHECK-NEXT:    vfwcvtbf16.f.f.v v12, v8
-; CHECK-NEXT:    vmv.v.x v8, a1
-; CHECK-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
-; CHECK-NEXT:    vmfne.vv v10, v12, v12, v0.t
+; CHECK-NEXT:    fcvt.s.bf16 fa5, fa0
+; CHECK-NEXT:    vsetvli a1, zero, e32, m4, ta, ma
+; CHECK-NEXT:    vfmv.v.f v12, fa5
+; CHECK-NEXT:    vsetvli zero, a0, e32, m4, ta, ma
+; CHECK-NEXT:    vmfne.vf v10, v12, fa5, v0.t
 ; CHECK-NEXT:    vsetvli zero, zero, e16, m2, ta, ma
 ; CHECK-NEXT:    vfwcvtbf16.f.f.v v12, v8
 ; CHECK-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
 ; CHECK-NEXT:    vmfne.vv v8, v12, v12, v0.t
-; CHECK-NEXT:    vmor.mm v0, v10, v8
+; CHECK-NEXT:    vmor.mm v0, v8, v10
 ; CHECK-NEXT:    ret
   %elt.head = insertelement <vscale x 8 x bfloat> poison, bfloat %b, i32 0
   %vb = shufflevector <vscale x 8 x bfloat> %elt.head, <vscale x 8 x bfloat> poison, <vscale x 8 x i32> zeroinitializer
@@ -1440,17 +1333,16 @@ define <vscale x 8 x i1> @fcmp_uno_vf_nxv8bf16(<vscale x 8 x bfloat> %va, bfloat
 define <vscale x 8 x i1> @fcmp_uno_vf_swap_nxv8bf16(<vscale x 8 x bfloat> %va, bfloat %b, <vscale x 8 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: fcmp_uno_vf_swap_nxv8bf16:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    fmv.x.h a1, fa0
-; CHECK-NEXT:    vsetvli zero, a0, e16, m2, ta, ma
-; CHECK-NEXT:    vfwcvtbf16.f.f.v v12, v8
-; CHECK-NEXT:    vmv.v.x v8, a1
-; CHECK-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
-; CHECK-NEXT:    vmfne.vv v10, v12, v12, v0.t
+; CHECK-NEXT:    fcvt.s.bf16 fa5, fa0
+; CHECK-NEXT:    vsetvli a1, zero, e32, m4, ta, ma
+; CHECK-NEXT:    vfmv.v.f v12, fa5
+; CHECK-NEXT:    vsetvli zero, a0, e32, m4, ta, ma
+; CHECK-NEXT:    vmfne.vf v10, v12, fa5, v0.t
 ; CHECK-NEXT:    vsetvli zero, zero, e16, m2, ta, ma
 ; CHECK-NEXT:    vfwcvtbf16.f.f.v v12, v8
 ; CHECK-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
 ; CHECK-NEXT:    vmfne.vv v8, v12, v12, v0.t
-; CHECK-NEXT:    vmor.mm v0, v8, v10
+; CHECK-NEXT:    vmor.mm v0, v10, v8
 ; CHECK-NEXT:    ret
   %elt.head = insertelement <vscale x 8 x bfloat> poison, bfloat %b, i32 0
   %vb = shufflevector <vscale x 8 x bfloat> %elt.head, <vscale x 8 x bfloat> poison, <vscale x 8 x i32> zeroinitializer
@@ -1677,13 +1569,11 @@ define <vscale x 1 x i1> @fcmp_oeq_vf_nxv1f16(<vscale x 1 x half> %va, half %b,
 ;
 ; ZVFHMIN-LABEL: fcmp_oeq_vf_nxv1f16:
 ; ZVFHMIN:       # %bb.0:
-; ZVFHMIN-NEXT:    fmv.x.h a1, fa0
+; ZVFHMIN-NEXT:    fcvt.s.h fa5, fa0
 ; ZVFHMIN-NEXT:    vsetvli zero, a0, e16, mf4, ta, ma
-; ZVFHMIN-NEXT:    vmv.v.x v9, a1
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v10, v8
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v8, v9
+; ZVFHMIN-NEXT:    vfwcvt.f.f.v v9, v8
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, mf2, ta, ma
-; ZVFHMIN-NEXT:    vmfeq.vv v0, v10, v8, v0.t
+; ZVFHMIN-NEXT:    vmfeq.vf v0, v9, fa5, v0.t
 ; ZVFHMIN-NEXT:    ret
   %elt.head = insertelement <vscale x 1 x half> poison, half %b, i32 0
   %vb = shufflevector <vscale x 1 x half> %elt.head, <vscale x 1 x half> poison, <vscale x 1 x i32> zeroinitializer
@@ -1700,13 +1590,11 @@ define <vscale x 1 x i1> @fcmp_oeq_vf_swap_nxv1f16(<vscale x 1 x half> %va, half
 ;
 ; ZVFHMIN-LABEL: fcmp_oeq_vf_swap_nxv1f16:
 ; ZVFHMIN:       # %bb.0:
-; ZVFHMIN-NEXT:    fmv.x.h a1, fa0
+; ZVFHMIN-NEXT:    fcvt.s.h fa5, fa0
 ; ZVFHMIN-NEXT:    vsetvli zero, a0, e16, mf4, ta, ma
-; ZVFHMIN-NEXT:    vmv.v.x v9, a1
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v10, v8
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v8, v9
+; ZVFHMIN-NEXT:    vfwcvt.f.f.v v9, v8
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, mf2, ta, ma
-; ZVFHMIN-NEXT:    vmfeq.vv v0, v8, v10, v0.t
+; ZVFHMIN-NEXT:    vmfeq.vf v0, v9, fa5, v0.t
 ; ZVFHMIN-NEXT:    ret
   %elt.head = insertelement <vscale x 1 x half> poison, half %b, i32 0
   %vb = shufflevector <vscale x 1 x half> %elt.head, <vscale x 1 x half> poison, <vscale x 1 x i32> zeroinitializer
@@ -1742,13 +1630,11 @@ define <vscale x 1 x i1> @fcmp_ogt_vf_nxv1f16(<vscale x 1 x half> %va, half %b,
 ;
 ; ZVFHMIN-LABEL: fcmp_ogt_vf_nxv1f16:
 ; ZVFHMIN:       # %bb.0:
-; ZVFHMIN-NEXT:    fmv.x.h a1, fa0
+; ZVFHMIN-NEXT:    fcvt.s.h fa5, fa0
 ; ZVFHMIN-NEXT:    vsetvli zero, a0, e16, mf4, ta, ma
-; ZVFHMIN-NEXT:    vmv.v.x v9, a1
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v10, v8
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v8, v9
+; ZVFHMIN-NEXT:    vfwcvt.f.f.v v9, v8
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, mf2, ta, ma
-; ZVFHMIN-NEXT:    vmflt.vv v0, v8, v10, v0.t
+; ZVFHMIN-NEXT:    vmfgt.vf v0, v9, fa5, v0.t
 ; ZVFHMIN-NEXT:    ret
   %elt.head = insertelement <vscale x 1 x half> poison, half %b, i32 0
   %vb = shufflevector <vscale x 1 x half> %elt.head, <vscale x 1 x half> poison, <vscale x 1 x i32> zeroinitializer
@@ -1765,13 +1651,11 @@ define <vscale x 1 x i1> @fcmp_ogt_vf_swap_nxv1f16(<vscale x 1 x half> %va, half
 ;
 ; ZVFHMIN-LABEL: fcmp_ogt_vf_swap_nxv1f16:
 ; ZVFHMIN:       # %bb.0:
-; ZVFHMIN-NEXT:    fmv.x.h a1, fa0
+; ZVFHMIN-NEXT:    fcvt.s.h fa5, fa0
 ; ZVFHMIN-NEXT:    vsetvli zero, a0, e16, mf4, ta, ma
-; ZVFHMIN-NEXT:    vmv.v.x v9, a1
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v10, v8
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v8, v9
+; ZVFHMIN-NEXT:    vfwcvt.f.f.v v9, v8
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, mf2, ta, ma
-; ZVFHMIN-NEXT:    vmflt.vv v0, v10, v8, v0.t
+; ZVFHMIN-NEXT:    vmflt.vf v0, v9, fa5, v0.t
 ; ZVFHMIN-NEXT:    ret
   %elt.head = insertelement <vscale x 1 x half> poison, half %b, i32 0
   %vb = shufflevector <vscale x 1 x half> %elt.head, <vscale x 1 x half> poison, <vscale x 1 x i32> zeroinitializer
@@ -1807,13 +1691,11 @@ define <vscale x 1 x i1> @fcmp_oge_vf_nxv1f16(<vscale x 1 x half> %va, half %b,
 ;
 ; ZVFHMIN-LABEL: fcmp_oge_vf_nxv1f16:
 ; ZVFHMIN:       # %bb.0:
-; ZVFHMIN-NEXT:    fmv.x.h a1, fa0
+; ZVFHMIN-NEXT:    fcvt.s.h fa5, fa0
 ; ZVFHMIN-NEXT:    vsetvli zero, a0, e16, mf4, ta, ma
-; ZVFHMIN-NEXT:    vmv.v.x v9, a1
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v10, v8
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v8, v9
+; ZVFHMIN-NEXT:    vfwcvt.f.f.v v9, v8
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, mf2, ta, ma
-; ZVFHMIN-NEXT:    vmfle.vv v0, v8, v10, v0.t
+; ZVFHMIN-NEXT:    vmfge.vf v0, v9, fa5, v0.t
 ; ZVFHMIN-NEXT:    ret
   %elt.head = insertelement <vscale x 1 x half> poison, half %b, i32 0
   %vb = shufflevector <vscale x 1 x half> %elt.head, <vscale x 1 x half> poison, <vscale x 1 x i32> zeroinitializer
@@ -1830,13 +1712,11 @@ define <vscale x 1 x i1> @fcmp_oge_vf_swap_nxv1f16(<vscale x 1 x half> %va, half
 ;
 ; ZVFHMIN-LABEL: fcmp_oge_vf_swap_nxv1f16:
 ; ZVFHMIN:       # %bb.0:
-; ZVFHMIN-NEXT:    fmv.x.h a1, fa0
+; ZVFHMIN-NEXT:    fcvt.s.h fa5, fa0
 ; ZVFHMIN-NEXT:    vsetvli zero, a0, e16, mf4, ta, ma
-; ZVFHMIN-NEXT:    vmv.v.x v9, a1
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v10, v8
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v8, v9
+; ZVFHMIN-NEXT:    vfwcvt.f.f.v v9, v8
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, mf2, ta, ma
-; ZVFHMIN-NEXT:    vmfle.vv v0, v10, v8, v0.t
+; ZVFHMIN-NEXT:    vmfle.vf v0, v9, fa5, v0.t
 ; ZVFHMIN-NEXT:    ret
   %elt.head = insertelement <vscale x 1 x half> poison, half %b, i32 0
   %vb = shufflevector <vscale x 1 x half> %elt.head, <vscale x 1 x half> poison, <vscale x 1 x i32> zeroinitializer
@@ -1872,13 +1752,11 @@ define <vscale x 1 x i1> @fcmp_olt_vf_nxv1f16(<vscale x 1 x half> %va, half %b,
 ;
 ; ZVFHMIN-LABEL: fcmp_olt_vf_nxv1f16:
 ; ZVFHMIN:       # %bb.0:
-; ZVFHMIN-NEXT:    fmv.x.h a1, fa0
+; ZVFHMIN-NEXT:    fcvt.s.h fa5, fa0
 ; ZVFHMIN-NEXT:    vsetvli zero, a0, e16, mf4, ta, ma
-; ZVFHMIN-NEXT:    vmv.v.x v9, a1
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v10, v8
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v8, v9
+; ZVFHMIN-NEXT:    vfwcvt.f.f.v v9, v8
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, mf2, ta, ma
-; ZVFHMIN-NEXT:    vmflt.vv v0, v10, v8, v0.t
+; ZVFHMIN-NEXT:    vmflt.vf v0, v9, fa5, v0.t
 ; ZVFHMIN-NEXT:    ret
   %elt.head = insertelement <vscale x 1 x half> poison, half %b, i32 0
   %vb = shufflevector <vscale x 1 x half> %elt.head, <vscale x 1 x half> poison, <vscale x 1 x i32> zeroinitializer
@@ -1895,13 +1773,11 @@ define <vscale x 1 x i1> @fcmp_olt_vf_swap_nxv1f16(<vscale x 1 x half> %va, half
 ;
 ; ZVFHMIN-LABEL: fcmp_olt_vf_swap_nxv1f16:
 ; ZVFHMIN:       # %bb.0:
-; ZVFHMIN-NEXT:    fmv.x.h a1, fa0
+; ZVFHMIN-NEXT:    fcvt.s.h fa5, fa0
 ; ZVFHMIN-NEXT:    vsetvli zero, a0, e16, mf4, ta, ma
-; ZVFHMIN-NEXT:    vmv.v.x v9, a1
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v10, v8
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v8, v9
+; ZVFHMIN-NEXT:    vfwcvt.f.f.v v9, v8
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, mf2, ta, ma
-; ZVFHMIN-NEXT:    vmflt.vv v0, v8, v10, v0.t
+; ZVFHMIN-NEXT:    vmfgt.vf v0, v9, fa5, v0.t
 ; ZVFHMIN-NEXT:    ret
   %elt.head = insertelement <vscale x 1 x half> poison, half %b, i32 0
   %vb = shufflevector <vscale x 1 x half> %elt.head, <vscale x 1 x half> poison, <vscale x 1 x i32> zeroinitializer
@@ -1937,13 +1813,11 @@ define <vscale x 1 x i1> @fcmp_ole_vf_nxv1f16(<vscale x 1 x half> %va, half %b,
 ;
 ; ZVFHMIN-LABEL: fcmp_ole_vf_nxv1f16:
 ; ZVFHMIN:       # %bb.0:
-; ZVFHMIN-NEXT:    fmv.x.h a1, fa0
+; ZVFHMIN-NEXT:    fcvt.s.h fa5, fa0
 ; ZVFHMIN-NEXT:    vsetvli zero, a0, e16, mf4, ta, ma
-; ZVFHMIN-NEXT:    vmv.v.x v9, a1
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v10, v8
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v8, v9
+; ZVFHMIN-NEXT:    vfwcvt.f.f.v v9, v8
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, mf2, ta, ma
-; ZVFHMIN-NEXT:    vmfle.vv v0, v10, v8, v0.t
+; ZVFHMIN-NEXT:    vmfle.vf v0, v9, fa5, v0.t
 ; ZVFHMIN-NEXT:    ret
   %elt.head = insertelement <vscale x 1 x half> poison, half %b, i32 0
   %vb = shufflevector <vscale x 1 x half> %elt.head, <vscale x 1 x half> poison, <vscale x 1 x i32> zeroinitializer
@@ -1960,13 +1834,11 @@ define <vscale x 1 x i1> @fcmp_ole_vf_swap_nxv1f16(<vscale x 1 x half> %va, half
 ;
 ; ZVFHMIN-LABEL: fcmp_ole_vf_swap_nxv1f16:
 ; ZVFHMIN:       # %bb.0:
-; ZVFHMIN-NEXT:    fmv.x.h a1, fa0
+; ZVFHMIN-NEXT:    fcvt.s.h fa5, fa0
 ; ZVFHMIN-NEXT:    vsetvli zero, a0, e16, mf4, ta, ma
-; ZVFHMIN-NEXT:    vmv.v.x v9, a1
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v10, v8
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v8, v9
+; ZVFHMIN-NEXT:    vfwcvt.f.f.v v9, v8
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, mf2, ta, ma
-; ZVFHMIN-NEXT:    vmfle.vv v0, v8, v10, v0.t
+; ZVFHMIN-NEXT:    vmfge.vf v0, v9, fa5, v0.t
 ; ZVFHMIN-NEXT:    ret
   %elt.head = insertelement <vscale x 1 x half> poison, half %b, i32 0
   %vb = shufflevector <vscale x 1 x half> %elt.head, <vscale x 1 x half> poison, <vscale x 1 x i32> zeroinitializer
@@ -2008,14 +1880,12 @@ define <vscale x 1 x i1> @fcmp_one_vf_nxv1f16(<vscale x 1 x half> %va, half %b,
 ;
 ; ZVFHMIN-LABEL: fcmp_one_vf_nxv1f16:
 ; ZVFHMIN:       # %bb.0:
-; ZVFHMIN-NEXT:    fmv.x.h a1, fa0
+; ZVFHMIN-NEXT:    fcvt.s.h fa5, fa0
 ; ZVFHMIN-NEXT:    vsetvli zero, a0, e16, mf4, ta, ma
 ; ZVFHMIN-NEXT:    vfwcvt.f.f.v v9, v8
-; ZVFHMIN-NEXT:    vmv.v.x v8, a1
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v10, v8
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, mf2, ta, ma
-; ZVFHMIN-NEXT:    vmflt.vv v8, v9, v10, v0.t
-; ZVFHMIN-NEXT:    vmflt.vv v9, v10, v9, v0.t
+; ZVFHMIN-NEXT:    vmflt.vf v8, v9, fa5, v0.t
+; ZVFHMIN-NEXT:    vmfgt.vf v9, v9, fa5, v0.t
 ; ZVFHMIN-NEXT:    vmor.mm v0, v9, v8
 ; ZVFHMIN-NEXT:    ret
   %elt.head = insertelement <vscale x 1 x half> poison, half %b, i32 0
@@ -2035,14 +1905,12 @@ define <vscale x 1 x i1> @fcmp_one_vf_swap_nxv1f16(<vscale x 1 x half> %va, half
 ;
 ; ZVFHMIN-LABEL: fcmp_one_vf_swap_nxv1f16:
 ; ZVFHMIN:       # %bb.0:
-; ZVFHMIN-NEXT:    fmv.x.h a1, fa0
+; ZVFHMIN-NEXT:    fcvt.s.h fa5, fa0
 ; ZVFHMIN-NEXT:    vsetvli zero, a0, e16, mf4, ta, ma
 ; ZVFHMIN-NEXT:    vfwcvt.f.f.v v9, v8
-; ZVFHMIN-NEXT:    vmv.v.x v8, a1
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v10, v8
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, mf2, ta, ma
-; ZVFHMIN-NEXT:    vmflt.vv v8, v10, v9, v0.t
-; ZVFHMIN-NEXT:    vmflt.vv v9, v9, v10, v0.t
+; ZVFHMIN-NEXT:    vmfgt.vf v8, v9, fa5, v0.t
+; ZVFHMIN-NEXT:    vmflt.vf v9, v9, fa5, v0.t
 ; ZVFHMIN-NEXT:    vmor.mm v0, v9, v8
 ; ZVFHMIN-NEXT:    ret
   %elt.head = insertelement <vscale x 1 x half> poison, half %b, i32 0
@@ -2087,16 +1955,14 @@ define <vscale x 1 x i1> @fcmp_ord_vf_nxv1f16(<vscale x 1 x half> %va, half %b,
 ;
 ; ZVFHMIN-LABEL: fcmp_ord_vf_nxv1f16:
 ; ZVFHMIN:       # %bb.0:
-; ZVFHMIN-NEXT:    fmv.x.h a1, fa0
+; ZVFHMIN-NEXT:    fcvt.s.h fa5, fa0
 ; ZVFHMIN-NEXT:    vsetvli zero, a0, e16, mf4, ta, ma
 ; ZVFHMIN-NEXT:    vfwcvt.f.f.v v9, v8
-; ZVFHMIN-NEXT:    vmv.v.x v8, a1
-; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, mf2, ta, ma
+; ZVFHMIN-NEXT:    vsetvli a1, zero, e32, mf2, ta, ma
+; ZVFHMIN-NEXT:    vfmv.v.f v8, fa5
+; ZVFHMIN-NEXT:    vsetvli zero, a0, e32, mf2, ta, ma
+; ZVFHMIN-NEXT:    vmfeq.vf v8, v8, fa5, v0.t
 ; ZVFHMIN-NEXT:    vmfeq.vv v9, v9, v9, v0.t
-; ZVFHMIN-NEXT:    vsetvli zero, zero, e16, mf4, ta, ma
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v10, v8
-; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, mf2, ta, ma
-; ZVFHMIN-NEXT:    vmfeq.vv v8, v10, v10, v0.t
 ; ZVFHMIN-NEXT:    vmand.mm v0, v9, v8
 ; ZVFHMIN-NEXT:    ret
   %elt.head = insertelement <vscale x 1 x half> poison, half %b, i32 0
@@ -2118,16 +1984,14 @@ define <vscale x 1 x i1> @fcmp_ord_vf_swap_nxv1f16(<vscale x 1 x half> %va, half
 ;
 ; ZVFHMIN-LABEL: fcmp_ord_vf_swap_nxv1f16:
 ; ZVFHMIN:       # %bb.0:
-; ZVFHMIN-NEXT:    fmv.x.h a1, fa0
+; ZVFHMIN-NEXT:    fcvt.s.h fa5, fa0
 ; ZVFHMIN-NEXT:    vsetvli zero, a0, e16, mf4, ta, ma
 ; ZVFHMIN-NEXT:    vfwcvt.f.f.v v9, v8
-; ZVFHMIN-NEXT:    vmv.v.x v8, a1
-; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, mf2, ta, ma
+; ZVFHMIN-NEXT:    vsetvli a1, zero, e32, mf2, ta, ma
+; ZVFHMIN-NEXT:    vfmv.v.f v8, fa5
+; ZVFHMIN-NEXT:    vsetvli zero, a0, e32, mf2, ta, ma
+; ZVFHMIN-NEXT:    vmfeq.vf v8, v8, fa5, v0.t
 ; ZVFHMIN-NEXT:    vmfeq.vv v9, v9, v9, v0.t
-; ZVFHMIN-NEXT:    vsetvli zero, zero, e16, mf4, ta, ma
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v10, v8
-; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, mf2, ta, ma
-; ZVFHMIN-NEXT:    vmfeq.vv v8, v10, v10, v0.t
 ; ZVFHMIN-NEXT:    vmand.mm v0, v8, v9
 ; ZVFHMIN-NEXT:    ret
   %elt.head = insertelement <vscale x 1 x half> poison, half %b, i32 0
@@ -2170,14 +2034,12 @@ define <vscale x 1 x i1> @fcmp_ueq_vf_nxv1f16(<vscale x 1 x half> %va, half %b,
 ;
 ; ZVFHMIN-LABEL: fcmp_ueq_vf_nxv1f16:
 ; ZVFHMIN:       # %bb.0:
-; ZVFHMIN-NEXT:    fmv.x.h a1, fa0
+; ZVFHMIN-NEXT:    fcvt.s.h fa5, fa0
 ; ZVFHMIN-NEXT:    vsetvli zero, a0, e16, mf4, ta, ma
 ; ZVFHMIN-NEXT:    vfwcvt.f.f.v v9, v8
-; ZVFHMIN-NEXT:    vmv.v.x v8, a1
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v10, v8
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, mf2, ta, ma
-; ZVFHMIN-NEXT:    vmflt.vv v8, v9, v10, v0.t
-; ZVFHMIN-NEXT:    vmflt.vv v9, v10, v9, v0.t
+; ZVFHMIN-NEXT:    vmflt.vf v8, v9, fa5, v0.t
+; ZVFHMIN-NEXT:    vmfgt.vf v9, v9, fa5, v0.t
 ; ZVFHMIN-NEXT:    vmnor.mm v0, v9, v8
 ; ZVFHMIN-NEXT:    ret
   %elt.head = insertelement <vscale x 1 x half> poison, half %b, i32 0
@@ -2197,14 +2059,12 @@ define <vscale x 1 x i1> @fcmp_ueq_vf_swap_nxv1f16(<vscale x 1 x half> %va, half
 ;
 ; ZVFHMIN-LABEL: fcmp_ueq_vf_swap_nxv1f16:
 ; ZVFHMIN:       # %bb.0:
-; ZVFHMIN-NEXT:    fmv.x.h a1, fa0
+; ZVFHMIN-NEXT:    fcvt.s.h fa5, fa0
 ; ZVFHMIN-NEXT:    vsetvli zero, a0, e16, mf4, ta, ma
 ; ZVFHMIN-NEXT:    vfwcvt.f.f.v v9, v8
-; ZVFHMIN-NEXT:    vmv.v.x v8, a1
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v10, v8
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, mf2, ta, ma
-; ZVFHMIN-NEXT:    vmflt.vv v8, v10, v9, v0.t
-; ZVFHMIN-NEXT:    vmflt.vv v9, v9, v10, v0.t
+; ZVFHMIN-NEXT:    vmfgt.vf v8, v9, fa5, v0.t
+; ZVFHMIN-NEXT:    vmflt.vf v9, v9, fa5, v0.t
 ; ZVFHMIN-NEXT:    vmnor.mm v0, v9, v8
 ; ZVFHMIN-NEXT:    ret
   %elt.head = insertelement <vscale x 1 x half> poison, half %b, i32 0
@@ -2244,13 +2104,11 @@ define <vscale x 1 x i1> @fcmp_ugt_vf_nxv1f16(<vscale x 1 x half> %va, half %b,
 ;
 ; ZVFHMIN-LABEL: fcmp_ugt_vf_nxv1f16:
 ; ZVFHMIN:       # %bb.0:
-; ZVFHMIN-NEXT:    fmv.x.h a1, fa0
+; ZVFHMIN-NEXT:    fcvt.s.h fa5, fa0
 ; ZVFHMIN-NEXT:    vsetvli zero, a0, e16, mf4, ta, ma
-; ZVFHMIN-NEXT:    vmv.v.x v9, a1
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v10, v8
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v8, v9
+; ZVFHMIN-NEXT:    vfwcvt.f.f.v v9, v8
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, mf2, ta, ma
-; ZVFHMIN-NEXT:    vmfle.vv v8, v10, v8, v0.t
+; ZVFHMIN-NEXT:    vmfle.vf v8, v9, fa5, v0.t
 ; ZVFHMIN-NEXT:    vmnot.m v0, v8
 ; ZVFHMIN-NEXT:    ret
   %elt.head = insertelement <vscale x 1 x half> poison, half %b, i32 0
@@ -2269,13 +2127,11 @@ define <vscale x 1 x i1> @fcmp_ugt_vf_swap_nxv1f16(<vscale x 1 x half> %va, half
 ;
 ; ZVFHMIN-LABEL: fcmp_ugt_vf_swap_nxv1f16:
 ; ZVFHMIN:       # %bb.0:
-; ZVFHMIN-NEXT:    fmv.x.h a1, fa0
+; ZVFHMIN-NEXT:    fcvt.s.h fa5, fa0
 ; ZVFHMIN-NEXT:    vsetvli zero, a0, e16, mf4, ta, ma
-; ZVFHMIN-NEXT:    vmv.v.x v9, a1
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v10, v8
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v8, v9
+; ZVFHMIN-NEXT:    vfwcvt.f.f.v v9, v8
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, mf2, ta, ma
-; ZVFHMIN-NEXT:    vmfle.vv v8, v8, v10, v0.t
+; ZVFHMIN-NEXT:    vmfge.vf v8, v9, fa5, v0.t
 ; ZVFHMIN-NEXT:    vmnot.m v0, v8
 ; ZVFHMIN-NEXT:    ret
   %elt.head = insertelement <vscale x 1 x half> poison, half %b, i32 0
@@ -2315,13 +2171,11 @@ define <vscale x 1 x i1> @fcmp_uge_vf_nxv1f16(<vscale x 1 x half> %va, half %b,
 ;
 ; ZVFHMIN-LABEL: fcmp_uge_vf_nxv1f16:
 ; ZVFHMIN:       # %bb.0:
-; ZVFHMIN-NEXT:    fmv.x.h a1, fa0
+; ZVFHMIN-NEXT:    fcvt.s.h fa5, fa0
 ; ZVFHMIN-NEXT:    vsetvli zero, a0, e16, mf4, ta, ma
-; ZVFHMIN-NEXT:    vmv.v.x v9, a1
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v10, v8
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v8, v9
+; ZVFHMIN-NEXT:    vfwcvt.f.f.v v9, v8
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, mf2, ta, ma
-; ZVFHMIN-NEXT:    vmflt.vv v8, v10, v8, v0.t
+; ZVFHMIN-NEXT:    vmflt.vf v8, v9, fa5, v0.t
 ; ZVFHMIN-NEXT:    vmnot.m v0, v8
 ; ZVFHMIN-NEXT:    ret
   %elt.head = insertelement <vscale x 1 x half> poison, half %b, i32 0
@@ -2340,13 +2194,11 @@ define <vscale x 1 x i1> @fcmp_uge_vf_swap_nxv1f16(<vscale x 1 x half> %va, half
 ;
 ; ZVFHMIN-LABEL: fcmp_uge_vf_swap_nxv1f16:
 ; ZVFHMIN:       # %bb.0:
-; ZVFHMIN-NEXT:    fmv.x.h a1, fa0
+; ZVFHMIN-NEXT:    fcvt.s.h fa5, fa0
 ; ZVFHMIN-NEXT:    vsetvli zero, a0, e16, mf4, ta, ma
-; ZVFHMIN-NEXT:    vmv.v.x v9, a1
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v10, v8
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v8, v9
+; ZVFHMIN-NEXT:    vfwcvt.f.f.v v9, v8
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, mf2, ta, ma
-; ZVFHMIN-NEXT:    vmflt.vv v8, v8, v10, v0.t
+; ZVFHMIN-NEXT:    vmfgt.vf v8, v9, fa5, v0.t
 ; ZVFHMIN-NEXT:    vmnot.m v0, v8
 ; ZVFHMIN-NEXT:    ret
   %elt.head = insertelement <vscale x 1 x half> poison, half %b, i32 0
@@ -2386,13 +2238,11 @@ define <vscale x 1 x i1> @fcmp_ult_vf_nxv1f16(<vscale x 1 x half> %va, half %b,
 ;
 ; ZVFHMIN-LABEL: fcmp_ult_vf_nxv1f16:
 ; ZVFHMIN:       # %bb.0:
-; ZVFHMIN-NEXT:    fmv.x.h a1, fa0
+; ZVFHMIN-NEXT:    fcvt.s.h fa5, fa0
 ; ZVFHMIN-NEXT:    vsetvli zero, a0, e16, mf4, ta, ma
-; ZVFHMIN-NEXT:    vmv.v.x v9, a1
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v10, v8
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v8, v9
+; ZVFHMIN-NEXT:    vfwcvt.f.f.v v9, v8
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, mf2, ta, ma
-; ZVFHMIN-NEXT:    vmfle.vv v8, v8, v10, v0.t
+; ZVFHMIN-NEXT:    vmfge.vf v8, v9, fa5, v0.t
 ; ZVFHMIN-NEXT:    vmnot.m v0, v8
 ; ZVFHMIN-NEXT:    ret
   %elt.head = insertelement <vscale x 1 x half> poison, half %b, i32 0
@@ -2411,13 +2261,11 @@ define <vscale x 1 x i1> @fcmp_ult_vf_swap_nxv1f16(<vscale x 1 x half> %va, half
 ;
 ; ZVFHMIN-LABEL: fcmp_ult_vf_swap_nxv1f16:
 ; ZVFHMIN:       # %bb.0:
-; ZVFHMIN-NEXT:    fmv.x.h a1, fa0
+; ZVFHMIN-NEXT:    fcvt.s.h fa5, fa0
 ; ZVFHMIN-NEXT:    vsetvli zero, a0, e16, mf4, ta, ma
-; ZVFHMIN-NEXT:    vmv.v.x v9, a1
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v10, v8
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v8, v9
+; ZVFHMIN-NEXT:    vfwcvt.f.f.v v9, v8
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, mf2, ta, ma
-; ZVFHMIN-NEXT:    vmfle.vv v8, v10, v8, v0.t
+; ZVFHMIN-NEXT:    vmfle.vf v8, v9, fa5, v0.t
 ; ZVFHMIN-NEXT:    vmnot.m v0, v8
 ; ZVFHMIN-NEXT:    ret
   %elt.head = insertelement <vscale x 1 x half> poison, half %b, i32 0
@@ -2457,13 +2305,11 @@ define <vscale x 1 x i1> @fcmp_ule_vf_nxv1f16(<vscale x 1 x half> %va, half %b,
 ;
 ; ZVFHMIN-LABEL: fcmp_ule_vf_nxv1f16:
 ; ZVFHMIN:       # %bb.0:
-; ZVFHMIN-NEXT:    fmv.x.h a1, fa0
+; ZVFHMIN-NEXT:    fcvt.s.h fa5, fa0
 ; ZVFHMIN-NEXT:    vsetvli zero, a0, e16, mf4, ta, ma
-; ZVFHMIN-NEXT:    vmv.v.x v9, a1
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v10, v8
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v8, v9
+; ZVFHMIN-NEXT:    vfwcvt.f.f.v v9, v8
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, mf2, ta, ma
-; ZVFHMIN-NEXT:    vmflt.vv v8, v8, v10, v0.t
+; ZVFHMIN-NEXT:    vmfgt.vf v8, v9, fa5, v0.t
 ; ZVFHMIN-NEXT:    vmnot.m v0, v8
 ; ZVFHMIN-NEXT:    ret
   %elt.head = insertelement <vscale x 1 x half> poison, half %b, i32 0
@@ -2482,13 +2328,11 @@ define <vscale x 1 x i1> @fcmp_ule_vf_swap_nxv1f16(<vscale x 1 x half> %va, half
 ;
 ; ZVFHMIN-LABEL: fcmp_ule_vf_swap_nxv1f16:
 ; ZVFHMIN:       # %bb.0:
-; ZVFHMIN-NEXT:    fmv.x.h a1, fa0
+; ZVFHMIN-NEXT:    fcvt.s.h fa5, fa0
 ; ZVFHMIN-NEXT:    vsetvli zero, a0, e16, mf4, ta, ma
-; ZVFHMIN-NEXT:    vmv.v.x v9, a1
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v10, v8
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v8, v9
+; ZVFHMIN-NEXT:    vfwcvt.f.f.v v9, v8
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, mf2, ta, ma
-; ZVFHMIN-NEXT:    vmflt.vv v8, v10, v8, v0.t
+; ZVFHMIN-NEXT:    vmflt.vf v8, v9, fa5, v0.t
 ; ZVFHMIN-NEXT:    vmnot.m v0, v8
 ; ZVFHMIN-NEXT:    ret
   %elt.head = insertelement <vscale x 1 x half> poison, half %b, i32 0
@@ -2525,13 +2369,11 @@ define <vscale x 1 x i1> @fcmp_une_vf_nxv1f16(<vscale x 1 x half> %va, half %b,
 ;
 ; ZVFHMIN-LABEL: fcmp_une_vf_nxv1f16:
 ; ZVFHMIN:       # %bb.0:
-; ZVFHMIN-NEXT:    fmv.x.h a1, fa0
+; ZVFHMIN-NEXT:    fcvt.s.h fa5, fa0
 ; ZVFHMIN-NEXT:    vsetvli zero, a0, e16, mf4, ta, ma
-; ZVFHMIN-NEXT:    vmv.v.x v9, a1
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v10, v8
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v8, v9
+; ZVFHMIN-NEXT:    vfwcvt.f.f.v v9, v8
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, mf2, ta, ma
-; ZVFHMIN-NEXT:    vmfne.vv v0, v10, v8, v0.t
+; ZVFHMIN-NEXT:    vmfne.vf v0, v9, fa5, v0.t
 ; ZVFHMIN-NEXT:    ret
   %elt.head = insertelement <vscale x 1 x half> poison, half %b, i32 0
   %vb = shufflevector <vscale x 1 x half> %elt.head, <vscale x 1 x half> poison, <vscale x 1 x i32> zeroinitializer
@@ -2548,13 +2390,11 @@ define <vscale x 1 x i1> @fcmp_une_vf_swap_nxv1f16(<vscale x 1 x half> %va, half
 ;
 ; ZVFHMIN-LABEL: fcmp_une_vf_swap_nxv1f16:
 ; ZVFHMIN:       # %bb.0:
-; ZVFHMIN-NEXT:    fmv.x.h a1, fa0
+; ZVFHMIN-NEXT:    fcvt.s.h fa5, fa0
 ; ZVFHMIN-NEXT:    vsetvli zero, a0, e16, mf4, ta, ma
-; ZVFHMIN-NEXT:    vmv.v.x v9, a1
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v10, v8
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v8, v9
+; ZVFHMIN-NEXT:    vfwcvt.f.f.v v9, v8
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, mf2, ta, ma
-; ZVFHMIN-NEXT:    vmfne.vv v0, v8, v10, v0.t
+; ZVFHMIN-NEXT:    vmfne.vf v0, v9, fa5, v0.t
 ; ZVFHMIN-NEXT:    ret
   %elt.head = insertelement <vscale x 1 x half> poison, half %b, i32 0
   %vb = shufflevector <vscale x 1 x half> %elt.head, <vscale x 1 x half> poison, <vscale x 1 x i32> zeroinitializer
@@ -2598,16 +2438,14 @@ define <vscale x 1 x i1> @fcmp_uno_vf_nxv1f16(<vscale x 1 x half> %va, half %b,
 ;
 ; ZVFHMIN-LABEL: fcmp_uno_vf_nxv1f16:
 ; ZVFHMIN:       # %bb.0:
-; ZVFHMIN-NEXT:    fmv.x.h a1, fa0
+; ZVFHMIN-NEXT:    fcvt.s.h fa5, fa0
 ; ZVFHMIN-NEXT:    vsetvli zero, a0, e16, mf4, ta, ma
 ; ZVFHMIN-NEXT:    vfwcvt.f.f.v v9, v8
-; ZVFHMIN-NEXT:    vmv.v.x v8, a1
-; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, mf2, ta, ma
+; ZVFHMIN-NEXT:    vsetvli a1, zero, e32, mf2, ta, ma
+; ZVFHMIN-NEXT:    vfmv.v.f v8, fa5
+; ZVFHMIN-NEXT:    vsetvli zero, a0, e32, mf2, ta, ma
+; ZVFHMIN-NEXT:    vmfne.vf v8, v8, fa5, v0.t
 ; ZVFHMIN-NEXT:    vmfne.vv v9, v9, v9, v0.t
-; ZVFHMIN-NEXT:    vsetvli zero, zero, e16, mf4, ta, ma
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v10, v8
-; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, mf2, ta, ma
-; ZVFHMIN-NEXT:    vmfne.vv v8, v10, v10, v0.t
 ; ZVFHMIN-NEXT:    vmor.mm v0, v9, v8
 ; ZVFHMIN-NEXT:    ret
   %elt.head = insertelement <vscale x 1 x half> poison, half %b, i32 0
@@ -2629,16 +2467,14 @@ define <vscale x 1 x i1> @fcmp_uno_vf_swap_nxv1f16(<vscale x 1 x half> %va, half
 ;
 ; ZVFHMIN-LABEL: fcmp_uno_vf_swap_nxv1f16:
 ; ZVFHMIN:       # %bb.0:
-; ZVFHMIN-NEXT:    fmv.x.h a1, fa0
+; ZVFHMIN-NEXT:    fcvt.s.h fa5, fa0
 ; ZVFHMIN-NEXT:    vsetvli zero, a0, e16, mf4, ta, ma
 ; ZVFHMIN-NEXT:    vfwcvt.f.f.v v9, v8
-; ZVFHMIN-NEXT:    vmv.v.x v8, a1
-; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, mf2, ta, ma
+; ZVFHMIN-NEXT:    vsetvli a1, zero, e32, mf2, ta, ma
+; ZVFHMIN-NEXT:    vfmv.v.f v8, fa5
+; ZVFHMIN-NEXT:    vsetvli zero, a0, e32, mf2, ta, ma
+; ZVFHMIN-NEXT:    vmfne.vf v8, v8, fa5, v0.t
 ; ZVFHMIN-NEXT:    vmfne.vv v9, v9, v9, v0.t
-; ZVFHMIN-NEXT:    vsetvli zero, zero, e16, mf4, ta, ma
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v10, v8
-; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, mf2, ta, ma
-; ZVFHMIN-NEXT:    vmfne.vv v8, v10, v10, v0.t
 ; ZVFHMIN-NEXT:    vmor.mm v0, v8, v9
 ; ZVFHMIN-NEXT:    ret
   %elt.head = insertelement <vscale x 1 x half> poison, half %b, i32 0
@@ -2702,13 +2538,11 @@ define <vscale x 8 x i1> @fcmp_oeq_vf_nxv8f16(<vscale x 8 x half> %va, half %b,
 ;
 ; ZVFHMIN-LABEL: fcmp_oeq_vf_nxv8f16:
 ; ZVFHMIN:       # %bb.0:
-; ZVFHMIN-NEXT:    fmv.x.h a1, fa0
+; ZVFHMIN-NEXT:    fcvt.s.h fa5, fa0
 ; ZVFHMIN-NEXT:    vsetvli zero, a0, e16, m2, ta, ma
-; ZVFHMIN-NEXT:    vmv.v.x v10, a1
 ; ZVFHMIN-NEXT:    vfwcvt.f.f.v v12, v8
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v16, v10
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
-; ZVFHMIN-NEXT:    vmfeq.vv v8, v12, v16, v0.t
+; ZVFHMIN-NEXT:    vmfeq.vf v8, v12, fa5, v0.t
 ; ZVFHMIN-NEXT:    vmv1r.v v0, v8
 ; ZVFHMIN-NEXT:    ret
   %elt.head = insertelement <vscale x 8 x half> poison, half %b, i32 0
@@ -2727,13 +2561,11 @@ define <vscale x 8 x i1> @fcmp_oeq_vf_swap_nxv8f16(<vscale x 8 x half> %va, half
 ;
 ; ZVFHMIN-LABEL: fcmp_oeq_vf_swap_nxv8f16:
 ; ZVFHMIN:       # %bb.0:
-; ZVFHMIN-NEXT:    fmv.x.h a1, fa0
+; ZVFHMIN-NEXT:    fcvt.s.h fa5, fa0
 ; ZVFHMIN-NEXT:    vsetvli zero, a0, e16, m2, ta, ma
-; ZVFHMIN-NEXT:    vmv.v.x v10, a1
 ; ZVFHMIN-NEXT:    vfwcvt.f.f.v v12, v8
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v16, v10
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
-; ZVFHMIN-NEXT:    vmfeq.vv v8, v16, v12, v0.t
+; ZVFHMIN-NEXT:    vmfeq.vf v8, v12, fa5, v0.t
 ; ZVFHMIN-NEXT:    vmv1r.v v0, v8
 ; ZVFHMIN-NEXT:    ret
   %elt.head = insertelement <vscale x 8 x half> poison, half %b, i32 0
@@ -2773,13 +2605,11 @@ define <vscale x 8 x i1> @fcmp_ogt_vf_nxv8f16(<vscale x 8 x half> %va, half %b,
 ;
 ; ZVFHMIN-LABEL: fcmp_ogt_vf_nxv8f16:
 ; ZVFHMIN:       # %bb.0:
-; ZVFHMIN-NEXT:    fmv.x.h a1, fa0
+; ZVFHMIN-NEXT:    fcvt.s.h fa5, fa0
 ; ZVFHMIN-NEXT:    vsetvli zero, a0, e16, m2, ta, ma
-; ZVFHMIN-NEXT:    vmv.v.x v10, a1
 ; ZVFHMIN-NEXT:    vfwcvt.f.f.v v12, v8
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v16, v10
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
-; ZVFHMIN-NEXT:    vmflt.vv v8, v16, v12, v0.t
+; ZVFHMIN-NEXT:    vmfgt.vf v8, v12, fa5, v0.t
 ; ZVFHMIN-NEXT:    vmv1r.v v0, v8
 ; ZVFHMIN-NEXT:    ret
   %elt.head = insertelement <vscale x 8 x half> poison, half %b, i32 0
@@ -2798,13 +2628,11 @@ define <vscale x 8 x i1> @fcmp_ogt_vf_swap_nxv8f16(<vscale x 8 x half> %va, half
 ;
 ; ZVFHMIN-LABEL: fcmp_ogt_vf_swap_nxv8f16:
 ; ZVFHMIN:       # %bb.0:
-; ZVFHMIN-NEXT:    fmv.x.h a1, fa0
+; ZVFHMIN-NEXT:    fcvt.s.h fa5, fa0
 ; ZVFHMIN-NEXT:    vsetvli zero, a0, e16, m2, ta, ma
-; ZVFHMIN-NEXT:    vmv.v.x v10, a1
 ; ZVFHMIN-NEXT:    vfwcvt.f.f.v v12, v8
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v16, v10
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
-; ZVFHMIN-NEXT:    vmflt.vv v8, v12, v16, v0.t
+; ZVFHMIN-NEXT:    vmflt.vf v8, v12, fa5, v0.t
 ; ZVFHMIN-NEXT:    vmv1r.v v0, v8
 ; ZVFHMIN-NEXT:    ret
   %elt.head = insertelement <vscale x 8 x half> poison, half %b, i32 0
@@ -2844,13 +2672,11 @@ define <vscale x 8 x i1> @fcmp_oge_vf_nxv8f16(<vscale x 8 x half> %va, half %b,
 ;
 ; ZVFHMIN-LABEL: fcmp_oge_vf_nxv8f16:
 ; ZVFHMIN:       # %bb.0:
-; ZVFHMIN-NEXT:    fmv.x.h a1, fa0
+; ZVFHMIN-NEXT:    fcvt.s.h fa5, fa0
 ; ZVFHMIN-NEXT:    vsetvli zero, a0, e16, m2, ta, ma
-; ZVFHMIN-NEXT:    vmv.v.x v10, a1
 ; ZVFHMIN-NEXT:    vfwcvt.f.f.v v12, v8
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v16, v10
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
-; ZVFHMIN-NEXT:    vmfle.vv v8, v16, v12, v0.t
+; ZVFHMIN-NEXT:    vmfge.vf v8, v12, fa5, v0.t
 ; ZVFHMIN-NEXT:    vmv1r.v v0, v8
 ; ZVFHMIN-NEXT:    ret
   %elt.head = insertelement <vscale x 8 x half> poison, half %b, i32 0
@@ -2869,13 +2695,11 @@ define <vscale x 8 x i1> @fcmp_oge_vf_swap_nxv8f16(<vscale x 8 x half> %va, half
 ;
 ; ZVFHMIN-LABEL: fcmp_oge_vf_swap_nxv8f16:
 ; ZVFHMIN:       # %bb.0:
-; ZVFHMIN-NEXT:    fmv.x.h a1, fa0
+; ZVFHMIN-NEXT:    fcvt.s.h fa5, fa0
 ; ZVFHMIN-NEXT:    vsetvli zero, a0, e16, m2, ta, ma
-; ZVFHMIN-NEXT:    vmv.v.x v10, a1
 ; ZVFHMIN-NEXT:    vfwcvt.f.f.v v12, v8
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v16, v10
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
-; ZVFHMIN-NEXT:    vmfle.vv v8, v12, v16, v0.t
+; ZVFHMIN-NEXT:    vmfle.vf v8, v12, fa5, v0.t
 ; ZVFHMIN-NEXT:    vmv1r.v v0, v8
 ; ZVFHMIN-NEXT:    ret
   %elt.head = insertelement <vscale x 8 x half> poison, half %b, i32 0
@@ -2915,13 +2739,11 @@ define <vscale x 8 x i1> @fcmp_olt_vf_nxv8f16(<vscale x 8 x half> %va, half %b,
 ;
 ; ZVFHMIN-LABEL: fcmp_olt_vf_nxv8f16:
 ; ZVFHMIN:       # %bb.0:
-; ZVFHMIN-NEXT:    fmv.x.h a1, fa0
+; ZVFHMIN-NEXT:    fcvt.s.h fa5, fa0
 ; ZVFHMIN-NEXT:    vsetvli zero, a0, e16, m2, ta, ma
-; ZVFHMIN-NEXT:    vmv.v.x v10, a1
 ; ZVFHMIN-NEXT:    vfwcvt.f.f.v v12, v8
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v16, v10
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
-; ZVFHMIN-NEXT:    vmflt.vv v8, v12, v16, v0.t
+; ZVFHMIN-NEXT:    vmflt.vf v8, v12, fa5, v0.t
 ; ZVFHMIN-NEXT:    vmv1r.v v0, v8
 ; ZVFHMIN-NEXT:    ret
   %elt.head = insertelement <vscale x 8 x half> poison, half %b, i32 0
@@ -2940,13 +2762,11 @@ define <vscale x 8 x i1> @fcmp_olt_vf_swap_nxv8f16(<vscale x 8 x half> %va, half
 ;
 ; ZVFHMIN-LABEL: fcmp_olt_vf_swap_nxv8f16:
 ; ZVFHMIN:       # %bb.0:
-; ZVFHMIN-NEXT:    fmv.x.h a1, fa0
+; ZVFHMIN-NEXT:    fcvt.s.h fa5, fa0
 ; ZVFHMIN-NEXT:    vsetvli zero, a0, e16, m2, ta, ma
-; ZVFHMIN-NEXT:    vmv.v.x v10, a1
 ; ZVFHMIN-NEXT:    vfwcvt.f.f.v v12, v8
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v16, v10
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
-; ZVFHMIN-NEXT:    vmflt.vv v8, v16, v12, v0.t
+; ZVFHMIN-NEXT:    vmfgt.vf v8, v12, fa5, v0.t
 ; ZVFHMIN-NEXT:    vmv1r.v v0, v8
 ; ZVFHMIN-NEXT:    ret
   %elt.head = insertelement <vscale x 8 x half> poison, half %b, i32 0
@@ -2986,13 +2806,11 @@ define <vscale x 8 x i1> @fcmp_ole_vf_nxv8f16(<vscale x 8 x half> %va, half %b,
 ;
 ; ZVFHMIN-LABEL: fcmp_ole_vf_nxv8f16:
 ; ZVFHMIN:       # %bb.0:
-; ZVFHMIN-NEXT:    fmv.x.h a1, fa0
+; ZVFHMIN-NEXT:    fcvt.s.h fa5, fa0
 ; ZVFHMIN-NEXT:    vsetvli zero, a0, e16, m2, ta, ma
-; ZVFHMIN-NEXT:    vmv.v.x v10, a1
 ; ZVFHMIN-NEXT:    vfwcvt.f.f.v v12, v8
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v16, v10
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
-; ZVFHMIN-NEXT:    vmfle.vv v8, v12, v16, v0.t
+; ZVFHMIN-NEXT:    vmfle.vf v8, v12, fa5, v0.t
 ; ZVFHMIN-NEXT:    vmv1r.v v0, v8
 ; ZVFHMIN-NEXT:    ret
   %elt.head = insertelement <vscale x 8 x half> poison, half %b, i32 0
@@ -3011,13 +2829,11 @@ define <vscale x 8 x i1> @fcmp_ole_vf_swap_nxv8f16(<vscale x 8 x half> %va, half
 ;
 ; ZVFHMIN-LABEL: fcmp_ole_vf_swap_nxv8f16:
 ; ZVFHMIN:       # %bb.0:
-; ZVFHMIN-NEXT:    fmv.x.h a1, fa0
+; ZVFHMIN-NEXT:    fcvt.s.h fa5, fa0
 ; ZVFHMIN-NEXT:    vsetvli zero, a0, e16, m2, ta, ma
-; ZVFHMIN-NEXT:    vmv.v.x v10, a1
 ; ZVFHMIN-NEXT:    vfwcvt.f.f.v v12, v8
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v16, v10
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
-; ZVFHMIN-NEXT:    vmfle.vv v8, v16, v12, v0.t
+; ZVFHMIN-NEXT:    vmfge.vf v8, v12, fa5, v0.t
 ; ZVFHMIN-NEXT:    vmv1r.v v0, v8
 ; ZVFHMIN-NEXT:    ret
   %elt.head = insertelement <vscale x 8 x half> poison, half %b, i32 0
@@ -3060,14 +2876,12 @@ define <vscale x 8 x i1> @fcmp_one_vf_nxv8f16(<vscale x 8 x half> %va, half %b,
 ;
 ; ZVFHMIN-LABEL: fcmp_one_vf_nxv8f16:
 ; ZVFHMIN:       # %bb.0:
-; ZVFHMIN-NEXT:    fmv.x.h a1, fa0
+; ZVFHMIN-NEXT:    fcvt.s.h fa5, fa0
 ; ZVFHMIN-NEXT:    vsetvli zero, a0, e16, m2, ta, ma
 ; ZVFHMIN-NEXT:    vfwcvt.f.f.v v12, v8
-; ZVFHMIN-NEXT:    vmv.v.x v8, a1
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v16, v8
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
-; ZVFHMIN-NEXT:    vmflt.vv v8, v12, v16, v0.t
-; ZVFHMIN-NEXT:    vmflt.vv v9, v16, v12, v0.t
+; ZVFHMIN-NEXT:    vmflt.vf v8, v12, fa5, v0.t
+; ZVFHMIN-NEXT:    vmfgt.vf v9, v12, fa5, v0.t
 ; ZVFHMIN-NEXT:    vmor.mm v0, v9, v8
 ; ZVFHMIN-NEXT:    ret
   %elt.head = insertelement <vscale x 8 x half> poison, half %b, i32 0
@@ -3087,14 +2901,12 @@ define <vscale x 8 x i1> @fcmp_one_vf_swap_nxv8f16(<vscale x 8 x half> %va, half
 ;
 ; ZVFHMIN-LABEL: fcmp_one_vf_swap_nxv8f16:
 ; ZVFHMIN:       # %bb.0:
-; ZVFHMIN-NEXT:    fmv.x.h a1, fa0
+; ZVFHMIN-NEXT:    fcvt.s.h fa5, fa0
 ; ZVFHMIN-NEXT:    vsetvli zero, a0, e16, m2, ta, ma
 ; ZVFHMIN-NEXT:    vfwcvt.f.f.v v12, v8
-; ZVFHMIN-NEXT:    vmv.v.x v8, a1
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v16, v8
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
-; ZVFHMIN-NEXT:    vmflt.vv v8, v16, v12, v0.t
-; ZVFHMIN-NEXT:    vmflt.vv v9, v12, v16, v0.t
+; ZVFHMIN-NEXT:    vmfgt.vf v8, v12, fa5, v0.t
+; ZVFHMIN-NEXT:    vmflt.vf v9, v12, fa5, v0.t
 ; ZVFHMIN-NEXT:    vmor.mm v0, v9, v8
 ; ZVFHMIN-NEXT:    ret
   %elt.head = insertelement <vscale x 8 x half> poison, half %b, i32 0
@@ -3141,17 +2953,16 @@ define <vscale x 8 x i1> @fcmp_ord_vf_nxv8f16(<vscale x 8 x half> %va, half %b,
 ;
 ; ZVFHMIN-LABEL: fcmp_ord_vf_nxv8f16:
 ; ZVFHMIN:       # %bb.0:
-; ZVFHMIN-NEXT:    fmv.x.h a1, fa0
-; ZVFHMIN-NEXT:    vsetvli zero, a0, e16, m2, ta, ma
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v12, v8
-; ZVFHMIN-NEXT:    vmv.v.x v8, a1
-; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
-; ZVFHMIN-NEXT:    vmfeq.vv v10, v12, v12, v0.t
+; ZVFHMIN-NEXT:    fcvt.s.h fa5, fa0
+; ZVFHMIN-NEXT:    vsetvli a1, zero, e32, m4, ta, ma
+; ZVFHMIN-NEXT:    vfmv.v.f v12, fa5
+; ZVFHMIN-NEXT:    vsetvli zero, a0, e32, m4, ta, ma
+; ZVFHMIN-NEXT:    vmfeq.vf v10, v12, fa5, v0.t
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e16, m2, ta, ma
 ; ZVFHMIN-NEXT:    vfwcvt.f.f.v v12, v8
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
 ; ZVFHMIN-NEXT:    vmfeq.vv v8, v12, v12, v0.t
-; ZVFHMIN-NEXT:    vmand.mm v0, v10, v8
+; ZVFHMIN-NEXT:    vmand.mm v0, v8, v10
 ; ZVFHMIN-NEXT:    ret
   %elt.head = insertelement <vscale x 8 x half> poison, half %b, i32 0
   %vb = shufflevector <vscale x 8 x half> %elt.head, <vscale x 8 x half> poison, <vscale x 8 x i32> zeroinitializer
@@ -3172,17 +2983,16 @@ define <vscale x 8 x i1> @fcmp_ord_vf_swap_nxv8f16(<vscale x 8 x half> %va, half
 ;
 ; ZVFHMIN-LABEL: fcmp_ord_vf_swap_nxv8f16:
 ; ZVFHMIN:       # %bb.0:
-; ZVFHMIN-NEXT:    fmv.x.h a1, fa0
-; ZVFHMIN-NEXT:    vsetvli zero, a0, e16, m2, ta, ma
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v12, v8
-; ZVFHMIN-NEXT:    vmv.v.x v8, a1
-; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
-; ZVFHMIN-NEXT:    vmfeq.vv v10, v12, v12, v0.t
+; ZVFHMIN-NEXT:    fcvt.s.h fa5, fa0
+; ZVFHMIN-NEXT:    vsetvli a1, zero, e32, m4, ta, ma
+; ZVFHMIN-NEXT:    vfmv.v.f v12, fa5
+; ZVFHMIN-NEXT:    vsetvli zero, a0, e32, m4, ta, ma
+; ZVFHMIN-NEXT:    vmfeq.vf v10, v12, fa5, v0.t
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e16, m2, ta, ma
 ; ZVFHMIN-NEXT:    vfwcvt.f.f.v v12, v8
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
 ; ZVFHMIN-NEXT:    vmfeq.vv v8, v12, v12, v0.t
-; ZVFHMIN-NEXT:    vmand.mm v0, v8, v10
+; ZVFHMIN-NEXT:    vmand.mm v0, v10, v8
 ; ZVFHMIN-NEXT:    ret
   %elt.head = insertelement <vscale x 8 x half> poison, half %b, i32 0
   %vb = shufflevector <vscale x 8 x half> %elt.head, <vscale x 8 x half> poison, <vscale x 8 x i32> zeroinitializer
@@ -3224,14 +3034,12 @@ define <vscale x 8 x i1> @fcmp_ueq_vf_nxv8f16(<vscale x 8 x half> %va, half %b,
 ;
 ; ZVFHMIN-LABEL: fcmp_ueq_vf_nxv8f16:
 ; ZVFHMIN:       # %bb.0:
-; ZVFHMIN-NEXT:    fmv.x.h a1, fa0
+; ZVFHMIN-NEXT:    fcvt.s.h fa5, fa0
 ; ZVFHMIN-NEXT:    vsetvli zero, a0, e16, m2, ta, ma
 ; ZVFHMIN-NEXT:    vfwcvt.f.f.v v12, v8
-; ZVFHMIN-NEXT:    vmv.v.x v8, a1
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v16, v8
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
-; ZVFHMIN-NEXT:    vmflt.vv v8, v12, v16, v0.t
-; ZVFHMIN-NEXT:    vmflt.vv v9, v16, v12, v0.t
+; ZVFHMIN-NEXT:    vmflt.vf v8, v12, fa5, v0.t
+; ZVFHMIN-NEXT:    vmfgt.vf v9, v12, fa5, v0.t
 ; ZVFHMIN-NEXT:    vmnor.mm v0, v9, v8
 ; ZVFHMIN-NEXT:    ret
   %elt.head = insertelement <vscale x 8 x half> poison, half %b, i32 0
@@ -3251,14 +3059,12 @@ define <vscale x 8 x i1> @fcmp_ueq_vf_swap_nxv8f16(<vscale x 8 x half> %va, half
 ;
 ; ZVFHMIN-LABEL: fcmp_ueq_vf_swap_nxv8f16:
 ; ZVFHMIN:       # %bb.0:
-; ZVFHMIN-NEXT:    fmv.x.h a1, fa0
+; ZVFHMIN-NEXT:    fcvt.s.h fa5, fa0
 ; ZVFHMIN-NEXT:    vsetvli zero, a0, e16, m2, ta, ma
 ; ZVFHMIN-NEXT:    vfwcvt.f.f.v v12, v8
-; ZVFHMIN-NEXT:    vmv.v.x v8, a1
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v16, v8
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
-; ZVFHMIN-NEXT:    vmflt.vv v8, v16, v12, v0.t
-; ZVFHMIN-NEXT:    vmflt.vv v9, v12, v16, v0.t
+; ZVFHMIN-NEXT:    vmfgt.vf v8, v12, fa5, v0.t
+; ZVFHMIN-NEXT:    vmflt.vf v9, v12, fa5, v0.t
 ; ZVFHMIN-NEXT:    vmnor.mm v0, v9, v8
 ; ZVFHMIN-NEXT:    ret
   %elt.head = insertelement <vscale x 8 x half> poison, half %b, i32 0
@@ -3298,13 +3104,11 @@ define <vscale x 8 x i1> @fcmp_ugt_vf_nxv8f16(<vscale x 8 x half> %va, half %b,
 ;
 ; ZVFHMIN-LABEL: fcmp_ugt_vf_nxv8f16:
 ; ZVFHMIN:       # %bb.0:
-; ZVFHMIN-NEXT:    fmv.x.h a1, fa0
+; ZVFHMIN-NEXT:    fcvt.s.h fa5, fa0
 ; ZVFHMIN-NEXT:    vsetvli zero, a0, e16, m2, ta, ma
-; ZVFHMIN-NEXT:    vmv.v.x v10, a1
 ; ZVFHMIN-NEXT:    vfwcvt.f.f.v v12, v8
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v16, v10
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
-; ZVFHMIN-NEXT:    vmfle.vv v8, v12, v16, v0.t
+; ZVFHMIN-NEXT:    vmfle.vf v8, v12, fa5, v0.t
 ; ZVFHMIN-NEXT:    vmnot.m v0, v8
 ; ZVFHMIN-NEXT:    ret
   %elt.head = insertelement <vscale x 8 x half> poison, half %b, i32 0
@@ -3323,13 +3127,11 @@ define <vscale x 8 x i1> @fcmp_ugt_vf_swap_nxv8f16(<vscale x 8 x half> %va, half
 ;
 ; ZVFHMIN-LABEL: fcmp_ugt_vf_swap_nxv8f16:
 ; ZVFHMIN:       # %bb.0:
-; ZVFHMIN-NEXT:    fmv.x.h a1, fa0
+; ZVFHMIN-NEXT:    fcvt.s.h fa5, fa0
 ; ZVFHMIN-NEXT:    vsetvli zero, a0, e16, m2, ta, ma
-; ZVFHMIN-NEXT:    vmv.v.x v10, a1
 ; ZVFHMIN-NEXT:    vfwcvt.f.f.v v12, v8
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v16, v10
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
-; ZVFHMIN-NEXT:    vmfle.vv v8, v16, v12, v0.t
+; ZVFHMIN-NEXT:    vmfge.vf v8, v12, fa5, v0.t
 ; ZVFHMIN-NEXT:    vmnot.m v0, v8
 ; ZVFHMIN-NEXT:    ret
   %elt.head = insertelement <vscale x 8 x half> poison, half %b, i32 0
@@ -3369,13 +3171,11 @@ define <vscale x 8 x i1> @fcmp_uge_vf_nxv8f16(<vscale x 8 x half> %va, half %b,
 ;
 ; ZVFHMIN-LABEL: fcmp_uge_vf_nxv8f16:
 ; ZVFHMIN:       # %bb.0:
-; ZVFHMIN-NEXT:    fmv.x.h a1, fa0
+; ZVFHMIN-NEXT:    fcvt.s.h fa5, fa0
 ; ZVFHMIN-NEXT:    vsetvli zero, a0, e16, m2, ta, ma
-; ZVFHMIN-NEXT:    vmv.v.x v10, a1
 ; ZVFHMIN-NEXT:    vfwcvt.f.f.v v12, v8
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v16, v10
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
-; ZVFHMIN-NEXT:    vmflt.vv v8, v12, v16, v0.t
+; ZVFHMIN-NEXT:    vmflt.vf v8, v12, fa5, v0.t
 ; ZVFHMIN-NEXT:    vmnot.m v0, v8
 ; ZVFHMIN-NEXT:    ret
   %elt.head = insertelement <vscale x 8 x half> poison, half %b, i32 0
@@ -3394,13 +3194,11 @@ define <vscale x 8 x i1> @fcmp_uge_vf_swap_nxv8f16(<vscale x 8 x half> %va, half
 ;
 ; ZVFHMIN-LABEL: fcmp_uge_vf_swap_nxv8f16:
 ; ZVFHMIN:       # %bb.0:
-; ZVFHMIN-NEXT:    fmv.x.h a1, fa0
+; ZVFHMIN-NEXT:    fcvt.s.h fa5, fa0
 ; ZVFHMIN-NEXT:    vsetvli zero, a0, e16, m2, ta, ma
-; ZVFHMIN-NEXT:    vmv.v.x v10, a1
 ; ZVFHMIN-NEXT:    vfwcvt.f.f.v v12, v8
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v16, v10
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
-; ZVFHMIN-NEXT:    vmflt.vv v8, v16, v12, v0.t
+; ZVFHMIN-NEXT:    vmfgt.vf v8, v12, fa5, v0.t
 ; ZVFHMIN-NEXT:    vmnot.m v0, v8
 ; ZVFHMIN-NEXT:    ret
   %elt.head = insertelement <vscale x 8 x half> poison, half %b, i32 0
@@ -3440,13 +3238,11 @@ define <vscale x 8 x i1> @fcmp_ult_vf_nxv8f16(<vscale x 8 x half> %va, half %b,
 ;
 ; ZVFHMIN-LABEL: fcmp_ult_vf_nxv8f16:
 ; ZVFHMIN:       # %bb.0:
-; ZVFHMIN-NEXT:    fmv.x.h a1, fa0
+; ZVFHMIN-NEXT:    fcvt.s.h fa5, fa0
 ; ZVFHMIN-NEXT:    vsetvli zero, a0, e16, m2, ta, ma
-; ZVFHMIN-NEXT:    vmv.v.x v10, a1
 ; ZVFHMIN-NEXT:    vfwcvt.f.f.v v12, v8
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v16, v10
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
-; ZVFHMIN-NEXT:    vmfle.vv v8, v16, v12, v0.t
+; ZVFHMIN-NEXT:    vmfge.vf v8, v12, fa5, v0.t
 ; ZVFHMIN-NEXT:    vmnot.m v0, v8
 ; ZVFHMIN-NEXT:    ret
   %elt.head = insertelement <vscale x 8 x half> poison, half %b, i32 0
@@ -3465,13 +3261,11 @@ define <vscale x 8 x i1> @fcmp_ult_vf_swap_nxv8f16(<vscale x 8 x half> %va, half
 ;
 ; ZVFHMIN-LABEL: fcmp_ult_vf_swap_nxv8f16:
 ; ZVFHMIN:       # %bb.0:
-; ZVFHMIN-NEXT:    fmv.x.h a1, fa0
+; ZVFHMIN-NEXT:    fcvt.s.h fa5, fa0
 ; ZVFHMIN-NEXT:    vsetvli zero, a0, e16, m2, ta, ma
-; ZVFHMIN-NEXT:    vmv.v.x v10, a1
 ; ZVFHMIN-NEXT:    vfwcvt.f.f.v v12, v8
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v16, v10
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
-; ZVFHMIN-NEXT:    vmfle.vv v8, v12, v16, v0.t
+; ZVFHMIN-NEXT:    vmfle.vf v8, v12, fa5, v0.t
 ; ZVFHMIN-NEXT:    vmnot.m v0, v8
 ; ZVFHMIN-NEXT:    ret
   %elt.head = insertelement <vscale x 8 x half> poison, half %b, i32 0
@@ -3511,13 +3305,11 @@ define <vscale x 8 x i1> @fcmp_ule_vf_nxv8f16(<vscale x 8 x half> %va, half %b,
 ;
 ; ZVFHMIN-LABEL: fcmp_ule_vf_nxv8f16:
 ; ZVFHMIN:       # %bb.0:
-; ZVFHMIN-NEXT:    fmv.x.h a1, fa0
+; ZVFHMIN-NEXT:    fcvt.s.h fa5, fa0
 ; ZVFHMIN-NEXT:    vsetvli zero, a0, e16, m2, ta, ma
-; ZVFHMIN-NEXT:    vmv.v.x v10, a1
 ; ZVFHMIN-NEXT:    vfwcvt.f.f.v v12, v8
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v16, v10
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
-; ZVFHMIN-NEXT:    vmflt.vv v8, v16, v12, v0.t
+; ZVFHMIN-NEXT:    vmfgt.vf v8, v12, fa5, v0.t
 ; ZVFHMIN-NEXT:    vmnot.m v0, v8
 ; ZVFHMIN-NEXT:    ret
   %elt.head = insertelement <vscale x 8 x half> poison, half %b, i32 0
@@ -3536,13 +3328,11 @@ define <vscale x 8 x i1> @fcmp_ule_vf_swap_nxv8f16(<vscale x 8 x half> %va, half
 ;
 ; ZVFHMIN-LABEL: fcmp_ule_vf_swap_nxv8f16:
 ; ZVFHMIN:       # %bb.0:
-; ZVFHMIN-NEXT:    fmv.x.h a1, fa0
+; ZVFHMIN-NEXT:    fcvt.s.h fa5, fa0
 ; ZVFHMIN-NEXT:    vsetvli zero, a0, e16, m2, ta, ma
-; ZVFHMIN-NEXT:    vmv.v.x v10, a1
 ; ZVFHMIN-NEXT:    vfwcvt.f.f.v v12, v8
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v16, v10
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
-; ZVFHMIN-NEXT:    vmflt.vv v8, v12, v16, v0.t
+; ZVFHMIN-NEXT:    vmflt.vf v8, v12, fa5, v0.t
 ; ZVFHMIN-NEXT:    vmnot.m v0, v8
 ; ZVFHMIN-NEXT:    ret
   %elt.head = insertelement <vscale x 8 x half> poison, half %b, i32 0
@@ -3582,13 +3372,11 @@ define <vscale x 8 x i1> @fcmp_une_vf_nxv8f16(<vscale x 8 x half> %va, half %b,
 ;
 ; ZVFHMIN-LABEL: fcmp_une_vf_nxv8f16:
 ; ZVFHMIN:       # %bb.0:
-; ZVFHMIN-NEXT:    fmv.x.h a1, fa0
+; ZVFHMIN-NEXT:    fcvt.s.h fa5, fa0
 ; ZVFHMIN-NEXT:    vsetvli zero, a0, e16, m2, ta, ma
-; ZVFHMIN-NEXT:    vmv.v.x v10, a1
 ; ZVFHMIN-NEXT:    vfwcvt.f.f.v v12, v8
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v16, v10
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
-; ZVFHMIN-NEXT:    vmfne.vv v8, v12, v16, v0.t
+; ZVFHMIN-NEXT:    vmfne.vf v8, v12, fa5, v0.t
 ; ZVFHMIN-NEXT:    vmv1r.v v0, v8
 ; ZVFHMIN-NEXT:    ret
   %elt.head = insertelement <vscale x 8 x half> poison, half %b, i32 0
@@ -3607,13 +3395,11 @@ define <vscale x 8 x i1> @fcmp_une_vf_swap_nxv8f16(<vscale x 8 x half> %va, half
 ;
 ; ZVFHMIN-LABEL: fcmp_une_vf_swap_nxv8f16:
 ; ZVFHMIN:       # %bb.0:
-; ZVFHMIN-NEXT:    fmv.x.h a1, fa0
+; ZVFHMIN-NEXT:    fcvt.s.h fa5, fa0
 ; ZVFHMIN-NEXT:    vsetvli zero, a0, e16, m2, ta, ma
-; ZVFHMIN-NEXT:    vmv.v.x v10, a1
 ; ZVFHMIN-NEXT:    vfwcvt.f.f.v v12, v8
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v16, v10
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
-; ZVFHMIN-NEXT:    vmfne.vv v8, v16, v12, v0.t
+; ZVFHMIN-NEXT:    vmfne.vf v8, v12, fa5, v0.t
 ; ZVFHMIN-NEXT:    vmv1r.v v0, v8
 ; ZVFHMIN-NEXT:    ret
   %elt.head = insertelement <vscale x 8 x half> poison, half %b, i32 0
@@ -3660,17 +3446,16 @@ define <vscale x 8 x i1> @fcmp_uno_vf_nxv8f16(<vscale x 8 x half> %va, half %b,
 ;
 ; ZVFHMIN-LABEL: fcmp_uno_vf_nxv8f16:
 ; ZVFHMIN:       # %bb.0:
-; ZVFHMIN-NEXT:    fmv.x.h a1, fa0
-; ZVFHMIN-NEXT:    vsetvli zero, a0, e16, m2, ta, ma
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v12, v8
-; ZVFHMIN-NEXT:    vmv.v.x v8, a1
-; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
-; ZVFHMIN-NEXT:    vmfne.vv v10, v12, v12, v0.t
+; ZVFHMIN-NEXT:    fcvt.s.h fa5, fa0
+; ZVFHMIN-NEXT:    vsetvli a1, zero, e32, m4, ta, ma
+; ZVFHMIN-NEXT:    vfmv.v.f v12, fa5
+; ZVFHMIN-NEXT:    vsetvli zero, a0, e32, m4, ta, ma
+; ZVFHMIN-NEXT:    vmfne.vf v10, v12, fa5, v0.t
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e16, m2, ta, ma
 ; ZVFHMIN-NEXT:    vfwcvt.f.f.v v12, v8
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
 ; ZVFHMIN-NEXT:    vmfne.vv v8, v12, v12, v0.t
-; ZVFHMIN-NEXT:    vmor.mm v0, v10, v8
+; ZVFHMIN-NEXT:    vmor.mm v0, v8, v10
 ; ZVFHMIN-NEXT:    ret
   %elt.head = insertelement <vscale x 8 x half> poison, half %b, i32 0
   %vb = shufflevector <vscale x 8 x half> %elt.head, <vscale x 8 x half> poison, <vscale x 8 x i32> zeroinitializer
@@ -3691,17 +3476,16 @@ define <vscale x 8 x i1> @fcmp_uno_vf_swap_nxv8f16(<vscale x 8 x half> %va, half
 ;
 ; ZVFHMIN-LABEL: fcmp_uno_vf_swap_nxv8f16:
 ; ZVFHMIN:       # %bb.0:
-; ZVFHMIN-NEXT:    fmv.x.h a1, fa0
-; ZVFHMIN-NEXT:    vsetvli zero, a0, e16, m2, ta, ma
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v12, v8
-; ZVFHMIN-NEXT:    vmv.v.x v8, a1
-; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
-; ZVFHMIN-NEXT:    vmfne.vv v10, v12, v12, v0.t
+; ZVFHMIN-NEXT:    fcvt.s.h fa5, fa0
+; ZVFHMIN-NEXT:    vsetvli a1, zero, e32, m4, ta, ma
+; ZVFHMIN-NEXT:    vfmv.v.f v12, fa5
+; ZVFHMIN-NEXT:    vsetvli zero, a0, e32, m4, ta, ma
+; ZVFHMIN-NEXT:    vmfne.vf v10, v12, fa5, v0.t
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e16, m2, ta, ma
 ; ZVFHMIN-NEXT:    vfwcvt.f.f.v v12, v8
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
 ; ZVFHMIN-NEXT:    vmfne.vv v8, v12, v12, v0.t
-; ZVFHMIN-NEXT:    vmor.mm v0, v8, v10
+; ZVFHMIN-NEXT:    vmor.mm v0, v10, v8
 ; ZVFHMIN-NEXT:    ret
   %elt.head = insertelement <vscale x 8 x half> poison, half %b, i32 0
   %vb = shufflevector <vscale x 8 x half> %elt.head, <vscale x 8 x half> poison, <vscale x 8 x i32> zeroinitializer
diff --git a/llvm/test/CodeGen/RISCV/rvv/setcc-fp.ll b/llvm/test/CodeGen/RISCV/rvv/setcc-fp.ll
index 7ef46a7372e22..ae868fed68cab 100644
--- a/llvm/test/CodeGen/RISCV/rvv/setcc-fp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/setcc-fp.ll
@@ -31,13 +31,11 @@ define <vscale x 8 x i1> @fcmp_oeq_vv_nxv8bf16(<vscale x 8 x bfloat> %va, <vscal
 define <vscale x 8 x i1> @fcmp_oeq_vf_nxv8bf16(<vscale x 8 x bfloat> %va, bfloat %b) {
 ; CHECK-LABEL: fcmp_oeq_vf_nxv8bf16:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    fmv.x.h a0, fa0
-; CHECK-NEXT:    vsetvli a1, zero, e16, m2, ta, ma
-; CHECK-NEXT:    vmv.v.x v10, a0
+; CHECK-NEXT:    fcvt.s.bf16 fa5, fa0
+; CHECK-NEXT:    vsetvli a0, zero, e16, m2, ta, ma
 ; CHECK-NEXT:    vfwcvtbf16.f.f.v v12, v8
-; CHECK-NEXT:    vfwcvtbf16.f.f.v v16, v10
 ; CHECK-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
-; CHECK-NEXT:    vmfeq.vv v0, v12, v16
+; CHECK-NEXT:    vmfeq.vf v0, v12, fa5
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 8 x bfloat> poison, bfloat %b, i32 0
   %splat = shufflevector <vscale x 8 x bfloat> %head, <vscale x 8 x bfloat> poison, <vscale x 8 x i32> zeroinitializer
@@ -48,13 +46,11 @@ define <vscale x 8 x i1> @fcmp_oeq_vf_nxv8bf16(<vscale x 8 x bfloat> %va, bfloat
 define <vscale x 8 x i1> @fcmp_oeq_fv_nxv8bf16(<vscale x 8 x bfloat> %va, bfloat %b) {
 ; CHECK-LABEL: fcmp_oeq_fv_nxv8bf16:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    fmv.x.h a0, fa0
-; CHECK-NEXT:    vsetvli a1, zero, e16, m2, ta, ma
-; CHECK-NEXT:    vmv.v.x v10, a0
+; CHECK-NEXT:    fcvt.s.bf16 fa5, fa0
+; CHECK-NEXT:    vsetvli a0, zero, e16, m2, ta, ma
 ; CHECK-NEXT:    vfwcvtbf16.f.f.v v12, v8
-; CHECK-NEXT:    vfwcvtbf16.f.f.v v16, v10
 ; CHECK-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
-; CHECK-NEXT:    vmfeq.vv v0, v16, v12
+; CHECK-NEXT:    vmfeq.vf v0, v12, fa5
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 8 x bfloat> poison, bfloat %b, i32 0
   %splat = shufflevector <vscale x 8 x bfloat> %head, <vscale x 8 x bfloat> poison, <vscale x 8 x i32> zeroinitializer
@@ -78,13 +74,11 @@ define <vscale x 8 x i1> @fcmp_oeq_vv_nxv8bf16_nonans(<vscale x 8 x bfloat> %va,
 define <vscale x 8 x i1> @fcmp_oeq_vf_nxv8bf16_nonans(<vscale x 8 x bfloat> %va, bfloat %b) #0 {
 ; CHECK-LABEL: fcmp_oeq_vf_nxv8bf16_nonans:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    fmv.x.h a0, fa0
-; CHECK-NEXT:    vsetvli a1, zero, e16, m2, ta, ma
-; CHECK-NEXT:    vmv.v.x v10, a0
+; CHECK-NEXT:    fcvt.s.bf16 fa5, fa0
+; CHECK-NEXT:    vsetvli a0, zero, e16, m2, ta, ma
 ; CHECK-NEXT:    vfwcvtbf16.f.f.v v12, v8
-; CHECK-NEXT:    vfwcvtbf16.f.f.v v16, v10
 ; CHECK-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
-; CHECK-NEXT:    vmfeq.vv v0, v12, v16
+; CHECK-NEXT:    vmfeq.vf v0, v12, fa5
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 8 x bfloat> poison, bfloat %b, i32 0
   %splat = shufflevector <vscale x 8 x bfloat> %head, <vscale x 8 x bfloat> poison, <vscale x 8 x i32> zeroinitializer
@@ -108,13 +102,11 @@ define <vscale x 8 x i1> @fcmp_ogt_vv_nxv8bf16(<vscale x 8 x bfloat> %va, <vscal
 define <vscale x 8 x i1> @fcmp_ogt_vf_nxv8bf16(<vscale x 8 x bfloat> %va, bfloat %b) {
 ; CHECK-LABEL: fcmp_ogt_vf_nxv8bf16:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    fmv.x.h a0, fa0
-; CHECK-NEXT:    vsetvli a1, zero, e16, m2, ta, ma
-; CHECK-NEXT:    vmv.v.x v10, a0
+; CHECK-NEXT:    fcvt.s.bf16 fa5, fa0
+; CHECK-NEXT:    vsetvli a0, zero, e16, m2, ta, ma
 ; CHECK-NEXT:    vfwcvtbf16.f.f.v v12, v8
-; CHECK-NEXT:    vfwcvtbf16.f.f.v v16, v10
 ; CHECK-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
-; CHECK-NEXT:    vmflt.vv v0, v16, v12
+; CHECK-NEXT:    vmfgt.vf v0, v12, fa5
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 8 x bfloat> poison, bfloat %b, i32 0
   %splat = shufflevector <vscale x 8 x bfloat> %head, <vscale x 8 x bfloat> poison, <vscale x 8 x i32> zeroinitializer
@@ -125,13 +117,11 @@ define <vscale x 8 x i1> @fcmp_ogt_vf_nxv8bf16(<vscale x 8 x bfloat> %va, bfloat
 define <vscale x 8 x i1> @fcmp_ogt_fv_nxv8bf16(<vscale x 8 x bfloat> %va, bfloat %b) {
 ; CHECK-LABEL: fcmp_ogt_fv_nxv8bf16:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    fmv.x.h a0, fa0
-; CHECK-NEXT:    vsetvli a1, zero, e16, m2, ta, ma
-; CHECK-NEXT:    vmv.v.x v10, a0
+; CHECK-NEXT:    fcvt.s.bf16 fa5, fa0
+; CHECK-NEXT:    vsetvli a0, zero, e16, m2, ta, ma
 ; CHECK-NEXT:    vfwcvtbf16.f.f.v v12, v8
-; CHECK-NEXT:    vfwcvtbf16.f.f.v v16, v10
 ; CHECK-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
-; CHECK-NEXT:    vmflt.vv v0, v12, v16
+; CHECK-NEXT:    vmflt.vf v0, v12, fa5
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 8 x bfloat> poison, bfloat %b, i32 0
   %splat = shufflevector <vscale x 8 x bfloat> %head, <vscale x 8 x bfloat> poison, <vscale x 8 x i32> zeroinitializer
@@ -155,13 +145,11 @@ define <vscale x 8 x i1> @fcmp_ogt_vv_nxv8bf16_nonans(<vscale x 8 x bfloat> %va,
 define <vscale x 8 x i1> @fcmp_ogt_vf_nxv8bf16_nonans(<vscale x 8 x bfloat> %va, bfloat %b) #0 {
 ; CHECK-LABEL: fcmp_ogt_vf_nxv8bf16_nonans:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    fmv.x.h a0, fa0
-; CHECK-NEXT:    vsetvli a1, zero, e16, m2, ta, ma
-; CHECK-NEXT:    vmv.v.x v10, a0
+; CHECK-NEXT:    fcvt.s.bf16 fa5, fa0
+; CHECK-NEXT:    vsetvli a0, zero, e16, m2, ta, ma
 ; CHECK-NEXT:    vfwcvtbf16.f.f.v v12, v8
-; CHECK-NEXT:    vfwcvtbf16.f.f.v v16, v10
 ; CHECK-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
-; CHECK-NEXT:    vmflt.vv v0, v16, v12
+; CHECK-NEXT:    vmfgt.vf v0, v12, fa5
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 8 x bfloat> poison, bfloat %b, i32 0
   %splat = shufflevector <vscale x 8 x bfloat> %head, <vscale x 8 x bfloat> poison, <vscale x 8 x i32> zeroinitializer
@@ -185,13 +173,11 @@ define <vscale x 8 x i1> @fcmp_oge_vv_nxv8bf16(<vscale x 8 x bfloat> %va, <vscal
 define <vscale x 8 x i1> @fcmp_oge_vf_nxv8bf16(<vscale x 8 x bfloat> %va, bfloat %b) {
 ; CHECK-LABEL: fcmp_oge_vf_nxv8bf16:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    fmv.x.h a0, fa0
-; CHECK-NEXT:    vsetvli a1, zero, e16, m2, ta, ma
-; CHECK-NEXT:    vmv.v.x v10, a0
+; CHECK-NEXT:    fcvt.s.bf16 fa5, fa0
+; CHECK-NEXT:    vsetvli a0, zero, e16, m2, ta, ma
 ; CHECK-NEXT:    vfwcvtbf16.f.f.v v12, v8
-; CHECK-NEXT:    vfwcvtbf16.f.f.v v16, v10
 ; CHECK-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
-; CHECK-NEXT:    vmfle.vv v0, v16, v12
+; CHECK-NEXT:    vmfge.vf v0, v12, fa5
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 8 x bfloat> poison, bfloat %b, i32 0
   %splat = shufflevector <vscale x 8 x bfloat> %head, <vscale x 8 x bfloat> poison, <vscale x 8 x i32> zeroinitializer
@@ -202,13 +188,11 @@ define <vscale x 8 x i1> @fcmp_oge_vf_nxv8bf16(<vscale x 8 x bfloat> %va, bfloat
 define <vscale x 8 x i1> @fcmp_oge_fv_nxv8bf16(<vscale x 8 x bfloat> %va, bfloat %b) {
 ; CHECK-LABEL: fcmp_oge_fv_nxv8bf16:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    fmv.x.h a0, fa0
-; CHECK-NEXT:    vsetvli a1, zero, e16, m2, ta, ma
-; CHECK-NEXT:    vmv.v.x v10, a0
+; CHECK-NEXT:    fcvt.s.bf16 fa5, fa0
+; CHECK-NEXT:    vsetvli a0, zero, e16, m2, ta, ma
 ; CHECK-NEXT:    vfwcvtbf16.f.f.v v12, v8
-; CHECK-NEXT:    vfwcvtbf16.f.f.v v16, v10
 ; CHECK-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
-; CHECK-NEXT:    vmfle.vv v0, v12, v16
+; CHECK-NEXT:    vmfle.vf v0, v12, fa5
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 8 x bfloat> poison, bfloat %b, i32 0
   %splat = shufflevector <vscale x 8 x bfloat> %head, <vscale x 8 x bfloat> poison, <vscale x 8 x i32> zeroinitializer
@@ -232,13 +216,11 @@ define <vscale x 8 x i1> @fcmp_oge_vv_nxv8bf16_nonans(<vscale x 8 x bfloat> %va,
 define <vscale x 8 x i1> @fcmp_oge_vf_nxv8bf16_nonans(<vscale x 8 x bfloat> %va, bfloat %b) #0 {
 ; CHECK-LABEL: fcmp_oge_vf_nxv8bf16_nonans:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    fmv.x.h a0, fa0
-; CHECK-NEXT:    vsetvli a1, zero, e16, m2, ta, ma
-; CHECK-NEXT:    vmv.v.x v10, a0
+; CHECK-NEXT:    fcvt.s.bf16 fa5, fa0
+; CHECK-NEXT:    vsetvli a0, zero, e16, m2, ta, ma
 ; CHECK-NEXT:    vfwcvtbf16.f.f.v v12, v8
-; CHECK-NEXT:    vfwcvtbf16.f.f.v v16, v10
 ; CHECK-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
-; CHECK-NEXT:    vmfle.vv v0, v16, v12
+; CHECK-NEXT:    vmfge.vf v0, v12, fa5
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 8 x bfloat> poison, bfloat %b, i32 0
   %splat = shufflevector <vscale x 8 x bfloat> %head, <vscale x 8 x bfloat> poison, <vscale x 8 x i32> zeroinitializer
@@ -262,13 +244,11 @@ define <vscale x 8 x i1> @fcmp_olt_vv_nxv8bf16(<vscale x 8 x bfloat> %va, <vscal
 define <vscale x 8 x i1> @fcmp_olt_vf_nxv8bf16(<vscale x 8 x bfloat> %va, bfloat %b) {
 ; CHECK-LABEL: fcmp_olt_vf_nxv8bf16:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    fmv.x.h a0, fa0
-; CHECK-NEXT:    vsetvli a1, zero, e16, m2, ta, ma
-; CHECK-NEXT:    vmv.v.x v10, a0
+; CHECK-NEXT:    fcvt.s.bf16 fa5, fa0
+; CHECK-NEXT:    vsetvli a0, zero, e16, m2, ta, ma
 ; CHECK-NEXT:    vfwcvtbf16.f.f.v v12, v8
-; CHECK-NEXT:    vfwcvtbf16.f.f.v v16, v10
 ; CHECK-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
-; CHECK-NEXT:    vmflt.vv v0, v12, v16
+; CHECK-NEXT:    vmflt.vf v0, v12, fa5
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 8 x bfloat> poison, bfloat %b, i32 0
   %splat = shufflevector <vscale x 8 x bfloat> %head, <vscale x 8 x bfloat> poison, <vscale x 8 x i32> zeroinitializer
@@ -279,13 +259,11 @@ define <vscale x 8 x i1> @fcmp_olt_vf_nxv8bf16(<vscale x 8 x bfloat> %va, bfloat
 define <vscale x 8 x i1> @fcmp_olt_fv_nxv8bf16(<vscale x 8 x bfloat> %va, bfloat %b) {
 ; CHECK-LABEL: fcmp_olt_fv_nxv8bf16:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    fmv.x.h a0, fa0
-; CHECK-NEXT:    vsetvli a1, zero, e16, m2, ta, ma
-; CHECK-NEXT:    vmv.v.x v10, a0
+; CHECK-NEXT:    fcvt.s.bf16 fa5, fa0
+; CHECK-NEXT:    vsetvli a0, zero, e16, m2, ta, ma
 ; CHECK-NEXT:    vfwcvtbf16.f.f.v v12, v8
-; CHECK-NEXT:    vfwcvtbf16.f.f.v v16, v10
 ; CHECK-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
-; CHECK-NEXT:    vmflt.vv v0, v16, v12
+; CHECK-NEXT:    vmfgt.vf v0, v12, fa5
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 8 x bfloat> poison, bfloat %b, i32 0
   %splat = shufflevector <vscale x 8 x bfloat> %head, <vscale x 8 x bfloat> poison, <vscale x 8 x i32> zeroinitializer
@@ -309,13 +287,11 @@ define <vscale x 8 x i1> @fcmp_olt_vv_nxv8bf16_nonans(<vscale x 8 x bfloat> %va,
 define <vscale x 8 x i1> @fcmp_olt_vf_nxv8bf16_nonans(<vscale x 8 x bfloat> %va, bfloat %b) #0 {
 ; CHECK-LABEL: fcmp_olt_vf_nxv8bf16_nonans:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    fmv.x.h a0, fa0
-; CHECK-NEXT:    vsetvli a1, zero, e16, m2, ta, ma
-; CHECK-NEXT:    vmv.v.x v10, a0
+; CHECK-NEXT:    fcvt.s.bf16 fa5, fa0
+; CHECK-NEXT:    vsetvli a0, zero, e16, m2, ta, ma
 ; CHECK-NEXT:    vfwcvtbf16.f.f.v v12, v8
-; CHECK-NEXT:    vfwcvtbf16.f.f.v v16, v10
 ; CHECK-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
-; CHECK-NEXT:    vmflt.vv v0, v12, v16
+; CHECK-NEXT:    vmflt.vf v0, v12, fa5
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 8 x bfloat> poison, bfloat %b, i32 0
   %splat = shufflevector <vscale x 8 x bfloat> %head, <vscale x 8 x bfloat> poison, <vscale x 8 x i32> zeroinitializer
@@ -339,13 +315,11 @@ define <vscale x 8 x i1> @fcmp_ole_vv_nxv8bf16(<vscale x 8 x bfloat> %va, <vscal
 define <vscale x 8 x i1> @fcmp_ole_vf_nxv8bf16(<vscale x 8 x bfloat> %va, bfloat %b) {
 ; CHECK-LABEL: fcmp_ole_vf_nxv8bf16:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    fmv.x.h a0, fa0
-; CHECK-NEXT:    vsetvli a1, zero, e16, m2, ta, ma
-; CHECK-NEXT:    vmv.v.x v10, a0
+; CHECK-NEXT:    fcvt.s.bf16 fa5, fa0
+; CHECK-NEXT:    vsetvli a0, zero, e16, m2, ta, ma
 ; CHECK-NEXT:    vfwcvtbf16.f.f.v v12, v8
-; CHECK-NEXT:    vfwcvtbf16.f.f.v v16, v10
 ; CHECK-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
-; CHECK-NEXT:    vmfle.vv v0, v12, v16
+; CHECK-NEXT:    vmfle.vf v0, v12, fa5
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 8 x bfloat> poison, bfloat %b, i32 0
   %splat = shufflevector <vscale x 8 x bfloat> %head, <vscale x 8 x bfloat> poison, <vscale x 8 x i32> zeroinitializer
@@ -356,13 +330,11 @@ define <vscale x 8 x i1> @fcmp_ole_vf_nxv8bf16(<vscale x 8 x bfloat> %va, bfloat
 define <vscale x 8 x i1> @fcmp_ole_fv_nxv8bf16(<vscale x 8 x bfloat> %va, bfloat %b) {
 ; CHECK-LABEL: fcmp_ole_fv_nxv8bf16:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    fmv.x.h a0, fa0
-; CHECK-NEXT:    vsetvli a1, zero, e16, m2, ta, ma
-; CHECK-NEXT:    vmv.v.x v10, a0
+; CHECK-NEXT:    fcvt.s.bf16 fa5, fa0
+; CHECK-NEXT:    vsetvli a0, zero, e16, m2, ta, ma
 ; CHECK-NEXT:    vfwcvtbf16.f.f.v v12, v8
-; CHECK-NEXT:    vfwcvtbf16.f.f.v v16, v10
 ; CHECK-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
-; CHECK-NEXT:    vmfle.vv v0, v16, v12
+; CHECK-NEXT:    vmfge.vf v0, v12, fa5
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 8 x bfloat> poison, bfloat %b, i32 0
   %splat = shufflevector <vscale x 8 x bfloat> %head, <vscale x 8 x bfloat> poison, <vscale x 8 x i32> zeroinitializer
@@ -386,13 +358,11 @@ define <vscale x 8 x i1> @fcmp_ole_vv_nxv8bf16_nonans(<vscale x 8 x bfloat> %va,
 define <vscale x 8 x i1> @fcmp_ole_vf_nxv8bf16_nonans(<vscale x 8 x bfloat> %va, bfloat %b) #0 {
 ; CHECK-LABEL: fcmp_ole_vf_nxv8bf16_nonans:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    fmv.x.h a0, fa0
-; CHECK-NEXT:    vsetvli a1, zero, e16, m2, ta, ma
-; CHECK-NEXT:    vmv.v.x v10, a0
+; CHECK-NEXT:    fcvt.s.bf16 fa5, fa0
+; CHECK-NEXT:    vsetvli a0, zero, e16, m2, ta, ma
 ; CHECK-NEXT:    vfwcvtbf16.f.f.v v12, v8
-; CHECK-NEXT:    vfwcvtbf16.f.f.v v16, v10
 ; CHECK-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
-; CHECK-NEXT:    vmfle.vv v0, v12, v16
+; CHECK-NEXT:    vmfle.vf v0, v12, fa5
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 8 x bfloat> poison, bfloat %b, i32 0
   %splat = shufflevector <vscale x 8 x bfloat> %head, <vscale x 8 x bfloat> poison, <vscale x 8 x i32> zeroinitializer
@@ -418,14 +388,12 @@ define <vscale x 8 x i1> @fcmp_one_vv_nxv8bf16(<vscale x 8 x bfloat> %va, <vscal
 define <vscale x 8 x i1> @fcmp_one_vf_nxv8bf16(<vscale x 8 x bfloat> %va, bfloat %b) {
 ; CHECK-LABEL: fcmp_one_vf_nxv8bf16:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    fmv.x.h a0, fa0
-; CHECK-NEXT:    vsetvli a1, zero, e16, m2, ta, ma
+; CHECK-NEXT:    fcvt.s.bf16 fa5, fa0
+; CHECK-NEXT:    vsetvli a0, zero, e16, m2, ta, ma
 ; CHECK-NEXT:    vfwcvtbf16.f.f.v v12, v8
-; CHECK-NEXT:    vmv.v.x v8, a0
-; CHECK-NEXT:    vfwcvtbf16.f.f.v v16, v8
 ; CHECK-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
-; CHECK-NEXT:    vmflt.vv v8, v12, v16
-; CHECK-NEXT:    vmflt.vv v9, v16, v12
+; CHECK-NEXT:    vmflt.vf v8, v12, fa5
+; CHECK-NEXT:    vmfgt.vf v9, v12, fa5
 ; CHECK-NEXT:    vmor.mm v0, v9, v8
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 8 x bfloat> poison, bfloat %b, i32 0
@@ -437,14 +405,12 @@ define <vscale x 8 x i1> @fcmp_one_vf_nxv8bf16(<vscale x 8 x bfloat> %va, bfloat
 define <vscale x 8 x i1> @fcmp_one_fv_nxv8bf16(<vscale x 8 x bfloat> %va, bfloat %b) {
 ; CHECK-LABEL: fcmp_one_fv_nxv8bf16:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    fmv.x.h a0, fa0
-; CHECK-NEXT:    vsetvli a1, zero, e16, m2, ta, ma
+; CHECK-NEXT:    fcvt.s.bf16 fa5, fa0
+; CHECK-NEXT:    vsetvli a0, zero, e16, m2, ta, ma
 ; CHECK-NEXT:    vfwcvtbf16.f.f.v v12, v8
-; CHECK-NEXT:    vmv.v.x v8, a0
-; CHECK-NEXT:    vfwcvtbf16.f.f.v v16, v8
 ; CHECK-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
-; CHECK-NEXT:    vmflt.vv v8, v16, v12
-; CHECK-NEXT:    vmflt.vv v9, v12, v16
+; CHECK-NEXT:    vmfgt.vf v8, v12, fa5
+; CHECK-NEXT:    vmflt.vf v9, v12, fa5
 ; CHECK-NEXT:    vmor.mm v0, v9, v8
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 8 x bfloat> poison, bfloat %b, i32 0
@@ -469,13 +435,11 @@ define <vscale x 8 x i1> @fcmp_one_vv_nxv8bf16_nonans(<vscale x 8 x bfloat> %va,
 define <vscale x 8 x i1> @fcmp_one_vf_nxv8bf16_nonans(<vscale x 8 x bfloat> %va, bfloat %b) #0 {
 ; CHECK-LABEL: fcmp_one_vf_nxv8bf16_nonans:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    fmv.x.h a0, fa0
-; CHECK-NEXT:    vsetvli a1, zero, e16, m2, ta, ma
-; CHECK-NEXT:    vmv.v.x v10, a0
+; CHECK-NEXT:    fcvt.s.bf16 fa5, fa0
+; CHECK-NEXT:    vsetvli a0, zero, e16, m2, ta, ma
 ; CHECK-NEXT:    vfwcvtbf16.f.f.v v12, v8
-; CHECK-NEXT:    vfwcvtbf16.f.f.v v16, v10
 ; CHECK-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
-; CHECK-NEXT:    vmfne.vv v0, v12, v16
+; CHECK-NEXT:    vmfne.vf v0, v12, fa5
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 8 x bfloat> poison, bfloat %b, i32 0
   %splat = shufflevector <vscale x 8 x bfloat> %head, <vscale x 8 x bfloat> poison, <vscale x 8 x i32> zeroinitializer
@@ -503,17 +467,15 @@ define <vscale x 8 x i1> @fcmp_ord_vv_nxv8bf16(<vscale x 8 x bfloat> %va, <vscal
 define <vscale x 8 x i1> @fcmp_ord_vf_nxv8bf16(<vscale x 8 x bfloat> %va, bfloat %b) {
 ; CHECK-LABEL: fcmp_ord_vf_nxv8bf16:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    fmv.x.h a0, fa0
-; CHECK-NEXT:    vsetvli a1, zero, e16, m2, ta, ma
-; CHECK-NEXT:    vfwcvtbf16.f.f.v v12, v8
-; CHECK-NEXT:    vmv.v.x v8, a0
-; CHECK-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
-; CHECK-NEXT:    vmfeq.vv v10, v12, v12
+; CHECK-NEXT:    fcvt.s.bf16 fa5, fa0
+; CHECK-NEXT:    vsetvli a0, zero, e32, m4, ta, ma
+; CHECK-NEXT:    vfmv.v.f v12, fa5
+; CHECK-NEXT:    vmfeq.vf v10, v12, fa5
 ; CHECK-NEXT:    vsetvli zero, zero, e16, m2, ta, ma
 ; CHECK-NEXT:    vfwcvtbf16.f.f.v v12, v8
 ; CHECK-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
 ; CHECK-NEXT:    vmfeq.vv v8, v12, v12
-; CHECK-NEXT:    vmand.mm v0, v10, v8
+; CHECK-NEXT:    vmand.mm v0, v8, v10
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 8 x bfloat> poison, bfloat %b, i32 0
   %splat = shufflevector <vscale x 8 x bfloat> %head, <vscale x 8 x bfloat> poison, <vscale x 8 x i32> zeroinitializer
@@ -524,17 +486,15 @@ define <vscale x 8 x i1> @fcmp_ord_vf_nxv8bf16(<vscale x 8 x bfloat> %va, bfloat
 define <vscale x 8 x i1> @fcmp_ord_fv_nxv8bf16(<vscale x 8 x bfloat> %va, bfloat %b) {
 ; CHECK-LABEL: fcmp_ord_fv_nxv8bf16:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    fmv.x.h a0, fa0
-; CHECK-NEXT:    vsetvli a1, zero, e16, m2, ta, ma
-; CHECK-NEXT:    vfwcvtbf16.f.f.v v12, v8
-; CHECK-NEXT:    vmv.v.x v8, a0
-; CHECK-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
-; CHECK-NEXT:    vmfeq.vv v10, v12, v12
+; CHECK-NEXT:    fcvt.s.bf16 fa5, fa0
+; CHECK-NEXT:    vsetvli a0, zero, e32, m4, ta, ma
+; CHECK-NEXT:    vfmv.v.f v12, fa5
+; CHECK-NEXT:    vmfeq.vf v10, v12, fa5
 ; CHECK-NEXT:    vsetvli zero, zero, e16, m2, ta, ma
 ; CHECK-NEXT:    vfwcvtbf16.f.f.v v12, v8
 ; CHECK-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
 ; CHECK-NEXT:    vmfeq.vv v8, v12, v12
-; CHECK-NEXT:    vmand.mm v0, v8, v10
+; CHECK-NEXT:    vmand.mm v0, v10, v8
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 8 x bfloat> poison, bfloat %b, i32 0
   %splat = shufflevector <vscale x 8 x bfloat> %head, <vscale x 8 x bfloat> poison, <vscale x 8 x i32> zeroinitializer
@@ -562,17 +522,15 @@ define <vscale x 8 x i1> @fcmp_ord_vv_nxv8bf16_nonans(<vscale x 8 x bfloat> %va,
 define <vscale x 8 x i1> @fcmp_ord_vf_nxv8bf16_nonans(<vscale x 8 x bfloat> %va, bfloat %b) #0 {
 ; CHECK-LABEL: fcmp_ord_vf_nxv8bf16_nonans:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    fmv.x.h a0, fa0
-; CHECK-NEXT:    vsetvli a1, zero, e16, m2, ta, ma
-; CHECK-NEXT:    vfwcvtbf16.f.f.v v12, v8
-; CHECK-NEXT:    vmv.v.x v8, a0
-; CHECK-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
-; CHECK-NEXT:    vmfeq.vv v10, v12, v12
+; CHECK-NEXT:    fcvt.s.bf16 fa5, fa0
+; CHECK-NEXT:    vsetvli a0, zero, e32, m4, ta, ma
+; CHECK-NEXT:    vfmv.v.f v12, fa5
+; CHECK-NEXT:    vmfeq.vf v10, v12, fa5
 ; CHECK-NEXT:    vsetvli zero, zero, e16, m2, ta, ma
 ; CHECK-NEXT:    vfwcvtbf16.f.f.v v12, v8
 ; CHECK-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
 ; CHECK-NEXT:    vmfeq.vv v8, v12, v12
-; CHECK-NEXT:    vmand.mm v0, v10, v8
+; CHECK-NEXT:    vmand.mm v0, v8, v10
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 8 x bfloat> poison, bfloat %b, i32 0
   %splat = shufflevector <vscale x 8 x bfloat> %head, <vscale x 8 x bfloat> poison, <vscale x 8 x i32> zeroinitializer
@@ -598,14 +556,12 @@ define <vscale x 8 x i1> @fcmp_ueq_vv_nxv8bf16(<vscale x 8 x bfloat> %va, <vscal
 define <vscale x 8 x i1> @fcmp_ueq_vf_nxv8bf16(<vscale x 8 x bfloat> %va, bfloat %b) {
 ; CHECK-LABEL: fcmp_ueq_vf_nxv8bf16:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    fmv.x.h a0, fa0
-; CHECK-NEXT:    vsetvli a1, zero, e16, m2, ta, ma
+; CHECK-NEXT:    fcvt.s.bf16 fa5, fa0
+; CHECK-NEXT:    vsetvli a0, zero, e16, m2, ta, ma
 ; CHECK-NEXT:    vfwcvtbf16.f.f.v v12, v8
-; CHECK-NEXT:    vmv.v.x v8, a0
-; CHECK-NEXT:    vfwcvtbf16.f.f.v v16, v8
 ; CHECK-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
-; CHECK-NEXT:    vmflt.vv v8, v12, v16
-; CHECK-NEXT:    vmflt.vv v9, v16, v12
+; CHECK-NEXT:    vmflt.vf v8, v12, fa5
+; CHECK-NEXT:    vmfgt.vf v9, v12, fa5
 ; CHECK-NEXT:    vmnor.mm v0, v9, v8
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 8 x bfloat> poison, bfloat %b, i32 0
@@ -617,14 +573,12 @@ define <vscale x 8 x i1> @fcmp_ueq_vf_nxv8bf16(<vscale x 8 x bfloat> %va, bfloat
 define <vscale x 8 x i1> @fcmp_ueq_fv_nxv8bf16(<vscale x 8 x bfloat> %va, bfloat %b) {
 ; CHECK-LABEL: fcmp_ueq_fv_nxv8bf16:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    fmv.x.h a0, fa0
-; CHECK-NEXT:    vsetvli a1, zero, e16, m2, ta, ma
+; CHECK-NEXT:    fcvt.s.bf16 fa5, fa0
+; CHECK-NEXT:    vsetvli a0, zero, e16, m2, ta, ma
 ; CHECK-NEXT:    vfwcvtbf16.f.f.v v12, v8
-; CHECK-NEXT:    vmv.v.x v8, a0
-; CHECK-NEXT:    vfwcvtbf16.f.f.v v16, v8
 ; CHECK-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
-; CHECK-NEXT:    vmflt.vv v8, v16, v12
-; CHECK-NEXT:    vmflt.vv v9, v12, v16
+; CHECK-NEXT:    vmfgt.vf v8, v12, fa5
+; CHECK-NEXT:    vmflt.vf v9, v12, fa5
 ; CHECK-NEXT:    vmnor.mm v0, v9, v8
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 8 x bfloat> poison, bfloat %b, i32 0
@@ -649,13 +603,11 @@ define <vscale x 8 x i1> @fcmp_ueq_vv_nxv8bf16_nonans(<vscale x 8 x bfloat> %va,
 define <vscale x 8 x i1> @fcmp_ueq_vf_nxv8bf16_nonans(<vscale x 8 x bfloat> %va, bfloat %b) #0 {
 ; CHECK-LABEL: fcmp_ueq_vf_nxv8bf16_nonans:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    fmv.x.h a0, fa0
-; CHECK-NEXT:    vsetvli a1, zero, e16, m2, ta, ma
-; CHECK-NEXT:    vmv.v.x v10, a0
+; CHECK-NEXT:    fcvt.s.bf16 fa5, fa0
+; CHECK-NEXT:    vsetvli a0, zero, e16, m2, ta, ma
 ; CHECK-NEXT:    vfwcvtbf16.f.f.v v12, v8
-; CHECK-NEXT:    vfwcvtbf16.f.f.v v16, v10
 ; CHECK-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
-; CHECK-NEXT:    vmfeq.vv v0, v12, v16
+; CHECK-NEXT:    vmfeq.vf v0, v12, fa5
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 8 x bfloat> poison, bfloat %b, i32 0
   %splat = shufflevector <vscale x 8 x bfloat> %head, <vscale x 8 x bfloat> poison, <vscale x 8 x i32> zeroinitializer
@@ -680,13 +632,11 @@ define <vscale x 8 x i1> @fcmp_ugt_vv_nxv8bf16(<vscale x 8 x bfloat> %va, <vscal
 define <vscale x 8 x i1> @fcmp_ugt_vf_nxv8bf16(<vscale x 8 x bfloat> %va, bfloat %b) {
 ; CHECK-LABEL: fcmp_ugt_vf_nxv8bf16:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    fmv.x.h a0, fa0
-; CHECK-NEXT:    vsetvli a1, zero, e16, m2, ta, ma
-; CHECK-NEXT:    vmv.v.x v10, a0
+; CHECK-NEXT:    fcvt.s.bf16 fa5, fa0
+; CHECK-NEXT:    vsetvli a0, zero, e16, m2, ta, ma
 ; CHECK-NEXT:    vfwcvtbf16.f.f.v v12, v8
-; CHECK-NEXT:    vfwcvtbf16.f.f.v v16, v10
 ; CHECK-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
-; CHECK-NEXT:    vmfle.vv v8, v12, v16
+; CHECK-NEXT:    vmfle.vf v8, v12, fa5
 ; CHECK-NEXT:    vmnot.m v0, v8
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 8 x bfloat> poison, bfloat %b, i32 0
@@ -698,13 +648,11 @@ define <vscale x 8 x i1> @fcmp_ugt_vf_nxv8bf16(<vscale x 8 x bfloat> %va, bfloat
 define <vscale x 8 x i1> @fcmp_ugt_fv_nxv8bf16(<vscale x 8 x bfloat> %va, bfloat %b) {
 ; CHECK-LABEL: fcmp_ugt_fv_nxv8bf16:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    fmv.x.h a0, fa0
-; CHECK-NEXT:    vsetvli a1, zero, e16, m2, ta, ma
-; CHECK-NEXT:    vmv.v.x v10, a0
+; CHECK-NEXT:    fcvt.s.bf16 fa5, fa0
+; CHECK-NEXT:    vsetvli a0, zero, e16, m2, ta, ma
 ; CHECK-NEXT:    vfwcvtbf16.f.f.v v12, v8
-; CHECK-NEXT:    vfwcvtbf16.f.f.v v16, v10
 ; CHECK-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
-; CHECK-NEXT:    vmfle.vv v8, v16, v12
+; CHECK-NEXT:    vmfge.vf v8, v12, fa5
 ; CHECK-NEXT:    vmnot.m v0, v8
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 8 x bfloat> poison, bfloat %b, i32 0
@@ -729,13 +677,11 @@ define <vscale x 8 x i1> @fcmp_ugt_vv_nxv8bf16_nonans(<vscale x 8 x bfloat> %va,
 define <vscale x 8 x i1> @fcmp_ugt_vf_nxv8bf16_nonans(<vscale x 8 x bfloat> %va, bfloat %b) #0 {
 ; CHECK-LABEL: fcmp_ugt_vf_nxv8bf16_nonans:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    fmv.x.h a0, fa0
-; CHECK-NEXT:    vsetvli a1, zero, e16, m2, ta, ma
-; CHECK-NEXT:    vmv.v.x v10, a0
+; CHECK-NEXT:    fcvt.s.bf16 fa5, fa0
+; CHECK-NEXT:    vsetvli a0, zero, e16, m2, ta, ma
 ; CHECK-NEXT:    vfwcvtbf16.f.f.v v12, v8
-; CHECK-NEXT:    vfwcvtbf16.f.f.v v16, v10
 ; CHECK-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
-; CHECK-NEXT:    vmflt.vv v0, v16, v12
+; CHECK-NEXT:    vmfgt.vf v0, v12, fa5
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 8 x bfloat> poison, bfloat %b, i32 0
   %splat = shufflevector <vscale x 8 x bfloat> %head, <vscale x 8 x bfloat> poison, <vscale x 8 x i32> zeroinitializer
@@ -760,13 +706,11 @@ define <vscale x 8 x i1> @fcmp_uge_vv_nxv8bf16(<vscale x 8 x bfloat> %va, <vscal
 define <vscale x 8 x i1> @fcmp_uge_vf_nxv8bf16(<vscale x 8 x bfloat> %va, bfloat %b) {
 ; CHECK-LABEL: fcmp_uge_vf_nxv8bf16:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    fmv.x.h a0, fa0
-; CHECK-NEXT:    vsetvli a1, zero, e16, m2, ta, ma
-; CHECK-NEXT:    vmv.v.x v10, a0
+; CHECK-NEXT:    fcvt.s.bf16 fa5, fa0
+; CHECK-NEXT:    vsetvli a0, zero, e16, m2, ta, ma
 ; CHECK-NEXT:    vfwcvtbf16.f.f.v v12, v8
-; CHECK-NEXT:    vfwcvtbf16.f.f.v v16, v10
 ; CHECK-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
-; CHECK-NEXT:    vmflt.vv v8, v12, v16
+; CHECK-NEXT:    vmflt.vf v8, v12, fa5
 ; CHECK-NEXT:    vmnot.m v0, v8
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 8 x bfloat> poison, bfloat %b, i32 0
@@ -778,13 +722,11 @@ define <vscale x 8 x i1> @fcmp_uge_vf_nxv8bf16(<vscale x 8 x bfloat> %va, bfloat
 define <vscale x 8 x i1> @fcmp_uge_fv_nxv8bf16(<vscale x 8 x bfloat> %va, bfloat %b) {
 ; CHECK-LABEL: fcmp_uge_fv_nxv8bf16:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    fmv.x.h a0, fa0
-; CHECK-NEXT:    vsetvli a1, zero, e16, m2, ta, ma
-; CHECK-NEXT:    vmv.v.x v10, a0
+; CHECK-NEXT:    fcvt.s.bf16 fa5, fa0
+; CHECK-NEXT:    vsetvli a0, zero, e16, m2, ta, ma
 ; CHECK-NEXT:    vfwcvtbf16.f.f.v v12, v8
-; CHECK-NEXT:    vfwcvtbf16.f.f.v v16, v10
 ; CHECK-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
-; CHECK-NEXT:    vmflt.vv v8, v16, v12
+; CHECK-NEXT:    vmfgt.vf v8, v12, fa5
 ; CHECK-NEXT:    vmnot.m v0, v8
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 8 x bfloat> poison, bfloat %b, i32 0
@@ -809,13 +751,11 @@ define <vscale x 8 x i1> @fcmp_uge_vv_nxv8bf16_nonans(<vscale x 8 x bfloat> %va,
 define <vscale x 8 x i1> @fcmp_uge_vf_nxv8bf16_nonans(<vscale x 8 x bfloat> %va, bfloat %b) #0 {
 ; CHECK-LABEL: fcmp_uge_vf_nxv8bf16_nonans:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    fmv.x.h a0, fa0
-; CHECK-NEXT:    vsetvli a1, zero, e16, m2, ta, ma
-; CHECK-NEXT:    vmv.v.x v10, a0
+; CHECK-NEXT:    fcvt.s.bf16 fa5, fa0
+; CHECK-NEXT:    vsetvli a0, zero, e16, m2, ta, ma
 ; CHECK-NEXT:    vfwcvtbf16.f.f.v v12, v8
-; CHECK-NEXT:    vfwcvtbf16.f.f.v v16, v10
 ; CHECK-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
-; CHECK-NEXT:    vmfle.vv v0, v16, v12
+; CHECK-NEXT:    vmfge.vf v0, v12, fa5
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 8 x bfloat> poison, bfloat %b, i32 0
   %splat = shufflevector <vscale x 8 x bfloat> %head, <vscale x 8 x bfloat> poison, <vscale x 8 x i32> zeroinitializer
@@ -840,13 +780,11 @@ define <vscale x 8 x i1> @fcmp_ult_vv_nxv8bf16(<vscale x 8 x bfloat> %va, <vscal
 define <vscale x 8 x i1> @fcmp_ult_vf_nxv8bf16(<vscale x 8 x bfloat> %va, bfloat %b) {
 ; CHECK-LABEL: fcmp_ult_vf_nxv8bf16:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    fmv.x.h a0, fa0
-; CHECK-NEXT:    vsetvli a1, zero, e16, m2, ta, ma
-; CHECK-NEXT:    vmv.v.x v10, a0
+; CHECK-NEXT:    fcvt.s.bf16 fa5, fa0
+; CHECK-NEXT:    vsetvli a0, zero, e16, m2, ta, ma
 ; CHECK-NEXT:    vfwcvtbf16.f.f.v v12, v8
-; CHECK-NEXT:    vfwcvtbf16.f.f.v v16, v10
 ; CHECK-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
-; CHECK-NEXT:    vmfle.vv v8, v16, v12
+; CHECK-NEXT:    vmfge.vf v8, v12, fa5
 ; CHECK-NEXT:    vmnot.m v0, v8
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 8 x bfloat> poison, bfloat %b, i32 0
@@ -858,13 +796,11 @@ define <vscale x 8 x i1> @fcmp_ult_vf_nxv8bf16(<vscale x 8 x bfloat> %va, bfloat
 define <vscale x 8 x i1> @fcmp_ult_fv_nxv8bf16(<vscale x 8 x bfloat> %va, bfloat %b) {
 ; CHECK-LABEL: fcmp_ult_fv_nxv8bf16:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    fmv.x.h a0, fa0
-; CHECK-NEXT:    vsetvli a1, zero, e16, m2, ta, ma
-; CHECK-NEXT:    vmv.v.x v10, a0
+; CHECK-NEXT:    fcvt.s.bf16 fa5, fa0
+; CHECK-NEXT:    vsetvli a0, zero, e16, m2, ta, ma
 ; CHECK-NEXT:    vfwcvtbf16.f.f.v v12, v8
-; CHECK-NEXT:    vfwcvtbf16.f.f.v v16, v10
 ; CHECK-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
-; CHECK-NEXT:    vmfle.vv v8, v12, v16
+; CHECK-NEXT:    vmfle.vf v8, v12, fa5
 ; CHECK-NEXT:    vmnot.m v0, v8
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 8 x bfloat> poison, bfloat %b, i32 0
@@ -889,13 +825,11 @@ define <vscale x 8 x i1> @fcmp_ult_vv_nxv8bf16_nonans(<vscale x 8 x bfloat> %va,
 define <vscale x 8 x i1> @fcmp_ult_vf_nxv8bf16_nonans(<vscale x 8 x bfloat> %va, bfloat %b) #0 {
 ; CHECK-LABEL: fcmp_ult_vf_nxv8bf16_nonans:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    fmv.x.h a0, fa0
-; CHECK-NEXT:    vsetvli a1, zero, e16, m2, ta, ma
-; CHECK-NEXT:    vmv.v.x v10, a0
+; CHECK-NEXT:    fcvt.s.bf16 fa5, fa0
+; CHECK-NEXT:    vsetvli a0, zero, e16, m2, ta, ma
 ; CHECK-NEXT:    vfwcvtbf16.f.f.v v12, v8
-; CHECK-NEXT:    vfwcvtbf16.f.f.v v16, v10
 ; CHECK-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
-; CHECK-NEXT:    vmflt.vv v0, v12, v16
+; CHECK-NEXT:    vmflt.vf v0, v12, fa5
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 8 x bfloat> poison, bfloat %b, i32 0
   %splat = shufflevector <vscale x 8 x bfloat> %head, <vscale x 8 x bfloat> poison, <vscale x 8 x i32> zeroinitializer
@@ -920,13 +854,11 @@ define <vscale x 8 x i1> @fcmp_ule_vv_nxv8bf16(<vscale x 8 x bfloat> %va, <vscal
 define <vscale x 8 x i1> @fcmp_ule_vf_nxv8bf16(<vscale x 8 x bfloat> %va, bfloat %b) {
 ; CHECK-LABEL: fcmp_ule_vf_nxv8bf16:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    fmv.x.h a0, fa0
-; CHECK-NEXT:    vsetvli a1, zero, e16, m2, ta, ma
-; CHECK-NEXT:    vmv.v.x v10, a0
+; CHECK-NEXT:    fcvt.s.bf16 fa5, fa0
+; CHECK-NEXT:    vsetvli a0, zero, e16, m2, ta, ma
 ; CHECK-NEXT:    vfwcvtbf16.f.f.v v12, v8
-; CHECK-NEXT:    vfwcvtbf16.f.f.v v16, v10
 ; CHECK-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
-; CHECK-NEXT:    vmflt.vv v8, v16, v12
+; CHECK-NEXT:    vmfgt.vf v8, v12, fa5
 ; CHECK-NEXT:    vmnot.m v0, v8
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 8 x bfloat> poison, bfloat %b, i32 0
@@ -938,13 +870,11 @@ define <vscale x 8 x i1> @fcmp_ule_vf_nxv8bf16(<vscale x 8 x bfloat> %va, bfloat
 define <vscale x 8 x i1> @fcmp_ule_fv_nxv8bf16(<vscale x 8 x bfloat> %va, bfloat %b) {
 ; CHECK-LABEL: fcmp_ule_fv_nxv8bf16:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    fmv.x.h a0, fa0
-; CHECK-NEXT:    vsetvli a1, zero, e16, m2, ta, ma
-; CHECK-NEXT:    vmv.v.x v10, a0
+; CHECK-NEXT:    fcvt.s.bf16 fa5, fa0
+; CHECK-NEXT:    vsetvli a0, zero, e16, m2, ta, ma
 ; CHECK-NEXT:    vfwcvtbf16.f.f.v v12, v8
-; CHECK-NEXT:    vfwcvtbf16.f.f.v v16, v10
 ; CHECK-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
-; CHECK-NEXT:    vmflt.vv v8, v12, v16
+; CHECK-NEXT:    vmflt.vf v8, v12, fa5
 ; CHECK-NEXT:    vmnot.m v0, v8
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 8 x bfloat> poison, bfloat %b, i32 0
@@ -969,13 +899,11 @@ define <vscale x 8 x i1> @fcmp_ule_vv_nxv8bf16_nonans(<vscale x 8 x bfloat> %va,
 define <vscale x 8 x i1> @fcmp_ule_vf_nxv8bf16_nonans(<vscale x 8 x bfloat> %va, bfloat %b) #0 {
 ; CHECK-LABEL: fcmp_ule_vf_nxv8bf16_nonans:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    fmv.x.h a0, fa0
-; CHECK-NEXT:    vsetvli a1, zero, e16, m2, ta, ma
-; CHECK-NEXT:    vmv.v.x v10, a0
+; CHECK-NEXT:    fcvt.s.bf16 fa5, fa0
+; CHECK-NEXT:    vsetvli a0, zero, e16, m2, ta, ma
 ; CHECK-NEXT:    vfwcvtbf16.f.f.v v12, v8
-; CHECK-NEXT:    vfwcvtbf16.f.f.v v16, v10
 ; CHECK-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
-; CHECK-NEXT:    vmfle.vv v0, v12, v16
+; CHECK-NEXT:    vmfle.vf v0, v12, fa5
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 8 x bfloat> poison, bfloat %b, i32 0
   %splat = shufflevector <vscale x 8 x bfloat> %head, <vscale x 8 x bfloat> poison, <vscale x 8 x i32> zeroinitializer
@@ -999,13 +927,11 @@ define <vscale x 8 x i1> @fcmp_une_vv_nxv8bf16(<vscale x 8 x bfloat> %va, <vscal
 define <vscale x 8 x i1> @fcmp_une_vf_nxv8bf16(<vscale x 8 x bfloat> %va, bfloat %b) {
 ; CHECK-LABEL: fcmp_une_vf_nxv8bf16:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    fmv.x.h a0, fa0
-; CHECK-NEXT:    vsetvli a1, zero, e16, m2, ta, ma
-; CHECK-NEXT:    vmv.v.x v10, a0
+; CHECK-NEXT:    fcvt.s.bf16 fa5, fa0
+; CHECK-NEXT:    vsetvli a0, zero, e16, m2, ta, ma
 ; CHECK-NEXT:    vfwcvtbf16.f.f.v v12, v8
-; CHECK-NEXT:    vfwcvtbf16.f.f.v v16, v10
 ; CHECK-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
-; CHECK-NEXT:    vmfne.vv v0, v12, v16
+; CHECK-NEXT:    vmfne.vf v0, v12, fa5
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 8 x bfloat> poison, bfloat %b, i32 0
   %splat = shufflevector <vscale x 8 x bfloat> %head, <vscale x 8 x bfloat> poison, <vscale x 8 x i32> zeroinitializer
@@ -1016,13 +942,11 @@ define <vscale x 8 x i1> @fcmp_une_vf_nxv8bf16(<vscale x 8 x bfloat> %va, bfloat
 define <vscale x 8 x i1> @fcmp_une_fv_nxv8bf16(<vscale x 8 x bfloat> %va, bfloat %b) {
 ; CHECK-LABEL: fcmp_une_fv_nxv8bf16:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    fmv.x.h a0, fa0
-; CHECK-NEXT:    vsetvli a1, zero, e16, m2, ta, ma
-; CHECK-NEXT:    vmv.v.x v10, a0
+; CHECK-NEXT:    fcvt.s.bf16 fa5, fa0
+; CHECK-NEXT:    vsetvli a0, zero, e16, m2, ta, ma
 ; CHECK-NEXT:    vfwcvtbf16.f.f.v v12, v8
-; CHECK-NEXT:    vfwcvtbf16.f.f.v v16, v10
 ; CHECK-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
-; CHECK-NEXT:    vmfne.vv v0, v16, v12
+; CHECK-NEXT:    vmfne.vf v0, v12, fa5
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 8 x bfloat> poison, bfloat %b, i32 0
   %splat = shufflevector <vscale x 8 x bfloat> %head, <vscale x 8 x bfloat> poison, <vscale x 8 x i32> zeroinitializer
@@ -1046,13 +970,11 @@ define <vscale x 8 x i1> @fcmp_une_vv_nxv8bf16_nonans(<vscale x 8 x bfloat> %va,
 define <vscale x 8 x i1> @fcmp_une_vf_nxv8bf16_nonans(<vscale x 8 x bfloat> %va, bfloat %b) #0 {
 ; CHECK-LABEL: fcmp_une_vf_nxv8bf16_nonans:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    fmv.x.h a0, fa0
-; CHECK-NEXT:    vsetvli a1, zero, e16, m2, ta, ma
-; CHECK-NEXT:    vmv.v.x v10, a0
+; CHECK-NEXT:    fcvt.s.bf16 fa5, fa0
+; CHECK-NEXT:    vsetvli a0, zero, e16, m2, ta, ma
 ; CHECK-NEXT:    vfwcvtbf16.f.f.v v12, v8
-; CHECK-NEXT:    vfwcvtbf16.f.f.v v16, v10
 ; CHECK-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
-; CHECK-NEXT:    vmfne.vv v0, v12, v16
+; CHECK-NEXT:    vmfne.vf v0, v12, fa5
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 8 x bfloat> poison, bfloat %b, i32 0
   %splat = shufflevector <vscale x 8 x bfloat> %head, <vscale x 8 x bfloat> poison, <vscale x 8 x i32> zeroinitializer
@@ -1080,17 +1002,15 @@ define <vscale x 8 x i1> @fcmp_uno_vv_nxv8bf16(<vscale x 8 x bfloat> %va, <vscal
 define <vscale x 8 x i1> @fcmp_uno_vf_nxv8bf16(<vscale x 8 x bfloat> %va, bfloat %b) {
 ; CHECK-LABEL: fcmp_uno_vf_nxv8bf16:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    fmv.x.h a0, fa0
-; CHECK-NEXT:    vsetvli a1, zero, e16, m2, ta, ma
-; CHECK-NEXT:    vfwcvtbf16.f.f.v v12, v8
-; CHECK-NEXT:    vmv.v.x v8, a0
-; CHECK-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
-; CHECK-NEXT:    vmfne.vv v10, v12, v12
+; CHECK-NEXT:    fcvt.s.bf16 fa5, fa0
+; CHECK-NEXT:    vsetvli a0, zero, e32, m4, ta, ma
+; CHECK-NEXT:    vfmv.v.f v12, fa5
+; CHECK-NEXT:    vmfne.vf v10, v12, fa5
 ; CHECK-NEXT:    vsetvli zero, zero, e16, m2, ta, ma
 ; CHECK-NEXT:    vfwcvtbf16.f.f.v v12, v8
 ; CHECK-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
 ; CHECK-NEXT:    vmfne.vv v8, v12, v12
-; CHECK-NEXT:    vmor.mm v0, v10, v8
+; CHECK-NEXT:    vmor.mm v0, v8, v10
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 8 x bfloat> poison, bfloat %b, i32 0
   %splat = shufflevector <vscale x 8 x bfloat> %head, <vscale x 8 x bfloat> poison, <vscale x 8 x i32> zeroinitializer
@@ -1101,17 +1021,15 @@ define <vscale x 8 x i1> @fcmp_uno_vf_nxv8bf16(<vscale x 8 x bfloat> %va, bfloat
 define <vscale x 8 x i1> @fcmp_uno_fv_nxv8bf16(<vscale x 8 x bfloat> %va, bfloat %b) {
 ; CHECK-LABEL: fcmp_uno_fv_nxv8bf16:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    fmv.x.h a0, fa0
-; CHECK-NEXT:    vsetvli a1, zero, e16, m2, ta, ma
-; CHECK-NEXT:    vfwcvtbf16.f.f.v v12, v8
-; CHECK-NEXT:    vmv.v.x v8, a0
-; CHECK-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
-; CHECK-NEXT:    vmfne.vv v10, v12, v12
+; CHECK-NEXT:    fcvt.s.bf16 fa5, fa0
+; CHECK-NEXT:    vsetvli a0, zero, e32, m4, ta, ma
+; CHECK-NEXT:    vfmv.v.f v12, fa5
+; CHECK-NEXT:    vmfne.vf v10, v12, fa5
 ; CHECK-NEXT:    vsetvli zero, zero, e16, m2, ta, ma
 ; CHECK-NEXT:    vfwcvtbf16.f.f.v v12, v8
 ; CHECK-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
 ; CHECK-NEXT:    vmfne.vv v8, v12, v12
-; CHECK-NEXT:    vmor.mm v0, v8, v10
+; CHECK-NEXT:    vmor.mm v0, v10, v8
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 8 x bfloat> poison, bfloat %b, i32 0
   %splat = shufflevector <vscale x 8 x bfloat> %head, <vscale x 8 x bfloat> poison, <vscale x 8 x i32> zeroinitializer
@@ -1139,17 +1057,15 @@ define <vscale x 8 x i1> @fcmp_uno_vv_nxv8bf16_nonans(<vscale x 8 x bfloat> %va,
 define <vscale x 8 x i1> @fcmp_uno_vf_nxv8bf16_nonans(<vscale x 8 x bfloat> %va, bfloat %b) #0 {
 ; CHECK-LABEL: fcmp_uno_vf_nxv8bf16_nonans:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    fmv.x.h a0, fa0
-; CHECK-NEXT:    vsetvli a1, zero, e16, m2, ta, ma
-; CHECK-NEXT:    vfwcvtbf16.f.f.v v12, v8
-; CHECK-NEXT:    vmv.v.x v8, a0
-; CHECK-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
-; CHECK-NEXT:    vmfne.vv v10, v12, v12
+; CHECK-NEXT:    fcvt.s.bf16 fa5, fa0
+; CHECK-NEXT:    vsetvli a0, zero, e32, m4, ta, ma
+; CHECK-NEXT:    vfmv.v.f v12, fa5
+; CHECK-NEXT:    vmfne.vf v10, v12, fa5
 ; CHECK-NEXT:    vsetvli zero, zero, e16, m2, ta, ma
 ; CHECK-NEXT:    vfwcvtbf16.f.f.v v12, v8
 ; CHECK-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
 ; CHECK-NEXT:    vmfne.vv v8, v12, v12
-; CHECK-NEXT:    vmor.mm v0, v10, v8
+; CHECK-NEXT:    vmor.mm v0, v8, v10
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 8 x bfloat> poison, bfloat %b, i32 0
   %splat = shufflevector <vscale x 8 x bfloat> %head, <vscale x 8 x bfloat> poison, <vscale x 8 x i32> zeroinitializer
@@ -1185,13 +1101,11 @@ define <vscale x 8 x i1> @fcmp_oeq_vf_nxv8f16(<vscale x 8 x half> %va, half %b)
 ;
 ; ZVFHMIN-LABEL: fcmp_oeq_vf_nxv8f16:
 ; ZVFHMIN:       # %bb.0:
-; ZVFHMIN-NEXT:    fmv.x.h a0, fa0
-; ZVFHMIN-NEXT:    vsetvli a1, zero, e16, m2, ta, ma
-; ZVFHMIN-NEXT:    vmv.v.x v10, a0
+; ZVFHMIN-NEXT:    fcvt.s.h fa5, fa0
+; ZVFHMIN-NEXT:    vsetvli a0, zero, e16, m2, ta, ma
 ; ZVFHMIN-NEXT:    vfwcvt.f.f.v v12, v8
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v16, v10
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
-; ZVFHMIN-NEXT:    vmfeq.vv v0, v12, v16
+; ZVFHMIN-NEXT:    vmfeq.vf v0, v12, fa5
 ; ZVFHMIN-NEXT:    ret
   %head = insertelement <vscale x 8 x half> poison, half %b, i32 0
   %splat = shufflevector <vscale x 8 x half> %head, <vscale x 8 x half> poison, <vscale x 8 x i32> zeroinitializer
@@ -1208,13 +1122,11 @@ define <vscale x 8 x i1> @fcmp_oeq_fv_nxv8f16(<vscale x 8 x half> %va, half %b)
 ;
 ; ZVFHMIN-LABEL: fcmp_oeq_fv_nxv8f16:
 ; ZVFHMIN:       # %bb.0:
-; ZVFHMIN-NEXT:    fmv.x.h a0, fa0
-; ZVFHMIN-NEXT:    vsetvli a1, zero, e16, m2, ta, ma
-; ZVFHMIN-NEXT:    vmv.v.x v10, a0
+; ZVFHMIN-NEXT:    fcvt.s.h fa5, fa0
+; ZVFHMIN-NEXT:    vsetvli a0, zero, e16, m2, ta, ma
 ; ZVFHMIN-NEXT:    vfwcvt.f.f.v v12, v8
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v16, v10
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
-; ZVFHMIN-NEXT:    vmfeq.vv v0, v16, v12
+; ZVFHMIN-NEXT:    vmfeq.vf v0, v12, fa5
 ; ZVFHMIN-NEXT:    ret
   %head = insertelement <vscale x 8 x half> poison, half %b, i32 0
   %splat = shufflevector <vscale x 8 x half> %head, <vscale x 8 x half> poison, <vscale x 8 x i32> zeroinitializer
@@ -1250,13 +1162,11 @@ define <vscale x 8 x i1> @fcmp_oeq_vf_nxv8f16_nonans(<vscale x 8 x half> %va, ha
 ;
 ; ZVFHMIN-LABEL: fcmp_oeq_vf_nxv8f16_nonans:
 ; ZVFHMIN:       # %bb.0:
-; ZVFHMIN-NEXT:    fmv.x.h a0, fa0
-; ZVFHMIN-NEXT:    vsetvli a1, zero, e16, m2, ta, ma
-; ZVFHMIN-NEXT:    vmv.v.x v10, a0
+; ZVFHMIN-NEXT:    fcvt.s.h fa5, fa0
+; ZVFHMIN-NEXT:    vsetvli a0, zero, e16, m2, ta, ma
 ; ZVFHMIN-NEXT:    vfwcvt.f.f.v v12, v8
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v16, v10
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
-; ZVFHMIN-NEXT:    vmfeq.vv v0, v12, v16
+; ZVFHMIN-NEXT:    vmfeq.vf v0, v12, fa5
 ; ZVFHMIN-NEXT:    ret
   %head = insertelement <vscale x 8 x half> poison, half %b, i32 0
   %splat = shufflevector <vscale x 8 x half> %head, <vscale x 8 x half> poison, <vscale x 8 x i32> zeroinitializer
@@ -1292,13 +1202,11 @@ define <vscale x 8 x i1> @fcmp_ogt_vf_nxv8f16(<vscale x 8 x half> %va, half %b)
 ;
 ; ZVFHMIN-LABEL: fcmp_ogt_vf_nxv8f16:
 ; ZVFHMIN:       # %bb.0:
-; ZVFHMIN-NEXT:    fmv.x.h a0, fa0
-; ZVFHMIN-NEXT:    vsetvli a1, zero, e16, m2, ta, ma
-; ZVFHMIN-NEXT:    vmv.v.x v10, a0
+; ZVFHMIN-NEXT:    fcvt.s.h fa5, fa0
+; ZVFHMIN-NEXT:    vsetvli a0, zero, e16, m2, ta, ma
 ; ZVFHMIN-NEXT:    vfwcvt.f.f.v v12, v8
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v16, v10
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
-; ZVFHMIN-NEXT:    vmflt.vv v0, v16, v12
+; ZVFHMIN-NEXT:    vmfgt.vf v0, v12, fa5
 ; ZVFHMIN-NEXT:    ret
   %head = insertelement <vscale x 8 x half> poison, half %b, i32 0
   %splat = shufflevector <vscale x 8 x half> %head, <vscale x 8 x half> poison, <vscale x 8 x i32> zeroinitializer
@@ -1315,13 +1223,11 @@ define <vscale x 8 x i1> @fcmp_ogt_fv_nxv8f16(<vscale x 8 x half> %va, half %b)
 ;
 ; ZVFHMIN-LABEL: fcmp_ogt_fv_nxv8f16:
 ; ZVFHMIN:       # %bb.0:
-; ZVFHMIN-NEXT:    fmv.x.h a0, fa0
-; ZVFHMIN-NEXT:    vsetvli a1, zero, e16, m2, ta, ma
-; ZVFHMIN-NEXT:    vmv.v.x v10, a0
+; ZVFHMIN-NEXT:    fcvt.s.h fa5, fa0
+; ZVFHMIN-NEXT:    vsetvli a0, zero, e16, m2, ta, ma
 ; ZVFHMIN-NEXT:    vfwcvt.f.f.v v12, v8
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v16, v10
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
-; ZVFHMIN-NEXT:    vmflt.vv v0, v12, v16
+; ZVFHMIN-NEXT:    vmflt.vf v0, v12, fa5
 ; ZVFHMIN-NEXT:    ret
   %head = insertelement <vscale x 8 x half> poison, half %b, i32 0
   %splat = shufflevector <vscale x 8 x half> %head, <vscale x 8 x half> poison, <vscale x 8 x i32> zeroinitializer
@@ -1357,13 +1263,11 @@ define <vscale x 8 x i1> @fcmp_ogt_vf_nxv8f16_nonans(<vscale x 8 x half> %va, ha
 ;
 ; ZVFHMIN-LABEL: fcmp_ogt_vf_nxv8f16_nonans:
 ; ZVFHMIN:       # %bb.0:
-; ZVFHMIN-NEXT:    fmv.x.h a0, fa0
-; ZVFHMIN-NEXT:    vsetvli a1, zero, e16, m2, ta, ma
-; ZVFHMIN-NEXT:    vmv.v.x v10, a0
+; ZVFHMIN-NEXT:    fcvt.s.h fa5, fa0
+; ZVFHMIN-NEXT:    vsetvli a0, zero, e16, m2, ta, ma
 ; ZVFHMIN-NEXT:    vfwcvt.f.f.v v12, v8
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v16, v10
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
-; ZVFHMIN-NEXT:    vmflt.vv v0, v16, v12
+; ZVFHMIN-NEXT:    vmfgt.vf v0, v12, fa5
 ; ZVFHMIN-NEXT:    ret
   %head = insertelement <vscale x 8 x half> poison, half %b, i32 0
   %splat = shufflevector <vscale x 8 x half> %head, <vscale x 8 x half> poison, <vscale x 8 x i32> zeroinitializer
@@ -1399,13 +1303,11 @@ define <vscale x 8 x i1> @fcmp_oge_vf_nxv8f16(<vscale x 8 x half> %va, half %b)
 ;
 ; ZVFHMIN-LABEL: fcmp_oge_vf_nxv8f16:
 ; ZVFHMIN:       # %bb.0:
-; ZVFHMIN-NEXT:    fmv.x.h a0, fa0
-; ZVFHMIN-NEXT:    vsetvli a1, zero, e16, m2, ta, ma
-; ZVFHMIN-NEXT:    vmv.v.x v10, a0
+; ZVFHMIN-NEXT:    fcvt.s.h fa5, fa0
+; ZVFHMIN-NEXT:    vsetvli a0, zero, e16, m2, ta, ma
 ; ZVFHMIN-NEXT:    vfwcvt.f.f.v v12, v8
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v16, v10
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
-; ZVFHMIN-NEXT:    vmfle.vv v0, v16, v12
+; ZVFHMIN-NEXT:    vmfge.vf v0, v12, fa5
 ; ZVFHMIN-NEXT:    ret
   %head = insertelement <vscale x 8 x half> poison, half %b, i32 0
   %splat = shufflevector <vscale x 8 x half> %head, <vscale x 8 x half> poison, <vscale x 8 x i32> zeroinitializer
@@ -1422,13 +1324,11 @@ define <vscale x 8 x i1> @fcmp_oge_fv_nxv8f16(<vscale x 8 x half> %va, half %b)
 ;
 ; ZVFHMIN-LABEL: fcmp_oge_fv_nxv8f16:
 ; ZVFHMIN:       # %bb.0:
-; ZVFHMIN-NEXT:    fmv.x.h a0, fa0
-; ZVFHMIN-NEXT:    vsetvli a1, zero, e16, m2, ta, ma
-; ZVFHMIN-NEXT:    vmv.v.x v10, a0
+; ZVFHMIN-NEXT:    fcvt.s.h fa5, fa0
+; ZVFHMIN-NEXT:    vsetvli a0, zero, e16, m2, ta, ma
 ; ZVFHMIN-NEXT:    vfwcvt.f.f.v v12, v8
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v16, v10
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
-; ZVFHMIN-NEXT:    vmfle.vv v0, v12, v16
+; ZVFHMIN-NEXT:    vmfle.vf v0, v12, fa5
 ; ZVFHMIN-NEXT:    ret
   %head = insertelement <vscale x 8 x half> poison, half %b, i32 0
   %splat = shufflevector <vscale x 8 x half> %head, <vscale x 8 x half> poison, <vscale x 8 x i32> zeroinitializer
@@ -1464,13 +1364,11 @@ define <vscale x 8 x i1> @fcmp_oge_vf_nxv8f16_nonans(<vscale x 8 x half> %va, ha
 ;
 ; ZVFHMIN-LABEL: fcmp_oge_vf_nxv8f16_nonans:
 ; ZVFHMIN:       # %bb.0:
-; ZVFHMIN-NEXT:    fmv.x.h a0, fa0
-; ZVFHMIN-NEXT:    vsetvli a1, zero, e16, m2, ta, ma
-; ZVFHMIN-NEXT:    vmv.v.x v10, a0
+; ZVFHMIN-NEXT:    fcvt.s.h fa5, fa0
+; ZVFHMIN-NEXT:    vsetvli a0, zero, e16, m2, ta, ma
 ; ZVFHMIN-NEXT:    vfwcvt.f.f.v v12, v8
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v16, v10
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
-; ZVFHMIN-NEXT:    vmfle.vv v0, v16, v12
+; ZVFHMIN-NEXT:    vmfge.vf v0, v12, fa5
 ; ZVFHMIN-NEXT:    ret
   %head = insertelement <vscale x 8 x half> poison, half %b, i32 0
   %splat = shufflevector <vscale x 8 x half> %head, <vscale x 8 x half> poison, <vscale x 8 x i32> zeroinitializer
@@ -1506,13 +1404,11 @@ define <vscale x 8 x i1> @fcmp_olt_vf_nxv8f16(<vscale x 8 x half> %va, half %b)
 ;
 ; ZVFHMIN-LABEL: fcmp_olt_vf_nxv8f16:
 ; ZVFHMIN:       # %bb.0:
-; ZVFHMIN-NEXT:    fmv.x.h a0, fa0
-; ZVFHMIN-NEXT:    vsetvli a1, zero, e16, m2, ta, ma
-; ZVFHMIN-NEXT:    vmv.v.x v10, a0
+; ZVFHMIN-NEXT:    fcvt.s.h fa5, fa0
+; ZVFHMIN-NEXT:    vsetvli a0, zero, e16, m2, ta, ma
 ; ZVFHMIN-NEXT:    vfwcvt.f.f.v v12, v8
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v16, v10
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
-; ZVFHMIN-NEXT:    vmflt.vv v0, v12, v16
+; ZVFHMIN-NEXT:    vmflt.vf v0, v12, fa5
 ; ZVFHMIN-NEXT:    ret
   %head = insertelement <vscale x 8 x half> poison, half %b, i32 0
   %splat = shufflevector <vscale x 8 x half> %head, <vscale x 8 x half> poison, <vscale x 8 x i32> zeroinitializer
@@ -1529,13 +1425,11 @@ define <vscale x 8 x i1> @fcmp_olt_fv_nxv8f16(<vscale x 8 x half> %va, half %b)
 ;
 ; ZVFHMIN-LABEL: fcmp_olt_fv_nxv8f16:
 ; ZVFHMIN:       # %bb.0:
-; ZVFHMIN-NEXT:    fmv.x.h a0, fa0
-; ZVFHMIN-NEXT:    vsetvli a1, zero, e16, m2, ta, ma
-; ZVFHMIN-NEXT:    vmv.v.x v10, a0
+; ZVFHMIN-NEXT:    fcvt.s.h fa5, fa0
+; ZVFHMIN-NEXT:    vsetvli a0, zero, e16, m2, ta, ma
 ; ZVFHMIN-NEXT:    vfwcvt.f.f.v v12, v8
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v16, v10
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
-; ZVFHMIN-NEXT:    vmflt.vv v0, v16, v12
+; ZVFHMIN-NEXT:    vmfgt.vf v0, v12, fa5
 ; ZVFHMIN-NEXT:    ret
   %head = insertelement <vscale x 8 x half> poison, half %b, i32 0
   %splat = shufflevector <vscale x 8 x half> %head, <vscale x 8 x half> poison, <vscale x 8 x i32> zeroinitializer
@@ -1571,13 +1465,11 @@ define <vscale x 8 x i1> @fcmp_olt_vf_nxv8f16_nonans(<vscale x 8 x half> %va, ha
 ;
 ; ZVFHMIN-LABEL: fcmp_olt_vf_nxv8f16_nonans:
 ; ZVFHMIN:       # %bb.0:
-; ZVFHMIN-NEXT:    fmv.x.h a0, fa0
-; ZVFHMIN-NEXT:    vsetvli a1, zero, e16, m2, ta, ma
-; ZVFHMIN-NEXT:    vmv.v.x v10, a0
+; ZVFHMIN-NEXT:    fcvt.s.h fa5, fa0
+; ZVFHMIN-NEXT:    vsetvli a0, zero, e16, m2, ta, ma
 ; ZVFHMIN-NEXT:    vfwcvt.f.f.v v12, v8
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v16, v10
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
-; ZVFHMIN-NEXT:    vmflt.vv v0, v12, v16
+; ZVFHMIN-NEXT:    vmflt.vf v0, v12, fa5
 ; ZVFHMIN-NEXT:    ret
   %head = insertelement <vscale x 8 x half> poison, half %b, i32 0
   %splat = shufflevector <vscale x 8 x half> %head, <vscale x 8 x half> poison, <vscale x 8 x i32> zeroinitializer
@@ -1613,13 +1505,11 @@ define <vscale x 8 x i1> @fcmp_ole_vf_nxv8f16(<vscale x 8 x half> %va, half %b)
 ;
 ; ZVFHMIN-LABEL: fcmp_ole_vf_nxv8f16:
 ; ZVFHMIN:       # %bb.0:
-; ZVFHMIN-NEXT:    fmv.x.h a0, fa0
-; ZVFHMIN-NEXT:    vsetvli a1, zero, e16, m2, ta, ma
-; ZVFHMIN-NEXT:    vmv.v.x v10, a0
+; ZVFHMIN-NEXT:    fcvt.s.h fa5, fa0
+; ZVFHMIN-NEXT:    vsetvli a0, zero, e16, m2, ta, ma
 ; ZVFHMIN-NEXT:    vfwcvt.f.f.v v12, v8
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v16, v10
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
-; ZVFHMIN-NEXT:    vmfle.vv v0, v12, v16
+; ZVFHMIN-NEXT:    vmfle.vf v0, v12, fa5
 ; ZVFHMIN-NEXT:    ret
   %head = insertelement <vscale x 8 x half> poison, half %b, i32 0
   %splat = shufflevector <vscale x 8 x half> %head, <vscale x 8 x half> poison, <vscale x 8 x i32> zeroinitializer
@@ -1636,13 +1526,11 @@ define <vscale x 8 x i1> @fcmp_ole_fv_nxv8f16(<vscale x 8 x half> %va, half %b)
 ;
 ; ZVFHMIN-LABEL: fcmp_ole_fv_nxv8f16:
 ; ZVFHMIN:       # %bb.0:
-; ZVFHMIN-NEXT:    fmv.x.h a0, fa0
-; ZVFHMIN-NEXT:    vsetvli a1, zero, e16, m2, ta, ma
-; ZVFHMIN-NEXT:    vmv.v.x v10, a0
+; ZVFHMIN-NEXT:    fcvt.s.h fa5, fa0
+; ZVFHMIN-NEXT:    vsetvli a0, zero, e16, m2, ta, ma
 ; ZVFHMIN-NEXT:    vfwcvt.f.f.v v12, v8
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v16, v10
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
-; ZVFHMIN-NEXT:    vmfle.vv v0, v16, v12
+; ZVFHMIN-NEXT:    vmfge.vf v0, v12, fa5
 ; ZVFHMIN-NEXT:    ret
   %head = insertelement <vscale x 8 x half> poison, half %b, i32 0
   %splat = shufflevector <vscale x 8 x half> %head, <vscale x 8 x half> poison, <vscale x 8 x i32> zeroinitializer
@@ -1678,13 +1566,11 @@ define <vscale x 8 x i1> @fcmp_ole_vf_nxv8f16_nonans(<vscale x 8 x half> %va, ha
 ;
 ; ZVFHMIN-LABEL: fcmp_ole_vf_nxv8f16_nonans:
 ; ZVFHMIN:       # %bb.0:
-; ZVFHMIN-NEXT:    fmv.x.h a0, fa0
-; ZVFHMIN-NEXT:    vsetvli a1, zero, e16, m2, ta, ma
-; ZVFHMIN-NEXT:    vmv.v.x v10, a0
+; ZVFHMIN-NEXT:    fcvt.s.h fa5, fa0
+; ZVFHMIN-NEXT:    vsetvli a0, zero, e16, m2, ta, ma
 ; ZVFHMIN-NEXT:    vfwcvt.f.f.v v12, v8
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v16, v10
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
-; ZVFHMIN-NEXT:    vmfle.vv v0, v12, v16
+; ZVFHMIN-NEXT:    vmfle.vf v0, v12, fa5
 ; ZVFHMIN-NEXT:    ret
   %head = insertelement <vscale x 8 x half> poison, half %b, i32 0
   %splat = shufflevector <vscale x 8 x half> %head, <vscale x 8 x half> poison, <vscale x 8 x i32> zeroinitializer
@@ -1726,14 +1612,12 @@ define <vscale x 8 x i1> @fcmp_one_vf_nxv8f16(<vscale x 8 x half> %va, half %b)
 ;
 ; ZVFHMIN-LABEL: fcmp_one_vf_nxv8f16:
 ; ZVFHMIN:       # %bb.0:
-; ZVFHMIN-NEXT:    fmv.x.h a0, fa0
-; ZVFHMIN-NEXT:    vsetvli a1, zero, e16, m2, ta, ma
+; ZVFHMIN-NEXT:    fcvt.s.h fa5, fa0
+; ZVFHMIN-NEXT:    vsetvli a0, zero, e16, m2, ta, ma
 ; ZVFHMIN-NEXT:    vfwcvt.f.f.v v12, v8
-; ZVFHMIN-NEXT:    vmv.v.x v8, a0
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v16, v8
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
-; ZVFHMIN-NEXT:    vmflt.vv v8, v12, v16
-; ZVFHMIN-NEXT:    vmflt.vv v9, v16, v12
+; ZVFHMIN-NEXT:    vmflt.vf v8, v12, fa5
+; ZVFHMIN-NEXT:    vmfgt.vf v9, v12, fa5
 ; ZVFHMIN-NEXT:    vmor.mm v0, v9, v8
 ; ZVFHMIN-NEXT:    ret
   %head = insertelement <vscale x 8 x half> poison, half %b, i32 0
@@ -1753,14 +1637,12 @@ define <vscale x 8 x i1> @fcmp_one_fv_nxv8f16(<vscale x 8 x half> %va, half %b)
 ;
 ; ZVFHMIN-LABEL: fcmp_one_fv_nxv8f16:
 ; ZVFHMIN:       # %bb.0:
-; ZVFHMIN-NEXT:    fmv.x.h a0, fa0
-; ZVFHMIN-NEXT:    vsetvli a1, zero, e16, m2, ta, ma
+; ZVFHMIN-NEXT:    fcvt.s.h fa5, fa0
+; ZVFHMIN-NEXT:    vsetvli a0, zero, e16, m2, ta, ma
 ; ZVFHMIN-NEXT:    vfwcvt.f.f.v v12, v8
-; ZVFHMIN-NEXT:    vmv.v.x v8, a0
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v16, v8
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
-; ZVFHMIN-NEXT:    vmflt.vv v8, v16, v12
-; ZVFHMIN-NEXT:    vmflt.vv v9, v12, v16
+; ZVFHMIN-NEXT:    vmfgt.vf v8, v12, fa5
+; ZVFHMIN-NEXT:    vmflt.vf v9, v12, fa5
 ; ZVFHMIN-NEXT:    vmor.mm v0, v9, v8
 ; ZVFHMIN-NEXT:    ret
   %head = insertelement <vscale x 8 x half> poison, half %b, i32 0
@@ -1797,13 +1679,11 @@ define <vscale x 8 x i1> @fcmp_one_vf_nxv8f16_nonans(<vscale x 8 x half> %va, ha
 ;
 ; ZVFHMIN-LABEL: fcmp_one_vf_nxv8f16_nonans:
 ; ZVFHMIN:       # %bb.0:
-; ZVFHMIN-NEXT:    fmv.x.h a0, fa0
-; ZVFHMIN-NEXT:    vsetvli a1, zero, e16, m2, ta, ma
-; ZVFHMIN-NEXT:    vmv.v.x v10, a0
+; ZVFHMIN-NEXT:    fcvt.s.h fa5, fa0
+; ZVFHMIN-NEXT:    vsetvli a0, zero, e16, m2, ta, ma
 ; ZVFHMIN-NEXT:    vfwcvt.f.f.v v12, v8
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v16, v10
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
-; ZVFHMIN-NEXT:    vmfne.vv v0, v12, v16
+; ZVFHMIN-NEXT:    vmfne.vf v0, v12, fa5
 ; ZVFHMIN-NEXT:    ret
   %head = insertelement <vscale x 8 x half> poison, half %b, i32 0
   %splat = shufflevector <vscale x 8 x half> %head, <vscale x 8 x half> poison, <vscale x 8 x i32> zeroinitializer
@@ -1848,17 +1728,15 @@ define <vscale x 8 x i1> @fcmp_ord_vf_nxv8f16(<vscale x 8 x half> %va, half %b)
 ;
 ; ZVFHMIN-LABEL: fcmp_ord_vf_nxv8f16:
 ; ZVFHMIN:       # %bb.0:
-; ZVFHMIN-NEXT:    fmv.x.h a0, fa0
-; ZVFHMIN-NEXT:    vsetvli a1, zero, e16, m2, ta, ma
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v12, v8
-; ZVFHMIN-NEXT:    vmv.v.x v8, a0
-; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
-; ZVFHMIN-NEXT:    vmfeq.vv v10, v12, v12
+; ZVFHMIN-NEXT:    fcvt.s.h fa5, fa0
+; ZVFHMIN-NEXT:    vsetvli a0, zero, e32, m4, ta, ma
+; ZVFHMIN-NEXT:    vfmv.v.f v12, fa5
+; ZVFHMIN-NEXT:    vmfeq.vf v10, v12, fa5
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e16, m2, ta, ma
 ; ZVFHMIN-NEXT:    vfwcvt.f.f.v v12, v8
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
 ; ZVFHMIN-NEXT:    vmfeq.vv v8, v12, v12
-; ZVFHMIN-NEXT:    vmand.mm v0, v10, v8
+; ZVFHMIN-NEXT:    vmand.mm v0, v8, v10
 ; ZVFHMIN-NEXT:    ret
   %head = insertelement <vscale x 8 x half> poison, half %b, i32 0
   %splat = shufflevector <vscale x 8 x half> %head, <vscale x 8 x half> poison, <vscale x 8 x i32> zeroinitializer
@@ -1878,17 +1756,15 @@ define <vscale x 8 x i1> @fcmp_ord_fv_nxv8f16(<vscale x 8 x half> %va, half %b)
 ;
 ; ZVFHMIN-LABEL: fcmp_ord_fv_nxv8f16:
 ; ZVFHMIN:       # %bb.0:
-; ZVFHMIN-NEXT:    fmv.x.h a0, fa0
-; ZVFHMIN-NEXT:    vsetvli a1, zero, e16, m2, ta, ma
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v12, v8
-; ZVFHMIN-NEXT:    vmv.v.x v8, a0
-; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
-; ZVFHMIN-NEXT:    vmfeq.vv v10, v12, v12
+; ZVFHMIN-NEXT:    fcvt.s.h fa5, fa0
+; ZVFHMIN-NEXT:    vsetvli a0, zero, e32, m4, ta, ma
+; ZVFHMIN-NEXT:    vfmv.v.f v12, fa5
+; ZVFHMIN-NEXT:    vmfeq.vf v10, v12, fa5
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e16, m2, ta, ma
 ; ZVFHMIN-NEXT:    vfwcvt.f.f.v v12, v8
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
 ; ZVFHMIN-NEXT:    vmfeq.vv v8, v12, v12
-; ZVFHMIN-NEXT:    vmand.mm v0, v8, v10
+; ZVFHMIN-NEXT:    vmand.mm v0, v10, v8
 ; ZVFHMIN-NEXT:    ret
   %head = insertelement <vscale x 8 x half> poison, half %b, i32 0
   %splat = shufflevector <vscale x 8 x half> %head, <vscale x 8 x half> poison, <vscale x 8 x i32> zeroinitializer
@@ -1933,17 +1809,15 @@ define <vscale x 8 x i1> @fcmp_ord_vf_nxv8f16_nonans(<vscale x 8 x half> %va, ha
 ;
 ; ZVFHMIN-LABEL: fcmp_ord_vf_nxv8f16_nonans:
 ; ZVFHMIN:       # %bb.0:
-; ZVFHMIN-NEXT:    fmv.x.h a0, fa0
-; ZVFHMIN-NEXT:    vsetvli a1, zero, e16, m2, ta, ma
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v12, v8
-; ZVFHMIN-NEXT:    vmv.v.x v8, a0
-; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
-; ZVFHMIN-NEXT:    vmfeq.vv v10, v12, v12
+; ZVFHMIN-NEXT:    fcvt.s.h fa5, fa0
+; ZVFHMIN-NEXT:    vsetvli a0, zero, e32, m4, ta, ma
+; ZVFHMIN-NEXT:    vfmv.v.f v12, fa5
+; ZVFHMIN-NEXT:    vmfeq.vf v10, v12, fa5
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e16, m2, ta, ma
 ; ZVFHMIN-NEXT:    vfwcvt.f.f.v v12, v8
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
 ; ZVFHMIN-NEXT:    vmfeq.vv v8, v12, v12
-; ZVFHMIN-NEXT:    vmand.mm v0, v10, v8
+; ZVFHMIN-NEXT:    vmand.mm v0, v8, v10
 ; ZVFHMIN-NEXT:    ret
   %head = insertelement <vscale x 8 x half> poison, half %b, i32 0
   %splat = shufflevector <vscale x 8 x half> %head, <vscale x 8 x half> poison, <vscale x 8 x i32> zeroinitializer
@@ -1985,14 +1859,12 @@ define <vscale x 8 x i1> @fcmp_ueq_vf_nxv8f16(<vscale x 8 x half> %va, half %b)
 ;
 ; ZVFHMIN-LABEL: fcmp_ueq_vf_nxv8f16:
 ; ZVFHMIN:       # %bb.0:
-; ZVFHMIN-NEXT:    fmv.x.h a0, fa0
-; ZVFHMIN-NEXT:    vsetvli a1, zero, e16, m2, ta, ma
+; ZVFHMIN-NEXT:    fcvt.s.h fa5, fa0
+; ZVFHMIN-NEXT:    vsetvli a0, zero, e16, m2, ta, ma
 ; ZVFHMIN-NEXT:    vfwcvt.f.f.v v12, v8
-; ZVFHMIN-NEXT:    vmv.v.x v8, a0
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v16, v8
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
-; ZVFHMIN-NEXT:    vmflt.vv v8, v12, v16
-; ZVFHMIN-NEXT:    vmflt.vv v9, v16, v12
+; ZVFHMIN-NEXT:    vmflt.vf v8, v12, fa5
+; ZVFHMIN-NEXT:    vmfgt.vf v9, v12, fa5
 ; ZVFHMIN-NEXT:    vmnor.mm v0, v9, v8
 ; ZVFHMIN-NEXT:    ret
   %head = insertelement <vscale x 8 x half> poison, half %b, i32 0
@@ -2012,14 +1884,12 @@ define <vscale x 8 x i1> @fcmp_ueq_fv_nxv8f16(<vscale x 8 x half> %va, half %b)
 ;
 ; ZVFHMIN-LABEL: fcmp_ueq_fv_nxv8f16:
 ; ZVFHMIN:       # %bb.0:
-; ZVFHMIN-NEXT:    fmv.x.h a0, fa0
-; ZVFHMIN-NEXT:    vsetvli a1, zero, e16, m2, ta, ma
+; ZVFHMIN-NEXT:    fcvt.s.h fa5, fa0
+; ZVFHMIN-NEXT:    vsetvli a0, zero, e16, m2, ta, ma
 ; ZVFHMIN-NEXT:    vfwcvt.f.f.v v12, v8
-; ZVFHMIN-NEXT:    vmv.v.x v8, a0
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v16, v8
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
-; ZVFHMIN-NEXT:    vmflt.vv v8, v16, v12
-; ZVFHMIN-NEXT:    vmflt.vv v9, v12, v16
+; ZVFHMIN-NEXT:    vmfgt.vf v8, v12, fa5
+; ZVFHMIN-NEXT:    vmflt.vf v9, v12, fa5
 ; ZVFHMIN-NEXT:    vmnor.mm v0, v9, v8
 ; ZVFHMIN-NEXT:    ret
   %head = insertelement <vscale x 8 x half> poison, half %b, i32 0
@@ -2056,13 +1926,11 @@ define <vscale x 8 x i1> @fcmp_ueq_vf_nxv8f16_nonans(<vscale x 8 x half> %va, ha
 ;
 ; ZVFHMIN-LABEL: fcmp_ueq_vf_nxv8f16_nonans:
 ; ZVFHMIN:       # %bb.0:
-; ZVFHMIN-NEXT:    fmv.x.h a0, fa0
-; ZVFHMIN-NEXT:    vsetvli a1, zero, e16, m2, ta, ma
-; ZVFHMIN-NEXT:    vmv.v.x v10, a0
+; ZVFHMIN-NEXT:    fcvt.s.h fa5, fa0
+; ZVFHMIN-NEXT:    vsetvli a0, zero, e16, m2, ta, ma
 ; ZVFHMIN-NEXT:    vfwcvt.f.f.v v12, v8
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v16, v10
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
-; ZVFHMIN-NEXT:    vmfeq.vv v0, v12, v16
+; ZVFHMIN-NEXT:    vmfeq.vf v0, v12, fa5
 ; ZVFHMIN-NEXT:    ret
   %head = insertelement <vscale x 8 x half> poison, half %b, i32 0
   %splat = shufflevector <vscale x 8 x half> %head, <vscale x 8 x half> poison, <vscale x 8 x i32> zeroinitializer
@@ -2101,13 +1969,11 @@ define <vscale x 8 x i1> @fcmp_ugt_vf_nxv8f16(<vscale x 8 x half> %va, half %b)
 ;
 ; ZVFHMIN-LABEL: fcmp_ugt_vf_nxv8f16:
 ; ZVFHMIN:       # %bb.0:
-; ZVFHMIN-NEXT:    fmv.x.h a0, fa0
-; ZVFHMIN-NEXT:    vsetvli a1, zero, e16, m2, ta, ma
-; ZVFHMIN-NEXT:    vmv.v.x v10, a0
+; ZVFHMIN-NEXT:    fcvt.s.h fa5, fa0
+; ZVFHMIN-NEXT:    vsetvli a0, zero, e16, m2, ta, ma
 ; ZVFHMIN-NEXT:    vfwcvt.f.f.v v12, v8
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v16, v10
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
-; ZVFHMIN-NEXT:    vmfle.vv v8, v12, v16
+; ZVFHMIN-NEXT:    vmfle.vf v8, v12, fa5
 ; ZVFHMIN-NEXT:    vmnot.m v0, v8
 ; ZVFHMIN-NEXT:    ret
   %head = insertelement <vscale x 8 x half> poison, half %b, i32 0
@@ -2126,13 +1992,11 @@ define <vscale x 8 x i1> @fcmp_ugt_fv_nxv8f16(<vscale x 8 x half> %va, half %b)
 ;
 ; ZVFHMIN-LABEL: fcmp_ugt_fv_nxv8f16:
 ; ZVFHMIN:       # %bb.0:
-; ZVFHMIN-NEXT:    fmv.x.h a0, fa0
-; ZVFHMIN-NEXT:    vsetvli a1, zero, e16, m2, ta, ma
-; ZVFHMIN-NEXT:    vmv.v.x v10, a0
+; ZVFHMIN-NEXT:    fcvt.s.h fa5, fa0
+; ZVFHMIN-NEXT:    vsetvli a0, zero, e16, m2, ta, ma
 ; ZVFHMIN-NEXT:    vfwcvt.f.f.v v12, v8
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v16, v10
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
-; ZVFHMIN-NEXT:    vmfle.vv v8, v16, v12
+; ZVFHMIN-NEXT:    vmfge.vf v8, v12, fa5
 ; ZVFHMIN-NEXT:    vmnot.m v0, v8
 ; ZVFHMIN-NEXT:    ret
   %head = insertelement <vscale x 8 x half> poison, half %b, i32 0
@@ -2169,13 +2033,11 @@ define <vscale x 8 x i1> @fcmp_ugt_vf_nxv8f16_nonans(<vscale x 8 x half> %va, ha
 ;
 ; ZVFHMIN-LABEL: fcmp_ugt_vf_nxv8f16_nonans:
 ; ZVFHMIN:       # %bb.0:
-; ZVFHMIN-NEXT:    fmv.x.h a0, fa0
-; ZVFHMIN-NEXT:    vsetvli a1, zero, e16, m2, ta, ma
-; ZVFHMIN-NEXT:    vmv.v.x v10, a0
+; ZVFHMIN-NEXT:    fcvt.s.h fa5, fa0
+; ZVFHMIN-NEXT:    vsetvli a0, zero, e16, m2, ta, ma
 ; ZVFHMIN-NEXT:    vfwcvt.f.f.v v12, v8
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v16, v10
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
-; ZVFHMIN-NEXT:    vmflt.vv v0, v16, v12
+; ZVFHMIN-NEXT:    vmfgt.vf v0, v12, fa5
 ; ZVFHMIN-NEXT:    ret
   %head = insertelement <vscale x 8 x half> poison, half %b, i32 0
   %splat = shufflevector <vscale x 8 x half> %head, <vscale x 8 x half> poison, <vscale x 8 x i32> zeroinitializer
@@ -2214,13 +2076,11 @@ define <vscale x 8 x i1> @fcmp_uge_vf_nxv8f16(<vscale x 8 x half> %va, half %b)
 ;
 ; ZVFHMIN-LABEL: fcmp_uge_vf_nxv8f16:
 ; ZVFHMIN:       # %bb.0:
-; ZVFHMIN-NEXT:    fmv.x.h a0, fa0
-; ZVFHMIN-NEXT:    vsetvli a1, zero, e16, m2, ta, ma
-; ZVFHMIN-NEXT:    vmv.v.x v10, a0
+; ZVFHMIN-NEXT:    fcvt.s.h fa5, fa0
+; ZVFHMIN-NEXT:    vsetvli a0, zero, e16, m2, ta, ma
 ; ZVFHMIN-NEXT:    vfwcvt.f.f.v v12, v8
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v16, v10
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
-; ZVFHMIN-NEXT:    vmflt.vv v8, v12, v16
+; ZVFHMIN-NEXT:    vmflt.vf v8, v12, fa5
 ; ZVFHMIN-NEXT:    vmnot.m v0, v8
 ; ZVFHMIN-NEXT:    ret
   %head = insertelement <vscale x 8 x half> poison, half %b, i32 0
@@ -2239,13 +2099,11 @@ define <vscale x 8 x i1> @fcmp_uge_fv_nxv8f16(<vscale x 8 x half> %va, half %b)
 ;
 ; ZVFHMIN-LABEL: fcmp_uge_fv_nxv8f16:
 ; ZVFHMIN:       # %bb.0:
-; ZVFHMIN-NEXT:    fmv.x.h a0, fa0
-; ZVFHMIN-NEXT:    vsetvli a1, zero, e16, m2, ta, ma
-; ZVFHMIN-NEXT:    vmv.v.x v10, a0
+; ZVFHMIN-NEXT:    fcvt.s.h fa5, fa0
+; ZVFHMIN-NEXT:    vsetvli a0, zero, e16, m2, ta, ma
 ; ZVFHMIN-NEXT:    vfwcvt.f.f.v v12, v8
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v16, v10
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
-; ZVFHMIN-NEXT:    vmflt.vv v8, v16, v12
+; ZVFHMIN-NEXT:    vmfgt.vf v8, v12, fa5
 ; ZVFHMIN-NEXT:    vmnot.m v0, v8
 ; ZVFHMIN-NEXT:    ret
   %head = insertelement <vscale x 8 x half> poison, half %b, i32 0
@@ -2282,13 +2140,11 @@ define <vscale x 8 x i1> @fcmp_uge_vf_nxv8f16_nonans(<vscale x 8 x half> %va, ha
 ;
 ; ZVFHMIN-LABEL: fcmp_uge_vf_nxv8f16_nonans:
 ; ZVFHMIN:       # %bb.0:
-; ZVFHMIN-NEXT:    fmv.x.h a0, fa0
-; ZVFHMIN-NEXT:    vsetvli a1, zero, e16, m2, ta, ma
-; ZVFHMIN-NEXT:    vmv.v.x v10, a0
+; ZVFHMIN-NEXT:    fcvt.s.h fa5, fa0
+; ZVFHMIN-NEXT:    vsetvli a0, zero, e16, m2, ta, ma
 ; ZVFHMIN-NEXT:    vfwcvt.f.f.v v12, v8
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v16, v10
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
-; ZVFHMIN-NEXT:    vmfle.vv v0, v16, v12
+; ZVFHMIN-NEXT:    vmfge.vf v0, v12, fa5
 ; ZVFHMIN-NEXT:    ret
   %head = insertelement <vscale x 8 x half> poison, half %b, i32 0
   %splat = shufflevector <vscale x 8 x half> %head, <vscale x 8 x half> poison, <vscale x 8 x i32> zeroinitializer
@@ -2327,13 +2183,11 @@ define <vscale x 8 x i1> @fcmp_ult_vf_nxv8f16(<vscale x 8 x half> %va, half %b)
 ;
 ; ZVFHMIN-LABEL: fcmp_ult_vf_nxv8f16:
 ; ZVFHMIN:       # %bb.0:
-; ZVFHMIN-NEXT:    fmv.x.h a0, fa0
-; ZVFHMIN-NEXT:    vsetvli a1, zero, e16, m2, ta, ma
-; ZVFHMIN-NEXT:    vmv.v.x v10, a0
+; ZVFHMIN-NEXT:    fcvt.s.h fa5, fa0
+; ZVFHMIN-NEXT:    vsetvli a0, zero, e16, m2, ta, ma
 ; ZVFHMIN-NEXT:    vfwcvt.f.f.v v12, v8
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v16, v10
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
-; ZVFHMIN-NEXT:    vmfle.vv v8, v16, v12
+; ZVFHMIN-NEXT:    vmfge.vf v8, v12, fa5
 ; ZVFHMIN-NEXT:    vmnot.m v0, v8
 ; ZVFHMIN-NEXT:    ret
   %head = insertelement <vscale x 8 x half> poison, half %b, i32 0
@@ -2352,13 +2206,11 @@ define <vscale x 8 x i1> @fcmp_ult_fv_nxv8f16(<vscale x 8 x half> %va, half %b)
 ;
 ; ZVFHMIN-LABEL: fcmp_ult_fv_nxv8f16:
 ; ZVFHMIN:       # %bb.0:
-; ZVFHMIN-NEXT:    fmv.x.h a0, fa0
-; ZVFHMIN-NEXT:    vsetvli a1, zero, e16, m2, ta, ma
-; ZVFHMIN-NEXT:    vmv.v.x v10, a0
+; ZVFHMIN-NEXT:    fcvt.s.h fa5, fa0
+; ZVFHMIN-NEXT:    vsetvli a0, zero, e16, m2, ta, ma
 ; ZVFHMIN-NEXT:    vfwcvt.f.f.v v12, v8
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v16, v10
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
-; ZVFHMIN-NEXT:    vmfle.vv v8, v12, v16
+; ZVFHMIN-NEXT:    vmfle.vf v8, v12, fa5
 ; ZVFHMIN-NEXT:    vmnot.m v0, v8
 ; ZVFHMIN-NEXT:    ret
   %head = insertelement <vscale x 8 x half> poison, half %b, i32 0
@@ -2395,13 +2247,11 @@ define <vscale x 8 x i1> @fcmp_ult_vf_nxv8f16_nonans(<vscale x 8 x half> %va, ha
 ;
 ; ZVFHMIN-LABEL: fcmp_ult_vf_nxv8f16_nonans:
 ; ZVFHMIN:       # %bb.0:
-; ZVFHMIN-NEXT:    fmv.x.h a0, fa0
-; ZVFHMIN-NEXT:    vsetvli a1, zero, e16, m2, ta, ma
-; ZVFHMIN-NEXT:    vmv.v.x v10, a0
+; ZVFHMIN-NEXT:    fcvt.s.h fa5, fa0
+; ZVFHMIN-NEXT:    vsetvli a0, zero, e16, m2, ta, ma
 ; ZVFHMIN-NEXT:    vfwcvt.f.f.v v12, v8
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v16, v10
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
-; ZVFHMIN-NEXT:    vmflt.vv v0, v12, v16
+; ZVFHMIN-NEXT:    vmflt.vf v0, v12, fa5
 ; ZVFHMIN-NEXT:    ret
   %head = insertelement <vscale x 8 x half> poison, half %b, i32 0
   %splat = shufflevector <vscale x 8 x half> %head, <vscale x 8 x half> poison, <vscale x 8 x i32> zeroinitializer
@@ -2440,13 +2290,11 @@ define <vscale x 8 x i1> @fcmp_ule_vf_nxv8f16(<vscale x 8 x half> %va, half %b)
 ;
 ; ZVFHMIN-LABEL: fcmp_ule_vf_nxv8f16:
 ; ZVFHMIN:       # %bb.0:
-; ZVFHMIN-NEXT:    fmv.x.h a0, fa0
-; ZVFHMIN-NEXT:    vsetvli a1, zero, e16, m2, ta, ma
-; ZVFHMIN-NEXT:    vmv.v.x v10, a0
+; ZVFHMIN-NEXT:    fcvt.s.h fa5, fa0
+; ZVFHMIN-NEXT:    vsetvli a0, zero, e16, m2, ta, ma
 ; ZVFHMIN-NEXT:    vfwcvt.f.f.v v12, v8
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v16, v10
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
-; ZVFHMIN-NEXT:    vmflt.vv v8, v16, v12
+; ZVFHMIN-NEXT:    vmfgt.vf v8, v12, fa5
 ; ZVFHMIN-NEXT:    vmnot.m v0, v8
 ; ZVFHMIN-NEXT:    ret
   %head = insertelement <vscale x 8 x half> poison, half %b, i32 0
@@ -2465,13 +2313,11 @@ define <vscale x 8 x i1> @fcmp_ule_fv_nxv8f16(<vscale x 8 x half> %va, half %b)
 ;
 ; ZVFHMIN-LABEL: fcmp_ule_fv_nxv8f16:
 ; ZVFHMIN:       # %bb.0:
-; ZVFHMIN-NEXT:    fmv.x.h a0, fa0
-; ZVFHMIN-NEXT:    vsetvli a1, zero, e16, m2, ta, ma
-; ZVFHMIN-NEXT:    vmv.v.x v10, a0
+; ZVFHMIN-NEXT:    fcvt.s.h fa5, fa0
+; ZVFHMIN-NEXT:    vsetvli a0, zero, e16, m2, ta, ma
 ; ZVFHMIN-NEXT:    vfwcvt.f.f.v v12, v8
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v16, v10
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
-; ZVFHMIN-NEXT:    vmflt.vv v8, v12, v16
+; ZVFHMIN-NEXT:    vmflt.vf v8, v12, fa5
 ; ZVFHMIN-NEXT:    vmnot.m v0, v8
 ; ZVFHMIN-NEXT:    ret
   %head = insertelement <vscale x 8 x half> poison, half %b, i32 0
@@ -2508,13 +2354,11 @@ define <vscale x 8 x i1> @fcmp_ule_vf_nxv8f16_nonans(<vscale x 8 x half> %va, ha
 ;
 ; ZVFHMIN-LABEL: fcmp_ule_vf_nxv8f16_nonans:
 ; ZVFHMIN:       # %bb.0:
-; ZVFHMIN-NEXT:    fmv.x.h a0, fa0
-; ZVFHMIN-NEXT:    vsetvli a1, zero, e16, m2, ta, ma
-; ZVFHMIN-NEXT:    vmv.v.x v10, a0
+; ZVFHMIN-NEXT:    fcvt.s.h fa5, fa0
+; ZVFHMIN-NEXT:    vsetvli a0, zero, e16, m2, ta, ma
 ; ZVFHMIN-NEXT:    vfwcvt.f.f.v v12, v8
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v16, v10
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
-; ZVFHMIN-NEXT:    vmfle.vv v0, v12, v16
+; ZVFHMIN-NEXT:    vmfle.vf v0, v12, fa5
 ; ZVFHMIN-NEXT:    ret
   %head = insertelement <vscale x 8 x half> poison, half %b, i32 0
   %splat = shufflevector <vscale x 8 x half> %head, <vscale x 8 x half> poison, <vscale x 8 x i32> zeroinitializer
@@ -2550,13 +2394,11 @@ define <vscale x 8 x i1> @fcmp_une_vf_nxv8f16(<vscale x 8 x half> %va, half %b)
 ;
 ; ZVFHMIN-LABEL: fcmp_une_vf_nxv8f16:
 ; ZVFHMIN:       # %bb.0:
-; ZVFHMIN-NEXT:    fmv.x.h a0, fa0
-; ZVFHMIN-NEXT:    vsetvli a1, zero, e16, m2, ta, ma
-; ZVFHMIN-NEXT:    vmv.v.x v10, a0
+; ZVFHMIN-NEXT:    fcvt.s.h fa5, fa0
+; ZVFHMIN-NEXT:    vsetvli a0, zero, e16, m2, ta, ma
 ; ZVFHMIN-NEXT:    vfwcvt.f.f.v v12, v8
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v16, v10
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
-; ZVFHMIN-NEXT:    vmfne.vv v0, v12, v16
+; ZVFHMIN-NEXT:    vmfne.vf v0, v12, fa5
 ; ZVFHMIN-NEXT:    ret
   %head = insertelement <vscale x 8 x half> poison, half %b, i32 0
   %splat = shufflevector <vscale x 8 x half> %head, <vscale x 8 x half> poison, <vscale x 8 x i32> zeroinitializer
@@ -2573,13 +2415,11 @@ define <vscale x 8 x i1> @fcmp_une_fv_nxv8f16(<vscale x 8 x half> %va, half %b)
 ;
 ; ZVFHMIN-LABEL: fcmp_une_fv_nxv8f16:
 ; ZVFHMIN:       # %bb.0:
-; ZVFHMIN-NEXT:    fmv.x.h a0, fa0
-; ZVFHMIN-NEXT:    vsetvli a1, zero, e16, m2, ta, ma
-; ZVFHMIN-NEXT:    vmv.v.x v10, a0
+; ZVFHMIN-NEXT:    fcvt.s.h fa5, fa0
+; ZVFHMIN-NEXT:    vsetvli a0, zero, e16, m2, ta, ma
 ; ZVFHMIN-NEXT:    vfwcvt.f.f.v v12, v8
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v16, v10
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
-; ZVFHMIN-NEXT:    vmfne.vv v0, v16, v12
+; ZVFHMIN-NEXT:    vmfne.vf v0, v12, fa5
 ; ZVFHMIN-NEXT:    ret
   %head = insertelement <vscale x 8 x half> poison, half %b, i32 0
   %splat = shufflevector <vscale x 8 x half> %head, <vscale x 8 x half> poison, <vscale x 8 x i32> zeroinitializer
@@ -2615,13 +2455,11 @@ define <vscale x 8 x i1> @fcmp_une_vf_nxv8f16_nonans(<vscale x 8 x half> %va, ha
 ;
 ; ZVFHMIN-LABEL: fcmp_une_vf_nxv8f16_nonans:
 ; ZVFHMIN:       # %bb.0:
-; ZVFHMIN-NEXT:    fmv.x.h a0, fa0
-; ZVFHMIN-NEXT:    vsetvli a1, zero, e16, m2, ta, ma
-; ZVFHMIN-NEXT:    vmv.v.x v10, a0
+; ZVFHMIN-NEXT:    fcvt.s.h fa5, fa0
+; ZVFHMIN-NEXT:    vsetvli a0, zero, e16, m2, ta, ma
 ; ZVFHMIN-NEXT:    vfwcvt.f.f.v v12, v8
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v16, v10
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
-; ZVFHMIN-NEXT:    vmfne.vv v0, v12, v16
+; ZVFHMIN-NEXT:    vmfne.vf v0, v12, fa5
 ; ZVFHMIN-NEXT:    ret
   %head = insertelement <vscale x 8 x half> poison, half %b, i32 0
   %splat = shufflevector <vscale x 8 x half> %head, <vscale x 8 x half> poison, <vscale x 8 x i32> zeroinitializer
@@ -2666,17 +2504,15 @@ define <vscale x 8 x i1> @fcmp_uno_vf_nxv8f16(<vscale x 8 x half> %va, half %b)
 ;
 ; ZVFHMIN-LABEL: fcmp_uno_vf_nxv8f16:
 ; ZVFHMIN:       # %bb.0:
-; ZVFHMIN-NEXT:    fmv.x.h a0, fa0
-; ZVFHMIN-NEXT:    vsetvli a1, zero, e16, m2, ta, ma
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v12, v8
-; ZVFHMIN-NEXT:    vmv.v.x v8, a0
-; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
-; ZVFHMIN-NEXT:    vmfne.vv v10, v12, v12
+; ZVFHMIN-NEXT:    fcvt.s.h fa5, fa0
+; ZVFHMIN-NEXT:    vsetvli a0, zero, e32, m4, ta, ma
+; ZVFHMIN-NEXT:    vfmv.v.f v12, fa5
+; ZVFHMIN-NEXT:    vmfne.vf v10, v12, fa5
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e16, m2, ta, ma
 ; ZVFHMIN-NEXT:    vfwcvt.f.f.v v12, v8
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
 ; ZVFHMIN-NEXT:    vmfne.vv v8, v12, v12
-; ZVFHMIN-NEXT:    vmor.mm v0, v10, v8
+; ZVFHMIN-NEXT:    vmor.mm v0, v8, v10
 ; ZVFHMIN-NEXT:    ret
   %head = insertelement <vscale x 8 x half> poison, half %b, i32 0
   %splat = shufflevector <vscale x 8 x half> %head, <vscale x 8 x half> poison, <vscale x 8 x i32> zeroinitializer
@@ -2696,17 +2532,15 @@ define <vscale x 8 x i1> @fcmp_uno_fv_nxv8f16(<vscale x 8 x half> %va, half %b)
 ;
 ; ZVFHMIN-LABEL: fcmp_uno_fv_nxv8f16:
 ; ZVFHMIN:       # %bb.0:
-; ZVFHMIN-NEXT:    fmv.x.h a0, fa0
-; ZVFHMIN-NEXT:    vsetvli a1, zero, e16, m2, ta, ma
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v12, v8
-; ZVFHMIN-NEXT:    vmv.v.x v8, a0
-; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
-; ZVFHMIN-NEXT:    vmfne.vv v10, v12, v12
+; ZVFHMIN-NEXT:    fcvt.s.h fa5, fa0
+; ZVFHMIN-NEXT:    vsetvli a0, zero, e32, m4, ta, ma
+; ZVFHMIN-NEXT:    vfmv.v.f v12, fa5
+; ZVFHMIN-NEXT:    vmfne.vf v10, v12, fa5
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e16, m2, ta, ma
 ; ZVFHMIN-NEXT:    vfwcvt.f.f.v v12, v8
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
 ; ZVFHMIN-NEXT:    vmfne.vv v8, v12, v12
-; ZVFHMIN-NEXT:    vmor.mm v0, v8, v10
+; ZVFHMIN-NEXT:    vmor.mm v0, v10, v8
 ; ZVFHMIN-NEXT:    ret
   %head = insertelement <vscale x 8 x half> poison, half %b, i32 0
   %splat = shufflevector <vscale x 8 x half> %head, <vscale x 8 x half> poison, <vscale x 8 x i32> zeroinitializer
@@ -2751,17 +2585,15 @@ define <vscale x 8 x i1> @fcmp_uno_vf_nxv8f16_nonans(<vscale x 8 x half> %va, ha
 ;
 ; ZVFHMIN-LABEL: fcmp_uno_vf_nxv8f16_nonans:
 ; ZVFHMIN:       # %bb.0:
-; ZVFHMIN-NEXT:    fmv.x.h a0, fa0
-; ZVFHMIN-NEXT:    vsetvli a1, zero, e16, m2, ta, ma
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v12, v8
-; ZVFHMIN-NEXT:    vmv.v.x v8, a0
-; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
-; ZVFHMIN-NEXT:    vmfne.vv v10, v12, v12
+; ZVFHMIN-NEXT:    fcvt.s.h fa5, fa0
+; ZVFHMIN-NEXT:    vsetvli a0, zero, e32, m4, ta, ma
+; ZVFHMIN-NEXT:    vfmv.v.f v12, fa5
+; ZVFHMIN-NEXT:    vmfne.vf v10, v12, fa5
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e16, m2, ta, ma
 ; ZVFHMIN-NEXT:    vfwcvt.f.f.v v12, v8
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
 ; ZVFHMIN-NEXT:    vmfne.vv v8, v12, v12
-; ZVFHMIN-NEXT:    vmor.mm v0, v10, v8
+; ZVFHMIN-NEXT:    vmor.mm v0, v8, v10
 ; ZVFHMIN-NEXT:    ret
   %head = insertelement <vscale x 8 x half> poison, half %b, i32 0
   %splat = shufflevector <vscale x 8 x half> %head, <vscale x 8 x half> poison, <vscale x 8 x i32> zeroinitializer
diff --git a/llvm/test/CodeGen/RISCV/rvv/sink-splat-operands.ll b/llvm/test/CodeGen/RISCV/rvv/sink-splat-operands.ll
index 735621aa4390e..1948675ae9cf0 100644
--- a/llvm/test/CodeGen/RISCV/rvv/sink-splat-operands.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/sink-splat-operands.ll
@@ -5741,3 +5741,152 @@ vector.body:
 for.cond.cleanup:
   ret void
 }
+
+define void @sink_splat_vfwadd_vf(ptr nocapture %a, ptr nocapture %b, float %f) {
+; CHECK-LABEL: sink_splat_vfwadd_vf:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    li a1, 0
+; CHECK-NEXT:    li a2, 1020
+; CHECK-NEXT:    vsetvli a3, zero, e32, m1, ta, ma
+; CHECK-NEXT:  .LBB125_1: # %vector.body
+; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vl1re32.v v8, (a0)
+; CHECK-NEXT:    addi a1, a1, 4
+; CHECK-NEXT:    addi a2, a2, -4
+; CHECK-NEXT:    vfwadd.vf v10, v8, fa0
+; CHECK-NEXT:    vs2r.v v10, (a0)
+; CHECK-NEXT:    addi a0, a0, 16
+; CHECK-NEXT:    j .LBB125_1
+entry:
+  %f.ext = fpext float %f to double
+  %broadcast.splatinsert = insertelement <vscale x 2 x double> poison, double %f.ext, i32 0
+  %broadcast.splat = shufflevector <vscale x 2 x double> %broadcast.splatinsert, <vscale x 2 x double> poison, <vscale x 2 x i32> zeroinitializer
+  br label %vector.body
+
+vector.body:                                      ; preds = %vector.body, %entry
+  %index = phi i64 [ 0, %entry ], [ %index.next, %vector.body ]
+  %0 = getelementptr float, ptr %a, i64 %index
+  %wide.load = load <vscale x 2 x float>, ptr %0
+  %ext = fpext <vscale x 2 x float> %wide.load to <vscale x 2 x double>
+  %1 = fadd <vscale x 2 x double> %ext, %broadcast.splat
+  %2 = getelementptr double, ptr %b, i64 %index
+  store <vscale x 2 x double> %1, ptr %0
+  %index.next = add i64 %index, 4
+  %3 = icmp eq i64 %index.next, 1024
+  br i1 32, label %for.cond.cleanup, label %vector.body
+
+for.cond.cleanup:                                 ; preds = %vector.body
+  ret void
+}
+
+define void @sink_splat_vfwadd_wf(ptr nocapture %a, ptr nocapture %b, float %f) {
+; CHECK-LABEL: sink_splat_vfwadd_wf:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    li a1, 0
+; CHECK-NEXT:    li a2, 1020
+; CHECK-NEXT:    vsetvli a3, zero, e32, m1, ta, ma
+; CHECK-NEXT:  .LBB126_1: # %vector.body
+; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vl2re64.v v8, (a0)
+; CHECK-NEXT:    addi a1, a1, 4
+; CHECK-NEXT:    addi a2, a2, -4
+; CHECK-NEXT:    vfwadd.wf v8, v8, fa0
+; CHECK-NEXT:    vs2r.v v8, (a0)
+; CHECK-NEXT:    addi a0, a0, 32
+; CHECK-NEXT:    j .LBB126_1
+entry:
+  %f.ext = fpext float %f to double
+  %broadcast.splatinsert = insertelement <vscale x 2 x double> poison, double %f.ext, i32 0
+  %broadcast.splat = shufflevector <vscale x 2 x double> %broadcast.splatinsert, <vscale x 2 x double> poison, <vscale x 2 x i32> zeroinitializer
+  br label %vector.body
+
+vector.body:                                      ; preds = %vector.body, %entry
+  %index = phi i64 [ 0, %entry ], [ %index.next, %vector.body ]
+  %0 = getelementptr double, ptr %a, i64 %index
+  %wide.load = load <vscale x 2 x double>, ptr %0
+  %1 = fadd <vscale x 2 x double> %wide.load, %broadcast.splat
+  %2 = getelementptr double, ptr %b, i64 %index
+  store <vscale x 2 x double> %1, ptr %0
+  %index.next = add i64 %index, 4
+  %3 = icmp eq i64 %index.next, 1024
+  br i1 32, label %for.cond.cleanup, label %vector.body
+
+for.cond.cleanup:                                 ; preds = %vector.body
+  ret void
+}
+
+define void @sink_splat_vfwmul_vf(ptr nocapture %a, ptr nocapture %b, float %f) {
+; CHECK-LABEL: sink_splat_vfwmul_vf:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    li a1, 0
+; CHECK-NEXT:    li a2, 1020
+; CHECK-NEXT:    vsetvli a3, zero, e32, m1, ta, ma
+; CHECK-NEXT:  .LBB127_1: # %vector.body
+; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vl1re32.v v8, (a0)
+; CHECK-NEXT:    addi a1, a1, 4
+; CHECK-NEXT:    addi a2, a2, -4
+; CHECK-NEXT:    vfwmul.vf v10, v8, fa0
+; CHECK-NEXT:    vs2r.v v10, (a0)
+; CHECK-NEXT:    addi a0, a0, 16
+; CHECK-NEXT:    j .LBB127_1
+entry:
+  %f.ext = fpext float %f to double
+  %broadcast.splatinsert = insertelement <vscale x 2 x double> poison, double %f.ext, i32 0
+  %broadcast.splat = shufflevector <vscale x 2 x double> %broadcast.splatinsert, <vscale x 2 x double> poison, <vscale x 2 x i32> zeroinitializer
+  br label %vector.body
+
+vector.body:                                      ; preds = %vector.body, %entry
+  %index = phi i64 [ 0, %entry ], [ %index.next, %vector.body ]
+  %0 = getelementptr float, ptr %a, i64 %index
+  %wide.load = load <vscale x 2 x float>, ptr %0
+  %ext = fpext <vscale x 2 x float> %wide.load to <vscale x 2 x double>
+  %1 = fmul <vscale x 2 x double> %ext, %broadcast.splat
+  %2 = getelementptr double, ptr %b, i64 %index
+  store <vscale x 2 x double> %1, ptr %0
+  %index.next = add i64 %index, 4
+  %3 = icmp eq i64 %index.next, 1024
+  br i1 32, label %for.cond.cleanup, label %vector.body
+
+for.cond.cleanup:                                 ; preds = %vector.body
+  ret void
+}
+
+; Even though there's no vfwmul.wf we'll sink the fcvt.d.s. Make sure
+; early-machinelicm undos the sink after isel.
+define void @sink_splat_vfwmul_wf(ptr nocapture %a, ptr nocapture %b, float %f) {
+; CHECK-LABEL: sink_splat_vfwmul_wf:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    li a1, 0
+; CHECK-NEXT:    li a2, 1020
+; CHECK-NEXT:    fcvt.d.s fa5, fa0
+; CHECK-NEXT:    vsetvli a3, zero, e64, m2, ta, ma
+; CHECK-NEXT:  .LBB128_1: # %vector.body
+; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vl2re64.v v8, (a0)
+; CHECK-NEXT:    addi a1, a1, 4
+; CHECK-NEXT:    addi a2, a2, -4
+; CHECK-NEXT:    vfmul.vf v8, v8, fa5
+; CHECK-NEXT:    vs2r.v v8, (a0)
+; CHECK-NEXT:    addi a0, a0, 16
+; CHECK-NEXT:    j .LBB128_1
+entry:
+  %f.ext = fpext float %f to double
+  %broadcast.splatinsert = insertelement <vscale x 2 x double> poison, double %f.ext, i32 0
+  %broadcast.splat = shufflevector <vscale x 2 x double> %broadcast.splatinsert, <vscale x 2 x double> poison, <vscale x 2 x i32> zeroinitializer
+  br label %vector.body
+
+vector.body:                                      ; preds = %vector.body, %entry
+  %index = phi i64 [ 0, %entry ], [ %index.next, %vector.body ]
+  %0 = getelementptr float, ptr %a, i64 %index
+  %wide.load = load <vscale x 2 x double>, ptr %0
+  %1 = fmul <vscale x 2 x double> %wide.load, %broadcast.splat
+  %2 = getelementptr double, ptr %b, i64 %index
+  store <vscale x 2 x double> %1, ptr %0
+  %index.next = add i64 %index, 4
+  %3 = icmp eq i64 %index.next, 1024
+  br i1 32, label %for.cond.cleanup, label %vector.body
+
+for.cond.cleanup:                                 ; preds = %vector.body
+  ret void
+}
diff --git a/llvm/test/CodeGen/RISCV/rvv/vector-deinterleave-fixed.ll b/llvm/test/CodeGen/RISCV/rvv/vector-deinterleave-fixed.ll
index 2fc5b40a89afa..9f0b2b3914836 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vector-deinterleave-fixed.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vector-deinterleave-fixed.ll
@@ -1,6 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s -mtriple=riscv32 -mattr=+v,+zvfh | FileCheck %s
-; RUN: llc < %s -mtriple=riscv64 -mattr=+v,+zvfh | FileCheck %s
+; RUN: llc < %s -mtriple=riscv32 -mattr=+v,+zvfh | FileCheck %s --check-prefixes=CHECK,RV32
+; RUN: llc < %s -mtriple=riscv64 -mattr=+v,+zvfh | FileCheck %s --check-prefixes=CHECK,RV64
 
 ; Integers
 
@@ -152,11 +152,268 @@ define {<8 x i64>, <8 x i64>} @vector_deinterleave_v8i64_v16i64(<16 x i64> %vec)
   ret {<8 x i64>, <8 x i64>} %retval
 }
 
-declare {<16 x i1>, <16 x i1>} @llvm.vector.deinterleave2.v32i1(<32 x i1>)
-declare {<16 x i8>, <16 x i8>} @llvm.vector.deinterleave2.v32i8(<32 x i8>)
-declare {<8 x i16>, <8 x i16>} @llvm.vector.deinterleave2.v16i16(<16 x i16>)
-declare {<4 x i32>, <4 x i32>} @llvm.vector.deinterleave2.v8i32(<8 x i32>)
-declare {<2 x i64>, <2 x i64>} @llvm.vector.deinterleave2.v4i64(<4 x i64>)
+define {<2 x i32>, <2 x i32>, <2 x i32>} @vector_deinterleave3_v2i32_v6i32(<6 x i32> %v) {
+; CHECK-LABEL: vector_deinterleave3_v2i32_v6i32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    addi sp, sp, -16
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    csrr a0, vlenb
+; CHECK-NEXT:    slli a0, a0, 1
+; CHECK-NEXT:    sub sp, sp, a0
+; CHECK-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x02, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 2 * vlenb
+; CHECK-NEXT:    csrr a0, vlenb
+; CHECK-NEXT:    vsetivli zero, 2, e32, m1, ta, ma
+; CHECK-NEXT:    vslidedown.vi v10, v8, 2
+; CHECK-NEXT:    vsetivli zero, 2, e32, m2, ta, ma
+; CHECK-NEXT:    vslidedown.vi v12, v8, 4
+; CHECK-NEXT:    srli a0, a0, 3
+; CHECK-NEXT:    add a1, a0, a0
+; CHECK-NEXT:    vsetvli zero, a1, e32, m1, ta, ma
+; CHECK-NEXT:    vslideup.vx v8, v10, a0
+; CHECK-NEXT:    addi a0, sp, 16
+; CHECK-NEXT:    vmv1r.v v9, v12
+; CHECK-NEXT:    vs2r.v v8, (a0)
+; CHECK-NEXT:    vsetvli a1, zero, e32, mf2, ta, ma
+; CHECK-NEXT:    vlseg3e32.v v8, (a0)
+; CHECK-NEXT:    csrr a0, vlenb
+; CHECK-NEXT:    slli a0, a0, 1
+; CHECK-NEXT:    add sp, sp, a0
+; CHECK-NEXT:    .cfi_def_cfa sp, 16
+; CHECK-NEXT:    addi sp, sp, 16
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-NEXT:    ret
+	   %res = call {<2 x i32>, <2 x i32>, <2 x i32>} @llvm.vector.deinterleave3.v6i32(<6 x i32> %v)
+	   ret {<2 x i32>, <2 x i32>, <2 x i32>} %res
+}
+
+
+define {<2 x i16>, <2 x i16>, <2 x i16>, <2 x i16>, <2 x i16>} @vector_deinterleave5_v2i16_v10i16(<10 x i16> %v) {
+; CHECK-LABEL: vector_deinterleave5_v2i16_v10i16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    addi sp, sp, -16
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    csrr a0, vlenb
+; CHECK-NEXT:    slli a0, a0, 1
+; CHECK-NEXT:    sub sp, sp, a0
+; CHECK-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x02, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 2 * vlenb
+; CHECK-NEXT:    csrr a0, vlenb
+; CHECK-NEXT:    vsetivli zero, 2, e16, m1, ta, ma
+; CHECK-NEXT:    vslidedown.vi v10, v8, 6
+; CHECK-NEXT:    vslidedown.vi v11, v8, 4
+; CHECK-NEXT:    vslidedown.vi v12, v8, 2
+; CHECK-NEXT:    vsetivli zero, 2, e16, m2, ta, ma
+; CHECK-NEXT:    vslidedown.vi v14, v8, 8
+; CHECK-NEXT:    srli a1, a0, 3
+; CHECK-NEXT:    srli a0, a0, 2
+; CHECK-NEXT:    add a2, a1, a1
+; CHECK-NEXT:    vsetvli zero, a2, e16, mf2, ta, ma
+; CHECK-NEXT:    vslideup.vx v11, v10, a1
+; CHECK-NEXT:    vslideup.vx v8, v12, a1
+; CHECK-NEXT:    add a1, a0, a0
+; CHECK-NEXT:    vsetvli zero, a1, e16, m1, ta, ma
+; CHECK-NEXT:    vslideup.vx v8, v11, a0
+; CHECK-NEXT:    addi a0, sp, 16
+; CHECK-NEXT:    vmv1r.v v9, v14
+; CHECK-NEXT:    vs2r.v v8, (a0)
+; CHECK-NEXT:    vsetvli a1, zero, e16, mf4, ta, ma
+; CHECK-NEXT:    vlseg5e16.v v8, (a0)
+; CHECK-NEXT:    csrr a0, vlenb
+; CHECK-NEXT:    slli a0, a0, 1
+; CHECK-NEXT:    add sp, sp, a0
+; CHECK-NEXT:    .cfi_def_cfa sp, 16
+; CHECK-NEXT:    addi sp, sp, 16
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-NEXT:    ret
+	   %res = call {<2 x i16>, <2 x i16>, <2 x i16>, <2 x i16>, <2 x i16>} @llvm.vector.deinterleave5.v10i16(<10 x i16> %v)
+	   ret {<2 x i16>, <2 x i16>, <2 x i16>, <2 x i16>, <2 x i16>} %res
+}
+
+define {<2 x i8>, <2 x i8>, <2 x i8>, <2 x i8>, <2 x i8>, <2 x i8>, <2 x i8>} @vector_deinterleave7_v14i8_v2i8(<14 x i8> %v) {
+; RV32-LABEL: vector_deinterleave7_v14i8_v2i8:
+; RV32:       # %bb.0:
+; RV32-NEXT:    addi sp, sp, -48
+; RV32-NEXT:    .cfi_def_cfa_offset 48
+; RV32-NEXT:    sw ra, 44(sp) # 4-byte Folded Spill
+; RV32-NEXT:    sw s0, 40(sp) # 4-byte Folded Spill
+; RV32-NEXT:    sw s1, 36(sp) # 4-byte Folded Spill
+; RV32-NEXT:    .cfi_offset ra, -4
+; RV32-NEXT:    .cfi_offset s0, -8
+; RV32-NEXT:    .cfi_offset s1, -12
+; RV32-NEXT:    csrr a0, vlenb
+; RV32-NEXT:    slli a0, a0, 2
+; RV32-NEXT:    sub sp, sp, a0
+; RV32-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x30, 0x22, 0x11, 0x04, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 48 + 4 * vlenb
+; RV32-NEXT:    addi a0, sp, 32
+; RV32-NEXT:    vs1r.v v8, (a0) # Unknown-size Folded Spill
+; RV32-NEXT:    csrr s1, vlenb
+; RV32-NEXT:    vsetivli zero, 2, e8, m1, ta, ma
+; RV32-NEXT:    vslidedown.vi v11, v8, 10
+; RV32-NEXT:    vslidedown.vi v10, v8, 8
+; RV32-NEXT:    vslidedown.vi v9, v8, 2
+; RV32-NEXT:    srli s0, s1, 3
+; RV32-NEXT:    add a0, s0, s0
+; RV32-NEXT:    vsetvli zero, a0, e8, mf2, tu, ma
+; RV32-NEXT:    vslideup.vx v10, v11, s0
+; RV32-NEXT:    vmv1r.v v11, v8
+; RV32-NEXT:    vslideup.vx v11, v9, s0
+; RV32-NEXT:    vsetivli zero, 2, e8, m1, ta, ma
+; RV32-NEXT:    vslidedown.vi v9, v8, 12
+; RV32-NEXT:    srli a0, s1, 2
+; RV32-NEXT:    add a1, a0, s0
+; RV32-NEXT:    vsetvli zero, a1, e8, mf2, tu, ma
+; RV32-NEXT:    vslideup.vx v10, v9, a0
+; RV32-NEXT:    csrr a2, vlenb
+; RV32-NEXT:    slli a2, a2, 1
+; RV32-NEXT:    add a2, sp, a2
+; RV32-NEXT:    addi a2, a2, 32
+; RV32-NEXT:    vs1r.v v10, (a2) # Unknown-size Folded Spill
+; RV32-NEXT:    vsetivli zero, 2, e8, m1, ta, ma
+; RV32-NEXT:    vslidedown.vi v9, v8, 4
+; RV32-NEXT:    vsetvli zero, a1, e8, mf2, tu, ma
+; RV32-NEXT:    vslideup.vx v11, v9, a0
+; RV32-NEXT:    csrr a0, vlenb
+; RV32-NEXT:    add a0, sp, a0
+; RV32-NEXT:    addi a0, a0, 32
+; RV32-NEXT:    vs1r.v v11, (a0) # Unknown-size Folded Spill
+; RV32-NEXT:    li a1, 3
+; RV32-NEXT:    mv a0, s0
+; RV32-NEXT:    call __mulsi3
+; RV32-NEXT:    add s0, a0, s0
+; RV32-NEXT:    addi a1, sp, 32
+; RV32-NEXT:    vl1r.v v8, (a1) # Unknown-size Folded Reload
+; RV32-NEXT:    vsetivli zero, 2, e8, m1, ta, ma
+; RV32-NEXT:    vslidedown.vi v8, v8, 6
+; RV32-NEXT:    srli s1, s1, 1
+; RV32-NEXT:    csrr a1, vlenb
+; RV32-NEXT:    add a1, sp, a1
+; RV32-NEXT:    addi a1, a1, 32
+; RV32-NEXT:    vl1r.v v9, (a1) # Unknown-size Folded Reload
+; RV32-NEXT:    vsetvli zero, s0, e8, mf2, ta, ma
+; RV32-NEXT:    vslideup.vx v9, v8, a0
+; RV32-NEXT:    add a0, s1, s1
+; RV32-NEXT:    csrr a1, vlenb
+; RV32-NEXT:    slli a1, a1, 1
+; RV32-NEXT:    add a1, sp, a1
+; RV32-NEXT:    addi a1, a1, 32
+; RV32-NEXT:    vl1r.v v8, (a1) # Unknown-size Folded Reload
+; RV32-NEXT:    vsetvli zero, a0, e8, m1, ta, ma
+; RV32-NEXT:    vslideup.vx v9, v8, s1
+; RV32-NEXT:    csrr a0, vlenb
+; RV32-NEXT:    slli a1, a0, 1
+; RV32-NEXT:    add a0, a1, a0
+; RV32-NEXT:    add a0, sp, a0
+; RV32-NEXT:    addi a0, a0, 32
+; RV32-NEXT:    vs1r.v v9, (a0)
+; RV32-NEXT:    vsetvli a1, zero, e8, mf8, ta, ma
+; RV32-NEXT:    vlseg7e8.v v8, (a0)
+; RV32-NEXT:    csrr a0, vlenb
+; RV32-NEXT:    slli a0, a0, 2
+; RV32-NEXT:    add sp, sp, a0
+; RV32-NEXT:    .cfi_def_cfa sp, 48
+; RV32-NEXT:    lw ra, 44(sp) # 4-byte Folded Reload
+; RV32-NEXT:    lw s0, 40(sp) # 4-byte Folded Reload
+; RV32-NEXT:    lw s1, 36(sp) # 4-byte Folded Reload
+; RV32-NEXT:    .cfi_restore ra
+; RV32-NEXT:    .cfi_restore s0
+; RV32-NEXT:    .cfi_restore s1
+; RV32-NEXT:    addi sp, sp, 48
+; RV32-NEXT:    .cfi_def_cfa_offset 0
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: vector_deinterleave7_v14i8_v2i8:
+; RV64:       # %bb.0:
+; RV64-NEXT:    addi sp, sp, -64
+; RV64-NEXT:    .cfi_def_cfa_offset 64
+; RV64-NEXT:    sd ra, 56(sp) # 8-byte Folded Spill
+; RV64-NEXT:    sd s0, 48(sp) # 8-byte Folded Spill
+; RV64-NEXT:    sd s1, 40(sp) # 8-byte Folded Spill
+; RV64-NEXT:    .cfi_offset ra, -8
+; RV64-NEXT:    .cfi_offset s0, -16
+; RV64-NEXT:    .cfi_offset s1, -24
+; RV64-NEXT:    csrr a0, vlenb
+; RV64-NEXT:    slli a0, a0, 2
+; RV64-NEXT:    sub sp, sp, a0
+; RV64-NEXT:    .cfi_escape 0x0f, 0x0e, 0x72, 0x00, 0x11, 0xc0, 0x00, 0x22, 0x11, 0x04, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 64 + 4 * vlenb
+; RV64-NEXT:    addi a0, sp, 32
+; RV64-NEXT:    vs1r.v v8, (a0) # Unknown-size Folded Spill
+; RV64-NEXT:    csrr s1, vlenb
+; RV64-NEXT:    vsetivli zero, 2, e8, m1, ta, ma
+; RV64-NEXT:    vslidedown.vi v11, v8, 10
+; RV64-NEXT:    vslidedown.vi v10, v8, 8
+; RV64-NEXT:    vslidedown.vi v9, v8, 2
+; RV64-NEXT:    srli s0, s1, 3
+; RV64-NEXT:    add a0, s0, s0
+; RV64-NEXT:    vsetvli zero, a0, e8, mf2, tu, ma
+; RV64-NEXT:    vslideup.vx v10, v11, s0
+; RV64-NEXT:    vmv1r.v v11, v8
+; RV64-NEXT:    vslideup.vx v11, v9, s0
+; RV64-NEXT:    vsetivli zero, 2, e8, m1, ta, ma
+; RV64-NEXT:    vslidedown.vi v9, v8, 12
+; RV64-NEXT:    srli a0, s1, 2
+; RV64-NEXT:    add a1, a0, s0
+; RV64-NEXT:    vsetvli zero, a1, e8, mf2, tu, ma
+; RV64-NEXT:    vslideup.vx v10, v9, a0
+; RV64-NEXT:    csrr a2, vlenb
+; RV64-NEXT:    slli a2, a2, 1
+; RV64-NEXT:    add a2, sp, a2
+; RV64-NEXT:    addi a2, a2, 32
+; RV64-NEXT:    vs1r.v v10, (a2) # Unknown-size Folded Spill
+; RV64-NEXT:    vsetivli zero, 2, e8, m1, ta, ma
+; RV64-NEXT:    vslidedown.vi v9, v8, 4
+; RV64-NEXT:    vsetvli zero, a1, e8, mf2, tu, ma
+; RV64-NEXT:    vslideup.vx v11, v9, a0
+; RV64-NEXT:    csrr a0, vlenb
+; RV64-NEXT:    add a0, sp, a0
+; RV64-NEXT:    addi a0, a0, 32
+; RV64-NEXT:    vs1r.v v11, (a0) # Unknown-size Folded Spill
+; RV64-NEXT:    li a1, 3
+; RV64-NEXT:    mv a0, s0
+; RV64-NEXT:    call __muldi3
+; RV64-NEXT:    add s0, a0, s0
+; RV64-NEXT:    addi a1, sp, 32
+; RV64-NEXT:    vl1r.v v8, (a1) # Unknown-size Folded Reload
+; RV64-NEXT:    vsetivli zero, 2, e8, m1, ta, ma
+; RV64-NEXT:    vslidedown.vi v8, v8, 6
+; RV64-NEXT:    srli s1, s1, 1
+; RV64-NEXT:    csrr a1, vlenb
+; RV64-NEXT:    add a1, sp, a1
+; RV64-NEXT:    addi a1, a1, 32
+; RV64-NEXT:    vl1r.v v9, (a1) # Unknown-size Folded Reload
+; RV64-NEXT:    vsetvli zero, s0, e8, mf2, ta, ma
+; RV64-NEXT:    vslideup.vx v9, v8, a0
+; RV64-NEXT:    add a0, s1, s1
+; RV64-NEXT:    csrr a1, vlenb
+; RV64-NEXT:    slli a1, a1, 1
+; RV64-NEXT:    add a1, sp, a1
+; RV64-NEXT:    addi a1, a1, 32
+; RV64-NEXT:    vl1r.v v8, (a1) # Unknown-size Folded Reload
+; RV64-NEXT:    vsetvli zero, a0, e8, m1, ta, ma
+; RV64-NEXT:    vslideup.vx v9, v8, s1
+; RV64-NEXT:    csrr a0, vlenb
+; RV64-NEXT:    slli a1, a0, 1
+; RV64-NEXT:    add a0, a1, a0
+; RV64-NEXT:    add a0, sp, a0
+; RV64-NEXT:    addi a0, a0, 32
+; RV64-NEXT:    vs1r.v v9, (a0)
+; RV64-NEXT:    vsetvli a1, zero, e8, mf8, ta, ma
+; RV64-NEXT:    vlseg7e8.v v8, (a0)
+; RV64-NEXT:    csrr a0, vlenb
+; RV64-NEXT:    slli a0, a0, 2
+; RV64-NEXT:    add sp, sp, a0
+; RV64-NEXT:    .cfi_def_cfa sp, 64
+; RV64-NEXT:    ld ra, 56(sp) # 8-byte Folded Reload
+; RV64-NEXT:    ld s0, 48(sp) # 8-byte Folded Reload
+; RV64-NEXT:    ld s1, 40(sp) # 8-byte Folded Reload
+; RV64-NEXT:    .cfi_restore ra
+; RV64-NEXT:    .cfi_restore s0
+; RV64-NEXT:    .cfi_restore s1
+; RV64-NEXT:    addi sp, sp, 64
+; RV64-NEXT:    .cfi_def_cfa_offset 0
+; RV64-NEXT:    ret
+	   %res = call {<2 x i8>, <2 x i8>, <2 x i8>, <2 x i8>, <2 x i8>, <2 x i8>, <2 x i8>} @llvm.vector.deinterleave7.v14i8(<14 x i8> %v)
+	   ret {<2 x i8>, <2 x i8>, <2 x i8>, <2 x i8>, <2 x i8>, <2 x i8>, <2 x i8>} %res
+}
+
 
 ; Floats
 
@@ -267,9 +524,125 @@ define {<4 x double>, <4 x double>} @vector_deinterleave_v4f64_v8f64(<8 x double
 ret {<4 x double>, <4 x double>} %retval
 }
 
-declare {<2 x half>,<2 x half>} @llvm.vector.deinterleave2.v4f16(<4 x half>)
-declare {<4 x half>, <4 x half>} @llvm.vector.deinterleave2.v8f16(<8 x half>)
-declare {<2 x float>, <2 x float>} @llvm.vector.deinterleave2.v4f32(<4 x float>)
-declare {<8 x half>, <8 x half>} @llvm.vector.deinterleave2.v16f16(<16 x half>)
-declare {<4 x float>, <4 x float>} @llvm.vector.deinterleave2.v8f32(<8 x float>)
-declare {<2 x double>, <2 x double>} @llvm.vector.deinterleave2.v4f64(<4 x double>)
+define {<2 x float>, <2 x float>, <2 x float>} @vector_deinterleave3_v632_v2f32(<6 x float> %v) {
+; CHECK-LABEL: vector_deinterleave3_v632_v2f32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    addi sp, sp, -16
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    csrr a0, vlenb
+; CHECK-NEXT:    slli a0, a0, 1
+; CHECK-NEXT:    sub sp, sp, a0
+; CHECK-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x02, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 2 * vlenb
+; CHECK-NEXT:    csrr a0, vlenb
+; CHECK-NEXT:    vsetivli zero, 2, e32, m1, ta, ma
+; CHECK-NEXT:    vslidedown.vi v10, v8, 2
+; CHECK-NEXT:    vsetivli zero, 2, e32, m2, ta, ma
+; CHECK-NEXT:    vslidedown.vi v12, v8, 4
+; CHECK-NEXT:    srli a0, a0, 3
+; CHECK-NEXT:    add a1, a0, a0
+; CHECK-NEXT:    vsetvli zero, a1, e32, m1, ta, ma
+; CHECK-NEXT:    vslideup.vx v8, v10, a0
+; CHECK-NEXT:    addi a0, sp, 16
+; CHECK-NEXT:    vmv1r.v v9, v12
+; CHECK-NEXT:    vs2r.v v8, (a0)
+; CHECK-NEXT:    vsetvli a1, zero, e32, mf2, ta, ma
+; CHECK-NEXT:    vlseg3e32.v v8, (a0)
+; CHECK-NEXT:    csrr a0, vlenb
+; CHECK-NEXT:    slli a0, a0, 1
+; CHECK-NEXT:    add sp, sp, a0
+; CHECK-NEXT:    .cfi_def_cfa sp, 16
+; CHECK-NEXT:    addi sp, sp, 16
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-NEXT:    ret
+	   %res = call {<2 x float>, <2 x float>, <2 x float>} @llvm.vector.deinterleave3.v6f32(<6 x float> %v)
+	   ret {<2 x float>, <2 x float>, <2 x float>} %res
+}
+
+
+define {<2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>} @vector_deinterleave5_v10f16_v2f16(<10 x half> %v) {
+; CHECK-LABEL: vector_deinterleave5_v10f16_v2f16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    addi sp, sp, -16
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    csrr a0, vlenb
+; CHECK-NEXT:    slli a0, a0, 1
+; CHECK-NEXT:    sub sp, sp, a0
+; CHECK-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x02, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 2 * vlenb
+; CHECK-NEXT:    csrr a0, vlenb
+; CHECK-NEXT:    vsetivli zero, 2, e16, m1, ta, ma
+; CHECK-NEXT:    vslidedown.vi v10, v8, 6
+; CHECK-NEXT:    vslidedown.vi v11, v8, 4
+; CHECK-NEXT:    vslidedown.vi v12, v8, 2
+; CHECK-NEXT:    vsetivli zero, 2, e16, m2, ta, ma
+; CHECK-NEXT:    vslidedown.vi v14, v8, 8
+; CHECK-NEXT:    srli a1, a0, 3
+; CHECK-NEXT:    srli a0, a0, 2
+; CHECK-NEXT:    add a2, a1, a1
+; CHECK-NEXT:    vsetvli zero, a2, e16, mf2, ta, ma
+; CHECK-NEXT:    vslideup.vx v11, v10, a1
+; CHECK-NEXT:    vslideup.vx v8, v12, a1
+; CHECK-NEXT:    add a1, a0, a0
+; CHECK-NEXT:    vsetvli zero, a1, e16, m1, ta, ma
+; CHECK-NEXT:    vslideup.vx v8, v11, a0
+; CHECK-NEXT:    addi a0, sp, 16
+; CHECK-NEXT:    vmv1r.v v9, v14
+; CHECK-NEXT:    vs2r.v v8, (a0)
+; CHECK-NEXT:    vsetvli a1, zero, e16, mf4, ta, ma
+; CHECK-NEXT:    vlseg5e16.v v8, (a0)
+; CHECK-NEXT:    csrr a0, vlenb
+; CHECK-NEXT:    slli a0, a0, 1
+; CHECK-NEXT:    add sp, sp, a0
+; CHECK-NEXT:    .cfi_def_cfa sp, 16
+; CHECK-NEXT:    addi sp, sp, 16
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-NEXT:    ret
+	   %res = call {<2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>} @llvm.vector.deinterleave5.v10f16(<10 x half> %v)
+	   ret {<2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>} %res
+}
+
+define {<1 x half>, <1 x half>, <1 x half>, <1 x half>, <1 x half>, <1 x half>, <1 x half>} @vector_deinterleave7_v7f16_v1f16(<7 x half> %v) {
+; CHECK-LABEL: vector_deinterleave7_v7f16_v1f16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    addi sp, sp, -16
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    csrr a0, vlenb
+; CHECK-NEXT:    slli a0, a0, 1
+; CHECK-NEXT:    sub sp, sp, a0
+; CHECK-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x02, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 2 * vlenb
+; CHECK-NEXT:    csrr a0, vlenb
+; CHECK-NEXT:    vsetivli zero, 1, e16, m1, ta, ma
+; CHECK-NEXT:    vslidedown.vi v9, v8, 3
+; CHECK-NEXT:    vslidedown.vi v10, v8, 2
+; CHECK-NEXT:    vslidedown.vi v11, v8, 1
+; CHECK-NEXT:    vmv1r.v v12, v8
+; CHECK-NEXT:    vslidedown.vi v14, v8, 5
+; CHECK-NEXT:    vslidedown.vi v15, v8, 6
+; CHECK-NEXT:    srli a1, a0, 3
+; CHECK-NEXT:    srli a0, a0, 2
+; CHECK-NEXT:    add a2, a1, a1
+; CHECK-NEXT:    add a3, a0, a0
+; CHECK-NEXT:    vsetvli zero, a2, e16, mf2, ta, ma
+; CHECK-NEXT:    vslideup.vx v10, v9, a1
+; CHECK-NEXT:    vslideup.vx v12, v11, a1
+; CHECK-NEXT:    vsetvli zero, a3, e16, m1, ta, ma
+; CHECK-NEXT:    vslideup.vx v12, v10, a0
+; CHECK-NEXT:    vsetivli zero, 1, e16, m1, ta, ma
+; CHECK-NEXT:    vslidedown.vi v13, v8, 4
+; CHECK-NEXT:    vsetvli zero, a2, e16, mf2, ta, ma
+; CHECK-NEXT:    vslideup.vx v13, v14, a1
+; CHECK-NEXT:    vsetvli zero, a3, e16, m1, ta, ma
+; CHECK-NEXT:    vslideup.vx v13, v15, a0
+; CHECK-NEXT:    addi a0, sp, 16
+; CHECK-NEXT:    vs2r.v v12, (a0)
+; CHECK-NEXT:    vsetvli a1, zero, e16, mf4, ta, ma
+; CHECK-NEXT:    vlseg7e16.v v8, (a0)
+; CHECK-NEXT:    csrr a0, vlenb
+; CHECK-NEXT:    slli a0, a0, 1
+; CHECK-NEXT:    add sp, sp, a0
+; CHECK-NEXT:    .cfi_def_cfa sp, 16
+; CHECK-NEXT:    addi sp, sp, 16
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-NEXT:    ret
+	   %res = call {<1 x half>, <1 x half>, <1 x half>, <1 x half>, <1 x half>, <1 x half>, <1 x half>} @llvm.vector.deinterleave7.v7f16(<7 x half> %v)
+	   ret {<1 x half>, <1 x half>, <1 x half>, <1 x half>, <1 x half>, <1 x half>, <1 x half>} %res
+}
diff --git a/llvm/test/CodeGen/RISCV/rvv/vector-deinterleave.ll b/llvm/test/CodeGen/RISCV/rvv/vector-deinterleave.ll
index bb71c2973bb57..81b6de9e662d5 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vector-deinterleave.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vector-deinterleave.ll
@@ -1,8 +1,8 @@
-; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s -mtriple=riscv32 -mattr=+m,+v,+zvfh,+zvfbfmin | FileCheck %s
-; RUN: llc < %s -mtriple=riscv64 -mattr=+m,+v,+zvfh,+zvfbfmin | FileCheck %s
-; RUN: llc < %s -mtriple=riscv32 -mattr=+m,+v,+zvfhmin,+zvfbfmin | FileCheck %s
-; RUN: llc < %s -mtriple=riscv64 -mattr=+m,+v,+zvfhmin,+zvfbfmin | FileCheck %s
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc < %s -mtriple=riscv32 -mattr=+m,+v,+zvfh,+zvfbfmin | FileCheck %s --check-prefixes=CHECK,RV32
+; RUN: llc < %s -mtriple=riscv64 -mattr=+m,+v,+zvfh,+zvfbfmin | FileCheck %s --check-prefixes=CHECK,RV64
+; RUN: llc < %s -mtriple=riscv32 -mattr=+m,+v,+zvfhmin,+zvfbfmin | FileCheck %s --check-prefixes=CHECK,RV32
+; RUN: llc < %s -mtriple=riscv64 -mattr=+m,+v,+zvfhmin,+zvfbfmin | FileCheck %s --check-prefixes=CHECK,RV64
 
 ; Integers
 
@@ -104,11 +104,6 @@ define {<vscale x 4 x i64>, <vscale x 4 x i64>} @vector_deinterleave_nxv4i64_nxv
 ret {<vscale x 4 x i64>, <vscale x 4 x i64>} %retval
 }
 
-declare {<vscale x 16 x i1>, <vscale x 16 x i1>} @llvm.vector.deinterleave2.nxv32i1(<vscale x 32 x i1>)
-declare {<vscale x 16 x i8>, <vscale x 16 x i8>} @llvm.vector.deinterleave2.nxv32i8(<vscale x 32 x i8>)
-declare {<vscale x 8 x i16>, <vscale x 8 x i16>} @llvm.vector.deinterleave2.nxv16i16(<vscale x 16 x i16>)
-declare {<vscale x 4 x i32>, <vscale x 4 x i32>} @llvm.vector.deinterleave2.nxv8i32(<vscale x 8 x i32>)
-declare {<vscale x 2 x i64>, <vscale x 2 x i64>} @llvm.vector.deinterleave2.nxv4i64(<vscale x 4 x i64>)
 
 define {<vscale x 64 x i1>, <vscale x 64 x i1>} @vector_deinterleave_nxv64i1_nxv128i1(<vscale x 128 x i1> %vec) {
 ; CHECK-LABEL: vector_deinterleave_nxv64i1_nxv128i1:
@@ -228,11 +223,6 @@ define {<vscale x 8 x i64>, <vscale x 8 x i64>} @vector_deinterleave_nxv8i64_nxv
 ret {<vscale x 8 x i64>, <vscale x 8 x i64>} %retval
 }
 
-declare {<vscale x 64 x i1>, <vscale x 64 x i1>} @llvm.vector.deinterleave2.nxv128i1(<vscale x 128 x i1>)
-declare {<vscale x 64 x i8>, <vscale x 64 x i8>} @llvm.vector.deinterleave2.nxv128i8(<vscale x 128 x i8>)
-declare {<vscale x 32 x i16>, <vscale x 32 x i16>} @llvm.vector.deinterleave2.nxv64i16(<vscale x 64 x i16>)
-declare {<vscale x 16 x i32>, <vscale x 16 x i32>} @llvm.vector.deinterleave2.nxv32i32(<vscale x 32 x i32>)
-declare {<vscale x 8 x i64>, <vscale x 8 x i64>} @llvm.vector.deinterleave2.nxv16i64(<vscale x 16 x i64>)
 
 ; Floats
 
@@ -358,12 +348,6 @@ define {<vscale x 2 x double>, <vscale x 2 x double>} @vector_deinterleave_nxv2f
 ret {<vscale x 2 x double>, <vscale x 2 x double>} %retval
 }
 
-declare {<vscale x 2 x half>,<vscale x 2 x half>} @llvm.vector.deinterleave2.nxv4f16(<vscale x 4 x half>)
-declare {<vscale x 4 x half>, <vscale x 4 x half>} @llvm.vector.deinterleave2.nxv8f16(<vscale x 8 x half>)
-declare {<vscale x 2 x float>, <vscale x 2 x float>} @llvm.vector.deinterleave2.nxv4f32(<vscale x 4 x float>)
-declare {<vscale x 8 x half>, <vscale x 8 x half>} @llvm.vector.deinterleave2.nxv16f16(<vscale x 16 x half>)
-declare {<vscale x 4 x float>, <vscale x 4 x float>} @llvm.vector.deinterleave2.nxv8f32(<vscale x 8 x float>)
-declare {<vscale x 2 x double>, <vscale x 2 x double>} @llvm.vector.deinterleave2.nxv4f64(<vscale x 4 x double>)
 
 define {<vscale x 32 x bfloat>, <vscale x 32 x bfloat>} @vector_deinterleave_nxv32bf16_nxv64bf16(<vscale x 64 x bfloat> %vec) {
 ; CHECK-LABEL: vector_deinterleave_nxv32bf16_nxv64bf16:
@@ -461,6 +445,629 @@ define {<vscale x 8 x double>, <vscale x 8 x double>} @vector_deinterleave_nxv8f
 ret {<vscale x 8 x double>, <vscale x 8 x double>} %retval
 }
 
-declare {<vscale x 32 x half>, <vscale x 32 x half>} @llvm.vector.deinterleave2.nxv64f16(<vscale x 64 x half>)
-declare {<vscale x 16 x float>, <vscale x 16 x float>} @llvm.vector.deinterleave2.nxv32f32(<vscale x 32 x float>)
-declare {<vscale x 8 x double>, <vscale x 8 x double>} @llvm.vector.deinterleave2.nxv16f64(<vscale x 16 x double>)
+define {<vscale x 16 x i1>, <vscale x 16 x i1>, <vscale x 16 x i1>} @vector_deinterleave_nxv16i1_nxv48i1(<vscale x 48 x i1> %vec) nounwind {
+; CHECK-LABEL: vector_deinterleave_nxv16i1_nxv48i1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    addi sp, sp, -16
+; CHECK-NEXT:    csrr a0, vlenb
+; CHECK-NEXT:    slli a0, a0, 3
+; CHECK-NEXT:    sub sp, sp, a0
+; CHECK-NEXT:    vsetvli a0, zero, e8, m2, ta, ma
+; CHECK-NEXT:    vmv.v.i v10, 0
+; CHECK-NEXT:    csrr a0, vlenb
+; CHECK-NEXT:    srli a1, a0, 2
+; CHECK-NEXT:    vsetvli a2, zero, e8, m1, ta, ma
+; CHECK-NEXT:    vslidedown.vx v8, v0, a1
+; CHECK-NEXT:    addi a1, sp, 16
+; CHECK-NEXT:    vsetvli a2, zero, e8, m2, ta, ma
+; CHECK-NEXT:    vmerge.vim v16, v10, 1, v0
+; CHECK-NEXT:    srli a0, a0, 1
+; CHECK-NEXT:    vsetvli a2, zero, e8, m1, ta, ma
+; CHECK-NEXT:    vslidedown.vx v9, v0, a0
+; CHECK-NEXT:    vmv1r.v v0, v8
+; CHECK-NEXT:    vsetvli a0, zero, e8, m2, ta, ma
+; CHECK-NEXT:    vmerge.vim v18, v10, 1, v0
+; CHECK-NEXT:    vmv1r.v v0, v9
+; CHECK-NEXT:    vmerge.vim v20, v10, 1, v0
+; CHECK-NEXT:    vs8r.v v16, (a1)
+; CHECK-NEXT:    vlseg3e8.v v8, (a1)
+; CHECK-NEXT:    vmsne.vi v0, v8, 0
+; CHECK-NEXT:    vmsne.vi v8, v10, 0
+; CHECK-NEXT:    vmsne.vi v9, v12, 0
+; CHECK-NEXT:    csrr a0, vlenb
+; CHECK-NEXT:    slli a0, a0, 3
+; CHECK-NEXT:    add sp, sp, a0
+; CHECK-NEXT:    addi sp, sp, 16
+; CHECK-NEXT:    ret
+  %retval = call {<vscale x 16 x i1>, <vscale x 16 x i1>, <vscale x 16 x i1>} @llvm.vector.deinterleave3.nxv48i1(<vscale x 48 x i1> %vec)
+  ret {<vscale x 16 x i1>, <vscale x 16 x i1>, <vscale x 16 x i1>} %retval
+}
+
+
+define {<vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>} @vector_deinterleave_nxv16i8_nxv48i8(<vscale x 48 x i8> %vec) nounwind {
+; CHECK-LABEL: vector_deinterleave_nxv16i8_nxv48i8:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    addi sp, sp, -16
+; CHECK-NEXT:    csrr a0, vlenb
+; CHECK-NEXT:    slli a0, a0, 3
+; CHECK-NEXT:    sub sp, sp, a0
+; CHECK-NEXT:    addi a0, sp, 16
+; CHECK-NEXT:    vs8r.v v8, (a0)
+; CHECK-NEXT:    vsetvli a1, zero, e8, m2, ta, ma
+; CHECK-NEXT:    vlseg3e8.v v8, (a0)
+; CHECK-NEXT:    csrr a0, vlenb
+; CHECK-NEXT:    slli a0, a0, 3
+; CHECK-NEXT:    add sp, sp, a0
+; CHECK-NEXT:    addi sp, sp, 16
+; CHECK-NEXT:    ret
+  %retval = call {<vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>} @llvm.vector.deinterleave3.nxv48i8(<vscale x 48 x i8> %vec)
+  ret {<vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>} %retval
+}
+
+
+define {<vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>} @vector_deinterleave_nxv8i16_nxv24i16(<vscale x 24 x i16> %vec) nounwind {
+; CHECK-LABEL: vector_deinterleave_nxv8i16_nxv24i16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    addi sp, sp, -16
+; CHECK-NEXT:    csrr a0, vlenb
+; CHECK-NEXT:    slli a0, a0, 3
+; CHECK-NEXT:    sub sp, sp, a0
+; CHECK-NEXT:    addi a0, sp, 16
+; CHECK-NEXT:    vs8r.v v8, (a0)
+; CHECK-NEXT:    vsetvli a1, zero, e16, m2, ta, ma
+; CHECK-NEXT:    vlseg3e16.v v8, (a0)
+; CHECK-NEXT:    csrr a0, vlenb
+; CHECK-NEXT:    slli a0, a0, 3
+; CHECK-NEXT:    add sp, sp, a0
+; CHECK-NEXT:    addi sp, sp, 16
+; CHECK-NEXT:    ret
+  %retval = call {<vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>} @llvm.vector.deinterleave3.nxv24i16(<vscale x 24 x i16> %vec)
+  ret {<vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>} %retval
+}
+
+
+define {<vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>} @vector_deinterleave_nxv4i32_nxv12i32(<vscale x 12 x i32> %vec) nounwind {
+; CHECK-LABEL: vector_deinterleave_nxv4i32_nxv12i32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    addi sp, sp, -16
+; CHECK-NEXT:    csrr a0, vlenb
+; CHECK-NEXT:    slli a0, a0, 3
+; CHECK-NEXT:    sub sp, sp, a0
+; CHECK-NEXT:    addi a0, sp, 16
+; CHECK-NEXT:    vs8r.v v8, (a0)
+; CHECK-NEXT:    vsetvli a1, zero, e32, m2, ta, ma
+; CHECK-NEXT:    vlseg3e32.v v8, (a0)
+; CHECK-NEXT:    csrr a0, vlenb
+; CHECK-NEXT:    slli a0, a0, 3
+; CHECK-NEXT:    add sp, sp, a0
+; CHECK-NEXT:    addi sp, sp, 16
+; CHECK-NEXT:    ret
+  %retval = call {<vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>} @llvm.vector.deinterleave3.nxv12i32(<vscale x 12 x i32> %vec)
+  ret {<vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>} %retval
+}
+
+
+define {<vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>} @vector_deinterleave_nxv2i64_nxv6i64(<vscale x 6 x i64> %vec) nounwind {
+; CHECK-LABEL: vector_deinterleave_nxv2i64_nxv6i64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    addi sp, sp, -16
+; CHECK-NEXT:    csrr a0, vlenb
+; CHECK-NEXT:    slli a0, a0, 3
+; CHECK-NEXT:    sub sp, sp, a0
+; CHECK-NEXT:    addi a0, sp, 16
+; CHECK-NEXT:    vs8r.v v8, (a0)
+; CHECK-NEXT:    vsetvli a1, zero, e64, m2, ta, ma
+; CHECK-NEXT:    vlseg3e64.v v8, (a0)
+; CHECK-NEXT:    csrr a0, vlenb
+; CHECK-NEXT:    slli a0, a0, 3
+; CHECK-NEXT:    add sp, sp, a0
+; CHECK-NEXT:    addi sp, sp, 16
+; CHECK-NEXT:    ret
+  %retval = call {<vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>} @llvm.vector.deinterleave3.nxv6i64(<vscale x 6 x i64> %vec)
+  ret {<vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>} %retval
+}
+
+define {<vscale x 16 x i1>, <vscale x 16 x i1>, <vscale x 16 x i1>, <vscale x 16 x i1>, <vscale x 16 x i1>} @vector_deinterleave_nxv16i1_nxv80i1(<vscale x 80 x i1> %vec) nounwind {
+; CHECK-LABEL: vector_deinterleave_nxv16i1_nxv80i1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    addi sp, sp, -16
+; CHECK-NEXT:    csrr a0, vlenb
+; CHECK-NEXT:    slli a0, a0, 4
+; CHECK-NEXT:    sub sp, sp, a0
+; CHECK-NEXT:    vsetvli a0, zero, e8, m2, ta, ma
+; CHECK-NEXT:    vmv1r.v v9, v0
+; CHECK-NEXT:    vmv.v.i v12, 0
+; CHECK-NEXT:    csrr a0, vlenb
+; CHECK-NEXT:    vmerge.vim v16, v12, 1, v0
+; CHECK-NEXT:    srli a1, a0, 2
+; CHECK-NEXT:    srli a2, a0, 1
+; CHECK-NEXT:    vsetvli a3, zero, e8, m1, ta, ma
+; CHECK-NEXT:    vslidedown.vx v0, v0, a1
+; CHECK-NEXT:    srli a1, a0, 3
+; CHECK-NEXT:    vslidedown.vx v10, v9, a2
+; CHECK-NEXT:    slli a1, a1, 1
+; CHECK-NEXT:    vsetvli a2, zero, e8, m2, ta, ma
+; CHECK-NEXT:    vmerge.vim v18, v12, 1, v0
+; CHECK-NEXT:    sub a0, a0, a1
+; CHECK-NEXT:    vsetvli a1, zero, e8, m1, ta, ma
+; CHECK-NEXT:    vslidedown.vx v9, v9, a0
+; CHECK-NEXT:    vmv1r.v v0, v10
+; CHECK-NEXT:    vsetvli a0, zero, e8, m2, ta, ma
+; CHECK-NEXT:    vmerge.vim v20, v12, 1, v0
+; CHECK-NEXT:    vmv1r.v v0, v9
+; CHECK-NEXT:    vmerge.vim v14, v12, 1, v0
+; CHECK-NEXT:    vmv1r.v v10, v15
+; CHECK-NEXT:    vmv1r.v v0, v8
+; CHECK-NEXT:    vmerge.vim v12, v12, 1, v0
+; CHECK-NEXT:    vmv1r.v v11, v12
+; CHECK-NEXT:    addi a0, sp, 16
+; CHECK-NEXT:    vmv1r.v v8, v21
+; CHECK-NEXT:    csrr a1, vlenb
+; CHECK-NEXT:    slli a1, a1, 3
+; CHECK-NEXT:    add a1, sp, a1
+; CHECK-NEXT:    addi a1, a1, 16
+; CHECK-NEXT:    vmv1r.v v9, v14
+; CHECK-NEXT:    vs8r.v v16, (a0)
+; CHECK-NEXT:    vmv1r.v v12, v13
+; CHECK-NEXT:    vs8r.v v8, (a1)
+; CHECK-NEXT:    vsetvli a2, zero, e8, m1, ta, ma
+; CHECK-NEXT:    vlseg5e8.v v8, (a0)
+; CHECK-NEXT:    vlseg5e8.v v14, (a1)
+; CHECK-NEXT:    vmv2r.v v20, v8
+; CHECK-NEXT:    vmv2r.v v22, v10
+; CHECK-NEXT:    vmv1r.v v21, v14
+; CHECK-NEXT:    vsetvli a0, zero, e8, m2, ta, ma
+; CHECK-NEXT:    vmsne.vi v0, v20, 0
+; CHECK-NEXT:    vmv1r.v v14, v9
+; CHECK-NEXT:    vmsne.vi v8, v14, 0
+; CHECK-NEXT:    vmv1r.v v23, v16
+; CHECK-NEXT:    vmsne.vi v9, v22, 0
+; CHECK-NEXT:    vmv1r.v v16, v11
+; CHECK-NEXT:    vmsne.vi v10, v16, 0
+; CHECK-NEXT:    vmv1r.v v13, v18
+; CHECK-NEXT:    vmsne.vi v11, v12, 0
+; CHECK-NEXT:    csrr a0, vlenb
+; CHECK-NEXT:    slli a0, a0, 4
+; CHECK-NEXT:    add sp, sp, a0
+; CHECK-NEXT:    addi sp, sp, 16
+; CHECK-NEXT:    ret
+  %retval = call {<vscale x 16 x i1>, <vscale x 16 x i1>, <vscale x 16 x i1>, <vscale x 16 x i1>, <vscale x 16 x i1>} @llvm.vector.deinterleave5.nxv80i1(<vscale x 80 x i1> %vec)
+  ret {<vscale x 16 x i1>, <vscale x 16 x i1>, <vscale x 16 x i1>, <vscale x 16 x i1>, <vscale x 16 x i1>} %retval
+}
+
+
+define {<vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>} @vector_deinterleave_nxv16i8_nxv80i8(<vscale x 80 x i8> %vec) nounwind {
+; CHECK-LABEL: vector_deinterleave_nxv16i8_nxv80i8:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    addi sp, sp, -16
+; CHECK-NEXT:    csrr a0, vlenb
+; CHECK-NEXT:    slli a0, a0, 4
+; CHECK-NEXT:    sub sp, sp, a0
+; CHECK-NEXT:    vsetvli a0, zero, e8, m1, ta, ma
+; CHECK-NEXT:    vmv1r.v v26, v15
+; CHECK-NEXT:    vmv1r.v v27, v16
+; CHECK-NEXT:    addi a0, sp, 16
+; CHECK-NEXT:    vmv1r.v v24, v13
+; CHECK-NEXT:    csrr a1, vlenb
+; CHECK-NEXT:    slli a1, a1, 3
+; CHECK-NEXT:    add a1, sp, a1
+; CHECK-NEXT:    addi a1, a1, 16
+; CHECK-NEXT:    vmv1r.v v25, v14
+; CHECK-NEXT:    vs8r.v v8, (a0)
+; CHECK-NEXT:    vmv1r.v v28, v17
+; CHECK-NEXT:    vs8r.v v24, (a1)
+; CHECK-NEXT:    vlseg5e8.v v12, (a0)
+; CHECK-NEXT:    vlseg5e8.v v18, (a1)
+; CHECK-NEXT:    vmv2r.v v8, v12
+; CHECK-NEXT:    vmv1r.v v9, v18
+; CHECK-NEXT:    vmv1r.v v18, v13
+; CHECK-NEXT:    vmv2r.v v12, v14
+; CHECK-NEXT:    vmv1r.v v13, v20
+; CHECK-NEXT:    vmv1r.v v20, v15
+; CHECK-NEXT:    vmv1r.v v17, v22
+; CHECK-NEXT:    vmv2r.v v10, v18
+; CHECK-NEXT:    vmv2r.v v14, v20
+; CHECK-NEXT:    csrr a0, vlenb
+; CHECK-NEXT:    slli a0, a0, 4
+; CHECK-NEXT:    add sp, sp, a0
+; CHECK-NEXT:    addi sp, sp, 16
+; CHECK-NEXT:    ret
+  %retval = call {<vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>} @llvm.vector.deinterleave5.nxv80i8(<vscale x 80 x i8> %vec)
+  ret {<vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>} %retval
+}
+
+
+define {<vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>} @vector_deinterleave_nxv8i16_nxv40i16(<vscale x 40 x i16> %vec) nounwind {
+; CHECK-LABEL: vector_deinterleave_nxv8i16_nxv40i16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    addi sp, sp, -16
+; CHECK-NEXT:    csrr a0, vlenb
+; CHECK-NEXT:    slli a0, a0, 4
+; CHECK-NEXT:    sub sp, sp, a0
+; CHECK-NEXT:    vsetvli a0, zero, e16, m1, ta, ma
+; CHECK-NEXT:    vmv1r.v v26, v15
+; CHECK-NEXT:    vmv1r.v v27, v16
+; CHECK-NEXT:    addi a0, sp, 16
+; CHECK-NEXT:    vmv1r.v v24, v13
+; CHECK-NEXT:    csrr a1, vlenb
+; CHECK-NEXT:    slli a1, a1, 3
+; CHECK-NEXT:    add a1, sp, a1
+; CHECK-NEXT:    addi a1, a1, 16
+; CHECK-NEXT:    vmv1r.v v25, v14
+; CHECK-NEXT:    vs8r.v v8, (a0)
+; CHECK-NEXT:    vmv1r.v v28, v17
+; CHECK-NEXT:    vs8r.v v24, (a1)
+; CHECK-NEXT:    vlseg5e16.v v12, (a0)
+; CHECK-NEXT:    vlseg5e16.v v18, (a1)
+; CHECK-NEXT:    vmv2r.v v8, v12
+; CHECK-NEXT:    vmv1r.v v9, v18
+; CHECK-NEXT:    vmv1r.v v18, v13
+; CHECK-NEXT:    vmv2r.v v12, v14
+; CHECK-NEXT:    vmv1r.v v13, v20
+; CHECK-NEXT:    vmv1r.v v20, v15
+; CHECK-NEXT:    vmv1r.v v17, v22
+; CHECK-NEXT:    vmv2r.v v10, v18
+; CHECK-NEXT:    vmv2r.v v14, v20
+; CHECK-NEXT:    csrr a0, vlenb
+; CHECK-NEXT:    slli a0, a0, 4
+; CHECK-NEXT:    add sp, sp, a0
+; CHECK-NEXT:    addi sp, sp, 16
+; CHECK-NEXT:    ret
+  %retval = call {<vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>} @llvm.vector.deinterleave5.nxv40i16(<vscale x 40 x i16> %vec)
+  ret {<vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>} %retval
+}
+
+
+define {<vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>} @vector_deinterleave_nxv4i32_nxv20i32(<vscale x 20 x i32> %vec) nounwind {
+; CHECK-LABEL: vector_deinterleave_nxv4i32_nxv20i32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    addi sp, sp, -16
+; CHECK-NEXT:    csrr a0, vlenb
+; CHECK-NEXT:    slli a0, a0, 4
+; CHECK-NEXT:    sub sp, sp, a0
+; CHECK-NEXT:    vsetvli a0, zero, e32, m1, ta, ma
+; CHECK-NEXT:    vmv1r.v v26, v15
+; CHECK-NEXT:    vmv1r.v v27, v16
+; CHECK-NEXT:    addi a0, sp, 16
+; CHECK-NEXT:    vmv1r.v v24, v13
+; CHECK-NEXT:    csrr a1, vlenb
+; CHECK-NEXT:    slli a1, a1, 3
+; CHECK-NEXT:    add a1, sp, a1
+; CHECK-NEXT:    addi a1, a1, 16
+; CHECK-NEXT:    vmv1r.v v25, v14
+; CHECK-NEXT:    vs8r.v v8, (a0)
+; CHECK-NEXT:    vmv1r.v v28, v17
+; CHECK-NEXT:    vs8r.v v24, (a1)
+; CHECK-NEXT:    vlseg5e32.v v12, (a0)
+; CHECK-NEXT:    vlseg5e32.v v18, (a1)
+; CHECK-NEXT:    vmv2r.v v8, v12
+; CHECK-NEXT:    vmv1r.v v9, v18
+; CHECK-NEXT:    vmv1r.v v18, v13
+; CHECK-NEXT:    vmv2r.v v12, v14
+; CHECK-NEXT:    vmv1r.v v13, v20
+; CHECK-NEXT:    vmv1r.v v20, v15
+; CHECK-NEXT:    vmv1r.v v17, v22
+; CHECK-NEXT:    vmv2r.v v10, v18
+; CHECK-NEXT:    vmv2r.v v14, v20
+; CHECK-NEXT:    csrr a0, vlenb
+; CHECK-NEXT:    slli a0, a0, 4
+; CHECK-NEXT:    add sp, sp, a0
+; CHECK-NEXT:    addi sp, sp, 16
+; CHECK-NEXT:    ret
+  %retval = call {<vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>} @llvm.vector.deinterleave5.nxv20i32(<vscale x 20 x i32> %vec)
+  ret {<vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>} %retval
+}
+
+
+define {<vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>} @vector_deinterleave_nxv2i64_nxv10i64(<vscale x 10 x i64> %vec) nounwind {
+; CHECK-LABEL: vector_deinterleave_nxv2i64_nxv10i64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    addi sp, sp, -16
+; CHECK-NEXT:    csrr a0, vlenb
+; CHECK-NEXT:    slli a0, a0, 4
+; CHECK-NEXT:    sub sp, sp, a0
+; CHECK-NEXT:    vsetvli a0, zero, e64, m1, ta, ma
+; CHECK-NEXT:    vmv1r.v v26, v15
+; CHECK-NEXT:    vmv1r.v v27, v16
+; CHECK-NEXT:    addi a0, sp, 16
+; CHECK-NEXT:    vmv1r.v v24, v13
+; CHECK-NEXT:    csrr a1, vlenb
+; CHECK-NEXT:    slli a1, a1, 3
+; CHECK-NEXT:    add a1, sp, a1
+; CHECK-NEXT:    addi a1, a1, 16
+; CHECK-NEXT:    vmv1r.v v25, v14
+; CHECK-NEXT:    vs8r.v v8, (a0)
+; CHECK-NEXT:    vmv1r.v v28, v17
+; CHECK-NEXT:    vs8r.v v24, (a1)
+; CHECK-NEXT:    vlseg5e64.v v12, (a0)
+; CHECK-NEXT:    vlseg5e64.v v18, (a1)
+; CHECK-NEXT:    vmv2r.v v8, v12
+; CHECK-NEXT:    vmv1r.v v9, v18
+; CHECK-NEXT:    vmv1r.v v18, v13
+; CHECK-NEXT:    vmv2r.v v12, v14
+; CHECK-NEXT:    vmv1r.v v13, v20
+; CHECK-NEXT:    vmv1r.v v20, v15
+; CHECK-NEXT:    vmv1r.v v17, v22
+; CHECK-NEXT:    vmv2r.v v10, v18
+; CHECK-NEXT:    vmv2r.v v14, v20
+; CHECK-NEXT:    csrr a0, vlenb
+; CHECK-NEXT:    slli a0, a0, 4
+; CHECK-NEXT:    add sp, sp, a0
+; CHECK-NEXT:    addi sp, sp, 16
+; CHECK-NEXT:    ret
+  %retval = call {<vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>} @llvm.vector.deinterleave5.nxv10i64(<vscale x 10 x i64> %vec)
+  ret {<vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>} %retval
+}
+
+define {<vscale x 16 x i1>, <vscale x 16 x i1>, <vscale x 16 x i1>, <vscale x 16 x i1>, <vscale x 16 x i1>, <vscale x 16 x i1>, <vscale x 16 x i1>} @vector_deinterleave_nxv16i1_nxv112i1(<vscale x 112 x i1> %vec) nounwind {
+; CHECK-LABEL: vector_deinterleave_nxv16i1_nxv112i1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    addi sp, sp, -16
+; CHECK-NEXT:    csrr a0, vlenb
+; CHECK-NEXT:    slli a0, a0, 4
+; CHECK-NEXT:    sub sp, sp, a0
+; CHECK-NEXT:    vsetvli a0, zero, e8, m2, ta, ma
+; CHECK-NEXT:    vmv1r.v v9, v0
+; CHECK-NEXT:    vmv.v.i v12, 0
+; CHECK-NEXT:    csrr a1, vlenb
+; CHECK-NEXT:    vmerge.vim v16, v12, 1, v0
+; CHECK-NEXT:    srli a2, a1, 2
+; CHECK-NEXT:    srli a0, a1, 1
+; CHECK-NEXT:    srli a3, a1, 3
+; CHECK-NEXT:    vsetvli a4, zero, e8, m1, ta, ma
+; CHECK-NEXT:    vslidedown.vx v0, v0, a2
+; CHECK-NEXT:    vslidedown.vx v10, v9, a0
+; CHECK-NEXT:    slli a3, a3, 1
+; CHECK-NEXT:    vslidedown.vx v11, v8, a2
+; CHECK-NEXT:    vsetvli a2, zero, e8, m2, ta, ma
+; CHECK-NEXT:    vmerge.vim v18, v12, 1, v0
+; CHECK-NEXT:    sub a1, a1, a3
+; CHECK-NEXT:    vsetvli a2, zero, e8, m1, ta, ma
+; CHECK-NEXT:    vslidedown.vx v9, v9, a1
+; CHECK-NEXT:    vmv1r.v v0, v10
+; CHECK-NEXT:    vsetvli a1, zero, e8, m2, ta, ma
+; CHECK-NEXT:    vmerge.vim v20, v12, 1, v0
+; CHECK-NEXT:    vmv1r.v v0, v9
+; CHECK-NEXT:    vmerge.vim v22, v12, 1, v0
+; CHECK-NEXT:    vmv1r.v v0, v8
+; CHECK-NEXT:    vmerge.vim v14, v12, 1, v0
+; CHECK-NEXT:    vmv1r.v v10, v15
+; CHECK-NEXT:    vmv1r.v v0, v11
+; CHECK-NEXT:    vmerge.vim v24, v12, 1, v0
+; CHECK-NEXT:    vmv1r.v v11, v24
+; CHECK-NEXT:    vsetvli a1, zero, e8, m1, ta, ma
+; CHECK-NEXT:    vslidedown.vx v0, v8, a0
+; CHECK-NEXT:    vmv1r.v v8, v23
+; CHECK-NEXT:    addi a0, sp, 16
+; CHECK-NEXT:    vmv1r.v v9, v14
+; CHECK-NEXT:    vsetvli a1, zero, e8, m2, ta, ma
+; CHECK-NEXT:    vmerge.vim v14, v12, 1, v0
+; CHECK-NEXT:    vmv1r.v v12, v25
+; CHECK-NEXT:    csrr a1, vlenb
+; CHECK-NEXT:    slli a1, a1, 3
+; CHECK-NEXT:    add a1, sp, a1
+; CHECK-NEXT:    addi a1, a1, 16
+; CHECK-NEXT:    vmv1r.v v13, v14
+; CHECK-NEXT:    vs8r.v v16, (a0)
+; CHECK-NEXT:    vmv1r.v v14, v15
+; CHECK-NEXT:    vs8r.v v8, (a1)
+; CHECK-NEXT:    vsetvli a2, zero, e8, m1, ta, ma
+; CHECK-NEXT:    vlseg7e8.v v8, (a0)
+; CHECK-NEXT:    vlseg7e8.v v16, (a1)
+; CHECK-NEXT:    vmv2r.v v24, v8
+; CHECK-NEXT:    vmv2r.v v26, v10
+; CHECK-NEXT:    vmv2r.v v28, v12
+; CHECK-NEXT:    vmv1r.v v25, v16
+; CHECK-NEXT:    vsetvli a0, zero, e8, m2, ta, ma
+; CHECK-NEXT:    vmsne.vi v0, v24, 0
+; CHECK-NEXT:    vmv1r.v v16, v9
+; CHECK-NEXT:    vmsne.vi v8, v16, 0
+; CHECK-NEXT:    vmv1r.v v27, v18
+; CHECK-NEXT:    vmsne.vi v9, v26, 0
+; CHECK-NEXT:    vmv1r.v v18, v11
+; CHECK-NEXT:    vmsne.vi v10, v18, 0
+; CHECK-NEXT:    vmv1r.v v29, v20
+; CHECK-NEXT:    vmsne.vi v11, v28, 0
+; CHECK-NEXT:    vmv1r.v v20, v13
+; CHECK-NEXT:    vmsne.vi v12, v20, 0
+; CHECK-NEXT:    vmv1r.v v15, v22
+; CHECK-NEXT:    vmsne.vi v13, v14, 0
+; CHECK-NEXT:    csrr a0, vlenb
+; CHECK-NEXT:    slli a0, a0, 4
+; CHECK-NEXT:    add sp, sp, a0
+; CHECK-NEXT:    addi sp, sp, 16
+; CHECK-NEXT:    ret
+  %retval = call {<vscale x 16 x i1>, <vscale x 16 x i1>, <vscale x 16 x i1>, <vscale x 16 x i1>, <vscale x 16 x i1>, <vscale x 16 x i1>, <vscale x 16 x i1>} @llvm.vector.deinterleave7.nxv112i1(<vscale x 112 x i1> %vec)
+  ret {<vscale x 16 x i1>, <vscale x 16 x i1>, <vscale x 16 x i1>, <vscale x 16 x i1>, <vscale x 16 x i1>, <vscale x 16 x i1>, <vscale x 16 x i1>} %retval
+}
+
+
+define {<vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>} @vector_deinterleave_nxv16i8_nxv112i8(<vscale x 112 x i8> %vec) nounwind {
+; CHECK-LABEL: vector_deinterleave_nxv16i8_nxv112i8:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    addi sp, sp, -16
+; CHECK-NEXT:    csrr a0, vlenb
+; CHECK-NEXT:    slli a0, a0, 4
+; CHECK-NEXT:    sub sp, sp, a0
+; CHECK-NEXT:    vsetvli a0, zero, e8, m1, ta, ma
+; CHECK-NEXT:    vmv1r.v v30, v21
+; CHECK-NEXT:    vmv1r.v v28, v19
+; CHECK-NEXT:    vmv1r.v v29, v20
+; CHECK-NEXT:    vmv1r.v v26, v17
+; CHECK-NEXT:    addi a0, sp, 16
+; CHECK-NEXT:    vmv1r.v v27, v18
+; CHECK-NEXT:    csrr a1, vlenb
+; CHECK-NEXT:    slli a1, a1, 3
+; CHECK-NEXT:    add a1, sp, a1
+; CHECK-NEXT:    addi a1, a1, 16
+; CHECK-NEXT:    vmv1r.v v24, v15
+; CHECK-NEXT:    vs8r.v v8, (a0)
+; CHECK-NEXT:    vmv1r.v v25, v16
+; CHECK-NEXT:    vs8r.v v24, (a1)
+; CHECK-NEXT:    vlseg7e8.v v14, (a0)
+; CHECK-NEXT:    vlseg7e8.v v22, (a1)
+; CHECK-NEXT:    vmv2r.v v8, v14
+; CHECK-NEXT:    vmv1r.v v9, v22
+; CHECK-NEXT:    vmv1r.v v22, v15
+; CHECK-NEXT:    vmv2r.v v12, v16
+; CHECK-NEXT:    vmv1r.v v13, v24
+; CHECK-NEXT:    vmv1r.v v24, v17
+; CHECK-NEXT:    vmv2r.v v16, v18
+; CHECK-NEXT:    vmv1r.v v17, v26
+; CHECK-NEXT:    vmv1r.v v26, v19
+; CHECK-NEXT:    vmv1r.v v21, v28
+; CHECK-NEXT:    vmv2r.v v10, v22
+; CHECK-NEXT:    vmv2r.v v14, v24
+; CHECK-NEXT:    vmv2r.v v18, v26
+; CHECK-NEXT:    csrr a0, vlenb
+; CHECK-NEXT:    slli a0, a0, 4
+; CHECK-NEXT:    add sp, sp, a0
+; CHECK-NEXT:    addi sp, sp, 16
+; CHECK-NEXT:    ret
+  %retval = call {<vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>} @llvm.vector.deinterleave7.nxv112i8(<vscale x 112 x i8> %vec)
+  ret {<vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>} %retval
+}
+
+
+define {<vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>} @vector_deinterleave_nxv8i16_nxv56i16(<vscale x 56 x i16> %vec) nounwind {
+; CHECK-LABEL: vector_deinterleave_nxv8i16_nxv56i16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    addi sp, sp, -16
+; CHECK-NEXT:    csrr a0, vlenb
+; CHECK-NEXT:    slli a0, a0, 4
+; CHECK-NEXT:    sub sp, sp, a0
+; CHECK-NEXT:    vsetvli a0, zero, e16, m1, ta, ma
+; CHECK-NEXT:    vmv1r.v v30, v21
+; CHECK-NEXT:    vmv1r.v v28, v19
+; CHECK-NEXT:    vmv1r.v v29, v20
+; CHECK-NEXT:    vmv1r.v v26, v17
+; CHECK-NEXT:    addi a0, sp, 16
+; CHECK-NEXT:    vmv1r.v v27, v18
+; CHECK-NEXT:    csrr a1, vlenb
+; CHECK-NEXT:    slli a1, a1, 3
+; CHECK-NEXT:    add a1, sp, a1
+; CHECK-NEXT:    addi a1, a1, 16
+; CHECK-NEXT:    vmv1r.v v24, v15
+; CHECK-NEXT:    vs8r.v v8, (a0)
+; CHECK-NEXT:    vmv1r.v v25, v16
+; CHECK-NEXT:    vs8r.v v24, (a1)
+; CHECK-NEXT:    vlseg7e16.v v14, (a0)
+; CHECK-NEXT:    vlseg7e16.v v22, (a1)
+; CHECK-NEXT:    vmv2r.v v8, v14
+; CHECK-NEXT:    vmv1r.v v9, v22
+; CHECK-NEXT:    vmv1r.v v22, v15
+; CHECK-NEXT:    vmv2r.v v12, v16
+; CHECK-NEXT:    vmv1r.v v13, v24
+; CHECK-NEXT:    vmv1r.v v24, v17
+; CHECK-NEXT:    vmv2r.v v16, v18
+; CHECK-NEXT:    vmv1r.v v17, v26
+; CHECK-NEXT:    vmv1r.v v26, v19
+; CHECK-NEXT:    vmv1r.v v21, v28
+; CHECK-NEXT:    vmv2r.v v10, v22
+; CHECK-NEXT:    vmv2r.v v14, v24
+; CHECK-NEXT:    vmv2r.v v18, v26
+; CHECK-NEXT:    csrr a0, vlenb
+; CHECK-NEXT:    slli a0, a0, 4
+; CHECK-NEXT:    add sp, sp, a0
+; CHECK-NEXT:    addi sp, sp, 16
+; CHECK-NEXT:    ret
+  %retval = call {<vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>} @llvm.vector.deinterleave7.nxv56i16(<vscale x 56 x i16> %vec)
+  ret {<vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>} %retval
+}
+
+
+define {<vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>} @vector_deinterleave_nxv4i32_nxv28i32(<vscale x 28 x i32> %vec) nounwind {
+; CHECK-LABEL: vector_deinterleave_nxv4i32_nxv28i32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    addi sp, sp, -16
+; CHECK-NEXT:    csrr a0, vlenb
+; CHECK-NEXT:    slli a0, a0, 4
+; CHECK-NEXT:    sub sp, sp, a0
+; CHECK-NEXT:    vsetvli a0, zero, e32, m1, ta, ma
+; CHECK-NEXT:    vmv1r.v v30, v21
+; CHECK-NEXT:    vmv1r.v v28, v19
+; CHECK-NEXT:    vmv1r.v v29, v20
+; CHECK-NEXT:    vmv1r.v v26, v17
+; CHECK-NEXT:    addi a0, sp, 16
+; CHECK-NEXT:    vmv1r.v v27, v18
+; CHECK-NEXT:    csrr a1, vlenb
+; CHECK-NEXT:    slli a1, a1, 3
+; CHECK-NEXT:    add a1, sp, a1
+; CHECK-NEXT:    addi a1, a1, 16
+; CHECK-NEXT:    vmv1r.v v24, v15
+; CHECK-NEXT:    vs8r.v v8, (a0)
+; CHECK-NEXT:    vmv1r.v v25, v16
+; CHECK-NEXT:    vs8r.v v24, (a1)
+; CHECK-NEXT:    vlseg7e32.v v14, (a0)
+; CHECK-NEXT:    vlseg7e32.v v22, (a1)
+; CHECK-NEXT:    vmv2r.v v8, v14
+; CHECK-NEXT:    vmv1r.v v9, v22
+; CHECK-NEXT:    vmv1r.v v22, v15
+; CHECK-NEXT:    vmv2r.v v12, v16
+; CHECK-NEXT:    vmv1r.v v13, v24
+; CHECK-NEXT:    vmv1r.v v24, v17
+; CHECK-NEXT:    vmv2r.v v16, v18
+; CHECK-NEXT:    vmv1r.v v17, v26
+; CHECK-NEXT:    vmv1r.v v26, v19
+; CHECK-NEXT:    vmv1r.v v21, v28
+; CHECK-NEXT:    vmv2r.v v10, v22
+; CHECK-NEXT:    vmv2r.v v14, v24
+; CHECK-NEXT:    vmv2r.v v18, v26
+; CHECK-NEXT:    csrr a0, vlenb
+; CHECK-NEXT:    slli a0, a0, 4
+; CHECK-NEXT:    add sp, sp, a0
+; CHECK-NEXT:    addi sp, sp, 16
+; CHECK-NEXT:    ret
+  %retval = call {<vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>} @llvm.vector.deinterleave7.nxv28i32(<vscale x 28 x i32> %vec)
+  ret {<vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>} %retval
+}
+
+
+define {<vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>} @vector_deinterleave_nxv2i64_nxv14i64(<vscale x 14 x i64> %vec) nounwind {
+; CHECK-LABEL: vector_deinterleave_nxv2i64_nxv14i64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    addi sp, sp, -16
+; CHECK-NEXT:    csrr a0, vlenb
+; CHECK-NEXT:    slli a0, a0, 4
+; CHECK-NEXT:    sub sp, sp, a0
+; CHECK-NEXT:    vsetvli a0, zero, e64, m1, ta, ma
+; CHECK-NEXT:    vmv1r.v v30, v21
+; CHECK-NEXT:    vmv1r.v v28, v19
+; CHECK-NEXT:    vmv1r.v v29, v20
+; CHECK-NEXT:    vmv1r.v v26, v17
+; CHECK-NEXT:    addi a0, sp, 16
+; CHECK-NEXT:    vmv1r.v v27, v18
+; CHECK-NEXT:    csrr a1, vlenb
+; CHECK-NEXT:    slli a1, a1, 3
+; CHECK-NEXT:    add a1, sp, a1
+; CHECK-NEXT:    addi a1, a1, 16
+; CHECK-NEXT:    vmv1r.v v24, v15
+; CHECK-NEXT:    vs8r.v v8, (a0)
+; CHECK-NEXT:    vmv1r.v v25, v16
+; CHECK-NEXT:    vs8r.v v24, (a1)
+; CHECK-NEXT:    vlseg7e64.v v14, (a0)
+; CHECK-NEXT:    vlseg7e64.v v22, (a1)
+; CHECK-NEXT:    vmv2r.v v8, v14
+; CHECK-NEXT:    vmv1r.v v9, v22
+; CHECK-NEXT:    vmv1r.v v22, v15
+; CHECK-NEXT:    vmv2r.v v12, v16
+; CHECK-NEXT:    vmv1r.v v13, v24
+; CHECK-NEXT:    vmv1r.v v24, v17
+; CHECK-NEXT:    vmv2r.v v16, v18
+; CHECK-NEXT:    vmv1r.v v17, v26
+; CHECK-NEXT:    vmv1r.v v26, v19
+; CHECK-NEXT:    vmv1r.v v21, v28
+; CHECK-NEXT:    vmv2r.v v10, v22
+; CHECK-NEXT:    vmv2r.v v14, v24
+; CHECK-NEXT:    vmv2r.v v18, v26
+; CHECK-NEXT:    csrr a0, vlenb
+; CHECK-NEXT:    slli a0, a0, 4
+; CHECK-NEXT:    add sp, sp, a0
+; CHECK-NEXT:    addi sp, sp, 16
+; CHECK-NEXT:    ret
+  %retval = call {<vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>} @llvm.vector.deinterleave7.nxv14i64(<vscale x 14 x i64> %vec)
+  ret {<vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>} %retval
+}
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; RV32: {{.*}}
+; RV64: {{.*}}
diff --git a/llvm/test/CodeGen/RISCV/rvv/vector-interleave-fixed.ll b/llvm/test/CodeGen/RISCV/rvv/vector-interleave-fixed.ll
index 08aa02c7e869a..1e4cb06480163 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vector-interleave-fixed.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vector-interleave-fixed.ll
@@ -122,10 +122,237 @@ define <4 x i64> @vector_interleave_v4i64_v2i64(<2 x i64> %a, <2 x i64> %b) {
 	   ret <4 x i64> %res
 }
 
-declare <32 x i1> @llvm.vector.interleave2.v32i1(<16 x i1>, <16 x i1>)
-declare <16 x i16> @llvm.vector.interleave2.v16i16(<8 x i16>, <8 x i16>)
-declare <8 x i32> @llvm.vector.interleave2.v8i32(<4 x i32>, <4 x i32>)
-declare <4 x i64> @llvm.vector.interleave2.v4i64(<2 x i64>, <2 x i64>)
+define <6 x i32> @vector_interleave3_v6i32_v2i32(<2 x i32> %a, <2 x i32> %b, <2 x i32> %c) {
+; CHECK-LABEL: vector_interleave3_v6i32_v2i32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    addi sp, sp, -16
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    csrr a0, vlenb
+; CHECK-NEXT:    slli a0, a0, 1
+; CHECK-NEXT:    sub sp, sp, a0
+; CHECK-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x02, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 2 * vlenb
+; CHECK-NEXT:    addi a0, sp, 16
+; CHECK-NEXT:    csrr a1, vlenb
+; CHECK-NEXT:    srli a1, a1, 1
+; CHECK-NEXT:    vsetvli a2, zero, e32, mf2, ta, ma
+; CHECK-NEXT:    vsseg3e32.v v8, (a0)
+; CHECK-NEXT:    add a2, a0, a1
+; CHECK-NEXT:    vle32.v v9, (a2)
+; CHECK-NEXT:    vle32.v v8, (a0)
+; CHECK-NEXT:    add a1, a2, a1
+; CHECK-NEXT:    vle32.v v10, (a1)
+; CHECK-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
+; CHECK-NEXT:    vslideup.vi v8, v9, 2
+; CHECK-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
+; CHECK-NEXT:    vslideup.vi v8, v10, 4
+; CHECK-NEXT:    csrr a0, vlenb
+; CHECK-NEXT:    slli a0, a0, 1
+; CHECK-NEXT:    add sp, sp, a0
+; CHECK-NEXT:    .cfi_def_cfa sp, 16
+; CHECK-NEXT:    addi sp, sp, 16
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-NEXT:    ret
+;
+; ZVBB-LABEL: vector_interleave3_v6i32_v2i32:
+; ZVBB:       # %bb.0:
+; ZVBB-NEXT:    addi sp, sp, -16
+; ZVBB-NEXT:    .cfi_def_cfa_offset 16
+; ZVBB-NEXT:    csrr a0, vlenb
+; ZVBB-NEXT:    slli a0, a0, 1
+; ZVBB-NEXT:    sub sp, sp, a0
+; ZVBB-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x02, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 2 * vlenb
+; ZVBB-NEXT:    addi a0, sp, 16
+; ZVBB-NEXT:    csrr a1, vlenb
+; ZVBB-NEXT:    srli a1, a1, 1
+; ZVBB-NEXT:    vsetvli a2, zero, e32, mf2, ta, ma
+; ZVBB-NEXT:    vsseg3e32.v v8, (a0)
+; ZVBB-NEXT:    add a2, a0, a1
+; ZVBB-NEXT:    vle32.v v9, (a2)
+; ZVBB-NEXT:    vle32.v v8, (a0)
+; ZVBB-NEXT:    add a1, a2, a1
+; ZVBB-NEXT:    vle32.v v10, (a1)
+; ZVBB-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
+; ZVBB-NEXT:    vslideup.vi v8, v9, 2
+; ZVBB-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
+; ZVBB-NEXT:    vslideup.vi v8, v10, 4
+; ZVBB-NEXT:    csrr a0, vlenb
+; ZVBB-NEXT:    slli a0, a0, 1
+; ZVBB-NEXT:    add sp, sp, a0
+; ZVBB-NEXT:    .cfi_def_cfa sp, 16
+; ZVBB-NEXT:    addi sp, sp, 16
+; ZVBB-NEXT:    .cfi_def_cfa_offset 0
+; ZVBB-NEXT:    ret
+	   %res = call <6 x i32> @llvm.vector.interleave3.v6i32(<2 x i32> %a, <2 x i32> %b, <2 x i32> %c)
+	   ret <6 x i32> %res
+}
+
+
+define <10 x i16> @vector_interleave5_v10i16_v2i16(<2 x i16> %a, <2 x i16> %b, <2 x i16> %c, <2 x i16> %d, <2 x i16> %e) {
+; CHECK-LABEL: vector_interleave5_v10i16_v2i16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    addi sp, sp, -16
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    csrr a0, vlenb
+; CHECK-NEXT:    slli a0, a0, 1
+; CHECK-NEXT:    sub sp, sp, a0
+; CHECK-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x02, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 2 * vlenb
+; CHECK-NEXT:    addi a0, sp, 16
+; CHECK-NEXT:    csrr a1, vlenb
+; CHECK-NEXT:    srli a1, a1, 2
+; CHECK-NEXT:    add a2, a0, a1
+; CHECK-NEXT:    add a3, a2, a1
+; CHECK-NEXT:    vsetvli a4, zero, e16, mf4, ta, ma
+; CHECK-NEXT:    vsseg5e16.v v8, (a0)
+; CHECK-NEXT:    add a4, a3, a1
+; CHECK-NEXT:    vle16.v v9, (a2)
+; CHECK-NEXT:    vle16.v v10, (a4)
+; CHECK-NEXT:    vle16.v v11, (a3)
+; CHECK-NEXT:    vle16.v v8, (a0)
+; CHECK-NEXT:    add a1, a4, a1
+; CHECK-NEXT:    vle16.v v12, (a1)
+; CHECK-NEXT:    vsetivli zero, 4, e16, mf2, ta, ma
+; CHECK-NEXT:    vslideup.vi v11, v10, 2
+; CHECK-NEXT:    vslideup.vi v8, v9, 2
+; CHECK-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; CHECK-NEXT:    vslideup.vi v8, v11, 4
+; CHECK-NEXT:    vsetivli zero, 16, e16, m2, ta, ma
+; CHECK-NEXT:    vslideup.vi v8, v12, 8
+; CHECK-NEXT:    csrr a0, vlenb
+; CHECK-NEXT:    slli a0, a0, 1
+; CHECK-NEXT:    add sp, sp, a0
+; CHECK-NEXT:    .cfi_def_cfa sp, 16
+; CHECK-NEXT:    addi sp, sp, 16
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-NEXT:    ret
+;
+; ZVBB-LABEL: vector_interleave5_v10i16_v2i16:
+; ZVBB:       # %bb.0:
+; ZVBB-NEXT:    addi sp, sp, -16
+; ZVBB-NEXT:    .cfi_def_cfa_offset 16
+; ZVBB-NEXT:    csrr a0, vlenb
+; ZVBB-NEXT:    slli a0, a0, 1
+; ZVBB-NEXT:    sub sp, sp, a0
+; ZVBB-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x02, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 2 * vlenb
+; ZVBB-NEXT:    addi a0, sp, 16
+; ZVBB-NEXT:    csrr a1, vlenb
+; ZVBB-NEXT:    srli a1, a1, 2
+; ZVBB-NEXT:    add a2, a0, a1
+; ZVBB-NEXT:    add a3, a2, a1
+; ZVBB-NEXT:    vsetvli a4, zero, e16, mf4, ta, ma
+; ZVBB-NEXT:    vsseg5e16.v v8, (a0)
+; ZVBB-NEXT:    add a4, a3, a1
+; ZVBB-NEXT:    vle16.v v9, (a2)
+; ZVBB-NEXT:    vle16.v v10, (a4)
+; ZVBB-NEXT:    vle16.v v11, (a3)
+; ZVBB-NEXT:    vle16.v v8, (a0)
+; ZVBB-NEXT:    add a1, a4, a1
+; ZVBB-NEXT:    vle16.v v12, (a1)
+; ZVBB-NEXT:    vsetivli zero, 4, e16, mf2, ta, ma
+; ZVBB-NEXT:    vslideup.vi v11, v10, 2
+; ZVBB-NEXT:    vslideup.vi v8, v9, 2
+; ZVBB-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; ZVBB-NEXT:    vslideup.vi v8, v11, 4
+; ZVBB-NEXT:    vsetivli zero, 16, e16, m2, ta, ma
+; ZVBB-NEXT:    vslideup.vi v8, v12, 8
+; ZVBB-NEXT:    csrr a0, vlenb
+; ZVBB-NEXT:    slli a0, a0, 1
+; ZVBB-NEXT:    add sp, sp, a0
+; ZVBB-NEXT:    .cfi_def_cfa sp, 16
+; ZVBB-NEXT:    addi sp, sp, 16
+; ZVBB-NEXT:    .cfi_def_cfa_offset 0
+; ZVBB-NEXT:    ret
+	   %res = call <10 x i16> @llvm.vector.interleave5.v10i16(<2 x i16> %a, <2 x i16> %b, <2 x i16> %c, <2 x i16> %d, <2 x i16> %e)
+	   ret <10 x i16> %res
+}
+
+define <14 x i8> @vector_interleave7_v14i8_v2i8(<2 x i8> %a, <2 x i8> %b, <2 x i8> %c, <2 x i8> %d, <2 x i8> %e, <2 x i8> %f, <2 x i8> %g) {
+; CHECK-LABEL: vector_interleave7_v14i8_v2i8:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    addi sp, sp, -16
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    csrr a0, vlenb
+; CHECK-NEXT:    sub sp, sp, a0
+; CHECK-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x01, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 1 * vlenb
+; CHECK-NEXT:    addi a0, sp, 16
+; CHECK-NEXT:    csrr a1, vlenb
+; CHECK-NEXT:    srli a1, a1, 3
+; CHECK-NEXT:    add a2, a0, a1
+; CHECK-NEXT:    add a3, a2, a1
+; CHECK-NEXT:    add a4, a3, a1
+; CHECK-NEXT:    vsetvli a5, zero, e8, mf8, ta, ma
+; CHECK-NEXT:    vsseg7e8.v v8, (a0)
+; CHECK-NEXT:    vle8.v v9, (a4)
+; CHECK-NEXT:    add a4, a4, a1
+; CHECK-NEXT:    vle8.v v10, (a2)
+; CHECK-NEXT:    add a2, a4, a1
+; CHECK-NEXT:    add a1, a2, a1
+; CHECK-NEXT:    vle8.v v11, (a2)
+; CHECK-NEXT:    vle8.v v12, (a4)
+; CHECK-NEXT:    vle8.v v8, (a0)
+; CHECK-NEXT:    vle8.v v13, (a1)
+; CHECK-NEXT:    vle8.v v14, (a3)
+; CHECK-NEXT:    vsetivli zero, 4, e8, mf2, tu, ma
+; CHECK-NEXT:    vslideup.vi v12, v11, 2
+; CHECK-NEXT:    vslideup.vi v8, v10, 2
+; CHECK-NEXT:    vsetivli zero, 6, e8, mf2, tu, ma
+; CHECK-NEXT:    vslideup.vi v12, v13, 4
+; CHECK-NEXT:    vslideup.vi v8, v14, 4
+; CHECK-NEXT:    vsetivli zero, 8, e8, mf2, ta, ma
+; CHECK-NEXT:    vslideup.vi v8, v9, 6
+; CHECK-NEXT:    vsetivli zero, 16, e8, m1, ta, ma
+; CHECK-NEXT:    vslideup.vi v8, v12, 8
+; CHECK-NEXT:    csrr a0, vlenb
+; CHECK-NEXT:    add sp, sp, a0
+; CHECK-NEXT:    .cfi_def_cfa sp, 16
+; CHECK-NEXT:    addi sp, sp, 16
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-NEXT:    ret
+;
+; ZVBB-LABEL: vector_interleave7_v14i8_v2i8:
+; ZVBB:       # %bb.0:
+; ZVBB-NEXT:    addi sp, sp, -16
+; ZVBB-NEXT:    .cfi_def_cfa_offset 16
+; ZVBB-NEXT:    csrr a0, vlenb
+; ZVBB-NEXT:    sub sp, sp, a0
+; ZVBB-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x01, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 1 * vlenb
+; ZVBB-NEXT:    addi a0, sp, 16
+; ZVBB-NEXT:    csrr a1, vlenb
+; ZVBB-NEXT:    srli a1, a1, 3
+; ZVBB-NEXT:    add a2, a0, a1
+; ZVBB-NEXT:    add a3, a2, a1
+; ZVBB-NEXT:    add a4, a3, a1
+; ZVBB-NEXT:    vsetvli a5, zero, e8, mf8, ta, ma
+; ZVBB-NEXT:    vsseg7e8.v v8, (a0)
+; ZVBB-NEXT:    vle8.v v9, (a4)
+; ZVBB-NEXT:    add a4, a4, a1
+; ZVBB-NEXT:    vle8.v v10, (a2)
+; ZVBB-NEXT:    add a2, a4, a1
+; ZVBB-NEXT:    add a1, a2, a1
+; ZVBB-NEXT:    vle8.v v11, (a2)
+; ZVBB-NEXT:    vle8.v v12, (a4)
+; ZVBB-NEXT:    vle8.v v8, (a0)
+; ZVBB-NEXT:    vle8.v v13, (a1)
+; ZVBB-NEXT:    vle8.v v14, (a3)
+; ZVBB-NEXT:    vsetivli zero, 4, e8, mf2, tu, ma
+; ZVBB-NEXT:    vslideup.vi v12, v11, 2
+; ZVBB-NEXT:    vslideup.vi v8, v10, 2
+; ZVBB-NEXT:    vsetivli zero, 6, e8, mf2, tu, ma
+; ZVBB-NEXT:    vslideup.vi v12, v13, 4
+; ZVBB-NEXT:    vslideup.vi v8, v14, 4
+; ZVBB-NEXT:    vsetivli zero, 8, e8, mf2, ta, ma
+; ZVBB-NEXT:    vslideup.vi v8, v9, 6
+; ZVBB-NEXT:    vsetivli zero, 16, e8, m1, ta, ma
+; ZVBB-NEXT:    vslideup.vi v8, v12, 8
+; ZVBB-NEXT:    csrr a0, vlenb
+; ZVBB-NEXT:    add sp, sp, a0
+; ZVBB-NEXT:    .cfi_def_cfa sp, 16
+; ZVBB-NEXT:    addi sp, sp, 16
+; ZVBB-NEXT:    .cfi_def_cfa_offset 0
+; ZVBB-NEXT:    ret
+	   %res = call <14 x i8> @llvm.vector.interleave7.v14i8(<2 x i8> %a, <2 x i8> %b, <2 x i8> %c, <2 x i8> %d, <2 x i8> %e, <2 x i8> %f, <2 x i8> %g)
+	   ret <14 x i8> %res
+}
+
 
 ; Floats
 
@@ -270,13 +497,240 @@ define <4 x double> @vector_interleave_v4f64_v2f64(<2 x double> %a, <2 x double>
 	   ret <4 x double> %res
 }
 
+define <6 x float> @vector_interleave3_v632_v2f32(<2 x float> %a, <2 x float> %b, <2 x float> %c) {
+; CHECK-LABEL: vector_interleave3_v632_v2f32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    addi sp, sp, -16
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    csrr a0, vlenb
+; CHECK-NEXT:    slli a0, a0, 1
+; CHECK-NEXT:    sub sp, sp, a0
+; CHECK-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x02, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 2 * vlenb
+; CHECK-NEXT:    addi a0, sp, 16
+; CHECK-NEXT:    csrr a1, vlenb
+; CHECK-NEXT:    srli a1, a1, 1
+; CHECK-NEXT:    vsetvli a2, zero, e32, mf2, ta, ma
+; CHECK-NEXT:    vsseg3e32.v v8, (a0)
+; CHECK-NEXT:    add a2, a0, a1
+; CHECK-NEXT:    vle32.v v9, (a2)
+; CHECK-NEXT:    vle32.v v8, (a0)
+; CHECK-NEXT:    add a1, a2, a1
+; CHECK-NEXT:    vle32.v v10, (a1)
+; CHECK-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
+; CHECK-NEXT:    vslideup.vi v8, v9, 2
+; CHECK-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
+; CHECK-NEXT:    vslideup.vi v8, v10, 4
+; CHECK-NEXT:    csrr a0, vlenb
+; CHECK-NEXT:    slli a0, a0, 1
+; CHECK-NEXT:    add sp, sp, a0
+; CHECK-NEXT:    .cfi_def_cfa sp, 16
+; CHECK-NEXT:    addi sp, sp, 16
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-NEXT:    ret
+;
+; ZVBB-LABEL: vector_interleave3_v632_v2f32:
+; ZVBB:       # %bb.0:
+; ZVBB-NEXT:    addi sp, sp, -16
+; ZVBB-NEXT:    .cfi_def_cfa_offset 16
+; ZVBB-NEXT:    csrr a0, vlenb
+; ZVBB-NEXT:    slli a0, a0, 1
+; ZVBB-NEXT:    sub sp, sp, a0
+; ZVBB-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x02, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 2 * vlenb
+; ZVBB-NEXT:    addi a0, sp, 16
+; ZVBB-NEXT:    csrr a1, vlenb
+; ZVBB-NEXT:    srli a1, a1, 1
+; ZVBB-NEXT:    vsetvli a2, zero, e32, mf2, ta, ma
+; ZVBB-NEXT:    vsseg3e32.v v8, (a0)
+; ZVBB-NEXT:    add a2, a0, a1
+; ZVBB-NEXT:    vle32.v v9, (a2)
+; ZVBB-NEXT:    vle32.v v8, (a0)
+; ZVBB-NEXT:    add a1, a2, a1
+; ZVBB-NEXT:    vle32.v v10, (a1)
+; ZVBB-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
+; ZVBB-NEXT:    vslideup.vi v8, v9, 2
+; ZVBB-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
+; ZVBB-NEXT:    vslideup.vi v8, v10, 4
+; ZVBB-NEXT:    csrr a0, vlenb
+; ZVBB-NEXT:    slli a0, a0, 1
+; ZVBB-NEXT:    add sp, sp, a0
+; ZVBB-NEXT:    .cfi_def_cfa sp, 16
+; ZVBB-NEXT:    addi sp, sp, 16
+; ZVBB-NEXT:    .cfi_def_cfa_offset 0
+; ZVBB-NEXT:    ret
+	   %res = call <6 x float> @llvm.vector.interleave3.v6f32(<2 x float> %a, <2 x float> %b, <2 x float> %c)
+	   ret <6 x float> %res
+}
+
 
-declare <4 x half> @llvm.vector.interleave2.v4f16(<2 x half>, <2 x half>)
-declare <8 x half> @llvm.vector.interleave2.v8f16(<4 x half>, <4 x half>)
-declare <4 x float> @llvm.vector.interleave2.v4f32(<2 x float>, <2 x float>)
-declare <16 x half> @llvm.vector.interleave2.v16f16(<8 x half>, <8 x half>)
-declare <8 x float> @llvm.vector.interleave2.v8f32(<4 x float>, <4 x float>)
-declare <4 x double> @llvm.vector.interleave2.v4f64(<2 x double>, <2 x double>)
+define <10 x half> @vector_interleave5_v10f16_v2f16(<2 x half> %a, <2 x half> %b, <2 x half> %c, <2 x half> %d, <2 x half> %e) {
+; CHECK-LABEL: vector_interleave5_v10f16_v2f16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    addi sp, sp, -16
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    csrr a0, vlenb
+; CHECK-NEXT:    slli a0, a0, 1
+; CHECK-NEXT:    sub sp, sp, a0
+; CHECK-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x02, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 2 * vlenb
+; CHECK-NEXT:    addi a0, sp, 16
+; CHECK-NEXT:    csrr a1, vlenb
+; CHECK-NEXT:    srli a1, a1, 2
+; CHECK-NEXT:    add a2, a0, a1
+; CHECK-NEXT:    add a3, a2, a1
+; CHECK-NEXT:    vsetvli a4, zero, e16, mf4, ta, ma
+; CHECK-NEXT:    vsseg5e16.v v8, (a0)
+; CHECK-NEXT:    add a4, a3, a1
+; CHECK-NEXT:    vle16.v v9, (a2)
+; CHECK-NEXT:    vle16.v v10, (a4)
+; CHECK-NEXT:    vle16.v v11, (a3)
+; CHECK-NEXT:    vle16.v v8, (a0)
+; CHECK-NEXT:    add a1, a4, a1
+; CHECK-NEXT:    vle16.v v12, (a1)
+; CHECK-NEXT:    vsetivli zero, 4, e16, mf2, ta, ma
+; CHECK-NEXT:    vslideup.vi v11, v10, 2
+; CHECK-NEXT:    vslideup.vi v8, v9, 2
+; CHECK-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; CHECK-NEXT:    vslideup.vi v8, v11, 4
+; CHECK-NEXT:    vsetivli zero, 16, e16, m2, ta, ma
+; CHECK-NEXT:    vslideup.vi v8, v12, 8
+; CHECK-NEXT:    csrr a0, vlenb
+; CHECK-NEXT:    slli a0, a0, 1
+; CHECK-NEXT:    add sp, sp, a0
+; CHECK-NEXT:    .cfi_def_cfa sp, 16
+; CHECK-NEXT:    addi sp, sp, 16
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-NEXT:    ret
+;
+; ZVBB-LABEL: vector_interleave5_v10f16_v2f16:
+; ZVBB:       # %bb.0:
+; ZVBB-NEXT:    addi sp, sp, -16
+; ZVBB-NEXT:    .cfi_def_cfa_offset 16
+; ZVBB-NEXT:    csrr a0, vlenb
+; ZVBB-NEXT:    slli a0, a0, 1
+; ZVBB-NEXT:    sub sp, sp, a0
+; ZVBB-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x02, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 2 * vlenb
+; ZVBB-NEXT:    addi a0, sp, 16
+; ZVBB-NEXT:    csrr a1, vlenb
+; ZVBB-NEXT:    srli a1, a1, 2
+; ZVBB-NEXT:    add a2, a0, a1
+; ZVBB-NEXT:    add a3, a2, a1
+; ZVBB-NEXT:    vsetvli a4, zero, e16, mf4, ta, ma
+; ZVBB-NEXT:    vsseg5e16.v v8, (a0)
+; ZVBB-NEXT:    add a4, a3, a1
+; ZVBB-NEXT:    vle16.v v9, (a2)
+; ZVBB-NEXT:    vle16.v v10, (a4)
+; ZVBB-NEXT:    vle16.v v11, (a3)
+; ZVBB-NEXT:    vle16.v v8, (a0)
+; ZVBB-NEXT:    add a1, a4, a1
+; ZVBB-NEXT:    vle16.v v12, (a1)
+; ZVBB-NEXT:    vsetivli zero, 4, e16, mf2, ta, ma
+; ZVBB-NEXT:    vslideup.vi v11, v10, 2
+; ZVBB-NEXT:    vslideup.vi v8, v9, 2
+; ZVBB-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; ZVBB-NEXT:    vslideup.vi v8, v11, 4
+; ZVBB-NEXT:    vsetivli zero, 16, e16, m2, ta, ma
+; ZVBB-NEXT:    vslideup.vi v8, v12, 8
+; ZVBB-NEXT:    csrr a0, vlenb
+; ZVBB-NEXT:    slli a0, a0, 1
+; ZVBB-NEXT:    add sp, sp, a0
+; ZVBB-NEXT:    .cfi_def_cfa sp, 16
+; ZVBB-NEXT:    addi sp, sp, 16
+; ZVBB-NEXT:    .cfi_def_cfa_offset 0
+; ZVBB-NEXT:    ret
+	   %res = call <10 x half> @llvm.vector.interleave5.v10f16(<2 x half> %a, <2 x half> %b, <2 x half> %c, <2 x half> %d, <2 x half> %e)
+	   ret <10 x half> %res
+}
+
+define <7 x half> @vector_interleave7_v7f16_v1f16(<1 x half> %a, <1 x half> %b, <1 x half> %c, <1 x half> %d, <1 x half> %e, <1 x half> %f, <1 x half> %g) {
+; CHECK-LABEL: vector_interleave7_v7f16_v1f16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    addi sp, sp, -16
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    csrr a0, vlenb
+; CHECK-NEXT:    slli a0, a0, 1
+; CHECK-NEXT:    sub sp, sp, a0
+; CHECK-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x02, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 2 * vlenb
+; CHECK-NEXT:    addi a0, sp, 16
+; CHECK-NEXT:    csrr a1, vlenb
+; CHECK-NEXT:    srli a1, a1, 2
+; CHECK-NEXT:    add a2, a0, a1
+; CHECK-NEXT:    add a3, a2, a1
+; CHECK-NEXT:    add a4, a3, a1
+; CHECK-NEXT:    vsetvli a5, zero, e16, mf4, ta, ma
+; CHECK-NEXT:    vsseg7e16.v v8, (a0)
+; CHECK-NEXT:    vle16.v v9, (a4)
+; CHECK-NEXT:    add a4, a4, a1
+; CHECK-NEXT:    vle16.v v10, (a2)
+; CHECK-NEXT:    add a2, a4, a1
+; CHECK-NEXT:    add a1, a2, a1
+; CHECK-NEXT:    vle16.v v11, (a2)
+; CHECK-NEXT:    vle16.v v12, (a4)
+; CHECK-NEXT:    vle16.v v8, (a0)
+; CHECK-NEXT:    vle16.v v13, (a1)
+; CHECK-NEXT:    vle16.v v14, (a3)
+; CHECK-NEXT:    vsetivli zero, 2, e16, mf2, tu, ma
+; CHECK-NEXT:    vslideup.vi v12, v11, 1
+; CHECK-NEXT:    vslideup.vi v8, v10, 1
+; CHECK-NEXT:    vsetivli zero, 3, e16, mf2, tu, ma
+; CHECK-NEXT:    vslideup.vi v12, v13, 2
+; CHECK-NEXT:    vslideup.vi v8, v14, 2
+; CHECK-NEXT:    vsetivli zero, 4, e16, mf2, ta, ma
+; CHECK-NEXT:    vslideup.vi v8, v9, 3
+; CHECK-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; CHECK-NEXT:    vslideup.vi v8, v12, 4
+; CHECK-NEXT:    csrr a0, vlenb
+; CHECK-NEXT:    slli a0, a0, 1
+; CHECK-NEXT:    add sp, sp, a0
+; CHECK-NEXT:    .cfi_def_cfa sp, 16
+; CHECK-NEXT:    addi sp, sp, 16
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-NEXT:    ret
+;
+; ZVBB-LABEL: vector_interleave7_v7f16_v1f16:
+; ZVBB:       # %bb.0:
+; ZVBB-NEXT:    addi sp, sp, -16
+; ZVBB-NEXT:    .cfi_def_cfa_offset 16
+; ZVBB-NEXT:    csrr a0, vlenb
+; ZVBB-NEXT:    slli a0, a0, 1
+; ZVBB-NEXT:    sub sp, sp, a0
+; ZVBB-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x02, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 2 * vlenb
+; ZVBB-NEXT:    addi a0, sp, 16
+; ZVBB-NEXT:    csrr a1, vlenb
+; ZVBB-NEXT:    srli a1, a1, 2
+; ZVBB-NEXT:    add a2, a0, a1
+; ZVBB-NEXT:    add a3, a2, a1
+; ZVBB-NEXT:    add a4, a3, a1
+; ZVBB-NEXT:    vsetvli a5, zero, e16, mf4, ta, ma
+; ZVBB-NEXT:    vsseg7e16.v v8, (a0)
+; ZVBB-NEXT:    vle16.v v9, (a4)
+; ZVBB-NEXT:    add a4, a4, a1
+; ZVBB-NEXT:    vle16.v v10, (a2)
+; ZVBB-NEXT:    add a2, a4, a1
+; ZVBB-NEXT:    add a1, a2, a1
+; ZVBB-NEXT:    vle16.v v11, (a2)
+; ZVBB-NEXT:    vle16.v v12, (a4)
+; ZVBB-NEXT:    vle16.v v8, (a0)
+; ZVBB-NEXT:    vle16.v v13, (a1)
+; ZVBB-NEXT:    vle16.v v14, (a3)
+; ZVBB-NEXT:    vsetivli zero, 2, e16, mf2, tu, ma
+; ZVBB-NEXT:    vslideup.vi v12, v11, 1
+; ZVBB-NEXT:    vslideup.vi v8, v10, 1
+; ZVBB-NEXT:    vsetivli zero, 3, e16, mf2, tu, ma
+; ZVBB-NEXT:    vslideup.vi v12, v13, 2
+; ZVBB-NEXT:    vslideup.vi v8, v14, 2
+; ZVBB-NEXT:    vsetivli zero, 4, e16, mf2, ta, ma
+; ZVBB-NEXT:    vslideup.vi v8, v9, 3
+; ZVBB-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; ZVBB-NEXT:    vslideup.vi v8, v12, 4
+; ZVBB-NEXT:    csrr a0, vlenb
+; ZVBB-NEXT:    slli a0, a0, 1
+; ZVBB-NEXT:    add sp, sp, a0
+; ZVBB-NEXT:    .cfi_def_cfa sp, 16
+; ZVBB-NEXT:    addi sp, sp, 16
+; ZVBB-NEXT:    .cfi_def_cfa_offset 0
+; ZVBB-NEXT:    ret
+	   %res = call <7 x half> @llvm.vector.interleave7.v7f16(<1 x half> %a, <1 x half> %b, <1 x half> %c, <1 x half> %d, <1 x half> %e, <1 x half> %f, <1 x half> %g)
+	   ret <7 x half> %res
+}
 ;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
 ; RV32: {{.*}}
 ; RV64: {{.*}}
diff --git a/llvm/test/CodeGen/RISCV/rvv/vector-interleave.ll b/llvm/test/CodeGen/RISCV/rvv/vector-interleave.ll
index 864acb320d8fe..6aa62c2256925 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vector-interleave.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vector-interleave.ll
@@ -1,10 +1,10 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s -mtriple=riscv32 -mattr=+v,+zvfh,+zvfbfmin | FileCheck %s
-; RUN: llc < %s -mtriple=riscv64 -mattr=+v,+zvfh,+zvfbfmin | FileCheck %s
-; RUN: llc < %s -mtriple=riscv32 -mattr=+v,+zvfhmin,+zvfbfmin | FileCheck %s
-; RUN: llc < %s -mtriple=riscv64 -mattr=+v,+zvfhmin,+zvfbfmin | FileCheck %s
-; RUN: llc < %s -mtriple=riscv32 -mattr=+v,+zvbb,+zvfh,+zvfbfmin | FileCheck %s --check-prefix=ZVBB
-; RUN: llc < %s -mtriple=riscv64 -mattr=+v,+zvbb,+zvfh,+zvfbfmin | FileCheck %s --check-prefix=ZVBB
+; RUN: llc < %s -mtriple=riscv32 -mattr=+m,+v,+zvfh,+zvfbfmin | FileCheck %s --check-prefixes=CHECK,RV32
+; RUN: llc < %s -mtriple=riscv64 -mattr=+m,+v,+zvfh,+zvfbfmin | FileCheck %s --check-prefixes=CHECK,RV64
+; RUN: llc < %s -mtriple=riscv32 -mattr=+m,+v,+zvfhmin,+zvfbfmin | FileCheck %s --check-prefixes=CHECK,RV32
+; RUN: llc < %s -mtriple=riscv64 -mattr=+m,+v,+zvfhmin,+zvfbfmin | FileCheck %s --check-prefixes=CHECK,RV64
+; RUN: llc < %s -mtriple=riscv32 -mattr=+m,+v,+zvbb,+zvfh,+zvfbfmin | FileCheck %s --check-prefixes=ZVBB,ZVBB-RV32
+; RUN: llc < %s -mtriple=riscv64 -mattr=+m,+v,+zvbb,+zvfh,+zvfbfmin | FileCheck %s --check-prefixes=ZVBB,ZVBB-RV64
 
 ; Integers
 
@@ -151,11 +151,6 @@ define <vscale x 4 x i64> @vector_interleave_nxv4i64_nxv2i64(<vscale x 2 x i64>
   ret <vscale x 4 x i64> %res
 }
 
-declare <vscale x 32 x i1> @llvm.vector.interleave2.nxv32i1(<vscale x 16 x i1>, <vscale x 16 x i1>)
-declare <vscale x 32 x i8> @llvm.vector.interleave2.nxv32i8(<vscale x 16 x i8>, <vscale x 16 x i8>)
-declare <vscale x 16 x i16> @llvm.vector.interleave2.nxv16i16(<vscale x 8 x i16>, <vscale x 8 x i16>)
-declare <vscale x 8 x i32> @llvm.vector.interleave2.nxv8i32(<vscale x 4 x i32>, <vscale x 4 x i32>)
-declare <vscale x 4 x i64> @llvm.vector.interleave2.nxv4i64(<vscale x 2 x i64>, <vscale x 2 x i64>)
 
 define <vscale x 128 x i1> @vector_interleave_nxv128i1_nxv64i1(<vscale x 64 x i1> %a, <vscale x 64 x i1> %b) {
 ; CHECK-LABEL: vector_interleave_nxv128i1_nxv64i1:
@@ -216,11 +211,11 @@ define <vscale x 128 x i8> @vector_interleave_nxv128i8_nxv64i8(<vscale x 64 x i8
 ; ZVBB-LABEL: vector_interleave_nxv128i8_nxv64i8:
 ; ZVBB:       # %bb.0:
 ; ZVBB-NEXT:    vsetvli a0, zero, e8, m4, ta, ma
-; ZVBB-NEXT:    vmv8r.v v24, v8
-; ZVBB-NEXT:    vwsll.vi v8, v16, 8
+; ZVBB-NEXT:    vwsll.vi v24, v16, 8
 ; ZVBB-NEXT:    vwsll.vi v0, v20, 8
-; ZVBB-NEXT:    vwaddu.wv v8, v8, v24
-; ZVBB-NEXT:    vwaddu.wv v0, v0, v28
+; ZVBB-NEXT:    vwaddu.wv v24, v24, v8
+; ZVBB-NEXT:    vwaddu.wv v0, v0, v12
+; ZVBB-NEXT:    vmv8r.v v8, v24
 ; ZVBB-NEXT:    vmv8r.v v16, v0
 ; ZVBB-NEXT:    ret
   %res = call <vscale x 128 x i8> @llvm.vector.interleave2.nxv128i8(<vscale x 64 x i8> %a, <vscale x 64 x i8> %b)
@@ -243,11 +238,11 @@ define <vscale x 64 x i16> @vector_interleave_nxv64i16_nxv32i16(<vscale x 32 x i
 ; ZVBB-LABEL: vector_interleave_nxv64i16_nxv32i16:
 ; ZVBB:       # %bb.0:
 ; ZVBB-NEXT:    vsetvli a0, zero, e16, m4, ta, ma
-; ZVBB-NEXT:    vmv8r.v v24, v8
-; ZVBB-NEXT:    vwsll.vi v8, v16, 16
+; ZVBB-NEXT:    vwsll.vi v24, v16, 16
 ; ZVBB-NEXT:    vwsll.vi v0, v20, 16
-; ZVBB-NEXT:    vwaddu.wv v8, v8, v24
-; ZVBB-NEXT:    vwaddu.wv v0, v0, v28
+; ZVBB-NEXT:    vwaddu.wv v24, v24, v8
+; ZVBB-NEXT:    vwaddu.wv v0, v0, v12
+; ZVBB-NEXT:    vmv8r.v v8, v24
 ; ZVBB-NEXT:    vmv8r.v v16, v0
 ; ZVBB-NEXT:    ret
   %res = call <vscale x 64 x i16> @llvm.vector.interleave2.nxv64i16(<vscale x 32 x i16> %a, <vscale x 32 x i16> %b)
@@ -269,13 +264,13 @@ define <vscale x 32 x i32> @vector_interleave_nxv32i32_nxv16i32(<vscale x 16 x i
 ;
 ; ZVBB-LABEL: vector_interleave_nxv32i32_nxv16i32:
 ; ZVBB:       # %bb.0:
-; ZVBB-NEXT:    vsetvli a0, zero, e32, m4, ta, ma
-; ZVBB-NEXT:    vmv8r.v v24, v8
 ; ZVBB-NEXT:    li a0, 32
-; ZVBB-NEXT:    vwsll.vx v8, v16, a0
+; ZVBB-NEXT:    vsetvli a1, zero, e32, m4, ta, ma
+; ZVBB-NEXT:    vwsll.vx v24, v16, a0
 ; ZVBB-NEXT:    vwsll.vx v0, v20, a0
-; ZVBB-NEXT:    vwaddu.wv v8, v8, v24
-; ZVBB-NEXT:    vwaddu.wv v0, v0, v28
+; ZVBB-NEXT:    vwaddu.wv v24, v24, v8
+; ZVBB-NEXT:    vwaddu.wv v0, v0, v12
+; ZVBB-NEXT:    vmv8r.v v8, v24
 ; ZVBB-NEXT:    vmv8r.v v16, v0
 ; ZVBB-NEXT:    ret
   %res = call <vscale x 32 x i32> @llvm.vector.interleave2.nxv32i32(<vscale x 16 x i32> %a, <vscale x 16 x i32> %b)
@@ -324,11 +319,6 @@ define <vscale x 16 x i64> @vector_interleave_nxv16i64_nxv8i64(<vscale x 8 x i64
   ret <vscale x 16 x i64> %res
 }
 
-declare <vscale x 128 x i1> @llvm.vector.interleave2.nxv128i1(<vscale x 64 x i1>, <vscale x 64 x i1>)
-declare <vscale x 128 x i8> @llvm.vector.interleave2.nxv128i8(<vscale x 64 x i8>, <vscale x 64 x i8>)
-declare <vscale x 64 x i16> @llvm.vector.interleave2.nxv64i16(<vscale x 32 x i16>, <vscale x 32 x i16>)
-declare <vscale x 32 x i32> @llvm.vector.interleave2.nxv32i32(<vscale x 16 x i32>, <vscale x 16 x i32>)
-declare <vscale x 16 x i64> @llvm.vector.interleave2.nxv16i64(<vscale x 8 x i64>, <vscale x 8 x i64>)
 
 ; Floats
 
@@ -565,12 +555,6 @@ define <vscale x 4 x double> @vector_interleave_nxv4f64_nxv2f64(<vscale x 2 x do
 }
 
 
-declare <vscale x 4 x half> @llvm.vector.interleave2.nxv4f16(<vscale x 2 x half>, <vscale x 2 x half>)
-declare <vscale x 8 x half> @llvm.vector.interleave2.nxv8f16(<vscale x 4 x half>, <vscale x 4 x half>)
-declare <vscale x 4 x float> @llvm.vector.interleave2.nxv4f32(<vscale x 2 x float>, <vscale x 2 x float>)
-declare <vscale x 16 x half> @llvm.vector.interleave2.nxv16f16(<vscale x 8 x half>, <vscale x 8 x half>)
-declare <vscale x 8 x float> @llvm.vector.interleave2.nxv8f32(<vscale x 4 x float>, <vscale x 4 x float>)
-declare <vscale x 4 x double> @llvm.vector.interleave2.nxv4f64(<vscale x 2 x double>, <vscale x 2 x double>)
 
 define <vscale x 64 x bfloat> @vector_interleave_nxv64bf16_nxv32bf16(<vscale x 32 x bfloat> %a, <vscale x 32 x bfloat> %b) {
 ; CHECK-LABEL: vector_interleave_nxv64bf16_nxv32bf16:
@@ -588,11 +572,11 @@ define <vscale x 64 x bfloat> @vector_interleave_nxv64bf16_nxv32bf16(<vscale x 3
 ; ZVBB-LABEL: vector_interleave_nxv64bf16_nxv32bf16:
 ; ZVBB:       # %bb.0:
 ; ZVBB-NEXT:    vsetvli a0, zero, e16, m4, ta, ma
-; ZVBB-NEXT:    vmv8r.v v24, v8
-; ZVBB-NEXT:    vwsll.vi v8, v16, 16
+; ZVBB-NEXT:    vwsll.vi v24, v16, 16
 ; ZVBB-NEXT:    vwsll.vi v0, v20, 16
-; ZVBB-NEXT:    vwaddu.wv v8, v8, v24
-; ZVBB-NEXT:    vwaddu.wv v0, v0, v28
+; ZVBB-NEXT:    vwaddu.wv v24, v24, v8
+; ZVBB-NEXT:    vwaddu.wv v0, v0, v12
+; ZVBB-NEXT:    vmv8r.v v8, v24
 ; ZVBB-NEXT:    vmv8r.v v16, v0
 ; ZVBB-NEXT:    ret
   %res = call <vscale x 64 x bfloat> @llvm.vector.interleave2.nxv64bf16(<vscale x 32 x bfloat> %a, <vscale x 32 x bfloat> %b)
@@ -615,11 +599,11 @@ define <vscale x 64 x half> @vector_interleave_nxv64f16_nxv32f16(<vscale x 32 x
 ; ZVBB-LABEL: vector_interleave_nxv64f16_nxv32f16:
 ; ZVBB:       # %bb.0:
 ; ZVBB-NEXT:    vsetvli a0, zero, e16, m4, ta, ma
-; ZVBB-NEXT:    vmv8r.v v24, v8
-; ZVBB-NEXT:    vwsll.vi v8, v16, 16
+; ZVBB-NEXT:    vwsll.vi v24, v16, 16
 ; ZVBB-NEXT:    vwsll.vi v0, v20, 16
-; ZVBB-NEXT:    vwaddu.wv v8, v8, v24
-; ZVBB-NEXT:    vwaddu.wv v0, v0, v28
+; ZVBB-NEXT:    vwaddu.wv v24, v24, v8
+; ZVBB-NEXT:    vwaddu.wv v0, v0, v12
+; ZVBB-NEXT:    vmv8r.v v8, v24
 ; ZVBB-NEXT:    vmv8r.v v16, v0
 ; ZVBB-NEXT:    ret
   %res = call <vscale x 64 x half> @llvm.vector.interleave2.nxv64f16(<vscale x 32 x half> %a, <vscale x 32 x half> %b)
@@ -641,13 +625,13 @@ define <vscale x 32 x float> @vector_interleave_nxv32f32_nxv16f32(<vscale x 16 x
 ;
 ; ZVBB-LABEL: vector_interleave_nxv32f32_nxv16f32:
 ; ZVBB:       # %bb.0:
-; ZVBB-NEXT:    vsetvli a0, zero, e32, m4, ta, ma
-; ZVBB-NEXT:    vmv8r.v v24, v8
 ; ZVBB-NEXT:    li a0, 32
-; ZVBB-NEXT:    vwsll.vx v8, v16, a0
+; ZVBB-NEXT:    vsetvli a1, zero, e32, m4, ta, ma
+; ZVBB-NEXT:    vwsll.vx v24, v16, a0
 ; ZVBB-NEXT:    vwsll.vx v0, v20, a0
-; ZVBB-NEXT:    vwaddu.wv v8, v8, v24
-; ZVBB-NEXT:    vwaddu.wv v0, v0, v28
+; ZVBB-NEXT:    vwaddu.wv v24, v24, v8
+; ZVBB-NEXT:    vwaddu.wv v0, v0, v12
+; ZVBB-NEXT:    vmv8r.v v8, v24
 ; ZVBB-NEXT:    vmv8r.v v16, v0
 ; ZVBB-NEXT:    ret
   %res = call <vscale x 32 x float> @llvm.vector.interleave2.nxv32f32(<vscale x 16 x float> %a, <vscale x 16 x float> %b)
@@ -734,6 +718,2856 @@ define <vscale x 8 x i32> @vector_interleave_nxv8i32_nxv4i32_poison2(<vscale x 4
   ret <vscale x 8 x i32> %res
 }
 
-declare <vscale x 64 x half> @llvm.vector.interleave2.nxv64f16(<vscale x 32 x half>, <vscale x 32 x half>)
-declare <vscale x 32 x float> @llvm.vector.interleave2.nxv32f32(<vscale x 16 x float>, <vscale x 16 x float>)
-declare <vscale x 16 x double> @llvm.vector.interleave2.nxv16f64(<vscale x 8 x double>, <vscale x 8 x double>)
+define <vscale x 48 x i1> @vector_interleave_nxv48i1_nxv16i1(<vscale x 16 x i1> %a, <vscale x 16 x i1> %b, <vscale x 16 x i1> %c) nounwind {
+; CHECK-LABEL: vector_interleave_nxv48i1_nxv16i1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    addi sp, sp, -16
+; CHECK-NEXT:    csrr a0, vlenb
+; CHECK-NEXT:    li a1, 6
+; CHECK-NEXT:    mul a0, a0, a1
+; CHECK-NEXT:    sub sp, sp, a0
+; CHECK-NEXT:    vsetvli a0, zero, e8, m2, ta, ma
+; CHECK-NEXT:    vmv1r.v v10, v0
+; CHECK-NEXT:    vmv1r.v v0, v8
+; CHECK-NEXT:    vmv.v.i v12, 0
+; CHECK-NEXT:    addi a0, sp, 16
+; CHECK-NEXT:    csrr a1, vlenb
+; CHECK-NEXT:    vmerge.vim v16, v12, 1, v0
+; CHECK-NEXT:    slli a2, a1, 1
+; CHECK-NEXT:    vmv1r.v v0, v10
+; CHECK-NEXT:    vmerge.vim v14, v12, 1, v0
+; CHECK-NEXT:    add a3, a0, a2
+; CHECK-NEXT:    vmv1r.v v0, v9
+; CHECK-NEXT:    vmerge.vim v18, v12, 1, v0
+; CHECK-NEXT:    add a2, a3, a2
+; CHECK-NEXT:    vsseg3e8.v v14, (a0)
+; CHECK-NEXT:    vl2r.v v8, (a2)
+; CHECK-NEXT:    srli a2, a1, 2
+; CHECK-NEXT:    srli a1, a1, 1
+; CHECK-NEXT:    vl2r.v v10, (a3)
+; CHECK-NEXT:    vl2r.v v12, (a0)
+; CHECK-NEXT:    add a0, a2, a2
+; CHECK-NEXT:    vmsne.vi v14, v8, 0
+; CHECK-NEXT:    vmsne.vi v8, v10, 0
+; CHECK-NEXT:    vmsne.vi v0, v12, 0
+; CHECK-NEXT:    vsetvli zero, a0, e8, mf2, ta, ma
+; CHECK-NEXT:    vslideup.vx v0, v8, a2
+; CHECK-NEXT:    add a0, a1, a1
+; CHECK-NEXT:    vsetvli zero, a0, e8, m1, ta, ma
+; CHECK-NEXT:    vslideup.vx v0, v14, a1
+; CHECK-NEXT:    csrr a0, vlenb
+; CHECK-NEXT:    li a1, 6
+; CHECK-NEXT:    mul a0, a0, a1
+; CHECK-NEXT:    add sp, sp, a0
+; CHECK-NEXT:    addi sp, sp, 16
+; CHECK-NEXT:    ret
+;
+; ZVBB-LABEL: vector_interleave_nxv48i1_nxv16i1:
+; ZVBB:       # %bb.0:
+; ZVBB-NEXT:    addi sp, sp, -16
+; ZVBB-NEXT:    csrr a0, vlenb
+; ZVBB-NEXT:    li a1, 6
+; ZVBB-NEXT:    mul a0, a0, a1
+; ZVBB-NEXT:    sub sp, sp, a0
+; ZVBB-NEXT:    vsetvli a0, zero, e8, m2, ta, ma
+; ZVBB-NEXT:    vmv1r.v v10, v0
+; ZVBB-NEXT:    vmv1r.v v0, v8
+; ZVBB-NEXT:    vmv.v.i v12, 0
+; ZVBB-NEXT:    addi a0, sp, 16
+; ZVBB-NEXT:    csrr a1, vlenb
+; ZVBB-NEXT:    vmerge.vim v16, v12, 1, v0
+; ZVBB-NEXT:    slli a2, a1, 1
+; ZVBB-NEXT:    vmv1r.v v0, v10
+; ZVBB-NEXT:    vmerge.vim v14, v12, 1, v0
+; ZVBB-NEXT:    add a3, a0, a2
+; ZVBB-NEXT:    vmv1r.v v0, v9
+; ZVBB-NEXT:    vmerge.vim v18, v12, 1, v0
+; ZVBB-NEXT:    add a2, a3, a2
+; ZVBB-NEXT:    vsseg3e8.v v14, (a0)
+; ZVBB-NEXT:    vl2r.v v8, (a2)
+; ZVBB-NEXT:    srli a2, a1, 2
+; ZVBB-NEXT:    srli a1, a1, 1
+; ZVBB-NEXT:    vl2r.v v10, (a3)
+; ZVBB-NEXT:    vl2r.v v12, (a0)
+; ZVBB-NEXT:    add a0, a2, a2
+; ZVBB-NEXT:    vmsne.vi v14, v8, 0
+; ZVBB-NEXT:    vmsne.vi v8, v10, 0
+; ZVBB-NEXT:    vmsne.vi v0, v12, 0
+; ZVBB-NEXT:    vsetvli zero, a0, e8, mf2, ta, ma
+; ZVBB-NEXT:    vslideup.vx v0, v8, a2
+; ZVBB-NEXT:    add a0, a1, a1
+; ZVBB-NEXT:    vsetvli zero, a0, e8, m1, ta, ma
+; ZVBB-NEXT:    vslideup.vx v0, v14, a1
+; ZVBB-NEXT:    csrr a0, vlenb
+; ZVBB-NEXT:    li a1, 6
+; ZVBB-NEXT:    mul a0, a0, a1
+; ZVBB-NEXT:    add sp, sp, a0
+; ZVBB-NEXT:    addi sp, sp, 16
+; ZVBB-NEXT:    ret
+  %res = call <vscale x 48 x i1> @llvm.vector.interleave3.nxv48i1(<vscale x 16 x i1> %a, <vscale x 16 x i1> %b, <vscale x 16 x i1> %c)
+  ret <vscale x 48 x i1> %res
+}
+
+
+define <vscale x 48 x i8> @vector_interleave_nxv48i8_nxv16i8(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b, <vscale x 16 x i8> %c) nounwind {
+; CHECK-LABEL: vector_interleave_nxv48i8_nxv16i8:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    addi sp, sp, -16
+; CHECK-NEXT:    csrr a0, vlenb
+; CHECK-NEXT:    li a1, 6
+; CHECK-NEXT:    mul a0, a0, a1
+; CHECK-NEXT:    sub sp, sp, a0
+; CHECK-NEXT:    addi a0, sp, 16
+; CHECK-NEXT:    csrr a1, vlenb
+; CHECK-NEXT:    slli a1, a1, 1
+; CHECK-NEXT:    vsetvli a2, zero, e8, m2, ta, ma
+; CHECK-NEXT:    vsseg3e8.v v8, (a0)
+; CHECK-NEXT:    vl2r.v v8, (a0)
+; CHECK-NEXT:    add a0, a0, a1
+; CHECK-NEXT:    vl2r.v v10, (a0)
+; CHECK-NEXT:    add a0, a0, a1
+; CHECK-NEXT:    vl2r.v v12, (a0)
+; CHECK-NEXT:    csrr a0, vlenb
+; CHECK-NEXT:    li a1, 6
+; CHECK-NEXT:    mul a0, a0, a1
+; CHECK-NEXT:    add sp, sp, a0
+; CHECK-NEXT:    addi sp, sp, 16
+; CHECK-NEXT:    ret
+;
+; ZVBB-LABEL: vector_interleave_nxv48i8_nxv16i8:
+; ZVBB:       # %bb.0:
+; ZVBB-NEXT:    addi sp, sp, -16
+; ZVBB-NEXT:    csrr a0, vlenb
+; ZVBB-NEXT:    li a1, 6
+; ZVBB-NEXT:    mul a0, a0, a1
+; ZVBB-NEXT:    sub sp, sp, a0
+; ZVBB-NEXT:    addi a0, sp, 16
+; ZVBB-NEXT:    csrr a1, vlenb
+; ZVBB-NEXT:    slli a1, a1, 1
+; ZVBB-NEXT:    vsetvli a2, zero, e8, m2, ta, ma
+; ZVBB-NEXT:    vsseg3e8.v v8, (a0)
+; ZVBB-NEXT:    vl2r.v v8, (a0)
+; ZVBB-NEXT:    add a0, a0, a1
+; ZVBB-NEXT:    vl2r.v v10, (a0)
+; ZVBB-NEXT:    add a0, a0, a1
+; ZVBB-NEXT:    vl2r.v v12, (a0)
+; ZVBB-NEXT:    csrr a0, vlenb
+; ZVBB-NEXT:    li a1, 6
+; ZVBB-NEXT:    mul a0, a0, a1
+; ZVBB-NEXT:    add sp, sp, a0
+; ZVBB-NEXT:    addi sp, sp, 16
+; ZVBB-NEXT:    ret
+  %res = call <vscale x 48 x i8> @llvm.vector.interleave3.nxv48i8(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b, <vscale x 16 x i8> %c)
+  ret <vscale x 48 x i8> %res
+}
+
+
+define <vscale x 24 x i16> @vector_interleave_nxv24i16_nxv8i16(<vscale x 8 x i16> %a, <vscale x 8 x i16> %b, <vscale x 8 x i16> %c) nounwind {
+; CHECK-LABEL: vector_interleave_nxv24i16_nxv8i16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    addi sp, sp, -16
+; CHECK-NEXT:    csrr a0, vlenb
+; CHECK-NEXT:    li a1, 6
+; CHECK-NEXT:    mul a0, a0, a1
+; CHECK-NEXT:    sub sp, sp, a0
+; CHECK-NEXT:    addi a0, sp, 16
+; CHECK-NEXT:    csrr a1, vlenb
+; CHECK-NEXT:    slli a1, a1, 1
+; CHECK-NEXT:    vsetvli a2, zero, e16, m2, ta, ma
+; CHECK-NEXT:    vsseg3e16.v v8, (a0)
+; CHECK-NEXT:    vl2re16.v v8, (a0)
+; CHECK-NEXT:    add a0, a0, a1
+; CHECK-NEXT:    vl2re16.v v10, (a0)
+; CHECK-NEXT:    add a0, a0, a1
+; CHECK-NEXT:    vl2re16.v v12, (a0)
+; CHECK-NEXT:    csrr a0, vlenb
+; CHECK-NEXT:    li a1, 6
+; CHECK-NEXT:    mul a0, a0, a1
+; CHECK-NEXT:    add sp, sp, a0
+; CHECK-NEXT:    addi sp, sp, 16
+; CHECK-NEXT:    ret
+;
+; ZVBB-LABEL: vector_interleave_nxv24i16_nxv8i16:
+; ZVBB:       # %bb.0:
+; ZVBB-NEXT:    addi sp, sp, -16
+; ZVBB-NEXT:    csrr a0, vlenb
+; ZVBB-NEXT:    li a1, 6
+; ZVBB-NEXT:    mul a0, a0, a1
+; ZVBB-NEXT:    sub sp, sp, a0
+; ZVBB-NEXT:    addi a0, sp, 16
+; ZVBB-NEXT:    csrr a1, vlenb
+; ZVBB-NEXT:    slli a1, a1, 1
+; ZVBB-NEXT:    vsetvli a2, zero, e16, m2, ta, ma
+; ZVBB-NEXT:    vsseg3e16.v v8, (a0)
+; ZVBB-NEXT:    vl2re16.v v8, (a0)
+; ZVBB-NEXT:    add a0, a0, a1
+; ZVBB-NEXT:    vl2re16.v v10, (a0)
+; ZVBB-NEXT:    add a0, a0, a1
+; ZVBB-NEXT:    vl2re16.v v12, (a0)
+; ZVBB-NEXT:    csrr a0, vlenb
+; ZVBB-NEXT:    li a1, 6
+; ZVBB-NEXT:    mul a0, a0, a1
+; ZVBB-NEXT:    add sp, sp, a0
+; ZVBB-NEXT:    addi sp, sp, 16
+; ZVBB-NEXT:    ret
+  %res = call <vscale x 24 x i16> @llvm.vector.interleave3.nxv24i16(<vscale x 8 x i16> %a, <vscale x 8 x i16> %b, <vscale x 8 x i16> %c)
+  ret <vscale x 24 x i16> %res
+}
+
+
+define <vscale x 12 x i32> @vector_interleave_nxv12i32_nxv4i32(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b, <vscale x 4 x i32> %c) nounwind {
+; CHECK-LABEL: vector_interleave_nxv12i32_nxv4i32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    addi sp, sp, -16
+; CHECK-NEXT:    csrr a0, vlenb
+; CHECK-NEXT:    li a1, 6
+; CHECK-NEXT:    mul a0, a0, a1
+; CHECK-NEXT:    sub sp, sp, a0
+; CHECK-NEXT:    addi a0, sp, 16
+; CHECK-NEXT:    csrr a1, vlenb
+; CHECK-NEXT:    slli a1, a1, 1
+; CHECK-NEXT:    vsetvli a2, zero, e32, m2, ta, ma
+; CHECK-NEXT:    vsseg3e32.v v8, (a0)
+; CHECK-NEXT:    vl2re32.v v8, (a0)
+; CHECK-NEXT:    add a0, a0, a1
+; CHECK-NEXT:    vl2re32.v v10, (a0)
+; CHECK-NEXT:    add a0, a0, a1
+; CHECK-NEXT:    vl2re32.v v12, (a0)
+; CHECK-NEXT:    csrr a0, vlenb
+; CHECK-NEXT:    li a1, 6
+; CHECK-NEXT:    mul a0, a0, a1
+; CHECK-NEXT:    add sp, sp, a0
+; CHECK-NEXT:    addi sp, sp, 16
+; CHECK-NEXT:    ret
+;
+; ZVBB-LABEL: vector_interleave_nxv12i32_nxv4i32:
+; ZVBB:       # %bb.0:
+; ZVBB-NEXT:    addi sp, sp, -16
+; ZVBB-NEXT:    csrr a0, vlenb
+; ZVBB-NEXT:    li a1, 6
+; ZVBB-NEXT:    mul a0, a0, a1
+; ZVBB-NEXT:    sub sp, sp, a0
+; ZVBB-NEXT:    addi a0, sp, 16
+; ZVBB-NEXT:    csrr a1, vlenb
+; ZVBB-NEXT:    slli a1, a1, 1
+; ZVBB-NEXT:    vsetvli a2, zero, e32, m2, ta, ma
+; ZVBB-NEXT:    vsseg3e32.v v8, (a0)
+; ZVBB-NEXT:    vl2re32.v v8, (a0)
+; ZVBB-NEXT:    add a0, a0, a1
+; ZVBB-NEXT:    vl2re32.v v10, (a0)
+; ZVBB-NEXT:    add a0, a0, a1
+; ZVBB-NEXT:    vl2re32.v v12, (a0)
+; ZVBB-NEXT:    csrr a0, vlenb
+; ZVBB-NEXT:    li a1, 6
+; ZVBB-NEXT:    mul a0, a0, a1
+; ZVBB-NEXT:    add sp, sp, a0
+; ZVBB-NEXT:    addi sp, sp, 16
+; ZVBB-NEXT:    ret
+  %res = call <vscale x 12 x i32> @llvm.vector.interleave3.nxv12i32(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b, <vscale x 4 x i32> %c)
+  ret <vscale x 12 x i32> %res
+}
+
+
+define <vscale x 6 x i64> @vector_interleave_nxv6i64_nxv2i64(<vscale x 2 x i64> %a, <vscale x 2 x i64> %b, <vscale x 2 x i64> %c) nounwind {
+; CHECK-LABEL: vector_interleave_nxv6i64_nxv2i64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    addi sp, sp, -16
+; CHECK-NEXT:    csrr a0, vlenb
+; CHECK-NEXT:    li a1, 6
+; CHECK-NEXT:    mul a0, a0, a1
+; CHECK-NEXT:    sub sp, sp, a0
+; CHECK-NEXT:    addi a0, sp, 16
+; CHECK-NEXT:    csrr a1, vlenb
+; CHECK-NEXT:    slli a1, a1, 1
+; CHECK-NEXT:    vsetvli a2, zero, e64, m2, ta, ma
+; CHECK-NEXT:    vsseg3e64.v v8, (a0)
+; CHECK-NEXT:    vl2re64.v v8, (a0)
+; CHECK-NEXT:    add a0, a0, a1
+; CHECK-NEXT:    vl2re64.v v10, (a0)
+; CHECK-NEXT:    add a0, a0, a1
+; CHECK-NEXT:    vl2re64.v v12, (a0)
+; CHECK-NEXT:    csrr a0, vlenb
+; CHECK-NEXT:    li a1, 6
+; CHECK-NEXT:    mul a0, a0, a1
+; CHECK-NEXT:    add sp, sp, a0
+; CHECK-NEXT:    addi sp, sp, 16
+; CHECK-NEXT:    ret
+;
+; ZVBB-LABEL: vector_interleave_nxv6i64_nxv2i64:
+; ZVBB:       # %bb.0:
+; ZVBB-NEXT:    addi sp, sp, -16
+; ZVBB-NEXT:    csrr a0, vlenb
+; ZVBB-NEXT:    li a1, 6
+; ZVBB-NEXT:    mul a0, a0, a1
+; ZVBB-NEXT:    sub sp, sp, a0
+; ZVBB-NEXT:    addi a0, sp, 16
+; ZVBB-NEXT:    csrr a1, vlenb
+; ZVBB-NEXT:    slli a1, a1, 1
+; ZVBB-NEXT:    vsetvli a2, zero, e64, m2, ta, ma
+; ZVBB-NEXT:    vsseg3e64.v v8, (a0)
+; ZVBB-NEXT:    vl2re64.v v8, (a0)
+; ZVBB-NEXT:    add a0, a0, a1
+; ZVBB-NEXT:    vl2re64.v v10, (a0)
+; ZVBB-NEXT:    add a0, a0, a1
+; ZVBB-NEXT:    vl2re64.v v12, (a0)
+; ZVBB-NEXT:    csrr a0, vlenb
+; ZVBB-NEXT:    li a1, 6
+; ZVBB-NEXT:    mul a0, a0, a1
+; ZVBB-NEXT:    add sp, sp, a0
+; ZVBB-NEXT:    addi sp, sp, 16
+; ZVBB-NEXT:    ret
+  %res = call <vscale x 6 x i64> @llvm.vector.interleave3.nxv6i64(<vscale x 2 x i64> %a, <vscale x 2 x i64> %b, <vscale x 2 x i64> %c)
+  ret <vscale x 6 x i64> %res
+}
+
+define <vscale x 80 x i1> @vector_interleave_nxv80i1_nxv16i1(<vscale x 16 x i1> %a, <vscale x 16 x i1> %b, <vscale x 16 x i1> %c, <vscale x 16 x i1> %d, <vscale x 16 x i1> %e) nounwind {
+; CHECK-LABEL: vector_interleave_nxv80i1_nxv16i1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    addi sp, sp, -16
+; CHECK-NEXT:    csrr a0, vlenb
+; CHECK-NEXT:    li a1, 10
+; CHECK-NEXT:    mul a0, a0, a1
+; CHECK-NEXT:    sub sp, sp, a0
+; CHECK-NEXT:    vsetvli a0, zero, e8, m2, ta, ma
+; CHECK-NEXT:    vmv.v.i v12, 0
+; CHECK-NEXT:    addi a4, sp, 16
+; CHECK-NEXT:    csrr a0, vlenb
+; CHECK-NEXT:    slli a1, a0, 2
+; CHECK-NEXT:    add a0, a1, a0
+; CHECK-NEXT:    add a0, sp, a0
+; CHECK-NEXT:    addi a0, a0, 16
+; CHECK-NEXT:    csrr a1, vlenb
+; CHECK-NEXT:    vmerge.vim v14, v12, 1, v0
+; CHECK-NEXT:    vmv1r.v v0, v8
+; CHECK-NEXT:    vmerge.vim v18, v12, 1, v0
+; CHECK-NEXT:    add a2, a4, a1
+; CHECK-NEXT:    srli a3, a1, 2
+; CHECK-NEXT:    vmv2r.v v20, v14
+; CHECK-NEXT:    vmv1r.v v0, v9
+; CHECK-NEXT:    vmerge.vim v16, v12, 1, v0
+; CHECK-NEXT:    vmv1r.v v21, v18
+; CHECK-NEXT:    vmv1r.v v0, v10
+; CHECK-NEXT:    vmerge.vim v8, v12, 1, v0
+; CHECK-NEXT:    vmv1r.v v22, v16
+; CHECK-NEXT:    vmv1r.v v16, v19
+; CHECK-NEXT:    add a5, a2, a1
+; CHECK-NEXT:    vmv1r.v v23, v8
+; CHECK-NEXT:    vmv1r.v v18, v9
+; CHECK-NEXT:    vmv1r.v v0, v11
+; CHECK-NEXT:    vmerge.vim v24, v12, 1, v0
+; CHECK-NEXT:    vsetvli a6, zero, e8, m1, ta, ma
+; CHECK-NEXT:    vsseg5e8.v v20, (a4)
+; CHECK-NEXT:    vmv1r.v v19, v25
+; CHECK-NEXT:    vsseg5e8.v v15, (a0)
+; CHECK-NEXT:    vl1r.v v8, (a5)
+; CHECK-NEXT:    add a5, a5, a1
+; CHECK-NEXT:    vl1r.v v10, (a4)
+; CHECK-NEXT:    add a4, a5, a1
+; CHECK-NEXT:    vl1r.v v12, (a4)
+; CHECK-NEXT:    add a4, a0, a1
+; CHECK-NEXT:    vl1r.v v14, (a4)
+; CHECK-NEXT:    add a4, a4, a1
+; CHECK-NEXT:    vl1r.v v9, (a5)
+; CHECK-NEXT:    add a5, a4, a1
+; CHECK-NEXT:    vl1r.v v16, (a5)
+; CHECK-NEXT:    add a5, a5, a1
+; CHECK-NEXT:    srli a1, a1, 1
+; CHECK-NEXT:    vl1r.v v11, (a2)
+; CHECK-NEXT:    add a2, a3, a3
+; CHECK-NEXT:    vl1r.v v15, (a4)
+; CHECK-NEXT:    add a4, a1, a1
+; CHECK-NEXT:    vl1r.v v13, (a0)
+; CHECK-NEXT:    vl1r.v v17, (a5)
+; CHECK-NEXT:    vsetvli a0, zero, e8, m2, ta, ma
+; CHECK-NEXT:    vmsne.vi v18, v8, 0
+; CHECK-NEXT:    vmsne.vi v0, v10, 0
+; CHECK-NEXT:    vmsne.vi v8, v14, 0
+; CHECK-NEXT:    vmsne.vi v9, v12, 0
+; CHECK-NEXT:    vsetvli zero, a2, e8, mf2, ta, ma
+; CHECK-NEXT:    vslideup.vx v0, v18, a3
+; CHECK-NEXT:    vslideup.vx v9, v8, a3
+; CHECK-NEXT:    vsetvli zero, a4, e8, m1, ta, ma
+; CHECK-NEXT:    vslideup.vx v0, v9, a1
+; CHECK-NEXT:    vsetvli a0, zero, e8, m2, ta, ma
+; CHECK-NEXT:    vmsne.vi v8, v16, 0
+; CHECK-NEXT:    csrr a0, vlenb
+; CHECK-NEXT:    li a1, 10
+; CHECK-NEXT:    mul a0, a0, a1
+; CHECK-NEXT:    add sp, sp, a0
+; CHECK-NEXT:    addi sp, sp, 16
+; CHECK-NEXT:    ret
+;
+; ZVBB-LABEL: vector_interleave_nxv80i1_nxv16i1:
+; ZVBB:       # %bb.0:
+; ZVBB-NEXT:    addi sp, sp, -16
+; ZVBB-NEXT:    csrr a0, vlenb
+; ZVBB-NEXT:    li a1, 10
+; ZVBB-NEXT:    mul a0, a0, a1
+; ZVBB-NEXT:    sub sp, sp, a0
+; ZVBB-NEXT:    vsetvli a0, zero, e8, m2, ta, ma
+; ZVBB-NEXT:    vmv.v.i v12, 0
+; ZVBB-NEXT:    addi a4, sp, 16
+; ZVBB-NEXT:    csrr a0, vlenb
+; ZVBB-NEXT:    slli a1, a0, 2
+; ZVBB-NEXT:    add a0, a1, a0
+; ZVBB-NEXT:    add a0, sp, a0
+; ZVBB-NEXT:    addi a0, a0, 16
+; ZVBB-NEXT:    csrr a1, vlenb
+; ZVBB-NEXT:    vmerge.vim v14, v12, 1, v0
+; ZVBB-NEXT:    vmv1r.v v0, v8
+; ZVBB-NEXT:    vmerge.vim v18, v12, 1, v0
+; ZVBB-NEXT:    add a2, a4, a1
+; ZVBB-NEXT:    srli a3, a1, 2
+; ZVBB-NEXT:    vmv2r.v v20, v14
+; ZVBB-NEXT:    vmv1r.v v0, v9
+; ZVBB-NEXT:    vmerge.vim v16, v12, 1, v0
+; ZVBB-NEXT:    vmv1r.v v21, v18
+; ZVBB-NEXT:    vmv1r.v v0, v10
+; ZVBB-NEXT:    vmerge.vim v8, v12, 1, v0
+; ZVBB-NEXT:    vmv1r.v v22, v16
+; ZVBB-NEXT:    vmv1r.v v16, v19
+; ZVBB-NEXT:    add a5, a2, a1
+; ZVBB-NEXT:    vmv1r.v v23, v8
+; ZVBB-NEXT:    vmv1r.v v18, v9
+; ZVBB-NEXT:    vmv1r.v v0, v11
+; ZVBB-NEXT:    vmerge.vim v24, v12, 1, v0
+; ZVBB-NEXT:    vsetvli a6, zero, e8, m1, ta, ma
+; ZVBB-NEXT:    vsseg5e8.v v20, (a4)
+; ZVBB-NEXT:    vmv1r.v v19, v25
+; ZVBB-NEXT:    vsseg5e8.v v15, (a0)
+; ZVBB-NEXT:    vl1r.v v8, (a5)
+; ZVBB-NEXT:    add a5, a5, a1
+; ZVBB-NEXT:    vl1r.v v10, (a4)
+; ZVBB-NEXT:    add a4, a5, a1
+; ZVBB-NEXT:    vl1r.v v12, (a4)
+; ZVBB-NEXT:    add a4, a0, a1
+; ZVBB-NEXT:    vl1r.v v14, (a4)
+; ZVBB-NEXT:    add a4, a4, a1
+; ZVBB-NEXT:    vl1r.v v9, (a5)
+; ZVBB-NEXT:    add a5, a4, a1
+; ZVBB-NEXT:    vl1r.v v16, (a5)
+; ZVBB-NEXT:    add a5, a5, a1
+; ZVBB-NEXT:    srli a1, a1, 1
+; ZVBB-NEXT:    vl1r.v v11, (a2)
+; ZVBB-NEXT:    add a2, a3, a3
+; ZVBB-NEXT:    vl1r.v v15, (a4)
+; ZVBB-NEXT:    add a4, a1, a1
+; ZVBB-NEXT:    vl1r.v v13, (a0)
+; ZVBB-NEXT:    vl1r.v v17, (a5)
+; ZVBB-NEXT:    vsetvli a0, zero, e8, m2, ta, ma
+; ZVBB-NEXT:    vmsne.vi v18, v8, 0
+; ZVBB-NEXT:    vmsne.vi v0, v10, 0
+; ZVBB-NEXT:    vmsne.vi v8, v14, 0
+; ZVBB-NEXT:    vmsne.vi v9, v12, 0
+; ZVBB-NEXT:    vsetvli zero, a2, e8, mf2, ta, ma
+; ZVBB-NEXT:    vslideup.vx v0, v18, a3
+; ZVBB-NEXT:    vslideup.vx v9, v8, a3
+; ZVBB-NEXT:    vsetvli zero, a4, e8, m1, ta, ma
+; ZVBB-NEXT:    vslideup.vx v0, v9, a1
+; ZVBB-NEXT:    vsetvli a0, zero, e8, m2, ta, ma
+; ZVBB-NEXT:    vmsne.vi v8, v16, 0
+; ZVBB-NEXT:    csrr a0, vlenb
+; ZVBB-NEXT:    li a1, 10
+; ZVBB-NEXT:    mul a0, a0, a1
+; ZVBB-NEXT:    add sp, sp, a0
+; ZVBB-NEXT:    addi sp, sp, 16
+; ZVBB-NEXT:    ret
+  %res = call <vscale x 80 x i1> @llvm.vector.interleave5.nxv80i1(<vscale x 16 x i1> %a, <vscale x 16 x i1> %b, <vscale x 16 x i1> %c, <vscale x 16 x i1> %d, <vscale x 16 x i1> %e)
+  ret <vscale x 80 x i1> %res
+}
+
+
+define <vscale x 80 x i8> @vector_interleave_nxv80i8_nxv16i8(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b, <vscale x 16 x i8> %c, <vscale x 16 x i8> %d, <vscale x 16 x i8> %e) nounwind {
+;
+; RV32-LABEL: vector_interleave_nxv80i8_nxv16i8:
+; RV32:       # %bb.0:
+; RV32-NEXT:    addi sp, sp, -80
+; RV32-NEXT:    sw ra, 76(sp) # 4-byte Folded Spill
+; RV32-NEXT:    sw s0, 72(sp) # 4-byte Folded Spill
+; RV32-NEXT:    addi s0, sp, 80
+; RV32-NEXT:    csrr a0, vlenb
+; RV32-NEXT:    li a1, 28
+; RV32-NEXT:    mul a0, a0, a1
+; RV32-NEXT:    sub sp, sp, a0
+; RV32-NEXT:    andi sp, sp, -64
+; RV32-NEXT:    vsetvli a0, zero, e8, m1, ta, ma
+; RV32-NEXT:    vmv2r.v v20, v16
+; RV32-NEXT:    addi a0, sp, 64
+; RV32-NEXT:    vmv2r.v v18, v12
+; RV32-NEXT:    csrr a1, vlenb
+; RV32-NEXT:    slli a2, a1, 2
+; RV32-NEXT:    add a1, a2, a1
+; RV32-NEXT:    add a1, sp, a1
+; RV32-NEXT:    addi a1, a1, 64
+; RV32-NEXT:    csrr a2, vlenb
+; RV32-NEXT:    vmv2r.v v16, v8
+; RV32-NEXT:    vmv2r.v v22, v16
+; RV32-NEXT:    vmv2r.v v24, v18
+; RV32-NEXT:    vmv1r.v v26, v20
+; RV32-NEXT:    add a3, a0, a2
+; RV32-NEXT:    vmv1r.v v23, v10
+; RV32-NEXT:    add a4, a1, a2
+; RV32-NEXT:    add a5, a4, a2
+; RV32-NEXT:    vmv1r.v v25, v14
+; RV32-NEXT:    add a6, a5, a2
+; RV32-NEXT:    vmv1r.v v18, v11
+; RV32-NEXT:    vsseg5e8.v v22, (a0)
+; RV32-NEXT:    vmv1r.v v20, v15
+; RV32-NEXT:    vsseg5e8.v v17, (a1)
+; RV32-NEXT:    vl1r.v v16, (a6)
+; RV32-NEXT:    add a6, a6, a2
+; RV32-NEXT:    vl1r.v v17, (a6)
+; RV32-NEXT:    add a6, a3, a2
+; RV32-NEXT:    vl1r.v v10, (a6)
+; RV32-NEXT:    add a6, a6, a2
+; RV32-NEXT:    vl1r.v v11, (a6)
+; RV32-NEXT:    vl1r.v v8, (a0)
+; RV32-NEXT:    vl1r.v v9, (a3)
+; RV32-NEXT:    vl1r.v v14, (a4)
+; RV32-NEXT:    csrr a0, vlenb
+; RV32-NEXT:    li a3, 10
+; RV32-NEXT:    mul a0, a0, a3
+; RV32-NEXT:    add a0, sp, a0
+; RV32-NEXT:    addi a0, a0, 64
+; RV32-NEXT:    add a6, a6, a2
+; RV32-NEXT:    vl1r.v v15, (a5)
+; RV32-NEXT:    vl1r.v v12, (a6)
+; RV32-NEXT:    vl1r.v v13, (a1)
+; RV32-NEXT:    slli a2, a2, 3
+; RV32-NEXT:    add a2, a0, a2
+; RV32-NEXT:    vs2r.v v16, (a2)
+; RV32-NEXT:    vs8r.v v8, (a0)
+; RV32-NEXT:    vl8r.v v16, (a2)
+; RV32-NEXT:    vl8r.v v8, (a0)
+; RV32-NEXT:    addi sp, s0, -80
+; RV32-NEXT:    lw ra, 76(sp) # 4-byte Folded Reload
+; RV32-NEXT:    lw s0, 72(sp) # 4-byte Folded Reload
+; RV32-NEXT:    addi sp, sp, 80
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: vector_interleave_nxv80i8_nxv16i8:
+; RV64:       # %bb.0:
+; RV64-NEXT:    addi sp, sp, -80
+; RV64-NEXT:    sd ra, 72(sp) # 8-byte Folded Spill
+; RV64-NEXT:    sd s0, 64(sp) # 8-byte Folded Spill
+; RV64-NEXT:    addi s0, sp, 80
+; RV64-NEXT:    csrr a0, vlenb
+; RV64-NEXT:    li a1, 28
+; RV64-NEXT:    mul a0, a0, a1
+; RV64-NEXT:    sub sp, sp, a0
+; RV64-NEXT:    andi sp, sp, -64
+; RV64-NEXT:    vsetvli a0, zero, e8, m1, ta, ma
+; RV64-NEXT:    vmv2r.v v20, v16
+; RV64-NEXT:    addi a0, sp, 64
+; RV64-NEXT:    vmv2r.v v18, v12
+; RV64-NEXT:    csrr a1, vlenb
+; RV64-NEXT:    slli a2, a1, 2
+; RV64-NEXT:    add a1, a2, a1
+; RV64-NEXT:    add a1, sp, a1
+; RV64-NEXT:    addi a1, a1, 64
+; RV64-NEXT:    csrr a2, vlenb
+; RV64-NEXT:    vmv2r.v v16, v8
+; RV64-NEXT:    vmv2r.v v22, v16
+; RV64-NEXT:    vmv2r.v v24, v18
+; RV64-NEXT:    vmv1r.v v26, v20
+; RV64-NEXT:    add a3, a0, a2
+; RV64-NEXT:    vmv1r.v v23, v10
+; RV64-NEXT:    add a4, a1, a2
+; RV64-NEXT:    add a5, a4, a2
+; RV64-NEXT:    vmv1r.v v25, v14
+; RV64-NEXT:    add a6, a5, a2
+; RV64-NEXT:    vmv1r.v v18, v11
+; RV64-NEXT:    vsseg5e8.v v22, (a0)
+; RV64-NEXT:    vmv1r.v v20, v15
+; RV64-NEXT:    vsseg5e8.v v17, (a1)
+; RV64-NEXT:    vl1r.v v16, (a6)
+; RV64-NEXT:    add a6, a6, a2
+; RV64-NEXT:    vl1r.v v17, (a6)
+; RV64-NEXT:    add a6, a3, a2
+; RV64-NEXT:    vl1r.v v10, (a6)
+; RV64-NEXT:    add a6, a6, a2
+; RV64-NEXT:    vl1r.v v11, (a6)
+; RV64-NEXT:    vl1r.v v8, (a0)
+; RV64-NEXT:    vl1r.v v9, (a3)
+; RV64-NEXT:    vl1r.v v14, (a4)
+; RV64-NEXT:    csrr a0, vlenb
+; RV64-NEXT:    li a3, 10
+; RV64-NEXT:    mul a0, a0, a3
+; RV64-NEXT:    add a0, sp, a0
+; RV64-NEXT:    addi a0, a0, 64
+; RV64-NEXT:    add a6, a6, a2
+; RV64-NEXT:    vl1r.v v15, (a5)
+; RV64-NEXT:    vl1r.v v12, (a6)
+; RV64-NEXT:    vl1r.v v13, (a1)
+; RV64-NEXT:    slli a2, a2, 3
+; RV64-NEXT:    add a2, a0, a2
+; RV64-NEXT:    vs2r.v v16, (a2)
+; RV64-NEXT:    vs8r.v v8, (a0)
+; RV64-NEXT:    vl8r.v v16, (a2)
+; RV64-NEXT:    vl8r.v v8, (a0)
+; RV64-NEXT:    addi sp, s0, -80
+; RV64-NEXT:    ld ra, 72(sp) # 8-byte Folded Reload
+; RV64-NEXT:    ld s0, 64(sp) # 8-byte Folded Reload
+; RV64-NEXT:    addi sp, sp, 80
+; RV64-NEXT:    ret
+;
+; ZVBB-RV32-LABEL: vector_interleave_nxv80i8_nxv16i8:
+; ZVBB-RV32:       # %bb.0:
+; ZVBB-RV32-NEXT:    addi sp, sp, -80
+; ZVBB-RV32-NEXT:    sw ra, 76(sp) # 4-byte Folded Spill
+; ZVBB-RV32-NEXT:    sw s0, 72(sp) # 4-byte Folded Spill
+; ZVBB-RV32-NEXT:    addi s0, sp, 80
+; ZVBB-RV32-NEXT:    csrr a0, vlenb
+; ZVBB-RV32-NEXT:    li a1, 28
+; ZVBB-RV32-NEXT:    mul a0, a0, a1
+; ZVBB-RV32-NEXT:    sub sp, sp, a0
+; ZVBB-RV32-NEXT:    andi sp, sp, -64
+; ZVBB-RV32-NEXT:    vsetvli a0, zero, e8, m1, ta, ma
+; ZVBB-RV32-NEXT:    vmv2r.v v20, v16
+; ZVBB-RV32-NEXT:    addi a0, sp, 64
+; ZVBB-RV32-NEXT:    vmv2r.v v18, v12
+; ZVBB-RV32-NEXT:    csrr a1, vlenb
+; ZVBB-RV32-NEXT:    slli a2, a1, 2
+; ZVBB-RV32-NEXT:    add a1, a2, a1
+; ZVBB-RV32-NEXT:    add a1, sp, a1
+; ZVBB-RV32-NEXT:    addi a1, a1, 64
+; ZVBB-RV32-NEXT:    csrr a2, vlenb
+; ZVBB-RV32-NEXT:    vmv2r.v v16, v8
+; ZVBB-RV32-NEXT:    vmv2r.v v22, v16
+; ZVBB-RV32-NEXT:    vmv2r.v v24, v18
+; ZVBB-RV32-NEXT:    vmv1r.v v26, v20
+; ZVBB-RV32-NEXT:    add a3, a0, a2
+; ZVBB-RV32-NEXT:    vmv1r.v v23, v10
+; ZVBB-RV32-NEXT:    add a4, a1, a2
+; ZVBB-RV32-NEXT:    add a5, a4, a2
+; ZVBB-RV32-NEXT:    vmv1r.v v25, v14
+; ZVBB-RV32-NEXT:    add a6, a5, a2
+; ZVBB-RV32-NEXT:    vmv1r.v v18, v11
+; ZVBB-RV32-NEXT:    vsseg5e8.v v22, (a0)
+; ZVBB-RV32-NEXT:    vmv1r.v v20, v15
+; ZVBB-RV32-NEXT:    vsseg5e8.v v17, (a1)
+; ZVBB-RV32-NEXT:    vl1r.v v16, (a6)
+; ZVBB-RV32-NEXT:    add a6, a6, a2
+; ZVBB-RV32-NEXT:    vl1r.v v17, (a6)
+; ZVBB-RV32-NEXT:    add a6, a3, a2
+; ZVBB-RV32-NEXT:    vl1r.v v10, (a6)
+; ZVBB-RV32-NEXT:    add a6, a6, a2
+; ZVBB-RV32-NEXT:    vl1r.v v11, (a6)
+; ZVBB-RV32-NEXT:    vl1r.v v8, (a0)
+; ZVBB-RV32-NEXT:    vl1r.v v9, (a3)
+; ZVBB-RV32-NEXT:    vl1r.v v14, (a4)
+; ZVBB-RV32-NEXT:    csrr a0, vlenb
+; ZVBB-RV32-NEXT:    li a3, 10
+; ZVBB-RV32-NEXT:    mul a0, a0, a3
+; ZVBB-RV32-NEXT:    add a0, sp, a0
+; ZVBB-RV32-NEXT:    addi a0, a0, 64
+; ZVBB-RV32-NEXT:    add a6, a6, a2
+; ZVBB-RV32-NEXT:    vl1r.v v15, (a5)
+; ZVBB-RV32-NEXT:    vl1r.v v12, (a6)
+; ZVBB-RV32-NEXT:    vl1r.v v13, (a1)
+; ZVBB-RV32-NEXT:    slli a2, a2, 3
+; ZVBB-RV32-NEXT:    add a2, a0, a2
+; ZVBB-RV32-NEXT:    vs2r.v v16, (a2)
+; ZVBB-RV32-NEXT:    vs8r.v v8, (a0)
+; ZVBB-RV32-NEXT:    vl8r.v v16, (a2)
+; ZVBB-RV32-NEXT:    vl8r.v v8, (a0)
+; ZVBB-RV32-NEXT:    addi sp, s0, -80
+; ZVBB-RV32-NEXT:    lw ra, 76(sp) # 4-byte Folded Reload
+; ZVBB-RV32-NEXT:    lw s0, 72(sp) # 4-byte Folded Reload
+; ZVBB-RV32-NEXT:    addi sp, sp, 80
+; ZVBB-RV32-NEXT:    ret
+;
+; ZVBB-RV64-LABEL: vector_interleave_nxv80i8_nxv16i8:
+; ZVBB-RV64:       # %bb.0:
+; ZVBB-RV64-NEXT:    addi sp, sp, -80
+; ZVBB-RV64-NEXT:    sd ra, 72(sp) # 8-byte Folded Spill
+; ZVBB-RV64-NEXT:    sd s0, 64(sp) # 8-byte Folded Spill
+; ZVBB-RV64-NEXT:    addi s0, sp, 80
+; ZVBB-RV64-NEXT:    csrr a0, vlenb
+; ZVBB-RV64-NEXT:    li a1, 28
+; ZVBB-RV64-NEXT:    mul a0, a0, a1
+; ZVBB-RV64-NEXT:    sub sp, sp, a0
+; ZVBB-RV64-NEXT:    andi sp, sp, -64
+; ZVBB-RV64-NEXT:    vsetvli a0, zero, e8, m1, ta, ma
+; ZVBB-RV64-NEXT:    vmv2r.v v20, v16
+; ZVBB-RV64-NEXT:    addi a0, sp, 64
+; ZVBB-RV64-NEXT:    vmv2r.v v18, v12
+; ZVBB-RV64-NEXT:    csrr a1, vlenb
+; ZVBB-RV64-NEXT:    slli a2, a1, 2
+; ZVBB-RV64-NEXT:    add a1, a2, a1
+; ZVBB-RV64-NEXT:    add a1, sp, a1
+; ZVBB-RV64-NEXT:    addi a1, a1, 64
+; ZVBB-RV64-NEXT:    csrr a2, vlenb
+; ZVBB-RV64-NEXT:    vmv2r.v v16, v8
+; ZVBB-RV64-NEXT:    vmv2r.v v22, v16
+; ZVBB-RV64-NEXT:    vmv2r.v v24, v18
+; ZVBB-RV64-NEXT:    vmv1r.v v26, v20
+; ZVBB-RV64-NEXT:    add a3, a0, a2
+; ZVBB-RV64-NEXT:    vmv1r.v v23, v10
+; ZVBB-RV64-NEXT:    add a4, a1, a2
+; ZVBB-RV64-NEXT:    add a5, a4, a2
+; ZVBB-RV64-NEXT:    vmv1r.v v25, v14
+; ZVBB-RV64-NEXT:    add a6, a5, a2
+; ZVBB-RV64-NEXT:    vmv1r.v v18, v11
+; ZVBB-RV64-NEXT:    vsseg5e8.v v22, (a0)
+; ZVBB-RV64-NEXT:    vmv1r.v v20, v15
+; ZVBB-RV64-NEXT:    vsseg5e8.v v17, (a1)
+; ZVBB-RV64-NEXT:    vl1r.v v16, (a6)
+; ZVBB-RV64-NEXT:    add a6, a6, a2
+; ZVBB-RV64-NEXT:    vl1r.v v17, (a6)
+; ZVBB-RV64-NEXT:    add a6, a3, a2
+; ZVBB-RV64-NEXT:    vl1r.v v10, (a6)
+; ZVBB-RV64-NEXT:    add a6, a6, a2
+; ZVBB-RV64-NEXT:    vl1r.v v11, (a6)
+; ZVBB-RV64-NEXT:    vl1r.v v8, (a0)
+; ZVBB-RV64-NEXT:    vl1r.v v9, (a3)
+; ZVBB-RV64-NEXT:    vl1r.v v14, (a4)
+; ZVBB-RV64-NEXT:    csrr a0, vlenb
+; ZVBB-RV64-NEXT:    li a3, 10
+; ZVBB-RV64-NEXT:    mul a0, a0, a3
+; ZVBB-RV64-NEXT:    add a0, sp, a0
+; ZVBB-RV64-NEXT:    addi a0, a0, 64
+; ZVBB-RV64-NEXT:    add a6, a6, a2
+; ZVBB-RV64-NEXT:    vl1r.v v15, (a5)
+; ZVBB-RV64-NEXT:    vl1r.v v12, (a6)
+; ZVBB-RV64-NEXT:    vl1r.v v13, (a1)
+; ZVBB-RV64-NEXT:    slli a2, a2, 3
+; ZVBB-RV64-NEXT:    add a2, a0, a2
+; ZVBB-RV64-NEXT:    vs2r.v v16, (a2)
+; ZVBB-RV64-NEXT:    vs8r.v v8, (a0)
+; ZVBB-RV64-NEXT:    vl8r.v v16, (a2)
+; ZVBB-RV64-NEXT:    vl8r.v v8, (a0)
+; ZVBB-RV64-NEXT:    addi sp, s0, -80
+; ZVBB-RV64-NEXT:    ld ra, 72(sp) # 8-byte Folded Reload
+; ZVBB-RV64-NEXT:    ld s0, 64(sp) # 8-byte Folded Reload
+; ZVBB-RV64-NEXT:    addi sp, sp, 80
+; ZVBB-RV64-NEXT:    ret
+  %res = call <vscale x 80 x i8> @llvm.vector.interleave5.nxv80i8(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b, <vscale x 16 x i8> %c, <vscale x 16 x i8> %d, <vscale x 16 x i8> %e)
+  ret <vscale x 80 x i8> %res
+}
+
+
+define <vscale x 40 x i8> @vector_interleave_nxv40i8_nxv8i8(<vscale x 8 x i8> %a, <vscale x 8 x i8> %b, <vscale x 8 x i8> %c, <vscale x 8 x i8> %d, <vscale x 8 x i8> %e) nounwind {
+; CHECK-LABEL: vector_interleave_nxv40i8_nxv8i8:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    addi sp, sp, -16
+; CHECK-NEXT:    csrr a0, vlenb
+; CHECK-NEXT:    slli a1, a0, 2
+; CHECK-NEXT:    add a0, a1, a0
+; CHECK-NEXT:    sub sp, sp, a0
+; CHECK-NEXT:    addi a0, sp, 16
+; CHECK-NEXT:    csrr a1, vlenb
+; CHECK-NEXT:    add a2, a0, a1
+; CHECK-NEXT:    add a3, a2, a1
+; CHECK-NEXT:    vsetvli a4, zero, e8, m1, ta, ma
+; CHECK-NEXT:    vsseg5e8.v v8, (a0)
+; CHECK-NEXT:    vl1r.v v10, (a3)
+; CHECK-NEXT:    add a3, a3, a1
+; CHECK-NEXT:    vl1r.v v11, (a3)
+; CHECK-NEXT:    vl1r.v v8, (a0)
+; CHECK-NEXT:    vl1r.v v9, (a2)
+; CHECK-NEXT:    add a1, a3, a1
+; CHECK-NEXT:    vl1r.v v12, (a1)
+; CHECK-NEXT:    csrr a0, vlenb
+; CHECK-NEXT:    slli a1, a0, 2
+; CHECK-NEXT:    add a0, a1, a0
+; CHECK-NEXT:    add sp, sp, a0
+; CHECK-NEXT:    addi sp, sp, 16
+; CHECK-NEXT:    ret
+;
+; ZVBB-LABEL: vector_interleave_nxv40i8_nxv8i8:
+; ZVBB:       # %bb.0:
+; ZVBB-NEXT:    addi sp, sp, -16
+; ZVBB-NEXT:    csrr a0, vlenb
+; ZVBB-NEXT:    slli a1, a0, 2
+; ZVBB-NEXT:    add a0, a1, a0
+; ZVBB-NEXT:    sub sp, sp, a0
+; ZVBB-NEXT:    addi a0, sp, 16
+; ZVBB-NEXT:    csrr a1, vlenb
+; ZVBB-NEXT:    add a2, a0, a1
+; ZVBB-NEXT:    add a3, a2, a1
+; ZVBB-NEXT:    vsetvli a4, zero, e8, m1, ta, ma
+; ZVBB-NEXT:    vsseg5e8.v v8, (a0)
+; ZVBB-NEXT:    vl1r.v v10, (a3)
+; ZVBB-NEXT:    add a3, a3, a1
+; ZVBB-NEXT:    vl1r.v v11, (a3)
+; ZVBB-NEXT:    vl1r.v v8, (a0)
+; ZVBB-NEXT:    vl1r.v v9, (a2)
+; ZVBB-NEXT:    add a1, a3, a1
+; ZVBB-NEXT:    vl1r.v v12, (a1)
+; ZVBB-NEXT:    csrr a0, vlenb
+; ZVBB-NEXT:    slli a1, a0, 2
+; ZVBB-NEXT:    add a0, a1, a0
+; ZVBB-NEXT:    add sp, sp, a0
+; ZVBB-NEXT:    addi sp, sp, 16
+; ZVBB-NEXT:    ret
+  %res = call <vscale x 40 x i8> @llvm.vector.interleave5.nxv40i8(<vscale x 8 x i8> %a, <vscale x 8 x i8> %b, <vscale x 8 x i8> %c, <vscale x 8 x i8> %d, <vscale x 8 x i8> %e)
+  ret <vscale x 40 x i8> %res
+}
+
+
+define <vscale x 20 x i32> @vector_interleave_nxv20i32_nxv4i32(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b, <vscale x 4 x i32> %c, <vscale x 4 x i32> %d, <vscale x 4 x i32> %e) nounwind {
+;
+; RV32-LABEL: vector_interleave_nxv20i32_nxv4i32:
+; RV32:       # %bb.0:
+; RV32-NEXT:    addi sp, sp, -80
+; RV32-NEXT:    sw ra, 76(sp) # 4-byte Folded Spill
+; RV32-NEXT:    sw s0, 72(sp) # 4-byte Folded Spill
+; RV32-NEXT:    addi s0, sp, 80
+; RV32-NEXT:    csrr a0, vlenb
+; RV32-NEXT:    li a1, 28
+; RV32-NEXT:    mul a0, a0, a1
+; RV32-NEXT:    sub sp, sp, a0
+; RV32-NEXT:    andi sp, sp, -64
+; RV32-NEXT:    vsetvli a0, zero, e32, m1, ta, ma
+; RV32-NEXT:    vmv2r.v v20, v16
+; RV32-NEXT:    addi a0, sp, 64
+; RV32-NEXT:    vmv2r.v v18, v12
+; RV32-NEXT:    csrr a1, vlenb
+; RV32-NEXT:    slli a2, a1, 2
+; RV32-NEXT:    add a1, a2, a1
+; RV32-NEXT:    add a1, sp, a1
+; RV32-NEXT:    addi a1, a1, 64
+; RV32-NEXT:    csrr a2, vlenb
+; RV32-NEXT:    vmv2r.v v16, v8
+; RV32-NEXT:    vmv2r.v v22, v16
+; RV32-NEXT:    vmv2r.v v24, v18
+; RV32-NEXT:    vmv1r.v v26, v20
+; RV32-NEXT:    add a3, a0, a2
+; RV32-NEXT:    vmv1r.v v23, v10
+; RV32-NEXT:    add a4, a1, a2
+; RV32-NEXT:    add a5, a4, a2
+; RV32-NEXT:    vmv1r.v v25, v14
+; RV32-NEXT:    add a6, a5, a2
+; RV32-NEXT:    vmv1r.v v18, v11
+; RV32-NEXT:    vsseg5e32.v v22, (a0)
+; RV32-NEXT:    vmv1r.v v20, v15
+; RV32-NEXT:    vsseg5e32.v v17, (a1)
+; RV32-NEXT:    vl1re32.v v16, (a6)
+; RV32-NEXT:    add a6, a6, a2
+; RV32-NEXT:    vl1re32.v v17, (a6)
+; RV32-NEXT:    add a6, a3, a2
+; RV32-NEXT:    vl1re32.v v10, (a6)
+; RV32-NEXT:    add a6, a6, a2
+; RV32-NEXT:    vl1re32.v v11, (a6)
+; RV32-NEXT:    vl1re32.v v8, (a0)
+; RV32-NEXT:    vl1re32.v v9, (a3)
+; RV32-NEXT:    vl1re32.v v14, (a4)
+; RV32-NEXT:    csrr a0, vlenb
+; RV32-NEXT:    li a3, 10
+; RV32-NEXT:    mul a0, a0, a3
+; RV32-NEXT:    add a0, sp, a0
+; RV32-NEXT:    addi a0, a0, 64
+; RV32-NEXT:    add a6, a6, a2
+; RV32-NEXT:    vl1re32.v v15, (a5)
+; RV32-NEXT:    vl1re32.v v12, (a6)
+; RV32-NEXT:    vl1re32.v v13, (a1)
+; RV32-NEXT:    slli a2, a2, 3
+; RV32-NEXT:    add a2, a0, a2
+; RV32-NEXT:    vs2r.v v16, (a2)
+; RV32-NEXT:    vs8r.v v8, (a0)
+; RV32-NEXT:    vl8re32.v v16, (a2)
+; RV32-NEXT:    vl8re32.v v8, (a0)
+; RV32-NEXT:    addi sp, s0, -80
+; RV32-NEXT:    lw ra, 76(sp) # 4-byte Folded Reload
+; RV32-NEXT:    lw s0, 72(sp) # 4-byte Folded Reload
+; RV32-NEXT:    addi sp, sp, 80
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: vector_interleave_nxv20i32_nxv4i32:
+; RV64:       # %bb.0:
+; RV64-NEXT:    addi sp, sp, -80
+; RV64-NEXT:    sd ra, 72(sp) # 8-byte Folded Spill
+; RV64-NEXT:    sd s0, 64(sp) # 8-byte Folded Spill
+; RV64-NEXT:    addi s0, sp, 80
+; RV64-NEXT:    csrr a0, vlenb
+; RV64-NEXT:    li a1, 28
+; RV64-NEXT:    mul a0, a0, a1
+; RV64-NEXT:    sub sp, sp, a0
+; RV64-NEXT:    andi sp, sp, -64
+; RV64-NEXT:    vsetvli a0, zero, e32, m1, ta, ma
+; RV64-NEXT:    vmv2r.v v20, v16
+; RV64-NEXT:    addi a0, sp, 64
+; RV64-NEXT:    vmv2r.v v18, v12
+; RV64-NEXT:    csrr a1, vlenb
+; RV64-NEXT:    slli a2, a1, 2
+; RV64-NEXT:    add a1, a2, a1
+; RV64-NEXT:    add a1, sp, a1
+; RV64-NEXT:    addi a1, a1, 64
+; RV64-NEXT:    csrr a2, vlenb
+; RV64-NEXT:    vmv2r.v v16, v8
+; RV64-NEXT:    vmv2r.v v22, v16
+; RV64-NEXT:    vmv2r.v v24, v18
+; RV64-NEXT:    vmv1r.v v26, v20
+; RV64-NEXT:    add a3, a0, a2
+; RV64-NEXT:    vmv1r.v v23, v10
+; RV64-NEXT:    add a4, a1, a2
+; RV64-NEXT:    add a5, a4, a2
+; RV64-NEXT:    vmv1r.v v25, v14
+; RV64-NEXT:    add a6, a5, a2
+; RV64-NEXT:    vmv1r.v v18, v11
+; RV64-NEXT:    vsseg5e32.v v22, (a0)
+; RV64-NEXT:    vmv1r.v v20, v15
+; RV64-NEXT:    vsseg5e32.v v17, (a1)
+; RV64-NEXT:    vl1re32.v v16, (a6)
+; RV64-NEXT:    add a6, a6, a2
+; RV64-NEXT:    vl1re32.v v17, (a6)
+; RV64-NEXT:    add a6, a3, a2
+; RV64-NEXT:    vl1re32.v v10, (a6)
+; RV64-NEXT:    add a6, a6, a2
+; RV64-NEXT:    vl1re32.v v11, (a6)
+; RV64-NEXT:    vl1re32.v v8, (a0)
+; RV64-NEXT:    vl1re32.v v9, (a3)
+; RV64-NEXT:    vl1re32.v v14, (a4)
+; RV64-NEXT:    csrr a0, vlenb
+; RV64-NEXT:    li a3, 10
+; RV64-NEXT:    mul a0, a0, a3
+; RV64-NEXT:    add a0, sp, a0
+; RV64-NEXT:    addi a0, a0, 64
+; RV64-NEXT:    add a6, a6, a2
+; RV64-NEXT:    vl1re32.v v15, (a5)
+; RV64-NEXT:    vl1re32.v v12, (a6)
+; RV64-NEXT:    vl1re32.v v13, (a1)
+; RV64-NEXT:    slli a2, a2, 3
+; RV64-NEXT:    add a2, a0, a2
+; RV64-NEXT:    vs2r.v v16, (a2)
+; RV64-NEXT:    vs8r.v v8, (a0)
+; RV64-NEXT:    vl8re32.v v16, (a2)
+; RV64-NEXT:    vl8re32.v v8, (a0)
+; RV64-NEXT:    addi sp, s0, -80
+; RV64-NEXT:    ld ra, 72(sp) # 8-byte Folded Reload
+; RV64-NEXT:    ld s0, 64(sp) # 8-byte Folded Reload
+; RV64-NEXT:    addi sp, sp, 80
+; RV64-NEXT:    ret
+;
+; ZVBB-RV32-LABEL: vector_interleave_nxv20i32_nxv4i32:
+; ZVBB-RV32:       # %bb.0:
+; ZVBB-RV32-NEXT:    addi sp, sp, -80
+; ZVBB-RV32-NEXT:    sw ra, 76(sp) # 4-byte Folded Spill
+; ZVBB-RV32-NEXT:    sw s0, 72(sp) # 4-byte Folded Spill
+; ZVBB-RV32-NEXT:    addi s0, sp, 80
+; ZVBB-RV32-NEXT:    csrr a0, vlenb
+; ZVBB-RV32-NEXT:    li a1, 28
+; ZVBB-RV32-NEXT:    mul a0, a0, a1
+; ZVBB-RV32-NEXT:    sub sp, sp, a0
+; ZVBB-RV32-NEXT:    andi sp, sp, -64
+; ZVBB-RV32-NEXT:    vsetvli a0, zero, e32, m1, ta, ma
+; ZVBB-RV32-NEXT:    vmv2r.v v20, v16
+; ZVBB-RV32-NEXT:    addi a0, sp, 64
+; ZVBB-RV32-NEXT:    vmv2r.v v18, v12
+; ZVBB-RV32-NEXT:    csrr a1, vlenb
+; ZVBB-RV32-NEXT:    slli a2, a1, 2
+; ZVBB-RV32-NEXT:    add a1, a2, a1
+; ZVBB-RV32-NEXT:    add a1, sp, a1
+; ZVBB-RV32-NEXT:    addi a1, a1, 64
+; ZVBB-RV32-NEXT:    csrr a2, vlenb
+; ZVBB-RV32-NEXT:    vmv2r.v v16, v8
+; ZVBB-RV32-NEXT:    vmv2r.v v22, v16
+; ZVBB-RV32-NEXT:    vmv2r.v v24, v18
+; ZVBB-RV32-NEXT:    vmv1r.v v26, v20
+; ZVBB-RV32-NEXT:    add a3, a0, a2
+; ZVBB-RV32-NEXT:    vmv1r.v v23, v10
+; ZVBB-RV32-NEXT:    add a4, a1, a2
+; ZVBB-RV32-NEXT:    add a5, a4, a2
+; ZVBB-RV32-NEXT:    vmv1r.v v25, v14
+; ZVBB-RV32-NEXT:    add a6, a5, a2
+; ZVBB-RV32-NEXT:    vmv1r.v v18, v11
+; ZVBB-RV32-NEXT:    vsseg5e32.v v22, (a0)
+; ZVBB-RV32-NEXT:    vmv1r.v v20, v15
+; ZVBB-RV32-NEXT:    vsseg5e32.v v17, (a1)
+; ZVBB-RV32-NEXT:    vl1re32.v v16, (a6)
+; ZVBB-RV32-NEXT:    add a6, a6, a2
+; ZVBB-RV32-NEXT:    vl1re32.v v17, (a6)
+; ZVBB-RV32-NEXT:    add a6, a3, a2
+; ZVBB-RV32-NEXT:    vl1re32.v v10, (a6)
+; ZVBB-RV32-NEXT:    add a6, a6, a2
+; ZVBB-RV32-NEXT:    vl1re32.v v11, (a6)
+; ZVBB-RV32-NEXT:    vl1re32.v v8, (a0)
+; ZVBB-RV32-NEXT:    vl1re32.v v9, (a3)
+; ZVBB-RV32-NEXT:    vl1re32.v v14, (a4)
+; ZVBB-RV32-NEXT:    csrr a0, vlenb
+; ZVBB-RV32-NEXT:    li a3, 10
+; ZVBB-RV32-NEXT:    mul a0, a0, a3
+; ZVBB-RV32-NEXT:    add a0, sp, a0
+; ZVBB-RV32-NEXT:    addi a0, a0, 64
+; ZVBB-RV32-NEXT:    add a6, a6, a2
+; ZVBB-RV32-NEXT:    vl1re32.v v15, (a5)
+; ZVBB-RV32-NEXT:    vl1re32.v v12, (a6)
+; ZVBB-RV32-NEXT:    vl1re32.v v13, (a1)
+; ZVBB-RV32-NEXT:    slli a2, a2, 3
+; ZVBB-RV32-NEXT:    add a2, a0, a2
+; ZVBB-RV32-NEXT:    vs2r.v v16, (a2)
+; ZVBB-RV32-NEXT:    vs8r.v v8, (a0)
+; ZVBB-RV32-NEXT:    vl8re32.v v16, (a2)
+; ZVBB-RV32-NEXT:    vl8re32.v v8, (a0)
+; ZVBB-RV32-NEXT:    addi sp, s0, -80
+; ZVBB-RV32-NEXT:    lw ra, 76(sp) # 4-byte Folded Reload
+; ZVBB-RV32-NEXT:    lw s0, 72(sp) # 4-byte Folded Reload
+; ZVBB-RV32-NEXT:    addi sp, sp, 80
+; ZVBB-RV32-NEXT:    ret
+;
+; ZVBB-RV64-LABEL: vector_interleave_nxv20i32_nxv4i32:
+; ZVBB-RV64:       # %bb.0:
+; ZVBB-RV64-NEXT:    addi sp, sp, -80
+; ZVBB-RV64-NEXT:    sd ra, 72(sp) # 8-byte Folded Spill
+; ZVBB-RV64-NEXT:    sd s0, 64(sp) # 8-byte Folded Spill
+; ZVBB-RV64-NEXT:    addi s0, sp, 80
+; ZVBB-RV64-NEXT:    csrr a0, vlenb
+; ZVBB-RV64-NEXT:    li a1, 28
+; ZVBB-RV64-NEXT:    mul a0, a0, a1
+; ZVBB-RV64-NEXT:    sub sp, sp, a0
+; ZVBB-RV64-NEXT:    andi sp, sp, -64
+; ZVBB-RV64-NEXT:    vsetvli a0, zero, e32, m1, ta, ma
+; ZVBB-RV64-NEXT:    vmv2r.v v20, v16
+; ZVBB-RV64-NEXT:    addi a0, sp, 64
+; ZVBB-RV64-NEXT:    vmv2r.v v18, v12
+; ZVBB-RV64-NEXT:    csrr a1, vlenb
+; ZVBB-RV64-NEXT:    slli a2, a1, 2
+; ZVBB-RV64-NEXT:    add a1, a2, a1
+; ZVBB-RV64-NEXT:    add a1, sp, a1
+; ZVBB-RV64-NEXT:    addi a1, a1, 64
+; ZVBB-RV64-NEXT:    csrr a2, vlenb
+; ZVBB-RV64-NEXT:    vmv2r.v v16, v8
+; ZVBB-RV64-NEXT:    vmv2r.v v22, v16
+; ZVBB-RV64-NEXT:    vmv2r.v v24, v18
+; ZVBB-RV64-NEXT:    vmv1r.v v26, v20
+; ZVBB-RV64-NEXT:    add a3, a0, a2
+; ZVBB-RV64-NEXT:    vmv1r.v v23, v10
+; ZVBB-RV64-NEXT:    add a4, a1, a2
+; ZVBB-RV64-NEXT:    add a5, a4, a2
+; ZVBB-RV64-NEXT:    vmv1r.v v25, v14
+; ZVBB-RV64-NEXT:    add a6, a5, a2
+; ZVBB-RV64-NEXT:    vmv1r.v v18, v11
+; ZVBB-RV64-NEXT:    vsseg5e32.v v22, (a0)
+; ZVBB-RV64-NEXT:    vmv1r.v v20, v15
+; ZVBB-RV64-NEXT:    vsseg5e32.v v17, (a1)
+; ZVBB-RV64-NEXT:    vl1re32.v v16, (a6)
+; ZVBB-RV64-NEXT:    add a6, a6, a2
+; ZVBB-RV64-NEXT:    vl1re32.v v17, (a6)
+; ZVBB-RV64-NEXT:    add a6, a3, a2
+; ZVBB-RV64-NEXT:    vl1re32.v v10, (a6)
+; ZVBB-RV64-NEXT:    add a6, a6, a2
+; ZVBB-RV64-NEXT:    vl1re32.v v11, (a6)
+; ZVBB-RV64-NEXT:    vl1re32.v v8, (a0)
+; ZVBB-RV64-NEXT:    vl1re32.v v9, (a3)
+; ZVBB-RV64-NEXT:    vl1re32.v v14, (a4)
+; ZVBB-RV64-NEXT:    csrr a0, vlenb
+; ZVBB-RV64-NEXT:    li a3, 10
+; ZVBB-RV64-NEXT:    mul a0, a0, a3
+; ZVBB-RV64-NEXT:    add a0, sp, a0
+; ZVBB-RV64-NEXT:    addi a0, a0, 64
+; ZVBB-RV64-NEXT:    add a6, a6, a2
+; ZVBB-RV64-NEXT:    vl1re32.v v15, (a5)
+; ZVBB-RV64-NEXT:    vl1re32.v v12, (a6)
+; ZVBB-RV64-NEXT:    vl1re32.v v13, (a1)
+; ZVBB-RV64-NEXT:    slli a2, a2, 3
+; ZVBB-RV64-NEXT:    add a2, a0, a2
+; ZVBB-RV64-NEXT:    vs2r.v v16, (a2)
+; ZVBB-RV64-NEXT:    vs8r.v v8, (a0)
+; ZVBB-RV64-NEXT:    vl8re32.v v16, (a2)
+; ZVBB-RV64-NEXT:    vl8re32.v v8, (a0)
+; ZVBB-RV64-NEXT:    addi sp, s0, -80
+; ZVBB-RV64-NEXT:    ld ra, 72(sp) # 8-byte Folded Reload
+; ZVBB-RV64-NEXT:    ld s0, 64(sp) # 8-byte Folded Reload
+; ZVBB-RV64-NEXT:    addi sp, sp, 80
+; ZVBB-RV64-NEXT:    ret
+  %res = call <vscale x 20 x i32> @llvm.vector.interleave5.nxv20i32(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b, <vscale x 4 x i32> %c, <vscale x 4 x i32> %d, <vscale x 4 x i32> %e)
+  ret <vscale x 20 x i32> %res
+}
+
+
+define <vscale x 10 x i64> @vector_interleave_nxv10i64_nxv2i64(<vscale x 2 x i64> %a, <vscale x 2 x i64> %b, <vscale x 2 x i64> %c, <vscale x 2 x i64> %d, <vscale x 2 x i64> %e) nounwind {
+;
+; RV32-LABEL: vector_interleave_nxv10i64_nxv2i64:
+; RV32:       # %bb.0:
+; RV32-NEXT:    addi sp, sp, -80
+; RV32-NEXT:    sw ra, 76(sp) # 4-byte Folded Spill
+; RV32-NEXT:    sw s0, 72(sp) # 4-byte Folded Spill
+; RV32-NEXT:    addi s0, sp, 80
+; RV32-NEXT:    csrr a0, vlenb
+; RV32-NEXT:    li a1, 28
+; RV32-NEXT:    mul a0, a0, a1
+; RV32-NEXT:    sub sp, sp, a0
+; RV32-NEXT:    andi sp, sp, -64
+; RV32-NEXT:    vsetvli a0, zero, e64, m1, ta, ma
+; RV32-NEXT:    vmv2r.v v20, v16
+; RV32-NEXT:    addi a0, sp, 64
+; RV32-NEXT:    vmv2r.v v18, v12
+; RV32-NEXT:    csrr a1, vlenb
+; RV32-NEXT:    slli a2, a1, 2
+; RV32-NEXT:    add a1, a2, a1
+; RV32-NEXT:    add a1, sp, a1
+; RV32-NEXT:    addi a1, a1, 64
+; RV32-NEXT:    csrr a2, vlenb
+; RV32-NEXT:    vmv2r.v v16, v8
+; RV32-NEXT:    vmv2r.v v22, v16
+; RV32-NEXT:    vmv2r.v v24, v18
+; RV32-NEXT:    vmv1r.v v26, v20
+; RV32-NEXT:    add a3, a0, a2
+; RV32-NEXT:    vmv1r.v v23, v10
+; RV32-NEXT:    add a4, a1, a2
+; RV32-NEXT:    add a5, a4, a2
+; RV32-NEXT:    vmv1r.v v25, v14
+; RV32-NEXT:    add a6, a5, a2
+; RV32-NEXT:    vmv1r.v v18, v11
+; RV32-NEXT:    vsseg5e64.v v22, (a0)
+; RV32-NEXT:    vmv1r.v v20, v15
+; RV32-NEXT:    vsseg5e64.v v17, (a1)
+; RV32-NEXT:    vl1re64.v v16, (a6)
+; RV32-NEXT:    add a6, a6, a2
+; RV32-NEXT:    vl1re64.v v17, (a6)
+; RV32-NEXT:    add a6, a3, a2
+; RV32-NEXT:    vl1re64.v v10, (a6)
+; RV32-NEXT:    add a6, a6, a2
+; RV32-NEXT:    vl1re64.v v11, (a6)
+; RV32-NEXT:    vl1re64.v v8, (a0)
+; RV32-NEXT:    vl1re64.v v9, (a3)
+; RV32-NEXT:    vl1re64.v v14, (a4)
+; RV32-NEXT:    csrr a0, vlenb
+; RV32-NEXT:    li a3, 10
+; RV32-NEXT:    mul a0, a0, a3
+; RV32-NEXT:    add a0, sp, a0
+; RV32-NEXT:    addi a0, a0, 64
+; RV32-NEXT:    add a6, a6, a2
+; RV32-NEXT:    vl1re64.v v15, (a5)
+; RV32-NEXT:    vl1re64.v v12, (a6)
+; RV32-NEXT:    vl1re64.v v13, (a1)
+; RV32-NEXT:    slli a2, a2, 3
+; RV32-NEXT:    add a2, a0, a2
+; RV32-NEXT:    vs2r.v v16, (a2)
+; RV32-NEXT:    vs8r.v v8, (a0)
+; RV32-NEXT:    vl8re64.v v16, (a2)
+; RV32-NEXT:    vl8re64.v v8, (a0)
+; RV32-NEXT:    addi sp, s0, -80
+; RV32-NEXT:    lw ra, 76(sp) # 4-byte Folded Reload
+; RV32-NEXT:    lw s0, 72(sp) # 4-byte Folded Reload
+; RV32-NEXT:    addi sp, sp, 80
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: vector_interleave_nxv10i64_nxv2i64:
+; RV64:       # %bb.0:
+; RV64-NEXT:    addi sp, sp, -80
+; RV64-NEXT:    sd ra, 72(sp) # 8-byte Folded Spill
+; RV64-NEXT:    sd s0, 64(sp) # 8-byte Folded Spill
+; RV64-NEXT:    addi s0, sp, 80
+; RV64-NEXT:    csrr a0, vlenb
+; RV64-NEXT:    li a1, 28
+; RV64-NEXT:    mul a0, a0, a1
+; RV64-NEXT:    sub sp, sp, a0
+; RV64-NEXT:    andi sp, sp, -64
+; RV64-NEXT:    vsetvli a0, zero, e64, m1, ta, ma
+; RV64-NEXT:    vmv2r.v v20, v16
+; RV64-NEXT:    addi a0, sp, 64
+; RV64-NEXT:    vmv2r.v v18, v12
+; RV64-NEXT:    csrr a1, vlenb
+; RV64-NEXT:    slli a2, a1, 2
+; RV64-NEXT:    add a1, a2, a1
+; RV64-NEXT:    add a1, sp, a1
+; RV64-NEXT:    addi a1, a1, 64
+; RV64-NEXT:    csrr a2, vlenb
+; RV64-NEXT:    vmv2r.v v16, v8
+; RV64-NEXT:    vmv2r.v v22, v16
+; RV64-NEXT:    vmv2r.v v24, v18
+; RV64-NEXT:    vmv1r.v v26, v20
+; RV64-NEXT:    add a3, a0, a2
+; RV64-NEXT:    vmv1r.v v23, v10
+; RV64-NEXT:    add a4, a1, a2
+; RV64-NEXT:    add a5, a4, a2
+; RV64-NEXT:    vmv1r.v v25, v14
+; RV64-NEXT:    add a6, a5, a2
+; RV64-NEXT:    vmv1r.v v18, v11
+; RV64-NEXT:    vsseg5e64.v v22, (a0)
+; RV64-NEXT:    vmv1r.v v20, v15
+; RV64-NEXT:    vsseg5e64.v v17, (a1)
+; RV64-NEXT:    vl1re64.v v16, (a6)
+; RV64-NEXT:    add a6, a6, a2
+; RV64-NEXT:    vl1re64.v v17, (a6)
+; RV64-NEXT:    add a6, a3, a2
+; RV64-NEXT:    vl1re64.v v10, (a6)
+; RV64-NEXT:    add a6, a6, a2
+; RV64-NEXT:    vl1re64.v v11, (a6)
+; RV64-NEXT:    vl1re64.v v8, (a0)
+; RV64-NEXT:    vl1re64.v v9, (a3)
+; RV64-NEXT:    vl1re64.v v14, (a4)
+; RV64-NEXT:    csrr a0, vlenb
+; RV64-NEXT:    li a3, 10
+; RV64-NEXT:    mul a0, a0, a3
+; RV64-NEXT:    add a0, sp, a0
+; RV64-NEXT:    addi a0, a0, 64
+; RV64-NEXT:    add a6, a6, a2
+; RV64-NEXT:    vl1re64.v v15, (a5)
+; RV64-NEXT:    vl1re64.v v12, (a6)
+; RV64-NEXT:    vl1re64.v v13, (a1)
+; RV64-NEXT:    slli a2, a2, 3
+; RV64-NEXT:    add a2, a0, a2
+; RV64-NEXT:    vs2r.v v16, (a2)
+; RV64-NEXT:    vs8r.v v8, (a0)
+; RV64-NEXT:    vl8re64.v v16, (a2)
+; RV64-NEXT:    vl8re64.v v8, (a0)
+; RV64-NEXT:    addi sp, s0, -80
+; RV64-NEXT:    ld ra, 72(sp) # 8-byte Folded Reload
+; RV64-NEXT:    ld s0, 64(sp) # 8-byte Folded Reload
+; RV64-NEXT:    addi sp, sp, 80
+; RV64-NEXT:    ret
+;
+; ZVBB-RV32-LABEL: vector_interleave_nxv10i64_nxv2i64:
+; ZVBB-RV32:       # %bb.0:
+; ZVBB-RV32-NEXT:    addi sp, sp, -80
+; ZVBB-RV32-NEXT:    sw ra, 76(sp) # 4-byte Folded Spill
+; ZVBB-RV32-NEXT:    sw s0, 72(sp) # 4-byte Folded Spill
+; ZVBB-RV32-NEXT:    addi s0, sp, 80
+; ZVBB-RV32-NEXT:    csrr a0, vlenb
+; ZVBB-RV32-NEXT:    li a1, 28
+; ZVBB-RV32-NEXT:    mul a0, a0, a1
+; ZVBB-RV32-NEXT:    sub sp, sp, a0
+; ZVBB-RV32-NEXT:    andi sp, sp, -64
+; ZVBB-RV32-NEXT:    vsetvli a0, zero, e64, m1, ta, ma
+; ZVBB-RV32-NEXT:    vmv2r.v v20, v16
+; ZVBB-RV32-NEXT:    addi a0, sp, 64
+; ZVBB-RV32-NEXT:    vmv2r.v v18, v12
+; ZVBB-RV32-NEXT:    csrr a1, vlenb
+; ZVBB-RV32-NEXT:    slli a2, a1, 2
+; ZVBB-RV32-NEXT:    add a1, a2, a1
+; ZVBB-RV32-NEXT:    add a1, sp, a1
+; ZVBB-RV32-NEXT:    addi a1, a1, 64
+; ZVBB-RV32-NEXT:    csrr a2, vlenb
+; ZVBB-RV32-NEXT:    vmv2r.v v16, v8
+; ZVBB-RV32-NEXT:    vmv2r.v v22, v16
+; ZVBB-RV32-NEXT:    vmv2r.v v24, v18
+; ZVBB-RV32-NEXT:    vmv1r.v v26, v20
+; ZVBB-RV32-NEXT:    add a3, a0, a2
+; ZVBB-RV32-NEXT:    vmv1r.v v23, v10
+; ZVBB-RV32-NEXT:    add a4, a1, a2
+; ZVBB-RV32-NEXT:    add a5, a4, a2
+; ZVBB-RV32-NEXT:    vmv1r.v v25, v14
+; ZVBB-RV32-NEXT:    add a6, a5, a2
+; ZVBB-RV32-NEXT:    vmv1r.v v18, v11
+; ZVBB-RV32-NEXT:    vsseg5e64.v v22, (a0)
+; ZVBB-RV32-NEXT:    vmv1r.v v20, v15
+; ZVBB-RV32-NEXT:    vsseg5e64.v v17, (a1)
+; ZVBB-RV32-NEXT:    vl1re64.v v16, (a6)
+; ZVBB-RV32-NEXT:    add a6, a6, a2
+; ZVBB-RV32-NEXT:    vl1re64.v v17, (a6)
+; ZVBB-RV32-NEXT:    add a6, a3, a2
+; ZVBB-RV32-NEXT:    vl1re64.v v10, (a6)
+; ZVBB-RV32-NEXT:    add a6, a6, a2
+; ZVBB-RV32-NEXT:    vl1re64.v v11, (a6)
+; ZVBB-RV32-NEXT:    vl1re64.v v8, (a0)
+; ZVBB-RV32-NEXT:    vl1re64.v v9, (a3)
+; ZVBB-RV32-NEXT:    vl1re64.v v14, (a4)
+; ZVBB-RV32-NEXT:    csrr a0, vlenb
+; ZVBB-RV32-NEXT:    li a3, 10
+; ZVBB-RV32-NEXT:    mul a0, a0, a3
+; ZVBB-RV32-NEXT:    add a0, sp, a0
+; ZVBB-RV32-NEXT:    addi a0, a0, 64
+; ZVBB-RV32-NEXT:    add a6, a6, a2
+; ZVBB-RV32-NEXT:    vl1re64.v v15, (a5)
+; ZVBB-RV32-NEXT:    vl1re64.v v12, (a6)
+; ZVBB-RV32-NEXT:    vl1re64.v v13, (a1)
+; ZVBB-RV32-NEXT:    slli a2, a2, 3
+; ZVBB-RV32-NEXT:    add a2, a0, a2
+; ZVBB-RV32-NEXT:    vs2r.v v16, (a2)
+; ZVBB-RV32-NEXT:    vs8r.v v8, (a0)
+; ZVBB-RV32-NEXT:    vl8re64.v v16, (a2)
+; ZVBB-RV32-NEXT:    vl8re64.v v8, (a0)
+; ZVBB-RV32-NEXT:    addi sp, s0, -80
+; ZVBB-RV32-NEXT:    lw ra, 76(sp) # 4-byte Folded Reload
+; ZVBB-RV32-NEXT:    lw s0, 72(sp) # 4-byte Folded Reload
+; ZVBB-RV32-NEXT:    addi sp, sp, 80
+; ZVBB-RV32-NEXT:    ret
+;
+; ZVBB-RV64-LABEL: vector_interleave_nxv10i64_nxv2i64:
+; ZVBB-RV64:       # %bb.0:
+; ZVBB-RV64-NEXT:    addi sp, sp, -80
+; ZVBB-RV64-NEXT:    sd ra, 72(sp) # 8-byte Folded Spill
+; ZVBB-RV64-NEXT:    sd s0, 64(sp) # 8-byte Folded Spill
+; ZVBB-RV64-NEXT:    addi s0, sp, 80
+; ZVBB-RV64-NEXT:    csrr a0, vlenb
+; ZVBB-RV64-NEXT:    li a1, 28
+; ZVBB-RV64-NEXT:    mul a0, a0, a1
+; ZVBB-RV64-NEXT:    sub sp, sp, a0
+; ZVBB-RV64-NEXT:    andi sp, sp, -64
+; ZVBB-RV64-NEXT:    vsetvli a0, zero, e64, m1, ta, ma
+; ZVBB-RV64-NEXT:    vmv2r.v v20, v16
+; ZVBB-RV64-NEXT:    addi a0, sp, 64
+; ZVBB-RV64-NEXT:    vmv2r.v v18, v12
+; ZVBB-RV64-NEXT:    csrr a1, vlenb
+; ZVBB-RV64-NEXT:    slli a2, a1, 2
+; ZVBB-RV64-NEXT:    add a1, a2, a1
+; ZVBB-RV64-NEXT:    add a1, sp, a1
+; ZVBB-RV64-NEXT:    addi a1, a1, 64
+; ZVBB-RV64-NEXT:    csrr a2, vlenb
+; ZVBB-RV64-NEXT:    vmv2r.v v16, v8
+; ZVBB-RV64-NEXT:    vmv2r.v v22, v16
+; ZVBB-RV64-NEXT:    vmv2r.v v24, v18
+; ZVBB-RV64-NEXT:    vmv1r.v v26, v20
+; ZVBB-RV64-NEXT:    add a3, a0, a2
+; ZVBB-RV64-NEXT:    vmv1r.v v23, v10
+; ZVBB-RV64-NEXT:    add a4, a1, a2
+; ZVBB-RV64-NEXT:    add a5, a4, a2
+; ZVBB-RV64-NEXT:    vmv1r.v v25, v14
+; ZVBB-RV64-NEXT:    add a6, a5, a2
+; ZVBB-RV64-NEXT:    vmv1r.v v18, v11
+; ZVBB-RV64-NEXT:    vsseg5e64.v v22, (a0)
+; ZVBB-RV64-NEXT:    vmv1r.v v20, v15
+; ZVBB-RV64-NEXT:    vsseg5e64.v v17, (a1)
+; ZVBB-RV64-NEXT:    vl1re64.v v16, (a6)
+; ZVBB-RV64-NEXT:    add a6, a6, a2
+; ZVBB-RV64-NEXT:    vl1re64.v v17, (a6)
+; ZVBB-RV64-NEXT:    add a6, a3, a2
+; ZVBB-RV64-NEXT:    vl1re64.v v10, (a6)
+; ZVBB-RV64-NEXT:    add a6, a6, a2
+; ZVBB-RV64-NEXT:    vl1re64.v v11, (a6)
+; ZVBB-RV64-NEXT:    vl1re64.v v8, (a0)
+; ZVBB-RV64-NEXT:    vl1re64.v v9, (a3)
+; ZVBB-RV64-NEXT:    vl1re64.v v14, (a4)
+; ZVBB-RV64-NEXT:    csrr a0, vlenb
+; ZVBB-RV64-NEXT:    li a3, 10
+; ZVBB-RV64-NEXT:    mul a0, a0, a3
+; ZVBB-RV64-NEXT:    add a0, sp, a0
+; ZVBB-RV64-NEXT:    addi a0, a0, 64
+; ZVBB-RV64-NEXT:    add a6, a6, a2
+; ZVBB-RV64-NEXT:    vl1re64.v v15, (a5)
+; ZVBB-RV64-NEXT:    vl1re64.v v12, (a6)
+; ZVBB-RV64-NEXT:    vl1re64.v v13, (a1)
+; ZVBB-RV64-NEXT:    slli a2, a2, 3
+; ZVBB-RV64-NEXT:    add a2, a0, a2
+; ZVBB-RV64-NEXT:    vs2r.v v16, (a2)
+; ZVBB-RV64-NEXT:    vs8r.v v8, (a0)
+; ZVBB-RV64-NEXT:    vl8re64.v v16, (a2)
+; ZVBB-RV64-NEXT:    vl8re64.v v8, (a0)
+; ZVBB-RV64-NEXT:    addi sp, s0, -80
+; ZVBB-RV64-NEXT:    ld ra, 72(sp) # 8-byte Folded Reload
+; ZVBB-RV64-NEXT:    ld s0, 64(sp) # 8-byte Folded Reload
+; ZVBB-RV64-NEXT:    addi sp, sp, 80
+; ZVBB-RV64-NEXT:    ret
+  %res = call <vscale x 10 x i64> @llvm.vector.interleave5.nxv10i64(<vscale x 2 x i64> %a, <vscale x 2 x i64> %b, <vscale x 2 x i64> %c, <vscale x 2 x i64> %d, <vscale x 2 x i64> %e)
+  ret <vscale x 10 x i64> %res
+}
+
+define <vscale x 112 x i1> @vector_interleave_nxv112i1_nxv16i1(<vscale x 16 x i1> %a, <vscale x 16 x i1> %b, <vscale x 16 x i1> %c, <vscale x 16 x i1> %d, <vscale x 16 x i1> %e, <vscale x 16 x i1> %f, <vscale x 16 x i1> %g) nounwind {
+; CHECK-LABEL: vector_interleave_nxv112i1_nxv16i1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    addi sp, sp, -16
+; CHECK-NEXT:    csrr a0, vlenb
+; CHECK-NEXT:    li a1, 14
+; CHECK-NEXT:    mul a0, a0, a1
+; CHECK-NEXT:    sub sp, sp, a0
+; CHECK-NEXT:    vsetvli a0, zero, e8, m2, ta, ma
+; CHECK-NEXT:    vmv.v.i v14, 0
+; CHECK-NEXT:    addi a4, sp, 16
+; CHECK-NEXT:    csrr a0, vlenb
+; CHECK-NEXT:    slli a1, a0, 3
+; CHECK-NEXT:    sub a0, a1, a0
+; CHECK-NEXT:    add a0, sp, a0
+; CHECK-NEXT:    addi a0, a0, 16
+; CHECK-NEXT:    csrr a2, vlenb
+; CHECK-NEXT:    vmerge.vim v16, v14, 1, v0
+; CHECK-NEXT:    vmv1r.v v0, v8
+; CHECK-NEXT:    vmerge.vim v22, v14, 1, v0
+; CHECK-NEXT:    add a3, a4, a2
+; CHECK-NEXT:    srli a1, a2, 2
+; CHECK-NEXT:    add a5, a0, a2
+; CHECK-NEXT:    vmv4r.v v24, v16
+; CHECK-NEXT:    vmv1r.v v0, v9
+; CHECK-NEXT:    vmerge.vim v18, v14, 1, v0
+; CHECK-NEXT:    add a6, a3, a2
+; CHECK-NEXT:    vmv1r.v v25, v22
+; CHECK-NEXT:    vmv1r.v v0, v10
+; CHECK-NEXT:    vmerge.vim v8, v14, 1, v0
+; CHECK-NEXT:    vmv1r.v v26, v18
+; CHECK-NEXT:    vmv1r.v v0, v11
+; CHECK-NEXT:    vmerge.vim v20, v14, 1, v0
+; CHECK-NEXT:    vmv1r.v v27, v8
+; CHECK-NEXT:    vmv1r.v v0, v12
+; CHECK-NEXT:    vmerge.vim v10, v14, 1, v0
+; CHECK-NEXT:    vmv1r.v v28, v20
+; CHECK-NEXT:    vmv1r.v v18, v23
+; CHECK-NEXT:    add a7, a6, a2
+; CHECK-NEXT:    vmv1r.v v29, v10
+; CHECK-NEXT:    vmv1r.v v20, v9
+; CHECK-NEXT:    vmv1r.v v0, v13
+; CHECK-NEXT:    vmerge.vim v30, v14, 1, v0
+; CHECK-NEXT:    vmv1r.v v22, v11
+; CHECK-NEXT:    vsetvli t0, zero, e8, m1, ta, ma
+; CHECK-NEXT:    vsseg7e8.v v24, (a4)
+; CHECK-NEXT:    vmv1r.v v23, v31
+; CHECK-NEXT:    vsseg7e8.v v17, (a0)
+; CHECK-NEXT:    vl1r.v v8, (a6)
+; CHECK-NEXT:    add a6, a7, a2
+; CHECK-NEXT:    vl1r.v v10, (a4)
+; CHECK-NEXT:    add a4, a6, a2
+; CHECK-NEXT:    vl1r.v v12, (a6)
+; CHECK-NEXT:    add a6, a4, a2
+; CHECK-NEXT:    vl1r.v v14, (a6)
+; CHECK-NEXT:    add a6, a5, a2
+; CHECK-NEXT:    vl1r.v v16, (a5)
+; CHECK-NEXT:    add a5, a6, a2
+; CHECK-NEXT:    vl1r.v v18, (a5)
+; CHECK-NEXT:    add a5, a5, a2
+; CHECK-NEXT:    vl1r.v v9, (a7)
+; CHECK-NEXT:    add a7, a5, a2
+; CHECK-NEXT:    vl1r.v v20, (a7)
+; CHECK-NEXT:    add a7, a7, a2
+; CHECK-NEXT:    srli a2, a2, 1
+; CHECK-NEXT:    vl1r.v v11, (a3)
+; CHECK-NEXT:    add a3, a1, a1
+; CHECK-NEXT:    vl1r.v v13, (a4)
+; CHECK-NEXT:    add a4, a2, a2
+; CHECK-NEXT:    vl1r.v v15, (a0)
+; CHECK-NEXT:    vl1r.v v19, (a5)
+; CHECK-NEXT:    vl1r.v v17, (a6)
+; CHECK-NEXT:    vl1r.v v21, (a7)
+; CHECK-NEXT:    vsetvli a0, zero, e8, m2, ta, ma
+; CHECK-NEXT:    vmsne.vi v22, v8, 0
+; CHECK-NEXT:    vmsne.vi v0, v10, 0
+; CHECK-NEXT:    vmsne.vi v9, v12, 0
+; CHECK-NEXT:    vmsne.vi v10, v14, 0
+; CHECK-NEXT:    vmsne.vi v11, v18, 0
+; CHECK-NEXT:    vmsne.vi v8, v16, 0
+; CHECK-NEXT:    vmsne.vi v12, v20, 0
+; CHECK-NEXT:    vsetvli zero, a3, e8, mf2, ta, ma
+; CHECK-NEXT:    vslideup.vx v0, v22, a1
+; CHECK-NEXT:    vslideup.vx v9, v10, a1
+; CHECK-NEXT:    vslideup.vx v8, v11, a1
+; CHECK-NEXT:    vsetvli zero, a4, e8, m1, ta, ma
+; CHECK-NEXT:    vslideup.vx v0, v9, a2
+; CHECK-NEXT:    vslideup.vx v8, v12, a2
+; CHECK-NEXT:    csrr a0, vlenb
+; CHECK-NEXT:    li a1, 14
+; CHECK-NEXT:    mul a0, a0, a1
+; CHECK-NEXT:    add sp, sp, a0
+; CHECK-NEXT:    addi sp, sp, 16
+; CHECK-NEXT:    ret
+;
+; ZVBB-LABEL: vector_interleave_nxv112i1_nxv16i1:
+; ZVBB:       # %bb.0:
+; ZVBB-NEXT:    addi sp, sp, -16
+; ZVBB-NEXT:    csrr a0, vlenb
+; ZVBB-NEXT:    li a1, 14
+; ZVBB-NEXT:    mul a0, a0, a1
+; ZVBB-NEXT:    sub sp, sp, a0
+; ZVBB-NEXT:    vsetvli a0, zero, e8, m2, ta, ma
+; ZVBB-NEXT:    vmv.v.i v14, 0
+; ZVBB-NEXT:    addi a4, sp, 16
+; ZVBB-NEXT:    csrr a0, vlenb
+; ZVBB-NEXT:    slli a1, a0, 3
+; ZVBB-NEXT:    sub a0, a1, a0
+; ZVBB-NEXT:    add a0, sp, a0
+; ZVBB-NEXT:    addi a0, a0, 16
+; ZVBB-NEXT:    csrr a2, vlenb
+; ZVBB-NEXT:    vmerge.vim v16, v14, 1, v0
+; ZVBB-NEXT:    vmv1r.v v0, v8
+; ZVBB-NEXT:    vmerge.vim v22, v14, 1, v0
+; ZVBB-NEXT:    add a3, a4, a2
+; ZVBB-NEXT:    srli a1, a2, 2
+; ZVBB-NEXT:    add a5, a0, a2
+; ZVBB-NEXT:    vmv4r.v v24, v16
+; ZVBB-NEXT:    vmv1r.v v0, v9
+; ZVBB-NEXT:    vmerge.vim v18, v14, 1, v0
+; ZVBB-NEXT:    add a6, a3, a2
+; ZVBB-NEXT:    vmv1r.v v25, v22
+; ZVBB-NEXT:    vmv1r.v v0, v10
+; ZVBB-NEXT:    vmerge.vim v8, v14, 1, v0
+; ZVBB-NEXT:    vmv1r.v v26, v18
+; ZVBB-NEXT:    vmv1r.v v0, v11
+; ZVBB-NEXT:    vmerge.vim v20, v14, 1, v0
+; ZVBB-NEXT:    vmv1r.v v27, v8
+; ZVBB-NEXT:    vmv1r.v v0, v12
+; ZVBB-NEXT:    vmerge.vim v10, v14, 1, v0
+; ZVBB-NEXT:    vmv1r.v v28, v20
+; ZVBB-NEXT:    vmv1r.v v18, v23
+; ZVBB-NEXT:    add a7, a6, a2
+; ZVBB-NEXT:    vmv1r.v v29, v10
+; ZVBB-NEXT:    vmv1r.v v20, v9
+; ZVBB-NEXT:    vmv1r.v v0, v13
+; ZVBB-NEXT:    vmerge.vim v30, v14, 1, v0
+; ZVBB-NEXT:    vmv1r.v v22, v11
+; ZVBB-NEXT:    vsetvli t0, zero, e8, m1, ta, ma
+; ZVBB-NEXT:    vsseg7e8.v v24, (a4)
+; ZVBB-NEXT:    vmv1r.v v23, v31
+; ZVBB-NEXT:    vsseg7e8.v v17, (a0)
+; ZVBB-NEXT:    vl1r.v v8, (a6)
+; ZVBB-NEXT:    add a6, a7, a2
+; ZVBB-NEXT:    vl1r.v v10, (a4)
+; ZVBB-NEXT:    add a4, a6, a2
+; ZVBB-NEXT:    vl1r.v v12, (a6)
+; ZVBB-NEXT:    add a6, a4, a2
+; ZVBB-NEXT:    vl1r.v v14, (a6)
+; ZVBB-NEXT:    add a6, a5, a2
+; ZVBB-NEXT:    vl1r.v v16, (a5)
+; ZVBB-NEXT:    add a5, a6, a2
+; ZVBB-NEXT:    vl1r.v v18, (a5)
+; ZVBB-NEXT:    add a5, a5, a2
+; ZVBB-NEXT:    vl1r.v v9, (a7)
+; ZVBB-NEXT:    add a7, a5, a2
+; ZVBB-NEXT:    vl1r.v v20, (a7)
+; ZVBB-NEXT:    add a7, a7, a2
+; ZVBB-NEXT:    srli a2, a2, 1
+; ZVBB-NEXT:    vl1r.v v11, (a3)
+; ZVBB-NEXT:    add a3, a1, a1
+; ZVBB-NEXT:    vl1r.v v13, (a4)
+; ZVBB-NEXT:    add a4, a2, a2
+; ZVBB-NEXT:    vl1r.v v15, (a0)
+; ZVBB-NEXT:    vl1r.v v19, (a5)
+; ZVBB-NEXT:    vl1r.v v17, (a6)
+; ZVBB-NEXT:    vl1r.v v21, (a7)
+; ZVBB-NEXT:    vsetvli a0, zero, e8, m2, ta, ma
+; ZVBB-NEXT:    vmsne.vi v22, v8, 0
+; ZVBB-NEXT:    vmsne.vi v0, v10, 0
+; ZVBB-NEXT:    vmsne.vi v9, v12, 0
+; ZVBB-NEXT:    vmsne.vi v10, v14, 0
+; ZVBB-NEXT:    vmsne.vi v11, v18, 0
+; ZVBB-NEXT:    vmsne.vi v8, v16, 0
+; ZVBB-NEXT:    vmsne.vi v12, v20, 0
+; ZVBB-NEXT:    vsetvli zero, a3, e8, mf2, ta, ma
+; ZVBB-NEXT:    vslideup.vx v0, v22, a1
+; ZVBB-NEXT:    vslideup.vx v9, v10, a1
+; ZVBB-NEXT:    vslideup.vx v8, v11, a1
+; ZVBB-NEXT:    vsetvli zero, a4, e8, m1, ta, ma
+; ZVBB-NEXT:    vslideup.vx v0, v9, a2
+; ZVBB-NEXT:    vslideup.vx v8, v12, a2
+; ZVBB-NEXT:    csrr a0, vlenb
+; ZVBB-NEXT:    li a1, 14
+; ZVBB-NEXT:    mul a0, a0, a1
+; ZVBB-NEXT:    add sp, sp, a0
+; ZVBB-NEXT:    addi sp, sp, 16
+; ZVBB-NEXT:    ret
+  %res = call <vscale x 112 x i1> @llvm.vector.interleave7.nxv112i1(<vscale x 16 x i1> %a, <vscale x 16 x i1> %b, <vscale x 16 x i1> %c, <vscale x 16 x i1> %d, <vscale x 16 x i1> %e, <vscale x 16 x i1> %f, <vscale x 16 x i1> %g)
+  ret <vscale x 112 x i1> %res
+}
+
+
+define <vscale x 112 x i8> @vector_interleave_nxv112i8_nxv16i8(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b, <vscale x 16 x i8> %c, <vscale x 16 x i8> %d, <vscale x 16 x i8> %e, <vscale x 16 x i8> %f, <vscale x 16 x i8> %g) nounwind {
+;
+; RV32-LABEL: vector_interleave_nxv112i8_nxv16i8:
+; RV32:       # %bb.0:
+; RV32-NEXT:    addi sp, sp, -80
+; RV32-NEXT:    sw ra, 76(sp) # 4-byte Folded Spill
+; RV32-NEXT:    sw s0, 72(sp) # 4-byte Folded Spill
+; RV32-NEXT:    addi s0, sp, 80
+; RV32-NEXT:    csrr a0, vlenb
+; RV32-NEXT:    slli a0, a0, 5
+; RV32-NEXT:    sub sp, sp, a0
+; RV32-NEXT:    andi sp, sp, -64
+; RV32-NEXT:    vsetvli a0, zero, e8, m1, ta, ma
+; RV32-NEXT:    vmv2r.v v26, v20
+; RV32-NEXT:    addi a0, sp, 64
+; RV32-NEXT:    vmv2r.v v24, v16
+; RV32-NEXT:    csrr a1, vlenb
+; RV32-NEXT:    slli a2, a1, 3
+; RV32-NEXT:    sub a1, a2, a1
+; RV32-NEXT:    add a1, sp, a1
+; RV32-NEXT:    addi a1, a1, 64
+; RV32-NEXT:    vmv2r.v v22, v12
+; RV32-NEXT:    csrr a2, vlenb
+; RV32-NEXT:    vmv2r.v v20, v8
+; RV32-NEXT:    vmv1r.v v1, v20
+; RV32-NEXT:    vmv1r.v v3, v22
+; RV32-NEXT:    vmv1r.v v5, v24
+; RV32-NEXT:    vmv1r.v v7, v26
+; RV32-NEXT:    add a3, a0, a2
+; RV32-NEXT:    vmv1r.v v2, v10
+; RV32-NEXT:    add a4, a1, a2
+; RV32-NEXT:    slli a5, a2, 2
+; RV32-NEXT:    vmv1r.v v4, v14
+; RV32-NEXT:    slli a6, a2, 4
+; RV32-NEXT:    add a7, a4, a2
+; RV32-NEXT:    vmv1r.v v6, v18
+; RV32-NEXT:    sub a5, a6, a5
+; RV32-NEXT:    vmv1r.v v22, v11
+; RV32-NEXT:    add a6, a7, a2
+; RV32-NEXT:    vmv1r.v v24, v15
+; RV32-NEXT:    vsseg7e8.v v1, (a0)
+; RV32-NEXT:    vmv1r.v v26, v19
+; RV32-NEXT:    vsseg7e8.v v21, (a1)
+; RV32-NEXT:    vl1r.v v10, (a6)
+; RV32-NEXT:    add a6, a6, a2
+; RV32-NEXT:    vl1r.v v11, (a6)
+; RV32-NEXT:    add a6, a6, a2
+; RV32-NEXT:    vl1r.v v12, (a6)
+; RV32-NEXT:    add a6, a6, a2
+; RV32-NEXT:    vl1r.v v13, (a6)
+; RV32-NEXT:    add a6, a3, a2
+; RV32-NEXT:    vl1r.v v18, (a6)
+; RV32-NEXT:    add a6, a6, a2
+; RV32-NEXT:    vl1r.v v19, (a6)
+; RV32-NEXT:    vl1r.v v16, (a0)
+; RV32-NEXT:    vl1r.v v8, (a4)
+; RV32-NEXT:    vl1r.v v17, (a3)
+; RV32-NEXT:    vl1r.v v9, (a7)
+; RV32-NEXT:    csrr a0, vlenb
+; RV32-NEXT:    li a3, 14
+; RV32-NEXT:    mul a0, a0, a3
+; RV32-NEXT:    add a0, sp, a0
+; RV32-NEXT:    addi a0, a0, 64
+; RV32-NEXT:    add a6, a6, a2
+; RV32-NEXT:    vl1r.v v20, (a6)
+; RV32-NEXT:    add a6, a6, a2
+; RV32-NEXT:    vl1r.v v21, (a6)
+; RV32-NEXT:    add a6, a6, a2
+; RV32-NEXT:    slli a2, a2, 3
+; RV32-NEXT:    add a2, a0, a2
+; RV32-NEXT:    vl1r.v v22, (a6)
+; RV32-NEXT:    vl1r.v v23, (a1)
+; RV32-NEXT:    add a5, a0, a5
+; RV32-NEXT:    vs2r.v v12, (a5)
+; RV32-NEXT:    vs4r.v v8, (a2)
+; RV32-NEXT:    vs8r.v v16, (a0)
+; RV32-NEXT:    vl8r.v v16, (a2)
+; RV32-NEXT:    vl8r.v v8, (a0)
+; RV32-NEXT:    addi sp, s0, -80
+; RV32-NEXT:    lw ra, 76(sp) # 4-byte Folded Reload
+; RV32-NEXT:    lw s0, 72(sp) # 4-byte Folded Reload
+; RV32-NEXT:    addi sp, sp, 80
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: vector_interleave_nxv112i8_nxv16i8:
+; RV64:       # %bb.0:
+; RV64-NEXT:    addi sp, sp, -80
+; RV64-NEXT:    sd ra, 72(sp) # 8-byte Folded Spill
+; RV64-NEXT:    sd s0, 64(sp) # 8-byte Folded Spill
+; RV64-NEXT:    addi s0, sp, 80
+; RV64-NEXT:    csrr a0, vlenb
+; RV64-NEXT:    slli a0, a0, 5
+; RV64-NEXT:    sub sp, sp, a0
+; RV64-NEXT:    andi sp, sp, -64
+; RV64-NEXT:    vsetvli a0, zero, e8, m1, ta, ma
+; RV64-NEXT:    vmv2r.v v26, v20
+; RV64-NEXT:    addi a0, sp, 64
+; RV64-NEXT:    vmv2r.v v24, v16
+; RV64-NEXT:    csrr a1, vlenb
+; RV64-NEXT:    slli a2, a1, 3
+; RV64-NEXT:    sub a1, a2, a1
+; RV64-NEXT:    add a1, sp, a1
+; RV64-NEXT:    addi a1, a1, 64
+; RV64-NEXT:    vmv2r.v v22, v12
+; RV64-NEXT:    csrr a2, vlenb
+; RV64-NEXT:    vmv2r.v v20, v8
+; RV64-NEXT:    vmv1r.v v1, v20
+; RV64-NEXT:    vmv1r.v v3, v22
+; RV64-NEXT:    vmv1r.v v5, v24
+; RV64-NEXT:    vmv1r.v v7, v26
+; RV64-NEXT:    add a3, a0, a2
+; RV64-NEXT:    vmv1r.v v2, v10
+; RV64-NEXT:    add a4, a1, a2
+; RV64-NEXT:    slli a5, a2, 2
+; RV64-NEXT:    vmv1r.v v4, v14
+; RV64-NEXT:    slli a6, a2, 4
+; RV64-NEXT:    add a7, a4, a2
+; RV64-NEXT:    vmv1r.v v6, v18
+; RV64-NEXT:    sub a5, a6, a5
+; RV64-NEXT:    vmv1r.v v22, v11
+; RV64-NEXT:    add a6, a7, a2
+; RV64-NEXT:    vmv1r.v v24, v15
+; RV64-NEXT:    vsseg7e8.v v1, (a0)
+; RV64-NEXT:    vmv1r.v v26, v19
+; RV64-NEXT:    vsseg7e8.v v21, (a1)
+; RV64-NEXT:    vl1r.v v10, (a6)
+; RV64-NEXT:    add a6, a6, a2
+; RV64-NEXT:    vl1r.v v11, (a6)
+; RV64-NEXT:    add a6, a6, a2
+; RV64-NEXT:    vl1r.v v12, (a6)
+; RV64-NEXT:    add a6, a6, a2
+; RV64-NEXT:    vl1r.v v13, (a6)
+; RV64-NEXT:    add a6, a3, a2
+; RV64-NEXT:    vl1r.v v18, (a6)
+; RV64-NEXT:    add a6, a6, a2
+; RV64-NEXT:    vl1r.v v19, (a6)
+; RV64-NEXT:    vl1r.v v16, (a0)
+; RV64-NEXT:    vl1r.v v8, (a4)
+; RV64-NEXT:    vl1r.v v17, (a3)
+; RV64-NEXT:    vl1r.v v9, (a7)
+; RV64-NEXT:    csrr a0, vlenb
+; RV64-NEXT:    li a3, 14
+; RV64-NEXT:    mul a0, a0, a3
+; RV64-NEXT:    add a0, sp, a0
+; RV64-NEXT:    addi a0, a0, 64
+; RV64-NEXT:    add a6, a6, a2
+; RV64-NEXT:    vl1r.v v20, (a6)
+; RV64-NEXT:    add a6, a6, a2
+; RV64-NEXT:    vl1r.v v21, (a6)
+; RV64-NEXT:    add a6, a6, a2
+; RV64-NEXT:    slli a2, a2, 3
+; RV64-NEXT:    add a2, a0, a2
+; RV64-NEXT:    vl1r.v v22, (a6)
+; RV64-NEXT:    vl1r.v v23, (a1)
+; RV64-NEXT:    add a5, a0, a5
+; RV64-NEXT:    vs2r.v v12, (a5)
+; RV64-NEXT:    vs4r.v v8, (a2)
+; RV64-NEXT:    vs8r.v v16, (a0)
+; RV64-NEXT:    vl8r.v v16, (a2)
+; RV64-NEXT:    vl8r.v v8, (a0)
+; RV64-NEXT:    addi sp, s0, -80
+; RV64-NEXT:    ld ra, 72(sp) # 8-byte Folded Reload
+; RV64-NEXT:    ld s0, 64(sp) # 8-byte Folded Reload
+; RV64-NEXT:    addi sp, sp, 80
+; RV64-NEXT:    ret
+;
+; ZVBB-RV32-LABEL: vector_interleave_nxv112i8_nxv16i8:
+; ZVBB-RV32:       # %bb.0:
+; ZVBB-RV32-NEXT:    addi sp, sp, -80
+; ZVBB-RV32-NEXT:    sw ra, 76(sp) # 4-byte Folded Spill
+; ZVBB-RV32-NEXT:    sw s0, 72(sp) # 4-byte Folded Spill
+; ZVBB-RV32-NEXT:    addi s0, sp, 80
+; ZVBB-RV32-NEXT:    csrr a0, vlenb
+; ZVBB-RV32-NEXT:    slli a0, a0, 5
+; ZVBB-RV32-NEXT:    sub sp, sp, a0
+; ZVBB-RV32-NEXT:    andi sp, sp, -64
+; ZVBB-RV32-NEXT:    vsetvli a0, zero, e8, m1, ta, ma
+; ZVBB-RV32-NEXT:    vmv2r.v v26, v20
+; ZVBB-RV32-NEXT:    addi a0, sp, 64
+; ZVBB-RV32-NEXT:    vmv2r.v v24, v16
+; ZVBB-RV32-NEXT:    csrr a1, vlenb
+; ZVBB-RV32-NEXT:    slli a2, a1, 3
+; ZVBB-RV32-NEXT:    sub a1, a2, a1
+; ZVBB-RV32-NEXT:    add a1, sp, a1
+; ZVBB-RV32-NEXT:    addi a1, a1, 64
+; ZVBB-RV32-NEXT:    vmv2r.v v22, v12
+; ZVBB-RV32-NEXT:    csrr a2, vlenb
+; ZVBB-RV32-NEXT:    vmv2r.v v20, v8
+; ZVBB-RV32-NEXT:    vmv1r.v v1, v20
+; ZVBB-RV32-NEXT:    vmv1r.v v3, v22
+; ZVBB-RV32-NEXT:    vmv1r.v v5, v24
+; ZVBB-RV32-NEXT:    vmv1r.v v7, v26
+; ZVBB-RV32-NEXT:    add a3, a0, a2
+; ZVBB-RV32-NEXT:    vmv1r.v v2, v10
+; ZVBB-RV32-NEXT:    add a4, a1, a2
+; ZVBB-RV32-NEXT:    slli a5, a2, 2
+; ZVBB-RV32-NEXT:    vmv1r.v v4, v14
+; ZVBB-RV32-NEXT:    slli a6, a2, 4
+; ZVBB-RV32-NEXT:    add a7, a4, a2
+; ZVBB-RV32-NEXT:    vmv1r.v v6, v18
+; ZVBB-RV32-NEXT:    sub a5, a6, a5
+; ZVBB-RV32-NEXT:    vmv1r.v v22, v11
+; ZVBB-RV32-NEXT:    add a6, a7, a2
+; ZVBB-RV32-NEXT:    vmv1r.v v24, v15
+; ZVBB-RV32-NEXT:    vsseg7e8.v v1, (a0)
+; ZVBB-RV32-NEXT:    vmv1r.v v26, v19
+; ZVBB-RV32-NEXT:    vsseg7e8.v v21, (a1)
+; ZVBB-RV32-NEXT:    vl1r.v v10, (a6)
+; ZVBB-RV32-NEXT:    add a6, a6, a2
+; ZVBB-RV32-NEXT:    vl1r.v v11, (a6)
+; ZVBB-RV32-NEXT:    add a6, a6, a2
+; ZVBB-RV32-NEXT:    vl1r.v v12, (a6)
+; ZVBB-RV32-NEXT:    add a6, a6, a2
+; ZVBB-RV32-NEXT:    vl1r.v v13, (a6)
+; ZVBB-RV32-NEXT:    add a6, a3, a2
+; ZVBB-RV32-NEXT:    vl1r.v v18, (a6)
+; ZVBB-RV32-NEXT:    add a6, a6, a2
+; ZVBB-RV32-NEXT:    vl1r.v v19, (a6)
+; ZVBB-RV32-NEXT:    vl1r.v v16, (a0)
+; ZVBB-RV32-NEXT:    vl1r.v v8, (a4)
+; ZVBB-RV32-NEXT:    vl1r.v v17, (a3)
+; ZVBB-RV32-NEXT:    vl1r.v v9, (a7)
+; ZVBB-RV32-NEXT:    csrr a0, vlenb
+; ZVBB-RV32-NEXT:    li a3, 14
+; ZVBB-RV32-NEXT:    mul a0, a0, a3
+; ZVBB-RV32-NEXT:    add a0, sp, a0
+; ZVBB-RV32-NEXT:    addi a0, a0, 64
+; ZVBB-RV32-NEXT:    add a6, a6, a2
+; ZVBB-RV32-NEXT:    vl1r.v v20, (a6)
+; ZVBB-RV32-NEXT:    add a6, a6, a2
+; ZVBB-RV32-NEXT:    vl1r.v v21, (a6)
+; ZVBB-RV32-NEXT:    add a6, a6, a2
+; ZVBB-RV32-NEXT:    slli a2, a2, 3
+; ZVBB-RV32-NEXT:    add a2, a0, a2
+; ZVBB-RV32-NEXT:    vl1r.v v22, (a6)
+; ZVBB-RV32-NEXT:    vl1r.v v23, (a1)
+; ZVBB-RV32-NEXT:    add a5, a0, a5
+; ZVBB-RV32-NEXT:    vs2r.v v12, (a5)
+; ZVBB-RV32-NEXT:    vs4r.v v8, (a2)
+; ZVBB-RV32-NEXT:    vs8r.v v16, (a0)
+; ZVBB-RV32-NEXT:    vl8r.v v16, (a2)
+; ZVBB-RV32-NEXT:    vl8r.v v8, (a0)
+; ZVBB-RV32-NEXT:    addi sp, s0, -80
+; ZVBB-RV32-NEXT:    lw ra, 76(sp) # 4-byte Folded Reload
+; ZVBB-RV32-NEXT:    lw s0, 72(sp) # 4-byte Folded Reload
+; ZVBB-RV32-NEXT:    addi sp, sp, 80
+; ZVBB-RV32-NEXT:    ret
+;
+; ZVBB-RV64-LABEL: vector_interleave_nxv112i8_nxv16i8:
+; ZVBB-RV64:       # %bb.0:
+; ZVBB-RV64-NEXT:    addi sp, sp, -80
+; ZVBB-RV64-NEXT:    sd ra, 72(sp) # 8-byte Folded Spill
+; ZVBB-RV64-NEXT:    sd s0, 64(sp) # 8-byte Folded Spill
+; ZVBB-RV64-NEXT:    addi s0, sp, 80
+; ZVBB-RV64-NEXT:    csrr a0, vlenb
+; ZVBB-RV64-NEXT:    slli a0, a0, 5
+; ZVBB-RV64-NEXT:    sub sp, sp, a0
+; ZVBB-RV64-NEXT:    andi sp, sp, -64
+; ZVBB-RV64-NEXT:    vsetvli a0, zero, e8, m1, ta, ma
+; ZVBB-RV64-NEXT:    vmv2r.v v26, v20
+; ZVBB-RV64-NEXT:    addi a0, sp, 64
+; ZVBB-RV64-NEXT:    vmv2r.v v24, v16
+; ZVBB-RV64-NEXT:    csrr a1, vlenb
+; ZVBB-RV64-NEXT:    slli a2, a1, 3
+; ZVBB-RV64-NEXT:    sub a1, a2, a1
+; ZVBB-RV64-NEXT:    add a1, sp, a1
+; ZVBB-RV64-NEXT:    addi a1, a1, 64
+; ZVBB-RV64-NEXT:    vmv2r.v v22, v12
+; ZVBB-RV64-NEXT:    csrr a2, vlenb
+; ZVBB-RV64-NEXT:    vmv2r.v v20, v8
+; ZVBB-RV64-NEXT:    vmv1r.v v1, v20
+; ZVBB-RV64-NEXT:    vmv1r.v v3, v22
+; ZVBB-RV64-NEXT:    vmv1r.v v5, v24
+; ZVBB-RV64-NEXT:    vmv1r.v v7, v26
+; ZVBB-RV64-NEXT:    add a3, a0, a2
+; ZVBB-RV64-NEXT:    vmv1r.v v2, v10
+; ZVBB-RV64-NEXT:    add a4, a1, a2
+; ZVBB-RV64-NEXT:    slli a5, a2, 2
+; ZVBB-RV64-NEXT:    vmv1r.v v4, v14
+; ZVBB-RV64-NEXT:    slli a6, a2, 4
+; ZVBB-RV64-NEXT:    add a7, a4, a2
+; ZVBB-RV64-NEXT:    vmv1r.v v6, v18
+; ZVBB-RV64-NEXT:    sub a5, a6, a5
+; ZVBB-RV64-NEXT:    vmv1r.v v22, v11
+; ZVBB-RV64-NEXT:    add a6, a7, a2
+; ZVBB-RV64-NEXT:    vmv1r.v v24, v15
+; ZVBB-RV64-NEXT:    vsseg7e8.v v1, (a0)
+; ZVBB-RV64-NEXT:    vmv1r.v v26, v19
+; ZVBB-RV64-NEXT:    vsseg7e8.v v21, (a1)
+; ZVBB-RV64-NEXT:    vl1r.v v10, (a6)
+; ZVBB-RV64-NEXT:    add a6, a6, a2
+; ZVBB-RV64-NEXT:    vl1r.v v11, (a6)
+; ZVBB-RV64-NEXT:    add a6, a6, a2
+; ZVBB-RV64-NEXT:    vl1r.v v12, (a6)
+; ZVBB-RV64-NEXT:    add a6, a6, a2
+; ZVBB-RV64-NEXT:    vl1r.v v13, (a6)
+; ZVBB-RV64-NEXT:    add a6, a3, a2
+; ZVBB-RV64-NEXT:    vl1r.v v18, (a6)
+; ZVBB-RV64-NEXT:    add a6, a6, a2
+; ZVBB-RV64-NEXT:    vl1r.v v19, (a6)
+; ZVBB-RV64-NEXT:    vl1r.v v16, (a0)
+; ZVBB-RV64-NEXT:    vl1r.v v8, (a4)
+; ZVBB-RV64-NEXT:    vl1r.v v17, (a3)
+; ZVBB-RV64-NEXT:    vl1r.v v9, (a7)
+; ZVBB-RV64-NEXT:    csrr a0, vlenb
+; ZVBB-RV64-NEXT:    li a3, 14
+; ZVBB-RV64-NEXT:    mul a0, a0, a3
+; ZVBB-RV64-NEXT:    add a0, sp, a0
+; ZVBB-RV64-NEXT:    addi a0, a0, 64
+; ZVBB-RV64-NEXT:    add a6, a6, a2
+; ZVBB-RV64-NEXT:    vl1r.v v20, (a6)
+; ZVBB-RV64-NEXT:    add a6, a6, a2
+; ZVBB-RV64-NEXT:    vl1r.v v21, (a6)
+; ZVBB-RV64-NEXT:    add a6, a6, a2
+; ZVBB-RV64-NEXT:    slli a2, a2, 3
+; ZVBB-RV64-NEXT:    add a2, a0, a2
+; ZVBB-RV64-NEXT:    vl1r.v v22, (a6)
+; ZVBB-RV64-NEXT:    vl1r.v v23, (a1)
+; ZVBB-RV64-NEXT:    add a5, a0, a5
+; ZVBB-RV64-NEXT:    vs2r.v v12, (a5)
+; ZVBB-RV64-NEXT:    vs4r.v v8, (a2)
+; ZVBB-RV64-NEXT:    vs8r.v v16, (a0)
+; ZVBB-RV64-NEXT:    vl8r.v v16, (a2)
+; ZVBB-RV64-NEXT:    vl8r.v v8, (a0)
+; ZVBB-RV64-NEXT:    addi sp, s0, -80
+; ZVBB-RV64-NEXT:    ld ra, 72(sp) # 8-byte Folded Reload
+; ZVBB-RV64-NEXT:    ld s0, 64(sp) # 8-byte Folded Reload
+; ZVBB-RV64-NEXT:    addi sp, sp, 80
+; ZVBB-RV64-NEXT:    ret
+  %res = call <vscale x 112 x i8> @llvm.vector.interleave7.nxv112i8(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b, <vscale x 16 x i8> %c, <vscale x 16 x i8> %d, <vscale x 16 x i8> %e, <vscale x 16 x i8> %f, <vscale x 16 x i8> %g)
+  ret <vscale x 112 x i8> %res
+}
+
+
+define <vscale x 56 x i16> @vector_interleave_nxv56i16_nxv8i16(<vscale x 8 x i16> %a, <vscale x 8 x i16> %b, <vscale x 8 x i16> %c, <vscale x 8 x i16> %d, <vscale x 8 x i16> %e, <vscale x 8 x i16> %f, <vscale x 8 x i16> %g) nounwind {
+;
+; RV32-LABEL: vector_interleave_nxv56i16_nxv8i16:
+; RV32:       # %bb.0:
+; RV32-NEXT:    addi sp, sp, -80
+; RV32-NEXT:    sw ra, 76(sp) # 4-byte Folded Spill
+; RV32-NEXT:    sw s0, 72(sp) # 4-byte Folded Spill
+; RV32-NEXT:    addi s0, sp, 80
+; RV32-NEXT:    csrr a0, vlenb
+; RV32-NEXT:    slli a0, a0, 5
+; RV32-NEXT:    sub sp, sp, a0
+; RV32-NEXT:    andi sp, sp, -64
+; RV32-NEXT:    vsetvli a0, zero, e16, m1, ta, ma
+; RV32-NEXT:    vmv2r.v v26, v20
+; RV32-NEXT:    addi a0, sp, 64
+; RV32-NEXT:    vmv2r.v v24, v16
+; RV32-NEXT:    csrr a1, vlenb
+; RV32-NEXT:    slli a2, a1, 3
+; RV32-NEXT:    sub a1, a2, a1
+; RV32-NEXT:    add a1, sp, a1
+; RV32-NEXT:    addi a1, a1, 64
+; RV32-NEXT:    vmv2r.v v22, v12
+; RV32-NEXT:    csrr a2, vlenb
+; RV32-NEXT:    vmv2r.v v20, v8
+; RV32-NEXT:    vmv1r.v v1, v20
+; RV32-NEXT:    vmv1r.v v3, v22
+; RV32-NEXT:    vmv1r.v v5, v24
+; RV32-NEXT:    vmv1r.v v7, v26
+; RV32-NEXT:    add a3, a0, a2
+; RV32-NEXT:    vmv1r.v v2, v10
+; RV32-NEXT:    add a4, a1, a2
+; RV32-NEXT:    slli a5, a2, 2
+; RV32-NEXT:    vmv1r.v v4, v14
+; RV32-NEXT:    slli a6, a2, 4
+; RV32-NEXT:    add a7, a4, a2
+; RV32-NEXT:    vmv1r.v v6, v18
+; RV32-NEXT:    sub a5, a6, a5
+; RV32-NEXT:    vmv1r.v v22, v11
+; RV32-NEXT:    add a6, a7, a2
+; RV32-NEXT:    vmv1r.v v24, v15
+; RV32-NEXT:    vsseg7e16.v v1, (a0)
+; RV32-NEXT:    vmv1r.v v26, v19
+; RV32-NEXT:    vsseg7e16.v v21, (a1)
+; RV32-NEXT:    vl1re16.v v10, (a6)
+; RV32-NEXT:    add a6, a6, a2
+; RV32-NEXT:    vl1re16.v v11, (a6)
+; RV32-NEXT:    add a6, a6, a2
+; RV32-NEXT:    vl1re16.v v12, (a6)
+; RV32-NEXT:    add a6, a6, a2
+; RV32-NEXT:    vl1re16.v v13, (a6)
+; RV32-NEXT:    add a6, a3, a2
+; RV32-NEXT:    vl1re16.v v18, (a6)
+; RV32-NEXT:    add a6, a6, a2
+; RV32-NEXT:    vl1re16.v v19, (a6)
+; RV32-NEXT:    vl1re16.v v16, (a0)
+; RV32-NEXT:    vl1re16.v v8, (a4)
+; RV32-NEXT:    vl1re16.v v17, (a3)
+; RV32-NEXT:    vl1re16.v v9, (a7)
+; RV32-NEXT:    csrr a0, vlenb
+; RV32-NEXT:    li a3, 14
+; RV32-NEXT:    mul a0, a0, a3
+; RV32-NEXT:    add a0, sp, a0
+; RV32-NEXT:    addi a0, a0, 64
+; RV32-NEXT:    add a6, a6, a2
+; RV32-NEXT:    vl1re16.v v20, (a6)
+; RV32-NEXT:    add a6, a6, a2
+; RV32-NEXT:    vl1re16.v v21, (a6)
+; RV32-NEXT:    add a6, a6, a2
+; RV32-NEXT:    slli a2, a2, 3
+; RV32-NEXT:    add a2, a0, a2
+; RV32-NEXT:    vl1re16.v v22, (a6)
+; RV32-NEXT:    vl1re16.v v23, (a1)
+; RV32-NEXT:    add a5, a0, a5
+; RV32-NEXT:    vs2r.v v12, (a5)
+; RV32-NEXT:    vs4r.v v8, (a2)
+; RV32-NEXT:    vs8r.v v16, (a0)
+; RV32-NEXT:    vl8re16.v v16, (a2)
+; RV32-NEXT:    vl8re16.v v8, (a0)
+; RV32-NEXT:    addi sp, s0, -80
+; RV32-NEXT:    lw ra, 76(sp) # 4-byte Folded Reload
+; RV32-NEXT:    lw s0, 72(sp) # 4-byte Folded Reload
+; RV32-NEXT:    addi sp, sp, 80
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: vector_interleave_nxv56i16_nxv8i16:
+; RV64:       # %bb.0:
+; RV64-NEXT:    addi sp, sp, -80
+; RV64-NEXT:    sd ra, 72(sp) # 8-byte Folded Spill
+; RV64-NEXT:    sd s0, 64(sp) # 8-byte Folded Spill
+; RV64-NEXT:    addi s0, sp, 80
+; RV64-NEXT:    csrr a0, vlenb
+; RV64-NEXT:    slli a0, a0, 5
+; RV64-NEXT:    sub sp, sp, a0
+; RV64-NEXT:    andi sp, sp, -64
+; RV64-NEXT:    vsetvli a0, zero, e16, m1, ta, ma
+; RV64-NEXT:    vmv2r.v v26, v20
+; RV64-NEXT:    addi a0, sp, 64
+; RV64-NEXT:    vmv2r.v v24, v16
+; RV64-NEXT:    csrr a1, vlenb
+; RV64-NEXT:    slli a2, a1, 3
+; RV64-NEXT:    sub a1, a2, a1
+; RV64-NEXT:    add a1, sp, a1
+; RV64-NEXT:    addi a1, a1, 64
+; RV64-NEXT:    vmv2r.v v22, v12
+; RV64-NEXT:    csrr a2, vlenb
+; RV64-NEXT:    vmv2r.v v20, v8
+; RV64-NEXT:    vmv1r.v v1, v20
+; RV64-NEXT:    vmv1r.v v3, v22
+; RV64-NEXT:    vmv1r.v v5, v24
+; RV64-NEXT:    vmv1r.v v7, v26
+; RV64-NEXT:    add a3, a0, a2
+; RV64-NEXT:    vmv1r.v v2, v10
+; RV64-NEXT:    add a4, a1, a2
+; RV64-NEXT:    slli a5, a2, 2
+; RV64-NEXT:    vmv1r.v v4, v14
+; RV64-NEXT:    slli a6, a2, 4
+; RV64-NEXT:    add a7, a4, a2
+; RV64-NEXT:    vmv1r.v v6, v18
+; RV64-NEXT:    sub a5, a6, a5
+; RV64-NEXT:    vmv1r.v v22, v11
+; RV64-NEXT:    add a6, a7, a2
+; RV64-NEXT:    vmv1r.v v24, v15
+; RV64-NEXT:    vsseg7e16.v v1, (a0)
+; RV64-NEXT:    vmv1r.v v26, v19
+; RV64-NEXT:    vsseg7e16.v v21, (a1)
+; RV64-NEXT:    vl1re16.v v10, (a6)
+; RV64-NEXT:    add a6, a6, a2
+; RV64-NEXT:    vl1re16.v v11, (a6)
+; RV64-NEXT:    add a6, a6, a2
+; RV64-NEXT:    vl1re16.v v12, (a6)
+; RV64-NEXT:    add a6, a6, a2
+; RV64-NEXT:    vl1re16.v v13, (a6)
+; RV64-NEXT:    add a6, a3, a2
+; RV64-NEXT:    vl1re16.v v18, (a6)
+; RV64-NEXT:    add a6, a6, a2
+; RV64-NEXT:    vl1re16.v v19, (a6)
+; RV64-NEXT:    vl1re16.v v16, (a0)
+; RV64-NEXT:    vl1re16.v v8, (a4)
+; RV64-NEXT:    vl1re16.v v17, (a3)
+; RV64-NEXT:    vl1re16.v v9, (a7)
+; RV64-NEXT:    csrr a0, vlenb
+; RV64-NEXT:    li a3, 14
+; RV64-NEXT:    mul a0, a0, a3
+; RV64-NEXT:    add a0, sp, a0
+; RV64-NEXT:    addi a0, a0, 64
+; RV64-NEXT:    add a6, a6, a2
+; RV64-NEXT:    vl1re16.v v20, (a6)
+; RV64-NEXT:    add a6, a6, a2
+; RV64-NEXT:    vl1re16.v v21, (a6)
+; RV64-NEXT:    add a6, a6, a2
+; RV64-NEXT:    slli a2, a2, 3
+; RV64-NEXT:    add a2, a0, a2
+; RV64-NEXT:    vl1re16.v v22, (a6)
+; RV64-NEXT:    vl1re16.v v23, (a1)
+; RV64-NEXT:    add a5, a0, a5
+; RV64-NEXT:    vs2r.v v12, (a5)
+; RV64-NEXT:    vs4r.v v8, (a2)
+; RV64-NEXT:    vs8r.v v16, (a0)
+; RV64-NEXT:    vl8re16.v v16, (a2)
+; RV64-NEXT:    vl8re16.v v8, (a0)
+; RV64-NEXT:    addi sp, s0, -80
+; RV64-NEXT:    ld ra, 72(sp) # 8-byte Folded Reload
+; RV64-NEXT:    ld s0, 64(sp) # 8-byte Folded Reload
+; RV64-NEXT:    addi sp, sp, 80
+; RV64-NEXT:    ret
+;
+; ZVBB-RV32-LABEL: vector_interleave_nxv56i16_nxv8i16:
+; ZVBB-RV32:       # %bb.0:
+; ZVBB-RV32-NEXT:    addi sp, sp, -80
+; ZVBB-RV32-NEXT:    sw ra, 76(sp) # 4-byte Folded Spill
+; ZVBB-RV32-NEXT:    sw s0, 72(sp) # 4-byte Folded Spill
+; ZVBB-RV32-NEXT:    addi s0, sp, 80
+; ZVBB-RV32-NEXT:    csrr a0, vlenb
+; ZVBB-RV32-NEXT:    slli a0, a0, 5
+; ZVBB-RV32-NEXT:    sub sp, sp, a0
+; ZVBB-RV32-NEXT:    andi sp, sp, -64
+; ZVBB-RV32-NEXT:    vsetvli a0, zero, e16, m1, ta, ma
+; ZVBB-RV32-NEXT:    vmv2r.v v26, v20
+; ZVBB-RV32-NEXT:    addi a0, sp, 64
+; ZVBB-RV32-NEXT:    vmv2r.v v24, v16
+; ZVBB-RV32-NEXT:    csrr a1, vlenb
+; ZVBB-RV32-NEXT:    slli a2, a1, 3
+; ZVBB-RV32-NEXT:    sub a1, a2, a1
+; ZVBB-RV32-NEXT:    add a1, sp, a1
+; ZVBB-RV32-NEXT:    addi a1, a1, 64
+; ZVBB-RV32-NEXT:    vmv2r.v v22, v12
+; ZVBB-RV32-NEXT:    csrr a2, vlenb
+; ZVBB-RV32-NEXT:    vmv2r.v v20, v8
+; ZVBB-RV32-NEXT:    vmv1r.v v1, v20
+; ZVBB-RV32-NEXT:    vmv1r.v v3, v22
+; ZVBB-RV32-NEXT:    vmv1r.v v5, v24
+; ZVBB-RV32-NEXT:    vmv1r.v v7, v26
+; ZVBB-RV32-NEXT:    add a3, a0, a2
+; ZVBB-RV32-NEXT:    vmv1r.v v2, v10
+; ZVBB-RV32-NEXT:    add a4, a1, a2
+; ZVBB-RV32-NEXT:    slli a5, a2, 2
+; ZVBB-RV32-NEXT:    vmv1r.v v4, v14
+; ZVBB-RV32-NEXT:    slli a6, a2, 4
+; ZVBB-RV32-NEXT:    add a7, a4, a2
+; ZVBB-RV32-NEXT:    vmv1r.v v6, v18
+; ZVBB-RV32-NEXT:    sub a5, a6, a5
+; ZVBB-RV32-NEXT:    vmv1r.v v22, v11
+; ZVBB-RV32-NEXT:    add a6, a7, a2
+; ZVBB-RV32-NEXT:    vmv1r.v v24, v15
+; ZVBB-RV32-NEXT:    vsseg7e16.v v1, (a0)
+; ZVBB-RV32-NEXT:    vmv1r.v v26, v19
+; ZVBB-RV32-NEXT:    vsseg7e16.v v21, (a1)
+; ZVBB-RV32-NEXT:    vl1re16.v v10, (a6)
+; ZVBB-RV32-NEXT:    add a6, a6, a2
+; ZVBB-RV32-NEXT:    vl1re16.v v11, (a6)
+; ZVBB-RV32-NEXT:    add a6, a6, a2
+; ZVBB-RV32-NEXT:    vl1re16.v v12, (a6)
+; ZVBB-RV32-NEXT:    add a6, a6, a2
+; ZVBB-RV32-NEXT:    vl1re16.v v13, (a6)
+; ZVBB-RV32-NEXT:    add a6, a3, a2
+; ZVBB-RV32-NEXT:    vl1re16.v v18, (a6)
+; ZVBB-RV32-NEXT:    add a6, a6, a2
+; ZVBB-RV32-NEXT:    vl1re16.v v19, (a6)
+; ZVBB-RV32-NEXT:    vl1re16.v v16, (a0)
+; ZVBB-RV32-NEXT:    vl1re16.v v8, (a4)
+; ZVBB-RV32-NEXT:    vl1re16.v v17, (a3)
+; ZVBB-RV32-NEXT:    vl1re16.v v9, (a7)
+; ZVBB-RV32-NEXT:    csrr a0, vlenb
+; ZVBB-RV32-NEXT:    li a3, 14
+; ZVBB-RV32-NEXT:    mul a0, a0, a3
+; ZVBB-RV32-NEXT:    add a0, sp, a0
+; ZVBB-RV32-NEXT:    addi a0, a0, 64
+; ZVBB-RV32-NEXT:    add a6, a6, a2
+; ZVBB-RV32-NEXT:    vl1re16.v v20, (a6)
+; ZVBB-RV32-NEXT:    add a6, a6, a2
+; ZVBB-RV32-NEXT:    vl1re16.v v21, (a6)
+; ZVBB-RV32-NEXT:    add a6, a6, a2
+; ZVBB-RV32-NEXT:    slli a2, a2, 3
+; ZVBB-RV32-NEXT:    add a2, a0, a2
+; ZVBB-RV32-NEXT:    vl1re16.v v22, (a6)
+; ZVBB-RV32-NEXT:    vl1re16.v v23, (a1)
+; ZVBB-RV32-NEXT:    add a5, a0, a5
+; ZVBB-RV32-NEXT:    vs2r.v v12, (a5)
+; ZVBB-RV32-NEXT:    vs4r.v v8, (a2)
+; ZVBB-RV32-NEXT:    vs8r.v v16, (a0)
+; ZVBB-RV32-NEXT:    vl8re16.v v16, (a2)
+; ZVBB-RV32-NEXT:    vl8re16.v v8, (a0)
+; ZVBB-RV32-NEXT:    addi sp, s0, -80
+; ZVBB-RV32-NEXT:    lw ra, 76(sp) # 4-byte Folded Reload
+; ZVBB-RV32-NEXT:    lw s0, 72(sp) # 4-byte Folded Reload
+; ZVBB-RV32-NEXT:    addi sp, sp, 80
+; ZVBB-RV32-NEXT:    ret
+;
+; ZVBB-RV64-LABEL: vector_interleave_nxv56i16_nxv8i16:
+; ZVBB-RV64:       # %bb.0:
+; ZVBB-RV64-NEXT:    addi sp, sp, -80
+; ZVBB-RV64-NEXT:    sd ra, 72(sp) # 8-byte Folded Spill
+; ZVBB-RV64-NEXT:    sd s0, 64(sp) # 8-byte Folded Spill
+; ZVBB-RV64-NEXT:    addi s0, sp, 80
+; ZVBB-RV64-NEXT:    csrr a0, vlenb
+; ZVBB-RV64-NEXT:    slli a0, a0, 5
+; ZVBB-RV64-NEXT:    sub sp, sp, a0
+; ZVBB-RV64-NEXT:    andi sp, sp, -64
+; ZVBB-RV64-NEXT:    vsetvli a0, zero, e16, m1, ta, ma
+; ZVBB-RV64-NEXT:    vmv2r.v v26, v20
+; ZVBB-RV64-NEXT:    addi a0, sp, 64
+; ZVBB-RV64-NEXT:    vmv2r.v v24, v16
+; ZVBB-RV64-NEXT:    csrr a1, vlenb
+; ZVBB-RV64-NEXT:    slli a2, a1, 3
+; ZVBB-RV64-NEXT:    sub a1, a2, a1
+; ZVBB-RV64-NEXT:    add a1, sp, a1
+; ZVBB-RV64-NEXT:    addi a1, a1, 64
+; ZVBB-RV64-NEXT:    vmv2r.v v22, v12
+; ZVBB-RV64-NEXT:    csrr a2, vlenb
+; ZVBB-RV64-NEXT:    vmv2r.v v20, v8
+; ZVBB-RV64-NEXT:    vmv1r.v v1, v20
+; ZVBB-RV64-NEXT:    vmv1r.v v3, v22
+; ZVBB-RV64-NEXT:    vmv1r.v v5, v24
+; ZVBB-RV64-NEXT:    vmv1r.v v7, v26
+; ZVBB-RV64-NEXT:    add a3, a0, a2
+; ZVBB-RV64-NEXT:    vmv1r.v v2, v10
+; ZVBB-RV64-NEXT:    add a4, a1, a2
+; ZVBB-RV64-NEXT:    slli a5, a2, 2
+; ZVBB-RV64-NEXT:    vmv1r.v v4, v14
+; ZVBB-RV64-NEXT:    slli a6, a2, 4
+; ZVBB-RV64-NEXT:    add a7, a4, a2
+; ZVBB-RV64-NEXT:    vmv1r.v v6, v18
+; ZVBB-RV64-NEXT:    sub a5, a6, a5
+; ZVBB-RV64-NEXT:    vmv1r.v v22, v11
+; ZVBB-RV64-NEXT:    add a6, a7, a2
+; ZVBB-RV64-NEXT:    vmv1r.v v24, v15
+; ZVBB-RV64-NEXT:    vsseg7e16.v v1, (a0)
+; ZVBB-RV64-NEXT:    vmv1r.v v26, v19
+; ZVBB-RV64-NEXT:    vsseg7e16.v v21, (a1)
+; ZVBB-RV64-NEXT:    vl1re16.v v10, (a6)
+; ZVBB-RV64-NEXT:    add a6, a6, a2
+; ZVBB-RV64-NEXT:    vl1re16.v v11, (a6)
+; ZVBB-RV64-NEXT:    add a6, a6, a2
+; ZVBB-RV64-NEXT:    vl1re16.v v12, (a6)
+; ZVBB-RV64-NEXT:    add a6, a6, a2
+; ZVBB-RV64-NEXT:    vl1re16.v v13, (a6)
+; ZVBB-RV64-NEXT:    add a6, a3, a2
+; ZVBB-RV64-NEXT:    vl1re16.v v18, (a6)
+; ZVBB-RV64-NEXT:    add a6, a6, a2
+; ZVBB-RV64-NEXT:    vl1re16.v v19, (a6)
+; ZVBB-RV64-NEXT:    vl1re16.v v16, (a0)
+; ZVBB-RV64-NEXT:    vl1re16.v v8, (a4)
+; ZVBB-RV64-NEXT:    vl1re16.v v17, (a3)
+; ZVBB-RV64-NEXT:    vl1re16.v v9, (a7)
+; ZVBB-RV64-NEXT:    csrr a0, vlenb
+; ZVBB-RV64-NEXT:    li a3, 14
+; ZVBB-RV64-NEXT:    mul a0, a0, a3
+; ZVBB-RV64-NEXT:    add a0, sp, a0
+; ZVBB-RV64-NEXT:    addi a0, a0, 64
+; ZVBB-RV64-NEXT:    add a6, a6, a2
+; ZVBB-RV64-NEXT:    vl1re16.v v20, (a6)
+; ZVBB-RV64-NEXT:    add a6, a6, a2
+; ZVBB-RV64-NEXT:    vl1re16.v v21, (a6)
+; ZVBB-RV64-NEXT:    add a6, a6, a2
+; ZVBB-RV64-NEXT:    slli a2, a2, 3
+; ZVBB-RV64-NEXT:    add a2, a0, a2
+; ZVBB-RV64-NEXT:    vl1re16.v v22, (a6)
+; ZVBB-RV64-NEXT:    vl1re16.v v23, (a1)
+; ZVBB-RV64-NEXT:    add a5, a0, a5
+; ZVBB-RV64-NEXT:    vs2r.v v12, (a5)
+; ZVBB-RV64-NEXT:    vs4r.v v8, (a2)
+; ZVBB-RV64-NEXT:    vs8r.v v16, (a0)
+; ZVBB-RV64-NEXT:    vl8re16.v v16, (a2)
+; ZVBB-RV64-NEXT:    vl8re16.v v8, (a0)
+; ZVBB-RV64-NEXT:    addi sp, s0, -80
+; ZVBB-RV64-NEXT:    ld ra, 72(sp) # 8-byte Folded Reload
+; ZVBB-RV64-NEXT:    ld s0, 64(sp) # 8-byte Folded Reload
+; ZVBB-RV64-NEXT:    addi sp, sp, 80
+; ZVBB-RV64-NEXT:    ret
+  %res = call <vscale x 56 x i16> @llvm.vector.interleave7.nxv56i16(<vscale x 8 x i16> %a, <vscale x 8 x i16> %b, <vscale x 8 x i16> %c, <vscale x 8 x i16> %d, <vscale x 8 x i16> %e, <vscale x 8 x i16> %f, <vscale x 8 x i16> %g)
+  ret <vscale x 56 x i16> %res
+}
+
+
+define <vscale x 28 x i32> @vector_interleave_nxv28i32_nxv4i32(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b, <vscale x 4 x i32> %c, <vscale x 4 x i32> %d, <vscale x 4 x i32> %e, <vscale x 4 x i32> %f, <vscale x 4 x i32> %g) nounwind {
+;
+; RV32-LABEL: vector_interleave_nxv28i32_nxv4i32:
+; RV32:       # %bb.0:
+; RV32-NEXT:    addi sp, sp, -80
+; RV32-NEXT:    sw ra, 76(sp) # 4-byte Folded Spill
+; RV32-NEXT:    sw s0, 72(sp) # 4-byte Folded Spill
+; RV32-NEXT:    addi s0, sp, 80
+; RV32-NEXT:    csrr a0, vlenb
+; RV32-NEXT:    slli a0, a0, 5
+; RV32-NEXT:    sub sp, sp, a0
+; RV32-NEXT:    andi sp, sp, -64
+; RV32-NEXT:    vsetvli a0, zero, e32, m1, ta, ma
+; RV32-NEXT:    vmv2r.v v26, v20
+; RV32-NEXT:    addi a0, sp, 64
+; RV32-NEXT:    vmv2r.v v24, v16
+; RV32-NEXT:    csrr a1, vlenb
+; RV32-NEXT:    slli a2, a1, 3
+; RV32-NEXT:    sub a1, a2, a1
+; RV32-NEXT:    add a1, sp, a1
+; RV32-NEXT:    addi a1, a1, 64
+; RV32-NEXT:    vmv2r.v v22, v12
+; RV32-NEXT:    csrr a2, vlenb
+; RV32-NEXT:    vmv2r.v v20, v8
+; RV32-NEXT:    vmv1r.v v1, v20
+; RV32-NEXT:    vmv1r.v v3, v22
+; RV32-NEXT:    vmv1r.v v5, v24
+; RV32-NEXT:    vmv1r.v v7, v26
+; RV32-NEXT:    add a3, a0, a2
+; RV32-NEXT:    vmv1r.v v2, v10
+; RV32-NEXT:    add a4, a1, a2
+; RV32-NEXT:    slli a5, a2, 2
+; RV32-NEXT:    vmv1r.v v4, v14
+; RV32-NEXT:    slli a6, a2, 4
+; RV32-NEXT:    add a7, a4, a2
+; RV32-NEXT:    vmv1r.v v6, v18
+; RV32-NEXT:    sub a5, a6, a5
+; RV32-NEXT:    vmv1r.v v22, v11
+; RV32-NEXT:    add a6, a7, a2
+; RV32-NEXT:    vmv1r.v v24, v15
+; RV32-NEXT:    vsseg7e32.v v1, (a0)
+; RV32-NEXT:    vmv1r.v v26, v19
+; RV32-NEXT:    vsseg7e32.v v21, (a1)
+; RV32-NEXT:    vl1re32.v v10, (a6)
+; RV32-NEXT:    add a6, a6, a2
+; RV32-NEXT:    vl1re32.v v11, (a6)
+; RV32-NEXT:    add a6, a6, a2
+; RV32-NEXT:    vl1re32.v v12, (a6)
+; RV32-NEXT:    add a6, a6, a2
+; RV32-NEXT:    vl1re32.v v13, (a6)
+; RV32-NEXT:    add a6, a3, a2
+; RV32-NEXT:    vl1re32.v v18, (a6)
+; RV32-NEXT:    add a6, a6, a2
+; RV32-NEXT:    vl1re32.v v19, (a6)
+; RV32-NEXT:    vl1re32.v v16, (a0)
+; RV32-NEXT:    vl1re32.v v8, (a4)
+; RV32-NEXT:    vl1re32.v v17, (a3)
+; RV32-NEXT:    vl1re32.v v9, (a7)
+; RV32-NEXT:    csrr a0, vlenb
+; RV32-NEXT:    li a3, 14
+; RV32-NEXT:    mul a0, a0, a3
+; RV32-NEXT:    add a0, sp, a0
+; RV32-NEXT:    addi a0, a0, 64
+; RV32-NEXT:    add a6, a6, a2
+; RV32-NEXT:    vl1re32.v v20, (a6)
+; RV32-NEXT:    add a6, a6, a2
+; RV32-NEXT:    vl1re32.v v21, (a6)
+; RV32-NEXT:    add a6, a6, a2
+; RV32-NEXT:    slli a2, a2, 3
+; RV32-NEXT:    add a2, a0, a2
+; RV32-NEXT:    vl1re32.v v22, (a6)
+; RV32-NEXT:    vl1re32.v v23, (a1)
+; RV32-NEXT:    add a5, a0, a5
+; RV32-NEXT:    vs2r.v v12, (a5)
+; RV32-NEXT:    vs4r.v v8, (a2)
+; RV32-NEXT:    vs8r.v v16, (a0)
+; RV32-NEXT:    vl8re32.v v16, (a2)
+; RV32-NEXT:    vl8re32.v v8, (a0)
+; RV32-NEXT:    addi sp, s0, -80
+; RV32-NEXT:    lw ra, 76(sp) # 4-byte Folded Reload
+; RV32-NEXT:    lw s0, 72(sp) # 4-byte Folded Reload
+; RV32-NEXT:    addi sp, sp, 80
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: vector_interleave_nxv28i32_nxv4i32:
+; RV64:       # %bb.0:
+; RV64-NEXT:    addi sp, sp, -80
+; RV64-NEXT:    sd ra, 72(sp) # 8-byte Folded Spill
+; RV64-NEXT:    sd s0, 64(sp) # 8-byte Folded Spill
+; RV64-NEXT:    addi s0, sp, 80
+; RV64-NEXT:    csrr a0, vlenb
+; RV64-NEXT:    slli a0, a0, 5
+; RV64-NEXT:    sub sp, sp, a0
+; RV64-NEXT:    andi sp, sp, -64
+; RV64-NEXT:    vsetvli a0, zero, e32, m1, ta, ma
+; RV64-NEXT:    vmv2r.v v26, v20
+; RV64-NEXT:    addi a0, sp, 64
+; RV64-NEXT:    vmv2r.v v24, v16
+; RV64-NEXT:    csrr a1, vlenb
+; RV64-NEXT:    slli a2, a1, 3
+; RV64-NEXT:    sub a1, a2, a1
+; RV64-NEXT:    add a1, sp, a1
+; RV64-NEXT:    addi a1, a1, 64
+; RV64-NEXT:    vmv2r.v v22, v12
+; RV64-NEXT:    csrr a2, vlenb
+; RV64-NEXT:    vmv2r.v v20, v8
+; RV64-NEXT:    vmv1r.v v1, v20
+; RV64-NEXT:    vmv1r.v v3, v22
+; RV64-NEXT:    vmv1r.v v5, v24
+; RV64-NEXT:    vmv1r.v v7, v26
+; RV64-NEXT:    add a3, a0, a2
+; RV64-NEXT:    vmv1r.v v2, v10
+; RV64-NEXT:    add a4, a1, a2
+; RV64-NEXT:    slli a5, a2, 2
+; RV64-NEXT:    vmv1r.v v4, v14
+; RV64-NEXT:    slli a6, a2, 4
+; RV64-NEXT:    add a7, a4, a2
+; RV64-NEXT:    vmv1r.v v6, v18
+; RV64-NEXT:    sub a5, a6, a5
+; RV64-NEXT:    vmv1r.v v22, v11
+; RV64-NEXT:    add a6, a7, a2
+; RV64-NEXT:    vmv1r.v v24, v15
+; RV64-NEXT:    vsseg7e32.v v1, (a0)
+; RV64-NEXT:    vmv1r.v v26, v19
+; RV64-NEXT:    vsseg7e32.v v21, (a1)
+; RV64-NEXT:    vl1re32.v v10, (a6)
+; RV64-NEXT:    add a6, a6, a2
+; RV64-NEXT:    vl1re32.v v11, (a6)
+; RV64-NEXT:    add a6, a6, a2
+; RV64-NEXT:    vl1re32.v v12, (a6)
+; RV64-NEXT:    add a6, a6, a2
+; RV64-NEXT:    vl1re32.v v13, (a6)
+; RV64-NEXT:    add a6, a3, a2
+; RV64-NEXT:    vl1re32.v v18, (a6)
+; RV64-NEXT:    add a6, a6, a2
+; RV64-NEXT:    vl1re32.v v19, (a6)
+; RV64-NEXT:    vl1re32.v v16, (a0)
+; RV64-NEXT:    vl1re32.v v8, (a4)
+; RV64-NEXT:    vl1re32.v v17, (a3)
+; RV64-NEXT:    vl1re32.v v9, (a7)
+; RV64-NEXT:    csrr a0, vlenb
+; RV64-NEXT:    li a3, 14
+; RV64-NEXT:    mul a0, a0, a3
+; RV64-NEXT:    add a0, sp, a0
+; RV64-NEXT:    addi a0, a0, 64
+; RV64-NEXT:    add a6, a6, a2
+; RV64-NEXT:    vl1re32.v v20, (a6)
+; RV64-NEXT:    add a6, a6, a2
+; RV64-NEXT:    vl1re32.v v21, (a6)
+; RV64-NEXT:    add a6, a6, a2
+; RV64-NEXT:    slli a2, a2, 3
+; RV64-NEXT:    add a2, a0, a2
+; RV64-NEXT:    vl1re32.v v22, (a6)
+; RV64-NEXT:    vl1re32.v v23, (a1)
+; RV64-NEXT:    add a5, a0, a5
+; RV64-NEXT:    vs2r.v v12, (a5)
+; RV64-NEXT:    vs4r.v v8, (a2)
+; RV64-NEXT:    vs8r.v v16, (a0)
+; RV64-NEXT:    vl8re32.v v16, (a2)
+; RV64-NEXT:    vl8re32.v v8, (a0)
+; RV64-NEXT:    addi sp, s0, -80
+; RV64-NEXT:    ld ra, 72(sp) # 8-byte Folded Reload
+; RV64-NEXT:    ld s0, 64(sp) # 8-byte Folded Reload
+; RV64-NEXT:    addi sp, sp, 80
+; RV64-NEXT:    ret
+;
+; ZVBB-RV32-LABEL: vector_interleave_nxv28i32_nxv4i32:
+; ZVBB-RV32:       # %bb.0:
+; ZVBB-RV32-NEXT:    addi sp, sp, -80
+; ZVBB-RV32-NEXT:    sw ra, 76(sp) # 4-byte Folded Spill
+; ZVBB-RV32-NEXT:    sw s0, 72(sp) # 4-byte Folded Spill
+; ZVBB-RV32-NEXT:    addi s0, sp, 80
+; ZVBB-RV32-NEXT:    csrr a0, vlenb
+; ZVBB-RV32-NEXT:    slli a0, a0, 5
+; ZVBB-RV32-NEXT:    sub sp, sp, a0
+; ZVBB-RV32-NEXT:    andi sp, sp, -64
+; ZVBB-RV32-NEXT:    vsetvli a0, zero, e32, m1, ta, ma
+; ZVBB-RV32-NEXT:    vmv2r.v v26, v20
+; ZVBB-RV32-NEXT:    addi a0, sp, 64
+; ZVBB-RV32-NEXT:    vmv2r.v v24, v16
+; ZVBB-RV32-NEXT:    csrr a1, vlenb
+; ZVBB-RV32-NEXT:    slli a2, a1, 3
+; ZVBB-RV32-NEXT:    sub a1, a2, a1
+; ZVBB-RV32-NEXT:    add a1, sp, a1
+; ZVBB-RV32-NEXT:    addi a1, a1, 64
+; ZVBB-RV32-NEXT:    vmv2r.v v22, v12
+; ZVBB-RV32-NEXT:    csrr a2, vlenb
+; ZVBB-RV32-NEXT:    vmv2r.v v20, v8
+; ZVBB-RV32-NEXT:    vmv1r.v v1, v20
+; ZVBB-RV32-NEXT:    vmv1r.v v3, v22
+; ZVBB-RV32-NEXT:    vmv1r.v v5, v24
+; ZVBB-RV32-NEXT:    vmv1r.v v7, v26
+; ZVBB-RV32-NEXT:    add a3, a0, a2
+; ZVBB-RV32-NEXT:    vmv1r.v v2, v10
+; ZVBB-RV32-NEXT:    add a4, a1, a2
+; ZVBB-RV32-NEXT:    slli a5, a2, 2
+; ZVBB-RV32-NEXT:    vmv1r.v v4, v14
+; ZVBB-RV32-NEXT:    slli a6, a2, 4
+; ZVBB-RV32-NEXT:    add a7, a4, a2
+; ZVBB-RV32-NEXT:    vmv1r.v v6, v18
+; ZVBB-RV32-NEXT:    sub a5, a6, a5
+; ZVBB-RV32-NEXT:    vmv1r.v v22, v11
+; ZVBB-RV32-NEXT:    add a6, a7, a2
+; ZVBB-RV32-NEXT:    vmv1r.v v24, v15
+; ZVBB-RV32-NEXT:    vsseg7e32.v v1, (a0)
+; ZVBB-RV32-NEXT:    vmv1r.v v26, v19
+; ZVBB-RV32-NEXT:    vsseg7e32.v v21, (a1)
+; ZVBB-RV32-NEXT:    vl1re32.v v10, (a6)
+; ZVBB-RV32-NEXT:    add a6, a6, a2
+; ZVBB-RV32-NEXT:    vl1re32.v v11, (a6)
+; ZVBB-RV32-NEXT:    add a6, a6, a2
+; ZVBB-RV32-NEXT:    vl1re32.v v12, (a6)
+; ZVBB-RV32-NEXT:    add a6, a6, a2
+; ZVBB-RV32-NEXT:    vl1re32.v v13, (a6)
+; ZVBB-RV32-NEXT:    add a6, a3, a2
+; ZVBB-RV32-NEXT:    vl1re32.v v18, (a6)
+; ZVBB-RV32-NEXT:    add a6, a6, a2
+; ZVBB-RV32-NEXT:    vl1re32.v v19, (a6)
+; ZVBB-RV32-NEXT:    vl1re32.v v16, (a0)
+; ZVBB-RV32-NEXT:    vl1re32.v v8, (a4)
+; ZVBB-RV32-NEXT:    vl1re32.v v17, (a3)
+; ZVBB-RV32-NEXT:    vl1re32.v v9, (a7)
+; ZVBB-RV32-NEXT:    csrr a0, vlenb
+; ZVBB-RV32-NEXT:    li a3, 14
+; ZVBB-RV32-NEXT:    mul a0, a0, a3
+; ZVBB-RV32-NEXT:    add a0, sp, a0
+; ZVBB-RV32-NEXT:    addi a0, a0, 64
+; ZVBB-RV32-NEXT:    add a6, a6, a2
+; ZVBB-RV32-NEXT:    vl1re32.v v20, (a6)
+; ZVBB-RV32-NEXT:    add a6, a6, a2
+; ZVBB-RV32-NEXT:    vl1re32.v v21, (a6)
+; ZVBB-RV32-NEXT:    add a6, a6, a2
+; ZVBB-RV32-NEXT:    slli a2, a2, 3
+; ZVBB-RV32-NEXT:    add a2, a0, a2
+; ZVBB-RV32-NEXT:    vl1re32.v v22, (a6)
+; ZVBB-RV32-NEXT:    vl1re32.v v23, (a1)
+; ZVBB-RV32-NEXT:    add a5, a0, a5
+; ZVBB-RV32-NEXT:    vs2r.v v12, (a5)
+; ZVBB-RV32-NEXT:    vs4r.v v8, (a2)
+; ZVBB-RV32-NEXT:    vs8r.v v16, (a0)
+; ZVBB-RV32-NEXT:    vl8re32.v v16, (a2)
+; ZVBB-RV32-NEXT:    vl8re32.v v8, (a0)
+; ZVBB-RV32-NEXT:    addi sp, s0, -80
+; ZVBB-RV32-NEXT:    lw ra, 76(sp) # 4-byte Folded Reload
+; ZVBB-RV32-NEXT:    lw s0, 72(sp) # 4-byte Folded Reload
+; ZVBB-RV32-NEXT:    addi sp, sp, 80
+; ZVBB-RV32-NEXT:    ret
+;
+; ZVBB-RV64-LABEL: vector_interleave_nxv28i32_nxv4i32:
+; ZVBB-RV64:       # %bb.0:
+; ZVBB-RV64-NEXT:    addi sp, sp, -80
+; ZVBB-RV64-NEXT:    sd ra, 72(sp) # 8-byte Folded Spill
+; ZVBB-RV64-NEXT:    sd s0, 64(sp) # 8-byte Folded Spill
+; ZVBB-RV64-NEXT:    addi s0, sp, 80
+; ZVBB-RV64-NEXT:    csrr a0, vlenb
+; ZVBB-RV64-NEXT:    slli a0, a0, 5
+; ZVBB-RV64-NEXT:    sub sp, sp, a0
+; ZVBB-RV64-NEXT:    andi sp, sp, -64
+; ZVBB-RV64-NEXT:    vsetvli a0, zero, e32, m1, ta, ma
+; ZVBB-RV64-NEXT:    vmv2r.v v26, v20
+; ZVBB-RV64-NEXT:    addi a0, sp, 64
+; ZVBB-RV64-NEXT:    vmv2r.v v24, v16
+; ZVBB-RV64-NEXT:    csrr a1, vlenb
+; ZVBB-RV64-NEXT:    slli a2, a1, 3
+; ZVBB-RV64-NEXT:    sub a1, a2, a1
+; ZVBB-RV64-NEXT:    add a1, sp, a1
+; ZVBB-RV64-NEXT:    addi a1, a1, 64
+; ZVBB-RV64-NEXT:    vmv2r.v v22, v12
+; ZVBB-RV64-NEXT:    csrr a2, vlenb
+; ZVBB-RV64-NEXT:    vmv2r.v v20, v8
+; ZVBB-RV64-NEXT:    vmv1r.v v1, v20
+; ZVBB-RV64-NEXT:    vmv1r.v v3, v22
+; ZVBB-RV64-NEXT:    vmv1r.v v5, v24
+; ZVBB-RV64-NEXT:    vmv1r.v v7, v26
+; ZVBB-RV64-NEXT:    add a3, a0, a2
+; ZVBB-RV64-NEXT:    vmv1r.v v2, v10
+; ZVBB-RV64-NEXT:    add a4, a1, a2
+; ZVBB-RV64-NEXT:    slli a5, a2, 2
+; ZVBB-RV64-NEXT:    vmv1r.v v4, v14
+; ZVBB-RV64-NEXT:    slli a6, a2, 4
+; ZVBB-RV64-NEXT:    add a7, a4, a2
+; ZVBB-RV64-NEXT:    vmv1r.v v6, v18
+; ZVBB-RV64-NEXT:    sub a5, a6, a5
+; ZVBB-RV64-NEXT:    vmv1r.v v22, v11
+; ZVBB-RV64-NEXT:    add a6, a7, a2
+; ZVBB-RV64-NEXT:    vmv1r.v v24, v15
+; ZVBB-RV64-NEXT:    vsseg7e32.v v1, (a0)
+; ZVBB-RV64-NEXT:    vmv1r.v v26, v19
+; ZVBB-RV64-NEXT:    vsseg7e32.v v21, (a1)
+; ZVBB-RV64-NEXT:    vl1re32.v v10, (a6)
+; ZVBB-RV64-NEXT:    add a6, a6, a2
+; ZVBB-RV64-NEXT:    vl1re32.v v11, (a6)
+; ZVBB-RV64-NEXT:    add a6, a6, a2
+; ZVBB-RV64-NEXT:    vl1re32.v v12, (a6)
+; ZVBB-RV64-NEXT:    add a6, a6, a2
+; ZVBB-RV64-NEXT:    vl1re32.v v13, (a6)
+; ZVBB-RV64-NEXT:    add a6, a3, a2
+; ZVBB-RV64-NEXT:    vl1re32.v v18, (a6)
+; ZVBB-RV64-NEXT:    add a6, a6, a2
+; ZVBB-RV64-NEXT:    vl1re32.v v19, (a6)
+; ZVBB-RV64-NEXT:    vl1re32.v v16, (a0)
+; ZVBB-RV64-NEXT:    vl1re32.v v8, (a4)
+; ZVBB-RV64-NEXT:    vl1re32.v v17, (a3)
+; ZVBB-RV64-NEXT:    vl1re32.v v9, (a7)
+; ZVBB-RV64-NEXT:    csrr a0, vlenb
+; ZVBB-RV64-NEXT:    li a3, 14
+; ZVBB-RV64-NEXT:    mul a0, a0, a3
+; ZVBB-RV64-NEXT:    add a0, sp, a0
+; ZVBB-RV64-NEXT:    addi a0, a0, 64
+; ZVBB-RV64-NEXT:    add a6, a6, a2
+; ZVBB-RV64-NEXT:    vl1re32.v v20, (a6)
+; ZVBB-RV64-NEXT:    add a6, a6, a2
+; ZVBB-RV64-NEXT:    vl1re32.v v21, (a6)
+; ZVBB-RV64-NEXT:    add a6, a6, a2
+; ZVBB-RV64-NEXT:    slli a2, a2, 3
+; ZVBB-RV64-NEXT:    add a2, a0, a2
+; ZVBB-RV64-NEXT:    vl1re32.v v22, (a6)
+; ZVBB-RV64-NEXT:    vl1re32.v v23, (a1)
+; ZVBB-RV64-NEXT:    add a5, a0, a5
+; ZVBB-RV64-NEXT:    vs2r.v v12, (a5)
+; ZVBB-RV64-NEXT:    vs4r.v v8, (a2)
+; ZVBB-RV64-NEXT:    vs8r.v v16, (a0)
+; ZVBB-RV64-NEXT:    vl8re32.v v16, (a2)
+; ZVBB-RV64-NEXT:    vl8re32.v v8, (a0)
+; ZVBB-RV64-NEXT:    addi sp, s0, -80
+; ZVBB-RV64-NEXT:    ld ra, 72(sp) # 8-byte Folded Reload
+; ZVBB-RV64-NEXT:    ld s0, 64(sp) # 8-byte Folded Reload
+; ZVBB-RV64-NEXT:    addi sp, sp, 80
+; ZVBB-RV64-NEXT:    ret
+  %res = call <vscale x 28 x i32> @llvm.vector.interleave7.nxv28i32(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b, <vscale x 4 x i32> %c, <vscale x 4 x i32> %d, <vscale x 4 x i32> %e, <vscale x 4 x i32> %f, <vscale x 4 x i32> %g)
+  ret <vscale x 28 x i32> %res
+}
+
+define <vscale x 14 x i64> @vector_interleave_nxv14i64_nxv2i64(<vscale x 2 x i64> %a, <vscale x 2 x i64> %b, <vscale x 2 x i64> %c, <vscale x 2 x i64> %d, <vscale x 2 x i64> %e, <vscale x 2 x i64> %f, <vscale x 2 x i64> %g) nounwind {
+;
+; RV32-LABEL: vector_interleave_nxv14i64_nxv2i64:
+; RV32:       # %bb.0:
+; RV32-NEXT:    addi sp, sp, -80
+; RV32-NEXT:    sw ra, 76(sp) # 4-byte Folded Spill
+; RV32-NEXT:    sw s0, 72(sp) # 4-byte Folded Spill
+; RV32-NEXT:    addi s0, sp, 80
+; RV32-NEXT:    csrr a0, vlenb
+; RV32-NEXT:    slli a0, a0, 5
+; RV32-NEXT:    sub sp, sp, a0
+; RV32-NEXT:    andi sp, sp, -64
+; RV32-NEXT:    vsetvli a0, zero, e64, m1, ta, ma
+; RV32-NEXT:    vmv2r.v v26, v20
+; RV32-NEXT:    addi a0, sp, 64
+; RV32-NEXT:    vmv2r.v v24, v16
+; RV32-NEXT:    csrr a1, vlenb
+; RV32-NEXT:    slli a2, a1, 3
+; RV32-NEXT:    sub a1, a2, a1
+; RV32-NEXT:    add a1, sp, a1
+; RV32-NEXT:    addi a1, a1, 64
+; RV32-NEXT:    vmv2r.v v22, v12
+; RV32-NEXT:    csrr a2, vlenb
+; RV32-NEXT:    vmv2r.v v20, v8
+; RV32-NEXT:    vmv1r.v v1, v20
+; RV32-NEXT:    vmv1r.v v3, v22
+; RV32-NEXT:    vmv1r.v v5, v24
+; RV32-NEXT:    vmv1r.v v7, v26
+; RV32-NEXT:    add a3, a0, a2
+; RV32-NEXT:    vmv1r.v v2, v10
+; RV32-NEXT:    add a4, a1, a2
+; RV32-NEXT:    slli a5, a2, 2
+; RV32-NEXT:    vmv1r.v v4, v14
+; RV32-NEXT:    slli a6, a2, 4
+; RV32-NEXT:    add a7, a4, a2
+; RV32-NEXT:    vmv1r.v v6, v18
+; RV32-NEXT:    sub a5, a6, a5
+; RV32-NEXT:    vmv1r.v v22, v11
+; RV32-NEXT:    add a6, a7, a2
+; RV32-NEXT:    vmv1r.v v24, v15
+; RV32-NEXT:    vsseg7e64.v v1, (a0)
+; RV32-NEXT:    vmv1r.v v26, v19
+; RV32-NEXT:    vsseg7e64.v v21, (a1)
+; RV32-NEXT:    vl1re64.v v10, (a6)
+; RV32-NEXT:    add a6, a6, a2
+; RV32-NEXT:    vl1re64.v v11, (a6)
+; RV32-NEXT:    add a6, a6, a2
+; RV32-NEXT:    vl1re64.v v12, (a6)
+; RV32-NEXT:    add a6, a6, a2
+; RV32-NEXT:    vl1re64.v v13, (a6)
+; RV32-NEXT:    add a6, a3, a2
+; RV32-NEXT:    vl1re64.v v18, (a6)
+; RV32-NEXT:    add a6, a6, a2
+; RV32-NEXT:    vl1re64.v v19, (a6)
+; RV32-NEXT:    vl1re64.v v16, (a0)
+; RV32-NEXT:    vl1re64.v v8, (a4)
+; RV32-NEXT:    vl1re64.v v17, (a3)
+; RV32-NEXT:    vl1re64.v v9, (a7)
+; RV32-NEXT:    csrr a0, vlenb
+; RV32-NEXT:    li a3, 14
+; RV32-NEXT:    mul a0, a0, a3
+; RV32-NEXT:    add a0, sp, a0
+; RV32-NEXT:    addi a0, a0, 64
+; RV32-NEXT:    add a6, a6, a2
+; RV32-NEXT:    vl1re64.v v20, (a6)
+; RV32-NEXT:    add a6, a6, a2
+; RV32-NEXT:    vl1re64.v v21, (a6)
+; RV32-NEXT:    add a6, a6, a2
+; RV32-NEXT:    slli a2, a2, 3
+; RV32-NEXT:    add a2, a0, a2
+; RV32-NEXT:    vl1re64.v v22, (a6)
+; RV32-NEXT:    vl1re64.v v23, (a1)
+; RV32-NEXT:    add a5, a0, a5
+; RV32-NEXT:    vs2r.v v12, (a5)
+; RV32-NEXT:    vs4r.v v8, (a2)
+; RV32-NEXT:    vs8r.v v16, (a0)
+; RV32-NEXT:    vl8re64.v v16, (a2)
+; RV32-NEXT:    vl8re64.v v8, (a0)
+; RV32-NEXT:    addi sp, s0, -80
+; RV32-NEXT:    lw ra, 76(sp) # 4-byte Folded Reload
+; RV32-NEXT:    lw s0, 72(sp) # 4-byte Folded Reload
+; RV32-NEXT:    addi sp, sp, 80
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: vector_interleave_nxv14i64_nxv2i64:
+; RV64:       # %bb.0:
+; RV64-NEXT:    addi sp, sp, -80
+; RV64-NEXT:    sd ra, 72(sp) # 8-byte Folded Spill
+; RV64-NEXT:    sd s0, 64(sp) # 8-byte Folded Spill
+; RV64-NEXT:    addi s0, sp, 80
+; RV64-NEXT:    csrr a0, vlenb
+; RV64-NEXT:    slli a0, a0, 5
+; RV64-NEXT:    sub sp, sp, a0
+; RV64-NEXT:    andi sp, sp, -64
+; RV64-NEXT:    vsetvli a0, zero, e64, m1, ta, ma
+; RV64-NEXT:    vmv2r.v v26, v20
+; RV64-NEXT:    addi a0, sp, 64
+; RV64-NEXT:    vmv2r.v v24, v16
+; RV64-NEXT:    csrr a1, vlenb
+; RV64-NEXT:    slli a2, a1, 3
+; RV64-NEXT:    sub a1, a2, a1
+; RV64-NEXT:    add a1, sp, a1
+; RV64-NEXT:    addi a1, a1, 64
+; RV64-NEXT:    vmv2r.v v22, v12
+; RV64-NEXT:    csrr a2, vlenb
+; RV64-NEXT:    vmv2r.v v20, v8
+; RV64-NEXT:    vmv1r.v v1, v20
+; RV64-NEXT:    vmv1r.v v3, v22
+; RV64-NEXT:    vmv1r.v v5, v24
+; RV64-NEXT:    vmv1r.v v7, v26
+; RV64-NEXT:    add a3, a0, a2
+; RV64-NEXT:    vmv1r.v v2, v10
+; RV64-NEXT:    add a4, a1, a2
+; RV64-NEXT:    slli a5, a2, 2
+; RV64-NEXT:    vmv1r.v v4, v14
+; RV64-NEXT:    slli a6, a2, 4
+; RV64-NEXT:    add a7, a4, a2
+; RV64-NEXT:    vmv1r.v v6, v18
+; RV64-NEXT:    sub a5, a6, a5
+; RV64-NEXT:    vmv1r.v v22, v11
+; RV64-NEXT:    add a6, a7, a2
+; RV64-NEXT:    vmv1r.v v24, v15
+; RV64-NEXT:    vsseg7e64.v v1, (a0)
+; RV64-NEXT:    vmv1r.v v26, v19
+; RV64-NEXT:    vsseg7e64.v v21, (a1)
+; RV64-NEXT:    vl1re64.v v10, (a6)
+; RV64-NEXT:    add a6, a6, a2
+; RV64-NEXT:    vl1re64.v v11, (a6)
+; RV64-NEXT:    add a6, a6, a2
+; RV64-NEXT:    vl1re64.v v12, (a6)
+; RV64-NEXT:    add a6, a6, a2
+; RV64-NEXT:    vl1re64.v v13, (a6)
+; RV64-NEXT:    add a6, a3, a2
+; RV64-NEXT:    vl1re64.v v18, (a6)
+; RV64-NEXT:    add a6, a6, a2
+; RV64-NEXT:    vl1re64.v v19, (a6)
+; RV64-NEXT:    vl1re64.v v16, (a0)
+; RV64-NEXT:    vl1re64.v v8, (a4)
+; RV64-NEXT:    vl1re64.v v17, (a3)
+; RV64-NEXT:    vl1re64.v v9, (a7)
+; RV64-NEXT:    csrr a0, vlenb
+; RV64-NEXT:    li a3, 14
+; RV64-NEXT:    mul a0, a0, a3
+; RV64-NEXT:    add a0, sp, a0
+; RV64-NEXT:    addi a0, a0, 64
+; RV64-NEXT:    add a6, a6, a2
+; RV64-NEXT:    vl1re64.v v20, (a6)
+; RV64-NEXT:    add a6, a6, a2
+; RV64-NEXT:    vl1re64.v v21, (a6)
+; RV64-NEXT:    add a6, a6, a2
+; RV64-NEXT:    slli a2, a2, 3
+; RV64-NEXT:    add a2, a0, a2
+; RV64-NEXT:    vl1re64.v v22, (a6)
+; RV64-NEXT:    vl1re64.v v23, (a1)
+; RV64-NEXT:    add a5, a0, a5
+; RV64-NEXT:    vs2r.v v12, (a5)
+; RV64-NEXT:    vs4r.v v8, (a2)
+; RV64-NEXT:    vs8r.v v16, (a0)
+; RV64-NEXT:    vl8re64.v v16, (a2)
+; RV64-NEXT:    vl8re64.v v8, (a0)
+; RV64-NEXT:    addi sp, s0, -80
+; RV64-NEXT:    ld ra, 72(sp) # 8-byte Folded Reload
+; RV64-NEXT:    ld s0, 64(sp) # 8-byte Folded Reload
+; RV64-NEXT:    addi sp, sp, 80
+; RV64-NEXT:    ret
+;
+; ZVBB-RV32-LABEL: vector_interleave_nxv14i64_nxv2i64:
+; ZVBB-RV32:       # %bb.0:
+; ZVBB-RV32-NEXT:    addi sp, sp, -80
+; ZVBB-RV32-NEXT:    sw ra, 76(sp) # 4-byte Folded Spill
+; ZVBB-RV32-NEXT:    sw s0, 72(sp) # 4-byte Folded Spill
+; ZVBB-RV32-NEXT:    addi s0, sp, 80
+; ZVBB-RV32-NEXT:    csrr a0, vlenb
+; ZVBB-RV32-NEXT:    slli a0, a0, 5
+; ZVBB-RV32-NEXT:    sub sp, sp, a0
+; ZVBB-RV32-NEXT:    andi sp, sp, -64
+; ZVBB-RV32-NEXT:    vsetvli a0, zero, e64, m1, ta, ma
+; ZVBB-RV32-NEXT:    vmv2r.v v26, v20
+; ZVBB-RV32-NEXT:    addi a0, sp, 64
+; ZVBB-RV32-NEXT:    vmv2r.v v24, v16
+; ZVBB-RV32-NEXT:    csrr a1, vlenb
+; ZVBB-RV32-NEXT:    slli a2, a1, 3
+; ZVBB-RV32-NEXT:    sub a1, a2, a1
+; ZVBB-RV32-NEXT:    add a1, sp, a1
+; ZVBB-RV32-NEXT:    addi a1, a1, 64
+; ZVBB-RV32-NEXT:    vmv2r.v v22, v12
+; ZVBB-RV32-NEXT:    csrr a2, vlenb
+; ZVBB-RV32-NEXT:    vmv2r.v v20, v8
+; ZVBB-RV32-NEXT:    vmv1r.v v1, v20
+; ZVBB-RV32-NEXT:    vmv1r.v v3, v22
+; ZVBB-RV32-NEXT:    vmv1r.v v5, v24
+; ZVBB-RV32-NEXT:    vmv1r.v v7, v26
+; ZVBB-RV32-NEXT:    add a3, a0, a2
+; ZVBB-RV32-NEXT:    vmv1r.v v2, v10
+; ZVBB-RV32-NEXT:    add a4, a1, a2
+; ZVBB-RV32-NEXT:    slli a5, a2, 2
+; ZVBB-RV32-NEXT:    vmv1r.v v4, v14
+; ZVBB-RV32-NEXT:    slli a6, a2, 4
+; ZVBB-RV32-NEXT:    add a7, a4, a2
+; ZVBB-RV32-NEXT:    vmv1r.v v6, v18
+; ZVBB-RV32-NEXT:    sub a5, a6, a5
+; ZVBB-RV32-NEXT:    vmv1r.v v22, v11
+; ZVBB-RV32-NEXT:    add a6, a7, a2
+; ZVBB-RV32-NEXT:    vmv1r.v v24, v15
+; ZVBB-RV32-NEXT:    vsseg7e64.v v1, (a0)
+; ZVBB-RV32-NEXT:    vmv1r.v v26, v19
+; ZVBB-RV32-NEXT:    vsseg7e64.v v21, (a1)
+; ZVBB-RV32-NEXT:    vl1re64.v v10, (a6)
+; ZVBB-RV32-NEXT:    add a6, a6, a2
+; ZVBB-RV32-NEXT:    vl1re64.v v11, (a6)
+; ZVBB-RV32-NEXT:    add a6, a6, a2
+; ZVBB-RV32-NEXT:    vl1re64.v v12, (a6)
+; ZVBB-RV32-NEXT:    add a6, a6, a2
+; ZVBB-RV32-NEXT:    vl1re64.v v13, (a6)
+; ZVBB-RV32-NEXT:    add a6, a3, a2
+; ZVBB-RV32-NEXT:    vl1re64.v v18, (a6)
+; ZVBB-RV32-NEXT:    add a6, a6, a2
+; ZVBB-RV32-NEXT:    vl1re64.v v19, (a6)
+; ZVBB-RV32-NEXT:    vl1re64.v v16, (a0)
+; ZVBB-RV32-NEXT:    vl1re64.v v8, (a4)
+; ZVBB-RV32-NEXT:    vl1re64.v v17, (a3)
+; ZVBB-RV32-NEXT:    vl1re64.v v9, (a7)
+; ZVBB-RV32-NEXT:    csrr a0, vlenb
+; ZVBB-RV32-NEXT:    li a3, 14
+; ZVBB-RV32-NEXT:    mul a0, a0, a3
+; ZVBB-RV32-NEXT:    add a0, sp, a0
+; ZVBB-RV32-NEXT:    addi a0, a0, 64
+; ZVBB-RV32-NEXT:    add a6, a6, a2
+; ZVBB-RV32-NEXT:    vl1re64.v v20, (a6)
+; ZVBB-RV32-NEXT:    add a6, a6, a2
+; ZVBB-RV32-NEXT:    vl1re64.v v21, (a6)
+; ZVBB-RV32-NEXT:    add a6, a6, a2
+; ZVBB-RV32-NEXT:    slli a2, a2, 3
+; ZVBB-RV32-NEXT:    add a2, a0, a2
+; ZVBB-RV32-NEXT:    vl1re64.v v22, (a6)
+; ZVBB-RV32-NEXT:    vl1re64.v v23, (a1)
+; ZVBB-RV32-NEXT:    add a5, a0, a5
+; ZVBB-RV32-NEXT:    vs2r.v v12, (a5)
+; ZVBB-RV32-NEXT:    vs4r.v v8, (a2)
+; ZVBB-RV32-NEXT:    vs8r.v v16, (a0)
+; ZVBB-RV32-NEXT:    vl8re64.v v16, (a2)
+; ZVBB-RV32-NEXT:    vl8re64.v v8, (a0)
+; ZVBB-RV32-NEXT:    addi sp, s0, -80
+; ZVBB-RV32-NEXT:    lw ra, 76(sp) # 4-byte Folded Reload
+; ZVBB-RV32-NEXT:    lw s0, 72(sp) # 4-byte Folded Reload
+; ZVBB-RV32-NEXT:    addi sp, sp, 80
+; ZVBB-RV32-NEXT:    ret
+;
+; ZVBB-RV64-LABEL: vector_interleave_nxv14i64_nxv2i64:
+; ZVBB-RV64:       # %bb.0:
+; ZVBB-RV64-NEXT:    addi sp, sp, -80
+; ZVBB-RV64-NEXT:    sd ra, 72(sp) # 8-byte Folded Spill
+; ZVBB-RV64-NEXT:    sd s0, 64(sp) # 8-byte Folded Spill
+; ZVBB-RV64-NEXT:    addi s0, sp, 80
+; ZVBB-RV64-NEXT:    csrr a0, vlenb
+; ZVBB-RV64-NEXT:    slli a0, a0, 5
+; ZVBB-RV64-NEXT:    sub sp, sp, a0
+; ZVBB-RV64-NEXT:    andi sp, sp, -64
+; ZVBB-RV64-NEXT:    vsetvli a0, zero, e64, m1, ta, ma
+; ZVBB-RV64-NEXT:    vmv2r.v v26, v20
+; ZVBB-RV64-NEXT:    addi a0, sp, 64
+; ZVBB-RV64-NEXT:    vmv2r.v v24, v16
+; ZVBB-RV64-NEXT:    csrr a1, vlenb
+; ZVBB-RV64-NEXT:    slli a2, a1, 3
+; ZVBB-RV64-NEXT:    sub a1, a2, a1
+; ZVBB-RV64-NEXT:    add a1, sp, a1
+; ZVBB-RV64-NEXT:    addi a1, a1, 64
+; ZVBB-RV64-NEXT:    vmv2r.v v22, v12
+; ZVBB-RV64-NEXT:    csrr a2, vlenb
+; ZVBB-RV64-NEXT:    vmv2r.v v20, v8
+; ZVBB-RV64-NEXT:    vmv1r.v v1, v20
+; ZVBB-RV64-NEXT:    vmv1r.v v3, v22
+; ZVBB-RV64-NEXT:    vmv1r.v v5, v24
+; ZVBB-RV64-NEXT:    vmv1r.v v7, v26
+; ZVBB-RV64-NEXT:    add a3, a0, a2
+; ZVBB-RV64-NEXT:    vmv1r.v v2, v10
+; ZVBB-RV64-NEXT:    add a4, a1, a2
+; ZVBB-RV64-NEXT:    slli a5, a2, 2
+; ZVBB-RV64-NEXT:    vmv1r.v v4, v14
+; ZVBB-RV64-NEXT:    slli a6, a2, 4
+; ZVBB-RV64-NEXT:    add a7, a4, a2
+; ZVBB-RV64-NEXT:    vmv1r.v v6, v18
+; ZVBB-RV64-NEXT:    sub a5, a6, a5
+; ZVBB-RV64-NEXT:    vmv1r.v v22, v11
+; ZVBB-RV64-NEXT:    add a6, a7, a2
+; ZVBB-RV64-NEXT:    vmv1r.v v24, v15
+; ZVBB-RV64-NEXT:    vsseg7e64.v v1, (a0)
+; ZVBB-RV64-NEXT:    vmv1r.v v26, v19
+; ZVBB-RV64-NEXT:    vsseg7e64.v v21, (a1)
+; ZVBB-RV64-NEXT:    vl1re64.v v10, (a6)
+; ZVBB-RV64-NEXT:    add a6, a6, a2
+; ZVBB-RV64-NEXT:    vl1re64.v v11, (a6)
+; ZVBB-RV64-NEXT:    add a6, a6, a2
+; ZVBB-RV64-NEXT:    vl1re64.v v12, (a6)
+; ZVBB-RV64-NEXT:    add a6, a6, a2
+; ZVBB-RV64-NEXT:    vl1re64.v v13, (a6)
+; ZVBB-RV64-NEXT:    add a6, a3, a2
+; ZVBB-RV64-NEXT:    vl1re64.v v18, (a6)
+; ZVBB-RV64-NEXT:    add a6, a6, a2
+; ZVBB-RV64-NEXT:    vl1re64.v v19, (a6)
+; ZVBB-RV64-NEXT:    vl1re64.v v16, (a0)
+; ZVBB-RV64-NEXT:    vl1re64.v v8, (a4)
+; ZVBB-RV64-NEXT:    vl1re64.v v17, (a3)
+; ZVBB-RV64-NEXT:    vl1re64.v v9, (a7)
+; ZVBB-RV64-NEXT:    csrr a0, vlenb
+; ZVBB-RV64-NEXT:    li a3, 14
+; ZVBB-RV64-NEXT:    mul a0, a0, a3
+; ZVBB-RV64-NEXT:    add a0, sp, a0
+; ZVBB-RV64-NEXT:    addi a0, a0, 64
+; ZVBB-RV64-NEXT:    add a6, a6, a2
+; ZVBB-RV64-NEXT:    vl1re64.v v20, (a6)
+; ZVBB-RV64-NEXT:    add a6, a6, a2
+; ZVBB-RV64-NEXT:    vl1re64.v v21, (a6)
+; ZVBB-RV64-NEXT:    add a6, a6, a2
+; ZVBB-RV64-NEXT:    slli a2, a2, 3
+; ZVBB-RV64-NEXT:    add a2, a0, a2
+; ZVBB-RV64-NEXT:    vl1re64.v v22, (a6)
+; ZVBB-RV64-NEXT:    vl1re64.v v23, (a1)
+; ZVBB-RV64-NEXT:    add a5, a0, a5
+; ZVBB-RV64-NEXT:    vs2r.v v12, (a5)
+; ZVBB-RV64-NEXT:    vs4r.v v8, (a2)
+; ZVBB-RV64-NEXT:    vs8r.v v16, (a0)
+; ZVBB-RV64-NEXT:    vl8re64.v v16, (a2)
+; ZVBB-RV64-NEXT:    vl8re64.v v8, (a0)
+; ZVBB-RV64-NEXT:    addi sp, s0, -80
+; ZVBB-RV64-NEXT:    ld ra, 72(sp) # 8-byte Folded Reload
+; ZVBB-RV64-NEXT:    ld s0, 64(sp) # 8-byte Folded Reload
+; ZVBB-RV64-NEXT:    addi sp, sp, 80
+; ZVBB-RV64-NEXT:    ret
+  %res = call <vscale x 14 x i64> @llvm.vector.interleave7.nxv14i64(<vscale x 2 x i64> %a, <vscale x 2 x i64> %b, <vscale x 2 x i64> %c, <vscale x 2 x i64> %d, <vscale x 2 x i64> %e, <vscale x 2 x i64> %f, <vscale x 2 x i64> %g)
+  ret <vscale x 14 x i64> %res
+}
diff --git a/llvm/test/CodeGen/RISCV/rvv/vfadd-sdnode.ll b/llvm/test/CodeGen/RISCV/rvv/vfadd-sdnode.ll
index 19c5ee4a85ed6..a21918e679b62 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vfadd-sdnode.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vfadd-sdnode.ll
@@ -30,13 +30,11 @@ define <vscale x 1 x bfloat> @vfadd_vv_nxv1bf16(<vscale x 1 x bfloat> %va, <vsca
 define <vscale x 1 x bfloat> @vfadd_vf_nxv1bf16(<vscale x 1 x bfloat> %va, bfloat %b) {
 ; CHECK-LABEL: vfadd_vf_nxv1bf16:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    fmv.x.h a0, fa0
-; CHECK-NEXT:    vsetvli a1, zero, e16, mf4, ta, ma
-; CHECK-NEXT:    vmv.v.x v9, a0
-; CHECK-NEXT:    vfwcvtbf16.f.f.v v10, v8
-; CHECK-NEXT:    vfwcvtbf16.f.f.v v8, v9
+; CHECK-NEXT:    fcvt.s.bf16 fa5, fa0
+; CHECK-NEXT:    vsetvli a0, zero, e16, mf4, ta, ma
+; CHECK-NEXT:    vfwcvtbf16.f.f.v v9, v8
 ; CHECK-NEXT:    vsetvli zero, zero, e32, mf2, ta, ma
-; CHECK-NEXT:    vfadd.vv v9, v10, v8
+; CHECK-NEXT:    vfadd.vf v9, v9, fa5
 ; CHECK-NEXT:    vsetvli zero, zero, e16, mf4, ta, ma
 ; CHECK-NEXT:    vfncvtbf16.f.f.w v8, v9
 ; CHECK-NEXT:    ret
@@ -64,13 +62,11 @@ define <vscale x 2 x bfloat> @vfadd_vv_nxv2bf16(<vscale x 2 x bfloat> %va, <vsca
 define <vscale x 2 x bfloat> @vfadd_vf_nxv2bf16(<vscale x 2 x bfloat> %va, bfloat %b) {
 ; CHECK-LABEL: vfadd_vf_nxv2bf16:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    fmv.x.h a0, fa0
-; CHECK-NEXT:    vsetvli a1, zero, e16, mf2, ta, ma
-; CHECK-NEXT:    vmv.v.x v9, a0
-; CHECK-NEXT:    vfwcvtbf16.f.f.v v10, v8
-; CHECK-NEXT:    vfwcvtbf16.f.f.v v8, v9
+; CHECK-NEXT:    fcvt.s.bf16 fa5, fa0
+; CHECK-NEXT:    vsetvli a0, zero, e16, mf2, ta, ma
+; CHECK-NEXT:    vfwcvtbf16.f.f.v v9, v8
 ; CHECK-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
-; CHECK-NEXT:    vfadd.vv v9, v10, v8
+; CHECK-NEXT:    vfadd.vf v9, v9, fa5
 ; CHECK-NEXT:    vsetvli zero, zero, e16, mf2, ta, ma
 ; CHECK-NEXT:    vfncvtbf16.f.f.w v8, v9
 ; CHECK-NEXT:    ret
@@ -98,13 +94,11 @@ define <vscale x 4 x bfloat> @vfadd_vv_nxv4bf16(<vscale x 4 x bfloat> %va, <vsca
 define <vscale x 4 x bfloat> @vfadd_vf_nxv4bf16(<vscale x 4 x bfloat> %va, bfloat %b) {
 ; CHECK-LABEL: vfadd_vf_nxv4bf16:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    fmv.x.h a0, fa0
-; CHECK-NEXT:    vsetvli a1, zero, e16, m1, ta, ma
-; CHECK-NEXT:    vmv.v.x v9, a0
+; CHECK-NEXT:    fcvt.s.bf16 fa5, fa0
+; CHECK-NEXT:    vsetvli a0, zero, e16, m1, ta, ma
 ; CHECK-NEXT:    vfwcvtbf16.f.f.v v10, v8
-; CHECK-NEXT:    vfwcvtbf16.f.f.v v12, v9
 ; CHECK-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
-; CHECK-NEXT:    vfadd.vv v10, v10, v12
+; CHECK-NEXT:    vfadd.vf v10, v10, fa5
 ; CHECK-NEXT:    vsetvli zero, zero, e16, m1, ta, ma
 ; CHECK-NEXT:    vfncvtbf16.f.f.w v8, v10
 ; CHECK-NEXT:    ret
@@ -132,13 +126,11 @@ define <vscale x 8 x bfloat> @vfadd_vv_nxv8bf16(<vscale x 8 x bfloat> %va, <vsca
 define <vscale x 8 x bfloat> @vfadd_vf_nxv8bf16(<vscale x 8 x bfloat> %va, bfloat %b) {
 ; CHECK-LABEL: vfadd_vf_nxv8bf16:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    fmv.x.h a0, fa0
-; CHECK-NEXT:    vsetvli a1, zero, e16, m2, ta, ma
-; CHECK-NEXT:    vmv.v.x v10, a0
+; CHECK-NEXT:    fcvt.s.bf16 fa5, fa0
+; CHECK-NEXT:    vsetvli a0, zero, e16, m2, ta, ma
 ; CHECK-NEXT:    vfwcvtbf16.f.f.v v12, v8
-; CHECK-NEXT:    vfwcvtbf16.f.f.v v16, v10
 ; CHECK-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
-; CHECK-NEXT:    vfadd.vv v12, v12, v16
+; CHECK-NEXT:    vfadd.vf v12, v12, fa5
 ; CHECK-NEXT:    vsetvli zero, zero, e16, m2, ta, ma
 ; CHECK-NEXT:    vfncvtbf16.f.f.w v8, v12
 ; CHECK-NEXT:    ret
@@ -151,13 +143,11 @@ define <vscale x 8 x bfloat> @vfadd_vf_nxv8bf16(<vscale x 8 x bfloat> %va, bfloa
 define <vscale x 8 x bfloat> @vfadd_fv_nxv8bf16(<vscale x 8 x bfloat> %va, bfloat %b) {
 ; CHECK-LABEL: vfadd_fv_nxv8bf16:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    fmv.x.h a0, fa0
-; CHECK-NEXT:    vsetvli a1, zero, e16, m2, ta, ma
-; CHECK-NEXT:    vmv.v.x v10, a0
+; CHECK-NEXT:    fcvt.s.bf16 fa5, fa0
+; CHECK-NEXT:    vsetvli a0, zero, e16, m2, ta, ma
 ; CHECK-NEXT:    vfwcvtbf16.f.f.v v12, v8
-; CHECK-NEXT:    vfwcvtbf16.f.f.v v16, v10
 ; CHECK-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
-; CHECK-NEXT:    vfadd.vv v12, v16, v12
+; CHECK-NEXT:    vfadd.vf v12, v12, fa5
 ; CHECK-NEXT:    vsetvli zero, zero, e16, m2, ta, ma
 ; CHECK-NEXT:    vfncvtbf16.f.f.w v8, v12
 ; CHECK-NEXT:    ret
@@ -185,13 +175,11 @@ define <vscale x 16 x bfloat> @vfadd_vv_nxv16bf16(<vscale x 16 x bfloat> %va, <v
 define <vscale x 16 x bfloat> @vfadd_vf_nxv16bf16(<vscale x 16 x bfloat> %va, bfloat %b) {
 ; CHECK-LABEL: vfadd_vf_nxv16bf16:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    fmv.x.h a0, fa0
-; CHECK-NEXT:    vsetvli a1, zero, e16, m4, ta, ma
-; CHECK-NEXT:    vmv.v.x v12, a0
+; CHECK-NEXT:    fcvt.s.bf16 fa5, fa0
+; CHECK-NEXT:    vsetvli a0, zero, e16, m4, ta, ma
 ; CHECK-NEXT:    vfwcvtbf16.f.f.v v16, v8
-; CHECK-NEXT:    vfwcvtbf16.f.f.v v24, v12
 ; CHECK-NEXT:    vsetvli zero, zero, e32, m8, ta, ma
-; CHECK-NEXT:    vfadd.vv v16, v16, v24
+; CHECK-NEXT:    vfadd.vf v16, v16, fa5
 ; CHECK-NEXT:    vsetvli zero, zero, e16, m4, ta, ma
 ; CHECK-NEXT:    vfncvtbf16.f.f.w v8, v16
 ; CHECK-NEXT:    ret
@@ -310,13 +298,11 @@ define <vscale x 1 x half> @vfadd_vf_nxv1f16(<vscale x 1 x half> %va, half %b) {
 ;
 ; ZVFHMIN-LABEL: vfadd_vf_nxv1f16:
 ; ZVFHMIN:       # %bb.0:
-; ZVFHMIN-NEXT:    fmv.x.h a0, fa0
-; ZVFHMIN-NEXT:    vsetvli a1, zero, e16, mf4, ta, ma
-; ZVFHMIN-NEXT:    vmv.v.x v9, a0
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v10, v8
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v8, v9
+; ZVFHMIN-NEXT:    fcvt.s.h fa5, fa0
+; ZVFHMIN-NEXT:    vsetvli a0, zero, e16, mf4, ta, ma
+; ZVFHMIN-NEXT:    vfwcvt.f.f.v v9, v8
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, mf2, ta, ma
-; ZVFHMIN-NEXT:    vfadd.vv v9, v10, v8
+; ZVFHMIN-NEXT:    vfadd.vf v9, v9, fa5
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e16, mf4, ta, ma
 ; ZVFHMIN-NEXT:    vfncvt.f.f.w v8, v9
 ; ZVFHMIN-NEXT:    ret
@@ -356,13 +342,11 @@ define <vscale x 2 x half> @vfadd_vf_nxv2f16(<vscale x 2 x half> %va, half %b) {
 ;
 ; ZVFHMIN-LABEL: vfadd_vf_nxv2f16:
 ; ZVFHMIN:       # %bb.0:
-; ZVFHMIN-NEXT:    fmv.x.h a0, fa0
-; ZVFHMIN-NEXT:    vsetvli a1, zero, e16, mf2, ta, ma
-; ZVFHMIN-NEXT:    vmv.v.x v9, a0
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v10, v8
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v8, v9
+; ZVFHMIN-NEXT:    fcvt.s.h fa5, fa0
+; ZVFHMIN-NEXT:    vsetvli a0, zero, e16, mf2, ta, ma
+; ZVFHMIN-NEXT:    vfwcvt.f.f.v v9, v8
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
-; ZVFHMIN-NEXT:    vfadd.vv v9, v10, v8
+; ZVFHMIN-NEXT:    vfadd.vf v9, v9, fa5
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e16, mf2, ta, ma
 ; ZVFHMIN-NEXT:    vfncvt.f.f.w v8, v9
 ; ZVFHMIN-NEXT:    ret
@@ -402,13 +386,11 @@ define <vscale x 4 x half> @vfadd_vf_nxv4f16(<vscale x 4 x half> %va, half %b) {
 ;
 ; ZVFHMIN-LABEL: vfadd_vf_nxv4f16:
 ; ZVFHMIN:       # %bb.0:
-; ZVFHMIN-NEXT:    fmv.x.h a0, fa0
-; ZVFHMIN-NEXT:    vsetvli a1, zero, e16, m1, ta, ma
-; ZVFHMIN-NEXT:    vmv.v.x v9, a0
+; ZVFHMIN-NEXT:    fcvt.s.h fa5, fa0
+; ZVFHMIN-NEXT:    vsetvli a0, zero, e16, m1, ta, ma
 ; ZVFHMIN-NEXT:    vfwcvt.f.f.v v10, v8
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v12, v9
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
-; ZVFHMIN-NEXT:    vfadd.vv v10, v10, v12
+; ZVFHMIN-NEXT:    vfadd.vf v10, v10, fa5
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e16, m1, ta, ma
 ; ZVFHMIN-NEXT:    vfncvt.f.f.w v8, v10
 ; ZVFHMIN-NEXT:    ret
@@ -448,13 +430,11 @@ define <vscale x 8 x half> @vfadd_vf_nxv8f16(<vscale x 8 x half> %va, half %b) {
 ;
 ; ZVFHMIN-LABEL: vfadd_vf_nxv8f16:
 ; ZVFHMIN:       # %bb.0:
-; ZVFHMIN-NEXT:    fmv.x.h a0, fa0
-; ZVFHMIN-NEXT:    vsetvli a1, zero, e16, m2, ta, ma
-; ZVFHMIN-NEXT:    vmv.v.x v10, a0
+; ZVFHMIN-NEXT:    fcvt.s.h fa5, fa0
+; ZVFHMIN-NEXT:    vsetvli a0, zero, e16, m2, ta, ma
 ; ZVFHMIN-NEXT:    vfwcvt.f.f.v v12, v8
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v16, v10
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
-; ZVFHMIN-NEXT:    vfadd.vv v12, v12, v16
+; ZVFHMIN-NEXT:    vfadd.vf v12, v12, fa5
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e16, m2, ta, ma
 ; ZVFHMIN-NEXT:    vfncvt.f.f.w v8, v12
 ; ZVFHMIN-NEXT:    ret
@@ -473,13 +453,11 @@ define <vscale x 8 x half> @vfadd_fv_nxv8f16(<vscale x 8 x half> %va, half %b) {
 ;
 ; ZVFHMIN-LABEL: vfadd_fv_nxv8f16:
 ; ZVFHMIN:       # %bb.0:
-; ZVFHMIN-NEXT:    fmv.x.h a0, fa0
-; ZVFHMIN-NEXT:    vsetvli a1, zero, e16, m2, ta, ma
-; ZVFHMIN-NEXT:    vmv.v.x v10, a0
+; ZVFHMIN-NEXT:    fcvt.s.h fa5, fa0
+; ZVFHMIN-NEXT:    vsetvli a0, zero, e16, m2, ta, ma
 ; ZVFHMIN-NEXT:    vfwcvt.f.f.v v12, v8
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v16, v10
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
-; ZVFHMIN-NEXT:    vfadd.vv v12, v16, v12
+; ZVFHMIN-NEXT:    vfadd.vf v12, v12, fa5
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e16, m2, ta, ma
 ; ZVFHMIN-NEXT:    vfncvt.f.f.w v8, v12
 ; ZVFHMIN-NEXT:    ret
@@ -519,13 +497,11 @@ define <vscale x 16 x half> @vfadd_vf_nxv16f16(<vscale x 16 x half> %va, half %b
 ;
 ; ZVFHMIN-LABEL: vfadd_vf_nxv16f16:
 ; ZVFHMIN:       # %bb.0:
-; ZVFHMIN-NEXT:    fmv.x.h a0, fa0
-; ZVFHMIN-NEXT:    vsetvli a1, zero, e16, m4, ta, ma
-; ZVFHMIN-NEXT:    vmv.v.x v12, a0
+; ZVFHMIN-NEXT:    fcvt.s.h fa5, fa0
+; ZVFHMIN-NEXT:    vsetvli a0, zero, e16, m4, ta, ma
 ; ZVFHMIN-NEXT:    vfwcvt.f.f.v v16, v8
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v24, v12
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m8, ta, ma
-; ZVFHMIN-NEXT:    vfadd.vv v16, v16, v24
+; ZVFHMIN-NEXT:    vfadd.vf v16, v16, fa5
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e16, m4, ta, ma
 ; ZVFHMIN-NEXT:    vfncvt.f.f.w v8, v16
 ; ZVFHMIN-NEXT:    ret
diff --git a/llvm/test/CodeGen/RISCV/rvv/vfdiv-sdnode.ll b/llvm/test/CodeGen/RISCV/rvv/vfdiv-sdnode.ll
index b4a9b1fe3fcf8..e671ba850415b 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vfdiv-sdnode.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vfdiv-sdnode.ll
@@ -26,13 +26,11 @@ define <vscale x 1 x bfloat> @vfdiv_vv_nxv1bf16(<vscale x 1 x bfloat> %va, <vsca
 define <vscale x 1 x bfloat> @vfdiv_vf_nxv1bf16(<vscale x 1 x bfloat> %va, bfloat %b) {
 ; CHECK-LABEL: vfdiv_vf_nxv1bf16:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    fmv.x.h a0, fa0
-; CHECK-NEXT:    vsetvli a1, zero, e16, mf4, ta, ma
-; CHECK-NEXT:    vmv.v.x v9, a0
-; CHECK-NEXT:    vfwcvtbf16.f.f.v v10, v8
-; CHECK-NEXT:    vfwcvtbf16.f.f.v v8, v9
+; CHECK-NEXT:    fcvt.s.bf16 fa5, fa0
+; CHECK-NEXT:    vsetvli a0, zero, e16, mf4, ta, ma
+; CHECK-NEXT:    vfwcvtbf16.f.f.v v9, v8
 ; CHECK-NEXT:    vsetvli zero, zero, e32, mf2, ta, ma
-; CHECK-NEXT:    vfdiv.vv v9, v10, v8
+; CHECK-NEXT:    vfdiv.vf v9, v9, fa5
 ; CHECK-NEXT:    vsetvli zero, zero, e16, mf4, ta, ma
 ; CHECK-NEXT:    vfncvtbf16.f.f.w v8, v9
 ; CHECK-NEXT:    ret
@@ -60,13 +58,11 @@ define <vscale x 2 x bfloat> @vfdiv_vv_nxv2bf16(<vscale x 2 x bfloat> %va, <vsca
 define <vscale x 2 x bfloat> @vfdiv_vf_nxv2bf16(<vscale x 2 x bfloat> %va, bfloat %b) {
 ; CHECK-LABEL: vfdiv_vf_nxv2bf16:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    fmv.x.h a0, fa0
-; CHECK-NEXT:    vsetvli a1, zero, e16, mf2, ta, ma
-; CHECK-NEXT:    vmv.v.x v9, a0
-; CHECK-NEXT:    vfwcvtbf16.f.f.v v10, v8
-; CHECK-NEXT:    vfwcvtbf16.f.f.v v8, v9
+; CHECK-NEXT:    fcvt.s.bf16 fa5, fa0
+; CHECK-NEXT:    vsetvli a0, zero, e16, mf2, ta, ma
+; CHECK-NEXT:    vfwcvtbf16.f.f.v v9, v8
 ; CHECK-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
-; CHECK-NEXT:    vfdiv.vv v9, v10, v8
+; CHECK-NEXT:    vfdiv.vf v9, v9, fa5
 ; CHECK-NEXT:    vsetvli zero, zero, e16, mf2, ta, ma
 ; CHECK-NEXT:    vfncvtbf16.f.f.w v8, v9
 ; CHECK-NEXT:    ret
@@ -94,13 +90,11 @@ define <vscale x 4 x bfloat> @vfdiv_vv_nxv4bf16(<vscale x 4 x bfloat> %va, <vsca
 define <vscale x 4 x bfloat> @vfdiv_vf_nxv4bf16(<vscale x 4 x bfloat> %va, bfloat %b) {
 ; CHECK-LABEL: vfdiv_vf_nxv4bf16:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    fmv.x.h a0, fa0
-; CHECK-NEXT:    vsetvli a1, zero, e16, m1, ta, ma
-; CHECK-NEXT:    vmv.v.x v9, a0
+; CHECK-NEXT:    fcvt.s.bf16 fa5, fa0
+; CHECK-NEXT:    vsetvli a0, zero, e16, m1, ta, ma
 ; CHECK-NEXT:    vfwcvtbf16.f.f.v v10, v8
-; CHECK-NEXT:    vfwcvtbf16.f.f.v v12, v9
 ; CHECK-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
-; CHECK-NEXT:    vfdiv.vv v10, v10, v12
+; CHECK-NEXT:    vfdiv.vf v10, v10, fa5
 ; CHECK-NEXT:    vsetvli zero, zero, e16, m1, ta, ma
 ; CHECK-NEXT:    vfncvtbf16.f.f.w v8, v10
 ; CHECK-NEXT:    ret
@@ -128,13 +122,11 @@ define <vscale x 8 x bfloat> @vfdiv_vv_nxv8bf16(<vscale x 8 x bfloat> %va, <vsca
 define <vscale x 8 x bfloat> @vfdiv_vf_nxv8bf16(<vscale x 8 x bfloat> %va, bfloat %b) {
 ; CHECK-LABEL: vfdiv_vf_nxv8bf16:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    fmv.x.h a0, fa0
-; CHECK-NEXT:    vsetvli a1, zero, e16, m2, ta, ma
-; CHECK-NEXT:    vmv.v.x v10, a0
+; CHECK-NEXT:    fcvt.s.bf16 fa5, fa0
+; CHECK-NEXT:    vsetvli a0, zero, e16, m2, ta, ma
 ; CHECK-NEXT:    vfwcvtbf16.f.f.v v12, v8
-; CHECK-NEXT:    vfwcvtbf16.f.f.v v16, v10
 ; CHECK-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
-; CHECK-NEXT:    vfdiv.vv v12, v12, v16
+; CHECK-NEXT:    vfdiv.vf v12, v12, fa5
 ; CHECK-NEXT:    vsetvli zero, zero, e16, m2, ta, ma
 ; CHECK-NEXT:    vfncvtbf16.f.f.w v8, v12
 ; CHECK-NEXT:    ret
@@ -147,13 +139,11 @@ define <vscale x 8 x bfloat> @vfdiv_vf_nxv8bf16(<vscale x 8 x bfloat> %va, bfloa
 define <vscale x 8 x bfloat> @vfdiv_fv_nxv8bf16(<vscale x 8 x bfloat> %va, bfloat %b) {
 ; CHECK-LABEL: vfdiv_fv_nxv8bf16:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    fmv.x.h a0, fa0
-; CHECK-NEXT:    vsetvli a1, zero, e16, m2, ta, ma
-; CHECK-NEXT:    vmv.v.x v10, a0
+; CHECK-NEXT:    fcvt.s.bf16 fa5, fa0
+; CHECK-NEXT:    vsetvli a0, zero, e16, m2, ta, ma
 ; CHECK-NEXT:    vfwcvtbf16.f.f.v v12, v8
-; CHECK-NEXT:    vfwcvtbf16.f.f.v v16, v10
 ; CHECK-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
-; CHECK-NEXT:    vfdiv.vv v12, v16, v12
+; CHECK-NEXT:    vfrdiv.vf v12, v12, fa5
 ; CHECK-NEXT:    vsetvli zero, zero, e16, m2, ta, ma
 ; CHECK-NEXT:    vfncvtbf16.f.f.w v8, v12
 ; CHECK-NEXT:    ret
@@ -181,13 +171,11 @@ define <vscale x 16 x bfloat> @vfdiv_vv_nxv16bf16(<vscale x 16 x bfloat> %va, <v
 define <vscale x 16 x bfloat> @vfdiv_vf_nxv16bf16(<vscale x 16 x bfloat> %va, bfloat %b) {
 ; CHECK-LABEL: vfdiv_vf_nxv16bf16:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    fmv.x.h a0, fa0
-; CHECK-NEXT:    vsetvli a1, zero, e16, m4, ta, ma
-; CHECK-NEXT:    vmv.v.x v12, a0
+; CHECK-NEXT:    fcvt.s.bf16 fa5, fa0
+; CHECK-NEXT:    vsetvli a0, zero, e16, m4, ta, ma
 ; CHECK-NEXT:    vfwcvtbf16.f.f.v v16, v8
-; CHECK-NEXT:    vfwcvtbf16.f.f.v v24, v12
 ; CHECK-NEXT:    vsetvli zero, zero, e32, m8, ta, ma
-; CHECK-NEXT:    vfdiv.vv v16, v16, v24
+; CHECK-NEXT:    vfdiv.vf v16, v16, fa5
 ; CHECK-NEXT:    vsetvli zero, zero, e16, m4, ta, ma
 ; CHECK-NEXT:    vfncvtbf16.f.f.w v8, v16
 ; CHECK-NEXT:    ret
@@ -306,13 +294,11 @@ define <vscale x 1 x half> @vfdiv_vf_nxv1f16(<vscale x 1 x half> %va, half %b) {
 ;
 ; ZVFHMIN-LABEL: vfdiv_vf_nxv1f16:
 ; ZVFHMIN:       # %bb.0:
-; ZVFHMIN-NEXT:    fmv.x.h a0, fa0
-; ZVFHMIN-NEXT:    vsetvli a1, zero, e16, mf4, ta, ma
-; ZVFHMIN-NEXT:    vmv.v.x v9, a0
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v10, v8
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v8, v9
+; ZVFHMIN-NEXT:    fcvt.s.h fa5, fa0
+; ZVFHMIN-NEXT:    vsetvli a0, zero, e16, mf4, ta, ma
+; ZVFHMIN-NEXT:    vfwcvt.f.f.v v9, v8
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, mf2, ta, ma
-; ZVFHMIN-NEXT:    vfdiv.vv v9, v10, v8
+; ZVFHMIN-NEXT:    vfdiv.vf v9, v9, fa5
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e16, mf4, ta, ma
 ; ZVFHMIN-NEXT:    vfncvt.f.f.w v8, v9
 ; ZVFHMIN-NEXT:    ret
@@ -352,13 +338,11 @@ define <vscale x 2 x half> @vfdiv_vf_nxv2f16(<vscale x 2 x half> %va, half %b) {
 ;
 ; ZVFHMIN-LABEL: vfdiv_vf_nxv2f16:
 ; ZVFHMIN:       # %bb.0:
-; ZVFHMIN-NEXT:    fmv.x.h a0, fa0
-; ZVFHMIN-NEXT:    vsetvli a1, zero, e16, mf2, ta, ma
-; ZVFHMIN-NEXT:    vmv.v.x v9, a0
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v10, v8
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v8, v9
+; ZVFHMIN-NEXT:    fcvt.s.h fa5, fa0
+; ZVFHMIN-NEXT:    vsetvli a0, zero, e16, mf2, ta, ma
+; ZVFHMIN-NEXT:    vfwcvt.f.f.v v9, v8
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
-; ZVFHMIN-NEXT:    vfdiv.vv v9, v10, v8
+; ZVFHMIN-NEXT:    vfdiv.vf v9, v9, fa5
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e16, mf2, ta, ma
 ; ZVFHMIN-NEXT:    vfncvt.f.f.w v8, v9
 ; ZVFHMIN-NEXT:    ret
@@ -398,13 +382,11 @@ define <vscale x 4 x half> @vfdiv_vf_nxv4f16(<vscale x 4 x half> %va, half %b) {
 ;
 ; ZVFHMIN-LABEL: vfdiv_vf_nxv4f16:
 ; ZVFHMIN:       # %bb.0:
-; ZVFHMIN-NEXT:    fmv.x.h a0, fa0
-; ZVFHMIN-NEXT:    vsetvli a1, zero, e16, m1, ta, ma
-; ZVFHMIN-NEXT:    vmv.v.x v9, a0
+; ZVFHMIN-NEXT:    fcvt.s.h fa5, fa0
+; ZVFHMIN-NEXT:    vsetvli a0, zero, e16, m1, ta, ma
 ; ZVFHMIN-NEXT:    vfwcvt.f.f.v v10, v8
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v12, v9
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
-; ZVFHMIN-NEXT:    vfdiv.vv v10, v10, v12
+; ZVFHMIN-NEXT:    vfdiv.vf v10, v10, fa5
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e16, m1, ta, ma
 ; ZVFHMIN-NEXT:    vfncvt.f.f.w v8, v10
 ; ZVFHMIN-NEXT:    ret
@@ -444,13 +426,11 @@ define <vscale x 8 x half> @vfdiv_vf_nxv8f16(<vscale x 8 x half> %va, half %b) {
 ;
 ; ZVFHMIN-LABEL: vfdiv_vf_nxv8f16:
 ; ZVFHMIN:       # %bb.0:
-; ZVFHMIN-NEXT:    fmv.x.h a0, fa0
-; ZVFHMIN-NEXT:    vsetvli a1, zero, e16, m2, ta, ma
-; ZVFHMIN-NEXT:    vmv.v.x v10, a0
+; ZVFHMIN-NEXT:    fcvt.s.h fa5, fa0
+; ZVFHMIN-NEXT:    vsetvli a0, zero, e16, m2, ta, ma
 ; ZVFHMIN-NEXT:    vfwcvt.f.f.v v12, v8
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v16, v10
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
-; ZVFHMIN-NEXT:    vfdiv.vv v12, v12, v16
+; ZVFHMIN-NEXT:    vfdiv.vf v12, v12, fa5
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e16, m2, ta, ma
 ; ZVFHMIN-NEXT:    vfncvt.f.f.w v8, v12
 ; ZVFHMIN-NEXT:    ret
@@ -469,13 +449,11 @@ define <vscale x 8 x half> @vfdiv_fv_nxv8f16(<vscale x 8 x half> %va, half %b) {
 ;
 ; ZVFHMIN-LABEL: vfdiv_fv_nxv8f16:
 ; ZVFHMIN:       # %bb.0:
-; ZVFHMIN-NEXT:    fmv.x.h a0, fa0
-; ZVFHMIN-NEXT:    vsetvli a1, zero, e16, m2, ta, ma
-; ZVFHMIN-NEXT:    vmv.v.x v10, a0
+; ZVFHMIN-NEXT:    fcvt.s.h fa5, fa0
+; ZVFHMIN-NEXT:    vsetvli a0, zero, e16, m2, ta, ma
 ; ZVFHMIN-NEXT:    vfwcvt.f.f.v v12, v8
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v16, v10
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
-; ZVFHMIN-NEXT:    vfdiv.vv v12, v16, v12
+; ZVFHMIN-NEXT:    vfrdiv.vf v12, v12, fa5
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e16, m2, ta, ma
 ; ZVFHMIN-NEXT:    vfncvt.f.f.w v8, v12
 ; ZVFHMIN-NEXT:    ret
@@ -515,13 +493,11 @@ define <vscale x 16 x half> @vfdiv_vf_nxv16f16(<vscale x 16 x half> %va, half %b
 ;
 ; ZVFHMIN-LABEL: vfdiv_vf_nxv16f16:
 ; ZVFHMIN:       # %bb.0:
-; ZVFHMIN-NEXT:    fmv.x.h a0, fa0
-; ZVFHMIN-NEXT:    vsetvli a1, zero, e16, m4, ta, ma
-; ZVFHMIN-NEXT:    vmv.v.x v12, a0
+; ZVFHMIN-NEXT:    fcvt.s.h fa5, fa0
+; ZVFHMIN-NEXT:    vsetvli a0, zero, e16, m4, ta, ma
 ; ZVFHMIN-NEXT:    vfwcvt.f.f.v v16, v8
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v24, v12
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m8, ta, ma
-; ZVFHMIN-NEXT:    vfdiv.vv v16, v16, v24
+; ZVFHMIN-NEXT:    vfdiv.vf v16, v16, fa5
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e16, m4, ta, ma
 ; ZVFHMIN-NEXT:    vfncvt.f.f.w v8, v16
 ; ZVFHMIN-NEXT:    ret
diff --git a/llvm/test/CodeGen/RISCV/rvv/vfmadd-sdnode.ll b/llvm/test/CodeGen/RISCV/rvv/vfmadd-sdnode.ll
index 109023032e6ab..42166be34c88b 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vfmadd-sdnode.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vfmadd-sdnode.ll
@@ -46,16 +46,14 @@ define <vscale x 1 x bfloat> @vfmadd_vv_nxv1bf16_commuted(<vscale x 1 x bfloat>
 define <vscale x 1 x bfloat> @vfmadd_vf_nxv1bf16(<vscale x 1 x bfloat> %va, <vscale x 1 x bfloat> %vb, bfloat %c) {
 ; CHECK-LABEL: vfmadd_vf_nxv1bf16:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    fmv.x.h a0, fa0
-; CHECK-NEXT:    vsetvli a1, zero, e16, mf4, ta, ma
+; CHECK-NEXT:    vsetvli a0, zero, e16, mf4, ta, ma
 ; CHECK-NEXT:    vfwcvtbf16.f.f.v v10, v9
-; CHECK-NEXT:    vmv.v.x v9, a0
-; CHECK-NEXT:    vfwcvtbf16.f.f.v v11, v8
-; CHECK-NEXT:    vfwcvtbf16.f.f.v v12, v9
+; CHECK-NEXT:    fcvt.s.bf16 fa5, fa0
+; CHECK-NEXT:    vfwcvtbf16.f.f.v v9, v8
 ; CHECK-NEXT:    vsetvli zero, zero, e32, mf2, ta, ma
-; CHECK-NEXT:    vfmadd.vv v12, v11, v10
+; CHECK-NEXT:    vfmadd.vf v9, fa5, v10
 ; CHECK-NEXT:    vsetvli zero, zero, e16, mf4, ta, ma
-; CHECK-NEXT:    vfncvtbf16.f.f.w v8, v12
+; CHECK-NEXT:    vfncvtbf16.f.f.w v8, v9
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 1 x bfloat> poison, bfloat %c, i32 0
   %splat = shufflevector <vscale x 1 x bfloat> %head, <vscale x 1 x bfloat> poison, <vscale x 1 x i32> zeroinitializer
@@ -84,16 +82,14 @@ define <vscale x 2 x bfloat> @vfmadd_vv_nxv2bf16(<vscale x 2 x bfloat> %va, <vsc
 define <vscale x 2 x bfloat> @vfmadd_vf_nxv2bf16(<vscale x 2 x bfloat> %va, <vscale x 2 x bfloat> %vb, bfloat %c) {
 ; CHECK-LABEL: vfmadd_vf_nxv2bf16:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    fmv.x.h a0, fa0
-; CHECK-NEXT:    vsetvli a1, zero, e16, mf2, ta, ma
+; CHECK-NEXT:    vsetvli a0, zero, e16, mf2, ta, ma
 ; CHECK-NEXT:    vfwcvtbf16.f.f.v v10, v8
-; CHECK-NEXT:    vmv.v.x v8, a0
+; CHECK-NEXT:    fcvt.s.bf16 fa5, fa0
 ; CHECK-NEXT:    vfwcvtbf16.f.f.v v11, v9
-; CHECK-NEXT:    vfwcvtbf16.f.f.v v9, v8
 ; CHECK-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
-; CHECK-NEXT:    vfmadd.vv v9, v11, v10
+; CHECK-NEXT:    vfmadd.vf v11, fa5, v10
 ; CHECK-NEXT:    vsetvli zero, zero, e16, mf2, ta, ma
-; CHECK-NEXT:    vfncvtbf16.f.f.w v8, v9
+; CHECK-NEXT:    vfncvtbf16.f.f.w v8, v11
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 2 x bfloat> poison, bfloat %c, i32 0
   %splat = shufflevector <vscale x 2 x bfloat> %head, <vscale x 2 x bfloat> poison, <vscale x 2 x i32> zeroinitializer
@@ -122,16 +118,14 @@ define <vscale x 4 x bfloat> @vfmadd_vv_nxv4bf16(<vscale x 4 x bfloat> %va, <vsc
 define <vscale x 4 x bfloat> @vfmadd_vf_nxv4bf16(<vscale x 4 x bfloat> %va, <vscale x 4 x bfloat> %vb, bfloat %c) {
 ; CHECK-LABEL: vfmadd_vf_nxv4bf16:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    fmv.x.h a0, fa0
-; CHECK-NEXT:    vsetvli a1, zero, e16, m1, ta, ma
+; CHECK-NEXT:    vsetvli a0, zero, e16, m1, ta, ma
 ; CHECK-NEXT:    vfwcvtbf16.f.f.v v10, v9
-; CHECK-NEXT:    vmv.v.x v9, a0
+; CHECK-NEXT:    fcvt.s.bf16 fa5, fa0
 ; CHECK-NEXT:    vfwcvtbf16.f.f.v v12, v8
-; CHECK-NEXT:    vfwcvtbf16.f.f.v v14, v9
 ; CHECK-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
-; CHECK-NEXT:    vfmadd.vv v14, v12, v10
+; CHECK-NEXT:    vfmadd.vf v12, fa5, v10
 ; CHECK-NEXT:    vsetvli zero, zero, e16, m1, ta, ma
-; CHECK-NEXT:    vfncvtbf16.f.f.w v8, v14
+; CHECK-NEXT:    vfncvtbf16.f.f.w v8, v12
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 4 x bfloat> poison, bfloat %c, i32 0
   %splat = shufflevector <vscale x 4 x bfloat> %head, <vscale x 4 x bfloat> poison, <vscale x 4 x i32> zeroinitializer
@@ -160,16 +154,14 @@ define <vscale x 8 x bfloat> @vfmadd_vv_nxv8bf16(<vscale x 8 x bfloat> %va, <vsc
 define <vscale x 8 x bfloat> @vfmadd_vf_nxv8bf16(<vscale x 8 x bfloat> %va, <vscale x 8 x bfloat> %vb, bfloat %c) {
 ; CHECK-LABEL: vfmadd_vf_nxv8bf16:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    fmv.x.h a0, fa0
-; CHECK-NEXT:    vsetvli a1, zero, e16, m2, ta, ma
+; CHECK-NEXT:    vsetvli a0, zero, e16, m2, ta, ma
 ; CHECK-NEXT:    vfwcvtbf16.f.f.v v12, v8
-; CHECK-NEXT:    vmv.v.x v8, a0
+; CHECK-NEXT:    fcvt.s.bf16 fa5, fa0
 ; CHECK-NEXT:    vfwcvtbf16.f.f.v v16, v10
-; CHECK-NEXT:    vfwcvtbf16.f.f.v v20, v8
 ; CHECK-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
-; CHECK-NEXT:    vfmadd.vv v20, v16, v12
+; CHECK-NEXT:    vfmadd.vf v16, fa5, v12
 ; CHECK-NEXT:    vsetvli zero, zero, e16, m2, ta, ma
-; CHECK-NEXT:    vfncvtbf16.f.f.w v8, v20
+; CHECK-NEXT:    vfncvtbf16.f.f.w v8, v16
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 8 x bfloat> poison, bfloat %c, i32 0
   %splat = shufflevector <vscale x 8 x bfloat> %head, <vscale x 8 x bfloat> poison, <vscale x 8 x i32> zeroinitializer
@@ -198,16 +190,14 @@ define <vscale x 16 x bfloat> @vfmadd_vv_nxv16bf16(<vscale x 16 x bfloat> %va, <
 define <vscale x 16 x bfloat> @vfmadd_vf_nxv16bf16(<vscale x 16 x bfloat> %va, <vscale x 16 x bfloat> %vb, bfloat %c) {
 ; CHECK-LABEL: vfmadd_vf_nxv16bf16:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    fmv.x.h a0, fa0
-; CHECK-NEXT:    vsetvli a1, zero, e16, m4, ta, ma
+; CHECK-NEXT:    vsetvli a0, zero, e16, m4, ta, ma
 ; CHECK-NEXT:    vfwcvtbf16.f.f.v v16, v12
-; CHECK-NEXT:    vmv.v.x v12, a0
+; CHECK-NEXT:    fcvt.s.bf16 fa5, fa0
 ; CHECK-NEXT:    vfwcvtbf16.f.f.v v24, v8
-; CHECK-NEXT:    vfwcvtbf16.f.f.v v0, v12
 ; CHECK-NEXT:    vsetvli zero, zero, e32, m8, ta, ma
-; CHECK-NEXT:    vfmadd.vv v0, v24, v16
+; CHECK-NEXT:    vfmadd.vf v24, fa5, v16
 ; CHECK-NEXT:    vsetvli zero, zero, e16, m4, ta, ma
-; CHECK-NEXT:    vfncvtbf16.f.f.w v8, v0
+; CHECK-NEXT:    vfncvtbf16.f.f.w v8, v24
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 16 x bfloat> poison, bfloat %c, i32 0
   %splat = shufflevector <vscale x 16 x bfloat> %head, <vscale x 16 x bfloat> poison, <vscale x 16 x i32> zeroinitializer
@@ -640,16 +630,14 @@ define <vscale x 1 x half> @vfmadd_vf_nxv1f16(<vscale x 1 x half> %va, <vscale x
 ;
 ; ZVFHMIN-LABEL: vfmadd_vf_nxv1f16:
 ; ZVFHMIN:       # %bb.0:
-; ZVFHMIN-NEXT:    fmv.x.h a0, fa0
-; ZVFHMIN-NEXT:    vsetvli a1, zero, e16, mf4, ta, ma
+; ZVFHMIN-NEXT:    vsetvli a0, zero, e16, mf4, ta, ma
 ; ZVFHMIN-NEXT:    vfwcvt.f.f.v v10, v9
-; ZVFHMIN-NEXT:    vmv.v.x v9, a0
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v11, v8
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v12, v9
+; ZVFHMIN-NEXT:    fcvt.s.h fa5, fa0
+; ZVFHMIN-NEXT:    vfwcvt.f.f.v v9, v8
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, mf2, ta, ma
-; ZVFHMIN-NEXT:    vfmadd.vv v12, v11, v10
+; ZVFHMIN-NEXT:    vfmadd.vf v9, fa5, v10
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e16, mf4, ta, ma
-; ZVFHMIN-NEXT:    vfncvt.f.f.w v8, v12
+; ZVFHMIN-NEXT:    vfncvt.f.f.w v8, v9
 ; ZVFHMIN-NEXT:    ret
   %head = insertelement <vscale x 1 x half> poison, half %c, i32 0
   %splat = shufflevector <vscale x 1 x half> %head, <vscale x 1 x half> poison, <vscale x 1 x i32> zeroinitializer
@@ -690,16 +678,14 @@ define <vscale x 2 x half> @vfmadd_vf_nxv2f16(<vscale x 2 x half> %va, <vscale x
 ;
 ; ZVFHMIN-LABEL: vfmadd_vf_nxv2f16:
 ; ZVFHMIN:       # %bb.0:
-; ZVFHMIN-NEXT:    fmv.x.h a0, fa0
-; ZVFHMIN-NEXT:    vsetvli a1, zero, e16, mf2, ta, ma
+; ZVFHMIN-NEXT:    vsetvli a0, zero, e16, mf2, ta, ma
 ; ZVFHMIN-NEXT:    vfwcvt.f.f.v v10, v8
-; ZVFHMIN-NEXT:    vmv.v.x v8, a0
+; ZVFHMIN-NEXT:    fcvt.s.h fa5, fa0
 ; ZVFHMIN-NEXT:    vfwcvt.f.f.v v11, v9
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v9, v8
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
-; ZVFHMIN-NEXT:    vfmadd.vv v9, v11, v10
+; ZVFHMIN-NEXT:    vfmadd.vf v11, fa5, v10
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e16, mf2, ta, ma
-; ZVFHMIN-NEXT:    vfncvt.f.f.w v8, v9
+; ZVFHMIN-NEXT:    vfncvt.f.f.w v8, v11
 ; ZVFHMIN-NEXT:    ret
   %head = insertelement <vscale x 2 x half> poison, half %c, i32 0
   %splat = shufflevector <vscale x 2 x half> %head, <vscale x 2 x half> poison, <vscale x 2 x i32> zeroinitializer
@@ -740,16 +726,14 @@ define <vscale x 4 x half> @vfmadd_vf_nxv4f16(<vscale x 4 x half> %va, <vscale x
 ;
 ; ZVFHMIN-LABEL: vfmadd_vf_nxv4f16:
 ; ZVFHMIN:       # %bb.0:
-; ZVFHMIN-NEXT:    fmv.x.h a0, fa0
-; ZVFHMIN-NEXT:    vsetvli a1, zero, e16, m1, ta, ma
+; ZVFHMIN-NEXT:    vsetvli a0, zero, e16, m1, ta, ma
 ; ZVFHMIN-NEXT:    vfwcvt.f.f.v v10, v9
-; ZVFHMIN-NEXT:    vmv.v.x v9, a0
+; ZVFHMIN-NEXT:    fcvt.s.h fa5, fa0
 ; ZVFHMIN-NEXT:    vfwcvt.f.f.v v12, v8
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v14, v9
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
-; ZVFHMIN-NEXT:    vfmadd.vv v14, v12, v10
+; ZVFHMIN-NEXT:    vfmadd.vf v12, fa5, v10
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e16, m1, ta, ma
-; ZVFHMIN-NEXT:    vfncvt.f.f.w v8, v14
+; ZVFHMIN-NEXT:    vfncvt.f.f.w v8, v12
 ; ZVFHMIN-NEXT:    ret
   %head = insertelement <vscale x 4 x half> poison, half %c, i32 0
   %splat = shufflevector <vscale x 4 x half> %head, <vscale x 4 x half> poison, <vscale x 4 x i32> zeroinitializer
@@ -790,16 +774,14 @@ define <vscale x 8 x half> @vfmadd_vf_nxv8f16(<vscale x 8 x half> %va, <vscale x
 ;
 ; ZVFHMIN-LABEL: vfmadd_vf_nxv8f16:
 ; ZVFHMIN:       # %bb.0:
-; ZVFHMIN-NEXT:    fmv.x.h a0, fa0
-; ZVFHMIN-NEXT:    vsetvli a1, zero, e16, m2, ta, ma
+; ZVFHMIN-NEXT:    vsetvli a0, zero, e16, m2, ta, ma
 ; ZVFHMIN-NEXT:    vfwcvt.f.f.v v12, v8
-; ZVFHMIN-NEXT:    vmv.v.x v8, a0
+; ZVFHMIN-NEXT:    fcvt.s.h fa5, fa0
 ; ZVFHMIN-NEXT:    vfwcvt.f.f.v v16, v10
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v20, v8
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
-; ZVFHMIN-NEXT:    vfmadd.vv v20, v16, v12
+; ZVFHMIN-NEXT:    vfmadd.vf v16, fa5, v12
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e16, m2, ta, ma
-; ZVFHMIN-NEXT:    vfncvt.f.f.w v8, v20
+; ZVFHMIN-NEXT:    vfncvt.f.f.w v8, v16
 ; ZVFHMIN-NEXT:    ret
   %head = insertelement <vscale x 8 x half> poison, half %c, i32 0
   %splat = shufflevector <vscale x 8 x half> %head, <vscale x 8 x half> poison, <vscale x 8 x i32> zeroinitializer
@@ -840,16 +822,14 @@ define <vscale x 16 x half> @vfmadd_vf_nxv16f16(<vscale x 16 x half> %va, <vscal
 ;
 ; ZVFHMIN-LABEL: vfmadd_vf_nxv16f16:
 ; ZVFHMIN:       # %bb.0:
-; ZVFHMIN-NEXT:    fmv.x.h a0, fa0
-; ZVFHMIN-NEXT:    vsetvli a1, zero, e16, m4, ta, ma
+; ZVFHMIN-NEXT:    vsetvli a0, zero, e16, m4, ta, ma
 ; ZVFHMIN-NEXT:    vfwcvt.f.f.v v16, v12
-; ZVFHMIN-NEXT:    vmv.v.x v12, a0
+; ZVFHMIN-NEXT:    fcvt.s.h fa5, fa0
 ; ZVFHMIN-NEXT:    vfwcvt.f.f.v v24, v8
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v0, v12
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m8, ta, ma
-; ZVFHMIN-NEXT:    vfmadd.vv v0, v24, v16
+; ZVFHMIN-NEXT:    vfmadd.vf v24, fa5, v16
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e16, m4, ta, ma
-; ZVFHMIN-NEXT:    vfncvt.f.f.w v8, v0
+; ZVFHMIN-NEXT:    vfncvt.f.f.w v8, v24
 ; ZVFHMIN-NEXT:    ret
   %head = insertelement <vscale x 16 x half> poison, half %c, i32 0
   %splat = shufflevector <vscale x 16 x half> %head, <vscale x 16 x half> poison, <vscale x 16 x i32> zeroinitializer
diff --git a/llvm/test/CodeGen/RISCV/rvv/vfmax-sdnode.ll b/llvm/test/CodeGen/RISCV/rvv/vfmax-sdnode.ll
index db034fb4862d8..395eebb44ebb9 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vfmax-sdnode.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vfmax-sdnode.ll
@@ -30,13 +30,11 @@ define <vscale x 1 x bfloat> @vfmax_nxv1bf16_vv(<vscale x 1 x bfloat> %a, <vscal
 define <vscale x 1 x bfloat> @vfmax_nxv1bf16_vf(<vscale x 1 x bfloat> %a, bfloat %b) {
 ; CHECK-LABEL: vfmax_nxv1bf16_vf:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    fmv.x.h a0, fa0
-; CHECK-NEXT:    vsetvli a1, zero, e16, mf4, ta, ma
-; CHECK-NEXT:    vmv.v.x v9, a0
-; CHECK-NEXT:    vfwcvtbf16.f.f.v v10, v8
-; CHECK-NEXT:    vfwcvtbf16.f.f.v v8, v9
+; CHECK-NEXT:    fcvt.s.bf16 fa5, fa0
+; CHECK-NEXT:    vsetvli a0, zero, e16, mf4, ta, ma
+; CHECK-NEXT:    vfwcvtbf16.f.f.v v9, v8
 ; CHECK-NEXT:    vsetvli zero, zero, e32, mf2, ta, ma
-; CHECK-NEXT:    vfmax.vv v9, v10, v8
+; CHECK-NEXT:    vfmax.vf v9, v9, fa5
 ; CHECK-NEXT:    vsetvli zero, zero, e16, mf4, ta, ma
 ; CHECK-NEXT:    vfncvtbf16.f.f.w v8, v9
 ; CHECK-NEXT:    ret
@@ -66,13 +64,11 @@ define <vscale x 2 x bfloat> @vfmax_nxv2bf16_vv(<vscale x 2 x bfloat> %a, <vscal
 define <vscale x 2 x bfloat> @vfmax_nxv2bf16_vf(<vscale x 2 x bfloat> %a, bfloat %b) {
 ; CHECK-LABEL: vfmax_nxv2bf16_vf:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    fmv.x.h a0, fa0
-; CHECK-NEXT:    vsetvli a1, zero, e16, mf2, ta, ma
-; CHECK-NEXT:    vmv.v.x v9, a0
-; CHECK-NEXT:    vfwcvtbf16.f.f.v v10, v8
-; CHECK-NEXT:    vfwcvtbf16.f.f.v v8, v9
+; CHECK-NEXT:    fcvt.s.bf16 fa5, fa0
+; CHECK-NEXT:    vsetvli a0, zero, e16, mf2, ta, ma
+; CHECK-NEXT:    vfwcvtbf16.f.f.v v9, v8
 ; CHECK-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
-; CHECK-NEXT:    vfmax.vv v9, v10, v8
+; CHECK-NEXT:    vfmax.vf v9, v9, fa5
 ; CHECK-NEXT:    vsetvli zero, zero, e16, mf2, ta, ma
 ; CHECK-NEXT:    vfncvtbf16.f.f.w v8, v9
 ; CHECK-NEXT:    ret
@@ -102,13 +98,11 @@ define <vscale x 4 x bfloat> @vfmax_nxv4bf16_vv(<vscale x 4 x bfloat> %a, <vscal
 define <vscale x 4 x bfloat> @vfmax_nxv4bf16_vf(<vscale x 4 x bfloat> %a, bfloat %b) {
 ; CHECK-LABEL: vfmax_nxv4bf16_vf:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    fmv.x.h a0, fa0
-; CHECK-NEXT:    vsetvli a1, zero, e16, m1, ta, ma
-; CHECK-NEXT:    vmv.v.x v9, a0
+; CHECK-NEXT:    fcvt.s.bf16 fa5, fa0
+; CHECK-NEXT:    vsetvli a0, zero, e16, m1, ta, ma
 ; CHECK-NEXT:    vfwcvtbf16.f.f.v v10, v8
-; CHECK-NEXT:    vfwcvtbf16.f.f.v v12, v9
 ; CHECK-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
-; CHECK-NEXT:    vfmax.vv v10, v10, v12
+; CHECK-NEXT:    vfmax.vf v10, v10, fa5
 ; CHECK-NEXT:    vsetvli zero, zero, e16, m1, ta, ma
 ; CHECK-NEXT:    vfncvtbf16.f.f.w v8, v10
 ; CHECK-NEXT:    ret
@@ -138,13 +132,11 @@ define <vscale x 8 x bfloat> @vfmax_nxv8bf16_vv(<vscale x 8 x bfloat> %a, <vscal
 define <vscale x 8 x bfloat> @vfmax_nxv8bf16_vf(<vscale x 8 x bfloat> %a, bfloat %b) {
 ; CHECK-LABEL: vfmax_nxv8bf16_vf:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    fmv.x.h a0, fa0
-; CHECK-NEXT:    vsetvli a1, zero, e16, m2, ta, ma
-; CHECK-NEXT:    vmv.v.x v10, a0
+; CHECK-NEXT:    fcvt.s.bf16 fa5, fa0
+; CHECK-NEXT:    vsetvli a0, zero, e16, m2, ta, ma
 ; CHECK-NEXT:    vfwcvtbf16.f.f.v v12, v8
-; CHECK-NEXT:    vfwcvtbf16.f.f.v v16, v10
 ; CHECK-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
-; CHECK-NEXT:    vfmax.vv v12, v12, v16
+; CHECK-NEXT:    vfmax.vf v12, v12, fa5
 ; CHECK-NEXT:    vsetvli zero, zero, e16, m2, ta, ma
 ; CHECK-NEXT:    vfncvtbf16.f.f.w v8, v12
 ; CHECK-NEXT:    ret
@@ -174,13 +166,11 @@ define <vscale x 16 x bfloat> @vfmax_nxv16bf16_vv(<vscale x 16 x bfloat> %a, <vs
 define <vscale x 16 x bfloat> @vfmax_nxv16bf16_vf(<vscale x 16 x bfloat> %a, bfloat %b) {
 ; CHECK-LABEL: vfmax_nxv16bf16_vf:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    fmv.x.h a0, fa0
-; CHECK-NEXT:    vsetvli a1, zero, e16, m4, ta, ma
-; CHECK-NEXT:    vmv.v.x v12, a0
+; CHECK-NEXT:    fcvt.s.bf16 fa5, fa0
+; CHECK-NEXT:    vsetvli a0, zero, e16, m4, ta, ma
 ; CHECK-NEXT:    vfwcvtbf16.f.f.v v16, v8
-; CHECK-NEXT:    vfwcvtbf16.f.f.v v24, v12
 ; CHECK-NEXT:    vsetvli zero, zero, e32, m8, ta, ma
-; CHECK-NEXT:    vfmax.vv v16, v16, v24
+; CHECK-NEXT:    vfmax.vf v16, v16, fa5
 ; CHECK-NEXT:    vsetvli zero, zero, e16, m4, ta, ma
 ; CHECK-NEXT:    vfncvtbf16.f.f.w v8, v16
 ; CHECK-NEXT:    ret
@@ -303,13 +293,11 @@ define <vscale x 1 x half> @vfmax_nxv1f16_vf(<vscale x 1 x half> %a, half %b) {
 ;
 ; ZVFHMIN-LABEL: vfmax_nxv1f16_vf:
 ; ZVFHMIN:       # %bb.0:
-; ZVFHMIN-NEXT:    fmv.x.h a0, fa0
-; ZVFHMIN-NEXT:    vsetvli a1, zero, e16, mf4, ta, ma
-; ZVFHMIN-NEXT:    vmv.v.x v9, a0
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v10, v8
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v8, v9
+; ZVFHMIN-NEXT:    fcvt.s.h fa5, fa0
+; ZVFHMIN-NEXT:    vsetvli a0, zero, e16, mf4, ta, ma
+; ZVFHMIN-NEXT:    vfwcvt.f.f.v v9, v8
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, mf2, ta, ma
-; ZVFHMIN-NEXT:    vfmax.vv v9, v10, v8
+; ZVFHMIN-NEXT:    vfmax.vf v9, v9, fa5
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e16, mf4, ta, ma
 ; ZVFHMIN-NEXT:    vfncvt.f.f.w v8, v9
 ; ZVFHMIN-NEXT:    ret
@@ -351,13 +339,11 @@ define <vscale x 2 x half> @vfmax_nxv2f16_vf(<vscale x 2 x half> %a, half %b) {
 ;
 ; ZVFHMIN-LABEL: vfmax_nxv2f16_vf:
 ; ZVFHMIN:       # %bb.0:
-; ZVFHMIN-NEXT:    fmv.x.h a0, fa0
-; ZVFHMIN-NEXT:    vsetvli a1, zero, e16, mf2, ta, ma
-; ZVFHMIN-NEXT:    vmv.v.x v9, a0
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v10, v8
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v8, v9
+; ZVFHMIN-NEXT:    fcvt.s.h fa5, fa0
+; ZVFHMIN-NEXT:    vsetvli a0, zero, e16, mf2, ta, ma
+; ZVFHMIN-NEXT:    vfwcvt.f.f.v v9, v8
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
-; ZVFHMIN-NEXT:    vfmax.vv v9, v10, v8
+; ZVFHMIN-NEXT:    vfmax.vf v9, v9, fa5
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e16, mf2, ta, ma
 ; ZVFHMIN-NEXT:    vfncvt.f.f.w v8, v9
 ; ZVFHMIN-NEXT:    ret
@@ -399,13 +385,11 @@ define <vscale x 4 x half> @vfmax_nxv4f16_vf(<vscale x 4 x half> %a, half %b) {
 ;
 ; ZVFHMIN-LABEL: vfmax_nxv4f16_vf:
 ; ZVFHMIN:       # %bb.0:
-; ZVFHMIN-NEXT:    fmv.x.h a0, fa0
-; ZVFHMIN-NEXT:    vsetvli a1, zero, e16, m1, ta, ma
-; ZVFHMIN-NEXT:    vmv.v.x v9, a0
+; ZVFHMIN-NEXT:    fcvt.s.h fa5, fa0
+; ZVFHMIN-NEXT:    vsetvli a0, zero, e16, m1, ta, ma
 ; ZVFHMIN-NEXT:    vfwcvt.f.f.v v10, v8
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v12, v9
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
-; ZVFHMIN-NEXT:    vfmax.vv v10, v10, v12
+; ZVFHMIN-NEXT:    vfmax.vf v10, v10, fa5
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e16, m1, ta, ma
 ; ZVFHMIN-NEXT:    vfncvt.f.f.w v8, v10
 ; ZVFHMIN-NEXT:    ret
@@ -447,13 +431,11 @@ define <vscale x 8 x half> @vfmax_nxv8f16_vf(<vscale x 8 x half> %a, half %b) {
 ;
 ; ZVFHMIN-LABEL: vfmax_nxv8f16_vf:
 ; ZVFHMIN:       # %bb.0:
-; ZVFHMIN-NEXT:    fmv.x.h a0, fa0
-; ZVFHMIN-NEXT:    vsetvli a1, zero, e16, m2, ta, ma
-; ZVFHMIN-NEXT:    vmv.v.x v10, a0
+; ZVFHMIN-NEXT:    fcvt.s.h fa5, fa0
+; ZVFHMIN-NEXT:    vsetvli a0, zero, e16, m2, ta, ma
 ; ZVFHMIN-NEXT:    vfwcvt.f.f.v v12, v8
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v16, v10
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
-; ZVFHMIN-NEXT:    vfmax.vv v12, v12, v16
+; ZVFHMIN-NEXT:    vfmax.vf v12, v12, fa5
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e16, m2, ta, ma
 ; ZVFHMIN-NEXT:    vfncvt.f.f.w v8, v12
 ; ZVFHMIN-NEXT:    ret
@@ -495,13 +477,11 @@ define <vscale x 16 x half> @vfmax_nxv16f16_vf(<vscale x 16 x half> %a, half %b)
 ;
 ; ZVFHMIN-LABEL: vfmax_nxv16f16_vf:
 ; ZVFHMIN:       # %bb.0:
-; ZVFHMIN-NEXT:    fmv.x.h a0, fa0
-; ZVFHMIN-NEXT:    vsetvli a1, zero, e16, m4, ta, ma
-; ZVFHMIN-NEXT:    vmv.v.x v12, a0
+; ZVFHMIN-NEXT:    fcvt.s.h fa5, fa0
+; ZVFHMIN-NEXT:    vsetvli a0, zero, e16, m4, ta, ma
 ; ZVFHMIN-NEXT:    vfwcvt.f.f.v v16, v8
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v24, v12
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m8, ta, ma
-; ZVFHMIN-NEXT:    vfmax.vv v16, v16, v24
+; ZVFHMIN-NEXT:    vfmax.vf v16, v16, fa5
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e16, m4, ta, ma
 ; ZVFHMIN-NEXT:    vfncvt.f.f.w v8, v16
 ; ZVFHMIN-NEXT:    ret
diff --git a/llvm/test/CodeGen/RISCV/rvv/vfmin-sdnode.ll b/llvm/test/CodeGen/RISCV/rvv/vfmin-sdnode.ll
index 3ee82c33485f6..283c510cf56cd 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vfmin-sdnode.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vfmin-sdnode.ll
@@ -30,13 +30,11 @@ define <vscale x 1 x bfloat> @vfmin_nxv1bf16_vv(<vscale x 1 x bfloat> %a, <vscal
 define <vscale x 1 x bfloat> @vfmin_nxv1bf16_vf(<vscale x 1 x bfloat> %a, bfloat %b) {
 ; CHECK-LABEL: vfmin_nxv1bf16_vf:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    fmv.x.h a0, fa0
-; CHECK-NEXT:    vsetvli a1, zero, e16, mf4, ta, ma
-; CHECK-NEXT:    vmv.v.x v9, a0
-; CHECK-NEXT:    vfwcvtbf16.f.f.v v10, v8
-; CHECK-NEXT:    vfwcvtbf16.f.f.v v8, v9
+; CHECK-NEXT:    fcvt.s.bf16 fa5, fa0
+; CHECK-NEXT:    vsetvli a0, zero, e16, mf4, ta, ma
+; CHECK-NEXT:    vfwcvtbf16.f.f.v v9, v8
 ; CHECK-NEXT:    vsetvli zero, zero, e32, mf2, ta, ma
-; CHECK-NEXT:    vfmin.vv v9, v10, v8
+; CHECK-NEXT:    vfmin.vf v9, v9, fa5
 ; CHECK-NEXT:    vsetvli zero, zero, e16, mf4, ta, ma
 ; CHECK-NEXT:    vfncvtbf16.f.f.w v8, v9
 ; CHECK-NEXT:    ret
@@ -66,13 +64,11 @@ define <vscale x 2 x bfloat> @vfmin_nxv2bf16_vv(<vscale x 2 x bfloat> %a, <vscal
 define <vscale x 2 x bfloat> @vfmin_nxv2bf16_vf(<vscale x 2 x bfloat> %a, bfloat %b) {
 ; CHECK-LABEL: vfmin_nxv2bf16_vf:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    fmv.x.h a0, fa0
-; CHECK-NEXT:    vsetvli a1, zero, e16, mf2, ta, ma
-; CHECK-NEXT:    vmv.v.x v9, a0
-; CHECK-NEXT:    vfwcvtbf16.f.f.v v10, v8
-; CHECK-NEXT:    vfwcvtbf16.f.f.v v8, v9
+; CHECK-NEXT:    fcvt.s.bf16 fa5, fa0
+; CHECK-NEXT:    vsetvli a0, zero, e16, mf2, ta, ma
+; CHECK-NEXT:    vfwcvtbf16.f.f.v v9, v8
 ; CHECK-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
-; CHECK-NEXT:    vfmin.vv v9, v10, v8
+; CHECK-NEXT:    vfmin.vf v9, v9, fa5
 ; CHECK-NEXT:    vsetvli zero, zero, e16, mf2, ta, ma
 ; CHECK-NEXT:    vfncvtbf16.f.f.w v8, v9
 ; CHECK-NEXT:    ret
@@ -102,13 +98,11 @@ define <vscale x 4 x bfloat> @vfmin_nxv4bf16_vv(<vscale x 4 x bfloat> %a, <vscal
 define <vscale x 4 x bfloat> @vfmin_nxv4bf16_vf(<vscale x 4 x bfloat> %a, bfloat %b) {
 ; CHECK-LABEL: vfmin_nxv4bf16_vf:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    fmv.x.h a0, fa0
-; CHECK-NEXT:    vsetvli a1, zero, e16, m1, ta, ma
-; CHECK-NEXT:    vmv.v.x v9, a0
+; CHECK-NEXT:    fcvt.s.bf16 fa5, fa0
+; CHECK-NEXT:    vsetvli a0, zero, e16, m1, ta, ma
 ; CHECK-NEXT:    vfwcvtbf16.f.f.v v10, v8
-; CHECK-NEXT:    vfwcvtbf16.f.f.v v12, v9
 ; CHECK-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
-; CHECK-NEXT:    vfmin.vv v10, v10, v12
+; CHECK-NEXT:    vfmin.vf v10, v10, fa5
 ; CHECK-NEXT:    vsetvli zero, zero, e16, m1, ta, ma
 ; CHECK-NEXT:    vfncvtbf16.f.f.w v8, v10
 ; CHECK-NEXT:    ret
@@ -138,13 +132,11 @@ define <vscale x 8 x bfloat> @vfmin_nxv8bf16_vv(<vscale x 8 x bfloat> %a, <vscal
 define <vscale x 8 x bfloat> @vfmin_nxv8bf16_vf(<vscale x 8 x bfloat> %a, bfloat %b) {
 ; CHECK-LABEL: vfmin_nxv8bf16_vf:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    fmv.x.h a0, fa0
-; CHECK-NEXT:    vsetvli a1, zero, e16, m2, ta, ma
-; CHECK-NEXT:    vmv.v.x v10, a0
+; CHECK-NEXT:    fcvt.s.bf16 fa5, fa0
+; CHECK-NEXT:    vsetvli a0, zero, e16, m2, ta, ma
 ; CHECK-NEXT:    vfwcvtbf16.f.f.v v12, v8
-; CHECK-NEXT:    vfwcvtbf16.f.f.v v16, v10
 ; CHECK-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
-; CHECK-NEXT:    vfmin.vv v12, v12, v16
+; CHECK-NEXT:    vfmin.vf v12, v12, fa5
 ; CHECK-NEXT:    vsetvli zero, zero, e16, m2, ta, ma
 ; CHECK-NEXT:    vfncvtbf16.f.f.w v8, v12
 ; CHECK-NEXT:    ret
@@ -174,13 +166,11 @@ define <vscale x 16 x bfloat> @vfmin_nxv16bf16_vv(<vscale x 16 x bfloat> %a, <vs
 define <vscale x 16 x bfloat> @vfmin_nxv16bf16_vf(<vscale x 16 x bfloat> %a, bfloat %b) {
 ; CHECK-LABEL: vfmin_nxv16bf16_vf:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    fmv.x.h a0, fa0
-; CHECK-NEXT:    vsetvli a1, zero, e16, m4, ta, ma
-; CHECK-NEXT:    vmv.v.x v12, a0
+; CHECK-NEXT:    fcvt.s.bf16 fa5, fa0
+; CHECK-NEXT:    vsetvli a0, zero, e16, m4, ta, ma
 ; CHECK-NEXT:    vfwcvtbf16.f.f.v v16, v8
-; CHECK-NEXT:    vfwcvtbf16.f.f.v v24, v12
 ; CHECK-NEXT:    vsetvli zero, zero, e32, m8, ta, ma
-; CHECK-NEXT:    vfmin.vv v16, v16, v24
+; CHECK-NEXT:    vfmin.vf v16, v16, fa5
 ; CHECK-NEXT:    vsetvli zero, zero, e16, m4, ta, ma
 ; CHECK-NEXT:    vfncvtbf16.f.f.w v8, v16
 ; CHECK-NEXT:    ret
@@ -303,13 +293,11 @@ define <vscale x 1 x half> @vfmin_nxv1f16_vf(<vscale x 1 x half> %a, half %b) {
 ;
 ; ZVFHMIN-LABEL: vfmin_nxv1f16_vf:
 ; ZVFHMIN:       # %bb.0:
-; ZVFHMIN-NEXT:    fmv.x.h a0, fa0
-; ZVFHMIN-NEXT:    vsetvli a1, zero, e16, mf4, ta, ma
-; ZVFHMIN-NEXT:    vmv.v.x v9, a0
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v10, v8
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v8, v9
+; ZVFHMIN-NEXT:    fcvt.s.h fa5, fa0
+; ZVFHMIN-NEXT:    vsetvli a0, zero, e16, mf4, ta, ma
+; ZVFHMIN-NEXT:    vfwcvt.f.f.v v9, v8
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, mf2, ta, ma
-; ZVFHMIN-NEXT:    vfmin.vv v9, v10, v8
+; ZVFHMIN-NEXT:    vfmin.vf v9, v9, fa5
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e16, mf4, ta, ma
 ; ZVFHMIN-NEXT:    vfncvt.f.f.w v8, v9
 ; ZVFHMIN-NEXT:    ret
@@ -351,13 +339,11 @@ define <vscale x 2 x half> @vfmin_nxv2f16_vf(<vscale x 2 x half> %a, half %b) {
 ;
 ; ZVFHMIN-LABEL: vfmin_nxv2f16_vf:
 ; ZVFHMIN:       # %bb.0:
-; ZVFHMIN-NEXT:    fmv.x.h a0, fa0
-; ZVFHMIN-NEXT:    vsetvli a1, zero, e16, mf2, ta, ma
-; ZVFHMIN-NEXT:    vmv.v.x v9, a0
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v10, v8
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v8, v9
+; ZVFHMIN-NEXT:    fcvt.s.h fa5, fa0
+; ZVFHMIN-NEXT:    vsetvli a0, zero, e16, mf2, ta, ma
+; ZVFHMIN-NEXT:    vfwcvt.f.f.v v9, v8
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
-; ZVFHMIN-NEXT:    vfmin.vv v9, v10, v8
+; ZVFHMIN-NEXT:    vfmin.vf v9, v9, fa5
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e16, mf2, ta, ma
 ; ZVFHMIN-NEXT:    vfncvt.f.f.w v8, v9
 ; ZVFHMIN-NEXT:    ret
@@ -399,13 +385,11 @@ define <vscale x 4 x half> @vfmin_nxv4f16_vf(<vscale x 4 x half> %a, half %b) {
 ;
 ; ZVFHMIN-LABEL: vfmin_nxv4f16_vf:
 ; ZVFHMIN:       # %bb.0:
-; ZVFHMIN-NEXT:    fmv.x.h a0, fa0
-; ZVFHMIN-NEXT:    vsetvli a1, zero, e16, m1, ta, ma
-; ZVFHMIN-NEXT:    vmv.v.x v9, a0
+; ZVFHMIN-NEXT:    fcvt.s.h fa5, fa0
+; ZVFHMIN-NEXT:    vsetvli a0, zero, e16, m1, ta, ma
 ; ZVFHMIN-NEXT:    vfwcvt.f.f.v v10, v8
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v12, v9
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
-; ZVFHMIN-NEXT:    vfmin.vv v10, v10, v12
+; ZVFHMIN-NEXT:    vfmin.vf v10, v10, fa5
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e16, m1, ta, ma
 ; ZVFHMIN-NEXT:    vfncvt.f.f.w v8, v10
 ; ZVFHMIN-NEXT:    ret
@@ -447,13 +431,11 @@ define <vscale x 8 x half> @vfmin_nxv8f16_vf(<vscale x 8 x half> %a, half %b) {
 ;
 ; ZVFHMIN-LABEL: vfmin_nxv8f16_vf:
 ; ZVFHMIN:       # %bb.0:
-; ZVFHMIN-NEXT:    fmv.x.h a0, fa0
-; ZVFHMIN-NEXT:    vsetvli a1, zero, e16, m2, ta, ma
-; ZVFHMIN-NEXT:    vmv.v.x v10, a0
+; ZVFHMIN-NEXT:    fcvt.s.h fa5, fa0
+; ZVFHMIN-NEXT:    vsetvli a0, zero, e16, m2, ta, ma
 ; ZVFHMIN-NEXT:    vfwcvt.f.f.v v12, v8
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v16, v10
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
-; ZVFHMIN-NEXT:    vfmin.vv v12, v12, v16
+; ZVFHMIN-NEXT:    vfmin.vf v12, v12, fa5
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e16, m2, ta, ma
 ; ZVFHMIN-NEXT:    vfncvt.f.f.w v8, v12
 ; ZVFHMIN-NEXT:    ret
@@ -495,13 +477,11 @@ define <vscale x 16 x half> @vfmin_nxv16f16_vf(<vscale x 16 x half> %a, half %b)
 ;
 ; ZVFHMIN-LABEL: vfmin_nxv16f16_vf:
 ; ZVFHMIN:       # %bb.0:
-; ZVFHMIN-NEXT:    fmv.x.h a0, fa0
-; ZVFHMIN-NEXT:    vsetvli a1, zero, e16, m4, ta, ma
-; ZVFHMIN-NEXT:    vmv.v.x v12, a0
+; ZVFHMIN-NEXT:    fcvt.s.h fa5, fa0
+; ZVFHMIN-NEXT:    vsetvli a0, zero, e16, m4, ta, ma
 ; ZVFHMIN-NEXT:    vfwcvt.f.f.v v16, v8
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v24, v12
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m8, ta, ma
-; ZVFHMIN-NEXT:    vfmin.vv v16, v16, v24
+; ZVFHMIN-NEXT:    vfmin.vf v16, v16, fa5
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e16, m4, ta, ma
 ; ZVFHMIN-NEXT:    vfncvt.f.f.w v8, v16
 ; ZVFHMIN-NEXT:    ret
diff --git a/llvm/test/CodeGen/RISCV/rvv/vfmul-sdnode.ll b/llvm/test/CodeGen/RISCV/rvv/vfmul-sdnode.ll
index fb8ed3f943fff..6fc9ccb3b3e39 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vfmul-sdnode.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vfmul-sdnode.ll
@@ -30,13 +30,11 @@ define <vscale x 1 x bfloat> @vfmul_vv_nxv1bf16(<vscale x 1 x bfloat> %va, <vsca
 define <vscale x 1 x bfloat> @vfmul_vf_nxv1bf16(<vscale x 1 x bfloat> %va, bfloat %b) {
 ; CHECK-LABEL: vfmul_vf_nxv1bf16:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    fmv.x.h a0, fa0
-; CHECK-NEXT:    vsetvli a1, zero, e16, mf4, ta, ma
-; CHECK-NEXT:    vmv.v.x v9, a0
-; CHECK-NEXT:    vfwcvtbf16.f.f.v v10, v8
-; CHECK-NEXT:    vfwcvtbf16.f.f.v v8, v9
+; CHECK-NEXT:    fcvt.s.bf16 fa5, fa0
+; CHECK-NEXT:    vsetvli a0, zero, e16, mf4, ta, ma
+; CHECK-NEXT:    vfwcvtbf16.f.f.v v9, v8
 ; CHECK-NEXT:    vsetvli zero, zero, e32, mf2, ta, ma
-; CHECK-NEXT:    vfmul.vv v9, v10, v8
+; CHECK-NEXT:    vfmul.vf v9, v9, fa5
 ; CHECK-NEXT:    vsetvli zero, zero, e16, mf4, ta, ma
 ; CHECK-NEXT:    vfncvtbf16.f.f.w v8, v9
 ; CHECK-NEXT:    ret
@@ -64,13 +62,11 @@ define <vscale x 2 x bfloat> @vfmul_vv_nxv2bf16(<vscale x 2 x bfloat> %va, <vsca
 define <vscale x 2 x bfloat> @vfmul_vf_nxv2bf16(<vscale x 2 x bfloat> %va, bfloat %b) {
 ; CHECK-LABEL: vfmul_vf_nxv2bf16:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    fmv.x.h a0, fa0
-; CHECK-NEXT:    vsetvli a1, zero, e16, mf2, ta, ma
-; CHECK-NEXT:    vmv.v.x v9, a0
-; CHECK-NEXT:    vfwcvtbf16.f.f.v v10, v8
-; CHECK-NEXT:    vfwcvtbf16.f.f.v v8, v9
+; CHECK-NEXT:    fcvt.s.bf16 fa5, fa0
+; CHECK-NEXT:    vsetvli a0, zero, e16, mf2, ta, ma
+; CHECK-NEXT:    vfwcvtbf16.f.f.v v9, v8
 ; CHECK-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
-; CHECK-NEXT:    vfmul.vv v9, v10, v8
+; CHECK-NEXT:    vfmul.vf v9, v9, fa5
 ; CHECK-NEXT:    vsetvli zero, zero, e16, mf2, ta, ma
 ; CHECK-NEXT:    vfncvtbf16.f.f.w v8, v9
 ; CHECK-NEXT:    ret
@@ -98,13 +94,11 @@ define <vscale x 4 x bfloat> @vfmul_vv_nxv4bf16(<vscale x 4 x bfloat> %va, <vsca
 define <vscale x 4 x bfloat> @vfmul_vf_nxv4bf16(<vscale x 4 x bfloat> %va, bfloat %b) {
 ; CHECK-LABEL: vfmul_vf_nxv4bf16:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    fmv.x.h a0, fa0
-; CHECK-NEXT:    vsetvli a1, zero, e16, m1, ta, ma
-; CHECK-NEXT:    vmv.v.x v9, a0
+; CHECK-NEXT:    fcvt.s.bf16 fa5, fa0
+; CHECK-NEXT:    vsetvli a0, zero, e16, m1, ta, ma
 ; CHECK-NEXT:    vfwcvtbf16.f.f.v v10, v8
-; CHECK-NEXT:    vfwcvtbf16.f.f.v v12, v9
 ; CHECK-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
-; CHECK-NEXT:    vfmul.vv v10, v10, v12
+; CHECK-NEXT:    vfmul.vf v10, v10, fa5
 ; CHECK-NEXT:    vsetvli zero, zero, e16, m1, ta, ma
 ; CHECK-NEXT:    vfncvtbf16.f.f.w v8, v10
 ; CHECK-NEXT:    ret
@@ -132,13 +126,11 @@ define <vscale x 8 x bfloat> @vfmul_vv_nxv8bf16(<vscale x 8 x bfloat> %va, <vsca
 define <vscale x 8 x bfloat> @vfmul_vf_nxv8bf16(<vscale x 8 x bfloat> %va, bfloat %b) {
 ; CHECK-LABEL: vfmul_vf_nxv8bf16:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    fmv.x.h a0, fa0
-; CHECK-NEXT:    vsetvli a1, zero, e16, m2, ta, ma
-; CHECK-NEXT:    vmv.v.x v10, a0
+; CHECK-NEXT:    fcvt.s.bf16 fa5, fa0
+; CHECK-NEXT:    vsetvli a0, zero, e16, m2, ta, ma
 ; CHECK-NEXT:    vfwcvtbf16.f.f.v v12, v8
-; CHECK-NEXT:    vfwcvtbf16.f.f.v v16, v10
 ; CHECK-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
-; CHECK-NEXT:    vfmul.vv v12, v12, v16
+; CHECK-NEXT:    vfmul.vf v12, v12, fa5
 ; CHECK-NEXT:    vsetvli zero, zero, e16, m2, ta, ma
 ; CHECK-NEXT:    vfncvtbf16.f.f.w v8, v12
 ; CHECK-NEXT:    ret
@@ -151,13 +143,11 @@ define <vscale x 8 x bfloat> @vfmul_vf_nxv8bf16(<vscale x 8 x bfloat> %va, bfloa
 define <vscale x 8 x bfloat> @vfmul_fv_nxv8bf16(<vscale x 8 x bfloat> %va, bfloat %b) {
 ; CHECK-LABEL: vfmul_fv_nxv8bf16:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    fmv.x.h a0, fa0
-; CHECK-NEXT:    vsetvli a1, zero, e16, m2, ta, ma
-; CHECK-NEXT:    vmv.v.x v10, a0
+; CHECK-NEXT:    fcvt.s.bf16 fa5, fa0
+; CHECK-NEXT:    vsetvli a0, zero, e16, m2, ta, ma
 ; CHECK-NEXT:    vfwcvtbf16.f.f.v v12, v8
-; CHECK-NEXT:    vfwcvtbf16.f.f.v v16, v10
 ; CHECK-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
-; CHECK-NEXT:    vfmul.vv v12, v16, v12
+; CHECK-NEXT:    vfmul.vf v12, v12, fa5
 ; CHECK-NEXT:    vsetvli zero, zero, e16, m2, ta, ma
 ; CHECK-NEXT:    vfncvtbf16.f.f.w v8, v12
 ; CHECK-NEXT:    ret
@@ -185,13 +175,11 @@ define <vscale x 16 x bfloat> @vfmul_vv_nxv16bf16(<vscale x 16 x bfloat> %va, <v
 define <vscale x 16 x bfloat> @vfmul_vf_nxv16bf16(<vscale x 16 x bfloat> %va, bfloat %b) {
 ; CHECK-LABEL: vfmul_vf_nxv16bf16:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    fmv.x.h a0, fa0
-; CHECK-NEXT:    vsetvli a1, zero, e16, m4, ta, ma
-; CHECK-NEXT:    vmv.v.x v12, a0
+; CHECK-NEXT:    fcvt.s.bf16 fa5, fa0
+; CHECK-NEXT:    vsetvli a0, zero, e16, m4, ta, ma
 ; CHECK-NEXT:    vfwcvtbf16.f.f.v v16, v8
-; CHECK-NEXT:    vfwcvtbf16.f.f.v v24, v12
 ; CHECK-NEXT:    vsetvli zero, zero, e32, m8, ta, ma
-; CHECK-NEXT:    vfmul.vv v16, v16, v24
+; CHECK-NEXT:    vfmul.vf v16, v16, fa5
 ; CHECK-NEXT:    vsetvli zero, zero, e16, m4, ta, ma
 ; CHECK-NEXT:    vfncvtbf16.f.f.w v8, v16
 ; CHECK-NEXT:    ret
@@ -310,13 +298,11 @@ define <vscale x 1 x half> @vfmul_vf_nxv1f16(<vscale x 1 x half> %va, half %b) {
 ;
 ; ZVFHMIN-LABEL: vfmul_vf_nxv1f16:
 ; ZVFHMIN:       # %bb.0:
-; ZVFHMIN-NEXT:    fmv.x.h a0, fa0
-; ZVFHMIN-NEXT:    vsetvli a1, zero, e16, mf4, ta, ma
-; ZVFHMIN-NEXT:    vmv.v.x v9, a0
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v10, v8
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v8, v9
+; ZVFHMIN-NEXT:    fcvt.s.h fa5, fa0
+; ZVFHMIN-NEXT:    vsetvli a0, zero, e16, mf4, ta, ma
+; ZVFHMIN-NEXT:    vfwcvt.f.f.v v9, v8
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, mf2, ta, ma
-; ZVFHMIN-NEXT:    vfmul.vv v9, v10, v8
+; ZVFHMIN-NEXT:    vfmul.vf v9, v9, fa5
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e16, mf4, ta, ma
 ; ZVFHMIN-NEXT:    vfncvt.f.f.w v8, v9
 ; ZVFHMIN-NEXT:    ret
@@ -356,13 +342,11 @@ define <vscale x 2 x half> @vfmul_vf_nxv2f16(<vscale x 2 x half> %va, half %b) {
 ;
 ; ZVFHMIN-LABEL: vfmul_vf_nxv2f16:
 ; ZVFHMIN:       # %bb.0:
-; ZVFHMIN-NEXT:    fmv.x.h a0, fa0
-; ZVFHMIN-NEXT:    vsetvli a1, zero, e16, mf2, ta, ma
-; ZVFHMIN-NEXT:    vmv.v.x v9, a0
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v10, v8
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v8, v9
+; ZVFHMIN-NEXT:    fcvt.s.h fa5, fa0
+; ZVFHMIN-NEXT:    vsetvli a0, zero, e16, mf2, ta, ma
+; ZVFHMIN-NEXT:    vfwcvt.f.f.v v9, v8
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
-; ZVFHMIN-NEXT:    vfmul.vv v9, v10, v8
+; ZVFHMIN-NEXT:    vfmul.vf v9, v9, fa5
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e16, mf2, ta, ma
 ; ZVFHMIN-NEXT:    vfncvt.f.f.w v8, v9
 ; ZVFHMIN-NEXT:    ret
@@ -402,13 +386,11 @@ define <vscale x 4 x half> @vfmul_vf_nxv4f16(<vscale x 4 x half> %va, half %b) {
 ;
 ; ZVFHMIN-LABEL: vfmul_vf_nxv4f16:
 ; ZVFHMIN:       # %bb.0:
-; ZVFHMIN-NEXT:    fmv.x.h a0, fa0
-; ZVFHMIN-NEXT:    vsetvli a1, zero, e16, m1, ta, ma
-; ZVFHMIN-NEXT:    vmv.v.x v9, a0
+; ZVFHMIN-NEXT:    fcvt.s.h fa5, fa0
+; ZVFHMIN-NEXT:    vsetvli a0, zero, e16, m1, ta, ma
 ; ZVFHMIN-NEXT:    vfwcvt.f.f.v v10, v8
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v12, v9
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
-; ZVFHMIN-NEXT:    vfmul.vv v10, v10, v12
+; ZVFHMIN-NEXT:    vfmul.vf v10, v10, fa5
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e16, m1, ta, ma
 ; ZVFHMIN-NEXT:    vfncvt.f.f.w v8, v10
 ; ZVFHMIN-NEXT:    ret
@@ -448,13 +430,11 @@ define <vscale x 8 x half> @vfmul_vf_nxv8f16(<vscale x 8 x half> %va, half %b) {
 ;
 ; ZVFHMIN-LABEL: vfmul_vf_nxv8f16:
 ; ZVFHMIN:       # %bb.0:
-; ZVFHMIN-NEXT:    fmv.x.h a0, fa0
-; ZVFHMIN-NEXT:    vsetvli a1, zero, e16, m2, ta, ma
-; ZVFHMIN-NEXT:    vmv.v.x v10, a0
+; ZVFHMIN-NEXT:    fcvt.s.h fa5, fa0
+; ZVFHMIN-NEXT:    vsetvli a0, zero, e16, m2, ta, ma
 ; ZVFHMIN-NEXT:    vfwcvt.f.f.v v12, v8
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v16, v10
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
-; ZVFHMIN-NEXT:    vfmul.vv v12, v12, v16
+; ZVFHMIN-NEXT:    vfmul.vf v12, v12, fa5
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e16, m2, ta, ma
 ; ZVFHMIN-NEXT:    vfncvt.f.f.w v8, v12
 ; ZVFHMIN-NEXT:    ret
@@ -473,13 +453,11 @@ define <vscale x 8 x half> @vfmul_fv_nxv8f16(<vscale x 8 x half> %va, half %b) {
 ;
 ; ZVFHMIN-LABEL: vfmul_fv_nxv8f16:
 ; ZVFHMIN:       # %bb.0:
-; ZVFHMIN-NEXT:    fmv.x.h a0, fa0
-; ZVFHMIN-NEXT:    vsetvli a1, zero, e16, m2, ta, ma
-; ZVFHMIN-NEXT:    vmv.v.x v10, a0
+; ZVFHMIN-NEXT:    fcvt.s.h fa5, fa0
+; ZVFHMIN-NEXT:    vsetvli a0, zero, e16, m2, ta, ma
 ; ZVFHMIN-NEXT:    vfwcvt.f.f.v v12, v8
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v16, v10
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
-; ZVFHMIN-NEXT:    vfmul.vv v12, v16, v12
+; ZVFHMIN-NEXT:    vfmul.vf v12, v12, fa5
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e16, m2, ta, ma
 ; ZVFHMIN-NEXT:    vfncvt.f.f.w v8, v12
 ; ZVFHMIN-NEXT:    ret
@@ -519,13 +497,11 @@ define <vscale x 16 x half> @vfmul_vf_nxv16f16(<vscale x 16 x half> %va, half %b
 ;
 ; ZVFHMIN-LABEL: vfmul_vf_nxv16f16:
 ; ZVFHMIN:       # %bb.0:
-; ZVFHMIN-NEXT:    fmv.x.h a0, fa0
-; ZVFHMIN-NEXT:    vsetvli a1, zero, e16, m4, ta, ma
-; ZVFHMIN-NEXT:    vmv.v.x v12, a0
+; ZVFHMIN-NEXT:    fcvt.s.h fa5, fa0
+; ZVFHMIN-NEXT:    vsetvli a0, zero, e16, m4, ta, ma
 ; ZVFHMIN-NEXT:    vfwcvt.f.f.v v16, v8
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v24, v12
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m8, ta, ma
-; ZVFHMIN-NEXT:    vfmul.vv v16, v16, v24
+; ZVFHMIN-NEXT:    vfmul.vf v16, v16, fa5
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e16, m4, ta, ma
 ; ZVFHMIN-NEXT:    vfncvt.f.f.w v8, v16
 ; ZVFHMIN-NEXT:    ret
diff --git a/llvm/test/CodeGen/RISCV/rvv/vfsub-sdnode.ll b/llvm/test/CodeGen/RISCV/rvv/vfsub-sdnode.ll
index f80644777c72a..9ab41ce978fc3 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vfsub-sdnode.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vfsub-sdnode.ll
@@ -30,13 +30,11 @@ define <vscale x 1 x bfloat> @vfsub_vv_nxv1bf16(<vscale x 1 x bfloat> %va, <vsca
 define <vscale x 1 x bfloat> @vfsub_vf_nxv1bf16(<vscale x 1 x bfloat> %va, bfloat %b) {
 ; CHECK-LABEL: vfsub_vf_nxv1bf16:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    fmv.x.h a0, fa0
-; CHECK-NEXT:    vsetvli a1, zero, e16, mf4, ta, ma
-; CHECK-NEXT:    vmv.v.x v9, a0
-; CHECK-NEXT:    vfwcvtbf16.f.f.v v10, v8
-; CHECK-NEXT:    vfwcvtbf16.f.f.v v8, v9
+; CHECK-NEXT:    fcvt.s.bf16 fa5, fa0
+; CHECK-NEXT:    vsetvli a0, zero, e16, mf4, ta, ma
+; CHECK-NEXT:    vfwcvtbf16.f.f.v v9, v8
 ; CHECK-NEXT:    vsetvli zero, zero, e32, mf2, ta, ma
-; CHECK-NEXT:    vfsub.vv v9, v10, v8
+; CHECK-NEXT:    vfsub.vf v9, v9, fa5
 ; CHECK-NEXT:    vsetvli zero, zero, e16, mf4, ta, ma
 ; CHECK-NEXT:    vfncvtbf16.f.f.w v8, v9
 ; CHECK-NEXT:    ret
@@ -64,13 +62,11 @@ define <vscale x 2 x bfloat> @vfsub_vv_nxv2bf16(<vscale x 2 x bfloat> %va, <vsca
 define <vscale x 2 x bfloat> @vfsub_vf_nxv2bf16(<vscale x 2 x bfloat> %va, bfloat %b) {
 ; CHECK-LABEL: vfsub_vf_nxv2bf16:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    fmv.x.h a0, fa0
-; CHECK-NEXT:    vsetvli a1, zero, e16, mf2, ta, ma
-; CHECK-NEXT:    vmv.v.x v9, a0
-; CHECK-NEXT:    vfwcvtbf16.f.f.v v10, v8
-; CHECK-NEXT:    vfwcvtbf16.f.f.v v8, v9
+; CHECK-NEXT:    fcvt.s.bf16 fa5, fa0
+; CHECK-NEXT:    vsetvli a0, zero, e16, mf2, ta, ma
+; CHECK-NEXT:    vfwcvtbf16.f.f.v v9, v8
 ; CHECK-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
-; CHECK-NEXT:    vfsub.vv v9, v10, v8
+; CHECK-NEXT:    vfsub.vf v9, v9, fa5
 ; CHECK-NEXT:    vsetvli zero, zero, e16, mf2, ta, ma
 ; CHECK-NEXT:    vfncvtbf16.f.f.w v8, v9
 ; CHECK-NEXT:    ret
@@ -98,13 +94,11 @@ define <vscale x 4 x bfloat> @vfsub_vv_nxv4bf16(<vscale x 4 x bfloat> %va, <vsca
 define <vscale x 4 x bfloat> @vfsub_vf_nxv4bf16(<vscale x 4 x bfloat> %va, bfloat %b) {
 ; CHECK-LABEL: vfsub_vf_nxv4bf16:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    fmv.x.h a0, fa0
-; CHECK-NEXT:    vsetvli a1, zero, e16, m1, ta, ma
-; CHECK-NEXT:    vmv.v.x v9, a0
+; CHECK-NEXT:    fcvt.s.bf16 fa5, fa0
+; CHECK-NEXT:    vsetvli a0, zero, e16, m1, ta, ma
 ; CHECK-NEXT:    vfwcvtbf16.f.f.v v10, v8
-; CHECK-NEXT:    vfwcvtbf16.f.f.v v12, v9
 ; CHECK-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
-; CHECK-NEXT:    vfsub.vv v10, v10, v12
+; CHECK-NEXT:    vfsub.vf v10, v10, fa5
 ; CHECK-NEXT:    vsetvli zero, zero, e16, m1, ta, ma
 ; CHECK-NEXT:    vfncvtbf16.f.f.w v8, v10
 ; CHECK-NEXT:    ret
@@ -132,13 +126,11 @@ define <vscale x 8 x bfloat> @vfsub_vv_nxv8bf16(<vscale x 8 x bfloat> %va, <vsca
 define <vscale x 8 x bfloat> @vfsub_vf_nxv8bf16(<vscale x 8 x bfloat> %va, bfloat %b) {
 ; CHECK-LABEL: vfsub_vf_nxv8bf16:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    fmv.x.h a0, fa0
-; CHECK-NEXT:    vsetvli a1, zero, e16, m2, ta, ma
-; CHECK-NEXT:    vmv.v.x v10, a0
+; CHECK-NEXT:    fcvt.s.bf16 fa5, fa0
+; CHECK-NEXT:    vsetvli a0, zero, e16, m2, ta, ma
 ; CHECK-NEXT:    vfwcvtbf16.f.f.v v12, v8
-; CHECK-NEXT:    vfwcvtbf16.f.f.v v16, v10
 ; CHECK-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
-; CHECK-NEXT:    vfsub.vv v12, v12, v16
+; CHECK-NEXT:    vfsub.vf v12, v12, fa5
 ; CHECK-NEXT:    vsetvli zero, zero, e16, m2, ta, ma
 ; CHECK-NEXT:    vfncvtbf16.f.f.w v8, v12
 ; CHECK-NEXT:    ret
@@ -151,13 +143,11 @@ define <vscale x 8 x bfloat> @vfsub_vf_nxv8bf16(<vscale x 8 x bfloat> %va, bfloa
 define <vscale x 8 x bfloat> @vfsub_fv_nxv8bf16(<vscale x 8 x bfloat> %va, bfloat %b) {
 ; CHECK-LABEL: vfsub_fv_nxv8bf16:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    fmv.x.h a0, fa0
-; CHECK-NEXT:    vsetvli a1, zero, e16, m2, ta, ma
-; CHECK-NEXT:    vmv.v.x v10, a0
+; CHECK-NEXT:    fcvt.s.bf16 fa5, fa0
+; CHECK-NEXT:    vsetvli a0, zero, e16, m2, ta, ma
 ; CHECK-NEXT:    vfwcvtbf16.f.f.v v12, v8
-; CHECK-NEXT:    vfwcvtbf16.f.f.v v16, v10
 ; CHECK-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
-; CHECK-NEXT:    vfsub.vv v12, v16, v12
+; CHECK-NEXT:    vfrsub.vf v12, v12, fa5
 ; CHECK-NEXT:    vsetvli zero, zero, e16, m2, ta, ma
 ; CHECK-NEXT:    vfncvtbf16.f.f.w v8, v12
 ; CHECK-NEXT:    ret
@@ -185,13 +175,11 @@ define <vscale x 16 x bfloat> @vfsub_vv_nxv16bf16(<vscale x 16 x bfloat> %va, <v
 define <vscale x 16 x bfloat> @vfsub_vf_nxv16bf16(<vscale x 16 x bfloat> %va, bfloat %b) {
 ; CHECK-LABEL: vfsub_vf_nxv16bf16:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    fmv.x.h a0, fa0
-; CHECK-NEXT:    vsetvli a1, zero, e16, m4, ta, ma
-; CHECK-NEXT:    vmv.v.x v12, a0
+; CHECK-NEXT:    fcvt.s.bf16 fa5, fa0
+; CHECK-NEXT:    vsetvli a0, zero, e16, m4, ta, ma
 ; CHECK-NEXT:    vfwcvtbf16.f.f.v v16, v8
-; CHECK-NEXT:    vfwcvtbf16.f.f.v v24, v12
 ; CHECK-NEXT:    vsetvli zero, zero, e32, m8, ta, ma
-; CHECK-NEXT:    vfsub.vv v16, v16, v24
+; CHECK-NEXT:    vfsub.vf v16, v16, fa5
 ; CHECK-NEXT:    vsetvli zero, zero, e16, m4, ta, ma
 ; CHECK-NEXT:    vfncvtbf16.f.f.w v8, v16
 ; CHECK-NEXT:    ret
@@ -310,13 +298,11 @@ define <vscale x 1 x half> @vfsub_vf_nxv1f16(<vscale x 1 x half> %va, half %b) {
 ;
 ; ZVFHMIN-LABEL: vfsub_vf_nxv1f16:
 ; ZVFHMIN:       # %bb.0:
-; ZVFHMIN-NEXT:    fmv.x.h a0, fa0
-; ZVFHMIN-NEXT:    vsetvli a1, zero, e16, mf4, ta, ma
-; ZVFHMIN-NEXT:    vmv.v.x v9, a0
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v10, v8
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v8, v9
+; ZVFHMIN-NEXT:    fcvt.s.h fa5, fa0
+; ZVFHMIN-NEXT:    vsetvli a0, zero, e16, mf4, ta, ma
+; ZVFHMIN-NEXT:    vfwcvt.f.f.v v9, v8
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, mf2, ta, ma
-; ZVFHMIN-NEXT:    vfsub.vv v9, v10, v8
+; ZVFHMIN-NEXT:    vfsub.vf v9, v9, fa5
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e16, mf4, ta, ma
 ; ZVFHMIN-NEXT:    vfncvt.f.f.w v8, v9
 ; ZVFHMIN-NEXT:    ret
@@ -356,13 +342,11 @@ define <vscale x 2 x half> @vfsub_vf_nxv2f16(<vscale x 2 x half> %va, half %b) {
 ;
 ; ZVFHMIN-LABEL: vfsub_vf_nxv2f16:
 ; ZVFHMIN:       # %bb.0:
-; ZVFHMIN-NEXT:    fmv.x.h a0, fa0
-; ZVFHMIN-NEXT:    vsetvli a1, zero, e16, mf2, ta, ma
-; ZVFHMIN-NEXT:    vmv.v.x v9, a0
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v10, v8
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v8, v9
+; ZVFHMIN-NEXT:    fcvt.s.h fa5, fa0
+; ZVFHMIN-NEXT:    vsetvli a0, zero, e16, mf2, ta, ma
+; ZVFHMIN-NEXT:    vfwcvt.f.f.v v9, v8
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
-; ZVFHMIN-NEXT:    vfsub.vv v9, v10, v8
+; ZVFHMIN-NEXT:    vfsub.vf v9, v9, fa5
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e16, mf2, ta, ma
 ; ZVFHMIN-NEXT:    vfncvt.f.f.w v8, v9
 ; ZVFHMIN-NEXT:    ret
@@ -402,13 +386,11 @@ define <vscale x 4 x half> @vfsub_vf_nxv4f16(<vscale x 4 x half> %va, half %b) {
 ;
 ; ZVFHMIN-LABEL: vfsub_vf_nxv4f16:
 ; ZVFHMIN:       # %bb.0:
-; ZVFHMIN-NEXT:    fmv.x.h a0, fa0
-; ZVFHMIN-NEXT:    vsetvli a1, zero, e16, m1, ta, ma
-; ZVFHMIN-NEXT:    vmv.v.x v9, a0
+; ZVFHMIN-NEXT:    fcvt.s.h fa5, fa0
+; ZVFHMIN-NEXT:    vsetvli a0, zero, e16, m1, ta, ma
 ; ZVFHMIN-NEXT:    vfwcvt.f.f.v v10, v8
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v12, v9
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
-; ZVFHMIN-NEXT:    vfsub.vv v10, v10, v12
+; ZVFHMIN-NEXT:    vfsub.vf v10, v10, fa5
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e16, m1, ta, ma
 ; ZVFHMIN-NEXT:    vfncvt.f.f.w v8, v10
 ; ZVFHMIN-NEXT:    ret
@@ -448,13 +430,11 @@ define <vscale x 8 x half> @vfsub_vf_nxv8f16(<vscale x 8 x half> %va, half %b) {
 ;
 ; ZVFHMIN-LABEL: vfsub_vf_nxv8f16:
 ; ZVFHMIN:       # %bb.0:
-; ZVFHMIN-NEXT:    fmv.x.h a0, fa0
-; ZVFHMIN-NEXT:    vsetvli a1, zero, e16, m2, ta, ma
-; ZVFHMIN-NEXT:    vmv.v.x v10, a0
+; ZVFHMIN-NEXT:    fcvt.s.h fa5, fa0
+; ZVFHMIN-NEXT:    vsetvli a0, zero, e16, m2, ta, ma
 ; ZVFHMIN-NEXT:    vfwcvt.f.f.v v12, v8
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v16, v10
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
-; ZVFHMIN-NEXT:    vfsub.vv v12, v12, v16
+; ZVFHMIN-NEXT:    vfsub.vf v12, v12, fa5
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e16, m2, ta, ma
 ; ZVFHMIN-NEXT:    vfncvt.f.f.w v8, v12
 ; ZVFHMIN-NEXT:    ret
@@ -473,13 +453,11 @@ define <vscale x 8 x half> @vfsub_fv_nxv8f16(<vscale x 8 x half> %va, half %b) {
 ;
 ; ZVFHMIN-LABEL: vfsub_fv_nxv8f16:
 ; ZVFHMIN:       # %bb.0:
-; ZVFHMIN-NEXT:    fmv.x.h a0, fa0
-; ZVFHMIN-NEXT:    vsetvli a1, zero, e16, m2, ta, ma
-; ZVFHMIN-NEXT:    vmv.v.x v10, a0
+; ZVFHMIN-NEXT:    fcvt.s.h fa5, fa0
+; ZVFHMIN-NEXT:    vsetvli a0, zero, e16, m2, ta, ma
 ; ZVFHMIN-NEXT:    vfwcvt.f.f.v v12, v8
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v16, v10
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
-; ZVFHMIN-NEXT:    vfsub.vv v12, v16, v12
+; ZVFHMIN-NEXT:    vfrsub.vf v12, v12, fa5
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e16, m2, ta, ma
 ; ZVFHMIN-NEXT:    vfncvt.f.f.w v8, v12
 ; ZVFHMIN-NEXT:    ret
@@ -519,13 +497,11 @@ define <vscale x 16 x half> @vfsub_vf_nxv16f16(<vscale x 16 x half> %va, half %b
 ;
 ; ZVFHMIN-LABEL: vfsub_vf_nxv16f16:
 ; ZVFHMIN:       # %bb.0:
-; ZVFHMIN-NEXT:    fmv.x.h a0, fa0
-; ZVFHMIN-NEXT:    vsetvli a1, zero, e16, m4, ta, ma
-; ZVFHMIN-NEXT:    vmv.v.x v12, a0
+; ZVFHMIN-NEXT:    fcvt.s.h fa5, fa0
+; ZVFHMIN-NEXT:    vsetvli a0, zero, e16, m4, ta, ma
 ; ZVFHMIN-NEXT:    vfwcvt.f.f.v v16, v8
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v24, v12
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m8, ta, ma
-; ZVFHMIN-NEXT:    vfsub.vv v16, v16, v24
+; ZVFHMIN-NEXT:    vfsub.vf v16, v16, fa5
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e16, m4, ta, ma
 ; ZVFHMIN-NEXT:    vfncvt.f.f.w v8, v16
 ; ZVFHMIN-NEXT:    ret
diff --git a/llvm/test/CodeGen/RISCV/rvv/vfwadd-sdnode.ll b/llvm/test/CodeGen/RISCV/rvv/vfwadd-sdnode.ll
index 68014ff4206f8..f7d287a088cc3 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vfwadd-sdnode.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vfwadd-sdnode.ll
@@ -323,3 +323,15 @@ define <vscale x 8 x double> @vfwadd_wf_nxv8f64_2(<vscale x 8 x double> %va, flo
   %vd = fadd <vscale x 8 x double> %va, %splat
   ret <vscale x 8 x double> %vd
 }
+
+define <vscale x 1 x double> @vfwadd_vv_nxv1f64_same_op(<vscale x 1 x float> %va) {
+; CHECK-LABEL: vfwadd_vv_nxv1f64_same_op:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e32, mf2, ta, ma
+; CHECK-NEXT:    vfwadd.vv v9, v8, v8
+; CHECK-NEXT:    vmv1r.v v8, v9
+; CHECK-NEXT:    ret
+  %vb = fpext <vscale x 1 x float> %va to <vscale x 1 x double>
+  %vc = fadd <vscale x 1 x double> %vb, %vb
+  ret <vscale x 1 x double> %vc
+}
diff --git a/llvm/test/CodeGen/RISCV/rvv/vfwmacc-sdnode.ll b/llvm/test/CodeGen/RISCV/rvv/vfwmacc-sdnode.ll
index f69b2346226ee..63113b8780989 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vfwmacc-sdnode.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vfwmacc-sdnode.ll
@@ -1764,3 +1764,28 @@ define <vscale x 8 x double> @vfwnmsac_fv_nxv8f64(<vscale x 8 x double> %va, <vs
   %vg = call <vscale x 8 x double> @llvm.fma.v8f64(<vscale x 8 x double> %vd, <vscale x 8 x double> %vf, <vscale x 8 x double> %va)
   ret <vscale x 8 x double> %vg
 }
+
+define <vscale x 1 x double> @vfwma_vv_nxv1f64_same_op(<vscale x 1 x float> %va, <vscale x 1 x double> %vb) {
+; CHECK-LABEL: vfwma_vv_nxv1f64_same_op:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e32, mf2, ta, ma
+; CHECK-NEXT:    vfwmacc.vv v9, v8, v8
+; CHECK-NEXT:    vmv1r.v v8, v9
+; CHECK-NEXT:    ret
+  %vc = fpext <vscale x 1 x float> %va to <vscale x 1 x double>
+  %vd = call <vscale x 1 x double> @llvm.fma(<vscale x 1 x double> %vc, <vscale x 1 x double> %vc, <vscale x 1 x double> %vb)
+  ret <vscale x 1 x double> %vd
+}
+
+define <vscale x 1 x double> @vfwmsac_vv_nxv1f64_same_op(<vscale x 1 x float> %va, <vscale x 1 x double> %vb) {
+; CHECK-LABEL: vfwmsac_vv_nxv1f64_same_op:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e32, mf2, ta, ma
+; CHECK-NEXT:    vfwmsac.vv v9, v8, v8
+; CHECK-NEXT:    vmv1r.v v8, v9
+; CHECK-NEXT:    ret
+  %vc = fpext <vscale x 1 x float> %va to <vscale x 1 x double>
+  %vd = fneg <vscale x 1 x double> %vb
+  %ve = call <vscale x 1 x double> @llvm.fma(<vscale x 1 x double> %vc, <vscale x 1 x double> %vc, <vscale x 1 x double> %vd)
+  ret <vscale x 1 x double> %ve
+}
diff --git a/llvm/test/CodeGen/RISCV/rvv/vfwmul-sdnode.ll b/llvm/test/CodeGen/RISCV/rvv/vfwmul-sdnode.ll
index f00ff4b6d2cec..8cc8c5cffca6b 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vfwmul-sdnode.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vfwmul-sdnode.ll
@@ -175,3 +175,15 @@ define <vscale x 8 x double> @vfwmul_vf_nxv8f64_2(<vscale x 8 x float> %va, floa
   %ve = fmul <vscale x 8 x double> %vc, %splat
   ret <vscale x 8 x double> %ve
 }
+
+define <vscale x 1 x double> @vfwmul_vv_nxv1f64_same_op(<vscale x 1 x float> %va) {
+; CHECK-LABEL: vfwmul_vv_nxv1f64_same_op:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e32, mf2, ta, ma
+; CHECK-NEXT:    vfwmul.vv v9, v8, v8
+; CHECK-NEXT:    vmv1r.v v8, v9
+; CHECK-NEXT:    ret
+  %vb = fpext <vscale x 1 x float> %va to <vscale x 1 x double>
+  %vc = fmul <vscale x 1 x double> %vb, %vb
+  ret <vscale x 1 x double> %vc
+}
diff --git a/llvm/test/CodeGen/RISCV/rvv/vfwsub-sdnode.ll b/llvm/test/CodeGen/RISCV/rvv/vfwsub-sdnode.ll
index b9f66d5d30825..d0cb64d986661 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vfwsub-sdnode.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vfwsub-sdnode.ll
@@ -323,3 +323,15 @@ define <vscale x 8 x double> @vfwsub_wf_nxv8f64_2(<vscale x 8 x double> %va, flo
   %vd = fsub <vscale x 8 x double> %va, %splat
   ret <vscale x 8 x double> %vd
 }
+
+define <vscale x 1 x double> @vfwsub_vv_nxv1f64_same_op(<vscale x 1 x float> %va) {
+; CHECK-LABEL: vfwsub_vv_nxv1f64_same_op:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e32, mf2, ta, ma
+; CHECK-NEXT:    vfwsub.vv v9, v8, v8
+; CHECK-NEXT:    vmv1r.v v8, v9
+; CHECK-NEXT:    ret
+  %vb = fpext <vscale x 1 x float> %va to <vscale x 1 x double>
+  %vc = fsub <vscale x 1 x double> %vb, %vb
+  ret <vscale x 1 x double> %vc
+}
diff --git a/llvm/test/CodeGen/RISCV/rvv/vl-opt-instrs.ll b/llvm/test/CodeGen/RISCV/rvv/vl-opt-instrs.ll
index 163166c1df743..c6ee9e34dc207 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vl-opt-instrs.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vl-opt-instrs.ll
@@ -1,8 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
-; RUN: sed 's/iXLen/i32/g' %s | llc -mtriple=riscv32 -mattr=+v,+zvbb -riscv-enable-vl-optimizer=false -verify-machineinstrs | FileCheck %s --check-prefixes=NOVLOPT
-; RUN: sed 's/iXLen/i64/g' %s | llc -mtriple=riscv64 -mattr=+v,+zvbb -riscv-enable-vl-optimizer=false -verify-machineinstrs | FileCheck %s --check-prefixes=NOVLOPT
-; RUN: sed 's/iXLen/i32/g' %s | llc -mtriple=riscv32 -mattr=+v,+zvbb -riscv-enable-vl-optimizer -verify-machineinstrs | FileCheck %s --check-prefixes=VLOPT
-; RUN: sed 's/iXLen/i64/g' %s | llc -mtriple=riscv64 -mattr=+v,+zvbb -riscv-enable-vl-optimizer -verify-machineinstrs | FileCheck %s --check-prefixes=VLOPT
+; RUN: sed 's/iXLen/i32/g' %s | llc -mtriple=riscv32 -mattr=+v,+zvbb,+zvfbfwma -riscv-enable-vl-optimizer=false -verify-machineinstrs | FileCheck %s --check-prefixes=NOVLOPT
+; RUN: sed 's/iXLen/i64/g' %s | llc -mtriple=riscv64 -mattr=+v,+zvbb,+zvfbfwma -riscv-enable-vl-optimizer=false -verify-machineinstrs | FileCheck %s --check-prefixes=NOVLOPT
+; RUN: sed 's/iXLen/i32/g' %s | llc -mtriple=riscv32 -mattr=+v,+zvbb,+zvfbfwma -riscv-enable-vl-optimizer -verify-machineinstrs | FileCheck %s --check-prefixes=VLOPT
+; RUN: sed 's/iXLen/i64/g' %s | llc -mtriple=riscv64 -mattr=+v,+zvbb,+zvfbfwma -riscv-enable-vl-optimizer -verify-machineinstrs | FileCheck %s --check-prefixes=VLOPT
 
 ; The purpose of this file is to check the behavior of specific instructions as it relates to the VL optimizer
 
@@ -2511,1523 +2511,2561 @@ define <vscale x 4 x i32> @vwmaccus_vx(<vscale x 4 x i32> %a, i16 %b, <vscale x
   ret <vscale x 4 x i32> %2
 }
 
-define <vscale x 4 x i32> @vmv_v_i(<vscale x 4 x i32> %a, i32 %x, iXLen %vl) {
-; NOVLOPT-LABEL: vmv_v_i:
+define <vscale x 4 x i32> @vsaddu_vv(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b, iXLen %vl) {
+; NOVLOPT-LABEL: vsaddu_vv:
 ; NOVLOPT:       # %bb.0:
-; NOVLOPT-NEXT:    vsetvli a0, zero, e32, m2, ta, ma
-; NOVLOPT-NEXT:    vmv.v.i v10, 5
-; NOVLOPT-NEXT:    vsetvli zero, a1, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vsetvli a1, zero, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vsaddu.vv v10, v8, v10
+; NOVLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
 ; NOVLOPT-NEXT:    vadd.vv v8, v10, v8
 ; NOVLOPT-NEXT:    ret
 ;
-; VLOPT-LABEL: vmv_v_i:
+; VLOPT-LABEL: vsaddu_vv:
 ; VLOPT:       # %bb.0:
-; VLOPT-NEXT:    vsetvli zero, a1, e32, m2, ta, ma
-; VLOPT-NEXT:    vmv.v.i v10, 5
+; VLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
+; VLOPT-NEXT:    vsaddu.vv v10, v8, v10
 ; VLOPT-NEXT:    vadd.vv v8, v10, v8
 ; VLOPT-NEXT:    ret
-  %1 = call <vscale x 4 x i32> @llvm.riscv.vmv.v.x.nxv4i32(<vscale x 4 x i32> poison, i32 5, iXLen -1)
-  %2 = call <vscale x 4 x i32> @llvm.riscv.vadd.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %1, <vscale x 4 x i32> %a, iXLen %vl)
+  %1 = call <vscale x 4 x i32> @llvm.riscv.vsaddu(<vscale x 4 x i32> poison, <vscale x 4 x i32> %a, <vscale x 4 x i32> %b, iXLen -1)
+  %2 = call <vscale x 4 x i32> @llvm.riscv.vadd(<vscale x 4 x i32> poison, <vscale x 4 x i32> %1, <vscale x 4 x i32> %a, iXLen %vl)
   ret <vscale x 4 x i32> %2
 }
 
-define <vscale x 4 x i32> @vmv_v_x(<vscale x 4 x i32> %a, i32 %x, iXLen %vl) {
-; NOVLOPT-LABEL: vmv_v_x:
+define <vscale x 4 x i32> @vsaddu_vx(<vscale x 4 x i32> %a, i32 %b, iXLen %vl) {
+; NOVLOPT-LABEL: vsaddu_vx:
 ; NOVLOPT:       # %bb.0:
 ; NOVLOPT-NEXT:    vsetvli a2, zero, e32, m2, ta, ma
-; NOVLOPT-NEXT:    vmv.v.x v10, a0
+; NOVLOPT-NEXT:    vsaddu.vx v10, v8, a0
 ; NOVLOPT-NEXT:    vsetvli zero, a1, e32, m2, ta, ma
 ; NOVLOPT-NEXT:    vadd.vv v8, v10, v8
 ; NOVLOPT-NEXT:    ret
 ;
-; VLOPT-LABEL: vmv_v_x:
+; VLOPT-LABEL: vsaddu_vx:
 ; VLOPT:       # %bb.0:
 ; VLOPT-NEXT:    vsetvli zero, a1, e32, m2, ta, ma
-; VLOPT-NEXT:    vmv.v.x v10, a0
+; VLOPT-NEXT:    vsaddu.vx v10, v8, a0
 ; VLOPT-NEXT:    vadd.vv v8, v10, v8
 ; VLOPT-NEXT:    ret
-  %1 = call <vscale x 4 x i32> @llvm.riscv.vmv.v.x.nxv4i32(<vscale x 4 x i32> poison, i32 %x, iXLen -1)
-  %2 = call <vscale x 4 x i32> @llvm.riscv.vadd.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %1, <vscale x 4 x i32> %a, iXLen %vl)
+  %1 = call <vscale x 4 x i32> @llvm.riscv.vsaddu(<vscale x 4 x i32> poison, <vscale x 4 x i32> %a, i32 %b, iXLen -1)
+  %2 = call <vscale x 4 x i32> @llvm.riscv.vadd(<vscale x 4 x i32> poison, <vscale x 4 x i32> %1, <vscale x 4 x i32> %a, iXLen %vl)
   ret <vscale x 4 x i32> %2
 }
 
-; The vmv.v.v is optimized away if we use a vadd as the user.
-define <vscale x 1 x i8> @vmv_v_v(<vscale x 1 x i8> %a, <vscale x 1 x i8> %b, <vscale x 1 x i8> %c, <vscale x 1 x i1> %m, iXLen %vl) {
-; NOVLOPT-LABEL: vmv_v_v:
+define <vscale x 4 x i32> @vsaddu_vi(<vscale x 4 x i32> %a, iXLen %vl) {
+; NOVLOPT-LABEL: vsaddu_vi:
 ; NOVLOPT:       # %bb.0:
-; NOVLOPT-NEXT:    vsetvli a1, zero, e8, mf8, tu, ma
-; NOVLOPT-NEXT:    vmv.v.v v8, v9
-; NOVLOPT-NEXT:    vsetvli zero, a0, e8, mf8, ta, ma
-; NOVLOPT-NEXT:    vmerge.vvm v8, v8, v10, v0
+; NOVLOPT-NEXT:    vsetvli a1, zero, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vsaddu.vi v10, v8, 5
+; NOVLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vadd.vv v8, v10, v8
 ; NOVLOPT-NEXT:    ret
 ;
-; VLOPT-LABEL: vmv_v_v:
+; VLOPT-LABEL: vsaddu_vi:
 ; VLOPT:       # %bb.0:
-; VLOPT-NEXT:    vsetvli zero, a0, e8, mf8, tu, ma
-; VLOPT-NEXT:    vmv.v.v v8, v9
-; VLOPT-NEXT:    vsetvli zero, zero, e8, mf8, ta, ma
-; VLOPT-NEXT:    vmerge.vvm v8, v8, v10, v0
+; VLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
+; VLOPT-NEXT:    vsaddu.vi v10, v8, 5
+; VLOPT-NEXT:    vadd.vv v8, v10, v8
 ; VLOPT-NEXT:    ret
-  %2 = call <vscale x 1 x i8> @llvm.riscv.vmv.v.v.nxv1i8.nxv1i8(<vscale x 1 x i8> %a, <vscale x 1 x i8> %b, iXLen -1)
-  %3 = call <vscale x 1 x i8> @llvm.riscv.vmerge.nxv1i8.nxv1i8(<vscale x 1 x i8> undef, <vscale x 1 x i8> %2, <vscale x 1 x i8> %c, <vscale x 1 x i1> %m, iXLen %vl)
-  ret <vscale x 1 x i8> %3
+  %1 = call <vscale x 4 x i32> @llvm.riscv.vsaddu(<vscale x 4 x i32> poison, <vscale x 4 x i32> %a, i32 5, iXLen -1)
+  %2 = call <vscale x 4 x i32> @llvm.riscv.vadd(<vscale x 4 x i32> poison, <vscale x 4 x i32> %1, <vscale x 4 x i32> %a, iXLen %vl)
+  ret <vscale x 4 x i32> %2
 }
 
-define <vscale x 4 x i32> @vwsll_vi(<vscale x 4 x i16> %a, <vscale x 4 x i32> %b, iXLen %vl) {
-; NOVLOPT-LABEL: vwsll_vi:
+define <vscale x 4 x i32> @vsadd_vv(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b, iXLen %vl) {
+; NOVLOPT-LABEL: vsadd_vv:
 ; NOVLOPT:       # %bb.0:
-; NOVLOPT-NEXT:    vsetvli a1, zero, e16, m1, ta, ma
-; NOVLOPT-NEXT:    vwsll.vi v12, v8, 1
+; NOVLOPT-NEXT:    vsetvli a1, zero, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vsadd.vv v10, v8, v10
 ; NOVLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
-; NOVLOPT-NEXT:    vadd.vv v8, v12, v10
+; NOVLOPT-NEXT:    vadd.vv v8, v10, v8
 ; NOVLOPT-NEXT:    ret
 ;
-; VLOPT-LABEL: vwsll_vi:
+; VLOPT-LABEL: vsadd_vv:
 ; VLOPT:       # %bb.0:
-; VLOPT-NEXT:    vsetvli zero, a0, e16, m1, ta, ma
-; VLOPT-NEXT:    vwsll.vi v12, v8, 1
-; VLOPT-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
-; VLOPT-NEXT:    vadd.vv v8, v12, v10
+; VLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
+; VLOPT-NEXT:    vsadd.vv v10, v8, v10
+; VLOPT-NEXT:    vadd.vv v8, v10, v8
 ; VLOPT-NEXT:    ret
-  %1 = call <vscale x 4 x i32> @llvm.riscv.vwsll.nxv4i32.nxv4i16(<vscale x 4 x i32> poison, <vscale x 4 x i16> %a,iXLen 1, iXLen -1)
-  %2 = call <vscale x 4 x i32> @llvm.riscv.vadd.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %1, <vscale x 4 x i32> %b, iXLen %vl)
+  %1 = call <vscale x 4 x i32> @llvm.riscv.vsadd(<vscale x 4 x i32> poison, <vscale x 4 x i32> %a, <vscale x 4 x i32> %b, iXLen -1)
+  %2 = call <vscale x 4 x i32> @llvm.riscv.vadd(<vscale x 4 x i32> poison, <vscale x 4 x i32> %1, <vscale x 4 x i32> %a, iXLen %vl)
   ret <vscale x 4 x i32> %2
 }
 
-define <vscale x 1 x i32> @vmand_mm(<vscale x 1 x i1> %a, <vscale x 1 x i1> %b, <vscale x 1 x i32> %c, iXLen %vl) {
-; NOVLOPT-LABEL: vmand_mm:
+define <vscale x 4 x i32> @vsadd_vx(<vscale x 4 x i32> %a, i32 %b, iXLen %vl) {
+; NOVLOPT-LABEL: vsadd_vx:
 ; NOVLOPT:       # %bb.0:
-; NOVLOPT-NEXT:    vsetvli a1, zero, e8, mf8, ta, ma
-; NOVLOPT-NEXT:    vmand.mm v8, v0, v8
-; NOVLOPT-NEXT:    vsetvli zero, a0, e8, mf8, ta, ma
-; NOVLOPT-NEXT:    vmand.mm v0, v0, v8
-; NOVLOPT-NEXT:    vmv1r.v v8, v9
-; NOVLOPT-NEXT:    vsetvli zero, zero, e32, mf2, tu, mu
-; NOVLOPT-NEXT:    vadd.vv v8, v9, v9, v0.t
+; NOVLOPT-NEXT:    vsetvli a2, zero, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vsadd.vx v10, v8, a0
+; NOVLOPT-NEXT:    vsetvli zero, a1, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vadd.vv v8, v10, v8
 ; NOVLOPT-NEXT:    ret
 ;
-; VLOPT-LABEL: vmand_mm:
+; VLOPT-LABEL: vsadd_vx:
 ; VLOPT:       # %bb.0:
-; VLOPT-NEXT:    vsetvli zero, a0, e8, mf8, ta, ma
-; VLOPT-NEXT:    vmand.mm v8, v0, v8
-; VLOPT-NEXT:    vmand.mm v0, v0, v8
-; VLOPT-NEXT:    vmv1r.v v8, v9
-; VLOPT-NEXT:    vsetvli zero, zero, e32, mf2, tu, mu
-; VLOPT-NEXT:    vadd.vv v8, v9, v9, v0.t
+; VLOPT-NEXT:    vsetvli zero, a1, e32, m2, ta, ma
+; VLOPT-NEXT:    vsadd.vx v10, v8, a0
+; VLOPT-NEXT:    vadd.vv v8, v10, v8
 ; VLOPT-NEXT:    ret
-  %1 = call <vscale x 1 x i1> @llvm.riscv.vmand.nxv1i1(<vscale x 1 x i1> %a, <vscale x 1 x i1> %b, iXLen -1)
-  %2 = call <vscale x 1 x i1> @llvm.riscv.vmand.nxv1i1(<vscale x 1 x i1> %a, <vscale x 1 x i1> %1, iXLen %vl)
-  %3 = call <vscale x 1 x i32> @llvm.riscv.vadd.mask.nxv1i32.nxv1i32(<vscale x 1 x i32> %c, <vscale x 1 x i32> %c, <vscale x 1 x i32> %c, <vscale x 1 x i1> %2, iXLen %vl, iXLen 0)
-  ret <vscale x 1 x i32> %3
+  %1 = call <vscale x 4 x i32> @llvm.riscv.vsadd(<vscale x 4 x i32> poison, <vscale x 4 x i32> %a, i32 %b, iXLen -1)
+  %2 = call <vscale x 4 x i32> @llvm.riscv.vadd(<vscale x 4 x i32> poison, <vscale x 4 x i32> %1, <vscale x 4 x i32> %a, iXLen %vl)
+  ret <vscale x 4 x i32> %2
 }
 
-define <vscale x 1 x i32> @vmnand_mm(<vscale x 1 x i1> %a, <vscale x 1 x i1> %b, <vscale x 1 x i32> %c, iXLen %vl) {
-; NOVLOPT-LABEL: vmnand_mm:
+define <vscale x 4 x i32> @vsadd_vi(<vscale x 4 x i32> %a, iXLen %vl) {
+; NOVLOPT-LABEL: vsadd_vi:
 ; NOVLOPT:       # %bb.0:
-; NOVLOPT-NEXT:    vsetvli a1, zero, e8, mf8, ta, ma
-; NOVLOPT-NEXT:    vmnand.mm v8, v0, v8
-; NOVLOPT-NEXT:    vsetvli zero, a0, e8, mf8, ta, ma
-; NOVLOPT-NEXT:    vmand.mm v0, v0, v8
-; NOVLOPT-NEXT:    vmv1r.v v8, v9
-; NOVLOPT-NEXT:    vsetvli zero, zero, e32, mf2, tu, mu
-; NOVLOPT-NEXT:    vadd.vv v8, v9, v9, v0.t
+; NOVLOPT-NEXT:    vsetvli a1, zero, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vsadd.vi v10, v8, 5
+; NOVLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vadd.vv v8, v10, v8
 ; NOVLOPT-NEXT:    ret
 ;
-; VLOPT-LABEL: vmnand_mm:
+; VLOPT-LABEL: vsadd_vi:
 ; VLOPT:       # %bb.0:
-; VLOPT-NEXT:    vsetvli zero, a0, e8, mf8, ta, ma
-; VLOPT-NEXT:    vmnand.mm v8, v0, v8
-; VLOPT-NEXT:    vmand.mm v0, v0, v8
-; VLOPT-NEXT:    vmv1r.v v8, v9
-; VLOPT-NEXT:    vsetvli zero, zero, e32, mf2, tu, mu
-; VLOPT-NEXT:    vadd.vv v8, v9, v9, v0.t
+; VLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
+; VLOPT-NEXT:    vsadd.vi v10, v8, 5
+; VLOPT-NEXT:    vadd.vv v8, v10, v8
 ; VLOPT-NEXT:    ret
-  %1 = call <vscale x 1 x i1> @llvm.riscv.vmnand.nxv1i1(<vscale x 1 x i1> %a, <vscale x 1 x i1> %b, iXLen -1)
-  %2 = call <vscale x 1 x i1> @llvm.riscv.vmand.nxv1i1(<vscale x 1 x i1> %a, <vscale x 1 x i1> %1, iXLen %vl)
-  %3 = call <vscale x 1 x i32> @llvm.riscv.vadd.mask.nxv1i32.nxv1i32(<vscale x 1 x i32> %c, <vscale x 1 x i32> %c, <vscale x 1 x i32> %c, <vscale x 1 x i1> %2, iXLen %vl, iXLen 0)
-  ret <vscale x 1 x i32> %3
+  %1 = call <vscale x 4 x i32> @llvm.riscv.vsadd(<vscale x 4 x i32> poison, <vscale x 4 x i32> %a, i32 5, iXLen -1)
+  %2 = call <vscale x 4 x i32> @llvm.riscv.vadd(<vscale x 4 x i32> poison, <vscale x 4 x i32> %1, <vscale x 4 x i32> %a, iXLen %vl)
+  ret <vscale x 4 x i32> %2
 }
 
-define <vscale x 1 x i32> @vmandn_mm(<vscale x 1 x i1> %a, <vscale x 1 x i1> %b, <vscale x 1 x i32> %c, iXLen %vl) {
-; NOVLOPT-LABEL: vmandn_mm:
+define <vscale x 4 x i32> @vssubu_vv(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b, iXLen %vl) {
+; NOVLOPT-LABEL: vssubu_vv:
 ; NOVLOPT:       # %bb.0:
-; NOVLOPT-NEXT:    vsetvli a1, zero, e8, mf8, ta, ma
-; NOVLOPT-NEXT:    vmandn.mm v8, v0, v8
-; NOVLOPT-NEXT:    vsetvli zero, a0, e8, mf8, ta, ma
-; NOVLOPT-NEXT:    vmand.mm v0, v0, v8
-; NOVLOPT-NEXT:    vmv1r.v v8, v9
-; NOVLOPT-NEXT:    vsetvli zero, zero, e32, mf2, tu, mu
-; NOVLOPT-NEXT:    vadd.vv v8, v9, v9, v0.t
+; NOVLOPT-NEXT:    vsetvli a1, zero, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vssubu.vv v10, v8, v10
+; NOVLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vadd.vv v8, v10, v8
 ; NOVLOPT-NEXT:    ret
 ;
-; VLOPT-LABEL: vmandn_mm:
+; VLOPT-LABEL: vssubu_vv:
 ; VLOPT:       # %bb.0:
-; VLOPT-NEXT:    vsetvli zero, a0, e8, mf8, ta, ma
-; VLOPT-NEXT:    vmandn.mm v8, v0, v8
-; VLOPT-NEXT:    vmand.mm v0, v0, v8
-; VLOPT-NEXT:    vmv1r.v v8, v9
-; VLOPT-NEXT:    vsetvli zero, zero, e32, mf2, tu, mu
-; VLOPT-NEXT:    vadd.vv v8, v9, v9, v0.t
+; VLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
+; VLOPT-NEXT:    vssubu.vv v10, v8, v10
+; VLOPT-NEXT:    vadd.vv v8, v10, v8
 ; VLOPT-NEXT:    ret
-  %1 = call <vscale x 1 x i1> @llvm.riscv.vmandn.nxv1i1(<vscale x 1 x i1> %a, <vscale x 1 x i1> %b, iXLen -1)
-  %2 = call <vscale x 1 x i1> @llvm.riscv.vmand.nxv1i1(<vscale x 1 x i1> %a, <vscale x 1 x i1> %1, iXLen %vl)
-  %3 = call <vscale x 1 x i32> @llvm.riscv.vadd.mask.nxv1i32.nxv1i32(<vscale x 1 x i32> %c, <vscale x 1 x i32> %c, <vscale x 1 x i32> %c, <vscale x 1 x i1> %2, iXLen %vl, iXLen 0)
-  ret <vscale x 1 x i32> %3
+  %1 = call <vscale x 4 x i32> @llvm.riscv.vssubu(<vscale x 4 x i32> poison, <vscale x 4 x i32> %a, <vscale x 4 x i32> %b, iXLen -1)
+  %2 = call <vscale x 4 x i32> @llvm.riscv.vadd(<vscale x 4 x i32> poison, <vscale x 4 x i32> %1, <vscale x 4 x i32> %a, iXLen %vl)
+  ret <vscale x 4 x i32> %2
 }
 
-define <vscale x 1 x i32> @vmxor_mm(<vscale x 1 x i1> %a, <vscale x 1 x i1> %b, <vscale x 1 x i32> %c, iXLen %vl) {
-; NOVLOPT-LABEL: vmxor_mm:
+define <vscale x 4 x i32> @vssubu_vx(<vscale x 4 x i32> %a, i32 %b, iXLen %vl) {
+; NOVLOPT-LABEL: vssubu_vx:
 ; NOVLOPT:       # %bb.0:
-; NOVLOPT-NEXT:    vsetvli a1, zero, e8, mf8, ta, ma
-; NOVLOPT-NEXT:    vmxor.mm v8, v0, v8
-; NOVLOPT-NEXT:    vsetvli zero, a0, e8, mf8, ta, ma
-; NOVLOPT-NEXT:    vmand.mm v0, v0, v8
-; NOVLOPT-NEXT:    vmv1r.v v8, v9
-; NOVLOPT-NEXT:    vsetvli zero, zero, e32, mf2, tu, mu
-; NOVLOPT-NEXT:    vadd.vv v8, v9, v9, v0.t
+; NOVLOPT-NEXT:    vsetvli a2, zero, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vssubu.vx v10, v8, a0
+; NOVLOPT-NEXT:    vsetvli zero, a1, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vadd.vv v8, v10, v8
 ; NOVLOPT-NEXT:    ret
 ;
-; VLOPT-LABEL: vmxor_mm:
+; VLOPT-LABEL: vssubu_vx:
 ; VLOPT:       # %bb.0:
-; VLOPT-NEXT:    vsetvli zero, a0, e8, mf8, ta, ma
-; VLOPT-NEXT:    vmxor.mm v8, v0, v8
-; VLOPT-NEXT:    vmand.mm v0, v0, v8
-; VLOPT-NEXT:    vmv1r.v v8, v9
-; VLOPT-NEXT:    vsetvli zero, zero, e32, mf2, tu, mu
-; VLOPT-NEXT:    vadd.vv v8, v9, v9, v0.t
+; VLOPT-NEXT:    vsetvli zero, a1, e32, m2, ta, ma
+; VLOPT-NEXT:    vssubu.vx v10, v8, a0
+; VLOPT-NEXT:    vadd.vv v8, v10, v8
 ; VLOPT-NEXT:    ret
-  %1 = call <vscale x 1 x i1> @llvm.riscv.vmxor.nxv1i1(<vscale x 1 x i1> %a, <vscale x 1 x i1> %b, iXLen -1)
-  %2 = call <vscale x 1 x i1> @llvm.riscv.vmand.nxv1i1(<vscale x 1 x i1> %a, <vscale x 1 x i1> %1, iXLen %vl)
-  %3 = call <vscale x 1 x i32> @llvm.riscv.vadd.mask.nxv1i32.nxv1i32(<vscale x 1 x i32> %c, <vscale x 1 x i32> %c, <vscale x 1 x i32> %c, <vscale x 1 x i1> %2, iXLen %vl, iXLen 0)
-  ret <vscale x 1 x i32> %3
+  %1 = call <vscale x 4 x i32> @llvm.riscv.vssubu(<vscale x 4 x i32> poison, <vscale x 4 x i32> %a, i32 %b, iXLen -1)
+  %2 = call <vscale x 4 x i32> @llvm.riscv.vadd(<vscale x 4 x i32> poison, <vscale x 4 x i32> %1, <vscale x 4 x i32> %a, iXLen %vl)
+  ret <vscale x 4 x i32> %2
 }
 
-define <vscale x 1 x i32> @vmor_mm(<vscale x 1 x i1> %a, <vscale x 1 x i1> %b, <vscale x 1 x i32> %c, iXLen %vl) {
-; NOVLOPT-LABEL: vmor_mm:
+define <vscale x 4 x i32> @vssub_vv(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b, iXLen %vl) {
+; NOVLOPT-LABEL: vssub_vv:
 ; NOVLOPT:       # %bb.0:
-; NOVLOPT-NEXT:    vsetvli a1, zero, e8, mf8, ta, ma
-; NOVLOPT-NEXT:    vmor.mm v8, v0, v8
-; NOVLOPT-NEXT:    vsetvli zero, a0, e8, mf8, ta, ma
-; NOVLOPT-NEXT:    vmand.mm v0, v0, v8
-; NOVLOPT-NEXT:    vmv1r.v v8, v9
-; NOVLOPT-NEXT:    vsetvli zero, zero, e32, mf2, tu, mu
-; NOVLOPT-NEXT:    vadd.vv v8, v9, v9, v0.t
+; NOVLOPT-NEXT:    vsetvli a1, zero, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vssub.vv v10, v8, v10
+; NOVLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vadd.vv v8, v10, v8
 ; NOVLOPT-NEXT:    ret
 ;
-; VLOPT-LABEL: vmor_mm:
+; VLOPT-LABEL: vssub_vv:
 ; VLOPT:       # %bb.0:
-; VLOPT-NEXT:    vsetvli zero, a0, e8, mf8, ta, ma
-; VLOPT-NEXT:    vmor.mm v8, v0, v8
-; VLOPT-NEXT:    vmand.mm v0, v0, v8
-; VLOPT-NEXT:    vmv1r.v v8, v9
-; VLOPT-NEXT:    vsetvli zero, zero, e32, mf2, tu, mu
-; VLOPT-NEXT:    vadd.vv v8, v9, v9, v0.t
+; VLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
+; VLOPT-NEXT:    vssub.vv v10, v8, v10
+; VLOPT-NEXT:    vadd.vv v8, v10, v8
 ; VLOPT-NEXT:    ret
-  %1 = call <vscale x 1 x i1> @llvm.riscv.vmor.nxv1i1(<vscale x 1 x i1> %a, <vscale x 1 x i1> %b, iXLen -1)
-  %2 = call <vscale x 1 x i1> @llvm.riscv.vmand.nxv1i1(<vscale x 1 x i1> %a, <vscale x 1 x i1> %1, iXLen %vl)
-  %3 = call <vscale x 1 x i32> @llvm.riscv.vadd.mask.nxv1i32.nxv1i32(<vscale x 1 x i32> %c, <vscale x 1 x i32> %c, <vscale x 1 x i32> %c, <vscale x 1 x i1> %2, iXLen %vl, iXLen 0)
-  ret <vscale x 1 x i32> %3
+  %1 = call <vscale x 4 x i32> @llvm.riscv.vssub(<vscale x 4 x i32> poison, <vscale x 4 x i32> %a, <vscale x 4 x i32> %b, iXLen -1)
+  %2 = call <vscale x 4 x i32> @llvm.riscv.vadd(<vscale x 4 x i32> poison, <vscale x 4 x i32> %1, <vscale x 4 x i32> %a, iXLen %vl)
+  ret <vscale x 4 x i32> %2
 }
 
-
-define <vscale x 1 x i32> @vmnor_mm(<vscale x 1 x i1> %a, <vscale x 1 x i1> %b, <vscale x 1 x i32> %c, iXLen %vl) {
-; NOVLOPT-LABEL: vmnor_mm:
+define <vscale x 4 x i32> @vssub_vx(<vscale x 4 x i32> %a, i32 %b, iXLen %vl) {
+; NOVLOPT-LABEL: vssub_vx:
 ; NOVLOPT:       # %bb.0:
-; NOVLOPT-NEXT:    vsetvli a1, zero, e8, mf8, ta, ma
-; NOVLOPT-NEXT:    vmnor.mm v8, v0, v8
-; NOVLOPT-NEXT:    vsetvli zero, a0, e8, mf8, ta, ma
-; NOVLOPT-NEXT:    vmand.mm v0, v0, v8
-; NOVLOPT-NEXT:    vmv1r.v v8, v9
-; NOVLOPT-NEXT:    vsetvli zero, zero, e32, mf2, tu, mu
-; NOVLOPT-NEXT:    vadd.vv v8, v9, v9, v0.t
+; NOVLOPT-NEXT:    vsetvli a2, zero, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vssub.vx v10, v8, a0
+; NOVLOPT-NEXT:    vsetvli zero, a1, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vadd.vv v8, v10, v8
 ; NOVLOPT-NEXT:    ret
 ;
-; VLOPT-LABEL: vmnor_mm:
+; VLOPT-LABEL: vssub_vx:
 ; VLOPT:       # %bb.0:
-; VLOPT-NEXT:    vsetvli zero, a0, e8, mf8, ta, ma
-; VLOPT-NEXT:    vmnor.mm v8, v0, v8
-; VLOPT-NEXT:    vmand.mm v0, v0, v8
-; VLOPT-NEXT:    vmv1r.v v8, v9
-; VLOPT-NEXT:    vsetvli zero, zero, e32, mf2, tu, mu
-; VLOPT-NEXT:    vadd.vv v8, v9, v9, v0.t
+; VLOPT-NEXT:    vsetvli zero, a1, e32, m2, ta, ma
+; VLOPT-NEXT:    vssub.vx v10, v8, a0
+; VLOPT-NEXT:    vadd.vv v8, v10, v8
 ; VLOPT-NEXT:    ret
-  %1 = call <vscale x 1 x i1> @llvm.riscv.vmnor.nxv1i1(<vscale x 1 x i1> %a, <vscale x 1 x i1> %b, iXLen -1)
-  %2 = call <vscale x 1 x i1> @llvm.riscv.vmand.nxv1i1(<vscale x 1 x i1> %a, <vscale x 1 x i1> %1, iXLen %vl)
-  %3 = call <vscale x 1 x i32> @llvm.riscv.vadd.mask.nxv1i32.nxv1i32(<vscale x 1 x i32> %c, <vscale x 1 x i32> %c, <vscale x 1 x i32> %c, <vscale x 1 x i1> %2, iXLen %vl, iXLen 0)
-  ret <vscale x 1 x i32> %3
+  %1 = call <vscale x 4 x i32> @llvm.riscv.vssub(<vscale x 4 x i32> poison, <vscale x 4 x i32> %a, i32 %b, iXLen -1)
+  %2 = call <vscale x 4 x i32> @llvm.riscv.vadd(<vscale x 4 x i32> poison, <vscale x 4 x i32> %1, <vscale x 4 x i32> %a, iXLen %vl)
+  ret <vscale x 4 x i32> %2
 }
 
-define <vscale x 1 x i32> @vmorn_mm(<vscale x 1 x i1> %a, <vscale x 1 x i1> %b, <vscale x 1 x i32> %c, iXLen %vl) {
-; NOVLOPT-LABEL: vmorn_mm:
+define <vscale x 4 x i32> @vsmul_vv(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b, iXLen %vl) {
+; NOVLOPT-LABEL: vsmul_vv:
 ; NOVLOPT:       # %bb.0:
-; NOVLOPT-NEXT:    vsetvli a1, zero, e8, mf8, ta, ma
-; NOVLOPT-NEXT:    vmorn.mm v8, v0, v8
-; NOVLOPT-NEXT:    vsetvli zero, a0, e8, mf8, ta, ma
-; NOVLOPT-NEXT:    vmand.mm v0, v0, v8
-; NOVLOPT-NEXT:    vmv1r.v v8, v9
-; NOVLOPT-NEXT:    vsetvli zero, zero, e32, mf2, tu, mu
-; NOVLOPT-NEXT:    vadd.vv v8, v9, v9, v0.t
+; NOVLOPT-NEXT:    csrwi vxrm, 0
+; NOVLOPT-NEXT:    vsetvli a1, zero, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vsmul.vv v8, v8, v10
+; NOVLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vadd.vv v8, v8, v10
 ; NOVLOPT-NEXT:    ret
 ;
-; VLOPT-LABEL: vmorn_mm:
+; VLOPT-LABEL: vsmul_vv:
 ; VLOPT:       # %bb.0:
-; VLOPT-NEXT:    vsetvli zero, a0, e8, mf8, ta, ma
-; VLOPT-NEXT:    vmorn.mm v8, v0, v8
-; VLOPT-NEXT:    vmand.mm v0, v0, v8
-; VLOPT-NEXT:    vmv1r.v v8, v9
-; VLOPT-NEXT:    vsetvli zero, zero, e32, mf2, tu, mu
-; VLOPT-NEXT:    vadd.vv v8, v9, v9, v0.t
+; VLOPT-NEXT:    csrwi vxrm, 0
+; VLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
+; VLOPT-NEXT:    vsmul.vv v8, v8, v10
+; VLOPT-NEXT:    vadd.vv v8, v8, v10
 ; VLOPT-NEXT:    ret
-  %1 = call <vscale x 1 x i1> @llvm.riscv.vmorn.nxv1i1(<vscale x 1 x i1> %a, <vscale x 1 x i1> %b, iXLen -1)
-  %2 = call <vscale x 1 x i1> @llvm.riscv.vmand.nxv1i1(<vscale x 1 x i1> %a, <vscale x 1 x i1> %1, iXLen %vl)
-  %3 = call <vscale x 1 x i32> @llvm.riscv.vadd.mask.nxv1i32.nxv1i32(<vscale x 1 x i32> %c, <vscale x 1 x i32> %c, <vscale x 1 x i32> %c, <vscale x 1 x i1> %2, iXLen %vl, iXLen 0)
-  ret <vscale x 1 x i32> %3
+  %1 = call <vscale x 4 x i32> @llvm.riscv.vsmul.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %a, <vscale x 4 x i32> %b, iXLen 0, iXLen -1)
+  %2 = call <vscale x 4 x i32> @llvm.riscv.vadd.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %1, <vscale x 4 x i32> %b, iXLen %vl)
+  ret <vscale x 4 x i32> %2
 }
 
-define <vscale x 1 x i32> @vmxnor_mm(<vscale x 1 x i1> %a, <vscale x 1 x i1> %b, <vscale x 1 x i32> %c, iXLen %vl) {
-; NOVLOPT-LABEL: vmxnor_mm:
+define <vscale x 4 x i32> @vsmul_vx(<vscale x 4 x i32> %a, i32 %b, iXLen %vl) {
+; NOVLOPT-LABEL: vsmul_vx:
 ; NOVLOPT:       # %bb.0:
-; NOVLOPT-NEXT:    vsetvli a1, zero, e8, mf8, ta, ma
-; NOVLOPT-NEXT:    vmxnor.mm v8, v0, v8
-; NOVLOPT-NEXT:    vsetvli zero, a0, e8, mf8, ta, ma
-; NOVLOPT-NEXT:    vmand.mm v0, v0, v8
-; NOVLOPT-NEXT:    vmv1r.v v8, v9
-; NOVLOPT-NEXT:    vsetvli zero, zero, e32, mf2, tu, mu
-; NOVLOPT-NEXT:    vadd.vv v8, v9, v9, v0.t
+; NOVLOPT-NEXT:    csrwi vxrm, 0
+; NOVLOPT-NEXT:    vsetvli a2, zero, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vsmul.vx v10, v8, a0
+; NOVLOPT-NEXT:    vsetvli zero, a1, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vadd.vv v8, v10, v8
 ; NOVLOPT-NEXT:    ret
 ;
-; VLOPT-LABEL: vmxnor_mm:
+; VLOPT-LABEL: vsmul_vx:
 ; VLOPT:       # %bb.0:
-; VLOPT-NEXT:    vsetvli zero, a0, e8, mf8, ta, ma
-; VLOPT-NEXT:    vmxnor.mm v8, v0, v8
-; VLOPT-NEXT:    vmand.mm v0, v0, v8
-; VLOPT-NEXT:    vmv1r.v v8, v9
-; VLOPT-NEXT:    vsetvli zero, zero, e32, mf2, tu, mu
-; VLOPT-NEXT:    vadd.vv v8, v9, v9, v0.t
+; VLOPT-NEXT:    csrwi vxrm, 0
+; VLOPT-NEXT:    vsetvli zero, a1, e32, m2, ta, ma
+; VLOPT-NEXT:    vsmul.vx v10, v8, a0
+; VLOPT-NEXT:    vadd.vv v8, v10, v8
 ; VLOPT-NEXT:    ret
-  %1 = call <vscale x 1 x i1> @llvm.riscv.vmxnor.nxv1i1(<vscale x 1 x i1> %a, <vscale x 1 x i1> %b, iXLen -1)
-  %2 = call <vscale x 1 x i1> @llvm.riscv.vmand.nxv1i1(<vscale x 1 x i1> %a, <vscale x 1 x i1> %1, iXLen %vl)
-  %3 = call <vscale x 1 x i32> @llvm.riscv.vadd.mask.nxv1i32.nxv1i32(<vscale x 1 x i32> %c, <vscale x 1 x i32> %c, <vscale x 1 x i32> %c, <vscale x 1 x i1> %2, iXLen %vl, iXLen 0)
-  ret <vscale x 1 x i32> %3
+  %1 = call <vscale x 4 x i32> @llvm.riscv.vsmul.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %a, i32 %b, iXLen 0, iXLen -1)
+  %2 = call <vscale x 4 x i32> @llvm.riscv.vadd.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %1, <vscale x 4 x i32> %a, iXLen %vl)
+  ret <vscale x 4 x i32> %2
 }
 
-define <vscale x 1 x i32> @vmsbf_m(<vscale x 1 x i1> %a, <vscale x 1 x i32> %c, iXLen %vl) {
-; NOVLOPT-LABEL: vmsbf_m:
+define <vscale x 4 x i32> @vssrl_vv(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b, iXLen %vl) {
+; NOVLOPT-LABEL: vssrl_vv:
 ; NOVLOPT:       # %bb.0:
-; NOVLOPT-NEXT:    vsetvli a1, zero, e8, mf8, ta, ma
-; NOVLOPT-NEXT:    vmsbf.m v9, v0
-; NOVLOPT-NEXT:    vsetvli zero, a0, e8, mf8, ta, ma
-; NOVLOPT-NEXT:    vmand.mm v0, v0, v9
-; NOVLOPT-NEXT:    vsetvli zero, zero, e32, mf2, tu, mu
-; NOVLOPT-NEXT:    vadd.vv v8, v8, v8, v0.t
+; NOVLOPT-NEXT:    csrwi vxrm, 0
+; NOVLOPT-NEXT:    vsetvli a1, zero, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vssrl.vv v8, v8, v10
+; NOVLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vadd.vv v8, v8, v10
 ; NOVLOPT-NEXT:    ret
 ;
-; VLOPT-LABEL: vmsbf_m:
+; VLOPT-LABEL: vssrl_vv:
 ; VLOPT:       # %bb.0:
-; VLOPT-NEXT:    vsetvli zero, a0, e8, mf8, ta, ma
-; VLOPT-NEXT:    vmsbf.m v9, v0
-; VLOPT-NEXT:    vmand.mm v0, v0, v9
-; VLOPT-NEXT:    vsetvli zero, zero, e32, mf2, tu, mu
-; VLOPT-NEXT:    vadd.vv v8, v8, v8, v0.t
+; VLOPT-NEXT:    csrwi vxrm, 0
+; VLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
+; VLOPT-NEXT:    vssrl.vv v8, v8, v10
+; VLOPT-NEXT:    vadd.vv v8, v8, v10
 ; VLOPT-NEXT:    ret
-  %1 = call <vscale x 1 x i1> @llvm.riscv.vmsbf.nxv1i1(<vscale x 1 x i1> %a, iXLen -1)
-  %2 = call <vscale x 1 x i1> @llvm.riscv.vmand.nxv1i1(<vscale x 1 x i1> %a, <vscale x 1 x i1> %1, iXLen %vl)
-  %3 = call <vscale x 1 x i32> @llvm.riscv.vadd.mask.nxv1i32.nxv1i32(<vscale x 1 x i32> %c, <vscale x 1 x i32> %c, <vscale x 1 x i32> %c, <vscale x 1 x i1> %2, iXLen %vl, iXLen 0)
-  ret <vscale x 1 x i32> %3
+  %1 = call <vscale x 4 x i32> @llvm.riscv.vssrl.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %a, <vscale x 4 x i32> %b, iXLen 0, iXLen -1)
+  %2 = call <vscale x 4 x i32> @llvm.riscv.vadd.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %1, <vscale x 4 x i32> %b, iXLen %vl)
+  ret <vscale x 4 x i32> %2
 }
 
-define <vscale x 1 x i32> @vmsif_m(<vscale x 1 x i1> %a, <vscale x 1 x i32> %c, iXLen %vl) {
-; NOVLOPT-LABEL: vmsif_m:
+define <vscale x 4 x i32> @vssrl_vx(<vscale x 4 x i32> %a, iXLen %b, iXLen %vl) {
+; NOVLOPT-LABEL: vssrl_vx:
 ; NOVLOPT:       # %bb.0:
-; NOVLOPT-NEXT:    vsetvli a1, zero, e8, mf8, ta, ma
-; NOVLOPT-NEXT:    vmsif.m v9, v0
-; NOVLOPT-NEXT:    vsetvli zero, a0, e8, mf8, ta, ma
-; NOVLOPT-NEXT:    vmand.mm v0, v0, v9
-; NOVLOPT-NEXT:    vsetvli zero, zero, e32, mf2, tu, mu
-; NOVLOPT-NEXT:    vadd.vv v8, v8, v8, v0.t
+; NOVLOPT-NEXT:    csrwi vxrm, 0
+; NOVLOPT-NEXT:    vsetvli a2, zero, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vssrl.vx v10, v8, a0
+; NOVLOPT-NEXT:    vsetvli zero, a1, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vadd.vv v8, v10, v8
 ; NOVLOPT-NEXT:    ret
 ;
-; VLOPT-LABEL: vmsif_m:
+; VLOPT-LABEL: vssrl_vx:
 ; VLOPT:       # %bb.0:
-; VLOPT-NEXT:    vsetvli zero, a0, e8, mf8, ta, ma
-; VLOPT-NEXT:    vmsif.m v9, v0
-; VLOPT-NEXT:    vmand.mm v0, v0, v9
-; VLOPT-NEXT:    vsetvli zero, zero, e32, mf2, tu, mu
-; VLOPT-NEXT:    vadd.vv v8, v8, v8, v0.t
+; VLOPT-NEXT:    csrwi vxrm, 0
+; VLOPT-NEXT:    vsetvli zero, a1, e32, m2, ta, ma
+; VLOPT-NEXT:    vssrl.vx v10, v8, a0
+; VLOPT-NEXT:    vadd.vv v8, v10, v8
 ; VLOPT-NEXT:    ret
-  %1 = call <vscale x 1 x i1> @llvm.riscv.vmsif.nxv1i1(<vscale x 1 x i1> %a, iXLen -1)
-  %2 = call <vscale x 1 x i1> @llvm.riscv.vmand.nxv1i1(<vscale x 1 x i1> %a, <vscale x 1 x i1> %1, iXLen %vl)
-  %3 = call <vscale x 1 x i32> @llvm.riscv.vadd.mask.nxv1i32.nxv1i32(<vscale x 1 x i32> %c, <vscale x 1 x i32> %c, <vscale x 1 x i32> %c, <vscale x 1 x i1> %2, iXLen %vl, iXLen 0)
-  ret <vscale x 1 x i32> %3
+  %1 = call <vscale x 4 x i32> @llvm.riscv.vssrl.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %a, iXLen %b, iXLen 0, iXLen -1)
+  %2 = call <vscale x 4 x i32> @llvm.riscv.vadd.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %1, <vscale x 4 x i32> %a, iXLen %vl)
+  ret <vscale x 4 x i32> %2
 }
 
-define <vscale x 1 x i32> @vmsof_m(<vscale x 1 x i1> %a, <vscale x 1 x i32> %c, iXLen %vl) {
-; NOVLOPT-LABEL: vmsof_m:
+define <vscale x 4 x i32> @vssrl_vi(<vscale x 4 x i32> %a, iXLen %vl) {
+; NOVLOPT-LABEL: vssrl_vi:
 ; NOVLOPT:       # %bb.0:
-; NOVLOPT-NEXT:    vsetvli a1, zero, e8, mf8, ta, ma
-; NOVLOPT-NEXT:    vmsof.m v9, v0
-; NOVLOPT-NEXT:    vsetvli zero, a0, e8, mf8, ta, ma
-; NOVLOPT-NEXT:    vmand.mm v0, v0, v9
-; NOVLOPT-NEXT:    vsetvli zero, zero, e32, mf2, tu, mu
-; NOVLOPT-NEXT:    vadd.vv v8, v8, v8, v0.t
+; NOVLOPT-NEXT:    csrwi vxrm, 0
+; NOVLOPT-NEXT:    vsetvli a1, zero, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vssrl.vi v10, v8, 5
+; NOVLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vadd.vv v8, v10, v8
 ; NOVLOPT-NEXT:    ret
 ;
-; VLOPT-LABEL: vmsof_m:
+; VLOPT-LABEL: vssrl_vi:
 ; VLOPT:       # %bb.0:
-; VLOPT-NEXT:    vsetvli zero, a0, e8, mf8, ta, ma
-; VLOPT-NEXT:    vmsof.m v9, v0
-; VLOPT-NEXT:    vmand.mm v0, v0, v9
-; VLOPT-NEXT:    vsetvli zero, zero, e32, mf2, tu, mu
-; VLOPT-NEXT:    vadd.vv v8, v8, v8, v0.t
+; VLOPT-NEXT:    csrwi vxrm, 0
+; VLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
+; VLOPT-NEXT:    vssrl.vi v10, v8, 5
+; VLOPT-NEXT:    vadd.vv v8, v10, v8
 ; VLOPT-NEXT:    ret
-  %1 = call <vscale x 1 x i1> @llvm.riscv.vmsof.nxv1i1(<vscale x 1 x i1> %a, iXLen -1)
-  %2 = call <vscale x 1 x i1> @llvm.riscv.vmand.nxv1i1(<vscale x 1 x i1> %a, <vscale x 1 x i1> %1, iXLen %vl)
-  %3 = call <vscale x 1 x i32> @llvm.riscv.vadd.mask.nxv1i32.nxv1i32(<vscale x 1 x i32> %c, <vscale x 1 x i32> %c, <vscale x 1 x i32> %c, <vscale x 1 x i1> %2, iXLen %vl, iXLen 0)
-  ret <vscale x 1 x i32> %3
+  %1 = call <vscale x 4 x i32> @llvm.riscv.vssrl.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %a, iXLen 5, iXLen 0, iXLen -1)
+  %2 = call <vscale x 4 x i32> @llvm.riscv.vadd.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %1, <vscale x 4 x i32> %a, iXLen %vl)
+  ret <vscale x 4 x i32> %2
 }
 
-define <vscale x 4 x i32> @viota_m(<vscale x 4 x i1> %a, <vscale x 4 x i32> %c, iXLen %vl) {
-; NOVLOPT-LABEL: viota_m:
+define <vscale x 4 x i32> @vssra_vv(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b, iXLen %vl) {
+; NOVLOPT-LABEL: vssra_vv:
 ; NOVLOPT:       # %bb.0:
+; NOVLOPT-NEXT:    csrwi vxrm, 0
 ; NOVLOPT-NEXT:    vsetvli a1, zero, e32, m2, ta, ma
-; NOVLOPT-NEXT:    viota.m v10, v0
+; NOVLOPT-NEXT:    vssra.vv v8, v8, v10
 ; NOVLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
-; NOVLOPT-NEXT:    vadd.vv v8, v10, v8
+; NOVLOPT-NEXT:    vadd.vv v8, v8, v10
 ; NOVLOPT-NEXT:    ret
 ;
-; VLOPT-LABEL: viota_m:
+; VLOPT-LABEL: vssra_vv:
 ; VLOPT:       # %bb.0:
+; VLOPT-NEXT:    csrwi vxrm, 0
 ; VLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
-; VLOPT-NEXT:    viota.m v10, v0
-; VLOPT-NEXT:    vadd.vv v8, v10, v8
+; VLOPT-NEXT:    vssra.vv v8, v8, v10
+; VLOPT-NEXT:    vadd.vv v8, v8, v10
 ; VLOPT-NEXT:    ret
-  %1 = call <vscale x 4 x i32> @llvm.riscv.viota.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i1> %a, iXLen -1)
-  %2 = call <vscale x 4 x i32> @llvm.riscv.vadd.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %1, <vscale x 4 x i32> %c, iXLen %vl)
+  %1 = call <vscale x 4 x i32> @llvm.riscv.vssra.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %a, <vscale x 4 x i32> %b, iXLen 0, iXLen -1)
+  %2 = call <vscale x 4 x i32> @llvm.riscv.vadd.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %1, <vscale x 4 x i32> %b, iXLen %vl)
   ret <vscale x 4 x i32> %2
 }
 
-define <vscale x 4 x i32> @vid.v(<vscale x 4 x i32> %c, iXLen %vl) {
-; NOVLOPT-LABEL: vid.v:
+define <vscale x 4 x i32> @vssra_vx(<vscale x 4 x i32> %a, iXLen %b, iXLen %vl) {
+; NOVLOPT-LABEL: vssra_vx:
 ; NOVLOPT:       # %bb.0:
-; NOVLOPT-NEXT:    vsetvli a1, zero, e32, m2, ta, ma
-; NOVLOPT-NEXT:    vid.v v10
-; NOVLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
+; NOVLOPT-NEXT:    csrwi vxrm, 0
+; NOVLOPT-NEXT:    vsetvli a2, zero, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vssra.vx v10, v8, a0
+; NOVLOPT-NEXT:    vsetvli zero, a1, e32, m2, ta, ma
 ; NOVLOPT-NEXT:    vadd.vv v8, v10, v8
 ; NOVLOPT-NEXT:    ret
 ;
-; VLOPT-LABEL: vid.v:
+; VLOPT-LABEL: vssra_vx:
 ; VLOPT:       # %bb.0:
-; VLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
-; VLOPT-NEXT:    vid.v v10
+; VLOPT-NEXT:    csrwi vxrm, 0
+; VLOPT-NEXT:    vsetvli zero, a1, e32, m2, ta, ma
+; VLOPT-NEXT:    vssra.vx v10, v8, a0
 ; VLOPT-NEXT:    vadd.vv v8, v10, v8
 ; VLOPT-NEXT:    ret
-  %1 = call <vscale x 4 x i32> @llvm.riscv.vid.nxv4i32(<vscale x 4 x i32> poison, iXLen -1)
-  %2 = call <vscale x 4 x i32> @llvm.riscv.vadd.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %1, <vscale x 4 x i32> %c, iXLen %vl)
+  %1 = call <vscale x 4 x i32> @llvm.riscv.vssra.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %a, iXLen %b, iXLen 0, iXLen -1)
+  %2 = call <vscale x 4 x i32> @llvm.riscv.vadd.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %1, <vscale x 4 x i32> %a, iXLen %vl)
   ret <vscale x 4 x i32> %2
 }
 
-define <vscale x 4 x float> @vfadd_vv(<vscale x 4 x float> %a, <vscale x 4 x float> %b, iXLen %vl) {
-; NOVLOPT-LABEL: vfadd_vv:
+define <vscale x 4 x i32> @vssra_vi(<vscale x 4 x i32> %a, iXLen %vl) {
+; NOVLOPT-LABEL: vssra_vi:
 ; NOVLOPT:       # %bb.0:
+; NOVLOPT-NEXT:    csrwi vxrm, 0
 ; NOVLOPT-NEXT:    vsetvli a1, zero, e32, m2, ta, ma
-; NOVLOPT-NEXT:    vfadd.vv v8, v8, v10
+; NOVLOPT-NEXT:    vssra.vi v10, v8, 5
 ; NOVLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
-; NOVLOPT-NEXT:    vfadd.vv v8, v8, v10
+; NOVLOPT-NEXT:    vadd.vv v8, v10, v8
 ; NOVLOPT-NEXT:    ret
 ;
-; VLOPT-LABEL: vfadd_vv:
+; VLOPT-LABEL: vssra_vi:
 ; VLOPT:       # %bb.0:
+; VLOPT-NEXT:    csrwi vxrm, 0
 ; VLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
-; VLOPT-NEXT:    vfadd.vv v8, v8, v10
-; VLOPT-NEXT:    vfadd.vv v8, v8, v10
+; VLOPT-NEXT:    vssra.vi v10, v8, 5
+; VLOPT-NEXT:    vadd.vv v8, v10, v8
 ; VLOPT-NEXT:    ret
-  %1 = call <vscale x 4 x float> @llvm.riscv.vfadd.nxv4f32.nxv4f32(<vscale x 4 x float> poison, <vscale x 4 x float> %a, <vscale x 4 x float> %b, iXLen 7, iXLen -1)
-  %2 = call <vscale x 4 x float> @llvm.riscv.vfadd.nxv4f32.nxv4f32(<vscale x 4 x float> poison, <vscale x 4 x float> %1, <vscale x 4 x float> %b, iXLen 7, iXLen %vl)
-  ret <vscale x 4 x float> %2
+  %1 = call <vscale x 4 x i32> @llvm.riscv.vssra.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %a, iXLen 5, iXLen 0, iXLen -1)
+  %2 = call <vscale x 4 x i32> @llvm.riscv.vadd.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %1, <vscale x 4 x i32> %a, iXLen %vl)
+  ret <vscale x 4 x i32> %2
 }
 
-define <vscale x 4 x float> @vfadd_vf(<vscale x 4 x float> %a, float %b, iXLen %vl) {
-; NOVLOPT-LABEL: vfadd_vf:
+define <vscale x 4 x i32> @vnclipu_vv(<vscale x 4 x i64> %a, <vscale x 4 x i32> %b, iXLen %vl) {
+; NOVLOPT-LABEL: vnclipu_vv:
 ; NOVLOPT:       # %bb.0:
+; NOVLOPT-NEXT:    csrwi vxrm, 0
 ; NOVLOPT-NEXT:    vsetvli a1, zero, e32, m2, ta, ma
-; NOVLOPT-NEXT:    vfadd.vf v10, v8, fa0
+; NOVLOPT-NEXT:    vnclipu.wv v14, v8, v12
 ; NOVLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
-; NOVLOPT-NEXT:    vfadd.vv v8, v10, v8
+; NOVLOPT-NEXT:    vadd.vv v8, v14, v14
 ; NOVLOPT-NEXT:    ret
 ;
-; VLOPT-LABEL: vfadd_vf:
+; VLOPT-LABEL: vnclipu_vv:
 ; VLOPT:       # %bb.0:
+; VLOPT-NEXT:    csrwi vxrm, 0
 ; VLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
-; VLOPT-NEXT:    vfadd.vf v10, v8, fa0
-; VLOPT-NEXT:    vfadd.vv v8, v10, v8
+; VLOPT-NEXT:    vnclipu.wv v14, v8, v12
+; VLOPT-NEXT:    vadd.vv v8, v14, v14
 ; VLOPT-NEXT:    ret
-  %1 = call <vscale x 4 x float> @llvm.riscv.vfadd.nxv4f32.nxv4f32(<vscale x 4 x float> poison, <vscale x 4 x float> %a, float %b, iXLen 7, iXLen -1)
-  %2 = call <vscale x 4 x float> @llvm.riscv.vfadd.nxv4f32.nxv4f32(<vscale x 4 x float> poison, <vscale x 4 x float> %1, <vscale x 4 x float> %a, iXLen 7, iXLen %vl)
-  ret <vscale x 4 x float> %2
+  %1 = call <vscale x 4 x i32> @llvm.riscv.vnclipu(<vscale x 4 x i32> poison, <vscale x 4 x i64> %a, <vscale x 4 x i32> %b, iXLen 0, iXLen -1)
+  %2 = call <vscale x 4 x i32> @llvm.riscv.vadd(<vscale x 4 x i32> poison, <vscale x 4 x i32> %1, <vscale x 4 x i32> %1, iXLen %vl)
+  ret <vscale x 4 x i32> %2
 }
 
-define <vscale x 4 x float> @vfsub_vv(<vscale x 4 x float> %a, <vscale x 4 x float> %b, iXLen %vl) {
-; NOVLOPT-LABEL: vfsub_vv:
+define <vscale x 4 x i32> @vnclipu_vx(<vscale x 4 x i64> %a, iXLen %b, iXLen %vl) {
+; NOVLOPT-LABEL: vnclipu_vx:
 ; NOVLOPT:       # %bb.0:
-; NOVLOPT-NEXT:    vsetvli a1, zero, e32, m2, ta, ma
-; NOVLOPT-NEXT:    vfsub.vv v8, v8, v10
-; NOVLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
-; NOVLOPT-NEXT:    vfadd.vv v8, v8, v10
+; NOVLOPT-NEXT:    csrwi vxrm, 0
+; NOVLOPT-NEXT:    vsetvli a2, zero, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vnclipu.wx v12, v8, a0
+; NOVLOPT-NEXT:    vsetvli zero, a1, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vadd.vv v8, v12, v12
 ; NOVLOPT-NEXT:    ret
 ;
-; VLOPT-LABEL: vfsub_vv:
+; VLOPT-LABEL: vnclipu_vx:
 ; VLOPT:       # %bb.0:
-; VLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
-; VLOPT-NEXT:    vfsub.vv v8, v8, v10
-; VLOPT-NEXT:    vfadd.vv v8, v8, v10
+; VLOPT-NEXT:    csrwi vxrm, 0
+; VLOPT-NEXT:    vsetvli zero, a1, e32, m2, ta, ma
+; VLOPT-NEXT:    vnclipu.wx v12, v8, a0
+; VLOPT-NEXT:    vadd.vv v8, v12, v12
 ; VLOPT-NEXT:    ret
-  %1 = call <vscale x 4 x float> @llvm.riscv.vfsub.nxv4f32.nxv4f32(<vscale x 4 x float> poison, <vscale x 4 x float> %a, <vscale x 4 x float> %b, iXLen 7, iXLen -1)
-  %2 = call <vscale x 4 x float> @llvm.riscv.vfadd.nxv4f32.nxv4f32(<vscale x 4 x float> poison, <vscale x 4 x float> %1, <vscale x 4 x float> %b, iXLen 7, iXLen %vl)
-  ret <vscale x 4 x float> %2
+  %1 = call <vscale x 4 x i32> @llvm.riscv.vnclipu(<vscale x 4 x i32> poison, <vscale x 4 x i64> %a, iXLen %b, iXLen 0, iXLen -1)
+  %2 = call <vscale x 4 x i32> @llvm.riscv.vadd(<vscale x 4 x i32> poison, <vscale x 4 x i32> %1, <vscale x 4 x i32> %1, iXLen %vl)
+  ret <vscale x 4 x i32> %2
 }
 
-define <vscale x 4 x float> @vfsub_vf(<vscale x 4 x float> %a, float %b, iXLen %vl) {
-; NOVLOPT-LABEL: vfsub_vf:
+define <vscale x 4 x i32> @vnclipu_vi(<vscale x 4 x i64> %a, iXLen %vl) {
+; NOVLOPT-LABEL: vnclipu_vi:
 ; NOVLOPT:       # %bb.0:
+; NOVLOPT-NEXT:    csrwi vxrm, 0
 ; NOVLOPT-NEXT:    vsetvli a1, zero, e32, m2, ta, ma
-; NOVLOPT-NEXT:    vfsub.vf v10, v8, fa0
+; NOVLOPT-NEXT:    vnclipu.wi v12, v8, 5
 ; NOVLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
-; NOVLOPT-NEXT:    vfadd.vv v8, v10, v8
+; NOVLOPT-NEXT:    vadd.vv v8, v12, v12
 ; NOVLOPT-NEXT:    ret
 ;
-; VLOPT-LABEL: vfsub_vf:
+; VLOPT-LABEL: vnclipu_vi:
 ; VLOPT:       # %bb.0:
+; VLOPT-NEXT:    csrwi vxrm, 0
 ; VLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
-; VLOPT-NEXT:    vfsub.vf v10, v8, fa0
-; VLOPT-NEXT:    vfadd.vv v8, v10, v8
+; VLOPT-NEXT:    vnclipu.wi v12, v8, 5
+; VLOPT-NEXT:    vadd.vv v8, v12, v12
 ; VLOPT-NEXT:    ret
-  %1 = call <vscale x 4 x float> @llvm.riscv.vfsub.nxv4f32.nxv4f32(<vscale x 4 x float> poison, <vscale x 4 x float> %a, float %b, iXLen 7, iXLen -1)
-  %2 = call <vscale x 4 x float> @llvm.riscv.vfadd.nxv4f32.nxv4f32(<vscale x 4 x float> poison, <vscale x 4 x float> %1, <vscale x 4 x float> %a, iXLen 7, iXLen %vl)
-  ret <vscale x 4 x float> %2
+  %1 = call <vscale x 4 x i32> @llvm.riscv.vnclipu(<vscale x 4 x i32> poison, <vscale x 4 x i64> %a, iXLen 5, iXLen 0, iXLen -1)
+  %2 = call <vscale x 4 x i32> @llvm.riscv.vadd(<vscale x 4 x i32> poison, <vscale x 4 x i32> %1, <vscale x 4 x i32> %1, iXLen %vl)
+  ret <vscale x 4 x i32> %2
 }
 
-define <vscale x 4 x float> @vfrsub_vf(<vscale x 4 x float> %a, float %b, iXLen %vl) {
-; NOVLOPT-LABEL: vfrsub_vf:
+define <vscale x 4 x i32> @vnclip_vv(<vscale x 4 x i64> %a, <vscale x 4 x i32> %b, iXLen %vl) {
+; NOVLOPT-LABEL: vnclip_vv:
 ; NOVLOPT:       # %bb.0:
+; NOVLOPT-NEXT:    csrwi vxrm, 0
 ; NOVLOPT-NEXT:    vsetvli a1, zero, e32, m2, ta, ma
-; NOVLOPT-NEXT:    vfrsub.vf v10, v8, fa0
+; NOVLOPT-NEXT:    vnclip.wv v14, v8, v12
 ; NOVLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
-; NOVLOPT-NEXT:    vfadd.vv v8, v10, v8
+; NOVLOPT-NEXT:    vadd.vv v8, v14, v14
 ; NOVLOPT-NEXT:    ret
 ;
-; VLOPT-LABEL: vfrsub_vf:
+; VLOPT-LABEL: vnclip_vv:
 ; VLOPT:       # %bb.0:
+; VLOPT-NEXT:    csrwi vxrm, 0
 ; VLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
-; VLOPT-NEXT:    vfrsub.vf v10, v8, fa0
-; VLOPT-NEXT:    vfadd.vv v8, v10, v8
+; VLOPT-NEXT:    vnclip.wv v14, v8, v12
+; VLOPT-NEXT:    vadd.vv v8, v14, v14
 ; VLOPT-NEXT:    ret
-  %1 = call <vscale x 4 x float> @llvm.riscv.vfrsub.nxv4f32.nxv4f32(<vscale x 4 x float> poison, <vscale x 4 x float> %a, float %b, iXLen 7, iXLen -1)
-  %2 = call <vscale x 4 x float> @llvm.riscv.vfadd.nxv4f32.nxv4f32(<vscale x 4 x float> poison, <vscale x 4 x float> %1, <vscale x 4 x float> %a, iXLen 7, iXLen %vl)
-  ret <vscale x 4 x float> %2
+  %1 = call <vscale x 4 x i32> @llvm.riscv.vnclip(<vscale x 4 x i32> poison, <vscale x 4 x i64> %a, <vscale x 4 x i32> %b, iXLen 0, iXLen -1)
+  %2 = call <vscale x 4 x i32> @llvm.riscv.vadd(<vscale x 4 x i32> poison, <vscale x 4 x i32> %1, <vscale x 4 x i32> %1, iXLen %vl)
+  ret <vscale x 4 x i32> %2
 }
 
-define <vscale x 4 x double> @vfwadd_vv(<vscale x 4 x float> %a, <vscale x 4 x float> %b, iXLen %vl) {
-; NOVLOPT-LABEL: vfwadd_vv:
+define <vscale x 4 x i32> @vnclip_vx(<vscale x 4 x i64> %a, iXLen %b, iXLen %vl) {
+; NOVLOPT-LABEL: vnclip_vx:
 ; NOVLOPT:       # %bb.0:
-; NOVLOPT-NEXT:    vsetvli a1, zero, e32, m2, ta, ma
-; NOVLOPT-NEXT:    vfwadd.vv v12, v8, v10
-; NOVLOPT-NEXT:    vsetvli zero, a0, e64, m4, ta, ma
-; NOVLOPT-NEXT:    vfadd.vv v8, v12, v12
+; NOVLOPT-NEXT:    csrwi vxrm, 0
+; NOVLOPT-NEXT:    vsetvli a2, zero, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vnclip.wx v12, v8, a0
+; NOVLOPT-NEXT:    vsetvli zero, a1, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vadd.vv v8, v12, v12
 ; NOVLOPT-NEXT:    ret
 ;
-; VLOPT-LABEL: vfwadd_vv:
+; VLOPT-LABEL: vnclip_vx:
 ; VLOPT:       # %bb.0:
-; VLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
-; VLOPT-NEXT:    vfwadd.vv v12, v8, v10
-; VLOPT-NEXT:    vsetvli zero, zero, e64, m4, ta, ma
-; VLOPT-NEXT:    vfadd.vv v8, v12, v12
+; VLOPT-NEXT:    csrwi vxrm, 0
+; VLOPT-NEXT:    vsetvli zero, a1, e32, m2, ta, ma
+; VLOPT-NEXT:    vnclip.wx v12, v8, a0
+; VLOPT-NEXT:    vadd.vv v8, v12, v12
 ; VLOPT-NEXT:    ret
-  %1 = call <vscale x 4 x double> @llvm.riscv.vfwadd.nxv4f64.nxv4f32.nxv4f32(<vscale x 4 x double> poison, <vscale x 4 x float> %a, <vscale x 4 x float> %b, iXLen 7, iXLen -1)
-  %2 = call <vscale x 4 x double> @llvm.riscv.vfadd.nxv4f64.nxv4f64(<vscale x 4 x double> poison, <vscale x 4 x double> %1, <vscale x 4 x double> %1, iXLen 7, iXLen %vl)
-  ret <vscale x 4 x double> %2
+  %1 = call <vscale x 4 x i32> @llvm.riscv.vnclip(<vscale x 4 x i32> poison, <vscale x 4 x i64> %a, iXLen %b, iXLen 0, iXLen -1)
+  %2 = call <vscale x 4 x i32> @llvm.riscv.vadd(<vscale x 4 x i32> poison, <vscale x 4 x i32> %1, <vscale x 4 x i32> %1, iXLen %vl)
+  ret <vscale x 4 x i32> %2
 }
 
-define <vscale x 4 x double> @vfwadd_vf(<vscale x 4 x float> %a, float %b, iXLen %vl) {
-; NOVLOPT-LABEL: vfwadd_vf:
+define <vscale x 4 x i32> @vnclip_vi(<vscale x 4 x i64> %a, iXLen %vl) {
+; NOVLOPT-LABEL: vnclip_vi:
 ; NOVLOPT:       # %bb.0:
+; NOVLOPT-NEXT:    csrwi vxrm, 0
 ; NOVLOPT-NEXT:    vsetvli a1, zero, e32, m2, ta, ma
-; NOVLOPT-NEXT:    vfwadd.vf v12, v8, fa0
-; NOVLOPT-NEXT:    vsetvli zero, a0, e64, m4, ta, ma
-; NOVLOPT-NEXT:    vfadd.vv v8, v12, v12
+; NOVLOPT-NEXT:    vnclip.wi v12, v8, 5
+; NOVLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vadd.vv v8, v12, v12
 ; NOVLOPT-NEXT:    ret
 ;
-; VLOPT-LABEL: vfwadd_vf:
+; VLOPT-LABEL: vnclip_vi:
 ; VLOPT:       # %bb.0:
+; VLOPT-NEXT:    csrwi vxrm, 0
 ; VLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
-; VLOPT-NEXT:    vfwadd.vf v12, v8, fa0
-; VLOPT-NEXT:    vsetvli zero, zero, e64, m4, ta, ma
-; VLOPT-NEXT:    vfadd.vv v8, v12, v12
+; VLOPT-NEXT:    vnclip.wi v12, v8, 5
+; VLOPT-NEXT:    vadd.vv v8, v12, v12
 ; VLOPT-NEXT:    ret
-  %1 = call <vscale x 4 x double> @llvm.riscv.vfwadd.nxv4f64.nxv4f32.f32(<vscale x 4 x double> poison, <vscale x 4 x float> %a, float %b, iXLen 7, iXLen -1)
-  %2 = call <vscale x 4 x double> @llvm.riscv.vfadd.nxv4f64.nxv4f64(<vscale x 4 x double> poison, <vscale x 4 x double> %1, <vscale x 4 x double> %1, iXLen 7, iXLen %vl)
-  ret <vscale x 4 x double> %2
+  %1 = call <vscale x 4 x i32> @llvm.riscv.vnclip(<vscale x 4 x i32> poison, <vscale x 4 x i64> %a, iXLen 5, iXLen 0, iXLen -1)
+  %2 = call <vscale x 4 x i32> @llvm.riscv.vadd(<vscale x 4 x i32> poison, <vscale x 4 x i32> %1, <vscale x 4 x i32> %1, iXLen %vl)
+  ret <vscale x 4 x i32> %2
 }
 
-define <vscale x 4 x double> @vfwsub_vv(<vscale x 4 x float> %a, <vscale x 4 x float> %b, iXLen %vl) {
-; NOVLOPT-LABEL: vfwsub_vv:
+define <vscale x 4 x i32> @vmv_v_i(<vscale x 4 x i32> %a, i32 %x, iXLen %vl) {
+; NOVLOPT-LABEL: vmv_v_i:
 ; NOVLOPT:       # %bb.0:
-; NOVLOPT-NEXT:    vsetvli a1, zero, e32, m2, ta, ma
-; NOVLOPT-NEXT:    vfwsub.vv v12, v8, v10
-; NOVLOPT-NEXT:    vsetvli zero, a0, e64, m4, ta, ma
-; NOVLOPT-NEXT:    vfadd.vv v8, v12, v12
-; NOVLOPT-NEXT:    ret
-;
-; VLOPT-LABEL: vfwsub_vv:
+; NOVLOPT-NEXT:    vsetvli a0, zero, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vmv.v.i v10, 5
+; NOVLOPT-NEXT:    vsetvli zero, a1, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vadd.vv v8, v10, v8
+; NOVLOPT-NEXT:    ret
+;
+; VLOPT-LABEL: vmv_v_i:
+; VLOPT:       # %bb.0:
+; VLOPT-NEXT:    vsetvli zero, a1, e32, m2, ta, ma
+; VLOPT-NEXT:    vmv.v.i v10, 5
+; VLOPT-NEXT:    vadd.vv v8, v10, v8
+; VLOPT-NEXT:    ret
+  %1 = call <vscale x 4 x i32> @llvm.riscv.vmv.v.x.nxv4i32(<vscale x 4 x i32> poison, i32 5, iXLen -1)
+  %2 = call <vscale x 4 x i32> @llvm.riscv.vadd.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %1, <vscale x 4 x i32> %a, iXLen %vl)
+  ret <vscale x 4 x i32> %2
+}
+
+define <vscale x 4 x i32> @vmv_v_x(<vscale x 4 x i32> %a, i32 %x, iXLen %vl) {
+; NOVLOPT-LABEL: vmv_v_x:
+; NOVLOPT:       # %bb.0:
+; NOVLOPT-NEXT:    vsetvli a2, zero, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vmv.v.x v10, a0
+; NOVLOPT-NEXT:    vsetvli zero, a1, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vadd.vv v8, v10, v8
+; NOVLOPT-NEXT:    ret
+;
+; VLOPT-LABEL: vmv_v_x:
+; VLOPT:       # %bb.0:
+; VLOPT-NEXT:    vsetvli zero, a1, e32, m2, ta, ma
+; VLOPT-NEXT:    vmv.v.x v10, a0
+; VLOPT-NEXT:    vadd.vv v8, v10, v8
+; VLOPT-NEXT:    ret
+  %1 = call <vscale x 4 x i32> @llvm.riscv.vmv.v.x.nxv4i32(<vscale x 4 x i32> poison, i32 %x, iXLen -1)
+  %2 = call <vscale x 4 x i32> @llvm.riscv.vadd.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %1, <vscale x 4 x i32> %a, iXLen %vl)
+  ret <vscale x 4 x i32> %2
+}
+
+; The vmv.v.v is optimized away if we use a vadd as the user.
+define <vscale x 1 x i8> @vmv_v_v(<vscale x 1 x i8> %a, <vscale x 1 x i8> %b, <vscale x 1 x i8> %c, <vscale x 1 x i1> %m, iXLen %vl) {
+; NOVLOPT-LABEL: vmv_v_v:
+; NOVLOPT:       # %bb.0:
+; NOVLOPT-NEXT:    vsetvli a1, zero, e8, mf8, tu, ma
+; NOVLOPT-NEXT:    vmv.v.v v8, v9
+; NOVLOPT-NEXT:    vsetvli zero, a0, e8, mf8, ta, ma
+; NOVLOPT-NEXT:    vmerge.vvm v8, v8, v10, v0
+; NOVLOPT-NEXT:    ret
+;
+; VLOPT-LABEL: vmv_v_v:
+; VLOPT:       # %bb.0:
+; VLOPT-NEXT:    vsetvli zero, a0, e8, mf8, tu, ma
+; VLOPT-NEXT:    vmv.v.v v8, v9
+; VLOPT-NEXT:    vsetvli zero, zero, e8, mf8, ta, ma
+; VLOPT-NEXT:    vmerge.vvm v8, v8, v10, v0
+; VLOPT-NEXT:    ret
+  %2 = call <vscale x 1 x i8> @llvm.riscv.vmv.v.v.nxv1i8.nxv1i8(<vscale x 1 x i8> %a, <vscale x 1 x i8> %b, iXLen -1)
+  %3 = call <vscale x 1 x i8> @llvm.riscv.vmerge.nxv1i8.nxv1i8(<vscale x 1 x i8> undef, <vscale x 1 x i8> %2, <vscale x 1 x i8> %c, <vscale x 1 x i1> %m, iXLen %vl)
+  ret <vscale x 1 x i8> %3
+}
+
+define <vscale x 4 x i32> @vwsll_vi(<vscale x 4 x i16> %a, <vscale x 4 x i32> %b, iXLen %vl) {
+; NOVLOPT-LABEL: vwsll_vi:
+; NOVLOPT:       # %bb.0:
+; NOVLOPT-NEXT:    vsetvli a1, zero, e16, m1, ta, ma
+; NOVLOPT-NEXT:    vwsll.vi v12, v8, 1
+; NOVLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vadd.vv v8, v12, v10
+; NOVLOPT-NEXT:    ret
+;
+; VLOPT-LABEL: vwsll_vi:
+; VLOPT:       # %bb.0:
+; VLOPT-NEXT:    vsetvli zero, a0, e16, m1, ta, ma
+; VLOPT-NEXT:    vwsll.vi v12, v8, 1
+; VLOPT-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
+; VLOPT-NEXT:    vadd.vv v8, v12, v10
+; VLOPT-NEXT:    ret
+  %1 = call <vscale x 4 x i32> @llvm.riscv.vwsll.nxv4i32.nxv4i16(<vscale x 4 x i32> poison, <vscale x 4 x i16> %a,iXLen 1, iXLen -1)
+  %2 = call <vscale x 4 x i32> @llvm.riscv.vadd.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %1, <vscale x 4 x i32> %b, iXLen %vl)
+  ret <vscale x 4 x i32> %2
+}
+
+define <vscale x 1 x i32> @vmand_mm(<vscale x 1 x i1> %a, <vscale x 1 x i1> %b, <vscale x 1 x i32> %c, iXLen %vl) {
+; NOVLOPT-LABEL: vmand_mm:
+; NOVLOPT:       # %bb.0:
+; NOVLOPT-NEXT:    vsetvli a1, zero, e8, mf8, ta, ma
+; NOVLOPT-NEXT:    vmand.mm v8, v0, v8
+; NOVLOPT-NEXT:    vsetvli zero, a0, e8, mf8, ta, ma
+; NOVLOPT-NEXT:    vmand.mm v0, v0, v8
+; NOVLOPT-NEXT:    vmv1r.v v8, v9
+; NOVLOPT-NEXT:    vsetvli zero, zero, e32, mf2, tu, mu
+; NOVLOPT-NEXT:    vadd.vv v8, v9, v9, v0.t
+; NOVLOPT-NEXT:    ret
+;
+; VLOPT-LABEL: vmand_mm:
+; VLOPT:       # %bb.0:
+; VLOPT-NEXT:    vsetvli zero, a0, e8, mf8, ta, ma
+; VLOPT-NEXT:    vmand.mm v8, v0, v8
+; VLOPT-NEXT:    vmand.mm v0, v0, v8
+; VLOPT-NEXT:    vmv1r.v v8, v9
+; VLOPT-NEXT:    vsetvli zero, zero, e32, mf2, tu, mu
+; VLOPT-NEXT:    vadd.vv v8, v9, v9, v0.t
+; VLOPT-NEXT:    ret
+  %1 = call <vscale x 1 x i1> @llvm.riscv.vmand.nxv1i1(<vscale x 1 x i1> %a, <vscale x 1 x i1> %b, iXLen -1)
+  %2 = call <vscale x 1 x i1> @llvm.riscv.vmand.nxv1i1(<vscale x 1 x i1> %a, <vscale x 1 x i1> %1, iXLen %vl)
+  %3 = call <vscale x 1 x i32> @llvm.riscv.vadd.mask.nxv1i32.nxv1i32(<vscale x 1 x i32> %c, <vscale x 1 x i32> %c, <vscale x 1 x i32> %c, <vscale x 1 x i1> %2, iXLen %vl, iXLen 0)
+  ret <vscale x 1 x i32> %3
+}
+
+define <vscale x 1 x i32> @vmnand_mm(<vscale x 1 x i1> %a, <vscale x 1 x i1> %b, <vscale x 1 x i32> %c, iXLen %vl) {
+; NOVLOPT-LABEL: vmnand_mm:
+; NOVLOPT:       # %bb.0:
+; NOVLOPT-NEXT:    vsetvli a1, zero, e8, mf8, ta, ma
+; NOVLOPT-NEXT:    vmnand.mm v8, v0, v8
+; NOVLOPT-NEXT:    vsetvli zero, a0, e8, mf8, ta, ma
+; NOVLOPT-NEXT:    vmand.mm v0, v0, v8
+; NOVLOPT-NEXT:    vmv1r.v v8, v9
+; NOVLOPT-NEXT:    vsetvli zero, zero, e32, mf2, tu, mu
+; NOVLOPT-NEXT:    vadd.vv v8, v9, v9, v0.t
+; NOVLOPT-NEXT:    ret
+;
+; VLOPT-LABEL: vmnand_mm:
+; VLOPT:       # %bb.0:
+; VLOPT-NEXT:    vsetvli zero, a0, e8, mf8, ta, ma
+; VLOPT-NEXT:    vmnand.mm v8, v0, v8
+; VLOPT-NEXT:    vmand.mm v0, v0, v8
+; VLOPT-NEXT:    vmv1r.v v8, v9
+; VLOPT-NEXT:    vsetvli zero, zero, e32, mf2, tu, mu
+; VLOPT-NEXT:    vadd.vv v8, v9, v9, v0.t
+; VLOPT-NEXT:    ret
+  %1 = call <vscale x 1 x i1> @llvm.riscv.vmnand.nxv1i1(<vscale x 1 x i1> %a, <vscale x 1 x i1> %b, iXLen -1)
+  %2 = call <vscale x 1 x i1> @llvm.riscv.vmand.nxv1i1(<vscale x 1 x i1> %a, <vscale x 1 x i1> %1, iXLen %vl)
+  %3 = call <vscale x 1 x i32> @llvm.riscv.vadd.mask.nxv1i32.nxv1i32(<vscale x 1 x i32> %c, <vscale x 1 x i32> %c, <vscale x 1 x i32> %c, <vscale x 1 x i1> %2, iXLen %vl, iXLen 0)
+  ret <vscale x 1 x i32> %3
+}
+
+define <vscale x 1 x i32> @vmandn_mm(<vscale x 1 x i1> %a, <vscale x 1 x i1> %b, <vscale x 1 x i32> %c, iXLen %vl) {
+; NOVLOPT-LABEL: vmandn_mm:
+; NOVLOPT:       # %bb.0:
+; NOVLOPT-NEXT:    vsetvli a1, zero, e8, mf8, ta, ma
+; NOVLOPT-NEXT:    vmandn.mm v8, v0, v8
+; NOVLOPT-NEXT:    vsetvli zero, a0, e8, mf8, ta, ma
+; NOVLOPT-NEXT:    vmand.mm v0, v0, v8
+; NOVLOPT-NEXT:    vmv1r.v v8, v9
+; NOVLOPT-NEXT:    vsetvli zero, zero, e32, mf2, tu, mu
+; NOVLOPT-NEXT:    vadd.vv v8, v9, v9, v0.t
+; NOVLOPT-NEXT:    ret
+;
+; VLOPT-LABEL: vmandn_mm:
+; VLOPT:       # %bb.0:
+; VLOPT-NEXT:    vsetvli zero, a0, e8, mf8, ta, ma
+; VLOPT-NEXT:    vmandn.mm v8, v0, v8
+; VLOPT-NEXT:    vmand.mm v0, v0, v8
+; VLOPT-NEXT:    vmv1r.v v8, v9
+; VLOPT-NEXT:    vsetvli zero, zero, e32, mf2, tu, mu
+; VLOPT-NEXT:    vadd.vv v8, v9, v9, v0.t
+; VLOPT-NEXT:    ret
+  %1 = call <vscale x 1 x i1> @llvm.riscv.vmandn.nxv1i1(<vscale x 1 x i1> %a, <vscale x 1 x i1> %b, iXLen -1)
+  %2 = call <vscale x 1 x i1> @llvm.riscv.vmand.nxv1i1(<vscale x 1 x i1> %a, <vscale x 1 x i1> %1, iXLen %vl)
+  %3 = call <vscale x 1 x i32> @llvm.riscv.vadd.mask.nxv1i32.nxv1i32(<vscale x 1 x i32> %c, <vscale x 1 x i32> %c, <vscale x 1 x i32> %c, <vscale x 1 x i1> %2, iXLen %vl, iXLen 0)
+  ret <vscale x 1 x i32> %3
+}
+
+define <vscale x 1 x i32> @vmxor_mm(<vscale x 1 x i1> %a, <vscale x 1 x i1> %b, <vscale x 1 x i32> %c, iXLen %vl) {
+; NOVLOPT-LABEL: vmxor_mm:
+; NOVLOPT:       # %bb.0:
+; NOVLOPT-NEXT:    vsetvli a1, zero, e8, mf8, ta, ma
+; NOVLOPT-NEXT:    vmxor.mm v8, v0, v8
+; NOVLOPT-NEXT:    vsetvli zero, a0, e8, mf8, ta, ma
+; NOVLOPT-NEXT:    vmand.mm v0, v0, v8
+; NOVLOPT-NEXT:    vmv1r.v v8, v9
+; NOVLOPT-NEXT:    vsetvli zero, zero, e32, mf2, tu, mu
+; NOVLOPT-NEXT:    vadd.vv v8, v9, v9, v0.t
+; NOVLOPT-NEXT:    ret
+;
+; VLOPT-LABEL: vmxor_mm:
+; VLOPT:       # %bb.0:
+; VLOPT-NEXT:    vsetvli zero, a0, e8, mf8, ta, ma
+; VLOPT-NEXT:    vmxor.mm v8, v0, v8
+; VLOPT-NEXT:    vmand.mm v0, v0, v8
+; VLOPT-NEXT:    vmv1r.v v8, v9
+; VLOPT-NEXT:    vsetvli zero, zero, e32, mf2, tu, mu
+; VLOPT-NEXT:    vadd.vv v8, v9, v9, v0.t
+; VLOPT-NEXT:    ret
+  %1 = call <vscale x 1 x i1> @llvm.riscv.vmxor.nxv1i1(<vscale x 1 x i1> %a, <vscale x 1 x i1> %b, iXLen -1)
+  %2 = call <vscale x 1 x i1> @llvm.riscv.vmand.nxv1i1(<vscale x 1 x i1> %a, <vscale x 1 x i1> %1, iXLen %vl)
+  %3 = call <vscale x 1 x i32> @llvm.riscv.vadd.mask.nxv1i32.nxv1i32(<vscale x 1 x i32> %c, <vscale x 1 x i32> %c, <vscale x 1 x i32> %c, <vscale x 1 x i1> %2, iXLen %vl, iXLen 0)
+  ret <vscale x 1 x i32> %3
+}
+
+define <vscale x 1 x i32> @vmor_mm(<vscale x 1 x i1> %a, <vscale x 1 x i1> %b, <vscale x 1 x i32> %c, iXLen %vl) {
+; NOVLOPT-LABEL: vmor_mm:
+; NOVLOPT:       # %bb.0:
+; NOVLOPT-NEXT:    vsetvli a1, zero, e8, mf8, ta, ma
+; NOVLOPT-NEXT:    vmor.mm v8, v0, v8
+; NOVLOPT-NEXT:    vsetvli zero, a0, e8, mf8, ta, ma
+; NOVLOPT-NEXT:    vmand.mm v0, v0, v8
+; NOVLOPT-NEXT:    vmv1r.v v8, v9
+; NOVLOPT-NEXT:    vsetvli zero, zero, e32, mf2, tu, mu
+; NOVLOPT-NEXT:    vadd.vv v8, v9, v9, v0.t
+; NOVLOPT-NEXT:    ret
+;
+; VLOPT-LABEL: vmor_mm:
+; VLOPT:       # %bb.0:
+; VLOPT-NEXT:    vsetvli zero, a0, e8, mf8, ta, ma
+; VLOPT-NEXT:    vmor.mm v8, v0, v8
+; VLOPT-NEXT:    vmand.mm v0, v0, v8
+; VLOPT-NEXT:    vmv1r.v v8, v9
+; VLOPT-NEXT:    vsetvli zero, zero, e32, mf2, tu, mu
+; VLOPT-NEXT:    vadd.vv v8, v9, v9, v0.t
+; VLOPT-NEXT:    ret
+  %1 = call <vscale x 1 x i1> @llvm.riscv.vmor.nxv1i1(<vscale x 1 x i1> %a, <vscale x 1 x i1> %b, iXLen -1)
+  %2 = call <vscale x 1 x i1> @llvm.riscv.vmand.nxv1i1(<vscale x 1 x i1> %a, <vscale x 1 x i1> %1, iXLen %vl)
+  %3 = call <vscale x 1 x i32> @llvm.riscv.vadd.mask.nxv1i32.nxv1i32(<vscale x 1 x i32> %c, <vscale x 1 x i32> %c, <vscale x 1 x i32> %c, <vscale x 1 x i1> %2, iXLen %vl, iXLen 0)
+  ret <vscale x 1 x i32> %3
+}
+
+
+define <vscale x 1 x i32> @vmnor_mm(<vscale x 1 x i1> %a, <vscale x 1 x i1> %b, <vscale x 1 x i32> %c, iXLen %vl) {
+; NOVLOPT-LABEL: vmnor_mm:
+; NOVLOPT:       # %bb.0:
+; NOVLOPT-NEXT:    vsetvli a1, zero, e8, mf8, ta, ma
+; NOVLOPT-NEXT:    vmnor.mm v8, v0, v8
+; NOVLOPT-NEXT:    vsetvli zero, a0, e8, mf8, ta, ma
+; NOVLOPT-NEXT:    vmand.mm v0, v0, v8
+; NOVLOPT-NEXT:    vmv1r.v v8, v9
+; NOVLOPT-NEXT:    vsetvli zero, zero, e32, mf2, tu, mu
+; NOVLOPT-NEXT:    vadd.vv v8, v9, v9, v0.t
+; NOVLOPT-NEXT:    ret
+;
+; VLOPT-LABEL: vmnor_mm:
+; VLOPT:       # %bb.0:
+; VLOPT-NEXT:    vsetvli zero, a0, e8, mf8, ta, ma
+; VLOPT-NEXT:    vmnor.mm v8, v0, v8
+; VLOPT-NEXT:    vmand.mm v0, v0, v8
+; VLOPT-NEXT:    vmv1r.v v8, v9
+; VLOPT-NEXT:    vsetvli zero, zero, e32, mf2, tu, mu
+; VLOPT-NEXT:    vadd.vv v8, v9, v9, v0.t
+; VLOPT-NEXT:    ret
+  %1 = call <vscale x 1 x i1> @llvm.riscv.vmnor.nxv1i1(<vscale x 1 x i1> %a, <vscale x 1 x i1> %b, iXLen -1)
+  %2 = call <vscale x 1 x i1> @llvm.riscv.vmand.nxv1i1(<vscale x 1 x i1> %a, <vscale x 1 x i1> %1, iXLen %vl)
+  %3 = call <vscale x 1 x i32> @llvm.riscv.vadd.mask.nxv1i32.nxv1i32(<vscale x 1 x i32> %c, <vscale x 1 x i32> %c, <vscale x 1 x i32> %c, <vscale x 1 x i1> %2, iXLen %vl, iXLen 0)
+  ret <vscale x 1 x i32> %3
+}
+
+define <vscale x 1 x i32> @vmorn_mm(<vscale x 1 x i1> %a, <vscale x 1 x i1> %b, <vscale x 1 x i32> %c, iXLen %vl) {
+; NOVLOPT-LABEL: vmorn_mm:
+; NOVLOPT:       # %bb.0:
+; NOVLOPT-NEXT:    vsetvli a1, zero, e8, mf8, ta, ma
+; NOVLOPT-NEXT:    vmorn.mm v8, v0, v8
+; NOVLOPT-NEXT:    vsetvli zero, a0, e8, mf8, ta, ma
+; NOVLOPT-NEXT:    vmand.mm v0, v0, v8
+; NOVLOPT-NEXT:    vmv1r.v v8, v9
+; NOVLOPT-NEXT:    vsetvli zero, zero, e32, mf2, tu, mu
+; NOVLOPT-NEXT:    vadd.vv v8, v9, v9, v0.t
+; NOVLOPT-NEXT:    ret
+;
+; VLOPT-LABEL: vmorn_mm:
+; VLOPT:       # %bb.0:
+; VLOPT-NEXT:    vsetvli zero, a0, e8, mf8, ta, ma
+; VLOPT-NEXT:    vmorn.mm v8, v0, v8
+; VLOPT-NEXT:    vmand.mm v0, v0, v8
+; VLOPT-NEXT:    vmv1r.v v8, v9
+; VLOPT-NEXT:    vsetvli zero, zero, e32, mf2, tu, mu
+; VLOPT-NEXT:    vadd.vv v8, v9, v9, v0.t
+; VLOPT-NEXT:    ret
+  %1 = call <vscale x 1 x i1> @llvm.riscv.vmorn.nxv1i1(<vscale x 1 x i1> %a, <vscale x 1 x i1> %b, iXLen -1)
+  %2 = call <vscale x 1 x i1> @llvm.riscv.vmand.nxv1i1(<vscale x 1 x i1> %a, <vscale x 1 x i1> %1, iXLen %vl)
+  %3 = call <vscale x 1 x i32> @llvm.riscv.vadd.mask.nxv1i32.nxv1i32(<vscale x 1 x i32> %c, <vscale x 1 x i32> %c, <vscale x 1 x i32> %c, <vscale x 1 x i1> %2, iXLen %vl, iXLen 0)
+  ret <vscale x 1 x i32> %3
+}
+
+define <vscale x 1 x i32> @vmxnor_mm(<vscale x 1 x i1> %a, <vscale x 1 x i1> %b, <vscale x 1 x i32> %c, iXLen %vl) {
+; NOVLOPT-LABEL: vmxnor_mm:
+; NOVLOPT:       # %bb.0:
+; NOVLOPT-NEXT:    vsetvli a1, zero, e8, mf8, ta, ma
+; NOVLOPT-NEXT:    vmxnor.mm v8, v0, v8
+; NOVLOPT-NEXT:    vsetvli zero, a0, e8, mf8, ta, ma
+; NOVLOPT-NEXT:    vmand.mm v0, v0, v8
+; NOVLOPT-NEXT:    vmv1r.v v8, v9
+; NOVLOPT-NEXT:    vsetvli zero, zero, e32, mf2, tu, mu
+; NOVLOPT-NEXT:    vadd.vv v8, v9, v9, v0.t
+; NOVLOPT-NEXT:    ret
+;
+; VLOPT-LABEL: vmxnor_mm:
+; VLOPT:       # %bb.0:
+; VLOPT-NEXT:    vsetvli zero, a0, e8, mf8, ta, ma
+; VLOPT-NEXT:    vmxnor.mm v8, v0, v8
+; VLOPT-NEXT:    vmand.mm v0, v0, v8
+; VLOPT-NEXT:    vmv1r.v v8, v9
+; VLOPT-NEXT:    vsetvli zero, zero, e32, mf2, tu, mu
+; VLOPT-NEXT:    vadd.vv v8, v9, v9, v0.t
+; VLOPT-NEXT:    ret
+  %1 = call <vscale x 1 x i1> @llvm.riscv.vmxnor.nxv1i1(<vscale x 1 x i1> %a, <vscale x 1 x i1> %b, iXLen -1)
+  %2 = call <vscale x 1 x i1> @llvm.riscv.vmand.nxv1i1(<vscale x 1 x i1> %a, <vscale x 1 x i1> %1, iXLen %vl)
+  %3 = call <vscale x 1 x i32> @llvm.riscv.vadd.mask.nxv1i32.nxv1i32(<vscale x 1 x i32> %c, <vscale x 1 x i32> %c, <vscale x 1 x i32> %c, <vscale x 1 x i1> %2, iXLen %vl, iXLen 0)
+  ret <vscale x 1 x i32> %3
+}
+
+define <vscale x 1 x i32> @vmsbf_m(<vscale x 1 x i1> %a, <vscale x 1 x i32> %c, iXLen %vl) {
+; NOVLOPT-LABEL: vmsbf_m:
+; NOVLOPT:       # %bb.0:
+; NOVLOPT-NEXT:    vsetvli a1, zero, e8, mf8, ta, ma
+; NOVLOPT-NEXT:    vmsbf.m v9, v0
+; NOVLOPT-NEXT:    vsetvli zero, a0, e8, mf8, ta, ma
+; NOVLOPT-NEXT:    vmand.mm v0, v0, v9
+; NOVLOPT-NEXT:    vsetvli zero, zero, e32, mf2, tu, mu
+; NOVLOPT-NEXT:    vadd.vv v8, v8, v8, v0.t
+; NOVLOPT-NEXT:    ret
+;
+; VLOPT-LABEL: vmsbf_m:
+; VLOPT:       # %bb.0:
+; VLOPT-NEXT:    vsetvli zero, a0, e8, mf8, ta, ma
+; VLOPT-NEXT:    vmsbf.m v9, v0
+; VLOPT-NEXT:    vmand.mm v0, v0, v9
+; VLOPT-NEXT:    vsetvli zero, zero, e32, mf2, tu, mu
+; VLOPT-NEXT:    vadd.vv v8, v8, v8, v0.t
+; VLOPT-NEXT:    ret
+  %1 = call <vscale x 1 x i1> @llvm.riscv.vmsbf.nxv1i1(<vscale x 1 x i1> %a, iXLen -1)
+  %2 = call <vscale x 1 x i1> @llvm.riscv.vmand.nxv1i1(<vscale x 1 x i1> %a, <vscale x 1 x i1> %1, iXLen %vl)
+  %3 = call <vscale x 1 x i32> @llvm.riscv.vadd.mask.nxv1i32.nxv1i32(<vscale x 1 x i32> %c, <vscale x 1 x i32> %c, <vscale x 1 x i32> %c, <vscale x 1 x i1> %2, iXLen %vl, iXLen 0)
+  ret <vscale x 1 x i32> %3
+}
+
+define <vscale x 1 x i32> @vmsif_m(<vscale x 1 x i1> %a, <vscale x 1 x i32> %c, iXLen %vl) {
+; NOVLOPT-LABEL: vmsif_m:
+; NOVLOPT:       # %bb.0:
+; NOVLOPT-NEXT:    vsetvli a1, zero, e8, mf8, ta, ma
+; NOVLOPT-NEXT:    vmsif.m v9, v0
+; NOVLOPT-NEXT:    vsetvli zero, a0, e8, mf8, ta, ma
+; NOVLOPT-NEXT:    vmand.mm v0, v0, v9
+; NOVLOPT-NEXT:    vsetvli zero, zero, e32, mf2, tu, mu
+; NOVLOPT-NEXT:    vadd.vv v8, v8, v8, v0.t
+; NOVLOPT-NEXT:    ret
+;
+; VLOPT-LABEL: vmsif_m:
+; VLOPT:       # %bb.0:
+; VLOPT-NEXT:    vsetvli zero, a0, e8, mf8, ta, ma
+; VLOPT-NEXT:    vmsif.m v9, v0
+; VLOPT-NEXT:    vmand.mm v0, v0, v9
+; VLOPT-NEXT:    vsetvli zero, zero, e32, mf2, tu, mu
+; VLOPT-NEXT:    vadd.vv v8, v8, v8, v0.t
+; VLOPT-NEXT:    ret
+  %1 = call <vscale x 1 x i1> @llvm.riscv.vmsif.nxv1i1(<vscale x 1 x i1> %a, iXLen -1)
+  %2 = call <vscale x 1 x i1> @llvm.riscv.vmand.nxv1i1(<vscale x 1 x i1> %a, <vscale x 1 x i1> %1, iXLen %vl)
+  %3 = call <vscale x 1 x i32> @llvm.riscv.vadd.mask.nxv1i32.nxv1i32(<vscale x 1 x i32> %c, <vscale x 1 x i32> %c, <vscale x 1 x i32> %c, <vscale x 1 x i1> %2, iXLen %vl, iXLen 0)
+  ret <vscale x 1 x i32> %3
+}
+
+define <vscale x 1 x i32> @vmsof_m(<vscale x 1 x i1> %a, <vscale x 1 x i32> %c, iXLen %vl) {
+; NOVLOPT-LABEL: vmsof_m:
+; NOVLOPT:       # %bb.0:
+; NOVLOPT-NEXT:    vsetvli a1, zero, e8, mf8, ta, ma
+; NOVLOPT-NEXT:    vmsof.m v9, v0
+; NOVLOPT-NEXT:    vsetvli zero, a0, e8, mf8, ta, ma
+; NOVLOPT-NEXT:    vmand.mm v0, v0, v9
+; NOVLOPT-NEXT:    vsetvli zero, zero, e32, mf2, tu, mu
+; NOVLOPT-NEXT:    vadd.vv v8, v8, v8, v0.t
+; NOVLOPT-NEXT:    ret
+;
+; VLOPT-LABEL: vmsof_m:
+; VLOPT:       # %bb.0:
+; VLOPT-NEXT:    vsetvli zero, a0, e8, mf8, ta, ma
+; VLOPT-NEXT:    vmsof.m v9, v0
+; VLOPT-NEXT:    vmand.mm v0, v0, v9
+; VLOPT-NEXT:    vsetvli zero, zero, e32, mf2, tu, mu
+; VLOPT-NEXT:    vadd.vv v8, v8, v8, v0.t
+; VLOPT-NEXT:    ret
+  %1 = call <vscale x 1 x i1> @llvm.riscv.vmsof.nxv1i1(<vscale x 1 x i1> %a, iXLen -1)
+  %2 = call <vscale x 1 x i1> @llvm.riscv.vmand.nxv1i1(<vscale x 1 x i1> %a, <vscale x 1 x i1> %1, iXLen %vl)
+  %3 = call <vscale x 1 x i32> @llvm.riscv.vadd.mask.nxv1i32.nxv1i32(<vscale x 1 x i32> %c, <vscale x 1 x i32> %c, <vscale x 1 x i32> %c, <vscale x 1 x i1> %2, iXLen %vl, iXLen 0)
+  ret <vscale x 1 x i32> %3
+}
+
+define <vscale x 4 x i32> @viota_m(<vscale x 4 x i1> %a, <vscale x 4 x i32> %c, iXLen %vl) {
+; NOVLOPT-LABEL: viota_m:
+; NOVLOPT:       # %bb.0:
+; NOVLOPT-NEXT:    vsetvli a1, zero, e32, m2, ta, ma
+; NOVLOPT-NEXT:    viota.m v10, v0
+; NOVLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vadd.vv v8, v10, v8
+; NOVLOPT-NEXT:    ret
+;
+; VLOPT-LABEL: viota_m:
+; VLOPT:       # %bb.0:
+; VLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
+; VLOPT-NEXT:    viota.m v10, v0
+; VLOPT-NEXT:    vadd.vv v8, v10, v8
+; VLOPT-NEXT:    ret
+  %1 = call <vscale x 4 x i32> @llvm.riscv.viota.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i1> %a, iXLen -1)
+  %2 = call <vscale x 4 x i32> @llvm.riscv.vadd.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %1, <vscale x 4 x i32> %c, iXLen %vl)
+  ret <vscale x 4 x i32> %2
+}
+
+define <vscale x 4 x i32> @vid.v(<vscale x 4 x i32> %c, iXLen %vl) {
+; NOVLOPT-LABEL: vid.v:
+; NOVLOPT:       # %bb.0:
+; NOVLOPT-NEXT:    vsetvli a1, zero, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vid.v v10
+; NOVLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vadd.vv v8, v10, v8
+; NOVLOPT-NEXT:    ret
+;
+; VLOPT-LABEL: vid.v:
+; VLOPT:       # %bb.0:
+; VLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
+; VLOPT-NEXT:    vid.v v10
+; VLOPT-NEXT:    vadd.vv v8, v10, v8
+; VLOPT-NEXT:    ret
+  %1 = call <vscale x 4 x i32> @llvm.riscv.vid.nxv4i32(<vscale x 4 x i32> poison, iXLen -1)
+  %2 = call <vscale x 4 x i32> @llvm.riscv.vadd.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %1, <vscale x 4 x i32> %c, iXLen %vl)
+  ret <vscale x 4 x i32> %2
+}
+
+define <vscale x 4 x float> @vfadd_vv(<vscale x 4 x float> %a, <vscale x 4 x float> %b, iXLen %vl) {
+; NOVLOPT-LABEL: vfadd_vv:
+; NOVLOPT:       # %bb.0:
+; NOVLOPT-NEXT:    vsetvli a1, zero, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vfadd.vv v8, v8, v10
+; NOVLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vfadd.vv v8, v8, v10
+; NOVLOPT-NEXT:    ret
+;
+; VLOPT-LABEL: vfadd_vv:
+; VLOPT:       # %bb.0:
+; VLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
+; VLOPT-NEXT:    vfadd.vv v8, v8, v10
+; VLOPT-NEXT:    vfadd.vv v8, v8, v10
+; VLOPT-NEXT:    ret
+  %1 = call <vscale x 4 x float> @llvm.riscv.vfadd.nxv4f32.nxv4f32(<vscale x 4 x float> poison, <vscale x 4 x float> %a, <vscale x 4 x float> %b, iXLen 7, iXLen -1)
+  %2 = call <vscale x 4 x float> @llvm.riscv.vfadd.nxv4f32.nxv4f32(<vscale x 4 x float> poison, <vscale x 4 x float> %1, <vscale x 4 x float> %b, iXLen 7, iXLen %vl)
+  ret <vscale x 4 x float> %2
+}
+
+define <vscale x 4 x float> @vfadd_vf(<vscale x 4 x float> %a, float %b, iXLen %vl) {
+; NOVLOPT-LABEL: vfadd_vf:
+; NOVLOPT:       # %bb.0:
+; NOVLOPT-NEXT:    vsetvli a1, zero, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vfadd.vf v10, v8, fa0
+; NOVLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vfadd.vv v8, v10, v8
+; NOVLOPT-NEXT:    ret
+;
+; VLOPT-LABEL: vfadd_vf:
+; VLOPT:       # %bb.0:
+; VLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
+; VLOPT-NEXT:    vfadd.vf v10, v8, fa0
+; VLOPT-NEXT:    vfadd.vv v8, v10, v8
+; VLOPT-NEXT:    ret
+  %1 = call <vscale x 4 x float> @llvm.riscv.vfadd.nxv4f32.nxv4f32(<vscale x 4 x float> poison, <vscale x 4 x float> %a, float %b, iXLen 7, iXLen -1)
+  %2 = call <vscale x 4 x float> @llvm.riscv.vfadd.nxv4f32.nxv4f32(<vscale x 4 x float> poison, <vscale x 4 x float> %1, <vscale x 4 x float> %a, iXLen 7, iXLen %vl)
+  ret <vscale x 4 x float> %2
+}
+
+define <vscale x 4 x float> @vfsub_vv(<vscale x 4 x float> %a, <vscale x 4 x float> %b, iXLen %vl) {
+; NOVLOPT-LABEL: vfsub_vv:
+; NOVLOPT:       # %bb.0:
+; NOVLOPT-NEXT:    vsetvli a1, zero, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vfsub.vv v8, v8, v10
+; NOVLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vfadd.vv v8, v8, v10
+; NOVLOPT-NEXT:    ret
+;
+; VLOPT-LABEL: vfsub_vv:
+; VLOPT:       # %bb.0:
+; VLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
+; VLOPT-NEXT:    vfsub.vv v8, v8, v10
+; VLOPT-NEXT:    vfadd.vv v8, v8, v10
+; VLOPT-NEXT:    ret
+  %1 = call <vscale x 4 x float> @llvm.riscv.vfsub.nxv4f32.nxv4f32(<vscale x 4 x float> poison, <vscale x 4 x float> %a, <vscale x 4 x float> %b, iXLen 7, iXLen -1)
+  %2 = call <vscale x 4 x float> @llvm.riscv.vfadd.nxv4f32.nxv4f32(<vscale x 4 x float> poison, <vscale x 4 x float> %1, <vscale x 4 x float> %b, iXLen 7, iXLen %vl)
+  ret <vscale x 4 x float> %2
+}
+
+define <vscale x 4 x float> @vfsub_vf(<vscale x 4 x float> %a, float %b, iXLen %vl) {
+; NOVLOPT-LABEL: vfsub_vf:
+; NOVLOPT:       # %bb.0:
+; NOVLOPT-NEXT:    vsetvli a1, zero, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vfsub.vf v10, v8, fa0
+; NOVLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vfadd.vv v8, v10, v8
+; NOVLOPT-NEXT:    ret
+;
+; VLOPT-LABEL: vfsub_vf:
+; VLOPT:       # %bb.0:
+; VLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
+; VLOPT-NEXT:    vfsub.vf v10, v8, fa0
+; VLOPT-NEXT:    vfadd.vv v8, v10, v8
+; VLOPT-NEXT:    ret
+  %1 = call <vscale x 4 x float> @llvm.riscv.vfsub.nxv4f32.nxv4f32(<vscale x 4 x float> poison, <vscale x 4 x float> %a, float %b, iXLen 7, iXLen -1)
+  %2 = call <vscale x 4 x float> @llvm.riscv.vfadd.nxv4f32.nxv4f32(<vscale x 4 x float> poison, <vscale x 4 x float> %1, <vscale x 4 x float> %a, iXLen 7, iXLen %vl)
+  ret <vscale x 4 x float> %2
+}
+
+define <vscale x 4 x float> @vfrsub_vf(<vscale x 4 x float> %a, float %b, iXLen %vl) {
+; NOVLOPT-LABEL: vfrsub_vf:
+; NOVLOPT:       # %bb.0:
+; NOVLOPT-NEXT:    vsetvli a1, zero, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vfrsub.vf v10, v8, fa0
+; NOVLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vfadd.vv v8, v10, v8
+; NOVLOPT-NEXT:    ret
+;
+; VLOPT-LABEL: vfrsub_vf:
+; VLOPT:       # %bb.0:
+; VLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
+; VLOPT-NEXT:    vfrsub.vf v10, v8, fa0
+; VLOPT-NEXT:    vfadd.vv v8, v10, v8
+; VLOPT-NEXT:    ret
+  %1 = call <vscale x 4 x float> @llvm.riscv.vfrsub.nxv4f32.nxv4f32(<vscale x 4 x float> poison, <vscale x 4 x float> %a, float %b, iXLen 7, iXLen -1)
+  %2 = call <vscale x 4 x float> @llvm.riscv.vfadd.nxv4f32.nxv4f32(<vscale x 4 x float> poison, <vscale x 4 x float> %1, <vscale x 4 x float> %a, iXLen 7, iXLen %vl)
+  ret <vscale x 4 x float> %2
+}
+
+define <vscale x 4 x double> @vfwadd_vv(<vscale x 4 x float> %a, <vscale x 4 x float> %b, iXLen %vl) {
+; NOVLOPT-LABEL: vfwadd_vv:
+; NOVLOPT:       # %bb.0:
+; NOVLOPT-NEXT:    vsetvli a1, zero, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vfwadd.vv v12, v8, v10
+; NOVLOPT-NEXT:    vsetvli zero, a0, e64, m4, ta, ma
+; NOVLOPT-NEXT:    vfadd.vv v8, v12, v12
+; NOVLOPT-NEXT:    ret
+;
+; VLOPT-LABEL: vfwadd_vv:
+; VLOPT:       # %bb.0:
+; VLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
+; VLOPT-NEXT:    vfwadd.vv v12, v8, v10
+; VLOPT-NEXT:    vsetvli zero, zero, e64, m4, ta, ma
+; VLOPT-NEXT:    vfadd.vv v8, v12, v12
+; VLOPT-NEXT:    ret
+  %1 = call <vscale x 4 x double> @llvm.riscv.vfwadd.nxv4f64.nxv4f32.nxv4f32(<vscale x 4 x double> poison, <vscale x 4 x float> %a, <vscale x 4 x float> %b, iXLen 7, iXLen -1)
+  %2 = call <vscale x 4 x double> @llvm.riscv.vfadd.nxv4f64.nxv4f64(<vscale x 4 x double> poison, <vscale x 4 x double> %1, <vscale x 4 x double> %1, iXLen 7, iXLen %vl)
+  ret <vscale x 4 x double> %2
+}
+
+define <vscale x 4 x double> @vfwadd_vf(<vscale x 4 x float> %a, float %b, iXLen %vl) {
+; NOVLOPT-LABEL: vfwadd_vf:
+; NOVLOPT:       # %bb.0:
+; NOVLOPT-NEXT:    vsetvli a1, zero, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vfwadd.vf v12, v8, fa0
+; NOVLOPT-NEXT:    vsetvli zero, a0, e64, m4, ta, ma
+; NOVLOPT-NEXT:    vfadd.vv v8, v12, v12
+; NOVLOPT-NEXT:    ret
+;
+; VLOPT-LABEL: vfwadd_vf:
+; VLOPT:       # %bb.0:
+; VLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
+; VLOPT-NEXT:    vfwadd.vf v12, v8, fa0
+; VLOPT-NEXT:    vsetvli zero, zero, e64, m4, ta, ma
+; VLOPT-NEXT:    vfadd.vv v8, v12, v12
+; VLOPT-NEXT:    ret
+  %1 = call <vscale x 4 x double> @llvm.riscv.vfwadd.nxv4f64.nxv4f32.f32(<vscale x 4 x double> poison, <vscale x 4 x float> %a, float %b, iXLen 7, iXLen -1)
+  %2 = call <vscale x 4 x double> @llvm.riscv.vfadd.nxv4f64.nxv4f64(<vscale x 4 x double> poison, <vscale x 4 x double> %1, <vscale x 4 x double> %1, iXLen 7, iXLen %vl)
+  ret <vscale x 4 x double> %2
+}
+
+define <vscale x 4 x double> @vfwsub_vv(<vscale x 4 x float> %a, <vscale x 4 x float> %b, iXLen %vl) {
+; NOVLOPT-LABEL: vfwsub_vv:
+; NOVLOPT:       # %bb.0:
+; NOVLOPT-NEXT:    vsetvli a1, zero, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vfwsub.vv v12, v8, v10
+; NOVLOPT-NEXT:    vsetvli zero, a0, e64, m4, ta, ma
+; NOVLOPT-NEXT:    vfadd.vv v8, v12, v12
+; NOVLOPT-NEXT:    ret
+;
+; VLOPT-LABEL: vfwsub_vv:
+; VLOPT:       # %bb.0:
+; VLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
+; VLOPT-NEXT:    vfwsub.vv v12, v8, v10
+; VLOPT-NEXT:    vsetvli zero, zero, e64, m4, ta, ma
+; VLOPT-NEXT:    vfadd.vv v8, v12, v12
+; VLOPT-NEXT:    ret
+  %1 = call <vscale x 4 x double> @llvm.riscv.vfwsub.nxv4f64.nxv4f32.nxv4f32(<vscale x 4 x double> poison, <vscale x 4 x float> %a, <vscale x 4 x float> %b, iXLen 7, iXLen -1)
+  %2 = call <vscale x 4 x double> @llvm.riscv.vfadd.nxv4f64.nxv4f64(<vscale x 4 x double> poison, <vscale x 4 x double> %1, <vscale x 4 x double> %1, iXLen 7, iXLen %vl)
+  ret <vscale x 4 x double> %2
+}
+
+define <vscale x 4 x double> @vfwsub_vf(<vscale x 4 x float> %a, float %b, iXLen %vl) {
+; NOVLOPT-LABEL: vfwsub_vf:
+; NOVLOPT:       # %bb.0:
+; NOVLOPT-NEXT:    vsetvli a1, zero, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vfwsub.vf v12, v8, fa0
+; NOVLOPT-NEXT:    vsetvli zero, a0, e64, m4, ta, ma
+; NOVLOPT-NEXT:    vfadd.vv v8, v12, v12
+; NOVLOPT-NEXT:    ret
+;
+; VLOPT-LABEL: vfwsub_vf:
+; VLOPT:       # %bb.0:
+; VLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
+; VLOPT-NEXT:    vfwsub.vf v12, v8, fa0
+; VLOPT-NEXT:    vsetvli zero, zero, e64, m4, ta, ma
+; VLOPT-NEXT:    vfadd.vv v8, v12, v12
+; VLOPT-NEXT:    ret
+  %1 = call <vscale x 4 x double> @llvm.riscv.vfwsub.nxv4f64.nxv4f32.f32(<vscale x 4 x double> poison, <vscale x 4 x float> %a, float %b, iXLen 7, iXLen -1)
+  %2 = call <vscale x 4 x double> @llvm.riscv.vfadd.nxv4f64.nxv4f64(<vscale x 4 x double> poison, <vscale x 4 x double> %1, <vscale x 4 x double> %1, iXLen 7, iXLen %vl)
+  ret <vscale x 4 x double> %2
+}
+
+define <vscale x 4 x double> @vfwadd_wv(<vscale x 4 x double> %a, <vscale x 4 x float> %b, iXLen %vl) {
+; NOVLOPT-LABEL: vfwadd_wv:
+; NOVLOPT:       # %bb.0:
+; NOVLOPT-NEXT:    vsetvli a1, zero, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vfwadd.wv v8, v8, v12
+; NOVLOPT-NEXT:    vsetvli zero, a0, e64, m4, ta, ma
+; NOVLOPT-NEXT:    vfadd.vv v8, v8, v8
+; NOVLOPT-NEXT:    ret
+;
+; VLOPT-LABEL: vfwadd_wv:
+; VLOPT:       # %bb.0:
+; VLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
+; VLOPT-NEXT:    vfwadd.wv v8, v8, v12
+; VLOPT-NEXT:    vsetvli zero, zero, e64, m4, ta, ma
+; VLOPT-NEXT:    vfadd.vv v8, v8, v8
+; VLOPT-NEXT:    ret
+  %1 = call <vscale x 4 x double> @llvm.riscv.vfwadd.w.nxv4f64.nxv4f32.nxv4f32(<vscale x 4 x double> poison, <vscale x 4 x double> %a, <vscale x 4 x float> %b, iXLen 7, iXLen -1)
+  %2 = call <vscale x 4 x double> @llvm.riscv.vfadd.nxv4f64.nxv4f64(<vscale x 4 x double> poison, <vscale x 4 x double> %1, <vscale x 4 x double> %1, iXLen 7, iXLen %vl)
+  ret <vscale x 4 x double> %2
+}
+
+define <vscale x 4 x double> @vfwadd_wf(<vscale x 4 x double> %a, float %b, iXLen %vl) {
+; NOVLOPT-LABEL: vfwadd_wf:
+; NOVLOPT:       # %bb.0:
+; NOVLOPT-NEXT:    vsetvli a1, zero, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vfwadd.wf v8, v8, fa0
+; NOVLOPT-NEXT:    vsetvli zero, a0, e64, m4, ta, ma
+; NOVLOPT-NEXT:    vfadd.vv v8, v8, v8
+; NOVLOPT-NEXT:    ret
+;
+; VLOPT-LABEL: vfwadd_wf:
+; VLOPT:       # %bb.0:
+; VLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
+; VLOPT-NEXT:    vfwadd.wf v8, v8, fa0
+; VLOPT-NEXT:    vsetvli zero, zero, e64, m4, ta, ma
+; VLOPT-NEXT:    vfadd.vv v8, v8, v8
+; VLOPT-NEXT:    ret
+  %1 = call <vscale x 4 x double> @llvm.riscv.vfwadd.w.nxv4f64.nxv4f32.f32(<vscale x 4 x double> poison, <vscale x 4 x double> %a, float %b, iXLen 7, iXLen -1)
+  %2 = call <vscale x 4 x double> @llvm.riscv.vfadd.nxv4f64.nxv4f64(<vscale x 4 x double> poison, <vscale x 4 x double> %1, <vscale x 4 x double> %1, iXLen 7, iXLen %vl)
+  ret <vscale x 4 x double> %2
+}
+
+define <vscale x 4 x double> @vfwsub_wv(<vscale x 4 x double> %a, <vscale x 4 x float> %b, iXLen %vl) {
+; NOVLOPT-LABEL: vfwsub_wv:
+; NOVLOPT:       # %bb.0:
+; NOVLOPT-NEXT:    vsetvli a1, zero, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vfwsub.wv v8, v8, v12
+; NOVLOPT-NEXT:    vsetvli zero, a0, e64, m4, ta, ma
+; NOVLOPT-NEXT:    vfadd.vv v8, v8, v8
+; NOVLOPT-NEXT:    ret
+;
+; VLOPT-LABEL: vfwsub_wv:
+; VLOPT:       # %bb.0:
+; VLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
+; VLOPT-NEXT:    vfwsub.wv v8, v8, v12
+; VLOPT-NEXT:    vsetvli zero, zero, e64, m4, ta, ma
+; VLOPT-NEXT:    vfadd.vv v8, v8, v8
+; VLOPT-NEXT:    ret
+  %1 = call <vscale x 4 x double> @llvm.riscv.vfwsub.w.nxv4f64.nxv4f32.nxv4f32(<vscale x 4 x double> poison, <vscale x 4 x double> %a, <vscale x 4 x float> %b, iXLen 7, iXLen -1)
+  %2 = call <vscale x 4 x double> @llvm.riscv.vfadd.nxv4f64.nxv4f64(<vscale x 4 x double> poison, <vscale x 4 x double> %1, <vscale x 4 x double> %1, iXLen 7, iXLen %vl)
+  ret <vscale x 4 x double> %2
+}
+
+define <vscale x 4 x double> @vfwsub_wf(<vscale x 4 x double> %a, float %b, iXLen %vl) {
+; NOVLOPT-LABEL: vfwsub_wf:
+; NOVLOPT:       # %bb.0:
+; NOVLOPT-NEXT:    vsetvli a1, zero, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vfwsub.wf v8, v8, fa0
+; NOVLOPT-NEXT:    vsetvli zero, a0, e64, m4, ta, ma
+; NOVLOPT-NEXT:    vfadd.vv v8, v8, v8
+; NOVLOPT-NEXT:    ret
+;
+; VLOPT-LABEL: vfwsub_wf:
+; VLOPT:       # %bb.0:
+; VLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
+; VLOPT-NEXT:    vfwsub.wf v8, v8, fa0
+; VLOPT-NEXT:    vsetvli zero, zero, e64, m4, ta, ma
+; VLOPT-NEXT:    vfadd.vv v8, v8, v8
+; VLOPT-NEXT:    ret
+  %1 = call <vscale x 4 x double> @llvm.riscv.vfwsub.w.nxv4f64.nxv4f32.f32(<vscale x 4 x double> poison, <vscale x 4 x double> %a, float %b, iXLen 7, iXLen -1)
+  %2 = call <vscale x 4 x double> @llvm.riscv.vfadd.nxv4f64.nxv4f64(<vscale x 4 x double> poison, <vscale x 4 x double> %1, <vscale x 4 x double> %1, iXLen 7, iXLen %vl)
+  ret <vscale x 4 x double> %2
+}
+
+define <vscale x 4 x float> @vfmul_vv(<vscale x 4 x float> %a, <vscale x 4 x float> %b, iXLen %vl) {
+; NOVLOPT-LABEL: vfmul_vv:
+; NOVLOPT:       # %bb.0:
+; NOVLOPT-NEXT:    vsetvli a1, zero, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vfmul.vv v8, v8, v10
+; NOVLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vfadd.vv v8, v8, v10
+; NOVLOPT-NEXT:    ret
+;
+; VLOPT-LABEL: vfmul_vv:
+; VLOPT:       # %bb.0:
+; VLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
+; VLOPT-NEXT:    vfmul.vv v8, v8, v10
+; VLOPT-NEXT:    vfadd.vv v8, v8, v10
+; VLOPT-NEXT:    ret
+  %1 = call <vscale x 4 x float> @llvm.riscv.vfmul.nxv4f32.nxv4f32(<vscale x 4 x float> poison, <vscale x 4 x float> %a, <vscale x 4 x float> %b, iXLen 7, iXLen -1)
+  %2 = call <vscale x 4 x float> @llvm.riscv.vfadd.nxv4f32.nxv4f32(<vscale x 4 x float> poison, <vscale x 4 x float> %1, <vscale x 4 x float> %b, iXLen 7, iXLen %vl)
+  ret <vscale x 4 x float> %2
+}
+
+define <vscale x 4 x float> @vfmul_vf(<vscale x 4 x float> %a, float %b, iXLen %vl) {
+; NOVLOPT-LABEL: vfmul_vf:
+; NOVLOPT:       # %bb.0:
+; NOVLOPT-NEXT:    vsetvli a1, zero, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vfmul.vf v10, v8, fa0
+; NOVLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vfadd.vv v8, v10, v8
+; NOVLOPT-NEXT:    ret
+;
+; VLOPT-LABEL: vfmul_vf:
+; VLOPT:       # %bb.0:
+; VLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
+; VLOPT-NEXT:    vfmul.vf v10, v8, fa0
+; VLOPT-NEXT:    vfadd.vv v8, v10, v8
+; VLOPT-NEXT:    ret
+  %1 = call <vscale x 4 x float> @llvm.riscv.vfmul.nxv4f32.nxv4f32(<vscale x 4 x float> poison, <vscale x 4 x float> %a, float %b, iXLen 7, iXLen -1)
+  %2 = call <vscale x 4 x float> @llvm.riscv.vfadd.nxv4f32.nxv4f32(<vscale x 4 x float> poison, <vscale x 4 x float> %1, <vscale x 4 x float> %a, iXLen 7, iXLen %vl)
+  ret <vscale x 4 x float> %2
+}
+
+define <vscale x 4 x float> @vfdiv_vv(<vscale x 4 x float> %a, <vscale x 4 x float> %b, iXLen %vl) {
+; NOVLOPT-LABEL: vfdiv_vv:
+; NOVLOPT:       # %bb.0:
+; NOVLOPT-NEXT:    vsetvli a1, zero, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vfdiv.vv v8, v8, v10
+; NOVLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vfadd.vv v8, v8, v10
+; NOVLOPT-NEXT:    ret
+;
+; VLOPT-LABEL: vfdiv_vv:
+; VLOPT:       # %bb.0:
+; VLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
+; VLOPT-NEXT:    vfdiv.vv v8, v8, v10
+; VLOPT-NEXT:    vfadd.vv v8, v8, v10
+; VLOPT-NEXT:    ret
+  %1 = call <vscale x 4 x float> @llvm.riscv.vfdiv.nxv4f32.nxv4f32(<vscale x 4 x float> poison, <vscale x 4 x float> %a, <vscale x 4 x float> %b, iXLen 7, iXLen -1)
+  %2 = call <vscale x 4 x float> @llvm.riscv.vfadd.nxv4f32.nxv4f32(<vscale x 4 x float> poison, <vscale x 4 x float> %1, <vscale x 4 x float> %b, iXLen 7, iXLen %vl)
+  ret <vscale x 4 x float> %2
+}
+
+define <vscale x 4 x float> @vfdiv_vf(<vscale x 4 x float> %a, float %b, iXLen %vl) {
+; NOVLOPT-LABEL: vfdiv_vf:
+; NOVLOPT:       # %bb.0:
+; NOVLOPT-NEXT:    vsetvli a1, zero, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vfdiv.vf v10, v8, fa0
+; NOVLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vfadd.vv v8, v10, v8
+; NOVLOPT-NEXT:    ret
+;
+; VLOPT-LABEL: vfdiv_vf:
+; VLOPT:       # %bb.0:
+; VLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
+; VLOPT-NEXT:    vfdiv.vf v10, v8, fa0
+; VLOPT-NEXT:    vfadd.vv v8, v10, v8
+; VLOPT-NEXT:    ret
+  %1 = call <vscale x 4 x float> @llvm.riscv.vfdiv.nxv4f32.nxv4f32(<vscale x 4 x float> poison, <vscale x 4 x float> %a, float %b, iXLen 7, iXLen -1)
+  %2 = call <vscale x 4 x float> @llvm.riscv.vfadd.nxv4f32.nxv4f32(<vscale x 4 x float> poison, <vscale x 4 x float> %1, <vscale x 4 x float> %a, iXLen 7, iXLen %vl)
+  ret <vscale x 4 x float> %2
+}
+
+define <vscale x 4 x float> @vfrdiv_vf(<vscale x 4 x float> %a, float %b, iXLen %vl) {
+; NOVLOPT-LABEL: vfrdiv_vf:
+; NOVLOPT:       # %bb.0:
+; NOVLOPT-NEXT:    vsetvli a1, zero, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vfrdiv.vf v10, v8, fa0
+; NOVLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vfadd.vv v8, v10, v8
+; NOVLOPT-NEXT:    ret
+;
+; VLOPT-LABEL: vfrdiv_vf:
+; VLOPT:       # %bb.0:
+; VLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
+; VLOPT-NEXT:    vfrdiv.vf v10, v8, fa0
+; VLOPT-NEXT:    vfadd.vv v8, v10, v8
+; VLOPT-NEXT:    ret
+  %1 = call <vscale x 4 x float> @llvm.riscv.vfrdiv.nxv4f32.nxv4f32(<vscale x 4 x float> poison, <vscale x 4 x float> %a, float %b, iXLen 7, iXLen -1)
+  %2 = call <vscale x 4 x float> @llvm.riscv.vfadd.nxv4f32.nxv4f32(<vscale x 4 x float> poison, <vscale x 4 x float> %1, <vscale x 4 x float> %a, iXLen 7, iXLen %vl)
+  ret <vscale x 4 x float> %2
+}
+
+define <vscale x 4 x double> @vfwmul_vv(<vscale x 4 x float> %a, <vscale x 4 x float> %b, iXLen %vl) {
+; NOVLOPT-LABEL: vfwmul_vv:
+; NOVLOPT:       # %bb.0:
+; NOVLOPT-NEXT:    vsetvli a1, zero, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vfwmul.vv v12, v8, v10
+; NOVLOPT-NEXT:    vsetvli zero, a0, e64, m4, ta, ma
+; NOVLOPT-NEXT:    vfadd.vv v8, v12, v12
+; NOVLOPT-NEXT:    ret
+;
+; VLOPT-LABEL: vfwmul_vv:
+; VLOPT:       # %bb.0:
+; VLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
+; VLOPT-NEXT:    vfwmul.vv v12, v8, v10
+; VLOPT-NEXT:    vsetvli zero, zero, e64, m4, ta, ma
+; VLOPT-NEXT:    vfadd.vv v8, v12, v12
+; VLOPT-NEXT:    ret
+  %1 = call <vscale x 4 x double> @llvm.riscv.vfwmul.nxv4f64.nxv4f32.nxv4f32(<vscale x 4 x double> poison, <vscale x 4 x float> %a, <vscale x 4 x float> %b, iXLen 7, iXLen -1)
+  %2 = call <vscale x 4 x double> @llvm.riscv.vfadd.nxv4f64.nxv4f64(<vscale x 4 x double> poison, <vscale x 4 x double> %1, <vscale x 4 x double> %1, iXLen 7, iXLen %vl)
+  ret <vscale x 4 x double> %2
+}
+
+define <vscale x 4 x double> @vfwmul_vf(<vscale x 4 x float> %a, float %b, iXLen %vl) {
+; NOVLOPT-LABEL: vfwmul_vf:
+; NOVLOPT:       # %bb.0:
+; NOVLOPT-NEXT:    vsetvli a1, zero, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vfwmul.vf v12, v8, fa0
+; NOVLOPT-NEXT:    vsetvli zero, a0, e64, m4, ta, ma
+; NOVLOPT-NEXT:    vfadd.vv v8, v12, v12
+; NOVLOPT-NEXT:    ret
+;
+; VLOPT-LABEL: vfwmul_vf:
+; VLOPT:       # %bb.0:
+; VLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
+; VLOPT-NEXT:    vfwmul.vf v12, v8, fa0
+; VLOPT-NEXT:    vsetvli zero, zero, e64, m4, ta, ma
+; VLOPT-NEXT:    vfadd.vv v8, v12, v12
+; VLOPT-NEXT:    ret
+  %1 = call <vscale x 4 x double> @llvm.riscv.vfwmul.nxv4f64.nxv4f32.f32(<vscale x 4 x double> poison, <vscale x 4 x float> %a, float %b, iXLen 7, iXLen -1)
+  %2 = call <vscale x 4 x double> @llvm.riscv.vfadd.nxv4f64.nxv4f64(<vscale x 4 x double> poison, <vscale x 4 x double> %1, <vscale x 4 x double> %1, iXLen 7, iXLen %vl)
+  ret <vscale x 4 x double> %2
+}
+
+define <vscale x 4 x i1> @vmfeq_vf(<vscale x 4 x float> %a, <vscale x 4 x i1> %b, float%c, iXLen %vl) {
+; NOVLOPT-LABEL: vmfeq_vf:
+; NOVLOPT:       # %bb.0:
+; NOVLOPT-NEXT:    vsetvli a1, zero, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vmfeq.vf v10, v8, fa0
+; NOVLOPT-NEXT:    vsetvli zero, a0, e8, mf2, ta, ma
+; NOVLOPT-NEXT:    vmand.mm v0, v10, v0
+; NOVLOPT-NEXT:    ret
+;
+; VLOPT-LABEL: vmfeq_vf:
+; VLOPT:       # %bb.0:
+; VLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
+; VLOPT-NEXT:    vmfeq.vf v10, v8, fa0
+; VLOPT-NEXT:    vmand.mm v0, v10, v0
+; VLOPT-NEXT:    ret
+  %1 = call <vscale x 4 x i1> @llvm.riscv.vmfeq.nxv4f32.f32(<vscale x 4 x float> %a, float %c, iXLen -1)
+  %2 = call <vscale x 4 x i1> @llvm.riscv.vmand.nxv4i1(<vscale x 4 x i1> %1, <vscale x 4 x i1> %b, iXLen %vl)
+  ret <vscale x 4 x i1> %2
+}
+
+define <vscale x 4 x i1> @vmfeq_vv(<vscale x 4 x float> %a, <vscale x 4 x i1> %b, <vscale x 4 x float> %c, iXLen %vl) {
+; NOVLOPT-LABEL: vmfeq_vv:
+; NOVLOPT:       # %bb.0:
+; NOVLOPT-NEXT:    vsetvli a1, zero, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vmfeq.vv v12, v8, v10
+; NOVLOPT-NEXT:    vsetvli zero, a0, e8, mf2, ta, ma
+; NOVLOPT-NEXT:    vmand.mm v0, v12, v0
+; NOVLOPT-NEXT:    ret
+;
+; VLOPT-LABEL: vmfeq_vv:
+; VLOPT:       # %bb.0:
+; VLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
+; VLOPT-NEXT:    vmfeq.vv v12, v8, v10
+; VLOPT-NEXT:    vmand.mm v0, v12, v0
+; VLOPT-NEXT:    ret
+  %1 = call <vscale x 4 x i1> @llvm.riscv.vmfeq.nxv4f32.nxv4f32(<vscale x 4 x float> %a, <vscale x 4 x float> %c, iXLen -1)
+  %2 = call <vscale x 4 x i1> @llvm.riscv.vmand.nxv4i1(<vscale x 4 x i1> %1, <vscale x 4 x i1> %b, iXLen %vl)
+  ret <vscale x 4 x i1> %2
+}
+
+define <vscale x 4 x i1> @vmfne_vf(<vscale x 4 x float> %a, <vscale x 4 x i1> %b, float%c, iXLen %vl) {
+; NOVLOPT-LABEL: vmfne_vf:
+; NOVLOPT:       # %bb.0:
+; NOVLOPT-NEXT:    vsetvli a1, zero, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vmfne.vf v10, v8, fa0
+; NOVLOPT-NEXT:    vsetvli zero, a0, e8, mf2, ta, ma
+; NOVLOPT-NEXT:    vmand.mm v0, v10, v0
+; NOVLOPT-NEXT:    ret
+;
+; VLOPT-LABEL: vmfne_vf:
+; VLOPT:       # %bb.0:
+; VLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
+; VLOPT-NEXT:    vmfne.vf v10, v8, fa0
+; VLOPT-NEXT:    vmand.mm v0, v10, v0
+; VLOPT-NEXT:    ret
+  %1 = call <vscale x 4 x i1> @llvm.riscv.vmfne.nxv4f32.f32(<vscale x 4 x float> %a, float %c, iXLen -1)
+  %2 = call <vscale x 4 x i1> @llvm.riscv.vmand.nxv4i1(<vscale x 4 x i1> %1, <vscale x 4 x i1> %b, iXLen %vl)
+  ret <vscale x 4 x i1> %2
+}
+
+define <vscale x 4 x i1> @vmfne_vv(<vscale x 4 x float> %a, <vscale x 4 x i1> %b, <vscale x 4 x float> %c, iXLen %vl) {
+; NOVLOPT-LABEL: vmfne_vv:
+; NOVLOPT:       # %bb.0:
+; NOVLOPT-NEXT:    vsetvli a1, zero, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vmfne.vv v12, v8, v10
+; NOVLOPT-NEXT:    vsetvli zero, a0, e8, mf2, ta, ma
+; NOVLOPT-NEXT:    vmand.mm v0, v12, v0
+; NOVLOPT-NEXT:    ret
+;
+; VLOPT-LABEL: vmfne_vv:
 ; VLOPT:       # %bb.0:
 ; VLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
-; VLOPT-NEXT:    vfwsub.vv v12, v8, v10
-; VLOPT-NEXT:    vsetvli zero, zero, e64, m4, ta, ma
-; VLOPT-NEXT:    vfadd.vv v8, v12, v12
+; VLOPT-NEXT:    vmfne.vv v12, v8, v10
+; VLOPT-NEXT:    vmand.mm v0, v12, v0
 ; VLOPT-NEXT:    ret
-  %1 = call <vscale x 4 x double> @llvm.riscv.vfwsub.nxv4f64.nxv4f32.nxv4f32(<vscale x 4 x double> poison, <vscale x 4 x float> %a, <vscale x 4 x float> %b, iXLen 7, iXLen -1)
-  %2 = call <vscale x 4 x double> @llvm.riscv.vfadd.nxv4f64.nxv4f64(<vscale x 4 x double> poison, <vscale x 4 x double> %1, <vscale x 4 x double> %1, iXLen 7, iXLen %vl)
-  ret <vscale x 4 x double> %2
+  %1 = call <vscale x 4 x i1> @llvm.riscv.vmfne.nxv4f32.nxv4f32(<vscale x 4 x float> %a, <vscale x 4 x float> %c, iXLen -1)
+  %2 = call <vscale x 4 x i1> @llvm.riscv.vmand.nxv4i1(<vscale x 4 x i1> %1, <vscale x 4 x i1> %b, iXLen %vl)
+  ret <vscale x 4 x i1> %2
 }
 
-define <vscale x 4 x double> @vfwsub_vf(<vscale x 4 x float> %a, float %b, iXLen %vl) {
-; NOVLOPT-LABEL: vfwsub_vf:
+define <vscale x 4 x i1> @vmflt_vf(<vscale x 4 x float> %a, <vscale x 4 x i1> %b, float%c, iXLen %vl) {
+; NOVLOPT-LABEL: vmflt_vf:
 ; NOVLOPT:       # %bb.0:
 ; NOVLOPT-NEXT:    vsetvli a1, zero, e32, m2, ta, ma
-; NOVLOPT-NEXT:    vfwsub.vf v12, v8, fa0
-; NOVLOPT-NEXT:    vsetvli zero, a0, e64, m4, ta, ma
-; NOVLOPT-NEXT:    vfadd.vv v8, v12, v12
+; NOVLOPT-NEXT:    vmflt.vf v10, v8, fa0
+; NOVLOPT-NEXT:    vsetvli zero, a0, e8, mf2, ta, ma
+; NOVLOPT-NEXT:    vmand.mm v0, v10, v0
 ; NOVLOPT-NEXT:    ret
 ;
-; VLOPT-LABEL: vfwsub_vf:
+; VLOPT-LABEL: vmflt_vf:
 ; VLOPT:       # %bb.0:
 ; VLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
-; VLOPT-NEXT:    vfwsub.vf v12, v8, fa0
-; VLOPT-NEXT:    vsetvli zero, zero, e64, m4, ta, ma
-; VLOPT-NEXT:    vfadd.vv v8, v12, v12
+; VLOPT-NEXT:    vmflt.vf v10, v8, fa0
+; VLOPT-NEXT:    vmand.mm v0, v10, v0
 ; VLOPT-NEXT:    ret
-  %1 = call <vscale x 4 x double> @llvm.riscv.vfwsub.nxv4f64.nxv4f32.f32(<vscale x 4 x double> poison, <vscale x 4 x float> %a, float %b, iXLen 7, iXLen -1)
-  %2 = call <vscale x 4 x double> @llvm.riscv.vfadd.nxv4f64.nxv4f64(<vscale x 4 x double> poison, <vscale x 4 x double> %1, <vscale x 4 x double> %1, iXLen 7, iXLen %vl)
-  ret <vscale x 4 x double> %2
+  %1 = call <vscale x 4 x i1> @llvm.riscv.vmflt.nxv4f32.f32(<vscale x 4 x float> %a, float %c, iXLen -1)
+  %2 = call <vscale x 4 x i1> @llvm.riscv.vmand.nxv4i1(<vscale x 4 x i1> %1, <vscale x 4 x i1> %b, iXLen %vl)
+  ret <vscale x 4 x i1> %2
 }
 
-define <vscale x 4 x double> @vfwadd_wv(<vscale x 4 x double> %a, <vscale x 4 x float> %b, iXLen %vl) {
-; NOVLOPT-LABEL: vfwadd_wv:
+define <vscale x 4 x i1> @vmflt_vv(<vscale x 4 x float> %a, <vscale x 4 x i1> %b, <vscale x 4 x float> %c, iXLen %vl) {
+; NOVLOPT-LABEL: vmflt_vv:
 ; NOVLOPT:       # %bb.0:
 ; NOVLOPT-NEXT:    vsetvli a1, zero, e32, m2, ta, ma
-; NOVLOPT-NEXT:    vfwadd.wv v8, v8, v12
-; NOVLOPT-NEXT:    vsetvli zero, a0, e64, m4, ta, ma
-; NOVLOPT-NEXT:    vfadd.vv v8, v8, v8
+; NOVLOPT-NEXT:    vmflt.vv v12, v8, v10
+; NOVLOPT-NEXT:    vsetvli zero, a0, e8, mf2, ta, ma
+; NOVLOPT-NEXT:    vmand.mm v0, v12, v0
 ; NOVLOPT-NEXT:    ret
 ;
-; VLOPT-LABEL: vfwadd_wv:
+; VLOPT-LABEL: vmflt_vv:
 ; VLOPT:       # %bb.0:
 ; VLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
-; VLOPT-NEXT:    vfwadd.wv v8, v8, v12
-; VLOPT-NEXT:    vsetvli zero, zero, e64, m4, ta, ma
-; VLOPT-NEXT:    vfadd.vv v8, v8, v8
+; VLOPT-NEXT:    vmflt.vv v12, v8, v10
+; VLOPT-NEXT:    vmand.mm v0, v12, v0
 ; VLOPT-NEXT:    ret
-  %1 = call <vscale x 4 x double> @llvm.riscv.vfwadd.w.nxv4f64.nxv4f32.nxv4f32(<vscale x 4 x double> poison, <vscale x 4 x double> %a, <vscale x 4 x float> %b, iXLen 7, iXLen -1)
-  %2 = call <vscale x 4 x double> @llvm.riscv.vfadd.nxv4f64.nxv4f64(<vscale x 4 x double> poison, <vscale x 4 x double> %1, <vscale x 4 x double> %1, iXLen 7, iXLen %vl)
-  ret <vscale x 4 x double> %2
+  %1 = call <vscale x 4 x i1> @llvm.riscv.vmflt.nxv4f32.nxv4f32(<vscale x 4 x float> %a, <vscale x 4 x float> %c, iXLen -1)
+  %2 = call <vscale x 4 x i1> @llvm.riscv.vmand.nxv4i1(<vscale x 4 x i1> %1, <vscale x 4 x i1> %b, iXLen %vl)
+  ret <vscale x 4 x i1> %2
 }
 
-define <vscale x 4 x double> @vfwadd_wf(<vscale x 4 x double> %a, float %b, iXLen %vl) {
-; NOVLOPT-LABEL: vfwadd_wf:
+define <vscale x 4 x i1> @vmfle_vf(<vscale x 4 x float> %a, <vscale x 4 x i1> %b, float%c, iXLen %vl) {
+; NOVLOPT-LABEL: vmfle_vf:
 ; NOVLOPT:       # %bb.0:
 ; NOVLOPT-NEXT:    vsetvli a1, zero, e32, m2, ta, ma
-; NOVLOPT-NEXT:    vfwadd.wf v8, v8, fa0
-; NOVLOPT-NEXT:    vsetvli zero, a0, e64, m4, ta, ma
-; NOVLOPT-NEXT:    vfadd.vv v8, v8, v8
+; NOVLOPT-NEXT:    vmfle.vf v10, v8, fa0
+; NOVLOPT-NEXT:    vsetvli zero, a0, e8, mf2, ta, ma
+; NOVLOPT-NEXT:    vmand.mm v0, v10, v0
 ; NOVLOPT-NEXT:    ret
 ;
-; VLOPT-LABEL: vfwadd_wf:
+; VLOPT-LABEL: vmfle_vf:
 ; VLOPT:       # %bb.0:
 ; VLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
-; VLOPT-NEXT:    vfwadd.wf v8, v8, fa0
-; VLOPT-NEXT:    vsetvli zero, zero, e64, m4, ta, ma
-; VLOPT-NEXT:    vfadd.vv v8, v8, v8
+; VLOPT-NEXT:    vmfle.vf v10, v8, fa0
+; VLOPT-NEXT:    vmand.mm v0, v10, v0
 ; VLOPT-NEXT:    ret
-  %1 = call <vscale x 4 x double> @llvm.riscv.vfwadd.w.nxv4f64.nxv4f32.f32(<vscale x 4 x double> poison, <vscale x 4 x double> %a, float %b, iXLen 7, iXLen -1)
-  %2 = call <vscale x 4 x double> @llvm.riscv.vfadd.nxv4f64.nxv4f64(<vscale x 4 x double> poison, <vscale x 4 x double> %1, <vscale x 4 x double> %1, iXLen 7, iXLen %vl)
-  ret <vscale x 4 x double> %2
+  %1 = call <vscale x 4 x i1> @llvm.riscv.vmfle.nxv4f32.f32(<vscale x 4 x float> %a, float %c, iXLen -1)
+  %2 = call <vscale x 4 x i1> @llvm.riscv.vmand.nxv4i1(<vscale x 4 x i1> %1, <vscale x 4 x i1> %b, iXLen %vl)
+  ret <vscale x 4 x i1> %2
 }
 
-define <vscale x 4 x double> @vfwsub_wv(<vscale x 4 x double> %a, <vscale x 4 x float> %b, iXLen %vl) {
-; NOVLOPT-LABEL: vfwsub_wv:
+define <vscale x 4 x i1> @vmfle_vv(<vscale x 4 x float> %a, <vscale x 4 x i1> %b, <vscale x 4 x float> %c, iXLen %vl) {
+; NOVLOPT-LABEL: vmfle_vv:
 ; NOVLOPT:       # %bb.0:
 ; NOVLOPT-NEXT:    vsetvli a1, zero, e32, m2, ta, ma
-; NOVLOPT-NEXT:    vfwsub.wv v8, v8, v12
-; NOVLOPT-NEXT:    vsetvli zero, a0, e64, m4, ta, ma
-; NOVLOPT-NEXT:    vfadd.vv v8, v8, v8
+; NOVLOPT-NEXT:    vmfle.vv v12, v8, v10
+; NOVLOPT-NEXT:    vsetvli zero, a0, e8, mf2, ta, ma
+; NOVLOPT-NEXT:    vmand.mm v0, v12, v0
 ; NOVLOPT-NEXT:    ret
 ;
-; VLOPT-LABEL: vfwsub_wv:
+; VLOPT-LABEL: vmfle_vv:
 ; VLOPT:       # %bb.0:
 ; VLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
-; VLOPT-NEXT:    vfwsub.wv v8, v8, v12
-; VLOPT-NEXT:    vsetvli zero, zero, e64, m4, ta, ma
-; VLOPT-NEXT:    vfadd.vv v8, v8, v8
+; VLOPT-NEXT:    vmfle.vv v12, v8, v10
+; VLOPT-NEXT:    vmand.mm v0, v12, v0
 ; VLOPT-NEXT:    ret
-  %1 = call <vscale x 4 x double> @llvm.riscv.vfwsub.w.nxv4f64.nxv4f32.nxv4f32(<vscale x 4 x double> poison, <vscale x 4 x double> %a, <vscale x 4 x float> %b, iXLen 7, iXLen -1)
-  %2 = call <vscale x 4 x double> @llvm.riscv.vfadd.nxv4f64.nxv4f64(<vscale x 4 x double> poison, <vscale x 4 x double> %1, <vscale x 4 x double> %1, iXLen 7, iXLen %vl)
-  ret <vscale x 4 x double> %2
+  %1 = call <vscale x 4 x i1> @llvm.riscv.vmfle.nxv4f32.nxv4f32(<vscale x 4 x float> %a, <vscale x 4 x float> %c, iXLen -1)
+  %2 = call <vscale x 4 x i1> @llvm.riscv.vmand.nxv4i1(<vscale x 4 x i1> %1, <vscale x 4 x i1> %b, iXLen %vl)
+  ret <vscale x 4 x i1> %2
 }
 
-define <vscale x 4 x double> @vfwsub_wf(<vscale x 4 x double> %a, float %b, iXLen %vl) {
-; NOVLOPT-LABEL: vfwsub_wf:
+define <vscale x 4 x i1> @vmfgt_vf(<vscale x 4 x float> %a, <vscale x 4 x i1> %b, float%c, iXLen %vl) {
+; NOVLOPT-LABEL: vmfgt_vf:
 ; NOVLOPT:       # %bb.0:
 ; NOVLOPT-NEXT:    vsetvli a1, zero, e32, m2, ta, ma
-; NOVLOPT-NEXT:    vfwsub.wf v8, v8, fa0
-; NOVLOPT-NEXT:    vsetvli zero, a0, e64, m4, ta, ma
-; NOVLOPT-NEXT:    vfadd.vv v8, v8, v8
+; NOVLOPT-NEXT:    vmfgt.vf v10, v8, fa0
+; NOVLOPT-NEXT:    vsetvli zero, a0, e8, mf2, ta, ma
+; NOVLOPT-NEXT:    vmand.mm v0, v10, v0
 ; NOVLOPT-NEXT:    ret
 ;
-; VLOPT-LABEL: vfwsub_wf:
+; VLOPT-LABEL: vmfgt_vf:
 ; VLOPT:       # %bb.0:
 ; VLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
-; VLOPT-NEXT:    vfwsub.wf v8, v8, fa0
-; VLOPT-NEXT:    vsetvli zero, zero, e64, m4, ta, ma
-; VLOPT-NEXT:    vfadd.vv v8, v8, v8
+; VLOPT-NEXT:    vmfgt.vf v10, v8, fa0
+; VLOPT-NEXT:    vmand.mm v0, v10, v0
 ; VLOPT-NEXT:    ret
-  %1 = call <vscale x 4 x double> @llvm.riscv.vfwsub.w.nxv4f64.nxv4f32.f32(<vscale x 4 x double> poison, <vscale x 4 x double> %a, float %b, iXLen 7, iXLen -1)
-  %2 = call <vscale x 4 x double> @llvm.riscv.vfadd.nxv4f64.nxv4f64(<vscale x 4 x double> poison, <vscale x 4 x double> %1, <vscale x 4 x double> %1, iXLen 7, iXLen %vl)
-  ret <vscale x 4 x double> %2
+  %1 = call <vscale x 4 x i1> @llvm.riscv.vmfgt.nxv4f32.f32(<vscale x 4 x float> %a, float %c, iXLen -1)
+  %2 = call <vscale x 4 x i1> @llvm.riscv.vmand.nxv4i1(<vscale x 4 x i1> %1, <vscale x 4 x i1> %b, iXLen %vl)
+  ret <vscale x 4 x i1> %2
 }
 
-define <vscale x 4 x float> @vfmul_vv(<vscale x 4 x float> %a, <vscale x 4 x float> %b, iXLen %vl) {
-; NOVLOPT-LABEL: vfmul_vv:
+define <vscale x 4 x i1> @vmfgt_vv(<vscale x 4 x float> %a, <vscale x 4 x i1> %b, <vscale x 4 x float> %c, iXLen %vl) {
+; NOVLOPT-LABEL: vmfgt_vv:
 ; NOVLOPT:       # %bb.0:
 ; NOVLOPT-NEXT:    vsetvli a1, zero, e32, m2, ta, ma
-; NOVLOPT-NEXT:    vfmul.vv v8, v8, v10
-; NOVLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
-; NOVLOPT-NEXT:    vfadd.vv v8, v8, v10
+; NOVLOPT-NEXT:    vmflt.vv v12, v10, v8
+; NOVLOPT-NEXT:    vsetvli zero, a0, e8, mf2, ta, ma
+; NOVLOPT-NEXT:    vmand.mm v0, v12, v0
 ; NOVLOPT-NEXT:    ret
 ;
-; VLOPT-LABEL: vfmul_vv:
+; VLOPT-LABEL: vmfgt_vv:
 ; VLOPT:       # %bb.0:
 ; VLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
-; VLOPT-NEXT:    vfmul.vv v8, v8, v10
-; VLOPT-NEXT:    vfadd.vv v8, v8, v10
+; VLOPT-NEXT:    vmflt.vv v12, v10, v8
+; VLOPT-NEXT:    vmand.mm v0, v12, v0
 ; VLOPT-NEXT:    ret
-  %1 = call <vscale x 4 x float> @llvm.riscv.vfmul.nxv4f32.nxv4f32(<vscale x 4 x float> poison, <vscale x 4 x float> %a, <vscale x 4 x float> %b, iXLen 7, iXLen -1)
-  %2 = call <vscale x 4 x float> @llvm.riscv.vfadd.nxv4f32.nxv4f32(<vscale x 4 x float> poison, <vscale x 4 x float> %1, <vscale x 4 x float> %b, iXLen 7, iXLen %vl)
-  ret <vscale x 4 x float> %2
+  %1 = call <vscale x 4 x i1> @llvm.riscv.vmfgt.nxv4f32.nxv4f32(<vscale x 4 x float> %a, <vscale x 4 x float> %c, iXLen -1)
+  %2 = call <vscale x 4 x i1> @llvm.riscv.vmand.nxv4i1(<vscale x 4 x i1> %1, <vscale x 4 x i1> %b, iXLen %vl)
+  ret <vscale x 4 x i1> %2
 }
 
-define <vscale x 4 x float> @vfmul_vf(<vscale x 4 x float> %a, float %b, iXLen %vl) {
-; NOVLOPT-LABEL: vfmul_vf:
+define <vscale x 4 x i32> @vmerge_vvm(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b, <vscale x 4 x i1> %c, iXLen %vl) {
+; NOVLOPT-LABEL: vmerge_vvm:
 ; NOVLOPT:       # %bb.0:
 ; NOVLOPT-NEXT:    vsetvli a1, zero, e32, m2, ta, ma
-; NOVLOPT-NEXT:    vfmul.vf v10, v8, fa0
+; NOVLOPT-NEXT:    vmerge.vvm v8, v8, v10, v0
 ; NOVLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
-; NOVLOPT-NEXT:    vfadd.vv v8, v10, v8
+; NOVLOPT-NEXT:    vadd.vv v8, v8, v10
 ; NOVLOPT-NEXT:    ret
 ;
-; VLOPT-LABEL: vfmul_vf:
+; VLOPT-LABEL: vmerge_vvm:
 ; VLOPT:       # %bb.0:
 ; VLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
-; VLOPT-NEXT:    vfmul.vf v10, v8, fa0
-; VLOPT-NEXT:    vfadd.vv v8, v10, v8
+; VLOPT-NEXT:    vmerge.vvm v8, v8, v10, v0
+; VLOPT-NEXT:    vadd.vv v8, v8, v10
 ; VLOPT-NEXT:    ret
-  %1 = call <vscale x 4 x float> @llvm.riscv.vfmul.nxv4f32.nxv4f32(<vscale x 4 x float> poison, <vscale x 4 x float> %a, float %b, iXLen 7, iXLen -1)
-  %2 = call <vscale x 4 x float> @llvm.riscv.vfadd.nxv4f32.nxv4f32(<vscale x 4 x float> poison, <vscale x 4 x float> %1, <vscale x 4 x float> %a, iXLen 7, iXLen %vl)
-  ret <vscale x 4 x float> %2
+  %1 = call <vscale x 4 x i32> @llvm.riscv.vmerge.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %a, <vscale x 4 x i32> %b, <vscale x 4 x i1> %c, iXLen -1)
+  %2 = call <vscale x 4 x i32> @llvm.riscv.vadd.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %1, <vscale x 4 x i32> %b, iXLen %vl)
+  ret <vscale x 4 x i32> %2
 }
 
-define <vscale x 4 x float> @vfdiv_vv(<vscale x 4 x float> %a, <vscale x 4 x float> %b, iXLen %vl) {
-; NOVLOPT-LABEL: vfdiv_vv:
+define <vscale x 4 x i32> @vmerge_vxm(<vscale x 4 x i32> %a, i32 %b, <vscale x 4 x i1> %c, iXLen %vl) {
+; NOVLOPT-LABEL: vmerge_vxm:
 ; NOVLOPT:       # %bb.0:
-; NOVLOPT-NEXT:    vsetvli a1, zero, e32, m2, ta, ma
-; NOVLOPT-NEXT:    vfdiv.vv v8, v8, v10
-; NOVLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
-; NOVLOPT-NEXT:    vfadd.vv v8, v8, v10
+; NOVLOPT-NEXT:    vsetvli a2, zero, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vmerge.vxm v8, v8, a0, v0
+; NOVLOPT-NEXT:    vsetvli zero, a1, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vadd.vv v8, v8, v8
 ; NOVLOPT-NEXT:    ret
 ;
-; VLOPT-LABEL: vfdiv_vv:
+; VLOPT-LABEL: vmerge_vxm:
 ; VLOPT:       # %bb.0:
-; VLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
-; VLOPT-NEXT:    vfdiv.vv v8, v8, v10
-; VLOPT-NEXT:    vfadd.vv v8, v8, v10
+; VLOPT-NEXT:    vsetvli zero, a1, e32, m2, ta, ma
+; VLOPT-NEXT:    vmerge.vxm v8, v8, a0, v0
+; VLOPT-NEXT:    vadd.vv v8, v8, v8
 ; VLOPT-NEXT:    ret
-  %1 = call <vscale x 4 x float> @llvm.riscv.vfdiv.nxv4f32.nxv4f32(<vscale x 4 x float> poison, <vscale x 4 x float> %a, <vscale x 4 x float> %b, iXLen 7, iXLen -1)
-  %2 = call <vscale x 4 x float> @llvm.riscv.vfadd.nxv4f32.nxv4f32(<vscale x 4 x float> poison, <vscale x 4 x float> %1, <vscale x 4 x float> %b, iXLen 7, iXLen %vl)
-  ret <vscale x 4 x float> %2
+  %1 = call <vscale x 4 x i32> @llvm.riscv.vmerge.nxv4i32.i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %a, i32 %b, <vscale x 4 x i1> %c, iXLen -1)
+  %2 = call <vscale x 4 x i32> @llvm.riscv.vadd.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %1, <vscale x 4 x i32> %1, iXLen %vl)
+  ret <vscale x 4 x i32> %2
 }
 
-define <vscale x 4 x float> @vfdiv_vf(<vscale x 4 x float> %a, float %b, iXLen %vl) {
-; NOVLOPT-LABEL: vfdiv_vf:
+define <vscale x 4 x i32> @vmerge_vim(<vscale x 4 x i32> %a, <vscale x 4 x i1> %c, iXLen %vl) {
+; NOVLOPT-LABEL: vmerge_vim:
 ; NOVLOPT:       # %bb.0:
 ; NOVLOPT-NEXT:    vsetvli a1, zero, e32, m2, ta, ma
-; NOVLOPT-NEXT:    vfdiv.vf v10, v8, fa0
+; NOVLOPT-NEXT:    vmerge.vim v8, v8, 9, v0
 ; NOVLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
-; NOVLOPT-NEXT:    vfadd.vv v8, v10, v8
+; NOVLOPT-NEXT:    vadd.vv v8, v8, v8
 ; NOVLOPT-NEXT:    ret
 ;
-; VLOPT-LABEL: vfdiv_vf:
+; VLOPT-LABEL: vmerge_vim:
 ; VLOPT:       # %bb.0:
 ; VLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
-; VLOPT-NEXT:    vfdiv.vf v10, v8, fa0
-; VLOPT-NEXT:    vfadd.vv v8, v10, v8
+; VLOPT-NEXT:    vmerge.vim v8, v8, 9, v0
+; VLOPT-NEXT:    vadd.vv v8, v8, v8
 ; VLOPT-NEXT:    ret
-  %1 = call <vscale x 4 x float> @llvm.riscv.vfdiv.nxv4f32.nxv4f32(<vscale x 4 x float> poison, <vscale x 4 x float> %a, float %b, iXLen 7, iXLen -1)
-  %2 = call <vscale x 4 x float> @llvm.riscv.vfadd.nxv4f32.nxv4f32(<vscale x 4 x float> poison, <vscale x 4 x float> %1, <vscale x 4 x float> %a, iXLen 7, iXLen %vl)
-  ret <vscale x 4 x float> %2
+  %1 = call <vscale x 4 x i32> @llvm.riscv.vmerge.nxv4i32.i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %a, i32 9, <vscale x 4 x i1> %c, iXLen -1)
+  %2 = call <vscale x 4 x i32> @llvm.riscv.vadd.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %1, <vscale x 4 x i32> %1, iXLen %vl)
+  ret <vscale x 4 x i32> %2
 }
 
-define <vscale x 4 x float> @vfrdiv_vf(<vscale x 4 x float> %a, float %b, iXLen %vl) {
-; NOVLOPT-LABEL: vfrdiv_vf:
+define <vscale x 4 x i32> @vadc_vvm(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b, <vscale x 4 x i1> %c, iXLen %vl) {
+; NOVLOPT-LABEL: vadc_vvm:
 ; NOVLOPT:       # %bb.0:
 ; NOVLOPT-NEXT:    vsetvli a1, zero, e32, m2, ta, ma
-; NOVLOPT-NEXT:    vfrdiv.vf v10, v8, fa0
+; NOVLOPT-NEXT:    vadc.vvm v8, v8, v10, v0
 ; NOVLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
-; NOVLOPT-NEXT:    vfadd.vv v8, v10, v8
+; NOVLOPT-NEXT:    vadd.vv v8, v8, v10
 ; NOVLOPT-NEXT:    ret
 ;
-; VLOPT-LABEL: vfrdiv_vf:
+; VLOPT-LABEL: vadc_vvm:
 ; VLOPT:       # %bb.0:
 ; VLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
-; VLOPT-NEXT:    vfrdiv.vf v10, v8, fa0
-; VLOPT-NEXT:    vfadd.vv v8, v10, v8
+; VLOPT-NEXT:    vadc.vvm v8, v8, v10, v0
+; VLOPT-NEXT:    vadd.vv v8, v8, v10
 ; VLOPT-NEXT:    ret
-  %1 = call <vscale x 4 x float> @llvm.riscv.vfrdiv.nxv4f32.nxv4f32(<vscale x 4 x float> poison, <vscale x 4 x float> %a, float %b, iXLen 7, iXLen -1)
-  %2 = call <vscale x 4 x float> @llvm.riscv.vfadd.nxv4f32.nxv4f32(<vscale x 4 x float> poison, <vscale x 4 x float> %1, <vscale x 4 x float> %a, iXLen 7, iXLen %vl)
-  ret <vscale x 4 x float> %2
+  %1 = call <vscale x 4 x i32> @llvm.riscv.vadc.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %a, <vscale x 4 x i32> %b, <vscale x 4 x i1> %c, iXLen -1)
+  %2 = call <vscale x 4 x i32> @llvm.riscv.vadd.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %1, <vscale x 4 x i32> %b, iXLen %vl)
+  ret <vscale x 4 x i32> %2
+}
+
+define <vscale x 4 x i32> @vadc_vxm(<vscale x 4 x i32> %a, i32 %b, <vscale x 4 x i1> %c, iXLen %vl) {
+; NOVLOPT-LABEL: vadc_vxm:
+; NOVLOPT:       # %bb.0:
+; NOVLOPT-NEXT:    vsetvli a2, zero, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vadc.vxm v8, v8, a0, v0
+; NOVLOPT-NEXT:    vsetvli zero, a1, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vadd.vv v8, v8, v8
+; NOVLOPT-NEXT:    ret
+;
+; VLOPT-LABEL: vadc_vxm:
+; VLOPT:       # %bb.0:
+; VLOPT-NEXT:    vsetvli zero, a1, e32, m2, ta, ma
+; VLOPT-NEXT:    vadc.vxm v8, v8, a0, v0
+; VLOPT-NEXT:    vadd.vv v8, v8, v8
+; VLOPT-NEXT:    ret
+  %1 = call <vscale x 4 x i32> @llvm.riscv.vadc.nxv4i32.i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %a, i32 %b, <vscale x 4 x i1> %c, iXLen -1)
+  %2 = call <vscale x 4 x i32> @llvm.riscv.vadd.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %1, <vscale x 4 x i32> %1, iXLen %vl)
+  ret <vscale x 4 x i32> %2
 }
 
-define <vscale x 4 x double> @vfwmul_vv(<vscale x 4 x float> %a, <vscale x 4 x float> %b, iXLen %vl) {
-; NOVLOPT-LABEL: vfwmul_vv:
+define <vscale x 4 x i32> @vadc_vim(<vscale x 4 x i32> %a, <vscale x 4 x i1> %c, iXLen %vl) {
+; NOVLOPT-LABEL: vadc_vim:
 ; NOVLOPT:       # %bb.0:
 ; NOVLOPT-NEXT:    vsetvli a1, zero, e32, m2, ta, ma
-; NOVLOPT-NEXT:    vfwmul.vv v12, v8, v10
-; NOVLOPT-NEXT:    vsetvli zero, a0, e64, m4, ta, ma
-; NOVLOPT-NEXT:    vfadd.vv v8, v12, v12
+; NOVLOPT-NEXT:    vadc.vim v8, v8, 9, v0
+; NOVLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vadd.vv v8, v8, v8
 ; NOVLOPT-NEXT:    ret
 ;
-; VLOPT-LABEL: vfwmul_vv:
+; VLOPT-LABEL: vadc_vim:
 ; VLOPT:       # %bb.0:
 ; VLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
-; VLOPT-NEXT:    vfwmul.vv v12, v8, v10
-; VLOPT-NEXT:    vsetvli zero, zero, e64, m4, ta, ma
-; VLOPT-NEXT:    vfadd.vv v8, v12, v12
+; VLOPT-NEXT:    vadc.vim v8, v8, 9, v0
+; VLOPT-NEXT:    vadd.vv v8, v8, v8
 ; VLOPT-NEXT:    ret
-  %1 = call <vscale x 4 x double> @llvm.riscv.vfwmul.nxv4f64.nxv4f32.nxv4f32(<vscale x 4 x double> poison, <vscale x 4 x float> %a, <vscale x 4 x float> %b, iXLen 7, iXLen -1)
-  %2 = call <vscale x 4 x double> @llvm.riscv.vfadd.nxv4f64.nxv4f64(<vscale x 4 x double> poison, <vscale x 4 x double> %1, <vscale x 4 x double> %1, iXLen 7, iXLen %vl)
-  ret <vscale x 4 x double> %2
+  %1 = call <vscale x 4 x i32> @llvm.riscv.vadc.nxv4i32.i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %a, i32 9, <vscale x 4 x i1> %c, iXLen -1)
+  %2 = call <vscale x 4 x i32> @llvm.riscv.vadd.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %1, <vscale x 4 x i32> %1, iXLen %vl)
+  ret <vscale x 4 x i32> %2
 }
 
-define <vscale x 4 x double> @vfwmul_vf(<vscale x 4 x float> %a, float %b, iXLen %vl) {
-; NOVLOPT-LABEL: vfwmul_vf:
+define <vscale x 4 x i32> @vaadd_vv(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b, iXLen %vl) {
+; NOVLOPT-LABEL: vaadd_vv:
 ; NOVLOPT:       # %bb.0:
+; NOVLOPT-NEXT:    csrwi vxrm, 0
 ; NOVLOPT-NEXT:    vsetvli a1, zero, e32, m2, ta, ma
-; NOVLOPT-NEXT:    vfwmul.vf v12, v8, fa0
-; NOVLOPT-NEXT:    vsetvli zero, a0, e64, m4, ta, ma
-; NOVLOPT-NEXT:    vfadd.vv v8, v12, v12
+; NOVLOPT-NEXT:    vaadd.vv v8, v8, v10
+; NOVLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vadd.vv v8, v8, v10
 ; NOVLOPT-NEXT:    ret
 ;
-; VLOPT-LABEL: vfwmul_vf:
+; VLOPT-LABEL: vaadd_vv:
 ; VLOPT:       # %bb.0:
+; VLOPT-NEXT:    csrwi vxrm, 0
 ; VLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
-; VLOPT-NEXT:    vfwmul.vf v12, v8, fa0
-; VLOPT-NEXT:    vsetvli zero, zero, e64, m4, ta, ma
-; VLOPT-NEXT:    vfadd.vv v8, v12, v12
+; VLOPT-NEXT:    vaadd.vv v8, v8, v10
+; VLOPT-NEXT:    vadd.vv v8, v8, v10
 ; VLOPT-NEXT:    ret
-  %1 = call <vscale x 4 x double> @llvm.riscv.vfwmul.nxv4f64.nxv4f32.f32(<vscale x 4 x double> poison, <vscale x 4 x float> %a, float %b, iXLen 7, iXLen -1)
-  %2 = call <vscale x 4 x double> @llvm.riscv.vfadd.nxv4f64.nxv4f64(<vscale x 4 x double> poison, <vscale x 4 x double> %1, <vscale x 4 x double> %1, iXLen 7, iXLen %vl)
-  ret <vscale x 4 x double> %2
+  %1 = call <vscale x 4 x i32> @llvm.riscv.vaadd.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %a, <vscale x 4 x i32> %b, iXLen 0, iXLen -1)
+  %2 = call <vscale x 4 x i32> @llvm.riscv.vadd.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %1, <vscale x 4 x i32> %b, iXLen %vl)
+  ret <vscale x 4 x i32> %2
 }
 
-define <vscale x 4 x i1> @vmfeq_vf(<vscale x 4 x float> %a, <vscale x 4 x i1> %b, float%c, iXLen %vl) {
-; NOVLOPT-LABEL: vmfeq_vf:
+define <vscale x 4 x i32> @vaadd_vx(<vscale x 4 x i32> %a, i32 %b, iXLen %vl) {
+; NOVLOPT-LABEL: vaadd_vx:
 ; NOVLOPT:       # %bb.0:
-; NOVLOPT-NEXT:    vsetvli a1, zero, e32, m2, ta, ma
-; NOVLOPT-NEXT:    vmfeq.vf v10, v8, fa0
-; NOVLOPT-NEXT:    vsetvli zero, a0, e8, mf2, ta, ma
-; NOVLOPT-NEXT:    vmand.mm v0, v10, v0
+; NOVLOPT-NEXT:    csrwi vxrm, 0
+; NOVLOPT-NEXT:    vsetvli a2, zero, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vaadd.vx v10, v8, a0
+; NOVLOPT-NEXT:    vsetvli zero, a1, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vadd.vv v8, v10, v8
 ; NOVLOPT-NEXT:    ret
 ;
-; VLOPT-LABEL: vmfeq_vf:
+; VLOPT-LABEL: vaadd_vx:
 ; VLOPT:       # %bb.0:
-; VLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
-; VLOPT-NEXT:    vmfeq.vf v10, v8, fa0
-; VLOPT-NEXT:    vmand.mm v0, v10, v0
+; VLOPT-NEXT:    csrwi vxrm, 0
+; VLOPT-NEXT:    vsetvli zero, a1, e32, m2, ta, ma
+; VLOPT-NEXT:    vaadd.vx v10, v8, a0
+; VLOPT-NEXT:    vadd.vv v8, v10, v8
 ; VLOPT-NEXT:    ret
-  %1 = call <vscale x 4 x i1> @llvm.riscv.vmfeq.nxv4f32.f32(<vscale x 4 x float> %a, float %c, iXLen -1)
-  %2 = call <vscale x 4 x i1> @llvm.riscv.vmand.nxv4i1(<vscale x 4 x i1> %1, <vscale x 4 x i1> %b, iXLen %vl)
-  ret <vscale x 4 x i1> %2
+  %1 = call <vscale x 4 x i32> @llvm.riscv.vaadd.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %a, i32 %b, iXLen 0, iXLen -1)
+  %2 = call <vscale x 4 x i32> @llvm.riscv.vadd.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %1, <vscale x 4 x i32> %a, iXLen %vl)
+  ret <vscale x 4 x i32> %2
 }
 
-define <vscale x 4 x i1> @vmfeq_vv(<vscale x 4 x float> %a, <vscale x 4 x i1> %b, <vscale x 4 x float> %c, iXLen %vl) {
-; NOVLOPT-LABEL: vmfeq_vv:
+define <vscale x 4 x i32> @vasub_vv(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b, iXLen %vl) {
+; NOVLOPT-LABEL: vasub_vv:
 ; NOVLOPT:       # %bb.0:
+; NOVLOPT-NEXT:    csrwi vxrm, 0
 ; NOVLOPT-NEXT:    vsetvli a1, zero, e32, m2, ta, ma
-; NOVLOPT-NEXT:    vmfeq.vv v12, v8, v10
-; NOVLOPT-NEXT:    vsetvli zero, a0, e8, mf2, ta, ma
-; NOVLOPT-NEXT:    vmand.mm v0, v12, v0
+; NOVLOPT-NEXT:    vasub.vv v8, v8, v10
+; NOVLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vadd.vv v8, v8, v10
 ; NOVLOPT-NEXT:    ret
 ;
-; VLOPT-LABEL: vmfeq_vv:
+; VLOPT-LABEL: vasub_vv:
 ; VLOPT:       # %bb.0:
+; VLOPT-NEXT:    csrwi vxrm, 0
 ; VLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
-; VLOPT-NEXT:    vmfeq.vv v12, v8, v10
-; VLOPT-NEXT:    vmand.mm v0, v12, v0
+; VLOPT-NEXT:    vasub.vv v8, v8, v10
+; VLOPT-NEXT:    vadd.vv v8, v8, v10
 ; VLOPT-NEXT:    ret
-  %1 = call <vscale x 4 x i1> @llvm.riscv.vmfeq.nxv4f32.nxv4f32(<vscale x 4 x float> %a, <vscale x 4 x float> %c, iXLen -1)
-  %2 = call <vscale x 4 x i1> @llvm.riscv.vmand.nxv4i1(<vscale x 4 x i1> %1, <vscale x 4 x i1> %b, iXLen %vl)
-  ret <vscale x 4 x i1> %2
+  %1 = call <vscale x 4 x i32> @llvm.riscv.vasub.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %a, <vscale x 4 x i32> %b, iXLen 0, iXLen -1)
+  %2 = call <vscale x 4 x i32> @llvm.riscv.vadd.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %1, <vscale x 4 x i32> %b, iXLen %vl)
+  ret <vscale x 4 x i32> %2
 }
 
-define <vscale x 4 x i1> @vmfne_vf(<vscale x 4 x float> %a, <vscale x 4 x i1> %b, float%c, iXLen %vl) {
-; NOVLOPT-LABEL: vmfne_vf:
+define <vscale x 4 x i32> @vasub_vx(<vscale x 4 x i32> %a, i32 %b, iXLen %vl) {
+; NOVLOPT-LABEL: vasub_vx:
 ; NOVLOPT:       # %bb.0:
-; NOVLOPT-NEXT:    vsetvli a1, zero, e32, m2, ta, ma
-; NOVLOPT-NEXT:    vmfne.vf v10, v8, fa0
-; NOVLOPT-NEXT:    vsetvli zero, a0, e8, mf2, ta, ma
-; NOVLOPT-NEXT:    vmand.mm v0, v10, v0
+; NOVLOPT-NEXT:    csrwi vxrm, 0
+; NOVLOPT-NEXT:    vsetvli a2, zero, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vasub.vx v10, v8, a0
+; NOVLOPT-NEXT:    vsetvli zero, a1, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vadd.vv v8, v10, v8
 ; NOVLOPT-NEXT:    ret
 ;
-; VLOPT-LABEL: vmfne_vf:
+; VLOPT-LABEL: vasub_vx:
 ; VLOPT:       # %bb.0:
-; VLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
-; VLOPT-NEXT:    vmfne.vf v10, v8, fa0
-; VLOPT-NEXT:    vmand.mm v0, v10, v0
+; VLOPT-NEXT:    csrwi vxrm, 0
+; VLOPT-NEXT:    vsetvli zero, a1, e32, m2, ta, ma
+; VLOPT-NEXT:    vasub.vx v10, v8, a0
+; VLOPT-NEXT:    vadd.vv v8, v10, v8
 ; VLOPT-NEXT:    ret
-  %1 = call <vscale x 4 x i1> @llvm.riscv.vmfne.nxv4f32.f32(<vscale x 4 x float> %a, float %c, iXLen -1)
-  %2 = call <vscale x 4 x i1> @llvm.riscv.vmand.nxv4i1(<vscale x 4 x i1> %1, <vscale x 4 x i1> %b, iXLen %vl)
-  ret <vscale x 4 x i1> %2
+  %1 = call <vscale x 4 x i32> @llvm.riscv.vasub.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %a, i32 %b, iXLen 0, iXLen -1)
+  %2 = call <vscale x 4 x i32> @llvm.riscv.vadd.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %1, <vscale x 4 x i32> %a, iXLen %vl)
+  ret <vscale x 4 x i32> %2
 }
 
-define <vscale x 4 x i1> @vmfne_vv(<vscale x 4 x float> %a, <vscale x 4 x i1> %b, <vscale x 4 x float> %c, iXLen %vl) {
-; NOVLOPT-LABEL: vmfne_vv:
+define <vscale x 4 x i32> @vaaddu_vv(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b, iXLen %vl) {
+; NOVLOPT-LABEL: vaaddu_vv:
 ; NOVLOPT:       # %bb.0:
+; NOVLOPT-NEXT:    csrwi vxrm, 0
 ; NOVLOPT-NEXT:    vsetvli a1, zero, e32, m2, ta, ma
-; NOVLOPT-NEXT:    vmfne.vv v12, v8, v10
-; NOVLOPT-NEXT:    vsetvli zero, a0, e8, mf2, ta, ma
-; NOVLOPT-NEXT:    vmand.mm v0, v12, v0
+; NOVLOPT-NEXT:    vaaddu.vv v8, v8, v10
+; NOVLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vadd.vv v8, v8, v10
 ; NOVLOPT-NEXT:    ret
 ;
-; VLOPT-LABEL: vmfne_vv:
+; VLOPT-LABEL: vaaddu_vv:
 ; VLOPT:       # %bb.0:
+; VLOPT-NEXT:    csrwi vxrm, 0
 ; VLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
-; VLOPT-NEXT:    vmfne.vv v12, v8, v10
-; VLOPT-NEXT:    vmand.mm v0, v12, v0
+; VLOPT-NEXT:    vaaddu.vv v8, v8, v10
+; VLOPT-NEXT:    vadd.vv v8, v8, v10
 ; VLOPT-NEXT:    ret
-  %1 = call <vscale x 4 x i1> @llvm.riscv.vmfne.nxv4f32.nxv4f32(<vscale x 4 x float> %a, <vscale x 4 x float> %c, iXLen -1)
-  %2 = call <vscale x 4 x i1> @llvm.riscv.vmand.nxv4i1(<vscale x 4 x i1> %1, <vscale x 4 x i1> %b, iXLen %vl)
-  ret <vscale x 4 x i1> %2
+  %1 = call <vscale x 4 x i32> @llvm.riscv.vaaddu.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %a, <vscale x 4 x i32> %b, iXLen 0, iXLen -1)
+  %2 = call <vscale x 4 x i32> @llvm.riscv.vadd.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %1, <vscale x 4 x i32> %b, iXLen %vl)
+  ret <vscale x 4 x i32> %2
 }
 
-define <vscale x 4 x i1> @vmflt_vf(<vscale x 4 x float> %a, <vscale x 4 x i1> %b, float%c, iXLen %vl) {
-; NOVLOPT-LABEL: vmflt_vf:
+define <vscale x 4 x i32> @vaaddu_vx(<vscale x 4 x i32> %a, i32 %b, iXLen %vl) {
+; NOVLOPT-LABEL: vaaddu_vx:
 ; NOVLOPT:       # %bb.0:
-; NOVLOPT-NEXT:    vsetvli a1, zero, e32, m2, ta, ma
-; NOVLOPT-NEXT:    vmflt.vf v10, v8, fa0
-; NOVLOPT-NEXT:    vsetvli zero, a0, e8, mf2, ta, ma
-; NOVLOPT-NEXT:    vmand.mm v0, v10, v0
+; NOVLOPT-NEXT:    csrwi vxrm, 0
+; NOVLOPT-NEXT:    vsetvli a2, zero, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vaaddu.vx v10, v8, a0
+; NOVLOPT-NEXT:    vsetvli zero, a1, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vadd.vv v8, v10, v8
 ; NOVLOPT-NEXT:    ret
 ;
-; VLOPT-LABEL: vmflt_vf:
+; VLOPT-LABEL: vaaddu_vx:
 ; VLOPT:       # %bb.0:
-; VLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
-; VLOPT-NEXT:    vmflt.vf v10, v8, fa0
-; VLOPT-NEXT:    vmand.mm v0, v10, v0
+; VLOPT-NEXT:    csrwi vxrm, 0
+; VLOPT-NEXT:    vsetvli zero, a1, e32, m2, ta, ma
+; VLOPT-NEXT:    vaaddu.vx v10, v8, a0
+; VLOPT-NEXT:    vadd.vv v8, v10, v8
 ; VLOPT-NEXT:    ret
-  %1 = call <vscale x 4 x i1> @llvm.riscv.vmflt.nxv4f32.f32(<vscale x 4 x float> %a, float %c, iXLen -1)
-  %2 = call <vscale x 4 x i1> @llvm.riscv.vmand.nxv4i1(<vscale x 4 x i1> %1, <vscale x 4 x i1> %b, iXLen %vl)
-  ret <vscale x 4 x i1> %2
+  %1 = call <vscale x 4 x i32> @llvm.riscv.vaaddu.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %a, i32 %b, iXLen 0, iXLen -1)
+  %2 = call <vscale x 4 x i32> @llvm.riscv.vadd.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %1, <vscale x 4 x i32> %a, iXLen %vl)
+  ret <vscale x 4 x i32> %2
 }
 
-define <vscale x 4 x i1> @vmflt_vv(<vscale x 4 x float> %a, <vscale x 4 x i1> %b, <vscale x 4 x float> %c, iXLen %vl) {
-; NOVLOPT-LABEL: vmflt_vv:
+define <vscale x 4 x i32> @vasubu_vv(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b, iXLen %vl) {
+; NOVLOPT-LABEL: vasubu_vv:
 ; NOVLOPT:       # %bb.0:
+; NOVLOPT-NEXT:    csrwi vxrm, 0
 ; NOVLOPT-NEXT:    vsetvli a1, zero, e32, m2, ta, ma
-; NOVLOPT-NEXT:    vmflt.vv v12, v8, v10
-; NOVLOPT-NEXT:    vsetvli zero, a0, e8, mf2, ta, ma
-; NOVLOPT-NEXT:    vmand.mm v0, v12, v0
+; NOVLOPT-NEXT:    vasubu.vv v8, v8, v10
+; NOVLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vadd.vv v8, v8, v10
 ; NOVLOPT-NEXT:    ret
 ;
-; VLOPT-LABEL: vmflt_vv:
+; VLOPT-LABEL: vasubu_vv:
 ; VLOPT:       # %bb.0:
+; VLOPT-NEXT:    csrwi vxrm, 0
 ; VLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
-; VLOPT-NEXT:    vmflt.vv v12, v8, v10
-; VLOPT-NEXT:    vmand.mm v0, v12, v0
+; VLOPT-NEXT:    vasubu.vv v8, v8, v10
+; VLOPT-NEXT:    vadd.vv v8, v8, v10
 ; VLOPT-NEXT:    ret
-  %1 = call <vscale x 4 x i1> @llvm.riscv.vmflt.nxv4f32.nxv4f32(<vscale x 4 x float> %a, <vscale x 4 x float> %c, iXLen -1)
-  %2 = call <vscale x 4 x i1> @llvm.riscv.vmand.nxv4i1(<vscale x 4 x i1> %1, <vscale x 4 x i1> %b, iXLen %vl)
-  ret <vscale x 4 x i1> %2
+  %1 = call <vscale x 4 x i32> @llvm.riscv.vasubu.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %a, <vscale x 4 x i32> %b, iXLen 0, iXLen -1)
+  %2 = call <vscale x 4 x i32> @llvm.riscv.vadd.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %1, <vscale x 4 x i32> %b, iXLen %vl)
+  ret <vscale x 4 x i32> %2
 }
 
-define <vscale x 4 x i1> @vmfle_vf(<vscale x 4 x float> %a, <vscale x 4 x i1> %b, float%c, iXLen %vl) {
-; NOVLOPT-LABEL: vmfle_vf:
+define <vscale x 4 x i32> @vasubu_vx(<vscale x 4 x i32> %a, i32 %b, iXLen %vl) {
+; NOVLOPT-LABEL: vasubu_vx:
 ; NOVLOPT:       # %bb.0:
-; NOVLOPT-NEXT:    vsetvli a1, zero, e32, m2, ta, ma
-; NOVLOPT-NEXT:    vmfle.vf v10, v8, fa0
-; NOVLOPT-NEXT:    vsetvli zero, a0, e8, mf2, ta, ma
-; NOVLOPT-NEXT:    vmand.mm v0, v10, v0
+; NOVLOPT-NEXT:    csrwi vxrm, 0
+; NOVLOPT-NEXT:    vsetvli a2, zero, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vasubu.vx v10, v8, a0
+; NOVLOPT-NEXT:    vsetvli zero, a1, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vadd.vv v8, v10, v8
 ; NOVLOPT-NEXT:    ret
 ;
-; VLOPT-LABEL: vmfle_vf:
+; VLOPT-LABEL: vasubu_vx:
 ; VLOPT:       # %bb.0:
-; VLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
-; VLOPT-NEXT:    vmfle.vf v10, v8, fa0
-; VLOPT-NEXT:    vmand.mm v0, v10, v0
+; VLOPT-NEXT:    csrwi vxrm, 0
+; VLOPT-NEXT:    vsetvli zero, a1, e32, m2, ta, ma
+; VLOPT-NEXT:    vasubu.vx v10, v8, a0
+; VLOPT-NEXT:    vadd.vv v8, v10, v8
 ; VLOPT-NEXT:    ret
-  %1 = call <vscale x 4 x i1> @llvm.riscv.vmfle.nxv4f32.f32(<vscale x 4 x float> %a, float %c, iXLen -1)
-  %2 = call <vscale x 4 x i1> @llvm.riscv.vmand.nxv4i1(<vscale x 4 x i1> %1, <vscale x 4 x i1> %b, iXLen %vl)
-  ret <vscale x 4 x i1> %2
+  %1 = call <vscale x 4 x i32> @llvm.riscv.vasubu.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %a, i32 %b, iXLen 0, iXLen -1)
+  %2 = call <vscale x 4 x i32> @llvm.riscv.vadd.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %1, <vscale x 4 x i32> %a, iXLen %vl)
+  ret <vscale x 4 x i32> %2
 }
 
-define <vscale x 4 x i1> @vmfle_vv(<vscale x 4 x float> %a, <vscale x 4 x i1> %b, <vscale x 4 x float> %c, iXLen %vl) {
-; NOVLOPT-LABEL: vmfle_vv:
+define <vscale x 4 x float> @vfmax_vv(<vscale x 4 x float> %a, <vscale x 4 x float> %b, iXLen %vl) {
+; NOVLOPT-LABEL: vfmax_vv:
 ; NOVLOPT:       # %bb.0:
 ; NOVLOPT-NEXT:    vsetvli a1, zero, e32, m2, ta, ma
-; NOVLOPT-NEXT:    vmfle.vv v12, v8, v10
-; NOVLOPT-NEXT:    vsetvli zero, a0, e8, mf2, ta, ma
-; NOVLOPT-NEXT:    vmand.mm v0, v12, v0
+; NOVLOPT-NEXT:    vfmax.vv v8, v8, v10
+; NOVLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vfadd.vv v8, v8, v10
 ; NOVLOPT-NEXT:    ret
 ;
-; VLOPT-LABEL: vmfle_vv:
+; VLOPT-LABEL: vfmax_vv:
 ; VLOPT:       # %bb.0:
 ; VLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
-; VLOPT-NEXT:    vmfle.vv v12, v8, v10
-; VLOPT-NEXT:    vmand.mm v0, v12, v0
+; VLOPT-NEXT:    vfmax.vv v8, v8, v10
+; VLOPT-NEXT:    vfadd.vv v8, v8, v10
 ; VLOPT-NEXT:    ret
-  %1 = call <vscale x 4 x i1> @llvm.riscv.vmfle.nxv4f32.nxv4f32(<vscale x 4 x float> %a, <vscale x 4 x float> %c, iXLen -1)
-  %2 = call <vscale x 4 x i1> @llvm.riscv.vmand.nxv4i1(<vscale x 4 x i1> %1, <vscale x 4 x i1> %b, iXLen %vl)
-  ret <vscale x 4 x i1> %2
+  %1 = call <vscale x 4 x float> @llvm.riscv.vfmax.nxv4f32.nxv4f32(<vscale x 4 x float> poison, <vscale x 4 x float> %a, <vscale x 4 x float> %b, iXLen -1)
+  %2 = call <vscale x 4 x float> @llvm.riscv.vfadd.nxv4f32.nxv4f32(<vscale x 4 x float> poison, <vscale x 4 x float> %1, <vscale x 4 x float> %b, iXLen 7, iXLen %vl)
+  ret <vscale x 4 x float> %2
 }
 
-define <vscale x 4 x i1> @vmfgt_vf(<vscale x 4 x float> %a, <vscale x 4 x i1> %b, float%c, iXLen %vl) {
-; NOVLOPT-LABEL: vmfgt_vf:
+define <vscale x 4 x float> @vfmax_vf(<vscale x 4 x float> %a, float %b, iXLen %vl) {
+; NOVLOPT-LABEL: vfmax_vf:
 ; NOVLOPT:       # %bb.0:
 ; NOVLOPT-NEXT:    vsetvli a1, zero, e32, m2, ta, ma
-; NOVLOPT-NEXT:    vmfgt.vf v10, v8, fa0
-; NOVLOPT-NEXT:    vsetvli zero, a0, e8, mf2, ta, ma
-; NOVLOPT-NEXT:    vmand.mm v0, v10, v0
+; NOVLOPT-NEXT:    vfmax.vf v10, v8, fa0
+; NOVLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vfadd.vv v8, v10, v8
 ; NOVLOPT-NEXT:    ret
 ;
-; VLOPT-LABEL: vmfgt_vf:
+; VLOPT-LABEL: vfmax_vf:
 ; VLOPT:       # %bb.0:
 ; VLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
-; VLOPT-NEXT:    vmfgt.vf v10, v8, fa0
-; VLOPT-NEXT:    vmand.mm v0, v10, v0
+; VLOPT-NEXT:    vfmax.vf v10, v8, fa0
+; VLOPT-NEXT:    vfadd.vv v8, v10, v8
 ; VLOPT-NEXT:    ret
-  %1 = call <vscale x 4 x i1> @llvm.riscv.vmfgt.nxv4f32.f32(<vscale x 4 x float> %a, float %c, iXLen -1)
-  %2 = call <vscale x 4 x i1> @llvm.riscv.vmand.nxv4i1(<vscale x 4 x i1> %1, <vscale x 4 x i1> %b, iXLen %vl)
-  ret <vscale x 4 x i1> %2
+  %1 = call <vscale x 4 x float> @llvm.riscv.vfmax.nxv4f32.f32(<vscale x 4 x float> poison, <vscale x 4 x float> %a, float %b, iXLen -1)
+  %2 = call <vscale x 4 x float> @llvm.riscv.vfadd.nxv4f32.nxv4f32(<vscale x 4 x float> poison, <vscale x 4 x float> %1, <vscale x 4 x float> %a, iXLen 7, iXLen %vl)
+  ret <vscale x 4 x float> %2
 }
 
-define <vscale x 4 x i1> @vmfgt_vv(<vscale x 4 x float> %a, <vscale x 4 x i1> %b, <vscale x 4 x float> %c, iXLen %vl) {
-; NOVLOPT-LABEL: vmfgt_vv:
+define <vscale x 4 x float> @vfmin_vv(<vscale x 4 x float> %a, <vscale x 4 x float> %b, iXLen %vl) {
+; NOVLOPT-LABEL: vfmin_vv:
 ; NOVLOPT:       # %bb.0:
 ; NOVLOPT-NEXT:    vsetvli a1, zero, e32, m2, ta, ma
-; NOVLOPT-NEXT:    vmflt.vv v12, v10, v8
-; NOVLOPT-NEXT:    vsetvli zero, a0, e8, mf2, ta, ma
-; NOVLOPT-NEXT:    vmand.mm v0, v12, v0
+; NOVLOPT-NEXT:    vfmin.vv v8, v8, v10
+; NOVLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vfadd.vv v8, v8, v10
 ; NOVLOPT-NEXT:    ret
 ;
-; VLOPT-LABEL: vmfgt_vv:
+; VLOPT-LABEL: vfmin_vv:
 ; VLOPT:       # %bb.0:
 ; VLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
-; VLOPT-NEXT:    vmflt.vv v12, v10, v8
-; VLOPT-NEXT:    vmand.mm v0, v12, v0
+; VLOPT-NEXT:    vfmin.vv v8, v8, v10
+; VLOPT-NEXT:    vfadd.vv v8, v8, v10
 ; VLOPT-NEXT:    ret
-  %1 = call <vscale x 4 x i1> @llvm.riscv.vmfgt.nxv4f32.nxv4f32(<vscale x 4 x float> %a, <vscale x 4 x float> %c, iXLen -1)
-  %2 = call <vscale x 4 x i1> @llvm.riscv.vmand.nxv4i1(<vscale x 4 x i1> %1, <vscale x 4 x i1> %b, iXLen %vl)
-  ret <vscale x 4 x i1> %2
+  %1 = call <vscale x 4 x float> @llvm.riscv.vfmin.nxv4f32.nxv4f32(<vscale x 4 x float> poison, <vscale x 4 x float> %a, <vscale x 4 x float> %b, iXLen -1)
+  %2 = call <vscale x 4 x float> @llvm.riscv.vfadd.nxv4f32.nxv4f32(<vscale x 4 x float> poison, <vscale x 4 x float> %1, <vscale x 4 x float> %b, iXLen 7, iXLen %vl)
+  ret <vscale x 4 x float> %2
 }
 
-define <vscale x 4 x i32> @vmerge_vvm(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b, <vscale x 4 x i1> %c, iXLen %vl) {
-; NOVLOPT-LABEL: vmerge_vvm:
+define <vscale x 4 x float> @vfmin_vf(<vscale x 4 x float> %a, float %b, iXLen %vl) {
+; NOVLOPT-LABEL: vfmin_vf:
 ; NOVLOPT:       # %bb.0:
 ; NOVLOPT-NEXT:    vsetvli a1, zero, e32, m2, ta, ma
-; NOVLOPT-NEXT:    vmerge.vvm v8, v8, v10, v0
+; NOVLOPT-NEXT:    vfmin.vf v10, v8, fa0
 ; NOVLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
-; NOVLOPT-NEXT:    vadd.vv v8, v8, v10
+; NOVLOPT-NEXT:    vfadd.vv v8, v10, v8
 ; NOVLOPT-NEXT:    ret
 ;
-; VLOPT-LABEL: vmerge_vvm:
+; VLOPT-LABEL: vfmin_vf:
 ; VLOPT:       # %bb.0:
 ; VLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
-; VLOPT-NEXT:    vmerge.vvm v8, v8, v10, v0
-; VLOPT-NEXT:    vadd.vv v8, v8, v10
+; VLOPT-NEXT:    vfmin.vf v10, v8, fa0
+; VLOPT-NEXT:    vfadd.vv v8, v10, v8
 ; VLOPT-NEXT:    ret
-  %1 = call <vscale x 4 x i32> @llvm.riscv.vmerge.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %a, <vscale x 4 x i32> %b, <vscale x 4 x i1> %c, iXLen -1)
-  %2 = call <vscale x 4 x i32> @llvm.riscv.vadd.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %1, <vscale x 4 x i32> %b, iXLen %vl)
-  ret <vscale x 4 x i32> %2
+  %1 = call <vscale x 4 x float> @llvm.riscv.vfmin.nxv4f32.f32(<vscale x 4 x float> poison, <vscale x 4 x float> %a, float %b, iXLen -1)
+  %2 = call <vscale x 4 x float> @llvm.riscv.vfadd.nxv4f32.nxv4f32(<vscale x 4 x float> poison, <vscale x 4 x float> %1, <vscale x 4 x float> %a, iXLen 7, iXLen %vl)
+  ret <vscale x 4 x float> %2
 }
 
-define <vscale x 4 x i32> @vmerge_vxm(<vscale x 4 x i32> %a, i32 %b, <vscale x 4 x i1> %c, iXLen %vl) {
-; NOVLOPT-LABEL: vmerge_vxm:
+define <vscale x 4 x float> @vfsgnj_vv(<vscale x 4 x float> %a, <vscale x 4 x float> %b, iXLen %vl) {
+; NOVLOPT-LABEL: vfsgnj_vv:
 ; NOVLOPT:       # %bb.0:
-; NOVLOPT-NEXT:    vsetvli a2, zero, e32, m2, ta, ma
-; NOVLOPT-NEXT:    vmerge.vxm v8, v8, a0, v0
-; NOVLOPT-NEXT:    vsetvli zero, a1, e32, m2, ta, ma
-; NOVLOPT-NEXT:    vadd.vv v8, v8, v8
+; NOVLOPT-NEXT:    vsetvli a1, zero, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vfsgnj.vv v8, v8, v10
+; NOVLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vfadd.vv v8, v8, v10
 ; NOVLOPT-NEXT:    ret
 ;
-; VLOPT-LABEL: vmerge_vxm:
+; VLOPT-LABEL: vfsgnj_vv:
 ; VLOPT:       # %bb.0:
-; VLOPT-NEXT:    vsetvli zero, a1, e32, m2, ta, ma
-; VLOPT-NEXT:    vmerge.vxm v8, v8, a0, v0
-; VLOPT-NEXT:    vadd.vv v8, v8, v8
+; VLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
+; VLOPT-NEXT:    vfsgnj.vv v8, v8, v10
+; VLOPT-NEXT:    vfadd.vv v8, v8, v10
 ; VLOPT-NEXT:    ret
-  %1 = call <vscale x 4 x i32> @llvm.riscv.vmerge.nxv4i32.i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %a, i32 %b, <vscale x 4 x i1> %c, iXLen -1)
-  %2 = call <vscale x 4 x i32> @llvm.riscv.vadd.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %1, <vscale x 4 x i32> %1, iXLen %vl)
-  ret <vscale x 4 x i32> %2
+  %1 = call <vscale x 4 x float> @llvm.riscv.vfsgnj.nxv4f32.nxv4f32(<vscale x 4 x float> poison, <vscale x 4 x float> %a, <vscale x 4 x float> %b, iXLen -1)
+  %2 = call <vscale x 4 x float> @llvm.riscv.vfadd.nxv4f32.nxv4f32(<vscale x 4 x float> poison, <vscale x 4 x float> %1, <vscale x 4 x float> %b, iXLen 7, iXLen %vl)
+  ret <vscale x 4 x float> %2
 }
 
-define <vscale x 4 x i32> @vmerge_vim(<vscale x 4 x i32> %a, <vscale x 4 x i1> %c, iXLen %vl) {
-; NOVLOPT-LABEL: vmerge_vim:
+define <vscale x 4 x float> @vfsgnj_vf(<vscale x 4 x float> %a, float %b, iXLen %vl) {
+; NOVLOPT-LABEL: vfsgnj_vf:
 ; NOVLOPT:       # %bb.0:
 ; NOVLOPT-NEXT:    vsetvli a1, zero, e32, m2, ta, ma
-; NOVLOPT-NEXT:    vmerge.vim v8, v8, 9, v0
+; NOVLOPT-NEXT:    vfsgnj.vf v10, v8, fa0
 ; NOVLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
-; NOVLOPT-NEXT:    vadd.vv v8, v8, v8
+; NOVLOPT-NEXT:    vfadd.vv v8, v10, v8
 ; NOVLOPT-NEXT:    ret
 ;
-; VLOPT-LABEL: vmerge_vim:
+; VLOPT-LABEL: vfsgnj_vf:
 ; VLOPT:       # %bb.0:
 ; VLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
-; VLOPT-NEXT:    vmerge.vim v8, v8, 9, v0
-; VLOPT-NEXT:    vadd.vv v8, v8, v8
+; VLOPT-NEXT:    vfsgnj.vf v10, v8, fa0
+; VLOPT-NEXT:    vfadd.vv v8, v10, v8
 ; VLOPT-NEXT:    ret
-  %1 = call <vscale x 4 x i32> @llvm.riscv.vmerge.nxv4i32.i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %a, i32 9, <vscale x 4 x i1> %c, iXLen -1)
-  %2 = call <vscale x 4 x i32> @llvm.riscv.vadd.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %1, <vscale x 4 x i32> %1, iXLen %vl)
-  ret <vscale x 4 x i32> %2
+  %1 = call <vscale x 4 x float> @llvm.riscv.vfsgnj.nxv4f32.nxv4f32(<vscale x 4 x float> poison, <vscale x 4 x float> %a, float %b, iXLen -1)
+  %2 = call <vscale x 4 x float> @llvm.riscv.vfadd.nxv4f32.nxv4f32(<vscale x 4 x float> poison, <vscale x 4 x float> %1, <vscale x 4 x float> %a, iXLen 7, iXLen %vl)
+  ret <vscale x 4 x float> %2
 }
 
-define <vscale x 4 x i32> @vadc_vvm(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b, <vscale x 4 x i1> %c, iXLen %vl) {
-; NOVLOPT-LABEL: vadc_vvm:
+define <vscale x 4 x float> @vfsgnjn_vv(<vscale x 4 x float> %a, <vscale x 4 x float> %b, iXLen %vl) {
+; NOVLOPT-LABEL: vfsgnjn_vv:
 ; NOVLOPT:       # %bb.0:
 ; NOVLOPT-NEXT:    vsetvli a1, zero, e32, m2, ta, ma
-; NOVLOPT-NEXT:    vadc.vvm v8, v8, v10, v0
+; NOVLOPT-NEXT:    vfsgnjn.vv v8, v8, v10
 ; NOVLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
-; NOVLOPT-NEXT:    vadd.vv v8, v8, v10
+; NOVLOPT-NEXT:    vfadd.vv v8, v8, v10
 ; NOVLOPT-NEXT:    ret
 ;
-; VLOPT-LABEL: vadc_vvm:
+; VLOPT-LABEL: vfsgnjn_vv:
 ; VLOPT:       # %bb.0:
 ; VLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
-; VLOPT-NEXT:    vadc.vvm v8, v8, v10, v0
-; VLOPT-NEXT:    vadd.vv v8, v8, v10
+; VLOPT-NEXT:    vfsgnjn.vv v8, v8, v10
+; VLOPT-NEXT:    vfadd.vv v8, v8, v10
 ; VLOPT-NEXT:    ret
-  %1 = call <vscale x 4 x i32> @llvm.riscv.vadc.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %a, <vscale x 4 x i32> %b, <vscale x 4 x i1> %c, iXLen -1)
-  %2 = call <vscale x 4 x i32> @llvm.riscv.vadd.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %1, <vscale x 4 x i32> %b, iXLen %vl)
-  ret <vscale x 4 x i32> %2
+  %1 = call <vscale x 4 x float> @llvm.riscv.vfsgnjn.nxv4f32.nxv4f32(<vscale x 4 x float> poison, <vscale x 4 x float> %a, <vscale x 4 x float> %b, iXLen -1)
+  %2 = call <vscale x 4 x float> @llvm.riscv.vfadd.nxv4f32.nxv4f32(<vscale x 4 x float> poison, <vscale x 4 x float> %1, <vscale x 4 x float> %b, iXLen 7, iXLen %vl)
+  ret <vscale x 4 x float> %2
 }
 
-define <vscale x 4 x i32> @vadc_vxm(<vscale x 4 x i32> %a, i32 %b, <vscale x 4 x i1> %c, iXLen %vl) {
-; NOVLOPT-LABEL: vadc_vxm:
+define <vscale x 4 x float> @vfsgnjn_vf(<vscale x 4 x float> %a, float %b, iXLen %vl) {
+; NOVLOPT-LABEL: vfsgnjn_vf:
 ; NOVLOPT:       # %bb.0:
-; NOVLOPT-NEXT:    vsetvli a2, zero, e32, m2, ta, ma
-; NOVLOPT-NEXT:    vadc.vxm v8, v8, a0, v0
-; NOVLOPT-NEXT:    vsetvli zero, a1, e32, m2, ta, ma
-; NOVLOPT-NEXT:    vadd.vv v8, v8, v8
+; NOVLOPT-NEXT:    vsetvli a1, zero, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vfsgnjn.vf v10, v8, fa0
+; NOVLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vfadd.vv v8, v10, v8
 ; NOVLOPT-NEXT:    ret
 ;
-; VLOPT-LABEL: vadc_vxm:
+; VLOPT-LABEL: vfsgnjn_vf:
 ; VLOPT:       # %bb.0:
-; VLOPT-NEXT:    vsetvli zero, a1, e32, m2, ta, ma
-; VLOPT-NEXT:    vadc.vxm v8, v8, a0, v0
-; VLOPT-NEXT:    vadd.vv v8, v8, v8
+; VLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
+; VLOPT-NEXT:    vfsgnjn.vf v10, v8, fa0
+; VLOPT-NEXT:    vfadd.vv v8, v10, v8
 ; VLOPT-NEXT:    ret
-  %1 = call <vscale x 4 x i32> @llvm.riscv.vadc.nxv4i32.i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %a, i32 %b, <vscale x 4 x i1> %c, iXLen -1)
-  %2 = call <vscale x 4 x i32> @llvm.riscv.vadd.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %1, <vscale x 4 x i32> %1, iXLen %vl)
-  ret <vscale x 4 x i32> %2
+  %1 = call <vscale x 4 x float> @llvm.riscv.vfsgnjn.nxv4f32.nxv4f32(<vscale x 4 x float> poison, <vscale x 4 x float> %a, float %b, iXLen -1)
+  %2 = call <vscale x 4 x float> @llvm.riscv.vfadd.nxv4f32.nxv4f32(<vscale x 4 x float> poison, <vscale x 4 x float> %1, <vscale x 4 x float> %a, iXLen 7, iXLen %vl)
+  ret <vscale x 4 x float> %2
 }
 
-define <vscale x 4 x i32> @vadc_vim(<vscale x 4 x i32> %a, <vscale x 4 x i1> %c, iXLen %vl) {
-; NOVLOPT-LABEL: vadc_vim:
+define <vscale x 4 x float> @vfsgnjx_vv(<vscale x 4 x float> %a, <vscale x 4 x float> %b, iXLen %vl) {
+; NOVLOPT-LABEL: vfsgnjx_vv:
 ; NOVLOPT:       # %bb.0:
 ; NOVLOPT-NEXT:    vsetvli a1, zero, e32, m2, ta, ma
-; NOVLOPT-NEXT:    vadc.vim v8, v8, 9, v0
+; NOVLOPT-NEXT:    vfsgnjx.vv v8, v8, v10
 ; NOVLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
-; NOVLOPT-NEXT:    vadd.vv v8, v8, v8
+; NOVLOPT-NEXT:    vfadd.vv v8, v8, v10
 ; NOVLOPT-NEXT:    ret
 ;
-; VLOPT-LABEL: vadc_vim:
+; VLOPT-LABEL: vfsgnjx_vv:
 ; VLOPT:       # %bb.0:
 ; VLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
-; VLOPT-NEXT:    vadc.vim v8, v8, 9, v0
-; VLOPT-NEXT:    vadd.vv v8, v8, v8
+; VLOPT-NEXT:    vfsgnjx.vv v8, v8, v10
+; VLOPT-NEXT:    vfadd.vv v8, v8, v10
 ; VLOPT-NEXT:    ret
-  %1 = call <vscale x 4 x i32> @llvm.riscv.vadc.nxv4i32.i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %a, i32 9, <vscale x 4 x i1> %c, iXLen -1)
-  %2 = call <vscale x 4 x i32> @llvm.riscv.vadd.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %1, <vscale x 4 x i32> %1, iXLen %vl)
-  ret <vscale x 4 x i32> %2
+  %1 = call <vscale x 4 x float> @llvm.riscv.vfsgnjx.nxv4f32.nxv4f32(<vscale x 4 x float> poison, <vscale x 4 x float> %a, <vscale x 4 x float> %b, iXLen -1)
+  %2 = call <vscale x 4 x float> @llvm.riscv.vfadd.nxv4f32.nxv4f32(<vscale x 4 x float> poison, <vscale x 4 x float> %1, <vscale x 4 x float> %b, iXLen 7, iXLen %vl)
+  ret <vscale x 4 x float> %2
 }
 
-define <vscale x 4 x i32> @vaadd_vv(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b, iXLen %vl) {
-; NOVLOPT-LABEL: vaadd_vv:
+define <vscale x 4 x float> @vfsgnjx_vf(<vscale x 4 x float> %a, float %b, iXLen %vl) {
+; NOVLOPT-LABEL: vfsgnjx_vf:
 ; NOVLOPT:       # %bb.0:
-; NOVLOPT-NEXT:    csrwi vxrm, 0
 ; NOVLOPT-NEXT:    vsetvli a1, zero, e32, m2, ta, ma
-; NOVLOPT-NEXT:    vaadd.vv v8, v8, v10
+; NOVLOPT-NEXT:    vfsgnjx.vf v10, v8, fa0
 ; NOVLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
-; NOVLOPT-NEXT:    vadd.vv v8, v8, v10
+; NOVLOPT-NEXT:    vfadd.vv v8, v10, v8
 ; NOVLOPT-NEXT:    ret
 ;
-; VLOPT-LABEL: vaadd_vv:
+; VLOPT-LABEL: vfsgnjx_vf:
 ; VLOPT:       # %bb.0:
-; VLOPT-NEXT:    csrwi vxrm, 0
 ; VLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
-; VLOPT-NEXT:    vaadd.vv v8, v8, v10
-; VLOPT-NEXT:    vadd.vv v8, v8, v10
+; VLOPT-NEXT:    vfsgnjx.vf v10, v8, fa0
+; VLOPT-NEXT:    vfadd.vv v8, v10, v8
 ; VLOPT-NEXT:    ret
-  %1 = call <vscale x 4 x i32> @llvm.riscv.vaadd.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %a, <vscale x 4 x i32> %b, iXLen 0, iXLen -1)
-  %2 = call <vscale x 4 x i32> @llvm.riscv.vadd.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %1, <vscale x 4 x i32> %b, iXLen %vl)
-  ret <vscale x 4 x i32> %2
+  %1 = call <vscale x 4 x float> @llvm.riscv.vfsgnjx.nxv4f32.nxv4f32(<vscale x 4 x float> poison, <vscale x 4 x float> %a, float %b, iXLen -1)
+  %2 = call <vscale x 4 x float> @llvm.riscv.vfadd.nxv4f32.nxv4f32(<vscale x 4 x float> poison, <vscale x 4 x float> %1, <vscale x 4 x float> %a, iXLen 7, iXLen %vl)
+  ret <vscale x 4 x float> %2
 }
 
-define <vscale x 4 x i32> @vaadd_vx(<vscale x 4 x i32> %a, i32 %b, iXLen %vl) {
-; NOVLOPT-LABEL: vaadd_vx:
+define <vscale x 4 x float> @vfmacc_vv(<vscale x 4 x float> %a, <vscale x 4 x float> %b, <vscale x 4 x float> %c, iXLen %vl) {
+; NOVLOPT-LABEL: vfmacc_vv:
 ; NOVLOPT:       # %bb.0:
-; NOVLOPT-NEXT:    csrwi vxrm, 0
-; NOVLOPT-NEXT:    vsetvli a2, zero, e32, m2, ta, ma
-; NOVLOPT-NEXT:    vaadd.vx v10, v8, a0
-; NOVLOPT-NEXT:    vsetvli zero, a1, e32, m2, ta, ma
-; NOVLOPT-NEXT:    vadd.vv v8, v10, v8
+; NOVLOPT-NEXT:    vsetvli a1, zero, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vfmacc.vv v8, v12, v10
+; NOVLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vfadd.vv v8, v8, v12
 ; NOVLOPT-NEXT:    ret
 ;
-; VLOPT-LABEL: vaadd_vx:
+; VLOPT-LABEL: vfmacc_vv:
 ; VLOPT:       # %bb.0:
-; VLOPT-NEXT:    csrwi vxrm, 0
-; VLOPT-NEXT:    vsetvli zero, a1, e32, m2, ta, ma
-; VLOPT-NEXT:    vaadd.vx v10, v8, a0
-; VLOPT-NEXT:    vadd.vv v8, v10, v8
+; VLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
+; VLOPT-NEXT:    vfmacc.vv v8, v12, v10
+; VLOPT-NEXT:    vfadd.vv v8, v8, v12
 ; VLOPT-NEXT:    ret
-  %1 = call <vscale x 4 x i32> @llvm.riscv.vaadd.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %a, i32 %b, iXLen 0, iXLen -1)
-  %2 = call <vscale x 4 x i32> @llvm.riscv.vadd.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %1, <vscale x 4 x i32> %a, iXLen %vl)
-  ret <vscale x 4 x i32> %2
+  %1 = call <vscale x 4 x float> @llvm.riscv.vfmacc(<vscale x 4 x float> %a, <vscale x 4 x float> %b, <vscale x 4 x float> %c, iXLen 7, iXLen -1, iXLen 3)
+  %2 = call <vscale x 4 x float> @llvm.riscv.vfadd(<vscale x 4 x float> poison, <vscale x 4 x float> %1, <vscale x 4 x float> %c, iXLen 7, iXLen %vl)
+  ret <vscale x 4 x float> %2
 }
 
-define <vscale x 4 x i32> @vasub_vv(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b, iXLen %vl) {
-; NOVLOPT-LABEL: vasub_vv:
+define <vscale x 4 x float> @vfmacc_vf(<vscale x 4 x float> %a, float %b, <vscale x 4 x float> %c, iXLen %vl) {
+; NOVLOPT-LABEL: vfmacc_vf:
 ; NOVLOPT:       # %bb.0:
-; NOVLOPT-NEXT:    csrwi vxrm, 0
 ; NOVLOPT-NEXT:    vsetvli a1, zero, e32, m2, ta, ma
-; NOVLOPT-NEXT:    vasub.vv v8, v8, v10
+; NOVLOPT-NEXT:    vfmacc.vf v8, fa0, v10
 ; NOVLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
-; NOVLOPT-NEXT:    vadd.vv v8, v8, v10
+; NOVLOPT-NEXT:    vfadd.vv v8, v8, v10
 ; NOVLOPT-NEXT:    ret
 ;
-; VLOPT-LABEL: vasub_vv:
+; VLOPT-LABEL: vfmacc_vf:
 ; VLOPT:       # %bb.0:
-; VLOPT-NEXT:    csrwi vxrm, 0
 ; VLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
-; VLOPT-NEXT:    vasub.vv v8, v8, v10
-; VLOPT-NEXT:    vadd.vv v8, v8, v10
+; VLOPT-NEXT:    vfmacc.vf v8, fa0, v10
+; VLOPT-NEXT:    vfadd.vv v8, v8, v10
 ; VLOPT-NEXT:    ret
-  %1 = call <vscale x 4 x i32> @llvm.riscv.vasub.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %a, <vscale x 4 x i32> %b, iXLen 0, iXLen -1)
-  %2 = call <vscale x 4 x i32> @llvm.riscv.vadd.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %1, <vscale x 4 x i32> %b, iXLen %vl)
-  ret <vscale x 4 x i32> %2
+  %1 = call <vscale x 4 x float> @llvm.riscv.vfmacc(<vscale x 4 x float> %a, float %b, <vscale x 4 x float> %c, iXLen 7, iXLen -1, iXLen 3)
+  %2 = call <vscale x 4 x float> @llvm.riscv.vfadd(<vscale x 4 x float> poison, <vscale x 4 x float> %1, <vscale x 4 x float> %c, iXLen 7, iXLen %vl)
+  ret <vscale x 4 x float> %2
 }
 
-define <vscale x 4 x i32> @vasub_vx(<vscale x 4 x i32> %a, i32 %b, iXLen %vl) {
-; NOVLOPT-LABEL: vasub_vx:
+define <vscale x 4 x float> @vfnmacc_vv(<vscale x 4 x float> %a, <vscale x 4 x float> %b, <vscale x 4 x float> %c, iXLen %vl) {
+; NOVLOPT-LABEL: vfnmacc_vv:
 ; NOVLOPT:       # %bb.0:
-; NOVLOPT-NEXT:    csrwi vxrm, 0
-; NOVLOPT-NEXT:    vsetvli a2, zero, e32, m2, ta, ma
-; NOVLOPT-NEXT:    vasub.vx v10, v8, a0
-; NOVLOPT-NEXT:    vsetvli zero, a1, e32, m2, ta, ma
-; NOVLOPT-NEXT:    vadd.vv v8, v10, v8
+; NOVLOPT-NEXT:    vsetvli a1, zero, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vfnmacc.vv v8, v12, v10
+; NOVLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vfadd.vv v8, v8, v12
 ; NOVLOPT-NEXT:    ret
 ;
-; VLOPT-LABEL: vasub_vx:
+; VLOPT-LABEL: vfnmacc_vv:
 ; VLOPT:       # %bb.0:
-; VLOPT-NEXT:    csrwi vxrm, 0
-; VLOPT-NEXT:    vsetvli zero, a1, e32, m2, ta, ma
-; VLOPT-NEXT:    vasub.vx v10, v8, a0
-; VLOPT-NEXT:    vadd.vv v8, v10, v8
+; VLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
+; VLOPT-NEXT:    vfnmacc.vv v8, v12, v10
+; VLOPT-NEXT:    vfadd.vv v8, v8, v12
 ; VLOPT-NEXT:    ret
-  %1 = call <vscale x 4 x i32> @llvm.riscv.vasub.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %a, i32 %b, iXLen 0, iXLen -1)
-  %2 = call <vscale x 4 x i32> @llvm.riscv.vadd.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %1, <vscale x 4 x i32> %a, iXLen %vl)
-  ret <vscale x 4 x i32> %2
+  %1 = call <vscale x 4 x float> @llvm.riscv.vfnmacc(<vscale x 4 x float> %a, <vscale x 4 x float> %b, <vscale x 4 x float> %c, iXLen 7, iXLen -1, iXLen 3)
+  %2 = call <vscale x 4 x float> @llvm.riscv.vfadd(<vscale x 4 x float> poison, <vscale x 4 x float> %1, <vscale x 4 x float> %c, iXLen 7, iXLen %vl)
+  ret <vscale x 4 x float> %2
 }
 
-define <vscale x 4 x i32> @vaaddu_vv(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b, iXLen %vl) {
-; NOVLOPT-LABEL: vaaddu_vv:
+define <vscale x 4 x float> @vfnmacc_vf(<vscale x 4 x float> %a, float %b, <vscale x 4 x float> %c, iXLen %vl) {
+; NOVLOPT-LABEL: vfnmacc_vf:
 ; NOVLOPT:       # %bb.0:
-; NOVLOPT-NEXT:    csrwi vxrm, 0
 ; NOVLOPT-NEXT:    vsetvli a1, zero, e32, m2, ta, ma
-; NOVLOPT-NEXT:    vaaddu.vv v8, v8, v10
+; NOVLOPT-NEXT:    vfnmacc.vf v8, fa0, v10
 ; NOVLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
-; NOVLOPT-NEXT:    vadd.vv v8, v8, v10
+; NOVLOPT-NEXT:    vfadd.vv v8, v8, v10
 ; NOVLOPT-NEXT:    ret
 ;
-; VLOPT-LABEL: vaaddu_vv:
+; VLOPT-LABEL: vfnmacc_vf:
 ; VLOPT:       # %bb.0:
-; VLOPT-NEXT:    csrwi vxrm, 0
 ; VLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
-; VLOPT-NEXT:    vaaddu.vv v8, v8, v10
-; VLOPT-NEXT:    vadd.vv v8, v8, v10
+; VLOPT-NEXT:    vfnmacc.vf v8, fa0, v10
+; VLOPT-NEXT:    vfadd.vv v8, v8, v10
 ; VLOPT-NEXT:    ret
-  %1 = call <vscale x 4 x i32> @llvm.riscv.vaaddu.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %a, <vscale x 4 x i32> %b, iXLen 0, iXLen -1)
-  %2 = call <vscale x 4 x i32> @llvm.riscv.vadd.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %1, <vscale x 4 x i32> %b, iXLen %vl)
-  ret <vscale x 4 x i32> %2
+  %1 = call <vscale x 4 x float> @llvm.riscv.vfnmacc(<vscale x 4 x float> %a, float %b, <vscale x 4 x float> %c, iXLen 7, iXLen -1, iXLen 3)
+  %2 = call <vscale x 4 x float> @llvm.riscv.vfadd(<vscale x 4 x float> poison, <vscale x 4 x float> %1, <vscale x 4 x float> %c, iXLen 7, iXLen %vl)
+  ret <vscale x 4 x float> %2
 }
 
-define <vscale x 4 x i32> @vaaddu_vx(<vscale x 4 x i32> %a, i32 %b, iXLen %vl) {
-; NOVLOPT-LABEL: vaaddu_vx:
+define <vscale x 4 x float> @vfmsac_vv(<vscale x 4 x float> %a, <vscale x 4 x float> %b, <vscale x 4 x float> %c, iXLen %vl) {
+; NOVLOPT-LABEL: vfmsac_vv:
 ; NOVLOPT:       # %bb.0:
-; NOVLOPT-NEXT:    csrwi vxrm, 0
-; NOVLOPT-NEXT:    vsetvli a2, zero, e32, m2, ta, ma
-; NOVLOPT-NEXT:    vaaddu.vx v10, v8, a0
-; NOVLOPT-NEXT:    vsetvli zero, a1, e32, m2, ta, ma
-; NOVLOPT-NEXT:    vadd.vv v8, v10, v8
+; NOVLOPT-NEXT:    vsetvli a1, zero, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vfmsac.vv v8, v12, v10
+; NOVLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vfadd.vv v8, v8, v12
 ; NOVLOPT-NEXT:    ret
 ;
-; VLOPT-LABEL: vaaddu_vx:
+; VLOPT-LABEL: vfmsac_vv:
 ; VLOPT:       # %bb.0:
-; VLOPT-NEXT:    csrwi vxrm, 0
-; VLOPT-NEXT:    vsetvli zero, a1, e32, m2, ta, ma
-; VLOPT-NEXT:    vaaddu.vx v10, v8, a0
-; VLOPT-NEXT:    vadd.vv v8, v10, v8
+; VLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
+; VLOPT-NEXT:    vfmsac.vv v8, v12, v10
+; VLOPT-NEXT:    vfadd.vv v8, v8, v12
 ; VLOPT-NEXT:    ret
-  %1 = call <vscale x 4 x i32> @llvm.riscv.vaaddu.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %a, i32 %b, iXLen 0, iXLen -1)
-  %2 = call <vscale x 4 x i32> @llvm.riscv.vadd.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %1, <vscale x 4 x i32> %a, iXLen %vl)
-  ret <vscale x 4 x i32> %2
+  %1 = call <vscale x 4 x float> @llvm.riscv.vfmsac(<vscale x 4 x float> %a, <vscale x 4 x float> %b, <vscale x 4 x float> %c, iXLen 7, iXLen -1, iXLen 3)
+  %2 = call <vscale x 4 x float> @llvm.riscv.vfadd(<vscale x 4 x float> poison, <vscale x 4 x float> %1, <vscale x 4 x float> %c, iXLen 7, iXLen %vl)
+  ret <vscale x 4 x float> %2
 }
 
-define <vscale x 4 x i32> @vasubu_vv(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b, iXLen %vl) {
-; NOVLOPT-LABEL: vasubu_vv:
+define <vscale x 4 x float> @vfmsac_vf(<vscale x 4 x float> %a, float %b, <vscale x 4 x float> %c, iXLen %vl) {
+; NOVLOPT-LABEL: vfmsac_vf:
 ; NOVLOPT:       # %bb.0:
-; NOVLOPT-NEXT:    csrwi vxrm, 0
 ; NOVLOPT-NEXT:    vsetvli a1, zero, e32, m2, ta, ma
-; NOVLOPT-NEXT:    vasubu.vv v8, v8, v10
+; NOVLOPT-NEXT:    vfmsac.vf v8, fa0, v10
 ; NOVLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
-; NOVLOPT-NEXT:    vadd.vv v8, v8, v10
+; NOVLOPT-NEXT:    vfadd.vv v8, v8, v10
 ; NOVLOPT-NEXT:    ret
 ;
-; VLOPT-LABEL: vasubu_vv:
+; VLOPT-LABEL: vfmsac_vf:
 ; VLOPT:       # %bb.0:
-; VLOPT-NEXT:    csrwi vxrm, 0
 ; VLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
-; VLOPT-NEXT:    vasubu.vv v8, v8, v10
-; VLOPT-NEXT:    vadd.vv v8, v8, v10
+; VLOPT-NEXT:    vfmsac.vf v8, fa0, v10
+; VLOPT-NEXT:    vfadd.vv v8, v8, v10
 ; VLOPT-NEXT:    ret
-  %1 = call <vscale x 4 x i32> @llvm.riscv.vasubu.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %a, <vscale x 4 x i32> %b, iXLen 0, iXLen -1)
-  %2 = call <vscale x 4 x i32> @llvm.riscv.vadd.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %1, <vscale x 4 x i32> %b, iXLen %vl)
-  ret <vscale x 4 x i32> %2
+  %1 = call <vscale x 4 x float> @llvm.riscv.vfmsac(<vscale x 4 x float> %a, float %b, <vscale x 4 x float> %c, iXLen 7, iXLen -1, iXLen 3)
+  %2 = call <vscale x 4 x float> @llvm.riscv.vfadd(<vscale x 4 x float> poison, <vscale x 4 x float> %1, <vscale x 4 x float> %c, iXLen 7, iXLen %vl)
+  ret <vscale x 4 x float> %2
 }
 
-define <vscale x 4 x i32> @vasubu_vx(<vscale x 4 x i32> %a, i32 %b, iXLen %vl) {
-; NOVLOPT-LABEL: vasubu_vx:
+define <vscale x 4 x float> @vfnmsac_vv(<vscale x 4 x float> %a, <vscale x 4 x float> %b, <vscale x 4 x float> %c, iXLen %vl) {
+; NOVLOPT-LABEL: vfnmsac_vv:
 ; NOVLOPT:       # %bb.0:
-; NOVLOPT-NEXT:    csrwi vxrm, 0
-; NOVLOPT-NEXT:    vsetvli a2, zero, e32, m2, ta, ma
-; NOVLOPT-NEXT:    vasubu.vx v10, v8, a0
-; NOVLOPT-NEXT:    vsetvli zero, a1, e32, m2, ta, ma
-; NOVLOPT-NEXT:    vadd.vv v8, v10, v8
+; NOVLOPT-NEXT:    vsetvli a1, zero, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vfnmsac.vv v8, v12, v10
+; NOVLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vfadd.vv v8, v8, v12
 ; NOVLOPT-NEXT:    ret
 ;
-; VLOPT-LABEL: vasubu_vx:
+; VLOPT-LABEL: vfnmsac_vv:
 ; VLOPT:       # %bb.0:
-; VLOPT-NEXT:    csrwi vxrm, 0
-; VLOPT-NEXT:    vsetvli zero, a1, e32, m2, ta, ma
-; VLOPT-NEXT:    vasubu.vx v10, v8, a0
-; VLOPT-NEXT:    vadd.vv v8, v10, v8
+; VLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
+; VLOPT-NEXT:    vfnmsac.vv v8, v12, v10
+; VLOPT-NEXT:    vfadd.vv v8, v8, v12
 ; VLOPT-NEXT:    ret
-  %1 = call <vscale x 4 x i32> @llvm.riscv.vasubu.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %a, i32 %b, iXLen 0, iXLen -1)
-  %2 = call <vscale x 4 x i32> @llvm.riscv.vadd.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %1, <vscale x 4 x i32> %a, iXLen %vl)
-  ret <vscale x 4 x i32> %2
+  %1 = call <vscale x 4 x float> @llvm.riscv.vfnmsac(<vscale x 4 x float> %a, <vscale x 4 x float> %b, <vscale x 4 x float> %c, iXLen 7, iXLen -1, iXLen 3)
+  %2 = call <vscale x 4 x float> @llvm.riscv.vfadd(<vscale x 4 x float> poison, <vscale x 4 x float> %1, <vscale x 4 x float> %c, iXLen 7, iXLen %vl)
+  ret <vscale x 4 x float> %2
 }
 
-define <vscale x 4 x float> @vfmax_vv(<vscale x 4 x float> %a, <vscale x 4 x float> %b, iXLen %vl) {
-; NOVLOPT-LABEL: vfmax_vv:
+define <vscale x 4 x float> @vfnmsac_vf(<vscale x 4 x float> %a, float %b, <vscale x 4 x float> %c, iXLen %vl) {
+; NOVLOPT-LABEL: vfnmsac_vf:
 ; NOVLOPT:       # %bb.0:
 ; NOVLOPT-NEXT:    vsetvli a1, zero, e32, m2, ta, ma
-; NOVLOPT-NEXT:    vfmax.vv v8, v8, v10
+; NOVLOPT-NEXT:    vfnmsac.vf v8, fa0, v10
 ; NOVLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
 ; NOVLOPT-NEXT:    vfadd.vv v8, v8, v10
 ; NOVLOPT-NEXT:    ret
 ;
-; VLOPT-LABEL: vfmax_vv:
+; VLOPT-LABEL: vfnmsac_vf:
 ; VLOPT:       # %bb.0:
 ; VLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
-; VLOPT-NEXT:    vfmax.vv v8, v8, v10
+; VLOPT-NEXT:    vfnmsac.vf v8, fa0, v10
 ; VLOPT-NEXT:    vfadd.vv v8, v8, v10
 ; VLOPT-NEXT:    ret
-  %1 = call <vscale x 4 x float> @llvm.riscv.vfmax.nxv4f32.nxv4f32(<vscale x 4 x float> poison, <vscale x 4 x float> %a, <vscale x 4 x float> %b, iXLen -1)
-  %2 = call <vscale x 4 x float> @llvm.riscv.vfadd.nxv4f32.nxv4f32(<vscale x 4 x float> poison, <vscale x 4 x float> %1, <vscale x 4 x float> %b, iXLen 7, iXLen %vl)
+  %1 = call <vscale x 4 x float> @llvm.riscv.vfnmsac(<vscale x 4 x float> %a, float %b, <vscale x 4 x float> %c, iXLen 7, iXLen -1, iXLen 3)
+  %2 = call <vscale x 4 x float> @llvm.riscv.vfadd(<vscale x 4 x float> poison, <vscale x 4 x float> %1, <vscale x 4 x float> %c, iXLen 7, iXLen %vl)
   ret <vscale x 4 x float> %2
 }
 
-define <vscale x 4 x float> @vfmax_vf(<vscale x 4 x float> %a, float %b, iXLen %vl) {
-; NOVLOPT-LABEL: vfmax_vf:
+define <vscale x 4 x float> @vfmadd_vv(<vscale x 4 x float> %a, <vscale x 4 x float> %b, <vscale x 4 x float> %c, iXLen %vl) {
+; NOVLOPT-LABEL: vfmadd_vv:
 ; NOVLOPT:       # %bb.0:
 ; NOVLOPT-NEXT:    vsetvli a1, zero, e32, m2, ta, ma
-; NOVLOPT-NEXT:    vfmax.vf v10, v8, fa0
+; NOVLOPT-NEXT:    vfmadd.vv v8, v10, v12
 ; NOVLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
-; NOVLOPT-NEXT:    vfadd.vv v8, v10, v8
+; NOVLOPT-NEXT:    vfadd.vv v8, v8, v12
 ; NOVLOPT-NEXT:    ret
 ;
-; VLOPT-LABEL: vfmax_vf:
+; VLOPT-LABEL: vfmadd_vv:
 ; VLOPT:       # %bb.0:
 ; VLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
-; VLOPT-NEXT:    vfmax.vf v10, v8, fa0
-; VLOPT-NEXT:    vfadd.vv v8, v10, v8
+; VLOPT-NEXT:    vfmadd.vv v8, v10, v12
+; VLOPT-NEXT:    vfadd.vv v8, v8, v12
 ; VLOPT-NEXT:    ret
-  %1 = call <vscale x 4 x float> @llvm.riscv.vfmax.nxv4f32.f32(<vscale x 4 x float> poison, <vscale x 4 x float> %a, float %b, iXLen -1)
-  %2 = call <vscale x 4 x float> @llvm.riscv.vfadd.nxv4f32.nxv4f32(<vscale x 4 x float> poison, <vscale x 4 x float> %1, <vscale x 4 x float> %a, iXLen 7, iXLen %vl)
+  %1 = call <vscale x 4 x float> @llvm.riscv.vfmadd(<vscale x 4 x float> %a, <vscale x 4 x float> %b, <vscale x 4 x float> %c, iXLen 7, iXLen -1, iXLen 3)
+  %2 = call <vscale x 4 x float> @llvm.riscv.vfadd(<vscale x 4 x float> poison, <vscale x 4 x float> %1, <vscale x 4 x float> %c, iXLen 7, iXLen %vl)
   ret <vscale x 4 x float> %2
 }
 
-define <vscale x 4 x float> @vfmin_vv(<vscale x 4 x float> %a, <vscale x 4 x float> %b, iXLen %vl) {
-; NOVLOPT-LABEL: vfmin_vv:
+define <vscale x 4 x float> @vfmadd_vf(<vscale x 4 x float> %a, float %b, <vscale x 4 x float> %c, iXLen %vl) {
+; NOVLOPT-LABEL: vfmadd_vf:
 ; NOVLOPT:       # %bb.0:
 ; NOVLOPT-NEXT:    vsetvli a1, zero, e32, m2, ta, ma
-; NOVLOPT-NEXT:    vfmin.vv v8, v8, v10
+; NOVLOPT-NEXT:    vfmadd.vf v8, fa0, v10
 ; NOVLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
 ; NOVLOPT-NEXT:    vfadd.vv v8, v8, v10
 ; NOVLOPT-NEXT:    ret
 ;
-; VLOPT-LABEL: vfmin_vv:
+; VLOPT-LABEL: vfmadd_vf:
 ; VLOPT:       # %bb.0:
 ; VLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
-; VLOPT-NEXT:    vfmin.vv v8, v8, v10
+; VLOPT-NEXT:    vfmadd.vf v8, fa0, v10
 ; VLOPT-NEXT:    vfadd.vv v8, v8, v10
 ; VLOPT-NEXT:    ret
-  %1 = call <vscale x 4 x float> @llvm.riscv.vfmin.nxv4f32.nxv4f32(<vscale x 4 x float> poison, <vscale x 4 x float> %a, <vscale x 4 x float> %b, iXLen -1)
-  %2 = call <vscale x 4 x float> @llvm.riscv.vfadd.nxv4f32.nxv4f32(<vscale x 4 x float> poison, <vscale x 4 x float> %1, <vscale x 4 x float> %b, iXLen 7, iXLen %vl)
+  %1 = call <vscale x 4 x float> @llvm.riscv.vfmadd(<vscale x 4 x float> %a, float %b, <vscale x 4 x float> %c, iXLen 7, iXLen -1, iXLen 3)
+  %2 = call <vscale x 4 x float> @llvm.riscv.vfadd(<vscale x 4 x float> poison, <vscale x 4 x float> %1, <vscale x 4 x float> %c, iXLen 7, iXLen %vl)
   ret <vscale x 4 x float> %2
 }
 
-define <vscale x 4 x float> @vfmin_vf(<vscale x 4 x float> %a, float %b, iXLen %vl) {
-; NOVLOPT-LABEL: vfmin_vf:
+define <vscale x 4 x float> @vfnmadd_vv(<vscale x 4 x float> %a, <vscale x 4 x float> %b, <vscale x 4 x float> %c, iXLen %vl) {
+; NOVLOPT-LABEL: vfnmadd_vv:
 ; NOVLOPT:       # %bb.0:
 ; NOVLOPT-NEXT:    vsetvli a1, zero, e32, m2, ta, ma
-; NOVLOPT-NEXT:    vfmin.vf v10, v8, fa0
+; NOVLOPT-NEXT:    vfnmadd.vv v8, v10, v12
 ; NOVLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
-; NOVLOPT-NEXT:    vfadd.vv v8, v10, v8
+; NOVLOPT-NEXT:    vfadd.vv v8, v8, v12
 ; NOVLOPT-NEXT:    ret
 ;
-; VLOPT-LABEL: vfmin_vf:
+; VLOPT-LABEL: vfnmadd_vv:
 ; VLOPT:       # %bb.0:
 ; VLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
-; VLOPT-NEXT:    vfmin.vf v10, v8, fa0
-; VLOPT-NEXT:    vfadd.vv v8, v10, v8
+; VLOPT-NEXT:    vfnmadd.vv v8, v10, v12
+; VLOPT-NEXT:    vfadd.vv v8, v8, v12
 ; VLOPT-NEXT:    ret
-  %1 = call <vscale x 4 x float> @llvm.riscv.vfmin.nxv4f32.f32(<vscale x 4 x float> poison, <vscale x 4 x float> %a, float %b, iXLen -1)
-  %2 = call <vscale x 4 x float> @llvm.riscv.vfadd.nxv4f32.nxv4f32(<vscale x 4 x float> poison, <vscale x 4 x float> %1, <vscale x 4 x float> %a, iXLen 7, iXLen %vl)
+  %1 = call <vscale x 4 x float> @llvm.riscv.vfnmadd(<vscale x 4 x float> %a, <vscale x 4 x float> %b, <vscale x 4 x float> %c, iXLen 7, iXLen -1, iXLen 3)
+  %2 = call <vscale x 4 x float> @llvm.riscv.vfadd(<vscale x 4 x float> poison, <vscale x 4 x float> %1, <vscale x 4 x float> %c, iXLen 7, iXLen %vl)
   ret <vscale x 4 x float> %2
 }
 
-define <vscale x 4 x float> @vfsgnj_vv(<vscale x 4 x float> %a, <vscale x 4 x float> %b, iXLen %vl) {
-; NOVLOPT-LABEL: vfsgnj_vv:
+define <vscale x 4 x float> @vfnmadd_vf(<vscale x 4 x float> %a, float %b, <vscale x 4 x float> %c, iXLen %vl) {
+; NOVLOPT-LABEL: vfnmadd_vf:
 ; NOVLOPT:       # %bb.0:
 ; NOVLOPT-NEXT:    vsetvli a1, zero, e32, m2, ta, ma
-; NOVLOPT-NEXT:    vfsgnj.vv v8, v8, v10
+; NOVLOPT-NEXT:    vfnmadd.vf v8, fa0, v10
 ; NOVLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
 ; NOVLOPT-NEXT:    vfadd.vv v8, v8, v10
 ; NOVLOPT-NEXT:    ret
 ;
-; VLOPT-LABEL: vfsgnj_vv:
+; VLOPT-LABEL: vfnmadd_vf:
 ; VLOPT:       # %bb.0:
 ; VLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
-; VLOPT-NEXT:    vfsgnj.vv v8, v8, v10
+; VLOPT-NEXT:    vfnmadd.vf v8, fa0, v10
 ; VLOPT-NEXT:    vfadd.vv v8, v8, v10
 ; VLOPT-NEXT:    ret
-  %1 = call <vscale x 4 x float> @llvm.riscv.vfsgnj.nxv4f32.nxv4f32(<vscale x 4 x float> poison, <vscale x 4 x float> %a, <vscale x 4 x float> %b, iXLen -1)
-  %2 = call <vscale x 4 x float> @llvm.riscv.vfadd.nxv4f32.nxv4f32(<vscale x 4 x float> poison, <vscale x 4 x float> %1, <vscale x 4 x float> %b, iXLen 7, iXLen %vl)
+  %1 = call <vscale x 4 x float> @llvm.riscv.vfnmadd(<vscale x 4 x float> %a, float %b, <vscale x 4 x float> %c, iXLen 7, iXLen -1, iXLen 3)
+  %2 = call <vscale x 4 x float> @llvm.riscv.vfadd(<vscale x 4 x float> poison, <vscale x 4 x float> %1, <vscale x 4 x float> %c, iXLen 7, iXLen %vl)
   ret <vscale x 4 x float> %2
 }
 
-define <vscale x 4 x float> @vfsgnj_vf(<vscale x 4 x float> %a, float %b, iXLen %vl) {
-; NOVLOPT-LABEL: vfsgnj_vf:
+define <vscale x 4 x float> @vfmsub_vv(<vscale x 4 x float> %a, <vscale x 4 x float> %b, <vscale x 4 x float> %c, iXLen %vl) {
+; NOVLOPT-LABEL: vfmsub_vv:
 ; NOVLOPT:       # %bb.0:
 ; NOVLOPT-NEXT:    vsetvli a1, zero, e32, m2, ta, ma
-; NOVLOPT-NEXT:    vfsgnj.vf v10, v8, fa0
+; NOVLOPT-NEXT:    vfmsub.vv v8, v10, v12
 ; NOVLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
-; NOVLOPT-NEXT:    vfadd.vv v8, v10, v8
+; NOVLOPT-NEXT:    vfadd.vv v8, v8, v12
 ; NOVLOPT-NEXT:    ret
 ;
-; VLOPT-LABEL: vfsgnj_vf:
+; VLOPT-LABEL: vfmsub_vv:
 ; VLOPT:       # %bb.0:
 ; VLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
-; VLOPT-NEXT:    vfsgnj.vf v10, v8, fa0
-; VLOPT-NEXT:    vfadd.vv v8, v10, v8
+; VLOPT-NEXT:    vfmsub.vv v8, v10, v12
+; VLOPT-NEXT:    vfadd.vv v8, v8, v12
 ; VLOPT-NEXT:    ret
-  %1 = call <vscale x 4 x float> @llvm.riscv.vfsgnj.nxv4f32.nxv4f32(<vscale x 4 x float> poison, <vscale x 4 x float> %a, float %b, iXLen -1)
-  %2 = call <vscale x 4 x float> @llvm.riscv.vfadd.nxv4f32.nxv4f32(<vscale x 4 x float> poison, <vscale x 4 x float> %1, <vscale x 4 x float> %a, iXLen 7, iXLen %vl)
+  %1 = call <vscale x 4 x float> @llvm.riscv.vfmsub(<vscale x 4 x float> %a, <vscale x 4 x float> %b, <vscale x 4 x float> %c, iXLen 7, iXLen -1, iXLen 3)
+  %2 = call <vscale x 4 x float> @llvm.riscv.vfadd(<vscale x 4 x float> poison, <vscale x 4 x float> %1, <vscale x 4 x float> %c, iXLen 7, iXLen %vl)
   ret <vscale x 4 x float> %2
 }
 
-define <vscale x 4 x float> @vfsgnjn_vv(<vscale x 4 x float> %a, <vscale x 4 x float> %b, iXLen %vl) {
-; NOVLOPT-LABEL: vfsgnjn_vv:
+define <vscale x 4 x float> @vfmsub_vf(<vscale x 4 x float> %a, float %b, <vscale x 4 x float> %c, iXLen %vl) {
+; NOVLOPT-LABEL: vfmsub_vf:
 ; NOVLOPT:       # %bb.0:
 ; NOVLOPT-NEXT:    vsetvli a1, zero, e32, m2, ta, ma
-; NOVLOPT-NEXT:    vfsgnjn.vv v8, v8, v10
+; NOVLOPT-NEXT:    vfmsub.vf v8, fa0, v10
 ; NOVLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
 ; NOVLOPT-NEXT:    vfadd.vv v8, v8, v10
 ; NOVLOPT-NEXT:    ret
 ;
-; VLOPT-LABEL: vfsgnjn_vv:
+; VLOPT-LABEL: vfmsub_vf:
 ; VLOPT:       # %bb.0:
 ; VLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
-; VLOPT-NEXT:    vfsgnjn.vv v8, v8, v10
+; VLOPT-NEXT:    vfmsub.vf v8, fa0, v10
 ; VLOPT-NEXT:    vfadd.vv v8, v8, v10
 ; VLOPT-NEXT:    ret
-  %1 = call <vscale x 4 x float> @llvm.riscv.vfsgnjn.nxv4f32.nxv4f32(<vscale x 4 x float> poison, <vscale x 4 x float> %a, <vscale x 4 x float> %b, iXLen -1)
-  %2 = call <vscale x 4 x float> @llvm.riscv.vfadd.nxv4f32.nxv4f32(<vscale x 4 x float> poison, <vscale x 4 x float> %1, <vscale x 4 x float> %b, iXLen 7, iXLen %vl)
+  %1 = call <vscale x 4 x float> @llvm.riscv.vfmsub(<vscale x 4 x float> %a, float %b, <vscale x 4 x float> %c, iXLen 7, iXLen -1, iXLen 3)
+  %2 = call <vscale x 4 x float> @llvm.riscv.vfadd(<vscale x 4 x float> poison, <vscale x 4 x float> %1, <vscale x 4 x float> %c, iXLen 7, iXLen %vl)
   ret <vscale x 4 x float> %2
 }
 
-define <vscale x 4 x float> @vfsgnjn_vf(<vscale x 4 x float> %a, float %b, iXLen %vl) {
-; NOVLOPT-LABEL: vfsgnjn_vf:
+define <vscale x 4 x float> @vfnmsub_vv(<vscale x 4 x float> %a, <vscale x 4 x float> %b, <vscale x 4 x float> %c, iXLen %vl) {
+; NOVLOPT-LABEL: vfnmsub_vv:
 ; NOVLOPT:       # %bb.0:
 ; NOVLOPT-NEXT:    vsetvli a1, zero, e32, m2, ta, ma
-; NOVLOPT-NEXT:    vfsgnjn.vf v10, v8, fa0
+; NOVLOPT-NEXT:    vfnmsub.vv v8, v10, v12
 ; NOVLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
-; NOVLOPT-NEXT:    vfadd.vv v8, v10, v8
+; NOVLOPT-NEXT:    vfadd.vv v8, v8, v12
 ; NOVLOPT-NEXT:    ret
 ;
-; VLOPT-LABEL: vfsgnjn_vf:
+; VLOPT-LABEL: vfnmsub_vv:
 ; VLOPT:       # %bb.0:
 ; VLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
-; VLOPT-NEXT:    vfsgnjn.vf v10, v8, fa0
-; VLOPT-NEXT:    vfadd.vv v8, v10, v8
+; VLOPT-NEXT:    vfnmsub.vv v8, v10, v12
+; VLOPT-NEXT:    vfadd.vv v8, v8, v12
 ; VLOPT-NEXT:    ret
-  %1 = call <vscale x 4 x float> @llvm.riscv.vfsgnjn.nxv4f32.nxv4f32(<vscale x 4 x float> poison, <vscale x 4 x float> %a, float %b, iXLen -1)
-  %2 = call <vscale x 4 x float> @llvm.riscv.vfadd.nxv4f32.nxv4f32(<vscale x 4 x float> poison, <vscale x 4 x float> %1, <vscale x 4 x float> %a, iXLen 7, iXLen %vl)
+  %1 = call <vscale x 4 x float> @llvm.riscv.vfnmsub(<vscale x 4 x float> %a, <vscale x 4 x float> %b, <vscale x 4 x float> %c, iXLen 7, iXLen -1, iXLen 3)
+  %2 = call <vscale x 4 x float> @llvm.riscv.vfadd(<vscale x 4 x float> poison, <vscale x 4 x float> %1, <vscale x 4 x float> %c, iXLen 7, iXLen %vl)
   ret <vscale x 4 x float> %2
 }
 
-define <vscale x 4 x float> @vfsgnjx_vv(<vscale x 4 x float> %a, <vscale x 4 x float> %b, iXLen %vl) {
-; NOVLOPT-LABEL: vfsgnjx_vv:
+define <vscale x 4 x float> @vfnmsub_vf(<vscale x 4 x float> %a, float %b, <vscale x 4 x float> %c, iXLen %vl) {
+; NOVLOPT-LABEL: vfnmsub_vf:
 ; NOVLOPT:       # %bb.0:
 ; NOVLOPT-NEXT:    vsetvli a1, zero, e32, m2, ta, ma
-; NOVLOPT-NEXT:    vfsgnjx.vv v8, v8, v10
+; NOVLOPT-NEXT:    vfnmsub.vf v8, fa0, v10
 ; NOVLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
 ; NOVLOPT-NEXT:    vfadd.vv v8, v8, v10
 ; NOVLOPT-NEXT:    ret
 ;
-; VLOPT-LABEL: vfsgnjx_vv:
+; VLOPT-LABEL: vfnmsub_vf:
 ; VLOPT:       # %bb.0:
 ; VLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
-; VLOPT-NEXT:    vfsgnjx.vv v8, v8, v10
+; VLOPT-NEXT:    vfnmsub.vf v8, fa0, v10
 ; VLOPT-NEXT:    vfadd.vv v8, v8, v10
 ; VLOPT-NEXT:    ret
-  %1 = call <vscale x 4 x float> @llvm.riscv.vfsgnjx.nxv4f32.nxv4f32(<vscale x 4 x float> poison, <vscale x 4 x float> %a, <vscale x 4 x float> %b, iXLen -1)
-  %2 = call <vscale x 4 x float> @llvm.riscv.vfadd.nxv4f32.nxv4f32(<vscale x 4 x float> poison, <vscale x 4 x float> %1, <vscale x 4 x float> %b, iXLen 7, iXLen %vl)
+  %1 = call <vscale x 4 x float> @llvm.riscv.vfnmsub(<vscale x 4 x float> %a, float %b, <vscale x 4 x float> %c, iXLen 7, iXLen -1, iXLen 3)
+  %2 = call <vscale x 4 x float> @llvm.riscv.vfadd(<vscale x 4 x float> poison, <vscale x 4 x float> %1, <vscale x 4 x float> %c, iXLen 7, iXLen %vl)
   ret <vscale x 4 x float> %2
 }
 
-define <vscale x 4 x float> @vfsgnjx_vf(<vscale x 4 x float> %a, float %b, iXLen %vl) {
-; NOVLOPT-LABEL: vfsgnjx_vf:
+define <vscale x 4 x double> @vfwmacc_vv(<vscale x 4 x double> %a, <vscale x 4 x float> %b, <vscale x 4 x float> %c, <vscale x 4 x double> %d, iXLen %vl) {
+; NOVLOPT-LABEL: vfwmacc_vv:
 ; NOVLOPT:       # %bb.0:
-; NOVLOPT-NEXT:    vsetvli a1, zero, e32, m2, ta, ma
-; NOVLOPT-NEXT:    vfsgnjx.vf v10, v8, fa0
+; NOVLOPT-NEXT:    vsetvli a1, zero, e32, m2, tu, ma
+; NOVLOPT-NEXT:    vfwmacc.vv v8, v12, v14
+; NOVLOPT-NEXT:    vsetvli zero, a0, e64, m4, ta, ma
+; NOVLOPT-NEXT:    vfadd.vv v8, v8, v16
+; NOVLOPT-NEXT:    ret
+;
+; VLOPT-LABEL: vfwmacc_vv:
+; VLOPT:       # %bb.0:
+; VLOPT-NEXT:    vsetvli zero, a0, e32, m2, tu, ma
+; VLOPT-NEXT:    vfwmacc.vv v8, v12, v14
+; VLOPT-NEXT:    vsetvli zero, zero, e64, m4, ta, ma
+; VLOPT-NEXT:    vfadd.vv v8, v8, v16
+; VLOPT-NEXT:    ret
+  %1 = call <vscale x 4 x double> @llvm.riscv.vfwmacc(<vscale x 4 x double> %a, <vscale x 4 x float> %b, <vscale x 4 x float> %c, iXLen 7, iXLen -1, iXLen 0)
+  %2 = call <vscale x 4 x double> @llvm.riscv.vfadd(<vscale x 4 x double> poison, <vscale x 4 x double> %1, <vscale x 4 x double> %d, iXLen 7, iXLen %vl)
+  ret <vscale x 4 x double> %2
+}
+
+define <vscale x 4 x double> @vfwmacc_vf(<vscale x 4 x double> %a, float %b, <vscale x 4 x float> %c, <vscale x 4 x double> %d, iXLen %vl) {
+; NOVLOPT-LABEL: vfwmacc_vf:
+; NOVLOPT:       # %bb.0:
+; NOVLOPT-NEXT:    vsetvli a1, zero, e32, m2, tu, ma
+; NOVLOPT-NEXT:    vfwmacc.vf v8, fa0, v12
+; NOVLOPT-NEXT:    vsetvli zero, a0, e64, m4, ta, ma
+; NOVLOPT-NEXT:    vfadd.vv v8, v8, v16
+; NOVLOPT-NEXT:    ret
+;
+; VLOPT-LABEL: vfwmacc_vf:
+; VLOPT:       # %bb.0:
+; VLOPT-NEXT:    vsetvli zero, a0, e32, m2, tu, ma
+; VLOPT-NEXT:    vfwmacc.vf v8, fa0, v12
+; VLOPT-NEXT:    vsetvli zero, zero, e64, m4, ta, ma
+; VLOPT-NEXT:    vfadd.vv v8, v8, v16
+; VLOPT-NEXT:    ret
+  %1 = call <vscale x 4 x double> @llvm.riscv.vfwmacc(<vscale x 4 x double> %a, float %b, <vscale x 4 x float> %c, iXLen 7, iXLen -1, iXLen 0)
+  %2 = call <vscale x 4 x double> @llvm.riscv.vfadd(<vscale x 4 x double> poison, <vscale x 4 x double> %1, <vscale x 4 x double> %d, iXLen 7, iXLen %vl)
+  ret <vscale x 4 x double> %2
+}
+
+define <vscale x 4 x double> @vfwnmacc_vv(<vscale x 4 x double> %a, <vscale x 4 x float> %b, <vscale x 4 x float> %c, <vscale x 4 x double> %d, iXLen %vl) {
+; NOVLOPT-LABEL: vfwnmacc_vv:
+; NOVLOPT:       # %bb.0:
+; NOVLOPT-NEXT:    vsetvli a1, zero, e32, m2, tu, ma
+; NOVLOPT-NEXT:    vfwnmacc.vv v8, v12, v14
+; NOVLOPT-NEXT:    vsetvli zero, a0, e64, m4, ta, ma
+; NOVLOPT-NEXT:    vfadd.vv v8, v8, v16
+; NOVLOPT-NEXT:    ret
+;
+; VLOPT-LABEL: vfwnmacc_vv:
+; VLOPT:       # %bb.0:
+; VLOPT-NEXT:    vsetvli zero, a0, e32, m2, tu, ma
+; VLOPT-NEXT:    vfwnmacc.vv v8, v12, v14
+; VLOPT-NEXT:    vsetvli zero, zero, e64, m4, ta, ma
+; VLOPT-NEXT:    vfadd.vv v8, v8, v16
+; VLOPT-NEXT:    ret
+  %1 = call <vscale x 4 x double> @llvm.riscv.vfwnmacc(<vscale x 4 x double> %a, <vscale x 4 x float> %b, <vscale x 4 x float> %c, iXLen 7, iXLen -1, iXLen 0)
+  %2 = call <vscale x 4 x double> @llvm.riscv.vfadd(<vscale x 4 x double> poison, <vscale x 4 x double> %1, <vscale x 4 x double> %d, iXLen 7, iXLen %vl)
+  ret <vscale x 4 x double> %2
+}
+
+define <vscale x 4 x double> @vfwnmacc_vf(<vscale x 4 x double> %a, float %b, <vscale x 4 x float> %c, <vscale x 4 x double> %d, iXLen %vl) {
+; NOVLOPT-LABEL: vfwnmacc_vf:
+; NOVLOPT:       # %bb.0:
+; NOVLOPT-NEXT:    vsetvli a1, zero, e32, m2, tu, ma
+; NOVLOPT-NEXT:    vfwnmacc.vf v8, fa0, v12
+; NOVLOPT-NEXT:    vsetvli zero, a0, e64, m4, ta, ma
+; NOVLOPT-NEXT:    vfadd.vv v8, v8, v16
+; NOVLOPT-NEXT:    ret
+;
+; VLOPT-LABEL: vfwnmacc_vf:
+; VLOPT:       # %bb.0:
+; VLOPT-NEXT:    vsetvli zero, a0, e32, m2, tu, ma
+; VLOPT-NEXT:    vfwnmacc.vf v8, fa0, v12
+; VLOPT-NEXT:    vsetvli zero, zero, e64, m4, ta, ma
+; VLOPT-NEXT:    vfadd.vv v8, v8, v16
+; VLOPT-NEXT:    ret
+  %1 = call <vscale x 4 x double> @llvm.riscv.vfwnmacc(<vscale x 4 x double> %a, float %b, <vscale x 4 x float> %c, iXLen 7, iXLen -1, iXLen 0)
+  %2 = call <vscale x 4 x double> @llvm.riscv.vfadd(<vscale x 4 x double> poison, <vscale x 4 x double> %1, <vscale x 4 x double> %d, iXLen 7, iXLen %vl)
+  ret <vscale x 4 x double> %2
+}
+
+define <vscale x 4 x double> @vfwmsac_vv(<vscale x 4 x double> %a, <vscale x 4 x float> %b, <vscale x 4 x float> %c, <vscale x 4 x double> %d, iXLen %vl) {
+; NOVLOPT-LABEL: vfwmsac_vv:
+; NOVLOPT:       # %bb.0:
+; NOVLOPT-NEXT:    vsetvli a1, zero, e32, m2, tu, ma
+; NOVLOPT-NEXT:    vfwmsac.vv v8, v12, v14
+; NOVLOPT-NEXT:    vsetvli zero, a0, e64, m4, ta, ma
+; NOVLOPT-NEXT:    vfadd.vv v8, v8, v16
+; NOVLOPT-NEXT:    ret
+;
+; VLOPT-LABEL: vfwmsac_vv:
+; VLOPT:       # %bb.0:
+; VLOPT-NEXT:    vsetvli zero, a0, e32, m2, tu, ma
+; VLOPT-NEXT:    vfwmsac.vv v8, v12, v14
+; VLOPT-NEXT:    vsetvli zero, zero, e64, m4, ta, ma
+; VLOPT-NEXT:    vfadd.vv v8, v8, v16
+; VLOPT-NEXT:    ret
+  %1 = call <vscale x 4 x double> @llvm.riscv.vfwmsac(<vscale x 4 x double> %a, <vscale x 4 x float> %b, <vscale x 4 x float> %c, iXLen 7, iXLen -1, iXLen 0)
+  %2 = call <vscale x 4 x double> @llvm.riscv.vfadd(<vscale x 4 x double> poison, <vscale x 4 x double> %1, <vscale x 4 x double> %d, iXLen 7, iXLen %vl)
+  ret <vscale x 4 x double> %2
+}
+
+define <vscale x 4 x double> @vfwmsac_vf(<vscale x 4 x double> %a, float %b, <vscale x 4 x float> %c, <vscale x 4 x double> %d, iXLen %vl) {
+; NOVLOPT-LABEL: vfwmsac_vf:
+; NOVLOPT:       # %bb.0:
+; NOVLOPT-NEXT:    vsetvli a1, zero, e32, m2, tu, ma
+; NOVLOPT-NEXT:    vfwmsac.vf v8, fa0, v12
+; NOVLOPT-NEXT:    vsetvli zero, a0, e64, m4, ta, ma
+; NOVLOPT-NEXT:    vfadd.vv v8, v8, v16
+; NOVLOPT-NEXT:    ret
+;
+; VLOPT-LABEL: vfwmsac_vf:
+; VLOPT:       # %bb.0:
+; VLOPT-NEXT:    vsetvli zero, a0, e32, m2, tu, ma
+; VLOPT-NEXT:    vfwmsac.vf v8, fa0, v12
+; VLOPT-NEXT:    vsetvli zero, zero, e64, m4, ta, ma
+; VLOPT-NEXT:    vfadd.vv v8, v8, v16
+; VLOPT-NEXT:    ret
+  %1 = call <vscale x 4 x double> @llvm.riscv.vfwmsac(<vscale x 4 x double> %a, float %b, <vscale x 4 x float> %c, iXLen 7, iXLen -1, iXLen 0)
+  %2 = call <vscale x 4 x double> @llvm.riscv.vfadd(<vscale x 4 x double> poison, <vscale x 4 x double> %1, <vscale x 4 x double> %d, iXLen 7, iXLen %vl)
+  ret <vscale x 4 x double> %2
+}
+
+define <vscale x 4 x double> @vfwnmsac_vv(<vscale x 4 x double> %a, <vscale x 4 x float> %b, <vscale x 4 x float> %c, <vscale x 4 x double> %d, iXLen %vl) {
+; NOVLOPT-LABEL: vfwnmsac_vv:
+; NOVLOPT:       # %bb.0:
+; NOVLOPT-NEXT:    vsetvli a1, zero, e32, m2, tu, ma
+; NOVLOPT-NEXT:    vfwnmsac.vv v8, v12, v14
+; NOVLOPT-NEXT:    vsetvli zero, a0, e64, m4, ta, ma
+; NOVLOPT-NEXT:    vfadd.vv v8, v8, v16
+; NOVLOPT-NEXT:    ret
+;
+; VLOPT-LABEL: vfwnmsac_vv:
+; VLOPT:       # %bb.0:
+; VLOPT-NEXT:    vsetvli zero, a0, e32, m2, tu, ma
+; VLOPT-NEXT:    vfwnmsac.vv v8, v12, v14
+; VLOPT-NEXT:    vsetvli zero, zero, e64, m4, ta, ma
+; VLOPT-NEXT:    vfadd.vv v8, v8, v16
+; VLOPT-NEXT:    ret
+  %1 = call <vscale x 4 x double> @llvm.riscv.vfwnmsac(<vscale x 4 x double> %a, <vscale x 4 x float> %b, <vscale x 4 x float> %c, iXLen 7, iXLen -1, iXLen 0)
+  %2 = call <vscale x 4 x double> @llvm.riscv.vfadd(<vscale x 4 x double> poison, <vscale x 4 x double> %1, <vscale x 4 x double> %d, iXLen 7, iXLen %vl)
+  ret <vscale x 4 x double> %2
+}
+
+define <vscale x 4 x double> @vfwnmsac_vf(<vscale x 4 x double> %a, float %b, <vscale x 4 x float> %c, <vscale x 4 x double> %d, iXLen %vl) {
+; NOVLOPT-LABEL: vfwnmsac_vf:
+; NOVLOPT:       # %bb.0:
+; NOVLOPT-NEXT:    vsetvli a1, zero, e32, m2, tu, ma
+; NOVLOPT-NEXT:    vfwnmsac.vf v8, fa0, v12
+; NOVLOPT-NEXT:    vsetvli zero, a0, e64, m4, ta, ma
+; NOVLOPT-NEXT:    vfadd.vv v8, v8, v16
+; NOVLOPT-NEXT:    ret
+;
+; VLOPT-LABEL: vfwnmsac_vf:
+; VLOPT:       # %bb.0:
+; VLOPT-NEXT:    vsetvli zero, a0, e32, m2, tu, ma
+; VLOPT-NEXT:    vfwnmsac.vf v8, fa0, v12
+; VLOPT-NEXT:    vsetvli zero, zero, e64, m4, ta, ma
+; VLOPT-NEXT:    vfadd.vv v8, v8, v16
+; VLOPT-NEXT:    ret
+  %1 = call <vscale x 4 x double> @llvm.riscv.vfwnmsac(<vscale x 4 x double> %a, float %b, <vscale x 4 x float> %c, iXLen 7, iXLen -1, iXLen 0)
+  %2 = call <vscale x 4 x double> @llvm.riscv.vfadd(<vscale x 4 x double> poison, <vscale x 4 x double> %1, <vscale x 4 x double> %d, iXLen 7, iXLen %vl)
+  ret <vscale x 4 x double> %2
+}
+
+define <vscale x 4 x float> @vfwmaccbf16_vv(<vscale x 4 x float> %a, <vscale x 4 x bfloat> %b, <vscale x 4 x bfloat> %c, <vscale x 4 x float> %d, iXLen %vl) {
+; NOVLOPT-LABEL: vfwmaccbf16_vv:
+; NOVLOPT:       # %bb.0:
+; NOVLOPT-NEXT:    vsetvli a1, zero, e16, m1, tu, ma
+; NOVLOPT-NEXT:    vfwmaccbf16.vv v8, v10, v11
 ; NOVLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
-; NOVLOPT-NEXT:    vfadd.vv v8, v10, v8
+; NOVLOPT-NEXT:    vfadd.vv v8, v8, v12
 ; NOVLOPT-NEXT:    ret
 ;
-; VLOPT-LABEL: vfsgnjx_vf:
+; VLOPT-LABEL: vfwmaccbf16_vv:
 ; VLOPT:       # %bb.0:
-; VLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
-; VLOPT-NEXT:    vfsgnjx.vf v10, v8, fa0
-; VLOPT-NEXT:    vfadd.vv v8, v10, v8
+; VLOPT-NEXT:    vsetvli zero, a0, e16, m1, tu, ma
+; VLOPT-NEXT:    vfwmaccbf16.vv v8, v10, v11
+; VLOPT-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
+; VLOPT-NEXT:    vfadd.vv v8, v8, v12
 ; VLOPT-NEXT:    ret
-  %1 = call <vscale x 4 x float> @llvm.riscv.vfsgnjx.nxv4f32.nxv4f32(<vscale x 4 x float> poison, <vscale x 4 x float> %a, float %b, iXLen -1)
-  %2 = call <vscale x 4 x float> @llvm.riscv.vfadd.nxv4f32.nxv4f32(<vscale x 4 x float> poison, <vscale x 4 x float> %1, <vscale x 4 x float> %a, iXLen 7, iXLen %vl)
+  %1 = call <vscale x 4 x float> @llvm.riscv.vfwmaccbf16(<vscale x 4 x float> %a, <vscale x 4 x bfloat> %b, <vscale x 4 x bfloat> %c, iXLen 7, iXLen -1, iXLen 0)
+  %2 = call <vscale x 4 x float> @llvm.riscv.vfadd(<vscale x 4 x float> poison, <vscale x 4 x float> %1, <vscale x 4 x float> %d, iXLen 7, iXLen %vl)
+  ret <vscale x 4 x float> %2
+}
+
+define <vscale x 4 x float> @vfwmaccbf16_vf(<vscale x 4 x float> %a, bfloat %b, <vscale x 4 x bfloat> %c, <vscale x 4 x float> %d, iXLen %vl) {
+; NOVLOPT-LABEL: vfwmaccbf16_vf:
+; NOVLOPT:       # %bb.0:
+; NOVLOPT-NEXT:    vsetvli a1, zero, e16, m1, tu, ma
+; NOVLOPT-NEXT:    vfwmaccbf16.vf v8, fa0, v10
+; NOVLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vfadd.vv v8, v8, v12
+; NOVLOPT-NEXT:    ret
+;
+; VLOPT-LABEL: vfwmaccbf16_vf:
+; VLOPT:       # %bb.0:
+; VLOPT-NEXT:    vsetvli zero, a0, e16, m1, tu, ma
+; VLOPT-NEXT:    vfwmaccbf16.vf v8, fa0, v10
+; VLOPT-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
+; VLOPT-NEXT:    vfadd.vv v8, v8, v12
+; VLOPT-NEXT:    ret
+  %1 = call <vscale x 4 x float> @llvm.riscv.vfwmaccbf16(<vscale x 4 x float> %a, bfloat %b, <vscale x 4 x bfloat> %c, iXLen 7, iXLen -1, iXLen 0)
+  %2 = call <vscale x 4 x float> @llvm.riscv.vfadd(<vscale x 4 x float> poison, <vscale x 4 x float> %1, <vscale x 4 x float> %d, iXLen 7, iXLen %vl)
   ret <vscale x 4 x float> %2
 }
diff --git a/llvm/test/CodeGen/RISCV/rvv/vl-opt-op-info.mir b/llvm/test/CodeGen/RISCV/rvv/vl-opt-op-info.mir
index edcd32c4098bc..d2906c4613295 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vl-opt-op-info.mir
+++ b/llvm/test/CodeGen/RISCV/rvv/vl-opt-op-info.mir
@@ -93,6 +93,42 @@ body: |
     %y:vr = PseudoVADD_VV_MF2 $noreg, %x, $noreg, 1, 4 /* e8 */, 0
 ...
 ---
+name: vwop_vv_vd_passthru_use
+body: |
+  bb.0:
+    ; CHECK-LABEL: name: vwop_vv_vd_passthru_use
+    ; CHECK: %x:vr = PseudoVADD_VV_M1 $noreg, $noreg, $noreg, 1, 4 /* e16 */, 0 /* tu, mu */
+    ; CHECK-NEXT: early-clobber %y:vr = PseudoVWADD_VV_MF2 %x, $noreg, $noreg, 1, 3 /* e8 */, 0 /* tu, mu */
+    ; CHECK-NEXT: %z:vr = PseudoVADD_VV_M1 $noreg, %y, $noreg, 1, 4 /* e16 */, 0 /* tu, mu */
+    %x:vr = PseudoVADD_VV_M1 $noreg, $noreg, $noreg, -1, 4 /* e16 */, 0
+    %y:vr = PseudoVWADD_VV_MF2 %x, $noreg, $noreg, 1, 3 /* e8 */, 0
+    %z:vr = PseudoVADD_VV_M1 $noreg, %y, $noreg, 1, 4 /* e16 */, 0
+...
+---
+name: vwop_vv_vd_passthru_use_incompatible_eew
+body: |
+  bb.0:
+    ; CHECK-LABEL: name: vwop_vv_vd_passthru_use_incompatible_eew
+    ; CHECK: %x:vr = PseudoVADD_VV_M1 $noreg, $noreg, $noreg, -1, 4 /* e16 */, 0 /* tu, mu */
+    ; CHECK-NEXT: early-clobber %y:vr = PseudoVWADD_VV_MF2 %x, $noreg, $noreg, 1, 4 /* e16 */, 0 /* tu, mu */
+    ; CHECK-NEXT: %z:vr = PseudoVADD_VV_M1 $noreg, %y, $noreg, 1, 4 /* e16 */, 0 /* tu, mu */
+    %x:vr = PseudoVADD_VV_M1 $noreg, $noreg, $noreg, -1, 4 /* e16 */, 0
+    %y:vr = PseudoVWADD_VV_MF2 %x, $noreg, $noreg, 1, 4 /* e16 */, 0
+    %z:vr = PseudoVADD_VV_M1 $noreg, %y, $noreg, 1, 4 /* e16 */, 0
+...
+---
+name: vwop_vv_vd_passthru_use_incompatible_emul
+body: |
+  bb.0:
+    ; CHECK-LABEL: name: vwop_vv_vd_passthru_use_incompatible_emul
+    ; CHECK: %x:vr = PseudoVADD_VV_M1 $noreg, $noreg, $noreg, -1, 4 /* e16 */, 0 /* tu, mu */
+    ; CHECK-NEXT: early-clobber %y:vr = PseudoVWADD_VV_MF4 %x, $noreg, $noreg, 1, 3 /* e8 */, 0 /* tu, mu */
+    ; CHECK-NEXT: %z:vr = PseudoVADD_VV_MF2 $noreg, %y, $noreg, 1, 4 /* e16 */, 0 /* tu, mu */
+    %x:vr = PseudoVADD_VV_M1 $noreg, $noreg, $noreg, -1, 4 /* e16 */, 0
+    %y:vr = PseudoVWADD_VV_MF4 %x, $noreg, $noreg, 1, 3 /* e8 */, 0
+    %z:vr = PseudoVADD_VV_MF2 $noreg, %y, $noreg, 1, 4 /* e16 */, 0
+...
+---
 name: vwop_vv_vs2
 body: |
   bb.0:
@@ -183,6 +219,42 @@ body: |
     %y:vr = PseudoVADD_VV_MF2 $noreg, %x, $noreg, 1, 4 /* e8 */, 0
 ...
 ---
+name: vwop_wv_vd_passthru_use
+body: |
+  bb.0:
+    ; CHECK-LABEL: name: vwop_wv_vd_passthru_use
+    ; CHECK: %x:vr = PseudoVADD_VV_M1 $noreg, $noreg, $noreg, 1, 4 /* e16 */, 0 /* tu, mu */
+    ; CHECK-NEXT: early-clobber %y:vr = PseudoVWADD_WV_MF2 %x, $noreg, $noreg, 1, 3 /* e8 */, 0 /* tu, mu */
+    ; CHECK-NEXT: %z:vr = PseudoVADD_VV_M1 $noreg, %y, $noreg, 1, 4 /* e16 */, 0 /* tu, mu */
+    %x:vr = PseudoVADD_VV_M1 $noreg, $noreg, $noreg, -1, 4 /* e16 */, 0
+    %y:vr = PseudoVWADD_WV_MF2 %x, $noreg, $noreg, 1, 3 /* e8 */, 0
+    %z:vr = PseudoVADD_VV_M1 $noreg, %y, $noreg, 1, 4 /* e16 */, 0
+...
+---
+name: vwop_wv_vd_passthru_use_incompatible_eew
+body: |
+  bb.0:
+    ; CHECK-LABEL: name: vwop_wv_vd_passthru_use_incompatible_eew
+    ; CHECK: %x:vr = PseudoVADD_VV_M1 $noreg, $noreg, $noreg, -1, 4 /* e16 */, 0 /* tu, mu */
+    ; CHECK-NEXT: early-clobber %y:vr = PseudoVWADD_WV_MF2 %x, $noreg, $noreg, 1, 4 /* e16 */, 0 /* tu, mu */
+    ; CHECK-NEXT: %z:vr = PseudoVADD_VV_M1 $noreg, %y, $noreg, 1, 4 /* e16 */, 0 /* tu, mu */
+    %x:vr = PseudoVADD_VV_M1 $noreg, $noreg, $noreg, -1, 4 /* e16 */, 0
+    %y:vr = PseudoVWADD_WV_MF2 %x, $noreg, $noreg, 1, 4 /* e16 */, 0
+    %z:vr = PseudoVADD_VV_M1 $noreg, %y, $noreg, 1, 4 /* e16 */, 0
+...
+---
+name: vwop_wv_vd_passthru_use_incompatible_emul
+body: |
+  bb.0:
+    ; CHECK-LABEL: name: vwop_wv_vd_passthru_use_incompatible_emul
+    ; CHECK: %x:vr = PseudoVADD_VV_M1 $noreg, $noreg, $noreg, -1, 4 /* e16 */, 0 /* tu, mu */
+    ; CHECK-NEXT: early-clobber %y:vr = PseudoVWADD_WV_MF4 %x, $noreg, $noreg, 1, 3 /* e8 */, 0 /* tu, mu */
+    ; CHECK-NEXT: %z:vr = PseudoVADD_VV_MF2 $noreg, %y, $noreg, 1, 4 /* e16 */, 0 /* tu, mu */
+    %x:vr = PseudoVADD_VV_M1 $noreg, $noreg, $noreg, -1, 4 /* e16 */, 0
+    %y:vr = PseudoVWADD_WV_MF4 %x, $noreg, $noreg, 1, 3 /* e8 */, 0
+    %z:vr = PseudoVADD_VV_MF2 $noreg, %y, $noreg, 1, 4 /* e16 */, 0
+...
+---
 name: vwop_wv_vs2
 body: |
   bb.0:
@@ -483,6 +555,42 @@ body: |
     %y:vr = PseudoVADD_VV_M1 $noreg, %x, $noreg, 1, 3 /* e8 */, 0
 ...
 ---
+name: vnop_wv_vd_passthru_use
+body: |
+  bb.0:
+    ; CHECK-LABEL: name: vnop_wv_vd_passthru_use
+    ; CHECK: %x:vr = PseudoVADD_VV_M1 $noreg, $noreg, $noreg, 1, 3 /* e8 */, 0 /* tu, mu */
+    ; CHECK-NEXT: early-clobber %y:vr = PseudoVNSRL_WV_M1 %x, $noreg, $noreg, 1, 3 /* e8 */, 0 /* tu, mu */
+    ; CHECK-NEXT: %z:vr = PseudoVADD_VV_M1 $noreg, %y, $noreg, 1, 3 /* e8 */, 0 /* tu, mu */
+    %x:vr = PseudoVADD_VV_M1 $noreg, $noreg, $noreg, -1, 3 /* e8 */, 0
+    %y:vr = PseudoVNSRL_WV_M1 %x, $noreg, $noreg, 1, 3 /* e8 */, 0
+    %z:vr = PseudoVADD_VV_M1 $noreg, %y, $noreg, 1, 3 /* e8 */, 0
+...
+---
+name: vnop_wv_vd_passthru_use_incompatible_eew
+body: |
+  bb.0:
+    ; CHECK-LABEL: name: vnop_wv_vd_passthru_use_incompatible_eew
+    ; CHECK: %x:vr = PseudoVADD_VV_M1 $noreg, $noreg, $noreg, -1, 3 /* e8 */, 0 /* tu, mu */
+    ; CHECK-NEXT: early-clobber %y:vr = PseudoVNSRL_WV_M1 %x, $noreg, $noreg, 1, 4 /* e16 */, 0 /* tu, mu */
+    ; CHECK-NEXT: %z:vr = PseudoVADD_VV_M1 $noreg, %y, $noreg, 1, 4 /* e16 */, 0 /* tu, mu */
+    %x:vr = PseudoVADD_VV_M1 $noreg, $noreg, $noreg, -1, 3 /* e8 */, 0
+    %y:vr = PseudoVNSRL_WV_M1 %x, $noreg, $noreg, 1, 4 /* e16 */, 0
+    %z:vr = PseudoVADD_VV_M1 $noreg, %y, $noreg, 1, 4 /* e16 */, 0
+...
+---
+name: vnop_wv_vd_passthru_use_unsupported_emul
+body: |
+  bb.0:
+    ; CHECK-LABEL: name: vnop_wv_vd_passthru_use_unsupported_emul
+    ; CHECK: %x:vr = PseudoVADD_VV_M1 $noreg, $noreg, $noreg, -1, 3 /* e8 */, 0 /* tu, mu */
+    ; CHECK-NEXT: %y:vr = PseudoVNSRL_WV_MF2 %x, $noreg, $noreg, 1, 3 /* e8 */, 0 /* tu, mu */
+    ; CHECK-NEXT: %z:vr = PseudoVADD_VV_MF2 $noreg, %y, $noreg, 1, 3 /* e8 */, 0 /* tu, mu */
+    %x:vr = PseudoVADD_VV_M1 $noreg, $noreg, $noreg, -1, 3 /* e8 */, 0
+    %y:vr = PseudoVNSRL_WV_MF2 %x, $noreg, $noreg, 1, 3 /* e8 */, 0
+    %z:vr = PseudoVADD_VV_MF2 $noreg, %y, $noreg, 1, 3 /* e8 */, 0
+...
+---
 name: vnop_wv_vs2
 body: |
   bb.0:
@@ -648,9 +756,9 @@ body: |
   bb.0:
     ; CHECK-LABEL: name: vleN_v_incompatible_emul
     ; CHECK: %x:vr = PseudoVLE8_V_M1 $noreg, $noreg, -1, 3 /* e8 */, 0 /* tu, mu */
-    ; CHECK-NEXT: %x:vr = PseudoVADD_VV_MF2 $noreg, %x, $noreg, 1, 3 /* e8 */, 0 /* tu, mu */
+    ; CHECK-NEXT: %y:vr = PseudoVADD_VV_MF2 $noreg, %x, $noreg, 1, 3 /* e8 */, 0 /* tu, mu */
     %x:vr = PseudoVLE8_V_M1 $noreg, $noreg, -1, 3 /* e8 */, 0
-    %x:vr = PseudoVADD_VV_MF2 $noreg, %x, $noreg, 1, 3 /* e8 */, 0
+    %y:vr = PseudoVADD_VV_MF2 $noreg, %x, $noreg, 1, 3 /* e8 */, 0
 ...
 ---
 name: vlm_v
@@ -758,10 +866,8 @@ body: |
   bb.0:
     ; CHECK-LABEL: name: vsuxeiN_v_idx_incompatible_eew
     ; CHECK: %x:vr = PseudoVADD_VV_M1 $noreg, $noreg, $noreg, -1, 4 /* e16 */, 0 /* tu, mu */
-    ; CHECK-NEXT: %y:vr = PseudoVADD_VV_M1 $noreg, $noreg, $noreg, -1, 3 /* e8 */, 0 /* tu, mu */
     ; CHECK-NEXT: PseudoVSUXEI8_V_M1_M1 $noreg, $noreg, %x, 1, 3 /* e8 */
     %x:vr = PseudoVADD_VV_M1 $noreg, $noreg, $noreg, -1, 4 /* e16 */, 0
-    %y:vr = PseudoVADD_VV_M1 $noreg, $noreg, $noreg, -1, 3 /* e8 */, 0
     PseudoVSUXEI8_V_M1_M1 $noreg, $noreg, %x, 1, 3 /* e8 */
 ...
 ---
@@ -1005,6 +1111,42 @@ body: |
     %y:vr = PseudoVMSEQ_VV_MF2 $noreg, %x, 1, 3 /* e8 */
 ...
 ---
+name: vmop_vv_passthru_use
+body: |
+  bb.0:
+    ; CHECK-LABEL: name: vmop_vv_passthru_use
+    ; CHECK: %x:vrnov0 = PseudoVMAND_MM_B8 $noreg, $noreg, 1, 0 /* e8 */
+    ; CHECK-NEXT: %y:vrnov0 = PseudoVMSEQ_VV_M1_MASK %x, $noreg, $noreg, $noreg, 1, 3 /* e8 */
+    ; CHECK-NEXT: %z:vr = PseudoVMAND_MM_B8 %y, $noreg, 1, 0 /* e8 */
+    %x:vrnov0 = PseudoVMAND_MM_B8 $noreg, $noreg, -1, 0 /* e1 */
+    %y:vrnov0 = PseudoVMSEQ_VV_M1_MASK %x, $noreg, $noreg, $noreg, 1, 3 /* e8 */
+    %z:vr = PseudoVMAND_MM_B8 %y, $noreg, 1, 0 /* e1 */
+...
+---
+name: vmop_vv_passthru_use_incompatible_eew
+body: |
+  bb.0:
+    ; CHECK-LABEL: name: vmop_vv_passthru_use_incompatible_eew
+    ; CHECK: %x:vrnov0 = PseudoVADD_VV_M1 $noreg, $noreg, $noreg, -1, 3 /* e8 */, 0 /* tu, mu */
+    ; CHECK-NEXT: %y:vrnov0 = PseudoVMSEQ_VV_M1_MASK %x, $noreg, $noreg, $noreg, 1, 3 /* e8 */
+    ; CHECK-NEXT: %z:vr = PseudoVMAND_MM_B8 %y, $noreg, 1, 0 /* e8 */
+    %x:vrnov0 = PseudoVADD_VV_M1 $noreg, $noreg, $noreg, -1, 3 /* e8 */, 0
+    %y:vrnov0 = PseudoVMSEQ_VV_M1_MASK %x, $noreg, $noreg, $noreg, 1, 3 /* e8 */
+    %z:vr = PseudoVMAND_MM_B8 %y, $noreg, 1, 0 /* e1 */
+...
+---
+name: vmop_vv_passthru_use_incompatible_emul
+body: |
+  bb.0:
+    ; CHECK-LABEL: name: vmop_vv_passthru_use_incompatible_emul
+    ; CHECK: %x:vrnov0 = PseudoVMAND_MM_B16 $noreg, $noreg, -1, 0 /* e8 */
+    ; CHECK-NEXT: %y:vrnov0 = PseudoVMSEQ_VV_M1_MASK %x, $noreg, $noreg, $noreg, 1, 3 /* e8 */
+    ; CHECK-NEXT: %z:vr = PseudoVMAND_MM_B8 %y, $noreg, 1, 0 /* e8 */
+    %x:vrnov0 = PseudoVMAND_MM_B16 $noreg, $noreg, -1, 0 /* e1 */
+    %y:vrnov0 = PseudoVMSEQ_VV_M1_MASK %x, $noreg, $noreg, $noreg, 1, 3 /* e8 */
+    %z:vr = PseudoVMAND_MM_B8 %y, $noreg, 1, 0 /* e1 */
+...
+---
 name: vmerge_vim
 body: |
   bb.0:
@@ -1215,6 +1357,42 @@ body: |
     %y:vr = PseudoVADD_VV_MF2 $noreg, %x, $noreg, 1, 3 /* e8 */, 0
 ...
 ---
+name: viota_m_dest_passthru_use
+body: |
+  bb.0:
+    ; CHECK-LABEL: name: viota_m_dest_passthru_use
+    ; CHECK: %x:vr = PseudoVADD_VV_M1 $noreg, $noreg, $noreg, 1, 3 /* e8 */, 0 /* tu, mu */
+    ; CHECK-NEXT: early-clobber %y:vr = PseudoVIOTA_M_M1 %x, $noreg, 1, 3 /* e8 */, 0 /* tu, mu */
+    ; CHECK-NEXT: %z:vr = PseudoVADD_VV_M1 $noreg, %y, $noreg, 1, 3 /* e8 */, 0 /* tu, mu */
+    %x:vr = PseudoVADD_VV_M1 $noreg, $noreg, $noreg, -1, 3 /* e8 */, 0
+    %y:vr = PseudoVIOTA_M_M1 %x, $noreg, 1, 3 /* e8 */, 0
+    %z:vr = PseudoVADD_VV_M1 $noreg, %y, $noreg, 1, 3 /* e8 */, 0
+...
+---
+name: viota_m_dest_passthru_use_incompatible_eew
+body: |
+  bb.0:
+    ; CHECK-LABEL: name: viota_m_dest_passthru_use_incompatible_eew
+    ; CHECK: %x:vr = PseudoVADD_VV_M1 $noreg, $noreg, $noreg, -1, 3 /* e8 */, 0 /* tu, mu */
+    ; CHECK-NEXT: early-clobber %y:vr = PseudoVIOTA_M_M1 %x, $noreg, 1, 4 /* e16 */, 0 /* tu, mu */
+    ; CHECK-NEXT: %z:vr = PseudoVADD_VV_M1 $noreg, %y, $noreg, 1, 4 /* e16 */, 0 /* tu, mu */
+    %x:vr = PseudoVADD_VV_M1 $noreg, $noreg, $noreg, -1, 3 /* e8 */, 0
+    %y:vr = PseudoVIOTA_M_M1 %x, $noreg, 1, 4 /* e16 */, 0
+    %z:vr = PseudoVADD_VV_M1 $noreg, %y, $noreg, 1, 4 /* e16 */, 0
+...
+---
+name: viota_m_dest_passthru_use_incompatible_emul
+body: |
+  bb.0:
+    ; CHECK-LABEL: name: viota_m_dest_passthru_use_incompatible_emul
+    ; CHECK: %x:vr = PseudoVADD_VV_M1 $noreg, $noreg, $noreg, -1, 3 /* e8 */, 0 /* tu, mu */
+    ; CHECK-NEXT: early-clobber %y:vr = PseudoVIOTA_M_MF2 %x, $noreg, 1, 3 /* e8 */, 0 /* tu, mu */
+    ; CHECK-NEXT: %z:vr = PseudoVADD_VV_MF2 $noreg, %y, $noreg, 1, 3 /* e8 */, 0 /* tu, mu */
+    %x:vr = PseudoVADD_VV_M1 $noreg, $noreg, $noreg, -1, 3 /* e8 */, 0
+    %y:vr = PseudoVIOTA_M_MF2 %x, $noreg, 1, 3 /* e8 */, 0
+    %z:vr = PseudoVADD_VV_MF2 $noreg, %y, $noreg, 1, 3 /* e8 */, 0
+...
+---
 name: viota_m_mask
 body: |
   bb.0:
@@ -1469,6 +1647,42 @@ body: |
     %y:vr = PseudoVFREDMAX_VS_MF2_E32 $noreg, %x, %x, 1, 5 /* e32 */, 0
 ...
 ---
+name: vwred_passthru_use
+body: |
+  bb.0:
+    ; CHECK-LABEL: name: vwred_passthru_use
+    ; CHECK: %x:vr = PseudoVADD_VV_M1 $noreg, $noreg, $noreg, -1, 4 /* e16 */, 0 /* tu, mu */
+    ; CHECK-NEXT: %y:vr = PseudoVWREDSUM_VS_MF2_E8 %x, $noreg, $noreg, 1, 3 /* e8 */, 0 /* tu, mu */
+    ; CHECK-NEXT: %z:vr = PseudoVADD_VV_M1 $noreg, %y, $noreg, 1, 4 /* e16 */, 0 /* tu, mu */
+    %x:vr = PseudoVADD_VV_M1 $noreg, $noreg, $noreg, -1, 4 /* e16 */, 0
+    %y:vr = PseudoVWREDSUM_VS_MF2_E8 %x, $noreg, $noreg, 1, 3 /* e8 */, 0
+    %z:vr = PseudoVADD_VV_M1 $noreg, %y, $noreg, 1, 4 /* e16 */, 0
+...
+---
+name: vwred_passthru_use_incompatible_eew
+body: |
+  bb.0:
+    ; CHECK-LABEL: name: vwred_passthru_use_incompatible_eew
+    ; CHECK: %x:vr = PseudoVADD_VV_M1 $noreg, $noreg, $noreg, -1, 3 /* e8 */, 0 /* tu, mu */
+    ; CHECK-NEXT: %y:vr = PseudoVWREDSUM_VS_MF2_E8 %x, $noreg, $noreg, 1, 3 /* e8 */, 0 /* tu, mu */
+    ; CHECK-NEXT: %z:vr = PseudoVADD_VV_M1 $noreg, %y, $noreg, 1, 4 /* e16 */, 0 /* tu, mu */
+    %x:vr = PseudoVADD_VV_M1 $noreg, $noreg, $noreg, -1, 3 /* e8 */, 0
+    %y:vr = PseudoVWREDSUM_VS_MF2_E8 %x, $noreg, $noreg, 1, 3 /* e8 */, 0
+    %z:vr = PseudoVADD_VV_M1 $noreg, %y, $noreg, 1, 4 /* e16 */, 0
+...
+---
+name: vwred_passthru_use_incompatible_emul
+body: |
+  bb.0:
+    ; CHECK-LABEL: name: vwred_passthru_use_incompatible_emul
+    ; CHECK: %x:vr = PseudoVADD_VV_M1 $noreg, $noreg, $noreg, -1, 4 /* e16 */, 0 /* tu, mu */
+    ; CHECK-NEXT: %y:vr = PseudoVWREDSUM_VS_MF4_E8 %x, $noreg, $noreg, 1, 3 /* e8 */, 0 /* tu, mu */
+    ; CHECK-NEXT: %z:vr = PseudoVADD_VV_MF2 $noreg, %y, $noreg, 1, 4 /* e16 */, 0 /* tu, mu */
+    %x:vr = PseudoVADD_VV_M1 $noreg, $noreg, $noreg, -1, 4 /* e16 */, 0
+    %y:vr = PseudoVWREDSUM_VS_MF4_E8 %x, $noreg, $noreg, 1, 3 /* e8 */, 0
+    %z:vr = PseudoVADD_VV_MF2 $noreg, %y, $noreg, 1, 4 /* e16 */, 0
+...
+---
 name: vfirst_v
 body: |
   bb.0:
diff --git a/llvm/test/CodeGen/RISCV/rvv/vl-opt.ll b/llvm/test/CodeGen/RISCV/rvv/vl-opt.ll
index 1cc30f077feb4..f0b05d2420b1a 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vl-opt.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vl-opt.ll
@@ -107,7 +107,8 @@ define <vscale x 4 x i32> @different_vl_with_ta(<vscale x 4 x i32> %a, <vscale x
   ret <vscale x 4 x i32> %w
 }
 
-; Test case to make sure VL won't propgate if using tail-undisturbed policy.
+; We can propagate VL to a tail-undisturbed policy, provided none of its users
+; are passthrus (i.e. read past VL).
 define <vscale x 4 x i32> @different_vl_with_tu(<vscale x 4 x i32> %passthru, <vscale x 4 x i32> %a, <vscale x 4 x i32> %b, iXLen %vl1, iXLen %vl2) {
 ; CHECK-LABEL: different_vl_with_tu:
 ; CHECK:       # %bb.0:
@@ -118,22 +119,65 @@ define <vscale x 4 x i32> @different_vl_with_tu(<vscale x 4 x i32> %passthru, <v
 ; CHECK-NEXT:    vadd.vv v8, v14, v10
 ; CHECK-NEXT:    ret
   %v = call <vscale x 4 x i32> @llvm.riscv.vadd.nxv4i32.nxv4i32(<vscale x 4 x i32> %a, <vscale x 4 x i32> %a, <vscale x 4 x i32> %b, iXLen %vl1)
-  %w = call <vscale x 4 x i32> @llvm.riscv.vadd.nxv4i32.nxv4i32(<vscale x 4 x i32> %passthru, <vscale x 4 x i32> %v, <vscale x 4 x i32> %a,iXLen %vl2)
+  %w = call <vscale x 4 x i32> @llvm.riscv.vadd.nxv4i32.nxv4i32(<vscale x 4 x i32> %passthru, <vscale x 4 x i32> %v, <vscale x 4 x i32> %a, iXLen %vl2)
   ret <vscale x 4 x i32> %w
 }
 
-; Test case to make sure VL won't propgate if using tail-undisturbed policy.
+; We can propagate VL to a tail-undisturbed policy, provided none of its users
+; are passthrus (i.e. read past VL).
 define <vscale x 4 x i32> @different_imm_vl_with_tu(<vscale x 4 x i32> %passthru, <vscale x 4 x i32> %a, <vscale x 4 x i32> %b, iXLen %vl1, iXLen %vl2) {
-; CHECK-LABEL: different_imm_vl_with_tu:
+; NOVLOPT-LABEL: different_imm_vl_with_tu:
+; NOVLOPT:       # %bb.0:
+; NOVLOPT-NEXT:    vsetivli zero, 5, e32, m2, tu, ma
+; NOVLOPT-NEXT:    vmv2r.v v14, v10
+; NOVLOPT-NEXT:    vadd.vv v14, v10, v12
+; NOVLOPT-NEXT:    vsetivli zero, 4, e32, m2, tu, ma
+; NOVLOPT-NEXT:    vadd.vv v8, v14, v10
+; NOVLOPT-NEXT:    ret
+;
+; VLOPT-LABEL: different_imm_vl_with_tu:
+; VLOPT:       # %bb.0:
+; VLOPT-NEXT:    vsetivli zero, 4, e32, m2, tu, ma
+; VLOPT-NEXT:    vmv2r.v v14, v10
+; VLOPT-NEXT:    vadd.vv v14, v10, v12
+; VLOPT-NEXT:    vadd.vv v8, v14, v10
+; VLOPT-NEXT:    ret
+  %v = call <vscale x 4 x i32> @llvm.riscv.vadd.nxv4i32.nxv4i32(<vscale x 4 x i32> %a, <vscale x 4 x i32> %a, <vscale x 4 x i32> %b, iXLen 5)
+  %w = call <vscale x 4 x i32> @llvm.riscv.vadd.nxv4i32.nxv4i32(<vscale x 4 x i32> %passthru, <vscale x 4 x i32> %v, <vscale x 4 x i32> %a, iXLen 4)
+  ret <vscale x 4 x i32> %w
+}
+
+; We can't reduce the VL as %v is used as a passthru, i.e. the elements past VL
+; are demanded.
+define <vscale x 4 x i32> @different_vl_as_passthru(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b, iXLen %vl1, iXLen %vl2) {
+; CHECK-LABEL: different_vl_as_passthru:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli zero, a0, e32, m2, tu, ma
+; CHECK-NEXT:    vmv2r.v v12, v8
+; CHECK-NEXT:    vadd.vv v12, v8, v10
+; CHECK-NEXT:    vsetvli zero, a1, e32, m2, tu, ma
+; CHECK-NEXT:    vadd.vv v12, v8, v10
+; CHECK-NEXT:    vmv2r.v v8, v12
+; CHECK-NEXT:    ret
+  %v = call <vscale x 4 x i32> @llvm.riscv.vadd.nxv4i32.nxv4i32(<vscale x 4 x i32> %a, <vscale x 4 x i32> %a, <vscale x 4 x i32> %b, iXLen %vl1)
+  %w = call <vscale x 4 x i32> @llvm.riscv.vadd.nxv4i32.nxv4i32(<vscale x 4 x i32> %v, <vscale x 4 x i32> %a, <vscale x 4 x i32> %b, iXLen %vl2)
+  ret <vscale x 4 x i32> %w
+}
+
+; We can't reduce the VL as %v is used as a passthru, i.e. the elements past VL
+; are demanded.
+define <vscale x 4 x i32> @different_imm_vl_as_passthru(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b, iXLen %vl1, iXLen %vl2) {
+; CHECK-LABEL: different_imm_vl_as_passthru:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetivli zero, 5, e32, m2, tu, ma
-; CHECK-NEXT:    vmv2r.v v14, v10
-; CHECK-NEXT:    vadd.vv v14, v10, v12
+; CHECK-NEXT:    vmv2r.v v12, v8
+; CHECK-NEXT:    vadd.vv v12, v8, v10
 ; CHECK-NEXT:    vsetivli zero, 4, e32, m2, tu, ma
-; CHECK-NEXT:    vadd.vv v8, v14, v10
+; CHECK-NEXT:    vadd.vv v12, v8, v10
+; CHECK-NEXT:    vmv2r.v v8, v12
 ; CHECK-NEXT:    ret
   %v = call <vscale x 4 x i32> @llvm.riscv.vadd.nxv4i32.nxv4i32(<vscale x 4 x i32> %a, <vscale x 4 x i32> %a, <vscale x 4 x i32> %b, iXLen 5)
-  %w = call <vscale x 4 x i32> @llvm.riscv.vadd.nxv4i32.nxv4i32(<vscale x 4 x i32> %passthru, <vscale x 4 x i32> %v, <vscale x 4 x i32> %a,iXLen 4)
+  %w = call <vscale x 4 x i32> @llvm.riscv.vadd.nxv4i32.nxv4i32(<vscale x 4 x i32> %v, <vscale x 4 x i32> %a, <vscale x 4 x i32> %b, iXLen 4)
   ret <vscale x 4 x i32> %w
 }
 
@@ -150,3 +194,26 @@ define <vscale x 4 x i32> @dont_optimize_tied_def(<vscale x 4 x i32> %a, <vscale
   ret <vscale x 4 x i32> %2
 }
 
+define void @optimize_ternary_use(<vscale x 4 x i16> %a, <vscale x 4 x i32> %b, <vscale x 4 x i32> %c, ptr %p, iXLen %vl) {
+; NOVLOPT-LABEL: optimize_ternary_use:
+; NOVLOPT:       # %bb.0:
+; NOVLOPT-NEXT:    vsetvli a2, zero, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vzext.vf2 v14, v8
+; NOVLOPT-NEXT:    vsetvli zero, a1, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vmadd.vv v14, v10, v12
+; NOVLOPT-NEXT:    vse32.v v14, (a0)
+; NOVLOPT-NEXT:    ret
+;
+; VLOPT-LABEL: optimize_ternary_use:
+; VLOPT:       # %bb.0:
+; VLOPT-NEXT:    vsetvli zero, a1, e32, m2, ta, ma
+; VLOPT-NEXT:    vzext.vf2 v14, v8
+; VLOPT-NEXT:    vmadd.vv v14, v10, v12
+; VLOPT-NEXT:    vse32.v v14, (a0)
+; VLOPT-NEXT:    ret
+  %1 = zext <vscale x 4 x i16> %a to <vscale x 4 x i32>
+  %2 = mul <vscale x 4 x i32> %b, %1
+  %3 = add <vscale x 4 x i32> %2, %c
+  call void @llvm.riscv.vse(<vscale x 4 x i32> %3, ptr %p, iXLen %vl)
+  ret void
+}
diff --git a/llvm/test/CodeGen/RISCV/rvv/vl-opt.mir b/llvm/test/CodeGen/RISCV/rvv/vl-opt.mir
index 027eb8ca3c17f..0475a988e9851 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vl-opt.mir
+++ b/llvm/test/CodeGen/RISCV/rvv/vl-opt.mir
@@ -149,3 +149,144 @@ body: |
     ; CHECK-NEXT: early-clobber %y:vrm2 = PseudoVWADD_WV_M1_TIED $noreg, %x, 1, 3 /* e8 */, 0 /* tu, mu */
     %x:vr = PseudoVADD_VV_M1 $noreg, $noreg, $noreg, -1, 3 /* e8 */, 0 /* tu, mu */
     %y:vrm2 = PseudoVWADD_WV_M1_TIED $noreg, %x, 1, 3 /* e8 */, 0 /* tu, mu */
+...
+---
+name: crossbb
+body: |
+  ; CHECK-LABEL: name: crossbb
+  ; CHECK: bb.0:
+  ; CHECK-NEXT:   successors: %bb.3(0x80000000)
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   PseudoBR %bb.3
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.1:
+  ; CHECK-NEXT:   %a1:vr = PseudoVADD_VV_M1 $noreg, %c, $noreg, 1, 3 /* e8 */, 0 /* tu, mu */
+  ; CHECK-NEXT:   %a2:vr = PseudoVADD_VV_M1 $noreg, %a1, $noreg, 1, 3 /* e8 */, 0 /* tu, mu */
+  ; CHECK-NEXT:   PseudoRET
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.2:
+  ; CHECK-NEXT:   %b1:vr = PseudoVADD_VV_M1 $noreg, %c, $noreg, 1, 3 /* e8 */, 0 /* tu, mu */
+  ; CHECK-NEXT:   %b2:vr = PseudoVADD_VV_M1 $noreg, %b1, $noreg, 1, 3 /* e8 */, 0 /* tu, mu */
+  ; CHECK-NEXT:   PseudoRET
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.3:
+  ; CHECK-NEXT:   successors: %bb.1(0x40000000), %bb.2(0x40000000)
+  ; CHECK-NEXT:   liveins: $x1
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   %c:vr = PseudoVADD_VV_M1 $noreg, $noreg, $noreg, 1, 3 /* e8 */, 0 /* tu, mu */
+  ; CHECK-NEXT:   BEQ $x1, $x0, %bb.1
+  ; CHECK-NEXT:   PseudoBR %bb.2
+  bb.0:
+    PseudoBR %bb.3
+  bb.1:
+    %a1:vr = PseudoVADD_VV_M1 $noreg, %c, $noreg, -1, 3 /* e8 */, 0 /* tu, mu */
+    %a2:vr = PseudoVADD_VV_M1 $noreg, %a1, $noreg, 1, 3 /* e8 */, 0 /* tu, mu */
+    PseudoRET
+  bb.2:
+    %b1:vr = PseudoVADD_VV_M1 $noreg, %c, $noreg, -1, 3 /* e8 */, 0 /* tu, mu */
+    %b2:vr = PseudoVADD_VV_M1 $noreg, %b1, $noreg, 1, 3 /* e8 */, 0 /* tu, mu */
+    PseudoRET
+  bb.3:
+    liveins: $x1
+    %c:vr = PseudoVADD_VV_M1 $noreg, $noreg, $noreg, -1, 3 /* e8 */, 0 /* tu, mu */
+    BEQ $x1, $x0, %bb.1
+    PseudoBR %bb.2
+...
+---
+name: unreachable
+body: |
+  ; CHECK-LABEL: name: unreachable
+  ; CHECK: bb.0:
+  ; CHECK-NEXT:   %x:vr = PseudoVADD_VV_M1 $noreg, $noreg, $noreg, 1, 3 /* e8 */, 0 /* tu, mu */
+  ; CHECK-NEXT:   PseudoRET
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.1:
+  ; CHECK-NEXT:   %y:vr = PseudoVADD_VV_M1 $noreg, %x, $noreg, 1, 3 /* e8 */, 0 /* tu, mu */
+  ; CHECK-NEXT:   PseudoRET
+  bb.0:
+    %x:vr = PseudoVADD_VV_M1 $noreg, $noreg, $noreg, -1, 3 /* e8 */, 0 /* tu, mu */
+    PseudoRET
+  bb.1:
+    %y:vr = PseudoVADD_VV_M1 $noreg, %x, $noreg, 1, 3 /* e8 */, 0 /* tu, mu */
+    PseudoRET
+...
+---
+# Can reduce %x even though %y uses it as a passthru, because %y's inactive elements aren't demanded
+name: passthru_not_demanded
+body: |
+  bb.0:
+    ; CHECK-LABEL: name: passthru_not_demanded
+    ; CHECK: %x:vr = PseudoVADD_VV_M1 $noreg, $noreg, $noreg, 1, 3 /* e8 */, 0 /* tu, mu */
+    ; CHECK-NEXT: %y:vr = PseudoVADD_VV_M1 %x, $noreg, $noreg, 1, 3 /* e8 */, 0 /* tu, mu */
+    ; CHECK-NEXT: %z:vr = PseudoVADD_VV_M1 $noreg, %y, $noreg, 1, 3 /* e8 */, 0 /* tu, mu */
+    %x:vr = PseudoVADD_VV_M1 $noreg, $noreg, $noreg, -1, 3 /* e8 */, 0 /* tu, mu */
+    %y:vr = PseudoVADD_VV_M1 %x, $noreg, $noreg, 1, 3 /* e8 */, 0 /* tu, mu */
+    %z:vr = PseudoVADD_VV_M1 $noreg, %y, $noreg, 1, 3 /* e8 */, 0 /* tu, mu */
+...
+---
+# Can't reduce %x because %y uses it as a passthru, and %y's inactive elements are demanded by %z
+name: passthru_demanded
+body: |
+  bb.0:
+    ; CHECK-LABEL: name: passthru_demanded
+    ; CHECK: %x:vr = PseudoVADD_VV_M1 $noreg, $noreg, $noreg, -1, 3 /* e8 */, 0 /* tu, mu */
+    ; CHECK-NEXT: %y:vr = PseudoVADD_VV_M1 %x, $noreg, $noreg, 1, 3 /* e8 */, 0 /* tu, mu */
+    ; CHECK-NEXT: %z:vr = PseudoVADD_VV_M1 $noreg, %y, $noreg, 2, 3 /* e8 */, 0 /* tu, mu */
+    %x:vr = PseudoVADD_VV_M1 $noreg, $noreg, $noreg, -1, 3 /* e8 */, 0 /* tu, mu */
+    %y:vr = PseudoVADD_VV_M1 %x, $noreg, $noreg, 1, 3 /* e8 */, 0 /* tu, mu */
+    %z:vr = PseudoVADD_VV_M1 $noreg, %y, $noreg, 2, 3 /* e8 */, 0 /* tu, mu */
+...
+---
+# Can reduce %x even though %y uses it as a passthru, because %y's inactive elements aren't demanded
+name: passthru_not_demanded_passthru_chain
+body: |
+  bb.0:
+    ; CHECK-LABEL: name: passthru_not_demanded_passthru_chain
+    ; CHECK: %x:vr = PseudoVADD_VV_M1 $noreg, $noreg, $noreg, 1, 3 /* e8 */, 0 /* tu, mu */
+    ; CHECK-NEXT: %y:vr = PseudoVADD_VV_M1 %x, $noreg, $noreg, 1, 3 /* e8 */, 0 /* tu, mu */
+    ; CHECK-NEXT: %z:vr = PseudoVADD_VV_M1 %y, $noreg, $noreg, 1, 3 /* e8 */, 0 /* tu, mu */
+    ; CHECK-NEXT: %a:vr = PseudoVADD_VV_M1 %z, $noreg, $noreg, 1, 3 /* e8 */, 0 /* tu, mu */
+    ; CHECK-NEXT: %b:vr = PseudoVADD_VV_M1 $noreg, %a, $noreg, 1, 3 /* e8 */, 0 /* tu, mu */
+    %x:vr = PseudoVADD_VV_M1 $noreg, $noreg, $noreg, -1, 3 /* e8 */, 0 /* tu, mu */
+    %y:vr = PseudoVADD_VV_M1 %x, $noreg, $noreg, 1, 3 /* e8 */, 0 /* tu, mu */
+    %z:vr = PseudoVADD_VV_M1 %y, $noreg, $noreg, 1, 3 /* e8 */, 0 /* tu, mu */
+    %a:vr = PseudoVADD_VV_M1 %z, $noreg, $noreg, 1, 3 /* e8 */, 0 /* tu, mu */
+    %b:vr = PseudoVADD_VV_M1 $noreg, %a, $noreg, 1, 3 /* e8 */, 0 /* tu, mu */
+...
+---
+# Can't reduce %x because %y uses it as a passthru, and %y's inactive elements are ultimately demanded in %b
+name: passthru_demanded_passthru_chain
+body: |
+  bb.0:
+    ; CHECK-LABEL: name: passthru_demanded_passthru_chain
+    ; CHECK: %x:vr = PseudoVADD_VV_M1 $noreg, $noreg, $noreg, -1, 3 /* e8 */, 0 /* tu, mu */
+    ; CHECK-NEXT: %y:vr = PseudoVADD_VV_M1 %x, $noreg, $noreg, 1, 3 /* e8 */, 0 /* tu, mu */
+    ; CHECK-NEXT: %z:vr = PseudoVADD_VV_M1 %y, $noreg, $noreg, 1, 3 /* e8 */, 0 /* tu, mu */
+    ; CHECK-NEXT: %a:vr = PseudoVADD_VV_M1 %z, $noreg, $noreg, 1, 3 /* e8 */, 0 /* tu, mu */
+    ; CHECK-NEXT: %b:vr = PseudoVADD_VV_M1 $noreg, %a, $noreg, 2, 3 /* e8 */, 0 /* tu, mu */
+    %x:vr = PseudoVADD_VV_M1 $noreg, $noreg, $noreg, -1, 3 /* e8 */, 0 /* tu, mu */
+    %y:vr = PseudoVADD_VV_M1 %x, $noreg, $noreg, 1, 3 /* e8 */, 0 /* tu, mu */
+    %z:vr = PseudoVADD_VV_M1 %y, $noreg, $noreg, 1, 3 /* e8 */, 0 /* tu, mu */
+    %a:vr = PseudoVADD_VV_M1 %z, $noreg, $noreg, 1, 3 /* e8 */, 0 /* tu, mu */
+    %b:vr = PseudoVADD_VV_M1 $noreg, %a, $noreg, 2, 3 /* e8 */, 0 /* tu, mu */
+...
+---
+name: vxsat_dead
+body: |
+  bb.0:
+    ; CHECK-LABEL: name: vxsat_dead
+    ; CHECK: %x:vr = PseudoVSADDU_VV_M1 $noreg, $noreg, $noreg, 1, 3 /* e8 */, 0 /* tu, mu */, implicit-def dead $vxsat
+    ; CHECK-NEXT: %y:vr = PseudoVADD_VV_M1 $noreg, %x, $noreg, 1, 3 /* e8 */, 0 /* tu, mu */
+    %x:vr = PseudoVSADDU_VV_M1 $noreg, $noreg, $noreg, -1, 3 /* e8 */, 0 /* tu, mu */, implicit-def dead $vxsat
+    %y:vr = PseudoVADD_VV_M1 $noreg, %x, $noreg, 1, 3 /* e8 */, 0 /* tu, mu */
+...
+---
+name: vxsat_not_dead
+body: |
+  bb.0:
+    ; CHECK-LABEL: name: vxsat_not_dead
+    ; CHECK: %x:vr = PseudoVSADDU_VV_M1 $noreg, $noreg, $noreg, -1, 3 /* e8 */, 0 /* tu, mu */, implicit-def $vxsat
+    ; CHECK-NEXT: %y:vr = PseudoVADD_VV_M1 $noreg, %x, $noreg, 1, 3 /* e8 */, 0 /* tu, mu */
+    %x:vr = PseudoVSADDU_VV_M1 $noreg, $noreg, $noreg, -1, 3 /* e8 */, 0 /* tu, mu */, implicit-def $vxsat
+    %y:vr = PseudoVADD_VV_M1 $noreg, %x, $noreg, 1, 3 /* e8 */, 0 /* tu, mu */
+...
diff --git a/llvm/test/CodeGen/RISCV/rvv/vlopt-same-vl.ll b/llvm/test/CodeGen/RISCV/rvv/vlopt-same-vl.ll
index 65e6eddfb3cd6..a14268af85bc8 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vlopt-same-vl.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vlopt-same-vl.ll
@@ -12,7 +12,7 @@
 
 define <vscale x 4 x i32> @same_vl_imm(<vscale x 4 x i32> %passthru, <vscale x 4 x i32> %a, <vscale x 4 x i32> %b) {
   ; CHECK: User VL is: 4
-  ; CHECK-NEXT: Abort due to CommonVL == VLOp, no point in reducing.
+  ; CHECK: Abort due to CommonVL == VLOp, no point in reducing.
   %v = call <vscale x 4 x i32> @llvm.riscv.vadd.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %a, <vscale x 4 x i32> %b, i64 4)
   %w = call <vscale x 4 x i32> @llvm.riscv.vadd.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %v, <vscale x 4 x i32> %a, i64 4)
   ret <vscale x 4 x i32> %w
@@ -20,7 +20,7 @@ define <vscale x 4 x i32> @same_vl_imm(<vscale x 4 x i32> %passthru, <vscale x 4
 
 define <vscale x 4 x i32> @same_vl_reg(<vscale x 4 x i32> %passthru, <vscale x 4 x i32> %a, <vscale x 4 x i32> %b, i64 %vl) {
   ; CHECK: User VL is: %3:gprnox0
-  ; CHECK-NEXT: Abort due to CommonVL == VLOp, no point in reducing.
+  ; CHECK: Abort due to CommonVL == VLOp, no point in reducing.
   %v = call <vscale x 4 x i32> @llvm.riscv.vadd.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %a, <vscale x 4 x i32> %b, i64 %vl)
   %w = call <vscale x 4 x i32> @llvm.riscv.vadd.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %v, <vscale x 4 x i32> %a, i64 %vl)
   ret <vscale x 4 x i32> %w
diff --git a/llvm/test/CodeGen/RISCV/rvv/vmadd-vp.ll b/llvm/test/CodeGen/RISCV/rvv/vmadd-vp.ll
index eb74b238c01b9..a29af3d5b54b0 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vmadd-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vmadd-vp.ll
@@ -1638,9 +1638,8 @@ define <vscale x 1 x i64> @vmadd_vx_nxv1i64(<vscale x 1 x i64> %a, i64 %b, <vsca
 ; RV32-NEXT:    sw a0, 8(sp)
 ; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
-; RV32-NEXT:    vsetvli a1, zero, e64, m1, ta, ma
-; RV32-NEXT:    vlse64.v v10, (a0), zero
 ; RV32-NEXT:    vsetvli zero, a2, e64, m1, ta, ma
+; RV32-NEXT:    vlse64.v v10, (a0), zero
 ; RV32-NEXT:    vmadd.vv v10, v8, v9
 ; RV32-NEXT:    vsetvli zero, zero, e64, m1, tu, ma
 ; RV32-NEXT:    vmerge.vvm v8, v8, v10, v0
@@ -1669,9 +1668,8 @@ define <vscale x 1 x i64> @vmadd_vx_nxv1i64_unmasked(<vscale x 1 x i64> %a, i64
 ; RV32-NEXT:    sw a0, 8(sp)
 ; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
-; RV32-NEXT:    vsetvli a1, zero, e64, m1, ta, ma
-; RV32-NEXT:    vlse64.v v10, (a0), zero
 ; RV32-NEXT:    vsetvli zero, a2, e64, m1, ta, ma
+; RV32-NEXT:    vlse64.v v10, (a0), zero
 ; RV32-NEXT:    vmadd.vv v10, v8, v9
 ; RV32-NEXT:    vsetvli zero, zero, e64, m1, tu, ma
 ; RV32-NEXT:    vmv.v.v v8, v10
@@ -1713,9 +1711,8 @@ define <vscale x 1 x i64> @vmadd_vx_nxv1i64_ta(<vscale x 1 x i64> %a, i64 %b, <v
 ; RV32-NEXT:    sw a0, 8(sp)
 ; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
-; RV32-NEXT:    vsetvli a1, zero, e64, m1, ta, ma
-; RV32-NEXT:    vlse64.v v10, (a0), zero
 ; RV32-NEXT:    vsetvli zero, a2, e64, m1, ta, ma
+; RV32-NEXT:    vlse64.v v10, (a0), zero
 ; RV32-NEXT:    vmadd.vv v10, v8, v9
 ; RV32-NEXT:    vmerge.vvm v8, v8, v10, v0
 ; RV32-NEXT:    addi sp, sp, 16
@@ -1776,9 +1773,8 @@ define <vscale x 2 x i64> @vmadd_vx_nxv2i64(<vscale x 2 x i64> %a, i64 %b, <vsca
 ; RV32-NEXT:    sw a0, 8(sp)
 ; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
-; RV32-NEXT:    vsetvli a1, zero, e64, m2, ta, ma
-; RV32-NEXT:    vlse64.v v12, (a0), zero
 ; RV32-NEXT:    vsetvli zero, a2, e64, m2, ta, ma
+; RV32-NEXT:    vlse64.v v12, (a0), zero
 ; RV32-NEXT:    vmadd.vv v12, v8, v10
 ; RV32-NEXT:    vsetvli zero, zero, e64, m2, tu, ma
 ; RV32-NEXT:    vmerge.vvm v8, v8, v12, v0
@@ -1807,9 +1803,8 @@ define <vscale x 2 x i64> @vmadd_vx_nxv2i64_unmasked(<vscale x 2 x i64> %a, i64
 ; RV32-NEXT:    sw a0, 8(sp)
 ; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
-; RV32-NEXT:    vsetvli a1, zero, e64, m2, ta, ma
-; RV32-NEXT:    vlse64.v v12, (a0), zero
 ; RV32-NEXT:    vsetvli zero, a2, e64, m2, ta, ma
+; RV32-NEXT:    vlse64.v v12, (a0), zero
 ; RV32-NEXT:    vmadd.vv v12, v8, v10
 ; RV32-NEXT:    vsetvli zero, zero, e64, m2, tu, ma
 ; RV32-NEXT:    vmv.v.v v8, v12
@@ -1851,9 +1846,8 @@ define <vscale x 2 x i64> @vmadd_vx_nxv2i64_ta(<vscale x 2 x i64> %a, i64 %b, <v
 ; RV32-NEXT:    sw a0, 8(sp)
 ; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
-; RV32-NEXT:    vsetvli a1, zero, e64, m2, ta, ma
-; RV32-NEXT:    vlse64.v v12, (a0), zero
 ; RV32-NEXT:    vsetvli zero, a2, e64, m2, ta, ma
+; RV32-NEXT:    vlse64.v v12, (a0), zero
 ; RV32-NEXT:    vmadd.vv v12, v8, v10
 ; RV32-NEXT:    vmerge.vvm v8, v8, v12, v0
 ; RV32-NEXT:    addi sp, sp, 16
@@ -1914,9 +1908,8 @@ define <vscale x 4 x i64> @vmadd_vx_nxv4i64(<vscale x 4 x i64> %a, i64 %b, <vsca
 ; RV32-NEXT:    sw a0, 8(sp)
 ; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
-; RV32-NEXT:    vsetvli a1, zero, e64, m4, ta, ma
-; RV32-NEXT:    vlse64.v v16, (a0), zero
 ; RV32-NEXT:    vsetvli zero, a2, e64, m4, ta, ma
+; RV32-NEXT:    vlse64.v v16, (a0), zero
 ; RV32-NEXT:    vmadd.vv v16, v8, v12
 ; RV32-NEXT:    vsetvli zero, zero, e64, m4, tu, ma
 ; RV32-NEXT:    vmerge.vvm v8, v8, v16, v0
@@ -1945,9 +1938,8 @@ define <vscale x 4 x i64> @vmadd_vx_nxv4i64_unmasked(<vscale x 4 x i64> %a, i64
 ; RV32-NEXT:    sw a0, 8(sp)
 ; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
-; RV32-NEXT:    vsetvli a1, zero, e64, m4, ta, ma
-; RV32-NEXT:    vlse64.v v16, (a0), zero
 ; RV32-NEXT:    vsetvli zero, a2, e64, m4, ta, ma
+; RV32-NEXT:    vlse64.v v16, (a0), zero
 ; RV32-NEXT:    vmadd.vv v16, v8, v12
 ; RV32-NEXT:    vsetvli zero, zero, e64, m4, tu, ma
 ; RV32-NEXT:    vmv.v.v v8, v16
@@ -1989,9 +1981,8 @@ define <vscale x 4 x i64> @vmadd_vx_nxv4i64_ta(<vscale x 4 x i64> %a, i64 %b, <v
 ; RV32-NEXT:    sw a0, 8(sp)
 ; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
-; RV32-NEXT:    vsetvli a1, zero, e64, m4, ta, ma
-; RV32-NEXT:    vlse64.v v16, (a0), zero
 ; RV32-NEXT:    vsetvli zero, a2, e64, m4, ta, ma
+; RV32-NEXT:    vlse64.v v16, (a0), zero
 ; RV32-NEXT:    vmadd.vv v16, v8, v12
 ; RV32-NEXT:    vmerge.vvm v8, v8, v16, v0
 ; RV32-NEXT:    addi sp, sp, 16
@@ -2054,9 +2045,8 @@ define <vscale x 8 x i64> @vmadd_vx_nxv8i64(<vscale x 8 x i64> %a, i64 %b, <vsca
 ; RV32-NEXT:    sw a0, 8(sp)
 ; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
-; RV32-NEXT:    vsetvli a1, zero, e64, m8, ta, ma
-; RV32-NEXT:    vlse64.v v24, (a0), zero
 ; RV32-NEXT:    vsetvli zero, a2, e64, m8, ta, ma
+; RV32-NEXT:    vlse64.v v24, (a0), zero
 ; RV32-NEXT:    vmadd.vv v24, v8, v16
 ; RV32-NEXT:    vsetvli zero, zero, e64, m8, tu, ma
 ; RV32-NEXT:    vmerge.vvm v8, v8, v24, v0
@@ -2085,9 +2075,8 @@ define <vscale x 8 x i64> @vmadd_vx_nxv8i64_unmasked(<vscale x 8 x i64> %a, i64
 ; RV32-NEXT:    sw a0, 8(sp)
 ; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
-; RV32-NEXT:    vsetvli a1, zero, e64, m8, ta, ma
-; RV32-NEXT:    vlse64.v v24, (a0), zero
 ; RV32-NEXT:    vsetvli zero, a2, e64, m8, ta, ma
+; RV32-NEXT:    vlse64.v v24, (a0), zero
 ; RV32-NEXT:    vmadd.vv v24, v8, v16
 ; RV32-NEXT:    vsetvli zero, zero, e64, m8, tu, ma
 ; RV32-NEXT:    vmv.v.v v8, v24
@@ -2130,9 +2119,8 @@ define <vscale x 8 x i64> @vmadd_vx_nxv8i64_ta(<vscale x 8 x i64> %a, i64 %b, <v
 ; RV32-NEXT:    sw a0, 8(sp)
 ; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
-; RV32-NEXT:    vsetvli a1, zero, e64, m8, ta, ma
-; RV32-NEXT:    vlse64.v v24, (a0), zero
 ; RV32-NEXT:    vsetvli zero, a2, e64, m8, ta, ma
+; RV32-NEXT:    vlse64.v v24, (a0), zero
 ; RV32-NEXT:    vmadd.vv v24, v8, v16
 ; RV32-NEXT:    vmerge.vvm v8, v8, v24, v0
 ; RV32-NEXT:    addi sp, sp, 16
diff --git a/llvm/test/CodeGen/RISCV/rvv/vp-combine-reverse-load.ll b/llvm/test/CodeGen/RISCV/rvv/vp-combine-reverse-load.ll
index 50e26bd141070..24d8e56fa17fe 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vp-combine-reverse-load.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vp-combine-reverse-load.ll
@@ -1,4 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=riscv32 -mattr=+f,+v -verify-machineinstrs < %s | FileCheck %s
 ; RUN: llc -mtriple=riscv64 -mattr=+f,+v -verify-machineinstrs < %s | FileCheck %s
 
 define <vscale x 2 x float> @test_reverse_load_combiner(<vscale x 2 x float>* %ptr, i32 zeroext %evl) {
diff --git a/llvm/test/CodeGen/RISCV/rvv/vp-combine-store-reverse.ll b/llvm/test/CodeGen/RISCV/rvv/vp-combine-store-reverse.ll
index 4896a1367935a..a2466c48b0ab7 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vp-combine-store-reverse.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vp-combine-store-reverse.ll
@@ -1,4 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=riscv32 -mattr=+f,+v -verify-machineinstrs < %s | FileCheck %s
 ; RUN: llc -mtriple=riscv64 -mattr=+f,+v -verify-machineinstrs < %s | FileCheck %s
 
 define void @test_store_reverse_combiner(<vscale x 2 x float> %val, <vscale x 2 x float>* %ptr, i32 zeroext %evl) {
diff --git a/llvm/test/CodeGen/RISCV/rvv/vp-vaaddu.ll b/llvm/test/CodeGen/RISCV/rvv/vp-vaaddu.ll
new file mode 100644
index 0000000000000..989fbb7fcea8b
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/rvv/vp-vaaddu.ll
@@ -0,0 +1,169 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 4
+; RUN: llc < %s -mtriple=riscv64 -mattr=+v | FileCheck %s
+
+declare <vscale x 2 x i16> @llvm.vp.zext.nxv2i16.nxv2i8(<vscale x 2 x i8>, <vscale x 2 x i1>, i32)
+declare <vscale x 2 x i32> @llvm.vp.zext.nxv2i32.nxv2i8(<vscale x 2 x i8>, <vscale x 2 x i1>, i32)
+declare <vscale x 2 x i32> @llvm.vp.zext.nxv2i32.nxv2i16(<vscale x 2 x i16>, <vscale x 2 x i1>, i32)
+declare <vscale x 2 x i8> @llvm.vp.trunc.nxv2i8.nxv2i16(<vscale x 2 x i16>, <vscale x 2 x i1>, i32)
+declare <vscale x 2 x i16> @llvm.vp.trunc.nxv2i16.nxv2i32(<vscale x 2 x i32>, <vscale x 2 x i1>, i32)
+declare <vscale x 2 x i8> @llvm.vp.trunc.nxv2i8.nxv2i32(<vscale x 2 x i32>, <vscale x 2 x i1>, i32)
+declare <vscale x 2 x i16> @llvm.vp.add.nxv2i16(<vscale x 2 x i16>, <vscale x 2 x i16>, <vscale x 2 x i1>, i32)
+declare <vscale x 2 x i16> @llvm.vp.lshr.nxv2i16(<vscale x 2 x i16>, <vscale x 2 x i16>, <vscale x 2 x i1>, i32)
+declare <vscale x 2 x i32> @llvm.vp.add.nxv2i32(<vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i1>, i32)
+declare <vscale x 2 x i32> @llvm.vp.lshr.nxv2i32(<vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i1>, i32)
+
+define <vscale x 2 x i8> @vaaddu_1(<vscale x 2 x i8> %x, <vscale x 2 x i8> %y, <vscale x 2 x i1> %m, i32 zeroext %vl) {
+; CHECK-LABEL: vaaddu_1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e8, mf4, ta, ma
+; CHECK-NEXT:    vaaddu.vv v8, v8, v9, v0.t
+; CHECK-NEXT:    ret
+  %xz = call <vscale x 2 x i16> @llvm.vp.zext.nxv2i16.nxv2i8(<vscale x 2 x i8> %x, <vscale x 2 x i1> %m, i32 %vl)
+  %yz = call <vscale x 2 x i16> @llvm.vp.zext.nxv2i16.nxv2i8(<vscale x 2 x i8> %y, <vscale x 2 x i1> %m, i32 %vl)
+  %a = call <vscale x 2 x i16> @llvm.vp.add.nxv2i16(<vscale x 2 x i16> %xz, <vscale x 2 x i16> %yz, <vscale x 2 x i1> %m, i32 %vl)
+  %b = call <vscale x 2 x i16> @llvm.vp.add.nxv2i16(<vscale x 2 x i16> %a, <vscale x 2 x i16> splat (i16 1), <vscale x 2 x i1> %m, i32 %vl)
+  %c = call <vscale x 2 x i16> @llvm.vp.lshr.nxv2i16(<vscale x 2 x i16> %b, <vscale x 2 x i16> splat (i16 1), <vscale x 2 x i1> %m, i32 %vl)
+  %d = call <vscale x 2 x i8> @llvm.vp.trunc.nxv2i8.nxv2i16(<vscale x 2 x i16> %c, <vscale x 2 x i1> %m, i32 %vl)
+  ret <vscale x 2 x i8> %d
+}
+
+define <vscale x 2 x i8> @vaaddu_2(<vscale x 2 x i8> %x, <vscale x 2 x i8> %y, <vscale x 2 x i1> %m, i32 zeroext %vl) {
+; CHECK-LABEL: vaaddu_2:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e8, mf4, ta, ma
+; CHECK-NEXT:    vaaddu.vv v8, v8, v9, v0.t
+; CHECK-NEXT:    ret
+  %xz = call <vscale x 2 x i16> @llvm.vp.zext.nxv2i16.nxv2i8(<vscale x 2 x i8> %x, <vscale x 2 x i1> %m, i32 %vl)
+  %yz = call <vscale x 2 x i16> @llvm.vp.zext.nxv2i16.nxv2i8(<vscale x 2 x i8> %y, <vscale x 2 x i1> %m, i32 %vl)
+  %a = call <vscale x 2 x i16> @llvm.vp.add.nxv2i16(<vscale x 2 x i16> %xz, <vscale x 2 x i16> %yz, <vscale x 2 x i1> %m, i32 %vl)
+  %b = call <vscale x 2 x i16> @llvm.vp.add.nxv2i16(<vscale x 2 x i16> splat (i16 1), <vscale x 2 x i16> %a, <vscale x 2 x i1> %m, i32 %vl)
+  %c = call <vscale x 2 x i16> @llvm.vp.lshr.nxv2i16(<vscale x 2 x i16> %b, <vscale x 2 x i16> splat (i16 1), <vscale x 2 x i1> %m, i32 %vl)
+  %d = call <vscale x 2 x i8> @llvm.vp.trunc.nxv2i8.nxv2i16(<vscale x 2 x i16> %c, <vscale x 2 x i1> %m, i32 %vl)
+  ret <vscale x 2 x i8> %d
+}
+
+define <vscale x 2 x i8> @vaaddu_3(<vscale x 2 x i8> %x, <vscale x 2 x i8> %y, <vscale x 2 x i1> %m, i32 zeroext %vl) {
+; CHECK-LABEL: vaaddu_3:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e8, mf4, ta, ma
+; CHECK-NEXT:    vaaddu.vv v8, v9, v8, v0.t
+; CHECK-NEXT:    ret
+  %xz = call <vscale x 2 x i16> @llvm.vp.zext.nxv2i16.nxv2i8(<vscale x 2 x i8> %x, <vscale x 2 x i1> %m, i32 %vl)
+  %yz = call <vscale x 2 x i16> @llvm.vp.zext.nxv2i16.nxv2i8(<vscale x 2 x i8> %y, <vscale x 2 x i1> %m, i32 %vl)
+  %a = call <vscale x 2 x i16> @llvm.vp.add.nxv2i16(<vscale x 2 x i16> %xz, <vscale x 2 x i16> splat (i16 1), <vscale x 2 x i1> %m, i32 %vl)
+  %b = call <vscale x 2 x i16> @llvm.vp.add.nxv2i16(<vscale x 2 x i16> %a, <vscale x 2 x i16> %yz, <vscale x 2 x i1> %m, i32 %vl)
+  %c = call <vscale x 2 x i16> @llvm.vp.lshr.nxv2i16(<vscale x 2 x i16> %b, <vscale x 2 x i16> splat (i16 1), <vscale x 2 x i1> %m, i32 %vl)
+  %d = call <vscale x 2 x i8> @llvm.vp.trunc.nxv2i8.nxv2i16(<vscale x 2 x i16> %c, <vscale x 2 x i1> %m, i32 %vl)
+  ret <vscale x 2 x i8> %d
+}
+
+define <vscale x 2 x i8> @vaaddu_4(<vscale x 2 x i8> %x, <vscale x 2 x i8> %y, <vscale x 2 x i1> %m, i32 zeroext %vl) {
+; CHECK-LABEL: vaaddu_4:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e8, mf4, ta, ma
+; CHECK-NEXT:    vaaddu.vv v8, v9, v8, v0.t
+; CHECK-NEXT:    ret
+  %xz = call <vscale x 2 x i16> @llvm.vp.zext.nxv2i16.nxv2i8(<vscale x 2 x i8> %x, <vscale x 2 x i1> %m, i32 %vl)
+  %yz = call <vscale x 2 x i16> @llvm.vp.zext.nxv2i16.nxv2i8(<vscale x 2 x i8> %y, <vscale x 2 x i1> %m, i32 %vl)
+  %a = call <vscale x 2 x i16> @llvm.vp.add.nxv2i16(<vscale x 2 x i16> %xz, <vscale x 2 x i16> splat (i16 1), <vscale x 2 x i1> %m, i32 %vl)
+  %b = call <vscale x 2 x i16> @llvm.vp.add.nxv2i16(<vscale x 2 x i16> %yz, <vscale x 2 x i16> %a, <vscale x 2 x i1> %m, i32 %vl)
+  %c = call <vscale x 2 x i16> @llvm.vp.lshr.nxv2i16(<vscale x 2 x i16> %b, <vscale x 2 x i16> splat (i16 1), <vscale x 2 x i1> %m, i32 %vl)
+  %d = call <vscale x 2 x i8> @llvm.vp.trunc.nxv2i8.nxv2i16(<vscale x 2 x i16> %c, <vscale x 2 x i1> %m, i32 %vl)
+  ret <vscale x 2 x i8> %d
+}
+
+define <vscale x 2 x i8> @vaaddu_5(<vscale x 2 x i8> %x, <vscale x 2 x i8> %y, <vscale x 2 x i1> %m, i32 zeroext %vl) {
+; CHECK-LABEL: vaaddu_5:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e8, mf4, ta, ma
+; CHECK-NEXT:    vaaddu.vv v8, v9, v8, v0.t
+; CHECK-NEXT:    ret
+  %xz = call <vscale x 2 x i16> @llvm.vp.zext.nxv2i16.nxv2i8(<vscale x 2 x i8> %x, <vscale x 2 x i1> %m, i32 %vl)
+  %yz = call <vscale x 2 x i16> @llvm.vp.zext.nxv2i16.nxv2i8(<vscale x 2 x i8> %y, <vscale x 2 x i1> %m, i32 %vl)
+  %a = call <vscale x 2 x i16> @llvm.vp.add.nxv2i16(<vscale x 2 x i16> splat (i16 1), <vscale x 2 x i16> %xz, <vscale x 2 x i1> %m, i32 %vl)
+  %b = call <vscale x 2 x i16> @llvm.vp.add.nxv2i16(<vscale x 2 x i16> %a, <vscale x 2 x i16> %yz, <vscale x 2 x i1> %m, i32 %vl)
+  %c = call <vscale x 2 x i16> @llvm.vp.lshr.nxv2i16(<vscale x 2 x i16> %b, <vscale x 2 x i16> splat (i16 1), <vscale x 2 x i1> %m, i32 %vl)
+  %d = call <vscale x 2 x i8> @llvm.vp.trunc.nxv2i8.nxv2i16(<vscale x 2 x i16> %c, <vscale x 2 x i1> %m, i32 %vl)
+  ret <vscale x 2 x i8> %d
+}
+
+define <vscale x 2 x i8> @vaaddu_6(<vscale x 2 x i8> %x, <vscale x 2 x i8> %y, <vscale x 2 x i1> %m, i32 zeroext %vl) {
+; CHECK-LABEL: vaaddu_6:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e8, mf4, ta, ma
+; CHECK-NEXT:    vaaddu.vv v8, v9, v8, v0.t
+; CHECK-NEXT:    ret
+  %xz = call <vscale x 2 x i16> @llvm.vp.zext.nxv2i16.nxv2i8(<vscale x 2 x i8> %x, <vscale x 2 x i1> %m, i32 %vl)
+  %yz = call <vscale x 2 x i16> @llvm.vp.zext.nxv2i16.nxv2i8(<vscale x 2 x i8> %y, <vscale x 2 x i1> %m, i32 %vl)
+  %a = call <vscale x 2 x i16> @llvm.vp.add.nxv2i16(<vscale x 2 x i16> splat (i16 1), <vscale x 2 x i16> %xz, <vscale x 2 x i1> %m, i32 %vl)
+  %b = call <vscale x 2 x i16> @llvm.vp.add.nxv2i16(<vscale x 2 x i16> %yz, <vscale x 2 x i16> %a, <vscale x 2 x i1> %m, i32 %vl)
+  %c = call <vscale x 2 x i16> @llvm.vp.lshr.nxv2i16(<vscale x 2 x i16> %b, <vscale x 2 x i16> splat (i16 1), <vscale x 2 x i1> %m, i32 %vl)
+  %d = call <vscale x 2 x i8> @llvm.vp.trunc.nxv2i8.nxv2i16(<vscale x 2 x i16> %c, <vscale x 2 x i1> %m, i32 %vl)
+  ret <vscale x 2 x i8> %d
+}
+
+; Test where the size is reduced by 4x instead of 2x.
+define <vscale x 2 x i8> @vaaddu_7(<vscale x 2 x i8> %x, <vscale x 2 x i8> %y, <vscale x 2 x i1> %m, i32 zeroext %vl) {
+; CHECK-LABEL: vaaddu_7:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e8, mf4, ta, ma
+; CHECK-NEXT:    vaaddu.vv v8, v8, v9, v0.t
+; CHECK-NEXT:    ret
+  %xz = call <vscale x 2 x i32> @llvm.vp.zext.nxv2i32.nxv2i8(<vscale x 2 x i8> %x, <vscale x 2 x i1> %m, i32 %vl)
+  %yz = call <vscale x 2 x i32> @llvm.vp.zext.nxv2i32.nxv2i8(<vscale x 2 x i8> %y, <vscale x 2 x i1> %m, i32 %vl)
+  %a = call <vscale x 2 x i32> @llvm.vp.add.nxv2i32(<vscale x 2 x i32> %xz, <vscale x 2 x i32> %yz, <vscale x 2 x i1> %m, i32 %vl)
+  %b = call <vscale x 2 x i32> @llvm.vp.add.nxv2i32(<vscale x 2 x i32> %a, <vscale x 2 x i32> splat (i32 1), <vscale x 2 x i1> %m, i32 %vl)
+  %c = call <vscale x 2 x i32> @llvm.vp.lshr.nxv2i32(<vscale x 2 x i32> %b, <vscale x 2 x i32> splat (i32 1), <vscale x 2 x i1> %m, i32 %vl)
+  %d = call <vscale x 2 x i8> @llvm.vp.trunc.nxv2i8.nxv2i32(<vscale x 2 x i32> %c, <vscale x 2 x i1> %m, i32 %vl)
+  ret <vscale x 2 x i8> %d
+}
+
+; Test where the zext can't be completely removed.
+define <vscale x 2 x i16> @vaaddu_8(<vscale x 2 x i8> %x, <vscale x 2 x i8> %y, <vscale x 2 x i1> %m, i32 zeroext %vl) {
+; CHECK-LABEL: vaaddu_8:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, ta, ma
+; CHECK-NEXT:    vzext.vf2 v10, v8, v0.t
+; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vzext.vf2 v8, v9, v0.t
+; CHECK-NEXT:    vaaddu.vv v8, v10, v8, v0.t
+; CHECK-NEXT:    ret
+  %xz = call <vscale x 2 x i32> @llvm.vp.zext.nxv2i32.nxv2i8(<vscale x 2 x i8> %x, <vscale x 2 x i1> %m, i32 %vl)
+  %yz = call <vscale x 2 x i32> @llvm.vp.zext.nxv2i32.nxv2i8(<vscale x 2 x i8> %y, <vscale x 2 x i1> %m, i32 %vl)
+  %a = call <vscale x 2 x i32> @llvm.vp.add.nxv2i32(<vscale x 2 x i32> %xz, <vscale x 2 x i32> %yz, <vscale x 2 x i1> %m, i32 %vl)
+  %b = call <vscale x 2 x i32> @llvm.vp.add.nxv2i32(<vscale x 2 x i32> %a, <vscale x 2 x i32> splat (i32 1), <vscale x 2 x i1> %m, i32 %vl)
+  %c = call <vscale x 2 x i32> @llvm.vp.lshr.nxv2i32(<vscale x 2 x i32> %b, <vscale x 2 x i32> splat (i32 1), <vscale x 2 x i1> %m, i32 %vl)
+  %d = call <vscale x 2 x i16> @llvm.vp.trunc.nxv2i16.nxv2i32(<vscale x 2 x i32> %c, <vscale x 2 x i1> %m, i32 %vl)
+  ret <vscale x 2 x i16> %d
+}
+
+; Negative test. The truncate has a smaller type than the zero extend.
+; TODO: Could still handle this by truncating after an i16 vaaddu.
+define <vscale x 2 x i8> @vaaddu_9(<vscale x 2 x i16> %x, <vscale x 2 x i16> %y, <vscale x 2 x i1> %m, i32 zeroext %vl) {
+; CHECK-LABEL: vaaddu_9:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, ta, ma
+; CHECK-NEXT:    vwaddu.vv v10, v8, v9, v0.t
+; CHECK-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
+; CHECK-NEXT:    vadd.vi v8, v10, 1, v0.t
+; CHECK-NEXT:    vsrl.vi v8, v8, 1, v0.t
+; CHECK-NEXT:    vsetvli zero, zero, e16, mf2, ta, ma
+; CHECK-NEXT:    vnsrl.wi v8, v8, 0, v0.t
+; CHECK-NEXT:    vsetvli zero, zero, e8, mf4, ta, ma
+; CHECK-NEXT:    vnsrl.wi v8, v8, 0, v0.t
+; CHECK-NEXT:    ret
+  %xz = call <vscale x 2 x i32> @llvm.vp.zext.nxv2i32.nxv2i16(<vscale x 2 x i16> %x, <vscale x 2 x i1> %m, i32 %vl)
+  %yz = call <vscale x 2 x i32> @llvm.vp.zext.nxv2i32.nxv2i16(<vscale x 2 x i16> %y, <vscale x 2 x i1> %m, i32 %vl)
+  %a = call <vscale x 2 x i32> @llvm.vp.add.nxv2i32(<vscale x 2 x i32> %xz, <vscale x 2 x i32> %yz, <vscale x 2 x i1> %m, i32 %vl)
+  %b = call <vscale x 2 x i32> @llvm.vp.add.nxv2i32(<vscale x 2 x i32> %a, <vscale x 2 x i32> splat (i32 1), <vscale x 2 x i1> %m, i32 %vl)
+  %c = call <vscale x 2 x i32> @llvm.vp.lshr.nxv2i32(<vscale x 2 x i32> %b, <vscale x 2 x i32> splat (i32 1), <vscale x 2 x i1> %m, i32 %vl)
+  %d = call <vscale x 2 x i8> @llvm.vp.trunc.nxv2i8.nxv2i32(<vscale x 2 x i32> %c, <vscale x 2 x i1> %m, i32 %vl)
+  ret <vscale x 2 x i8> %d
+}
diff --git a/llvm/test/CodeGen/RISCV/rvv/vp-vector-interleaved-access.ll b/llvm/test/CodeGen/RISCV/rvv/vp-vector-interleaved-access.ll
new file mode 100644
index 0000000000000..e481891dfd52f
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/rvv/vp-vector-interleaved-access.ll
@@ -0,0 +1,816 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 2
+; RUN: llc < %s -mtriple=riscv32 -mattr=+v,m -O2 | FileCheck -check-prefixes=CHECK,RV32 %s
+; RUN: llc < %s -mtriple=riscv64 -mattr=+v,m -O2 | FileCheck -check-prefixes=CHECK,RV64 %s
+
+define {<vscale x 2 x i32>, <vscale x 2 x i32>} @load_factor2_v2(ptr %ptr, i32 %evl) {
+; RV32-LABEL: load_factor2_v2:
+; RV32:       # %bb.0:
+; RV32-NEXT:    slli a1, a1, 1
+; RV32-NEXT:    srli a1, a1, 1
+; RV32-NEXT:    vsetvli zero, a1, e32, m1, ta, ma
+; RV32-NEXT:    vlseg2e32.v v8, (a0)
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: load_factor2_v2:
+; RV64:       # %bb.0:
+; RV64-NEXT:    slli a1, a1, 33
+; RV64-NEXT:    srli a1, a1, 33
+; RV64-NEXT:    vsetvli zero, a1, e32, m1, ta, ma
+; RV64-NEXT:    vlseg2e32.v v8, (a0)
+; RV64-NEXT:    ret
+  %rvl = mul i32 %evl, 2
+  %wide.masked.load = call <vscale x 4 x i32> @llvm.vp.load.nxv4i32.p0(ptr %ptr, <vscale x 4 x i1> splat (i1 true), i32 %rvl)
+  %deinterleaved.results = call { <vscale x 2 x i32>, <vscale x 2 x i32> } @llvm.vector.deinterleave2.nxv4i32(<vscale x 4 x i32> %wide.masked.load)
+  %t0 = extractvalue { <vscale x 2 x i32>, <vscale x 2 x i32> } %deinterleaved.results, 0
+  %t1 = extractvalue { <vscale x 2 x i32>, <vscale x 2 x i32> } %deinterleaved.results, 1
+  %res0 = insertvalue { <vscale x 2 x i32>, <vscale x 2 x i32> } poison, <vscale x 2 x i32> %t0, 0
+  %res1 = insertvalue { <vscale x 2 x i32>, <vscale x 2 x i32> } %res0, <vscale x 2 x i32> %t1, 1
+  ret { <vscale x 2 x i32>, <vscale x 2 x i32> } %res1
+}
+
+define {<vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>} @load_factor4_v2(ptr %ptr, i32 %evl) {
+; RV32-LABEL: load_factor4_v2:
+; RV32:       # %bb.0:
+; RV32-NEXT:    slli a1, a1, 2
+; RV32-NEXT:    srli a1, a1, 2
+; RV32-NEXT:    vsetvli zero, a1, e32, m1, ta, ma
+; RV32-NEXT:    vlseg4e32.v v8, (a0)
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: load_factor4_v2:
+; RV64:       # %bb.0:
+; RV64-NEXT:    slli a1, a1, 34
+; RV64-NEXT:    srli a1, a1, 34
+; RV64-NEXT:    vsetvli zero, a1, e32, m1, ta, ma
+; RV64-NEXT:    vlseg4e32.v v8, (a0)
+; RV64-NEXT:    ret
+  %rvl = mul i32 %evl, 4
+  %wide.masked.load = call <vscale x 8 x i32> @llvm.vp.load.nxv8i32.p0(ptr %ptr, <vscale x 8 x i1> splat (i1 true), i32 %rvl)
+  %d0 = call { <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.vector.deinterleave2.nxv8i32(<vscale x 8 x i32> %wide.masked.load)
+  %d0.0 = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } %d0, 0
+  %d0.1 = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } %d0, 1
+  %d1 = call { <vscale x 2 x i32>, <vscale x 2 x i32> } @llvm.vector.deinterleave2.nxv4i32(<vscale x 4 x i32> %d0.0)
+  %t0 = extractvalue { <vscale x 2 x i32>, <vscale x 2 x i32> } %d1, 0
+  %t2 = extractvalue { <vscale x 2 x i32>, <vscale x 2 x i32> } %d1, 1
+  %d2 = call { <vscale x 2 x i32>, <vscale x 2 x i32> } @llvm.vector.deinterleave2.nxv4i32(<vscale x 4 x i32> %d0.1)
+  %t1 = extractvalue { <vscale x 2 x i32>, <vscale x 2 x i32> } %d2, 0
+  %t3 = extractvalue { <vscale x 2 x i32>, <vscale x 2 x i32> } %d2, 1
+
+  %res0 = insertvalue { <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32> } poison, <vscale x 2 x i32> %t0, 0
+  %res1 = insertvalue { <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32> } %res0, <vscale x 2 x i32> %t1, 1
+  %res2 = insertvalue { <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32> } %res1, <vscale x 2 x i32> %t2, 2
+  %res3 = insertvalue { <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32> } %res2, <vscale x 2 x i32> %t3, 3
+  ret { <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32> } %res3
+}
+
+define {<vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>} @load_factor8_v2(ptr %ptr, i32 %evl) {
+; RV32-LABEL: load_factor8_v2:
+; RV32:       # %bb.0:
+; RV32-NEXT:    slli a1, a1, 3
+; RV32-NEXT:    srli a1, a1, 3
+; RV32-NEXT:    vsetvli zero, a1, e32, m1, ta, ma
+; RV32-NEXT:    vlseg8e32.v v8, (a0)
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: load_factor8_v2:
+; RV64:       # %bb.0:
+; RV64-NEXT:    slli a1, a1, 35
+; RV64-NEXT:    srli a1, a1, 35
+; RV64-NEXT:    vsetvli zero, a1, e32, m1, ta, ma
+; RV64-NEXT:    vlseg8e32.v v8, (a0)
+; RV64-NEXT:    ret
+  %rvl = mul i32 %evl, 8
+  %wide.masked.load = call <vscale x 16 x i32> @llvm.vp.load.nxv16i32.p0(ptr %ptr, <vscale x 16 x i1> splat (i1 true), i32 %rvl)
+  %d0 = call { <vscale x 8 x i32>, <vscale x 8 x i32> } @llvm.vector.deinterleave2.nxv16i32(<vscale x 16 x i32> %wide.masked.load)
+  %d0.0 = extractvalue { <vscale x 8 x i32>, <vscale x 8 x i32> } %d0, 0
+  %d0.1 = extractvalue { <vscale x 8 x i32>, <vscale x 8 x i32> } %d0, 1
+  %d1 = call { <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.vector.deinterleave2.nxv8i32(<vscale x 8 x i32> %d0.0)
+  %d1.0 = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } %d1, 0
+  %d1.1 = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } %d1, 1
+  %d2 = call { <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.vector.deinterleave2.nxv8i32(<vscale x 8 x i32> %d0.1)
+  %d2.0 = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } %d2, 0
+  %d2.1 = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } %d2, 1
+
+  %d3 = call { <vscale x 2 x i32>, <vscale x 2 x i32> } @llvm.vector.deinterleave2.nxv4i32(<vscale x 4 x i32> %d1.0)
+  %t0 = extractvalue { <vscale x 2 x i32>, <vscale x 2 x i32> } %d3, 0
+  %t4 = extractvalue { <vscale x 2 x i32>, <vscale x 2 x i32> } %d3, 1
+  %d4 = call { <vscale x 2 x i32>, <vscale x 2 x i32> } @llvm.vector.deinterleave2.nxv4i32(<vscale x 4 x i32> %d1.1)
+  %t2 = extractvalue { <vscale x 2 x i32>, <vscale x 2 x i32> } %d4, 0
+  %t6 = extractvalue { <vscale x 2 x i32>, <vscale x 2 x i32> } %d4, 1
+  %d5 = call { <vscale x 2 x i32>, <vscale x 2 x i32> } @llvm.vector.deinterleave2.nxv4i32(<vscale x 4 x i32> %d2.0)
+  %t1 = extractvalue { <vscale x 2 x i32>, <vscale x 2 x i32> } %d5, 0
+  %t5 = extractvalue { <vscale x 2 x i32>, <vscale x 2 x i32> } %d5, 1
+  %d6 = call { <vscale x 2 x i32>, <vscale x 2 x i32> } @llvm.vector.deinterleave2.nxv4i32(<vscale x 4 x i32> %d2.1)
+  %t3 = extractvalue { <vscale x 2 x i32>, <vscale x 2 x i32> } %d6, 0
+  %t7 = extractvalue { <vscale x 2 x i32>, <vscale x 2 x i32> } %d6, 1
+
+  %res0 = insertvalue { <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32> } poison, <vscale x 2 x i32> %t0, 0
+  %res1 = insertvalue { <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32> } %res0, <vscale x 2 x i32> %t1, 1
+  %res2 = insertvalue { <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32> } %res1, <vscale x 2 x i32> %t2, 2
+  %res3 = insertvalue { <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32> } %res2, <vscale x 2 x i32> %t3, 3
+  %res4 = insertvalue { <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32> } %res3, <vscale x 2 x i32> %t4, 4
+  %res5 = insertvalue { <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32> } %res4, <vscale x 2 x i32> %t5, 5
+  %res6 = insertvalue { <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32> } %res5, <vscale x 2 x i32> %t6, 6
+  %res7 = insertvalue { <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32> } %res6, <vscale x 2 x i32> %t7, 7
+  ret { <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32> } %res7
+}
+
+define void @store_factor2_v2(<vscale x 1 x i32> %v0, <vscale x 1 x i32> %v1, ptr %ptr, i32 %evl) {
+; RV32-LABEL: store_factor2_v2:
+; RV32:       # %bb.0:
+; RV32-NEXT:    slli a1, a1, 1
+; RV32-NEXT:    srli a1, a1, 1
+; RV32-NEXT:    vsetvli zero, a1, e32, mf2, ta, ma
+; RV32-NEXT:    vsseg2e32.v v8, (a0)
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: store_factor2_v2:
+; RV64:       # %bb.0:
+; RV64-NEXT:    slli a1, a1, 33
+; RV64-NEXT:    srli a1, a1, 33
+; RV64-NEXT:    vsetvli zero, a1, e32, mf2, ta, ma
+; RV64-NEXT:    vsseg2e32.v v8, (a0)
+; RV64-NEXT:    ret
+  %rvl = mul i32 %evl, 2
+  %interleaved.vec = call <vscale x 2 x i32> @llvm.vector.interleave2.nxv2i32(<vscale x 1 x i32> %v0, <vscale x 1 x i32> %v1)
+  call void @llvm.vp.store.nxv2i32.p0(<vscale x 2 x i32> %interleaved.vec, ptr %ptr, <vscale x 2 x i1> splat (i1 true), i32 %rvl)
+  ret void
+}
+
+define void @store_factor4_v2(<vscale x 1 x i32> %v0, <vscale x 1 x i32> %v1, ptr %ptr, i32 %evl) {
+; RV32-LABEL: store_factor4_v2:
+; RV32:       # %bb.0:
+; RV32-NEXT:    slli a1, a1, 3
+; RV32-NEXT:    srli a1, a1, 2
+; RV32-NEXT:    vsetvli zero, a1, e32, mf2, ta, ma
+; RV32-NEXT:    vmv1r.v v10, v8
+; RV32-NEXT:    vmv1r.v v11, v9
+; RV32-NEXT:    vsseg4e32.v v8, (a0)
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: store_factor4_v2:
+; RV64:       # %bb.0:
+; RV64-NEXT:    slli a1, a1, 35
+; RV64-NEXT:    srli a1, a1, 34
+; RV64-NEXT:    vsetvli zero, a1, e32, mf2, ta, ma
+; RV64-NEXT:    vmv1r.v v10, v8
+; RV64-NEXT:    vmv1r.v v11, v9
+; RV64-NEXT:    vsseg4e32.v v8, (a0)
+; RV64-NEXT:    ret
+  %rvl = mul i32 %evl, 8
+  %interleaved.vec0 = call <vscale x 2 x i32> @llvm.vector.interleave2.nxv2i32(<vscale x 1 x i32> %v0, <vscale x 1 x i32> %v0)
+  %interleaved.vec1 = call <vscale x 2 x i32> @llvm.vector.interleave2.nxv2i32(<vscale x 1 x i32> %v1, <vscale x 1 x i32> %v1)
+  %interleaved.vec2 = call <vscale x 4 x i32> @llvm.vector.interleave2.nxv4i32(<vscale x 2 x i32> %interleaved.vec0, <vscale x 2 x i32> %interleaved.vec1)
+  call void @llvm.vp.store.nxv4i32.p0(<vscale x 4 x i32> %interleaved.vec2, ptr %ptr, <vscale x 4 x i1> splat (i1 true), i32 %rvl)
+  ret void
+}
+
+define void @store_factor8_v2(<vscale x 1 x i32> %v0, <vscale x 1 x i32> %v1, ptr %ptr, i32 %evl) {
+; RV32-LABEL: store_factor8_v2:
+; RV32:       # %bb.0:
+; RV32-NEXT:    slli a1, a1, 3
+; RV32-NEXT:    srli a1, a1, 3
+; RV32-NEXT:    vsetvli zero, a1, e32, mf2, ta, ma
+; RV32-NEXT:    vmv1r.v v10, v8
+; RV32-NEXT:    vmv1r.v v11, v9
+; RV32-NEXT:    vmv1r.v v12, v8
+; RV32-NEXT:    vmv1r.v v13, v9
+; RV32-NEXT:    vmv1r.v v14, v8
+; RV32-NEXT:    vmv1r.v v15, v9
+; RV32-NEXT:    vsseg8e32.v v8, (a0)
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: store_factor8_v2:
+; RV64:       # %bb.0:
+; RV64-NEXT:    slli a1, a1, 35
+; RV64-NEXT:    srli a1, a1, 35
+; RV64-NEXT:    vsetvli zero, a1, e32, mf2, ta, ma
+; RV64-NEXT:    vmv1r.v v10, v8
+; RV64-NEXT:    vmv1r.v v11, v9
+; RV64-NEXT:    vmv1r.v v12, v8
+; RV64-NEXT:    vmv1r.v v13, v9
+; RV64-NEXT:    vmv1r.v v14, v8
+; RV64-NEXT:    vmv1r.v v15, v9
+; RV64-NEXT:    vsseg8e32.v v8, (a0)
+; RV64-NEXT:    ret
+  %rvl = mul i32 %evl, 8
+  %interleaved.vec0 = call <vscale x 2 x i32> @llvm.vector.interleave2.nxv2i32(<vscale x 1 x i32> %v0, <vscale x 1 x i32> %v0)
+  %interleaved.vec1 = call <vscale x 2 x i32> @llvm.vector.interleave2.nxv2i32(<vscale x 1 x i32> %v0, <vscale x 1 x i32> %v0)
+  %interleaved.vec2 = call <vscale x 4 x i32> @llvm.vector.interleave2.nxv4i32(<vscale x 2 x i32> %interleaved.vec0, <vscale x 2 x i32> %interleaved.vec1)
+  %interleaved.vec3 = call <vscale x 2 x i32> @llvm.vector.interleave2.nxv2i32(<vscale x 1 x i32> %v1, <vscale x 1 x i32> %v1)
+  %interleaved.vec4 = call <vscale x 2 x i32> @llvm.vector.interleave2.nxv2i32(<vscale x 1 x i32> %v1, <vscale x 1 x i32> %v1)
+  %interleaved.vec5 = call <vscale x 4 x i32> @llvm.vector.interleave2.nxv4i32(<vscale x 2 x i32> %interleaved.vec3, <vscale x 2 x i32> %interleaved.vec4)
+  %interleaved.vec6 = call <vscale x 8 x i32> @llvm.vector.interleave2.nxv8i32(<vscale x 4 x i32> %interleaved.vec2, <vscale x 4 x i32> %interleaved.vec5)
+  call void @llvm.vp.store.nxv8i32.p0(<vscale x 8 x i32> %interleaved.vec6, ptr %ptr, <vscale x 8 x i1> splat (i1 true), i32 %rvl)
+  ret void
+}
+
+define {<vscale x 2 x i32>, <vscale x 2 x i32>} @masked_load_factor2_v2(<vscale x 2 x i1> %mask, ptr %ptr, i32 %evl) {
+; RV32-LABEL: masked_load_factor2_v2:
+; RV32:       # %bb.0:
+; RV32-NEXT:    slli a1, a1, 1
+; RV32-NEXT:    srli a1, a1, 1
+; RV32-NEXT:    vsetvli zero, a1, e32, m1, ta, ma
+; RV32-NEXT:    vlseg2e32.v v8, (a0), v0.t
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: masked_load_factor2_v2:
+; RV64:       # %bb.0:
+; RV64-NEXT:    slli a1, a1, 33
+; RV64-NEXT:    srli a1, a1, 33
+; RV64-NEXT:    vsetvli zero, a1, e32, m1, ta, ma
+; RV64-NEXT:    vlseg2e32.v v8, (a0), v0.t
+; RV64-NEXT:    ret
+  %rvl = mul i32 %evl, 2
+  %interleaved.mask = tail call <vscale x 4 x i1> @llvm.vector.interleave2.nxv4i1(<vscale x 2 x i1> %mask, <vscale x 2 x i1> %mask)
+  %wide.masked.load = tail call <vscale x 4 x i32> @llvm.vp.load.nxv4i32.p0(ptr %ptr, <vscale x 4 x i1> %interleaved.mask, i32 %rvl)
+  %deinterleaved.results = tail call { <vscale x 2 x i32>, <vscale x 2 x i32> } @llvm.vector.deinterleave2.nxv16i32(<vscale x 4 x i32> %wide.masked.load)
+  %t0 = extractvalue { <vscale x 2 x i32>, <vscale x 2 x i32> } %deinterleaved.results, 0
+  %t1 = extractvalue { <vscale x 2 x i32>, <vscale x 2 x i32> } %deinterleaved.results, 1
+  %res0 = insertvalue { <vscale x 2 x i32>, <vscale x 2 x i32> } poison, <vscale x 2 x i32> %t0, 0
+  %res1 = insertvalue { <vscale x 2 x i32>, <vscale x 2 x i32> } %res0, <vscale x 2 x i32> %t1, 1
+  ret { <vscale x 2 x i32>, <vscale x 2 x i32> } %res1
+}
+
+define {<vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>} @masked_load_factor4_v2(<vscale x 2 x i1> %mask, ptr %ptr, i32 %evl) {
+; RV32-LABEL: masked_load_factor4_v2:
+; RV32:       # %bb.0:
+; RV32-NEXT:    slli a1, a1, 2
+; RV32-NEXT:    srli a1, a1, 2
+; RV32-NEXT:    vsetvli zero, a1, e32, m1, ta, ma
+; RV32-NEXT:    vlseg4e32.v v8, (a0), v0.t
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: masked_load_factor4_v2:
+; RV64:       # %bb.0:
+; RV64-NEXT:    slli a1, a1, 34
+; RV64-NEXT:    srli a1, a1, 34
+; RV64-NEXT:    vsetvli zero, a1, e32, m1, ta, ma
+; RV64-NEXT:    vlseg4e32.v v8, (a0), v0.t
+; RV64-NEXT:    ret
+  %rvl = mul i32 %evl, 4
+  %interleaved.mask0 = call <vscale x 4 x i1> @llvm.vector.interleave2.nxv4i1(<vscale x 2 x i1> %mask, <vscale x 2 x i1> %mask)
+  %interleaved.mask1 = call <vscale x 4 x i1> @llvm.vector.interleave2.nxv4i1(<vscale x 2 x i1> %mask, <vscale x 2 x i1> %mask)
+  %interleaved.mask2 = call <vscale x 8 x i1> @llvm.vector.interleave2.nxv8i1(<vscale x 4 x i1> %interleaved.mask0, <vscale x 4 x i1> %interleaved.mask1)
+  %wide.masked.load = call <vscale x 8 x i32> @llvm.vp.load.nxv8i32.p0(ptr %ptr, <vscale x 8 x i1> %interleaved.mask2, i32 %rvl)
+  %d0 = call { <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.vector.deinterleave2.nxv8i32(<vscale x 8 x i32> %wide.masked.load)
+  %d0.0 = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } %d0, 0
+  %d0.1 = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } %d0, 1
+  %d1 = call { <vscale x 2 x i32>, <vscale x 2 x i32> } @llvm.vector.deinterleave2.nxv4i32(<vscale x 4 x i32> %d0.0)
+  %t0 = extractvalue { <vscale x 2 x i32>, <vscale x 2 x i32> } %d1, 0
+  %t2 = extractvalue { <vscale x 2 x i32>, <vscale x 2 x i32> } %d1, 1
+  %d2 = call { <vscale x 2 x i32>, <vscale x 2 x i32> } @llvm.vector.deinterleave2.nxv4i32(<vscale x 4 x i32> %d0.1)
+  %t1 = extractvalue { <vscale x 2 x i32>, <vscale x 2 x i32> } %d2, 0
+  %t3 = extractvalue { <vscale x 2 x i32>, <vscale x 2 x i32> } %d2, 1
+
+  %res0 = insertvalue { <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32> } poison, <vscale x 2 x i32> %t0, 0
+  %res1 = insertvalue { <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32> } %res0, <vscale x 2 x i32> %t1, 1
+  %res2 = insertvalue { <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32> } %res1, <vscale x 2 x i32> %t2, 2
+  %res3 = insertvalue { <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32> } %res2, <vscale x 2 x i32> %t3, 3
+  ret { <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32> } %res3
+}
+
+define void @masked_store_factor2_v2(<vscale x 1 x i1> %mask, <vscale x 1 x i32> %v0, <vscale x 1 x i32> %v1, ptr %ptr, i32 %evl) {
+; RV32-LABEL: masked_store_factor2_v2:
+; RV32:       # %bb.0:
+; RV32-NEXT:    slli a1, a1, 1
+; RV32-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
+; RV32-NEXT:    vmv1r.v v9, v8
+; RV32-NEXT:    srli a1, a1, 1
+; RV32-NEXT:    vsetvli zero, a1, e32, mf2, ta, ma
+; RV32-NEXT:    vsseg2e32.v v8, (a0), v0.t
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: masked_store_factor2_v2:
+; RV64:       # %bb.0:
+; RV64-NEXT:    slli a1, a1, 33
+; RV64-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
+; RV64-NEXT:    vmv1r.v v9, v8
+; RV64-NEXT:    srli a1, a1, 33
+; RV64-NEXT:    vsetvli zero, a1, e32, mf2, ta, ma
+; RV64-NEXT:    vsseg2e32.v v8, (a0), v0.t
+; RV64-NEXT:    ret
+  %rvl = mul i32 %evl, 2
+  %interleaved.mask = tail call <vscale x 2 x i1> @llvm.vector.interleave2.nxv2i1(<vscale x 1 x i1> %mask, <vscale x 1 x i1> %mask)
+  %interleaved.vec = tail call <vscale x 2 x i32> @llvm.vector.interleave2.nxv2i32(<vscale x 1 x i32> %v0, <vscale x 1 x i32> %v0)
+  tail call void @llvm.vp.store.nxv2i32.p0(<vscale x 2 x i32> %interleaved.vec, ptr %ptr, <vscale x 2 x i1> %interleaved.mask, i32 %rvl)
+  ret void
+}
+
+define void @masked_load_store_factor2_v2_shared_mask(<vscale x 2 x i1> %mask, ptr %ptr, i32 %evl) {
+; RV32-LABEL: masked_load_store_factor2_v2_shared_mask:
+; RV32:       # %bb.0:
+; RV32-NEXT:    slli a1, a1, 1
+; RV32-NEXT:    srli a1, a1, 1
+; RV32-NEXT:    vsetvli zero, a1, e32, m1, ta, ma
+; RV32-NEXT:    vlseg2e32.v v8, (a0), v0.t
+; RV32-NEXT:    vsseg2e32.v v8, (a0), v0.t
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: masked_load_store_factor2_v2_shared_mask:
+; RV64:       # %bb.0:
+; RV64-NEXT:    slli a1, a1, 33
+; RV64-NEXT:    srli a1, a1, 33
+; RV64-NEXT:    vsetvli zero, a1, e32, m1, ta, ma
+; RV64-NEXT:    vlseg2e32.v v8, (a0), v0.t
+; RV64-NEXT:    vsseg2e32.v v8, (a0), v0.t
+; RV64-NEXT:    ret
+  %rvl = mul i32 %evl, 2
+  %interleaved.mask = tail call <vscale x 4 x i1> @llvm.vector.interleave2.nxv4i1(<vscale x 2 x i1> %mask, <vscale x 2 x i1> %mask)
+  %wide.masked.load = tail call <vscale x 4 x i32> @llvm.vp.load.nxv4i32.p0(ptr %ptr, <vscale x 4 x i1> %interleaved.mask, i32 %rvl)
+  %deinterleaved.results = tail call { <vscale x 2 x i32>, <vscale x 2 x i32> } @llvm.vector.deinterleave2.nxv16i32(<vscale x 4 x i32> %wide.masked.load)
+  %t0 = extractvalue { <vscale x 2 x i32>, <vscale x 2 x i32> } %deinterleaved.results, 0
+  %t1 = extractvalue { <vscale x 2 x i32>, <vscale x 2 x i32> } %deinterleaved.results, 1
+  %interleaved.vec = tail call <vscale x 4 x i32> @llvm.vector.interleave2.nxv4i32(<vscale x 2 x i32> %t0, <vscale x 2 x i32> %t1)
+  tail call void @llvm.vp.store.nxv4i32.p0(<vscale x 4 x i32> %interleaved.vec, ptr %ptr, <vscale x 4 x i1> %interleaved.mask, i32 %rvl)
+  ret void
+}
+
+define i32 @masked_load_store_factor2_v2_shared_mask_extract(<vscale x 2 x i1> %mask, ptr %ptr, i32 %evl) {
+; RV32-LABEL: masked_load_store_factor2_v2_shared_mask_extract:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetvli a2, zero, e8, mf4, ta, ma
+; RV32-NEXT:    vmv1r.v v8, v0
+; RV32-NEXT:    vmv.v.i v9, 0
+; RV32-NEXT:    li a2, -1
+; RV32-NEXT:    vsetvli a3, zero, e8, mf2, ta, ma
+; RV32-NEXT:    vmv.v.i v10, 0
+; RV32-NEXT:    csrr a3, vlenb
+; RV32-NEXT:    vsetvli a4, zero, e8, mf4, ta, ma
+; RV32-NEXT:    vmerge.vim v11, v9, 1, v0
+; RV32-NEXT:    srli a3, a3, 2
+; RV32-NEXT:    vwaddu.vv v12, v11, v11
+; RV32-NEXT:    vwmaccu.vx v12, a2, v11
+; RV32-NEXT:    vmsne.vi v0, v12, 0
+; RV32-NEXT:    vsetvli a2, zero, e8, mf2, ta, ma
+; RV32-NEXT:    vslidedown.vx v11, v12, a3
+; RV32-NEXT:    vmerge.vim v10, v10, 1, v0
+; RV32-NEXT:    vsetvli a2, zero, e8, mf4, ta, ma
+; RV32-NEXT:    vmsne.vi v0, v11, 0
+; RV32-NEXT:    add a2, a3, a3
+; RV32-NEXT:    vmerge.vim v9, v9, 1, v0
+; RV32-NEXT:    vsetvli zero, a2, e8, mf2, ta, ma
+; RV32-NEXT:    vslideup.vx v10, v9, a3
+; RV32-NEXT:    vsetvli a2, zero, e8, mf2, ta, ma
+; RV32-NEXT:    vmsne.vi v0, v10, 0
+; RV32-NEXT:    slli a2, a1, 1
+; RV32-NEXT:    vsetvli zero, a2, e32, m2, ta, ma
+; RV32-NEXT:    vle32.v v10, (a0), v0.t
+; RV32-NEXT:    li a1, 32
+; RV32-NEXT:    vsetvli a3, zero, e32, m1, ta, ma
+; RV32-NEXT:    vnsrl.wx v13, v10, a1
+; RV32-NEXT:    vmv.x.s a1, v10
+; RV32-NEXT:    vnsrl.wi v12, v10, 0
+; RV32-NEXT:    srli a2, a2, 1
+; RV32-NEXT:    vmv1r.v v0, v8
+; RV32-NEXT:    vsetvli zero, a2, e32, m1, ta, ma
+; RV32-NEXT:    vsseg2e32.v v12, (a0), v0.t
+; RV32-NEXT:    mv a0, a1
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: masked_load_store_factor2_v2_shared_mask_extract:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetvli a2, zero, e8, mf4, ta, ma
+; RV64-NEXT:    vmv1r.v v8, v0
+; RV64-NEXT:    vmv.v.i v9, 0
+; RV64-NEXT:    li a2, -1
+; RV64-NEXT:    vsetvli a3, zero, e8, mf2, ta, ma
+; RV64-NEXT:    vmv.v.i v10, 0
+; RV64-NEXT:    csrr a3, vlenb
+; RV64-NEXT:    slli a4, a1, 33
+; RV64-NEXT:    vsetvli a1, zero, e8, mf4, ta, ma
+; RV64-NEXT:    vmerge.vim v11, v9, 1, v0
+; RV64-NEXT:    srli a3, a3, 2
+; RV64-NEXT:    vwaddu.vv v12, v11, v11
+; RV64-NEXT:    vwmaccu.vx v12, a2, v11
+; RV64-NEXT:    vmsne.vi v0, v12, 0
+; RV64-NEXT:    vsetvli a1, zero, e8, mf2, ta, ma
+; RV64-NEXT:    vslidedown.vx v11, v12, a3
+; RV64-NEXT:    vmerge.vim v10, v10, 1, v0
+; RV64-NEXT:    vsetvli a1, zero, e8, mf4, ta, ma
+; RV64-NEXT:    vmsne.vi v0, v11, 0
+; RV64-NEXT:    add a1, a3, a3
+; RV64-NEXT:    vmerge.vim v9, v9, 1, v0
+; RV64-NEXT:    vsetvli zero, a1, e8, mf2, ta, ma
+; RV64-NEXT:    vslideup.vx v10, v9, a3
+; RV64-NEXT:    vsetvli a1, zero, e8, mf2, ta, ma
+; RV64-NEXT:    vmsne.vi v0, v10, 0
+; RV64-NEXT:    srli a1, a4, 32
+; RV64-NEXT:    vsetvli zero, a1, e32, m2, ta, ma
+; RV64-NEXT:    vle32.v v10, (a0), v0.t
+; RV64-NEXT:    li a1, 32
+; RV64-NEXT:    vsetvli a2, zero, e32, m1, ta, ma
+; RV64-NEXT:    vnsrl.wx v13, v10, a1
+; RV64-NEXT:    vmv.x.s a1, v10
+; RV64-NEXT:    vnsrl.wi v12, v10, 0
+; RV64-NEXT:    srli a4, a4, 33
+; RV64-NEXT:    vmv1r.v v0, v8
+; RV64-NEXT:    vsetvli zero, a4, e32, m1, ta, ma
+; RV64-NEXT:    vsseg2e32.v v12, (a0), v0.t
+; RV64-NEXT:    mv a0, a1
+; RV64-NEXT:    ret
+  %rvl = mul i32 %evl, 2
+  %interleaved.mask = tail call <vscale x 4 x i1> @llvm.vector.interleave2.nxv4i1(<vscale x 2 x i1> %mask, <vscale x 2 x i1> %mask)
+  %wide.masked.load = tail call <vscale x 4 x i32> @llvm.vp.load.nxv4i32.p0(ptr %ptr, <vscale x 4 x i1> %interleaved.mask, i32 %rvl)
+  %deinterleaved.results = tail call { <vscale x 2 x i32>, <vscale x 2 x i32> } @llvm.vector.deinterleave2.nxv16i32(<vscale x 4 x i32> %wide.masked.load)
+  %t0 = extractvalue { <vscale x 2 x i32>, <vscale x 2 x i32> } %deinterleaved.results, 0
+  %t1 = extractvalue { <vscale x 2 x i32>, <vscale x 2 x i32> } %deinterleaved.results, 1
+  %r0 = extractelement <vscale x 4 x i32> %wide.masked.load, i32 0
+  %interleaved.vec = tail call <vscale x 4 x i32> @llvm.vector.interleave2.nxv4i32(<vscale x 2 x i32> %t0, <vscale x 2 x i32> %t1)
+  tail call void @llvm.vp.store.nxv4i32.p0(<vscale x 4 x i32> %interleaved.vec, ptr %ptr, <vscale x 4 x i1> %interleaved.mask, i32 %rvl)
+  ret i32 %r0
+}
+
+define void @masked_store_factor4_v2(<vscale x 1 x i1> %mask, <vscale x 1 x i32> %v0, <vscale x 1 x i32> %v1, ptr %ptr, i32 %evl) {
+; RV32-LABEL: masked_store_factor4_v2:
+; RV32:       # %bb.0:
+; RV32-NEXT:    slli a1, a1, 2
+; RV32-NEXT:    srli a1, a1, 2
+; RV32-NEXT:    vsetvli zero, a1, e32, mf2, ta, ma
+; RV32-NEXT:    vmv1r.v v10, v8
+; RV32-NEXT:    vmv1r.v v11, v9
+; RV32-NEXT:    vsseg4e32.v v8, (a0), v0.t
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: masked_store_factor4_v2:
+; RV64:       # %bb.0:
+; RV64-NEXT:    slli a1, a1, 34
+; RV64-NEXT:    srli a1, a1, 34
+; RV64-NEXT:    vsetvli zero, a1, e32, mf2, ta, ma
+; RV64-NEXT:    vmv1r.v v10, v8
+; RV64-NEXT:    vmv1r.v v11, v9
+; RV64-NEXT:    vsseg4e32.v v8, (a0), v0.t
+; RV64-NEXT:    ret
+  %rvl = mul i32 %evl, 4
+  %interleaved.mask0 = call <vscale x 2 x i1> @llvm.vector.interleave2.nxv2i1(<vscale x 1 x i1> %mask, <vscale x 1 x i1> %mask)
+  %interleaved.mask1 = call <vscale x 2 x i1> @llvm.vector.interleave2.nxv2i1(<vscale x 1 x i1> %mask, <vscale x 1 x i1> %mask)
+  %interleaved.mask2 = call <vscale x 4 x i1> @llvm.vector.interleave2.nxv4i1(<vscale x 2 x i1> %interleaved.mask0, <vscale x 2 x i1> %interleaved.mask1)
+  %interleaved.vec0 = call <vscale x 2 x i32> @llvm.vector.interleave2.nxv2i32(<vscale x 1 x i32> %v0, <vscale x 1 x i32> %v0)
+  %interleaved.vec1 = call <vscale x 2 x i32> @llvm.vector.interleave2.nxv2i32(<vscale x 1 x i32> %v1, <vscale x 1 x i32> %v1)
+  %interleaved.vec2 = call <vscale x 4 x i32> @llvm.vector.interleave2.nxv4i32(<vscale x 2 x i32> %interleaved.vec0, <vscale x 2 x i32> %interleaved.vec1)
+  call void @llvm.vp.store.nxv4i32.p0(<vscale x 4 x i32> %interleaved.vec2, ptr %ptr, <vscale x 4 x i1> %interleaved.mask2, i32 %rvl)
+  ret void
+}
+
+; Negative tests
+
+; We should not transform this function because the deinterleave tree is not in a desired form.
+define {<vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>} @incorrect_extract_value_index(ptr %ptr, i32 %evl) {
+; RV32-LABEL: incorrect_extract_value_index:
+; RV32:       # %bb.0:
+; RV32-NEXT:    slli a1, a1, 2
+; RV32-NEXT:    vsetvli zero, a1, e32, m4, ta, ma
+; RV32-NEXT:    vle32.v v8, (a0)
+; RV32-NEXT:    li a0, 32
+; RV32-NEXT:    vsetvli a1, zero, e32, m2, ta, ma
+; RV32-NEXT:    vnsrl.wi v12, v8, 0
+; RV32-NEXT:    vsetvli a1, zero, e32, m1, ta, ma
+; RV32-NEXT:    vnsrl.wx v9, v12, a0
+; RV32-NEXT:    vnsrl.wi v8, v12, 0
+; RV32-NEXT:    vmv.v.v v10, v9
+; RV32-NEXT:    vmv.v.v v11, v9
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: incorrect_extract_value_index:
+; RV64:       # %bb.0:
+; RV64-NEXT:    slli a1, a1, 34
+; RV64-NEXT:    srli a1, a1, 32
+; RV64-NEXT:    vsetvli zero, a1, e32, m4, ta, ma
+; RV64-NEXT:    vle32.v v8, (a0)
+; RV64-NEXT:    li a0, 32
+; RV64-NEXT:    vsetvli a1, zero, e32, m2, ta, ma
+; RV64-NEXT:    vnsrl.wi v12, v8, 0
+; RV64-NEXT:    vsetvli a1, zero, e32, m1, ta, ma
+; RV64-NEXT:    vnsrl.wx v9, v12, a0
+; RV64-NEXT:    vnsrl.wi v8, v12, 0
+; RV64-NEXT:    vmv.v.v v10, v9
+; RV64-NEXT:    vmv.v.v v11, v9
+; RV64-NEXT:    ret
+  %rvl = mul i32 %evl, 4
+  %wide.masked.load = call <vscale x 8 x i32> @llvm.vp.load.nxv8i32.p0(ptr %ptr, <vscale x 8 x i1> splat (i1 true), i32 %rvl)
+  %d0 = call { <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.vector.deinterleave2.nxv8i32(<vscale x 8 x i32> %wide.masked.load)
+  %d0.0 = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } %d0, 0
+  %d0.1 = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } %d0, 0
+  %d1 = call { <vscale x 2 x i32>, <vscale x 2 x i32> } @llvm.vector.deinterleave2.nxv4i32(<vscale x 4 x i32> %d0.0)
+  %t0 = extractvalue { <vscale x 2 x i32>, <vscale x 2 x i32> } %d1, 0
+  %t2 = extractvalue { <vscale x 2 x i32>, <vscale x 2 x i32> } %d1, 1
+  %d2 = call { <vscale x 2 x i32>, <vscale x 2 x i32> } @llvm.vector.deinterleave2.nxv4i32(<vscale x 4 x i32> %d0.1)
+  %t1 = extractvalue { <vscale x 2 x i32>, <vscale x 2 x i32> } %d2, 1
+  %t3 = extractvalue { <vscale x 2 x i32>, <vscale x 2 x i32> } %d2, 1
+
+  %res0 = insertvalue { <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32> } poison, <vscale x 2 x i32> %t0, 0
+  %res1 = insertvalue { <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32> } %res0, <vscale x 2 x i32> %t1, 1
+  %res2 = insertvalue { <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32> } %res1, <vscale x 2 x i32> %t2, 2
+  %res3 = insertvalue { <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32> } %res2, <vscale x 2 x i32> %t3, 3
+  ret { <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32> } %res3
+}
+
+; We should not transform this function because the expression is not a balanced tree.
+define {<vscale x 4 x i32>, <vscale x 2 x i32>, <vscale x 1 x i32>, <vscale x 1 x i32>} @not_balanced_load_tree(ptr %ptr, i32 %evl) {
+; RV32-LABEL: not_balanced_load_tree:
+; RV32:       # %bb.0:
+; RV32-NEXT:    slli a1, a1, 2
+; RV32-NEXT:    vsetvli zero, a1, e32, m4, ta, ma
+; RV32-NEXT:    vle32.v v12, (a0)
+; RV32-NEXT:    li a0, 32
+; RV32-NEXT:    vsetvli a1, zero, e32, m2, ta, ma
+; RV32-NEXT:    vnsrl.wx v8, v12, a0
+; RV32-NEXT:    vnsrl.wi v16, v12, 0
+; RV32-NEXT:    vsetvli a1, zero, e32, m1, ta, ma
+; RV32-NEXT:    vnsrl.wi v10, v16, 0
+; RV32-NEXT:    vnsrl.wx v11, v16, a0
+; RV32-NEXT:    vsetvli a1, zero, e32, mf2, ta, ma
+; RV32-NEXT:    vnsrl.wx v12, v11, a0
+; RV32-NEXT:    vnsrl.wi v11, v11, 0
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: not_balanced_load_tree:
+; RV64:       # %bb.0:
+; RV64-NEXT:    slli a1, a1, 34
+; RV64-NEXT:    srli a1, a1, 32
+; RV64-NEXT:    vsetvli zero, a1, e32, m4, ta, ma
+; RV64-NEXT:    vle32.v v12, (a0)
+; RV64-NEXT:    li a0, 32
+; RV64-NEXT:    vsetvli a1, zero, e32, m2, ta, ma
+; RV64-NEXT:    vnsrl.wx v8, v12, a0
+; RV64-NEXT:    vnsrl.wi v16, v12, 0
+; RV64-NEXT:    vsetvli a1, zero, e32, m1, ta, ma
+; RV64-NEXT:    vnsrl.wi v10, v16, 0
+; RV64-NEXT:    vnsrl.wx v11, v16, a0
+; RV64-NEXT:    vsetvli a1, zero, e32, mf2, ta, ma
+; RV64-NEXT:    vnsrl.wx v12, v11, a0
+; RV64-NEXT:    vnsrl.wi v11, v11, 0
+; RV64-NEXT:    ret
+  %rvl = mul i32 %evl, 4
+  %wide.masked.load = call <vscale x 8 x i32> @llvm.vp.load.nxv8i32.p0(ptr %ptr, <vscale x 8 x i1> splat (i1 true), i32 %rvl)
+  %d0 = call { <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.vector.deinterleave2.nxv8i32(<vscale x 8 x i32> %wide.masked.load)
+  %d0.0 = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } %d0, 0
+  %t0 = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } %d0, 1
+  %d1 = call { <vscale x 2 x i32>, <vscale x 2 x i32> } @llvm.vector.deinterleave2.nxv4i32(<vscale x 4 x i32> %d0.0)
+  %t1 = extractvalue { <vscale x 2 x i32>, <vscale x 2 x i32> } %d1, 0
+  %d1.1 = extractvalue { <vscale x 2 x i32>, <vscale x 2 x i32> } %d1, 1
+  %d2 = call { <vscale x 1 x i32>, <vscale x 1 x i32> } @llvm.vector.deinterleave2.nxv4i32(<vscale x 2 x i32> %d1.1)
+  %t2 = extractvalue { <vscale x 1 x i32>, <vscale x 1 x i32> } %d2, 0
+  %t3 = extractvalue { <vscale x 1 x i32>, <vscale x 1 x i32> } %d2, 1
+
+  %res0 = insertvalue { <vscale x 4 x i32>, <vscale x 2 x i32>, <vscale x 1 x i32>, <vscale x 1 x i32> } poison, <vscale x 4 x i32> %t0, 0
+  %res1 = insertvalue { <vscale x 4 x i32>, <vscale x 2 x i32>, <vscale x 1 x i32>, <vscale x 1 x i32> } %res0, <vscale x 2 x i32> %t1, 1
+  %res2 = insertvalue { <vscale x 4 x i32>, <vscale x 2 x i32>, <vscale x 1 x i32>, <vscale x 1 x i32> } %res1, <vscale x 1 x i32> %t2, 2
+  %res3 = insertvalue { <vscale x 4 x i32>, <vscale x 2 x i32>, <vscale x 1 x i32>, <vscale x 1 x i32> } %res2, <vscale x 1 x i32> %t3, 3
+  ret { <vscale x 4 x i32>, <vscale x 2 x i32>, <vscale x 1 x i32>, <vscale x 1 x i32> } %res3
+}
+
+define void @not_balanced_store_tree(<vscale x 1 x i32> %v0, <vscale x 2 x i32> %v1, <vscale x 4 x i32> %v2, ptr %ptr, i32 %evl) {
+; RV32-LABEL: not_balanced_store_tree:
+; RV32:       # %bb.0:
+; RV32-NEXT:    slli a1, a1, 2
+; RV32-NEXT:    vsetvli a2, zero, e32, mf2, ta, ma
+; RV32-NEXT:    vwaddu.vv v12, v8, v8
+; RV32-NEXT:    li a2, -1
+; RV32-NEXT:    csrr a3, vlenb
+; RV32-NEXT:    vwmaccu.vx v12, a2, v8
+; RV32-NEXT:    srli a3, a3, 3
+; RV32-NEXT:    vsetvli a4, zero, e32, m1, ta, ma
+; RV32-NEXT:    vslidedown.vx v8, v12, a3
+; RV32-NEXT:    add a4, a3, a3
+; RV32-NEXT:    vsetvli zero, a4, e32, m1, ta, ma
+; RV32-NEXT:    vslideup.vx v12, v8, a3
+; RV32-NEXT:    vsetvli a3, zero, e32, m1, ta, ma
+; RV32-NEXT:    vwaddu.vv v14, v12, v9
+; RV32-NEXT:    vwmaccu.vx v14, a2, v9
+; RV32-NEXT:    vsetvli a3, zero, e32, m2, ta, ma
+; RV32-NEXT:    vwaddu.vv v16, v14, v10
+; RV32-NEXT:    vwmaccu.vx v16, a2, v10
+; RV32-NEXT:    vsetvli zero, a1, e32, m4, ta, ma
+; RV32-NEXT:    vse32.v v16, (a0)
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: not_balanced_store_tree:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetvli a2, zero, e32, mf2, ta, ma
+; RV64-NEXT:    vwaddu.vv v12, v8, v8
+; RV64-NEXT:    li a2, -1
+; RV64-NEXT:    csrr a3, vlenb
+; RV64-NEXT:    slli a1, a1, 34
+; RV64-NEXT:    vwmaccu.vx v12, a2, v8
+; RV64-NEXT:    srli a3, a3, 3
+; RV64-NEXT:    vsetvli a4, zero, e32, m1, ta, ma
+; RV64-NEXT:    vslidedown.vx v8, v12, a3
+; RV64-NEXT:    add a4, a3, a3
+; RV64-NEXT:    vsetvli zero, a4, e32, m1, ta, ma
+; RV64-NEXT:    vslideup.vx v12, v8, a3
+; RV64-NEXT:    vsetvli a3, zero, e32, m1, ta, ma
+; RV64-NEXT:    vwaddu.vv v14, v12, v9
+; RV64-NEXT:    vwmaccu.vx v14, a2, v9
+; RV64-NEXT:    vsetvli a3, zero, e32, m2, ta, ma
+; RV64-NEXT:    vwaddu.vv v16, v14, v10
+; RV64-NEXT:    vwmaccu.vx v16, a2, v10
+; RV64-NEXT:    srli a1, a1, 32
+; RV64-NEXT:    vsetvli zero, a1, e32, m4, ta, ma
+; RV64-NEXT:    vse32.v v16, (a0)
+; RV64-NEXT:    ret
+  %rvl = mul i32 %evl, 4
+  %interleaved.vec0 = call <vscale x 2 x i32> @llvm.vector.interleave2.nxv2i32(<vscale x 1 x i32> %v0, <vscale x 1 x i32> %v0)
+  %interleaved.vec1 = call <vscale x 4 x i32> @llvm.vector.interleave2.nxv2i32(<vscale x 2 x i32> %interleaved.vec0, <vscale x 2 x i32> %v1)
+  %interleaved.vec2 = call <vscale x 8 x i32> @llvm.vector.interleave2.nxv4i32(<vscale x 4 x i32> %interleaved.vec1, <vscale x 4 x i32> %v2)
+  call void @llvm.vp.store.nxv8i32.p0(<vscale x 8 x i32> %interleaved.vec2, ptr %ptr, <vscale x 8 x i1> splat (i1 true), i32 %rvl)
+  ret void
+}
+
+; We only support scalable vectors for now.
+define {<2 x i32>, <2 x i32>, <2 x i32>, <2 x i32>} @not_scalable_vectors(ptr %ptr, i32 %evl) {
+; RV32-LABEL: not_scalable_vectors:
+; RV32:       # %bb.0:
+; RV32-NEXT:    slli a1, a1, 2
+; RV32-NEXT:    vsetvli zero, a1, e32, m2, ta, ma
+; RV32-NEXT:    vle32.v v8, (a0)
+; RV32-NEXT:    li a0, 32
+; RV32-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
+; RV32-NEXT:    vnsrl.wx v12, v8, a0
+; RV32-NEXT:    vnsrl.wi v11, v8, 0
+; RV32-NEXT:    vsetivli zero, 2, e32, mf2, ta, ma
+; RV32-NEXT:    vnsrl.wx v10, v11, a0
+; RV32-NEXT:    vnsrl.wi v8, v11, 0
+; RV32-NEXT:    vnsrl.wx v11, v12, a0
+; RV32-NEXT:    vnsrl.wi v9, v12, 0
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: not_scalable_vectors:
+; RV64:       # %bb.0:
+; RV64-NEXT:    slli a1, a1, 34
+; RV64-NEXT:    srli a1, a1, 32
+; RV64-NEXT:    vsetvli zero, a1, e32, m2, ta, ma
+; RV64-NEXT:    vle32.v v8, (a0)
+; RV64-NEXT:    li a0, 32
+; RV64-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
+; RV64-NEXT:    vnsrl.wx v12, v8, a0
+; RV64-NEXT:    vnsrl.wi v11, v8, 0
+; RV64-NEXT:    vsetivli zero, 2, e32, mf2, ta, ma
+; RV64-NEXT:    vnsrl.wx v10, v11, a0
+; RV64-NEXT:    vnsrl.wi v8, v11, 0
+; RV64-NEXT:    vnsrl.wx v11, v12, a0
+; RV64-NEXT:    vnsrl.wi v9, v12, 0
+; RV64-NEXT:    ret
+  %rvl = mul i32 %evl, 4
+  %wide.masked.load = call <8 x i32> @llvm.vp.load.v8i32.p0(ptr %ptr, <8 x i1> splat (i1 true), i32 %rvl)
+  %d0 = call { <4 x i32>, <4 x i32> } @llvm.vector.deinterleave2.v8i32(<8 x i32> %wide.masked.load)
+  %d0.0 = extractvalue { <4 x i32>, <4 x i32> } %d0, 0
+  %d0.1 = extractvalue { <4 x i32>, <4 x i32> } %d0, 1
+  %d1 = call { <2 x i32>, <2 x i32> } @llvm.vector.deinterleave2.v4i32(<4 x i32> %d0.0)
+  %t0 = extractvalue { <2 x i32>, <2 x i32> } %d1, 0
+  %t2 = extractvalue { <2 x i32>, <2 x i32> } %d1, 1
+  %d2 = call { <2 x i32>, <2 x i32> } @llvm.vector.deinterleave2.v4i32(<4 x i32> %d0.1)
+  %t1 = extractvalue { <2 x i32>, <2 x i32> } %d2, 0
+  %t3 = extractvalue { <2 x i32>, <2 x i32> } %d2, 1
+
+  %res0 = insertvalue { <2 x i32>, <2 x i32>, <2 x i32>, <2 x i32> } poison, <2 x i32> %t0, 0
+  %res1 = insertvalue { <2 x i32>, <2 x i32>, <2 x i32>, <2 x i32> } %res0, <2 x i32> %t1, 1
+  %res2 = insertvalue { <2 x i32>, <2 x i32>, <2 x i32>, <2 x i32> } %res1, <2 x i32> %t2, 2
+  %res3 = insertvalue { <2 x i32>, <2 x i32>, <2 x i32>, <2 x i32> } %res2, <2 x i32> %t3, 3
+  ret { <2 x i32>, <2 x i32>, <2 x i32>, <2 x i32> } %res3
+}
+
+define {<vscale x 2 x i32>, <vscale x 2 x i32>} @not_same_mask(<vscale x 2 x i1> %mask0, <vscale x 2 x i1> %mask1, ptr %ptr, i32 %evl) {
+; RV32-LABEL: not_same_mask:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetvli a2, zero, e8, mf4, ta, ma
+; RV32-NEXT:    vmv1r.v v9, v0
+; RV32-NEXT:    vmv1r.v v0, v8
+; RV32-NEXT:    vmv.v.i v8, 0
+; RV32-NEXT:    li a2, -1
+; RV32-NEXT:    vsetvli a3, zero, e8, mf2, ta, ma
+; RV32-NEXT:    vmv.v.i v10, 0
+; RV32-NEXT:    csrr a3, vlenb
+; RV32-NEXT:    vsetvli a4, zero, e8, mf4, ta, ma
+; RV32-NEXT:    vmerge.vim v11, v8, 1, v0
+; RV32-NEXT:    vmv1r.v v0, v9
+; RV32-NEXT:    vmerge.vim v9, v8, 1, v0
+; RV32-NEXT:    srli a3, a3, 2
+; RV32-NEXT:    vwaddu.vv v12, v9, v11
+; RV32-NEXT:    vwmaccu.vx v12, a2, v11
+; RV32-NEXT:    vmsne.vi v0, v12, 0
+; RV32-NEXT:    vsetvli a2, zero, e8, mf2, ta, ma
+; RV32-NEXT:    vslidedown.vx v9, v12, a3
+; RV32-NEXT:    vmerge.vim v10, v10, 1, v0
+; RV32-NEXT:    vsetvli a2, zero, e8, mf4, ta, ma
+; RV32-NEXT:    vmsne.vi v0, v9, 0
+; RV32-NEXT:    add a2, a3, a3
+; RV32-NEXT:    vmerge.vim v8, v8, 1, v0
+; RV32-NEXT:    vsetvli zero, a2, e8, mf2, ta, ma
+; RV32-NEXT:    vslideup.vx v10, v8, a3
+; RV32-NEXT:    vsetvli a2, zero, e8, mf2, ta, ma
+; RV32-NEXT:    vmsne.vi v0, v10, 0
+; RV32-NEXT:    slli a1, a1, 1
+; RV32-NEXT:    vsetvli zero, a1, e32, m2, ta, ma
+; RV32-NEXT:    vle32.v v10, (a0), v0.t
+; RV32-NEXT:    li a0, 32
+; RV32-NEXT:    vsetvli a1, zero, e32, m1, ta, ma
+; RV32-NEXT:    vnsrl.wx v9, v10, a0
+; RV32-NEXT:    vnsrl.wi v8, v10, 0
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: not_same_mask:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetvli a2, zero, e8, mf4, ta, ma
+; RV64-NEXT:    vmv1r.v v9, v0
+; RV64-NEXT:    vmv1r.v v0, v8
+; RV64-NEXT:    vmv.v.i v8, 0
+; RV64-NEXT:    li a2, -1
+; RV64-NEXT:    vsetvli a3, zero, e8, mf2, ta, ma
+; RV64-NEXT:    vmv.v.i v10, 0
+; RV64-NEXT:    csrr a3, vlenb
+; RV64-NEXT:    slli a1, a1, 33
+; RV64-NEXT:    vsetvli a4, zero, e8, mf4, ta, ma
+; RV64-NEXT:    vmerge.vim v11, v8, 1, v0
+; RV64-NEXT:    vmv1r.v v0, v9
+; RV64-NEXT:    vmerge.vim v9, v8, 1, v0
+; RV64-NEXT:    srli a3, a3, 2
+; RV64-NEXT:    vwaddu.vv v12, v9, v11
+; RV64-NEXT:    vwmaccu.vx v12, a2, v11
+; RV64-NEXT:    vmsne.vi v0, v12, 0
+; RV64-NEXT:    vsetvli a2, zero, e8, mf2, ta, ma
+; RV64-NEXT:    vslidedown.vx v9, v12, a3
+; RV64-NEXT:    vmerge.vim v10, v10, 1, v0
+; RV64-NEXT:    vsetvli a2, zero, e8, mf4, ta, ma
+; RV64-NEXT:    vmsne.vi v0, v9, 0
+; RV64-NEXT:    add a2, a3, a3
+; RV64-NEXT:    vmerge.vim v8, v8, 1, v0
+; RV64-NEXT:    vsetvli zero, a2, e8, mf2, ta, ma
+; RV64-NEXT:    vslideup.vx v10, v8, a3
+; RV64-NEXT:    vsetvli a2, zero, e8, mf2, ta, ma
+; RV64-NEXT:    vmsne.vi v0, v10, 0
+; RV64-NEXT:    srli a1, a1, 32
+; RV64-NEXT:    vsetvli zero, a1, e32, m2, ta, ma
+; RV64-NEXT:    vle32.v v10, (a0), v0.t
+; RV64-NEXT:    li a0, 32
+; RV64-NEXT:    vsetvli a1, zero, e32, m1, ta, ma
+; RV64-NEXT:    vnsrl.wx v9, v10, a0
+; RV64-NEXT:    vnsrl.wi v8, v10, 0
+; RV64-NEXT:    ret
+  %rvl = mul i32 %evl, 2
+  %interleaved.mask = tail call <vscale x 4 x i1> @llvm.vector.interleave2.nxv4i1(<vscale x 2 x i1> %mask0, <vscale x 2 x i1> %mask1)
+  %wide.masked.load = tail call <vscale x 4 x i32> @llvm.vp.load.nxv4i32.p0(ptr %ptr, <vscale x 4 x i1> %interleaved.mask, i32 %rvl)
+  %deinterleaved.results = tail call { <vscale x 2 x i32>, <vscale x 2 x i32> } @llvm.vector.deinterleave2.nxv16i32(<vscale x 4 x i32> %wide.masked.load)
+  %t0 = extractvalue { <vscale x 2 x i32>, <vscale x 2 x i32> } %deinterleaved.results, 0
+  %t1 = extractvalue { <vscale x 2 x i32>, <vscale x 2 x i32> } %deinterleaved.results, 1
+  %res0 = insertvalue { <vscale x 2 x i32>, <vscale x 2 x i32> } poison, <vscale x 2 x i32> %t0, 0
+  %res1 = insertvalue { <vscale x 2 x i32>, <vscale x 2 x i32> } %res0, <vscale x 2 x i32> %t1, 1
+  ret { <vscale x 2 x i32>, <vscale x 2 x i32> } %res1
+}
+
+; EVL should be a multiple of factor
+define {<vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>} @invalid_evl(ptr %ptr, i32 %evl) {
+; RV32-LABEL: invalid_evl:
+; RV32:       # %bb.0:
+; RV32-NEXT:    ori a1, a1, 1
+; RV32-NEXT:    vsetvli zero, a1, e32, m4, ta, ma
+; RV32-NEXT:    vle32.v v8, (a0)
+; RV32-NEXT:    li a0, 32
+; RV32-NEXT:    vsetvli a1, zero, e32, m2, ta, ma
+; RV32-NEXT:    vnsrl.wx v12, v8, a0
+; RV32-NEXT:    vnsrl.wi v14, v8, 0
+; RV32-NEXT:    vsetvli a1, zero, e32, m1, ta, ma
+; RV32-NEXT:    vnsrl.wx v10, v14, a0
+; RV32-NEXT:    vnsrl.wi v8, v14, 0
+; RV32-NEXT:    vnsrl.wx v11, v12, a0
+; RV32-NEXT:    vnsrl.wi v9, v12, 0
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: invalid_evl:
+; RV64:       # %bb.0:
+; RV64-NEXT:    ori a1, a1, 1
+; RV64-NEXT:    slli a1, a1, 32
+; RV64-NEXT:    srli a1, a1, 32
+; RV64-NEXT:    vsetvli zero, a1, e32, m4, ta, ma
+; RV64-NEXT:    vle32.v v8, (a0)
+; RV64-NEXT:    li a0, 32
+; RV64-NEXT:    vsetvli a1, zero, e32, m2, ta, ma
+; RV64-NEXT:    vnsrl.wx v12, v8, a0
+; RV64-NEXT:    vnsrl.wi v14, v8, 0
+; RV64-NEXT:    vsetvli a1, zero, e32, m1, ta, ma
+; RV64-NEXT:    vnsrl.wx v10, v14, a0
+; RV64-NEXT:    vnsrl.wi v8, v14, 0
+; RV64-NEXT:    vnsrl.wx v11, v12, a0
+; RV64-NEXT:    vnsrl.wi v9, v12, 0
+; RV64-NEXT:    ret
+  %rvl = or i32 %evl, 1
+  %wide.masked.load = call <vscale x 8 x i32> @llvm.vp.load.nxv8i32.p0(ptr %ptr, <vscale x 8 x i1> splat (i1 true), i32 %rvl)
+  %d0 = call { <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.vector.deinterleave2.nxv8i32(<vscale x 8 x i32> %wide.masked.load)
+  %d0.0 = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } %d0, 0
+  %d0.1 = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } %d0, 1
+  %d1 = call { <vscale x 2 x i32>, <vscale x 2 x i32> } @llvm.vector.deinterleave2.nxv4i32(<vscale x 4 x i32> %d0.0)
+  %t0 = extractvalue { <vscale x 2 x i32>, <vscale x 2 x i32> } %d1, 0
+  %t2 = extractvalue { <vscale x 2 x i32>, <vscale x 2 x i32> } %d1, 1
+  %d2 = call { <vscale x 2 x i32>, <vscale x 2 x i32> } @llvm.vector.deinterleave2.nxv4i32(<vscale x 4 x i32> %d0.1)
+  %t1 = extractvalue { <vscale x 2 x i32>, <vscale x 2 x i32> } %d2, 0
+  %t3 = extractvalue { <vscale x 2 x i32>, <vscale x 2 x i32> } %d2, 1
+
+  %res0 = insertvalue { <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32> } poison, <vscale x 2 x i32> %t0, 0
+  %res1 = insertvalue { <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32> } %res0, <vscale x 2 x i32> %t1, 1
+  %res2 = insertvalue { <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32> } %res1, <vscale x 2 x i32> %t2, 2
+  %res3 = insertvalue { <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32> } %res2, <vscale x 2 x i32> %t3, 3
+  ret { <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32> } %res3
+}
+
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; CHECK: {{.*}}
diff --git a/llvm/test/CodeGen/RISCV/rvv/vpgather-sdnode.ll b/llvm/test/CodeGen/RISCV/rvv/vpgather-sdnode.ll
index abe7bdad8125a..1007d1ce649cc 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vpgather-sdnode.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vpgather-sdnode.ll
@@ -799,18 +799,18 @@ define <vscale x 8 x i32> @vpgather_baseidx_sext_nxv8i8_nxv8i32(ptr %base, <vsca
 define <vscale x 8 x i32> @vpgather_baseidx_zext_nxv8i8_nxv8i32(ptr %base, <vscale x 8 x i8> %idxs, <vscale x 8 x i1> %m, i32 zeroext %evl) {
 ; RV32-LABEL: vpgather_baseidx_zext_nxv8i8_nxv8i32:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    vsetvli zero, a1, e16, m2, ta, ma
-; RV32-NEXT:    vzext.vf2 v10, v8
-; RV32-NEXT:    vsll.vi v12, v10, 2
+; RV32-NEXT:    li a2, 4
+; RV32-NEXT:    vsetvli zero, a1, e8, m1, ta, ma
+; RV32-NEXT:    vwmulu.vx v12, v8, a2
 ; RV32-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
 ; RV32-NEXT:    vluxei16.v v8, (a0), v12, v0.t
 ; RV32-NEXT:    ret
 ;
 ; RV64-LABEL: vpgather_baseidx_zext_nxv8i8_nxv8i32:
 ; RV64:       # %bb.0:
-; RV64-NEXT:    vsetvli zero, a1, e16, m2, ta, ma
-; RV64-NEXT:    vzext.vf2 v10, v8
-; RV64-NEXT:    vsll.vi v12, v10, 2
+; RV64-NEXT:    li a2, 4
+; RV64-NEXT:    vsetvli zero, a1, e8, m1, ta, ma
+; RV64-NEXT:    vwmulu.vx v12, v8, a2
 ; RV64-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
 ; RV64-NEXT:    vluxei16.v v8, (a0), v12, v0.t
 ; RV64-NEXT:    ret
@@ -823,10 +823,11 @@ define <vscale x 8 x i32> @vpgather_baseidx_zext_nxv8i8_nxv8i32(ptr %base, <vsca
 define <vscale x 8 x i32> @vpgather_baseidx_nxv8i16_nxv8i32(ptr %base, <vscale x 8 x i16> %idxs, <vscale x 8 x i1> %m, i32 zeroext %evl) {
 ; RV32-LABEL: vpgather_baseidx_nxv8i16_nxv8i32:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    vsetvli zero, a1, e32, m4, ta, ma
-; RV32-NEXT:    vsext.vf2 v12, v8
-; RV32-NEXT:    vsll.vi v8, v12, 2
-; RV32-NEXT:    vluxei32.v v8, (a0), v8, v0.t
+; RV32-NEXT:    li a2, 4
+; RV32-NEXT:    vsetvli zero, a1, e16, m2, ta, ma
+; RV32-NEXT:    vwmulsu.vx v12, v8, a2
+; RV32-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
+; RV32-NEXT:    vluxei32.v v8, (a0), v12, v0.t
 ; RV32-NEXT:    ret
 ;
 ; RV64-LABEL: vpgather_baseidx_nxv8i16_nxv8i32:
@@ -845,10 +846,11 @@ define <vscale x 8 x i32> @vpgather_baseidx_nxv8i16_nxv8i32(ptr %base, <vscale x
 define <vscale x 8 x i32> @vpgather_baseidx_sext_nxv8i16_nxv8i32(ptr %base, <vscale x 8 x i16> %idxs, <vscale x 8 x i1> %m, i32 zeroext %evl) {
 ; RV32-LABEL: vpgather_baseidx_sext_nxv8i16_nxv8i32:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    vsetvli zero, a1, e32, m4, ta, ma
-; RV32-NEXT:    vsext.vf2 v12, v8
-; RV32-NEXT:    vsll.vi v8, v12, 2
-; RV32-NEXT:    vluxei32.v v8, (a0), v8, v0.t
+; RV32-NEXT:    li a2, 4
+; RV32-NEXT:    vsetvli zero, a1, e16, m2, ta, ma
+; RV32-NEXT:    vwmulsu.vx v12, v8, a2
+; RV32-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
+; RV32-NEXT:    vluxei32.v v8, (a0), v12, v0.t
 ; RV32-NEXT:    ret
 ;
 ; RV64-LABEL: vpgather_baseidx_sext_nxv8i16_nxv8i32:
@@ -868,18 +870,20 @@ define <vscale x 8 x i32> @vpgather_baseidx_sext_nxv8i16_nxv8i32(ptr %base, <vsc
 define <vscale x 8 x i32> @vpgather_baseidx_zext_nxv8i16_nxv8i32(ptr %base, <vscale x 8 x i16> %idxs, <vscale x 8 x i1> %m, i32 zeroext %evl) {
 ; RV32-LABEL: vpgather_baseidx_zext_nxv8i16_nxv8i32:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    vsetvli zero, a1, e32, m4, ta, ma
-; RV32-NEXT:    vzext.vf2 v12, v8
-; RV32-NEXT:    vsll.vi v8, v12, 2
-; RV32-NEXT:    vluxei32.v v8, (a0), v8, v0.t
+; RV32-NEXT:    li a2, 4
+; RV32-NEXT:    vsetvli zero, a1, e16, m2, ta, ma
+; RV32-NEXT:    vwmulu.vx v12, v8, a2
+; RV32-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
+; RV32-NEXT:    vluxei32.v v8, (a0), v12, v0.t
 ; RV32-NEXT:    ret
 ;
 ; RV64-LABEL: vpgather_baseidx_zext_nxv8i16_nxv8i32:
 ; RV64:       # %bb.0:
-; RV64-NEXT:    vsetvli zero, a1, e32, m4, ta, ma
-; RV64-NEXT:    vzext.vf2 v12, v8
-; RV64-NEXT:    vsll.vi v8, v12, 2
-; RV64-NEXT:    vluxei32.v v8, (a0), v8, v0.t
+; RV64-NEXT:    li a2, 4
+; RV64-NEXT:    vsetvli zero, a1, e16, m2, ta, ma
+; RV64-NEXT:    vwmulu.vx v12, v8, a2
+; RV64-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
+; RV64-NEXT:    vluxei32.v v8, (a0), v12, v0.t
 ; RV64-NEXT:    ret
   %eidxs = zext <vscale x 8 x i16> %idxs to <vscale x 8 x i32>
   %ptrs = getelementptr inbounds i32, ptr %base, <vscale x 8 x i32> %eidxs
@@ -897,10 +901,9 @@ define <vscale x 8 x i32> @vpgather_baseidx_nxv8i32(ptr %base, <vscale x 8 x i32
 ;
 ; RV64-LABEL: vpgather_baseidx_nxv8i32:
 ; RV64:       # %bb.0:
-; RV64-NEXT:    vsetvli zero, a1, e64, m8, ta, ma
-; RV64-NEXT:    vsext.vf2 v16, v8
-; RV64-NEXT:    vsll.vi v16, v16, 2
-; RV64-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
+; RV64-NEXT:    li a2, 4
+; RV64-NEXT:    vsetvli zero, a1, e32, m4, ta, ma
+; RV64-NEXT:    vwmulsu.vx v16, v8, a2
 ; RV64-NEXT:    vluxei64.v v8, (a0), v16, v0.t
 ; RV64-NEXT:    ret
   %ptrs = getelementptr inbounds i32, ptr %base, <vscale x 8 x i32> %idxs
@@ -1049,18 +1052,18 @@ define <vscale x 8 x i64> @vpgather_baseidx_sext_nxv8i8_nxv8i64(ptr %base, <vsca
 define <vscale x 8 x i64> @vpgather_baseidx_zext_nxv8i8_nxv8i64(ptr %base, <vscale x 8 x i8> %idxs, <vscale x 8 x i1> %m, i32 zeroext %evl) {
 ; RV32-LABEL: vpgather_baseidx_zext_nxv8i8_nxv8i64:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    vsetvli zero, a1, e16, m2, ta, ma
-; RV32-NEXT:    vzext.vf2 v10, v8
-; RV32-NEXT:    vsll.vi v16, v10, 3
+; RV32-NEXT:    li a2, 8
+; RV32-NEXT:    vsetvli zero, a1, e8, m1, ta, ma
+; RV32-NEXT:    vwmulu.vx v16, v8, a2
 ; RV32-NEXT:    vsetvli zero, zero, e64, m8, ta, ma
 ; RV32-NEXT:    vluxei16.v v8, (a0), v16, v0.t
 ; RV32-NEXT:    ret
 ;
 ; RV64-LABEL: vpgather_baseidx_zext_nxv8i8_nxv8i64:
 ; RV64:       # %bb.0:
-; RV64-NEXT:    vsetvli zero, a1, e16, m2, ta, ma
-; RV64-NEXT:    vzext.vf2 v10, v8
-; RV64-NEXT:    vsll.vi v16, v10, 3
+; RV64-NEXT:    li a2, 8
+; RV64-NEXT:    vsetvli zero, a1, e8, m1, ta, ma
+; RV64-NEXT:    vwmulu.vx v16, v8, a2
 ; RV64-NEXT:    vsetvli zero, zero, e64, m8, ta, ma
 ; RV64-NEXT:    vluxei16.v v8, (a0), v16, v0.t
 ; RV64-NEXT:    ret
@@ -1073,9 +1076,9 @@ define <vscale x 8 x i64> @vpgather_baseidx_zext_nxv8i8_nxv8i64(ptr %base, <vsca
 define <vscale x 8 x i64> @vpgather_baseidx_nxv8i16_nxv8i64(ptr %base, <vscale x 8 x i16> %idxs, <vscale x 8 x i1> %m, i32 zeroext %evl) {
 ; RV32-LABEL: vpgather_baseidx_nxv8i16_nxv8i64:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    vsetvli zero, a1, e32, m4, ta, ma
-; RV32-NEXT:    vsext.vf2 v12, v8
-; RV32-NEXT:    vsll.vi v16, v12, 3
+; RV32-NEXT:    li a2, 8
+; RV32-NEXT:    vsetvli zero, a1, e16, m2, ta, ma
+; RV32-NEXT:    vwmulsu.vx v16, v8, a2
 ; RV32-NEXT:    vsetvli zero, zero, e64, m8, ta, ma
 ; RV32-NEXT:    vluxei32.v v8, (a0), v16, v0.t
 ; RV32-NEXT:    ret
@@ -1095,9 +1098,9 @@ define <vscale x 8 x i64> @vpgather_baseidx_nxv8i16_nxv8i64(ptr %base, <vscale x
 define <vscale x 8 x i64> @vpgather_baseidx_sext_nxv8i16_nxv8i64(ptr %base, <vscale x 8 x i16> %idxs, <vscale x 8 x i1> %m, i32 zeroext %evl) {
 ; RV32-LABEL: vpgather_baseidx_sext_nxv8i16_nxv8i64:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    vsetvli zero, a1, e32, m4, ta, ma
-; RV32-NEXT:    vsext.vf2 v12, v8
-; RV32-NEXT:    vsll.vi v16, v12, 3
+; RV32-NEXT:    li a2, 8
+; RV32-NEXT:    vsetvli zero, a1, e16, m2, ta, ma
+; RV32-NEXT:    vwmulsu.vx v16, v8, a2
 ; RV32-NEXT:    vsetvli zero, zero, e64, m8, ta, ma
 ; RV32-NEXT:    vluxei32.v v8, (a0), v16, v0.t
 ; RV32-NEXT:    ret
@@ -1118,18 +1121,18 @@ define <vscale x 8 x i64> @vpgather_baseidx_sext_nxv8i16_nxv8i64(ptr %base, <vsc
 define <vscale x 8 x i64> @vpgather_baseidx_zext_nxv8i16_nxv8i64(ptr %base, <vscale x 8 x i16> %idxs, <vscale x 8 x i1> %m, i32 zeroext %evl) {
 ; RV32-LABEL: vpgather_baseidx_zext_nxv8i16_nxv8i64:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    vsetvli zero, a1, e32, m4, ta, ma
-; RV32-NEXT:    vzext.vf2 v12, v8
-; RV32-NEXT:    vsll.vi v16, v12, 3
+; RV32-NEXT:    li a2, 8
+; RV32-NEXT:    vsetvli zero, a1, e16, m2, ta, ma
+; RV32-NEXT:    vwmulu.vx v16, v8, a2
 ; RV32-NEXT:    vsetvli zero, zero, e64, m8, ta, ma
 ; RV32-NEXT:    vluxei32.v v8, (a0), v16, v0.t
 ; RV32-NEXT:    ret
 ;
 ; RV64-LABEL: vpgather_baseidx_zext_nxv8i16_nxv8i64:
 ; RV64:       # %bb.0:
-; RV64-NEXT:    vsetvli zero, a1, e32, m4, ta, ma
-; RV64-NEXT:    vzext.vf2 v12, v8
-; RV64-NEXT:    vsll.vi v16, v12, 3
+; RV64-NEXT:    li a2, 8
+; RV64-NEXT:    vsetvli zero, a1, e16, m2, ta, ma
+; RV64-NEXT:    vwmulu.vx v16, v8, a2
 ; RV64-NEXT:    vsetvli zero, zero, e64, m8, ta, ma
 ; RV64-NEXT:    vluxei32.v v8, (a0), v16, v0.t
 ; RV64-NEXT:    ret
@@ -1150,10 +1153,11 @@ define <vscale x 8 x i64> @vpgather_baseidx_nxv8i32_nxv8i64(ptr %base, <vscale x
 ;
 ; RV64-LABEL: vpgather_baseidx_nxv8i32_nxv8i64:
 ; RV64:       # %bb.0:
-; RV64-NEXT:    vsetvli zero, a1, e64, m8, ta, ma
-; RV64-NEXT:    vsext.vf2 v16, v8
-; RV64-NEXT:    vsll.vi v8, v16, 3
-; RV64-NEXT:    vluxei64.v v8, (a0), v8, v0.t
+; RV64-NEXT:    li a2, 8
+; RV64-NEXT:    vsetvli zero, a1, e32, m4, ta, ma
+; RV64-NEXT:    vwmulsu.vx v16, v8, a2
+; RV64-NEXT:    vsetvli zero, zero, e64, m8, ta, ma
+; RV64-NEXT:    vluxei64.v v8, (a0), v16, v0.t
 ; RV64-NEXT:    ret
   %ptrs = getelementptr inbounds i64, ptr %base, <vscale x 8 x i32> %idxs
   %v = call <vscale x 8 x i64> @llvm.vp.gather.nxv8i64.nxv8p0(<vscale x 8 x ptr> %ptrs, <vscale x 8 x i1> %m, i32 %evl)
@@ -1171,10 +1175,11 @@ define <vscale x 8 x i64> @vpgather_baseidx_sext_nxv8i32_nxv8i64(ptr %base, <vsc
 ;
 ; RV64-LABEL: vpgather_baseidx_sext_nxv8i32_nxv8i64:
 ; RV64:       # %bb.0:
-; RV64-NEXT:    vsetvli zero, a1, e64, m8, ta, ma
-; RV64-NEXT:    vsext.vf2 v16, v8
-; RV64-NEXT:    vsll.vi v8, v16, 3
-; RV64-NEXT:    vluxei64.v v8, (a0), v8, v0.t
+; RV64-NEXT:    li a2, 8
+; RV64-NEXT:    vsetvli zero, a1, e32, m4, ta, ma
+; RV64-NEXT:    vwmulsu.vx v16, v8, a2
+; RV64-NEXT:    vsetvli zero, zero, e64, m8, ta, ma
+; RV64-NEXT:    vluxei64.v v8, (a0), v16, v0.t
 ; RV64-NEXT:    ret
   %eidxs = sext <vscale x 8 x i32> %idxs to <vscale x 8 x i64>
   %ptrs = getelementptr inbounds i64, ptr %base, <vscale x 8 x i64> %eidxs
@@ -1193,10 +1198,11 @@ define <vscale x 8 x i64> @vpgather_baseidx_zext_nxv8i32_nxv8i64(ptr %base, <vsc
 ;
 ; RV64-LABEL: vpgather_baseidx_zext_nxv8i32_nxv8i64:
 ; RV64:       # %bb.0:
-; RV64-NEXT:    vsetvli zero, a1, e64, m8, ta, ma
-; RV64-NEXT:    vzext.vf2 v16, v8
-; RV64-NEXT:    vsll.vi v8, v16, 3
-; RV64-NEXT:    vluxei64.v v8, (a0), v8, v0.t
+; RV64-NEXT:    li a2, 8
+; RV64-NEXT:    vsetvli zero, a1, e32, m4, ta, ma
+; RV64-NEXT:    vwmulu.vx v16, v8, a2
+; RV64-NEXT:    vsetvli zero, zero, e64, m8, ta, ma
+; RV64-NEXT:    vluxei64.v v8, (a0), v16, v0.t
 ; RV64-NEXT:    ret
   %eidxs = zext <vscale x 8 x i32> %idxs to <vscale x 8 x i64>
   %ptrs = getelementptr inbounds i64, ptr %base, <vscale x 8 x i64> %eidxs
@@ -1742,18 +1748,18 @@ define <vscale x 8 x float> @vpgather_baseidx_sext_nxv8i8_nxv8f32(ptr %base, <vs
 define <vscale x 8 x float> @vpgather_baseidx_zext_nxv8i8_nxv8f32(ptr %base, <vscale x 8 x i8> %idxs, <vscale x 8 x i1> %m, i32 zeroext %evl) {
 ; RV32-LABEL: vpgather_baseidx_zext_nxv8i8_nxv8f32:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    vsetvli zero, a1, e16, m2, ta, ma
-; RV32-NEXT:    vzext.vf2 v10, v8
-; RV32-NEXT:    vsll.vi v12, v10, 2
+; RV32-NEXT:    li a2, 4
+; RV32-NEXT:    vsetvli zero, a1, e8, m1, ta, ma
+; RV32-NEXT:    vwmulu.vx v12, v8, a2
 ; RV32-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
 ; RV32-NEXT:    vluxei16.v v8, (a0), v12, v0.t
 ; RV32-NEXT:    ret
 ;
 ; RV64-LABEL: vpgather_baseidx_zext_nxv8i8_nxv8f32:
 ; RV64:       # %bb.0:
-; RV64-NEXT:    vsetvli zero, a1, e16, m2, ta, ma
-; RV64-NEXT:    vzext.vf2 v10, v8
-; RV64-NEXT:    vsll.vi v12, v10, 2
+; RV64-NEXT:    li a2, 4
+; RV64-NEXT:    vsetvli zero, a1, e8, m1, ta, ma
+; RV64-NEXT:    vwmulu.vx v12, v8, a2
 ; RV64-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
 ; RV64-NEXT:    vluxei16.v v8, (a0), v12, v0.t
 ; RV64-NEXT:    ret
@@ -1766,10 +1772,11 @@ define <vscale x 8 x float> @vpgather_baseidx_zext_nxv8i8_nxv8f32(ptr %base, <vs
 define <vscale x 8 x float> @vpgather_baseidx_nxv8i16_nxv8f32(ptr %base, <vscale x 8 x i16> %idxs, <vscale x 8 x i1> %m, i32 zeroext %evl) {
 ; RV32-LABEL: vpgather_baseidx_nxv8i16_nxv8f32:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    vsetvli zero, a1, e32, m4, ta, ma
-; RV32-NEXT:    vsext.vf2 v12, v8
-; RV32-NEXT:    vsll.vi v8, v12, 2
-; RV32-NEXT:    vluxei32.v v8, (a0), v8, v0.t
+; RV32-NEXT:    li a2, 4
+; RV32-NEXT:    vsetvli zero, a1, e16, m2, ta, ma
+; RV32-NEXT:    vwmulsu.vx v12, v8, a2
+; RV32-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
+; RV32-NEXT:    vluxei32.v v8, (a0), v12, v0.t
 ; RV32-NEXT:    ret
 ;
 ; RV64-LABEL: vpgather_baseidx_nxv8i16_nxv8f32:
@@ -1788,10 +1795,11 @@ define <vscale x 8 x float> @vpgather_baseidx_nxv8i16_nxv8f32(ptr %base, <vscale
 define <vscale x 8 x float> @vpgather_baseidx_sext_nxv8i16_nxv8f32(ptr %base, <vscale x 8 x i16> %idxs, <vscale x 8 x i1> %m, i32 zeroext %evl) {
 ; RV32-LABEL: vpgather_baseidx_sext_nxv8i16_nxv8f32:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    vsetvli zero, a1, e32, m4, ta, ma
-; RV32-NEXT:    vsext.vf2 v12, v8
-; RV32-NEXT:    vsll.vi v8, v12, 2
-; RV32-NEXT:    vluxei32.v v8, (a0), v8, v0.t
+; RV32-NEXT:    li a2, 4
+; RV32-NEXT:    vsetvli zero, a1, e16, m2, ta, ma
+; RV32-NEXT:    vwmulsu.vx v12, v8, a2
+; RV32-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
+; RV32-NEXT:    vluxei32.v v8, (a0), v12, v0.t
 ; RV32-NEXT:    ret
 ;
 ; RV64-LABEL: vpgather_baseidx_sext_nxv8i16_nxv8f32:
@@ -1811,18 +1819,20 @@ define <vscale x 8 x float> @vpgather_baseidx_sext_nxv8i16_nxv8f32(ptr %base, <v
 define <vscale x 8 x float> @vpgather_baseidx_zext_nxv8i16_nxv8f32(ptr %base, <vscale x 8 x i16> %idxs, <vscale x 8 x i1> %m, i32 zeroext %evl) {
 ; RV32-LABEL: vpgather_baseidx_zext_nxv8i16_nxv8f32:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    vsetvli zero, a1, e32, m4, ta, ma
-; RV32-NEXT:    vzext.vf2 v12, v8
-; RV32-NEXT:    vsll.vi v8, v12, 2
-; RV32-NEXT:    vluxei32.v v8, (a0), v8, v0.t
+; RV32-NEXT:    li a2, 4
+; RV32-NEXT:    vsetvli zero, a1, e16, m2, ta, ma
+; RV32-NEXT:    vwmulu.vx v12, v8, a2
+; RV32-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
+; RV32-NEXT:    vluxei32.v v8, (a0), v12, v0.t
 ; RV32-NEXT:    ret
 ;
 ; RV64-LABEL: vpgather_baseidx_zext_nxv8i16_nxv8f32:
 ; RV64:       # %bb.0:
-; RV64-NEXT:    vsetvli zero, a1, e32, m4, ta, ma
-; RV64-NEXT:    vzext.vf2 v12, v8
-; RV64-NEXT:    vsll.vi v8, v12, 2
-; RV64-NEXT:    vluxei32.v v8, (a0), v8, v0.t
+; RV64-NEXT:    li a2, 4
+; RV64-NEXT:    vsetvli zero, a1, e16, m2, ta, ma
+; RV64-NEXT:    vwmulu.vx v12, v8, a2
+; RV64-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
+; RV64-NEXT:    vluxei32.v v8, (a0), v12, v0.t
 ; RV64-NEXT:    ret
   %eidxs = zext <vscale x 8 x i16> %idxs to <vscale x 8 x i32>
   %ptrs = getelementptr inbounds float, ptr %base, <vscale x 8 x i32> %eidxs
@@ -1840,10 +1850,9 @@ define <vscale x 8 x float> @vpgather_baseidx_nxv8f32(ptr %base, <vscale x 8 x i
 ;
 ; RV64-LABEL: vpgather_baseidx_nxv8f32:
 ; RV64:       # %bb.0:
-; RV64-NEXT:    vsetvli zero, a1, e64, m8, ta, ma
-; RV64-NEXT:    vsext.vf2 v16, v8
-; RV64-NEXT:    vsll.vi v16, v16, 2
-; RV64-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
+; RV64-NEXT:    li a2, 4
+; RV64-NEXT:    vsetvli zero, a1, e32, m4, ta, ma
+; RV64-NEXT:    vwmulsu.vx v16, v8, a2
 ; RV64-NEXT:    vluxei64.v v8, (a0), v16, v0.t
 ; RV64-NEXT:    ret
   %ptrs = getelementptr inbounds float, ptr %base, <vscale x 8 x i32> %idxs
@@ -1992,18 +2001,18 @@ define <vscale x 6 x double> @vpgather_baseidx_sext_nxv6i8_nxv6f64(ptr %base, <v
 define <vscale x 6 x double> @vpgather_baseidx_zext_nxv6i8_nxv6f64(ptr %base, <vscale x 6 x i8> %idxs, <vscale x 6 x i1> %m, i32 zeroext %evl) {
 ; RV32-LABEL: vpgather_baseidx_zext_nxv6i8_nxv6f64:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    vsetvli zero, a1, e16, m2, ta, ma
-; RV32-NEXT:    vzext.vf2 v10, v8
-; RV32-NEXT:    vsll.vi v16, v10, 3
+; RV32-NEXT:    li a2, 8
+; RV32-NEXT:    vsetvli zero, a1, e8, m1, ta, ma
+; RV32-NEXT:    vwmulu.vx v16, v8, a2
 ; RV32-NEXT:    vsetvli zero, zero, e64, m8, ta, ma
 ; RV32-NEXT:    vluxei16.v v8, (a0), v16, v0.t
 ; RV32-NEXT:    ret
 ;
 ; RV64-LABEL: vpgather_baseidx_zext_nxv6i8_nxv6f64:
 ; RV64:       # %bb.0:
-; RV64-NEXT:    vsetvli zero, a1, e16, m2, ta, ma
-; RV64-NEXT:    vzext.vf2 v10, v8
-; RV64-NEXT:    vsll.vi v16, v10, 3
+; RV64-NEXT:    li a2, 8
+; RV64-NEXT:    vsetvli zero, a1, e8, m1, ta, ma
+; RV64-NEXT:    vwmulu.vx v16, v8, a2
 ; RV64-NEXT:    vsetvli zero, zero, e64, m8, ta, ma
 ; RV64-NEXT:    vluxei16.v v8, (a0), v16, v0.t
 ; RV64-NEXT:    ret
@@ -2016,9 +2025,9 @@ define <vscale x 6 x double> @vpgather_baseidx_zext_nxv6i8_nxv6f64(ptr %base, <v
 define <vscale x 6 x double> @vpgather_baseidx_nxv6i16_nxv6f64(ptr %base, <vscale x 6 x i16> %idxs, <vscale x 6 x i1> %m, i32 zeroext %evl) {
 ; RV32-LABEL: vpgather_baseidx_nxv6i16_nxv6f64:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    vsetvli zero, a1, e32, m4, ta, ma
-; RV32-NEXT:    vsext.vf2 v12, v8
-; RV32-NEXT:    vsll.vi v16, v12, 3
+; RV32-NEXT:    li a2, 8
+; RV32-NEXT:    vsetvli zero, a1, e16, m2, ta, ma
+; RV32-NEXT:    vwmulsu.vx v16, v8, a2
 ; RV32-NEXT:    vsetvli zero, zero, e64, m8, ta, ma
 ; RV32-NEXT:    vluxei32.v v8, (a0), v16, v0.t
 ; RV32-NEXT:    ret
@@ -2038,9 +2047,9 @@ define <vscale x 6 x double> @vpgather_baseidx_nxv6i16_nxv6f64(ptr %base, <vscal
 define <vscale x 6 x double> @vpgather_baseidx_sext_nxv6i16_nxv6f64(ptr %base, <vscale x 6 x i16> %idxs, <vscale x 6 x i1> %m, i32 zeroext %evl) {
 ; RV32-LABEL: vpgather_baseidx_sext_nxv6i16_nxv6f64:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    vsetvli zero, a1, e32, m4, ta, ma
-; RV32-NEXT:    vsext.vf2 v12, v8
-; RV32-NEXT:    vsll.vi v16, v12, 3
+; RV32-NEXT:    li a2, 8
+; RV32-NEXT:    vsetvli zero, a1, e16, m2, ta, ma
+; RV32-NEXT:    vwmulsu.vx v16, v8, a2
 ; RV32-NEXT:    vsetvli zero, zero, e64, m8, ta, ma
 ; RV32-NEXT:    vluxei32.v v8, (a0), v16, v0.t
 ; RV32-NEXT:    ret
@@ -2061,18 +2070,18 @@ define <vscale x 6 x double> @vpgather_baseidx_sext_nxv6i16_nxv6f64(ptr %base, <
 define <vscale x 6 x double> @vpgather_baseidx_zext_nxv6i16_nxv6f64(ptr %base, <vscale x 6 x i16> %idxs, <vscale x 6 x i1> %m, i32 zeroext %evl) {
 ; RV32-LABEL: vpgather_baseidx_zext_nxv6i16_nxv6f64:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    vsetvli zero, a1, e32, m4, ta, ma
-; RV32-NEXT:    vzext.vf2 v12, v8
-; RV32-NEXT:    vsll.vi v16, v12, 3
+; RV32-NEXT:    li a2, 8
+; RV32-NEXT:    vsetvli zero, a1, e16, m2, ta, ma
+; RV32-NEXT:    vwmulu.vx v16, v8, a2
 ; RV32-NEXT:    vsetvli zero, zero, e64, m8, ta, ma
 ; RV32-NEXT:    vluxei32.v v8, (a0), v16, v0.t
 ; RV32-NEXT:    ret
 ;
 ; RV64-LABEL: vpgather_baseidx_zext_nxv6i16_nxv6f64:
 ; RV64:       # %bb.0:
-; RV64-NEXT:    vsetvli zero, a1, e32, m4, ta, ma
-; RV64-NEXT:    vzext.vf2 v12, v8
-; RV64-NEXT:    vsll.vi v16, v12, 3
+; RV64-NEXT:    li a2, 8
+; RV64-NEXT:    vsetvli zero, a1, e16, m2, ta, ma
+; RV64-NEXT:    vwmulu.vx v16, v8, a2
 ; RV64-NEXT:    vsetvli zero, zero, e64, m8, ta, ma
 ; RV64-NEXT:    vluxei32.v v8, (a0), v16, v0.t
 ; RV64-NEXT:    ret
@@ -2093,10 +2102,11 @@ define <vscale x 6 x double> @vpgather_baseidx_nxv6i32_nxv6f64(ptr %base, <vscal
 ;
 ; RV64-LABEL: vpgather_baseidx_nxv6i32_nxv6f64:
 ; RV64:       # %bb.0:
-; RV64-NEXT:    vsetvli zero, a1, e64, m8, ta, ma
-; RV64-NEXT:    vsext.vf2 v16, v8
-; RV64-NEXT:    vsll.vi v8, v16, 3
-; RV64-NEXT:    vluxei64.v v8, (a0), v8, v0.t
+; RV64-NEXT:    li a2, 8
+; RV64-NEXT:    vsetvli zero, a1, e32, m4, ta, ma
+; RV64-NEXT:    vwmulsu.vx v16, v8, a2
+; RV64-NEXT:    vsetvli zero, zero, e64, m8, ta, ma
+; RV64-NEXT:    vluxei64.v v8, (a0), v16, v0.t
 ; RV64-NEXT:    ret
   %ptrs = getelementptr inbounds double, ptr %base, <vscale x 6 x i32> %idxs
   %v = call <vscale x 6 x double> @llvm.vp.gather.nxv6f64.nxv6p0(<vscale x 6 x ptr> %ptrs, <vscale x 6 x i1> %m, i32 %evl)
@@ -2114,10 +2124,11 @@ define <vscale x 6 x double> @vpgather_baseidx_sext_nxv6i32_nxv6f64(ptr %base, <
 ;
 ; RV64-LABEL: vpgather_baseidx_sext_nxv6i32_nxv6f64:
 ; RV64:       # %bb.0:
-; RV64-NEXT:    vsetvli zero, a1, e64, m8, ta, ma
-; RV64-NEXT:    vsext.vf2 v16, v8
-; RV64-NEXT:    vsll.vi v8, v16, 3
-; RV64-NEXT:    vluxei64.v v8, (a0), v8, v0.t
+; RV64-NEXT:    li a2, 8
+; RV64-NEXT:    vsetvli zero, a1, e32, m4, ta, ma
+; RV64-NEXT:    vwmulsu.vx v16, v8, a2
+; RV64-NEXT:    vsetvli zero, zero, e64, m8, ta, ma
+; RV64-NEXT:    vluxei64.v v8, (a0), v16, v0.t
 ; RV64-NEXT:    ret
   %eidxs = sext <vscale x 6 x i32> %idxs to <vscale x 6 x i64>
   %ptrs = getelementptr inbounds double, ptr %base, <vscale x 6 x i64> %eidxs
@@ -2136,10 +2147,11 @@ define <vscale x 6 x double> @vpgather_baseidx_zext_nxv6i32_nxv6f64(ptr %base, <
 ;
 ; RV64-LABEL: vpgather_baseidx_zext_nxv6i32_nxv6f64:
 ; RV64:       # %bb.0:
-; RV64-NEXT:    vsetvli zero, a1, e64, m8, ta, ma
-; RV64-NEXT:    vzext.vf2 v16, v8
-; RV64-NEXT:    vsll.vi v8, v16, 3
-; RV64-NEXT:    vluxei64.v v8, (a0), v8, v0.t
+; RV64-NEXT:    li a2, 8
+; RV64-NEXT:    vsetvli zero, a1, e32, m4, ta, ma
+; RV64-NEXT:    vwmulu.vx v16, v8, a2
+; RV64-NEXT:    vsetvli zero, zero, e64, m8, ta, ma
+; RV64-NEXT:    vluxei64.v v8, (a0), v16, v0.t
 ; RV64-NEXT:    ret
   %eidxs = zext <vscale x 6 x i32> %idxs to <vscale x 6 x i64>
   %ptrs = getelementptr inbounds double, ptr %base, <vscale x 6 x i64> %eidxs
@@ -2235,18 +2247,18 @@ define <vscale x 8 x double> @vpgather_baseidx_sext_nxv8i8_nxv8f64(ptr %base, <v
 define <vscale x 8 x double> @vpgather_baseidx_zext_nxv8i8_nxv8f64(ptr %base, <vscale x 8 x i8> %idxs, <vscale x 8 x i1> %m, i32 zeroext %evl) {
 ; RV32-LABEL: vpgather_baseidx_zext_nxv8i8_nxv8f64:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    vsetvli zero, a1, e16, m2, ta, ma
-; RV32-NEXT:    vzext.vf2 v10, v8
-; RV32-NEXT:    vsll.vi v16, v10, 3
+; RV32-NEXT:    li a2, 8
+; RV32-NEXT:    vsetvli zero, a1, e8, m1, ta, ma
+; RV32-NEXT:    vwmulu.vx v16, v8, a2
 ; RV32-NEXT:    vsetvli zero, zero, e64, m8, ta, ma
 ; RV32-NEXT:    vluxei16.v v8, (a0), v16, v0.t
 ; RV32-NEXT:    ret
 ;
 ; RV64-LABEL: vpgather_baseidx_zext_nxv8i8_nxv8f64:
 ; RV64:       # %bb.0:
-; RV64-NEXT:    vsetvli zero, a1, e16, m2, ta, ma
-; RV64-NEXT:    vzext.vf2 v10, v8
-; RV64-NEXT:    vsll.vi v16, v10, 3
+; RV64-NEXT:    li a2, 8
+; RV64-NEXT:    vsetvli zero, a1, e8, m1, ta, ma
+; RV64-NEXT:    vwmulu.vx v16, v8, a2
 ; RV64-NEXT:    vsetvli zero, zero, e64, m8, ta, ma
 ; RV64-NEXT:    vluxei16.v v8, (a0), v16, v0.t
 ; RV64-NEXT:    ret
@@ -2259,9 +2271,9 @@ define <vscale x 8 x double> @vpgather_baseidx_zext_nxv8i8_nxv8f64(ptr %base, <v
 define <vscale x 8 x double> @vpgather_baseidx_nxv8i16_nxv8f64(ptr %base, <vscale x 8 x i16> %idxs, <vscale x 8 x i1> %m, i32 zeroext %evl) {
 ; RV32-LABEL: vpgather_baseidx_nxv8i16_nxv8f64:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    vsetvli zero, a1, e32, m4, ta, ma
-; RV32-NEXT:    vsext.vf2 v12, v8
-; RV32-NEXT:    vsll.vi v16, v12, 3
+; RV32-NEXT:    li a2, 8
+; RV32-NEXT:    vsetvli zero, a1, e16, m2, ta, ma
+; RV32-NEXT:    vwmulsu.vx v16, v8, a2
 ; RV32-NEXT:    vsetvli zero, zero, e64, m8, ta, ma
 ; RV32-NEXT:    vluxei32.v v8, (a0), v16, v0.t
 ; RV32-NEXT:    ret
@@ -2281,9 +2293,9 @@ define <vscale x 8 x double> @vpgather_baseidx_nxv8i16_nxv8f64(ptr %base, <vscal
 define <vscale x 8 x double> @vpgather_baseidx_sext_nxv8i16_nxv8f64(ptr %base, <vscale x 8 x i16> %idxs, <vscale x 8 x i1> %m, i32 zeroext %evl) {
 ; RV32-LABEL: vpgather_baseidx_sext_nxv8i16_nxv8f64:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    vsetvli zero, a1, e32, m4, ta, ma
-; RV32-NEXT:    vsext.vf2 v12, v8
-; RV32-NEXT:    vsll.vi v16, v12, 3
+; RV32-NEXT:    li a2, 8
+; RV32-NEXT:    vsetvli zero, a1, e16, m2, ta, ma
+; RV32-NEXT:    vwmulsu.vx v16, v8, a2
 ; RV32-NEXT:    vsetvli zero, zero, e64, m8, ta, ma
 ; RV32-NEXT:    vluxei32.v v8, (a0), v16, v0.t
 ; RV32-NEXT:    ret
@@ -2304,18 +2316,18 @@ define <vscale x 8 x double> @vpgather_baseidx_sext_nxv8i16_nxv8f64(ptr %base, <
 define <vscale x 8 x double> @vpgather_baseidx_zext_nxv8i16_nxv8f64(ptr %base, <vscale x 8 x i16> %idxs, <vscale x 8 x i1> %m, i32 zeroext %evl) {
 ; RV32-LABEL: vpgather_baseidx_zext_nxv8i16_nxv8f64:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    vsetvli zero, a1, e32, m4, ta, ma
-; RV32-NEXT:    vzext.vf2 v12, v8
-; RV32-NEXT:    vsll.vi v16, v12, 3
+; RV32-NEXT:    li a2, 8
+; RV32-NEXT:    vsetvli zero, a1, e16, m2, ta, ma
+; RV32-NEXT:    vwmulu.vx v16, v8, a2
 ; RV32-NEXT:    vsetvli zero, zero, e64, m8, ta, ma
 ; RV32-NEXT:    vluxei32.v v8, (a0), v16, v0.t
 ; RV32-NEXT:    ret
 ;
 ; RV64-LABEL: vpgather_baseidx_zext_nxv8i16_nxv8f64:
 ; RV64:       # %bb.0:
-; RV64-NEXT:    vsetvli zero, a1, e32, m4, ta, ma
-; RV64-NEXT:    vzext.vf2 v12, v8
-; RV64-NEXT:    vsll.vi v16, v12, 3
+; RV64-NEXT:    li a2, 8
+; RV64-NEXT:    vsetvli zero, a1, e16, m2, ta, ma
+; RV64-NEXT:    vwmulu.vx v16, v8, a2
 ; RV64-NEXT:    vsetvli zero, zero, e64, m8, ta, ma
 ; RV64-NEXT:    vluxei32.v v8, (a0), v16, v0.t
 ; RV64-NEXT:    ret
@@ -2336,10 +2348,11 @@ define <vscale x 8 x double> @vpgather_baseidx_nxv8i32_nxv8f64(ptr %base, <vscal
 ;
 ; RV64-LABEL: vpgather_baseidx_nxv8i32_nxv8f64:
 ; RV64:       # %bb.0:
-; RV64-NEXT:    vsetvli zero, a1, e64, m8, ta, ma
-; RV64-NEXT:    vsext.vf2 v16, v8
-; RV64-NEXT:    vsll.vi v8, v16, 3
-; RV64-NEXT:    vluxei64.v v8, (a0), v8, v0.t
+; RV64-NEXT:    li a2, 8
+; RV64-NEXT:    vsetvli zero, a1, e32, m4, ta, ma
+; RV64-NEXT:    vwmulsu.vx v16, v8, a2
+; RV64-NEXT:    vsetvli zero, zero, e64, m8, ta, ma
+; RV64-NEXT:    vluxei64.v v8, (a0), v16, v0.t
 ; RV64-NEXT:    ret
   %ptrs = getelementptr inbounds double, ptr %base, <vscale x 8 x i32> %idxs
   %v = call <vscale x 8 x double> @llvm.vp.gather.nxv8f64.nxv8p0(<vscale x 8 x ptr> %ptrs, <vscale x 8 x i1> %m, i32 %evl)
@@ -2357,10 +2370,11 @@ define <vscale x 8 x double> @vpgather_baseidx_sext_nxv8i32_nxv8f64(ptr %base, <
 ;
 ; RV64-LABEL: vpgather_baseidx_sext_nxv8i32_nxv8f64:
 ; RV64:       # %bb.0:
-; RV64-NEXT:    vsetvli zero, a1, e64, m8, ta, ma
-; RV64-NEXT:    vsext.vf2 v16, v8
-; RV64-NEXT:    vsll.vi v8, v16, 3
-; RV64-NEXT:    vluxei64.v v8, (a0), v8, v0.t
+; RV64-NEXT:    li a2, 8
+; RV64-NEXT:    vsetvli zero, a1, e32, m4, ta, ma
+; RV64-NEXT:    vwmulsu.vx v16, v8, a2
+; RV64-NEXT:    vsetvli zero, zero, e64, m8, ta, ma
+; RV64-NEXT:    vluxei64.v v8, (a0), v16, v0.t
 ; RV64-NEXT:    ret
   %eidxs = sext <vscale x 8 x i32> %idxs to <vscale x 8 x i64>
   %ptrs = getelementptr inbounds double, ptr %base, <vscale x 8 x i64> %eidxs
@@ -2379,10 +2393,11 @@ define <vscale x 8 x double> @vpgather_baseidx_zext_nxv8i32_nxv8f64(ptr %base, <
 ;
 ; RV64-LABEL: vpgather_baseidx_zext_nxv8i32_nxv8f64:
 ; RV64:       # %bb.0:
-; RV64-NEXT:    vsetvli zero, a1, e64, m8, ta, ma
-; RV64-NEXT:    vzext.vf2 v16, v8
-; RV64-NEXT:    vsll.vi v8, v16, 3
-; RV64-NEXT:    vluxei64.v v8, (a0), v8, v0.t
+; RV64-NEXT:    li a2, 8
+; RV64-NEXT:    vsetvli zero, a1, e32, m4, ta, ma
+; RV64-NEXT:    vwmulu.vx v16, v8, a2
+; RV64-NEXT:    vsetvli zero, zero, e64, m8, ta, ma
+; RV64-NEXT:    vluxei64.v v8, (a0), v16, v0.t
 ; RV64-NEXT:    ret
   %eidxs = zext <vscale x 8 x i32> %idxs to <vscale x 8 x i64>
   %ptrs = getelementptr inbounds double, ptr %base, <vscale x 8 x i64> %eidxs
@@ -2465,27 +2480,26 @@ define <vscale x 16 x double> @vpgather_nxv16f64(<vscale x 16 x ptr> %ptrs, <vsc
 define <vscale x 16 x double> @vpgather_baseidx_nxv16i16_nxv16f64(ptr %base, <vscale x 16 x i16> %idxs, <vscale x 16 x i1> %m, i32 zeroext %evl) {
 ; RV32-LABEL: vpgather_baseidx_nxv16i16_nxv16f64:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    vsetvli a2, zero, e32, m8, ta, ma
-; RV32-NEXT:    vmv1r.v v12, v0
-; RV32-NEXT:    vsext.vf2 v16, v8
+; RV32-NEXT:    li a3, 8
 ; RV32-NEXT:    csrr a2, vlenb
-; RV32-NEXT:    vsll.vi v24, v16, 3
-; RV32-NEXT:    sub a3, a1, a2
-; RV32-NEXT:    srli a4, a2, 3
-; RV32-NEXT:    vsetvli a5, zero, e8, mf4, ta, ma
-; RV32-NEXT:    vslidedown.vx v0, v0, a4
-; RV32-NEXT:    sltu a4, a1, a3
-; RV32-NEXT:    addi a4, a4, -1
-; RV32-NEXT:    and a3, a4, a3
-; RV32-NEXT:    vsetvli zero, a3, e64, m8, ta, ma
-; RV32-NEXT:    vluxei32.v v16, (a0), v28, v0.t
+; RV32-NEXT:    vsetvli a4, zero, e16, m4, ta, ma
+; RV32-NEXT:    vwmulsu.vx v24, v8, a3
+; RV32-NEXT:    mv a3, a1
 ; RV32-NEXT:    bltu a1, a2, .LBB112_2
 ; RV32-NEXT:  # %bb.1:
-; RV32-NEXT:    mv a1, a2
+; RV32-NEXT:    mv a3, a2
 ; RV32-NEXT:  .LBB112_2:
-; RV32-NEXT:    vmv1r.v v0, v12
-; RV32-NEXT:    vsetvli zero, a1, e64, m8, ta, ma
+; RV32-NEXT:    vsetvli zero, a3, e64, m8, ta, ma
 ; RV32-NEXT:    vluxei32.v v8, (a0), v24, v0.t
+; RV32-NEXT:    sub a3, a1, a2
+; RV32-NEXT:    srli a2, a2, 3
+; RV32-NEXT:    sltu a1, a1, a3
+; RV32-NEXT:    addi a1, a1, -1
+; RV32-NEXT:    vsetvli a4, zero, e8, mf4, ta, ma
+; RV32-NEXT:    vslidedown.vx v0, v0, a2
+; RV32-NEXT:    and a1, a1, a3
+; RV32-NEXT:    vsetvli zero, a1, e64, m8, ta, ma
+; RV32-NEXT:    vluxei32.v v16, (a0), v28, v0.t
 ; RV32-NEXT:    ret
 ;
 ; RV64-LABEL: vpgather_baseidx_nxv16i16_nxv16f64:
@@ -2523,27 +2537,26 @@ define <vscale x 16 x double> @vpgather_baseidx_nxv16i16_nxv16f64(ptr %base, <vs
 define <vscale x 16 x double> @vpgather_baseidx_sext_nxv16i16_nxv16f64(ptr %base, <vscale x 16 x i16> %idxs, <vscale x 16 x i1> %m, i32 zeroext %evl) {
 ; RV32-LABEL: vpgather_baseidx_sext_nxv16i16_nxv16f64:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    vsetvli a2, zero, e32, m8, ta, ma
-; RV32-NEXT:    vmv1r.v v12, v0
-; RV32-NEXT:    vsext.vf2 v16, v8
+; RV32-NEXT:    li a3, 8
 ; RV32-NEXT:    csrr a2, vlenb
-; RV32-NEXT:    vsll.vi v24, v16, 3
-; RV32-NEXT:    sub a3, a1, a2
-; RV32-NEXT:    srli a4, a2, 3
-; RV32-NEXT:    vsetvli a5, zero, e8, mf4, ta, ma
-; RV32-NEXT:    vslidedown.vx v0, v0, a4
-; RV32-NEXT:    sltu a4, a1, a3
-; RV32-NEXT:    addi a4, a4, -1
-; RV32-NEXT:    and a3, a4, a3
-; RV32-NEXT:    vsetvli zero, a3, e64, m8, ta, ma
-; RV32-NEXT:    vluxei32.v v16, (a0), v28, v0.t
+; RV32-NEXT:    vsetvli a4, zero, e16, m4, ta, ma
+; RV32-NEXT:    vwmulsu.vx v24, v8, a3
+; RV32-NEXT:    mv a3, a1
 ; RV32-NEXT:    bltu a1, a2, .LBB113_2
 ; RV32-NEXT:  # %bb.1:
-; RV32-NEXT:    mv a1, a2
+; RV32-NEXT:    mv a3, a2
 ; RV32-NEXT:  .LBB113_2:
-; RV32-NEXT:    vmv1r.v v0, v12
-; RV32-NEXT:    vsetvli zero, a1, e64, m8, ta, ma
+; RV32-NEXT:    vsetvli zero, a3, e64, m8, ta, ma
 ; RV32-NEXT:    vluxei32.v v8, (a0), v24, v0.t
+; RV32-NEXT:    sub a3, a1, a2
+; RV32-NEXT:    srli a2, a2, 3
+; RV32-NEXT:    sltu a1, a1, a3
+; RV32-NEXT:    addi a1, a1, -1
+; RV32-NEXT:    vsetvli a4, zero, e8, mf4, ta, ma
+; RV32-NEXT:    vslidedown.vx v0, v0, a2
+; RV32-NEXT:    and a1, a1, a3
+; RV32-NEXT:    vsetvli zero, a1, e64, m8, ta, ma
+; RV32-NEXT:    vluxei32.v v16, (a0), v28, v0.t
 ; RV32-NEXT:    ret
 ;
 ; RV64-LABEL: vpgather_baseidx_sext_nxv16i16_nxv16f64:
@@ -2582,52 +2595,50 @@ define <vscale x 16 x double> @vpgather_baseidx_sext_nxv16i16_nxv16f64(ptr %base
 define <vscale x 16 x double> @vpgather_baseidx_zext_nxv16i16_nxv16f64(ptr %base, <vscale x 16 x i16> %idxs, <vscale x 16 x i1> %m, i32 zeroext %evl) {
 ; RV32-LABEL: vpgather_baseidx_zext_nxv16i16_nxv16f64:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    vsetvli a2, zero, e32, m8, ta, ma
-; RV32-NEXT:    vmv1r.v v12, v0
-; RV32-NEXT:    vzext.vf2 v16, v8
+; RV32-NEXT:    li a3, 8
 ; RV32-NEXT:    csrr a2, vlenb
-; RV32-NEXT:    vsll.vi v24, v16, 3
-; RV32-NEXT:    sub a3, a1, a2
-; RV32-NEXT:    srli a4, a2, 3
-; RV32-NEXT:    vsetvli a5, zero, e8, mf4, ta, ma
-; RV32-NEXT:    vslidedown.vx v0, v0, a4
-; RV32-NEXT:    sltu a4, a1, a3
-; RV32-NEXT:    addi a4, a4, -1
-; RV32-NEXT:    and a3, a4, a3
-; RV32-NEXT:    vsetvli zero, a3, e64, m8, ta, ma
-; RV32-NEXT:    vluxei32.v v16, (a0), v28, v0.t
+; RV32-NEXT:    vsetvli a4, zero, e16, m4, ta, ma
+; RV32-NEXT:    vwmulu.vx v24, v8, a3
+; RV32-NEXT:    mv a3, a1
 ; RV32-NEXT:    bltu a1, a2, .LBB114_2
 ; RV32-NEXT:  # %bb.1:
-; RV32-NEXT:    mv a1, a2
+; RV32-NEXT:    mv a3, a2
 ; RV32-NEXT:  .LBB114_2:
-; RV32-NEXT:    vmv1r.v v0, v12
-; RV32-NEXT:    vsetvli zero, a1, e64, m8, ta, ma
+; RV32-NEXT:    vsetvli zero, a3, e64, m8, ta, ma
 ; RV32-NEXT:    vluxei32.v v8, (a0), v24, v0.t
+; RV32-NEXT:    sub a3, a1, a2
+; RV32-NEXT:    srli a2, a2, 3
+; RV32-NEXT:    sltu a1, a1, a3
+; RV32-NEXT:    addi a1, a1, -1
+; RV32-NEXT:    vsetvli a4, zero, e8, mf4, ta, ma
+; RV32-NEXT:    vslidedown.vx v0, v0, a2
+; RV32-NEXT:    and a1, a1, a3
+; RV32-NEXT:    vsetvli zero, a1, e64, m8, ta, ma
+; RV32-NEXT:    vluxei32.v v16, (a0), v28, v0.t
 ; RV32-NEXT:    ret
 ;
 ; RV64-LABEL: vpgather_baseidx_zext_nxv16i16_nxv16f64:
 ; RV64:       # %bb.0:
-; RV64-NEXT:    vsetvli a2, zero, e32, m8, ta, ma
-; RV64-NEXT:    vmv1r.v v12, v0
-; RV64-NEXT:    vzext.vf2 v16, v8
+; RV64-NEXT:    li a3, 8
 ; RV64-NEXT:    csrr a2, vlenb
-; RV64-NEXT:    vsll.vi v24, v16, 3
-; RV64-NEXT:    sub a3, a1, a2
-; RV64-NEXT:    srli a4, a2, 3
-; RV64-NEXT:    vsetvli a5, zero, e8, mf4, ta, ma
-; RV64-NEXT:    vslidedown.vx v0, v0, a4
-; RV64-NEXT:    sltu a4, a1, a3
-; RV64-NEXT:    addi a4, a4, -1
-; RV64-NEXT:    and a3, a4, a3
-; RV64-NEXT:    vsetvli zero, a3, e64, m8, ta, ma
-; RV64-NEXT:    vluxei32.v v16, (a0), v28, v0.t
+; RV64-NEXT:    vsetvli a4, zero, e16, m4, ta, ma
+; RV64-NEXT:    vwmulu.vx v24, v8, a3
+; RV64-NEXT:    mv a3, a1
 ; RV64-NEXT:    bltu a1, a2, .LBB114_2
 ; RV64-NEXT:  # %bb.1:
-; RV64-NEXT:    mv a1, a2
+; RV64-NEXT:    mv a3, a2
 ; RV64-NEXT:  .LBB114_2:
-; RV64-NEXT:    vmv1r.v v0, v12
-; RV64-NEXT:    vsetvli zero, a1, e64, m8, ta, ma
+; RV64-NEXT:    vsetvli zero, a3, e64, m8, ta, ma
 ; RV64-NEXT:    vluxei32.v v8, (a0), v24, v0.t
+; RV64-NEXT:    sub a3, a1, a2
+; RV64-NEXT:    srli a2, a2, 3
+; RV64-NEXT:    sltu a1, a1, a3
+; RV64-NEXT:    addi a1, a1, -1
+; RV64-NEXT:    vsetvli a4, zero, e8, mf4, ta, ma
+; RV64-NEXT:    vslidedown.vx v0, v0, a2
+; RV64-NEXT:    and a1, a1, a3
+; RV64-NEXT:    vsetvli zero, a1, e64, m8, ta, ma
+; RV64-NEXT:    vluxei32.v v16, (a0), v28, v0.t
 ; RV64-NEXT:    ret
   %eidxs = zext <vscale x 16 x i16> %idxs to <vscale x 16 x i64>
   %ptrs = getelementptr inbounds double, ptr %base, <vscale x 16 x i64> %eidxs
diff --git a/llvm/test/CodeGen/RISCV/rvv/vpscatter-sdnode.ll b/llvm/test/CodeGen/RISCV/rvv/vpscatter-sdnode.ll
index 647e3965b7ec2..2cf6248c17598 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vpscatter-sdnode.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vpscatter-sdnode.ll
@@ -659,20 +659,20 @@ define void @vpscatter_baseidx_sext_nxv8i8_nxv8i32(<vscale x 8 x i32> %val, ptr
 define void @vpscatter_baseidx_zext_nxv8i8_nxv8i32(<vscale x 8 x i32> %val, ptr %base, <vscale x 8 x i8> %idxs, <vscale x 8 x i1> %m, i32 zeroext %evl) {
 ; RV32-LABEL: vpscatter_baseidx_zext_nxv8i8_nxv8i32:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    vsetvli zero, a1, e16, m2, ta, ma
-; RV32-NEXT:    vzext.vf2 v14, v12
-; RV32-NEXT:    vsll.vi v12, v14, 2
+; RV32-NEXT:    li a2, 4
+; RV32-NEXT:    vsetvli zero, a1, e8, m1, ta, ma
+; RV32-NEXT:    vwmulu.vx v14, v12, a2
 ; RV32-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
-; RV32-NEXT:    vsoxei16.v v8, (a0), v12, v0.t
+; RV32-NEXT:    vsoxei16.v v8, (a0), v14, v0.t
 ; RV32-NEXT:    ret
 ;
 ; RV64-LABEL: vpscatter_baseidx_zext_nxv8i8_nxv8i32:
 ; RV64:       # %bb.0:
-; RV64-NEXT:    vsetvli zero, a1, e16, m2, ta, ma
-; RV64-NEXT:    vzext.vf2 v14, v12
-; RV64-NEXT:    vsll.vi v12, v14, 2
+; RV64-NEXT:    li a2, 4
+; RV64-NEXT:    vsetvli zero, a1, e8, m1, ta, ma
+; RV64-NEXT:    vwmulu.vx v14, v12, a2
 ; RV64-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
-; RV64-NEXT:    vsoxei16.v v8, (a0), v12, v0.t
+; RV64-NEXT:    vsoxei16.v v8, (a0), v14, v0.t
 ; RV64-NEXT:    ret
   %eidxs = zext <vscale x 8 x i8> %idxs to <vscale x 8 x i32>
   %ptrs = getelementptr inbounds i32, ptr %base, <vscale x 8 x i32> %eidxs
@@ -683,10 +683,11 @@ define void @vpscatter_baseidx_zext_nxv8i8_nxv8i32(<vscale x 8 x i32> %val, ptr
 define void @vpscatter_baseidx_nxv8i16_nxv8i32(<vscale x 8 x i32> %val, ptr %base, <vscale x 8 x i16> %idxs, <vscale x 8 x i1> %m, i32 zeroext %evl) {
 ; RV32-LABEL: vpscatter_baseidx_nxv8i16_nxv8i32:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    vsetvli zero, a1, e32, m4, ta, ma
-; RV32-NEXT:    vsext.vf2 v16, v12
-; RV32-NEXT:    vsll.vi v12, v16, 2
-; RV32-NEXT:    vsoxei32.v v8, (a0), v12, v0.t
+; RV32-NEXT:    li a2, 4
+; RV32-NEXT:    vsetvli zero, a1, e16, m2, ta, ma
+; RV32-NEXT:    vwmulsu.vx v16, v12, a2
+; RV32-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
+; RV32-NEXT:    vsoxei32.v v8, (a0), v16, v0.t
 ; RV32-NEXT:    ret
 ;
 ; RV64-LABEL: vpscatter_baseidx_nxv8i16_nxv8i32:
@@ -705,10 +706,11 @@ define void @vpscatter_baseidx_nxv8i16_nxv8i32(<vscale x 8 x i32> %val, ptr %bas
 define void @vpscatter_baseidx_sext_nxv8i16_nxv8i32(<vscale x 8 x i32> %val, ptr %base, <vscale x 8 x i16> %idxs, <vscale x 8 x i1> %m, i32 zeroext %evl) {
 ; RV32-LABEL: vpscatter_baseidx_sext_nxv8i16_nxv8i32:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    vsetvli zero, a1, e32, m4, ta, ma
-; RV32-NEXT:    vsext.vf2 v16, v12
-; RV32-NEXT:    vsll.vi v12, v16, 2
-; RV32-NEXT:    vsoxei32.v v8, (a0), v12, v0.t
+; RV32-NEXT:    li a2, 4
+; RV32-NEXT:    vsetvli zero, a1, e16, m2, ta, ma
+; RV32-NEXT:    vwmulsu.vx v16, v12, a2
+; RV32-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
+; RV32-NEXT:    vsoxei32.v v8, (a0), v16, v0.t
 ; RV32-NEXT:    ret
 ;
 ; RV64-LABEL: vpscatter_baseidx_sext_nxv8i16_nxv8i32:
@@ -728,18 +730,20 @@ define void @vpscatter_baseidx_sext_nxv8i16_nxv8i32(<vscale x 8 x i32> %val, ptr
 define void @vpscatter_baseidx_zext_nxv8i16_nxv8i32(<vscale x 8 x i32> %val, ptr %base, <vscale x 8 x i16> %idxs, <vscale x 8 x i1> %m, i32 zeroext %evl) {
 ; RV32-LABEL: vpscatter_baseidx_zext_nxv8i16_nxv8i32:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    vsetvli zero, a1, e32, m4, ta, ma
-; RV32-NEXT:    vzext.vf2 v16, v12
-; RV32-NEXT:    vsll.vi v12, v16, 2
-; RV32-NEXT:    vsoxei32.v v8, (a0), v12, v0.t
+; RV32-NEXT:    li a2, 4
+; RV32-NEXT:    vsetvli zero, a1, e16, m2, ta, ma
+; RV32-NEXT:    vwmulu.vx v16, v12, a2
+; RV32-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
+; RV32-NEXT:    vsoxei32.v v8, (a0), v16, v0.t
 ; RV32-NEXT:    ret
 ;
 ; RV64-LABEL: vpscatter_baseidx_zext_nxv8i16_nxv8i32:
 ; RV64:       # %bb.0:
-; RV64-NEXT:    vsetvli zero, a1, e32, m4, ta, ma
-; RV64-NEXT:    vzext.vf2 v16, v12
-; RV64-NEXT:    vsll.vi v12, v16, 2
-; RV64-NEXT:    vsoxei32.v v8, (a0), v12, v0.t
+; RV64-NEXT:    li a2, 4
+; RV64-NEXT:    vsetvli zero, a1, e16, m2, ta, ma
+; RV64-NEXT:    vwmulu.vx v16, v12, a2
+; RV64-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
+; RV64-NEXT:    vsoxei32.v v8, (a0), v16, v0.t
 ; RV64-NEXT:    ret
   %eidxs = zext <vscale x 8 x i16> %idxs to <vscale x 8 x i32>
   %ptrs = getelementptr inbounds i32, ptr %base, <vscale x 8 x i32> %eidxs
@@ -757,10 +761,9 @@ define void @vpscatter_baseidx_nxv8i32(<vscale x 8 x i32> %val, ptr %base, <vsca
 ;
 ; RV64-LABEL: vpscatter_baseidx_nxv8i32:
 ; RV64:       # %bb.0:
-; RV64-NEXT:    vsetvli zero, a1, e64, m8, ta, ma
-; RV64-NEXT:    vsext.vf2 v16, v12
-; RV64-NEXT:    vsll.vi v16, v16, 2
-; RV64-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
+; RV64-NEXT:    li a2, 4
+; RV64-NEXT:    vsetvli zero, a1, e32, m4, ta, ma
+; RV64-NEXT:    vwmulsu.vx v16, v12, a2
 ; RV64-NEXT:    vsoxei64.v v8, (a0), v16, v0.t
 ; RV64-NEXT:    ret
   %ptrs = getelementptr inbounds i32, ptr %base, <vscale x 8 x i32> %idxs
@@ -904,20 +907,20 @@ define void @vpscatter_baseidx_sext_nxv8i8_nxv8i64(<vscale x 8 x i64> %val, ptr
 define void @vpscatter_baseidx_zext_nxv8i8_nxv8i64(<vscale x 8 x i64> %val, ptr %base, <vscale x 8 x i8> %idxs, <vscale x 8 x i1> %m, i32 zeroext %evl) {
 ; RV32-LABEL: vpscatter_baseidx_zext_nxv8i8_nxv8i64:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    vsetvli zero, a1, e16, m2, ta, ma
-; RV32-NEXT:    vzext.vf2 v18, v16
-; RV32-NEXT:    vsll.vi v16, v18, 3
+; RV32-NEXT:    li a2, 8
+; RV32-NEXT:    vsetvli zero, a1, e8, m1, ta, ma
+; RV32-NEXT:    vwmulu.vx v18, v16, a2
 ; RV32-NEXT:    vsetvli zero, zero, e64, m8, ta, ma
-; RV32-NEXT:    vsoxei16.v v8, (a0), v16, v0.t
+; RV32-NEXT:    vsoxei16.v v8, (a0), v18, v0.t
 ; RV32-NEXT:    ret
 ;
 ; RV64-LABEL: vpscatter_baseidx_zext_nxv8i8_nxv8i64:
 ; RV64:       # %bb.0:
-; RV64-NEXT:    vsetvli zero, a1, e16, m2, ta, ma
-; RV64-NEXT:    vzext.vf2 v18, v16
-; RV64-NEXT:    vsll.vi v16, v18, 3
+; RV64-NEXT:    li a2, 8
+; RV64-NEXT:    vsetvli zero, a1, e8, m1, ta, ma
+; RV64-NEXT:    vwmulu.vx v18, v16, a2
 ; RV64-NEXT:    vsetvli zero, zero, e64, m8, ta, ma
-; RV64-NEXT:    vsoxei16.v v8, (a0), v16, v0.t
+; RV64-NEXT:    vsoxei16.v v8, (a0), v18, v0.t
 ; RV64-NEXT:    ret
   %eidxs = zext <vscale x 8 x i8> %idxs to <vscale x 8 x i64>
   %ptrs = getelementptr inbounds i64, ptr %base, <vscale x 8 x i64> %eidxs
@@ -928,11 +931,11 @@ define void @vpscatter_baseidx_zext_nxv8i8_nxv8i64(<vscale x 8 x i64> %val, ptr
 define void @vpscatter_baseidx_nxv8i16_nxv8i64(<vscale x 8 x i64> %val, ptr %base, <vscale x 8 x i16> %idxs, <vscale x 8 x i1> %m, i32 zeroext %evl) {
 ; RV32-LABEL: vpscatter_baseidx_nxv8i16_nxv8i64:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    vsetvli zero, a1, e32, m4, ta, ma
-; RV32-NEXT:    vsext.vf2 v20, v16
-; RV32-NEXT:    vsll.vi v16, v20, 3
+; RV32-NEXT:    li a2, 8
+; RV32-NEXT:    vsetvli zero, a1, e16, m2, ta, ma
+; RV32-NEXT:    vwmulsu.vx v20, v16, a2
 ; RV32-NEXT:    vsetvli zero, zero, e64, m8, ta, ma
-; RV32-NEXT:    vsoxei32.v v8, (a0), v16, v0.t
+; RV32-NEXT:    vsoxei32.v v8, (a0), v20, v0.t
 ; RV32-NEXT:    ret
 ;
 ; RV64-LABEL: vpscatter_baseidx_nxv8i16_nxv8i64:
@@ -950,11 +953,11 @@ define void @vpscatter_baseidx_nxv8i16_nxv8i64(<vscale x 8 x i64> %val, ptr %bas
 define void @vpscatter_baseidx_sext_nxv8i16_nxv8i64(<vscale x 8 x i64> %val, ptr %base, <vscale x 8 x i16> %idxs, <vscale x 8 x i1> %m, i32 zeroext %evl) {
 ; RV32-LABEL: vpscatter_baseidx_sext_nxv8i16_nxv8i64:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    vsetvli zero, a1, e32, m4, ta, ma
-; RV32-NEXT:    vsext.vf2 v20, v16
-; RV32-NEXT:    vsll.vi v16, v20, 3
+; RV32-NEXT:    li a2, 8
+; RV32-NEXT:    vsetvli zero, a1, e16, m2, ta, ma
+; RV32-NEXT:    vwmulsu.vx v20, v16, a2
 ; RV32-NEXT:    vsetvli zero, zero, e64, m8, ta, ma
-; RV32-NEXT:    vsoxei32.v v8, (a0), v16, v0.t
+; RV32-NEXT:    vsoxei32.v v8, (a0), v20, v0.t
 ; RV32-NEXT:    ret
 ;
 ; RV64-LABEL: vpscatter_baseidx_sext_nxv8i16_nxv8i64:
@@ -973,20 +976,20 @@ define void @vpscatter_baseidx_sext_nxv8i16_nxv8i64(<vscale x 8 x i64> %val, ptr
 define void @vpscatter_baseidx_zext_nxv8i16_nxv8i64(<vscale x 8 x i64> %val, ptr %base, <vscale x 8 x i16> %idxs, <vscale x 8 x i1> %m, i32 zeroext %evl) {
 ; RV32-LABEL: vpscatter_baseidx_zext_nxv8i16_nxv8i64:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    vsetvli zero, a1, e32, m4, ta, ma
-; RV32-NEXT:    vzext.vf2 v20, v16
-; RV32-NEXT:    vsll.vi v16, v20, 3
+; RV32-NEXT:    li a2, 8
+; RV32-NEXT:    vsetvli zero, a1, e16, m2, ta, ma
+; RV32-NEXT:    vwmulu.vx v20, v16, a2
 ; RV32-NEXT:    vsetvli zero, zero, e64, m8, ta, ma
-; RV32-NEXT:    vsoxei32.v v8, (a0), v16, v0.t
+; RV32-NEXT:    vsoxei32.v v8, (a0), v20, v0.t
 ; RV32-NEXT:    ret
 ;
 ; RV64-LABEL: vpscatter_baseidx_zext_nxv8i16_nxv8i64:
 ; RV64:       # %bb.0:
-; RV64-NEXT:    vsetvli zero, a1, e32, m4, ta, ma
-; RV64-NEXT:    vzext.vf2 v20, v16
-; RV64-NEXT:    vsll.vi v16, v20, 3
+; RV64-NEXT:    li a2, 8
+; RV64-NEXT:    vsetvli zero, a1, e16, m2, ta, ma
+; RV64-NEXT:    vwmulu.vx v20, v16, a2
 ; RV64-NEXT:    vsetvli zero, zero, e64, m8, ta, ma
-; RV64-NEXT:    vsoxei32.v v8, (a0), v16, v0.t
+; RV64-NEXT:    vsoxei32.v v8, (a0), v20, v0.t
 ; RV64-NEXT:    ret
   %eidxs = zext <vscale x 8 x i16> %idxs to <vscale x 8 x i64>
   %ptrs = getelementptr inbounds i64, ptr %base, <vscale x 8 x i64> %eidxs
@@ -1005,10 +1008,11 @@ define void @vpscatter_baseidx_nxv8i32_nxv8i64(<vscale x 8 x i64> %val, ptr %bas
 ;
 ; RV64-LABEL: vpscatter_baseidx_nxv8i32_nxv8i64:
 ; RV64:       # %bb.0:
-; RV64-NEXT:    vsetvli zero, a1, e64, m8, ta, ma
-; RV64-NEXT:    vsext.vf2 v24, v16
-; RV64-NEXT:    vsll.vi v16, v24, 3
-; RV64-NEXT:    vsoxei64.v v8, (a0), v16, v0.t
+; RV64-NEXT:    li a2, 8
+; RV64-NEXT:    vsetvli zero, a1, e32, m4, ta, ma
+; RV64-NEXT:    vwmulsu.vx v24, v16, a2
+; RV64-NEXT:    vsetvli zero, zero, e64, m8, ta, ma
+; RV64-NEXT:    vsoxei64.v v8, (a0), v24, v0.t
 ; RV64-NEXT:    ret
   %ptrs = getelementptr inbounds i64, ptr %base, <vscale x 8 x i32> %idxs
   call void @llvm.vp.scatter.nxv8i64.nxv8p0(<vscale x 8 x i64> %val, <vscale x 8 x ptr> %ptrs, <vscale x 8 x i1> %m, i32 %evl)
@@ -1026,10 +1030,11 @@ define void @vpscatter_baseidx_sext_nxv8i32_nxv8i64(<vscale x 8 x i64> %val, ptr
 ;
 ; RV64-LABEL: vpscatter_baseidx_sext_nxv8i32_nxv8i64:
 ; RV64:       # %bb.0:
-; RV64-NEXT:    vsetvli zero, a1, e64, m8, ta, ma
-; RV64-NEXT:    vsext.vf2 v24, v16
-; RV64-NEXT:    vsll.vi v16, v24, 3
-; RV64-NEXT:    vsoxei64.v v8, (a0), v16, v0.t
+; RV64-NEXT:    li a2, 8
+; RV64-NEXT:    vsetvli zero, a1, e32, m4, ta, ma
+; RV64-NEXT:    vwmulsu.vx v24, v16, a2
+; RV64-NEXT:    vsetvli zero, zero, e64, m8, ta, ma
+; RV64-NEXT:    vsoxei64.v v8, (a0), v24, v0.t
 ; RV64-NEXT:    ret
   %eidxs = sext <vscale x 8 x i32> %idxs to <vscale x 8 x i64>
   %ptrs = getelementptr inbounds i64, ptr %base, <vscale x 8 x i64> %eidxs
@@ -1048,10 +1053,11 @@ define void @vpscatter_baseidx_zext_nxv8i32_nxv8i64(<vscale x 8 x i64> %val, ptr
 ;
 ; RV64-LABEL: vpscatter_baseidx_zext_nxv8i32_nxv8i64:
 ; RV64:       # %bb.0:
-; RV64-NEXT:    vsetvli zero, a1, e64, m8, ta, ma
-; RV64-NEXT:    vzext.vf2 v24, v16
-; RV64-NEXT:    vsll.vi v16, v24, 3
-; RV64-NEXT:    vsoxei64.v v8, (a0), v16, v0.t
+; RV64-NEXT:    li a2, 8
+; RV64-NEXT:    vsetvli zero, a1, e32, m4, ta, ma
+; RV64-NEXT:    vwmulu.vx v24, v16, a2
+; RV64-NEXT:    vsetvli zero, zero, e64, m8, ta, ma
+; RV64-NEXT:    vsoxei64.v v8, (a0), v24, v0.t
 ; RV64-NEXT:    ret
   %eidxs = zext <vscale x 8 x i32> %idxs to <vscale x 8 x i64>
   %ptrs = getelementptr inbounds i64, ptr %base, <vscale x 8 x i64> %eidxs
@@ -1572,20 +1578,20 @@ define void @vpscatter_baseidx_sext_nxv8i8_nxv8f32(<vscale x 8 x float> %val, pt
 define void @vpscatter_baseidx_zext_nxv8i8_nxv8f32(<vscale x 8 x float> %val, ptr %base, <vscale x 8 x i8> %idxs, <vscale x 8 x i1> %m, i32 zeroext %evl) {
 ; RV32-LABEL: vpscatter_baseidx_zext_nxv8i8_nxv8f32:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    vsetvli zero, a1, e16, m2, ta, ma
-; RV32-NEXT:    vzext.vf2 v14, v12
-; RV32-NEXT:    vsll.vi v12, v14, 2
+; RV32-NEXT:    li a2, 4
+; RV32-NEXT:    vsetvli zero, a1, e8, m1, ta, ma
+; RV32-NEXT:    vwmulu.vx v14, v12, a2
 ; RV32-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
-; RV32-NEXT:    vsoxei16.v v8, (a0), v12, v0.t
+; RV32-NEXT:    vsoxei16.v v8, (a0), v14, v0.t
 ; RV32-NEXT:    ret
 ;
 ; RV64-LABEL: vpscatter_baseidx_zext_nxv8i8_nxv8f32:
 ; RV64:       # %bb.0:
-; RV64-NEXT:    vsetvli zero, a1, e16, m2, ta, ma
-; RV64-NEXT:    vzext.vf2 v14, v12
-; RV64-NEXT:    vsll.vi v12, v14, 2
+; RV64-NEXT:    li a2, 4
+; RV64-NEXT:    vsetvli zero, a1, e8, m1, ta, ma
+; RV64-NEXT:    vwmulu.vx v14, v12, a2
 ; RV64-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
-; RV64-NEXT:    vsoxei16.v v8, (a0), v12, v0.t
+; RV64-NEXT:    vsoxei16.v v8, (a0), v14, v0.t
 ; RV64-NEXT:    ret
   %eidxs = zext <vscale x 8 x i8> %idxs to <vscale x 8 x i32>
   %ptrs = getelementptr inbounds float, ptr %base, <vscale x 8 x i32> %eidxs
@@ -1596,10 +1602,11 @@ define void @vpscatter_baseidx_zext_nxv8i8_nxv8f32(<vscale x 8 x float> %val, pt
 define void @vpscatter_baseidx_nxv8i16_nxv8f32(<vscale x 8 x float> %val, ptr %base, <vscale x 8 x i16> %idxs, <vscale x 8 x i1> %m, i32 zeroext %evl) {
 ; RV32-LABEL: vpscatter_baseidx_nxv8i16_nxv8f32:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    vsetvli zero, a1, e32, m4, ta, ma
-; RV32-NEXT:    vsext.vf2 v16, v12
-; RV32-NEXT:    vsll.vi v12, v16, 2
-; RV32-NEXT:    vsoxei32.v v8, (a0), v12, v0.t
+; RV32-NEXT:    li a2, 4
+; RV32-NEXT:    vsetvli zero, a1, e16, m2, ta, ma
+; RV32-NEXT:    vwmulsu.vx v16, v12, a2
+; RV32-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
+; RV32-NEXT:    vsoxei32.v v8, (a0), v16, v0.t
 ; RV32-NEXT:    ret
 ;
 ; RV64-LABEL: vpscatter_baseidx_nxv8i16_nxv8f32:
@@ -1618,10 +1625,11 @@ define void @vpscatter_baseidx_nxv8i16_nxv8f32(<vscale x 8 x float> %val, ptr %b
 define void @vpscatter_baseidx_sext_nxv8i16_nxv8f32(<vscale x 8 x float> %val, ptr %base, <vscale x 8 x i16> %idxs, <vscale x 8 x i1> %m, i32 zeroext %evl) {
 ; RV32-LABEL: vpscatter_baseidx_sext_nxv8i16_nxv8f32:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    vsetvli zero, a1, e32, m4, ta, ma
-; RV32-NEXT:    vsext.vf2 v16, v12
-; RV32-NEXT:    vsll.vi v12, v16, 2
-; RV32-NEXT:    vsoxei32.v v8, (a0), v12, v0.t
+; RV32-NEXT:    li a2, 4
+; RV32-NEXT:    vsetvli zero, a1, e16, m2, ta, ma
+; RV32-NEXT:    vwmulsu.vx v16, v12, a2
+; RV32-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
+; RV32-NEXT:    vsoxei32.v v8, (a0), v16, v0.t
 ; RV32-NEXT:    ret
 ;
 ; RV64-LABEL: vpscatter_baseidx_sext_nxv8i16_nxv8f32:
@@ -1641,18 +1649,20 @@ define void @vpscatter_baseidx_sext_nxv8i16_nxv8f32(<vscale x 8 x float> %val, p
 define void @vpscatter_baseidx_zext_nxv8i16_nxv8f32(<vscale x 8 x float> %val, ptr %base, <vscale x 8 x i16> %idxs, <vscale x 8 x i1> %m, i32 zeroext %evl) {
 ; RV32-LABEL: vpscatter_baseidx_zext_nxv8i16_nxv8f32:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    vsetvli zero, a1, e32, m4, ta, ma
-; RV32-NEXT:    vzext.vf2 v16, v12
-; RV32-NEXT:    vsll.vi v12, v16, 2
-; RV32-NEXT:    vsoxei32.v v8, (a0), v12, v0.t
+; RV32-NEXT:    li a2, 4
+; RV32-NEXT:    vsetvli zero, a1, e16, m2, ta, ma
+; RV32-NEXT:    vwmulu.vx v16, v12, a2
+; RV32-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
+; RV32-NEXT:    vsoxei32.v v8, (a0), v16, v0.t
 ; RV32-NEXT:    ret
 ;
 ; RV64-LABEL: vpscatter_baseidx_zext_nxv8i16_nxv8f32:
 ; RV64:       # %bb.0:
-; RV64-NEXT:    vsetvli zero, a1, e32, m4, ta, ma
-; RV64-NEXT:    vzext.vf2 v16, v12
-; RV64-NEXT:    vsll.vi v12, v16, 2
-; RV64-NEXT:    vsoxei32.v v8, (a0), v12, v0.t
+; RV64-NEXT:    li a2, 4
+; RV64-NEXT:    vsetvli zero, a1, e16, m2, ta, ma
+; RV64-NEXT:    vwmulu.vx v16, v12, a2
+; RV64-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
+; RV64-NEXT:    vsoxei32.v v8, (a0), v16, v0.t
 ; RV64-NEXT:    ret
   %eidxs = zext <vscale x 8 x i16> %idxs to <vscale x 8 x i32>
   %ptrs = getelementptr inbounds float, ptr %base, <vscale x 8 x i32> %eidxs
@@ -1670,10 +1680,9 @@ define void @vpscatter_baseidx_nxv8f32(<vscale x 8 x float> %val, ptr %base, <vs
 ;
 ; RV64-LABEL: vpscatter_baseidx_nxv8f32:
 ; RV64:       # %bb.0:
-; RV64-NEXT:    vsetvli zero, a1, e64, m8, ta, ma
-; RV64-NEXT:    vsext.vf2 v16, v12
-; RV64-NEXT:    vsll.vi v16, v16, 2
-; RV64-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
+; RV64-NEXT:    li a2, 4
+; RV64-NEXT:    vsetvli zero, a1, e32, m4, ta, ma
+; RV64-NEXT:    vwmulsu.vx v16, v12, a2
 ; RV64-NEXT:    vsoxei64.v v8, (a0), v16, v0.t
 ; RV64-NEXT:    ret
   %ptrs = getelementptr inbounds float, ptr %base, <vscale x 8 x i32> %idxs
@@ -1817,20 +1826,20 @@ define void @vpscatter_baseidx_sext_nxv6i8_nxv6f64(<vscale x 6 x double> %val, p
 define void @vpscatter_baseidx_zext_nxv6i8_nxv6f64(<vscale x 6 x double> %val, ptr %base, <vscale x 6 x i8> %idxs, <vscale x 6 x i1> %m, i32 zeroext %evl) {
 ; RV32-LABEL: vpscatter_baseidx_zext_nxv6i8_nxv6f64:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    vsetvli zero, a1, e16, m2, ta, ma
-; RV32-NEXT:    vzext.vf2 v18, v16
-; RV32-NEXT:    vsll.vi v16, v18, 3
+; RV32-NEXT:    li a2, 8
+; RV32-NEXT:    vsetvli zero, a1, e8, m1, ta, ma
+; RV32-NEXT:    vwmulu.vx v18, v16, a2
 ; RV32-NEXT:    vsetvli zero, zero, e64, m8, ta, ma
-; RV32-NEXT:    vsoxei16.v v8, (a0), v16, v0.t
+; RV32-NEXT:    vsoxei16.v v8, (a0), v18, v0.t
 ; RV32-NEXT:    ret
 ;
 ; RV64-LABEL: vpscatter_baseidx_zext_nxv6i8_nxv6f64:
 ; RV64:       # %bb.0:
-; RV64-NEXT:    vsetvli zero, a1, e16, m2, ta, ma
-; RV64-NEXT:    vzext.vf2 v18, v16
-; RV64-NEXT:    vsll.vi v16, v18, 3
+; RV64-NEXT:    li a2, 8
+; RV64-NEXT:    vsetvli zero, a1, e8, m1, ta, ma
+; RV64-NEXT:    vwmulu.vx v18, v16, a2
 ; RV64-NEXT:    vsetvli zero, zero, e64, m8, ta, ma
-; RV64-NEXT:    vsoxei16.v v8, (a0), v16, v0.t
+; RV64-NEXT:    vsoxei16.v v8, (a0), v18, v0.t
 ; RV64-NEXT:    ret
   %eidxs = zext <vscale x 6 x i8> %idxs to <vscale x 6 x i64>
   %ptrs = getelementptr inbounds double, ptr %base, <vscale x 6 x i64> %eidxs
@@ -1841,11 +1850,11 @@ define void @vpscatter_baseidx_zext_nxv6i8_nxv6f64(<vscale x 6 x double> %val, p
 define void @vpscatter_baseidx_nxv6i16_nxv6f64(<vscale x 6 x double> %val, ptr %base, <vscale x 6 x i16> %idxs, <vscale x 6 x i1> %m, i32 zeroext %evl) {
 ; RV32-LABEL: vpscatter_baseidx_nxv6i16_nxv6f64:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    vsetvli zero, a1, e32, m4, ta, ma
-; RV32-NEXT:    vsext.vf2 v20, v16
-; RV32-NEXT:    vsll.vi v16, v20, 3
+; RV32-NEXT:    li a2, 8
+; RV32-NEXT:    vsetvli zero, a1, e16, m2, ta, ma
+; RV32-NEXT:    vwmulsu.vx v20, v16, a2
 ; RV32-NEXT:    vsetvli zero, zero, e64, m8, ta, ma
-; RV32-NEXT:    vsoxei32.v v8, (a0), v16, v0.t
+; RV32-NEXT:    vsoxei32.v v8, (a0), v20, v0.t
 ; RV32-NEXT:    ret
 ;
 ; RV64-LABEL: vpscatter_baseidx_nxv6i16_nxv6f64:
@@ -1863,11 +1872,11 @@ define void @vpscatter_baseidx_nxv6i16_nxv6f64(<vscale x 6 x double> %val, ptr %
 define void @vpscatter_baseidx_sext_nxv6i16_nxv6f64(<vscale x 6 x double> %val, ptr %base, <vscale x 6 x i16> %idxs, <vscale x 6 x i1> %m, i32 zeroext %evl) {
 ; RV32-LABEL: vpscatter_baseidx_sext_nxv6i16_nxv6f64:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    vsetvli zero, a1, e32, m4, ta, ma
-; RV32-NEXT:    vsext.vf2 v20, v16
-; RV32-NEXT:    vsll.vi v16, v20, 3
+; RV32-NEXT:    li a2, 8
+; RV32-NEXT:    vsetvli zero, a1, e16, m2, ta, ma
+; RV32-NEXT:    vwmulsu.vx v20, v16, a2
 ; RV32-NEXT:    vsetvli zero, zero, e64, m8, ta, ma
-; RV32-NEXT:    vsoxei32.v v8, (a0), v16, v0.t
+; RV32-NEXT:    vsoxei32.v v8, (a0), v20, v0.t
 ; RV32-NEXT:    ret
 ;
 ; RV64-LABEL: vpscatter_baseidx_sext_nxv6i16_nxv6f64:
@@ -1886,20 +1895,20 @@ define void @vpscatter_baseidx_sext_nxv6i16_nxv6f64(<vscale x 6 x double> %val,
 define void @vpscatter_baseidx_zext_nxv6i16_nxv6f64(<vscale x 6 x double> %val, ptr %base, <vscale x 6 x i16> %idxs, <vscale x 6 x i1> %m, i32 zeroext %evl) {
 ; RV32-LABEL: vpscatter_baseidx_zext_nxv6i16_nxv6f64:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    vsetvli zero, a1, e32, m4, ta, ma
-; RV32-NEXT:    vzext.vf2 v20, v16
-; RV32-NEXT:    vsll.vi v16, v20, 3
+; RV32-NEXT:    li a2, 8
+; RV32-NEXT:    vsetvli zero, a1, e16, m2, ta, ma
+; RV32-NEXT:    vwmulu.vx v20, v16, a2
 ; RV32-NEXT:    vsetvli zero, zero, e64, m8, ta, ma
-; RV32-NEXT:    vsoxei32.v v8, (a0), v16, v0.t
+; RV32-NEXT:    vsoxei32.v v8, (a0), v20, v0.t
 ; RV32-NEXT:    ret
 ;
 ; RV64-LABEL: vpscatter_baseidx_zext_nxv6i16_nxv6f64:
 ; RV64:       # %bb.0:
-; RV64-NEXT:    vsetvli zero, a1, e32, m4, ta, ma
-; RV64-NEXT:    vzext.vf2 v20, v16
-; RV64-NEXT:    vsll.vi v16, v20, 3
+; RV64-NEXT:    li a2, 8
+; RV64-NEXT:    vsetvli zero, a1, e16, m2, ta, ma
+; RV64-NEXT:    vwmulu.vx v20, v16, a2
 ; RV64-NEXT:    vsetvli zero, zero, e64, m8, ta, ma
-; RV64-NEXT:    vsoxei32.v v8, (a0), v16, v0.t
+; RV64-NEXT:    vsoxei32.v v8, (a0), v20, v0.t
 ; RV64-NEXT:    ret
   %eidxs = zext <vscale x 6 x i16> %idxs to <vscale x 6 x i64>
   %ptrs = getelementptr inbounds double, ptr %base, <vscale x 6 x i64> %eidxs
@@ -1918,10 +1927,11 @@ define void @vpscatter_baseidx_nxv6i32_nxv6f64(<vscale x 6 x double> %val, ptr %
 ;
 ; RV64-LABEL: vpscatter_baseidx_nxv6i32_nxv6f64:
 ; RV64:       # %bb.0:
-; RV64-NEXT:    vsetvli zero, a1, e64, m8, ta, ma
-; RV64-NEXT:    vsext.vf2 v24, v16
-; RV64-NEXT:    vsll.vi v16, v24, 3
-; RV64-NEXT:    vsoxei64.v v8, (a0), v16, v0.t
+; RV64-NEXT:    li a2, 8
+; RV64-NEXT:    vsetvli zero, a1, e32, m4, ta, ma
+; RV64-NEXT:    vwmulsu.vx v24, v16, a2
+; RV64-NEXT:    vsetvli zero, zero, e64, m8, ta, ma
+; RV64-NEXT:    vsoxei64.v v8, (a0), v24, v0.t
 ; RV64-NEXT:    ret
   %ptrs = getelementptr inbounds double, ptr %base, <vscale x 6 x i32> %idxs
   call void @llvm.vp.scatter.nxv6f64.nxv6p0(<vscale x 6 x double> %val, <vscale x 6 x ptr> %ptrs, <vscale x 6 x i1> %m, i32 %evl)
@@ -1939,10 +1949,11 @@ define void @vpscatter_baseidx_sext_nxv6i32_nxv6f64(<vscale x 6 x double> %val,
 ;
 ; RV64-LABEL: vpscatter_baseidx_sext_nxv6i32_nxv6f64:
 ; RV64:       # %bb.0:
-; RV64-NEXT:    vsetvli zero, a1, e64, m8, ta, ma
-; RV64-NEXT:    vsext.vf2 v24, v16
-; RV64-NEXT:    vsll.vi v16, v24, 3
-; RV64-NEXT:    vsoxei64.v v8, (a0), v16, v0.t
+; RV64-NEXT:    li a2, 8
+; RV64-NEXT:    vsetvli zero, a1, e32, m4, ta, ma
+; RV64-NEXT:    vwmulsu.vx v24, v16, a2
+; RV64-NEXT:    vsetvli zero, zero, e64, m8, ta, ma
+; RV64-NEXT:    vsoxei64.v v8, (a0), v24, v0.t
 ; RV64-NEXT:    ret
   %eidxs = sext <vscale x 6 x i32> %idxs to <vscale x 6 x i64>
   %ptrs = getelementptr inbounds double, ptr %base, <vscale x 6 x i64> %eidxs
@@ -1961,10 +1972,11 @@ define void @vpscatter_baseidx_zext_nxv6i32_nxv6f64(<vscale x 6 x double> %val,
 ;
 ; RV64-LABEL: vpscatter_baseidx_zext_nxv6i32_nxv6f64:
 ; RV64:       # %bb.0:
-; RV64-NEXT:    vsetvli zero, a1, e64, m8, ta, ma
-; RV64-NEXT:    vzext.vf2 v24, v16
-; RV64-NEXT:    vsll.vi v16, v24, 3
-; RV64-NEXT:    vsoxei64.v v8, (a0), v16, v0.t
+; RV64-NEXT:    li a2, 8
+; RV64-NEXT:    vsetvli zero, a1, e32, m4, ta, ma
+; RV64-NEXT:    vwmulu.vx v24, v16, a2
+; RV64-NEXT:    vsetvli zero, zero, e64, m8, ta, ma
+; RV64-NEXT:    vsoxei64.v v8, (a0), v24, v0.t
 ; RV64-NEXT:    ret
   %eidxs = zext <vscale x 6 x i32> %idxs to <vscale x 6 x i64>
   %ptrs = getelementptr inbounds double, ptr %base, <vscale x 6 x i64> %eidxs
@@ -2059,20 +2071,20 @@ define void @vpscatter_baseidx_sext_nxv8i8_nxv8f64(<vscale x 8 x double> %val, p
 define void @vpscatter_baseidx_zext_nxv8i8_nxv8f64(<vscale x 8 x double> %val, ptr %base, <vscale x 8 x i8> %idxs, <vscale x 8 x i1> %m, i32 zeroext %evl) {
 ; RV32-LABEL: vpscatter_baseidx_zext_nxv8i8_nxv8f64:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    vsetvli zero, a1, e16, m2, ta, ma
-; RV32-NEXT:    vzext.vf2 v18, v16
-; RV32-NEXT:    vsll.vi v16, v18, 3
+; RV32-NEXT:    li a2, 8
+; RV32-NEXT:    vsetvli zero, a1, e8, m1, ta, ma
+; RV32-NEXT:    vwmulu.vx v18, v16, a2
 ; RV32-NEXT:    vsetvli zero, zero, e64, m8, ta, ma
-; RV32-NEXT:    vsoxei16.v v8, (a0), v16, v0.t
+; RV32-NEXT:    vsoxei16.v v8, (a0), v18, v0.t
 ; RV32-NEXT:    ret
 ;
 ; RV64-LABEL: vpscatter_baseidx_zext_nxv8i8_nxv8f64:
 ; RV64:       # %bb.0:
-; RV64-NEXT:    vsetvli zero, a1, e16, m2, ta, ma
-; RV64-NEXT:    vzext.vf2 v18, v16
-; RV64-NEXT:    vsll.vi v16, v18, 3
+; RV64-NEXT:    li a2, 8
+; RV64-NEXT:    vsetvli zero, a1, e8, m1, ta, ma
+; RV64-NEXT:    vwmulu.vx v18, v16, a2
 ; RV64-NEXT:    vsetvli zero, zero, e64, m8, ta, ma
-; RV64-NEXT:    vsoxei16.v v8, (a0), v16, v0.t
+; RV64-NEXT:    vsoxei16.v v8, (a0), v18, v0.t
 ; RV64-NEXT:    ret
   %eidxs = zext <vscale x 8 x i8> %idxs to <vscale x 8 x i64>
   %ptrs = getelementptr inbounds double, ptr %base, <vscale x 8 x i64> %eidxs
@@ -2083,11 +2095,11 @@ define void @vpscatter_baseidx_zext_nxv8i8_nxv8f64(<vscale x 8 x double> %val, p
 define void @vpscatter_baseidx_nxv8i16_nxv8f64(<vscale x 8 x double> %val, ptr %base, <vscale x 8 x i16> %idxs, <vscale x 8 x i1> %m, i32 zeroext %evl) {
 ; RV32-LABEL: vpscatter_baseidx_nxv8i16_nxv8f64:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    vsetvli zero, a1, e32, m4, ta, ma
-; RV32-NEXT:    vsext.vf2 v20, v16
-; RV32-NEXT:    vsll.vi v16, v20, 3
+; RV32-NEXT:    li a2, 8
+; RV32-NEXT:    vsetvli zero, a1, e16, m2, ta, ma
+; RV32-NEXT:    vwmulsu.vx v20, v16, a2
 ; RV32-NEXT:    vsetvli zero, zero, e64, m8, ta, ma
-; RV32-NEXT:    vsoxei32.v v8, (a0), v16, v0.t
+; RV32-NEXT:    vsoxei32.v v8, (a0), v20, v0.t
 ; RV32-NEXT:    ret
 ;
 ; RV64-LABEL: vpscatter_baseidx_nxv8i16_nxv8f64:
@@ -2105,11 +2117,11 @@ define void @vpscatter_baseidx_nxv8i16_nxv8f64(<vscale x 8 x double> %val, ptr %
 define void @vpscatter_baseidx_sext_nxv8i16_nxv8f64(<vscale x 8 x double> %val, ptr %base, <vscale x 8 x i16> %idxs, <vscale x 8 x i1> %m, i32 zeroext %evl) {
 ; RV32-LABEL: vpscatter_baseidx_sext_nxv8i16_nxv8f64:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    vsetvli zero, a1, e32, m4, ta, ma
-; RV32-NEXT:    vsext.vf2 v20, v16
-; RV32-NEXT:    vsll.vi v16, v20, 3
+; RV32-NEXT:    li a2, 8
+; RV32-NEXT:    vsetvli zero, a1, e16, m2, ta, ma
+; RV32-NEXT:    vwmulsu.vx v20, v16, a2
 ; RV32-NEXT:    vsetvli zero, zero, e64, m8, ta, ma
-; RV32-NEXT:    vsoxei32.v v8, (a0), v16, v0.t
+; RV32-NEXT:    vsoxei32.v v8, (a0), v20, v0.t
 ; RV32-NEXT:    ret
 ;
 ; RV64-LABEL: vpscatter_baseidx_sext_nxv8i16_nxv8f64:
@@ -2128,20 +2140,20 @@ define void @vpscatter_baseidx_sext_nxv8i16_nxv8f64(<vscale x 8 x double> %val,
 define void @vpscatter_baseidx_zext_nxv8i16_nxv8f64(<vscale x 8 x double> %val, ptr %base, <vscale x 8 x i16> %idxs, <vscale x 8 x i1> %m, i32 zeroext %evl) {
 ; RV32-LABEL: vpscatter_baseidx_zext_nxv8i16_nxv8f64:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    vsetvli zero, a1, e32, m4, ta, ma
-; RV32-NEXT:    vzext.vf2 v20, v16
-; RV32-NEXT:    vsll.vi v16, v20, 3
+; RV32-NEXT:    li a2, 8
+; RV32-NEXT:    vsetvli zero, a1, e16, m2, ta, ma
+; RV32-NEXT:    vwmulu.vx v20, v16, a2
 ; RV32-NEXT:    vsetvli zero, zero, e64, m8, ta, ma
-; RV32-NEXT:    vsoxei32.v v8, (a0), v16, v0.t
+; RV32-NEXT:    vsoxei32.v v8, (a0), v20, v0.t
 ; RV32-NEXT:    ret
 ;
 ; RV64-LABEL: vpscatter_baseidx_zext_nxv8i16_nxv8f64:
 ; RV64:       # %bb.0:
-; RV64-NEXT:    vsetvli zero, a1, e32, m4, ta, ma
-; RV64-NEXT:    vzext.vf2 v20, v16
-; RV64-NEXT:    vsll.vi v16, v20, 3
+; RV64-NEXT:    li a2, 8
+; RV64-NEXT:    vsetvli zero, a1, e16, m2, ta, ma
+; RV64-NEXT:    vwmulu.vx v20, v16, a2
 ; RV64-NEXT:    vsetvli zero, zero, e64, m8, ta, ma
-; RV64-NEXT:    vsoxei32.v v8, (a0), v16, v0.t
+; RV64-NEXT:    vsoxei32.v v8, (a0), v20, v0.t
 ; RV64-NEXT:    ret
   %eidxs = zext <vscale x 8 x i16> %idxs to <vscale x 8 x i64>
   %ptrs = getelementptr inbounds double, ptr %base, <vscale x 8 x i64> %eidxs
@@ -2160,10 +2172,11 @@ define void @vpscatter_baseidx_nxv8i32_nxv8f64(<vscale x 8 x double> %val, ptr %
 ;
 ; RV64-LABEL: vpscatter_baseidx_nxv8i32_nxv8f64:
 ; RV64:       # %bb.0:
-; RV64-NEXT:    vsetvli zero, a1, e64, m8, ta, ma
-; RV64-NEXT:    vsext.vf2 v24, v16
-; RV64-NEXT:    vsll.vi v16, v24, 3
-; RV64-NEXT:    vsoxei64.v v8, (a0), v16, v0.t
+; RV64-NEXT:    li a2, 8
+; RV64-NEXT:    vsetvli zero, a1, e32, m4, ta, ma
+; RV64-NEXT:    vwmulsu.vx v24, v16, a2
+; RV64-NEXT:    vsetvli zero, zero, e64, m8, ta, ma
+; RV64-NEXT:    vsoxei64.v v8, (a0), v24, v0.t
 ; RV64-NEXT:    ret
   %ptrs = getelementptr inbounds double, ptr %base, <vscale x 8 x i32> %idxs
   call void @llvm.vp.scatter.nxv8f64.nxv8p0(<vscale x 8 x double> %val, <vscale x 8 x ptr> %ptrs, <vscale x 8 x i1> %m, i32 %evl)
@@ -2181,10 +2194,11 @@ define void @vpscatter_baseidx_sext_nxv8i32_nxv8f64(<vscale x 8 x double> %val,
 ;
 ; RV64-LABEL: vpscatter_baseidx_sext_nxv8i32_nxv8f64:
 ; RV64:       # %bb.0:
-; RV64-NEXT:    vsetvli zero, a1, e64, m8, ta, ma
-; RV64-NEXT:    vsext.vf2 v24, v16
-; RV64-NEXT:    vsll.vi v16, v24, 3
-; RV64-NEXT:    vsoxei64.v v8, (a0), v16, v0.t
+; RV64-NEXT:    li a2, 8
+; RV64-NEXT:    vsetvli zero, a1, e32, m4, ta, ma
+; RV64-NEXT:    vwmulsu.vx v24, v16, a2
+; RV64-NEXT:    vsetvli zero, zero, e64, m8, ta, ma
+; RV64-NEXT:    vsoxei64.v v8, (a0), v24, v0.t
 ; RV64-NEXT:    ret
   %eidxs = sext <vscale x 8 x i32> %idxs to <vscale x 8 x i64>
   %ptrs = getelementptr inbounds double, ptr %base, <vscale x 8 x i64> %eidxs
@@ -2203,10 +2217,11 @@ define void @vpscatter_baseidx_zext_nxv8i32_nxv8f64(<vscale x 8 x double> %val,
 ;
 ; RV64-LABEL: vpscatter_baseidx_zext_nxv8i32_nxv8f64:
 ; RV64:       # %bb.0:
-; RV64-NEXT:    vsetvli zero, a1, e64, m8, ta, ma
-; RV64-NEXT:    vzext.vf2 v24, v16
-; RV64-NEXT:    vsll.vi v16, v24, 3
-; RV64-NEXT:    vsoxei64.v v8, (a0), v16, v0.t
+; RV64-NEXT:    li a2, 8
+; RV64-NEXT:    vsetvli zero, a1, e32, m4, ta, ma
+; RV64-NEXT:    vwmulu.vx v24, v16, a2
+; RV64-NEXT:    vsetvli zero, zero, e64, m8, ta, ma
+; RV64-NEXT:    vsoxei64.v v8, (a0), v24, v0.t
 ; RV64-NEXT:    ret
   %eidxs = zext <vscale x 8 x i32> %idxs to <vscale x 8 x i64>
   %ptrs = getelementptr inbounds double, ptr %base, <vscale x 8 x i64> %eidxs
@@ -2307,25 +2322,16 @@ define void @vpscatter_nxv16f64(<vscale x 16 x double> %val, <vscale x 16 x ptr>
 define void @vpscatter_baseidx_nxv16i16_nxv16f64(<vscale x 16 x double> %val, ptr %base, <vscale x 16 x i16> %idxs, <vscale x 16 x i1> %m, i32 zeroext %evl) {
 ; RV32-LABEL: vpscatter_baseidx_nxv16i16_nxv16f64:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    csrr a3, vlenb
-; RV32-NEXT:    sub sp, sp, a3
-; RV32-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x01, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 1 * vlenb
-; RV32-NEXT:    addi a3, sp, 16
-; RV32-NEXT:    vs1r.v v0, (a3) # Unknown-size Folded Spill
-; RV32-NEXT:    vl4re16.v v24, (a1)
-; RV32-NEXT:    vsetvli a1, zero, e32, m8, ta, ma
-; RV32-NEXT:    vsext.vf2 v0, v24
+; RV32-NEXT:    vl4re16.v v4, (a1)
+; RV32-NEXT:    li a3, 8
 ; RV32-NEXT:    csrr a1, vlenb
-; RV32-NEXT:    vsll.vi v24, v0, 3
+; RV32-NEXT:    vsetvli a4, zero, e16, m4, ta, ma
+; RV32-NEXT:    vwmulsu.vx v24, v4, a3
 ; RV32-NEXT:    mv a3, a2
 ; RV32-NEXT:    bltu a2, a1, .LBB109_2
 ; RV32-NEXT:  # %bb.1:
 ; RV32-NEXT:    mv a3, a1
 ; RV32-NEXT:  .LBB109_2:
-; RV32-NEXT:    addi a4, sp, 16
-; RV32-NEXT:    vl1r.v v0, (a4) # Unknown-size Folded Reload
 ; RV32-NEXT:    vsetvli zero, a3, e64, m8, ta, ma
 ; RV32-NEXT:    vsoxei32.v v8, (a0), v24, v0.t
 ; RV32-NEXT:    sub a3, a2, a1
@@ -2337,11 +2343,6 @@ define void @vpscatter_baseidx_nxv16i16_nxv16f64(<vscale x 16 x double> %val, pt
 ; RV32-NEXT:    and a2, a2, a3
 ; RV32-NEXT:    vsetvli zero, a2, e64, m8, ta, ma
 ; RV32-NEXT:    vsoxei32.v v16, (a0), v28, v0.t
-; RV32-NEXT:    csrr a0, vlenb
-; RV32-NEXT:    add sp, sp, a0
-; RV32-NEXT:    .cfi_def_cfa sp, 16
-; RV32-NEXT:    addi sp, sp, 16
-; RV32-NEXT:    .cfi_def_cfa_offset 0
 ; RV32-NEXT:    ret
 ;
 ; RV64-LABEL: vpscatter_baseidx_nxv16i16_nxv16f64:
@@ -2404,25 +2405,16 @@ define void @vpscatter_baseidx_nxv16i16_nxv16f64(<vscale x 16 x double> %val, pt
 define void @vpscatter_baseidx_sext_nxv16i16_nxv16f64(<vscale x 16 x double> %val, ptr %base, <vscale x 16 x i16> %idxs, <vscale x 16 x i1> %m, i32 zeroext %evl) {
 ; RV32-LABEL: vpscatter_baseidx_sext_nxv16i16_nxv16f64:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    csrr a3, vlenb
-; RV32-NEXT:    sub sp, sp, a3
-; RV32-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x01, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 1 * vlenb
-; RV32-NEXT:    addi a3, sp, 16
-; RV32-NEXT:    vs1r.v v0, (a3) # Unknown-size Folded Spill
-; RV32-NEXT:    vl4re16.v v24, (a1)
-; RV32-NEXT:    vsetvli a1, zero, e32, m8, ta, ma
-; RV32-NEXT:    vsext.vf2 v0, v24
+; RV32-NEXT:    vl4re16.v v4, (a1)
+; RV32-NEXT:    li a3, 8
 ; RV32-NEXT:    csrr a1, vlenb
-; RV32-NEXT:    vsll.vi v24, v0, 3
+; RV32-NEXT:    vsetvli a4, zero, e16, m4, ta, ma
+; RV32-NEXT:    vwmulsu.vx v24, v4, a3
 ; RV32-NEXT:    mv a3, a2
 ; RV32-NEXT:    bltu a2, a1, .LBB110_2
 ; RV32-NEXT:  # %bb.1:
 ; RV32-NEXT:    mv a3, a1
 ; RV32-NEXT:  .LBB110_2:
-; RV32-NEXT:    addi a4, sp, 16
-; RV32-NEXT:    vl1r.v v0, (a4) # Unknown-size Folded Reload
 ; RV32-NEXT:    vsetvli zero, a3, e64, m8, ta, ma
 ; RV32-NEXT:    vsoxei32.v v8, (a0), v24, v0.t
 ; RV32-NEXT:    sub a3, a2, a1
@@ -2434,11 +2426,6 @@ define void @vpscatter_baseidx_sext_nxv16i16_nxv16f64(<vscale x 16 x double> %va
 ; RV32-NEXT:    and a2, a2, a3
 ; RV32-NEXT:    vsetvli zero, a2, e64, m8, ta, ma
 ; RV32-NEXT:    vsoxei32.v v16, (a0), v28, v0.t
-; RV32-NEXT:    csrr a0, vlenb
-; RV32-NEXT:    add sp, sp, a0
-; RV32-NEXT:    .cfi_def_cfa sp, 16
-; RV32-NEXT:    addi sp, sp, 16
-; RV32-NEXT:    .cfi_def_cfa_offset 0
 ; RV32-NEXT:    ret
 ;
 ; RV64-LABEL: vpscatter_baseidx_sext_nxv16i16_nxv16f64:
@@ -2502,25 +2489,16 @@ define void @vpscatter_baseidx_sext_nxv16i16_nxv16f64(<vscale x 16 x double> %va
 define void @vpscatter_baseidx_zext_nxv16i16_nxv16f64(<vscale x 16 x double> %val, ptr %base, <vscale x 16 x i16> %idxs, <vscale x 16 x i1> %m, i32 zeroext %evl) {
 ; RV32-LABEL: vpscatter_baseidx_zext_nxv16i16_nxv16f64:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    csrr a3, vlenb
-; RV32-NEXT:    sub sp, sp, a3
-; RV32-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x01, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 1 * vlenb
-; RV32-NEXT:    addi a3, sp, 16
-; RV32-NEXT:    vs1r.v v0, (a3) # Unknown-size Folded Spill
-; RV32-NEXT:    vl4re16.v v24, (a1)
-; RV32-NEXT:    vsetvli a1, zero, e32, m8, ta, ma
-; RV32-NEXT:    vzext.vf2 v0, v24
+; RV32-NEXT:    vl4re16.v v4, (a1)
+; RV32-NEXT:    li a3, 8
 ; RV32-NEXT:    csrr a1, vlenb
-; RV32-NEXT:    vsll.vi v24, v0, 3
+; RV32-NEXT:    vsetvli a4, zero, e16, m4, ta, ma
+; RV32-NEXT:    vwmulu.vx v24, v4, a3
 ; RV32-NEXT:    mv a3, a2
 ; RV32-NEXT:    bltu a2, a1, .LBB111_2
 ; RV32-NEXT:  # %bb.1:
 ; RV32-NEXT:    mv a3, a1
 ; RV32-NEXT:  .LBB111_2:
-; RV32-NEXT:    addi a4, sp, 16
-; RV32-NEXT:    vl1r.v v0, (a4) # Unknown-size Folded Reload
 ; RV32-NEXT:    vsetvli zero, a3, e64, m8, ta, ma
 ; RV32-NEXT:    vsoxei32.v v8, (a0), v24, v0.t
 ; RV32-NEXT:    sub a3, a2, a1
@@ -2532,34 +2510,20 @@ define void @vpscatter_baseidx_zext_nxv16i16_nxv16f64(<vscale x 16 x double> %va
 ; RV32-NEXT:    and a2, a2, a3
 ; RV32-NEXT:    vsetvli zero, a2, e64, m8, ta, ma
 ; RV32-NEXT:    vsoxei32.v v16, (a0), v28, v0.t
-; RV32-NEXT:    csrr a0, vlenb
-; RV32-NEXT:    add sp, sp, a0
-; RV32-NEXT:    .cfi_def_cfa sp, 16
-; RV32-NEXT:    addi sp, sp, 16
-; RV32-NEXT:    .cfi_def_cfa_offset 0
 ; RV32-NEXT:    ret
 ;
 ; RV64-LABEL: vpscatter_baseidx_zext_nxv16i16_nxv16f64:
 ; RV64:       # %bb.0:
-; RV64-NEXT:    addi sp, sp, -16
-; RV64-NEXT:    .cfi_def_cfa_offset 16
-; RV64-NEXT:    csrr a3, vlenb
-; RV64-NEXT:    sub sp, sp, a3
-; RV64-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x01, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 1 * vlenb
-; RV64-NEXT:    addi a3, sp, 16
-; RV64-NEXT:    vs1r.v v0, (a3) # Unknown-size Folded Spill
-; RV64-NEXT:    vl4re16.v v24, (a1)
-; RV64-NEXT:    vsetvli a1, zero, e32, m8, ta, ma
-; RV64-NEXT:    vzext.vf2 v0, v24
+; RV64-NEXT:    vl4re16.v v4, (a1)
+; RV64-NEXT:    li a3, 8
 ; RV64-NEXT:    csrr a1, vlenb
-; RV64-NEXT:    vsll.vi v24, v0, 3
+; RV64-NEXT:    vsetvli a4, zero, e16, m4, ta, ma
+; RV64-NEXT:    vwmulu.vx v24, v4, a3
 ; RV64-NEXT:    mv a3, a2
 ; RV64-NEXT:    bltu a2, a1, .LBB111_2
 ; RV64-NEXT:  # %bb.1:
 ; RV64-NEXT:    mv a3, a1
 ; RV64-NEXT:  .LBB111_2:
-; RV64-NEXT:    addi a4, sp, 16
-; RV64-NEXT:    vl1r.v v0, (a4) # Unknown-size Folded Reload
 ; RV64-NEXT:    vsetvli zero, a3, e64, m8, ta, ma
 ; RV64-NEXT:    vsoxei32.v v8, (a0), v24, v0.t
 ; RV64-NEXT:    sub a3, a2, a1
@@ -2571,11 +2535,6 @@ define void @vpscatter_baseidx_zext_nxv16i16_nxv16f64(<vscale x 16 x double> %va
 ; RV64-NEXT:    and a2, a2, a3
 ; RV64-NEXT:    vsetvli zero, a2, e64, m8, ta, ma
 ; RV64-NEXT:    vsoxei32.v v16, (a0), v28, v0.t
-; RV64-NEXT:    csrr a0, vlenb
-; RV64-NEXT:    add sp, sp, a0
-; RV64-NEXT:    .cfi_def_cfa sp, 16
-; RV64-NEXT:    addi sp, sp, 16
-; RV64-NEXT:    .cfi_def_cfa_offset 0
 ; RV64-NEXT:    ret
   %eidxs = zext <vscale x 16 x i16> %idxs to <vscale x 16 x i64>
   %ptrs = getelementptr inbounds double, ptr %base, <vscale x 16 x i64> %eidxs
diff --git a/llvm/test/CodeGen/RISCV/rvv/vreductions-fp-vp.ll b/llvm/test/CodeGen/RISCV/rvv/vreductions-fp-vp.ll
index a20f88c697716..ccea5b05fb03c 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vreductions-fp-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vreductions-fp-vp.ll
@@ -465,3 +465,127 @@ define float @vreduce_fmaximum_v4f32(float %start, <4 x float> %val, <4 x i1> %m
   %s = call float @llvm.vp.reduce.fmaximum.v4f32(float %start, <4 x float> %val, <4 x i1> %m, i32 %evl)
   ret float %s
 }
+
+define float @vpreduce_fadd_fpext_vp_nxv1f16_nxv1f32(float %s, <vscale x 1 x half> %v, <vscale x 1 x i1> %m, i32 zeroext %evl) {
+; CHECK-LABEL: vpreduce_fadd_fpext_vp_nxv1f16_nxv1f32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, ma
+; CHECK-NEXT:    vfwcvt.f.f.v v9, v8, v0.t
+; CHECK-NEXT:    vsetivli zero, 1, e32, mf2, ta, ma
+; CHECK-NEXT:    vfmv.s.f v8, fa0
+; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, ta, ma
+; CHECK-NEXT:    vfredusum.vs v8, v9, v8, v0.t
+; CHECK-NEXT:    vfmv.f.s fa0, v8
+; CHECK-NEXT:    ret
+  %w = call <vscale x 1 x float> @llvm.vp.fpext(<vscale x 1 x half> %v, <vscale x 1 x i1> %m, i32 %evl)
+  %r = call reassoc float @llvm.vp.reduce.fadd(float %s, <vscale x 1 x float> %w, <vscale x 1 x i1> %m, i32 %evl)
+  ret float %r
+}
+
+define float @vpreduce_ord_fadd_fpext_vp_fpext_nxv1f16_nxv1f32(float %s, <vscale x 1 x half> %v, <vscale x 1 x i1> %m, i32 zeroext %evl) {
+; CHECK-LABEL: vpreduce_ord_fadd_fpext_vp_fpext_nxv1f16_nxv1f32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, ma
+; CHECK-NEXT:    vfwcvt.f.f.v v9, v8, v0.t
+; CHECK-NEXT:    vsetivli zero, 1, e32, mf2, ta, ma
+; CHECK-NEXT:    vfmv.s.f v8, fa0
+; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, ta, ma
+; CHECK-NEXT:    vfredosum.vs v8, v9, v8, v0.t
+; CHECK-NEXT:    vfmv.f.s fa0, v8
+; CHECK-NEXT:    ret
+  %w = call <vscale x 1 x float> @llvm.vp.fpext(<vscale x 1 x half> %v, <vscale x 1 x i1> %m, i32 %evl)
+  %r = call float @llvm.vp.reduce.fadd(float %s, <vscale x 1 x float> %w, <vscale x 1 x i1> %m, i32 %evl)
+  ret float %r
+}
+
+define double @vpreduce_fadd_fpext_vp_nxv1f32_nxv1f64(double %s, <vscale x 1 x float> %v, <vscale x 1 x i1> %m, i32 zeroext %evl) {
+; CHECK-LABEL: vpreduce_fadd_fpext_vp_nxv1f32_nxv1f64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, ta, ma
+; CHECK-NEXT:    vfwcvt.f.f.v v9, v8, v0.t
+; CHECK-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
+; CHECK-NEXT:    vfmv.s.f v8, fa0
+; CHECK-NEXT:    vsetvli zero, a0, e64, m1, ta, ma
+; CHECK-NEXT:    vfredusum.vs v8, v9, v8, v0.t
+; CHECK-NEXT:    vfmv.f.s fa0, v8
+; CHECK-NEXT:    ret
+  %w = call <vscale x 1 x double> @llvm.vp.fpext(<vscale x 1 x float> %v, <vscale x 1 x i1> %m, i32 %evl)
+  %r = call reassoc double @llvm.vp.reduce.fadd(double %s, <vscale x 1 x double> %w, <vscale x 1 x i1> %m, i32 %evl)
+  ret double %r
+}
+
+define double @vpreduce_ord_fadd_fpext_vp_fpext_nxv1f32_nxv1f64(double %s, <vscale x 1 x float> %v, <vscale x 1 x i1> %m, i32 zeroext %evl) {
+; CHECK-LABEL: vpreduce_ord_fadd_fpext_vp_fpext_nxv1f32_nxv1f64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, ta, ma
+; CHECK-NEXT:    vfwcvt.f.f.v v9, v8, v0.t
+; CHECK-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
+; CHECK-NEXT:    vfmv.s.f v8, fa0
+; CHECK-NEXT:    vsetvli zero, a0, e64, m1, ta, ma
+; CHECK-NEXT:    vfredosum.vs v8, v9, v8, v0.t
+; CHECK-NEXT:    vfmv.f.s fa0, v8
+; CHECK-NEXT:    ret
+  %w = call <vscale x 1 x double> @llvm.vp.fpext(<vscale x 1 x float> %v, <vscale x 1 x i1> %m, i32 %evl)
+  %r = call double @llvm.vp.reduce.fadd(double %s, <vscale x 1 x double> %w, <vscale x 1 x i1> %m, i32 %evl)
+  ret double %r
+}
+
+define float @vpreduce_fadd_fpext_nxv1f16_nxv1f32(float %s, <vscale x 1 x half> %v, <vscale x 1 x i1> %m, i32 zeroext %evl) {
+; CHECK-LABEL: vpreduce_fadd_fpext_nxv1f16_nxv1f32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
+; CHECK-NEXT:    vfmv.s.f v9, fa0
+; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, ma
+; CHECK-NEXT:    vfwredusum.vs v9, v8, v9, v0.t
+; CHECK-NEXT:    vsetvli zero, zero, e32, mf2, ta, ma
+; CHECK-NEXT:    vfmv.f.s fa0, v9
+; CHECK-NEXT:    ret
+  %w = fpext <vscale x 1 x half> %v to <vscale x 1 x float>
+  %r = call reassoc float @llvm.vp.reduce.fadd(float %s, <vscale x 1 x float> %w, <vscale x 1 x i1> %m, i32 %evl)
+  ret float %r
+}
+
+define float @vpreduce_ord_fadd_fpext_nxv1f16_nxv1f32(float %s, <vscale x 1 x half> %v, <vscale x 1 x i1> %m, i32 zeroext %evl) {
+; CHECK-LABEL: vpreduce_ord_fadd_fpext_nxv1f16_nxv1f32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
+; CHECK-NEXT:    vfmv.s.f v9, fa0
+; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, ma
+; CHECK-NEXT:    vfwredosum.vs v9, v8, v9, v0.t
+; CHECK-NEXT:    vsetvli zero, zero, e32, mf2, ta, ma
+; CHECK-NEXT:    vfmv.f.s fa0, v9
+; CHECK-NEXT:    ret
+  %w = fpext <vscale x 1 x half> %v to <vscale x 1 x float>
+  %r = call float @llvm.vp.reduce.fadd(float %s, <vscale x 1 x float> %w, <vscale x 1 x i1> %m, i32 %evl)
+  ret float %r
+}
+
+define double @vpreduce_fadd_fpext_nxv1f32_nxv1f64(double %s, <vscale x 1 x float> %v, <vscale x 1 x i1> %m, i32 zeroext %evl) {
+; CHECK-LABEL: vpreduce_fadd_fpext_nxv1f32_nxv1f64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
+; CHECK-NEXT:    vfmv.s.f v9, fa0
+; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, ta, ma
+; CHECK-NEXT:    vfwredusum.vs v9, v8, v9, v0.t
+; CHECK-NEXT:    vsetvli zero, zero, e64, m1, ta, ma
+; CHECK-NEXT:    vfmv.f.s fa0, v9
+; CHECK-NEXT:    ret
+  %w = fpext <vscale x 1 x float> %v to <vscale x 1 x double>
+  %r = call reassoc double @llvm.vp.reduce.fadd(double %s, <vscale x 1 x double> %w, <vscale x 1 x i1> %m, i32 %evl)
+  ret double %r
+}
+
+define double @vpreduce_ord_fadd_fpext_nxv1f32_nxv1f64(double %s, <vscale x 1 x float> %v, <vscale x 1 x i1> %m, i32 zeroext %evl) {
+; CHECK-LABEL: vpreduce_ord_fadd_fpext_nxv1f32_nxv1f64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
+; CHECK-NEXT:    vfmv.s.f v9, fa0
+; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, ta, ma
+; CHECK-NEXT:    vfwredosum.vs v9, v8, v9, v0.t
+; CHECK-NEXT:    vsetvli zero, zero, e64, m1, ta, ma
+; CHECK-NEXT:    vfmv.f.s fa0, v9
+; CHECK-NEXT:    ret
+  %w = fpext <vscale x 1 x float> %v to <vscale x 1 x double>
+  %r = call double @llvm.vp.reduce.fadd(double %s, <vscale x 1 x double> %w, <vscale x 1 x i1> %m, i32 %evl)
+  ret double %r
+}
diff --git a/llvm/test/CodeGen/RISCV/rvv/vsetvli-insert.mir b/llvm/test/CodeGen/RISCV/rvv/vsetvli-insert.mir
index a9cf741b1cb2a..b7704e5d03099 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vsetvli-insert.mir
+++ b/llvm/test/CodeGen/RISCV/rvv/vsetvli-insert.mir
@@ -139,7 +139,7 @@ body:             |
     ; CHECK-NEXT: [[COPY2:%[0-9]+]]:vr = COPY $v8
     ; CHECK-NEXT: dead $x0 = PseudoVSETVLI [[COPY]], 216 /* e64, m1, ta, ma */, implicit-def $vl, implicit-def $vtype
     ; CHECK-NEXT: [[PseudoVADD_VV_M1_:%[0-9]+]]:vr = PseudoVADD_VV_M1 undef $noreg, [[COPY2]], [[COPY1]], $noreg, 6 /* e64 */, 0 /* tu, mu */, implicit $vl, implicit $vtype
-    ; CHECK-NEXT: $v8 = COPY [[PseudoVADD_VV_M1_]]
+    ; CHECK-NEXT: $v8 = COPY [[PseudoVADD_VV_M1_]], implicit $vtype
     ; CHECK-NEXT: PseudoRET implicit $v8
     %2:gprnox0 = COPY $x10
     %1:vr = COPY $v9
@@ -179,7 +179,7 @@ body:             |
     ; CHECK-NEXT: dead $x0 = PseudoVSETVLI [[COPY]], 216 /* e64, m1, ta, ma */, implicit-def $vl, implicit-def $vtype
     ; CHECK-NEXT: [[PseudoVLE64_V_M1_:%[0-9]+]]:vr = PseudoVLE64_V_M1 undef $noreg, [[COPY2]], $noreg, 6 /* e64 */, 0 /* tu, mu */, implicit $vl, implicit $vtype
     ; CHECK-NEXT: [[PseudoVADD_VV_M1_:%[0-9]+]]:vr = PseudoVADD_VV_M1 undef $noreg, [[PseudoVLE64_V_M1_]], [[COPY1]], $noreg, 6 /* e64 */, 0 /* tu, mu */, implicit $vl, implicit $vtype
-    ; CHECK-NEXT: $v8 = COPY [[PseudoVADD_VV_M1_]]
+    ; CHECK-NEXT: $v8 = COPY [[PseudoVADD_VV_M1_]], implicit $vtype
     ; CHECK-NEXT: PseudoRET implicit $v8
     %2:gprnox0 = COPY $x11
     %1:vr = COPY $v8
@@ -217,7 +217,7 @@ body:             |
     ; CHECK-NEXT: dead $x0 = PseudoVSETVLI [[COPY]], 216 /* e64, m1, ta, ma */, implicit-def $vl, implicit-def $vtype
     ; CHECK-NEXT: [[PseudoVLE32_V_MF2_:%[0-9]+]]:vr = PseudoVLE32_V_MF2 undef $noreg, [[COPY1]], $noreg, 5 /* e32 */, 0 /* tu, mu */, implicit $vl, implicit $vtype
     ; CHECK-NEXT: early-clobber %3:vr = PseudoVZEXT_VF2_M1 undef $noreg, [[PseudoVLE32_V_MF2_]], $noreg, 6 /* e64 */, 0 /* tu, mu */, implicit $vl, implicit $vtype
-    ; CHECK-NEXT: $v8 = COPY %3
+    ; CHECK-NEXT: $v8 = COPY %3, implicit $vtype
     ; CHECK-NEXT: PseudoRET implicit $v8
     %1:gprnox0 = COPY $x11
     %0:gpr = COPY $x10
@@ -368,7 +368,7 @@ body:             |
     ; CHECK-NEXT: [[COPY2:%[0-9]+]]:vr = COPY $v8
     ; CHECK-NEXT: dead [[PseudoVSETVLI:%[0-9]+]]:gprnox0 = PseudoVSETVLI [[COPY]], 88 /* e64, m1, ta, mu */, implicit-def $vl, implicit-def $vtype
     ; CHECK-NEXT: [[PseudoVADD_VV_M1_:%[0-9]+]]:vr = PseudoVADD_VV_M1 undef $noreg, [[COPY2]], [[COPY1]], $noreg, 6 /* e64 */, 0 /* tu, mu */, implicit $vl, implicit $vtype
-    ; CHECK-NEXT: $v8 = COPY [[PseudoVADD_VV_M1_]]
+    ; CHECK-NEXT: $v8 = COPY [[PseudoVADD_VV_M1_]], implicit $vtype
     ; CHECK-NEXT: PseudoRET implicit $v8
     %2:gprnox0 = COPY $x10
     %1:vr = COPY $v9
@@ -411,7 +411,7 @@ body:             |
     ; CHECK-NEXT: INLINEASM &"", 1 /* sideeffect attdialect */
     ; CHECK-NEXT: dead $x0 = PseudoVSETVLI [[COPY]], 216 /* e64, m1, ta, ma */, implicit-def $vl, implicit-def $vtype
     ; CHECK-NEXT: [[PseudoVADD_VV_M1_:%[0-9]+]]:vr = PseudoVADD_VV_M1 undef $noreg, [[PseudoVLE64_V_M1_]], [[COPY1]], $noreg, 6 /* e64 */, 0 /* tu, mu */, implicit $vl, implicit $vtype
-    ; CHECK-NEXT: $v8 = COPY [[PseudoVADD_VV_M1_]]
+    ; CHECK-NEXT: $v8 = COPY [[PseudoVADD_VV_M1_]], implicit $vtype
     ; CHECK-NEXT: PseudoRET implicit $v8
     %2:gprnox0 = COPY $x11
     %1:vr = COPY $v8
diff --git a/llvm/test/CodeGen/RISCV/rvv/vwsll-sdnode.ll b/llvm/test/CodeGen/RISCV/rvv/vwsll-sdnode.ll
index ff807adf0e59f..fd09fe791b4fd 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vwsll-sdnode.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vwsll-sdnode.ll
@@ -224,9 +224,10 @@ define <vscale x 2 x i64> @vwsll_vx_i8_nxv2i64_zext(<vscale x 2 x i32> %a, i8 %b
 define <vscale x 2 x i64> @vwsll_vi_nxv2i64(<vscale x 2 x i32> %a) {
 ; CHECK-LABEL: vwsll_vi_nxv2i64:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a0, zero, e64, m2, ta, ma
-; CHECK-NEXT:    vzext.vf2 v10, v8
-; CHECK-NEXT:    vsll.vi v8, v10, 2
+; CHECK-NEXT:    li a0, 4
+; CHECK-NEXT:    vsetvli a1, zero, e32, m1, ta, ma
+; CHECK-NEXT:    vwmulu.vx v10, v8, a0
+; CHECK-NEXT:    vmv2r.v v8, v10
 ; CHECK-NEXT:    ret
 ;
 ; CHECK-ZVBB-LABEL: vwsll_vi_nxv2i64:
@@ -432,9 +433,10 @@ define <vscale x 4 x i32> @vwsll_vx_i8_nxv4i32_zext(<vscale x 4 x i16> %a, i8 %b
 define <vscale x 4 x i32> @vwsll_vi_nxv4i32(<vscale x 4 x i16> %a) {
 ; CHECK-LABEL: vwsll_vi_nxv4i32:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a0, zero, e32, m2, ta, ma
-; CHECK-NEXT:    vzext.vf2 v10, v8
-; CHECK-NEXT:    vsll.vi v8, v10, 2
+; CHECK-NEXT:    li a0, 4
+; CHECK-NEXT:    vsetvli a1, zero, e16, m1, ta, ma
+; CHECK-NEXT:    vwmulu.vx v10, v8, a0
+; CHECK-NEXT:    vmv2r.v v8, v10
 ; CHECK-NEXT:    ret
 ;
 ; CHECK-ZVBB-LABEL: vwsll_vi_nxv4i32:
@@ -612,9 +614,10 @@ define <vscale x 8 x i16> @vwsll_vx_i8_nxv8i16_zext(<vscale x 8 x i8> %a, i8 %b)
 define <vscale x 8 x i16> @vwsll_vi_nxv8i16(<vscale x 8 x i8> %a) {
 ; CHECK-LABEL: vwsll_vi_nxv8i16:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a0, zero, e16, m2, ta, ma
-; CHECK-NEXT:    vzext.vf2 v10, v8
-; CHECK-NEXT:    vsll.vi v8, v10, 2
+; CHECK-NEXT:    li a0, 4
+; CHECK-NEXT:    vsetvli a1, zero, e8, m1, ta, ma
+; CHECK-NEXT:    vwmulu.vx v10, v8, a0
+; CHECK-NEXT:    vmv2r.v v8, v10
 ; CHECK-NEXT:    ret
 ;
 ; CHECK-ZVBB-LABEL: vwsll_vi_nxv8i16:
diff --git a/llvm/test/CodeGen/RISCV/rvv/vwsll-vp.ll b/llvm/test/CodeGen/RISCV/rvv/vwsll-vp.ll
index c30c4763dd46d..1358a7c69cb8a 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vwsll-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vwsll-vp.ll
@@ -240,9 +240,10 @@ define <vscale x 2 x i64> @vwsll_vx_i8_nxv2i64_zext(<vscale x 2 x i32> %a, i8 %b
 define <vscale x 2 x i64> @vwsll_vi_nxv2i64(<vscale x 2 x i32> %a, <vscale x 2 x i1> %m, i32 zeroext %vl) {
 ; CHECK-LABEL: vwsll_vi_nxv2i64:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli zero, a0, e64, m2, ta, ma
-; CHECK-NEXT:    vzext.vf2 v10, v8
-; CHECK-NEXT:    vsll.vi v8, v10, 2, v0.t
+; CHECK-NEXT:    li a1, 4
+; CHECK-NEXT:    vsetvli zero, a0, e32, m1, ta, ma
+; CHECK-NEXT:    vwmulu.vx v10, v8, a1, v0.t
+; CHECK-NEXT:    vmv2r.v v8, v10
 ; CHECK-NEXT:    ret
 ;
 ; CHECK-ZVBB-LABEL: vwsll_vi_nxv2i64:
@@ -464,9 +465,10 @@ define <vscale x 4 x i32> @vwsll_vx_i8_nxv4i32_zext(<vscale x 4 x i16> %a, i8 %b
 define <vscale x 4 x i32> @vwsll_vi_nxv4i32(<vscale x 4 x i16> %a, <vscale x 4 x i1> %m, i32 zeroext %vl) {
 ; CHECK-LABEL: vwsll_vi_nxv4i32:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
-; CHECK-NEXT:    vzext.vf2 v10, v8
-; CHECK-NEXT:    vsll.vi v8, v10, 2, v0.t
+; CHECK-NEXT:    li a1, 4
+; CHECK-NEXT:    vsetvli zero, a0, e16, m1, ta, ma
+; CHECK-NEXT:    vwmulu.vx v10, v8, a1, v0.t
+; CHECK-NEXT:    vmv2r.v v8, v10
 ; CHECK-NEXT:    ret
 ;
 ; CHECK-ZVBB-LABEL: vwsll_vi_nxv4i32:
@@ -661,9 +663,10 @@ define <vscale x 8 x i16> @vwsll_vx_i8_nxv8i16_zext(<vscale x 8 x i8> %a, i8 %b,
 define <vscale x 8 x i16> @vwsll_vi_nxv8i16(<vscale x 8 x i8> %a, <vscale x 8 x i1> %m, i32 zeroext %vl) {
 ; CHECK-LABEL: vwsll_vi_nxv8i16:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli zero, a0, e16, m2, ta, ma
-; CHECK-NEXT:    vzext.vf2 v10, v8
-; CHECK-NEXT:    vsll.vi v8, v10, 2, v0.t
+; CHECK-NEXT:    li a1, 4
+; CHECK-NEXT:    vsetvli zero, a0, e8, m1, ta, ma
+; CHECK-NEXT:    vwmulu.vx v10, v8, a1, v0.t
+; CHECK-NEXT:    vmv2r.v v8, v10
 ; CHECK-NEXT:    ret
 ;
 ; CHECK-ZVBB-LABEL: vwsll_vi_nxv8i16:
diff --git a/llvm/test/CodeGen/RISCV/select-and.ll b/llvm/test/CodeGen/RISCV/select-and.ll
index d305993f0e966..f827e840f4a36 100644
--- a/llvm/test/CodeGen/RISCV/select-and.ll
+++ b/llvm/test/CodeGen/RISCV/select-and.ll
@@ -3,6 +3,8 @@
 ; RUN:   | FileCheck -check-prefix=RV32I %s
 ; RUN: llc -mtriple=riscv64 -verify-machineinstrs < %s \
 ; RUN:   | FileCheck -check-prefix=RV64I %s
+; RUN: llc -mtriple=riscv64 -mattr=+xmipscmove -verify-machineinstrs < %s \
+; RUN:   | FileCheck -check-prefix=RV64I-CCMOV %s
 
 ;; There are a few different ways to lower (select (and A, B), X, Y). This test
 ;; ensures that we do so with as few branches as possible.
@@ -27,6 +29,12 @@ define signext i32 @select_of_and(i1 zeroext %a, i1 zeroext %b, i32 signext %c,
 ; RV64I-NEXT:    mv a0, a3
 ; RV64I-NEXT:  .LBB0_2:
 ; RV64I-NEXT:    ret
+;
+; RV64I-CCMOV-LABEL: select_of_and:
+; RV64I-CCMOV:       # %bb.0:
+; RV64I-CCMOV-NEXT:    and a0, a0, a1
+; RV64I-CCMOV-NEXT:    mips.ccmov a0, a0, a2, a3
+; RV64I-CCMOV-NEXT:    ret
   %1 = and i1 %a, %b
   %2 = select i1 %1, i32 %c, i32 %d
   ret i32 %2
@@ -69,6 +77,23 @@ define signext i32 @if_of_and(i1 zeroext %a, i1 zeroext %b) nounwind {
 ; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
+;
+; RV64I-CCMOV-LABEL: if_of_and:
+; RV64I-CCMOV:       # %bb.0:
+; RV64I-CCMOV-NEXT:    addi sp, sp, -16
+; RV64I-CCMOV-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64I-CCMOV-NEXT:    beqz a0, .LBB1_3
+; RV64I-CCMOV-NEXT:  # %bb.1:
+; RV64I-CCMOV-NEXT:    beqz a1, .LBB1_3
+; RV64I-CCMOV-NEXT:  # %bb.2: # %if.then
+; RV64I-CCMOV-NEXT:    call both
+; RV64I-CCMOV-NEXT:    j .LBB1_4
+; RV64I-CCMOV-NEXT:  .LBB1_3: # %if.else
+; RV64I-CCMOV-NEXT:    call neither
+; RV64I-CCMOV-NEXT:  .LBB1_4: # %if.end
+; RV64I-CCMOV-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64I-CCMOV-NEXT:    addi sp, sp, 16
+; RV64I-CCMOV-NEXT:    ret
   %1 = and i1 %a, %b
   br i1 %1, label %if.then, label %if.else
 
diff --git a/llvm/test/CodeGen/RISCV/select-bare.ll b/llvm/test/CodeGen/RISCV/select-bare.ll
index cf8fe96742bfb..c9e108a1ca9d0 100644
--- a/llvm/test/CodeGen/RISCV/select-bare.ll
+++ b/llvm/test/CodeGen/RISCV/select-bare.ll
@@ -1,6 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mtriple=riscv32 -verify-machineinstrs < %s \
 ; RUN:   | FileCheck %s -check-prefix=RV32I
+; RUN: llc -mtriple=riscv64 -mattr=+xmipscmove -verify-machineinstrs < %s \
+; RUN:   | FileCheck -check-prefix=RV64I-CCMOV %s
 
 define i32 @bare_select(i1 %a, i32 %b, i32 %c) nounwind {
 ; RV32I-LABEL: bare_select:
@@ -12,6 +14,12 @@ define i32 @bare_select(i1 %a, i32 %b, i32 %c) nounwind {
 ; RV32I-NEXT:    mv a0, a2
 ; RV32I-NEXT:  .LBB0_2:
 ; RV32I-NEXT:    ret
+;
+; RV64I-CCMOV-LABEL: bare_select:
+; RV64I-CCMOV:       # %bb.0:
+; RV64I-CCMOV-NEXT:    andi a0, a0, 1
+; RV64I-CCMOV-NEXT:    mips.ccmov a0, a0, a1, a2
+; RV64I-CCMOV-NEXT:    ret
   %1 = select i1 %a, i32 %b, i32 %c
   ret i32 %1
 }
@@ -26,6 +34,12 @@ define float @bare_select_float(i1 %a, float %b, float %c) nounwind {
 ; RV32I-NEXT:    mv a0, a2
 ; RV32I-NEXT:  .LBB1_2:
 ; RV32I-NEXT:    ret
+;
+; RV64I-CCMOV-LABEL: bare_select_float:
+; RV64I-CCMOV:       # %bb.0:
+; RV64I-CCMOV-NEXT:    andi a0, a0, 1
+; RV64I-CCMOV-NEXT:    mips.ccmov a0, a0, a1, a2
+; RV64I-CCMOV-NEXT:    ret
   %1 = select i1 %a, float %b, float %c
   ret float %1
 }
diff --git a/llvm/test/CodeGen/RISCV/select-cc.ll b/llvm/test/CodeGen/RISCV/select-cc.ll
index 31e25702da8ba..1c2a0cf007d11 100644
--- a/llvm/test/CodeGen/RISCV/select-cc.ll
+++ b/llvm/test/CodeGen/RISCV/select-cc.ll
@@ -3,6 +3,8 @@
 ; RUN:   | FileCheck -check-prefixes=RV32I %s
 ; RUN: llc -mtriple=riscv64 -disable-block-placement -verify-machineinstrs < %s \
 ; RUN:   | FileCheck -check-prefixes=RV64I %s
+; RUN: llc -mtriple=riscv64 -mattr=+xmipscmove -verify-machineinstrs < %s \
+; RUN:   | FileCheck -check-prefix=RV64I-CCMOV %s
 
 define signext i32 @foo(i32 signext %a, ptr %b) nounwind {
 ; RV32I-LABEL: foo:
@@ -156,6 +158,57 @@ define signext i32 @foo(i32 signext %a, ptr %b) nounwind {
 ; RV64I-NEXT:    mv a0, a1
 ; RV64I-NEXT:  .LBB0_28:
 ; RV64I-NEXT:    ret
+;
+; RV64I-CCMOV-LABEL: foo:
+; RV64I-CCMOV:       # %bb.0:
+; RV64I-CCMOV-NEXT:    lw a2, 0(a1)
+; RV64I-CCMOV-NEXT:    lw a3, 0(a1)
+; RV64I-CCMOV-NEXT:    lw a4, 0(a1)
+; RV64I-CCMOV-NEXT:    lw a5, 0(a1)
+; RV64I-CCMOV-NEXT:    xor a6, a0, a2
+; RV64I-CCMOV-NEXT:    mips.ccmov a0, a6, a2, a0
+; RV64I-CCMOV-NEXT:    xor a2, a0, a3
+; RV64I-CCMOV-NEXT:    mips.ccmov a0, a2, a0, a3
+; RV64I-CCMOV-NEXT:    lw a2, 0(a1)
+; RV64I-CCMOV-NEXT:    sltu a3, a4, a0
+; RV64I-CCMOV-NEXT:    mips.ccmov a0, a3, a0, a4
+; RV64I-CCMOV-NEXT:    lw a3, 0(a1)
+; RV64I-CCMOV-NEXT:    sltu a4, a0, a5
+; RV64I-CCMOV-NEXT:    mips.ccmov a0, a4, a5, a0
+; RV64I-CCMOV-NEXT:    lw a4, 0(a1)
+; RV64I-CCMOV-NEXT:    sltu a5, a0, a2
+; RV64I-CCMOV-NEXT:    mips.ccmov a0, a5, a0, a2
+; RV64I-CCMOV-NEXT:    lw a2, 0(a1)
+; RV64I-CCMOV-NEXT:    sltu a5, a3, a0
+; RV64I-CCMOV-NEXT:    mips.ccmov a0, a5, a3, a0
+; RV64I-CCMOV-NEXT:    lw a3, 0(a1)
+; RV64I-CCMOV-NEXT:    sext.w a5, a0
+; RV64I-CCMOV-NEXT:    slt a5, a4, a5
+; RV64I-CCMOV-NEXT:    mips.ccmov a0, a5, a0, a4
+; RV64I-CCMOV-NEXT:    lw a4, 0(a1)
+; RV64I-CCMOV-NEXT:    sext.w a5, a0
+; RV64I-CCMOV-NEXT:    slt a5, a5, a2
+; RV64I-CCMOV-NEXT:    mips.ccmov a0, a5, a2, a0
+; RV64I-CCMOV-NEXT:    lw a2, 0(a1)
+; RV64I-CCMOV-NEXT:    sext.w a5, a0
+; RV64I-CCMOV-NEXT:    slt a5, a5, a3
+; RV64I-CCMOV-NEXT:    mips.ccmov a0, a5, a0, a3
+; RV64I-CCMOV-NEXT:    lw a3, 0(a1)
+; RV64I-CCMOV-NEXT:    sext.w a5, a0
+; RV64I-CCMOV-NEXT:    slt a5, a4, a5
+; RV64I-CCMOV-NEXT:    mips.ccmov a0, a5, a4, a0
+; RV64I-CCMOV-NEXT:    lw a4, 0(a1)
+; RV64I-CCMOV-NEXT:    slti a5, a2, 1
+; RV64I-CCMOV-NEXT:    mips.ccmov a0, a5, a0, a2
+; RV64I-CCMOV-NEXT:    slti a5, a2, 0
+; RV64I-CCMOV-NEXT:    mips.ccmov a0, a5, a3, a0
+; RV64I-CCMOV-NEXT:    lw a1, 0(a1)
+; RV64I-CCMOV-NEXT:    slti a3, a4, 1025
+; RV64I-CCMOV-NEXT:    mips.ccmov a0, a3, a4, a0
+; RV64I-CCMOV-NEXT:    sltiu a2, a2, 2047
+; RV64I-CCMOV-NEXT:    mips.ccmov a0, a2, a1, a0
+; RV64I-CCMOV-NEXT:    sext.w a0, a0
+; RV64I-CCMOV-NEXT:    ret
   %val1 = load volatile i32, ptr %b
   %tst1 = icmp eq i32 %a, %val1
   %val2 = select i1 %tst1, i32 %a, i32 %val1
@@ -258,6 +311,23 @@ define signext i16 @numsignbits(i16 signext %0, i16 signext %1, i16 signext %2,
 ; RV64I-NEXT:    ld s0, 0(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
+;
+; RV64I-CCMOV-LABEL: numsignbits:
+; RV64I-CCMOV:       # %bb.0:
+; RV64I-CCMOV-NEXT:    addi sp, sp, -16
+; RV64I-CCMOV-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64I-CCMOV-NEXT:    sd s0, 0(sp) # 8-byte Folded Spill
+; RV64I-CCMOV-NEXT:    mips.ccmov s0, a0, a2, a3
+; RV64I-CCMOV-NEXT:    beqz a1, .LBB1_2
+; RV64I-CCMOV-NEXT:  # %bb.1:
+; RV64I-CCMOV-NEXT:    mv a0, s0
+; RV64I-CCMOV-NEXT:    call bar
+; RV64I-CCMOV-NEXT:  .LBB1_2:
+; RV64I-CCMOV-NEXT:    mv a0, s0
+; RV64I-CCMOV-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64I-CCMOV-NEXT:    ld s0, 0(sp) # 8-byte Folded Reload
+; RV64I-CCMOV-NEXT:    addi sp, sp, 16
+; RV64I-CCMOV-NEXT:    ret
   %5 = icmp eq i16 %0, 0
   %6 = select i1 %5, i16 %3, i16 %2
   %7 = icmp eq i16 %1, 0
@@ -295,6 +365,14 @@ define i32 @select_sge_int16min(i32 signext %x, i32 signext %y, i32 signext %z)
 ; RV64I-NEXT:  .LBB2_2:
 ; RV64I-NEXT:    mv a0, a1
 ; RV64I-NEXT:    ret
+;
+; RV64I-CCMOV-LABEL: select_sge_int16min:
+; RV64I-CCMOV:       # %bb.0:
+; RV64I-CCMOV-NEXT:    lui a3, 1048560
+; RV64I-CCMOV-NEXT:    addiw a3, a3, -1
+; RV64I-CCMOV-NEXT:    slt a0, a3, a0
+; RV64I-CCMOV-NEXT:    mips.ccmov a0, a0, a1, a2
+; RV64I-CCMOV-NEXT:    ret
   %a = icmp sge i32 %x, -65536
   %b = select i1 %a, i32 %y, i32 %z
   ret i32 %b
@@ -331,6 +409,14 @@ define i64 @select_sge_int32min(i64 %x, i64 %y, i64 %z) {
 ; RV64I-NEXT:  .LBB3_2:
 ; RV64I-NEXT:    mv a0, a1
 ; RV64I-NEXT:    ret
+;
+; RV64I-CCMOV-LABEL: select_sge_int32min:
+; RV64I-CCMOV:       # %bb.0:
+; RV64I-CCMOV-NEXT:    lui a3, 524288
+; RV64I-CCMOV-NEXT:    addi a3, a3, -1
+; RV64I-CCMOV-NEXT:    slt a0, a3, a0
+; RV64I-CCMOV-NEXT:    mips.ccmov a0, a0, a1, a2
+; RV64I-CCMOV-NEXT:    ret
   %a = icmp sge i64 %x, -2147483648
   %b = select i1 %a, i64 %y, i64 %z
   ret i64 %b
diff --git a/llvm/test/CodeGen/RISCV/select-const.ll b/llvm/test/CodeGen/RISCV/select-const.ll
index 6a24d03de8749..90a81c535cef2 100644
--- a/llvm/test/CodeGen/RISCV/select-const.ll
+++ b/llvm/test/CodeGen/RISCV/select-const.ll
@@ -3,10 +3,14 @@
 ; RUN:   | FileCheck -check-prefixes=RV32,RV32I %s
 ; RUN: llc -mtriple=riscv32 -mattr=+f -target-abi=ilp32 -verify-machineinstrs < %s \
 ; RUN:   | FileCheck -check-prefixes=RV32,RV32IF %s
+; RUN: llc -mtriple=riscv32 -mattr=+zicond -target-abi=ilp32 -verify-machineinstrs < %s \
+; RUN:   | FileCheck -check-prefixes=RV32,RV32ZICOND %s
 ; RUN: llc -mtriple=riscv64 -target-abi=lp64 -verify-machineinstrs < %s \
 ; RUN:   | FileCheck -check-prefixes=RV64,RV64I %s
 ; RUN: llc -mtriple=riscv64 -mattr=+f,+d -target-abi=lp64 -verify-machineinstrs < %s \
 ; RUN:   | FileCheck -check-prefixes=RV64,RV64IFD %s
+; RUN: llc -mtriple=riscv64 -mattr=+zicond -target-abi=lp64 -verify-machineinstrs < %s \
+; RUN:   | FileCheck -check-prefixes=RV64,RV64ZICOND %s
 
 ;; This tests how good we are at materialising constants using `select`. The aim
 ;; is that we do so without a branch if possible (at the moment our lowering of
@@ -59,25 +63,59 @@ define signext i32 @select_const_int_pow2_zero(i1 zeroext %a) nounwind {
 }
 
 define signext i32 @select_const_int_harder(i1 zeroext %a) nounwind {
-; RV32-LABEL: select_const_int_harder:
-; RV32:       # %bb.0:
-; RV32-NEXT:    bnez a0, .LBB3_2
-; RV32-NEXT:  # %bb.1:
-; RV32-NEXT:    li a0, 38
-; RV32-NEXT:    ret
-; RV32-NEXT:  .LBB3_2:
-; RV32-NEXT:    li a0, 6
-; RV32-NEXT:    ret
+; RV32I-LABEL: select_const_int_harder:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    bnez a0, .LBB3_2
+; RV32I-NEXT:  # %bb.1:
+; RV32I-NEXT:    li a0, 38
+; RV32I-NEXT:    ret
+; RV32I-NEXT:  .LBB3_2:
+; RV32I-NEXT:    li a0, 6
+; RV32I-NEXT:    ret
 ;
-; RV64-LABEL: select_const_int_harder:
-; RV64:       # %bb.0:
-; RV64-NEXT:    bnez a0, .LBB3_2
-; RV64-NEXT:  # %bb.1:
-; RV64-NEXT:    li a0, 38
-; RV64-NEXT:    ret
-; RV64-NEXT:  .LBB3_2:
-; RV64-NEXT:    li a0, 6
-; RV64-NEXT:    ret
+; RV32IF-LABEL: select_const_int_harder:
+; RV32IF:       # %bb.0:
+; RV32IF-NEXT:    bnez a0, .LBB3_2
+; RV32IF-NEXT:  # %bb.1:
+; RV32IF-NEXT:    li a0, 38
+; RV32IF-NEXT:    ret
+; RV32IF-NEXT:  .LBB3_2:
+; RV32IF-NEXT:    li a0, 6
+; RV32IF-NEXT:    ret
+;
+; RV32ZICOND-LABEL: select_const_int_harder:
+; RV32ZICOND:       # %bb.0:
+; RV32ZICOND-NEXT:    li a1, 32
+; RV32ZICOND-NEXT:    czero.nez a0, a1, a0
+; RV32ZICOND-NEXT:    addi a0, a0, 6
+; RV32ZICOND-NEXT:    ret
+;
+; RV64I-LABEL: select_const_int_harder:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    bnez a0, .LBB3_2
+; RV64I-NEXT:  # %bb.1:
+; RV64I-NEXT:    li a0, 38
+; RV64I-NEXT:    ret
+; RV64I-NEXT:  .LBB3_2:
+; RV64I-NEXT:    li a0, 6
+; RV64I-NEXT:    ret
+;
+; RV64IFD-LABEL: select_const_int_harder:
+; RV64IFD:       # %bb.0:
+; RV64IFD-NEXT:    bnez a0, .LBB3_2
+; RV64IFD-NEXT:  # %bb.1:
+; RV64IFD-NEXT:    li a0, 38
+; RV64IFD-NEXT:    ret
+; RV64IFD-NEXT:  .LBB3_2:
+; RV64IFD-NEXT:    li a0, 6
+; RV64IFD-NEXT:    ret
+;
+; RV64ZICOND-LABEL: select_const_int_harder:
+; RV64ZICOND:       # %bb.0:
+; RV64ZICOND-NEXT:    li a1, 32
+; RV64ZICOND-NEXT:    czero.nez a0, a1, a0
+; RV64ZICOND-NEXT:    addi a0, a0, 6
+; RV64ZICOND-NEXT:    ret
   %1 = select i1 %a, i32 6, i32 38
   ret i32 %1
 }
@@ -106,6 +144,14 @@ define float @select_const_fp(i1 zeroext %a) nounwind {
 ; RV32IF-NEXT:    fmv.x.w a0, fa5
 ; RV32IF-NEXT:    ret
 ;
+; RV32ZICOND-LABEL: select_const_fp:
+; RV32ZICOND:       # %bb.0:
+; RV32ZICOND-NEXT:    lui a1, 1024
+; RV32ZICOND-NEXT:    czero.nez a0, a1, a0
+; RV32ZICOND-NEXT:    lui a1, 263168
+; RV32ZICOND-NEXT:    add a0, a0, a1
+; RV32ZICOND-NEXT:    ret
+;
 ; RV64I-LABEL: select_const_fp:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    mv a1, a0
@@ -128,6 +174,14 @@ define float @select_const_fp(i1 zeroext %a) nounwind {
 ; RV64IFD-NEXT:    fmv.w.x fa5, a0
 ; RV64IFD-NEXT:    fmv.x.w a0, fa5
 ; RV64IFD-NEXT:    ret
+;
+; RV64ZICOND-LABEL: select_const_fp:
+; RV64ZICOND:       # %bb.0:
+; RV64ZICOND-NEXT:    lui a1, 1024
+; RV64ZICOND-NEXT:    czero.nez a0, a1, a0
+; RV64ZICOND-NEXT:    lui a1, 263168
+; RV64ZICOND-NEXT:    add a0, a0, a1
+; RV64ZICOND-NEXT:    ret
   %1 = select i1 %a, float 3.0, float 4.0
   ret float %1
 }
diff --git a/llvm/test/CodeGen/RISCV/select-or.ll b/llvm/test/CodeGen/RISCV/select-or.ll
index 20a5ec15290cd..338c7c06c3ab8 100644
--- a/llvm/test/CodeGen/RISCV/select-or.ll
+++ b/llvm/test/CodeGen/RISCV/select-or.ll
@@ -3,6 +3,8 @@
 ; RUN:   | FileCheck -check-prefix=RV32I %s
 ; RUN: llc -mtriple=riscv64 -verify-machineinstrs < %s \
 ; RUN:   | FileCheck -check-prefix=RV64I %s
+; RUN: llc -mtriple=riscv64 -mattr=+xmipscmove -verify-machineinstrs < %s \
+; RUN:   | FileCheck -check-prefix=RV64I-CCMOV %s
 
 ;; There are a few different ways to lower (select (or A, B), X, Y). This test
 ;; ensures that we do so with as few branches as possible.
@@ -27,6 +29,12 @@ define signext i32 @select_of_or(i1 zeroext %a, i1 zeroext %b, i32 signext %c, i
 ; RV64I-NEXT:    mv a0, a3
 ; RV64I-NEXT:  .LBB0_2:
 ; RV64I-NEXT:    ret
+;
+; RV64I-CCMOV-LABEL: select_of_or:
+; RV64I-CCMOV:       # %bb.0:
+; RV64I-CCMOV-NEXT:    or a0, a0, a1
+; RV64I-CCMOV-NEXT:    mips.ccmov a0, a0, a2, a3
+; RV64I-CCMOV-NEXT:    ret
   %1 = or i1 %a, %b
   %2 = select i1 %1, i32 %c, i32 %d
   ret i32 %2
@@ -69,6 +77,23 @@ define signext i32 @if_of_or(i1 zeroext %a, i1 zeroext %b) nounwind {
 ; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
+;
+; RV64I-CCMOV-LABEL: if_of_or:
+; RV64I-CCMOV:       # %bb.0:
+; RV64I-CCMOV-NEXT:    addi sp, sp, -16
+; RV64I-CCMOV-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64I-CCMOV-NEXT:    bnez a0, .LBB1_3
+; RV64I-CCMOV-NEXT:  # %bb.1:
+; RV64I-CCMOV-NEXT:    bnez a1, .LBB1_3
+; RV64I-CCMOV-NEXT:  # %bb.2: # %if.else
+; RV64I-CCMOV-NEXT:    call neither
+; RV64I-CCMOV-NEXT:    j .LBB1_4
+; RV64I-CCMOV-NEXT:  .LBB1_3: # %if.then
+; RV64I-CCMOV-NEXT:    call either
+; RV64I-CCMOV-NEXT:  .LBB1_4: # %if.end
+; RV64I-CCMOV-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64I-CCMOV-NEXT:    addi sp, sp, 16
+; RV64I-CCMOV-NEXT:    ret
   %1 = or i1 %a, %b
   br i1 %1, label %if.then, label %if.else
 
diff --git a/llvm/test/CodeGen/RISCV/shrinkwrap-jump-table.ll b/llvm/test/CodeGen/RISCV/shrinkwrap-jump-table.ll
index 20a0d6d07b09c..3efd38c14878e 100644
--- a/llvm/test/CodeGen/RISCV/shrinkwrap-jump-table.ll
+++ b/llvm/test/CodeGen/RISCV/shrinkwrap-jump-table.ll
@@ -19,7 +19,7 @@ define dso_local signext i32 @test_shrinkwrap_jump_table(ptr noundef %m) local_u
 ; CHECK-NEXT:    slli a1, a1, 2
 ; CHECK-NEXT:    lui a2, %hi(.LJTI0_0)
 ; CHECK-NEXT:    addi a2, a2, %lo(.LJTI0_0)
-; CHECK-NEXT:    add a1, a1, a2
+; CHECK-NEXT:    add a1, a2, a1
 ; CHECK-NEXT:    lw a1, 0(a1)
 ; CHECK-NEXT:    jr a1
 ; CHECK-NEXT:  .LBB0_2: # %sw.bb
diff --git a/llvm/test/CodeGen/RISCV/stack-inst-compress.mir b/llvm/test/CodeGen/RISCV/stack-inst-compress.mir
index 2e6d888e65ba6..fe84d29963353 100644
--- a/llvm/test/CodeGen/RISCV/stack-inst-compress.mir
+++ b/llvm/test/CodeGen/RISCV/stack-inst-compress.mir
@@ -47,7 +47,7 @@ body:             |
     ; CHECK-RV32-NO-COM-NEXT: {{  $}}
     ; CHECK-RV32-NO-COM-NEXT: $x2 = frame-setup ADDI $x2, -2032
     ; CHECK-RV32-NO-COM-NEXT: frame-setup CFI_INSTRUCTION def_cfa_offset 2032
-    ; CHECK-RV32-NO-COM-NEXT: SW killed $x1, $x2, 2028 :: (store (s32) into %stack.1)
+    ; CHECK-RV32-NO-COM-NEXT: frame-setup SW killed $x1, $x2, 2028 :: (store (s32) into %stack.1)
     ; CHECK-RV32-NO-COM-NEXT: frame-setup CFI_INSTRUCTION offset $x1, -4
     ; CHECK-RV32-NO-COM-NEXT: $x2 = frame-setup ADDI $x2, -32
     ; CHECK-RV32-NO-COM-NEXT: frame-setup CFI_INSTRUCTION def_cfa_offset 2064
@@ -55,7 +55,7 @@ body:             |
     ; CHECK-RV32-NO-COM-NEXT: PseudoCALL target-flags(riscv-call) @_Z6calleePi, csr_ilp32_lp64, implicit-def dead $x1, implicit killed $x10, implicit-def $x2
     ; CHECK-RV32-NO-COM-NEXT: $x2 = frame-destroy ADDI $x2, 32
     ; CHECK-RV32-NO-COM-NEXT: frame-destroy CFI_INSTRUCTION def_cfa_offset 2032
-    ; CHECK-RV32-NO-COM-NEXT: $x1 = LW $x2, 2028 :: (load (s32) from %stack.1)
+    ; CHECK-RV32-NO-COM-NEXT: $x1 = frame-destroy LW $x2, 2028 :: (load (s32) from %stack.1)
     ; CHECK-RV32-NO-COM-NEXT: frame-destroy CFI_INSTRUCTION restore $x1
     ; CHECK-RV32-NO-COM-NEXT: $x2 = frame-destroy ADDI $x2, 2032
     ; CHECK-RV32-NO-COM-NEXT: frame-destroy CFI_INSTRUCTION def_cfa_offset 0
@@ -66,7 +66,7 @@ body:             |
     ; CHECK-RV32-COM-NEXT: {{  $}}
     ; CHECK-RV32-COM-NEXT: $x2 = frame-setup ADDI $x2, -256
     ; CHECK-RV32-COM-NEXT: frame-setup CFI_INSTRUCTION def_cfa_offset 256
-    ; CHECK-RV32-COM-NEXT: SW killed $x1, $x2, 252 :: (store (s32) into %stack.1)
+    ; CHECK-RV32-COM-NEXT: frame-setup SW killed $x1, $x2, 252 :: (store (s32) into %stack.1)
     ; CHECK-RV32-COM-NEXT: frame-setup CFI_INSTRUCTION offset $x1, -4
     ; CHECK-RV32-COM-NEXT: $x2 = frame-setup ADDI $x2, -1808
     ; CHECK-RV32-COM-NEXT: frame-setup CFI_INSTRUCTION def_cfa_offset 2064
@@ -74,7 +74,7 @@ body:             |
     ; CHECK-RV32-COM-NEXT: PseudoCALL target-flags(riscv-call) @_Z6calleePi, csr_ilp32_lp64, implicit-def dead $x1, implicit killed $x10, implicit-def $x2
     ; CHECK-RV32-COM-NEXT: $x2 = frame-destroy ADDI $x2, 1808
     ; CHECK-RV32-COM-NEXT: frame-destroy CFI_INSTRUCTION def_cfa_offset 256
-    ; CHECK-RV32-COM-NEXT: $x1 = LW $x2, 252 :: (load (s32) from %stack.1)
+    ; CHECK-RV32-COM-NEXT: $x1 = frame-destroy LW $x2, 252 :: (load (s32) from %stack.1)
     ; CHECK-RV32-COM-NEXT: frame-destroy CFI_INSTRUCTION restore $x1
     ; CHECK-RV32-COM-NEXT: $x2 = frame-destroy ADDI $x2, 256
     ; CHECK-RV32-COM-NEXT: frame-destroy CFI_INSTRUCTION def_cfa_offset 0
@@ -85,7 +85,7 @@ body:             |
     ; CHECK-RV64-NO-COM-NEXT: {{  $}}
     ; CHECK-RV64-NO-COM-NEXT: $x2 = frame-setup ADDI $x2, -2032
     ; CHECK-RV64-NO-COM-NEXT: frame-setup CFI_INSTRUCTION def_cfa_offset 2032
-    ; CHECK-RV64-NO-COM-NEXT: SD killed $x1, $x2, 2024 :: (store (s64) into %stack.1)
+    ; CHECK-RV64-NO-COM-NEXT: frame-setup SD killed $x1, $x2, 2024 :: (store (s64) into %stack.1)
     ; CHECK-RV64-NO-COM-NEXT: frame-setup CFI_INSTRUCTION offset $x1, -8
     ; CHECK-RV64-NO-COM-NEXT: $x2 = frame-setup ADDI $x2, -32
     ; CHECK-RV64-NO-COM-NEXT: frame-setup CFI_INSTRUCTION def_cfa_offset 2064
@@ -93,7 +93,7 @@ body:             |
     ; CHECK-RV64-NO-COM-NEXT: PseudoCALL target-flags(riscv-call) @_Z6calleePi, csr_ilp32_lp64, implicit-def dead $x1, implicit killed $x10, implicit-def $x2
     ; CHECK-RV64-NO-COM-NEXT: $x2 = frame-destroy ADDI $x2, 32
     ; CHECK-RV64-NO-COM-NEXT: frame-destroy CFI_INSTRUCTION def_cfa_offset 2032
-    ; CHECK-RV64-NO-COM-NEXT: $x1 = LD $x2, 2024 :: (load (s64) from %stack.1)
+    ; CHECK-RV64-NO-COM-NEXT: $x1 = frame-destroy LD $x2, 2024 :: (load (s64) from %stack.1)
     ; CHECK-RV64-NO-COM-NEXT: frame-destroy CFI_INSTRUCTION restore $x1
     ; CHECK-RV64-NO-COM-NEXT: $x2 = frame-destroy ADDI $x2, 2032
     ; CHECK-RV64-NO-COM-NEXT: frame-destroy CFI_INSTRUCTION def_cfa_offset 0
@@ -104,7 +104,7 @@ body:             |
     ; CHECK-RV64-COM-NEXT: {{  $}}
     ; CHECK-RV64-COM-NEXT: $x2 = frame-setup ADDI $x2, -496
     ; CHECK-RV64-COM-NEXT: frame-setup CFI_INSTRUCTION def_cfa_offset 496
-    ; CHECK-RV64-COM-NEXT: SD killed $x1, $x2, 488 :: (store (s64) into %stack.1)
+    ; CHECK-RV64-COM-NEXT: frame-setup SD killed $x1, $x2, 488 :: (store (s64) into %stack.1)
     ; CHECK-RV64-COM-NEXT: frame-setup CFI_INSTRUCTION offset $x1, -8
     ; CHECK-RV64-COM-NEXT: $x2 = frame-setup ADDI $x2, -1568
     ; CHECK-RV64-COM-NEXT: frame-setup CFI_INSTRUCTION def_cfa_offset 2064
@@ -112,7 +112,7 @@ body:             |
     ; CHECK-RV64-COM-NEXT: PseudoCALL target-flags(riscv-call) @_Z6calleePi, csr_ilp32_lp64, implicit-def dead $x1, implicit killed $x10, implicit-def $x2
     ; CHECK-RV64-COM-NEXT: $x2 = frame-destroy ADDI $x2, 1568
     ; CHECK-RV64-COM-NEXT: frame-destroy CFI_INSTRUCTION def_cfa_offset 496
-    ; CHECK-RV64-COM-NEXT: $x1 = LD $x2, 488 :: (load (s64) from %stack.1)
+    ; CHECK-RV64-COM-NEXT: $x1 = frame-destroy LD $x2, 488 :: (load (s64) from %stack.1)
     ; CHECK-RV64-COM-NEXT: frame-destroy CFI_INSTRUCTION restore $x1
     ; CHECK-RV64-COM-NEXT: $x2 = frame-destroy ADDI $x2, 496
     ; CHECK-RV64-COM-NEXT: frame-destroy CFI_INSTRUCTION def_cfa_offset 0
@@ -145,7 +145,7 @@ body:             |
     ; CHECK-RV32-NO-COM-NEXT: {{  $}}
     ; CHECK-RV32-NO-COM-NEXT: $x2 = frame-setup ADDI $x2, -2032
     ; CHECK-RV32-NO-COM-NEXT: frame-setup CFI_INSTRUCTION def_cfa_offset 2032
-    ; CHECK-RV32-NO-COM-NEXT: SW killed $x1, $x2, 2028 :: (store (s32) into %stack.1)
+    ; CHECK-RV32-NO-COM-NEXT: frame-setup SW killed $x1, $x2, 2028 :: (store (s32) into %stack.1)
     ; CHECK-RV32-NO-COM-NEXT: frame-setup CFI_INSTRUCTION offset $x1, -4
     ; CHECK-RV32-NO-COM-NEXT: $x2 = frame-setup ADDI $x2, -2048
     ; CHECK-RV32-NO-COM-NEXT: $x2 = frame-setup ADDI killed $x2, -32
@@ -155,7 +155,7 @@ body:             |
     ; CHECK-RV32-NO-COM-NEXT: $x2 = frame-destroy ADDI $x2, 2032
     ; CHECK-RV32-NO-COM-NEXT: $x2 = frame-destroy ADDI killed $x2, 48
     ; CHECK-RV32-NO-COM-NEXT: frame-destroy CFI_INSTRUCTION def_cfa_offset 2032
-    ; CHECK-RV32-NO-COM-NEXT: $x1 = LW $x2, 2028 :: (load (s32) from %stack.1)
+    ; CHECK-RV32-NO-COM-NEXT: $x1 = frame-destroy LW $x2, 2028 :: (load (s32) from %stack.1)
     ; CHECK-RV32-NO-COM-NEXT: frame-destroy CFI_INSTRUCTION restore $x1
     ; CHECK-RV32-NO-COM-NEXT: $x2 = frame-destroy ADDI $x2, 2032
     ; CHECK-RV32-NO-COM-NEXT: frame-destroy CFI_INSTRUCTION def_cfa_offset 0
@@ -166,7 +166,7 @@ body:             |
     ; CHECK-RV32-COM-NEXT: {{  $}}
     ; CHECK-RV32-COM-NEXT: $x2 = frame-setup ADDI $x2, -256
     ; CHECK-RV32-COM-NEXT: frame-setup CFI_INSTRUCTION def_cfa_offset 256
-    ; CHECK-RV32-COM-NEXT: SW killed $x1, $x2, 252 :: (store (s32) into %stack.1)
+    ; CHECK-RV32-COM-NEXT: frame-setup SW killed $x1, $x2, 252 :: (store (s32) into %stack.1)
     ; CHECK-RV32-COM-NEXT: frame-setup CFI_INSTRUCTION offset $x1, -4
     ; CHECK-RV32-COM-NEXT: $x2 = frame-setup ADDI $x2, -2048
     ; CHECK-RV32-COM-NEXT: $x2 = frame-setup ADDI killed $x2, -1808
@@ -176,7 +176,7 @@ body:             |
     ; CHECK-RV32-COM-NEXT: $x2 = frame-destroy ADDI $x2, 2032
     ; CHECK-RV32-COM-NEXT: $x2 = frame-destroy ADDI killed $x2, 1824
     ; CHECK-RV32-COM-NEXT: frame-destroy CFI_INSTRUCTION def_cfa_offset 256
-    ; CHECK-RV32-COM-NEXT: $x1 = LW $x2, 252 :: (load (s32) from %stack.1)
+    ; CHECK-RV32-COM-NEXT: $x1 = frame-destroy LW $x2, 252 :: (load (s32) from %stack.1)
     ; CHECK-RV32-COM-NEXT: frame-destroy CFI_INSTRUCTION restore $x1
     ; CHECK-RV32-COM-NEXT: $x2 = frame-destroy ADDI $x2, 256
     ; CHECK-RV32-COM-NEXT: frame-destroy CFI_INSTRUCTION def_cfa_offset 0
@@ -187,7 +187,7 @@ body:             |
     ; CHECK-RV64-NO-COM-NEXT: {{  $}}
     ; CHECK-RV64-NO-COM-NEXT: $x2 = frame-setup ADDI $x2, -2032
     ; CHECK-RV64-NO-COM-NEXT: frame-setup CFI_INSTRUCTION def_cfa_offset 2032
-    ; CHECK-RV64-NO-COM-NEXT: SD killed $x1, $x2, 2024 :: (store (s64) into %stack.1)
+    ; CHECK-RV64-NO-COM-NEXT: frame-setup SD killed $x1, $x2, 2024 :: (store (s64) into %stack.1)
     ; CHECK-RV64-NO-COM-NEXT: frame-setup CFI_INSTRUCTION offset $x1, -8
     ; CHECK-RV64-NO-COM-NEXT: $x2 = frame-setup ADDI $x2, -2048
     ; CHECK-RV64-NO-COM-NEXT: $x2 = frame-setup ADDI killed $x2, -32
@@ -197,7 +197,7 @@ body:             |
     ; CHECK-RV64-NO-COM-NEXT: $x2 = frame-destroy ADDI $x2, 2032
     ; CHECK-RV64-NO-COM-NEXT: $x2 = frame-destroy ADDI killed $x2, 48
     ; CHECK-RV64-NO-COM-NEXT: frame-destroy CFI_INSTRUCTION def_cfa_offset 2032
-    ; CHECK-RV64-NO-COM-NEXT: $x1 = LD $x2, 2024 :: (load (s64) from %stack.1)
+    ; CHECK-RV64-NO-COM-NEXT: $x1 = frame-destroy LD $x2, 2024 :: (load (s64) from %stack.1)
     ; CHECK-RV64-NO-COM-NEXT: frame-destroy CFI_INSTRUCTION restore $x1
     ; CHECK-RV64-NO-COM-NEXT: $x2 = frame-destroy ADDI $x2, 2032
     ; CHECK-RV64-NO-COM-NEXT: frame-destroy CFI_INSTRUCTION def_cfa_offset 0
@@ -208,7 +208,7 @@ body:             |
     ; CHECK-RV64-COM-NEXT: {{  $}}
     ; CHECK-RV64-COM-NEXT: $x2 = frame-setup ADDI $x2, -496
     ; CHECK-RV64-COM-NEXT: frame-setup CFI_INSTRUCTION def_cfa_offset 496
-    ; CHECK-RV64-COM-NEXT: SD killed $x1, $x2, 488 :: (store (s64) into %stack.1)
+    ; CHECK-RV64-COM-NEXT: frame-setup SD killed $x1, $x2, 488 :: (store (s64) into %stack.1)
     ; CHECK-RV64-COM-NEXT: frame-setup CFI_INSTRUCTION offset $x1, -8
     ; CHECK-RV64-COM-NEXT: $x2 = frame-setup ADDI $x2, -2048
     ; CHECK-RV64-COM-NEXT: $x2 = frame-setup ADDI killed $x2, -1568
@@ -218,7 +218,7 @@ body:             |
     ; CHECK-RV64-COM-NEXT: $x2 = frame-destroy ADDI $x2, 2032
     ; CHECK-RV64-COM-NEXT: $x2 = frame-destroy ADDI killed $x2, 1584
     ; CHECK-RV64-COM-NEXT: frame-destroy CFI_INSTRUCTION def_cfa_offset 496
-    ; CHECK-RV64-COM-NEXT: $x1 = LD $x2, 488 :: (load (s64) from %stack.1)
+    ; CHECK-RV64-COM-NEXT: $x1 = frame-destroy LD $x2, 488 :: (load (s64) from %stack.1)
     ; CHECK-RV64-COM-NEXT: frame-destroy CFI_INSTRUCTION restore $x1
     ; CHECK-RV64-COM-NEXT: $x2 = frame-destroy ADDI $x2, 496
     ; CHECK-RV64-COM-NEXT: frame-destroy CFI_INSTRUCTION def_cfa_offset 0
@@ -251,7 +251,7 @@ body:             |
     ; CHECK-RV32-NO-COM-NEXT: {{  $}}
     ; CHECK-RV32-NO-COM-NEXT: $x2 = frame-setup ADDI $x2, -2032
     ; CHECK-RV32-NO-COM-NEXT: frame-setup CFI_INSTRUCTION def_cfa_offset 2032
-    ; CHECK-RV32-NO-COM-NEXT: SW killed $x1, $x2, 2028 :: (store (s32) into %stack.1)
+    ; CHECK-RV32-NO-COM-NEXT: frame-setup SW killed $x1, $x2, 2028 :: (store (s32) into %stack.1)
     ; CHECK-RV32-NO-COM-NEXT: frame-setup CFI_INSTRUCTION offset $x1, -4
     ; CHECK-RV32-NO-COM-NEXT: $x10 = frame-setup LUI 2
     ; CHECK-RV32-NO-COM-NEXT: $x10 = frame-setup ADDI killed $x10, -2016
@@ -263,7 +263,7 @@ body:             |
     ; CHECK-RV32-NO-COM-NEXT: $x10 = frame-destroy ADDI killed $x10, -2016
     ; CHECK-RV32-NO-COM-NEXT: $x2 = frame-destroy ADD $x2, killed $x10
     ; CHECK-RV32-NO-COM-NEXT: frame-destroy CFI_INSTRUCTION def_cfa_offset 2032
-    ; CHECK-RV32-NO-COM-NEXT: $x1 = LW $x2, 2028 :: (load (s32) from %stack.1)
+    ; CHECK-RV32-NO-COM-NEXT: $x1 = frame-destroy LW $x2, 2028 :: (load (s32) from %stack.1)
     ; CHECK-RV32-NO-COM-NEXT: frame-destroy CFI_INSTRUCTION restore $x1
     ; CHECK-RV32-NO-COM-NEXT: $x2 = frame-destroy ADDI $x2, 2032
     ; CHECK-RV32-NO-COM-NEXT: frame-destroy CFI_INSTRUCTION def_cfa_offset 0
@@ -274,7 +274,7 @@ body:             |
     ; CHECK-RV32-COM-NEXT: {{  $}}
     ; CHECK-RV32-COM-NEXT: $x2 = frame-setup ADDI $x2, -256
     ; CHECK-RV32-COM-NEXT: frame-setup CFI_INSTRUCTION def_cfa_offset 256
-    ; CHECK-RV32-COM-NEXT: SW killed $x1, $x2, 252 :: (store (s32) into %stack.1)
+    ; CHECK-RV32-COM-NEXT: frame-setup SW killed $x1, $x2, 252 :: (store (s32) into %stack.1)
     ; CHECK-RV32-COM-NEXT: frame-setup CFI_INSTRUCTION offset $x1, -4
     ; CHECK-RV32-COM-NEXT: $x10 = frame-setup LUI 2
     ; CHECK-RV32-COM-NEXT: $x10 = frame-setup ADDI killed $x10, -240
@@ -286,7 +286,7 @@ body:             |
     ; CHECK-RV32-COM-NEXT: $x10 = frame-destroy ADDI killed $x10, -240
     ; CHECK-RV32-COM-NEXT: $x2 = frame-destroy ADD $x2, killed $x10
     ; CHECK-RV32-COM-NEXT: frame-destroy CFI_INSTRUCTION def_cfa_offset 256
-    ; CHECK-RV32-COM-NEXT: $x1 = LW $x2, 252 :: (load (s32) from %stack.1)
+    ; CHECK-RV32-COM-NEXT: $x1 = frame-destroy LW $x2, 252 :: (load (s32) from %stack.1)
     ; CHECK-RV32-COM-NEXT: frame-destroy CFI_INSTRUCTION restore $x1
     ; CHECK-RV32-COM-NEXT: $x2 = frame-destroy ADDI $x2, 256
     ; CHECK-RV32-COM-NEXT: frame-destroy CFI_INSTRUCTION def_cfa_offset 0
@@ -297,7 +297,7 @@ body:             |
     ; CHECK-RV64-NO-COM-NEXT: {{  $}}
     ; CHECK-RV64-NO-COM-NEXT: $x2 = frame-setup ADDI $x2, -2032
     ; CHECK-RV64-NO-COM-NEXT: frame-setup CFI_INSTRUCTION def_cfa_offset 2032
-    ; CHECK-RV64-NO-COM-NEXT: SD killed $x1, $x2, 2024 :: (store (s64) into %stack.1)
+    ; CHECK-RV64-NO-COM-NEXT: frame-setup SD killed $x1, $x2, 2024 :: (store (s64) into %stack.1)
     ; CHECK-RV64-NO-COM-NEXT: frame-setup CFI_INSTRUCTION offset $x1, -8
     ; CHECK-RV64-NO-COM-NEXT: $x10 = frame-setup LUI 2
     ; CHECK-RV64-NO-COM-NEXT: $x10 = frame-setup ADDIW killed $x10, -2016
@@ -309,7 +309,7 @@ body:             |
     ; CHECK-RV64-NO-COM-NEXT: $x10 = frame-destroy ADDIW killed $x10, -2016
     ; CHECK-RV64-NO-COM-NEXT: $x2 = frame-destroy ADD $x2, killed $x10
     ; CHECK-RV64-NO-COM-NEXT: frame-destroy CFI_INSTRUCTION def_cfa_offset 2032
-    ; CHECK-RV64-NO-COM-NEXT: $x1 = LD $x2, 2024 :: (load (s64) from %stack.1)
+    ; CHECK-RV64-NO-COM-NEXT: $x1 = frame-destroy LD $x2, 2024 :: (load (s64) from %stack.1)
     ; CHECK-RV64-NO-COM-NEXT: frame-destroy CFI_INSTRUCTION restore $x1
     ; CHECK-RV64-NO-COM-NEXT: $x2 = frame-destroy ADDI $x2, 2032
     ; CHECK-RV64-NO-COM-NEXT: frame-destroy CFI_INSTRUCTION def_cfa_offset 0
@@ -320,7 +320,7 @@ body:             |
     ; CHECK-RV64-COM-NEXT: {{  $}}
     ; CHECK-RV64-COM-NEXT: $x2 = frame-setup ADDI $x2, -496
     ; CHECK-RV64-COM-NEXT: frame-setup CFI_INSTRUCTION def_cfa_offset 496
-    ; CHECK-RV64-COM-NEXT: SD killed $x1, $x2, 488 :: (store (s64) into %stack.1)
+    ; CHECK-RV64-COM-NEXT: frame-setup SD killed $x1, $x2, 488 :: (store (s64) into %stack.1)
     ; CHECK-RV64-COM-NEXT: frame-setup CFI_INSTRUCTION offset $x1, -8
     ; CHECK-RV64-COM-NEXT: $x10 = frame-setup LUI 2
     ; CHECK-RV64-COM-NEXT: $x10 = frame-setup ADDIW killed $x10, -480
@@ -332,7 +332,7 @@ body:             |
     ; CHECK-RV64-COM-NEXT: $x10 = frame-destroy ADDIW killed $x10, -480
     ; CHECK-RV64-COM-NEXT: $x2 = frame-destroy ADD $x2, killed $x10
     ; CHECK-RV64-COM-NEXT: frame-destroy CFI_INSTRUCTION def_cfa_offset 496
-    ; CHECK-RV64-COM-NEXT: $x1 = LD $x2, 488 :: (load (s64) from %stack.1)
+    ; CHECK-RV64-COM-NEXT: $x1 = frame-destroy LD $x2, 488 :: (load (s64) from %stack.1)
     ; CHECK-RV64-COM-NEXT: frame-destroy CFI_INSTRUCTION restore $x1
     ; CHECK-RV64-COM-NEXT: $x2 = frame-destroy ADDI $x2, 496
     ; CHECK-RV64-COM-NEXT: frame-destroy CFI_INSTRUCTION def_cfa_offset 0
diff --git a/llvm/test/CodeGen/RISCV/zcmp-additional-stack.ll b/llvm/test/CodeGen/RISCV/zcmp-additional-stack.ll
index c98b9b80378fd..61c1de588a6e1 100644
--- a/llvm/test/CodeGen/RISCV/zcmp-additional-stack.ll
+++ b/llvm/test/CodeGen/RISCV/zcmp-additional-stack.ll
@@ -8,8 +8,8 @@ define ptr @func(ptr %s, i32 %_c, ptr %incdec.ptr, i1 %0, i8 %conv14) #0 {
 ; RV32-NEXT:    .cfi_offset ra, -12
 ; RV32-NEXT:    .cfi_offset s0, -8
 ; RV32-NEXT:    .cfi_offset s1, -4
-; RV32-NEXT:    addi sp, sp, -8
-; RV32-NEXT:    .cfi_def_cfa_offset 24
+; RV32-NEXT:    addi sp, sp, -4
+; RV32-NEXT:    .cfi_def_cfa_offset 20
 ; RV32-NEXT:    sw a4, 4(sp) # 4-byte Folded Spill
 ; RV32-NEXT:    sw a2, 0(sp) # 4-byte Folded Spill
 ; RV32-NEXT:    mv a2, a1
@@ -33,7 +33,7 @@ define ptr @func(ptr %s, i32 %_c, ptr %incdec.ptr, i1 %0, i8 %conv14) #0 {
 ; RV32-NEXT:    lw a0, 4(sp) # 4-byte Folded Reload
 ; RV32-NEXT:    sb a0, 0(s0)
 ; RV32-NEXT:    mv a0, s1
-; RV32-NEXT:    addi sp, sp, 8
+; RV32-NEXT:    addi sp, sp, 4
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
 ; RV32-NEXT:    cm.popret {ra, s0-s1}, 16
 entry:
diff --git a/llvm/test/CodeGen/RISCV/zcmp-cm-popretz.mir b/llvm/test/CodeGen/RISCV/zcmp-cm-popretz.mir
index e737ee0994968..ba2a333f6c9ba 100644
--- a/llvm/test/CodeGen/RISCV/zcmp-cm-popretz.mir
+++ b/llvm/test/CodeGen/RISCV/zcmp-cm-popretz.mir
@@ -65,14 +65,14 @@ body:                   |
     ; CHECK-NO-ZCMP32-NEXT: {{  $}}
     ; CHECK-NO-ZCMP32-NEXT: $x2 = frame-setup ADDI $x2, -16
     ; CHECK-NO-ZCMP32-NEXT: frame-setup CFI_INSTRUCTION def_cfa_offset 16
-    ; CHECK-NO-ZCMP32-NEXT: SW killed $x1, $x2, 12 :: (store (s32) into %stack.0)
-    ; CHECK-NO-ZCMP32-NEXT: SW killed $x8, $x2, 8 :: (store (s32) into %stack.1)
+    ; CHECK-NO-ZCMP32-NEXT: frame-setup SW killed $x1, $x2, 12 :: (store (s32) into %stack.0)
+    ; CHECK-NO-ZCMP32-NEXT: frame-setup SW killed $x8, $x2, 8 :: (store (s32) into %stack.1)
     ; CHECK-NO-ZCMP32-NEXT: frame-setup CFI_INSTRUCTION offset $x1, -4
     ; CHECK-NO-ZCMP32-NEXT: frame-setup CFI_INSTRUCTION offset $x8, -8
     ; CHECK-NO-ZCMP32-NEXT: $x1 = IMPLICIT_DEF
     ; CHECK-NO-ZCMP32-NEXT: $x8 = IMPLICIT_DEF
-    ; CHECK-NO-ZCMP32-NEXT: $x1 = LW $x2, 12 :: (load (s32) from %stack.0)
-    ; CHECK-NO-ZCMP32-NEXT: $x8 = LW $x2, 8 :: (load (s32) from %stack.1)
+    ; CHECK-NO-ZCMP32-NEXT: $x1 = frame-destroy LW $x2, 12 :: (load (s32) from %stack.0)
+    ; CHECK-NO-ZCMP32-NEXT: $x8 = frame-destroy LW $x2, 8 :: (load (s32) from %stack.1)
     ; CHECK-NO-ZCMP32-NEXT: frame-destroy CFI_INSTRUCTION restore $x1
     ; CHECK-NO-ZCMP32-NEXT: frame-destroy CFI_INSTRUCTION restore $x8
     ; CHECK-NO-ZCMP32-NEXT: $x2 = frame-destroy ADDI $x2, 16
@@ -84,14 +84,14 @@ body:                   |
     ; CHECK-NO-ZCMP64-NEXT: {{  $}}
     ; CHECK-NO-ZCMP64-NEXT: $x2 = frame-setup ADDI $x2, -16
     ; CHECK-NO-ZCMP64-NEXT: frame-setup CFI_INSTRUCTION def_cfa_offset 16
-    ; CHECK-NO-ZCMP64-NEXT: SD killed $x1, $x2, 8 :: (store (s64) into %stack.0)
-    ; CHECK-NO-ZCMP64-NEXT: SD killed $x8, $x2, 0 :: (store (s64) into %stack.1)
+    ; CHECK-NO-ZCMP64-NEXT: frame-setup SD killed $x1, $x2, 8 :: (store (s64) into %stack.0)
+    ; CHECK-NO-ZCMP64-NEXT: frame-setup SD killed $x8, $x2, 0 :: (store (s64) into %stack.1)
     ; CHECK-NO-ZCMP64-NEXT: frame-setup CFI_INSTRUCTION offset $x1, -8
     ; CHECK-NO-ZCMP64-NEXT: frame-setup CFI_INSTRUCTION offset $x8, -16
     ; CHECK-NO-ZCMP64-NEXT: $x1 = IMPLICIT_DEF
     ; CHECK-NO-ZCMP64-NEXT: $x8 = IMPLICIT_DEF
-    ; CHECK-NO-ZCMP64-NEXT: $x1 = LD $x2, 8 :: (load (s64) from %stack.0)
-    ; CHECK-NO-ZCMP64-NEXT: $x8 = LD $x2, 0 :: (load (s64) from %stack.1)
+    ; CHECK-NO-ZCMP64-NEXT: $x1 = frame-destroy LD $x2, 8 :: (load (s64) from %stack.0)
+    ; CHECK-NO-ZCMP64-NEXT: $x8 = frame-destroy LD $x2, 0 :: (load (s64) from %stack.1)
     ; CHECK-NO-ZCMP64-NEXT: frame-destroy CFI_INSTRUCTION restore $x1
     ; CHECK-NO-ZCMP64-NEXT: frame-destroy CFI_INSTRUCTION restore $x8
     ; CHECK-NO-ZCMP64-NEXT: $x2 = frame-destroy ADDI $x2, 16
@@ -157,15 +157,15 @@ body:                   |
     ; CHECK-NO-ZCMP32-NEXT: {{  $}}
     ; CHECK-NO-ZCMP32-NEXT: $x2 = frame-setup ADDI $x2, -16
     ; CHECK-NO-ZCMP32-NEXT: frame-setup CFI_INSTRUCTION def_cfa_offset 16
-    ; CHECK-NO-ZCMP32-NEXT: SW killed $x1, $x2, 12 :: (store (s32) into %stack.0)
-    ; CHECK-NO-ZCMP32-NEXT: SW killed $x8, $x2, 8 :: (store (s32) into %stack.1)
+    ; CHECK-NO-ZCMP32-NEXT: frame-setup SW killed $x1, $x2, 12 :: (store (s32) into %stack.0)
+    ; CHECK-NO-ZCMP32-NEXT: frame-setup SW killed $x8, $x2, 8 :: (store (s32) into %stack.1)
     ; CHECK-NO-ZCMP32-NEXT: frame-setup CFI_INSTRUCTION offset $x1, -4
     ; CHECK-NO-ZCMP32-NEXT: frame-setup CFI_INSTRUCTION offset $x8, -8
     ; CHECK-NO-ZCMP32-NEXT: $x1 = IMPLICIT_DEF
     ; CHECK-NO-ZCMP32-NEXT: $x8 = IMPLICIT_DEF
     ; CHECK-NO-ZCMP32-NEXT: $x10 = ADDI $x0, 0
-    ; CHECK-NO-ZCMP32-NEXT: $x1 = LW $x2, 12 :: (load (s32) from %stack.0)
-    ; CHECK-NO-ZCMP32-NEXT: $x8 = LW $x2, 8 :: (load (s32) from %stack.1)
+    ; CHECK-NO-ZCMP32-NEXT: $x1 = frame-destroy LW $x2, 12 :: (load (s32) from %stack.0)
+    ; CHECK-NO-ZCMP32-NEXT: $x8 = frame-destroy LW $x2, 8 :: (load (s32) from %stack.1)
     ; CHECK-NO-ZCMP32-NEXT: frame-destroy CFI_INSTRUCTION restore $x1
     ; CHECK-NO-ZCMP32-NEXT: frame-destroy CFI_INSTRUCTION restore $x8
     ; CHECK-NO-ZCMP32-NEXT: $x2 = frame-destroy ADDI $x2, 16
@@ -177,15 +177,15 @@ body:                   |
     ; CHECK-NO-ZCMP64-NEXT: {{  $}}
     ; CHECK-NO-ZCMP64-NEXT: $x2 = frame-setup ADDI $x2, -16
     ; CHECK-NO-ZCMP64-NEXT: frame-setup CFI_INSTRUCTION def_cfa_offset 16
-    ; CHECK-NO-ZCMP64-NEXT: SD killed $x1, $x2, 8 :: (store (s64) into %stack.0)
-    ; CHECK-NO-ZCMP64-NEXT: SD killed $x8, $x2, 0 :: (store (s64) into %stack.1)
+    ; CHECK-NO-ZCMP64-NEXT: frame-setup SD killed $x1, $x2, 8 :: (store (s64) into %stack.0)
+    ; CHECK-NO-ZCMP64-NEXT: frame-setup SD killed $x8, $x2, 0 :: (store (s64) into %stack.1)
     ; CHECK-NO-ZCMP64-NEXT: frame-setup CFI_INSTRUCTION offset $x1, -8
     ; CHECK-NO-ZCMP64-NEXT: frame-setup CFI_INSTRUCTION offset $x8, -16
     ; CHECK-NO-ZCMP64-NEXT: $x1 = IMPLICIT_DEF
     ; CHECK-NO-ZCMP64-NEXT: $x8 = IMPLICIT_DEF
     ; CHECK-NO-ZCMP64-NEXT: $x10 = ADDI $x0, 0
-    ; CHECK-NO-ZCMP64-NEXT: $x1 = LD $x2, 8 :: (load (s64) from %stack.0)
-    ; CHECK-NO-ZCMP64-NEXT: $x8 = LD $x2, 0 :: (load (s64) from %stack.1)
+    ; CHECK-NO-ZCMP64-NEXT: $x1 = frame-destroy LD $x2, 8 :: (load (s64) from %stack.0)
+    ; CHECK-NO-ZCMP64-NEXT: $x8 = frame-destroy LD $x2, 0 :: (load (s64) from %stack.1)
     ; CHECK-NO-ZCMP64-NEXT: frame-destroy CFI_INSTRUCTION restore $x1
     ; CHECK-NO-ZCMP64-NEXT: frame-destroy CFI_INSTRUCTION restore $x8
     ; CHECK-NO-ZCMP64-NEXT: $x2 = frame-destroy ADDI $x2, 16
diff --git a/llvm/test/CodeGen/RISCV/zcmp-cm-push-pop.mir b/llvm/test/CodeGen/RISCV/zcmp-cm-push-pop.mir
index 00cca9645ecb3..f78031e62f049 100644
--- a/llvm/test/CodeGen/RISCV/zcmp-cm-push-pop.mir
+++ b/llvm/test/CodeGen/RISCV/zcmp-cm-push-pop.mir
@@ -155,19 +155,19 @@ body:                   |
     ; CHECK-NO-ZCMP32-NEXT: {{  $}}
     ; CHECK-NO-ZCMP32-NEXT: $x2 = frame-setup ADDI $x2, -64
     ; CHECK-NO-ZCMP32-NEXT: frame-setup CFI_INSTRUCTION def_cfa_offset 64
-    ; CHECK-NO-ZCMP32-NEXT: SW killed $x1, $x2, 60 :: (store (s32) into %stack.0)
-    ; CHECK-NO-ZCMP32-NEXT: SW killed $x8, $x2, 56 :: (store (s32) into %stack.1)
-    ; CHECK-NO-ZCMP32-NEXT: SW killed $x9, $x2, 52 :: (store (s32) into %stack.2)
-    ; CHECK-NO-ZCMP32-NEXT: SW killed $x18, $x2, 48 :: (store (s32) into %stack.3)
-    ; CHECK-NO-ZCMP32-NEXT: SW killed $x19, $x2, 44 :: (store (s32) into %stack.4)
-    ; CHECK-NO-ZCMP32-NEXT: SW killed $x20, $x2, 40 :: (store (s32) into %stack.5)
-    ; CHECK-NO-ZCMP32-NEXT: SW killed $x21, $x2, 36 :: (store (s32) into %stack.6)
-    ; CHECK-NO-ZCMP32-NEXT: SW killed $x22, $x2, 32 :: (store (s32) into %stack.7)
-    ; CHECK-NO-ZCMP32-NEXT: SW killed $x23, $x2, 28 :: (store (s32) into %stack.8)
-    ; CHECK-NO-ZCMP32-NEXT: SW killed $x24, $x2, 24 :: (store (s32) into %stack.9)
-    ; CHECK-NO-ZCMP32-NEXT: SW killed $x25, $x2, 20 :: (store (s32) into %stack.10)
-    ; CHECK-NO-ZCMP32-NEXT: SW killed $x26, $x2, 16 :: (store (s32) into %stack.11)
-    ; CHECK-NO-ZCMP32-NEXT: SW killed $x27, $x2, 12 :: (store (s32) into %stack.12)
+    ; CHECK-NO-ZCMP32-NEXT: frame-setup SW killed $x1, $x2, 60 :: (store (s32) into %stack.0)
+    ; CHECK-NO-ZCMP32-NEXT: frame-setup SW killed $x8, $x2, 56 :: (store (s32) into %stack.1)
+    ; CHECK-NO-ZCMP32-NEXT: frame-setup SW killed $x9, $x2, 52 :: (store (s32) into %stack.2)
+    ; CHECK-NO-ZCMP32-NEXT: frame-setup SW killed $x18, $x2, 48 :: (store (s32) into %stack.3)
+    ; CHECK-NO-ZCMP32-NEXT: frame-setup SW killed $x19, $x2, 44 :: (store (s32) into %stack.4)
+    ; CHECK-NO-ZCMP32-NEXT: frame-setup SW killed $x20, $x2, 40 :: (store (s32) into %stack.5)
+    ; CHECK-NO-ZCMP32-NEXT: frame-setup SW killed $x21, $x2, 36 :: (store (s32) into %stack.6)
+    ; CHECK-NO-ZCMP32-NEXT: frame-setup SW killed $x22, $x2, 32 :: (store (s32) into %stack.7)
+    ; CHECK-NO-ZCMP32-NEXT: frame-setup SW killed $x23, $x2, 28 :: (store (s32) into %stack.8)
+    ; CHECK-NO-ZCMP32-NEXT: frame-setup SW killed $x24, $x2, 24 :: (store (s32) into %stack.9)
+    ; CHECK-NO-ZCMP32-NEXT: frame-setup SW killed $x25, $x2, 20 :: (store (s32) into %stack.10)
+    ; CHECK-NO-ZCMP32-NEXT: frame-setup SW killed $x26, $x2, 16 :: (store (s32) into %stack.11)
+    ; CHECK-NO-ZCMP32-NEXT: frame-setup SW killed $x27, $x2, 12 :: (store (s32) into %stack.12)
     ; CHECK-NO-ZCMP32-NEXT: frame-setup CFI_INSTRUCTION offset $x1, -4
     ; CHECK-NO-ZCMP32-NEXT: frame-setup CFI_INSTRUCTION offset $x8, -8
     ; CHECK-NO-ZCMP32-NEXT: frame-setup CFI_INSTRUCTION offset $x9, -12
@@ -194,19 +194,19 @@ body:                   |
     ; CHECK-NO-ZCMP32-NEXT: $x25 = IMPLICIT_DEF
     ; CHECK-NO-ZCMP32-NEXT: $x26 = IMPLICIT_DEF
     ; CHECK-NO-ZCMP32-NEXT: $x27 = IMPLICIT_DEF
-    ; CHECK-NO-ZCMP32-NEXT: $x1 = LW $x2, 60 :: (load (s32) from %stack.0)
-    ; CHECK-NO-ZCMP32-NEXT: $x8 = LW $x2, 56 :: (load (s32) from %stack.1)
-    ; CHECK-NO-ZCMP32-NEXT: $x9 = LW $x2, 52 :: (load (s32) from %stack.2)
-    ; CHECK-NO-ZCMP32-NEXT: $x18 = LW $x2, 48 :: (load (s32) from %stack.3)
-    ; CHECK-NO-ZCMP32-NEXT: $x19 = LW $x2, 44 :: (load (s32) from %stack.4)
-    ; CHECK-NO-ZCMP32-NEXT: $x20 = LW $x2, 40 :: (load (s32) from %stack.5)
-    ; CHECK-NO-ZCMP32-NEXT: $x21 = LW $x2, 36 :: (load (s32) from %stack.6)
-    ; CHECK-NO-ZCMP32-NEXT: $x22 = LW $x2, 32 :: (load (s32) from %stack.7)
-    ; CHECK-NO-ZCMP32-NEXT: $x23 = LW $x2, 28 :: (load (s32) from %stack.8)
-    ; CHECK-NO-ZCMP32-NEXT: $x24 = LW $x2, 24 :: (load (s32) from %stack.9)
-    ; CHECK-NO-ZCMP32-NEXT: $x25 = LW $x2, 20 :: (load (s32) from %stack.10)
-    ; CHECK-NO-ZCMP32-NEXT: $x26 = LW $x2, 16 :: (load (s32) from %stack.11)
-    ; CHECK-NO-ZCMP32-NEXT: $x27 = LW $x2, 12 :: (load (s32) from %stack.12)
+    ; CHECK-NO-ZCMP32-NEXT: $x1 = frame-destroy LW $x2, 60 :: (load (s32) from %stack.0)
+    ; CHECK-NO-ZCMP32-NEXT: $x8 = frame-destroy LW $x2, 56 :: (load (s32) from %stack.1)
+    ; CHECK-NO-ZCMP32-NEXT: $x9 = frame-destroy LW $x2, 52 :: (load (s32) from %stack.2)
+    ; CHECK-NO-ZCMP32-NEXT: $x18 = frame-destroy LW $x2, 48 :: (load (s32) from %stack.3)
+    ; CHECK-NO-ZCMP32-NEXT: $x19 = frame-destroy LW $x2, 44 :: (load (s32) from %stack.4)
+    ; CHECK-NO-ZCMP32-NEXT: $x20 = frame-destroy LW $x2, 40 :: (load (s32) from %stack.5)
+    ; CHECK-NO-ZCMP32-NEXT: $x21 = frame-destroy LW $x2, 36 :: (load (s32) from %stack.6)
+    ; CHECK-NO-ZCMP32-NEXT: $x22 = frame-destroy LW $x2, 32 :: (load (s32) from %stack.7)
+    ; CHECK-NO-ZCMP32-NEXT: $x23 = frame-destroy LW $x2, 28 :: (load (s32) from %stack.8)
+    ; CHECK-NO-ZCMP32-NEXT: $x24 = frame-destroy LW $x2, 24 :: (load (s32) from %stack.9)
+    ; CHECK-NO-ZCMP32-NEXT: $x25 = frame-destroy LW $x2, 20 :: (load (s32) from %stack.10)
+    ; CHECK-NO-ZCMP32-NEXT: $x26 = frame-destroy LW $x2, 16 :: (load (s32) from %stack.11)
+    ; CHECK-NO-ZCMP32-NEXT: $x27 = frame-destroy LW $x2, 12 :: (load (s32) from %stack.12)
     ; CHECK-NO-ZCMP32-NEXT: frame-destroy CFI_INSTRUCTION restore $x1
     ; CHECK-NO-ZCMP32-NEXT: frame-destroy CFI_INSTRUCTION restore $x8
     ; CHECK-NO-ZCMP32-NEXT: frame-destroy CFI_INSTRUCTION restore $x9
@@ -229,19 +229,19 @@ body:                   |
     ; CHECK-NO-ZCMP64-NEXT: {{  $}}
     ; CHECK-NO-ZCMP64-NEXT: $x2 = frame-setup ADDI $x2, -112
     ; CHECK-NO-ZCMP64-NEXT: frame-setup CFI_INSTRUCTION def_cfa_offset 112
-    ; CHECK-NO-ZCMP64-NEXT: SD killed $x1, $x2, 104 :: (store (s64) into %stack.0)
-    ; CHECK-NO-ZCMP64-NEXT: SD killed $x8, $x2, 96 :: (store (s64) into %stack.1)
-    ; CHECK-NO-ZCMP64-NEXT: SD killed $x9, $x2, 88 :: (store (s64) into %stack.2)
-    ; CHECK-NO-ZCMP64-NEXT: SD killed $x18, $x2, 80 :: (store (s64) into %stack.3)
-    ; CHECK-NO-ZCMP64-NEXT: SD killed $x19, $x2, 72 :: (store (s64) into %stack.4)
-    ; CHECK-NO-ZCMP64-NEXT: SD killed $x20, $x2, 64 :: (store (s64) into %stack.5)
-    ; CHECK-NO-ZCMP64-NEXT: SD killed $x21, $x2, 56 :: (store (s64) into %stack.6)
-    ; CHECK-NO-ZCMP64-NEXT: SD killed $x22, $x2, 48 :: (store (s64) into %stack.7)
-    ; CHECK-NO-ZCMP64-NEXT: SD killed $x23, $x2, 40 :: (store (s64) into %stack.8)
-    ; CHECK-NO-ZCMP64-NEXT: SD killed $x24, $x2, 32 :: (store (s64) into %stack.9)
-    ; CHECK-NO-ZCMP64-NEXT: SD killed $x25, $x2, 24 :: (store (s64) into %stack.10)
-    ; CHECK-NO-ZCMP64-NEXT: SD killed $x26, $x2, 16 :: (store (s64) into %stack.11)
-    ; CHECK-NO-ZCMP64-NEXT: SD killed $x27, $x2, 8 :: (store (s64) into %stack.12)
+    ; CHECK-NO-ZCMP64-NEXT: frame-setup SD killed $x1, $x2, 104 :: (store (s64) into %stack.0)
+    ; CHECK-NO-ZCMP64-NEXT: frame-setup SD killed $x8, $x2, 96 :: (store (s64) into %stack.1)
+    ; CHECK-NO-ZCMP64-NEXT: frame-setup SD killed $x9, $x2, 88 :: (store (s64) into %stack.2)
+    ; CHECK-NO-ZCMP64-NEXT: frame-setup SD killed $x18, $x2, 80 :: (store (s64) into %stack.3)
+    ; CHECK-NO-ZCMP64-NEXT: frame-setup SD killed $x19, $x2, 72 :: (store (s64) into %stack.4)
+    ; CHECK-NO-ZCMP64-NEXT: frame-setup SD killed $x20, $x2, 64 :: (store (s64) into %stack.5)
+    ; CHECK-NO-ZCMP64-NEXT: frame-setup SD killed $x21, $x2, 56 :: (store (s64) into %stack.6)
+    ; CHECK-NO-ZCMP64-NEXT: frame-setup SD killed $x22, $x2, 48 :: (store (s64) into %stack.7)
+    ; CHECK-NO-ZCMP64-NEXT: frame-setup SD killed $x23, $x2, 40 :: (store (s64) into %stack.8)
+    ; CHECK-NO-ZCMP64-NEXT: frame-setup SD killed $x24, $x2, 32 :: (store (s64) into %stack.9)
+    ; CHECK-NO-ZCMP64-NEXT: frame-setup SD killed $x25, $x2, 24 :: (store (s64) into %stack.10)
+    ; CHECK-NO-ZCMP64-NEXT: frame-setup SD killed $x26, $x2, 16 :: (store (s64) into %stack.11)
+    ; CHECK-NO-ZCMP64-NEXT: frame-setup SD killed $x27, $x2, 8 :: (store (s64) into %stack.12)
     ; CHECK-NO-ZCMP64-NEXT: frame-setup CFI_INSTRUCTION offset $x1, -8
     ; CHECK-NO-ZCMP64-NEXT: frame-setup CFI_INSTRUCTION offset $x8, -16
     ; CHECK-NO-ZCMP64-NEXT: frame-setup CFI_INSTRUCTION offset $x9, -24
@@ -268,19 +268,19 @@ body:                   |
     ; CHECK-NO-ZCMP64-NEXT: $x25 = IMPLICIT_DEF
     ; CHECK-NO-ZCMP64-NEXT: $x26 = IMPLICIT_DEF
     ; CHECK-NO-ZCMP64-NEXT: $x27 = IMPLICIT_DEF
-    ; CHECK-NO-ZCMP64-NEXT: $x1 = LD $x2, 104 :: (load (s64) from %stack.0)
-    ; CHECK-NO-ZCMP64-NEXT: $x8 = LD $x2, 96 :: (load (s64) from %stack.1)
-    ; CHECK-NO-ZCMP64-NEXT: $x9 = LD $x2, 88 :: (load (s64) from %stack.2)
-    ; CHECK-NO-ZCMP64-NEXT: $x18 = LD $x2, 80 :: (load (s64) from %stack.3)
-    ; CHECK-NO-ZCMP64-NEXT: $x19 = LD $x2, 72 :: (load (s64) from %stack.4)
-    ; CHECK-NO-ZCMP64-NEXT: $x20 = LD $x2, 64 :: (load (s64) from %stack.5)
-    ; CHECK-NO-ZCMP64-NEXT: $x21 = LD $x2, 56 :: (load (s64) from %stack.6)
-    ; CHECK-NO-ZCMP64-NEXT: $x22 = LD $x2, 48 :: (load (s64) from %stack.7)
-    ; CHECK-NO-ZCMP64-NEXT: $x23 = LD $x2, 40 :: (load (s64) from %stack.8)
-    ; CHECK-NO-ZCMP64-NEXT: $x24 = LD $x2, 32 :: (load (s64) from %stack.9)
-    ; CHECK-NO-ZCMP64-NEXT: $x25 = LD $x2, 24 :: (load (s64) from %stack.10)
-    ; CHECK-NO-ZCMP64-NEXT: $x26 = LD $x2, 16 :: (load (s64) from %stack.11)
-    ; CHECK-NO-ZCMP64-NEXT: $x27 = LD $x2, 8 :: (load (s64) from %stack.12)
+    ; CHECK-NO-ZCMP64-NEXT: $x1 = frame-destroy LD $x2, 104 :: (load (s64) from %stack.0)
+    ; CHECK-NO-ZCMP64-NEXT: $x8 = frame-destroy LD $x2, 96 :: (load (s64) from %stack.1)
+    ; CHECK-NO-ZCMP64-NEXT: $x9 = frame-destroy LD $x2, 88 :: (load (s64) from %stack.2)
+    ; CHECK-NO-ZCMP64-NEXT: $x18 = frame-destroy LD $x2, 80 :: (load (s64) from %stack.3)
+    ; CHECK-NO-ZCMP64-NEXT: $x19 = frame-destroy LD $x2, 72 :: (load (s64) from %stack.4)
+    ; CHECK-NO-ZCMP64-NEXT: $x20 = frame-destroy LD $x2, 64 :: (load (s64) from %stack.5)
+    ; CHECK-NO-ZCMP64-NEXT: $x21 = frame-destroy LD $x2, 56 :: (load (s64) from %stack.6)
+    ; CHECK-NO-ZCMP64-NEXT: $x22 = frame-destroy LD $x2, 48 :: (load (s64) from %stack.7)
+    ; CHECK-NO-ZCMP64-NEXT: $x23 = frame-destroy LD $x2, 40 :: (load (s64) from %stack.8)
+    ; CHECK-NO-ZCMP64-NEXT: $x24 = frame-destroy LD $x2, 32 :: (load (s64) from %stack.9)
+    ; CHECK-NO-ZCMP64-NEXT: $x25 = frame-destroy LD $x2, 24 :: (load (s64) from %stack.10)
+    ; CHECK-NO-ZCMP64-NEXT: $x26 = frame-destroy LD $x2, 16 :: (load (s64) from %stack.11)
+    ; CHECK-NO-ZCMP64-NEXT: $x27 = frame-destroy LD $x2, 8 :: (load (s64) from %stack.12)
     ; CHECK-NO-ZCMP64-NEXT: frame-destroy CFI_INSTRUCTION restore $x1
     ; CHECK-NO-ZCMP64-NEXT: frame-destroy CFI_INSTRUCTION restore $x8
     ; CHECK-NO-ZCMP64-NEXT: frame-destroy CFI_INSTRUCTION restore $x9
diff --git a/llvm/test/CodeGen/RISCV/zcmp-with-float.ll b/llvm/test/CodeGen/RISCV/zcmp-with-float.ll
index d2ecba2fe8d18..638a3af00eec8 100644
--- a/llvm/test/CodeGen/RISCV/zcmp-with-float.ll
+++ b/llvm/test/CodeGen/RISCV/zcmp-with-float.ll
@@ -8,31 +8,31 @@ declare void @callee()
 define float @foo(float %arg) {
 ; RV32-LABEL: foo:
 ; RV32:       # %bb.0: # %entry
-; RV32-NEXT:    cm.push {ra}, -32
-; RV32-NEXT:    .cfi_def_cfa_offset 32
+; RV32-NEXT:    cm.push {ra}, -16
+; RV32-NEXT:    .cfi_def_cfa_offset 16
 ; RV32-NEXT:    .cfi_offset ra, -4
-; RV32-NEXT:    fsw fs0, 12(sp) # 4-byte Folded Spill
-; RV32-NEXT:    .cfi_offset fs0, -20
+; RV32-NEXT:    fsw fs0, 8(sp) # 4-byte Folded Spill
+; RV32-NEXT:    .cfi_offset fs0, -8
 ; RV32-NEXT:    fmv.s fs0, fa0
 ; RV32-NEXT:    call callee
 ; RV32-NEXT:    fmv.s fa0, fs0
-; RV32-NEXT:    flw fs0, 12(sp) # 4-byte Folded Reload
+; RV32-NEXT:    flw fs0, 8(sp) # 4-byte Folded Reload
 ; RV32-NEXT:    .cfi_restore fs0
-; RV32-NEXT:    cm.popret {ra}, 32
+; RV32-NEXT:    cm.popret {ra}, 16
 ;
 ; RV64-LABEL: foo:
 ; RV64:       # %bb.0: # %entry
-; RV64-NEXT:    cm.push {ra}, -32
-; RV64-NEXT:    .cfi_def_cfa_offset 32
+; RV64-NEXT:    cm.push {ra}, -16
+; RV64-NEXT:    .cfi_def_cfa_offset 16
 ; RV64-NEXT:    .cfi_offset ra, -8
-; RV64-NEXT:    fsw fs0, 12(sp) # 4-byte Folded Spill
-; RV64-NEXT:    .cfi_offset fs0, -20
+; RV64-NEXT:    fsw fs0, 4(sp) # 4-byte Folded Spill
+; RV64-NEXT:    .cfi_offset fs0, -12
 ; RV64-NEXT:    fmv.s fs0, fa0
 ; RV64-NEXT:    call callee
 ; RV64-NEXT:    fmv.s fa0, fs0
-; RV64-NEXT:    flw fs0, 12(sp) # 4-byte Folded Reload
+; RV64-NEXT:    flw fs0, 4(sp) # 4-byte Folded Reload
 ; RV64-NEXT:    .cfi_restore fs0
-; RV64-NEXT:    cm.popret {ra}, 32
+; RV64-NEXT:    cm.popret {ra}, 16
 entry:
   call void @callee()
   ret float %arg
@@ -41,20 +41,20 @@ entry:
 define void @foo2(i32 %x, float %y) {
 ; RV32-LABEL: foo2:
 ; RV32:       # %bb.0: # %entry
-; RV32-NEXT:    cm.push {ra, s0}, -32
-; RV32-NEXT:    .cfi_def_cfa_offset 32
+; RV32-NEXT:    cm.push {ra, s0}, -16
+; RV32-NEXT:    .cfi_def_cfa_offset 16
 ; RV32-NEXT:    .cfi_offset ra, -8
 ; RV32-NEXT:    .cfi_offset s0, -4
-; RV32-NEXT:    fsw fs0, 12(sp) # 4-byte Folded Spill
-; RV32-NEXT:    .cfi_offset fs0, -20
+; RV32-NEXT:    fsw fs0, 4(sp) # 4-byte Folded Spill
+; RV32-NEXT:    .cfi_offset fs0, -12
 ; RV32-NEXT:    fmv.s fs0, fa0
 ; RV32-NEXT:    mv s0, a0
 ; RV32-NEXT:    call bar
 ; RV32-NEXT:    mv a0, s0
 ; RV32-NEXT:    fmv.s fa0, fs0
-; RV32-NEXT:    flw fs0, 12(sp) # 4-byte Folded Reload
+; RV32-NEXT:    flw fs0, 4(sp) # 4-byte Folded Reload
 ; RV32-NEXT:    .cfi_restore fs0
-; RV32-NEXT:    cm.pop {ra, s0}, 32
+; RV32-NEXT:    cm.pop {ra, s0}, 16
 ; RV32-NEXT:    .cfi_restore ra
 ; RV32-NEXT:    .cfi_restore s0
 ; RV32-NEXT:    .cfi_def_cfa_offset 0
diff --git a/llvm/test/CodeGen/RISCV/zdinx-memoperand.ll b/llvm/test/CodeGen/RISCV/zdinx-memoperand.ll
new file mode 100644
index 0000000000000..d618253912470
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/zdinx-memoperand.ll
@@ -0,0 +1,37 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc < %s -mtriple=riscv32 -mattr=+zdinx | FileCheck %s
+
+; This test previously asserted because TailMerge created a PseudoRV32ZdinxSD
+; with 2 memoperands which RISCVExpandPseudo could not handle.
+
+define i32 @foo(double %x, ptr %y, i64 %0, i64 %1, i1 %cmp6.not, ptr %arrayidx13, ptr %arrayidx20) {
+; CHECK-LABEL: foo:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    andi a0, a7, 1
+; CHECK-NEXT:    beqz a0, .LBB0_2
+; CHECK-NEXT:  # %bb.1: # %if.else
+; CHECK-NEXT:    lw a0, 4(sp)
+; CHECK-NEXT:    j .LBB0_3
+; CHECK-NEXT:  .LBB0_2: # %if.then7
+; CHECK-NEXT:    lw a0, 0(sp)
+; CHECK-NEXT:  .LBB0_3: # %common.ret
+; CHECK-NEXT:    fcvt.d.w a2, zero
+; CHECK-NEXT:    sw a2, 0(a0)
+; CHECK-NEXT:    sw a3, 4(a0)
+; CHECK-NEXT:    li a0, 0
+; CHECK-NEXT:    ret
+entry:
+  br i1 %cmp6.not, label %if.else, label %if.then7
+
+common.ret:                                       ; preds = %if.else, %if.then7
+  ret i32 0
+
+if.then7:                                         ; preds = %entry
+  store double 0.000000e+00, ptr %arrayidx13, align 8
+  br label %common.ret
+
+if.else:                                          ; preds = %entry
+  store double 0.000000e+00, ptr %arrayidx20, align 8
+  br label %common.ret
+}
+
diff --git a/llvm/test/CodeGen/SPARC/fmuladd-soft-float.ll b/llvm/test/CodeGen/SPARC/fmuladd-soft-float.ll
index a9e666e3c9b4d..b2ea38f294335 100644
--- a/llvm/test/CodeGen/SPARC/fmuladd-soft-float.ll
+++ b/llvm/test/CodeGen/SPARC/fmuladd-soft-float.ll
@@ -165,7 +165,7 @@ define <4 x float> @fmuladd_contract_v4f32(<4 x float> %a, <4 x float> %b, <4 x
 ; SOFT-FLOAT-32-NEXT:    mov %i0, %o0
 ; SOFT-FLOAT-32-NEXT:    call __mulsf3
 ; SOFT-FLOAT-32-NEXT:    mov %i4, %o1
-; SOFT-FLOAT-32-NEXT:    mov %o0, %l6
+; SOFT-FLOAT-32-NEXT:    mov %o0, %i0
 ; SOFT-FLOAT-32-NEXT:    mov %i1, %o0
 ; SOFT-FLOAT-32-NEXT:    call __mulsf3
 ; SOFT-FLOAT-32-NEXT:    mov %i5, %o1
@@ -173,26 +173,28 @@ define <4 x float> @fmuladd_contract_v4f32(<4 x float> %a, <4 x float> %b, <4 x
 ; SOFT-FLOAT-32-NEXT:    mov %i2, %o0
 ; SOFT-FLOAT-32-NEXT:    call __mulsf3
 ; SOFT-FLOAT-32-NEXT:    mov %l5, %o1
-; SOFT-FLOAT-32-NEXT:    mov %o0, %i4
+; SOFT-FLOAT-32-NEXT:    mov %o0, %i2
 ; SOFT-FLOAT-32-NEXT:    mov %i3, %o0
 ; SOFT-FLOAT-32-NEXT:    call __mulsf3
 ; SOFT-FLOAT-32-NEXT:    mov %l4, %o1
 ; SOFT-FLOAT-32-NEXT:    call __addsf3
 ; SOFT-FLOAT-32-NEXT:    mov %l3, %o1
 ; SOFT-FLOAT-32-NEXT:    mov %o0, %i3
-; SOFT-FLOAT-32-NEXT:    mov %i4, %o0
+; SOFT-FLOAT-32-NEXT:    mov %i2, %o0
 ; SOFT-FLOAT-32-NEXT:    call __addsf3
 ; SOFT-FLOAT-32-NEXT:    mov %l2, %o1
-; SOFT-FLOAT-32-NEXT:    mov %o0, %i2
+; SOFT-FLOAT-32-NEXT:    mov %o0, %i4
 ; SOFT-FLOAT-32-NEXT:    mov %i1, %o0
 ; SOFT-FLOAT-32-NEXT:    call __addsf3
 ; SOFT-FLOAT-32-NEXT:    mov %l1, %o1
 ; SOFT-FLOAT-32-NEXT:    mov %o0, %i1
-; SOFT-FLOAT-32-NEXT:    mov %l6, %o0
+; SOFT-FLOAT-32-NEXT:    mov %i0, %o0
 ; SOFT-FLOAT-32-NEXT:    call __addsf3
 ; SOFT-FLOAT-32-NEXT:    mov %l0, %o1
+; SOFT-FLOAT-32-NEXT:    ! kill: def $o0 killed $o0 def $o0_o1
+; SOFT-FLOAT-32-NEXT:    mov %o0, %i0
 ; SOFT-FLOAT-32-NEXT:    ret
-; SOFT-FLOAT-32-NEXT:    restore %g0, %o0, %o0
+; SOFT-FLOAT-32-NEXT:    restore %g0, %i4, %o2
 ;
 ; SOFT-FLOAT-64-LABEL: fmuladd_contract_v4f32:
 ; SOFT-FLOAT-64:         .cfi_startproc
diff --git a/llvm/test/CodeGen/SPARC/fp128.ll b/llvm/test/CodeGen/SPARC/fp128.ll
index 521e33399ac28..99bfb8d742711 100644
--- a/llvm/test/CodeGen/SPARC/fp128.ll
+++ b/llvm/test/CodeGen/SPARC/fp128.ll
@@ -54,11 +54,11 @@ entry:
 
 ; CHECK-LABEL: f128_spill_large:
 ; CHECK:       sethi 4, %g1
-; CHECK:       std %f{{.+}}, [%fp+-16]
-; CHECK-NEXT:  std %f{{.+}}, [%fp+-8]
-; CHECK:       ldd [%fp+-16], %f{{.+}}
-; CHECK-NEXT:  ldd [%fp+-8], %f{{.+}}
 
+; CHECK:       std %f{{.+}}, [%[[S0:.+]]]
+; CHECK:       std %f{{.+}}, [%[[S1:.+]]]
+; CHECK-DAG:   ldd [%[[S0]]], %f{{.+}}
+; CHECK-DAG:   ldd [%[[S1]]], %f{{.+}}
 define void @f128_spill_large(ptr noalias sret(<251 x fp128>) %scalar.result, ptr byval(<251 x fp128>) %a) {
 entry:
   %0 = load <251 x fp128>, ptr %a, align 8
@@ -102,10 +102,10 @@ entry:
 
 
 ; CHECK-LABEL: f128_abs:
-; CHECK:       ldd [%o0], %f0
-; CHECK:       ldd [%o0+8], %f2
-; BE:          fabss %f0, %f0
-; EL:          fabss %f3, %f3
+; CHECK-DAG:       ldd [%o0], [[REG:%f[0-9]+]]
+; CHECK-DAG:       ldd [%o0+8], %f{{[0-9]+}}
+; BE:          fabss [[REG]], [[REG]]
+; EL:          fabss %f1, %f1
 
 define void @f128_abs(ptr noalias sret(fp128) %scalar.result, ptr byval(fp128) %a) {
 entry:
@@ -229,10 +229,10 @@ entry:
 }
 
 ; CHECK-LABEL: f128_neg:
-; CHECK:       ldd [%o0], %f0
-; CHECK:       ldd [%o0+8], %f2
-; BE:          fnegs %f0, %f0
-; EL:          fnegs %f3, %f3
+; CHECK-DAG:       ldd [%o0], [[REG:%f[0-9]+]]
+; CHECK-DAG:       ldd [%o0+8], %f{{[0-9]+}}
+; BE:          fnegs [[REG]], [[REG]]
+; LE:          fnegs [[REG]], [[REG]]
 
 define void @f128_neg(ptr noalias sret(fp128) %scalar.result, ptr byval(fp128) %a) {
 entry:
diff --git a/llvm/test/CodeGen/SPARC/fp16-promote.ll b/llvm/test/CodeGen/SPARC/fp16-promote.ll
index f09c37b790844..a15104c7b8cff 100644
--- a/llvm/test/CodeGen/SPARC/fp16-promote.ll
+++ b/llvm/test/CodeGen/SPARC/fp16-promote.ll
@@ -89,10 +89,10 @@ define void @test_fpextend_fp128(ptr %p, ptr %out) nounwind {
 ; V8-OPT-NEXT:    call _Q_stoq
 ; V8-OPT-NEXT:    ld [%fp+-20], %o0
 ; V8-OPT-NEXT:    unimp 16
-; V8-OPT-NEXT:    ldd [%fp+-16], %f0
-; V8-OPT-NEXT:    ldd [%fp+-8], %f2
-; V8-OPT-NEXT:    std %f2, [%i1+8]
-; V8-OPT-NEXT:    std %f0, [%i1]
+; V8-OPT-NEXT:    ldd [%fp+-8], %f0
+; V8-OPT-NEXT:    ldd [%fp+-16], %f4
+; V8-OPT-NEXT:    std %f0, [%i1+8]
+; V8-OPT-NEXT:    std %f4, [%i1]
 ; V8-OPT-NEXT:    ret
 ; V8-OPT-NEXT:    restore
 ;
@@ -133,10 +133,10 @@ define void @test_fpextend_fp128(ptr %p, ptr %out) nounwind {
 ; V9-NEXT:    call _Q_stoq
 ; V9-NEXT:    ld [%fp+-20], %o0
 ; V9-NEXT:    unimp 16
-; V9-NEXT:    ldd [%fp+-16], %f0
-; V9-NEXT:    ldd [%fp+-8], %f2
-; V9-NEXT:    std %f2, [%i1+8]
-; V9-NEXT:    std %f0, [%i1]
+; V9-NEXT:    ldd [%fp+-8], %f0
+; V9-NEXT:    ldd [%fp+-16], %f4
+; V9-NEXT:    std %f0, [%i1+8]
+; V9-NEXT:    std %f4, [%i1]
 ; V9-NEXT:    ret
 ; V9-NEXT:    restore
 ;
@@ -149,10 +149,10 @@ define void @test_fpextend_fp128(ptr %p, ptr %out) nounwind {
 ; SPARC64-NEXT:    fmovs %f0, %f3
 ; SPARC64-NEXT:    call _Qp_stoq
 ; SPARC64-NEXT:    nop
-; SPARC64-NEXT:    ldd [%fp+2031], %f0
-; SPARC64-NEXT:    ldd [%fp+2039], %f2
-; SPARC64-NEXT:    std %f2, [%i1+8]
-; SPARC64-NEXT:    std %f0, [%i1]
+; SPARC64-NEXT:    ldd [%fp+2039], %f0
+; SPARC64-NEXT:    ldd [%fp+2031], %f4
+; SPARC64-NEXT:    std %f0, [%i1+8]
+; SPARC64-NEXT:    std %f4, [%i1]
 ; SPARC64-NEXT:    ret
 ; SPARC64-NEXT:    restore
   %a = load half, ptr %p
@@ -270,8 +270,8 @@ define void @test_fptrunc_fp128(ptr %dp, ptr %p) nounwind {
 ; V8-OPT:       ! %bb.0:
 ; V8-OPT-NEXT:    save %sp, -104, %sp
 ; V8-OPT-NEXT:    ldd [%i0], %f0
-; V8-OPT-NEXT:    ldd [%i0+8], %f2
-; V8-OPT-NEXT:    std %f2, [%sp+100]
+; V8-OPT-NEXT:    ldd [%i0+8], %f4
+; V8-OPT-NEXT:    std %f4, [%sp+100]
 ; V8-OPT-NEXT:    call __trunctfhf2
 ; V8-OPT-NEXT:    std %f0, [%sp+92]
 ; V8-OPT-NEXT:    sth %o0, [%i1]
@@ -302,8 +302,8 @@ define void @test_fptrunc_fp128(ptr %dp, ptr %p) nounwind {
 ; V9:       ! %bb.0:
 ; V9-NEXT:    save %sp, -104, %sp
 ; V9-NEXT:    ldd [%i0], %f0
-; V9-NEXT:    ldd [%i0+8], %f2
-; V9-NEXT:    std %f2, [%sp+100]
+; V9-NEXT:    ldd [%i0+8], %f4
+; V9-NEXT:    std %f4, [%sp+100]
 ; V9-NEXT:    call __trunctfhf2
 ; V9-NEXT:    std %f0, [%sp+92]
 ; V9-NEXT:    sth %o0, [%i1]
diff --git a/llvm/test/CodeGen/SPIRV/extensions/SPV_KHR_bit_instructions/cl_khr_extended_bit_ops.ll b/llvm/test/CodeGen/SPIRV/extensions/SPV_KHR_bit_instructions/cl_khr_extended_bit_ops.ll
new file mode 100644
index 0000000000000..14e5964a949da
--- /dev/null
+++ b/llvm/test/CodeGen/SPIRV/extensions/SPV_KHR_bit_instructions/cl_khr_extended_bit_ops.ll
@@ -0,0 +1,2764 @@
+; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv32-unknown-unknown %s --spirv-ext=+SPV_KHR_bit_instructions -o - | FileCheck %s --check-prefix=CHECK-EXTENSION
+; RUN: not llc -verify-machineinstrs -O0 -mtriple=spirv32-unknown-unknown %s -o %t.spvt 2>&1 | FileCheck %s --check-prefix=CHECK-NO-EXTENSION
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv64-unknown-unknown %s --spirv-ext=+SPV_KHR_bit_instructions -o - -filetype=obj | spirv-val %} 
+;
+; CHECK-EXTENSION: Capability BitInstructions
+; CHECK-EXTENSION: Extension "SPV_KHR_bit_instructions"
+; CHECK-NO-EXTENSION: LLVM ERROR: bitfield_insert: the builtin requires the following SPIR-V extension: SPV_KHR_bit_instructions
+;
+; CHECK-EXTENSION: %[[#]] = OpFunction %[[#]] None %[[#]]
+; CHECK-EXTENSION: %[[#insertbase_long:]] = OpFunctionParameter %[[#]]
+; CHECK-EXTENSION: %[[#insertinsert_long:]] = OpFunctionParameter %[[#]]
+; CHECK-EXTENSION: %[[#]] = OpBitFieldInsert %[[#]] %[[#insertbase_long]] %[[#insertinsert_long]]
+;
+; OpenCL equivalent.
+; kernel void testInsert_long(long b, long i, global long *res) {
+;   *res = bitfield_insert(b, i, 4, 2);
+; }
+
+define dso_local spir_kernel void @testInsert_long(i64 noundef %b, i64 noundef %i, ptr addrspace(1) nocapture noundef writeonly align 8 initializes((0, 8)) %res) {
+entry:
+  %call = tail call spir_func i64 @_Z15bitfield_insertlljj(i64 noundef %b, i64 noundef %i, i32 noundef 4, i32 noundef 2) #2
+  store i64 %call, ptr addrspace(1) %res, align 8, !tbaa !7
+  ret void
+}
+
+declare spir_func i64 @_Z15bitfield_insertlljj(i64 noundef, i64 noundef, i32 noundef, i32 noundef) 
+
+; CHECK-EXTENSION: %[[#]] = OpFunction %[[#]] None %[[#]]
+; CHECK-EXTENSION: %[[#insertbase_ulong:]] = OpFunctionParameter %[[#]]
+; CHECK-EXTENSION: %[[#insertinsert_ulong:]] = OpFunctionParameter %[[#]]
+; CHECK-EXTENSION: %[[#]] = OpBitFieldInsert %[[#]] %[[#insertbase_ulong]] %[[#insertinsert_ulong]]
+; OpenCL equivalent.
+; kernel void testInsert_ulong(ulong b, ulong i, global ulong *res) {
+;   *res = bitfield_insert(b, i, 4, 2);
+; }
+
+define dso_local spir_kernel void @testInsert_ulong(i64 noundef %b, i64 noundef %i, ptr addrspace(1) nocapture noundef writeonly align 8 initializes((0, 8)) %res) {
+entry:
+  %call = tail call spir_func i64 @_Z15bitfield_insertmmjj(i64 noundef %b, i64 noundef %i, i32 noundef 4, i32 noundef 2) #2
+  store i64 %call, ptr addrspace(1) %res, align 8, !tbaa !7
+  ret void
+}
+
+declare spir_func i64 @_Z15bitfield_insertmmjj(i64 noundef, i64 noundef, i32 noundef, i32 noundef) 
+
+; CHECK-EXTENSION: %[[#]] = OpFunction %[[#]] None %[[#]]
+; CHECK-EXTENSION: %[[#insertbase_int:]] = OpFunctionParameter %[[#]]
+; CHECK-EXTENSION: %[[#insertinsert_int:]] = OpFunctionParameter %[[#]]
+; CHECK-EXTENSION: %[[#]] = OpBitFieldInsert %[[#]] %[[#insertbase_int]] %[[#insertinsert_int]]
+; OpenCL equivalent.
+; kernel void testInsert_int(int b, int i, global int *res) {
+;   *res = bitfield_insert(b, i, 4, 2);
+; }
+
+define dso_local spir_kernel void @testInsert_int(i32 noundef %b, i32 noundef %i, ptr addrspace(1) nocapture noundef writeonly align 4 initializes((0, 4)) %res) {
+entry:
+  %call = tail call spir_func i32 @_Z15bitfield_insertiijj(i32 noundef %b, i32 noundef %i, i32 noundef 4, i32 noundef 2) #2
+  store i32 %call, ptr addrspace(1) %res, align 4, !tbaa !13
+  ret void
+}
+
+declare spir_func i32 @_Z15bitfield_insertiijj(i32 noundef, i32 noundef, i32 noundef, i32 noundef) 
+
+; CHECK-EXTENSION: %[[#]] = OpFunction %[[#]] None %[[#]]
+; CHECK-EXTENSION: %[[#insertbase_uint:]] = OpFunctionParameter %[[#]]
+; CHECK-EXTENSION: %[[#insertinsert_uint:]] = OpFunctionParameter %[[#]]
+; CHECK-EXTENSION: %[[#]] = OpBitFieldInsert %[[#]] %[[#insertbase_uint]] %[[#insertinsert_uint]]
+; OpenCL equivalent.
+; kernel void testInsert_uint(uint b, uint i, global uint *res) {
+;   *res = bitfield_insert(b, i, 4, 2);
+; }
+
+define dso_local spir_kernel void @testInsert_uint(i32 noundef %b, i32 noundef %i, ptr addrspace(1) nocapture noundef writeonly align 4 initializes((0, 4)) %res) {
+entry:
+  %call = tail call spir_func i32 @_Z15bitfield_insertjjjj(i32 noundef %b, i32 noundef %i, i32 noundef 4, i32 noundef 2) #2
+  store i32 %call, ptr addrspace(1) %res, align 4, !tbaa !13
+  ret void
+}
+
+declare spir_func i32 @_Z15bitfield_insertjjjj(i32 noundef, i32 noundef, i32 noundef, i32 noundef) 
+
+; CHECK-EXTENSION: %[[#]] = OpFunction %[[#]] None %[[#]]
+; CHECK-EXTENSION: %[[#insertbase_short:]] = OpFunctionParameter %[[#]]
+; CHECK-EXTENSION: %[[#insertinsert_short:]] = OpFunctionParameter %[[#]]
+; CHECK-EXTENSION: %[[#]] = OpBitFieldInsert %[[#]] %[[#insertbase_short]] %[[#insertinsert_short]]
+; OpenCL equivalent.
+; kernel void testInsert_short(short b, short i, global short *res) {
+;   *res = bitfield_insert(b, i, 4, 2);
+; }
+
+define dso_local spir_kernel void @testInsert_short(i16 noundef signext %b, i16 noundef signext %i, ptr addrspace(1) nocapture noundef writeonly align 2 initializes((0, 2)) %res) {
+entry:
+  %call = tail call spir_func signext i16 @_Z15bitfield_insertssjj(i16 noundef signext %b, i16 noundef signext %i, i32 noundef 4, i32 noundef 2) #2
+  store i16 %call, ptr addrspace(1) %res, align 2, !tbaa !17
+  ret void
+}
+
+declare spir_func signext i16 @_Z15bitfield_insertssjj(i16 noundef signext, i16 noundef signext, i32 noundef, i32 noundef) 
+
+; CHECK-EXTENSION: %[[#]] = OpFunction %[[#]] None %[[#]]
+; CHECK-EXTENSION: %[[#insertbase_ushort:]] = OpFunctionParameter %[[#]]
+; CHECK-EXTENSION: %[[#insertinsert_ushort:]] = OpFunctionParameter %[[#]]
+; CHECK-EXTENSION: %[[#]] = OpBitFieldInsert %[[#]] %[[#insertbase_ushort]] %[[#insertinsert_ushort]]
+; OpenCL equivalentxr.
+; kernel void testInsert_ushort(ushort b, ushort i, global ushort *res) {
+;   *res = bitfield_insert(b, i, 4, 2);
+; }
+
+define dso_local spir_kernel void @testInsert_ushort(i16 noundef zeroext %b, i16 noundef zeroext %i, ptr addrspace(1) nocapture noundef writeonly align 2 initializes((0, 2)) %res) {
+entry:
+  %call = tail call spir_func zeroext i16 @_Z15bitfield_insertttjj(i16 noundef zeroext %b, i16 noundef zeroext %i, i32 noundef 4, i32 noundef 2) #2
+  store i16 %call, ptr addrspace(1) %res, align 2, !tbaa !17
+  ret void
+}
+
+declare spir_func zeroext i16 @_Z15bitfield_insertttjj(i16 noundef zeroext, i16 noundef zeroext, i32 noundef, i32 noundef) 
+
+; CHECK-EXTENSION: %[[#]] = OpFunction %[[#]] None %[[#]]
+; CHECK-EXTENSION: %[[#insertbase_long2:]] = OpFunctionParameter %[[#]]
+; CHECK-EXTENSION: %[[#insertinsert_long2:]] = OpFunctionParameter %[[#]]
+; CHECK-EXTENSION: %[[#]] = OpBitFieldInsert %[[#]] %[[#insertbase_long2]] %[[#insertinsert_long2]]
+; OpenCL equivalent.
+; kernel void testInsert_long2(long2 b, long2 i, global long2 *res) {
+;   *res = bitfield_insert(b, i, 4, 2);
+; }
+
+define dso_local spir_kernel void @testInsert_long2(<2 x i64> noundef %b, <2 x i64> noundef %i, ptr addrspace(1) nocapture noundef writeonly align 16 initializes((0, 16)) %res) {
+entry:
+  %call = tail call spir_func <2 x i64> @_Z15bitfield_insertDv2_lS_jj(<2 x i64> noundef %b, <2 x i64> noundef %i, i32 noundef 4, i32 noundef 2) #2
+  store <2 x i64> %call, ptr addrspace(1) %res, align 16, !tbaa !22
+  ret void
+}
+
+declare spir_func <2 x i64> @_Z15bitfield_insertDv2_lS_jj(<2 x i64> noundef, <2 x i64> noundef, i32 noundef, i32 noundef) 
+
+; CHECK-EXTENSION: %[[#]] = OpFunction %[[#]] None %[[#]]
+; CHECK-EXTENSION: %[[#insertbase_ulong2:]] = OpFunctionParameter %[[#]]
+; CHECK-EXTENSION: %[[#insertinsert_ulong2:]] = OpFunctionParameter %[[#]]
+; CHECK-EXTENSION: %[[#]] = OpBitFieldInsert %[[#]] %[[#insertbase_ulong2]] %[[#insertinsert_ulong2]]
+; OpenCL equivalent.
+; kernel void testInsert_ulong2(ulong2 b, ulong2 i, global ulong2 *res) {
+;   *res = bitfield_insert(b, i, 4, 2);
+; }
+
+define dso_local spir_kernel void @testInsert_ulong2(<2 x i64> noundef %b, <2 x i64> noundef %i, ptr addrspace(1) nocapture noundef writeonly align 16 initializes((0, 16)) %res) {
+entry:
+  %call = tail call spir_func <2 x i64> @_Z15bitfield_insertDv2_mS_jj(<2 x i64> noundef %b, <2 x i64> noundef %i, i32 noundef 4, i32 noundef 2) #2
+  store <2 x i64> %call, ptr addrspace(1) %res, align 16, !tbaa !22
+  ret void
+}
+
+declare spir_func <2 x i64> @_Z15bitfield_insertDv2_mS_jj(<2 x i64> noundef, <2 x i64> noundef, i32 noundef, i32 noundef) 
+
+; CHECK-EXTENSION: %[[#]] = OpFunction %[[#]] None %[[#]]
+; CHECK-EXTENSION: %[[#insertbase_int2:]] = OpFunctionParameter %[[#]]
+; CHECK-EXTENSION: %[[#insertinsert_int2:]] = OpFunctionParameter %[[#]]
+; CHECK-EXTENSION: %[[#]] = OpBitFieldInsert %[[#]] %[[#insertbase_int2]] %[[#insertinsert_int2]]
+; OpenCL equivalent.
+; kernel void testInsert_int2(int2 b, int2 i, global int2 *res) {
+;   *res = bitfield_insert(b, i, 4, 2);
+; }
+
+define dso_local spir_kernel void @testInsert_int2(<2 x i32> noundef %b, <2 x i32> noundef %i, ptr addrspace(1) nocapture noundef writeonly align 8 initializes((0, 8)) %res) {
+entry:
+  %call = tail call spir_func <2 x i32> @_Z15bitfield_insertDv2_iS_jj(<2 x i32> noundef %b, <2 x i32> noundef %i, i32 noundef 4, i32 noundef 2) #2
+  store <2 x i32> %call, ptr addrspace(1) %res, align 8, !tbaa !22
+  ret void
+}
+
+declare spir_func <2 x i32> @_Z15bitfield_insertDv2_iS_jj(<2 x i32> noundef, <2 x i32> noundef, i32 noundef, i32 noundef) 
+
+; CHECK-EXTENSION: %[[#]] = OpFunction %[[#]] None %[[#]]
+; CHECK-EXTENSION: %[[#insertbase_uint2:]] = OpFunctionParameter %[[#]]
+; CHECK-EXTENSION: %[[#insertinsert_uint2:]] = OpFunctionParameter %[[#]]
+; CHECK-EXTENSION: %[[#]] = OpBitFieldInsert %[[#]] %[[#insertbase_uint2]] %[[#insertinsert_uint2]]
+; OpenCL equivalent.
+; kernel void testInsert_uint2(uint2 b, uint2 i, global uint2 *res) {
+;   *res = bitfield_insert(b, i, 4, 2);
+; }
+
+define dso_local spir_kernel void @testInsert_uint2(<2 x i32> noundef %b, <2 x i32> noundef %i, ptr addrspace(1) nocapture noundef writeonly align 8 initializes((0, 8)) %res) {
+entry:
+  %call = tail call spir_func <2 x i32> @_Z15bitfield_insertDv2_jS_jj(<2 x i32> noundef %b, <2 x i32> noundef %i, i32 noundef 4, i32 noundef 2) #2
+  store <2 x i32> %call, ptr addrspace(1) %res, align 8, !tbaa !22
+  ret void
+}
+
+declare spir_func <2 x i32> @_Z15bitfield_insertDv2_jS_jj(<2 x i32> noundef, <2 x i32> noundef, i32 noundef, i32 noundef) 
+
+; CHECK-EXTENSION: %[[#]] = OpFunction %[[#]] None %[[#]]
+; CHECK-EXTENSION: %[[#insertbase_short2:]] = OpFunctionParameter %[[#]]
+; CHECK-EXTENSION: %[[#insertinsert_short2:]] = OpFunctionParameter %[[#]]
+; CHECK-EXTENSION: %[[#]] = OpBitFieldInsert %[[#]] %[[#insertbase_short2]] %[[#insertinsert_short2]]
+; OpenCL equivalent.
+; kernel void testInsert_short2(short2 b, short2 i, global short2 *res) {
+;   *res = bitfield_insert(b, i, 4, 2);
+; }
+
+define dso_local spir_kernel void @testInsert_short2(<2 x i16> noundef %b, <2 x i16> noundef %i, ptr addrspace(1) nocapture noundef writeonly align 4 initializes((0, 4)) %res) {
+entry:
+  %call = tail call spir_func <2 x i16> @_Z15bitfield_insertDv2_sS_jj(<2 x i16> noundef %b, <2 x i16> noundef %i, i32 noundef 4, i32 noundef 2) #2
+  store <2 x i16> %call, ptr addrspace(1) %res, align 4, !tbaa !22
+  ret void
+}
+
+declare spir_func <2 x i16> @_Z15bitfield_insertDv2_sS_jj(<2 x i16> noundef, <2 x i16> noundef, i32 noundef, i32 noundef) 
+
+; CHECK-EXTENSION: %[[#]] = OpFunction %[[#]] None %[[#]]
+; CHECK-EXTENSION: %[[#insertbase_ushort2:]] = OpFunctionParameter %[[#]]
+; CHECK-EXTENSION: %[[#insertinsert_ushort2:]] = OpFunctionParameter %[[#]]
+; CHECK-EXTENSION: %[[#]] = OpBitFieldInsert %[[#]] %[[#insertbase_ushort2]] %[[#insertinsert_ushort2]]
+; OpenCL equivalent.
+; kernel void testInsert_ushort2(ushort2 b, ushort2 i, global ushort2 *res) {
+;   *res = bitfield_insert(b, i, 4, 2);
+; }
+
+define dso_local spir_kernel void @testInsert_ushort2(<2 x i16> noundef %b, <2 x i16> noundef %i, ptr addrspace(1) nocapture noundef writeonly align 4 initializes((0, 4)) %res) {
+entry:
+  %call = tail call spir_func <2 x i16> @_Z15bitfield_insertDv2_tS_jj(<2 x i16> noundef %b, <2 x i16> noundef %i, i32 noundef 4, i32 noundef 2) #2
+  store <2 x i16> %call, ptr addrspace(1) %res, align 4, !tbaa !22
+  ret void
+}
+
+declare spir_func <2 x i16> @_Z15bitfield_insertDv2_tS_jj(<2 x i16> noundef, <2 x i16> noundef, i32 noundef, i32 noundef) 
+
+; CHECK-EXTENSION: %[[#]] = OpFunction %[[#]] None %[[#]]
+; CHECK-EXTENSION: %[[#insertbase_char2:]] = OpFunctionParameter %[[#]]
+; CHECK-EXTENSION: %[[#insertinsert_char2:]] = OpFunctionParameter %[[#]]
+; CHECK-EXTENSION: %[[#]] = OpBitFieldInsert %[[#]] %[[#insertbase_char2]] %[[#insertinsert_char2]]
+; OpenCL equivalent.
+; kernel void testInsert_char2(char2 b, char2 i, global char2 *res) {
+;   *res = bitfield_insert(b, i, 4, 2);
+; }
+
+define dso_local spir_kernel void @testInsert_char2(<2 x i8> noundef %b, <2 x i8> noundef %i, ptr addrspace(1) nocapture noundef writeonly align 2 initializes((0, 2)) %res) {
+entry:
+  %call = tail call spir_func <2 x i8> @_Z15bitfield_insertDv2_cS_jj(<2 x i8> noundef %b, <2 x i8> noundef %i, i32 noundef 4, i32 noundef 2) #2
+  store <2 x i8> %call, ptr addrspace(1) %res, align 2, !tbaa !22
+  ret void
+}
+
+declare spir_func <2 x i8> @_Z15bitfield_insertDv2_cS_jj(<2 x i8> noundef, <2 x i8> noundef, i32 noundef, i32 noundef) 
+
+; CHECK-EXTENSION: %[[#]] = OpFunction %[[#]] None %[[#]]
+; CHECK-EXTENSION: %[[#insertbase_uchar2:]] = OpFunctionParameter %[[#]]
+; CHECK-EXTENSION: %[[#insertinsert_uchar2:]] = OpFunctionParameter %[[#]]
+; CHECK-EXTENSION: %[[#]] = OpBitFieldInsert %[[#]] %[[#insertbase_uchar2]] %[[#insertinsert_uchar2]]
+; OpenCL equivalent.
+; kernel void testInsert_uchar2(uchar2 b, uchar2 i, global uchar2 *res) {
+;   *res = bitfield_insert(b, i, 4, 2);
+; }
+
+define dso_local spir_kernel void @testInsert_uchar2(<2 x i8> noundef %b, <2 x i8> noundef %i, ptr addrspace(1) nocapture noundef writeonly align 2 initializes((0, 2)) %res) {
+entry:
+  %call = tail call spir_func <2 x i8> @_Z15bitfield_insertDv2_hS_jj(<2 x i8> noundef %b, <2 x i8> noundef %i, i32 noundef 4, i32 noundef 2) #2
+  store <2 x i8> %call, ptr addrspace(1) %res, align 2, !tbaa !22
+  ret void
+}
+
+declare spir_func <2 x i8> @_Z15bitfield_insertDv2_hS_jj(<2 x i8> noundef, <2 x i8> noundef, i32 noundef, i32 noundef) 
+
+; CHECK-EXTENSION: %[[#]] = OpFunction %[[#]] None %[[#]]
+; CHECK-EXTENSION: %[[#insertbase_long3:]] = OpFunctionParameter %[[#]]
+; CHECK-EXTENSION: %[[#insertinsert_long3:]] = OpFunctionParameter %[[#]]
+; CHECK-EXTENSION: %[[#]] = OpBitFieldInsert %[[#]] %[[#insertbase_long3]] %[[#insertinsert_long3]]
+; OpenCL equivalent.
+; kernel void testInsert_long3(long3 b, long3 i, global long3 *res) {
+;   *res = bitfield_insert(b, i, 4, 2);
+; }
+
+define dso_local spir_kernel void @testInsert_long3(<3 x i64> noundef %b, <3 x i64> noundef %i, ptr addrspace(1) nocapture noundef writeonly align 32 initializes((0, 32)) %res) {
+entry:
+  %call = tail call spir_func <3 x i64> @_Z15bitfield_insertDv3_lS_jj(<3 x i64> noundef %b, <3 x i64> noundef %i, i32 noundef 4, i32 noundef 2) #2
+  %extractVec5 = shufflevector <3 x i64> %call, <3 x i64> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 poison>
+  store <4 x i64> %extractVec5, ptr addrspace(1) %res, align 32, !tbaa !22
+  ret void
+}
+
+declare spir_func <3 x i64> @_Z15bitfield_insertDv3_lS_jj(<3 x i64> noundef, <3 x i64> noundef, i32 noundef, i32 noundef) 
+
+; CHECK-EXTENSION: %[[#]] = OpFunction %[[#]] None %[[#]]
+; CHECK-EXTENSION: %[[#insertbase_ulong3:]] = OpFunctionParameter %[[#]]
+; CHECK-EXTENSION: %[[#insertinsert_ulong3:]] = OpFunctionParameter %[[#]]
+; CHECK-EXTENSION: %[[#]] = OpBitFieldInsert %[[#]] %[[#insertbase_ulong3]] %[[#insertinsert_ulong3]]
+; OpenCL equivalent.
+; kernel void testInsert_ulong3(ulong3 b, ulong3 i, global ulong3 *res) {
+;   *res = bitfield_insert(b, i, 4, 2);
+; }
+
+define dso_local spir_kernel void @testInsert_ulong3(<3 x i64> noundef %b, <3 x i64> noundef %i, ptr addrspace(1) nocapture noundef writeonly align 32 initializes((0, 32)) %res) {
+entry:
+  %call = tail call spir_func <3 x i64> @_Z15bitfield_insertDv3_mS_jj(<3 x i64> noundef %b, <3 x i64> noundef %i, i32 noundef 4, i32 noundef 2) #2
+  %extractVec5 = shufflevector <3 x i64> %call, <3 x i64> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 poison>
+  store <4 x i64> %extractVec5, ptr addrspace(1) %res, align 32, !tbaa !22
+  ret void
+}
+
+declare spir_func <3 x i64> @_Z15bitfield_insertDv3_mS_jj(<3 x i64> noundef, <3 x i64> noundef, i32 noundef, i32 noundef) 
+
+; CHECK-EXTENSION: %[[#]] = OpFunction %[[#]] None %[[#]]
+; CHECK-EXTENSION: %[[#insertbase_int3:]] = OpFunctionParameter %[[#]]
+; CHECK-EXTENSION: %[[#insertinsert_int3:]] = OpFunctionParameter %[[#]]
+; CHECK-EXTENSION: %[[#]] = OpBitFieldInsert %[[#]] %[[#insertbase_int3]] %[[#insertinsert_int3]]
+; OpenCL equivalent.
+; kernel void testInsert_int3(int3 b, int3 i, global int3 *res) {
+;   *res = bitfield_insert(b, i, 4, 2);
+; }
+
+define dso_local spir_kernel void @testInsert_int3(<3 x i32> noundef %b, <3 x i32> noundef %i, ptr addrspace(1) nocapture noundef writeonly align 16 initializes((0, 16)) %res) {
+entry:
+  %call = tail call spir_func <3 x i32> @_Z15bitfield_insertDv3_iS_jj(<3 x i32> noundef %b, <3 x i32> noundef %i, i32 noundef 4, i32 noundef 2) #2
+  %extractVec5 = shufflevector <3 x i32> %call, <3 x i32> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 poison>
+  store <4 x i32> %extractVec5, ptr addrspace(1) %res, align 16, !tbaa !22
+  ret void
+}
+
+declare spir_func <3 x i32> @_Z15bitfield_insertDv3_iS_jj(<3 x i32> noundef, <3 x i32> noundef, i32 noundef, i32 noundef) 
+
+; CHECK-EXTENSION: %[[#]] = OpFunction %[[#]] None %[[#]]
+; CHECK-EXTENSION: %[[#insertbase_uint3:]] = OpFunctionParameter %[[#]]
+; CHECK-EXTENSION: %[[#insertinsert_uint3:]] = OpFunctionParameter %[[#]]
+; CHECK-EXTENSION: %[[#]] = OpBitFieldInsert %[[#]] %[[#insertbase_uint3]] %[[#insertinsert_uint3]]
+; OpenCL equivalent.
+; kernel void testInsert_uint3(uint3 b, uint3 i, global uint3 *res) {
+;   *res = bitfield_insert(b, i, 4, 2);
+; }
+
+define dso_local spir_kernel void @testInsert_uint3(<3 x i32> noundef %b, <3 x i32> noundef %i, ptr addrspace(1) nocapture noundef writeonly align 16 initializes((0, 16)) %res) {
+entry:
+  %call = tail call spir_func <3 x i32> @_Z15bitfield_insertDv3_jS_jj(<3 x i32> noundef %b, <3 x i32> noundef %i, i32 noundef 4, i32 noundef 2) #2
+  %extractVec5 = shufflevector <3 x i32> %call, <3 x i32> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 poison>
+  store <4 x i32> %extractVec5, ptr addrspace(1) %res, align 16, !tbaa !22
+  ret void
+}
+
+declare spir_func <3 x i32> @_Z15bitfield_insertDv3_jS_jj(<3 x i32> noundef, <3 x i32> noundef, i32 noundef, i32 noundef) 
+
+; CHECK-EXTENSION: %[[#]] = OpFunction %[[#]] None %[[#]]
+; CHECK-EXTENSION: %[[#insertbase_short3:]] = OpFunctionParameter %[[#]]
+; CHECK-EXTENSION: %[[#insertinsert_short3:]] = OpFunctionParameter %[[#]]
+; CHECK-EXTENSION: %[[#]] = OpBitFieldInsert %[[#]] %[[#insertbase_short3]] %[[#insertinsert_short3]]
+; OpenCL equivalent.
+; kernel void testInsert_short3(short3 b, short3 i, global short3 *res) {
+;   *res = bitfield_insert(b, i, 4, 2);
+; }
+
+define dso_local spir_kernel void @testInsert_short3(<3 x i16> noundef %b, <3 x i16> noundef %i, ptr addrspace(1) nocapture noundef writeonly align 8 initializes((0, 8)) %res) {
+entry:
+  %call = tail call spir_func <3 x i16> @_Z15bitfield_insertDv3_sS_jj(<3 x i16> noundef %b, <3 x i16> noundef %i, i32 noundef 4, i32 noundef 2) #2
+  %extractVec5 = shufflevector <3 x i16> %call, <3 x i16> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 poison>
+  store <4 x i16> %extractVec5, ptr addrspace(1) %res, align 8, !tbaa !22
+  ret void
+}
+
+declare spir_func <3 x i16> @_Z15bitfield_insertDv3_sS_jj(<3 x i16> noundef, <3 x i16> noundef, i32 noundef, i32 noundef) 
+
+; CHECK-EXTENSION: %[[#]] = OpFunction %[[#]] None %[[#]]
+; CHECK-EXTENSION: %[[#insertbase_ushort3:]] = OpFunctionParameter %[[#]]
+; CHECK-EXTENSION: %[[#insertinsert_ushort3:]] = OpFunctionParameter %[[#]]
+; CHECK-EXTENSION: %[[#]] = OpBitFieldInsert %[[#]] %[[#insertbase_ushort3]] %[[#insertinsert_ushort3]]
+; OpenCL equivalent.
+; kernel void testInsert_ushort3(ushort3 b, ushort3 i, global ushort3 *res) {
+;   *res = bitfield_insert(b, i, 4, 2);
+; }
+
+define dso_local spir_kernel void @testInsert_ushort3(<3 x i16> noundef %b, <3 x i16> noundef %i, ptr addrspace(1) nocapture noundef writeonly align 8 initializes((0, 8)) %res) {
+entry:
+  %call = tail call spir_func <3 x i16> @_Z15bitfield_insertDv3_tS_jj(<3 x i16> noundef %b, <3 x i16> noundef %i, i32 noundef 4, i32 noundef 2) #2
+  %extractVec5 = shufflevector <3 x i16> %call, <3 x i16> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 poison>
+  store <4 x i16> %extractVec5, ptr addrspace(1) %res, align 8, !tbaa !22
+  ret void
+}
+
+declare spir_func <3 x i16> @_Z15bitfield_insertDv3_tS_jj(<3 x i16> noundef, <3 x i16> noundef, i32 noundef, i32 noundef) 
+
+; CHECK-EXTENSION: %[[#]] = OpFunction %[[#]] None %[[#]]
+; CHECK-EXTENSION: %[[#insertbase_char3:]] = OpFunctionParameter %[[#]]
+; CHECK-EXTENSION: %[[#insertinsert_char3:]] = OpFunctionParameter %[[#]]
+; CHECK-EXTENSION: %[[#]] = OpBitFieldInsert %[[#]] %[[#insertbase_char3]] %[[#insertinsert_char3]]
+; OpenCL equivalent.
+; kernel void testInsert_char3(char3 b, char3 i, global char3 *res) {
+;   *res = bitfield_insert(b, i, 4, 2);
+; }
+
+define dso_local spir_kernel void @testInsert_char3(<3 x i8> noundef %b, <3 x i8> noundef %i, ptr addrspace(1) nocapture noundef writeonly align 4 initializes((0, 4)) %res) {
+entry:
+  %call = tail call spir_func <3 x i8> @_Z15bitfield_insertDv3_cS_jj(<3 x i8> noundef %b, <3 x i8> noundef %i, i32 noundef 4, i32 noundef 2) #2
+  %extractVec5 = shufflevector <3 x i8> %call, <3 x i8> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 poison>
+  store <4 x i8> %extractVec5, ptr addrspace(1) %res, align 4, !tbaa !22
+  ret void
+}
+
+declare spir_func <3 x i8> @_Z15bitfield_insertDv3_cS_jj(<3 x i8> noundef, <3 x i8> noundef, i32 noundef, i32 noundef) 
+
+; CHECK-EXTENSION: %[[#]] = OpFunction %[[#]] None %[[#]]
+; CHECK-EXTENSION: %[[#insertbase_uchar3:]] = OpFunctionParameter %[[#]]
+; CHECK-EXTENSION: %[[#insertinsert_uchar3:]] = OpFunctionParameter %[[#]]
+; CHECK-EXTENSION: %[[#]] = OpBitFieldInsert %[[#]] %[[#insertbase_uchar3]] %[[#insertinsert_uchar3]]
+; OpenCL equivalent.
+; kernel void testInsert_uchar3(uchar3 b, uchar3 i, global uchar3 *res) {
+;   *res = bitfield_insert(b, i, 4, 2);
+; }
+
+define dso_local spir_kernel void @testInsert_uchar3(<3 x i8> noundef %b, <3 x i8> noundef %i, ptr addrspace(1) nocapture noundef writeonly align 4 initializes((0, 4)) %res) {
+entry:
+  %call = tail call spir_func <3 x i8> @_Z15bitfield_insertDv3_hS_jj(<3 x i8> noundef %b, <3 x i8> noundef %i, i32 noundef 4, i32 noundef 2) #2
+  %extractVec5 = shufflevector <3 x i8> %call, <3 x i8> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 poison>
+  store <4 x i8> %extractVec5, ptr addrspace(1) %res, align 4, !tbaa !22
+  ret void
+}
+
+declare spir_func <3 x i8> @_Z15bitfield_insertDv3_hS_jj(<3 x i8> noundef, <3 x i8> noundef, i32 noundef, i32 noundef) 
+
+; CHECK-EXTENSION: %[[#]] = OpFunction %[[#]] None %[[#]]
+; CHECK-EXTENSION: %[[#insertbase_long4:]] = OpFunctionParameter %[[#]]
+; CHECK-EXTENSION: %[[#insertinsert_long4:]] = OpFunctionParameter %[[#]]
+; CHECK-EXTENSION: %[[#]] = OpBitFieldInsert %[[#]] %[[#insertbase_long4]] %[[#insertinsert_long4]]
+; OpenCL equivalent.
+; kernel void testInsert_long4(long4 b, long4 i, global long4 *res) {
+;   *res = bitfield_insert(b, i, 4, 2);
+; }
+
+define dso_local spir_kernel void @testInsert_long4(<4 x i64> noundef %b, <4 x i64> noundef %i, ptr addrspace(1) nocapture noundef writeonly align 32 initializes((0, 32)) %res) {
+entry:
+  %call = tail call spir_func <4 x i64> @_Z15bitfield_insertDv4_lS_jj(<4 x i64> noundef %b, <4 x i64> noundef %i, i32 noundef 4, i32 noundef 2) #2
+  store <4 x i64> %call, ptr addrspace(1) %res, align 32, !tbaa !22
+  ret void
+}
+
+declare spir_func <4 x i64> @_Z15bitfield_insertDv4_lS_jj(<4 x i64> noundef, <4 x i64> noundef, i32 noundef, i32 noundef) 
+
+; CHECK-EXTENSION: %[[#]] = OpFunction %[[#]] None %[[#]]
+; CHECK-EXTENSION: %[[#insertbase_ulong4:]] = OpFunctionParameter %[[#]]
+; CHECK-EXTENSION: %[[#insertinsert_ulong4:]] = OpFunctionParameter %[[#]]
+; CHECK-EXTENSION: %[[#]] = OpBitFieldInsert %[[#]] %[[#insertbase_ulong4]] %[[#insertinsert_ulong4]]
+; OpenCL equivalent.
+; kernel void testInsert_ulong4(ulong4 b, ulong4 i, global ulong4 *res) {
+;   *res = bitfield_insert(b, i, 4, 2);
+; }
+
+define dso_local spir_kernel void @testInsert_ulong4(<4 x i64> noundef %b, <4 x i64> noundef %i, ptr addrspace(1) nocapture noundef writeonly align 32 initializes((0, 32)) %res) {
+entry:
+  %call = tail call spir_func <4 x i64> @_Z15bitfield_insertDv4_mS_jj(<4 x i64> noundef %b, <4 x i64> noundef %i, i32 noundef 4, i32 noundef 2) #2
+  store <4 x i64> %call, ptr addrspace(1) %res, align 32, !tbaa !22
+  ret void
+}
+
+declare spir_func <4 x i64> @_Z15bitfield_insertDv4_mS_jj(<4 x i64> noundef, <4 x i64> noundef, i32 noundef, i32 noundef) 
+
+; CHECK-EXTENSION: %[[#]] = OpFunction %[[#]] None %[[#]]
+; CHECK-EXTENSION: %[[#insertbase_int4:]] = OpFunctionParameter %[[#]]
+; CHECK-EXTENSION: %[[#insertinsert_int4:]] = OpFunctionParameter %[[#]]
+; CHECK-EXTENSION: %[[#]] = OpBitFieldInsert %[[#]] %[[#insertbase_int4]] %[[#insertinsert_int4]]
+; OpenCL equivalent.
+; kernel void testInsert_int4(int4 b, int4 i, global int4 *res) {
+;   *res = bitfield_insert(b, i, 4, 2);
+; }
+
+define dso_local spir_kernel void @testInsert_int4(<4 x i32> noundef %b, <4 x i32> noundef %i, ptr addrspace(1) nocapture noundef writeonly align 16 initializes((0, 16)) %res) {
+entry:
+  %call = tail call spir_func <4 x i32> @_Z15bitfield_insertDv4_iS_jj(<4 x i32> noundef %b, <4 x i32> noundef %i, i32 noundef 4, i32 noundef 2) #2
+  store <4 x i32> %call, ptr addrspace(1) %res, align 16, !tbaa !22
+  ret void
+}
+
+declare spir_func <4 x i32> @_Z15bitfield_insertDv4_iS_jj(<4 x i32> noundef, <4 x i32> noundef, i32 noundef, i32 noundef) 
+
+; CHECK-EXTENSION: %[[#]] = OpFunction %[[#]] None %[[#]]
+; CHECK-EXTENSION: %[[#insertbase_uint4:]] = OpFunctionParameter %[[#]]
+; CHECK-EXTENSION: %[[#insertinsert_uint4:]] = OpFunctionParameter %[[#]]
+; CHECK-EXTENSION: %[[#]] = OpBitFieldInsert %[[#]] %[[#insertbase_uint4]] %[[#insertinsert_uint4]]
+; OpenCL equivalent.
+; kernel void testInsert_uint4(uint4 b, uint4 i, global uint4 *res) {
+;   *res = bitfield_insert(b, i, 4, 2);
+; }
+
+define dso_local spir_kernel void @testInsert_uint4(<4 x i32> noundef %b, <4 x i32> noundef %i, ptr addrspace(1) nocapture noundef writeonly align 16 initializes((0, 16)) %res) {
+entry:
+  %call = tail call spir_func <4 x i32> @_Z15bitfield_insertDv4_jS_jj(<4 x i32> noundef %b, <4 x i32> noundef %i, i32 noundef 4, i32 noundef 2) #2
+  store <4 x i32> %call, ptr addrspace(1) %res, align 16, !tbaa !22
+  ret void
+}
+
+declare spir_func <4 x i32> @_Z15bitfield_insertDv4_jS_jj(<4 x i32> noundef, <4 x i32> noundef, i32 noundef, i32 noundef) 
+
+; CHECK-EXTENSION: %[[#]] = OpFunction %[[#]] None %[[#]]
+; CHECK-EXTENSION: %[[#insertbase_short4:]] = OpFunctionParameter %[[#]]
+; CHECK-EXTENSION: %[[#insertinsert_short4:]] = OpFunctionParameter %[[#]]
+; CHECK-EXTENSION: %[[#]] = OpBitFieldInsert %[[#]] %[[#insertbase_short4]] %[[#insertinsert_short4]]
+; OpenCL equivalent.
+; kernel void testInsert_short4(short4 b, short4 i, global short4 *res) {
+;   *res = bitfield_insert(b, i, 4, 2);
+; }
+
+define dso_local spir_kernel void @testInsert_short4(<4 x i16> noundef %b, <4 x i16> noundef %i, ptr addrspace(1) nocapture noundef writeonly align 8 initializes((0, 8)) %res) {
+entry:
+  %call = tail call spir_func <4 x i16> @_Z15bitfield_insertDv4_sS_jj(<4 x i16> noundef %b, <4 x i16> noundef %i, i32 noundef 4, i32 noundef 2) #2
+  store <4 x i16> %call, ptr addrspace(1) %res, align 8, !tbaa !22
+  ret void
+}
+
+declare spir_func <4 x i16> @_Z15bitfield_insertDv4_sS_jj(<4 x i16> noundef, <4 x i16> noundef, i32 noundef, i32 noundef) 
+
+; CHECK-EXTENSION: %[[#]] = OpFunction %[[#]] None %[[#]]
+; CHECK-EXTENSION: %[[#insertbase_ushort4:]] = OpFunctionParameter %[[#]]
+; CHECK-EXTENSION: %[[#insertinsert_ushort4:]] = OpFunctionParameter %[[#]]
+; CHECK-EXTENSION: %[[#]] = OpBitFieldInsert %[[#]] %[[#insertbase_ushort4]] %[[#insertinsert_ushort4]]
+; OpenCL equivalent.
+; kernel void testInsert_ushort4(ushort4 b, ushort4 i, global ushort4 *res) {
+;   *res = bitfield_insert(b, i, 4, 2);
+; }
+
+define dso_local spir_kernel void @testInsert_ushort4(<4 x i16> noundef %b, <4 x i16> noundef %i, ptr addrspace(1) nocapture noundef writeonly align 8 initializes((0, 8)) %res) {
+entry:
+  %call = tail call spir_func <4 x i16> @_Z15bitfield_insertDv4_tS_jj(<4 x i16> noundef %b, <4 x i16> noundef %i, i32 noundef 4, i32 noundef 2) #2
+  store <4 x i16> %call, ptr addrspace(1) %res, align 8, !tbaa !22
+  ret void
+}
+
+declare spir_func <4 x i16> @_Z15bitfield_insertDv4_tS_jj(<4 x i16> noundef, <4 x i16> noundef, i32 noundef, i32 noundef) 
+
+; CHECK-EXTENSION: %[[#]] = OpFunction %[[#]] None %[[#]]
+; CHECK-EXTENSION: %[[#insertbase_char4:]] = OpFunctionParameter %[[#]]
+; CHECK-EXTENSION: %[[#insertinsert_char4:]] = OpFunctionParameter %[[#]]
+; CHECK-EXTENSION: %[[#]] = OpBitFieldInsert %[[#]] %[[#insertbase_char4]] %[[#insertinsert_char4]]
+; OpenCL equivalent.
+; kernel void testInsert_char4(char4 b, char4 i, global char4 *res) {
+;   *res = bitfield_insert(b, i, 4, 2);
+; }
+
+define dso_local spir_kernel void @testInsert_char4(<4 x i8> noundef %b, <4 x i8> noundef %i, ptr addrspace(1) nocapture noundef writeonly align 4 initializes((0, 4)) %res) {
+entry:
+  %call = tail call spir_func <4 x i8> @_Z15bitfield_insertDv4_cS_jj(<4 x i8> noundef %b, <4 x i8> noundef %i, i32 noundef 4, i32 noundef 2) #2
+  store <4 x i8> %call, ptr addrspace(1) %res, align 4, !tbaa !22
+  ret void
+}
+
+declare spir_func <4 x i8> @_Z15bitfield_insertDv4_cS_jj(<4 x i8> noundef, <4 x i8> noundef, i32 noundef, i32 noundef) 
+
+; CHECK-EXTENSION: %[[#]] = OpFunction %[[#]] None %[[#]]
+; CHECK-EXTENSION: %[[#insertbase_uchar4:]] = OpFunctionParameter %[[#]]
+; CHECK-EXTENSION: %[[#insertinsert_uchar4:]] = OpFunctionParameter %[[#]]
+; CHECK-EXTENSION: %[[#]] = OpBitFieldInsert %[[#]] %[[#insertbase_uchar4]] %[[#insertinsert_uchar4]]
+; OpenCL equivalent.
+; kernel void testInsert_uchar4(uchar4 b, uchar4 i, global uchar4 *res) {
+;   *res = bitfield_insert(b, i, 4, 2);
+; }
+
+define dso_local spir_kernel void @testInsert_uchar4(<4 x i8> noundef %b, <4 x i8> noundef %i, ptr addrspace(1) nocapture noundef writeonly align 4 initializes((0, 4)) %res) {
+entry:
+  %call = tail call spir_func <4 x i8> @_Z15bitfield_insertDv4_hS_jj(<4 x i8> noundef %b, <4 x i8> noundef %i, i32 noundef 4, i32 noundef 2) #2
+  store <4 x i8> %call, ptr addrspace(1) %res, align 4, !tbaa !22
+  ret void
+}
+
+declare spir_func <4 x i8> @_Z15bitfield_insertDv4_hS_jj(<4 x i8> noundef, <4 x i8> noundef, i32 noundef, i32 noundef) 
+
+; CHECK-EXTENSION: %[[#]] = OpFunction %[[#]] None %[[#]]
+; CHECK-EXTENSION: %[[#insertbase_long8:]] = OpFunctionParameter %[[#]]
+; CHECK-EXTENSION: %[[#insertinsert_long8:]] = OpFunctionParameter %[[#]]
+; CHECK-EXTENSION: %[[#]] = OpBitFieldInsert %[[#]] %[[#insertbase_long8]] %[[#insertinsert_long8]]
+; OpenCL equivalent.
+; kernel void testInsert_long8(long8 b, long8 i, global long8 *res) {
+;   *res = bitfield_insert(b, i, 4, 2);
+; }
+
+define dso_local spir_kernel void @testInsert_long8(<8 x i64> noundef %b, <8 x i64> noundef %i, ptr addrspace(1) nocapture noundef writeonly align 64 initializes((0, 64)) %res) {
+entry:
+  %call = tail call spir_func <8 x i64> @_Z15bitfield_insertDv8_lS_jj(<8 x i64> noundef %b, <8 x i64> noundef %i, i32 noundef 4, i32 noundef 2) #2
+  store <8 x i64> %call, ptr addrspace(1) %res, align 64, !tbaa !22
+  ret void
+}
+
+declare spir_func <8 x i64> @_Z15bitfield_insertDv8_lS_jj(<8 x i64> noundef, <8 x i64> noundef, i32 noundef, i32 noundef) 
+
+; CHECK-EXTENSION: %[[#]] = OpFunction %[[#]] None %[[#]]
+; CHECK-EXTENSION: %[[#insertbase_ulong8:]] = OpFunctionParameter %[[#]]
+; CHECK-EXTENSION: %[[#insertinsert_ulong8:]] = OpFunctionParameter %[[#]]
+; CHECK-EXTENSION: %[[#]] = OpBitFieldInsert %[[#]] %[[#insertbase_ulong8]] %[[#insertinsert_ulong8]]
+; OpenCL equivalent.
+; kernel void testInsert_ulong8(ulong8 b, ulong8 i, global ulong8 *res) {
+;   *res = bitfield_insert(b, i, 4, 2);
+; }
+
+define dso_local spir_kernel void @testInsert_ulong8(<8 x i64> noundef %b, <8 x i64> noundef %i, ptr addrspace(1) nocapture noundef writeonly align 64 initializes((0, 64)) %res) {
+entry:
+  %call = tail call spir_func <8 x i64> @_Z15bitfield_insertDv8_mS_jj(<8 x i64> noundef %b, <8 x i64> noundef %i, i32 noundef 4, i32 noundef 2) #2
+  store <8 x i64> %call, ptr addrspace(1) %res, align 64, !tbaa !22
+  ret void
+}
+
+declare spir_func <8 x i64> @_Z15bitfield_insertDv8_mS_jj(<8 x i64> noundef, <8 x i64> noundef, i32 noundef, i32 noundef) 
+
+; CHECK-EXTENSION: %[[#]] = OpFunction %[[#]] None %[[#]]
+; CHECK-EXTENSION: %[[#insertbase_int8:]] = OpFunctionParameter %[[#]]
+; CHECK-EXTENSION: %[[#insertinsert_int8:]] = OpFunctionParameter %[[#]]
+; CHECK-EXTENSION: %[[#]] = OpBitFieldInsert %[[#]] %[[#insertbase_int8]] %[[#insertinsert_int8]]
+; OpenCL equivalent.
+; kernel void testInsert_int8(int8 b, int8 i, global int8 *res) {
+;   *res = bitfield_insert(b, i, 4, 2);
+; }
+
+define dso_local spir_kernel void @testInsert_int8(<8 x i32> noundef %b, <8 x i32> noundef %i, ptr addrspace(1) nocapture noundef writeonly align 32 initializes((0, 32)) %res) {
+entry:
+  %call = tail call spir_func <8 x i32> @_Z15bitfield_insertDv8_iS_jj(<8 x i32> noundef %b, <8 x i32> noundef %i, i32 noundef 4, i32 noundef 2) #2
+  store <8 x i32> %call, ptr addrspace(1) %res, align 32, !tbaa !22
+  ret void
+}
+
+declare spir_func <8 x i32> @_Z15bitfield_insertDv8_iS_jj(<8 x i32> noundef, <8 x i32> noundef, i32 noundef, i32 noundef) 
+
+; CHECK-EXTENSION: %[[#]] = OpFunction %[[#]] None %[[#]]
+; CHECK-EXTENSION: %[[#insertbase_uint8:]] = OpFunctionParameter %[[#]]
+; CHECK-EXTENSION: %[[#insertinsert_uint8:]] = OpFunctionParameter %[[#]]
+; CHECK-EXTENSION: %[[#]] = OpBitFieldInsert %[[#]] %[[#insertbase_uint8]] %[[#insertinsert_uint8]]
+; OpenCL equivalent.
+; kernel void testInsert_uint8(uint8 b, uint8 i, global uint8 *res) {
+;   *res = bitfield_insert(b, i, 4, 2);
+; }
+
+define dso_local spir_kernel void @testInsert_uint8(<8 x i32> noundef %b, <8 x i32> noundef %i, ptr addrspace(1) nocapture noundef writeonly align 32 initializes((0, 32)) %res) {
+entry:
+  %call = tail call spir_func <8 x i32> @_Z15bitfield_insertDv8_jS_jj(<8 x i32> noundef %b, <8 x i32> noundef %i, i32 noundef 4, i32 noundef 2) #2
+  store <8 x i32> %call, ptr addrspace(1) %res, align 32, !tbaa !22
+  ret void
+}
+
+declare spir_func <8 x i32> @_Z15bitfield_insertDv8_jS_jj(<8 x i32> noundef, <8 x i32> noundef, i32 noundef, i32 noundef) 
+
+; CHECK-EXTENSION: %[[#]] = OpFunction %[[#]] None %[[#]]
+; CHECK-EXTENSION: %[[#insertbase_short8:]] = OpFunctionParameter %[[#]]
+; CHECK-EXTENSION: %[[#insertinsert_short8:]] = OpFunctionParameter %[[#]]
+; CHECK-EXTENSION: %[[#]] = OpBitFieldInsert %[[#]] %[[#insertbase_short8]] %[[#insertinsert_short8]]
+; OpenCL equivalent.
+; kernel void testInsert_short8(short8 b, short8 i, global short8 *res) {
+;   *res = bitfield_insert(b, i, 4, 2);
+; }
+
+define dso_local spir_kernel void @testInsert_short8(<8 x i16> noundef %b, <8 x i16> noundef %i, ptr addrspace(1) nocapture noundef writeonly align 16 initializes((0, 16)) %res) {
+entry:
+  %call = tail call spir_func <8 x i16> @_Z15bitfield_insertDv8_sS_jj(<8 x i16> noundef %b, <8 x i16> noundef %i, i32 noundef 4, i32 noundef 2) #2
+  store <8 x i16> %call, ptr addrspace(1) %res, align 16, !tbaa !22
+  ret void
+}
+
+declare spir_func <8 x i16> @_Z15bitfield_insertDv8_sS_jj(<8 x i16> noundef, <8 x i16> noundef, i32 noundef, i32 noundef) 
+
+; CHECK-EXTENSION: %[[#]] = OpFunction %[[#]] None %[[#]]
+; CHECK-EXTENSION: %[[#insertbase_ushort8:]] = OpFunctionParameter %[[#]]
+; CHECK-EXTENSION: %[[#insertinsert_ushort8:]] = OpFunctionParameter %[[#]]
+; CHECK-EXTENSION: %[[#]] = OpBitFieldInsert %[[#]] %[[#insertbase_ushort8]] %[[#insertinsert_ushort8]]
+; OpenCL equivalent.
+; kernel void testInsert_ushort8(ushort8 b, ushort8 i, global ushort8 *res) {
+;   *res = bitfield_insert(b, i, 4, 2);
+; }
+
+define dso_local spir_kernel void @testInsert_ushort8(<8 x i16> noundef %b, <8 x i16> noundef %i, ptr addrspace(1) nocapture noundef writeonly align 16 initializes((0, 16)) %res) {
+entry:
+  %call = tail call spir_func <8 x i16> @_Z15bitfield_insertDv8_tS_jj(<8 x i16> noundef %b, <8 x i16> noundef %i, i32 noundef 4, i32 noundef 2) #2
+  store <8 x i16> %call, ptr addrspace(1) %res, align 16, !tbaa !22
+  ret void
+}
+
+declare spir_func <8 x i16> @_Z15bitfield_insertDv8_tS_jj(<8 x i16> noundef, <8 x i16> noundef, i32 noundef, i32 noundef) 
+
+; CHECK-EXTENSION: %[[#]] = OpFunction %[[#]] None %[[#]]
+; CHECK-EXTENSION: %[[#insertbase_char8:]] = OpFunctionParameter %[[#]]
+; CHECK-EXTENSION: %[[#insertinsert_char8:]] = OpFunctionParameter %[[#]]
+; CHECK-EXTENSION: %[[#]] = OpBitFieldInsert %[[#]] %[[#insertbase_char8]] %[[#insertinsert_char8]]
+; OpenCL equivalent.
+; kernel void testInsert_char8(char8 b, char8 i, global char8 *res) {
+;   *res = bitfield_insert(b, i, 4, 2);
+; }
+
+define dso_local spir_kernel void @testInsert_char8(<8 x i8> noundef %b, <8 x i8> noundef %i, ptr addrspace(1) nocapture noundef writeonly align 8 initializes((0, 8)) %res) {
+entry:
+  %call = tail call spir_func <8 x i8> @_Z15bitfield_insertDv8_cS_jj(<8 x i8> noundef %b, <8 x i8> noundef %i, i32 noundef 4, i32 noundef 2) #2
+  store <8 x i8> %call, ptr addrspace(1) %res, align 8, !tbaa !22
+  ret void
+}
+
+declare spir_func <8 x i8> @_Z15bitfield_insertDv8_cS_jj(<8 x i8> noundef, <8 x i8> noundef, i32 noundef, i32 noundef) 
+
+; CHECK-EXTENSION: %[[#]] = OpFunction %[[#]] None %[[#]]
+; CHECK-EXTENSION: %[[#insertbase_uchar8:]] = OpFunctionParameter %[[#]]
+; CHECK-EXTENSION: %[[#insertinsert_uchar8:]] = OpFunctionParameter %[[#]]
+; CHECK-EXTENSION: %[[#]] = OpBitFieldInsert %[[#]] %[[#insertbase_uchar8]] %[[#insertinsert_uchar8]]
+; OpenCL equivalent.
+; kernel void testInsert_uchar8(uchar8 b, uchar8 i, global uchar8 *res) {
+;   *res = bitfield_insert(b, i, 4, 2);
+; }
+
+define dso_local spir_kernel void @testInsert_uchar8(<8 x i8> noundef %b, <8 x i8> noundef %i, ptr addrspace(1) nocapture noundef writeonly align 8 initializes((0, 8)) %res) {
+entry:
+  %call = tail call spir_func <8 x i8> @_Z15bitfield_insertDv8_hS_jj(<8 x i8> noundef %b, <8 x i8> noundef %i, i32 noundef 4, i32 noundef 2) #2
+  store <8 x i8> %call, ptr addrspace(1) %res, align 8, !tbaa !22
+  ret void
+}
+
+declare spir_func <8 x i8> @_Z15bitfield_insertDv8_hS_jj(<8 x i8> noundef, <8 x i8> noundef, i32 noundef, i32 noundef) 
+
+; CHECK-EXTENSION: %[[#]] = OpFunction %[[#]] None %[[#]]
+; CHECK-EXTENSION: %[[#insertbase_long16:]] = OpFunctionParameter %[[#]]
+; CHECK-EXTENSION: %[[#insertinsert_long16:]] = OpFunctionParameter %[[#]]
+; CHECK-EXTENSION: %[[#]] = OpBitFieldInsert %[[#]] %[[#insertbase_long16]] %[[#insertinsert_long16]]
+; OpenCL equivalent.
+; kernel void testInsert_long16(long16 b, long16 i, global long16 *res) {
+;   *res = bitfield_insert(b, i, 4, 2);
+; }
+
+define dso_local spir_kernel void @testInsert_long16(<16 x i64> noundef %b, <16 x i64> noundef %i, ptr addrspace(1) nocapture noundef writeonly align 128 initializes((0, 128)) %res) {
+entry:
+  %call = tail call spir_func <16 x i64> @_Z15bitfield_insertDv16_lS_jj(<16 x i64> noundef %b, <16 x i64> noundef %i, i32 noundef 4, i32 noundef 2) #2
+  store <16 x i64> %call, ptr addrspace(1) %res, align 128, !tbaa !22
+  ret void
+}
+
+declare spir_func <16 x i64> @_Z15bitfield_insertDv16_lS_jj(<16 x i64> noundef, <16 x i64> noundef, i32 noundef, i32 noundef) 
+
+; CHECK-EXTENSION: %[[#]] = OpFunction %[[#]] None %[[#]]
+; CHECK-EXTENSION: %[[#insertbase_ulong16:]] = OpFunctionParameter %[[#]]
+; CHECK-EXTENSION: %[[#insertinsert_ulong16:]] = OpFunctionParameter %[[#]]
+; CHECK-EXTENSION: %[[#]] = OpBitFieldInsert %[[#]] %[[#insertbase_ulong16]] %[[#insertinsert_ulong16]]
+; OpenCL equivalent.
+; kernel void testInsert_ulong16(ulong16 b, ulong16 i, global ulong16 *res) {
+;   *res = bitfield_insert(b, i, 4, 2);
+; }
+
+define dso_local spir_kernel void @testInsert_ulong16(<16 x i64> noundef %b, <16 x i64> noundef %i, ptr addrspace(1) nocapture noundef writeonly align 128 initializes((0, 128)) %res) {
+entry:
+  %call = tail call spir_func <16 x i64> @_Z15bitfield_insertDv16_mS_jj(<16 x i64> noundef %b, <16 x i64> noundef %i, i32 noundef 4, i32 noundef 2) #2
+  store <16 x i64> %call, ptr addrspace(1) %res, align 128, !tbaa !22
+  ret void
+}
+
+declare spir_func <16 x i64> @_Z15bitfield_insertDv16_mS_jj(<16 x i64> noundef, <16 x i64> noundef, i32 noundef, i32 noundef) 
+
+; CHECK-EXTENSION: %[[#]] = OpFunction %[[#]] None %[[#]]
+; CHECK-EXTENSION: %[[#insertbase_int16:]] = OpFunctionParameter %[[#]]
+; CHECK-EXTENSION: %[[#insertinsert_int16:]] = OpFunctionParameter %[[#]]
+; CHECK-EXTENSION: %[[#]] = OpBitFieldInsert %[[#]] %[[#insertbase_int16]] %[[#insertinsert_int16]]
+; OpenCL equivalent.
+; kernel void testInsert_int16(int16 b, int16 i, global int16 *res) {
+;   *res = bitfield_insert(b, i, 4, 2);
+; }
+
+define dso_local spir_kernel void @testInsert_int16(<16 x i32> noundef %b, <16 x i32> noundef %i, ptr addrspace(1) nocapture noundef writeonly align 64 initializes((0, 64)) %res) {
+entry:
+  %call = tail call spir_func <16 x i32> @_Z15bitfield_insertDv16_iS_jj(<16 x i32> noundef %b, <16 x i32> noundef %i, i32 noundef 4, i32 noundef 2) #2
+  store <16 x i32> %call, ptr addrspace(1) %res, align 64, !tbaa !22
+  ret void
+}
+
+declare spir_func <16 x i32> @_Z15bitfield_insertDv16_iS_jj(<16 x i32> noundef, <16 x i32> noundef, i32 noundef, i32 noundef) 
+
+; CHECK-EXTENSION: %[[#]] = OpFunction %[[#]] None %[[#]]
+; CHECK-EXTENSION: %[[#insertbase_uint16:]] = OpFunctionParameter %[[#]]
+; CHECK-EXTENSION: %[[#insertinsert_uint16:]] = OpFunctionParameter %[[#]]
+; CHECK-EXTENSION: %[[#]] = OpBitFieldInsert %[[#]] %[[#insertbase_uint16]] %[[#insertinsert_uint16]]
+; OpenCL equivalent.
+; kernel void testInsert_uint16(uint16 b, uint16 i, global uint16 *res) {
+;   *res = bitfield_insert(b, i, 4, 2);
+; }
+
+define dso_local spir_kernel void @testInsert_uint16(<16 x i32> noundef %b, <16 x i32> noundef %i, ptr addrspace(1) nocapture noundef writeonly align 64 initializes((0, 64)) %res) {
+entry:
+  %call = tail call spir_func <16 x i32> @_Z15bitfield_insertDv16_jS_jj(<16 x i32> noundef %b, <16 x i32> noundef %i, i32 noundef 4, i32 noundef 2) #2
+  store <16 x i32> %call, ptr addrspace(1) %res, align 64, !tbaa !22
+  ret void
+}
+
+declare spir_func <16 x i32> @_Z15bitfield_insertDv16_jS_jj(<16 x i32> noundef, <16 x i32> noundef, i32 noundef, i32 noundef) 
+
+; CHECK-EXTENSION: %[[#]] = OpFunction %[[#]] None %[[#]]
+; CHECK-EXTENSION: %[[#insertbase_short16:]] = OpFunctionParameter %[[#]]
+; CHECK-EXTENSION: %[[#insertinsert_short16:]] = OpFunctionParameter %[[#]]
+; CHECK-EXTENSION: %[[#]] = OpBitFieldInsert %[[#]] %[[#insertbase_short16]] %[[#insertinsert_short16]]
+; OpenCL equivalent.
+; kernel void testInsert_short16(short16 b, short16 i, global short16 *res) {
+;   *res = bitfield_insert(b, i, 4, 2);
+; }
+
+define dso_local spir_kernel void @testInsert_short16(<16 x i16> noundef %b, <16 x i16> noundef %i, ptr addrspace(1) nocapture noundef writeonly align 32 initializes((0, 32)) %res) {
+entry:
+  %call = tail call spir_func <16 x i16> @_Z15bitfield_insertDv16_sS_jj(<16 x i16> noundef %b, <16 x i16> noundef %i, i32 noundef 4, i32 noundef 2) #2
+  store <16 x i16> %call, ptr addrspace(1) %res, align 32, !tbaa !22
+  ret void
+}
+
+declare spir_func <16 x i16> @_Z15bitfield_insertDv16_sS_jj(<16 x i16> noundef, <16 x i16> noundef, i32 noundef, i32 noundef) 
+
+; CHECK-EXTENSION: %[[#]] = OpFunction %[[#]] None %[[#]]
+; CHECK-EXTENSION: %[[#insertbase_ushort16:]] = OpFunctionParameter %[[#]]
+; CHECK-EXTENSION: %[[#insertinsert_ushort16:]] = OpFunctionParameter %[[#]]
+; CHECK-EXTENSION: %[[#]] = OpBitFieldInsert %[[#]] %[[#insertbase_ushort16]] %[[#insertinsert_ushort16]]
+; OpenCL equivalent.
+; kernel void testInsert_ushort16(ushort16 b, ushort16 i, global ushort16 *res) {
+;   *res = bitfield_insert(b, i, 4, 2);
+; }
+
+define dso_local spir_kernel void @testInsert_ushort16(<16 x i16> noundef %b, <16 x i16> noundef %i, ptr addrspace(1) nocapture noundef writeonly align 32 initializes((0, 32)) %res) {
+entry:
+  %call = tail call spir_func <16 x i16> @_Z15bitfield_insertDv16_tS_jj(<16 x i16> noundef %b, <16 x i16> noundef %i, i32 noundef 4, i32 noundef 2) #2
+  store <16 x i16> %call, ptr addrspace(1) %res, align 32, !tbaa !22
+  ret void
+}
+
+declare spir_func <16 x i16> @_Z15bitfield_insertDv16_tS_jj(<16 x i16> noundef, <16 x i16> noundef, i32 noundef, i32 noundef) 
+
+; CHECK-EXTENSION: %[[#]] = OpFunction %[[#]] None %[[#]]
+; CHECK-EXTENSION: %[[#insertbase_char16:]] = OpFunctionParameter %[[#]]
+; CHECK-EXTENSION: %[[#insertinsert_char16:]] = OpFunctionParameter %[[#]]
+; CHECK-EXTENSION: %[[#]] = OpBitFieldInsert %[[#]] %[[#insertbase_char16]] %[[#insertinsert_char16]]
+; OpenCL equivalent.
+; kernel void testInsert_char16(char16 b, char16 i, global char16 *res) {
+;   *res = bitfield_insert(b, i, 4, 2);
+; }
+
+define dso_local spir_kernel void @testInsert_char16(<16 x i8> noundef %b, <16 x i8> noundef %i, ptr addrspace(1) nocapture noundef writeonly align 16 initializes((0, 16)) %res) {
+entry:
+  %call = tail call spir_func <16 x i8> @_Z15bitfield_insertDv16_cS_jj(<16 x i8> noundef %b, <16 x i8> noundef %i, i32 noundef 4, i32 noundef 2) #2
+  store <16 x i8> %call, ptr addrspace(1) %res, align 16, !tbaa !22
+  ret void
+}
+
+declare spir_func <16 x i8> @_Z15bitfield_insertDv16_cS_jj(<16 x i8> noundef, <16 x i8> noundef, i32 noundef, i32 noundef) 
+
+; CHECK-EXTENSION: %[[#]] = OpFunction %[[#]] None %[[#]]
+; CHECK-EXTENSION: %[[#insertbase_uchar16:]] = OpFunctionParameter %[[#]]
+; CHECK-EXTENSION: %[[#insertinsert_uchar16:]] = OpFunctionParameter %[[#]]
+; CHECK-EXTENSION: %[[#]] = OpBitFieldInsert %[[#]] %[[#insertbase_uchar16]] %[[#insertinsert_uchar16]]
+; OpenCL equivalent.
+; kernel void testInsert_uchar16(uchar16 b, uchar16 i, global uchar16 *res) {
+;   *res = bitfield_insert(b, i, 4, 2);
+; }
+
+define dso_local spir_kernel void @testInsert_uchar16(<16 x i8> noundef %b, <16 x i8> noundef %i, ptr addrspace(1) nocapture noundef writeonly align 16 initializes((0, 16)) %res) {
+entry:
+  %call = tail call spir_func <16 x i8> @_Z15bitfield_insertDv16_hS_jj(<16 x i8> noundef %b, <16 x i8> noundef %i, i32 noundef 4, i32 noundef 2) #2
+  store <16 x i8> %call, ptr addrspace(1) %res, align 16, !tbaa !22
+  ret void
+}
+
+declare spir_func <16 x i8> @_Z15bitfield_insertDv16_hS_jj(<16 x i8> noundef, <16 x i8> noundef, i32 noundef, i32 noundef) 
+
+; CHECK-EXTENSION: %[[#]] = OpFunction %[[#]] None %[[#]]
+; CHECK-EXTENSION: %[[#sextractbase:]] = OpFunctionParameter %[[#]]
+; CHECK-EXTENSION: %[[#sextractbaseu:]] = OpFunctionParameter %[[#]]
+; CHECK-EXTENSION: %[[#]] = OpBitFieldSExtract %[[#]] %[[#sextractbase]]
+; CHECK-EXTENSION: %[[#]] = OpBitFieldSExtract %[[#]] %[[#sextractbaseu]]
+; OpenCL equivalent.
+; kernel void testExtractS_long(long b, ulong bu, global long *res) {
+;   *res = bitfield_extract_signed(b, 5, 4);
+;   *res += bitfield_extract_signed(bu, 5, 4);
+; }
+
+define dso_local spir_kernel void @testExtractS_long(i64 noundef %b, i64 noundef %bu, ptr addrspace(1) nocapture noundef writeonly align 8 initializes((0, 8)) %res) {
+entry:
+  %call = tail call spir_func i64 @_Z23bitfield_extract_signedljj(i64 noundef %b, i32 noundef 5, i32 noundef 4) #2
+  %call1 = tail call spir_func i64 @_Z23bitfield_extract_signedmjj(i64 noundef %bu, i32 noundef 5, i32 noundef 4) #2
+  %add = add nsw i64 %call1, %call
+  store i64 %add, ptr addrspace(1) %res, align 8, !tbaa !7
+  ret void
+}
+
+declare spir_func i64 @_Z23bitfield_extract_signedljj(i64 noundef, i32 noundef, i32 noundef) 
+
+declare spir_func i64 @_Z23bitfield_extract_signedmjj(i64 noundef, i32 noundef, i32 noundef) 
+
+; CHECK-EXTENSION: %[[#]] = OpFunction %[[#]] None %[[#]]
+; CHECK-EXTENSION: %[[#sextractbase:]] = OpFunctionParameter %[[#]]
+; CHECK-EXTENSION: %[[#sextractbaseu:]] = OpFunctionParameter %[[#]]
+; CHECK-EXTENSION: %[[#]] = OpBitFieldSExtract %[[#]] %[[#sextractbase]]
+; CHECK-EXTENSION: %[[#]] = OpBitFieldSExtract %[[#]] %[[#sextractbaseu]]
+; OpenCL equivalent.
+; kernel void testExtractS_int(int b, uint bu, global int *res) {
+;   *res = bitfield_extract_signed(b, 5, 4);
+;   *res += bitfield_extract_signed(bu, 5, 4);
+; }
+
+define dso_local spir_kernel void @testExtractS_int(i32 noundef %b, i32 noundef %bu, ptr addrspace(1) nocapture noundef writeonly align 4 initializes((0, 4)) %res) {
+entry:
+  %call = tail call spir_func i32 @_Z23bitfield_extract_signedijj(i32 noundef %b, i32 noundef 5, i32 noundef 4) #2
+  %call1 = tail call spir_func i32 @_Z23bitfield_extract_signedjjj(i32 noundef %bu, i32 noundef 5, i32 noundef 4) #2
+  %add = add nsw i32 %call1, %call
+  store i32 %add, ptr addrspace(1) %res, align 4, !tbaa !13
+  ret void
+}
+
+declare spir_func i32 @_Z23bitfield_extract_signedijj(i32 noundef, i32 noundef, i32 noundef) 
+
+declare spir_func i32 @_Z23bitfield_extract_signedjjj(i32 noundef, i32 noundef, i32 noundef) 
+
+; CHECK-EXTENSION: %[[#]] = OpFunction %[[#]] None %[[#]]
+; CHECK-EXTENSION: %[[#sextractbase:]] = OpFunctionParameter %[[#]]
+; CHECK-EXTENSION: %[[#sextractbaseu:]] = OpFunctionParameter %[[#]]
+; CHECK-EXTENSION: %[[#]] = OpBitFieldSExtract %[[#]] %[[#sextractbase]]
+; CHECK-EXTENSION: %[[#]] = OpBitFieldSExtract %[[#]] %[[#sextractbaseu]]
+; OpenCL equivalent.
+; kernel void testExtractS_short(short b, ushort bu, global short *res) {
+;   *res = bitfield_extract_signed(b, 5, 4);
+;   *res += bitfield_extract_signed(bu, 5, 4);
+; }
+
+define dso_local spir_kernel void @testExtractS_short(i16 noundef signext %b, i16 noundef zeroext %bu, ptr addrspace(1) nocapture noundef writeonly align 2 initializes((0, 2)) %res) {
+entry:
+  %call = tail call spir_func signext i16 @_Z23bitfield_extract_signedsjj(i16 noundef signext %b, i32 noundef 5, i32 noundef 4) #2
+  %call1 = tail call spir_func signext i16 @_Z23bitfield_extract_signedtjj(i16 noundef zeroext %bu, i32 noundef 5, i32 noundef 4) #2
+  %add = add i16 %call1, %call
+  store i16 %add, ptr addrspace(1) %res, align 2, !tbaa !17
+  ret void
+}
+
+declare spir_func signext i16 @_Z23bitfield_extract_signedsjj(i16 noundef signext, i32 noundef, i32 noundef) 
+
+declare spir_func signext i16 @_Z23bitfield_extract_signedtjj(i16 noundef zeroext, i32 noundef, i32 noundef) 
+
+; CHECK-EXTENSION: %[[#]] = OpFunction %[[#]] None %[[#]]
+; CHECK-EXTENSION: %[[#sextractbase:]] = OpFunctionParameter %[[#]]
+; CHECK-EXTENSION: %[[#sextractbaseu:]] = OpFunctionParameter %[[#]]
+; CHECK-EXTENSION: %[[#]] = OpBitFieldSExtract %[[#]] %[[#sextractbase]]
+; CHECK-EXTENSION: %[[#]] = OpBitFieldSExtract %[[#]] %[[#sextractbaseu]]
+; OpenCL equivalent.
+; kernel void testExtractS_char(char b, uchar bu, global char *res) {
+;   *res = bitfield_extract_signed(b, 5, 4);
+;   *res += bitfield_extract_signed(bu, 5, 4);
+; }
+
+define dso_local spir_kernel void @testExtractS_char(i8 noundef signext %b, i8 noundef zeroext %bu, ptr addrspace(1) nocapture noundef writeonly align 1 initializes((0, 1)) %res) {
+entry:
+  %call = tail call spir_func signext i8 @_Z23bitfield_extract_signedcjj(i8 noundef signext %b, i32 noundef 5, i32 noundef 4) #2
+  %call1 = tail call spir_func signext i8 @_Z23bitfield_extract_signedhjj(i8 noundef zeroext %bu, i32 noundef 5, i32 noundef 4) #2
+  %add = add i8 %call1, %call
+  store i8 %add, ptr addrspace(1) %res, align 1, !tbaa !22
+  ret void
+}
+
+declare spir_func signext i8 @_Z23bitfield_extract_signedcjj(i8 noundef signext, i32 noundef, i32 noundef) 
+
+declare spir_func signext i8 @_Z23bitfield_extract_signedhjj(i8 noundef zeroext, i32 noundef, i32 noundef) 
+
+; CHECK-EXTENSION: %[[#]] = OpFunction %[[#]] None %[[#]]
+; CHECK-EXTENSION: %[[#sextractbase:]] = OpFunctionParameter %[[#]]
+; CHECK-EXTENSION: %[[#sextractbaseu:]] = OpFunctionParameter %[[#]]
+; CHECK-EXTENSION: %[[#]] = OpBitFieldSExtract %[[#]] %[[#sextractbase]]
+; CHECK-EXTENSION: %[[#]] = OpBitFieldSExtract %[[#]] %[[#sextractbaseu]]
+; OpenCL equivalent.
+; kernel void testExtractS_long2(long b, ulong bu, global long *res) {
+;   *res = bitfield_extract_signed(b, 5, 4);
+;   *res += bitfield_extract_signed(bu, 5, 4);
+; }
+
+define dso_local spir_kernel void @testExtractS_long2(i64 noundef %b, i64 noundef %bu, ptr addrspace(1) nocapture noundef writeonly align 8 initializes((0, 8)) %res) {
+entry:
+  %call = tail call spir_func i64 @_Z23bitfield_extract_signedljj(i64 noundef %b, i32 noundef 5, i32 noundef 4) #2
+  %call1 = tail call spir_func i64 @_Z23bitfield_extract_signedmjj(i64 noundef %bu, i32 noundef 5, i32 noundef 4) #2
+  %add = add nsw i64 %call1, %call
+  store i64 %add, ptr addrspace(1) %res, align 8, !tbaa !7
+  ret void
+}
+
+; CHECK-EXTENSION: %[[#]] = OpFunction %[[#]] None %[[#]]
+; CHECK-EXTENSION: %[[#sextractbase:]] = OpFunctionParameter %[[#]]
+; CHECK-EXTENSION: %[[#sextractbaseu:]] = OpFunctionParameter %[[#]]
+; CHECK-EXTENSION: %[[#]] = OpBitFieldSExtract %[[#]] %[[#sextractbase]]
+; CHECK-EXTENSION: %[[#]] = OpBitFieldSExtract %[[#]] %[[#sextractbaseu]]
+; OpenCL equivalent.
+; kernel void testExtractS_int2(int b, uint bu, global int *res) {
+;   *res = bitfield_extract_signed(b, 5, 4);
+;   *res += bitfield_extract_signed(bu, 5, 4);
+; }
+
+define dso_local spir_kernel void @testExtractS_int2(i32 noundef %b, i32 noundef %bu, ptr addrspace(1) nocapture noundef writeonly align 4 initializes((0, 4)) %res) {
+entry:
+  %call = tail call spir_func i32 @_Z23bitfield_extract_signedijj(i32 noundef %b, i32 noundef 5, i32 noundef 4) #2
+  %call1 = tail call spir_func i32 @_Z23bitfield_extract_signedjjj(i32 noundef %bu, i32 noundef 5, i32 noundef 4) #2
+  %add = add nsw i32 %call1, %call
+  store i32 %add, ptr addrspace(1) %res, align 4, !tbaa !13
+  ret void
+}
+
+; CHECK-EXTENSION: %[[#]] = OpFunction %[[#]] None %[[#]]
+; CHECK-EXTENSION: %[[#sextractbase:]] = OpFunctionParameter %[[#]]
+; CHECK-EXTENSION: %[[#sextractbaseu:]] = OpFunctionParameter %[[#]]
+; CHECK-EXTENSION: %[[#]] = OpBitFieldSExtract %[[#]] %[[#sextractbase]]
+; CHECK-EXTENSION: %[[#]] = OpBitFieldSExtract %[[#]] %[[#sextractbaseu]]
+; OpenCL equivalent.
+; kernel void testExtractS_short2(short b, ushort bu, global short *res) {
+;   *res = bitfield_extract_signed(b, 5, 4);
+;   *res += bitfield_extract_signed(bu, 5, 4);
+; }
+
+define dso_local spir_kernel void @testExtractS_short2(i16 noundef signext %b, i16 noundef zeroext %bu, ptr addrspace(1) nocapture noundef writeonly align 2 initializes((0, 2)) %res) {
+entry:
+  %call = tail call spir_func signext i16 @_Z23bitfield_extract_signedsjj(i16 noundef signext %b, i32 noundef 5, i32 noundef 4) #2
+  %call1 = tail call spir_func signext i16 @_Z23bitfield_extract_signedtjj(i16 noundef zeroext %bu, i32 noundef 5, i32 noundef 4) #2
+  %add = add i16 %call1, %call
+  store i16 %add, ptr addrspace(1) %res, align 2, !tbaa !17
+  ret void
+}
+
+; CHECK-EXTENSION: %[[#]] = OpFunction %[[#]] None %[[#]]
+; CHECK-EXTENSION: %[[#sextractbase:]] = OpFunctionParameter %[[#]]
+; CHECK-EXTENSION: %[[#sextractbaseu:]] = OpFunctionParameter %[[#]]
+; CHECK-EXTENSION: %[[#]] = OpBitFieldSExtract %[[#]] %[[#sextractbase]]
+; CHECK-EXTENSION: %[[#]] = OpBitFieldSExtract %[[#]] %[[#sextractbaseu]]
+; OpenCL equivalent.
+; kernel void testExtractS_char2(char b, uchar bu, global char *res) {
+;   *res = bitfield_extract_signed(b, 5, 4);
+;   *res += bitfield_extract_signed(bu, 5, 4);
+; }
+
+define dso_local spir_kernel void @testExtractS_char2(i8 noundef signext %b, i8 noundef zeroext %bu, ptr addrspace(1) nocapture noundef writeonly align 1 initializes((0, 1)) %res) {
+entry:
+  %call = tail call spir_func signext i8 @_Z23bitfield_extract_signedcjj(i8 noundef signext %b, i32 noundef 5, i32 noundef 4) #2
+  %call1 = tail call spir_func signext i8 @_Z23bitfield_extract_signedhjj(i8 noundef zeroext %bu, i32 noundef 5, i32 noundef 4) #2
+  %add = add i8 %call1, %call
+  store i8 %add, ptr addrspace(1) %res, align 1, !tbaa !22
+  ret void
+}
+
+; CHECK-EXTENSION: %[[#]] = OpFunction %[[#]] None %[[#]]
+; CHECK-EXTENSION: %[[#sextractbase:]] = OpFunctionParameter %[[#]]
+; CHECK-EXTENSION: %[[#sextractbaseu:]] = OpFunctionParameter %[[#]]
+; CHECK-EXTENSION: %[[#]] = OpBitFieldSExtract %[[#]] %[[#sextractbase]]
+; CHECK-EXTENSION: %[[#]] = OpBitFieldSExtract %[[#]] %[[#sextractbaseu]]
+; OpenCL equivalent.
+; kernel void testExtractS_long3(long b, ulong bu, global long *res) {
+;   *res = bitfield_extract_signed(b, 5, 4);
+;   *res += bitfield_extract_signed(bu, 5, 4);
+; }
+
+define dso_local spir_kernel void @testExtractS_long3(i64 noundef %b, i64 noundef %bu, ptr addrspace(1) nocapture noundef writeonly align 8 initializes((0, 8)) %res) {
+entry:
+  %call = tail call spir_func i64 @_Z23bitfield_extract_signedljj(i64 noundef %b, i32 noundef 5, i32 noundef 4) #2
+  %call1 = tail call spir_func i64 @_Z23bitfield_extract_signedmjj(i64 noundef %bu, i32 noundef 5, i32 noundef 4) #2
+  %add = add nsw i64 %call1, %call
+  store i64 %add, ptr addrspace(1) %res, align 8, !tbaa !7
+  ret void
+}
+
+; CHECK-EXTENSION: %[[#]] = OpFunction %[[#]] None %[[#]]
+; CHECK-EXTENSION: %[[#sextractbase:]] = OpFunctionParameter %[[#]]
+; CHECK-EXTENSION: %[[#sextractbaseu:]] = OpFunctionParameter %[[#]]
+; CHECK-EXTENSION: %[[#]] = OpBitFieldSExtract %[[#]] %[[#sextractbase]]
+; CHECK-EXTENSION: %[[#]] = OpBitFieldSExtract %[[#]] %[[#sextractbaseu]]
+; OpenCL equivalent.
+; kernel void testExtractS_int3(int b, uint bu, global int *res) {
+;   *res = bitfield_extract_signed(b, 5, 4);
+;   *res += bitfield_extract_signed(bu, 5, 4);
+; }
+
+define dso_local spir_kernel void @testExtractS_int3(i32 noundef %b, i32 noundef %bu, ptr addrspace(1) nocapture noundef writeonly align 4 initializes((0, 4)) %res) {
+entry:
+  %call = tail call spir_func i32 @_Z23bitfield_extract_signedijj(i32 noundef %b, i32 noundef 5, i32 noundef 4) #2
+  %call1 = tail call spir_func i32 @_Z23bitfield_extract_signedjjj(i32 noundef %bu, i32 noundef 5, i32 noundef 4) #2
+  %add = add nsw i32 %call1, %call
+  store i32 %add, ptr addrspace(1) %res, align 4, !tbaa !13
+  ret void
+}
+
+; CHECK-EXTENSION: %[[#]] = OpFunction %[[#]] None %[[#]]
+; CHECK-EXTENSION: %[[#sextractbase:]] = OpFunctionParameter %[[#]]
+; CHECK-EXTENSION: %[[#sextractbaseu:]] = OpFunctionParameter %[[#]]
+; CHECK-EXTENSION: %[[#]] = OpBitFieldSExtract %[[#]] %[[#sextractbase]]
+; CHECK-EXTENSION: %[[#]] = OpBitFieldSExtract %[[#]] %[[#sextractbaseu]]
+; OpenCL equivalent.
+; kernel void testExtractS_short3(short b, ushort bu, global short *res) {
+;   *res = bitfield_extract_signed(b, 5, 4);
+;   *res += bitfield_extract_signed(bu, 5, 4);
+; }
+
+define dso_local spir_kernel void @testExtractS_short3(i16 noundef signext %b, i16 noundef zeroext %bu, ptr addrspace(1) nocapture noundef writeonly align 2 initializes((0, 2)) %res) {
+entry:
+  %call = tail call spir_func signext i16 @_Z23bitfield_extract_signedsjj(i16 noundef signext %b, i32 noundef 5, i32 noundef 4) #2
+  %call1 = tail call spir_func signext i16 @_Z23bitfield_extract_signedtjj(i16 noundef zeroext %bu, i32 noundef 5, i32 noundef 4) #2
+  %add = add i16 %call1, %call
+  store i16 %add, ptr addrspace(1) %res, align 2, !tbaa !17
+  ret void
+}
+
+; CHECK-EXTENSION: %[[#]] = OpFunction %[[#]] None %[[#]]
+; CHECK-EXTENSION: %[[#sextractbase:]] = OpFunctionParameter %[[#]]
+; CHECK-EXTENSION: %[[#sextractbaseu:]] = OpFunctionParameter %[[#]]
+; CHECK-EXTENSION: %[[#]] = OpBitFieldSExtract %[[#]] %[[#sextractbase]]
+; CHECK-EXTENSION: %[[#]] = OpBitFieldSExtract %[[#]] %[[#sextractbaseu]]
+; OpenCL equivalent.
+; kernel void testExtractS_char3(char b, uchar bu, global char *res) {
+;   *res = bitfield_extract_signed(b, 5, 4);
+;   *res += bitfield_extract_signed(bu, 5, 4);
+; }
+
+define dso_local spir_kernel void @testExtractS_char3(i8 noundef signext %b, i8 noundef zeroext %bu, ptr addrspace(1) nocapture noundef writeonly align 1 initializes((0, 1)) %res) {
+entry:
+  %call = tail call spir_func signext i8 @_Z23bitfield_extract_signedcjj(i8 noundef signext %b, i32 noundef 5, i32 noundef 4) #2
+  %call1 = tail call spir_func signext i8 @_Z23bitfield_extract_signedhjj(i8 noundef zeroext %bu, i32 noundef 5, i32 noundef 4) #2
+  %add = add i8 %call1, %call
+  store i8 %add, ptr addrspace(1) %res, align 1, !tbaa !22
+  ret void
+}
+
+; CHECK-EXTENSION: %[[#]] = OpFunction %[[#]] None %[[#]]
+; CHECK-EXTENSION: %[[#sextractbase:]] = OpFunctionParameter %[[#]]
+; CHECK-EXTENSION: %[[#sextractbaseu:]] = OpFunctionParameter %[[#]]
+; CHECK-EXTENSION: %[[#]] = OpBitFieldSExtract %[[#]] %[[#sextractbase]]
+; CHECK-EXTENSION: %[[#]] = OpBitFieldSExtract %[[#]] %[[#sextractbaseu]]
+; OpenCL equivalent.
+; kernel void testExtractS_long4(long b, ulong bu, global long *res) {
+;   *res = bitfield_extract_signed(b, 5, 4);
+;   *res += bitfield_extract_signed(bu, 5, 4);
+; }
+
+define dso_local spir_kernel void @testExtractS_long4(i64 noundef %b, i64 noundef %bu, ptr addrspace(1) nocapture noundef writeonly align 8 initializes((0, 8)) %res) {
+entry:
+  %call = tail call spir_func i64 @_Z23bitfield_extract_signedljj(i64 noundef %b, i32 noundef 5, i32 noundef 4) #2
+  %call1 = tail call spir_func i64 @_Z23bitfield_extract_signedmjj(i64 noundef %bu, i32 noundef 5, i32 noundef 4) #2
+  %add = add nsw i64 %call1, %call
+  store i64 %add, ptr addrspace(1) %res, align 8, !tbaa !7
+  ret void
+}
+
+; CHECK-EXTENSION: %[[#]] = OpFunction %[[#]] None %[[#]]
+; CHECK-EXTENSION: %[[#sextractbase:]] = OpFunctionParameter %[[#]]
+; CHECK-EXTENSION: %[[#sextractbaseu:]] = OpFunctionParameter %[[#]]
+; CHECK-EXTENSION: %[[#]] = OpBitFieldSExtract %[[#]] %[[#sextractbase]]
+; CHECK-EXTENSION: %[[#]] = OpBitFieldSExtract %[[#]] %[[#sextractbaseu]]
+; OpenCL equivalent.
+; kernel void testExtractS_int4(int b, uint bu, global int *res) {
+;   *res = bitfield_extract_signed(b, 5, 4);
+;   *res += bitfield_extract_signed(bu, 5, 4);
+; }
+
+define dso_local spir_kernel void @testExtractS_int4(i32 noundef %b, i32 noundef %bu, ptr addrspace(1) nocapture noundef writeonly align 4 initializes((0, 4)) %res) {
+entry:
+  %call = tail call spir_func i32 @_Z23bitfield_extract_signedijj(i32 noundef %b, i32 noundef 5, i32 noundef 4) #2
+  %call1 = tail call spir_func i32 @_Z23bitfield_extract_signedjjj(i32 noundef %bu, i32 noundef 5, i32 noundef 4) #2
+  %add = add nsw i32 %call1, %call
+  store i32 %add, ptr addrspace(1) %res, align 4, !tbaa !13
+  ret void
+}
+
+; CHECK-EXTENSION: %[[#]] = OpFunction %[[#]] None %[[#]]
+; CHECK-EXTENSION: %[[#sextractbase:]] = OpFunctionParameter %[[#]]
+; CHECK-EXTENSION: %[[#sextractbaseu:]] = OpFunctionParameter %[[#]]
+; CHECK-EXTENSION: %[[#]] = OpBitFieldSExtract %[[#]] %[[#sextractbase]]
+; CHECK-EXTENSION: %[[#]] = OpBitFieldSExtract %[[#]] %[[#sextractbaseu]]
+; OpenCL equivalent.
+; kernel void testExtractS_short4(short b, ushort bu, global short *res) {
+;   *res = bitfield_extract_signed(b, 5, 4);
+;   *res += bitfield_extract_signed(bu, 5, 4);
+; }
+
+define dso_local spir_kernel void @testExtractS_short4(i16 noundef signext %b, i16 noundef zeroext %bu, ptr addrspace(1) nocapture noundef writeonly align 2 initializes((0, 2)) %res) {
+entry:
+  %call = tail call spir_func signext i16 @_Z23bitfield_extract_signedsjj(i16 noundef signext %b, i32 noundef 5, i32 noundef 4) #2
+  %call1 = tail call spir_func signext i16 @_Z23bitfield_extract_signedtjj(i16 noundef zeroext %bu, i32 noundef 5, i32 noundef 4) #2
+  %add = add i16 %call1, %call
+  store i16 %add, ptr addrspace(1) %res, align 2, !tbaa !17
+  ret void
+}
+
+; CHECK-EXTENSION: %[[#]] = OpFunction %[[#]] None %[[#]]
+; CHECK-EXTENSION: %[[#sextractbase:]] = OpFunctionParameter %[[#]]
+; CHECK-EXTENSION: %[[#sextractbaseu:]] = OpFunctionParameter %[[#]]
+; CHECK-EXTENSION: %[[#]] = OpBitFieldSExtract %[[#]] %[[#sextractbase]]
+; CHECK-EXTENSION: %[[#]] = OpBitFieldSExtract %[[#]] %[[#sextractbaseu]]
+; OpenCL equivalent.
+; kernel void testExtractS_char4(char b, uchar bu, global char *res) {
+;   *res = bitfield_extract_signed(b, 5, 4);
+;   *res += bitfield_extract_signed(bu, 5, 4);
+; }
+
+define dso_local spir_kernel void @testExtractS_char4(i8 noundef signext %b, i8 noundef zeroext %bu, ptr addrspace(1) nocapture noundef writeonly align 1 initializes((0, 1)) %res) {
+entry:
+  %call = tail call spir_func signext i8 @_Z23bitfield_extract_signedcjj(i8 noundef signext %b, i32 noundef 5, i32 noundef 4) #2
+  %call1 = tail call spir_func signext i8 @_Z23bitfield_extract_signedhjj(i8 noundef zeroext %bu, i32 noundef 5, i32 noundef 4) #2
+  %add = add i8 %call1, %call
+  store i8 %add, ptr addrspace(1) %res, align 1, !tbaa !22
+  ret void
+}
+
+; CHECK-EXTENSION: %[[#]] = OpFunction %[[#]] None %[[#]]
+; CHECK-EXTENSION: %[[#sextractbase:]] = OpFunctionParameter %[[#]]
+; CHECK-EXTENSION: %[[#sextractbaseu:]] = OpFunctionParameter %[[#]]
+; CHECK-EXTENSION: %[[#]] = OpBitFieldSExtract %[[#]] %[[#sextractbase]]
+; CHECK-EXTENSION: %[[#]] = OpBitFieldSExtract %[[#]] %[[#sextractbaseu]]
+; OpenCL equivalent.
+; kernel void testExtractS_long8(long b, ulong bu, global long *res) {
+;   *res = bitfield_extract_signed(b, 5, 4);
+;   *res += bitfield_extract_signed(bu, 5, 4);
+; }
+
+define dso_local spir_kernel void @testExtractS_long8(i64 noundef %b, i64 noundef %bu, ptr addrspace(1) nocapture noundef writeonly align 8 initializes((0, 8)) %res) {
+entry:
+  %call = tail call spir_func i64 @_Z23bitfield_extract_signedljj(i64 noundef %b, i32 noundef 5, i32 noundef 4) #2
+  %call1 = tail call spir_func i64 @_Z23bitfield_extract_signedmjj(i64 noundef %bu, i32 noundef 5, i32 noundef 4) #2
+  %add = add nsw i64 %call1, %call
+  store i64 %add, ptr addrspace(1) %res, align 8, !tbaa !7
+  ret void
+}
+
+; CHECK-EXTENSION: %[[#]] = OpFunction %[[#]] None %[[#]]
+; CHECK-EXTENSION: %[[#sextractbase:]] = OpFunctionParameter %[[#]]
+; CHECK-EXTENSION: %[[#sextractbaseu:]] = OpFunctionParameter %[[#]]
+; CHECK-EXTENSION: %[[#]] = OpBitFieldSExtract %[[#]] %[[#sextractbase]]
+; CHECK-EXTENSION: %[[#]] = OpBitFieldSExtract %[[#]] %[[#sextractbaseu]]
+; OpenCL equivalent.
+; kernel void testExtractS_int8(int b, uint bu, global int *res) {
+;   *res = bitfield_extract_signed(b, 5, 4);
+;   *res += bitfield_extract_signed(bu, 5, 4);
+; }
+
+define dso_local spir_kernel void @testExtractS_int8(i32 noundef %b, i32 noundef %bu, ptr addrspace(1) nocapture noundef writeonly align 4 initializes((0, 4)) %res) {
+entry:
+  %call = tail call spir_func i32 @_Z23bitfield_extract_signedijj(i32 noundef %b, i32 noundef 5, i32 noundef 4) #2
+  %call1 = tail call spir_func i32 @_Z23bitfield_extract_signedjjj(i32 noundef %bu, i32 noundef 5, i32 noundef 4) #2
+  %add = add nsw i32 %call1, %call
+  store i32 %add, ptr addrspace(1) %res, align 4, !tbaa !13
+  ret void
+}
+
+; CHECK-EXTENSION: %[[#]] = OpFunction %[[#]] None %[[#]]
+; CHECK-EXTENSION: %[[#sextractbase:]] = OpFunctionParameter %[[#]]
+; CHECK-EXTENSION: %[[#sextractbaseu:]] = OpFunctionParameter %[[#]]
+; CHECK-EXTENSION: %[[#]] = OpBitFieldSExtract %[[#]] %[[#sextractbase]]
+; CHECK-EXTENSION: %[[#]] = OpBitFieldSExtract %[[#]] %[[#sextractbaseu]]
+; OpenCL equivalent.
+; kernel void testExtractS_short8(short b, ushort bu, global short *res) {
+;   *res = bitfield_extract_signed(b, 5, 4);
+;   *res += bitfield_extract_signed(bu, 5, 4);
+; }
+
+define dso_local spir_kernel void @testExtractS_short8(i16 noundef signext %b, i16 noundef zeroext %bu, ptr addrspace(1) nocapture noundef writeonly align 2 initializes((0, 2)) %res) {
+entry:
+  %call = tail call spir_func signext i16 @_Z23bitfield_extract_signedsjj(i16 noundef signext %b, i32 noundef 5, i32 noundef 4) #2
+  %call1 = tail call spir_func signext i16 @_Z23bitfield_extract_signedtjj(i16 noundef zeroext %bu, i32 noundef 5, i32 noundef 4) #2
+  %add = add i16 %call1, %call
+  store i16 %add, ptr addrspace(1) %res, align 2, !tbaa !17
+  ret void
+}
+
+; CHECK-EXTENSION: %[[#]] = OpFunction %[[#]] None %[[#]]
+; CHECK-EXTENSION: %[[#sextractbase:]] = OpFunctionParameter %[[#]]
+; CHECK-EXTENSION: %[[#sextractbaseu:]] = OpFunctionParameter %[[#]]
+; CHECK-EXTENSION: %[[#]] = OpBitFieldSExtract %[[#]] %[[#sextractbase]]
+; CHECK-EXTENSION: %[[#]] = OpBitFieldSExtract %[[#]] %[[#sextractbaseu]]
+; OpenCL equivalent.
+; kernel void testExtractS_char8(char b, uchar bu, global char *res) {
+;   *res = bitfield_extract_signed(b, 5, 4);
+;   *res += bitfield_extract_signed(bu, 5, 4);
+; }
+
+define dso_local spir_kernel void @testExtractS_char8(i8 noundef signext %b, i8 noundef zeroext %bu, ptr addrspace(1) nocapture noundef writeonly align 1 initializes((0, 1)) %res) {
+entry:
+  %call = tail call spir_func signext i8 @_Z23bitfield_extract_signedcjj(i8 noundef signext %b, i32 noundef 5, i32 noundef 4) #2
+  %call1 = tail call spir_func signext i8 @_Z23bitfield_extract_signedhjj(i8 noundef zeroext %bu, i32 noundef 5, i32 noundef 4) #2
+  %add = add i8 %call1, %call
+  store i8 %add, ptr addrspace(1) %res, align 1, !tbaa !22
+  ret void
+}
+
+; CHECK-EXTENSION: %[[#]] = OpFunction %[[#]] None %[[#]]
+; CHECK-EXTENSION: %[[#sextractbase:]] = OpFunctionParameter %[[#]]
+; CHECK-EXTENSION: %[[#sextractbaseu:]] = OpFunctionParameter %[[#]]
+; CHECK-EXTENSION: %[[#]] = OpBitFieldSExtract %[[#]] %[[#sextractbase]]
+; CHECK-EXTENSION: %[[#]] = OpBitFieldSExtract %[[#]] %[[#sextractbaseu]]
+; OpenCL equivalent.
+; kernel void testExtractS_long16(long b, ulong bu, global long *res) {
+;   *res = bitfield_extract_signed(b, 5, 4);
+;   *res += bitfield_extract_signed(bu, 5, 4);
+; }
+
+define dso_local spir_kernel void @testExtractS_long16(i64 noundef %b, i64 noundef %bu, ptr addrspace(1) nocapture noundef writeonly align 8 initializes((0, 8)) %res) {
+entry:
+  %call = tail call spir_func i64 @_Z23bitfield_extract_signedljj(i64 noundef %b, i32 noundef 5, i32 noundef 4) #2
+  %call1 = tail call spir_func i64 @_Z23bitfield_extract_signedmjj(i64 noundef %bu, i32 noundef 5, i32 noundef 4) #2
+  %add = add nsw i64 %call1, %call
+  store i64 %add, ptr addrspace(1) %res, align 8, !tbaa !7
+  ret void
+}
+
+; CHECK-EXTENSION: %[[#]] = OpFunction %[[#]] None %[[#]]
+; CHECK-EXTENSION: %[[#sextractbase:]] = OpFunctionParameter %[[#]]
+; CHECK-EXTENSION: %[[#sextractbaseu:]] = OpFunctionParameter %[[#]]
+; CHECK-EXTENSION: %[[#]] = OpBitFieldSExtract %[[#]] %[[#sextractbase]]
+; CHECK-EXTENSION: %[[#]] = OpBitFieldSExtract %[[#]] %[[#sextractbaseu]]
+; OpenCL equivalent.
+; kernel void testExtractS_int16(int b, uint bu, global int *res) {
+;   *res = bitfield_extract_signed(b, 5, 4);
+;   *res += bitfield_extract_signed(bu, 5, 4);
+; }
+
+define dso_local spir_kernel void @testExtractS_int16(i32 noundef %b, i32 noundef %bu, ptr addrspace(1) nocapture noundef writeonly align 4 initializes((0, 4)) %res) {
+entry:
+  %call = tail call spir_func i32 @_Z23bitfield_extract_signedijj(i32 noundef %b, i32 noundef 5, i32 noundef 4) #2
+  %call1 = tail call spir_func i32 @_Z23bitfield_extract_signedjjj(i32 noundef %bu, i32 noundef 5, i32 noundef 4) #2
+  %add = add nsw i32 %call1, %call
+  store i32 %add, ptr addrspace(1) %res, align 4, !tbaa !13
+  ret void
+}
+
+; CHECK-EXTENSION: %[[#]] = OpFunction %[[#]] None %[[#]]
+; CHECK-EXTENSION: %[[#sextractbase:]] = OpFunctionParameter %[[#]]
+; CHECK-EXTENSION: %[[#sextractbaseu:]] = OpFunctionParameter %[[#]]
+; CHECK-EXTENSION: %[[#]] = OpBitFieldSExtract %[[#]] %[[#sextractbase]]
+; CHECK-EXTENSION: %[[#]] = OpBitFieldSExtract %[[#]] %[[#sextractbaseu]]
+; OpenCL equivalent.
+; kernel void testExtractS_short16(short b, ushort bu, global short *res) {
+;   *res = bitfield_extract_signed(b, 5, 4);
+;   *res += bitfield_extract_signed(bu, 5, 4);
+; }
+
+define dso_local spir_kernel void @testExtractS_short16(i16 noundef signext %b, i16 noundef zeroext %bu, ptr addrspace(1) nocapture noundef writeonly align 2 initializes((0, 2)) %res) {
+entry:
+  %call = tail call spir_func signext i16 @_Z23bitfield_extract_signedsjj(i16 noundef signext %b, i32 noundef 5, i32 noundef 4) #2
+  %call1 = tail call spir_func signext i16 @_Z23bitfield_extract_signedtjj(i16 noundef zeroext %bu, i32 noundef 5, i32 noundef 4) #2
+  %add = add i16 %call1, %call
+  store i16 %add, ptr addrspace(1) %res, align 2, !tbaa !17
+  ret void
+}
+
+; CHECK-EXTENSION: %[[#]] = OpFunction %[[#]] None %[[#]]
+; CHECK-EXTENSION: %[[#sextractbase:]] = OpFunctionParameter %[[#]]
+; CHECK-EXTENSION: %[[#sextractbaseu:]] = OpFunctionParameter %[[#]]
+; CHECK-EXTENSION: %[[#]] = OpBitFieldSExtract %[[#]] %[[#sextractbase]]
+; CHECK-EXTENSION: %[[#]] = OpBitFieldSExtract %[[#]] %[[#sextractbaseu]]
+; OpenCL equivalent.
+; kernel void testExtractS_char16(char b, uchar bu, global char *res) {
+;   *res = bitfield_extract_signed(b, 5, 4);
+;   *res += bitfield_extract_signed(bu, 5, 4);
+; }
+
+define dso_local spir_kernel void @testExtractS_char16(i8 noundef signext %b, i8 noundef zeroext %bu, ptr addrspace(1) nocapture noundef writeonly align 1 initializes((0, 1)) %res) {
+entry:
+  %call = tail call spir_func signext i8 @_Z23bitfield_extract_signedcjj(i8 noundef signext %b, i32 noundef 5, i32 noundef 4) #2
+  %call1 = tail call spir_func signext i8 @_Z23bitfield_extract_signedhjj(i8 noundef zeroext %bu, i32 noundef 5, i32 noundef 4) #2
+  %add = add i8 %call1, %call
+  store i8 %add, ptr addrspace(1) %res, align 1, !tbaa !22
+  ret void
+}
+
+; CHECK-EXTENSION: %[[#]] = OpFunction %[[#]] None %[[#]]
+; CHECK-EXTENSION: %[[#uextractbase:]] = OpFunctionParameter %[[#]]
+; CHECK-EXTENSION: %[[#uextractbaseu:]] = OpFunctionParameter %[[#]]
+; CHECK-EXTENSION: %[[#]] = OpBitFieldUExtract %[[#]] %[[#uextractbase]]
+; CHECK-EXTENSION: %[[#]] = OpBitFieldUExtract %[[#]] %[[#uextractbaseu]]
+; OpenCL equivalent.
+; kernel void testExtractU_long(long b, ulong bu, global ulong *res) {
+;   *res = bitfield_extract_unsigned(b, 3, 4);
+;   *res += bitfield_extract_unsigned(bu, 3, 4);
+; }
+
+define dso_local spir_kernel void @testExtractU_long(i64 noundef %b, i64 noundef %bu, ptr addrspace(1) nocapture noundef writeonly align 8 initializes((0, 8)) %res) {
+entry:
+  %call = tail call spir_func i64 @_Z25bitfield_extract_unsignedljj(i64 noundef %b, i32 noundef 3, i32 noundef 4) #2
+  %call1 = tail call spir_func i64 @_Z25bitfield_extract_unsignedmjj(i64 noundef %bu, i32 noundef 3, i32 noundef 4) #2
+  %add = add i64 %call1, %call
+  store i64 %add, ptr addrspace(1) %res, align 8, !tbaa !7
+  ret void
+}
+
+declare spir_func i64 @_Z25bitfield_extract_unsignedljj(i64 noundef, i32 noundef, i32 noundef) 
+
+declare spir_func i64 @_Z25bitfield_extract_unsignedmjj(i64 noundef, i32 noundef, i32 noundef) 
+
+; CHECK-EXTENSION: %[[#]] = OpFunction %[[#]] None %[[#]]
+; CHECK-EXTENSION: %[[#uextractbase:]] = OpFunctionParameter %[[#]]
+; CHECK-EXTENSION: %[[#uextractbaseu:]] = OpFunctionParameter %[[#]]
+; CHECK-EXTENSION: %[[#]] = OpBitFieldUExtract %[[#]] %[[#uextractbase]]
+; CHECK-EXTENSION: %[[#]] = OpBitFieldUExtract %[[#]] %[[#uextractbaseu]]
+; OpenCL equivalent.
+; kernel void testExtractU_int(int b, uint bu, global uint *res) {
+;   *res = bitfield_extract_unsigned(b, 3, 4);
+;   *res += bitfield_extract_unsigned(bu, 3, 4);
+; }
+
+define dso_local spir_kernel void @testExtractU_int(i32 noundef %b, i32 noundef %bu, ptr addrspace(1) nocapture noundef writeonly align 4 initializes((0, 4)) %res) {
+entry:
+  %call = tail call spir_func i32 @_Z25bitfield_extract_unsignedijj(i32 noundef %b, i32 noundef 3, i32 noundef 4) #2
+  %call1 = tail call spir_func i32 @_Z25bitfield_extract_unsignedjjj(i32 noundef %bu, i32 noundef 3, i32 noundef 4) #2
+  %add = add i32 %call1, %call
+  store i32 %add, ptr addrspace(1) %res, align 4, !tbaa !13
+  ret void
+}
+
+declare spir_func i32 @_Z25bitfield_extract_unsignedijj(i32 noundef, i32 noundef, i32 noundef) 
+
+declare spir_func i32 @_Z25bitfield_extract_unsignedjjj(i32 noundef, i32 noundef, i32 noundef) 
+
+; CHECK-EXTENSION: %[[#]] = OpFunction %[[#]] None %[[#]]
+; CHECK-EXTENSION: %[[#uextractbase:]] = OpFunctionParameter %[[#]]
+; CHECK-EXTENSION: %[[#uextractbaseu:]] = OpFunctionParameter %[[#]]
+; CHECK-EXTENSION: %[[#]] = OpBitFieldUExtract %[[#]] %[[#uextractbase]]
+; CHECK-EXTENSION: %[[#]] = OpBitFieldUExtract %[[#]] %[[#uextractbaseu]]
+; OpenCL equivalent.
+; kernel void testExtractU_short(short b, ushort bu, global ushort *res) {
+;   *res = bitfield_extract_unsigned(b, 3, 4);
+;   *res += bitfield_extract_unsigned(bu, 3, 4);
+; }
+
+define dso_local spir_kernel void @testExtractU_short(i16 noundef signext %b, i16 noundef zeroext %bu, ptr addrspace(1) nocapture noundef writeonly align 2 initializes((0, 2)) %res) {
+entry:
+  %call = tail call spir_func zeroext i16 @_Z25bitfield_extract_unsignedsjj(i16 noundef signext %b, i32 noundef 3, i32 noundef 4) #2
+  %call1 = tail call spir_func zeroext i16 @_Z25bitfield_extract_unsignedtjj(i16 noundef zeroext %bu, i32 noundef 3, i32 noundef 4) #2
+  %add = add i16 %call1, %call
+  store i16 %add, ptr addrspace(1) %res, align 2, !tbaa !17
+  ret void
+}
+
+declare spir_func zeroext i16 @_Z25bitfield_extract_unsignedsjj(i16 noundef signext, i32 noundef, i32 noundef) 
+
+declare spir_func zeroext i16 @_Z25bitfield_extract_unsignedtjj(i16 noundef zeroext, i32 noundef, i32 noundef) 
+
+; CHECK-EXTENSION: %[[#]] = OpFunction %[[#]] None %[[#]]
+; CHECK-EXTENSION: %[[#uextractbase:]] = OpFunctionParameter %[[#]]
+; CHECK-EXTENSION: %[[#uextractbaseu:]] = OpFunctionParameter %[[#]]
+; CHECK-EXTENSION: %[[#]] = OpBitFieldUExtract %[[#]] %[[#uextractbase]]
+; CHECK-EXTENSION: %[[#]] = OpBitFieldUExtract %[[#]] %[[#uextractbaseu]]
+; OpenCL equivalent.
+; kernel void testExtractU_char(char b, uchar bu, global uchar *res) {
+;   *res = bitfield_extract_unsigned(b, 3, 4);
+;   *res += bitfield_extract_unsigned(bu, 3, 4);
+; }
+
+define dso_local spir_kernel void @testExtractU_char(i8 noundef signext %b, i8 noundef zeroext %bu, ptr addrspace(1) nocapture noundef writeonly align 1 initializes((0, 1)) %res) {
+entry:
+  %call = tail call spir_func zeroext i8 @_Z25bitfield_extract_unsignedcjj(i8 noundef signext %b, i32 noundef 3, i32 noundef 4) #2
+  %call1 = tail call spir_func zeroext i8 @_Z25bitfield_extract_unsignedhjj(i8 noundef zeroext %bu, i32 noundef 3, i32 noundef 4) #2
+  %add = add i8 %call1, %call
+  store i8 %add, ptr addrspace(1) %res, align 1, !tbaa !22
+  ret void
+}
+
+declare spir_func zeroext i8 @_Z25bitfield_extract_unsignedcjj(i8 noundef signext, i32 noundef, i32 noundef) 
+
+declare spir_func zeroext i8 @_Z25bitfield_extract_unsignedhjj(i8 noundef zeroext, i32 noundef, i32 noundef) 
+
+; CHECK-EXTENSION: %[[#]] = OpFunction %[[#]] None %[[#]]
+; CHECK-EXTENSION: %[[#uextractbase:]] = OpFunctionParameter %[[#]]
+; CHECK-EXTENSION: %[[#uextractbaseu:]] = OpFunctionParameter %[[#]]
+; CHECK-EXTENSION: %[[#]] = OpBitFieldUExtract %[[#]] %[[#uextractbase]]
+; CHECK-EXTENSION: %[[#]] = OpBitFieldUExtract %[[#]] %[[#uextractbaseu]]
+; OpenCL equivalent.
+; kernel void testExtractU_long2(long2 b, ulong2 bu, global ulong2 *res) {
+;   *res = bitfield_extract_unsigned(b, 3, 4);
+;   *res += bitfield_extract_unsigned(bu, 3, 4);
+; }
+
+define dso_local spir_kernel void @testExtractU_long2(<2 x i64> noundef %b, <2 x i64> noundef %bu, ptr addrspace(1) nocapture noundef writeonly align 16 initializes((0, 16)) %res) {
+entry:
+  %call = tail call spir_func <2 x i64> @_Z25bitfield_extract_unsignedDv2_ljj(<2 x i64> noundef %b, i32 noundef 3, i32 noundef 4) #2
+  %call1 = tail call spir_func <2 x i64> @_Z25bitfield_extract_unsignedDv2_mjj(<2 x i64> noundef %bu, i32 noundef 3, i32 noundef 4) #2
+  %add = add <2 x i64> %call1, %call
+  store <2 x i64> %add, ptr addrspace(1) %res, align 16, !tbaa !22
+  ret void
+}
+
+declare spir_func <2 x i64> @_Z25bitfield_extract_unsignedDv2_ljj(<2 x i64> noundef, i32 noundef, i32 noundef) 
+
+declare spir_func <2 x i64> @_Z25bitfield_extract_unsignedDv2_mjj(<2 x i64> noundef, i32 noundef, i32 noundef) 
+
+; CHECK-EXTENSION: %[[#]] = OpFunction %[[#]] None %[[#]]
+; CHECK-EXTENSION: %[[#uextractbase:]] = OpFunctionParameter %[[#]]
+; CHECK-EXTENSION: %[[#uextractbaseu:]] = OpFunctionParameter %[[#]]
+; CHECK-EXTENSION: %[[#]] = OpBitFieldUExtract %[[#]] %[[#uextractbase]]
+; CHECK-EXTENSION: %[[#]] = OpBitFieldUExtract %[[#]] %[[#uextractbaseu]]
+; OpenCL equivalent.
+; kernel void testExtractU_int2(int2 b, uint2 bu, global uint2 *res) {
+;   *res = bitfield_extract_unsigned(b, 3, 4);
+;   *res += bitfield_extract_unsigned(bu, 3, 4);
+; }
+
+define dso_local spir_kernel void @testExtractU_int2(<2 x i32> noundef %b, <2 x i32> noundef %bu, ptr addrspace(1) nocapture noundef writeonly align 8 initializes((0, 8)) %res) {
+entry:
+  %call = tail call spir_func <2 x i32> @_Z25bitfield_extract_unsignedDv2_ijj(<2 x i32> noundef %b, i32 noundef 3, i32 noundef 4) #2
+  %call1 = tail call spir_func <2 x i32> @_Z25bitfield_extract_unsignedDv2_jjj(<2 x i32> noundef %bu, i32 noundef 3, i32 noundef 4) #2
+  %add = add <2 x i32> %call1, %call
+  store <2 x i32> %add, ptr addrspace(1) %res, align 8, !tbaa !22
+  ret void
+}
+
+declare spir_func <2 x i32> @_Z25bitfield_extract_unsignedDv2_ijj(<2 x i32> noundef, i32 noundef, i32 noundef) 
+
+declare spir_func <2 x i32> @_Z25bitfield_extract_unsignedDv2_jjj(<2 x i32> noundef, i32 noundef, i32 noundef) 
+
+; CHECK-EXTENSION: %[[#]] = OpFunction %[[#]] None %[[#]]
+; CHECK-EXTENSION: %[[#uextractbase:]] = OpFunctionParameter %[[#]]
+; CHECK-EXTENSION: %[[#uextractbaseu:]] = OpFunctionParameter %[[#]]
+; CHECK-EXTENSION: %[[#]] = OpBitFieldUExtract %[[#]] %[[#uextractbase]]
+; CHECK-EXTENSION: %[[#]] = OpBitFieldUExtract %[[#]] %[[#uextractbaseu]]
+; OpenCL equivalent.
+; kernel void testExtractU_short2(short2 b, ushort2 bu, global ushort2 *res) {
+;   *res = bitfield_extract_unsigned(b, 3, 4);
+;   *res += bitfield_extract_unsigned(bu, 3, 4);
+; }
+
+define dso_local spir_kernel void @testExtractU_short2(<2 x i16> noundef %b, <2 x i16> noundef %bu, ptr addrspace(1) nocapture noundef writeonly align 4 initializes((0, 4)) %res) {
+entry:
+  %call = tail call spir_func <2 x i16> @_Z25bitfield_extract_unsignedDv2_sjj(<2 x i16> noundef %b, i32 noundef 3, i32 noundef 4) #2
+  %call1 = tail call spir_func <2 x i16> @_Z25bitfield_extract_unsignedDv2_tjj(<2 x i16> noundef %bu, i32 noundef 3, i32 noundef 4) #2
+  %add = add <2 x i16> %call1, %call
+  store <2 x i16> %add, ptr addrspace(1) %res, align 4, !tbaa !22
+  ret void
+}
+
+declare spir_func <2 x i16> @_Z25bitfield_extract_unsignedDv2_sjj(<2 x i16> noundef, i32 noundef, i32 noundef) 
+
+declare spir_func <2 x i16> @_Z25bitfield_extract_unsignedDv2_tjj(<2 x i16> noundef, i32 noundef, i32 noundef) 
+
+; CHECK-EXTENSION: %[[#]] = OpFunction %[[#]] None %[[#]]
+; CHECK-EXTENSION: %[[#uextractbase:]] = OpFunctionParameter %[[#]]
+; CHECK-EXTENSION: %[[#uextractbaseu:]] = OpFunctionParameter %[[#]]
+; CHECK-EXTENSION: %[[#]] = OpBitFieldUExtract %[[#]] %[[#uextractbase]]
+; CHECK-EXTENSION: %[[#]] = OpBitFieldUExtract %[[#]] %[[#uextractbaseu]]
+; OpenCL equivalent.
+; kernel void testExtractU_char2(char2 b, uchar2 bu, global uchar2 *res) {
+;   *res = bitfield_extract_unsigned(b, 3, 4);
+;   *res += bitfield_extract_unsigned(bu, 3, 4);
+; }
+
+define dso_local spir_kernel void @testExtractU_char2(<2 x i8> noundef %b, <2 x i8> noundef %bu, ptr addrspace(1) nocapture noundef writeonly align 2 initializes((0, 2)) %res) {
+entry:
+  %call = tail call spir_func <2 x i8> @_Z25bitfield_extract_unsignedDv2_cjj(<2 x i8> noundef %b, i32 noundef 3, i32 noundef 4) #2
+  %call1 = tail call spir_func <2 x i8> @_Z25bitfield_extract_unsignedDv2_hjj(<2 x i8> noundef %bu, i32 noundef 3, i32 noundef 4) #2
+  %add = add <2 x i8> %call1, %call
+  store <2 x i8> %add, ptr addrspace(1) %res, align 2, !tbaa !22
+  ret void
+}
+
+declare spir_func <2 x i8> @_Z25bitfield_extract_unsignedDv2_cjj(<2 x i8> noundef, i32 noundef, i32 noundef) 
+
+declare spir_func <2 x i8> @_Z25bitfield_extract_unsignedDv2_hjj(<2 x i8> noundef, i32 noundef, i32 noundef) 
+
+; CHECK-EXTENSION: %[[#]] = OpFunction %[[#]] None %[[#]]
+; CHECK-EXTENSION: %[[#uextractbase:]] = OpFunctionParameter %[[#]]
+; CHECK-EXTENSION: %[[#uextractbaseu:]] = OpFunctionParameter %[[#]]
+; CHECK-EXTENSION: %[[#]] = OpBitFieldUExtract %[[#]] %[[#uextractbase]]
+; CHECK-EXTENSION: %[[#]] = OpBitFieldUExtract %[[#]] %[[#uextractbaseu]]
+; OpenCL equivalent.
+; kernel void testExtractU_long3(long3 b, ulong3 bu, global ulong3 *res) {
+;   *res = bitfield_extract_unsigned(b, 3, 4);
+;   *res += bitfield_extract_unsigned(bu, 3, 4);
+; }
+
+define dso_local spir_kernel void @testExtractU_long3(<3 x i64> noundef %b, <3 x i64> noundef %bu, ptr addrspace(1) nocapture noundef writeonly align 32 initializes((0, 32)) %res) {
+entry:
+  %call = tail call spir_func <3 x i64> @_Z25bitfield_extract_unsignedDv3_ljj(<3 x i64> noundef %b, i32 noundef 3, i32 noundef 4) #2
+  %call6 = tail call spir_func <3 x i64> @_Z25bitfield_extract_unsignedDv3_mjj(<3 x i64> noundef %bu, i32 noundef 3, i32 noundef 4) #2
+  %add = add <3 x i64> %call6, %call
+  %extractVec9 = shufflevector <3 x i64> %add, <3 x i64> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 poison>
+  store <4 x i64> %extractVec9, ptr addrspace(1) %res, align 32, !tbaa !22
+  ret void
+}
+
+declare spir_func <3 x i64> @_Z25bitfield_extract_unsignedDv3_ljj(<3 x i64> noundef, i32 noundef, i32 noundef) 
+
+declare spir_func <3 x i64> @_Z25bitfield_extract_unsignedDv3_mjj(<3 x i64> noundef, i32 noundef, i32 noundef) 
+
+; CHECK-EXTENSION: %[[#]] = OpFunction %[[#]] None %[[#]]
+; CHECK-EXTENSION: %[[#uextractbase:]] = OpFunctionParameter %[[#]]
+; CHECK-EXTENSION: %[[#uextractbaseu:]] = OpFunctionParameter %[[#]]
+; CHECK-EXTENSION: %[[#]] = OpBitFieldUExtract %[[#]] %[[#uextractbase]]
+; CHECK-EXTENSION: %[[#]] = OpBitFieldUExtract %[[#]] %[[#uextractbaseu]]
+; OpenCL equivalent.
+; kernel void testExtractU_int3(int3 b, uint3 bu, global uint3 *res) {
+;   *res = bitfield_extract_unsigned(b, 3, 4);
+;   *res += bitfield_extract_unsigned(bu, 3, 4);
+; }
+
+define dso_local spir_kernel void @testExtractU_int3(<3 x i32> noundef %b, <3 x i32> noundef %bu, ptr addrspace(1) nocapture noundef writeonly align 16 initializes((0, 16)) %res) {
+entry:
+  %call = tail call spir_func <3 x i32> @_Z25bitfield_extract_unsignedDv3_ijj(<3 x i32> noundef %b, i32 noundef 3, i32 noundef 4) #2
+  %call6 = tail call spir_func <3 x i32> @_Z25bitfield_extract_unsignedDv3_jjj(<3 x i32> noundef %bu, i32 noundef 3, i32 noundef 4) #2
+  %add = add <3 x i32> %call6, %call
+  %extractVec9 = shufflevector <3 x i32> %add, <3 x i32> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 poison>
+  store <4 x i32> %extractVec9, ptr addrspace(1) %res, align 16, !tbaa !22
+  ret void
+}
+
+declare spir_func <3 x i32> @_Z25bitfield_extract_unsignedDv3_ijj(<3 x i32> noundef, i32 noundef, i32 noundef) 
+
+declare spir_func <3 x i32> @_Z25bitfield_extract_unsignedDv3_jjj(<3 x i32> noundef, i32 noundef, i32 noundef) 
+
+; CHECK-EXTENSION: %[[#]] = OpFunction %[[#]] None %[[#]]
+; CHECK-EXTENSION: %[[#uextractbase:]] = OpFunctionParameter %[[#]]
+; CHECK-EXTENSION: %[[#uextractbaseu:]] = OpFunctionParameter %[[#]]
+; CHECK-EXTENSION: %[[#]] = OpBitFieldUExtract %[[#]] %[[#uextractbase]]
+; CHECK-EXTENSION: %[[#]] = OpBitFieldUExtract %[[#]] %[[#uextractbaseu]]
+; OpenCL equivalent.
+; kernel void testExtractU_short3(short3 b, ushort3 bu, global ushort3 *res) {
+;   *res = bitfield_extract_unsigned(b, 3, 4);
+;   *res += bitfield_extract_unsigned(bu, 3, 4);
+; }
+
+define dso_local spir_kernel void @testExtractU_short3(<3 x i16> noundef %b, <3 x i16> noundef %bu, ptr addrspace(1) nocapture noundef writeonly align 8 initializes((0, 8)) %res) {
+entry:
+  %call = tail call spir_func <3 x i16> @_Z25bitfield_extract_unsignedDv3_sjj(<3 x i16> noundef %b, i32 noundef 3, i32 noundef 4) #2
+  %call6 = tail call spir_func <3 x i16> @_Z25bitfield_extract_unsignedDv3_tjj(<3 x i16> noundef %bu, i32 noundef 3, i32 noundef 4) #2
+  %add = add <3 x i16> %call6, %call
+  %extractVec9 = shufflevector <3 x i16> %add, <3 x i16> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 poison>
+  store <4 x i16> %extractVec9, ptr addrspace(1) %res, align 8, !tbaa !22
+  ret void
+}
+
+declare spir_func <3 x i16> @_Z25bitfield_extract_unsignedDv3_sjj(<3 x i16> noundef, i32 noundef, i32 noundef) 
+
+declare spir_func <3 x i16> @_Z25bitfield_extract_unsignedDv3_tjj(<3 x i16> noundef, i32 noundef, i32 noundef) 
+
+; CHECK-EXTENSION: %[[#]] = OpFunction %[[#]] None %[[#]]
+; CHECK-EXTENSION: %[[#uextractbase:]] = OpFunctionParameter %[[#]]
+; CHECK-EXTENSION: %[[#uextractbaseu:]] = OpFunctionParameter %[[#]]
+; CHECK-EXTENSION: %[[#]] = OpBitFieldUExtract %[[#]] %[[#uextractbase]]
+; CHECK-EXTENSION: %[[#]] = OpBitFieldUExtract %[[#]] %[[#uextractbaseu]]
+; OpenCL equivalent.
+; kernel void testExtractU_char3(char3 b, uchar3 bu, global uchar3 *res) {
+;   *res = bitfield_extract_unsigned(b, 3, 4);
+;   *res += bitfield_extract_unsigned(bu, 3, 4);
+; }
+
+define dso_local spir_kernel void @testExtractU_char3(<3 x i8> noundef %b, <3 x i8> noundef %bu, ptr addrspace(1) nocapture noundef writeonly align 4 initializes((0, 4)) %res) {
+entry:
+  %call = tail call spir_func <3 x i8> @_Z25bitfield_extract_unsignedDv3_cjj(<3 x i8> noundef %b, i32 noundef 3, i32 noundef 4) #2
+  %call6 = tail call spir_func <3 x i8> @_Z25bitfield_extract_unsignedDv3_hjj(<3 x i8> noundef %bu, i32 noundef 3, i32 noundef 4) #2
+  %add = add <3 x i8> %call6, %call
+  %extractVec9 = shufflevector <3 x i8> %add, <3 x i8> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 poison>
+  store <4 x i8> %extractVec9, ptr addrspace(1) %res, align 4, !tbaa !22
+  ret void
+}
+
+declare spir_func <3 x i8> @_Z25bitfield_extract_unsignedDv3_cjj(<3 x i8> noundef, i32 noundef, i32 noundef) 
+
+declare spir_func <3 x i8> @_Z25bitfield_extract_unsignedDv3_hjj(<3 x i8> noundef, i32 noundef, i32 noundef) 
+
+; CHECK-EXTENSION: %[[#]] = OpFunction %[[#]] None %[[#]]
+; CHECK-EXTENSION: %[[#uextractbase:]] = OpFunctionParameter %[[#]]
+; CHECK-EXTENSION: %[[#uextractbaseu:]] = OpFunctionParameter %[[#]]
+; CHECK-EXTENSION: %[[#]] = OpBitFieldUExtract %[[#]] %[[#uextractbase]]
+; CHECK-EXTENSION: %[[#]] = OpBitFieldUExtract %[[#]] %[[#uextractbaseu]]
+; OpenCL equivalent.
+; kernel void testExtractU_long4(long4 b, ulong4 bu, global ulong4 *res) {
+;   *res = bitfield_extract_unsigned(b, 3, 4);
+;   *res += bitfield_extract_unsigned(bu, 3, 4);
+; }
+
+define dso_local spir_kernel void @testExtractU_long4(<4 x i64> noundef %b, <4 x i64> noundef %bu, ptr addrspace(1) nocapture noundef writeonly align 32 initializes((0, 32)) %res) {
+entry:
+  %call = tail call spir_func <4 x i64> @_Z25bitfield_extract_unsignedDv4_ljj(<4 x i64> noundef %b, i32 noundef 3, i32 noundef 4) #2
+  %call1 = tail call spir_func <4 x i64> @_Z25bitfield_extract_unsignedDv4_mjj(<4 x i64> noundef %bu, i32 noundef 3, i32 noundef 4) #2
+  %add = add <4 x i64> %call1, %call
+  store <4 x i64> %add, ptr addrspace(1) %res, align 32, !tbaa !22
+  ret void
+}
+
+declare spir_func <4 x i64> @_Z25bitfield_extract_unsignedDv4_ljj(<4 x i64> noundef, i32 noundef, i32 noundef) 
+
+declare spir_func <4 x i64> @_Z25bitfield_extract_unsignedDv4_mjj(<4 x i64> noundef, i32 noundef, i32 noundef) 
+
+; CHECK-EXTENSION: %[[#]] = OpFunction %[[#]] None %[[#]]
+; CHECK-EXTENSION: %[[#uextractbase:]] = OpFunctionParameter %[[#]]
+; CHECK-EXTENSION: %[[#uextractbaseu:]] = OpFunctionParameter %[[#]]
+; CHECK-EXTENSION: %[[#]] = OpBitFieldUExtract %[[#]] %[[#uextractbase]]
+; CHECK-EXTENSION: %[[#]] = OpBitFieldUExtract %[[#]] %[[#uextractbaseu]]
+; OpenCL equivalent.
+; kernel void testExtractU_int4(int4 b, uint4 bu, global uint4 *res) {
+;   *res = bitfield_extract_unsigned(b, 3, 4);
+;   *res += bitfield_extract_unsigned(bu, 3, 4);
+; }
+
+define dso_local spir_kernel void @testExtractU_int4(<4 x i32> noundef %b, <4 x i32> noundef %bu, ptr addrspace(1) nocapture noundef writeonly align 16 initializes((0, 16)) %res) {
+entry:
+  %call = tail call spir_func <4 x i32> @_Z25bitfield_extract_unsignedDv4_ijj(<4 x i32> noundef %b, i32 noundef 3, i32 noundef 4) #2
+  %call1 = tail call spir_func <4 x i32> @_Z25bitfield_extract_unsignedDv4_jjj(<4 x i32> noundef %bu, i32 noundef 3, i32 noundef 4) #2
+  %add = add <4 x i32> %call1, %call
+  store <4 x i32> %add, ptr addrspace(1) %res, align 16, !tbaa !22
+  ret void
+}
+
+declare spir_func <4 x i32> @_Z25bitfield_extract_unsignedDv4_ijj(<4 x i32> noundef, i32 noundef, i32 noundef) 
+
+declare spir_func <4 x i32> @_Z25bitfield_extract_unsignedDv4_jjj(<4 x i32> noundef, i32 noundef, i32 noundef) 
+
+; CHECK-EXTENSION: %[[#]] = OpFunction %[[#]] None %[[#]]
+; CHECK-EXTENSION: %[[#uextractbase:]] = OpFunctionParameter %[[#]]
+; CHECK-EXTENSION: %[[#uextractbaseu:]] = OpFunctionParameter %[[#]]
+; CHECK-EXTENSION: %[[#]] = OpBitFieldUExtract %[[#]] %[[#uextractbase]]
+; CHECK-EXTENSION: %[[#]] = OpBitFieldUExtract %[[#]] %[[#uextractbaseu]]
+; OpenCL equivalent.
+; kernel void testExtractU_short4(short4 b, ushort4 bu, global ushort4 *res) {
+;   *res = bitfield_extract_unsigned(b, 3, 4);
+;   *res += bitfield_extract_unsigned(bu, 3, 4);
+; }
+
+define dso_local spir_kernel void @testExtractU_short4(<4 x i16> noundef %b, <4 x i16> noundef %bu, ptr addrspace(1) nocapture noundef writeonly align 8 initializes((0, 8)) %res) {
+entry:
+  %call = tail call spir_func <4 x i16> @_Z25bitfield_extract_unsignedDv4_sjj(<4 x i16> noundef %b, i32 noundef 3, i32 noundef 4) #2
+  %call1 = tail call spir_func <4 x i16> @_Z25bitfield_extract_unsignedDv4_tjj(<4 x i16> noundef %bu, i32 noundef 3, i32 noundef 4) #2
+  %add = add <4 x i16> %call1, %call
+  store <4 x i16> %add, ptr addrspace(1) %res, align 8, !tbaa !22
+  ret void
+}
+
+declare spir_func <4 x i16> @_Z25bitfield_extract_unsignedDv4_sjj(<4 x i16> noundef, i32 noundef, i32 noundef) 
+
+declare spir_func <4 x i16> @_Z25bitfield_extract_unsignedDv4_tjj(<4 x i16> noundef, i32 noundef, i32 noundef) 
+
+; CHECK-EXTENSION: %[[#]] = OpFunction %[[#]] None %[[#]]
+; CHECK-EXTENSION: %[[#uextractbase:]] = OpFunctionParameter %[[#]]
+; CHECK-EXTENSION: %[[#uextractbaseu:]] = OpFunctionParameter %[[#]]
+; CHECK-EXTENSION: %[[#]] = OpBitFieldUExtract %[[#]] %[[#uextractbase]]
+; CHECK-EXTENSION: %[[#]] = OpBitFieldUExtract %[[#]] %[[#uextractbaseu]]
+; OpenCL equivalent.
+; kernel void testExtractU_char4(char4 b, uchar4 bu, global uchar4 *res) {
+;   *res = bitfield_extract_unsigned(b, 3, 4);
+;   *res += bitfield_extract_unsigned(bu, 3, 4);
+; }
+
+define dso_local spir_kernel void @testExtractU_char4(<4 x i8> noundef %b, <4 x i8> noundef %bu, ptr addrspace(1) nocapture noundef writeonly align 4 initializes((0, 4)) %res) {
+entry:
+  %call = tail call spir_func <4 x i8> @_Z25bitfield_extract_unsignedDv4_cjj(<4 x i8> noundef %b, i32 noundef 3, i32 noundef 4) #2
+  %call1 = tail call spir_func <4 x i8> @_Z25bitfield_extract_unsignedDv4_hjj(<4 x i8> noundef %bu, i32 noundef 3, i32 noundef 4) #2
+  %add = add <4 x i8> %call1, %call
+  store <4 x i8> %add, ptr addrspace(1) %res, align 4, !tbaa !22
+  ret void
+}
+
+declare spir_func <4 x i8> @_Z25bitfield_extract_unsignedDv4_cjj(<4 x i8> noundef, i32 noundef, i32 noundef) 
+
+declare spir_func <4 x i8> @_Z25bitfield_extract_unsignedDv4_hjj(<4 x i8> noundef, i32 noundef, i32 noundef) 
+
+; CHECK-EXTENSION: %[[#]] = OpFunction %[[#]] None %[[#]]
+; CHECK-EXTENSION: %[[#uextractbase:]] = OpFunctionParameter %[[#]]
+; CHECK-EXTENSION: %[[#uextractbaseu:]] = OpFunctionParameter %[[#]]
+; CHECK-EXTENSION: %[[#]] = OpBitFieldUExtract %[[#]] %[[#uextractbase]]
+; CHECK-EXTENSION: %[[#]] = OpBitFieldUExtract %[[#]] %[[#uextractbaseu]]
+; OpenCL equivalent.
+; kernel void testExtractU_long8(long8 b, ulong8 bu, global ulong8 *res) {
+;   *res = bitfield_extract_unsigned(b, 3, 4);
+;   *res += bitfield_extract_unsigned(bu, 3, 4);
+; }
+
+define dso_local spir_kernel void @testExtractU_long8(<8 x i64> noundef %b, <8 x i64> noundef %bu, ptr addrspace(1) nocapture noundef writeonly align 64 initializes((0, 64)) %res) {
+entry:
+  %call = tail call spir_func <8 x i64> @_Z25bitfield_extract_unsignedDv8_ljj(<8 x i64> noundef %b, i32 noundef 3, i32 noundef 4) #2
+  %call1 = tail call spir_func <8 x i64> @_Z25bitfield_extract_unsignedDv8_mjj(<8 x i64> noundef %bu, i32 noundef 3, i32 noundef 4) #2
+  %add = add <8 x i64> %call1, %call
+  store <8 x i64> %add, ptr addrspace(1) %res, align 64, !tbaa !22
+  ret void
+}
+
+declare spir_func <8 x i64> @_Z25bitfield_extract_unsignedDv8_ljj(<8 x i64> noundef, i32 noundef, i32 noundef) 
+
+declare spir_func <8 x i64> @_Z25bitfield_extract_unsignedDv8_mjj(<8 x i64> noundef, i32 noundef, i32 noundef) 
+
+; CHECK-EXTENSION: %[[#]] = OpFunction %[[#]] None %[[#]]
+; CHECK-EXTENSION: %[[#uextractbase:]] = OpFunctionParameter %[[#]]
+; CHECK-EXTENSION: %[[#uextractbaseu:]] = OpFunctionParameter %[[#]]
+; CHECK-EXTENSION: %[[#]] = OpBitFieldUExtract %[[#]] %[[#uextractbase]]
+; CHECK-EXTENSION: %[[#]] = OpBitFieldUExtract %[[#]] %[[#uextractbaseu]]
+; OpenCL equivalent.
+; kernel void testExtractU_int8(int8 b, uint8 bu, global uint8 *res) {
+;   *res = bitfield_extract_unsigned(b, 3, 4);
+;   *res += bitfield_extract_unsigned(bu, 3, 4);
+; }
+
+define dso_local spir_kernel void @testExtractU_int8(<8 x i32> noundef %b, <8 x i32> noundef %bu, ptr addrspace(1) nocapture noundef writeonly align 32 initializes((0, 32)) %res) {
+entry:
+  %call = tail call spir_func <8 x i32> @_Z25bitfield_extract_unsignedDv8_ijj(<8 x i32> noundef %b, i32 noundef 3, i32 noundef 4) #2
+  %call1 = tail call spir_func <8 x i32> @_Z25bitfield_extract_unsignedDv8_jjj(<8 x i32> noundef %bu, i32 noundef 3, i32 noundef 4) #2
+  %add = add <8 x i32> %call1, %call
+  store <8 x i32> %add, ptr addrspace(1) %res, align 32, !tbaa !22
+  ret void
+}
+
+declare spir_func <8 x i32> @_Z25bitfield_extract_unsignedDv8_ijj(<8 x i32> noundef, i32 noundef, i32 noundef) 
+
+declare spir_func <8 x i32> @_Z25bitfield_extract_unsignedDv8_jjj(<8 x i32> noundef, i32 noundef, i32 noundef) 
+
+; CHECK-EXTENSION: %[[#]] = OpFunction %[[#]] None %[[#]]
+; CHECK-EXTENSION: %[[#uextractbase:]] = OpFunctionParameter %[[#]]
+; CHECK-EXTENSION: %[[#uextractbaseu:]] = OpFunctionParameter %[[#]]
+; CHECK-EXTENSION: %[[#]] = OpBitFieldUExtract %[[#]] %[[#uextractbase]]
+; CHECK-EXTENSION: %[[#]] = OpBitFieldUExtract %[[#]] %[[#uextractbaseu]]
+; OpenCL equivalent.
+; kernel void testExtractU_short8(short8 b, ushort8 bu, global ushort8 *res) {
+;   *res = bitfield_extract_unsigned(b, 3, 4);
+;   *res += bitfield_extract_unsigned(bu, 3, 4);
+; }
+
+define dso_local spir_kernel void @testExtractU_short8(<8 x i16> noundef %b, <8 x i16> noundef %bu, ptr addrspace(1) nocapture noundef writeonly align 16 initializes((0, 16)) %res) {
+entry:
+  %call = tail call spir_func <8 x i16> @_Z25bitfield_extract_unsignedDv8_sjj(<8 x i16> noundef %b, i32 noundef 3, i32 noundef 4) #2
+  %call1 = tail call spir_func <8 x i16> @_Z25bitfield_extract_unsignedDv8_tjj(<8 x i16> noundef %bu, i32 noundef 3, i32 noundef 4) #2
+  %add = add <8 x i16> %call1, %call
+  store <8 x i16> %add, ptr addrspace(1) %res, align 16, !tbaa !22
+  ret void
+}
+
+declare spir_func <8 x i16> @_Z25bitfield_extract_unsignedDv8_sjj(<8 x i16> noundef, i32 noundef, i32 noundef) 
+
+declare spir_func <8 x i16> @_Z25bitfield_extract_unsignedDv8_tjj(<8 x i16> noundef, i32 noundef, i32 noundef) 
+
+; CHECK-EXTENSION: %[[#]] = OpFunction %[[#]] None %[[#]]
+; CHECK-EXTENSION: %[[#uextractbase:]] = OpFunctionParameter %[[#]]
+; CHECK-EXTENSION: %[[#uextractbaseu:]] = OpFunctionParameter %[[#]]
+; CHECK-EXTENSION: %[[#]] = OpBitFieldUExtract %[[#]] %[[#uextractbase]]
+; CHECK-EXTENSION: %[[#]] = OpBitFieldUExtract %[[#]] %[[#uextractbaseu]]
+; OpenCL equivalent.
+; kernel void testExtractU_char8(char8 b, uchar8 bu, global uchar8 *res) {
+;   *res = bitfield_extract_unsigned(b, 3, 4);
+;   *res += bitfield_extract_unsigned(bu, 3, 4);
+; }
+
+define dso_local spir_kernel void @testExtractU_char8(<8 x i8> noundef %b, <8 x i8> noundef %bu, ptr addrspace(1) nocapture noundef writeonly align 8 initializes((0, 8)) %res) {
+entry:
+  %call = tail call spir_func <8 x i8> @_Z25bitfield_extract_unsignedDv8_cjj(<8 x i8> noundef %b, i32 noundef 3, i32 noundef 4) #2
+  %call1 = tail call spir_func <8 x i8> @_Z25bitfield_extract_unsignedDv8_hjj(<8 x i8> noundef %bu, i32 noundef 3, i32 noundef 4) #2
+  %add = add <8 x i8> %call1, %call
+  store <8 x i8> %add, ptr addrspace(1) %res, align 8, !tbaa !22
+  ret void
+}
+
+declare spir_func <8 x i8> @_Z25bitfield_extract_unsignedDv8_cjj(<8 x i8> noundef, i32 noundef, i32 noundef) 
+
+declare spir_func <8 x i8> @_Z25bitfield_extract_unsignedDv8_hjj(<8 x i8> noundef, i32 noundef, i32 noundef) 
+
+; CHECK-EXTENSION: %[[#]] = OpFunction %[[#]] None %[[#]]
+; CHECK-EXTENSION: %[[#uextractbase:]] = OpFunctionParameter %[[#]]
+; CHECK-EXTENSION: %[[#uextractbaseu:]] = OpFunctionParameter %[[#]]
+; CHECK-EXTENSION: %[[#]] = OpBitFieldUExtract %[[#]] %[[#uextractbase]]
+; CHECK-EXTENSION: %[[#]] = OpBitFieldUExtract %[[#]] %[[#uextractbaseu]]
+; OpenCL equivalent.
+; kernel void testExtractU_long16(long16 b, ulong16 bu, global ulong16 *res) {
+;   *res = bitfield_extract_unsigned(b, 3, 4);
+;   *res += bitfield_extract_unsigned(bu, 3, 4);
+; }
+
+define dso_local spir_kernel void @testExtractU_long16(<16 x i64> noundef %b, <16 x i64> noundef %bu, ptr addrspace(1) nocapture noundef writeonly align 128 initializes((0, 128)) %res) {
+entry:
+  %call = tail call spir_func <16 x i64> @_Z25bitfield_extract_unsignedDv16_ljj(<16 x i64> noundef %b, i32 noundef 3, i32 noundef 4) #2
+  %call1 = tail call spir_func <16 x i64> @_Z25bitfield_extract_unsignedDv16_mjj(<16 x i64> noundef %bu, i32 noundef 3, i32 noundef 4) #2
+  %add = add <16 x i64> %call1, %call
+  store <16 x i64> %add, ptr addrspace(1) %res, align 128, !tbaa !22
+  ret void
+}
+
+declare spir_func <16 x i64> @_Z25bitfield_extract_unsignedDv16_ljj(<16 x i64> noundef, i32 noundef, i32 noundef) 
+
+declare spir_func <16 x i64> @_Z25bitfield_extract_unsignedDv16_mjj(<16 x i64> noundef, i32 noundef, i32 noundef) 
+
+; CHECK-EXTENSION: %[[#]] = OpFunction %[[#]] None %[[#]]
+; CHECK-EXTENSION: %[[#uextractbase:]] = OpFunctionParameter %[[#]]
+; CHECK-EXTENSION: %[[#uextractbaseu:]] = OpFunctionParameter %[[#]]
+; CHECK-EXTENSION: %[[#]] = OpBitFieldUExtract %[[#]] %[[#uextractbase]]
+; CHECK-EXTENSION: %[[#]] = OpBitFieldUExtract %[[#]] %[[#uextractbaseu]]
+; OpenCL equivalent.
+; kernel void testExtractU_int16(int16 b, uint16 bu, global uint16 *res) {
+;   *res = bitfield_extract_unsigned(b, 3, 4);
+;   *res += bitfield_extract_unsigned(bu, 3, 4);
+; }
+
+define dso_local spir_kernel void @testExtractU_int16(<16 x i32> noundef %b, <16 x i32> noundef %bu, ptr addrspace(1) nocapture noundef writeonly align 64 initializes((0, 64)) %res) {
+entry:
+  %call = tail call spir_func <16 x i32> @_Z25bitfield_extract_unsignedDv16_ijj(<16 x i32> noundef %b, i32 noundef 3, i32 noundef 4) #2
+  %call1 = tail call spir_func <16 x i32> @_Z25bitfield_extract_unsignedDv16_jjj(<16 x i32> noundef %bu, i32 noundef 3, i32 noundef 4) #2
+  %add = add <16 x i32> %call1, %call
+  store <16 x i32> %add, ptr addrspace(1) %res, align 64, !tbaa !22
+  ret void
+}
+
+declare spir_func <16 x i32> @_Z25bitfield_extract_unsignedDv16_ijj(<16 x i32> noundef, i32 noundef, i32 noundef) 
+
+declare spir_func <16 x i32> @_Z25bitfield_extract_unsignedDv16_jjj(<16 x i32> noundef, i32 noundef, i32 noundef) 
+
+; CHECK-EXTENSION: %[[#]] = OpFunction %[[#]] None %[[#]]
+; CHECK-EXTENSION: %[[#uextractbase:]] = OpFunctionParameter %[[#]]
+; CHECK-EXTENSION: %[[#uextractbaseu:]] = OpFunctionParameter %[[#]]
+; CHECK-EXTENSION: %[[#]] = OpBitFieldUExtract %[[#]] %[[#uextractbase]]
+; CHECK-EXTENSION: %[[#]] = OpBitFieldUExtract %[[#]] %[[#uextractbaseu]]
+; OpenCL equivalent.
+; kernel void testExtractU_short16(short16 b, ushort16 bu, global ushort16 *res) {
+;   *res = bitfield_extract_unsigned(b, 3, 4);
+;   *res += bitfield_extract_unsigned(bu, 3, 4);
+; }
+
+define dso_local spir_kernel void @testExtractU_short16(<16 x i16> noundef %b, <16 x i16> noundef %bu, ptr addrspace(1) nocapture noundef writeonly align 32 initializes((0, 32)) %res) {
+entry:
+  %call = tail call spir_func <16 x i16> @_Z25bitfield_extract_unsignedDv16_sjj(<16 x i16> noundef %b, i32 noundef 3, i32 noundef 4) #2
+  %call1 = tail call spir_func <16 x i16> @_Z25bitfield_extract_unsignedDv16_tjj(<16 x i16> noundef %bu, i32 noundef 3, i32 noundef 4) #2
+  %add = add <16 x i16> %call1, %call
+  store <16 x i16> %add, ptr addrspace(1) %res, align 32, !tbaa !22
+  ret void
+}
+
+declare spir_func <16 x i16> @_Z25bitfield_extract_unsignedDv16_sjj(<16 x i16> noundef, i32 noundef, i32 noundef) 
+
+declare spir_func <16 x i16> @_Z25bitfield_extract_unsignedDv16_tjj(<16 x i16> noundef, i32 noundef, i32 noundef) 
+
+; CHECK-EXTENSION: %[[#]] = OpFunction %[[#]] None %[[#]]
+; CHECK-EXTENSION: %[[#uextractbase:]] = OpFunctionParameter %[[#]]
+; CHECK-EXTENSION: %[[#uextractbaseu:]] = OpFunctionParameter %[[#]]
+; CHECK-EXTENSION: %[[#]] = OpBitFieldUExtract %[[#]] %[[#uextractbase]]
+; CHECK-EXTENSION: %[[#]] = OpBitFieldUExtract %[[#]] %[[#uextractbaseu]]
+; OpenCL equivalent.
+; kernel void testExtractU_char16(char16 b, uchar16 bu, global uchar16 *res) {
+;   *res = bitfield_extract_unsigned(b, 3, 4);
+;   *res += bitfield_extract_unsigned(bu, 3, 4);
+; }
+
+define dso_local spir_kernel void @testExtractU_char16(<16 x i8> noundef %b, <16 x i8> noundef %bu, ptr addrspace(1) nocapture noundef writeonly align 16 initializes((0, 16)) %res) {
+entry:
+  %call = tail call spir_func <16 x i8> @_Z25bitfield_extract_unsignedDv16_cjj(<16 x i8> noundef %b, i32 noundef 3, i32 noundef 4) #2
+  %call1 = tail call spir_func <16 x i8> @_Z25bitfield_extract_unsignedDv16_hjj(<16 x i8> noundef %bu, i32 noundef 3, i32 noundef 4) #2
+  %add = add <16 x i8> %call1, %call
+  store <16 x i8> %add, ptr addrspace(1) %res, align 16, !tbaa !22
+  ret void
+}
+
+declare spir_func <16 x i8> @_Z25bitfield_extract_unsignedDv16_cjj(<16 x i8> noundef, i32 noundef, i32 noundef) 
+
+declare spir_func <16 x i8> @_Z25bitfield_extract_unsignedDv16_hjj(<16 x i8> noundef, i32 noundef, i32 noundef) 
+
+; CHECK-EXTENSION: %[[#]] = OpFunction %[[#]] None %[[#]]
+; CHECK-EXTENSION: %[[#reversebase:]] = OpFunctionParameter %[[#]]
+; CHECK-EXTENSION: %[[#]] = OpBitReverse %[[#]] %[[#reversebase]]
+; OpenCL equivalent.
+; kernel void testBitReverse_long(long b, global long *res) {
+;   *res = bit_reverse(b);
+; }
+
+define dso_local spir_kernel void @testBitReverse_long(i64 noundef %b, ptr addrspace(1) nocapture noundef writeonly align 8 initializes((0, 8)) %res) {
+entry:
+  %call = tail call spir_func i64 @_Z11bit_reversel(i64 noundef %b) #2
+  store i64 %call, ptr addrspace(1) %res, align 8, !tbaa !7
+  ret void
+}
+
+declare spir_func i64 @_Z11bit_reversel(i64 noundef) 
+
+; CHECK-EXTENSION: %[[#]] = OpFunction %[[#]] None %[[#]]
+; CHECK-EXTENSION: %[[#reversebase:]] = OpFunctionParameter %[[#]]
+; CHECK-EXTENSION: %[[#]] = OpBitReverse %[[#]] %[[#reversebase]]
+; OpenCL equivalent.
+; kernel void testBitReverse_ulong(ulong b, global ulong *res) {
+;   *res = bit_reverse(b);
+; }
+
+define dso_local spir_kernel void @testBitReverse_ulong(i64 noundef %b, ptr addrspace(1) nocapture noundef writeonly align 8 initializes((0, 8)) %res) {
+entry:
+  %call = tail call spir_func i64 @_Z11bit_reversem(i64 noundef %b) #2
+  store i64 %call, ptr addrspace(1) %res, align 8, !tbaa !7
+  ret void
+}
+
+declare spir_func i64 @_Z11bit_reversem(i64 noundef) 
+
+; CHECK-EXTENSION: %[[#]] = OpFunction %[[#]] None %[[#]]
+; CHECK-EXTENSION: %[[#reversebase:]] = OpFunctionParameter %[[#]]
+; CHECK-EXTENSION: %[[#]] = OpBitReverse %[[#]] %[[#reversebase]]
+; OpenCL equivalent.
+; kernel void testBitReverse_int(int b, global int *res) {
+;   *res = bit_reverse(b);
+; }
+
+define dso_local spir_kernel void @testBitReverse_int(i32 noundef %b, ptr addrspace(1) nocapture noundef writeonly align 4 initializes((0, 4)) %res) {
+entry:
+  %call = tail call spir_func i32 @_Z11bit_reversei(i32 noundef %b) #2
+  store i32 %call, ptr addrspace(1) %res, align 4, !tbaa !13
+  ret void
+}
+
+declare spir_func i32 @_Z11bit_reversei(i32 noundef) 
+
+; CHECK-EXTENSION: %[[#]] = OpFunction %[[#]] None %[[#]]
+; CHECK-EXTENSION: %[[#reversebase:]] = OpFunctionParameter %[[#]]
+; CHECK-EXTENSION: %[[#]] = OpBitReverse %[[#]] %[[#reversebase]]
+; OpenCL equivalent.
+; kernel void testBitReverse_uint(uint b, global uint *res) {
+;   *res = bit_reverse(b);
+; }
+
+define dso_local spir_kernel void @testBitReverse_uint(i32 noundef %b, ptr addrspace(1) nocapture noundef writeonly align 4 initializes((0, 4)) %res) {
+entry:
+  %call = tail call spir_func i32 @_Z11bit_reversej(i32 noundef %b) #2
+  store i32 %call, ptr addrspace(1) %res, align 4, !tbaa !13
+  ret void
+}
+
+declare spir_func i32 @_Z11bit_reversej(i32 noundef) 
+
+; CHECK-EXTENSION: %[[#]] = OpFunction %[[#]] None %[[#]]
+; CHECK-EXTENSION: %[[#reversebase:]] = OpFunctionParameter %[[#]]
+; CHECK-EXTENSION: %[[#]] = OpBitReverse %[[#]] %[[#reversebase]]
+; OpenCL equivalent.
+; kernel void testBitReverse_short(short b, global short *res) {
+;   *res = bit_reverse(b);
+; }
+
+define dso_local spir_kernel void @testBitReverse_short(i16 noundef signext %b, ptr addrspace(1) nocapture noundef writeonly align 2 initializes((0, 2)) %res) {
+entry:
+  %call = tail call spir_func signext i16 @_Z11bit_reverses(i16 noundef signext %b) #2
+  store i16 %call, ptr addrspace(1) %res, align 2, !tbaa !17
+  ret void
+}
+
+declare spir_func signext i16 @_Z11bit_reverses(i16 noundef signext) 
+
+; CHECK-EXTENSION: %[[#]] = OpFunction %[[#]] None %[[#]]
+; CHECK-EXTENSION: %[[#reversebase:]] = OpFunctionParameter %[[#]]
+; CHECK-EXTENSION: %[[#]] = OpBitReverse %[[#]] %[[#reversebase]]
+; OpenCL equivalent.
+; kernel void testBitReverse_ushort(ushort b, global ushort *res) {
+;   *res = bit_reverse(b);
+; }
+
+define dso_local spir_kernel void @testBitReverse_ushort(i16 noundef zeroext %b, ptr addrspace(1) nocapture noundef writeonly align 2 initializes((0, 2)) %res) {
+entry:
+  %call = tail call spir_func zeroext i16 @_Z11bit_reverset(i16 noundef zeroext %b) #2
+  store i16 %call, ptr addrspace(1) %res, align 2, !tbaa !17
+  ret void
+}
+
+declare spir_func zeroext i16 @_Z11bit_reverset(i16 noundef zeroext) 
+
+; CHECK-EXTENSION: %[[#]] = OpFunction %[[#]] None %[[#]]
+; CHECK-EXTENSION: %[[#reversebase:]] = OpFunctionParameter %[[#]]
+; CHECK-EXTENSION: %[[#]] = OpBitReverse %[[#]] %[[#reversebase]]
+; OpenCL equivalent.
+; kernel void testBitReverse_char(char b, global char *res) {
+;   *res = bit_reverse(b);
+; }
+
+define dso_local spir_kernel void @testBitReverse_char(i8 noundef signext %b, ptr addrspace(1) nocapture noundef writeonly align 1 initializes((0, 1)) %res) {
+entry:
+  %call = tail call spir_func signext i8 @_Z11bit_reversec(i8 noundef signext %b) #2
+  store i8 %call, ptr addrspace(1) %res, align 1, !tbaa !22
+  ret void
+}
+
+declare spir_func signext i8 @_Z11bit_reversec(i8 noundef signext) 
+
+; CHECK-EXTENSION: %[[#]] = OpFunction %[[#]] None %[[#]]
+; CHECK-EXTENSION: %[[#reversebase:]] = OpFunctionParameter %[[#]]
+; CHECK-EXTENSION: %[[#]] = OpBitReverse %[[#]] %[[#reversebase]]
+; OpenCL equivalent.
+; kernel void testBitReverse_uchar(uchar b, global uchar *res) {
+;   *res = bit_reverse(b);
+; }
+
+define dso_local spir_kernel void @testBitReverse_uchar(i8 noundef zeroext %b, ptr addrspace(1) nocapture noundef writeonly align 1 initializes((0, 1)) %res) {
+entry:
+  %call = tail call spir_func zeroext i8 @_Z11bit_reverseh(i8 noundef zeroext %b) #2
+  store i8 %call, ptr addrspace(1) %res, align 1, !tbaa !22
+  ret void
+}
+
+declare spir_func zeroext i8 @_Z11bit_reverseh(i8 noundef zeroext) 
+
+; CHECK-EXTENSION: %[[#]] = OpFunction %[[#]] None %[[#]]
+; CHECK-EXTENSION: %[[#reversebase:]] = OpFunctionParameter %[[#]]
+; CHECK-EXTENSION: %[[#]] = OpBitReverse %[[#]] %[[#reversebase]]
+; OpenCL equivalent.
+; kernel void testBitReverse_long2(long2 b, global long2 *res) {
+;   *res = bit_reverse(b);
+; }
+
+define dso_local spir_kernel void @testBitReverse_long2(<2 x i64> noundef %b, ptr addrspace(1) nocapture noundef writeonly align 16 initializes((0, 16)) %res) {
+entry:
+  %call = tail call spir_func <2 x i64> @_Z11bit_reverseDv2_l(<2 x i64> noundef %b) #2
+  store <2 x i64> %call, ptr addrspace(1) %res, align 16, !tbaa !22
+  ret void
+}
+
+declare spir_func <2 x i64> @_Z11bit_reverseDv2_l(<2 x i64> noundef) 
+
+; CHECK-EXTENSION: %[[#]] = OpFunction %[[#]] None %[[#]]
+; CHECK-EXTENSION: %[[#reversebase:]] = OpFunctionParameter %[[#]]
+; CHECK-EXTENSION: %[[#]] = OpBitReverse %[[#]] %[[#reversebase]]
+; OpenCL equivalent.
+; kernel void testBitReverse_ulong2(ulong2 b, global ulong2 *res) {
+;   *res = bit_reverse(b);
+; }
+
+define dso_local spir_kernel void @testBitReverse_ulong2(<2 x i64> noundef %b, ptr addrspace(1) nocapture noundef writeonly align 16 initializes((0, 16)) %res) {
+entry:
+  %call = tail call spir_func <2 x i64> @_Z11bit_reverseDv2_m(<2 x i64> noundef %b) #2
+  store <2 x i64> %call, ptr addrspace(1) %res, align 16, !tbaa !22
+  ret void
+}
+
+declare spir_func <2 x i64> @_Z11bit_reverseDv2_m(<2 x i64> noundef) 
+
+; CHECK-EXTENSION: %[[#]] = OpFunction %[[#]] None %[[#]]
+; CHECK-EXTENSION: %[[#reversebase:]] = OpFunctionParameter %[[#]]
+; CHECK-EXTENSION: %[[#]] = OpBitReverse %[[#]] %[[#reversebase]]
+; OpenCL equivalent.
+; kernel void testBitReverse_int2(int2 b, global int2 *res) {
+;   *res = bit_reverse(b);
+; }
+
+define dso_local spir_kernel void @testBitReverse_int2(<2 x i32> noundef %b, ptr addrspace(1) nocapture noundef writeonly align 8 initializes((0, 8)) %res) {
+entry:
+  %call = tail call spir_func <2 x i32> @_Z11bit_reverseDv2_i(<2 x i32> noundef %b) #2
+  store <2 x i32> %call, ptr addrspace(1) %res, align 8, !tbaa !22
+  ret void
+}
+
+declare spir_func <2 x i32> @_Z11bit_reverseDv2_i(<2 x i32> noundef) 
+
+; CHECK-EXTENSION: %[[#]] = OpFunction %[[#]] None %[[#]]
+; CHECK-EXTENSION: %[[#reversebase:]] = OpFunctionParameter %[[#]]
+; CHECK-EXTENSION: %[[#]] = OpBitReverse %[[#]] %[[#reversebase]]
+; OpenCL equivalent.
+; kernel void testBitReverse_uint2(uint2 b, global uint2 *res) {
+;   *res = bit_reverse(b);
+; }
+
+define dso_local spir_kernel void @testBitReverse_uint2(<2 x i32> noundef %b, ptr addrspace(1) nocapture noundef writeonly align 8 initializes((0, 8)) %res) {
+entry:
+  %call = tail call spir_func <2 x i32> @_Z11bit_reverseDv2_j(<2 x i32> noundef %b) #2
+  store <2 x i32> %call, ptr addrspace(1) %res, align 8, !tbaa !22
+  ret void
+}
+
+declare spir_func <2 x i32> @_Z11bit_reverseDv2_j(<2 x i32> noundef) 
+
+; CHECK-EXTENSION: %[[#]] = OpFunction %[[#]] None %[[#]]
+; CHECK-EXTENSION: %[[#reversebase:]] = OpFunctionParameter %[[#]]
+; CHECK-EXTENSION: %[[#]] = OpBitReverse %[[#]] %[[#reversebase]]
+; OpenCL equivalent.
+; kernel void testBitReverse_short2(short2 b, global short2 *res) {
+;   *res = bit_reverse(b);
+; }
+
+define dso_local spir_kernel void @testBitReverse_short2(<2 x i16> noundef %b, ptr addrspace(1) nocapture noundef writeonly align 4 initializes((0, 4)) %res) {
+entry:
+  %call = tail call spir_func <2 x i16> @_Z11bit_reverseDv2_s(<2 x i16> noundef %b) #2
+  store <2 x i16> %call, ptr addrspace(1) %res, align 4, !tbaa !22
+  ret void
+}
+
+declare spir_func <2 x i16> @_Z11bit_reverseDv2_s(<2 x i16> noundef) 
+
+; CHECK-EXTENSION: %[[#]] = OpFunction %[[#]] None %[[#]]
+; CHECK-EXTENSION: %[[#reversebase:]] = OpFunctionParameter %[[#]]
+; CHECK-EXTENSION: %[[#]] = OpBitReverse %[[#]] %[[#reversebase]]
+; OpenCL equivalent.
+; kernel void testBitReverse_ushort2(ushort2 b, global ushort2 *res) {
+;   *res = bit_reverse(b);
+; }
+
+define dso_local spir_kernel void @testBitReverse_ushort2(<2 x i16> noundef %b, ptr addrspace(1) nocapture noundef writeonly align 4 initializes((0, 4)) %res) {
+entry:
+  %call = tail call spir_func <2 x i16> @_Z11bit_reverseDv2_t(<2 x i16> noundef %b) #2
+  store <2 x i16> %call, ptr addrspace(1) %res, align 4, !tbaa !22
+  ret void
+}
+
+declare spir_func <2 x i16> @_Z11bit_reverseDv2_t(<2 x i16> noundef) 
+
+; CHECK-EXTENSION: %[[#]] = OpFunction %[[#]] None %[[#]]
+; CHECK-EXTENSION: %[[#reversebase:]] = OpFunctionParameter %[[#]]
+; CHECK-EXTENSION: %[[#]] = OpBitReverse %[[#]] %[[#reversebase]]
+; OpenCL equivalent.
+; kernel void testBitReverse_char2(char2 b, global char2 *res) {
+;   *res = bit_reverse(b);
+; }
+
+define dso_local spir_kernel void @testBitReverse_char2(<2 x i8> noundef %b, ptr addrspace(1) nocapture noundef writeonly align 2 initializes((0, 2)) %res) {
+entry:
+  %call = tail call spir_func <2 x i8> @_Z11bit_reverseDv2_c(<2 x i8> noundef %b) #2
+  store <2 x i8> %call, ptr addrspace(1) %res, align 2, !tbaa !22
+  ret void
+}
+
+declare spir_func <2 x i8> @_Z11bit_reverseDv2_c(<2 x i8> noundef) 
+
+; CHECK-EXTENSION: %[[#]] = OpFunction %[[#]] None %[[#]]
+; CHECK-EXTENSION: %[[#reversebase:]] = OpFunctionParameter %[[#]]
+; CHECK-EXTENSION: %[[#]] = OpBitReverse %[[#]] %[[#reversebase]]
+; OpenCL equivalent.
+; kernel void testBitReverse_uchar2(uchar2 b, global uchar2 *res) {
+;   *res = bit_reverse(b);
+; }
+
+define dso_local spir_kernel void @testBitReverse_uchar2(<2 x i8> noundef %b, ptr addrspace(1) nocapture noundef writeonly align 2 initializes((0, 2)) %res) {
+entry:
+  %call = tail call spir_func <2 x i8> @_Z11bit_reverseDv2_h(<2 x i8> noundef %b) #2
+  store <2 x i8> %call, ptr addrspace(1) %res, align 2, !tbaa !22
+  ret void
+}
+
+declare spir_func <2 x i8> @_Z11bit_reverseDv2_h(<2 x i8> noundef) 
+
+; CHECK-EXTENSION: %[[#]] = OpFunction %[[#]] None %[[#]]
+; CHECK-EXTENSION: %[[#reversebase:]] = OpFunctionParameter %[[#]]
+; CHECK-EXTENSION: %[[#]] = OpBitReverse %[[#]] %[[#reversebase]]
+; OpenCL equivalent.
+; kernel void testBitReverse_long3(long3 b, global long3 *res) {
+;   *res = bit_reverse(b);
+; }
+
+define dso_local spir_kernel void @testBitReverse_long3(<3 x i64> noundef %b, ptr addrspace(1) nocapture noundef writeonly align 32 initializes((0, 32)) %res) {
+entry:
+  %call = tail call spir_func <3 x i64> @_Z11bit_reverseDv3_l(<3 x i64> noundef %b) #2
+  %extractVec2 = shufflevector <3 x i64> %call, <3 x i64> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 poison>
+  store <4 x i64> %extractVec2, ptr addrspace(1) %res, align 32, !tbaa !22
+  ret void
+}
+
+declare spir_func <3 x i64> @_Z11bit_reverseDv3_l(<3 x i64> noundef) 
+
+; CHECK-EXTENSION: %[[#]] = OpFunction %[[#]] None %[[#]]
+; CHECK-EXTENSION: %[[#reversebase:]] = OpFunctionParameter %[[#]]
+; CHECK-EXTENSION: %[[#]] = OpBitReverse %[[#]] %[[#reversebase]]
+; OpenCL equivalent.
+; kernel void testBitReverse_ulong3(ulong3 b, global ulong3 *res) {
+;   *res = bit_reverse(b);
+; }
+
+define dso_local spir_kernel void @testBitReverse_ulong3(<3 x i64> noundef %b, ptr addrspace(1) nocapture noundef writeonly align 32 initializes((0, 32)) %res) {
+entry:
+  %call = tail call spir_func <3 x i64> @_Z11bit_reverseDv3_m(<3 x i64> noundef %b) #2
+  %extractVec2 = shufflevector <3 x i64> %call, <3 x i64> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 poison>
+  store <4 x i64> %extractVec2, ptr addrspace(1) %res, align 32, !tbaa !22
+  ret void
+}
+
+declare spir_func <3 x i64> @_Z11bit_reverseDv3_m(<3 x i64> noundef) 
+
+; CHECK-EXTENSION: %[[#]] = OpFunction %[[#]] None %[[#]]
+; CHECK-EXTENSION: %[[#reversebase:]] = OpFunctionParameter %[[#]]
+; CHECK-EXTENSION: %[[#]] = OpBitReverse %[[#]] %[[#reversebase]]
+; OpenCL equivalent.
+; kernel void testBitReverse_int3(int3 b, global int3 *res) {
+;   *res = bit_reverse(b);
+; }
+
+define dso_local spir_kernel void @testBitReverse_int3(<3 x i32> noundef %b, ptr addrspace(1) nocapture noundef writeonly align 16 initializes((0, 16)) %res) {
+entry:
+  %call = tail call spir_func <3 x i32> @_Z11bit_reverseDv3_i(<3 x i32> noundef %b) #2
+  %extractVec2 = shufflevector <3 x i32> %call, <3 x i32> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 poison>
+  store <4 x i32> %extractVec2, ptr addrspace(1) %res, align 16, !tbaa !22
+  ret void
+}
+
+declare spir_func <3 x i32> @_Z11bit_reverseDv3_i(<3 x i32> noundef) 
+
+; CHECK-EXTENSION: %[[#]] = OpFunction %[[#]] None %[[#]]
+; CHECK-EXTENSION: %[[#reversebase:]] = OpFunctionParameter %[[#]]
+; CHECK-EXTENSION: %[[#]] = OpBitReverse %[[#]] %[[#reversebase]]
+; OpenCL equivalent.
+; kernel void testBitReverse_uint3(uint3 b, global uint3 *res) {
+;   *res = bit_reverse(b);
+; }
+
+define dso_local spir_kernel void @testBitReverse_uint3(<3 x i32> noundef %b, ptr addrspace(1) nocapture noundef writeonly align 16 initializes((0, 16)) %res) {
+entry:
+  %call = tail call spir_func <3 x i32> @_Z11bit_reverseDv3_j(<3 x i32> noundef %b) #2
+  %extractVec2 = shufflevector <3 x i32> %call, <3 x i32> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 poison>
+  store <4 x i32> %extractVec2, ptr addrspace(1) %res, align 16, !tbaa !22
+  ret void
+}
+
+declare spir_func <3 x i32> @_Z11bit_reverseDv3_j(<3 x i32> noundef) 
+
+; CHECK-EXTENSION: %[[#]] = OpFunction %[[#]] None %[[#]]
+; CHECK-EXTENSION: %[[#reversebase:]] = OpFunctionParameter %[[#]]
+; CHECK-EXTENSION: %[[#]] = OpBitReverse %[[#]] %[[#reversebase]]
+; OpenCL equivalent.
+; kernel void testBitReverse_short3(short3 b, global short3 *res) {
+;   *res = bit_reverse(b);
+; }
+
+define dso_local spir_kernel void @testBitReverse_short3(<3 x i16> noundef %b, ptr addrspace(1) nocapture noundef writeonly align 8 initializes((0, 8)) %res) {
+entry:
+  %call = tail call spir_func <3 x i16> @_Z11bit_reverseDv3_s(<3 x i16> noundef %b) #2
+  %extractVec2 = shufflevector <3 x i16> %call, <3 x i16> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 poison>
+  store <4 x i16> %extractVec2, ptr addrspace(1) %res, align 8, !tbaa !22
+  ret void
+}
+
+declare spir_func <3 x i16> @_Z11bit_reverseDv3_s(<3 x i16> noundef) 
+
+; CHECK-EXTENSION: %[[#]] = OpFunction %[[#]] None %[[#]]
+; CHECK-EXTENSION: %[[#reversebase:]] = OpFunctionParameter %[[#]]
+; CHECK-EXTENSION: %[[#]] = OpBitReverse %[[#]] %[[#reversebase]]
+; OpenCL equivalent.
+; kernel void testBitReverse_ushort3(ushort3 b, global ushort3 *res) {
+;   *res = bit_reverse(b);
+; }
+
+define dso_local spir_kernel void @testBitReverse_ushort3(<3 x i16> noundef %b, ptr addrspace(1) nocapture noundef writeonly align 8 initializes((0, 8)) %res) {
+entry:
+  %call = tail call spir_func <3 x i16> @_Z11bit_reverseDv3_t(<3 x i16> noundef %b) #2
+  %extractVec2 = shufflevector <3 x i16> %call, <3 x i16> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 poison>
+  store <4 x i16> %extractVec2, ptr addrspace(1) %res, align 8, !tbaa !22
+  ret void
+}
+
+declare spir_func <3 x i16> @_Z11bit_reverseDv3_t(<3 x i16> noundef) 
+
+; CHECK-EXTENSION: %[[#]] = OpFunction %[[#]] None %[[#]]
+; CHECK-EXTENSION: %[[#reversebase:]] = OpFunctionParameter %[[#]]
+; CHECK-EXTENSION: %[[#]] = OpBitReverse %[[#]] %[[#reversebase]]
+; OpenCL equivalent.
+; kernel void testBitReverse_char3(char3 b, global char3 *res) {
+;   *res = bit_reverse(b);
+; }
+
+define dso_local spir_kernel void @testBitReverse_char3(<3 x i8> noundef %b, ptr addrspace(1) nocapture noundef writeonly align 4 initializes((0, 4)) %res) {
+entry:
+  %call = tail call spir_func <3 x i8> @_Z11bit_reverseDv3_c(<3 x i8> noundef %b) #2
+  %extractVec2 = shufflevector <3 x i8> %call, <3 x i8> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 poison>
+  store <4 x i8> %extractVec2, ptr addrspace(1) %res, align 4, !tbaa !22
+  ret void
+}
+
+declare spir_func <3 x i8> @_Z11bit_reverseDv3_c(<3 x i8> noundef) 
+
+; CHECK-EXTENSION: %[[#]] = OpFunction %[[#]] None %[[#]]
+; CHECK-EXTENSION: %[[#reversebase:]] = OpFunctionParameter %[[#]]
+; CHECK-EXTENSION: %[[#]] = OpBitReverse %[[#]] %[[#reversebase]]
+; OpenCL equivalent.
+; kernel void testBitReverse_uchar3(uchar3 b, global uchar3 *res) {
+;   *res = bit_reverse(b);
+; }
+
+define dso_local spir_kernel void @testBitReverse_uchar3(<3 x i8> noundef %b, ptr addrspace(1) nocapture noundef writeonly align 4 initializes((0, 4)) %res) {
+entry:
+  %call = tail call spir_func <3 x i8> @_Z11bit_reverseDv3_h(<3 x i8> noundef %b) #2
+  %extractVec2 = shufflevector <3 x i8> %call, <3 x i8> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 poison>
+  store <4 x i8> %extractVec2, ptr addrspace(1) %res, align 4, !tbaa !22
+  ret void
+}
+
+declare spir_func <3 x i8> @_Z11bit_reverseDv3_h(<3 x i8> noundef) 
+
+; CHECK-EXTENSION: %[[#]] = OpFunction %[[#]] None %[[#]]
+; CHECK-EXTENSION: %[[#reversebase:]] = OpFunctionParameter %[[#]]
+; CHECK-EXTENSION: %[[#]] = OpBitReverse %[[#]] %[[#reversebase]]
+; OpenCL equivalent.
+; kernel void testBitReverse_long4(long4 b, global long4 *res) {
+;   *res = bit_reverse(b);
+; }
+
+define dso_local spir_kernel void @testBitReverse_long4(<4 x i64> noundef %b, ptr addrspace(1) nocapture noundef writeonly align 32 initializes((0, 32)) %res) {
+entry:
+  %call = tail call spir_func <4 x i64> @_Z11bit_reverseDv4_l(<4 x i64> noundef %b) #2
+  store <4 x i64> %call, ptr addrspace(1) %res, align 32, !tbaa !22
+  ret void
+}
+
+declare spir_func <4 x i64> @_Z11bit_reverseDv4_l(<4 x i64> noundef) 
+
+; CHECK-EXTENSION: %[[#]] = OpFunction %[[#]] None %[[#]]
+; CHECK-EXTENSION: %[[#reversebase:]] = OpFunctionParameter %[[#]]
+; CHECK-EXTENSION: %[[#]] = OpBitReverse %[[#]] %[[#reversebase]]
+; OpenCL equivalent.
+; kernel void testBitReverse_ulong4(ulong4 b, global ulong4 *res) {
+;   *res = bit_reverse(b);
+; }
+
+define dso_local spir_kernel void @testBitReverse_ulong4(<4 x i64> noundef %b, ptr addrspace(1) nocapture noundef writeonly align 32 initializes((0, 32)) %res) {
+entry:
+  %call = tail call spir_func <4 x i64> @_Z11bit_reverseDv4_m(<4 x i64> noundef %b) #2
+  store <4 x i64> %call, ptr addrspace(1) %res, align 32, !tbaa !22
+  ret void
+}
+
+declare spir_func <4 x i64> @_Z11bit_reverseDv4_m(<4 x i64> noundef) 
+
+; CHECK-EXTENSION: %[[#]] = OpFunction %[[#]] None %[[#]]
+; CHECK-EXTENSION: %[[#reversebase:]] = OpFunctionParameter %[[#]]
+; CHECK-EXTENSION: %[[#]] = OpBitReverse %[[#]] %[[#reversebase]]
+; OpenCL equivalent.
+; kernel void testBitReverse_int4(int4 b, global int4 *res) {
+;   *res = bit_reverse(b);
+; }
+
+define dso_local spir_kernel void @testBitReverse_int4(<4 x i32> noundef %b, ptr addrspace(1) nocapture noundef writeonly align 16 initializes((0, 16)) %res) {
+entry:
+  %call = tail call spir_func <4 x i32> @_Z11bit_reverseDv4_i(<4 x i32> noundef %b) #2
+  store <4 x i32> %call, ptr addrspace(1) %res, align 16, !tbaa !22
+  ret void
+}
+
+declare spir_func <4 x i32> @_Z11bit_reverseDv4_i(<4 x i32> noundef) 
+
+; CHECK-EXTENSION: %[[#]] = OpFunction %[[#]] None %[[#]]
+; CHECK-EXTENSION: %[[#reversebase:]] = OpFunctionParameter %[[#]]
+; CHECK-EXTENSION: %[[#]] = OpBitReverse %[[#]] %[[#reversebase]]
+; OpenCL equivalent.
+; kernel void testBitReverse_uint4(uint4 b, global uint4 *res) {
+;   *res = bit_reverse(b);
+; }
+
+define dso_local spir_kernel void @testBitReverse_uint4(<4 x i32> noundef %b, ptr addrspace(1) nocapture noundef writeonly align 16 initializes((0, 16)) %res) {
+entry:
+  %call = tail call spir_func <4 x i32> @_Z11bit_reverseDv4_j(<4 x i32> noundef %b) #2
+  store <4 x i32> %call, ptr addrspace(1) %res, align 16, !tbaa !22
+  ret void
+}
+
+declare spir_func <4 x i32> @_Z11bit_reverseDv4_j(<4 x i32> noundef) 
+
+; CHECK-EXTENSION: %[[#]] = OpFunction %[[#]] None %[[#]]
+; CHECK-EXTENSION: %[[#reversebase:]] = OpFunctionParameter %[[#]]
+; CHECK-EXTENSION: %[[#]] = OpBitReverse %[[#]] %[[#reversebase]]
+; OpenCL equivalent.
+; kernel void testBitReverse_short4(short4 b, global short4 *res) {
+;   *res = bit_reverse(b);
+; }
+
+define dso_local spir_kernel void @testBitReverse_short4(<4 x i16> noundef %b, ptr addrspace(1) nocapture noundef writeonly align 8 initializes((0, 8)) %res) {
+entry:
+  %call = tail call spir_func <4 x i16> @_Z11bit_reverseDv4_s(<4 x i16> noundef %b) #2
+  store <4 x i16> %call, ptr addrspace(1) %res, align 8, !tbaa !22
+  ret void
+}
+
+declare spir_func <4 x i16> @_Z11bit_reverseDv4_s(<4 x i16> noundef) 
+
+; CHECK-EXTENSION: %[[#]] = OpFunction %[[#]] None %[[#]]
+; CHECK-EXTENSION: %[[#reversebase:]] = OpFunctionParameter %[[#]]
+; CHECK-EXTENSION: %[[#]] = OpBitReverse %[[#]] %[[#reversebase]]
+; OpenCL equivalent.
+; kernel void testBitReverse_ushort4(ushort4 b, global ushort4 *res) {
+;   *res = bit_reverse(b);
+; }
+
+define dso_local spir_kernel void @testBitReverse_ushort4(<4 x i16> noundef %b, ptr addrspace(1) nocapture noundef writeonly align 8 initializes((0, 8)) %res) {
+entry:
+  %call = tail call spir_func <4 x i16> @_Z11bit_reverseDv4_t(<4 x i16> noundef %b) #2
+  store <4 x i16> %call, ptr addrspace(1) %res, align 8, !tbaa !22
+  ret void
+}
+
+declare spir_func <4 x i16> @_Z11bit_reverseDv4_t(<4 x i16> noundef) 
+
+; CHECK-EXTENSION: %[[#]] = OpFunction %[[#]] None %[[#]]
+; CHECK-EXTENSION: %[[#reversebase:]] = OpFunctionParameter %[[#]]
+; CHECK-EXTENSION: %[[#]] = OpBitReverse %[[#]] %[[#reversebase]]
+; OpenCL equivalent.
+; kernel void testBitReverse_char4(char4 b, global char4 *res) {
+;   *res = bit_reverse(b);
+; }
+
+define dso_local spir_kernel void @testBitReverse_char4(<4 x i8> noundef %b, ptr addrspace(1) nocapture noundef writeonly align 4 initializes((0, 4)) %res) {
+entry:
+  %call = tail call spir_func <4 x i8> @_Z11bit_reverseDv4_c(<4 x i8> noundef %b) #2
+  store <4 x i8> %call, ptr addrspace(1) %res, align 4, !tbaa !22
+  ret void
+}
+
+declare spir_func <4 x i8> @_Z11bit_reverseDv4_c(<4 x i8> noundef) 
+
+; CHECK-EXTENSION: %[[#]] = OpFunction %[[#]] None %[[#]]
+; CHECK-EXTENSION: %[[#reversebase:]] = OpFunctionParameter %[[#]]
+; CHECK-EXTENSION: %[[#]] = OpBitReverse %[[#]] %[[#reversebase]]
+; OpenCL equivalent.
+; kernel void testBitReverse_uchar4(uchar4 b, global uchar4 *res) {
+;   *res = bit_reverse(b);
+; }
+
+define dso_local spir_kernel void @testBitReverse_uchar4(<4 x i8> noundef %b, ptr addrspace(1) nocapture noundef writeonly align 4 initializes((0, 4)) %res) {
+entry:
+  %call = tail call spir_func <4 x i8> @_Z11bit_reverseDv4_h(<4 x i8> noundef %b) #2
+  store <4 x i8> %call, ptr addrspace(1) %res, align 4, !tbaa !22
+  ret void
+}
+
+declare spir_func <4 x i8> @_Z11bit_reverseDv4_h(<4 x i8> noundef) 
+
+; CHECK-EXTENSION: %[[#]] = OpFunction %[[#]] None %[[#]]
+; CHECK-EXTENSION: %[[#reversebase:]] = OpFunctionParameter %[[#]]
+; CHECK-EXTENSION: %[[#]] = OpBitReverse %[[#]] %[[#reversebase]]
+; OpenCL equivalent.
+; kernel void testBitReverse_long8(long8 b, global long8 *res) {
+;   *res = bit_reverse(b);
+; }
+
+define dso_local spir_kernel void @testBitReverse_long8(<8 x i64> noundef %b, ptr addrspace(1) nocapture noundef writeonly align 64 initializes((0, 64)) %res) {
+entry:
+  %call = tail call spir_func <8 x i64> @_Z11bit_reverseDv8_l(<8 x i64> noundef %b) #2
+  store <8 x i64> %call, ptr addrspace(1) %res, align 64, !tbaa !22
+  ret void
+}
+
+declare spir_func <8 x i64> @_Z11bit_reverseDv8_l(<8 x i64> noundef) 
+
+; CHECK-EXTENSION: %[[#]] = OpFunction %[[#]] None %[[#]]
+; CHECK-EXTENSION: %[[#reversebase:]] = OpFunctionParameter %[[#]]
+; CHECK-EXTENSION: %[[#]] = OpBitReverse %[[#]] %[[#reversebase]]
+; OpenCL equivalent.
+; kernel void testBitReverse_ulong8(ulong8 b, global ulong8 *res) {
+;   *res = bit_reverse(b);
+; }
+
+define dso_local spir_kernel void @testBitReverse_ulong8(<8 x i64> noundef %b, ptr addrspace(1) nocapture noundef writeonly align 64 initializes((0, 64)) %res) {
+entry:
+  %call = tail call spir_func <8 x i64> @_Z11bit_reverseDv8_m(<8 x i64> noundef %b) #2
+  store <8 x i64> %call, ptr addrspace(1) %res, align 64, !tbaa !22
+  ret void
+}
+
+declare spir_func <8 x i64> @_Z11bit_reverseDv8_m(<8 x i64> noundef) 
+
+; CHECK-EXTENSION: %[[#]] = OpFunction %[[#]] None %[[#]]
+; CHECK-EXTENSION: %[[#reversebase:]] = OpFunctionParameter %[[#]]
+; CHECK-EXTENSION: %[[#]] = OpBitReverse %[[#]] %[[#reversebase]]
+; OpenCL equivalent.
+; kernel void testBitReverse_int8(int8 b, global int8 *res) {
+;   *res = bit_reverse(b);
+; }
+
+define dso_local spir_kernel void @testBitReverse_int8(<8 x i32> noundef %b, ptr addrspace(1) nocapture noundef writeonly align 32 initializes((0, 32)) %res) {
+entry:
+  %call = tail call spir_func <8 x i32> @_Z11bit_reverseDv8_i(<8 x i32> noundef %b) #2
+  store <8 x i32> %call, ptr addrspace(1) %res, align 32, !tbaa !22
+  ret void
+}
+
+declare spir_func <8 x i32> @_Z11bit_reverseDv8_i(<8 x i32> noundef) 
+
+; CHECK-EXTENSION: %[[#]] = OpFunction %[[#]] None %[[#]]
+; CHECK-EXTENSION: %[[#reversebase:]] = OpFunctionParameter %[[#]]
+; CHECK-EXTENSION: %[[#]] = OpBitReverse %[[#]] %[[#reversebase]]
+; OpenCL equivalent.
+; kernel void testBitReverse_uint8(uint8 b, global uint8 *res) {
+;   *res = bit_reverse(b);
+; }
+
+define dso_local spir_kernel void @testBitReverse_uint8(<8 x i32> noundef %b, ptr addrspace(1) nocapture noundef writeonly align 32 initializes((0, 32)) %res) {
+entry:
+  %call = tail call spir_func <8 x i32> @_Z11bit_reverseDv8_j(<8 x i32> noundef %b) #2
+  store <8 x i32> %call, ptr addrspace(1) %res, align 32, !tbaa !22
+  ret void
+}
+
+declare spir_func <8 x i32> @_Z11bit_reverseDv8_j(<8 x i32> noundef) 
+
+; CHECK-EXTENSION: %[[#]] = OpFunction %[[#]] None %[[#]]
+; CHECK-EXTENSION: %[[#reversebase:]] = OpFunctionParameter %[[#]]
+; CHECK-EXTENSION: %[[#]] = OpBitReverse %[[#]] %[[#reversebase]]
+; OpenCL equivalent.
+; kernel void testBitReverse_short8(short8 b, global short8 *res) {
+;   *res = bit_reverse(b);
+; }
+
+define dso_local spir_kernel void @testBitReverse_short8(<8 x i16> noundef %b, ptr addrspace(1) nocapture noundef writeonly align 16 initializes((0, 16)) %res) {
+entry:
+  %call = tail call spir_func <8 x i16> @_Z11bit_reverseDv8_s(<8 x i16> noundef %b) #2
+  store <8 x i16> %call, ptr addrspace(1) %res, align 16, !tbaa !22
+  ret void
+}
+
+declare spir_func <8 x i16> @_Z11bit_reverseDv8_s(<8 x i16> noundef) 
+
+; CHECK-EXTENSION: %[[#]] = OpFunction %[[#]] None %[[#]]
+; CHECK-EXTENSION: %[[#reversebase:]] = OpFunctionParameter %[[#]]
+; CHECK-EXTENSION: %[[#]] = OpBitReverse %[[#]] %[[#reversebase]]
+; OpenCL equivalent.
+; kernel void testBitReverse_ushort8(ushort8 b, global ushort8 *res) {
+;   *res = bit_reverse(b);
+; }
+
+define dso_local spir_kernel void @testBitReverse_ushort8(<8 x i16> noundef %b, ptr addrspace(1) nocapture noundef writeonly align 16 initializes((0, 16)) %res) {
+entry:
+  %call = tail call spir_func <8 x i16> @_Z11bit_reverseDv8_t(<8 x i16> noundef %b) #2
+  store <8 x i16> %call, ptr addrspace(1) %res, align 16, !tbaa !22
+  ret void
+}
+
+declare spir_func <8 x i16> @_Z11bit_reverseDv8_t(<8 x i16> noundef) 
+
+; CHECK-EXTENSION: %[[#]] = OpFunction %[[#]] None %[[#]]
+; CHECK-EXTENSION: %[[#reversebase:]] = OpFunctionParameter %[[#]]
+; CHECK-EXTENSION: %[[#]] = OpBitReverse %[[#]] %[[#reversebase]]
+; OpenCL equivalent.
+; kernel void testBitReverse_char8(char8 b, global char8 *res) {
+;   *res = bit_reverse(b);
+; }
+
+define dso_local spir_kernel void @testBitReverse_char8(<8 x i8> noundef %b, ptr addrspace(1) nocapture noundef writeonly align 8 initializes((0, 8)) %res) {
+entry:
+  %call = tail call spir_func <8 x i8> @_Z11bit_reverseDv8_c(<8 x i8> noundef %b) #2
+  store <8 x i8> %call, ptr addrspace(1) %res, align 8, !tbaa !22
+  ret void
+}
+
+declare spir_func <8 x i8> @_Z11bit_reverseDv8_c(<8 x i8> noundef) 
+
+; CHECK-EXTENSION: %[[#]] = OpFunction %[[#]] None %[[#]]
+; CHECK-EXTENSION: %[[#reversebase:]] = OpFunctionParameter %[[#]]
+; CHECK-EXTENSION: %[[#]] = OpBitReverse %[[#]] %[[#reversebase]]
+; OpenCL equivalent.
+; kernel void testBitReverse_uchar8(uchar8 b, global uchar8 *res) {
+;   *res = bit_reverse(b);
+; }
+
+define dso_local spir_kernel void @testBitReverse_uchar8(<8 x i8> noundef %b, ptr addrspace(1) nocapture noundef writeonly align 8 initializes((0, 8)) %res) {
+entry:
+  %call = tail call spir_func <8 x i8> @_Z11bit_reverseDv8_h(<8 x i8> noundef %b) #2
+  store <8 x i8> %call, ptr addrspace(1) %res, align 8, !tbaa !22
+  ret void
+}
+
+declare spir_func <8 x i8> @_Z11bit_reverseDv8_h(<8 x i8> noundef) 
+
+; CHECK-EXTENSION: %[[#]] = OpFunction %[[#]] None %[[#]]
+; CHECK-EXTENSION: %[[#reversebase:]] = OpFunctionParameter %[[#]]
+; CHECK-EXTENSION: %[[#]] = OpBitReverse %[[#]] %[[#reversebase]]
+; OpenCL equivalent.
+; kernel void testBitReverse_long16(long16 b, global long16 *res) {
+;   *res = bit_reverse(b);
+; }
+
+define dso_local spir_kernel void @testBitReverse_long16(<16 x i64> noundef %b, ptr addrspace(1) nocapture noundef writeonly align 128 initializes((0, 128)) %res) {
+entry:
+  %call = tail call spir_func <16 x i64> @_Z11bit_reverseDv16_l(<16 x i64> noundef %b) #2
+  store <16 x i64> %call, ptr addrspace(1) %res, align 128, !tbaa !22
+  ret void
+}
+
+declare spir_func <16 x i64> @_Z11bit_reverseDv16_l(<16 x i64> noundef) 
+
+; CHECK-EXTENSION: %[[#]] = OpFunction %[[#]] None %[[#]]
+; CHECK-EXTENSION: %[[#reversebase:]] = OpFunctionParameter %[[#]]
+; CHECK-EXTENSION: %[[#]] = OpBitReverse %[[#]] %[[#reversebase]]
+; OpenCL equivalent.
+; kernel void testBitReverse_ulong16(ulong16 b, global ulong16 *res) {
+;   *res = bit_reverse(b);
+; }
+
+define dso_local spir_kernel void @testBitReverse_ulong16(<16 x i64> noundef %b, ptr addrspace(1) nocapture noundef writeonly align 128 initializes((0, 128)) %res) {
+entry:
+  %call = tail call spir_func <16 x i64> @_Z11bit_reverseDv16_m(<16 x i64> noundef %b) #2
+  store <16 x i64> %call, ptr addrspace(1) %res, align 128, !tbaa !22
+  ret void
+}
+
+declare spir_func <16 x i64> @_Z11bit_reverseDv16_m(<16 x i64> noundef) 
+
+; CHECK-EXTENSION: %[[#]] = OpFunction %[[#]] None %[[#]]
+; CHECK-EXTENSION: %[[#reversebase:]] = OpFunctionParameter %[[#]]
+; CHECK-EXTENSION: %[[#]] = OpBitReverse %[[#]] %[[#reversebase]]
+; OpenCL equivalent.
+; kernel void testBitReverse_int16(int16 b, global int16 *res) {
+;   *res = bit_reverse(b);
+; }
+
+define dso_local spir_kernel void @testBitReverse_int16(<16 x i32> noundef %b, ptr addrspace(1) nocapture noundef writeonly align 64 initializes((0, 64)) %res) {
+entry:
+  %call = tail call spir_func <16 x i32> @_Z11bit_reverseDv16_i(<16 x i32> noundef %b) #2
+  store <16 x i32> %call, ptr addrspace(1) %res, align 64, !tbaa !22
+  ret void
+}
+
+declare spir_func <16 x i32> @_Z11bit_reverseDv16_i(<16 x i32> noundef) 
+
+; CHECK-EXTENSION: %[[#]] = OpFunction %[[#]] None %[[#]]
+; CHECK-EXTENSION: %[[#reversebase:]] = OpFunctionParameter %[[#]]
+; CHECK-EXTENSION: %[[#]] = OpBitReverse %[[#]] %[[#reversebase]]
+; OpenCL equivalent.
+; kernel void testBitReverse_uint16(uint16 b, global uint16 *res) {
+;   *res = bit_reverse(b);
+; }
+
+define dso_local spir_kernel void @testBitReverse_uint16(<16 x i32> noundef %b, ptr addrspace(1) nocapture noundef writeonly align 64 initializes((0, 64)) %res) {
+entry:
+  %call = tail call spir_func <16 x i32> @_Z11bit_reverseDv16_j(<16 x i32> noundef %b) #2
+  store <16 x i32> %call, ptr addrspace(1) %res, align 64, !tbaa !22
+  ret void
+}
+
+declare spir_func <16 x i32> @_Z11bit_reverseDv16_j(<16 x i32> noundef) 
+
+; CHECK-EXTENSION: %[[#]] = OpFunction %[[#]] None %[[#]]
+; CHECK-EXTENSION: %[[#reversebase:]] = OpFunctionParameter %[[#]]
+; CHECK-EXTENSION: %[[#]] = OpBitReverse %[[#]] %[[#reversebase]]
+; OpenCL equivalent.
+; kernel void testBitReverse_short16(short16 b, global short16 *res) {
+;   *res = bit_reverse(b);
+; }
+
+define dso_local spir_kernel void @testBitReverse_short16(<16 x i16> noundef %b, ptr addrspace(1) nocapture noundef writeonly align 32 initializes((0, 32)) %res) {
+entry:
+  %call = tail call spir_func <16 x i16> @_Z11bit_reverseDv16_s(<16 x i16> noundef %b) #2
+  store <16 x i16> %call, ptr addrspace(1) %res, align 32, !tbaa !22
+  ret void
+}
+
+declare spir_func <16 x i16> @_Z11bit_reverseDv16_s(<16 x i16> noundef) 
+
+; CHECK-EXTENSION: %[[#]] = OpFunction %[[#]] None %[[#]]
+; CHECK-EXTENSION: %[[#reversebase:]] = OpFunctionParameter %[[#]]
+; CHECK-EXTENSION: %[[#]] = OpBitReverse %[[#]] %[[#reversebase]]
+; OpenCL equivalent.
+; kernel void testBitReverse_ushort16(ushort16 b, global ushort16 *res) {
+;   *res = bit_reverse(b);
+; }
+
+define dso_local spir_kernel void @testBitReverse_ushort16(<16 x i16> noundef %b, ptr addrspace(1) nocapture noundef writeonly align 32 initializes((0, 32)) %res) {
+entry:
+  %call = tail call spir_func <16 x i16> @_Z11bit_reverseDv16_t(<16 x i16> noundef %b) #2
+  store <16 x i16> %call, ptr addrspace(1) %res, align 32, !tbaa !22
+  ret void
+}
+
+declare spir_func <16 x i16> @_Z11bit_reverseDv16_t(<16 x i16> noundef) 
+
+; CHECK-EXTENSION: %[[#]] = OpFunction %[[#]] None %[[#]]
+; CHECK-EXTENSION: %[[#reversebase:]] = OpFunctionParameter %[[#]]
+; CHECK-EXTENSION: %[[#]] = OpBitReverse %[[#]] %[[#reversebase]]
+; OpenCL equivalent.
+; kernel void testBitReverse_char16(char16 b, global char16 *res) {
+;   *res = bit_reverse(b);
+; }
+
+define dso_local spir_kernel void @testBitReverse_char16(<16 x i8> noundef %b, ptr addrspace(1) nocapture noundef writeonly align 16 initializes((0, 16)) %res) {
+entry:
+  %call = tail call spir_func <16 x i8> @_Z11bit_reverseDv16_c(<16 x i8> noundef %b) #2
+  store <16 x i8> %call, ptr addrspace(1) %res, align 16, !tbaa !22
+  ret void
+}
+
+declare spir_func <16 x i8> @_Z11bit_reverseDv16_c(<16 x i8> noundef) 
+
+; CHECK-EXTENSION: %[[#]] = OpFunction %[[#]] None %[[#]]
+; CHECK-EXTENSION: %[[#reversebase:]] = OpFunctionParameter %[[#]]
+; CHECK-EXTENSION: %[[#]] = OpBitReverse %[[#]] %[[#reversebase]]
+; OpenCL equivalent.
+; kernel void testBitReverse_uchar16(uchar16 b, global uchar16 *res) {
+;   *res = bit_reverse(b);
+; }
+
+define dso_local spir_kernel void @testBitReverse_uchar16(<16 x i8> noundef %b, ptr addrspace(1) nocapture noundef writeonly align 16 initializes((0, 16)) %res) {
+entry:
+  %call = tail call spir_func <16 x i8> @_Z11bit_reverseDv16_h(<16 x i8> noundef %b) #2
+  store <16 x i8> %call, ptr addrspace(1) %res, align 16, !tbaa !22
+  ret void
+}
+
+declare spir_func <16 x i8> @_Z11bit_reverseDv16_h(<16 x i8> noundef) 
+
+attributes #2 = { convergent nounwind willreturn memory(none) }
+
+!llvm.module.flags = !{!0}
+!opencl.ocl.version = !{!1}
+!opencl.spir.version = !{!1}
+!llvm.ident = !{!2}
+
+!0 = !{i32 1, !"wchar_size", i32 4}
+!1 = !{i32 2, i32 0}
+!2 = !{!"clang version 20.0.0git (https://github.com/llvm/llvm-project.git cc61409d353a40f62d3a137f3c7436aa00df779d)"}
+!7 = !{!8, !8, i64 0}
+!8 = !{!"long", !9, i64 0}
+!9 = !{!"omnipotent char", !10, i64 0}
+!10 = !{!"Simple C/C++ TBAA"}
+!13 = !{!14, !14, i64 0}
+!14 = !{!"int", !9, i64 0}
+!17 = !{!18, !18, i64 0}
+!18 = !{!"short", !9, i64 0}
+!22 = !{!9, !9, i64 0}
diff --git a/llvm/test/CodeGen/SPIRV/extensions/SPV_KHR_bit_instructions/cl_khr_extended_bit_ops_ocl_only.ll b/llvm/test/CodeGen/SPIRV/extensions/SPV_KHR_bit_instructions/cl_khr_extended_bit_ops_ocl_only.ll
new file mode 100644
index 0000000000000..221ecb4fef6d9
--- /dev/null
+++ b/llvm/test/CodeGen/SPIRV/extensions/SPV_KHR_bit_instructions/cl_khr_extended_bit_ops_ocl_only.ll
@@ -0,0 +1,43 @@
+; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv32-unknown-unknown %s --spirv-ext=+SPV_KHR_bit_instructions -o - | FileCheck %s --check-prefix=CHECK-EXTENSION
+; RUN: not llc -verify-machineinstrs -O0 -mtriple=spirv32-unknown-unknown %s -o %t.spvt 2>&1 | FileCheck %s --check-prefix=CHECK-NO-EXTENSION
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv64-unknown-unknown %s --spirv-ext=+SPV_KHR_bit_instructions -o - -filetype=obj | spirv-val %} 
+;
+; if OpenCL builtin calls present in LLVM IR input and SPV_KHR_bit_instructions is enabled, no error and BitInstructions capability generated.
+; CHECK-EXTENSION: Capability BitInstructions
+; CHECK-EXTENSION-NOT: Capability Shader
+; CHECK-EXTENSION: Extension "SPV_KHR_bit_instructions"
+; if OpenCL builtin calls present in LLVM IR input and SPV_KHR_bit_instructions is NOT enabled, error.
+; CHECK-NO-EXTENSION: LLVM ERROR: bitfield_insert: the builtin requires the following SPIR-V extension: SPV_KHR_bit_instructions
+;
+; CHECK-EXTENSION: %[[#]] = OpFunction %[[#]] None %[[#]]
+; CHECK-EXTENSION: %[[#insertbase_long:]] = OpFunctionParameter %[[#]]
+; CHECK-EXTENSION: %[[#insertinsert_long:]] = OpFunctionParameter %[[#]]
+; CHECK-EXTENSION: %[[#]] = OpBitFieldInsert %[[#]] %[[#insertbase_long]] %[[#insertinsert_long]]
+;
+; OpenCL equivalent.
+; kernel void testInsert_long(long b, long i, global long *res) {
+;   *res = bitfield_insert(b, i, 4, 2);
+; }
+define dso_local spir_kernel void @testInsert_long(i64 noundef %b, i64 noundef %i, ptr addrspace(1) nocapture noundef writeonly align 8 initializes((0, 8)) %res) {
+entry:
+  %call = tail call spir_func i64 @_Z15bitfield_insertlljj(i64 noundef %b, i64 noundef %i, i32 noundef 4, i32 noundef 2) #2
+  store i64 %call, ptr addrspace(1) %res, align 8, !tbaa !7
+  ret void
+}
+
+declare spir_func i64 @_Z15bitfield_insertlljj(i64 noundef, i64 noundef, i32 noundef, i32 noundef) 
+
+attributes #2 = { convergent nounwind willreturn memory(none) }
+
+!llvm.module.flags = !{!0}
+!opencl.ocl.version = !{!1}
+!opencl.spir.version = !{!1}
+!llvm.ident = !{!2}
+
+!0 = !{i32 1, !"wchar_size", i32 4}
+!1 = !{i32 2, i32 0}
+!2 = !{!"clang version 20.0.0git (https://github.com/llvm/llvm-project.git cc61409d353a40f62d3a137f3c7436aa00df779d)"}
+!7 = !{!8, !8, i64 0}
+!8 = !{!"long", !9, i64 0}
+!9 = !{!"omnipotent char", !10, i64 0}
+!10 = !{!"Simple C/C++ TBAA"}
diff --git a/llvm/test/CodeGen/SPIRV/extensions/SPV_KHR_bit_instructions/cl_khr_extended_bit_ops_spv-friendly_only.ll b/llvm/test/CodeGen/SPIRV/extensions/SPV_KHR_bit_instructions/cl_khr_extended_bit_ops_spv-friendly_only.ll
new file mode 100644
index 0000000000000..65cccc83a3e02
--- /dev/null
+++ b/llvm/test/CodeGen/SPIRV/extensions/SPV_KHR_bit_instructions/cl_khr_extended_bit_ops_spv-friendly_only.ll
@@ -0,0 +1,38 @@
+; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv32-unknown-unknown %s --spirv-ext=+SPV_KHR_bit_instructions -o - | FileCheck %s --check-prefix=CHECK-EXTENSION
+; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv32-unknown-unknown %s -o - | FileCheck %s --check-prefix=CHECK-NO-EXTENSION
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv64-unknown-unknown %s --spirv-ext=+SPV_KHR_bit_instructions -o - -filetype=obj | spirv-val %} 
+;
+; CHECK-EXTENSION: Capability BitInstructions
+; CHECK-EXTENSION: Extension "SPV_KHR_bit_instructions"
+; CHECK-NO-EXTENSION-NOT: Capability BitInstructions 
+; CHECK-NO-EXTENSION-NOT: Extension "SPV_KHR_bit_instructions"
+; CHECK-NO-EXTENSION: Capability Shader 
+;
+; CHECK-EXTENSION: %[[#]] = OpFunction %[[#]] None %[[#]]
+; CHECK-EXTENSION: %[[#reversebase:]] = OpFunctionParameter %[[#]]
+; CHECK-EXTENSION: %[[#]] = OpBitReverse %[[#]] %[[#reversebase]]
+; OpenCL equivalent.
+; kernel void testBitReverse_SPIRVFriendly(long4 b, global long4 *res) {
+;   *res = bit_reverse(b);
+; }
+define spir_kernel void @testBitReverse_SPIRVFriendly(<4 x i64> %b, ptr addrspace(1) nocapture align 32 %res) #3 {
+entry:
+  %call = call <4 x i64> @llvm.bitreverse.v4i64(<4 x i64> %b)
+  store <4 x i64> %call, ptr addrspace(1) %res, align 32
+  ret void
+}
+
+declare <4 x i64> @llvm.bitreverse.v4i64(<4 x i64>) #4
+
+
+attributes #3 = { nounwind }
+attributes #4 = { nocallback nofree nosync nounwind speculatable willreturn memory(none) }
+
+!llvm.module.flags = !{!0}
+!opencl.ocl.version = !{!1}
+!opencl.spir.version = !{!1}
+!llvm.ident = !{!2}
+
+!0 = !{i32 1, !"wchar_size", i32 4}
+!1 = !{i32 2, i32 0}
+!2 = !{!"clang version 20.0.0git (https://github.com/llvm/llvm-project.git cc61409d353a40f62d3a137f3c7436aa00df779d)"}
diff --git a/llvm/test/CodeGen/SPIRV/extensions/SPV_KHR_bit_instructions/spirv-friendly_extended_bit_ops.ll b/llvm/test/CodeGen/SPIRV/extensions/SPV_KHR_bit_instructions/spirv-friendly_extended_bit_ops.ll
new file mode 100644
index 0000000000000..e49a55956aff0
--- /dev/null
+++ b/llvm/test/CodeGen/SPIRV/extensions/SPV_KHR_bit_instructions/spirv-friendly_extended_bit_ops.ll
@@ -0,0 +1,98 @@
+; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv32-unknown-unknown %s --spirv-ext=+SPV_KHR_bit_instructions -o - | FileCheck %s --check-prefix=CHECK-EXTENSION
+; RUN: not llc -verify-machineinstrs -O0 -mtriple=spirv32-unknown-unknown %s -o %t.spvt 2>&1 | FileCheck %s --check-prefix=CHECK-NO-EXTENSION
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv64-unknown-unknown %s --spirv-ext=+SPV_KHR_bit_instructions -o - -filetype=obj | spirv-val %} 
+;
+; CHECK-EXTENSION: Capability BitInstructions
+; CHECK-EXTENSION: Extension "SPV_KHR_bit_instructions"
+; CHECK-NO-EXTENSION: LLVM ERROR: __spirv_BitFieldInsert: the builtin requires the following SPIR-V extension: SPV_KHR_bit_instructions
+
+; Test SPIRV-friendly builtins.
+; CHECK-EXTENSION: %[[#]] = OpFunction %[[#]] None %[[#]]
+; CHECK-EXTENSION: %[[#insertbase_int2:]] = OpFunctionParameter %[[#]]
+; CHECK-EXTENSION: %[[#insertinsert_int2:]] = OpFunctionParameter %[[#]]
+; CHECK-EXTENSION: %[[#]] = OpBitFieldInsert %[[#]] %[[#insertbase_int2]] %[[#insertinsert_int2]]
+; OpenCL equivalent.
+; kernel void testInsert_SPIRVFriendly(int2 b, int2 i, global int2 *res) {
+;   *res = bitfield_insert(b, i, 4, 2);
+; }
+define spir_kernel void @testInsert_SPIRVFriendly(<2 x i32> %b, <2 x i32> %i, ptr addrspace(1) nocapture align 8 %res) #3 {
+entry:
+  %call = call spir_func <2 x i32> @_Z22__spirv_BitFieldInsertDv2_iS_jj(<2 x i32> %b, <2 x i32> %i, i32 4, i32 2) #3
+  store <2 x i32> %call, ptr addrspace(1) %res, align 8
+  ret void
+}
+
+declare spir_func <2 x i32> @_Z22__spirv_BitFieldInsertDv2_iS_jj(<2 x i32>, <2 x i32>, i32, i32) #3
+
+
+; CHECK-EXTENSION: %[[#]] = OpFunction %[[#]] None %[[#]]
+; CHECK-EXTENSION: %[[#sextractbase:]] = OpFunctionParameter %[[#]]
+; CHECK-EXTENSION: %[[#sextractbaseu:]] = OpFunctionParameter %[[#]]
+; CHECK-EXTENSION: %[[#]] = OpBitFieldSExtract %[[#]] %[[#sextractbase]]
+; CHECK-EXTENSION: %[[#]] = OpBitFieldSExtract %[[#]] %[[#sextractbaseu]]
+; OpenCL equivalent.
+; kernel void testExtractS_SPIRVFriendly(short b, ushort bu, global short *res) {
+;   *res = bitfield_extract_signed(b, 5, 4);
+;   *res += bitfield_extract_signed(bu, 5, 4);
+; }
+define spir_kernel void @testExtractS_SPIRVFriendly(i16 signext %b, i16 zeroext %bu, ptr addrspace(1) nocapture align 2 %res) #3 {
+entry:
+  %call = call spir_func i16 @_Z24__spirv_BitFieldSExtractsjj(i16 %b, i32 5, i32 4) #3
+  %call1 = call spir_func i16 @_Z24__spirv_BitFieldSExtractsjj(i16 %bu, i32 5, i32 4) #3
+  %add = add i16 %call1, %call
+  store i16 %add, ptr addrspace(1) %res, align 2
+  ret void
+}
+
+declare spir_func i16 @_Z24__spirv_BitFieldSExtractsjj(i16, i32, i32) #3
+
+; CHECK-EXTENSION: %[[#]] = OpFunction %[[#]] None %[[#]]
+; CHECK-EXTENSION: %[[#uextractbase:]] = OpFunctionParameter %[[#]]
+; CHECK-EXTENSION: %[[#uextractbaseu:]] = OpFunctionParameter %[[#]]
+; CHECK-EXTENSION: %[[#]] = OpBitFieldUExtract %[[#]] %[[#uextractbase]]
+; CHECK-EXTENSION: %[[#]] = OpBitFieldUExtract %[[#]] %[[#uextractbaseu]]
+; OpenCL equivalent.
+; kernel void testExtractU_SPIRVFriendly(char8 b, uchar8 bu, global uchar8 *res) {
+;   *res = bitfield_extract_unsigned(b, 3, 4);
+;   *res += bitfield_extract_unsigned(bu, 3, 4);
+; }
+define spir_kernel void @testExtractU_SPIRVFriendly(<8 x i8> %b, <8 x i8> %bu, ptr addrspace(1) nocapture align 8 %res) #3 {
+entry:
+  %call = call spir_func <8 x i8> @_Z24__spirv_BitFieldUExtractDv8_hjj(<8 x i8> %b, i32 3, i32 4) #3
+  %call1 = call spir_func <8 x i8> @_Z24__spirv_BitFieldUExtractDv8_hjj(<8 x i8> %bu, i32 3, i32 4) #3
+  %add = add <8 x i8> %call1, %call
+  store <8 x i8> %add, ptr addrspace(1) %res, align 8
+  ret void
+}
+
+declare spir_func <8 x i8> @_Z24__spirv_BitFieldUExtractDv8_hjj(<8 x i8>, i32, i32) #3
+
+; CHECK-EXTENSION: %[[#]] = OpFunction %[[#]] None %[[#]]
+; CHECK-EXTENSION: %[[#reversebase:]] = OpFunctionParameter %[[#]]
+; CHECK-EXTENSION: %[[#]] = OpBitReverse %[[#]] %[[#reversebase]]
+; OpenCL equivalent.
+; kernel void testBitReverse_SPIRVFriendly(long4 b, global long4 *res) {
+;   *res = bit_reverse(b);
+; }
+define spir_kernel void @testBitReverse_SPIRVFriendly(<4 x i64> %b, ptr addrspace(1) nocapture align 32 %res) #3 {
+entry:
+  %call = call <4 x i64> @llvm.bitreverse.v4i64(<4 x i64> %b)
+  store <4 x i64> %call, ptr addrspace(1) %res, align 32
+  ret void
+}
+
+declare <4 x i64> @llvm.bitreverse.v4i64(<4 x i64>) #4
+
+
+
+attributes #3 = { nounwind }
+attributes #4 = { nocallback nofree nosync nounwind speculatable willreturn memory(none) }
+
+!llvm.module.flags = !{!0}
+!opencl.ocl.version = !{!1}
+!opencl.spir.version = !{!1}
+!llvm.ident = !{!2}
+
+!0 = !{i32 1, !"wchar_size", i32 4}
+!1 = !{i32 2, i32 0}
+!2 = !{!"clang version 20.0.0git (https://github.com/llvm/llvm-project.git cc61409d353a40f62d3a137f3c7436aa00df779d)"}
diff --git a/llvm/test/CodeGen/SPIRV/extensions/SPV_KHR_integer_dot_product/SPV_KHR_integer_dot_product_OCLtoSPIRV_char4.ll b/llvm/test/CodeGen/SPIRV/extensions/SPV_KHR_integer_dot_product/SPV_KHR_integer_dot_product_OCLtoSPIRV_char4.ll
new file mode 100644
index 0000000000000..8c68017b00fd4
--- /dev/null
+++ b/llvm/test/CodeGen/SPIRV/extensions/SPV_KHR_integer_dot_product/SPV_KHR_integer_dot_product_OCLtoSPIRV_char4.ll
@@ -0,0 +1,53 @@
+; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv32v1.6-unknown-unknown %s -o - | FileCheck %s
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv32v1.6-unknown-unknown %s -o - -filetype=obj | spirv-val %}
+; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv32-unknown-unknown --spirv-ext=+SPV_KHR_integer_dot_product %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-EXT
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv32-unknown-unknown --spirv-ext=+SPV_KHR_integer_dot_product %s -o - -filetype=obj | spirv-val %}
+
+; CHECK: Capability DotProduct
+; CHECK: Capability DotProductInput4x8Bit
+; CHECK-EXT: OpExtension "SPV_KHR_integer_dot_product"
+; CHECK-NOT: OpExtension "SPV_KHR_integer_dot_product"
+
+; CHECK: Name %[[#SignedA:]] "ia"
+; CHECK: Name %[[#UnsignedA:]] "ua"
+; CHECK: Name %[[#SignedB:]] "ib"
+; CHECK: Name %[[#UnsignedB:]] "ub"
+
+; CHECK: SDot %[[#]] %[[#SignedA]] %[[#SignedB]]
+; CHECK: SUDot %[[#]] %[[#SignedA]] %[[#UnsignedB]]
+; CHECK: SUDot %[[#]] %[[#SignedB]] %[[#UnsignedA]]
+; CHECK: UDot %[[#]] %[[#UnsignedA]] %[[#UnsignedB]]
+
+; CHECK: SDotAccSat %[[#]] %[[#SignedA]] %[[#SignedB]] %[[#]]
+; CHECK: SUDotAccSat %[[#]] %[[#SignedA]] %[[#UnsignedB]] %[[#]]
+; CHECK: SUDotAccSat %[[#]] %[[#SignedB]] %[[#UnsignedA]] %[[#]]
+; CHECK: UDotAccSat %[[#]] %[[#UnsignedA]] %[[#UnsignedB]] %[[#]]
+
+define spir_kernel void @test(<4 x i8> %ia, <4 x i8> %ua, <4 x i8> %ib, <4 x i8> %ub, <4 x i8> %ires, <4 x i8> %ures) {
+entry:
+  %call = tail call spir_func i32 @_Z3dotDv4_cS_(<4 x i8> %ia, <4 x i8> %ib) #2
+  %call1 = tail call spir_func i32 @_Z3dotDv4_cDv4_h(<4 x i8> %ia, <4 x i8> %ub) #2
+  %call2 = tail call spir_func i32 @_Z3dotDv4_hDv4_c(<4 x i8> %ua, <4 x i8> %ib) #2
+  %call3 = tail call spir_func i32 @_Z3dotDv4_hS_(<4 x i8> %ua, <4 x i8> %ub) #2
+  %call4 = tail call spir_func i32 @_Z11dot_acc_satDv4_cS_i(<4 x i8> %ia, <4 x i8> %ib, i32 %call2) #2
+  %call5 = tail call spir_func i32 @_Z11dot_acc_satDv4_cDv4_hi(<4 x i8> %ia, <4 x i8> %ub, i32 %call4) #2
+  %call6 = tail call spir_func i32 @_Z11dot_acc_satDv4_hDv4_ci(<4 x i8> %ua, <4 x i8> %ib, i32 %call5) #2
+  %call7 = tail call spir_func i32 @_Z11dot_acc_satDv4_hS_j(<4 x i8> %ua, <4 x i8> %ub, i32 %call3) #2
+  ret void
+}
+
+declare spir_func i32 @_Z3dotDv4_cS_(<4 x i8>, <4 x i8>)
+declare spir_func i32 @_Z3dotDv4_cDv4_h(<4 x i8>, <4 x i8>)
+declare spir_func i32 @_Z3dotDv4_hDv4_c(<4 x i8>, <4 x i8>)
+declare spir_func i32 @_Z3dotDv4_hS_(<4 x i8>, <4 x i8>)
+declare spir_func i32 @_Z11dot_acc_satDv4_cS_i(<4 x i8>, <4 x i8>, i32)
+declare spir_func i32 @_Z11dot_acc_satDv4_cDv4_hi(<4 x i8>, <4 x i8>, i32)
+declare spir_func i32 @_Z11dot_acc_satDv4_hDv4_ci(<4 x i8>, <4 x i8>, i32)
+declare spir_func i32 @_Z11dot_acc_satDv4_hS_j(<4 x i8>, <4 x i8>, i32)
+
+!llvm.module.flags = !{!0}
+!opencl.ocl.version = !{!1}
+!opencl.spir.version = !{!1}
+
+!0 = !{i32 1, !"wchar_size", i32 4}
+!1 = !{i32 2, i32 0}
diff --git a/llvm/test/CodeGen/SPIRV/extensions/SPV_KHR_integer_dot_product/SPV_KHR_integer_dot_product_OCLtoSPIRV_int.ll b/llvm/test/CodeGen/SPIRV/extensions/SPV_KHR_integer_dot_product/SPV_KHR_integer_dot_product_OCLtoSPIRV_int.ll
new file mode 100644
index 0000000000000..284f5c34671b7
--- /dev/null
+++ b/llvm/test/CodeGen/SPIRV/extensions/SPV_KHR_integer_dot_product/SPV_KHR_integer_dot_product_OCLtoSPIRV_int.ll
@@ -0,0 +1,53 @@
+; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv32v1.6-unknown-unknown %s -o - | FileCheck %s
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv32v1.6-unknown-unknown %s -o - -filetype=obj | spirv-val %}
+; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv32-unknown-unknown --spirv-ext=+SPV_KHR_integer_dot_product %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-EXT
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv32-unknown-unknown --spirv-ext=+SPV_KHR_integer_dot_product %s -o - -filetype=obj | spirv-val %}
+
+; CHECK: Capability DotProduct
+; CHECK: Capability DotProductInput4x8BitPacked
+; CHECK-EXT: OpExtension "SPV_KHR_integer_dot_product"
+; CHECK-NOT: OpExtension "SPV_KHR_integer_dot_product"
+
+; CHECK: Name %[[#SignedA:]] "ia"
+; CHECK: Name %[[#UnsignedA:]] "ua"
+; CHECK: Name %[[#SignedB:]] "ib"
+; CHECK: Name %[[#UnsignedB:]] "ub"
+
+; CHECK: SDot %[[#]] %[[#SignedA]] %[[#SignedB]] 0
+; CHECK: SUDot %[[#]] %[[#SignedA]] %[[#UnsignedB]] 0
+; CHECK: SUDot %[[#]] %[[#SignedB]] %[[#UnsignedA]] 0
+; CHECK: UDot %[[#]] %[[#UnsignedA]] %[[#UnsignedB]] 0
+
+; CHECK: SDotAccSat %[[#]] %[[#SignedA]] %[[#SignedB]] %[[#]] 0
+; CHECK: SUDotAccSat %[[#]] %[[#SignedA]] %[[#UnsignedB]] %[[#]] 0
+; CHECK: SUDotAccSat %[[#]] %[[#SignedB]] %[[#UnsignedA]] %[[#]] 0
+; CHECK: UDotAccSat %[[#]] %[[#UnsignedA]] %[[#UnsignedB]] %[[#]] 0
+
+define spir_kernel void @test(i32 %ia, i32 %ua, i32 %ib, i32 %ub, i32 %ires, i32 %ures) {
+entry:
+  %call = tail call spir_func i32 @_Z20dot_4x8packed_ss_intjj(i32 %ia, i32 %ib) #2
+  %call1 = tail call spir_func i32 @_Z20dot_4x8packed_su_intjj(i32 %ia, i32 %ub) #2
+  %call2 = tail call spir_func i32 @_Z20dot_4x8packed_us_intjj(i32 %ua, i32 %ib) #2
+  %call3 = tail call spir_func i32 @_Z21dot_4x8packed_uu_uintjj(i32 %ua, i32 %ub) #2
+  %call4 = tail call spir_func i32 @_Z28dot_acc_sat_4x8packed_ss_intjji(i32 %ia, i32 %ib, i32 %ires) #2
+  %call5 = tail call spir_func i32 @_Z28dot_acc_sat_4x8packed_su_intjji(i32 %ia, i32 %ub, i32 %ires) #2
+  %call6 = tail call spir_func i32 @_Z28dot_acc_sat_4x8packed_us_intjji(i32 %ua, i32 %ib, i32 %ires) #2
+  %call7 = tail call spir_func i32 @_Z29dot_acc_sat_4x8packed_uu_uintjjj(i32 %ua, i32 %ub, i32 %ures) #2
+  ret void
+}
+
+declare spir_func i32 @_Z20dot_4x8packed_ss_intjj(i32, i32)
+declare spir_func i32 @_Z20dot_4x8packed_su_intjj(i32, i32)
+declare spir_func i32 @_Z20dot_4x8packed_us_intjj(i32, i32)
+declare spir_func i32 @_Z21dot_4x8packed_uu_uintjj(i32, i32)
+declare spir_func i32 @_Z28dot_acc_sat_4x8packed_ss_intjji(i32, i32, i32)
+declare spir_func i32 @_Z28dot_acc_sat_4x8packed_su_intjji(i32, i32, i32)
+declare spir_func i32 @_Z28dot_acc_sat_4x8packed_us_intjji(i32, i32, i32)
+declare spir_func i32 @_Z29dot_acc_sat_4x8packed_uu_uintjjj(i32, i32, i32)
+
+!llvm.module.flags = !{!0}
+!opencl.ocl.version = !{!1}
+!opencl.spir.version = !{!1}
+
+!0 = !{i32 1, !"wchar_size", i32 4}
+!1 = !{i32 2, i32 0}
diff --git a/llvm/test/CodeGen/SPIRV/extensions/SPV_KHR_integer_dot_product/SPV_KHR_integer_dot_product_OCLtoSPIRV_short2.ll b/llvm/test/CodeGen/SPIRV/extensions/SPV_KHR_integer_dot_product/SPV_KHR_integer_dot_product_OCLtoSPIRV_short2.ll
new file mode 100644
index 0000000000000..8849453fe4d68
--- /dev/null
+++ b/llvm/test/CodeGen/SPIRV/extensions/SPV_KHR_integer_dot_product/SPV_KHR_integer_dot_product_OCLtoSPIRV_short2.ll
@@ -0,0 +1,53 @@
+; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv32v1.6-unknown-unknown %s -o - | FileCheck %s
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv32v1.6-unknown-unknown %s -o - -filetype=obj | spirv-val %}
+; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv32-unknown-unknown --spirv-ext=+SPV_KHR_integer_dot_product %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-EXT
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv32-unknown-unknown --spirv-ext=+SPV_KHR_integer_dot_product %s -o - -filetype=obj | spirv-val %}
+
+; CHECK: Capability DotProduct
+; CHECK: Capability DotProductInputAll
+; CHECK-EXT: OpExtension "SPV_KHR_integer_dot_product"
+; CHECK-NOT: OpExtension "SPV_KHR_integer_dot_product"
+
+; CHECK: Name %[[#SignedA:]] "ia"
+; CHECK: Name %[[#UnsignedA:]] "ua"
+; CHECK: Name %[[#SignedB:]] "ib"
+; CHECK: Name %[[#UnsignedB:]] "ub"
+
+; CHECK: SDot %[[#]] %[[#SignedA]] %[[#SignedB]]
+; CHECK: SUDot %[[#]] %[[#SignedA]] %[[#UnsignedB]]
+; CHECK: SUDot %[[#]] %[[#SignedB]] %[[#UnsignedA]]
+; CHECK: UDot %[[#]] %[[#UnsignedA]] %[[#UnsignedB]]
+
+; CHECK: SDotAccSat %[[#]] %[[#SignedA]] %[[#SignedB]] %[[#]]
+; CHECK: SUDotAccSat %[[#]] %[[#SignedA]] %[[#UnsignedB]] %[[#]]
+; CHECK: SUDotAccSat %[[#]] %[[#SignedB]] %[[#UnsignedA]] %[[#]]
+; CHECK: UDotAccSat %[[#]] %[[#UnsignedA]] %[[#UnsignedB]] %[[#]]
+
+define spir_kernel void @test(<2 x i16> %ia, <2 x i16> %ua, <2 x i16> %ib, <2 x i16> %ub, <2 x i16> %ires, <2 x i16> %ures) {
+entry:
+  %call = tail call spir_func i32 @_Z3dotDv2_sS_(<2 x i16> %ia, <2 x i16> %ib) #2
+  %call1 = tail call spir_func i32 @_Z3dotDv2_sDv2_t(<2 x i16> %ia, <2 x i16> %ub) #2
+  %call2 = tail call spir_func i32 @_Z3dotDv2_tDv2_s(<2 x i16> %ua, <2 x i16> %ib) #2
+  %call3 = tail call spir_func i32 @_Z3dotDv2_tS_(<2 x i16> %ua, <2 x i16> %ub) #2
+  %call4 = tail call spir_func i32 @_Z11dot_acc_satDv2_sS_i(<2 x i16> %ia, <2 x i16> %ib, i32 %call2) #2
+  %call5 = tail call spir_func i32 @_Z11dot_acc_satDv2_sDv2_ti(<2 x i16> %ia, <2 x i16> %ub, i32 %call4) #2
+  %call6 = tail call spir_func i32 @_Z11dot_acc_satDv2_tDv2_si(<2 x i16> %ua, <2 x i16> %ib, i32 %call5) #2
+  %call7 = tail call spir_func i32 @_Z11dot_acc_satDv2_tS_j(<2 x i16> %ua, <2 x i16> %ub, i32 %call3) #2
+  ret void
+}
+
+declare spir_func i32 @_Z3dotDv2_sS_(<2 x i16>, <2 x i16>)
+declare spir_func i32 @_Z3dotDv2_sDv2_t(<2 x i16>, <2 x i16>)
+declare spir_func i32 @_Z3dotDv2_tDv2_s(<2 x i16>, <2 x i16>)
+declare spir_func i32 @_Z3dotDv2_tS_(<2 x i16>, <2 x i16>)
+declare spir_func i32 @_Z11dot_acc_satDv2_sS_i(<2 x i16>, <2 x i16>, i32)
+declare spir_func i32 @_Z11dot_acc_satDv2_sDv2_ti(<2 x i16>, <2 x i16>, i32)
+declare spir_func i32 @_Z11dot_acc_satDv2_tDv2_si(<2 x i16>, <2 x i16>, i32)
+declare spir_func i32 @_Z11dot_acc_satDv2_tS_j(<2 x i16>, <2 x i16>, i32)
+
+!llvm.module.flags = !{!0}
+!opencl.ocl.version = !{!1}
+!opencl.spir.version = !{!1}
+
+!0 = !{i32 1, !"wchar_size", i32 4}
+!1 = !{i32 2, i32 0}
diff --git a/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/WaveActiveMax.ll b/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/WaveActiveMax.ll
new file mode 100644
index 0000000000000..b74e8c39a97aa
--- /dev/null
+++ b/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/WaveActiveMax.ll
@@ -0,0 +1,57 @@
+; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv-vulkan-unknown %s -o - | FileCheck %s
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-vulkan-unknown %s -o - -filetype=obj | spirv-val %}
+
+; Test lowering to spir-v backend for various types and scalar/vector
+
+; CHECK: OpCapability GroupNonUniformArithmetic
+
+; CHECK-DAG:   %[[#f16:]] = OpTypeFloat 16
+; CHECK-DAG:   %[[#f32:]] = OpTypeFloat 32
+; CHECK-DAG:   %[[#uint:]] = OpTypeInt 32 0
+; CHECK-DAG:   %[[#v4_half:]] = OpTypeVector %[[#f16]] 4
+; CHECK-DAG:   %[[#scope:]] = OpConstant %[[#uint]] 3
+
+; CHECK-LABEL: Begin function test_float
+; CHECK:   %[[#fexpr:]] = OpFunctionParameter %[[#f32]]
+define float @test_float(float %fexpr) {
+entry:
+; CHECK:   %[[#fret:]] = OpGroupNonUniformFMax %[[#f32]] %[[#scope]] Reduce %[[#fexpr]]
+  %0 = call float @llvm.spv.wave.reduce.max.f32(float %fexpr)
+  ret float %0
+}
+
+; CHECK-LABEL: Begin function test_int_signed
+; CHECK:   %[[#iexpr:]] = OpFunctionParameter %[[#uint]]
+define i32 @test_int_signed(i32 %iexpr) {
+entry:
+; CHECK:   %[[#iret:]] = OpGroupNonUniformSMax %[[#uint]] %[[#scope]] Reduce %[[#iexpr]]
+  %0 = call i32 @llvm.spv.wave.reduce.max.i32(i32 %iexpr)
+  ret i32 %0
+}
+
+; CHECK-LABEL: Begin function test_int_unsigned
+; CHECK:   %[[#iexpr:]] = OpFunctionParameter %[[#uint]]
+define i32 @test_int_unsigned(i32 %iexpr) {
+entry:
+; CHECK:   %[[#iret:]] = OpGroupNonUniformUMax %[[#uint]] %[[#scope]] Reduce %[[#iexpr]]
+  %0 = call i32 @llvm.spv.wave.reduce.umax.i32(i32 %iexpr)
+  ret i32 %0
+}
+
+; CHECK-LABEL: Begin function test_vhalf
+; CHECK:   %[[#vbexpr:]] = OpFunctionParameter %[[#v4_half]]
+define <4 x half> @test_vhalf(<4 x half> %vbexpr) {
+entry:
+; CHECK:   %[[#vhalfret:]] = OpGroupNonUniformFMax %[[#v4_half]] %[[#scope]] Reduce %[[#vbexpr]]
+  %0 = call <4 x half> @llvm.spv.wave.reduce.max.v4half(<4 x half> %vbexpr)
+  ret <4 x half> %0
+}
+
+declare float @llvm.spv.wave.reduce.max.f32(float)
+declare i32 @llvm.spv.wave.reduce.max.i32(i32)
+declare <4 x half> @llvm.spv.wave.reduce.max.v4half(<4 x half>)
+
+declare float @llvm.spv.wave.reduce.umax.f32(float)
+declare i32 @llvm.spv.wave.reduce.umax.i32(i32)
+declare <4 x half> @llvm.spv.wave.reduce.umax.v4half(<4 x half>)
+
diff --git a/llvm/test/CodeGen/SPIRV/hlsl-resources/BufferLoadStore.ll b/llvm/test/CodeGen/SPIRV/hlsl-resources/BufferLoadStore.ll
index 25dcc90cb61cd..d810ef9ccecc4 100644
--- a/llvm/test/CodeGen/SPIRV/hlsl-resources/BufferLoadStore.ll
+++ b/llvm/test/CodeGen/SPIRV/hlsl-resources/BufferLoadStore.ll
@@ -2,6 +2,7 @@
 ; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-vulkan-library %s -o - -filetype=obj | spirv-val %}
 
 ; CHECK-DAG: [[float:%[0-9]+]] = OpTypeFloat 32
+; CHECK-DAG: [[v2float:%[0-9]+]] = OpTypeVector [[float]] 2
 ; CHECK-DAG: [[v4float:%[0-9]+]] = OpTypeVector [[float]] 4
 ; CHECK-DAG: [[int:%[0-9]+]] = OpTypeInt 32 0
 ; CHECK-DAG: [[zero:%[0-9]+]] = OpConstant [[int]] 0
@@ -10,10 +11,11 @@
 ; CHECK-DAG: [[twenty_three:%[0-9]+]] = OpConstant [[int]] 23
 ; CHECK-DAG: [[ImageType:%[0-9]+]] = OpTypeImage [[float]] Buffer 2 0 0 2 Rgba32f 
 ; CHECK-DAG: [[ImagePtr:%[0-9]+]] = OpTypePointer UniformConstant [[ImageType]]
-; CHECK: [[Var:%[0-9]+]] = OpVariable [[ImagePtr]] UniformConstant
+; CHECK-DAG: [[Var:%[0-9]+]] = OpVariable [[ImagePtr]] UniformConstant
 
 ; Function Attrs: mustprogress nofree noinline norecurse nosync nounwind willreturn memory(readwrite, inaccessiblemem: none)
-define void @main() local_unnamed_addr #0 {
+; CHECK: OpFunction
+define void @main_scalar() local_unnamed_addr #0 {
 entry:
 ; CHECK: [[H:%[0-9]+]] = OpLoad [[ImageType]] [[Var]]
   %s_h.i = tail call target("spirv.Image", float, 5, 2, 0, 0, 2, 1) @llvm.spv.resource.handlefrombinding.tspirv.Image_f32_5_2_0_0_2_0t(i32 3, i32 5, i32 1, i32 0, i1 false)
@@ -50,6 +52,86 @@ bb_both:
   ret void
 }
 
+; Function Attrs: mustprogress nofree noinline norecurse nosync nounwind willreturn memory(readwrite, inaccessiblemem: none)
+; CHECK: OpFunction
+define void @main_vector2() local_unnamed_addr #0 {
+entry:
+; CHECK: [[H:%[0-9]+]] = OpLoad [[ImageType]] [[Var]]
+  %s_h.i = tail call target("spirv.Image", float, 5, 2, 0, 0, 2, 1) @llvm.spv.resource.handlefrombinding.tspirv.Image_f32_5_2_0_0_2_0t(i32 3, i32 5, i32 1, i32 0, i1 false)
+
+; CHECK: [[R:%[0-9]+]] = OpImageRead [[v4float]] [[H]] [[one]]
+; CHECK: [[E0:%[0-9]+]] = OpCompositeExtract [[float]] [[R]] 0
+; CHECK: [[E1:%[0-9]+]] = OpCompositeExtract [[float]] [[R]] 1
+; CHECK: [[V:%[0-9]+]] = OpCompositeConstruct [[v2float]] [[E0]] [[E1]]
+  %0 = tail call noundef nonnull align 4 dereferenceable(4) ptr @llvm.spv.resource.getpointer.p0.tspirv.Image_f32_5_2_0_0_2_0t(target("spirv.Image", float, 5, 2, 0, 0, 2, 1) %s_h.i, i32 1)
+  %1 = load <2 x float>, ptr %0, align 4
+; CHECK: OpBranch [[bb_store:%[0-9]+]]
+  br label %bb_store
+
+; CHECK: [[bb_store]] = OpLabel
+bb_store:
+
+; CHECK: [[H:%[0-9]+]] = OpLoad [[ImageType]] [[Var]]
+; CHECK: OpImageWrite [[H]] [[zero]] [[V]]
+  %2 = tail call noundef nonnull align 4 dereferenceable(4) ptr @llvm.spv.resource.getpointer.p0.tspirv.Image_f32_5_2_0_0_2_0t(target("spirv.Image", float, 5, 2, 0, 0, 2, 1) %s_h.i, i32 0)
+  store <2 x float> %1, ptr %2, align 4
+; CHECK: OpBranch [[bb_both:%[0-9]+]]
+  br label %bb_both
+
+; CHECK: [[bb_both]] = OpLabel
+bb_both:
+; CHECK: [[H:%[0-9]+]] = OpLoad [[ImageType]] [[Var]]
+; CHECK: [[R:%[0-9]+]] = OpImageRead [[v4float]] [[H]] [[twenty_three]]
+; CHECK: [[E0:%[0-9]+]] = OpCompositeExtract [[float]] [[R]] 0
+; CHECK: [[E1:%[0-9]+]] = OpCompositeExtract [[float]] [[R]] 1
+; CHECK: [[V:%[0-9]+]] = OpCompositeConstruct [[v2float]] [[E0]] [[E1]]
+  %3 = tail call noundef nonnull align 4 dereferenceable(4) ptr @llvm.spv.resource.getpointer.p0.tspirv.Image_f32_5_2_0_0_2_0t(target("spirv.Image", float, 5, 2, 0, 0, 2, 1) %s_h.i, i32 23)
+  %4 = load <2 x float>, ptr %3, align 4
+
+; CHECK: [[H:%[0-9]+]] = OpLoad [[ImageType]] [[Var]]
+; CHECK: OpImageWrite [[H]] [[twenty]] [[V]]
+  %5 = tail call noundef nonnull align 4 dereferenceable(4) ptr @llvm.spv.resource.getpointer.p0.tspirv.Image_f32_5_2_0_0_2_0t(target("spirv.Image", float, 5, 2, 0, 0, 2, 1) %s_h.i, i32 20)
+  store <2 x float> %4, ptr %5, align 4
+  ret void
+}
+
+; Function Attrs: mustprogress nofree noinline norecurse nosync nounwind willreturn memory(readwrite, inaccessiblemem: none)
+; CHECK: OpFunction
+define void @main_vector4() local_unnamed_addr #0 {
+entry:
+; CHECK: [[H:%[0-9]+]] = OpLoad [[ImageType]] [[Var]]
+  %s_h.i = tail call target("spirv.Image", float, 5, 2, 0, 0, 2, 1) @llvm.spv.resource.handlefrombinding.tspirv.Image_f32_5_2_0_0_2_0t(i32 3, i32 5, i32 1, i32 0, i1 false)
+
+; CHECK: [[R:%[0-9]+]] = OpImageRead [[v4float]] [[H]] [[one]]
+  %0 = tail call noundef nonnull align 4 dereferenceable(4) ptr @llvm.spv.resource.getpointer.p0.tspirv.Image_f32_5_2_0_0_2_0t(target("spirv.Image", float, 5, 2, 0, 0, 2, 1) %s_h.i, i32 1)
+  %1 = load <4 x float>, ptr %0, align 4
+; CHECK: OpBranch [[bb_store:%[0-9]+]]
+  br label %bb_store
+
+; CHECK: [[bb_store]] = OpLabel
+bb_store:
+
+; CHECK: [[H:%[0-9]+]] = OpLoad [[ImageType]] [[Var]]
+; CHECK: OpImageWrite [[H]] [[zero]] [[R]]
+  %2 = tail call noundef nonnull align 4 dereferenceable(4) ptr @llvm.spv.resource.getpointer.p0.tspirv.Image_f32_5_2_0_0_2_0t(target("spirv.Image", float, 5, 2, 0, 0, 2, 1) %s_h.i, i32 0)
+  store <4 x float> %1, ptr %2, align 4
+; CHECK: OpBranch [[bb_both:%[0-9]+]]
+  br label %bb_both
+
+; CHECK: [[bb_both]] = OpLabel
+bb_both:
+; CHECK: [[H:%[0-9]+]] = OpLoad [[ImageType]] [[Var]]
+; CHECK: [[R:%[0-9]+]] = OpImageRead [[v4float]] [[H]] [[twenty_three]]
+  %3 = tail call noundef nonnull align 4 dereferenceable(4) ptr @llvm.spv.resource.getpointer.p0.tspirv.Image_f32_5_2_0_0_2_0t(target("spirv.Image", float, 5, 2, 0, 0, 2, 1) %s_h.i, i32 23)
+  %4 = load <4 x float>, ptr %3, align 4
+
+; CHECK: [[H:%[0-9]+]] = OpLoad [[ImageType]] [[Var]]
+; CHECK: OpImageWrite [[H]] [[twenty]] [[R]]
+  %5 = tail call noundef nonnull align 4 dereferenceable(4) ptr @llvm.spv.resource.getpointer.p0.tspirv.Image_f32_5_2_0_0_2_0t(target("spirv.Image", float, 5, 2, 0, 0, 2, 1) %s_h.i, i32 20)
+  store <4 x float> %4, ptr %5, align 4
+  ret void
+}
+
 ; Function Attrs: mustprogress nocallback nofree nosync nounwind willreturn memory(none)
 declare ptr @llvm.spv.resource.getpointer.p0.tspirv.Image_f32_5_2_0_0_2_0t(target("spirv.Image", float, 5, 2, 0, 0, 2, 1), i32) #1
 
diff --git a/llvm/test/CodeGen/SPIRV/lit.local.cfg b/llvm/test/CodeGen/SPIRV/lit.local.cfg
index 4655633a25682..f139d13268af2 100644
--- a/llvm/test/CodeGen/SPIRV/lit.local.cfg
+++ b/llvm/test/CodeGen/SPIRV/lit.local.cfg
@@ -16,3 +16,4 @@ if config.spirv_tools_tests:
     config.substitutions.append(("spirv-dis", os.path.join(config.llvm_tools_dir, "spirv-dis")))
     config.substitutions.append(("spirv-val", os.path.join(config.llvm_tools_dir, "spirv-val")))
     config.substitutions.append(("spirv-as", os.path.join(config.llvm_tools_dir, "spirv-as")))
+    config.substitutions.append(("spirv-link", os.path.join(config.llvm_tools_dir, "spirv-link")))
diff --git a/llvm/test/CodeGen/SystemZ/branch-05.ll b/llvm/test/CodeGen/SystemZ/branch-05.ll
index 4a4aa2a9eb36f..30fbce0c3f31f 100644
--- a/llvm/test/CodeGen/SystemZ/branch-05.ll
+++ b/llvm/test/CodeGen/SystemZ/branch-05.ll
@@ -9,7 +9,7 @@ define i32 @f1(i32 %x, i32 %y, i32 %op) {
 ; CHECK: llgfr [[OP64:%r[0-5]]], %r4
 ; CHECK: sllg [[INDEX:%r[1-5]]], [[OP64]], 3
 ; CHECK: larl [[BASE:%r[1-5]]]
-; CHECK: lg [[TARGET:%r[1-5]]], 0([[BASE]],[[INDEX]])
+; CHECK: lg [[TARGET:%r[1-5]]], 0([[INDEX]],[[BASE]])
 ; CHECK: br [[TARGET]]
 entry:
   switch i32 %op, label %exit [
diff --git a/llvm/test/CodeGen/SystemZ/branch-11.ll b/llvm/test/CodeGen/SystemZ/branch-11.ll
index ce7b3ef267b49..75ba4a3e541a8 100644
--- a/llvm/test/CodeGen/SystemZ/branch-11.ll
+++ b/llvm/test/CodeGen/SystemZ/branch-11.ll
@@ -9,7 +9,7 @@ define i32 @f1(i32 %x, i32 %y, i32 %op) {
 ; CHECK: llgfr [[OP64:%r[0-5]]], %r4
 ; CHECK: sllg [[INDEX:%r[1-5]]], [[OP64]], 3
 ; CHECK: larl [[BASE:%r[1-5]]]
-; CHECK: bi 0([[BASE]],[[INDEX]])
+; CHECK: bi 0([[INDEX]],[[BASE]])
 entry:
   switch i32 %op, label %exit [
     i32 1, label %b.add
diff --git a/llvm/test/CodeGen/SystemZ/cond-move-10.mir b/llvm/test/CodeGen/SystemZ/cond-move-10.mir
new file mode 100644
index 0000000000000..1db960829729e
--- /dev/null
+++ b/llvm/test/CodeGen/SystemZ/cond-move-10.mir
@@ -0,0 +1,21 @@
+# RUN: llc -o - %s -mtriple=s390x-linux-gnu -mcpu=z15 -run-pass=systemz-post-rewrite \
+# RUN:   2>&1 | FileCheck %s
+
+# The SELRMux has two identical sources - replace with a copy instruction.
+# CHECK:      name: fun0
+# CHECK:      renamable $r1l = AHIMuxK killed renamable $r1l, -1, implicit-def dead $cc
+# CHECK-NEXT: CHIMux renamable $r5h, 9, implicit-def $cc
+# CHECK-NEXT: $r14h = COPY killed renamable $r1l
+---
+name:            fun0
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    liveins: $r1l, $r5h
+    renamable $r1l = AHIMuxK killed renamable $r1l, -1, implicit-def dead $cc
+    CHIMux renamable $r5h, 9, implicit-def $cc
+    renamable $r14h = SELRMux killed renamable $r1l, renamable $r1l, 14, 8, implicit $cc
+    $r14l = COPY killed renamable $r14h
+    $r14d = LGFR $r14l
+    Return implicit $r14d
+...
diff --git a/llvm/test/CodeGen/SystemZ/copy-phys-reg-fp64-to-gr64.mir b/llvm/test/CodeGen/SystemZ/copy-phys-reg-fp64-to-gr64.mir
new file mode 100644
index 0000000000000..6fd09d25216e7
--- /dev/null
+++ b/llvm/test/CodeGen/SystemZ/copy-phys-reg-fp64-to-gr64.mir
@@ -0,0 +1,48 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 4
+# RUN: llc -mtriple=s390x-ibm-linux -mcpu=z13 -run-pass=postrapseudos -o - %s | FileCheck %s
+
+---
+name:            copy_fp64_to_gr64__r1d_to_f3d
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    liveins: $r1d
+    ; CHECK-LABEL: name: copy_fp64_to_gr64__r1d_to_f3d
+    ; CHECK: liveins: $r1d
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: $f3d = LDGR $r1d
+    ; CHECK-NEXT: Return implicit $f3d
+    $f3d = COPY $r1d
+    Return implicit $f3d
+...
+
+---
+name:            copy_fp64_to_gr64__r1d_to_f3d_undef
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    liveins: $r1d
+    ; CHECK-LABEL: name: copy_fp64_to_gr64__r1d_to_f3d_undef
+    ; CHECK: liveins: $r1d
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: $f3d = KILL undef $r1d
+    ; CHECK-NEXT: Return implicit $f3d
+    $f3d = COPY undef $r1d
+    Return implicit $f3d
+...
+
+---
+name:            copy_fp64_to_gr64__r1d_to_f3d_killed
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    liveins: $r1d
+    ; CHECK-LABEL: name: copy_fp64_to_gr64__r1d_to_f3d_killed
+    ; CHECK: liveins: $r1d
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: $f3d = LDGR killed $r1d
+    ; CHECK-NEXT: Return implicit $f3d
+    $f3d = COPY killed $r1d
+    Return implicit $f3d
+...
+
diff --git a/llvm/test/CodeGen/SystemZ/copy-phys-reg-gr64-to-fp64.mir b/llvm/test/CodeGen/SystemZ/copy-phys-reg-gr64-to-fp64.mir
new file mode 100644
index 0000000000000..07ef93415bb79
--- /dev/null
+++ b/llvm/test/CodeGen/SystemZ/copy-phys-reg-gr64-to-fp64.mir
@@ -0,0 +1,47 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 4
+# RUN: llc -mtriple=s390x-ibm-linux -mcpu=z13 -run-pass=postrapseudos -o - %s | FileCheck %s
+---
+name:            copy_fp64_to_gr64__f3d_to_r1d
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    liveins: $f3d
+    ; CHECK-LABEL: name: copy_fp64_to_gr64__f3d_to_r1d
+    ; CHECK: liveins: $f3d
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: $r1d = LGDR $f3d
+    ; CHECK-NEXT: Return implicit $r1d
+    $r1d = COPY $f3d
+    Return implicit $r1d
+...
+
+---
+name:            copy_fp64_to_gr64__f3d_to_r1d_undef
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    liveins: $f3d
+    ; CHECK-LABEL: name: copy_fp64_to_gr64__f3d_to_r1d_undef
+    ; CHECK: liveins: $f3d
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: $r1d = KILL undef $f3d
+    ; CHECK-NEXT: Return implicit $r1d
+    $r1d = COPY undef $f3d
+    Return implicit $r1d
+...
+
+---
+name:            copy_fp64_to_gr64__f3d_to_r1d_killed
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    liveins: $f3d
+    ; CHECK-LABEL: name: copy_fp64_to_gr64__f3d_to_r1d_killed
+    ; CHECK: liveins: $f3d
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: $r1d = LGDR killed $f3d
+    ; CHECK-NEXT: Return implicit $r1d
+    $r1d = COPY killed $f3d
+    Return implicit $r1d
+...
+
diff --git a/llvm/test/CodeGen/SystemZ/int-uadd-01.ll b/llvm/test/CodeGen/SystemZ/int-uadd-01.ll
index 7d4aa9f321848..ec36065737fbf 100644
--- a/llvm/test/CodeGen/SystemZ/int-uadd-01.ll
+++ b/llvm/test/CodeGen/SystemZ/int-uadd-01.ll
@@ -271,10 +271,10 @@ define zeroext i1 @f14(ptr %ptr0) {
 ; CHECK-NEXT:    .cfi_offset %r15, -40
 ; CHECK-NEXT:    aghi %r15, -168
 ; CHECK-NEXT:    .cfi_def_cfa_offset 328
-; CHECK-NEXT:    l %r6, 0(%r2)
-; CHECK-NEXT:    l %r13, 8(%r2)
-; CHECK-NEXT:    l %r12, 16(%r2)
-; CHECK-NEXT:    l %r7, 24(%r2)
+; CHECK-NEXT:    l %r7, 0(%r2)
+; CHECK-NEXT:    l %r6, 8(%r2)
+; CHECK-NEXT:    l %r13, 16(%r2)
+; CHECK-NEXT:    l %r12, 24(%r2)
 ; CHECK-NEXT:    l %r8, 32(%r2)
 ; CHECK-NEXT:    l %r9, 40(%r2)
 ; CHECK-NEXT:    l %r10, 48(%r2)
@@ -282,16 +282,16 @@ define zeroext i1 @f14(ptr %ptr0) {
 ; CHECK-NEXT:    mvc 160(4,%r15), 64(%r2) # 4-byte Folded Spill
 ; CHECK-NEXT:    mvc 164(4,%r15), 72(%r2) # 4-byte Folded Spill
 ; CHECK-NEXT:    brasl %r14, foo@PLT
-; CHECK-NEXT:    alr %r2, %r6
+; CHECK-NEXT:    alr %r2, %r7
 ; CHECK-NEXT:    ipm %r0
 ; CHECK-NEXT:    risbg %r0, %r0, 63, 191, 35
-; CHECK-NEXT:    alr %r2, %r13
+; CHECK-NEXT:    alr %r2, %r6
 ; CHECK-NEXT:    ipm %r1
 ; CHECK-NEXT:    rosbg %r0, %r1, 63, 63, 35
-; CHECK-NEXT:    alr %r2, %r12
+; CHECK-NEXT:    alr %r2, %r13
 ; CHECK-NEXT:    ipm %r1
 ; CHECK-NEXT:    rosbg %r0, %r1, 63, 63, 35
-; CHECK-NEXT:    alr %r2, %r7
+; CHECK-NEXT:    alr %r2, %r12
 ; CHECK-NEXT:    ipm %r1
 ; CHECK-NEXT:    rosbg %r0, %r1, 63, 63, 35
 ; CHECK-NEXT:    alr %r2, %r8
diff --git a/llvm/test/CodeGen/SystemZ/int-uadd-02.ll b/llvm/test/CodeGen/SystemZ/int-uadd-02.ll
index 46c5b4ff35fc0..ad6aa79feac47 100644
--- a/llvm/test/CodeGen/SystemZ/int-uadd-02.ll
+++ b/llvm/test/CodeGen/SystemZ/int-uadd-02.ll
@@ -215,10 +215,10 @@ define zeroext i1 @f11(ptr %ptr0) {
 ; CHECK-NEXT:    .cfi_offset %r15, -40
 ; CHECK-NEXT:    aghi %r15, -176
 ; CHECK-NEXT:    .cfi_def_cfa_offset 336
-; CHECK-NEXT:    lg %r6, 0(%r2)
-; CHECK-NEXT:    lg %r13, 16(%r2)
-; CHECK-NEXT:    lg %r12, 32(%r2)
-; CHECK-NEXT:    lg %r7, 48(%r2)
+; CHECK-NEXT:    lg %r7, 0(%r2)
+; CHECK-NEXT:    lg %r6, 16(%r2)
+; CHECK-NEXT:    lg %r13, 32(%r2)
+; CHECK-NEXT:    lg %r12, 48(%r2)
 ; CHECK-NEXT:    lg %r8, 64(%r2)
 ; CHECK-NEXT:    lg %r9, 80(%r2)
 ; CHECK-NEXT:    lg %r10, 96(%r2)
@@ -226,16 +226,16 @@ define zeroext i1 @f11(ptr %ptr0) {
 ; CHECK-NEXT:    mvc 160(8,%r15), 128(%r2) # 8-byte Folded Spill
 ; CHECK-NEXT:    mvc 168(8,%r15), 144(%r2) # 8-byte Folded Spill
 ; CHECK-NEXT:    brasl %r14, foo@PLT
-; CHECK-NEXT:    algr %r2, %r6
+; CHECK-NEXT:    algr %r2, %r7
 ; CHECK-NEXT:    ipm %r0
 ; CHECK-NEXT:    risbg %r0, %r0, 63, 191, 35
-; CHECK-NEXT:    algr %r2, %r13
+; CHECK-NEXT:    algr %r2, %r6
 ; CHECK-NEXT:    ipm %r1
 ; CHECK-NEXT:    rosbg %r0, %r1, 63, 63, 35
-; CHECK-NEXT:    algr %r2, %r12
+; CHECK-NEXT:    algr %r2, %r13
 ; CHECK-NEXT:    ipm %r1
 ; CHECK-NEXT:    rosbg %r0, %r1, 63, 63, 35
-; CHECK-NEXT:    algr %r2, %r7
+; CHECK-NEXT:    algr %r2, %r12
 ; CHECK-NEXT:    ipm %r1
 ; CHECK-NEXT:    rosbg %r0, %r1, 63, 63, 35
 ; CHECK-NEXT:    algr %r2, %r8
diff --git a/llvm/test/CodeGen/SystemZ/pr60413.ll b/llvm/test/CodeGen/SystemZ/pr60413.ll
index 62f5d49192ea9..6dee3d303a6f2 100644
--- a/llvm/test/CodeGen/SystemZ/pr60413.ll
+++ b/llvm/test/CodeGen/SystemZ/pr60413.ll
@@ -40,38 +40,38 @@ define dso_local void @m() local_unnamed_addr #1 {
 ; CHECK-NEXT:    vn %v0, %v0, %v6
 ; CHECK-NEXT:    vn %v4, %v4, %v6
 ; CHECK-NEXT:    vperm %v1, %v1, %v1, %v5
-; CHECK-NEXT:    vn %v5, %v1, %v6
-; CHECK-NEXT:    vperm %v1, %v0, %v3, %v2
-; CHECK-NEXT:    vn %v2, %v1, %v6
-; CHECK-NEXT:    vrepif %v1, 127
-; CHECK-NEXT:    vchlf %v3, %v5, %v1
-; CHECK-NEXT:    vlgvf %r3, %v3, 1
-; CHECK-NEXT:    vlgvf %r2, %v3, 0
+; CHECK-NEXT:    vn %v1, %v1, %v6
+; CHECK-NEXT:    vperm %v2, %v0, %v3, %v2
+; CHECK-NEXT:    vn %v2, %v2, %v6
+; CHECK-NEXT:    vrepif %v3, 127
+; CHECK-NEXT:    vchlf %v1, %v1, %v3
+; CHECK-NEXT:    vlgvf %r3, %v1, 1
+; CHECK-NEXT:    vlgvf %r2, %v1, 0
 ; CHECK-NEXT:    risbg %r2, %r2, 48, 176, 15
 ; CHECK-NEXT:    rosbg %r2, %r3, 49, 49, 14
-; CHECK-NEXT:    vlgvf %r3, %v3, 2
+; CHECK-NEXT:    vlgvf %r3, %v1, 2
 ; CHECK-NEXT:    rosbg %r2, %r3, 50, 50, 13
-; CHECK-NEXT:    vlgvf %r3, %v3, 3
+; CHECK-NEXT:    vlgvf %r3, %v1, 3
 ; CHECK-NEXT:    rosbg %r2, %r3, 51, 51, 12
-; CHECK-NEXT:    vchlf %v3, %v4, %v1
-; CHECK-NEXT:    vlgvf %r3, %v3, 0
+; CHECK-NEXT:    vchlf %v1, %v4, %v3
+; CHECK-NEXT:    vlgvf %r3, %v1, 0
 ; CHECK-NEXT:    rosbg %r2, %r3, 52, 52, 11
-; CHECK-NEXT:    vlgvf %r3, %v3, 1
+; CHECK-NEXT:    vlgvf %r3, %v1, 1
 ; CHECK-NEXT:    rosbg %r2, %r3, 53, 53, 10
-; CHECK-NEXT:    vlgvf %r3, %v3, 2
+; CHECK-NEXT:    vlgvf %r3, %v1, 2
 ; CHECK-NEXT:    rosbg %r2, %r3, 54, 54, 9
-; CHECK-NEXT:    vlgvf %r3, %v3, 3
+; CHECK-NEXT:    vlgvf %r3, %v1, 3
 ; CHECK-NEXT:    rosbg %r2, %r3, 55, 55, 8
-; CHECK-NEXT:    vchlf %v2, %v2, %v1
-; CHECK-NEXT:    vlgvf %r3, %v2, 0
+; CHECK-NEXT:    vchlf %v1, %v2, %v3
+; CHECK-NEXT:    vlgvf %r3, %v1, 0
 ; CHECK-NEXT:    rosbg %r2, %r3, 56, 56, 7
-; CHECK-NEXT:    vlgvf %r3, %v2, 1
+; CHECK-NEXT:    vlgvf %r3, %v1, 1
 ; CHECK-NEXT:    rosbg %r2, %r3, 57, 57, 6
-; CHECK-NEXT:    vlgvf %r3, %v2, 2
+; CHECK-NEXT:    vlgvf %r3, %v1, 2
 ; CHECK-NEXT:    rosbg %r2, %r3, 58, 58, 5
-; CHECK-NEXT:    vlgvf %r3, %v2, 3
+; CHECK-NEXT:    vlgvf %r3, %v1, 3
 ; CHECK-NEXT:    rosbg %r2, %r3, 59, 59, 4
-; CHECK-NEXT:    vchlf %v0, %v0, %v1
+; CHECK-NEXT:    vchlf %v0, %v0, %v3
 ; CHECK-NEXT:    vlgvf %r3, %v0, 0
 ; CHECK-NEXT:    rosbg %r2, %r3, 60, 60, 3
 ; CHECK-NEXT:    vlgvf %r3, %v0, 1
diff --git a/llvm/test/CodeGen/SystemZ/rda-stack-copy.mir b/llvm/test/CodeGen/SystemZ/rda-stack-copy.mir
new file mode 100644
index 0000000000000..a93b59324a8e5
--- /dev/null
+++ b/llvm/test/CodeGen/SystemZ/rda-stack-copy.mir
@@ -0,0 +1,30 @@
+# RUN: llc %s -mtriple=s390x-linux-gnu -run-pass=reaching-defs-analysis -print-all-reaching-defs -o - 2>&1 | FileCheck %s
+
+---
+name:            test0
+tracksRegLiveness: true
+stack:
+  - { id: 0, name: '', type: default, offset: 0, size: 4, alignment: 4,
+      stack-id: default, callee-saved-register: '', callee-saved-restored: true,
+      debug-info-variable: '', debug-info-expression: '', debug-info-location: '' }
+  - { id: 1, name: '', type: default, offset: 0, size: 4, alignment: 4,
+      stack-id: default, callee-saved-register: '', callee-saved-restored: true,
+      debug-info-variable: '', debug-info-expression: '', debug-info-location: '' }
+body:             |
+  ; CHECK-LABEL: RDA results for test0
+  ; CHECK-NEXT: %stack.1:{ }
+  ; CHECK-NEXT: %stack.0:{ }
+  ; CHECK-NEXT: 0: MVC %stack.1, 0, 4, %stack.0, 0 :: (store (s32) into %stack.1), (load (s32) from %stack.0)
+  ; CHECK-EMPTY: 
+  ; CHECK-NEXT: %stack.1:{ 0 }
+  ; CHECK-NEXT: 1: $r2l = L %stack.1, 0, $noreg
+  ; CHECK-EMPTY: 
+  ; CHECK-NEXT: implicit $r2l:{ 1 }
+  ; CHECK-NEXT: 2: Return implicit $r2l
+
+  bb.0:
+    MVC %stack.1, 0 , 4, %stack.0, 0:: (store (s32) into %stack.1), (load (s32) from %stack.0)
+    $r2l = L %stack.1, 0, $noreg
+    Return implicit $r2l
+
+...
diff --git a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/fast-fp-loops.ll b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/fast-fp-loops.ll
index 6cb98557c9bc1..ca1f1036ea674 100644
--- a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/fast-fp-loops.ll
+++ b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/fast-fp-loops.ll
@@ -24,8 +24,8 @@ define arm_aapcs_vfpcc void @fast_float_mul(ptr nocapture %a, ptr nocapture read
 ; CHECK-NEXT:    cmpeq.w r12, #0
 ; CHECK-NEXT:    beq .LBB0_4
 ; CHECK-NEXT:  @ %bb.2: @ %for.body.preheader
-; CHECK-NEXT:    subs r4, r3, #1
 ; CHECK-NEXT:    and r12, r3, #3
+; CHECK-NEXT:    subs r4, r3, #1
 ; CHECK-NEXT:    cmp r4, #3
 ; CHECK-NEXT:    bhs .LBB0_6
 ; CHECK-NEXT:  @ %bb.3:
diff --git a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/minloop.ll b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/minloop.ll
index 44cbd7d65125e..9c36bae6fac13 100644
--- a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/minloop.ll
+++ b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/minloop.ll
@@ -10,8 +10,8 @@ define void @arm_min_q31(ptr nocapture readonly %pSrc, i32 %blockSize, ptr nocap
 ; CHECK-NEXT:    subs.w r9, r1, #1
 ; CHECK-NEXT:    beq .LBB0_3
 ; CHECK-NEXT:  @ %bb.1: @ %while.body.preheader
-; CHECK-NEXT:    subs r7, r1, #2
 ; CHECK-NEXT:    and r8, r9, #3
+; CHECK-NEXT:    subs r7, r1, #2
 ; CHECK-NEXT:    cmp r7, #3
 ; CHECK-NEXT:    bhs .LBB0_4
 ; CHECK-NEXT:  @ %bb.2:
diff --git a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/mve-float-loops.ll b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/mve-float-loops.ll
index c0bc34c2b06ef..7c6c7e90413b1 100644
--- a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/mve-float-loops.ll
+++ b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/mve-float-loops.ll
@@ -1411,8 +1411,8 @@ define arm_aapcs_vfpcc float @half_half_mac(ptr nocapture readonly %a, ptr nocap
 ; CHECK-NEXT:    push {r4, r5, r7, lr}
 ; CHECK-NEXT:    cbz r2, .LBB9_3
 ; CHECK-NEXT:  @ %bb.1: @ %for.body.preheader
-; CHECK-NEXT:    subs r3, r2, #1
 ; CHECK-NEXT:    and r12, r2, #3
+; CHECK-NEXT:    subs r3, r2, #1
 ; CHECK-NEXT:    cmp r3, #3
 ; CHECK-NEXT:    bhs .LBB9_4
 ; CHECK-NEXT:  @ %bb.2:
@@ -1566,8 +1566,8 @@ define arm_aapcs_vfpcc float @half_half_acc(ptr nocapture readonly %a, ptr nocap
 ; CHECK-NEXT:    push {r4, r5, r7, lr}
 ; CHECK-NEXT:    cbz r2, .LBB10_3
 ; CHECK-NEXT:  @ %bb.1: @ %for.body.preheader
-; CHECK-NEXT:    subs r3, r2, #1
 ; CHECK-NEXT:    and r12, r2, #3
+; CHECK-NEXT:    subs r3, r2, #1
 ; CHECK-NEXT:    cmp r3, #3
 ; CHECK-NEXT:    bhs .LBB10_4
 ; CHECK-NEXT:  @ %bb.2:
@@ -1721,8 +1721,8 @@ define arm_aapcs_vfpcc float @half_short_mac(ptr nocapture readonly %a, ptr noca
 ; CHECK-NEXT:    push {r4, r5, r6, lr}
 ; CHECK-NEXT:    cbz r2, .LBB11_3
 ; CHECK-NEXT:  @ %bb.1: @ %for.body.preheader
-; CHECK-NEXT:    subs r3, r2, #1
 ; CHECK-NEXT:    and r12, r2, #3
+; CHECK-NEXT:    subs r3, r2, #1
 ; CHECK-NEXT:    cmp r3, #3
 ; CHECK-NEXT:    bhs .LBB11_4
 ; CHECK-NEXT:  @ %bb.2:
diff --git a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/mve-tail-data-types.ll b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/mve-tail-data-types.ll
index 715f6565199ec..8a5a15a57912c 100644
--- a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/mve-tail-data-types.ll
+++ b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/mve-tail-data-types.ll
@@ -348,8 +348,8 @@ define arm_aapcs_vfpcc void @test_vec_mul_scalar_add_char(ptr nocapture readonly
 ; CHECK-NEXT:    cmpeq r7, #0
 ; CHECK-NEXT:    beq .LBB5_4
 ; CHECK-NEXT:  @ %bb.2: @ %for.body.preheader
-; CHECK-NEXT:    subs r7, r4, #1
 ; CHECK-NEXT:    and r12, r4, #3
+; CHECK-NEXT:    subs r7, r4, #1
 ; CHECK-NEXT:    cmp r7, #3
 ; CHECK-NEXT:    bhs .LBB5_6
 ; CHECK-NEXT:  @ %bb.3:
@@ -624,8 +624,8 @@ define arm_aapcs_vfpcc void @test_vec_mul_scalar_add_uchar(ptr nocapture readonl
 ; CHECK-NEXT:    cmpeq r7, #0
 ; CHECK-NEXT:    beq .LBB7_4
 ; CHECK-NEXT:  @ %bb.2: @ %for.body.preheader
-; CHECK-NEXT:    subs r7, r4, #1
 ; CHECK-NEXT:    and r12, r4, #3
+; CHECK-NEXT:    subs r7, r4, #1
 ; CHECK-NEXT:    cmp r7, #3
 ; CHECK-NEXT:    bhs .LBB7_6
 ; CHECK-NEXT:  @ %bb.3:
@@ -900,8 +900,8 @@ define arm_aapcs_vfpcc void @test_vec_mul_scalar_add_int(ptr nocapture readonly
 ; CHECK-NEXT:    cmpeq r7, #0
 ; CHECK-NEXT:    beq .LBB9_4
 ; CHECK-NEXT:  @ %bb.2: @ %for.body.preheader
-; CHECK-NEXT:    subs r7, r4, #1
 ; CHECK-NEXT:    and r12, r4, #3
+; CHECK-NEXT:    subs r7, r4, #1
 ; CHECK-NEXT:    cmp r7, #3
 ; CHECK-NEXT:    bhs .LBB9_6
 ; CHECK-NEXT:  @ %bb.3:
diff --git a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/reductions.ll b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/reductions.ll
index 3c4af10b12438..6f986ce28381b 100644
--- a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/reductions.ll
+++ b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/reductions.ll
@@ -446,8 +446,8 @@ define dso_local arm_aapcs_vfpcc i32 @two_loops_mul_add_v4i32(ptr nocapture read
 ; CHECK-NEXT:    movs r3, #0
 ; CHECK-NEXT:    vdup.32 q0, r3
 ; CHECK-NEXT:    movs r3, #1
-; CHECK-NEXT:    add.w r3, r3, r7, lsr #2
 ; CHECK-NEXT:    vmov.32 q0[0], r12
+; CHECK-NEXT:    add.w r3, r3, r7, lsr #2
 ; CHECK-NEXT:    dls lr, r3
 ; CHECK-NEXT:  .LBB6_5: @ %vector.body46
 ; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
diff --git a/llvm/test/CodeGen/Thumb2/avoidmuls.mir b/llvm/test/CodeGen/Thumb2/avoidmuls.mir
index 865152068fdf7..7bfa4b9060063 100644
--- a/llvm/test/CodeGen/Thumb2/avoidmuls.mir
+++ b/llvm/test/CodeGen/Thumb2/avoidmuls.mir
@@ -1,4 +1,5 @@
 # RUN: llc -mtriple=thumbv7m-none-eabi -mcpu=cortex-m33 -run-pass=thumb2-reduce-size %s -o - | FileCheck %s --check-prefix=MUL
+# RUN: llc -mtriple=thumbv7m-none-eabi -mcpu=star-mc1 -run-pass=thumb2-reduce-size %s -o - | FileCheck %s --check-prefix=MUL
 # RUN: llc -mtriple=thumbv7m-none-eabi --run-pass=thumb2-reduce-size %s -o - | FileCheck %s --check-prefix=MULS
 
 ---
@@ -17,4 +18,4 @@ body:             |
 
 # MULS-LABEL: test
 # MULS: tMUL
-# MULS-NOT: t2MUL
\ No newline at end of file
+# MULS-NOT: t2MUL
diff --git a/llvm/test/CodeGen/Thumb2/mve-complex-deinterleaving-uniform-cases.ll b/llvm/test/CodeGen/Thumb2/mve-complex-deinterleaving-uniform-cases.ll
index c5f61b7fcdde5..dc67abc1be07e 100644
--- a/llvm/test/CodeGen/Thumb2/mve-complex-deinterleaving-uniform-cases.ll
+++ b/llvm/test/CodeGen/Thumb2/mve-complex-deinterleaving-uniform-cases.ll
@@ -212,70 +212,70 @@ define arm_aapcs_vfpcc <12 x float> @abp90c12(<12 x float> %a, <12 x float> %b,
 ; CHECK-NEXT:    vpush {d8, d9, d10, d11, d12, d13, d14, d15}
 ; CHECK-NEXT:    .pad #64
 ; CHECK-NEXT:    sub sp, #64
-; CHECK-NEXT:    vldr s23, [sp, #140]
-; CHECK-NEXT:    vmov.f32 s20, s13
-; CHECK-NEXT:    vldr s22, [sp, #132]
-; CHECK-NEXT:    vmov.f32 s25, s11
-; CHECK-NEXT:    vmov.f32 s13, s10
+; CHECK-NEXT:    vmov.f32 s21, s11
+; CHECK-NEXT:    vldr s31, [sp, #140]
+; CHECK-NEXT:    vmov.f32 s20, s9
+; CHECK-NEXT:    vldr s30, [sp, #132]
+; CHECK-NEXT:    vstrw.32 q5, [sp, #32] @ 16-byte Spill
+; CHECK-NEXT:    vmov.f32 s28, s13
+; CHECK-NEXT:    vmov.f32 s23, s6
 ; CHECK-NEXT:    vldr s19, [sp, #136]
-; CHECK-NEXT:    vmov.f32 s11, s6
-; CHECK-NEXT:    vldr s18, [sp, #128]
 ; CHECK-NEXT:    vmov.f32 s6, s5
-; CHECK-NEXT:    vldr s31, [sp, #188]
-; CHECK-NEXT:    vmov.f32 s10, s4
-; CHECK-NEXT:    vldr s30, [sp, #180]
-; CHECK-NEXT:    vmov.f32 s21, s15
-; CHECK-NEXT:    vldr s29, [sp, #172]
+; CHECK-NEXT:    vldr s18, [sp, #128]
+; CHECK-NEXT:    vmov.f32 s22, s4
+; CHECK-NEXT:    vldr s27, [sp, #184]
+; CHECK-NEXT:    vmov.f32 s29, s15
+; CHECK-NEXT:    vldr s26, [sp, #176]
 ; CHECK-NEXT:    vmov.f32 s5, s3
-; CHECK-NEXT:    vldr s28, [sp, #164]
+; CHECK-NEXT:    vldr s25, [sp, #168]
 ; CHECK-NEXT:    vmov.f32 s4, s1
-; CHECK-NEXT:    vmov.f32 s24, s9
+; CHECK-NEXT:    vldr s24, [sp, #160]
 ; CHECK-NEXT:    vmov.f32 s16, s12
-; CHECK-NEXT:    vstrw.32 q6, [sp, #32] @ 16-byte Spill
-; CHECK-NEXT:    vmov.f32 s12, s8
-; CHECK-NEXT:    vldr s27, [sp, #184]
+; CHECK-NEXT:    vldr s11, [sp, #188]
 ; CHECK-NEXT:    vmov.f32 s17, s14
-; CHECK-NEXT:    vldr s26, [sp, #176]
-; CHECK-NEXT:    vmov.f32 s9, s2
-; CHECK-NEXT:    vldr s25, [sp, #168]
-; CHECK-NEXT:    vmov.f32 s8, s0
-; CHECK-NEXT:    vmul.f32 q0, q5, q1
+; CHECK-NEXT:    vldr s9, [sp, #172]
+; CHECK-NEXT:    vmov.f32 s21, s2
+; CHECK-NEXT:    vmov.f32 s20, s0
+; CHECK-NEXT:    vmul.f32 q0, q7, q1
 ; CHECK-NEXT:    vmul.f32 q1, q4, q1
 ; CHECK-NEXT:    vneg.f32 q0, q0
-; CHECK-NEXT:    vldr s24, [sp, #160]
-; CHECK-NEXT:    vfma.f32 q1, q5, q2
 ; CHECK-NEXT:    vstrw.32 q0, [sp, #16] @ 16-byte Spill
-; CHECK-NEXT:    vstrw.32 q3, [sp, #48] @ 16-byte Spill
-; CHECK-NEXT:    vsub.f32 q6, q6, q1
+; CHECK-NEXT:    vfma.f32 q1, q7, q5
+; CHECK-NEXT:    vsub.f32 q7, q6, q1
 ; CHECK-NEXT:    vldrw.u32 q1, [sp, #16] @ 16-byte Reload
+; CHECK-NEXT:    vmov.f32 s13, s10
+; CHECK-NEXT:    vldr s10, [sp, #180]
+; CHECK-NEXT:    vmov.f32 s12, s8
+; CHECK-NEXT:    vfma.f32 q1, q4, q5
+; CHECK-NEXT:    vstrw.32 q3, [sp, #48] @ 16-byte Spill
+; CHECK-NEXT:    vldr s8, [sp, #164]
 ; CHECK-NEXT:    vldr s13, [sp, #156]
-; CHECK-NEXT:    vfma.f32 q1, q4, q2
+; CHECK-NEXT:    vldrw.u32 q5, [sp, #32] @ 16-byte Reload
 ; CHECK-NEXT:    vldr s12, [sp, #148]
-; CHECK-NEXT:    vadd.f32 q1, q7, q1
-; CHECK-NEXT:    vldrw.u32 q7, [sp, #32] @ 16-byte Reload
+; CHECK-NEXT:    vadd.f32 q1, q2, q1
 ; CHECK-NEXT:    vldr s1, [sp, #152]
+; CHECK-NEXT:    vmul.f32 q2, q3, q5
 ; CHECK-NEXT:    vstrw.32 q3, [sp] @ 16-byte Spill
-; CHECK-NEXT:    vmul.f32 q2, q3, q7
 ; CHECK-NEXT:    vldr s0, [sp, #144]
 ; CHECK-NEXT:    vldrw.u32 q3, [sp, #48] @ 16-byte Reload
 ; CHECK-NEXT:    vneg.f32 q2, q2
-; CHECK-NEXT:    vldr s21, [sp, #200]
+; CHECK-NEXT:    vldr s25, [sp, #200]
 ; CHECK-NEXT:    vfma.f32 q2, q0, q3
-; CHECK-NEXT:    vmul.f32 q0, q0, q7
-; CHECK-NEXT:    vldrw.u32 q7, [sp] @ 16-byte Reload
-; CHECK-NEXT:    vldr s20, [sp, #192]
+; CHECK-NEXT:    vmul.f32 q0, q0, q5
+; CHECK-NEXT:    vldrw.u32 q5, [sp] @ 16-byte Reload
+; CHECK-NEXT:    vldr s24, [sp, #192]
 ; CHECK-NEXT:    vldr s17, [sp, #204]
 ; CHECK-NEXT:    vldr s16, [sp, #196]
-; CHECK-NEXT:    vfma.f32 q0, q7, q3
-; CHECK-NEXT:    vsub.f32 q3, q5, q0
+; CHECK-NEXT:    vfma.f32 q0, q5, q3
+; CHECK-NEXT:    vsub.f32 q3, q6, q0
 ; CHECK-NEXT:    vmov.f32 s1, s4
 ; CHECK-NEXT:    vadd.f32 q4, q4, q2
 ; CHECK-NEXT:    vmov.f32 s3, s5
 ; CHECK-NEXT:    vmov.f32 s5, s6
-; CHECK-NEXT:    vmov.f32 s0, s24
-; CHECK-NEXT:    vmov.f32 s2, s25
-; CHECK-NEXT:    vmov.f32 s4, s26
-; CHECK-NEXT:    vmov.f32 s6, s27
+; CHECK-NEXT:    vmov.f32 s0, s28
+; CHECK-NEXT:    vmov.f32 s2, s29
+; CHECK-NEXT:    vmov.f32 s4, s30
+; CHECK-NEXT:    vmov.f32 s6, s31
 ; CHECK-NEXT:    vmov.f32 s8, s12
 ; CHECK-NEXT:    vmov.f32 s9, s16
 ; CHECK-NEXT:    vmov.f32 s10, s13
diff --git a/llvm/test/CodeGen/Thumb2/mve-float32regloops.ll b/llvm/test/CodeGen/Thumb2/mve-float32regloops.ll
index 495ffe809f70f..28166e455aba2 100644
--- a/llvm/test/CodeGen/Thumb2/mve-float32regloops.ll
+++ b/llvm/test/CodeGen/Thumb2/mve-float32regloops.ll
@@ -1060,12 +1060,12 @@ define void @fir(ptr nocapture readonly %S, ptr nocapture readonly %pSrc, ptr no
 ; CHECK-NEXT:    vfma.f32 q0, q4, r5
 ; CHECK-NEXT:    vldrw.u32 q3, [r4, #-8]
 ; CHECK-NEXT:    vfma.f32 q0, q5, r6
-; CHECK-NEXT:    ldr r0, [sp, #16] @ 4-byte Reload
-; CHECK-NEXT:    vfma.f32 q0, q2, lr
 ; CHECK-NEXT:    vldrw.u32 q1, [r4, #-4]
+; CHECK-NEXT:    vfma.f32 q0, q2, lr
+; CHECK-NEXT:    ldr r0, [sp, #16] @ 4-byte Reload
 ; CHECK-NEXT:    vfma.f32 q0, q3, r11
-; CHECK-NEXT:    cmp r0, #16
 ; CHECK-NEXT:    vfma.f32 q0, q1, r8
+; CHECK-NEXT:    cmp r0, #16
 ; CHECK-NEXT:    blo .LBB16_9
 ; CHECK-NEXT:  @ %bb.7: @ %for.body.preheader
 ; CHECK-NEXT:    @ in Loop: Header=BB16_6 Depth=1
@@ -1603,8 +1603,8 @@ define arm_aapcs_vfpcc void @arm_biquad_cascade_df1_f32(ptr nocapture readonly %
 ; CHECK-NEXT:  .LBB19_3: @ %do.body
 ; CHECK-NEXT:    @ =>This Loop Header: Depth=1
 ; CHECK-NEXT:    @ Child Loop BB19_5 Depth 2
-; CHECK-NEXT:    mov r6, r2
 ; CHECK-NEXT:    ldrd r5, r11, [r9]
+; CHECK-NEXT:    mov r6, r2
 ; CHECK-NEXT:    ldrd r8, r10, [r9, #8]
 ; CHECK-NEXT:    ldr r2, [sp] @ 4-byte Reload
 ; CHECK-NEXT:    str r7, [sp, #12] @ 4-byte Spill
@@ -1912,11 +1912,11 @@ define void @arm_biquad_cascade_df2T_f32(ptr nocapture readonly %S, ptr nocaptur
 ; CHECK-NEXT:    b .LBB20_3
 ; CHECK-NEXT:  .LBB20_1: @ %if.else
 ; CHECK-NEXT:    @ in Loop: Header=BB20_3 Depth=1
-; CHECK-NEXT:    vmov.f32 s6, s5
-; CHECK-NEXT:    vstr s4, [r6]
+; CHECK-NEXT:    vmov.f32 s14, s13
+; CHECK-NEXT:    vstr s12, [r6]
 ; CHECK-NEXT:  .LBB20_2: @ %if.end
 ; CHECK-NEXT:    @ in Loop: Header=BB20_3 Depth=1
-; CHECK-NEXT:    vstr s6, [r6, #4]
+; CHECK-NEXT:    vstr s14, [r6, #4]
 ; CHECK-NEXT:    add.w r12, r12, #20
 ; CHECK-NEXT:    adds r6, #8
 ; CHECK-NEXT:    subs r0, #1
@@ -1925,41 +1925,41 @@ define void @arm_biquad_cascade_df2T_f32(ptr nocapture readonly %S, ptr nocaptur
 ; CHECK-NEXT:  .LBB20_3: @ %do.body
 ; CHECK-NEXT:    @ =>This Loop Header: Depth=1
 ; CHECK-NEXT:    @ Child Loop BB20_5 Depth 2
-; CHECK-NEXT:    vldrw.u32 q3, [r12]
+; CHECK-NEXT:    vldrw.u32 q2, [r12]
 ; CHECK-NEXT:    movs r5, #0
-; CHECK-NEXT:    vmov q4, q3
+; CHECK-NEXT:    vmov q4, q2
 ; CHECK-NEXT:    vshlc q4, r5, #32
-; CHECK-NEXT:    vldrw.u32 q2, [r12, #8]
-; CHECK-NEXT:    vmov q5, q2
+; CHECK-NEXT:    vldrw.u32 q1, [r12, #8]
+; CHECK-NEXT:    vmov q5, q1
 ; CHECK-NEXT:    vshlc q5, r5, #32
-; CHECK-NEXT:    vldrw.u32 q1, [r6]
-; CHECK-NEXT:    vmov.f32 s6, s0
+; CHECK-NEXT:    vldrw.u32 q3, [r6]
+; CHECK-NEXT:    vmov.f32 s14, s0
 ; CHECK-NEXT:    mov r5, r2
-; CHECK-NEXT:    vmov.f32 s7, s0
+; CHECK-NEXT:    vmov.f32 s15, s0
 ; CHECK-NEXT:    wls lr, r8, .LBB20_6
 ; CHECK-NEXT:  @ %bb.4: @ %while.body.preheader
 ; CHECK-NEXT:    @ in Loop: Header=BB20_3 Depth=1
-; CHECK-NEXT:    vmov q6, q1
+; CHECK-NEXT:    vmov q6, q3
 ; CHECK-NEXT:    mov r5, r2
 ; CHECK-NEXT:  .LBB20_5: @ %while.body
 ; CHECK-NEXT:    @ Parent Loop BB20_3 Depth=1
 ; CHECK-NEXT:    @ => This Inner Loop Header: Depth=2
 ; CHECK-NEXT:    ldrd r7, r4, [r1], #8
-; CHECK-NEXT:    vfma.f32 q6, q3, r7
+; CHECK-NEXT:    vfma.f32 q6, q2, r7
 ; CHECK-NEXT:    vmov r7, s24
-; CHECK-NEXT:    vmov q1, q6
-; CHECK-NEXT:    vfma.f32 q1, q2, r7
+; CHECK-NEXT:    vmov q3, q6
+; CHECK-NEXT:    vfma.f32 q3, q1, r7
 ; CHECK-NEXT:    vstr s24, [r5]
-; CHECK-NEXT:    vmov.f32 s7, s0
-; CHECK-NEXT:    vfma.f32 q1, q4, r4
-; CHECK-NEXT:    vmov r4, s5
-; CHECK-NEXT:    vstr s5, [r5, #4]
-; CHECK-NEXT:    vfma.f32 q1, q5, r4
+; CHECK-NEXT:    vmov.f32 s15, s0
+; CHECK-NEXT:    vfma.f32 q3, q4, r4
+; CHECK-NEXT:    vmov r4, s13
+; CHECK-NEXT:    vstr s13, [r5, #4]
+; CHECK-NEXT:    vfma.f32 q3, q5, r4
 ; CHECK-NEXT:    adds r5, #8
-; CHECK-NEXT:    vmov.f32 s4, s6
-; CHECK-NEXT:    vmov.f32 s5, s7
-; CHECK-NEXT:    vmov.f32 s6, s0
-; CHECK-NEXT:    vmov q6, q1
+; CHECK-NEXT:    vmov.f32 s12, s14
+; CHECK-NEXT:    vmov.f32 s13, s15
+; CHECK-NEXT:    vmov.f32 s14, s0
+; CHECK-NEXT:    vmov q6, q3
 ; CHECK-NEXT:    le lr, .LBB20_5
 ; CHECK-NEXT:  .LBB20_6: @ %while.end
 ; CHECK-NEXT:    @ in Loop: Header=BB20_3 Depth=1
@@ -1968,11 +1968,11 @@ define void @arm_biquad_cascade_df2T_f32(ptr nocapture readonly %S, ptr nocaptur
 ; CHECK-NEXT:  @ %bb.7: @ %if.then
 ; CHECK-NEXT:    @ in Loop: Header=BB20_3 Depth=1
 ; CHECK-NEXT:    ldr r1, [r1]
-; CHECK-NEXT:    vfma.f32 q1, q3, r1
-; CHECK-NEXT:    vmov r1, s4
-; CHECK-NEXT:    vstr s4, [r5]
-; CHECK-NEXT:    vfma.f32 q1, q2, r1
-; CHECK-NEXT:    vstr s5, [r6]
+; CHECK-NEXT:    vfma.f32 q3, q2, r1
+; CHECK-NEXT:    vmov r1, s12
+; CHECK-NEXT:    vstr s12, [r5]
+; CHECK-NEXT:    vfma.f32 q3, q1, r1
+; CHECK-NEXT:    vstr s13, [r6]
 ; CHECK-NEXT:    b .LBB20_2
 ; CHECK-NEXT:  .LBB20_8: @ %do.end
 ; CHECK-NEXT:    vpop {d8, d9, d10, d11, d12, d13}
diff --git a/llvm/test/CodeGen/Thumb2/mve-gather-increment.ll b/llvm/test/CodeGen/Thumb2/mve-gather-increment.ll
index e63c62574dafb..e8b49c1067379 100644
--- a/llvm/test/CodeGen/Thumb2/mve-gather-increment.ll
+++ b/llvm/test/CodeGen/Thumb2/mve-gather-increment.ll
@@ -1376,8 +1376,8 @@ define arm_aapcs_vfpcc void @gather_inc_v16i8_simple(ptr noalias nocapture reado
 ; CHECK-NEXT:    bne .LBB16_3
 ; CHECK-NEXT:  @ %bb.4: @ %middle.block
 ; CHECK-NEXT:    @ in Loop: Header=BB16_2 Depth=1
-; CHECK-NEXT:    ldr r1, [sp, #60] @ 4-byte Reload
 ; CHECK-NEXT:    ldr.w r9, [sp, #52] @ 4-byte Reload
+; CHECK-NEXT:    ldr r1, [sp, #60] @ 4-byte Reload
 ; CHECK-NEXT:    cmp r9, r1
 ; CHECK-NEXT:    bne .LBB16_2
 ; CHECK-NEXT:  .LBB16_5: @ %for.cond.cleanup
diff --git a/llvm/test/CodeGen/Thumb2/mve-gather-scatter-optimisation.ll b/llvm/test/CodeGen/Thumb2/mve-gather-scatter-optimisation.ll
index 7b8b884576d13..eedca2cd4a5d3 100644
--- a/llvm/test/CodeGen/Thumb2/mve-gather-scatter-optimisation.ll
+++ b/llvm/test/CodeGen/Thumb2/mve-gather-scatter-optimisation.ll
@@ -712,8 +712,8 @@ define dso_local void @arm_mat_mult_q15(ptr noalias nocapture readonly %A, ptr n
 ; CHECK-NEXT:  @ %bb.12: @ %middle.block
 ; CHECK-NEXT:    @ in Loop: Header=BB10_8 Depth=2
 ; CHECK-NEXT:    vaddv.u32 r10, q4
-; CHECK-NEXT:    cmp r2, r12
 ; CHECK-NEXT:    mov r4, r2
+; CHECK-NEXT:    cmp r2, r12
 ; CHECK-NEXT:    beq .LBB10_7
 ; CHECK-NEXT:  .LBB10_13: @ %for.body8.us.us.preheader
 ; CHECK-NEXT:    @ in Loop: Header=BB10_8 Depth=2
diff --git a/llvm/test/CodeGen/Thumb2/mve-laneinterleaving-reduct.ll b/llvm/test/CodeGen/Thumb2/mve-laneinterleaving-reduct.ll
index 77980be905207..652d25af02e7c 100644
--- a/llvm/test/CodeGen/Thumb2/mve-laneinterleaving-reduct.ll
+++ b/llvm/test/CodeGen/Thumb2/mve-laneinterleaving-reduct.ll
@@ -180,9 +180,9 @@ define void @correlate(ptr nocapture noundef readonly %ID, ptr nocapture noundef
 ; CHECK-NEXT:    @ in Loop: Header=BB4_4 Depth=1
 ; CHECK-NEXT:    ldr r1, [sp] @ 4-byte Reload
 ; CHECK-NEXT:    add.w r2, r9, r10
-; CHECK-NEXT:    sub.w r5, r8, r9
 ; CHECK-NEXT:    add.w r7, r1, r9, lsl #1
 ; CHECK-NEXT:    add.w r2, r1, r2, lsl #1
+; CHECK-NEXT:    sub.w r5, r8, r9
 ; CHECK-NEXT:    dlstp.32 lr, r5
 ; CHECK-NEXT:  .LBB4_11: @ %vec.epilog.vector.body
 ; CHECK-NEXT:    @ Parent Loop BB4_4 Depth=1
diff --git a/llvm/test/CodeGen/Thumb2/mve-masked-ldst.ll b/llvm/test/CodeGen/Thumb2/mve-masked-ldst.ll
index 4934d22320903..67910e40e7705 100644
--- a/llvm/test/CodeGen/Thumb2/mve-masked-ldst.ll
+++ b/llvm/test/CodeGen/Thumb2/mve-masked-ldst.ll
@@ -762,10 +762,10 @@ define void @foo_v4f32_v4f16(ptr %dest, ptr %mask, ptr %src) {
 ; CHECK-LE-NEXT:  .LBB18_5: @ %else8
 ; CHECK-LE-NEXT:    vmrs r2, p0
 ; CHECK-LE-NEXT:    movs r1, #0
-; CHECK-LE-NEXT:    vcvtt.f32.f16 s3, s1
-; CHECK-LE-NEXT:    vcvtb.f32.f16 s2, s1
-; CHECK-LE-NEXT:    vcvtt.f32.f16 s1, s0
+; CHECK-LE-NEXT:    vcvtt.f32.f16 s6, s0
 ; CHECK-LE-NEXT:    vcvtb.f32.f16 s0, s0
+; CHECK-LE-NEXT:    vcvtt.f32.f16 s2, s1
+; CHECK-LE-NEXT:    vcvtb.f32.f16 s4, s1
 ; CHECK-LE-NEXT:    and r3, r2, #1
 ; CHECK-LE-NEXT:    rsbs r3, r3, #0
 ; CHECK-LE-NEXT:    bfi r1, r3, #0, #1
@@ -784,15 +784,15 @@ define void @foo_v4f32_v4f16(ptr %dest, ptr %mask, ptr %src) {
 ; CHECK-LE-NEXT:    strne r2, [r0]
 ; CHECK-LE-NEXT:    lsls r2, r1, #30
 ; CHECK-LE-NEXT:    itt mi
-; CHECK-LE-NEXT:    vmovmi r2, s1
+; CHECK-LE-NEXT:    vmovmi r2, s6
 ; CHECK-LE-NEXT:    strmi r2, [r0, #4]
 ; CHECK-LE-NEXT:    lsls r2, r1, #29
 ; CHECK-LE-NEXT:    itt mi
-; CHECK-LE-NEXT:    vmovmi r2, s2
+; CHECK-LE-NEXT:    vmovmi r2, s4
 ; CHECK-LE-NEXT:    strmi r2, [r0, #8]
 ; CHECK-LE-NEXT:    lsls r1, r1, #28
 ; CHECK-LE-NEXT:    itt mi
-; CHECK-LE-NEXT:    vmovmi r1, s3
+; CHECK-LE-NEXT:    vmovmi r1, s2
 ; CHECK-LE-NEXT:    strmi r1, [r0, #12]
 ; CHECK-LE-NEXT:    add sp, #4
 ; CHECK-LE-NEXT:    pop {r7, pc}
@@ -853,10 +853,10 @@ define void @foo_v4f32_v4f16(ptr %dest, ptr %mask, ptr %src) {
 ; CHECK-BE-NEXT:  .LBB18_5: @ %else8
 ; CHECK-BE-NEXT:    vmrs r2, p0
 ; CHECK-BE-NEXT:    movs r1, #0
-; CHECK-BE-NEXT:    vcvtt.f32.f16 s3, s1
-; CHECK-BE-NEXT:    vcvtb.f32.f16 s2, s1
-; CHECK-BE-NEXT:    vcvtt.f32.f16 s1, s0
+; CHECK-BE-NEXT:    vcvtt.f32.f16 s6, s0
 ; CHECK-BE-NEXT:    vcvtb.f32.f16 s0, s0
+; CHECK-BE-NEXT:    vcvtt.f32.f16 s2, s1
+; CHECK-BE-NEXT:    vcvtb.f32.f16 s4, s1
 ; CHECK-BE-NEXT:    ubfx r3, r2, #12, #1
 ; CHECK-BE-NEXT:    rsbs r3, r3, #0
 ; CHECK-BE-NEXT:    bfi r1, r3, #0, #1
@@ -875,15 +875,15 @@ define void @foo_v4f32_v4f16(ptr %dest, ptr %mask, ptr %src) {
 ; CHECK-BE-NEXT:    strmi r2, [r0]
 ; CHECK-BE-NEXT:    lsls r2, r1, #29
 ; CHECK-BE-NEXT:    itt mi
-; CHECK-BE-NEXT:    vmovmi r2, s1
+; CHECK-BE-NEXT:    vmovmi r2, s6
 ; CHECK-BE-NEXT:    strmi r2, [r0, #4]
 ; CHECK-BE-NEXT:    lsls r2, r1, #30
 ; CHECK-BE-NEXT:    itt mi
-; CHECK-BE-NEXT:    vmovmi r2, s2
+; CHECK-BE-NEXT:    vmovmi r2, s4
 ; CHECK-BE-NEXT:    strmi r2, [r0, #8]
 ; CHECK-BE-NEXT:    lsls r1, r1, #31
 ; CHECK-BE-NEXT:    itt ne
-; CHECK-BE-NEXT:    vmovne r1, s3
+; CHECK-BE-NEXT:    vmovne r1, s2
 ; CHECK-BE-NEXT:    strne r1, [r0, #12]
 ; CHECK-BE-NEXT:    add sp, #4
 ; CHECK-BE-NEXT:    pop {r7, pc}
@@ -953,10 +953,10 @@ define void @foo_v4f32_v4f16_unaligned(ptr %dest, ptr %mask, ptr %src) {
 ; CHECK-LE-NEXT:  .LBB19_5: @ %else8
 ; CHECK-LE-NEXT:    vmrs r2, p0
 ; CHECK-LE-NEXT:    movs r1, #0
-; CHECK-LE-NEXT:    vcvtt.f32.f16 s3, s1
-; CHECK-LE-NEXT:    vcvtb.f32.f16 s2, s1
-; CHECK-LE-NEXT:    vcvtt.f32.f16 s1, s0
+; CHECK-LE-NEXT:    vcvtt.f32.f16 s6, s0
 ; CHECK-LE-NEXT:    vcvtb.f32.f16 s0, s0
+; CHECK-LE-NEXT:    vcvtt.f32.f16 s2, s1
+; CHECK-LE-NEXT:    vcvtb.f32.f16 s4, s1
 ; CHECK-LE-NEXT:    and r3, r2, #1
 ; CHECK-LE-NEXT:    rsbs r3, r3, #0
 ; CHECK-LE-NEXT:    bfi r1, r3, #0, #1
@@ -975,15 +975,15 @@ define void @foo_v4f32_v4f16_unaligned(ptr %dest, ptr %mask, ptr %src) {
 ; CHECK-LE-NEXT:    strne r2, [r0]
 ; CHECK-LE-NEXT:    lsls r2, r1, #30
 ; CHECK-LE-NEXT:    itt mi
-; CHECK-LE-NEXT:    vmovmi r2, s1
+; CHECK-LE-NEXT:    vmovmi r2, s6
 ; CHECK-LE-NEXT:    strmi r2, [r0, #4]
 ; CHECK-LE-NEXT:    lsls r2, r1, #29
 ; CHECK-LE-NEXT:    itt mi
-; CHECK-LE-NEXT:    vmovmi r2, s2
+; CHECK-LE-NEXT:    vmovmi r2, s4
 ; CHECK-LE-NEXT:    strmi r2, [r0, #8]
 ; CHECK-LE-NEXT:    lsls r1, r1, #28
 ; CHECK-LE-NEXT:    itt mi
-; CHECK-LE-NEXT:    vmovmi r1, s3
+; CHECK-LE-NEXT:    vmovmi r1, s2
 ; CHECK-LE-NEXT:    strmi r1, [r0, #12]
 ; CHECK-LE-NEXT:    add sp, #4
 ; CHECK-LE-NEXT:    pop {r7, pc}
@@ -1044,10 +1044,10 @@ define void @foo_v4f32_v4f16_unaligned(ptr %dest, ptr %mask, ptr %src) {
 ; CHECK-BE-NEXT:  .LBB19_5: @ %else8
 ; CHECK-BE-NEXT:    vmrs r2, p0
 ; CHECK-BE-NEXT:    movs r1, #0
-; CHECK-BE-NEXT:    vcvtt.f32.f16 s3, s1
-; CHECK-BE-NEXT:    vcvtb.f32.f16 s2, s1
-; CHECK-BE-NEXT:    vcvtt.f32.f16 s1, s0
+; CHECK-BE-NEXT:    vcvtt.f32.f16 s6, s0
 ; CHECK-BE-NEXT:    vcvtb.f32.f16 s0, s0
+; CHECK-BE-NEXT:    vcvtt.f32.f16 s2, s1
+; CHECK-BE-NEXT:    vcvtb.f32.f16 s4, s1
 ; CHECK-BE-NEXT:    ubfx r3, r2, #12, #1
 ; CHECK-BE-NEXT:    rsbs r3, r3, #0
 ; CHECK-BE-NEXT:    bfi r1, r3, #0, #1
@@ -1066,15 +1066,15 @@ define void @foo_v4f32_v4f16_unaligned(ptr %dest, ptr %mask, ptr %src) {
 ; CHECK-BE-NEXT:    strmi r2, [r0]
 ; CHECK-BE-NEXT:    lsls r2, r1, #29
 ; CHECK-BE-NEXT:    itt mi
-; CHECK-BE-NEXT:    vmovmi r2, s1
+; CHECK-BE-NEXT:    vmovmi r2, s6
 ; CHECK-BE-NEXT:    strmi r2, [r0, #4]
 ; CHECK-BE-NEXT:    lsls r2, r1, #30
 ; CHECK-BE-NEXT:    itt mi
-; CHECK-BE-NEXT:    vmovmi r2, s2
+; CHECK-BE-NEXT:    vmovmi r2, s4
 ; CHECK-BE-NEXT:    strmi r2, [r0, #8]
 ; CHECK-BE-NEXT:    lsls r1, r1, #31
 ; CHECK-BE-NEXT:    itt ne
-; CHECK-BE-NEXT:    vmovne r1, s3
+; CHECK-BE-NEXT:    vmovne r1, s2
 ; CHECK-BE-NEXT:    strne r1, [r0, #12]
 ; CHECK-BE-NEXT:    add sp, #4
 ; CHECK-BE-NEXT:    pop {r7, pc}
diff --git a/llvm/test/CodeGen/Thumb2/mve-postinc-dct.ll b/llvm/test/CodeGen/Thumb2/mve-postinc-dct.ll
index 45bb70ec44b73..f90af3cc5ba24 100644
--- a/llvm/test/CodeGen/Thumb2/mve-postinc-dct.ll
+++ b/llvm/test/CodeGen/Thumb2/mve-postinc-dct.ll
@@ -258,11 +258,11 @@ define void @DCT_mve3(ptr nocapture readonly %S, ptr nocapture readonly %pIn, pt
 ; CHECK-NEXT:    @ Child Loop BB2_3 Depth 2
 ; CHECK-NEXT:    ldr r6, [sp, #16] @ 4-byte Reload
 ; CHECK-NEXT:    vmov.i32 q0, #0x0
-; CHECK-NEXT:    ldr r1, [sp] @ 4-byte Reload
 ; CHECK-NEXT:    adds r0, r5, #2
 ; CHECK-NEXT:    adds r2, r5, #1
-; CHECK-NEXT:    str r0, [sp, #20] @ 4-byte Spill
+; CHECK-NEXT:    ldr r1, [sp] @ 4-byte Reload
 ; CHECK-NEXT:    mov r3, r9
+; CHECK-NEXT:    str r0, [sp, #20] @ 4-byte Spill
 ; CHECK-NEXT:    mov r0, r12
 ; CHECK-NEXT:    mov r4, r10
 ; CHECK-NEXT:    vmov q2, q0
@@ -618,13 +618,13 @@ define void @DCT_mve5(ptr nocapture readonly %S, ptr nocapture readonly %pIn, pt
 ; CHECK-NEXT:    adds r1, r0, #4
 ; CHECK-NEXT:    ldr r4, [sp, #20] @ 4-byte Reload
 ; CHECK-NEXT:    vmov.i32 q1, #0x0
-; CHECK-NEXT:    ldr r6, [sp, #8] @ 4-byte Reload
-; CHECK-NEXT:    add.w r10, r0, #2
 ; CHECK-NEXT:    str r1, [sp, #28] @ 4-byte Spill
 ; CHECK-NEXT:    adds r1, r0, #3
+; CHECK-NEXT:    add.w r10, r0, #2
 ; CHECK-NEXT:    add.w r11, r0, #1
-; CHECK-NEXT:    str r1, [sp, #24] @ 4-byte Spill
+; CHECK-NEXT:    ldr r6, [sp, #8] @ 4-byte Reload
 ; CHECK-NEXT:    mov r3, r8
+; CHECK-NEXT:    str r1, [sp, #24] @ 4-byte Spill
 ; CHECK-NEXT:    vmov q0, q1
 ; CHECK-NEXT:    vmov q3, q1
 ; CHECK-NEXT:    vmov q2, q1
@@ -833,8 +833,8 @@ define void @DCT_mve6(ptr nocapture readonly %S, ptr nocapture readonly %pIn, pt
 ; CHECK-NEXT:    vmov.i32 q1, #0x0
 ; CHECK-NEXT:    ldr r1, [sp, #16] @ 4-byte Reload
 ; CHECK-NEXT:    add.w r11, r0, #2
-; CHECK-NEXT:    ldr r6, [sp, #4] @ 4-byte Reload
 ; CHECK-NEXT:    adds r4, r0, #1
+; CHECK-NEXT:    ldr r6, [sp, #4] @ 4-byte Reload
 ; CHECK-NEXT:    mov r3, r8
 ; CHECK-NEXT:    vmov q3, q1
 ; CHECK-NEXT:    vmov q4, q1
@@ -1068,8 +1068,8 @@ define void @DCT_mve7(ptr nocapture readonly %S, ptr nocapture readonly %pIn, pt
 ; CHECK-NEXT:    vmov.i32 q2, #0x0
 ; CHECK-NEXT:    ldr r1, [sp, #20] @ 4-byte Reload
 ; CHECK-NEXT:    adds r4, r0, #2
-; CHECK-NEXT:    ldr r6, [sp, #8] @ 4-byte Reload
 ; CHECK-NEXT:    add.w r8, r0, #1
+; CHECK-NEXT:    ldr r6, [sp, #8] @ 4-byte Reload
 ; CHECK-NEXT:    mov r3, r9
 ; CHECK-NEXT:    vmov q4, q2
 ; CHECK-NEXT:    vmov q5, q2
@@ -1347,11 +1347,11 @@ define void @DCT_mve8(ptr nocapture readonly %S, ptr nocapture readonly %pIn, pt
 ; CHECK-NEXT:    adds r1, r0, #4
 ; CHECK-NEXT:    ldr.w r9, [sp, #20] @ 4-byte Reload
 ; CHECK-NEXT:    vmov.i32 q3, #0x0
-; CHECK-NEXT:    ldr r5, [sp, #8] @ 4-byte Reload
-; CHECK-NEXT:    adds r4, r0, #3
 ; CHECK-NEXT:    str r1, [sp, #24] @ 4-byte Spill
+; CHECK-NEXT:    adds r4, r0, #3
 ; CHECK-NEXT:    add.w r8, r0, #2
 ; CHECK-NEXT:    adds r1, r0, #1
+; CHECK-NEXT:    ldr r5, [sp, #8] @ 4-byte Reload
 ; CHECK-NEXT:    mov r3, r12
 ; CHECK-NEXT:    vmov q5, q3
 ; CHECK-NEXT:    vmov q6, q3
diff --git a/llvm/test/CodeGen/Thumb2/mve-postinc-distribute.ll b/llvm/test/CodeGen/Thumb2/mve-postinc-distribute.ll
index dd63b8564bdb4..096d4382d2c35 100644
--- a/llvm/test/CodeGen/Thumb2/mve-postinc-distribute.ll
+++ b/llvm/test/CodeGen/Thumb2/mve-postinc-distribute.ll
@@ -100,8 +100,8 @@ define void @arm_cmplx_dot_prod_q15(ptr nocapture readonly %pSrcA, ptr nocapture
 ; CHECK-NEXT:    ldr.w r8, [sp, #36]
 ; CHECK-NEXT:    mov r6, r12
 ; CHECK-NEXT:    mov r5, r7
-; CHECK-NEXT:    and r2, r2, #3
 ; CHECK-NEXT:    lsrl r6, r5, #6
+; CHECK-NEXT:    and r2, r2, #3
 ; CHECK-NEXT:    wls lr, r2, .LBB1_7
 ; CHECK-NEXT:  .LBB1_5: @ %while.body11
 ; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
diff --git a/llvm/test/CodeGen/Thumb2/mve-postinc-lsr.ll b/llvm/test/CodeGen/Thumb2/mve-postinc-lsr.ll
index c03339b52f264..cba0f9cbba2ca 100644
--- a/llvm/test/CodeGen/Thumb2/mve-postinc-lsr.ll
+++ b/llvm/test/CodeGen/Thumb2/mve-postinc-lsr.ll
@@ -708,14 +708,14 @@ define ptr @signext(ptr %input_row, ptr %input_col, i16 zeroext %output_ch, i16
 ; CHECK-NEXT:    mov r6, r12
 ; CHECK-NEXT:  .LBB5_4: @ %for.cond.cleanup23
 ; CHECK-NEXT:    @ in Loop: Header=BB5_5 Depth=1
-; CHECK-NEXT:    add.w r0, r8, r10
 ; CHECK-NEXT:    ldr r1, [sp, #100]
+; CHECK-NEXT:    add.w r0, r8, r10
 ; CHECK-NEXT:    add r0, r6
 ; CHECK-NEXT:    add r0, r12
 ; CHECK-NEXT:    strb.w r0, [r1, r11]
 ; CHECK-NEXT:    add.w r11, r11, #1
-; CHECK-NEXT:    ldr r0, [sp, #20] @ 4-byte Reload
 ; CHECK-NEXT:    ldr r1, [sp, #24] @ 4-byte Reload
+; CHECK-NEXT:    ldr r0, [sp, #20] @ 4-byte Reload
 ; CHECK-NEXT:    cmp r11, r0
 ; CHECK-NEXT:    beq .LBB5_8
 ; CHECK-NEXT:  .LBB5_5: @ %for.body
@@ -933,14 +933,14 @@ define ptr @signext_optsize(ptr %input_row, ptr %input_col, i16 zeroext %output_
 ; CHECK-NEXT:    mov r6, r12
 ; CHECK-NEXT:  .LBB6_7: @ %for.cond.cleanup23
 ; CHECK-NEXT:    @ in Loop: Header=BB6_3 Depth=1
-; CHECK-NEXT:    add.w r0, r8, r10
 ; CHECK-NEXT:    ldr r1, [sp, #100]
+; CHECK-NEXT:    add.w r0, r8, r10
 ; CHECK-NEXT:    add r0, r6
 ; CHECK-NEXT:    add r0, r12
 ; CHECK-NEXT:    strb.w r0, [r1, r11]
 ; CHECK-NEXT:    add.w r11, r11, #1
-; CHECK-NEXT:    ldr r0, [sp, #20] @ 4-byte Reload
 ; CHECK-NEXT:    ldr r1, [sp, #24] @ 4-byte Reload
+; CHECK-NEXT:    ldr r0, [sp, #20] @ 4-byte Reload
 ; CHECK-NEXT:    cmp r11, r0
 ; CHECK-NEXT:    bne .LBB6_3
 ; CHECK-NEXT:  .LBB6_8: @ %if.end
diff --git a/llvm/test/CodeGen/Thumb2/mve-pred-vctpvpsel.ll b/llvm/test/CodeGen/Thumb2/mve-pred-vctpvpsel.ll
index 9400f24e7192c..723cbff1ab9d3 100644
--- a/llvm/test/CodeGen/Thumb2/mve-pred-vctpvpsel.ll
+++ b/llvm/test/CodeGen/Thumb2/mve-pred-vctpvpsel.ll
@@ -11,8 +11,8 @@ define void @arm_min_helium_f32(ptr %pSrc, i32 %blockSize, ptr nocapture %pResul
 ; CHECK-NEXT:    vidup.u32 q2, r4, #1
 ; CHECK-NEXT:    movw r5, #54437
 ; CHECK-NEXT:    movt r5, #21352
-; CHECK-NEXT:    vdup.32 q1, r5
 ; CHECK-NEXT:    vmov.i32 q0, #0x0
+; CHECK-NEXT:    vdup.32 q1, r5
 ; CHECK-NEXT:    dlstp.32 lr, r1
 ; CHECK-NEXT:  .LBB0_1: @ %do.body
 ; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
diff --git a/llvm/test/CodeGen/Thumb2/mve-shuffle.ll b/llvm/test/CodeGen/Thumb2/mve-shuffle.ll
index f4643f8c6c4a1..33816fec69bac 100644
--- a/llvm/test/CodeGen/Thumb2/mve-shuffle.ll
+++ b/llvm/test/CodeGen/Thumb2/mve-shuffle.ll
@@ -225,15 +225,15 @@ entry:
 define arm_aapcs_vfpcc <8 x i16> @shuffle3_i16(<8 x i16> %src) {
 ; CHECK-LABEL: shuffle3_i16:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vmov q1, q0
-; CHECK-NEXT:    vmovx.f16 s2, s5
-; CHECK-NEXT:    vmovx.f16 s0, s4
-; CHECK-NEXT:    vins.f16 s5, s4
-; CHECK-NEXT:    vins.f16 s2, s0
-; CHECK-NEXT:    vmov.f32 s3, s5
-; CHECK-NEXT:    vmovx.f16 s1, s7
-; CHECK-NEXT:    vmov.f32 s0, s6
-; CHECK-NEXT:    vins.f16 s1, s7
+; CHECK-NEXT:    vmovx.f16 s5, s3
+; CHECK-NEXT:    vmovx.f16 s6, s1
+; CHECK-NEXT:    vmovx.f16 s4, s0
+; CHECK-NEXT:    vins.f16 s1, s0
+; CHECK-NEXT:    vins.f16 s6, s4
+; CHECK-NEXT:    vins.f16 s5, s3
+; CHECK-NEXT:    vmov.f32 s4, s2
+; CHECK-NEXT:    vmov.f32 s7, s1
+; CHECK-NEXT:    vmov q0, q1
 ; CHECK-NEXT:    bx lr
 entry:
   %out = shufflevector <8 x i16> %src, <8 x i16> undef, <8 x i32> <i32 4, i32 5, i32 7, i32 6, i32 3, i32 1, i32 2, i32 0>
@@ -357,22 +357,22 @@ define arm_aapcs_vfpcc <8 x i16> @shuffle3step_i16(<32 x i16> %src) {
 ; CHECK-NEXT:    vmov.f32 s17, s4
 ; CHECK-NEXT:    vmovx.f16 s13, s3
 ; CHECK-NEXT:    vins.f16 s17, s2
-; CHECK-NEXT:    vmov.f32 s18, s7
 ; CHECK-NEXT:    vmovx.f16 s2, s8
-; CHECK-NEXT:    vmov.f32 s19, s10
+; CHECK-NEXT:    vmov.f32 s18, s7
+; CHECK-NEXT:    vmovx.f16 s14, s6
 ; CHECK-NEXT:    vins.f16 s18, s2
 ; CHECK-NEXT:    vmovx.f16 s2, s11
+; CHECK-NEXT:    vmov.f32 s19, s10
+; CHECK-NEXT:    vmovx.f16 s15, s9
 ; CHECK-NEXT:    vins.f16 s19, s2
 ; CHECK-NEXT:    vmovx.f16 s2, s1
 ; CHECK-NEXT:    vins.f16 s0, s2
 ; CHECK-NEXT:    vmovx.f16 s2, s4
 ; CHECK-NEXT:    vins.f16 s3, s2
 ; CHECK-NEXT:    vmovx.f16 s2, s7
-; CHECK-NEXT:    vmovx.f16 s4, s10
-; CHECK-NEXT:    vmovx.f16 s14, s6
-; CHECK-NEXT:    vmovx.f16 s15, s9
 ; CHECK-NEXT:    vins.f16 s6, s2
-; CHECK-NEXT:    vins.f16 s9, s4
+; CHECK-NEXT:    vmovx.f16 s2, s10
+; CHECK-NEXT:    vins.f16 s9, s2
 ; CHECK-NEXT:    vmov.f32 s1, s3
 ; CHECK-NEXT:    vins.f16 s14, s8
 ; CHECK-NEXT:    vins.f16 s15, s11
@@ -416,8 +416,8 @@ define arm_aapcs_vfpcc <8 x i16> @shuffle4step_i16(<32 x i16> %src) {
 ; CHECK-NEXT:    vmov.f32 s20, s1
 ; CHECK-NEXT:    vmovx.f16 s1, s10
 ; CHECK-NEXT:    vmov.f32 s22, s9
-; CHECK-NEXT:    vmov.f32 s23, s13
 ; CHECK-NEXT:    vmov.f32 s21, s5
+; CHECK-NEXT:    vmov.f32 s23, s13
 ; CHECK-NEXT:    vadd.i16 q4, q5, q4
 ; CHECK-NEXT:    vmovx.f16 s22, s8
 ; CHECK-NEXT:    vins.f16 s22, s1
@@ -434,9 +434,9 @@ define arm_aapcs_vfpcc <8 x i16> @shuffle4step_i16(<32 x i16> %src) {
 ; CHECK-NEXT:    vins.f16 s4, s6
 ; CHECK-NEXT:    vins.f16 s21, s1
 ; CHECK-NEXT:    vins.f16 s0, s2
-; CHECK-NEXT:    vmov.f32 s3, s12
 ; CHECK-NEXT:    vmov.f32 s1, s4
 ; CHECK-NEXT:    vmov.f32 s2, s8
+; CHECK-NEXT:    vmov.f32 s3, s12
 ; CHECK-NEXT:    vadd.i16 q0, q0, q5
 ; CHECK-NEXT:    vadd.i16 q0, q0, q4
 ; CHECK-NEXT:    vpop {d8, d9, d10, d11}
@@ -1147,15 +1147,15 @@ entry:
 define arm_aapcs_vfpcc <8 x half> @shuffle3_f16(<8 x half> %src) {
 ; CHECK-LABEL: shuffle3_f16:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vmov q1, q0
-; CHECK-NEXT:    vmovx.f16 s2, s5
-; CHECK-NEXT:    vmovx.f16 s0, s4
-; CHECK-NEXT:    vins.f16 s5, s4
-; CHECK-NEXT:    vins.f16 s2, s0
-; CHECK-NEXT:    vmov.f32 s3, s5
-; CHECK-NEXT:    vmovx.f16 s1, s7
-; CHECK-NEXT:    vmov.f32 s0, s6
-; CHECK-NEXT:    vins.f16 s1, s7
+; CHECK-NEXT:    vmovx.f16 s5, s3
+; CHECK-NEXT:    vmovx.f16 s6, s1
+; CHECK-NEXT:    vmovx.f16 s4, s0
+; CHECK-NEXT:    vins.f16 s1, s0
+; CHECK-NEXT:    vins.f16 s6, s4
+; CHECK-NEXT:    vins.f16 s5, s3
+; CHECK-NEXT:    vmov.f32 s4, s2
+; CHECK-NEXT:    vmov.f32 s7, s1
+; CHECK-NEXT:    vmov q0, q1
 ; CHECK-NEXT:    bx lr
 entry:
   %out = shufflevector <8 x half> %src, <8 x half> undef, <8 x i32> <i32 4, i32 5, i32 7, i32 6, i32 3, i32 1, i32 2, i32 0>
@@ -1348,12 +1348,12 @@ define arm_aapcs_vfpcc <8 x half> @shuffle4step_f16(<32 x half> %src) {
 ; CHECKFP-NEXT:    vins.f16 s0, s2
 ; CHECKFP-NEXT:    vmov.f32 s24, s1
 ; CHECKFP-NEXT:    vmov.f32 s26, s9
-; CHECKFP-NEXT:    vmov.f32 s27, s13
 ; CHECKFP-NEXT:    vmov.f32 s25, s5
+; CHECKFP-NEXT:    vmov.f32 s27, s13
 ; CHECKFP-NEXT:    vmov.f32 s2, s8
 ; CHECKFP-NEXT:    vadd.f16 q4, q6, q4
-; CHECKFP-NEXT:    vmov.f32 s3, s12
 ; CHECKFP-NEXT:    vmov.f32 s1, s4
+; CHECKFP-NEXT:    vmov.f32 s3, s12
 ; CHECKFP-NEXT:    vadd.f16 q0, q0, q5
 ; CHECKFP-NEXT:    vadd.f16 q0, q0, q4
 ; CHECKFP-NEXT:    vpop {d8, d9, d10, d11, d12, d13}
diff --git a/llvm/test/CodeGen/Thumb2/mve-tailpred-nonzerostart.ll b/llvm/test/CodeGen/Thumb2/mve-tailpred-nonzerostart.ll
index 42a00b61b4183..244a96595eaec 100644
--- a/llvm/test/CodeGen/Thumb2/mve-tailpred-nonzerostart.ll
+++ b/llvm/test/CodeGen/Thumb2/mve-tailpred-nonzerostart.ll
@@ -11,10 +11,10 @@ define arm_aapcs_vfpcc void @start12(ptr nocapture readonly %x, ptr nocapture re
 ; CHECK-NEXT:    poplt {r4, pc}
 ; CHECK-NEXT:  .LBB0_1: @ %vector.ph
 ; CHECK-NEXT:    vmov r12, s0
-; CHECK-NEXT:    subs r3, #12
 ; CHECK-NEXT:    adds r0, #48
 ; CHECK-NEXT:    adds r1, #48
 ; CHECK-NEXT:    adds r2, #48
+; CHECK-NEXT:    subs r3, #12
 ; CHECK-NEXT:    dlstp.32 lr, r3
 ; CHECK-NEXT:  .LBB0_2: @ %vector.body
 ; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
diff --git a/llvm/test/CodeGen/Thumb2/mve-vcvt16.ll b/llvm/test/CodeGen/Thumb2/mve-vcvt16.ll
index a5725a2a30048..cdaf4468340f7 100644
--- a/llvm/test/CodeGen/Thumb2/mve-vcvt16.ll
+++ b/llvm/test/CodeGen/Thumb2/mve-vcvt16.ll
@@ -327,11 +327,12 @@ define arm_aapcs_vfpcc <4 x float> @load_ext_4(ptr %src) {
 ; CHECK-MVE:       @ %bb.0: @ %entry
 ; CHECK-MVE-NEXT:    ldrd r0, r1, [r0]
 ; CHECK-MVE-NEXT:    vmov.32 q0[0], r0
-; CHECK-MVE-NEXT:    vmov.32 q0[1], r1
-; CHECK-MVE-NEXT:    vcvtt.f32.f16 s3, s1
-; CHECK-MVE-NEXT:    vcvtb.f32.f16 s2, s1
+; CHECK-MVE-NEXT:    vmov q1, q0
 ; CHECK-MVE-NEXT:    vcvtt.f32.f16 s1, s0
+; CHECK-MVE-NEXT:    vmov.32 q1[1], r1
 ; CHECK-MVE-NEXT:    vcvtb.f32.f16 s0, s0
+; CHECK-MVE-NEXT:    vcvtt.f32.f16 s3, s5
+; CHECK-MVE-NEXT:    vcvtb.f32.f16 s2, s5
 ; CHECK-MVE-NEXT:    bx lr
 ;
 ; CHECK-MVEFP-LABEL: load_ext_4:
diff --git a/llvm/test/CodeGen/Thumb2/mve-vld2.ll b/llvm/test/CodeGen/Thumb2/mve-vld2.ll
index 633aef46bbffc..36a035d56bcfa 100644
--- a/llvm/test/CodeGen/Thumb2/mve-vld2.ll
+++ b/llvm/test/CodeGen/Thumb2/mve-vld2.ll
@@ -514,11 +514,12 @@ define void @vld2_v2f16(ptr %src, ptr %dst) {
 ; CHECK-NEXT:    ldr r2, [r0]
 ; CHECK-NEXT:    ldr r0, [r0, #4]
 ; CHECK-NEXT:    vmov.32 q0[0], r2
-; CHECK-NEXT:    vmov.32 q0[1], r0
+; CHECK-NEXT:    vmov q1, q0
+; CHECK-NEXT:    vmov.32 q1[1], r0
 ; CHECK-NEXT:    vmovx.f16 s4, s0
-; CHECK-NEXT:    vmovx.f16 s2, s1
+; CHECK-NEXT:    vmovx.f16 s2, s5
 ; CHECK-NEXT:    vins.f16 s4, s2
-; CHECK-NEXT:    vins.f16 s0, s1
+; CHECK-NEXT:    vins.f16 s0, s5
 ; CHECK-NEXT:    vadd.f16 q0, q0, q1
 ; CHECK-NEXT:    vmov r0, s0
 ; CHECK-NEXT:    str r0, [r1]
diff --git a/llvm/test/CodeGen/Thumb2/mve-vld3.ll b/llvm/test/CodeGen/Thumb2/mve-vld3.ll
index ccdc996d75970..b207ce7bdefd1 100644
--- a/llvm/test/CodeGen/Thumb2/mve-vld3.ll
+++ b/llvm/test/CodeGen/Thumb2/mve-vld3.ll
@@ -313,11 +313,11 @@ define void @vld3_v8i16(ptr %src, ptr %dst) {
 ; CHECK-NEXT:    vmovx.f16 s2, s8
 ; CHECK-NEXT:    vins.f16 s3, s2
 ; CHECK-NEXT:    vmovx.f16 s2, s11
-; CHECK-NEXT:    vmovx.f16 s8, s14
 ; CHECK-NEXT:    vmovx.f16 s18, s10
-; CHECK-NEXT:    vmovx.f16 s19, s13
 ; CHECK-NEXT:    vins.f16 s10, s2
-; CHECK-NEXT:    vins.f16 s13, s8
+; CHECK-NEXT:    vmovx.f16 s2, s14
+; CHECK-NEXT:    vmovx.f16 s19, s13
+; CHECK-NEXT:    vins.f16 s13, s2
 ; CHECK-NEXT:    vmov.f32 s1, s3
 ; CHECK-NEXT:    vins.f16 s18, s12
 ; CHECK-NEXT:    vins.f16 s19, s15
@@ -368,55 +368,55 @@ define void @vld3_v16i16(ptr %src, ptr %dst) {
 ; CHECK-LV-NEXT:    vmovx.f16 s2, s8
 ; CHECK-LV-NEXT:    vins.f16 s3, s2
 ; CHECK-LV-NEXT:    vmovx.f16 s2, s11
-; CHECK-LV-NEXT:    vmovx.f16 s8, s14
 ; CHECK-LV-NEXT:    vmovx.f16 s18, s10
-; CHECK-LV-NEXT:    vmovx.f16 s19, s13
 ; CHECK-LV-NEXT:    vins.f16 s10, s2
-; CHECK-LV-NEXT:    vins.f16 s13, s8
+; CHECK-LV-NEXT:    vmovx.f16 s2, s14
+; CHECK-LV-NEXT:    vmovx.f16 s19, s13
+; CHECK-LV-NEXT:    vins.f16 s13, s2
 ; CHECK-LV-NEXT:    vmov.f32 s1, s3
 ; CHECK-LV-NEXT:    vins.f16 s18, s12
 ; CHECK-LV-NEXT:    vins.f16 s19, s15
-; CHECK-LV-NEXT:    vmov.f32 s3, s13
-; CHECK-LV-NEXT:    vldrw.u32 q3, [r0, #16]
 ; CHECK-LV-NEXT:    vins.f16 s17, s9
 ; CHECK-LV-NEXT:    vmov.f32 s2, s10
+; CHECK-LV-NEXT:    vmov.f32 s3, s13
+; CHECK-LV-NEXT:    vldrw.u32 q2, [r0, #16]
 ; CHECK-LV-NEXT:    vadd.i16 q0, q0, q4
-; CHECK-LV-NEXT:    vldrw.u32 q2, [r0, #32]
-; CHECK-LV-NEXT:    vadd.i16 q0, q0, q1
-; CHECK-LV-NEXT:    vmovx.f16 s6, s14
 ; CHECK-LV-NEXT:    vldrw.u32 q4, [r0]
-; CHECK-LV-NEXT:    vins.f16 s6, s8
-; CHECK-LV-NEXT:    vmov.f32 s22, s15
-; CHECK-LV-NEXT:    vmovx.f16 s8, s8
-; CHECK-LV-NEXT:    vins.f16 s22, s8
-; CHECK-LV-NEXT:    vmovx.f16 s8, s11
-; CHECK-LV-NEXT:    vmov.f32 s23, s10
-; CHECK-LV-NEXT:    vmovx.f16 s4, s16
-; CHECK-LV-NEXT:    vins.f16 s23, s8
-; CHECK-LV-NEXT:    vmovx.f16 s8, s17
-; CHECK-LV-NEXT:    vins.f16 s16, s8
-; CHECK-LV-NEXT:    vmovx.f16 s8, s12
-; CHECK-LV-NEXT:    vmovx.f16 s5, s19
-; CHECK-LV-NEXT:    vins.f16 s19, s8
-; CHECK-LV-NEXT:    vmovx.f16 s8, s15
-; CHECK-LV-NEXT:    vmovx.f16 s7, s9
-; CHECK-LV-NEXT:    vins.f16 s14, s8
-; CHECK-LV-NEXT:    vmovx.f16 s8, s10
-; CHECK-LV-NEXT:    vins.f16 s4, s18
+; CHECK-LV-NEXT:    vadd.i16 q0, q0, q1
+; CHECK-LV-NEXT:    vldrw.u32 q1, [r0, #32]
+; CHECK-LV-NEXT:    vmovx.f16 s14, s10
+; CHECK-LV-NEXT:    vmov.f32 s22, s11
+; CHECK-LV-NEXT:    vins.f16 s14, s4
+; CHECK-LV-NEXT:    vmovx.f16 s4, s4
+; CHECK-LV-NEXT:    vins.f16 s22, s4
+; CHECK-LV-NEXT:    vmovx.f16 s4, s7
+; CHECK-LV-NEXT:    vmov.f32 s23, s6
+; CHECK-LV-NEXT:    vmovx.f16 s12, s16
+; CHECK-LV-NEXT:    vins.f16 s23, s4
+; CHECK-LV-NEXT:    vmovx.f16 s4, s17
+; CHECK-LV-NEXT:    vins.f16 s16, s4
+; CHECK-LV-NEXT:    vmovx.f16 s4, s8
+; CHECK-LV-NEXT:    vmovx.f16 s13, s19
+; CHECK-LV-NEXT:    vins.f16 s19, s4
+; CHECK-LV-NEXT:    vmovx.f16 s4, s11
+; CHECK-LV-NEXT:    vmovx.f16 s15, s5
+; CHECK-LV-NEXT:    vins.f16 s10, s4
+; CHECK-LV-NEXT:    vmovx.f16 s4, s6
+; CHECK-LV-NEXT:    vins.f16 s12, s18
 ; CHECK-LV-NEXT:    vmov.f32 s20, s17
 ; CHECK-LV-NEXT:    vmovx.f16 s18, s18
-; CHECK-LV-NEXT:    vins.f16 s9, s8
-; CHECK-LV-NEXT:    vins.f16 s5, s13
+; CHECK-LV-NEXT:    vins.f16 s5, s4
+; CHECK-LV-NEXT:    vins.f16 s13, s9
 ; CHECK-LV-NEXT:    vins.f16 s20, s18
 ; CHECK-LV-NEXT:    vmov.f32 s17, s19
-; CHECK-LV-NEXT:    vins.f16 s7, s11
-; CHECK-LV-NEXT:    vmovx.f16 s13, s13
-; CHECK-LV-NEXT:    vmov.f32 s21, s12
-; CHECK-LV-NEXT:    vmov.f32 s18, s14
-; CHECK-LV-NEXT:    vins.f16 s21, s13
-; CHECK-LV-NEXT:    vmov.f32 s19, s9
+; CHECK-LV-NEXT:    vins.f16 s15, s7
+; CHECK-LV-NEXT:    vmovx.f16 s9, s9
+; CHECK-LV-NEXT:    vmov.f32 s21, s8
+; CHECK-LV-NEXT:    vmov.f32 s18, s10
+; CHECK-LV-NEXT:    vins.f16 s21, s9
+; CHECK-LV-NEXT:    vmov.f32 s19, s5
 ; CHECK-LV-NEXT:    vstrw.32 q0, [r1, #16]
-; CHECK-LV-NEXT:    vadd.i16 q1, q4, q1
+; CHECK-LV-NEXT:    vadd.i16 q1, q4, q3
 ; CHECK-LV-NEXT:    vadd.i16 q1, q1, q5
 ; CHECK-LV-NEXT:    vstrw.32 q1, [r1]
 ; CHECK-LV-NEXT:    vpop {d8, d9, d10, d11}
@@ -449,55 +449,55 @@ define void @vld3_v16i16(ptr %src, ptr %dst) {
 ; CHECK-LIS-NEXT:    vmovx.f16 s2, s8
 ; CHECK-LIS-NEXT:    vins.f16 s3, s2
 ; CHECK-LIS-NEXT:    vmovx.f16 s2, s11
-; CHECK-LIS-NEXT:    vmovx.f16 s8, s14
 ; CHECK-LIS-NEXT:    vmovx.f16 s18, s10
-; CHECK-LIS-NEXT:    vmovx.f16 s19, s13
 ; CHECK-LIS-NEXT:    vins.f16 s10, s2
-; CHECK-LIS-NEXT:    vins.f16 s13, s8
+; CHECK-LIS-NEXT:    vmovx.f16 s2, s14
+; CHECK-LIS-NEXT:    vmovx.f16 s19, s13
+; CHECK-LIS-NEXT:    vins.f16 s13, s2
 ; CHECK-LIS-NEXT:    vmov.f32 s1, s3
 ; CHECK-LIS-NEXT:    vins.f16 s18, s12
 ; CHECK-LIS-NEXT:    vins.f16 s19, s15
 ; CHECK-LIS-NEXT:    vmov.f32 s3, s13
 ; CHECK-LIS-NEXT:    vins.f16 s17, s9
 ; CHECK-LIS-NEXT:    vmov.f32 s2, s10
-; CHECK-LIS-NEXT:    vldrw.u32 q2, [r0, #32]
+; CHECK-LIS-NEXT:    vldrw.u32 q3, [r0, #16]
 ; CHECK-LIS-NEXT:    vadd.i16 q0, q0, q4
-; CHECK-LIS-NEXT:    vldrw.u32 q4, [r0, #16]
+; CHECK-LIS-NEXT:    vldrw.u32 q4, [r0]
 ; CHECK-LIS-NEXT:    vadd.i16 q0, q0, q1
-; CHECK-LIS-NEXT:    vldrw.u32 q3, [r0]
-; CHECK-LIS-NEXT:    vmovx.f16 s6, s18
-; CHECK-LIS-NEXT:    vmov.f32 s22, s19
-; CHECK-LIS-NEXT:    vins.f16 s6, s8
-; CHECK-LIS-NEXT:    vmovx.f16 s8, s8
-; CHECK-LIS-NEXT:    vins.f16 s22, s8
-; CHECK-LIS-NEXT:    vmovx.f16 s8, s11
-; CHECK-LIS-NEXT:    vmov.f32 s23, s10
-; CHECK-LIS-NEXT:    vmovx.f16 s4, s12
-; CHECK-LIS-NEXT:    vins.f16 s23, s8
-; CHECK-LIS-NEXT:    vmovx.f16 s8, s13
-; CHECK-LIS-NEXT:    vins.f16 s12, s8
+; CHECK-LIS-NEXT:    vldrw.u32 q1, [r0, #32]
+; CHECK-LIS-NEXT:    vmovx.f16 s10, s14
+; CHECK-LIS-NEXT:    vmov.f32 s22, s15
+; CHECK-LIS-NEXT:    vins.f16 s10, s4
+; CHECK-LIS-NEXT:    vmovx.f16 s4, s4
+; CHECK-LIS-NEXT:    vins.f16 s22, s4
+; CHECK-LIS-NEXT:    vmovx.f16 s4, s7
+; CHECK-LIS-NEXT:    vmov.f32 s23, s6
 ; CHECK-LIS-NEXT:    vmovx.f16 s8, s16
-; CHECK-LIS-NEXT:    vmovx.f16 s5, s15
-; CHECK-LIS-NEXT:    vins.f16 s15, s8
-; CHECK-LIS-NEXT:    vmovx.f16 s8, s19
-; CHECK-LIS-NEXT:    vins.f16 s4, s14
-; CHECK-LIS-NEXT:    vmov.f32 s20, s13
-; CHECK-LIS-NEXT:    vmovx.f16 s14, s14
-; CHECK-LIS-NEXT:    vins.f16 s18, s8
-; CHECK-LIS-NEXT:    vmovx.f16 s8, s10
-; CHECK-LIS-NEXT:    vmovx.f16 s7, s9
-; CHECK-LIS-NEXT:    vins.f16 s20, s14
-; CHECK-LIS-NEXT:    vmovx.f16 s14, s17
-; CHECK-LIS-NEXT:    vmov.f32 s21, s16
-; CHECK-LIS-NEXT:    vins.f16 s9, s8
-; CHECK-LIS-NEXT:    vins.f16 s21, s14
-; CHECK-LIS-NEXT:    vmov.f32 s13, s15
-; CHECK-LIS-NEXT:    vins.f16 s7, s11
-; CHECK-LIS-NEXT:    vins.f16 s5, s17
-; CHECK-LIS-NEXT:    vmov.f32 s14, s18
-; CHECK-LIS-NEXT:    vmov.f32 s15, s9
+; CHECK-LIS-NEXT:    vins.f16 s23, s4
+; CHECK-LIS-NEXT:    vmovx.f16 s4, s17
+; CHECK-LIS-NEXT:    vins.f16 s16, s4
+; CHECK-LIS-NEXT:    vmovx.f16 s4, s12
+; CHECK-LIS-NEXT:    vmovx.f16 s9, s19
+; CHECK-LIS-NEXT:    vins.f16 s19, s4
+; CHECK-LIS-NEXT:    vmovx.f16 s4, s15
+; CHECK-LIS-NEXT:    vmovx.f16 s11, s5
+; CHECK-LIS-NEXT:    vins.f16 s14, s4
+; CHECK-LIS-NEXT:    vmovx.f16 s4, s6
+; CHECK-LIS-NEXT:    vins.f16 s8, s18
+; CHECK-LIS-NEXT:    vmov.f32 s20, s17
+; CHECK-LIS-NEXT:    vmovx.f16 s18, s18
+; CHECK-LIS-NEXT:    vins.f16 s5, s4
+; CHECK-LIS-NEXT:    vins.f16 s9, s13
+; CHECK-LIS-NEXT:    vins.f16 s20, s18
+; CHECK-LIS-NEXT:    vmov.f32 s17, s19
+; CHECK-LIS-NEXT:    vins.f16 s11, s7
+; CHECK-LIS-NEXT:    vmovx.f16 s13, s13
+; CHECK-LIS-NEXT:    vmov.f32 s21, s12
+; CHECK-LIS-NEXT:    vmov.f32 s18, s14
+; CHECK-LIS-NEXT:    vins.f16 s21, s13
+; CHECK-LIS-NEXT:    vmov.f32 s19, s5
 ; CHECK-LIS-NEXT:    vstrw.32 q0, [r1, #16]
-; CHECK-LIS-NEXT:    vadd.i16 q1, q3, q1
+; CHECK-LIS-NEXT:    vadd.i16 q1, q4, q2
 ; CHECK-LIS-NEXT:    vadd.i16 q1, q1, q5
 ; CHECK-LIS-NEXT:    vstrw.32 q1, [r1]
 ; CHECK-LIS-NEXT:    vpop {d8, d9, d10, d11}
@@ -1194,24 +1194,25 @@ define void @vld3_v4f16(ptr %src, ptr %dst) {
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    ldrd r2, r3, [r0, #16]
 ; CHECK-NEXT:    vldrw.u32 q1, [r0]
-; CHECK-NEXT:    vmov.32 q2[0], r2
-; CHECK-NEXT:    vmovx.f16 s12, s4
-; CHECK-NEXT:    vmov.32 q2[1], r3
-; CHECK-NEXT:    vmovx.f16 s13, s7
-; CHECK-NEXT:    vmovx.f16 s0, s9
-; CHECK-NEXT:    vmov.f32 s1, s8
-; CHECK-NEXT:    vins.f16 s1, s0
-; CHECK-NEXT:    vmovx.f16 s0, s5
-; CHECK-NEXT:    vins.f16 s4, s0
-; CHECK-NEXT:    vmovx.f16 s0, s6
-; CHECK-NEXT:    vins.f16 s5, s0
-; CHECK-NEXT:    vmovx.f16 s0, s8
+; CHECK-NEXT:    vmov.32 q0[0], r2
+; CHECK-NEXT:    vmovx.f16 s8, s4
+; CHECK-NEXT:    vmov q3, q0
+; CHECK-NEXT:    vmov.f32 s1, s0
+; CHECK-NEXT:    vmov.32 q3[1], r3
+; CHECK-NEXT:    vmovx.f16 s0, s0
+; CHECK-NEXT:    vmovx.f16 s2, s13
+; CHECK-NEXT:    vmovx.f16 s9, s7
+; CHECK-NEXT:    vins.f16 s1, s2
+; CHECK-NEXT:    vmovx.f16 s2, s5
+; CHECK-NEXT:    vins.f16 s4, s2
+; CHECK-NEXT:    vmovx.f16 s2, s6
+; CHECK-NEXT:    vins.f16 s5, s2
 ; CHECK-NEXT:    vins.f16 s7, s0
 ; CHECK-NEXT:    vmov.f32 s0, s5
-; CHECK-NEXT:    vins.f16 s12, s6
-; CHECK-NEXT:    vins.f16 s13, s9
+; CHECK-NEXT:    vins.f16 s8, s6
+; CHECK-NEXT:    vins.f16 s9, s13
 ; CHECK-NEXT:    vmov.f32 s5, s7
-; CHECK-NEXT:    vadd.f16 q1, q1, q3
+; CHECK-NEXT:    vadd.f16 q1, q1, q2
 ; CHECK-NEXT:    vadd.f16 q0, q1, q0
 ; CHECK-NEXT:    vmov r0, r2, d0
 ; CHECK-NEXT:    strd r0, r2, [r1]
@@ -1228,93 +1229,49 @@ entry:
 }
 
 define void @vld3_v8f16(ptr %src, ptr %dst) {
-; CHECK-LV-LABEL: vld3_v8f16:
-; CHECK-LV:       @ %bb.0: @ %entry
-; CHECK-LV-NEXT:    .vsave {d8, d9}
-; CHECK-LV-NEXT:    vpush {d8, d9}
-; CHECK-LV-NEXT:    vldrw.u32 q2, [r0, #16]
-; CHECK-LV-NEXT:    vldrw.u32 q0, [r0]
-; CHECK-LV-NEXT:    vldrw.u32 q3, [r0, #32]
-; CHECK-LV-NEXT:    vmov.f32 s5, s8
-; CHECK-LV-NEXT:    vmovx.f16 s8, s8
-; CHECK-LV-NEXT:    vmovx.f16 s17, s3
-; CHECK-LV-NEXT:    vins.f16 s3, s8
-; CHECK-LV-NEXT:    vmovx.f16 s8, s11
-; CHECK-LV-NEXT:    vmovx.f16 s18, s10
-; CHECK-LV-NEXT:    vmovx.f16 s16, s0
-; CHECK-LV-NEXT:    vins.f16 s10, s8
-; CHECK-LV-NEXT:    vmovx.f16 s6, s2
-; CHECK-LV-NEXT:    vmov.f32 s4, s1
-; CHECK-LV-NEXT:    vmovx.f16 s8, s14
-; CHECK-LV-NEXT:    vmovx.f16 s19, s13
-; CHECK-LV-NEXT:    vins.f16 s4, s6
-; CHECK-LV-NEXT:    vmovx.f16 s6, s9
-; CHECK-LV-NEXT:    vins.f16 s16, s2
-; CHECK-LV-NEXT:    vmovx.f16 s2, s15
-; CHECK-LV-NEXT:    vmovx.f16 s7, s12
-; CHECK-LV-NEXT:    vins.f16 s18, s12
-; CHECK-LV-NEXT:    vmovx.f16 s12, s1
-; CHECK-LV-NEXT:    vins.f16 s13, s8
-; CHECK-LV-NEXT:    vins.f16 s5, s6
-; CHECK-LV-NEXT:    vmov.f32 s6, s11
-; CHECK-LV-NEXT:    vins.f16 s14, s2
-; CHECK-LV-NEXT:    vmov.f32 s1, s3
-; CHECK-LV-NEXT:    vins.f16 s19, s15
-; CHECK-LV-NEXT:    vins.f16 s17, s9
-; CHECK-LV-NEXT:    vins.f16 s0, s12
-; CHECK-LV-NEXT:    vmov.f32 s2, s10
-; CHECK-LV-NEXT:    vmov.f32 s3, s13
-; CHECK-LV-NEXT:    vins.f16 s6, s7
-; CHECK-LV-NEXT:    vmov.f32 s7, s14
-; CHECK-LV-NEXT:    vadd.f16 q0, q0, q4
-; CHECK-LV-NEXT:    vadd.f16 q0, q0, q1
-; CHECK-LV-NEXT:    vstrw.32 q0, [r1]
-; CHECK-LV-NEXT:    vpop {d8, d9}
-; CHECK-LV-NEXT:    bx lr
-;
-; CHECK-LIS-LABEL: vld3_v8f16:
-; CHECK-LIS:       @ %bb.0: @ %entry
-; CHECK-LIS-NEXT:    .vsave {d8, d9}
-; CHECK-LIS-NEXT:    vpush {d8, d9}
-; CHECK-LIS-NEXT:    vldrw.u32 q0, [r0]
-; CHECK-LIS-NEXT:    vldrw.u32 q2, [r0, #16]
-; CHECK-LIS-NEXT:    vldrw.u32 q4, [r0, #32]
-; CHECK-LIS-NEXT:    vmov.f32 s4, s1
-; CHECK-LIS-NEXT:    vmovx.f16 s6, s2
-; CHECK-LIS-NEXT:    vins.f16 s4, s6
-; CHECK-LIS-NEXT:    vmov.f32 s5, s8
-; CHECK-LIS-NEXT:    vmovx.f16 s6, s9
-; CHECK-LIS-NEXT:    vmovx.f16 s8, s8
-; CHECK-LIS-NEXT:    vmovx.f16 s13, s3
-; CHECK-LIS-NEXT:    vins.f16 s5, s6
-; CHECK-LIS-NEXT:    vins.f16 s3, s8
-; CHECK-LIS-NEXT:    vmov.f32 s6, s11
-; CHECK-LIS-NEXT:    vmovx.f16 s12, s16
-; CHECK-LIS-NEXT:    vmovx.f16 s8, s11
-; CHECK-LIS-NEXT:    vmovx.f16 s14, s10
-; CHECK-LIS-NEXT:    vins.f16 s6, s12
-; CHECK-LIS-NEXT:    vmovx.f16 s12, s0
-; CHECK-LIS-NEXT:    vins.f16 s10, s8
-; CHECK-LIS-NEXT:    vmovx.f16 s8, s18
-; CHECK-LIS-NEXT:    vmovx.f16 s15, s17
-; CHECK-LIS-NEXT:    vins.f16 s12, s2
-; CHECK-LIS-NEXT:    vmovx.f16 s2, s19
-; CHECK-LIS-NEXT:    vmovx.f16 s1, s1
-; CHECK-LIS-NEXT:    vins.f16 s17, s8
-; CHECK-LIS-NEXT:    vins.f16 s18, s2
-; CHECK-LIS-NEXT:    vins.f16 s0, s1
-; CHECK-LIS-NEXT:    vmov.f32 s1, s3
-; CHECK-LIS-NEXT:    vins.f16 s14, s16
-; CHECK-LIS-NEXT:    vins.f16 s15, s19
-; CHECK-LIS-NEXT:    vins.f16 s13, s9
-; CHECK-LIS-NEXT:    vmov.f32 s2, s10
-; CHECK-LIS-NEXT:    vmov.f32 s3, s17
-; CHECK-LIS-NEXT:    vmov.f32 s7, s18
-; CHECK-LIS-NEXT:    vadd.f16 q0, q0, q3
-; CHECK-LIS-NEXT:    vadd.f16 q0, q0, q1
-; CHECK-LIS-NEXT:    vstrw.32 q0, [r1]
-; CHECK-LIS-NEXT:    vpop {d8, d9}
-; CHECK-LIS-NEXT:    bx lr
+; CHECK-LABEL: vld3_v8f16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .vsave {d8, d9}
+; CHECK-NEXT:    vpush {d8, d9}
+; CHECK-NEXT:    vldrw.u32 q2, [r0, #16]
+; CHECK-NEXT:    vldrw.u32 q0, [r0]
+; CHECK-NEXT:    vldrw.u32 q3, [r0, #32]
+; CHECK-NEXT:    vmov.f32 s5, s8
+; CHECK-NEXT:    vmovx.f16 s8, s8
+; CHECK-NEXT:    vmovx.f16 s17, s3
+; CHECK-NEXT:    vins.f16 s3, s8
+; CHECK-NEXT:    vmovx.f16 s8, s11
+; CHECK-NEXT:    vmovx.f16 s18, s10
+; CHECK-NEXT:    vmovx.f16 s16, s0
+; CHECK-NEXT:    vins.f16 s10, s8
+; CHECK-NEXT:    vmovx.f16 s6, s2
+; CHECK-NEXT:    vmov.f32 s4, s1
+; CHECK-NEXT:    vmovx.f16 s8, s14
+; CHECK-NEXT:    vmovx.f16 s19, s13
+; CHECK-NEXT:    vins.f16 s4, s6
+; CHECK-NEXT:    vmovx.f16 s6, s9
+; CHECK-NEXT:    vins.f16 s16, s2
+; CHECK-NEXT:    vmovx.f16 s2, s15
+; CHECK-NEXT:    vmovx.f16 s7, s12
+; CHECK-NEXT:    vins.f16 s18, s12
+; CHECK-NEXT:    vmovx.f16 s12, s1
+; CHECK-NEXT:    vins.f16 s13, s8
+; CHECK-NEXT:    vins.f16 s5, s6
+; CHECK-NEXT:    vmov.f32 s6, s11
+; CHECK-NEXT:    vins.f16 s14, s2
+; CHECK-NEXT:    vmov.f32 s1, s3
+; CHECK-NEXT:    vins.f16 s19, s15
+; CHECK-NEXT:    vins.f16 s17, s9
+; CHECK-NEXT:    vins.f16 s0, s12
+; CHECK-NEXT:    vmov.f32 s2, s10
+; CHECK-NEXT:    vmov.f32 s3, s13
+; CHECK-NEXT:    vins.f16 s6, s7
+; CHECK-NEXT:    vmov.f32 s7, s14
+; CHECK-NEXT:    vadd.f16 q0, q0, q4
+; CHECK-NEXT:    vadd.f16 q0, q0, q1
+; CHECK-NEXT:    vstrw.32 q0, [r1]
+; CHECK-NEXT:    vpop {d8, d9}
+; CHECK-NEXT:    bx lr
 entry:
   %l1 = load <24 x half>, ptr %src, align 4
   %s1 = shufflevector <24 x half> %l1, <24 x half> undef, <8 x i32> <i32 0, i32 3, i32 6, i32 9, i32 12, i32 15, i32 18, i32 21>
@@ -1327,167 +1284,86 @@ entry:
 }
 
 define void @vld3_v16f16(ptr %src, ptr %dst) {
-; CHECK-LV-LABEL: vld3_v16f16:
-; CHECK-LV:       @ %bb.0: @ %entry
-; CHECK-LV-NEXT:    .vsave {d8, d9}
-; CHECK-LV-NEXT:    vpush {d8, d9}
-; CHECK-LV-NEXT:    vldrw.u32 q0, [r0, #48]
-; CHECK-LV-NEXT:    vldrw.u32 q2, [r0, #64]
-; CHECK-LV-NEXT:    vldrw.u32 q3, [r0, #80]
-; CHECK-LV-NEXT:    vmovx.f16 s6, s2
-; CHECK-LV-NEXT:    vmov.f32 s4, s1
-; CHECK-LV-NEXT:    vins.f16 s4, s6
-; CHECK-LV-NEXT:    vmovx.f16 s6, s9
-; CHECK-LV-NEXT:    vmov.f32 s5, s8
-; CHECK-LV-NEXT:    vmovx.f16 s7, s12
-; CHECK-LV-NEXT:    vins.f16 s5, s6
-; CHECK-LV-NEXT:    vmov.f32 s6, s11
-; CHECK-LV-NEXT:    vins.f16 s6, s7
-; CHECK-LV-NEXT:    vmovx.f16 s16, s15
-; CHECK-LV-NEXT:    vmov.f32 s7, s14
-; CHECK-LV-NEXT:    vmovx.f16 s17, s3
-; CHECK-LV-NEXT:    vins.f16 s7, s16
-; CHECK-LV-NEXT:    vmovx.f16 s16, s0
-; CHECK-LV-NEXT:    vins.f16 s16, s2
-; CHECK-LV-NEXT:    vmovx.f16 s2, s1
-; CHECK-LV-NEXT:    vins.f16 s0, s2
-; CHECK-LV-NEXT:    vmovx.f16 s2, s8
-; CHECK-LV-NEXT:    vins.f16 s3, s2
-; CHECK-LV-NEXT:    vmovx.f16 s2, s11
-; CHECK-LV-NEXT:    vmovx.f16 s18, s10
-; CHECK-LV-NEXT:    vins.f16 s10, s2
-; CHECK-LV-NEXT:    vmovx.f16 s2, s14
-; CHECK-LV-NEXT:    vmovx.f16 s19, s13
-; CHECK-LV-NEXT:    vins.f16 s13, s2
-; CHECK-LV-NEXT:    vmov.f32 s1, s3
-; CHECK-LV-NEXT:    vins.f16 s18, s12
-; CHECK-LV-NEXT:    vins.f16 s19, s15
-; CHECK-LV-NEXT:    vmov.f32 s3, s13
-; CHECK-LV-NEXT:    vins.f16 s17, s9
-; CHECK-LV-NEXT:    vmov.f32 s2, s10
-; CHECK-LV-NEXT:    vldrw.u32 q3, [r0, #16]
-; CHECK-LV-NEXT:    vadd.f16 q0, q0, q4
-; CHECK-LV-NEXT:    vadd.f16 q2, q0, q1
-; CHECK-LV-NEXT:    vldrw.u32 q0, [r0]
-; CHECK-LV-NEXT:    vldrw.u32 q1, [r0, #32]
-; CHECK-LV-NEXT:    vstrw.32 q2, [r1, #16]
-; CHECK-LV-NEXT:    vmovx.f16 s10, s2
-; CHECK-LV-NEXT:    vmov.f32 s8, s1
-; CHECK-LV-NEXT:    vins.f16 s8, s10
-; CHECK-LV-NEXT:    vmovx.f16 s10, s13
-; CHECK-LV-NEXT:    vmov.f32 s9, s12
-; CHECK-LV-NEXT:    vmovx.f16 s11, s4
-; CHECK-LV-NEXT:    vins.f16 s9, s10
-; CHECK-LV-NEXT:    vmov.f32 s10, s15
-; CHECK-LV-NEXT:    vins.f16 s10, s11
-; CHECK-LV-NEXT:    vmovx.f16 s16, s7
-; CHECK-LV-NEXT:    vmov.f32 s11, s6
-; CHECK-LV-NEXT:    vmovx.f16 s17, s3
-; CHECK-LV-NEXT:    vins.f16 s11, s16
-; CHECK-LV-NEXT:    vmovx.f16 s16, s0
-; CHECK-LV-NEXT:    vins.f16 s16, s2
-; CHECK-LV-NEXT:    vmovx.f16 s2, s1
-; CHECK-LV-NEXT:    vins.f16 s0, s2
-; CHECK-LV-NEXT:    vmovx.f16 s2, s12
-; CHECK-LV-NEXT:    vins.f16 s3, s2
-; CHECK-LV-NEXT:    vmovx.f16 s2, s15
-; CHECK-LV-NEXT:    vmovx.f16 s18, s14
-; CHECK-LV-NEXT:    vins.f16 s14, s2
-; CHECK-LV-NEXT:    vmovx.f16 s2, s6
-; CHECK-LV-NEXT:    vmovx.f16 s19, s5
-; CHECK-LV-NEXT:    vins.f16 s5, s2
-; CHECK-LV-NEXT:    vmov.f32 s1, s3
-; CHECK-LV-NEXT:    vins.f16 s18, s4
-; CHECK-LV-NEXT:    vins.f16 s19, s7
-; CHECK-LV-NEXT:    vins.f16 s17, s13
-; CHECK-LV-NEXT:    vmov.f32 s2, s14
-; CHECK-LV-NEXT:    vmov.f32 s3, s5
-; CHECK-LV-NEXT:    vadd.f16 q0, q0, q4
-; CHECK-LV-NEXT:    vadd.f16 q0, q0, q2
-; CHECK-LV-NEXT:    vstrw.32 q0, [r1]
-; CHECK-LV-NEXT:    vpop {d8, d9}
-; CHECK-LV-NEXT:    bx lr
-;
-; CHECK-LIS-LABEL: vld3_v16f16:
-; CHECK-LIS:       @ %bb.0: @ %entry
-; CHECK-LIS-NEXT:    .vsave {d8, d9}
-; CHECK-LIS-NEXT:    vpush {d8, d9}
-; CHECK-LIS-NEXT:    vldrw.u32 q0, [r0, #48]
-; CHECK-LIS-NEXT:    vldrw.u32 q2, [r0, #64]
-; CHECK-LIS-NEXT:    vldrw.u32 q3, [r0, #80]
-; CHECK-LIS-NEXT:    vmovx.f16 s6, s2
-; CHECK-LIS-NEXT:    vmov.f32 s4, s1
-; CHECK-LIS-NEXT:    vins.f16 s4, s6
-; CHECK-LIS-NEXT:    vmovx.f16 s6, s9
-; CHECK-LIS-NEXT:    vmov.f32 s5, s8
-; CHECK-LIS-NEXT:    vmovx.f16 s7, s12
-; CHECK-LIS-NEXT:    vins.f16 s5, s6
-; CHECK-LIS-NEXT:    vmov.f32 s6, s11
-; CHECK-LIS-NEXT:    vins.f16 s6, s7
-; CHECK-LIS-NEXT:    vmovx.f16 s16, s15
-; CHECK-LIS-NEXT:    vmov.f32 s7, s14
-; CHECK-LIS-NEXT:    vmovx.f16 s17, s3
-; CHECK-LIS-NEXT:    vins.f16 s7, s16
-; CHECK-LIS-NEXT:    vmovx.f16 s16, s0
-; CHECK-LIS-NEXT:    vins.f16 s16, s2
-; CHECK-LIS-NEXT:    vmovx.f16 s2, s1
-; CHECK-LIS-NEXT:    vins.f16 s0, s2
-; CHECK-LIS-NEXT:    vmovx.f16 s2, s8
-; CHECK-LIS-NEXT:    vins.f16 s3, s2
-; CHECK-LIS-NEXT:    vmovx.f16 s2, s11
-; CHECK-LIS-NEXT:    vmovx.f16 s18, s10
-; CHECK-LIS-NEXT:    vins.f16 s10, s2
-; CHECK-LIS-NEXT:    vmovx.f16 s2, s14
-; CHECK-LIS-NEXT:    vmovx.f16 s19, s13
-; CHECK-LIS-NEXT:    vins.f16 s13, s2
-; CHECK-LIS-NEXT:    vmov.f32 s1, s3
-; CHECK-LIS-NEXT:    vins.f16 s18, s12
-; CHECK-LIS-NEXT:    vins.f16 s19, s15
-; CHECK-LIS-NEXT:    vmov.f32 s3, s13
-; CHECK-LIS-NEXT:    vins.f16 s17, s9
-; CHECK-LIS-NEXT:    vmov.f32 s2, s10
-; CHECK-LIS-NEXT:    vldrw.u32 q3, [r0, #16]
-; CHECK-LIS-NEXT:    vadd.f16 q0, q0, q4
-; CHECK-LIS-NEXT:    vldrw.u32 q2, [r0, #32]
-; CHECK-LIS-NEXT:    vadd.f16 q1, q0, q1
-; CHECK-LIS-NEXT:    vldrw.u32 q0, [r0]
-; CHECK-LIS-NEXT:    vstrw.32 q1, [r1, #16]
-; CHECK-LIS-NEXT:    vmov.f32 s5, s12
-; CHECK-LIS-NEXT:    vmovx.f16 s6, s2
-; CHECK-LIS-NEXT:    vmov.f32 s4, s1
-; CHECK-LIS-NEXT:    vins.f16 s4, s6
-; CHECK-LIS-NEXT:    vmovx.f16 s6, s13
-; CHECK-LIS-NEXT:    vins.f16 s5, s6
-; CHECK-LIS-NEXT:    vmov.f32 s6, s15
-; CHECK-LIS-NEXT:    vmovx.f16 s7, s8
-; CHECK-LIS-NEXT:    vmovx.f16 s16, s11
-; CHECK-LIS-NEXT:    vins.f16 s6, s7
-; CHECK-LIS-NEXT:    vmov.f32 s7, s10
-; CHECK-LIS-NEXT:    vins.f16 s7, s16
-; CHECK-LIS-NEXT:    vmovx.f16 s16, s0
-; CHECK-LIS-NEXT:    vins.f16 s16, s2
-; CHECK-LIS-NEXT:    vmovx.f16 s2, s1
-; CHECK-LIS-NEXT:    vins.f16 s0, s2
-; CHECK-LIS-NEXT:    vmovx.f16 s2, s12
-; CHECK-LIS-NEXT:    vmovx.f16 s17, s3
-; CHECK-LIS-NEXT:    vins.f16 s3, s2
-; CHECK-LIS-NEXT:    vmovx.f16 s2, s15
-; CHECK-LIS-NEXT:    vmovx.f16 s18, s14
-; CHECK-LIS-NEXT:    vins.f16 s14, s2
-; CHECK-LIS-NEXT:    vmovx.f16 s2, s10
-; CHECK-LIS-NEXT:    vmovx.f16 s19, s9
-; CHECK-LIS-NEXT:    vins.f16 s9, s2
-; CHECK-LIS-NEXT:    vmov.f32 s1, s3
-; CHECK-LIS-NEXT:    vins.f16 s18, s8
-; CHECK-LIS-NEXT:    vins.f16 s19, s11
-; CHECK-LIS-NEXT:    vins.f16 s17, s13
-; CHECK-LIS-NEXT:    vmov.f32 s2, s14
-; CHECK-LIS-NEXT:    vmov.f32 s3, s9
-; CHECK-LIS-NEXT:    vadd.f16 q0, q0, q4
-; CHECK-LIS-NEXT:    vadd.f16 q0, q0, q1
-; CHECK-LIS-NEXT:    vstrw.32 q0, [r1]
-; CHECK-LIS-NEXT:    vpop {d8, d9}
-; CHECK-LIS-NEXT:    bx lr
+; CHECK-LABEL: vld3_v16f16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .vsave {d8, d9}
+; CHECK-NEXT:    vpush {d8, d9}
+; CHECK-NEXT:    vldrw.u32 q0, [r0, #48]
+; CHECK-NEXT:    vldrw.u32 q2, [r0, #64]
+; CHECK-NEXT:    vldrw.u32 q3, [r0, #80]
+; CHECK-NEXT:    vmovx.f16 s6, s2
+; CHECK-NEXT:    vmov.f32 s4, s1
+; CHECK-NEXT:    vins.f16 s4, s6
+; CHECK-NEXT:    vmovx.f16 s6, s9
+; CHECK-NEXT:    vmov.f32 s5, s8
+; CHECK-NEXT:    vmovx.f16 s7, s12
+; CHECK-NEXT:    vins.f16 s5, s6
+; CHECK-NEXT:    vmov.f32 s6, s11
+; CHECK-NEXT:    vins.f16 s6, s7
+; CHECK-NEXT:    vmovx.f16 s16, s15
+; CHECK-NEXT:    vmov.f32 s7, s14
+; CHECK-NEXT:    vmovx.f16 s17, s3
+; CHECK-NEXT:    vins.f16 s7, s16
+; CHECK-NEXT:    vmovx.f16 s16, s0
+; CHECK-NEXT:    vins.f16 s16, s2
+; CHECK-NEXT:    vmovx.f16 s2, s1
+; CHECK-NEXT:    vins.f16 s0, s2
+; CHECK-NEXT:    vmovx.f16 s2, s8
+; CHECK-NEXT:    vins.f16 s3, s2
+; CHECK-NEXT:    vmovx.f16 s2, s11
+; CHECK-NEXT:    vmovx.f16 s18, s10
+; CHECK-NEXT:    vins.f16 s10, s2
+; CHECK-NEXT:    vmovx.f16 s2, s14
+; CHECK-NEXT:    vmovx.f16 s19, s13
+; CHECK-NEXT:    vins.f16 s13, s2
+; CHECK-NEXT:    vmov.f32 s1, s3
+; CHECK-NEXT:    vins.f16 s18, s12
+; CHECK-NEXT:    vins.f16 s19, s15
+; CHECK-NEXT:    vins.f16 s17, s9
+; CHECK-NEXT:    vmov.f32 s2, s10
+; CHECK-NEXT:    vmov.f32 s3, s13
+; CHECK-NEXT:    vldrw.u32 q2, [r0, #16]
+; CHECK-NEXT:    vadd.f16 q0, q0, q4
+; CHECK-NEXT:    vadd.f16 q3, q0, q1
+; CHECK-NEXT:    vldrw.u32 q1, [r0]
+; CHECK-NEXT:    vldrw.u32 q0, [r0, #32]
+; CHECK-NEXT:    vstrw.32 q3, [r1, #16]
+; CHECK-NEXT:    vmovx.f16 s14, s6
+; CHECK-NEXT:    vmov.f32 s12, s5
+; CHECK-NEXT:    vins.f16 s12, s14
+; CHECK-NEXT:    vmovx.f16 s14, s9
+; CHECK-NEXT:    vmov.f32 s13, s8
+; CHECK-NEXT:    vmovx.f16 s18, s10
+; CHECK-NEXT:    vins.f16 s13, s14
+; CHECK-NEXT:    vmovx.f16 s15, s0
+; CHECK-NEXT:    vmov.f32 s14, s11
+; CHECK-NEXT:    vins.f16 s18, s0
+; CHECK-NEXT:    vins.f16 s14, s15
+; CHECK-NEXT:    vmovx.f16 s16, s3
+; CHECK-NEXT:    vmov.f32 s15, s2
+; CHECK-NEXT:    vmovx.f16 s0, s5
+; CHECK-NEXT:    vins.f16 s15, s16
+; CHECK-NEXT:    vmovx.f16 s16, s4
+; CHECK-NEXT:    vins.f16 s4, s0
+; CHECK-NEXT:    vmovx.f16 s0, s8
+; CHECK-NEXT:    vmovx.f16 s17, s7
+; CHECK-NEXT:    vins.f16 s7, s0
+; CHECK-NEXT:    vmovx.f16 s0, s11
+; CHECK-NEXT:    vmovx.f16 s19, s1
+; CHECK-NEXT:    vins.f16 s10, s0
+; CHECK-NEXT:    vmovx.f16 s0, s2
+; CHECK-NEXT:    vins.f16 s1, s0
+; CHECK-NEXT:    vins.f16 s16, s6
+; CHECK-NEXT:    vmov.f32 s5, s7
+; CHECK-NEXT:    vins.f16 s19, s3
+; CHECK-NEXT:    vins.f16 s17, s9
+; CHECK-NEXT:    vmov.f32 s6, s10
+; CHECK-NEXT:    vmov.f32 s7, s1
+; CHECK-NEXT:    vadd.f16 q0, q1, q4
+; CHECK-NEXT:    vadd.f16 q0, q0, q3
+; CHECK-NEXT:    vstrw.32 q0, [r1]
+; CHECK-NEXT:    vpop {d8, d9}
+; CHECK-NEXT:    bx lr
 entry:
   %l1 = load <48 x half>, ptr %src, align 4
   %s1 = shufflevector <48 x half> %l1, <48 x half> undef, <16 x i32> <i32 0, i32 3, i32 6, i32 9, i32 12, i32 15, i32 18, i32 21, i32 24, i32 27, i32 30, i32 33, i32 36, i32 39, i32 42, i32 45>
diff --git a/llvm/test/CodeGen/Thumb2/mve-vld4.ll b/llvm/test/CodeGen/Thumb2/mve-vld4.ll
index b49f19e55c895..bf16c5b7d10ff 100644
--- a/llvm/test/CodeGen/Thumb2/mve-vld4.ll
+++ b/llvm/test/CodeGen/Thumb2/mve-vld4.ll
@@ -360,50 +360,50 @@ define void @vld4_v8i16_align1(ptr %src, ptr %dst) {
 ; CHECK-NEXT:    .vsave {d8, d9, d10, d11}
 ; CHECK-NEXT:    vpush {d8, d9, d10, d11}
 ; CHECK-NEXT:    vldrb.u8 q1, [r0, #32]
-; CHECK-NEXT:    vldrb.u8 q2, [r0, #48]
 ; CHECK-NEXT:    vmovx.f16 s18, s5
 ; CHECK-NEXT:    vmovx.f16 s0, s7
 ; CHECK-NEXT:    vins.f16 s18, s0
-; CHECK-NEXT:    vmovx.f16 s19, s9
-; CHECK-NEXT:    vmovx.f16 s0, s11
+; CHECK-NEXT:    vldrb.u8 q0, [r0, #48]
 ; CHECK-NEXT:    vins.f16 s5, s7
-; CHECK-NEXT:    vins.f16 s19, s0
-; CHECK-NEXT:    vldrb.u8 q0, [r0]
-; CHECK-NEXT:    vins.f16 s9, s11
+; CHECK-NEXT:    vmovx.f16 s19, s1
+; CHECK-NEXT:    vmovx.f16 s8, s3
+; CHECK-NEXT:    vins.f16 s19, s8
+; CHECK-NEXT:    vldrb.u8 q2, [r0]
+; CHECK-NEXT:    vins.f16 s1, s3
 ; CHECK-NEXT:    vmov.f32 s22, s5
-; CHECK-NEXT:    vmovx.f16 s16, s1
-; CHECK-NEXT:    vmovx.f16 s12, s3
+; CHECK-NEXT:    vmovx.f16 s16, s9
+; CHECK-NEXT:    vmovx.f16 s12, s11
 ; CHECK-NEXT:    vins.f16 s16, s12
 ; CHECK-NEXT:    vldrb.u8 q3, [r0, #16]
-; CHECK-NEXT:    vins.f16 s1, s3
-; CHECK-NEXT:    vmov.f32 s23, s9
+; CHECK-NEXT:    vins.f16 s9, s11
+; CHECK-NEXT:    vmov.f32 s23, s1
 ; CHECK-NEXT:    vmovx.f16 s17, s13
 ; CHECK-NEXT:    vmovx.f16 s20, s15
 ; CHECK-NEXT:    vins.f16 s13, s15
 ; CHECK-NEXT:    vins.f16 s17, s20
-; CHECK-NEXT:    vmov.f32 s20, s1
+; CHECK-NEXT:    vmov.f32 s20, s9
 ; CHECK-NEXT:    vmovx.f16 s1, s6
 ; CHECK-NEXT:    vmov.f32 s21, s13
 ; CHECK-NEXT:    vadd.i16 q4, q5, q4
 ; CHECK-NEXT:    vmovx.f16 s22, s4
 ; CHECK-NEXT:    vins.f16 s22, s1
-; CHECK-NEXT:    vmovx.f16 s23, s8
-; CHECK-NEXT:    vmovx.f16 s1, s10
-; CHECK-NEXT:    vmovx.f16 s20, s0
-; CHECK-NEXT:    vins.f16 s23, s1
+; CHECK-NEXT:    vmovx.f16 s23, s0
 ; CHECK-NEXT:    vmovx.f16 s1, s2
+; CHECK-NEXT:    vmovx.f16 s20, s8
+; CHECK-NEXT:    vins.f16 s23, s1
+; CHECK-NEXT:    vmovx.f16 s1, s10
 ; CHECK-NEXT:    vins.f16 s20, s1
 ; CHECK-NEXT:    vmovx.f16 s21, s12
 ; CHECK-NEXT:    vmovx.f16 s1, s14
-; CHECK-NEXT:    vins.f16 s8, s10
-; CHECK-NEXT:    vins.f16 s4, s6
+; CHECK-NEXT:    vins.f16 s0, s2
 ; CHECK-NEXT:    vins.f16 s12, s14
+; CHECK-NEXT:    vins.f16 s4, s6
+; CHECK-NEXT:    vins.f16 s8, s10
 ; CHECK-NEXT:    vins.f16 s21, s1
-; CHECK-NEXT:    vins.f16 s0, s2
-; CHECK-NEXT:    vmov.f32 s3, s8
-; CHECK-NEXT:    vmov.f32 s1, s12
-; CHECK-NEXT:    vmov.f32 s2, s4
-; CHECK-NEXT:    vadd.i16 q0, q0, q5
+; CHECK-NEXT:    vmov.f32 s9, s12
+; CHECK-NEXT:    vmov.f32 s10, s4
+; CHECK-NEXT:    vmov.f32 s11, s0
+; CHECK-NEXT:    vadd.i16 q0, q2, q5
 ; CHECK-NEXT:    vadd.i16 q0, q0, q4
 ; CHECK-NEXT:    vstrw.32 q0, [r1]
 ; CHECK-NEXT:    vpop {d8, d9, d10, d11}
@@ -1081,51 +1081,51 @@ define void @vld4_v8f16_align1(ptr %src, ptr %dst) {
 ; CHECK-NEXT:    .vsave {d8, d9, d10, d11, d12, d13}
 ; CHECK-NEXT:    vpush {d8, d9, d10, d11, d12, d13}
 ; CHECK-NEXT:    vldrb.u8 q0, [r0, #32]
-; CHECK-NEXT:    vldrb.u8 q2, [r0, #48]
-; CHECK-NEXT:    vmovx.f16 s18, s1
+; CHECK-NEXT:    vldrb.u8 q4, [r0, #16]
+; CHECK-NEXT:    vmovx.f16 s14, s1
 ; CHECK-NEXT:    vmovx.f16 s4, s3
-; CHECK-NEXT:    vins.f16 s18, s4
-; CHECK-NEXT:    vmovx.f16 s19, s9
-; CHECK-NEXT:    vmovx.f16 s4, s11
+; CHECK-NEXT:    vins.f16 s14, s4
+; CHECK-NEXT:    vldrb.u8 q1, [r0, #48]
 ; CHECK-NEXT:    vins.f16 s1, s3
-; CHECK-NEXT:    vins.f16 s19, s4
-; CHECK-NEXT:    vldrb.u8 q1, [r0]
 ; CHECK-NEXT:    vmovx.f16 s22, s0
+; CHECK-NEXT:    vmovx.f16 s15, s5
+; CHECK-NEXT:    vmovx.f16 s8, s7
+; CHECK-NEXT:    vins.f16 s15, s8
+; CHECK-NEXT:    vldrb.u8 q2, [r0]
 ; CHECK-NEXT:    vmovx.f16 s3, s2
-; CHECK-NEXT:    vmovx.f16 s16, s5
-; CHECK-NEXT:    vmovx.f16 s12, s7
-; CHECK-NEXT:    vins.f16 s16, s12
-; CHECK-NEXT:    vldrb.u8 q3, [r0, #16]
+; CHECK-NEXT:    vmovx.f16 s23, s4
+; CHECK-NEXT:    vmovx.f16 s12, s9
+; CHECK-NEXT:    vmovx.f16 s13, s11
+; CHECK-NEXT:    vins.f16 s12, s13
+; CHECK-NEXT:    vmovx.f16 s13, s17
+; CHECK-NEXT:    vmovx.f16 s20, s19
 ; CHECK-NEXT:    vins.f16 s22, s3
-; CHECK-NEXT:    vmovx.f16 s23, s8
-; CHECK-NEXT:    vmovx.f16 s17, s13
-; CHECK-NEXT:    vmovx.f16 s20, s15
-; CHECK-NEXT:    vmovx.f16 s3, s10
-; CHECK-NEXT:    vins.f16 s17, s20
-; CHECK-NEXT:    vins.f16 s23, s3
-; CHECK-NEXT:    vmovx.f16 s20, s4
 ; CHECK-NEXT:    vmovx.f16 s3, s6
-; CHECK-NEXT:    vins.f16 s9, s11
+; CHECK-NEXT:    vins.f16 s13, s20
+; CHECK-NEXT:    vins.f16 s23, s3
+; CHECK-NEXT:    vmovx.f16 s20, s8
+; CHECK-NEXT:    vmovx.f16 s3, s10
 ; CHECK-NEXT:    vins.f16 s5, s7
-; CHECK-NEXT:    vins.f16 s13, s15
+; CHECK-NEXT:    vins.f16 s9, s11
+; CHECK-NEXT:    vins.f16 s17, s19
 ; CHECK-NEXT:    vins.f16 s20, s3
-; CHECK-NEXT:    vmovx.f16 s21, s12
-; CHECK-NEXT:    vmovx.f16 s3, s14
-; CHECK-NEXT:    vins.f16 s8, s10
-; CHECK-NEXT:    vins.f16 s0, s2
-; CHECK-NEXT:    vins.f16 s12, s14
+; CHECK-NEXT:    vmovx.f16 s21, s16
+; CHECK-NEXT:    vmovx.f16 s3, s18
 ; CHECK-NEXT:    vins.f16 s4, s6
-; CHECK-NEXT:    vmov.f32 s24, s5
+; CHECK-NEXT:    vins.f16 s0, s2
+; CHECK-NEXT:    vins.f16 s16, s18
+; CHECK-NEXT:    vins.f16 s8, s10
+; CHECK-NEXT:    vmov.f32 s24, s9
 ; CHECK-NEXT:    vins.f16 s21, s3
 ; CHECK-NEXT:    vmov.f32 s26, s1
-; CHECK-NEXT:    vmov.f32 s27, s9
-; CHECK-NEXT:    vmov.f32 s25, s13
-; CHECK-NEXT:    vmov.f32 s6, s0
-; CHECK-NEXT:    vadd.f16 q4, q6, q4
-; CHECK-NEXT:    vmov.f32 s7, s8
-; CHECK-NEXT:    vmov.f32 s5, s12
-; CHECK-NEXT:    vadd.f16 q0, q1, q5
-; CHECK-NEXT:    vadd.f16 q0, q0, q4
+; CHECK-NEXT:    vmov.f32 s25, s17
+; CHECK-NEXT:    vmov.f32 s27, s5
+; CHECK-NEXT:    vmov.f32 s10, s0
+; CHECK-NEXT:    vadd.f16 q3, q6, q3
+; CHECK-NEXT:    vmov.f32 s9, s16
+; CHECK-NEXT:    vmov.f32 s11, s4
+; CHECK-NEXT:    vadd.f16 q0, q2, q5
+; CHECK-NEXT:    vadd.f16 q0, q0, q3
 ; CHECK-NEXT:    vstrw.32 q0, [r1]
 ; CHECK-NEXT:    vpop {d8, d9, d10, d11, d12, d13}
 ; CHECK-NEXT:    bx lr
diff --git a/llvm/test/CodeGen/Thumb2/mve-vldst4.ll b/llvm/test/CodeGen/Thumb2/mve-vldst4.ll
index 2e51e9e059f65..c82bc582fa87c 100644
--- a/llvm/test/CodeGen/Thumb2/mve-vldst4.ll
+++ b/llvm/test/CodeGen/Thumb2/mve-vldst4.ll
@@ -6,118 +6,114 @@ define void @vldst4(ptr nocapture readonly %pIn, ptr nocapture %pOut, i32 %numRo
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    .save {r7, lr}
 ; CHECK-NEXT:    push {r7, lr}
-; CHECK-NEXT:    .vsave {d8, d9, d10, d11, d12, d13}
-; CHECK-NEXT:    vpush {d8, d9, d10, d11, d12, d13}
+; CHECK-NEXT:    .vsave {d8, d9, d10, d11, d12, d13, d14, d15}
+; CHECK-NEXT:    vpush {d8, d9, d10, d11, d12, d13, d14, d15}
 ; CHECK-NEXT:    mul r12, r3, r2
 ; CHECK-NEXT:    lsrs.w r2, r12, #2
 ; CHECK-NEXT:    beq.w .LBB0_3
 ; CHECK-NEXT:  @ %bb.1: @ %vector.ph
 ; CHECK-NEXT:    mvn r3, #7
-; CHECK-NEXT:    ldr r2, [sp, #56]
+; CHECK-NEXT:    ldr r2, [sp, #72]
 ; CHECK-NEXT:    and.w r3, r3, r12, lsr #2
 ; CHECK-NEXT:    sub.w r12, r3, #8
 ; CHECK-NEXT:    movs r3, #1
 ; CHECK-NEXT:    add.w lr, r3, r12, lsr #3
 ; CHECK-NEXT:  .LBB0_2: @ %vector.body
 ; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    vldrh.u16 q1, [r0, #32]
-; CHECK-NEXT:    vldrh.u16 q4, [r0, #48]
-; CHECK-NEXT:    vldrh.u16 q3, [r0], #64
-; CHECK-NEXT:    vmovx.f16 s26, s4
-; CHECK-NEXT:    vins.f16 s4, s6
-; CHECK-NEXT:    vmovx.f16 s6, s6
+; CHECK-NEXT:    vldrh.u16 q3, [r0, #32]
+; CHECK-NEXT:    vldrh.u16 q0, [r0, #48]
+; CHECK-NEXT:    vldrh.u16 q4, [r0], #64
+; CHECK-NEXT:    vmovx.f16 s8, s15
+; CHECK-NEXT:    vmovx.f16 s10, s13
 ; CHECK-NEXT:    vldrh.u16 q5, [r0, #-48]
-; CHECK-NEXT:    vmovx.f16 s27, s16
-; CHECK-NEXT:    vins.f16 s26, s6
-; CHECK-NEXT:    vmovx.f16 s6, s18
-; CHECK-NEXT:    vmovx.f16 s8, s7
-; CHECK-NEXT:    vmovx.f16 s10, s5
-; CHECK-NEXT:    vmovx.f16 s24, s12
 ; CHECK-NEXT:    vins.f16 s10, s8
-; CHECK-NEXT:    vins.f16 s27, s6
-; CHECK-NEXT:    vmovx.f16 s6, s14
-; CHECK-NEXT:    vmovx.f16 s8, s19
-; CHECK-NEXT:    vmovx.f16 s11, s17
-; CHECK-NEXT:    vmov.f32 s0, s13
+; CHECK-NEXT:    vmovx.f16 s8, s3
+; CHECK-NEXT:    vmovx.f16 s11, s1
+; CHECK-NEXT:    vmov.f32 s7, s1
 ; CHECK-NEXT:    vins.f16 s11, s8
-; CHECK-NEXT:    vmovx.f16 s25, s20
-; CHECK-NEXT:    vins.f16 s24, s6
-; CHECK-NEXT:    vmovx.f16 s6, s22
-; CHECK-NEXT:    vmovx.f16 s1, s15
-; CHECK-NEXT:    vmovx.f16 s8, s13
-; CHECK-NEXT:    vins.f16 s20, s22
-; CHECK-NEXT:    vins.f16 s16, s18
-; CHECK-NEXT:    vmov.f32 s2, s5
-; CHECK-NEXT:    vins.f16 s25, s6
-; CHECK-NEXT:    vmov.f32 s3, s17
-; CHECK-NEXT:    vins.f16 s0, s15
+; CHECK-NEXT:    vmovx.f16 s1, s19
+; CHECK-NEXT:    vmovx.f16 s8, s17
+; CHECK-NEXT:    vmov.f32 s6, s13
 ; CHECK-NEXT:    vmovx.f16 s9, s21
+; CHECK-NEXT:    vmov.f32 s4, s17
+; CHECK-NEXT:    vins.f16 s21, s23
 ; CHECK-NEXT:    vins.f16 s8, s1
 ; CHECK-NEXT:    vmovx.f16 s1, s23
-; CHECK-NEXT:    vins.f16 s12, s14
-; CHECK-NEXT:    vins.f16 s21, s23
-; CHECK-NEXT:    vmov.f32 s14, s4
-; CHECK-NEXT:    vmov.f32 s15, s16
 ; CHECK-NEXT:    vins.f16 s9, s1
-; CHECK-NEXT:    vmov.f32 s13, s20
-; CHECK-NEXT:    vmul.f16 q6, q6, r2
-; CHECK-NEXT:    vmul.f16 q3, q3, r2
-; CHECK-NEXT:    vins.f16 s2, s7
-; CHECK-NEXT:    vins.f16 s3, s19
-; CHECK-NEXT:    vmov.f32 s1, s21
-; CHECK-NEXT:    vmul.f16 q0, q0, r2
-; CHECK-NEXT:    vmovx.f16 s4, s12
-; CHECK-NEXT:    vmovx.f16 s6, s24
+; CHECK-NEXT:    vins.f16 s6, s15
+; CHECK-NEXT:    vins.f16 s7, s3
+; CHECK-NEXT:    vins.f16 s4, s19
+; CHECK-NEXT:    vmov.f32 s5, s21
+; CHECK-NEXT:    vmovx.f16 s27, s0
+; CHECK-NEXT:    vmul.f16 q1, q1, r2
+; CHECK-NEXT:    vins.f16 s0, s2
+; CHECK-NEXT:    vmovx.f16 s2, s2
 ; CHECK-NEXT:    vmul.f16 q2, q2, r2
-; CHECK-NEXT:    vmovx.f16 s7, s0
-; CHECK-NEXT:    vins.f16 s0, s8
+; CHECK-NEXT:    vmovx.f16 s3, s4
+; CHECK-NEXT:    vins.f16 s4, s8
 ; CHECK-NEXT:    vmovx.f16 s8, s8
-; CHECK-NEXT:    vins.f16 s4, s6
-; CHECK-NEXT:    vmovx.f16 s5, s1
-; CHECK-NEXT:    vmovx.f16 s6, s9
-; CHECK-NEXT:    vins.f16 s7, s8
-; CHECK-NEXT:    vins.f16 s5, s6
-; CHECK-NEXT:    vmovx.f16 s6, s13
-; CHECK-NEXT:    vmovx.f16 s8, s25
-; CHECK-NEXT:    vins.f16 s6, s8
-; CHECK-NEXT:    vmovx.f16 s19, s2
-; CHECK-NEXT:    vmovx.f16 s8, s10
-; CHECK-NEXT:    vmovx.f16 s18, s14
-; CHECK-NEXT:    vins.f16 s19, s8
-; CHECK-NEXT:    vmovx.f16 s8, s26
-; CHECK-NEXT:    vins.f16 s18, s8
-; CHECK-NEXT:    vmovx.f16 s23, s3
-; CHECK-NEXT:    vmovx.f16 s8, s11
-; CHECK-NEXT:    vins.f16 s14, s26
-; CHECK-NEXT:    vins.f16 s23, s8
-; CHECK-NEXT:    vmovx.f16 s22, s15
-; CHECK-NEXT:    vins.f16 s15, s27
-; CHECK-NEXT:    vmovx.f16 s8, s27
-; CHECK-NEXT:    vins.f16 s12, s24
-; CHECK-NEXT:    vins.f16 s13, s25
-; CHECK-NEXT:    vins.f16 s2, s10
-; CHECK-NEXT:    vins.f16 s3, s11
-; CHECK-NEXT:    vins.f16 s1, s9
-; CHECK-NEXT:    vins.f16 s22, s8
-; CHECK-NEXT:    vmov q2, q3
-; CHECK-NEXT:    vmov q6, q0
-; CHECK-NEXT:    vmov.f32 s10, s4
-; CHECK-NEXT:    vmov.f32 s11, s7
-; CHECK-NEXT:    vmov.f32 s9, s0
-; CHECK-NEXT:    vmov.f32 s17, s2
-; CHECK-NEXT:    vmov.f32 s16, s14
-; CHECK-NEXT:    vmov.f32 s21, s3
-; CHECK-NEXT:    vstrh.16 q4, [r1, #32]
-; CHECK-NEXT:    vmov.f32 s20, s15
-; CHECK-NEXT:    vmov.f32 s7, s5
-; CHECK-NEXT:    vstrh.16 q5, [r1, #48]
-; CHECK-NEXT:    vstrh.16 q2, [r1], #64
-; CHECK-NEXT:    vmov.f32 s4, s13
-; CHECK-NEXT:    vmov.f32 s5, s25
-; CHECK-NEXT:    vstrh.16 q1, [r1, #-48]
+; CHECK-NEXT:    vmovx.f16 s24, s16
+; CHECK-NEXT:    vins.f16 s27, s2
+; CHECK-NEXT:    vmovx.f16 s2, s18
+; CHECK-NEXT:    vins.f16 s3, s8
+; CHECK-NEXT:    vmovx.f16 s26, s12
+; CHECK-NEXT:    vmovx.f16 s25, s20
+; CHECK-NEXT:    vins.f16 s20, s22
+; CHECK-NEXT:    vmovx.f16 s8, s14
+; CHECK-NEXT:    vins.f16 s24, s2
+; CHECK-NEXT:    vmovx.f16 s2, s22
+; CHECK-NEXT:    vins.f16 s12, s14
+; CHECK-NEXT:    vins.f16 s16, s18
+; CHECK-NEXT:    vins.f16 s26, s8
+; CHECK-NEXT:    vins.f16 s25, s2
+; CHECK-NEXT:    vmov.f32 s18, s12
+; CHECK-NEXT:    vmov.f32 s17, s20
+; CHECK-NEXT:    vmul.f16 q6, q6, r2
+; CHECK-NEXT:    vmov.f32 s19, s0
+; CHECK-NEXT:    vmovx.f16 s0, s24
+; CHECK-NEXT:    vmul.f16 q4, q4, r2
+; CHECK-NEXT:    vmovx.f16 s15, s5
+; CHECK-NEXT:    vmovx.f16 s2, s16
+; CHECK-NEXT:    vmovx.f16 s14, s17
+; CHECK-NEXT:    vins.f16 s2, s0
+; CHECK-NEXT:    vmovx.f16 s0, s9
+; CHECK-NEXT:    vins.f16 s15, s0
+; CHECK-NEXT:    vmovx.f16 s0, s25
+; CHECK-NEXT:    vins.f16 s14, s0
+; CHECK-NEXT:    vmovx.f16 s23, s6
+; CHECK-NEXT:    vmovx.f16 s0, s10
+; CHECK-NEXT:    vmovx.f16 s22, s18
+; CHECK-NEXT:    vins.f16 s23, s0
+; CHECK-NEXT:    vmovx.f16 s0, s26
+; CHECK-NEXT:    vins.f16 s22, s0
+; CHECK-NEXT:    vmovx.f16 s31, s7
+; CHECK-NEXT:    vmovx.f16 s0, s11
+; CHECK-NEXT:    vins.f16 s6, s10
+; CHECK-NEXT:    vins.f16 s18, s26
+; CHECK-NEXT:    vins.f16 s7, s11
+; CHECK-NEXT:    vins.f16 s31, s0
+; CHECK-NEXT:    vmovx.f16 s30, s19
+; CHECK-NEXT:    vins.f16 s19, s27
+; CHECK-NEXT:    vmovx.f16 s0, s27
+; CHECK-NEXT:    vins.f16 s16, s24
+; CHECK-NEXT:    vins.f16 s5, s9
+; CHECK-NEXT:    vins.f16 s17, s25
+; CHECK-NEXT:    vins.f16 s30, s0
+; CHECK-NEXT:    vmov.f32 s1, s4
+; CHECK-NEXT:    vmov.f32 s0, s16
+; CHECK-NEXT:    vmov.f32 s21, s6
+; CHECK-NEXT:    vmov.f32 s20, s18
+; CHECK-NEXT:    vmov.f32 s29, s7
+; CHECK-NEXT:    vstrh.16 q5, [r1, #32]
+; CHECK-NEXT:    vmov.f32 s28, s19
+; CHECK-NEXT:    vstrh.16 q7, [r1, #48]
+; CHECK-NEXT:    vstrh.16 q0, [r1], #64
+; CHECK-NEXT:    vmov.f32 s12, s17
+; CHECK-NEXT:    vmov.f32 s13, s5
+; CHECK-NEXT:    vstrh.16 q3, [r1, #-48]
 ; CHECK-NEXT:    le lr, .LBB0_2
 ; CHECK-NEXT:  .LBB0_3: @ %while.end
-; CHECK-NEXT:    vpop {d8, d9, d10, d11, d12, d13}
+; CHECK-NEXT:    vpop {d8, d9, d10, d11, d12, d13, d14, d15}
 ; CHECK-NEXT:    pop {r7, pc}
 entry:
   %tmp.0.extract.trunc = trunc i32 %scale.coerce to i16
diff --git a/llvm/test/CodeGen/Thumb2/mve-vst2.ll b/llvm/test/CodeGen/Thumb2/mve-vst2.ll
index 57d08a7f3c4b2..f5a129fa2dc94 100644
--- a/llvm/test/CodeGen/Thumb2/mve-vst2.ll
+++ b/llvm/test/CodeGen/Thumb2/mve-vst2.ll
@@ -503,25 +503,28 @@ entry:
 define void @vst2_v4f16(ptr %src, ptr %dst) {
 ; CHECK-LABEL: vst2_v4f16:
 ; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .vsave {d8, d9}
+; CHECK-NEXT:    vpush {d8, d9}
 ; CHECK-NEXT:    ldrd r2, r12, [r0]
 ; CHECK-NEXT:    ldrd r3, r0, [r0, #8]
-; CHECK-NEXT:    vmov.32 q0[0], r2
-; CHECK-NEXT:    vmov.32 q1[0], r3
+; CHECK-NEXT:    vmov.32 q4[0], r2
+; CHECK-NEXT:    vmov q0, q4
+; CHECK-NEXT:    vmov.32 q3[0], r3
+; CHECK-NEXT:    vmov q2, q3
 ; CHECK-NEXT:    vmov.32 q0[1], r12
-; CHECK-NEXT:    vmov.32 q1[1], r0
-; CHECK-NEXT:    vmovx.f16 s2, s0
-; CHECK-NEXT:    vins.f16 s0, s4
-; CHECK-NEXT:    vmovx.f16 s4, s4
-; CHECK-NEXT:    vins.f16 s2, s4
-; CHECK-NEXT:    vmovx.f16 s4, s1
-; CHECK-NEXT:    vins.f16 s1, s5
-; CHECK-NEXT:    vmovx.f16 s6, s5
-; CHECK-NEXT:    vmov q2, q0
-; CHECK-NEXT:    vins.f16 s4, s6
-; CHECK-NEXT:    vmov.f32 s9, s2
-; CHECK-NEXT:    vmov.f32 s10, s1
-; CHECK-NEXT:    vmov.f32 s11, s4
-; CHECK-NEXT:    vstrh.16 q2, [r1]
+; CHECK-NEXT:    vmov.32 q2[1], r0
+; CHECK-NEXT:    vmovx.f16 s0, s12
+; CHECK-NEXT:    vmovx.f16 s5, s16
+; CHECK-NEXT:    vmov.f32 s4, s16
+; CHECK-NEXT:    vins.f16 s5, s0
+; CHECK-NEXT:    vmovx.f16 s7, s1
+; CHECK-NEXT:    vins.f16 s1, s9
+; CHECK-NEXT:    vmovx.f16 s0, s9
+; CHECK-NEXT:    vins.f16 s4, s12
+; CHECK-NEXT:    vins.f16 s7, s0
+; CHECK-NEXT:    vmov.f32 s6, s1
+; CHECK-NEXT:    vstrh.16 q1, [r1]
+; CHECK-NEXT:    vpop {d8, d9}
 ; CHECK-NEXT:    bx lr
 entry:
   %l1 = load <4 x half>, ptr %src, align 4
diff --git a/llvm/test/CodeGen/Thumb2/mve-vst3.ll b/llvm/test/CodeGen/Thumb2/mve-vst3.ll
index 85317e1fe4626..903069b7e776d 100644
--- a/llvm/test/CodeGen/Thumb2/mve-vst3.ll
+++ b/llvm/test/CodeGen/Thumb2/mve-vst3.ll
@@ -12,13 +12,11 @@ define void @vst3_v2i32(ptr %src, ptr %dst) {
 ; CHECK-NEXT:    ldrd r3, r2, [r0, #8]
 ; CHECK-NEXT:    ldrd r4, r0, [r0, #16]
 ; CHECK-NEXT:    vmov q1[2], q1[0], lr, r3
-; CHECK-NEXT:    str r2, [r1, #16]
-; CHECK-NEXT:    vmov.32 q0[0], r4
+; CHECK-NEXT:    strd r2, r0, [r1, #16]
 ; CHECK-NEXT:    vmov q1[3], q1[1], r12, r2
-; CHECK-NEXT:    vmov.32 q0[1], r0
+; CHECK-NEXT:    vmov.32 q0[0], r4
 ; CHECK-NEXT:    vmov.f32 s8, s4
 ; CHECK-NEXT:    vmov.f32 s9, s6
-; CHECK-NEXT:    str r0, [r1, #20]
 ; CHECK-NEXT:    vmov.f32 s10, s0
 ; CHECK-NEXT:    vmov.f32 s11, s5
 ; CHECK-NEXT:    vstrw.32 q2, [r1]
@@ -337,51 +335,49 @@ define void @vst3_v8i16(ptr %src, ptr %dst) {
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    .vsave {d8, d9, d10, d11}
 ; CHECK-NEXT:    vpush {d8, d9, d10, d11}
-; CHECK-NEXT:    vldrw.u32 q1, [r0]
+; CHECK-NEXT:    vldrw.u32 q0, [r0]
 ; CHECK-NEXT:    vldrw.u32 q2, [r0, #16]
-; CHECK-NEXT:    vmov.f32 s12, s7
+; CHECK-NEXT:    vmov.f32 s12, s3
 ; CHECK-NEXT:    vmov.u16 r2, q2[5]
-; CHECK-NEXT:    vmov.16 q0[0], r2
+; CHECK-NEXT:    vmov.16 q1[0], r2
 ; CHECK-NEXT:    vins.f16 s12, s11
-; CHECK-NEXT:    vmov.f32 s1, s12
+; CHECK-NEXT:    vmov.f32 s5, s12
 ; CHECK-NEXT:    vmov.u16 r2, q2[7]
 ; CHECK-NEXT:    vldrw.u32 q3, [r0, #32]
-; CHECK-NEXT:    vmov.16 q0[6], r2
-; CHECK-NEXT:    vmov.f32 s2, s7
+; CHECK-NEXT:    vmov.16 q1[6], r2
+; CHECK-NEXT:    vmov.f32 s20, s0
 ; CHECK-NEXT:    vmov.u16 r0, q2[3]
-; CHECK-NEXT:    vmovx.f16 s7, s14
+; CHECK-NEXT:    vmovx.f16 s6, s14
 ; CHECK-NEXT:    vmov.16 q4[2], r0
-; CHECK-NEXT:    vins.f16 s0, s7
-; CHECK-NEXT:    vmovx.f16 s7, s15
-; CHECK-NEXT:    vins.f16 s3, s7
-; CHECK-NEXT:    vmov.f32 s7, s6
+; CHECK-NEXT:    vins.f16 s4, s6
+; CHECK-NEXT:    vmovx.f16 s6, s3
+; CHECK-NEXT:    vmovx.f16 s3, s15
+; CHECK-NEXT:    vins.f16 s15, s6
+; CHECK-NEXT:    vmov.f32 s6, s15
+; CHECK-NEXT:    vins.f16 s7, s3
+; CHECK-NEXT:    vmov.f32 s3, s2
+; CHECK-NEXT:    vmovx.f16 s15, s1
 ; CHECK-NEXT:    vmovx.f16 s2, s2
-; CHECK-NEXT:    vins.f16 s7, s10
-; CHECK-NEXT:    vmov.f32 s20, s4
-; CHECK-NEXT:    vins.f16 s15, s2
-; CHECK-NEXT:    vmov.f32 s18, s7
 ; CHECK-NEXT:    vins.f16 s20, s8
-; CHECK-NEXT:    vmov.f32 s7, s6
-; CHECK-NEXT:    vmovx.f16 s6, s5
-; CHECK-NEXT:    vmov.f32 s2, s15
-; CHECK-NEXT:    vmovx.f16 s15, s13
-; CHECK-NEXT:    vins.f16 s13, s6
-; CHECK-NEXT:    vmovx.f16 s6, s7
 ; CHECK-NEXT:    vmov.u16 r0, q2[1]
-; CHECK-NEXT:    vmovx.f16 s4, s4
-; CHECK-NEXT:    vins.f16 s14, s6
-; CHECK-NEXT:    vmovx.f16 s6, s12
+; CHECK-NEXT:    vmovx.f16 s0, s0
+; CHECK-NEXT:    vmovx.f16 s16, s13
+; CHECK-NEXT:    vins.f16 s14, s2
+; CHECK-NEXT:    vins.f16 s3, s10
+; CHECK-NEXT:    vmovx.f16 s2, s12
 ; CHECK-NEXT:    vmov.16 q5[4], r0
-; CHECK-NEXT:    vins.f16 s5, s9
-; CHECK-NEXT:    vins.f16 s12, s4
-; CHECK-NEXT:    vins.f16 s17, s15
+; CHECK-NEXT:    vins.f16 s1, s9
+; CHECK-NEXT:    vins.f16 s12, s0
+; CHECK-NEXT:    vins.f16 s13, s15
+; CHECK-NEXT:    vins.f16 s17, s16
 ; CHECK-NEXT:    vmov.f32 s16, s13
-; CHECK-NEXT:    vins.f16 s22, s6
+; CHECK-NEXT:    vmov.f32 s18, s3
+; CHECK-NEXT:    vins.f16 s22, s2
 ; CHECK-NEXT:    vmov.f32 s19, s14
-; CHECK-NEXT:    vstrw.32 q0, [r1, #32]
-; CHECK-NEXT:    vmov.f32 s23, s5
-; CHECK-NEXT:    vstrw.32 q4, [r1, #16]
+; CHECK-NEXT:    vstrw.32 q1, [r1, #32]
 ; CHECK-NEXT:    vmov.f32 s21, s12
+; CHECK-NEXT:    vstrw.32 q4, [r1, #16]
+; CHECK-NEXT:    vmov.f32 s23, s1
 ; CHECK-NEXT:    vstrw.32 q5, [r1]
 ; CHECK-NEXT:    vpop {d8, d9, d10, d11}
 ; CHECK-NEXT:    bx lr
@@ -405,110 +401,105 @@ define void @vst3_v16i16(ptr %src, ptr %dst) {
 ; CHECK-NEXT:    vpush {d8, d9, d10, d11, d12, d13, d14, d15}
 ; CHECK-NEXT:    .pad #48
 ; CHECK-NEXT:    sub sp, #48
-; CHECK-NEXT:    vldrw.u32 q2, [r0]
-; CHECK-NEXT:    vldrw.u32 q1, [r0, #32]
-; CHECK-NEXT:    vldrw.u32 q7, [r0, #80]
-; CHECK-NEXT:    vmov.f32 s0, s11
-; CHECK-NEXT:    vmov.u16 r2, q1[5]
-; CHECK-NEXT:    vmov.16 q3[0], r2
-; CHECK-NEXT:    vins.f16 s0, s7
-; CHECK-NEXT:    vmov.f32 s2, s11
-; CHECK-NEXT:    vmov.u16 r2, q1[7]
-; CHECK-NEXT:    vmov.f64 d12, d4
-; CHECK-NEXT:    vstrw.32 q1, [sp, #32] @ 16-byte Spill
-; CHECK-NEXT:    vldrw.u32 q1, [r0, #16]
-; CHECK-NEXT:    vmov.f32 s26, s10
+; CHECK-NEXT:    vldrw.u32 q2, [r0, #32]
+; CHECK-NEXT:    vldrw.u32 q7, [r0]
+; CHECK-NEXT:    vldrw.u32 q3, [r0, #16]
+; CHECK-NEXT:    vldrw.u32 q6, [r0, #80]
+; CHECK-NEXT:    vmov.f32 s0, s31
+; CHECK-NEXT:    vmov.u16 r2, q2[5]
+; CHECK-NEXT:    vmov.16 q1[0], r2
+; CHECK-NEXT:    vins.f16 s0, s11
+; CHECK-NEXT:    vstrw.32 q2, [sp, #32] @ 16-byte Spill
+; CHECK-NEXT:    vmov.u16 r2, q2[7]
 ; CHECK-NEXT:    vldrw.u32 q2, [r0, #64]
-; CHECK-NEXT:    vmov.f32 s13, s0
-; CHECK-NEXT:    vstrw.32 q6, [sp] @ 16-byte Spill
-; CHECK-NEXT:    vmov.16 q3[6], r2
+; CHECK-NEXT:    vmov.f32 s5, s0
+; CHECK-NEXT:    vmov.16 q1[6], r2
+; CHECK-NEXT:    vmov.f32 s20, s15
 ; CHECK-NEXT:    vmovx.f16 s0, s10
-; CHECK-NEXT:    vins.f16 s12, s0
-; CHECK-NEXT:    vmovx.f16 s0, s2
-; CHECK-NEXT:    vmov.f32 s14, s11
-; CHECK-NEXT:    vins.f16 s14, s0
-; CHECK-NEXT:    vmov.f32 s20, s7
-; CHECK-NEXT:    vmov q0, q3
-; CHECK-NEXT:    vldrw.u32 q3, [r0, #48]
-; CHECK-NEXT:    vmov.u16 r2, q3[5]
-; CHECK-NEXT:    vins.f16 s20, s15
+; CHECK-NEXT:    vmov.f32 s6, s11
+; CHECK-NEXT:    vins.f16 s4, s0
+; CHECK-NEXT:    vmovx.f16 s0, s31
+; CHECK-NEXT:    vins.f16 s6, s0
+; CHECK-NEXT:    vmovx.f16 s11, s11
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    vldrw.u32 q1, [r0, #48]
+; CHECK-NEXT:    vins.f16 s3, s11
+; CHECK-NEXT:    vmovx.f16 s15, s15
+; CHECK-NEXT:    vmov.u16 r2, q1[5]
+; CHECK-NEXT:    vins.f16 s20, s7
 ; CHECK-NEXT:    vmov.16 q4[0], r2
-; CHECK-NEXT:    vmov.u16 r2, q3[7]
+; CHECK-NEXT:    vmov.u16 r2, q1[7]
 ; CHECK-NEXT:    vmov.f32 s17, s20
-; CHECK-NEXT:    vmovx.f16 s20, s31
+; CHECK-NEXT:    vmovx.f16 s20, s27
 ; CHECK-NEXT:    vmov.16 q4[6], r2
-; CHECK-NEXT:    vmov.f32 s18, s7
-; CHECK-NEXT:    vmovx.f16 s7, s30
-; CHECK-NEXT:    vins.f16 s16, s7
-; CHECK-NEXT:    vmovx.f16 s7, s18
-; CHECK-NEXT:    vins.f16 s31, s7
-; CHECK-NEXT:    vmovx.f16 s7, s11
-; CHECK-NEXT:    vins.f16 s3, s7
-; CHECK-NEXT:    vins.f16 s19, s20
 ; CHECK-NEXT:    vstrw.32 q0, [sp, #16] @ 16-byte Spill
+; CHECK-NEXT:    vins.f16 s19, s20
+; CHECK-NEXT:    vmov.f64 d10, d14
 ; CHECK-NEXT:    vldrw.u32 q0, [sp, #32] @ 16-byte Reload
-; CHECK-NEXT:    vmov.f32 s20, s24
-; CHECK-NEXT:    vmovx.f16 s11, s8
-; CHECK-NEXT:    vmov.f32 s7, s25
-; CHECK-NEXT:    vins.f16 s20, s0
+; CHECK-NEXT:    vins.f16 s27, s15
+; CHECK-NEXT:    vmovx.f16 s18, s26
+; CHECK-NEXT:    vins.f16 s28, s0
 ; CHECK-NEXT:    vmov.u16 r0, q0[1]
-; CHECK-NEXT:    vins.f16 s7, s1
+; CHECK-NEXT:    vmov.f32 s22, s30
+; CHECK-NEXT:    vmov.16 q7[4], r0
+; CHECK-NEXT:    vmovx.f16 s15, s20
+; CHECK-NEXT:    vstrw.32 q5, [sp] @ 16-byte Spill
+; CHECK-NEXT:    vmovx.f16 s20, s8
+; CHECK-NEXT:    vmov.u16 r0, q1[1]
+; CHECK-NEXT:    vins.f16 s30, s20
+; CHECK-NEXT:    vmov.f32 s20, s12
+; CHECK-NEXT:    vins.f16 s20, s4
+; CHECK-NEXT:    vmov.f32 s11, s21
 ; CHECK-NEXT:    vmov.16 q5[4], r0
-; CHECK-NEXT:    vmov.u16 r0, q3[1]
-; CHECK-NEXT:    vmov.f32 s23, s7
-; CHECK-NEXT:    vmovx.f16 s7, s24
-; CHECK-NEXT:    vmov.f32 s24, s4
-; CHECK-NEXT:    vins.f16 s8, s7
-; CHECK-NEXT:    vins.f16 s24, s12
-; CHECK-NEXT:    vmov.f32 s21, s8
-; CHECK-NEXT:    vmov.f32 s8, s5
-; CHECK-NEXT:    vmov.16 q6[4], r0
-; CHECK-NEXT:    vins.f16 s8, s13
-; CHECK-NEXT:    vmovx.f16 s4, s4
-; CHECK-NEXT:    vmov.f32 s27, s8
-; CHECK-NEXT:    vmovx.f16 s8, s28
-; CHECK-NEXT:    vins.f16 s28, s4
-; CHECK-NEXT:    vmov.f32 s4, s6
-; CHECK-NEXT:    vmov.u16 r0, q3[3]
-; CHECK-NEXT:    vins.f16 s4, s14
+; CHECK-NEXT:    vmov.u16 r0, q1[3]
+; CHECK-NEXT:    vins.f16 s11, s1
 ; CHECK-NEXT:    vmov.16 q0[2], r0
-; CHECK-NEXT:    vins.f16 s26, s8
-; CHECK-NEXT:    vmov.f32 s2, s4
-; CHECK-NEXT:    vmovx.f16 s4, s29
-; CHECK-NEXT:    vins.f16 s1, s4
-; CHECK-NEXT:    vmovx.f16 s4, s6
-; CHECK-NEXT:    vmovx.f16 s0, s5
-; CHECK-NEXT:    vins.f16 s30, s4
+; CHECK-NEXT:    vmovx.f16 s12, s12
+; CHECK-NEXT:    vins.f16 s8, s15
+; CHECK-NEXT:    vmov.f32 s31, s11
+; CHECK-NEXT:    vmovx.f16 s11, s24
+; CHECK-NEXT:    vins.f16 s24, s12
+; CHECK-NEXT:    vmov.f32 s12, s14
+; CHECK-NEXT:    vmovx.f16 s2, s25
+; CHECK-NEXT:    vmov.f32 s29, s8
+; CHECK-NEXT:    vmov.f32 s8, s13
+; CHECK-NEXT:    vins.f16 s1, s2
+; CHECK-NEXT:    vmovx.f16 s2, s14
+; CHECK-NEXT:    vins.f16 s12, s6
+; CHECK-NEXT:    vins.f16 s8, s5
+; CHECK-NEXT:    vins.f16 s26, s2
+; CHECK-NEXT:    vmovx.f16 s0, s13
+; CHECK-NEXT:    vmov.f32 s2, s12
+; CHECK-NEXT:    vldrw.u32 q3, [sp, #32] @ 16-byte Reload
 ; CHECK-NEXT:    vldrw.u32 q1, [sp] @ 16-byte Reload
-; CHECK-NEXT:    vins.f16 s29, s0
-; CHECK-NEXT:    vmov.f32 s0, s29
-; CHECK-NEXT:    vins.f16 s22, s11
-; CHECK-NEXT:    vmov.f32 s3, s30
-; CHECK-NEXT:    vstrw.32 q5, [r1]
-; CHECK-NEXT:    vmov.f32 s29, s5
-; CHECK-NEXT:    vstrw.32 q0, [r1, #64]
-; CHECK-NEXT:    vmov.f32 s30, s6
+; CHECK-NEXT:    vmov.f32 s23, s8
+; CHECK-NEXT:    vins.f16 s25, s0
 ; CHECK-NEXT:    vmov.f32 s8, s6
-; CHECK-NEXT:    vldrw.u32 q1, [sp, #32] @ 16-byte Reload
-; CHECK-NEXT:    vmov.f32 s18, s31
-; CHECK-NEXT:    vmov.u16 r0, q1[3]
-; CHECK-NEXT:    vins.f16 s8, s6
-; CHECK-NEXT:    vmov.16 q1[2], r0
-; CHECK-NEXT:    vmov.f32 s25, s28
-; CHECK-NEXT:    vmov.f32 s6, s8
-; CHECK-NEXT:    vmovx.f16 s8, s9
-; CHECK-NEXT:    vmovx.f16 s4, s29
-; CHECK-NEXT:    vins.f16 s5, s8
-; CHECK-NEXT:    vmovx.f16 s8, s30
-; CHECK-NEXT:    vins.f16 s9, s4
-; CHECK-NEXT:    vins.f16 s10, s8
-; CHECK-NEXT:    vmov.f32 s4, s9
-; CHECK-NEXT:    vmov.f32 s7, s10
-; CHECK-NEXT:    vstrw.32 q6, [r1, #48]
-; CHECK-NEXT:    vstrw.32 q1, [r1, #16]
+; CHECK-NEXT:    vmov.u16 r0, q3[3]
+; CHECK-NEXT:    vins.f16 s8, s14
+; CHECK-NEXT:    vmov.16 q3[2], r0
+; CHECK-NEXT:    vmovx.f16 s14, s9
+; CHECK-NEXT:    vmovx.f16 s12, s5
+; CHECK-NEXT:    vins.f16 s13, s14
+; CHECK-NEXT:    vmovx.f16 s14, s6
+; CHECK-NEXT:    vins.f16 s9, s12
+; CHECK-NEXT:    vins.f16 s10, s14
 ; CHECK-NEXT:    vldrw.u32 q1, [sp, #16] @ 16-byte Reload
-; CHECK-NEXT:    vstrw.32 q4, [r1, #80]
+; CHECK-NEXT:    vins.f16 s16, s18
+; CHECK-NEXT:    vmov.f32 s18, s27
+; CHECK-NEXT:    vins.f16 s22, s11
+; CHECK-NEXT:    vmov.f32 s21, s24
 ; CHECK-NEXT:    vstrw.32 q1, [r1, #32]
+; CHECK-NEXT:    vmov.f32 s0, s25
+; CHECK-NEXT:    vstrw.32 q5, [r1, #48]
+; CHECK-NEXT:    vmov.f32 s3, s26
+; CHECK-NEXT:    vstrw.32 q4, [r1, #80]
+; CHECK-NEXT:    vmov.f32 s12, s9
+; CHECK-NEXT:    vstrw.32 q0, [r1, #64]
+; CHECK-NEXT:    vmov.f32 s14, s8
+; CHECK-NEXT:    vstrw.32 q7, [r1]
+; CHECK-NEXT:    vmov.f32 s15, s10
+; CHECK-NEXT:    vstrw.32 q3, [r1, #16]
 ; CHECK-NEXT:    add sp, #48
 ; CHECK-NEXT:    vpop {d8, d9, d10, d11, d12, d13, d14, d15}
 ; CHECK-NEXT:    bx lr
@@ -628,13 +619,13 @@ define void @vst3_v8i8(ptr %src, ptr %dst) {
 ; CHECK-NEXT:    vmovx.f16 s2, s7
 ; CHECK-NEXT:    vmovx.f16 s3, s11
 ; CHECK-NEXT:    vins.f16 s3, s2
-; CHECK-NEXT:    vmovx.f16 s16, s1
+; CHECK-NEXT:    vmovx.f16 s2, s1
 ; CHECK-NEXT:    vmov.f32 s1, s15
-; CHECK-NEXT:    vmovx.f16 s18, s15
+; CHECK-NEXT:    vmovx.f16 s16, s15
+; CHECK-NEXT:    vins.f16 s1, s2
 ; CHECK-NEXT:    vmov.f32 s2, s7
 ; CHECK-NEXT:    vmov.u16 r0, q3[0]
-; CHECK-NEXT:    vins.f16 s1, s16
-; CHECK-NEXT:    vins.f16 s2, s18
+; CHECK-NEXT:    vins.f16 s2, s16
 ; CHECK-NEXT:    vmov.8 q4[0], r0
 ; CHECK-NEXT:    vmov.u16 r0, q2[0]
 ; CHECK-NEXT:    vmov.8 q4[1], r0
@@ -1183,16 +1174,18 @@ define void @vst3_v2f16(ptr %src, ptr %dst) {
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    ldrd r2, r3, [r0]
 ; CHECK-NEXT:    ldr r0, [r0, #8]
-; CHECK-NEXT:    vmov.32 q0[0], r2
+; CHECK-NEXT:    vmov.32 q1[0], r2
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    vmov.32 q2[0], r0
 ; CHECK-NEXT:    vmov.32 q0[1], r3
-; CHECK-NEXT:    vmov.32 q1[0], r0
-; CHECK-NEXT:    vmovx.f16 s2, s0
-; CHECK-NEXT:    vmovx.f16 s6, s4
-; CHECK-NEXT:    vins.f16 s4, s2
+; CHECK-NEXT:    vmovx.f16 s2, s4
+; CHECK-NEXT:    vmov.f32 s0, s4
+; CHECK-NEXT:    vmovx.f16 s4, s8
+; CHECK-NEXT:    vins.f16 s8, s2
 ; CHECK-NEXT:    vmovx.f16 s2, s1
 ; CHECK-NEXT:    vins.f16 s0, s1
-; CHECK-NEXT:    vmov.f32 s1, s4
-; CHECK-NEXT:    vins.f16 s2, s6
+; CHECK-NEXT:    vmov.f32 s1, s8
+; CHECK-NEXT:    vins.f16 s2, s4
 ; CHECK-NEXT:    vmov r3, s2
 ; CHECK-NEXT:    vmov r0, r2, d0
 ; CHECK-NEXT:    stm r1!, {r0, r2, r3}
@@ -1219,25 +1212,27 @@ define void @vst3_v4f16(ptr %src, ptr %dst) {
 ; CHECK-NEXT:    ldrd r3, r2, [r0, #8]
 ; CHECK-NEXT:    ldrd r4, r0, [r0, #16]
 ; CHECK-NEXT:    vmov q0[2], q0[0], lr, r3
-; CHECK-NEXT:    vmov.32 q1[0], r4
+; CHECK-NEXT:    vmov.32 q2[0], r4
 ; CHECK-NEXT:    vmov q0[3], q0[1], r12, r2
-; CHECK-NEXT:    vmov.32 q1[1], r0
+; CHECK-NEXT:    vmov q1, q2
 ; CHECK-NEXT:    vmovx.f16 s9, s3
-; CHECK-NEXT:    vmovx.f16 s6, s0
+; CHECK-NEXT:    vmov.32 q1[1], r0
+; CHECK-NEXT:    vmovx.f16 s4, s0
+; CHECK-NEXT:    vmov.f32 s6, s8
 ; CHECK-NEXT:    vins.f16 s0, s2
-; CHECK-NEXT:    vmovx.f16 s8, s4
+; CHECK-NEXT:    vins.f16 s6, s4
+; CHECK-NEXT:    vmovx.f16 s4, s8
 ; CHECK-NEXT:    vmovx.f16 s2, s2
-; CHECK-NEXT:    vins.f16 s4, s6
-; CHECK-NEXT:    vmovx.f16 s6, s1
-; CHECK-NEXT:    vins.f16 s2, s8
+; CHECK-NEXT:    vins.f16 s2, s4
+; CHECK-NEXT:    vmovx.f16 s4, s1
 ; CHECK-NEXT:    vmovx.f16 s8, s5
-; CHECK-NEXT:    vins.f16 s5, s6
+; CHECK-NEXT:    vins.f16 s5, s4
 ; CHECK-NEXT:    vins.f16 s9, s8
 ; CHECK-NEXT:    vmov.f32 s8, s5
 ; CHECK-NEXT:    vins.f16 s1, s3
 ; CHECK-NEXT:    vmov r0, r2, d4
 ; CHECK-NEXT:    vmov q2, q0
-; CHECK-NEXT:    vmov.f32 s9, s4
+; CHECK-NEXT:    vmov.f32 s9, s6
 ; CHECK-NEXT:    vmov.f32 s10, s2
 ; CHECK-NEXT:    vmov.f32 s11, s1
 ; CHECK-NEXT:    vstrw.32 q2, [r1]
@@ -1259,58 +1254,58 @@ entry:
 define void @vst3_v8f16(ptr %src, ptr %dst) {
 ; CHECK-LABEL: vst3_v8f16:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    .vsave {d8, d9}
-; CHECK-NEXT:    vpush {d8, d9}
-; CHECK-NEXT:    vldrw.u32 q4, [r0, #16]
-; CHECK-NEXT:    vldrw.u32 q3, [r0]
-; CHECK-NEXT:    vmovx.f16 s0, s18
-; CHECK-NEXT:    vmov.f32 s4, s15
+; CHECK-NEXT:    .vsave {d8, d9, d10, d11}
+; CHECK-NEXT:    vpush {d8, d9, d10, d11}
+; CHECK-NEXT:    vldrw.u32 q5, [r0, #16]
+; CHECK-NEXT:    vldrw.u32 q1, [r0]
+; CHECK-NEXT:    vmovx.f16 s0, s22
+; CHECK-NEXT:    vmov.f32 s8, s7
 ; CHECK-NEXT:    vmov r2, s0
-; CHECK-NEXT:    vins.f16 s4, s19
+; CHECK-NEXT:    vins.f16 s8, s23
 ; CHECK-NEXT:    vmov.16 q0[0], r2
-; CHECK-NEXT:    vmovx.f16 s10, s16
-; CHECK-NEXT:    vmov.f32 s1, s4
-; CHECK-NEXT:    vmovx.f16 s4, s19
-; CHECK-NEXT:    vmov r2, s4
-; CHECK-NEXT:    vldrw.u32 q1, [r0, #32]
+; CHECK-NEXT:    vmov.f32 s16, s4
+; CHECK-NEXT:    vmov.f32 s1, s8
+; CHECK-NEXT:    vmovx.f16 s8, s23
+; CHECK-NEXT:    vmov r2, s8
+; CHECK-NEXT:    vldrw.u32 q2, [r0, #32]
 ; CHECK-NEXT:    vmov.16 q0[6], r2
-; CHECK-NEXT:    vmov r0, s10
-; CHECK-NEXT:    vmovx.f16 s8, s6
-; CHECK-NEXT:    vmovx.f16 s2, s15
-; CHECK-NEXT:    vins.f16 s0, s8
-; CHECK-NEXT:    vmovx.f16 s8, s7
-; CHECK-NEXT:    vins.f16 s3, s8
-; CHECK-NEXT:    vmov.f32 s8, s12
-; CHECK-NEXT:    vins.f16 s8, s16
-; CHECK-NEXT:    vins.f16 s7, s2
-; CHECK-NEXT:    vmov.f32 s2, s13
-; CHECK-NEXT:    vmov.16 q2[4], r0
-; CHECK-NEXT:    vins.f16 s2, s17
-; CHECK-NEXT:    vmov.f32 s11, s2
-; CHECK-NEXT:    vmovx.f16 s2, s12
-; CHECK-NEXT:    vmovx.f16 s12, s4
-; CHECK-NEXT:    vins.f16 s4, s2
-; CHECK-NEXT:    vins.f16 s10, s12
-; CHECK-NEXT:    vmovx.f16 s12, s17
-; CHECK-NEXT:    vmov.f32 s2, s14
-; CHECK-NEXT:    vmov r0, s12
-; CHECK-NEXT:    vins.f16 s2, s18
-; CHECK-NEXT:    vmov.16 q4[2], r0
-; CHECK-NEXT:    vmovx.f16 s12, s5
-; CHECK-NEXT:    vmov.f32 s18, s2
-; CHECK-NEXT:    vmovx.f16 s2, s13
-; CHECK-NEXT:    vins.f16 s5, s2
-; CHECK-NEXT:    vmovx.f16 s2, s14
-; CHECK-NEXT:    vins.f16 s6, s2
-; CHECK-NEXT:    vmov.f32 s2, s7
-; CHECK-NEXT:    vmov.f32 s9, s4
-; CHECK-NEXT:    vins.f16 s17, s12
-; CHECK-NEXT:    vmov.f32 s16, s5
+; CHECK-NEXT:    vins.f16 s16, s20
+; CHECK-NEXT:    vmovx.f16 s2, s10
+; CHECK-NEXT:    vmovx.f16 s12, s11
+; CHECK-NEXT:    vins.f16 s0, s2
+; CHECK-NEXT:    vmovx.f16 s2, s7
+; CHECK-NEXT:    vins.f16 s11, s2
+; CHECK-NEXT:    vmovx.f16 s2, s20
+; CHECK-NEXT:    vmov r0, s2
+; CHECK-NEXT:    vmovx.f16 s2, s4
+; CHECK-NEXT:    vmovx.f16 s4, s8
+; CHECK-NEXT:    vins.f16 s8, s2
+; CHECK-NEXT:    vmovx.f16 s2, s21
+; CHECK-NEXT:    vmov.16 q4[4], r0
+; CHECK-NEXT:    vmov r0, s2
+; CHECK-NEXT:    vmovx.f16 s2, s5
+; CHECK-NEXT:    vins.f16 s3, s12
+; CHECK-NEXT:    vmov.f32 s12, s5
+; CHECK-NEXT:    vins.f16 s18, s4
+; CHECK-NEXT:    vmov.f32 s4, s6
+; CHECK-NEXT:    vmovx.f16 s14, s9
+; CHECK-NEXT:    vins.f16 s9, s2
+; CHECK-NEXT:    vmovx.f16 s2, s6
+; CHECK-NEXT:    vins.f16 s12, s21
+; CHECK-NEXT:    vins.f16 s4, s22
+; CHECK-NEXT:    vmov.16 q5[2], r0
+; CHECK-NEXT:    vins.f16 s10, s2
+; CHECK-NEXT:    vmov.f32 s2, s11
+; CHECK-NEXT:    vins.f16 s21, s14
+; CHECK-NEXT:    vmov.f32 s20, s9
+; CHECK-NEXT:    vmov.f32 s22, s4
 ; CHECK-NEXT:    vstrw.32 q0, [r1, #32]
-; CHECK-NEXT:    vmov.f32 s19, s6
-; CHECK-NEXT:    vstrw.32 q2, [r1]
-; CHECK-NEXT:    vstrw.32 q4, [r1, #16]
-; CHECK-NEXT:    vpop {d8, d9}
+; CHECK-NEXT:    vmov.f32 s23, s10
+; CHECK-NEXT:    vmov.f32 s17, s8
+; CHECK-NEXT:    vstrw.32 q5, [r1, #16]
+; CHECK-NEXT:    vmov.f32 s19, s12
+; CHECK-NEXT:    vstrw.32 q4, [r1]
+; CHECK-NEXT:    vpop {d8, d9, d10, d11}
 ; CHECK-NEXT:    bx lr
 entry:
   %l1 = load <8 x half>, ptr %src, align 4
@@ -1330,128 +1325,118 @@ define void @vst3_v16f16(ptr %src, ptr %dst) {
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    .vsave {d8, d9, d10, d11, d12, d13, d14, d15}
 ; CHECK-NEXT:    vpush {d8, d9, d10, d11, d12, d13, d14, d15}
-; CHECK-NEXT:    .pad #48
-; CHECK-NEXT:    sub sp, #48
-; CHECK-NEXT:    vldrw.u32 q3, [r0, #16]
+; CHECK-NEXT:    .pad #72
+; CHECK-NEXT:    sub sp, #72
+; CHECK-NEXT:    vldrw.u32 q5, [r0, #16]
 ; CHECK-NEXT:    vldrw.u32 q1, [r0, #48]
-; CHECK-NEXT:    vldrw.u32 q6, [r0, #32]
-; CHECK-NEXT:    vmov.f32 s8, s12
-; CHECK-NEXT:    vmovx.f16 s2, s4
-; CHECK-NEXT:    vmov.f32 s0, s13
-; CHECK-NEXT:    vins.f16 s8, s4
-; CHECK-NEXT:    vmov r2, s2
-; CHECK-NEXT:    vins.f16 s0, s5
-; CHECK-NEXT:    vmov.16 q2[4], r2
-; CHECK-NEXT:    vmov q4, q3
-; CHECK-NEXT:    vmov.f32 s11, s0
-; CHECK-NEXT:    vmovx.f16 s0, s16
-; CHECK-NEXT:    vmov.f32 s12, s8
-; CHECK-NEXT:    vmov.f64 d11, d9
-; CHECK-NEXT:    vmov.f32 s21, s17
-; CHECK-NEXT:    vmov.f64 d7, d5
 ; CHECK-NEXT:    vldrw.u32 q2, [r0, #80]
+; CHECK-NEXT:    vldrw.u32 q7, [r0, #32]
+; CHECK-NEXT:    vmov.f32 s0, s20
+; CHECK-NEXT:    vldrw.u32 q4, [r0, #64]
+; CHECK-NEXT:    vins.f16 s0, s4
 ; CHECK-NEXT:    vmovx.f16 s2, s8
+; CHECK-NEXT:    vmov.f32 s12, s0
+; CHECK-NEXT:    vmov.f32 s0, s21
+; CHECK-NEXT:    vins.f16 s0, s5
+; CHECK-NEXT:    vstr s0, [sp, #68] @ 4-byte Spill
+; CHECK-NEXT:    vmovx.f16 s0, s4
+; CHECK-NEXT:    vmov r2, s0
+; CHECK-NEXT:    vmovx.f16 s0, s20
+; CHECK-NEXT:    vmov.16 q3[4], r2
 ; CHECK-NEXT:    vins.f16 s8, s0
 ; CHECK-NEXT:    vins.f16 s14, s2
-; CHECK-NEXT:    vmovx.f16 s2, s24
-; CHECK-NEXT:    vstrw.32 q3, [sp, #16] @ 16-byte Spill
-; CHECK-NEXT:    vldrw.u32 q3, [r0]
-; CHECK-NEXT:    vmov r2, s2
-; CHECK-NEXT:    vmov.f32 s16, s12
-; CHECK-NEXT:    vmov.f32 s0, s13
-; CHECK-NEXT:    vins.f16 s16, s24
-; CHECK-NEXT:    vmov.16 q4[4], r2
-; CHECK-NEXT:    vins.f16 s0, s25
-; CHECK-NEXT:    vmov.f32 s19, s0
-; CHECK-NEXT:    vmovx.f16 s0, s12
-; CHECK-NEXT:    vmov.f64 d15, d13
-; CHECK-NEXT:    vmov.f32 s17, s13
-; CHECK-NEXT:    vmov.f32 s24, s16
-; CHECK-NEXT:    vmov.f64 d13, d9
-; CHECK-NEXT:    vmov.f64 d9, d7
-; CHECK-NEXT:    vldrw.u32 q3, [r0, #64]
-; CHECK-NEXT:    vmovx.f16 s2, s12
-; CHECK-NEXT:    vins.f16 s12, s0
-; CHECK-NEXT:    vins.f16 s26, s2
+; CHECK-NEXT:    vldrw.u32 q0, [r0]
+; CHECK-NEXT:    vstrw.32 q3, [sp, #48] @ 16-byte Spill
+; CHECK-NEXT:    vmov.f32 s12, s0
+; CHECK-NEXT:    vmov q6, q0
+; CHECK-NEXT:    vmovx.f16 s0, s28
+; CHECK-NEXT:    vins.f16 s12, s28
+; CHECK-NEXT:    vmov r2, s0
+; CHECK-NEXT:    vmovx.f16 s2, s16
+; CHECK-NEXT:    vmov.16 q3[4], r2
+; CHECK-NEXT:    vmovx.f16 s0, s24
+; CHECK-NEXT:    vins.f16 s14, s2
 ; CHECK-NEXT:    vmovx.f16 s2, s30
-; CHECK-NEXT:    vmov.f32 s0, s19
-; CHECK-NEXT:    vstrw.32 q6, [sp, #32] @ 16-byte Spill
+; CHECK-NEXT:    vins.f16 s16, s0
+; CHECK-NEXT:    vmov.f32 s0, s27
 ; CHECK-NEXT:    vmov r0, s2
+; CHECK-NEXT:    vstrw.32 q3, [sp, #32] @ 16-byte Spill
 ; CHECK-NEXT:    vins.f16 s0, s31
-; CHECK-NEXT:    vmov.f32 s29, s25
-; CHECK-NEXT:    vmov.16 q6[0], r0
-; CHECK-NEXT:    vmov.f32 s25, s0
+; CHECK-NEXT:    vmov.16 q3[0], r0
+; CHECK-NEXT:    vmov.f32 s13, s0
 ; CHECK-NEXT:    vmovx.f16 s0, s31
 ; CHECK-NEXT:    vmov r0, s0
-; CHECK-NEXT:    vmovx.f16 s0, s14
-; CHECK-NEXT:    vmov.16 q6[6], r0
-; CHECK-NEXT:    vmovx.f16 s2, s15
-; CHECK-NEXT:    vins.f16 s24, s0
-; CHECK-NEXT:    vmovx.f16 s0, s19
-; CHECK-NEXT:    vins.f16 s15, s0
+; CHECK-NEXT:    vmovx.f16 s0, s18
+; CHECK-NEXT:    vmov.16 q3[6], r0
+; CHECK-NEXT:    vmov.f32 s4, s1
+; CHECK-NEXT:    vins.f16 s12, s0
+; CHECK-NEXT:    vmovx.f16 s0, s27
+; CHECK-NEXT:    vmovx.f16 s2, s19
+; CHECK-NEXT:    vins.f16 s19, s0
+; CHECK-NEXT:    vins.f16 s4, s29
 ; CHECK-NEXT:    vmovx.f16 s0, s6
+; CHECK-NEXT:    vstr s4, [sp, #4] @ 4-byte Spill
 ; CHECK-NEXT:    vmov.f32 s4, s23
-; CHECK-NEXT:    vins.f16 s27, s2
 ; CHECK-NEXT:    vmov r0, s0
+; CHECK-NEXT:    vins.f16 s15, s2
 ; CHECK-NEXT:    vins.f16 s4, s7
 ; CHECK-NEXT:    vmov.16 q0[0], r0
-; CHECK-NEXT:    vstrw.32 q7, [sp] @ 16-byte Spill
 ; CHECK-NEXT:    vmov.f32 s1, s4
 ; CHECK-NEXT:    vmovx.f16 s4, s7
 ; CHECK-NEXT:    vmov r0, s4
-; CHECK-NEXT:    vmovx.f16 s4, s10
-; CHECK-NEXT:    vmov.16 q0[6], r0
-; CHECK-NEXT:    vldrw.u32 q7, [sp, #32] @ 16-byte Reload
-; CHECK-NEXT:    vins.f16 s0, s4
 ; CHECK-NEXT:    vmovx.f16 s4, s11
-; CHECK-NEXT:    vmovx.f16 s2, s23
+; CHECK-NEXT:    vmov.16 q0[6], r0
+; CHECK-NEXT:    vstrw.32 q6, [sp, #8] @ 16-byte Spill
+; CHECK-NEXT:    vmovx.f16 s2, s10
 ; CHECK-NEXT:    vins.f16 s3, s4
-; CHECK-NEXT:    vmovx.f16 s4, s5
+; CHECK-NEXT:    vins.f16 s0, s2
+; CHECK-NEXT:    vmovx.f16 s2, s23
 ; CHECK-NEXT:    vins.f16 s11, s2
 ; CHECK-NEXT:    vmov.f32 s2, s22
-; CHECK-NEXT:    vmov r0, s4
 ; CHECK-NEXT:    vins.f16 s2, s6
+; CHECK-NEXT:    vldrw.u32 q6, [sp, #32] @ 16-byte Reload
+; CHECK-NEXT:    vstr s2, [sp, #28] @ 4-byte Spill
+; CHECK-NEXT:    vmovx.f16 s2, s5
+; CHECK-NEXT:    vmov r0, s2
+; CHECK-NEXT:    vldr s27, [sp, #4] @ 4-byte Reload
 ; CHECK-NEXT:    vmov.16 q1[2], r0
-; CHECK-NEXT:    vmov.f32 s29, s12
+; CHECK-NEXT:    vmov.f32 s2, s11
 ; CHECK-NEXT:    vmovx.f16 s4, s21
-; CHECK-NEXT:    vmovx.f16 s12, s9
+; CHECK-NEXT:    vmovx.f16 s6, s9
 ; CHECK-NEXT:    vins.f16 s9, s4
 ; CHECK-NEXT:    vmovx.f16 s4, s22
+; CHECK-NEXT:    vldrw.u32 q5, [sp, #8] @ 16-byte Reload
 ; CHECK-NEXT:    vins.f16 s10, s4
-; CHECK-NEXT:    vmov.f32 s21, s17
-; CHECK-NEXT:    vmov.f32 s22, s18
-; CHECK-NEXT:    vins.f16 s5, s12
-; CHECK-NEXT:    vmov.f32 s4, s18
-; CHECK-NEXT:    vldrw.u32 q4, [sp] @ 16-byte Reload
-; CHECK-NEXT:    vstrw.32 q7, [sp, #32] @ 16-byte Spill
-; CHECK-NEXT:    vmov.f32 s6, s2
-; CHECK-NEXT:    vmovx.f16 s12, s17
-; CHECK-NEXT:    vins.f16 s4, s18
-; CHECK-NEXT:    vmov r0, s12
-; CHECK-NEXT:    vmovx.f16 s12, s13
-; CHECK-NEXT:    vmov.16 q7[2], r0
-; CHECK-NEXT:    vmov.f32 s2, s11
-; CHECK-NEXT:    vmov.f32 s30, s4
+; CHECK-NEXT:    vmovx.f16 s4, s29
+; CHECK-NEXT:    vins.f16 s5, s6
+; CHECK-NEXT:    vmov.f32 s11, s22
+; CHECK-NEXT:    vmovx.f16 s6, s17
+; CHECK-NEXT:    vmov r0, s4
 ; CHECK-NEXT:    vmovx.f16 s4, s21
-; CHECK-NEXT:    vins.f16 s13, s4
+; CHECK-NEXT:    vins.f16 s11, s30
+; CHECK-NEXT:    vmov.16 q7[2], r0
+; CHECK-NEXT:    vins.f16 s17, s4
 ; CHECK-NEXT:    vmovx.f16 s4, s22
-; CHECK-NEXT:    vins.f16 s14, s4
-; CHECK-NEXT:    vldrw.u32 q5, [sp, #16] @ 16-byte Reload
+; CHECK-NEXT:    vldrw.u32 q5, [sp, #48] @ 16-byte Reload
+; CHECK-NEXT:    vins.f16 s18, s4
+; CHECK-NEXT:    vins.f16 s29, s6
+; CHECK-NEXT:    vldr s23, [sp, #68] @ 4-byte Reload
+; CHECK-NEXT:    vldr s6, [sp, #28] @ 4-byte Reload
+; CHECK-NEXT:    vmov.f32 s25, s16
+; CHECK-NEXT:    vmov.f32 s14, s19
 ; CHECK-NEXT:    vstrw.32 q0, [r1, #80]
-; CHECK-NEXT:    vldrw.u32 q0, [sp, #32] @ 16-byte Reload
-; CHECK-NEXT:    vmov.f32 s26, s15
-; CHECK-NEXT:    vins.f16 s29, s12
 ; CHECK-NEXT:    vmov.f32 s21, s8
-; CHECK-NEXT:    vstrw.32 q6, [r1, #32]
+; CHECK-NEXT:    vstrw.32 q3, [r1, #32]
 ; CHECK-NEXT:    vmov.f32 s4, s9
 ; CHECK-NEXT:    vstrw.32 q5, [r1, #48]
 ; CHECK-NEXT:    vmov.f32 s7, s10
-; CHECK-NEXT:    vstrw.32 q0, [r1]
-; CHECK-NEXT:    vmov.f32 s28, s13
+; CHECK-NEXT:    vstrw.32 q6, [r1]
+; CHECK-NEXT:    vmov.f32 s28, s17
 ; CHECK-NEXT:    vstrw.32 q1, [r1, #64]
-; CHECK-NEXT:    vmov.f32 s31, s14
+; CHECK-NEXT:    vmov.f32 s30, s11
+; CHECK-NEXT:    vmov.f32 s31, s18
 ; CHECK-NEXT:    vstrw.32 q7, [r1, #16]
-; CHECK-NEXT:    add sp, #48
+; CHECK-NEXT:    add sp, #72
 ; CHECK-NEXT:    vpop {d8, d9, d10, d11, d12, d13, d14, d15}
 ; CHECK-NEXT:    bx lr
 entry:
diff --git a/llvm/test/CodeGen/Thumb2/mve-vst4.ll b/llvm/test/CodeGen/Thumb2/mve-vst4.ll
index b36904495e878..9dba1cc2d6ed7 100644
--- a/llvm/test/CodeGen/Thumb2/mve-vst4.ll
+++ b/llvm/test/CodeGen/Thumb2/mve-vst4.ll
@@ -397,55 +397,51 @@ define void @vst4_v8i16_align1(ptr %src, ptr %dst) {
 ; CHECK-NEXT:    vpush {d8, d9, d10, d11, d12, d13}
 ; CHECK-NEXT:    vldrw.u32 q1, [r0, #32]
 ; CHECK-NEXT:    vldrw.u32 q2, [r0, #48]
-; CHECK-NEXT:    vldrw.u32 q4, [r0, #16]
-; CHECK-NEXT:    vmovx.f16 s12, s5
-; CHECK-NEXT:    vmovx.f16 s0, s9
-; CHECK-NEXT:    vins.f16 s5, s9
-; CHECK-NEXT:    vins.f16 s12, s0
-; CHECK-NEXT:    vmov q0, q1
-; CHECK-NEXT:    vmovx.f16 s27, s4
-; CHECK-NEXT:    vins.f16 s4, s8
-; CHECK-NEXT:    vmov.f32 s3, s12
 ; CHECK-NEXT:    vldrw.u32 q3, [r0]
-; CHECK-NEXT:    vmov.f32 s5, s4
-; CHECK-NEXT:    vmovx.f16 s8, s8
-; CHECK-NEXT:    vmovx.f16 s0, s17
-; CHECK-NEXT:    vmovx.f16 s2, s13
-; CHECK-NEXT:    vins.f16 s27, s8
-; CHECK-NEXT:    vmovx.f16 s4, s12
-; CHECK-NEXT:    vmovx.f16 s8, s16
-; CHECK-NEXT:    vins.f16 s13, s17
-; CHECK-NEXT:    vins.f16 s12, s16
-; CHECK-NEXT:    vmov q5, q3
+; CHECK-NEXT:    vldrw.u32 q4, [r0, #16]
+; CHECK-NEXT:    vmovx.f16 s23, s4
 ; CHECK-NEXT:    vins.f16 s4, s8
-; CHECK-NEXT:    vmov.f32 s22, s4
-; CHECK-NEXT:    vmovx.f16 s4, s11
-; CHECK-NEXT:    vmov.f32 s23, s27
+; CHECK-NEXT:    vmov.f32 s21, s4
+; CHECK-NEXT:    vmovx.f16 s22, s12
+; CHECK-NEXT:    vmovx.f16 s4, s16
 ; CHECK-NEXT:    vmovx.f16 s27, s7
+; CHECK-NEXT:    vins.f16 s22, s4
+; CHECK-NEXT:    vmovx.f16 s4, s11
 ; CHECK-NEXT:    vins.f16 s7, s11
 ; CHECK-NEXT:    vins.f16 s27, s4
 ; CHECK-NEXT:    vmovx.f16 s26, s15
 ; CHECK-NEXT:    vmovx.f16 s4, s19
+; CHECK-NEXT:    vmovx.f16 s3, s5
+; CHECK-NEXT:    vins.f16 s5, s9
+; CHECK-NEXT:    vmovx.f16 s0, s9
 ; CHECK-NEXT:    vmov.f32 s25, s7
 ; CHECK-NEXT:    vins.f16 s26, s4
 ; CHECK-NEXT:    vmovx.f16 s7, s6
 ; CHECK-NEXT:    vmovx.f16 s4, s10
 ; CHECK-NEXT:    vins.f16 s6, s10
-; CHECK-NEXT:    vmov.f32 s21, s5
+; CHECK-NEXT:    vins.f16 s3, s0
+; CHECK-NEXT:    vmovx.f16 s2, s13
+; CHECK-NEXT:    vmovx.f16 s0, s17
+; CHECK-NEXT:    vmov.f32 s1, s5
+; CHECK-NEXT:    vmovx.f16 s8, s8
+; CHECK-NEXT:    vins.f16 s12, s16
 ; CHECK-NEXT:    vins.f16 s15, s19
 ; CHECK-NEXT:    vins.f16 s7, s4
 ; CHECK-NEXT:    vmov.f32 s5, s6
 ; CHECK-NEXT:    vmovx.f16 s6, s14
 ; CHECK-NEXT:    vmovx.f16 s4, s18
+; CHECK-NEXT:    vins.f16 s13, s17
 ; CHECK-NEXT:    vins.f16 s14, s18
 ; CHECK-NEXT:    vins.f16 s2, s0
 ; CHECK-NEXT:    vmov.f32 s0, s13
-; CHECK-NEXT:    vmov.f32 s24, s15
 ; CHECK-NEXT:    vins.f16 s6, s4
-; CHECK-NEXT:    vmov.f32 s4, s14
+; CHECK-NEXT:    vmov.f32 s24, s15
+; CHECK-NEXT:    vins.f16 s23, s8
+; CHECK-NEXT:    vmov.f32 s20, s12
 ; CHECK-NEXT:    vstrb.8 q6, [r1, #48]
-; CHECK-NEXT:    vstrb.8 q1, [r1, #32]
+; CHECK-NEXT:    vmov.f32 s4, s14
 ; CHECK-NEXT:    vstrb.8 q0, [r1, #16]
+; CHECK-NEXT:    vstrb.8 q1, [r1, #32]
 ; CHECK-NEXT:    vstrb.8 q5, [r1]
 ; CHECK-NEXT:    vpop {d8, d9, d10, d11, d12, d13}
 ; CHECK-NEXT:    bx lr
@@ -1002,22 +998,28 @@ entry:
 define void @vst4_v2f16(ptr %src, ptr %dst) {
 ; CHECK-LABEL: vst4_v2f16:
 ; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .vsave {d8, d9}
+; CHECK-NEXT:    vpush {d8, d9}
 ; CHECK-NEXT:    ldm.w r0, {r2, r3, r12}
-; CHECK-NEXT:    vmov.32 q1[0], r12
+; CHECK-NEXT:    vmov.32 q3[0], r12
+; CHECK-NEXT:    vmov q4, q3
+; CHECK-NEXT:    vmov.f32 s1, s12
 ; CHECK-NEXT:    ldr r0, [r0, #12]
-; CHECK-NEXT:    vmov.32 q0[0], r2
-; CHECK-NEXT:    vmov.32 q0[1], r3
-; CHECK-NEXT:    vmov.32 q1[1], r0
-; CHECK-NEXT:    vmovx.f16 s2, s0
-; CHECK-NEXT:    vmovx.f16 s6, s1
-; CHECK-NEXT:    vmovx.f16 s3, s4
+; CHECK-NEXT:    vmovx.f16 s3, s12
+; CHECK-NEXT:    vmov.32 q1[0], r2
+; CHECK-NEXT:    vmov q2, q1
+; CHECK-NEXT:    vmov.32 q4[1], r0
+; CHECK-NEXT:    vmov.32 q2[1], r3
+; CHECK-NEXT:    vmov.f32 s0, s4
+; CHECK-NEXT:    vmovx.f16 s2, s4
+; CHECK-NEXT:    vmovx.f16 s6, s9
+; CHECK-NEXT:    vmovx.f16 s4, s17
+; CHECK-NEXT:    vins.f16 s0, s9
+; CHECK-NEXT:    vins.f16 s1, s17
 ; CHECK-NEXT:    vins.f16 s2, s6
-; CHECK-NEXT:    vmovx.f16 s6, s5
-; CHECK-NEXT:    vins.f16 s4, s5
-; CHECK-NEXT:    vins.f16 s0, s1
-; CHECK-NEXT:    vins.f16 s3, s6
-; CHECK-NEXT:    vmov.f32 s1, s4
+; CHECK-NEXT:    vins.f16 s3, s4
 ; CHECK-NEXT:    vstrh.16 q0, [r1]
+; CHECK-NEXT:    vpop {d8, d9}
 ; CHECK-NEXT:    bx lr
 entry:
   %l1 = load <2 x half>, ptr %src, align 4
@@ -1155,61 +1157,57 @@ entry:
 define void @vst4_v8f16_align1(ptr %src, ptr %dst) {
 ; CHECK-LABEL: vst4_v8f16_align1:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    .vsave {d9, d10, d11, d12, d13}
-; CHECK-NEXT:    vpush {d9, d10, d11, d12, d13}
+; CHECK-NEXT:    .vsave {d8, d9, d10, d11, d12, d13}
+; CHECK-NEXT:    vpush {d8, d9, d10, d11, d12, d13}
 ; CHECK-NEXT:    vldrw.u32 q1, [r0, #32]
 ; CHECK-NEXT:    vldrw.u32 q5, [r0, #48]
-; CHECK-NEXT:    vldrw.u32 q6, [r0, #16]
 ; CHECK-NEXT:    vldrw.u32 q2, [r0]
-; CHECK-NEXT:    vmovx.f16 s0, s5
-; CHECK-NEXT:    vmovx.f16 s2, s21
-; CHECK-NEXT:    vins.f16 s0, s2
+; CHECK-NEXT:    vldrw.u32 q6, [r0, #16]
+; CHECK-NEXT:    vmovx.f16 s3, s5
+; CHECK-NEXT:    vmovx.f16 s0, s21
+; CHECK-NEXT:    vins.f16 s3, s0
 ; CHECK-NEXT:    vmovx.f16 s2, s9
-; CHECK-NEXT:    vmovx.f16 s12, s25
-; CHECK-NEXT:    vmovx.f16 s19, s4
-; CHECK-NEXT:    vins.f16 s2, s12
-; CHECK-NEXT:    vmovx.f16 s12, s20
-; CHECK-NEXT:    vins.f16 s19, s12
-; CHECK-NEXT:    vmovx.f16 s12, s8
-; CHECK-NEXT:    vmovx.f16 s14, s24
-; CHECK-NEXT:    vmovx.f16 s15, s7
-; CHECK-NEXT:    vins.f16 s12, s14
-; CHECK-NEXT:    vmovx.f16 s14, s23
-; CHECK-NEXT:    vins.f16 s15, s14
-; CHECK-NEXT:    vmovx.f16 s14, s11
-; CHECK-NEXT:    vmovx.f16 s1, s27
+; CHECK-NEXT:    vmovx.f16 s0, s25
+; CHECK-NEXT:    vmovx.f16 s15, s4
+; CHECK-NEXT:    vins.f16 s2, s0
+; CHECK-NEXT:    vmovx.f16 s0, s20
+; CHECK-NEXT:    vins.f16 s15, s0
+; CHECK-NEXT:    vmovx.f16 s14, s8
+; CHECK-NEXT:    vmovx.f16 s0, s24
+; CHECK-NEXT:    vmovx.f16 s19, s7
+; CHECK-NEXT:    vins.f16 s14, s0
+; CHECK-NEXT:    vmovx.f16 s0, s23
+; CHECK-NEXT:    vins.f16 s19, s0
+; CHECK-NEXT:    vmovx.f16 s18, s11
+; CHECK-NEXT:    vmovx.f16 s0, s27
 ; CHECK-NEXT:    vins.f16 s7, s23
-; CHECK-NEXT:    vins.f16 s14, s1
+; CHECK-NEXT:    vins.f16 s18, s0
 ; CHECK-NEXT:    vmovx.f16 s23, s6
-; CHECK-NEXT:    vmovx.f16 s1, s22
-; CHECK-NEXT:    vins.f16 s6, s22
+; CHECK-NEXT:    vmovx.f16 s0, s22
 ; CHECK-NEXT:    vins.f16 s5, s21
+; CHECK-NEXT:    vins.f16 s6, s22
 ; CHECK-NEXT:    vins.f16 s4, s20
-; CHECK-NEXT:    vins.f16 s23, s1
+; CHECK-NEXT:    vins.f16 s8, s24
+; CHECK-NEXT:    vins.f16 s11, s27
+; CHECK-NEXT:    vins.f16 s23, s0
 ; CHECK-NEXT:    vmovx.f16 s22, s10
 ; CHECK-NEXT:    vins.f16 s10, s26
-; CHECK-NEXT:    vmovx.f16 s1, s26
+; CHECK-NEXT:    vmovx.f16 s0, s26
 ; CHECK-NEXT:    vins.f16 s9, s25
-; CHECK-NEXT:    vins.f16 s8, s24
-; CHECK-NEXT:    vins.f16 s11, s27
-; CHECK-NEXT:    vmov q6, q1
-; CHECK-NEXT:    vins.f16 s22, s1
-; CHECK-NEXT:    vmov.f32 s1, s25
-; CHECK-NEXT:    vmov q6, q2
-; CHECK-NEXT:    vmov.f32 s3, s0
+; CHECK-NEXT:    vins.f16 s22, s0
 ; CHECK-NEXT:    vmov.f32 s0, s9
-; CHECK-NEXT:    vmov.f32 s26, s12
+; CHECK-NEXT:    vmov.f32 s1, s5
+; CHECK-NEXT:    vmov.f32 s13, s4
 ; CHECK-NEXT:    vstrb.8 q0, [r1, #16]
-; CHECK-NEXT:    vmov.f32 s25, s4
-; CHECK-NEXT:    vmov.f32 s27, s19
-; CHECK-NEXT:    vmov.f32 s13, s7
-; CHECK-NEXT:    vstrb.8 q6, [r1]
-; CHECK-NEXT:    vmov.f32 s12, s11
+; CHECK-NEXT:    vmov.f32 s12, s8
+; CHECK-NEXT:    vmov.f32 s17, s7
+; CHECK-NEXT:    vstrb.8 q3, [r1]
+; CHECK-NEXT:    vmov.f32 s16, s11
 ; CHECK-NEXT:    vmov.f32 s21, s6
-; CHECK-NEXT:    vstrb.8 q3, [r1, #48]
+; CHECK-NEXT:    vstrb.8 q4, [r1, #48]
 ; CHECK-NEXT:    vmov.f32 s20, s10
 ; CHECK-NEXT:    vstrb.8 q5, [r1, #32]
-; CHECK-NEXT:    vpop {d9, d10, d11, d12, d13}
+; CHECK-NEXT:    vpop {d8, d9, d10, d11, d12, d13}
 ; CHECK-NEXT:    bx lr
 entry:
   %l1 = load <8 x half>, ptr %src, align 4
diff --git a/llvm/test/CodeGen/Thumb2/pacbti-m-varargs-2.ll b/llvm/test/CodeGen/Thumb2/pacbti-m-varargs-2.ll
index 03b769f256bc2..7bb0c74fb0163 100644
--- a/llvm/test/CodeGen/Thumb2/pacbti-m-varargs-2.ll
+++ b/llvm/test/CodeGen/Thumb2/pacbti-m-varargs-2.ll
@@ -42,9 +42,9 @@ define hidden i32 @_Z1fiz(i32 %n, ...) local_unnamed_addr #0 {
 ; CHECK-NEXT:    mov r4, r0
 ; CHECK-NEXT:    add r0, sp, #28
 ; CHECK-NEXT:    movs r5, #0
-; CHECK-NEXT:    cmp r4, #1
 ; CHECK-NEXT:    stm r0!, {r1, r2, r3}
 ; CHECK-NEXT:    add r0, sp, #28
+; CHECK-NEXT:    cmp r4, #1
 ; CHECK-NEXT:    str r0, [sp, #4]
 ; CHECK-NEXT:    blt .LBB0_2
 ; CHECK-NEXT:  .LBB0_1: @ %for.body
diff --git a/llvm/test/CodeGen/Thumb2/pacbti-m-vla.ll b/llvm/test/CodeGen/Thumb2/pacbti-m-vla.ll
index 5eb5990be7c11..e6fcf56af6e8d 100644
--- a/llvm/test/CodeGen/Thumb2/pacbti-m-vla.ll
+++ b/llvm/test/CodeGen/Thumb2/pacbti-m-vla.ll
@@ -46,8 +46,8 @@ define hidden i32 @f(i32 %n) local_unnamed_addr #0 {
 ; CHECK-NEXT:    cmp r5, #1
 ; CHECK-NEXT:    blt .LBB0_3
 ; CHECK-NEXT:  @ %bb.1: @ %for.body.preheader
-; CHECK-NEXT:    subs r0, r5, #1
 ; CHECK-NEXT:    and r12, r5, #3
+; CHECK-NEXT:    subs r0, r5, #1
 ; CHECK-NEXT:    cmp r0, #3
 ; CHECK-NEXT:    bhs .LBB0_4
 ; CHECK-NEXT:  @ %bb.2:
diff --git a/llvm/test/CodeGen/Thumb2/setjmp_longjmp.ll b/llvm/test/CodeGen/Thumb2/setjmp_longjmp.ll
index 4e700ce493b35..e260286fb692a 100644
--- a/llvm/test/CodeGen/Thumb2/setjmp_longjmp.ll
+++ b/llvm/test/CodeGen/Thumb2/setjmp_longjmp.ll
@@ -58,8 +58,8 @@ define void @double_foobar() {
 ; CHECK-NEXT:    ldr r1, [sp] @ 4-byte Reload
 ; CHECK-NEXT:    movs r0, #2
 ; CHECK-NEXT:    str r0, [r1]
-; CHECK-NEXT:    add r1, sp, #4
 ; CHECK-NEXT:    movs r0, #0
+; CHECK-NEXT:    add r1, sp, #4
 ; CHECK-NEXT:    ldr r0, [r1, #8]
 ; CHECK-NEXT:    mov sp, r0
 ; CHECK-NEXT:    ldr r0, [r1, #4]
diff --git a/llvm/test/CodeGen/VE/Scalar/br_jt.ll b/llvm/test/CodeGen/VE/Scalar/br_jt.ll
index 5674376a615dd..fd880a7f42912 100644
--- a/llvm/test/CodeGen/VE/Scalar/br_jt.ll
+++ b/llvm/test/CodeGen/VE/Scalar/br_jt.ll
@@ -376,7 +376,7 @@ define signext i32 @br_jt4_m(i32 signext %0, i32 signext %1) {
 ; CHECK-NEXT:    lea %s2, .LJTI5_0@lo
 ; CHECK-NEXT:    and %s2, %s2, (32)0
 ; CHECK-NEXT:    lea.sl %s2, .LJTI5_0@hi(, %s2)
-; CHECK-NEXT:    ld %s2, (%s2, %s0)
+; CHECK-NEXT:    ld %s2, (%s0, %s2)
 ; CHECK-NEXT:    or %s0, 3, (0)1
 ; CHECK-NEXT:    b.l.t (, %s2)
 ; CHECK-NEXT:  .LBB5_2:
@@ -463,7 +463,7 @@ define signext i32 @br_jt7_m(i32 signext %0, i32 signext %1) {
 ; CHECK-NEXT:    lea %s3, .LJTI6_0@lo
 ; CHECK-NEXT:    and %s3, %s3, (32)0
 ; CHECK-NEXT:    lea.sl %s3, .LJTI6_0@hi(, %s3)
-; CHECK-NEXT:    ld %s3, (%s3, %s0)
+; CHECK-NEXT:    ld %s3, (%s0, %s3)
 ; CHECK-NEXT:    and %s1, %s1, (32)0
 ; CHECK-NEXT:    or %s0, 3, (0)1
 ; CHECK-NEXT:    b.l.t (, %s3)
@@ -599,7 +599,7 @@ define signext i32 @br_jt8_m(i32 signext %0, i32 signext %1) {
 ; CHECK-NEXT:    lea %s3, .LJTI7_0@lo
 ; CHECK-NEXT:    and %s3, %s3, (32)0
 ; CHECK-NEXT:    lea.sl %s3, .LJTI7_0@hi(, %s3)
-; CHECK-NEXT:    ld %s3, (%s3, %s0)
+; CHECK-NEXT:    ld %s3, (%s0, %s3)
 ; CHECK-NEXT:    and %s1, %s1, (32)0
 ; CHECK-NEXT:    or %s0, 3, (0)1
 ; CHECK-NEXT:    b.l.t (, %s3)
@@ -656,11 +656,11 @@ define signext i32 @br_jt8_m(i32 signext %0, i32 signext %1) {
 ; PIC-NEXT:    lea %s3, .LJTI7_0@gotoff_lo
 ; PIC-NEXT:    and %s3, %s3, (32)0
 ; PIC-NEXT:    lea.sl %s3, .LJTI7_0@gotoff_hi(%s3, %s15)
-; PIC-NEXT:    ldl.sx %s0, (%s3, %s0)
+; PIC-NEXT:    ldl.sx %s0, (%s0, %s3)
 ; PIC-NEXT:    lea %s3, br_jt8_m@gotoff_lo
 ; PIC-NEXT:    and %s3, %s3, (32)0
 ; PIC-NEXT:    lea.sl %s3, br_jt8_m@gotoff_hi(%s3, %s15)
-; PIC-NEXT:    adds.l %s3, %s0, %s3
+; PIC-NEXT:    adds.l %s3, %s3, %s0
 ; PIC-NEXT:    or %s0, 3, (0)1
 ; PIC-NEXT:    b.l.t (, %s3)
 ; PIC-NEXT:  .LBB7_2:
diff --git a/llvm/test/CodeGen/VE/Scalar/select.ll b/llvm/test/CodeGen/VE/Scalar/select.ll
index 184513a3f820b..0a98b91e78020 100644
--- a/llvm/test/CodeGen/VE/Scalar/select.ll
+++ b/llvm/test/CodeGen/VE/Scalar/select.ll
@@ -352,9 +352,9 @@ define fp128 @select_quad_mimm(i1 zeroext %0, fp128 %1) {
 ; CHECK-NEXT:    and %s1, %s1, (32)0
 ; CHECK-NEXT:    lea.sl %s1, .LCPI{{[0-9]+}}_0@hi(, %s1)
 ; CHECK-NEXT:    ld %s4, 8(, %s1)
-; CHECK-NEXT:    ld %s5, (, %s1)
+; CHECK-NEXT:    ld %s1, (, %s1)
 ; CHECK-NEXT:    cmov.w.ne %s2, %s4, %s0
-; CHECK-NEXT:    cmov.w.ne %s3, %s5, %s0
+; CHECK-NEXT:    cmov.w.ne %s3, %s1, %s0
 ; CHECK-NEXT:    or %s0, 0, %s2
 ; CHECK-NEXT:    or %s1, 0, %s3
 ; CHECK-NEXT:    b.l.t (, %s10)
diff --git a/llvm/test/CodeGen/VE/Scalar/va_caller.ll b/llvm/test/CodeGen/VE/Scalar/va_caller.ll
index 3cffc5e6c9fcd..421a419547d92 100644
--- a/llvm/test/CodeGen/VE/Scalar/va_caller.ll
+++ b/llvm/test/CodeGen/VE/Scalar/va_caller.ll
@@ -24,15 +24,15 @@ define i32 @caller() {
 ; CHECK-NEXT:    st %s1, 192(, %s11)
 ; CHECK-NEXT:    or %s1, 1, (0)1
 ; CHECK-NEXT:    st %s1, 184(, %s11)
-; CHECK-NEXT:    lea %s1, .LCPI{{[0-9]+}}_0@lo
-; CHECK-NEXT:    and %s1, %s1, (32)0
-; CHECK-NEXT:    lea.sl %s1, .LCPI{{[0-9]+}}_0@hi(, %s1)
-; CHECK-NEXT:    ld %s34, 8(, %s1)
-; CHECK-NEXT:    ld %s35, (, %s1)
 ; CHECK-NEXT:    st %s0, 176(, %s11)
-; CHECK-NEXT:    lea.sl %s0, 1086324736
-; CHECK-NEXT:    st %s0, 224(, %s11)
-; CHECK-NEXT:    st %s34, 280(, %s11)
+; CHECK-NEXT:    lea %s0, .LCPI{{[0-9]+}}_0@lo
+; CHECK-NEXT:    and %s0, %s0, (32)0
+; CHECK-NEXT:    lea.sl %s0,  .LCPI{{[0-9]+}}_0@hi(, %s0)
+; CHECK-NEXT:    ld %s2, 8(, %s0)
+; CHECK-NEXT:    lea.sl %s1, 1086324736
+; CHECK-NEXT:    st %s1, 224(, %s11)
+; CHECK-NEXT:    ld %s34, (, %s0)
+; CHECK-NEXT:    st %s2, 280(, %s11)
 ; CHECK-NEXT:    lea %s0, func@lo
 ; CHECK-NEXT:    and %s0, %s0, (32)0
 ; CHECK-NEXT:    lea.sl %s12, func@hi(, %s0)
@@ -44,10 +44,13 @@ define i32 @caller() {
 ; CHECK-NEXT:    or %s5, 5, (0)1
 ; CHECK-NEXT:    lea.sl %s6, 1086324736
 ; CHECK-NEXT:    or %s7, 0, (0)1
-; CHECK-NEXT:    st %s35, 272(, %s11)
+; CHECK-NEXT:    st %s34, 272(, %s11)
 ; CHECK-NEXT:    bsic %s10, (, %s12)
 ; CHECK-NEXT:    or %s0, 0, (0)1
 ; CHECK-NEXT:    or %s11, 0, %s9
+; CHECK-NEXT:    ld %s10, 8(, %s11)
+; CHECK-NEXT:    ld %s9, (, %s11)
+; CHECK-NEXT:    b.l.t (, %s10)
   call i32 (i32, ...) @func(i32 0, i16 1, i8 2, i32 3, i16 4, i8 5, float 6.0, ptr null, i64 8, double 9.0, i128 10, fp128 0xLA000000000000000)
   ret i32 0
 }
diff --git a/llvm/test/CodeGen/WebAssembly/exception.ll b/llvm/test/CodeGen/WebAssembly/exception.ll
index 304664b622e80..febab822a6a9e 100644
--- a/llvm/test/CodeGen/WebAssembly/exception.ll
+++ b/llvm/test/CodeGen/WebAssembly/exception.ll
@@ -2,6 +2,7 @@
 ; RUN: llc < %s -asm-verbose=false -wasm-enable-eh -wasm-use-legacy-eh=false -exception-model=wasm -mattr=+exception-handling -verify-machineinstrs -O0
 ; RUN: llc < %s -wasm-enable-eh -wasm-use-legacy-eh=false -exception-model=wasm -mattr=+exception-handling
 ; RUN: llc < %s -wasm-enable-eh -wasm-use-legacy-eh=false -exception-model=wasm -mattr=+exception-handling -filetype=obj
+; RUN: llc < %s -mtriple=wasm64-unknown-unknown -wasm-enable-eh -wasm-use-legacy-eh=false -exception-model=wasm -mattr=+exception-handling -verify-machineinstrs | FileCheck --implicit-check-not=ehgcr -allow-deprecated-dag-overlap %s --check-prefix=WASM64
 
 target triple = "wasm32-unknown-unknown"
 
@@ -30,11 +31,13 @@ define void @throw(ptr %p) {
 ; }
 
 ; CHECK-LABEL: catch:
+; WASM64-LABEL: catch:
 ; CHECK: global.get  __stack_pointer
 ; CHECK: local.set  0
 ; CHECK: block
 ; CHECK:   block     () -> (i32, exnref)
 ; CHECK:     try_table    (catch_ref __cpp_exception 0)
+; WASM64:  block     () -> (i64, exnref)
 ; CHECK:       call  foo
 ; CHECK:       br        2
 ; CHECK:     end_try_table
@@ -138,8 +141,10 @@ ehcleanup:                                        ; preds = %entry
 ; }
 
 ; CHECK-LABEL: terminatepad
+; WASM64-LABEL: terminatepad
 ; CHECK: block
 ; CHECK:   block     i32
+; WASM64:  block     i64
 ; CHECK:     try_table    (catch __cpp_exception 0)
 ; CHECK:       call  foo
 ; CHECK:       br        2
diff --git a/llvm/test/CodeGen/WebAssembly/int-mac-reduction-loops.ll b/llvm/test/CodeGen/WebAssembly/int-mac-reduction-loops.ll
new file mode 100644
index 0000000000000..c9af8e2268f1c
--- /dev/null
+++ b/llvm/test/CodeGen/WebAssembly/int-mac-reduction-loops.ll
@@ -0,0 +1,407 @@
+; RUN: opt -mattr=+simd128 -passes=loop-vectorize %s | llc -mtriple=wasm32 -mattr=+simd128 -verify-machineinstrs -o - | FileCheck %s
+; RUN: opt -mattr=+simd128 -passes=loop-vectorize -vectorizer-maximize-bandwidth %s | llc -mtriple=wasm32 -mattr=+simd128 -verify-machineinstrs -o - | FileCheck %s --check-prefix=MAX-BANDWIDTH
+
+target triple = "wasm32"
+
+define hidden i32 @i32_mac_s8(ptr nocapture noundef readonly %a, ptr nocapture noundef readonly %b, i32 noundef %N) {
+; CHECK-LABEL: i32_mac_s8:
+; CHECK:    v128.load32_zero 0:p2align=0
+; CHECK:    i16x8.extend_low_i8x16_s
+; CHECK:    v128.load32_zero 0:p2align=0
+; CHECK:    i16x8.extend_low_i8x16_s
+; CHECK:    i32x4.extmul_low_i16x8_s
+; CHECK:    i32x4.add
+
+; MAX-BANDWIDTH: v128.load
+; MAX-BANDWIDTH: i16x8.extend_low_i8x16_s
+; MAX-BANDWIDTH: v128.load
+; MAX-BANDWIDTH: i16x8.extend_low_i8x16_s
+; MAX-BANDWIDTH: i32x4.dot_i16x8_s
+; MAX-BANDWIDTH: i16x8.extend_high_i8x16_s
+; MAX-BANDWIDTH: i16x8.extend_high_i8x16_s
+; MAX-BANDWIDTH: i32x4.dot_i16x8_s
+; MAX-BANDWIDTH: i32x4.add
+; MAX-BANDWIDTH: i32x4.add
+
+entry:
+  %cmp7.not = icmp eq i32 %N, 0
+  br i1 %cmp7.not, label %for.cond.cleanup, label %for.body
+
+for.cond.cleanup:                                 ; preds = %for.body, %entry
+  %res.0.lcssa = phi i32 [ 0, %entry ], [ %add, %for.body ]
+  ret i32 %res.0.lcssa
+
+for.body:                                         ; preds = %entry, %for.body
+  %i.09 = phi i32 [ %inc, %for.body ], [ 0, %entry ]
+  %res.08 = phi i32 [ %add, %for.body ], [ 0, %entry ]
+  %arrayidx = getelementptr inbounds i8, ptr %a, i32 %i.09
+  %0 = load i8, ptr %arrayidx, align 1
+  %conv = sext i8 %0 to i32
+  %arrayidx1 = getelementptr inbounds i8, ptr %b, i32 %i.09
+  %1 = load i8, ptr %arrayidx1, align 1
+  %conv2 = sext i8 %1 to i32
+  %mul = mul nsw i32 %conv2, %conv
+  %add = add nsw i32 %mul, %res.08
+  %inc = add nuw i32 %i.09, 1
+  %exitcond.not = icmp eq i32 %inc, %N
+  br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
+
+define hidden i32 @i32_mac_s16(ptr nocapture noundef readonly %a, ptr nocapture noundef readonly %b, i32 noundef %N) {
+; CHECK-LABEL: i32_mac_s16:
+; CHECK:    i32x4.load16x4_s 0:p2align=1
+; CHECK:    i32x4.load16x4_s 0:p2align=1
+; CHECK:    i32x4.mul
+; CHECK:    i32x4.add
+
+; MAX-BANDWIDTH: v128.load
+; MAX-BANDWIDTH: v128.load
+; MAX-BANDWIDTH: i32x4.dot_i16x8_s
+
+entry:
+  %cmp7.not = icmp eq i32 %N, 0
+  br i1 %cmp7.not, label %for.cond.cleanup, label %for.body
+
+for.cond.cleanup:                                 ; preds = %for.body, %entry
+  %res.0.lcssa = phi i32 [ 0, %entry ], [ %add, %for.body ]
+  ret i32 %res.0.lcssa
+
+for.body:                                         ; preds = %entry, %for.body
+  %i.09 = phi i32 [ %inc, %for.body ], [ 0, %entry ]
+  %res.08 = phi i32 [ %add, %for.body ], [ 0, %entry ]
+  %arrayidx = getelementptr inbounds i16, ptr %a, i32 %i.09
+  %0 = load i16, ptr %arrayidx, align 2
+  %conv = sext i16 %0 to i32
+  %arrayidx1 = getelementptr inbounds i16, ptr %b, i32 %i.09
+  %1 = load i16, ptr %arrayidx1, align 2
+  %conv2 = sext i16 %1 to i32
+  %mul = mul nsw i32 %conv2, %conv
+  %add = add nsw i32 %mul, %res.08
+  %inc = add nuw i32 %i.09, 1
+  %exitcond.not = icmp eq i32 %inc, %N
+  br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
+
+define hidden i64 @i64_mac_s16(ptr nocapture noundef readonly %a, ptr nocapture noundef readonly %b, i32 noundef %N) {
+; CHECK-LABEL: i64_mac_s16:
+; CHECK:    v128.load32_zero 0:p2align=1
+; CHECK:    i32x4.extend_low_i16x8_s
+; CHECK:    v128.load32_zero 0:p2align=1
+; CHECK:    i32x4.extend_low_i16x8_s
+; CHECK:    i64x2.extmul_low_i32x4_s
+; CHECK:    i64x2.add
+
+; MAX-BANDWIDTH: v128.load
+; MAX-BANDWIDTH: i8x16.shuffle	12, 13, 14, 15, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1
+; MAX-BANDWIDTH: i32x4.extend_low_i16x8_s
+; MAX-BANDWIDTH: v128.load
+; MAX-BANDWIDTH: i8x16.shuffle	12, 13, 14, 15, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1
+; MAX-BANDWIDTH: i32x4.extend_low_i16x8_s
+; MAX-BANDWIDTH: i64x2.extmul_low_i32x4_s
+; MAX-BANDWIDTH: i64x2.add
+; MAX-BANDWIDTH: i8x16.shuffle	8, 9, 10, 11, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1
+; MAX-BANDWIDTH: i32x4.extend_low_i16x8_s
+; MAX-BANDWIDTH: i8x16.shuffle	8, 9, 10, 11, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1
+; MAX-BANDWIDTH: i32x4.extend_low_i16x8_s
+; MAX-BANDWIDTH: i64x2.extmul_low_i32x4_s
+; MAX-BANDWIDTH: i64x2.add
+; MAX-BANDWIDTH: i8x16.shuffle	4, 5, 6, 7, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1
+; MAX-BANDWIDTH: i32x4.extend_low_i16x8_s
+; MAX-BANDWIDTH: i8x16.shuffle	4, 5, 6, 7, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1
+; MAX-BANDWIDTH: i32x4.extend_low_i16x8_s
+; MAX-BANDWIDTH: i64x2.extmul_low_i32x4_s
+; MAX-BANDWIDTH: i64x2.add
+; MAX-BANDWIDTH: i32x4.extend_low_i16x8_s
+; MAX-BANDWIDTH: i32x4.extend_low_i16x8_s
+; MAX-BANDWIDTH: i64x2.extmul_low_i32x4_s
+; MAX-BANDWIDTH: i64x2.add
+
+entry:
+  %cmp7.not = icmp eq i32 %N, 0
+  br i1 %cmp7.not, label %for.cond.cleanup, label %for.body
+
+for.cond.cleanup:                                 ; preds = %for.body, %entry
+  %res.0.lcssa = phi i64 [ 0, %entry ], [ %add, %for.body ]
+  ret i64 %res.0.lcssa
+
+for.body:                                         ; preds = %entry, %for.body
+  %i.09 = phi i32 [ %inc, %for.body ], [ 0, %entry ]
+  %res.08 = phi i64 [ %add, %for.body ], [ 0, %entry ]
+  %arrayidx = getelementptr inbounds i16, ptr %a, i32 %i.09
+  %0 = load i16, ptr %arrayidx, align 2
+  %conv = sext i16 %0 to i64
+  %arrayidx1 = getelementptr inbounds i16, ptr %b, i32 %i.09
+  %1 = load i16, ptr %arrayidx1, align 2
+  %conv2 = sext i16 %1 to i64
+  %mul = mul nsw i64 %conv2, %conv
+  %add = add nsw i64 %mul, %res.08
+  %inc = add nuw i32 %i.09, 1
+  %exitcond.not = icmp eq i32 %inc, %N
+  br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
+
+define hidden i64 @i64_mac_s32(ptr nocapture noundef readonly %a, ptr nocapture noundef readonly %b, i32 noundef %N) {
+; CHECK-LABEL: i64_mac_s32:
+; CHECK:    v128.load64_zero 0:p2align=2
+; CHECK:    v128.load64_zero 0:p2align=2
+; CHECK:    i32x4.mul
+; CHECK:    i64x2.extend_low_i32x4_s
+; CHECK:    i64x2.add
+
+; MAX-BANDWIDTH: v128.load
+; MAX-BANDWIDTH: v128.load
+; MAX-BANDWIDTH: i32x4.mul
+; MAX-BANDWIDTH: i64x2.extend_low_i32x4_s
+; MAX-BANDWIDTH: i64x2.add
+; MAX-BANDWIDTH: i8x16.shuffle	8, 9, 10, 11, 12, 13, 14, 15, 0, 1, 2, 3, 0, 1, 2, 3
+; MAX-BANDWIDTH: i64x2.extend_low_i32x4_s
+; MAX-BANDWIDTH: i64x2.add
+
+entry:
+  %cmp6.not = icmp eq i32 %N, 0
+  br i1 %cmp6.not, label %for.cond.cleanup, label %for.body
+
+for.cond.cleanup:                                 ; preds = %for.body, %entry
+  %res.0.lcssa = phi i64 [ 0, %entry ], [ %add, %for.body ]
+  ret i64 %res.0.lcssa
+
+for.body:                                         ; preds = %entry, %for.body
+  %i.08 = phi i32 [ %inc, %for.body ], [ 0, %entry ]
+  %res.07 = phi i64 [ %add, %for.body ], [ 0, %entry ]
+  %arrayidx = getelementptr inbounds i32, ptr %a, i32 %i.08
+  %0 = load i32, ptr %arrayidx, align 4
+  %arrayidx1 = getelementptr inbounds i32, ptr %b, i32 %i.08
+  %1 = load i32, ptr %arrayidx1, align 4
+  %mul = mul i32 %1, %0
+  %conv = sext i32 %mul to i64
+  %add = add i64 %res.07, %conv
+  %inc = add nuw i32 %i.08, 1
+  %exitcond.not = icmp eq i32 %inc, %N
+  br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
+
+define hidden i32 @i32_mac_u8(ptr nocapture noundef readonly %a, ptr nocapture noundef readonly %b, i32 noundef %N) {
+; CHECK-LABEL: i32_mac_u8:
+; CHECK:    v128.load32_zero 0:p2align=0
+; CHECK:    i16x8.extend_low_i8x16_u
+; CHECK:    v128.load32_zero 0:p2align=0
+; CHECK:    i16x8.extend_low_i8x16_u
+; CHECK:    i32x4.extmul_low_i16x8_u
+; CHECK:    i32x4.add
+
+; MAX-BANDWIDTH: v128.load
+; MAX-BANDWIDTH: v128.load
+; MAX-BANDWIDTH: i16x8.extmul_low_i8x16_u
+; MAX-BANDWIDTH: i32x4.extend_low_i16x8_u
+; MAX-BANDWIDTH: i32x4.extend_high_i16x8_u
+; MAX-BANDWIDTH: i32x4.add
+; MAX-BANDWIDTH: i16x8.extmul_high_i8x16_u
+; MAX-BANDWIDTH: i32x4.extend_low_i16x8_u
+; MAX-BANDWIDTH: i32x4.extend_high_i16x8_u
+; MAX-BANDWIDTH: i32x4.add
+; MAX-BANDWIDTH: i32x4.add
+; MAX-BANDWIDTH: i32x4.add
+
+entry:
+  %cmp7.not = icmp eq i32 %N, 0
+  br i1 %cmp7.not, label %for.cond.cleanup, label %for.body
+
+for.cond.cleanup:                                 ; preds = %for.body, %entry
+  %res.0.lcssa = phi i32 [ 0, %entry ], [ %add, %for.body ]
+  ret i32 %res.0.lcssa
+
+for.body:                                         ; preds = %entry, %for.body
+  %i.09 = phi i32 [ %inc, %for.body ], [ 0, %entry ]
+  %res.08 = phi i32 [ %add, %for.body ], [ 0, %entry ]
+  %arrayidx = getelementptr inbounds i8, ptr %a, i32 %i.09
+  %0 = load i8, ptr %arrayidx, align 1
+  %conv = zext i8 %0 to i32
+  %arrayidx1 = getelementptr inbounds i8, ptr %b, i32 %i.09
+  %1 = load i8, ptr %arrayidx1, align 1
+  %conv2 = zext i8 %1 to i32
+  %mul = mul nuw nsw i32 %conv2, %conv
+  %add = add i32 %mul, %res.08
+  %inc = add nuw i32 %i.09, 1
+  %exitcond.not = icmp eq i32 %inc, %N
+  br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
+
+define hidden i32 @i32_mac_u16(ptr nocapture noundef readonly %a, ptr nocapture noundef readonly %b, i32 noundef %N) {
+; CHECK-LABEL: i32_mac_u16:
+; CHECK:    i32x4.load16x4_u 0:p2align=1
+; CHECK:    i32x4.load16x4_u 0:p2align=1
+; CHECK:    i32x4.mul
+; CHECK:    i32x4.add
+
+; MAX-BANDWIDTH: v128.load
+; MAX-BANDWIDTH: v128.load
+; MAX-BANDWIDTH: i32x4.extmul_low_i16x8_u
+; MAX-BANDWIDTH: i32x4.extmul_high_i16x8_u
+; MAX-BANDWIDTH: i32x4.add
+; MAX-BANDWIDTH: i32x4.add
+
+entry:
+  %cmp7.not = icmp eq i32 %N, 0
+  br i1 %cmp7.not, label %for.cond.cleanup, label %for.body
+
+for.cond.cleanup:                                 ; preds = %for.body, %entry
+  %res.0.lcssa = phi i32 [ 0, %entry ], [ %add, %for.body ]
+  ret i32 %res.0.lcssa
+
+for.body:                                         ; preds = %entry, %for.body
+  %i.09 = phi i32 [ %inc, %for.body ], [ 0, %entry ]
+  %res.08 = phi i32 [ %add, %for.body ], [ 0, %entry ]
+  %arrayidx = getelementptr inbounds i16, ptr %a, i32 %i.09
+  %0 = load i16, ptr %arrayidx, align 2
+  %conv = zext i16 %0 to i32
+  %arrayidx1 = getelementptr inbounds i16, ptr %b, i32 %i.09
+  %1 = load i16, ptr %arrayidx1, align 2
+  %conv2 = zext i16 %1 to i32
+  %mul = mul nuw nsw i32 %conv2, %conv
+  %add = add i32 %mul, %res.08
+  %inc = add nuw i32 %i.09, 1
+  %exitcond.not = icmp eq i32 %inc, %N
+  br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
+
+define hidden i32 @i32_mac_u16_s16(ptr nocapture noundef readonly %a, ptr nocapture noundef readonly %b, i32 noundef %N) {
+; CHECK-LABEL: i32_mac_u16_s16:
+; CHECK:    i32x4.load16x4_s 0:p2align=1
+; CHECK:    i32x4.load16x4_u 0:p2align=1
+; CHECK:    i32x4.mul
+; CHECK:    i32x4.add
+
+; MAX-BANDWIDTH: v128.load
+; MAX-BANDWIDTH: i8x16.shuffle	8, 9, 10, 11, 12, 13, 14, 15, 0, 1, 0, 1, 0, 1, 0, 1
+; MAX-BANDWIDTH: i32x4.extend_low_i16x8_s
+; MAX-BANDWIDTH: v128.load
+; MAX-BANDWIDTH: i8x16.shuffle	8, 9, 10, 11, 12, 13, 14, 15, 0, 1, 0, 1, 0, 1, 0, 1
+; MAX-BANDWIDTH: i32x4.extend_low_i16x8_u
+; MAX-BANDWIDTH: i32x4.mul
+; MAX-BANDWIDTH: i32x4.extend_low_i16x8_s
+; MAX-BANDWIDTH: i32x4.extend_low_i16x8_u
+; MAX-BANDWIDTH: i32x4.mul
+; MAX-BANDWIDTH: i32x4.add
+; MAX-BANDWIDTH: i32x4.add
+
+entry:
+  %cmp7.not = icmp eq i32 %N, 0
+  br i1 %cmp7.not, label %for.cond.cleanup, label %for.body
+
+for.cond.cleanup:                                 ; preds = %for.body, %entry
+  %res.0.lcssa = phi i32 [ 0, %entry ], [ %add, %for.body ]
+  ret i32 %res.0.lcssa
+
+for.body:                                         ; preds = %entry, %for.body
+  %i.09 = phi i32 [ %inc, %for.body ], [ 0, %entry ]
+  %res.08 = phi i32 [ %add, %for.body ], [ 0, %entry ]
+  %arrayidx = getelementptr inbounds i16, ptr %a, i32 %i.09
+  %0 = load i16, ptr %arrayidx, align 2
+  %conv = zext i16 %0 to i32
+  %arrayidx1 = getelementptr inbounds i16, ptr %b, i32 %i.09
+  %1 = load i16, ptr %arrayidx1, align 2
+  %conv2 = sext i16 %1 to i32
+  %mul = mul nuw nsw i32 %conv2, %conv
+  %add = add i32 %mul, %res.08
+  %inc = add nuw i32 %i.09, 1
+  %exitcond.not = icmp eq i32 %inc, %N
+  br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
+
+define hidden i64 @i64_mac_u16(ptr nocapture noundef readonly %a, ptr nocapture noundef readonly %b, i32 noundef %N) {
+; CHECK-LABEL: i64_mac_u16:
+; CHECK:    v128.load32_zero 0:p2align=1
+; CHECK:    i32x4.extend_low_i16x8_u
+; CHECK:    v128.load32_zero 0:p2align=1
+; CHECK:    i32x4.extend_low_i16x8_u
+; CHECK:    i64x2.extmul_low_i32x4_u
+; CHECK:    i64x2.add
+
+; MAX-BANDWIDTH: v128.load
+; MAX-BANDWIDTH: i8x16.shuffle	12, 13, 14, 15, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1
+; MAX-BANDWIDTH: i32x4.extend_low_i16x8_u
+; MAX-BANDWIDTH: v128.load
+; MAX-BANDWIDTH: i8x16.shuffle	12, 13, 14, 15, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1
+; MAX-BANDWIDTH: i32x4.extend_low_i16x8_u
+; MAX-BANDWIDTH: i64x2.extmul_low_i32x4_u
+; MAX-BANDWIDTH: i64x2.add
+; MAX-BANDWIDTH: i8x16.shuffle	8, 9, 10, 11, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1
+; MAX-BANDWIDTH: i32x4.extend_low_i16x8_u
+; MAX-BANDWIDTH: i8x16.shuffle	8, 9, 10, 11, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1
+; MAX-BANDWIDTH: i32x4.extend_low_i16x8_u
+; MAX-BANDWIDTH: i64x2.extmul_low_i32x4_u
+; MAX-BANDWIDTH: i64x2.add
+; MAX-BANDWIDTH: i8x16.shuffle	4, 5, 6, 7, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1
+; MAX-BANDWIDTH: i32x4.extend_low_i16x8_u
+; MAX-BANDWIDTH: i8x16.shuffle	4, 5, 6, 7, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1
+; MAX-BANDWIDTH: i32x4.extend_low_i16x8_u
+; MAX-BANDWIDTH: i64x2.extmul_low_i32x4_u
+; MAX-BANDWIDTH: i64x2.add
+; MAX-BANDWIDTH: i32x4.extend_low_i16x8_u
+; MAX-BANDWIDTH: i32x4.extend_low_i16x8_u
+; MAX-BANDWIDTH: i64x2.extmul_low_i32x4_u
+; MAX-BANDWIDTH: i64x2.add
+
+entry:
+  %cmp8.not = icmp eq i32 %N, 0
+  br i1 %cmp8.not, label %for.cond.cleanup, label %for.body
+
+for.cond.cleanup:                                 ; preds = %for.body, %entry
+  %res.0.lcssa = phi i64 [ 0, %entry ], [ %add, %for.body ]
+  ret i64 %res.0.lcssa
+
+for.body:                                         ; preds = %entry, %for.body
+  %i.010 = phi i32 [ %inc, %for.body ], [ 0, %entry ]
+  %res.09 = phi i64 [ %add, %for.body ], [ 0, %entry ]
+  %arrayidx = getelementptr inbounds i16, ptr %a, i32 %i.010
+  %0 = load i16, ptr %arrayidx, align 2
+  %conv = zext i16 %0 to i64
+  %arrayidx1 = getelementptr inbounds i16, ptr %b, i32 %i.010
+  %1 = load i16, ptr %arrayidx1, align 2
+  %conv2 = zext i16 %1 to i64
+  %mul = mul nuw nsw i64 %conv2, %conv
+  %add = add i64 %mul, %res.09
+  %inc = add nuw i32 %i.010, 1
+  %exitcond.not = icmp eq i32 %inc, %N
+  br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
+
+define hidden i64 @i64_mac_u32(ptr nocapture noundef readonly %a, ptr nocapture noundef readonly %b, i32 noundef %N) {
+; CHECK-LABEL: i64_mac_u32:
+; CHECK:    v128.load64_zero 0:p2align=2
+; CHECK:    v128.load64_zero 0:p2align=2
+; CHECK:    i32x4.mul
+; CHECK:    i64x2.extend_low_i32x4_u
+; CHECK:    i64x2.add
+
+; MAX-BANDWIDTH: v128.load
+; MAX-BANDWIDTH: v128.load
+; MAX-BANDWIDTH: i32x4.mul
+; MAX-BANDWIDTH: i64x2.extend_low_i32x4_u
+; MAX-BANDWIDTH: i64x2.add
+; MAX-BANDWIDTH: i8x16.shuffle	8, 9, 10, 11, 12, 13, 14, 15, 0, 1, 2, 3, 0, 1, 2, 3
+; MAX-BANDWIDTH: i64x2.extend_low_i32x4_u
+; MAX-BANDWIDTH: i64x2.add
+
+entry:
+  %cmp6.not = icmp eq i32 %N, 0
+  br i1 %cmp6.not, label %for.cond.cleanup, label %for.body
+
+for.cond.cleanup:                                 ; preds = %for.body, %entry
+  %res.0.lcssa = phi i64 [ 0, %entry ], [ %add, %for.body ]
+  ret i64 %res.0.lcssa
+
+for.body:                                         ; preds = %entry, %for.body
+  %i.08 = phi i32 [ %inc, %for.body ], [ 0, %entry ]
+  %res.07 = phi i64 [ %add, %for.body ], [ 0, %entry ]
+  %arrayidx = getelementptr inbounds i32, ptr %a, i32 %i.08
+  %0 = load i32, ptr %arrayidx, align 4
+  %arrayidx1 = getelementptr inbounds i32, ptr %b, i32 %i.08
+  %1 = load i32, ptr %arrayidx1, align 4
+  %mul = mul i32 %1, %0
+  %conv = zext i32 %mul to i64
+  %add = add i64 %res.07, %conv
+  %inc = add nuw i32 %i.08, 1
+  %exitcond.not = icmp eq i32 %inc, %N
+  br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
diff --git a/llvm/test/CodeGen/X86/AMX/amx-ldtilecfg-insert.ll b/llvm/test/CodeGen/X86/AMX/amx-ldtilecfg-insert.ll
index 8595024103dbd..06cf968512db8 100644
--- a/llvm/test/CodeGen/X86/AMX/amx-ldtilecfg-insert.ll
+++ b/llvm/test/CodeGen/X86/AMX/amx-ldtilecfg-insert.ll
@@ -230,7 +230,6 @@ exit:
 define dso_local void @test5(i16 signext %0, i16 signext %1) nounwind {
 ; CHECK-LABEL: test5:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    # kill: def $esi killed $esi def $rsi
 ; CHECK-NEXT:    vxorps %xmm0, %xmm0, %xmm0
 ; CHECK-NEXT:    vmovups %zmm0, -{{[0-9]+}}(%rsp)
 ; CHECK-NEXT:    movb $1, -{{[0-9]+}}(%rsp)
@@ -239,7 +238,8 @@ define dso_local void @test5(i16 signext %0, i16 signext %1) nounwind {
 ; CHECK-NEXT:    xorl %eax, %eax
 ; CHECK-NEXT:    movl $buf, %ecx
 ; CHECK-NEXT:    movl $32, %edx
-; CHECK-NEXT:    leal -1(%rsi), %r8d
+; CHECK-NEXT:    movl %esi, %r8d
+; CHECK-NEXT:    decl %r8d
 ; CHECK-NEXT:    jmp .LBB4_1
 ; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB4_3: # %if.false
diff --git a/llvm/test/CodeGen/X86/GlobalISel/isel-fcmp-i686.mir b/llvm/test/CodeGen/X86/GlobalISel/isel-fcmp-i686.mir
new file mode 100644
index 0000000000000..fb4e25f2a5ced
--- /dev/null
+++ b/llvm/test/CodeGen/X86/GlobalISel/isel-fcmp-i686.mir
@@ -0,0 +1,399 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 5
+# NOTE: This MIR test is required because the support for 64 bit memory ops is missing in i686 mode, Due to distinction between float/int types, support is expected in near future and there is this RFC in place https://discourse.llvm.org/t/rfc-globalisel-adding-fp-type-information-to-llt/83349. Once this support is introduced this test must be dropped and integrated into the LLVM IR tests.
+# RUN: llc -mtriple=i686-linux-gnu -mattr=+x87,-sse,-sse2 -run-pass=regbankselect,instruction-select -disable-gisel-legality-check -global-isel %s -o - | FileCheck %s --check-prefixes GISEL-X86
+
+---
+name:            fcmp_double_oeq
+alignment:       16
+exposesReturnsTwice: false
+legalized:       true
+tracksRegLiveness: true
+fixedStack:
+  - { id: 0, type: default, offset: 8, size: 8, alignment: 8, stack-id: default,
+      isImmutable: true, isAliased: false, callee-saved-register: '', callee-saved-restored: true,
+      debug-info-variable: '', debug-info-expression: '', debug-info-location: '' }
+
+stack:           []
+body:             |
+  bb.1.entry:
+    ; GISEL-X86-LABEL: name: fcmp_double_oeq
+    ; GISEL-X86: [[LD_Fp64m:%[0-9]+]]:rfp64 = LD_Fp64m %fixed-stack.0, 1, $noreg, 0, $noreg, implicit-def $fpsw, implicit $fpcw :: (invariant load (s64) from %fixed-stack.0)
+    ; GISEL-X86-NEXT: [[LD_Fp64m1:%[0-9]+]]:rfp64 = LD_Fp64m %fixed-stack.0, 1, $noreg, 0, $noreg, implicit-def $fpsw, implicit $fpcw :: (invariant load (s64) from %fixed-stack.0 + 8)
+    ; GISEL-X86-NEXT: UCOM_FpIr64 [[LD_Fp64m]], [[LD_Fp64m1]], implicit-def $eflags, implicit-def $fpsw, implicit $fpcw
+    ; GISEL-X86-NEXT: [[SETCCr:%[0-9]+]]:gr8 = SETCCr 4, implicit $eflags
+    ; GISEL-X86-NEXT: [[SETCCr1:%[0-9]+]]:gr8 = SETCCr 11, implicit $eflags
+    ; GISEL-X86-NEXT: [[AND8rr:%[0-9]+]]:gr8 = AND8rr [[SETCCr]], [[SETCCr1]], implicit-def $eflags
+    ; GISEL-X86-NEXT: $al = COPY [[AND8rr]]
+    ; GISEL-X86-NEXT: RET 0, implicit $al
+    %3:_(p0) = G_FRAME_INDEX %fixed-stack.0
+    %2:_(s64) = G_LOAD %3(p0) :: (invariant load (s64) from %fixed-stack.0, align 8)
+    %1:_(s64) = G_LOAD %3(p0) :: (invariant load (s64) from %fixed-stack.0 + 8, basealign 8)
+    %4:_(s8) = G_FCMP floatpred(oeq), %2(s64), %1
+    $al = COPY %4(s8)
+    RET 0, implicit $al
+
+...
+---
+name:            fcmp_double_ogt
+alignment:       16
+legalized:       true
+tracksRegLiveness: true
+fixedStack:
+  - { id: 0, type: default, offset: 8, size: 8, alignment: 8, stack-id: default,
+      isImmutable: true, isAliased: false, callee-saved-register: '', callee-saved-restored: true,
+      debug-info-variable: '', debug-info-expression: '', debug-info-location: '' }
+
+stack:           []
+body:             |
+  bb.1.entry:
+    ; GISEL-X86-LABEL: name: fcmp_double_ogt
+    ; GISEL-X86: [[LD_Fp64m:%[0-9]+]]:rfp64 = LD_Fp64m %fixed-stack.0, 1, $noreg, 0, $noreg, implicit-def $fpsw, implicit $fpcw :: (invariant load (s64) from %fixed-stack.0)
+    ; GISEL-X86-NEXT: [[LD_Fp64m1:%[0-9]+]]:rfp64 = LD_Fp64m %fixed-stack.0, 1, $noreg, 0, $noreg, implicit-def $fpsw, implicit $fpcw :: (invariant load (s64) from %fixed-stack.0 + 8)
+    ; GISEL-X86-NEXT: UCOM_FpIr64 [[LD_Fp64m]], [[LD_Fp64m1]], implicit-def $eflags, implicit-def $fpsw, implicit $fpcw
+    ; GISEL-X86-NEXT: [[SETCCr:%[0-9]+]]:gr8 = SETCCr 7, implicit $eflags
+    ; GISEL-X86-NEXT: $al = COPY [[SETCCr]]
+    ; GISEL-X86-NEXT: RET 0, implicit $al
+    %3:_(p0) = G_FRAME_INDEX %fixed-stack.0
+    %2:_(s64) = G_LOAD %3(p0) :: (invariant load (s64) from %fixed-stack.0, align 8)
+    %1:_(s64) = G_LOAD %3(p0) :: (invariant load (s64) from %fixed-stack.0 + 8, basealign 8)
+    %4:_(s8) = G_FCMP floatpred(ogt), %2(s64), %1
+    $al = COPY %4(s8)
+    RET 0, implicit $al
+
+...
+---
+name:            fcmp_double_oge
+alignment:       16
+exposesReturnsTwice: false
+legalized:       true
+tracksRegLiveness: true
+fixedStack:
+  - { id: 0, type: default, offset: 8, size: 8, alignment: 8, stack-id: default,
+      isImmutable: true, isAliased: false, callee-saved-register: '', callee-saved-restored: true,
+      debug-info-variable: '', debug-info-expression: '', debug-info-location: '' }
+stack:           []
+body:             |
+  bb.1.entry:
+    ; GISEL-X86-LABEL: name: fcmp_double_oge
+    ; GISEL-X86: [[LD_Fp64m:%[0-9]+]]:rfp64 = LD_Fp64m %fixed-stack.0, 1, $noreg, 0, $noreg, implicit-def $fpsw, implicit $fpcw :: (invariant load (s64) from %fixed-stack.0)
+    ; GISEL-X86-NEXT: [[LD_Fp64m1:%[0-9]+]]:rfp64 = LD_Fp64m %fixed-stack.0, 1, $noreg, 0, $noreg, implicit-def $fpsw, implicit $fpcw :: (invariant load (s64) from %fixed-stack.0 + 8)
+    ; GISEL-X86-NEXT: UCOM_FpIr64 [[LD_Fp64m]], [[LD_Fp64m1]], implicit-def $eflags, implicit-def $fpsw, implicit $fpcw
+    ; GISEL-X86-NEXT: [[SETCCr:%[0-9]+]]:gr8 = SETCCr 3, implicit $eflags
+    ; GISEL-X86-NEXT: $al = COPY [[SETCCr]]
+    ; GISEL-X86-NEXT: RET 0, implicit $al
+    %3:_(p0) = G_FRAME_INDEX %fixed-stack.0
+    %2:_(s64) = G_LOAD %3(p0) :: (invariant load (s64) from %fixed-stack.0, align 8)
+    %1:_(s64) = G_LOAD %3(p0) :: (invariant load (s64) from %fixed-stack.0 + 8, basealign 8)
+    %4:_(s8) = G_FCMP floatpred(oge), %2(s64), %1
+    $al = COPY %4(s8)
+    RET 0, implicit $al
+
+...
+---
+name:            fcmp_double_olt
+alignment:       16
+exposesReturnsTwice: false
+legalized:       true
+tracksRegLiveness: true
+fixedStack:
+  - { id: 0, type: default, offset: 8, size: 8, alignment: 8, stack-id: default,
+      isImmutable: true, isAliased: false, callee-saved-register: '', callee-saved-restored: true,
+      debug-info-variable: '', debug-info-expression: '', debug-info-location: '' }
+
+body:             |
+  bb.1.entry:
+    ; GISEL-X86-LABEL: name: fcmp_double_olt
+    ; GISEL-X86: [[LD_Fp64m:%[0-9]+]]:rfp64 = LD_Fp64m %fixed-stack.0, 1, $noreg, 0, $noreg, implicit-def $fpsw, implicit $fpcw :: (invariant load (s64) from %fixed-stack.0)
+    ; GISEL-X86-NEXT: [[LD_Fp64m1:%[0-9]+]]:rfp64 = LD_Fp64m %fixed-stack.0, 1, $noreg, 0, $noreg, implicit-def $fpsw, implicit $fpcw :: (invariant load (s64) from %fixed-stack.0 + 8)
+    ; GISEL-X86-NEXT: UCOM_FpIr64 [[LD_Fp64m1]], [[LD_Fp64m]], implicit-def $eflags, implicit-def $fpsw, implicit $fpcw
+    ; GISEL-X86-NEXT: [[SETCCr:%[0-9]+]]:gr8 = SETCCr 7, implicit $eflags
+    ; GISEL-X86-NEXT: $al = COPY [[SETCCr]]
+    ; GISEL-X86-NEXT: RET 0, implicit $al
+    %3:_(p0) = G_FRAME_INDEX %fixed-stack.0
+    %2:_(s64) = G_LOAD %3(p0) :: (invariant load (s64) from %fixed-stack.0, align 8)
+    %1:_(s64) = G_LOAD %3(p0) :: (invariant load (s64) from %fixed-stack.0 + 8, basealign 8)
+    %4:_(s8) = G_FCMP floatpred(olt), %2(s64), %1
+    $al = COPY %4(s8)
+    RET 0, implicit $al
+
+...
+---
+name:            fcmp_double_ole
+alignment:       16
+exposesReturnsTwice: false
+legalized:       true
+tracksRegLiveness: true
+fixedStack:
+  - { id: 0, type: default, offset: 8, size: 8, alignment: 8, stack-id: default,
+      isImmutable: true, isAliased: false, callee-saved-register: '', callee-saved-restored: true,
+      debug-info-variable: '', debug-info-expression: '', debug-info-location: '' }
+stack:           []
+body:             |
+  bb.1.entry:
+    ; GISEL-X86-LABEL: name: fcmp_double_ole
+    ; GISEL-X86: [[LD_Fp64m:%[0-9]+]]:rfp64 = LD_Fp64m %fixed-stack.0, 1, $noreg, 0, $noreg, implicit-def $fpsw, implicit $fpcw :: (invariant load (s64) from %fixed-stack.0)
+    ; GISEL-X86-NEXT: [[LD_Fp64m1:%[0-9]+]]:rfp64 = LD_Fp64m %fixed-stack.0, 1, $noreg, 0, $noreg, implicit-def $fpsw, implicit $fpcw :: (invariant load (s64) from %fixed-stack.0 + 8)
+    ; GISEL-X86-NEXT: UCOM_FpIr64 [[LD_Fp64m1]], [[LD_Fp64m]], implicit-def $eflags, implicit-def $fpsw, implicit $fpcw
+    ; GISEL-X86-NEXT: [[SETCCr:%[0-9]+]]:gr8 = SETCCr 3, implicit $eflags
+    ; GISEL-X86-NEXT: $al = COPY [[SETCCr]]
+    ; GISEL-X86-NEXT: RET 0, implicit $al
+    %3:_(p0) = G_FRAME_INDEX %fixed-stack.0
+    %2:_(s64) = G_LOAD %3(p0) :: (invariant load (s64) from %fixed-stack.0, align 8)
+    %1:_(s64) = G_LOAD %3(p0) :: (invariant load (s64) from %fixed-stack.0 + 8, basealign 8)
+    %4:_(s8) = G_FCMP floatpred(ole), %2(s64), %1
+    $al = COPY %4(s8)
+    RET 0, implicit $al
+
+...
+---
+name:            fcmp_double_one
+alignment:       16
+exposesReturnsTwice: false
+legalized:       true
+tracksRegLiveness: true
+fixedStack:
+  - { id: 0, type: default, offset: 8, size: 8, alignment: 8, stack-id: default,
+      isImmutable: true, isAliased: false, callee-saved-register: '', callee-saved-restored: true,
+      debug-info-variable: '', debug-info-expression: '', debug-info-location: '' }
+body:             |
+  bb.1.entry:
+    ; GISEL-X86-LABEL: name: fcmp_double_one
+    ; GISEL-X86: [[LD_Fp64m:%[0-9]+]]:rfp64 = LD_Fp64m %fixed-stack.0, 1, $noreg, 0, $noreg, implicit-def $fpsw, implicit $fpcw :: (invariant load (s64) from %fixed-stack.0)
+    ; GISEL-X86-NEXT: [[LD_Fp64m1:%[0-9]+]]:rfp64 = LD_Fp64m %fixed-stack.0, 1, $noreg, 0, $noreg, implicit-def $fpsw, implicit $fpcw :: (invariant load (s64) from %fixed-stack.0 + 8)
+    ; GISEL-X86-NEXT: UCOM_FpIr64 [[LD_Fp64m]], [[LD_Fp64m1]], implicit-def $eflags, implicit-def $fpsw, implicit $fpcw
+    ; GISEL-X86-NEXT: [[SETCCr:%[0-9]+]]:gr8 = SETCCr 5, implicit $eflags
+    ; GISEL-X86-NEXT: $al = COPY [[SETCCr]]
+    ; GISEL-X86-NEXT: RET 0, implicit $al
+    %3:_(p0) = G_FRAME_INDEX %fixed-stack.0
+    %2:_(s64) = G_LOAD %3(p0) :: (invariant load (s64) from %fixed-stack.0, align 8)
+    %1:_(s64) = G_LOAD %3(p0) :: (invariant load (s64) from %fixed-stack.0 + 8, basealign 8)
+    %4:_(s8) = G_FCMP floatpred(one), %2(s64), %1
+    $al = COPY %4(s8)
+    RET 0, implicit $al
+
+...
+---
+name:            fcmp_double_ord
+alignment:       16
+exposesReturnsTwice: false
+legalized:       true
+tracksRegLiveness: true
+fixedStack:
+  - { id: 0, type: default, offset: 8, size: 8, alignment: 8, stack-id: default,
+      isImmutable: true, isAliased: false, callee-saved-register: '', callee-saved-restored: true,
+      debug-info-variable: '', debug-info-expression: '', debug-info-location: '' }
+stack:           []
+body:             |
+  bb.1.entry:
+    ; GISEL-X86-LABEL: name: fcmp_double_ord
+    ; GISEL-X86: [[LD_Fp64m:%[0-9]+]]:rfp64 = LD_Fp64m %fixed-stack.0, 1, $noreg, 0, $noreg, implicit-def $fpsw, implicit $fpcw :: (invariant load (s64) from %fixed-stack.0)
+    ; GISEL-X86-NEXT: [[LD_Fp64m1:%[0-9]+]]:rfp64 = LD_Fp64m %fixed-stack.0, 1, $noreg, 0, $noreg, implicit-def $fpsw, implicit $fpcw :: (invariant load (s64) from %fixed-stack.0 + 8)
+    ; GISEL-X86-NEXT: UCOM_FpIr64 [[LD_Fp64m]], [[LD_Fp64m1]], implicit-def $eflags, implicit-def $fpsw, implicit $fpcw
+    ; GISEL-X86-NEXT: [[SETCCr:%[0-9]+]]:gr8 = SETCCr 11, implicit $eflags
+    ; GISEL-X86-NEXT: $al = COPY [[SETCCr]]
+    ; GISEL-X86-NEXT: RET 0, implicit $al
+    %3:_(p0) = G_FRAME_INDEX %fixed-stack.0
+    %2:_(s64) = G_LOAD %3(p0) :: (invariant load (s64) from %fixed-stack.0, align 8)
+    %1:_(s64) = G_LOAD %3(p0) :: (invariant load (s64) from %fixed-stack.0 + 8, basealign 8)
+    %4:_(s8) = G_FCMP floatpred(ord), %2(s64), %1
+    $al = COPY %4(s8)
+    RET 0, implicit $al
+
+...
+---
+name:            fcmp_double_uno
+alignment:       16
+exposesReturnsTwice: false
+legalized:       true
+fixedStack:
+  - { id: 0, type: default, offset: 8, size: 8, alignment: 8, stack-id: default,
+      isImmutable: true, isAliased: false, callee-saved-register: '', callee-saved-restored: true,
+      debug-info-variable: '', debug-info-expression: '', debug-info-location: '' }
+stack:           []
+body:             |
+  bb.1.entry:
+    ; GISEL-X86-LABEL: name: fcmp_double_uno
+    ; GISEL-X86: [[LD_Fp64m:%[0-9]+]]:rfp64 = LD_Fp64m %fixed-stack.0, 1, $noreg, 0, $noreg, implicit-def $fpsw, implicit $fpcw :: (invariant load (s64) from %fixed-stack.0)
+    ; GISEL-X86-NEXT: [[LD_Fp64m1:%[0-9]+]]:rfp64 = LD_Fp64m %fixed-stack.0, 1, $noreg, 0, $noreg, implicit-def $fpsw, implicit $fpcw :: (invariant load (s64) from %fixed-stack.0 + 8)
+    ; GISEL-X86-NEXT: UCOM_FpIr64 [[LD_Fp64m]], [[LD_Fp64m1]], implicit-def $eflags, implicit-def $fpsw, implicit $fpcw
+    ; GISEL-X86-NEXT: [[SETCCr:%[0-9]+]]:gr8 = SETCCr 10, implicit $eflags
+    ; GISEL-X86-NEXT: $al = COPY [[SETCCr]]
+    ; GISEL-X86-NEXT: RET 0, implicit $al
+    %3:_(p0) = G_FRAME_INDEX %fixed-stack.0
+    %2:_(s64) = G_LOAD %3(p0) :: (invariant load (s64) from %fixed-stack.0, align 8)
+    %1:_(s64) = G_LOAD %3(p0) :: (invariant load (s64) from %fixed-stack.0 + 8, basealign 8)
+    %4:_(s8) = G_FCMP floatpred(uno), %2(s64), %1
+    $al = COPY %4(s8)
+    RET 0, implicit $al
+
+...
+---
+name:            fcmp_double_ueq
+alignment:       16
+exposesReturnsTwice: false
+legalized:       true
+tracksRegLiveness: true
+fixedStack:
+  - { id: 0, type: default, offset: 8, size: 8, alignment: 8, stack-id: default,
+      isImmutable: true, isAliased: false, callee-saved-register: '', callee-saved-restored: true,
+      debug-info-variable: '', debug-info-expression: '', debug-info-location: '' }
+stack:           []
+body:             |
+  bb.1.entry:
+    ; GISEL-X86-LABEL: name: fcmp_double_ueq
+    ; GISEL-X86: [[LD_Fp64m:%[0-9]+]]:rfp64 = LD_Fp64m %fixed-stack.0, 1, $noreg, 0, $noreg, implicit-def $fpsw, implicit $fpcw :: (invariant load (s64) from %fixed-stack.0)
+    ; GISEL-X86-NEXT: [[LD_Fp64m1:%[0-9]+]]:rfp64 = LD_Fp64m %fixed-stack.0, 1, $noreg, 0, $noreg, implicit-def $fpsw, implicit $fpcw :: (invariant load (s64) from %fixed-stack.0 + 8)
+    ; GISEL-X86-NEXT: UCOM_FpIr64 [[LD_Fp64m]], [[LD_Fp64m1]], implicit-def $eflags, implicit-def $fpsw, implicit $fpcw
+    ; GISEL-X86-NEXT: [[SETCCr:%[0-9]+]]:gr8 = SETCCr 4, implicit $eflags
+    ; GISEL-X86-NEXT: $al = COPY [[SETCCr]]
+    ; GISEL-X86-NEXT: RET 0, implicit $al
+    %3:_(p0) = G_FRAME_INDEX %fixed-stack.0
+    %2:_(s64) = G_LOAD %3(p0) :: (invariant load (s64) from %fixed-stack.0, align 8)
+    %1:_(s64) = G_LOAD %3(p0) :: (invariant load (s64) from %fixed-stack.0 + 8, basealign 8)
+    %4:_(s8) = G_FCMP floatpred(ueq), %2(s64), %1
+    $al = COPY %4(s8)
+    RET 0, implicit $al
+
+...
+---
+name:            fcmp_double_ugt
+alignment:       16
+exposesReturnsTwice: false
+legalized:       true
+tracksRegLiveness: true
+fixedStack:
+  - { id: 0, type: default, offset: 8, size: 8, alignment: 8, stack-id: default,
+      isImmutable: true, isAliased: false, callee-saved-register: '', callee-saved-restored: true,
+      debug-info-variable: '', debug-info-expression: '', debug-info-location: '' }
+
+stack:           []
+body:             |
+  bb.1.entry:
+    ; GISEL-X86-LABEL: name: fcmp_double_ugt
+    ; GISEL-X86: [[LD_Fp64m:%[0-9]+]]:rfp64 = LD_Fp64m %fixed-stack.0, 1, $noreg, 0, $noreg, implicit-def $fpsw, implicit $fpcw :: (invariant load (s64) from %fixed-stack.0)
+    ; GISEL-X86-NEXT: [[LD_Fp64m1:%[0-9]+]]:rfp64 = LD_Fp64m %fixed-stack.0, 1, $noreg, 0, $noreg, implicit-def $fpsw, implicit $fpcw :: (invariant load (s64) from %fixed-stack.0 + 8)
+    ; GISEL-X86-NEXT: UCOM_FpIr64 [[LD_Fp64m1]], [[LD_Fp64m]], implicit-def $eflags, implicit-def $fpsw, implicit $fpcw
+    ; GISEL-X86-NEXT: [[SETCCr:%[0-9]+]]:gr8 = SETCCr 2, implicit $eflags
+    ; GISEL-X86-NEXT: $al = COPY [[SETCCr]]
+    ; GISEL-X86-NEXT: RET 0, implicit $al
+    %3:_(p0) = G_FRAME_INDEX %fixed-stack.0
+    %2:_(s64) = G_LOAD %3(p0) :: (invariant load (s64) from %fixed-stack.0, align 8)
+    %1:_(s64) = G_LOAD %3(p0) :: (invariant load (s64) from %fixed-stack.0 + 8, basealign 8)
+    %4:_(s8) = G_FCMP floatpred(ugt), %2(s64), %1
+    $al = COPY %4(s8)
+    RET 0, implicit $al
+
+...
+---
+name:            fcmp_double_uge
+alignment:       16
+exposesReturnsTwice: false
+legalized:       true
+tracksRegLiveness: true
+fixedStack:
+  - { id: 0, type: default, offset: 8, size: 8, alignment: 8, stack-id: default,
+      isImmutable: true, isAliased: false, callee-saved-register: '', callee-saved-restored: true,
+      debug-info-variable: '', debug-info-expression: '', debug-info-location: '' }
+stack:           []
+body:             |
+  bb.1.entry:
+    ; GISEL-X86-LABEL: name: fcmp_double_uge
+    ; GISEL-X86: [[LD_Fp64m:%[0-9]+]]:rfp64 = LD_Fp64m %fixed-stack.0, 1, $noreg, 0, $noreg, implicit-def $fpsw, implicit $fpcw :: (invariant load (s64) from %fixed-stack.0)
+    ; GISEL-X86-NEXT: [[LD_Fp64m1:%[0-9]+]]:rfp64 = LD_Fp64m %fixed-stack.0, 1, $noreg, 0, $noreg, implicit-def $fpsw, implicit $fpcw :: (invariant load (s64) from %fixed-stack.0 + 8)
+    ; GISEL-X86-NEXT: UCOM_FpIr64 [[LD_Fp64m1]], [[LD_Fp64m]], implicit-def $eflags, implicit-def $fpsw, implicit $fpcw
+    ; GISEL-X86-NEXT: [[SETCCr:%[0-9]+]]:gr8 = SETCCr 6, implicit $eflags
+    ; GISEL-X86-NEXT: $al = COPY [[SETCCr]]
+    ; GISEL-X86-NEXT: RET 0, implicit $al
+    %3:_(p0) = G_FRAME_INDEX %fixed-stack.0
+    %2:_(s64) = G_LOAD %3(p0) :: (invariant load (s64) from %fixed-stack.0, align 8)
+    %1:_(s64) = G_LOAD %3(p0) :: (invariant load (s64) from %fixed-stack.0 + 8, basealign 8)
+    %4:_(s8) = G_FCMP floatpred(uge), %2(s64), %1
+    $al = COPY %4(s8)
+    RET 0, implicit $al
+
+...
+---
+name:            fcmp_double_ult
+alignment:       16
+exposesReturnsTwice: false
+legalized:       true
+tracksRegLiveness: true
+
+fixedStack:
+  - { id: 0, type: default, offset: 8, size: 8, alignment: 8, stack-id: default,
+      isImmutable: true, isAliased: false, callee-saved-register: '', callee-saved-restored: true,
+      debug-info-variable: '', debug-info-expression: '', debug-info-location: '' }
+stack:           []
+body:             |
+  bb.1.entry:
+    ; GISEL-X86-LABEL: name: fcmp_double_ult
+    ; GISEL-X86: [[LD_Fp64m:%[0-9]+]]:rfp64 = LD_Fp64m %fixed-stack.0, 1, $noreg, 0, $noreg, implicit-def $fpsw, implicit $fpcw :: (invariant load (s64) from %fixed-stack.0)
+    ; GISEL-X86-NEXT: [[LD_Fp64m1:%[0-9]+]]:rfp64 = LD_Fp64m %fixed-stack.0, 1, $noreg, 0, $noreg, implicit-def $fpsw, implicit $fpcw :: (invariant load (s64) from %fixed-stack.0 + 8)
+    ; GISEL-X86-NEXT: UCOM_FpIr64 [[LD_Fp64m]], [[LD_Fp64m1]], implicit-def $eflags, implicit-def $fpsw, implicit $fpcw
+    ; GISEL-X86-NEXT: [[SETCCr:%[0-9]+]]:gr8 = SETCCr 2, implicit $eflags
+    ; GISEL-X86-NEXT: $al = COPY [[SETCCr]]
+    ; GISEL-X86-NEXT: RET 0, implicit $al
+    %3:_(p0) = G_FRAME_INDEX %fixed-stack.0
+    %2:_(s64) = G_LOAD %3(p0) :: (invariant load (s64) from %fixed-stack.0, align 8)
+    %1:_(s64) = G_LOAD %3(p0) :: (invariant load (s64) from %fixed-stack.0 + 8, basealign 8)
+    %4:_(s8) = G_FCMP floatpred(ult), %2(s64), %1
+    $al = COPY %4(s8)
+    RET 0, implicit $al
+
+...
+---
+name:            fcmp_double_ule
+alignment:       16
+exposesReturnsTwice: false
+legalized:       true
+tracksRegLiveness: true
+fixedStack:
+  - { id: 0, type: default, offset: 8, size: 8, alignment: 8, stack-id: default,
+      isImmutable: true, isAliased: false, callee-saved-register: '', callee-saved-restored: true,
+      debug-info-variable: '', debug-info-expression: '', debug-info-location: '' }
+stack:           []
+body:             |
+  bb.1.entry:
+    ; GISEL-X86-LABEL: name: fcmp_double_ule
+    ; GISEL-X86: [[LD_Fp64m:%[0-9]+]]:rfp64 = LD_Fp64m %fixed-stack.0, 1, $noreg, 0, $noreg, implicit-def $fpsw, implicit $fpcw :: (invariant load (s64) from %fixed-stack.0)
+    ; GISEL-X86-NEXT: [[LD_Fp64m1:%[0-9]+]]:rfp64 = LD_Fp64m %fixed-stack.0, 1, $noreg, 0, $noreg, implicit-def $fpsw, implicit $fpcw :: (invariant load (s64) from %fixed-stack.0 + 8)
+    ; GISEL-X86-NEXT: UCOM_FpIr64 [[LD_Fp64m]], [[LD_Fp64m1]], implicit-def $eflags, implicit-def $fpsw, implicit $fpcw
+    ; GISEL-X86-NEXT: [[SETCCr:%[0-9]+]]:gr8 = SETCCr 6, implicit $eflags
+    ; GISEL-X86-NEXT: $al = COPY [[SETCCr]]
+    ; GISEL-X86-NEXT: RET 0, implicit $al
+    %3:_(p0) = G_FRAME_INDEX %fixed-stack.0
+    %2:_(s64) = G_LOAD %3(p0) :: (invariant load (s64) from %fixed-stack.0, align 8)
+    %1:_(s64) = G_LOAD %3(p0) :: (invariant load (s64) from %fixed-stack.0 + 8, basealign 8)
+    %4:_(s8) = G_FCMP floatpred(ule), %2(s64), %1
+    $al = COPY %4(s8)
+    RET 0, implicit $al
+
+...
+---
+name:            fcmp_double_une
+alignment:       16
+exposesReturnsTwice: false
+legalized:       true
+tracksRegLiveness: true
+fixedStack:
+  - { id: 0, type: default, offset: 8, size: 8, alignment: 8, stack-id: default,
+      isImmutable: true, isAliased: false, callee-saved-register: '', callee-saved-restored: true,
+      debug-info-variable: '', debug-info-expression: '', debug-info-location: '' }
+body:             |
+  bb.1.entry:
+    ; GISEL-X86-LABEL: name: fcmp_double_une
+    ; GISEL-X86: [[LD_Fp64m:%[0-9]+]]:rfp64 = LD_Fp64m %fixed-stack.0, 1, $noreg, 0, $noreg, implicit-def $fpsw, implicit $fpcw :: (invariant load (s64) from %fixed-stack.0)
+    ; GISEL-X86-NEXT: [[LD_Fp64m1:%[0-9]+]]:rfp64 = LD_Fp64m %fixed-stack.0, 1, $noreg, 0, $noreg, implicit-def $fpsw, implicit $fpcw :: (invariant load (s64) from %fixed-stack.0 + 8)
+    ; GISEL-X86-NEXT: UCOM_FpIr64 [[LD_Fp64m]], [[LD_Fp64m1]], implicit-def $eflags, implicit-def $fpsw, implicit $fpcw
+    ; GISEL-X86-NEXT: [[SETCCr:%[0-9]+]]:gr8 = SETCCr 5, implicit $eflags
+    ; GISEL-X86-NEXT: [[SETCCr1:%[0-9]+]]:gr8 = SETCCr 10, implicit $eflags
+    ; GISEL-X86-NEXT: [[OR8rr:%[0-9]+]]:gr8 = OR8rr [[SETCCr]], [[SETCCr1]], implicit-def $eflags
+    ; GISEL-X86-NEXT: $al = COPY [[OR8rr]]
+    ; GISEL-X86-NEXT: RET 0, implicit $al
+    %3:_(p0) = G_FRAME_INDEX %fixed-stack.0
+    %2:_(s64) = G_LOAD %3(p0) :: (invariant load (s64) from %fixed-stack.0, align 8)
+    %1:_(s64) = G_LOAD %3(p0) :: (invariant load (s64) from %fixed-stack.0 + 8, basealign 8)
+    %4:_(s8) = G_FCMP floatpred(une), %2(s64), %1
+    $al = COPY %4(s8)
+    RET 0, implicit $al
+...
diff --git a/llvm/test/CodeGen/X86/apx/push2-pop2-cfi-seh.ll b/llvm/test/CodeGen/X86/apx/push2-pop2-cfi-seh.ll
index 6c9fdc2adce2f..ad24608d338ad 100644
--- a/llvm/test/CodeGen/X86/apx/push2-pop2-cfi-seh.ll
+++ b/llvm/test/CodeGen/X86/apx/push2-pop2-cfi-seh.ll
@@ -142,6 +142,7 @@ define i32 @csr6_alloc16(ptr %argv) {
 ; WIN-REF-NEXT:    xorl %eax, %eax
 ; WIN-REF-NEXT:    callq *%rax
 ; WIN-REF-NEXT:    nop
+; WIN-REF-NEXT:    .seh_startepilogue
 ; WIN-REF-NEXT:    addq $56, %rsp
 ; WIN-REF-NEXT:    popq %rbx
 ; WIN-REF-NEXT:    popq %rbp
@@ -149,6 +150,7 @@ define i32 @csr6_alloc16(ptr %argv) {
 ; WIN-REF-NEXT:    popq %r13
 ; WIN-REF-NEXT:    popq %r14
 ; WIN-REF-NEXT:    popq %r15
+; WIN-REF-NEXT:    .seh_endepilogue
 ; WIN-REF-NEXT:    retq
 ; WIN-REF-NEXT:    .seh_endproc
 ;
@@ -173,11 +175,13 @@ define i32 @csr6_alloc16(ptr %argv) {
 ; WIN-NEXT:    xorl %eax, %eax
 ; WIN-NEXT:    callq *%rax
 ; WIN-NEXT:    nop
+; WIN-NEXT:    .seh_startepilogue
 ; WIN-NEXT:    addq $64, %rsp
 ; WIN-NEXT:    pop2 %rbp, %rbx
 ; WIN-NEXT:    pop2 %r13, %r12
 ; WIN-NEXT:    pop2 %r15, %r14
 ; WIN-NEXT:    popq %rcx
+; WIN-NEXT:    .seh_endepilogue
 ; WIN-NEXT:    retq
 ; WIN-NEXT:    .seh_endproc
 ;
@@ -202,11 +206,13 @@ define i32 @csr6_alloc16(ptr %argv) {
 ; WIN-PPX-NEXT:    xorl %eax, %eax
 ; WIN-PPX-NEXT:    callq *%rax
 ; WIN-PPX-NEXT:    nop
+; WIN-PPX-NEXT:    .seh_startepilogue
 ; WIN-PPX-NEXT:    addq $64, %rsp
 ; WIN-PPX-NEXT:    pop2p %rbp, %rbx
 ; WIN-PPX-NEXT:    pop2p %r13, %r12
 ; WIN-PPX-NEXT:    pop2p %r15, %r14
 ; WIN-PPX-NEXT:    popq %rcx
+; WIN-PPX-NEXT:    .seh_endepilogue
 ; WIN-PPX-NEXT:    retq
 ; WIN-PPX-NEXT:    .seh_endproc
 entry:
diff --git a/llvm/test/CodeGen/X86/avx512-arith.ll b/llvm/test/CodeGen/X86/avx512-arith.ll
index 0fae921b1ca83..20550fc4eb9fa 100644
--- a/llvm/test/CodeGen/X86/avx512-arith.ll
+++ b/llvm/test/CodeGen/X86/avx512-arith.ll
@@ -1158,7 +1158,7 @@ define <16 x i32> @masked_inc_test(<16 x i32> %i, <16 x i32> %mask1) nounwind re
 ; CHECK-LABEL: masked_inc_test:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vptestmd %zmm1, %zmm1, %k1
-; CHECK-NEXT:    vpternlogd $255, %zmm1, %zmm1, %zmm1
+; CHECK-NEXT:    vpternlogd {{.*#+}} zmm1 = -1
 ; CHECK-NEXT:    vpsubd %zmm1, %zmm0, %zmm0 {%k1}
 ; CHECK-NEXT:    retq
   %mask = icmp ne <16 x i32> %mask1, zeroinitializer
@@ -1171,7 +1171,7 @@ define <16 x i32> @masked_dec_test(<16 x i32> %i, <16 x i32> %mask1) nounwind re
 ; CHECK-LABEL: masked_dec_test:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vptestmd %zmm1, %zmm1, %k1
-; CHECK-NEXT:    vpternlogd $255, %zmm1, %zmm1, %zmm1
+; CHECK-NEXT:    vpternlogd {{.*#+}} zmm1 = -1
 ; CHECK-NEXT:    vpaddd %zmm1, %zmm0, %zmm0 {%k1}
 ; CHECK-NEXT:    retq
   %mask = icmp ne <16 x i32> %mask1, zeroinitializer
diff --git a/llvm/test/CodeGen/X86/avx512-broadcast-arith.ll b/llvm/test/CodeGen/X86/avx512-broadcast-arith.ll
index 7e48b3719cf0f..5d7c337c94dee 100644
--- a/llvm/test/CodeGen/X86/avx512-broadcast-arith.ll
+++ b/llvm/test/CodeGen/X86/avx512-broadcast-arith.ll
@@ -18,18 +18,18 @@ define <64 x i8> @add_v64i8_broadcasts(<64 x i8> %a0, i64 %a1, i8 %a2) {
 ; AVX512F-NEXT:    kmovw %ecx, %k2
 ; AVX512F-NEXT:    kmovw %eax, %k3
 ; AVX512F-NEXT:    kmovw %edi, %k4
-; AVX512F-NEXT:    vpternlogd $255, %zmm2, %zmm2, %zmm2 {%k4} {z}
+; AVX512F-NEXT:    vpternlogd {{.*#+}} zmm2 {%k4} {z} = -1
 ; AVX512F-NEXT:    vpmovdb %zmm2, %xmm2
-; AVX512F-NEXT:    vpternlogd $255, %zmm3, %zmm3, %zmm3 {%k3} {z}
+; AVX512F-NEXT:    vpternlogd {{.*#+}} zmm3 {%k3} {z} = -1
 ; AVX512F-NEXT:    vpmovdb %zmm3, %xmm3
 ; AVX512F-NEXT:    vinserti128 $1, %xmm3, %ymm2, %ymm2
-; AVX512F-NEXT:    vpternlogd $255, %zmm3, %zmm3, %zmm3 {%k1} {z}
+; AVX512F-NEXT:    vpternlogd {{.*#+}} zmm3 {%k1} {z} = -1
 ; AVX512F-NEXT:    vpmovdb %zmm3, %xmm3
-; AVX512F-NEXT:    vpternlogd $255, %zmm4, %zmm4, %zmm4 {%k2} {z}
+; AVX512F-NEXT:    vpternlogd {{.*#+}} zmm4 {%k2} {z} = -1
 ; AVX512F-NEXT:    vpmovdb %zmm4, %xmm4
 ; AVX512F-NEXT:    vinserti128 $1, %xmm4, %ymm3, %ymm3
 ; AVX512F-NEXT:    vinserti64x4 $1, %ymm2, %zmm3, %zmm2
-; AVX512F-NEXT:    vpternlogq $216, %zmm2, %zmm1, %zmm0
+; AVX512F-NEXT:    vpternlogq {{.*#+}} zmm0 = zmm0 ^ (zmm2 & (zmm0 ^ zmm1))
 ; AVX512F-NEXT:    vextracti64x4 $1, %zmm0, %ymm3
 ; AVX512F-NEXT:    vpaddb %ymm1, %ymm3, %ymm3
 ; AVX512F-NEXT:    vpaddb %ymm1, %ymm0, %ymm0
@@ -37,7 +37,7 @@ define <64 x i8> @add_v64i8_broadcasts(<64 x i8> %a0, i64 %a1, i8 %a2) {
 ; AVX512F-NEXT:    vpaddb %ymm1, %ymm3, %ymm3
 ; AVX512F-NEXT:    vpaddb %ymm1, %ymm0, %ymm0
 ; AVX512F-NEXT:    vinserti64x4 $1, %ymm3, %zmm0, %zmm0
-; AVX512F-NEXT:    vpternlogq $226, %zmm4, %zmm2, %zmm0
+; AVX512F-NEXT:    vpternlogq {{.*#+}} zmm0 = zmm4 ^ (zmm2 & (zmm0 ^ zmm4))
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512BW-LABEL: add_v64i8_broadcasts:
diff --git a/llvm/test/CodeGen/X86/avx512-calling-conv.ll b/llvm/test/CodeGen/X86/avx512-calling-conv.ll
index c27cced9d5ffa..23b46ee59154f 100644
--- a/llvm/test/CodeGen/X86/avx512-calling-conv.ll
+++ b/llvm/test/CodeGen/X86/avx512-calling-conv.ll
@@ -166,7 +166,7 @@ define <16 x i32> @test6(<16 x i32>%a, <16 x i32>%b) {
 ; KNL-NEXT:    pushq %rax
 ; KNL-NEXT:    .cfi_def_cfa_offset 16
 ; KNL-NEXT:    vpcmpgtd %zmm1, %zmm0, %k1
-; KNL-NEXT:    vpternlogd $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
+; KNL-NEXT:    vpternlogd {{.*#+}} zmm0 {%k1} {z} = -1
 ; KNL-NEXT:    vpmovdb %zmm0, %xmm0
 ; KNL-NEXT:    callq _func16xi1
 ; KNL-NEXT:    vpmovzxbd {{.*#+}} zmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero,xmm0[8],zero,zero,zero,xmm0[9],zero,zero,zero,xmm0[10],zero,zero,zero,xmm0[11],zero,zero,zero,xmm0[12],zero,zero,zero,xmm0[13],zero,zero,zero,xmm0[14],zero,zero,zero,xmm0[15],zero,zero,zero
@@ -194,7 +194,7 @@ define <16 x i32> @test6(<16 x i32>%a, <16 x i32>%b) {
 ; KNL_X32-NEXT:    subl $12, %esp
 ; KNL_X32-NEXT:    .cfi_def_cfa_offset 16
 ; KNL_X32-NEXT:    vpcmpgtd %zmm1, %zmm0, %k1
-; KNL_X32-NEXT:    vpternlogd $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
+; KNL_X32-NEXT:    vpternlogd {{.*#+}} zmm0 {%k1} {z} = -1
 ; KNL_X32-NEXT:    vpmovdb %zmm0, %xmm0
 ; KNL_X32-NEXT:    calll _func16xi1
 ; KNL_X32-NEXT:    vpmovzxbd {{.*#+}} zmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero,xmm0[8],zero,zero,zero,xmm0[9],zero,zero,zero,xmm0[10],zero,zero,zero,xmm0[11],zero,zero,zero,xmm0[12],zero,zero,zero,xmm0[13],zero,zero,zero,xmm0[14],zero,zero,zero,xmm0[15],zero,zero,zero
diff --git a/llvm/test/CodeGen/X86/avx512-cmp.ll b/llvm/test/CodeGen/X86/avx512-cmp.ll
index e46378804be5e..56d6d136fdb61 100644
--- a/llvm/test/CodeGen/X86/avx512-cmp.ll
+++ b/llvm/test/CodeGen/X86/avx512-cmp.ll
@@ -190,7 +190,7 @@ define <8 x i32> @legalize_loop(<8 x double> %arg) {
 ; KNL:       ## %bb.0:
 ; KNL-NEXT:    vxorpd %xmm1, %xmm1, %xmm1
 ; KNL-NEXT:    vcmpnltpd %zmm0, %zmm1, %k1
-; KNL-NEXT:    vpternlogd $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
+; KNL-NEXT:    vpternlogd {{.*#+}} zmm0 {%k1} {z} = -1
 ; KNL-NEXT:    vpshufd {{.*#+}} ymm1 = ymm0[3,2,1,0,7,6,5,4]
 ; KNL-NEXT:    vpsrld $31, %ymm1, %ymm1
 ; KNL-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[2,3,0,1]
diff --git a/llvm/test/CodeGen/X86/avx512-ext.ll b/llvm/test/CodeGen/X86/avx512-ext.ll
index 79e59fdcf4a12..c60d9a3ff17d3 100644
--- a/llvm/test/CodeGen/X86/avx512-ext.ll
+++ b/llvm/test/CodeGen/X86/avx512-ext.ll
@@ -1603,7 +1603,7 @@ define   <16 x i32> @zext_16i1_to_16xi32(i16 %b) {
 ; KNL-LABEL: zext_16i1_to_16xi32:
 ; KNL:       # %bb.0:
 ; KNL-NEXT:    kmovw %edi, %k1
-; KNL-NEXT:    vpternlogd $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
+; KNL-NEXT:    vpternlogd {{.*#+}} zmm0 {%k1} {z} = -1
 ; KNL-NEXT:    vpsrld $31, %zmm0, %zmm0
 ; KNL-NEXT:    retq
 ;
@@ -1629,7 +1629,7 @@ define   <8 x i64> @zext_8i1_to_8xi64(i8 %b) {
 ; KNL-LABEL: zext_8i1_to_8xi64:
 ; KNL:       # %bb.0:
 ; KNL-NEXT:    kmovw %edi, %k1
-; KNL-NEXT:    vpternlogq $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
+; KNL-NEXT:    vpternlogq {{.*#+}} zmm0 {%k1} {z} = -1
 ; KNL-NEXT:    vpsrlq $63, %zmm0, %zmm0
 ; KNL-NEXT:    retq
 ;
@@ -1747,14 +1747,14 @@ define <8 x i32> @sext_8i1_8i32(<8 x i32> %a1, <8 x i32> %a2) nounwind {
 ; KNL-LABEL: sext_8i1_8i32:
 ; KNL:       # %bb.0:
 ; KNL-NEXT:    vpcmpgtd %ymm0, %ymm1, %ymm0
-; KNL-NEXT:    vpternlogq $15, %zmm0, %zmm0, %zmm0
+; KNL-NEXT:    vpternlogq {{.*#+}} zmm0 = ~zmm0
 ; KNL-NEXT:    # kill: def $ymm0 killed $ymm0 killed $zmm0
 ; KNL-NEXT:    retq
 ;
 ; AVX512DQ-LABEL: sext_8i1_8i32:
 ; AVX512DQ:       # %bb.0:
 ; AVX512DQ-NEXT:    vpcmpgtd %ymm0, %ymm1, %ymm0
-; AVX512DQ-NEXT:    vpternlogq $15, %ymm0, %ymm0, %ymm0
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} ymm0 = ~ymm0
 ; AVX512DQ-NEXT:    retq
   %x = icmp slt <8 x i32> %a1, %a2
   %x1 = xor <8 x i1>%x, <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>
@@ -1840,7 +1840,7 @@ define <16 x i32> @sext_16i1_16i32(<16 x i32> %a1, <16 x i32> %a2) nounwind {
 ; KNL-LABEL: sext_16i1_16i32:
 ; KNL:       # %bb.0:
 ; KNL-NEXT:    vpcmpgtd %zmm0, %zmm1, %k1
-; KNL-NEXT:    vpternlogd $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
+; KNL-NEXT:    vpternlogd {{.*#+}} zmm0 {%k1} {z} = -1
 ; KNL-NEXT:    retq
 ;
 ; AVX512DQ-LABEL: sext_16i1_16i32:
@@ -1897,7 +1897,7 @@ define <64 x i16> @test21(<64 x i16> %x , <64 x i1> %mask) nounwind readnone {
 ; KNL-NEXT:    movw $-5, %ax
 ; KNL-NEXT:    kmovw %eax, %k1
 ; KNL-NEXT:    kandw %k1, %k0, %k0
-; KNL-NEXT:    kmovw %k1, %k7
+; KNL-NEXT:    kmovw %k1, %k2
 ; KNL-NEXT:    kmovw %k1, {{[-0-9]+}}(%r{{[sb]}}p) # 2-byte Spill
 ; KNL-NEXT:    movzbl {{[0-9]+}}(%rsp), %eax
 ; KNL-NEXT:    kmovw %eax, %k1
@@ -1906,8 +1906,8 @@ define <64 x i16> @test21(<64 x i16> %x , <64 x i1> %mask) nounwind readnone {
 ; KNL-NEXT:    korw %k1, %k0, %k0
 ; KNL-NEXT:    movw $-9, %ax
 ; KNL-NEXT:    kmovw %eax, %k1
-; KNL-NEXT:    kmovw %k1, {{[-0-9]+}}(%r{{[sb]}}p) # 2-byte Spill
 ; KNL-NEXT:    kandw %k1, %k0, %k0
+; KNL-NEXT:    kmovw %k1, %k7
 ; KNL-NEXT:    movzbl {{[0-9]+}}(%rsp), %eax
 ; KNL-NEXT:    kmovw %eax, %k1
 ; KNL-NEXT:    kshiftlw $15, %k1, %k1
@@ -1924,9 +1924,8 @@ define <64 x i16> @test21(<64 x i16> %x , <64 x i1> %mask) nounwind readnone {
 ; KNL-NEXT:    korw %k1, %k0, %k0
 ; KNL-NEXT:    movw $-33, %ax
 ; KNL-NEXT:    kmovw %eax, %k1
-; KNL-NEXT:    kandw %k1, %k0, %k0
-; KNL-NEXT:    kmovw %k1, %k2
 ; KNL-NEXT:    kmovw %k1, {{[-0-9]+}}(%r{{[sb]}}p) # 2-byte Spill
+; KNL-NEXT:    kandw %k1, %k0, %k0
 ; KNL-NEXT:    movzbl {{[0-9]+}}(%rsp), %eax
 ; KNL-NEXT:    kmovw %eax, %k1
 ; KNL-NEXT:    kshiftlw $15, %k1, %k1
@@ -1934,8 +1933,9 @@ define <64 x i16> @test21(<64 x i16> %x , <64 x i1> %mask) nounwind readnone {
 ; KNL-NEXT:    korw %k1, %k0, %k0
 ; KNL-NEXT:    movw $-65, %ax
 ; KNL-NEXT:    kmovw %eax, %k1
-; KNL-NEXT:    kmovw %k1, {{[-0-9]+}}(%r{{[sb]}}p) # 2-byte Spill
 ; KNL-NEXT:    kandw %k1, %k0, %k0
+; KNL-NEXT:    kmovw %k1, %k3
+; KNL-NEXT:    kmovw %k1, {{[-0-9]+}}(%r{{[sb]}}p) # 2-byte Spill
 ; KNL-NEXT:    movzbl {{[0-9]+}}(%rsp), %eax
 ; KNL-NEXT:    kmovw %eax, %k1
 ; KNL-NEXT:    kshiftlw $15, %k1, %k1
@@ -1943,9 +1943,8 @@ define <64 x i16> @test21(<64 x i16> %x , <64 x i1> %mask) nounwind readnone {
 ; KNL-NEXT:    korw %k1, %k0, %k0
 ; KNL-NEXT:    movw $-129, %ax
 ; KNL-NEXT:    kmovw %eax, %k1
-; KNL-NEXT:    kandw %k1, %k0, %k0
-; KNL-NEXT:    kmovw %k1, %k3
 ; KNL-NEXT:    kmovw %k1, {{[-0-9]+}}(%r{{[sb]}}p) # 2-byte Spill
+; KNL-NEXT:    kandw %k1, %k0, %k0
 ; KNL-NEXT:    movzbl {{[0-9]+}}(%rsp), %eax
 ; KNL-NEXT:    kmovw %eax, %k1
 ; KNL-NEXT:    kshiftlw $15, %k1, %k1
@@ -1953,18 +1952,17 @@ define <64 x i16> @test21(<64 x i16> %x , <64 x i1> %mask) nounwind readnone {
 ; KNL-NEXT:    korw %k1, %k0, %k0
 ; KNL-NEXT:    movw $-257, %ax # imm = 0xFEFF
 ; KNL-NEXT:    kmovw %eax, %k1
-; KNL-NEXT:    kmovw %k1, {{[-0-9]+}}(%r{{[sb]}}p) # 2-byte Spill
 ; KNL-NEXT:    kandw %k1, %k0, %k0
+; KNL-NEXT:    kmovw %k1, %k4
+; KNL-NEXT:    kmovw %k1, {{[-0-9]+}}(%r{{[sb]}}p) # 2-byte Spill
 ; KNL-NEXT:    movzbl {{[0-9]+}}(%rsp), %eax
 ; KNL-NEXT:    kmovw %eax, %k1
 ; KNL-NEXT:    kshiftlw $15, %k1, %k1
 ; KNL-NEXT:    kshiftrw $7, %k1, %k1
 ; KNL-NEXT:    korw %k1, %k0, %k0
 ; KNL-NEXT:    movw $-513, %ax # imm = 0xFDFF
-; KNL-NEXT:    kmovw %eax, %k1
-; KNL-NEXT:    kandw %k1, %k0, %k0
-; KNL-NEXT:    kmovw %k1, %k4
-; KNL-NEXT:    kmovw %k1, {{[-0-9]+}}(%r{{[sb]}}p) # 2-byte Spill
+; KNL-NEXT:    kmovw %eax, %k5
+; KNL-NEXT:    kandw %k5, %k0, %k0
 ; KNL-NEXT:    movzbl {{[0-9]+}}(%rsp), %eax
 ; KNL-NEXT:    kmovw %eax, %k1
 ; KNL-NEXT:    kshiftlw $15, %k1, %k1
@@ -1980,8 +1978,9 @@ define <64 x i16> @test21(<64 x i16> %x , <64 x i1> %mask) nounwind readnone {
 ; KNL-NEXT:    kshiftrw $5, %k1, %k1
 ; KNL-NEXT:    korw %k1, %k0, %k0
 ; KNL-NEXT:    movw $-2049, %ax # imm = 0xF7FF
-; KNL-NEXT:    kmovw %eax, %k5
-; KNL-NEXT:    kandw %k5, %k0, %k0
+; KNL-NEXT:    kmovw %eax, %k1
+; KNL-NEXT:    kmovw %k1, {{[-0-9]+}}(%r{{[sb]}}p) # 2-byte Spill
+; KNL-NEXT:    kandw %k1, %k0, %k0
 ; KNL-NEXT:    movzbl {{[0-9]+}}(%rsp), %eax
 ; KNL-NEXT:    kmovw %eax, %k1
 ; KNL-NEXT:    kshiftlw $15, %k1, %k1
@@ -2026,49 +2025,50 @@ define <64 x i16> @test21(<64 x i16> %x , <64 x i1> %mask) nounwind readnone {
 ; KNL-NEXT:    kshiftrw $14, %k1, %k1
 ; KNL-NEXT:    kmovw %edi, %k6
 ; KNL-NEXT:    korw %k1, %k6, %k1
-; KNL-NEXT:    kandw %k7, %k1, %k1
+; KNL-NEXT:    kandw %k2, %k1, %k1
 ; KNL-NEXT:    kmovw %edx, %k6
 ; KNL-NEXT:    kshiftlw $15, %k6, %k6
 ; KNL-NEXT:    kshiftrw $13, %k6, %k6
 ; KNL-NEXT:    korw %k6, %k1, %k1
-; KNL-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k7 # 2-byte Reload
+; KNL-NEXT:    kmovw %k7, %k0
+; KNL-NEXT:    kmovw %k7, {{[-0-9]+}}(%r{{[sb]}}p) # 2-byte Spill
 ; KNL-NEXT:    kandw %k7, %k1, %k1
 ; KNL-NEXT:    kmovw %ecx, %k6
 ; KNL-NEXT:    kshiftlw $15, %k6, %k6
 ; KNL-NEXT:    kshiftrw $12, %k6, %k6
 ; KNL-NEXT:    korw %k6, %k1, %k1
-; KNL-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k0 # 2-byte Reload
-; KNL-NEXT:    kandw %k0, %k1, %k1
+; KNL-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k2 # 2-byte Reload
+; KNL-NEXT:    kandw %k2, %k1, %k1
 ; KNL-NEXT:    kmovw %r8d, %k6
 ; KNL-NEXT:    kshiftlw $15, %k6, %k6
 ; KNL-NEXT:    kshiftrw $11, %k6, %k6
 ; KNL-NEXT:    korw %k6, %k1, %k1
-; KNL-NEXT:    kandw %k2, %k1, %k1
+; KNL-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k7 # 2-byte Reload
+; KNL-NEXT:    kandw %k7, %k1, %k1
 ; KNL-NEXT:    kmovw %r9d, %k6
 ; KNL-NEXT:    kshiftlw $15, %k6, %k6
 ; KNL-NEXT:    kshiftrw $10, %k6, %k6
 ; KNL-NEXT:    korw %k6, %k1, %k1
-; KNL-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k2 # 2-byte Reload
-; KNL-NEXT:    kandw %k2, %k1, %k1
+; KNL-NEXT:    kandw %k3, %k1, %k1
 ; KNL-NEXT:    movzbl {{[0-9]+}}(%rsp), %eax
 ; KNL-NEXT:    kmovw %eax, %k6
 ; KNL-NEXT:    kshiftlw $15, %k6, %k6
 ; KNL-NEXT:    kshiftrw $9, %k6, %k6
 ; KNL-NEXT:    korw %k6, %k1, %k1
+; KNL-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k3 # 2-byte Reload
 ; KNL-NEXT:    kandw %k3, %k1, %k1
 ; KNL-NEXT:    movzbl {{[0-9]+}}(%rsp), %eax
 ; KNL-NEXT:    kmovw %eax, %k6
 ; KNL-NEXT:    kshiftlw $15, %k6, %k6
 ; KNL-NEXT:    kshiftrw $8, %k6, %k6
 ; KNL-NEXT:    korw %k6, %k1, %k1
-; KNL-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k3 # 2-byte Reload
-; KNL-NEXT:    kandw %k3, %k1, %k1
+; KNL-NEXT:    kandw %k4, %k1, %k1
 ; KNL-NEXT:    movzbl {{[0-9]+}}(%rsp), %eax
 ; KNL-NEXT:    kmovw %eax, %k6
 ; KNL-NEXT:    kshiftlw $15, %k6, %k6
 ; KNL-NEXT:    kshiftrw $7, %k6, %k6
 ; KNL-NEXT:    korw %k6, %k1, %k1
-; KNL-NEXT:    kandw %k4, %k1, %k1
+; KNL-NEXT:    kandw %k5, %k1, %k1
 ; KNL-NEXT:    movzbl {{[0-9]+}}(%rsp), %eax
 ; KNL-NEXT:    kmovw %eax, %k6
 ; KNL-NEXT:    kshiftlw $15, %k6, %k6
@@ -2081,8 +2081,8 @@ define <64 x i16> @test21(<64 x i16> %x , <64 x i1> %mask) nounwind readnone {
 ; KNL-NEXT:    kshiftlw $15, %k6, %k6
 ; KNL-NEXT:    kshiftrw $5, %k6, %k6
 ; KNL-NEXT:    korw %k6, %k1, %k1
-; KNL-NEXT:    kmovw %k5, {{[-0-9]+}}(%r{{[sb]}}p) # 2-byte Spill
-; KNL-NEXT:    kandw %k5, %k1, %k1
+; KNL-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k6 # 2-byte Reload
+; KNL-NEXT:    kandw %k6, %k1, %k1
 ; KNL-NEXT:    movzbl {{[0-9]+}}(%rsp), %eax
 ; KNL-NEXT:    kmovw %eax, %k6
 ; KNL-NEXT:    kshiftlw $15, %k6, %k6
@@ -2130,46 +2130,45 @@ define <64 x i16> @test21(<64 x i16> %x , <64 x i1> %mask) nounwind readnone {
 ; KNL-NEXT:    kshiftlw $15, %k6, %k6
 ; KNL-NEXT:    kshiftrw $13, %k6, %k6
 ; KNL-NEXT:    korw %k6, %k1, %k1
-; KNL-NEXT:    kandw %k7, %k1, %k1
+; KNL-NEXT:    kandw %k0, %k1, %k1
 ; KNL-NEXT:    movzbl {{[0-9]+}}(%rsp), %eax
 ; KNL-NEXT:    kmovw %eax, %k6
 ; KNL-NEXT:    kshiftlw $15, %k6, %k6
 ; KNL-NEXT:    kshiftrw $12, %k6, %k6
 ; KNL-NEXT:    korw %k6, %k1, %k1
-; KNL-NEXT:    kandw %k0, %k1, %k1
+; KNL-NEXT:    kandw %k2, %k1, %k1
 ; KNL-NEXT:    movzbl {{[0-9]+}}(%rsp), %eax
 ; KNL-NEXT:    kmovw %eax, %k6
 ; KNL-NEXT:    kshiftlw $15, %k6, %k6
 ; KNL-NEXT:    kshiftrw $11, %k6, %k6
 ; KNL-NEXT:    korw %k6, %k1, %k1
-; KNL-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k0 # 2-byte Reload
-; KNL-NEXT:    kandw %k0, %k1, %k1
+; KNL-NEXT:    kandw %k7, %k1, %k1
 ; KNL-NEXT:    movzbl {{[0-9]+}}(%rsp), %eax
 ; KNL-NEXT:    kmovw %eax, %k6
 ; KNL-NEXT:    kshiftlw $15, %k6, %k6
 ; KNL-NEXT:    kshiftrw $10, %k6, %k6
 ; KNL-NEXT:    korw %k6, %k1, %k1
-; KNL-NEXT:    kandw %k2, %k1, %k1
+; KNL-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k0 # 2-byte Reload
+; KNL-NEXT:    kandw %k0, %k1, %k1
 ; KNL-NEXT:    movzbl {{[0-9]+}}(%rsp), %eax
 ; KNL-NEXT:    kmovw %eax, %k6
 ; KNL-NEXT:    kshiftlw $15, %k6, %k6
 ; KNL-NEXT:    kshiftrw $9, %k6, %k6
 ; KNL-NEXT:    korw %k6, %k1, %k1
-; KNL-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k0 # 2-byte Reload
-; KNL-NEXT:    kandw %k0, %k1, %k1
+; KNL-NEXT:    kandw %k3, %k1, %k1
 ; KNL-NEXT:    movzbl {{[0-9]+}}(%rsp), %eax
 ; KNL-NEXT:    kmovw %eax, %k6
 ; KNL-NEXT:    kshiftlw $15, %k6, %k6
 ; KNL-NEXT:    kshiftrw $8, %k6, %k6
 ; KNL-NEXT:    korw %k6, %k1, %k1
+; KNL-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k3 # 2-byte Reload
 ; KNL-NEXT:    kandw %k3, %k1, %k1
 ; KNL-NEXT:    movzbl {{[0-9]+}}(%rsp), %eax
 ; KNL-NEXT:    kmovw %eax, %k6
 ; KNL-NEXT:    kshiftlw $15, %k6, %k6
 ; KNL-NEXT:    kshiftrw $7, %k6, %k6
 ; KNL-NEXT:    korw %k6, %k1, %k1
-; KNL-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k3 # 2-byte Reload
-; KNL-NEXT:    kandw %k3, %k1, %k1
+; KNL-NEXT:    kandw %k5, %k1, %k1
 ; KNL-NEXT:    movzbl {{[0-9]+}}(%rsp), %eax
 ; KNL-NEXT:    kmovw %eax, %k6
 ; KNL-NEXT:    kshiftlw $15, %k6, %k6
@@ -2181,28 +2180,29 @@ define <64 x i16> @test21(<64 x i16> %x , <64 x i1> %mask) nounwind readnone {
 ; KNL-NEXT:    kshiftlw $15, %k6, %k6
 ; KNL-NEXT:    kshiftrw $5, %k6, %k6
 ; KNL-NEXT:    korw %k6, %k1, %k1
-; KNL-NEXT:    kandw %k5, %k1, %k1
+; KNL-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k0 # 2-byte Reload
+; KNL-NEXT:    kandw %k0, %k1, %k1
 ; KNL-NEXT:    movzbl {{[0-9]+}}(%rsp), %eax
 ; KNL-NEXT:    kmovw %eax, %k6
 ; KNL-NEXT:    kshiftlw $15, %k6, %k6
 ; KNL-NEXT:    kshiftrw $4, %k6, %k6
 ; KNL-NEXT:    korw %k6, %k1, %k1
-; KNL-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k0 # 2-byte Reload
-; KNL-NEXT:    kandw %k0, %k1, %k1
+; KNL-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k2 # 2-byte Reload
+; KNL-NEXT:    kandw %k2, %k1, %k1
 ; KNL-NEXT:    movzbl {{[0-9]+}}(%rsp), %eax
 ; KNL-NEXT:    kmovw %eax, %k6
 ; KNL-NEXT:    kshiftlw $15, %k6, %k6
 ; KNL-NEXT:    kshiftrw $3, %k6, %k6
 ; KNL-NEXT:    korw %k6, %k1, %k1
-; KNL-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k2 # 2-byte Reload
-; KNL-NEXT:    kandw %k2, %k1, %k1
+; KNL-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k6 # 2-byte Reload
+; KNL-NEXT:    kandw %k6, %k1, %k1
 ; KNL-NEXT:    movzbl {{[0-9]+}}(%rsp), %eax
 ; KNL-NEXT:    kmovw %eax, %k6
 ; KNL-NEXT:    kshiftlw $15, %k6, %k6
 ; KNL-NEXT:    kshiftrw $2, %k6, %k6
 ; KNL-NEXT:    korw %k6, %k1, %k1
-; KNL-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k5 # 2-byte Reload
-; KNL-NEXT:    kandw %k5, %k1, %k1
+; KNL-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k6 # 2-byte Reload
+; KNL-NEXT:    kandw %k6, %k1, %k1
 ; KNL-NEXT:    movzbl {{[0-9]+}}(%rsp), %eax
 ; KNL-NEXT:    kmovw %eax, %k6
 ; KNL-NEXT:    kshiftlw $14, %k6, %k6
@@ -2221,56 +2221,55 @@ define <64 x i16> @test21(<64 x i16> %x , <64 x i1> %mask) nounwind readnone {
 ; KNL-NEXT:    kshiftrw $14, %k6, %k6
 ; KNL-NEXT:    kmovw %eax, %k7
 ; KNL-NEXT:    korw %k6, %k7, %k6
-; KNL-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k5 # 2-byte Reload
-; KNL-NEXT:    kandw %k5, %k6, %k6
+; KNL-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k7 # 2-byte Reload
+; KNL-NEXT:    kandw %k7, %k6, %k6
 ; KNL-NEXT:    movzbl {{[0-9]+}}(%rsp), %eax
 ; KNL-NEXT:    kmovw %eax, %k7
 ; KNL-NEXT:    kshiftlw $15, %k7, %k7
 ; KNL-NEXT:    kshiftrw $13, %k7, %k7
 ; KNL-NEXT:    korw %k7, %k6, %k6
-; KNL-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k5 # 2-byte Reload
-; KNL-NEXT:    kandw %k5, %k6, %k6
+; KNL-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k7 # 2-byte Reload
+; KNL-NEXT:    kandw %k7, %k6, %k6
 ; KNL-NEXT:    movzbl {{[0-9]+}}(%rsp), %eax
 ; KNL-NEXT:    kmovw %eax, %k7
 ; KNL-NEXT:    kshiftlw $15, %k7, %k7
 ; KNL-NEXT:    kshiftrw $12, %k7, %k7
 ; KNL-NEXT:    korw %k7, %k6, %k6
-; KNL-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k5 # 2-byte Reload
-; KNL-NEXT:    kandw %k5, %k6, %k6
+; KNL-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k7 # 2-byte Reload
+; KNL-NEXT:    kandw %k7, %k6, %k6
 ; KNL-NEXT:    movzbl {{[0-9]+}}(%rsp), %eax
 ; KNL-NEXT:    kmovw %eax, %k7
 ; KNL-NEXT:    kshiftlw $15, %k7, %k7
 ; KNL-NEXT:    kshiftrw $11, %k7, %k7
 ; KNL-NEXT:    korw %k7, %k6, %k6
-; KNL-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k5 # 2-byte Reload
-; KNL-NEXT:    kandw %k5, %k6, %k6
+; KNL-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k7 # 2-byte Reload
+; KNL-NEXT:    kandw %k7, %k6, %k6
 ; KNL-NEXT:    movzbl {{[0-9]+}}(%rsp), %eax
 ; KNL-NEXT:    kmovw %eax, %k7
 ; KNL-NEXT:    kshiftlw $15, %k7, %k7
 ; KNL-NEXT:    kshiftrw $10, %k7, %k7
 ; KNL-NEXT:    korw %k7, %k6, %k6
-; KNL-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k5 # 2-byte Reload
-; KNL-NEXT:    kandw %k5, %k6, %k6
+; KNL-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k7 # 2-byte Reload
+; KNL-NEXT:    kandw %k7, %k6, %k6
 ; KNL-NEXT:    movzbl {{[0-9]+}}(%rsp), %eax
 ; KNL-NEXT:    kmovw %eax, %k7
 ; KNL-NEXT:    kshiftlw $15, %k7, %k7
 ; KNL-NEXT:    kshiftrw $9, %k7, %k7
 ; KNL-NEXT:    korw %k7, %k6, %k6
-; KNL-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k5 # 2-byte Reload
-; KNL-NEXT:    kandw %k5, %k6, %k6
+; KNL-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k7 # 2-byte Reload
+; KNL-NEXT:    kandw %k7, %k6, %k6
 ; KNL-NEXT:    movzbl {{[0-9]+}}(%rsp), %eax
 ; KNL-NEXT:    kmovw %eax, %k7
 ; KNL-NEXT:    kshiftlw $15, %k7, %k7
 ; KNL-NEXT:    kshiftrw $8, %k7, %k7
 ; KNL-NEXT:    korw %k7, %k6, %k6
-; KNL-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k5 # 2-byte Reload
-; KNL-NEXT:    kandw %k5, %k6, %k6
+; KNL-NEXT:    kandw %k3, %k6, %k6
 ; KNL-NEXT:    movzbl {{[0-9]+}}(%rsp), %eax
 ; KNL-NEXT:    kmovw %eax, %k7
 ; KNL-NEXT:    kshiftlw $15, %k7, %k7
 ; KNL-NEXT:    kshiftrw $7, %k7, %k7
 ; KNL-NEXT:    korw %k7, %k6, %k6
-; KNL-NEXT:    kandw %k3, %k6, %k6
+; KNL-NEXT:    kandw %k5, %k6, %k6
 ; KNL-NEXT:    movzbl {{[0-9]+}}(%rsp), %eax
 ; KNL-NEXT:    kmovw %eax, %k7
 ; KNL-NEXT:    kshiftlw $15, %k7, %k7
@@ -2282,20 +2281,20 @@ define <64 x i16> @test21(<64 x i16> %x , <64 x i1> %mask) nounwind readnone {
 ; KNL-NEXT:    kshiftlw $15, %k6, %k6
 ; KNL-NEXT:    kshiftrw $5, %k6, %k6
 ; KNL-NEXT:    korw %k6, %k5, %k5
-; KNL-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k3 # 2-byte Reload
-; KNL-NEXT:    kandw %k3, %k5, %k4
+; KNL-NEXT:    kandw %k0, %k5, %k4
 ; KNL-NEXT:    movzbl {{[0-9]+}}(%rsp), %eax
 ; KNL-NEXT:    kmovw %eax, %k5
 ; KNL-NEXT:    kshiftlw $15, %k5, %k5
 ; KNL-NEXT:    kshiftrw $4, %k5, %k5
 ; KNL-NEXT:    korw %k5, %k4, %k4
-; KNL-NEXT:    kandw %k0, %k4, %k3
+; KNL-NEXT:    kandw %k2, %k4, %k3
 ; KNL-NEXT:    movzbl {{[0-9]+}}(%rsp), %eax
 ; KNL-NEXT:    kmovw %eax, %k4
 ; KNL-NEXT:    kshiftlw $15, %k4, %k4
 ; KNL-NEXT:    kshiftrw $3, %k4, %k4
 ; KNL-NEXT:    korw %k4, %k3, %k3
-; KNL-NEXT:    kandw %k2, %k3, %k2
+; KNL-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k0 # 2-byte Reload
+; KNL-NEXT:    kandw %k0, %k3, %k2
 ; KNL-NEXT:    movzbl {{[0-9]+}}(%rsp), %eax
 ; KNL-NEXT:    kmovw %eax, %k3
 ; KNL-NEXT:    kshiftlw $15, %k3, %k3
@@ -2313,12 +2312,12 @@ define <64 x i16> @test21(<64 x i16> %x , <64 x i1> %mask) nounwind readnone {
 ; KNL-NEXT:    kmovw %eax, %k2
 ; KNL-NEXT:    kshiftlw $15, %k2, %k2
 ; KNL-NEXT:    korw %k2, %k0, %k2
-; KNL-NEXT:    vpternlogd $255, %zmm2, %zmm2, %zmm2 {%k2} {z}
-; KNL-NEXT:    vpternlogd $255, %zmm3, %zmm3, %zmm3 {%k1} {z}
+; KNL-NEXT:    vpternlogd {{.*#+}} zmm2 {%k2} {z} = -1
+; KNL-NEXT:    vpternlogd {{.*#+}} zmm3 {%k1} {z} = -1
 ; KNL-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k1 # 2-byte Reload
-; KNL-NEXT:    vpternlogd $255, %zmm4, %zmm4, %zmm4 {%k1} {z}
+; KNL-NEXT:    vpternlogd {{.*#+}} zmm4 {%k1} {z} = -1
 ; KNL-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k1 # 2-byte Reload
-; KNL-NEXT:    vpternlogd $255, %zmm5, %zmm5, %zmm5 {%k1} {z}
+; KNL-NEXT:    vpternlogd {{.*#+}} zmm5 {%k1} {z} = -1
 ; KNL-NEXT:    vpmovdw %zmm2, %ymm2
 ; KNL-NEXT:    vpmovdw %zmm3, %ymm3
 ; KNL-NEXT:    vinserti64x4 $1, %ymm3, %zmm2, %zmm2
@@ -2351,7 +2350,7 @@ define <64 x i16> @test21(<64 x i16> %x , <64 x i1> %mask) nounwind readnone {
 ; AVX512DQNOBW-NEXT:    movw $-5, %ax
 ; AVX512DQNOBW-NEXT:    kmovw %eax, %k1
 ; AVX512DQNOBW-NEXT:    kandw %k1, %k0, %k0
-; AVX512DQNOBW-NEXT:    kmovw %k1, %k7
+; AVX512DQNOBW-NEXT:    kmovw %k1, %k2
 ; AVX512DQNOBW-NEXT:    kmovw %k1, {{[-0-9]+}}(%r{{[sb]}}p) # 2-byte Spill
 ; AVX512DQNOBW-NEXT:    movzbl {{[0-9]+}}(%rsp), %eax
 ; AVX512DQNOBW-NEXT:    kmovw %eax, %k1
@@ -2360,8 +2359,8 @@ define <64 x i16> @test21(<64 x i16> %x , <64 x i1> %mask) nounwind readnone {
 ; AVX512DQNOBW-NEXT:    korw %k1, %k0, %k0
 ; AVX512DQNOBW-NEXT:    movw $-9, %ax
 ; AVX512DQNOBW-NEXT:    kmovw %eax, %k1
-; AVX512DQNOBW-NEXT:    kmovw %k1, {{[-0-9]+}}(%r{{[sb]}}p) # 2-byte Spill
 ; AVX512DQNOBW-NEXT:    kandw %k1, %k0, %k0
+; AVX512DQNOBW-NEXT:    kmovw %k1, %k7
 ; AVX512DQNOBW-NEXT:    movzbl {{[0-9]+}}(%rsp), %eax
 ; AVX512DQNOBW-NEXT:    kmovw %eax, %k1
 ; AVX512DQNOBW-NEXT:    kshiftlw $15, %k1, %k1
@@ -2378,9 +2377,8 @@ define <64 x i16> @test21(<64 x i16> %x , <64 x i1> %mask) nounwind readnone {
 ; AVX512DQNOBW-NEXT:    korw %k1, %k0, %k0
 ; AVX512DQNOBW-NEXT:    movw $-33, %ax
 ; AVX512DQNOBW-NEXT:    kmovw %eax, %k1
-; AVX512DQNOBW-NEXT:    kandw %k1, %k0, %k0
-; AVX512DQNOBW-NEXT:    kmovw %k1, %k2
 ; AVX512DQNOBW-NEXT:    kmovw %k1, {{[-0-9]+}}(%r{{[sb]}}p) # 2-byte Spill
+; AVX512DQNOBW-NEXT:    kandw %k1, %k0, %k0
 ; AVX512DQNOBW-NEXT:    movzbl {{[0-9]+}}(%rsp), %eax
 ; AVX512DQNOBW-NEXT:    kmovw %eax, %k1
 ; AVX512DQNOBW-NEXT:    kshiftlw $15, %k1, %k1
@@ -2388,8 +2386,9 @@ define <64 x i16> @test21(<64 x i16> %x , <64 x i1> %mask) nounwind readnone {
 ; AVX512DQNOBW-NEXT:    korw %k1, %k0, %k0
 ; AVX512DQNOBW-NEXT:    movw $-65, %ax
 ; AVX512DQNOBW-NEXT:    kmovw %eax, %k1
-; AVX512DQNOBW-NEXT:    kmovw %k1, {{[-0-9]+}}(%r{{[sb]}}p) # 2-byte Spill
 ; AVX512DQNOBW-NEXT:    kandw %k1, %k0, %k0
+; AVX512DQNOBW-NEXT:    kmovw %k1, %k3
+; AVX512DQNOBW-NEXT:    kmovw %k1, {{[-0-9]+}}(%r{{[sb]}}p) # 2-byte Spill
 ; AVX512DQNOBW-NEXT:    movzbl {{[0-9]+}}(%rsp), %eax
 ; AVX512DQNOBW-NEXT:    kmovw %eax, %k1
 ; AVX512DQNOBW-NEXT:    kshiftlw $15, %k1, %k1
@@ -2397,9 +2396,8 @@ define <64 x i16> @test21(<64 x i16> %x , <64 x i1> %mask) nounwind readnone {
 ; AVX512DQNOBW-NEXT:    korw %k1, %k0, %k0
 ; AVX512DQNOBW-NEXT:    movw $-129, %ax
 ; AVX512DQNOBW-NEXT:    kmovw %eax, %k1
-; AVX512DQNOBW-NEXT:    kandw %k1, %k0, %k0
-; AVX512DQNOBW-NEXT:    kmovw %k1, %k3
 ; AVX512DQNOBW-NEXT:    kmovw %k1, {{[-0-9]+}}(%r{{[sb]}}p) # 2-byte Spill
+; AVX512DQNOBW-NEXT:    kandw %k1, %k0, %k0
 ; AVX512DQNOBW-NEXT:    movzbl {{[0-9]+}}(%rsp), %eax
 ; AVX512DQNOBW-NEXT:    kmovw %eax, %k1
 ; AVX512DQNOBW-NEXT:    kshiftlw $15, %k1, %k1
@@ -2407,18 +2405,17 @@ define <64 x i16> @test21(<64 x i16> %x , <64 x i1> %mask) nounwind readnone {
 ; AVX512DQNOBW-NEXT:    korw %k1, %k0, %k0
 ; AVX512DQNOBW-NEXT:    movw $-257, %ax # imm = 0xFEFF
 ; AVX512DQNOBW-NEXT:    kmovw %eax, %k1
-; AVX512DQNOBW-NEXT:    kmovw %k1, {{[-0-9]+}}(%r{{[sb]}}p) # 2-byte Spill
 ; AVX512DQNOBW-NEXT:    kandw %k1, %k0, %k0
+; AVX512DQNOBW-NEXT:    kmovw %k1, %k4
+; AVX512DQNOBW-NEXT:    kmovw %k1, {{[-0-9]+}}(%r{{[sb]}}p) # 2-byte Spill
 ; AVX512DQNOBW-NEXT:    movzbl {{[0-9]+}}(%rsp), %eax
 ; AVX512DQNOBW-NEXT:    kmovw %eax, %k1
 ; AVX512DQNOBW-NEXT:    kshiftlw $15, %k1, %k1
 ; AVX512DQNOBW-NEXT:    kshiftrw $7, %k1, %k1
 ; AVX512DQNOBW-NEXT:    korw %k1, %k0, %k0
 ; AVX512DQNOBW-NEXT:    movw $-513, %ax # imm = 0xFDFF
-; AVX512DQNOBW-NEXT:    kmovw %eax, %k1
-; AVX512DQNOBW-NEXT:    kandw %k1, %k0, %k0
-; AVX512DQNOBW-NEXT:    kmovw %k1, %k4
-; AVX512DQNOBW-NEXT:    kmovw %k1, {{[-0-9]+}}(%r{{[sb]}}p) # 2-byte Spill
+; AVX512DQNOBW-NEXT:    kmovw %eax, %k5
+; AVX512DQNOBW-NEXT:    kandw %k5, %k0, %k0
 ; AVX512DQNOBW-NEXT:    movzbl {{[0-9]+}}(%rsp), %eax
 ; AVX512DQNOBW-NEXT:    kmovw %eax, %k1
 ; AVX512DQNOBW-NEXT:    kshiftlw $15, %k1, %k1
@@ -2434,8 +2431,9 @@ define <64 x i16> @test21(<64 x i16> %x , <64 x i1> %mask) nounwind readnone {
 ; AVX512DQNOBW-NEXT:    kshiftrw $5, %k1, %k1
 ; AVX512DQNOBW-NEXT:    korw %k1, %k0, %k0
 ; AVX512DQNOBW-NEXT:    movw $-2049, %ax # imm = 0xF7FF
-; AVX512DQNOBW-NEXT:    kmovw %eax, %k5
-; AVX512DQNOBW-NEXT:    kandw %k5, %k0, %k0
+; AVX512DQNOBW-NEXT:    kmovw %eax, %k1
+; AVX512DQNOBW-NEXT:    kmovw %k1, {{[-0-9]+}}(%r{{[sb]}}p) # 2-byte Spill
+; AVX512DQNOBW-NEXT:    kandw %k1, %k0, %k0
 ; AVX512DQNOBW-NEXT:    movzbl {{[0-9]+}}(%rsp), %eax
 ; AVX512DQNOBW-NEXT:    kmovw %eax, %k1
 ; AVX512DQNOBW-NEXT:    kshiftlw $15, %k1, %k1
@@ -2458,215 +2456,216 @@ define <64 x i16> @test21(<64 x i16> %x , <64 x i1> %mask) nounwind readnone {
 ; AVX512DQNOBW-NEXT:    kmovw %eax, %k1
 ; AVX512DQNOBW-NEXT:    kshiftlw $15, %k1, %k1
 ; AVX512DQNOBW-NEXT:    kshiftrw $2, %k1, %k1
-; AVX512DQNOBW-NEXT:    korw %k1, %k0, %k0
+; AVX512DQNOBW-NEXT:    korw %k1, %k0, %k1
 ; AVX512DQNOBW-NEXT:    movw $-16385, %ax # imm = 0xBFFF
-; AVX512DQNOBW-NEXT:    kmovw %eax, %k1
-; AVX512DQNOBW-NEXT:    kmovw %k1, {{[-0-9]+}}(%r{{[sb]}}p) # 2-byte Spill
-; AVX512DQNOBW-NEXT:    kandw %k1, %k0, %k0
+; AVX512DQNOBW-NEXT:    kmovw %eax, %k0
+; AVX512DQNOBW-NEXT:    kmovw %k0, {{[-0-9]+}}(%r{{[sb]}}p) # 2-byte Spill
+; AVX512DQNOBW-NEXT:    kandw %k0, %k1, %k1
 ; AVX512DQNOBW-NEXT:    movzbl {{[0-9]+}}(%rsp), %eax
 ; AVX512DQNOBW-NEXT:    kmovw %eax, %k6
 ; AVX512DQNOBW-NEXT:    kshiftlw $14, %k6, %k6
-; AVX512DQNOBW-NEXT:    korw %k6, %k0, %k0
-; AVX512DQNOBW-NEXT:    kshiftlw $1, %k0, %k0
-; AVX512DQNOBW-NEXT:    kshiftrw $1, %k0, %k0
+; AVX512DQNOBW-NEXT:    korw %k6, %k1, %k1
+; AVX512DQNOBW-NEXT:    kshiftlw $1, %k1, %k1
+; AVX512DQNOBW-NEXT:    kshiftrw $1, %k1, %k1
 ; AVX512DQNOBW-NEXT:    movzbl {{[0-9]+}}(%rsp), %eax
 ; AVX512DQNOBW-NEXT:    kmovw %eax, %k6
 ; AVX512DQNOBW-NEXT:    kshiftlw $15, %k6, %k6
-; AVX512DQNOBW-NEXT:    korw %k6, %k0, %k0
+; AVX512DQNOBW-NEXT:    korw %k6, %k1, %k0
 ; AVX512DQNOBW-NEXT:    kmovw %k0, {{[-0-9]+}}(%r{{[sb]}}p) # 2-byte Spill
 ; AVX512DQNOBW-NEXT:    andl $1, %edi
-; AVX512DQNOBW-NEXT:    kmovw %esi, %k0
-; AVX512DQNOBW-NEXT:    kshiftlw $15, %k0, %k0
-; AVX512DQNOBW-NEXT:    kshiftrw $14, %k0, %k0
+; AVX512DQNOBW-NEXT:    kmovw %esi, %k1
+; AVX512DQNOBW-NEXT:    kshiftlw $15, %k1, %k1
+; AVX512DQNOBW-NEXT:    kshiftrw $14, %k1, %k1
 ; AVX512DQNOBW-NEXT:    kmovw %edi, %k6
-; AVX512DQNOBW-NEXT:    korw %k0, %k6, %k0
-; AVX512DQNOBW-NEXT:    kandw %k7, %k0, %k0
+; AVX512DQNOBW-NEXT:    korw %k1, %k6, %k1
+; AVX512DQNOBW-NEXT:    kandw %k2, %k1, %k1
 ; AVX512DQNOBW-NEXT:    kmovw %edx, %k6
 ; AVX512DQNOBW-NEXT:    kshiftlw $15, %k6, %k6
 ; AVX512DQNOBW-NEXT:    kshiftrw $13, %k6, %k6
-; AVX512DQNOBW-NEXT:    korw %k6, %k0, %k0
-; AVX512DQNOBW-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k7 # 2-byte Reload
-; AVX512DQNOBW-NEXT:    kandw %k7, %k0, %k0
+; AVX512DQNOBW-NEXT:    korw %k6, %k1, %k1
+; AVX512DQNOBW-NEXT:    kmovw %k7, %k0
+; AVX512DQNOBW-NEXT:    kmovw %k7, {{[-0-9]+}}(%r{{[sb]}}p) # 2-byte Spill
+; AVX512DQNOBW-NEXT:    kandw %k7, %k1, %k1
 ; AVX512DQNOBW-NEXT:    kmovw %ecx, %k6
 ; AVX512DQNOBW-NEXT:    kshiftlw $15, %k6, %k6
 ; AVX512DQNOBW-NEXT:    kshiftrw $12, %k6, %k6
-; AVX512DQNOBW-NEXT:    korw %k6, %k0, %k0
-; AVX512DQNOBW-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k1 # 2-byte Reload
-; AVX512DQNOBW-NEXT:    kandw %k1, %k0, %k0
+; AVX512DQNOBW-NEXT:    korw %k6, %k1, %k1
+; AVX512DQNOBW-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k2 # 2-byte Reload
+; AVX512DQNOBW-NEXT:    kandw %k2, %k1, %k1
 ; AVX512DQNOBW-NEXT:    kmovw %r8d, %k6
 ; AVX512DQNOBW-NEXT:    kshiftlw $15, %k6, %k6
 ; AVX512DQNOBW-NEXT:    kshiftrw $11, %k6, %k6
-; AVX512DQNOBW-NEXT:    korw %k6, %k0, %k0
-; AVX512DQNOBW-NEXT:    kandw %k2, %k0, %k0
+; AVX512DQNOBW-NEXT:    korw %k6, %k1, %k1
+; AVX512DQNOBW-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k7 # 2-byte Reload
+; AVX512DQNOBW-NEXT:    kandw %k7, %k1, %k1
 ; AVX512DQNOBW-NEXT:    kmovw %r9d, %k6
 ; AVX512DQNOBW-NEXT:    kshiftlw $15, %k6, %k6
 ; AVX512DQNOBW-NEXT:    kshiftrw $10, %k6, %k6
-; AVX512DQNOBW-NEXT:    korw %k6, %k0, %k0
-; AVX512DQNOBW-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k2 # 2-byte Reload
-; AVX512DQNOBW-NEXT:    kandw %k2, %k0, %k0
+; AVX512DQNOBW-NEXT:    korw %k6, %k1, %k1
+; AVX512DQNOBW-NEXT:    kandw %k3, %k1, %k1
 ; AVX512DQNOBW-NEXT:    movzbl {{[0-9]+}}(%rsp), %eax
 ; AVX512DQNOBW-NEXT:    kmovw %eax, %k6
 ; AVX512DQNOBW-NEXT:    kshiftlw $15, %k6, %k6
 ; AVX512DQNOBW-NEXT:    kshiftrw $9, %k6, %k6
-; AVX512DQNOBW-NEXT:    korw %k6, %k0, %k0
-; AVX512DQNOBW-NEXT:    kandw %k3, %k0, %k0
+; AVX512DQNOBW-NEXT:    korw %k6, %k1, %k1
+; AVX512DQNOBW-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k3 # 2-byte Reload
+; AVX512DQNOBW-NEXT:    kandw %k3, %k1, %k1
 ; AVX512DQNOBW-NEXT:    movzbl {{[0-9]+}}(%rsp), %eax
 ; AVX512DQNOBW-NEXT:    kmovw %eax, %k6
 ; AVX512DQNOBW-NEXT:    kshiftlw $15, %k6, %k6
 ; AVX512DQNOBW-NEXT:    kshiftrw $8, %k6, %k6
-; AVX512DQNOBW-NEXT:    korw %k6, %k0, %k0
-; AVX512DQNOBW-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k3 # 2-byte Reload
-; AVX512DQNOBW-NEXT:    kandw %k3, %k0, %k0
+; AVX512DQNOBW-NEXT:    korw %k6, %k1, %k1
+; AVX512DQNOBW-NEXT:    kandw %k4, %k1, %k1
 ; AVX512DQNOBW-NEXT:    movzbl {{[0-9]+}}(%rsp), %eax
 ; AVX512DQNOBW-NEXT:    kmovw %eax, %k6
 ; AVX512DQNOBW-NEXT:    kshiftlw $15, %k6, %k6
 ; AVX512DQNOBW-NEXT:    kshiftrw $7, %k6, %k6
-; AVX512DQNOBW-NEXT:    korw %k6, %k0, %k0
-; AVX512DQNOBW-NEXT:    kandw %k4, %k0, %k0
+; AVX512DQNOBW-NEXT:    korw %k6, %k1, %k1
+; AVX512DQNOBW-NEXT:    kandw %k5, %k1, %k1
 ; AVX512DQNOBW-NEXT:    movzbl {{[0-9]+}}(%rsp), %eax
 ; AVX512DQNOBW-NEXT:    kmovw %eax, %k6
 ; AVX512DQNOBW-NEXT:    kshiftlw $15, %k6, %k6
 ; AVX512DQNOBW-NEXT:    kshiftrw $6, %k6, %k6
-; AVX512DQNOBW-NEXT:    korw %k6, %k0, %k0
+; AVX512DQNOBW-NEXT:    korw %k6, %k1, %k1
 ; AVX512DQNOBW-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k4 # 2-byte Reload
-; AVX512DQNOBW-NEXT:    kandw %k4, %k0, %k0
+; AVX512DQNOBW-NEXT:    kandw %k4, %k1, %k1
 ; AVX512DQNOBW-NEXT:    movzbl {{[0-9]+}}(%rsp), %eax
 ; AVX512DQNOBW-NEXT:    kmovw %eax, %k6
 ; AVX512DQNOBW-NEXT:    kshiftlw $15, %k6, %k6
 ; AVX512DQNOBW-NEXT:    kshiftrw $5, %k6, %k6
-; AVX512DQNOBW-NEXT:    korw %k6, %k0, %k0
-; AVX512DQNOBW-NEXT:    kmovw %k5, {{[-0-9]+}}(%r{{[sb]}}p) # 2-byte Spill
-; AVX512DQNOBW-NEXT:    kandw %k5, %k0, %k0
+; AVX512DQNOBW-NEXT:    korw %k6, %k1, %k1
+; AVX512DQNOBW-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k6 # 2-byte Reload
+; AVX512DQNOBW-NEXT:    kandw %k6, %k1, %k1
 ; AVX512DQNOBW-NEXT:    movzbl {{[0-9]+}}(%rsp), %eax
 ; AVX512DQNOBW-NEXT:    kmovw %eax, %k6
 ; AVX512DQNOBW-NEXT:    kshiftlw $15, %k6, %k6
 ; AVX512DQNOBW-NEXT:    kshiftrw $4, %k6, %k6
-; AVX512DQNOBW-NEXT:    korw %k6, %k0, %k0
+; AVX512DQNOBW-NEXT:    korw %k6, %k1, %k1
 ; AVX512DQNOBW-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k6 # 2-byte Reload
-; AVX512DQNOBW-NEXT:    kandw %k6, %k0, %k0
+; AVX512DQNOBW-NEXT:    kandw %k6, %k1, %k1
 ; AVX512DQNOBW-NEXT:    movzbl {{[0-9]+}}(%rsp), %eax
 ; AVX512DQNOBW-NEXT:    kmovw %eax, %k6
 ; AVX512DQNOBW-NEXT:    kshiftlw $15, %k6, %k6
 ; AVX512DQNOBW-NEXT:    kshiftrw $3, %k6, %k6
-; AVX512DQNOBW-NEXT:    korw %k6, %k0, %k0
+; AVX512DQNOBW-NEXT:    korw %k6, %k1, %k1
 ; AVX512DQNOBW-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k6 # 2-byte Reload
-; AVX512DQNOBW-NEXT:    kandw %k6, %k0, %k0
+; AVX512DQNOBW-NEXT:    kandw %k6, %k1, %k1
 ; AVX512DQNOBW-NEXT:    movzbl {{[0-9]+}}(%rsp), %eax
 ; AVX512DQNOBW-NEXT:    kmovw %eax, %k6
 ; AVX512DQNOBW-NEXT:    kshiftlw $15, %k6, %k6
 ; AVX512DQNOBW-NEXT:    kshiftrw $2, %k6, %k6
-; AVX512DQNOBW-NEXT:    korw %k6, %k0, %k0
+; AVX512DQNOBW-NEXT:    korw %k6, %k1, %k1
 ; AVX512DQNOBW-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k6 # 2-byte Reload
-; AVX512DQNOBW-NEXT:    kandw %k6, %k0, %k0
+; AVX512DQNOBW-NEXT:    kandw %k6, %k1, %k1
 ; AVX512DQNOBW-NEXT:    movzbl {{[0-9]+}}(%rsp), %eax
 ; AVX512DQNOBW-NEXT:    kmovw %eax, %k6
 ; AVX512DQNOBW-NEXT:    kshiftlw $14, %k6, %k6
-; AVX512DQNOBW-NEXT:    korw %k6, %k0, %k0
-; AVX512DQNOBW-NEXT:    kshiftlw $1, %k0, %k0
-; AVX512DQNOBW-NEXT:    kshiftrw $1, %k0, %k0
+; AVX512DQNOBW-NEXT:    korw %k6, %k1, %k1
+; AVX512DQNOBW-NEXT:    kshiftlw $1, %k1, %k1
+; AVX512DQNOBW-NEXT:    kshiftrw $1, %k1, %k1
 ; AVX512DQNOBW-NEXT:    movzbl {{[0-9]+}}(%rsp), %eax
 ; AVX512DQNOBW-NEXT:    kmovw %eax, %k6
 ; AVX512DQNOBW-NEXT:    kshiftlw $15, %k6, %k6
-; AVX512DQNOBW-NEXT:    korw %k6, %k0, %k0
-; AVX512DQNOBW-NEXT:    kmovw %k0, {{[-0-9]+}}(%r{{[sb]}}p) # 2-byte Spill
+; AVX512DQNOBW-NEXT:    korw %k6, %k1, %k1
+; AVX512DQNOBW-NEXT:    kmovw %k1, {{[-0-9]+}}(%r{{[sb]}}p) # 2-byte Spill
 ; AVX512DQNOBW-NEXT:    movzbl {{[0-9]+}}(%rsp), %eax
 ; AVX512DQNOBW-NEXT:    andl $1, %eax
 ; AVX512DQNOBW-NEXT:    movzbl {{[0-9]+}}(%rsp), %ecx
-; AVX512DQNOBW-NEXT:    kmovw %ecx, %k0
-; AVX512DQNOBW-NEXT:    kshiftlw $15, %k0, %k0
-; AVX512DQNOBW-NEXT:    kshiftrw $14, %k0, %k0
+; AVX512DQNOBW-NEXT:    kmovw %ecx, %k1
+; AVX512DQNOBW-NEXT:    kshiftlw $15, %k1, %k1
+; AVX512DQNOBW-NEXT:    kshiftrw $14, %k1, %k1
 ; AVX512DQNOBW-NEXT:    kmovw %eax, %k6
-; AVX512DQNOBW-NEXT:    korw %k0, %k6, %k0
+; AVX512DQNOBW-NEXT:    korw %k1, %k6, %k1
 ; AVX512DQNOBW-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k6 # 2-byte Reload
-; AVX512DQNOBW-NEXT:    kandw %k6, %k0, %k0
+; AVX512DQNOBW-NEXT:    kandw %k6, %k1, %k1
 ; AVX512DQNOBW-NEXT:    movzbl {{[0-9]+}}(%rsp), %eax
 ; AVX512DQNOBW-NEXT:    kmovw %eax, %k6
 ; AVX512DQNOBW-NEXT:    kshiftlw $15, %k6, %k6
 ; AVX512DQNOBW-NEXT:    kshiftrw $13, %k6, %k6
-; AVX512DQNOBW-NEXT:    korw %k6, %k0, %k0
-; AVX512DQNOBW-NEXT:    kandw %k7, %k0, %k0
+; AVX512DQNOBW-NEXT:    korw %k6, %k1, %k1
+; AVX512DQNOBW-NEXT:    kandw %k0, %k1, %k1
 ; AVX512DQNOBW-NEXT:    movzbl {{[0-9]+}}(%rsp), %eax
 ; AVX512DQNOBW-NEXT:    kmovw %eax, %k6
 ; AVX512DQNOBW-NEXT:    kshiftlw $15, %k6, %k6
 ; AVX512DQNOBW-NEXT:    kshiftrw $12, %k6, %k6
-; AVX512DQNOBW-NEXT:    korw %k6, %k0, %k0
-; AVX512DQNOBW-NEXT:    kandw %k1, %k0, %k0
+; AVX512DQNOBW-NEXT:    korw %k6, %k1, %k1
+; AVX512DQNOBW-NEXT:    kandw %k2, %k1, %k1
 ; AVX512DQNOBW-NEXT:    movzbl {{[0-9]+}}(%rsp), %eax
 ; AVX512DQNOBW-NEXT:    kmovw %eax, %k6
 ; AVX512DQNOBW-NEXT:    kshiftlw $15, %k6, %k6
 ; AVX512DQNOBW-NEXT:    kshiftrw $11, %k6, %k6
-; AVX512DQNOBW-NEXT:    korw %k6, %k0, %k0
-; AVX512DQNOBW-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k1 # 2-byte Reload
-; AVX512DQNOBW-NEXT:    kandw %k1, %k0, %k0
+; AVX512DQNOBW-NEXT:    korw %k6, %k1, %k1
+; AVX512DQNOBW-NEXT:    kandw %k7, %k1, %k1
 ; AVX512DQNOBW-NEXT:    movzbl {{[0-9]+}}(%rsp), %eax
 ; AVX512DQNOBW-NEXT:    kmovw %eax, %k6
 ; AVX512DQNOBW-NEXT:    kshiftlw $15, %k6, %k6
 ; AVX512DQNOBW-NEXT:    kshiftrw $10, %k6, %k6
-; AVX512DQNOBW-NEXT:    korw %k6, %k0, %k0
-; AVX512DQNOBW-NEXT:    kandw %k2, %k0, %k0
+; AVX512DQNOBW-NEXT:    korw %k6, %k1, %k1
+; AVX512DQNOBW-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k0 # 2-byte Reload
+; AVX512DQNOBW-NEXT:    kandw %k0, %k1, %k1
 ; AVX512DQNOBW-NEXT:    movzbl {{[0-9]+}}(%rsp), %eax
 ; AVX512DQNOBW-NEXT:    kmovw %eax, %k6
 ; AVX512DQNOBW-NEXT:    kshiftlw $15, %k6, %k6
 ; AVX512DQNOBW-NEXT:    kshiftrw $9, %k6, %k6
-; AVX512DQNOBW-NEXT:    korw %k6, %k0, %k0
-; AVX512DQNOBW-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k1 # 2-byte Reload
-; AVX512DQNOBW-NEXT:    kandw %k1, %k0, %k0
+; AVX512DQNOBW-NEXT:    korw %k6, %k1, %k1
+; AVX512DQNOBW-NEXT:    kandw %k3, %k1, %k1
 ; AVX512DQNOBW-NEXT:    movzbl {{[0-9]+}}(%rsp), %eax
 ; AVX512DQNOBW-NEXT:    kmovw %eax, %k6
 ; AVX512DQNOBW-NEXT:    kshiftlw $15, %k6, %k6
 ; AVX512DQNOBW-NEXT:    kshiftrw $8, %k6, %k6
-; AVX512DQNOBW-NEXT:    korw %k6, %k0, %k0
-; AVX512DQNOBW-NEXT:    kandw %k3, %k0, %k0
+; AVX512DQNOBW-NEXT:    korw %k6, %k1, %k1
+; AVX512DQNOBW-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k3 # 2-byte Reload
+; AVX512DQNOBW-NEXT:    kandw %k3, %k1, %k1
 ; AVX512DQNOBW-NEXT:    movzbl {{[0-9]+}}(%rsp), %eax
 ; AVX512DQNOBW-NEXT:    kmovw %eax, %k6
 ; AVX512DQNOBW-NEXT:    kshiftlw $15, %k6, %k6
 ; AVX512DQNOBW-NEXT:    kshiftrw $7, %k6, %k6
-; AVX512DQNOBW-NEXT:    korw %k6, %k0, %k0
-; AVX512DQNOBW-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k3 # 2-byte Reload
-; AVX512DQNOBW-NEXT:    kandw %k3, %k0, %k0
+; AVX512DQNOBW-NEXT:    korw %k6, %k1, %k1
+; AVX512DQNOBW-NEXT:    kandw %k5, %k1, %k1
 ; AVX512DQNOBW-NEXT:    movzbl {{[0-9]+}}(%rsp), %eax
 ; AVX512DQNOBW-NEXT:    kmovw %eax, %k6
 ; AVX512DQNOBW-NEXT:    kshiftlw $15, %k6, %k6
 ; AVX512DQNOBW-NEXT:    kshiftrw $6, %k6, %k6
-; AVX512DQNOBW-NEXT:    korw %k6, %k0, %k0
-; AVX512DQNOBW-NEXT:    kandw %k4, %k0, %k0
+; AVX512DQNOBW-NEXT:    korw %k6, %k1, %k1
+; AVX512DQNOBW-NEXT:    kandw %k4, %k1, %k1
 ; AVX512DQNOBW-NEXT:    movzbl {{[0-9]+}}(%rsp), %eax
 ; AVX512DQNOBW-NEXT:    kmovw %eax, %k6
 ; AVX512DQNOBW-NEXT:    kshiftlw $15, %k6, %k6
 ; AVX512DQNOBW-NEXT:    kshiftrw $5, %k6, %k6
-; AVX512DQNOBW-NEXT:    korw %k6, %k0, %k0
-; AVX512DQNOBW-NEXT:    kandw %k5, %k0, %k0
+; AVX512DQNOBW-NEXT:    korw %k6, %k1, %k1
+; AVX512DQNOBW-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k0 # 2-byte Reload
+; AVX512DQNOBW-NEXT:    kandw %k0, %k1, %k1
 ; AVX512DQNOBW-NEXT:    movzbl {{[0-9]+}}(%rsp), %eax
 ; AVX512DQNOBW-NEXT:    kmovw %eax, %k6
 ; AVX512DQNOBW-NEXT:    kshiftlw $15, %k6, %k6
 ; AVX512DQNOBW-NEXT:    kshiftrw $4, %k6, %k6
-; AVX512DQNOBW-NEXT:    korw %k6, %k0, %k0
-; AVX512DQNOBW-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k1 # 2-byte Reload
-; AVX512DQNOBW-NEXT:    kandw %k1, %k0, %k0
+; AVX512DQNOBW-NEXT:    korw %k6, %k1, %k1
+; AVX512DQNOBW-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k2 # 2-byte Reload
+; AVX512DQNOBW-NEXT:    kandw %k2, %k1, %k1
 ; AVX512DQNOBW-NEXT:    movzbl {{[0-9]+}}(%rsp), %eax
 ; AVX512DQNOBW-NEXT:    kmovw %eax, %k6
 ; AVX512DQNOBW-NEXT:    kshiftlw $15, %k6, %k6
 ; AVX512DQNOBW-NEXT:    kshiftrw $3, %k6, %k6
-; AVX512DQNOBW-NEXT:    korw %k6, %k0, %k0
-; AVX512DQNOBW-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k2 # 2-byte Reload
-; AVX512DQNOBW-NEXT:    kandw %k2, %k0, %k0
+; AVX512DQNOBW-NEXT:    korw %k6, %k1, %k1
+; AVX512DQNOBW-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k6 # 2-byte Reload
+; AVX512DQNOBW-NEXT:    kandw %k6, %k1, %k1
 ; AVX512DQNOBW-NEXT:    movzbl {{[0-9]+}}(%rsp), %eax
 ; AVX512DQNOBW-NEXT:    kmovw %eax, %k6
 ; AVX512DQNOBW-NEXT:    kshiftlw $15, %k6, %k6
 ; AVX512DQNOBW-NEXT:    kshiftrw $2, %k6, %k6
-; AVX512DQNOBW-NEXT:    korw %k6, %k0, %k0
-; AVX512DQNOBW-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k5 # 2-byte Reload
-; AVX512DQNOBW-NEXT:    kandw %k5, %k0, %k0
+; AVX512DQNOBW-NEXT:    korw %k6, %k1, %k1
+; AVX512DQNOBW-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k6 # 2-byte Reload
+; AVX512DQNOBW-NEXT:    kandw %k6, %k1, %k1
 ; AVX512DQNOBW-NEXT:    movzbl {{[0-9]+}}(%rsp), %eax
 ; AVX512DQNOBW-NEXT:    kmovw %eax, %k6
 ; AVX512DQNOBW-NEXT:    kshiftlw $14, %k6, %k6
-; AVX512DQNOBW-NEXT:    korw %k6, %k0, %k0
-; AVX512DQNOBW-NEXT:    kshiftlw $1, %k0, %k0
-; AVX512DQNOBW-NEXT:    kshiftrw $1, %k0, %k0
+; AVX512DQNOBW-NEXT:    korw %k6, %k1, %k1
+; AVX512DQNOBW-NEXT:    kshiftlw $1, %k1, %k1
+; AVX512DQNOBW-NEXT:    kshiftrw $1, %k1, %k1
 ; AVX512DQNOBW-NEXT:    movzbl {{[0-9]+}}(%rsp), %eax
 ; AVX512DQNOBW-NEXT:    kmovw %eax, %k6
 ; AVX512DQNOBW-NEXT:    kshiftlw $15, %k6, %k6
-; AVX512DQNOBW-NEXT:    korw %k6, %k0, %k0
+; AVX512DQNOBW-NEXT:    korw %k6, %k1, %k1
 ; AVX512DQNOBW-NEXT:    movzbl {{[0-9]+}}(%rsp), %eax
 ; AVX512DQNOBW-NEXT:    andl $1, %eax
 ; AVX512DQNOBW-NEXT:    movzbl {{[0-9]+}}(%rsp), %ecx
@@ -2675,56 +2674,55 @@ define <64 x i16> @test21(<64 x i16> %x , <64 x i1> %mask) nounwind readnone {
 ; AVX512DQNOBW-NEXT:    kshiftrw $14, %k6, %k6
 ; AVX512DQNOBW-NEXT:    kmovw %eax, %k7
 ; AVX512DQNOBW-NEXT:    korw %k6, %k7, %k6
-; AVX512DQNOBW-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k5 # 2-byte Reload
-; AVX512DQNOBW-NEXT:    kandw %k5, %k6, %k6
+; AVX512DQNOBW-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k7 # 2-byte Reload
+; AVX512DQNOBW-NEXT:    kandw %k7, %k6, %k6
 ; AVX512DQNOBW-NEXT:    movzbl {{[0-9]+}}(%rsp), %eax
 ; AVX512DQNOBW-NEXT:    kmovw %eax, %k7
 ; AVX512DQNOBW-NEXT:    kshiftlw $15, %k7, %k7
 ; AVX512DQNOBW-NEXT:    kshiftrw $13, %k7, %k7
 ; AVX512DQNOBW-NEXT:    korw %k7, %k6, %k6
-; AVX512DQNOBW-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k5 # 2-byte Reload
-; AVX512DQNOBW-NEXT:    kandw %k5, %k6, %k6
+; AVX512DQNOBW-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k7 # 2-byte Reload
+; AVX512DQNOBW-NEXT:    kandw %k7, %k6, %k6
 ; AVX512DQNOBW-NEXT:    movzbl {{[0-9]+}}(%rsp), %eax
 ; AVX512DQNOBW-NEXT:    kmovw %eax, %k7
 ; AVX512DQNOBW-NEXT:    kshiftlw $15, %k7, %k7
 ; AVX512DQNOBW-NEXT:    kshiftrw $12, %k7, %k7
 ; AVX512DQNOBW-NEXT:    korw %k7, %k6, %k6
-; AVX512DQNOBW-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k5 # 2-byte Reload
-; AVX512DQNOBW-NEXT:    kandw %k5, %k6, %k6
+; AVX512DQNOBW-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k7 # 2-byte Reload
+; AVX512DQNOBW-NEXT:    kandw %k7, %k6, %k6
 ; AVX512DQNOBW-NEXT:    movzbl {{[0-9]+}}(%rsp), %eax
 ; AVX512DQNOBW-NEXT:    kmovw %eax, %k7
 ; AVX512DQNOBW-NEXT:    kshiftlw $15, %k7, %k7
 ; AVX512DQNOBW-NEXT:    kshiftrw $11, %k7, %k7
 ; AVX512DQNOBW-NEXT:    korw %k7, %k6, %k6
-; AVX512DQNOBW-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k5 # 2-byte Reload
-; AVX512DQNOBW-NEXT:    kandw %k5, %k6, %k6
+; AVX512DQNOBW-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k7 # 2-byte Reload
+; AVX512DQNOBW-NEXT:    kandw %k7, %k6, %k6
 ; AVX512DQNOBW-NEXT:    movzbl {{[0-9]+}}(%rsp), %eax
 ; AVX512DQNOBW-NEXT:    kmovw %eax, %k7
 ; AVX512DQNOBW-NEXT:    kshiftlw $15, %k7, %k7
 ; AVX512DQNOBW-NEXT:    kshiftrw $10, %k7, %k7
 ; AVX512DQNOBW-NEXT:    korw %k7, %k6, %k6
-; AVX512DQNOBW-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k5 # 2-byte Reload
-; AVX512DQNOBW-NEXT:    kandw %k5, %k6, %k6
+; AVX512DQNOBW-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k7 # 2-byte Reload
+; AVX512DQNOBW-NEXT:    kandw %k7, %k6, %k6
 ; AVX512DQNOBW-NEXT:    movzbl {{[0-9]+}}(%rsp), %eax
 ; AVX512DQNOBW-NEXT:    kmovw %eax, %k7
 ; AVX512DQNOBW-NEXT:    kshiftlw $15, %k7, %k7
 ; AVX512DQNOBW-NEXT:    kshiftrw $9, %k7, %k7
 ; AVX512DQNOBW-NEXT:    korw %k7, %k6, %k6
-; AVX512DQNOBW-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k5 # 2-byte Reload
-; AVX512DQNOBW-NEXT:    kandw %k5, %k6, %k6
+; AVX512DQNOBW-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k7 # 2-byte Reload
+; AVX512DQNOBW-NEXT:    kandw %k7, %k6, %k6
 ; AVX512DQNOBW-NEXT:    movzbl {{[0-9]+}}(%rsp), %eax
 ; AVX512DQNOBW-NEXT:    kmovw %eax, %k7
 ; AVX512DQNOBW-NEXT:    kshiftlw $15, %k7, %k7
 ; AVX512DQNOBW-NEXT:    kshiftrw $8, %k7, %k7
 ; AVX512DQNOBW-NEXT:    korw %k7, %k6, %k6
-; AVX512DQNOBW-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k5 # 2-byte Reload
-; AVX512DQNOBW-NEXT:    kandw %k5, %k6, %k6
+; AVX512DQNOBW-NEXT:    kandw %k3, %k6, %k6
 ; AVX512DQNOBW-NEXT:    movzbl {{[0-9]+}}(%rsp), %eax
 ; AVX512DQNOBW-NEXT:    kmovw %eax, %k7
 ; AVX512DQNOBW-NEXT:    kshiftlw $15, %k7, %k7
 ; AVX512DQNOBW-NEXT:    kshiftrw $7, %k7, %k7
 ; AVX512DQNOBW-NEXT:    korw %k7, %k6, %k6
-; AVX512DQNOBW-NEXT:    kandw %k3, %k6, %k6
+; AVX512DQNOBW-NEXT:    kandw %k5, %k6, %k6
 ; AVX512DQNOBW-NEXT:    movzbl {{[0-9]+}}(%rsp), %eax
 ; AVX512DQNOBW-NEXT:    kmovw %eax, %k7
 ; AVX512DQNOBW-NEXT:    kshiftlw $15, %k7, %k7
@@ -2736,39 +2734,39 @@ define <64 x i16> @test21(<64 x i16> %x , <64 x i1> %mask) nounwind readnone {
 ; AVX512DQNOBW-NEXT:    kshiftlw $15, %k6, %k6
 ; AVX512DQNOBW-NEXT:    kshiftrw $5, %k6, %k6
 ; AVX512DQNOBW-NEXT:    korw %k6, %k5, %k5
-; AVX512DQNOBW-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k3 # 2-byte Reload
-; AVX512DQNOBW-NEXT:    kandw %k3, %k5, %k4
+; AVX512DQNOBW-NEXT:    kandw %k0, %k5, %k4
 ; AVX512DQNOBW-NEXT:    movzbl {{[0-9]+}}(%rsp), %eax
 ; AVX512DQNOBW-NEXT:    kmovw %eax, %k5
 ; AVX512DQNOBW-NEXT:    kshiftlw $15, %k5, %k5
 ; AVX512DQNOBW-NEXT:    kshiftrw $4, %k5, %k5
 ; AVX512DQNOBW-NEXT:    korw %k5, %k4, %k4
-; AVX512DQNOBW-NEXT:    kandw %k1, %k4, %k3
+; AVX512DQNOBW-NEXT:    kandw %k2, %k4, %k3
 ; AVX512DQNOBW-NEXT:    movzbl {{[0-9]+}}(%rsp), %eax
 ; AVX512DQNOBW-NEXT:    kmovw %eax, %k4
 ; AVX512DQNOBW-NEXT:    kshiftlw $15, %k4, %k4
 ; AVX512DQNOBW-NEXT:    kshiftrw $3, %k4, %k4
 ; AVX512DQNOBW-NEXT:    korw %k4, %k3, %k3
-; AVX512DQNOBW-NEXT:    kandw %k2, %k3, %k2
+; AVX512DQNOBW-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k0 # 2-byte Reload
+; AVX512DQNOBW-NEXT:    kandw %k0, %k3, %k2
 ; AVX512DQNOBW-NEXT:    movzbl {{[0-9]+}}(%rsp), %eax
 ; AVX512DQNOBW-NEXT:    kmovw %eax, %k3
 ; AVX512DQNOBW-NEXT:    kshiftlw $15, %k3, %k3
 ; AVX512DQNOBW-NEXT:    kshiftrw $2, %k3, %k3
 ; AVX512DQNOBW-NEXT:    korw %k3, %k2, %k2
-; AVX512DQNOBW-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k1 # 2-byte Reload
-; AVX512DQNOBW-NEXT:    kandw %k1, %k2, %k1
+; AVX512DQNOBW-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k0 # 2-byte Reload
+; AVX512DQNOBW-NEXT:    kandw %k0, %k2, %k0
 ; AVX512DQNOBW-NEXT:    movzbl {{[0-9]+}}(%rsp), %eax
 ; AVX512DQNOBW-NEXT:    kmovw %eax, %k2
 ; AVX512DQNOBW-NEXT:    kshiftlw $14, %k2, %k2
-; AVX512DQNOBW-NEXT:    korw %k2, %k1, %k1
-; AVX512DQNOBW-NEXT:    kshiftlw $1, %k1, %k1
-; AVX512DQNOBW-NEXT:    kshiftrw $1, %k1, %k1
+; AVX512DQNOBW-NEXT:    korw %k2, %k0, %k0
+; AVX512DQNOBW-NEXT:    kshiftlw $1, %k0, %k0
+; AVX512DQNOBW-NEXT:    kshiftrw $1, %k0, %k0
 ; AVX512DQNOBW-NEXT:    movzbl {{[0-9]+}}(%rsp), %eax
 ; AVX512DQNOBW-NEXT:    kmovw %eax, %k2
 ; AVX512DQNOBW-NEXT:    kshiftlw $15, %k2, %k2
-; AVX512DQNOBW-NEXT:    korw %k2, %k1, %k1
-; AVX512DQNOBW-NEXT:    vpmovm2d %k1, %zmm2
-; AVX512DQNOBW-NEXT:    vpmovm2d %k0, %zmm3
+; AVX512DQNOBW-NEXT:    korw %k2, %k0, %k0
+; AVX512DQNOBW-NEXT:    vpmovm2d %k0, %zmm2
+; AVX512DQNOBW-NEXT:    vpmovm2d %k1, %zmm3
 ; AVX512DQNOBW-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k0 # 2-byte Reload
 ; AVX512DQNOBW-NEXT:    vpmovm2d %k0, %zmm4
 ; AVX512DQNOBW-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k0 # 2-byte Reload
diff --git a/llvm/test/CodeGen/X86/avx512-extract-subvector-load-store.ll b/llvm/test/CodeGen/X86/avx512-extract-subvector-load-store.ll
index 6c661eb771d1b..5115c3cdc259a 100644
--- a/llvm/test/CodeGen/X86/avx512-extract-subvector-load-store.ll
+++ b/llvm/test/CodeGen/X86/avx512-extract-subvector-load-store.ll
@@ -485,7 +485,7 @@ define void @load_v64i1_broadcast_32_v16i1(ptr %a0,<16 x float> %a1,<16 x float>
 ; AVX512NOTDQ-LABEL: load_v64i1_broadcast_32_v16i1:
 ; AVX512NOTDQ:       # %bb.0:
 ; AVX512NOTDQ-NEXT:    kmovw 4(%rdi), %k1
-; AVX512NOTDQ-NEXT:    vpternlogd $255, %zmm2, %zmm2, %zmm2 {%k1} {z}
+; AVX512NOTDQ-NEXT:    vpternlogd {{.*#+}} zmm2 {%k1} {z} = -1
 ; AVX512NOTDQ-NEXT:    vpbroadcastd %xmm2, %zmm2
 ; AVX512NOTDQ-NEXT:    vptestmd %zmm2, %zmm2, %k1
 ; AVX512NOTDQ-NEXT:    vmovaps %zmm0, %zmm1 {%k1}
@@ -642,7 +642,7 @@ define void @load_v64i1_broadcast_63_v16i1(ptr %a0,<16 x float> %a1,<16 x float>
 ; AVX512NOTDQ-FAST-LABEL: load_v64i1_broadcast_63_v16i1:
 ; AVX512NOTDQ-FAST:       # %bb.0:
 ; AVX512NOTDQ-FAST-NEXT:    kmovw 6(%rdi), %k1
-; AVX512NOTDQ-FAST-NEXT:    vpternlogd $255, %zmm2, %zmm2, %zmm2 {%k1} {z}
+; AVX512NOTDQ-FAST-NEXT:    vpternlogd {{.*#+}} zmm2 {%k1} {z} = -1
 ; AVX512NOTDQ-FAST-NEXT:    vpbroadcastd {{.*#+}} zmm3 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
 ; AVX512NOTDQ-FAST-NEXT:    vpermd %zmm2, %zmm3, %zmm2
 ; AVX512NOTDQ-FAST-NEXT:    vptestmd %zmm2, %zmm2, %k1
@@ -654,7 +654,7 @@ define void @load_v64i1_broadcast_63_v16i1(ptr %a0,<16 x float> %a1,<16 x float>
 ; AVX512NOTDQ-FAST-PERLANE-LABEL: load_v64i1_broadcast_63_v16i1:
 ; AVX512NOTDQ-FAST-PERLANE:       # %bb.0:
 ; AVX512NOTDQ-FAST-PERLANE-NEXT:    kmovw 6(%rdi), %k1
-; AVX512NOTDQ-FAST-PERLANE-NEXT:    vpternlogd $255, %zmm2, %zmm2, %zmm2 {%k1} {z}
+; AVX512NOTDQ-FAST-PERLANE-NEXT:    vpternlogd {{.*#+}} zmm2 {%k1} {z} = -1
 ; AVX512NOTDQ-FAST-PERLANE-NEXT:    vpshufd {{.*#+}} zmm2 = zmm2[3,3,3,3,7,7,7,7,11,11,11,11,15,15,15,15]
 ; AVX512NOTDQ-FAST-PERLANE-NEXT:    vshufi64x2 {{.*#+}} zmm2 = zmm2[6,7,6,7,6,7,6,7]
 ; AVX512NOTDQ-FAST-PERLANE-NEXT:    vptestmd %zmm2, %zmm2, %k1
@@ -1426,7 +1426,7 @@ define void @load_v64i1_broadcast_32_v16i1_store(ptr %a0,ptr %a1) {
 ; AVX512NOTDQ-LABEL: load_v64i1_broadcast_32_v16i1_store:
 ; AVX512NOTDQ:       # %bb.0:
 ; AVX512NOTDQ-NEXT:    kmovw 4(%rdi), %k1
-; AVX512NOTDQ-NEXT:    vpternlogd $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
+; AVX512NOTDQ-NEXT:    vpternlogd {{.*#+}} zmm0 {%k1} {z} = -1
 ; AVX512NOTDQ-NEXT:    vpbroadcastd %xmm0, %zmm0
 ; AVX512NOTDQ-NEXT:    vptestmd %zmm0, %zmm0, %k0
 ; AVX512NOTDQ-NEXT:    kmovw %k0, (%rsi)
@@ -1596,7 +1596,7 @@ define void @load_v64i1_broadcast_63_v16i1_store(ptr %a0,ptr %a1) {
 ; AVX512NOTDQ-FAST-LABEL: load_v64i1_broadcast_63_v16i1_store:
 ; AVX512NOTDQ-FAST:       # %bb.0:
 ; AVX512NOTDQ-FAST-NEXT:    kmovw 6(%rdi), %k1
-; AVX512NOTDQ-FAST-NEXT:    vpternlogd $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
+; AVX512NOTDQ-FAST-NEXT:    vpternlogd {{.*#+}} zmm0 {%k1} {z} = -1
 ; AVX512NOTDQ-FAST-NEXT:    vpbroadcastd {{.*#+}} zmm1 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
 ; AVX512NOTDQ-FAST-NEXT:    vpermd %zmm0, %zmm1, %zmm0
 ; AVX512NOTDQ-FAST-NEXT:    vptestmd %zmm0, %zmm0, %k0
@@ -1607,7 +1607,7 @@ define void @load_v64i1_broadcast_63_v16i1_store(ptr %a0,ptr %a1) {
 ; AVX512NOTDQ-FAST-PERLANE-LABEL: load_v64i1_broadcast_63_v16i1_store:
 ; AVX512NOTDQ-FAST-PERLANE:       # %bb.0:
 ; AVX512NOTDQ-FAST-PERLANE-NEXT:    kmovw 6(%rdi), %k1
-; AVX512NOTDQ-FAST-PERLANE-NEXT:    vpternlogd $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
+; AVX512NOTDQ-FAST-PERLANE-NEXT:    vpternlogd {{.*#+}} zmm0 {%k1} {z} = -1
 ; AVX512NOTDQ-FAST-PERLANE-NEXT:    vpshufd {{.*#+}} zmm0 = zmm0[3,3,3,3,7,7,7,7,11,11,11,11,15,15,15,15]
 ; AVX512NOTDQ-FAST-PERLANE-NEXT:    vshufi64x2 {{.*#+}} zmm0 = zmm0[6,7,6,7,6,7,6,7]
 ; AVX512NOTDQ-FAST-PERLANE-NEXT:    vptestmd %zmm0, %zmm0, %k0
diff --git a/llvm/test/CodeGen/X86/avx512-intel-ocl.ll b/llvm/test/CodeGen/X86/avx512-intel-ocl.ll
index 6c68279b8d04a..eb8bff26f3b77 100644
--- a/llvm/test/CodeGen/X86/avx512-intel-ocl.ll
+++ b/llvm/test/CodeGen/X86/avx512-intel-ocl.ll
@@ -429,7 +429,9 @@ define <16 x float> @testf16_inp_mask(<16 x float> %a, i16 %mask)  {
 ; WIN64-KNL-NEXT:    kmovw %edx, %k1
 ; WIN64-KNL-NEXT:    callq func_float16_mask
 ; WIN64-KNL-NEXT:    nop
+; WIN64-KNL-NEXT:    .seh_startepilogue
 ; WIN64-KNL-NEXT:    addq $40, %rsp
+; WIN64-KNL-NEXT:    .seh_endepilogue
 ; WIN64-KNL-NEXT:    retq
 ; WIN64-KNL-NEXT:    .seh_endproc
 ;
@@ -443,7 +445,9 @@ define <16 x float> @testf16_inp_mask(<16 x float> %a, i16 %mask)  {
 ; WIN64-SKX-NEXT:    kmovd %edx, %k1
 ; WIN64-SKX-NEXT:    callq func_float16_mask
 ; WIN64-SKX-NEXT:    nop
+; WIN64-SKX-NEXT:    .seh_startepilogue
 ; WIN64-SKX-NEXT:    addq $40, %rsp
+; WIN64-SKX-NEXT:    .seh_endepilogue
 ; WIN64-SKX-NEXT:    retq
 ; WIN64-SKX-NEXT:    .seh_endproc
 ;
diff --git a/llvm/test/CodeGen/X86/avx512-load-store.ll b/llvm/test/CodeGen/X86/avx512-load-store.ll
index c32c3d9b85503..ce6bfa90d88a7 100644
--- a/llvm/test/CodeGen/X86/avx512-load-store.ll
+++ b/llvm/test/CodeGen/X86/avx512-load-store.ll
@@ -143,7 +143,7 @@ define <4 x float> @test_mm_mask_load_ss(<4 x float> %__A, i8 zeroext %__U, ptr
 ; CHECK64-LABEL: test_mm_mask_load_ss:
 ; CHECK64:       # %bb.0: # %entry
 ; CHECK64-NEXT:    kmovw %edi, %k1
-; CHECK64-NEXT:    vmovss (%rsi), %xmm0 {%k1}
+; CHECK64-NEXT:    vmovss {{.*#+}} xmm0 {%k1} = mem[0],zero,zero,zero
 ; CHECK64-NEXT:    retq
 ;
 ; CHECK32-LABEL: test_mm_mask_load_ss:
@@ -151,7 +151,7 @@ define <4 x float> @test_mm_mask_load_ss(<4 x float> %__A, i8 zeroext %__U, ptr
 ; CHECK32-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; CHECK32-NEXT:    movzbl {{[0-9]+}}(%esp), %ecx
 ; CHECK32-NEXT:    kmovw %ecx, %k1
-; CHECK32-NEXT:    vmovss (%eax), %xmm0 {%k1}
+; CHECK32-NEXT:    vmovss {{.*#+}} xmm0 {%k1} = mem[0],zero,zero,zero
 ; CHECK32-NEXT:    retl
 entry:
   %shuffle.i = shufflevector <4 x float> %__A, <4 x float> <float 0.000000e+00, float undef, float undef, float undef>, <4 x i32> <i32 0, i32 4, i32 4, i32 4>
@@ -168,7 +168,7 @@ define <2 x double> @test_mm_mask_load_sd(<2 x double> %__A, i8 zeroext %__U, pt
 ; CHECK64-LABEL: test_mm_mask_load_sd:
 ; CHECK64:       # %bb.0: # %entry
 ; CHECK64-NEXT:    kmovw %edi, %k1
-; CHECK64-NEXT:    vmovsd (%rsi), %xmm0 {%k1}
+; CHECK64-NEXT:    vmovsd {{.*#+}} xmm0 {%k1} = mem[0],zero
 ; CHECK64-NEXT:    retq
 ;
 ; CHECK32-LABEL: test_mm_mask_load_sd:
@@ -176,7 +176,7 @@ define <2 x double> @test_mm_mask_load_sd(<2 x double> %__A, i8 zeroext %__U, pt
 ; CHECK32-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; CHECK32-NEXT:    movzbl {{[0-9]+}}(%esp), %ecx
 ; CHECK32-NEXT:    kmovw %ecx, %k1
-; CHECK32-NEXT:    vmovsd (%eax), %xmm0 {%k1}
+; CHECK32-NEXT:    vmovsd {{.*#+}} xmm0 {%k1} = mem[0],zero
 ; CHECK32-NEXT:    retl
 entry:
   %shuffle5.i = insertelement <2 x double> %__A, double 0.000000e+00, i32 1
@@ -192,7 +192,7 @@ define <4 x float> @test_mm_maskz_load_ss(i8 zeroext %__U, ptr %__W) local_unnam
 ; CHECK64-LABEL: test_mm_maskz_load_ss:
 ; CHECK64:       # %bb.0: # %entry
 ; CHECK64-NEXT:    kmovw %edi, %k1
-; CHECK64-NEXT:    vmovss (%rsi), %xmm0 {%k1} {z}
+; CHECK64-NEXT:    vmovss {{.*#+}} xmm0 {%k1} {z} = mem[0],zero,zero,zero
 ; CHECK64-NEXT:    retq
 ;
 ; CHECK32-LABEL: test_mm_maskz_load_ss:
@@ -200,7 +200,7 @@ define <4 x float> @test_mm_maskz_load_ss(i8 zeroext %__U, ptr %__W) local_unnam
 ; CHECK32-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; CHECK32-NEXT:    movzbl {{[0-9]+}}(%esp), %ecx
 ; CHECK32-NEXT:    kmovw %ecx, %k1
-; CHECK32-NEXT:    vmovss (%eax), %xmm0 {%k1} {z}
+; CHECK32-NEXT:    vmovss {{.*#+}} xmm0 {%k1} {z} = mem[0],zero,zero,zero
 ; CHECK32-NEXT:    retl
 entry:
   %0 = and i8 %__U, 1
@@ -215,7 +215,7 @@ define <2 x double> @test_mm_maskz_load_sd(i8 zeroext %__U, ptr %__W) local_unna
 ; CHECK64-LABEL: test_mm_maskz_load_sd:
 ; CHECK64:       # %bb.0: # %entry
 ; CHECK64-NEXT:    kmovw %edi, %k1
-; CHECK64-NEXT:    vmovsd (%rsi), %xmm0 {%k1} {z}
+; CHECK64-NEXT:    vmovsd {{.*#+}} xmm0 {%k1} {z} = mem[0],zero
 ; CHECK64-NEXT:    retq
 ;
 ; CHECK32-LABEL: test_mm_maskz_load_sd:
@@ -223,7 +223,7 @@ define <2 x double> @test_mm_maskz_load_sd(i8 zeroext %__U, ptr %__W) local_unna
 ; CHECK32-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; CHECK32-NEXT:    movzbl {{[0-9]+}}(%esp), %ecx
 ; CHECK32-NEXT:    kmovw %ecx, %k1
-; CHECK32-NEXT:    vmovsd (%eax), %xmm0 {%k1} {z}
+; CHECK32-NEXT:    vmovsd {{.*#+}} xmm0 {%k1} {z} = mem[0],zero
 ; CHECK32-NEXT:    retl
 entry:
   %0 = and i8 %__U, 1
@@ -283,7 +283,7 @@ define <4 x float> @test_mm_mask_load_ss_2(<4 x float> %__A, i8 zeroext %__U, pt
 ; CHECK64-LABEL: test_mm_mask_load_ss_2:
 ; CHECK64:       # %bb.0: # %entry
 ; CHECK64-NEXT:    kmovw %edi, %k1
-; CHECK64-NEXT:    vmovss (%rsi), %xmm0 {%k1}
+; CHECK64-NEXT:    vmovss {{.*#+}} xmm0 {%k1} = mem[0],zero,zero,zero
 ; CHECK64-NEXT:    retq
 ;
 ; CHECK32-LABEL: test_mm_mask_load_ss_2:
@@ -291,7 +291,7 @@ define <4 x float> @test_mm_mask_load_ss_2(<4 x float> %__A, i8 zeroext %__U, pt
 ; CHECK32-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; CHECK32-NEXT:    movzbl {{[0-9]+}}(%esp), %ecx
 ; CHECK32-NEXT:    kmovw %ecx, %k1
-; CHECK32-NEXT:    vmovss (%eax), %xmm0 {%k1}
+; CHECK32-NEXT:    vmovss {{.*#+}} xmm0 {%k1} = mem[0],zero,zero,zero
 ; CHECK32-NEXT:    retl
 entry:
   %shuffle.i = shufflevector <4 x float> %__A, <4 x float> <float 0.000000e+00, float undef, float undef, float undef>, <4 x i32> <i32 0, i32 4, i32 4, i32 4>
@@ -306,7 +306,7 @@ define <4 x float> @test_mm_maskz_load_ss_2(i8 zeroext %__U, ptr readonly %__W)
 ; CHECK64-LABEL: test_mm_maskz_load_ss_2:
 ; CHECK64:       # %bb.0: # %entry
 ; CHECK64-NEXT:    kmovw %edi, %k1
-; CHECK64-NEXT:    vmovss (%rsi), %xmm0 {%k1} {z}
+; CHECK64-NEXT:    vmovss {{.*#+}} xmm0 {%k1} {z} = mem[0],zero,zero,zero
 ; CHECK64-NEXT:    retq
 ;
 ; CHECK32-LABEL: test_mm_maskz_load_ss_2:
@@ -314,7 +314,7 @@ define <4 x float> @test_mm_maskz_load_ss_2(i8 zeroext %__U, ptr readonly %__W)
 ; CHECK32-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; CHECK32-NEXT:    movzbl {{[0-9]+}}(%esp), %ecx
 ; CHECK32-NEXT:    kmovw %ecx, %k1
-; CHECK32-NEXT:    vmovss (%eax), %xmm0 {%k1} {z}
+; CHECK32-NEXT:    vmovss {{.*#+}} xmm0 {%k1} {z} = mem[0],zero,zero,zero
 ; CHECK32-NEXT:    retl
 entry:
   %0 = and i8 %__U, 1
@@ -328,7 +328,7 @@ define <2 x double> @test_mm_mask_load_sd_2(<2 x double> %__A, i8 zeroext %__U,
 ; CHECK64-LABEL: test_mm_mask_load_sd_2:
 ; CHECK64:       # %bb.0: # %entry
 ; CHECK64-NEXT:    kmovw %edi, %k1
-; CHECK64-NEXT:    vmovsd (%rsi), %xmm0 {%k1}
+; CHECK64-NEXT:    vmovsd {{.*#+}} xmm0 {%k1} = mem[0],zero
 ; CHECK64-NEXT:    retq
 ;
 ; CHECK32-LABEL: test_mm_mask_load_sd_2:
@@ -336,7 +336,7 @@ define <2 x double> @test_mm_mask_load_sd_2(<2 x double> %__A, i8 zeroext %__U,
 ; CHECK32-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; CHECK32-NEXT:    movzbl {{[0-9]+}}(%esp), %ecx
 ; CHECK32-NEXT:    kmovw %ecx, %k1
-; CHECK32-NEXT:    vmovsd (%eax), %xmm0 {%k1}
+; CHECK32-NEXT:    vmovsd {{.*#+}} xmm0 {%k1} = mem[0],zero
 ; CHECK32-NEXT:    retl
 entry:
   %shuffle3.i = insertelement <2 x double> %__A, double 0.000000e+00, i32 1
@@ -351,7 +351,7 @@ define <2 x double> @test_mm_maskz_load_sd_2(i8 zeroext %__U, ptr readonly %__W)
 ; CHECK64-LABEL: test_mm_maskz_load_sd_2:
 ; CHECK64:       # %bb.0: # %entry
 ; CHECK64-NEXT:    kmovw %edi, %k1
-; CHECK64-NEXT:    vmovsd (%rsi), %xmm0 {%k1} {z}
+; CHECK64-NEXT:    vmovsd {{.*#+}} xmm0 {%k1} {z} = mem[0],zero
 ; CHECK64-NEXT:    retq
 ;
 ; CHECK32-LABEL: test_mm_maskz_load_sd_2:
@@ -359,7 +359,7 @@ define <2 x double> @test_mm_maskz_load_sd_2(i8 zeroext %__U, ptr readonly %__W)
 ; CHECK32-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; CHECK32-NEXT:    movzbl {{[0-9]+}}(%esp), %ecx
 ; CHECK32-NEXT:    kmovw %ecx, %k1
-; CHECK32-NEXT:    vmovsd (%eax), %xmm0 {%k1} {z}
+; CHECK32-NEXT:    vmovsd {{.*#+}} xmm0 {%k1} {z} = mem[0],zero
 ; CHECK32-NEXT:    retl
 entry:
   %0 = and i8 %__U, 1
diff --git a/llvm/test/CodeGen/X86/avx512-logic.ll b/llvm/test/CodeGen/X86/avx512-logic.ll
index e53e194ba05c2..23f4fcb1c77c6 100644
--- a/llvm/test/CodeGen/X86/avx512-logic.ll
+++ b/llvm/test/CodeGen/X86/avx512-logic.ll
@@ -856,7 +856,7 @@ entry:
 define <16 x i32> @ternlog_and_andn(<16 x i32> %x, <16 x i32> %y, <16 x i32> %z) {
 ; ALL-LABEL: ternlog_and_andn:
 ; ALL:       ## %bb.0:
-; ALL-NEXT:    vpternlogd $8, %zmm1, %zmm2, %zmm0
+; ALL-NEXT:    vpternlogd {{.*#+}} zmm0 = zmm2 & zmm1 & ~zmm0
 ; ALL-NEXT:    retq
   %a = xor <16 x i32> %x, <i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1>
   %b = and <16 x i32> %y, %a
@@ -867,7 +867,7 @@ define <16 x i32> @ternlog_and_andn(<16 x i32> %x, <16 x i32> %y, <16 x i32> %z)
 define <16 x i32> @ternlog_or_andn(<16 x i32> %x, <16 x i32> %y, <16 x i32> %z) {
 ; ALL-LABEL: ternlog_or_andn:
 ; ALL:       ## %bb.0:
-; ALL-NEXT:    vpternlogd $206, %zmm1, %zmm2, %zmm0
+; ALL-NEXT:    vpternlogd {{.*#+}} zmm0 = (zmm1 & ~zmm0) | zmm2
 ; ALL-NEXT:    retq
   %a = xor <16 x i32> %x, <i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1>
   %b = and <16 x i32> %y, %a
@@ -878,7 +878,7 @@ define <16 x i32> @ternlog_or_andn(<16 x i32> %x, <16 x i32> %y, <16 x i32> %z)
 define <16 x i32> @ternlog_xor_andn(<16 x i32> %x, <16 x i32> %y, <16 x i32> %z) {
 ; ALL-LABEL: ternlog_xor_andn:
 ; ALL:       ## %bb.0:
-; ALL-NEXT:    vpternlogd $198, %zmm1, %zmm2, %zmm0
+; ALL-NEXT:    vpternlogd {{.*#+}} zmm0 = zmm2 ^ (zmm1 & ~zmm0)
 ; ALL-NEXT:    retq
   %a = xor <16 x i32> %x, <i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1>
   %b = and <16 x i32> %y, %a
@@ -889,7 +889,7 @@ define <16 x i32> @ternlog_xor_andn(<16 x i32> %x, <16 x i32> %y, <16 x i32> %z)
 define <16 x i32> @ternlog_or_and_mask(<16 x i32> %x, <16 x i32> %y) {
 ; ALL-LABEL: ternlog_or_and_mask:
 ; ALL:       ## %bb.0:
-; ALL-NEXT:    vpternlogd $236, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm1, %zmm0
+; ALL-NEXT:    vpternlogd {{.*#+}} zmm0 = (zmm0 & mem) | zmm1
 ; ALL-NEXT:    retq
   %a = and <16 x i32> %x, <i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255>
   %b = or <16 x i32> %a, %y
@@ -899,7 +899,7 @@ define <16 x i32> @ternlog_or_and_mask(<16 x i32> %x, <16 x i32> %y) {
 define <8 x i64> @ternlog_xor_and_mask(<8 x i64> %x, <8 x i64> %y) {
 ; ALL-LABEL: ternlog_xor_and_mask:
 ; ALL:       ## %bb.0:
-; ALL-NEXT:    vpternlogq $108, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm1, %zmm0
+; ALL-NEXT:    vpternlogq {{.*#+}} zmm0 = zmm1 ^ (zmm0 & mem)
 ; ALL-NEXT:    retq
   %a = and <8 x i64> %x, <i64 4294967295, i64 4294967295, i64 4294967295, i64 4294967295, i64 4294967295, i64 4294967295, i64 4294967295, i64 4294967295>
   %b = xor <8 x i64> %a, %y
@@ -911,7 +911,7 @@ define <16 x i32> @ternlog_maskz_or_and_mask(<16 x i32> %x, <16 x i32> %y, <16 x
 ; ALL:       ## %bb.0:
 ; ALL-NEXT:    vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm0, %zmm3
 ; ALL-NEXT:    vpsrad $31, %zmm2, %zmm0
-; ALL-NEXT:    vpternlogd $224, %zmm1, %zmm3, %zmm0
+; ALL-NEXT:    vpternlogd {{.*#+}} zmm0 = zmm0 & (zmm3 | zmm1)
 ; ALL-NEXT:    retq
   %m = icmp slt <16 x i32> %mask, zeroinitializer
   %a = and <16 x i32> %x, <i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255>
@@ -925,7 +925,7 @@ define <8 x i64> @ternlog_maskz_xor_and_mask(<8 x i64> %x, <8 x i64> %y, <8 x i6
 ; ALL:       ## %bb.0:
 ; ALL-NEXT:    vpandq {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm0, %zmm3
 ; ALL-NEXT:    vpsraq $63, %zmm2, %zmm0
-; ALL-NEXT:    vpternlogq $96, %zmm1, %zmm3, %zmm0
+; ALL-NEXT:    vpternlogq {{.*#+}} zmm0 = zmm0 & (zmm3 ^ zmm1)
 ; ALL-NEXT:    retq
   %m = icmp slt <8 x i64> %mask, zeroinitializer
   %a = and <8 x i64> %x, <i64 4294967295, i64 4294967295, i64 4294967295, i64 4294967295, i64 4294967295, i64 4294967295, i64 4294967295, i64 4294967295>
diff --git a/llvm/test/CodeGen/X86/avx512-regcall-Mask.ll b/llvm/test/CodeGen/X86/avx512-regcall-Mask.ll
index b3a0c7dffae11..162f5efd78f6d 100644
--- a/llvm/test/CodeGen/X86/avx512-regcall-Mask.ll
+++ b/llvm/test/CodeGen/X86/avx512-regcall-Mask.ll
@@ -149,12 +149,14 @@ define dso_local i64 @caller_argv64i1() #0 {
 ; WIN64-NEXT:    callq test_argv64i1
 ; WIN64-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
 ; WIN64-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm7 # 16-byte Reload
+; WIN64-NEXT:    .seh_startepilogue
 ; WIN64-NEXT:    addq $48, %rsp
 ; WIN64-NEXT:    popq %rdi
 ; WIN64-NEXT:    popq %rsi
 ; WIN64-NEXT:    popq %r12
 ; WIN64-NEXT:    popq %r14
 ; WIN64-NEXT:    popq %r15
+; WIN64-NEXT:    .seh_endepilogue
 ; WIN64-NEXT:    retq
 ; WIN64-NEXT:    .seh_endproc
 ;
@@ -256,9 +258,11 @@ define dso_local <64 x i1> @caller_retv64i1() #0 {
 ; WIN64-NEXT:    vpmovm2b %k0, %zmm0
 ; WIN64-NEXT:    vmovaps (%rsp), %xmm6 # 16-byte Reload
 ; WIN64-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm7 # 16-byte Reload
+; WIN64-NEXT:    .seh_startepilogue
 ; WIN64-NEXT:    addq $40, %rsp
 ; WIN64-NEXT:    popq %rdi
 ; WIN64-NEXT:    popq %rsi
+; WIN64-NEXT:    .seh_endepilogue
 ; WIN64-NEXT:    retq
 ; WIN64-NEXT:    .seh_endproc
 ;
@@ -334,10 +338,12 @@ define dso_local x86_regcallcc i32 @test_argv32i1(<32 x i1> %x0, <32 x i1> %x1,
 ; WIN64-NEXT:    vzeroupper
 ; WIN64-NEXT:    callq test_argv32i1helper
 ; WIN64-NEXT:    nop
+; WIN64-NEXT:    .seh_startepilogue
 ; WIN64-NEXT:    movq %rbp, %rsp
 ; WIN64-NEXT:    popq %r10
 ; WIN64-NEXT:    popq %r11
 ; WIN64-NEXT:    popq %rbp
+; WIN64-NEXT:    .seh_endepilogue
 ; WIN64-NEXT:    retq
 ; WIN64-NEXT:    .seh_endproc
 ;
@@ -417,9 +423,11 @@ define dso_local i32 @caller_argv32i1() #0 {
 ; WIN64-NEXT:    callq test_argv32i1
 ; WIN64-NEXT:    vmovaps (%rsp), %xmm6 # 16-byte Reload
 ; WIN64-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm7 # 16-byte Reload
+; WIN64-NEXT:    .seh_startepilogue
 ; WIN64-NEXT:    addq $40, %rsp
 ; WIN64-NEXT:    popq %rdi
 ; WIN64-NEXT:    popq %rsi
+; WIN64-NEXT:    .seh_endepilogue
 ; WIN64-NEXT:    retq
 ; WIN64-NEXT:    .seh_endproc
 ;
@@ -480,9 +488,11 @@ define dso_local i32 @caller_retv32i1() #0 {
 ; WIN64-NEXT:    incl %eax
 ; WIN64-NEXT:    vmovaps (%rsp), %xmm6 # 16-byte Reload
 ; WIN64-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm7 # 16-byte Reload
+; WIN64-NEXT:    .seh_startepilogue
 ; WIN64-NEXT:    addq $40, %rsp
 ; WIN64-NEXT:    popq %rdi
 ; WIN64-NEXT:    popq %rsi
+; WIN64-NEXT:    .seh_endepilogue
 ; WIN64-NEXT:    retq
 ; WIN64-NEXT:    .seh_endproc
 ;
@@ -554,9 +564,11 @@ define dso_local x86_regcallcc i16 @test_argv16i1(<16 x i1> %x0, <16 x i1> %x1,
 ; WIN64-NEXT:    vzeroupper
 ; WIN64-NEXT:    callq test_argv16i1helper
 ; WIN64-NEXT:    nop
+; WIN64-NEXT:    .seh_startepilogue
 ; WIN64-NEXT:    addq $88, %rsp
 ; WIN64-NEXT:    popq %r10
 ; WIN64-NEXT:    popq %r11
+; WIN64-NEXT:    .seh_endepilogue
 ; WIN64-NEXT:    retq
 ; WIN64-NEXT:    .seh_endproc
 ;
@@ -635,9 +647,11 @@ define dso_local i16 @caller_argv16i1() #0 {
 ; WIN64-NEXT:    callq test_argv16i1
 ; WIN64-NEXT:    vmovaps (%rsp), %xmm6 # 16-byte Reload
 ; WIN64-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm7 # 16-byte Reload
+; WIN64-NEXT:    .seh_startepilogue
 ; WIN64-NEXT:    addq $40, %rsp
 ; WIN64-NEXT:    popq %rdi
 ; WIN64-NEXT:    popq %rsi
+; WIN64-NEXT:    .seh_endepilogue
 ; WIN64-NEXT:    retq
 ; WIN64-NEXT:    .seh_endproc
 ;
@@ -702,9 +716,11 @@ define dso_local i16 @caller_retv16i1() #0 {
 ; WIN64-NEXT:    # kill: def $ax killed $ax killed $eax
 ; WIN64-NEXT:    vmovaps (%rsp), %xmm6 # 16-byte Reload
 ; WIN64-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm7 # 16-byte Reload
+; WIN64-NEXT:    .seh_startepilogue
 ; WIN64-NEXT:    addq $40, %rsp
 ; WIN64-NEXT:    popq %rdi
 ; WIN64-NEXT:    popq %rsi
+; WIN64-NEXT:    .seh_endepilogue
 ; WIN64-NEXT:    retq
 ; WIN64-NEXT:    .seh_endproc
 ;
@@ -778,9 +794,11 @@ define dso_local x86_regcallcc i8 @test_argv8i1(<8 x i1> %x0, <8 x i1> %x1, <8 x
 ; WIN64-NEXT:    vzeroupper
 ; WIN64-NEXT:    callq test_argv8i1helper
 ; WIN64-NEXT:    nop
+; WIN64-NEXT:    .seh_startepilogue
 ; WIN64-NEXT:    addq $88, %rsp
 ; WIN64-NEXT:    popq %r10
 ; WIN64-NEXT:    popq %r11
+; WIN64-NEXT:    .seh_endepilogue
 ; WIN64-NEXT:    retq
 ; WIN64-NEXT:    .seh_endproc
 ;
@@ -859,9 +877,11 @@ define dso_local i8 @caller_argv8i1() #0 {
 ; WIN64-NEXT:    callq test_argv8i1
 ; WIN64-NEXT:    vmovaps (%rsp), %xmm6 # 16-byte Reload
 ; WIN64-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm7 # 16-byte Reload
+; WIN64-NEXT:    .seh_startepilogue
 ; WIN64-NEXT:    addq $40, %rsp
 ; WIN64-NEXT:    popq %rdi
 ; WIN64-NEXT:    popq %rsi
+; WIN64-NEXT:    .seh_endepilogue
 ; WIN64-NEXT:    retq
 ; WIN64-NEXT:    .seh_endproc
 ;
@@ -929,9 +949,11 @@ define dso_local <8 x i1> @caller_retv8i1() #0 {
 ; WIN64-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
 ; WIN64-NEXT:    vmovaps (%rsp), %xmm6 # 16-byte Reload
 ; WIN64-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm7 # 16-byte Reload
+; WIN64-NEXT:    .seh_startepilogue
 ; WIN64-NEXT:    addq $40, %rsp
 ; WIN64-NEXT:    popq %rdi
 ; WIN64-NEXT:    popq %rsi
+; WIN64-NEXT:    .seh_endepilogue
 ; WIN64-NEXT:    vzeroupper
 ; WIN64-NEXT:    retq
 ; WIN64-NEXT:    .seh_endproc
diff --git a/llvm/test/CodeGen/X86/avx512-regcall-NoMask.ll b/llvm/test/CodeGen/X86/avx512-regcall-NoMask.ll
index 2081d201704f3..88c99a06326ab 100644
--- a/llvm/test/CodeGen/X86/avx512-regcall-NoMask.ll
+++ b/llvm/test/CodeGen/X86/avx512-regcall-NoMask.ll
@@ -7,19 +7,19 @@
 define dso_local x86_regcallcc i1 @test_argReti1(i1 %a)  {
 ; X32-LABEL: test_argReti1:
 ; X32:       # %bb.0:
-; X32-NEXT:    incb %al
+; X32-NEXT:    xorb $1, %al
 ; X32-NEXT:    # kill: def $al killed $al killed $eax
 ; X32-NEXT:    retl
 ;
 ; WIN64-LABEL: test_argReti1:
 ; WIN64:       # %bb.0:
-; WIN64-NEXT:    incb %al
+; WIN64-NEXT:    xorb $1, %al
 ; WIN64-NEXT:    # kill: def $al killed $al killed $eax
 ; WIN64-NEXT:    retq
 ;
 ; LINUXOSX64-LABEL: test_argReti1:
 ; LINUXOSX64:       # %bb.0:
-; LINUXOSX64-NEXT:    incb %al
+; LINUXOSX64-NEXT:    xorb $1, %al
 ; LINUXOSX64-NEXT:    # kill: def $al killed $al killed $eax
 ; LINUXOSX64-NEXT:    retq
   %add = add i1 %a, 1
@@ -30,10 +30,10 @@ define dso_local x86_regcallcc i1 @test_argReti1(i1 %a)  {
 define dso_local x86_regcallcc i1 @test_CallargReti1(i1 %a)  {
 ; X32-LABEL: test_CallargReti1:
 ; X32:       # %bb.0:
-; X32-NEXT:    incb %al
+; X32-NEXT:    xorb $1, %al
 ; X32-NEXT:    movzbl %al, %eax
 ; X32-NEXT:    calll _test_argReti1
-; X32-NEXT:    incb %al
+; X32-NEXT:    xorb $1, %al
 ; X32-NEXT:    retl
 ;
 ; WIN64-LABEL: test_CallargReti1:
@@ -41,11 +41,13 @@ define dso_local x86_regcallcc i1 @test_CallargReti1(i1 %a)  {
 ; WIN64-NEXT:    pushq %rax
 ; WIN64-NEXT:    .seh_stackalloc 8
 ; WIN64-NEXT:    .seh_endprologue
-; WIN64-NEXT:    incb %al
+; WIN64-NEXT:    xorb $1, %al
 ; WIN64-NEXT:    movzbl %al, %eax
 ; WIN64-NEXT:    callq test_argReti1
-; WIN64-NEXT:    incb %al
+; WIN64-NEXT:    xorb $1, %al
+; WIN64-NEXT:    .seh_startepilogue
 ; WIN64-NEXT:    popq %rcx
+; WIN64-NEXT:    .seh_endepilogue
 ; WIN64-NEXT:    retq
 ; WIN64-NEXT:    .seh_endproc
 ;
@@ -53,10 +55,10 @@ define dso_local x86_regcallcc i1 @test_CallargReti1(i1 %a)  {
 ; LINUXOSX64:       # %bb.0:
 ; LINUXOSX64-NEXT:    pushq %rax
 ; LINUXOSX64-NEXT:    .cfi_def_cfa_offset 16
-; LINUXOSX64-NEXT:    incb %al
+; LINUXOSX64-NEXT:    xorb $1, %al
 ; LINUXOSX64-NEXT:    movzbl %al, %eax
 ; LINUXOSX64-NEXT:    callq test_argReti1
-; LINUXOSX64-NEXT:    incb %al
+; LINUXOSX64-NEXT:    xorb $1, %al
 ; LINUXOSX64-NEXT:    popq %rcx
 ; LINUXOSX64-NEXT:    .cfi_def_cfa_offset 8
 ; LINUXOSX64-NEXT:    retq
@@ -108,7 +110,9 @@ define dso_local x86_regcallcc i8 @test_CallargReti8(i8 %a)  {
 ; WIN64-NEXT:    movzbl %al, %eax
 ; WIN64-NEXT:    callq test_argReti8
 ; WIN64-NEXT:    incb %al
+; WIN64-NEXT:    .seh_startepilogue
 ; WIN64-NEXT:    popq %rcx
+; WIN64-NEXT:    .seh_endepilogue
 ; WIN64-NEXT:    retq
 ; WIN64-NEXT:    .seh_endproc
 ;
@@ -173,7 +177,9 @@ define dso_local x86_regcallcc i16 @test_CallargReti16(i16 %a)  {
 ; WIN64-NEXT:    # kill: def $ax killed $ax def $eax
 ; WIN64-NEXT:    incl %eax
 ; WIN64-NEXT:    # kill: def $ax killed $ax killed $eax
+; WIN64-NEXT:    .seh_startepilogue
 ; WIN64-NEXT:    popq %rcx
+; WIN64-NEXT:    .seh_endepilogue
 ; WIN64-NEXT:    retq
 ; WIN64-NEXT:    .seh_endproc
 ;
@@ -232,7 +238,9 @@ define dso_local x86_regcallcc i32 @test_CallargReti32(i32 %a)  {
 ; WIN64-NEXT:    incl %eax
 ; WIN64-NEXT:    callq test_argReti32
 ; WIN64-NEXT:    incl %eax
+; WIN64-NEXT:    .seh_startepilogue
 ; WIN64-NEXT:    popq %rcx
+; WIN64-NEXT:    .seh_endepilogue
 ; WIN64-NEXT:    retq
 ; WIN64-NEXT:    .seh_endproc
 ;
@@ -294,7 +302,9 @@ define dso_local x86_regcallcc i64 @test_CallargReti64(i64 %a)  {
 ; WIN64-NEXT:    incq %rax
 ; WIN64-NEXT:    callq test_argReti64
 ; WIN64-NEXT:    incq %rax
+; WIN64-NEXT:    .seh_startepilogue
 ; WIN64-NEXT:    popq %rcx
+; WIN64-NEXT:    .seh_endepilogue
 ; WIN64-NEXT:    retq
 ; WIN64-NEXT:    .seh_endproc
 ;
@@ -360,7 +370,9 @@ define dso_local x86_regcallcc float @test_CallargRetFloat(float %a)  {
 ; WIN64-NEXT:    callq test_argRetFloat
 ; WIN64-NEXT:    vaddss %xmm0, %xmm8, %xmm0
 ; WIN64-NEXT:    vmovaps (%rsp), %xmm8 # 16-byte Reload
+; WIN64-NEXT:    .seh_startepilogue
 ; WIN64-NEXT:    addq $24, %rsp
+; WIN64-NEXT:    .seh_endepilogue
 ; WIN64-NEXT:    retq
 ; WIN64-NEXT:    .seh_endproc
 ;
@@ -430,7 +442,9 @@ define dso_local x86_regcallcc double @test_CallargRetDouble(double %a)  {
 ; WIN64-NEXT:    callq test_argRetDouble
 ; WIN64-NEXT:    vaddsd %xmm0, %xmm8, %xmm0
 ; WIN64-NEXT:    vmovaps (%rsp), %xmm8 # 16-byte Reload
+; WIN64-NEXT:    .seh_startepilogue
 ; WIN64-NEXT:    addq $24, %rsp
+; WIN64-NEXT:    .seh_endepilogue
 ; WIN64-NEXT:    retq
 ; WIN64-NEXT:    .seh_endproc
 ;
@@ -522,7 +536,9 @@ define x86_regcallcc x86_fp80 @test_CallargRetf80(x86_fp80 %a)  {
 ; WIN64-NEXT:    fadd %st, %st(0)
 ; WIN64-NEXT:    callq test_argRetf80
 ; WIN64-NEXT:    fadd %st, %st(0)
+; WIN64-NEXT:    .seh_startepilogue
 ; WIN64-NEXT:    popq %rax
+; WIN64-NEXT:    .seh_endepilogue
 ; WIN64-NEXT:    retq
 ; WIN64-NEXT:    .seh_endproc
 ;
@@ -558,7 +574,9 @@ define dso_local x86_regcallcc double @test_CallargParamf80(x86_fp80 %a)  {
 ; WIN64-NEXT:    fadd %st, %st(0)
 ; WIN64-NEXT:    callq test_argParamf80
 ; WIN64-NEXT:    vaddsd %xmm0, %xmm0, %xmm0
+; WIN64-NEXT:    .seh_startepilogue
 ; WIN64-NEXT:    popq %rax
+; WIN64-NEXT:    .seh_endepilogue
 ; WIN64-NEXT:    retq
 ; WIN64-NEXT:    .seh_endproc
 ;
@@ -617,7 +635,9 @@ define dso_local x86_regcallcc ptr @test_CallargRetPointer(ptr %a)  {
 ; WIN64-NEXT:    incl %eax
 ; WIN64-NEXT:    callq test_argRetPointer
 ; WIN64-NEXT:    incl %eax
+; WIN64-NEXT:    .seh_startepilogue
 ; WIN64-NEXT:    popq %rcx
+; WIN64-NEXT:    .seh_endepilogue
 ; WIN64-NEXT:    retq
 ; WIN64-NEXT:    .seh_endproc
 ;
@@ -703,7 +723,9 @@ define dso_local x86_regcallcc <4 x i32> @test_CallargRet128Vector(<4 x i1> %x,
 ; WIN64-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k1 # 2-byte Reload
 ; WIN64-NEXT:    vmovdqa32 %xmm8, %xmm0 {%k1}
 ; WIN64-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm8 # 16-byte Reload
+; WIN64-NEXT:    .seh_startepilogue
 ; WIN64-NEXT:    addq $40, %rsp
+; WIN64-NEXT:    .seh_endepilogue
 ; WIN64-NEXT:    retq
 ; WIN64-NEXT:    .seh_endproc
 ;
@@ -783,7 +805,9 @@ define dso_local x86_regcallcc <8 x i32> @test_CallargRet256Vector(<8 x i1> %x,
 ; WIN64-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k1 # 2-byte Reload
 ; WIN64-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
 ; WIN64-NEXT:    vmovdqa32 %ymm1, %ymm0 {%k1}
+; WIN64-NEXT:    .seh_startepilogue
 ; WIN64-NEXT:    addq $56, %rsp
+; WIN64-NEXT:    .seh_endepilogue
 ; WIN64-NEXT:    retq
 ; WIN64-NEXT:    .seh_endproc
 ;
@@ -859,7 +883,9 @@ define dso_local x86_regcallcc <16 x i32> @test_CallargRet512Vector(<16 x i1> %x
 ; WIN64-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k1 # 2-byte Reload
 ; WIN64-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
 ; WIN64-NEXT:    vmovdqa32 %zmm1, %zmm0 {%k1}
+; WIN64-NEXT:    .seh_startepilogue
 ; WIN64-NEXT:    addq $88, %rsp
+; WIN64-NEXT:    .seh_endepilogue
 ; WIN64-NEXT:    retq
 ; WIN64-NEXT:    .seh_endproc
 ;
diff --git a/llvm/test/CodeGen/X86/avx512-select.ll b/llvm/test/CodeGen/X86/avx512-select.ll
index 536c667c7ec90..721ffbe1ceb79 100644
--- a/llvm/test/CodeGen/X86/avx512-select.ll
+++ b/llvm/test/CodeGen/X86/avx512-select.ll
@@ -502,7 +502,7 @@ define <16 x i64> @narrowExtractedVectorSelect_crash(<16 x i64> %arg, <16 x i16>
 ; X86-AVX512F-NEXT:    vptestmq %zmm0, %zmm0, %k0
 ; X86-AVX512F-NEXT:    vptestmq %zmm1, %zmm1, %k1
 ; X86-AVX512F-NEXT:    kunpckbw %k0, %k1, %k1
-; X86-AVX512F-NEXT:    vpternlogd $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
+; X86-AVX512F-NEXT:    vpternlogd {{.*#+}} zmm0 {%k1} {z} = -1
 ; X86-AVX512F-NEXT:    vpmovdw %zmm0, %ymm0
 ; X86-AVX512F-NEXT:    vpand %ymm2, %ymm0, %ymm1
 ; X86-AVX512F-NEXT:    vpmovzxwq {{.*#+}} zmm0 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero,xmm1[4],zero,zero,zero,xmm1[5],zero,zero,zero,xmm1[6],zero,zero,zero,xmm1[7],zero,zero,zero
@@ -515,7 +515,7 @@ define <16 x i64> @narrowExtractedVectorSelect_crash(<16 x i64> %arg, <16 x i16>
 ; X64-AVX512F-NEXT:    vptestmq %zmm0, %zmm0, %k0
 ; X64-AVX512F-NEXT:    vptestmq %zmm1, %zmm1, %k1
 ; X64-AVX512F-NEXT:    kunpckbw %k0, %k1, %k1
-; X64-AVX512F-NEXT:    vpternlogd $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
+; X64-AVX512F-NEXT:    vpternlogd {{.*#+}} zmm0 {%k1} {z} = -1
 ; X64-AVX512F-NEXT:    vpmovdw %zmm0, %ymm0
 ; X64-AVX512F-NEXT:    vpand %ymm2, %ymm0, %ymm1
 ; X64-AVX512F-NEXT:    vpmovzxwq {{.*#+}} zmm0 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero,xmm1[4],zero,zero,zero,xmm1[5],zero,zero,zero,xmm1[6],zero,zero,zero,xmm1[7],zero,zero,zero
diff --git a/llvm/test/CodeGen/X86/bfloat.ll b/llvm/test/CodeGen/X86/bfloat.ll
index d67cd6b62c2b9..4d269cfff2afe 100644
--- a/llvm/test/CodeGen/X86/bfloat.ll
+++ b/llvm/test/CodeGen/X86/bfloat.ll
@@ -82,8 +82,6 @@ define bfloat @add2(bfloat %a, bfloat %b) nounwind {
 ; X86-NEXT:    vmovd %eax, %xmm1
 ; X86-NEXT:    vaddss %xmm0, %xmm1, %xmm0
 ; X86-NEXT:    vcvtneps2bf16 %xmm0, %xmm0
-; X86-NEXT:    vmovw %xmm0, %eax
-; X86-NEXT:    vmovw %eax, %xmm0
 ; X86-NEXT:    retl
 ;
 ; SSE2-LABEL: add2:
@@ -110,8 +108,6 @@ define bfloat @add2(bfloat %a, bfloat %b) nounwind {
 ; FP16-NEXT:    vmovd %eax, %xmm1
 ; FP16-NEXT:    vaddss %xmm0, %xmm1, %xmm0
 ; FP16-NEXT:    vcvtneps2bf16 %xmm0, %xmm0
-; FP16-NEXT:    vmovw %xmm0, %eax
-; FP16-NEXT:    vmovw %eax, %xmm0
 ; FP16-NEXT:    retq
 ;
 ; AVXNC-LABEL: add2:
@@ -124,8 +120,6 @@ define bfloat @add2(bfloat %a, bfloat %b) nounwind {
 ; AVXNC-NEXT:    vmovd %eax, %xmm1
 ; AVXNC-NEXT:    vaddss %xmm0, %xmm1, %xmm0
 ; AVXNC-NEXT:    {vex} vcvtneps2bf16 %xmm0, %xmm0
-; AVXNC-NEXT:    vmovd %xmm0, %eax
-; AVXNC-NEXT:    vpinsrw $0, %eax, %xmm0, %xmm0
 ; AVXNC-NEXT:    retq
   %add = fadd bfloat %a, %b
   ret bfloat %add
@@ -432,8 +426,6 @@ define bfloat @add_constant2(bfloat %a) nounwind {
 ; X86-NEXT:    vmovd %eax, %xmm0
 ; X86-NEXT:    vaddss {{\.?LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0
 ; X86-NEXT:    vcvtneps2bf16 %xmm0, %xmm0
-; X86-NEXT:    vmovw %xmm0, %eax
-; X86-NEXT:    vmovw %eax, %xmm0
 ; X86-NEXT:    retl
 ;
 ; SSE2-LABEL: add_constant2:
@@ -454,8 +446,6 @@ define bfloat @add_constant2(bfloat %a) nounwind {
 ; FP16-NEXT:    vmovd %eax, %xmm0
 ; FP16-NEXT:    vaddss {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
 ; FP16-NEXT:    vcvtneps2bf16 %xmm0, %xmm0
-; FP16-NEXT:    vmovw %xmm0, %eax
-; FP16-NEXT:    vmovw %eax, %xmm0
 ; FP16-NEXT:    retq
 ;
 ; AVXNC-LABEL: add_constant2:
@@ -465,8 +455,6 @@ define bfloat @add_constant2(bfloat %a) nounwind {
 ; AVXNC-NEXT:    vmovd %eax, %xmm0
 ; AVXNC-NEXT:    vaddss {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
 ; AVXNC-NEXT:    {vex} vcvtneps2bf16 %xmm0, %xmm0
-; AVXNC-NEXT:    vmovd %xmm0, %eax
-; AVXNC-NEXT:    vpinsrw $0, %eax, %xmm0, %xmm0
 ; AVXNC-NEXT:    retq
   %add = fadd bfloat %a, 1.0
   ret bfloat %add
diff --git a/llvm/test/CodeGen/X86/bitcast-vector-bool.ll b/llvm/test/CodeGen/X86/bitcast-vector-bool.ll
index 501e73c46af9c..86d7df0c2d648 100644
--- a/llvm/test/CodeGen/X86/bitcast-vector-bool.ll
+++ b/llvm/test/CodeGen/X86/bitcast-vector-bool.ll
@@ -16,7 +16,7 @@ define i1 @bitcast_v2i64_to_v2i1(<2 x i64> %a0) nounwind {
 ; SSE-NEXT:    movmskpd %xmm0, %ecx
 ; SSE-NEXT:    movl %ecx, %eax
 ; SSE-NEXT:    shrb %al
-; SSE-NEXT:    addb %cl, %al
+; SSE-NEXT:    xorb %cl, %al
 ; SSE-NEXT:    retq
 ;
 ; AVX12-LABEL: bitcast_v2i64_to_v2i1:
@@ -24,7 +24,7 @@ define i1 @bitcast_v2i64_to_v2i1(<2 x i64> %a0) nounwind {
 ; AVX12-NEXT:    vmovmskpd %xmm0, %ecx
 ; AVX12-NEXT:    movl %ecx, %eax
 ; AVX12-NEXT:    shrb %al
-; AVX12-NEXT:    addb %cl, %al
+; AVX12-NEXT:    xorb %cl, %al
 ; AVX12-NEXT:    retq
 ;
 ; AVX512-LABEL: bitcast_v2i64_to_v2i1:
@@ -34,7 +34,7 @@ define i1 @bitcast_v2i64_to_v2i1(<2 x i64> %a0) nounwind {
 ; AVX512-NEXT:    kshiftrw $1, %k0, %k1
 ; AVX512-NEXT:    kmovd %k1, %ecx
 ; AVX512-NEXT:    kmovd %k0, %eax
-; AVX512-NEXT:    addb %cl, %al
+; AVX512-NEXT:    xorb %cl, %al
 ; AVX512-NEXT:    # kill: def $al killed $al killed $eax
 ; AVX512-NEXT:    retq
   %1 = icmp slt <2 x i64> %a0, zeroinitializer
diff --git a/llvm/test/CodeGen/X86/break-false-dep.ll b/llvm/test/CodeGen/X86/break-false-dep.ll
index 0a367e1916ff6..5acbccf41c5d3 100644
--- a/llvm/test/CodeGen/X86/break-false-dep.ll
+++ b/llvm/test/CodeGen/X86/break-false-dep.ll
@@ -516,8 +516,10 @@ define dso_local void @loopdep3() {
 ; SSE-WIN-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm13 # 16-byte Reload
 ; SSE-WIN-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm14 # 16-byte Reload
 ; SSE-WIN-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm15 # 16-byte Reload
+; SSE-WIN-NEXT:    .seh_startepilogue
 ; SSE-WIN-NEXT:    addq $160, %rsp
 ; SSE-WIN-NEXT:    popq %rsi
+; SSE-WIN-NEXT:    .seh_endepilogue
 ; SSE-WIN-NEXT:    retq
 ; SSE-WIN-NEXT:    .seh_endproc
 ;
@@ -592,8 +594,10 @@ define dso_local void @loopdep3() {
 ; AVX-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm13 # 16-byte Reload
 ; AVX-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm14 # 16-byte Reload
 ; AVX-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm15 # 16-byte Reload
+; AVX-NEXT:    .seh_startepilogue
 ; AVX-NEXT:    addq $160, %rsp
 ; AVX-NEXT:    popq %rsi
+; AVX-NEXT:    .seh_endepilogue
 ; AVX-NEXT:    retq
 ; AVX-NEXT:    .seh_endproc
 entry:
@@ -710,7 +714,9 @@ define dso_local double @inlineasmdep(i64 %arg) {
 ; SSE-WIN-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm13 # 16-byte Reload
 ; SSE-WIN-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm14 # 16-byte Reload
 ; SSE-WIN-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm15 # 16-byte Reload
+; SSE-WIN-NEXT:    .seh_startepilogue
 ; SSE-WIN-NEXT:    addq $168, %rsp
+; SSE-WIN-NEXT:    .seh_endepilogue
 ; SSE-WIN-NEXT:    retq
 ; SSE-WIN-NEXT:    .seh_endproc
 ;
@@ -767,7 +773,9 @@ define dso_local double @inlineasmdep(i64 %arg) {
 ; AVX-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm13 # 16-byte Reload
 ; AVX-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm14 # 16-byte Reload
 ; AVX-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm15 # 16-byte Reload
+; AVX-NEXT:    .seh_startepilogue
 ; AVX-NEXT:    addq $168, %rsp
+; AVX-NEXT:    .seh_endepilogue
 ; AVX-NEXT:    retq
 ; AVX-NEXT:    .seh_endproc
 top:
@@ -869,7 +877,9 @@ define dso_local double @truedeps(float %arg) {
 ; SSE-WIN-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm13 # 16-byte Reload
 ; SSE-WIN-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm14 # 16-byte Reload
 ; SSE-WIN-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm15 # 16-byte Reload
+; SSE-WIN-NEXT:    .seh_startepilogue
 ; SSE-WIN-NEXT:    addq $184, %rsp
+; SSE-WIN-NEXT:    .seh_endepilogue
 ; SSE-WIN-NEXT:    retq
 ; SSE-WIN-NEXT:    .seh_endproc
 ;
@@ -930,7 +940,9 @@ define dso_local double @truedeps(float %arg) {
 ; AVX-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm13 # 16-byte Reload
 ; AVX-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm14 # 16-byte Reload
 ; AVX-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm15 # 16-byte Reload
+; AVX-NEXT:    .seh_startepilogue
 ; AVX-NEXT:    addq $184, %rsp
+; AVX-NEXT:    .seh_endepilogue
 ; AVX-NEXT:    retq
 ; AVX-NEXT:    .seh_endproc
 top:
@@ -1029,7 +1041,9 @@ define dso_local double @clearence(i64 %arg) {
 ; SSE-WIN-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm13 # 16-byte Reload
 ; SSE-WIN-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm14 # 16-byte Reload
 ; SSE-WIN-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm15 # 16-byte Reload
+; SSE-WIN-NEXT:    .seh_startepilogue
 ; SSE-WIN-NEXT:    addq $168, %rsp
+; SSE-WIN-NEXT:    .seh_endepilogue
 ; SSE-WIN-NEXT:    retq
 ; SSE-WIN-NEXT:    .seh_endproc
 ;
@@ -1088,7 +1102,9 @@ define dso_local double @clearence(i64 %arg) {
 ; AVX-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm13 # 16-byte Reload
 ; AVX-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm14 # 16-byte Reload
 ; AVX-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm15 # 16-byte Reload
+; AVX-NEXT:    .seh_startepilogue
 ; AVX-NEXT:    addq $168, %rsp
+; AVX-NEXT:    .seh_endepilogue
 ; AVX-NEXT:    retq
 ; AVX-NEXT:    .seh_endproc
 top:
@@ -1397,7 +1413,9 @@ define dso_local void @loopclearance2(ptr nocapture %y, ptr %x, double %c1, doub
 ; SSE-WIN-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm13 # 16-byte Reload
 ; SSE-WIN-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm14 # 16-byte Reload
 ; SSE-WIN-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm15 # 16-byte Reload
+; SSE-WIN-NEXT:    .seh_startepilogue
 ; SSE-WIN-NEXT:    addq $152, %rsp
+; SSE-WIN-NEXT:    .seh_endepilogue
 ; SSE-WIN-NEXT:    retq
 ; SSE-WIN-NEXT:    .seh_endproc
 ;
@@ -1477,7 +1495,9 @@ define dso_local void @loopclearance2(ptr nocapture %y, ptr %x, double %c1, doub
 ; AVX1-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm13 # 16-byte Reload
 ; AVX1-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm14 # 16-byte Reload
 ; AVX1-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm15 # 16-byte Reload
+; AVX1-NEXT:    .seh_startepilogue
 ; AVX1-NEXT:    addq $152, %rsp
+; AVX1-NEXT:    .seh_endepilogue
 ; AVX1-NEXT:    retq
 ; AVX1-NEXT:    .seh_endproc
 ;
@@ -1557,7 +1577,9 @@ define dso_local void @loopclearance2(ptr nocapture %y, ptr %x, double %c1, doub
 ; AVX512VL-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm13 # 16-byte Reload
 ; AVX512VL-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm14 # 16-byte Reload
 ; AVX512VL-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm15 # 16-byte Reload
+; AVX512VL-NEXT:    .seh_startepilogue
 ; AVX512VL-NEXT:    addq $152, %rsp
+; AVX512VL-NEXT:    .seh_endepilogue
 ; AVX512VL-NEXT:    retq
 ; AVX512VL-NEXT:    .seh_endproc
 entry:
diff --git a/llvm/test/CodeGen/X86/canonicalize-vars-f16-type.ll b/llvm/test/CodeGen/X86/canonicalize-vars-f16-type.ll
index 04087c4f0dd5e..556b0deaf4c83 100644
--- a/llvm/test/CodeGen/X86/canonicalize-vars-f16-type.ll
+++ b/llvm/test/CodeGen/X86/canonicalize-vars-f16-type.ll
@@ -154,8 +154,6 @@ define half @complex_canonicalize_fmul_half(half %a, half %b) nounwind {
 ; AVX512-NEXT:    vcvtph2ps %xmm0, %xmm0
 ; AVX512-NEXT:    vsubss %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vcvtps2ph $4, %xmm0, %xmm0
-; AVX512-NEXT:    vmovd %xmm0, %eax
-; AVX512-NEXT:    vpinsrw $0, %eax, %xmm0, %xmm0
 ; AVX512-NEXT:    retq
 entry:
 
@@ -239,15 +237,11 @@ define void @v_test_canonicalize_v2half(<2 x half> addrspace(1)* %out) nounwind
 ; AVX512-NEXT:    vxorps %xmm3, %xmm3, %xmm3
 ; AVX512-NEXT:    vblendps {{.*#+}} xmm2 = xmm2[0],xmm3[1,2,3]
 ; AVX512-NEXT:    vcvtps2ph $4, %xmm2, %xmm2
-; AVX512-NEXT:    vmovd %xmm2, %eax
-; AVX512-NEXT:    vpinsrw $0, %eax, %xmm0, %xmm2
 ; AVX512-NEXT:    vpmovzxwq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero
 ; AVX512-NEXT:    vcvtph2ps %xmm0, %xmm0
 ; AVX512-NEXT:    vmulss %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0],xmm3[1,2,3]
 ; AVX512-NEXT:    vcvtps2ph $4, %xmm0, %xmm0
-; AVX512-NEXT:    vmovd %xmm0, %eax
-; AVX512-NEXT:    vpinsrw $0, %eax, %xmm0, %xmm0
 ; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3]
 ; AVX512-NEXT:    vmovd %xmm0, (%rdi)
 ; AVX512-NEXT:    retq
diff --git a/llvm/test/CodeGen/X86/canonicalize-vars.ll b/llvm/test/CodeGen/X86/canonicalize-vars.ll
index 951ea1b72f439..67213b38277dc 100644
--- a/llvm/test/CodeGen/X86/canonicalize-vars.ll
+++ b/llvm/test/CodeGen/X86/canonicalize-vars.ll
@@ -1,6 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --default-march x86_64-unknown-linux-gnu --version 5
 ; RUN: llc -mtriple=i686-- < %s | FileCheck %s -check-prefixes=X87
-; RUN: llc -mattr=+sse2 -mtriple=x86_64 < %s | FileCheck %s -check-prefixes=SSE,SSE2
+; RUN: llc -mattr=+sse2 -mtriple=i686-- < %s | FileCheck %s -check-prefixes=X86-SSE
+; RUN: llc -mattr=+avx -mtriple=i686-- < %s | FileCheck %s -check-prefixes=X86-AVX
+; RUN: llc -mattr=+sse2 -mtriple=x86_64 < %s | FileCheck %s -check-prefixes=SSE
 ; RUN: llc -mattr=+avx -mtriple=x86_64 < %s | FileCheck %s -check-prefixes=AVX,AVX1
 ; RUN: llc -mattr=+avx2 -mtriple=x86_64 < %s | FileCheck %s -check-prefixes=AVX,AVX2
 ; RUN: llc -mattr=+avx512f -mtriple=x86_64 < %s | FileCheck %s -check-prefixes=AVX,AVX512F
@@ -12,6 +14,30 @@ define float @canon_fp32_varargsf32(float %a) {
 ; X87-NEXT:    fmuls {{[0-9]+}}(%esp)
 ; X87-NEXT:    retl
 ;
+; X86-SSE-LABEL: canon_fp32_varargsf32:
+; X86-SSE:       # %bb.0:
+; X86-SSE-NEXT:    pushl %eax
+; X86-SSE-NEXT:    .cfi_def_cfa_offset 8
+; X86-SSE-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; X86-SSE-NEXT:    mulss {{\.?LCPI[0-9]+_[0-9]+}}, %xmm0
+; X86-SSE-NEXT:    movss %xmm0, (%esp)
+; X86-SSE-NEXT:    flds (%esp)
+; X86-SSE-NEXT:    popl %eax
+; X86-SSE-NEXT:    .cfi_def_cfa_offset 4
+; X86-SSE-NEXT:    retl
+;
+; X86-AVX-LABEL: canon_fp32_varargsf32:
+; X86-AVX:       # %bb.0:
+; X86-AVX-NEXT:    pushl %eax
+; X86-AVX-NEXT:    .cfi_def_cfa_offset 8
+; X86-AVX-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; X86-AVX-NEXT:    vmulss {{\.?LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0
+; X86-AVX-NEXT:    vmovss %xmm0, (%esp)
+; X86-AVX-NEXT:    flds (%esp)
+; X86-AVX-NEXT:    popl %eax
+; X86-AVX-NEXT:    .cfi_def_cfa_offset 4
+; X86-AVX-NEXT:    retl
+;
 ; SSE-LABEL: canon_fp32_varargsf32:
 ; SSE:       # %bb.0:
 ; SSE-NEXT:    mulss {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
@@ -33,6 +59,20 @@ define x86_fp80 @canon_fp32_varargsf80(x86_fp80 %a) {
 ; X87-NEXT:    fmulp %st, %st(1)
 ; X87-NEXT:    retl
 ;
+; X86-SSE-LABEL: canon_fp32_varargsf80:
+; X86-SSE:       # %bb.0:
+; X86-SSE-NEXT:    fldt {{[0-9]+}}(%esp)
+; X86-SSE-NEXT:    fld1
+; X86-SSE-NEXT:    fmulp %st, %st(1)
+; X86-SSE-NEXT:    retl
+;
+; X86-AVX-LABEL: canon_fp32_varargsf80:
+; X86-AVX:       # %bb.0:
+; X86-AVX-NEXT:    fldt {{[0-9]+}}(%esp)
+; X86-AVX-NEXT:    fld1
+; X86-AVX-NEXT:    fmulp %st, %st(1)
+; X86-AVX-NEXT:    retl
+;
 ; SSE-LABEL: canon_fp32_varargsf80:
 ; SSE:       # %bb.0:
 ; SSE-NEXT:    fldt {{[0-9]+}}(%rsp)
@@ -64,6 +104,32 @@ define x86_fp80 @complex_canonicalize_fmul_x86_fp80(x86_fp80 %a, x86_fp80 %b) {
 ; X87-NEXT:    fsubp %st, %st(1)
 ; X87-NEXT:    retl
 ;
+; X86-SSE-LABEL: complex_canonicalize_fmul_x86_fp80:
+; X86-SSE:       # %bb.0: # %entry
+; X86-SSE-NEXT:    fldt {{[0-9]+}}(%esp)
+; X86-SSE-NEXT:    fldt {{[0-9]+}}(%esp)
+; X86-SSE-NEXT:    fsub %st(1), %st
+; X86-SSE-NEXT:    fld %st(0)
+; X86-SSE-NEXT:    fadd %st(2), %st
+; X86-SSE-NEXT:    fsubp %st, %st(1)
+; X86-SSE-NEXT:    fld1
+; X86-SSE-NEXT:    fmulp %st, %st(1)
+; X86-SSE-NEXT:    fsubp %st, %st(1)
+; X86-SSE-NEXT:    retl
+;
+; X86-AVX-LABEL: complex_canonicalize_fmul_x86_fp80:
+; X86-AVX:       # %bb.0: # %entry
+; X86-AVX-NEXT:    fldt {{[0-9]+}}(%esp)
+; X86-AVX-NEXT:    fldt {{[0-9]+}}(%esp)
+; X86-AVX-NEXT:    fsub %st(1), %st
+; X86-AVX-NEXT:    fld %st(0)
+; X86-AVX-NEXT:    fadd %st(2), %st
+; X86-AVX-NEXT:    fsubp %st, %st(1)
+; X86-AVX-NEXT:    fld1
+; X86-AVX-NEXT:    fmulp %st, %st(1)
+; X86-AVX-NEXT:    fsubp %st, %st(1)
+; X86-AVX-NEXT:    retl
+;
 ; SSE-LABEL: complex_canonicalize_fmul_x86_fp80:
 ; SSE:       # %bb.0: # %entry
 ; SSE-NEXT:    fldt {{[0-9]+}}(%rsp)
@@ -130,6 +196,54 @@ define double @canonicalize_fp64(double %a, double %b) unnamed_addr #0 {
 ; X87-NEXT:    fmulp %st, %st(1)
 ; X87-NEXT:    retl
 ;
+; X86-SSE-LABEL: canonicalize_fp64:
+; X86-SSE:       # %bb.0: # %start
+; X86-SSE-NEXT:    pushl %ebp
+; X86-SSE-NEXT:    .cfi_def_cfa_offset 8
+; X86-SSE-NEXT:    .cfi_offset %ebp, -8
+; X86-SSE-NEXT:    movl %esp, %ebp
+; X86-SSE-NEXT:    .cfi_def_cfa_register %ebp
+; X86-SSE-NEXT:    andl $-8, %esp
+; X86-SSE-NEXT:    subl $8, %esp
+; X86-SSE-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; X86-SSE-NEXT:    movsd {{.*#+}} xmm1 = mem[0],zero
+; X86-SSE-NEXT:    movapd %xmm0, %xmm2
+; X86-SSE-NEXT:    cmpunordsd %xmm0, %xmm2
+; X86-SSE-NEXT:    movapd %xmm2, %xmm3
+; X86-SSE-NEXT:    andpd %xmm1, %xmm3
+; X86-SSE-NEXT:    maxsd %xmm0, %xmm1
+; X86-SSE-NEXT:    andnpd %xmm1, %xmm2
+; X86-SSE-NEXT:    orpd %xmm3, %xmm2
+; X86-SSE-NEXT:    mulsd {{\.?LCPI[0-9]+_[0-9]+}}, %xmm2
+; X86-SSE-NEXT:    movsd %xmm2, (%esp)
+; X86-SSE-NEXT:    fldl (%esp)
+; X86-SSE-NEXT:    movl %ebp, %esp
+; X86-SSE-NEXT:    popl %ebp
+; X86-SSE-NEXT:    .cfi_def_cfa %esp, 4
+; X86-SSE-NEXT:    retl
+;
+; X86-AVX-LABEL: canonicalize_fp64:
+; X86-AVX:       # %bb.0: # %start
+; X86-AVX-NEXT:    pushl %ebp
+; X86-AVX-NEXT:    .cfi_def_cfa_offset 8
+; X86-AVX-NEXT:    .cfi_offset %ebp, -8
+; X86-AVX-NEXT:    movl %esp, %ebp
+; X86-AVX-NEXT:    .cfi_def_cfa_register %ebp
+; X86-AVX-NEXT:    andl $-8, %esp
+; X86-AVX-NEXT:    subl $8, %esp
+; X86-AVX-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
+; X86-AVX-NEXT:    vmovsd {{.*#+}} xmm1 = mem[0],zero
+; X86-AVX-NEXT:    vmaxsd %xmm0, %xmm1, %xmm2
+; X86-AVX-NEXT:    vcmpunordsd %xmm0, %xmm0, %xmm0
+; X86-AVX-NEXT:    vblendvpd %xmm0, %xmm1, %xmm2, %xmm0
+; X86-AVX-NEXT:    vmulsd {{\.?LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0
+; X86-AVX-NEXT:    vmovsd %xmm0, (%esp)
+; X86-AVX-NEXT:    fldl (%esp)
+; X86-AVX-NEXT:    movl %ebp, %esp
+; X86-AVX-NEXT:    popl %ebp
+; X86-AVX-NEXT:    .cfi_def_cfa %esp, 4
+; X86-AVX-NEXT:    retl
+;
 ; SSE-LABEL: canonicalize_fp64:
 ; SSE:       # %bb.0: # %start
 ; SSE-NEXT:    movapd %xmm0, %xmm2
@@ -207,6 +321,42 @@ define float @canonicalize_fp32(float %aa, float %bb) unnamed_addr #0 {
 ; X87-NEXT:    fmulp %st, %st(1)
 ; X87-NEXT:    retl
 ;
+; X86-SSE-LABEL: canonicalize_fp32:
+; X86-SSE:       # %bb.0: # %start
+; X86-SSE-NEXT:    pushl %eax
+; X86-SSE-NEXT:    .cfi_def_cfa_offset 8
+; X86-SSE-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; X86-SSE-NEXT:    movss {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; X86-SSE-NEXT:    movaps %xmm0, %xmm2
+; X86-SSE-NEXT:    cmpunordss %xmm0, %xmm2
+; X86-SSE-NEXT:    movaps %xmm2, %xmm3
+; X86-SSE-NEXT:    andps %xmm1, %xmm3
+; X86-SSE-NEXT:    maxss %xmm0, %xmm1
+; X86-SSE-NEXT:    andnps %xmm1, %xmm2
+; X86-SSE-NEXT:    orps %xmm3, %xmm2
+; X86-SSE-NEXT:    mulss {{\.?LCPI[0-9]+_[0-9]+}}, %xmm2
+; X86-SSE-NEXT:    movss %xmm2, (%esp)
+; X86-SSE-NEXT:    flds (%esp)
+; X86-SSE-NEXT:    popl %eax
+; X86-SSE-NEXT:    .cfi_def_cfa_offset 4
+; X86-SSE-NEXT:    retl
+;
+; X86-AVX-LABEL: canonicalize_fp32:
+; X86-AVX:       # %bb.0: # %start
+; X86-AVX-NEXT:    pushl %eax
+; X86-AVX-NEXT:    .cfi_def_cfa_offset 8
+; X86-AVX-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; X86-AVX-NEXT:    vmovss {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; X86-AVX-NEXT:    vmaxss %xmm0, %xmm1, %xmm2
+; X86-AVX-NEXT:    vcmpunordss %xmm0, %xmm0, %xmm0
+; X86-AVX-NEXT:    vblendvps %xmm0, %xmm1, %xmm2, %xmm0
+; X86-AVX-NEXT:    vmulss {{\.?LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0
+; X86-AVX-NEXT:    vmovss %xmm0, (%esp)
+; X86-AVX-NEXT:    flds (%esp)
+; X86-AVX-NEXT:    popl %eax
+; X86-AVX-NEXT:    .cfi_def_cfa_offset 4
+; X86-AVX-NEXT:    retl
+;
 ; SSE-LABEL: canonicalize_fp32:
 ; SSE:       # %bb.0: # %start
 ; SSE-NEXT:    movaps %xmm0, %xmm2
@@ -261,6 +411,22 @@ define void @v_test_canonicalize_var_f32(float addrspace(1)* %out) #1 {
 ; X87-NEXT:    fstps (%eax)
 ; X87-NEXT:    retl
 ;
+; X86-SSE-LABEL: v_test_canonicalize_var_f32:
+; X86-SSE:       # %bb.0:
+; X86-SSE-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-SSE-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; X86-SSE-NEXT:    mulss {{\.?LCPI[0-9]+_[0-9]+}}, %xmm0
+; X86-SSE-NEXT:    movss %xmm0, (%eax)
+; X86-SSE-NEXT:    retl
+;
+; X86-AVX-LABEL: v_test_canonicalize_var_f32:
+; X86-AVX:       # %bb.0:
+; X86-AVX-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-AVX-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; X86-AVX-NEXT:    vmulss {{\.?LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0
+; X86-AVX-NEXT:    vmovss %xmm0, (%eax)
+; X86-AVX-NEXT:    retl
+;
 ; SSE-LABEL: v_test_canonicalize_var_f32:
 ; SSE:       # %bb.0:
 ; SSE-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
@@ -290,6 +456,24 @@ define void @v_test_canonicalize_x86_fp80(x86_fp80 addrspace(1)* %out) #1 {
 ; X87-NEXT:    fstpt (%eax)
 ; X87-NEXT:    retl
 ;
+; X86-SSE-LABEL: v_test_canonicalize_x86_fp80:
+; X86-SSE:       # %bb.0:
+; X86-SSE-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-SSE-NEXT:    fldt (%eax)
+; X86-SSE-NEXT:    fld1
+; X86-SSE-NEXT:    fmulp %st, %st(1)
+; X86-SSE-NEXT:    fstpt (%eax)
+; X86-SSE-NEXT:    retl
+;
+; X86-AVX-LABEL: v_test_canonicalize_x86_fp80:
+; X86-AVX:       # %bb.0:
+; X86-AVX-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-AVX-NEXT:    fldt (%eax)
+; X86-AVX-NEXT:    fld1
+; X86-AVX-NEXT:    fmulp %st, %st(1)
+; X86-AVX-NEXT:    fstpt (%eax)
+; X86-AVX-NEXT:    retl
+;
 ; SSE-LABEL: v_test_canonicalize_x86_fp80:
 ; SSE:       # %bb.0:
 ; SSE-NEXT:    fldt (%rdi)
@@ -320,6 +504,22 @@ define void @v_test_canonicalize_var_f64(double addrspace(1)* %out) #1 {
 ; X87-NEXT:    fstpl (%eax)
 ; X87-NEXT:    retl
 ;
+; X86-SSE-LABEL: v_test_canonicalize_var_f64:
+; X86-SSE:       # %bb.0:
+; X86-SSE-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-SSE-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; X86-SSE-NEXT:    mulsd {{\.?LCPI[0-9]+_[0-9]+}}, %xmm0
+; X86-SSE-NEXT:    movsd %xmm0, (%eax)
+; X86-SSE-NEXT:    retl
+;
+; X86-AVX-LABEL: v_test_canonicalize_var_f64:
+; X86-AVX:       # %bb.0:
+; X86-AVX-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-AVX-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
+; X86-AVX-NEXT:    vmulsd {{\.?LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0
+; X86-AVX-NEXT:    vmovsd %xmm0, (%eax)
+; X86-AVX-NEXT:    retl
+;
 ; SSE-LABEL: v_test_canonicalize_var_f64:
 ; SSE:       # %bb.0:
 ; SSE-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
@@ -347,6 +547,20 @@ define void @canonicalize_undef(double addrspace(1)* %out) {
 ; X87-NEXT:    movl $0, (%eax)
 ; X87-NEXT:    retl
 ;
+; X86-SSE-LABEL: canonicalize_undef:
+; X86-SSE:       # %bb.0:
+; X86-SSE-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-SSE-NEXT:    movl $2146959360, 4(%eax) # imm = 0x7FF80000
+; X86-SSE-NEXT:    movl $0, (%eax)
+; X86-SSE-NEXT:    retl
+;
+; X86-AVX-LABEL: canonicalize_undef:
+; X86-AVX:       # %bb.0:
+; X86-AVX-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-AVX-NEXT:    movl $2146959360, 4(%eax) # imm = 0x7FF80000
+; X86-AVX-NEXT:    movl $0, (%eax)
+; X86-AVX-NEXT:    retl
+;
 ; SSE-LABEL: canonicalize_undef:
 ; SSE:       # %bb.0:
 ; SSE-NEXT:    movabsq $9221120237041090560, %rax # imm = 0x7FF8000000000000
@@ -384,6 +598,16 @@ define <4 x float> @canon_fp32_varargsv4f32(<4 x float> %a) {
 ; X87-NEXT:    fstps (%eax)
 ; X87-NEXT:    retl $4
 ;
+; X86-SSE-LABEL: canon_fp32_varargsv4f32:
+; X86-SSE:       # %bb.0:
+; X86-SSE-NEXT:    mulps {{\.?LCPI[0-9]+_[0-9]+}}, %xmm0
+; X86-SSE-NEXT:    retl
+;
+; X86-AVX-LABEL: canon_fp32_varargsv4f32:
+; X86-AVX:       # %bb.0:
+; X86-AVX-NEXT:    vmulps {{\.?LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0
+; X86-AVX-NEXT:    retl
+;
 ; SSE-LABEL: canon_fp32_varargsv4f32:
 ; SSE:       # %bb.0:
 ; SSE-NEXT:    mulps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
@@ -430,6 +654,18 @@ define <4 x double> @canon_fp64_varargsv4f64(<4 x double> %a) {
 ; X87-NEXT:    fstpl (%eax)
 ; X87-NEXT:    retl $4
 ;
+; X86-SSE-LABEL: canon_fp64_varargsv4f64:
+; X86-SSE:       # %bb.0:
+; X86-SSE-NEXT:    movapd {{.*#+}} xmm2 = [1.0E+0,1.0E+0]
+; X86-SSE-NEXT:    mulpd %xmm2, %xmm0
+; X86-SSE-NEXT:    mulpd %xmm2, %xmm1
+; X86-SSE-NEXT:    retl
+;
+; X86-AVX-LABEL: canon_fp64_varargsv4f64:
+; X86-AVX:       # %bb.0:
+; X86-AVX-NEXT:    vmulpd {{\.?LCPI[0-9]+_[0-9]+}}, %ymm0, %ymm0
+; X86-AVX-NEXT:    retl
+;
 ; SSE-LABEL: canon_fp64_varargsv4f64:
 ; SSE:       # %bb.0:
 ; SSE-NEXT:    movapd {{.*#+}} xmm2 = [1.0E+0,1.0E+0]
@@ -468,6 +704,26 @@ define <2 x x86_fp80> @canon_fp80_varargsv2fp80(<2 x x86_fp80> %a) {
 ; X87-NEXT:    fxch %st(1)
 ; X87-NEXT:    retl
 ;
+; X86-SSE-LABEL: canon_fp80_varargsv2fp80:
+; X86-SSE:       # %bb.0:
+; X86-SSE-NEXT:    fldt {{[0-9]+}}(%esp)
+; X86-SSE-NEXT:    fldt {{[0-9]+}}(%esp)
+; X86-SSE-NEXT:    fld1
+; X86-SSE-NEXT:    fmul %st, %st(1)
+; X86-SSE-NEXT:    fmulp %st, %st(2)
+; X86-SSE-NEXT:    fxch %st(1)
+; X86-SSE-NEXT:    retl
+;
+; X86-AVX-LABEL: canon_fp80_varargsv2fp80:
+; X86-AVX:       # %bb.0:
+; X86-AVX-NEXT:    fldt {{[0-9]+}}(%esp)
+; X86-AVX-NEXT:    fldt {{[0-9]+}}(%esp)
+; X86-AVX-NEXT:    fld1
+; X86-AVX-NEXT:    fmul %st, %st(1)
+; X86-AVX-NEXT:    fmulp %st, %st(2)
+; X86-AVX-NEXT:    fxch %st(1)
+; X86-AVX-NEXT:    retl
+;
 ; SSE-LABEL: canon_fp80_varargsv2fp80:
 ; SSE:       # %bb.0:
 ; SSE-NEXT:    fldt {{[0-9]+}}(%rsp)
@@ -512,6 +768,22 @@ define void @vec_canonicalize_var_v4f32(<4 x float> addrspace(1)* %out) #1 {
 ; X87-NEXT:    fstps (%eax)
 ; X87-NEXT:    retl
 ;
+; X86-SSE-LABEL: vec_canonicalize_var_v4f32:
+; X86-SSE:       # %bb.0:
+; X86-SSE-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-SSE-NEXT:    movaps (%eax), %xmm0
+; X86-SSE-NEXT:    mulps {{\.?LCPI[0-9]+_[0-9]+}}, %xmm0
+; X86-SSE-NEXT:    movaps %xmm0, (%eax)
+; X86-SSE-NEXT:    retl
+;
+; X86-AVX-LABEL: vec_canonicalize_var_v4f32:
+; X86-AVX:       # %bb.0:
+; X86-AVX-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-AVX-NEXT:    vmovaps (%eax), %xmm0
+; X86-AVX-NEXT:    vmulps {{\.?LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0
+; X86-AVX-NEXT:    vmovaps %xmm0, (%eax)
+; X86-AVX-NEXT:    retl
+;
 ; SSE-LABEL: vec_canonicalize_var_v4f32:
 ; SSE:       # %bb.0:
 ; SSE-NEXT:    movaps (%rdi), %xmm0
@@ -566,6 +838,26 @@ define void @vec_canonicalize_var_v4f64(<4 x double> addrspace(1)* %out) #1 {
 ; X87-NEXT:    fstpl (%eax)
 ; X87-NEXT:    retl
 ;
+; X86-SSE-LABEL: vec_canonicalize_var_v4f64:
+; X86-SSE:       # %bb.0:
+; X86-SSE-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-SSE-NEXT:    movapd {{.*#+}} xmm0 = [1.0E+0,1.0E+0]
+; X86-SSE-NEXT:    movapd 16(%eax), %xmm1
+; X86-SSE-NEXT:    mulpd %xmm0, %xmm1
+; X86-SSE-NEXT:    mulpd (%eax), %xmm0
+; X86-SSE-NEXT:    movapd %xmm0, (%eax)
+; X86-SSE-NEXT:    movapd %xmm1, 16(%eax)
+; X86-SSE-NEXT:    retl
+;
+; X86-AVX-LABEL: vec_canonicalize_var_v4f64:
+; X86-AVX:       # %bb.0:
+; X86-AVX-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-AVX-NEXT:    vmovapd (%eax), %ymm0
+; X86-AVX-NEXT:    vmulpd {{\.?LCPI[0-9]+_[0-9]+}}, %ymm0, %ymm0
+; X86-AVX-NEXT:    vmovapd %ymm0, (%eax)
+; X86-AVX-NEXT:    vzeroupper
+; X86-AVX-NEXT:    retl
+;
 ; SSE-LABEL: vec_canonicalize_var_v4f64:
 ; SSE:       # %bb.0:
 ; SSE-NEXT:    movapd {{.*#+}} xmm0 = [1.0E+0,1.0E+0]
@@ -626,6 +918,46 @@ define void @vec_canonicalize_x86_fp80(<4 x x86_fp80> addrspace(1)* %out) #1 {
 ; X87-NEXT:    fstpt (%eax)
 ; X87-NEXT:    retl
 ;
+; X86-SSE-LABEL: vec_canonicalize_x86_fp80:
+; X86-SSE:       # %bb.0:
+; X86-SSE-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-SSE-NEXT:    fldt 30(%eax)
+; X86-SSE-NEXT:    fldt 20(%eax)
+; X86-SSE-NEXT:    fldt 10(%eax)
+; X86-SSE-NEXT:    fldt (%eax)
+; X86-SSE-NEXT:    fld1
+; X86-SSE-NEXT:    fmul %st, %st(1)
+; X86-SSE-NEXT:    fmul %st, %st(2)
+; X86-SSE-NEXT:    fmul %st, %st(3)
+; X86-SSE-NEXT:    fmulp %st, %st(4)
+; X86-SSE-NEXT:    fxch %st(3)
+; X86-SSE-NEXT:    fstpt 30(%eax)
+; X86-SSE-NEXT:    fxch %st(1)
+; X86-SSE-NEXT:    fstpt 20(%eax)
+; X86-SSE-NEXT:    fstpt 10(%eax)
+; X86-SSE-NEXT:    fstpt (%eax)
+; X86-SSE-NEXT:    retl
+;
+; X86-AVX-LABEL: vec_canonicalize_x86_fp80:
+; X86-AVX:       # %bb.0:
+; X86-AVX-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-AVX-NEXT:    fldt 30(%eax)
+; X86-AVX-NEXT:    fldt 20(%eax)
+; X86-AVX-NEXT:    fldt 10(%eax)
+; X86-AVX-NEXT:    fldt (%eax)
+; X86-AVX-NEXT:    fld1
+; X86-AVX-NEXT:    fmul %st, %st(1)
+; X86-AVX-NEXT:    fmul %st, %st(2)
+; X86-AVX-NEXT:    fmul %st, %st(3)
+; X86-AVX-NEXT:    fmulp %st, %st(4)
+; X86-AVX-NEXT:    fxch %st(3)
+; X86-AVX-NEXT:    fstpt 30(%eax)
+; X86-AVX-NEXT:    fxch %st(1)
+; X86-AVX-NEXT:    fstpt 20(%eax)
+; X86-AVX-NEXT:    fstpt 10(%eax)
+; X86-AVX-NEXT:    fstpt (%eax)
+; X86-AVX-NEXT:    retl
+;
 ; SSE-LABEL: vec_canonicalize_x86_fp80:
 ; SSE:       # %bb.0:
 ; SSE-NEXT:    fldt 30(%rdi)
@@ -668,5 +1000,3 @@ define void @vec_canonicalize_x86_fp80(<4 x x86_fp80> addrspace(1)* %out) #1 {
   store <4 x x86_fp80> %canonicalized, <4 x x86_fp80> addrspace(1)* %out
   ret void
 }
-;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
-; SSE2: {{.*}}
diff --git a/llvm/test/CodeGen/X86/catchpad-realign-savexmm.ll b/llvm/test/CodeGen/X86/catchpad-realign-savexmm.ll
index 84267acf3db21..7e2651da43c31 100644
--- a/llvm/test/CodeGen/X86/catchpad-realign-savexmm.ll
+++ b/llvm/test/CodeGen/X86/catchpad-realign-savexmm.ll
@@ -47,8 +47,10 @@ catch:
 ; CHECK: .Ltmp{{.*}}
 ; CHECK: .LBB{{.*}} # Block address taken
 ; CHECK: movaps  -16(%rbp), %xmm6
+; CHECK: .seh_startepilogue
 ; CHECK: addq    $64, %rsp
 ; CHECK: popq    %rbp
+; CHECK: .seh_endepilogue
 ; CHECK: retq
 ; CHECK: .seh_handlerdata
 ; CHECK: # %catch
@@ -63,6 +65,8 @@ catch:
 ; CHECK: .seh_endprologue
 ; CHECK: movapd  32(%rsp), %xmm6
 ; CHECK: leaq    .LBB0_1(%rip), %rax
+; CHECK: .seh_startepilogue
 ; CHECK: addq    $48, %rsp
 ; CHECK: popq    %rbp
+; CHECK: .seh_endepilogue
 ; CHECK: retq # CATCHRET
diff --git a/llvm/test/CodeGen/X86/cfguard-x86-64-vectorcall.ll b/llvm/test/CodeGen/X86/cfguard-x86-64-vectorcall.ll
index fbba8f4bcaaee..8dc33fb98bf0c 100644
--- a/llvm/test/CodeGen/X86/cfguard-x86-64-vectorcall.ll
+++ b/llvm/test/CodeGen/X86/cfguard-x86-64-vectorcall.ll
@@ -22,7 +22,9 @@ define void @func_cf_vector_x64(ptr %0, ptr %1) #0 {
 ; X64-NEXT:    movsd {{.*#+}} xmm3 = mem[0],zero
 ; X64-NEXT:    callq *__guard_dispatch_icall_fptr(%rip)
 ; X64-NEXT:    nop
+; X64-NEXT:    .seh_startepilogue
 ; X64-NEXT:    addq $72, %rsp
+; X64-NEXT:    .seh_endepilogue
 ; X64-NEXT:    retq
 ; X64-NEXT:    .seh_endproc
 entry:
diff --git a/llvm/test/CodeGen/X86/cleanuppad-realign.ll b/llvm/test/CodeGen/X86/cleanuppad-realign.ll
index be766425da9cf..7aad5a30522f7 100644
--- a/llvm/test/CodeGen/X86/cleanuppad-realign.ll
+++ b/llvm/test/CodeGen/X86/cleanuppad-realign.ll
@@ -74,5 +74,7 @@ ehcleanup:                                        ; preds = %entry
 ; X64: 	       andq    $-32, %rdx
 ; X64: 	       movq    %rdx, %rbx
 ; X64-NOT: 	mov{{.*}}, %rbx
+; X64:         .seh_startepilogue
 ; X64:         popq    %rbp
+; X64:         .seh_endepilogue
 ; X64:         retq                            # CLEANUPRET
diff --git a/llvm/test/CodeGen/X86/cmp-xor.ll b/llvm/test/CodeGen/X86/cmp-xor.ll
new file mode 100644
index 0000000000000..a8a7da4871ba9
--- /dev/null
+++ b/llvm/test/CodeGen/X86/cmp-xor.ll
@@ -0,0 +1,56 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=i686--| FileCheck %s --check-prefixes=X86
+; RUN: llc < %s -mtriple=x86_64--| FileCheck %s --check-prefixes=X64
+
+;
+; PR6146
+;
+
+define i32 @cmp_xor_i32(i32 %a, i32 %b, i32 %c)
+; X86-LABEL: cmp_xor_i32:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    xorl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    jne .LBB0_2
+; X86-NEXT:  # %bb.1:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:  .LBB0_2:
+; X86-NEXT:    retl
+;
+; X64-LABEL: cmp_xor_i32:
+; X64:       # %bb.0:
+; X64-NEXT:    movl %edi, %eax
+; X64-NEXT:    xorl %esi, %eax
+; X64-NEXT:    cmovel %edx, %eax
+; X64-NEXT:    retq
+{
+  %xor = xor i32 %a, %b
+  %cmp = icmp eq i32 %a, %b
+  %sel = select i1 %cmp, i32 %c, i32 %xor
+  ret i32 %sel
+}
+
+define i32 @cmp_xor_i32_commute(i32 %a, i32 %b, i32 %c)
+; X86-LABEL: cmp_xor_i32_commute:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    xorl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    jne .LBB1_2
+; X86-NEXT:  # %bb.1:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:  .LBB1_2:
+; X86-NEXT:    retl
+;
+; X64-LABEL: cmp_xor_i32_commute:
+; X64:       # %bb.0:
+; X64-NEXT:    movl %edi, %eax
+; X64-NEXT:    xorl %esi, %eax
+; X64-NEXT:    cmovel %edx, %eax
+; X64-NEXT:    retq
+{
+  %xor = xor i32 %b, %a
+  %cmp = icmp eq i32 %a, %b
+  %sel = select i1 %cmp, i32 %c, i32 %xor
+  ret i32 %sel
+}
+
diff --git a/llvm/test/CodeGen/X86/coalesce-dead-lanes.mir b/llvm/test/CodeGen/X86/coalesce-dead-lanes.mir
index f599ed00a3e5c..e4af5995c3a53 100644
--- a/llvm/test/CodeGen/X86/coalesce-dead-lanes.mir
+++ b/llvm/test/CodeGen/X86/coalesce-dead-lanes.mir
@@ -1,4 +1,5 @@
 # RUN: llc -run-pass register-coalescer -O0 -mtriple x86_64-pc-linux-gnu -o - %s | FileCheck %s
+# RUN: llc -passes register-coalescer -O0 -mtriple x86_64-pc-linux-gnu -o - %s | FileCheck %s
 
 ---
 name:            foo
diff --git a/llvm/test/CodeGen/X86/codegen-prepare-addrmode-sext.ll b/llvm/test/CodeGen/X86/codegen-prepare-addrmode-sext.ll
index f3070cd55903b..3d6b96cd2aa10 100644
--- a/llvm/test/CodeGen/X86/codegen-prepare-addrmode-sext.ll
+++ b/llvm/test/CodeGen/X86/codegen-prepare-addrmode-sext.ll
@@ -410,7 +410,7 @@ end:
 ; Use it at the starting point for the matching.
 define signext i16 @fn3(ptr nocapture readonly %P) {
 ; CHECK-LABEL: define signext i16 @fn3(
-; CHECK-SAME: ptr nocapture readonly [[P:%.*]]) {
+; CHECK-SAME: ptr readonly captures(none) [[P:%.*]]) {
 ; CHECK-NEXT:  [[ENTRY:.*]]:
 ; CHECK-NEXT:    br label %[[WHILE_BODY_I_I:.*]]
 ; CHECK:       [[WHILE_BODY_I_I]]:
@@ -620,3 +620,51 @@ define i8 @oneArgPromotionBlockSExtZExt(i1 %arg1, ptr %base) {
   %res = load i8, ptr %arrayidx
   ret i8 %res
 }
+
+; Check that we replace the deleted sext with the promoted value.
+define void @pr70938(ptr %f) {
+; CHECK-LABEL: define void @pr70938(
+; CHECK-SAME: ptr [[F:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[ADD:%.*]] = add nsw i64 0, 1
+; CHECK-NEXT:    [[SUNKADDR:%.*]] = mul i64 [[ADD]], 2
+; CHECK-NEXT:    [[SUNKADDR1:%.*]] = getelementptr i8, ptr [[F]], i64 [[SUNKADDR]]
+; CHECK-NEXT:    [[SUNKADDR2:%.*]] = getelementptr i8, ptr [[SUNKADDR1]], i64 1
+; CHECK-NEXT:    store i8 0, ptr [[SUNKADDR2]], align 1
+; CHECK-NEXT:    ret void
+;
+entry:
+  %add = add nsw i32 0, 1
+  %idxprom3 = sext i32 %add to i64
+  %arrayidx4 = getelementptr [2 x [1 x [2 x i8]]], ptr %f, i64 0, i64 %idxprom3
+  %arrayidx8 = getelementptr [2 x i8], ptr %arrayidx4, i64 0, i64 %idxprom3
+  br label %if.end
+
+if.end:                                           ; preds = %entry
+  store i8 0, ptr %arrayidx8, align 1
+  ret void
+}
+
+define void @pr119429() {
+; CHECK-LABEL: define void @pr119429() {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[AND:%.*]] = and i64 0, 0
+; CHECK-NEXT:    [[SUNKADDR:%.*]] = inttoptr i64 [[AND]] to ptr
+; CHECK-NEXT:    [[SUNKADDR1:%.*]] = mul i64 [[AND]], 2
+; CHECK-NEXT:    [[SUNKADDR2:%.*]] = getelementptr i8, ptr [[SUNKADDR]], i64 [[SUNKADDR1]]
+; CHECK-NEXT:    store i64 0, ptr [[SUNKADDR2]], align 8
+; CHECK-NEXT:    ret void
+;
+entry:
+  %and = and i32 0, 0
+  %conv1 = zext i32 %and to i64
+  %sub = add i64 %conv1, 0
+  br label %if.end
+
+if.end:
+  %mul = shl i64 %sub, 1
+  %add = add i64 %mul, %conv1
+  %ptr = inttoptr i64 %add to ptr
+  store i64 0, ptr %ptr, align 8
+  ret void
+}
diff --git a/llvm/test/CodeGen/X86/combine-add.ll b/llvm/test/CodeGen/X86/combine-add.ll
index 01a0320bc6b2f..ff9f995c4765b 100644
--- a/llvm/test/CodeGen/X86/combine-add.ll
+++ b/llvm/test/CodeGen/X86/combine-add.ll
@@ -501,10 +501,8 @@ define i1 @PR51238(i1 %b, i8 %x, i8 %y, i8 %z) {
 ; CHECK-LABEL: PR51238:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    notb %cl
-; CHECK-NEXT:    xorl %eax, %eax
 ; CHECK-NEXT:    addb %dl, %cl
-; CHECK-NEXT:    adcb $1, %al
-; CHECK-NEXT:    # kill: def $al killed $al killed $eax
+; CHECK-NEXT:    setae %al
 ; CHECK-NEXT:    retq
    %ny = xor i8 %y, -1
    %nz = xor i8 %z, -1
diff --git a/llvm/test/CodeGen/X86/conditional-tailcall-pgso.ll b/llvm/test/CodeGen/X86/conditional-tailcall-pgso.ll
index 396e0e7b06570..b3d5d836874cb 100644
--- a/llvm/test/CodeGen/X86/conditional-tailcall-pgso.ll
+++ b/llvm/test/CodeGen/X86/conditional-tailcall-pgso.ll
@@ -114,13 +114,17 @@ define void @f_non_leaf(i32 %x, i32 %y) !prof !14 {
 ; WIN64-NEXT:    jne .LBB1_2 # encoding: [0x75,A]
 ; WIN64-NEXT:    # fixup A - offset: 1, value: .LBB1_2-1, kind: FK_PCRel_1
 ; WIN64-NEXT:  # %bb.1: # %bb1
+; WIN64-NEXT:    .seh_startepilogue
 ; WIN64-NEXT:    popq %rbx # encoding: [0x5b]
+; WIN64-NEXT:    .seh_endepilogue
 ; WIN64-NEXT:    jmp foo # TAILCALL
 ; WIN64-NEXT:    # encoding: [0xeb,A]
 ; WIN64-NEXT:    # fixup A - offset: 1, value: foo-1, kind: FK_PCRel_1
 ; WIN64-NEXT:  .LBB1_2: # %bb2
 ; WIN64-NEXT:    nop # encoding: [0x90]
+; WIN64-NEXT:    .seh_startepilogue
 ; WIN64-NEXT:    popq %rbx # encoding: [0x5b]
+; WIN64-NEXT:    .seh_endepilogue
 ; WIN64-NEXT:    jmp bar # TAILCALL
 ; WIN64-NEXT:    # encoding: [0xeb,A]
 ; WIN64-NEXT:    # fixup A - offset: 1, value: bar-1, kind: FK_PCRel_1
diff --git a/llvm/test/CodeGen/X86/conditional-tailcall.ll b/llvm/test/CodeGen/X86/conditional-tailcall.ll
index 9e0a19f9a504f..4c990d81810be 100644
--- a/llvm/test/CodeGen/X86/conditional-tailcall.ll
+++ b/llvm/test/CodeGen/X86/conditional-tailcall.ll
@@ -114,13 +114,17 @@ define void @f_non_leaf(i32 %x, i32 %y) optsize {
 ; WIN64-NEXT:    jne .LBB1_2 # encoding: [0x75,A]
 ; WIN64-NEXT:    # fixup A - offset: 1, value: .LBB1_2-1, kind: FK_PCRel_1
 ; WIN64-NEXT:  # %bb.1: # %bb1
+; WIN64-NEXT:    .seh_startepilogue
 ; WIN64-NEXT:    popq %rbx # encoding: [0x5b]
+; WIN64-NEXT:    .seh_endepilogue
 ; WIN64-NEXT:    jmp foo # TAILCALL
 ; WIN64-NEXT:    # encoding: [0xeb,A]
 ; WIN64-NEXT:    # fixup A - offset: 1, value: foo-1, kind: FK_PCRel_1
 ; WIN64-NEXT:  .LBB1_2: # %bb2
 ; WIN64-NEXT:    nop # encoding: [0x90]
+; WIN64-NEXT:    .seh_startepilogue
 ; WIN64-NEXT:    popq %rbx # encoding: [0x5b]
+; WIN64-NEXT:    .seh_endepilogue
 ; WIN64-NEXT:    jmp bar # TAILCALL
 ; WIN64-NEXT:    # encoding: [0xeb,A]
 ; WIN64-NEXT:    # fixup A - offset: 1, value: bar-1, kind: FK_PCRel_1
diff --git a/llvm/test/CodeGen/X86/copy-low-subvec-elt-to-high-subvec-elt.ll b/llvm/test/CodeGen/X86/copy-low-subvec-elt-to-high-subvec-elt.ll
index 1baaab0931cb9..26a88ab15e3cc 100644
--- a/llvm/test/CodeGen/X86/copy-low-subvec-elt-to-high-subvec-elt.ll
+++ b/llvm/test/CodeGen/X86/copy-low-subvec-elt-to-high-subvec-elt.ll
@@ -151,8 +151,8 @@ define <4 x double> @vec256_eltty_double_source_subvec_0_target_subvec_mask_2_un
 define <4 x double> @vec256_eltty_double_source_subvec_0_target_subvec_mask_2_binary(<4 x double> %x, <4 x double> %y) nounwind {
 ; CHECK-LABEL: vec256_eltty_double_source_subvec_0_target_subvec_mask_2_binary:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm1
-; CHECK-NEXT:    vshufpd {{.*#+}} ymm0 = ymm0[0],ymm1[1],ymm0[2],ymm1[2]
+; CHECK-NEXT:    vbroadcastsd %xmm1, %ymm1
+; CHECK-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm1[6,7]
 ; CHECK-NEXT:    retq
   %r = shufflevector <4 x double> %x, <4 x double> %y, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
   ret <4 x double> %r
diff --git a/llvm/test/CodeGen/X86/data-section-prefix.ll b/llvm/test/CodeGen/X86/data-section-prefix.ll
new file mode 100644
index 0000000000000..4812fc70758fb
--- /dev/null
+++ b/llvm/test/CodeGen/X86/data-section-prefix.ll
@@ -0,0 +1,27 @@
+; RUN: llc -mtriple x86_64-linux-gnu -data-sections %s -o - | FileCheck %s --check-prefix=ELF
+; RUN: llc -mtriple x86_64-linux-gnu -unique-section-names=0 -data-sections %s -o - | FileCheck %s --check-prefix=ELF-NOUNIQ
+
+; RUN: llc -mtriple x86_64-windows-msvc -data-sections %s -o - | FileCheck %s --check-prefix=COFF-MSVC
+
+; ELF: .section .data.hot.foo,
+; ELF: .section .data.bar,
+; ELF: .section .bss.unlikely.baz,
+; ELF: .section .bss.quz,
+
+; ELF-NOUNIQ: .section    .data.hot.,"aw",@progbits,unique,1
+; ELF-NOUNIQ: .section    .data,"aw",@progbits,unique,2
+; ELF-NOUNIQ: .section    .bss.unlikely.,"aw",@nobits,unique,3
+; ELF-NOUNIQ: .section    .bss,"aw",@nobits,unique,4
+
+; COFF-MSVC: .section .data,"dw",one_only,foo
+; COFF-MSVC: .section .data,"dw",one_only,bar
+; COFF-MSVC: .section .bss,"bw",one_only,baz
+; COFF-MSVC: .section .bss,"bw",one_only,quz
+
+@foo = global i32 1, !section_prefix !0
+@bar = global i32 2
+@baz = global i32 0, !section_prefix !1
+@quz = global i32 0
+
+!0 = !{!"section_prefix", !"hot"}
+!1 = !{!"section_prefix", !"unlikely"}
diff --git a/llvm/test/CodeGen/X86/div-rem-pair-recomposition-signed.ll b/llvm/test/CodeGen/X86/div-rem-pair-recomposition-signed.ll
index f66d81c781fe0..880062bbc4f9e 100644
--- a/llvm/test/CodeGen/X86/div-rem-pair-recomposition-signed.ll
+++ b/llvm/test/CodeGen/X86/div-rem-pair-recomposition-signed.ll
@@ -200,10 +200,9 @@ define i128 @scalar_i128(i128 %x, i128 %y, ptr %divdst) nounwind {
 ; X86-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X86-NEXT:    sbbl %eax, %edi
 ; X86-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT:    movl 40(%ebp), %ecx
-; X86-NEXT:    movl %ecx, %edx
+; X86-NEXT:    movl 40(%ebp), %esi
+; X86-NEXT:    movl %esi, %edx
 ; X86-NEXT:    sarl $31, %edx
-; X86-NEXT:    movl %ecx, %esi
 ; X86-NEXT:    xorl %edx, %esi
 ; X86-NEXT:    movl 36(%ebp), %ecx
 ; X86-NEXT:    xorl %edx, %ecx
@@ -408,9 +407,9 @@ define i128 @scalar_i128(i128 %x, i128 %y, ptr %divdst) nounwind {
 ; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
 ; X86-NEXT:    adcl $-1, %eax
 ; X86-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NEXT:    adcl $-1, %eax
-; X86-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT:    adcl $-1, %ecx
+; X86-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
 ; X86-NEXT:    adcl $-1, %ecx
 ; X86-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
diff --git a/llvm/test/CodeGen/X86/dwo-stats.ll b/llvm/test/CodeGen/X86/dwo-stats.ll
new file mode 100644
index 0000000000000..fccfd55029c8b
--- /dev/null
+++ b/llvm/test/CodeGen/X86/dwo-stats.ll
@@ -0,0 +1,30 @@
+; REQUIRES: asserts
+; RUN: llc %s -mtriple=x86_64-linux --split-dwarf-file=%t.dwo --split-dwarf-output=%t.dwo --filetype=obj -o /dev/null -stats 2>&1 | FileCheck %s --check-prefixes=SPLIT,CHECK
+; RUN: llc %s -mtriple=x86_64-linux --filetype=obj -o /dev/null -stats 2>&1 | FileCheck %s --check-prefixes=NOTSPLIT,CHECK
+
+; NOTSPLIT-NOT: {{[0-9]+}} elf-object-writer - Total size of sections written to .dwo file
+; CHECK-DAG: {{[0-9]+}} elf-object-writer - Total size of debug info sections
+; SPLIT-DAG: {{[0-9]+}} elf-object-writer - Total size of sections written to .dwo file
+; NOTSPLIT-NOT: {{[0-9]+}} elf-object-writer - Total size of sections written to .dwo file
+
+define void @banana() !dbg !8 {
+  ret void, !dbg !12
+}
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!3, !4, !5, !6}
+!llvm.ident = !{!7}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 11.0.1", isOptimized: true, runtimeVersion: 0, splitDebugFilename: "test.dwo", emissionKind: FullDebug, enums: !2, splitDebugInlining: false, nameTableKind: GNU)
+!1 = !DIFile(filename: "/tmp/test.c", directory: "/tmp")
+!2 = !{}
+!3 = !{i32 7, !"Dwarf Version", i32 4}
+!4 = !{i32 2, !"Debug Info Version", i32 3}
+!5 = !{i32 1, !"wchar_size", i32 4}
+!6 = !{i32 7, !"PIC Level", i32 2}
+!7 = !{!"clang version 11.0.1"}
+!8 = distinct !DISubprogram(name: "banana", scope: !9, file: !9, line: 1, type: !10, scopeLine: 1, flags: DIFlagPrototyped | DIFlagAllCallsDescribed, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, retainedNodes: !2)
+!9 = !DIFile(filename: "test.c", directory: "/tmp")
+!10 = !DISubroutineType(types: !11)
+!11 = !{null}
+!12 = !DILocation(line: 1, column: 20, scope: !8)
diff --git a/llvm/test/CodeGen/X86/fake-use-remove-loads.mir b/llvm/test/CodeGen/X86/fake-use-remove-loads.mir
new file mode 100644
index 0000000000000..3f67f03c9a63d
--- /dev/null
+++ b/llvm/test/CodeGen/X86/fake-use-remove-loads.mir
@@ -0,0 +1,171 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 5
+# Ensure that loads into FAKE_USEs are correctly removed by the
+# remove-loads-into-fake-uses pass, and that if the function does not use
+# instruction referencing then no changes are made.
+# RUN: llc %s -run-pass remove-loads-into-fake-uses -mtriple=x86_64-unknown-linux -debug-only=remove-loads-into-fake-uses 2>&1 -o - | FileCheck %s
+# REQUIRES: asserts
+#
+## We verify that:
+##   - The load into the FAKE_USE is removed, along with the FAKE_USE itself,
+##     even when the FAKE_USE is for a subregister of the move.
+##   - We correctly handle situations where FAKE_USE has additional `killed`
+##     operands added by other passes.
+##   - The store to the stack slot still exists.
+##   - When the register has a use between the restore and the FAKE_USE, we do
+##     not delete the load or fake use.
+
+
+---
+name:            enabled
+alignment:       16
+tracksRegLiveness: true
+noPhis:          true
+noVRegs:         true
+hasFakeUses:     true
+tracksDebugUserValues: true
+debugInstrRef: true
+liveins:
+  - { reg: '$rdi', virtual-reg: '' }
+  - { reg: '$esi', virtual-reg: '' }
+  - { reg: '$rdx', virtual-reg: '' }
+frameInfo:
+  isCalleeSavedInfoValid: true
+stack:
+  - { id: 0, name: '', type: spill-slot, offset: -8, size: 8, alignment: 8,
+      stack-id: default, callee-saved-register: '', callee-saved-restored: true,
+      debug-info-variable: '', debug-info-expression: '', debug-info-location: '' }
+  - { id: 1, name: '', type: spill-slot, offset: -16, size: 8, alignment: 8,
+      stack-id: default, callee-saved-register: '', callee-saved-restored: true,
+      debug-info-variable: '', debug-info-expression: '', debug-info-location: '' }
+body:             |
+  bb.0:
+    liveins: $esi, $rdi, $rdx, $r15, $r14, $r13, $r12, $r11, $rbx
+
+    ; CHECK-LABEL: name: enabled
+    ; CHECK: liveins: $esi, $rdi, $rdx, $r15, $r14, $r13, $r12, $r11, $rbx
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: $rbx = MOV64rr $rdx
+    ; CHECK-NEXT: $r14d = MOV32rr $esi
+    ; CHECK-NEXT: $r15 = MOV64rr $rdi
+    ; CHECK-NEXT: renamable $r12d = XOR32rr undef $r12d, undef $r12d, implicit-def dead $eflags, implicit-def $r12
+    ; CHECK-NEXT: renamable $eax = XOR32rr undef $eax, undef $eax, implicit-def dead $eflags, implicit-def $rax
+
+    ;; The store to the stack slot is still present.
+    ; CHECK-NEXT: MOV64mr $rbp, 1, $noreg, -48, $noreg, killed renamable $rax :: (store (s64) into %stack.0)
+
+    ; CHECK-NEXT: MOV64mr $rbp, 1, $noreg, -40, $noreg, killed renamable $r11 :: (store (s64) into %stack.1)
+    ; CHECK-NEXT: renamable $eax = XOR32rr undef $eax, undef $eax, implicit-def dead $eflags
+    ; CHECK-NEXT: $r13d = MOV32rr killed $eax
+    ; CHECK-NEXT: $rdi = MOV64rr $r15
+    ; CHECK-NEXT: CALL64r renamable $r12, csr_64, implicit $rsp, implicit $ssp, implicit $rdi, implicit-def $rsp, implicit-def $ssp
+    ; CHECK-NEXT: dead renamable $eax = MOV32rm renamable $rbx, 1, $noreg, 0, $noreg
+    ; CHECK-NEXT: renamable $eax = MOV32ri 1
+    ; CHECK-NEXT: TEST8ri renamable $r14b, 1, implicit-def $eflags
+    
+    ;; First FAKE_USE and its corresponding load are removed; second FAKE_USE of
+    ;; a restored value that is also used is preserved.
+    ; CHECK-NEXT: renamable $r11 = MOV64rm $rbp, 1, $noreg, -40, $noreg :: (load (s64) from %stack.1)
+    ; CHECK-NEXT: renamable $r12d = XOR32rr $r12d, $r11d, implicit-def dead $eflags
+    ; CHECK-NEXT: FAKE_USE killed renamable $r11d
+
+    ; CHECK-NEXT: TEST32rr killed renamable $r13d, renamable $r13d, implicit-def $eflags
+    ; CHECK-NEXT: RET64
+
+    $rbx = MOV64rr $rdx
+    $r14d = MOV32rr $esi
+    $r15 = MOV64rr $rdi
+    renamable $r12d = XOR32rr undef $r12d, undef $r12d, implicit-def dead $eflags, implicit-def $r12
+    renamable $eax = XOR32rr undef $eax, undef $eax, implicit-def dead $eflags, implicit-def $rax
+    MOV64mr $rbp, 1, $noreg, -48, $noreg, killed renamable $rax :: (store (s64) into %stack.0)
+    MOV64mr $rbp, 1, $noreg, -40, $noreg, killed renamable $r11 :: (store (s64) into %stack.1)
+    renamable $eax = XOR32rr undef $eax, undef $eax, implicit-def dead $eflags
+    $r13d = MOV32rr killed $eax
+    $rdi = MOV64rr $r15
+    CALL64r renamable $r12, csr_64, implicit $rsp, implicit $ssp, implicit $rdi, implicit-def $rsp, implicit-def $ssp
+    dead renamable $eax = MOV32rm renamable $rbx, 1, $noreg, 0, $noreg
+    renamable $eax = MOV32ri 1
+    TEST8ri renamable $r14b, 1, implicit-def $eflags
+    renamable $rax = MOV64rm $rbp, 1, $noreg, -48, $noreg :: (load (s64) from %stack.0)
+    FAKE_USE renamable $eax, implicit killed $rax
+    renamable $r11 = MOV64rm $rbp, 1, $noreg, -40, $noreg :: (load (s64) from %stack.1)
+    renamable $r12d = XOR32rr $r12d, $r11d, implicit-def dead $eflags
+    FAKE_USE killed renamable $r11d
+    TEST32rr killed renamable $r13d, renamable $r13d, implicit-def $eflags
+    RET64
+
+...
+---
+name:            disabled
+alignment:       16
+tracksRegLiveness: true
+noPhis:          true
+noVRegs:         true
+hasFakeUses:     true
+tracksDebugUserValues: true
+debugInstrRef: false
+liveins:
+  - { reg: '$rdi', virtual-reg: '' }
+  - { reg: '$esi', virtual-reg: '' }
+  - { reg: '$rdx', virtual-reg: '' }
+frameInfo:
+  isCalleeSavedInfoValid: true
+stack:
+  - { id: 0, name: '', type: spill-slot, offset: -8, size: 8, alignment: 8,
+      stack-id: default, callee-saved-register: '', callee-saved-restored: true,
+      debug-info-variable: '', debug-info-expression: '', debug-info-location: '' }
+  - { id: 1, name: '', type: spill-slot, offset: -16, size: 8, alignment: 8,
+      stack-id: default, callee-saved-register: '', callee-saved-restored: true,
+      debug-info-variable: '', debug-info-expression: '', debug-info-location: '' }
+body:             |
+  bb.0:
+    liveins: $esi, $rdi, $rdx, $r15, $r14, $r13, $r12, $r11, $rbx
+
+    ; CHECK-LABEL: name: disabled
+    ; CHECK: liveins: $esi, $rdi, $rdx, $r15, $r14, $r13, $r12, $r11, $rbx
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: $rbx = MOV64rr $rdx
+    ; CHECK-NEXT: $r14d = MOV32rr $esi
+    ; CHECK-NEXT: $r15 = MOV64rr $rdi
+    ; CHECK-NEXT: renamable $r12d = XOR32rr undef $r12d, undef $r12d, implicit-def dead $eflags, implicit-def $r12
+    ; CHECK-NEXT: renamable $eax = XOR32rr undef $eax, undef $eax, implicit-def dead $eflags, implicit-def $rax
+    ; CHECK-NEXT: MOV64mr $rbp, 1, $noreg, -48, $noreg, killed renamable $rax :: (store (s64) into %stack.0)
+    ; CHECK-NEXT: MOV64mr $rbp, 1, $noreg, -40, $noreg, killed renamable $r11 :: (store (s64) into %stack.1)
+    ; CHECK-NEXT: renamable $eax = XOR32rr undef $eax, undef $eax, implicit-def dead $eflags
+    ; CHECK-NEXT: $r13d = MOV32rr killed $eax
+    ; CHECK-NEXT: $rdi = MOV64rr $r15
+    ; CHECK-NEXT: CALL64r renamable $r12, csr_64, implicit $rsp, implicit $ssp, implicit $rdi, implicit-def $rsp, implicit-def $ssp
+    ; CHECK-NEXT: dead renamable $eax = MOV32rm renamable $rbx, 1, $noreg, 0, $noreg
+    ; CHECK-NEXT: renamable $eax = MOV32ri 1
+    ; CHECK-NEXT: TEST8ri renamable $r14b, 1, implicit-def $eflags
+
+    ;; Verify that when instr-ref is disabled, we do not remove fake uses.
+    ; CHECK-NEXT: renamable $rax = MOV64rm $rbp, 1, $noreg, -48, $noreg :: (load (s64) from %stack.0)
+    ; CHECK-NEXT: FAKE_USE renamable $eax, implicit killed $rax
+    ; CHECK-NEXT: renamable $r11 = MOV64rm $rbp, 1, $noreg, -40, $noreg :: (load (s64) from %stack.1)
+    ; CHECK-NEXT: renamable $r12d = XOR32rr $r12d, $r11d, implicit-def dead $eflags
+    ; CHECK-NEXT: FAKE_USE killed renamable $r11d
+    ; CHECK-NEXT: TEST32rr killed renamable $r13d, renamable $r13d, implicit-def $eflags
+    ; CHECK-NEXT: RET64
+    $rbx = MOV64rr $rdx
+    $r14d = MOV32rr $esi
+    $r15 = MOV64rr $rdi
+    renamable $r12d = XOR32rr undef $r12d, undef $r12d, implicit-def dead $eflags, implicit-def $r12
+    renamable $eax = XOR32rr undef $eax, undef $eax, implicit-def dead $eflags, implicit-def $rax
+    MOV64mr $rbp, 1, $noreg, -48, $noreg, killed renamable $rax :: (store (s64) into %stack.0)
+    MOV64mr $rbp, 1, $noreg, -40, $noreg, killed renamable $r11 :: (store (s64) into %stack.1)
+    renamable $eax = XOR32rr undef $eax, undef $eax, implicit-def dead $eflags
+    $r13d = MOV32rr killed $eax
+    $rdi = MOV64rr $r15
+    CALL64r renamable $r12, csr_64, implicit $rsp, implicit $ssp, implicit $rdi, implicit-def $rsp, implicit-def $ssp
+    dead renamable $eax = MOV32rm renamable $rbx, 1, $noreg, 0, $noreg
+    renamable $eax = MOV32ri 1
+    TEST8ri renamable $r14b, 1, implicit-def $eflags
+    renamable $rax = MOV64rm $rbp, 1, $noreg, -48, $noreg :: (load (s64) from %stack.0)
+    FAKE_USE renamable $eax, implicit killed $rax
+    renamable $r11 = MOV64rm $rbp, 1, $noreg, -40, $noreg :: (load (s64) from %stack.1)
+    renamable $r12d = XOR32rr $r12d, $r11d, implicit-def dead $eflags
+    FAKE_USE killed renamable $r11d
+    TEST32rr killed renamable $r13d, renamable $r13d, implicit-def $eflags
+    RET64
+
+...
diff --git a/llvm/test/CodeGen/X86/fake-use-simple-tail-call.ll b/llvm/test/CodeGen/X86/fake-use-simple-tail-call.ll
index 45a210ef39100..fb01da9f12c8b 100644
--- a/llvm/test/CodeGen/X86/fake-use-simple-tail-call.ll
+++ b/llvm/test/CodeGen/X86/fake-use-simple-tail-call.ll
@@ -1,6 +1,6 @@
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -O2 -o - \
 ; RUN:   | FileCheck %s --implicit-check-not=TAILCALL
-; Generated with: clang -emit-llvm -O2 -S -fextend-lifetimes test.cpp -o -
+; Generated with: clang -emit-llvm -O2 -S -fextend-variable-liveness test.cpp -o -
 ; =========== test.cpp ===============
 ; extern int bar(int);
 ; int foo1(int i)
diff --git a/llvm/test/CodeGen/X86/fake-use-zero-length.ll b/llvm/test/CodeGen/X86/fake-use-zero-length.ll
index e3bdd2659dd91..71cb1029dcad6 100644
--- a/llvm/test/CodeGen/X86/fake-use-zero-length.ll
+++ b/llvm/test/CodeGen/X86/fake-use-zero-length.ll
@@ -4,7 +4,7 @@
 ; and structs. Check also that they are not propagated.
 ;
 ; Generated from the following source with
-; clang -fextend-lifetimes -S -emit-llvm -O2 -mllvm -stop-after=safe-stack -o test.mir test.cpp
+; clang -fextend-variable-liveness -S -emit-llvm -O2 -mllvm -stop-after=safe-stack -o test.mir test.cpp
 ;
 ; int main ()
 ; { int array[0]; }
diff --git a/llvm/test/CodeGen/X86/fast-isel-select.ll b/llvm/test/CodeGen/X86/fast-isel-select.ll
index 94477f3286e48..83e8d0a9d418b 100644
--- a/llvm/test/CodeGen/X86/fast-isel-select.ll
+++ b/llvm/test/CodeGen/X86/fast-isel-select.ll
@@ -11,7 +11,7 @@ define i32 @fastisel_select(i1 %exchSub2211_, i1 %trunc_8766) {
 ; CHECK:       ## %bb.0:
 ; CHECK-NEXT:    movb %sil, %al
 ; CHECK-NEXT:    movb %dil, %dl
-; CHECK-NEXT:    subb %al, %dl
+; CHECK-NEXT:    xorb %al, %dl
 ; CHECK-NEXT:    xorl %eax, %eax
 ; CHECK-NEXT:    movl $1204476887, %ecx ## imm = 0x47CADBD7
 ; CHECK-NEXT:    testb $1, %dl
diff --git a/llvm/test/CodeGen/X86/fmaxnum.ll b/llvm/test/CodeGen/X86/fmaxnum.ll
index 2e1af1e84e076..d6252cc85e8b4 100644
--- a/llvm/test/CodeGen/X86/fmaxnum.ll
+++ b/llvm/test/CodeGen/X86/fmaxnum.ll
@@ -22,17 +22,26 @@ declare <8 x double> @llvm.maxnum.v8f64(<8 x double>, <8 x double>)
 ; FIXME: As the vector tests show, the SSE run shouldn't need this many moves.
 
 define float @test_fmaxf(float %x, float %y) {
-; SSE-LABEL: test_fmaxf:
-; SSE:       # %bb.0:
-; SSE-NEXT:    movaps %xmm0, %xmm2
-; SSE-NEXT:    cmpunordss %xmm0, %xmm2
-; SSE-NEXT:    movaps %xmm2, %xmm3
-; SSE-NEXT:    andps %xmm1, %xmm3
-; SSE-NEXT:    maxss %xmm0, %xmm1
-; SSE-NEXT:    andnps %xmm1, %xmm2
-; SSE-NEXT:    orps %xmm3, %xmm2
-; SSE-NEXT:    movaps %xmm2, %xmm0
-; SSE-NEXT:    retq
+; SSE2-LABEL: test_fmaxf:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    movaps %xmm0, %xmm2
+; SSE2-NEXT:    cmpunordss %xmm0, %xmm2
+; SSE2-NEXT:    movaps %xmm2, %xmm3
+; SSE2-NEXT:    andps %xmm1, %xmm3
+; SSE2-NEXT:    maxss %xmm0, %xmm1
+; SSE2-NEXT:    andnps %xmm1, %xmm2
+; SSE2-NEXT:    orps %xmm3, %xmm2
+; SSE2-NEXT:    movaps %xmm2, %xmm0
+; SSE2-NEXT:    retq
+;
+; SSE4-LABEL: test_fmaxf:
+; SSE4:       # %bb.0:
+; SSE4-NEXT:    movaps %xmm1, %xmm2
+; SSE4-NEXT:    maxss %xmm0, %xmm2
+; SSE4-NEXT:    cmpunordss %xmm0, %xmm0
+; SSE4-NEXT:    blendvps %xmm0, %xmm1, %xmm2
+; SSE4-NEXT:    movaps %xmm2, %xmm0
+; SSE4-NEXT:    retq
 ;
 ; AVX1-LABEL: test_fmaxf:
 ; AVX1:       # %bb.0:
@@ -63,17 +72,26 @@ define float @test_fmaxf_minsize(float %x, float %y) minsize {
 ; FIXME: As the vector tests show, the SSE run shouldn't need this many moves.
 
 define double @test_fmax(double %x, double %y) {
-; SSE-LABEL: test_fmax:
-; SSE:       # %bb.0:
-; SSE-NEXT:    movapd %xmm0, %xmm2
-; SSE-NEXT:    cmpunordsd %xmm0, %xmm2
-; SSE-NEXT:    movapd %xmm2, %xmm3
-; SSE-NEXT:    andpd %xmm1, %xmm3
-; SSE-NEXT:    maxsd %xmm0, %xmm1
-; SSE-NEXT:    andnpd %xmm1, %xmm2
-; SSE-NEXT:    orpd %xmm3, %xmm2
-; SSE-NEXT:    movapd %xmm2, %xmm0
-; SSE-NEXT:    retq
+; SSE2-LABEL: test_fmax:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    movapd %xmm0, %xmm2
+; SSE2-NEXT:    cmpunordsd %xmm0, %xmm2
+; SSE2-NEXT:    movapd %xmm2, %xmm3
+; SSE2-NEXT:    andpd %xmm1, %xmm3
+; SSE2-NEXT:    maxsd %xmm0, %xmm1
+; SSE2-NEXT:    andnpd %xmm1, %xmm2
+; SSE2-NEXT:    orpd %xmm3, %xmm2
+; SSE2-NEXT:    movapd %xmm2, %xmm0
+; SSE2-NEXT:    retq
+;
+; SSE4-LABEL: test_fmax:
+; SSE4:       # %bb.0:
+; SSE4-NEXT:    movapd %xmm1, %xmm2
+; SSE4-NEXT:    maxsd %xmm0, %xmm2
+; SSE4-NEXT:    cmpunordsd %xmm0, %xmm0
+; SSE4-NEXT:    blendvpd %xmm0, %xmm1, %xmm2
+; SSE4-NEXT:    movapd %xmm2, %xmm0
+; SSE4-NEXT:    retq
 ;
 ; AVX1-LABEL: test_fmax:
 ; AVX1:       # %bb.0:
@@ -111,17 +129,26 @@ define x86_fp80 @test_fmaxl(x86_fp80 %x, x86_fp80 %y) {
 }
 
 define float @test_intrinsic_fmaxf(float %x, float %y) {
-; SSE-LABEL: test_intrinsic_fmaxf:
-; SSE:       # %bb.0:
-; SSE-NEXT:    movaps %xmm0, %xmm2
-; SSE-NEXT:    cmpunordss %xmm0, %xmm2
-; SSE-NEXT:    movaps %xmm2, %xmm3
-; SSE-NEXT:    andps %xmm1, %xmm3
-; SSE-NEXT:    maxss %xmm0, %xmm1
-; SSE-NEXT:    andnps %xmm1, %xmm2
-; SSE-NEXT:    orps %xmm3, %xmm2
-; SSE-NEXT:    movaps %xmm2, %xmm0
-; SSE-NEXT:    retq
+; SSE2-LABEL: test_intrinsic_fmaxf:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    movaps %xmm0, %xmm2
+; SSE2-NEXT:    cmpunordss %xmm0, %xmm2
+; SSE2-NEXT:    movaps %xmm2, %xmm3
+; SSE2-NEXT:    andps %xmm1, %xmm3
+; SSE2-NEXT:    maxss %xmm0, %xmm1
+; SSE2-NEXT:    andnps %xmm1, %xmm2
+; SSE2-NEXT:    orps %xmm3, %xmm2
+; SSE2-NEXT:    movaps %xmm2, %xmm0
+; SSE2-NEXT:    retq
+;
+; SSE4-LABEL: test_intrinsic_fmaxf:
+; SSE4:       # %bb.0:
+; SSE4-NEXT:    movaps %xmm1, %xmm2
+; SSE4-NEXT:    maxss %xmm0, %xmm2
+; SSE4-NEXT:    cmpunordss %xmm0, %xmm0
+; SSE4-NEXT:    blendvps %xmm0, %xmm1, %xmm2
+; SSE4-NEXT:    movaps %xmm2, %xmm0
+; SSE4-NEXT:    retq
 ;
 ; AVX1-LABEL: test_intrinsic_fmaxf:
 ; AVX1:       # %bb.0:
@@ -142,17 +169,26 @@ define float @test_intrinsic_fmaxf(float %x, float %y) {
 }
 
 define double @test_intrinsic_fmax(double %x, double %y) {
-; SSE-LABEL: test_intrinsic_fmax:
-; SSE:       # %bb.0:
-; SSE-NEXT:    movapd %xmm0, %xmm2
-; SSE-NEXT:    cmpunordsd %xmm0, %xmm2
-; SSE-NEXT:    movapd %xmm2, %xmm3
-; SSE-NEXT:    andpd %xmm1, %xmm3
-; SSE-NEXT:    maxsd %xmm0, %xmm1
-; SSE-NEXT:    andnpd %xmm1, %xmm2
-; SSE-NEXT:    orpd %xmm3, %xmm2
-; SSE-NEXT:    movapd %xmm2, %xmm0
-; SSE-NEXT:    retq
+; SSE2-LABEL: test_intrinsic_fmax:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    movapd %xmm0, %xmm2
+; SSE2-NEXT:    cmpunordsd %xmm0, %xmm2
+; SSE2-NEXT:    movapd %xmm2, %xmm3
+; SSE2-NEXT:    andpd %xmm1, %xmm3
+; SSE2-NEXT:    maxsd %xmm0, %xmm1
+; SSE2-NEXT:    andnpd %xmm1, %xmm2
+; SSE2-NEXT:    orpd %xmm3, %xmm2
+; SSE2-NEXT:    movapd %xmm2, %xmm0
+; SSE2-NEXT:    retq
+;
+; SSE4-LABEL: test_intrinsic_fmax:
+; SSE4:       # %bb.0:
+; SSE4-NEXT:    movapd %xmm1, %xmm2
+; SSE4-NEXT:    maxsd %xmm0, %xmm2
+; SSE4-NEXT:    cmpunordsd %xmm0, %xmm0
+; SSE4-NEXT:    blendvpd %xmm0, %xmm1, %xmm2
+; SSE4-NEXT:    movapd %xmm2, %xmm0
+; SSE4-NEXT:    retq
 ;
 ; AVX1-LABEL: test_intrinsic_fmax:
 ; AVX1:       # %bb.0:
diff --git a/llvm/test/CodeGen/X86/fminimum-fmaximum.ll b/llvm/test/CodeGen/X86/fminimum-fmaximum.ll
index 257524e0d4db5..0530c843acfe6 100644
--- a/llvm/test/CodeGen/X86/fminimum-fmaximum.ll
+++ b/llvm/test/CodeGen/X86/fminimum-fmaximum.ll
@@ -2022,19 +2022,19 @@ define <4 x half> @test_fmaximum_v4f16(<4 x half> %x, <4 x half> %y) nounwind {
 ; X86-NEXT:    fstps {{[0-9]+}}(%esp)
 ; X86-NEXT:    fldt {{[-0-9]+}}(%e{{[sb]}}p) # 10-byte Folded Reload
 ; X86-NEXT:    fstps {{[0-9]+}}(%esp)
-; X86-NEXT:    vmovd {{.*#+}} xmm2 = mem[0],zero,zero,zero
-; X86-NEXT:    vmovd %xmm2, %eax
+; X86-NEXT:    vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; X86-NEXT:    vmovd %xmm1, %eax
 ; X86-NEXT:    vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
 ; X86-NEXT:    testl %eax, %eax
 ; X86-NEXT:    js .LBB33_1
 ; X86-NEXT:  # %bb.2:
-; X86-NEXT:    vmovdqa %xmm2, %xmm1
+; X86-NEXT:    vmovdqa %xmm1, %xmm2
 ; X86-NEXT:    jmp .LBB33_3
 ; X86-NEXT:  .LBB33_1:
-; X86-NEXT:    vmovdqa %xmm0, %xmm1
-; X86-NEXT:    vmovdqa %xmm2, %xmm0
+; X86-NEXT:    vmovdqa %xmm0, %xmm2
+; X86-NEXT:    vmovdqa %xmm1, %xmm0
 ; X86-NEXT:  .LBB33_3:
-; X86-NEXT:    vmaxss %xmm1, %xmm0, %xmm1
+; X86-NEXT:    vmaxss %xmm2, %xmm0, %xmm1
 ; X86-NEXT:    vcmpunordss %xmm0, %xmm0, %xmm2
 ; X86-NEXT:    vblendvps %xmm2, %xmm0, %xmm1, %xmm0
 ; X86-NEXT:    vmovups %xmm0, {{[-0-9]+}}(%e{{[sb]}}p) # 16-byte Spill
@@ -2154,43 +2154,43 @@ define <4 x bfloat> @test_fmaximum_v4bf16(<4 x bfloat> %x, <4 x bfloat> %y) {
 ; SSE2-NEXT:    .cfi_offset %r14, -32
 ; SSE2-NEXT:    .cfi_offset %r15, -24
 ; SSE2-NEXT:    .cfi_offset %rbp, -16
-; SSE2-NEXT:    movdqa %xmm1, %xmm4
-; SSE2-NEXT:    movdqa %xmm0, %xmm5
 ; SSE2-NEXT:    pextrw $0, %xmm1, %r14d
 ; SSE2-NEXT:    pextrw $0, %xmm0, %r15d
-; SSE2-NEXT:    movdqa %xmm1, %xmm0
-; SSE2-NEXT:    psrld $16, %xmm0
-; SSE2-NEXT:    pextrw $0, %xmm0, %eax
-; SSE2-NEXT:    movdqa %xmm5, %xmm0
-; SSE2-NEXT:    psrld $16, %xmm0
-; SSE2-NEXT:    pextrw $0, %xmm0, %ecx
+; SSE2-NEXT:    movdqa %xmm1, %xmm2
+; SSE2-NEXT:    psrld $16, %xmm2
+; SSE2-NEXT:    pextrw $0, %xmm2, %eax
+; SSE2-NEXT:    movdqa %xmm0, %xmm2
+; SSE2-NEXT:    psrld $16, %xmm2
+; SSE2-NEXT:    pextrw $0, %xmm2, %ecx
 ; SSE2-NEXT:    shll $16, %ecx
 ; SSE2-NEXT:    movd %ecx, %xmm3
 ; SSE2-NEXT:    shll $16, %eax
 ; SSE2-NEXT:    movd %eax, %xmm2
 ; SSE2-NEXT:    testl %ecx, %ecx
-; SSE2-NEXT:    movdqa %xmm3, %xmm1
+; SSE2-NEXT:    movdqa %xmm3, %xmm7
 ; SSE2-NEXT:    js .LBB34_2
 ; SSE2-NEXT:  # %bb.1:
-; SSE2-NEXT:    movdqa %xmm2, %xmm1
+; SSE2-NEXT:    movdqa %xmm2, %xmm7
 ; SSE2-NEXT:  .LBB34_2:
-; SSE2-NEXT:    movdqa %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; SSE2-NEXT:    shufps {{.*#+}} xmm4 = xmm4[1,1,1,1]
-; SSE2-NEXT:    movdqa %xmm5, (%rsp) # 16-byte Spill
-; SSE2-NEXT:    shufps {{.*#+}} xmm5 = xmm5[1,1,1,1]
-; SSE2-NEXT:    movdqa %xmm1, %xmm0
-; SSE2-NEXT:    cmpunordss %xmm1, %xmm0
-; SSE2-NEXT:    movaps %xmm0, %xmm6
-; SSE2-NEXT:    andps %xmm1, %xmm6
+; SSE2-NEXT:    movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; SSE2-NEXT:    movdqa %xmm1, %xmm5
+; SSE2-NEXT:    shufps {{.*#+}} xmm5 = xmm5[1,1],xmm1[1,1]
+; SSE2-NEXT:    movdqa %xmm0, (%rsp) # 16-byte Spill
+; SSE2-NEXT:    movdqa %xmm0, %xmm6
+; SSE2-NEXT:    shufps {{.*#+}} xmm6 = xmm6[1,1],xmm0[1,1]
+; SSE2-NEXT:    movdqa %xmm7, %xmm0
+; SSE2-NEXT:    cmpunordss %xmm7, %xmm0
+; SSE2-NEXT:    movaps %xmm0, %xmm4
+; SSE2-NEXT:    andps %xmm7, %xmm4
 ; SSE2-NEXT:    js .LBB34_4
 ; SSE2-NEXT:  # %bb.3:
 ; SSE2-NEXT:    movdqa %xmm3, %xmm2
 ; SSE2-NEXT:  .LBB34_4:
-; SSE2-NEXT:    pextrw $0, %xmm4, %ebp
-; SSE2-NEXT:    pextrw $0, %xmm5, %ebx
-; SSE2-NEXT:    maxss %xmm2, %xmm1
-; SSE2-NEXT:    andnps %xmm1, %xmm0
-; SSE2-NEXT:    orps %xmm6, %xmm0
+; SSE2-NEXT:    pextrw $0, %xmm5, %ebp
+; SSE2-NEXT:    pextrw $0, %xmm6, %ebx
+; SSE2-NEXT:    maxss %xmm2, %xmm7
+; SSE2-NEXT:    andnps %xmm7, %xmm0
+; SSE2-NEXT:    orps %xmm4, %xmm0
 ; SSE2-NEXT:    callq __truncsfbf2@PLT
 ; SSE2-NEXT:    movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
 ; SSE2-NEXT:    shll $16, %r15d
@@ -2312,7 +2312,7 @@ define <4 x bfloat> @test_fmaximum_v4bf16(<4 x bfloat> %x, <4 x bfloat> %y) {
 ; AVX1-NEXT:    vmovshdup {{.*#+}} xmm4 = xmm0[1,1,3,3]
 ; AVX1-NEXT:    vpextrw $0, %xmm4, %ebx
 ; AVX1-NEXT:    vmovshdup {{.*#+}} xmm4 = xmm1[1,1,3,3]
-; AVX1-NEXT:    vpextrw $0, %xmm4, %ebp
+; AVX1-NEXT:    vpextrw $0, %xmm4, %r14d
 ; AVX1-NEXT:    vpextrw $0, %xmm0, %r12d
 ; AVX1-NEXT:    vpextrw $0, %xmm1, %r13d
 ; AVX1-NEXT:    vpsrld $16, %xmm0, %xmm0
@@ -2331,7 +2331,7 @@ define <4 x bfloat> @test_fmaximum_v4bf16(<4 x bfloat> %x, <4 x bfloat> %y) {
 ; AVX1-NEXT:    vmovdqa %xmm0, %xmm1
 ; AVX1-NEXT:    vmovdqa %xmm4, %xmm0
 ; AVX1-NEXT:  .LBB34_3:
-; AVX1-NEXT:    vpextrw $0, %xmm2, %r14d
+; AVX1-NEXT:    vpextrw $0, %xmm2, %ebp
 ; AVX1-NEXT:    vpextrw $0, %xmm3, %r15d
 ; AVX1-NEXT:    vmaxss %xmm1, %xmm0, %xmm1
 ; AVX1-NEXT:    vcmpunordss %xmm0, %xmm0, %xmm2
@@ -2355,8 +2355,8 @@ define <4 x bfloat> @test_fmaximum_v4bf16(<4 x bfloat> %x, <4 x bfloat> %y) {
 ; AVX1-NEXT:    vblendvps %xmm2, %xmm0, %xmm1, %xmm0
 ; AVX1-NEXT:    callq __truncsfbf2@PLT
 ; AVX1-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX1-NEXT:    shll $16, %ebp
-; AVX1-NEXT:    vmovd %ebp, %xmm0
+; AVX1-NEXT:    shll $16, %r14d
+; AVX1-NEXT:    vmovd %r14d, %xmm0
 ; AVX1-NEXT:    shll $16, %ebx
 ; AVX1-NEXT:    vmovd %ebx, %xmm2
 ; AVX1-NEXT:    js .LBB34_7
@@ -2374,8 +2374,8 @@ define <4 x bfloat> @test_fmaximum_v4bf16(<4 x bfloat> %x, <4 x bfloat> %y) {
 ; AVX1-NEXT:    vmovaps %xmm0, (%rsp) # 16-byte Spill
 ; AVX1-NEXT:    shll $16, %r15d
 ; AVX1-NEXT:    vmovd %r15d, %xmm0
-; AVX1-NEXT:    shll $16, %r14d
-; AVX1-NEXT:    vmovd %r14d, %xmm2
+; AVX1-NEXT:    shll $16, %ebp
+; AVX1-NEXT:    vmovd %ebp, %xmm2
 ; AVX1-NEXT:    js .LBB34_10
 ; AVX1-NEXT:  # %bb.11:
 ; AVX1-NEXT:    vmovdqa %xmm2, %xmm1
diff --git a/llvm/test/CodeGen/X86/fminimumnum-fmaximumnum.ll b/llvm/test/CodeGen/X86/fminimumnum-fmaximumnum.ll
index bfff6ef41dbe0..c7f5e13cb7464 100644
--- a/llvm/test/CodeGen/X86/fminimumnum-fmaximumnum.ll
+++ b/llvm/test/CodeGen/X86/fminimumnum-fmaximumnum.ll
@@ -1812,212 +1812,186 @@ define <4 x half> @test_fmaximumnum_v4f16(<4 x half> %x, <4 x half> %y) nounwind
 ;
 ; AVX512-LABEL: test_fmaximumnum_v4f16:
 ; AVX512:       # %bb.0:
-; AVX512-NEXT:    subq $88, %rsp
+; AVX512-NEXT:    subq $72, %rsp
 ; AVX512-NEXT:    vmovdqa %xmm1, %xmm4
-; AVX512-NEXT:    vmovdqa %xmm0, %xmm6
+; AVX512-NEXT:    vmovdqa %xmm0, %xmm8
 ; AVX512-NEXT:    vpsrldq {{.*#+}} xmm0 = xmm1[14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
 ; AVX512-NEXT:    vcvtph2ps %xmm0, %xmm0
 ; AVX512-NEXT:    vucomiss %xmm0, %xmm0
 ; AVX512-NEXT:    setp %al
 ; AVX512-NEXT:    kmovw %eax, %k1
-; AVX512-NEXT:    vpsrldq {{.*#+}} xmm2 = xmm6[14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX512-NEXT:    vcvtph2ps %xmm2, %xmm2
-; AVX512-NEXT:    vucomiss %xmm2, %xmm2
+; AVX512-NEXT:    vpsrldq {{.*#+}} xmm1 = xmm8[14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; AVX512-NEXT:    vcvtph2ps %xmm1, %xmm1
+; AVX512-NEXT:    vucomiss %xmm1, %xmm1
 ; AVX512-NEXT:    setp %al
 ; AVX512-NEXT:    kmovw %eax, %k2
-; AVX512-NEXT:    vmovss %xmm0, %xmm2, %xmm2 {%k2}
-; AVX512-NEXT:    vcvtps2ph $4, %xmm2, %xmm1
-; AVX512-NEXT:    vmovaps %xmm1, (%rsp) # 16-byte Spill
-; AVX512-NEXT:    vcvtph2ps %xmm1, %xmm2
-; AVX512-NEXT:    vmovss %xmm2, %xmm0, %xmm0 {%k1}
+; AVX512-NEXT:    vmovss %xmm0, %xmm1, %xmm1 {%k2}
+; AVX512-NEXT:    vcvtps2ph $4, %xmm1, %xmm1
+; AVX512-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512-NEXT:    vcvtph2ps %xmm1, %xmm1
+; AVX512-NEXT:    vmovss %xmm1, %xmm0, %xmm0 {%k1}
 ; AVX512-NEXT:    vcvtps2ph $4, %xmm0, %xmm0
 ; AVX512-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
 ; AVX512-NEXT:    vcvtph2ps %xmm0, %xmm0
-; AVX512-NEXT:    vucomiss %xmm0, %xmm2
+; AVX512-NEXT:    vucomiss %xmm0, %xmm1
 ; AVX512-NEXT:    seta %al
 ; AVX512-NEXT:    kmovw %eax, %k1
-; AVX512-NEXT:    vmovss %xmm2, %xmm0, %xmm0 {%k1}
+; AVX512-NEXT:    vmovss %xmm1, %xmm0, %xmm0 {%k1}
 ; AVX512-NEXT:    vcvtps2ph $4, %xmm0, %xmm0
 ; AVX512-NEXT:    vpmovzxwq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero
 ; AVX512-NEXT:    vcvtph2ps %xmm0, %xmm0
 ; AVX512-NEXT:    movzwl {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %eax
-; AVX512-NEXT:    vmovd %eax, %xmm2
-; AVX512-NEXT:    vcvtph2ps %xmm2, %xmm9
+; AVX512-NEXT:    vmovd %eax, %xmm1
+; AVX512-NEXT:    vcvtph2ps %xmm1, %xmm9
 ; AVX512-NEXT:    vmulss %xmm0, %xmm9, %xmm0
-; AVX512-NEXT:    vxorps %xmm10, %xmm10, %xmm10
-; AVX512-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0],xmm10[1,2,3]
-; AVX512-NEXT:    vcvtps2ph $4, %xmm0, %xmm0
-; AVX512-NEXT:    vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512-NEXT:    vmovd %xmm0, %eax
-; AVX512-NEXT:    vpinsrw $0, %eax, %xmm0, %xmm0
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm2 = xmm4[3,3,3,3]
-; AVX512-NEXT:    vcvtph2ps %xmm2, %xmm2
-; AVX512-NEXT:    vucomiss %xmm2, %xmm2
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm4[3,3,3,3]
+; AVX512-NEXT:    vcvtph2ps %xmm1, %xmm1
+; AVX512-NEXT:    vucomiss %xmm1, %xmm1
 ; AVX512-NEXT:    setp %al
 ; AVX512-NEXT:    kmovw %eax, %k1
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm3 = xmm6[3,3,3,3]
-; AVX512-NEXT:    vcvtph2ps %xmm3, %xmm3
-; AVX512-NEXT:    vucomiss %xmm3, %xmm3
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm2 = xmm8[3,3,3,3]
+; AVX512-NEXT:    vcvtph2ps %xmm2, %xmm2
+; AVX512-NEXT:    vucomiss %xmm2, %xmm2
 ; AVX512-NEXT:    setp %al
 ; AVX512-NEXT:    kmovw %eax, %k2
-; AVX512-NEXT:    vmovss %xmm2, %xmm3, %xmm3 {%k2}
-; AVX512-NEXT:    vcvtps2ph $4, %xmm3, %xmm1
+; AVX512-NEXT:    vmovss %xmm1, %xmm2, %xmm2 {%k2}
+; AVX512-NEXT:    vcvtps2ph $4, %xmm2, %xmm2
+; AVX512-NEXT:    vmovaps %xmm2, (%rsp) # 16-byte Spill
+; AVX512-NEXT:    vcvtph2ps %xmm2, %xmm2
+; AVX512-NEXT:    vmovss %xmm2, %xmm1, %xmm1 {%k1}
+; AVX512-NEXT:    vcvtps2ph $4, %xmm1, %xmm1
 ; AVX512-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
 ; AVX512-NEXT:    vcvtph2ps %xmm1, %xmm3
-; AVX512-NEXT:    vmovss %xmm3, %xmm2, %xmm2 {%k1}
-; AVX512-NEXT:    vcvtps2ph $4, %xmm2, %xmm1
-; AVX512-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512-NEXT:    vcvtph2ps %xmm1, %xmm2
-; AVX512-NEXT:    vucomiss %xmm2, %xmm3
+; AVX512-NEXT:    vucomiss %xmm3, %xmm2
 ; AVX512-NEXT:    seta %al
 ; AVX512-NEXT:    kmovw %eax, %k1
-; AVX512-NEXT:    vmovss %xmm3, %xmm2, %xmm2 {%k1}
-; AVX512-NEXT:    vcvtps2ph $4, %xmm2, %xmm2
-; AVX512-NEXT:    vpmovzxwq {{.*#+}} xmm2 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero
-; AVX512-NEXT:    vcvtph2ps %xmm2, %xmm2
-; AVX512-NEXT:    vmulss %xmm2, %xmm9, %xmm2
-; AVX512-NEXT:    vblendps {{.*#+}} xmm2 = xmm2[0],xmm10[1,2,3]
-; AVX512-NEXT:    vcvtps2ph $4, %xmm2, %xmm1
-; AVX512-NEXT:    vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512-NEXT:    vmovd %xmm1, %eax
-; AVX512-NEXT:    vpinsrw $0, %eax, %xmm0, %xmm2
-; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3]
-; AVX512-NEXT:    vpsrldq {{.*#+}} xmm0 = xmm4[10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX512-NEXT:    vcvtph2ps %xmm0, %xmm0
-; AVX512-NEXT:    vucomiss %xmm0, %xmm0
+; AVX512-NEXT:    vmovss %xmm2, %xmm3, %xmm3 {%k1}
+; AVX512-NEXT:    vpsrldq {{.*#+}} xmm1 = xmm4[10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; AVX512-NEXT:    vcvtph2ps %xmm1, %xmm1
+; AVX512-NEXT:    vucomiss %xmm1, %xmm1
 ; AVX512-NEXT:    setp %al
 ; AVX512-NEXT:    kmovw %eax, %k1
-; AVX512-NEXT:    vpsrldq {{.*#+}} xmm3 = xmm6[10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX512-NEXT:    vcvtph2ps %xmm3, %xmm3
-; AVX512-NEXT:    vucomiss %xmm3, %xmm3
+; AVX512-NEXT:    vpsrldq {{.*#+}} xmm2 = xmm8[10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; AVX512-NEXT:    vcvtph2ps %xmm2, %xmm2
+; AVX512-NEXT:    vucomiss %xmm2, %xmm2
 ; AVX512-NEXT:    setp %al
 ; AVX512-NEXT:    kmovw %eax, %k2
-; AVX512-NEXT:    vmovss %xmm0, %xmm3, %xmm3 {%k2}
-; AVX512-NEXT:    vcvtps2ph $4, %xmm3, %xmm1
+; AVX512-NEXT:    vmovss %xmm1, %xmm2, %xmm2 {%k2}
+; AVX512-NEXT:    vcvtps2ph $4, %xmm2, %xmm2
+; AVX512-NEXT:    vmovaps %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512-NEXT:    vcvtph2ps %xmm2, %xmm2
+; AVX512-NEXT:    vmovss %xmm2, %xmm1, %xmm1 {%k1}
+; AVX512-NEXT:    vcvtps2ph $4, %xmm1, %xmm1
 ; AVX512-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512-NEXT:    vcvtph2ps %xmm1, %xmm5
-; AVX512-NEXT:    vmovss %xmm5, %xmm0, %xmm0 {%k1}
-; AVX512-NEXT:    vcvtps2ph $4, %xmm0, %xmm0
-; AVX512-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512-NEXT:    vcvtph2ps %xmm0, %xmm3
-; AVX512-NEXT:    vucomiss %xmm3, %xmm5
+; AVX512-NEXT:    vcvtph2ps %xmm1, %xmm1
+; AVX512-NEXT:    vucomiss %xmm1, %xmm2
 ; AVX512-NEXT:    seta %al
 ; AVX512-NEXT:    kmovw %eax, %k1
-; AVX512-NEXT:    vmovss %xmm5, %xmm3, %xmm3 {%k1}
-; AVX512-NEXT:    vshufpd {{.*#+}} xmm0 = xmm4[1,0]
-; AVX512-NEXT:    vcvtph2ps %xmm0, %xmm0
-; AVX512-NEXT:    vucomiss %xmm0, %xmm0
+; AVX512-NEXT:    vmovss %xmm2, %xmm1, %xmm1 {%k1}
+; AVX512-NEXT:    vshufpd {{.*#+}} xmm2 = xmm4[1,0]
+; AVX512-NEXT:    vcvtph2ps %xmm2, %xmm2
+; AVX512-NEXT:    vucomiss %xmm2, %xmm2
 ; AVX512-NEXT:    setp %al
 ; AVX512-NEXT:    kmovw %eax, %k1
-; AVX512-NEXT:    vshufpd {{.*#+}} xmm5 = xmm6[1,0]
-; AVX512-NEXT:    vcvtph2ps %xmm5, %xmm5
-; AVX512-NEXT:    vucomiss %xmm5, %xmm5
+; AVX512-NEXT:    vshufpd {{.*#+}} xmm7 = xmm8[1,0]
+; AVX512-NEXT:    vcvtph2ps %xmm7, %xmm7
+; AVX512-NEXT:    vucomiss %xmm7, %xmm7
 ; AVX512-NEXT:    setp %al
 ; AVX512-NEXT:    kmovw %eax, %k2
-; AVX512-NEXT:    vmovss %xmm0, %xmm5, %xmm5 {%k2}
-; AVX512-NEXT:    vcvtps2ph $4, %xmm5, %xmm15
-; AVX512-NEXT:    vcvtph2ps %xmm15, %xmm5
-; AVX512-NEXT:    vmovss %xmm5, %xmm0, %xmm0 {%k1}
-; AVX512-NEXT:    vcvtps2ph $4, %xmm0, %xmm0
-; AVX512-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512-NEXT:    vcvtph2ps %xmm0, %xmm0
-; AVX512-NEXT:    vucomiss %xmm0, %xmm5
+; AVX512-NEXT:    vmovss %xmm2, %xmm7, %xmm7 {%k2}
+; AVX512-NEXT:    vcvtps2ph $4, %xmm7, %xmm14
+; AVX512-NEXT:    vcvtph2ps %xmm14, %xmm7
+; AVX512-NEXT:    vmovss %xmm7, %xmm2, %xmm2 {%k1}
+; AVX512-NEXT:    vcvtps2ph $4, %xmm2, %xmm2
+; AVX512-NEXT:    vmovaps %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512-NEXT:    vcvtph2ps %xmm2, %xmm2
+; AVX512-NEXT:    vucomiss %xmm2, %xmm7
 ; AVX512-NEXT:    seta %al
 ; AVX512-NEXT:    kmovw %eax, %k1
-; AVX512-NEXT:    vmovss %xmm5, %xmm0, %xmm0 {%k1}
-; AVX512-NEXT:    vcvtps2ph $4, %xmm3, %xmm3
-; AVX512-NEXT:    vpmovzxwq {{.*#+}} xmm3 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero
-; AVX512-NEXT:    vcvtph2ps %xmm3, %xmm3
-; AVX512-NEXT:    vmulss %xmm3, %xmm9, %xmm3
-; AVX512-NEXT:    vblendps {{.*#+}} xmm3 = xmm3[0],xmm10[1,2,3]
-; AVX512-NEXT:    vcvtps2ph $4, %xmm3, %xmm1
+; AVX512-NEXT:    vmovss %xmm7, %xmm2, %xmm2 {%k1}
+; AVX512-NEXT:    vxorps %xmm15, %xmm15, %xmm15
+; AVX512-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0],xmm15[1,2,3]
+; AVX512-NEXT:    vcvtps2ph $4, %xmm0, %xmm5
+; AVX512-NEXT:    vmovdqa %xmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512-NEXT:    vcvtps2ph $4, %xmm3, %xmm0
+; AVX512-NEXT:    vpmovzxwq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero
+; AVX512-NEXT:    vcvtph2ps %xmm0, %xmm0
+; AVX512-NEXT:    vmulss %xmm0, %xmm9, %xmm0
+; AVX512-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0],xmm15[1,2,3]
+; AVX512-NEXT:    vcvtps2ph $4, %xmm0, %xmm3
+; AVX512-NEXT:    vmovdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512-NEXT:    vcvtps2ph $4, %xmm1, %xmm0
+; AVX512-NEXT:    vpmovzxwq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero
+; AVX512-NEXT:    vcvtph2ps %xmm0, %xmm0
+; AVX512-NEXT:    vmulss %xmm0, %xmm9, %xmm0
+; AVX512-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0],xmm15[1,2,3]
+; AVX512-NEXT:    vcvtps2ph $4, %xmm0, %xmm1
 ; AVX512-NEXT:    vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512-NEXT:    vmovd %xmm1, %eax
-; AVX512-NEXT:    vcvtps2ph $4, %xmm0, %xmm0
+; AVX512-NEXT:    vcvtps2ph $4, %xmm2, %xmm0
 ; AVX512-NEXT:    vpmovzxwq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero
 ; AVX512-NEXT:    vcvtph2ps %xmm0, %xmm0
 ; AVX512-NEXT:    vmulss %xmm0, %xmm9, %xmm0
-; AVX512-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0],xmm10[1,2,3]
-; AVX512-NEXT:    vcvtps2ph $4, %xmm0, %xmm0
-; AVX512-NEXT:    vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512-NEXT:    vmovd %xmm0, %ecx
-; AVX512-NEXT:    vpinsrw $0, %eax, %xmm0, %xmm0
-; AVX512-NEXT:    vpinsrw $0, %ecx, %xmm0, %xmm3
-; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm3[0],xmm0[0],xmm3[1],xmm0[1],xmm3[2],xmm0[2],xmm3[3],xmm0[3]
-; AVX512-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
+; AVX512-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0],xmm15[1,2,3]
+; AVX512-NEXT:    vcvtps2ph $4, %xmm0, %xmm2
+; AVX512-NEXT:    vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm3[0],xmm5[0],xmm3[1],xmm5[1],xmm3[2],xmm5[2],xmm3[3],xmm5[3]
+; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
+; AVX512-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
 ; AVX512-NEXT:    vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
 ; AVX512-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm4[3,3,3,3,4,5,6,7]
 ; AVX512-NEXT:    vcvtph2ps %xmm0, %xmm0
 ; AVX512-NEXT:    vucomiss %xmm0, %xmm0
 ; AVX512-NEXT:    setp %al
 ; AVX512-NEXT:    kmovw %eax, %k1
-; AVX512-NEXT:    vpshuflw {{.*#+}} xmm2 = xmm6[3,3,3,3,4,5,6,7]
-; AVX512-NEXT:    vcvtph2ps %xmm2, %xmm2
-; AVX512-NEXT:    vucomiss %xmm2, %xmm2
+; AVX512-NEXT:    vpshuflw {{.*#+}} xmm1 = xmm8[3,3,3,3,4,5,6,7]
+; AVX512-NEXT:    vcvtph2ps %xmm1, %xmm1
+; AVX512-NEXT:    vucomiss %xmm1, %xmm1
 ; AVX512-NEXT:    setp %al
 ; AVX512-NEXT:    kmovw %eax, %k2
-; AVX512-NEXT:    vmovss %xmm0, %xmm2, %xmm2 {%k2}
-; AVX512-NEXT:    vcvtps2ph $4, %xmm2, %xmm11
-; AVX512-NEXT:    vcvtph2ps %xmm11, %xmm3
-; AVX512-NEXT:    vmovss %xmm3, %xmm0, %xmm0 {%k1}
-; AVX512-NEXT:    vcvtps2ph $4, %xmm0, %xmm0
-; AVX512-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512-NEXT:    vcvtph2ps %xmm0, %xmm2
-; AVX512-NEXT:    vucomiss %xmm2, %xmm3
+; AVX512-NEXT:    vmovss %xmm0, %xmm1, %xmm1 {%k2}
+; AVX512-NEXT:    vcvtps2ph $4, %xmm1, %xmm12
+; AVX512-NEXT:    vcvtph2ps %xmm12, %xmm1
+; AVX512-NEXT:    vmovss %xmm1, %xmm0, %xmm0 {%k1}
+; AVX512-NEXT:    vcvtps2ph $4, %xmm0, %xmm13
+; AVX512-NEXT:    vcvtph2ps %xmm13, %xmm6
+; AVX512-NEXT:    vucomiss %xmm6, %xmm1
 ; AVX512-NEXT:    seta %al
 ; AVX512-NEXT:    kmovw %eax, %k1
-; AVX512-NEXT:    vmovss %xmm3, %xmm2, %xmm2 {%k1}
+; AVX512-NEXT:    vmovss %xmm1, %xmm6, %xmm6 {%k1}
 ; AVX512-NEXT:    vmovshdup {{.*#+}} xmm0 = xmm4[1,1,3,3]
 ; AVX512-NEXT:    vcvtph2ps %xmm0, %xmm0
 ; AVX512-NEXT:    vucomiss %xmm0, %xmm0
 ; AVX512-NEXT:    setp %al
 ; AVX512-NEXT:    kmovw %eax, %k1
-; AVX512-NEXT:    vmovshdup {{.*#+}} xmm3 = xmm6[1,1,3,3]
-; AVX512-NEXT:    vcvtph2ps %xmm3, %xmm3
-; AVX512-NEXT:    vucomiss %xmm3, %xmm3
+; AVX512-NEXT:    vmovshdup {{.*#+}} xmm1 = xmm8[1,1,3,3]
+; AVX512-NEXT:    vcvtph2ps %xmm1, %xmm1
+; AVX512-NEXT:    vucomiss %xmm1, %xmm1
 ; AVX512-NEXT:    setp %al
 ; AVX512-NEXT:    kmovw %eax, %k2
-; AVX512-NEXT:    vmovss %xmm0, %xmm3, %xmm3 {%k2}
-; AVX512-NEXT:    vcvtps2ph $4, %xmm3, %xmm7
-; AVX512-NEXT:    vcvtph2ps %xmm7, %xmm3
+; AVX512-NEXT:    vmovss %xmm0, %xmm1, %xmm1 {%k2}
+; AVX512-NEXT:    vcvtps2ph $4, %xmm1, %xmm10
+; AVX512-NEXT:    vcvtph2ps %xmm10, %xmm3
 ; AVX512-NEXT:    vmovss %xmm3, %xmm0, %xmm0 {%k1}
-; AVX512-NEXT:    vcvtps2ph $4, %xmm0, %xmm12
-; AVX512-NEXT:    vcvtph2ps %xmm12, %xmm0
-; AVX512-NEXT:    vucomiss %xmm0, %xmm3
+; AVX512-NEXT:    vcvtps2ph $4, %xmm0, %xmm11
+; AVX512-NEXT:    vcvtph2ps %xmm11, %xmm5
+; AVX512-NEXT:    vucomiss %xmm5, %xmm3
 ; AVX512-NEXT:    seta %al
 ; AVX512-NEXT:    kmovw %eax, %k1
-; AVX512-NEXT:    vmovss %xmm3, %xmm0, %xmm0 {%k1}
-; AVX512-NEXT:    vcvtps2ph $4, %xmm2, %xmm2
-; AVX512-NEXT:    vpmovzxwq {{.*#+}} xmm2 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero
-; AVX512-NEXT:    vcvtph2ps %xmm2, %xmm2
-; AVX512-NEXT:    vmulss %xmm2, %xmm9, %xmm2
-; AVX512-NEXT:    vblendps {{.*#+}} xmm2 = xmm2[0],xmm10[1,2,3]
-; AVX512-NEXT:    vcvtps2ph $4, %xmm2, %xmm14
-; AVX512-NEXT:    vmovd %xmm14, %eax
-; AVX512-NEXT:    vcvtps2ph $4, %xmm0, %xmm0
-; AVX512-NEXT:    vpmovzxwq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero
-; AVX512-NEXT:    vcvtph2ps %xmm0, %xmm0
-; AVX512-NEXT:    vmulss %xmm0, %xmm9, %xmm0
-; AVX512-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0],xmm10[1,2,3]
-; AVX512-NEXT:    vcvtps2ph $4, %xmm0, %xmm13
-; AVX512-NEXT:    vmovd %xmm13, %ecx
-; AVX512-NEXT:    vpinsrw $0, %eax, %xmm0, %xmm0
-; AVX512-NEXT:    vpinsrw $0, %ecx, %xmm0, %xmm2
-; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm5 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3]
+; AVX512-NEXT:    vmovss %xmm3, %xmm5, %xmm5 {%k1}
 ; AVX512-NEXT:    vcvtph2ps %xmm4, %xmm0
 ; AVX512-NEXT:    vucomiss %xmm0, %xmm0
 ; AVX512-NEXT:    setp %al
 ; AVX512-NEXT:    kmovw %eax, %k1
-; AVX512-NEXT:    vcvtph2ps %xmm6, %xmm2
-; AVX512-NEXT:    vucomiss %xmm2, %xmm2
+; AVX512-NEXT:    vcvtph2ps %xmm8, %xmm3
+; AVX512-NEXT:    vucomiss %xmm3, %xmm3
 ; AVX512-NEXT:    setp %al
 ; AVX512-NEXT:    kmovw %eax, %k2
-; AVX512-NEXT:    vmovss %xmm0, %xmm2, %xmm2 {%k2}
-; AVX512-NEXT:    vcvtps2ph $4, %xmm2, %xmm3
+; AVX512-NEXT:    vmovss %xmm0, %xmm3, %xmm3 {%k2}
+; AVX512-NEXT:    vcvtps2ph $4, %xmm3, %xmm3
 ; AVX512-NEXT:    vcvtph2ps %xmm3, %xmm1
 ; AVX512-NEXT:    vmovss %xmm1, %xmm0, %xmm0 {%k1}
-; AVX512-NEXT:    vcvtps2ph $4, %xmm0, %xmm8
-; AVX512-NEXT:    vcvtph2ps %xmm8, %xmm2
+; AVX512-NEXT:    vcvtps2ph $4, %xmm0, %xmm7
+; AVX512-NEXT:    vcvtph2ps %xmm7, %xmm2
 ; AVX512-NEXT:    vucomiss %xmm2, %xmm1
 ; AVX512-NEXT:    seta %al
 ; AVX512-NEXT:    kmovw %eax, %k1
@@ -2027,21 +2001,29 @@ define <4 x half> @test_fmaximumnum_v4f16(<4 x half> %x, <4 x half> %y) nounwind
 ; AVX512-NEXT:    vucomiss %xmm1, %xmm1
 ; AVX512-NEXT:    setp %al
 ; AVX512-NEXT:    kmovw %eax, %k1
-; AVX512-NEXT:    vpshuflw {{.*#+}} xmm4 = xmm6[1,1,1,1,4,5,6,7]
+; AVX512-NEXT:    vpshuflw {{.*#+}} xmm4 = xmm8[1,1,1,1,4,5,6,7]
 ; AVX512-NEXT:    vcvtph2ps %xmm4, %xmm4
 ; AVX512-NEXT:    vucomiss %xmm4, %xmm4
 ; AVX512-NEXT:    setp %al
 ; AVX512-NEXT:    kmovw %eax, %k2
 ; AVX512-NEXT:    vmovss %xmm1, %xmm4, %xmm4 {%k2}
-; AVX512-NEXT:    vcvtps2ph $4, %xmm4, %xmm4
-; AVX512-NEXT:    vcvtph2ps %xmm4, %xmm6
-; AVX512-NEXT:    vmovss %xmm6, %xmm1, %xmm1 {%k1}
+; AVX512-NEXT:    vcvtps2ph $4, %xmm4, %xmm8
+; AVX512-NEXT:    vcvtph2ps %xmm8, %xmm4
+; AVX512-NEXT:    vmovss %xmm4, %xmm1, %xmm1 {%k1}
 ; AVX512-NEXT:    vcvtps2ph $4, %xmm1, %xmm1
 ; AVX512-NEXT:    vcvtph2ps %xmm1, %xmm0
-; AVX512-NEXT:    vucomiss %xmm0, %xmm6
+; AVX512-NEXT:    vucomiss %xmm0, %xmm4
 ; AVX512-NEXT:    seta %al
 ; AVX512-NEXT:    kmovw %eax, %k1
-; AVX512-NEXT:    vmovss %xmm6, %xmm0, %xmm0 {%k1}
+; AVX512-NEXT:    vmovss %xmm4, %xmm0, %xmm0 {%k1}
+; AVX512-NEXT:    vcvtps2ph $4, %xmm6, %xmm4
+; AVX512-NEXT:    vpmovzxwq {{.*#+}} xmm4 = xmm4[0],zero,zero,zero,xmm4[1],zero,zero,zero
+; AVX512-NEXT:    vcvtph2ps %xmm4, %xmm4
+; AVX512-NEXT:    vmulss %xmm4, %xmm9, %xmm4
+; AVX512-NEXT:    vcvtps2ph $4, %xmm5, %xmm5
+; AVX512-NEXT:    vpmovzxwq {{.*#+}} xmm5 = xmm5[0],zero,zero,zero,xmm5[1],zero,zero,zero
+; AVX512-NEXT:    vcvtph2ps %xmm5, %xmm5
+; AVX512-NEXT:    vmulss %xmm5, %xmm9, %xmm5
 ; AVX512-NEXT:    vcvtps2ph $4, %xmm2, %xmm2
 ; AVX512-NEXT:    vpmovzxwq {{.*#+}} xmm2 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero
 ; AVX512-NEXT:    vcvtph2ps %xmm2, %xmm2
@@ -2050,121 +2032,89 @@ define <4 x half> @test_fmaximumnum_v4f16(<4 x half> %x, <4 x half> %y) nounwind
 ; AVX512-NEXT:    vpmovzxwq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero
 ; AVX512-NEXT:    vcvtph2ps %xmm0, %xmm0
 ; AVX512-NEXT:    vmulss %xmm0, %xmm9, %xmm0
-; AVX512-NEXT:    vblendps {{.*#+}} xmm2 = xmm2[0],xmm10[1,2,3]
-; AVX512-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0],xmm10[1,2,3]
-; AVX512-NEXT:    vcvtps2ph $4, %xmm2, %xmm9
-; AVX512-NEXT:    vmovd %xmm9, %eax
-; AVX512-NEXT:    vcvtps2ph $4, %xmm0, %xmm10
-; AVX512-NEXT:    vmovd %xmm10, %ecx
-; AVX512-NEXT:    vpinsrw $0, %eax, %xmm0, %xmm0
-; AVX512-NEXT:    vpinsrw $0, %ecx, %xmm0, %xmm2
-; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3]
-; AVX512-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm5[0],xmm0[1],xmm5[1]
-; AVX512-NEXT:    vpunpcklqdq {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm6 # 16-byte Folded Reload
-; AVX512-NEXT:    # xmm6 = xmm0[0],mem[0]
-; AVX512-NEXT:    vmovdqa (%rsp), %xmm0 # 16-byte Reload
-; AVX512-NEXT:    vmovd %xmm0, %eax
-; AVX512-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
-; AVX512-NEXT:    vmovd %xmm0, %ecx
-; AVX512-NEXT:    vpinsrw $0, %eax, %xmm0, %xmm0
-; AVX512-NEXT:    vpinsrw $0, %ecx, %xmm0, %xmm2
-; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3]
-; AVX512-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload
-; AVX512-NEXT:    vmovd %xmm2, %eax
-; AVX512-NEXT:    vmovd %xmm15, %ecx
-; AVX512-NEXT:    vpinsrw $0, %eax, %xmm0, %xmm2
-; AVX512-NEXT:    vpinsrw $0, %ecx, %xmm0, %xmm5
-; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm5[0],xmm2[0],xmm5[1],xmm2[1],xmm5[2],xmm2[2],xmm5[3],xmm2[3]
-; AVX512-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm2[0],xmm0[0],xmm2[1],xmm0[1]
-; AVX512-NEXT:    vmovd %xmm11, %eax
-; AVX512-NEXT:    vmovd %xmm7, %ecx
-; AVX512-NEXT:    vpinsrw $0, %eax, %xmm0, %xmm2
-; AVX512-NEXT:    vpinsrw $0, %ecx, %xmm0, %xmm5
-; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm5[0],xmm2[0],xmm5[1],xmm2[1],xmm5[2],xmm2[2],xmm5[3],xmm2[3]
-; AVX512-NEXT:    vmovd %xmm3, %eax
-; AVX512-NEXT:    vmovd %xmm4, %ecx
-; AVX512-NEXT:    vpinsrw $0, %eax, %xmm0, %xmm3
-; AVX512-NEXT:    vpinsrw $0, %ecx, %xmm0, %xmm4
-; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1],xmm3[2],xmm4[2],xmm3[3],xmm4[3]
-; AVX512-NEXT:    vpunpckldq {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]
-; AVX512-NEXT:    vpunpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm0[0]
-; AVX512-NEXT:    vpxor %xmm0, %xmm0, %xmm0
-; AVX512-NEXT:    vpcmpeqw %xmm0, %xmm2, %xmm3
-; AVX512-NEXT:    vpblendvb %xmm3, %xmm2, %xmm6, %xmm2
-; AVX512-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Reload
-; AVX512-NEXT:    vmovd %xmm3, %eax
-; AVX512-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Reload
-; AVX512-NEXT:    vmovd %xmm3, %ecx
-; AVX512-NEXT:    vpinsrw $0, %eax, %xmm0, %xmm3
-; AVX512-NEXT:    vpinsrw $0, %ecx, %xmm0, %xmm4
-; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
-; AVX512-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload
-; AVX512-NEXT:    vmovd %xmm4, %eax
-; AVX512-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload
-; AVX512-NEXT:    vmovd %xmm4, %ecx
-; AVX512-NEXT:    vpinsrw $0, %eax, %xmm0, %xmm4
-; AVX512-NEXT:    vpinsrw $0, %ecx, %xmm0, %xmm5
-; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm4 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3]
-; AVX512-NEXT:    vpunpckldq {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1]
-; AVX512-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload
-; AVX512-NEXT:    vmovd %xmm4, %eax
-; AVX512-NEXT:    vpinsrw $0, %eax, %xmm0, %xmm4
-; AVX512-NEXT:    vmovd %xmm12, %eax
-; AVX512-NEXT:    vpinsrw $0, %eax, %xmm0, %xmm5
-; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm4 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3]
-; AVX512-NEXT:    vmovd %xmm8, %eax
-; AVX512-NEXT:    vpinsrw $0, %eax, %xmm0, %xmm5
-; AVX512-NEXT:    vmovd %xmm1, %eax
-; AVX512-NEXT:    vpinsrw $0, %eax, %xmm0, %xmm1
-; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm5[0],xmm1[0],xmm5[1],xmm1[1],xmm5[2],xmm1[2],xmm5[3],xmm1[3]
-; AVX512-NEXT:    vpunpckldq {{.*#+}} xmm1 = xmm1[0],xmm4[0],xmm1[1],xmm4[1]
-; AVX512-NEXT:    vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm3[0]
-; AVX512-NEXT:    vpcmpeqw %xmm0, %xmm1, %xmm0
-; AVX512-NEXT:    vpblendvb %xmm0, %xmm1, %xmm2, %xmm0
-; AVX512-NEXT:    vcvtph2ps %xmm10, %xmm1
+; AVX512-NEXT:    vblendps {{.*#+}} xmm4 = xmm4[0],xmm15[1,2,3]
+; AVX512-NEXT:    vblendps {{.*#+}} xmm5 = xmm5[0],xmm15[1,2,3]
+; AVX512-NEXT:    vblendps {{.*#+}} xmm9 = xmm2[0],xmm15[1,2,3]
+; AVX512-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0],xmm15[1,2,3]
+; AVX512-NEXT:    vcvtps2ph $4, %xmm4, %xmm2
+; AVX512-NEXT:    vcvtps2ph $4, %xmm5, %xmm6
+; AVX512-NEXT:    vcvtps2ph $4, %xmm9, %xmm4
+; AVX512-NEXT:    vcvtps2ph $4, %xmm0, %xmm5
+; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm6[0],xmm2[0],xmm6[1],xmm2[1],xmm6[2],xmm2[2],xmm6[3],xmm2[3]
+; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm9 = xmm4[0],xmm5[0],xmm4[1],xmm5[1],xmm4[2],xmm5[2],xmm4[3],xmm5[3]
+; AVX512-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm9[0],xmm0[0],xmm9[1],xmm0[1]
+; AVX512-NEXT:    vpunpcklqdq {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; AVX512-NEXT:    # xmm0 = xmm0[0],mem[0]
+; AVX512-NEXT:    vmovdqa (%rsp), %xmm9 # 16-byte Reload
+; AVX512-NEXT:    vpunpcklwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm9, %xmm9 # 16-byte Folded Reload
+; AVX512-NEXT:    # xmm9 = xmm9[0],mem[0],xmm9[1],mem[1],xmm9[2],mem[2],xmm9[3],mem[3]
+; AVX512-NEXT:    vpunpcklwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm14, %xmm14 # 16-byte Folded Reload
+; AVX512-NEXT:    # xmm14 = xmm14[0],mem[0],xmm14[1],mem[1],xmm14[2],mem[2],xmm14[3],mem[3]
+; AVX512-NEXT:    vpunpckldq {{.*#+}} xmm9 = xmm14[0],xmm9[0],xmm14[1],xmm9[1]
+; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm10 = xmm10[0],xmm12[0],xmm10[1],xmm12[1],xmm10[2],xmm12[2],xmm10[3],xmm12[3]
+; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm3 = xmm3[0],xmm8[0],xmm3[1],xmm8[1],xmm3[2],xmm8[2],xmm3[3],xmm8[3]
+; AVX512-NEXT:    vpunpckldq {{.*#+}} xmm3 = xmm3[0],xmm10[0],xmm3[1],xmm10[1]
+; AVX512-NEXT:    vpunpcklqdq {{.*#+}} xmm3 = xmm3[0],xmm9[0]
+; AVX512-NEXT:    vpxor %xmm8, %xmm8, %xmm8
+; AVX512-NEXT:    vpcmpeqw %xmm3, %xmm8, %xmm9
+; AVX512-NEXT:    vpblendvb %xmm9, %xmm3, %xmm0, %xmm3
+; AVX512-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm9 # 16-byte Reload
+; AVX512-NEXT:    vpunpcklwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm9, %xmm9 # 16-byte Folded Reload
+; AVX512-NEXT:    # xmm9 = xmm9[0],mem[0],xmm9[1],mem[1],xmm9[2],mem[2],xmm9[3],mem[3]
+; AVX512-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm10 # 16-byte Reload
+; AVX512-NEXT:    vpunpcklwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm10, %xmm10 # 16-byte Folded Reload
+; AVX512-NEXT:    # xmm10 = xmm10[0],mem[0],xmm10[1],mem[1],xmm10[2],mem[2],xmm10[3],mem[3]
+; AVX512-NEXT:    vpunpckldq {{.*#+}} xmm9 = xmm10[0],xmm9[0],xmm10[1],xmm9[1]
+; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm10 = xmm11[0],xmm13[0],xmm11[1],xmm13[1],xmm11[2],xmm13[2],xmm11[3],xmm13[3]
+; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm7[0],xmm1[0],xmm7[1],xmm1[1],xmm7[2],xmm1[2],xmm7[3],xmm1[3]
+; AVX512-NEXT:    vpunpckldq {{.*#+}} xmm1 = xmm1[0],xmm10[0],xmm1[1],xmm10[1]
+; AVX512-NEXT:    vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm9[0]
+; AVX512-NEXT:    vpcmpeqw %xmm1, %xmm8, %xmm7
+; AVX512-NEXT:    vpblendvb %xmm7, %xmm1, %xmm3, %xmm1
+; AVX512-NEXT:    vcvtph2ps %xmm5, %xmm3
 ; AVX512-NEXT:    xorl %eax, %eax
-; AVX512-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512-NEXT:    vucomiss %xmm2, %xmm1
+; AVX512-NEXT:    vpxor %xmm5, %xmm5, %xmm5
+; AVX512-NEXT:    vucomiss %xmm5, %xmm3
 ; AVX512-NEXT:    movl $65535, %ecx # imm = 0xFFFF
 ; AVX512-NEXT:    movl $0, %edx
 ; AVX512-NEXT:    cmovel %ecx, %edx
-; AVX512-NEXT:    vcvtph2ps %xmm9, %xmm1
-; AVX512-NEXT:    vucomiss %xmm2, %xmm1
+; AVX512-NEXT:    vcvtph2ps %xmm4, %xmm3
+; AVX512-NEXT:    vucomiss %xmm5, %xmm3
 ; AVX512-NEXT:    movl $0, %esi
 ; AVX512-NEXT:    cmovel %ecx, %esi
-; AVX512-NEXT:    vcvtph2ps %xmm13, %xmm1
-; AVX512-NEXT:    vucomiss %xmm2, %xmm1
+; AVX512-NEXT:    vcvtph2ps %xmm6, %xmm3
+; AVX512-NEXT:    vucomiss %xmm5, %xmm3
 ; AVX512-NEXT:    movl $0, %edi
 ; AVX512-NEXT:    cmovel %ecx, %edi
-; AVX512-NEXT:    vcvtph2ps %xmm14, %xmm1
-; AVX512-NEXT:    vucomiss %xmm2, %xmm1
+; AVX512-NEXT:    vcvtph2ps %xmm2, %xmm2
+; AVX512-NEXT:    vucomiss %xmm5, %xmm2
 ; AVX512-NEXT:    movl $0, %r8d
 ; AVX512-NEXT:    cmovel %ecx, %r8d
-; AVX512-NEXT:    vcvtph2ps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
-; AVX512-NEXT:    vucomiss %xmm2, %xmm1
+; AVX512-NEXT:    vcvtph2ps {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Folded Reload
+; AVX512-NEXT:    vucomiss %xmm5, %xmm2
 ; AVX512-NEXT:    movl $0, %r9d
 ; AVX512-NEXT:    cmovel %ecx, %r9d
-; AVX512-NEXT:    vcvtph2ps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
-; AVX512-NEXT:    vucomiss %xmm2, %xmm1
+; AVX512-NEXT:    vcvtph2ps {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Folded Reload
+; AVX512-NEXT:    vucomiss %xmm5, %xmm2
 ; AVX512-NEXT:    movl $0, %r10d
 ; AVX512-NEXT:    cmovel %ecx, %r10d
-; AVX512-NEXT:    vcvtph2ps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
-; AVX512-NEXT:    vucomiss %xmm2, %xmm1
+; AVX512-NEXT:    vcvtph2ps {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Folded Reload
+; AVX512-NEXT:    vucomiss %xmm5, %xmm2
 ; AVX512-NEXT:    movl $0, %r11d
 ; AVX512-NEXT:    cmovel %ecx, %r11d
-; AVX512-NEXT:    vcvtph2ps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
-; AVX512-NEXT:    vucomiss %xmm2, %xmm1
-; AVX512-NEXT:    vmovd %esi, %xmm1
-; AVX512-NEXT:    vpinsrw $1, %edx, %xmm1, %xmm1
-; AVX512-NEXT:    vpinsrw $2, %edi, %xmm1, %xmm1
-; AVX512-NEXT:    vpinsrw $3, %r8d, %xmm1, %xmm1
-; AVX512-NEXT:    vpinsrw $4, %r9d, %xmm1, %xmm1
-; AVX512-NEXT:    vpinsrw $5, %r10d, %xmm1, %xmm1
-; AVX512-NEXT:    vpinsrw $6, %r11d, %xmm1, %xmm1
+; AVX512-NEXT:    vcvtph2ps {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Folded Reload
+; AVX512-NEXT:    vucomiss %xmm5, %xmm2
+; AVX512-NEXT:    vmovd %esi, %xmm2
+; AVX512-NEXT:    vpinsrw $1, %edx, %xmm2, %xmm2
+; AVX512-NEXT:    vpinsrw $2, %edi, %xmm2, %xmm2
+; AVX512-NEXT:    vpinsrw $3, %r8d, %xmm2, %xmm2
+; AVX512-NEXT:    vpinsrw $4, %r9d, %xmm2, %xmm2
+; AVX512-NEXT:    vpinsrw $5, %r10d, %xmm2, %xmm2
+; AVX512-NEXT:    vpinsrw $6, %r11d, %xmm2, %xmm2
 ; AVX512-NEXT:    cmovel %ecx, %eax
-; AVX512-NEXT:    vpinsrw $7, %eax, %xmm1, %xmm1
-; AVX512-NEXT:    vpblendvb %xmm1, %xmm0, %xmm6, %xmm0
-; AVX512-NEXT:    addq $88, %rsp
+; AVX512-NEXT:    vpinsrw $7, %eax, %xmm2, %xmm2
+; AVX512-NEXT:    vpblendvb %xmm2, %xmm1, %xmm0, %xmm0
+; AVX512-NEXT:    addq $72, %rsp
 ; AVX512-NEXT:    retq
 ;
 ; AVX10_2-LABEL: test_fmaximumnum_v4f16:
@@ -2204,19 +2154,19 @@ define <4 x half> @test_fmaximumnum_v4f16(<4 x half> %x, <4 x half> %y) nounwind
 ; X86-NEXT:    fstps {{[0-9]+}}(%esp)
 ; X86-NEXT:    fldt {{[-0-9]+}}(%e{{[sb]}}p) # 10-byte Folded Reload
 ; X86-NEXT:    fstps {{[0-9]+}}(%esp)
-; X86-NEXT:    vmovd {{.*#+}} xmm2 = mem[0],zero,zero,zero
-; X86-NEXT:    vmovd %xmm2, %eax
+; X86-NEXT:    vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; X86-NEXT:    vmovd %xmm1, %eax
 ; X86-NEXT:    vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
 ; X86-NEXT:    testl %eax, %eax
 ; X86-NEXT:    js .LBB33_1
 ; X86-NEXT:  # %bb.2:
-; X86-NEXT:    vmovdqa %xmm2, %xmm1
+; X86-NEXT:    vmovdqa %xmm1, %xmm2
 ; X86-NEXT:    jmp .LBB33_3
 ; X86-NEXT:  .LBB33_1:
-; X86-NEXT:    vmovdqa %xmm0, %xmm1
-; X86-NEXT:    vmovdqa %xmm2, %xmm0
+; X86-NEXT:    vmovdqa %xmm0, %xmm2
+; X86-NEXT:    vmovdqa %xmm1, %xmm0
 ; X86-NEXT:  .LBB33_3:
-; X86-NEXT:    vmaxss %xmm1, %xmm0, %xmm1
+; X86-NEXT:    vmaxss %xmm2, %xmm0, %xmm1
 ; X86-NEXT:    vcmpordss %xmm0, %xmm0, %xmm2
 ; X86-NEXT:    vblendvps %xmm2, %xmm0, %xmm1, %xmm0
 ; X86-NEXT:    vmovups %xmm0, {{[-0-9]+}}(%e{{[sb]}}p) # 16-byte Spill
@@ -2327,43 +2277,43 @@ define <4 x bfloat> @test_fmaximumnum_v4bf16(<4 x bfloat> %x, <4 x bfloat> %y) n
 ; SSE2-NEXT:    pushq %r14
 ; SSE2-NEXT:    pushq %rbx
 ; SSE2-NEXT:    subq $56, %rsp
-; SSE2-NEXT:    movdqa %xmm1, %xmm4
-; SSE2-NEXT:    movdqa %xmm0, %xmm5
 ; SSE2-NEXT:    pextrw $0, %xmm1, %r14d
 ; SSE2-NEXT:    pextrw $0, %xmm0, %r15d
-; SSE2-NEXT:    movdqa %xmm1, %xmm0
-; SSE2-NEXT:    psrld $16, %xmm0
-; SSE2-NEXT:    pextrw $0, %xmm0, %eax
-; SSE2-NEXT:    movdqa %xmm5, %xmm0
-; SSE2-NEXT:    psrld $16, %xmm0
-; SSE2-NEXT:    pextrw $0, %xmm0, %ecx
+; SSE2-NEXT:    movdqa %xmm1, %xmm2
+; SSE2-NEXT:    psrld $16, %xmm2
+; SSE2-NEXT:    pextrw $0, %xmm2, %eax
+; SSE2-NEXT:    movdqa %xmm0, %xmm2
+; SSE2-NEXT:    psrld $16, %xmm2
+; SSE2-NEXT:    pextrw $0, %xmm2, %ecx
 ; SSE2-NEXT:    shll $16, %ecx
 ; SSE2-NEXT:    movd %ecx, %xmm3
 ; SSE2-NEXT:    shll $16, %eax
 ; SSE2-NEXT:    movd %eax, %xmm2
 ; SSE2-NEXT:    testl %ecx, %ecx
-; SSE2-NEXT:    movdqa %xmm3, %xmm1
+; SSE2-NEXT:    movdqa %xmm3, %xmm7
 ; SSE2-NEXT:    js .LBB34_2
 ; SSE2-NEXT:  # %bb.1:
-; SSE2-NEXT:    movdqa %xmm2, %xmm1
+; SSE2-NEXT:    movdqa %xmm2, %xmm7
 ; SSE2-NEXT:  .LBB34_2:
-; SSE2-NEXT:    movdqa %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; SSE2-NEXT:    shufps {{.*#+}} xmm4 = xmm4[1,1,1,1]
-; SSE2-NEXT:    movdqa %xmm5, (%rsp) # 16-byte Spill
-; SSE2-NEXT:    shufps {{.*#+}} xmm5 = xmm5[1,1,1,1]
-; SSE2-NEXT:    movdqa %xmm1, %xmm0
-; SSE2-NEXT:    cmpordss %xmm1, %xmm0
-; SSE2-NEXT:    movaps %xmm0, %xmm6
-; SSE2-NEXT:    andps %xmm1, %xmm6
+; SSE2-NEXT:    movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; SSE2-NEXT:    movdqa %xmm1, %xmm5
+; SSE2-NEXT:    shufps {{.*#+}} xmm5 = xmm5[1,1],xmm1[1,1]
+; SSE2-NEXT:    movdqa %xmm0, (%rsp) # 16-byte Spill
+; SSE2-NEXT:    movdqa %xmm0, %xmm6
+; SSE2-NEXT:    shufps {{.*#+}} xmm6 = xmm6[1,1],xmm0[1,1]
+; SSE2-NEXT:    movdqa %xmm7, %xmm0
+; SSE2-NEXT:    cmpordss %xmm7, %xmm0
+; SSE2-NEXT:    movaps %xmm0, %xmm4
+; SSE2-NEXT:    andps %xmm7, %xmm4
 ; SSE2-NEXT:    js .LBB34_4
 ; SSE2-NEXT:  # %bb.3:
 ; SSE2-NEXT:    movdqa %xmm3, %xmm2
 ; SSE2-NEXT:  .LBB34_4:
-; SSE2-NEXT:    pextrw $0, %xmm4, %ebp
-; SSE2-NEXT:    pextrw $0, %xmm5, %ebx
-; SSE2-NEXT:    maxss %xmm2, %xmm1
-; SSE2-NEXT:    andnps %xmm1, %xmm0
-; SSE2-NEXT:    orps %xmm6, %xmm0
+; SSE2-NEXT:    pextrw $0, %xmm5, %ebp
+; SSE2-NEXT:    pextrw $0, %xmm6, %ebx
+; SSE2-NEXT:    maxss %xmm2, %xmm7
+; SSE2-NEXT:    andnps %xmm7, %xmm0
+; SSE2-NEXT:    orps %xmm4, %xmm0
 ; SSE2-NEXT:    callq __truncsfbf2@PLT
 ; SSE2-NEXT:    movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
 ; SSE2-NEXT:    shll $16, %r15d
@@ -2467,7 +2417,7 @@ define <4 x bfloat> @test_fmaximumnum_v4bf16(<4 x bfloat> %x, <4 x bfloat> %y) n
 ; AVX1-NEXT:    vmovshdup {{.*#+}} xmm4 = xmm0[1,1,3,3]
 ; AVX1-NEXT:    vpextrw $0, %xmm4, %ebx
 ; AVX1-NEXT:    vmovshdup {{.*#+}} xmm4 = xmm1[1,1,3,3]
-; AVX1-NEXT:    vpextrw $0, %xmm4, %ebp
+; AVX1-NEXT:    vpextrw $0, %xmm4, %r14d
 ; AVX1-NEXT:    vpextrw $0, %xmm0, %r12d
 ; AVX1-NEXT:    vpextrw $0, %xmm1, %r13d
 ; AVX1-NEXT:    vpsrld $16, %xmm0, %xmm0
@@ -2486,7 +2436,7 @@ define <4 x bfloat> @test_fmaximumnum_v4bf16(<4 x bfloat> %x, <4 x bfloat> %y) n
 ; AVX1-NEXT:    vmovdqa %xmm0, %xmm1
 ; AVX1-NEXT:    vmovdqa %xmm4, %xmm0
 ; AVX1-NEXT:  .LBB34_3:
-; AVX1-NEXT:    vpextrw $0, %xmm2, %r14d
+; AVX1-NEXT:    vpextrw $0, %xmm2, %ebp
 ; AVX1-NEXT:    vpextrw $0, %xmm3, %r15d
 ; AVX1-NEXT:    vmaxss %xmm1, %xmm0, %xmm1
 ; AVX1-NEXT:    vcmpordss %xmm0, %xmm0, %xmm2
@@ -2510,8 +2460,8 @@ define <4 x bfloat> @test_fmaximumnum_v4bf16(<4 x bfloat> %x, <4 x bfloat> %y) n
 ; AVX1-NEXT:    vblendvps %xmm2, %xmm0, %xmm1, %xmm0
 ; AVX1-NEXT:    callq __truncsfbf2@PLT
 ; AVX1-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX1-NEXT:    shll $16, %ebp
-; AVX1-NEXT:    vmovd %ebp, %xmm0
+; AVX1-NEXT:    shll $16, %r14d
+; AVX1-NEXT:    vmovd %r14d, %xmm0
 ; AVX1-NEXT:    shll $16, %ebx
 ; AVX1-NEXT:    vmovd %ebx, %xmm2
 ; AVX1-NEXT:    js .LBB34_7
@@ -2529,8 +2479,8 @@ define <4 x bfloat> @test_fmaximumnum_v4bf16(<4 x bfloat> %x, <4 x bfloat> %y) n
 ; AVX1-NEXT:    vmovaps %xmm0, (%rsp) # 16-byte Spill
 ; AVX1-NEXT:    shll $16, %r15d
 ; AVX1-NEXT:    vmovd %r15d, %xmm0
-; AVX1-NEXT:    shll $16, %r14d
-; AVX1-NEXT:    vmovd %r14d, %xmm2
+; AVX1-NEXT:    shll $16, %ebp
+; AVX1-NEXT:    vmovd %ebp, %xmm2
 ; AVX1-NEXT:    js .LBB34_10
 ; AVX1-NEXT:  # %bb.11:
 ; AVX1-NEXT:    vmovdqa %xmm2, %xmm1
diff --git a/llvm/test/CodeGen/X86/fminnum.ll b/llvm/test/CodeGen/X86/fminnum.ll
index 1290a7b819106..0ef8fdec33d93 100644
--- a/llvm/test/CodeGen/X86/fminnum.ll
+++ b/llvm/test/CodeGen/X86/fminnum.ll
@@ -22,17 +22,26 @@ declare <8 x double> @llvm.minnum.v8f64(<8 x double>, <8 x double>)
 ; FIXME: As the vector tests show, the SSE run shouldn't need this many moves.
 
 define float @test_fminf(float %x, float %y) {
-; SSE-LABEL: test_fminf:
-; SSE:       # %bb.0:
-; SSE-NEXT:    movaps %xmm0, %xmm2
-; SSE-NEXT:    cmpunordss %xmm0, %xmm2
-; SSE-NEXT:    movaps %xmm2, %xmm3
-; SSE-NEXT:    andps %xmm1, %xmm3
-; SSE-NEXT:    minss %xmm0, %xmm1
-; SSE-NEXT:    andnps %xmm1, %xmm2
-; SSE-NEXT:    orps %xmm3, %xmm2
-; SSE-NEXT:    movaps %xmm2, %xmm0
-; SSE-NEXT:    retq
+; SSE2-LABEL: test_fminf:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    movaps %xmm0, %xmm2
+; SSE2-NEXT:    cmpunordss %xmm0, %xmm2
+; SSE2-NEXT:    movaps %xmm2, %xmm3
+; SSE2-NEXT:    andps %xmm1, %xmm3
+; SSE2-NEXT:    minss %xmm0, %xmm1
+; SSE2-NEXT:    andnps %xmm1, %xmm2
+; SSE2-NEXT:    orps %xmm3, %xmm2
+; SSE2-NEXT:    movaps %xmm2, %xmm0
+; SSE2-NEXT:    retq
+;
+; SSE4-LABEL: test_fminf:
+; SSE4:       # %bb.0:
+; SSE4-NEXT:    movaps %xmm1, %xmm2
+; SSE4-NEXT:    minss %xmm0, %xmm2
+; SSE4-NEXT:    cmpunordss %xmm0, %xmm0
+; SSE4-NEXT:    blendvps %xmm0, %xmm1, %xmm2
+; SSE4-NEXT:    movaps %xmm2, %xmm0
+; SSE4-NEXT:    retq
 ;
 ; AVX1-LABEL: test_fminf:
 ; AVX1:       # %bb.0:
@@ -63,17 +72,26 @@ define float @test_fminf_minsize(float %x, float %y) minsize {
 ; FIXME: As the vector tests show, the SSE run shouldn't need this many moves.
 
 define double @test_fmin(double %x, double %y) {
-; SSE-LABEL: test_fmin:
-; SSE:       # %bb.0:
-; SSE-NEXT:    movapd %xmm0, %xmm2
-; SSE-NEXT:    cmpunordsd %xmm0, %xmm2
-; SSE-NEXT:    movapd %xmm2, %xmm3
-; SSE-NEXT:    andpd %xmm1, %xmm3
-; SSE-NEXT:    minsd %xmm0, %xmm1
-; SSE-NEXT:    andnpd %xmm1, %xmm2
-; SSE-NEXT:    orpd %xmm3, %xmm2
-; SSE-NEXT:    movapd %xmm2, %xmm0
-; SSE-NEXT:    retq
+; SSE2-LABEL: test_fmin:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    movapd %xmm0, %xmm2
+; SSE2-NEXT:    cmpunordsd %xmm0, %xmm2
+; SSE2-NEXT:    movapd %xmm2, %xmm3
+; SSE2-NEXT:    andpd %xmm1, %xmm3
+; SSE2-NEXT:    minsd %xmm0, %xmm1
+; SSE2-NEXT:    andnpd %xmm1, %xmm2
+; SSE2-NEXT:    orpd %xmm3, %xmm2
+; SSE2-NEXT:    movapd %xmm2, %xmm0
+; SSE2-NEXT:    retq
+;
+; SSE4-LABEL: test_fmin:
+; SSE4:       # %bb.0:
+; SSE4-NEXT:    movapd %xmm1, %xmm2
+; SSE4-NEXT:    minsd %xmm0, %xmm2
+; SSE4-NEXT:    cmpunordsd %xmm0, %xmm0
+; SSE4-NEXT:    blendvpd %xmm0, %xmm1, %xmm2
+; SSE4-NEXT:    movapd %xmm2, %xmm0
+; SSE4-NEXT:    retq
 ;
 ; AVX1-LABEL: test_fmin:
 ; AVX1:       # %bb.0:
@@ -111,17 +129,26 @@ define x86_fp80 @test_fminl(x86_fp80 %x, x86_fp80 %y) {
 }
 
 define float @test_intrinsic_fminf(float %x, float %y) {
-; SSE-LABEL: test_intrinsic_fminf:
-; SSE:       # %bb.0:
-; SSE-NEXT:    movaps %xmm0, %xmm2
-; SSE-NEXT:    cmpunordss %xmm0, %xmm2
-; SSE-NEXT:    movaps %xmm2, %xmm3
-; SSE-NEXT:    andps %xmm1, %xmm3
-; SSE-NEXT:    minss %xmm0, %xmm1
-; SSE-NEXT:    andnps %xmm1, %xmm2
-; SSE-NEXT:    orps %xmm3, %xmm2
-; SSE-NEXT:    movaps %xmm2, %xmm0
-; SSE-NEXT:    retq
+; SSE2-LABEL: test_intrinsic_fminf:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    movaps %xmm0, %xmm2
+; SSE2-NEXT:    cmpunordss %xmm0, %xmm2
+; SSE2-NEXT:    movaps %xmm2, %xmm3
+; SSE2-NEXT:    andps %xmm1, %xmm3
+; SSE2-NEXT:    minss %xmm0, %xmm1
+; SSE2-NEXT:    andnps %xmm1, %xmm2
+; SSE2-NEXT:    orps %xmm3, %xmm2
+; SSE2-NEXT:    movaps %xmm2, %xmm0
+; SSE2-NEXT:    retq
+;
+; SSE4-LABEL: test_intrinsic_fminf:
+; SSE4:       # %bb.0:
+; SSE4-NEXT:    movaps %xmm1, %xmm2
+; SSE4-NEXT:    minss %xmm0, %xmm2
+; SSE4-NEXT:    cmpunordss %xmm0, %xmm0
+; SSE4-NEXT:    blendvps %xmm0, %xmm1, %xmm2
+; SSE4-NEXT:    movaps %xmm2, %xmm0
+; SSE4-NEXT:    retq
 ;
 ; AVX1-LABEL: test_intrinsic_fminf:
 ; AVX1:       # %bb.0:
@@ -142,17 +169,26 @@ define float @test_intrinsic_fminf(float %x, float %y) {
 }
 
 define double @test_intrinsic_fmin(double %x, double %y) {
-; SSE-LABEL: test_intrinsic_fmin:
-; SSE:       # %bb.0:
-; SSE-NEXT:    movapd %xmm0, %xmm2
-; SSE-NEXT:    cmpunordsd %xmm0, %xmm2
-; SSE-NEXT:    movapd %xmm2, %xmm3
-; SSE-NEXT:    andpd %xmm1, %xmm3
-; SSE-NEXT:    minsd %xmm0, %xmm1
-; SSE-NEXT:    andnpd %xmm1, %xmm2
-; SSE-NEXT:    orpd %xmm3, %xmm2
-; SSE-NEXT:    movapd %xmm2, %xmm0
-; SSE-NEXT:    retq
+; SSE2-LABEL: test_intrinsic_fmin:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    movapd %xmm0, %xmm2
+; SSE2-NEXT:    cmpunordsd %xmm0, %xmm2
+; SSE2-NEXT:    movapd %xmm2, %xmm3
+; SSE2-NEXT:    andpd %xmm1, %xmm3
+; SSE2-NEXT:    minsd %xmm0, %xmm1
+; SSE2-NEXT:    andnpd %xmm1, %xmm2
+; SSE2-NEXT:    orpd %xmm3, %xmm2
+; SSE2-NEXT:    movapd %xmm2, %xmm0
+; SSE2-NEXT:    retq
+;
+; SSE4-LABEL: test_intrinsic_fmin:
+; SSE4:       # %bb.0:
+; SSE4-NEXT:    movapd %xmm1, %xmm2
+; SSE4-NEXT:    minsd %xmm0, %xmm2
+; SSE4-NEXT:    cmpunordsd %xmm0, %xmm0
+; SSE4-NEXT:    blendvpd %xmm0, %xmm1, %xmm2
+; SSE4-NEXT:    movapd %xmm2, %xmm0
+; SSE4-NEXT:    retq
 ;
 ; AVX1-LABEL: test_intrinsic_fmin:
 ; AVX1:       # %bb.0:
diff --git a/llvm/test/CodeGen/X86/fold-int-pow2-with-fmul-or-fdiv.ll b/llvm/test/CodeGen/X86/fold-int-pow2-with-fmul-or-fdiv.ll
index 2163121410553..53517373d3e4d 100644
--- a/llvm/test/CodeGen/X86/fold-int-pow2-with-fmul-or-fdiv.ll
+++ b/llvm/test/CodeGen/X86/fold-int-pow2-with-fmul-or-fdiv.ll
@@ -569,17 +569,13 @@ define <8 x half> @fmul_pow2_ldexp_8xhalf(<8 x i16> %i) {
 ; CHECK-AVX512F-NEXT:    vmovss {{.*#+}} xmm0 = [8.192E+3,0.0E+0,0.0E+0,0.0E+0]
 ; CHECK-AVX512F-NEXT:    callq ldexpf@PLT
 ; CHECK-AVX512F-NEXT:    vcvtps2ph $4, %xmm0, %xmm0
-; CHECK-AVX512F-NEXT:    vmovd %xmm0, %eax
-; CHECK-AVX512F-NEXT:    vpinsrw $0, %eax, %xmm0, %xmm0
-; CHECK-AVX512F-NEXT:    vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-AVX512F-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
 ; CHECK-AVX512F-NEXT:    vmovdqa (%rsp), %xmm0 # 16-byte Reload
 ; CHECK-AVX512F-NEXT:    vpextrw $6, %xmm0, %eax
 ; CHECK-AVX512F-NEXT:    movswl %ax, %edi
 ; CHECK-AVX512F-NEXT:    vmovss {{.*#+}} xmm0 = [8.192E+3,0.0E+0,0.0E+0,0.0E+0]
 ; CHECK-AVX512F-NEXT:    callq ldexpf@PLT
 ; CHECK-AVX512F-NEXT:    vcvtps2ph $4, %xmm0, %xmm0
-; CHECK-AVX512F-NEXT:    vmovd %xmm0, %eax
-; CHECK-AVX512F-NEXT:    vpinsrw $0, %eax, %xmm0, %xmm0
 ; CHECK-AVX512F-NEXT:    vpunpcklwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
 ; CHECK-AVX512F-NEXT:    # xmm0 = xmm0[0],mem[0],xmm0[1],mem[1],xmm0[2],mem[2],xmm0[3],mem[3]
 ; CHECK-AVX512F-NEXT:    vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
@@ -589,17 +585,13 @@ define <8 x half> @fmul_pow2_ldexp_8xhalf(<8 x i16> %i) {
 ; CHECK-AVX512F-NEXT:    vmovss {{.*#+}} xmm0 = [8.192E+3,0.0E+0,0.0E+0,0.0E+0]
 ; CHECK-AVX512F-NEXT:    callq ldexpf@PLT
 ; CHECK-AVX512F-NEXT:    vcvtps2ph $4, %xmm0, %xmm0
-; CHECK-AVX512F-NEXT:    vmovd %xmm0, %eax
-; CHECK-AVX512F-NEXT:    vpinsrw $0, %eax, %xmm0, %xmm0
-; CHECK-AVX512F-NEXT:    vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-AVX512F-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
 ; CHECK-AVX512F-NEXT:    vmovdqa (%rsp), %xmm0 # 16-byte Reload
 ; CHECK-AVX512F-NEXT:    vpextrw $4, %xmm0, %eax
 ; CHECK-AVX512F-NEXT:    movswl %ax, %edi
 ; CHECK-AVX512F-NEXT:    vmovss {{.*#+}} xmm0 = [8.192E+3,0.0E+0,0.0E+0,0.0E+0]
 ; CHECK-AVX512F-NEXT:    callq ldexpf@PLT
 ; CHECK-AVX512F-NEXT:    vcvtps2ph $4, %xmm0, %xmm0
-; CHECK-AVX512F-NEXT:    vmovd %xmm0, %eax
-; CHECK-AVX512F-NEXT:    vpinsrw $0, %eax, %xmm0, %xmm0
 ; CHECK-AVX512F-NEXT:    vpunpcklwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
 ; CHECK-AVX512F-NEXT:    # xmm0 = xmm0[0],mem[0],xmm0[1],mem[1],xmm0[2],mem[2],xmm0[3],mem[3]
 ; CHECK-AVX512F-NEXT:    vpunpckldq {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
@@ -611,17 +603,13 @@ define <8 x half> @fmul_pow2_ldexp_8xhalf(<8 x i16> %i) {
 ; CHECK-AVX512F-NEXT:    vmovss {{.*#+}} xmm0 = [8.192E+3,0.0E+0,0.0E+0,0.0E+0]
 ; CHECK-AVX512F-NEXT:    callq ldexpf@PLT
 ; CHECK-AVX512F-NEXT:    vcvtps2ph $4, %xmm0, %xmm0
-; CHECK-AVX512F-NEXT:    vmovd %xmm0, %eax
-; CHECK-AVX512F-NEXT:    vpinsrw $0, %eax, %xmm0, %xmm0
-; CHECK-AVX512F-NEXT:    vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-AVX512F-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
 ; CHECK-AVX512F-NEXT:    vmovdqa (%rsp), %xmm0 # 16-byte Reload
 ; CHECK-AVX512F-NEXT:    vpextrw $2, %xmm0, %eax
 ; CHECK-AVX512F-NEXT:    movswl %ax, %edi
 ; CHECK-AVX512F-NEXT:    vmovss {{.*#+}} xmm0 = [8.192E+3,0.0E+0,0.0E+0,0.0E+0]
 ; CHECK-AVX512F-NEXT:    callq ldexpf@PLT
 ; CHECK-AVX512F-NEXT:    vcvtps2ph $4, %xmm0, %xmm0
-; CHECK-AVX512F-NEXT:    vmovd %xmm0, %eax
-; CHECK-AVX512F-NEXT:    vpinsrw $0, %eax, %xmm0, %xmm0
 ; CHECK-AVX512F-NEXT:    vpunpcklwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
 ; CHECK-AVX512F-NEXT:    # xmm0 = xmm0[0],mem[0],xmm0[1],mem[1],xmm0[2],mem[2],xmm0[3],mem[3]
 ; CHECK-AVX512F-NEXT:    vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
@@ -631,17 +619,13 @@ define <8 x half> @fmul_pow2_ldexp_8xhalf(<8 x i16> %i) {
 ; CHECK-AVX512F-NEXT:    vmovss {{.*#+}} xmm0 = [8.192E+3,0.0E+0,0.0E+0,0.0E+0]
 ; CHECK-AVX512F-NEXT:    callq ldexpf@PLT
 ; CHECK-AVX512F-NEXT:    vcvtps2ph $4, %xmm0, %xmm0
-; CHECK-AVX512F-NEXT:    vmovd %xmm0, %eax
-; CHECK-AVX512F-NEXT:    vpinsrw $0, %eax, %xmm0, %xmm0
-; CHECK-AVX512F-NEXT:    vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-AVX512F-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
 ; CHECK-AVX512F-NEXT:    vmovdqa (%rsp), %xmm0 # 16-byte Reload
 ; CHECK-AVX512F-NEXT:    vmovd %xmm0, %eax
 ; CHECK-AVX512F-NEXT:    movswl %ax, %edi
 ; CHECK-AVX512F-NEXT:    vmovss {{.*#+}} xmm0 = [8.192E+3,0.0E+0,0.0E+0,0.0E+0]
 ; CHECK-AVX512F-NEXT:    callq ldexpf@PLT
 ; CHECK-AVX512F-NEXT:    vcvtps2ph $4, %xmm0, %xmm0
-; CHECK-AVX512F-NEXT:    vmovd %xmm0, %eax
-; CHECK-AVX512F-NEXT:    vpinsrw $0, %eax, %xmm0, %xmm0
 ; CHECK-AVX512F-NEXT:    vpunpcklwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
 ; CHECK-AVX512F-NEXT:    # xmm0 = xmm0[0],mem[0],xmm0[1],mem[1],xmm0[2],mem[2],xmm0[3],mem[3]
 ; CHECK-AVX512F-NEXT:    vpunpckldq {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
@@ -1429,8 +1413,6 @@ define half @fdiv_pow_shl_cnt_fail_out_of_bounds(i32 %cnt) nounwind {
 ; CHECK-NO-FASTFMA-NEXT:    vmovss {{.*#+}} xmm1 = [8.192E+3,0.0E+0,0.0E+0,0.0E+0]
 ; CHECK-NO-FASTFMA-NEXT:    vdivss %xmm0, %xmm1, %xmm0
 ; CHECK-NO-FASTFMA-NEXT:    vcvtps2ph $4, %xmm0, %xmm0
-; CHECK-NO-FASTFMA-NEXT:    vmovd %xmm0, %eax
-; CHECK-NO-FASTFMA-NEXT:    vpinsrw $0, %eax, %xmm0, %xmm0
 ; CHECK-NO-FASTFMA-NEXT:    retq
 ;
 ; CHECK-FMA-LABEL: fdiv_pow_shl_cnt_fail_out_of_bounds:
@@ -1443,8 +1425,6 @@ define half @fdiv_pow_shl_cnt_fail_out_of_bounds(i32 %cnt) nounwind {
 ; CHECK-FMA-NEXT:    vmovss {{.*#+}} xmm1 = [8.192E+3,0.0E+0,0.0E+0,0.0E+0]
 ; CHECK-FMA-NEXT:    vdivss %xmm0, %xmm1, %xmm0
 ; CHECK-FMA-NEXT:    vcvtps2ph $4, %xmm0, %xmm0
-; CHECK-FMA-NEXT:    vmovd %xmm0, %eax
-; CHECK-FMA-NEXT:    vpinsrw $0, %eax, %xmm0, %xmm0
 ; CHECK-FMA-NEXT:    retq
   %shl = shl nuw i32 1, %cnt
   %conv = uitofp i32 %shl to half
@@ -1545,8 +1525,6 @@ define half @fdiv_pow_shl_cnt_fail_out_of_bound2(i16 %cnt) nounwind {
 ; CHECK-NO-FASTFMA-NEXT:    vmovss {{.*#+}} xmm1 = [2.0E+0,0.0E+0,0.0E+0,0.0E+0]
 ; CHECK-NO-FASTFMA-NEXT:    vdivss %xmm0, %xmm1, %xmm0
 ; CHECK-NO-FASTFMA-NEXT:    vcvtps2ph $4, %xmm0, %xmm0
-; CHECK-NO-FASTFMA-NEXT:    vmovd %xmm0, %eax
-; CHECK-NO-FASTFMA-NEXT:    vpinsrw $0, %eax, %xmm0, %xmm0
 ; CHECK-NO-FASTFMA-NEXT:    retq
 ;
 ; CHECK-FMA-LABEL: fdiv_pow_shl_cnt_fail_out_of_bound2:
@@ -1560,8 +1538,6 @@ define half @fdiv_pow_shl_cnt_fail_out_of_bound2(i16 %cnt) nounwind {
 ; CHECK-FMA-NEXT:    vmovss {{.*#+}} xmm1 = [2.0E+0,0.0E+0,0.0E+0,0.0E+0]
 ; CHECK-FMA-NEXT:    vdivss %xmm0, %xmm1, %xmm0
 ; CHECK-FMA-NEXT:    vcvtps2ph $4, %xmm0, %xmm0
-; CHECK-FMA-NEXT:    vmovd %xmm0, %eax
-; CHECK-FMA-NEXT:    vpinsrw $0, %eax, %xmm0, %xmm0
 ; CHECK-FMA-NEXT:    retq
   %shl = shl nuw i16 1, %cnt
   %conv = uitofp i16 %shl to half
diff --git a/llvm/test/CodeGen/X86/fp-round.ll b/llvm/test/CodeGen/X86/fp-round.ll
index 8efd5819a6d22..58c4f71892e90 100644
--- a/llvm/test/CodeGen/X86/fp-round.ll
+++ b/llvm/test/CodeGen/X86/fp-round.ll
@@ -55,19 +55,17 @@ define half @round_f16(half %h) {
 ; AVX512F-NEXT:    vmovd %eax, %xmm0
 ; AVX512F-NEXT:    vcvtph2ps %xmm0, %xmm0
 ; AVX512F-NEXT:    vpbroadcastd {{.*#+}} xmm1 = [4.9999997E-1,4.9999997E-1,4.9999997E-1,4.9999997E-1]
-; AVX512F-NEXT:    vpternlogd $248, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to4}, %xmm0, %xmm1
+; AVX512F-NEXT:    vpternlogd {{.*#+}} xmm1 = xmm1 | (xmm0 & mem)
 ; AVX512F-NEXT:    vaddss %xmm1, %xmm0, %xmm0
 ; AVX512F-NEXT:    vroundss $11, %xmm0, %xmm0, %xmm0
 ; AVX512F-NEXT:    vcvtps2ph $4, %xmm0, %xmm0
-; AVX512F-NEXT:    vmovd %xmm0, %eax
-; AVX512F-NEXT:    vpinsrw $0, %eax, %xmm0, %xmm0
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512FP16-LABEL: round_f16:
 ; AVX512FP16:       ## %bb.0: ## %entry
 ; AVX512FP16-NEXT:    vpbroadcastw {{.*#+}} xmm1 = [-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0]
 ; AVX512FP16-NEXT:    vpbroadcastw {{.*#+}} xmm2 = [4.9976E-1,4.9976E-1,4.9976E-1,4.9976E-1,4.9976E-1,4.9976E-1,4.9976E-1,4.9976E-1]
-; AVX512FP16-NEXT:    vpternlogq $248, %xmm1, %xmm0, %xmm2
+; AVX512FP16-NEXT:    vpternlogq {{.*#+}} xmm2 = xmm2 | (xmm0 & xmm1)
 ; AVX512FP16-NEXT:    vaddsh %xmm2, %xmm0, %xmm0
 ; AVX512FP16-NEXT:    vrndscalesh $11, %xmm0, %xmm0, %xmm0
 ; AVX512FP16-NEXT:    retq
@@ -103,7 +101,7 @@ define float @round_f32(float %x) {
 ; AVX512F-LABEL: round_f32:
 ; AVX512F:       # %bb.0:
 ; AVX512F-NEXT:    vpbroadcastd {{.*#+}} xmm1 = [4.9999997E-1,4.9999997E-1,4.9999997E-1,4.9999997E-1]
-; AVX512F-NEXT:    vpternlogd $248, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to4}, %xmm0, %xmm1
+; AVX512F-NEXT:    vpternlogd {{.*#+}} xmm1 = xmm1 | (xmm0 & mem)
 ; AVX512F-NEXT:    vaddss %xmm1, %xmm0, %xmm0
 ; AVX512F-NEXT:    vroundss $11, %xmm0, %xmm0, %xmm0
 ; AVX512F-NEXT:    retq
@@ -111,7 +109,7 @@ define float @round_f32(float %x) {
 ; AVX512FP16-LABEL: round_f32:
 ; AVX512FP16:       ## %bb.0:
 ; AVX512FP16-NEXT:    vpbroadcastd {{.*#+}} xmm1 = [4.9999997E-1,4.9999997E-1,4.9999997E-1,4.9999997E-1]
-; AVX512FP16-NEXT:    vpternlogd $248, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to4}, %xmm0, %xmm1
+; AVX512FP16-NEXT:    vpternlogd {{.*#+}} xmm1 = xmm1 | (xmm0 & mem)
 ; AVX512FP16-NEXT:    vaddss %xmm1, %xmm0, %xmm0
 ; AVX512FP16-NEXT:    vroundss $11, %xmm0, %xmm0, %xmm0
 ; AVX512FP16-NEXT:    retq
@@ -147,7 +145,7 @@ define double @round_f64(double %x) {
 ; AVX512F-LABEL: round_f64:
 ; AVX512F:       # %bb.0:
 ; AVX512F-NEXT:    vpbroadcastq {{.*#+}} xmm1 = [4.9999999999999994E-1,4.9999999999999994E-1]
-; AVX512F-NEXT:    vpternlogq $248, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to2}, %xmm0, %xmm1
+; AVX512F-NEXT:    vpternlogq {{.*#+}} xmm1 = xmm1 | (xmm0 & mem)
 ; AVX512F-NEXT:    vaddsd %xmm1, %xmm0, %xmm0
 ; AVX512F-NEXT:    vroundsd $11, %xmm0, %xmm0, %xmm0
 ; AVX512F-NEXT:    retq
@@ -155,7 +153,7 @@ define double @round_f64(double %x) {
 ; AVX512FP16-LABEL: round_f64:
 ; AVX512FP16:       ## %bb.0:
 ; AVX512FP16-NEXT:    vpbroadcastq {{.*#+}} xmm1 = [4.9999999999999994E-1,4.9999999999999994E-1]
-; AVX512FP16-NEXT:    vpternlogq $248, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to2}, %xmm0, %xmm1
+; AVX512FP16-NEXT:    vpternlogq {{.*#+}} xmm1 = xmm1 | (xmm0 & mem)
 ; AVX512FP16-NEXT:    vaddsd %xmm1, %xmm0, %xmm0
 ; AVX512FP16-NEXT:    vroundsd $11, %xmm0, %xmm0, %xmm0
 ; AVX512FP16-NEXT:    retq
@@ -213,7 +211,7 @@ define <4 x float> @round_v4f32(<4 x float> %x) {
 ; AVX512F-LABEL: round_v4f32:
 ; AVX512F:       # %bb.0:
 ; AVX512F-NEXT:    vpbroadcastd {{.*#+}} xmm1 = [4.9999997E-1,4.9999997E-1,4.9999997E-1,4.9999997E-1]
-; AVX512F-NEXT:    vpternlogd $248, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to4}, %xmm0, %xmm1
+; AVX512F-NEXT:    vpternlogd {{.*#+}} xmm1 = xmm1 | (xmm0 & mem)
 ; AVX512F-NEXT:    vaddps %xmm1, %xmm0, %xmm0
 ; AVX512F-NEXT:    vroundps $11, %xmm0, %xmm0
 ; AVX512F-NEXT:    retq
@@ -221,7 +219,7 @@ define <4 x float> @round_v4f32(<4 x float> %x) {
 ; AVX512FP16-LABEL: round_v4f32:
 ; AVX512FP16:       ## %bb.0:
 ; AVX512FP16-NEXT:    vpbroadcastd {{.*#+}} xmm1 = [4.9999997E-1,4.9999997E-1,4.9999997E-1,4.9999997E-1]
-; AVX512FP16-NEXT:    vpternlogd $248, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to4}, %xmm0, %xmm1
+; AVX512FP16-NEXT:    vpternlogd {{.*#+}} xmm1 = xmm1 | (xmm0 & mem)
 ; AVX512FP16-NEXT:    vaddps %xmm1, %xmm0, %xmm0
 ; AVX512FP16-NEXT:    vroundps $11, %xmm0, %xmm0
 ; AVX512FP16-NEXT:    retq
@@ -267,7 +265,7 @@ define <2 x double> @round_v2f64(<2 x double> %x) {
 ; AVX512F-LABEL: round_v2f64:
 ; AVX512F:       # %bb.0:
 ; AVX512F-NEXT:    vpbroadcastq {{.*#+}} xmm1 = [4.9999999999999994E-1,4.9999999999999994E-1]
-; AVX512F-NEXT:    vpternlogq $248, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to2}, %xmm0, %xmm1
+; AVX512F-NEXT:    vpternlogq {{.*#+}} xmm1 = xmm1 | (xmm0 & mem)
 ; AVX512F-NEXT:    vaddpd %xmm1, %xmm0, %xmm0
 ; AVX512F-NEXT:    vroundpd $11, %xmm0, %xmm0
 ; AVX512F-NEXT:    retq
@@ -275,7 +273,7 @@ define <2 x double> @round_v2f64(<2 x double> %x) {
 ; AVX512FP16-LABEL: round_v2f64:
 ; AVX512FP16:       ## %bb.0:
 ; AVX512FP16-NEXT:    vpbroadcastq {{.*#+}} xmm1 = [4.9999999999999994E-1,4.9999999999999994E-1]
-; AVX512FP16-NEXT:    vpternlogq $248, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to2}, %xmm0, %xmm1
+; AVX512FP16-NEXT:    vpternlogq {{.*#+}} xmm1 = xmm1 | (xmm0 & mem)
 ; AVX512FP16-NEXT:    vaddpd %xmm1, %xmm0, %xmm0
 ; AVX512FP16-NEXT:    vroundpd $11, %xmm0, %xmm0
 ; AVX512FP16-NEXT:    retq
@@ -361,7 +359,7 @@ define <8 x float> @round_v8f32(<8 x float> %x) {
 ; AVX512F-LABEL: round_v8f32:
 ; AVX512F:       # %bb.0:
 ; AVX512F-NEXT:    vpbroadcastd {{.*#+}} ymm1 = [4.9999997E-1,4.9999997E-1,4.9999997E-1,4.9999997E-1,4.9999997E-1,4.9999997E-1,4.9999997E-1,4.9999997E-1]
-; AVX512F-NEXT:    vpternlogd $248, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm0, %ymm1
+; AVX512F-NEXT:    vpternlogd {{.*#+}} ymm1 = ymm1 | (ymm0 & mem)
 ; AVX512F-NEXT:    vaddps %ymm1, %ymm0, %ymm0
 ; AVX512F-NEXT:    vroundps $11, %ymm0, %ymm0
 ; AVX512F-NEXT:    retq
@@ -369,7 +367,7 @@ define <8 x float> @round_v8f32(<8 x float> %x) {
 ; AVX512FP16-LABEL: round_v8f32:
 ; AVX512FP16:       ## %bb.0:
 ; AVX512FP16-NEXT:    vpbroadcastd {{.*#+}} ymm1 = [4.9999997E-1,4.9999997E-1,4.9999997E-1,4.9999997E-1,4.9999997E-1,4.9999997E-1,4.9999997E-1,4.9999997E-1]
-; AVX512FP16-NEXT:    vpternlogd $248, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm0, %ymm1
+; AVX512FP16-NEXT:    vpternlogd {{.*#+}} ymm1 = ymm1 | (ymm0 & mem)
 ; AVX512FP16-NEXT:    vaddps %ymm1, %ymm0, %ymm0
 ; AVX512FP16-NEXT:    vroundps $11, %ymm0, %ymm0
 ; AVX512FP16-NEXT:    retq
@@ -431,7 +429,7 @@ define <4 x double> @round_v4f64(<4 x double> %x) {
 ; AVX512F-LABEL: round_v4f64:
 ; AVX512F:       # %bb.0:
 ; AVX512F-NEXT:    vpbroadcastq {{.*#+}} ymm1 = [4.9999999999999994E-1,4.9999999999999994E-1,4.9999999999999994E-1,4.9999999999999994E-1]
-; AVX512F-NEXT:    vpternlogq $248, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to4}, %ymm0, %ymm1
+; AVX512F-NEXT:    vpternlogq {{.*#+}} ymm1 = ymm1 | (ymm0 & mem)
 ; AVX512F-NEXT:    vaddpd %ymm1, %ymm0, %ymm0
 ; AVX512F-NEXT:    vroundpd $11, %ymm0, %ymm0
 ; AVX512F-NEXT:    retq
@@ -439,7 +437,7 @@ define <4 x double> @round_v4f64(<4 x double> %x) {
 ; AVX512FP16-LABEL: round_v4f64:
 ; AVX512FP16:       ## %bb.0:
 ; AVX512FP16-NEXT:    vpbroadcastq {{.*#+}} ymm1 = [4.9999999999999994E-1,4.9999999999999994E-1,4.9999999999999994E-1,4.9999999999999994E-1]
-; AVX512FP16-NEXT:    vpternlogq $248, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to4}, %ymm0, %ymm1
+; AVX512FP16-NEXT:    vpternlogq {{.*#+}} ymm1 = ymm1 | (ymm0 & mem)
 ; AVX512FP16-NEXT:    vaddpd %ymm1, %ymm0, %ymm0
 ; AVX512FP16-NEXT:    vroundpd $11, %ymm0, %ymm0
 ; AVX512FP16-NEXT:    retq
@@ -587,7 +585,7 @@ define <16 x float> @round_v16f32(<16 x float> %x) {
 ; AVX512F-LABEL: round_v16f32:
 ; AVX512F:       # %bb.0:
 ; AVX512F-NEXT:    vpbroadcastd {{.*#+}} zmm1 = [4.9999997E-1,4.9999997E-1,4.9999997E-1,4.9999997E-1,4.9999997E-1,4.9999997E-1,4.9999997E-1,4.9999997E-1,4.9999997E-1,4.9999997E-1,4.9999997E-1,4.9999997E-1,4.9999997E-1,4.9999997E-1,4.9999997E-1,4.9999997E-1]
-; AVX512F-NEXT:    vpternlogd $248, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm0, %zmm1
+; AVX512F-NEXT:    vpternlogd {{.*#+}} zmm1 = zmm1 | (zmm0 & mem)
 ; AVX512F-NEXT:    vaddps %zmm1, %zmm0, %zmm0
 ; AVX512F-NEXT:    vrndscaleps $11, %zmm0, %zmm0
 ; AVX512F-NEXT:    retq
@@ -595,7 +593,7 @@ define <16 x float> @round_v16f32(<16 x float> %x) {
 ; AVX512FP16-LABEL: round_v16f32:
 ; AVX512FP16:       ## %bb.0:
 ; AVX512FP16-NEXT:    vpbroadcastd {{.*#+}} zmm1 = [4.9999997E-1,4.9999997E-1,4.9999997E-1,4.9999997E-1,4.9999997E-1,4.9999997E-1,4.9999997E-1,4.9999997E-1,4.9999997E-1,4.9999997E-1,4.9999997E-1,4.9999997E-1,4.9999997E-1,4.9999997E-1,4.9999997E-1,4.9999997E-1]
-; AVX512FP16-NEXT:    vpternlogd $248, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm0, %zmm1
+; AVX512FP16-NEXT:    vpternlogd {{.*#+}} zmm1 = zmm1 | (zmm0 & mem)
 ; AVX512FP16-NEXT:    vaddps %zmm1, %zmm0, %zmm0
 ; AVX512FP16-NEXT:    vrndscaleps $11, %zmm0, %zmm0
 ; AVX512FP16-NEXT:    retq
@@ -695,7 +693,7 @@ define <8 x double> @round_v8f64(<8 x double> %x) {
 ; AVX512F-LABEL: round_v8f64:
 ; AVX512F:       # %bb.0:
 ; AVX512F-NEXT:    vpbroadcastq {{.*#+}} zmm1 = [4.9999999999999994E-1,4.9999999999999994E-1,4.9999999999999994E-1,4.9999999999999994E-1,4.9999999999999994E-1,4.9999999999999994E-1,4.9999999999999994E-1,4.9999999999999994E-1]
-; AVX512F-NEXT:    vpternlogq $248, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm0, %zmm1
+; AVX512F-NEXT:    vpternlogq {{.*#+}} zmm1 = zmm1 | (zmm0 & mem)
 ; AVX512F-NEXT:    vaddpd %zmm1, %zmm0, %zmm0
 ; AVX512F-NEXT:    vrndscalepd $11, %zmm0, %zmm0
 ; AVX512F-NEXT:    retq
@@ -703,7 +701,7 @@ define <8 x double> @round_v8f64(<8 x double> %x) {
 ; AVX512FP16-LABEL: round_v8f64:
 ; AVX512FP16:       ## %bb.0:
 ; AVX512FP16-NEXT:    vpbroadcastq {{.*#+}} zmm1 = [4.9999999999999994E-1,4.9999999999999994E-1,4.9999999999999994E-1,4.9999999999999994E-1,4.9999999999999994E-1,4.9999999999999994E-1,4.9999999999999994E-1,4.9999999999999994E-1]
-; AVX512FP16-NEXT:    vpternlogq $248, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm0, %zmm1
+; AVX512FP16-NEXT:    vpternlogq {{.*#+}} zmm1 = zmm1 | (zmm0 & mem)
 ; AVX512FP16-NEXT:    vaddpd %zmm1, %zmm0, %zmm0
 ; AVX512FP16-NEXT:    vrndscalepd $11, %zmm0, %zmm0
 ; AVX512FP16-NEXT:    retq
diff --git a/llvm/test/CodeGen/X86/fp-roundeven.ll b/llvm/test/CodeGen/X86/fp-roundeven.ll
index 7d1c52cd65451..ee298fd47f728 100644
--- a/llvm/test/CodeGen/X86/fp-roundeven.ll
+++ b/llvm/test/CodeGen/X86/fp-roundeven.ll
@@ -53,8 +53,6 @@ define half @roundeven_f16(half %h) {
 ; AVX512F-NEXT:    vcvtph2ps %xmm0, %xmm0
 ; AVX512F-NEXT:    vroundss $8, %xmm0, %xmm0, %xmm0
 ; AVX512F-NEXT:    vcvtps2ph $4, %xmm0, %xmm0
-; AVX512F-NEXT:    vmovd %xmm0, %eax
-; AVX512F-NEXT:    vpinsrw $0, %eax, %xmm0, %xmm0
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512FP16-LABEL: roundeven_f16:
diff --git a/llvm/test/CodeGen/X86/fp-select-cmp-and.ll b/llvm/test/CodeGen/X86/fp-select-cmp-and.ll
index 0f6159d36ea81..1d006f725ca34 100644
--- a/llvm/test/CodeGen/X86/fp-select-cmp-and.ll
+++ b/llvm/test/CodeGen/X86/fp-select-cmp-and.ll
@@ -189,10 +189,9 @@ define float @test17(float %a, float %b, float %c, float %eps) {
 ; CHECK-LABEL: test17:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    cmpless %xmm0, %xmm3
-; CHECK-NEXT:    andps %xmm3, %xmm2
-; CHECK-NEXT:    andnps %xmm1, %xmm3
-; CHECK-NEXT:    orps %xmm2, %xmm3
 ; CHECK-NEXT:    movaps %xmm3, %xmm0
+; CHECK-NEXT:    blendvps %xmm0, %xmm2, %xmm1
+; CHECK-NEXT:    movaps %xmm1, %xmm0
 ; CHECK-NEXT:    retq
   %cmp = fcmp oge float %a, %eps
   %cond = select i1 %cmp, float %c, float %b
@@ -203,10 +202,9 @@ define double @test18(double %a, double %b, double %c, double %eps) {
 ; CHECK-LABEL: test18:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    cmplesd %xmm0, %xmm3
-; CHECK-NEXT:    andpd %xmm3, %xmm2
-; CHECK-NEXT:    andnpd %xmm1, %xmm3
-; CHECK-NEXT:    orpd %xmm2, %xmm3
 ; CHECK-NEXT:    movapd %xmm3, %xmm0
+; CHECK-NEXT:    blendvpd %xmm0, %xmm2, %xmm1
+; CHECK-NEXT:    movapd %xmm1, %xmm0
 ; CHECK-NEXT:    retq
   %cmp = fcmp oge double %a, %eps
   %cond = select i1 %cmp, double %c, double %b
diff --git a/llvm/test/CodeGen/X86/fp-strict-scalar-fp16.ll b/llvm/test/CodeGen/X86/fp-strict-scalar-fp16.ll
index e1b677e855094..fbc798d8bbe48 100644
--- a/llvm/test/CodeGen/X86/fp-strict-scalar-fp16.ll
+++ b/llvm/test/CodeGen/X86/fp-strict-scalar-fp16.ll
@@ -46,8 +46,6 @@ define half @fadd_f16(half %a, half %b) nounwind strictfp {
 ; AVX-NEXT:    vxorps %xmm1, %xmm1, %xmm1
 ; AVX-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
 ; AVX-NEXT:    vcvtps2ph $4, %xmm0, %xmm0
-; AVX-NEXT:    vmovd %xmm0, %eax
-; AVX-NEXT:    vpinsrw $0, %eax, %xmm0, %xmm0
 ; AVX-NEXT:    retq
 ;
 ; X86-LABEL: fadd_f16:
@@ -96,8 +94,6 @@ define half @fsub_f16(half %a, half %b) nounwind strictfp {
 ; AVX-NEXT:    vxorps %xmm1, %xmm1, %xmm1
 ; AVX-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
 ; AVX-NEXT:    vcvtps2ph $4, %xmm0, %xmm0
-; AVX-NEXT:    vmovd %xmm0, %eax
-; AVX-NEXT:    vpinsrw $0, %eax, %xmm0, %xmm0
 ; AVX-NEXT:    retq
 ;
 ; X86-LABEL: fsub_f16:
@@ -146,8 +142,6 @@ define half @fmul_f16(half %a, half %b) nounwind strictfp {
 ; AVX-NEXT:    vxorps %xmm1, %xmm1, %xmm1
 ; AVX-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
 ; AVX-NEXT:    vcvtps2ph $4, %xmm0, %xmm0
-; AVX-NEXT:    vmovd %xmm0, %eax
-; AVX-NEXT:    vpinsrw $0, %eax, %xmm0, %xmm0
 ; AVX-NEXT:    retq
 ;
 ; X86-LABEL: fmul_f16:
@@ -196,8 +190,6 @@ define half @fdiv_f16(half %a, half %b) nounwind strictfp {
 ; AVX-NEXT:    vxorps %xmm1, %xmm1, %xmm1
 ; AVX-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
 ; AVX-NEXT:    vcvtps2ph $4, %xmm0, %xmm0
-; AVX-NEXT:    vmovd %xmm0, %eax
-; AVX-NEXT:    vpinsrw $0, %eax, %xmm0, %xmm0
 ; AVX-NEXT:    retq
 ;
 ; X86-LABEL: fdiv_f16:
@@ -479,8 +471,6 @@ define half @fma_f16(half %a, half %b, half %c) nounwind strictfp {
 ; F16C-NEXT:    vxorps %xmm1, %xmm1, %xmm1
 ; F16C-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
 ; F16C-NEXT:    vcvtps2ph $4, %xmm0, %xmm0
-; F16C-NEXT:    vmovd %xmm0, %eax
-; F16C-NEXT:    vpinsrw $0, %eax, %xmm0, %xmm0
 ; F16C-NEXT:    popq %rax
 ; F16C-NEXT:    retq
 ;
@@ -502,8 +492,6 @@ define half @fma_f16(half %a, half %b, half %c) nounwind strictfp {
 ; AVX512-NEXT:    vxorps %xmm0, %xmm0, %xmm0
 ; AVX512-NEXT:    vblendps {{.*#+}} xmm0 = xmm2[0],xmm0[1,2,3]
 ; AVX512-NEXT:    vcvtps2ph $4, %xmm0, %xmm0
-; AVX512-NEXT:    vmovd %xmm0, %eax
-; AVX512-NEXT:    vpinsrw $0, %eax, %xmm0, %xmm0
 ; AVX512-NEXT:    retq
 ;
 ; X86-LABEL: fma_f16:
diff --git a/llvm/test/CodeGen/X86/fp-strict-scalar-fptoint-fp16.ll b/llvm/test/CodeGen/X86/fp-strict-scalar-fptoint-fp16.ll
index bd3cb377bfce6..0498f9b7f9a3d 100644
--- a/llvm/test/CodeGen/X86/fp-strict-scalar-fptoint-fp16.ll
+++ b/llvm/test/CodeGen/X86/fp-strict-scalar-fptoint-fp16.ll
@@ -176,7 +176,7 @@ define i64 @fptosi_f16toi64(half %x) #0 {
 ;
 ; X86-LABEL: fptosi_f16toi64:
 ; X86:       # %bb.0:
-; X86-NEXT:    vmovsh {{[0-9]+}}(%esp), %xmm0
+; X86-NEXT:    vmovsh {{.*#+}} xmm0 = mem[0],zero,zero,zero,zero,zero,zero,zero
 ; X86-NEXT:    vcvttph2qq %xmm0, %xmm0
 ; X86-NEXT:    vmovd %xmm0, %eax
 ; X86-NEXT:    vpextrd $1, %xmm0, %edx
@@ -395,7 +395,7 @@ define i64 @fptoui_f16toi64(half %x) #0 {
 ;
 ; X86-LABEL: fptoui_f16toi64:
 ; X86:       # %bb.0:
-; X86-NEXT:    vmovsh {{[0-9]+}}(%esp), %xmm0
+; X86-NEXT:    vmovsh {{.*#+}} xmm0 = mem[0],zero,zero,zero,zero,zero,zero,zero
 ; X86-NEXT:    vcvttph2uqq %xmm0, %xmm0
 ; X86-NEXT:    vmovd %xmm0, %eax
 ; X86-NEXT:    vpextrd $1, %xmm0, %edx
diff --git a/llvm/test/CodeGen/X86/fp-strict-scalar-inttofp-fp16.ll b/llvm/test/CodeGen/X86/fp-strict-scalar-inttofp-fp16.ll
index 3c99d4daa806f..7c0386f0e784e 100644
--- a/llvm/test/CodeGen/X86/fp-strict-scalar-inttofp-fp16.ll
+++ b/llvm/test/CodeGen/X86/fp-strict-scalar-inttofp-fp16.ll
@@ -37,8 +37,6 @@ define half @sitofp_i1tof16(i1 %x) #0 {
 ; AVX-NEXT:    vxorps %xmm1, %xmm1, %xmm1
 ; AVX-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
 ; AVX-NEXT:    vcvtps2ph $4, %xmm0, %xmm0
-; AVX-NEXT:    vmovd %xmm0, %eax
-; AVX-NEXT:    vpinsrw $0, %eax, %xmm0, %xmm0
 ; AVX-NEXT:    retq
 ;
 ; X86-LABEL: sitofp_i1tof16:
@@ -80,8 +78,6 @@ define half @sitofp_i8tof16(i8 %x) #0 {
 ; AVX-NEXT:    vxorps %xmm1, %xmm1, %xmm1
 ; AVX-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
 ; AVX-NEXT:    vcvtps2ph $4, %xmm0, %xmm0
-; AVX-NEXT:    vmovd %xmm0, %eax
-; AVX-NEXT:    vpinsrw $0, %eax, %xmm0, %xmm0
 ; AVX-NEXT:    retq
 ;
 ; X86-LABEL: sitofp_i8tof16:
@@ -118,8 +114,6 @@ define half @sitofp_i16tof16(i16 %x) #0 {
 ; AVX-NEXT:    vxorps %xmm1, %xmm1, %xmm1
 ; AVX-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
 ; AVX-NEXT:    vcvtps2ph $4, %xmm0, %xmm0
-; AVX-NEXT:    vmovd %xmm0, %eax
-; AVX-NEXT:    vpinsrw $0, %eax, %xmm0, %xmm0
 ; AVX-NEXT:    retq
 ;
 ; X86-LABEL: sitofp_i16tof16:
@@ -154,8 +148,6 @@ define half @sitofp_i32tof16(i32 %x) #0 {
 ; AVX-NEXT:    vxorps %xmm1, %xmm1, %xmm1
 ; AVX-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
 ; AVX-NEXT:    vcvtps2ph $4, %xmm0, %xmm0
-; AVX-NEXT:    vmovd %xmm0, %eax
-; AVX-NEXT:    vpinsrw $0, %eax, %xmm0, %xmm0
 ; AVX-NEXT:    retq
 ;
 ; X86-LABEL: sitofp_i32tof16:
@@ -188,8 +180,6 @@ define half @sitofp_i64tof16(i64 %x) #0 {
 ; AVX-NEXT:    vxorps %xmm1, %xmm1, %xmm1
 ; AVX-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
 ; AVX-NEXT:    vcvtps2ph $4, %xmm0, %xmm0
-; AVX-NEXT:    vmovd %xmm0, %eax
-; AVX-NEXT:    vpinsrw $0, %eax, %xmm0, %xmm0
 ; AVX-NEXT:    retq
 ;
 ; X86-LABEL: sitofp_i64tof16:
@@ -225,8 +215,6 @@ define half @uitofp_i1tof16(i1 %x) #0 {
 ; AVX-NEXT:    vxorps %xmm1, %xmm1, %xmm1
 ; AVX-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
 ; AVX-NEXT:    vcvtps2ph $4, %xmm0, %xmm0
-; AVX-NEXT:    vmovd %xmm0, %eax
-; AVX-NEXT:    vpinsrw $0, %eax, %xmm0, %xmm0
 ; AVX-NEXT:    retq
 ;
 ; X86-LABEL: uitofp_i1tof16:
@@ -265,8 +253,6 @@ define half @uitofp_i8tof16(i8 %x) #0 {
 ; AVX-NEXT:    vxorps %xmm1, %xmm1, %xmm1
 ; AVX-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
 ; AVX-NEXT:    vcvtps2ph $4, %xmm0, %xmm0
-; AVX-NEXT:    vmovd %xmm0, %eax
-; AVX-NEXT:    vpinsrw $0, %eax, %xmm0, %xmm0
 ; AVX-NEXT:    retq
 ;
 ; X86-LABEL: uitofp_i8tof16:
@@ -303,8 +289,6 @@ define half @uitofp_i16tof16(i16 %x) #0 {
 ; AVX-NEXT:    vxorps %xmm1, %xmm1, %xmm1
 ; AVX-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
 ; AVX-NEXT:    vcvtps2ph $4, %xmm0, %xmm0
-; AVX-NEXT:    vmovd %xmm0, %eax
-; AVX-NEXT:    vpinsrw $0, %eax, %xmm0, %xmm0
 ; AVX-NEXT:    retq
 ;
 ; X86-LABEL: uitofp_i16tof16:
@@ -341,8 +325,6 @@ define half @uitofp_i32tof16(i32 %x) #0 {
 ; F16C-NEXT:    vxorps %xmm1, %xmm1, %xmm1
 ; F16C-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
 ; F16C-NEXT:    vcvtps2ph $4, %xmm0, %xmm0
-; F16C-NEXT:    vmovd %xmm0, %eax
-; F16C-NEXT:    vpinsrw $0, %eax, %xmm0, %xmm0
 ; F16C-NEXT:    retq
 ;
 ; AVX512-LABEL: uitofp_i32tof16:
@@ -351,8 +333,6 @@ define half @uitofp_i32tof16(i32 %x) #0 {
 ; AVX512-NEXT:    vxorps %xmm1, %xmm1, %xmm1
 ; AVX512-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
 ; AVX512-NEXT:    vcvtps2ph $4, %xmm0, %xmm0
-; AVX512-NEXT:    vmovd %xmm0, %eax
-; AVX512-NEXT:    vpinsrw $0, %eax, %xmm0, %xmm0
 ; AVX512-NEXT:    retq
 ;
 ; X86-LABEL: uitofp_i32tof16:
@@ -409,8 +389,6 @@ define half @uitofp_i64tof16(i64 %x) #0 {
 ; F16C-NEXT:    vxorps %xmm1, %xmm1, %xmm1
 ; F16C-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
 ; F16C-NEXT:    vcvtps2ph $4, %xmm0, %xmm0
-; F16C-NEXT:    vmovd %xmm0, %eax
-; F16C-NEXT:    vpinsrw $0, %eax, %xmm0, %xmm0
 ; F16C-NEXT:    retq
 ;
 ; AVX512-LABEL: uitofp_i64tof16:
@@ -419,8 +397,6 @@ define half @uitofp_i64tof16(i64 %x) #0 {
 ; AVX512-NEXT:    vxorps %xmm1, %xmm1, %xmm1
 ; AVX512-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
 ; AVX512-NEXT:    vcvtps2ph $4, %xmm0, %xmm0
-; AVX512-NEXT:    vmovd %xmm0, %eax
-; AVX512-NEXT:    vpinsrw $0, %eax, %xmm0, %xmm0
 ; AVX512-NEXT:    retq
 ;
 ; X86-LABEL: uitofp_i64tof16:
diff --git a/llvm/test/CodeGen/X86/fp-strict-scalar-round-fp16.ll b/llvm/test/CodeGen/X86/fp-strict-scalar-round-fp16.ll
index 6fe5dcd292930..1ab97dafb8514 100644
--- a/llvm/test/CodeGen/X86/fp-strict-scalar-round-fp16.ll
+++ b/llvm/test/CodeGen/X86/fp-strict-scalar-round-fp16.ll
@@ -33,8 +33,6 @@ define half @fceil32(half %f) #0 {
 ; AVX-NEXT:    vxorps %xmm1, %xmm1, %xmm1
 ; AVX-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
 ; AVX-NEXT:    vcvtps2ph $4, %xmm0, %xmm0
-; AVX-NEXT:    vmovd %xmm0, %eax
-; AVX-NEXT:    vpinsrw $0, %eax, %xmm0, %xmm0
 ; AVX-NEXT:    retq
 ;
 ; X86-LABEL: fceil32:
@@ -71,8 +69,6 @@ define half @ffloor32(half %f) #0 {
 ; AVX-NEXT:    vxorps %xmm1, %xmm1, %xmm1
 ; AVX-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
 ; AVX-NEXT:    vcvtps2ph $4, %xmm0, %xmm0
-; AVX-NEXT:    vmovd %xmm0, %eax
-; AVX-NEXT:    vpinsrw $0, %eax, %xmm0, %xmm0
 ; AVX-NEXT:    retq
 ;
 ; X86-LABEL: ffloor32:
@@ -109,8 +105,6 @@ define half @ftrunc32(half %f) #0 {
 ; AVX-NEXT:    vxorps %xmm1, %xmm1, %xmm1
 ; AVX-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
 ; AVX-NEXT:    vcvtps2ph $4, %xmm0, %xmm0
-; AVX-NEXT:    vmovd %xmm0, %eax
-; AVX-NEXT:    vpinsrw $0, %eax, %xmm0, %xmm0
 ; AVX-NEXT:    retq
 ;
 ; X86-LABEL: ftrunc32:
@@ -147,8 +141,6 @@ define half @frint32(half %f) #0 {
 ; AVX-NEXT:    vxorps %xmm1, %xmm1, %xmm1
 ; AVX-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
 ; AVX-NEXT:    vcvtps2ph $4, %xmm0, %xmm0
-; AVX-NEXT:    vmovd %xmm0, %eax
-; AVX-NEXT:    vpinsrw $0, %eax, %xmm0, %xmm0
 ; AVX-NEXT:    retq
 ;
 ; X86-LABEL: frint32:
@@ -186,8 +178,6 @@ define half @fnearbyint32(half %f) #0 {
 ; AVX-NEXT:    vxorps %xmm1, %xmm1, %xmm1
 ; AVX-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
 ; AVX-NEXT:    vcvtps2ph $4, %xmm0, %xmm0
-; AVX-NEXT:    vmovd %xmm0, %eax
-; AVX-NEXT:    vpinsrw $0, %eax, %xmm0, %xmm0
 ; AVX-NEXT:    retq
 ;
 ; X86-LABEL: fnearbyint32:
@@ -225,8 +215,6 @@ define half @froundeven16(half %f) #0 {
 ; AVX-NEXT:    vxorps %xmm1, %xmm1, %xmm1
 ; AVX-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
 ; AVX-NEXT:    vcvtps2ph $4, %xmm0, %xmm0
-; AVX-NEXT:    vmovd %xmm0, %eax
-; AVX-NEXT:    vpinsrw $0, %eax, %xmm0, %xmm0
 ; AVX-NEXT:    retq
 ;
 ; X86-LABEL: froundeven16:
@@ -265,8 +253,6 @@ define half @fround16(half %f) #0 {
 ; AVX-NEXT:    vxorps %xmm1, %xmm1, %xmm1
 ; AVX-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
 ; AVX-NEXT:    vcvtps2ph $4, %xmm0, %xmm0
-; AVX-NEXT:    vmovd %xmm0, %eax
-; AVX-NEXT:    vpinsrw $0, %eax, %xmm0, %xmm0
 ; AVX-NEXT:    popq %rax
 ; AVX-NEXT:    retq
 ;
diff --git a/llvm/test/CodeGen/X86/fshl.ll b/llvm/test/CodeGen/X86/fshl.ll
index 065b396e82ec3..e8c8ccfa8d37f 100644
--- a/llvm/test/CodeGen/X86/fshl.ll
+++ b/llvm/test/CodeGen/X86/fshl.ll
@@ -657,6 +657,29 @@ define i64 @combine_fshl_load_i64(ptr %p) nounwind {
   ret i64 %res
 }
 
+define i16 @combine_fshl_load_i16_chain_use(ptr %p, ptr %q, i16 %r) nounwind {
+; X86-LABEL: combine_fshl_load_i16_chain_use:
+; X86:       # %bb.0:
+; X86-NEXT:    movzwl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movzwl 1(%eax), %eax
+; X86-NEXT:    movw %cx, (%edx)
+; X86-NEXT:    retl
+;
+; X64-LABEL: combine_fshl_load_i16_chain_use:
+; X64:       # %bb.0:
+; X64-NEXT:    movzwl 1(%rdi), %eax
+; X64-NEXT:    movw %dx, (%rsi)
+; X64-NEXT:    retq
+  %p1 = getelementptr i16, ptr %p, i32 1
+  %ld0 = load i16, ptr %p
+  %ld1 = load i16, ptr %p1
+  %res = call i16 @llvm.fshl.i16(i16 %ld1, i16 %ld0, i16 8)
+  store i16 %r, ptr %q
+  ret i16 %res
+}
+
 !llvm.module.flags = !{!0}
 !0 = !{i32 1, !"ProfileSummary", !1}
 !1 = !{!2, !3, !4, !5, !6, !7, !8, !9}
diff --git a/llvm/test/CodeGen/X86/gfni-funnel-shifts.ll b/llvm/test/CodeGen/X86/gfni-funnel-shifts.ll
index c071f64dc66cd..0ca3380d188b7 100644
--- a/llvm/test/CodeGen/X86/gfni-funnel-shifts.ll
+++ b/llvm/test/CodeGen/X86/gfni-funnel-shifts.ll
@@ -492,7 +492,7 @@ define <16 x i8> @splatconstant_fshl_v16i8(<16 x i8> %a, <16 x i8> %b) nounwind
 ; GFNIAVX512:       # %bb.0:
 ; GFNIAVX512-NEXT:    vpsllw $3, %xmm0, %xmm2
 ; GFNIAVX512-NEXT:    vpsrlw $5, %xmm1, %xmm0
-; GFNIAVX512-NEXT:    vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to4}, %xmm2, %xmm0
+; GFNIAVX512-NEXT:    vpternlogd {{.*#+}} xmm0 = xmm0 ^ (mem & (xmm0 ^ xmm2))
 ; GFNIAVX512-NEXT:    retq
   %res = call <16 x i8> @llvm.fshl.v16i8(<16 x i8> %a, <16 x i8> %b, <16 x i8> <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>)
   ret <16 x i8> %res
@@ -518,7 +518,7 @@ define <16 x i8> @splatconstant_fshr_v16i8(<16 x i8> %a, <16 x i8> %b) nounwind
 ; GFNIAVX512:       # %bb.0:
 ; GFNIAVX512-NEXT:    vpaddw %xmm0, %xmm0, %xmm2
 ; GFNIAVX512-NEXT:    vpsrlw $7, %xmm1, %xmm0
-; GFNIAVX512-NEXT:    vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to4}, %xmm2, %xmm0
+; GFNIAVX512-NEXT:    vpternlogd {{.*#+}} xmm0 = xmm0 ^ (mem & (xmm0 ^ xmm2))
 ; GFNIAVX512-NEXT:    retq
   %res = call <16 x i8> @llvm.fshr.v16i8(<16 x i8> %a, <16 x i8> %b, <16 x i8> <i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7>)
   ret <16 x i8> %res
@@ -1311,7 +1311,7 @@ define <32 x i8> @splatconstant_fshl_v32i8(<32 x i8> %a, <32 x i8> %b) nounwind
 ; GFNIAVX512:       # %bb.0:
 ; GFNIAVX512-NEXT:    vpsllw $4, %ymm0, %ymm2
 ; GFNIAVX512-NEXT:    vpsrlw $4, %ymm1, %ymm0
-; GFNIAVX512-NEXT:    vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm2, %ymm0
+; GFNIAVX512-NEXT:    vpternlogd {{.*#+}} ymm0 = ymm0 ^ (mem & (ymm0 ^ ymm2))
 ; GFNIAVX512-NEXT:    retq
   %res = call <32 x i8> @llvm.fshl.v32i8(<32 x i8> %a, <32 x i8> %b, <32 x i8> <i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4>)
   ret <32 x i8> %res
@@ -1349,7 +1349,7 @@ define <32 x i8> @splatconstant_fshr_v32i8(<32 x i8> %a, <32 x i8> %b) nounwind
 ; GFNIAVX512:       # %bb.0:
 ; GFNIAVX512-NEXT:    vpsllw $2, %ymm0, %ymm2
 ; GFNIAVX512-NEXT:    vpsrlw $6, %ymm1, %ymm0
-; GFNIAVX512-NEXT:    vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm2, %ymm0
+; GFNIAVX512-NEXT:    vpternlogd {{.*#+}} ymm0 = ymm0 ^ (mem & (ymm0 ^ ymm2))
 ; GFNIAVX512-NEXT:    retq
   %res = call <32 x i8> @llvm.fshr.v32i8(<32 x i8> %a, <32 x i8> %b, <32 x i8> <i8 6, i8 6, i8 6, i8 6, i8 6, i8 6, i8 6, i8 6, i8 6, i8 6, i8 6, i8 6, i8 6, i8 6, i8 6, i8 6, i8 6, i8 6, i8 6, i8 6, i8 6, i8 6, i8 6, i8 6, i8 6, i8 6, i8 6, i8 6, i8 6, i8 6, i8 6, i8 6>)
   ret <32 x i8> %res
@@ -2775,7 +2775,7 @@ define <64 x i8> @splatconstant_fshl_v64i8(<64 x i8> %a, <64 x i8> %b) nounwind
 ; GFNIAVX512BW:       # %bb.0:
 ; GFNIAVX512BW-NEXT:    vpaddw %zmm0, %zmm0, %zmm2
 ; GFNIAVX512BW-NEXT:    vpsrlw $7, %zmm1, %zmm0
-; GFNIAVX512BW-NEXT:    vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm2, %zmm0
+; GFNIAVX512BW-NEXT:    vpternlogd {{.*#+}} zmm0 = zmm0 ^ (mem & (zmm0 ^ zmm2))
 ; GFNIAVX512BW-NEXT:    retq
   %res = call <64 x i8> @llvm.fshl.v64i8(<64 x i8> %a, <64 x i8> %b, <64 x i8> <i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1>)
   ret <64 x i8> %res
@@ -2836,7 +2836,7 @@ define <64 x i8> @splatconstant_fshr_v64i8(<64 x i8> %a, <64 x i8> %b) nounwind
 ; GFNIAVX512BW:       # %bb.0:
 ; GFNIAVX512BW-NEXT:    vpsllw $6, %zmm0, %zmm2
 ; GFNIAVX512BW-NEXT:    vpsrlw $2, %zmm1, %zmm0
-; GFNIAVX512BW-NEXT:    vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm2, %zmm0
+; GFNIAVX512BW-NEXT:    vpternlogd {{.*#+}} zmm0 = zmm0 ^ (mem & (zmm0 ^ zmm2))
 ; GFNIAVX512BW-NEXT:    retq
   %res = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %a, <64 x i8> %b, <64 x i8> <i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2>)
   ret <64 x i8> %res
diff --git a/llvm/test/CodeGen/X86/gfni-shifts.ll b/llvm/test/CodeGen/X86/gfni-shifts.ll
index 6ed524e406826..38f6cabd21fba 100644
--- a/llvm/test/CodeGen/X86/gfni-shifts.ll
+++ b/llvm/test/CodeGen/X86/gfni-shifts.ll
@@ -1194,7 +1194,7 @@ define <32 x i8> @splatvar_ashr_v32i8(<32 x i8> %a, <32 x i8> %b) nounwind {
 ; GFNIAVX512VL-NEXT:    vpsrlw %xmm1, %xmm3, %xmm1
 ; GFNIAVX512VL-NEXT:    vpsrlw $8, %xmm1, %xmm1
 ; GFNIAVX512VL-NEXT:    vpbroadcastb %xmm1, %ymm1
-; GFNIAVX512VL-NEXT:    vpternlogq $108, %ymm0, %ymm2, %ymm1
+; GFNIAVX512VL-NEXT:    vpternlogq {{.*#+}} ymm1 = ymm2 ^ (ymm1 & ymm0)
 ; GFNIAVX512VL-NEXT:    vpsubb %ymm2, %ymm1, %ymm0
 ; GFNIAVX512VL-NEXT:    retq
 ;
@@ -1263,7 +1263,7 @@ define <32 x i8> @constant_shl_v32i8(<32 x i8> %a) nounwind {
 ; GFNIAVX512VL-NEXT:    vpmaddubsw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm1 # [1,0,4,0,16,0,64,0,128,0,32,0,8,0,2,0,1,0,4,0,16,0,64,0,128,0,32,0,8,0,2,0]
 ; GFNIAVX512VL-NEXT:    vpmaddubsw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0 # [0,2,0,8,0,32,0,128,0,64,0,16,0,4,0,1,0,2,0,8,0,32,0,128,0,64,0,16,0,4,0,1]
 ; GFNIAVX512VL-NEXT:    vpsllw $8, %ymm0, %ymm0
-; GFNIAVX512VL-NEXT:    vpternlogd $248, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm1, %ymm0
+; GFNIAVX512VL-NEXT:    vpternlogd {{.*#+}} ymm0 = ymm0 | (ymm1 & mem)
 ; GFNIAVX512VL-NEXT:    retq
 ;
 ; GFNIAVX512BW-LABEL: constant_shl_v32i8:
@@ -2515,10 +2515,10 @@ define <64 x i8> @splatvar_ashr_v64i8(<64 x i8> %a, <64 x i8> %b) nounwind {
 ; GFNIAVX512VL-NEXT:    vpsrlw %xmm1, %xmm4, %xmm4
 ; GFNIAVX512VL-NEXT:    vpsrlw $8, %xmm4, %xmm4
 ; GFNIAVX512VL-NEXT:    vpbroadcastb %xmm4, %ymm4
-; GFNIAVX512VL-NEXT:    vpternlogq $108, %ymm4, %ymm3, %ymm2
+; GFNIAVX512VL-NEXT:    vpternlogq {{.*#+}} ymm2 = ymm3 ^ (ymm2 & ymm4)
 ; GFNIAVX512VL-NEXT:    vpsubb %ymm3, %ymm2, %ymm2
 ; GFNIAVX512VL-NEXT:    vpsrlw %xmm1, %ymm0, %ymm0
-; GFNIAVX512VL-NEXT:    vpternlogq $108, %ymm4, %ymm3, %ymm0
+; GFNIAVX512VL-NEXT:    vpternlogq {{.*#+}} ymm0 = ymm3 ^ (ymm0 & ymm4)
 ; GFNIAVX512VL-NEXT:    vpsubb %ymm3, %ymm0, %ymm0
 ; GFNIAVX512VL-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm0
 ; GFNIAVX512VL-NEXT:    retq
@@ -2533,7 +2533,7 @@ define <64 x i8> @splatvar_ashr_v64i8(<64 x i8> %a, <64 x i8> %b) nounwind {
 ; GFNIAVX512BW-NEXT:    vpsrlw %xmm1, %xmm3, %xmm1
 ; GFNIAVX512BW-NEXT:    vpsrlw $8, %xmm1, %xmm1
 ; GFNIAVX512BW-NEXT:    vpbroadcastb %xmm1, %zmm1
-; GFNIAVX512BW-NEXT:    vpternlogq $108, %zmm0, %zmm2, %zmm1
+; GFNIAVX512BW-NEXT:    vpternlogq {{.*#+}} zmm1 = zmm2 ^ (zmm1 & zmm0)
 ; GFNIAVX512BW-NEXT:    vpsubb %zmm2, %zmm1, %zmm0
 ; GFNIAVX512BW-NEXT:    retq
   %splat = shufflevector <64 x i8> %b, <64 x i8> undef, <64 x i32> zeroinitializer
@@ -2638,7 +2638,7 @@ define <64 x i8> @constant_shl_v64i8(<64 x i8> %a) nounwind {
 ; GFNIAVX512VL-NEXT:    vpmaddubsw %ymm3, %ymm1, %ymm1
 ; GFNIAVX512VL-NEXT:    vpsllw $8, %ymm1, %ymm1
 ; GFNIAVX512VL-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
-; GFNIAVX512VL-NEXT:    vpternlogd $248, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm2, %zmm0
+; GFNIAVX512VL-NEXT:    vpternlogd {{.*#+}} zmm0 = zmm0 | (zmm2 & mem)
 ; GFNIAVX512VL-NEXT:    retq
 ;
 ; GFNIAVX512BW-LABEL: constant_shl_v64i8:
@@ -2646,7 +2646,7 @@ define <64 x i8> @constant_shl_v64i8(<64 x i8> %a) nounwind {
 ; GFNIAVX512BW-NEXT:    vpmaddubsw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm1 # [1,0,4,0,16,0,64,0,128,0,32,0,8,0,2,0,1,0,4,0,16,0,64,0,128,0,32,0,8,0,2,0,1,0,4,0,16,0,64,0,128,0,32,0,8,0,2,0,1,0,4,0,16,0,64,0,128,0,32,0,8,0,2,0]
 ; GFNIAVX512BW-NEXT:    vpmaddubsw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm0 # [0,2,0,8,0,32,0,128,0,64,0,16,0,4,0,1,0,2,0,8,0,32,0,128,0,64,0,16,0,4,0,1,0,2,0,8,0,32,0,128,0,64,0,16,0,4,0,1,0,2,0,8,0,32,0,128,0,64,0,16,0,4,0,1]
 ; GFNIAVX512BW-NEXT:    vpsllw $8, %zmm0, %zmm0
-; GFNIAVX512BW-NEXT:    vpternlogd $248, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm1, %zmm0
+; GFNIAVX512BW-NEXT:    vpternlogd {{.*#+}} zmm0 = zmm0 | (zmm1 & mem)
 ; GFNIAVX512BW-NEXT:    retq
   %shift = shl <64 x i8> %a, <i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 7, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1, i8 0, i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 7, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1, i8 0, i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 7, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1, i8 0, i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 7, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1, i8 0>
   ret <64 x i8> %shift
diff --git a/llvm/test/CodeGen/X86/gpr-to-mask.ll b/llvm/test/CodeGen/X86/gpr-to-mask.ll
index c6e9606a7f7e3..d1513b584887f 100644
--- a/llvm/test/CodeGen/X86/gpr-to-mask.ll
+++ b/llvm/test/CodeGen/X86/gpr-to-mask.ll
@@ -106,7 +106,7 @@ define void @test_load_add(i1 %cond, ptr %fptr, ptr %iptr1, ptr %iptr2, float %f
 ; X86-64-NEXT:    je .LBB2_2
 ; X86-64-NEXT:  # %bb.1: # %if
 ; X86-64-NEXT:    movzbl (%rdx), %eax
-; X86-64-NEXT:    addb (%rcx), %al
+; X86-64-NEXT:    xorb (%rcx), %al
 ; X86-64-NEXT:    jmp .LBB2_3
 ; X86-64-NEXT:  .LBB2_2: # %else
 ; X86-64-NEXT:    movzbl (%rcx), %eax
@@ -127,7 +127,7 @@ define void @test_load_add(i1 %cond, ptr %fptr, ptr %iptr1, ptr %iptr2, float %f
 ; X86-32-NEXT:  # %bb.1: # %if
 ; X86-32-NEXT:    movl {{[0-9]+}}(%esp), %edx
 ; X86-32-NEXT:    movzbl (%edx), %edx
-; X86-32-NEXT:    addb (%ecx), %dl
+; X86-32-NEXT:    xorb (%ecx), %dl
 ; X86-32-NEXT:    jmp .LBB2_3
 ; X86-32-NEXT:  .LBB2_2: # %else
 ; X86-32-NEXT:    movzbl (%ecx), %edx
diff --git a/llvm/test/CodeGen/X86/half.ll b/llvm/test/CodeGen/X86/half.ll
index 6e7f109a5da5c..2472e6e19c862 100644
--- a/llvm/test/CodeGen/X86/half.ll
+++ b/llvm/test/CodeGen/X86/half.ll
@@ -800,8 +800,6 @@ define half @test_f80trunc_nodagcombine() #0 {
 ; BWON-F16C-NEXT:    pushq %rax
 ; BWON-F16C-NEXT:    callq test_floatret@PLT
 ; BWON-F16C-NEXT:    vcvtps2ph $4, %xmm0, %xmm0
-; BWON-F16C-NEXT:    vmovd %xmm0, %eax
-; BWON-F16C-NEXT:    vpinsrw $0, %eax, %xmm0, %xmm0
 ; BWON-F16C-NEXT:    popq %rax
 ; BWON-F16C-NEXT:    retq
 ;
@@ -1015,8 +1013,6 @@ define half @test_sqrt(half %0) #0 {
 ; BWON-F16C-NEXT:    vcvtph2ps %xmm0, %xmm0
 ; BWON-F16C-NEXT:    vsqrtss %xmm0, %xmm0, %xmm0
 ; BWON-F16C-NEXT:    vcvtps2ph $4, %xmm0, %xmm0
-; BWON-F16C-NEXT:    vmovd %xmm0, %eax
-; BWON-F16C-NEXT:    vpinsrw $0, %eax, %xmm0, %xmm0
 ; BWON-F16C-NEXT:    retq
 ;
 ; CHECK-I686-LABEL: test_sqrt:
@@ -1115,36 +1111,36 @@ entry:
 define void @main.45() #0 {
 ; CHECK-LIBCALL-LABEL: main.45:
 ; CHECK-LIBCALL:       # %bb.0: # %entry
-; CHECK-LIBCALL-NEXT:    pushq %rbp
 ; CHECK-LIBCALL-NEXT:    pushq %r15
 ; CHECK-LIBCALL-NEXT:    pushq %r14
 ; CHECK-LIBCALL-NEXT:    pushq %rbx
-; CHECK-LIBCALL-NEXT:    pushq %rax
+; CHECK-LIBCALL-NEXT:    subq $16, %rsp
 ; CHECK-LIBCALL-NEXT:    pinsrw $0, (%rax), %xmm0
 ; CHECK-LIBCALL-NEXT:    pshuflw {{.*#+}} xmm1 = xmm0[0,0,0,0,4,5,6,7]
+; CHECK-LIBCALL-NEXT:    movdqa %xmm1, (%rsp) # 16-byte Spill
 ; CHECK-LIBCALL-NEXT:    movq %xmm1, %rbx
 ; CHECK-LIBCALL-NEXT:    movq %rbx, %r14
 ; CHECK-LIBCALL-NEXT:    shrq $48, %r14
 ; CHECK-LIBCALL-NEXT:    movq %rbx, %r15
 ; CHECK-LIBCALL-NEXT:    shrq $32, %r15
-; CHECK-LIBCALL-NEXT:    movl %ebx, %ebp
-; CHECK-LIBCALL-NEXT:    shrl $16, %ebp
+; CHECK-LIBCALL-NEXT:    shrl $16, %ebx
 ; CHECK-LIBCALL-NEXT:    callq __extendhfsf2@PLT
 ; CHECK-LIBCALL-NEXT:    ucomiss %xmm0, %xmm0
-; CHECK-LIBCALL-NEXT:    movl $32256, %eax # imm = 0x7E00
-; CHECK-LIBCALL-NEXT:    cmovpl %eax, %ebp
-; CHECK-LIBCALL-NEXT:    cmovpl %eax, %r15d
-; CHECK-LIBCALL-NEXT:    cmovpl %eax, %r14d
-; CHECK-LIBCALL-NEXT:    cmovpl %eax, %ebx
-; CHECK-LIBCALL-NEXT:    movw %bx, (%rax)
+; CHECK-LIBCALL-NEXT:    movdqa (%rsp), %xmm0 # 16-byte Reload
+; CHECK-LIBCALL-NEXT:    pextrw $0, %xmm0, %eax
+; CHECK-LIBCALL-NEXT:    movl $32256, %ecx # imm = 0x7E00
+; CHECK-LIBCALL-NEXT:    cmovpl %ecx, %eax
+; CHECK-LIBCALL-NEXT:    cmovpl %ecx, %ebx
+; CHECK-LIBCALL-NEXT:    cmovpl %ecx, %r15d
+; CHECK-LIBCALL-NEXT:    cmovpl %ecx, %r14d
 ; CHECK-LIBCALL-NEXT:    movw %r14w, (%rax)
 ; CHECK-LIBCALL-NEXT:    movw %r15w, (%rax)
-; CHECK-LIBCALL-NEXT:    movw %bp, (%rax)
-; CHECK-LIBCALL-NEXT:    addq $8, %rsp
+; CHECK-LIBCALL-NEXT:    movw %bx, (%rax)
+; CHECK-LIBCALL-NEXT:    movw %ax, (%rax)
+; CHECK-LIBCALL-NEXT:    addq $16, %rsp
 ; CHECK-LIBCALL-NEXT:    popq %rbx
 ; CHECK-LIBCALL-NEXT:    popq %r14
 ; CHECK-LIBCALL-NEXT:    popq %r15
-; CHECK-LIBCALL-NEXT:    popq %rbp
 ; CHECK-LIBCALL-NEXT:    retq
 ;
 ; BWON-F16C-LABEL: main.45:
@@ -1161,28 +1157,20 @@ define void @main.45() #0 {
 ;
 ; CHECK-I686-LABEL: main.45:
 ; CHECK-I686:       # %bb.0: # %entry
-; CHECK-I686-NEXT:    pushl %edi
 ; CHECK-I686-NEXT:    pushl %esi
-; CHECK-I686-NEXT:    subl $20, %esp
+; CHECK-I686-NEXT:    subl $8, %esp
 ; CHECK-I686-NEXT:    pinsrw $0, (%eax), %xmm0
-; CHECK-I686-NEXT:    pshuflw {{.*#+}} xmm1 = xmm0[0,0,0,0,4,5,6,7]
-; CHECK-I686-NEXT:    movd %xmm1, %esi
-; CHECK-I686-NEXT:    movl %esi, %edi
-; CHECK-I686-NEXT:    shrl $16, %edi
-; CHECK-I686-NEXT:    pextrw $0, %xmm0, %eax
-; CHECK-I686-NEXT:    movw %ax, (%esp)
+; CHECK-I686-NEXT:    pextrw $0, %xmm0, %esi
+; CHECK-I686-NEXT:    movw %si, (%esp)
 ; CHECK-I686-NEXT:    calll __extendhfsf2
 ; CHECK-I686-NEXT:    fstps {{[0-9]+}}(%esp)
 ; CHECK-I686-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
 ; CHECK-I686-NEXT:    ucomiss %xmm0, %xmm0
 ; CHECK-I686-NEXT:    movl $32256, %eax # imm = 0x7E00
-; CHECK-I686-NEXT:    cmovpl %eax, %esi
-; CHECK-I686-NEXT:    cmovpl %eax, %edi
-; CHECK-I686-NEXT:    movw %di, (%eax)
-; CHECK-I686-NEXT:    movw %si, (%eax)
-; CHECK-I686-NEXT:    addl $20, %esp
+; CHECK-I686-NEXT:    cmovnpl %esi, %eax
+; CHECK-I686-NEXT:    movw %ax, (%eax)
+; CHECK-I686-NEXT:    addl $8, %esp
 ; CHECK-I686-NEXT:    popl %esi
-; CHECK-I686-NEXT:    popl %edi
 ; CHECK-I686-NEXT:    retl
 entry:
   %0 = load half, ptr undef, align 8
@@ -1319,8 +1307,6 @@ define half @pr61271(half %0, half %1) #0 {
 ; BWON-F16C-NEXT:    vcvtph2ps %xmm0, %xmm0
 ; BWON-F16C-NEXT:    vminss %xmm1, %xmm0, %xmm0
 ; BWON-F16C-NEXT:    vcvtps2ph $4, %xmm0, %xmm0
-; BWON-F16C-NEXT:    vmovd %xmm0, %eax
-; BWON-F16C-NEXT:    vpinsrw $0, %eax, %xmm0, %xmm0
 ; BWON-F16C-NEXT:    retq
 ;
 ; CHECK-I686-LABEL: pr61271:
@@ -1581,79 +1567,67 @@ define <8 x half> @maxnum_v8f16(<8 x half> %0, <8 x half> %1) #0 {
 ; BWON-F16C-LABEL: maxnum_v8f16:
 ; BWON-F16C:       # %bb.0:
 ; BWON-F16C-NEXT:    vpsrldq {{.*#+}} xmm2 = xmm1[14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; BWON-F16C-NEXT:    vcvtph2ps %xmm2, %xmm3
+; BWON-F16C-NEXT:    vpsrldq {{.*#+}} xmm2 = xmm0[14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
 ; BWON-F16C-NEXT:    vcvtph2ps %xmm2, %xmm2
-; BWON-F16C-NEXT:    vpsrldq {{.*#+}} xmm3 = xmm0[14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; BWON-F16C-NEXT:    vcvtph2ps %xmm3, %xmm3
-; BWON-F16C-NEXT:    vucomiss %xmm2, %xmm3
+; BWON-F16C-NEXT:    vucomiss %xmm3, %xmm2
 ; BWON-F16C-NEXT:    ja .LBB26_2
 ; BWON-F16C-NEXT:  # %bb.1:
-; BWON-F16C-NEXT:    vmovaps %xmm2, %xmm3
+; BWON-F16C-NEXT:    vmovaps %xmm3, %xmm2
 ; BWON-F16C-NEXT:  .LBB26_2:
-; BWON-F16C-NEXT:    vcvtps2ph $4, %xmm3, %xmm2
 ; BWON-F16C-NEXT:    vpshufd {{.*#+}} xmm3 = xmm1[3,3,3,3]
+; BWON-F16C-NEXT:    vcvtph2ps %xmm3, %xmm4
+; BWON-F16C-NEXT:    vpshufd {{.*#+}} xmm3 = xmm0[3,3,3,3]
 ; BWON-F16C-NEXT:    vcvtph2ps %xmm3, %xmm3
-; BWON-F16C-NEXT:    vpshufd {{.*#+}} xmm4 = xmm0[3,3,3,3]
-; BWON-F16C-NEXT:    vcvtph2ps %xmm4, %xmm4
-; BWON-F16C-NEXT:    vucomiss %xmm3, %xmm4
+; BWON-F16C-NEXT:    vucomiss %xmm4, %xmm3
 ; BWON-F16C-NEXT:    ja .LBB26_4
 ; BWON-F16C-NEXT:  # %bb.3:
-; BWON-F16C-NEXT:    vmovaps %xmm3, %xmm4
+; BWON-F16C-NEXT:    vmovaps %xmm4, %xmm3
 ; BWON-F16C-NEXT:  .LBB26_4:
-; BWON-F16C-NEXT:    vmovd %xmm2, %eax
-; BWON-F16C-NEXT:    vcvtps2ph $4, %xmm4, %xmm2
-; BWON-F16C-NEXT:    vmovd %xmm2, %ecx
-; BWON-F16C-NEXT:    vpsrldq {{.*#+}} xmm2 = xmm1[10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; BWON-F16C-NEXT:    vcvtph2ps %xmm2, %xmm3
-; BWON-F16C-NEXT:    vpsrldq {{.*#+}} xmm2 = xmm0[10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; BWON-F16C-NEXT:    vcvtph2ps %xmm2, %xmm2
-; BWON-F16C-NEXT:    vucomiss %xmm3, %xmm2
+; BWON-F16C-NEXT:    vpsrldq {{.*#+}} xmm4 = xmm1[10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; BWON-F16C-NEXT:    vcvtph2ps %xmm4, %xmm5
+; BWON-F16C-NEXT:    vpsrldq {{.*#+}} xmm4 = xmm0[10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; BWON-F16C-NEXT:    vcvtph2ps %xmm4, %xmm4
+; BWON-F16C-NEXT:    vucomiss %xmm5, %xmm4
 ; BWON-F16C-NEXT:    ja .LBB26_6
 ; BWON-F16C-NEXT:  # %bb.5:
-; BWON-F16C-NEXT:    vmovaps %xmm3, %xmm2
+; BWON-F16C-NEXT:    vmovaps %xmm5, %xmm4
 ; BWON-F16C-NEXT:  .LBB26_6:
-; BWON-F16C-NEXT:    vcvtps2ph $4, %xmm2, %xmm2
-; BWON-F16C-NEXT:    vmovd %xmm2, %edx
-; BWON-F16C-NEXT:    vshufpd {{.*#+}} xmm2 = xmm1[1,0]
-; BWON-F16C-NEXT:    vcvtph2ps %xmm2, %xmm3
-; BWON-F16C-NEXT:    vshufpd {{.*#+}} xmm2 = xmm0[1,0]
-; BWON-F16C-NEXT:    vcvtph2ps %xmm2, %xmm2
-; BWON-F16C-NEXT:    vucomiss %xmm3, %xmm2
+; BWON-F16C-NEXT:    vshufpd {{.*#+}} xmm5 = xmm1[1,0]
+; BWON-F16C-NEXT:    vcvtph2ps %xmm5, %xmm5
+; BWON-F16C-NEXT:    vshufpd {{.*#+}} xmm6 = xmm0[1,0]
+; BWON-F16C-NEXT:    vcvtph2ps %xmm6, %xmm6
+; BWON-F16C-NEXT:    vucomiss %xmm5, %xmm6
 ; BWON-F16C-NEXT:    ja .LBB26_8
 ; BWON-F16C-NEXT:  # %bb.7:
-; BWON-F16C-NEXT:    vmovaps %xmm3, %xmm2
+; BWON-F16C-NEXT:    vmovaps %xmm5, %xmm6
 ; BWON-F16C-NEXT:  .LBB26_8:
-; BWON-F16C-NEXT:    vcvtps2ph $4, %xmm2, %xmm2
-; BWON-F16C-NEXT:    vmovd %xmm2, %esi
-; BWON-F16C-NEXT:    vpshuflw {{.*#+}} xmm2 = xmm1[3,3,3,3,4,5,6,7]
-; BWON-F16C-NEXT:    vcvtph2ps %xmm2, %xmm2
-; BWON-F16C-NEXT:    vpshuflw {{.*#+}} xmm3 = xmm0[3,3,3,3,4,5,6,7]
-; BWON-F16C-NEXT:    vcvtph2ps %xmm3, %xmm6
-; BWON-F16C-NEXT:    vucomiss %xmm2, %xmm6
+; BWON-F16C-NEXT:    vpshuflw {{.*#+}} xmm5 = xmm1[3,3,3,3,4,5,6,7]
+; BWON-F16C-NEXT:    vcvtph2ps %xmm5, %xmm7
+; BWON-F16C-NEXT:    vpshuflw {{.*#+}} xmm5 = xmm0[3,3,3,3,4,5,6,7]
+; BWON-F16C-NEXT:    vcvtph2ps %xmm5, %xmm5
+; BWON-F16C-NEXT:    vucomiss %xmm7, %xmm5
 ; BWON-F16C-NEXT:    ja .LBB26_10
 ; BWON-F16C-NEXT:  # %bb.9:
-; BWON-F16C-NEXT:    vmovaps %xmm2, %xmm6
+; BWON-F16C-NEXT:    vmovaps %xmm7, %xmm5
 ; BWON-F16C-NEXT:  .LBB26_10:
-; BWON-F16C-NEXT:    vpinsrw $0, %eax, %xmm0, %xmm2
-; BWON-F16C-NEXT:    vpinsrw $0, %ecx, %xmm0, %xmm3
-; BWON-F16C-NEXT:    vpinsrw $0, %edx, %xmm0, %xmm4
-; BWON-F16C-NEXT:    vpinsrw $0, %esi, %xmm0, %xmm5
+; BWON-F16C-NEXT:    vcvtps2ph $4, %xmm2, %xmm2
+; BWON-F16C-NEXT:    vcvtps2ph $4, %xmm3, %xmm3
+; BWON-F16C-NEXT:    vcvtps2ph $4, %xmm4, %xmm4
 ; BWON-F16C-NEXT:    vcvtps2ph $4, %xmm6, %xmm6
-; BWON-F16C-NEXT:    vmovd %xmm6, %eax
-; BWON-F16C-NEXT:    vmovshdup {{.*#+}} xmm6 = xmm1[1,1,3,3]
-; BWON-F16C-NEXT:    vcvtph2ps %xmm6, %xmm7
-; BWON-F16C-NEXT:    vmovshdup {{.*#+}} xmm6 = xmm0[1,1,3,3]
-; BWON-F16C-NEXT:    vcvtph2ps %xmm6, %xmm6
-; BWON-F16C-NEXT:    vucomiss %xmm7, %xmm6
+; BWON-F16C-NEXT:    vmovshdup {{.*#+}} xmm7 = xmm1[1,1,3,3]
+; BWON-F16C-NEXT:    vcvtph2ps %xmm7, %xmm8
+; BWON-F16C-NEXT:    vmovshdup {{.*#+}} xmm7 = xmm0[1,1,3,3]
+; BWON-F16C-NEXT:    vcvtph2ps %xmm7, %xmm7
+; BWON-F16C-NEXT:    vucomiss %xmm8, %xmm7
 ; BWON-F16C-NEXT:    ja .LBB26_12
 ; BWON-F16C-NEXT:  # %bb.11:
-; BWON-F16C-NEXT:    vmovaps %xmm7, %xmm6
+; BWON-F16C-NEXT:    vmovaps %xmm8, %xmm7
 ; BWON-F16C-NEXT:  .LBB26_12:
 ; BWON-F16C-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]
-; BWON-F16C-NEXT:    vpunpcklwd {{.*#+}} xmm3 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3]
-; BWON-F16C-NEXT:    vpinsrw $0, %eax, %xmm0, %xmm4
-; BWON-F16C-NEXT:    vcvtps2ph $4, %xmm6, %xmm5
-; BWON-F16C-NEXT:    vmovd %xmm5, %eax
-; BWON-F16C-NEXT:    vpinsrw $0, %eax, %xmm0, %xmm5
+; BWON-F16C-NEXT:    vpunpcklwd {{.*#+}} xmm3 = xmm6[0],xmm4[0],xmm6[1],xmm4[1],xmm6[2],xmm4[2],xmm6[3],xmm4[3]
+; BWON-F16C-NEXT:    vcvtps2ph $4, %xmm5, %xmm4
+; BWON-F16C-NEXT:    vcvtps2ph $4, %xmm7, %xmm5
 ; BWON-F16C-NEXT:    vcvtph2ps %xmm1, %xmm7
 ; BWON-F16C-NEXT:    vcvtph2ps %xmm0, %xmm6
 ; BWON-F16C-NEXT:    vucomiss %xmm7, %xmm6
@@ -1664,8 +1638,6 @@ define <8 x half> @maxnum_v8f16(<8 x half> %0, <8 x half> %1) #0 {
 ; BWON-F16C-NEXT:    vpunpckldq {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]
 ; BWON-F16C-NEXT:    vpunpcklwd {{.*#+}} xmm3 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3]
 ; BWON-F16C-NEXT:    vcvtps2ph $4, %xmm6, %xmm4
-; BWON-F16C-NEXT:    vmovd %xmm4, %eax
-; BWON-F16C-NEXT:    vpinsrw $0, %eax, %xmm0, %xmm4
 ; BWON-F16C-NEXT:    vpshuflw {{.*#+}} xmm1 = xmm1[1,1,1,1,4,5,6,7]
 ; BWON-F16C-NEXT:    vcvtph2ps %xmm1, %xmm1
 ; BWON-F16C-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[1,1,1,1,4,5,6,7]
@@ -1676,8 +1648,6 @@ define <8 x half> @maxnum_v8f16(<8 x half> %0, <8 x half> %1) #0 {
 ; BWON-F16C-NEXT:    vmovaps %xmm1, %xmm0
 ; BWON-F16C-NEXT:  .LBB26_16:
 ; BWON-F16C-NEXT:    vcvtps2ph $4, %xmm0, %xmm0
-; BWON-F16C-NEXT:    vmovd %xmm0, %eax
-; BWON-F16C-NEXT:    vpinsrw $0, %eax, %xmm0, %xmm0
 ; BWON-F16C-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm4[0],xmm0[0],xmm4[1],xmm0[1],xmm4[2],xmm0[2],xmm4[3],xmm0[3]
 ; BWON-F16C-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1]
 ; BWON-F16C-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0]
diff --git a/llvm/test/CodeGen/X86/horizontal-sum.ll b/llvm/test/CodeGen/X86/horizontal-sum.ll
index 5fe1e2996ee9b..e2cc3ae0dca0a 100644
--- a/llvm/test/CodeGen/X86/horizontal-sum.ll
+++ b/llvm/test/CodeGen/X86/horizontal-sum.ll
@@ -256,11 +256,11 @@ define <8 x float> @pair_sum_v8f32_v4f32(<4 x float> %0, <4 x float> %1, <4 x fl
 ; AVX2-SLOW-NEXT:    vaddps %xmm2, %xmm1, %xmm1
 ; AVX2-SLOW-NEXT:    vmovlhps {{.*#+}} xmm0 = xmm0[0],xmm1[0]
 ; AVX2-SLOW-NEXT:    vshufpd {{.*#+}} xmm1 = xmm1[1,0]
-; AVX2-SLOW-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm1
-; AVX2-SLOW-NEXT:    vhaddps %xmm7, %xmm6, %xmm2
-; AVX2-SLOW-NEXT:    vhaddps %xmm2, %xmm2, %xmm2
-; AVX2-SLOW-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
-; AVX2-SLOW-NEXT:    vshufpd {{.*#+}} ymm0 = ymm1[0],ymm0[1],ymm1[2],ymm0[2]
+; AVX2-SLOW-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; AVX2-SLOW-NEXT:    vhaddps %xmm7, %xmm6, %xmm1
+; AVX2-SLOW-NEXT:    vhaddps %xmm1, %xmm1, %xmm1
+; AVX2-SLOW-NEXT:    vbroadcastsd %xmm1, %ymm1
+; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm1[6,7]
 ; AVX2-SLOW-NEXT:    retq
 ;
 ; AVX2-FAST-LABEL: pair_sum_v8f32_v4f32:
@@ -277,11 +277,11 @@ define <8 x float> @pair_sum_v8f32_v4f32(<4 x float> %0, <4 x float> %1, <4 x fl
 ; AVX2-FAST-NEXT:    vaddps %xmm2, %xmm1, %xmm1
 ; AVX2-FAST-NEXT:    vmovlhps {{.*#+}} xmm0 = xmm0[0],xmm1[0]
 ; AVX2-FAST-NEXT:    vshufpd {{.*#+}} xmm1 = xmm1[1,0]
-; AVX2-FAST-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm1
-; AVX2-FAST-NEXT:    vhaddps %xmm7, %xmm6, %xmm2
-; AVX2-FAST-NEXT:    vhaddps %xmm0, %xmm2, %xmm2
-; AVX2-FAST-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
-; AVX2-FAST-NEXT:    vshufpd {{.*#+}} ymm0 = ymm1[0],ymm0[1],ymm1[2],ymm0[2]
+; AVX2-FAST-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; AVX2-FAST-NEXT:    vhaddps %xmm7, %xmm6, %xmm1
+; AVX2-FAST-NEXT:    vhaddps %xmm0, %xmm1, %xmm1
+; AVX2-FAST-NEXT:    vbroadcastsd %xmm1, %ymm1
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm1[6,7]
 ; AVX2-FAST-NEXT:    retq
   %9 = shufflevector <4 x float> %0, <4 x float> poison, <2 x i32> <i32 0, i32 2>
   %10 = shufflevector <4 x float> %0, <4 x float> poison, <2 x i32> <i32 1, i32 3>
diff --git a/llvm/test/CodeGen/X86/huge-stack-offset.ll b/llvm/test/CodeGen/X86/huge-stack-offset.ll
index e825328ccd89a..d6080cfd3f753 100644
--- a/llvm/test/CodeGen/X86/huge-stack-offset.ll
+++ b/llvm/test/CodeGen/X86/huge-stack-offset.ll
@@ -1,5 +1,6 @@
 ; RUN: llc < %s -mtriple=x86_64-linux-unknown -verify-machineinstrs | FileCheck %s --check-prefix=CHECK-64
 ; RUN: llc < %s -mtriple=i386-linux-unknown -verify-machineinstrs | FileCheck %s --check-prefix=CHECK-32
+; RUN: llc < %s -mtriple=x86_64-linux-gnux32 -verify-machineinstrs | FileCheck %s -check-prefix=CHECK-32
 
 ; Test that a large stack offset uses a single add/sub instruction to
 ; adjust the stack pointer.
@@ -11,13 +12,11 @@ define void @foo() nounwind {
 ; CHECK-64-NOT:  subq    $2147483647, %rsp
 ; CHECK-64:      movabsq $50000000{{..}}, [[RAX:%r..]]
 ; CHECK-64-NEXT: addq    [[RAX]], %rsp
-
+;
 ; CHECK-32-LABEL: foo:
-; CHECK-32:      movl    $50000000{{..}}, %eax
-; CHECK-32-NEXT: subl    %eax, %esp
+; CHECK-32:      ud2
 ; CHECK-32-NOT:  subl    $2147483647, %esp
-; CHECK-32:      movl    $50000000{{..}}, [[EAX:%e..]]
-; CHECK-32-NEXT: addl    [[EAX]], %esp
+; CHECK-32:      ud2
   %1 = alloca [5000000000 x i8], align 16
   call void @bar(ptr %1)
   ret void
@@ -29,7 +28,7 @@ define i32 @foo2() nounwind {
 ; CHECK-64-LABEL: foo2:
 ; CHECK-64:     movl    $10, %eax
 ; CHECK-64-NOT: movabsq ${{.*}}, %rax
-
+;
 ; CHECK-32-LABEL: foo2:
 ; CHECK-32:     movl    $10, %eax
 ; CHECK-32-NOT: movl    ${{.*}}, %eax
@@ -44,9 +43,9 @@ define i32 @foo3(i32 inreg %x) nounwind {
 ; CHECK-64-LABEL: foo3:
 ; CHECK-64:      movabsq $50000000{{..}}, %rax
 ; CHECK-64-NEXT: subq    %rax, %rsp
-
+;
 ; CHECK-32-LABEL: foo3:
-; CHECK-32:      subl $2147483647, %esp
+; CHECK-32:      ud2
 ; CHECK-32-NOT:  movl ${{.*}}, %eax
   %1 = alloca [5000000000 x i8], align 16
   call void @bar(ptr %1)
diff --git a/llvm/test/CodeGen/X86/isel-fcmp-x87.ll b/llvm/test/CodeGen/X86/isel-fcmp-x87.ll
new file mode 100644
index 0000000000000..8c2a53082649a
--- /dev/null
+++ b/llvm/test/CodeGen/X86/isel-fcmp-x87.ll
@@ -0,0 +1,1496 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s                               -mtriple=x86_64-apple-darwin10 -mattr=+x87,-sse,-sse2 -verify-machineinstrs | FileCheck %s --check-prefixes=X64,SDAG-X64
+; RUN: llc < %s -fast-isel -fast-isel-abort=1 -mtriple=x86_64-apple-darwin10 -mattr=+x87,-sse,-sse2 -verify-machineinstrs | FileCheck %s --check-prefixes=X64,FAST-X64
+; RUN: llc < %s -global-isel -global-isel-abort=1 -mtriple=x86_64-apple-darwin10 -mattr=+x87,-sse,-sse2 -verify-machineinstrs | FileCheck %s --check-prefixes=GISEL-X64
+; RUN: llc < %s                               -mtriple=i686-apple-darwin10 -mattr=+x87,-sse,-sse2 -verify-machineinstrs | FileCheck %s --check-prefixes=X86,SDAG-X86
+; Allow fast-isel to fallback to selection dag on x86
+; RUN: llc < %s -fast-isel -mtriple=i686-apple-darwin10 -mattr=+x87,-sse,-sse2 -verify-machineinstrs | FileCheck %s --check-prefixes=X86,FAST-X86
+; RUN: llc < %s -global-isel -global-isel-abort=1 -mtriple=i686-apple-darwin10 -mattr=+x87,-sse,-sse2 -verify-machineinstrs | FileCheck %s --check-prefixes=GISEL-X86
+
+  define i1 @fcmp_x86_fp80_oeq(x86_fp80 %x, x86_fp80 %y) nounwind {
+; X64-LABEL: fcmp_x86_fp80_oeq:
+; X64:       ## %bb.0:
+; X64-NEXT:    fldt {{[0-9]+}}(%rsp)
+; X64-NEXT:    fldt {{[0-9]+}}(%rsp)
+; X64-NEXT:    fucompi %st(1), %st
+; X64-NEXT:    fstp %st(0)
+; X64-NEXT:    setnp %cl
+; X64-NEXT:    sete %al
+; X64-NEXT:    andb %cl, %al
+; X64-NEXT:    retq
+;
+; GISEL-X64-LABEL: fcmp_x86_fp80_oeq:
+; GISEL-X64:       ## %bb.0:
+; GISEL-X64-NEXT:    fldt {{[0-9]+}}(%rsp)
+; GISEL-X64-NEXT:    fldt {{[0-9]+}}(%rsp)
+; GISEL-X64-NEXT:    fxch %st(1)
+; GISEL-X64-NEXT:    fucompi %st(1), %st
+; GISEL-X64-NEXT:    fstp %st(0)
+; GISEL-X64-NEXT:    sete %cl
+; GISEL-X64-NEXT:    setnp %al
+; GISEL-X64-NEXT:    andb %cl, %al
+; GISEL-X64-NEXT:    retq
+;
+; X86-LABEL: fcmp_x86_fp80_oeq:
+; X86:       ## %bb.0:
+; X86-NEXT:    subl $12, %esp
+; X86-NEXT:    fldt {{[0-9]+}}(%esp)
+; X86-NEXT:    fldt {{[0-9]+}}(%esp)
+; X86-NEXT:    fucompp
+; X86-NEXT:    fnstsw %ax
+; X86-NEXT:    ## kill: def $ah killed $ah killed $ax
+; X86-NEXT:    sahf
+; X86-NEXT:    setnp %cl
+; X86-NEXT:    sete %al
+; X86-NEXT:    andb %cl, %al
+; X86-NEXT:    addl $12, %esp
+; X86-NEXT:    retl
+;
+; GISEL-X86-LABEL: fcmp_x86_fp80_oeq:
+; GISEL-X86:       ## %bb.0:
+; GISEL-X86-NEXT:    subl $12, %esp
+; GISEL-X86-NEXT:    fldt {{[0-9]+}}(%esp)
+; GISEL-X86-NEXT:    fldt {{[0-9]+}}(%esp)
+; GISEL-X86-NEXT:    fxch %st(1)
+; GISEL-X86-NEXT:    fucompi %st(1), %st
+; GISEL-X86-NEXT:    fstp %st(0)
+; GISEL-X86-NEXT:    sete %cl
+; GISEL-X86-NEXT:    setnp %al
+; GISEL-X86-NEXT:    andb %cl, %al
+; GISEL-X86-NEXT:    addl $12, %esp
+; GISEL-X86-NEXT:    retl
+    %1 = fcmp oeq x86_fp80 %x, %y
+    ret i1 %1
+  }
+
+  define i1 @fcmp_x86_fp80_ogt(x86_fp80 %x, x86_fp80 %y) nounwind {
+; X64-LABEL: fcmp_x86_fp80_ogt:
+; X64:       ## %bb.0:
+; X64-NEXT:    fldt {{[0-9]+}}(%rsp)
+; X64-NEXT:    fldt {{[0-9]+}}(%rsp)
+; X64-NEXT:    fucompi %st(1), %st
+; X64-NEXT:    fstp %st(0)
+; X64-NEXT:    seta %al
+; X64-NEXT:    retq
+;
+; GISEL-X64-LABEL: fcmp_x86_fp80_ogt:
+; GISEL-X64:       ## %bb.0:
+; GISEL-X64-NEXT:    fldt {{[0-9]+}}(%rsp)
+; GISEL-X64-NEXT:    fldt {{[0-9]+}}(%rsp)
+; GISEL-X64-NEXT:    fxch %st(1)
+; GISEL-X64-NEXT:    fucompi %st(1), %st
+; GISEL-X64-NEXT:    fstp %st(0)
+; GISEL-X64-NEXT:    seta %al
+; GISEL-X64-NEXT:    retq
+;
+; X86-LABEL: fcmp_x86_fp80_ogt:
+; X86:       ## %bb.0:
+; X86-NEXT:    subl $12, %esp
+; X86-NEXT:    fldt {{[0-9]+}}(%esp)
+; X86-NEXT:    fldt {{[0-9]+}}(%esp)
+; X86-NEXT:    fucompp
+; X86-NEXT:    fnstsw %ax
+; X86-NEXT:    ## kill: def $ah killed $ah killed $ax
+; X86-NEXT:    sahf
+; X86-NEXT:    seta %al
+; X86-NEXT:    addl $12, %esp
+; X86-NEXT:    retl
+;
+; GISEL-X86-LABEL: fcmp_x86_fp80_ogt:
+; GISEL-X86:       ## %bb.0:
+; GISEL-X86-NEXT:    subl $12, %esp
+; GISEL-X86-NEXT:    fldt {{[0-9]+}}(%esp)
+; GISEL-X86-NEXT:    fldt {{[0-9]+}}(%esp)
+; GISEL-X86-NEXT:    fxch %st(1)
+; GISEL-X86-NEXT:    fucompi %st(1), %st
+; GISEL-X86-NEXT:    fstp %st(0)
+; GISEL-X86-NEXT:    seta %al
+; GISEL-X86-NEXT:    addl $12, %esp
+; GISEL-X86-NEXT:    retl
+    %1 = fcmp ogt x86_fp80 %x, %y
+    ret i1 %1
+  }
+
+  define i1 @fcmp_x86_fp80_oge(x86_fp80 %x, x86_fp80 %y) nounwind {
+; X64-LABEL: fcmp_x86_fp80_oge:
+; X64:       ## %bb.0:
+; X64-NEXT:    fldt {{[0-9]+}}(%rsp)
+; X64-NEXT:    fldt {{[0-9]+}}(%rsp)
+; X64-NEXT:    fucompi %st(1), %st
+; X64-NEXT:    fstp %st(0)
+; X64-NEXT:    setae %al
+; X64-NEXT:    retq
+;
+; GISEL-X64-LABEL: fcmp_x86_fp80_oge:
+; GISEL-X64:       ## %bb.0:
+; GISEL-X64-NEXT:    fldt {{[0-9]+}}(%rsp)
+; GISEL-X64-NEXT:    fldt {{[0-9]+}}(%rsp)
+; GISEL-X64-NEXT:    fxch %st(1)
+; GISEL-X64-NEXT:    fucompi %st(1), %st
+; GISEL-X64-NEXT:    fstp %st(0)
+; GISEL-X64-NEXT:    setae %al
+; GISEL-X64-NEXT:    retq
+;
+; X86-LABEL: fcmp_x86_fp80_oge:
+; X86:       ## %bb.0:
+; X86-NEXT:    subl $12, %esp
+; X86-NEXT:    fldt {{[0-9]+}}(%esp)
+; X86-NEXT:    fldt {{[0-9]+}}(%esp)
+; X86-NEXT:    fucompp
+; X86-NEXT:    fnstsw %ax
+; X86-NEXT:    ## kill: def $ah killed $ah killed $ax
+; X86-NEXT:    sahf
+; X86-NEXT:    setae %al
+; X86-NEXT:    addl $12, %esp
+; X86-NEXT:    retl
+;
+; GISEL-X86-LABEL: fcmp_x86_fp80_oge:
+; GISEL-X86:       ## %bb.0:
+; GISEL-X86-NEXT:    subl $12, %esp
+; GISEL-X86-NEXT:    fldt {{[0-9]+}}(%esp)
+; GISEL-X86-NEXT:    fldt {{[0-9]+}}(%esp)
+; GISEL-X86-NEXT:    fxch %st(1)
+; GISEL-X86-NEXT:    fucompi %st(1), %st
+; GISEL-X86-NEXT:    fstp %st(0)
+; GISEL-X86-NEXT:    setae %al
+; GISEL-X86-NEXT:    addl $12, %esp
+; GISEL-X86-NEXT:    retl
+    %1 = fcmp oge x86_fp80 %x, %y
+    ret i1 %1
+  }
+
+  define i1 @fcmp_x86_fp80_olt(x86_fp80 %x, x86_fp80 %y) nounwind {
+; SDAG-X64-LABEL: fcmp_x86_fp80_olt:
+; SDAG-X64:       ## %bb.0:
+; SDAG-X64-NEXT:    fldt {{[0-9]+}}(%rsp)
+; SDAG-X64-NEXT:    fldt {{[0-9]+}}(%rsp)
+; SDAG-X64-NEXT:    fucompi %st(1), %st
+; SDAG-X64-NEXT:    fstp %st(0)
+; SDAG-X64-NEXT:    seta %al
+; SDAG-X64-NEXT:    retq
+;
+; FAST-X64-LABEL: fcmp_x86_fp80_olt:
+; FAST-X64:       ## %bb.0:
+; FAST-X64-NEXT:    fldt {{[0-9]+}}(%rsp)
+; FAST-X64-NEXT:    fldt {{[0-9]+}}(%rsp)
+; FAST-X64-NEXT:    fxch %st(1)
+; FAST-X64-NEXT:    fucompi %st(1), %st
+; FAST-X64-NEXT:    fstp %st(0)
+; FAST-X64-NEXT:    seta %al
+; FAST-X64-NEXT:    retq
+;
+; GISEL-X64-LABEL: fcmp_x86_fp80_olt:
+; GISEL-X64:       ## %bb.0:
+; GISEL-X64-NEXT:    fldt {{[0-9]+}}(%rsp)
+; GISEL-X64-NEXT:    fldt {{[0-9]+}}(%rsp)
+; GISEL-X64-NEXT:    fucompi %st(1), %st
+; GISEL-X64-NEXT:    fstp %st(0)
+; GISEL-X64-NEXT:    seta %al
+; GISEL-X64-NEXT:    retq
+;
+; SDAG-X86-LABEL: fcmp_x86_fp80_olt:
+; SDAG-X86:       ## %bb.0:
+; SDAG-X86-NEXT:    subl $12, %esp
+; SDAG-X86-NEXT:    fldt {{[0-9]+}}(%esp)
+; SDAG-X86-NEXT:    fldt {{[0-9]+}}(%esp)
+; SDAG-X86-NEXT:    fucompp
+; SDAG-X86-NEXT:    fnstsw %ax
+; SDAG-X86-NEXT:    ## kill: def $ah killed $ah killed $ax
+; SDAG-X86-NEXT:    sahf
+; SDAG-X86-NEXT:    seta %al
+; SDAG-X86-NEXT:    addl $12, %esp
+; SDAG-X86-NEXT:    retl
+;
+; FAST-X86-LABEL: fcmp_x86_fp80_olt:
+; FAST-X86:       ## %bb.0:
+; FAST-X86-NEXT:    subl $12, %esp
+; FAST-X86-NEXT:    fldt {{[0-9]+}}(%esp)
+; FAST-X86-NEXT:    fldt {{[0-9]+}}(%esp)
+; FAST-X86-NEXT:    fxch %st(1)
+; FAST-X86-NEXT:    fucompp
+; FAST-X86-NEXT:    fnstsw %ax
+; FAST-X86-NEXT:    ## kill: def $ah killed $ah killed $ax
+; FAST-X86-NEXT:    sahf
+; FAST-X86-NEXT:    seta %al
+; FAST-X86-NEXT:    addl $12, %esp
+; FAST-X86-NEXT:    retl
+;
+; GISEL-X86-LABEL: fcmp_x86_fp80_olt:
+; GISEL-X86:       ## %bb.0:
+; GISEL-X86-NEXT:    subl $12, %esp
+; GISEL-X86-NEXT:    fldt {{[0-9]+}}(%esp)
+; GISEL-X86-NEXT:    fldt {{[0-9]+}}(%esp)
+; GISEL-X86-NEXT:    fucompi %st(1), %st
+; GISEL-X86-NEXT:    fstp %st(0)
+; GISEL-X86-NEXT:    seta %al
+; GISEL-X86-NEXT:    addl $12, %esp
+; GISEL-X86-NEXT:    retl
+    %1 = fcmp olt x86_fp80 %x, %y
+    ret i1 %1
+  }
+
+  define i1 @fcmp_x86_fp80_ole(x86_fp80 %x, x86_fp80 %y) nounwind {
+; SDAG-X64-LABEL: fcmp_x86_fp80_ole:
+; SDAG-X64:       ## %bb.0:
+; SDAG-X64-NEXT:    fldt {{[0-9]+}}(%rsp)
+; SDAG-X64-NEXT:    fldt {{[0-9]+}}(%rsp)
+; SDAG-X64-NEXT:    fucompi %st(1), %st
+; SDAG-X64-NEXT:    fstp %st(0)
+; SDAG-X64-NEXT:    setae %al
+; SDAG-X64-NEXT:    retq
+;
+; FAST-X64-LABEL: fcmp_x86_fp80_ole:
+; FAST-X64:       ## %bb.0:
+; FAST-X64-NEXT:    fldt {{[0-9]+}}(%rsp)
+; FAST-X64-NEXT:    fldt {{[0-9]+}}(%rsp)
+; FAST-X64-NEXT:    fxch %st(1)
+; FAST-X64-NEXT:    fucompi %st(1), %st
+; FAST-X64-NEXT:    fstp %st(0)
+; FAST-X64-NEXT:    setae %al
+; FAST-X64-NEXT:    retq
+;
+; GISEL-X64-LABEL: fcmp_x86_fp80_ole:
+; GISEL-X64:       ## %bb.0:
+; GISEL-X64-NEXT:    fldt {{[0-9]+}}(%rsp)
+; GISEL-X64-NEXT:    fldt {{[0-9]+}}(%rsp)
+; GISEL-X64-NEXT:    fucompi %st(1), %st
+; GISEL-X64-NEXT:    fstp %st(0)
+; GISEL-X64-NEXT:    setae %al
+; GISEL-X64-NEXT:    retq
+;
+; SDAG-X86-LABEL: fcmp_x86_fp80_ole:
+; SDAG-X86:       ## %bb.0:
+; SDAG-X86-NEXT:    subl $12, %esp
+; SDAG-X86-NEXT:    fldt {{[0-9]+}}(%esp)
+; SDAG-X86-NEXT:    fldt {{[0-9]+}}(%esp)
+; SDAG-X86-NEXT:    fucompp
+; SDAG-X86-NEXT:    fnstsw %ax
+; SDAG-X86-NEXT:    ## kill: def $ah killed $ah killed $ax
+; SDAG-X86-NEXT:    sahf
+; SDAG-X86-NEXT:    setae %al
+; SDAG-X86-NEXT:    addl $12, %esp
+; SDAG-X86-NEXT:    retl
+;
+; FAST-X86-LABEL: fcmp_x86_fp80_ole:
+; FAST-X86:       ## %bb.0:
+; FAST-X86-NEXT:    subl $12, %esp
+; FAST-X86-NEXT:    fldt {{[0-9]+}}(%esp)
+; FAST-X86-NEXT:    fldt {{[0-9]+}}(%esp)
+; FAST-X86-NEXT:    fxch %st(1)
+; FAST-X86-NEXT:    fucompp
+; FAST-X86-NEXT:    fnstsw %ax
+; FAST-X86-NEXT:    ## kill: def $ah killed $ah killed $ax
+; FAST-X86-NEXT:    sahf
+; FAST-X86-NEXT:    setae %al
+; FAST-X86-NEXT:    addl $12, %esp
+; FAST-X86-NEXT:    retl
+;
+; GISEL-X86-LABEL: fcmp_x86_fp80_ole:
+; GISEL-X86:       ## %bb.0:
+; GISEL-X86-NEXT:    subl $12, %esp
+; GISEL-X86-NEXT:    fldt {{[0-9]+}}(%esp)
+; GISEL-X86-NEXT:    fldt {{[0-9]+}}(%esp)
+; GISEL-X86-NEXT:    fucompi %st(1), %st
+; GISEL-X86-NEXT:    fstp %st(0)
+; GISEL-X86-NEXT:    setae %al
+; GISEL-X86-NEXT:    addl $12, %esp
+; GISEL-X86-NEXT:    retl
+    %1 = fcmp ole x86_fp80 %x, %y
+    ret i1 %1
+  }
+
+  define i1 @fcmp_x86_fp80_one(x86_fp80 %x, x86_fp80 %y) nounwind {
+; X64-LABEL: fcmp_x86_fp80_one:
+; X64:       ## %bb.0:
+; X64-NEXT:    fldt {{[0-9]+}}(%rsp)
+; X64-NEXT:    fldt {{[0-9]+}}(%rsp)
+; X64-NEXT:    fucompi %st(1), %st
+; X64-NEXT:    fstp %st(0)
+; X64-NEXT:    setne %al
+; X64-NEXT:    retq
+;
+; GISEL-X64-LABEL: fcmp_x86_fp80_one:
+; GISEL-X64:       ## %bb.0:
+; GISEL-X64-NEXT:    fldt {{[0-9]+}}(%rsp)
+; GISEL-X64-NEXT:    fldt {{[0-9]+}}(%rsp)
+; GISEL-X64-NEXT:    fxch %st(1)
+; GISEL-X64-NEXT:    fucompi %st(1), %st
+; GISEL-X64-NEXT:    fstp %st(0)
+; GISEL-X64-NEXT:    setne %al
+; GISEL-X64-NEXT:    retq
+;
+; X86-LABEL: fcmp_x86_fp80_one:
+; X86:       ## %bb.0:
+; X86-NEXT:    subl $12, %esp
+; X86-NEXT:    fldt {{[0-9]+}}(%esp)
+; X86-NEXT:    fldt {{[0-9]+}}(%esp)
+; X86-NEXT:    fucompp
+; X86-NEXT:    fnstsw %ax
+; X86-NEXT:    ## kill: def $ah killed $ah killed $ax
+; X86-NEXT:    sahf
+; X86-NEXT:    setne %al
+; X86-NEXT:    addl $12, %esp
+; X86-NEXT:    retl
+;
+; GISEL-X86-LABEL: fcmp_x86_fp80_one:
+; GISEL-X86:       ## %bb.0:
+; GISEL-X86-NEXT:    subl $12, %esp
+; GISEL-X86-NEXT:    fldt {{[0-9]+}}(%esp)
+; GISEL-X86-NEXT:    fldt {{[0-9]+}}(%esp)
+; GISEL-X86-NEXT:    fxch %st(1)
+; GISEL-X86-NEXT:    fucompi %st(1), %st
+; GISEL-X86-NEXT:    fstp %st(0)
+; GISEL-X86-NEXT:    setne %al
+; GISEL-X86-NEXT:    addl $12, %esp
+; GISEL-X86-NEXT:    retl
+    %1 = fcmp one x86_fp80 %x, %y
+    ret i1 %1
+  }
+
+  define i1 @fcmp_x86_fp80_ord(x86_fp80 %x, x86_fp80 %y) nounwind {
+; X64-LABEL: fcmp_x86_fp80_ord:
+; X64:       ## %bb.0:
+; X64-NEXT:    fldt {{[0-9]+}}(%rsp)
+; X64-NEXT:    fldt {{[0-9]+}}(%rsp)
+; X64-NEXT:    fucompi %st(1), %st
+; X64-NEXT:    fstp %st(0)
+; X64-NEXT:    setnp %al
+; X64-NEXT:    retq
+;
+; GISEL-X64-LABEL: fcmp_x86_fp80_ord:
+; GISEL-X64:       ## %bb.0:
+; GISEL-X64-NEXT:    fldt {{[0-9]+}}(%rsp)
+; GISEL-X64-NEXT:    fldt {{[0-9]+}}(%rsp)
+; GISEL-X64-NEXT:    fxch %st(1)
+; GISEL-X64-NEXT:    fucompi %st(1), %st
+; GISEL-X64-NEXT:    fstp %st(0)
+; GISEL-X64-NEXT:    setnp %al
+; GISEL-X64-NEXT:    retq
+;
+; X86-LABEL: fcmp_x86_fp80_ord:
+; X86:       ## %bb.0:
+; X86-NEXT:    subl $12, %esp
+; X86-NEXT:    fldt {{[0-9]+}}(%esp)
+; X86-NEXT:    fldt {{[0-9]+}}(%esp)
+; X86-NEXT:    fucompp
+; X86-NEXT:    fnstsw %ax
+; X86-NEXT:    ## kill: def $ah killed $ah killed $ax
+; X86-NEXT:    sahf
+; X86-NEXT:    setnp %al
+; X86-NEXT:    addl $12, %esp
+; X86-NEXT:    retl
+;
+; GISEL-X86-LABEL: fcmp_x86_fp80_ord:
+; GISEL-X86:       ## %bb.0:
+; GISEL-X86-NEXT:    subl $12, %esp
+; GISEL-X86-NEXT:    fldt {{[0-9]+}}(%esp)
+; GISEL-X86-NEXT:    fldt {{[0-9]+}}(%esp)
+; GISEL-X86-NEXT:    fxch %st(1)
+; GISEL-X86-NEXT:    fucompi %st(1), %st
+; GISEL-X86-NEXT:    fstp %st(0)
+; GISEL-X86-NEXT:    setnp %al
+; GISEL-X86-NEXT:    addl $12, %esp
+; GISEL-X86-NEXT:    retl
+    %1 = fcmp ord x86_fp80 %x, %y
+    ret i1 %1
+  }
+
+  define i1 @fcmp_x86_fp80_uno(x86_fp80 %x, x86_fp80 %y) nounwind {
+; X64-LABEL: fcmp_x86_fp80_uno:
+; X64:       ## %bb.0:
+; X64-NEXT:    fldt {{[0-9]+}}(%rsp)
+; X64-NEXT:    fldt {{[0-9]+}}(%rsp)
+; X64-NEXT:    fucompi %st(1), %st
+; X64-NEXT:    fstp %st(0)
+; X64-NEXT:    setp %al
+; X64-NEXT:    retq
+;
+; GISEL-X64-LABEL: fcmp_x86_fp80_uno:
+; GISEL-X64:       ## %bb.0:
+; GISEL-X64-NEXT:    fldt {{[0-9]+}}(%rsp)
+; GISEL-X64-NEXT:    fldt {{[0-9]+}}(%rsp)
+; GISEL-X64-NEXT:    fxch %st(1)
+; GISEL-X64-NEXT:    fucompi %st(1), %st
+; GISEL-X64-NEXT:    fstp %st(0)
+; GISEL-X64-NEXT:    setp %al
+; GISEL-X64-NEXT:    retq
+;
+; X86-LABEL: fcmp_x86_fp80_uno:
+; X86:       ## %bb.0:
+; X86-NEXT:    subl $12, %esp
+; X86-NEXT:    fldt {{[0-9]+}}(%esp)
+; X86-NEXT:    fldt {{[0-9]+}}(%esp)
+; X86-NEXT:    fucompp
+; X86-NEXT:    fnstsw %ax
+; X86-NEXT:    ## kill: def $ah killed $ah killed $ax
+; X86-NEXT:    sahf
+; X86-NEXT:    setp %al
+; X86-NEXT:    addl $12, %esp
+; X86-NEXT:    retl
+;
+; GISEL-X86-LABEL: fcmp_x86_fp80_uno:
+; GISEL-X86:       ## %bb.0:
+; GISEL-X86-NEXT:    subl $12, %esp
+; GISEL-X86-NEXT:    fldt {{[0-9]+}}(%esp)
+; GISEL-X86-NEXT:    fldt {{[0-9]+}}(%esp)
+; GISEL-X86-NEXT:    fxch %st(1)
+; GISEL-X86-NEXT:    fucompi %st(1), %st
+; GISEL-X86-NEXT:    fstp %st(0)
+; GISEL-X86-NEXT:    setp %al
+; GISEL-X86-NEXT:    addl $12, %esp
+; GISEL-X86-NEXT:    retl
+    %1 = fcmp uno x86_fp80 %x, %y
+    ret i1 %1
+  }
+
+  define i1 @fcmp_x86_fp80_ueq(x86_fp80 %x, x86_fp80 %y) nounwind {
+; X64-LABEL: fcmp_x86_fp80_ueq:
+; X64:       ## %bb.0:
+; X64-NEXT:    fldt {{[0-9]+}}(%rsp)
+; X64-NEXT:    fldt {{[0-9]+}}(%rsp)
+; X64-NEXT:    fucompi %st(1), %st
+; X64-NEXT:    fstp %st(0)
+; X64-NEXT:    sete %al
+; X64-NEXT:    retq
+;
+; GISEL-X64-LABEL: fcmp_x86_fp80_ueq:
+; GISEL-X64:       ## %bb.0:
+; GISEL-X64-NEXT:    fldt {{[0-9]+}}(%rsp)
+; GISEL-X64-NEXT:    fldt {{[0-9]+}}(%rsp)
+; GISEL-X64-NEXT:    fxch %st(1)
+; GISEL-X64-NEXT:    fucompi %st(1), %st
+; GISEL-X64-NEXT:    fstp %st(0)
+; GISEL-X64-NEXT:    sete %al
+; GISEL-X64-NEXT:    retq
+;
+; X86-LABEL: fcmp_x86_fp80_ueq:
+; X86:       ## %bb.0:
+; X86-NEXT:    subl $12, %esp
+; X86-NEXT:    fldt {{[0-9]+}}(%esp)
+; X86-NEXT:    fldt {{[0-9]+}}(%esp)
+; X86-NEXT:    fucompp
+; X86-NEXT:    fnstsw %ax
+; X86-NEXT:    ## kill: def $ah killed $ah killed $ax
+; X86-NEXT:    sahf
+; X86-NEXT:    sete %al
+; X86-NEXT:    addl $12, %esp
+; X86-NEXT:    retl
+;
+; GISEL-X86-LABEL: fcmp_x86_fp80_ueq:
+; GISEL-X86:       ## %bb.0:
+; GISEL-X86-NEXT:    subl $12, %esp
+; GISEL-X86-NEXT:    fldt {{[0-9]+}}(%esp)
+; GISEL-X86-NEXT:    fldt {{[0-9]+}}(%esp)
+; GISEL-X86-NEXT:    fxch %st(1)
+; GISEL-X86-NEXT:    fucompi %st(1), %st
+; GISEL-X86-NEXT:    fstp %st(0)
+; GISEL-X86-NEXT:    sete %al
+; GISEL-X86-NEXT:    addl $12, %esp
+; GISEL-X86-NEXT:    retl
+    %1 = fcmp ueq x86_fp80 %x, %y
+    ret i1 %1
+  }
+
+  define i1 @fcmp_x86_fp80_ugt(x86_fp80 %x, x86_fp80 %y) nounwind {
+; SDAG-X64-LABEL: fcmp_x86_fp80_ugt:
+; SDAG-X64:       ## %bb.0:
+; SDAG-X64-NEXT:    fldt {{[0-9]+}}(%rsp)
+; SDAG-X64-NEXT:    fldt {{[0-9]+}}(%rsp)
+; SDAG-X64-NEXT:    fucompi %st(1), %st
+; SDAG-X64-NEXT:    fstp %st(0)
+; SDAG-X64-NEXT:    setb %al
+; SDAG-X64-NEXT:    retq
+;
+; FAST-X64-LABEL: fcmp_x86_fp80_ugt:
+; FAST-X64:       ## %bb.0:
+; FAST-X64-NEXT:    fldt {{[0-9]+}}(%rsp)
+; FAST-X64-NEXT:    fldt {{[0-9]+}}(%rsp)
+; FAST-X64-NEXT:    fxch %st(1)
+; FAST-X64-NEXT:    fucompi %st(1), %st
+; FAST-X64-NEXT:    fstp %st(0)
+; FAST-X64-NEXT:    setb %al
+; FAST-X64-NEXT:    retq
+;
+; GISEL-X64-LABEL: fcmp_x86_fp80_ugt:
+; GISEL-X64:       ## %bb.0:
+; GISEL-X64-NEXT:    fldt {{[0-9]+}}(%rsp)
+; GISEL-X64-NEXT:    fldt {{[0-9]+}}(%rsp)
+; GISEL-X64-NEXT:    fucompi %st(1), %st
+; GISEL-X64-NEXT:    fstp %st(0)
+; GISEL-X64-NEXT:    setb %al
+; GISEL-X64-NEXT:    retq
+;
+; SDAG-X86-LABEL: fcmp_x86_fp80_ugt:
+; SDAG-X86:       ## %bb.0:
+; SDAG-X86-NEXT:    subl $12, %esp
+; SDAG-X86-NEXT:    fldt {{[0-9]+}}(%esp)
+; SDAG-X86-NEXT:    fldt {{[0-9]+}}(%esp)
+; SDAG-X86-NEXT:    fucompp
+; SDAG-X86-NEXT:    fnstsw %ax
+; SDAG-X86-NEXT:    ## kill: def $ah killed $ah killed $ax
+; SDAG-X86-NEXT:    sahf
+; SDAG-X86-NEXT:    setb %al
+; SDAG-X86-NEXT:    addl $12, %esp
+; SDAG-X86-NEXT:    retl
+;
+; FAST-X86-LABEL: fcmp_x86_fp80_ugt:
+; FAST-X86:       ## %bb.0:
+; FAST-X86-NEXT:    subl $12, %esp
+; FAST-X86-NEXT:    fldt {{[0-9]+}}(%esp)
+; FAST-X86-NEXT:    fldt {{[0-9]+}}(%esp)
+; FAST-X86-NEXT:    fxch %st(1)
+; FAST-X86-NEXT:    fucompp
+; FAST-X86-NEXT:    fnstsw %ax
+; FAST-X86-NEXT:    ## kill: def $ah killed $ah killed $ax
+; FAST-X86-NEXT:    sahf
+; FAST-X86-NEXT:    setb %al
+; FAST-X86-NEXT:    addl $12, %esp
+; FAST-X86-NEXT:    retl
+;
+; GISEL-X86-LABEL: fcmp_x86_fp80_ugt:
+; GISEL-X86:       ## %bb.0:
+; GISEL-X86-NEXT:    subl $12, %esp
+; GISEL-X86-NEXT:    fldt {{[0-9]+}}(%esp)
+; GISEL-X86-NEXT:    fldt {{[0-9]+}}(%esp)
+; GISEL-X86-NEXT:    fucompi %st(1), %st
+; GISEL-X86-NEXT:    fstp %st(0)
+; GISEL-X86-NEXT:    setb %al
+; GISEL-X86-NEXT:    addl $12, %esp
+; GISEL-X86-NEXT:    retl
+    %1 = fcmp ugt x86_fp80 %x, %y
+    ret i1 %1
+  }
+
+  define i1 @fcmp_x86_fp80_uge(x86_fp80 %x, x86_fp80 %y) nounwind {
+; SDAG-X64-LABEL: fcmp_x86_fp80_uge:
+; SDAG-X64:       ## %bb.0:
+; SDAG-X64-NEXT:    fldt {{[0-9]+}}(%rsp)
+; SDAG-X64-NEXT:    fldt {{[0-9]+}}(%rsp)
+; SDAG-X64-NEXT:    fucompi %st(1), %st
+; SDAG-X64-NEXT:    fstp %st(0)
+; SDAG-X64-NEXT:    setbe %al
+; SDAG-X64-NEXT:    retq
+;
+; FAST-X64-LABEL: fcmp_x86_fp80_uge:
+; FAST-X64:       ## %bb.0:
+; FAST-X64-NEXT:    fldt {{[0-9]+}}(%rsp)
+; FAST-X64-NEXT:    fldt {{[0-9]+}}(%rsp)
+; FAST-X64-NEXT:    fxch %st(1)
+; FAST-X64-NEXT:    fucompi %st(1), %st
+; FAST-X64-NEXT:    fstp %st(0)
+; FAST-X64-NEXT:    setbe %al
+; FAST-X64-NEXT:    retq
+;
+; GISEL-X64-LABEL: fcmp_x86_fp80_uge:
+; GISEL-X64:       ## %bb.0:
+; GISEL-X64-NEXT:    fldt {{[0-9]+}}(%rsp)
+; GISEL-X64-NEXT:    fldt {{[0-9]+}}(%rsp)
+; GISEL-X64-NEXT:    fucompi %st(1), %st
+; GISEL-X64-NEXT:    fstp %st(0)
+; GISEL-X64-NEXT:    setbe %al
+; GISEL-X64-NEXT:    retq
+;
+; SDAG-X86-LABEL: fcmp_x86_fp80_uge:
+; SDAG-X86:       ## %bb.0:
+; SDAG-X86-NEXT:    subl $12, %esp
+; SDAG-X86-NEXT:    fldt {{[0-9]+}}(%esp)
+; SDAG-X86-NEXT:    fldt {{[0-9]+}}(%esp)
+; SDAG-X86-NEXT:    fucompp
+; SDAG-X86-NEXT:    fnstsw %ax
+; SDAG-X86-NEXT:    ## kill: def $ah killed $ah killed $ax
+; SDAG-X86-NEXT:    sahf
+; SDAG-X86-NEXT:    setbe %al
+; SDAG-X86-NEXT:    addl $12, %esp
+; SDAG-X86-NEXT:    retl
+;
+; FAST-X86-LABEL: fcmp_x86_fp80_uge:
+; FAST-X86:       ## %bb.0:
+; FAST-X86-NEXT:    subl $12, %esp
+; FAST-X86-NEXT:    fldt {{[0-9]+}}(%esp)
+; FAST-X86-NEXT:    fldt {{[0-9]+}}(%esp)
+; FAST-X86-NEXT:    fxch %st(1)
+; FAST-X86-NEXT:    fucompp
+; FAST-X86-NEXT:    fnstsw %ax
+; FAST-X86-NEXT:    ## kill: def $ah killed $ah killed $ax
+; FAST-X86-NEXT:    sahf
+; FAST-X86-NEXT:    setbe %al
+; FAST-X86-NEXT:    addl $12, %esp
+; FAST-X86-NEXT:    retl
+;
+; GISEL-X86-LABEL: fcmp_x86_fp80_uge:
+; GISEL-X86:       ## %bb.0:
+; GISEL-X86-NEXT:    subl $12, %esp
+; GISEL-X86-NEXT:    fldt {{[0-9]+}}(%esp)
+; GISEL-X86-NEXT:    fldt {{[0-9]+}}(%esp)
+; GISEL-X86-NEXT:    fucompi %st(1), %st
+; GISEL-X86-NEXT:    fstp %st(0)
+; GISEL-X86-NEXT:    setbe %al
+; GISEL-X86-NEXT:    addl $12, %esp
+; GISEL-X86-NEXT:    retl
+    %1 = fcmp uge x86_fp80 %x, %y
+    ret i1 %1
+  }
+
+  define i1 @fcmp_x86_fp80_ult(x86_fp80 %x, x86_fp80 %y) nounwind {
+; X64-LABEL: fcmp_x86_fp80_ult:
+; X64:       ## %bb.0:
+; X64-NEXT:    fldt {{[0-9]+}}(%rsp)
+; X64-NEXT:    fldt {{[0-9]+}}(%rsp)
+; X64-NEXT:    fucompi %st(1), %st
+; X64-NEXT:    fstp %st(0)
+; X64-NEXT:    setb %al
+; X64-NEXT:    retq
+;
+; GISEL-X64-LABEL: fcmp_x86_fp80_ult:
+; GISEL-X64:       ## %bb.0:
+; GISEL-X64-NEXT:    fldt {{[0-9]+}}(%rsp)
+; GISEL-X64-NEXT:    fldt {{[0-9]+}}(%rsp)
+; GISEL-X64-NEXT:    fxch %st(1)
+; GISEL-X64-NEXT:    fucompi %st(1), %st
+; GISEL-X64-NEXT:    fstp %st(0)
+; GISEL-X64-NEXT:    setb %al
+; GISEL-X64-NEXT:    retq
+;
+; X86-LABEL: fcmp_x86_fp80_ult:
+; X86:       ## %bb.0:
+; X86-NEXT:    subl $12, %esp
+; X86-NEXT:    fldt {{[0-9]+}}(%esp)
+; X86-NEXT:    fldt {{[0-9]+}}(%esp)
+; X86-NEXT:    fucompp
+; X86-NEXT:    fnstsw %ax
+; X86-NEXT:    ## kill: def $ah killed $ah killed $ax
+; X86-NEXT:    sahf
+; X86-NEXT:    setb %al
+; X86-NEXT:    addl $12, %esp
+; X86-NEXT:    retl
+;
+; GISEL-X86-LABEL: fcmp_x86_fp80_ult:
+; GISEL-X86:       ## %bb.0:
+; GISEL-X86-NEXT:    subl $12, %esp
+; GISEL-X86-NEXT:    fldt {{[0-9]+}}(%esp)
+; GISEL-X86-NEXT:    fldt {{[0-9]+}}(%esp)
+; GISEL-X86-NEXT:    fxch %st(1)
+; GISEL-X86-NEXT:    fucompi %st(1), %st
+; GISEL-X86-NEXT:    fstp %st(0)
+; GISEL-X86-NEXT:    setb %al
+; GISEL-X86-NEXT:    addl $12, %esp
+; GISEL-X86-NEXT:    retl
+    %1 = fcmp ult x86_fp80 %x, %y
+    ret i1 %1
+  }
+
+  define i1 @fcmp_x86_fp80_ule(x86_fp80 %x, x86_fp80 %y) nounwind {
+; X64-LABEL: fcmp_x86_fp80_ule:
+; X64:       ## %bb.0:
+; X64-NEXT:    fldt {{[0-9]+}}(%rsp)
+; X64-NEXT:    fldt {{[0-9]+}}(%rsp)
+; X64-NEXT:    fucompi %st(1), %st
+; X64-NEXT:    fstp %st(0)
+; X64-NEXT:    setbe %al
+; X64-NEXT:    retq
+;
+; GISEL-X64-LABEL: fcmp_x86_fp80_ule:
+; GISEL-X64:       ## %bb.0:
+; GISEL-X64-NEXT:    fldt {{[0-9]+}}(%rsp)
+; GISEL-X64-NEXT:    fldt {{[0-9]+}}(%rsp)
+; GISEL-X64-NEXT:    fxch %st(1)
+; GISEL-X64-NEXT:    fucompi %st(1), %st
+; GISEL-X64-NEXT:    fstp %st(0)
+; GISEL-X64-NEXT:    setbe %al
+; GISEL-X64-NEXT:    retq
+;
+; X86-LABEL: fcmp_x86_fp80_ule:
+; X86:       ## %bb.0:
+; X86-NEXT:    subl $12, %esp
+; X86-NEXT:    fldt {{[0-9]+}}(%esp)
+; X86-NEXT:    fldt {{[0-9]+}}(%esp)
+; X86-NEXT:    fucompp
+; X86-NEXT:    fnstsw %ax
+; X86-NEXT:    ## kill: def $ah killed $ah killed $ax
+; X86-NEXT:    sahf
+; X86-NEXT:    setbe %al
+; X86-NEXT:    addl $12, %esp
+; X86-NEXT:    retl
+;
+; GISEL-X86-LABEL: fcmp_x86_fp80_ule:
+; GISEL-X86:       ## %bb.0:
+; GISEL-X86-NEXT:    subl $12, %esp
+; GISEL-X86-NEXT:    fldt {{[0-9]+}}(%esp)
+; GISEL-X86-NEXT:    fldt {{[0-9]+}}(%esp)
+; GISEL-X86-NEXT:    fxch %st(1)
+; GISEL-X86-NEXT:    fucompi %st(1), %st
+; GISEL-X86-NEXT:    fstp %st(0)
+; GISEL-X86-NEXT:    setbe %al
+; GISEL-X86-NEXT:    addl $12, %esp
+; GISEL-X86-NEXT:    retl
+    %1 = fcmp ule x86_fp80 %x, %y
+    ret i1 %1
+  }
+
+  define i1 @fcmp_x86_fp80_une(x86_fp80 %x, x86_fp80 %y) nounwind {
+; X64-LABEL: fcmp_x86_fp80_une:
+; X64:       ## %bb.0:
+; X64-NEXT:    fldt {{[0-9]+}}(%rsp)
+; X64-NEXT:    fldt {{[0-9]+}}(%rsp)
+; X64-NEXT:    fucompi %st(1), %st
+; X64-NEXT:    fstp %st(0)
+; X64-NEXT:    setp %cl
+; X64-NEXT:    setne %al
+; X64-NEXT:    orb %cl, %al
+; X64-NEXT:    retq
+;
+; GISEL-X64-LABEL: fcmp_x86_fp80_une:
+; GISEL-X64:       ## %bb.0:
+; GISEL-X64-NEXT:    fldt {{[0-9]+}}(%rsp)
+; GISEL-X64-NEXT:    fldt {{[0-9]+}}(%rsp)
+; GISEL-X64-NEXT:    fxch %st(1)
+; GISEL-X64-NEXT:    fucompi %st(1), %st
+; GISEL-X64-NEXT:    fstp %st(0)
+; GISEL-X64-NEXT:    setne %cl
+; GISEL-X64-NEXT:    setp %al
+; GISEL-X64-NEXT:    orb %cl, %al
+; GISEL-X64-NEXT:    retq
+;
+; X86-LABEL: fcmp_x86_fp80_une:
+; X86:       ## %bb.0:
+; X86-NEXT:    subl $12, %esp
+; X86-NEXT:    fldt {{[0-9]+}}(%esp)
+; X86-NEXT:    fldt {{[0-9]+}}(%esp)
+; X86-NEXT:    fucompp
+; X86-NEXT:    fnstsw %ax
+; X86-NEXT:    ## kill: def $ah killed $ah killed $ax
+; X86-NEXT:    sahf
+; X86-NEXT:    setp %cl
+; X86-NEXT:    setne %al
+; X86-NEXT:    orb %cl, %al
+; X86-NEXT:    addl $12, %esp
+; X86-NEXT:    retl
+;
+; GISEL-X86-LABEL: fcmp_x86_fp80_une:
+; GISEL-X86:       ## %bb.0:
+; GISEL-X86-NEXT:    subl $12, %esp
+; GISEL-X86-NEXT:    fldt {{[0-9]+}}(%esp)
+; GISEL-X86-NEXT:    fldt {{[0-9]+}}(%esp)
+; GISEL-X86-NEXT:    fxch %st(1)
+; GISEL-X86-NEXT:    fucompi %st(1), %st
+; GISEL-X86-NEXT:    fstp %st(0)
+; GISEL-X86-NEXT:    setne %cl
+; GISEL-X86-NEXT:    setp %al
+; GISEL-X86-NEXT:    orb %cl, %al
+; GISEL-X86-NEXT:    addl $12, %esp
+; GISEL-X86-NEXT:    retl
+    %1 = fcmp une x86_fp80 %x, %y
+    ret i1 %1
+  }
+
+  define i1 @fcmp_float_oeq(float %x, float %y) {
+; X64-LABEL: fcmp_float_oeq:
+; X64:       ## %bb.0:
+; X64-NEXT:    flds {{[0-9]+}}(%rsp)
+; X64-NEXT:    flds {{[0-9]+}}(%rsp)
+; X64-NEXT:    fucompi %st(1), %st
+; X64-NEXT:    fstp %st(0)
+; X64-NEXT:    setnp %cl
+; X64-NEXT:    sete %al
+; X64-NEXT:    andb %cl, %al
+; X64-NEXT:    retq
+;
+; GISEL-X64-LABEL: fcmp_float_oeq:
+; GISEL-X64:       ## %bb.0:
+; GISEL-X64-NEXT:    flds {{[0-9]+}}(%rsp)
+; GISEL-X64-NEXT:    flds {{[0-9]+}}(%rsp)
+; GISEL-X64-NEXT:    fxch %st(1)
+; GISEL-X64-NEXT:    fucompi %st(1), %st
+; GISEL-X64-NEXT:    fstp %st(0)
+; GISEL-X64-NEXT:    sete %cl
+; GISEL-X64-NEXT:    setnp %al
+; GISEL-X64-NEXT:    andb %cl, %al
+; GISEL-X64-NEXT:    retq
+;
+; X86-LABEL: fcmp_float_oeq:
+; X86:       ## %bb.0:
+; X86-NEXT:    flds {{[0-9]+}}(%esp)
+; X86-NEXT:    flds {{[0-9]+}}(%esp)
+; X86-NEXT:    fucompp
+; X86-NEXT:    fnstsw %ax
+; X86-NEXT:    ## kill: def $ah killed $ah killed $ax
+; X86-NEXT:    sahf
+; X86-NEXT:    setnp %cl
+; X86-NEXT:    sete %al
+; X86-NEXT:    andb %cl, %al
+; X86-NEXT:    retl
+;
+; GISEL-X86-LABEL: fcmp_float_oeq:
+; GISEL-X86:       ## %bb.0:
+; GISEL-X86-NEXT:    flds {{[0-9]+}}(%esp)
+; GISEL-X86-NEXT:    flds {{[0-9]+}}(%esp)
+; GISEL-X86-NEXT:    fxch %st(1)
+; GISEL-X86-NEXT:    fucompi %st(1), %st
+; GISEL-X86-NEXT:    fstp %st(0)
+; GISEL-X86-NEXT:    sete %cl
+; GISEL-X86-NEXT:    setnp %al
+; GISEL-X86-NEXT:    andb %cl, %al
+; GISEL-X86-NEXT:    retl
+    %1 = fcmp oeq float %x, %y
+    ret i1 %1
+  }
+
+  define i1 @fcmp_float_ogt(float %x, float %y) {
+; X64-LABEL: fcmp_float_ogt:
+; X64:       ## %bb.0:
+; X64-NEXT:    flds {{[0-9]+}}(%rsp)
+; X64-NEXT:    flds {{[0-9]+}}(%rsp)
+; X64-NEXT:    fucompi %st(1), %st
+; X64-NEXT:    fstp %st(0)
+; X64-NEXT:    seta %al
+; X64-NEXT:    retq
+;
+; GISEL-X64-LABEL: fcmp_float_ogt:
+; GISEL-X64:       ## %bb.0:
+; GISEL-X64-NEXT:    flds {{[0-9]+}}(%rsp)
+; GISEL-X64-NEXT:    flds {{[0-9]+}}(%rsp)
+; GISEL-X64-NEXT:    fxch %st(1)
+; GISEL-X64-NEXT:    fucompi %st(1), %st
+; GISEL-X64-NEXT:    fstp %st(0)
+; GISEL-X64-NEXT:    seta %al
+; GISEL-X64-NEXT:    retq
+;
+; X86-LABEL: fcmp_float_ogt:
+; X86:       ## %bb.0:
+; X86-NEXT:    flds {{[0-9]+}}(%esp)
+; X86-NEXT:    flds {{[0-9]+}}(%esp)
+; X86-NEXT:    fucompp
+; X86-NEXT:    fnstsw %ax
+; X86-NEXT:    ## kill: def $ah killed $ah killed $ax
+; X86-NEXT:    sahf
+; X86-NEXT:    seta %al
+; X86-NEXT:    retl
+;
+; GISEL-X86-LABEL: fcmp_float_ogt:
+; GISEL-X86:       ## %bb.0:
+; GISEL-X86-NEXT:    flds {{[0-9]+}}(%esp)
+; GISEL-X86-NEXT:    flds {{[0-9]+}}(%esp)
+; GISEL-X86-NEXT:    fxch %st(1)
+; GISEL-X86-NEXT:    fucompi %st(1), %st
+; GISEL-X86-NEXT:    fstp %st(0)
+; GISEL-X86-NEXT:    seta %al
+; GISEL-X86-NEXT:    retl
+    %1 = fcmp ogt float %x, %y
+    ret i1 %1
+  }
+
+  define i1 @fcmp_float_oge(float %x, float %y) {
+; X64-LABEL: fcmp_float_oge:
+; X64:       ## %bb.0:
+; X64-NEXT:    flds {{[0-9]+}}(%rsp)
+; X64-NEXT:    flds {{[0-9]+}}(%rsp)
+; X64-NEXT:    fucompi %st(1), %st
+; X64-NEXT:    fstp %st(0)
+; X64-NEXT:    setae %al
+; X64-NEXT:    retq
+;
+; GISEL-X64-LABEL: fcmp_float_oge:
+; GISEL-X64:       ## %bb.0:
+; GISEL-X64-NEXT:    flds {{[0-9]+}}(%rsp)
+; GISEL-X64-NEXT:    flds {{[0-9]+}}(%rsp)
+; GISEL-X64-NEXT:    fxch %st(1)
+; GISEL-X64-NEXT:    fucompi %st(1), %st
+; GISEL-X64-NEXT:    fstp %st(0)
+; GISEL-X64-NEXT:    setae %al
+; GISEL-X64-NEXT:    retq
+;
+; X86-LABEL: fcmp_float_oge:
+; X86:       ## %bb.0:
+; X86-NEXT:    flds {{[0-9]+}}(%esp)
+; X86-NEXT:    flds {{[0-9]+}}(%esp)
+; X86-NEXT:    fucompp
+; X86-NEXT:    fnstsw %ax
+; X86-NEXT:    ## kill: def $ah killed $ah killed $ax
+; X86-NEXT:    sahf
+; X86-NEXT:    setae %al
+; X86-NEXT:    retl
+;
+; GISEL-X86-LABEL: fcmp_float_oge:
+; GISEL-X86:       ## %bb.0:
+; GISEL-X86-NEXT:    flds {{[0-9]+}}(%esp)
+; GISEL-X86-NEXT:    flds {{[0-9]+}}(%esp)
+; GISEL-X86-NEXT:    fxch %st(1)
+; GISEL-X86-NEXT:    fucompi %st(1), %st
+; GISEL-X86-NEXT:    fstp %st(0)
+; GISEL-X86-NEXT:    setae %al
+; GISEL-X86-NEXT:    retl
+    %1 = fcmp oge float %x, %y
+    ret i1 %1
+  }
+
+  define i1 @fcmp_float_olt(float %x, float %y) {
+; SDAG-X64-LABEL: fcmp_float_olt:
+; SDAG-X64:       ## %bb.0:
+; SDAG-X64-NEXT:    flds {{[0-9]+}}(%rsp)
+; SDAG-X64-NEXT:    flds {{[0-9]+}}(%rsp)
+; SDAG-X64-NEXT:    fucompi %st(1), %st
+; SDAG-X64-NEXT:    fstp %st(0)
+; SDAG-X64-NEXT:    seta %al
+; SDAG-X64-NEXT:    retq
+;
+; FAST-X64-LABEL: fcmp_float_olt:
+; FAST-X64:       ## %bb.0:
+; FAST-X64-NEXT:    flds {{[0-9]+}}(%rsp)
+; FAST-X64-NEXT:    flds {{[0-9]+}}(%rsp)
+; FAST-X64-NEXT:    fxch %st(1)
+; FAST-X64-NEXT:    fucompi %st(1), %st
+; FAST-X64-NEXT:    fstp %st(0)
+; FAST-X64-NEXT:    seta %al
+; FAST-X64-NEXT:    retq
+;
+; GISEL-X64-LABEL: fcmp_float_olt:
+; GISEL-X64:       ## %bb.0:
+; GISEL-X64-NEXT:    flds {{[0-9]+}}(%rsp)
+; GISEL-X64-NEXT:    flds {{[0-9]+}}(%rsp)
+; GISEL-X64-NEXT:    fucompi %st(1), %st
+; GISEL-X64-NEXT:    fstp %st(0)
+; GISEL-X64-NEXT:    seta %al
+; GISEL-X64-NEXT:    retq
+;
+; SDAG-X86-LABEL: fcmp_float_olt:
+; SDAG-X86:       ## %bb.0:
+; SDAG-X86-NEXT:    flds {{[0-9]+}}(%esp)
+; SDAG-X86-NEXT:    flds {{[0-9]+}}(%esp)
+; SDAG-X86-NEXT:    fucompp
+; SDAG-X86-NEXT:    fnstsw %ax
+; SDAG-X86-NEXT:    ## kill: def $ah killed $ah killed $ax
+; SDAG-X86-NEXT:    sahf
+; SDAG-X86-NEXT:    seta %al
+; SDAG-X86-NEXT:    retl
+;
+; FAST-X86-LABEL: fcmp_float_olt:
+; FAST-X86:       ## %bb.0:
+; FAST-X86-NEXT:    flds {{[0-9]+}}(%esp)
+; FAST-X86-NEXT:    flds {{[0-9]+}}(%esp)
+; FAST-X86-NEXT:    fxch %st(1)
+; FAST-X86-NEXT:    fucompp
+; FAST-X86-NEXT:    fnstsw %ax
+; FAST-X86-NEXT:    ## kill: def $ah killed $ah killed $ax
+; FAST-X86-NEXT:    sahf
+; FAST-X86-NEXT:    seta %al
+; FAST-X86-NEXT:    retl
+;
+; GISEL-X86-LABEL: fcmp_float_olt:
+; GISEL-X86:       ## %bb.0:
+; GISEL-X86-NEXT:    flds {{[0-9]+}}(%esp)
+; GISEL-X86-NEXT:    flds {{[0-9]+}}(%esp)
+; GISEL-X86-NEXT:    fucompi %st(1), %st
+; GISEL-X86-NEXT:    fstp %st(0)
+; GISEL-X86-NEXT:    seta %al
+; GISEL-X86-NEXT:    retl
+    %1 = fcmp olt float %x, %y
+    ret i1 %1
+  }
+
+  define i1 @fcmp_float_ole(float %x, float %y) {
+; SDAG-X64-LABEL: fcmp_float_ole:
+; SDAG-X64:       ## %bb.0:
+; SDAG-X64-NEXT:    flds {{[0-9]+}}(%rsp)
+; SDAG-X64-NEXT:    flds {{[0-9]+}}(%rsp)
+; SDAG-X64-NEXT:    fucompi %st(1), %st
+; SDAG-X64-NEXT:    fstp %st(0)
+; SDAG-X64-NEXT:    setae %al
+; SDAG-X64-NEXT:    retq
+;
+; FAST-X64-LABEL: fcmp_float_ole:
+; FAST-X64:       ## %bb.0:
+; FAST-X64-NEXT:    flds {{[0-9]+}}(%rsp)
+; FAST-X64-NEXT:    flds {{[0-9]+}}(%rsp)
+; FAST-X64-NEXT:    fxch %st(1)
+; FAST-X64-NEXT:    fucompi %st(1), %st
+; FAST-X64-NEXT:    fstp %st(0)
+; FAST-X64-NEXT:    setae %al
+; FAST-X64-NEXT:    retq
+;
+; GISEL-X64-LABEL: fcmp_float_ole:
+; GISEL-X64:       ## %bb.0:
+; GISEL-X64-NEXT:    flds {{[0-9]+}}(%rsp)
+; GISEL-X64-NEXT:    flds {{[0-9]+}}(%rsp)
+; GISEL-X64-NEXT:    fucompi %st(1), %st
+; GISEL-X64-NEXT:    fstp %st(0)
+; GISEL-X64-NEXT:    setae %al
+; GISEL-X64-NEXT:    retq
+;
+; SDAG-X86-LABEL: fcmp_float_ole:
+; SDAG-X86:       ## %bb.0:
+; SDAG-X86-NEXT:    flds {{[0-9]+}}(%esp)
+; SDAG-X86-NEXT:    flds {{[0-9]+}}(%esp)
+; SDAG-X86-NEXT:    fucompp
+; SDAG-X86-NEXT:    fnstsw %ax
+; SDAG-X86-NEXT:    ## kill: def $ah killed $ah killed $ax
+; SDAG-X86-NEXT:    sahf
+; SDAG-X86-NEXT:    setae %al
+; SDAG-X86-NEXT:    retl
+;
+; FAST-X86-LABEL: fcmp_float_ole:
+; FAST-X86:       ## %bb.0:
+; FAST-X86-NEXT:    flds {{[0-9]+}}(%esp)
+; FAST-X86-NEXT:    flds {{[0-9]+}}(%esp)
+; FAST-X86-NEXT:    fxch %st(1)
+; FAST-X86-NEXT:    fucompp
+; FAST-X86-NEXT:    fnstsw %ax
+; FAST-X86-NEXT:    ## kill: def $ah killed $ah killed $ax
+; FAST-X86-NEXT:    sahf
+; FAST-X86-NEXT:    setae %al
+; FAST-X86-NEXT:    retl
+;
+; GISEL-X86-LABEL: fcmp_float_ole:
+; GISEL-X86:       ## %bb.0:
+; GISEL-X86-NEXT:    flds {{[0-9]+}}(%esp)
+; GISEL-X86-NEXT:    flds {{[0-9]+}}(%esp)
+; GISEL-X86-NEXT:    fucompi %st(1), %st
+; GISEL-X86-NEXT:    fstp %st(0)
+; GISEL-X86-NEXT:    setae %al
+; GISEL-X86-NEXT:    retl
+    %1 = fcmp ole float %x, %y
+    ret i1 %1
+  }
+
+  define i1 @fcmp_float_one(float %x, float %y) {
+; X64-LABEL: fcmp_float_one:
+; X64:       ## %bb.0:
+; X64-NEXT:    flds {{[0-9]+}}(%rsp)
+; X64-NEXT:    flds {{[0-9]+}}(%rsp)
+; X64-NEXT:    fucompi %st(1), %st
+; X64-NEXT:    fstp %st(0)
+; X64-NEXT:    setne %al
+; X64-NEXT:    retq
+;
+; GISEL-X64-LABEL: fcmp_float_one:
+; GISEL-X64:       ## %bb.0:
+; GISEL-X64-NEXT:    flds {{[0-9]+}}(%rsp)
+; GISEL-X64-NEXT:    flds {{[0-9]+}}(%rsp)
+; GISEL-X64-NEXT:    fxch %st(1)
+; GISEL-X64-NEXT:    fucompi %st(1), %st
+; GISEL-X64-NEXT:    fstp %st(0)
+; GISEL-X64-NEXT:    setne %al
+; GISEL-X64-NEXT:    retq
+;
+; X86-LABEL: fcmp_float_one:
+; X86:       ## %bb.0:
+; X86-NEXT:    flds {{[0-9]+}}(%esp)
+; X86-NEXT:    flds {{[0-9]+}}(%esp)
+; X86-NEXT:    fucompp
+; X86-NEXT:    fnstsw %ax
+; X86-NEXT:    ## kill: def $ah killed $ah killed $ax
+; X86-NEXT:    sahf
+; X86-NEXT:    setne %al
+; X86-NEXT:    retl
+;
+; GISEL-X86-LABEL: fcmp_float_one:
+; GISEL-X86:       ## %bb.0:
+; GISEL-X86-NEXT:    flds {{[0-9]+}}(%esp)
+; GISEL-X86-NEXT:    flds {{[0-9]+}}(%esp)
+; GISEL-X86-NEXT:    fxch %st(1)
+; GISEL-X86-NEXT:    fucompi %st(1), %st
+; GISEL-X86-NEXT:    fstp %st(0)
+; GISEL-X86-NEXT:    setne %al
+; GISEL-X86-NEXT:    retl
+    %1 = fcmp one float %x, %y
+    ret i1 %1
+  }
+
+  define i1 @fcmp_float_ord(float %x, float %y) {
+; X64-LABEL: fcmp_float_ord:
+; X64:       ## %bb.0:
+; X64-NEXT:    flds {{[0-9]+}}(%rsp)
+; X64-NEXT:    flds {{[0-9]+}}(%rsp)
+; X64-NEXT:    fucompi %st(1), %st
+; X64-NEXT:    fstp %st(0)
+; X64-NEXT:    setnp %al
+; X64-NEXT:    retq
+;
+; GISEL-X64-LABEL: fcmp_float_ord:
+; GISEL-X64:       ## %bb.0:
+; GISEL-X64-NEXT:    flds {{[0-9]+}}(%rsp)
+; GISEL-X64-NEXT:    flds {{[0-9]+}}(%rsp)
+; GISEL-X64-NEXT:    fxch %st(1)
+; GISEL-X64-NEXT:    fucompi %st(1), %st
+; GISEL-X64-NEXT:    fstp %st(0)
+; GISEL-X64-NEXT:    setnp %al
+; GISEL-X64-NEXT:    retq
+;
+; X86-LABEL: fcmp_float_ord:
+; X86:       ## %bb.0:
+; X86-NEXT:    flds {{[0-9]+}}(%esp)
+; X86-NEXT:    flds {{[0-9]+}}(%esp)
+; X86-NEXT:    fucompp
+; X86-NEXT:    fnstsw %ax
+; X86-NEXT:    ## kill: def $ah killed $ah killed $ax
+; X86-NEXT:    sahf
+; X86-NEXT:    setnp %al
+; X86-NEXT:    retl
+;
+; GISEL-X86-LABEL: fcmp_float_ord:
+; GISEL-X86:       ## %bb.0:
+; GISEL-X86-NEXT:    flds {{[0-9]+}}(%esp)
+; GISEL-X86-NEXT:    flds {{[0-9]+}}(%esp)
+; GISEL-X86-NEXT:    fxch %st(1)
+; GISEL-X86-NEXT:    fucompi %st(1), %st
+; GISEL-X86-NEXT:    fstp %st(0)
+; GISEL-X86-NEXT:    setnp %al
+; GISEL-X86-NEXT:    retl
+    %1 = fcmp ord float %x, %y
+    ret i1 %1
+  }
+
+  define i1 @fcmp_float_uno(float %x, float %y) {
+; X64-LABEL: fcmp_float_uno:
+; X64:       ## %bb.0:
+; X64-NEXT:    flds {{[0-9]+}}(%rsp)
+; X64-NEXT:    flds {{[0-9]+}}(%rsp)
+; X64-NEXT:    fucompi %st(1), %st
+; X64-NEXT:    fstp %st(0)
+; X64-NEXT:    setp %al
+; X64-NEXT:    retq
+;
+; GISEL-X64-LABEL: fcmp_float_uno:
+; GISEL-X64:       ## %bb.0:
+; GISEL-X64-NEXT:    flds {{[0-9]+}}(%rsp)
+; GISEL-X64-NEXT:    flds {{[0-9]+}}(%rsp)
+; GISEL-X64-NEXT:    fxch %st(1)
+; GISEL-X64-NEXT:    fucompi %st(1), %st
+; GISEL-X64-NEXT:    fstp %st(0)
+; GISEL-X64-NEXT:    setp %al
+; GISEL-X64-NEXT:    retq
+;
+; X86-LABEL: fcmp_float_uno:
+; X86:       ## %bb.0:
+; X86-NEXT:    flds {{[0-9]+}}(%esp)
+; X86-NEXT:    flds {{[0-9]+}}(%esp)
+; X86-NEXT:    fucompp
+; X86-NEXT:    fnstsw %ax
+; X86-NEXT:    ## kill: def $ah killed $ah killed $ax
+; X86-NEXT:    sahf
+; X86-NEXT:    setp %al
+; X86-NEXT:    retl
+;
+; GISEL-X86-LABEL: fcmp_float_uno:
+; GISEL-X86:       ## %bb.0:
+; GISEL-X86-NEXT:    flds {{[0-9]+}}(%esp)
+; GISEL-X86-NEXT:    flds {{[0-9]+}}(%esp)
+; GISEL-X86-NEXT:    fxch %st(1)
+; GISEL-X86-NEXT:    fucompi %st(1), %st
+; GISEL-X86-NEXT:    fstp %st(0)
+; GISEL-X86-NEXT:    setp %al
+; GISEL-X86-NEXT:    retl
+    %1 = fcmp uno float %x, %y
+    ret i1 %1
+  }
+
+  define i1 @fcmp_float_ueq(float %x, float %y) {
+; X64-LABEL: fcmp_float_ueq:
+; X64:       ## %bb.0:
+; X64-NEXT:    flds {{[0-9]+}}(%rsp)
+; X64-NEXT:    flds {{[0-9]+}}(%rsp)
+; X64-NEXT:    fucompi %st(1), %st
+; X64-NEXT:    fstp %st(0)
+; X64-NEXT:    sete %al
+; X64-NEXT:    retq
+;
+; GISEL-X64-LABEL: fcmp_float_ueq:
+; GISEL-X64:       ## %bb.0:
+; GISEL-X64-NEXT:    flds {{[0-9]+}}(%rsp)
+; GISEL-X64-NEXT:    flds {{[0-9]+}}(%rsp)
+; GISEL-X64-NEXT:    fxch %st(1)
+; GISEL-X64-NEXT:    fucompi %st(1), %st
+; GISEL-X64-NEXT:    fstp %st(0)
+; GISEL-X64-NEXT:    sete %al
+; GISEL-X64-NEXT:    retq
+;
+; X86-LABEL: fcmp_float_ueq:
+; X86:       ## %bb.0:
+; X86-NEXT:    flds {{[0-9]+}}(%esp)
+; X86-NEXT:    flds {{[0-9]+}}(%esp)
+; X86-NEXT:    fucompp
+; X86-NEXT:    fnstsw %ax
+; X86-NEXT:    ## kill: def $ah killed $ah killed $ax
+; X86-NEXT:    sahf
+; X86-NEXT:    sete %al
+; X86-NEXT:    retl
+;
+; GISEL-X86-LABEL: fcmp_float_ueq:
+; GISEL-X86:       ## %bb.0:
+; GISEL-X86-NEXT:    flds {{[0-9]+}}(%esp)
+; GISEL-X86-NEXT:    flds {{[0-9]+}}(%esp)
+; GISEL-X86-NEXT:    fxch %st(1)
+; GISEL-X86-NEXT:    fucompi %st(1), %st
+; GISEL-X86-NEXT:    fstp %st(0)
+; GISEL-X86-NEXT:    sete %al
+; GISEL-X86-NEXT:    retl
+    %1 = fcmp ueq float %x, %y
+    ret i1 %1
+  }
+
+  define i1 @fcmp_float_ugt(float %x, float %y) {
+; SDAG-X64-LABEL: fcmp_float_ugt:
+; SDAG-X64:       ## %bb.0:
+; SDAG-X64-NEXT:    flds {{[0-9]+}}(%rsp)
+; SDAG-X64-NEXT:    flds {{[0-9]+}}(%rsp)
+; SDAG-X64-NEXT:    fucompi %st(1), %st
+; SDAG-X64-NEXT:    fstp %st(0)
+; SDAG-X64-NEXT:    setb %al
+; SDAG-X64-NEXT:    retq
+;
+; FAST-X64-LABEL: fcmp_float_ugt:
+; FAST-X64:       ## %bb.0:
+; FAST-X64-NEXT:    flds {{[0-9]+}}(%rsp)
+; FAST-X64-NEXT:    flds {{[0-9]+}}(%rsp)
+; FAST-X64-NEXT:    fxch %st(1)
+; FAST-X64-NEXT:    fucompi %st(1), %st
+; FAST-X64-NEXT:    fstp %st(0)
+; FAST-X64-NEXT:    setb %al
+; FAST-X64-NEXT:    retq
+;
+; GISEL-X64-LABEL: fcmp_float_ugt:
+; GISEL-X64:       ## %bb.0:
+; GISEL-X64-NEXT:    flds {{[0-9]+}}(%rsp)
+; GISEL-X64-NEXT:    flds {{[0-9]+}}(%rsp)
+; GISEL-X64-NEXT:    fucompi %st(1), %st
+; GISEL-X64-NEXT:    fstp %st(0)
+; GISEL-X64-NEXT:    setb %al
+; GISEL-X64-NEXT:    retq
+;
+; SDAG-X86-LABEL: fcmp_float_ugt:
+; SDAG-X86:       ## %bb.0:
+; SDAG-X86-NEXT:    flds {{[0-9]+}}(%esp)
+; SDAG-X86-NEXT:    flds {{[0-9]+}}(%esp)
+; SDAG-X86-NEXT:    fucompp
+; SDAG-X86-NEXT:    fnstsw %ax
+; SDAG-X86-NEXT:    ## kill: def $ah killed $ah killed $ax
+; SDAG-X86-NEXT:    sahf
+; SDAG-X86-NEXT:    setb %al
+; SDAG-X86-NEXT:    retl
+;
+; FAST-X86-LABEL: fcmp_float_ugt:
+; FAST-X86:       ## %bb.0:
+; FAST-X86-NEXT:    flds {{[0-9]+}}(%esp)
+; FAST-X86-NEXT:    flds {{[0-9]+}}(%esp)
+; FAST-X86-NEXT:    fxch %st(1)
+; FAST-X86-NEXT:    fucompp
+; FAST-X86-NEXT:    fnstsw %ax
+; FAST-X86-NEXT:    ## kill: def $ah killed $ah killed $ax
+; FAST-X86-NEXT:    sahf
+; FAST-X86-NEXT:    setb %al
+; FAST-X86-NEXT:    retl
+;
+; GISEL-X86-LABEL: fcmp_float_ugt:
+; GISEL-X86:       ## %bb.0:
+; GISEL-X86-NEXT:    flds {{[0-9]+}}(%esp)
+; GISEL-X86-NEXT:    flds {{[0-9]+}}(%esp)
+; GISEL-X86-NEXT:    fucompi %st(1), %st
+; GISEL-X86-NEXT:    fstp %st(0)
+; GISEL-X86-NEXT:    setb %al
+; GISEL-X86-NEXT:    retl
+    %1 = fcmp ugt float %x, %y
+    ret i1 %1
+  }
+
+  define i1 @fcmp_float_uge(float %x, float %y) {
+; SDAG-X64-LABEL: fcmp_float_uge:
+; SDAG-X64:       ## %bb.0:
+; SDAG-X64-NEXT:    flds {{[0-9]+}}(%rsp)
+; SDAG-X64-NEXT:    flds {{[0-9]+}}(%rsp)
+; SDAG-X64-NEXT:    fucompi %st(1), %st
+; SDAG-X64-NEXT:    fstp %st(0)
+; SDAG-X64-NEXT:    setbe %al
+; SDAG-X64-NEXT:    retq
+;
+; FAST-X64-LABEL: fcmp_float_uge:
+; FAST-X64:       ## %bb.0:
+; FAST-X64-NEXT:    flds {{[0-9]+}}(%rsp)
+; FAST-X64-NEXT:    flds {{[0-9]+}}(%rsp)
+; FAST-X64-NEXT:    fxch %st(1)
+; FAST-X64-NEXT:    fucompi %st(1), %st
+; FAST-X64-NEXT:    fstp %st(0)
+; FAST-X64-NEXT:    setbe %al
+; FAST-X64-NEXT:    retq
+;
+; GISEL-X64-LABEL: fcmp_float_uge:
+; GISEL-X64:       ## %bb.0:
+; GISEL-X64-NEXT:    flds {{[0-9]+}}(%rsp)
+; GISEL-X64-NEXT:    flds {{[0-9]+}}(%rsp)
+; GISEL-X64-NEXT:    fucompi %st(1), %st
+; GISEL-X64-NEXT:    fstp %st(0)
+; GISEL-X64-NEXT:    setbe %al
+; GISEL-X64-NEXT:    retq
+;
+; SDAG-X86-LABEL: fcmp_float_uge:
+; SDAG-X86:       ## %bb.0:
+; SDAG-X86-NEXT:    flds {{[0-9]+}}(%esp)
+; SDAG-X86-NEXT:    flds {{[0-9]+}}(%esp)
+; SDAG-X86-NEXT:    fucompp
+; SDAG-X86-NEXT:    fnstsw %ax
+; SDAG-X86-NEXT:    ## kill: def $ah killed $ah killed $ax
+; SDAG-X86-NEXT:    sahf
+; SDAG-X86-NEXT:    setbe %al
+; SDAG-X86-NEXT:    retl
+;
+; FAST-X86-LABEL: fcmp_float_uge:
+; FAST-X86:       ## %bb.0:
+; FAST-X86-NEXT:    flds {{[0-9]+}}(%esp)
+; FAST-X86-NEXT:    flds {{[0-9]+}}(%esp)
+; FAST-X86-NEXT:    fxch %st(1)
+; FAST-X86-NEXT:    fucompp
+; FAST-X86-NEXT:    fnstsw %ax
+; FAST-X86-NEXT:    ## kill: def $ah killed $ah killed $ax
+; FAST-X86-NEXT:    sahf
+; FAST-X86-NEXT:    setbe %al
+; FAST-X86-NEXT:    retl
+;
+; GISEL-X86-LABEL: fcmp_float_uge:
+; GISEL-X86:       ## %bb.0:
+; GISEL-X86-NEXT:    flds {{[0-9]+}}(%esp)
+; GISEL-X86-NEXT:    flds {{[0-9]+}}(%esp)
+; GISEL-X86-NEXT:    fucompi %st(1), %st
+; GISEL-X86-NEXT:    fstp %st(0)
+; GISEL-X86-NEXT:    setbe %al
+; GISEL-X86-NEXT:    retl
+    %1 = fcmp uge float %x, %y
+    ret i1 %1
+  }
+
+  define i1 @fcmp_float_ult(float %x, float %y) {
+; X64-LABEL: fcmp_float_ult:
+; X64:       ## %bb.0:
+; X64-NEXT:    flds {{[0-9]+}}(%rsp)
+; X64-NEXT:    flds {{[0-9]+}}(%rsp)
+; X64-NEXT:    fucompi %st(1), %st
+; X64-NEXT:    fstp %st(0)
+; X64-NEXT:    setb %al
+; X64-NEXT:    retq
+;
+; GISEL-X64-LABEL: fcmp_float_ult:
+; GISEL-X64:       ## %bb.0:
+; GISEL-X64-NEXT:    flds {{[0-9]+}}(%rsp)
+; GISEL-X64-NEXT:    flds {{[0-9]+}}(%rsp)
+; GISEL-X64-NEXT:    fxch %st(1)
+; GISEL-X64-NEXT:    fucompi %st(1), %st
+; GISEL-X64-NEXT:    fstp %st(0)
+; GISEL-X64-NEXT:    setb %al
+; GISEL-X64-NEXT:    retq
+;
+; X86-LABEL: fcmp_float_ult:
+; X86:       ## %bb.0:
+; X86-NEXT:    flds {{[0-9]+}}(%esp)
+; X86-NEXT:    flds {{[0-9]+}}(%esp)
+; X86-NEXT:    fucompp
+; X86-NEXT:    fnstsw %ax
+; X86-NEXT:    ## kill: def $ah killed $ah killed $ax
+; X86-NEXT:    sahf
+; X86-NEXT:    setb %al
+; X86-NEXT:    retl
+;
+; GISEL-X86-LABEL: fcmp_float_ult:
+; GISEL-X86:       ## %bb.0:
+; GISEL-X86-NEXT:    flds {{[0-9]+}}(%esp)
+; GISEL-X86-NEXT:    flds {{[0-9]+}}(%esp)
+; GISEL-X86-NEXT:    fxch %st(1)
+; GISEL-X86-NEXT:    fucompi %st(1), %st
+; GISEL-X86-NEXT:    fstp %st(0)
+; GISEL-X86-NEXT:    setb %al
+; GISEL-X86-NEXT:    retl
+    %1 = fcmp ult float %x, %y
+    ret i1 %1
+  }
+
+  define i1 @fcmp_float_ule(float %x, float %y) {
+; X64-LABEL: fcmp_float_ule:
+; X64:       ## %bb.0:
+; X64-NEXT:    flds {{[0-9]+}}(%rsp)
+; X64-NEXT:    flds {{[0-9]+}}(%rsp)
+; X64-NEXT:    fucompi %st(1), %st
+; X64-NEXT:    fstp %st(0)
+; X64-NEXT:    setbe %al
+; X64-NEXT:    retq
+;
+; GISEL-X64-LABEL: fcmp_float_ule:
+; GISEL-X64:       ## %bb.0:
+; GISEL-X64-NEXT:    flds {{[0-9]+}}(%rsp)
+; GISEL-X64-NEXT:    flds {{[0-9]+}}(%rsp)
+; GISEL-X64-NEXT:    fxch %st(1)
+; GISEL-X64-NEXT:    fucompi %st(1), %st
+; GISEL-X64-NEXT:    fstp %st(0)
+; GISEL-X64-NEXT:    setbe %al
+; GISEL-X64-NEXT:    retq
+;
+; X86-LABEL: fcmp_float_ule:
+; X86:       ## %bb.0:
+; X86-NEXT:    flds {{[0-9]+}}(%esp)
+; X86-NEXT:    flds {{[0-9]+}}(%esp)
+; X86-NEXT:    fucompp
+; X86-NEXT:    fnstsw %ax
+; X86-NEXT:    ## kill: def $ah killed $ah killed $ax
+; X86-NEXT:    sahf
+; X86-NEXT:    setbe %al
+; X86-NEXT:    retl
+;
+; GISEL-X86-LABEL: fcmp_float_ule:
+; GISEL-X86:       ## %bb.0:
+; GISEL-X86-NEXT:    flds {{[0-9]+}}(%esp)
+; GISEL-X86-NEXT:    flds {{[0-9]+}}(%esp)
+; GISEL-X86-NEXT:    fxch %st(1)
+; GISEL-X86-NEXT:    fucompi %st(1), %st
+; GISEL-X86-NEXT:    fstp %st(0)
+; GISEL-X86-NEXT:    setbe %al
+; GISEL-X86-NEXT:    retl
+    %1 = fcmp ule float %x, %y
+    ret i1 %1
+  }
+
+  define i1 @fcmp_float_une(float %x, float %y) {
+; X64-LABEL: fcmp_float_une:
+; X64:       ## %bb.0:
+; X64-NEXT:    flds {{[0-9]+}}(%rsp)
+; X64-NEXT:    flds {{[0-9]+}}(%rsp)
+; X64-NEXT:    fucompi %st(1), %st
+; X64-NEXT:    fstp %st(0)
+; X64-NEXT:    setp %cl
+; X64-NEXT:    setne %al
+; X64-NEXT:    orb %cl, %al
+; X64-NEXT:    retq
+;
+; GISEL-X64-LABEL: fcmp_float_une:
+; GISEL-X64:       ## %bb.0:
+; GISEL-X64-NEXT:    flds {{[0-9]+}}(%rsp)
+; GISEL-X64-NEXT:    flds {{[0-9]+}}(%rsp)
+; GISEL-X64-NEXT:    fxch %st(1)
+; GISEL-X64-NEXT:    fucompi %st(1), %st
+; GISEL-X64-NEXT:    fstp %st(0)
+; GISEL-X64-NEXT:    setne %cl
+; GISEL-X64-NEXT:    setp %al
+; GISEL-X64-NEXT:    orb %cl, %al
+; GISEL-X64-NEXT:    retq
+;
+; X86-LABEL: fcmp_float_une:
+; X86:       ## %bb.0:
+; X86-NEXT:    flds {{[0-9]+}}(%esp)
+; X86-NEXT:    flds {{[0-9]+}}(%esp)
+; X86-NEXT:    fucompp
+; X86-NEXT:    fnstsw %ax
+; X86-NEXT:    ## kill: def $ah killed $ah killed $ax
+; X86-NEXT:    sahf
+; X86-NEXT:    setp %cl
+; X86-NEXT:    setne %al
+; X86-NEXT:    orb %cl, %al
+; X86-NEXT:    retl
+;
+; GISEL-X86-LABEL: fcmp_float_une:
+; GISEL-X86:       ## %bb.0:
+; GISEL-X86-NEXT:    flds {{[0-9]+}}(%esp)
+; GISEL-X86-NEXT:    flds {{[0-9]+}}(%esp)
+; GISEL-X86-NEXT:    fxch %st(1)
+; GISEL-X86-NEXT:    fucompi %st(1), %st
+; GISEL-X86-NEXT:    fstp %st(0)
+; GISEL-X86-NEXT:    setne %cl
+; GISEL-X86-NEXT:    setp %al
+; GISEL-X86-NEXT:    orb %cl, %al
+; GISEL-X86-NEXT:    retl
+    %1 = fcmp une float %x, %y
+    ret i1 %1
+  }
diff --git a/llvm/test/CodeGen/X86/isel-fcmp.ll b/llvm/test/CodeGen/X86/isel-fcmp.ll
new file mode 100644
index 0000000000000..4a223aaa4149b
--- /dev/null
+++ b/llvm/test/CodeGen/X86/isel-fcmp.ll
@@ -0,0 +1,888 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s                               -mtriple=x86_64-apple-darwin10 -mattr=+sse,+sse2,-x87 -verify-machineinstrs | FileCheck %s --check-prefixes=X64,SDAG-X64
+; RUN: llc < %s -fast-isel -fast-isel-abort=1 -mtriple=x86_64-apple-darwin10 -mattr=+sse,+sse2,-x87 -verify-machineinstrs | FileCheck %s --check-prefixes=X64,FAST-X64
+; RUN: llc < %s -global-isel -global-isel-abort=1 -mtriple=x86_64-apple-darwin10 -mattr=+sse,+sse2,-x87 -verify-machineinstrs | FileCheck %s --check-prefixes=GISEL-X64
+; i686 with 64 bit store is issue.
+; RUN: llc < %s                               -mtriple=i686-apple-darwin10 -mattr=-sse,-sse2,+x87 -verify-machineinstrs | FileCheck %s --check-prefixes=X86,SDAG-X86
+; Allow fast-isel to fallback to selection dag on x86
+; RUN: llc < %s -fast-isel -mtriple=i686-apple-darwin10 -mattr=-sse,-sse2,+x87 -verify-machineinstrs | FileCheck %s --check-prefixes=X86,FAST-X86
+; llc < %s -global-isel -global-isel-abort=1 -mtriple=i686-apple-darwin10 -mattr=-sse,-sse2,+x87 -verify-machineinstrs | FileCheck %s --check-prefixes=GISEL-X86
+
+  define i1 @fcmp_float_oeq(float %x, float %y) nounwind {
+; X64-LABEL: fcmp_float_oeq:
+; X64:       ## %bb.0:
+; X64-NEXT:    cmpeqss %xmm1, %xmm0
+; X64-NEXT:    movd %xmm0, %eax
+; X64-NEXT:    andl $1, %eax
+; X64-NEXT:    ## kill: def $al killed $al killed $eax
+; X64-NEXT:    retq
+;
+; GISEL-X64-LABEL: fcmp_float_oeq:
+; GISEL-X64:       ## %bb.0:
+; GISEL-X64-NEXT:    ucomiss %xmm1, %xmm0
+; GISEL-X64-NEXT:    sete %cl
+; GISEL-X64-NEXT:    setnp %al
+; GISEL-X64-NEXT:    andb %cl, %al
+; GISEL-X64-NEXT:    retq
+;
+; X86-LABEL: fcmp_float_oeq:
+; X86:       ## %bb.0:
+; X86-NEXT:    flds {{[0-9]+}}(%esp)
+; X86-NEXT:    flds {{[0-9]+}}(%esp)
+; X86-NEXT:    fucompp
+; X86-NEXT:    fnstsw %ax
+; X86-NEXT:    ## kill: def $ah killed $ah killed $ax
+; X86-NEXT:    sahf
+; X86-NEXT:    setnp %cl
+; X86-NEXT:    sete %al
+; X86-NEXT:    andb %cl, %al
+; X86-NEXT:    retl
+    %1 = fcmp oeq float %x, %y
+    ret i1 %1
+  }
+
+  define i1 @fcmp_float_ogt(float %x, float %y) nounwind {
+; X64-LABEL: fcmp_float_ogt:
+; X64:       ## %bb.0:
+; X64-NEXT:    ucomiss %xmm1, %xmm0
+; X64-NEXT:    seta %al
+; X64-NEXT:    retq
+;
+; GISEL-X64-LABEL: fcmp_float_ogt:
+; GISEL-X64:       ## %bb.0:
+; GISEL-X64-NEXT:    ucomiss %xmm1, %xmm0
+; GISEL-X64-NEXT:    seta %al
+; GISEL-X64-NEXT:    retq
+;
+; X86-LABEL: fcmp_float_ogt:
+; X86:       ## %bb.0:
+; X86-NEXT:    flds {{[0-9]+}}(%esp)
+; X86-NEXT:    flds {{[0-9]+}}(%esp)
+; X86-NEXT:    fucompp
+; X86-NEXT:    fnstsw %ax
+; X86-NEXT:    ## kill: def $ah killed $ah killed $ax
+; X86-NEXT:    sahf
+; X86-NEXT:    seta %al
+; X86-NEXT:    retl
+    %1 = fcmp ogt float %x, %y
+    ret i1 %1
+  }
+
+  define i1 @fcmp_float_oge(float %x, float %y) nounwind {
+; X64-LABEL: fcmp_float_oge:
+; X64:       ## %bb.0:
+; X64-NEXT:    ucomiss %xmm1, %xmm0
+; X64-NEXT:    setae %al
+; X64-NEXT:    retq
+;
+; GISEL-X64-LABEL: fcmp_float_oge:
+; GISEL-X64:       ## %bb.0:
+; GISEL-X64-NEXT:    ucomiss %xmm1, %xmm0
+; GISEL-X64-NEXT:    setae %al
+; GISEL-X64-NEXT:    retq
+;
+; X86-LABEL: fcmp_float_oge:
+; X86:       ## %bb.0:
+; X86-NEXT:    flds {{[0-9]+}}(%esp)
+; X86-NEXT:    flds {{[0-9]+}}(%esp)
+; X86-NEXT:    fucompp
+; X86-NEXT:    fnstsw %ax
+; X86-NEXT:    ## kill: def $ah killed $ah killed $ax
+; X86-NEXT:    sahf
+; X86-NEXT:    setae %al
+; X86-NEXT:    retl
+    %1 = fcmp oge float %x, %y
+    ret i1 %1
+  }
+
+  define i1 @fcmp_float_olt(float %x, float %y) nounwind {
+; X64-LABEL: fcmp_float_olt:
+; X64:       ## %bb.0:
+; X64-NEXT:    ucomiss %xmm0, %xmm1
+; X64-NEXT:    seta %al
+; X64-NEXT:    retq
+;
+; GISEL-X64-LABEL: fcmp_float_olt:
+; GISEL-X64:       ## %bb.0:
+; GISEL-X64-NEXT:    ucomiss %xmm0, %xmm1
+; GISEL-X64-NEXT:    seta %al
+; GISEL-X64-NEXT:    retq
+;
+; SDAG-X86-LABEL: fcmp_float_olt:
+; SDAG-X86:       ## %bb.0:
+; SDAG-X86-NEXT:    flds {{[0-9]+}}(%esp)
+; SDAG-X86-NEXT:    flds {{[0-9]+}}(%esp)
+; SDAG-X86-NEXT:    fucompp
+; SDAG-X86-NEXT:    fnstsw %ax
+; SDAG-X86-NEXT:    ## kill: def $ah killed $ah killed $ax
+; SDAG-X86-NEXT:    sahf
+; SDAG-X86-NEXT:    seta %al
+; SDAG-X86-NEXT:    retl
+;
+; FAST-X86-LABEL: fcmp_float_olt:
+; FAST-X86:       ## %bb.0:
+; FAST-X86-NEXT:    flds {{[0-9]+}}(%esp)
+; FAST-X86-NEXT:    flds {{[0-9]+}}(%esp)
+; FAST-X86-NEXT:    fxch %st(1)
+; FAST-X86-NEXT:    fucompp
+; FAST-X86-NEXT:    fnstsw %ax
+; FAST-X86-NEXT:    ## kill: def $ah killed $ah killed $ax
+; FAST-X86-NEXT:    sahf
+; FAST-X86-NEXT:    seta %al
+; FAST-X86-NEXT:    retl
+    %1 = fcmp olt float %x, %y
+    ret i1 %1
+  }
+
+  define i1 @fcmp_float_ole(float %x, float %y) nounwind {
+; X64-LABEL: fcmp_float_ole:
+; X64:       ## %bb.0:
+; X64-NEXT:    ucomiss %xmm0, %xmm1
+; X64-NEXT:    setae %al
+; X64-NEXT:    retq
+;
+; GISEL-X64-LABEL: fcmp_float_ole:
+; GISEL-X64:       ## %bb.0:
+; GISEL-X64-NEXT:    ucomiss %xmm0, %xmm1
+; GISEL-X64-NEXT:    setae %al
+; GISEL-X64-NEXT:    retq
+;
+; SDAG-X86-LABEL: fcmp_float_ole:
+; SDAG-X86:       ## %bb.0:
+; SDAG-X86-NEXT:    flds {{[0-9]+}}(%esp)
+; SDAG-X86-NEXT:    flds {{[0-9]+}}(%esp)
+; SDAG-X86-NEXT:    fucompp
+; SDAG-X86-NEXT:    fnstsw %ax
+; SDAG-X86-NEXT:    ## kill: def $ah killed $ah killed $ax
+; SDAG-X86-NEXT:    sahf
+; SDAG-X86-NEXT:    setae %al
+; SDAG-X86-NEXT:    retl
+;
+; FAST-X86-LABEL: fcmp_float_ole:
+; FAST-X86:       ## %bb.0:
+; FAST-X86-NEXT:    flds {{[0-9]+}}(%esp)
+; FAST-X86-NEXT:    flds {{[0-9]+}}(%esp)
+; FAST-X86-NEXT:    fxch %st(1)
+; FAST-X86-NEXT:    fucompp
+; FAST-X86-NEXT:    fnstsw %ax
+; FAST-X86-NEXT:    ## kill: def $ah killed $ah killed $ax
+; FAST-X86-NEXT:    sahf
+; FAST-X86-NEXT:    setae %al
+; FAST-X86-NEXT:    retl
+    %1 = fcmp ole float %x, %y
+    ret i1 %1
+  }
+
+  define i1 @fcmp_float_one(float %x, float %y) nounwind {
+; X64-LABEL: fcmp_float_one:
+; X64:       ## %bb.0:
+; X64-NEXT:    ucomiss %xmm1, %xmm0
+; X64-NEXT:    setne %al
+; X64-NEXT:    retq
+;
+; GISEL-X64-LABEL: fcmp_float_one:
+; GISEL-X64:       ## %bb.0:
+; GISEL-X64-NEXT:    ucomiss %xmm1, %xmm0
+; GISEL-X64-NEXT:    setne %al
+; GISEL-X64-NEXT:    retq
+;
+; X86-LABEL: fcmp_float_one:
+; X86:       ## %bb.0:
+; X86-NEXT:    flds {{[0-9]+}}(%esp)
+; X86-NEXT:    flds {{[0-9]+}}(%esp)
+; X86-NEXT:    fucompp
+; X86-NEXT:    fnstsw %ax
+; X86-NEXT:    ## kill: def $ah killed $ah killed $ax
+; X86-NEXT:    sahf
+; X86-NEXT:    setne %al
+; X86-NEXT:    retl
+    %1 = fcmp one float %x, %y
+    ret i1 %1
+  }
+
+  define i1 @fcmp_float_ord(float %x, float %y) nounwind {
+; X64-LABEL: fcmp_float_ord:
+; X64:       ## %bb.0:
+; X64-NEXT:    ucomiss %xmm1, %xmm0
+; X64-NEXT:    setnp %al
+; X64-NEXT:    retq
+;
+; GISEL-X64-LABEL: fcmp_float_ord:
+; GISEL-X64:       ## %bb.0:
+; GISEL-X64-NEXT:    ucomiss %xmm1, %xmm0
+; GISEL-X64-NEXT:    setnp %al
+; GISEL-X64-NEXT:    retq
+;
+; X86-LABEL: fcmp_float_ord:
+; X86:       ## %bb.0:
+; X86-NEXT:    flds {{[0-9]+}}(%esp)
+; X86-NEXT:    flds {{[0-9]+}}(%esp)
+; X86-NEXT:    fucompp
+; X86-NEXT:    fnstsw %ax
+; X86-NEXT:    ## kill: def $ah killed $ah killed $ax
+; X86-NEXT:    sahf
+; X86-NEXT:    setnp %al
+; X86-NEXT:    retl
+    %1 = fcmp ord float %x, %y
+    ret i1 %1
+  }
+
+  define i1 @fcmp_float_uno(float %x, float %y) nounwind {
+; X64-LABEL: fcmp_float_uno:
+; X64:       ## %bb.0:
+; X64-NEXT:    ucomiss %xmm1, %xmm0
+; X64-NEXT:    setp %al
+; X64-NEXT:    retq
+;
+; GISEL-X64-LABEL: fcmp_float_uno:
+; GISEL-X64:       ## %bb.0:
+; GISEL-X64-NEXT:    ucomiss %xmm1, %xmm0
+; GISEL-X64-NEXT:    setp %al
+; GISEL-X64-NEXT:    retq
+;
+; X86-LABEL: fcmp_float_uno:
+; X86:       ## %bb.0:
+; X86-NEXT:    flds {{[0-9]+}}(%esp)
+; X86-NEXT:    flds {{[0-9]+}}(%esp)
+; X86-NEXT:    fucompp
+; X86-NEXT:    fnstsw %ax
+; X86-NEXT:    ## kill: def $ah killed $ah killed $ax
+; X86-NEXT:    sahf
+; X86-NEXT:    setp %al
+; X86-NEXT:    retl
+    %1 = fcmp uno float %x, %y
+    ret i1 %1
+  }
+
+  define i1 @fcmp_float_ueq(float %x, float %y) nounwind {
+; X64-LABEL: fcmp_float_ueq:
+; X64:       ## %bb.0:
+; X64-NEXT:    ucomiss %xmm1, %xmm0
+; X64-NEXT:    sete %al
+; X64-NEXT:    retq
+;
+; GISEL-X64-LABEL: fcmp_float_ueq:
+; GISEL-X64:       ## %bb.0:
+; GISEL-X64-NEXT:    ucomiss %xmm1, %xmm0
+; GISEL-X64-NEXT:    sete %al
+; GISEL-X64-NEXT:    retq
+;
+; X86-LABEL: fcmp_float_ueq:
+; X86:       ## %bb.0:
+; X86-NEXT:    flds {{[0-9]+}}(%esp)
+; X86-NEXT:    flds {{[0-9]+}}(%esp)
+; X86-NEXT:    fucompp
+; X86-NEXT:    fnstsw %ax
+; X86-NEXT:    ## kill: def $ah killed $ah killed $ax
+; X86-NEXT:    sahf
+; X86-NEXT:    sete %al
+; X86-NEXT:    retl
+    %1 = fcmp ueq float %x, %y
+    ret i1 %1
+  }
+
+  define i1 @fcmp_float_ugt(float %x, float %y) nounwind {
+; X64-LABEL: fcmp_float_ugt:
+; X64:       ## %bb.0:
+; X64-NEXT:    ucomiss %xmm0, %xmm1
+; X64-NEXT:    setb %al
+; X64-NEXT:    retq
+;
+; GISEL-X64-LABEL: fcmp_float_ugt:
+; GISEL-X64:       ## %bb.0:
+; GISEL-X64-NEXT:    ucomiss %xmm0, %xmm1
+; GISEL-X64-NEXT:    setb %al
+; GISEL-X64-NEXT:    retq
+;
+; SDAG-X86-LABEL: fcmp_float_ugt:
+; SDAG-X86:       ## %bb.0:
+; SDAG-X86-NEXT:    flds {{[0-9]+}}(%esp)
+; SDAG-X86-NEXT:    flds {{[0-9]+}}(%esp)
+; SDAG-X86-NEXT:    fucompp
+; SDAG-X86-NEXT:    fnstsw %ax
+; SDAG-X86-NEXT:    ## kill: def $ah killed $ah killed $ax
+; SDAG-X86-NEXT:    sahf
+; SDAG-X86-NEXT:    setb %al
+; SDAG-X86-NEXT:    retl
+;
+; FAST-X86-LABEL: fcmp_float_ugt:
+; FAST-X86:       ## %bb.0:
+; FAST-X86-NEXT:    flds {{[0-9]+}}(%esp)
+; FAST-X86-NEXT:    flds {{[0-9]+}}(%esp)
+; FAST-X86-NEXT:    fxch %st(1)
+; FAST-X86-NEXT:    fucompp
+; FAST-X86-NEXT:    fnstsw %ax
+; FAST-X86-NEXT:    ## kill: def $ah killed $ah killed $ax
+; FAST-X86-NEXT:    sahf
+; FAST-X86-NEXT:    setb %al
+; FAST-X86-NEXT:    retl
+    %1 = fcmp ugt float %x, %y
+    ret i1 %1
+  }
+
+  define i1 @fcmp_float_uge(float %x, float %y) nounwind {
+; X64-LABEL: fcmp_float_uge:
+; X64:       ## %bb.0:
+; X64-NEXT:    ucomiss %xmm0, %xmm1
+; X64-NEXT:    setbe %al
+; X64-NEXT:    retq
+;
+; GISEL-X64-LABEL: fcmp_float_uge:
+; GISEL-X64:       ## %bb.0:
+; GISEL-X64-NEXT:    ucomiss %xmm0, %xmm1
+; GISEL-X64-NEXT:    setbe %al
+; GISEL-X64-NEXT:    retq
+;
+; SDAG-X86-LABEL: fcmp_float_uge:
+; SDAG-X86:       ## %bb.0:
+; SDAG-X86-NEXT:    flds {{[0-9]+}}(%esp)
+; SDAG-X86-NEXT:    flds {{[0-9]+}}(%esp)
+; SDAG-X86-NEXT:    fucompp
+; SDAG-X86-NEXT:    fnstsw %ax
+; SDAG-X86-NEXT:    ## kill: def $ah killed $ah killed $ax
+; SDAG-X86-NEXT:    sahf
+; SDAG-X86-NEXT:    setbe %al
+; SDAG-X86-NEXT:    retl
+;
+; FAST-X86-LABEL: fcmp_float_uge:
+; FAST-X86:       ## %bb.0:
+; FAST-X86-NEXT:    flds {{[0-9]+}}(%esp)
+; FAST-X86-NEXT:    flds {{[0-9]+}}(%esp)
+; FAST-X86-NEXT:    fxch %st(1)
+; FAST-X86-NEXT:    fucompp
+; FAST-X86-NEXT:    fnstsw %ax
+; FAST-X86-NEXT:    ## kill: def $ah killed $ah killed $ax
+; FAST-X86-NEXT:    sahf
+; FAST-X86-NEXT:    setbe %al
+; FAST-X86-NEXT:    retl
+    %1 = fcmp uge float %x, %y
+    ret i1 %1
+  }
+
+  define i1 @fcmp_float_ult(float %x, float %y) nounwind {
+; X64-LABEL: fcmp_float_ult:
+; X64:       ## %bb.0:
+; X64-NEXT:    ucomiss %xmm1, %xmm0
+; X64-NEXT:    setb %al
+; X64-NEXT:    retq
+;
+; GISEL-X64-LABEL: fcmp_float_ult:
+; GISEL-X64:       ## %bb.0:
+; GISEL-X64-NEXT:    ucomiss %xmm1, %xmm0
+; GISEL-X64-NEXT:    setb %al
+; GISEL-X64-NEXT:    retq
+;
+; X86-LABEL: fcmp_float_ult:
+; X86:       ## %bb.0:
+; X86-NEXT:    flds {{[0-9]+}}(%esp)
+; X86-NEXT:    flds {{[0-9]+}}(%esp)
+; X86-NEXT:    fucompp
+; X86-NEXT:    fnstsw %ax
+; X86-NEXT:    ## kill: def $ah killed $ah killed $ax
+; X86-NEXT:    sahf
+; X86-NEXT:    setb %al
+; X86-NEXT:    retl
+    %1 = fcmp ult float %x, %y
+    ret i1 %1
+  }
+
+  define i1 @fcmp_float_ule(float %x, float %y) nounwind {
+; X64-LABEL: fcmp_float_ule:
+; X64:       ## %bb.0:
+; X64-NEXT:    ucomiss %xmm1, %xmm0
+; X64-NEXT:    setbe %al
+; X64-NEXT:    retq
+;
+; GISEL-X64-LABEL: fcmp_float_ule:
+; GISEL-X64:       ## %bb.0:
+; GISEL-X64-NEXT:    ucomiss %xmm1, %xmm0
+; GISEL-X64-NEXT:    setbe %al
+; GISEL-X64-NEXT:    retq
+;
+; X86-LABEL: fcmp_float_ule:
+; X86:       ## %bb.0:
+; X86-NEXT:    flds {{[0-9]+}}(%esp)
+; X86-NEXT:    flds {{[0-9]+}}(%esp)
+; X86-NEXT:    fucompp
+; X86-NEXT:    fnstsw %ax
+; X86-NEXT:    ## kill: def $ah killed $ah killed $ax
+; X86-NEXT:    sahf
+; X86-NEXT:    setbe %al
+; X86-NEXT:    retl
+    %1 = fcmp ule float %x, %y
+    ret i1 %1
+  }
+
+  define i1 @fcmp_float_une(float %x, float %y) nounwind {
+; X64-LABEL: fcmp_float_une:
+; X64:       ## %bb.0:
+; X64-NEXT:    cmpneqss %xmm1, %xmm0
+; X64-NEXT:    movd %xmm0, %eax
+; X64-NEXT:    andl $1, %eax
+; X64-NEXT:    ## kill: def $al killed $al killed $eax
+; X64-NEXT:    retq
+;
+; GISEL-X64-LABEL: fcmp_float_une:
+; GISEL-X64:       ## %bb.0:
+; GISEL-X64-NEXT:    ucomiss %xmm1, %xmm0
+; GISEL-X64-NEXT:    setne %cl
+; GISEL-X64-NEXT:    setp %al
+; GISEL-X64-NEXT:    orb %cl, %al
+; GISEL-X64-NEXT:    retq
+;
+; X86-LABEL: fcmp_float_une:
+; X86:       ## %bb.0:
+; X86-NEXT:    flds {{[0-9]+}}(%esp)
+; X86-NEXT:    flds {{[0-9]+}}(%esp)
+; X86-NEXT:    fucompp
+; X86-NEXT:    fnstsw %ax
+; X86-NEXT:    ## kill: def $ah killed $ah killed $ax
+; X86-NEXT:    sahf
+; X86-NEXT:    setp %cl
+; X86-NEXT:    setne %al
+; X86-NEXT:    orb %cl, %al
+; X86-NEXT:    retl
+    %1 = fcmp une float %x, %y
+    ret i1 %1
+  }
+
+  define i1 @fcmp_double_oeq(double %x, double %y) nounwind {
+; X64-LABEL: fcmp_double_oeq:
+; X64:       ## %bb.0:
+; X64-NEXT:    cmpeqsd %xmm1, %xmm0
+; X64-NEXT:    movq %xmm0, %rax
+; X64-NEXT:    andl $1, %eax
+; X64-NEXT:    ## kill: def $al killed $al killed $rax
+; X64-NEXT:    retq
+;
+; GISEL-X64-LABEL: fcmp_double_oeq:
+; GISEL-X64:       ## %bb.0:
+; GISEL-X64-NEXT:    ucomisd %xmm1, %xmm0
+; GISEL-X64-NEXT:    sete %cl
+; GISEL-X64-NEXT:    setnp %al
+; GISEL-X64-NEXT:    andb %cl, %al
+; GISEL-X64-NEXT:    retq
+;
+; X86-LABEL: fcmp_double_oeq:
+; X86:       ## %bb.0:
+; X86-NEXT:    fldl {{[0-9]+}}(%esp)
+; X86-NEXT:    fldl {{[0-9]+}}(%esp)
+; X86-NEXT:    fucompp
+; X86-NEXT:    fnstsw %ax
+; X86-NEXT:    ## kill: def $ah killed $ah killed $ax
+; X86-NEXT:    sahf
+; X86-NEXT:    setnp %cl
+; X86-NEXT:    sete %al
+; X86-NEXT:    andb %cl, %al
+; X86-NEXT:    retl
+    %1 = fcmp oeq double %x, %y
+    ret i1 %1
+  }
+
+  define i1 @fcmp_double_ogt(double %x, double %y) nounwind {
+; X64-LABEL: fcmp_double_ogt:
+; X64:       ## %bb.0:
+; X64-NEXT:    ucomisd %xmm1, %xmm0
+; X64-NEXT:    seta %al
+; X64-NEXT:    retq
+;
+; GISEL-X64-LABEL: fcmp_double_ogt:
+; GISEL-X64:       ## %bb.0:
+; GISEL-X64-NEXT:    ucomisd %xmm1, %xmm0
+; GISEL-X64-NEXT:    seta %al
+; GISEL-X64-NEXT:    retq
+;
+; X86-LABEL: fcmp_double_ogt:
+; X86:       ## %bb.0:
+; X86-NEXT:    fldl {{[0-9]+}}(%esp)
+; X86-NEXT:    fldl {{[0-9]+}}(%esp)
+; X86-NEXT:    fucompp
+; X86-NEXT:    fnstsw %ax
+; X86-NEXT:    ## kill: def $ah killed $ah killed $ax
+; X86-NEXT:    sahf
+; X86-NEXT:    seta %al
+; X86-NEXT:    retl
+    %1 = fcmp ogt double %x, %y
+    ret i1 %1
+  }
+
+  define i1 @fcmp_double_oge(double %x, double %y) nounwind {
+; X64-LABEL: fcmp_double_oge:
+; X64:       ## %bb.0:
+; X64-NEXT:    ucomisd %xmm1, %xmm0
+; X64-NEXT:    setae %al
+; X64-NEXT:    retq
+;
+; GISEL-X64-LABEL: fcmp_double_oge:
+; GISEL-X64:       ## %bb.0:
+; GISEL-X64-NEXT:    ucomisd %xmm1, %xmm0
+; GISEL-X64-NEXT:    setae %al
+; GISEL-X64-NEXT:    retq
+;
+; X86-LABEL: fcmp_double_oge:
+; X86:       ## %bb.0:
+; X86-NEXT:    fldl {{[0-9]+}}(%esp)
+; X86-NEXT:    fldl {{[0-9]+}}(%esp)
+; X86-NEXT:    fucompp
+; X86-NEXT:    fnstsw %ax
+; X86-NEXT:    ## kill: def $ah killed $ah killed $ax
+; X86-NEXT:    sahf
+; X86-NEXT:    setae %al
+; X86-NEXT:    retl
+    %1 = fcmp oge double %x, %y
+    ret i1 %1
+  }
+
+  define i1 @fcmp_double_olt(double %x, double %y) nounwind {
+; X64-LABEL: fcmp_double_olt:
+; X64:       ## %bb.0:
+; X64-NEXT:    ucomisd %xmm0, %xmm1
+; X64-NEXT:    seta %al
+; X64-NEXT:    retq
+;
+; GISEL-X64-LABEL: fcmp_double_olt:
+; GISEL-X64:       ## %bb.0:
+; GISEL-X64-NEXT:    ucomisd %xmm0, %xmm1
+; GISEL-X64-NEXT:    seta %al
+; GISEL-X64-NEXT:    retq
+;
+; SDAG-X86-LABEL: fcmp_double_olt:
+; SDAG-X86:       ## %bb.0:
+; SDAG-X86-NEXT:    fldl {{[0-9]+}}(%esp)
+; SDAG-X86-NEXT:    fldl {{[0-9]+}}(%esp)
+; SDAG-X86-NEXT:    fucompp
+; SDAG-X86-NEXT:    fnstsw %ax
+; SDAG-X86-NEXT:    ## kill: def $ah killed $ah killed $ax
+; SDAG-X86-NEXT:    sahf
+; SDAG-X86-NEXT:    seta %al
+; SDAG-X86-NEXT:    retl
+;
+; FAST-X86-LABEL: fcmp_double_olt:
+; FAST-X86:       ## %bb.0:
+; FAST-X86-NEXT:    fldl {{[0-9]+}}(%esp)
+; FAST-X86-NEXT:    fldl {{[0-9]+}}(%esp)
+; FAST-X86-NEXT:    fxch %st(1)
+; FAST-X86-NEXT:    fucompp
+; FAST-X86-NEXT:    fnstsw %ax
+; FAST-X86-NEXT:    ## kill: def $ah killed $ah killed $ax
+; FAST-X86-NEXT:    sahf
+; FAST-X86-NEXT:    seta %al
+; FAST-X86-NEXT:    retl
+    %1 = fcmp olt double %x, %y
+    ret i1 %1
+  }
+
+  define i1 @fcmp_double_ole(double %x, double %y) nounwind {
+; X64-LABEL: fcmp_double_ole:
+; X64:       ## %bb.0:
+; X64-NEXT:    ucomisd %xmm0, %xmm1
+; X64-NEXT:    setae %al
+; X64-NEXT:    retq
+;
+; GISEL-X64-LABEL: fcmp_double_ole:
+; GISEL-X64:       ## %bb.0:
+; GISEL-X64-NEXT:    ucomisd %xmm0, %xmm1
+; GISEL-X64-NEXT:    setae %al
+; GISEL-X64-NEXT:    retq
+;
+; SDAG-X86-LABEL: fcmp_double_ole:
+; SDAG-X86:       ## %bb.0:
+; SDAG-X86-NEXT:    fldl {{[0-9]+}}(%esp)
+; SDAG-X86-NEXT:    fldl {{[0-9]+}}(%esp)
+; SDAG-X86-NEXT:    fucompp
+; SDAG-X86-NEXT:    fnstsw %ax
+; SDAG-X86-NEXT:    ## kill: def $ah killed $ah killed $ax
+; SDAG-X86-NEXT:    sahf
+; SDAG-X86-NEXT:    setae %al
+; SDAG-X86-NEXT:    retl
+;
+; FAST-X86-LABEL: fcmp_double_ole:
+; FAST-X86:       ## %bb.0:
+; FAST-X86-NEXT:    fldl {{[0-9]+}}(%esp)
+; FAST-X86-NEXT:    fldl {{[0-9]+}}(%esp)
+; FAST-X86-NEXT:    fxch %st(1)
+; FAST-X86-NEXT:    fucompp
+; FAST-X86-NEXT:    fnstsw %ax
+; FAST-X86-NEXT:    ## kill: def $ah killed $ah killed $ax
+; FAST-X86-NEXT:    sahf
+; FAST-X86-NEXT:    setae %al
+; FAST-X86-NEXT:    retl
+    %1 = fcmp ole double %x, %y
+    ret i1 %1
+  }
+
+  define i1 @fcmp_double_one(double %x, double %y) nounwind {
+; X64-LABEL: fcmp_double_one:
+; X64:       ## %bb.0:
+; X64-NEXT:    ucomisd %xmm1, %xmm0
+; X64-NEXT:    setne %al
+; X64-NEXT:    retq
+;
+; GISEL-X64-LABEL: fcmp_double_one:
+; GISEL-X64:       ## %bb.0:
+; GISEL-X64-NEXT:    ucomisd %xmm1, %xmm0
+; GISEL-X64-NEXT:    setne %al
+; GISEL-X64-NEXT:    retq
+;
+; X86-LABEL: fcmp_double_one:
+; X86:       ## %bb.0:
+; X86-NEXT:    fldl {{[0-9]+}}(%esp)
+; X86-NEXT:    fldl {{[0-9]+}}(%esp)
+; X86-NEXT:    fucompp
+; X86-NEXT:    fnstsw %ax
+; X86-NEXT:    ## kill: def $ah killed $ah killed $ax
+; X86-NEXT:    sahf
+; X86-NEXT:    setne %al
+; X86-NEXT:    retl
+    %1 = fcmp one double %x, %y
+    ret i1 %1
+  }
+
+  define i1 @fcmp_double_ord(double %x, double %y) nounwind {
+; X64-LABEL: fcmp_double_ord:
+; X64:       ## %bb.0:
+; X64-NEXT:    ucomisd %xmm1, %xmm0
+; X64-NEXT:    setnp %al
+; X64-NEXT:    retq
+;
+; GISEL-X64-LABEL: fcmp_double_ord:
+; GISEL-X64:       ## %bb.0:
+; GISEL-X64-NEXT:    ucomisd %xmm1, %xmm0
+; GISEL-X64-NEXT:    setnp %al
+; GISEL-X64-NEXT:    retq
+;
+; X86-LABEL: fcmp_double_ord:
+; X86:       ## %bb.0:
+; X86-NEXT:    fldl {{[0-9]+}}(%esp)
+; X86-NEXT:    fldl {{[0-9]+}}(%esp)
+; X86-NEXT:    fucompp
+; X86-NEXT:    fnstsw %ax
+; X86-NEXT:    ## kill: def $ah killed $ah killed $ax
+; X86-NEXT:    sahf
+; X86-NEXT:    setnp %al
+; X86-NEXT:    retl
+    %1 = fcmp ord double %x, %y
+    ret i1 %1
+  }
+
+  define i1 @fcmp_double_uno(double %x, double %y) nounwind {
+; X64-LABEL: fcmp_double_uno:
+; X64:       ## %bb.0:
+; X64-NEXT:    ucomisd %xmm1, %xmm0
+; X64-NEXT:    setp %al
+; X64-NEXT:    retq
+;
+; GISEL-X64-LABEL: fcmp_double_uno:
+; GISEL-X64:       ## %bb.0:
+; GISEL-X64-NEXT:    ucomisd %xmm1, %xmm0
+; GISEL-X64-NEXT:    setp %al
+; GISEL-X64-NEXT:    retq
+;
+; X86-LABEL: fcmp_double_uno:
+; X86:       ## %bb.0:
+; X86-NEXT:    fldl {{[0-9]+}}(%esp)
+; X86-NEXT:    fldl {{[0-9]+}}(%esp)
+; X86-NEXT:    fucompp
+; X86-NEXT:    fnstsw %ax
+; X86-NEXT:    ## kill: def $ah killed $ah killed $ax
+; X86-NEXT:    sahf
+; X86-NEXT:    setp %al
+; X86-NEXT:    retl
+    %1 = fcmp uno double %x, %y
+    ret i1 %1
+  }
+
+  define i1 @fcmp_double_ueq(double %x, double %y) nounwind {
+; X64-LABEL: fcmp_double_ueq:
+; X64:       ## %bb.0:
+; X64-NEXT:    ucomisd %xmm1, %xmm0
+; X64-NEXT:    sete %al
+; X64-NEXT:    retq
+;
+; GISEL-X64-LABEL: fcmp_double_ueq:
+; GISEL-X64:       ## %bb.0:
+; GISEL-X64-NEXT:    ucomisd %xmm1, %xmm0
+; GISEL-X64-NEXT:    sete %al
+; GISEL-X64-NEXT:    retq
+;
+; X86-LABEL: fcmp_double_ueq:
+; X86:       ## %bb.0:
+; X86-NEXT:    fldl {{[0-9]+}}(%esp)
+; X86-NEXT:    fldl {{[0-9]+}}(%esp)
+; X86-NEXT:    fucompp
+; X86-NEXT:    fnstsw %ax
+; X86-NEXT:    ## kill: def $ah killed $ah killed $ax
+; X86-NEXT:    sahf
+; X86-NEXT:    sete %al
+; X86-NEXT:    retl
+    %1 = fcmp ueq double %x, %y
+    ret i1 %1
+  }
+
+  define i1 @fcmp_double_ugt(double %x, double %y) nounwind {
+; X64-LABEL: fcmp_double_ugt:
+; X64:       ## %bb.0:
+; X64-NEXT:    ucomisd %xmm0, %xmm1
+; X64-NEXT:    setb %al
+; X64-NEXT:    retq
+;
+; GISEL-X64-LABEL: fcmp_double_ugt:
+; GISEL-X64:       ## %bb.0:
+; GISEL-X64-NEXT:    ucomisd %xmm0, %xmm1
+; GISEL-X64-NEXT:    setb %al
+; GISEL-X64-NEXT:    retq
+;
+; SDAG-X86-LABEL: fcmp_double_ugt:
+; SDAG-X86:       ## %bb.0:
+; SDAG-X86-NEXT:    fldl {{[0-9]+}}(%esp)
+; SDAG-X86-NEXT:    fldl {{[0-9]+}}(%esp)
+; SDAG-X86-NEXT:    fucompp
+; SDAG-X86-NEXT:    fnstsw %ax
+; SDAG-X86-NEXT:    ## kill: def $ah killed $ah killed $ax
+; SDAG-X86-NEXT:    sahf
+; SDAG-X86-NEXT:    setb %al
+; SDAG-X86-NEXT:    retl
+;
+; FAST-X86-LABEL: fcmp_double_ugt:
+; FAST-X86:       ## %bb.0:
+; FAST-X86-NEXT:    fldl {{[0-9]+}}(%esp)
+; FAST-X86-NEXT:    fldl {{[0-9]+}}(%esp)
+; FAST-X86-NEXT:    fxch %st(1)
+; FAST-X86-NEXT:    fucompp
+; FAST-X86-NEXT:    fnstsw %ax
+; FAST-X86-NEXT:    ## kill: def $ah killed $ah killed $ax
+; FAST-X86-NEXT:    sahf
+; FAST-X86-NEXT:    setb %al
+; FAST-X86-NEXT:    retl
+    %1 = fcmp ugt double %x, %y
+    ret i1 %1
+  }
+
+  define i1 @fcmp_double_uge(double %x, double %y) nounwind {
+; X64-LABEL: fcmp_double_uge:
+; X64:       ## %bb.0:
+; X64-NEXT:    ucomisd %xmm0, %xmm1
+; X64-NEXT:    setbe %al
+; X64-NEXT:    retq
+;
+; GISEL-X64-LABEL: fcmp_double_uge:
+; GISEL-X64:       ## %bb.0:
+; GISEL-X64-NEXT:    ucomisd %xmm0, %xmm1
+; GISEL-X64-NEXT:    setbe %al
+; GISEL-X64-NEXT:    retq
+;
+; SDAG-X86-LABEL: fcmp_double_uge:
+; SDAG-X86:       ## %bb.0:
+; SDAG-X86-NEXT:    fldl {{[0-9]+}}(%esp)
+; SDAG-X86-NEXT:    fldl {{[0-9]+}}(%esp)
+; SDAG-X86-NEXT:    fucompp
+; SDAG-X86-NEXT:    fnstsw %ax
+; SDAG-X86-NEXT:    ## kill: def $ah killed $ah killed $ax
+; SDAG-X86-NEXT:    sahf
+; SDAG-X86-NEXT:    setbe %al
+; SDAG-X86-NEXT:    retl
+;
+; FAST-X86-LABEL: fcmp_double_uge:
+; FAST-X86:       ## %bb.0:
+; FAST-X86-NEXT:    fldl {{[0-9]+}}(%esp)
+; FAST-X86-NEXT:    fldl {{[0-9]+}}(%esp)
+; FAST-X86-NEXT:    fxch %st(1)
+; FAST-X86-NEXT:    fucompp
+; FAST-X86-NEXT:    fnstsw %ax
+; FAST-X86-NEXT:    ## kill: def $ah killed $ah killed $ax
+; FAST-X86-NEXT:    sahf
+; FAST-X86-NEXT:    setbe %al
+; FAST-X86-NEXT:    retl
+    %1 = fcmp uge double %x, %y
+    ret i1 %1
+  }
+
+  define i1 @fcmp_double_ult(double %x, double %y) nounwind {
+; X64-LABEL: fcmp_double_ult:
+; X64:       ## %bb.0:
+; X64-NEXT:    ucomisd %xmm1, %xmm0
+; X64-NEXT:    setb %al
+; X64-NEXT:    retq
+;
+; GISEL-X64-LABEL: fcmp_double_ult:
+; GISEL-X64:       ## %bb.0:
+; GISEL-X64-NEXT:    ucomisd %xmm1, %xmm0
+; GISEL-X64-NEXT:    setb %al
+; GISEL-X64-NEXT:    retq
+;
+; X86-LABEL: fcmp_double_ult:
+; X86:       ## %bb.0:
+; X86-NEXT:    fldl {{[0-9]+}}(%esp)
+; X86-NEXT:    fldl {{[0-9]+}}(%esp)
+; X86-NEXT:    fucompp
+; X86-NEXT:    fnstsw %ax
+; X86-NEXT:    ## kill: def $ah killed $ah killed $ax
+; X86-NEXT:    sahf
+; X86-NEXT:    setb %al
+; X86-NEXT:    retl
+    %1 = fcmp ult double %x, %y
+    ret i1 %1
+  }
+
+  define i1 @fcmp_double_ule(double %x, double %y) nounwind {
+; X64-LABEL: fcmp_double_ule:
+; X64:       ## %bb.0:
+; X64-NEXT:    ucomisd %xmm1, %xmm0
+; X64-NEXT:    setbe %al
+; X64-NEXT:    retq
+;
+; GISEL-X64-LABEL: fcmp_double_ule:
+; GISEL-X64:       ## %bb.0:
+; GISEL-X64-NEXT:    ucomisd %xmm1, %xmm0
+; GISEL-X64-NEXT:    setbe %al
+; GISEL-X64-NEXT:    retq
+;
+; X86-LABEL: fcmp_double_ule:
+; X86:       ## %bb.0:
+; X86-NEXT:    fldl {{[0-9]+}}(%esp)
+; X86-NEXT:    fldl {{[0-9]+}}(%esp)
+; X86-NEXT:    fucompp
+; X86-NEXT:    fnstsw %ax
+; X86-NEXT:    ## kill: def $ah killed $ah killed $ax
+; X86-NEXT:    sahf
+; X86-NEXT:    setbe %al
+; X86-NEXT:    retl
+    %1 = fcmp ule double %x, %y
+    ret i1 %1
+  }
+
+  define i1 @fcmp_double_une(double %x, double %y) nounwind {
+; X64-LABEL: fcmp_double_une:
+; X64:       ## %bb.0:
+; X64-NEXT:    cmpneqsd %xmm1, %xmm0
+; X64-NEXT:    movq %xmm0, %rax
+; X64-NEXT:    andl $1, %eax
+; X64-NEXT:    ## kill: def $al killed $al killed $rax
+; X64-NEXT:    retq
+;
+; GISEL-X64-LABEL: fcmp_double_une:
+; GISEL-X64:       ## %bb.0:
+; GISEL-X64-NEXT:    ucomisd %xmm1, %xmm0
+; GISEL-X64-NEXT:    setne %cl
+; GISEL-X64-NEXT:    setp %al
+; GISEL-X64-NEXT:    orb %cl, %al
+; GISEL-X64-NEXT:    retq
+;
+; X86-LABEL: fcmp_double_une:
+; X86:       ## %bb.0:
+; X86-NEXT:    fldl {{[0-9]+}}(%esp)
+; X86-NEXT:    fldl {{[0-9]+}}(%esp)
+; X86-NEXT:    fucompp
+; X86-NEXT:    fnstsw %ax
+; X86-NEXT:    ## kill: def $ah killed $ah killed $ax
+; X86-NEXT:    sahf
+; X86-NEXT:    setp %cl
+; X86-NEXT:    setne %al
+; X86-NEXT:    orb %cl, %al
+; X86-NEXT:    retl
+    %1 = fcmp une double %x, %y
+    ret i1 %1
+  }
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; FAST-X64: {{.*}}
+; SDAG-X64: {{.*}}
diff --git a/llvm/test/CodeGen/X86/jump-table-partition.ll b/llvm/test/CodeGen/X86/jump-table-partition.ll
index 8c7e726576bce..0d76f8a5a91ed 100644
--- a/llvm/test/CodeGen/X86/jump-table-partition.ll
+++ b/llvm/test/CodeGen/X86/jump-table-partition.ll
@@ -6,18 +6,55 @@
 ; RUN: llc -mtriple=x86_64-unknown-linux-gnu -stop-after=finalize-isel -min-jump-table-entries=2 %s -o %t.mir
 ; RUN: llc -mtriple=x86_64-unknown-linux-gnu --run-pass=static-data-splitter -stats -x mir %t.mir -o - 2>&1 | FileCheck %s --check-prefix=STAT
 
-; Tests stat messages are expected.
-; COM: Update test to verify section suffixes when target-lowering and assembler changes are implemented.
-; COM: Also run static-data-splitter pass with -static-data-default-hotness=cold and check data section suffix.
- 
+ ; @foo has 2 hot and 2 cold jump tables.
+ ; The two jump tables with unknown hotness come from @func_without_profile and
+ ; @bar respectively.
 ; STAT: 2 static-data-splitter - Number of cold jump tables seen
 ; STAT: 2 static-data-splitter - Number of hot jump tables seen
-; STAT: 1 static-data-splitter - Number of jump tables with unknown hotness
-
-; In function @foo, the 2 switch instructions to jt0.* and jt1.* get lowered to hot jump tables,
-; and the 2 switch instructions to jt2.* and jt3.* get lowered to cold jump tables.
-
-; @func_without_profile doesn't have profiles. It's jump table hotness is unknown.
+; STAT: 2 static-data-splitter - Number of jump tables with unknown hotness
+
+; RUN: llc -mtriple=x86_64-unknown-linux-gnu -enable-split-machine-functions \
+; RUN:     -partition-static-data-sections=true -function-sections=true \
+; RUN:     -min-jump-table-entries=2 -unique-section-names=false \
+; RUN:     %s -o - 2>&1 | FileCheck %s --check-prefixes=NUM,JT
+
+; Section names will optionally have `.<func>` if -function-sections is enabled.
+; RUN: llc -mtriple=x86_64-unknown-linux-gnu -enable-split-machine-functions \
+; RUN:     -partition-static-data-sections=true -function-sections=true \
+; RUN:     -min-jump-table-entries=2  %s -o - 2>&1 | FileCheck %s --check-prefixes=FUNC,JT
+
+; RUN: llc -mtriple=x86_64-unknown-linux-gnu -enable-split-machine-functions \
+; RUN:     -partition-static-data-sections=true -function-sections=false \
+; RUN:     -min-jump-table-entries=2 %s -o - 2>&1 | FileCheck %s --check-prefixes=FUNCLESS,JT
+
+; In function @foo, the 2 switch instructions to jt0.* and jt1.* are placed in
+; hot-prefixed sections, and the 2 switch instructions to jt2.* and jt3.* are
+; placed in cold-prefixed sections.
+; NUM:    .section .rodata.hot.,"a",@progbits,unique,2
+; FUNC:     .section .rodata.hot.foo,"a",@progbits
+; FUNCLESS: .section .rodata.hot.,"a",@progbits
+; JT: .LJTI0_0:
+; JT: .LJTI0_2:
+; NUM:    	.section	.rodata.unlikely.,"a",@progbits,unique,3
+; FUNC:       .section .rodata.unlikely.foo,"a",@progbits
+; FUNCLESS:   .section .rodata.unlikely.,"a",@progbits
+; JT: .LJTI0_1:
+; JT: .LJTI0_3:
+
+; @func_without_profile simulates the functions without profile information
+; (e.g., not instrumented or not profiled), its jump tables are placed in
+; sections without hot or unlikely prefixes.
+; NUM: .section .rodata,"a",@progbits,unique,5
+; FUNC: .section .rodata.func_without_profile,"a",@progbits
+; FUNCLESS: .section .rodata,"a",@progbits
+; JT: .LJTI1_0:
+
+; @bar doesn't have profile information and it has a section prefix.
+; Tests that its jump tables are placed in sections with function prefixes.
+; NUM: .section .rodata.bar_prefix.,"a",@progbits,unique,7
+; FUNC: .section .rodata.bar_prefix.bar
+; FUNCLESS: .section .rodata.bar_prefix.,"a"
+; JT: .LJTI2_0
 
 target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
@@ -31,6 +68,7 @@ target triple = "x86_64-unknown-linux-gnu"
 @default = private constant [8 x i8] c"default\00"
 @jt3 = private constant [4 x i8] c"jt3\00"
 
+; jt0 and jt2 are hot. jt1 and jt3 are cold.
 define i32 @foo(i32 %num) !prof !13 {
 entry:
   %mod3 = sdiv i32 %num, 3
@@ -53,9 +91,9 @@ jt0.default:
 
 jt0.epilog:
   %zero = icmp eq i32 %num, 0
-  br i1 %zero, label %cold, label %hot, !prof !15
+  br i1 %zero, label %hot, label %cold, !prof !17
 
-cold:
+hot:
  %c2 = call i32 @transform(i32 %num)
   switch i32 %c2, label %jt2.default [
     i32 1, label %jt2.bb1
@@ -76,9 +114,9 @@ jt2.default:
 
 jt2.epilog:
   %c2cmp = icmp ne i32 %c2, 0
-  br i1 %c2cmp, label %return, label %jt3.prologue, !prof !16
+  br i1 %c2cmp, label %return, label %jt3.prologue, !prof !18
 
-hot:
+cold:
   %c1 = call i32 @compute(i32 %num)
   switch i32 %c1, label %jt1.default [
     i32 1, label %jt1.bb1
@@ -150,6 +188,29 @@ sw.epilog:
   ret void
 }
 
+define void @bar(i32 %num) !section_prefix !20  {
+entry:
+  switch i32 %num, label %sw.default [
+    i32 1, label %sw.bb
+    i32 2, label %sw.bb1
+  ]
+
+sw.bb:
+  call i32 @puts(ptr @str.10)
+  br label %sw.epilog
+
+sw.bb1:
+  call i32 @puts(ptr @str.9)
+  br label %sw.epilog
+
+sw.default:
+  call i32 @puts(ptr @str.11)
+  br label %sw.epilog
+
+sw.epilog:
+  ret void
+}
+
 declare i32 @puts(ptr)
 declare i32 @printf(ptr, ...)
 declare i32 @compute(i32)
@@ -173,5 +234,9 @@ declare i32 @cleanup(i32)
 !12 = !{i32 999999, i64 1, i32 9}
 !13 = !{!"function_entry_count", i64 100000}
 !14 = !{!"branch_weights", i32 60000, i32 20000, i32 20000}
-!15 = !{!"branch_weights", i32 1, i32 99999}
-!16 = !{!"branch_weights", i32 99998, i32 1}
+!15 = !{!"function_entry_count", i64 1}
+!16 = !{!"branch_weights", i32 1, i32 0, i32 0, i32 0, i32 0, i32 0}
+!17 = !{!"branch_weights", i32 99999, i32 1}
+!18 = !{!"branch_weights", i32 99998, i32 1}
+!19 = !{!"branch_weights", i32 97000, i32 1000, i32 1000, i32 1000}
+!20 = !{!"function_section_prefix", !"bar_prefix"}
diff --git a/llvm/test/CodeGen/X86/kcfi-arity.ll b/llvm/test/CodeGen/X86/kcfi-arity.ll
new file mode 100644
index 0000000000000..68d90adaf2a17
--- /dev/null
+++ b/llvm/test/CodeGen/X86/kcfi-arity.ll
@@ -0,0 +1,205 @@
+; RUN: llc -mtriple=x86_64-unknown-linux-gnu -verify-machineinstrs < %s | FileCheck %s --check-prefix=ASM
+; RUN: llc -mtriple=x86_64-unknown-linux-gnu -verify-machineinstrs -stop-after=finalize-isel < %s | FileCheck %s --check-prefixes=MIR,ISEL
+; RUN: llc -mtriple=x86_64-unknown-linux-gnu -verify-machineinstrs -stop-after=kcfi < %s | FileCheck %s --check-prefixes=MIR,KCFI
+
+; ASM:       .p2align 4
+; ASM:       .type __cfi_f1,@function
+; ASM-LABEL: __cfi_f1:
+; ASM-NEXT:    nop
+; ASM-NEXT:    nop
+; ASM-NEXT:    nop
+; ASM-NEXT:    nop
+; ASM-NEXT:    nop
+; ASM-NEXT:    nop
+; ASM-NEXT:    nop
+; ASM-NEXT:    nop
+; ASM-NEXT:    nop
+; ASM-NEXT:    nop
+; ASM-NEXT:    nop
+; ASM-NEXT:    movl $12345678, %ecx
+; ASM-LABEL: .Lcfi_func_end0:
+; ASM-NEXT:  .size   __cfi_f1, .Lcfi_func_end0-__cfi_f1
+define void @f1(ptr noundef %x) !kcfi_type !1 {
+; ASM-LABEL: f1:
+; ASM:       # %bb.0:
+; ASM:         movl $4282621618, %r10d # imm = 0xFF439EB2
+; ASM-NEXT:    addl -4(%rdi), %r10d
+; ASM-NEXT:    je .Ltmp0
+; ASM-NEXT:  .Ltmp1:
+; ASM-NEXT:    ud2
+; ASM-NEXT:    .section .kcfi_traps,"ao",@progbits,.text
+; ASM-NEXT:  .Ltmp2:
+; ASM-NEXT:    .long .Ltmp1-.Ltmp2
+; ASM-NEXT:    .text
+; ASM-NEXT:  .Ltmp0:
+; ASM-NEXT:    callq *%rdi
+
+; MIR-LABEL: name: f1
+; MIR: body:
+; ISEL: CALL64r %0, csr_64, implicit $rsp, implicit $ssp, implicit-def $rsp, implicit-def $ssp, cfi-type 12345678
+; KCFI:       BUNDLE{{.*}} {
+; KCFI-NEXT:    KCFI_CHECK $rdi, 12345678, implicit-def $r10, implicit-def $r11, implicit-def $eflags
+; KCFI-NEXT:    CALL64r killed $rdi, csr_64, implicit $rsp, implicit $ssp, implicit-def $rsp, implicit-def $ssp
+; KCFI-NEXT:  }
+  call void %x() [ "kcfi"(i32 12345678) ]
+  ret void
+}
+
+; ASM-NOT: __cfi_f2:
+define void @f2(ptr noundef %x) {
+; ASM-LABEL: f2:
+
+; MIR-LABEL: name: f2
+; MIR: body:
+; ISEL: TCRETURNri64 %0, 0, csr_64, implicit $rsp, implicit $ssp, cfi-type 12345678
+; KCFI:       BUNDLE{{.*}} {
+; KCFI-NEXT:    KCFI_CHECK $rdi, 12345678, implicit-def $r10, implicit-def $r11, implicit-def $eflags
+; KCFI-NEXT:    TAILJMPr64 killed $rdi, csr_64, implicit $rsp, implicit $ssp, implicit $rsp, implicit $ssp
+; KCFI-NEXT:  }
+  tail call void %x() [ "kcfi"(i32 12345678) ]
+  ret void
+}
+
+; ASM-NOT: __cfi_f3:
+define void @f3(ptr noundef %x) #0 {
+; ASM-LABEL: f3:
+; MIR-LABEL: name: f3
+; MIR: body:
+; ISEL: CALL64pcrel32 &__llvm_retpoline_r11, csr_64, implicit $rsp, implicit $ssp, implicit-def $rsp, implicit-def $ssp, implicit killed $r11, cfi-type 12345678
+; KCFI:       BUNDLE{{.*}} {
+; KCFI-NEXT:    KCFI_CHECK $r11, 12345678, implicit-def $r10, implicit-def $r11, implicit-def $eflags
+; KCFI-NEXT:    CALL64pcrel32 &__llvm_retpoline_r11, csr_64, implicit $rsp, implicit $ssp, implicit-def $rsp, implicit-def $ssp, implicit internal killed $r11
+; KCFI-NEXT:  }
+  call void %x() [ "kcfi"(i32 12345678) ]
+  ret void
+}
+
+; ASM-NOT: __cfi_f4:
+define void @f4(ptr noundef %x) #0 {
+; ASM-LABEL: f4:
+; MIR-LABEL: name: f4
+; MIR: body:
+; ISEL: TCRETURNdi64 &__llvm_retpoline_r11, 0, csr_64, implicit $rsp, implicit $ssp, implicit killed $r11, cfi-type 12345678
+; KCFI:       BUNDLE{{.*}} {
+; KCFI-NEXT:    KCFI_CHECK $r11, 12345678, implicit-def $r10, implicit-def $r11, implicit-def $eflags
+; KCFI-NEXT:    TAILJMPd64 &__llvm_retpoline_r11, csr_64, implicit $rsp, implicit $ssp, implicit $rsp, implicit $ssp, implicit internal killed $r11
+; KCFI-NEXT:  }
+  tail call void %x() [ "kcfi"(i32 12345678) ]
+  ret void
+}
+
+;; Ensure we emit Value + 1 for unwanted values (e.g. endbr64 == 4196274163).
+; ASM-LABEL: __cfi_f5:
+; ASM: movl $4196274164, %ecx # imm = 0xFA1E0FF4
+define void @f5(ptr noundef %x) !kcfi_type !2 {
+; ASM-LABEL: f5:
+; ASM: movl $98693132, %r10d # imm = 0x5E1F00C
+  tail call void %x() [ "kcfi"(i32 4196274163) ]
+  ret void
+}
+
+;; Ensure we emit Value + 1 for unwanted values (e.g. -endbr64 == 98693133).
+; ASM-LABEL: __cfi_f6:
+; ASM: movl $98693134, %ecx # imm = 0x5E1F00E
+define void @f6(ptr noundef %x) !kcfi_type !3 {
+; ASM-LABEL: f6:
+; ASM: movl $4196274162, %r10d # imm = 0xFA1E0FF2
+  tail call void %x() [ "kcfi"(i32 98693133) ]
+  ret void
+}
+
+@g = external local_unnamed_addr global ptr, align 8
+
+define void @f7() {
+; MIR-LABEL: name: f7
+; MIR: body:
+; ISEL: TCRETURNmi64 killed %0, 1, $noreg, 0, $noreg, 0, csr_64, implicit $rsp, implicit $ssp, cfi-type 12345678
+; KCFI: $r11 = MOV64rm killed renamable $rax, 1, $noreg, 0, $noreg
+; KCFI-NEXT:  BUNDLE{{.*}} {
+; KCFI-NEXT:    KCFI_CHECK $r11, 12345678, implicit-def $r10, implicit-def $r11, implicit-def $eflags
+; KCFI-NEXT:    TAILJMPr64 internal $r11, csr_64, implicit $rsp, implicit $ssp, implicit $rsp, implicit $ssp
+; KCFI-NEXT:  }
+  %1 = load ptr, ptr @g, align 8
+  tail call void %1() [ "kcfi"(i32 12345678) ]
+  ret void
+}
+
+define void @f8() {
+; MIR-LABEL: name: f8
+; MIR: body:
+; ISEL: CALL64m killed %0, 1, $noreg, 0, $noreg, csr_64, implicit $rsp, implicit $ssp, implicit-def $rsp, implicit-def $ssp, cfi-type 12345678
+; KCFI: $r11 = MOV64rm killed renamable $rax, 1, $noreg, 0, $noreg
+; KCFI-NEXT:  BUNDLE{{.*}} {
+; KCFI-NEXT:    KCFI_CHECK $r11, 12345678, implicit-def $r10, implicit-def $r11, implicit-def $eflags
+; KCFI-NEXT:    CALL64r internal $r11, csr_64, implicit $rsp, implicit $ssp, implicit-def $rsp, implicit-def $ssp
+; KCFI-NEXT:  }
+  %1 = load ptr, ptr @g, align 8
+  call void %1() [ "kcfi"(i32 12345678) ]
+  ret void
+}
+
+%struct.S9 = type { [10 x i64] }
+
+;; Ensure that functions with large (e.g., greater than 8 bytes) arguments passed on the stack are assigned arity=7
+; ASM-LABEL: __cfi_f9:
+; ASM: movl	$199571451, %edi                # imm = 0xBE537FB
+define dso_local void @f9(ptr noundef byval(%struct.S9) align 8 %s) !kcfi_type !4  {
+entry:
+  ret void
+}
+
+;; Ensure that functions with fewer than 7 register arguments and no stack arguments are assigned arity<7
+; ASM-LABEL: __cfi_f10:
+; ASM: movl	$1046421190, %esi               # imm = 0x3E5F1EC6
+define dso_local void @f10(i32 noundef %v1, i32 noundef %v2, i32 noundef %v3, i32 noundef %v4, i32 noundef %v5, i32 noundef %v6) #0 !kcfi_type !5 {
+entry:
+  %v1.addr = alloca i32, align 4
+  %v2.addr = alloca i32, align 4
+  %v3.addr = alloca i32, align 4
+  %v4.addr = alloca i32, align 4
+  %v5.addr = alloca i32, align 4
+  %v6.addr = alloca i32, align 4
+  store i32 %v1, ptr %v1.addr, align 4
+  store i32 %v2, ptr %v2.addr, align 4
+  store i32 %v3, ptr %v3.addr, align 4
+  store i32 %v4, ptr %v4.addr, align 4
+  store i32 %v5, ptr %v5.addr, align 4
+  store i32 %v6, ptr %v6.addr, align 4
+  ret void
+}
+
+;; Ensure that functions with greater than 7 register arguments and no stack arguments are assigned arity=7
+; ASM-LABEL: __cfi_f11:
+; ASM: movl	$1342488295, %edi               # imm = 0x5004BEE7
+define dso_local void @f11(i32 noundef %v1, i32 noundef %v2, i32 noundef %v3, i32 noundef %v4, i32 noundef %v5, i32 noundef %v6, i32 noundef %v7, i32 noundef %v8) #0 !kcfi_type !6 {
+entry:
+  %v1.addr = alloca i32, align 4
+  %v2.addr = alloca i32, align 4
+  %v3.addr = alloca i32, align 4
+  %v4.addr = alloca i32, align 4
+  %v5.addr = alloca i32, align 4
+  %v6.addr = alloca i32, align 4
+  %v7.addr = alloca i32, align 4
+  %v8.addr = alloca i32, align 4
+  store i32 %v1, ptr %v1.addr, align 4
+  store i32 %v2, ptr %v2.addr, align 4
+  store i32 %v3, ptr %v3.addr, align 4
+  store i32 %v4, ptr %v4.addr, align 4
+  store i32 %v5, ptr %v5.addr, align 4
+  store i32 %v6, ptr %v6.addr, align 4
+  store i32 %v7, ptr %v7.addr, align 4
+  store i32 %v8, ptr %v8.addr, align 4
+  ret void
+}
+
+attributes #0 = { "target-features"="+retpoline-indirect-branches,+retpoline-indirect-calls" }
+
+!llvm.module.flags = !{!0, !7}
+!0 = !{i32 4, !"kcfi", i32 1}
+!1 = !{i32 12345678}
+!2 = !{i32 4196274163}
+!3 = !{i32 98693133}
+!4 = !{i32 199571451}
+!5 = !{i32 1046421190}
+!6 = !{i32 1342488295}
+!7 = !{i32 4, !"kcfi-arity", i32 1}
diff --git a/llvm/test/CodeGen/X86/late-remat-update.mir b/llvm/test/CodeGen/X86/late-remat-update.mir
index dd4e99c6df141..32123124a924c 100644
--- a/llvm/test/CodeGen/X86/late-remat-update.mir
+++ b/llvm/test/CodeGen/X86/late-remat-update.mir
@@ -1,5 +1,5 @@
 # REQUIRES: asserts
-# RUN: llc -mtriple=x86_64-- -run-pass=register-coalescer -late-remat-update-threshold=1 -stats %s -o /dev/null 2>&1 | FileCheck %s
+# RUN: llc -mtriple=x86_64-- -passes=register-coalescer -late-remat-update-threshold=1 -stats %s -o /dev/null 2>&1 | FileCheck %s
 # Check the test will rematerialize for three copies, but will call shrinkToUses
 # only once to update live range because of late rematerialization update.
 # CHECK: 3 regalloc - Number of instructions re-materialized
diff --git a/llvm/test/CodeGen/X86/ldexp.ll b/llvm/test/CodeGen/X86/ldexp.ll
index 0aa376195627d..3c6e14598571d 100644
--- a/llvm/test/CodeGen/X86/ldexp.ll
+++ b/llvm/test/CodeGen/X86/ldexp.ll
@@ -18,7 +18,9 @@ define float @ldexp_f32(i8 zeroext %x) {
 ; WIN64-NEXT:    movsd {{.*#+}} xmm0 = [1.0E+0,0.0E+0]
 ; WIN64-NEXT:    callq ldexp
 ; WIN64-NEXT:    cvtsd2ss %xmm0, %xmm0
+; WIN64-NEXT:    .seh_startepilogue
 ; WIN64-NEXT:    addq $40, %rsp
+; WIN64-NEXT:    .seh_endepilogue
 ; WIN64-NEXT:    retq
 ; WIN64-NEXT:    .seh_endproc
 ;
@@ -138,8 +140,10 @@ define <2 x float> @ldexp_v2f32(<2 x float> %val, <2 x i32> %exp) {
 ; WIN64-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
 ; WIN64-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm7 # 16-byte Reload
 ; WIN64-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm8 # 16-byte Reload
+; WIN64-NEXT:    .seh_startepilogue
 ; WIN64-NEXT:    addq $80, %rsp
 ; WIN64-NEXT:    popq %rsi
+; WIN64-NEXT:    .seh_endepilogue
 ; WIN64-NEXT:    retq
 ; WIN64-NEXT:    .seh_endproc
 ;
@@ -258,8 +262,10 @@ define <4 x float> @ldexp_v4f32(<4 x float> %val, <4 x i32> %exp) {
 ; WIN64-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
 ; WIN64-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm7 # 16-byte Reload
 ; WIN64-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm8 # 16-byte Reload
+; WIN64-NEXT:    .seh_startepilogue
 ; WIN64-NEXT:    addq $80, %rsp
 ; WIN64-NEXT:    popq %rsi
+; WIN64-NEXT:    .seh_endepilogue
 ; WIN64-NEXT:    retq
 ; WIN64-NEXT:    .seh_endproc
 ;
@@ -367,8 +373,10 @@ define <2 x double> @ldexp_v2f64(<2 x double> %val, <2 x i32> %exp) {
 ; WIN64-NEXT:    movaps %xmm7, %xmm0
 ; WIN64-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
 ; WIN64-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm7 # 16-byte Reload
+; WIN64-NEXT:    .seh_startepilogue
 ; WIN64-NEXT:    addq $64, %rsp
 ; WIN64-NEXT:    popq %rsi
+; WIN64-NEXT:    .seh_endepilogue
 ; WIN64-NEXT:    retq
 ; WIN64-NEXT:    .seh_endproc
 ;
@@ -493,10 +501,12 @@ define <4 x double> @ldexp_v4f64(<4 x double> %val, <4 x i32> %exp) {
 ; WIN64-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
 ; WIN64-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm7 # 16-byte Reload
 ; WIN64-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm8 # 16-byte Reload
+; WIN64-NEXT:    .seh_startepilogue
 ; WIN64-NEXT:    addq $80, %rsp
 ; WIN64-NEXT:    popq %rbx
 ; WIN64-NEXT:    popq %rdi
 ; WIN64-NEXT:    popq %rsi
+; WIN64-NEXT:    .seh_endepilogue
 ; WIN64-NEXT:    retq
 ; WIN64-NEXT:    .seh_endproc
 ;
@@ -584,8 +594,10 @@ define half @ldexp_f16(half %arg0, i32 %arg1) {
 ; WIN64-NEXT:    callq ldexp
 ; WIN64-NEXT:    callq __truncdfhf2
 ; WIN64-NEXT:    nop
+; WIN64-NEXT:    .seh_startepilogue
 ; WIN64-NEXT:    addq $32, %rsp
 ; WIN64-NEXT:    popq %rsi
+; WIN64-NEXT:    .seh_endepilogue
 ; WIN64-NEXT:    retq
 ; WIN64-NEXT:    .seh_endproc
 ;
diff --git a/llvm/test/CodeGen/X86/llround-conv.ll b/llvm/test/CodeGen/X86/llround-conv.ll
index 1ccda2bc27bbb..19a980b72809e 100644
--- a/llvm/test/CodeGen/X86/llround-conv.ll
+++ b/llvm/test/CodeGen/X86/llround-conv.ll
@@ -1,7 +1,9 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc < %s -mtriple=i686-unknown             | FileCheck %s --check-prefix=X86
 ; RUN: llc < %s -mtriple=i686-unknown -mattr=sse2 | FileCheck %s --check-prefix=SSE2
+; RUN: llc < %s -mtriple=i686-linux-gnu -global-isel -global-isel-abort=1 | FileCheck %s --check-prefixes=GISEL-X86
 ; RUN: llc < %s -mtriple=x86_64-unknown | FileCheck %s --check-prefix=X64
+; RUN: llc < %s -mtriple=x86_64-linux-gnu -global-isel -global-isel-abort=1 | FileCheck %s --check-prefixes=GISEL-X64
 
 define i64 @testmsxs(float %x) {
 ; X86-LABEL: testmsxs:
@@ -26,9 +28,29 @@ define i64 @testmsxs(float %x) {
 ; SSE2-NEXT:    .cfi_def_cfa_offset 4
 ; SSE2-NEXT:    retl
 ;
+; GISEL-X86-LABEL: testmsxs:
+; GISEL-X86:       # %bb.0: # %entry
+; GISEL-X86-NEXT:    subl $12, %esp
+; GISEL-X86-NEXT:    .cfi_def_cfa_offset 16
+; GISEL-X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; GISEL-X86-NEXT:    movl %eax, (%esp)
+; GISEL-X86-NEXT:    calll llroundf
+; GISEL-X86-NEXT:    addl $12, %esp
+; GISEL-X86-NEXT:    .cfi_def_cfa_offset 4
+; GISEL-X86-NEXT:    retl
+;
 ; X64-LABEL: testmsxs:
 ; X64:       # %bb.0: # %entry
 ; X64-NEXT:    jmp llroundf@PLT # TAILCALL
+;
+; GISEL-X64-LABEL: testmsxs:
+; GISEL-X64:       # %bb.0: # %entry
+; GISEL-X64-NEXT:    pushq %rax
+; GISEL-X64-NEXT:    .cfi_def_cfa_offset 16
+; GISEL-X64-NEXT:    callq llroundf
+; GISEL-X64-NEXT:    popq %rcx
+; GISEL-X64-NEXT:    .cfi_def_cfa_offset 8
+; GISEL-X64-NEXT:    retq
 entry:
   %0 = tail call i64 @llvm.llround.f32(float %x)
   ret i64 %0
@@ -57,9 +79,34 @@ define i64 @testmsxd(double %x) {
 ; SSE2-NEXT:    .cfi_def_cfa_offset 4
 ; SSE2-NEXT:    retl
 ;
+; GISEL-X86-LABEL: testmsxd:
+; GISEL-X86:       # %bb.0: # %entry
+; GISEL-X86-NEXT:    subl $12, %esp
+; GISEL-X86-NEXT:    .cfi_def_cfa_offset 16
+; GISEL-X86-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; GISEL-X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; GISEL-X86-NEXT:    movl 4(%eax), %eax
+; GISEL-X86-NEXT:    xorl %edx, %edx
+; GISEL-X86-NEXT:    addl %esp, %edx
+; GISEL-X86-NEXT:    movl %ecx, (%esp)
+; GISEL-X86-NEXT:    movl %eax, 4(%edx)
+; GISEL-X86-NEXT:    calll llround
+; GISEL-X86-NEXT:    addl $12, %esp
+; GISEL-X86-NEXT:    .cfi_def_cfa_offset 4
+; GISEL-X86-NEXT:    retl
+;
 ; X64-LABEL: testmsxd:
 ; X64:       # %bb.0: # %entry
 ; X64-NEXT:    jmp llround@PLT # TAILCALL
+;
+; GISEL-X64-LABEL: testmsxd:
+; GISEL-X64:       # %bb.0: # %entry
+; GISEL-X64-NEXT:    pushq %rax
+; GISEL-X64-NEXT:    .cfi_def_cfa_offset 16
+; GISEL-X64-NEXT:    callq llround
+; GISEL-X64-NEXT:    popq %rcx
+; GISEL-X64-NEXT:    .cfi_def_cfa_offset 8
+; GISEL-X64-NEXT:    retq
 entry:
   %0 = tail call i64 @llvm.llround.f64(double %x)
   ret i64 %0
@@ -88,14 +135,165 @@ define i64 @testmsll(x86_fp80 %x) {
 ; SSE2-NEXT:    .cfi_def_cfa_offset 4
 ; SSE2-NEXT:    retl
 ;
+; GISEL-X86-LABEL: testmsll:
+; GISEL-X86:       # %bb.0: # %entry
+; GISEL-X86-NEXT:    subl $12, %esp
+; GISEL-X86-NEXT:    .cfi_def_cfa_offset 16
+; GISEL-X86-NEXT:    fldt {{[0-9]+}}(%esp)
+; GISEL-X86-NEXT:    fstpt (%esp)
+; GISEL-X86-NEXT:    calll llroundl
+; GISEL-X86-NEXT:    addl $12, %esp
+; GISEL-X86-NEXT:    .cfi_def_cfa_offset 4
+; GISEL-X86-NEXT:    retl
+;
 ; X64-LABEL: testmsll:
 ; X64:       # %bb.0: # %entry
 ; X64-NEXT:    jmp llroundl@PLT # TAILCALL
+;
+; GISEL-X64-LABEL: testmsll:
+; GISEL-X64:       # %bb.0: # %entry
+; GISEL-X64-NEXT:    subq $24, %rsp
+; GISEL-X64-NEXT:    .cfi_def_cfa_offset 32
+; GISEL-X64-NEXT:    fldt {{[0-9]+}}(%rsp)
+; GISEL-X64-NEXT:    fstpt (%rsp)
+; GISEL-X64-NEXT:    callq llroundl
+; GISEL-X64-NEXT:    addq $24, %rsp
+; GISEL-X64-NEXT:    .cfi_def_cfa_offset 8
+; GISEL-X64-NEXT:    retq
 entry:
   %0 = tail call i64 @llvm.llround.f80(x86_fp80 %x)
   ret i64 %0
 }
 
-declare i64 @llvm.llround.f32(float) nounwind readnone
-declare i64 @llvm.llround.f64(double) nounwind readnone
-declare i64 @llvm.llround.f80(x86_fp80) nounwind readnone
+define i64 @test_llround_i64_f32(float %x) nounwind {
+; X86-LABEL: test_llround_i64_f32:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl %eax
+; X86-NEXT:    flds {{[0-9]+}}(%esp)
+; X86-NEXT:    fstps (%esp)
+; X86-NEXT:    calll llroundf
+; X86-NEXT:    popl %ecx
+; X86-NEXT:    retl
+;
+; SSE2-LABEL: test_llround_i64_f32:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    pushl %eax
+; SSE2-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; SSE2-NEXT:    movss %xmm0, (%esp)
+; SSE2-NEXT:    calll llroundf
+; SSE2-NEXT:    popl %ecx
+; SSE2-NEXT:    retl
+;
+; GISEL-X86-LABEL: test_llround_i64_f32:
+; GISEL-X86:       # %bb.0:
+; GISEL-X86-NEXT:    subl $12, %esp
+; GISEL-X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; GISEL-X86-NEXT:    movl %eax, (%esp)
+; GISEL-X86-NEXT:    calll llroundf
+; GISEL-X86-NEXT:    addl $12, %esp
+; GISEL-X86-NEXT:    retl
+;
+; X64-LABEL: test_llround_i64_f32:
+; X64:       # %bb.0:
+; X64-NEXT:    jmp llroundf@PLT # TAILCALL
+;
+; GISEL-X64-LABEL: test_llround_i64_f32:
+; GISEL-X64:       # %bb.0:
+; GISEL-X64-NEXT:    pushq %rax
+; GISEL-X64-NEXT:    callq llroundf
+; GISEL-X64-NEXT:    popq %rcx
+; GISEL-X64-NEXT:    retq
+  %conv = call i64 @llvm.llround.i64.f32(float %x)
+  ret i64 %conv
+}
+
+define i64 @test_llround_i64_f64(double %x) nounwind {
+; X86-LABEL: test_llround_i64_f64:
+; X86:       # %bb.0:
+; X86-NEXT:    subl $8, %esp
+; X86-NEXT:    fldl {{[0-9]+}}(%esp)
+; X86-NEXT:    fstpl (%esp)
+; X86-NEXT:    calll llround
+; X86-NEXT:    addl $8, %esp
+; X86-NEXT:    retl
+;
+; SSE2-LABEL: test_llround_i64_f64:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    subl $8, %esp
+; SSE2-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; SSE2-NEXT:    movsd %xmm0, (%esp)
+; SSE2-NEXT:    calll llround
+; SSE2-NEXT:    addl $8, %esp
+; SSE2-NEXT:    retl
+;
+; GISEL-X86-LABEL: test_llround_i64_f64:
+; GISEL-X86:       # %bb.0:
+; GISEL-X86-NEXT:    subl $12, %esp
+; GISEL-X86-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; GISEL-X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; GISEL-X86-NEXT:    movl 4(%eax), %eax
+; GISEL-X86-NEXT:    xorl %edx, %edx
+; GISEL-X86-NEXT:    addl %esp, %edx
+; GISEL-X86-NEXT:    movl %ecx, (%esp)
+; GISEL-X86-NEXT:    movl %eax, 4(%edx)
+; GISEL-X86-NEXT:    calll llround
+; GISEL-X86-NEXT:    addl $12, %esp
+; GISEL-X86-NEXT:    retl
+;
+; X64-LABEL: test_llround_i64_f64:
+; X64:       # %bb.0:
+; X64-NEXT:    jmp llround@PLT # TAILCALL
+;
+; GISEL-X64-LABEL: test_llround_i64_f64:
+; GISEL-X64:       # %bb.0:
+; GISEL-X64-NEXT:    pushq %rax
+; GISEL-X64-NEXT:    callq llround
+; GISEL-X64-NEXT:    popq %rcx
+; GISEL-X64-NEXT:    retq
+  %conv = call i64 @llvm.llround.i64.f64(double %x)
+  ret i64 %conv
+}
+
+define i64 @test_llround_i64_f80(x86_fp80 %x) nounwind {
+; X86-LABEL: test_llround_i64_f80:
+; X86:       # %bb.0:
+; X86-NEXT:    subl $12, %esp
+; X86-NEXT:    fldt {{[0-9]+}}(%esp)
+; X86-NEXT:    fstpt (%esp)
+; X86-NEXT:    calll llroundl
+; X86-NEXT:    addl $12, %esp
+; X86-NEXT:    retl
+;
+; SSE2-LABEL: test_llround_i64_f80:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    subl $12, %esp
+; SSE2-NEXT:    fldt {{[0-9]+}}(%esp)
+; SSE2-NEXT:    fstpt (%esp)
+; SSE2-NEXT:    calll llroundl
+; SSE2-NEXT:    addl $12, %esp
+; SSE2-NEXT:    retl
+;
+; GISEL-X86-LABEL: test_llround_i64_f80:
+; GISEL-X86:       # %bb.0:
+; GISEL-X86-NEXT:    subl $12, %esp
+; GISEL-X86-NEXT:    fldt {{[0-9]+}}(%esp)
+; GISEL-X86-NEXT:    fstpt (%esp)
+; GISEL-X86-NEXT:    calll llroundl
+; GISEL-X86-NEXT:    addl $12, %esp
+; GISEL-X86-NEXT:    retl
+;
+; X64-LABEL: test_llround_i64_f80:
+; X64:       # %bb.0:
+; X64-NEXT:    jmp llroundl@PLT # TAILCALL
+;
+; GISEL-X64-LABEL: test_llround_i64_f80:
+; GISEL-X64:       # %bb.0:
+; GISEL-X64-NEXT:    subq $24, %rsp
+; GISEL-X64-NEXT:    fldt {{[0-9]+}}(%rsp)
+; GISEL-X64-NEXT:    fstpt (%rsp)
+; GISEL-X64-NEXT:    callq llroundl
+; GISEL-X64-NEXT:    addq $24, %rsp
+; GISEL-X64-NEXT:    retq
+  %conv = call i64 @llvm.llround.i64.f80(x86_fp80 %x)
+  ret i64 %conv
+}
diff --git a/llvm/test/CodeGen/X86/localescape.ll b/llvm/test/CodeGen/X86/localescape.ll
index 399e1b38e017d..aee7613273f75 100644
--- a/llvm/test/CodeGen/X86/localescape.ll
+++ b/llvm/test/CodeGen/X86/localescape.ll
@@ -124,7 +124,9 @@ define void @alloc_func_no_frameaddr() {
 ; X64: movl $13, 32(%rsp)
 ; X64: xorl %ecx, %ecx
 ; X64: callq print_framealloc_from_fp
+; X64: .seh_startepilogue
 ; X64: addq $40, %rsp
+; X64: .seh_endepilogue
 ; X64: retq
 
 ; X86-LABEL: alloc_func_no_frameaddr:
diff --git a/llvm/test/CodeGen/X86/lrint-conv-i64.ll b/llvm/test/CodeGen/X86/lrint-conv-i64.ll
index 01b0af2f807f2..38fa09085e189 100644
--- a/llvm/test/CodeGen/X86/lrint-conv-i64.ll
+++ b/llvm/test/CodeGen/X86/lrint-conv-i64.ll
@@ -45,6 +45,24 @@ entry:
   ret i64 %0
 }
 
+define i32 @PR125324(float %x) {
+; SSE-LABEL: PR125324:
+; SSE:       # %bb.0: # %entry
+; SSE-NEXT:    cvtss2si %xmm0, %rax
+; SSE-NEXT:    # kill: def $eax killed $eax killed $rax
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: PR125324:
+; AVX:       # %bb.0: # %entry
+; AVX-NEXT:    vcvtss2si %xmm0, %rax
+; AVX-NEXT:    # kill: def $eax killed $eax killed $rax
+; AVX-NEXT:    retq
+entry:
+  %0 = tail call i64 @llvm.lrint.i64.f32(float %x)
+  %1 = trunc i64 %0 to i32
+  ret i32 %1
+}
+
 declare i64 @llvm.lrint.i64.f32(float) nounwind readnone
 declare i64 @llvm.lrint.i64.f64(double) nounwind readnone
 declare i64 @llvm.lrint.i64.f80(x86_fp80) nounwind readnone
diff --git a/llvm/test/CodeGen/X86/lround-conv-i32.ll b/llvm/test/CodeGen/X86/lround-conv-i32.ll
index 06baf1f273c95..c37536623143d 100644
--- a/llvm/test/CodeGen/X86/lround-conv-i32.ll
+++ b/llvm/test/CodeGen/X86/lround-conv-i32.ll
@@ -1,47 +1,100 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc < %s -mtriple=i686-unknown             | FileCheck %s
 ; RUN: llc < %s -mtriple=i686-unknown -mattr=sse2 | FileCheck %s
+; RUN: llc < %s -mtriple=i686-linux-gnu -global-isel -global-isel-abort=1 | FileCheck %s --check-prefixes=GISEL-X86
 ; RUN: llc < %s -mtriple=x86_64-unknown | FileCheck %s --check-prefix=X64
+; RUN: llc < %s -mtriple=x86_64-linux-gnu -global-isel -global-isel-abort=1 | FileCheck %s --check-prefixes=GISEL-X64
 
-define i32 @testmsws(float %x) {
+define i32 @testmsws(float %x) nounwind {
 ; CHECK-LABEL: testmsws:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    jmp lroundf # TAILCALL
 ;
+; GISEL-X86-LABEL: testmsws:
+; GISEL-X86:       # %bb.0: # %entry
+; GISEL-X86-NEXT:    subl $12, %esp
+; GISEL-X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; GISEL-X86-NEXT:    movl %eax, (%esp)
+; GISEL-X86-NEXT:    calll lroundf
+; GISEL-X86-NEXT:    addl $12, %esp
+; GISEL-X86-NEXT:    retl
+;
 ; X64-LABEL: testmsws:
 ; X64:       # %bb.0: # %entry
 ; X64-NEXT:    jmp lroundf@PLT # TAILCALL
+;
+; GISEL-X64-LABEL: testmsws:
+; GISEL-X64:       # %bb.0: # %entry
+; GISEL-X64-NEXT:    pushq %rax
+; GISEL-X64-NEXT:    callq lroundf
+; GISEL-X64-NEXT:    popq %rcx
+; GISEL-X64-NEXT:    retq
 entry:
   %0 = tail call i32 @llvm.lround.i32.f32(float %x)
   ret i32 %0
 }
 
-define i32 @testmswd(double %x) {
+define i32 @testmswd(double %x) nounwind {
 ; CHECK-LABEL: testmswd:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    jmp lround # TAILCALL
 ;
+; GISEL-X86-LABEL: testmswd:
+; GISEL-X86:       # %bb.0: # %entry
+; GISEL-X86-NEXT:    subl $12, %esp
+; GISEL-X86-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; GISEL-X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; GISEL-X86-NEXT:    movl 4(%eax), %eax
+; GISEL-X86-NEXT:    xorl %edx, %edx
+; GISEL-X86-NEXT:    addl %esp, %edx
+; GISEL-X86-NEXT:    movl %ecx, (%esp)
+; GISEL-X86-NEXT:    movl %eax, 4(%edx)
+; GISEL-X86-NEXT:    calll lround
+; GISEL-X86-NEXT:    addl $12, %esp
+; GISEL-X86-NEXT:    retl
+;
 ; X64-LABEL: testmswd:
 ; X64:       # %bb.0: # %entry
 ; X64-NEXT:    jmp lround@PLT # TAILCALL
+;
+; GISEL-X64-LABEL: testmswd:
+; GISEL-X64:       # %bb.0: # %entry
+; GISEL-X64-NEXT:    pushq %rax
+; GISEL-X64-NEXT:    callq lround
+; GISEL-X64-NEXT:    popq %rcx
+; GISEL-X64-NEXT:    retq
 entry:
   %0 = tail call i32 @llvm.lround.i32.f64(double %x)
   ret i32 %0
 }
 
-define i32 @testmsll(x86_fp80 %x) {
+define i32 @testmsll(x86_fp80 %x) nounwind {
 ; CHECK-LABEL: testmsll:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    jmp lroundl # TAILCALL
 ;
+; GISEL-X86-LABEL: testmsll:
+; GISEL-X86:       # %bb.0: # %entry
+; GISEL-X86-NEXT:    subl $12, %esp
+; GISEL-X86-NEXT:    fldt {{[0-9]+}}(%esp)
+; GISEL-X86-NEXT:    fstpt (%esp)
+; GISEL-X86-NEXT:    calll lroundl
+; GISEL-X86-NEXT:    addl $12, %esp
+; GISEL-X86-NEXT:    retl
+;
 ; X64-LABEL: testmsll:
 ; X64:       # %bb.0: # %entry
 ; X64-NEXT:    jmp lroundl@PLT # TAILCALL
+;
+; GISEL-X64-LABEL: testmsll:
+; GISEL-X64:       # %bb.0: # %entry
+; GISEL-X64-NEXT:    subq $24, %rsp
+; GISEL-X64-NEXT:    fldt {{[0-9]+}}(%rsp)
+; GISEL-X64-NEXT:    fstpt (%rsp)
+; GISEL-X64-NEXT:    callq lroundl
+; GISEL-X64-NEXT:    addq $24, %rsp
+; GISEL-X64-NEXT:    retq
 entry:
   %0 = tail call i32 @llvm.lround.i32.f80(x86_fp80 %x)
   ret i32 %0
 }
-
-declare i32 @llvm.lround.i32.f32(float) nounwind readnone
-declare i32 @llvm.lround.i32.f64(double) nounwind readnone
-declare i32 @llvm.lround.i32.f80(x86_fp80) nounwind readnone
diff --git a/llvm/test/CodeGen/X86/lround-conv-i64.ll b/llvm/test/CodeGen/X86/lround-conv-i64.ll
index 1cfa426736587..36b86f30ca133 100644
--- a/llvm/test/CodeGen/X86/lround-conv-i64.ll
+++ b/llvm/test/CodeGen/X86/lround-conv-i64.ll
@@ -1,33 +1,98 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=i686-linux-gnu -global-isel -global-isel-abort=1 | FileCheck %s --check-prefixes=GISEL-X86
 ; RUN: llc < %s -mtriple=x86_64-unknown | FileCheck %s
+; RUN: llc < %s -mtriple=x86_64-linux-gnu -global-isel -global-isel-abort=1 | FileCheck %s --check-prefixes=GISEL-X64
 
 define i64 @testmsxs(float %x) {
+; GISEL-X86-LABEL: testmsxs:
+; GISEL-X86:       # %bb.0: # %entry
+; GISEL-X86-NEXT:    subl $12, %esp
+; GISEL-X86-NEXT:    .cfi_def_cfa_offset 16
+; GISEL-X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; GISEL-X86-NEXT:    movl %eax, (%esp)
+; GISEL-X86-NEXT:    calll lroundf
+; GISEL-X86-NEXT:    addl $12, %esp
+; GISEL-X86-NEXT:    .cfi_def_cfa_offset 4
+; GISEL-X86-NEXT:    retl
+;
 ; CHECK-LABEL: testmsxs:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    jmp lroundf@PLT # TAILCALL
+;
+; GISEL-X64-LABEL: testmsxs:
+; GISEL-X64:       # %bb.0: # %entry
+; GISEL-X64-NEXT:    pushq %rax
+; GISEL-X64-NEXT:    .cfi_def_cfa_offset 16
+; GISEL-X64-NEXT:    callq lroundf
+; GISEL-X64-NEXT:    popq %rcx
+; GISEL-X64-NEXT:    .cfi_def_cfa_offset 8
+; GISEL-X64-NEXT:    retq
 entry:
   %0 = tail call i64 @llvm.lround.i64.f32(float %x)
   ret i64 %0
 }
 
 define i64 @testmsxd(double %x) {
+; GISEL-X86-LABEL: testmsxd:
+; GISEL-X86:       # %bb.0: # %entry
+; GISEL-X86-NEXT:    subl $12, %esp
+; GISEL-X86-NEXT:    .cfi_def_cfa_offset 16
+; GISEL-X86-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; GISEL-X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; GISEL-X86-NEXT:    movl 4(%eax), %eax
+; GISEL-X86-NEXT:    xorl %edx, %edx
+; GISEL-X86-NEXT:    addl %esp, %edx
+; GISEL-X86-NEXT:    movl %ecx, (%esp)
+; GISEL-X86-NEXT:    movl %eax, 4(%edx)
+; GISEL-X86-NEXT:    calll lround
+; GISEL-X86-NEXT:    addl $12, %esp
+; GISEL-X86-NEXT:    .cfi_def_cfa_offset 4
+; GISEL-X86-NEXT:    retl
+;
 ; CHECK-LABEL: testmsxd:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    jmp lround@PLT # TAILCALL
+;
+; GISEL-X64-LABEL: testmsxd:
+; GISEL-X64:       # %bb.0: # %entry
+; GISEL-X64-NEXT:    pushq %rax
+; GISEL-X64-NEXT:    .cfi_def_cfa_offset 16
+; GISEL-X64-NEXT:    callq lround
+; GISEL-X64-NEXT:    popq %rcx
+; GISEL-X64-NEXT:    .cfi_def_cfa_offset 8
+; GISEL-X64-NEXT:    retq
 entry:
   %0 = tail call i64 @llvm.lround.i64.f64(double %x)
   ret i64 %0
 }
 
 define i64 @testmsll(x86_fp80 %x) {
+; GISEL-X86-LABEL: testmsll:
+; GISEL-X86:       # %bb.0: # %entry
+; GISEL-X86-NEXT:    subl $12, %esp
+; GISEL-X86-NEXT:    .cfi_def_cfa_offset 16
+; GISEL-X86-NEXT:    fldt {{[0-9]+}}(%esp)
+; GISEL-X86-NEXT:    fstpt (%esp)
+; GISEL-X86-NEXT:    calll lroundl
+; GISEL-X86-NEXT:    addl $12, %esp
+; GISEL-X86-NEXT:    .cfi_def_cfa_offset 4
+; GISEL-X86-NEXT:    retl
+;
 ; CHECK-LABEL: testmsll:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    jmp lroundl@PLT # TAILCALL
+;
+; GISEL-X64-LABEL: testmsll:
+; GISEL-X64:       # %bb.0: # %entry
+; GISEL-X64-NEXT:    subq $24, %rsp
+; GISEL-X64-NEXT:    .cfi_def_cfa_offset 32
+; GISEL-X64-NEXT:    fldt {{[0-9]+}}(%rsp)
+; GISEL-X64-NEXT:    fstpt (%rsp)
+; GISEL-X64-NEXT:    callq lroundl
+; GISEL-X64-NEXT:    addq $24, %rsp
+; GISEL-X64-NEXT:    .cfi_def_cfa_offset 8
+; GISEL-X64-NEXT:    retq
 entry:
   %0 = tail call i64 @llvm.lround.i64.f80(x86_fp80 %x)
   ret i64 %0
 }
-
-declare i64 @llvm.lround.i64.f32(float) nounwind readnone
-declare i64 @llvm.lround.i64.f64(double) nounwind readnone
-declare i64 @llvm.lround.i64.f80(x86_fp80) nounwind readnone
diff --git a/llvm/test/CodeGen/X86/matrix-multiply.ll b/llvm/test/CodeGen/X86/matrix-multiply.ll
index bdc1ff4c157e4..a38ca339cd5e1 100644
--- a/llvm/test/CodeGen/X86/matrix-multiply.ll
+++ b/llvm/test/CodeGen/X86/matrix-multiply.ll
@@ -659,57 +659,57 @@ define <9 x double> @test_mul3x3_f64(<9 x double> %a0, <9 x double> %a1) nounwin
 ; AVX2:       # %bb.0: # %entry
 ; AVX2-NEXT:    movq %rdi, %rax
 ; AVX2-NEXT:    vmovsd {{.*#+}} xmm8 = mem[0],zero
-; AVX2-NEXT:    vunpcklpd {{.*#+}} xmm1 = xmm0[0],xmm1[0]
+; AVX2-NEXT:    vunpcklpd {{.*#+}} xmm0 = xmm0[0],xmm1[0]
 ; AVX2-NEXT:    vmovddup {{.*#+}} xmm9 = mem[0,0]
-; AVX2-NEXT:    vmulpd %xmm1, %xmm9, %xmm0
-; AVX2-NEXT:    vunpcklpd {{.*#+}} xmm3 = xmm3[0],xmm4[0]
-; AVX2-NEXT:    vmovddup {{.*#+}} xmm4 = mem[0,0]
-; AVX2-NEXT:    vmulpd %xmm4, %xmm3, %xmm10
-; AVX2-NEXT:    vaddpd %xmm0, %xmm10, %xmm0
+; AVX2-NEXT:    vmulpd %xmm0, %xmm9, %xmm10
+; AVX2-NEXT:    vunpcklpd {{.*#+}} xmm1 = xmm3[0],xmm4[0]
+; AVX2-NEXT:    vmovddup {{.*#+}} xmm3 = mem[0,0]
+; AVX2-NEXT:    vmulpd %xmm3, %xmm1, %xmm4
+; AVX2-NEXT:    vaddpd %xmm4, %xmm10, %xmm4
 ; AVX2-NEXT:    vunpcklpd {{.*#+}} xmm6 = xmm6[0],xmm7[0]
 ; AVX2-NEXT:    vmovddup {{.*#+}} xmm7 = mem[0,0]
 ; AVX2-NEXT:    vmulpd %xmm7, %xmm6, %xmm10
-; AVX2-NEXT:    vaddpd %xmm0, %xmm10, %xmm0
+; AVX2-NEXT:    vaddpd %xmm4, %xmm10, %xmm4
 ; AVX2-NEXT:    vmulsd %xmm2, %xmm9, %xmm9
-; AVX2-NEXT:    vmulsd %xmm4, %xmm5, %xmm4
-; AVX2-NEXT:    vaddsd %xmm4, %xmm9, %xmm4
+; AVX2-NEXT:    vmulsd %xmm3, %xmm5, %xmm3
+; AVX2-NEXT:    vaddsd %xmm3, %xmm9, %xmm3
 ; AVX2-NEXT:    vmulsd %xmm7, %xmm8, %xmm7
-; AVX2-NEXT:    vaddsd %xmm7, %xmm4, %xmm4
-; AVX2-NEXT:    vinsertf128 $1, %xmm4, %ymm0, %ymm4
-; AVX2-NEXT:    vmovddup {{.*#+}} xmm7 = mem[0,0]
-; AVX2-NEXT:    vmulpd %xmm7, %xmm1, %xmm9
+; AVX2-NEXT:    vaddsd %xmm7, %xmm3, %xmm3
+; AVX2-NEXT:    vinsertf128 $1, %xmm3, %ymm4, %ymm3
+; AVX2-NEXT:    vmovddup {{.*#+}} xmm4 = mem[0,0]
+; AVX2-NEXT:    vmulpd %xmm4, %xmm0, %xmm7
+; AVX2-NEXT:    vmovddup {{.*#+}} xmm9 = mem[0,0]
+; AVX2-NEXT:    vmulpd %xmm1, %xmm9, %xmm10
+; AVX2-NEXT:    vaddpd %xmm7, %xmm10, %xmm7
 ; AVX2-NEXT:    vmovddup {{.*#+}} xmm10 = mem[0,0]
-; AVX2-NEXT:    vmulpd %xmm3, %xmm10, %xmm11
-; AVX2-NEXT:    vaddpd %xmm11, %xmm9, %xmm9
-; AVX2-NEXT:    vmovddup {{.*#+}} xmm11 = mem[0,0]
-; AVX2-NEXT:    vmulpd %xmm6, %xmm11, %xmm12
-; AVX2-NEXT:    vaddpd %xmm12, %xmm9, %xmm9
-; AVX2-NEXT:    vmulsd %xmm7, %xmm2, %xmm7
-; AVX2-NEXT:    vmulsd %xmm5, %xmm10, %xmm10
-; AVX2-NEXT:    vaddsd %xmm7, %xmm10, %xmm7
-; AVX2-NEXT:    vmulsd %xmm11, %xmm8, %xmm10
-; AVX2-NEXT:    vaddsd %xmm7, %xmm10, %xmm7
+; AVX2-NEXT:    vmulpd %xmm6, %xmm10, %xmm11
+; AVX2-NEXT:    vaddpd %xmm7, %xmm11, %xmm7
+; AVX2-NEXT:    vmulsd %xmm4, %xmm2, %xmm4
+; AVX2-NEXT:    vmulsd %xmm5, %xmm9, %xmm9
+; AVX2-NEXT:    vaddsd %xmm4, %xmm9, %xmm4
+; AVX2-NEXT:    vmulsd %xmm10, %xmm8, %xmm9
+; AVX2-NEXT:    vaddsd %xmm4, %xmm9, %xmm4
+; AVX2-NEXT:    vmovddup {{.*#+}} xmm9 = mem[0,0]
+; AVX2-NEXT:    vmulpd %xmm0, %xmm9, %xmm0
 ; AVX2-NEXT:    vmovddup {{.*#+}} xmm10 = mem[0,0]
 ; AVX2-NEXT:    vmulpd %xmm1, %xmm10, %xmm1
-; AVX2-NEXT:    vmovddup {{.*#+}} xmm11 = mem[0,0]
-; AVX2-NEXT:    vmulpd %xmm3, %xmm11, %xmm3
-; AVX2-NEXT:    vaddpd %xmm3, %xmm1, %xmm1
-; AVX2-NEXT:    vmovddup {{.*#+}} xmm3 = mem[0,0]
-; AVX2-NEXT:    vmulpd %xmm3, %xmm6, %xmm6
-; AVX2-NEXT:    vaddpd %xmm6, %xmm1, %xmm1
-; AVX2-NEXT:    vmulsd %xmm2, %xmm10, %xmm2
-; AVX2-NEXT:    vmulsd %xmm5, %xmm11, %xmm5
+; AVX2-NEXT:    vaddpd %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vmovddup {{.*#+}} xmm1 = mem[0,0]
+; AVX2-NEXT:    vmulpd %xmm1, %xmm6, %xmm6
+; AVX2-NEXT:    vaddpd %xmm6, %xmm0, %xmm0
+; AVX2-NEXT:    vmulsd %xmm2, %xmm9, %xmm2
+; AVX2-NEXT:    vmulsd %xmm5, %xmm10, %xmm5
 ; AVX2-NEXT:    vaddsd %xmm5, %xmm2, %xmm2
-; AVX2-NEXT:    vmulsd %xmm3, %xmm8, %xmm3
-; AVX2-NEXT:    vaddsd %xmm3, %xmm2, %xmm2
-; AVX2-NEXT:    vinsertf128 $1, %xmm9, %ymm0, %ymm0
-; AVX2-NEXT:    vshufpd {{.*#+}} ymm0 = ymm4[0],ymm0[1],ymm4[2],ymm0[2]
-; AVX2-NEXT:    vinsertf128 $1, %xmm1, %ymm7, %ymm3
-; AVX2-NEXT:    vinsertf128 $1, %xmm1, %ymm9, %ymm1
-; AVX2-NEXT:    vshufpd {{.*#+}} ymm1 = ymm1[1],ymm3[0],ymm1[2],ymm3[3]
-; AVX2-NEXT:    vmovsd %xmm2, 64(%rdi)
-; AVX2-NEXT:    vmovapd %ymm1, 32(%rdi)
-; AVX2-NEXT:    vmovapd %ymm0, (%rdi)
+; AVX2-NEXT:    vmulsd %xmm1, %xmm8, %xmm1
+; AVX2-NEXT:    vaddsd %xmm1, %xmm2, %xmm1
+; AVX2-NEXT:    vbroadcastsd %xmm7, %ymm2
+; AVX2-NEXT:    vblendpd {{.*#+}} ymm2 = ymm3[0,1,2],ymm2[3]
+; AVX2-NEXT:    vinsertf128 $1, %xmm0, %ymm4, %ymm3
+; AVX2-NEXT:    vinsertf128 $1, %xmm0, %ymm7, %ymm0
+; AVX2-NEXT:    vshufpd {{.*#+}} ymm0 = ymm0[1],ymm3[0],ymm0[2],ymm3[3]
+; AVX2-NEXT:    vmovsd %xmm1, 64(%rdi)
+; AVX2-NEXT:    vmovapd %ymm0, 32(%rdi)
+; AVX2-NEXT:    vmovapd %ymm2, (%rdi)
 ; AVX2-NEXT:    vzeroupper
 ; AVX2-NEXT:    retq
 ;
diff --git a/llvm/test/CodeGen/X86/merge-huge-sp-updates.ll b/llvm/test/CodeGen/X86/merge-huge-sp-updates.ll
new file mode 100644
index 0000000000000..b26345e2d5bbc
--- /dev/null
+++ b/llvm/test/CodeGen/X86/merge-huge-sp-updates.ll
@@ -0,0 +1,32 @@
+; RUN: llc < %s -mtriple=x86_64-linux-unknown -verify-machineinstrs -o %t.s
+; RUN: FileCheck --input-file=%t.s %s
+
+; Double-check that we are able to assemble the generated '.s'. A symptom of the
+; problem that led to this test is an assembler failure when using
+; '-save-temps'. For example:
+;  
+; > ...s:683:7: error: invalid operand for instruction
+; >        addq    $2147483679, %rsp               # imm = 0x8000001F
+;
+; RUN: llvm-mc -triple x86_64-unknown-unknown %t.s
+
+; Check that the stack update after calling bar gets merged into the second add
+; and not the first which is already at the chunk size limit (0x7FFFFFFF).
+
+define void @foo(ptr %rhs) {
+; CHECK-LABEL: foo
+entry:
+  %lhs = alloca [5 x [5 x [3 x [162 x [161 x [161 x double]]]]]], align 16
+  store ptr %lhs, ptr %rhs, align 8
+  %0 = call i32 @baz()
+  call void @bar(i64 0, i64 0, i64 0, i64 0, i64 0, ptr null, ptr %rhs, ptr null, ptr %rhs)
+; CHECK: call{{.*}}bar
+; CHECK: addq{{.*}}$2147483647, %rsp
+; CHECK: addq{{.*}}$372037585, %rsp
+; CHECK: .cfi_adjust_cfa_offset -2519521232
+  ret void
+}
+
+declare void @bar(i64, i64, i64, i64, i64, ptr, ptr, ptr, ptr)
+
+declare i32 @baz()
diff --git a/llvm/test/CodeGen/X86/mixed-ptr-sizes.ll b/llvm/test/CodeGen/X86/mixed-ptr-sizes.ll
index 67539b07f5716..48d0ea49b70e6 100644
--- a/llvm/test/CodeGen/X86/mixed-ptr-sizes.ll
+++ b/llvm/test/CodeGen/X86/mixed-ptr-sizes.ll
@@ -1,6 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc < %s | FileCheck %s --check-prefixes=ALL,CHECK
 ; RUN: llc -O0 < %s | FileCheck %s --check-prefixes=ALL,CHECK-O0
+; RUN: llc --fast-isel < %s | FileCheck %s --check-prefixes=ALL,CHECK
+; RUN: llc --global-isel --global-isel-abort=2 < %s | FileCheck %s --check-prefixes=ALL,CHECK
 
 ; Source to regenerate:
 ; struct Foo {
@@ -131,7 +133,9 @@ define dso_local void @test_null_arg(ptr %f) {
 ; ALL-NEXT:    xorl %edx, %edx
 ; ALL-NEXT:    callq test_noop1
 ; ALL-NEXT:    nop
+; ALL-NEXT:    .seh_startepilogue
 ; ALL-NEXT:    addq $40, %rsp
+; ALL-NEXT:    .seh_endepilogue
 ; ALL-NEXT:    retq
 ; ALL-NEXT:    .seh_endproc
 entry:
diff --git a/llvm/test/CodeGen/X86/musttail-varargs.ll b/llvm/test/CodeGen/X86/musttail-varargs.ll
index d3ded0b2a03d8..f9b2ce79a35ca 100644
--- a/llvm/test/CodeGen/X86/musttail-varargs.ll
+++ b/llvm/test/CodeGen/X86/musttail-varargs.ll
@@ -228,11 +228,13 @@ define void @f_thunk(ptr %this, ...) {
 ; WINDOWS-NEXT:    movq %rbx, %rdx
 ; WINDOWS-NEXT:    movq %rdi, %r8
 ; WINDOWS-NEXT:    movq %rsi, %r9
+; WINDOWS-NEXT:    .seh_startepilogue
 ; WINDOWS-NEXT:    addq $72, %rsp
 ; WINDOWS-NEXT:    popq %rbx
 ; WINDOWS-NEXT:    popq %rdi
 ; WINDOWS-NEXT:    popq %rsi
 ; WINDOWS-NEXT:    popq %r14
+; WINDOWS-NEXT:    .seh_endepilogue
 ; WINDOWS-NEXT:    rex64 jmpq *%rax # TAILCALL
 ; WINDOWS-NEXT:    .seh_endproc
 ;
diff --git a/llvm/test/CodeGen/X86/no-plt-libcalls.ll b/llvm/test/CodeGen/X86/no-plt-libcalls.ll
index 8ccbfe967ef52..681df5a993185 100644
--- a/llvm/test/CodeGen/X86/no-plt-libcalls.ll
+++ b/llvm/test/CodeGen/X86/no-plt-libcalls.ll
@@ -15,7 +15,7 @@ define void @printf_call() {
 }
 
 ; CHECK: Function Attrs: nofree nounwind nonlazybind
-; CHECK-NEXT: declare noundef i32 @puts(ptr nocapture noundef readonly)
+; CHECK-NEXT: declare noundef i32 @puts(ptr noundef readonly captures(none))
 
 !llvm.module.flags = !{!0}
 !0 = !{i32 7, !"RtLibUseGOT", i32 1}
diff --git a/llvm/test/CodeGen/X86/no-sse-win64.ll b/llvm/test/CodeGen/X86/no-sse-win64.ll
index e8531dc8b9516..4ee2fc6514879 100644
--- a/llvm/test/CodeGen/X86/no-sse-win64.ll
+++ b/llvm/test/CodeGen/X86/no-sse-win64.ll
@@ -52,7 +52,9 @@ define void @pass_double(ptr %p) {
 ; CHECK-NEXT:    movq (%rcx), %rcx
 ; CHECK-NEXT:    callq take_double
 ; CHECK-NEXT:    nop
+; CHECK-NEXT:    .seh_startepilogue
 ; CHECK-NEXT:    addq $40, %rsp
+; CHECK-NEXT:    .seh_endepilogue
 ; CHECK-NEXT:    retq
 ; CHECK-NEXT:    .seh_endproc
   %v = load double, ptr %p
@@ -69,7 +71,9 @@ define void @pass_float(ptr %p) {
 ; CHECK-NEXT:    movl (%rcx), %ecx
 ; CHECK-NEXT:    callq take_float
 ; CHECK-NEXT:    nop
+; CHECK-NEXT:    .seh_startepilogue
 ; CHECK-NEXT:    addq $40, %rsp
+; CHECK-NEXT:    .seh_endepilogue
 ; CHECK-NEXT:    retq
 ; CHECK-NEXT:    .seh_endproc
   %v = load float, ptr %p
@@ -91,8 +95,10 @@ define void @call_double(ptr %p) {
 ; CHECK-NEXT:    movq %rcx, %rsi
 ; CHECK-NEXT:    callq produce_double
 ; CHECK-NEXT:    movq %rax, (%rsi)
+; CHECK-NEXT:    .seh_startepilogue
 ; CHECK-NEXT:    addq $32, %rsp
 ; CHECK-NEXT:    popq %rsi
+; CHECK-NEXT:    .seh_endepilogue
 ; CHECK-NEXT:    retq
 ; CHECK-NEXT:    .seh_endproc
   %v = call double @produce_double()
@@ -111,8 +117,10 @@ define void @call_float(ptr %p) {
 ; CHECK-NEXT:    movq %rcx, %rsi
 ; CHECK-NEXT:    callq produce_float
 ; CHECK-NEXT:    movl %eax, (%rsi)
+; CHECK-NEXT:    .seh_startepilogue
 ; CHECK-NEXT:    addq $32, %rsp
 ; CHECK-NEXT:    popq %rsi
+; CHECK-NEXT:    .seh_endepilogue
 ; CHECK-NEXT:    retq
 ; CHECK-NEXT:    .seh_endproc
   %v = call float @produce_float()
diff --git a/llvm/test/CodeGen/X86/post-ra-sched-with-debug.mir b/llvm/test/CodeGen/X86/post-ra-sched-with-debug.mir
index 65675ced011fd..bf725930f1f27 100644
--- a/llvm/test/CodeGen/X86/post-ra-sched-with-debug.mir
+++ b/llvm/test/CodeGen/X86/post-ra-sched-with-debug.mir
@@ -1,5 +1,7 @@
 # RUN: llc -mtriple=x86_64-unknown-unknown -mcpu=btver2 -run-pass=post-RA-sched -o - %s | FileCheck %s
+# RUN: llc -mtriple=x86_64-unknown-unknown -mcpu=btver2 -passes=post-RA-sched -o - %s | FileCheck %s
 # RUN: llc -mtriple=x86_64-unknown-unknown -mcpu=btver2 -run-pass=post-RA-sched -o - %s -experimental-debug-variable-locations| FileCheck %s
+# RUN: llc -mtriple=x86_64-unknown-unknown -mcpu=btver2 -passes=post-RA-sched -o - %s -experimental-debug-variable-locations| FileCheck %s
 
 # Test that multiple DBG_VALUE's and DBG_PHIs following an instruction whose
 # register needs # to be changed during the post-RA scheduler pass are updated
diff --git a/llvm/test/CodeGen/X86/pr124255.ll b/llvm/test/CodeGen/X86/pr124255.ll
new file mode 100644
index 0000000000000..da502442baddb
--- /dev/null
+++ b/llvm/test/CodeGen/X86/pr124255.ll
@@ -0,0 +1,47 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc < %s -mtriple=x86_64-- | FileCheck %s
+
+define <4 x i32> @insert_v2i32_in_v4i32_at_0(<4 x i32> %a, <2 x i32> %b) {
+; CHECK-LABEL: insert_v2i32_in_v4i32_at_0:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movsd {{.*#+}} xmm0 = xmm1[0],xmm0[1]
+; CHECK-NEXT:    retq
+  %result = tail call <4 x i32> @llvm.vector.insert.v4i32.v2i32(<4 x i32> %a, <2 x i32> %b, i64 0)
+  ret <4 x i32> %result
+}
+
+define <4 x i32> @insert_v2i32_in_v4i32_at_2(<4 x i32> %a, <2 x i32> %b) {
+; CHECK-LABEL: insert_v2i32_in_v4i32_at_2:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movlhps {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; CHECK-NEXT:    retq
+  %result = tail call <4 x i32> @llvm.vector.insert.v4i32.v2i32(<4 x i32> %a, <2 x i32> %b, i64 2)
+  ret <4 x i32> %result
+}
+
+define <4 x float> @insert_v2f32_in_v4f32_at_0(<4 x float> %a, <2 x float> %b) {
+; CHECK-LABEL: insert_v2f32_in_v4f32_at_0:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movsd {{.*#+}} xmm0 = xmm1[0],xmm0[1]
+; CHECK-NEXT:    retq
+  %result = tail call <4 x float> @llvm.vector.insert.v4float.v2float(<4 x float> %a, <2 x float> %b, i64 0)
+  ret <4 x float> %result
+}
+
+define <8 x i32> @insert_v2i32_in_v8i32_at_0(<8 x i32> %a, <2 x i32> %b) {
+; CHECK-LABEL: insert_v2i32_in_v8i32_at_0:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movsd {{.*#+}} xmm0 = xmm2[0],xmm0[1]
+; CHECK-NEXT:    retq
+  %result = tail call <8 x i32> @llvm.vector.insert.v8i32.v2i32(<8 x i32> %a, <2 x i32> %b, i64 0)
+  ret <8 x i32> %result
+}
+
+define <8 x i32> @insert_v2i32_in_v8i32_at_6(<8 x i32> %a, <2 x i32> %b) {
+; CHECK-LABEL: insert_v2i32_in_v8i32_at_6:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movlhps {{.*#+}} xmm1 = xmm1[0],xmm2[0]
+; CHECK-NEXT:    retq
+  %result = tail call <8 x i32> @llvm.vector.insert.v8i32.v2i32(<8 x i32> %a, <2 x i32> %b, i64 6)
+  ret <8 x i32> %result
+}
diff --git a/llvm/test/CodeGen/X86/pr27681.mir b/llvm/test/CodeGen/X86/pr27681.mir
index e7293fda709cf..d25cb65c3eda5 100644
--- a/llvm/test/CodeGen/X86/pr27681.mir
+++ b/llvm/test/CodeGen/X86/pr27681.mir
@@ -1,4 +1,5 @@
 # RUN: llc -mtriple=i386-unknown-linux-gnu -mcpu=slm -run-pass post-RA-sched -o - %s | FileCheck %s
+# RUN: llc -mtriple=i386-unknown-linux-gnu -mcpu=slm -passes=post-RA-sched -o - %s | FileCheck %s
 #
 # Verify that the critical antidependence breaker does not consider
 # a high byte register as available as a replacement register
diff --git a/llvm/test/CodeGen/X86/pr31088.ll b/llvm/test/CodeGen/X86/pr31088.ll
index 1c9c8e40c009d..5ecb67ba7ffa6 100644
--- a/llvm/test/CodeGen/X86/pr31088.ll
+++ b/llvm/test/CodeGen/X86/pr31088.ll
@@ -45,8 +45,6 @@ define <1 x half> @ir_fadd_v1f16(<1 x half> %arg0, <1 x half> %arg1) nounwind {
 ; F16C-NEXT:    vcvtph2ps %xmm0, %xmm0
 ; F16C-NEXT:    vaddss %xmm1, %xmm0, %xmm0
 ; F16C-NEXT:    vcvtps2ph $4, %xmm0, %xmm0
-; F16C-NEXT:    vmovd %xmm0, %eax
-; F16C-NEXT:    vpinsrw $0, %eax, %xmm0, %xmm0
 ; F16C-NEXT:    retq
 ;
 ; F16C-O0-LABEL: ir_fadd_v1f16:
@@ -55,12 +53,6 @@ define <1 x half> @ir_fadd_v1f16(<1 x half> %arg0, <1 x half> %arg1) nounwind {
 ; F16C-O0-NEXT:    vcvtph2ps %xmm0, %xmm0
 ; F16C-O0-NEXT:    vaddss %xmm1, %xmm0, %xmm0
 ; F16C-O0-NEXT:    vcvtps2ph $4, %xmm0, %xmm0
-; F16C-O0-NEXT:    vmovd %xmm0, %eax
-; F16C-O0-NEXT:    movw %ax, %cx
-; F16C-O0-NEXT:    # implicit-def: $eax
-; F16C-O0-NEXT:    movw %cx, %ax
-; F16C-O0-NEXT:    # implicit-def: $xmm0
-; F16C-O0-NEXT:    vpinsrw $0, %eax, %xmm0, %xmm0
 ; F16C-O0-NEXT:    retq
   %retval = fadd <1 x half> %arg0, %arg1
   ret <1 x half> %retval
diff --git a/llvm/test/CodeGen/X86/pr32284.ll b/llvm/test/CodeGen/X86/pr32284.ll
index 90fb76a23e9a5..8a726a469b61e 100644
--- a/llvm/test/CodeGen/X86/pr32284.ll
+++ b/llvm/test/CodeGen/X86/pr32284.ll
@@ -321,11 +321,9 @@ define void @f2() {
 ; X64-NEXT:    xorl %ecx, %ecx
 ; X64-NEXT:    testl %eax, %eax
 ; X64-NEXT:    sete %cl
-; X64-NEXT:    movl %eax, %edx
-; X64-NEXT:    xorl %ecx, %edx
-; X64-NEXT:    movw %dx, -{{[0-9]+}}(%rsp)
 ; X64-NEXT:    xorl %edx, %edx
-; X64-NEXT:    cmpl %eax, %ecx
+; X64-NEXT:    xorl %eax, %ecx
+; X64-NEXT:    movw %cx, -{{[0-9]+}}(%rsp)
 ; X64-NEXT:    sete %dl
 ; X64-NEXT:    movw %dx, (%rax)
 ; X64-NEXT:    retq
@@ -366,17 +364,15 @@ define void @f2() {
 ; X86:       # %bb.0: # %entry
 ; X86-NEXT:    subl $2, %esp
 ; X86-NEXT:    .cfi_def_cfa_offset 6
-; X86-NEXT:    movzbl var_7, %ecx
+; X86-NEXT:    movzbl var_7, %edx
 ; X86-NEXT:    xorl %eax, %eax
-; X86-NEXT:    testl %ecx, %ecx
+; X86-NEXT:    testl %edx, %edx
 ; X86-NEXT:    sete %al
-; X86-NEXT:    movl %ecx, %edx
-; X86-NEXT:    xorl %eax, %edx
-; X86-NEXT:    movw %dx, (%esp)
-; X86-NEXT:    xorl %edx, %edx
-; X86-NEXT:    cmpl %ecx, %eax
-; X86-NEXT:    sete %dl
-; X86-NEXT:    movw %dx, (%eax)
+; X86-NEXT:    xorl %ecx, %ecx
+; X86-NEXT:    xorl %edx, %eax
+; X86-NEXT:    movw %ax, (%esp)
+; X86-NEXT:    sete %cl
+; X86-NEXT:    movw %cx, (%eax)
 ; X86-NEXT:    addl $2, %esp
 ; X86-NEXT:    .cfi_def_cfa_offset 4
 ; X86-NEXT:    retl
diff --git a/llvm/test/CodeGen/X86/pr41619_reduced.mir b/llvm/test/CodeGen/X86/pr41619_reduced.mir
new file mode 100644
index 0000000000000..5dc2eb60e8141
--- /dev/null
+++ b/llvm/test/CodeGen/X86/pr41619_reduced.mir
@@ -0,0 +1,27 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 5
+# RUN: llc -mtriple=x86_64-- -mattr=+avx2 -run-pass=peephole-opt -o - %s | FileCheck %s
+
+# When trying to coalesce the operand of VMOVSDto64rr, a query would
+# be made with the same register class but the source has a
+# subregister and the result does not.
+---
+name:            uncoalescable_copy_queries_same_regclass_with_only_one_subreg
+tracksRegLiveness: true
+isSSA:           true
+body:             |
+  bb.0:
+    liveins: $rax
+
+    ; CHECK-LABEL: name: uncoalescable_copy_queries_same_regclass_with_only_one_subreg
+    ; CHECK: liveins: $rax
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:gr64 = COPY $rax
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:vr128 = COPY [[COPY]].sub_32bit
+    ; CHECK-NEXT: [[VMOVSDto64rr:%[0-9]+]]:gr64 = VMOVSDto64rr [[COPY1]]
+    ; CHECK-NEXT: RET 0, implicit [[VMOVSDto64rr]]
+    %0:gr64 = COPY $rax
+    %1:vr128 = COPY %0.sub_32bit
+    %2:gr64 = VMOVSDto64rr %1
+    RET 0, implicit %2
+
+...
diff --git a/llvm/test/CodeGen/X86/preserve_nonecc_call_win.ll b/llvm/test/CodeGen/X86/preserve_nonecc_call_win.ll
index 232ac34505782..8f933fbfd0568 100644
--- a/llvm/test/CodeGen/X86/preserve_nonecc_call_win.ll
+++ b/llvm/test/CodeGen/X86/preserve_nonecc_call_win.ll
@@ -12,7 +12,9 @@ define preserve_nonecc void @entry(ptr %r12, ptr %r13, ptr %r14, ptr %r15, ptr %
 ; CHECK-NEXT:    .seh_endprologue
 ; CHECK-NEXT:    callq boring
 ; CHECK-NEXT:    nop
+; CHECK-NEXT:    .seh_startepilogue
 ; CHECK-NEXT:    addq $40, %rsp
+; CHECK-NEXT:    .seh_endepilogue
 ; CHECK-NEXT:    jmp continuation # TAILCALL
 ; CHECK-NEXT:    .seh_endproc
   call void @boring()
diff --git a/llvm/test/CodeGen/X86/segmented-stacks.ll b/llvm/test/CodeGen/X86/segmented-stacks.ll
index 7304fedd28758..f8627ff56a1f9 100644
--- a/llvm/test/CodeGen/X86/segmented-stacks.ll
+++ b/llvm/test/CodeGen/X86/segmented-stacks.ll
@@ -257,7 +257,9 @@ define void @test_basic() #0 {
 ; X64-MinGW-NEXT:    movl $10, %edx
 ; X64-MinGW-NEXT:    callq dummy_use
 ; X64-MinGW-NEXT:    nop
+; X64-MinGW-NEXT:    .seh_startepilogue
 ; X64-MinGW-NEXT:    addq $72, %rsp
+; X64-MinGW-NEXT:    .seh_endepilogue
 ; X64-MinGW-NEXT:    retq
 ; X64-MinGW-NEXT:  .LBB0_1:
 ; X64-MinGW-NEXT:    movl $72, %r10d
@@ -603,8 +605,10 @@ define i32 @test_nested(ptr nest %closure, i32 %other) #0 {
 ; X64-MinGW-NEXT:    movl $10, %edx
 ; X64-MinGW-NEXT:    callq dummy_use
 ; X64-MinGW-NEXT:    movl %esi, %eax
+; X64-MinGW-NEXT:    .seh_startepilogue
 ; X64-MinGW-NEXT:    addq $80, %rsp
 ; X64-MinGW-NEXT:    popq %rsi
+; X64-MinGW-NEXT:    .seh_endepilogue
 ; X64-MinGW-NEXT:    retq
 ; X64-MinGW-NEXT:  .LBB1_1:
 ; X64-MinGW-NEXT:    movq %r10, %rax
@@ -859,7 +863,9 @@ define void @test_large() #0 {
 ; X64-MinGW-NEXT:    movl $3, %edx
 ; X64-MinGW-NEXT:    callq dummy_use
 ; X64-MinGW-NEXT:    nop
+; X64-MinGW-NEXT:    .seh_startepilogue
 ; X64-MinGW-NEXT:    addq $40040, %rsp # imm = 0x9C68
+; X64-MinGW-NEXT:    .seh_endepilogue
 ; X64-MinGW-NEXT:    retq
 ; X64-MinGW-NEXT:  .LBB2_1:
 ; X64-MinGW-NEXT:    movl $40040, %r10d # imm = 0x9C68
@@ -1096,7 +1102,9 @@ define fastcc void @test_fastcc() #0 {
 ; X64-MinGW-NEXT:    movl $10, %edx
 ; X64-MinGW-NEXT:    callq dummy_use
 ; X64-MinGW-NEXT:    nop
+; X64-MinGW-NEXT:    .seh_startepilogue
 ; X64-MinGW-NEXT:    addq $72, %rsp
+; X64-MinGW-NEXT:    .seh_endepilogue
 ; X64-MinGW-NEXT:    retq
 ; X64-MinGW-NEXT:  .LBB3_1:
 ; X64-MinGW-NEXT:    movl $72, %r10d
@@ -1347,7 +1355,9 @@ define fastcc void @test_fastcc_large() #0 {
 ; X64-MinGW-NEXT:    movl $3, %edx
 ; X64-MinGW-NEXT:    callq dummy_use
 ; X64-MinGW-NEXT:    nop
+; X64-MinGW-NEXT:    .seh_startepilogue
 ; X64-MinGW-NEXT:    addq $40040, %rsp # imm = 0x9C68
+; X64-MinGW-NEXT:    .seh_endepilogue
 ; X64-MinGW-NEXT:    retq
 ; X64-MinGW-NEXT:  .LBB4_1:
 ; X64-MinGW-NEXT:    movl $40040, %r10d # imm = 0x9C68
@@ -1602,7 +1612,9 @@ define fastcc void @test_fastcc_large_with_ecx_arg(i32 %a) #0 {
 ; X64-MinGW-NEXT:    leaq {{[0-9]+}}(%rsp), %rcx
 ; X64-MinGW-NEXT:    callq dummy_use
 ; X64-MinGW-NEXT:    nop
+; X64-MinGW-NEXT:    .seh_startepilogue
 ; X64-MinGW-NEXT:    addq $40040, %rsp # imm = 0x9C68
+; X64-MinGW-NEXT:    .seh_endepilogue
 ; X64-MinGW-NEXT:    retq
 ; X64-MinGW-NEXT:  .LBB5_1:
 ; X64-MinGW-NEXT:    movl $40040, %r10d # imm = 0x9C68
@@ -2087,7 +2099,9 @@ define i32 @test_nested_unused(ptr nest %unused) #0 {
 ; X64-MinGW-NEXT:    movl $10, %edx
 ; X64-MinGW-NEXT:    callq dummy_use
 ; X64-MinGW-NEXT:    movl $123, %eax
+; X64-MinGW-NEXT:    .seh_startepilogue
 ; X64-MinGW-NEXT:    addq $72, %rsp
+; X64-MinGW-NEXT:    .seh_endepilogue
 ; X64-MinGW-NEXT:    retq
 ; X64-MinGW-NEXT:  .LBB9_1:
 ; X64-MinGW-NEXT:    movl $72, %r10d
diff --git a/llvm/test/CodeGen/X86/seh-catchpad.ll b/llvm/test/CodeGen/X86/seh-catchpad.ll
index 62b431a830c79..7558c4389be59 100644
--- a/llvm/test/CodeGen/X86/seh-catchpad.ll
+++ b/llvm/test/CodeGen/X86/seh-catchpad.ll
@@ -100,8 +100,10 @@ __except.ret:                                     ; preds = %catch.dispatch.7
 ; CHECK: .Ltmp1:
 ; CHECK: .LBB1_[[epilogue:[0-9]+]]:                                # %__try.cont.12
 ; CHECK:         xorl    %eax, %eax
+; CHECK:         .seh_startepilogue
 ; CHECK:         addq    $32, %rsp
 ; CHECK:         popq    %rbp
+; CHECK:         .seh_endepilogue
 ; CHECK:         retq
 ; CHECK: .LBB1_[[except1bb:[0-9]+]]:                                # %__except
 ; CHECK: .Ltmp2:
diff --git a/llvm/test/CodeGen/X86/select-narrow-int-to-fp.ll b/llvm/test/CodeGen/X86/select-narrow-int-to-fp.ll
index 9bc0dbf5dcb58..28b405799dfd0 100644
--- a/llvm/test/CodeGen/X86/select-narrow-int-to-fp.ll
+++ b/llvm/test/CodeGen/X86/select-narrow-int-to-fp.ll
@@ -64,8 +64,6 @@ define half @uint8ToHalf(i8 %int8) {
 ; CHECK-NO_FP16-NEXT:    movzbl %dil, %eax
 ; CHECK-NO_FP16-NEXT:    vcvtsi2ss %eax, %xmm0, %xmm0
 ; CHECK-NO_FP16-NEXT:    vcvtps2ph $4, %xmm0, %xmm0
-; CHECK-NO_FP16-NEXT:    vmovd %xmm0, %eax
-; CHECK-NO_FP16-NEXT:    vpinsrw $0, %eax, %xmm0, %xmm0
 ; CHECK-NO_FP16-NEXT:    retq
 ;
 ; CHECK-WITH_FP16-LABEL: uint8ToHalf:
@@ -147,8 +145,6 @@ define half @sint8ToHalf(i8 %int8) {
 ; CHECK-NO_FP16-NEXT:    movsbl %dil, %eax
 ; CHECK-NO_FP16-NEXT:    vcvtsi2ss %eax, %xmm0, %xmm0
 ; CHECK-NO_FP16-NEXT:    vcvtps2ph $4, %xmm0, %xmm0
-; CHECK-NO_FP16-NEXT:    vmovd %xmm0, %eax
-; CHECK-NO_FP16-NEXT:    vpinsrw $0, %eax, %xmm0, %xmm0
 ; CHECK-NO_FP16-NEXT:    retq
 ;
 ; CHECK-WITH_FP16-LABEL: sint8ToHalf:
@@ -222,8 +218,6 @@ define half @uint16ToHalf(i16 %int16) {
 ; CHECK-NO_FP16-NEXT:    movzwl %di, %eax
 ; CHECK-NO_FP16-NEXT:    vcvtsi2ss %eax, %xmm0, %xmm0
 ; CHECK-NO_FP16-NEXT:    vcvtps2ph $4, %xmm0, %xmm0
-; CHECK-NO_FP16-NEXT:    vmovd %xmm0, %eax
-; CHECK-NO_FP16-NEXT:    vpinsrw $0, %eax, %xmm0, %xmm0
 ; CHECK-NO_FP16-NEXT:    retq
 ;
 ; CHECK-WITH_FP16-LABEL: uint16ToHalf:
@@ -289,8 +283,6 @@ define half @sint16ToHalf(i16 %int16) {
 ; CHECK-NO_FP16-NEXT:    movswl %di, %eax
 ; CHECK-NO_FP16-NEXT:    vcvtsi2ss %eax, %xmm0, %xmm0
 ; CHECK-NO_FP16-NEXT:    vcvtps2ph $4, %xmm0, %xmm0
-; CHECK-NO_FP16-NEXT:    vmovd %xmm0, %eax
-; CHECK-NO_FP16-NEXT:    vpinsrw $0, %eax, %xmm0, %xmm0
 ; CHECK-NO_FP16-NEXT:    retq
 ;
 ; CHECK-WITH_FP16-LABEL: sint16ToHalf:
diff --git a/llvm/test/CodeGen/X86/setcc-combine.ll b/llvm/test/CodeGen/X86/setcc-combine.ll
index 0745881b2f3a3..f526db00df606 100644
--- a/llvm/test/CodeGen/X86/setcc-combine.ll
+++ b/llvm/test/CodeGen/X86/setcc-combine.ll
@@ -262,8 +262,8 @@ define void @test_i1_uge(ptr%A2) {
 define i64 @PR40657(i8 %var2, i8 %var9) {
 ; CHECK-LABEL: PR40657:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addb %sil, %dil
-; CHECK-NEXT:    incb %dil
+; CHECK-NEXT:    xorl %esi, %edi
+; CHECK-NEXT:    notb %dil
 ; CHECK-NEXT:    movzbl %dil, %eax
 ; CHECK-NEXT:    andl $1, %eax
 ; CHECK-NEXT:    retq
@@ -283,12 +283,7 @@ define i64 @PR40657(i8 %var2, i8 %var9) {
 define i64 @PR40657_commute(i8 %var7, i8 %var8, i8 %var9) {
 ; CHECK-LABEL: PR40657_commute:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    subb %dil, %sil
-; CHECK-NEXT:    subb %sil, %dl
-; CHECK-NEXT:    subb %dl, %sil
-; CHECK-NEXT:    xorb %dl, %sil
-; CHECK-NEXT:    subb %sil, %dl
-; CHECK-NEXT:    movzbl %dl, %eax
+; CHECK-NEXT:    movl %edx, %eax
 ; CHECK-NEXT:    andl $1, %eax
 ; CHECK-NEXT:    retq
   %var4 = trunc i8 %var9 to i1
@@ -468,14 +463,23 @@ define <2 x double> @oge(<2 x double> %x) {
 ; negative test - don't create an fneg to replace 0.0 operand
 
 define double @ogt_no_fneg(double %x, double %y) {
-; CHECK-LABEL: ogt_no_fneg:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    xorpd %xmm2, %xmm2
-; CHECK-NEXT:    cmpltsd %xmm0, %xmm2
-; CHECK-NEXT:    andpd %xmm2, %xmm0
-; CHECK-NEXT:    andnpd %xmm1, %xmm2
-; CHECK-NEXT:    orpd %xmm2, %xmm0
-; CHECK-NEXT:    retq
+; SSE2-LABEL: ogt_no_fneg:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    xorpd %xmm2, %xmm2
+; SSE2-NEXT:    cmpltsd %xmm0, %xmm2
+; SSE2-NEXT:    andpd %xmm2, %xmm0
+; SSE2-NEXT:    andnpd %xmm1, %xmm2
+; SSE2-NEXT:    orpd %xmm2, %xmm0
+; SSE2-NEXT:    retq
+;
+; SSE41-LABEL: ogt_no_fneg:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    movapd %xmm0, %xmm2
+; SSE41-NEXT:    xorpd %xmm0, %xmm0
+; SSE41-NEXT:    cmpltsd %xmm2, %xmm0
+; SSE41-NEXT:    blendvpd %xmm0, %xmm2, %xmm1
+; SSE41-NEXT:    movapd %xmm1, %xmm0
+; SSE41-NEXT:    retq
   %cmp = fcmp ogt double %x, 0.0
   %r = select i1 %cmp, double %x, double %y
   ret double %r
@@ -484,16 +488,27 @@ define double @ogt_no_fneg(double %x, double %y) {
 ; negative test - can't change the setcc for non-zero constant
 
 define double @ogt_no_zero(double %x) {
-; CHECK-LABEL: ogt_no_zero:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    movapd {{.*#+}} xmm1 = [-0.0E+0,-0.0E+0]
-; CHECK-NEXT:    xorpd %xmm0, %xmm1
-; CHECK-NEXT:    movsd {{.*#+}} xmm2 = [1.0E+0,0.0E+0]
-; CHECK-NEXT:    cmpltsd %xmm0, %xmm2
-; CHECK-NEXT:    andpd %xmm2, %xmm0
-; CHECK-NEXT:    andnpd %xmm1, %xmm2
-; CHECK-NEXT:    orpd %xmm2, %xmm0
-; CHECK-NEXT:    retq
+; SSE2-LABEL: ogt_no_zero:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    movapd {{.*#+}} xmm1 = [-0.0E+0,-0.0E+0]
+; SSE2-NEXT:    xorpd %xmm0, %xmm1
+; SSE2-NEXT:    movsd {{.*#+}} xmm2 = [1.0E+0,0.0E+0]
+; SSE2-NEXT:    cmpltsd %xmm0, %xmm2
+; SSE2-NEXT:    andpd %xmm2, %xmm0
+; SSE2-NEXT:    andnpd %xmm1, %xmm2
+; SSE2-NEXT:    orpd %xmm2, %xmm0
+; SSE2-NEXT:    retq
+;
+; SSE41-LABEL: ogt_no_zero:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    movapd %xmm0, %xmm1
+; SSE41-NEXT:    movapd {{.*#+}} xmm2 = [-0.0E+0,-0.0E+0]
+; SSE41-NEXT:    xorpd %xmm0, %xmm2
+; SSE41-NEXT:    movsd {{.*#+}} xmm0 = [1.0E+0,0.0E+0]
+; SSE41-NEXT:    cmpltsd %xmm1, %xmm0
+; SSE41-NEXT:    blendvpd %xmm0, %xmm1, %xmm2
+; SSE41-NEXT:    movapd %xmm2, %xmm0
+; SSE41-NEXT:    retq
   %neg = fneg double %x
   %cmp = fcmp ogt double %x, 1.0
   %r = select i1 %cmp, double %x, double %neg
diff --git a/llvm/test/CodeGen/X86/smax.ll b/llvm/test/CodeGen/X86/smax.ll
index 2d59422953eb3..86891e964d96d 100644
--- a/llvm/test/CodeGen/X86/smax.ll
+++ b/llvm/test/CodeGen/X86/smax.ll
@@ -642,11 +642,11 @@ define i16 @test_signbits_i16(i16 %a, i16 %b) nounwind {
 ;
 ; X86-LABEL: test_signbits_i16:
 ; X86:       # %bb.0:
-; X86-NEXT:    movsbl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT:    movswl {{[0-9]+}}(%esp), %eax
-; X86-NEXT:    shrl $15, %eax
-; X86-NEXT:    cmpw %cx, %ax
-; X86-NEXT:    cmovlel %ecx, %eax
+; X86-NEXT:    movsbl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movswl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    shrl $15, %ecx
+; X86-NEXT:    cmpw %ax, %cx
+; X86-NEXT:    cmovgl %ecx, %eax
 ; X86-NEXT:    # kill: def $ax killed $ax killed $eax
 ; X86-NEXT:    retl
   %ax = ashr i16 %a, 15
diff --git a/llvm/test/CodeGen/X86/smin.ll b/llvm/test/CodeGen/X86/smin.ll
index bde61d5738ed5..8907f6c4cd598 100644
--- a/llvm/test/CodeGen/X86/smin.ll
+++ b/llvm/test/CodeGen/X86/smin.ll
@@ -643,11 +643,11 @@ define i16 @test_signbits_i16(i16 %a, i16 %b) nounwind {
 ;
 ; X86-LABEL: test_signbits_i16:
 ; X86:       # %bb.0:
-; X86-NEXT:    movsbl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT:    movswl {{[0-9]+}}(%esp), %eax
-; X86-NEXT:    shrl $15, %eax
-; X86-NEXT:    cmpw %cx, %ax
-; X86-NEXT:    cmovgel %ecx, %eax
+; X86-NEXT:    movsbl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movswl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    shrl $15, %ecx
+; X86-NEXT:    cmpw %ax, %cx
+; X86-NEXT:    cmovll %ecx, %eax
 ; X86-NEXT:    # kill: def $ax killed $ax killed $eax
 ; X86-NEXT:    retl
   %ax = ashr i16 %a, 15
diff --git a/llvm/test/CodeGen/X86/sse-minmax.ll b/llvm/test/CodeGen/X86/sse-minmax.ll
index 1c14b7400a358..7904b21a3b1fa 100644
--- a/llvm/test/CodeGen/X86/sse-minmax.ll
+++ b/llvm/test/CodeGen/X86/sse-minmax.ll
@@ -80,11 +80,11 @@ define double @olt_inverse(double %x, double %y)  {
 define double @oge(double %x, double %y)  {
 ; STRICT-LABEL: oge:
 ; STRICT:       # %bb.0:
-; STRICT-NEXT:    movapd %xmm1, %xmm2
-; STRICT-NEXT:    cmplesd %xmm0, %xmm2
-; STRICT-NEXT:    andpd %xmm2, %xmm0
-; STRICT-NEXT:    andnpd %xmm1, %xmm2
-; STRICT-NEXT:    orpd %xmm2, %xmm0
+; STRICT-NEXT:    movapd %xmm0, %xmm2
+; STRICT-NEXT:    movapd %xmm1, %xmm0
+; STRICT-NEXT:    cmplesd %xmm2, %xmm0
+; STRICT-NEXT:    blendvpd %xmm0, %xmm2, %xmm1
+; STRICT-NEXT:    movapd %xmm1, %xmm0
 ; STRICT-NEXT:    retq
 ;
 ; RELAX-LABEL: oge:
@@ -100,10 +100,9 @@ define double @ole(double %x, double %y)  {
 ; STRICT-LABEL: ole:
 ; STRICT:       # %bb.0:
 ; STRICT-NEXT:    movapd %xmm0, %xmm2
-; STRICT-NEXT:    cmplesd %xmm1, %xmm2
-; STRICT-NEXT:    andpd %xmm2, %xmm0
-; STRICT-NEXT:    andnpd %xmm1, %xmm2
-; STRICT-NEXT:    orpd %xmm2, %xmm0
+; STRICT-NEXT:    cmplesd %xmm1, %xmm0
+; STRICT-NEXT:    blendvpd %xmm0, %xmm2, %xmm1
+; STRICT-NEXT:    movapd %xmm1, %xmm0
 ; STRICT-NEXT:    retq
 ;
 ; RELAX-LABEL: ole:
@@ -118,11 +117,10 @@ define double @ole(double %x, double %y)  {
 define double @oge_inverse(double %x, double %y)  {
 ; STRICT-LABEL: oge_inverse:
 ; STRICT:       # %bb.0:
-; STRICT-NEXT:    movapd %xmm1, %xmm2
-; STRICT-NEXT:    cmplesd %xmm0, %xmm2
-; STRICT-NEXT:    andpd %xmm2, %xmm1
-; STRICT-NEXT:    andnpd %xmm0, %xmm2
-; STRICT-NEXT:    orpd %xmm1, %xmm2
+; STRICT-NEXT:    movapd %xmm0, %xmm2
+; STRICT-NEXT:    movapd %xmm1, %xmm0
+; STRICT-NEXT:    cmplesd %xmm2, %xmm0
+; STRICT-NEXT:    blendvpd %xmm0, %xmm1, %xmm2
 ; STRICT-NEXT:    movapd %xmm2, %xmm0
 ; STRICT-NEXT:    retq
 ;
@@ -145,10 +143,8 @@ define double @ole_inverse(double %x, double %y)  {
 ; STRICT-LABEL: ole_inverse:
 ; STRICT:       # %bb.0:
 ; STRICT-NEXT:    movapd %xmm0, %xmm2
-; STRICT-NEXT:    cmplesd %xmm1, %xmm2
-; STRICT-NEXT:    andpd %xmm2, %xmm1
-; STRICT-NEXT:    andnpd %xmm0, %xmm2
-; STRICT-NEXT:    orpd %xmm1, %xmm2
+; STRICT-NEXT:    cmplesd %xmm1, %xmm0
+; STRICT-NEXT:    blendvpd %xmm0, %xmm1, %xmm2
 ; STRICT-NEXT:    movapd %xmm2, %xmm0
 ; STRICT-NEXT:    retq
 ;
@@ -333,10 +329,9 @@ define double @ugt(double %x, double %y)  {
 ; STRICT-LABEL: ugt:
 ; STRICT:       # %bb.0:
 ; STRICT-NEXT:    movapd %xmm0, %xmm2
-; STRICT-NEXT:    cmpnlesd %xmm1, %xmm2
-; STRICT-NEXT:    andpd %xmm2, %xmm0
-; STRICT-NEXT:    andnpd %xmm1, %xmm2
-; STRICT-NEXT:    orpd %xmm2, %xmm0
+; STRICT-NEXT:    cmpnlesd %xmm1, %xmm0
+; STRICT-NEXT:    blendvpd %xmm0, %xmm2, %xmm1
+; STRICT-NEXT:    movapd %xmm1, %xmm0
 ; STRICT-NEXT:    retq
 ;
 ; RELAX-LABEL: ugt:
@@ -351,11 +346,11 @@ define double @ugt(double %x, double %y)  {
 define double @ult(double %x, double %y)  {
 ; STRICT-LABEL: ult:
 ; STRICT:       # %bb.0:
-; STRICT-NEXT:    movapd %xmm1, %xmm2
-; STRICT-NEXT:    cmpnlesd %xmm0, %xmm2
-; STRICT-NEXT:    andpd %xmm2, %xmm0
-; STRICT-NEXT:    andnpd %xmm1, %xmm2
-; STRICT-NEXT:    orpd %xmm2, %xmm0
+; STRICT-NEXT:    movapd %xmm0, %xmm2
+; STRICT-NEXT:    movapd %xmm1, %xmm0
+; STRICT-NEXT:    cmpnlesd %xmm2, %xmm0
+; STRICT-NEXT:    blendvpd %xmm0, %xmm2, %xmm1
+; STRICT-NEXT:    movapd %xmm1, %xmm0
 ; STRICT-NEXT:    retq
 ;
 ; RELAX-LABEL: ult:
@@ -371,10 +366,8 @@ define double @ugt_inverse(double %x, double %y)  {
 ; STRICT-LABEL: ugt_inverse:
 ; STRICT:       # %bb.0:
 ; STRICT-NEXT:    movapd %xmm0, %xmm2
-; STRICT-NEXT:    cmpnlesd %xmm1, %xmm2
-; STRICT-NEXT:    andpd %xmm2, %xmm1
-; STRICT-NEXT:    andnpd %xmm0, %xmm2
-; STRICT-NEXT:    orpd %xmm1, %xmm2
+; STRICT-NEXT:    cmpnlesd %xmm1, %xmm0
+; STRICT-NEXT:    blendvpd %xmm0, %xmm1, %xmm2
 ; STRICT-NEXT:    movapd %xmm2, %xmm0
 ; STRICT-NEXT:    retq
 ;
@@ -396,11 +389,10 @@ define double @ugt_inverse(double %x, double %y)  {
 define double @ult_inverse(double %x, double %y)  {
 ; STRICT-LABEL: ult_inverse:
 ; STRICT:       # %bb.0:
-; STRICT-NEXT:    movapd %xmm1, %xmm2
-; STRICT-NEXT:    cmpnlesd %xmm0, %xmm2
-; STRICT-NEXT:    andpd %xmm2, %xmm1
-; STRICT-NEXT:    andnpd %xmm0, %xmm2
-; STRICT-NEXT:    orpd %xmm1, %xmm2
+; STRICT-NEXT:    movapd %xmm0, %xmm2
+; STRICT-NEXT:    movapd %xmm1, %xmm0
+; STRICT-NEXT:    cmpnlesd %xmm2, %xmm0
+; STRICT-NEXT:    blendvpd %xmm0, %xmm1, %xmm2
 ; STRICT-NEXT:    movapd %xmm2, %xmm0
 ; STRICT-NEXT:    retq
 ;
@@ -738,12 +730,12 @@ define double @olt_inverse_y(double %x)  {
 define double @oge_y(double %x)  {
 ; STRICT-LABEL: oge_y:
 ; STRICT:       # %bb.0:
-; STRICT-NEXT:    movsd {{.*#+}} xmm1 = [-0.0E+0,0.0E+0]
-; STRICT-NEXT:    movapd %xmm1, %xmm2
-; STRICT-NEXT:    cmplesd %xmm0, %xmm2
-; STRICT-NEXT:    andpd %xmm2, %xmm0
-; STRICT-NEXT:    andnpd %xmm1, %xmm2
-; STRICT-NEXT:    orpd %xmm2, %xmm0
+; STRICT-NEXT:    movapd %xmm0, %xmm1
+; STRICT-NEXT:    movsd {{.*#+}} xmm0 = [-0.0E+0,0.0E+0]
+; STRICT-NEXT:    cmplesd %xmm1, %xmm0
+; STRICT-NEXT:    movapd {{.*#+}} xmm2 = [-0.0E+0,-0.0E+0]
+; STRICT-NEXT:    blendvpd %xmm0, %xmm1, %xmm2
+; STRICT-NEXT:    movapd %xmm2, %xmm0
 ; STRICT-NEXT:    retq
 ;
 ; RELAX-LABEL: oge_y:
@@ -758,12 +750,11 @@ define double @oge_y(double %x)  {
 define double @ole_y(double %x)  {
 ; STRICT-LABEL: ole_y:
 ; STRICT:       # %bb.0:
-; STRICT-NEXT:    movsd {{.*#+}} xmm1 = [-0.0E+0,0.0E+0]
-; STRICT-NEXT:    movapd %xmm0, %xmm2
-; STRICT-NEXT:    cmplesd %xmm1, %xmm2
-; STRICT-NEXT:    andpd %xmm2, %xmm0
-; STRICT-NEXT:    andnpd %xmm1, %xmm2
-; STRICT-NEXT:    orpd %xmm2, %xmm0
+; STRICT-NEXT:    movapd %xmm0, %xmm1
+; STRICT-NEXT:    cmplesd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; STRICT-NEXT:    movapd {{.*#+}} xmm2 = [-0.0E+0,-0.0E+0]
+; STRICT-NEXT:    blendvpd %xmm0, %xmm1, %xmm2
+; STRICT-NEXT:    movapd %xmm2, %xmm0
 ; STRICT-NEXT:    retq
 ;
 ; RELAX-LABEL: ole_y:
@@ -778,12 +769,10 @@ define double @ole_y(double %x)  {
 define double @oge_inverse_y(double %x)  {
 ; STRICT-LABEL: oge_inverse_y:
 ; STRICT:       # %bb.0:
-; STRICT-NEXT:    movsd {{.*#+}} xmm2 = [-0.0E+0,0.0E+0]
-; STRICT-NEXT:    movapd %xmm2, %xmm1
-; STRICT-NEXT:    cmplesd %xmm0, %xmm1
-; STRICT-NEXT:    andpd %xmm1, %xmm2
-; STRICT-NEXT:    andnpd %xmm0, %xmm1
-; STRICT-NEXT:    orpd %xmm2, %xmm1
+; STRICT-NEXT:    movapd %xmm0, %xmm1
+; STRICT-NEXT:    movsd {{.*#+}} xmm0 = [-0.0E+0,0.0E+0]
+; STRICT-NEXT:    cmplesd %xmm1, %xmm0
+; STRICT-NEXT:    blendvpd %xmm0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; STRICT-NEXT:    movapd %xmm1, %xmm0
 ; STRICT-NEXT:    retq
 ;
@@ -806,12 +795,9 @@ define double @oge_inverse_y(double %x)  {
 define double @ole_inverse_y(double %x)  {
 ; STRICT-LABEL: ole_inverse_y:
 ; STRICT:       # %bb.0:
-; STRICT-NEXT:    movsd {{.*#+}} xmm2 = [-0.0E+0,0.0E+0]
 ; STRICT-NEXT:    movapd %xmm0, %xmm1
-; STRICT-NEXT:    cmplesd %xmm2, %xmm1
-; STRICT-NEXT:    andpd %xmm1, %xmm2
-; STRICT-NEXT:    andnpd %xmm0, %xmm1
-; STRICT-NEXT:    orpd %xmm2, %xmm1
+; STRICT-NEXT:    cmplesd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; STRICT-NEXT:    blendvpd %xmm0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; STRICT-NEXT:    movapd %xmm1, %xmm0
 ; STRICT-NEXT:    retq
 ;
@@ -834,12 +820,11 @@ define double @ole_inverse_y(double %x)  {
 define double @ugt_y(double %x)  {
 ; STRICT-LABEL: ugt_y:
 ; STRICT:       # %bb.0:
-; STRICT-NEXT:    movsd {{.*#+}} xmm1 = [-0.0E+0,0.0E+0]
-; STRICT-NEXT:    movapd %xmm0, %xmm2
-; STRICT-NEXT:    cmpnlesd %xmm1, %xmm2
-; STRICT-NEXT:    andpd %xmm2, %xmm0
-; STRICT-NEXT:    andnpd %xmm1, %xmm2
-; STRICT-NEXT:    orpd %xmm2, %xmm0
+; STRICT-NEXT:    movapd %xmm0, %xmm1
+; STRICT-NEXT:    cmpnlesd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; STRICT-NEXT:    movapd {{.*#+}} xmm2 = [-0.0E+0,-0.0E+0]
+; STRICT-NEXT:    blendvpd %xmm0, %xmm1, %xmm2
+; STRICT-NEXT:    movapd %xmm2, %xmm0
 ; STRICT-NEXT:    retq
 ;
 ; RELAX-LABEL: ugt_y:
@@ -854,12 +839,12 @@ define double @ugt_y(double %x)  {
 define double @ult_y(double %x)  {
 ; STRICT-LABEL: ult_y:
 ; STRICT:       # %bb.0:
-; STRICT-NEXT:    movsd {{.*#+}} xmm1 = [-0.0E+0,0.0E+0]
-; STRICT-NEXT:    movapd %xmm1, %xmm2
-; STRICT-NEXT:    cmpnlesd %xmm0, %xmm2
-; STRICT-NEXT:    andpd %xmm2, %xmm0
-; STRICT-NEXT:    andnpd %xmm1, %xmm2
-; STRICT-NEXT:    orpd %xmm2, %xmm0
+; STRICT-NEXT:    movapd %xmm0, %xmm1
+; STRICT-NEXT:    movsd {{.*#+}} xmm0 = [-0.0E+0,0.0E+0]
+; STRICT-NEXT:    cmpnlesd %xmm1, %xmm0
+; STRICT-NEXT:    movapd {{.*#+}} xmm2 = [-0.0E+0,-0.0E+0]
+; STRICT-NEXT:    blendvpd %xmm0, %xmm1, %xmm2
+; STRICT-NEXT:    movapd %xmm2, %xmm0
 ; STRICT-NEXT:    retq
 ;
 ; RELAX-LABEL: ult_y:
@@ -874,12 +859,9 @@ define double @ult_y(double %x)  {
 define double @ugt_inverse_y(double %x)  {
 ; STRICT-LABEL: ugt_inverse_y:
 ; STRICT:       # %bb.0:
-; STRICT-NEXT:    movsd {{.*#+}} xmm2 = [-0.0E+0,0.0E+0]
 ; STRICT-NEXT:    movapd %xmm0, %xmm1
-; STRICT-NEXT:    cmpnlesd %xmm2, %xmm1
-; STRICT-NEXT:    andpd %xmm1, %xmm2
-; STRICT-NEXT:    andnpd %xmm0, %xmm1
-; STRICT-NEXT:    orpd %xmm2, %xmm1
+; STRICT-NEXT:    cmpnlesd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; STRICT-NEXT:    blendvpd %xmm0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; STRICT-NEXT:    movapd %xmm1, %xmm0
 ; STRICT-NEXT:    retq
 ;
@@ -902,12 +884,10 @@ define double @ugt_inverse_y(double %x)  {
 define double @ult_inverse_y(double %x)  {
 ; STRICT-LABEL: ult_inverse_y:
 ; STRICT:       # %bb.0:
-; STRICT-NEXT:    movsd {{.*#+}} xmm2 = [-0.0E+0,0.0E+0]
-; STRICT-NEXT:    movapd %xmm2, %xmm1
-; STRICT-NEXT:    cmpnlesd %xmm0, %xmm1
-; STRICT-NEXT:    andpd %xmm1, %xmm2
-; STRICT-NEXT:    andnpd %xmm0, %xmm1
-; STRICT-NEXT:    orpd %xmm2, %xmm1
+; STRICT-NEXT:    movapd %xmm0, %xmm1
+; STRICT-NEXT:    movsd {{.*#+}} xmm0 = [-0.0E+0,0.0E+0]
+; STRICT-NEXT:    cmpnlesd %xmm1, %xmm0
+; STRICT-NEXT:    blendvpd %xmm0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; STRICT-NEXT:    movapd %xmm1, %xmm0
 ; STRICT-NEXT:    retq
 ;
diff --git a/llvm/test/CodeGen/X86/sse-regcall.ll b/llvm/test/CodeGen/X86/sse-regcall.ll
index 6f0293392eef2..03b9e123eea48 100644
--- a/llvm/test/CodeGen/X86/sse-regcall.ll
+++ b/llvm/test/CodeGen/X86/sse-regcall.ll
@@ -7,19 +7,19 @@
 define x86_regcallcc i1 @test_argReti1(i1 %a)  {
 ; WIN32-LABEL: test_argReti1:
 ; WIN32:       # %bb.0:
-; WIN32-NEXT:    incb %al
+; WIN32-NEXT:    xorb $1, %al
 ; WIN32-NEXT:    # kill: def $al killed $al killed $eax
 ; WIN32-NEXT:    retl
 ;
 ; WIN64-LABEL: test_argReti1:
 ; WIN64:       # %bb.0:
-; WIN64-NEXT:    incb %al
+; WIN64-NEXT:    xorb $1, %al
 ; WIN64-NEXT:    # kill: def $al killed $al killed $eax
 ; WIN64-NEXT:    retq
 ;
 ; LINUXOSX-LABEL: test_argReti1:
 ; LINUXOSX:       # %bb.0:
-; LINUXOSX-NEXT:    incb %al
+; LINUXOSX-NEXT:    xorb $1, %al
 ; LINUXOSX-NEXT:    # kill: def $al killed $al killed $eax
 ; LINUXOSX-NEXT:    retq
   %add = add i1 %a, 1
@@ -30,10 +30,10 @@ define x86_regcallcc i1 @test_argReti1(i1 %a)  {
 define x86_regcallcc i1 @test_CallargReti1(i1 %a)  {
 ; WIN32-LABEL: test_CallargReti1:
 ; WIN32:       # %bb.0:
-; WIN32-NEXT:    incb %al
+; WIN32-NEXT:    xorb $1, %al
 ; WIN32-NEXT:    movzbl %al, %eax
 ; WIN32-NEXT:    calll _test_argReti1
-; WIN32-NEXT:    incb %al
+; WIN32-NEXT:    xorb $1, %al
 ; WIN32-NEXT:    retl
 ;
 ; WIN64-LABEL: test_CallargReti1:
@@ -41,11 +41,13 @@ define x86_regcallcc i1 @test_CallargReti1(i1 %a)  {
 ; WIN64-NEXT:    pushq %rax
 ; WIN64-NEXT:    .seh_stackalloc 8
 ; WIN64-NEXT:    .seh_endprologue
-; WIN64-NEXT:    incb %al
+; WIN64-NEXT:    xorb $1, %al
 ; WIN64-NEXT:    movzbl %al, %eax
 ; WIN64-NEXT:    callq test_argReti1
-; WIN64-NEXT:    incb %al
+; WIN64-NEXT:    xorb $1, %al
+; WIN64-NEXT:    .seh_startepilogue
 ; WIN64-NEXT:    popq %rcx
+; WIN64-NEXT:    .seh_endepilogue
 ; WIN64-NEXT:    retq
 ; WIN64-NEXT:    .seh_endproc
 ;
@@ -53,10 +55,10 @@ define x86_regcallcc i1 @test_CallargReti1(i1 %a)  {
 ; LINUXOSX:       # %bb.0:
 ; LINUXOSX-NEXT:    pushq %rax
 ; LINUXOSX-NEXT:    .cfi_def_cfa_offset 16
-; LINUXOSX-NEXT:    incb %al
+; LINUXOSX-NEXT:    xorb $1, %al
 ; LINUXOSX-NEXT:    movzbl %al, %eax
 ; LINUXOSX-NEXT:    callq *test_argReti1@GOTPCREL(%rip)
-; LINUXOSX-NEXT:    incb %al
+; LINUXOSX-NEXT:    xorb $1, %al
 ; LINUXOSX-NEXT:    popq %rcx
 ; LINUXOSX-NEXT:    .cfi_def_cfa_offset 8
 ; LINUXOSX-NEXT:    retq
diff --git a/llvm/test/CodeGen/X86/sse-regcall4.ll b/llvm/test/CodeGen/X86/sse-regcall4.ll
index c8df7a233d7e3..6f964f0a88ea3 100644
--- a/llvm/test/CodeGen/X86/sse-regcall4.ll
+++ b/llvm/test/CodeGen/X86/sse-regcall4.ll
@@ -7,19 +7,19 @@
 define x86_regcallcc i1 @test_argReti1(i1 %a)  {
 ; WIN32-LABEL: test_argReti1:
 ; WIN32:       # %bb.0:
-; WIN32-NEXT:    incb %cl
+; WIN32-NEXT:    xorb $1, %cl
 ; WIN32-NEXT:    # kill: def $cl killed $cl killed $ecx
 ; WIN32-NEXT:    retl
 ;
 ; WIN64-LABEL: test_argReti1:
 ; WIN64:       # %bb.0:
-; WIN64-NEXT:    incb %al
+; WIN64-NEXT:    xorb $1, %al
 ; WIN64-NEXT:    # kill: def $al killed $al killed $eax
 ; WIN64-NEXT:    retq
 ;
 ; LINUXOSX-LABEL: test_argReti1:
 ; LINUXOSX:       # %bb.0:
-; LINUXOSX-NEXT:    incb %al
+; LINUXOSX-NEXT:    xorb $1, %al
 ; LINUXOSX-NEXT:    # kill: def $al killed $al killed $eax
 ; LINUXOSX-NEXT:    retq
   %add = add i1 %a, 1
@@ -30,10 +30,10 @@ define x86_regcallcc i1 @test_argReti1(i1 %a)  {
 define x86_regcallcc i1 @test_CallargReti1(i1 %a)  {
 ; WIN32-LABEL: test_CallargReti1:
 ; WIN32:       # %bb.0:
-; WIN32-NEXT:    incb %cl
+; WIN32-NEXT:    xorb $1, %cl
 ; WIN32-NEXT:    movzbl %cl, %ecx
 ; WIN32-NEXT:    calll _test_argReti1
-; WIN32-NEXT:    incb %cl
+; WIN32-NEXT:    xorb $1, %cl
 ; WIN32-NEXT:    retl
 ;
 ; WIN64-LABEL: test_CallargReti1:
@@ -41,11 +41,13 @@ define x86_regcallcc i1 @test_CallargReti1(i1 %a)  {
 ; WIN64-NEXT:    pushq %rax
 ; WIN64-NEXT:    .seh_stackalloc 8
 ; WIN64-NEXT:    .seh_endprologue
-; WIN64-NEXT:    incb %al
+; WIN64-NEXT:    xorb $1, %al
 ; WIN64-NEXT:    movzbl %al, %eax
 ; WIN64-NEXT:    callq test_argReti1
-; WIN64-NEXT:    incb %al
+; WIN64-NEXT:    xorb $1, %al
+; WIN64-NEXT:    .seh_startepilogue
 ; WIN64-NEXT:    popq %rcx
+; WIN64-NEXT:    .seh_endepilogue
 ; WIN64-NEXT:    retq
 ; WIN64-NEXT:    .seh_endproc
 ;
@@ -53,10 +55,10 @@ define x86_regcallcc i1 @test_CallargReti1(i1 %a)  {
 ; LINUXOSX:       # %bb.0:
 ; LINUXOSX-NEXT:    pushq %rax
 ; LINUXOSX-NEXT:    .cfi_def_cfa_offset 16
-; LINUXOSX-NEXT:    incb %al
+; LINUXOSX-NEXT:    xorb $1, %al
 ; LINUXOSX-NEXT:    movzbl %al, %eax
 ; LINUXOSX-NEXT:    callq *test_argReti1@GOTPCREL(%rip)
-; LINUXOSX-NEXT:    incb %al
+; LINUXOSX-NEXT:    xorb $1, %al
 ; LINUXOSX-NEXT:    popq %rcx
 ; LINUXOSX-NEXT:    .cfi_def_cfa_offset 8
 ; LINUXOSX-NEXT:    retq
diff --git a/llvm/test/CodeGen/X86/stack-clash-extra-huge.ll b/llvm/test/CodeGen/X86/stack-clash-extra-huge.ll
index b8031056fd6b0..d9b20f50e9a88 100644
--- a/llvm/test/CodeGen/X86/stack-clash-extra-huge.ll
+++ b/llvm/test/CodeGen/X86/stack-clash-extra-huge.ll
@@ -30,44 +30,22 @@ define i32 @foo() local_unnamed_addr #0 {
 ; CHECK-X86-LABEL: foo:
 ; CHECK-X86:       # %bb.0:
 ; CHECK-X86-NEXT:    ud2
-; CHECK-X86-NEXT:    .cfi_def_cfa_register %eax
-; CHECK-X86-NEXT:    .cfi_adjust_cfa_offset 4800000000
-; CHECK-X86-NEXT:  .LBB0_1: # =>This Inner Loop Header: Depth=1
-; CHECK-X86-NEXT:    subl $4096, %esp # imm = 0x1000
-; CHECK-X86-NEXT:    movl $0, (%esp)
-; CHECK-X86-NEXT:    cmpl %eax, %esp
-; CHECK-X86-NEXT:    jne .LBB0_1
-; CHECK-X86-NEXT:  # %bb.2:
-; CHECK-X86-NEXT:    subl $12, %esp
-; CHECK-X86-NEXT:    .cfi_def_cfa_register %esp
 ; CHECK-X86-NEXT:    .cfi_def_cfa_offset 4800000016
 ; CHECK-X86-NEXT:    movl $1, 392(%esp)
 ; CHECK-X86-NEXT:    movl $1, 28792(%esp)
 ; CHECK-X86-NEXT:    movl (%esp), %eax
-; CHECK-X86-NEXT:    movl $4800000012, %ecx # imm = 0x11E1A300C
-; CHECK-X86-NEXT:    addl %ecx, %esp
+; CHECK-X86-NEXT:    ud2
 ; CHECK-X86-NEXT:    .cfi_def_cfa_offset 4
 ; CHECK-X86-NEXT:    retl
 ;
 ; CHECK-X32-LABEL: foo:
 ; CHECK-X32:       # %bb.0:
 ; CHECK-X32-NEXT:    ud2
-; CHECK-X32-NEXT:    .cfi_def_cfa_register %r11
-; CHECK-X32-NEXT:    .cfi_adjust_cfa_offset 4799995904
-; CHECK-X32-NEXT:  .LBB0_1: # =>This Inner Loop Header: Depth=1
-; CHECK-X32-NEXT:    subl $4096, %esp # imm = 0x1000
-; CHECK-X32-NEXT:    movq $0, (%esp)
-; CHECK-X32-NEXT:    cmpl %r11d, %esp
-; CHECK-X32-NEXT:    jne .LBB0_1
-; CHECK-X32-NEXT:  # %bb.2:
-; CHECK-X32-NEXT:    subl $3976, %esp # imm = 0xF88
-; CHECK-X32-NEXT:    .cfi_def_cfa_register %rsp
 ; CHECK-X32-NEXT:    .cfi_def_cfa_offset 4799999888
 ; CHECK-X32-NEXT:    movl $1, 264(%esp)
 ; CHECK-X32-NEXT:    movl $1, 28664(%esp)
 ; CHECK-X32-NEXT:    movl -128(%esp), %eax
-; CHECK-X32-NEXT:    movl $4799999880, %ecx # imm = 0x11E1A2F88
-; CHECK-X32-NEXT:    addl %ecx, %esp
+; CHECK-X32-NEXT:    ud2
 ; CHECK-X32-NEXT:    .cfi_def_cfa_offset 8
 ; CHECK-X32-NEXT:    retq
   %a = alloca i32, i64 1200000000, align 16
diff --git a/llvm/test/CodeGen/X86/stack-coloring-wineh.ll b/llvm/test/CodeGen/X86/stack-coloring-wineh.ll
index aa2f9e1cd2b1e..198f1bf198620 100644
--- a/llvm/test/CodeGen/X86/stack-coloring-wineh.ll
+++ b/llvm/test/CodeGen/X86/stack-coloring-wineh.ll
@@ -90,8 +90,10 @@ define void @pr66984(ptr %arg) personality ptr @__CxxFrameHandler3 {
 ; X86_64-NEXT:    # %exit
 ; X86_64-NEXT:  $ehgcr_0_3:
 ; X86_64-NEXT:    nop
+; X86_64-NEXT:    .seh_startepilogue
 ; X86_64-NEXT:    addq $64, %rsp
 ; X86_64-NEXT:    popq %rbp
+; X86_64-NEXT:    .seh_endepilogue
 ; X86_64-NEXT:    retq
 ; X86_64-NEXT:    .seh_handlerdata
 ; X86_64-NEXT:    .long ($cppxdata$pr66984)@IMGREL
@@ -116,8 +118,10 @@ define void @pr66984(ptr %arg) personality ptr @__CxxFrameHandler3 {
 ; X86_64-NEXT:    movq -8(%rbp), %rcx
 ; X86_64-NEXT:    callq cleanup
 ; X86_64-NEXT:    leaq .LBB0_3(%rip), %rax
+; X86_64-NEXT:    .seh_startepilogue
 ; X86_64-NEXT:    addq $32, %rsp
 ; X86_64-NEXT:    popq %rbp
+; X86_64-NEXT:    .seh_endepilogue
 ; X86_64-NEXT:    retq # CATCHRET
 ; X86_64-NEXT:    .seh_handlerdata
 ; X86_64-NEXT:    .long ($cppxdata$pr66984)@IMGREL
@@ -143,8 +147,10 @@ define void @pr66984(ptr %arg) personality ptr @__CxxFrameHandler3 {
 ; X86_64-NEXT:    leaq -32(%rbp), %rcx
 ; X86_64-NEXT:    callq foo
 ; X86_64-NEXT:    nop
+; X86_64-NEXT:    .seh_startepilogue
 ; X86_64-NEXT:    addq $32, %rsp
 ; X86_64-NEXT:    popq %rbp
+; X86_64-NEXT:    .seh_endepilogue
 ; X86_64-NEXT:    retq # CLEANUPRET
 ; X86_64-NEXT:  .Lfunc_end0:
 ; X86_64-NEXT:    .seh_handlerdata
diff --git a/llvm/test/CodeGen/X86/subcarry.ll b/llvm/test/CodeGen/X86/subcarry.ll
index 9538ea1061cd1..7d5db07c0172a 100644
--- a/llvm/test/CodeGen/X86/subcarry.ll
+++ b/llvm/test/CodeGen/X86/subcarry.ll
@@ -310,15 +310,10 @@ define { i64, i64, i1 } @subcarry_2x64_add_reversed(i64 %x0, i64 %x1, i64 %y0, i
 ; CHECK-LABEL: subcarry_2x64_add_reversed:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    movq %rdi, %rax
-; CHECK-NEXT:    movq %rsi, %rdi
-; CHECK-NEXT:    subq %rcx, %rdi
 ; CHECK-NEXT:    subq %rdx, %rax
-; CHECK-NEXT:    sbbq $0, %rdi
-; CHECK-NEXT:    setb %r8b
-; CHECK-NEXT:    cmpq %rcx, %rsi
-; CHECK-NEXT:    adcb $0, %r8b
-; CHECK-NEXT:    movq %rdi, %rdx
-; CHECK-NEXT:    movl %r8d, %ecx
+; CHECK-NEXT:    sbbq %rcx, %rsi
+; CHECK-NEXT:    setb %cl
+; CHECK-NEXT:    movq %rsi, %rdx
 ; CHECK-NEXT:    retq
   %t0 = call { i64, i1 } @llvm.usub.with.overflow.i64(i64 %x0, i64 %y0)
   %s0 = extractvalue { i64, i1 } %t0, 0
diff --git a/llvm/test/CodeGen/X86/swift-async-win64.ll b/llvm/test/CodeGen/X86/swift-async-win64.ll
index 843118ba43558..1546a0fee2ec5 100644
--- a/llvm/test/CodeGen/X86/swift-async-win64.ll
+++ b/llvm/test/CodeGen/X86/swift-async-win64.ll
@@ -35,9 +35,11 @@ define void @more_csrs(ptr swiftasync %context) "frame-pointer"="all" {
 ; CHECK64: .seh_endprologue
 ; CHECK64: movq    %r14, (%rbp)
 ; [...]
+; CHECK64: .seh_startepilogue
 ; CHECK64: addq    $8, %rsp
 ; CHECK64: popq    %r15
 ; CHECK64: popq    %rbp
+; CHECK64: .seh_endepilogue
 ; CHECK64: retq
 
 declare void @f(ptr)
diff --git a/llvm/test/CodeGen/X86/swifttailcc-store-ret-address-aliasing-stack-slot.ll b/llvm/test/CodeGen/X86/swifttailcc-store-ret-address-aliasing-stack-slot.ll
new file mode 100644
index 0000000000000..cd669768705e5
--- /dev/null
+++ b/llvm/test/CodeGen/X86/swifttailcc-store-ret-address-aliasing-stack-slot.ll
@@ -0,0 +1,44 @@
+; RUN: llc %s -o - | FileCheck %s
+
+target triple = "x86_64-apple-macosx"
+
+declare swifttailcc void @tc_fn(ptr swiftasync, i64, ptr, i8, ptr swiftself)
+
+declare void @foo()
+
+; FIXME: Currently the return address is written to the stack before loading the
+; argument from an aliasing stack slot.
+define swifttailcc void @test(ptr %0, ptr swiftasync %1, i64 %2, i64 %3, ptr %4, ptr %5, i64 %6, ptr %7, i8 %8) {
+; CHECK-LABEL: test:
+; CHECK:       ## %bb.0: ## %entry
+; CHECK-NEXT:    pushq %r15
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    pushq %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 24
+; CHECK-NEXT:    pushq %rax
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    .cfi_offset %rbx, -24
+; CHECK-NEXT:    .cfi_offset %r15, -16
+; CHECK-NEXT:    movq %r9, %r13
+; CHECK-NEXT:    movq %r8, %rbx
+; CHECK-NEXT:    movq {{[0-9]+}}(%rsp), %r15
+; CHECK-NEXT:    callq _foo
+; CHECK-NEXT:    movq %r14, (%rax)
+; CHECK-NEXT:    movl [[OFF:[0-9]+]](%rsp), %edx
+; CHECK-NEXT:    movq {{[0-9]+}}(%rsp), %rcx
+; CHECK-NEXT:    movq %rcx, [[OFF]](%rsp)
+; CHECK-NEXT:    movq %rax, %r14
+; CHECK-NEXT:    movq %r13, %rdi
+; CHECK-NEXT:    movq %r15, %rsi
+; CHECK-NEXT:    movq %rbx, %r13
+; CHECK-NEXT:    addq $8, %rsp
+; CHECK-NEXT:    popq %rbx
+; CHECK-NEXT:    popq %r15
+; CHECK-NEXT:    addq $16, %rsp
+; CHECK-NEXT:    jmp _tc_fn ## TAILCALL
+entry:
+  %res = tail call ptr @foo()
+  store ptr %1, ptr %res, align 8
+  musttail call swifttailcc void @tc_fn(ptr swiftasync %res, i64 %6, ptr %7, i8 %8, ptr swiftself %5)
+  ret void
+}
diff --git a/llvm/test/CodeGen/X86/tailcc-ssp.ll b/llvm/test/CodeGen/X86/tailcc-ssp.ll
index 81b6c9882fd99..914af1466147a 100644
--- a/llvm/test/CodeGen/X86/tailcc-ssp.ll
+++ b/llvm/test/CodeGen/X86/tailcc-ssp.ll
@@ -21,7 +21,9 @@ define tailcc void @tailcall_frame(ptr %0, i64 %1) sspreq {
 ; WINDOWS-NEXT:    xorl %ecx, %ecx
 ; WINDOWS-NEXT:    xorl %edx, %edx
 ; WINDOWS-NEXT:    xorl %r8d, %r8d
+; WINDOWS-NEXT:    .seh_startepilogue
 ; WINDOWS-NEXT:    addq $56, %rsp
+; WINDOWS-NEXT:    .seh_endepilogue
 ; WINDOWS-NEXT:    jmp h # TAILCALL
 ; WINDOWS-NEXT:  .LBB0_1:
 ; WINDOWS-NEXT:    callq __security_check_cookie
@@ -68,7 +70,9 @@ define void @tailcall_unrelated_frame() sspreq {
 ; WINDOWS-NEXT:    cmpq __security_cookie(%rip), %rcx
 ; WINDOWS-NEXT:    jne .LBB1_1
 ; WINDOWS-NEXT:  # %bb.2:
+; WINDOWS-NEXT:    .seh_startepilogue
 ; WINDOWS-NEXT:    addq $40, %rsp
+; WINDOWS-NEXT:    .seh_endepilogue
 ; WINDOWS-NEXT:    jmp bar # TAILCALL
 ; WINDOWS-NEXT:  .LBB1_1:
 ; WINDOWS-NEXT:    callq __security_check_cookie
diff --git a/llvm/test/CodeGen/X86/taildup-callsiteinfo.mir b/llvm/test/CodeGen/X86/taildup-callsiteinfo.mir
index def9e2e1eee0d..93991799aeb2a 100644
--- a/llvm/test/CodeGen/X86/taildup-callsiteinfo.mir
+++ b/llvm/test/CodeGen/X86/taildup-callsiteinfo.mir
@@ -68,8 +68,9 @@ body:             |
     liveins: $ecx
 
     CALL64pcrel32 @alloc, csr_win64, implicit $rsp, implicit $ssp, implicit $ecx, implicit-def $rsp, implicit-def $ssp, implicit-def dead $rax
-    SEH_Epilogue
+    SEH_BeginEpilogue
     $rsp = frame-destroy ADD64ri8 $rsp, 40, implicit-def dead $eflags
+    SEH_EndEpilogue
     TAILJMPd64 @f2, csr_win64, implicit $rsp, implicit $ssp, implicit $rsp, implicit $ssp
 
 ...
diff --git a/llvm/test/CodeGen/X86/test-shrink.ll b/llvm/test/CodeGen/X86/test-shrink.ll
index 6a7a0ad24ed8e..03bba9c67455c 100644
--- a/llvm/test/CodeGen/X86/test-shrink.ll
+++ b/llvm/test/CodeGen/X86/test-shrink.ll
@@ -546,7 +546,6 @@ define void @testw(i16 inreg %x) nounwind minsize {
 ; CHECK-WIN32-64-LABEL: testw:
 ; CHECK-WIN32-64:       # %bb.0:
 ; CHECK-WIN32-64-NEXT:    subq $40, %rsp
-; CHECK-WIN32-64-NEXT:    # kill: def $cx killed $cx def $ecx
 ; CHECK-WIN32-64-NEXT:    testw $2049, %cx # imm = 0x801
 ; CHECK-WIN32-64-NEXT:    jne .LBB12_2
 ; CHECK-WIN32-64-NEXT:  # %bb.1: # %yes
diff --git a/llvm/test/CodeGen/X86/umax.ll b/llvm/test/CodeGen/X86/umax.ll
index f0479aea1b82c..f589d4a7b04a9 100644
--- a/llvm/test/CodeGen/X86/umax.ll
+++ b/llvm/test/CodeGen/X86/umax.ll
@@ -1237,11 +1237,11 @@ define i16 @test_signbits_i16(i16 %a, i16 %b) nounwind {
 ;
 ; X86-LABEL: test_signbits_i16:
 ; X86:       # %bb.0:
-; X86-NEXT:    movsbl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT:    movswl {{[0-9]+}}(%esp), %eax
-; X86-NEXT:    shrl $15, %eax
-; X86-NEXT:    cmpw %cx, %ax
-; X86-NEXT:    cmovbel %ecx, %eax
+; X86-NEXT:    movsbl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movswl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    shrl $15, %ecx
+; X86-NEXT:    cmpw %ax, %cx
+; X86-NEXT:    cmoval %ecx, %eax
 ; X86-NEXT:    # kill: def $ax killed $ax killed $eax
 ; X86-NEXT:    retl
   %ax = ashr i16 %a, 15
diff --git a/llvm/test/CodeGen/X86/umin.ll b/llvm/test/CodeGen/X86/umin.ll
index e4ce08966a894..7a5cdbb9ce758 100644
--- a/llvm/test/CodeGen/X86/umin.ll
+++ b/llvm/test/CodeGen/X86/umin.ll
@@ -652,11 +652,11 @@ define i16 @test_signbits_i16(i16 %a, i16 %b) nounwind {
 ;
 ; X86-LABEL: test_signbits_i16:
 ; X86:       # %bb.0:
-; X86-NEXT:    movsbl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT:    movswl {{[0-9]+}}(%esp), %eax
-; X86-NEXT:    shrl $15, %eax
-; X86-NEXT:    cmpw %cx, %ax
-; X86-NEXT:    cmovael %ecx, %eax
+; X86-NEXT:    movsbl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movswl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    shrl $15, %ecx
+; X86-NEXT:    cmpw %ax, %cx
+; X86-NEXT:    cmovbl %ecx, %eax
 ; X86-NEXT:    # kill: def $ax killed $ax killed $eax
 ; X86-NEXT:    retl
   %ax = ashr i16 %a, 15
diff --git a/llvm/test/CodeGen/X86/vec_floor.ll b/llvm/test/CodeGen/X86/vec_floor.ll
index 65cde6ac91106..abb85ac83464c 100644
--- a/llvm/test/CodeGen/X86/vec_floor.ll
+++ b/llvm/test/CodeGen/X86/vec_floor.ll
@@ -1679,10 +1679,9 @@ define <4 x float> @floor_mask_ss_mask8(<4 x float> %x, <4 x float> %y, <4 x flo
 ; SSE41:       ## %bb.0:
 ; SSE41-NEXT:    roundss $9, %xmm0, %xmm3
 ; SSE41-NEXT:    cmpeqss %xmm1, %xmm0
-; SSE41-NEXT:    andps %xmm0, %xmm3
-; SSE41-NEXT:    andnps %xmm2, %xmm0
-; SSE41-NEXT:    orps %xmm3, %xmm0
-; SSE41-NEXT:    blendps {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
+; SSE41-NEXT:    blendvps %xmm0, %xmm3, %xmm2
+; SSE41-NEXT:    blendps {{.*#+}} xmm2 = xmm2[0],xmm1[1,2,3]
+; SSE41-NEXT:    movaps %xmm2, %xmm0
 ; SSE41-NEXT:    retq
 ;
 ; AVX-LABEL: floor_mask_ss_mask8:
@@ -1747,10 +1746,9 @@ define <2 x double> @floor_mask_sd_mask8(<2 x double> %x, <2 x double> %y, <2 x
 ; SSE41:       ## %bb.0:
 ; SSE41-NEXT:    roundsd $9, %xmm0, %xmm3
 ; SSE41-NEXT:    cmpeqsd %xmm1, %xmm0
-; SSE41-NEXT:    andpd %xmm0, %xmm3
-; SSE41-NEXT:    andnpd %xmm2, %xmm0
-; SSE41-NEXT:    orpd %xmm3, %xmm0
-; SSE41-NEXT:    blendpd {{.*#+}} xmm0 = xmm0[0],xmm1[1]
+; SSE41-NEXT:    blendvpd %xmm0, %xmm3, %xmm2
+; SSE41-NEXT:    blendpd {{.*#+}} xmm2 = xmm2[0],xmm1[1]
+; SSE41-NEXT:    movapd %xmm2, %xmm0
 ; SSE41-NEXT:    retq
 ;
 ; AVX-LABEL: floor_mask_sd_mask8:
@@ -2671,10 +2669,9 @@ define <4 x float> @ceil_mask_ss_mask8(<4 x float> %x, <4 x float> %y, <4 x floa
 ; SSE41:       ## %bb.0:
 ; SSE41-NEXT:    roundss $10, %xmm0, %xmm3
 ; SSE41-NEXT:    cmpeqss %xmm1, %xmm0
-; SSE41-NEXT:    andps %xmm0, %xmm3
-; SSE41-NEXT:    andnps %xmm2, %xmm0
-; SSE41-NEXT:    orps %xmm3, %xmm0
-; SSE41-NEXT:    blendps {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
+; SSE41-NEXT:    blendvps %xmm0, %xmm3, %xmm2
+; SSE41-NEXT:    blendps {{.*#+}} xmm2 = xmm2[0],xmm1[1,2,3]
+; SSE41-NEXT:    movaps %xmm2, %xmm0
 ; SSE41-NEXT:    retq
 ;
 ; AVX-LABEL: ceil_mask_ss_mask8:
@@ -2739,10 +2736,9 @@ define <2 x double> @ceil_mask_sd_mask8(<2 x double> %x, <2 x double> %y, <2 x d
 ; SSE41:       ## %bb.0:
 ; SSE41-NEXT:    roundsd $10, %xmm0, %xmm3
 ; SSE41-NEXT:    cmpeqsd %xmm1, %xmm0
-; SSE41-NEXT:    andpd %xmm0, %xmm3
-; SSE41-NEXT:    andnpd %xmm2, %xmm0
-; SSE41-NEXT:    orpd %xmm3, %xmm0
-; SSE41-NEXT:    blendpd {{.*#+}} xmm0 = xmm0[0],xmm1[1]
+; SSE41-NEXT:    blendvpd %xmm0, %xmm3, %xmm2
+; SSE41-NEXT:    blendpd {{.*#+}} xmm2 = xmm2[0],xmm1[1]
+; SSE41-NEXT:    movapd %xmm2, %xmm0
 ; SSE41-NEXT:    retq
 ;
 ; AVX-LABEL: ceil_mask_sd_mask8:
diff --git a/llvm/test/CodeGen/X86/vector-compress.ll b/llvm/test/CodeGen/X86/vector-compress.ll
index 87a948a4f1f7e..e88387a8b7c69 100644
--- a/llvm/test/CodeGen/X86/vector-compress.ll
+++ b/llvm/test/CodeGen/X86/vector-compress.ll
@@ -2390,25 +2390,24 @@ define <64 x i8> @test_compress_v64i8(<64 x i8> %vec, <64 x i1> %mask, <64 x i8>
 ; AVX512F-NEXT:    kshiftrw $13, %k1, %k1
 ; AVX512F-NEXT:    korw %k1, %k0, %k0
 ; AVX512F-NEXT:    movw $-9, %ax
-; AVX512F-NEXT:    kmovw %eax, %k7
-; AVX512F-NEXT:    kandw %k7, %k0, %k0
-; AVX512F-NEXT:    kmovw %k7, {{[-0-9]+}}(%r{{[sb]}}p) # 2-byte Spill
+; AVX512F-NEXT:    kmovw %eax, %k5
+; AVX512F-NEXT:    kandw %k5, %k0, %k0
 ; AVX512F-NEXT:    movzbl 376(%rbp), %eax
 ; AVX512F-NEXT:    kmovw %eax, %k1
 ; AVX512F-NEXT:    kshiftlw $15, %k1, %k1
 ; AVX512F-NEXT:    kshiftrw $12, %k1, %k1
 ; AVX512F-NEXT:    korw %k1, %k0, %k0
 ; AVX512F-NEXT:    movw $-17, %ax
-; AVX512F-NEXT:    kmovw %eax, %k5
-; AVX512F-NEXT:    kandw %k5, %k0, %k0
+; AVX512F-NEXT:    kmovw %eax, %k6
+; AVX512F-NEXT:    kandw %k6, %k0, %k0
 ; AVX512F-NEXT:    movzbl 384(%rbp), %eax
 ; AVX512F-NEXT:    kmovw %eax, %k1
 ; AVX512F-NEXT:    kshiftlw $15, %k1, %k1
 ; AVX512F-NEXT:    kshiftrw $11, %k1, %k1
 ; AVX512F-NEXT:    korw %k1, %k0, %k0
 ; AVX512F-NEXT:    movw $-33, %ax
-; AVX512F-NEXT:    kmovw %eax, %k6
-; AVX512F-NEXT:    kandw %k6, %k0, %k0
+; AVX512F-NEXT:    kmovw %eax, %k7
+; AVX512F-NEXT:    kandw %k7, %k0, %k0
 ; AVX512F-NEXT:    movzbl 392(%rbp), %eax
 ; AVX512F-NEXT:    kmovw %eax, %k1
 ; AVX512F-NEXT:    kshiftlw $15, %k1, %k1
@@ -2516,26 +2515,26 @@ define <64 x i8> @test_compress_v64i8(<64 x i8> %vec, <64 x i1> %mask, <64 x i8>
 ; AVX512F-NEXT:    kshiftlw $15, %k2, %k2
 ; AVX512F-NEXT:    kshiftrw $13, %k2, %k2
 ; AVX512F-NEXT:    korw %k2, %k1, %k1
-; AVX512F-NEXT:    kandw %k7, %k1, %k1
+; AVX512F-NEXT:    kandw %k5, %k1, %k1
 ; AVX512F-NEXT:    movzbl 248(%rbp), %eax
 ; AVX512F-NEXT:    kmovw %eax, %k2
 ; AVX512F-NEXT:    kshiftlw $15, %k2, %k2
 ; AVX512F-NEXT:    kshiftrw $12, %k2, %k2
 ; AVX512F-NEXT:    korw %k2, %k1, %k1
-; AVX512F-NEXT:    kandw %k5, %k1, %k1
+; AVX512F-NEXT:    kandw %k6, %k1, %k1
 ; AVX512F-NEXT:    movzbl 256(%rbp), %eax
 ; AVX512F-NEXT:    kmovw %eax, %k2
 ; AVX512F-NEXT:    kshiftlw $15, %k2, %k2
 ; AVX512F-NEXT:    kshiftrw $11, %k2, %k2
 ; AVX512F-NEXT:    korw %k2, %k1, %k1
-; AVX512F-NEXT:    kandw %k6, %k1, %k1
+; AVX512F-NEXT:    kandw %k7, %k1, %k1
 ; AVX512F-NEXT:    movzbl 264(%rbp), %eax
 ; AVX512F-NEXT:    kmovw %eax, %k2
 ; AVX512F-NEXT:    kshiftlw $15, %k2, %k2
 ; AVX512F-NEXT:    kshiftrw $10, %k2, %k2
 ; AVX512F-NEXT:    korw %k2, %k1, %k1
-; AVX512F-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k7 # 2-byte Reload
-; AVX512F-NEXT:    kandw %k7, %k1, %k1
+; AVX512F-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k0 # 2-byte Reload
+; AVX512F-NEXT:    kandw %k0, %k1, %k1
 ; AVX512F-NEXT:    movzbl 272(%rbp), %eax
 ; AVX512F-NEXT:    kmovw %eax, %k2
 ; AVX512F-NEXT:    kshiftlw $15, %k2, %k2
@@ -2626,27 +2625,31 @@ define <64 x i8> @test_compress_v64i8(<64 x i8> %vec, <64 x i1> %mask, <64 x i8>
 ; AVX512F-NEXT:    kshiftlw $15, %k2, %k2
 ; AVX512F-NEXT:    kshiftrw $13, %k2, %k2
 ; AVX512F-NEXT:    korw %k2, %k1, %k1
-; AVX512F-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k4 # 2-byte Reload
-; AVX512F-NEXT:    kandw %k4, %k1, %k1
+; AVX512F-NEXT:    kmovw %k5, %k4
+; AVX512F-NEXT:    kmovw %k5, {{[-0-9]+}}(%r{{[sb]}}p) # 2-byte Spill
+; AVX512F-NEXT:    kandw %k5, %k1, %k1
 ; AVX512F-NEXT:    movzbl 120(%rbp), %eax
 ; AVX512F-NEXT:    kmovw %eax, %k2
 ; AVX512F-NEXT:    kshiftlw $15, %k2, %k2
 ; AVX512F-NEXT:    kshiftrw $12, %k2, %k2
 ; AVX512F-NEXT:    korw %k2, %k1, %k1
-; AVX512F-NEXT:    kmovw %k5, {{[-0-9]+}}(%r{{[sb]}}p) # 2-byte Spill
-; AVX512F-NEXT:    kandw %k5, %k1, %k1
+; AVX512F-NEXT:    kmovw %k6, %k5
+; AVX512F-NEXT:    kmovw %k6, {{[-0-9]+}}(%r{{[sb]}}p) # 2-byte Spill
+; AVX512F-NEXT:    kandw %k6, %k1, %k1
 ; AVX512F-NEXT:    movzbl 128(%rbp), %eax
 ; AVX512F-NEXT:    kmovw %eax, %k2
 ; AVX512F-NEXT:    kshiftlw $15, %k2, %k2
 ; AVX512F-NEXT:    kshiftrw $11, %k2, %k2
 ; AVX512F-NEXT:    korw %k2, %k1, %k1
-; AVX512F-NEXT:    kmovw %k6, {{[-0-9]+}}(%r{{[sb]}}p) # 2-byte Spill
-; AVX512F-NEXT:    kandw %k6, %k1, %k1
+; AVX512F-NEXT:    kmovw %k7, %k6
+; AVX512F-NEXT:    kmovw %k7, {{[-0-9]+}}(%r{{[sb]}}p) # 2-byte Spill
+; AVX512F-NEXT:    kandw %k7, %k1, %k1
 ; AVX512F-NEXT:    movzbl 136(%rbp), %eax
 ; AVX512F-NEXT:    kmovw %eax, %k2
 ; AVX512F-NEXT:    kshiftlw $15, %k2, %k2
 ; AVX512F-NEXT:    kshiftrw $10, %k2, %k2
 ; AVX512F-NEXT:    korw %k2, %k1, %k1
+; AVX512F-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k7 # 2-byte Reload
 ; AVX512F-NEXT:    kandw %k7, %k1, %k1
 ; AVX512F-NEXT:    movzbl 144(%rbp), %eax
 ; AVX512F-NEXT:    kmovw %eax, %k2
@@ -2667,8 +2670,8 @@ define <64 x i8> @test_compress_v64i8(<64 x i8> %vec, <64 x i1> %mask, <64 x i8>
 ; AVX512F-NEXT:    kshiftlw $15, %k2, %k2
 ; AVX512F-NEXT:    kshiftrw $7, %k2, %k2
 ; AVX512F-NEXT:    korw %k2, %k1, %k1
-; AVX512F-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k7 # 2-byte Reload
-; AVX512F-NEXT:    kandw %k7, %k1, %k1
+; AVX512F-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k2 # 2-byte Reload
+; AVX512F-NEXT:    kandw %k2, %k1, %k1
 ; AVX512F-NEXT:    movzbl 168(%rbp), %eax
 ; AVX512F-NEXT:    kmovw %eax, %k2
 ; AVX512F-NEXT:    kshiftlw $15, %k2, %k2
@@ -2741,8 +2744,7 @@ define <64 x i8> @test_compress_v64i8(<64 x i8> %vec, <64 x i1> %mask, <64 x i8>
 ; AVX512F-NEXT:    kshiftlw $15, %k2, %k2
 ; AVX512F-NEXT:    kshiftrw $10, %k2, %k2
 ; AVX512F-NEXT:    korw %k2, %k1, %k1
-; AVX512F-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k2 # 2-byte Reload
-; AVX512F-NEXT:    kandw %k2, %k1, %k1
+; AVX512F-NEXT:    kandw %k7, %k1, %k1
 ; AVX512F-NEXT:    movzbl 16(%rbp), %eax
 ; AVX512F-NEXT:    kmovw %eax, %k2
 ; AVX512F-NEXT:    kshiftlw $15, %k2, %k2
@@ -2764,7 +2766,8 @@ define <64 x i8> @test_compress_v64i8(<64 x i8> %vec, <64 x i1> %mask, <64 x i8>
 ; AVX512F-NEXT:    kshiftlw $15, %k2, %k2
 ; AVX512F-NEXT:    kshiftrw $7, %k2, %k2
 ; AVX512F-NEXT:    korw %k2, %k1, %k1
-; AVX512F-NEXT:    kandw %k7, %k1, %k1
+; AVX512F-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k2 # 2-byte Reload
+; AVX512F-NEXT:    kandw %k2, %k1, %k1
 ; AVX512F-NEXT:    movzbl 40(%rbp), %eax
 ; AVX512F-NEXT:    kmovw %eax, %k2
 ; AVX512F-NEXT:    kshiftlw $15, %k2, %k2
@@ -3763,25 +3766,24 @@ define <64 x i32> @test_compress_large(<64 x i1> %mask, <64 x i32> %vec, <64 x i
 ; AVX512F-NEXT:    kshiftrw $13, %k1, %k1
 ; AVX512F-NEXT:    korw %k1, %k0, %k0
 ; AVX512F-NEXT:    movw $-9, %ax
-; AVX512F-NEXT:    kmovw %eax, %k7
-; AVX512F-NEXT:    kandw %k7, %k0, %k0
-; AVX512F-NEXT:    kmovw %k7, {{[-0-9]+}}(%r{{[sb]}}p) # 2-byte Spill
+; AVX512F-NEXT:    kmovw %eax, %k5
+; AVX512F-NEXT:    kandw %k5, %k0, %k0
 ; AVX512F-NEXT:    movzbl 376(%rbp), %eax
 ; AVX512F-NEXT:    kmovw %eax, %k1
 ; AVX512F-NEXT:    kshiftlw $15, %k1, %k1
 ; AVX512F-NEXT:    kshiftrw $12, %k1, %k1
 ; AVX512F-NEXT:    korw %k1, %k0, %k0
 ; AVX512F-NEXT:    movw $-17, %ax
-; AVX512F-NEXT:    kmovw %eax, %k5
-; AVX512F-NEXT:    kandw %k5, %k0, %k0
+; AVX512F-NEXT:    kmovw %eax, %k6
+; AVX512F-NEXT:    kandw %k6, %k0, %k0
 ; AVX512F-NEXT:    movzbl 384(%rbp), %eax
 ; AVX512F-NEXT:    kmovw %eax, %k1
 ; AVX512F-NEXT:    kshiftlw $15, %k1, %k1
 ; AVX512F-NEXT:    kshiftrw $11, %k1, %k1
 ; AVX512F-NEXT:    korw %k1, %k0, %k0
 ; AVX512F-NEXT:    movw $-33, %ax
-; AVX512F-NEXT:    kmovw %eax, %k6
-; AVX512F-NEXT:    kandw %k6, %k0, %k0
+; AVX512F-NEXT:    kmovw %eax, %k7
+; AVX512F-NEXT:    kandw %k7, %k0, %k0
 ; AVX512F-NEXT:    movzbl 392(%rbp), %eax
 ; AVX512F-NEXT:    kmovw %eax, %k1
 ; AVX512F-NEXT:    kshiftlw $15, %k1, %k1
@@ -3889,26 +3891,26 @@ define <64 x i32> @test_compress_large(<64 x i1> %mask, <64 x i32> %vec, <64 x i
 ; AVX512F-NEXT:    kshiftlw $15, %k2, %k2
 ; AVX512F-NEXT:    kshiftrw $13, %k2, %k2
 ; AVX512F-NEXT:    korw %k2, %k1, %k1
-; AVX512F-NEXT:    kandw %k7, %k1, %k1
+; AVX512F-NEXT:    kandw %k5, %k1, %k1
 ; AVX512F-NEXT:    movzbl 248(%rbp), %eax
 ; AVX512F-NEXT:    kmovw %eax, %k2
 ; AVX512F-NEXT:    kshiftlw $15, %k2, %k2
 ; AVX512F-NEXT:    kshiftrw $12, %k2, %k2
 ; AVX512F-NEXT:    korw %k2, %k1, %k1
-; AVX512F-NEXT:    kandw %k5, %k1, %k1
+; AVX512F-NEXT:    kandw %k6, %k1, %k1
 ; AVX512F-NEXT:    movzbl 256(%rbp), %eax
 ; AVX512F-NEXT:    kmovw %eax, %k2
 ; AVX512F-NEXT:    kshiftlw $15, %k2, %k2
 ; AVX512F-NEXT:    kshiftrw $11, %k2, %k2
 ; AVX512F-NEXT:    korw %k2, %k1, %k1
-; AVX512F-NEXT:    kandw %k6, %k1, %k1
+; AVX512F-NEXT:    kandw %k7, %k1, %k1
 ; AVX512F-NEXT:    movzbl 264(%rbp), %eax
 ; AVX512F-NEXT:    kmovw %eax, %k2
 ; AVX512F-NEXT:    kshiftlw $15, %k2, %k2
 ; AVX512F-NEXT:    kshiftrw $10, %k2, %k2
 ; AVX512F-NEXT:    korw %k2, %k1, %k1
-; AVX512F-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k7 # 2-byte Reload
-; AVX512F-NEXT:    kandw %k7, %k1, %k1
+; AVX512F-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k0 # 2-byte Reload
+; AVX512F-NEXT:    kandw %k0, %k1, %k1
 ; AVX512F-NEXT:    movzbl 272(%rbp), %eax
 ; AVX512F-NEXT:    kmovw %eax, %k2
 ; AVX512F-NEXT:    kshiftlw $15, %k2, %k2
@@ -3999,27 +4001,31 @@ define <64 x i32> @test_compress_large(<64 x i1> %mask, <64 x i32> %vec, <64 x i
 ; AVX512F-NEXT:    kshiftlw $15, %k2, %k2
 ; AVX512F-NEXT:    kshiftrw $13, %k2, %k2
 ; AVX512F-NEXT:    korw %k2, %k1, %k1
-; AVX512F-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k4 # 2-byte Reload
-; AVX512F-NEXT:    kandw %k4, %k1, %k1
+; AVX512F-NEXT:    kmovw %k5, %k4
+; AVX512F-NEXT:    kmovw %k5, {{[-0-9]+}}(%r{{[sb]}}p) # 2-byte Spill
+; AVX512F-NEXT:    kandw %k5, %k1, %k1
 ; AVX512F-NEXT:    movzbl 120(%rbp), %eax
 ; AVX512F-NEXT:    kmovw %eax, %k2
 ; AVX512F-NEXT:    kshiftlw $15, %k2, %k2
 ; AVX512F-NEXT:    kshiftrw $12, %k2, %k2
 ; AVX512F-NEXT:    korw %k2, %k1, %k1
-; AVX512F-NEXT:    kmovw %k5, {{[-0-9]+}}(%r{{[sb]}}p) # 2-byte Spill
-; AVX512F-NEXT:    kandw %k5, %k1, %k1
+; AVX512F-NEXT:    kmovw %k6, %k5
+; AVX512F-NEXT:    kmovw %k6, {{[-0-9]+}}(%r{{[sb]}}p) # 2-byte Spill
+; AVX512F-NEXT:    kandw %k6, %k1, %k1
 ; AVX512F-NEXT:    movzbl 128(%rbp), %eax
 ; AVX512F-NEXT:    kmovw %eax, %k2
 ; AVX512F-NEXT:    kshiftlw $15, %k2, %k2
 ; AVX512F-NEXT:    kshiftrw $11, %k2, %k2
 ; AVX512F-NEXT:    korw %k2, %k1, %k1
-; AVX512F-NEXT:    kmovw %k6, {{[-0-9]+}}(%r{{[sb]}}p) # 2-byte Spill
-; AVX512F-NEXT:    kandw %k6, %k1, %k1
+; AVX512F-NEXT:    kmovw %k7, %k6
+; AVX512F-NEXT:    kmovw %k7, {{[-0-9]+}}(%r{{[sb]}}p) # 2-byte Spill
+; AVX512F-NEXT:    kandw %k7, %k1, %k1
 ; AVX512F-NEXT:    movzbl 136(%rbp), %eax
 ; AVX512F-NEXT:    kmovw %eax, %k2
 ; AVX512F-NEXT:    kshiftlw $15, %k2, %k2
 ; AVX512F-NEXT:    kshiftrw $10, %k2, %k2
 ; AVX512F-NEXT:    korw %k2, %k1, %k1
+; AVX512F-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k7 # 2-byte Reload
 ; AVX512F-NEXT:    kandw %k7, %k1, %k1
 ; AVX512F-NEXT:    movzbl 144(%rbp), %eax
 ; AVX512F-NEXT:    kmovw %eax, %k2
@@ -4040,8 +4046,8 @@ define <64 x i32> @test_compress_large(<64 x i1> %mask, <64 x i32> %vec, <64 x i
 ; AVX512F-NEXT:    kshiftlw $15, %k2, %k2
 ; AVX512F-NEXT:    kshiftrw $7, %k2, %k2
 ; AVX512F-NEXT:    korw %k2, %k1, %k1
-; AVX512F-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k7 # 2-byte Reload
-; AVX512F-NEXT:    kandw %k7, %k1, %k1
+; AVX512F-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k2 # 2-byte Reload
+; AVX512F-NEXT:    kandw %k2, %k1, %k1
 ; AVX512F-NEXT:    movzbl 168(%rbp), %eax
 ; AVX512F-NEXT:    kmovw %eax, %k2
 ; AVX512F-NEXT:    kshiftlw $15, %k2, %k2
@@ -4114,8 +4120,7 @@ define <64 x i32> @test_compress_large(<64 x i1> %mask, <64 x i32> %vec, <64 x i
 ; AVX512F-NEXT:    kshiftlw $15, %k2, %k2
 ; AVX512F-NEXT:    kshiftrw $10, %k2, %k2
 ; AVX512F-NEXT:    korw %k2, %k1, %k1
-; AVX512F-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k2 # 2-byte Reload
-; AVX512F-NEXT:    kandw %k2, %k1, %k1
+; AVX512F-NEXT:    kandw %k7, %k1, %k1
 ; AVX512F-NEXT:    movzbl 16(%rbp), %eax
 ; AVX512F-NEXT:    kmovw %eax, %k2
 ; AVX512F-NEXT:    kshiftlw $15, %k2, %k2
@@ -4137,7 +4142,8 @@ define <64 x i32> @test_compress_large(<64 x i1> %mask, <64 x i32> %vec, <64 x i
 ; AVX512F-NEXT:    kshiftlw $15, %k2, %k2
 ; AVX512F-NEXT:    kshiftrw $7, %k2, %k2
 ; AVX512F-NEXT:    korw %k2, %k1, %k1
-; AVX512F-NEXT:    kandw %k7, %k1, %k1
+; AVX512F-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k2 # 2-byte Reload
+; AVX512F-NEXT:    kandw %k2, %k1, %k1
 ; AVX512F-NEXT:    movzbl 40(%rbp), %eax
 ; AVX512F-NEXT:    kmovw %eax, %k2
 ; AVX512F-NEXT:    kshiftlw $15, %k2, %k2
diff --git a/llvm/test/CodeGen/X86/vector-fshl-256.ll b/llvm/test/CodeGen/X86/vector-fshl-256.ll
index 193e570c5f9a8..32ad72b2aa56a 100644
--- a/llvm/test/CodeGen/X86/vector-fshl-256.ll
+++ b/llvm/test/CodeGen/X86/vector-fshl-256.ll
@@ -1312,7 +1312,6 @@ define void @fancierRotate2(ptr %arr, ptr %control, i32 %rot0, i32 %rot1) {
 ; AVX1-NEXT:    addq $8, %rax
 ; AVX1-NEXT:    jne .LBB8_1
 ; AVX1-NEXT:  # %bb.2: # %exit
-; AVX1-NEXT:    vzeroupper
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: fancierRotate2:
diff --git a/llvm/test/CodeGen/X86/vector-half-conversions.ll b/llvm/test/CodeGen/X86/vector-half-conversions.ll
index 4e50b56323311..1105909699d4f 100644
--- a/llvm/test/CodeGen/X86/vector-half-conversions.ll
+++ b/llvm/test/CodeGen/X86/vector-half-conversions.ll
@@ -39,35 +39,33 @@ define <4 x float> @cvt_4i16_to_4f32(<4 x i16> %a0) nounwind {
 ; AVX-LABEL: cvt_4i16_to_4f32:
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    subq $72, %rsp
+; AVX-NEXT:    vmovdqa %xmm0, (%rsp) # 16-byte Spill
 ; AVX-NEXT:    vmovq %xmm0, %rax
-; AVX-NEXT:    movq %rax, %rcx
+; AVX-NEXT:    movl %eax, %ecx
+; AVX-NEXT:    shrl $16, %ecx
 ; AVX-NEXT:    movq %rax, %rdx
+; AVX-NEXT:    shrq $32, %rdx
+; AVX-NEXT:    shrq $48, %rax
 ; AVX-NEXT:    vpinsrw $0, %eax, %xmm0, %xmm0
-; AVX-NEXT:    # kill: def $eax killed $eax killed $rax
-; AVX-NEXT:    shrl $16, %eax
-; AVX-NEXT:    shrq $32, %rcx
-; AVX-NEXT:    shrq $48, %rdx
-; AVX-NEXT:    vpinsrw $0, %edx, %xmm0, %xmm1
-; AVX-NEXT:    vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX-NEXT:    vpinsrw $0, %ecx, %xmm0, %xmm1
-; AVX-NEXT:    vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX-NEXT:    vpinsrw $0, %eax, %xmm0, %xmm1
-; AVX-NEXT:    vmovdqa %xmm1, (%rsp) # 16-byte Spill
+; AVX-NEXT:    vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX-NEXT:    vpinsrw $0, %edx, %xmm0, %xmm0
+; AVX-NEXT:    vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX-NEXT:    vpinsrw $0, %ecx, %xmm0, %xmm0
 ; AVX-NEXT:    callq __extendhfsf2@PLT
 ; AVX-NEXT:    vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
 ; AVX-NEXT:    vmovaps (%rsp), %xmm0 # 16-byte Reload
 ; AVX-NEXT:    callq __extendhfsf2@PLT
-; AVX-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
-; AVX-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[2,3]
+; AVX-NEXT:    vinsertps $16, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; AVX-NEXT:    # xmm0 = xmm0[0],mem[0],xmm0[2,3]
 ; AVX-NEXT:    vmovaps %xmm0, (%rsp) # 16-byte Spill
 ; AVX-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
 ; AVX-NEXT:    callq __extendhfsf2@PLT
 ; AVX-NEXT:    vmovaps (%rsp), %xmm1 # 16-byte Reload
 ; AVX-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0,1],xmm0[0],xmm1[3]
-; AVX-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX-NEXT:    vmovaps %xmm0, (%rsp) # 16-byte Spill
 ; AVX-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
 ; AVX-NEXT:    callq __extendhfsf2@PLT
-; AVX-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
+; AVX-NEXT:    vmovaps (%rsp), %xmm1 # 16-byte Reload
 ; AVX-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0,1,2],xmm0[0]
 ; AVX-NEXT:    addq $72, %rsp
 ; AVX-NEXT:    retq
@@ -90,35 +88,33 @@ define <4 x float> @cvt_8i16_to_4f32(<8 x i16> %a0) nounwind {
 ; AVX-LABEL: cvt_8i16_to_4f32:
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    subq $72, %rsp
+; AVX-NEXT:    vmovdqa %xmm0, (%rsp) # 16-byte Spill
 ; AVX-NEXT:    vmovq %xmm0, %rax
-; AVX-NEXT:    movq %rax, %rcx
+; AVX-NEXT:    movl %eax, %ecx
+; AVX-NEXT:    shrl $16, %ecx
 ; AVX-NEXT:    movq %rax, %rdx
+; AVX-NEXT:    shrq $32, %rdx
+; AVX-NEXT:    shrq $48, %rax
 ; AVX-NEXT:    vpinsrw $0, %eax, %xmm0, %xmm0
-; AVX-NEXT:    # kill: def $eax killed $eax killed $rax
-; AVX-NEXT:    shrl $16, %eax
-; AVX-NEXT:    shrq $32, %rcx
-; AVX-NEXT:    shrq $48, %rdx
-; AVX-NEXT:    vpinsrw $0, %edx, %xmm0, %xmm1
-; AVX-NEXT:    vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX-NEXT:    vpinsrw $0, %ecx, %xmm0, %xmm1
-; AVX-NEXT:    vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX-NEXT:    vpinsrw $0, %eax, %xmm0, %xmm1
-; AVX-NEXT:    vmovdqa %xmm1, (%rsp) # 16-byte Spill
+; AVX-NEXT:    vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX-NEXT:    vpinsrw $0, %edx, %xmm0, %xmm0
+; AVX-NEXT:    vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX-NEXT:    vpinsrw $0, %ecx, %xmm0, %xmm0
 ; AVX-NEXT:    callq __extendhfsf2@PLT
 ; AVX-NEXT:    vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
 ; AVX-NEXT:    vmovaps (%rsp), %xmm0 # 16-byte Reload
 ; AVX-NEXT:    callq __extendhfsf2@PLT
-; AVX-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
-; AVX-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[2,3]
+; AVX-NEXT:    vinsertps $16, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; AVX-NEXT:    # xmm0 = xmm0[0],mem[0],xmm0[2,3]
 ; AVX-NEXT:    vmovaps %xmm0, (%rsp) # 16-byte Spill
 ; AVX-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
 ; AVX-NEXT:    callq __extendhfsf2@PLT
 ; AVX-NEXT:    vmovaps (%rsp), %xmm1 # 16-byte Reload
 ; AVX-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0,1],xmm0[0],xmm1[3]
-; AVX-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX-NEXT:    vmovaps %xmm0, (%rsp) # 16-byte Spill
 ; AVX-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
 ; AVX-NEXT:    callq __extendhfsf2@PLT
-; AVX-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
+; AVX-NEXT:    vmovaps (%rsp), %xmm1 # 16-byte Reload
 ; AVX-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0,1,2],xmm0[0]
 ; AVX-NEXT:    addq $72, %rsp
 ; AVX-NEXT:    retq
@@ -430,17 +426,15 @@ define <2 x float> @cvt_2i16_to_2f32_constrained(<2 x i16> %a0) nounwind strictf
 ; AVX-LABEL: cvt_2i16_to_2f32_constrained:
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    subq $40, %rsp
-; AVX-NEXT:    vmovd %xmm0, %eax
+; AVX-NEXT:    vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX-NEXT:    vpextrw $1, %xmm0, %eax
 ; AVX-NEXT:    vpinsrw $0, %eax, %xmm0, %xmm0
-; AVX-NEXT:    shrl $16, %eax
-; AVX-NEXT:    vpinsrw $0, %eax, %xmm0, %xmm1
-; AVX-NEXT:    vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
 ; AVX-NEXT:    callq __extendhfsf2@PLT
 ; AVX-NEXT:    vmovdqa %xmm0, (%rsp) # 16-byte Spill
 ; AVX-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
 ; AVX-NEXT:    callq __extendhfsf2@PLT
-; AVX-NEXT:    vmovaps (%rsp), %xmm1 # 16-byte Reload
-; AVX-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[2,3]
+; AVX-NEXT:    vinsertps $16, (%rsp), %xmm0, %xmm0 # 16-byte Folded Reload
+; AVX-NEXT:    # xmm0 = xmm0[0],mem[0],xmm0[2,3]
 ; AVX-NEXT:    addq $40, %rsp
 ; AVX-NEXT:    retq
 ;
@@ -465,35 +459,33 @@ define <4 x float> @cvt_4i16_to_4f32_constrained(<4 x i16> %a0) nounwind strictf
 ; AVX-LABEL: cvt_4i16_to_4f32_constrained:
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    subq $72, %rsp
+; AVX-NEXT:    vmovdqa %xmm0, (%rsp) # 16-byte Spill
 ; AVX-NEXT:    vmovq %xmm0, %rax
-; AVX-NEXT:    movq %rax, %rcx
+; AVX-NEXT:    movl %eax, %ecx
+; AVX-NEXT:    shrl $16, %ecx
 ; AVX-NEXT:    movq %rax, %rdx
+; AVX-NEXT:    shrq $32, %rdx
+; AVX-NEXT:    shrq $48, %rax
 ; AVX-NEXT:    vpinsrw $0, %eax, %xmm0, %xmm0
-; AVX-NEXT:    # kill: def $eax killed $eax killed $rax
-; AVX-NEXT:    shrl $16, %eax
-; AVX-NEXT:    shrq $32, %rcx
-; AVX-NEXT:    shrq $48, %rdx
-; AVX-NEXT:    vpinsrw $0, %edx, %xmm0, %xmm1
-; AVX-NEXT:    vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX-NEXT:    vpinsrw $0, %ecx, %xmm0, %xmm1
-; AVX-NEXT:    vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX-NEXT:    vpinsrw $0, %eax, %xmm0, %xmm1
-; AVX-NEXT:    vmovdqa %xmm1, (%rsp) # 16-byte Spill
+; AVX-NEXT:    vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX-NEXT:    vpinsrw $0, %edx, %xmm0, %xmm0
+; AVX-NEXT:    vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX-NEXT:    vpinsrw $0, %ecx, %xmm0, %xmm0
 ; AVX-NEXT:    callq __extendhfsf2@PLT
 ; AVX-NEXT:    vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
 ; AVX-NEXT:    vmovaps (%rsp), %xmm0 # 16-byte Reload
 ; AVX-NEXT:    callq __extendhfsf2@PLT
-; AVX-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
-; AVX-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[2,3]
+; AVX-NEXT:    vinsertps $16, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; AVX-NEXT:    # xmm0 = xmm0[0],mem[0],xmm0[2,3]
 ; AVX-NEXT:    vmovaps %xmm0, (%rsp) # 16-byte Spill
 ; AVX-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
 ; AVX-NEXT:    callq __extendhfsf2@PLT
 ; AVX-NEXT:    vmovaps (%rsp), %xmm1 # 16-byte Reload
 ; AVX-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0,1],xmm0[0],xmm1[3]
-; AVX-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX-NEXT:    vmovaps %xmm0, (%rsp) # 16-byte Spill
 ; AVX-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
 ; AVX-NEXT:    callq __extendhfsf2@PLT
-; AVX-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
+; AVX-NEXT:    vmovaps (%rsp), %xmm1 # 16-byte Reload
 ; AVX-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0,1,2],xmm0[0]
 ; AVX-NEXT:    addq $72, %rsp
 ; AVX-NEXT:    retq
@@ -878,35 +870,34 @@ define <4 x float> @load_cvt_8i16_to_4f32(ptr %a0) nounwind {
 ; AVX-LABEL: load_cvt_8i16_to_4f32:
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    subq $72, %rsp
-; AVX-NEXT:    movq (%rdi), %rax
-; AVX-NEXT:    movq %rax, %rcx
+; AVX-NEXT:    vmovdqa (%rdi), %xmm0
+; AVX-NEXT:    vmovdqa %xmm0, (%rsp) # 16-byte Spill
+; AVX-NEXT:    vmovq %xmm0, %rax
+; AVX-NEXT:    movl %eax, %ecx
+; AVX-NEXT:    shrl $16, %ecx
 ; AVX-NEXT:    movq %rax, %rdx
+; AVX-NEXT:    shrq $32, %rdx
+; AVX-NEXT:    shrq $48, %rax
 ; AVX-NEXT:    vpinsrw $0, %eax, %xmm0, %xmm0
-; AVX-NEXT:    # kill: def $eax killed $eax killed $rax
-; AVX-NEXT:    shrl $16, %eax
-; AVX-NEXT:    shrq $32, %rcx
-; AVX-NEXT:    shrq $48, %rdx
-; AVX-NEXT:    vpinsrw $0, %edx, %xmm0, %xmm1
-; AVX-NEXT:    vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX-NEXT:    vpinsrw $0, %ecx, %xmm0, %xmm1
-; AVX-NEXT:    vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX-NEXT:    vpinsrw $0, %eax, %xmm0, %xmm1
-; AVX-NEXT:    vmovdqa %xmm1, (%rsp) # 16-byte Spill
+; AVX-NEXT:    vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX-NEXT:    vpinsrw $0, %edx, %xmm0, %xmm0
+; AVX-NEXT:    vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX-NEXT:    vpinsrw $0, %ecx, %xmm0, %xmm0
 ; AVX-NEXT:    callq __extendhfsf2@PLT
 ; AVX-NEXT:    vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
 ; AVX-NEXT:    vmovaps (%rsp), %xmm0 # 16-byte Reload
 ; AVX-NEXT:    callq __extendhfsf2@PLT
-; AVX-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
-; AVX-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[2,3]
+; AVX-NEXT:    vinsertps $16, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; AVX-NEXT:    # xmm0 = xmm0[0],mem[0],xmm0[2,3]
 ; AVX-NEXT:    vmovaps %xmm0, (%rsp) # 16-byte Spill
 ; AVX-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
 ; AVX-NEXT:    callq __extendhfsf2@PLT
 ; AVX-NEXT:    vmovaps (%rsp), %xmm1 # 16-byte Reload
 ; AVX-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0,1],xmm0[0],xmm1[3]
-; AVX-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX-NEXT:    vmovaps %xmm0, (%rsp) # 16-byte Spill
 ; AVX-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
 ; AVX-NEXT:    callq __extendhfsf2@PLT
-; AVX-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
+; AVX-NEXT:    vmovaps (%rsp), %xmm1 # 16-byte Reload
 ; AVX-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0,1,2],xmm0[0]
 ; AVX-NEXT:    addq $72, %rsp
 ; AVX-NEXT:    retq
@@ -1301,35 +1292,34 @@ define <4 x float> @load_cvt_8i16_to_4f32_constrained(ptr %a0) nounwind strictfp
 ; AVX-LABEL: load_cvt_8i16_to_4f32_constrained:
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    subq $72, %rsp
-; AVX-NEXT:    movq (%rdi), %rax
-; AVX-NEXT:    movq %rax, %rcx
+; AVX-NEXT:    vmovdqa (%rdi), %xmm0
+; AVX-NEXT:    vmovdqa %xmm0, (%rsp) # 16-byte Spill
+; AVX-NEXT:    vmovq %xmm0, %rax
+; AVX-NEXT:    movl %eax, %ecx
+; AVX-NEXT:    shrl $16, %ecx
 ; AVX-NEXT:    movq %rax, %rdx
+; AVX-NEXT:    shrq $32, %rdx
+; AVX-NEXT:    shrq $48, %rax
 ; AVX-NEXT:    vpinsrw $0, %eax, %xmm0, %xmm0
-; AVX-NEXT:    # kill: def $eax killed $eax killed $rax
-; AVX-NEXT:    shrl $16, %eax
-; AVX-NEXT:    shrq $32, %rcx
-; AVX-NEXT:    shrq $48, %rdx
-; AVX-NEXT:    vpinsrw $0, %edx, %xmm0, %xmm1
-; AVX-NEXT:    vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX-NEXT:    vpinsrw $0, %ecx, %xmm0, %xmm1
-; AVX-NEXT:    vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX-NEXT:    vpinsrw $0, %eax, %xmm0, %xmm1
-; AVX-NEXT:    vmovdqa %xmm1, (%rsp) # 16-byte Spill
+; AVX-NEXT:    vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX-NEXT:    vpinsrw $0, %edx, %xmm0, %xmm0
+; AVX-NEXT:    vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX-NEXT:    vpinsrw $0, %ecx, %xmm0, %xmm0
 ; AVX-NEXT:    callq __extendhfsf2@PLT
 ; AVX-NEXT:    vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
 ; AVX-NEXT:    vmovaps (%rsp), %xmm0 # 16-byte Reload
 ; AVX-NEXT:    callq __extendhfsf2@PLT
-; AVX-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
-; AVX-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[2,3]
+; AVX-NEXT:    vinsertps $16, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; AVX-NEXT:    # xmm0 = xmm0[0],mem[0],xmm0[2,3]
 ; AVX-NEXT:    vmovaps %xmm0, (%rsp) # 16-byte Spill
 ; AVX-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
 ; AVX-NEXT:    callq __extendhfsf2@PLT
 ; AVX-NEXT:    vmovaps (%rsp), %xmm1 # 16-byte Reload
 ; AVX-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0,1],xmm0[0],xmm1[3]
-; AVX-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX-NEXT:    vmovaps %xmm0, (%rsp) # 16-byte Spill
 ; AVX-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
 ; AVX-NEXT:    callq __extendhfsf2@PLT
-; AVX-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
+; AVX-NEXT:    vmovaps (%rsp), %xmm1 # 16-byte Reload
 ; AVX-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0,1,2],xmm0[0]
 ; AVX-NEXT:    addq $72, %rsp
 ; AVX-NEXT:    retq
@@ -1386,9 +1376,7 @@ define <2 x double> @cvt_2i16_to_2f64(<2 x i16> %a0) nounwind {
 ; AVX-LABEL: cvt_2i16_to_2f64:
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    subq $40, %rsp
-; AVX-NEXT:    vmovd %xmm0, %eax
-; AVX-NEXT:    vpinsrw $0, %eax, %xmm0, %xmm0
-; AVX-NEXT:    shrl $16, %eax
+; AVX-NEXT:    vpextrw $1, %xmm0, %eax
 ; AVX-NEXT:    vpinsrw $0, %eax, %xmm0, %xmm1
 ; AVX-NEXT:    vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
 ; AVX-NEXT:    callq __extendhfsf2@PLT
@@ -1422,19 +1410,18 @@ define <4 x double> @cvt_4i16_to_4f64(<4 x i16> %a0) nounwind {
 ; AVX-LABEL: cvt_4i16_to_4f64:
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    subq $72, %rsp
+; AVX-NEXT:    vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
 ; AVX-NEXT:    vmovq %xmm0, %rax
 ; AVX-NEXT:    movq %rax, %rcx
-; AVX-NEXT:    movl %eax, %edx
+; AVX-NEXT:    shrq $48, %rcx
+; AVX-NEXT:    movq %rax, %rdx
+; AVX-NEXT:    shrq $32, %rdx
+; AVX-NEXT:    shrl $16, %eax
 ; AVX-NEXT:    vpinsrw $0, %eax, %xmm0, %xmm0
 ; AVX-NEXT:    vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX-NEXT:    shrq $48, %rax
-; AVX-NEXT:    shrq $32, %rcx
-; AVX-NEXT:    shrl $16, %edx
 ; AVX-NEXT:    vpinsrw $0, %edx, %xmm0, %xmm0
-; AVX-NEXT:    vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX-NEXT:    vpinsrw $0, %ecx, %xmm0, %xmm0
 ; AVX-NEXT:    vmovdqa %xmm0, (%rsp) # 16-byte Spill
-; AVX-NEXT:    vpinsrw $0, %eax, %xmm0, %xmm0
+; AVX-NEXT:    vpinsrw $0, %ecx, %xmm0, %xmm0
 ; AVX-NEXT:    callq __extendhfsf2@PLT
 ; AVX-NEXT:    vcvtss2sd %xmm0, %xmm0, %xmm0
 ; AVX-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
@@ -1477,9 +1464,7 @@ define <2 x double> @cvt_8i16_to_2f64(<8 x i16> %a0) nounwind {
 ; AVX-LABEL: cvt_8i16_to_2f64:
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    subq $40, %rsp
-; AVX-NEXT:    vmovd %xmm0, %eax
-; AVX-NEXT:    vpinsrw $0, %eax, %xmm0, %xmm0
-; AVX-NEXT:    shrl $16, %eax
+; AVX-NEXT:    vpextrw $1, %xmm0, %eax
 ; AVX-NEXT:    vpinsrw $0, %eax, %xmm0, %xmm1
 ; AVX-NEXT:    vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
 ; AVX-NEXT:    callq __extendhfsf2@PLT
@@ -1514,19 +1499,18 @@ define <4 x double> @cvt_8i16_to_4f64(<8 x i16> %a0) nounwind {
 ; AVX-LABEL: cvt_8i16_to_4f64:
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    subq $72, %rsp
+; AVX-NEXT:    vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
 ; AVX-NEXT:    vmovq %xmm0, %rax
 ; AVX-NEXT:    movq %rax, %rcx
-; AVX-NEXT:    movl %eax, %edx
+; AVX-NEXT:    shrq $48, %rcx
+; AVX-NEXT:    movq %rax, %rdx
+; AVX-NEXT:    shrq $32, %rdx
+; AVX-NEXT:    shrl $16, %eax
 ; AVX-NEXT:    vpinsrw $0, %eax, %xmm0, %xmm0
 ; AVX-NEXT:    vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX-NEXT:    shrq $48, %rax
-; AVX-NEXT:    shrq $32, %rcx
-; AVX-NEXT:    shrl $16, %edx
 ; AVX-NEXT:    vpinsrw $0, %edx, %xmm0, %xmm0
-; AVX-NEXT:    vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX-NEXT:    vpinsrw $0, %ecx, %xmm0, %xmm0
 ; AVX-NEXT:    vmovdqa %xmm0, (%rsp) # 16-byte Spill
-; AVX-NEXT:    vpinsrw $0, %eax, %xmm0, %xmm0
+; AVX-NEXT:    vpinsrw $0, %ecx, %xmm0, %xmm0
 ; AVX-NEXT:    callq __extendhfsf2@PLT
 ; AVX-NEXT:    vcvtss2sd %xmm0, %xmm0, %xmm0
 ; AVX-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
@@ -1645,9 +1629,7 @@ define <2 x double> @cvt_2i16_to_2f64_constrained(<2 x i16> %a0) nounwind strict
 ; AVX-LABEL: cvt_2i16_to_2f64_constrained:
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    subq $40, %rsp
-; AVX-NEXT:    vmovd %xmm0, %eax
-; AVX-NEXT:    vpinsrw $0, %eax, %xmm0, %xmm0
-; AVX-NEXT:    shrl $16, %eax
+; AVX-NEXT:    vpextrw $1, %xmm0, %eax
 ; AVX-NEXT:    vpinsrw $0, %eax, %xmm0, %xmm1
 ; AVX-NEXT:    vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
 ; AVX-NEXT:    callq __extendhfsf2@PLT
@@ -1684,19 +1666,18 @@ define <4 x double> @cvt_4i16_to_4f64_constrained(<4 x i16> %a0) nounwind strict
 ; AVX-LABEL: cvt_4i16_to_4f64_constrained:
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    subq $72, %rsp
+; AVX-NEXT:    vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
 ; AVX-NEXT:    vmovq %xmm0, %rax
 ; AVX-NEXT:    movq %rax, %rcx
-; AVX-NEXT:    movl %eax, %edx
+; AVX-NEXT:    shrq $48, %rcx
+; AVX-NEXT:    movq %rax, %rdx
+; AVX-NEXT:    shrq $32, %rdx
+; AVX-NEXT:    shrl $16, %eax
 ; AVX-NEXT:    vpinsrw $0, %eax, %xmm0, %xmm0
 ; AVX-NEXT:    vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX-NEXT:    shrq $48, %rax
-; AVX-NEXT:    shrq $32, %rcx
-; AVX-NEXT:    shrl $16, %edx
 ; AVX-NEXT:    vpinsrw $0, %edx, %xmm0, %xmm0
-; AVX-NEXT:    vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX-NEXT:    vpinsrw $0, %ecx, %xmm0, %xmm0
 ; AVX-NEXT:    vmovdqa %xmm0, (%rsp) # 16-byte Spill
-; AVX-NEXT:    vpinsrw $0, %eax, %xmm0, %xmm0
+; AVX-NEXT:    vpinsrw $0, %ecx, %xmm0, %xmm0
 ; AVX-NEXT:    callq __extendhfsf2@PLT
 ; AVX-NEXT:    vcvtss2sd %xmm0, %xmm0, %xmm0
 ; AVX-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
@@ -1936,19 +1917,19 @@ define <4 x double> @load_cvt_8i16_to_4f64(ptr %a0) nounwind {
 ; AVX-LABEL: load_cvt_8i16_to_4f64:
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    subq $72, %rsp
-; AVX-NEXT:    movq (%rdi), %rax
+; AVX-NEXT:    vmovdqa (%rdi), %xmm0
+; AVX-NEXT:    vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX-NEXT:    vmovq %xmm0, %rax
 ; AVX-NEXT:    movq %rax, %rcx
-; AVX-NEXT:    movl %eax, %edx
+; AVX-NEXT:    shrq $48, %rcx
+; AVX-NEXT:    movq %rax, %rdx
+; AVX-NEXT:    shrq $32, %rdx
+; AVX-NEXT:    shrl $16, %eax
 ; AVX-NEXT:    vpinsrw $0, %eax, %xmm0, %xmm0
 ; AVX-NEXT:    vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX-NEXT:    shrq $48, %rax
-; AVX-NEXT:    shrq $32, %rcx
-; AVX-NEXT:    shrl $16, %edx
 ; AVX-NEXT:    vpinsrw $0, %edx, %xmm0, %xmm0
-; AVX-NEXT:    vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX-NEXT:    vpinsrw $0, %ecx, %xmm0, %xmm0
 ; AVX-NEXT:    vmovdqa %xmm0, (%rsp) # 16-byte Spill
-; AVX-NEXT:    vpinsrw $0, %eax, %xmm0, %xmm0
+; AVX-NEXT:    vpinsrw $0, %ecx, %xmm0, %xmm0
 ; AVX-NEXT:    callq __extendhfsf2@PLT
 ; AVX-NEXT:    vcvtss2sd %xmm0, %xmm0, %xmm0
 ; AVX-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
@@ -2142,14 +2123,14 @@ define i16 @cvt_f32_to_i16(float %a0) nounwind {
 ; F16C-LABEL: cvt_f32_to_i16:
 ; F16C:       # %bb.0:
 ; F16C-NEXT:    vcvtps2ph $4, %xmm0, %xmm0
-; F16C-NEXT:    vmovd %xmm0, %eax
+; F16C-NEXT:    vpextrw $0, %xmm0, %eax
 ; F16C-NEXT:    # kill: def $ax killed $ax killed $eax
 ; F16C-NEXT:    retq
 ;
 ; AVX512-LABEL: cvt_f32_to_i16:
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vcvtps2ph $4, %xmm0, %xmm0
-; AVX512-NEXT:    vmovd %xmm0, %eax
+; AVX512-NEXT:    vpextrw $0, %xmm0, %eax
 ; AVX512-NEXT:    # kill: def $ax killed $ax killed $eax
 ; AVX512-NEXT:    retq
   %1 = fptrunc float %a0 to half
diff --git a/llvm/test/CodeGen/X86/vector-idiv-sdiv-512.ll b/llvm/test/CodeGen/X86/vector-idiv-sdiv-512.ll
index 3dc145c7c2d69..6256c4f2acf7f 100644
--- a/llvm/test/CodeGen/X86/vector-idiv-sdiv-512.ll
+++ b/llvm/test/CodeGen/X86/vector-idiv-sdiv-512.ll
@@ -182,7 +182,7 @@ define <64 x i8> @test_div7_64i8(<64 x i8> %a) nounwind {
 ; AVX512BW-NEXT:    vpaddb %zmm0, %zmm1, %zmm0
 ; AVX512BW-NEXT:    vpsrlw $2, %zmm0, %zmm1
 ; AVX512BW-NEXT:    vpbroadcastb {{.*#+}} zmm2 = [32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32]
-; AVX512BW-NEXT:    vpternlogd $108, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm2, %zmm1
+; AVX512BW-NEXT:    vpternlogd {{.*#+}} zmm1 = zmm2 ^ (zmm1 & mem)
 ; AVX512BW-NEXT:    vpsrlw $7, %zmm0, %zmm0
 ; AVX512BW-NEXT:    vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm0, %zmm0
 ; AVX512BW-NEXT:    vpaddb %zmm0, %zmm1, %zmm0
@@ -509,7 +509,7 @@ define <64 x i8> @test_rem7_64i8(<64 x i8> %a) nounwind {
 ; AVX512BW-NEXT:    vpaddb %zmm0, %zmm1, %zmm1
 ; AVX512BW-NEXT:    vpsrlw $2, %zmm1, %zmm2
 ; AVX512BW-NEXT:    vpbroadcastb {{.*#+}} zmm3 = [32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32]
-; AVX512BW-NEXT:    vpternlogd $108, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm3, %zmm2
+; AVX512BW-NEXT:    vpternlogd {{.*#+}} zmm2 = zmm3 ^ (zmm2 & mem)
 ; AVX512BW-NEXT:    vpsrlw $7, %zmm1, %zmm1
 ; AVX512BW-NEXT:    vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm1, %zmm1
 ; AVX512BW-NEXT:    vpaddb %zmm1, %zmm2, %zmm1
@@ -618,7 +618,7 @@ define <64 x i8> @test_remconstant_64i8(<64 x i8> %a) nounwind {
 ; AVX512BW-NEXT:    vpmaddubsw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm1, %zmm2 # [7,0,9,0,11,0,13,0,15,0,17,0,19,0,21,0,23,0,25,0,27,0,29,0,31,0,33,0,35,0,37,0,38,0,36,0,34,0,32,0,30,0,28,0,26,0,24,0,22,0,20,0,18,0,16,0,14,0,12,0,10,0,8,0]
 ; AVX512BW-NEXT:    vpmaddubsw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm1, %zmm1 # [0,8,0,10,0,12,0,14,0,16,0,18,0,20,0,22,0,24,0,26,0,28,0,30,0,32,0,34,0,36,0,38,0,37,0,35,0,33,0,31,0,29,0,27,0,25,0,23,0,21,0,19,0,17,0,15,0,13,0,11,0,9,0,7]
 ; AVX512BW-NEXT:    vpsllw $8, %zmm1, %zmm1
-; AVX512BW-NEXT:    vpternlogd $248, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm2, %zmm1
+; AVX512BW-NEXT:    vpternlogd {{.*#+}} zmm1 = zmm1 | (zmm2 & mem)
 ; AVX512BW-NEXT:    vpsubb %zmm1, %zmm0, %zmm0
 ; AVX512BW-NEXT:    retq
   %res = srem <64 x i8> %a, <i8 7, i8 8, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15, i8 16, i8 17, i8 18, i8 19, i8 20, i8 21, i8 22, i8 23, i8 24, i8 25, i8 26, i8 27, i8 28, i8 29, i8 30, i8 31, i8 32, i8 33, i8 34, i8 35, i8 36, i8 37, i8 38, i8 38, i8 37, i8 36, i8 35, i8 34, i8 33, i8 32, i8 31, i8 30, i8 29, i8 28, i8 27, i8 26, i8 25, i8 24, i8 23, i8 22, i8 21, i8 20, i8 19, i8 18, i8 17, i8 16, i8 15, i8 14, i8 13, i8 12, i8 11, i8 10, i8 9, i8 8, i8 7>
diff --git a/llvm/test/CodeGen/X86/vector-lrint.ll b/llvm/test/CodeGen/X86/vector-lrint.ll
index 3612205bf1bfa..b1c8d46f497f3 100644
--- a/llvm/test/CodeGen/X86/vector-lrint.ll
+++ b/llvm/test/CodeGen/X86/vector-lrint.ll
@@ -269,31 +269,17 @@ declare <1 x iXLen> @llvm.lrint.v1iXLen.v1f64(<1 x double>)
 define <2 x iXLen> @lrint_v2f64(<2 x double> %x) {
 ; X86-SSE2-LABEL: lrint_v2f64:
 ; X86-SSE2:       # %bb.0:
-; X86-SSE2-NEXT:    cvtsd2si %xmm0, %eax
-; X86-SSE2-NEXT:    movd %eax, %xmm1
-; X86-SSE2-NEXT:    unpckhpd {{.*#+}} xmm0 = xmm0[1,1]
-; X86-SSE2-NEXT:    cvtsd2si %xmm0, %eax
-; X86-SSE2-NEXT:    movd %eax, %xmm0
-; X86-SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
-; X86-SSE2-NEXT:    movdqa %xmm1, %xmm0
+; X86-SSE2-NEXT:    cvtpd2dq %xmm0, %xmm0
 ; X86-SSE2-NEXT:    retl
 ;
 ; X86-AVX-LABEL: lrint_v2f64:
 ; X86-AVX:       # %bb.0:
-; X86-AVX-NEXT:    vshufpd {{.*#+}} xmm1 = xmm0[1,0]
-; X86-AVX-NEXT:    vcvtsd2si %xmm1, %eax
-; X86-AVX-NEXT:    vcvtsd2si %xmm0, %ecx
-; X86-AVX-NEXT:    vmovd %ecx, %xmm0
-; X86-AVX-NEXT:    vpinsrd $1, %eax, %xmm0, %xmm0
+; X86-AVX-NEXT:    vcvtpd2dq %xmm0, %xmm0
 ; X86-AVX-NEXT:    retl
 ;
 ; X64-AVX-i32-LABEL: lrint_v2f64:
 ; X64-AVX-i32:       # %bb.0:
-; X64-AVX-i32-NEXT:    vshufpd {{.*#+}} xmm1 = xmm0[1,0]
-; X64-AVX-i32-NEXT:    vcvtsd2si %xmm1, %eax
-; X64-AVX-i32-NEXT:    vcvtsd2si %xmm0, %ecx
-; X64-AVX-i32-NEXT:    vmovd %ecx, %xmm0
-; X64-AVX-i32-NEXT:    vpinsrd $1, %eax, %xmm0, %xmm0
+; X64-AVX-i32-NEXT:    vcvtpd2dq %xmm0, %xmm0
 ; X64-AVX-i32-NEXT:    retq
 ;
 ; X64-AVX1-i64-LABEL: lrint_v2f64:
@@ -328,20 +314,9 @@ declare <2 x iXLen> @llvm.lrint.v2iXLen.v2f64(<2 x double>)
 define <4 x iXLen> @lrint_v4f64(<4 x double> %x) {
 ; X86-SSE2-LABEL: lrint_v4f64:
 ; X86-SSE2:       # %bb.0:
-; X86-SSE2-NEXT:    cvtsd2si %xmm1, %eax
-; X86-SSE2-NEXT:    movd %eax, %xmm2
-; X86-SSE2-NEXT:    unpckhpd {{.*#+}} xmm1 = xmm1[1,1]
-; X86-SSE2-NEXT:    cvtsd2si %xmm1, %eax
-; X86-SSE2-NEXT:    movd %eax, %xmm1
-; X86-SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
-; X86-SSE2-NEXT:    cvtsd2si %xmm0, %eax
-; X86-SSE2-NEXT:    movd %eax, %xmm1
-; X86-SSE2-NEXT:    unpckhpd {{.*#+}} xmm0 = xmm0[1,1]
-; X86-SSE2-NEXT:    cvtsd2si %xmm0, %eax
-; X86-SSE2-NEXT:    movd %eax, %xmm0
-; X86-SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
-; X86-SSE2-NEXT:    punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm2[0]
-; X86-SSE2-NEXT:    movdqa %xmm1, %xmm0
+; X86-SSE2-NEXT:    cvtpd2dq %xmm1, %xmm1
+; X86-SSE2-NEXT:    cvtpd2dq %xmm0, %xmm0
+; X86-SSE2-NEXT:    movlhps {{.*#+}} xmm0 = xmm0[0],xmm1[0]
 ; X86-SSE2-NEXT:    retl
 ;
 ; X86-AVX-LABEL: lrint_v4f64:
@@ -411,34 +386,12 @@ define <8 x iXLen> @lrint_v8f64(<8 x double> %x) {
 ; X86-SSE2-NEXT:    .cfi_def_cfa_register %ebp
 ; X86-SSE2-NEXT:    andl $-16, %esp
 ; X86-SSE2-NEXT:    subl $16, %esp
-; X86-SSE2-NEXT:    movapd %xmm0, %xmm3
-; X86-SSE2-NEXT:    movapd 8(%ebp), %xmm4
-; X86-SSE2-NEXT:    cvtsd2si %xmm1, %eax
-; X86-SSE2-NEXT:    movd %eax, %xmm5
-; X86-SSE2-NEXT:    unpckhpd {{.*#+}} xmm1 = xmm1[1,1]
-; X86-SSE2-NEXT:    cvtsd2si %xmm1, %eax
-; X86-SSE2-NEXT:    movd %eax, %xmm0
-; X86-SSE2-NEXT:    punpckldq {{.*#+}} xmm5 = xmm5[0],xmm0[0],xmm5[1],xmm0[1]
-; X86-SSE2-NEXT:    cvtsd2si %xmm3, %eax
-; X86-SSE2-NEXT:    movd %eax, %xmm0
-; X86-SSE2-NEXT:    unpckhpd {{.*#+}} xmm3 = xmm3[1,1]
-; X86-SSE2-NEXT:    cvtsd2si %xmm3, %eax
-; X86-SSE2-NEXT:    movd %eax, %xmm1
-; X86-SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
-; X86-SSE2-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm5[0]
-; X86-SSE2-NEXT:    cvtsd2si %xmm4, %eax
-; X86-SSE2-NEXT:    movd %eax, %xmm3
-; X86-SSE2-NEXT:    unpckhpd {{.*#+}} xmm4 = xmm4[1,1]
-; X86-SSE2-NEXT:    cvtsd2si %xmm4, %eax
-; X86-SSE2-NEXT:    movd %eax, %xmm1
-; X86-SSE2-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1]
-; X86-SSE2-NEXT:    cvtsd2si %xmm2, %eax
-; X86-SSE2-NEXT:    movd %eax, %xmm1
-; X86-SSE2-NEXT:    unpckhpd {{.*#+}} xmm2 = xmm2[1,1]
-; X86-SSE2-NEXT:    cvtsd2si %xmm2, %eax
-; X86-SSE2-NEXT:    movd %eax, %xmm2
-; X86-SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
-; X86-SSE2-NEXT:    punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm3[0]
+; X86-SSE2-NEXT:    cvtpd2dq %xmm1, %xmm1
+; X86-SSE2-NEXT:    cvtpd2dq %xmm0, %xmm0
+; X86-SSE2-NEXT:    movlhps {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; X86-SSE2-NEXT:    cvtpd2dq %xmm2, %xmm1
+; X86-SSE2-NEXT:    cvtpd2dq 8(%ebp), %xmm2
+; X86-SSE2-NEXT:    movlhps {{.*#+}} xmm1 = xmm1[0],xmm2[0]
 ; X86-SSE2-NEXT:    movl %ebp, %esp
 ; X86-SSE2-NEXT:    popl %ebp
 ; X86-SSE2-NEXT:    .cfi_def_cfa %esp, 4
diff --git a/llvm/test/CodeGen/X86/vector-mul.ll b/llvm/test/CodeGen/X86/vector-mul.ll
index 19bbf7dc0a0e1..a166bebae721c 100644
--- a/llvm/test/CodeGen/X86/vector-mul.ll
+++ b/llvm/test/CodeGen/X86/vector-mul.ll
@@ -812,6 +812,69 @@ define <16 x i16> @mul_v16i16_neg9(<16 x i16> %a0) nounwind {
   ret <16 x i16> %1
 }
 
+; PR47422 - check mul-shl-add sequence expands to adds.
+define <16 x i16> @madd_v16i16_3(<16 x i16> %a0, <16 x i16> %a1) nounwind {
+; X86-SSE-LABEL: madd_v16i16_3:
+; X86-SSE:       # %bb.0:
+; X86-SSE-NEXT:    pushl %ebp
+; X86-SSE-NEXT:    movl %esp, %ebp
+; X86-SSE-NEXT:    andl $-16, %esp
+; X86-SSE-NEXT:    subl $16, %esp
+; X86-SSE-NEXT:    movdqa %xmm1, %xmm3
+; X86-SSE-NEXT:    paddw %xmm1, %xmm3
+; X86-SSE-NEXT:    paddw %xmm3, %xmm1
+; X86-SSE-NEXT:    movdqa %xmm0, %xmm3
+; X86-SSE-NEXT:    paddw %xmm0, %xmm3
+; X86-SSE-NEXT:    paddw %xmm2, %xmm0
+; X86-SSE-NEXT:    paddw %xmm3, %xmm0
+; X86-SSE-NEXT:    paddw 8(%ebp), %xmm1
+; X86-SSE-NEXT:    movl %ebp, %esp
+; X86-SSE-NEXT:    popl %ebp
+; X86-SSE-NEXT:    retl
+;
+; X64-SSE-LABEL: madd_v16i16_3:
+; X64-SSE:       # %bb.0:
+; X64-SSE-NEXT:    movdqa %xmm1, %xmm4
+; X64-SSE-NEXT:    paddw %xmm1, %xmm4
+; X64-SSE-NEXT:    movdqa %xmm0, %xmm5
+; X64-SSE-NEXT:    paddw %xmm0, %xmm5
+; X64-SSE-NEXT:    paddw %xmm2, %xmm0
+; X64-SSE-NEXT:    paddw %xmm5, %xmm0
+; X64-SSE-NEXT:    paddw %xmm3, %xmm1
+; X64-SSE-NEXT:    paddw %xmm4, %xmm1
+; X64-SSE-NEXT:    retq
+;
+; X64-XOP-LABEL: madd_v16i16_3:
+; X64-XOP:       # %bb.0:
+; X64-XOP-NEXT:    vpaddw %xmm0, %xmm0, %xmm2
+; X64-XOP-NEXT:    vextractf128 $1, %ymm0, %xmm3
+; X64-XOP-NEXT:    vpaddw %xmm3, %xmm3, %xmm4
+; X64-XOP-NEXT:    vextractf128 $1, %ymm1, %xmm5
+; X64-XOP-NEXT:    vpaddw %xmm5, %xmm3, %xmm3
+; X64-XOP-NEXT:    vpaddw %xmm3, %xmm4, %xmm3
+; X64-XOP-NEXT:    vpaddw %xmm1, %xmm0, %xmm0
+; X64-XOP-NEXT:    vpaddw %xmm0, %xmm2, %xmm0
+; X64-XOP-NEXT:    vinsertf128 $1, %xmm3, %ymm0, %ymm0
+; X64-XOP-NEXT:    retq
+;
+; X64-AVX2-LABEL: madd_v16i16_3:
+; X64-AVX2:       # %bb.0:
+; X64-AVX2-NEXT:    vpaddw %ymm0, %ymm0, %ymm2
+; X64-AVX2-NEXT:    vpaddw %ymm1, %ymm0, %ymm0
+; X64-AVX2-NEXT:    vpaddw %ymm2, %ymm0, %ymm0
+; X64-AVX2-NEXT:    retq
+;
+; X64-AVX512DQ-LABEL: madd_v16i16_3:
+; X64-AVX512DQ:       # %bb.0:
+; X64-AVX512DQ-NEXT:    vpaddw %ymm0, %ymm0, %ymm2
+; X64-AVX512DQ-NEXT:    vpaddw %ymm1, %ymm0, %ymm0
+; X64-AVX512DQ-NEXT:    vpaddw %ymm2, %ymm0, %ymm0
+; X64-AVX512DQ-NEXT:    retq
+  %mul = mul <16 x i16> %a0, splat (i16 3)
+  %add = add <16 x i16> %mul, %a1
+  ret <16 x i16> %add
+}
+
 define <32 x i8> @mul_v32i8_neg5(<32 x i8> %a0) nounwind {
 ; SSE-LABEL: mul_v32i8_neg5:
 ; SSE:       # %bb.0:
diff --git a/llvm/test/CodeGen/X86/vector-reduce-fmax.ll b/llvm/test/CodeGen/X86/vector-reduce-fmax.ll
index fe2c41f57cfab..7048b98227620 100644
--- a/llvm/test/CodeGen/X86/vector-reduce-fmax.ll
+++ b/llvm/test/CodeGen/X86/vector-reduce-fmax.ll
@@ -1,6 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse2 | FileCheck %s --check-prefixes=ALL,SSE,SSE2
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse4.1 | FileCheck %s --check-prefixes=ALL,SSE,SSE41
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse2 | FileCheck %s --check-prefixes=ALL,SSE2
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse4.1 | FileCheck %s --check-prefixes=ALL,SSE41
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx | FileCheck %s --check-prefixes=ALL,AVX
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx2 | FileCheck %s --check-prefixes=ALL,AVX
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512bw | FileCheck %s --check-prefixes=ALL,AVX512,AVX512BW
@@ -36,13 +36,10 @@ define float @test_v2f32(<2 x float> %a0) {
 ; SSE41-LABEL: test_v2f32:
 ; SSE41:       # %bb.0:
 ; SSE41-NEXT:    movshdup {{.*#+}} xmm2 = xmm0[1,1,3,3]
-; SSE41-NEXT:    movaps %xmm0, %xmm1
-; SSE41-NEXT:    cmpunordss %xmm0, %xmm1
-; SSE41-NEXT:    movaps %xmm1, %xmm3
-; SSE41-NEXT:    andps %xmm2, %xmm3
-; SSE41-NEXT:    maxss %xmm0, %xmm2
-; SSE41-NEXT:    andnps %xmm2, %xmm1
-; SSE41-NEXT:    orps %xmm3, %xmm1
+; SSE41-NEXT:    movaps %xmm2, %xmm1
+; SSE41-NEXT:    maxss %xmm0, %xmm1
+; SSE41-NEXT:    cmpunordss %xmm0, %xmm0
+; SSE41-NEXT:    blendvps %xmm0, %xmm2, %xmm1
 ; SSE41-NEXT:    movaps %xmm1, %xmm0
 ; SSE41-NEXT:    retq
 ;
@@ -89,21 +86,19 @@ define float @test_v3f32(<3 x float> %a0) {
 ;
 ; SSE41-LABEL: test_v3f32:
 ; SSE41:       # %bb.0:
-; SSE41-NEXT:    movshdup {{.*#+}} xmm1 = xmm0[1,1,3,3]
-; SSE41-NEXT:    movaps %xmm0, %xmm2
-; SSE41-NEXT:    cmpunordss %xmm0, %xmm2
-; SSE41-NEXT:    movaps %xmm2, %xmm3
-; SSE41-NEXT:    andps %xmm1, %xmm3
-; SSE41-NEXT:    maxss %xmm0, %xmm1
-; SSE41-NEXT:    andnps %xmm1, %xmm2
-; SSE41-NEXT:    orps %xmm3, %xmm2
-; SSE41-NEXT:    movhlps {{.*#+}} xmm0 = xmm0[1,1]
 ; SSE41-NEXT:    movaps %xmm0, %xmm1
-; SSE41-NEXT:    maxss %xmm2, %xmm1
+; SSE41-NEXT:    movshdup {{.*#+}} xmm3 = xmm0[1,1,3,3]
+; SSE41-NEXT:    movaps %xmm3, %xmm2
+; SSE41-NEXT:    maxss %xmm0, %xmm2
+; SSE41-NEXT:    cmpunordss %xmm0, %xmm0
+; SSE41-NEXT:    blendvps %xmm0, %xmm3, %xmm2
+; SSE41-NEXT:    movhlps {{.*#+}} xmm1 = xmm1[1,1]
+; SSE41-NEXT:    movaps %xmm1, %xmm3
+; SSE41-NEXT:    maxss %xmm2, %xmm3
 ; SSE41-NEXT:    cmpunordss %xmm2, %xmm2
-; SSE41-NEXT:    andps %xmm2, %xmm0
-; SSE41-NEXT:    andnps %xmm1, %xmm2
-; SSE41-NEXT:    orps %xmm2, %xmm0
+; SSE41-NEXT:    movaps %xmm2, %xmm0
+; SSE41-NEXT:    blendvps %xmm0, %xmm1, %xmm3
+; SSE41-NEXT:    movaps %xmm3, %xmm0
 ; SSE41-NEXT:    retq
 ;
 ; AVX-LABEL: test_v3f32:
@@ -166,31 +161,26 @@ define float @test_v4f32(<4 x float> %a0) {
 ;
 ; SSE41-LABEL: test_v4f32:
 ; SSE41:       # %bb.0:
-; SSE41-NEXT:    movaps %xmm0, %xmm1
-; SSE41-NEXT:    movaps %xmm0, %xmm2
-; SSE41-NEXT:    movshdup {{.*#+}} xmm3 = xmm0[1,1,3,3]
-; SSE41-NEXT:    cmpunordss %xmm0, %xmm0
-; SSE41-NEXT:    movaps %xmm0, %xmm4
-; SSE41-NEXT:    andps %xmm3, %xmm4
-; SSE41-NEXT:    maxss %xmm1, %xmm3
-; SSE41-NEXT:    shufps {{.*#+}} xmm1 = xmm1[3,3,3,3]
-; SSE41-NEXT:    movhlps {{.*#+}} xmm2 = xmm2[1,1]
-; SSE41-NEXT:    andnps %xmm3, %xmm0
-; SSE41-NEXT:    orps %xmm4, %xmm0
-; SSE41-NEXT:    movaps %xmm2, %xmm3
-; SSE41-NEXT:    maxss %xmm0, %xmm3
-; SSE41-NEXT:    cmpunordss %xmm0, %xmm0
+; SSE41-NEXT:    movaps %xmm0, %xmm3
+; SSE41-NEXT:    shufps {{.*#+}} xmm3 = xmm3[3,3],xmm0[3,3]
 ; SSE41-NEXT:    movaps %xmm0, %xmm4
-; SSE41-NEXT:    andnps %xmm3, %xmm4
-; SSE41-NEXT:    andps %xmm2, %xmm0
-; SSE41-NEXT:    orps %xmm4, %xmm0
-; SSE41-NEXT:    movaps %xmm1, %xmm2
-; SSE41-NEXT:    maxss %xmm0, %xmm2
+; SSE41-NEXT:    unpckhpd {{.*#+}} xmm4 = xmm4[1],xmm0[1]
+; SSE41-NEXT:    movshdup {{.*#+}} xmm2 = xmm0[1,1,3,3]
+; SSE41-NEXT:    movaps %xmm2, %xmm1
+; SSE41-NEXT:    maxss %xmm0, %xmm1
 ; SSE41-NEXT:    cmpunordss %xmm0, %xmm0
-; SSE41-NEXT:    movaps %xmm0, %xmm3
-; SSE41-NEXT:    andnps %xmm2, %xmm3
-; SSE41-NEXT:    andps %xmm1, %xmm0
-; SSE41-NEXT:    orps %xmm3, %xmm0
+; SSE41-NEXT:    blendvps %xmm0, %xmm2, %xmm1
+; SSE41-NEXT:    movaps %xmm4, %xmm2
+; SSE41-NEXT:    maxss %xmm1, %xmm2
+; SSE41-NEXT:    cmpunordss %xmm1, %xmm1
+; SSE41-NEXT:    movaps %xmm1, %xmm0
+; SSE41-NEXT:    blendvps %xmm0, %xmm4, %xmm2
+; SSE41-NEXT:    movaps %xmm3, %xmm1
+; SSE41-NEXT:    maxss %xmm2, %xmm1
+; SSE41-NEXT:    cmpunordss %xmm2, %xmm2
+; SSE41-NEXT:    movaps %xmm2, %xmm0
+; SSE41-NEXT:    blendvps %xmm0, %xmm3, %xmm1
+; SSE41-NEXT:    movaps %xmm1, %xmm0
 ; SSE41-NEXT:    retq
 ;
 ; AVX-LABEL: test_v4f32:
@@ -266,35 +256,30 @@ define float @test_v8f32(<8 x float> %a0) {
 ;
 ; SSE41-LABEL: test_v8f32:
 ; SSE41:       # %bb.0:
-; SSE41-NEXT:    movaps %xmm1, %xmm2
-; SSE41-NEXT:    maxps %xmm0, %xmm2
-; SSE41-NEXT:    cmpunordps %xmm0, %xmm0
-; SSE41-NEXT:    blendvps %xmm0, %xmm1, %xmm2
-; SSE41-NEXT:    movshdup {{.*#+}} xmm1 = xmm2[1,1,3,3]
-; SSE41-NEXT:    movaps %xmm2, %xmm0
-; SSE41-NEXT:    cmpunordss %xmm2, %xmm0
-; SSE41-NEXT:    movaps %xmm0, %xmm3
-; SSE41-NEXT:    andps %xmm1, %xmm3
-; SSE41-NEXT:    maxss %xmm2, %xmm1
-; SSE41-NEXT:    andnps %xmm1, %xmm0
-; SSE41-NEXT:    orps %xmm3, %xmm0
-; SSE41-NEXT:    movaps %xmm2, %xmm1
-; SSE41-NEXT:    unpckhpd {{.*#+}} xmm1 = xmm1[1],xmm2[1]
 ; SSE41-NEXT:    movaps %xmm1, %xmm3
-; SSE41-NEXT:    maxss %xmm0, %xmm3
-; SSE41-NEXT:    cmpunordss %xmm0, %xmm0
-; SSE41-NEXT:    movaps %xmm0, %xmm4
-; SSE41-NEXT:    andnps %xmm3, %xmm4
-; SSE41-NEXT:    andps %xmm1, %xmm0
-; SSE41-NEXT:    orps %xmm4, %xmm0
-; SSE41-NEXT:    shufps {{.*#+}} xmm2 = xmm2[3,3,3,3]
+; SSE41-NEXT:    maxps %xmm0, %xmm3
+; SSE41-NEXT:    cmpunordps %xmm0, %xmm0
+; SSE41-NEXT:    blendvps %xmm0, %xmm1, %xmm3
+; SSE41-NEXT:    movshdup {{.*#+}} xmm2 = xmm3[1,1,3,3]
 ; SSE41-NEXT:    movaps %xmm2, %xmm1
-; SSE41-NEXT:    maxss %xmm0, %xmm1
-; SSE41-NEXT:    cmpunordss %xmm0, %xmm0
-; SSE41-NEXT:    movaps %xmm0, %xmm3
-; SSE41-NEXT:    andnps %xmm1, %xmm3
-; SSE41-NEXT:    andps %xmm2, %xmm0
-; SSE41-NEXT:    orps %xmm3, %xmm0
+; SSE41-NEXT:    maxss %xmm3, %xmm1
+; SSE41-NEXT:    movaps %xmm3, %xmm0
+; SSE41-NEXT:    cmpunordss %xmm3, %xmm0
+; SSE41-NEXT:    blendvps %xmm0, %xmm2, %xmm1
+; SSE41-NEXT:    movaps %xmm3, %xmm4
+; SSE41-NEXT:    unpckhpd {{.*#+}} xmm4 = xmm4[1],xmm3[1]
+; SSE41-NEXT:    movaps %xmm4, %xmm2
+; SSE41-NEXT:    maxss %xmm1, %xmm2
+; SSE41-NEXT:    cmpunordss %xmm1, %xmm1
+; SSE41-NEXT:    movaps %xmm1, %xmm0
+; SSE41-NEXT:    blendvps %xmm0, %xmm4, %xmm2
+; SSE41-NEXT:    shufps {{.*#+}} xmm3 = xmm3[3,3,3,3]
+; SSE41-NEXT:    movaps %xmm3, %xmm1
+; SSE41-NEXT:    maxss %xmm2, %xmm1
+; SSE41-NEXT:    cmpunordss %xmm2, %xmm2
+; SSE41-NEXT:    movaps %xmm2, %xmm0
+; SSE41-NEXT:    blendvps %xmm0, %xmm3, %xmm1
+; SSE41-NEXT:    movaps %xmm1, %xmm0
 ; SSE41-NEXT:    retq
 ;
 ; AVX-LABEL: test_v8f32:
@@ -458,36 +443,31 @@ define float @test_v16f32(<16 x float> %a0) {
 ; SSE41-NEXT:    cmpunordps %xmm1, %xmm1
 ; SSE41-NEXT:    movaps %xmm1, %xmm0
 ; SSE41-NEXT:    blendvps %xmm0, %xmm3, %xmm2
-; SSE41-NEXT:    movaps %xmm2, %xmm1
-; SSE41-NEXT:    maxps %xmm4, %xmm1
+; SSE41-NEXT:    movaps %xmm2, %xmm3
+; SSE41-NEXT:    maxps %xmm4, %xmm3
 ; SSE41-NEXT:    cmpunordps %xmm4, %xmm4
 ; SSE41-NEXT:    movaps %xmm4, %xmm0
+; SSE41-NEXT:    blendvps %xmm0, %xmm2, %xmm3
+; SSE41-NEXT:    movshdup {{.*#+}} xmm2 = xmm3[1,1,3,3]
+; SSE41-NEXT:    movaps %xmm2, %xmm1
+; SSE41-NEXT:    maxss %xmm3, %xmm1
+; SSE41-NEXT:    movaps %xmm3, %xmm0
+; SSE41-NEXT:    cmpunordss %xmm3, %xmm0
 ; SSE41-NEXT:    blendvps %xmm0, %xmm2, %xmm1
-; SSE41-NEXT:    movshdup {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; SSE41-NEXT:    movaps %xmm1, %xmm0
-; SSE41-NEXT:    cmpunordss %xmm1, %xmm0
-; SSE41-NEXT:    movaps %xmm0, %xmm3
-; SSE41-NEXT:    andps %xmm2, %xmm3
+; SSE41-NEXT:    movaps %xmm3, %xmm4
+; SSE41-NEXT:    unpckhpd {{.*#+}} xmm4 = xmm4[1],xmm3[1]
+; SSE41-NEXT:    movaps %xmm4, %xmm2
 ; SSE41-NEXT:    maxss %xmm1, %xmm2
-; SSE41-NEXT:    andnps %xmm2, %xmm0
-; SSE41-NEXT:    orps %xmm3, %xmm0
-; SSE41-NEXT:    movaps %xmm1, %xmm2
-; SSE41-NEXT:    unpckhpd {{.*#+}} xmm2 = xmm2[1],xmm1[1]
-; SSE41-NEXT:    movaps %xmm2, %xmm3
-; SSE41-NEXT:    maxss %xmm0, %xmm3
-; SSE41-NEXT:    cmpunordss %xmm0, %xmm0
-; SSE41-NEXT:    movaps %xmm0, %xmm4
-; SSE41-NEXT:    andnps %xmm3, %xmm4
-; SSE41-NEXT:    andps %xmm2, %xmm0
-; SSE41-NEXT:    orps %xmm4, %xmm0
-; SSE41-NEXT:    shufps {{.*#+}} xmm1 = xmm1[3,3,3,3]
-; SSE41-NEXT:    movaps %xmm1, %xmm2
-; SSE41-NEXT:    maxss %xmm0, %xmm2
-; SSE41-NEXT:    cmpunordss %xmm0, %xmm0
-; SSE41-NEXT:    movaps %xmm0, %xmm3
-; SSE41-NEXT:    andnps %xmm2, %xmm3
-; SSE41-NEXT:    andps %xmm1, %xmm0
-; SSE41-NEXT:    orps %xmm3, %xmm0
+; SSE41-NEXT:    cmpunordss %xmm1, %xmm1
+; SSE41-NEXT:    movaps %xmm1, %xmm0
+; SSE41-NEXT:    blendvps %xmm0, %xmm4, %xmm2
+; SSE41-NEXT:    shufps {{.*#+}} xmm3 = xmm3[3,3,3,3]
+; SSE41-NEXT:    movaps %xmm3, %xmm1
+; SSE41-NEXT:    maxss %xmm2, %xmm1
+; SSE41-NEXT:    cmpunordss %xmm2, %xmm2
+; SSE41-NEXT:    movaps %xmm2, %xmm0
+; SSE41-NEXT:    blendvps %xmm0, %xmm3, %xmm1
+; SSE41-NEXT:    movaps %xmm1, %xmm0
 ; SSE41-NEXT:    retq
 ;
 ; AVX-LABEL: test_v16f32:
@@ -664,19 +644,30 @@ define float @test_v16f32(<16 x float> %a0) {
 ;
 
 define double @test_v2f64(<2 x double> %a0) {
-; SSE-LABEL: test_v2f64:
-; SSE:       # %bb.0:
-; SSE-NEXT:    movapd %xmm0, %xmm2
-; SSE-NEXT:    unpckhpd {{.*#+}} xmm2 = xmm2[1],xmm0[1]
-; SSE-NEXT:    movapd %xmm0, %xmm1
-; SSE-NEXT:    cmpunordsd %xmm0, %xmm1
-; SSE-NEXT:    movapd %xmm1, %xmm3
-; SSE-NEXT:    andpd %xmm2, %xmm3
-; SSE-NEXT:    maxsd %xmm0, %xmm2
-; SSE-NEXT:    andnpd %xmm2, %xmm1
-; SSE-NEXT:    orpd %xmm3, %xmm1
-; SSE-NEXT:    movapd %xmm1, %xmm0
-; SSE-NEXT:    retq
+; SSE2-LABEL: test_v2f64:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    movapd %xmm0, %xmm2
+; SSE2-NEXT:    unpckhpd {{.*#+}} xmm2 = xmm2[1],xmm0[1]
+; SSE2-NEXT:    movapd %xmm0, %xmm1
+; SSE2-NEXT:    cmpunordsd %xmm0, %xmm1
+; SSE2-NEXT:    movapd %xmm1, %xmm3
+; SSE2-NEXT:    andpd %xmm2, %xmm3
+; SSE2-NEXT:    maxsd %xmm0, %xmm2
+; SSE2-NEXT:    andnpd %xmm2, %xmm1
+; SSE2-NEXT:    orpd %xmm3, %xmm1
+; SSE2-NEXT:    movapd %xmm1, %xmm0
+; SSE2-NEXT:    retq
+;
+; SSE41-LABEL: test_v2f64:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    movapd %xmm0, %xmm2
+; SSE41-NEXT:    unpckhpd {{.*#+}} xmm2 = xmm2[1],xmm0[1]
+; SSE41-NEXT:    movapd %xmm2, %xmm1
+; SSE41-NEXT:    maxsd %xmm0, %xmm1
+; SSE41-NEXT:    cmpunordsd %xmm0, %xmm0
+; SSE41-NEXT:    blendvpd %xmm0, %xmm2, %xmm1
+; SSE41-NEXT:    movapd %xmm1, %xmm0
+; SSE41-NEXT:    retq
 ;
 ; AVX-LABEL: test_v2f64:
 ; AVX:       # %bb.0:
@@ -724,15 +715,14 @@ define double @test_v4f64(<4 x double> %a0) {
 ; SSE41-NEXT:    maxpd %xmm0, %xmm2
 ; SSE41-NEXT:    cmpunordpd %xmm0, %xmm0
 ; SSE41-NEXT:    blendvpd %xmm0, %xmm1, %xmm2
-; SSE41-NEXT:    movapd %xmm2, %xmm1
-; SSE41-NEXT:    unpckhpd {{.*#+}} xmm1 = xmm1[1],xmm2[1]
-; SSE41-NEXT:    movapd %xmm2, %xmm0
-; SSE41-NEXT:    cmpunordsd %xmm2, %xmm0
-; SSE41-NEXT:    movapd %xmm0, %xmm3
-; SSE41-NEXT:    andpd %xmm1, %xmm3
+; SSE41-NEXT:    movapd %xmm2, %xmm3
+; SSE41-NEXT:    unpckhpd {{.*#+}} xmm3 = xmm3[1],xmm2[1]
+; SSE41-NEXT:    movapd %xmm3, %xmm1
 ; SSE41-NEXT:    maxsd %xmm2, %xmm1
-; SSE41-NEXT:    andnpd %xmm1, %xmm0
-; SSE41-NEXT:    orpd %xmm3, %xmm0
+; SSE41-NEXT:    cmpunordsd %xmm2, %xmm2
+; SSE41-NEXT:    movapd %xmm2, %xmm0
+; SSE41-NEXT:    blendvpd %xmm0, %xmm3, %xmm1
+; SSE41-NEXT:    movapd %xmm1, %xmm0
 ; SSE41-NEXT:    retq
 ;
 ; AVX-LABEL: test_v4f64:
@@ -820,15 +810,14 @@ define double @test_v8f64(<8 x double> %a0) {
 ; SSE41-NEXT:    cmpunordpd %xmm4, %xmm4
 ; SSE41-NEXT:    movapd %xmm4, %xmm0
 ; SSE41-NEXT:    blendvpd %xmm0, %xmm2, %xmm1
-; SSE41-NEXT:    movapd %xmm1, %xmm2
-; SSE41-NEXT:    unpckhpd {{.*#+}} xmm2 = xmm2[1],xmm1[1]
-; SSE41-NEXT:    movapd %xmm1, %xmm0
-; SSE41-NEXT:    cmpunordsd %xmm1, %xmm0
-; SSE41-NEXT:    movapd %xmm0, %xmm3
-; SSE41-NEXT:    andpd %xmm2, %xmm3
+; SSE41-NEXT:    movapd %xmm1, %xmm3
+; SSE41-NEXT:    unpckhpd {{.*#+}} xmm3 = xmm3[1],xmm1[1]
+; SSE41-NEXT:    movapd %xmm3, %xmm2
 ; SSE41-NEXT:    maxsd %xmm1, %xmm2
-; SSE41-NEXT:    andnpd %xmm2, %xmm0
-; SSE41-NEXT:    orpd %xmm3, %xmm0
+; SSE41-NEXT:    cmpunordsd %xmm1, %xmm1
+; SSE41-NEXT:    movapd %xmm1, %xmm0
+; SSE41-NEXT:    blendvpd %xmm0, %xmm3, %xmm2
+; SSE41-NEXT:    movapd %xmm2, %xmm0
 ; SSE41-NEXT:    retq
 ;
 ; AVX-LABEL: test_v8f64:
@@ -1012,15 +1001,14 @@ define double @test_v16f64(<16 x double> %a0) {
 ; SSE41-NEXT:    cmpunordpd %xmm2, %xmm2
 ; SSE41-NEXT:    movapd %xmm2, %xmm0
 ; SSE41-NEXT:    blendvpd %xmm0, %xmm3, %xmm1
-; SSE41-NEXT:    movapd %xmm1, %xmm2
-; SSE41-NEXT:    unpckhpd {{.*#+}} xmm2 = xmm2[1],xmm1[1]
-; SSE41-NEXT:    movapd %xmm1, %xmm0
-; SSE41-NEXT:    cmpunordsd %xmm1, %xmm0
-; SSE41-NEXT:    movapd %xmm0, %xmm3
-; SSE41-NEXT:    andpd %xmm2, %xmm3
+; SSE41-NEXT:    movapd %xmm1, %xmm3
+; SSE41-NEXT:    unpckhpd {{.*#+}} xmm3 = xmm3[1],xmm1[1]
+; SSE41-NEXT:    movapd %xmm3, %xmm2
 ; SSE41-NEXT:    maxsd %xmm1, %xmm2
-; SSE41-NEXT:    andnpd %xmm2, %xmm0
-; SSE41-NEXT:    orpd %xmm3, %xmm0
+; SSE41-NEXT:    cmpunordsd %xmm1, %xmm1
+; SSE41-NEXT:    movapd %xmm1, %xmm0
+; SSE41-NEXT:    blendvpd %xmm0, %xmm3, %xmm2
+; SSE41-NEXT:    movapd %xmm2, %xmm0
 ; SSE41-NEXT:    retq
 ;
 ; AVX-LABEL: test_v16f64:
diff --git a/llvm/test/CodeGen/X86/vector-reduce-fmaximum.ll b/llvm/test/CodeGen/X86/vector-reduce-fmaximum.ll
index ec41657d2f248..008e3e4c217cb 100644
--- a/llvm/test/CodeGen/X86/vector-reduce-fmaximum.ll
+++ b/llvm/test/CodeGen/X86/vector-reduce-fmaximum.ll
@@ -49,22 +49,19 @@ define float @test_v2f32(<2 x float> %a0) {
 ; SSE41-NEXT:    movshdup {{.*#+}} xmm2 = xmm0[1,1,3,3]
 ; SSE41-NEXT:    movd %xmm0, %eax
 ; SSE41-NEXT:    testl %eax, %eax
+; SSE41-NEXT:    js .LBB1_1
+; SSE41-NEXT:  # %bb.2:
 ; SSE41-NEXT:    movaps %xmm0, %xmm3
-; SSE41-NEXT:    js .LBB1_2
-; SSE41-NEXT:  # %bb.1:
+; SSE41-NEXT:    jmp .LBB1_3
+; SSE41-NEXT:  .LBB1_1:
 ; SSE41-NEXT:    movaps %xmm2, %xmm3
-; SSE41-NEXT:  .LBB1_2:
-; SSE41-NEXT:    movaps %xmm3, %xmm1
-; SSE41-NEXT:    cmpunordss %xmm3, %xmm1
-; SSE41-NEXT:    movaps %xmm1, %xmm4
-; SSE41-NEXT:    andps %xmm3, %xmm4
-; SSE41-NEXT:    js .LBB1_4
-; SSE41-NEXT:  # %bb.3:
 ; SSE41-NEXT:    movaps %xmm0, %xmm2
-; SSE41-NEXT:  .LBB1_4:
-; SSE41-NEXT:    maxss %xmm2, %xmm3
-; SSE41-NEXT:    andnps %xmm3, %xmm1
-; SSE41-NEXT:    orps %xmm4, %xmm1
+; SSE41-NEXT:  .LBB1_3:
+; SSE41-NEXT:    movaps %xmm2, %xmm1
+; SSE41-NEXT:    maxss %xmm3, %xmm1
+; SSE41-NEXT:    movaps %xmm2, %xmm0
+; SSE41-NEXT:    cmpunordss %xmm2, %xmm0
+; SSE41-NEXT:    blendvps %xmm0, %xmm2, %xmm1
 ; SSE41-NEXT:    movaps %xmm1, %xmm0
 ; SSE41-NEXT:    retq
 ;
@@ -171,65 +168,57 @@ define float @test_v4f32(<4 x float> %a0) {
 ;
 ; SSE41-LABEL: test_v4f32:
 ; SSE41:       # %bb.0:
+; SSE41-NEXT:    movaps %xmm0, %xmm1
+; SSE41-NEXT:    movaps %xmm0, %xmm2
+; SSE41-NEXT:    unpckhpd {{.*#+}} xmm2 = xmm2[1],xmm0[1]
 ; SSE41-NEXT:    movshdup {{.*#+}} xmm3 = xmm0[1,1,3,3]
 ; SSE41-NEXT:    movd %xmm0, %eax
 ; SSE41-NEXT:    testl %eax, %eax
-; SSE41-NEXT:    movaps %xmm0, %xmm4
-; SSE41-NEXT:    js .LBB2_2
-; SSE41-NEXT:  # %bb.1:
+; SSE41-NEXT:    js .LBB2_1
+; SSE41-NEXT:  # %bb.2:
+; SSE41-NEXT:    movaps %xmm1, %xmm0
+; SSE41-NEXT:    jmp .LBB2_3
+; SSE41-NEXT:  .LBB2_1:
+; SSE41-NEXT:    movaps %xmm3, %xmm0
+; SSE41-NEXT:    movaps %xmm1, %xmm3
+; SSE41-NEXT:  .LBB2_3:
+; SSE41-NEXT:    shufps {{.*#+}} xmm1 = xmm1[3,3,3,3]
 ; SSE41-NEXT:    movaps %xmm3, %xmm4
-; SSE41-NEXT:  .LBB2_2:
-; SSE41-NEXT:    movaps %xmm0, %xmm1
-; SSE41-NEXT:    unpckhpd {{.*#+}} xmm1 = xmm1[1],xmm0[1]
-; SSE41-NEXT:    movaps %xmm4, %xmm2
-; SSE41-NEXT:    cmpunordss %xmm4, %xmm2
-; SSE41-NEXT:    movaps %xmm2, %xmm5
-; SSE41-NEXT:    andps %xmm4, %xmm5
-; SSE41-NEXT:    js .LBB2_4
-; SSE41-NEXT:  # %bb.3:
-; SSE41-NEXT:    movaps %xmm0, %xmm3
-; SSE41-NEXT:  .LBB2_4:
-; SSE41-NEXT:    maxss %xmm3, %xmm4
-; SSE41-NEXT:    andnps %xmm4, %xmm2
-; SSE41-NEXT:    orps %xmm5, %xmm2
-; SSE41-NEXT:    movd %xmm2, %eax
+; SSE41-NEXT:    maxss %xmm0, %xmm4
+; SSE41-NEXT:    movaps %xmm3, %xmm0
+; SSE41-NEXT:    cmpunordss %xmm3, %xmm0
+; SSE41-NEXT:    blendvps %xmm0, %xmm3, %xmm4
+; SSE41-NEXT:    movd %xmm4, %eax
 ; SSE41-NEXT:    testl %eax, %eax
-; SSE41-NEXT:    movaps %xmm2, %xmm4
-; SSE41-NEXT:    js .LBB2_6
+; SSE41-NEXT:    js .LBB2_4
 ; SSE41-NEXT:  # %bb.5:
-; SSE41-NEXT:    movaps %xmm1, %xmm4
+; SSE41-NEXT:    movaps %xmm4, %xmm0
+; SSE41-NEXT:    jmp .LBB2_6
+; SSE41-NEXT:  .LBB2_4:
+; SSE41-NEXT:    movaps %xmm2, %xmm0
+; SSE41-NEXT:    movaps %xmm4, %xmm2
 ; SSE41-NEXT:  .LBB2_6:
-; SSE41-NEXT:    shufps {{.*#+}} xmm0 = xmm0[3,3,3,3]
-; SSE41-NEXT:    movaps %xmm4, %xmm3
-; SSE41-NEXT:    cmpunordss %xmm4, %xmm3
-; SSE41-NEXT:    movaps %xmm3, %xmm5
-; SSE41-NEXT:    andps %xmm4, %xmm5
-; SSE41-NEXT:    js .LBB2_8
-; SSE41-NEXT:  # %bb.7:
-; SSE41-NEXT:    movaps %xmm2, %xmm1
-; SSE41-NEXT:  .LBB2_8:
-; SSE41-NEXT:    maxss %xmm1, %xmm4
-; SSE41-NEXT:    andnps %xmm4, %xmm3
-; SSE41-NEXT:    orps %xmm5, %xmm3
+; SSE41-NEXT:    movaps %xmm2, %xmm3
+; SSE41-NEXT:    maxss %xmm0, %xmm3
+; SSE41-NEXT:    movaps %xmm2, %xmm0
+; SSE41-NEXT:    cmpunordss %xmm2, %xmm0
+; SSE41-NEXT:    blendvps %xmm0, %xmm2, %xmm3
 ; SSE41-NEXT:    movd %xmm3, %eax
 ; SSE41-NEXT:    testl %eax, %eax
-; SSE41-NEXT:    movaps %xmm3, %xmm2
-; SSE41-NEXT:    js .LBB2_10
-; SSE41-NEXT:  # %bb.9:
-; SSE41-NEXT:    movaps %xmm0, %xmm2
-; SSE41-NEXT:  .LBB2_10:
-; SSE41-NEXT:    movaps %xmm2, %xmm1
-; SSE41-NEXT:    cmpunordss %xmm2, %xmm1
-; SSE41-NEXT:    movaps %xmm1, %xmm4
-; SSE41-NEXT:    andps %xmm2, %xmm4
-; SSE41-NEXT:    js .LBB2_12
-; SSE41-NEXT:  # %bb.11:
+; SSE41-NEXT:    js .LBB2_7
+; SSE41-NEXT:  # %bb.8:
 ; SSE41-NEXT:    movaps %xmm3, %xmm0
-; SSE41-NEXT:  .LBB2_12:
+; SSE41-NEXT:    jmp .LBB2_9
+; SSE41-NEXT:  .LBB2_7:
+; SSE41-NEXT:    movaps %xmm1, %xmm0
+; SSE41-NEXT:    movaps %xmm3, %xmm1
+; SSE41-NEXT:  .LBB2_9:
+; SSE41-NEXT:    movaps %xmm1, %xmm2
 ; SSE41-NEXT:    maxss %xmm0, %xmm2
-; SSE41-NEXT:    andnps %xmm2, %xmm1
-; SSE41-NEXT:    orps %xmm4, %xmm1
 ; SSE41-NEXT:    movaps %xmm1, %xmm0
+; SSE41-NEXT:    cmpunordss %xmm1, %xmm0
+; SSE41-NEXT:    blendvps %xmm0, %xmm1, %xmm2
+; SSE41-NEXT:    movaps %xmm2, %xmm0
 ; SSE41-NEXT:    retq
 ;
 ; AVX-LABEL: test_v4f32:
@@ -410,61 +399,53 @@ define float @test_v8f32(<8 x float> %a0) {
 ; SSE41-NEXT:    movshdup {{.*#+}} xmm1 = xmm2[1,1,3,3]
 ; SSE41-NEXT:    movd %xmm2, %eax
 ; SSE41-NEXT:    testl %eax, %eax
-; SSE41-NEXT:    movaps %xmm2, %xmm3
-; SSE41-NEXT:    js .LBB3_2
-; SSE41-NEXT:  # %bb.1:
+; SSE41-NEXT:    js .LBB3_1
+; SSE41-NEXT:  # %bb.2:
+; SSE41-NEXT:    movaps %xmm2, %xmm0
+; SSE41-NEXT:    jmp .LBB3_3
+; SSE41-NEXT:  .LBB3_1:
+; SSE41-NEXT:    movaps %xmm1, %xmm0
+; SSE41-NEXT:    movaps %xmm2, %xmm1
+; SSE41-NEXT:  .LBB3_3:
 ; SSE41-NEXT:    movaps %xmm1, %xmm3
-; SSE41-NEXT:  .LBB3_2:
-; SSE41-NEXT:    movaps %xmm3, %xmm0
-; SSE41-NEXT:    cmpunordss %xmm3, %xmm0
-; SSE41-NEXT:    movaps %xmm0, %xmm4
-; SSE41-NEXT:    andps %xmm3, %xmm4
-; SSE41-NEXT:    js .LBB3_4
-; SSE41-NEXT:  # %bb.3:
+; SSE41-NEXT:    maxss %xmm0, %xmm3
+; SSE41-NEXT:    movaps %xmm1, %xmm0
+; SSE41-NEXT:    cmpunordss %xmm1, %xmm0
+; SSE41-NEXT:    blendvps %xmm0, %xmm1, %xmm3
+; SSE41-NEXT:    movd %xmm3, %eax
 ; SSE41-NEXT:    movaps %xmm2, %xmm1
-; SSE41-NEXT:  .LBB3_4:
-; SSE41-NEXT:    maxss %xmm1, %xmm3
-; SSE41-NEXT:    andnps %xmm3, %xmm0
-; SSE41-NEXT:    orps %xmm4, %xmm0
-; SSE41-NEXT:    movd %xmm0, %eax
-; SSE41-NEXT:    movaps %xmm2, %xmm3
-; SSE41-NEXT:    unpckhpd {{.*#+}} xmm3 = xmm3[1],xmm2[1]
+; SSE41-NEXT:    unpckhpd {{.*#+}} xmm1 = xmm1[1],xmm2[1]
 ; SSE41-NEXT:    testl %eax, %eax
-; SSE41-NEXT:    movaps %xmm0, %xmm4
-; SSE41-NEXT:    js .LBB3_6
+; SSE41-NEXT:    js .LBB3_4
 ; SSE41-NEXT:  # %bb.5:
-; SSE41-NEXT:    movaps %xmm3, %xmm4
+; SSE41-NEXT:    movaps %xmm3, %xmm0
+; SSE41-NEXT:    jmp .LBB3_6
+; SSE41-NEXT:  .LBB3_4:
+; SSE41-NEXT:    movaps %xmm1, %xmm0
+; SSE41-NEXT:    movaps %xmm3, %xmm1
 ; SSE41-NEXT:  .LBB3_6:
-; SSE41-NEXT:    movaps %xmm4, %xmm1
-; SSE41-NEXT:    cmpunordss %xmm4, %xmm1
-; SSE41-NEXT:    movaps %xmm1, %xmm5
-; SSE41-NEXT:    andps %xmm4, %xmm5
-; SSE41-NEXT:    js .LBB3_8
-; SSE41-NEXT:  # %bb.7:
-; SSE41-NEXT:    movaps %xmm0, %xmm3
-; SSE41-NEXT:  .LBB3_8:
-; SSE41-NEXT:    maxss %xmm3, %xmm4
-; SSE41-NEXT:    andnps %xmm4, %xmm1
-; SSE41-NEXT:    orps %xmm5, %xmm1
-; SSE41-NEXT:    movd %xmm1, %eax
+; SSE41-NEXT:    movaps %xmm1, %xmm3
+; SSE41-NEXT:    maxss %xmm0, %xmm3
+; SSE41-NEXT:    movaps %xmm1, %xmm0
+; SSE41-NEXT:    cmpunordss %xmm1, %xmm0
+; SSE41-NEXT:    blendvps %xmm0, %xmm1, %xmm3
+; SSE41-NEXT:    movd %xmm3, %eax
 ; SSE41-NEXT:    shufps {{.*#+}} xmm2 = xmm2[3,3,3,3]
 ; SSE41-NEXT:    testl %eax, %eax
-; SSE41-NEXT:    movaps %xmm1, %xmm3
-; SSE41-NEXT:    js .LBB3_10
-; SSE41-NEXT:  # %bb.9:
-; SSE41-NEXT:    movaps %xmm2, %xmm3
-; SSE41-NEXT:  .LBB3_10:
+; SSE41-NEXT:    js .LBB3_7
+; SSE41-NEXT:  # %bb.8:
 ; SSE41-NEXT:    movaps %xmm3, %xmm0
-; SSE41-NEXT:    cmpunordss %xmm3, %xmm0
-; SSE41-NEXT:    movaps %xmm0, %xmm4
-; SSE41-NEXT:    andps %xmm3, %xmm4
-; SSE41-NEXT:    js .LBB3_12
-; SSE41-NEXT:  # %bb.11:
-; SSE41-NEXT:    movaps %xmm1, %xmm2
-; SSE41-NEXT:  .LBB3_12:
-; SSE41-NEXT:    maxss %xmm2, %xmm3
-; SSE41-NEXT:    andnps %xmm3, %xmm0
-; SSE41-NEXT:    orps %xmm4, %xmm0
+; SSE41-NEXT:    jmp .LBB3_9
+; SSE41-NEXT:  .LBB3_7:
+; SSE41-NEXT:    movaps %xmm2, %xmm0
+; SSE41-NEXT:    movaps %xmm3, %xmm2
+; SSE41-NEXT:  .LBB3_9:
+; SSE41-NEXT:    movaps %xmm2, %xmm1
+; SSE41-NEXT:    maxss %xmm0, %xmm1
+; SSE41-NEXT:    movaps %xmm2, %xmm0
+; SSE41-NEXT:    cmpunordss %xmm2, %xmm0
+; SSE41-NEXT:    blendvps %xmm0, %xmm2, %xmm1
+; SSE41-NEXT:    movaps %xmm1, %xmm0
 ; SSE41-NEXT:    retq
 ;
 ; AVX-LABEL: test_v8f32:
@@ -747,73 +728,65 @@ define float @test_v16f32(<16 x float> %a0) {
 ; SSE41-NEXT:    movaps %xmm2, %xmm0
 ; SSE41-NEXT:    cmpunordps %xmm2, %xmm0
 ; SSE41-NEXT:    blendvps %xmm0, %xmm2, %xmm1
-; SSE41-NEXT:    movaps %xmm1, %xmm2
+; SSE41-NEXT:    movaps %xmm1, %xmm3
 ; SSE41-NEXT:    movaps %xmm1, %xmm0
-; SSE41-NEXT:    blendvps %xmm0, %xmm5, %xmm2
+; SSE41-NEXT:    blendvps %xmm0, %xmm5, %xmm3
 ; SSE41-NEXT:    blendvps %xmm0, %xmm1, %xmm5
-; SSE41-NEXT:    movaps %xmm5, %xmm1
-; SSE41-NEXT:    maxps %xmm2, %xmm1
+; SSE41-NEXT:    movaps %xmm5, %xmm2
+; SSE41-NEXT:    maxps %xmm3, %xmm2
 ; SSE41-NEXT:    movaps %xmm5, %xmm0
 ; SSE41-NEXT:    cmpunordps %xmm5, %xmm0
-; SSE41-NEXT:    blendvps %xmm0, %xmm5, %xmm1
-; SSE41-NEXT:    movshdup {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; SSE41-NEXT:    movd %xmm1, %eax
+; SSE41-NEXT:    blendvps %xmm0, %xmm5, %xmm2
+; SSE41-NEXT:    movshdup {{.*#+}} xmm1 = xmm2[1,1,3,3]
+; SSE41-NEXT:    movd %xmm2, %eax
 ; SSE41-NEXT:    testl %eax, %eax
+; SSE41-NEXT:    js .LBB4_1
+; SSE41-NEXT:  # %bb.2:
+; SSE41-NEXT:    movaps %xmm2, %xmm0
+; SSE41-NEXT:    jmp .LBB4_3
+; SSE41-NEXT:  .LBB4_1:
+; SSE41-NEXT:    movaps %xmm1, %xmm0
+; SSE41-NEXT:    movaps %xmm2, %xmm1
+; SSE41-NEXT:  .LBB4_3:
 ; SSE41-NEXT:    movaps %xmm1, %xmm3
-; SSE41-NEXT:    js .LBB4_2
-; SSE41-NEXT:  # %bb.1:
-; SSE41-NEXT:    movaps %xmm2, %xmm3
-; SSE41-NEXT:  .LBB4_2:
-; SSE41-NEXT:    movaps %xmm3, %xmm0
-; SSE41-NEXT:    cmpunordss %xmm3, %xmm0
-; SSE41-NEXT:    movaps %xmm0, %xmm4
-; SSE41-NEXT:    andps %xmm3, %xmm4
-; SSE41-NEXT:    js .LBB4_4
-; SSE41-NEXT:  # %bb.3:
-; SSE41-NEXT:    movaps %xmm1, %xmm2
-; SSE41-NEXT:  .LBB4_4:
-; SSE41-NEXT:    maxss %xmm2, %xmm3
-; SSE41-NEXT:    andnps %xmm3, %xmm0
-; SSE41-NEXT:    orps %xmm4, %xmm0
-; SSE41-NEXT:    movd %xmm0, %eax
-; SSE41-NEXT:    movaps %xmm1, %xmm3
-; SSE41-NEXT:    unpckhpd {{.*#+}} xmm3 = xmm3[1],xmm1[1]
+; SSE41-NEXT:    maxss %xmm0, %xmm3
+; SSE41-NEXT:    movaps %xmm1, %xmm0
+; SSE41-NEXT:    cmpunordss %xmm1, %xmm0
+; SSE41-NEXT:    blendvps %xmm0, %xmm1, %xmm3
+; SSE41-NEXT:    movd %xmm3, %eax
+; SSE41-NEXT:    movaps %xmm2, %xmm1
+; SSE41-NEXT:    unpckhpd {{.*#+}} xmm1 = xmm1[1],xmm2[1]
 ; SSE41-NEXT:    testl %eax, %eax
-; SSE41-NEXT:    movaps %xmm0, %xmm4
-; SSE41-NEXT:    js .LBB4_6
+; SSE41-NEXT:    js .LBB4_4
 ; SSE41-NEXT:  # %bb.5:
-; SSE41-NEXT:    movaps %xmm3, %xmm4
+; SSE41-NEXT:    movaps %xmm3, %xmm0
+; SSE41-NEXT:    jmp .LBB4_6
+; SSE41-NEXT:  .LBB4_4:
+; SSE41-NEXT:    movaps %xmm1, %xmm0
+; SSE41-NEXT:    movaps %xmm3, %xmm1
 ; SSE41-NEXT:  .LBB4_6:
-; SSE41-NEXT:    movaps %xmm4, %xmm2
-; SSE41-NEXT:    cmpunordss %xmm4, %xmm2
-; SSE41-NEXT:    movaps %xmm2, %xmm5
-; SSE41-NEXT:    andps %xmm4, %xmm5
-; SSE41-NEXT:    js .LBB4_8
-; SSE41-NEXT:  # %bb.7:
-; SSE41-NEXT:    movaps %xmm0, %xmm3
-; SSE41-NEXT:  .LBB4_8:
-; SSE41-NEXT:    maxss %xmm3, %xmm4
-; SSE41-NEXT:    andnps %xmm4, %xmm2
-; SSE41-NEXT:    orps %xmm5, %xmm2
-; SSE41-NEXT:    movd %xmm2, %eax
-; SSE41-NEXT:    shufps {{.*#+}} xmm1 = xmm1[3,3,3,3]
-; SSE41-NEXT:    testl %eax, %eax
-; SSE41-NEXT:    movaps %xmm2, %xmm3
-; SSE41-NEXT:    js .LBB4_10
-; SSE41-NEXT:  # %bb.9:
 ; SSE41-NEXT:    movaps %xmm1, %xmm3
-; SSE41-NEXT:  .LBB4_10:
+; SSE41-NEXT:    maxss %xmm0, %xmm3
+; SSE41-NEXT:    movaps %xmm1, %xmm0
+; SSE41-NEXT:    cmpunordss %xmm1, %xmm0
+; SSE41-NEXT:    blendvps %xmm0, %xmm1, %xmm3
+; SSE41-NEXT:    movd %xmm3, %eax
+; SSE41-NEXT:    shufps {{.*#+}} xmm2 = xmm2[3,3,3,3]
+; SSE41-NEXT:    testl %eax, %eax
+; SSE41-NEXT:    js .LBB4_7
+; SSE41-NEXT:  # %bb.8:
 ; SSE41-NEXT:    movaps %xmm3, %xmm0
-; SSE41-NEXT:    cmpunordss %xmm3, %xmm0
-; SSE41-NEXT:    movaps %xmm0, %xmm4
-; SSE41-NEXT:    andps %xmm3, %xmm4
-; SSE41-NEXT:    js .LBB4_12
-; SSE41-NEXT:  # %bb.11:
+; SSE41-NEXT:    jmp .LBB4_9
+; SSE41-NEXT:  .LBB4_7:
+; SSE41-NEXT:    movaps %xmm2, %xmm0
+; SSE41-NEXT:    movaps %xmm3, %xmm2
+; SSE41-NEXT:  .LBB4_9:
 ; SSE41-NEXT:    movaps %xmm2, %xmm1
-; SSE41-NEXT:  .LBB4_12:
-; SSE41-NEXT:    maxss %xmm1, %xmm3
-; SSE41-NEXT:    andnps %xmm3, %xmm0
-; SSE41-NEXT:    orps %xmm4, %xmm0
+; SSE41-NEXT:    maxss %xmm0, %xmm1
+; SSE41-NEXT:    movaps %xmm2, %xmm0
+; SSE41-NEXT:    cmpunordss %xmm2, %xmm0
+; SSE41-NEXT:    blendvps %xmm0, %xmm2, %xmm1
+; SSE41-NEXT:    movaps %xmm1, %xmm0
 ; SSE41-NEXT:    retq
 ;
 ; AVX-LABEL: test_v16f32:
@@ -986,30 +959,52 @@ define float @test_v16f32(<16 x float> %a0) {
 ;
 
 define double @test_v2f64(<2 x double> %a0) {
-; SSE-LABEL: test_v2f64:
-; SSE:       # %bb.0:
-; SSE-NEXT:    movdqa %xmm0, %xmm2
-; SSE-NEXT:    punpckhqdq {{.*#+}} xmm2 = xmm2[1],xmm0[1]
-; SSE-NEXT:    movq %xmm0, %rax
-; SSE-NEXT:    testq %rax, %rax
-; SSE-NEXT:    movdqa %xmm0, %xmm3
-; SSE-NEXT:    js .LBB5_2
-; SSE-NEXT:  # %bb.1:
-; SSE-NEXT:    movdqa %xmm2, %xmm3
-; SSE-NEXT:  .LBB5_2:
-; SSE-NEXT:    movdqa %xmm3, %xmm1
-; SSE-NEXT:    cmpunordsd %xmm3, %xmm1
-; SSE-NEXT:    movapd %xmm1, %xmm4
-; SSE-NEXT:    andpd %xmm3, %xmm4
-; SSE-NEXT:    js .LBB5_4
-; SSE-NEXT:  # %bb.3:
-; SSE-NEXT:    movdqa %xmm0, %xmm2
-; SSE-NEXT:  .LBB5_4:
-; SSE-NEXT:    maxsd %xmm2, %xmm3
-; SSE-NEXT:    andnpd %xmm3, %xmm1
-; SSE-NEXT:    orpd %xmm4, %xmm1
-; SSE-NEXT:    movapd %xmm1, %xmm0
-; SSE-NEXT:    retq
+; SSE2-LABEL: test_v2f64:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    movdqa %xmm0, %xmm2
+; SSE2-NEXT:    punpckhqdq {{.*#+}} xmm2 = xmm2[1],xmm0[1]
+; SSE2-NEXT:    movq %xmm0, %rax
+; SSE2-NEXT:    testq %rax, %rax
+; SSE2-NEXT:    movdqa %xmm0, %xmm3
+; SSE2-NEXT:    js .LBB5_2
+; SSE2-NEXT:  # %bb.1:
+; SSE2-NEXT:    movdqa %xmm2, %xmm3
+; SSE2-NEXT:  .LBB5_2:
+; SSE2-NEXT:    movdqa %xmm3, %xmm1
+; SSE2-NEXT:    cmpunordsd %xmm3, %xmm1
+; SSE2-NEXT:    movapd %xmm1, %xmm4
+; SSE2-NEXT:    andpd %xmm3, %xmm4
+; SSE2-NEXT:    js .LBB5_4
+; SSE2-NEXT:  # %bb.3:
+; SSE2-NEXT:    movdqa %xmm0, %xmm2
+; SSE2-NEXT:  .LBB5_4:
+; SSE2-NEXT:    maxsd %xmm2, %xmm3
+; SSE2-NEXT:    andnpd %xmm3, %xmm1
+; SSE2-NEXT:    orpd %xmm4, %xmm1
+; SSE2-NEXT:    movapd %xmm1, %xmm0
+; SSE2-NEXT:    retq
+;
+; SSE41-LABEL: test_v2f64:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    movdqa %xmm0, %xmm2
+; SSE41-NEXT:    punpckhqdq {{.*#+}} xmm2 = xmm2[1],xmm0[1]
+; SSE41-NEXT:    movq %xmm0, %rax
+; SSE41-NEXT:    testq %rax, %rax
+; SSE41-NEXT:    js .LBB5_1
+; SSE41-NEXT:  # %bb.2:
+; SSE41-NEXT:    movdqa %xmm0, %xmm3
+; SSE41-NEXT:    jmp .LBB5_3
+; SSE41-NEXT:  .LBB5_1:
+; SSE41-NEXT:    movdqa %xmm2, %xmm3
+; SSE41-NEXT:    movdqa %xmm0, %xmm2
+; SSE41-NEXT:  .LBB5_3:
+; SSE41-NEXT:    movdqa %xmm2, %xmm1
+; SSE41-NEXT:    maxsd %xmm3, %xmm1
+; SSE41-NEXT:    movdqa %xmm2, %xmm0
+; SSE41-NEXT:    cmpunordsd %xmm2, %xmm0
+; SSE41-NEXT:    blendvpd %xmm0, %xmm2, %xmm1
+; SSE41-NEXT:    movapd %xmm1, %xmm0
+; SSE41-NEXT:    retq
 ;
 ; AVX-LABEL: test_v2f64:
 ; AVX:       # %bb.0:
@@ -1092,34 +1087,32 @@ define double @test_v4f64(<4 x double> %a0) {
 ;
 ; SSE41-LABEL: test_v4f64:
 ; SSE41:       # %bb.0:
-; SSE41-NEXT:    movapd %xmm0, %xmm3
-; SSE41-NEXT:    blendvpd %xmm0, %xmm1, %xmm3
+; SSE41-NEXT:    movapd %xmm0, %xmm2
+; SSE41-NEXT:    blendvpd %xmm0, %xmm1, %xmm2
 ; SSE41-NEXT:    blendvpd %xmm0, %xmm0, %xmm1
-; SSE41-NEXT:    movapd %xmm1, %xmm2
-; SSE41-NEXT:    maxpd %xmm3, %xmm2
+; SSE41-NEXT:    movapd %xmm1, %xmm3
+; SSE41-NEXT:    maxpd %xmm2, %xmm3
 ; SSE41-NEXT:    movapd %xmm1, %xmm0
 ; SSE41-NEXT:    cmpunordpd %xmm1, %xmm0
-; SSE41-NEXT:    blendvpd %xmm0, %xmm1, %xmm2
-; SSE41-NEXT:    movapd %xmm2, %xmm1
-; SSE41-NEXT:    unpckhpd {{.*#+}} xmm1 = xmm1[1],xmm2[1]
-; SSE41-NEXT:    movq %xmm2, %rax
+; SSE41-NEXT:    blendvpd %xmm0, %xmm1, %xmm3
+; SSE41-NEXT:    movapd %xmm3, %xmm2
+; SSE41-NEXT:    unpckhpd {{.*#+}} xmm2 = xmm2[1],xmm3[1]
+; SSE41-NEXT:    movq %xmm3, %rax
 ; SSE41-NEXT:    testq %rax, %rax
-; SSE41-NEXT:    movapd %xmm2, %xmm3
-; SSE41-NEXT:    js .LBB6_2
-; SSE41-NEXT:  # %bb.1:
-; SSE41-NEXT:    movapd %xmm1, %xmm3
-; SSE41-NEXT:  .LBB6_2:
+; SSE41-NEXT:    js .LBB6_1
+; SSE41-NEXT:  # %bb.2:
 ; SSE41-NEXT:    movapd %xmm3, %xmm0
-; SSE41-NEXT:    cmpunordsd %xmm3, %xmm0
-; SSE41-NEXT:    movapd %xmm0, %xmm4
-; SSE41-NEXT:    andpd %xmm3, %xmm4
-; SSE41-NEXT:    js .LBB6_4
-; SSE41-NEXT:  # %bb.3:
+; SSE41-NEXT:    jmp .LBB6_3
+; SSE41-NEXT:  .LBB6_1:
+; SSE41-NEXT:    movapd %xmm2, %xmm0
+; SSE41-NEXT:    movapd %xmm3, %xmm2
+; SSE41-NEXT:  .LBB6_3:
 ; SSE41-NEXT:    movapd %xmm2, %xmm1
-; SSE41-NEXT:  .LBB6_4:
-; SSE41-NEXT:    maxsd %xmm1, %xmm3
-; SSE41-NEXT:    andnpd %xmm3, %xmm0
-; SSE41-NEXT:    orpd %xmm4, %xmm0
+; SSE41-NEXT:    maxsd %xmm0, %xmm1
+; SSE41-NEXT:    movapd %xmm2, %xmm0
+; SSE41-NEXT:    cmpunordsd %xmm2, %xmm0
+; SSE41-NEXT:    blendvpd %xmm0, %xmm2, %xmm1
+; SSE41-NEXT:    movapd %xmm1, %xmm0
 ; SSE41-NEXT:    retq
 ;
 ; AVX-LABEL: test_v4f64:
@@ -1310,22 +1303,20 @@ define double @test_v8f64(<8 x double> %a0) {
 ; SSE41-NEXT:    unpckhpd {{.*#+}} xmm2 = xmm2[1],xmm1[1]
 ; SSE41-NEXT:    movq %xmm1, %rax
 ; SSE41-NEXT:    testq %rax, %rax
-; SSE41-NEXT:    movapd %xmm1, %xmm3
-; SSE41-NEXT:    js .LBB7_2
-; SSE41-NEXT:  # %bb.1:
-; SSE41-NEXT:    movapd %xmm2, %xmm3
-; SSE41-NEXT:  .LBB7_2:
-; SSE41-NEXT:    movapd %xmm3, %xmm0
-; SSE41-NEXT:    cmpunordsd %xmm3, %xmm0
-; SSE41-NEXT:    movapd %xmm0, %xmm4
-; SSE41-NEXT:    andpd %xmm3, %xmm4
-; SSE41-NEXT:    js .LBB7_4
-; SSE41-NEXT:  # %bb.3:
+; SSE41-NEXT:    js .LBB7_1
+; SSE41-NEXT:  # %bb.2:
+; SSE41-NEXT:    movapd %xmm1, %xmm0
+; SSE41-NEXT:    jmp .LBB7_3
+; SSE41-NEXT:  .LBB7_1:
+; SSE41-NEXT:    movapd %xmm2, %xmm0
 ; SSE41-NEXT:    movapd %xmm1, %xmm2
-; SSE41-NEXT:  .LBB7_4:
-; SSE41-NEXT:    maxsd %xmm2, %xmm3
-; SSE41-NEXT:    andnpd %xmm3, %xmm0
-; SSE41-NEXT:    orpd %xmm4, %xmm0
+; SSE41-NEXT:  .LBB7_3:
+; SSE41-NEXT:    movapd %xmm2, %xmm1
+; SSE41-NEXT:    maxsd %xmm0, %xmm1
+; SSE41-NEXT:    movapd %xmm2, %xmm0
+; SSE41-NEXT:    cmpunordsd %xmm2, %xmm0
+; SSE41-NEXT:    blendvpd %xmm0, %xmm2, %xmm1
+; SSE41-NEXT:    movapd %xmm1, %xmm0
 ; SSE41-NEXT:    retq
 ;
 ; AVX-LABEL: test_v8f64:
@@ -1646,22 +1637,20 @@ define double @test_v16f64(<16 x double> %a0) {
 ; SSE41-NEXT:    unpckhpd {{.*#+}} xmm2 = xmm2[1],xmm1[1]
 ; SSE41-NEXT:    movq %xmm1, %rax
 ; SSE41-NEXT:    testq %rax, %rax
-; SSE41-NEXT:    movapd %xmm1, %xmm3
-; SSE41-NEXT:    js .LBB8_2
-; SSE41-NEXT:  # %bb.1:
-; SSE41-NEXT:    movapd %xmm2, %xmm3
-; SSE41-NEXT:  .LBB8_2:
-; SSE41-NEXT:    movapd %xmm3, %xmm0
-; SSE41-NEXT:    cmpunordsd %xmm3, %xmm0
-; SSE41-NEXT:    movapd %xmm0, %xmm4
-; SSE41-NEXT:    andpd %xmm3, %xmm4
-; SSE41-NEXT:    js .LBB8_4
-; SSE41-NEXT:  # %bb.3:
+; SSE41-NEXT:    js .LBB8_1
+; SSE41-NEXT:  # %bb.2:
+; SSE41-NEXT:    movapd %xmm1, %xmm0
+; SSE41-NEXT:    jmp .LBB8_3
+; SSE41-NEXT:  .LBB8_1:
+; SSE41-NEXT:    movapd %xmm2, %xmm0
 ; SSE41-NEXT:    movapd %xmm1, %xmm2
-; SSE41-NEXT:  .LBB8_4:
-; SSE41-NEXT:    maxsd %xmm2, %xmm3
-; SSE41-NEXT:    andnpd %xmm3, %xmm0
-; SSE41-NEXT:    orpd %xmm4, %xmm0
+; SSE41-NEXT:  .LBB8_3:
+; SSE41-NEXT:    movapd %xmm2, %xmm1
+; SSE41-NEXT:    maxsd %xmm0, %xmm1
+; SSE41-NEXT:    movapd %xmm2, %xmm0
+; SSE41-NEXT:    cmpunordsd %xmm2, %xmm0
+; SSE41-NEXT:    blendvpd %xmm0, %xmm2, %xmm1
+; SSE41-NEXT:    movapd %xmm1, %xmm0
 ; SSE41-NEXT:    retq
 ;
 ; AVX-LABEL: test_v16f64:
@@ -1792,3 +1781,5 @@ declare double @llvm.vector.reduce.fmaximum.v2f64(<2 x double>)
 declare double @llvm.vector.reduce.fmaximum.v4f64(<4 x double>)
 declare double @llvm.vector.reduce.fmaximum.v8f64(<8 x double>)
 declare double @llvm.vector.reduce.fmaximum.v16f64(<16 x double>)
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; SSE: {{.*}}
diff --git a/llvm/test/CodeGen/X86/vector-reduce-fmin.ll b/llvm/test/CodeGen/X86/vector-reduce-fmin.ll
index 5ae9e552d0dcd..727af12217c67 100644
--- a/llvm/test/CodeGen/X86/vector-reduce-fmin.ll
+++ b/llvm/test/CodeGen/X86/vector-reduce-fmin.ll
@@ -1,6 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse2 | FileCheck %s --check-prefixes=ALL,SSE,SSE2
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse4.1 | FileCheck %s --check-prefixes=ALL,SSE,SSE41
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse2 | FileCheck %s --check-prefixes=ALL,SSE2
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse4.1 | FileCheck %s --check-prefixes=ALL,SSE41
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx | FileCheck %s --check-prefixes=ALL,AVX
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx2 | FileCheck %s --check-prefixes=ALL,AVX
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512bw | FileCheck %s --check-prefixes=ALL,AVX512,AVX512BW
@@ -36,13 +36,10 @@ define float @test_v2f32(<2 x float> %a0) {
 ; SSE41-LABEL: test_v2f32:
 ; SSE41:       # %bb.0:
 ; SSE41-NEXT:    movshdup {{.*#+}} xmm2 = xmm0[1,1,3,3]
-; SSE41-NEXT:    movaps %xmm0, %xmm1
-; SSE41-NEXT:    cmpunordss %xmm0, %xmm1
-; SSE41-NEXT:    movaps %xmm1, %xmm3
-; SSE41-NEXT:    andps %xmm2, %xmm3
-; SSE41-NEXT:    minss %xmm0, %xmm2
-; SSE41-NEXT:    andnps %xmm2, %xmm1
-; SSE41-NEXT:    orps %xmm3, %xmm1
+; SSE41-NEXT:    movaps %xmm2, %xmm1
+; SSE41-NEXT:    minss %xmm0, %xmm1
+; SSE41-NEXT:    cmpunordss %xmm0, %xmm0
+; SSE41-NEXT:    blendvps %xmm0, %xmm2, %xmm1
 ; SSE41-NEXT:    movaps %xmm1, %xmm0
 ; SSE41-NEXT:    retq
 ;
@@ -99,31 +96,26 @@ define float @test_v4f32(<4 x float> %a0) {
 ;
 ; SSE41-LABEL: test_v4f32:
 ; SSE41:       # %bb.0:
-; SSE41-NEXT:    movaps %xmm0, %xmm1
-; SSE41-NEXT:    movaps %xmm0, %xmm2
-; SSE41-NEXT:    movshdup {{.*#+}} xmm3 = xmm0[1,1,3,3]
-; SSE41-NEXT:    cmpunordss %xmm0, %xmm0
-; SSE41-NEXT:    movaps %xmm0, %xmm4
-; SSE41-NEXT:    andps %xmm3, %xmm4
-; SSE41-NEXT:    minss %xmm1, %xmm3
-; SSE41-NEXT:    shufps {{.*#+}} xmm1 = xmm1[3,3,3,3]
-; SSE41-NEXT:    movhlps {{.*#+}} xmm2 = xmm2[1,1]
-; SSE41-NEXT:    andnps %xmm3, %xmm0
-; SSE41-NEXT:    orps %xmm4, %xmm0
-; SSE41-NEXT:    movaps %xmm2, %xmm3
-; SSE41-NEXT:    minss %xmm0, %xmm3
-; SSE41-NEXT:    cmpunordss %xmm0, %xmm0
+; SSE41-NEXT:    movaps %xmm0, %xmm3
+; SSE41-NEXT:    shufps {{.*#+}} xmm3 = xmm3[3,3],xmm0[3,3]
 ; SSE41-NEXT:    movaps %xmm0, %xmm4
-; SSE41-NEXT:    andnps %xmm3, %xmm4
-; SSE41-NEXT:    andps %xmm2, %xmm0
-; SSE41-NEXT:    orps %xmm4, %xmm0
-; SSE41-NEXT:    movaps %xmm1, %xmm2
-; SSE41-NEXT:    minss %xmm0, %xmm2
+; SSE41-NEXT:    unpckhpd {{.*#+}} xmm4 = xmm4[1],xmm0[1]
+; SSE41-NEXT:    movshdup {{.*#+}} xmm2 = xmm0[1,1,3,3]
+; SSE41-NEXT:    movaps %xmm2, %xmm1
+; SSE41-NEXT:    minss %xmm0, %xmm1
 ; SSE41-NEXT:    cmpunordss %xmm0, %xmm0
-; SSE41-NEXT:    movaps %xmm0, %xmm3
-; SSE41-NEXT:    andnps %xmm2, %xmm3
-; SSE41-NEXT:    andps %xmm1, %xmm0
-; SSE41-NEXT:    orps %xmm3, %xmm0
+; SSE41-NEXT:    blendvps %xmm0, %xmm2, %xmm1
+; SSE41-NEXT:    movaps %xmm4, %xmm2
+; SSE41-NEXT:    minss %xmm1, %xmm2
+; SSE41-NEXT:    cmpunordss %xmm1, %xmm1
+; SSE41-NEXT:    movaps %xmm1, %xmm0
+; SSE41-NEXT:    blendvps %xmm0, %xmm4, %xmm2
+; SSE41-NEXT:    movaps %xmm3, %xmm1
+; SSE41-NEXT:    minss %xmm2, %xmm1
+; SSE41-NEXT:    cmpunordss %xmm2, %xmm2
+; SSE41-NEXT:    movaps %xmm2, %xmm0
+; SSE41-NEXT:    blendvps %xmm0, %xmm3, %xmm1
+; SSE41-NEXT:    movaps %xmm1, %xmm0
 ; SSE41-NEXT:    retq
 ;
 ; AVX-LABEL: test_v4f32:
@@ -199,35 +191,30 @@ define float @test_v8f32(<8 x float> %a0) {
 ;
 ; SSE41-LABEL: test_v8f32:
 ; SSE41:       # %bb.0:
-; SSE41-NEXT:    movaps %xmm1, %xmm2
-; SSE41-NEXT:    minps %xmm0, %xmm2
-; SSE41-NEXT:    cmpunordps %xmm0, %xmm0
-; SSE41-NEXT:    blendvps %xmm0, %xmm1, %xmm2
-; SSE41-NEXT:    movshdup {{.*#+}} xmm1 = xmm2[1,1,3,3]
-; SSE41-NEXT:    movaps %xmm2, %xmm0
-; SSE41-NEXT:    cmpunordss %xmm2, %xmm0
-; SSE41-NEXT:    movaps %xmm0, %xmm3
-; SSE41-NEXT:    andps %xmm1, %xmm3
-; SSE41-NEXT:    minss %xmm2, %xmm1
-; SSE41-NEXT:    andnps %xmm1, %xmm0
-; SSE41-NEXT:    orps %xmm3, %xmm0
-; SSE41-NEXT:    movaps %xmm2, %xmm1
-; SSE41-NEXT:    unpckhpd {{.*#+}} xmm1 = xmm1[1],xmm2[1]
 ; SSE41-NEXT:    movaps %xmm1, %xmm3
-; SSE41-NEXT:    minss %xmm0, %xmm3
-; SSE41-NEXT:    cmpunordss %xmm0, %xmm0
-; SSE41-NEXT:    movaps %xmm0, %xmm4
-; SSE41-NEXT:    andnps %xmm3, %xmm4
-; SSE41-NEXT:    andps %xmm1, %xmm0
-; SSE41-NEXT:    orps %xmm4, %xmm0
-; SSE41-NEXT:    shufps {{.*#+}} xmm2 = xmm2[3,3,3,3]
+; SSE41-NEXT:    minps %xmm0, %xmm3
+; SSE41-NEXT:    cmpunordps %xmm0, %xmm0
+; SSE41-NEXT:    blendvps %xmm0, %xmm1, %xmm3
+; SSE41-NEXT:    movshdup {{.*#+}} xmm2 = xmm3[1,1,3,3]
 ; SSE41-NEXT:    movaps %xmm2, %xmm1
-; SSE41-NEXT:    minss %xmm0, %xmm1
-; SSE41-NEXT:    cmpunordss %xmm0, %xmm0
-; SSE41-NEXT:    movaps %xmm0, %xmm3
-; SSE41-NEXT:    andnps %xmm1, %xmm3
-; SSE41-NEXT:    andps %xmm2, %xmm0
-; SSE41-NEXT:    orps %xmm3, %xmm0
+; SSE41-NEXT:    minss %xmm3, %xmm1
+; SSE41-NEXT:    movaps %xmm3, %xmm0
+; SSE41-NEXT:    cmpunordss %xmm3, %xmm0
+; SSE41-NEXT:    blendvps %xmm0, %xmm2, %xmm1
+; SSE41-NEXT:    movaps %xmm3, %xmm4
+; SSE41-NEXT:    unpckhpd {{.*#+}} xmm4 = xmm4[1],xmm3[1]
+; SSE41-NEXT:    movaps %xmm4, %xmm2
+; SSE41-NEXT:    minss %xmm1, %xmm2
+; SSE41-NEXT:    cmpunordss %xmm1, %xmm1
+; SSE41-NEXT:    movaps %xmm1, %xmm0
+; SSE41-NEXT:    blendvps %xmm0, %xmm4, %xmm2
+; SSE41-NEXT:    shufps {{.*#+}} xmm3 = xmm3[3,3,3,3]
+; SSE41-NEXT:    movaps %xmm3, %xmm1
+; SSE41-NEXT:    minss %xmm2, %xmm1
+; SSE41-NEXT:    cmpunordss %xmm2, %xmm2
+; SSE41-NEXT:    movaps %xmm2, %xmm0
+; SSE41-NEXT:    blendvps %xmm0, %xmm3, %xmm1
+; SSE41-NEXT:    movaps %xmm1, %xmm0
 ; SSE41-NEXT:    retq
 ;
 ; AVX-LABEL: test_v8f32:
@@ -391,36 +378,31 @@ define float @test_v16f32(<16 x float> %a0) {
 ; SSE41-NEXT:    cmpunordps %xmm1, %xmm1
 ; SSE41-NEXT:    movaps %xmm1, %xmm0
 ; SSE41-NEXT:    blendvps %xmm0, %xmm3, %xmm2
-; SSE41-NEXT:    movaps %xmm2, %xmm1
-; SSE41-NEXT:    minps %xmm4, %xmm1
+; SSE41-NEXT:    movaps %xmm2, %xmm3
+; SSE41-NEXT:    minps %xmm4, %xmm3
 ; SSE41-NEXT:    cmpunordps %xmm4, %xmm4
 ; SSE41-NEXT:    movaps %xmm4, %xmm0
+; SSE41-NEXT:    blendvps %xmm0, %xmm2, %xmm3
+; SSE41-NEXT:    movshdup {{.*#+}} xmm2 = xmm3[1,1,3,3]
+; SSE41-NEXT:    movaps %xmm2, %xmm1
+; SSE41-NEXT:    minss %xmm3, %xmm1
+; SSE41-NEXT:    movaps %xmm3, %xmm0
+; SSE41-NEXT:    cmpunordss %xmm3, %xmm0
 ; SSE41-NEXT:    blendvps %xmm0, %xmm2, %xmm1
-; SSE41-NEXT:    movshdup {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; SSE41-NEXT:    movaps %xmm1, %xmm0
-; SSE41-NEXT:    cmpunordss %xmm1, %xmm0
-; SSE41-NEXT:    movaps %xmm0, %xmm3
-; SSE41-NEXT:    andps %xmm2, %xmm3
+; SSE41-NEXT:    movaps %xmm3, %xmm4
+; SSE41-NEXT:    unpckhpd {{.*#+}} xmm4 = xmm4[1],xmm3[1]
+; SSE41-NEXT:    movaps %xmm4, %xmm2
 ; SSE41-NEXT:    minss %xmm1, %xmm2
-; SSE41-NEXT:    andnps %xmm2, %xmm0
-; SSE41-NEXT:    orps %xmm3, %xmm0
-; SSE41-NEXT:    movaps %xmm1, %xmm2
-; SSE41-NEXT:    unpckhpd {{.*#+}} xmm2 = xmm2[1],xmm1[1]
-; SSE41-NEXT:    movaps %xmm2, %xmm3
-; SSE41-NEXT:    minss %xmm0, %xmm3
-; SSE41-NEXT:    cmpunordss %xmm0, %xmm0
-; SSE41-NEXT:    movaps %xmm0, %xmm4
-; SSE41-NEXT:    andnps %xmm3, %xmm4
-; SSE41-NEXT:    andps %xmm2, %xmm0
-; SSE41-NEXT:    orps %xmm4, %xmm0
-; SSE41-NEXT:    shufps {{.*#+}} xmm1 = xmm1[3,3,3,3]
-; SSE41-NEXT:    movaps %xmm1, %xmm2
-; SSE41-NEXT:    minss %xmm0, %xmm2
-; SSE41-NEXT:    cmpunordss %xmm0, %xmm0
-; SSE41-NEXT:    movaps %xmm0, %xmm3
-; SSE41-NEXT:    andnps %xmm2, %xmm3
-; SSE41-NEXT:    andps %xmm1, %xmm0
-; SSE41-NEXT:    orps %xmm3, %xmm0
+; SSE41-NEXT:    cmpunordss %xmm1, %xmm1
+; SSE41-NEXT:    movaps %xmm1, %xmm0
+; SSE41-NEXT:    blendvps %xmm0, %xmm4, %xmm2
+; SSE41-NEXT:    shufps {{.*#+}} xmm3 = xmm3[3,3,3,3]
+; SSE41-NEXT:    movaps %xmm3, %xmm1
+; SSE41-NEXT:    minss %xmm2, %xmm1
+; SSE41-NEXT:    cmpunordss %xmm2, %xmm2
+; SSE41-NEXT:    movaps %xmm2, %xmm0
+; SSE41-NEXT:    blendvps %xmm0, %xmm3, %xmm1
+; SSE41-NEXT:    movaps %xmm1, %xmm0
 ; SSE41-NEXT:    retq
 ;
 ; AVX-LABEL: test_v16f32:
@@ -597,19 +579,30 @@ define float @test_v16f32(<16 x float> %a0) {
 ;
 
 define double @test_v2f64(<2 x double> %a0) {
-; SSE-LABEL: test_v2f64:
-; SSE:       # %bb.0:
-; SSE-NEXT:    movapd %xmm0, %xmm2
-; SSE-NEXT:    unpckhpd {{.*#+}} xmm2 = xmm2[1],xmm0[1]
-; SSE-NEXT:    movapd %xmm0, %xmm1
-; SSE-NEXT:    cmpunordsd %xmm0, %xmm1
-; SSE-NEXT:    movapd %xmm1, %xmm3
-; SSE-NEXT:    andpd %xmm2, %xmm3
-; SSE-NEXT:    minsd %xmm0, %xmm2
-; SSE-NEXT:    andnpd %xmm2, %xmm1
-; SSE-NEXT:    orpd %xmm3, %xmm1
-; SSE-NEXT:    movapd %xmm1, %xmm0
-; SSE-NEXT:    retq
+; SSE2-LABEL: test_v2f64:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    movapd %xmm0, %xmm2
+; SSE2-NEXT:    unpckhpd {{.*#+}} xmm2 = xmm2[1],xmm0[1]
+; SSE2-NEXT:    movapd %xmm0, %xmm1
+; SSE2-NEXT:    cmpunordsd %xmm0, %xmm1
+; SSE2-NEXT:    movapd %xmm1, %xmm3
+; SSE2-NEXT:    andpd %xmm2, %xmm3
+; SSE2-NEXT:    minsd %xmm0, %xmm2
+; SSE2-NEXT:    andnpd %xmm2, %xmm1
+; SSE2-NEXT:    orpd %xmm3, %xmm1
+; SSE2-NEXT:    movapd %xmm1, %xmm0
+; SSE2-NEXT:    retq
+;
+; SSE41-LABEL: test_v2f64:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    movapd %xmm0, %xmm2
+; SSE41-NEXT:    unpckhpd {{.*#+}} xmm2 = xmm2[1],xmm0[1]
+; SSE41-NEXT:    movapd %xmm2, %xmm1
+; SSE41-NEXT:    minsd %xmm0, %xmm1
+; SSE41-NEXT:    cmpunordsd %xmm0, %xmm0
+; SSE41-NEXT:    blendvpd %xmm0, %xmm2, %xmm1
+; SSE41-NEXT:    movapd %xmm1, %xmm0
+; SSE41-NEXT:    retq
 ;
 ; AVX-LABEL: test_v2f64:
 ; AVX:       # %bb.0:
@@ -661,15 +654,14 @@ define double @test_v3f64(<3 x double> %a0) {
 ; SSE41-NEXT:    minpd %xmm0, %xmm1
 ; SSE41-NEXT:    cmpunordpd %xmm0, %xmm0
 ; SSE41-NEXT:    blendvpd %xmm0, %xmm2, %xmm1
-; SSE41-NEXT:    movapd %xmm1, %xmm2
-; SSE41-NEXT:    unpckhpd {{.*#+}} xmm2 = xmm2[1],xmm1[1]
-; SSE41-NEXT:    movapd %xmm1, %xmm0
-; SSE41-NEXT:    cmpunordsd %xmm1, %xmm0
-; SSE41-NEXT:    movapd %xmm0, %xmm3
-; SSE41-NEXT:    andpd %xmm2, %xmm3
+; SSE41-NEXT:    movapd %xmm1, %xmm3
+; SSE41-NEXT:    unpckhpd {{.*#+}} xmm3 = xmm3[1],xmm1[1]
+; SSE41-NEXT:    movapd %xmm3, %xmm2
 ; SSE41-NEXT:    minsd %xmm1, %xmm2
-; SSE41-NEXT:    andnpd %xmm2, %xmm0
-; SSE41-NEXT:    orpd %xmm3, %xmm0
+; SSE41-NEXT:    cmpunordsd %xmm1, %xmm1
+; SSE41-NEXT:    movapd %xmm1, %xmm0
+; SSE41-NEXT:    blendvpd %xmm0, %xmm3, %xmm2
+; SSE41-NEXT:    movapd %xmm2, %xmm0
 ; SSE41-NEXT:    retq
 ;
 ; AVX-LABEL: test_v3f64:
@@ -727,15 +719,14 @@ define double @test_v4f64(<4 x double> %a0) {
 ; SSE41-NEXT:    minpd %xmm0, %xmm2
 ; SSE41-NEXT:    cmpunordpd %xmm0, %xmm0
 ; SSE41-NEXT:    blendvpd %xmm0, %xmm1, %xmm2
-; SSE41-NEXT:    movapd %xmm2, %xmm1
-; SSE41-NEXT:    unpckhpd {{.*#+}} xmm1 = xmm1[1],xmm2[1]
-; SSE41-NEXT:    movapd %xmm2, %xmm0
-; SSE41-NEXT:    cmpunordsd %xmm2, %xmm0
-; SSE41-NEXT:    movapd %xmm0, %xmm3
-; SSE41-NEXT:    andpd %xmm1, %xmm3
+; SSE41-NEXT:    movapd %xmm2, %xmm3
+; SSE41-NEXT:    unpckhpd {{.*#+}} xmm3 = xmm3[1],xmm2[1]
+; SSE41-NEXT:    movapd %xmm3, %xmm1
 ; SSE41-NEXT:    minsd %xmm2, %xmm1
-; SSE41-NEXT:    andnpd %xmm1, %xmm0
-; SSE41-NEXT:    orpd %xmm3, %xmm0
+; SSE41-NEXT:    cmpunordsd %xmm2, %xmm2
+; SSE41-NEXT:    movapd %xmm2, %xmm0
+; SSE41-NEXT:    blendvpd %xmm0, %xmm3, %xmm1
+; SSE41-NEXT:    movapd %xmm1, %xmm0
 ; SSE41-NEXT:    retq
 ;
 ; AVX-LABEL: test_v4f64:
@@ -823,15 +814,14 @@ define double @test_v8f64(<8 x double> %a0) {
 ; SSE41-NEXT:    cmpunordpd %xmm4, %xmm4
 ; SSE41-NEXT:    movapd %xmm4, %xmm0
 ; SSE41-NEXT:    blendvpd %xmm0, %xmm2, %xmm1
-; SSE41-NEXT:    movapd %xmm1, %xmm2
-; SSE41-NEXT:    unpckhpd {{.*#+}} xmm2 = xmm2[1],xmm1[1]
-; SSE41-NEXT:    movapd %xmm1, %xmm0
-; SSE41-NEXT:    cmpunordsd %xmm1, %xmm0
-; SSE41-NEXT:    movapd %xmm0, %xmm3
-; SSE41-NEXT:    andpd %xmm2, %xmm3
+; SSE41-NEXT:    movapd %xmm1, %xmm3
+; SSE41-NEXT:    unpckhpd {{.*#+}} xmm3 = xmm3[1],xmm1[1]
+; SSE41-NEXT:    movapd %xmm3, %xmm2
 ; SSE41-NEXT:    minsd %xmm1, %xmm2
-; SSE41-NEXT:    andnpd %xmm2, %xmm0
-; SSE41-NEXT:    orpd %xmm3, %xmm0
+; SSE41-NEXT:    cmpunordsd %xmm1, %xmm1
+; SSE41-NEXT:    movapd %xmm1, %xmm0
+; SSE41-NEXT:    blendvpd %xmm0, %xmm3, %xmm2
+; SSE41-NEXT:    movapd %xmm2, %xmm0
 ; SSE41-NEXT:    retq
 ;
 ; AVX-LABEL: test_v8f64:
@@ -1015,15 +1005,14 @@ define double @test_v16f64(<16 x double> %a0) {
 ; SSE41-NEXT:    cmpunordpd %xmm2, %xmm2
 ; SSE41-NEXT:    movapd %xmm2, %xmm0
 ; SSE41-NEXT:    blendvpd %xmm0, %xmm3, %xmm1
-; SSE41-NEXT:    movapd %xmm1, %xmm2
-; SSE41-NEXT:    unpckhpd {{.*#+}} xmm2 = xmm2[1],xmm1[1]
-; SSE41-NEXT:    movapd %xmm1, %xmm0
-; SSE41-NEXT:    cmpunordsd %xmm1, %xmm0
-; SSE41-NEXT:    movapd %xmm0, %xmm3
-; SSE41-NEXT:    andpd %xmm2, %xmm3
+; SSE41-NEXT:    movapd %xmm1, %xmm3
+; SSE41-NEXT:    unpckhpd {{.*#+}} xmm3 = xmm3[1],xmm1[1]
+; SSE41-NEXT:    movapd %xmm3, %xmm2
 ; SSE41-NEXT:    minsd %xmm1, %xmm2
-; SSE41-NEXT:    andnpd %xmm2, %xmm0
-; SSE41-NEXT:    orpd %xmm3, %xmm0
+; SSE41-NEXT:    cmpunordsd %xmm1, %xmm1
+; SSE41-NEXT:    movapd %xmm1, %xmm0
+; SSE41-NEXT:    blendvpd %xmm0, %xmm3, %xmm2
+; SSE41-NEXT:    movapd %xmm2, %xmm0
 ; SSE41-NEXT:    retq
 ;
 ; AVX-LABEL: test_v16f64:
diff --git a/llvm/test/CodeGen/X86/vector-shuffle-combining-avx.ll b/llvm/test/CodeGen/X86/vector-shuffle-combining-avx.ll
index 79602a18693db..00af58544e25c 100644
--- a/llvm/test/CodeGen/X86/vector-shuffle-combining-avx.ll
+++ b/llvm/test/CodeGen/X86/vector-shuffle-combining-avx.ll
@@ -493,11 +493,11 @@ define void @PR48908(<4 x double> %v0, <4 x double> %v1, <4 x double> %v2, ptr n
 ; X86-AVX2-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X86-AVX2-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; X86-AVX2-NEXT:    movl {{[0-9]+}}(%esp), %edx
-; X86-AVX2-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm3
+; X86-AVX2-NEXT:    vbroadcastsd %xmm1, %ymm3
 ; X86-AVX2-NEXT:    vperm2f128 {{.*#+}} ymm4 = ymm1[2,3],ymm2[0,1]
 ; X86-AVX2-NEXT:    vshufpd {{.*#+}} xmm5 = xmm1[1,0]
 ; X86-AVX2-NEXT:    vperm2f128 {{.*#+}} ymm6 = ymm0[0,1],ymm2[0,1]
-; X86-AVX2-NEXT:    vpermpd {{.*#+}} ymm3 = ymm3[0,2,2,1]
+; X86-AVX2-NEXT:    vperm2f128 {{.*#+}} ymm3 = ymm3[2,3],ymm0[0,1]
 ; X86-AVX2-NEXT:    vblendpd {{.*#+}} ymm3 = ymm6[0],ymm3[1],ymm6[2],ymm3[3]
 ; X86-AVX2-NEXT:    vmovapd %ymm3, (%edx)
 ; X86-AVX2-NEXT:    vblendpd {{.*#+}} ymm3 = ymm5[0,1],ymm0[2,3]
@@ -520,13 +520,13 @@ define void @PR48908(<4 x double> %v0, <4 x double> %v1, <4 x double> %v2, ptr n
 ; X86-AVX512-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X86-AVX512-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; X86-AVX512-NEXT:    movl {{[0-9]+}}(%esp), %edx
-; X86-AVX512-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm4
 ; X86-AVX512-NEXT:    vpmovsxbq {{.*#+}} ymm3 = [1,2,8,9]
 ; X86-AVX512-NEXT:    vpermi2pd %zmm2, %zmm1, %zmm3
-; X86-AVX512-NEXT:    vpmovsxbq {{.*#+}} ymm5 = [0,10,2,9]
-; X86-AVX512-NEXT:    vperm2f128 {{.*#+}} ymm6 = ymm0[0,1],ymm2[0,1]
-; X86-AVX512-NEXT:    vpermt2pd %zmm4, %zmm5, %zmm6
-; X86-AVX512-NEXT:    vmovapd %ymm6, (%edx)
+; X86-AVX512-NEXT:    vpmovsxbq {{.*#+}} ymm4 = [0,8,2,1]
+; X86-AVX512-NEXT:    vpermi2pd %zmm1, %zmm0, %zmm4
+; X86-AVX512-NEXT:    vperm2f128 {{.*#+}} ymm5 = ymm0[0,1],ymm2[0,1]
+; X86-AVX512-NEXT:    vblendpd {{.*#+}} ymm4 = ymm5[0],ymm4[1],ymm5[2],ymm4[3]
+; X86-AVX512-NEXT:    vmovapd %ymm4, (%edx)
 ; X86-AVX512-NEXT:    vpmovsxbq {{.*#+}} ymm4 = [0,3,10,1]
 ; X86-AVX512-NEXT:    vpermi2pd %zmm0, %zmm3, %zmm4
 ; X86-AVX512-NEXT:    vmovapd %ymm4, (%ecx)
@@ -563,11 +563,11 @@ define void @PR48908(<4 x double> %v0, <4 x double> %v1, <4 x double> %v2, ptr n
 ;
 ; X64-AVX2-LABEL: PR48908:
 ; X64-AVX2:       # %bb.0:
-; X64-AVX2-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm3
+; X64-AVX2-NEXT:    vbroadcastsd %xmm1, %ymm3
 ; X64-AVX2-NEXT:    vperm2f128 {{.*#+}} ymm4 = ymm1[2,3],ymm2[0,1]
 ; X64-AVX2-NEXT:    vshufpd {{.*#+}} xmm5 = xmm1[1,0]
 ; X64-AVX2-NEXT:    vperm2f128 {{.*#+}} ymm6 = ymm0[0,1],ymm2[0,1]
-; X64-AVX2-NEXT:    vpermpd {{.*#+}} ymm3 = ymm3[0,2,2,1]
+; X64-AVX2-NEXT:    vperm2f128 {{.*#+}} ymm3 = ymm3[2,3],ymm0[0,1]
 ; X64-AVX2-NEXT:    vblendpd {{.*#+}} ymm3 = ymm6[0],ymm3[1],ymm6[2],ymm3[3]
 ; X64-AVX2-NEXT:    vmovapd %ymm3, (%rdi)
 ; X64-AVX2-NEXT:    vblendpd {{.*#+}} ymm3 = ymm5[0,1],ymm0[2,3]
@@ -587,16 +587,16 @@ define void @PR48908(<4 x double> %v0, <4 x double> %v1, <4 x double> %v2, ptr n
 ; X64-AVX512-NEXT:    # kill: def $ymm2 killed $ymm2 def $zmm2
 ; X64-AVX512-NEXT:    # kill: def $ymm1 killed $ymm1 def $zmm1
 ; X64-AVX512-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
-; X64-AVX512-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm3
-; X64-AVX512-NEXT:    vpmovsxbq {{.*#+}} ymm4 = [1,2,8,9]
-; X64-AVX512-NEXT:    vpermi2pd %zmm2, %zmm1, %zmm4
-; X64-AVX512-NEXT:    vpmovsxbq {{.*#+}} ymm5 = [0,10,2,9]
-; X64-AVX512-NEXT:    vperm2f128 {{.*#+}} ymm6 = ymm0[0,1],ymm2[0,1]
-; X64-AVX512-NEXT:    vpermt2pd %zmm3, %zmm5, %zmm6
-; X64-AVX512-NEXT:    vmovapd %ymm6, (%rdi)
-; X64-AVX512-NEXT:    vpmovsxbq {{.*#+}} ymm3 = [0,3,10,1]
-; X64-AVX512-NEXT:    vpermi2pd %zmm0, %zmm4, %zmm3
-; X64-AVX512-NEXT:    vmovapd %ymm3, (%rsi)
+; X64-AVX512-NEXT:    vpmovsxbq {{.*#+}} ymm3 = [1,2,8,9]
+; X64-AVX512-NEXT:    vpermi2pd %zmm2, %zmm1, %zmm3
+; X64-AVX512-NEXT:    vpmovsxbq {{.*#+}} ymm4 = [0,8,2,1]
+; X64-AVX512-NEXT:    vpermi2pd %zmm1, %zmm0, %zmm4
+; X64-AVX512-NEXT:    vperm2f128 {{.*#+}} ymm5 = ymm0[0,1],ymm2[0,1]
+; X64-AVX512-NEXT:    vblendpd {{.*#+}} ymm4 = ymm5[0],ymm4[1],ymm5[2],ymm4[3]
+; X64-AVX512-NEXT:    vmovapd %ymm4, (%rdi)
+; X64-AVX512-NEXT:    vpmovsxbq {{.*#+}} ymm4 = [0,3,10,1]
+; X64-AVX512-NEXT:    vpermi2pd %zmm0, %zmm3, %zmm4
+; X64-AVX512-NEXT:    vmovapd %ymm4, (%rsi)
 ; X64-AVX512-NEXT:    vpmovsxbq {{.*#+}} xmm3 = [3,11]
 ; X64-AVX512-NEXT:    vpermi2pd %zmm1, %zmm0, %zmm3
 ; X64-AVX512-NEXT:    vpmovsxbq {{.*#+}} ymm0 = [2,8,9,3]
diff --git a/llvm/test/CodeGen/X86/vselect-zero.ll b/llvm/test/CodeGen/X86/vselect-zero.ll
index 50040593848a2..b3bb01137c70d 100644
--- a/llvm/test/CodeGen/X86/vselect-zero.ll
+++ b/llvm/test/CodeGen/X86/vselect-zero.ll
@@ -113,14 +113,22 @@ define float @fsel_zero_true_val(float %a, float %b, float %x) {
 }
 
 define double @fsel_nonzero_false_val(double %x, double %y, double %z) {
-; SSE-LABEL: fsel_nonzero_false_val:
-; SSE:       # %bb.0:
-; SSE-NEXT:    cmpeqsd %xmm1, %xmm0
-; SSE-NEXT:    andpd %xmm0, %xmm2
-; SSE-NEXT:    movsd {{.*#+}} xmm1 = [4.2E+1,0.0E+0]
-; SSE-NEXT:    andnpd %xmm1, %xmm0
-; SSE-NEXT:    orpd %xmm2, %xmm0
-; SSE-NEXT:    retq
+; SSE2-LABEL: fsel_nonzero_false_val:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    cmpeqsd %xmm1, %xmm0
+; SSE2-NEXT:    andpd %xmm0, %xmm2
+; SSE2-NEXT:    movsd {{.*#+}} xmm1 = [4.2E+1,0.0E+0]
+; SSE2-NEXT:    andnpd %xmm1, %xmm0
+; SSE2-NEXT:    orpd %xmm2, %xmm0
+; SSE2-NEXT:    retq
+;
+; SSE42-LABEL: fsel_nonzero_false_val:
+; SSE42:       # %bb.0:
+; SSE42-NEXT:    cmpeqsd %xmm1, %xmm0
+; SSE42-NEXT:    movapd {{.*#+}} xmm1 = [4.2E+1,4.2E+1]
+; SSE42-NEXT:    blendvpd %xmm0, %xmm2, %xmm1
+; SSE42-NEXT:    movapd %xmm1, %xmm0
+; SSE42-NEXT:    retq
 ;
 ; AVX-LABEL: fsel_nonzero_false_val:
 ; AVX:       # %bb.0:
@@ -142,14 +150,21 @@ define double @fsel_nonzero_false_val(double %x, double %y, double %z) {
 }
 
 define double @fsel_nonzero_true_val(double %x, double %y, double %z) {
-; SSE-LABEL: fsel_nonzero_true_val:
-; SSE:       # %bb.0:
-; SSE-NEXT:    cmpeqsd %xmm1, %xmm0
-; SSE-NEXT:    movsd {{.*#+}} xmm1 = [4.2E+1,0.0E+0]
-; SSE-NEXT:    andpd %xmm0, %xmm1
-; SSE-NEXT:    andnpd %xmm2, %xmm0
-; SSE-NEXT:    orpd %xmm1, %xmm0
-; SSE-NEXT:    retq
+; SSE2-LABEL: fsel_nonzero_true_val:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    cmpeqsd %xmm1, %xmm0
+; SSE2-NEXT:    movsd {{.*#+}} xmm1 = [4.2E+1,0.0E+0]
+; SSE2-NEXT:    andpd %xmm0, %xmm1
+; SSE2-NEXT:    andnpd %xmm2, %xmm0
+; SSE2-NEXT:    orpd %xmm1, %xmm0
+; SSE2-NEXT:    retq
+;
+; SSE42-LABEL: fsel_nonzero_true_val:
+; SSE42:       # %bb.0:
+; SSE42-NEXT:    cmpeqsd %xmm1, %xmm0
+; SSE42-NEXT:    blendvpd %xmm0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2
+; SSE42-NEXT:    movapd %xmm2, %xmm0
+; SSE42-NEXT:    retq
 ;
 ; AVX-LABEL: fsel_nonzero_true_val:
 ; AVX:       # %bb.0:
diff --git a/llvm/test/CodeGen/X86/wide-scalar-shift-legalization.ll b/llvm/test/CodeGen/X86/wide-scalar-shift-legalization.ll
index 43d2a997c81d2..338e104fbe8f0 100644
--- a/llvm/test/CodeGen/X86/wide-scalar-shift-legalization.ll
+++ b/llvm/test/CodeGen/X86/wide-scalar-shift-legalization.ll
@@ -3296,27 +3296,27 @@ define void @lshr_64bytes(ptr %src.ptr, ptr %bitOff.ptr, ptr %dst) nounwind {
 ; X64-HAVE-BMI2-NO-SHLD-NEXT:    andl $63, %ecx
 ; X64-HAVE-BMI2-NO-SHLD-NEXT:    shrl $3, %eax
 ; X64-HAVE-BMI2-NO-SHLD-NEXT:    andl $56, %eax
-; X64-HAVE-BMI2-NO-SHLD-NEXT:    movq -120(%rsp,%rax), %rdi
-; X64-HAVE-BMI2-NO-SHLD-NEXT:    movq -112(%rsp,%rax), %r8
-; X64-HAVE-BMI2-NO-SHLD-NEXT:    shrxq %rcx, %rdi, %r15
+; X64-HAVE-BMI2-NO-SHLD-NEXT:    movq -120(%rsp,%rax), %r8
+; X64-HAVE-BMI2-NO-SHLD-NEXT:    movq -112(%rsp,%rax), %rdi
+; X64-HAVE-BMI2-NO-SHLD-NEXT:    shrxq %rcx, %r8, %r15
 ; X64-HAVE-BMI2-NO-SHLD-NEXT:    shrxq %rcx, -128(%rsp,%rax), %rbx
 ; X64-HAVE-BMI2-NO-SHLD-NEXT:    movl %ecx, %esi
 ; X64-HAVE-BMI2-NO-SHLD-NEXT:    movq -104(%rsp,%rax), %r9
 ; X64-HAVE-BMI2-NO-SHLD-NEXT:    shrxq %rcx, %r9, %r13
-; X64-HAVE-BMI2-NO-SHLD-NEXT:    shrxq %rcx, %r8, %r10
+; X64-HAVE-BMI2-NO-SHLD-NEXT:    shrxq %rcx, %rdi, %r10
 ; X64-HAVE-BMI2-NO-SHLD-NEXT:    movq -88(%rsp,%rax), %r11
 ; X64-HAVE-BMI2-NO-SHLD-NEXT:    shrxq %rcx, %r11, %r14
 ; X64-HAVE-BMI2-NO-SHLD-NEXT:    movl %ecx, %r12d
 ; X64-HAVE-BMI2-NO-SHLD-NEXT:    notl %r12d
-; X64-HAVE-BMI2-NO-SHLD-NEXT:    addq %r8, %r8
-; X64-HAVE-BMI2-NO-SHLD-NEXT:    shlxq %r12, %r8, %r8
-; X64-HAVE-BMI2-NO-SHLD-NEXT:    orq %r15, %r8
+; X64-HAVE-BMI2-NO-SHLD-NEXT:    addq %rdi, %rdi
+; X64-HAVE-BMI2-NO-SHLD-NEXT:    shlxq %r12, %rdi, %rdi
+; X64-HAVE-BMI2-NO-SHLD-NEXT:    orq %r15, %rdi
 ; X64-HAVE-BMI2-NO-SHLD-NEXT:    movq -96(%rsp,%rax), %r15
 ; X64-HAVE-BMI2-NO-SHLD-NEXT:    shrxq %rcx, %r15, %rbp
 ; X64-HAVE-BMI2-NO-SHLD-NEXT:    xorb $63, %sil
-; X64-HAVE-BMI2-NO-SHLD-NEXT:    addq %rdi, %rdi
-; X64-HAVE-BMI2-NO-SHLD-NEXT:    shlxq %rsi, %rdi, %rdi
-; X64-HAVE-BMI2-NO-SHLD-NEXT:    orq %rbx, %rdi
+; X64-HAVE-BMI2-NO-SHLD-NEXT:    addq %r8, %r8
+; X64-HAVE-BMI2-NO-SHLD-NEXT:    shlxq %rsi, %r8, %r8
+; X64-HAVE-BMI2-NO-SHLD-NEXT:    orq %rbx, %r8
 ; X64-HAVE-BMI2-NO-SHLD-NEXT:    leaq (%r15,%r15), %rbx
 ; X64-HAVE-BMI2-NO-SHLD-NEXT:    shlxq %r12, %rbx, %rbx
 ; X64-HAVE-BMI2-NO-SHLD-NEXT:    orq %r13, %rbx
@@ -3342,8 +3342,8 @@ define void @lshr_64bytes(ptr %src.ptr, ptr %bitOff.ptr, ptr %dst) nounwind {
 ; X64-HAVE-BMI2-NO-SHLD-NEXT:    movq %r10, 40(%rdx)
 ; X64-HAVE-BMI2-NO-SHLD-NEXT:    movq %r9, 16(%rdx)
 ; X64-HAVE-BMI2-NO-SHLD-NEXT:    movq %rbx, 24(%rdx)
-; X64-HAVE-BMI2-NO-SHLD-NEXT:    movq %rdi, (%rdx)
-; X64-HAVE-BMI2-NO-SHLD-NEXT:    movq %r8, 8(%rdx)
+; X64-HAVE-BMI2-NO-SHLD-NEXT:    movq %r8, (%rdx)
+; X64-HAVE-BMI2-NO-SHLD-NEXT:    movq %rdi, 8(%rdx)
 ; X64-HAVE-BMI2-NO-SHLD-NEXT:    addq $8, %rsp
 ; X64-HAVE-BMI2-NO-SHLD-NEXT:    popq %rbx
 ; X64-HAVE-BMI2-NO-SHLD-NEXT:    popq %r12
@@ -5571,27 +5571,27 @@ define void @ashr_64bytes(ptr %src.ptr, ptr %bitOff.ptr, ptr %dst) nounwind {
 ; X64-HAVE-BMI2-NO-SHLD-NEXT:    andl $63, %ecx
 ; X64-HAVE-BMI2-NO-SHLD-NEXT:    shrl $3, %eax
 ; X64-HAVE-BMI2-NO-SHLD-NEXT:    andl $56, %eax
-; X64-HAVE-BMI2-NO-SHLD-NEXT:    movq -120(%rsp,%rax), %rdi
-; X64-HAVE-BMI2-NO-SHLD-NEXT:    movq -112(%rsp,%rax), %r8
-; X64-HAVE-BMI2-NO-SHLD-NEXT:    shrxq %rcx, %rdi, %r15
+; X64-HAVE-BMI2-NO-SHLD-NEXT:    movq -120(%rsp,%rax), %r8
+; X64-HAVE-BMI2-NO-SHLD-NEXT:    movq -112(%rsp,%rax), %rdi
+; X64-HAVE-BMI2-NO-SHLD-NEXT:    shrxq %rcx, %r8, %r15
 ; X64-HAVE-BMI2-NO-SHLD-NEXT:    shrxq %rcx, -128(%rsp,%rax), %rbx
 ; X64-HAVE-BMI2-NO-SHLD-NEXT:    movl %ecx, %esi
 ; X64-HAVE-BMI2-NO-SHLD-NEXT:    movq -104(%rsp,%rax), %r9
 ; X64-HAVE-BMI2-NO-SHLD-NEXT:    shrxq %rcx, %r9, %r13
-; X64-HAVE-BMI2-NO-SHLD-NEXT:    shrxq %rcx, %r8, %r10
+; X64-HAVE-BMI2-NO-SHLD-NEXT:    shrxq %rcx, %rdi, %r10
 ; X64-HAVE-BMI2-NO-SHLD-NEXT:    movq -88(%rsp,%rax), %r11
 ; X64-HAVE-BMI2-NO-SHLD-NEXT:    shrxq %rcx, %r11, %r14
 ; X64-HAVE-BMI2-NO-SHLD-NEXT:    movl %ecx, %r12d
 ; X64-HAVE-BMI2-NO-SHLD-NEXT:    notl %r12d
-; X64-HAVE-BMI2-NO-SHLD-NEXT:    addq %r8, %r8
-; X64-HAVE-BMI2-NO-SHLD-NEXT:    shlxq %r12, %r8, %r8
-; X64-HAVE-BMI2-NO-SHLD-NEXT:    orq %r15, %r8
+; X64-HAVE-BMI2-NO-SHLD-NEXT:    addq %rdi, %rdi
+; X64-HAVE-BMI2-NO-SHLD-NEXT:    shlxq %r12, %rdi, %rdi
+; X64-HAVE-BMI2-NO-SHLD-NEXT:    orq %r15, %rdi
 ; X64-HAVE-BMI2-NO-SHLD-NEXT:    movq -96(%rsp,%rax), %r15
 ; X64-HAVE-BMI2-NO-SHLD-NEXT:    shrxq %rcx, %r15, %rbp
 ; X64-HAVE-BMI2-NO-SHLD-NEXT:    xorb $63, %sil
-; X64-HAVE-BMI2-NO-SHLD-NEXT:    addq %rdi, %rdi
-; X64-HAVE-BMI2-NO-SHLD-NEXT:    shlxq %rsi, %rdi, %rdi
-; X64-HAVE-BMI2-NO-SHLD-NEXT:    orq %rbx, %rdi
+; X64-HAVE-BMI2-NO-SHLD-NEXT:    addq %r8, %r8
+; X64-HAVE-BMI2-NO-SHLD-NEXT:    shlxq %rsi, %r8, %r8
+; X64-HAVE-BMI2-NO-SHLD-NEXT:    orq %rbx, %r8
 ; X64-HAVE-BMI2-NO-SHLD-NEXT:    leaq (%r15,%r15), %rbx
 ; X64-HAVE-BMI2-NO-SHLD-NEXT:    shlxq %r12, %rbx, %rbx
 ; X64-HAVE-BMI2-NO-SHLD-NEXT:    orq %r13, %rbx
@@ -5617,8 +5617,8 @@ define void @ashr_64bytes(ptr %src.ptr, ptr %bitOff.ptr, ptr %dst) nounwind {
 ; X64-HAVE-BMI2-NO-SHLD-NEXT:    movq %r10, 40(%rdx)
 ; X64-HAVE-BMI2-NO-SHLD-NEXT:    movq %r9, 16(%rdx)
 ; X64-HAVE-BMI2-NO-SHLD-NEXT:    movq %rbx, 24(%rdx)
-; X64-HAVE-BMI2-NO-SHLD-NEXT:    movq %rdi, (%rdx)
-; X64-HAVE-BMI2-NO-SHLD-NEXT:    movq %r8, 8(%rdx)
+; X64-HAVE-BMI2-NO-SHLD-NEXT:    movq %r8, (%rdx)
+; X64-HAVE-BMI2-NO-SHLD-NEXT:    movq %rdi, 8(%rdx)
 ; X64-HAVE-BMI2-NO-SHLD-NEXT:    addq $8, %rsp
 ; X64-HAVE-BMI2-NO-SHLD-NEXT:    popq %rbx
 ; X64-HAVE-BMI2-NO-SHLD-NEXT:    popq %r12
diff --git a/llvm/test/CodeGen/X86/widen-load-of-small-alloca-with-zero-upper-half.ll b/llvm/test/CodeGen/X86/widen-load-of-small-alloca-with-zero-upper-half.ll
index fbbf2a6c127a5..81c4d5d71084c 100644
--- a/llvm/test/CodeGen/X86/widen-load-of-small-alloca-with-zero-upper-half.ll
+++ b/llvm/test/CodeGen/X86/widen-load-of-small-alloca-with-zero-upper-half.ll
@@ -1672,9 +1672,10 @@ define void @load_16byte_chunk_of_32byte_alloca_with_zero_upper_half(ptr %src, i
 ;
 ; X64-HAVE-BMI2-HAVE-SHLD-LABEL: load_16byte_chunk_of_32byte_alloca_with_zero_upper_half:
 ; X64-HAVE-BMI2-HAVE-SHLD:       # %bb.0:
+; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    movq %rsi, %rcx
 ; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    movups (%rdi), %xmm0
 ; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    xorps %xmm1, %xmm1
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    leal (,%rsi,8), %ecx
+; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    shll $3, %ecx
 ; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    movaps %xmm1, -{{[0-9]+}}(%rsp)
 ; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    movaps %xmm1, -{{[0-9]+}}(%rsp)
 ; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    movaps %xmm1, -{{[0-9]+}}(%rsp)
diff --git a/llvm/test/CodeGen/X86/widen-load-of-small-alloca.ll b/llvm/test/CodeGen/X86/widen-load-of-small-alloca.ll
index ff13f4ba577f2..8d36eef952a2b 100644
--- a/llvm/test/CodeGen/X86/widen-load-of-small-alloca.ll
+++ b/llvm/test/CodeGen/X86/widen-load-of-small-alloca.ll
@@ -1946,9 +1946,10 @@ define void @load_16byte_chunk_of_32byte_alloca(ptr %src, i64 %byteOff, ptr %dst
 ;
 ; X64-HAVE-BMI2-HAVE-SHLD-LABEL: load_16byte_chunk_of_32byte_alloca:
 ; X64-HAVE-BMI2-HAVE-SHLD:       # %bb.0:
+; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    movq %rsi, %rcx
 ; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    movups (%rdi), %xmm0
 ; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    movups 16(%rdi), %xmm1
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    leal (,%rsi,8), %ecx
+; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    shll $3, %ecx
 ; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    xorps %xmm2, %xmm2
 ; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    movaps %xmm2, -{{[0-9]+}}(%rsp)
 ; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    movaps %xmm2, -{{[0-9]+}}(%rsp)
diff --git a/llvm/test/CodeGen/X86/win-catchpad-csrs.ll b/llvm/test/CodeGen/X86/win-catchpad-csrs.ll
index 49f02197fede6..18546b838883b 100644
--- a/llvm/test/CodeGen/X86/win-catchpad-csrs.ll
+++ b/llvm/test/CodeGen/X86/win-catchpad-csrs.ll
@@ -109,8 +109,10 @@ handler1:
 ; X64: callq f
 ; X64: [[contbb:\.LBB0_[0-9]+]]: # Block address taken
 ; X64-NEXT:                      # %try.cont
+; X64: .seh_startepilogue
 ; X64: addq $40, %rsp
 ; X64: popq %rbp
+; X64: .seh_endepilogue
 ; X64: retq
 
 ; X64: "?catch$[[catch1bb:[0-9]+]]@?0?try_catch_catch@4HA":
@@ -131,11 +133,13 @@ handler1:
 ; X64: movl $2, %ecx
 ; X64: callq f
 ; X64: leaq [[contbb]](%rip), %rax
+; X64: .seh_startepilogue
 ; X64: addq $40, %rsp
 ; X64: popq %rbx
 ; X64: popq %rdi
 ; X64: popq %rsi
 ; X64: popq %rbp
+; X64: .seh_endepilogue
 ; X64: retq
 
 ; X64: $handlerMap$0$try_catch_catch:
@@ -182,10 +186,12 @@ try.cont:
 ; X64: callq f
 ; X64: [[contbb:\.LBB1_[0-9]+]]: # Block address taken
 ; X64-NEXT:                      # %try.cont
+; X64: .seh_startepilogue
 ; X64: addq $40, %rsp
 ; X64-NOT: popq
 ; X64: popq %rsi
 ; X64: popq %rbp
+; X64: .seh_endepilogue
 ; X64: retq
 
 ; X64: "?catch$[[catch1bb:[0-9]+]]@?0?try_one_csr@4HA":
@@ -200,9 +206,11 @@ try.cont:
 ; X64: leaq 32(%rdx), %rbp
 ; X64: .seh_endprologue
 ; X64: leaq [[contbb]](%rip), %rax
+; X64: .seh_startepilogue
 ; X64: addq $40, %rsp
 ; X64: popq %rsi
 ; X64: popq %rbp
+; X64: .seh_endepilogue
 ; X64: retq
 
 ; X64: $handlerMap$0$try_one_csr:
@@ -241,9 +249,11 @@ try.cont:
 ; X64: callq f
 ; X64: [[contbb:\.LBB2_[0-9]+]]: # Block address taken
 ; X64-NEXT:                      # %try.cont
+; X64: .seh_startepilogue
 ; X64: addq $48, %rsp
 ; X64-NOT: popq
 ; X64: popq %rbp
+; X64: .seh_endepilogue
 ; X64: retq
 
 ; X64: "?catch$[[catch1bb:[0-9]+]]@?0?try_no_csr@4HA":
@@ -256,8 +266,10 @@ try.cont:
 ; X64: leaq 48(%rdx), %rbp
 ; X64: .seh_endprologue
 ; X64: leaq [[contbb]](%rip), %rax
+; X64: .seh_startepilogue
 ; X64: addq $32, %rsp
 ; X64: popq %rbp
+; X64: .seh_endepilogue
 ; X64: retq
 
 ; X64: $handlerMap$0$try_no_csr:
diff --git a/llvm/test/CodeGen/X86/win-catchpad.ll b/llvm/test/CodeGen/X86/win-catchpad.ll
index d2067dd4e51c2..ceca37710e9ec 100644
--- a/llvm/test/CodeGen/X86/win-catchpad.ll
+++ b/llvm/test/CodeGen/X86/win-catchpad.ll
@@ -136,8 +136,10 @@ try.cont:
 ; X64: callq f
 ; X64: [[contbb:\.LBB0_[0-9]+]]: # Block address taken
 ; X64-NEXT:                      # %try.cont
+; X64: .seh_startepilogue
 ; X64: addq $[[STCK_ALLOC]], %rsp
 ; X64: popq %rbp
+; X64: .seh_endepilogue
 ; X64: retq
 
 ; X64: "?catch$[[catch1bb:[0-9]+]]@?0?try_catch_catch@4HA":
@@ -153,8 +155,10 @@ try.cont:
 ; X64-DAG: movl -4(%rbp), %ecx
 ; X64: callq f
 ; X64: leaq [[contbb]](%rip), %rax
+; X64-NEXT: .seh_startepilogue
 ; X64-NEXT: addq $32, %rsp
 ; X64-NEXT: popq %rbp
+; X64-NEXT: .seh_endepilogue
 ; X64-NEXT: retq
 
 ; X64: "?catch$[[catch2bb:[0-9]+]]@?0?try_catch_catch@4HA":
@@ -170,8 +174,10 @@ try.cont:
 ; X64-DAG: movl $3, %ecx
 ; X64: callq f
 ; X64: leaq [[contbb]](%rip), %rax
+; X64-NEXT: .seh_startepilogue
 ; X64-NEXT: addq $32, %rsp
 ; X64-NEXT: popq %rbp
+; X64-NEXT: .seh_endepilogue
 ; X64-NEXT: retq
 
 ; X64: $cppxdata$try_catch_catch:
@@ -289,8 +295,10 @@ try.cont:
 ; X64: .Ltmp[[after_call:[0-9]+]]:
 ; X64: [[contbb:\.LBB1_[0-9]+]]: # Block address taken
 ; X64-NEXT:                      # %try.cont
+; X64: .seh_startepilogue
 ; X64: addq $48, %rsp
 ; X64: popq %rbp
+; X64: .seh_endepilogue
 ; X64: retq
 
 ; X64: "?catch$[[catchbb:[0-9]+]]@?0?branch_to_normal_dest@4HA":
@@ -308,8 +316,10 @@ try.cont:
 ; X64: jne     .LBB1_[[normal_dest_bb]]
 ; X64: # %catch.done
 ; X64: leaq [[contbb]](%rip), %rax
+; X64-NEXT: .seh_startepilogue
 ; X64-NEXT: addq $32, %rsp
 ; X64-NEXT: popq %rbp
+; X64-NEXT: .seh_endepilogue
 ; X64-NEXT: retq
 
 ; X64-LABEL: $cppxdata$branch_to_normal_dest:
diff --git a/llvm/test/CodeGen/X86/win-funclet-cfi.ll b/llvm/test/CodeGen/X86/win-funclet-cfi.ll
index 7ae82e26601cd..f9a1e2f0d2880 100644
--- a/llvm/test/CodeGen/X86/win-funclet-cfi.ll
+++ b/llvm/test/CodeGen/X86/win-funclet-cfi.ll
@@ -57,6 +57,7 @@ declare i32 @__CxxFrameHandler3(...)
 ; Make sure there is a nop after a call if the call precedes the epilogue.
 ; CHECK: callq g
 ; CHECK-NEXT: nop
+; CHECK-NEXT: .seh_startepilogue
 
 ; Don't emit a reference to the LSDA.
 ; CHECK: .seh_handlerdata
@@ -87,6 +88,7 @@ declare i32 @__CxxFrameHandler3(...)
 ; Make sure there is at least one instruction after a call before the epilogue.
 ; CHECK: callq g
 ; CHECK-NEXT: leaq    .LBB0_{{[0-9]+}}(%rip), %rax
+; CHECK-NEXT: .seh_startepilogue
 
 ; Emit a reference to the LSDA.
 ; CHECK: .seh_handlerdata
diff --git a/llvm/test/CodeGen/X86/win-smallparams.ll b/llvm/test/CodeGen/X86/win-smallparams.ll
index 5ca8f6705479f..ecbc7e907a024 100644
--- a/llvm/test/CodeGen/X86/win-smallparams.ll
+++ b/llvm/test/CodeGen/X86/win-smallparams.ll
@@ -21,7 +21,9 @@ define void @call() {
 ; WIN64-NEXT:    movw $4, %r9w
 ; WIN64-NEXT:    callq manyargs
 ; WIN64-NEXT:    nop
+; WIN64-NEXT:    .seh_startepilogue
 ; WIN64-NEXT:    addq $56, %rsp
+; WIN64-NEXT:    .seh_endepilogue
 ; WIN64-NEXT:    retq
 ; WIN64-NEXT:    .seh_endproc
 ;
diff --git a/llvm/test/CodeGen/X86/win64-byval.ll b/llvm/test/CodeGen/X86/win64-byval.ll
index 6f48ce160cb6b..573a0016e8772 100644
--- a/llvm/test/CodeGen/X86/win64-byval.ll
+++ b/llvm/test/CodeGen/X86/win64-byval.ll
@@ -20,7 +20,9 @@ define void @bar() {
 ; CHECK-NEXT:    leaq {{[0-9]+}}(%rsp), %rcx
 ; CHECK-NEXT:    callq foo
 ; CHECK-NEXT:    nop
+; CHECK-NEXT:    .seh_startepilogue
 ; CHECK-NEXT:    addq $56, %rsp
+; CHECK-NEXT:    .seh_endepilogue
 ; CHECK-NEXT:    retq
 ; CHECK-NEXT:    .seh_endproc
     call void @foo(ptr byval({ float, double }) @G)
@@ -43,7 +45,9 @@ define void @baz(ptr byval({ float, double }) %arg) {
 ; CHECK-NEXT:    leaq {{[0-9]+}}(%rsp), %rcx
 ; CHECK-NEXT:    callq foo
 ; CHECK-NEXT:    nop
+; CHECK-NEXT:    .seh_startepilogue
 ; CHECK-NEXT:    addq $56, %rsp
+; CHECK-NEXT:    .seh_endepilogue
 ; CHECK-NEXT:    retq
 ; CHECK-NEXT:    .seh_endproc
     call void @foo(ptr byval({ float, double }) %arg)
@@ -81,7 +85,9 @@ define void @test() {
 ; CHECK-NEXT:    leaq {{[0-9]+}}(%rsp), %r9
 ; CHECK-NEXT:    callq foo2
 ; CHECK-NEXT:    nop
+; CHECK-NEXT:    .seh_startepilogue
 ; CHECK-NEXT:    addq $136, %rsp
+; CHECK-NEXT:    .seh_endepilogue
 ; CHECK-NEXT:    retq
 ; CHECK-NEXT:    .seh_endproc
   call void @foo2(ptr byval({ float, double }) @G, ptr byval({ float, double }) @G, ptr byval({ float, double }) @G, ptr byval({ float, double }) @G, ptr byval({ float, double }) @G, i64 10)
diff --git a/llvm/test/CodeGen/X86/win64-eh-empty-block-2.mir b/llvm/test/CodeGen/X86/win64-eh-empty-block-2.mir
index df3eb2f8aed72..6f2741faa856d 100644
--- a/llvm/test/CodeGen/X86/win64-eh-empty-block-2.mir
+++ b/llvm/test/CodeGen/X86/win64-eh-empty-block-2.mir
@@ -170,9 +170,10 @@ body:             |
 
   bb.8.return (machine-block-address-taken):
     $eax = MOV32rm $rbp, 1, $noreg, -12, $noreg :: (load (s32) from %stack.0)
-    SEH_Epilogue
+    SEH_BeginEpilogue
     $rsp = frame-destroy ADD64ri8 $rsp, 48, implicit-def dead $eflags
     $rbp = frame-destroy POP64r implicit-def $rsp, implicit $rsp
+    SEH_EndEpilogue
     RET64 $eax
 
   bb.1.if.then:
@@ -215,9 +216,10 @@ body:             |
     frame-setup SEH_EndPrologue
     MOV32mi $rbp, 1, $noreg, -12, $noreg, 1 :: (store (s32) into %stack.0)
     $rax = LEA64r $rip, 0, $noreg, %bb.8, $noreg
-    SEH_Epilogue
+    SEH_BeginEpilogue
     $rsp = frame-destroy ADD64ri8 $rsp, 32, implicit-def dead $eflags
     $rbp = frame-destroy POP64r implicit-def $rsp, implicit $rsp
+    SEH_EndEpilogue
     CATCHRET %bb.8, %bb.0
 
 ...
diff --git a/llvm/test/CodeGen/X86/win64-funclet-savexmm.ll b/llvm/test/CodeGen/X86/win64-funclet-savexmm.ll
index c8a84c15fb06b..ad675ab67963b 100644
--- a/llvm/test/CodeGen/X86/win64-funclet-savexmm.ll
+++ b/llvm/test/CodeGen/X86/win64-funclet-savexmm.ll
@@ -97,9 +97,11 @@ try.cont:                                         ; preds = %catchret.dest, %inv
 ; CHECK: vmovaps 48(%rsp), %xmm7
 ; CHECK: vmovaps 32(%rsp), %xmm8
 ; CHECK: leaq    .LBB0_1(%rip), %rax
+; CHECK: .seh_startepilogue
 ; CHECK: addq    $88, %rsp
 ; CHECK: popq    %rbx
 ; CHECK: popq    %rbp
+; CHECK: .seh_endepilogue
 ; CHECK: retq # CATCHRET
 
 ; CHECK-LABEL: "$handlerMap$0$?foo@@YAXHHHHH@Z":
diff --git a/llvm/test/CodeGen/X86/win64-seh-epilogue-statepoint.ll b/llvm/test/CodeGen/X86/win64-seh-epilogue-statepoint.ll
index 7f487a026fac2..bc5be7af6c7cf 100644
--- a/llvm/test/CodeGen/X86/win64-seh-epilogue-statepoint.ll
+++ b/llvm/test/CodeGen/X86/win64-seh-epilogue-statepoint.ll
@@ -10,7 +10,9 @@ define i32 @foobar() gc "statepoint-example" personality ptr @__gxx_personality_
 ; CHECK-NEXT:    callq bar
 ; CHECK-NEXT:  .Ltmp0:
 ; CHECK-NEXT:    nop
+; CHECK-NEXT:    .seh_startepilogue
 ; CHECK-NEXT:    addq $40, %rsp
+; CHECK-NEXT:    .seh_endepilogue
 ; CHECK-NEXT:    retq
 ; CHECK-NEXT:    .seh_endproc
     %statepoint_token = call token (i64, i32, ptr, i32, i32, ...) @llvm.experimental.gc.statepoint.p0(i64 2882400000, i32 0, ptr elementtype(i32 ()) @bar, i32 0, i32 0, i32 0, i32 0)
diff --git a/llvm/test/CodeGen/X86/win64_eh.ll b/llvm/test/CodeGen/X86/win64_eh.ll
index cbddf8c68f1cf..67088b49d1dc6 100644
--- a/llvm/test/CodeGen/X86/win64_eh.ll
+++ b/llvm/test/CodeGen/X86/win64_eh.ll
@@ -129,7 +129,9 @@ endtryfinally:
 ; ATOM:  leaq -40(%rsp), %rsp
 ; WIN64: .seh_stackalloc 40
 ; WIN64: .seh_endprologue
+; WIN64: .seh_startepilogue
 ; WIN64: addq $40, %rsp
+; WIN64: .seh_endepilogue
 ; WIN64: ret
 ; WIN64: .seh_handlerdata
 ; WIN64: .seh_endproc
@@ -163,9 +165,11 @@ entry:
 ; WIN64: andq  $-64, %rsp
 ; WIN64: movaps  -32(%rbp), %xmm6        # 16-byte Reload
 ; WIN64: movaps  -16(%rbp), %xmm7        # 16-byte Reload
+; WIN64: .seh_startepilogue
 ; WIN64: movq  %rbp, %rsp
 ; WIN64: popq  %rbx
 ; WIN64: popq  %rdi
 ; WIN64: popq  %rbp
+; WIN64: .seh_endepilogue
 ; WIN64: retq
 ; WIN64: .seh_endproc
diff --git a/llvm/test/CodeGen/X86/win64_frame.ll b/llvm/test/CodeGen/X86/win64_frame.ll
index 03b3f27705a4e..9cb3ba8e647db 100644
--- a/llvm/test/CodeGen/X86/win64_frame.ll
+++ b/llvm/test/CodeGen/X86/win64_frame.ll
@@ -11,7 +11,9 @@ define i32 @f1(i32 %p1, i32 %p2, i32 %p3, i32 %p4, i32 %p5) "frame-pointer"="all
 ; CHECK-NEXT:    .seh_setframe %rbp, 0
 ; CHECK-NEXT:    .seh_endprologue
 ; CHECK-NEXT:    movl 48(%rbp), %eax
+; CHECK-NEXT:    .seh_startepilogue
 ; CHECK-NEXT:    popq %rbp
+; CHECK-NEXT:    .seh_endepilogue
 ; CHECK-NEXT:    retq
 ; CHECK-NEXT:    .seh_endproc
   ret i32 %p5
@@ -32,8 +34,10 @@ define void @f2(i32 %p, ...) "frame-pointer"="all" {
 ; CHECK-NEXT:    movq %r9, 48(%rbp)
 ; CHECK-NEXT:    leaq 32(%rbp), %rax
 ; CHECK-NEXT:    movq %rax, (%rbp)
+; CHECK-NEXT:    .seh_startepilogue
 ; CHECK-NEXT:    addq $8, %rsp
 ; CHECK-NEXT:    popq %rbp
+; CHECK-NEXT:    .seh_endepilogue
 ; CHECK-NEXT:    retq
 ; CHECK-NEXT:    .seh_endproc
   %ap = alloca i8, align 8
@@ -50,7 +54,9 @@ define ptr @f3() "frame-pointer"="all" {
 ; CHECK-NEXT:    .seh_setframe %rbp, 0
 ; CHECK-NEXT:    .seh_endprologue
 ; CHECK-NEXT:    movq 8(%rbp), %rax
+; CHECK-NEXT:    .seh_startepilogue
 ; CHECK-NEXT:    popq %rbp
+; CHECK-NEXT:    .seh_endepilogue
 ; CHECK-NEXT:    retq
 ; CHECK-NEXT:    .seh_endproc
   %ra = call ptr @llvm.returnaddress(i32 0)
@@ -68,8 +74,10 @@ define ptr @f4() "frame-pointer"="all" {
 ; CHECK-NEXT:    .seh_setframe %rbp, 128
 ; CHECK-NEXT:    .seh_endprologue
 ; CHECK-NEXT:    movq 184(%rbp), %rax
+; CHECK-NEXT:    .seh_startepilogue
 ; CHECK-NEXT:    addq $304, %rsp # imm = 0x130
 ; CHECK-NEXT:    popq %rbp
+; CHECK-NEXT:    .seh_endepilogue
 ; CHECK-NEXT:    retq
 ; CHECK-NEXT:    .seh_endproc
   alloca [300 x i8]
@@ -92,8 +100,10 @@ define void @f5() "frame-pointer"="all" {
 ; CHECK-NEXT:    leaq -92(%rbp), %rcx
 ; CHECK-NEXT:    callq external
 ; CHECK-NEXT:    nop
+; CHECK-NEXT:    .seh_startepilogue
 ; CHECK-NEXT:    addq $336, %rsp # imm = 0x150
 ; CHECK-NEXT:    popq %rbp
+; CHECK-NEXT:    .seh_endepilogue
 ; CHECK-NEXT:    retq
 ; CHECK-NEXT:    .seh_endproc
   %a = alloca [300 x i8]
@@ -114,8 +124,10 @@ define void @f6(i32 %p, ...) "frame-pointer"="all" {
 ; CHECK-NEXT:    leaq -92(%rbp), %rcx
 ; CHECK-NEXT:    callq external
 ; CHECK-NEXT:    nop
+; CHECK-NEXT:    .seh_startepilogue
 ; CHECK-NEXT:    addq $336, %rsp # imm = 0x150
 ; CHECK-NEXT:    popq %rbp
+; CHECK-NEXT:    .seh_endepilogue
 ; CHECK-NEXT:    retq
 ; CHECK-NEXT:    .seh_endproc
   %a = alloca [300 x i8]
@@ -135,8 +147,10 @@ define i32 @f7(i32 %a, i32 %b, i32 %c, i32 %d, i32 %e) "frame-pointer"="all" {
 ; CHECK-NEXT:    .seh_endprologue
 ; CHECK-NEXT:    andq $-64, %rsp
 ; CHECK-NEXT:    movl 224(%rbp), %eax
+; CHECK-NEXT:    .seh_startepilogue
 ; CHECK-NEXT:    leaq 176(%rbp), %rsp
 ; CHECK-NEXT:    popq %rbp
+; CHECK-NEXT:    .seh_endepilogue
 ; CHECK-NEXT:    retq
 ; CHECK-NEXT:    .seh_endproc
   alloca [300 x i8], align 64
@@ -170,10 +184,12 @@ define i32 @f8(i32 %a, i32 %b, i32 %c, i32 %d, i32 %e) "frame-pointer"="all" {
 ; CHECK-NEXT:    callq external
 ; CHECK-NEXT:    addq $32, %rsp
 ; CHECK-NEXT:    movl %esi, %eax
+; CHECK-NEXT:    .seh_startepilogue
 ; CHECK-NEXT:    leaq 224(%rbp), %rsp
 ; CHECK-NEXT:    popq %rbx
 ; CHECK-NEXT:    popq %rsi
 ; CHECK-NEXT:    popq %rbp
+; CHECK-NEXT:    .seh_endepilogue
 ; CHECK-NEXT:    retq
 ; CHECK-NEXT:    .seh_endproc
   %alloca = alloca [300 x i8], align 64
@@ -192,7 +208,9 @@ define i64 @f9() {
 ; CHECK-NEXT:    .seh_endprologue
 ; CHECK-NEXT:    pushfq
 ; CHECK-NEXT:    popq %rax
+; CHECK-NEXT:    .seh_startepilogue
 ; CHECK-NEXT:    popq %rbp
+; CHECK-NEXT:    .seh_endepilogue
 ; CHECK-NEXT:    retq
 ; CHECK-NEXT:    .seh_endproc
 entry:
@@ -219,9 +237,11 @@ define i64 @f10(ptr %foo, i64 %bar, i64 %baz) {
 ; CHECK-NEXT:    callq dummy
 ; CHECK-NEXT:    testb %bl, %bl
 ; CHECK-NEXT:    cmoveq %rsi, %rax
+; CHECK-NEXT:    .seh_startepilogue
 ; CHECK-NEXT:    addq $40, %rsp
 ; CHECK-NEXT:    popq %rbx
 ; CHECK-NEXT:    popq %rsi
+; CHECK-NEXT:    .seh_endepilogue
 ; CHECK-NEXT:    retq
 ; CHECK-NEXT:    .seh_endproc
   %cx = cmpxchg ptr %foo, i64 %bar, i64 %baz seq_cst seq_cst
@@ -241,7 +261,9 @@ define ptr @f11() "frame-pointer"="all" {
 ; CHECK-NEXT:    .seh_setframe %rbp, 0
 ; CHECK-NEXT:    .seh_endprologue
 ; CHECK-NEXT:    leaq 8(%rbp), %rax
+; CHECK-NEXT:    .seh_startepilogue
 ; CHECK-NEXT:    popq %rbp
+; CHECK-NEXT:    .seh_endepilogue
 ; CHECK-NEXT:    retq
 ; CHECK-NEXT:    .seh_endproc
   %aora = call ptr @llvm.addressofreturnaddress()
diff --git a/llvm/test/CodeGen/X86/x86-64-flags-intrinsics.ll b/llvm/test/CodeGen/X86/x86-64-flags-intrinsics.ll
index e3bc77d4d5fa2..5ff645b57bd1e 100644
--- a/llvm/test/CodeGen/X86/x86-64-flags-intrinsics.ll
+++ b/llvm/test/CodeGen/X86/x86-64-flags-intrinsics.ll
@@ -21,7 +21,9 @@ define i64 @read_flags() {
 ; WIN64-NEXT:    .seh_endprologue
 ; WIN64-NEXT:    pushfq
 ; WIN64-NEXT:    popq %rax
+; WIN64-NEXT:    .seh_startepilogue
 ; WIN64-NEXT:    popq %rbp
+; WIN64-NEXT:    .seh_endepilogue
 ; WIN64-NEXT:    retq
 ; WIN64-NEXT:    .seh_endproc
 entry:
@@ -45,7 +47,9 @@ define void @write_flags(i64 %arg) {
 ; WIN64-NEXT:    .seh_endprologue
 ; WIN64-NEXT:    pushq %rcx
 ; WIN64-NEXT:    popfq
+; WIN64-NEXT:    .seh_startepilogue
 ; WIN64-NEXT:    popq %rbp
+; WIN64-NEXT:    .seh_endepilogue
 ; WIN64-NEXT:    retq
 ; WIN64-NEXT:    .seh_endproc
 entry:
diff --git a/llvm/test/CodeGen/X86/x86-win64-shrink-wrapping.ll b/llvm/test/CodeGen/X86/x86-win64-shrink-wrapping.ll
index 8309593896bf0..19637c8304dd8 100644
--- a/llvm/test/CodeGen/X86/x86-win64-shrink-wrapping.ll
+++ b/llvm/test/CodeGen/X86/x86-win64-shrink-wrapping.ll
@@ -38,12 +38,16 @@ define i32 @loopInfoSaveOutsideLoop(i32 %cond, i32 %N) #0 {
 ; ENABLE-NEXT:    nop
 ; ENABLE-NEXT:    #NO_APP
 ; ENABLE-NEXT:    shll $3, %eax
+; ENABLE-NEXT:    .seh_startepilogue
 ; ENABLE-NEXT:    popq %rbx
+; ENABLE-NEXT:    .seh_endepilogue
 ; ENABLE-NEXT:    retq
 ; ENABLE-NEXT:  .LBB0_5: # %if.else
 ; ENABLE-NEXT:    movl %edx, %eax
 ; ENABLE-NEXT:    addl %edx, %eax
+; ENABLE-NEXT:    .seh_startepilogue
 ; ENABLE-NEXT:    popq %rbx
+; ENABLE-NEXT:    .seh_endepilogue
 ; ENABLE-NEXT:    retq
 ; ENABLE-NEXT:    .seh_endproc
 ;
@@ -74,12 +78,16 @@ define i32 @loopInfoSaveOutsideLoop(i32 %cond, i32 %N) #0 {
 ; DISABLE-NEXT:    nop
 ; DISABLE-NEXT:    #NO_APP
 ; DISABLE-NEXT:    shll $3, %eax
+; DISABLE-NEXT:    .seh_startepilogue
 ; DISABLE-NEXT:    popq %rbx
+; DISABLE-NEXT:    .seh_endepilogue
 ; DISABLE-NEXT:    retq
 ; DISABLE-NEXT:  .LBB0_5: # %if.else
 ; DISABLE-NEXT:    movl %edx, %eax
 ; DISABLE-NEXT:    addl %edx, %eax
+; DISABLE-NEXT:    .seh_startepilogue
 ; DISABLE-NEXT:    popq %rbx
+; DISABLE-NEXT:    .seh_endepilogue
 ; DISABLE-NEXT:    retq
 ; DISABLE-NEXT:    .seh_endproc
 entry:
@@ -143,7 +151,9 @@ define i32 @loopInfoSaveOutsideLoop2(i32 %cond, i32 %N) #0 {
 ; ENABLE-NEXT:    nop
 ; ENABLE-NEXT:    #NO_APP
 ; ENABLE-NEXT:    shll $3, %eax
+; ENABLE-NEXT:    .seh_startepilogue
 ; ENABLE-NEXT:    popq %rbx
+; ENABLE-NEXT:    .seh_endepilogue
 ; ENABLE-NEXT:    retq
 ; ENABLE-NEXT:  .LBB1_4: # %if.else
 ; ENABLE-NEXT:    addl %edx, %edx
@@ -178,12 +188,16 @@ define i32 @loopInfoSaveOutsideLoop2(i32 %cond, i32 %N) #0 {
 ; DISABLE-NEXT:    nop
 ; DISABLE-NEXT:    #NO_APP
 ; DISABLE-NEXT:    shll $3, %eax
+; DISABLE-NEXT:    .seh_startepilogue
 ; DISABLE-NEXT:    popq %rbx
+; DISABLE-NEXT:    .seh_endepilogue
 ; DISABLE-NEXT:    retq
 ; DISABLE-NEXT:  .LBB1_5: # %if.else
 ; DISABLE-NEXT:    addl %edx, %edx
 ; DISABLE-NEXT:    movl %edx, %eax
+; DISABLE-NEXT:    .seh_startepilogue
 ; DISABLE-NEXT:    popq %rbx
+; DISABLE-NEXT:    .seh_endepilogue
 ; DISABLE-NEXT:    retq
 ; DISABLE-NEXT:    .seh_endproc
 entry:
diff --git a/llvm/test/DebugInfo/AArch64/DW_AT_APPLE_enum_kind.ll b/llvm/test/DebugInfo/AArch64/DW_AT_APPLE_enum_kind.ll
new file mode 100644
index 0000000000000..399d80c778072
--- /dev/null
+++ b/llvm/test/DebugInfo/AArch64/DW_AT_APPLE_enum_kind.ll
@@ -0,0 +1,56 @@
+; RUN: llc < %s -filetype=obj -o %t
+; RUN: llvm-dwarfdump -v %t | FileCheck %s
+
+; C++ source to regenerate:
+; enum __attribute__((enum_extensibility(open))) OpenEnum {
+;   oe1
+; } oe;
+; 
+; enum __attribute__((enum_extensibility(closed))) ClosedEnum {
+;   ce1
+; } ce;
+; 
+; $ clang++ -O0 -g debug-info-enum-kind.cpp -c
+
+
+; CHECK: .debug_abbrev contents:
+
+; CHECK: [3] DW_TAG_enumeration_type DW_CHILDREN_yes
+; CHECK: DW_AT_APPLE_enum_kind   DW_FORM_data1
+
+; CHECK: .debug_info contents:
+
+; CHECK: DW_TAG_enumeration_type [3]
+; CHECK-DAG: DW_AT_name {{.*}} string = "OpenEnum"
+; CHECK-DAG: DW_AT_APPLE_enum_kind [DW_FORM_data1]  (DW_APPLE_ENUM_KIND_Open)
+
+; CHECK: DW_TAG_enumeration_type [3]
+; CHECK-DAG: DW_AT_name {{.*}} string = "ClosedEnum"
+; CHECK-DAG: DW_AT_APPLE_enum_kind [DW_FORM_data1]  (DW_APPLE_ENUM_KIND_Closed)
+
+source_filename = "enum.cpp"
+target triple = "arm64-apple-macosx"
+
+@oe = global i32 0, align 4, !dbg !0
+@ce = global i32 0, align 4, !dbg !13
+
+!llvm.dbg.cu = !{!2}
+!llvm.module.flags = !{!15, !16}
+
+!0 = !DIGlobalVariableExpression(var: !1, expr: !DIExpression())
+!1 = distinct !DIGlobalVariable(name: "oe", scope: !2, file: !3, line: 3, type: !5, isLocal: false, isDefinition: true)
+!2 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus_14, file: !3, producer: "clang version 21.0.0git", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, enums: !4, globals: !12, splitDebugInlining: false, nameTableKind: Apple, sysroot: "/")
+!3 = !DIFile(filename: "enum.cpp", directory: "/tmp")
+!4 = !{!5, !9}
+!5 = !DICompositeType(tag: DW_TAG_enumeration_type, name: "OpenEnum", file: !3, line: 1, baseType: !6, size: 32, elements: !7, identifier: "_ZTS8OpenEnum", enumKind: DW_APPLE_ENUM_KIND_Open)
+!6 = !DIBasicType(name: "unsigned int", size: 32, encoding: DW_ATE_unsigned)
+!7 = !{!8}
+!8 = !DIEnumerator(name: "oe1", value: 0, isUnsigned: true)
+!9 = !DICompositeType(tag: DW_TAG_enumeration_type, name: "ClosedEnum", file: !3, line: 5, baseType: !6, size: 32, elements: !10, identifier: "_ZTS10ClosedEnum", enumKind: DW_APPLE_ENUM_KIND_Closed)
+!10 = !{!11}
+!11 = !DIEnumerator(name: "ce1", value: 0, isUnsigned: true)
+!12 = !{!0, !13}
+!13 = !DIGlobalVariableExpression(var: !14, expr: !DIExpression())
+!14 = distinct !DIGlobalVariable(name: "ce", scope: !2, file: !3, line: 7, type: !9, isLocal: false, isDefinition: true)
+!15 = !{i32 7, !"Dwarf Version", i32 5}
+!16 = !{i32 2, !"Debug Info Version", i32 3}
diff --git a/llvm/test/DebugInfo/AArch64/fake-use-global-isel.ll b/llvm/test/DebugInfo/AArch64/fake-use-global-isel.ll
index 65a6458309695..2339b067c71e4 100644
--- a/llvm/test/DebugInfo/AArch64/fake-use-global-isel.ll
+++ b/llvm/test/DebugInfo/AArch64/fake-use-global-isel.ll
@@ -12,7 +12,7 @@
 ; RUN: not %python %p/../Inputs/check-fake-use.py %t
 
 ; Generated with:
-; clang -O2 -g -S -emit-llvm -fextend-this-ptr fake-use.c
+; clang -O2 -g -S -emit-llvm -fextend-variable-liveness=this fake-use.c
 ;
 ; int glob[10];
 ; extern void bar();
diff --git a/llvm/test/DebugInfo/COFF/trailing-inlined-function.s b/llvm/test/DebugInfo/COFF/trailing-inlined-function.s
index 133f48e3038aa..2619222b8167c 100644
--- a/llvm/test/DebugInfo/COFF/trailing-inlined-function.s
+++ b/llvm/test/DebugInfo/COFF/trailing-inlined-function.s
@@ -94,7 +94,9 @@ add_numbers:
 	addl	4(%rdx), %eax
 .Ltmp4:
 	.cv_loc	0 1 6 0
+	.seh_startepilogue
 	addq	$40, %rsp
+	.seh_endepilogue
 	retq
 .LBB0_1:
 .Ltmp5:
diff --git a/llvm/test/DebugInfo/Generic/discriminated-union.ll b/llvm/test/DebugInfo/Generic/discriminated-union.ll
index d267d9b029e95..0267580eb0cbf 100644
--- a/llvm/test/DebugInfo/Generic/discriminated-union.ll
+++ b/llvm/test/DebugInfo/Generic/discriminated-union.ll
@@ -1,8 +1,8 @@
 ; RUN: %llc_dwarf -O0 -filetype=obj < %s > %t
-; RUN: llvm-dwarfdump -v -debug-info %t | FileCheck %s
+; RUN: llvm-dwarfdump -v -debug-info %t | FileCheck %s --check-prefix %if target-byteorder-big-endian %{ CHECK-BE %} %else %{ CHECK-LE %}
 
 ; RUN: %llc_dwarf --try-experimental-debuginfo-iterators -O0 -filetype=obj < %s > %t
-; RUN: llvm-dwarfdump -v -debug-info %t | FileCheck %s
+; RUN: llvm-dwarfdump -v -debug-info %t | FileCheck %s --check-prefix %if target-byteorder-big-endian %{ CHECK-BE %} %else %{ CHECK-LE %}
 
 ; Check for a variant part that has two members, one of which has a
 ; discriminant value.
@@ -22,7 +22,8 @@
 ;         CHECK: DW_AT_alignment
 ;         CHECK: DW_AT_data_member_location [DW_FORM_data1]	(0x00)
 ;     CHECK: DW_TAG_variant
-;       CHECK: DW_AT_discr_value [DW_FORM_data1]	(0x00)
+;       CHECK-LE: DW_AT_discr_value [DW_FORM_block1]	(<0x10> 00 00 00 00 00 00 00 00 01 00 00 00 00 00 00 00 )
+;       CHECK-BE: DW_AT_discr_value [DW_FORM_block1]	(<0x10> 00 00 00 00 00 00 00 01 00 00 00 00 00 00 00 00 )
 ;       CHECK: DW_TAG_member
 ;         CHECK: DW_AT_type
 ;         CHECK: DW_AT_alignment
@@ -71,7 +72,7 @@ attributes #0 = { nounwind uwtable }
 !21 = !DIBasicType(name: "u8", size: 8, encoding: DW_ATE_unsigned)
 !22 = !DIDerivedType(tag: DW_TAG_member, name: "__1", scope: !18, file: !7, baseType: !23, size: 64, align: 64)
 !23 = !DIDerivedType(tag: DW_TAG_pointer_type, name: "&u8", baseType: !21, size: 64, align: 64)
-!24 = !DIDerivedType(tag: DW_TAG_member, scope: !14, file: !7, baseType: !25, size: 128, align: 64, extraData: i64 0)
+!24 = !DIDerivedType(tag: DW_TAG_member, scope: !14, file: !7, baseType: !25, size: 128, align: 64, extraData: i128 18446744073709551616)
 !25 = !DICompositeType(tag: DW_TAG_structure_type, name: "Nope", scope: !12, file: !7, size: 128, align: 64, elements: !4, identifier: "7ce1efff6b82281ab9ceb730566e7e20::Nope")
 !27 = !DIBasicType(name: "u64", size: 64, encoding: DW_ATE_unsigned)
 !28 = !DIExpression()
diff --git a/llvm/test/DebugInfo/MIR/X86/instr-ref-join-def-vphi.mir b/llvm/test/DebugInfo/MIR/X86/instr-ref-join-def-vphi.mir
index a23d80ea64b39..b1531aa8ee878 100644
--- a/llvm/test/DebugInfo/MIR/X86/instr-ref-join-def-vphi.mir
+++ b/llvm/test/DebugInfo/MIR/X86/instr-ref-join-def-vphi.mir
@@ -233,10 +233,11 @@ body:             |
     liveins: $esi
 
     $eax = MOV32rr killed $esi, debug-location !35
-    SEH_Epilogue debug-location !35
+    SEH_BeginEpilogue debug-location !35
     $rsp = frame-destroy ADD64ri8 $rsp, 40, implicit-def dead $eflags, debug-location !35
     $rdi = frame-destroy POP64r implicit-def $rsp, implicit $rsp, debug-location !35
     $rsi = frame-destroy POP64r implicit-def $rsp, implicit $rsp, debug-location !35
+    SEH_EndEpilogue debug-location !35
     RET64 $eax, debug-location !35
 
 ...
diff --git a/llvm/test/DebugInfo/NVPTX/debug-addr-space.ll b/llvm/test/DebugInfo/NVPTX/debug-addr-space.ll
new file mode 100644
index 0000000000000..a96b51969a811
--- /dev/null
+++ b/llvm/test/DebugInfo/NVPTX/debug-addr-space.ll
@@ -0,0 +1,25 @@
+; RUN: llc < %s -mtriple=nvptx64-nvidia-cuda | FileCheck %s
+
+; Test that translateToNVVMDWARFAddrSpace() function translates NVVM IR address space
+; value `Shared` (3) to the corresponding DWARF DW_AT_address_class attribute for PTX.
+
+; CHECK: .section .debug_info
+; CHECK:      .b8 103                                 // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b32 55                                 // DW_AT_type
+; CHECK-NEXT: .b8 1                                   // DW_AT_decl_file
+; CHECK-NEXT: .b8 1                                   // DW_AT_decl_line
+; CHECK-NEXT: .b8 8                                   // DW_AT_address_class
+
+@g = internal addrspace(3) global i32 0, align 4, !dbg !0
+
+!llvm.dbg.cu = !{!2}
+!llvm.module.flags = !{!6}
+
+!0 = !DIGlobalVariableExpression(var: !1, expr: !DIExpression())
+!1 = distinct !DIGlobalVariable(name: "g", linkageName: "g", scope: !2, file: !3, line: 1, type: !5, isLocal: true, isDefinition: true)
+!2 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus, file: !3, isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, globals: !4)
+!3 = !DIFile(filename: "test.cu", directory: "test")
+!4 = !{!0}
+!5 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+!6 = !{i32 1, !"Debug Info Version", i32 3}
diff --git a/llvm/test/DebugInfo/NVPTX/debug-info.ll b/llvm/test/DebugInfo/NVPTX/debug-info.ll
index 51fb692789e22..62b30a1f15aff 100644
--- a/llvm/test/DebugInfo/NVPTX/debug-info.ll
+++ b/llvm/test/DebugInfo/NVPTX/debug-info.ll
@@ -98,2584 +98,2554 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-DAG: .file {{[0-9]+}} "{{.*}}/usr/local/cuda/include{{/|\\\\}}vector_types.h"
 
 ; CHECK:	.section	.debug_loc
-; CHECK-NEXT:	{
-; CHECK-NEXT:$L__debug_loc0:
-; CHECK-NEXT:.b64 $L__tmp8
-; CHECK-NEXT:.b64 $L__tmp10
-; CHECK-NEXT:.b8 5                                   // Loc expr size
-; CHECK-NEXT:.b8 0
-; CHECK-NEXT:.b8 144                                 // DW_OP_regx
-; CHECK-NEXT:.b8 177                                 // 2450993
-; CHECK-NEXT:.b8 204                                 //
-; CHECK-NEXT:.b8 149                                 //
-; CHECK-NEXT:.b8 1                                   //
-; CHECK-NEXT:.b64 0
-; CHECK-NEXT:.b64 0
-; CHECK-NEXT:$L__debug_loc1:
-; CHECK-NEXT:.b64 $L__tmp5
-; CHECK-NEXT:.b64 $L__func_end0
-; CHECK-NEXT:.b8 5                                   // Loc expr size
-; CHECK-NEXT:.b8 0
-; CHECK-NEXT:.b8 144                                 // DW_OP_regx
-; CHECK-NEXT:.b8 177                                 // 2454065
-; CHECK-NEXT:.b8 228                                 //
-; CHECK-NEXT:.b8 149                                 //
-; CHECK-NEXT:.b8 1                                   //
-; CHECK-NEXT:.b64 0
-; CHECK-NEXT:.b64 0
-; CHECK-NEXT:	}
-; CHECK-NEXT:	.section	.debug_abbrev
-; CHECK-NEXT:	{
-; CHECK-NEXT:.b8 1                                   // Abbreviation Code
-; CHECK-NEXT:.b8 17                                  // DW_TAG_compile_unit
-; CHECK-NEXT:.b8 1                                   // DW_CHILDREN_yes
-; CHECK-NEXT:.b8 37                                  // DW_AT_producer
-; CHECK-NEXT:.b8 8                                   // DW_FORM_string
-; CHECK-NEXT:.b8 19                                  // DW_AT_language
-; CHECK-NEXT:.b8 5                                   // DW_FORM_data2
-; CHECK-NEXT:.b8 3                                   // DW_AT_name
-; CHECK-NEXT:.b8 8                                   // DW_FORM_string
-; CHECK-NEXT:.b8 16                                  // DW_AT_stmt_list
-; CHECK-NEXT:.b8 6                                   // DW_FORM_data4
-; CHECK-NEXT:.b8 27                                  // DW_AT_comp_dir
-; CHECK-NEXT:.b8 8                                   // DW_FORM_string
-; CHECK-NEXT:.b8 0                                   // EOM(1)
-; CHECK-NEXT:.b8 0                                   // EOM(2)
-; CHECK-NEXT:.b8 2                                   // Abbreviation Code
-; CHECK-NEXT:.b8 19                                  // DW_TAG_structure_type
-; CHECK-NEXT:.b8 1                                   // DW_CHILDREN_yes
-; CHECK-NEXT:.b8 3                                   // DW_AT_name
-; CHECK-NEXT:.b8 8                                   // DW_FORM_string
-; CHECK-NEXT:.b8 11                                  // DW_AT_byte_size
-; CHECK-NEXT:.b8 11                                  // DW_FORM_data1
-; CHECK-NEXT:.b8 58                                  // DW_AT_decl_file
-; CHECK-NEXT:.b8 11                                  // DW_FORM_data1
-; CHECK-NEXT:.b8 59                                  // DW_AT_decl_line
-; CHECK-NEXT:.b8 11                                  // DW_FORM_data1
-; CHECK-NEXT:.b8 0                                   // EOM(1)
-; CHECK-NEXT:.b8 0                                   // EOM(2)
-; CHECK-NEXT:.b8 3                                   // Abbreviation Code
-; CHECK-NEXT:.b8 46                                  // DW_TAG_subprogram
-; CHECK-NEXT:.b8 0                                   // DW_CHILDREN_no
-; CHECK-NEXT:.b8 135                                 // DW_AT_MIPS_linkage_name
-; CHECK-NEXT:.b8 64
-; CHECK-NEXT:.b8 8                                   // DW_FORM_string
-; CHECK-NEXT:.b8 3                                   // DW_AT_name
-; CHECK-NEXT:.b8 8                                   // DW_FORM_string
-; CHECK-NEXT:.b8 58                                  // DW_AT_decl_file
-; CHECK-NEXT:.b8 11                                  // DW_FORM_data1
-; CHECK-NEXT:.b8 59                                  // DW_AT_decl_line
-; CHECK-NEXT:.b8 11                                  // DW_FORM_data1
-; CHECK-NEXT:.b8 73                                  // DW_AT_type
-; CHECK-NEXT:.b8 19                                  // DW_FORM_ref4
-; CHECK-NEXT:.b8 60                                  // DW_AT_declaration
-; CHECK-NEXT:.b8 12                                  // DW_FORM_flag
-; CHECK-NEXT:.b8 63                                  // DW_AT_external
-; CHECK-NEXT:.b8 12                                  // DW_FORM_flag
-; CHECK-NEXT:.b8 0                                   // EOM(1)
-; CHECK-NEXT:.b8 0                                   // EOM(2)
-; CHECK-NEXT:.b8 4                                   // Abbreviation Code
-; CHECK-NEXT:.b8 46                                  // DW_TAG_subprogram
-; CHECK-NEXT:.b8 1                                   // DW_CHILDREN_yes
-; CHECK-NEXT:.b8 135                                 // DW_AT_MIPS_linkage_name
-; CHECK-NEXT:.b8 64
-; CHECK-NEXT:.b8 8                                   // DW_FORM_string
-; CHECK-NEXT:.b8 3                                   // DW_AT_name
-; CHECK-NEXT:.b8 8                                   // DW_FORM_string
-; CHECK-NEXT:.b8 58                                  // DW_AT_decl_file
-; CHECK-NEXT:.b8 11                                  // DW_FORM_data1
-; CHECK-NEXT:.b8 59                                  // DW_AT_decl_line
-; CHECK-NEXT:.b8 11                                  // DW_FORM_data1
-; CHECK-NEXT:.b8 73                                  // DW_AT_type
-; CHECK-NEXT:.b8 19                                  // DW_FORM_ref4
-; CHECK-NEXT:.b8 60                                  // DW_AT_declaration
-; CHECK-NEXT:.b8 12                                  // DW_FORM_flag
-; CHECK-NEXT:.b8 100                                 // DW_AT_object_pointer
-; CHECK-NEXT:.b8 19                                  // DW_FORM_ref4
-; CHECK-NEXT:.b8 63                                  // DW_AT_external
-; CHECK-NEXT:.b8 12                                  // DW_FORM_flag
-; CHECK-NEXT:.b8 0                                   // EOM(1)
-; CHECK-NEXT:.b8 0                                   // EOM(2)
-; CHECK-NEXT:.b8 5                                   // Abbreviation Code
-; CHECK-NEXT:.b8 5                                   // DW_TAG_formal_parameter
-; CHECK-NEXT:.b8 0                                   // DW_CHILDREN_no
-; CHECK-NEXT:.b8 73                                  // DW_AT_type
-; CHECK-NEXT:.b8 19                                  // DW_FORM_ref4
-; CHECK-NEXT:.b8 52                                  // DW_AT_artificial
-; CHECK-NEXT:.b8 12                                  // DW_FORM_flag
-; CHECK-NEXT:.b8 0                                   // EOM(1)
-; CHECK-NEXT:.b8 0                                   // EOM(2)
-; CHECK-NEXT:.b8 6                                   // Abbreviation Code
-; CHECK-NEXT:.b8 46                                  // DW_TAG_subprogram
-; CHECK-NEXT:.b8 1                                   // DW_CHILDREN_yes
-; CHECK-NEXT:.b8 3                                   // DW_AT_name
-; CHECK-NEXT:.b8 8                                   // DW_FORM_string
-; CHECK-NEXT:.b8 58                                  // DW_AT_decl_file
-; CHECK-NEXT:.b8 11                                  // DW_FORM_data1
-; CHECK-NEXT:.b8 59                                  // DW_AT_decl_line
-; CHECK-NEXT:.b8 11                                  // DW_FORM_data1
-; CHECK-NEXT:.b8 60                                  // DW_AT_declaration
-; CHECK-NEXT:.b8 12                                  // DW_FORM_flag
-; CHECK-NEXT:.b8 100                                 // DW_AT_object_pointer
-; CHECK-NEXT:.b8 19                                  // DW_FORM_ref4
-; CHECK-NEXT:.b8 63                                  // DW_AT_external
-; CHECK-NEXT:.b8 12                                  // DW_FORM_flag
-; CHECK-NEXT:.b8 50                                  // DW_AT_accessibility
-; CHECK-NEXT:.b8 11                                  // DW_FORM_data1
-; CHECK-NEXT:.b8 0                                   // EOM(1)
-; CHECK-NEXT:.b8 0                                   // EOM(2)
-; CHECK-NEXT:.b8 7                                   // Abbreviation Code
-; CHECK-NEXT:.b8 5                                   // DW_TAG_formal_parameter
-; CHECK-NEXT:.b8 0                                   // DW_CHILDREN_no
-; CHECK-NEXT:.b8 73                                  // DW_AT_type
-; CHECK-NEXT:.b8 19                                  // DW_FORM_ref4
-; CHECK-NEXT:.b8 0                                   // EOM(1)
-; CHECK-NEXT:.b8 0                                   // EOM(2)
-; CHECK-NEXT:.b8 8                                   // Abbreviation Code
-; CHECK-NEXT:.b8 46                                  // DW_TAG_subprogram
-; CHECK-NEXT:.b8 1                                   // DW_CHILDREN_yes
-; CHECK-NEXT:.b8 135                                 // DW_AT_MIPS_linkage_name
-; CHECK-NEXT:.b8 64
-; CHECK-NEXT:.b8 8                                   // DW_FORM_string
-; CHECK-NEXT:.b8 3                                   // DW_AT_name
-; CHECK-NEXT:.b8 8                                   // DW_FORM_string
-; CHECK-NEXT:.b8 58                                  // DW_AT_decl_file
-; CHECK-NEXT:.b8 11                                  // DW_FORM_data1
-; CHECK-NEXT:.b8 59                                  // DW_AT_decl_line
-; CHECK-NEXT:.b8 11                                  // DW_FORM_data1
-; CHECK-NEXT:.b8 60                                  // DW_AT_declaration
-; CHECK-NEXT:.b8 12                                  // DW_FORM_flag
-; CHECK-NEXT:.b8 100                                 // DW_AT_object_pointer
-; CHECK-NEXT:.b8 19                                  // DW_FORM_ref4
-; CHECK-NEXT:.b8 63                                  // DW_AT_external
-; CHECK-NEXT:.b8 12                                  // DW_FORM_flag
-; CHECK-NEXT:.b8 50                                  // DW_AT_accessibility
-; CHECK-NEXT:.b8 11                                  // DW_FORM_data1
-; CHECK-NEXT:.b8 0                                   // EOM(1)
-; CHECK-NEXT:.b8 0                                   // EOM(2)
-; CHECK-NEXT:.b8 9                                   // Abbreviation Code
-; CHECK-NEXT:.b8 46                                  // DW_TAG_subprogram
-; CHECK-NEXT:.b8 1                                   // DW_CHILDREN_yes
-; CHECK-NEXT:.b8 135                                 // DW_AT_MIPS_linkage_name
-; CHECK-NEXT:.b8 64
-; CHECK-NEXT:.b8 8                                   // DW_FORM_string
-; CHECK-NEXT:.b8 3                                   // DW_AT_name
-; CHECK-NEXT:.b8 8                                   // DW_FORM_string
-; CHECK-NEXT:.b8 58                                  // DW_AT_decl_file
-; CHECK-NEXT:.b8 11                                  // DW_FORM_data1
-; CHECK-NEXT:.b8 59                                  // DW_AT_decl_line
-; CHECK-NEXT:.b8 11                                  // DW_FORM_data1
-; CHECK-NEXT:.b8 73                                  // DW_AT_type
-; CHECK-NEXT:.b8 19                                  // DW_FORM_ref4
-; CHECK-NEXT:.b8 60                                  // DW_AT_declaration
-; CHECK-NEXT:.b8 12                                  // DW_FORM_flag
-; CHECK-NEXT:.b8 100                                 // DW_AT_object_pointer
-; CHECK-NEXT:.b8 19                                  // DW_FORM_ref4
-; CHECK-NEXT:.b8 63                                  // DW_AT_external
-; CHECK-NEXT:.b8 12                                  // DW_FORM_flag
-; CHECK-NEXT:.b8 50                                  // DW_AT_accessibility
-; CHECK-NEXT:.b8 11                                  // DW_FORM_data1
-; CHECK-NEXT:.b8 0                                   // EOM(1)
-; CHECK-NEXT:.b8 0                                   // EOM(2)
-; CHECK-NEXT:.b8 10                                  // Abbreviation Code
-; CHECK-NEXT:.b8 36                                  // DW_TAG_base_type
-; CHECK-NEXT:.b8 0                                   // DW_CHILDREN_no
-; CHECK-NEXT:.b8 3                                   // DW_AT_name
-; CHECK-NEXT:.b8 8                                   // DW_FORM_string
-; CHECK-NEXT:.b8 62                                  // DW_AT_encoding
-; CHECK-NEXT:.b8 11                                  // DW_FORM_data1
-; CHECK-NEXT:.b8 11                                  // DW_AT_byte_size
-; CHECK-NEXT:.b8 11                                  // DW_FORM_data1
-; CHECK-NEXT:.b8 0                                   // EOM(1)
-; CHECK-NEXT:.b8 0                                   // EOM(2)
-; CHECK-NEXT:.b8 11                                  // Abbreviation Code
-; CHECK-NEXT:.b8 13                                  // DW_TAG_member
-; CHECK-NEXT:.b8 0                                   // DW_CHILDREN_no
-; CHECK-NEXT:.b8 3                                   // DW_AT_name
-; CHECK-NEXT:.b8 8                                   // DW_FORM_string
-; CHECK-NEXT:.b8 73                                  // DW_AT_type
-; CHECK-NEXT:.b8 19                                  // DW_FORM_ref4
-; CHECK-NEXT:.b8 58                                  // DW_AT_decl_file
-; CHECK-NEXT:.b8 11                                  // DW_FORM_data1
-; CHECK-NEXT:.b8 59                                  // DW_AT_decl_line
-; CHECK-NEXT:.b8 11                                  // DW_FORM_data1
-; CHECK-NEXT:.b8 56                                  // DW_AT_data_member_location
-; CHECK-NEXT:.b8 10                                  // DW_FORM_block1
-; CHECK-NEXT:.b8 0                                   // EOM(1)
-; CHECK-NEXT:.b8 0                                   // EOM(2)
-; CHECK-NEXT:.b8 12                                  // Abbreviation Code
-; CHECK-NEXT:.b8 15                                  // DW_TAG_pointer_type
-; CHECK-NEXT:.b8 0                                   // DW_CHILDREN_no
-; CHECK-NEXT:.b8 73                                  // DW_AT_type
-; CHECK-NEXT:.b8 19                                  // DW_FORM_ref4
-; CHECK-NEXT:.b8 0                                   // EOM(1)
-; CHECK-NEXT:.b8 0                                   // EOM(2)
-; CHECK-NEXT:.b8 13                                  // Abbreviation Code
-; CHECK-NEXT:.b8 38                                  // DW_TAG_const_type
-; CHECK-NEXT:.b8 0                                   // DW_CHILDREN_no
-; CHECK-NEXT:.b8 73                                  // DW_AT_type
-; CHECK-NEXT:.b8 19                                  // DW_FORM_ref4
-; CHECK-NEXT:.b8 0                                   // EOM(1)
-; CHECK-NEXT:.b8 0                                   // EOM(2)
-; CHECK-NEXT:.b8 14                                  // Abbreviation Code
-; CHECK-NEXT:.b8 16                                  // DW_TAG_reference_type
-; CHECK-NEXT:.b8 0                                   // DW_CHILDREN_no
-; CHECK-NEXT:.b8 73                                  // DW_AT_type
-; CHECK-NEXT:.b8 19                                  // DW_FORM_ref4
-; CHECK-NEXT:.b8 0                                   // EOM(1)
-; CHECK-NEXT:.b8 0                                   // EOM(2)
-; CHECK-NEXT:.b8 15                                  // Abbreviation Code
-; CHECK-NEXT:.b8 46                                  // DW_TAG_subprogram
-; CHECK-NEXT:.b8 0                                   // DW_CHILDREN_no
-; CHECK-NEXT:.b8 71                                  // DW_AT_specification
-; CHECK-NEXT:.b8 19                                  // DW_FORM_ref4
-; CHECK-NEXT:.b8 32                                  // DW_AT_inline
-; CHECK-NEXT:.b8 11                                  // DW_FORM_data1
-; CHECK-NEXT:.b8 0                                   // EOM(1)
-; CHECK-NEXT:.b8 0                                   // EOM(2)
-; CHECK-NEXT:.b8 16                                  // Abbreviation Code
-; CHECK-NEXT:.b8 19                                  // DW_TAG_structure_type
-; CHECK-NEXT:.b8 1                                   // DW_CHILDREN_yes
-; CHECK-NEXT:.b8 3                                   // DW_AT_name
-; CHECK-NEXT:.b8 8                                   // DW_FORM_string
-; CHECK-NEXT:.b8 11                                  // DW_AT_byte_size
-; CHECK-NEXT:.b8 11                                  // DW_FORM_data1
-; CHECK-NEXT:.b8 58                                  // DW_AT_decl_file
-; CHECK-NEXT:.b8 11                                  // DW_FORM_data1
-; CHECK-NEXT:.b8 59                                  // DW_AT_decl_line
-; CHECK-NEXT:.b8 5                                   // DW_FORM_data2
-; CHECK-NEXT:.b8 0                                   // EOM(1)
-; CHECK-NEXT:.b8 0                                   // EOM(2)
-; CHECK-NEXT:.b8 17                                  // Abbreviation Code
-; CHECK-NEXT:.b8 13                                  // DW_TAG_member
-; CHECK-NEXT:.b8 0                                   // DW_CHILDREN_no
-; CHECK-NEXT:.b8 3                                   // DW_AT_name
-; CHECK-NEXT:.b8 8                                   // DW_FORM_string
-; CHECK-NEXT:.b8 73                                  // DW_AT_type
-; CHECK-NEXT:.b8 19                                  // DW_FORM_ref4
-; CHECK-NEXT:.b8 58                                  // DW_AT_decl_file
-; CHECK-NEXT:.b8 11                                  // DW_FORM_data1
-; CHECK-NEXT:.b8 59                                  // DW_AT_decl_line
-; CHECK-NEXT:.b8 5                                   // DW_FORM_data2
-; CHECK-NEXT:.b8 56                                  // DW_AT_data_member_location
-; CHECK-NEXT:.b8 10                                  // DW_FORM_block1
-; CHECK-NEXT:.b8 0                                   // EOM(1)
-; CHECK-NEXT:.b8 0                                   // EOM(2)
-; CHECK-NEXT:.b8 18                                  // Abbreviation Code
-; CHECK-NEXT:.b8 46                                  // DW_TAG_subprogram
-; CHECK-NEXT:.b8 1                                   // DW_CHILDREN_yes
-; CHECK-NEXT:.b8 3                                   // DW_AT_name
-; CHECK-NEXT:.b8 8                                   // DW_FORM_string
-; CHECK-NEXT:.b8 58                                  // DW_AT_decl_file
-; CHECK-NEXT:.b8 11                                  // DW_FORM_data1
-; CHECK-NEXT:.b8 59                                  // DW_AT_decl_line
-; CHECK-NEXT:.b8 5                                   // DW_FORM_data2
-; CHECK-NEXT:.b8 60                                  // DW_AT_declaration
-; CHECK-NEXT:.b8 12                                  // DW_FORM_flag
-; CHECK-NEXT:.b8 100                                 // DW_AT_object_pointer
-; CHECK-NEXT:.b8 19                                  // DW_FORM_ref4
-; CHECK-NEXT:.b8 63                                  // DW_AT_external
-; CHECK-NEXT:.b8 12                                  // DW_FORM_flag
-; CHECK-NEXT:.b8 0                                   // EOM(1)
-; CHECK-NEXT:.b8 0                                   // EOM(2)
-; CHECK-NEXT:.b8 19                                  // Abbreviation Code
-; CHECK-NEXT:.b8 46                                  // DW_TAG_subprogram
-; CHECK-NEXT:.b8 1                                   // DW_CHILDREN_yes
-; CHECK-NEXT:.b8 135                                 // DW_AT_MIPS_linkage_name
-; CHECK-NEXT:.b8 64
-; CHECK-NEXT:.b8 8                                   // DW_FORM_string
-; CHECK-NEXT:.b8 3                                   // DW_AT_name
-; CHECK-NEXT:.b8 8                                   // DW_FORM_string
-; CHECK-NEXT:.b8 58                                  // DW_AT_decl_file
-; CHECK-NEXT:.b8 11                                  // DW_FORM_data1
-; CHECK-NEXT:.b8 59                                  // DW_AT_decl_line
-; CHECK-NEXT:.b8 5                                   // DW_FORM_data2
-; CHECK-NEXT:.b8 73                                  // DW_AT_type
-; CHECK-NEXT:.b8 19                                  // DW_FORM_ref4
-; CHECK-NEXT:.b8 60                                  // DW_AT_declaration
-; CHECK-NEXT:.b8 12                                  // DW_FORM_flag
-; CHECK-NEXT:.b8 100                                 // DW_AT_object_pointer
-; CHECK-NEXT:.b8 19                                  // DW_FORM_ref4
-; CHECK-NEXT:.b8 63                                  // DW_AT_external
-; CHECK-NEXT:.b8 12                                  // DW_FORM_flag
-; CHECK-NEXT:.b8 0                                   // EOM(1)
-; CHECK-NEXT:.b8 0                                   // EOM(2)
-; CHECK-NEXT:.b8 20                                  // Abbreviation Code
-; CHECK-NEXT:.b8 22                                  // DW_TAG_typedef
-; CHECK-NEXT:.b8 0                                   // DW_CHILDREN_no
-; CHECK-NEXT:.b8 73                                  // DW_AT_type
-; CHECK-NEXT:.b8 19                                  // DW_FORM_ref4
-; CHECK-NEXT:.b8 3                                   // DW_AT_name
-; CHECK-NEXT:.b8 8                                   // DW_FORM_string
-; CHECK-NEXT:.b8 58                                  // DW_AT_decl_file
-; CHECK-NEXT:.b8 11                                  // DW_FORM_data1
-; CHECK-NEXT:.b8 59                                  // DW_AT_decl_line
-; CHECK-NEXT:.b8 5                                   // DW_FORM_data2
-; CHECK-NEXT:.b8 0                                   // EOM(1)
-; CHECK-NEXT:.b8 0                                   // EOM(2)
-; CHECK-NEXT:.b8 21                                  // Abbreviation Code
-; CHECK-NEXT:.b8 46                                  // DW_TAG_subprogram
-; CHECK-NEXT:.b8 1                                   // DW_CHILDREN_yes
-; CHECK-NEXT:.b8 135                                 // DW_AT_MIPS_linkage_name
-; CHECK-NEXT:.b8 64
-; CHECK-NEXT:.b8 8                                   // DW_FORM_string
-; CHECK-NEXT:.b8 3                                   // DW_AT_name
-; CHECK-NEXT:.b8 8                                   // DW_FORM_string
-; CHECK-NEXT:.b8 58                                  // DW_AT_decl_file
-; CHECK-NEXT:.b8 11                                  // DW_FORM_data1
-; CHECK-NEXT:.b8 59                                  // DW_AT_decl_line
-; CHECK-NEXT:.b8 11                                  // DW_FORM_data1
-; CHECK-NEXT:.b8 63                                  // DW_AT_external
-; CHECK-NEXT:.b8 12                                  // DW_FORM_flag
-; CHECK-NEXT:.b8 32                                  // DW_AT_inline
-; CHECK-NEXT:.b8 11                                  // DW_FORM_data1
-; CHECK-NEXT:.b8 0                                   // EOM(1)
-; CHECK-NEXT:.b8 0                                   // EOM(2)
-; CHECK-NEXT:.b8 22                                  // Abbreviation Code
-; CHECK-NEXT:.b8 5                                   // DW_TAG_formal_parameter
-; CHECK-NEXT:.b8 0                                   // DW_CHILDREN_no
-; CHECK-NEXT:.b8 3                                   // DW_AT_name
-; CHECK-NEXT:.b8 8                                   // DW_FORM_string
-; CHECK-NEXT:.b8 58                                  // DW_AT_decl_file
-; CHECK-NEXT:.b8 11                                  // DW_FORM_data1
-; CHECK-NEXT:.b8 59                                  // DW_AT_decl_line
-; CHECK-NEXT:.b8 11                                  // DW_FORM_data1
-; CHECK-NEXT:.b8 73                                  // DW_AT_type
-; CHECK-NEXT:.b8 19                                  // DW_FORM_ref4
-; CHECK-NEXT:.b8 0                                   // EOM(1)
-; CHECK-NEXT:.b8 0                                   // EOM(2)
-; CHECK-NEXT:.b8 23                                  // Abbreviation Code
-; CHECK-NEXT:.b8 46                                  // DW_TAG_subprogram
-; CHECK-NEXT:.b8 1                                   // DW_CHILDREN_yes
-; CHECK-NEXT:.b8 17                                  // DW_AT_low_pc
-; CHECK-NEXT:.b8 1                                   // DW_FORM_addr
-; CHECK-NEXT:.b8 18                                  // DW_AT_high_pc
-; CHECK-NEXT:.b8 1                                   // DW_FORM_addr
-; CHECK-NEXT:.b8 64                                  // DW_AT_frame_base
-; CHECK-NEXT:.b8 10                                  // DW_FORM_block1
-; CHECK-NEXT:.b8 135                                 // DW_AT_MIPS_linkage_name
-; CHECK-NEXT:.b8 64
-; CHECK-NEXT:.b8 8                                   // DW_FORM_string
-; CHECK-NEXT:.b8 3                                   // DW_AT_name
-; CHECK-NEXT:.b8 8                                   // DW_FORM_string
-; CHECK-NEXT:.b8 58                                  // DW_AT_decl_file
-; CHECK-NEXT:.b8 11                                  // DW_FORM_data1
-; CHECK-NEXT:.b8 59                                  // DW_AT_decl_line
-; CHECK-NEXT:.b8 11                                  // DW_FORM_data1
-; CHECK-NEXT:.b8 63                                  // DW_AT_external
-; CHECK-NEXT:.b8 12                                  // DW_FORM_flag
-; CHECK-NEXT:.b8 0                                   // EOM(1)
-; CHECK-NEXT:.b8 0                                   // EOM(2)
-; CHECK-NEXT:.b8 24                                  // Abbreviation Code
-; CHECK-NEXT:.b8 5                                   // DW_TAG_formal_parameter
-; CHECK-NEXT:.b8 0                                   // DW_CHILDREN_no
-; CHECK-NEXT:.b8 51                                  // DW_AT_address_class
-; CHECK-NEXT:.b8 11                                  // DW_FORM_data1
-; CHECK-NEXT:.b8 2                                   // DW_AT_location
-; CHECK-NEXT:.b8 10                                  // DW_FORM_block1
-; CHECK-NEXT:.b8 3                                   // DW_AT_name
-; CHECK-NEXT:.b8 8                                   // DW_FORM_string
-; CHECK-NEXT:.b8 58                                  // DW_AT_decl_file
-; CHECK-NEXT:.b8 11                                  // DW_FORM_data1
-; CHECK-NEXT:.b8 59                                  // DW_AT_decl_line
-; CHECK-NEXT:.b8 11                                  // DW_FORM_data1
-; CHECK-NEXT:.b8 73                                  // DW_AT_type
-; CHECK-NEXT:.b8 19                                  // DW_FORM_ref4
-; CHECK-NEXT:.b8 0                                   // EOM(1)
-; CHECK-NEXT:.b8 0                                   // EOM(2)
-; CHECK-NEXT:.b8 25                                  // Abbreviation Code
-; CHECK-NEXT:.b8 5                                   // DW_TAG_formal_parameter
-; CHECK-NEXT:.b8 0                                   // DW_CHILDREN_no
-; CHECK-NEXT:.b8 2                                   // DW_AT_location
-; CHECK-NEXT:.b8 6                                   // DW_FORM_data4
-; CHECK-NEXT:.b8 3                                   // DW_AT_name
-; CHECK-NEXT:.b8 8                                   // DW_FORM_string
-; CHECK-NEXT:.b8 58                                  // DW_AT_decl_file
-; CHECK-NEXT:.b8 11                                  // DW_FORM_data1
-; CHECK-NEXT:.b8 59                                  // DW_AT_decl_line
-; CHECK-NEXT:.b8 11                                  // DW_FORM_data1
-; CHECK-NEXT:.b8 73                                  // DW_AT_type
-; CHECK-NEXT:.b8 19                                  // DW_FORM_ref4
-; CHECK-NEXT:.b8 0                                   // EOM(1)
-; CHECK-NEXT:.b8 0                                   // EOM(2)
-; CHECK-NEXT:.b8 26                                  // Abbreviation Code
-; CHECK-NEXT:.b8 52                                  // DW_TAG_variable
-; CHECK-NEXT:.b8 0                                   // DW_CHILDREN_no
-; CHECK-NEXT:.b8 2                                   // DW_AT_location
-; CHECK-NEXT:.b8 6                                   // DW_FORM_data4
-; CHECK-NEXT:.b8 3                                   // DW_AT_name
-; CHECK-NEXT:.b8 8                                   // DW_FORM_string
-; CHECK-NEXT:.b8 58                                  // DW_AT_decl_file
-; CHECK-NEXT:.b8 11                                  // DW_FORM_data1
-; CHECK-NEXT:.b8 59                                  // DW_AT_decl_line
-; CHECK-NEXT:.b8 11                                  // DW_FORM_data1
-; CHECK-NEXT:.b8 73                                  // DW_AT_type
-; CHECK-NEXT:.b8 19                                  // DW_FORM_ref4
-; CHECK-NEXT:.b8 0                                   // EOM(1)
-; CHECK-NEXT:.b8 0                                   // EOM(2)
-; CHECK-NEXT:.b8 27                                  // Abbreviation Code
-; CHECK-NEXT:.b8 29                                  // DW_TAG_inlined_subroutine
-; CHECK-NEXT:.b8 0                                   // DW_CHILDREN_no
-; CHECK-NEXT:.b8 49                                  // DW_AT_abstract_origin
-; CHECK-NEXT:.b8 19                                  // DW_FORM_ref4
-; CHECK-NEXT:.b8 17                                  // DW_AT_low_pc
-; CHECK-NEXT:.b8 1                                   // DW_FORM_addr
-; CHECK-NEXT:.b8 18                                  // DW_AT_high_pc
-; CHECK-NEXT:.b8 1                                   // DW_FORM_addr
-; CHECK-NEXT:.b8 88                                  // DW_AT_call_file
-; CHECK-NEXT:.b8 11                                  // DW_FORM_data1
-; CHECK-NEXT:.b8 89                                  // DW_AT_call_line
-; CHECK-NEXT:.b8 11                                  // DW_FORM_data1
-; CHECK-NEXT:.b8 87                                  // DW_AT_call_column
-; CHECK-NEXT:.b8 11                                  // DW_FORM_data1
-; CHECK-NEXT:.b8 0                                   // EOM(1)
-; CHECK-NEXT:.b8 0                                   // EOM(2)
-; CHECK-NEXT:.b8 28                                  // Abbreviation Code
-; CHECK-NEXT:.b8 29                                  // DW_TAG_inlined_subroutine
-; CHECK-NEXT:.b8 1                                   // DW_CHILDREN_yes
-; CHECK-NEXT:.b8 49                                  // DW_AT_abstract_origin
-; CHECK-NEXT:.b8 19                                  // DW_FORM_ref4
-; CHECK-NEXT:.b8 17                                  // DW_AT_low_pc
-; CHECK-NEXT:.b8 1                                   // DW_FORM_addr
-; CHECK-NEXT:.b8 18                                  // DW_AT_high_pc
-; CHECK-NEXT:.b8 1                                   // DW_FORM_addr
-; CHECK-NEXT:.b8 88                                  // DW_AT_call_file
-; CHECK-NEXT:.b8 11                                  // DW_FORM_data1
-; CHECK-NEXT:.b8 89                                  // DW_AT_call_line
-; CHECK-NEXT:.b8 11                                  // DW_FORM_data1
-; CHECK-NEXT:.b8 87                                  // DW_AT_call_column
-; CHECK-NEXT:.b8 11                                  // DW_FORM_data1
-; CHECK-NEXT:.b8 0                                   // EOM(1)
-; CHECK-NEXT:.b8 0                                   // EOM(2)
-; CHECK-NEXT:.b8 29                                  // Abbreviation Code
-; CHECK-NEXT:.b8 5                                   // DW_TAG_formal_parameter
-; CHECK-NEXT:.b8 0                                   // DW_CHILDREN_no
-; CHECK-NEXT:.b8 51                                  // DW_AT_address_class
-; CHECK-NEXT:.b8 11                                  // DW_FORM_data1
-; CHECK-NEXT:.b8 2                                   // DW_AT_location
-; CHECK-NEXT:.b8 10                                  // DW_FORM_block1
-; CHECK-NEXT:.b8 49                                  // DW_AT_abstract_origin
-; CHECK-NEXT:.b8 19                                  // DW_FORM_ref4
-; CHECK-NEXT:.b8 0                                   // EOM(1)
-; CHECK-NEXT:.b8 0                                   // EOM(2)
-; CHECK-NEXT:.b8 30                                  // Abbreviation Code
-; CHECK-NEXT:.b8 57                                  // DW_TAG_namespace
-; CHECK-NEXT:.b8 1                                   // DW_CHILDREN_yes
-; CHECK-NEXT:.b8 3                                   // DW_AT_name
-; CHECK-NEXT:.b8 8                                   // DW_FORM_string
-; CHECK-NEXT:.b8 0                                   // EOM(1)
-; CHECK-NEXT:.b8 0                                   // EOM(2)
-; CHECK-NEXT:.b8 31                                  // Abbreviation Code
-; CHECK-NEXT:.b8 8                                   // DW_TAG_imported_declaration
-; CHECK-NEXT:.b8 0                                   // DW_CHILDREN_no
-; CHECK-NEXT:.b8 58                                  // DW_AT_decl_file
-; CHECK-NEXT:.b8 11                                  // DW_FORM_data1
-; CHECK-NEXT:.b8 59                                  // DW_AT_decl_line
-; CHECK-NEXT:.b8 11                                  // DW_FORM_data1
-; CHECK-NEXT:.b8 24                                  // DW_AT_import
-; CHECK-NEXT:.b8 19                                  // DW_FORM_ref4
-; CHECK-NEXT:.b8 0                                   // EOM(1)
-; CHECK-NEXT:.b8 0                                   // EOM(2)
-; CHECK-NEXT:.b8 32                                  // Abbreviation Code
-; CHECK-NEXT:.b8 46                                  // DW_TAG_subprogram
-; CHECK-NEXT:.b8 1                                   // DW_CHILDREN_yes
-; CHECK-NEXT:.b8 135                                 // DW_AT_MIPS_linkage_name
-; CHECK-NEXT:.b8 64
-; CHECK-NEXT:.b8 8                                   // DW_FORM_string
-; CHECK-NEXT:.b8 3                                   // DW_AT_name
-; CHECK-NEXT:.b8 8                                   // DW_FORM_string
-; CHECK-NEXT:.b8 58                                  // DW_AT_decl_file
-; CHECK-NEXT:.b8 11                                  // DW_FORM_data1
-; CHECK-NEXT:.b8 59                                  // DW_AT_decl_line
-; CHECK-NEXT:.b8 11                                  // DW_FORM_data1
-; CHECK-NEXT:.b8 73                                  // DW_AT_type
-; CHECK-NEXT:.b8 19                                  // DW_FORM_ref4
-; CHECK-NEXT:.b8 60                                  // DW_AT_declaration
-; CHECK-NEXT:.b8 12                                  // DW_FORM_flag
-; CHECK-NEXT:.b8 0                                   // EOM(1)
-; CHECK-NEXT:.b8 0                                   // EOM(2)
-; CHECK-NEXT:.b8 0                                   // EOM(3)
-; CHECK-NEXT:	}
-; CHECK-NEXT:	.section	.debug_info
-; CHECK-NEXT:	{
-; CHECK-NEXT:.b32 2460                               // Length of Unit
-; CHECK-NEXT:.b8 2                                   // DWARF version number
-; CHECK-NEXT:.b8 0
-; CHECK-NEXT:.b32 .debug_abbrev                      // Offset Into Abbrev. Section
-; CHECK-NEXT:.b8 8                                   // Address Size (in bytes)
-; CHECK-NEXT:.b8 1                                   // Abbrev [1] 0xb:0x995 DW_TAG_compile_unit
-; CHECK-NEXT:.b8 0                                   // DW_AT_producer
-; CHECK-NEXT:.b8 4                                   // DW_AT_language
-; CHECK-NEXT:.b8 0
-; CHECK-NEXT:.b8 100                                 // DW_AT_name
-; CHECK-NEXT:.b8 101
-; CHECK-NEXT:.b8 98
-; CHECK-NEXT:.b8 117
-; CHECK-NEXT:.b8 103
-; CHECK-NEXT:.b8 45
-; CHECK-NEXT:.b8 105
-; CHECK-NEXT:.b8 110
-; CHECK-NEXT:.b8 102
-; CHECK-NEXT:.b8 111
-; CHECK-NEXT:.b8 46
-; CHECK-NEXT:.b8 99
-; CHECK-NEXT:.b8 117
-; CHECK-NEXT:.b8 0
-; CHECK-NEXT:.b32 .debug_line                        // DW_AT_stmt_list
-; CHECK-NEXT:.b8 47                                  // DW_AT_comp_dir
-; CHECK-NEXT:.b8 115
-; CHECK-NEXT:.b8 111
-; CHECK-NEXT:.b8 109
-; CHECK-NEXT:.b8 101
-; CHECK-NEXT:.b8 47
-; CHECK-NEXT:.b8 100
-; CHECK-NEXT:.b8 105
-; CHECK-NEXT:.b8 114
-; CHECK-NEXT:.b8 101
-; CHECK-NEXT:.b8 99
-; CHECK-NEXT:.b8 116
-; CHECK-NEXT:.b8 111
-; CHECK-NEXT:.b8 114
-; CHECK-NEXT:.b8 121
-; CHECK-NEXT:.b8 0
-; CHECK-NEXT:.b8 2                                   // Abbrev [2] 0x31:0x23e DW_TAG_structure_type
-; CHECK-NEXT:.b8 95                                  // DW_AT_name
-; CHECK-NEXT:.b8 95
-; CHECK-NEXT:.b8 99
-; CHECK-NEXT:.b8 117
-; CHECK-NEXT:.b8 100
-; CHECK-NEXT:.b8 97
-; CHECK-NEXT:.b8 95
-; CHECK-NEXT:.b8 98
-; CHECK-NEXT:.b8 117
-; CHECK-NEXT:.b8 105
-; CHECK-NEXT:.b8 108
-; CHECK-NEXT:.b8 116
-; CHECK-NEXT:.b8 105
-; CHECK-NEXT:.b8 110
-; CHECK-NEXT:.b8 95
-; CHECK-NEXT:.b8 98
-; CHECK-NEXT:.b8 108
-; CHECK-NEXT:.b8 111
-; CHECK-NEXT:.b8 99
-; CHECK-NEXT:.b8 107
-; CHECK-NEXT:.b8 73
-; CHECK-NEXT:.b8 100
-; CHECK-NEXT:.b8 120
-; CHECK-NEXT:.b8 95
-; CHECK-NEXT:.b8 116
-; CHECK-NEXT:.b8 0
-; CHECK-NEXT:.b8 1                                   // DW_AT_byte_size
-; CHECK-NEXT:.b8 2                                   // DW_AT_decl_file
-; CHECK-NEXT:.b8 77                                  // DW_AT_decl_line
-; CHECK-NEXT:.b8 3                                   // Abbrev [3] 0x4f:0x4f DW_TAG_subprogram
-; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
-; CHECK-NEXT:.b8 90
-; CHECK-NEXT:.b8 78
-; CHECK-NEXT:.b8 50
-; CHECK-NEXT:.b8 53
-; CHECK-NEXT:.b8 95
-; CHECK-NEXT:.b8 95
-; CHECK-NEXT:.b8 99
-; CHECK-NEXT:.b8 117
-; CHECK-NEXT:.b8 100
-; CHECK-NEXT:.b8 97
-; CHECK-NEXT:.b8 95
-; CHECK-NEXT:.b8 98
-; CHECK-NEXT:.b8 117
-; CHECK-NEXT:.b8 105
-; CHECK-NEXT:.b8 108
-; CHECK-NEXT:.b8 116
-; CHECK-NEXT:.b8 105
-; CHECK-NEXT:.b8 110
-; CHECK-NEXT:.b8 95
-; CHECK-NEXT:.b8 98
-; CHECK-NEXT:.b8 108
-; CHECK-NEXT:.b8 111
-; CHECK-NEXT:.b8 99
-; CHECK-NEXT:.b8 107
-; CHECK-NEXT:.b8 73
-; CHECK-NEXT:.b8 100
-; CHECK-NEXT:.b8 120
-; CHECK-NEXT:.b8 95
-; CHECK-NEXT:.b8 116
-; CHECK-NEXT:.b8 49
-; CHECK-NEXT:.b8 55
-; CHECK-NEXT:.b8 95
-; CHECK-NEXT:.b8 95
-; CHECK-NEXT:.b8 102
-; CHECK-NEXT:.b8 101
-; CHECK-NEXT:.b8 116
-; CHECK-NEXT:.b8 99
-; CHECK-NEXT:.b8 104
-; CHECK-NEXT:.b8 95
-; CHECK-NEXT:.b8 98
-; CHECK-NEXT:.b8 117
-; CHECK-NEXT:.b8 105
-; CHECK-NEXT:.b8 108
-; CHECK-NEXT:.b8 116
-; CHECK-NEXT:.b8 105
-; CHECK-NEXT:.b8 110
-; CHECK-NEXT:.b8 95
-; CHECK-NEXT:.b8 120
-; CHECK-NEXT:.b8 69
-; CHECK-NEXT:.b8 118
-; CHECK-NEXT:.b8 0
-; CHECK-NEXT:.b8 95                                  // DW_AT_name
-; CHECK-NEXT:.b8 95
-; CHECK-NEXT:.b8 102
-; CHECK-NEXT:.b8 101
-; CHECK-NEXT:.b8 116
-; CHECK-NEXT:.b8 99
-; CHECK-NEXT:.b8 104
-; CHECK-NEXT:.b8 95
-; CHECK-NEXT:.b8 98
-; CHECK-NEXT:.b8 117
-; CHECK-NEXT:.b8 105
-; CHECK-NEXT:.b8 108
-; CHECK-NEXT:.b8 116
-; CHECK-NEXT:.b8 105
-; CHECK-NEXT:.b8 110
-; CHECK-NEXT:.b8 95
-; CHECK-NEXT:.b8 120
-; CHECK-NEXT:.b8 0
-; CHECK-NEXT:.b8 2                                   // DW_AT_decl_file
-; CHECK-NEXT:.b8 78                                  // DW_AT_decl_line
-; CHECK-NEXT:.b32 623                                // DW_AT_type
-; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 1                                   // DW_AT_external
-; CHECK-NEXT:.b8 3                                   // Abbrev [3] 0x9e:0x4f DW_TAG_subprogram
-; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
-; CHECK-NEXT:.b8 90
-; CHECK-NEXT:.b8 78
-; CHECK-NEXT:.b8 50
-; CHECK-NEXT:.b8 53
-; CHECK-NEXT:.b8 95
-; CHECK-NEXT:.b8 95
-; CHECK-NEXT:.b8 99
-; CHECK-NEXT:.b8 117
-; CHECK-NEXT:.b8 100
-; CHECK-NEXT:.b8 97
-; CHECK-NEXT:.b8 95
-; CHECK-NEXT:.b8 98
-; CHECK-NEXT:.b8 117
-; CHECK-NEXT:.b8 105
-; CHECK-NEXT:.b8 108
-; CHECK-NEXT:.b8 116
-; CHECK-NEXT:.b8 105
-; CHECK-NEXT:.b8 110
-; CHECK-NEXT:.b8 95
-; CHECK-NEXT:.b8 98
-; CHECK-NEXT:.b8 108
-; CHECK-NEXT:.b8 111
-; CHECK-NEXT:.b8 99
-; CHECK-NEXT:.b8 107
-; CHECK-NEXT:.b8 73
-; CHECK-NEXT:.b8 100
-; CHECK-NEXT:.b8 120
-; CHECK-NEXT:.b8 95
-; CHECK-NEXT:.b8 116
-; CHECK-NEXT:.b8 49
-; CHECK-NEXT:.b8 55
-; CHECK-NEXT:.b8 95
-; CHECK-NEXT:.b8 95
-; CHECK-NEXT:.b8 102
-; CHECK-NEXT:.b8 101
-; CHECK-NEXT:.b8 116
-; CHECK-NEXT:.b8 99
-; CHECK-NEXT:.b8 104
-; CHECK-NEXT:.b8 95
-; CHECK-NEXT:.b8 98
-; CHECK-NEXT:.b8 117
-; CHECK-NEXT:.b8 105
-; CHECK-NEXT:.b8 108
-; CHECK-NEXT:.b8 116
-; CHECK-NEXT:.b8 105
-; CHECK-NEXT:.b8 110
-; CHECK-NEXT:.b8 95
-; CHECK-NEXT:.b8 121
-; CHECK-NEXT:.b8 69
-; CHECK-NEXT:.b8 118
-; CHECK-NEXT:.b8 0
-; CHECK-NEXT:.b8 95                                  // DW_AT_name
-; CHECK-NEXT:.b8 95
-; CHECK-NEXT:.b8 102
-; CHECK-NEXT:.b8 101
-; CHECK-NEXT:.b8 116
-; CHECK-NEXT:.b8 99
-; CHECK-NEXT:.b8 104
-; CHECK-NEXT:.b8 95
-; CHECK-NEXT:.b8 98
-; CHECK-NEXT:.b8 117
-; CHECK-NEXT:.b8 105
-; CHECK-NEXT:.b8 108
-; CHECK-NEXT:.b8 116
-; CHECK-NEXT:.b8 105
-; CHECK-NEXT:.b8 110
-; CHECK-NEXT:.b8 95
-; CHECK-NEXT:.b8 121
-; CHECK-NEXT:.b8 0
-; CHECK-NEXT:.b8 2                                   // DW_AT_decl_file
-; CHECK-NEXT:.b8 79                                  // DW_AT_decl_line
-; CHECK-NEXT:.b32 623                                // DW_AT_type
-; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 1                                   // DW_AT_external
-; CHECK-NEXT:.b8 3                                   // Abbrev [3] 0xed:0x4f DW_TAG_subprogram
-; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
-; CHECK-NEXT:.b8 90
-; CHECK-NEXT:.b8 78
-; CHECK-NEXT:.b8 50
-; CHECK-NEXT:.b8 53
-; CHECK-NEXT:.b8 95
-; CHECK-NEXT:.b8 95
-; CHECK-NEXT:.b8 99
-; CHECK-NEXT:.b8 117
-; CHECK-NEXT:.b8 100
-; CHECK-NEXT:.b8 97
-; CHECK-NEXT:.b8 95
-; CHECK-NEXT:.b8 98
-; CHECK-NEXT:.b8 117
-; CHECK-NEXT:.b8 105
-; CHECK-NEXT:.b8 108
-; CHECK-NEXT:.b8 116
-; CHECK-NEXT:.b8 105
-; CHECK-NEXT:.b8 110
-; CHECK-NEXT:.b8 95
-; CHECK-NEXT:.b8 98
-; CHECK-NEXT:.b8 108
-; CHECK-NEXT:.b8 111
-; CHECK-NEXT:.b8 99
-; CHECK-NEXT:.b8 107
-; CHECK-NEXT:.b8 73
-; CHECK-NEXT:.b8 100
-; CHECK-NEXT:.b8 120
-; CHECK-NEXT:.b8 95
-; CHECK-NEXT:.b8 116
-; CHECK-NEXT:.b8 49
-; CHECK-NEXT:.b8 55
-; CHECK-NEXT:.b8 95
-; CHECK-NEXT:.b8 95
-; CHECK-NEXT:.b8 102
-; CHECK-NEXT:.b8 101
-; CHECK-NEXT:.b8 116
-; CHECK-NEXT:.b8 99
-; CHECK-NEXT:.b8 104
-; CHECK-NEXT:.b8 95
-; CHECK-NEXT:.b8 98
-; CHECK-NEXT:.b8 117
-; CHECK-NEXT:.b8 105
-; CHECK-NEXT:.b8 108
-; CHECK-NEXT:.b8 116
-; CHECK-NEXT:.b8 105
-; CHECK-NEXT:.b8 110
-; CHECK-NEXT:.b8 95
-; CHECK-NEXT:.b8 122
-; CHECK-NEXT:.b8 69
-; CHECK-NEXT:.b8 118
-; CHECK-NEXT:.b8 0
-; CHECK-NEXT:.b8 95                                  // DW_AT_name
-; CHECK-NEXT:.b8 95
-; CHECK-NEXT:.b8 102
-; CHECK-NEXT:.b8 101
-; CHECK-NEXT:.b8 116
-; CHECK-NEXT:.b8 99
-; CHECK-NEXT:.b8 104
-; CHECK-NEXT:.b8 95
-; CHECK-NEXT:.b8 98
-; CHECK-NEXT:.b8 117
-; CHECK-NEXT:.b8 105
-; CHECK-NEXT:.b8 108
-; CHECK-NEXT:.b8 116
-; CHECK-NEXT:.b8 105
-; CHECK-NEXT:.b8 110
-; CHECK-NEXT:.b8 95
-; CHECK-NEXT:.b8 122
-; CHECK-NEXT:.b8 0
-; CHECK-NEXT:.b8 2                                   // DW_AT_decl_file
-; CHECK-NEXT:.b8 80                                  // DW_AT_decl_line
-; CHECK-NEXT:.b32 623                                // DW_AT_type
-; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 1                                   // DW_AT_external
-; CHECK-NEXT:.b8 4                                   // Abbrev [4] 0x13c:0x4d DW_TAG_subprogram
-; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
-; CHECK-NEXT:.b8 90
-; CHECK-NEXT:.b8 78
-; CHECK-NEXT:.b8 75
-; CHECK-NEXT:.b8 50
-; CHECK-NEXT:.b8 53
-; CHECK-NEXT:.b8 95
-; CHECK-NEXT:.b8 95
-; CHECK-NEXT:.b8 99
-; CHECK-NEXT:.b8 117
-; CHECK-NEXT:.b8 100
-; CHECK-NEXT:.b8 97
-; CHECK-NEXT:.b8 95
-; CHECK-NEXT:.b8 98
-; CHECK-NEXT:.b8 117
-; CHECK-NEXT:.b8 105
-; CHECK-NEXT:.b8 108
-; CHECK-NEXT:.b8 116
-; CHECK-NEXT:.b8 105
-; CHECK-NEXT:.b8 110
-; CHECK-NEXT:.b8 95
-; CHECK-NEXT:.b8 98
-; CHECK-NEXT:.b8 108
-; CHECK-NEXT:.b8 111
-; CHECK-NEXT:.b8 99
-; CHECK-NEXT:.b8 107
-; CHECK-NEXT:.b8 73
-; CHECK-NEXT:.b8 100
-; CHECK-NEXT:.b8 120
-; CHECK-NEXT:.b8 95
-; CHECK-NEXT:.b8 116
-; CHECK-NEXT:.b8 99
-; CHECK-NEXT:.b8 118
-; CHECK-NEXT:.b8 53
-; CHECK-NEXT:.b8 117
-; CHECK-NEXT:.b8 105
-; CHECK-NEXT:.b8 110
-; CHECK-NEXT:.b8 116
-; CHECK-NEXT:.b8 51
-; CHECK-NEXT:.b8 69
-; CHECK-NEXT:.b8 118
-; CHECK-NEXT:.b8 0
-; CHECK-NEXT:.b8 111                                 // DW_AT_name
-; CHECK-NEXT:.b8 112
-; CHECK-NEXT:.b8 101
-; CHECK-NEXT:.b8 114
-; CHECK-NEXT:.b8 97
-; CHECK-NEXT:.b8 116
-; CHECK-NEXT:.b8 111
-; CHECK-NEXT:.b8 114
-; CHECK-NEXT:.b8 32
-; CHECK-NEXT:.b8 117
-; CHECK-NEXT:.b8 105
-; CHECK-NEXT:.b8 110
-; CHECK-NEXT:.b8 116
-; CHECK-NEXT:.b8 51
-; CHECK-NEXT:.b8 0
-; CHECK-NEXT:.b8 2                                   // DW_AT_decl_file
-; CHECK-NEXT:.b8 83                                  // DW_AT_decl_line
-; CHECK-NEXT:.b32 639                                // DW_AT_type
-; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b32 386                                // DW_AT_object_pointer
-; CHECK-NEXT:.b8 1                                   // DW_AT_external
-; CHECK-NEXT:.b8 5                                   // Abbrev [5] 0x182:0x6 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 686                                // DW_AT_type
-; CHECK-NEXT:.b8 1                                   // DW_AT_artificial
-; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 6                                   // Abbrev [6] 0x189:0x2b DW_TAG_subprogram
-; CHECK-NEXT:.b8 95                                  // DW_AT_name
-; CHECK-NEXT:.b8 95
-; CHECK-NEXT:.b8 99
-; CHECK-NEXT:.b8 117
-; CHECK-NEXT:.b8 100
-; CHECK-NEXT:.b8 97
-; CHECK-NEXT:.b8 95
-; CHECK-NEXT:.b8 98
-; CHECK-NEXT:.b8 117
-; CHECK-NEXT:.b8 105
-; CHECK-NEXT:.b8 108
-; CHECK-NEXT:.b8 116
-; CHECK-NEXT:.b8 105
-; CHECK-NEXT:.b8 110
-; CHECK-NEXT:.b8 95
-; CHECK-NEXT:.b8 98
-; CHECK-NEXT:.b8 108
-; CHECK-NEXT:.b8 111
-; CHECK-NEXT:.b8 99
-; CHECK-NEXT:.b8 107
-; CHECK-NEXT:.b8 73
-; CHECK-NEXT:.b8 100
-; CHECK-NEXT:.b8 120
-; CHECK-NEXT:.b8 95
-; CHECK-NEXT:.b8 116
-; CHECK-NEXT:.b8 0
-; CHECK-NEXT:.b8 2                                   // DW_AT_decl_file
-; CHECK-NEXT:.b8 85                                  // DW_AT_decl_line
-; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b32 429                                // DW_AT_object_pointer
-; CHECK-NEXT:.b8 1                                   // DW_AT_external
-; CHECK-NEXT:.b8 3                                   // DW_AT_accessibility
-; CHECK-NEXT:                                        // DW_ACCESS_private
-; CHECK-NEXT:.b8 5                                   // Abbrev [5] 0x1ad:0x6 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 696                                // DW_AT_type
-; CHECK-NEXT:.b8 1                                   // DW_AT_artificial
-; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 6                                   // Abbrev [6] 0x1b4:0x30 DW_TAG_subprogram
-; CHECK-NEXT:.b8 95                                  // DW_AT_name
-; CHECK-NEXT:.b8 95
-; CHECK-NEXT:.b8 99
-; CHECK-NEXT:.b8 117
-; CHECK-NEXT:.b8 100
-; CHECK-NEXT:.b8 97
-; CHECK-NEXT:.b8 95
-; CHECK-NEXT:.b8 98
-; CHECK-NEXT:.b8 117
-; CHECK-NEXT:.b8 105
-; CHECK-NEXT:.b8 108
-; CHECK-NEXT:.b8 116
-; CHECK-NEXT:.b8 105
-; CHECK-NEXT:.b8 110
-; CHECK-NEXT:.b8 95
-; CHECK-NEXT:.b8 98
-; CHECK-NEXT:.b8 108
-; CHECK-NEXT:.b8 111
-; CHECK-NEXT:.b8 99
-; CHECK-NEXT:.b8 107
-; CHECK-NEXT:.b8 73
-; CHECK-NEXT:.b8 100
-; CHECK-NEXT:.b8 120
-; CHECK-NEXT:.b8 95
-; CHECK-NEXT:.b8 116
-; CHECK-NEXT:.b8 0
-; CHECK-NEXT:.b8 2                                   // DW_AT_decl_file
-; CHECK-NEXT:.b8 85                                  // DW_AT_decl_line
-; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b32 472                                // DW_AT_object_pointer
-; CHECK-NEXT:.b8 1                                   // DW_AT_external
-; CHECK-NEXT:.b8 3                                   // DW_AT_accessibility
-; CHECK-NEXT:                                        // DW_ACCESS_private
-; CHECK-NEXT:.b8 5                                   // Abbrev [5] 0x1d8:0x6 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 696                                // DW_AT_type
-; CHECK-NEXT:.b8 1                                   // DW_AT_artificial
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1de:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 701                                // DW_AT_type
-; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 8                                   // Abbrev [8] 0x1e4:0x47 DW_TAG_subprogram
-; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
-; CHECK-NEXT:.b8 90
-; CHECK-NEXT:.b8 78
-; CHECK-NEXT:.b8 75
-; CHECK-NEXT:.b8 50
-; CHECK-NEXT:.b8 53
-; CHECK-NEXT:.b8 95
-; CHECK-NEXT:.b8 95
-; CHECK-NEXT:.b8 99
-; CHECK-NEXT:.b8 117
-; CHECK-NEXT:.b8 100
-; CHECK-NEXT:.b8 97
-; CHECK-NEXT:.b8 95
-; CHECK-NEXT:.b8 98
-; CHECK-NEXT:.b8 117
-; CHECK-NEXT:.b8 105
-; CHECK-NEXT:.b8 108
-; CHECK-NEXT:.b8 116
-; CHECK-NEXT:.b8 105
-; CHECK-NEXT:.b8 110
-; CHECK-NEXT:.b8 95
-; CHECK-NEXT:.b8 98
-; CHECK-NEXT:.b8 108
-; CHECK-NEXT:.b8 111
-; CHECK-NEXT:.b8 99
-; CHECK-NEXT:.b8 107
-; CHECK-NEXT:.b8 73
-; CHECK-NEXT:.b8 100
-; CHECK-NEXT:.b8 120
-; CHECK-NEXT:.b8 95
-; CHECK-NEXT:.b8 116
-; CHECK-NEXT:.b8 97
-; CHECK-NEXT:.b8 83
-; CHECK-NEXT:.b8 69
-; CHECK-NEXT:.b8 82
-; CHECK-NEXT:.b8 75
-; CHECK-NEXT:.b8 83
-; CHECK-NEXT:.b8 95
-; CHECK-NEXT:.b8 0
-; CHECK-NEXT:.b8 111                                 // DW_AT_name
-; CHECK-NEXT:.b8 112
-; CHECK-NEXT:.b8 101
-; CHECK-NEXT:.b8 114
-; CHECK-NEXT:.b8 97
-; CHECK-NEXT:.b8 116
-; CHECK-NEXT:.b8 111
-; CHECK-NEXT:.b8 114
-; CHECK-NEXT:.b8 61
-; CHECK-NEXT:.b8 0
-; CHECK-NEXT:.b8 2                                   // DW_AT_decl_file
-; CHECK-NEXT:.b8 85                                  // DW_AT_decl_line
-; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b32 543                                // DW_AT_object_pointer
-; CHECK-NEXT:.b8 1                                   // DW_AT_external
-; CHECK-NEXT:.b8 3                                   // DW_AT_accessibility
-; CHECK-NEXT:                                        // DW_ACCESS_private
-; CHECK-NEXT:.b8 5                                   // Abbrev [5] 0x21f:0x6 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 686                                // DW_AT_type
-; CHECK-NEXT:.b8 1                                   // DW_AT_artificial
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x225:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 701                                // DW_AT_type
-; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 9                                   // Abbrev [9] 0x22b:0x43 DW_TAG_subprogram
-; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
-; CHECK-NEXT:.b8 90
-; CHECK-NEXT:.b8 78
-; CHECK-NEXT:.b8 75
-; CHECK-NEXT:.b8 50
-; CHECK-NEXT:.b8 53
-; CHECK-NEXT:.b8 95
-; CHECK-NEXT:.b8 95
-; CHECK-NEXT:.b8 99
-; CHECK-NEXT:.b8 117
-; CHECK-NEXT:.b8 100
-; CHECK-NEXT:.b8 97
-; CHECK-NEXT:.b8 95
-; CHECK-NEXT:.b8 98
-; CHECK-NEXT:.b8 117
-; CHECK-NEXT:.b8 105
-; CHECK-NEXT:.b8 108
-; CHECK-NEXT:.b8 116
-; CHECK-NEXT:.b8 105
-; CHECK-NEXT:.b8 110
-; CHECK-NEXT:.b8 95
-; CHECK-NEXT:.b8 98
-; CHECK-NEXT:.b8 108
-; CHECK-NEXT:.b8 111
-; CHECK-NEXT:.b8 99
-; CHECK-NEXT:.b8 107
-; CHECK-NEXT:.b8 73
-; CHECK-NEXT:.b8 100
-; CHECK-NEXT:.b8 120
-; CHECK-NEXT:.b8 95
-; CHECK-NEXT:.b8 116
-; CHECK-NEXT:.b8 97
-; CHECK-NEXT:.b8 100
-; CHECK-NEXT:.b8 69
-; CHECK-NEXT:.b8 118
-; CHECK-NEXT:.b8 0
-; CHECK-NEXT:.b8 111                                 // DW_AT_name
-; CHECK-NEXT:.b8 112
-; CHECK-NEXT:.b8 101
-; CHECK-NEXT:.b8 114
-; CHECK-NEXT:.b8 97
-; CHECK-NEXT:.b8 116
-; CHECK-NEXT:.b8 111
-; CHECK-NEXT:.b8 114
-; CHECK-NEXT:.b8 38
-; CHECK-NEXT:.b8 0
-; CHECK-NEXT:.b8 2                                   // DW_AT_decl_file
-; CHECK-NEXT:.b8 85                                  // DW_AT_decl_line
-; CHECK-NEXT:.b32 706                                // DW_AT_type
-; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b32 615                                // DW_AT_object_pointer
-; CHECK-NEXT:.b8 1                                   // DW_AT_external
-; CHECK-NEXT:.b8 3                                   // DW_AT_accessibility
-; CHECK-NEXT:                                        // DW_ACCESS_private
-; CHECK-NEXT:.b8 5                                   // Abbrev [5] 0x267:0x6 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 686                                // DW_AT_type
-; CHECK-NEXT:.b8 1                                   // DW_AT_artificial
-; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 10                                  // Abbrev [10] 0x26f:0x10 DW_TAG_base_type
-; CHECK-NEXT:.b8 117                                 // DW_AT_name
-; CHECK-NEXT:.b8 110
-; CHECK-NEXT:.b8 115
-; CHECK-NEXT:.b8 105
-; CHECK-NEXT:.b8 103
-; CHECK-NEXT:.b8 110
-; CHECK-NEXT:.b8 101
-; CHECK-NEXT:.b8 100
-; CHECK-NEXT:.b8 32
-; CHECK-NEXT:.b8 105
-; CHECK-NEXT:.b8 110
-; CHECK-NEXT:.b8 116
-; CHECK-NEXT:.b8 0
-; CHECK-NEXT:.b8 7                                   // DW_AT_encoding
-; CHECK-NEXT:.b8 4                                   // DW_AT_byte_size
-; CHECK-NEXT:.b8 2                                   // Abbrev [2] 0x27f:0x2f DW_TAG_structure_type
-; CHECK-NEXT:.b8 117                                 // DW_AT_name
-; CHECK-NEXT:.b8 105
-; CHECK-NEXT:.b8 110
-; CHECK-NEXT:.b8 116
-; CHECK-NEXT:.b8 51
-; CHECK-NEXT:.b8 0
-; CHECK-NEXT:.b8 12                                  // DW_AT_byte_size
-; CHECK-NEXT:.b8 3                                   // DW_AT_decl_file
-; CHECK-NEXT:.b8 190                                 // DW_AT_decl_line
-; CHECK-NEXT:.b8 11                                  // Abbrev [11] 0x289:0xc DW_TAG_member
-; CHECK-NEXT:.b8 120                                 // DW_AT_name
-; CHECK-NEXT:.b8 0
-; CHECK-NEXT:.b32 623                                // DW_AT_type
-; CHECK-NEXT:.b8 3                                   // DW_AT_decl_file
-; CHECK-NEXT:.b8 192                                 // DW_AT_decl_line
-; CHECK-NEXT:.b8 2                                   // DW_AT_data_member_location
-; CHECK-NEXT:.b8 35
-; CHECK-NEXT:.b8 0
-; CHECK-NEXT:.b8 11                                  // Abbrev [11] 0x295:0xc DW_TAG_member
-; CHECK-NEXT:.b8 121                                 // DW_AT_name
-; CHECK-NEXT:.b8 0
-; CHECK-NEXT:.b32 623                                // DW_AT_type
-; CHECK-NEXT:.b8 3                                   // DW_AT_decl_file
-; CHECK-NEXT:.b8 192                                 // DW_AT_decl_line
-; CHECK-NEXT:.b8 2                                   // DW_AT_data_member_location
-; CHECK-NEXT:.b8 35
-; CHECK-NEXT:.b8 4
-; CHECK-NEXT:.b8 11                                  // Abbrev [11] 0x2a1:0xc DW_TAG_member
-; CHECK-NEXT:.b8 122                                 // DW_AT_name
-; CHECK-NEXT:.b8 0
-; CHECK-NEXT:.b32 623                                // DW_AT_type
-; CHECK-NEXT:.b8 3                                   // DW_AT_decl_file
-; CHECK-NEXT:.b8 192                                 // DW_AT_decl_line
-; CHECK-NEXT:.b8 2                                   // DW_AT_data_member_location
-; CHECK-NEXT:.b8 35
-; CHECK-NEXT:.b8 8
-; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 12                                  // Abbrev [12] 0x2ae:0x5 DW_TAG_pointer_type
-; CHECK-NEXT:.b32 691                                // DW_AT_type
-; CHECK-NEXT:.b8 13                                  // Abbrev [13] 0x2b3:0x5 DW_TAG_const_type
-; CHECK-NEXT:.b32 49                                 // DW_AT_type
-; CHECK-NEXT:.b8 12                                  // Abbrev [12] 0x2b8:0x5 DW_TAG_pointer_type
-; CHECK-NEXT:.b32 49                                 // DW_AT_type
-; CHECK-NEXT:.b8 14                                  // Abbrev [14] 0x2bd:0x5 DW_TAG_reference_type
-; CHECK-NEXT:.b32 691                                // DW_AT_type
-; CHECK-NEXT:.b8 12                                  // Abbrev [12] 0x2c2:0x5 DW_TAG_pointer_type
-; CHECK-NEXT:.b32 49                                 // DW_AT_type
-; CHECK-NEXT:.b8 15                                  // Abbrev [15] 0x2c7:0x6 DW_TAG_subprogram
-; CHECK-NEXT:.b32 79                                 // DW_AT_specification
-; CHECK-NEXT:.b8 1                                   // DW_AT_inline
-; CHECK-NEXT:.b8 2                                   // Abbrev [2] 0x2cd:0x23c DW_TAG_structure_type
-; CHECK-NEXT:.b8 95                                  // DW_AT_name
-; CHECK-NEXT:.b8 95
-; CHECK-NEXT:.b8 99
-; CHECK-NEXT:.b8 117
-; CHECK-NEXT:.b8 100
-; CHECK-NEXT:.b8 97
-; CHECK-NEXT:.b8 95
-; CHECK-NEXT:.b8 98
-; CHECK-NEXT:.b8 117
-; CHECK-NEXT:.b8 105
-; CHECK-NEXT:.b8 108
-; CHECK-NEXT:.b8 116
-; CHECK-NEXT:.b8 105
-; CHECK-NEXT:.b8 110
-; CHECK-NEXT:.b8 95
-; CHECK-NEXT:.b8 98
-; CHECK-NEXT:.b8 108
-; CHECK-NEXT:.b8 111
-; CHECK-NEXT:.b8 99
-; CHECK-NEXT:.b8 107
-; CHECK-NEXT:.b8 68
-; CHECK-NEXT:.b8 105
-; CHECK-NEXT:.b8 109
-; CHECK-NEXT:.b8 95
-; CHECK-NEXT:.b8 116
-; CHECK-NEXT:.b8 0
-; CHECK-NEXT:.b8 1                                   // DW_AT_byte_size
-; CHECK-NEXT:.b8 2                                   // DW_AT_decl_file
-; CHECK-NEXT:.b8 88                                  // DW_AT_decl_line
-; CHECK-NEXT:.b8 3                                   // Abbrev [3] 0x2eb:0x4f DW_TAG_subprogram
-; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
-; CHECK-NEXT:.b8 90
-; CHECK-NEXT:.b8 78
-; CHECK-NEXT:.b8 50
-; CHECK-NEXT:.b8 53
-; CHECK-NEXT:.b8 95
-; CHECK-NEXT:.b8 95
-; CHECK-NEXT:.b8 99
-; CHECK-NEXT:.b8 117
-; CHECK-NEXT:.b8 100
-; CHECK-NEXT:.b8 97
-; CHECK-NEXT:.b8 95
-; CHECK-NEXT:.b8 98
-; CHECK-NEXT:.b8 117
-; CHECK-NEXT:.b8 105
-; CHECK-NEXT:.b8 108
-; CHECK-NEXT:.b8 116
-; CHECK-NEXT:.b8 105
-; CHECK-NEXT:.b8 110
-; CHECK-NEXT:.b8 95
-; CHECK-NEXT:.b8 98
-; CHECK-NEXT:.b8 108
-; CHECK-NEXT:.b8 111
-; CHECK-NEXT:.b8 99
-; CHECK-NEXT:.b8 107
-; CHECK-NEXT:.b8 68
-; CHECK-NEXT:.b8 105
-; CHECK-NEXT:.b8 109
-; CHECK-NEXT:.b8 95
-; CHECK-NEXT:.b8 116
-; CHECK-NEXT:.b8 49
-; CHECK-NEXT:.b8 55
-; CHECK-NEXT:.b8 95
-; CHECK-NEXT:.b8 95
-; CHECK-NEXT:.b8 102
-; CHECK-NEXT:.b8 101
-; CHECK-NEXT:.b8 116
-; CHECK-NEXT:.b8 99
-; CHECK-NEXT:.b8 104
-; CHECK-NEXT:.b8 95
-; CHECK-NEXT:.b8 98
-; CHECK-NEXT:.b8 117
-; CHECK-NEXT:.b8 105
-; CHECK-NEXT:.b8 108
-; CHECK-NEXT:.b8 116
-; CHECK-NEXT:.b8 105
-; CHECK-NEXT:.b8 110
-; CHECK-NEXT:.b8 95
-; CHECK-NEXT:.b8 120
-; CHECK-NEXT:.b8 69
-; CHECK-NEXT:.b8 118
-; CHECK-NEXT:.b8 0
-; CHECK-NEXT:.b8 95                                  // DW_AT_name
-; CHECK-NEXT:.b8 95
-; CHECK-NEXT:.b8 102
-; CHECK-NEXT:.b8 101
-; CHECK-NEXT:.b8 116
-; CHECK-NEXT:.b8 99
-; CHECK-NEXT:.b8 104
-; CHECK-NEXT:.b8 95
-; CHECK-NEXT:.b8 98
-; CHECK-NEXT:.b8 117
-; CHECK-NEXT:.b8 105
-; CHECK-NEXT:.b8 108
-; CHECK-NEXT:.b8 116
-; CHECK-NEXT:.b8 105
-; CHECK-NEXT:.b8 110
-; CHECK-NEXT:.b8 95
-; CHECK-NEXT:.b8 120
-; CHECK-NEXT:.b8 0
-; CHECK-NEXT:.b8 2                                   // DW_AT_decl_file
-; CHECK-NEXT:.b8 89                                  // DW_AT_decl_line
-; CHECK-NEXT:.b32 623                                // DW_AT_type
-; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 1                                   // DW_AT_external
-; CHECK-NEXT:.b8 3                                   // Abbrev [3] 0x33a:0x4f DW_TAG_subprogram
-; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
-; CHECK-NEXT:.b8 90
-; CHECK-NEXT:.b8 78
-; CHECK-NEXT:.b8 50
-; CHECK-NEXT:.b8 53
-; CHECK-NEXT:.b8 95
-; CHECK-NEXT:.b8 95
-; CHECK-NEXT:.b8 99
-; CHECK-NEXT:.b8 117
-; CHECK-NEXT:.b8 100
-; CHECK-NEXT:.b8 97
-; CHECK-NEXT:.b8 95
-; CHECK-NEXT:.b8 98
-; CHECK-NEXT:.b8 117
-; CHECK-NEXT:.b8 105
-; CHECK-NEXT:.b8 108
-; CHECK-NEXT:.b8 116
-; CHECK-NEXT:.b8 105
-; CHECK-NEXT:.b8 110
-; CHECK-NEXT:.b8 95
-; CHECK-NEXT:.b8 98
-; CHECK-NEXT:.b8 108
-; CHECK-NEXT:.b8 111
-; CHECK-NEXT:.b8 99
-; CHECK-NEXT:.b8 107
-; CHECK-NEXT:.b8 68
-; CHECK-NEXT:.b8 105
-; CHECK-NEXT:.b8 109
-; CHECK-NEXT:.b8 95
-; CHECK-NEXT:.b8 116
-; CHECK-NEXT:.b8 49
-; CHECK-NEXT:.b8 55
-; CHECK-NEXT:.b8 95
-; CHECK-NEXT:.b8 95
-; CHECK-NEXT:.b8 102
-; CHECK-NEXT:.b8 101
-; CHECK-NEXT:.b8 116
-; CHECK-NEXT:.b8 99
-; CHECK-NEXT:.b8 104
-; CHECK-NEXT:.b8 95
-; CHECK-NEXT:.b8 98
-; CHECK-NEXT:.b8 117
-; CHECK-NEXT:.b8 105
-; CHECK-NEXT:.b8 108
-; CHECK-NEXT:.b8 116
-; CHECK-NEXT:.b8 105
-; CHECK-NEXT:.b8 110
-; CHECK-NEXT:.b8 95
-; CHECK-NEXT:.b8 121
-; CHECK-NEXT:.b8 69
-; CHECK-NEXT:.b8 118
-; CHECK-NEXT:.b8 0
-; CHECK-NEXT:.b8 95                                  // DW_AT_name
-; CHECK-NEXT:.b8 95
-; CHECK-NEXT:.b8 102
-; CHECK-NEXT:.b8 101
-; CHECK-NEXT:.b8 116
-; CHECK-NEXT:.b8 99
-; CHECK-NEXT:.b8 104
-; CHECK-NEXT:.b8 95
-; CHECK-NEXT:.b8 98
-; CHECK-NEXT:.b8 117
-; CHECK-NEXT:.b8 105
-; CHECK-NEXT:.b8 108
-; CHECK-NEXT:.b8 116
-; CHECK-NEXT:.b8 105
-; CHECK-NEXT:.b8 110
-; CHECK-NEXT:.b8 95
-; CHECK-NEXT:.b8 121
-; CHECK-NEXT:.b8 0
-; CHECK-NEXT:.b8 2                                   // DW_AT_decl_file
-; CHECK-NEXT:.b8 90                                  // DW_AT_decl_line
-; CHECK-NEXT:.b32 623                                // DW_AT_type
-; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 1                                   // DW_AT_external
-; CHECK-NEXT:.b8 3                                   // Abbrev [3] 0x389:0x4f DW_TAG_subprogram
-; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
-; CHECK-NEXT:.b8 90
-; CHECK-NEXT:.b8 78
-; CHECK-NEXT:.b8 50
-; CHECK-NEXT:.b8 53
-; CHECK-NEXT:.b8 95
-; CHECK-NEXT:.b8 95
-; CHECK-NEXT:.b8 99
-; CHECK-NEXT:.b8 117
-; CHECK-NEXT:.b8 100
-; CHECK-NEXT:.b8 97
-; CHECK-NEXT:.b8 95
-; CHECK-NEXT:.b8 98
-; CHECK-NEXT:.b8 117
-; CHECK-NEXT:.b8 105
-; CHECK-NEXT:.b8 108
-; CHECK-NEXT:.b8 116
-; CHECK-NEXT:.b8 105
-; CHECK-NEXT:.b8 110
-; CHECK-NEXT:.b8 95
-; CHECK-NEXT:.b8 98
-; CHECK-NEXT:.b8 108
-; CHECK-NEXT:.b8 111
-; CHECK-NEXT:.b8 99
-; CHECK-NEXT:.b8 107
-; CHECK-NEXT:.b8 68
-; CHECK-NEXT:.b8 105
-; CHECK-NEXT:.b8 109
-; CHECK-NEXT:.b8 95
-; CHECK-NEXT:.b8 116
-; CHECK-NEXT:.b8 49
-; CHECK-NEXT:.b8 55
-; CHECK-NEXT:.b8 95
-; CHECK-NEXT:.b8 95
-; CHECK-NEXT:.b8 102
-; CHECK-NEXT:.b8 101
-; CHECK-NEXT:.b8 116
-; CHECK-NEXT:.b8 99
-; CHECK-NEXT:.b8 104
-; CHECK-NEXT:.b8 95
-; CHECK-NEXT:.b8 98
-; CHECK-NEXT:.b8 117
-; CHECK-NEXT:.b8 105
-; CHECK-NEXT:.b8 108
-; CHECK-NEXT:.b8 116
-; CHECK-NEXT:.b8 105
-; CHECK-NEXT:.b8 110
-; CHECK-NEXT:.b8 95
-; CHECK-NEXT:.b8 122
-; CHECK-NEXT:.b8 69
-; CHECK-NEXT:.b8 118
-; CHECK-NEXT:.b8 0
-; CHECK-NEXT:.b8 95                                  // DW_AT_name
-; CHECK-NEXT:.b8 95
-; CHECK-NEXT:.b8 102
-; CHECK-NEXT:.b8 101
-; CHECK-NEXT:.b8 116
-; CHECK-NEXT:.b8 99
-; CHECK-NEXT:.b8 104
-; CHECK-NEXT:.b8 95
-; CHECK-NEXT:.b8 98
-; CHECK-NEXT:.b8 117
-; CHECK-NEXT:.b8 105
-; CHECK-NEXT:.b8 108
-; CHECK-NEXT:.b8 116
-; CHECK-NEXT:.b8 105
-; CHECK-NEXT:.b8 110
-; CHECK-NEXT:.b8 95
-; CHECK-NEXT:.b8 122
-; CHECK-NEXT:.b8 0
-; CHECK-NEXT:.b8 2                                   // DW_AT_decl_file
-; CHECK-NEXT:.b8 91                                  // DW_AT_decl_line
-; CHECK-NEXT:.b32 623                                // DW_AT_type
-; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 1                                   // DW_AT_external
-; CHECK-NEXT:.b8 4                                   // Abbrev [4] 0x3d8:0x4b DW_TAG_subprogram
-; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
-; CHECK-NEXT:.b8 90
-; CHECK-NEXT:.b8 78
-; CHECK-NEXT:.b8 75
-; CHECK-NEXT:.b8 50
-; CHECK-NEXT:.b8 53
-; CHECK-NEXT:.b8 95
-; CHECK-NEXT:.b8 95
-; CHECK-NEXT:.b8 99
-; CHECK-NEXT:.b8 117
-; CHECK-NEXT:.b8 100
-; CHECK-NEXT:.b8 97
-; CHECK-NEXT:.b8 95
-; CHECK-NEXT:.b8 98
-; CHECK-NEXT:.b8 117
-; CHECK-NEXT:.b8 105
-; CHECK-NEXT:.b8 108
-; CHECK-NEXT:.b8 116
-; CHECK-NEXT:.b8 105
-; CHECK-NEXT:.b8 110
-; CHECK-NEXT:.b8 95
-; CHECK-NEXT:.b8 98
-; CHECK-NEXT:.b8 108
-; CHECK-NEXT:.b8 111
-; CHECK-NEXT:.b8 99
-; CHECK-NEXT:.b8 107
-; CHECK-NEXT:.b8 68
-; CHECK-NEXT:.b8 105
-; CHECK-NEXT:.b8 109
-; CHECK-NEXT:.b8 95
-; CHECK-NEXT:.b8 116
-; CHECK-NEXT:.b8 99
-; CHECK-NEXT:.b8 118
-; CHECK-NEXT:.b8 52
-; CHECK-NEXT:.b8 100
-; CHECK-NEXT:.b8 105
-; CHECK-NEXT:.b8 109
-; CHECK-NEXT:.b8 51
-; CHECK-NEXT:.b8 69
-; CHECK-NEXT:.b8 118
-; CHECK-NEXT:.b8 0
-; CHECK-NEXT:.b8 111                                 // DW_AT_name
-; CHECK-NEXT:.b8 112
-; CHECK-NEXT:.b8 101
-; CHECK-NEXT:.b8 114
-; CHECK-NEXT:.b8 97
-; CHECK-NEXT:.b8 116
-; CHECK-NEXT:.b8 111
-; CHECK-NEXT:.b8 114
-; CHECK-NEXT:.b8 32
-; CHECK-NEXT:.b8 100
-; CHECK-NEXT:.b8 105
-; CHECK-NEXT:.b8 109
-; CHECK-NEXT:.b8 51
-; CHECK-NEXT:.b8 0
-; CHECK-NEXT:.b8 2                                   // DW_AT_decl_file
-; CHECK-NEXT:.b8 94                                  // DW_AT_decl_line
-; CHECK-NEXT:.b32 1289                               // DW_AT_type
-; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b32 1052                               // DW_AT_object_pointer
-; CHECK-NEXT:.b8 1                                   // DW_AT_external
-; CHECK-NEXT:.b8 5                                   // Abbrev [5] 0x41c:0x6 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 1477                               // DW_AT_type
-; CHECK-NEXT:.b8 1                                   // DW_AT_artificial
-; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 6                                   // Abbrev [6] 0x423:0x2b DW_TAG_subprogram
-; CHECK-NEXT:.b8 95                                  // DW_AT_name
-; CHECK-NEXT:.b8 95
-; CHECK-NEXT:.b8 99
-; CHECK-NEXT:.b8 117
-; CHECK-NEXT:.b8 100
-; CHECK-NEXT:.b8 97
-; CHECK-NEXT:.b8 95
-; CHECK-NEXT:.b8 98
-; CHECK-NEXT:.b8 117
-; CHECK-NEXT:.b8 105
-; CHECK-NEXT:.b8 108
-; CHECK-NEXT:.b8 116
-; CHECK-NEXT:.b8 105
-; CHECK-NEXT:.b8 110
-; CHECK-NEXT:.b8 95
-; CHECK-NEXT:.b8 98
-; CHECK-NEXT:.b8 108
-; CHECK-NEXT:.b8 111
-; CHECK-NEXT:.b8 99
-; CHECK-NEXT:.b8 107
-; CHECK-NEXT:.b8 68
-; CHECK-NEXT:.b8 105
-; CHECK-NEXT:.b8 109
-; CHECK-NEXT:.b8 95
-; CHECK-NEXT:.b8 116
-; CHECK-NEXT:.b8 0
-; CHECK-NEXT:.b8 2                                   // DW_AT_decl_file
-; CHECK-NEXT:.b8 96                                  // DW_AT_decl_line
-; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b32 1095                               // DW_AT_object_pointer
-; CHECK-NEXT:.b8 1                                   // DW_AT_external
-; CHECK-NEXT:.b8 3                                   // DW_AT_accessibility
-; CHECK-NEXT:                                        // DW_ACCESS_private
-; CHECK-NEXT:.b8 5                                   // Abbrev [5] 0x447:0x6 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 1487                               // DW_AT_type
-; CHECK-NEXT:.b8 1                                   // DW_AT_artificial
-; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 6                                   // Abbrev [6] 0x44e:0x30 DW_TAG_subprogram
-; CHECK-NEXT:.b8 95                                  // DW_AT_name
-; CHECK-NEXT:.b8 95
-; CHECK-NEXT:.b8 99
-; CHECK-NEXT:.b8 117
-; CHECK-NEXT:.b8 100
-; CHECK-NEXT:.b8 97
-; CHECK-NEXT:.b8 95
-; CHECK-NEXT:.b8 98
-; CHECK-NEXT:.b8 117
-; CHECK-NEXT:.b8 105
-; CHECK-NEXT:.b8 108
-; CHECK-NEXT:.b8 116
-; CHECK-NEXT:.b8 105
-; CHECK-NEXT:.b8 110
-; CHECK-NEXT:.b8 95
-; CHECK-NEXT:.b8 98
-; CHECK-NEXT:.b8 108
-; CHECK-NEXT:.b8 111
-; CHECK-NEXT:.b8 99
-; CHECK-NEXT:.b8 107
-; CHECK-NEXT:.b8 68
-; CHECK-NEXT:.b8 105
-; CHECK-NEXT:.b8 109
-; CHECK-NEXT:.b8 95
-; CHECK-NEXT:.b8 116
-; CHECK-NEXT:.b8 0
-; CHECK-NEXT:.b8 2                                   // DW_AT_decl_file
-; CHECK-NEXT:.b8 96                                  // DW_AT_decl_line
-; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b32 1138                               // DW_AT_object_pointer
-; CHECK-NEXT:.b8 1                                   // DW_AT_external
-; CHECK-NEXT:.b8 3                                   // DW_AT_accessibility
-; CHECK-NEXT:                                        // DW_ACCESS_private
-; CHECK-NEXT:.b8 5                                   // Abbrev [5] 0x472:0x6 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 1487                               // DW_AT_type
-; CHECK-NEXT:.b8 1                                   // DW_AT_artificial
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x478:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 1492                               // DW_AT_type
-; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 8                                   // Abbrev [8] 0x47e:0x47 DW_TAG_subprogram
-; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
-; CHECK-NEXT:.b8 90
-; CHECK-NEXT:.b8 78
-; CHECK-NEXT:.b8 75
-; CHECK-NEXT:.b8 50
-; CHECK-NEXT:.b8 53
-; CHECK-NEXT:.b8 95
-; CHECK-NEXT:.b8 95
-; CHECK-NEXT:.b8 99
-; CHECK-NEXT:.b8 117
-; CHECK-NEXT:.b8 100
-; CHECK-NEXT:.b8 97
-; CHECK-NEXT:.b8 95
-; CHECK-NEXT:.b8 98
-; CHECK-NEXT:.b8 117
-; CHECK-NEXT:.b8 105
-; CHECK-NEXT:.b8 108
-; CHECK-NEXT:.b8 116
-; CHECK-NEXT:.b8 105
-; CHECK-NEXT:.b8 110
-; CHECK-NEXT:.b8 95
-; CHECK-NEXT:.b8 98
-; CHECK-NEXT:.b8 108
-; CHECK-NEXT:.b8 111
-; CHECK-NEXT:.b8 99
-; CHECK-NEXT:.b8 107
-; CHECK-NEXT:.b8 68
-; CHECK-NEXT:.b8 105
-; CHECK-NEXT:.b8 109
-; CHECK-NEXT:.b8 95
-; CHECK-NEXT:.b8 116
-; CHECK-NEXT:.b8 97
-; CHECK-NEXT:.b8 83
-; CHECK-NEXT:.b8 69
-; CHECK-NEXT:.b8 82
-; CHECK-NEXT:.b8 75
-; CHECK-NEXT:.b8 83
-; CHECK-NEXT:.b8 95
-; CHECK-NEXT:.b8 0
-; CHECK-NEXT:.b8 111                                 // DW_AT_name
-; CHECK-NEXT:.b8 112
-; CHECK-NEXT:.b8 101
-; CHECK-NEXT:.b8 114
-; CHECK-NEXT:.b8 97
-; CHECK-NEXT:.b8 116
-; CHECK-NEXT:.b8 111
-; CHECK-NEXT:.b8 114
-; CHECK-NEXT:.b8 61
-; CHECK-NEXT:.b8 0
-; CHECK-NEXT:.b8 2                                   // DW_AT_decl_file
-; CHECK-NEXT:.b8 96                                  // DW_AT_decl_line
-; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b32 1209                               // DW_AT_object_pointer
-; CHECK-NEXT:.b8 1                                   // DW_AT_external
-; CHECK-NEXT:.b8 3                                   // DW_AT_accessibility
-; CHECK-NEXT:                                        // DW_ACCESS_private
-; CHECK-NEXT:.b8 5                                   // Abbrev [5] 0x4b9:0x6 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 1477                               // DW_AT_type
-; CHECK-NEXT:.b8 1                                   // DW_AT_artificial
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x4bf:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 1492                               // DW_AT_type
-; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 9                                   // Abbrev [9] 0x4c5:0x43 DW_TAG_subprogram
-; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
-; CHECK-NEXT:.b8 90
-; CHECK-NEXT:.b8 78
-; CHECK-NEXT:.b8 75
-; CHECK-NEXT:.b8 50
-; CHECK-NEXT:.b8 53
-; CHECK-NEXT:.b8 95
-; CHECK-NEXT:.b8 95
-; CHECK-NEXT:.b8 99
-; CHECK-NEXT:.b8 117
-; CHECK-NEXT:.b8 100
-; CHECK-NEXT:.b8 97
-; CHECK-NEXT:.b8 95
-; CHECK-NEXT:.b8 98
-; CHECK-NEXT:.b8 117
-; CHECK-NEXT:.b8 105
-; CHECK-NEXT:.b8 108
-; CHECK-NEXT:.b8 116
-; CHECK-NEXT:.b8 105
-; CHECK-NEXT:.b8 110
-; CHECK-NEXT:.b8 95
-; CHECK-NEXT:.b8 98
-; CHECK-NEXT:.b8 108
-; CHECK-NEXT:.b8 111
-; CHECK-NEXT:.b8 99
-; CHECK-NEXT:.b8 107
-; CHECK-NEXT:.b8 68
-; CHECK-NEXT:.b8 105
-; CHECK-NEXT:.b8 109
-; CHECK-NEXT:.b8 95
-; CHECK-NEXT:.b8 116
-; CHECK-NEXT:.b8 97
-; CHECK-NEXT:.b8 100
-; CHECK-NEXT:.b8 69
-; CHECK-NEXT:.b8 118
-; CHECK-NEXT:.b8 0
-; CHECK-NEXT:.b8 111                                 // DW_AT_name
-; CHECK-NEXT:.b8 112
-; CHECK-NEXT:.b8 101
-; CHECK-NEXT:.b8 114
-; CHECK-NEXT:.b8 97
-; CHECK-NEXT:.b8 116
-; CHECK-NEXT:.b8 111
-; CHECK-NEXT:.b8 114
-; CHECK-NEXT:.b8 38
-; CHECK-NEXT:.b8 0
-; CHECK-NEXT:.b8 2                                   // DW_AT_decl_file
-; CHECK-NEXT:.b8 96                                  // DW_AT_decl_line
-; CHECK-NEXT:.b32 1497                               // DW_AT_type
-; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b32 1281                               // DW_AT_object_pointer
-; CHECK-NEXT:.b8 1                                   // DW_AT_external
-; CHECK-NEXT:.b8 3                                   // DW_AT_accessibility
-; CHECK-NEXT:                                        // DW_ACCESS_private
-; CHECK-NEXT:.b8 5                                   // Abbrev [5] 0x501:0x6 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 1477                               // DW_AT_type
-; CHECK-NEXT:.b8 1                                   // DW_AT_artificial
-; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 16                                  // Abbrev [16] 0x509:0xa9 DW_TAG_structure_type
-; CHECK-NEXT:.b8 100                                 // DW_AT_name
-; CHECK-NEXT:.b8 105
-; CHECK-NEXT:.b8 109
-; CHECK-NEXT:.b8 51
-; CHECK-NEXT:.b8 0
-; CHECK-NEXT:.b8 12                                  // DW_AT_byte_size
-; CHECK-NEXT:.b8 3                                   // DW_AT_decl_file
-; CHECK-NEXT:.b8 161                                 // DW_AT_decl_line
-; CHECK-NEXT:.b8 1
-; CHECK-NEXT:.b8 17                                  // Abbrev [17] 0x513:0xd DW_TAG_member
-; CHECK-NEXT:.b8 120                                 // DW_AT_name
-; CHECK-NEXT:.b8 0
-; CHECK-NEXT:.b32 623                                // DW_AT_type
-; CHECK-NEXT:.b8 3                                   // DW_AT_decl_file
-; CHECK-NEXT:.b8 163                                 // DW_AT_decl_line
-; CHECK-NEXT:.b8 1
-; CHECK-NEXT:.b8 2                                   // DW_AT_data_member_location
-; CHECK-NEXT:.b8 35
-; CHECK-NEXT:.b8 0
-; CHECK-NEXT:.b8 17                                  // Abbrev [17] 0x520:0xd DW_TAG_member
-; CHECK-NEXT:.b8 121                                 // DW_AT_name
-; CHECK-NEXT:.b8 0
-; CHECK-NEXT:.b32 623                                // DW_AT_type
-; CHECK-NEXT:.b8 3                                   // DW_AT_decl_file
-; CHECK-NEXT:.b8 163                                 // DW_AT_decl_line
-; CHECK-NEXT:.b8 1
-; CHECK-NEXT:.b8 2                                   // DW_AT_data_member_location
-; CHECK-NEXT:.b8 35
-; CHECK-NEXT:.b8 4
-; CHECK-NEXT:.b8 17                                  // Abbrev [17] 0x52d:0xd DW_TAG_member
-; CHECK-NEXT:.b8 122                                 // DW_AT_name
-; CHECK-NEXT:.b8 0
-; CHECK-NEXT:.b32 623                                // DW_AT_type
-; CHECK-NEXT:.b8 3                                   // DW_AT_decl_file
-; CHECK-NEXT:.b8 163                                 // DW_AT_decl_line
-; CHECK-NEXT:.b8 1
-; CHECK-NEXT:.b8 2                                   // DW_AT_data_member_location
-; CHECK-NEXT:.b8 35
-; CHECK-NEXT:.b8 8
-; CHECK-NEXT:.b8 18                                  // Abbrev [18] 0x53a:0x25 DW_TAG_subprogram
-; CHECK-NEXT:.b8 100                                 // DW_AT_name
-; CHECK-NEXT:.b8 105
-; CHECK-NEXT:.b8 109
-; CHECK-NEXT:.b8 51
-; CHECK-NEXT:.b8 0
-; CHECK-NEXT:.b8 3                                   // DW_AT_decl_file
-; CHECK-NEXT:.b8 165                                 // DW_AT_decl_line
-; CHECK-NEXT:.b8 1
-; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b32 1353                               // DW_AT_object_pointer
-; CHECK-NEXT:.b8 1                                   // DW_AT_external
-; CHECK-NEXT:.b8 5                                   // Abbrev [5] 0x549:0x6 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 1458                               // DW_AT_type
-; CHECK-NEXT:.b8 1                                   // DW_AT_artificial
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x54f:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 623                                // DW_AT_type
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x554:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 623                                // DW_AT_type
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x559:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 623                                // DW_AT_type
-; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 18                                  // Abbrev [18] 0x55f:0x1b DW_TAG_subprogram
-; CHECK-NEXT:.b8 100                                 // DW_AT_name
-; CHECK-NEXT:.b8 105
-; CHECK-NEXT:.b8 109
-; CHECK-NEXT:.b8 51
-; CHECK-NEXT:.b8 0
-; CHECK-NEXT:.b8 3                                   // DW_AT_decl_file
-; CHECK-NEXT:.b8 166                                 // DW_AT_decl_line
-; CHECK-NEXT:.b8 1
-; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b32 1390                               // DW_AT_object_pointer
-; CHECK-NEXT:.b8 1                                   // DW_AT_external
-; CHECK-NEXT:.b8 5                                   // Abbrev [5] 0x56e:0x6 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 1458                               // DW_AT_type
-; CHECK-NEXT:.b8 1                                   // DW_AT_artificial
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x574:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 1463                               // DW_AT_type
-; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 19                                  // Abbrev [19] 0x57a:0x37 DW_TAG_subprogram
-; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
-; CHECK-NEXT:.b8 90
-; CHECK-NEXT:.b8 78
-; CHECK-NEXT:.b8 52
-; CHECK-NEXT:.b8 100
-; CHECK-NEXT:.b8 105
-; CHECK-NEXT:.b8 109
-; CHECK-NEXT:.b8 51
-; CHECK-NEXT:.b8 99
-; CHECK-NEXT:.b8 118
-; CHECK-NEXT:.b8 53
-; CHECK-NEXT:.b8 117
-; CHECK-NEXT:.b8 105
-; CHECK-NEXT:.b8 110
-; CHECK-NEXT:.b8 116
-; CHECK-NEXT:.b8 51
-; CHECK-NEXT:.b8 69
-; CHECK-NEXT:.b8 118
-; CHECK-NEXT:.b8 0
-; CHECK-NEXT:.b8 111                                 // DW_AT_name
-; CHECK-NEXT:.b8 112
-; CHECK-NEXT:.b8 101
-; CHECK-NEXT:.b8 114
-; CHECK-NEXT:.b8 97
-; CHECK-NEXT:.b8 116
-; CHECK-NEXT:.b8 111
-; CHECK-NEXT:.b8 114
-; CHECK-NEXT:.b8 32
-; CHECK-NEXT:.b8 117
-; CHECK-NEXT:.b8 105
-; CHECK-NEXT:.b8 110
-; CHECK-NEXT:.b8 116
-; CHECK-NEXT:.b8 51
-; CHECK-NEXT:.b8 0
-; CHECK-NEXT:.b8 3                                   // DW_AT_decl_file
-; CHECK-NEXT:.b8 167                                 // DW_AT_decl_line
-; CHECK-NEXT:.b8 1
-; CHECK-NEXT:.b32 1463                               // DW_AT_type
-; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b32 1450                               // DW_AT_object_pointer
-; CHECK-NEXT:.b8 1                                   // DW_AT_external
-; CHECK-NEXT:.b8 5                                   // Abbrev [5] 0x5aa:0x6 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 1458                               // DW_AT_type
-; CHECK-NEXT:.b8 1                                   // DW_AT_artificial
-; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 12                                  // Abbrev [12] 0x5b2:0x5 DW_TAG_pointer_type
-; CHECK-NEXT:.b32 1289                               // DW_AT_type
-; CHECK-NEXT:.b8 20                                  // Abbrev [20] 0x5b7:0xe DW_TAG_typedef
-; CHECK-NEXT:.b32 639                                // DW_AT_type
-; CHECK-NEXT:.b8 117                                 // DW_AT_name
-; CHECK-NEXT:.b8 105
-; CHECK-NEXT:.b8 110
-; CHECK-NEXT:.b8 116
-; CHECK-NEXT:.b8 51
-; CHECK-NEXT:.b8 0
-; CHECK-NEXT:.b8 3                                   // DW_AT_decl_file
-; CHECK-NEXT:.b8 127                                 // DW_AT_decl_line
-; CHECK-NEXT:.b8 1
-; CHECK-NEXT:.b8 12                                  // Abbrev [12] 0x5c5:0x5 DW_TAG_pointer_type
-; CHECK-NEXT:.b32 1482                               // DW_AT_type
-; CHECK-NEXT:.b8 13                                  // Abbrev [13] 0x5ca:0x5 DW_TAG_const_type
-; CHECK-NEXT:.b32 717                                // DW_AT_type
-; CHECK-NEXT:.b8 12                                  // Abbrev [12] 0x5cf:0x5 DW_TAG_pointer_type
-; CHECK-NEXT:.b32 717                                // DW_AT_type
-; CHECK-NEXT:.b8 14                                  // Abbrev [14] 0x5d4:0x5 DW_TAG_reference_type
-; CHECK-NEXT:.b32 1482                               // DW_AT_type
-; CHECK-NEXT:.b8 12                                  // Abbrev [12] 0x5d9:0x5 DW_TAG_pointer_type
-; CHECK-NEXT:.b32 717                                // DW_AT_type
-; CHECK-NEXT:.b8 15                                  // Abbrev [15] 0x5de:0x6 DW_TAG_subprogram
-; CHECK-NEXT:.b32 747                                // DW_AT_specification
-; CHECK-NEXT:.b8 1                                   // DW_AT_inline
-; CHECK-NEXT:.b8 2                                   // Abbrev [2] 0x5e4:0x247 DW_TAG_structure_type
-; CHECK-NEXT:.b8 95                                  // DW_AT_name
-; CHECK-NEXT:.b8 95
-; CHECK-NEXT:.b8 99
-; CHECK-NEXT:.b8 117
-; CHECK-NEXT:.b8 100
-; CHECK-NEXT:.b8 97
-; CHECK-NEXT:.b8 95
-; CHECK-NEXT:.b8 98
-; CHECK-NEXT:.b8 117
-; CHECK-NEXT:.b8 105
-; CHECK-NEXT:.b8 108
-; CHECK-NEXT:.b8 116
-; CHECK-NEXT:.b8 105
-; CHECK-NEXT:.b8 110
-; CHECK-NEXT:.b8 95
-; CHECK-NEXT:.b8 116
-; CHECK-NEXT:.b8 104
-; CHECK-NEXT:.b8 114
-; CHECK-NEXT:.b8 101
-; CHECK-NEXT:.b8 97
-; CHECK-NEXT:.b8 100
-; CHECK-NEXT:.b8 73
-; CHECK-NEXT:.b8 100
-; CHECK-NEXT:.b8 120
-; CHECK-NEXT:.b8 95
-; CHECK-NEXT:.b8 116
-; CHECK-NEXT:.b8 0
-; CHECK-NEXT:.b8 1                                   // DW_AT_byte_size
-; CHECK-NEXT:.b8 2                                   // DW_AT_decl_file
-; CHECK-NEXT:.b8 66                                  // DW_AT_decl_line
-; CHECK-NEXT:.b8 3                                   // Abbrev [3] 0x603:0x50 DW_TAG_subprogram
-; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
-; CHECK-NEXT:.b8 90
-; CHECK-NEXT:.b8 78
-; CHECK-NEXT:.b8 50
-; CHECK-NEXT:.b8 54
-; CHECK-NEXT:.b8 95
-; CHECK-NEXT:.b8 95
-; CHECK-NEXT:.b8 99
-; CHECK-NEXT:.b8 117
-; CHECK-NEXT:.b8 100
-; CHECK-NEXT:.b8 97
-; CHECK-NEXT:.b8 95
-; CHECK-NEXT:.b8 98
-; CHECK-NEXT:.b8 117
-; CHECK-NEXT:.b8 105
-; CHECK-NEXT:.b8 108
-; CHECK-NEXT:.b8 116
-; CHECK-NEXT:.b8 105
-; CHECK-NEXT:.b8 110
-; CHECK-NEXT:.b8 95
-; CHECK-NEXT:.b8 116
-; CHECK-NEXT:.b8 104
-; CHECK-NEXT:.b8 114
-; CHECK-NEXT:.b8 101
-; CHECK-NEXT:.b8 97
-; CHECK-NEXT:.b8 100
-; CHECK-NEXT:.b8 73
-; CHECK-NEXT:.b8 100
-; CHECK-NEXT:.b8 120
-; CHECK-NEXT:.b8 95
-; CHECK-NEXT:.b8 116
-; CHECK-NEXT:.b8 49
-; CHECK-NEXT:.b8 55
-; CHECK-NEXT:.b8 95
-; CHECK-NEXT:.b8 95
-; CHECK-NEXT:.b8 102
-; CHECK-NEXT:.b8 101
-; CHECK-NEXT:.b8 116
-; CHECK-NEXT:.b8 99
-; CHECK-NEXT:.b8 104
-; CHECK-NEXT:.b8 95
-; CHECK-NEXT:.b8 98
-; CHECK-NEXT:.b8 117
-; CHECK-NEXT:.b8 105
-; CHECK-NEXT:.b8 108
-; CHECK-NEXT:.b8 116
-; CHECK-NEXT:.b8 105
-; CHECK-NEXT:.b8 110
-; CHECK-NEXT:.b8 95
-; CHECK-NEXT:.b8 120
-; CHECK-NEXT:.b8 69
-; CHECK-NEXT:.b8 118
-; CHECK-NEXT:.b8 0
-; CHECK-NEXT:.b8 95                                  // DW_AT_name
-; CHECK-NEXT:.b8 95
-; CHECK-NEXT:.b8 102
-; CHECK-NEXT:.b8 101
-; CHECK-NEXT:.b8 116
-; CHECK-NEXT:.b8 99
-; CHECK-NEXT:.b8 104
-; CHECK-NEXT:.b8 95
-; CHECK-NEXT:.b8 98
-; CHECK-NEXT:.b8 117
-; CHECK-NEXT:.b8 105
-; CHECK-NEXT:.b8 108
-; CHECK-NEXT:.b8 116
-; CHECK-NEXT:.b8 105
-; CHECK-NEXT:.b8 110
-; CHECK-NEXT:.b8 95
-; CHECK-NEXT:.b8 120
-; CHECK-NEXT:.b8 0
-; CHECK-NEXT:.b8 2                                   // DW_AT_decl_file
-; CHECK-NEXT:.b8 67                                  // DW_AT_decl_line
-; CHECK-NEXT:.b32 623                                // DW_AT_type
-; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 1                                   // DW_AT_external
-; CHECK-NEXT:.b8 3                                   // Abbrev [3] 0x653:0x50 DW_TAG_subprogram
-; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
-; CHECK-NEXT:.b8 90
-; CHECK-NEXT:.b8 78
-; CHECK-NEXT:.b8 50
-; CHECK-NEXT:.b8 54
-; CHECK-NEXT:.b8 95
-; CHECK-NEXT:.b8 95
-; CHECK-NEXT:.b8 99
-; CHECK-NEXT:.b8 117
-; CHECK-NEXT:.b8 100
-; CHECK-NEXT:.b8 97
-; CHECK-NEXT:.b8 95
-; CHECK-NEXT:.b8 98
-; CHECK-NEXT:.b8 117
-; CHECK-NEXT:.b8 105
-; CHECK-NEXT:.b8 108
-; CHECK-NEXT:.b8 116
-; CHECK-NEXT:.b8 105
-; CHECK-NEXT:.b8 110
-; CHECK-NEXT:.b8 95
-; CHECK-NEXT:.b8 116
-; CHECK-NEXT:.b8 104
-; CHECK-NEXT:.b8 114
-; CHECK-NEXT:.b8 101
-; CHECK-NEXT:.b8 97
-; CHECK-NEXT:.b8 100
-; CHECK-NEXT:.b8 73
-; CHECK-NEXT:.b8 100
-; CHECK-NEXT:.b8 120
-; CHECK-NEXT:.b8 95
-; CHECK-NEXT:.b8 116
-; CHECK-NEXT:.b8 49
-; CHECK-NEXT:.b8 55
-; CHECK-NEXT:.b8 95
-; CHECK-NEXT:.b8 95
-; CHECK-NEXT:.b8 102
-; CHECK-NEXT:.b8 101
-; CHECK-NEXT:.b8 116
-; CHECK-NEXT:.b8 99
-; CHECK-NEXT:.b8 104
-; CHECK-NEXT:.b8 95
-; CHECK-NEXT:.b8 98
-; CHECK-NEXT:.b8 117
-; CHECK-NEXT:.b8 105
-; CHECK-NEXT:.b8 108
-; CHECK-NEXT:.b8 116
-; CHECK-NEXT:.b8 105
-; CHECK-NEXT:.b8 110
-; CHECK-NEXT:.b8 95
-; CHECK-NEXT:.b8 121
-; CHECK-NEXT:.b8 69
-; CHECK-NEXT:.b8 118
-; CHECK-NEXT:.b8 0
-; CHECK-NEXT:.b8 95                                  // DW_AT_name
-; CHECK-NEXT:.b8 95
-; CHECK-NEXT:.b8 102
-; CHECK-NEXT:.b8 101
-; CHECK-NEXT:.b8 116
-; CHECK-NEXT:.b8 99
-; CHECK-NEXT:.b8 104
-; CHECK-NEXT:.b8 95
-; CHECK-NEXT:.b8 98
-; CHECK-NEXT:.b8 117
-; CHECK-NEXT:.b8 105
-; CHECK-NEXT:.b8 108
-; CHECK-NEXT:.b8 116
-; CHECK-NEXT:.b8 105
-; CHECK-NEXT:.b8 110
-; CHECK-NEXT:.b8 95
-; CHECK-NEXT:.b8 121
-; CHECK-NEXT:.b8 0
-; CHECK-NEXT:.b8 2                                   // DW_AT_decl_file
-; CHECK-NEXT:.b8 68                                  // DW_AT_decl_line
-; CHECK-NEXT:.b32 623                                // DW_AT_type
-; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 1                                   // DW_AT_external
-; CHECK-NEXT:.b8 3                                   // Abbrev [3] 0x6a3:0x50 DW_TAG_subprogram
-; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
-; CHECK-NEXT:.b8 90
-; CHECK-NEXT:.b8 78
-; CHECK-NEXT:.b8 50
-; CHECK-NEXT:.b8 54
-; CHECK-NEXT:.b8 95
-; CHECK-NEXT:.b8 95
-; CHECK-NEXT:.b8 99
-; CHECK-NEXT:.b8 117
-; CHECK-NEXT:.b8 100
-; CHECK-NEXT:.b8 97
-; CHECK-NEXT:.b8 95
-; CHECK-NEXT:.b8 98
-; CHECK-NEXT:.b8 117
-; CHECK-NEXT:.b8 105
-; CHECK-NEXT:.b8 108
-; CHECK-NEXT:.b8 116
-; CHECK-NEXT:.b8 105
-; CHECK-NEXT:.b8 110
-; CHECK-NEXT:.b8 95
-; CHECK-NEXT:.b8 116
-; CHECK-NEXT:.b8 104
-; CHECK-NEXT:.b8 114
-; CHECK-NEXT:.b8 101
-; CHECK-NEXT:.b8 97
-; CHECK-NEXT:.b8 100
-; CHECK-NEXT:.b8 73
-; CHECK-NEXT:.b8 100
-; CHECK-NEXT:.b8 120
-; CHECK-NEXT:.b8 95
-; CHECK-NEXT:.b8 116
-; CHECK-NEXT:.b8 49
-; CHECK-NEXT:.b8 55
-; CHECK-NEXT:.b8 95
-; CHECK-NEXT:.b8 95
-; CHECK-NEXT:.b8 102
-; CHECK-NEXT:.b8 101
-; CHECK-NEXT:.b8 116
-; CHECK-NEXT:.b8 99
-; CHECK-NEXT:.b8 104
-; CHECK-NEXT:.b8 95
-; CHECK-NEXT:.b8 98
-; CHECK-NEXT:.b8 117
-; CHECK-NEXT:.b8 105
-; CHECK-NEXT:.b8 108
-; CHECK-NEXT:.b8 116
-; CHECK-NEXT:.b8 105
-; CHECK-NEXT:.b8 110
-; CHECK-NEXT:.b8 95
-; CHECK-NEXT:.b8 122
-; CHECK-NEXT:.b8 69
-; CHECK-NEXT:.b8 118
-; CHECK-NEXT:.b8 0
-; CHECK-NEXT:.b8 95                                  // DW_AT_name
-; CHECK-NEXT:.b8 95
-; CHECK-NEXT:.b8 102
-; CHECK-NEXT:.b8 101
-; CHECK-NEXT:.b8 116
-; CHECK-NEXT:.b8 99
-; CHECK-NEXT:.b8 104
-; CHECK-NEXT:.b8 95
-; CHECK-NEXT:.b8 98
-; CHECK-NEXT:.b8 117
-; CHECK-NEXT:.b8 105
-; CHECK-NEXT:.b8 108
-; CHECK-NEXT:.b8 116
-; CHECK-NEXT:.b8 105
-; CHECK-NEXT:.b8 110
-; CHECK-NEXT:.b8 95
-; CHECK-NEXT:.b8 122
-; CHECK-NEXT:.b8 0
-; CHECK-NEXT:.b8 2                                   // DW_AT_decl_file
-; CHECK-NEXT:.b8 69                                  // DW_AT_decl_line
-; CHECK-NEXT:.b32 623                                // DW_AT_type
-; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 1                                   // DW_AT_external
-; CHECK-NEXT:.b8 4                                   // Abbrev [4] 0x6f3:0x4e DW_TAG_subprogram
-; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
-; CHECK-NEXT:.b8 90
-; CHECK-NEXT:.b8 78
-; CHECK-NEXT:.b8 75
-; CHECK-NEXT:.b8 50
-; CHECK-NEXT:.b8 54
-; CHECK-NEXT:.b8 95
-; CHECK-NEXT:.b8 95
-; CHECK-NEXT:.b8 99
-; CHECK-NEXT:.b8 117
-; CHECK-NEXT:.b8 100
-; CHECK-NEXT:.b8 97
-; CHECK-NEXT:.b8 95
-; CHECK-NEXT:.b8 98
-; CHECK-NEXT:.b8 117
-; CHECK-NEXT:.b8 105
-; CHECK-NEXT:.b8 108
-; CHECK-NEXT:.b8 116
-; CHECK-NEXT:.b8 105
-; CHECK-NEXT:.b8 110
-; CHECK-NEXT:.b8 95
-; CHECK-NEXT:.b8 116
-; CHECK-NEXT:.b8 104
-; CHECK-NEXT:.b8 114
-; CHECK-NEXT:.b8 101
-; CHECK-NEXT:.b8 97
-; CHECK-NEXT:.b8 100
-; CHECK-NEXT:.b8 73
-; CHECK-NEXT:.b8 100
-; CHECK-NEXT:.b8 120
-; CHECK-NEXT:.b8 95
-; CHECK-NEXT:.b8 116
-; CHECK-NEXT:.b8 99
-; CHECK-NEXT:.b8 118
-; CHECK-NEXT:.b8 53
-; CHECK-NEXT:.b8 117
-; CHECK-NEXT:.b8 105
-; CHECK-NEXT:.b8 110
-; CHECK-NEXT:.b8 116
-; CHECK-NEXT:.b8 51
-; CHECK-NEXT:.b8 69
-; CHECK-NEXT:.b8 118
-; CHECK-NEXT:.b8 0
-; CHECK-NEXT:.b8 111                                 // DW_AT_name
-; CHECK-NEXT:.b8 112
-; CHECK-NEXT:.b8 101
-; CHECK-NEXT:.b8 114
-; CHECK-NEXT:.b8 97
-; CHECK-NEXT:.b8 116
-; CHECK-NEXT:.b8 111
-; CHECK-NEXT:.b8 114
-; CHECK-NEXT:.b8 32
-; CHECK-NEXT:.b8 117
-; CHECK-NEXT:.b8 105
-; CHECK-NEXT:.b8 110
-; CHECK-NEXT:.b8 116
-; CHECK-NEXT:.b8 51
-; CHECK-NEXT:.b8 0
-; CHECK-NEXT:.b8 2                                   // DW_AT_decl_file
-; CHECK-NEXT:.b8 72                                  // DW_AT_decl_line
-; CHECK-NEXT:.b32 639                                // DW_AT_type
-; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b32 1850                               // DW_AT_object_pointer
-; CHECK-NEXT:.b8 1                                   // DW_AT_external
-; CHECK-NEXT:.b8 5                                   // Abbrev [5] 0x73a:0x6 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 2091                               // DW_AT_type
-; CHECK-NEXT:.b8 1                                   // DW_AT_artificial
-; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 6                                   // Abbrev [6] 0x741:0x2c DW_TAG_subprogram
-; CHECK-NEXT:.b8 95                                  // DW_AT_name
-; CHECK-NEXT:.b8 95
-; CHECK-NEXT:.b8 99
-; CHECK-NEXT:.b8 117
-; CHECK-NEXT:.b8 100
-; CHECK-NEXT:.b8 97
-; CHECK-NEXT:.b8 95
-; CHECK-NEXT:.b8 98
-; CHECK-NEXT:.b8 117
-; CHECK-NEXT:.b8 105
-; CHECK-NEXT:.b8 108
-; CHECK-NEXT:.b8 116
-; CHECK-NEXT:.b8 105
-; CHECK-NEXT:.b8 110
-; CHECK-NEXT:.b8 95
-; CHECK-NEXT:.b8 116
-; CHECK-NEXT:.b8 104
-; CHECK-NEXT:.b8 114
-; CHECK-NEXT:.b8 101
-; CHECK-NEXT:.b8 97
-; CHECK-NEXT:.b8 100
-; CHECK-NEXT:.b8 73
-; CHECK-NEXT:.b8 100
-; CHECK-NEXT:.b8 120
-; CHECK-NEXT:.b8 95
-; CHECK-NEXT:.b8 116
-; CHECK-NEXT:.b8 0
-; CHECK-NEXT:.b8 2                                   // DW_AT_decl_file
-; CHECK-NEXT:.b8 74                                  // DW_AT_decl_line
-; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b32 1894                               // DW_AT_object_pointer
-; CHECK-NEXT:.b8 1                                   // DW_AT_external
-; CHECK-NEXT:.b8 3                                   // DW_AT_accessibility
-; CHECK-NEXT:                                        // DW_ACCESS_private
-; CHECK-NEXT:.b8 5                                   // Abbrev [5] 0x766:0x6 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 2101                               // DW_AT_type
-; CHECK-NEXT:.b8 1                                   // DW_AT_artificial
-; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 6                                   // Abbrev [6] 0x76d:0x31 DW_TAG_subprogram
-; CHECK-NEXT:.b8 95                                  // DW_AT_name
-; CHECK-NEXT:.b8 95
-; CHECK-NEXT:.b8 99
-; CHECK-NEXT:.b8 117
-; CHECK-NEXT:.b8 100
-; CHECK-NEXT:.b8 97
-; CHECK-NEXT:.b8 95
-; CHECK-NEXT:.b8 98
-; CHECK-NEXT:.b8 117
-; CHECK-NEXT:.b8 105
-; CHECK-NEXT:.b8 108
-; CHECK-NEXT:.b8 116
-; CHECK-NEXT:.b8 105
-; CHECK-NEXT:.b8 110
-; CHECK-NEXT:.b8 95
-; CHECK-NEXT:.b8 116
-; CHECK-NEXT:.b8 104
-; CHECK-NEXT:.b8 114
-; CHECK-NEXT:.b8 101
-; CHECK-NEXT:.b8 97
-; CHECK-NEXT:.b8 100
-; CHECK-NEXT:.b8 73
-; CHECK-NEXT:.b8 100
-; CHECK-NEXT:.b8 120
-; CHECK-NEXT:.b8 95
-; CHECK-NEXT:.b8 116
-; CHECK-NEXT:.b8 0
-; CHECK-NEXT:.b8 2                                   // DW_AT_decl_file
-; CHECK-NEXT:.b8 74                                  // DW_AT_decl_line
-; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b32 1938                               // DW_AT_object_pointer
-; CHECK-NEXT:.b8 1                                   // DW_AT_external
-; CHECK-NEXT:.b8 3                                   // DW_AT_accessibility
-; CHECK-NEXT:                                        // DW_ACCESS_private
-; CHECK-NEXT:.b8 5                                   // Abbrev [5] 0x792:0x6 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 2101                               // DW_AT_type
-; CHECK-NEXT:.b8 1                                   // DW_AT_artificial
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x798:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 2106                               // DW_AT_type
-; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 8                                   // Abbrev [8] 0x79e:0x48 DW_TAG_subprogram
-; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
-; CHECK-NEXT:.b8 90
-; CHECK-NEXT:.b8 78
-; CHECK-NEXT:.b8 75
-; CHECK-NEXT:.b8 50
-; CHECK-NEXT:.b8 54
-; CHECK-NEXT:.b8 95
-; CHECK-NEXT:.b8 95
-; CHECK-NEXT:.b8 99
-; CHECK-NEXT:.b8 117
-; CHECK-NEXT:.b8 100
-; CHECK-NEXT:.b8 97
-; CHECK-NEXT:.b8 95
-; CHECK-NEXT:.b8 98
-; CHECK-NEXT:.b8 117
-; CHECK-NEXT:.b8 105
-; CHECK-NEXT:.b8 108
-; CHECK-NEXT:.b8 116
-; CHECK-NEXT:.b8 105
-; CHECK-NEXT:.b8 110
-; CHECK-NEXT:.b8 95
-; CHECK-NEXT:.b8 116
-; CHECK-NEXT:.b8 104
-; CHECK-NEXT:.b8 114
-; CHECK-NEXT:.b8 101
-; CHECK-NEXT:.b8 97
-; CHECK-NEXT:.b8 100
-; CHECK-NEXT:.b8 73
-; CHECK-NEXT:.b8 100
-; CHECK-NEXT:.b8 120
-; CHECK-NEXT:.b8 95
-; CHECK-NEXT:.b8 116
-; CHECK-NEXT:.b8 97
-; CHECK-NEXT:.b8 83
-; CHECK-NEXT:.b8 69
-; CHECK-NEXT:.b8 82
-; CHECK-NEXT:.b8 75
-; CHECK-NEXT:.b8 83
-; CHECK-NEXT:.b8 95
-; CHECK-NEXT:.b8 0
-; CHECK-NEXT:.b8 111                                 // DW_AT_name
-; CHECK-NEXT:.b8 112
-; CHECK-NEXT:.b8 101
-; CHECK-NEXT:.b8 114
-; CHECK-NEXT:.b8 97
-; CHECK-NEXT:.b8 116
-; CHECK-NEXT:.b8 111
-; CHECK-NEXT:.b8 114
-; CHECK-NEXT:.b8 61
-; CHECK-NEXT:.b8 0
-; CHECK-NEXT:.b8 2                                   // DW_AT_decl_file
-; CHECK-NEXT:.b8 74                                  // DW_AT_decl_line
-; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b32 2010                               // DW_AT_object_pointer
-; CHECK-NEXT:.b8 1                                   // DW_AT_external
-; CHECK-NEXT:.b8 3                                   // DW_AT_accessibility
-; CHECK-NEXT:                                        // DW_ACCESS_private
-; CHECK-NEXT:.b8 5                                   // Abbrev [5] 0x7da:0x6 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 2091                               // DW_AT_type
-; CHECK-NEXT:.b8 1                                   // DW_AT_artificial
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x7e0:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 2106                               // DW_AT_type
-; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 9                                   // Abbrev [9] 0x7e6:0x44 DW_TAG_subprogram
-; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
-; CHECK-NEXT:.b8 90
-; CHECK-NEXT:.b8 78
-; CHECK-NEXT:.b8 75
-; CHECK-NEXT:.b8 50
-; CHECK-NEXT:.b8 54
-; CHECK-NEXT:.b8 95
-; CHECK-NEXT:.b8 95
-; CHECK-NEXT:.b8 99
-; CHECK-NEXT:.b8 117
-; CHECK-NEXT:.b8 100
-; CHECK-NEXT:.b8 97
-; CHECK-NEXT:.b8 95
-; CHECK-NEXT:.b8 98
-; CHECK-NEXT:.b8 117
-; CHECK-NEXT:.b8 105
-; CHECK-NEXT:.b8 108
-; CHECK-NEXT:.b8 116
-; CHECK-NEXT:.b8 105
-; CHECK-NEXT:.b8 110
-; CHECK-NEXT:.b8 95
-; CHECK-NEXT:.b8 116
-; CHECK-NEXT:.b8 104
-; CHECK-NEXT:.b8 114
-; CHECK-NEXT:.b8 101
-; CHECK-NEXT:.b8 97
-; CHECK-NEXT:.b8 100
-; CHECK-NEXT:.b8 73
-; CHECK-NEXT:.b8 100
-; CHECK-NEXT:.b8 120
-; CHECK-NEXT:.b8 95
-; CHECK-NEXT:.b8 116
-; CHECK-NEXT:.b8 97
-; CHECK-NEXT:.b8 100
-; CHECK-NEXT:.b8 69
-; CHECK-NEXT:.b8 118
-; CHECK-NEXT:.b8 0
-; CHECK-NEXT:.b8 111                                 // DW_AT_name
-; CHECK-NEXT:.b8 112
-; CHECK-NEXT:.b8 101
-; CHECK-NEXT:.b8 114
-; CHECK-NEXT:.b8 97
-; CHECK-NEXT:.b8 116
-; CHECK-NEXT:.b8 111
-; CHECK-NEXT:.b8 114
-; CHECK-NEXT:.b8 38
-; CHECK-NEXT:.b8 0
-; CHECK-NEXT:.b8 2                                   // DW_AT_decl_file
-; CHECK-NEXT:.b8 74                                  // DW_AT_decl_line
-; CHECK-NEXT:.b32 2111                               // DW_AT_type
-; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b32 2083                               // DW_AT_object_pointer
-; CHECK-NEXT:.b8 1                                   // DW_AT_external
-; CHECK-NEXT:.b8 3                                   // DW_AT_accessibility
-; CHECK-NEXT:                                        // DW_ACCESS_private
-; CHECK-NEXT:.b8 5                                   // Abbrev [5] 0x823:0x6 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 2091                               // DW_AT_type
-; CHECK-NEXT:.b8 1                                   // DW_AT_artificial
-; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 12                                  // Abbrev [12] 0x82b:0x5 DW_TAG_pointer_type
-; CHECK-NEXT:.b32 2096                               // DW_AT_type
-; CHECK-NEXT:.b8 13                                  // Abbrev [13] 0x830:0x5 DW_TAG_const_type
-; CHECK-NEXT:.b32 1508                               // DW_AT_type
-; CHECK-NEXT:.b8 12                                  // Abbrev [12] 0x835:0x5 DW_TAG_pointer_type
-; CHECK-NEXT:.b32 1508                               // DW_AT_type
-; CHECK-NEXT:.b8 14                                  // Abbrev [14] 0x83a:0x5 DW_TAG_reference_type
-; CHECK-NEXT:.b32 2096                               // DW_AT_type
-; CHECK-NEXT:.b8 12                                  // Abbrev [12] 0x83f:0x5 DW_TAG_pointer_type
-; CHECK-NEXT:.b32 1508                               // DW_AT_type
-; CHECK-NEXT:.b8 15                                  // Abbrev [15] 0x844:0x6 DW_TAG_subprogram
-; CHECK-NEXT:.b32 1539                               // DW_AT_specification
-; CHECK-NEXT:.b8 1                                   // DW_AT_inline
-; CHECK-NEXT:.b8 21                                  // Abbrev [21] 0x84a:0x32 DW_TAG_subprogram
-; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
-; CHECK-NEXT:.b8 90
-; CHECK-NEXT:.b8 51
-; CHECK-NEXT:.b8 114
-; CHECK-NEXT:.b8 101
-; CHECK-NEXT:.b8 115
-; CHECK-NEXT:.b8 102
-; CHECK-NEXT:.b8 102
-; CHECK-NEXT:.b8 80
-; CHECK-NEXT:.b8 102
-; CHECK-NEXT:.b8 0
-; CHECK-NEXT:.b8 114                                 // DW_AT_name
-; CHECK-NEXT:.b8 101
-; CHECK-NEXT:.b8 115
-; CHECK-NEXT:.b8 0
-; CHECK-NEXT:.b8 1                                   // DW_AT_decl_file
-; CHECK-NEXT:.b8 3                                   // DW_AT_decl_line
-; CHECK-NEXT:.b8 1                                   // DW_AT_external
-; CHECK-NEXT:.b8 1                                   // DW_AT_inline
-; CHECK-NEXT:.b8 22                                  // Abbrev [22] 0x85e:0x9 DW_TAG_formal_parameter
-; CHECK-NEXT:.b8 120                                 // DW_AT_name
-; CHECK-NEXT:.b8 0
-; CHECK-NEXT:.b8 1                                   // DW_AT_decl_file
-; CHECK-NEXT:.b8 3                                   // DW_AT_decl_line
-; CHECK-NEXT:.b32 2172                               // DW_AT_type
-; CHECK-NEXT:.b8 22                                  // Abbrev [22] 0x867:0x9 DW_TAG_formal_parameter
-; CHECK-NEXT:.b8 121                                 // DW_AT_name
-; CHECK-NEXT:.b8 0
-; CHECK-NEXT:.b8 1                                   // DW_AT_decl_file
-; CHECK-NEXT:.b8 3                                   // DW_AT_decl_line
-; CHECK-NEXT:.b32 2172                               // DW_AT_type
-; CHECK-NEXT:.b8 22                                  // Abbrev [22] 0x870:0xb DW_TAG_formal_parameter
-; CHECK-NEXT:.b8 114                                 // DW_AT_name
-; CHECK-NEXT:.b8 101
-; CHECK-NEXT:.b8 115
-; CHECK-NEXT:.b8 0
-; CHECK-NEXT:.b8 1                                   // DW_AT_decl_file
-; CHECK-NEXT:.b8 3                                   // DW_AT_decl_line
-; CHECK-NEXT:.b32 2181                               // DW_AT_type
-; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 10                                  // Abbrev [10] 0x87c:0x9 DW_TAG_base_type
-; CHECK-NEXT:.b8 102                                 // DW_AT_name
-; CHECK-NEXT:.b8 108
-; CHECK-NEXT:.b8 111
-; CHECK-NEXT:.b8 97
-; CHECK-NEXT:.b8 116
-; CHECK-NEXT:.b8 0
-; CHECK-NEXT:.b8 4                                   // DW_AT_encoding
-; CHECK-NEXT:.b8 4                                   // DW_AT_byte_size
-; CHECK-NEXT:.b8 12                                  // Abbrev [12] 0x885:0x5 DW_TAG_pointer_type
-; CHECK-NEXT:.b32 2172                               // DW_AT_type
-; CHECK-NEXT:.b8 23                                  // Abbrev [23] 0x88a:0xd5 DW_TAG_subprogram
-; CHECK-NEXT:.b64 $L__func_begin0                    // DW_AT_low_pc
-; CHECK-NEXT:.b64 $L__func_end0                      // DW_AT_high_pc
-; CHECK-NEXT:.b8 1                                   // DW_AT_frame_base
-; CHECK-NEXT:.b8 156
-; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
-; CHECK-NEXT:.b8 90
-; CHECK-NEXT:.b8 53
-; CHECK-NEXT:.b8 115
-; CHECK-NEXT:.b8 97
-; CHECK-NEXT:.b8 120
-; CHECK-NEXT:.b8 112
-; CHECK-NEXT:.b8 121
-; CHECK-NEXT:.b8 105
-; CHECK-NEXT:.b8 102
-; CHECK-NEXT:.b8 80
-; CHECK-NEXT:.b8 102
-; CHECK-NEXT:.b8 83
-; CHECK-NEXT:.b8 95
-; CHECK-NEXT:.b8 0
-; CHECK-NEXT:.b8 115                                 // DW_AT_name
-; CHECK-NEXT:.b8 97
-; CHECK-NEXT:.b8 120
-; CHECK-NEXT:.b8 112
-; CHECK-NEXT:.b8 121
-; CHECK-NEXT:.b8 0
-; CHECK-NEXT:.b8 1                                   // DW_AT_decl_file
-; CHECK-NEXT:.b8 5                                   // DW_AT_decl_line
-; CHECK-NEXT:.b8 1                                   // DW_AT_external
-; CHECK-NEXT:.b8 24                                  // Abbrev [24] 0x8b5:0x10 DW_TAG_formal_parameter
-; CHECK-NEXT:.b8 2                                   // DW_AT_address_class
-; CHECK-NEXT:.b8 5                                   // DW_AT_location
-; CHECK-NEXT:.b8 144
-; CHECK-NEXT:.b8 178
-; CHECK-NEXT:.b8 228
-; CHECK-NEXT:.b8 149
-; CHECK-NEXT:.b8 1
-; CHECK-NEXT:.b8 110                                 // DW_AT_name
-; CHECK-NEXT:.b8 0
-; CHECK-NEXT:.b8 1                                   // DW_AT_decl_file
-; CHECK-NEXT:.b8 5                                   // DW_AT_decl_line
-; CHECK-NEXT:.b32 2456                               // DW_AT_type
-; CHECK-NEXT:.b8 25                                  // Abbrev [25] 0x8c5:0xd DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 $L__debug_loc0                     // DW_AT_location
-; CHECK-NEXT:.b8 97                                  // DW_AT_name
-; CHECK-NEXT:.b8 0
-; CHECK-NEXT:.b8 1                                   // DW_AT_decl_file
-; CHECK-NEXT:.b8 5                                   // DW_AT_decl_line
-; CHECK-NEXT:.b32 2172                               // DW_AT_type
-; CHECK-NEXT:.b8 22                                  // Abbrev [22] 0x8d2:0x9 DW_TAG_formal_parameter
-; CHECK-NEXT:.b8 120                                 // DW_AT_name
-; CHECK-NEXT:.b8 0
-; CHECK-NEXT:.b8 1                                   // DW_AT_decl_file
-; CHECK-NEXT:.b8 5                                   // DW_AT_decl_line
-; CHECK-NEXT:.b32 2181                               // DW_AT_type
-; CHECK-NEXT:.b8 22                                  // Abbrev [22] 0x8db:0x9 DW_TAG_formal_parameter
-; CHECK-NEXT:.b8 121                                 // DW_AT_name
-; CHECK-NEXT:.b8 0
-; CHECK-NEXT:.b8 1                                   // DW_AT_decl_file
-; CHECK-NEXT:.b8 5                                   // DW_AT_decl_line
-; CHECK-NEXT:.b32 2181                               // DW_AT_type
-; CHECK-NEXT:.b8 26                                  // Abbrev [26] 0x8e4:0xd DW_TAG_variable
-; CHECK-NEXT:.b32 $L__debug_loc1                     // DW_AT_location
-; CHECK-NEXT:.b8 105                                 // DW_AT_name
-; CHECK-NEXT:.b8 0
-; CHECK-NEXT:.b8 1                                   // DW_AT_decl_file
-; CHECK-NEXT:.b8 6                                   // DW_AT_decl_line
-; CHECK-NEXT:.b32 2456                               // DW_AT_type
-; CHECK-NEXT:.b8 27                                  // Abbrev [27] 0x8f1:0x18 DW_TAG_inlined_subroutine
-; CHECK-NEXT:.b32 711                                // DW_AT_abstract_origin
-; CHECK-NEXT:.b64 $L__tmp1                           // DW_AT_low_pc
-; CHECK-NEXT:.b64 $L__tmp2                           // DW_AT_high_pc
-; CHECK-NEXT:.b8 1                                   // DW_AT_call_file
-; CHECK-NEXT:.b8 6                                   // DW_AT_call_line
-; CHECK-NEXT:.b8 11                                  // DW_AT_call_column
-; CHECK-NEXT:.b8 27                                  // Abbrev [27] 0x909:0x18 DW_TAG_inlined_subroutine
-; CHECK-NEXT:.b32 1502                               // DW_AT_abstract_origin
-; CHECK-NEXT:.b64 $L__tmp2                           // DW_AT_low_pc
-; CHECK-NEXT:.b64 $L__tmp3                           // DW_AT_high_pc
-; CHECK-NEXT:.b8 1                                   // DW_AT_call_file
-; CHECK-NEXT:.b8 6                                   // DW_AT_call_line
-; CHECK-NEXT:.b8 24                                  // DW_AT_call_column
-; CHECK-NEXT:.b8 27                                  // Abbrev [27] 0x921:0x18 DW_TAG_inlined_subroutine
-; CHECK-NEXT:.b32 2116                               // DW_AT_abstract_origin
-; CHECK-NEXT:.b64 $L__tmp3                           // DW_AT_low_pc
-; CHECK-NEXT:.b64 $L__tmp4                           // DW_AT_high_pc
-; CHECK-NEXT:.b8 1                                   // DW_AT_call_file
-; CHECK-NEXT:.b8 6                                   // DW_AT_call_line
-; CHECK-NEXT:.b8 37                                  // DW_AT_call_column
-; CHECK-NEXT:.b8 28                                  // Abbrev [28] 0x939:0x25 DW_TAG_inlined_subroutine
-; CHECK-NEXT:.b32 2122                               // DW_AT_abstract_origin
-; CHECK-NEXT:.b64 $L__tmp9                           // DW_AT_low_pc
-; CHECK-NEXT:.b64 $L__tmp10                          // DW_AT_high_pc
-; CHECK-NEXT:.b8 1                                   // DW_AT_call_file
-; CHECK-NEXT:.b8 8                                   // DW_AT_call_line
-; CHECK-NEXT:.b8 5                                   // DW_AT_call_column
-; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0x951:0xc DW_TAG_formal_parameter
-; CHECK-NEXT:.b8 2                                   // DW_AT_address_class
-; CHECK-NEXT:.b8 5                                   // DW_AT_location
-; CHECK-NEXT:.b8 144
-; CHECK-NEXT:.b8 179
-; CHECK-NEXT:.b8 204
-; CHECK-NEXT:.b8 149
-; CHECK-NEXT:.b8 1
-; CHECK-NEXT:.b32 2151                               // DW_AT_abstract_origin
-; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0x95f:0xd DW_TAG_namespace
-; CHECK-NEXT:.b8 115                                 // DW_AT_name
-; CHECK-NEXT:.b8 116
-; CHECK-NEXT:.b8 100
-; CHECK-NEXT:.b8 0
-; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0x964:0x7 DW_TAG_imported_declaration
-; CHECK-NEXT:.b8 4                                   // DW_AT_decl_file
-; CHECK-NEXT:.b8 202                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 2412                               // DW_AT_import
-; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 32                                  // Abbrev [32] 0x96c:0x1b DW_TAG_subprogram
-; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
-; CHECK-NEXT:.b8 90
-; CHECK-NEXT:.b8 76
-; CHECK-NEXT:.b8 51
-; CHECK-NEXT:.b8 97
-; CHECK-NEXT:.b8 98
-; CHECK-NEXT:.b8 115
-; CHECK-NEXT:.b8 120
-; CHECK-NEXT:.b8 0
-; CHECK-NEXT:.b8 97                                  // DW_AT_name
-; CHECK-NEXT:.b8 98
-; CHECK-NEXT:.b8 115
-; CHECK-NEXT:.b8 0
-; CHECK-NEXT:.b8 4                                   // DW_AT_decl_file
-; CHECK-NEXT:.b8 44                                  // DW_AT_decl_line
-; CHECK-NEXT:.b32 2439                               // DW_AT_type
-; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x981:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 2439                               // DW_AT_type
-; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 10                                  // Abbrev [10] 0x987:0x11 DW_TAG_base_type
-; CHECK-NEXT:.b8 108                                 // DW_AT_name
-; CHECK-NEXT:.b8 111
-; CHECK-NEXT:.b8 110
-; CHECK-NEXT:.b8 103
-; CHECK-NEXT:.b8 32
-; CHECK-NEXT:.b8 108
-; CHECK-NEXT:.b8 111
-; CHECK-NEXT:.b8 110
-; CHECK-NEXT:.b8 103
-; CHECK-NEXT:.b8 32
-; CHECK-NEXT:.b8 105
-; CHECK-NEXT:.b8 110
-; CHECK-NEXT:.b8 116
-; CHECK-NEXT:.b8 0
-; CHECK-NEXT:.b8 5                                   // DW_AT_encoding
-; CHECK-NEXT:.b8 8                                   // DW_AT_byte_size
-; CHECK-NEXT:.b8 10                                  // Abbrev [10] 0x998:0x7 DW_TAG_base_type
-; CHECK-NEXT:.b8 105                                 // DW_AT_name
-; CHECK-NEXT:.b8 110
-; CHECK-NEXT:.b8 116
-; CHECK-NEXT:.b8 0
-; CHECK-NEXT:.b8 5                                   // DW_AT_encoding
-; CHECK-NEXT:.b8 4                                   // DW_AT_byte_size
-; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:	}
-; CHECK-NEXT:	.section	.debug_macinfo	{	}
+; CHECK-NEXT: 	{
+; CHECK-NEXT: $L__debug_loc0:
+; CHECK-NEXT: .b64 $L__tmp8
+; CHECK-NEXT: .b64 $L__tmp10
+; CHECK-NEXT: .b8 5                                   // Loc expr size
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 144                                 // DW_OP_regx
+; CHECK-NEXT: .b8 177                                 // 2450993
+; CHECK-NEXT: .b8 204                                 // 
+; CHECK-NEXT: .b8 149                                 // 
+; CHECK-NEXT: .b8 1                                   // 
+; CHECK-NEXT: .b64 0
+; CHECK-NEXT: .b64 0
+; CHECK-NEXT: $L__debug_loc1:
+; CHECK-NEXT: .b64 $L__tmp5
+; CHECK-NEXT: .b64 $L__func_end0
+; CHECK-NEXT: .b8 5                                   // Loc expr size
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 144                                 // DW_OP_regx
+; CHECK-NEXT: .b8 177                                 // 2454065
+; CHECK-NEXT: .b8 228                                 // 
+; CHECK-NEXT: .b8 149                                 // 
+; CHECK-NEXT: .b8 1                                   // 
+; CHECK-NEXT: .b64 0
+; CHECK-NEXT: .b64 0
+; CHECK-NEXT: 	}
+; CHECK-NEXT: 	.section	.debug_abbrev
+; CHECK-NEXT: 	{
+; CHECK-NEXT: .b8 1                                   // Abbreviation Code
+; CHECK-NEXT: .b8 17                                  // DW_TAG_compile_unit
+; CHECK-NEXT: .b8 1                                   // DW_CHILDREN_yes
+; CHECK-NEXT: .b8 37                                  // DW_AT_producer
+; CHECK-NEXT: .b8 8                                   // DW_FORM_string
+; CHECK-NEXT: .b8 19                                  // DW_AT_language
+; CHECK-NEXT: .b8 5                                   // DW_FORM_data2
+; CHECK-NEXT: .b8 3                                   // DW_AT_name
+; CHECK-NEXT: .b8 8                                   // DW_FORM_string
+; CHECK-NEXT: .b8 16                                  // DW_AT_stmt_list
+; CHECK-NEXT: .b8 6                                   // DW_FORM_data4
+; CHECK-NEXT: .b8 27                                  // DW_AT_comp_dir
+; CHECK-NEXT: .b8 8                                   // DW_FORM_string
+; CHECK-NEXT: .b8 0                                   // EOM(1)
+; CHECK-NEXT: .b8 0                                   // EOM(2)
+; CHECK-NEXT: .b8 2                                   // Abbreviation Code
+; CHECK-NEXT: .b8 19                                  // DW_TAG_structure_type
+; CHECK-NEXT: .b8 1                                   // DW_CHILDREN_yes
+; CHECK-NEXT: .b8 3                                   // DW_AT_name
+; CHECK-NEXT: .b8 8                                   // DW_FORM_string
+; CHECK-NEXT: .b8 11                                  // DW_AT_byte_size
+; CHECK-NEXT: .b8 11                                  // DW_FORM_data1
+; CHECK-NEXT: .b8 58                                  // DW_AT_decl_file
+; CHECK-NEXT: .b8 11                                  // DW_FORM_data1
+; CHECK-NEXT: .b8 59                                  // DW_AT_decl_line
+; CHECK-NEXT: .b8 11                                  // DW_FORM_data1
+; CHECK-NEXT: .b8 0                                   // EOM(1)
+; CHECK-NEXT: .b8 0                                   // EOM(2)
+; CHECK-NEXT: .b8 3                                   // Abbreviation Code
+; CHECK-NEXT: .b8 46                                  // DW_TAG_subprogram
+; CHECK-NEXT: .b8 0                                   // DW_CHILDREN_no
+; CHECK-NEXT: .b8 135                                 // DW_AT_MIPS_linkage_name
+; CHECK-NEXT: .b8 64
+; CHECK-NEXT: .b8 8                                   // DW_FORM_string
+; CHECK-NEXT: .b8 3                                   // DW_AT_name
+; CHECK-NEXT: .b8 8                                   // DW_FORM_string
+; CHECK-NEXT: .b8 58                                  // DW_AT_decl_file
+; CHECK-NEXT: .b8 11                                  // DW_FORM_data1
+; CHECK-NEXT: .b8 59                                  // DW_AT_decl_line
+; CHECK-NEXT: .b8 11                                  // DW_FORM_data1
+; CHECK-NEXT: .b8 73                                  // DW_AT_type
+; CHECK-NEXT: .b8 19                                  // DW_FORM_ref4
+; CHECK-NEXT: .b8 60                                  // DW_AT_declaration
+; CHECK-NEXT: .b8 12                                  // DW_FORM_flag
+; CHECK-NEXT: .b8 63                                  // DW_AT_external
+; CHECK-NEXT: .b8 12                                  // DW_FORM_flag
+; CHECK-NEXT: .b8 0                                   // EOM(1)
+; CHECK-NEXT: .b8 0                                   // EOM(2)
+; CHECK-NEXT: .b8 4                                   // Abbreviation Code
+; CHECK-NEXT: .b8 46                                  // DW_TAG_subprogram
+; CHECK-NEXT: .b8 1                                   // DW_CHILDREN_yes
+; CHECK-NEXT: .b8 135                                 // DW_AT_MIPS_linkage_name
+; CHECK-NEXT: .b8 64
+; CHECK-NEXT: .b8 8                                   // DW_FORM_string
+; CHECK-NEXT: .b8 3                                   // DW_AT_name
+; CHECK-NEXT: .b8 8                                   // DW_FORM_string
+; CHECK-NEXT: .b8 58                                  // DW_AT_decl_file
+; CHECK-NEXT: .b8 11                                  // DW_FORM_data1
+; CHECK-NEXT: .b8 59                                  // DW_AT_decl_line
+; CHECK-NEXT: .b8 11                                  // DW_FORM_data1
+; CHECK-NEXT: .b8 73                                  // DW_AT_type
+; CHECK-NEXT: .b8 19                                  // DW_FORM_ref4
+; CHECK-NEXT: .b8 60                                  // DW_AT_declaration
+; CHECK-NEXT: .b8 12                                  // DW_FORM_flag
+; CHECK-NEXT: .b8 63                                  // DW_AT_external
+; CHECK-NEXT: .b8 12                                  // DW_FORM_flag
+; CHECK-NEXT: .b8 0                                   // EOM(1)
+; CHECK-NEXT: .b8 0                                   // EOM(2)
+; CHECK-NEXT: .b8 5                                   // Abbreviation Code
+; CHECK-NEXT: .b8 5                                   // DW_TAG_formal_parameter
+; CHECK-NEXT: .b8 0                                   // DW_CHILDREN_no
+; CHECK-NEXT: .b8 73                                  // DW_AT_type
+; CHECK-NEXT: .b8 19                                  // DW_FORM_ref4
+; CHECK-NEXT: .b8 52                                  // DW_AT_artificial
+; CHECK-NEXT: .b8 12                                  // DW_FORM_flag
+; CHECK-NEXT: .b8 0                                   // EOM(1)
+; CHECK-NEXT: .b8 0                                   // EOM(2)
+; CHECK-NEXT: .b8 6                                   // Abbreviation Code
+; CHECK-NEXT: .b8 46                                  // DW_TAG_subprogram
+; CHECK-NEXT: .b8 1                                   // DW_CHILDREN_yes
+; CHECK-NEXT: .b8 3                                   // DW_AT_name
+; CHECK-NEXT: .b8 8                                   // DW_FORM_string
+; CHECK-NEXT: .b8 58                                  // DW_AT_decl_file
+; CHECK-NEXT: .b8 11                                  // DW_FORM_data1
+; CHECK-NEXT: .b8 59                                  // DW_AT_decl_line
+; CHECK-NEXT: .b8 11                                  // DW_FORM_data1
+; CHECK-NEXT: .b8 60                                  // DW_AT_declaration
+; CHECK-NEXT: .b8 12                                  // DW_FORM_flag
+; CHECK-NEXT: .b8 63                                  // DW_AT_external
+; CHECK-NEXT: .b8 12                                  // DW_FORM_flag
+; CHECK-NEXT: .b8 50                                  // DW_AT_accessibility
+; CHECK-NEXT: .b8 11                                  // DW_FORM_data1
+; CHECK-NEXT: .b8 0                                   // EOM(1)
+; CHECK-NEXT: .b8 0                                   // EOM(2)
+; CHECK-NEXT: .b8 7                                   // Abbreviation Code
+; CHECK-NEXT: .b8 5                                   // DW_TAG_formal_parameter
+; CHECK-NEXT: .b8 0                                   // DW_CHILDREN_no
+; CHECK-NEXT: .b8 73                                  // DW_AT_type
+; CHECK-NEXT: .b8 19                                  // DW_FORM_ref4
+; CHECK-NEXT: .b8 0                                   // EOM(1)
+; CHECK-NEXT: .b8 0                                   // EOM(2)
+; CHECK-NEXT: .b8 8                                   // Abbreviation Code
+; CHECK-NEXT: .b8 46                                  // DW_TAG_subprogram
+; CHECK-NEXT: .b8 1                                   // DW_CHILDREN_yes
+; CHECK-NEXT: .b8 135                                 // DW_AT_MIPS_linkage_name
+; CHECK-NEXT: .b8 64
+; CHECK-NEXT: .b8 8                                   // DW_FORM_string
+; CHECK-NEXT: .b8 3                                   // DW_AT_name
+; CHECK-NEXT: .b8 8                                   // DW_FORM_string
+; CHECK-NEXT: .b8 58                                  // DW_AT_decl_file
+; CHECK-NEXT: .b8 11                                  // DW_FORM_data1
+; CHECK-NEXT: .b8 59                                  // DW_AT_decl_line
+; CHECK-NEXT: .b8 11                                  // DW_FORM_data1
+; CHECK-NEXT: .b8 60                                  // DW_AT_declaration
+; CHECK-NEXT: .b8 12                                  // DW_FORM_flag
+; CHECK-NEXT: .b8 63                                  // DW_AT_external
+; CHECK-NEXT: .b8 12                                  // DW_FORM_flag
+; CHECK-NEXT: .b8 50                                  // DW_AT_accessibility
+; CHECK-NEXT: .b8 11                                  // DW_FORM_data1
+; CHECK-NEXT: .b8 0                                   // EOM(1)
+; CHECK-NEXT: .b8 0                                   // EOM(2)
+; CHECK-NEXT: .b8 9                                   // Abbreviation Code
+; CHECK-NEXT: .b8 46                                  // DW_TAG_subprogram
+; CHECK-NEXT: .b8 1                                   // DW_CHILDREN_yes
+; CHECK-NEXT: .b8 135                                 // DW_AT_MIPS_linkage_name
+; CHECK-NEXT: .b8 64
+; CHECK-NEXT: .b8 8                                   // DW_FORM_string
+; CHECK-NEXT: .b8 3                                   // DW_AT_name
+; CHECK-NEXT: .b8 8                                   // DW_FORM_string
+; CHECK-NEXT: .b8 58                                  // DW_AT_decl_file
+; CHECK-NEXT: .b8 11                                  // DW_FORM_data1
+; CHECK-NEXT: .b8 59                                  // DW_AT_decl_line
+; CHECK-NEXT: .b8 11                                  // DW_FORM_data1
+; CHECK-NEXT: .b8 73                                  // DW_AT_type
+; CHECK-NEXT: .b8 19                                  // DW_FORM_ref4
+; CHECK-NEXT: .b8 60                                  // DW_AT_declaration
+; CHECK-NEXT: .b8 12                                  // DW_FORM_flag
+; CHECK-NEXT: .b8 63                                  // DW_AT_external
+; CHECK-NEXT: .b8 12                                  // DW_FORM_flag
+; CHECK-NEXT: .b8 50                                  // DW_AT_accessibility
+; CHECK-NEXT: .b8 11                                  // DW_FORM_data1
+; CHECK-NEXT: .b8 0                                   // EOM(1)
+; CHECK-NEXT: .b8 0                                   // EOM(2)
+; CHECK-NEXT: .b8 10                                  // Abbreviation Code
+; CHECK-NEXT: .b8 36                                  // DW_TAG_base_type
+; CHECK-NEXT: .b8 0                                   // DW_CHILDREN_no
+; CHECK-NEXT: .b8 3                                   // DW_AT_name
+; CHECK-NEXT: .b8 8                                   // DW_FORM_string
+; CHECK-NEXT: .b8 62                                  // DW_AT_encoding
+; CHECK-NEXT: .b8 11                                  // DW_FORM_data1
+; CHECK-NEXT: .b8 11                                  // DW_AT_byte_size
+; CHECK-NEXT: .b8 11                                  // DW_FORM_data1
+; CHECK-NEXT: .b8 0                                   // EOM(1)
+; CHECK-NEXT: .b8 0                                   // EOM(2)
+; CHECK-NEXT: .b8 11                                  // Abbreviation Code
+; CHECK-NEXT: .b8 13                                  // DW_TAG_member
+; CHECK-NEXT: .b8 0                                   // DW_CHILDREN_no
+; CHECK-NEXT: .b8 3                                   // DW_AT_name
+; CHECK-NEXT: .b8 8                                   // DW_FORM_string
+; CHECK-NEXT: .b8 73                                  // DW_AT_type
+; CHECK-NEXT: .b8 19                                  // DW_FORM_ref4
+; CHECK-NEXT: .b8 58                                  // DW_AT_decl_file
+; CHECK-NEXT: .b8 11                                  // DW_FORM_data1
+; CHECK-NEXT: .b8 59                                  // DW_AT_decl_line
+; CHECK-NEXT: .b8 11                                  // DW_FORM_data1
+; CHECK-NEXT: .b8 56                                  // DW_AT_data_member_location
+; CHECK-NEXT: .b8 10                                  // DW_FORM_block1
+; CHECK-NEXT: .b8 0                                   // EOM(1)
+; CHECK-NEXT: .b8 0                                   // EOM(2)
+; CHECK-NEXT: .b8 12                                  // Abbreviation Code
+; CHECK-NEXT: .b8 15                                  // DW_TAG_pointer_type
+; CHECK-NEXT: .b8 0                                   // DW_CHILDREN_no
+; CHECK-NEXT: .b8 73                                  // DW_AT_type
+; CHECK-NEXT: .b8 19                                  // DW_FORM_ref4
+; CHECK-NEXT: .b8 0                                   // EOM(1)
+; CHECK-NEXT: .b8 0                                   // EOM(2)
+; CHECK-NEXT: .b8 13                                  // Abbreviation Code
+; CHECK-NEXT: .b8 38                                  // DW_TAG_const_type
+; CHECK-NEXT: .b8 0                                   // DW_CHILDREN_no
+; CHECK-NEXT: .b8 73                                  // DW_AT_type
+; CHECK-NEXT: .b8 19                                  // DW_FORM_ref4
+; CHECK-NEXT: .b8 0                                   // EOM(1)
+; CHECK-NEXT: .b8 0                                   // EOM(2)
+; CHECK-NEXT: .b8 14                                  // Abbreviation Code
+; CHECK-NEXT: .b8 16                                  // DW_TAG_reference_type
+; CHECK-NEXT: .b8 0                                   // DW_CHILDREN_no
+; CHECK-NEXT: .b8 73                                  // DW_AT_type
+; CHECK-NEXT: .b8 19                                  // DW_FORM_ref4
+; CHECK-NEXT: .b8 0                                   // EOM(1)
+; CHECK-NEXT: .b8 0                                   // EOM(2)
+; CHECK-NEXT: .b8 15                                  // Abbreviation Code
+; CHECK-NEXT: .b8 46                                  // DW_TAG_subprogram
+; CHECK-NEXT: .b8 0                                   // DW_CHILDREN_no
+; CHECK-NEXT: .b8 71                                  // DW_AT_specification
+; CHECK-NEXT: .b8 19                                  // DW_FORM_ref4
+; CHECK-NEXT: .b8 32                                  // DW_AT_inline
+; CHECK-NEXT: .b8 11                                  // DW_FORM_data1
+; CHECK-NEXT: .b8 0                                   // EOM(1)
+; CHECK-NEXT: .b8 0                                   // EOM(2)
+; CHECK-NEXT: .b8 16                                  // Abbreviation Code
+; CHECK-NEXT: .b8 19                                  // DW_TAG_structure_type
+; CHECK-NEXT: .b8 1                                   // DW_CHILDREN_yes
+; CHECK-NEXT: .b8 3                                   // DW_AT_name
+; CHECK-NEXT: .b8 8                                   // DW_FORM_string
+; CHECK-NEXT: .b8 11                                  // DW_AT_byte_size
+; CHECK-NEXT: .b8 11                                  // DW_FORM_data1
+; CHECK-NEXT: .b8 58                                  // DW_AT_decl_file
+; CHECK-NEXT: .b8 11                                  // DW_FORM_data1
+; CHECK-NEXT: .b8 59                                  // DW_AT_decl_line
+; CHECK-NEXT: .b8 5                                   // DW_FORM_data2
+; CHECK-NEXT: .b8 0                                   // EOM(1)
+; CHECK-NEXT: .b8 0                                   // EOM(2)
+; CHECK-NEXT: .b8 17                                  // Abbreviation Code
+; CHECK-NEXT: .b8 13                                  // DW_TAG_member
+; CHECK-NEXT: .b8 0                                   // DW_CHILDREN_no
+; CHECK-NEXT: .b8 3                                   // DW_AT_name
+; CHECK-NEXT: .b8 8                                   // DW_FORM_string
+; CHECK-NEXT: .b8 73                                  // DW_AT_type
+; CHECK-NEXT: .b8 19                                  // DW_FORM_ref4
+; CHECK-NEXT: .b8 58                                  // DW_AT_decl_file
+; CHECK-NEXT: .b8 11                                  // DW_FORM_data1
+; CHECK-NEXT: .b8 59                                  // DW_AT_decl_line
+; CHECK-NEXT: .b8 5                                   // DW_FORM_data2
+; CHECK-NEXT: .b8 56                                  // DW_AT_data_member_location
+; CHECK-NEXT: .b8 10                                  // DW_FORM_block1
+; CHECK-NEXT: .b8 0                                   // EOM(1)
+; CHECK-NEXT: .b8 0                                   // EOM(2)
+; CHECK-NEXT: .b8 18                                  // Abbreviation Code
+; CHECK-NEXT: .b8 46                                  // DW_TAG_subprogram
+; CHECK-NEXT: .b8 1                                   // DW_CHILDREN_yes
+; CHECK-NEXT: .b8 3                                   // DW_AT_name
+; CHECK-NEXT: .b8 8                                   // DW_FORM_string
+; CHECK-NEXT: .b8 58                                  // DW_AT_decl_file
+; CHECK-NEXT: .b8 11                                  // DW_FORM_data1
+; CHECK-NEXT: .b8 59                                  // DW_AT_decl_line
+; CHECK-NEXT: .b8 5                                   // DW_FORM_data2
+; CHECK-NEXT: .b8 60                                  // DW_AT_declaration
+; CHECK-NEXT: .b8 12                                  // DW_FORM_flag
+; CHECK-NEXT: .b8 63                                  // DW_AT_external
+; CHECK-NEXT: .b8 12                                  // DW_FORM_flag
+; CHECK-NEXT: .b8 0                                   // EOM(1)
+; CHECK-NEXT: .b8 0                                   // EOM(2)
+; CHECK-NEXT: .b8 19                                  // Abbreviation Code
+; CHECK-NEXT: .b8 46                                  // DW_TAG_subprogram
+; CHECK-NEXT: .b8 1                                   // DW_CHILDREN_yes
+; CHECK-NEXT: .b8 135                                 // DW_AT_MIPS_linkage_name
+; CHECK-NEXT: .b8 64
+; CHECK-NEXT: .b8 8                                   // DW_FORM_string
+; CHECK-NEXT: .b8 3                                   // DW_AT_name
+; CHECK-NEXT: .b8 8                                   // DW_FORM_string
+; CHECK-NEXT: .b8 58                                  // DW_AT_decl_file
+; CHECK-NEXT: .b8 11                                  // DW_FORM_data1
+; CHECK-NEXT: .b8 59                                  // DW_AT_decl_line
+; CHECK-NEXT: .b8 5                                   // DW_FORM_data2
+; CHECK-NEXT: .b8 73                                  // DW_AT_type
+; CHECK-NEXT: .b8 19                                  // DW_FORM_ref4
+; CHECK-NEXT: .b8 60                                  // DW_AT_declaration
+; CHECK-NEXT: .b8 12                                  // DW_FORM_flag
+; CHECK-NEXT: .b8 63                                  // DW_AT_external
+; CHECK-NEXT: .b8 12                                  // DW_FORM_flag
+; CHECK-NEXT: .b8 0                                   // EOM(1)
+; CHECK-NEXT: .b8 0                                   // EOM(2)
+; CHECK-NEXT: .b8 20                                  // Abbreviation Code
+; CHECK-NEXT: .b8 22                                  // DW_TAG_typedef
+; CHECK-NEXT: .b8 0                                   // DW_CHILDREN_no
+; CHECK-NEXT: .b8 73                                  // DW_AT_type
+; CHECK-NEXT: .b8 19                                  // DW_FORM_ref4
+; CHECK-NEXT: .b8 3                                   // DW_AT_name
+; CHECK-NEXT: .b8 8                                   // DW_FORM_string
+; CHECK-NEXT: .b8 58                                  // DW_AT_decl_file
+; CHECK-NEXT: .b8 11                                  // DW_FORM_data1
+; CHECK-NEXT: .b8 59                                  // DW_AT_decl_line
+; CHECK-NEXT: .b8 5                                   // DW_FORM_data2
+; CHECK-NEXT: .b8 0                                   // EOM(1)
+; CHECK-NEXT: .b8 0                                   // EOM(2)
+; CHECK-NEXT: .b8 21                                  // Abbreviation Code
+; CHECK-NEXT: .b8 46                                  // DW_TAG_subprogram
+; CHECK-NEXT: .b8 1                                   // DW_CHILDREN_yes
+; CHECK-NEXT: .b8 135                                 // DW_AT_MIPS_linkage_name
+; CHECK-NEXT: .b8 64
+; CHECK-NEXT: .b8 8                                   // DW_FORM_string
+; CHECK-NEXT: .b8 3                                   // DW_AT_name
+; CHECK-NEXT: .b8 8                                   // DW_FORM_string
+; CHECK-NEXT: .b8 58                                  // DW_AT_decl_file
+; CHECK-NEXT: .b8 11                                  // DW_FORM_data1
+; CHECK-NEXT: .b8 59                                  // DW_AT_decl_line
+; CHECK-NEXT: .b8 11                                  // DW_FORM_data1
+; CHECK-NEXT: .b8 63                                  // DW_AT_external
+; CHECK-NEXT: .b8 12                                  // DW_FORM_flag
+; CHECK-NEXT: .b8 32                                  // DW_AT_inline
+; CHECK-NEXT: .b8 11                                  // DW_FORM_data1
+; CHECK-NEXT: .b8 0                                   // EOM(1)
+; CHECK-NEXT: .b8 0                                   // EOM(2)
+; CHECK-NEXT: .b8 22                                  // Abbreviation Code
+; CHECK-NEXT: .b8 5                                   // DW_TAG_formal_parameter
+; CHECK-NEXT: .b8 0                                   // DW_CHILDREN_no
+; CHECK-NEXT: .b8 3                                   // DW_AT_name
+; CHECK-NEXT: .b8 8                                   // DW_FORM_string
+; CHECK-NEXT: .b8 58                                  // DW_AT_decl_file
+; CHECK-NEXT: .b8 11                                  // DW_FORM_data1
+; CHECK-NEXT: .b8 59                                  // DW_AT_decl_line
+; CHECK-NEXT: .b8 11                                  // DW_FORM_data1
+; CHECK-NEXT: .b8 73                                  // DW_AT_type
+; CHECK-NEXT: .b8 19                                  // DW_FORM_ref4
+; CHECK-NEXT: .b8 0                                   // EOM(1)
+; CHECK-NEXT: .b8 0                                   // EOM(2)
+; CHECK-NEXT: .b8 23                                  // Abbreviation Code
+; CHECK-NEXT: .b8 46                                  // DW_TAG_subprogram
+; CHECK-NEXT: .b8 1                                   // DW_CHILDREN_yes
+; CHECK-NEXT: .b8 17                                  // DW_AT_low_pc
+; CHECK-NEXT: .b8 1                                   // DW_FORM_addr
+; CHECK-NEXT: .b8 18                                  // DW_AT_high_pc
+; CHECK-NEXT: .b8 1                                   // DW_FORM_addr
+; CHECK-NEXT: .b8 64                                  // DW_AT_frame_base
+; CHECK-NEXT: .b8 10                                  // DW_FORM_block1
+; CHECK-NEXT: .b8 135                                 // DW_AT_MIPS_linkage_name
+; CHECK-NEXT: .b8 64
+; CHECK-NEXT: .b8 8                                   // DW_FORM_string
+; CHECK-NEXT: .b8 3                                   // DW_AT_name
+; CHECK-NEXT: .b8 8                                   // DW_FORM_string
+; CHECK-NEXT: .b8 58                                  // DW_AT_decl_file
+; CHECK-NEXT: .b8 11                                  // DW_FORM_data1
+; CHECK-NEXT: .b8 59                                  // DW_AT_decl_line
+; CHECK-NEXT: .b8 11                                  // DW_FORM_data1
+; CHECK-NEXT: .b8 63                                  // DW_AT_external
+; CHECK-NEXT: .b8 12                                  // DW_FORM_flag
+; CHECK-NEXT: .b8 0                                   // EOM(1)
+; CHECK-NEXT: .b8 0                                   // EOM(2)
+; CHECK-NEXT: .b8 24                                  // Abbreviation Code
+; CHECK-NEXT: .b8 5                                   // DW_TAG_formal_parameter
+; CHECK-NEXT: .b8 0                                   // DW_CHILDREN_no
+; CHECK-NEXT: .b8 51                                  // DW_AT_address_class
+; CHECK-NEXT: .b8 11                                  // DW_FORM_data1
+; CHECK-NEXT: .b8 2                                   // DW_AT_location
+; CHECK-NEXT: .b8 10                                  // DW_FORM_block1
+; CHECK-NEXT: .b8 3                                   // DW_AT_name
+; CHECK-NEXT: .b8 8                                   // DW_FORM_string
+; CHECK-NEXT: .b8 58                                  // DW_AT_decl_file
+; CHECK-NEXT: .b8 11                                  // DW_FORM_data1
+; CHECK-NEXT: .b8 59                                  // DW_AT_decl_line
+; CHECK-NEXT: .b8 11                                  // DW_FORM_data1
+; CHECK-NEXT: .b8 73                                  // DW_AT_type
+; CHECK-NEXT: .b8 19                                  // DW_FORM_ref4
+; CHECK-NEXT: .b8 0                                   // EOM(1)
+; CHECK-NEXT: .b8 0                                   // EOM(2)
+; CHECK-NEXT: .b8 25                                  // Abbreviation Code
+; CHECK-NEXT: .b8 5                                   // DW_TAG_formal_parameter
+; CHECK-NEXT: .b8 0                                   // DW_CHILDREN_no
+; CHECK-NEXT: .b8 2                                   // DW_AT_location
+; CHECK-NEXT: .b8 6                                   // DW_FORM_data4
+; CHECK-NEXT: .b8 3                                   // DW_AT_name
+; CHECK-NEXT: .b8 8                                   // DW_FORM_string
+; CHECK-NEXT: .b8 58                                  // DW_AT_decl_file
+; CHECK-NEXT: .b8 11                                  // DW_FORM_data1
+; CHECK-NEXT: .b8 59                                  // DW_AT_decl_line
+; CHECK-NEXT: .b8 11                                  // DW_FORM_data1
+; CHECK-NEXT: .b8 73                                  // DW_AT_type
+; CHECK-NEXT: .b8 19                                  // DW_FORM_ref4
+; CHECK-NEXT: .b8 0                                   // EOM(1)
+; CHECK-NEXT: .b8 0                                   // EOM(2)
+; CHECK-NEXT: .b8 26                                  // Abbreviation Code
+; CHECK-NEXT: .b8 52                                  // DW_TAG_variable
+; CHECK-NEXT: .b8 0                                   // DW_CHILDREN_no
+; CHECK-NEXT: .b8 2                                   // DW_AT_location
+; CHECK-NEXT: .b8 6                                   // DW_FORM_data4
+; CHECK-NEXT: .b8 3                                   // DW_AT_name
+; CHECK-NEXT: .b8 8                                   // DW_FORM_string
+; CHECK-NEXT: .b8 58                                  // DW_AT_decl_file
+; CHECK-NEXT: .b8 11                                  // DW_FORM_data1
+; CHECK-NEXT: .b8 59                                  // DW_AT_decl_line
+; CHECK-NEXT: .b8 11                                  // DW_FORM_data1
+; CHECK-NEXT: .b8 73                                  // DW_AT_type
+; CHECK-NEXT: .b8 19                                  // DW_FORM_ref4
+; CHECK-NEXT: .b8 0                                   // EOM(1)
+; CHECK-NEXT: .b8 0                                   // EOM(2)
+; CHECK-NEXT: .b8 27                                  // Abbreviation Code
+; CHECK-NEXT: .b8 29                                  // DW_TAG_inlined_subroutine
+; CHECK-NEXT: .b8 0                                   // DW_CHILDREN_no
+; CHECK-NEXT: .b8 49                                  // DW_AT_abstract_origin
+; CHECK-NEXT: .b8 19                                  // DW_FORM_ref4
+; CHECK-NEXT: .b8 17                                  // DW_AT_low_pc
+; CHECK-NEXT: .b8 1                                   // DW_FORM_addr
+; CHECK-NEXT: .b8 18                                  // DW_AT_high_pc
+; CHECK-NEXT: .b8 1                                   // DW_FORM_addr
+; CHECK-NEXT: .b8 88                                  // DW_AT_call_file
+; CHECK-NEXT: .b8 11                                  // DW_FORM_data1
+; CHECK-NEXT: .b8 89                                  // DW_AT_call_line
+; CHECK-NEXT: .b8 11                                  // DW_FORM_data1
+; CHECK-NEXT: .b8 87                                  // DW_AT_call_column
+; CHECK-NEXT: .b8 11                                  // DW_FORM_data1
+; CHECK-NEXT: .b8 0                                   // EOM(1)
+; CHECK-NEXT: .b8 0                                   // EOM(2)
+; CHECK-NEXT: .b8 28                                  // Abbreviation Code
+; CHECK-NEXT: .b8 29                                  // DW_TAG_inlined_subroutine
+; CHECK-NEXT: .b8 1                                   // DW_CHILDREN_yes
+; CHECK-NEXT: .b8 49                                  // DW_AT_abstract_origin
+; CHECK-NEXT: .b8 19                                  // DW_FORM_ref4
+; CHECK-NEXT: .b8 17                                  // DW_AT_low_pc
+; CHECK-NEXT: .b8 1                                   // DW_FORM_addr
+; CHECK-NEXT: .b8 18                                  // DW_AT_high_pc
+; CHECK-NEXT: .b8 1                                   // DW_FORM_addr
+; CHECK-NEXT: .b8 88                                  // DW_AT_call_file
+; CHECK-NEXT: .b8 11                                  // DW_FORM_data1
+; CHECK-NEXT: .b8 89                                  // DW_AT_call_line
+; CHECK-NEXT: .b8 11                                  // DW_FORM_data1
+; CHECK-NEXT: .b8 87                                  // DW_AT_call_column
+; CHECK-NEXT: .b8 11                                  // DW_FORM_data1
+; CHECK-NEXT: .b8 0                                   // EOM(1)
+; CHECK-NEXT: .b8 0                                   // EOM(2)
+; CHECK-NEXT: .b8 29                                  // Abbreviation Code
+; CHECK-NEXT: .b8 5                                   // DW_TAG_formal_parameter
+; CHECK-NEXT: .b8 0                                   // DW_CHILDREN_no
+; CHECK-NEXT: .b8 51                                  // DW_AT_address_class
+; CHECK-NEXT: .b8 11                                  // DW_FORM_data1
+; CHECK-NEXT: .b8 2                                   // DW_AT_location
+; CHECK-NEXT: .b8 10                                  // DW_FORM_block1
+; CHECK-NEXT: .b8 49                                  // DW_AT_abstract_origin
+; CHECK-NEXT: .b8 19                                  // DW_FORM_ref4
+; CHECK-NEXT: .b8 0                                   // EOM(1)
+; CHECK-NEXT: .b8 0                                   // EOM(2)
+; CHECK-NEXT: .b8 30                                  // Abbreviation Code
+; CHECK-NEXT: .b8 57                                  // DW_TAG_namespace
+; CHECK-NEXT: .b8 1                                   // DW_CHILDREN_yes
+; CHECK-NEXT: .b8 3                                   // DW_AT_name
+; CHECK-NEXT: .b8 8                                   // DW_FORM_string
+; CHECK-NEXT: .b8 0                                   // EOM(1)
+; CHECK-NEXT: .b8 0                                   // EOM(2)
+; CHECK-NEXT: .b8 31                                  // Abbreviation Code
+; CHECK-NEXT: .b8 8                                   // DW_TAG_imported_declaration
+; CHECK-NEXT: .b8 0                                   // DW_CHILDREN_no
+; CHECK-NEXT: .b8 58                                  // DW_AT_decl_file
+; CHECK-NEXT: .b8 11                                  // DW_FORM_data1
+; CHECK-NEXT: .b8 59                                  // DW_AT_decl_line
+; CHECK-NEXT: .b8 11                                  // DW_FORM_data1
+; CHECK-NEXT: .b8 24                                  // DW_AT_import
+; CHECK-NEXT: .b8 19                                  // DW_FORM_ref4
+; CHECK-NEXT: .b8 0                                   // EOM(1)
+; CHECK-NEXT: .b8 0                                   // EOM(2)
+; CHECK-NEXT: .b8 32                                  // Abbreviation Code
+; CHECK-NEXT: .b8 46                                  // DW_TAG_subprogram
+; CHECK-NEXT: .b8 1                                   // DW_CHILDREN_yes
+; CHECK-NEXT: .b8 135                                 // DW_AT_MIPS_linkage_name
+; CHECK-NEXT: .b8 64
+; CHECK-NEXT: .b8 8                                   // DW_FORM_string
+; CHECK-NEXT: .b8 3                                   // DW_AT_name
+; CHECK-NEXT: .b8 8                                   // DW_FORM_string
+; CHECK-NEXT: .b8 58                                  // DW_AT_decl_file
+; CHECK-NEXT: .b8 11                                  // DW_FORM_data1
+; CHECK-NEXT: .b8 59                                  // DW_AT_decl_line
+; CHECK-NEXT: .b8 11                                  // DW_FORM_data1
+; CHECK-NEXT: .b8 73                                  // DW_AT_type
+; CHECK-NEXT: .b8 19                                  // DW_FORM_ref4
+; CHECK-NEXT: .b8 60                                  // DW_AT_declaration
+; CHECK-NEXT: .b8 12                                  // DW_FORM_flag
+; CHECK-NEXT: .b8 0                                   // EOM(1)
+; CHECK-NEXT: .b8 0                                   // EOM(2)
+; CHECK-NEXT: .b8 0                                   // EOM(3)
+; CHECK-NEXT: 	}
+; CHECK-NEXT: 	.section	.debug_info
+; CHECK-NEXT: 	{
+; CHECK-NEXT: .b32 2388                               // Length of Unit
+; CHECK-NEXT: .b8 2                                   // DWARF version number
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b32 .debug_abbrev                      // Offset Into Abbrev. Section
+; CHECK-NEXT: .b8 8                                   // Address Size (in bytes)
+; CHECK-NEXT: .b8 1                                   // Abbrev [1] 0xb:0x94d DW_TAG_compile_unit
+; CHECK-NEXT: .b8 0                                   // DW_AT_producer
+; CHECK-NEXT: .b8 4                                   // DW_AT_language
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 100                                 // DW_AT_name
+; CHECK-NEXT: .b8 101
+; CHECK-NEXT: .b8 98
+; CHECK-NEXT: .b8 117
+; CHECK-NEXT: .b8 103
+; CHECK-NEXT: .b8 45
+; CHECK-NEXT: .b8 105
+; CHECK-NEXT: .b8 110
+; CHECK-NEXT: .b8 102
+; CHECK-NEXT: .b8 111
+; CHECK-NEXT: .b8 46
+; CHECK-NEXT: .b8 99
+; CHECK-NEXT: .b8 117
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b32 .debug_line                        // DW_AT_stmt_list
+; CHECK-NEXT: .b8 47                                  // DW_AT_comp_dir
+; CHECK-NEXT: .b8 115
+; CHECK-NEXT: .b8 111
+; CHECK-NEXT: .b8 109
+; CHECK-NEXT: .b8 101
+; CHECK-NEXT: .b8 47
+; CHECK-NEXT: .b8 100
+; CHECK-NEXT: .b8 105
+; CHECK-NEXT: .b8 114
+; CHECK-NEXT: .b8 101
+; CHECK-NEXT: .b8 99
+; CHECK-NEXT: .b8 116
+; CHECK-NEXT: .b8 111
+; CHECK-NEXT: .b8 114
+; CHECK-NEXT: .b8 121
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 2                                   // Abbrev [2] 0x31:0x22a DW_TAG_structure_type
+; CHECK-NEXT: .b8 95                                  // DW_AT_name
+; CHECK-NEXT: .b8 95
+; CHECK-NEXT: .b8 99
+; CHECK-NEXT: .b8 117
+; CHECK-NEXT: .b8 100
+; CHECK-NEXT: .b8 97
+; CHECK-NEXT: .b8 95
+; CHECK-NEXT: .b8 98
+; CHECK-NEXT: .b8 117
+; CHECK-NEXT: .b8 105
+; CHECK-NEXT: .b8 108
+; CHECK-NEXT: .b8 116
+; CHECK-NEXT: .b8 105
+; CHECK-NEXT: .b8 110
+; CHECK-NEXT: .b8 95
+; CHECK-NEXT: .b8 98
+; CHECK-NEXT: .b8 108
+; CHECK-NEXT: .b8 111
+; CHECK-NEXT: .b8 99
+; CHECK-NEXT: .b8 107
+; CHECK-NEXT: .b8 73
+; CHECK-NEXT: .b8 100
+; CHECK-NEXT: .b8 120
+; CHECK-NEXT: .b8 95
+; CHECK-NEXT: .b8 116
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 1                                   // DW_AT_byte_size
+; CHECK-NEXT: .b8 2                                   // DW_AT_decl_file
+; CHECK-NEXT: .b8 77                                  // DW_AT_decl_line
+; CHECK-NEXT: .b8 3                                   // Abbrev [3] 0x4f:0x4f DW_TAG_subprogram
+; CHECK-NEXT: .b8 95                                  // DW_AT_MIPS_linkage_name
+; CHECK-NEXT: .b8 90
+; CHECK-NEXT: .b8 78
+; CHECK-NEXT: .b8 50
+; CHECK-NEXT: .b8 53
+; CHECK-NEXT: .b8 95
+; CHECK-NEXT: .b8 95
+; CHECK-NEXT: .b8 99
+; CHECK-NEXT: .b8 117
+; CHECK-NEXT: .b8 100
+; CHECK-NEXT: .b8 97
+; CHECK-NEXT: .b8 95
+; CHECK-NEXT: .b8 98
+; CHECK-NEXT: .b8 117
+; CHECK-NEXT: .b8 105
+; CHECK-NEXT: .b8 108
+; CHECK-NEXT: .b8 116
+; CHECK-NEXT: .b8 105
+; CHECK-NEXT: .b8 110
+; CHECK-NEXT: .b8 95
+; CHECK-NEXT: .b8 98
+; CHECK-NEXT: .b8 108
+; CHECK-NEXT: .b8 111
+; CHECK-NEXT: .b8 99
+; CHECK-NEXT: .b8 107
+; CHECK-NEXT: .b8 73
+; CHECK-NEXT: .b8 100
+; CHECK-NEXT: .b8 120
+; CHECK-NEXT: .b8 95
+; CHECK-NEXT: .b8 116
+; CHECK-NEXT: .b8 49
+; CHECK-NEXT: .b8 55
+; CHECK-NEXT: .b8 95
+; CHECK-NEXT: .b8 95
+; CHECK-NEXT: .b8 102
+; CHECK-NEXT: .b8 101
+; CHECK-NEXT: .b8 116
+; CHECK-NEXT: .b8 99
+; CHECK-NEXT: .b8 104
+; CHECK-NEXT: .b8 95
+; CHECK-NEXT: .b8 98
+; CHECK-NEXT: .b8 117
+; CHECK-NEXT: .b8 105
+; CHECK-NEXT: .b8 108
+; CHECK-NEXT: .b8 116
+; CHECK-NEXT: .b8 105
+; CHECK-NEXT: .b8 110
+; CHECK-NEXT: .b8 95
+; CHECK-NEXT: .b8 120
+; CHECK-NEXT: .b8 69
+; CHECK-NEXT: .b8 118
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 95                                  // DW_AT_name
+; CHECK-NEXT: .b8 95
+; CHECK-NEXT: .b8 102
+; CHECK-NEXT: .b8 101
+; CHECK-NEXT: .b8 116
+; CHECK-NEXT: .b8 99
+; CHECK-NEXT: .b8 104
+; CHECK-NEXT: .b8 95
+; CHECK-NEXT: .b8 98
+; CHECK-NEXT: .b8 117
+; CHECK-NEXT: .b8 105
+; CHECK-NEXT: .b8 108
+; CHECK-NEXT: .b8 116
+; CHECK-NEXT: .b8 105
+; CHECK-NEXT: .b8 110
+; CHECK-NEXT: .b8 95
+; CHECK-NEXT: .b8 120
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 2                                   // DW_AT_decl_file
+; CHECK-NEXT: .b8 78                                  // DW_AT_decl_line
+; CHECK-NEXT: .b32 603                                // DW_AT_type
+; CHECK-NEXT: .b8 1                                   // DW_AT_declaration
+; CHECK-NEXT: .b8 1                                   // DW_AT_external
+; CHECK-NEXT: .b8 3                                   // Abbrev [3] 0x9e:0x4f DW_TAG_subprogram
+; CHECK-NEXT: .b8 95                                  // DW_AT_MIPS_linkage_name
+; CHECK-NEXT: .b8 90
+; CHECK-NEXT: .b8 78
+; CHECK-NEXT: .b8 50
+; CHECK-NEXT: .b8 53
+; CHECK-NEXT: .b8 95
+; CHECK-NEXT: .b8 95
+; CHECK-NEXT: .b8 99
+; CHECK-NEXT: .b8 117
+; CHECK-NEXT: .b8 100
+; CHECK-NEXT: .b8 97
+; CHECK-NEXT: .b8 95
+; CHECK-NEXT: .b8 98
+; CHECK-NEXT: .b8 117
+; CHECK-NEXT: .b8 105
+; CHECK-NEXT: .b8 108
+; CHECK-NEXT: .b8 116
+; CHECK-NEXT: .b8 105
+; CHECK-NEXT: .b8 110
+; CHECK-NEXT: .b8 95
+; CHECK-NEXT: .b8 98
+; CHECK-NEXT: .b8 108
+; CHECK-NEXT: .b8 111
+; CHECK-NEXT: .b8 99
+; CHECK-NEXT: .b8 107
+; CHECK-NEXT: .b8 73
+; CHECK-NEXT: .b8 100
+; CHECK-NEXT: .b8 120
+; CHECK-NEXT: .b8 95
+; CHECK-NEXT: .b8 116
+; CHECK-NEXT: .b8 49
+; CHECK-NEXT: .b8 55
+; CHECK-NEXT: .b8 95
+; CHECK-NEXT: .b8 95
+; CHECK-NEXT: .b8 102
+; CHECK-NEXT: .b8 101
+; CHECK-NEXT: .b8 116
+; CHECK-NEXT: .b8 99
+; CHECK-NEXT: .b8 104
+; CHECK-NEXT: .b8 95
+; CHECK-NEXT: .b8 98
+; CHECK-NEXT: .b8 117
+; CHECK-NEXT: .b8 105
+; CHECK-NEXT: .b8 108
+; CHECK-NEXT: .b8 116
+; CHECK-NEXT: .b8 105
+; CHECK-NEXT: .b8 110
+; CHECK-NEXT: .b8 95
+; CHECK-NEXT: .b8 121
+; CHECK-NEXT: .b8 69
+; CHECK-NEXT: .b8 118
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 95                                  // DW_AT_name
+; CHECK-NEXT: .b8 95
+; CHECK-NEXT: .b8 102
+; CHECK-NEXT: .b8 101
+; CHECK-NEXT: .b8 116
+; CHECK-NEXT: .b8 99
+; CHECK-NEXT: .b8 104
+; CHECK-NEXT: .b8 95
+; CHECK-NEXT: .b8 98
+; CHECK-NEXT: .b8 117
+; CHECK-NEXT: .b8 105
+; CHECK-NEXT: .b8 108
+; CHECK-NEXT: .b8 116
+; CHECK-NEXT: .b8 105
+; CHECK-NEXT: .b8 110
+; CHECK-NEXT: .b8 95
+; CHECK-NEXT: .b8 121
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 2                                   // DW_AT_decl_file
+; CHECK-NEXT: .b8 79                                  // DW_AT_decl_line
+; CHECK-NEXT: .b32 603                                // DW_AT_type
+; CHECK-NEXT: .b8 1                                   // DW_AT_declaration
+; CHECK-NEXT: .b8 1                                   // DW_AT_external
+; CHECK-NEXT: .b8 3                                   // Abbrev [3] 0xed:0x4f DW_TAG_subprogram
+; CHECK-NEXT: .b8 95                                  // DW_AT_MIPS_linkage_name
+; CHECK-NEXT: .b8 90
+; CHECK-NEXT: .b8 78
+; CHECK-NEXT: .b8 50
+; CHECK-NEXT: .b8 53
+; CHECK-NEXT: .b8 95
+; CHECK-NEXT: .b8 95
+; CHECK-NEXT: .b8 99
+; CHECK-NEXT: .b8 117
+; CHECK-NEXT: .b8 100
+; CHECK-NEXT: .b8 97
+; CHECK-NEXT: .b8 95
+; CHECK-NEXT: .b8 98
+; CHECK-NEXT: .b8 117
+; CHECK-NEXT: .b8 105
+; CHECK-NEXT: .b8 108
+; CHECK-NEXT: .b8 116
+; CHECK-NEXT: .b8 105
+; CHECK-NEXT: .b8 110
+; CHECK-NEXT: .b8 95
+; CHECK-NEXT: .b8 98
+; CHECK-NEXT: .b8 108
+; CHECK-NEXT: .b8 111
+; CHECK-NEXT: .b8 99
+; CHECK-NEXT: .b8 107
+; CHECK-NEXT: .b8 73
+; CHECK-NEXT: .b8 100
+; CHECK-NEXT: .b8 120
+; CHECK-NEXT: .b8 95
+; CHECK-NEXT: .b8 116
+; CHECK-NEXT: .b8 49
+; CHECK-NEXT: .b8 55
+; CHECK-NEXT: .b8 95
+; CHECK-NEXT: .b8 95
+; CHECK-NEXT: .b8 102
+; CHECK-NEXT: .b8 101
+; CHECK-NEXT: .b8 116
+; CHECK-NEXT: .b8 99
+; CHECK-NEXT: .b8 104
+; CHECK-NEXT: .b8 95
+; CHECK-NEXT: .b8 98
+; CHECK-NEXT: .b8 117
+; CHECK-NEXT: .b8 105
+; CHECK-NEXT: .b8 108
+; CHECK-NEXT: .b8 116
+; CHECK-NEXT: .b8 105
+; CHECK-NEXT: .b8 110
+; CHECK-NEXT: .b8 95
+; CHECK-NEXT: .b8 122
+; CHECK-NEXT: .b8 69
+; CHECK-NEXT: .b8 118
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 95                                  // DW_AT_name
+; CHECK-NEXT: .b8 95
+; CHECK-NEXT: .b8 102
+; CHECK-NEXT: .b8 101
+; CHECK-NEXT: .b8 116
+; CHECK-NEXT: .b8 99
+; CHECK-NEXT: .b8 104
+; CHECK-NEXT: .b8 95
+; CHECK-NEXT: .b8 98
+; CHECK-NEXT: .b8 117
+; CHECK-NEXT: .b8 105
+; CHECK-NEXT: .b8 108
+; CHECK-NEXT: .b8 116
+; CHECK-NEXT: .b8 105
+; CHECK-NEXT: .b8 110
+; CHECK-NEXT: .b8 95
+; CHECK-NEXT: .b8 122
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 2                                   // DW_AT_decl_file
+; CHECK-NEXT: .b8 80                                  // DW_AT_decl_line
+; CHECK-NEXT: .b32 603                                // DW_AT_type
+; CHECK-NEXT: .b8 1                                   // DW_AT_declaration
+; CHECK-NEXT: .b8 1                                   // DW_AT_external
+; CHECK-NEXT: .b8 4                                   // Abbrev [4] 0x13c:0x49 DW_TAG_subprogram
+; CHECK-NEXT: .b8 95                                  // DW_AT_MIPS_linkage_name
+; CHECK-NEXT: .b8 90
+; CHECK-NEXT: .b8 78
+; CHECK-NEXT: .b8 75
+; CHECK-NEXT: .b8 50
+; CHECK-NEXT: .b8 53
+; CHECK-NEXT: .b8 95
+; CHECK-NEXT: .b8 95
+; CHECK-NEXT: .b8 99
+; CHECK-NEXT: .b8 117
+; CHECK-NEXT: .b8 100
+; CHECK-NEXT: .b8 97
+; CHECK-NEXT: .b8 95
+; CHECK-NEXT: .b8 98
+; CHECK-NEXT: .b8 117
+; CHECK-NEXT: .b8 105
+; CHECK-NEXT: .b8 108
+; CHECK-NEXT: .b8 116
+; CHECK-NEXT: .b8 105
+; CHECK-NEXT: .b8 110
+; CHECK-NEXT: .b8 95
+; CHECK-NEXT: .b8 98
+; CHECK-NEXT: .b8 108
+; CHECK-NEXT: .b8 111
+; CHECK-NEXT: .b8 99
+; CHECK-NEXT: .b8 107
+; CHECK-NEXT: .b8 73
+; CHECK-NEXT: .b8 100
+; CHECK-NEXT: .b8 120
+; CHECK-NEXT: .b8 95
+; CHECK-NEXT: .b8 116
+; CHECK-NEXT: .b8 99
+; CHECK-NEXT: .b8 118
+; CHECK-NEXT: .b8 53
+; CHECK-NEXT: .b8 117
+; CHECK-NEXT: .b8 105
+; CHECK-NEXT: .b8 110
+; CHECK-NEXT: .b8 116
+; CHECK-NEXT: .b8 51
+; CHECK-NEXT: .b8 69
+; CHECK-NEXT: .b8 118
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 111                                 // DW_AT_name
+; CHECK-NEXT: .b8 112
+; CHECK-NEXT: .b8 101
+; CHECK-NEXT: .b8 114
+; CHECK-NEXT: .b8 97
+; CHECK-NEXT: .b8 116
+; CHECK-NEXT: .b8 111
+; CHECK-NEXT: .b8 114
+; CHECK-NEXT: .b8 32
+; CHECK-NEXT: .b8 117
+; CHECK-NEXT: .b8 105
+; CHECK-NEXT: .b8 110
+; CHECK-NEXT: .b8 116
+; CHECK-NEXT: .b8 51
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 2                                   // DW_AT_decl_file
+; CHECK-NEXT: .b8 83                                  // DW_AT_decl_line
+; CHECK-NEXT: .b32 619                                // DW_AT_type
+; CHECK-NEXT: .b8 1                                   // DW_AT_declaration
+; CHECK-NEXT: .b8 1                                   // DW_AT_external
+; CHECK-NEXT: .b8 5                                   // Abbrev [5] 0x17e:0x6 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 666                                // DW_AT_type
+; CHECK-NEXT: .b8 1                                   // DW_AT_artificial
+; CHECK-NEXT: .b8 0                                   // End Of Children Mark
+; CHECK-NEXT: .b8 6                                   // Abbrev [6] 0x185:0x27 DW_TAG_subprogram
+; CHECK-NEXT: .b8 95                                  // DW_AT_name
+; CHECK-NEXT: .b8 95
+; CHECK-NEXT: .b8 99
+; CHECK-NEXT: .b8 117
+; CHECK-NEXT: .b8 100
+; CHECK-NEXT: .b8 97
+; CHECK-NEXT: .b8 95
+; CHECK-NEXT: .b8 98
+; CHECK-NEXT: .b8 117
+; CHECK-NEXT: .b8 105
+; CHECK-NEXT: .b8 108
+; CHECK-NEXT: .b8 116
+; CHECK-NEXT: .b8 105
+; CHECK-NEXT: .b8 110
+; CHECK-NEXT: .b8 95
+; CHECK-NEXT: .b8 98
+; CHECK-NEXT: .b8 108
+; CHECK-NEXT: .b8 111
+; CHECK-NEXT: .b8 99
+; CHECK-NEXT: .b8 107
+; CHECK-NEXT: .b8 73
+; CHECK-NEXT: .b8 100
+; CHECK-NEXT: .b8 120
+; CHECK-NEXT: .b8 95
+; CHECK-NEXT: .b8 116
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 2                                   // DW_AT_decl_file
+; CHECK-NEXT: .b8 85                                  // DW_AT_decl_line
+; CHECK-NEXT: .b8 1                                   // DW_AT_declaration
+; CHECK-NEXT: .b8 1                                   // DW_AT_external
+; CHECK-NEXT: .b8 3                                   // DW_AT_accessibility
+; CHECK-NEXT:                                         // DW_ACCESS_private
+; CHECK-NEXT: .b8 5                                   // Abbrev [5] 0x1a5:0x6 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 676                                // DW_AT_type
+; CHECK-NEXT: .b8 1                                   // DW_AT_artificial
+; CHECK-NEXT: .b8 0                                   // End Of Children Mark
+; CHECK-NEXT: .b8 6                                   // Abbrev [6] 0x1ac:0x2c DW_TAG_subprogram
+; CHECK-NEXT: .b8 95                                  // DW_AT_name
+; CHECK-NEXT: .b8 95
+; CHECK-NEXT: .b8 99
+; CHECK-NEXT: .b8 117
+; CHECK-NEXT: .b8 100
+; CHECK-NEXT: .b8 97
+; CHECK-NEXT: .b8 95
+; CHECK-NEXT: .b8 98
+; CHECK-NEXT: .b8 117
+; CHECK-NEXT: .b8 105
+; CHECK-NEXT: .b8 108
+; CHECK-NEXT: .b8 116
+; CHECK-NEXT: .b8 105
+; CHECK-NEXT: .b8 110
+; CHECK-NEXT: .b8 95
+; CHECK-NEXT: .b8 98
+; CHECK-NEXT: .b8 108
+; CHECK-NEXT: .b8 111
+; CHECK-NEXT: .b8 99
+; CHECK-NEXT: .b8 107
+; CHECK-NEXT: .b8 73
+; CHECK-NEXT: .b8 100
+; CHECK-NEXT: .b8 120
+; CHECK-NEXT: .b8 95
+; CHECK-NEXT: .b8 116
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 2                                   // DW_AT_decl_file
+; CHECK-NEXT: .b8 85                                  // DW_AT_decl_line
+; CHECK-NEXT: .b8 1                                   // DW_AT_declaration
+; CHECK-NEXT: .b8 1                                   // DW_AT_external
+; CHECK-NEXT: .b8 3                                   // DW_AT_accessibility
+; CHECK-NEXT:                                         // DW_ACCESS_private
+; CHECK-NEXT: .b8 5                                   // Abbrev [5] 0x1cc:0x6 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 676                                // DW_AT_type
+; CHECK-NEXT: .b8 1                                   // DW_AT_artificial
+; CHECK-NEXT: .b8 7                                   // Abbrev [7] 0x1d2:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 681                                // DW_AT_type
+; CHECK-NEXT: .b8 0                                   // End Of Children Mark
+; CHECK-NEXT: .b8 8                                   // Abbrev [8] 0x1d8:0x43 DW_TAG_subprogram
+; CHECK-NEXT: .b8 95                                  // DW_AT_MIPS_linkage_name
+; CHECK-NEXT: .b8 90
+; CHECK-NEXT: .b8 78
+; CHECK-NEXT: .b8 75
+; CHECK-NEXT: .b8 50
+; CHECK-NEXT: .b8 53
+; CHECK-NEXT: .b8 95
+; CHECK-NEXT: .b8 95
+; CHECK-NEXT: .b8 99
+; CHECK-NEXT: .b8 117
+; CHECK-NEXT: .b8 100
+; CHECK-NEXT: .b8 97
+; CHECK-NEXT: .b8 95
+; CHECK-NEXT: .b8 98
+; CHECK-NEXT: .b8 117
+; CHECK-NEXT: .b8 105
+; CHECK-NEXT: .b8 108
+; CHECK-NEXT: .b8 116
+; CHECK-NEXT: .b8 105
+; CHECK-NEXT: .b8 110
+; CHECK-NEXT: .b8 95
+; CHECK-NEXT: .b8 98
+; CHECK-NEXT: .b8 108
+; CHECK-NEXT: .b8 111
+; CHECK-NEXT: .b8 99
+; CHECK-NEXT: .b8 107
+; CHECK-NEXT: .b8 73
+; CHECK-NEXT: .b8 100
+; CHECK-NEXT: .b8 120
+; CHECK-NEXT: .b8 95
+; CHECK-NEXT: .b8 116
+; CHECK-NEXT: .b8 97
+; CHECK-NEXT: .b8 83
+; CHECK-NEXT: .b8 69
+; CHECK-NEXT: .b8 82
+; CHECK-NEXT: .b8 75
+; CHECK-NEXT: .b8 83
+; CHECK-NEXT: .b8 95
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 111                                 // DW_AT_name
+; CHECK-NEXT: .b8 112
+; CHECK-NEXT: .b8 101
+; CHECK-NEXT: .b8 114
+; CHECK-NEXT: .b8 97
+; CHECK-NEXT: .b8 116
+; CHECK-NEXT: .b8 111
+; CHECK-NEXT: .b8 114
+; CHECK-NEXT: .b8 61
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 2                                   // DW_AT_decl_file
+; CHECK-NEXT: .b8 85                                  // DW_AT_decl_line
+; CHECK-NEXT: .b8 1                                   // DW_AT_declaration
+; CHECK-NEXT: .b8 1                                   // DW_AT_external
+; CHECK-NEXT: .b8 3                                   // DW_AT_accessibility
+; CHECK-NEXT:                                         // DW_ACCESS_private
+; CHECK-NEXT: .b8 5                                   // Abbrev [5] 0x20f:0x6 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 666                                // DW_AT_type
+; CHECK-NEXT: .b8 1                                   // DW_AT_artificial
+; CHECK-NEXT: .b8 7                                   // Abbrev [7] 0x215:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 681                                // DW_AT_type
+; CHECK-NEXT: .b8 0                                   // End Of Children Mark
+; CHECK-NEXT: .b8 9                                   // Abbrev [9] 0x21b:0x3f DW_TAG_subprogram
+; CHECK-NEXT: .b8 95                                  // DW_AT_MIPS_linkage_name
+; CHECK-NEXT: .b8 90
+; CHECK-NEXT: .b8 78
+; CHECK-NEXT: .b8 75
+; CHECK-NEXT: .b8 50
+; CHECK-NEXT: .b8 53
+; CHECK-NEXT: .b8 95
+; CHECK-NEXT: .b8 95
+; CHECK-NEXT: .b8 99
+; CHECK-NEXT: .b8 117
+; CHECK-NEXT: .b8 100
+; CHECK-NEXT: .b8 97
+; CHECK-NEXT: .b8 95
+; CHECK-NEXT: .b8 98
+; CHECK-NEXT: .b8 117
+; CHECK-NEXT: .b8 105
+; CHECK-NEXT: .b8 108
+; CHECK-NEXT: .b8 116
+; CHECK-NEXT: .b8 105
+; CHECK-NEXT: .b8 110
+; CHECK-NEXT: .b8 95
+; CHECK-NEXT: .b8 98
+; CHECK-NEXT: .b8 108
+; CHECK-NEXT: .b8 111
+; CHECK-NEXT: .b8 99
+; CHECK-NEXT: .b8 107
+; CHECK-NEXT: .b8 73
+; CHECK-NEXT: .b8 100
+; CHECK-NEXT: .b8 120
+; CHECK-NEXT: .b8 95
+; CHECK-NEXT: .b8 116
+; CHECK-NEXT: .b8 97
+; CHECK-NEXT: .b8 100
+; CHECK-NEXT: .b8 69
+; CHECK-NEXT: .b8 118
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 111                                 // DW_AT_name
+; CHECK-NEXT: .b8 112
+; CHECK-NEXT: .b8 101
+; CHECK-NEXT: .b8 114
+; CHECK-NEXT: .b8 97
+; CHECK-NEXT: .b8 116
+; CHECK-NEXT: .b8 111
+; CHECK-NEXT: .b8 114
+; CHECK-NEXT: .b8 38
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 2                                   // DW_AT_decl_file
+; CHECK-NEXT: .b8 85                                  // DW_AT_decl_line
+; CHECK-NEXT: .b32 686                                // DW_AT_type
+; CHECK-NEXT: .b8 1                                   // DW_AT_declaration
+; CHECK-NEXT: .b8 1                                   // DW_AT_external
+; CHECK-NEXT: .b8 3                                   // DW_AT_accessibility
+; CHECK-NEXT:                                         // DW_ACCESS_private
+; CHECK-NEXT: .b8 5                                   // Abbrev [5] 0x253:0x6 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 666                                // DW_AT_type
+; CHECK-NEXT: .b8 1                                   // DW_AT_artificial
+; CHECK-NEXT: .b8 0                                   // End Of Children Mark
+; CHECK-NEXT: .b8 0                                   // End Of Children Mark
+; CHECK-NEXT: .b8 10                                  // Abbrev [10] 0x25b:0x10 DW_TAG_base_type
+; CHECK-NEXT: .b8 117                                 // DW_AT_name
+; CHECK-NEXT: .b8 110
+; CHECK-NEXT: .b8 115
+; CHECK-NEXT: .b8 105
+; CHECK-NEXT: .b8 103
+; CHECK-NEXT: .b8 110
+; CHECK-NEXT: .b8 101
+; CHECK-NEXT: .b8 100
+; CHECK-NEXT: .b8 32
+; CHECK-NEXT: .b8 105
+; CHECK-NEXT: .b8 110
+; CHECK-NEXT: .b8 116
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 7                                   // DW_AT_encoding
+; CHECK-NEXT: .b8 4                                   // DW_AT_byte_size
+; CHECK-NEXT: .b8 2                                   // Abbrev [2] 0x26b:0x2f DW_TAG_structure_type
+; CHECK-NEXT: .b8 117                                 // DW_AT_name
+; CHECK-NEXT: .b8 105
+; CHECK-NEXT: .b8 110
+; CHECK-NEXT: .b8 116
+; CHECK-NEXT: .b8 51
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 12                                  // DW_AT_byte_size
+; CHECK-NEXT: .b8 3                                   // DW_AT_decl_file
+; CHECK-NEXT: .b8 190                                 // DW_AT_decl_line
+; CHECK-NEXT: .b8 11                                  // Abbrev [11] 0x275:0xc DW_TAG_member
+; CHECK-NEXT: .b8 120                                 // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b32 603                                // DW_AT_type
+; CHECK-NEXT: .b8 3                                   // DW_AT_decl_file
+; CHECK-NEXT: .b8 192                                 // DW_AT_decl_line
+; CHECK-NEXT: .b8 2                                   // DW_AT_data_member_location
+; CHECK-NEXT: .b8 35
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 11                                  // Abbrev [11] 0x281:0xc DW_TAG_member
+; CHECK-NEXT: .b8 121                                 // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b32 603                                // DW_AT_type
+; CHECK-NEXT: .b8 3                                   // DW_AT_decl_file
+; CHECK-NEXT: .b8 192                                 // DW_AT_decl_line
+; CHECK-NEXT: .b8 2                                   // DW_AT_data_member_location
+; CHECK-NEXT: .b8 35
+; CHECK-NEXT: .b8 4
+; CHECK-NEXT: .b8 11                                  // Abbrev [11] 0x28d:0xc DW_TAG_member
+; CHECK-NEXT: .b8 122                                 // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b32 603                                // DW_AT_type
+; CHECK-NEXT: .b8 3                                   // DW_AT_decl_file
+; CHECK-NEXT: .b8 192                                 // DW_AT_decl_line
+; CHECK-NEXT: .b8 2                                   // DW_AT_data_member_location
+; CHECK-NEXT: .b8 35
+; CHECK-NEXT: .b8 8
+; CHECK-NEXT: .b8 0                                   // End Of Children Mark
+; CHECK-NEXT: .b8 12                                  // Abbrev [12] 0x29a:0x5 DW_TAG_pointer_type
+; CHECK-NEXT: .b32 671                                // DW_AT_type
+; CHECK-NEXT: .b8 13                                  // Abbrev [13] 0x29f:0x5 DW_TAG_const_type
+; CHECK-NEXT: .b32 49                                 // DW_AT_type
+; CHECK-NEXT: .b8 12                                  // Abbrev [12] 0x2a4:0x5 DW_TAG_pointer_type
+; CHECK-NEXT: .b32 49                                 // DW_AT_type
+; CHECK-NEXT: .b8 14                                  // Abbrev [14] 0x2a9:0x5 DW_TAG_reference_type
+; CHECK-NEXT: .b32 671                                // DW_AT_type
+; CHECK-NEXT: .b8 12                                  // Abbrev [12] 0x2ae:0x5 DW_TAG_pointer_type
+; CHECK-NEXT: .b32 49                                 // DW_AT_type
+; CHECK-NEXT: .b8 15                                  // Abbrev [15] 0x2b3:0x6 DW_TAG_subprogram
+; CHECK-NEXT: .b32 79                                 // DW_AT_specification
+; CHECK-NEXT: .b8 1                                   // DW_AT_inline
+; CHECK-NEXT: .b8 2                                   // Abbrev [2] 0x2b9:0x228 DW_TAG_structure_type
+; CHECK-NEXT: .b8 95                                  // DW_AT_name
+; CHECK-NEXT: .b8 95
+; CHECK-NEXT: .b8 99
+; CHECK-NEXT: .b8 117
+; CHECK-NEXT: .b8 100
+; CHECK-NEXT: .b8 97
+; CHECK-NEXT: .b8 95
+; CHECK-NEXT: .b8 98
+; CHECK-NEXT: .b8 117
+; CHECK-NEXT: .b8 105
+; CHECK-NEXT: .b8 108
+; CHECK-NEXT: .b8 116
+; CHECK-NEXT: .b8 105
+; CHECK-NEXT: .b8 110
+; CHECK-NEXT: .b8 95
+; CHECK-NEXT: .b8 98
+; CHECK-NEXT: .b8 108
+; CHECK-NEXT: .b8 111
+; CHECK-NEXT: .b8 99
+; CHECK-NEXT: .b8 107
+; CHECK-NEXT: .b8 68
+; CHECK-NEXT: .b8 105
+; CHECK-NEXT: .b8 109
+; CHECK-NEXT: .b8 95
+; CHECK-NEXT: .b8 116
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 1                                   // DW_AT_byte_size
+; CHECK-NEXT: .b8 2                                   // DW_AT_decl_file
+; CHECK-NEXT: .b8 88                                  // DW_AT_decl_line
+; CHECK-NEXT: .b8 3                                   // Abbrev [3] 0x2d7:0x4f DW_TAG_subprogram
+; CHECK-NEXT: .b8 95                                  // DW_AT_MIPS_linkage_name
+; CHECK-NEXT: .b8 90
+; CHECK-NEXT: .b8 78
+; CHECK-NEXT: .b8 50
+; CHECK-NEXT: .b8 53
+; CHECK-NEXT: .b8 95
+; CHECK-NEXT: .b8 95
+; CHECK-NEXT: .b8 99
+; CHECK-NEXT: .b8 117
+; CHECK-NEXT: .b8 100
+; CHECK-NEXT: .b8 97
+; CHECK-NEXT: .b8 95
+; CHECK-NEXT: .b8 98
+; CHECK-NEXT: .b8 117
+; CHECK-NEXT: .b8 105
+; CHECK-NEXT: .b8 108
+; CHECK-NEXT: .b8 116
+; CHECK-NEXT: .b8 105
+; CHECK-NEXT: .b8 110
+; CHECK-NEXT: .b8 95
+; CHECK-NEXT: .b8 98
+; CHECK-NEXT: .b8 108
+; CHECK-NEXT: .b8 111
+; CHECK-NEXT: .b8 99
+; CHECK-NEXT: .b8 107
+; CHECK-NEXT: .b8 68
+; CHECK-NEXT: .b8 105
+; CHECK-NEXT: .b8 109
+; CHECK-NEXT: .b8 95
+; CHECK-NEXT: .b8 116
+; CHECK-NEXT: .b8 49
+; CHECK-NEXT: .b8 55
+; CHECK-NEXT: .b8 95
+; CHECK-NEXT: .b8 95
+; CHECK-NEXT: .b8 102
+; CHECK-NEXT: .b8 101
+; CHECK-NEXT: .b8 116
+; CHECK-NEXT: .b8 99
+; CHECK-NEXT: .b8 104
+; CHECK-NEXT: .b8 95
+; CHECK-NEXT: .b8 98
+; CHECK-NEXT: .b8 117
+; CHECK-NEXT: .b8 105
+; CHECK-NEXT: .b8 108
+; CHECK-NEXT: .b8 116
+; CHECK-NEXT: .b8 105
+; CHECK-NEXT: .b8 110
+; CHECK-NEXT: .b8 95
+; CHECK-NEXT: .b8 120
+; CHECK-NEXT: .b8 69
+; CHECK-NEXT: .b8 118
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 95                                  // DW_AT_name
+; CHECK-NEXT: .b8 95
+; CHECK-NEXT: .b8 102
+; CHECK-NEXT: .b8 101
+; CHECK-NEXT: .b8 116
+; CHECK-NEXT: .b8 99
+; CHECK-NEXT: .b8 104
+; CHECK-NEXT: .b8 95
+; CHECK-NEXT: .b8 98
+; CHECK-NEXT: .b8 117
+; CHECK-NEXT: .b8 105
+; CHECK-NEXT: .b8 108
+; CHECK-NEXT: .b8 116
+; CHECK-NEXT: .b8 105
+; CHECK-NEXT: .b8 110
+; CHECK-NEXT: .b8 95
+; CHECK-NEXT: .b8 120
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 2                                   // DW_AT_decl_file
+; CHECK-NEXT: .b8 89                                  // DW_AT_decl_line
+; CHECK-NEXT: .b32 603                                // DW_AT_type
+; CHECK-NEXT: .b8 1                                   // DW_AT_declaration
+; CHECK-NEXT: .b8 1                                   // DW_AT_external
+; CHECK-NEXT: .b8 3                                   // Abbrev [3] 0x326:0x4f DW_TAG_subprogram
+; CHECK-NEXT: .b8 95                                  // DW_AT_MIPS_linkage_name
+; CHECK-NEXT: .b8 90
+; CHECK-NEXT: .b8 78
+; CHECK-NEXT: .b8 50
+; CHECK-NEXT: .b8 53
+; CHECK-NEXT: .b8 95
+; CHECK-NEXT: .b8 95
+; CHECK-NEXT: .b8 99
+; CHECK-NEXT: .b8 117
+; CHECK-NEXT: .b8 100
+; CHECK-NEXT: .b8 97
+; CHECK-NEXT: .b8 95
+; CHECK-NEXT: .b8 98
+; CHECK-NEXT: .b8 117
+; CHECK-NEXT: .b8 105
+; CHECK-NEXT: .b8 108
+; CHECK-NEXT: .b8 116
+; CHECK-NEXT: .b8 105
+; CHECK-NEXT: .b8 110
+; CHECK-NEXT: .b8 95
+; CHECK-NEXT: .b8 98
+; CHECK-NEXT: .b8 108
+; CHECK-NEXT: .b8 111
+; CHECK-NEXT: .b8 99
+; CHECK-NEXT: .b8 107
+; CHECK-NEXT: .b8 68
+; CHECK-NEXT: .b8 105
+; CHECK-NEXT: .b8 109
+; CHECK-NEXT: .b8 95
+; CHECK-NEXT: .b8 116
+; CHECK-NEXT: .b8 49
+; CHECK-NEXT: .b8 55
+; CHECK-NEXT: .b8 95
+; CHECK-NEXT: .b8 95
+; CHECK-NEXT: .b8 102
+; CHECK-NEXT: .b8 101
+; CHECK-NEXT: .b8 116
+; CHECK-NEXT: .b8 99
+; CHECK-NEXT: .b8 104
+; CHECK-NEXT: .b8 95
+; CHECK-NEXT: .b8 98
+; CHECK-NEXT: .b8 117
+; CHECK-NEXT: .b8 105
+; CHECK-NEXT: .b8 108
+; CHECK-NEXT: .b8 116
+; CHECK-NEXT: .b8 105
+; CHECK-NEXT: .b8 110
+; CHECK-NEXT: .b8 95
+; CHECK-NEXT: .b8 121
+; CHECK-NEXT: .b8 69
+; CHECK-NEXT: .b8 118
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 95                                  // DW_AT_name
+; CHECK-NEXT: .b8 95
+; CHECK-NEXT: .b8 102
+; CHECK-NEXT: .b8 101
+; CHECK-NEXT: .b8 116
+; CHECK-NEXT: .b8 99
+; CHECK-NEXT: .b8 104
+; CHECK-NEXT: .b8 95
+; CHECK-NEXT: .b8 98
+; CHECK-NEXT: .b8 117
+; CHECK-NEXT: .b8 105
+; CHECK-NEXT: .b8 108
+; CHECK-NEXT: .b8 116
+; CHECK-NEXT: .b8 105
+; CHECK-NEXT: .b8 110
+; CHECK-NEXT: .b8 95
+; CHECK-NEXT: .b8 121
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 2                                   // DW_AT_decl_file
+; CHECK-NEXT: .b8 90                                  // DW_AT_decl_line
+; CHECK-NEXT: .b32 603                                // DW_AT_type
+; CHECK-NEXT: .b8 1                                   // DW_AT_declaration
+; CHECK-NEXT: .b8 1                                   // DW_AT_external
+; CHECK-NEXT: .b8 3                                   // Abbrev [3] 0x375:0x4f DW_TAG_subprogram
+; CHECK-NEXT: .b8 95                                  // DW_AT_MIPS_linkage_name
+; CHECK-NEXT: .b8 90
+; CHECK-NEXT: .b8 78
+; CHECK-NEXT: .b8 50
+; CHECK-NEXT: .b8 53
+; CHECK-NEXT: .b8 95
+; CHECK-NEXT: .b8 95
+; CHECK-NEXT: .b8 99
+; CHECK-NEXT: .b8 117
+; CHECK-NEXT: .b8 100
+; CHECK-NEXT: .b8 97
+; CHECK-NEXT: .b8 95
+; CHECK-NEXT: .b8 98
+; CHECK-NEXT: .b8 117
+; CHECK-NEXT: .b8 105
+; CHECK-NEXT: .b8 108
+; CHECK-NEXT: .b8 116
+; CHECK-NEXT: .b8 105
+; CHECK-NEXT: .b8 110
+; CHECK-NEXT: .b8 95
+; CHECK-NEXT: .b8 98
+; CHECK-NEXT: .b8 108
+; CHECK-NEXT: .b8 111
+; CHECK-NEXT: .b8 99
+; CHECK-NEXT: .b8 107
+; CHECK-NEXT: .b8 68
+; CHECK-NEXT: .b8 105
+; CHECK-NEXT: .b8 109
+; CHECK-NEXT: .b8 95
+; CHECK-NEXT: .b8 116
+; CHECK-NEXT: .b8 49
+; CHECK-NEXT: .b8 55
+; CHECK-NEXT: .b8 95
+; CHECK-NEXT: .b8 95
+; CHECK-NEXT: .b8 102
+; CHECK-NEXT: .b8 101
+; CHECK-NEXT: .b8 116
+; CHECK-NEXT: .b8 99
+; CHECK-NEXT: .b8 104
+; CHECK-NEXT: .b8 95
+; CHECK-NEXT: .b8 98
+; CHECK-NEXT: .b8 117
+; CHECK-NEXT: .b8 105
+; CHECK-NEXT: .b8 108
+; CHECK-NEXT: .b8 116
+; CHECK-NEXT: .b8 105
+; CHECK-NEXT: .b8 110
+; CHECK-NEXT: .b8 95
+; CHECK-NEXT: .b8 122
+; CHECK-NEXT: .b8 69
+; CHECK-NEXT: .b8 118
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 95                                  // DW_AT_name
+; CHECK-NEXT: .b8 95
+; CHECK-NEXT: .b8 102
+; CHECK-NEXT: .b8 101
+; CHECK-NEXT: .b8 116
+; CHECK-NEXT: .b8 99
+; CHECK-NEXT: .b8 104
+; CHECK-NEXT: .b8 95
+; CHECK-NEXT: .b8 98
+; CHECK-NEXT: .b8 117
+; CHECK-NEXT: .b8 105
+; CHECK-NEXT: .b8 108
+; CHECK-NEXT: .b8 116
+; CHECK-NEXT: .b8 105
+; CHECK-NEXT: .b8 110
+; CHECK-NEXT: .b8 95
+; CHECK-NEXT: .b8 122
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 2                                   // DW_AT_decl_file
+; CHECK-NEXT: .b8 91                                  // DW_AT_decl_line
+; CHECK-NEXT: .b32 603                                // DW_AT_type
+; CHECK-NEXT: .b8 1                                   // DW_AT_declaration
+; CHECK-NEXT: .b8 1                                   // DW_AT_external
+; CHECK-NEXT: .b8 4                                   // Abbrev [4] 0x3c4:0x47 DW_TAG_subprogram
+; CHECK-NEXT: .b8 95                                  // DW_AT_MIPS_linkage_name
+; CHECK-NEXT: .b8 90
+; CHECK-NEXT: .b8 78
+; CHECK-NEXT: .b8 75
+; CHECK-NEXT: .b8 50
+; CHECK-NEXT: .b8 53
+; CHECK-NEXT: .b8 95
+; CHECK-NEXT: .b8 95
+; CHECK-NEXT: .b8 99
+; CHECK-NEXT: .b8 117
+; CHECK-NEXT: .b8 100
+; CHECK-NEXT: .b8 97
+; CHECK-NEXT: .b8 95
+; CHECK-NEXT: .b8 98
+; CHECK-NEXT: .b8 117
+; CHECK-NEXT: .b8 105
+; CHECK-NEXT: .b8 108
+; CHECK-NEXT: .b8 116
+; CHECK-NEXT: .b8 105
+; CHECK-NEXT: .b8 110
+; CHECK-NEXT: .b8 95
+; CHECK-NEXT: .b8 98
+; CHECK-NEXT: .b8 108
+; CHECK-NEXT: .b8 111
+; CHECK-NEXT: .b8 99
+; CHECK-NEXT: .b8 107
+; CHECK-NEXT: .b8 68
+; CHECK-NEXT: .b8 105
+; CHECK-NEXT: .b8 109
+; CHECK-NEXT: .b8 95
+; CHECK-NEXT: .b8 116
+; CHECK-NEXT: .b8 99
+; CHECK-NEXT: .b8 118
+; CHECK-NEXT: .b8 52
+; CHECK-NEXT: .b8 100
+; CHECK-NEXT: .b8 105
+; CHECK-NEXT: .b8 109
+; CHECK-NEXT: .b8 51
+; CHECK-NEXT: .b8 69
+; CHECK-NEXT: .b8 118
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 111                                 // DW_AT_name
+; CHECK-NEXT: .b8 112
+; CHECK-NEXT: .b8 101
+; CHECK-NEXT: .b8 114
+; CHECK-NEXT: .b8 97
+; CHECK-NEXT: .b8 116
+; CHECK-NEXT: .b8 111
+; CHECK-NEXT: .b8 114
+; CHECK-NEXT: .b8 32
+; CHECK-NEXT: .b8 100
+; CHECK-NEXT: .b8 105
+; CHECK-NEXT: .b8 109
+; CHECK-NEXT: .b8 51
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 2                                   // DW_AT_decl_file
+; CHECK-NEXT: .b8 94                                  // DW_AT_decl_line
+; CHECK-NEXT: .b32 1249                               // DW_AT_type
+; CHECK-NEXT: .b8 1                                   // DW_AT_declaration
+; CHECK-NEXT: .b8 1                                   // DW_AT_external
+; CHECK-NEXT: .b8 5                                   // Abbrev [5] 0x404:0x6 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 1425                               // DW_AT_type
+; CHECK-NEXT: .b8 1                                   // DW_AT_artificial
+; CHECK-NEXT: .b8 0                                   // End Of Children Mark
+; CHECK-NEXT: .b8 6                                   // Abbrev [6] 0x40b:0x27 DW_TAG_subprogram
+; CHECK-NEXT: .b8 95                                  // DW_AT_name
+; CHECK-NEXT: .b8 95
+; CHECK-NEXT: .b8 99
+; CHECK-NEXT: .b8 117
+; CHECK-NEXT: .b8 100
+; CHECK-NEXT: .b8 97
+; CHECK-NEXT: .b8 95
+; CHECK-NEXT: .b8 98
+; CHECK-NEXT: .b8 117
+; CHECK-NEXT: .b8 105
+; CHECK-NEXT: .b8 108
+; CHECK-NEXT: .b8 116
+; CHECK-NEXT: .b8 105
+; CHECK-NEXT: .b8 110
+; CHECK-NEXT: .b8 95
+; CHECK-NEXT: .b8 98
+; CHECK-NEXT: .b8 108
+; CHECK-NEXT: .b8 111
+; CHECK-NEXT: .b8 99
+; CHECK-NEXT: .b8 107
+; CHECK-NEXT: .b8 68
+; CHECK-NEXT: .b8 105
+; CHECK-NEXT: .b8 109
+; CHECK-NEXT: .b8 95
+; CHECK-NEXT: .b8 116
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 2                                   // DW_AT_decl_file
+; CHECK-NEXT: .b8 96                                  // DW_AT_decl_line
+; CHECK-NEXT: .b8 1                                   // DW_AT_declaration
+; CHECK-NEXT: .b8 1                                   // DW_AT_external
+; CHECK-NEXT: .b8 3                                   // DW_AT_accessibility
+; CHECK-NEXT:                                         // DW_ACCESS_private
+; CHECK-NEXT: .b8 5                                   // Abbrev [5] 0x42b:0x6 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 1435                               // DW_AT_type
+; CHECK-NEXT: .b8 1                                   // DW_AT_artificial
+; CHECK-NEXT: .b8 0                                   // End Of Children Mark
+; CHECK-NEXT: .b8 6                                   // Abbrev [6] 0x432:0x2c DW_TAG_subprogram
+; CHECK-NEXT: .b8 95                                  // DW_AT_name
+; CHECK-NEXT: .b8 95
+; CHECK-NEXT: .b8 99
+; CHECK-NEXT: .b8 117
+; CHECK-NEXT: .b8 100
+; CHECK-NEXT: .b8 97
+; CHECK-NEXT: .b8 95
+; CHECK-NEXT: .b8 98
+; CHECK-NEXT: .b8 117
+; CHECK-NEXT: .b8 105
+; CHECK-NEXT: .b8 108
+; CHECK-NEXT: .b8 116
+; CHECK-NEXT: .b8 105
+; CHECK-NEXT: .b8 110
+; CHECK-NEXT: .b8 95
+; CHECK-NEXT: .b8 98
+; CHECK-NEXT: .b8 108
+; CHECK-NEXT: .b8 111
+; CHECK-NEXT: .b8 99
+; CHECK-NEXT: .b8 107
+; CHECK-NEXT: .b8 68
+; CHECK-NEXT: .b8 105
+; CHECK-NEXT: .b8 109
+; CHECK-NEXT: .b8 95
+; CHECK-NEXT: .b8 116
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 2                                   // DW_AT_decl_file
+; CHECK-NEXT: .b8 96                                  // DW_AT_decl_line
+; CHECK-NEXT: .b8 1                                   // DW_AT_declaration
+; CHECK-NEXT: .b8 1                                   // DW_AT_external
+; CHECK-NEXT: .b8 3                                   // DW_AT_accessibility
+; CHECK-NEXT:                                         // DW_ACCESS_private
+; CHECK-NEXT: .b8 5                                   // Abbrev [5] 0x452:0x6 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 1435                               // DW_AT_type
+; CHECK-NEXT: .b8 1                                   // DW_AT_artificial
+; CHECK-NEXT: .b8 7                                   // Abbrev [7] 0x458:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 1440                               // DW_AT_type
+; CHECK-NEXT: .b8 0                                   // End Of Children Mark
+; CHECK-NEXT: .b8 8                                   // Abbrev [8] 0x45e:0x43 DW_TAG_subprogram
+; CHECK-NEXT: .b8 95                                  // DW_AT_MIPS_linkage_name
+; CHECK-NEXT: .b8 90
+; CHECK-NEXT: .b8 78
+; CHECK-NEXT: .b8 75
+; CHECK-NEXT: .b8 50
+; CHECK-NEXT: .b8 53
+; CHECK-NEXT: .b8 95
+; CHECK-NEXT: .b8 95
+; CHECK-NEXT: .b8 99
+; CHECK-NEXT: .b8 117
+; CHECK-NEXT: .b8 100
+; CHECK-NEXT: .b8 97
+; CHECK-NEXT: .b8 95
+; CHECK-NEXT: .b8 98
+; CHECK-NEXT: .b8 117
+; CHECK-NEXT: .b8 105
+; CHECK-NEXT: .b8 108
+; CHECK-NEXT: .b8 116
+; CHECK-NEXT: .b8 105
+; CHECK-NEXT: .b8 110
+; CHECK-NEXT: .b8 95
+; CHECK-NEXT: .b8 98
+; CHECK-NEXT: .b8 108
+; CHECK-NEXT: .b8 111
+; CHECK-NEXT: .b8 99
+; CHECK-NEXT: .b8 107
+; CHECK-NEXT: .b8 68
+; CHECK-NEXT: .b8 105
+; CHECK-NEXT: .b8 109
+; CHECK-NEXT: .b8 95
+; CHECK-NEXT: .b8 116
+; CHECK-NEXT: .b8 97
+; CHECK-NEXT: .b8 83
+; CHECK-NEXT: .b8 69
+; CHECK-NEXT: .b8 82
+; CHECK-NEXT: .b8 75
+; CHECK-NEXT: .b8 83
+; CHECK-NEXT: .b8 95
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 111                                 // DW_AT_name
+; CHECK-NEXT: .b8 112
+; CHECK-NEXT: .b8 101
+; CHECK-NEXT: .b8 114
+; CHECK-NEXT: .b8 97
+; CHECK-NEXT: .b8 116
+; CHECK-NEXT: .b8 111
+; CHECK-NEXT: .b8 114
+; CHECK-NEXT: .b8 61
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 2                                   // DW_AT_decl_file
+; CHECK-NEXT: .b8 96                                  // DW_AT_decl_line
+; CHECK-NEXT: .b8 1                                   // DW_AT_declaration
+; CHECK-NEXT: .b8 1                                   // DW_AT_external
+; CHECK-NEXT: .b8 3                                   // DW_AT_accessibility
+; CHECK-NEXT:                                         // DW_ACCESS_private
+; CHECK-NEXT: .b8 5                                   // Abbrev [5] 0x495:0x6 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 1425                               // DW_AT_type
+; CHECK-NEXT: .b8 1                                   // DW_AT_artificial
+; CHECK-NEXT: .b8 7                                   // Abbrev [7] 0x49b:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 1440                               // DW_AT_type
+; CHECK-NEXT: .b8 0                                   // End Of Children Mark
+; CHECK-NEXT: .b8 9                                   // Abbrev [9] 0x4a1:0x3f DW_TAG_subprogram
+; CHECK-NEXT: .b8 95                                  // DW_AT_MIPS_linkage_name
+; CHECK-NEXT: .b8 90
+; CHECK-NEXT: .b8 78
+; CHECK-NEXT: .b8 75
+; CHECK-NEXT: .b8 50
+; CHECK-NEXT: .b8 53
+; CHECK-NEXT: .b8 95
+; CHECK-NEXT: .b8 95
+; CHECK-NEXT: .b8 99
+; CHECK-NEXT: .b8 117
+; CHECK-NEXT: .b8 100
+; CHECK-NEXT: .b8 97
+; CHECK-NEXT: .b8 95
+; CHECK-NEXT: .b8 98
+; CHECK-NEXT: .b8 117
+; CHECK-NEXT: .b8 105
+; CHECK-NEXT: .b8 108
+; CHECK-NEXT: .b8 116
+; CHECK-NEXT: .b8 105
+; CHECK-NEXT: .b8 110
+; CHECK-NEXT: .b8 95
+; CHECK-NEXT: .b8 98
+; CHECK-NEXT: .b8 108
+; CHECK-NEXT: .b8 111
+; CHECK-NEXT: .b8 99
+; CHECK-NEXT: .b8 107
+; CHECK-NEXT: .b8 68
+; CHECK-NEXT: .b8 105
+; CHECK-NEXT: .b8 109
+; CHECK-NEXT: .b8 95
+; CHECK-NEXT: .b8 116
+; CHECK-NEXT: .b8 97
+; CHECK-NEXT: .b8 100
+; CHECK-NEXT: .b8 69
+; CHECK-NEXT: .b8 118
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 111                                 // DW_AT_name
+; CHECK-NEXT: .b8 112
+; CHECK-NEXT: .b8 101
+; CHECK-NEXT: .b8 114
+; CHECK-NEXT: .b8 97
+; CHECK-NEXT: .b8 116
+; CHECK-NEXT: .b8 111
+; CHECK-NEXT: .b8 114
+; CHECK-NEXT: .b8 38
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 2                                   // DW_AT_decl_file
+; CHECK-NEXT: .b8 96                                  // DW_AT_decl_line
+; CHECK-NEXT: .b32 1445                               // DW_AT_type
+; CHECK-NEXT: .b8 1                                   // DW_AT_declaration
+; CHECK-NEXT: .b8 1                                   // DW_AT_external
+; CHECK-NEXT: .b8 3                                   // DW_AT_accessibility
+; CHECK-NEXT:                                         // DW_ACCESS_private
+; CHECK-NEXT: .b8 5                                   // Abbrev [5] 0x4d9:0x6 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 1425                               // DW_AT_type
+; CHECK-NEXT: .b8 1                                   // DW_AT_artificial
+; CHECK-NEXT: .b8 0                                   // End Of Children Mark
+; CHECK-NEXT: .b8 0                                   // End Of Children Mark
+; CHECK-NEXT: .b8 16                                  // Abbrev [16] 0x4e1:0x9d DW_TAG_structure_type
+; CHECK-NEXT: .b8 100                                 // DW_AT_name
+; CHECK-NEXT: .b8 105
+; CHECK-NEXT: .b8 109
+; CHECK-NEXT: .b8 51
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 12                                  // DW_AT_byte_size
+; CHECK-NEXT: .b8 3                                   // DW_AT_decl_file
+; CHECK-NEXT: .b8 161                                 // DW_AT_decl_line
+; CHECK-NEXT: .b8 1
+; CHECK-NEXT: .b8 17                                  // Abbrev [17] 0x4eb:0xd DW_TAG_member
+; CHECK-NEXT: .b8 120                                 // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b32 603                                // DW_AT_type
+; CHECK-NEXT: .b8 3                                   // DW_AT_decl_file
+; CHECK-NEXT: .b8 163                                 // DW_AT_decl_line
+; CHECK-NEXT: .b8 1
+; CHECK-NEXT: .b8 2                                   // DW_AT_data_member_location
+; CHECK-NEXT: .b8 35
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 17                                  // Abbrev [17] 0x4f8:0xd DW_TAG_member
+; CHECK-NEXT: .b8 121                                 // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b32 603                                // DW_AT_type
+; CHECK-NEXT: .b8 3                                   // DW_AT_decl_file
+; CHECK-NEXT: .b8 163                                 // DW_AT_decl_line
+; CHECK-NEXT: .b8 1
+; CHECK-NEXT: .b8 2                                   // DW_AT_data_member_location
+; CHECK-NEXT: .b8 35
+; CHECK-NEXT: .b8 4
+; CHECK-NEXT: .b8 17                                  // Abbrev [17] 0x505:0xd DW_TAG_member
+; CHECK-NEXT: .b8 122                                 // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b32 603                                // DW_AT_type
+; CHECK-NEXT: .b8 3                                   // DW_AT_decl_file
+; CHECK-NEXT: .b8 163                                 // DW_AT_decl_line
+; CHECK-NEXT: .b8 1
+; CHECK-NEXT: .b8 2                                   // DW_AT_data_member_location
+; CHECK-NEXT: .b8 35
+; CHECK-NEXT: .b8 8
+; CHECK-NEXT: .b8 18                                  // Abbrev [18] 0x512:0x21 DW_TAG_subprogram
+; CHECK-NEXT: .b8 100                                 // DW_AT_name
+; CHECK-NEXT: .b8 105
+; CHECK-NEXT: .b8 109
+; CHECK-NEXT: .b8 51
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 3                                   // DW_AT_decl_file
+; CHECK-NEXT: .b8 165                                 // DW_AT_decl_line
+; CHECK-NEXT: .b8 1
+; CHECK-NEXT: .b8 1                                   // DW_AT_declaration
+; CHECK-NEXT: .b8 1                                   // DW_AT_external
+; CHECK-NEXT: .b8 5                                   // Abbrev [5] 0x51d:0x6 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 1406                               // DW_AT_type
+; CHECK-NEXT: .b8 1                                   // DW_AT_artificial
+; CHECK-NEXT: .b8 7                                   // Abbrev [7] 0x523:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 603                                // DW_AT_type
+; CHECK-NEXT: .b8 7                                   // Abbrev [7] 0x528:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 603                                // DW_AT_type
+; CHECK-NEXT: .b8 7                                   // Abbrev [7] 0x52d:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 603                                // DW_AT_type
+; CHECK-NEXT: .b8 0                                   // End Of Children Mark
+; CHECK-NEXT: .b8 18                                  // Abbrev [18] 0x533:0x17 DW_TAG_subprogram
+; CHECK-NEXT: .b8 100                                 // DW_AT_name
+; CHECK-NEXT: .b8 105
+; CHECK-NEXT: .b8 109
+; CHECK-NEXT: .b8 51
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 3                                   // DW_AT_decl_file
+; CHECK-NEXT: .b8 166                                 // DW_AT_decl_line
+; CHECK-NEXT: .b8 1
+; CHECK-NEXT: .b8 1                                   // DW_AT_declaration
+; CHECK-NEXT: .b8 1                                   // DW_AT_external
+; CHECK-NEXT: .b8 5                                   // Abbrev [5] 0x53e:0x6 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 1406                               // DW_AT_type
+; CHECK-NEXT: .b8 1                                   // DW_AT_artificial
+; CHECK-NEXT: .b8 7                                   // Abbrev [7] 0x544:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 1411                               // DW_AT_type
+; CHECK-NEXT: .b8 0                                   // End Of Children Mark
+; CHECK-NEXT: .b8 19                                  // Abbrev [19] 0x54a:0x33 DW_TAG_subprogram
+; CHECK-NEXT: .b8 95                                  // DW_AT_MIPS_linkage_name
+; CHECK-NEXT: .b8 90
+; CHECK-NEXT: .b8 78
+; CHECK-NEXT: .b8 52
+; CHECK-NEXT: .b8 100
+; CHECK-NEXT: .b8 105
+; CHECK-NEXT: .b8 109
+; CHECK-NEXT: .b8 51
+; CHECK-NEXT: .b8 99
+; CHECK-NEXT: .b8 118
+; CHECK-NEXT: .b8 53
+; CHECK-NEXT: .b8 117
+; CHECK-NEXT: .b8 105
+; CHECK-NEXT: .b8 110
+; CHECK-NEXT: .b8 116
+; CHECK-NEXT: .b8 51
+; CHECK-NEXT: .b8 69
+; CHECK-NEXT: .b8 118
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 111                                 // DW_AT_name
+; CHECK-NEXT: .b8 112
+; CHECK-NEXT: .b8 101
+; CHECK-NEXT: .b8 114
+; CHECK-NEXT: .b8 97
+; CHECK-NEXT: .b8 116
+; CHECK-NEXT: .b8 111
+; CHECK-NEXT: .b8 114
+; CHECK-NEXT: .b8 32
+; CHECK-NEXT: .b8 117
+; CHECK-NEXT: .b8 105
+; CHECK-NEXT: .b8 110
+; CHECK-NEXT: .b8 116
+; CHECK-NEXT: .b8 51
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 3                                   // DW_AT_decl_file
+; CHECK-NEXT: .b8 167                                 // DW_AT_decl_line
+; CHECK-NEXT: .b8 1
+; CHECK-NEXT: .b32 1411                               // DW_AT_type
+; CHECK-NEXT: .b8 1                                   // DW_AT_declaration
+; CHECK-NEXT: .b8 1                                   // DW_AT_external
+; CHECK-NEXT: .b8 5                                   // Abbrev [5] 0x576:0x6 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 1406                               // DW_AT_type
+; CHECK-NEXT: .b8 1                                   // DW_AT_artificial
+; CHECK-NEXT: .b8 0                                   // End Of Children Mark
+; CHECK-NEXT: .b8 0                                   // End Of Children Mark
+; CHECK-NEXT: .b8 12                                  // Abbrev [12] 0x57e:0x5 DW_TAG_pointer_type
+; CHECK-NEXT: .b32 1249                               // DW_AT_type
+; CHECK-NEXT: .b8 20                                  // Abbrev [20] 0x583:0xe DW_TAG_typedef
+; CHECK-NEXT: .b32 619                                // DW_AT_type
+; CHECK-NEXT: .b8 117                                 // DW_AT_name
+; CHECK-NEXT: .b8 105
+; CHECK-NEXT: .b8 110
+; CHECK-NEXT: .b8 116
+; CHECK-NEXT: .b8 51
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 3                                   // DW_AT_decl_file
+; CHECK-NEXT: .b8 127                                 // DW_AT_decl_line
+; CHECK-NEXT: .b8 1
+; CHECK-NEXT: .b8 12                                  // Abbrev [12] 0x591:0x5 DW_TAG_pointer_type
+; CHECK-NEXT: .b32 1430                               // DW_AT_type
+; CHECK-NEXT: .b8 13                                  // Abbrev [13] 0x596:0x5 DW_TAG_const_type
+; CHECK-NEXT: .b32 697                                // DW_AT_type
+; CHECK-NEXT: .b8 12                                  // Abbrev [12] 0x59b:0x5 DW_TAG_pointer_type
+; CHECK-NEXT: .b32 697                                // DW_AT_type
+; CHECK-NEXT: .b8 14                                  // Abbrev [14] 0x5a0:0x5 DW_TAG_reference_type
+; CHECK-NEXT: .b32 1430                               // DW_AT_type
+; CHECK-NEXT: .b8 12                                  // Abbrev [12] 0x5a5:0x5 DW_TAG_pointer_type
+; CHECK-NEXT: .b32 697                                // DW_AT_type
+; CHECK-NEXT: .b8 15                                  // Abbrev [15] 0x5aa:0x6 DW_TAG_subprogram
+; CHECK-NEXT: .b32 727                                // DW_AT_specification
+; CHECK-NEXT: .b8 1                                   // DW_AT_inline
+; CHECK-NEXT: .b8 2                                   // Abbrev [2] 0x5b0:0x233 DW_TAG_structure_type
+; CHECK-NEXT: .b8 95                                  // DW_AT_name
+; CHECK-NEXT: .b8 95
+; CHECK-NEXT: .b8 99
+; CHECK-NEXT: .b8 117
+; CHECK-NEXT: .b8 100
+; CHECK-NEXT: .b8 97
+; CHECK-NEXT: .b8 95
+; CHECK-NEXT: .b8 98
+; CHECK-NEXT: .b8 117
+; CHECK-NEXT: .b8 105
+; CHECK-NEXT: .b8 108
+; CHECK-NEXT: .b8 116
+; CHECK-NEXT: .b8 105
+; CHECK-NEXT: .b8 110
+; CHECK-NEXT: .b8 95
+; CHECK-NEXT: .b8 116
+; CHECK-NEXT: .b8 104
+; CHECK-NEXT: .b8 114
+; CHECK-NEXT: .b8 101
+; CHECK-NEXT: .b8 97
+; CHECK-NEXT: .b8 100
+; CHECK-NEXT: .b8 73
+; CHECK-NEXT: .b8 100
+; CHECK-NEXT: .b8 120
+; CHECK-NEXT: .b8 95
+; CHECK-NEXT: .b8 116
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 1                                   // DW_AT_byte_size
+; CHECK-NEXT: .b8 2                                   // DW_AT_decl_file
+; CHECK-NEXT: .b8 66                                  // DW_AT_decl_line
+; CHECK-NEXT: .b8 3                                   // Abbrev [3] 0x5cf:0x50 DW_TAG_subprogram
+; CHECK-NEXT: .b8 95                                  // DW_AT_MIPS_linkage_name
+; CHECK-NEXT: .b8 90
+; CHECK-NEXT: .b8 78
+; CHECK-NEXT: .b8 50
+; CHECK-NEXT: .b8 54
+; CHECK-NEXT: .b8 95
+; CHECK-NEXT: .b8 95
+; CHECK-NEXT: .b8 99
+; CHECK-NEXT: .b8 117
+; CHECK-NEXT: .b8 100
+; CHECK-NEXT: .b8 97
+; CHECK-NEXT: .b8 95
+; CHECK-NEXT: .b8 98
+; CHECK-NEXT: .b8 117
+; CHECK-NEXT: .b8 105
+; CHECK-NEXT: .b8 108
+; CHECK-NEXT: .b8 116
+; CHECK-NEXT: .b8 105
+; CHECK-NEXT: .b8 110
+; CHECK-NEXT: .b8 95
+; CHECK-NEXT: .b8 116
+; CHECK-NEXT: .b8 104
+; CHECK-NEXT: .b8 114
+; CHECK-NEXT: .b8 101
+; CHECK-NEXT: .b8 97
+; CHECK-NEXT: .b8 100
+; CHECK-NEXT: .b8 73
+; CHECK-NEXT: .b8 100
+; CHECK-NEXT: .b8 120
+; CHECK-NEXT: .b8 95
+; CHECK-NEXT: .b8 116
+; CHECK-NEXT: .b8 49
+; CHECK-NEXT: .b8 55
+; CHECK-NEXT: .b8 95
+; CHECK-NEXT: .b8 95
+; CHECK-NEXT: .b8 102
+; CHECK-NEXT: .b8 101
+; CHECK-NEXT: .b8 116
+; CHECK-NEXT: .b8 99
+; CHECK-NEXT: .b8 104
+; CHECK-NEXT: .b8 95
+; CHECK-NEXT: .b8 98
+; CHECK-NEXT: .b8 117
+; CHECK-NEXT: .b8 105
+; CHECK-NEXT: .b8 108
+; CHECK-NEXT: .b8 116
+; CHECK-NEXT: .b8 105
+; CHECK-NEXT: .b8 110
+; CHECK-NEXT: .b8 95
+; CHECK-NEXT: .b8 120
+; CHECK-NEXT: .b8 69
+; CHECK-NEXT: .b8 118
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 95                                  // DW_AT_name
+; CHECK-NEXT: .b8 95
+; CHECK-NEXT: .b8 102
+; CHECK-NEXT: .b8 101
+; CHECK-NEXT: .b8 116
+; CHECK-NEXT: .b8 99
+; CHECK-NEXT: .b8 104
+; CHECK-NEXT: .b8 95
+; CHECK-NEXT: .b8 98
+; CHECK-NEXT: .b8 117
+; CHECK-NEXT: .b8 105
+; CHECK-NEXT: .b8 108
+; CHECK-NEXT: .b8 116
+; CHECK-NEXT: .b8 105
+; CHECK-NEXT: .b8 110
+; CHECK-NEXT: .b8 95
+; CHECK-NEXT: .b8 120
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 2                                   // DW_AT_decl_file
+; CHECK-NEXT: .b8 67                                  // DW_AT_decl_line
+; CHECK-NEXT: .b32 603                                // DW_AT_type
+; CHECK-NEXT: .b8 1                                   // DW_AT_declaration
+; CHECK-NEXT: .b8 1                                   // DW_AT_external
+; CHECK-NEXT: .b8 3                                   // Abbrev [3] 0x61f:0x50 DW_TAG_subprogram
+; CHECK-NEXT: .b8 95                                  // DW_AT_MIPS_linkage_name
+; CHECK-NEXT: .b8 90
+; CHECK-NEXT: .b8 78
+; CHECK-NEXT: .b8 50
+; CHECK-NEXT: .b8 54
+; CHECK-NEXT: .b8 95
+; CHECK-NEXT: .b8 95
+; CHECK-NEXT: .b8 99
+; CHECK-NEXT: .b8 117
+; CHECK-NEXT: .b8 100
+; CHECK-NEXT: .b8 97
+; CHECK-NEXT: .b8 95
+; CHECK-NEXT: .b8 98
+; CHECK-NEXT: .b8 117
+; CHECK-NEXT: .b8 105
+; CHECK-NEXT: .b8 108
+; CHECK-NEXT: .b8 116
+; CHECK-NEXT: .b8 105
+; CHECK-NEXT: .b8 110
+; CHECK-NEXT: .b8 95
+; CHECK-NEXT: .b8 116
+; CHECK-NEXT: .b8 104
+; CHECK-NEXT: .b8 114
+; CHECK-NEXT: .b8 101
+; CHECK-NEXT: .b8 97
+; CHECK-NEXT: .b8 100
+; CHECK-NEXT: .b8 73
+; CHECK-NEXT: .b8 100
+; CHECK-NEXT: .b8 120
+; CHECK-NEXT: .b8 95
+; CHECK-NEXT: .b8 116
+; CHECK-NEXT: .b8 49
+; CHECK-NEXT: .b8 55
+; CHECK-NEXT: .b8 95
+; CHECK-NEXT: .b8 95
+; CHECK-NEXT: .b8 102
+; CHECK-NEXT: .b8 101
+; CHECK-NEXT: .b8 116
+; CHECK-NEXT: .b8 99
+; CHECK-NEXT: .b8 104
+; CHECK-NEXT: .b8 95
+; CHECK-NEXT: .b8 98
+; CHECK-NEXT: .b8 117
+; CHECK-NEXT: .b8 105
+; CHECK-NEXT: .b8 108
+; CHECK-NEXT: .b8 116
+; CHECK-NEXT: .b8 105
+; CHECK-NEXT: .b8 110
+; CHECK-NEXT: .b8 95
+; CHECK-NEXT: .b8 121
+; CHECK-NEXT: .b8 69
+; CHECK-NEXT: .b8 118
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 95                                  // DW_AT_name
+; CHECK-NEXT: .b8 95
+; CHECK-NEXT: .b8 102
+; CHECK-NEXT: .b8 101
+; CHECK-NEXT: .b8 116
+; CHECK-NEXT: .b8 99
+; CHECK-NEXT: .b8 104
+; CHECK-NEXT: .b8 95
+; CHECK-NEXT: .b8 98
+; CHECK-NEXT: .b8 117
+; CHECK-NEXT: .b8 105
+; CHECK-NEXT: .b8 108
+; CHECK-NEXT: .b8 116
+; CHECK-NEXT: .b8 105
+; CHECK-NEXT: .b8 110
+; CHECK-NEXT: .b8 95
+; CHECK-NEXT: .b8 121
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 2                                   // DW_AT_decl_file
+; CHECK-NEXT: .b8 68                                  // DW_AT_decl_line
+; CHECK-NEXT: .b32 603                                // DW_AT_type
+; CHECK-NEXT: .b8 1                                   // DW_AT_declaration
+; CHECK-NEXT: .b8 1                                   // DW_AT_external
+; CHECK-NEXT: .b8 3                                   // Abbrev [3] 0x66f:0x50 DW_TAG_subprogram
+; CHECK-NEXT: .b8 95                                  // DW_AT_MIPS_linkage_name
+; CHECK-NEXT: .b8 90
+; CHECK-NEXT: .b8 78
+; CHECK-NEXT: .b8 50
+; CHECK-NEXT: .b8 54
+; CHECK-NEXT: .b8 95
+; CHECK-NEXT: .b8 95
+; CHECK-NEXT: .b8 99
+; CHECK-NEXT: .b8 117
+; CHECK-NEXT: .b8 100
+; CHECK-NEXT: .b8 97
+; CHECK-NEXT: .b8 95
+; CHECK-NEXT: .b8 98
+; CHECK-NEXT: .b8 117
+; CHECK-NEXT: .b8 105
+; CHECK-NEXT: .b8 108
+; CHECK-NEXT: .b8 116
+; CHECK-NEXT: .b8 105
+; CHECK-NEXT: .b8 110
+; CHECK-NEXT: .b8 95
+; CHECK-NEXT: .b8 116
+; CHECK-NEXT: .b8 104
+; CHECK-NEXT: .b8 114
+; CHECK-NEXT: .b8 101
+; CHECK-NEXT: .b8 97
+; CHECK-NEXT: .b8 100
+; CHECK-NEXT: .b8 73
+; CHECK-NEXT: .b8 100
+; CHECK-NEXT: .b8 120
+; CHECK-NEXT: .b8 95
+; CHECK-NEXT: .b8 116
+; CHECK-NEXT: .b8 49
+; CHECK-NEXT: .b8 55
+; CHECK-NEXT: .b8 95
+; CHECK-NEXT: .b8 95
+; CHECK-NEXT: .b8 102
+; CHECK-NEXT: .b8 101
+; CHECK-NEXT: .b8 116
+; CHECK-NEXT: .b8 99
+; CHECK-NEXT: .b8 104
+; CHECK-NEXT: .b8 95
+; CHECK-NEXT: .b8 98
+; CHECK-NEXT: .b8 117
+; CHECK-NEXT: .b8 105
+; CHECK-NEXT: .b8 108
+; CHECK-NEXT: .b8 116
+; CHECK-NEXT: .b8 105
+; CHECK-NEXT: .b8 110
+; CHECK-NEXT: .b8 95
+; CHECK-NEXT: .b8 122
+; CHECK-NEXT: .b8 69
+; CHECK-NEXT: .b8 118
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 95                                  // DW_AT_name
+; CHECK-NEXT: .b8 95
+; CHECK-NEXT: .b8 102
+; CHECK-NEXT: .b8 101
+; CHECK-NEXT: .b8 116
+; CHECK-NEXT: .b8 99
+; CHECK-NEXT: .b8 104
+; CHECK-NEXT: .b8 95
+; CHECK-NEXT: .b8 98
+; CHECK-NEXT: .b8 117
+; CHECK-NEXT: .b8 105
+; CHECK-NEXT: .b8 108
+; CHECK-NEXT: .b8 116
+; CHECK-NEXT: .b8 105
+; CHECK-NEXT: .b8 110
+; CHECK-NEXT: .b8 95
+; CHECK-NEXT: .b8 122
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 2                                   // DW_AT_decl_file
+; CHECK-NEXT: .b8 69                                  // DW_AT_decl_line
+; CHECK-NEXT: .b32 603                                // DW_AT_type
+; CHECK-NEXT: .b8 1                                   // DW_AT_declaration
+; CHECK-NEXT: .b8 1                                   // DW_AT_external
+; CHECK-NEXT: .b8 4                                   // Abbrev [4] 0x6bf:0x4a DW_TAG_subprogram
+; CHECK-NEXT: .b8 95                                  // DW_AT_MIPS_linkage_name
+; CHECK-NEXT: .b8 90
+; CHECK-NEXT: .b8 78
+; CHECK-NEXT: .b8 75
+; CHECK-NEXT: .b8 50
+; CHECK-NEXT: .b8 54
+; CHECK-NEXT: .b8 95
+; CHECK-NEXT: .b8 95
+; CHECK-NEXT: .b8 99
+; CHECK-NEXT: .b8 117
+; CHECK-NEXT: .b8 100
+; CHECK-NEXT: .b8 97
+; CHECK-NEXT: .b8 95
+; CHECK-NEXT: .b8 98
+; CHECK-NEXT: .b8 117
+; CHECK-NEXT: .b8 105
+; CHECK-NEXT: .b8 108
+; CHECK-NEXT: .b8 116
+; CHECK-NEXT: .b8 105
+; CHECK-NEXT: .b8 110
+; CHECK-NEXT: .b8 95
+; CHECK-NEXT: .b8 116
+; CHECK-NEXT: .b8 104
+; CHECK-NEXT: .b8 114
+; CHECK-NEXT: .b8 101
+; CHECK-NEXT: .b8 97
+; CHECK-NEXT: .b8 100
+; CHECK-NEXT: .b8 73
+; CHECK-NEXT: .b8 100
+; CHECK-NEXT: .b8 120
+; CHECK-NEXT: .b8 95
+; CHECK-NEXT: .b8 116
+; CHECK-NEXT: .b8 99
+; CHECK-NEXT: .b8 118
+; CHECK-NEXT: .b8 53
+; CHECK-NEXT: .b8 117
+; CHECK-NEXT: .b8 105
+; CHECK-NEXT: .b8 110
+; CHECK-NEXT: .b8 116
+; CHECK-NEXT: .b8 51
+; CHECK-NEXT: .b8 69
+; CHECK-NEXT: .b8 118
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 111                                 // DW_AT_name
+; CHECK-NEXT: .b8 112
+; CHECK-NEXT: .b8 101
+; CHECK-NEXT: .b8 114
+; CHECK-NEXT: .b8 97
+; CHECK-NEXT: .b8 116
+; CHECK-NEXT: .b8 111
+; CHECK-NEXT: .b8 114
+; CHECK-NEXT: .b8 32
+; CHECK-NEXT: .b8 117
+; CHECK-NEXT: .b8 105
+; CHECK-NEXT: .b8 110
+; CHECK-NEXT: .b8 116
+; CHECK-NEXT: .b8 51
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 2                                   // DW_AT_decl_file
+; CHECK-NEXT: .b8 72                                  // DW_AT_decl_line
+; CHECK-NEXT: .b32 619                                // DW_AT_type
+; CHECK-NEXT: .b8 1                                   // DW_AT_declaration
+; CHECK-NEXT: .b8 1                                   // DW_AT_external
+; CHECK-NEXT: .b8 5                                   // Abbrev [5] 0x702:0x6 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 2019                               // DW_AT_type
+; CHECK-NEXT: .b8 1                                   // DW_AT_artificial
+; CHECK-NEXT: .b8 0                                   // End Of Children Mark
+; CHECK-NEXT: .b8 6                                   // Abbrev [6] 0x709:0x28 DW_TAG_subprogram
+; CHECK-NEXT: .b8 95                                  // DW_AT_name
+; CHECK-NEXT: .b8 95
+; CHECK-NEXT: .b8 99
+; CHECK-NEXT: .b8 117
+; CHECK-NEXT: .b8 100
+; CHECK-NEXT: .b8 97
+; CHECK-NEXT: .b8 95
+; CHECK-NEXT: .b8 98
+; CHECK-NEXT: .b8 117
+; CHECK-NEXT: .b8 105
+; CHECK-NEXT: .b8 108
+; CHECK-NEXT: .b8 116
+; CHECK-NEXT: .b8 105
+; CHECK-NEXT: .b8 110
+; CHECK-NEXT: .b8 95
+; CHECK-NEXT: .b8 116
+; CHECK-NEXT: .b8 104
+; CHECK-NEXT: .b8 114
+; CHECK-NEXT: .b8 101
+; CHECK-NEXT: .b8 97
+; CHECK-NEXT: .b8 100
+; CHECK-NEXT: .b8 73
+; CHECK-NEXT: .b8 100
+; CHECK-NEXT: .b8 120
+; CHECK-NEXT: .b8 95
+; CHECK-NEXT: .b8 116
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 2                                   // DW_AT_decl_file
+; CHECK-NEXT: .b8 74                                  // DW_AT_decl_line
+; CHECK-NEXT: .b8 1                                   // DW_AT_declaration
+; CHECK-NEXT: .b8 1                                   // DW_AT_external
+; CHECK-NEXT: .b8 3                                   // DW_AT_accessibility
+; CHECK-NEXT:                                         // DW_ACCESS_private
+; CHECK-NEXT: .b8 5                                   // Abbrev [5] 0x72a:0x6 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 2029                               // DW_AT_type
+; CHECK-NEXT: .b8 1                                   // DW_AT_artificial
+; CHECK-NEXT: .b8 0                                   // End Of Children Mark
+; CHECK-NEXT: .b8 6                                   // Abbrev [6] 0x731:0x2d DW_TAG_subprogram
+; CHECK-NEXT: .b8 95                                  // DW_AT_name
+; CHECK-NEXT: .b8 95
+; CHECK-NEXT: .b8 99
+; CHECK-NEXT: .b8 117
+; CHECK-NEXT: .b8 100
+; CHECK-NEXT: .b8 97
+; CHECK-NEXT: .b8 95
+; CHECK-NEXT: .b8 98
+; CHECK-NEXT: .b8 117
+; CHECK-NEXT: .b8 105
+; CHECK-NEXT: .b8 108
+; CHECK-NEXT: .b8 116
+; CHECK-NEXT: .b8 105
+; CHECK-NEXT: .b8 110
+; CHECK-NEXT: .b8 95
+; CHECK-NEXT: .b8 116
+; CHECK-NEXT: .b8 104
+; CHECK-NEXT: .b8 114
+; CHECK-NEXT: .b8 101
+; CHECK-NEXT: .b8 97
+; CHECK-NEXT: .b8 100
+; CHECK-NEXT: .b8 73
+; CHECK-NEXT: .b8 100
+; CHECK-NEXT: .b8 120
+; CHECK-NEXT: .b8 95
+; CHECK-NEXT: .b8 116
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 2                                   // DW_AT_decl_file
+; CHECK-NEXT: .b8 74                                  // DW_AT_decl_line
+; CHECK-NEXT: .b8 1                                   // DW_AT_declaration
+; CHECK-NEXT: .b8 1                                   // DW_AT_external
+; CHECK-NEXT: .b8 3                                   // DW_AT_accessibility
+; CHECK-NEXT:                                         // DW_ACCESS_private
+; CHECK-NEXT: .b8 5                                   // Abbrev [5] 0x752:0x6 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 2029                               // DW_AT_type
+; CHECK-NEXT: .b8 1                                   // DW_AT_artificial
+; CHECK-NEXT: .b8 7                                   // Abbrev [7] 0x758:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 2034                               // DW_AT_type
+; CHECK-NEXT: .b8 0                                   // End Of Children Mark
+; CHECK-NEXT: .b8 8                                   // Abbrev [8] 0x75e:0x44 DW_TAG_subprogram
+; CHECK-NEXT: .b8 95                                  // DW_AT_MIPS_linkage_name
+; CHECK-NEXT: .b8 90
+; CHECK-NEXT: .b8 78
+; CHECK-NEXT: .b8 75
+; CHECK-NEXT: .b8 50
+; CHECK-NEXT: .b8 54
+; CHECK-NEXT: .b8 95
+; CHECK-NEXT: .b8 95
+; CHECK-NEXT: .b8 99
+; CHECK-NEXT: .b8 117
+; CHECK-NEXT: .b8 100
+; CHECK-NEXT: .b8 97
+; CHECK-NEXT: .b8 95
+; CHECK-NEXT: .b8 98
+; CHECK-NEXT: .b8 117
+; CHECK-NEXT: .b8 105
+; CHECK-NEXT: .b8 108
+; CHECK-NEXT: .b8 116
+; CHECK-NEXT: .b8 105
+; CHECK-NEXT: .b8 110
+; CHECK-NEXT: .b8 95
+; CHECK-NEXT: .b8 116
+; CHECK-NEXT: .b8 104
+; CHECK-NEXT: .b8 114
+; CHECK-NEXT: .b8 101
+; CHECK-NEXT: .b8 97
+; CHECK-NEXT: .b8 100
+; CHECK-NEXT: .b8 73
+; CHECK-NEXT: .b8 100
+; CHECK-NEXT: .b8 120
+; CHECK-NEXT: .b8 95
+; CHECK-NEXT: .b8 116
+; CHECK-NEXT: .b8 97
+; CHECK-NEXT: .b8 83
+; CHECK-NEXT: .b8 69
+; CHECK-NEXT: .b8 82
+; CHECK-NEXT: .b8 75
+; CHECK-NEXT: .b8 83
+; CHECK-NEXT: .b8 95
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 111                                 // DW_AT_name
+; CHECK-NEXT: .b8 112
+; CHECK-NEXT: .b8 101
+; CHECK-NEXT: .b8 114
+; CHECK-NEXT: .b8 97
+; CHECK-NEXT: .b8 116
+; CHECK-NEXT: .b8 111
+; CHECK-NEXT: .b8 114
+; CHECK-NEXT: .b8 61
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 2                                   // DW_AT_decl_file
+; CHECK-NEXT: .b8 74                                  // DW_AT_decl_line
+; CHECK-NEXT: .b8 1                                   // DW_AT_declaration
+; CHECK-NEXT: .b8 1                                   // DW_AT_external
+; CHECK-NEXT: .b8 3                                   // DW_AT_accessibility
+; CHECK-NEXT:                                         // DW_ACCESS_private
+; CHECK-NEXT: .b8 5                                   // Abbrev [5] 0x796:0x6 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 2019                               // DW_AT_type
+; CHECK-NEXT: .b8 1                                   // DW_AT_artificial
+; CHECK-NEXT: .b8 7                                   // Abbrev [7] 0x79c:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 2034                               // DW_AT_type
+; CHECK-NEXT: .b8 0                                   // End Of Children Mark
+; CHECK-NEXT: .b8 9                                   // Abbrev [9] 0x7a2:0x40 DW_TAG_subprogram
+; CHECK-NEXT: .b8 95                                  // DW_AT_MIPS_linkage_name
+; CHECK-NEXT: .b8 90
+; CHECK-NEXT: .b8 78
+; CHECK-NEXT: .b8 75
+; CHECK-NEXT: .b8 50
+; CHECK-NEXT: .b8 54
+; CHECK-NEXT: .b8 95
+; CHECK-NEXT: .b8 95
+; CHECK-NEXT: .b8 99
+; CHECK-NEXT: .b8 117
+; CHECK-NEXT: .b8 100
+; CHECK-NEXT: .b8 97
+; CHECK-NEXT: .b8 95
+; CHECK-NEXT: .b8 98
+; CHECK-NEXT: .b8 117
+; CHECK-NEXT: .b8 105
+; CHECK-NEXT: .b8 108
+; CHECK-NEXT: .b8 116
+; CHECK-NEXT: .b8 105
+; CHECK-NEXT: .b8 110
+; CHECK-NEXT: .b8 95
+; CHECK-NEXT: .b8 116
+; CHECK-NEXT: .b8 104
+; CHECK-NEXT: .b8 114
+; CHECK-NEXT: .b8 101
+; CHECK-NEXT: .b8 97
+; CHECK-NEXT: .b8 100
+; CHECK-NEXT: .b8 73
+; CHECK-NEXT: .b8 100
+; CHECK-NEXT: .b8 120
+; CHECK-NEXT: .b8 95
+; CHECK-NEXT: .b8 116
+; CHECK-NEXT: .b8 97
+; CHECK-NEXT: .b8 100
+; CHECK-NEXT: .b8 69
+; CHECK-NEXT: .b8 118
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 111                                 // DW_AT_name
+; CHECK-NEXT: .b8 112
+; CHECK-NEXT: .b8 101
+; CHECK-NEXT: .b8 114
+; CHECK-NEXT: .b8 97
+; CHECK-NEXT: .b8 116
+; CHECK-NEXT: .b8 111
+; CHECK-NEXT: .b8 114
+; CHECK-NEXT: .b8 38
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 2                                   // DW_AT_decl_file
+; CHECK-NEXT: .b8 74                                  // DW_AT_decl_line
+; CHECK-NEXT: .b32 2039                               // DW_AT_type
+; CHECK-NEXT: .b8 1                                   // DW_AT_declaration
+; CHECK-NEXT: .b8 1                                   // DW_AT_external
+; CHECK-NEXT: .b8 3                                   // DW_AT_accessibility
+; CHECK-NEXT:                                         // DW_ACCESS_private
+; CHECK-NEXT: .b8 5                                   // Abbrev [5] 0x7db:0x6 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 2019                               // DW_AT_type
+; CHECK-NEXT: .b8 1                                   // DW_AT_artificial
+; CHECK-NEXT: .b8 0                                   // End Of Children Mark
+; CHECK-NEXT: .b8 0                                   // End Of Children Mark
+; CHECK-NEXT: .b8 12                                  // Abbrev [12] 0x7e3:0x5 DW_TAG_pointer_type
+; CHECK-NEXT: .b32 2024                               // DW_AT_type
+; CHECK-NEXT: .b8 13                                  // Abbrev [13] 0x7e8:0x5 DW_TAG_const_type
+; CHECK-NEXT: .b32 1456                               // DW_AT_type
+; CHECK-NEXT: .b8 12                                  // Abbrev [12] 0x7ed:0x5 DW_TAG_pointer_type
+; CHECK-NEXT: .b32 1456                               // DW_AT_type
+; CHECK-NEXT: .b8 14                                  // Abbrev [14] 0x7f2:0x5 DW_TAG_reference_type
+; CHECK-NEXT: .b32 2024                               // DW_AT_type
+; CHECK-NEXT: .b8 12                                  // Abbrev [12] 0x7f7:0x5 DW_TAG_pointer_type
+; CHECK-NEXT: .b32 1456                               // DW_AT_type
+; CHECK-NEXT: .b8 15                                  // Abbrev [15] 0x7fc:0x6 DW_TAG_subprogram
+; CHECK-NEXT: .b32 1487                               // DW_AT_specification
+; CHECK-NEXT: .b8 1                                   // DW_AT_inline
+; CHECK-NEXT: .b8 21                                  // Abbrev [21] 0x802:0x32 DW_TAG_subprogram
+; CHECK-NEXT: .b8 95                                  // DW_AT_MIPS_linkage_name
+; CHECK-NEXT: .b8 90
+; CHECK-NEXT: .b8 51
+; CHECK-NEXT: .b8 114
+; CHECK-NEXT: .b8 101
+; CHECK-NEXT: .b8 115
+; CHECK-NEXT: .b8 102
+; CHECK-NEXT: .b8 102
+; CHECK-NEXT: .b8 80
+; CHECK-NEXT: .b8 102
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 114                                 // DW_AT_name
+; CHECK-NEXT: .b8 101
+; CHECK-NEXT: .b8 115
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 1                                   // DW_AT_decl_file
+; CHECK-NEXT: .b8 3                                   // DW_AT_decl_line
+; CHECK-NEXT: .b8 1                                   // DW_AT_external
+; CHECK-NEXT: .b8 1                                   // DW_AT_inline
+; CHECK-NEXT: .b8 22                                  // Abbrev [22] 0x816:0x9 DW_TAG_formal_parameter
+; CHECK-NEXT: .b8 120                                 // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 1                                   // DW_AT_decl_file
+; CHECK-NEXT: .b8 3                                   // DW_AT_decl_line
+; CHECK-NEXT: .b32 2100                               // DW_AT_type
+; CHECK-NEXT: .b8 22                                  // Abbrev [22] 0x81f:0x9 DW_TAG_formal_parameter
+; CHECK-NEXT: .b8 121                                 // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 1                                   // DW_AT_decl_file
+; CHECK-NEXT: .b8 3                                   // DW_AT_decl_line
+; CHECK-NEXT: .b32 2100                               // DW_AT_type
+; CHECK-NEXT: .b8 22                                  // Abbrev [22] 0x828:0xb DW_TAG_formal_parameter
+; CHECK-NEXT: .b8 114                                 // DW_AT_name
+; CHECK-NEXT: .b8 101
+; CHECK-NEXT: .b8 115
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 1                                   // DW_AT_decl_file
+; CHECK-NEXT: .b8 3                                   // DW_AT_decl_line
+; CHECK-NEXT: .b32 2109                               // DW_AT_type
+; CHECK-NEXT: .b8 0                                   // End Of Children Mark
+; CHECK-NEXT: .b8 10                                  // Abbrev [10] 0x834:0x9 DW_TAG_base_type
+; CHECK-NEXT: .b8 102                                 // DW_AT_name
+; CHECK-NEXT: .b8 108
+; CHECK-NEXT: .b8 111
+; CHECK-NEXT: .b8 97
+; CHECK-NEXT: .b8 116
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 4                                   // DW_AT_encoding
+; CHECK-NEXT: .b8 4                                   // DW_AT_byte_size
+; CHECK-NEXT: .b8 12                                  // Abbrev [12] 0x83d:0x5 DW_TAG_pointer_type
+; CHECK-NEXT: .b32 2100                               // DW_AT_type
+; CHECK-NEXT: .b8 23                                  // Abbrev [23] 0x842:0xd5 DW_TAG_subprogram
+; CHECK-NEXT: .b64 $L__func_begin0                    // DW_AT_low_pc
+; CHECK-NEXT: .b64 $L__func_end0                      // DW_AT_high_pc
+; CHECK-NEXT: .b8 1                                   // DW_AT_frame_base
+; CHECK-NEXT: .b8 156
+; CHECK-NEXT: .b8 95                                  // DW_AT_MIPS_linkage_name
+; CHECK-NEXT: .b8 90
+; CHECK-NEXT: .b8 53
+; CHECK-NEXT: .b8 115
+; CHECK-NEXT: .b8 97
+; CHECK-NEXT: .b8 120
+; CHECK-NEXT: .b8 112
+; CHECK-NEXT: .b8 121
+; CHECK-NEXT: .b8 105
+; CHECK-NEXT: .b8 102
+; CHECK-NEXT: .b8 80
+; CHECK-NEXT: .b8 102
+; CHECK-NEXT: .b8 83
+; CHECK-NEXT: .b8 95
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 115                                 // DW_AT_name
+; CHECK-NEXT: .b8 97
+; CHECK-NEXT: .b8 120
+; CHECK-NEXT: .b8 112
+; CHECK-NEXT: .b8 121
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 1                                   // DW_AT_decl_file
+; CHECK-NEXT: .b8 5                                   // DW_AT_decl_line
+; CHECK-NEXT: .b8 1                                   // DW_AT_external
+; CHECK-NEXT: .b8 24                                  // Abbrev [24] 0x86d:0x10 DW_TAG_formal_parameter
+; CHECK-NEXT: .b8 2                                   // DW_AT_address_class
+; CHECK-NEXT: .b8 5                                   // DW_AT_location
+; CHECK-NEXT: .b8 144
+; CHECK-NEXT: .b8 178
+; CHECK-NEXT: .b8 228
+; CHECK-NEXT: .b8 149
+; CHECK-NEXT: .b8 1
+; CHECK-NEXT: .b8 110                                 // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 1                                   // DW_AT_decl_file
+; CHECK-NEXT: .b8 5                                   // DW_AT_decl_line
+; CHECK-NEXT: .b32 2384                               // DW_AT_type
+; CHECK-NEXT: .b8 25                                  // Abbrev [25] 0x87d:0xd DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 $L__debug_loc0                     // DW_AT_location
+; CHECK-NEXT: .b8 97                                  // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 1                                   // DW_AT_decl_file
+; CHECK-NEXT: .b8 5                                   // DW_AT_decl_line
+; CHECK-NEXT: .b32 2100                               // DW_AT_type
+; CHECK-NEXT: .b8 22                                  // Abbrev [22] 0x88a:0x9 DW_TAG_formal_parameter
+; CHECK-NEXT: .b8 120                                 // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 1                                   // DW_AT_decl_file
+; CHECK-NEXT: .b8 5                                   // DW_AT_decl_line
+; CHECK-NEXT: .b32 2109                               // DW_AT_type
+; CHECK-NEXT: .b8 22                                  // Abbrev [22] 0x893:0x9 DW_TAG_formal_parameter
+; CHECK-NEXT: .b8 121                                 // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 1                                   // DW_AT_decl_file
+; CHECK-NEXT: .b8 5                                   // DW_AT_decl_line
+; CHECK-NEXT: .b32 2109                               // DW_AT_type
+; CHECK-NEXT: .b8 26                                  // Abbrev [26] 0x89c:0xd DW_TAG_variable
+; CHECK-NEXT: .b32 $L__debug_loc1                     // DW_AT_location
+; CHECK-NEXT: .b8 105                                 // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 1                                   // DW_AT_decl_file
+; CHECK-NEXT: .b8 6                                   // DW_AT_decl_line
+; CHECK-NEXT: .b32 2384                               // DW_AT_type
+; CHECK-NEXT: .b8 27                                  // Abbrev [27] 0x8a9:0x18 DW_TAG_inlined_subroutine
+; CHECK-NEXT: .b32 691                                // DW_AT_abstract_origin
+; CHECK-NEXT: .b64 $L__tmp1                           // DW_AT_low_pc
+; CHECK-NEXT: .b64 $L__tmp2                           // DW_AT_high_pc
+; CHECK-NEXT: .b8 1                                   // DW_AT_call_file
+; CHECK-NEXT: .b8 6                                   // DW_AT_call_line
+; CHECK-NEXT: .b8 11                                  // DW_AT_call_column
+; CHECK-NEXT: .b8 27                                  // Abbrev [27] 0x8c1:0x18 DW_TAG_inlined_subroutine
+; CHECK-NEXT: .b32 1450                               // DW_AT_abstract_origin
+; CHECK-NEXT: .b64 $L__tmp2                           // DW_AT_low_pc
+; CHECK-NEXT: .b64 $L__tmp3                           // DW_AT_high_pc
+; CHECK-NEXT: .b8 1                                   // DW_AT_call_file
+; CHECK-NEXT: .b8 6                                   // DW_AT_call_line
+; CHECK-NEXT: .b8 24                                  // DW_AT_call_column
+; CHECK-NEXT: .b8 27                                  // Abbrev [27] 0x8d9:0x18 DW_TAG_inlined_subroutine
+; CHECK-NEXT: .b32 2044                               // DW_AT_abstract_origin
+; CHECK-NEXT: .b64 $L__tmp3                           // DW_AT_low_pc
+; CHECK-NEXT: .b64 $L__tmp4                           // DW_AT_high_pc
+; CHECK-NEXT: .b8 1                                   // DW_AT_call_file
+; CHECK-NEXT: .b8 6                                   // DW_AT_call_line
+; CHECK-NEXT: .b8 37                                  // DW_AT_call_column
+; CHECK-NEXT: .b8 28                                  // Abbrev [28] 0x8f1:0x25 DW_TAG_inlined_subroutine
+; CHECK-NEXT: .b32 2050                               // DW_AT_abstract_origin
+; CHECK-NEXT: .b64 $L__tmp9                           // DW_AT_low_pc
+; CHECK-NEXT: .b64 $L__tmp10                          // DW_AT_high_pc
+; CHECK-NEXT: .b8 1                                   // DW_AT_call_file
+; CHECK-NEXT: .b8 8                                   // DW_AT_call_line
+; CHECK-NEXT: .b8 5                                   // DW_AT_call_column
+; CHECK-NEXT: .b8 29                                  // Abbrev [29] 0x909:0xc DW_TAG_formal_parameter
+; CHECK-NEXT: .b8 2                                   // DW_AT_address_class
+; CHECK-NEXT: .b8 5                                   // DW_AT_location
+; CHECK-NEXT: .b8 144
+; CHECK-NEXT: .b8 179
+; CHECK-NEXT: .b8 204
+; CHECK-NEXT: .b8 149
+; CHECK-NEXT: .b8 1
+; CHECK-NEXT: .b32 2079                               // DW_AT_abstract_origin
+; CHECK-NEXT: .b8 0                                   // End Of Children Mark
+; CHECK-NEXT: .b8 0                                   // End Of Children Mark
+; CHECK-NEXT: .b8 30                                  // Abbrev [30] 0x917:0xd DW_TAG_namespace
+; CHECK-NEXT: .b8 115                                 // DW_AT_name
+; CHECK-NEXT: .b8 116
+; CHECK-NEXT: .b8 100
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 31                                  // Abbrev [31] 0x91c:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT: .b8 4                                   // DW_AT_decl_file
+; CHECK-NEXT: .b8 202                                 // DW_AT_decl_line
+; CHECK-NEXT: .b32 2340                               // DW_AT_import
+; CHECK-NEXT: .b8 0                                   // End Of Children Mark
+; CHECK-NEXT: .b8 32                                  // Abbrev [32] 0x924:0x1b DW_TAG_subprogram
+; CHECK-NEXT: .b8 95                                  // DW_AT_MIPS_linkage_name
+; CHECK-NEXT: .b8 90
+; CHECK-NEXT: .b8 76
+; CHECK-NEXT: .b8 51
+; CHECK-NEXT: .b8 97
+; CHECK-NEXT: .b8 98
+; CHECK-NEXT: .b8 115
+; CHECK-NEXT: .b8 120
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 97                                  // DW_AT_name
+; CHECK-NEXT: .b8 98
+; CHECK-NEXT: .b8 115
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 4                                   // DW_AT_decl_file
+; CHECK-NEXT: .b8 44                                  // DW_AT_decl_line
+; CHECK-NEXT: .b32 2367                               // DW_AT_type
+; CHECK-NEXT: .b8 1                                   // DW_AT_declaration
+; CHECK-NEXT: .b8 7                                   // Abbrev [7] 0x939:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 2367                               // DW_AT_type
+; CHECK-NEXT: .b8 0                                   // End Of Children Mark
+; CHECK-NEXT: .b8 10                                  // Abbrev [10] 0x93f:0x11 DW_TAG_base_type
+; CHECK-NEXT: .b8 108                                 // DW_AT_name
+; CHECK-NEXT: .b8 111
+; CHECK-NEXT: .b8 110
+; CHECK-NEXT: .b8 103
+; CHECK-NEXT: .b8 32
+; CHECK-NEXT: .b8 108
+; CHECK-NEXT: .b8 111
+; CHECK-NEXT: .b8 110
+; CHECK-NEXT: .b8 103
+; CHECK-NEXT: .b8 32
+; CHECK-NEXT: .b8 105
+; CHECK-NEXT: .b8 110
+; CHECK-NEXT: .b8 116
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 5                                   // DW_AT_encoding
+; CHECK-NEXT: .b8 8                                   // DW_AT_byte_size
+; CHECK-NEXT: .b8 10                                  // Abbrev [10] 0x950:0x7 DW_TAG_base_type
+; CHECK-NEXT: .b8 105                                 // DW_AT_name
+; CHECK-NEXT: .b8 110
+; CHECK-NEXT: .b8 116
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 5                                   // DW_AT_encoding
+; CHECK-NEXT: .b8 4                                   // DW_AT_byte_size
+; CHECK-NEXT: .b8 0                                   // End Of Children Mark
+; CHECK-NEXT: 	}
+; CHECK-NEXT: 	.section	.debug_macinfo	{	}
 ; CHECK-NOT: debug_
 
 ; Function Attrs: nounwind readnone
diff --git a/llvm/test/DebugInfo/RISCV/dwarf-riscv-relocs.ll b/llvm/test/DebugInfo/RISCV/dwarf-riscv-relocs.ll
index 99594b5e01e95..14cb4b386a369 100644
--- a/llvm/test/DebugInfo/RISCV/dwarf-riscv-relocs.ll
+++ b/llvm/test/DebugInfo/RISCV/dwarf-riscv-relocs.ll
@@ -71,7 +71,7 @@
 ; DWARF-DUMP-NEXT: ------------------ ------ ------ ------ --- ------------- ------- -------------
 ; DWARF-DUMP-NEXT: 0x0000000000000000      2      0      0   0             0       0  is_stmt
 ; DWARF-DUMP-NEXT: 0x000000000000001c      3      5      0   0             0       0  is_stmt prologue_end
-; DWARF-DUMP-NEXT: 0x0000000000000028      3      5      0   0             0       0  epilogue_begin
+; DWARF-DUMP-NEXT: 0x0000000000000020      3      5      0   0             0       0  epilogue_begin
 ; DWARF-DUMP-NEXT: 0x0000000000000030      3      5      0   0             0       0  end_sequence
 
 ; ModuleID = 'dwarf-riscv-relocs.c'
diff --git a/llvm/test/DebugInfo/X86/DW_AT_object_pointer.ll b/llvm/test/DebugInfo/X86/DW_AT_object_pointer.ll
index 30d4203466766..d9988ac31451e 100644
--- a/llvm/test/DebugInfo/X86/DW_AT_object_pointer.ll
+++ b/llvm/test/DebugInfo/X86/DW_AT_object_pointer.ll
@@ -5,15 +5,7 @@
 ; CHECK-NOT: ""
 ; CHECK: DW_TAG
 ; CHECK: DW_TAG_class_type
-; CHECK: [[DECL:0x[0-9a-f]+]]: DW_TAG_subprogram
-; CHECK:                         DW_AT_name {{.*}} "A"
-; CHECK:                         DW_AT_object_pointer [DW_FORM_ref4] 
-; CHECK-SAME:                    (cu + 0x{{[0-9a-f]*}} => {[[DECL_PARAM:0x[0-9a-f]*]]})
-; CHECK: [[DECL_PARAM]]:         DW_TAG_formal_parameter
-;
-; CHECK: DW_TAG_subprogram
-; CHECK:   DW_AT_specification [DW_FORM_ref4] (cu + {{.*}} => {[[DECL]]}
-; CHECK:   DW_AT_object_pointer [DW_FORM_ref4]     (cu + 0x{{[0-9a-f]*}} => {[[PARAM:0x[0-9a-f]*]]})
+; CHECK: DW_AT_object_pointer [DW_FORM_ref4]     (cu + 0x{{[0-9a-f]*}} => {[[PARAM:0x[0-9a-f]*]]})
 ; CHECK: [[PARAM]]:     DW_TAG_formal_parameter
 ; CHECK-NOT: DW_TAG
 ; CHECK: DW_AT_name [DW_FORM_strp]     ( .debug_str[0x{{[0-9a-f]*}}] = "this")
diff --git a/llvm/test/DebugInfo/X86/dwarf-public-names.ll b/llvm/test/DebugInfo/X86/dwarf-public-names.ll
index a484c094892d0..c2274511d4191 100644
--- a/llvm/test/DebugInfo/X86/dwarf-public-names.ll
+++ b/llvm/test/DebugInfo/X86/dwarf-public-names.ll
@@ -61,7 +61,7 @@
 
 ; Skip the output to the header of the pubnames section.
 ; LINUX: debug_pubnames
-; LINUX-NEXT: unit_size =
+; LINUX-NEXT: unit_size = 0x00000128
 
 ; Check for each name in the output.
 ; LINUX-DAG: "ns"
diff --git a/llvm/test/DebugInfo/X86/fake-use.ll b/llvm/test/DebugInfo/X86/fake-use.ll
index 5ac5104a16711..0f24ab40ba9ad 100644
--- a/llvm/test/DebugInfo/X86/fake-use.ll
+++ b/llvm/test/DebugInfo/X86/fake-use.ll
@@ -9,7 +9,7 @@
 ; RUN: not %python %p/../Inputs/check-fake-use.py %t
 
 ; Generated with:
-; clang -O2 -g -S -emit-llvm -fextend-this-ptr fake-use.c
+; clang -O2 -g -S -emit-llvm -fextend-variable-liveness=this fake-use.c
 ;
 ; int glob[10];
 ; extern void bar();
diff --git a/llvm/test/ExecutionEngine/2010-01-15-UndefValue.ll b/llvm/test/ExecutionEngine/Interpreter/2010-01-15-UndefValue.ll
similarity index 100%
rename from llvm/test/ExecutionEngine/2010-01-15-UndefValue.ll
rename to llvm/test/ExecutionEngine/Interpreter/2010-01-15-UndefValue.ll
diff --git a/llvm/test/ExecutionEngine/test-interp-target-ext-type.ll b/llvm/test/ExecutionEngine/Interpreter/test-interp-target-ext-type.ll
similarity index 100%
rename from llvm/test/ExecutionEngine/test-interp-target-ext-type.ll
rename to llvm/test/ExecutionEngine/Interpreter/test-interp-target-ext-type.ll
diff --git a/llvm/test/ExecutionEngine/test-interp-vec-cast.ll b/llvm/test/ExecutionEngine/Interpreter/test-interp-vec-cast.ll
similarity index 100%
rename from llvm/test/ExecutionEngine/test-interp-vec-cast.ll
rename to llvm/test/ExecutionEngine/Interpreter/test-interp-vec-cast.ll
diff --git a/llvm/test/ExecutionEngine/test-interp-vec-insertelement.ll b/llvm/test/ExecutionEngine/Interpreter/test-interp-vec-insertelement.ll
similarity index 100%
rename from llvm/test/ExecutionEngine/test-interp-vec-insertelement.ll
rename to llvm/test/ExecutionEngine/Interpreter/test-interp-vec-insertelement.ll
diff --git a/llvm/test/ExecutionEngine/test-interp-vec-insertextractvalue.ll b/llvm/test/ExecutionEngine/Interpreter/test-interp-vec-insertextractvalue.ll
similarity index 100%
rename from llvm/test/ExecutionEngine/test-interp-vec-insertextractvalue.ll
rename to llvm/test/ExecutionEngine/Interpreter/test-interp-vec-insertextractvalue.ll
diff --git a/llvm/test/ExecutionEngine/test-interp-vec-loadstore.ll b/llvm/test/ExecutionEngine/Interpreter/test-interp-vec-loadstore.ll
similarity index 100%
rename from llvm/test/ExecutionEngine/test-interp-vec-loadstore.ll
rename to llvm/test/ExecutionEngine/Interpreter/test-interp-vec-loadstore.ll
diff --git a/llvm/test/ExecutionEngine/test-interp-vec-select.ll b/llvm/test/ExecutionEngine/Interpreter/test-interp-vec-select.ll
similarity index 100%
rename from llvm/test/ExecutionEngine/test-interp-vec-select.ll
rename to llvm/test/ExecutionEngine/Interpreter/test-interp-vec-select.ll
diff --git a/llvm/test/ExecutionEngine/test-interp-vec-shift.ll b/llvm/test/ExecutionEngine/Interpreter/test-interp-vec-shift.ll
similarity index 100%
rename from llvm/test/ExecutionEngine/test-interp-vec-shift.ll
rename to llvm/test/ExecutionEngine/Interpreter/test-interp-vec-shift.ll
diff --git a/llvm/test/ExecutionEngine/test-interp-vec-shuffle.ll b/llvm/test/ExecutionEngine/Interpreter/test-interp-vec-shuffle.ll
similarity index 100%
rename from llvm/test/ExecutionEngine/test-interp-vec-shuffle.ll
rename to llvm/test/ExecutionEngine/Interpreter/test-interp-vec-shuffle.ll
diff --git a/llvm/test/ExecutionEngine/JITLink/AArch64/MachO_ptrauth_dependencies.s b/llvm/test/ExecutionEngine/JITLink/AArch64/MachO_ptrauth_dependencies.s
new file mode 100644
index 0000000000000..454079923622e
--- /dev/null
+++ b/llvm/test/ExecutionEngine/JITLink/AArch64/MachO_ptrauth_dependencies.s
@@ -0,0 +1,21 @@
+# RUN: llvm-mc -triple=arm64e-apple-macosx -filetype=obj -o %t.o %s
+# RUN: llvm-jitlink -num-threads=0 -debug-only=orc -noexec \
+# RUN:              -abs _foo=0x1 %t.o 2>&1 \
+# RUN:              | FileCheck %s
+#
+# Ensure that we don't lose dependence tracking information when ptrauth edges
+# are lowered: _main should still depend on _foo.
+#
+# REQUIRES: asserts
+
+# CHECK:    Symbols: { _main }, Dependencies: { (main, { _foo }) }
+
+	.section	__TEXT,__text,regular,pure_instructions
+
+	.section	__DATA,__data
+	.globl	_main
+	.p2align	3, 0x0
+_main:
+	.quad	_foo@AUTH(ia,0)
+
+.subsections_via_symbols
diff --git a/llvm/test/ExecutionEngine/JITLink/x86-64/COFF_pdata_no_strip.s b/llvm/test/ExecutionEngine/JITLink/x86-64/COFF_pdata_no_strip.s
index 80f4f15afd169..584754a6684b7 100644
--- a/llvm/test/ExecutionEngine/JITLink/x86-64/COFF_pdata_no_strip.s
+++ b/llvm/test/ExecutionEngine/JITLink/x86-64/COFF_pdata_no_strip.s
@@ -26,6 +26,8 @@ main:
 	.seh_endprologue
 	movl	$0, 36(%rsp)
 	nop
+	.seh_startepilogue
 	addq	$40, %rsp
+	.seh_endepilogue
 	retq
 	.seh_endproc
diff --git a/llvm/test/ExecutionEngine/JITLink/x86-64/COFF_pdata_strip.s b/llvm/test/ExecutionEngine/JITLink/x86-64/COFF_pdata_strip.s
index 8b43e19d2ac4c..1d7529a700728 100644
--- a/llvm/test/ExecutionEngine/JITLink/x86-64/COFF_pdata_strip.s
+++ b/llvm/test/ExecutionEngine/JITLink/x86-64/COFF_pdata_strip.s
@@ -38,6 +38,8 @@ func:
 	.seh_endprologue
 	movl	$0, 36(%rsp)
 	nop
+	.seh_startepilogue
 	addq	$40, %rsp
+	.seh_endepilogue
 	retq
 	.seh_endproc
diff --git a/llvm/test/ExecutionEngine/fma3-jit.ll b/llvm/test/ExecutionEngine/MCJIT/fma3-jit.ll
similarity index 100%
rename from llvm/test/ExecutionEngine/fma3-jit.ll
rename to llvm/test/ExecutionEngine/MCJIT/fma3-jit.ll
diff --git a/llvm/test/ExecutionEngine/frem.ll b/llvm/test/ExecutionEngine/MCJIT/frem.ll
similarity index 90%
rename from llvm/test/ExecutionEngine/frem.ll
rename to llvm/test/ExecutionEngine/MCJIT/frem.ll
index d33e4fca876e1..b8739c249cf58 100644
--- a/llvm/test/ExecutionEngine/frem.ll
+++ b/llvm/test/ExecutionEngine/MCJIT/frem.ll
@@ -1,6 +1,3 @@
-; LoongArch does not support mcjit.
-; UNSUPPORTED: target=loongarch{{.*}}
-
 ; LLI.exe used to crash on Windows\X86 when certain single precession
 ; floating point intrinsics (defined as macros) are used.
 ; This unit test guards against the failure.
diff --git a/llvm/test/ExecutionEngine/mov64zext32.ll b/llvm/test/ExecutionEngine/MCJIT/mov64zext32.ll
similarity index 77%
rename from llvm/test/ExecutionEngine/mov64zext32.ll
rename to llvm/test/ExecutionEngine/MCJIT/mov64zext32.ll
index 43bd0fb2f7b43..bba1a1987350a 100644
--- a/llvm/test/ExecutionEngine/mov64zext32.ll
+++ b/llvm/test/ExecutionEngine/MCJIT/mov64zext32.ll
@@ -1,6 +1,3 @@
-; LoongArch does not support mcjit.
-; UNSUPPORTED: target=loongarch{{.*}}
-
 ; RUN: %lli -jit-kind=mcjit %s > /dev/null
 ; RUN: %lli %s > /dev/null
 
diff --git a/llvm/test/ExecutionEngine/test-interp-vec-arithm_float.ll b/llvm/test/ExecutionEngine/MCJIT/test-interp-vec-arithm_float.ll
similarity index 92%
rename from llvm/test/ExecutionEngine/test-interp-vec-arithm_float.ll
rename to llvm/test/ExecutionEngine/MCJIT/test-interp-vec-arithm_float.ll
index 99d95791c5ad2..6f784265a73e1 100644
--- a/llvm/test/ExecutionEngine/test-interp-vec-arithm_float.ll
+++ b/llvm/test/ExecutionEngine/MCJIT/test-interp-vec-arithm_float.ll
@@ -1,6 +1,3 @@
-; LoongArch does not support mcjit.
-; UNSUPPORTED: target=loongarch{{.*}}
-
 ; RUN: %lli -jit-kind=mcjit %s > /dev/null
 ; RUN: %lli %s > /dev/null
 
diff --git a/llvm/test/ExecutionEngine/test-interp-vec-arithm_int.ll b/llvm/test/ExecutionEngine/MCJIT/test-interp-vec-arithm_int.ll
similarity index 96%
rename from llvm/test/ExecutionEngine/test-interp-vec-arithm_int.ll
rename to llvm/test/ExecutionEngine/MCJIT/test-interp-vec-arithm_int.ll
index 2e5592d4d363c..6896af83c44fb 100644
--- a/llvm/test/ExecutionEngine/test-interp-vec-arithm_int.ll
+++ b/llvm/test/ExecutionEngine/MCJIT/test-interp-vec-arithm_int.ll
@@ -1,6 +1,3 @@
-; LoongArch does not support mcjit.
-; UNSUPPORTED: target=loongarch{{.*}}
-
 ; RUN: %lli -jit-kind=mcjit %s > /dev/null
 ; RUN: %lli %s > /dev/null
 
diff --git a/llvm/test/ExecutionEngine/test-interp-vec-logical.ll b/llvm/test/ExecutionEngine/MCJIT/test-interp-vec-logical.ll
similarity index 91%
rename from llvm/test/ExecutionEngine/test-interp-vec-logical.ll
rename to llvm/test/ExecutionEngine/MCJIT/test-interp-vec-logical.ll
index 1e11659b1898a..f654120eaf8e0 100644
--- a/llvm/test/ExecutionEngine/test-interp-vec-logical.ll
+++ b/llvm/test/ExecutionEngine/MCJIT/test-interp-vec-logical.ll
@@ -1,6 +1,3 @@
-; LoongArch does not support mcjit.
-; UNSUPPORTED: target=loongarch{{.*}}
-
 ; RUN: %lli -jit-kind=mcjit %s > /dev/null
 ; RUN: %lli %s > /dev/null
 
diff --git a/llvm/test/ExecutionEngine/test-interp-vec-setcond-fp.ll b/llvm/test/ExecutionEngine/MCJIT/test-interp-vec-setcond-fp.ll
similarity index 93%
rename from llvm/test/ExecutionEngine/test-interp-vec-setcond-fp.ll
rename to llvm/test/ExecutionEngine/MCJIT/test-interp-vec-setcond-fp.ll
index e919550de2689..84bdec1cf5597 100644
--- a/llvm/test/ExecutionEngine/test-interp-vec-setcond-fp.ll
+++ b/llvm/test/ExecutionEngine/MCJIT/test-interp-vec-setcond-fp.ll
@@ -1,6 +1,3 @@
-; LoongArch does not support mcjit.
-; UNSUPPORTED: target=loongarch{{.*}}
-
 ; RUN: %lli -jit-kind=mcjit %s > /dev/null
 ; RUN: %lli %s > /dev/null
 
diff --git a/llvm/test/ExecutionEngine/test-interp-vec-setcond-int.ll b/llvm/test/ExecutionEngine/MCJIT/test-interp-vec-setcond-int.ll
similarity index 97%
rename from llvm/test/ExecutionEngine/test-interp-vec-setcond-int.ll
rename to llvm/test/ExecutionEngine/MCJIT/test-interp-vec-setcond-int.ll
index 9862d6af1f6eb..5a20fc4f11721 100644
--- a/llvm/test/ExecutionEngine/test-interp-vec-setcond-int.ll
+++ b/llvm/test/ExecutionEngine/MCJIT/test-interp-vec-setcond-int.ll
@@ -1,6 +1,3 @@
-; LoongArch does not support mcjit.
-; UNSUPPORTED: target=loongarch{{.*}}
-
 ; RUN: %lli -jit-kind=mcjit %s > /dev/null
 ; RUN: %lli %s > /dev/null
 
diff --git a/llvm/test/ExecutionEngine/Orc/throw-catch-minimal.ll b/llvm/test/ExecutionEngine/Orc/throw-catch-minimal.ll
new file mode 100644
index 0000000000000..4ee55c61a81ea
--- /dev/null
+++ b/llvm/test/ExecutionEngine/Orc/throw-catch-minimal.ll
@@ -0,0 +1,52 @@
+; REQUIRES: system-darwin && host-unwind-supports-jit
+; RUN: lli -jit-kind=orc %s
+;
+; Basic correctness testing for eh-frame processing and registration.
+
+@_ZTIi = external constant ptr
+
+declare ptr @__cxa_allocate_exception(i64)
+declare void @__cxa_throw(ptr, ptr, ptr)
+
+declare i32 @__gxx_personality_v0(...)
+declare i32 @llvm.eh.typeid.for(ptr)
+declare ptr @__cxa_begin_catch(ptr)
+declare void @__cxa_end_catch()
+
+define void @explode() {
+entry:
+  %exception = tail call ptr @__cxa_allocate_exception(i64 4)
+  store i32 42, ptr %exception, align 16
+  tail call void @__cxa_throw(ptr %exception, ptr @_ZTIi, ptr null)
+  unreachable
+}
+
+define i32 @main(i32 %argc, ptr %argv) personality ptr @__gxx_personality_v0 {
+entry:
+  invoke void @explode()
+          to label %return unwind label %lpad
+
+lpad:
+  %0 = landingpad { ptr, i32 }
+          catch ptr @_ZTIi
+  %1 = extractvalue { ptr, i32 } %0, 1
+  %2 = tail call i32 @llvm.eh.typeid.for(ptr @_ZTIi)
+  %matches = icmp eq i32 %1, %2
+  br i1 %matches, label %catch, label %eh.resume
+
+catch:
+  %3 = extractvalue { ptr, i32 } %0, 0
+  %4 = tail call ptr @__cxa_begin_catch(ptr %3)
+  %5 = load i32, ptr %4, align 4
+  %cmp = icmp ne i32 %5, 42
+  %cond = zext i1 %cmp to i32
+  tail call void @__cxa_end_catch()
+  br label %return
+
+return:
+  %retval.0 = phi i32 [ %cond, %catch ], [ 2, %entry ]
+  ret i32 %retval.0
+
+eh.resume:
+  resume { ptr, i32 } %0
+}
diff --git a/llvm/test/ExecutionEngine/Orc/throw-catch-no-frame-pointer.ll b/llvm/test/ExecutionEngine/Orc/throw-catch-no-frame-pointer.ll
new file mode 100644
index 0000000000000..8032e71ad2f35
--- /dev/null
+++ b/llvm/test/ExecutionEngine/Orc/throw-catch-no-frame-pointer.ll
@@ -0,0 +1,51 @@
+; REQUIRES: system-darwin && host-unwind-supports-jit
+; RUN: lli -jit-kind=orc %s
+;
+; Check that we can throw exceptions from no-fp functions. On systems that
+; support compact-unwind this implicitly tests that we correctly handle
+; unwind-info records that depend on DWARF FDEs.
+
+@_ZTIi = external constant ptr
+
+declare ptr @__cxa_allocate_exception(i64)
+declare void @__cxa_throw(ptr, ptr, ptr)
+declare ptr @__cxa_begin_catch(ptr)
+declare void @__cxa_end_catch()
+declare i32 @__gxx_personality_v0(...)
+declare i32 @llvm.eh.typeid.for.p0(ptr)
+
+define void @_Z3foov() "frame-pointer"="none" {
+entry:
+  %exception = tail call ptr @__cxa_allocate_exception(i64 4)
+  store i32 42, ptr %exception
+  tail call void @__cxa_throw(ptr %exception, ptr nonnull @_ZTIi, ptr null)
+  unreachable
+}
+
+define i32 @main(i32 %argc, ptr %argv) "frame-pointer"="all" personality ptr @__gxx_personality_v0 {
+entry:
+  invoke void @_Z3foov()
+          to label %return.unreachable unwind label %lpad
+
+lpad:
+  %0 = landingpad { ptr, i32 }
+          catch ptr @_ZTIi
+  %1 = extractvalue { ptr, i32 } %0, 1
+  %2 = tail call i32 @llvm.eh.typeid.for.p0(ptr nonnull @_ZTIi)
+  %matches = icmp eq i32 %1, %2
+  br i1 %matches, label %catch, label %eh.resume
+
+catch:
+  %3 = extractvalue { ptr, i32 } %0, 0
+  %4 = tail call ptr @__cxa_begin_catch(ptr %3)
+  %5 = load i32, ptr %4
+  %sub = sub nsw i32 42, %5
+  tail call void @__cxa_end_catch()
+  ret i32 %sub
+
+return.unreachable:
+  unreachable
+
+eh.resume:
+  resume { ptr, i32 } %0
+}
diff --git a/llvm/test/ExecutionEngine/OrcLazy/minimal-throw-catch.ll b/llvm/test/ExecutionEngine/OrcLazy/minimal-throw-catch.ll
index 5bc5769a2c0d2..83d2d89a5d607 100644
--- a/llvm/test/ExecutionEngine/OrcLazy/minimal-throw-catch.ll
+++ b/llvm/test/ExecutionEngine/OrcLazy/minimal-throw-catch.ll
@@ -1,12 +1,8 @@
-; REQUIRES: x86_64-apple
+; REQUIRES: system-darwin && host-unwind-supports-jit
 ; RUN: lli -jit-kind=orc-lazy %s
 ;
 ; Basic correctness testing for eh-frame processing and registration.
 
-source_filename = "minimal-throw-catch.cpp"
-target datalayout = "e-m:o-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128"
-target triple = "x86_64-apple-macosx10.14.0"
-
 @_ZTIi = external constant ptr
 
 declare ptr @__cxa_allocate_exception(i64)
diff --git a/llvm/test/ExecutionEngine/lit.local.cfg b/llvm/test/ExecutionEngine/lit.local.cfg
index c748de14c8409..1951f140ea889 100644
--- a/llvm/test/ExecutionEngine/lit.local.cfg
+++ b/llvm/test/ExecutionEngine/lit.local.cfg
@@ -1,4 +1,4 @@
-if config.root.native_target in ['Sparc', 'SystemZ', 'Hexagon', 'RISCV']:
+if config.root.native_target in ['Sparc', 'SystemZ', 'Hexagon']:
     config.unsupported = True
 
 # ExecutionEngine tests are not expected to pass in a cross-compilation setup.
diff --git a/llvm/test/Feature/OperandBundles/function-attrs.ll b/llvm/test/Feature/OperandBundles/function-attrs.ll
index d3001fa7da42c..6db13d31a86d4 100644
--- a/llvm/test/Feature/OperandBundles/function-attrs.ll
+++ b/llvm/test/Feature/OperandBundles/function-attrs.ll
@@ -37,7 +37,7 @@ define void @test_2(ptr %x) {
 define void @test_3(ptr %x) {
 ; The "deopt" operand bundle does not capture or write to %x.
 
-; CHECK-LABEL: define void @test_3(ptr nocapture readonly %x)
+; CHECK-LABEL: define void @test_3(ptr readonly captures(none) %x)
  entry:
   call void @f_readonly() [ "deopt"(ptr %x) ]
   ret void
diff --git a/llvm/test/Instrumentation/BoundsChecking/runtimes.ll b/llvm/test/Instrumentation/BoundsChecking/runtimes.ll
index 7cf78a5d54e71..6c1acf6d13775 100644
--- a/llvm/test/Instrumentation/BoundsChecking/runtimes.ll
+++ b/llvm/test/Instrumentation/BoundsChecking/runtimes.ll
@@ -9,7 +9,8 @@
 ; RUN: opt < %s -passes='bounds-checking<min-rt>'       -S | FileCheck %s --check-prefixes=MINRT-NOMERGE
 ; RUN: opt < %s -passes='bounds-checking<min-rt-abort>' -S | FileCheck %s --check-prefixes=MINRTABORT-NOMERGE
 ;
-; RUN: opt < %s -passes='bounds-checking<trap;guard=3>'   -S | FileCheck %s --check-prefixes=TR-GUARD
+; RUN: opt < %s -passes='bounds-checking<trap;guard=3>'   -S | FileCheck %s --check-prefixes=TR-GUARD-COMMON,TR-GUARD-THREE
+; RUN: opt < %s -passes='bounds-checking<trap;guard=13>'   -S | FileCheck %s --check-prefixes=TR-GUARD-COMMON,TR-GUARD-THIRTEEN
 ; RUN: opt < %s -passes='bounds-checking<rt;guard=-5>'     -S | FileCheck %s --check-prefixes=RT-GUARD
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
 
@@ -126,23 +127,29 @@ define void @f1(i64 %x) nounwind {
 ; MINRTABORT-NOMERGE-NEXT:    call void @__ubsan_handle_local_out_of_bounds_minimal_abort() #[[ATTR2:[0-9]+]], !nosanitize [[META0]]
 ; MINRTABORT-NOMERGE-NEXT:    unreachable, !nosanitize [[META0]]
 ;
-; TR-GUARD-LABEL: define void @f1(
-; TR-GUARD-SAME: i64 [[X:%.*]]) #[[ATTR0:[0-9]+]] {
-; TR-GUARD-NEXT:    [[TMP1:%.*]] = mul i64 16, [[X]]
-; TR-GUARD-NEXT:    [[TMP2:%.*]] = alloca i128, i64 [[X]], align 8
-; TR-GUARD-NEXT:    [[TMP3:%.*]] = sub i64 [[TMP1]], 0, !nosanitize [[META0:![0-9]+]]
-; TR-GUARD-NEXT:    [[TMP4:%.*]] = icmp ult i64 [[TMP3]], 16, !nosanitize [[META0]]
-; TR-GUARD-NEXT:    [[TMP5:%.*]] = or i1 false, [[TMP4]], !nosanitize [[META0]]
-; TR-GUARD-NEXT:    [[TMP6:%.*]] = or i1 false, [[TMP5]], !nosanitize [[META0]]
-; TR-GUARD-NEXT:    [[TMP7:%.*]] = call i1 @llvm.allow.ubsan.check(i8 3), !nosanitize [[META0]]
-; TR-GUARD-NEXT:    [[TMP8:%.*]] = and i1 [[TMP6]], [[TMP7]], !nosanitize [[META0]]
-; TR-GUARD-NEXT:    br i1 [[TMP8]], label %[[TRAP:.*]], label %[[BB9:.*]]
-; TR-GUARD:       [[BB9]]:
-; TR-GUARD-NEXT:    [[TMP10:%.*]] = load i128, ptr [[TMP2]], align 4
-; TR-GUARD-NEXT:    ret void
-; TR-GUARD:       [[TRAP]]:
-; TR-GUARD-NEXT:    call void @llvm.ubsantrap(i8 3) #[[ATTR3:[0-9]+]], !nosanitize [[META0]]
-; TR-GUARD-NEXT:    unreachable, !nosanitize [[META0]]
+; TR-GUARD-COMMON-LABEL: define void @f1(
+; TR-GUARD-COMMON-SAME: i64 [[X:%.*]]) #[[ATTR0:[0-9]+]] {
+; TR-GUARD-COMMON-NEXT:    [[TMP1:%.*]] = mul i64 16, [[X]]
+; TR-GUARD-COMMON-NEXT:    [[TMP2:%.*]] = alloca i128, i64 [[X]], align 8
+; TR-GUARD-COMMON-NEXT:    [[TMP3:%.*]] = sub i64 [[TMP1]], 0, !nosanitize [[META0:![0-9]+]]
+; TR-GUARD-COMMON-NEXT:    [[TMP4:%.*]] = icmp ult i64 [[TMP3]], 16, !nosanitize [[META0]]
+; TR-GUARD-COMMON-NEXT:    [[TMP5:%.*]] = or i1 false, [[TMP4]], !nosanitize [[META0]]
+; TR-GUARD-COMMON-NEXT:    [[TMP6:%.*]] = or i1 false, [[TMP5]], !nosanitize [[META0]]
+;
+; TR-GUARD-THREE:          [[TMP7:%.*]] = call i1 @llvm.allow.ubsan.check(i8 3), !nosanitize [[META0]]
+; TR-GUARD-THIRTEEN:       [[TMP7:%.*]] = call i1 @llvm.allow.ubsan.check(i8 13), !nosanitize [[META0]]
+;
+; TR-GUARD-COMMON:         [[TMP8:%.*]] = and i1 [[TMP6]], [[TMP7]], !nosanitize [[META0]]
+; TR-GUARD-COMMON-NEXT:    br i1 [[TMP8]], label %[[TRAP:.*]], label %[[BB9:.*]]
+; TR-GUARD-COMMON:       [[BB9]]:
+; TR-GUARD-COMMON-NEXT:    [[TMP10:%.*]] = load i128, ptr [[TMP2]], align 4
+; TR-GUARD-COMMON-NEXT:    ret void
+; TR-GUARD-COMMON:       [[TRAP]]:
+;
+; TR-GUARD-THREE:       call void @llvm.ubsantrap(i8 3) #[[ATTR3:[0-9]+]], !nosanitize [[META0]]
+; TR-GUARD-THIRTEEN:    call void @llvm.ubsantrap(i8 13) #[[ATTR3:[0-9]+]], !nosanitize [[META0]]
+;
+; TR-GUARD-COMMON:      unreachable, !nosanitize [[META0]]
 ;
 ; RT-GUARD-LABEL: define void @f1(
 ; RT-GUARD-SAME: i64 [[X:%.*]]) #[[ATTR0:[0-9]+]] {
diff --git a/llvm/test/Instrumentation/HWAddressSanitizer/mem-attr.ll b/llvm/test/Instrumentation/HWAddressSanitizer/mem-attr.ll
index 724e6c5a0bdec..d891fdee265a5 100644
--- a/llvm/test/Instrumentation/HWAddressSanitizer/mem-attr.ll
+++ b/llvm/test/Instrumentation/HWAddressSanitizer/mem-attr.ll
@@ -4,15 +4,15 @@
 target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128-Fn32"
 target triple = "aarch64-unknown-linux-android30"
 
-; CHECK: define dso_local void @test_writeonly(ptr nocapture noundef %p) local_unnamed_addr #0
-define dso_local void @test_writeonly(ptr nocapture noundef writeonly %p) local_unnamed_addr #0 {
+; CHECK: define dso_local void @test_writeonly(ptr noundef captures(none) %p) local_unnamed_addr #0
+define dso_local void @test_writeonly(ptr noundef writeonly captures(none) %p) local_unnamed_addr #0 {
 entry:
   store i32 42, ptr %p, align 4
   ret void
 }
 
-; CHECK: define dso_local void @test_readonly(ptr nocapture noundef readonly %p) local_unnamed_addr #0
-define dso_local void @test_readonly(ptr nocapture noundef readonly %p) local_unnamed_addr #1 {
+; CHECK: define dso_local void @test_readonly(ptr noundef readonly captures(none) %p) local_unnamed_addr #0
+define dso_local void @test_readonly(ptr noundef readonly captures(none) %p) local_unnamed_addr #1 {
 entry:
   store i32 42, ptr %p, align 4
   ret void
diff --git a/llvm/test/Instrumentation/MemorySanitizer/AArch64/arm64-cvt.ll b/llvm/test/Instrumentation/MemorySanitizer/AArch64/arm64-cvt.ll
new file mode 100644
index 0000000000000..016e0114c83ff
--- /dev/null
+++ b/llvm/test/Instrumentation/MemorySanitizer/AArch64/arm64-cvt.ll
@@ -0,0 +1,829 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 4
+; RUN: opt < %s -passes=msan -S | FileCheck %s
+;
+; Suboptimally handled by visitInstruction: llvm.aarch64.neon.
+; - fcvtas, fcvtau
+; - fcvtms, fcvtmu
+; - fcvtns, fcvtnu
+; - fcvtps, fcvtpu
+; - fcvtzs, fcvtzu
+;
+; Forked from llvm/test/CodeGen/AArch64/arm64-cvt.ll
+
+target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
+target triple = "aarch64--linux-android9001"
+
+define i32 @fcvtas_1w1s(float %A) nounwind #0 {
+; CHECK-LABEL: define i32 @fcvtas_1w1s(
+; CHECK-SAME: float [[A:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i32, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i32 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP2:%.*]], label [[TMP4:%.*]], !prof [[PROF1:![0-9]+]]
+; CHECK:       2:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR3:[0-9]+]]
+; CHECK-NEXT:    unreachable
+; CHECK:       3:
+; CHECK-NEXT:    [[TMP3:%.*]] = call i32 @llvm.aarch64.neon.fcvtas.i32.f32(float [[A]])
+; CHECK-NEXT:    store i32 0, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i32 [[TMP3]]
+;
+  %tmpvar3 = call i32 @llvm.aarch64.neon.fcvtas.i32.f32(float %A)
+  ret i32 %tmpvar3
+}
+
+define i64 @fcvtas_1x1s(float %A) nounwind #0 {
+; CHECK-LABEL: define i64 @fcvtas_1x1s(
+; CHECK-SAME: float [[A:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i32, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i32 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP2:%.*]], label [[TMP4:%.*]], !prof [[PROF1]]
+; CHECK:       2:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR3]]
+; CHECK-NEXT:    unreachable
+; CHECK:       3:
+; CHECK-NEXT:    [[TMP3:%.*]] = call i64 @llvm.aarch64.neon.fcvtas.i64.f32(float [[A]])
+; CHECK-NEXT:    store i64 0, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i64 [[TMP3]]
+;
+  %tmpvar3 = call i64 @llvm.aarch64.neon.fcvtas.i64.f32(float %A)
+  ret i64 %tmpvar3
+}
+
+define i32 @fcvtas_1w1d(double %A) nounwind #0 {
+; CHECK-LABEL: define i32 @fcvtas_1w1d(
+; CHECK-SAME: double [[A:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP2:%.*]], label [[TMP4:%.*]], !prof [[PROF1]]
+; CHECK:       2:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR3]]
+; CHECK-NEXT:    unreachable
+; CHECK:       3:
+; CHECK-NEXT:    [[TMP3:%.*]] = call i32 @llvm.aarch64.neon.fcvtas.i32.f64(double [[A]])
+; CHECK-NEXT:    store i32 0, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i32 [[TMP3]]
+;
+  %tmpvar3 = call i32 @llvm.aarch64.neon.fcvtas.i32.f64(double %A)
+  ret i32 %tmpvar3
+}
+
+define i64 @fcvtas_1x1d(double %A) nounwind #0 {
+; CHECK-LABEL: define i64 @fcvtas_1x1d(
+; CHECK-SAME: double [[A:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP2:%.*]], label [[TMP4:%.*]], !prof [[PROF1]]
+; CHECK:       2:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR3]]
+; CHECK-NEXT:    unreachable
+; CHECK:       3:
+; CHECK-NEXT:    [[TMP3:%.*]] = call i64 @llvm.aarch64.neon.fcvtas.i64.f64(double [[A]])
+; CHECK-NEXT:    store i64 0, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i64 [[TMP3]]
+;
+  %tmpvar3 = call i64 @llvm.aarch64.neon.fcvtas.i64.f64(double %A)
+  ret i64 %tmpvar3
+}
+
+declare i32 @llvm.aarch64.neon.fcvtas.i32.f32(float) nounwind readnone
+declare i64 @llvm.aarch64.neon.fcvtas.i64.f32(float) nounwind readnone
+declare i32 @llvm.aarch64.neon.fcvtas.i32.f64(double) nounwind readnone
+declare i64 @llvm.aarch64.neon.fcvtas.i64.f64(double) nounwind readnone
+
+define i32 @fcvtau_1w1s(float %A) nounwind #0 {
+; CHECK-LABEL: define i32 @fcvtau_1w1s(
+; CHECK-SAME: float [[A:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i32, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i32 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP2:%.*]], label [[TMP4:%.*]], !prof [[PROF1]]
+; CHECK:       2:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR3]]
+; CHECK-NEXT:    unreachable
+; CHECK:       3:
+; CHECK-NEXT:    [[TMP3:%.*]] = call i32 @llvm.aarch64.neon.fcvtau.i32.f32(float [[A]])
+; CHECK-NEXT:    store i32 0, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i32 [[TMP3]]
+;
+  %tmpvar3 = call i32 @llvm.aarch64.neon.fcvtau.i32.f32(float %A)
+  ret i32 %tmpvar3
+}
+
+define i64 @fcvtau_1x1s(float %A) nounwind #0 {
+; CHECK-LABEL: define i64 @fcvtau_1x1s(
+; CHECK-SAME: float [[A:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i32, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i32 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP2:%.*]], label [[TMP4:%.*]], !prof [[PROF1]]
+; CHECK:       2:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR3]]
+; CHECK-NEXT:    unreachable
+; CHECK:       3:
+; CHECK-NEXT:    [[TMP3:%.*]] = call i64 @llvm.aarch64.neon.fcvtau.i64.f32(float [[A]])
+; CHECK-NEXT:    store i64 0, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i64 [[TMP3]]
+;
+  %tmpvar3 = call i64 @llvm.aarch64.neon.fcvtau.i64.f32(float %A)
+  ret i64 %tmpvar3
+}
+
+define i32 @fcvtau_1w1d(double %A) nounwind #0 {
+; CHECK-LABEL: define i32 @fcvtau_1w1d(
+; CHECK-SAME: double [[A:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP2:%.*]], label [[TMP4:%.*]], !prof [[PROF1]]
+; CHECK:       2:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR3]]
+; CHECK-NEXT:    unreachable
+; CHECK:       3:
+; CHECK-NEXT:    [[TMP3:%.*]] = call i32 @llvm.aarch64.neon.fcvtau.i32.f64(double [[A]])
+; CHECK-NEXT:    store i32 0, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i32 [[TMP3]]
+;
+  %tmpvar3 = call i32 @llvm.aarch64.neon.fcvtau.i32.f64(double %A)
+  ret i32 %tmpvar3
+}
+
+define i64 @fcvtau_1x1d(double %A) nounwind #0 {
+; CHECK-LABEL: define i64 @fcvtau_1x1d(
+; CHECK-SAME: double [[A:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP2:%.*]], label [[TMP4:%.*]], !prof [[PROF1]]
+; CHECK:       2:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR3]]
+; CHECK-NEXT:    unreachable
+; CHECK:       3:
+; CHECK-NEXT:    [[TMP3:%.*]] = call i64 @llvm.aarch64.neon.fcvtau.i64.f64(double [[A]])
+; CHECK-NEXT:    store i64 0, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i64 [[TMP3]]
+;
+  %tmpvar3 = call i64 @llvm.aarch64.neon.fcvtau.i64.f64(double %A)
+  ret i64 %tmpvar3
+}
+
+declare i32 @llvm.aarch64.neon.fcvtau.i32.f32(float) nounwind readnone
+declare i64 @llvm.aarch64.neon.fcvtau.i64.f32(float) nounwind readnone
+declare i32 @llvm.aarch64.neon.fcvtau.i32.f64(double) nounwind readnone
+declare i64 @llvm.aarch64.neon.fcvtau.i64.f64(double) nounwind readnone
+
+define i32 @fcvtms_1w1s(float %A) nounwind #0 {
+; CHECK-LABEL: define i32 @fcvtms_1w1s(
+; CHECK-SAME: float [[A:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i32, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i32 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP2:%.*]], label [[TMP4:%.*]], !prof [[PROF1]]
+; CHECK:       2:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR3]]
+; CHECK-NEXT:    unreachable
+; CHECK:       3:
+; CHECK-NEXT:    [[TMP3:%.*]] = call i32 @llvm.aarch64.neon.fcvtms.i32.f32(float [[A]])
+; CHECK-NEXT:    store i32 0, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i32 [[TMP3]]
+;
+  %tmpvar3 = call i32 @llvm.aarch64.neon.fcvtms.i32.f32(float %A)
+  ret i32 %tmpvar3
+}
+
+define i64 @fcvtms_1x1s(float %A) nounwind #0 {
+; CHECK-LABEL: define i64 @fcvtms_1x1s(
+; CHECK-SAME: float [[A:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i32, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i32 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP2:%.*]], label [[TMP4:%.*]], !prof [[PROF1]]
+; CHECK:       2:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR3]]
+; CHECK-NEXT:    unreachable
+; CHECK:       3:
+; CHECK-NEXT:    [[TMP3:%.*]] = call i64 @llvm.aarch64.neon.fcvtms.i64.f32(float [[A]])
+; CHECK-NEXT:    store i64 0, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i64 [[TMP3]]
+;
+  %tmpvar3 = call i64 @llvm.aarch64.neon.fcvtms.i64.f32(float %A)
+  ret i64 %tmpvar3
+}
+
+define i32 @fcvtms_1w1d(double %A) nounwind #0 {
+; CHECK-LABEL: define i32 @fcvtms_1w1d(
+; CHECK-SAME: double [[A:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP2:%.*]], label [[TMP4:%.*]], !prof [[PROF1]]
+; CHECK:       2:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR3]]
+; CHECK-NEXT:    unreachable
+; CHECK:       3:
+; CHECK-NEXT:    [[TMP3:%.*]] = call i32 @llvm.aarch64.neon.fcvtms.i32.f64(double [[A]])
+; CHECK-NEXT:    store i32 0, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i32 [[TMP3]]
+;
+  %tmpvar3 = call i32 @llvm.aarch64.neon.fcvtms.i32.f64(double %A)
+  ret i32 %tmpvar3
+}
+
+define i64 @fcvtms_1x1d(double %A) nounwind #0 {
+; CHECK-LABEL: define i64 @fcvtms_1x1d(
+; CHECK-SAME: double [[A:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP2:%.*]], label [[TMP4:%.*]], !prof [[PROF1]]
+; CHECK:       2:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR3]]
+; CHECK-NEXT:    unreachable
+; CHECK:       3:
+; CHECK-NEXT:    [[TMP3:%.*]] = call i64 @llvm.aarch64.neon.fcvtms.i64.f64(double [[A]])
+; CHECK-NEXT:    store i64 0, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i64 [[TMP3]]
+;
+  %tmpvar3 = call i64 @llvm.aarch64.neon.fcvtms.i64.f64(double %A)
+  ret i64 %tmpvar3
+}
+
+declare i32 @llvm.aarch64.neon.fcvtms.i32.f32(float) nounwind readnone
+declare i64 @llvm.aarch64.neon.fcvtms.i64.f32(float) nounwind readnone
+declare i32 @llvm.aarch64.neon.fcvtms.i32.f64(double) nounwind readnone
+declare i64 @llvm.aarch64.neon.fcvtms.i64.f64(double) nounwind readnone
+
+define i32 @fcvtmu_1w1s(float %A) nounwind #0 {
+; CHECK-LABEL: define i32 @fcvtmu_1w1s(
+; CHECK-SAME: float [[A:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i32, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i32 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP2:%.*]], label [[TMP4:%.*]], !prof [[PROF1]]
+; CHECK:       2:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR3]]
+; CHECK-NEXT:    unreachable
+; CHECK:       3:
+; CHECK-NEXT:    [[TMP3:%.*]] = call i32 @llvm.aarch64.neon.fcvtmu.i32.f32(float [[A]])
+; CHECK-NEXT:    store i32 0, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i32 [[TMP3]]
+;
+  %tmpvar3 = call i32 @llvm.aarch64.neon.fcvtmu.i32.f32(float %A)
+  ret i32 %tmpvar3
+}
+
+define i64 @fcvtmu_1x1s(float %A) nounwind #0 {
+; CHECK-LABEL: define i64 @fcvtmu_1x1s(
+; CHECK-SAME: float [[A:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i32, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i32 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP2:%.*]], label [[TMP4:%.*]], !prof [[PROF1]]
+; CHECK:       2:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR3]]
+; CHECK-NEXT:    unreachable
+; CHECK:       3:
+; CHECK-NEXT:    [[TMP3:%.*]] = call i64 @llvm.aarch64.neon.fcvtmu.i64.f32(float [[A]])
+; CHECK-NEXT:    store i64 0, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i64 [[TMP3]]
+;
+  %tmpvar3 = call i64 @llvm.aarch64.neon.fcvtmu.i64.f32(float %A)
+  ret i64 %tmpvar3
+}
+
+define i32 @fcvtmu_1w1d(double %A) nounwind #0 {
+; CHECK-LABEL: define i32 @fcvtmu_1w1d(
+; CHECK-SAME: double [[A:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP2:%.*]], label [[TMP4:%.*]], !prof [[PROF1]]
+; CHECK:       2:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR3]]
+; CHECK-NEXT:    unreachable
+; CHECK:       3:
+; CHECK-NEXT:    [[TMP3:%.*]] = call i32 @llvm.aarch64.neon.fcvtmu.i32.f64(double [[A]])
+; CHECK-NEXT:    store i32 0, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i32 [[TMP3]]
+;
+  %tmpvar3 = call i32 @llvm.aarch64.neon.fcvtmu.i32.f64(double %A)
+  ret i32 %tmpvar3
+}
+
+define i64 @fcvtmu_1x1d(double %A) nounwind #0 {
+; CHECK-LABEL: define i64 @fcvtmu_1x1d(
+; CHECK-SAME: double [[A:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP2:%.*]], label [[TMP4:%.*]], !prof [[PROF1]]
+; CHECK:       2:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR3]]
+; CHECK-NEXT:    unreachable
+; CHECK:       3:
+; CHECK-NEXT:    [[TMP3:%.*]] = call i64 @llvm.aarch64.neon.fcvtmu.i64.f64(double [[A]])
+; CHECK-NEXT:    store i64 0, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i64 [[TMP3]]
+;
+  %tmpvar3 = call i64 @llvm.aarch64.neon.fcvtmu.i64.f64(double %A)
+  ret i64 %tmpvar3
+}
+
+declare i32 @llvm.aarch64.neon.fcvtmu.i32.f32(float) nounwind readnone
+declare i64 @llvm.aarch64.neon.fcvtmu.i64.f32(float) nounwind readnone
+declare i32 @llvm.aarch64.neon.fcvtmu.i32.f64(double) nounwind readnone
+declare i64 @llvm.aarch64.neon.fcvtmu.i64.f64(double) nounwind readnone
+
+define i32 @fcvtns_1w1s(float %A) nounwind #0 {
+; CHECK-LABEL: define i32 @fcvtns_1w1s(
+; CHECK-SAME: float [[A:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i32, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i32 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP2:%.*]], label [[TMP4:%.*]], !prof [[PROF1]]
+; CHECK:       2:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR3]]
+; CHECK-NEXT:    unreachable
+; CHECK:       3:
+; CHECK-NEXT:    [[TMP3:%.*]] = call i32 @llvm.aarch64.neon.fcvtns.i32.f32(float [[A]])
+; CHECK-NEXT:    store i32 0, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i32 [[TMP3]]
+;
+  %tmpvar3 = call i32 @llvm.aarch64.neon.fcvtns.i32.f32(float %A)
+  ret i32 %tmpvar3
+}
+
+define i64 @fcvtns_1x1s(float %A) nounwind #0 {
+; CHECK-LABEL: define i64 @fcvtns_1x1s(
+; CHECK-SAME: float [[A:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i32, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i32 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP2:%.*]], label [[TMP4:%.*]], !prof [[PROF1]]
+; CHECK:       2:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR3]]
+; CHECK-NEXT:    unreachable
+; CHECK:       3:
+; CHECK-NEXT:    [[TMP3:%.*]] = call i64 @llvm.aarch64.neon.fcvtns.i64.f32(float [[A]])
+; CHECK-NEXT:    store i64 0, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i64 [[TMP3]]
+;
+  %tmpvar3 = call i64 @llvm.aarch64.neon.fcvtns.i64.f32(float %A)
+  ret i64 %tmpvar3
+}
+
+define i32 @fcvtns_1w1d(double %A) nounwind #0 {
+; CHECK-LABEL: define i32 @fcvtns_1w1d(
+; CHECK-SAME: double [[A:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP2:%.*]], label [[TMP4:%.*]], !prof [[PROF1]]
+; CHECK:       2:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR3]]
+; CHECK-NEXT:    unreachable
+; CHECK:       3:
+; CHECK-NEXT:    [[TMP3:%.*]] = call i32 @llvm.aarch64.neon.fcvtns.i32.f64(double [[A]])
+; CHECK-NEXT:    store i32 0, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i32 [[TMP3]]
+;
+  %tmpvar3 = call i32 @llvm.aarch64.neon.fcvtns.i32.f64(double %A)
+  ret i32 %tmpvar3
+}
+
+define i64 @fcvtns_1x1d(double %A) nounwind #0 {
+; CHECK-LABEL: define i64 @fcvtns_1x1d(
+; CHECK-SAME: double [[A:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP2:%.*]], label [[TMP4:%.*]], !prof [[PROF1]]
+; CHECK:       2:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR3]]
+; CHECK-NEXT:    unreachable
+; CHECK:       3:
+; CHECK-NEXT:    [[TMP3:%.*]] = call i64 @llvm.aarch64.neon.fcvtns.i64.f64(double [[A]])
+; CHECK-NEXT:    store i64 0, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i64 [[TMP3]]
+;
+  %tmpvar3 = call i64 @llvm.aarch64.neon.fcvtns.i64.f64(double %A)
+  ret i64 %tmpvar3
+}
+
+declare i32 @llvm.aarch64.neon.fcvtns.i32.f32(float) nounwind readnone
+declare i64 @llvm.aarch64.neon.fcvtns.i64.f32(float) nounwind readnone
+declare i32 @llvm.aarch64.neon.fcvtns.i32.f64(double) nounwind readnone
+declare i64 @llvm.aarch64.neon.fcvtns.i64.f64(double) nounwind readnone
+
+define i32 @fcvtnu_1w1s(float %A) nounwind #0 {
+; CHECK-LABEL: define i32 @fcvtnu_1w1s(
+; CHECK-SAME: float [[A:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i32, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i32 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP2:%.*]], label [[TMP4:%.*]], !prof [[PROF1]]
+; CHECK:       2:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR3]]
+; CHECK-NEXT:    unreachable
+; CHECK:       3:
+; CHECK-NEXT:    [[TMP3:%.*]] = call i32 @llvm.aarch64.neon.fcvtnu.i32.f32(float [[A]])
+; CHECK-NEXT:    store i32 0, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i32 [[TMP3]]
+;
+  %tmpvar3 = call i32 @llvm.aarch64.neon.fcvtnu.i32.f32(float %A)
+  ret i32 %tmpvar3
+}
+
+define i64 @fcvtnu_1x1s(float %A) nounwind #0 {
+; CHECK-LABEL: define i64 @fcvtnu_1x1s(
+; CHECK-SAME: float [[A:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i32, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i32 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP2:%.*]], label [[TMP4:%.*]], !prof [[PROF1]]
+; CHECK:       2:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR3]]
+; CHECK-NEXT:    unreachable
+; CHECK:       3:
+; CHECK-NEXT:    [[TMP3:%.*]] = call i64 @llvm.aarch64.neon.fcvtnu.i64.f32(float [[A]])
+; CHECK-NEXT:    store i64 0, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i64 [[TMP3]]
+;
+  %tmpvar3 = call i64 @llvm.aarch64.neon.fcvtnu.i64.f32(float %A)
+  ret i64 %tmpvar3
+}
+
+define i32 @fcvtnu_1w1d(double %A) nounwind #0 {
+; CHECK-LABEL: define i32 @fcvtnu_1w1d(
+; CHECK-SAME: double [[A:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP2:%.*]], label [[TMP4:%.*]], !prof [[PROF1]]
+; CHECK:       2:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR3]]
+; CHECK-NEXT:    unreachable
+; CHECK:       3:
+; CHECK-NEXT:    [[TMP3:%.*]] = call i32 @llvm.aarch64.neon.fcvtnu.i32.f64(double [[A]])
+; CHECK-NEXT:    store i32 0, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i32 [[TMP3]]
+;
+  %tmpvar3 = call i32 @llvm.aarch64.neon.fcvtnu.i32.f64(double %A)
+  ret i32 %tmpvar3
+}
+
+define i64 @fcvtnu_1x1d(double %A) nounwind #0 {
+; CHECK-LABEL: define i64 @fcvtnu_1x1d(
+; CHECK-SAME: double [[A:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP2:%.*]], label [[TMP4:%.*]], !prof [[PROF1]]
+; CHECK:       2:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR3]]
+; CHECK-NEXT:    unreachable
+; CHECK:       3:
+; CHECK-NEXT:    [[TMP3:%.*]] = call i64 @llvm.aarch64.neon.fcvtnu.i64.f64(double [[A]])
+; CHECK-NEXT:    store i64 0, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i64 [[TMP3]]
+;
+  %tmpvar3 = call i64 @llvm.aarch64.neon.fcvtnu.i64.f64(double %A)
+  ret i64 %tmpvar3
+}
+
+declare i32 @llvm.aarch64.neon.fcvtnu.i32.f32(float) nounwind readnone
+declare i64 @llvm.aarch64.neon.fcvtnu.i64.f32(float) nounwind readnone
+declare i32 @llvm.aarch64.neon.fcvtnu.i32.f64(double) nounwind readnone
+declare i64 @llvm.aarch64.neon.fcvtnu.i64.f64(double) nounwind readnone
+
+define i32 @fcvtps_1w1s(float %A) nounwind #0 {
+; CHECK-LABEL: define i32 @fcvtps_1w1s(
+; CHECK-SAME: float [[A:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i32, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i32 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP2:%.*]], label [[TMP4:%.*]], !prof [[PROF1]]
+; CHECK:       2:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR3]]
+; CHECK-NEXT:    unreachable
+; CHECK:       3:
+; CHECK-NEXT:    [[TMP3:%.*]] = call i32 @llvm.aarch64.neon.fcvtps.i32.f32(float [[A]])
+; CHECK-NEXT:    store i32 0, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i32 [[TMP3]]
+;
+  %tmpvar3 = call i32 @llvm.aarch64.neon.fcvtps.i32.f32(float %A)
+  ret i32 %tmpvar3
+}
+
+define i64 @fcvtps_1x1s(float %A) nounwind #0 {
+; CHECK-LABEL: define i64 @fcvtps_1x1s(
+; CHECK-SAME: float [[A:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i32, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i32 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP2:%.*]], label [[TMP4:%.*]], !prof [[PROF1]]
+; CHECK:       2:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR3]]
+; CHECK-NEXT:    unreachable
+; CHECK:       3:
+; CHECK-NEXT:    [[TMP3:%.*]] = call i64 @llvm.aarch64.neon.fcvtps.i64.f32(float [[A]])
+; CHECK-NEXT:    store i64 0, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i64 [[TMP3]]
+;
+  %tmpvar3 = call i64 @llvm.aarch64.neon.fcvtps.i64.f32(float %A)
+  ret i64 %tmpvar3
+}
+
+define i32 @fcvtps_1w1d(double %A) nounwind #0 {
+; CHECK-LABEL: define i32 @fcvtps_1w1d(
+; CHECK-SAME: double [[A:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP2:%.*]], label [[TMP4:%.*]], !prof [[PROF1]]
+; CHECK:       2:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR3]]
+; CHECK-NEXT:    unreachable
+; CHECK:       3:
+; CHECK-NEXT:    [[TMP3:%.*]] = call i32 @llvm.aarch64.neon.fcvtps.i32.f64(double [[A]])
+; CHECK-NEXT:    store i32 0, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i32 [[TMP3]]
+;
+  %tmpvar3 = call i32 @llvm.aarch64.neon.fcvtps.i32.f64(double %A)
+  ret i32 %tmpvar3
+}
+
+define i64 @fcvtps_1x1d(double %A) nounwind #0 {
+; CHECK-LABEL: define i64 @fcvtps_1x1d(
+; CHECK-SAME: double [[A:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP2:%.*]], label [[TMP4:%.*]], !prof [[PROF1]]
+; CHECK:       2:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR3]]
+; CHECK-NEXT:    unreachable
+; CHECK:       3:
+; CHECK-NEXT:    [[TMP3:%.*]] = call i64 @llvm.aarch64.neon.fcvtps.i64.f64(double [[A]])
+; CHECK-NEXT:    store i64 0, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i64 [[TMP3]]
+;
+  %tmpvar3 = call i64 @llvm.aarch64.neon.fcvtps.i64.f64(double %A)
+  ret i64 %tmpvar3
+}
+
+declare i32 @llvm.aarch64.neon.fcvtps.i32.f32(float) nounwind readnone
+declare i64 @llvm.aarch64.neon.fcvtps.i64.f32(float) nounwind readnone
+declare i32 @llvm.aarch64.neon.fcvtps.i32.f64(double) nounwind readnone
+declare i64 @llvm.aarch64.neon.fcvtps.i64.f64(double) nounwind readnone
+
+define i32 @fcvtpu_1w1s(float %A) nounwind #0 {
+; CHECK-LABEL: define i32 @fcvtpu_1w1s(
+; CHECK-SAME: float [[A:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i32, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i32 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP2:%.*]], label [[TMP4:%.*]], !prof [[PROF1]]
+; CHECK:       2:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR3]]
+; CHECK-NEXT:    unreachable
+; CHECK:       3:
+; CHECK-NEXT:    [[TMP3:%.*]] = call i32 @llvm.aarch64.neon.fcvtpu.i32.f32(float [[A]])
+; CHECK-NEXT:    store i32 0, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i32 [[TMP3]]
+;
+  %tmpvar3 = call i32 @llvm.aarch64.neon.fcvtpu.i32.f32(float %A)
+  ret i32 %tmpvar3
+}
+
+define i64 @fcvtpu_1x1s(float %A) nounwind #0 {
+; CHECK-LABEL: define i64 @fcvtpu_1x1s(
+; CHECK-SAME: float [[A:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i32, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i32 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP2:%.*]], label [[TMP4:%.*]], !prof [[PROF1]]
+; CHECK:       2:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR3]]
+; CHECK-NEXT:    unreachable
+; CHECK:       3:
+; CHECK-NEXT:    [[TMP3:%.*]] = call i64 @llvm.aarch64.neon.fcvtpu.i64.f32(float [[A]])
+; CHECK-NEXT:    store i64 0, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i64 [[TMP3]]
+;
+  %tmpvar3 = call i64 @llvm.aarch64.neon.fcvtpu.i64.f32(float %A)
+  ret i64 %tmpvar3
+}
+
+define i32 @fcvtpu_1w1d(double %A) nounwind #0 {
+; CHECK-LABEL: define i32 @fcvtpu_1w1d(
+; CHECK-SAME: double [[A:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP2:%.*]], label [[TMP4:%.*]], !prof [[PROF1]]
+; CHECK:       2:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR3]]
+; CHECK-NEXT:    unreachable
+; CHECK:       3:
+; CHECK-NEXT:    [[TMP3:%.*]] = call i32 @llvm.aarch64.neon.fcvtpu.i32.f64(double [[A]])
+; CHECK-NEXT:    store i32 0, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i32 [[TMP3]]
+;
+  %tmpvar3 = call i32 @llvm.aarch64.neon.fcvtpu.i32.f64(double %A)
+  ret i32 %tmpvar3
+}
+
+define i64 @fcvtpu_1x1d(double %A) nounwind #0 {
+; CHECK-LABEL: define i64 @fcvtpu_1x1d(
+; CHECK-SAME: double [[A:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP2:%.*]], label [[TMP4:%.*]], !prof [[PROF1]]
+; CHECK:       2:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR3]]
+; CHECK-NEXT:    unreachable
+; CHECK:       3:
+; CHECK-NEXT:    [[TMP3:%.*]] = call i64 @llvm.aarch64.neon.fcvtpu.i64.f64(double [[A]])
+; CHECK-NEXT:    store i64 0, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i64 [[TMP3]]
+;
+  %tmpvar3 = call i64 @llvm.aarch64.neon.fcvtpu.i64.f64(double %A)
+  ret i64 %tmpvar3
+}
+
+declare i32 @llvm.aarch64.neon.fcvtpu.i32.f32(float) nounwind readnone
+declare i64 @llvm.aarch64.neon.fcvtpu.i64.f32(float) nounwind readnone
+declare i32 @llvm.aarch64.neon.fcvtpu.i32.f64(double) nounwind readnone
+declare i64 @llvm.aarch64.neon.fcvtpu.i64.f64(double) nounwind readnone
+
+define i32 @fcvtzs_1w1s(float %A) nounwind #0 {
+; CHECK-LABEL: define i32 @fcvtzs_1w1s(
+; CHECK-SAME: float [[A:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i32, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i32 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP2:%.*]], label [[TMP4:%.*]], !prof [[PROF1]]
+; CHECK:       2:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR3]]
+; CHECK-NEXT:    unreachable
+; CHECK:       3:
+; CHECK-NEXT:    [[TMP3:%.*]] = call i32 @llvm.aarch64.neon.fcvtzs.i32.f32(float [[A]])
+; CHECK-NEXT:    store i32 0, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i32 [[TMP3]]
+;
+  %tmpvar3 = call i32 @llvm.aarch64.neon.fcvtzs.i32.f32(float %A)
+  ret i32 %tmpvar3
+}
+
+define i64 @fcvtzs_1x1s(float %A) nounwind #0 {
+; CHECK-LABEL: define i64 @fcvtzs_1x1s(
+; CHECK-SAME: float [[A:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i32, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i32 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP2:%.*]], label [[TMP4:%.*]], !prof [[PROF1]]
+; CHECK:       2:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR3]]
+; CHECK-NEXT:    unreachable
+; CHECK:       3:
+; CHECK-NEXT:    [[TMP3:%.*]] = call i64 @llvm.aarch64.neon.fcvtzs.i64.f32(float [[A]])
+; CHECK-NEXT:    store i64 0, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i64 [[TMP3]]
+;
+  %tmpvar3 = call i64 @llvm.aarch64.neon.fcvtzs.i64.f32(float %A)
+  ret i64 %tmpvar3
+}
+
+define i32 @fcvtzs_1w1d(double %A) nounwind #0 {
+; CHECK-LABEL: define i32 @fcvtzs_1w1d(
+; CHECK-SAME: double [[A:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP2:%.*]], label [[TMP4:%.*]], !prof [[PROF1]]
+; CHECK:       2:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR3]]
+; CHECK-NEXT:    unreachable
+; CHECK:       3:
+; CHECK-NEXT:    [[TMP3:%.*]] = call i32 @llvm.aarch64.neon.fcvtzs.i32.f64(double [[A]])
+; CHECK-NEXT:    store i32 0, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i32 [[TMP3]]
+;
+  %tmpvar3 = call i32 @llvm.aarch64.neon.fcvtzs.i32.f64(double %A)
+  ret i32 %tmpvar3
+}
+
+define i64 @fcvtzs_1x1d(double %A) nounwind #0 {
+; CHECK-LABEL: define i64 @fcvtzs_1x1d(
+; CHECK-SAME: double [[A:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP2:%.*]], label [[TMP4:%.*]], !prof [[PROF1]]
+; CHECK:       2:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR3]]
+; CHECK-NEXT:    unreachable
+; CHECK:       3:
+; CHECK-NEXT:    [[TMP3:%.*]] = call i64 @llvm.aarch64.neon.fcvtzs.i64.f64(double [[A]])
+; CHECK-NEXT:    store i64 0, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i64 [[TMP3]]
+;
+  %tmpvar3 = call i64 @llvm.aarch64.neon.fcvtzs.i64.f64(double %A)
+  ret i64 %tmpvar3
+}
+
+declare i32 @llvm.aarch64.neon.fcvtzs.i32.f32(float) nounwind readnone
+declare i64 @llvm.aarch64.neon.fcvtzs.i64.f32(float) nounwind readnone
+declare i32 @llvm.aarch64.neon.fcvtzs.i32.f64(double) nounwind readnone
+declare i64 @llvm.aarch64.neon.fcvtzs.i64.f64(double) nounwind readnone
+
+define i32 @fcvtzu_1w1s(float %A) nounwind #0 {
+; CHECK-LABEL: define i32 @fcvtzu_1w1s(
+; CHECK-SAME: float [[A:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i32, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i32 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP2:%.*]], label [[TMP4:%.*]], !prof [[PROF1]]
+; CHECK:       2:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR3]]
+; CHECK-NEXT:    unreachable
+; CHECK:       3:
+; CHECK-NEXT:    [[TMP3:%.*]] = call i32 @llvm.aarch64.neon.fcvtzu.i32.f32(float [[A]])
+; CHECK-NEXT:    store i32 0, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i32 [[TMP3]]
+;
+  %tmpvar3 = call i32 @llvm.aarch64.neon.fcvtzu.i32.f32(float %A)
+  ret i32 %tmpvar3
+}
+
+define i64 @fcvtzu_1x1s(float %A) nounwind #0 {
+; CHECK-LABEL: define i64 @fcvtzu_1x1s(
+; CHECK-SAME: float [[A:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i32, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i32 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP2:%.*]], label [[TMP4:%.*]], !prof [[PROF1]]
+; CHECK:       2:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR3]]
+; CHECK-NEXT:    unreachable
+; CHECK:       3:
+; CHECK-NEXT:    [[TMP3:%.*]] = call i64 @llvm.aarch64.neon.fcvtzu.i64.f32(float [[A]])
+; CHECK-NEXT:    store i64 0, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i64 [[TMP3]]
+;
+  %tmpvar3 = call i64 @llvm.aarch64.neon.fcvtzu.i64.f32(float %A)
+  ret i64 %tmpvar3
+}
+
+define i32 @fcvtzu_1w1d(double %A) nounwind #0 {
+; CHECK-LABEL: define i32 @fcvtzu_1w1d(
+; CHECK-SAME: double [[A:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP2:%.*]], label [[TMP4:%.*]], !prof [[PROF1]]
+; CHECK:       2:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR3]]
+; CHECK-NEXT:    unreachable
+; CHECK:       3:
+; CHECK-NEXT:    [[TMP3:%.*]] = call i32 @llvm.aarch64.neon.fcvtzu.i32.f64(double [[A]])
+; CHECK-NEXT:    store i32 0, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i32 [[TMP3]]
+;
+  %tmpvar3 = call i32 @llvm.aarch64.neon.fcvtzu.i32.f64(double %A)
+  ret i32 %tmpvar3
+}
+
+define i64 @fcvtzu_1x1d(double %A) nounwind #0 {
+; CHECK-LABEL: define i64 @fcvtzu_1x1d(
+; CHECK-SAME: double [[A:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP2:%.*]], label [[TMP4:%.*]], !prof [[PROF1]]
+; CHECK:       2:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR3]]
+; CHECK-NEXT:    unreachable
+; CHECK:       3:
+; CHECK-NEXT:    [[TMP3:%.*]] = call i64 @llvm.aarch64.neon.fcvtzu.i64.f64(double [[A]])
+; CHECK-NEXT:    store i64 0, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i64 [[TMP3]]
+;
+  %tmpvar3 = call i64 @llvm.aarch64.neon.fcvtzu.i64.f64(double %A)
+  ret i64 %tmpvar3
+}
+
+declare i32 @llvm.aarch64.neon.fcvtzu.i32.f32(float) nounwind readnone
+declare i64 @llvm.aarch64.neon.fcvtzu.i64.f32(float) nounwind readnone
+declare i32 @llvm.aarch64.neon.fcvtzu.i32.f64(double) nounwind readnone
+declare i64 @llvm.aarch64.neon.fcvtzu.i64.f64(double) nounwind readnone
+
+attributes #0 = { sanitize_memory }
+;.
+; CHECK: [[PROF1]] = !{!"branch_weights", i32 1, i32 1048575}
+;.
diff --git a/llvm/test/Instrumentation/MemorySanitizer/AArch64/arm64-fminv.ll b/llvm/test/Instrumentation/MemorySanitizer/AArch64/arm64-fminv.ll
new file mode 100644
index 0000000000000..ff5784ada8c48
--- /dev/null
+++ b/llvm/test/Instrumentation/MemorySanitizer/AArch64/arm64-fminv.ll
@@ -0,0 +1,193 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 4
+; RUN: opt < %s -passes=msan -S | FileCheck %s
+;
+; Forked from llvm/test/CodeGen/AArch64/arm64-fminv.ll
+
+target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
+target triple = "aarch64--linux-android9001"
+
+define float @test_fminv_v2f32(<2 x float> %in) #0 {
+; CHECK-LABEL: define float @test_fminv_v2f32(
+; CHECK-SAME: <2 x float> [[IN:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = call i32 @llvm.vector.reduce.or.v2i32(<2 x i32> [[TMP1]])
+; CHECK-NEXT:    [[MIN:%.*]] = call float @llvm.aarch64.neon.fminv.f32.v2f32(<2 x float> [[IN]])
+; CHECK-NEXT:    store i32 [[TMP2]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret float [[MIN]]
+;
+  %min = call float @llvm.aarch64.neon.fminv.f32.v2f32(<2 x float> %in)
+  ret float %min
+}
+
+define float @test_fminv_v4f32(<4 x float> %in) #0 {
+; CHECK-LABEL: define float @test_fminv_v4f32(
+; CHECK-SAME: <4 x float> [[IN:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = call i32 @llvm.vector.reduce.or.v4i32(<4 x i32> [[TMP1]])
+; CHECK-NEXT:    [[MIN:%.*]] = call float @llvm.aarch64.neon.fminv.f32.v4f32(<4 x float> [[IN]])
+; CHECK-NEXT:    store i32 [[TMP2]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret float [[MIN]]
+;
+  %min = call float @llvm.aarch64.neon.fminv.f32.v4f32(<4 x float> %in)
+  ret float %min
+}
+
+define double @test_fminv_v2f64(<2 x double> %in) #0 {
+; CHECK-LABEL: define double @test_fminv_v2f64(
+; CHECK-SAME: <2 x double> [[IN:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vector.reduce.or.v2i64(<2 x i64> [[TMP1]])
+; CHECK-NEXT:    [[MIN:%.*]] = call double @llvm.aarch64.neon.fminv.f64.v2f64(<2 x double> [[IN]])
+; CHECK-NEXT:    store i64 [[TMP2]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret double [[MIN]]
+;
+  %min = call double @llvm.aarch64.neon.fminv.f64.v2f64(<2 x double> %in)
+  ret double %min
+}
+
+declare float @llvm.aarch64.neon.fminv.f32.v2f32(<2 x float>)
+declare float @llvm.aarch64.neon.fminv.f32.v4f32(<4 x float>)
+declare double @llvm.aarch64.neon.fminv.f64.v2f64(<2 x double>)
+
+define float @test_fmaxv_v2f32(<2 x float> %in) #0 {
+; CHECK-LABEL: define float @test_fmaxv_v2f32(
+; CHECK-SAME: <2 x float> [[IN:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = call i32 @llvm.vector.reduce.or.v2i32(<2 x i32> [[TMP1]])
+; CHECK-NEXT:    [[MAX:%.*]] = call float @llvm.aarch64.neon.fmaxv.f32.v2f32(<2 x float> [[IN]])
+; CHECK-NEXT:    store i32 [[TMP2]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret float [[MAX]]
+;
+  %max = call float @llvm.aarch64.neon.fmaxv.f32.v2f32(<2 x float> %in)
+  ret float %max
+}
+
+define float @test_fmaxv_v4f32(<4 x float> %in) #0 {
+; CHECK-LABEL: define float @test_fmaxv_v4f32(
+; CHECK-SAME: <4 x float> [[IN:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = call i32 @llvm.vector.reduce.or.v4i32(<4 x i32> [[TMP1]])
+; CHECK-NEXT:    [[MAX:%.*]] = call float @llvm.aarch64.neon.fmaxv.f32.v4f32(<4 x float> [[IN]])
+; CHECK-NEXT:    store i32 [[TMP2]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret float [[MAX]]
+;
+  %max = call float @llvm.aarch64.neon.fmaxv.f32.v4f32(<4 x float> %in)
+  ret float %max
+}
+
+define double @test_fmaxv_v2f64(<2 x double> %shareholder_value) #0 {
+; CHECK-LABEL: define double @test_fmaxv_v2f64(
+; CHECK-SAME: <2 x double> [[SHAREHOLDER_VALUE:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vector.reduce.or.v2i64(<2 x i64> [[TMP1]])
+; CHECK-NEXT:    [[MAX:%.*]] = call double @llvm.aarch64.neon.fmaxv.f64.v2f64(<2 x double> [[SHAREHOLDER_VALUE]])
+; CHECK-NEXT:    store i64 [[TMP2]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret double [[MAX]]
+;
+  %max_sv = call double @llvm.aarch64.neon.fmaxv.f64.v2f64(<2 x double> %shareholder_value)
+  ret double %max_sv
+}
+
+declare float @llvm.aarch64.neon.fmaxv.f32.v2f32(<2 x float>)
+declare float @llvm.aarch64.neon.fmaxv.f32.v4f32(<4 x float>)
+declare double @llvm.aarch64.neon.fmaxv.f64.v2f64(<2 x double>)
+
+define float @test_fminnmv_v2f32(<2 x float> %in) #0 {
+; CHECK-LABEL: define float @test_fminnmv_v2f32(
+; CHECK-SAME: <2 x float> [[IN:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = call i32 @llvm.vector.reduce.or.v2i32(<2 x i32> [[TMP1]])
+; CHECK-NEXT:    [[MINNM:%.*]] = call float @llvm.aarch64.neon.fminnmv.f32.v2f32(<2 x float> [[IN]])
+; CHECK-NEXT:    store i32 [[TMP2]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret float [[MINNM]]
+;
+  %minnm = call float @llvm.aarch64.neon.fminnmv.f32.v2f32(<2 x float> %in)
+  ret float %minnm
+}
+
+define float @test_fminnmv_v4f32(<4 x float> %in) #0 {
+; CHECK-LABEL: define float @test_fminnmv_v4f32(
+; CHECK-SAME: <4 x float> [[IN:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = call i32 @llvm.vector.reduce.or.v4i32(<4 x i32> [[TMP1]])
+; CHECK-NEXT:    [[MINNM:%.*]] = call float @llvm.aarch64.neon.fminnmv.f32.v4f32(<4 x float> [[IN]])
+; CHECK-NEXT:    store i32 [[TMP2]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret float [[MINNM]]
+;
+  %minnm = call float @llvm.aarch64.neon.fminnmv.f32.v4f32(<4 x float> %in)
+  ret float %minnm
+}
+
+define double @test_fminnmv_v2f64(<2 x double> %in) #0 {
+; CHECK-LABEL: define double @test_fminnmv_v2f64(
+; CHECK-SAME: <2 x double> [[IN:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vector.reduce.or.v2i64(<2 x i64> [[TMP1]])
+; CHECK-NEXT:    [[MINNM:%.*]] = call double @llvm.aarch64.neon.fminnmv.f64.v2f64(<2 x double> [[IN]])
+; CHECK-NEXT:    store i64 [[TMP2]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret double [[MINNM]]
+;
+  %minnm = call double @llvm.aarch64.neon.fminnmv.f64.v2f64(<2 x double> %in)
+  ret double %minnm
+}
+
+declare float @llvm.aarch64.neon.fminnmv.f32.v2f32(<2 x float>)
+declare float @llvm.aarch64.neon.fminnmv.f32.v4f32(<4 x float>)
+declare double @llvm.aarch64.neon.fminnmv.f64.v2f64(<2 x double>)
+
+define float @test_fmaxnmv_v2f32(<2 x float> %in) #0 {
+; CHECK-LABEL: define float @test_fmaxnmv_v2f32(
+; CHECK-SAME: <2 x float> [[IN:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = call i32 @llvm.vector.reduce.or.v2i32(<2 x i32> [[TMP1]])
+; CHECK-NEXT:    [[MAXNM:%.*]] = call float @llvm.aarch64.neon.fmaxnmv.f32.v2f32(<2 x float> [[IN]])
+; CHECK-NEXT:    store i32 [[TMP2]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret float [[MAXNM]]
+;
+  %maxnm = call float @llvm.aarch64.neon.fmaxnmv.f32.v2f32(<2 x float> %in)
+  ret float %maxnm
+}
+
+define float @test_fmaxnmv_v4f32(<4 x float> %in) #0 {
+; CHECK-LABEL: define float @test_fmaxnmv_v4f32(
+; CHECK-SAME: <4 x float> [[IN:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = call i32 @llvm.vector.reduce.or.v4i32(<4 x i32> [[TMP1]])
+; CHECK-NEXT:    [[MAXNM:%.*]] = call float @llvm.aarch64.neon.fmaxnmv.f32.v4f32(<4 x float> [[IN]])
+; CHECK-NEXT:    store i32 [[TMP2]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret float [[MAXNM]]
+;
+  %maxnm = call float @llvm.aarch64.neon.fmaxnmv.f32.v4f32(<4 x float> %in)
+  ret float %maxnm
+}
+
+define double @test_fmaxnmv_v2f64(<2 x double> %in) #0 {
+; CHECK-LABEL: define double @test_fmaxnmv_v2f64(
+; CHECK-SAME: <2 x double> [[IN:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vector.reduce.or.v2i64(<2 x i64> [[TMP1]])
+; CHECK-NEXT:    [[MAXNM:%.*]] = call double @llvm.aarch64.neon.fmaxnmv.f64.v2f64(<2 x double> [[IN]])
+; CHECK-NEXT:    store i64 [[TMP2]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret double [[MAXNM]]
+;
+  %maxnm = call double @llvm.aarch64.neon.fmaxnmv.f64.v2f64(<2 x double> %in)
+  ret double %maxnm
+}
+
+declare float @llvm.aarch64.neon.fmaxnmv.f32.v2f32(<2 x float>)
+declare float @llvm.aarch64.neon.fmaxnmv.f32.v4f32(<4 x float>)
+declare double @llvm.aarch64.neon.fmaxnmv.f64.v2f64(<2 x double>)
+
+attributes #0 = { sanitize_memory }
diff --git a/llvm/test/Instrumentation/MemorySanitizer/AArch64/arm64-vadd.ll b/llvm/test/Instrumentation/MemorySanitizer/AArch64/arm64-vadd.ll
new file mode 100644
index 0000000000000..f9b223dc420b9
--- /dev/null
+++ b/llvm/test/Instrumentation/MemorySanitizer/AArch64/arm64-vadd.ll
@@ -0,0 +1,3824 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 4
+; RUN: opt < %s -passes=msan -S | FileCheck %s
+;
+; Handled heuristically but incorrectly:
+; - llvm.aarch64.neon.{addp, faddp}
+;
+; Unknown instructions handled by visitInstruction:
+; - Incorrectly: llvm.aarch64.neon.{addhn, raddhn}
+; - Suboptimally: llvm.aarch64.neon.{saddlp, uaddlp}
+;
+; Forked from llvm/test/CodeGen/AArch64/arm64-vadd.ll
+
+target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
+target triple = "aarch64--linux-android9001"
+
+define <8 x i8> @addhn8b(ptr %A, ptr %B) nounwind #0 {
+; CHECK-LABEL: define <8 x i8> @addhn8b(
+; CHECK-SAME: ptr [[A:%.*]], ptr [[B:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP17:%.*]], label [[TMP4:%.*]], !prof [[PROF1:![0-9]+]]
+; CHECK:       3:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4:[0-9]+]]
+; CHECK-NEXT:    unreachable
+; CHECK:       4:
+; CHECK-NEXT:    [[TMPVAR1:%.*]] = load <8 x i16>, ptr [[A]], align 16
+; CHECK-NEXT:    [[TMP5:%.*]] = ptrtoint ptr [[A]] to i64
+; CHECK-NEXT:    [[TMP6:%.*]] = xor i64 [[TMP5]], 193514046488576
+; CHECK-NEXT:    [[TMP7:%.*]] = inttoptr i64 [[TMP6]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <8 x i16>, ptr [[TMP7]], align 16
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i64 [[TMP2]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP2]], label [[TMP8:%.*]], label [[TMP9:%.*]], !prof [[PROF1]]
+; CHECK:       8:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       9:
+; CHECK-NEXT:    [[TMPVAR2:%.*]] = load <8 x i16>, ptr [[B]], align 16
+; CHECK-NEXT:    [[TMP10:%.*]] = ptrtoint ptr [[B]] to i64
+; CHECK-NEXT:    [[TMP11:%.*]] = xor i64 [[TMP10]], 193514046488576
+; CHECK-NEXT:    [[TMP12:%.*]] = inttoptr i64 [[TMP11]] to ptr
+; CHECK-NEXT:    [[_MSLD1:%.*]] = load <8 x i16>, ptr [[TMP12]], align 16
+; CHECK-NEXT:    [[TMP13:%.*]] = bitcast <8 x i16> [[_MSLD]] to i128
+; CHECK-NEXT:    [[_MSCMP3:%.*]] = icmp ne i128 [[TMP13]], 0
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast <8 x i16> [[_MSLD1]] to i128
+; CHECK-NEXT:    [[_MSCMP4:%.*]] = icmp ne i128 [[TMP14]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP3]], [[_MSCMP4]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP15:%.*]], label [[TMP16:%.*]], !prof [[PROF1]]
+; CHECK:       15:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       16:
+; CHECK-NEXT:    [[TMP3:%.*]] = call <8 x i8> @llvm.aarch64.neon.addhn.v8i8(<8 x i16> [[TMPVAR1]], <8 x i16> [[TMPVAR2]])
+; CHECK-NEXT:    store <8 x i8> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i8> [[TMP3]]
+;
+  %tmpvar1 = load <8 x i16>, ptr %A
+  %tmpvar2 = load <8 x i16>, ptr %B
+  %tmpvar3 = call <8 x i8> @llvm.aarch64.neon.addhn.v8i8(<8 x i16> %tmpvar1, <8 x i16> %tmpvar2)
+  ret <8 x i8> %tmpvar3
+}
+
+define <4 x i16> @addhn4h(ptr %A, ptr %B) nounwind #0 {
+; CHECK-LABEL: define <4 x i16> @addhn4h(
+; CHECK-SAME: ptr [[A:%.*]], ptr [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP17:%.*]], label [[TMP4:%.*]], !prof [[PROF1]]
+; CHECK:       3:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       4:
+; CHECK-NEXT:    [[TMPVAR1:%.*]] = load <4 x i32>, ptr [[A]], align 16
+; CHECK-NEXT:    [[TMP5:%.*]] = ptrtoint ptr [[A]] to i64
+; CHECK-NEXT:    [[TMP6:%.*]] = xor i64 [[TMP5]], 193514046488576
+; CHECK-NEXT:    [[TMP7:%.*]] = inttoptr i64 [[TMP6]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <4 x i32>, ptr [[TMP7]], align 16
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i64 [[TMP2]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP2]], label [[TMP8:%.*]], label [[TMP9:%.*]], !prof [[PROF1]]
+; CHECK:       8:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       9:
+; CHECK-NEXT:    [[TMPVAR2:%.*]] = load <4 x i32>, ptr [[B]], align 16
+; CHECK-NEXT:    [[TMP10:%.*]] = ptrtoint ptr [[B]] to i64
+; CHECK-NEXT:    [[TMP11:%.*]] = xor i64 [[TMP10]], 193514046488576
+; CHECK-NEXT:    [[TMP12:%.*]] = inttoptr i64 [[TMP11]] to ptr
+; CHECK-NEXT:    [[_MSLD1:%.*]] = load <4 x i32>, ptr [[TMP12]], align 16
+; CHECK-NEXT:    [[TMP13:%.*]] = bitcast <4 x i32> [[_MSLD]] to i128
+; CHECK-NEXT:    [[_MSCMP3:%.*]] = icmp ne i128 [[TMP13]], 0
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast <4 x i32> [[_MSLD1]] to i128
+; CHECK-NEXT:    [[_MSCMP4:%.*]] = icmp ne i128 [[TMP14]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP3]], [[_MSCMP4]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP15:%.*]], label [[TMP16:%.*]], !prof [[PROF1]]
+; CHECK:       15:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       16:
+; CHECK-NEXT:    [[TMP3:%.*]] = call <4 x i16> @llvm.aarch64.neon.addhn.v4i16(<4 x i32> [[TMPVAR1]], <4 x i32> [[TMPVAR2]])
+; CHECK-NEXT:    store <4 x i16> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i16> [[TMP3]]
+;
+  %tmpvar1 = load <4 x i32>, ptr %A
+  %tmpvar2 = load <4 x i32>, ptr %B
+  %tmpvar3 = call <4 x i16> @llvm.aarch64.neon.addhn.v4i16(<4 x i32> %tmpvar1, <4 x i32> %tmpvar2)
+  ret <4 x i16> %tmpvar3
+}
+
+define <2 x i32> @addhn2s(ptr %A, ptr %B) nounwind #0 {
+; CHECK-LABEL: define <2 x i32> @addhn2s(
+; CHECK-SAME: ptr [[A:%.*]], ptr [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP17:%.*]], label [[TMP4:%.*]], !prof [[PROF1]]
+; CHECK:       3:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       4:
+; CHECK-NEXT:    [[TMPVAR1:%.*]] = load <2 x i64>, ptr [[A]], align 16
+; CHECK-NEXT:    [[TMP5:%.*]] = ptrtoint ptr [[A]] to i64
+; CHECK-NEXT:    [[TMP6:%.*]] = xor i64 [[TMP5]], 193514046488576
+; CHECK-NEXT:    [[TMP7:%.*]] = inttoptr i64 [[TMP6]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <2 x i64>, ptr [[TMP7]], align 16
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i64 [[TMP2]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP2]], label [[TMP8:%.*]], label [[TMP9:%.*]], !prof [[PROF1]]
+; CHECK:       8:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       9:
+; CHECK-NEXT:    [[TMPVAR2:%.*]] = load <2 x i64>, ptr [[B]], align 16
+; CHECK-NEXT:    [[TMP10:%.*]] = ptrtoint ptr [[B]] to i64
+; CHECK-NEXT:    [[TMP11:%.*]] = xor i64 [[TMP10]], 193514046488576
+; CHECK-NEXT:    [[TMP12:%.*]] = inttoptr i64 [[TMP11]] to ptr
+; CHECK-NEXT:    [[_MSLD1:%.*]] = load <2 x i64>, ptr [[TMP12]], align 16
+; CHECK-NEXT:    [[TMP13:%.*]] = bitcast <2 x i64> [[_MSLD]] to i128
+; CHECK-NEXT:    [[_MSCMP3:%.*]] = icmp ne i128 [[TMP13]], 0
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast <2 x i64> [[_MSLD1]] to i128
+; CHECK-NEXT:    [[_MSCMP4:%.*]] = icmp ne i128 [[TMP14]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP3]], [[_MSCMP4]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP15:%.*]], label [[TMP16:%.*]], !prof [[PROF1]]
+; CHECK:       15:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       16:
+; CHECK-NEXT:    [[TMP3:%.*]] = call <2 x i32> @llvm.aarch64.neon.addhn.v2i32(<2 x i64> [[TMPVAR1]], <2 x i64> [[TMPVAR2]])
+; CHECK-NEXT:    store <2 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x i32> [[TMP3]]
+;
+  %tmpvar1 = load <2 x i64>, ptr %A
+  %tmpvar2 = load <2 x i64>, ptr %B
+  %tmpvar3 = call <2 x i32> @llvm.aarch64.neon.addhn.v2i32(<2 x i64> %tmpvar1, <2 x i64> %tmpvar2)
+  ret <2 x i32> %tmpvar3
+}
+
+define <16 x i8> @addhn2_16b(<8 x i16> %a, <8 x i16> %b) nounwind #0 {
+; CHECK-LABEL: define <16 x i8> @addhn2_16b(
+; CHECK-SAME: <8 x i16> [[A:%.*]], <8 x i16> [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <8 x i16> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP3]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <8 x i16> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP5:%.*]], label [[TMP6:%.*]], !prof [[PROF1]]
+; CHECK:       5:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       6:
+; CHECK-NEXT:    [[VADDHN2_I:%.*]] = tail call <8 x i8> @llvm.aarch64.neon.addhn.v8i8(<8 x i16> [[A]], <8 x i16> [[B]]) #[[ATTR3:[0-9]+]]
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <8 x i16> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i128 [[TMP7]], 0
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <8 x i16> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP3:%.*]] = icmp ne i128 [[TMP8]], 0
+; CHECK-NEXT:    [[_MSOR4:%.*]] = or i1 [[_MSCMP2]], [[_MSCMP3]]
+; CHECK-NEXT:    br i1 [[_MSOR4]], label [[TMP9:%.*]], label [[TMP10:%.*]], !prof [[PROF1]]
+; CHECK:       9:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       10:
+; CHECK-NEXT:    [[VADDHN_HIGH2_I:%.*]] = tail call <8 x i8> @llvm.aarch64.neon.addhn.v8i8(<8 x i16> [[A]], <8 x i16> [[B]]) #[[ATTR3]]
+; CHECK-NEXT:    [[RES:%.*]] = shufflevector <8 x i8> [[VADDHN2_I]], <8 x i8> [[VADDHN_HIGH2_I]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; CHECK-NEXT:    store <16 x i8> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i8> [[RES]]
+;
+  %vaddhn2.i = tail call <8 x i8> @llvm.aarch64.neon.addhn.v8i8(<8 x i16> %a, <8 x i16> %b) nounwind
+  %vaddhn_high2.i = tail call <8 x i8> @llvm.aarch64.neon.addhn.v8i8(<8 x i16> %a, <8 x i16> %b) nounwind
+  %res = shufflevector <8 x i8> %vaddhn2.i, <8 x i8> %vaddhn_high2.i, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+  ret <16 x i8> %res
+}
+
+define <8 x i16> @addhn2_8h(<4 x i32> %a, <4 x i32> %b) nounwind #0 {
+; CHECK-LABEL: define <8 x i16> @addhn2_8h(
+; CHECK-SAME: <4 x i32> [[A:%.*]], <4 x i32> [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <4 x i32> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP3]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <4 x i32> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP5:%.*]], label [[TMP6:%.*]], !prof [[PROF1]]
+; CHECK:       5:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       6:
+; CHECK-NEXT:    [[VADDHN2_I:%.*]] = tail call <4 x i16> @llvm.aarch64.neon.addhn.v4i16(<4 x i32> [[A]], <4 x i32> [[B]]) #[[ATTR3]]
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <4 x i32> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i128 [[TMP7]], 0
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <4 x i32> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP3:%.*]] = icmp ne i128 [[TMP8]], 0
+; CHECK-NEXT:    [[_MSOR4:%.*]] = or i1 [[_MSCMP2]], [[_MSCMP3]]
+; CHECK-NEXT:    br i1 [[_MSOR4]], label [[TMP9:%.*]], label [[TMP10:%.*]], !prof [[PROF1]]
+; CHECK:       9:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       10:
+; CHECK-NEXT:    [[VADDHN_HIGH3_I:%.*]] = tail call <4 x i16> @llvm.aarch64.neon.addhn.v4i16(<4 x i32> [[A]], <4 x i32> [[B]]) #[[ATTR3]]
+; CHECK-NEXT:    [[RES:%.*]] = shufflevector <4 x i16> [[VADDHN2_I]], <4 x i16> [[VADDHN_HIGH3_I]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:    store <8 x i16> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i16> [[RES]]
+;
+  %vaddhn2.i = tail call <4 x i16> @llvm.aarch64.neon.addhn.v4i16(<4 x i32> %a, <4 x i32> %b) nounwind
+  %vaddhn_high3.i = tail call <4 x i16> @llvm.aarch64.neon.addhn.v4i16(<4 x i32> %a, <4 x i32> %b) nounwind
+  %res = shufflevector <4 x i16> %vaddhn2.i, <4 x i16> %vaddhn_high3.i, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+  ret <8 x i16> %res
+}
+
+define <4 x i32> @addhn2_4s(<2 x i64> %a, <2 x i64> %b) nounwind #0 {
+; CHECK-LABEL: define <4 x i32> @addhn2_4s(
+; CHECK-SAME: <2 x i64> [[A:%.*]], <2 x i64> [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <2 x i64> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP3]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <2 x i64> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP5:%.*]], label [[TMP6:%.*]], !prof [[PROF1]]
+; CHECK:       5:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       6:
+; CHECK-NEXT:    [[VADDHN2_I:%.*]] = tail call <2 x i32> @llvm.aarch64.neon.addhn.v2i32(<2 x i64> [[A]], <2 x i64> [[B]]) #[[ATTR3]]
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <2 x i64> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i128 [[TMP7]], 0
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <2 x i64> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP3:%.*]] = icmp ne i128 [[TMP8]], 0
+; CHECK-NEXT:    [[_MSOR4:%.*]] = or i1 [[_MSCMP2]], [[_MSCMP3]]
+; CHECK-NEXT:    br i1 [[_MSOR4]], label [[TMP9:%.*]], label [[TMP10:%.*]], !prof [[PROF1]]
+; CHECK:       9:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       10:
+; CHECK-NEXT:    [[VADDHN_HIGH3_I:%.*]] = tail call <2 x i32> @llvm.aarch64.neon.addhn.v2i32(<2 x i64> [[A]], <2 x i64> [[B]]) #[[ATTR3]]
+; CHECK-NEXT:    [[RES:%.*]] = shufflevector <2 x i32> [[VADDHN2_I]], <2 x i32> [[VADDHN_HIGH3_I]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    store <4 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i32> [[RES]]
+;
+  %vaddhn2.i = tail call <2 x i32> @llvm.aarch64.neon.addhn.v2i32(<2 x i64> %a, <2 x i64> %b) nounwind
+  %vaddhn_high3.i = tail call <2 x i32> @llvm.aarch64.neon.addhn.v2i32(<2 x i64> %a, <2 x i64> %b) nounwind
+  %res = shufflevector <2 x i32> %vaddhn2.i, <2 x i32> %vaddhn_high3.i, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  ret <4 x i32> %res
+}
+
+declare <2 x i32> @llvm.aarch64.neon.addhn.v2i32(<2 x i64>, <2 x i64>) nounwind readnone
+declare <4 x i16> @llvm.aarch64.neon.addhn.v4i16(<4 x i32>, <4 x i32>) nounwind readnone
+declare <8 x i8> @llvm.aarch64.neon.addhn.v8i8(<8 x i16>, <8 x i16>) nounwind readnone
+
+
+define <8 x i8> @raddhn8b(ptr %A, ptr %B) nounwind #0 {
+; CHECK-LABEL: define <8 x i8> @raddhn8b(
+; CHECK-SAME: ptr [[A:%.*]], ptr [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP17:%.*]], label [[TMP4:%.*]], !prof [[PROF1]]
+; CHECK:       3:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       4:
+; CHECK-NEXT:    [[TMPVAR1:%.*]] = load <8 x i16>, ptr [[A]], align 16
+; CHECK-NEXT:    [[TMP5:%.*]] = ptrtoint ptr [[A]] to i64
+; CHECK-NEXT:    [[TMP6:%.*]] = xor i64 [[TMP5]], 193514046488576
+; CHECK-NEXT:    [[TMP7:%.*]] = inttoptr i64 [[TMP6]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <8 x i16>, ptr [[TMP7]], align 16
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i64 [[TMP2]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP2]], label [[TMP8:%.*]], label [[TMP9:%.*]], !prof [[PROF1]]
+; CHECK:       8:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       9:
+; CHECK-NEXT:    [[TMPVAR2:%.*]] = load <8 x i16>, ptr [[B]], align 16
+; CHECK-NEXT:    [[TMP10:%.*]] = ptrtoint ptr [[B]] to i64
+; CHECK-NEXT:    [[TMP11:%.*]] = xor i64 [[TMP10]], 193514046488576
+; CHECK-NEXT:    [[TMP12:%.*]] = inttoptr i64 [[TMP11]] to ptr
+; CHECK-NEXT:    [[_MSLD1:%.*]] = load <8 x i16>, ptr [[TMP12]], align 16
+; CHECK-NEXT:    [[TMP13:%.*]] = bitcast <8 x i16> [[_MSLD]] to i128
+; CHECK-NEXT:    [[_MSCMP3:%.*]] = icmp ne i128 [[TMP13]], 0
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast <8 x i16> [[_MSLD1]] to i128
+; CHECK-NEXT:    [[_MSCMP4:%.*]] = icmp ne i128 [[TMP14]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP3]], [[_MSCMP4]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP15:%.*]], label [[TMP16:%.*]], !prof [[PROF1]]
+; CHECK:       15:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       16:
+; CHECK-NEXT:    [[TMP3:%.*]] = call <8 x i8> @llvm.aarch64.neon.raddhn.v8i8(<8 x i16> [[TMPVAR1]], <8 x i16> [[TMPVAR2]])
+; CHECK-NEXT:    store <8 x i8> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i8> [[TMP3]]
+;
+  %tmpvar1 = load <8 x i16>, ptr %A
+  %tmpvar2 = load <8 x i16>, ptr %B
+  %tmpvar3 = call <8 x i8> @llvm.aarch64.neon.raddhn.v8i8(<8 x i16> %tmpvar1, <8 x i16> %tmpvar2)
+  ret <8 x i8> %tmpvar3
+}
+
+define <4 x i16> @raddhn4h(ptr %A, ptr %B) nounwind #0 {
+; CHECK-LABEL: define <4 x i16> @raddhn4h(
+; CHECK-SAME: ptr [[A:%.*]], ptr [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP17:%.*]], label [[TMP4:%.*]], !prof [[PROF1]]
+; CHECK:       3:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       4:
+; CHECK-NEXT:    [[TMPVAR1:%.*]] = load <4 x i32>, ptr [[A]], align 16
+; CHECK-NEXT:    [[TMP5:%.*]] = ptrtoint ptr [[A]] to i64
+; CHECK-NEXT:    [[TMP6:%.*]] = xor i64 [[TMP5]], 193514046488576
+; CHECK-NEXT:    [[TMP7:%.*]] = inttoptr i64 [[TMP6]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <4 x i32>, ptr [[TMP7]], align 16
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i64 [[TMP2]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP2]], label [[TMP8:%.*]], label [[TMP9:%.*]], !prof [[PROF1]]
+; CHECK:       8:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       9:
+; CHECK-NEXT:    [[TMPVAR2:%.*]] = load <4 x i32>, ptr [[B]], align 16
+; CHECK-NEXT:    [[TMP10:%.*]] = ptrtoint ptr [[B]] to i64
+; CHECK-NEXT:    [[TMP11:%.*]] = xor i64 [[TMP10]], 193514046488576
+; CHECK-NEXT:    [[TMP12:%.*]] = inttoptr i64 [[TMP11]] to ptr
+; CHECK-NEXT:    [[_MSLD1:%.*]] = load <4 x i32>, ptr [[TMP12]], align 16
+; CHECK-NEXT:    [[TMP13:%.*]] = bitcast <4 x i32> [[_MSLD]] to i128
+; CHECK-NEXT:    [[_MSCMP3:%.*]] = icmp ne i128 [[TMP13]], 0
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast <4 x i32> [[_MSLD1]] to i128
+; CHECK-NEXT:    [[_MSCMP4:%.*]] = icmp ne i128 [[TMP14]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP3]], [[_MSCMP4]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP15:%.*]], label [[TMP16:%.*]], !prof [[PROF1]]
+; CHECK:       15:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       16:
+; CHECK-NEXT:    [[TMP3:%.*]] = call <4 x i16> @llvm.aarch64.neon.raddhn.v4i16(<4 x i32> [[TMPVAR1]], <4 x i32> [[TMPVAR2]])
+; CHECK-NEXT:    store <4 x i16> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i16> [[TMP3]]
+;
+  %tmpvar1 = load <4 x i32>, ptr %A
+  %tmpvar2 = load <4 x i32>, ptr %B
+  %tmpvar3 = call <4 x i16> @llvm.aarch64.neon.raddhn.v4i16(<4 x i32> %tmpvar1, <4 x i32> %tmpvar2)
+  ret <4 x i16> %tmpvar3
+}
+
+define <2 x i32> @raddhn2s(ptr %A, ptr %B) nounwind #0 {
+; CHECK-LABEL: define <2 x i32> @raddhn2s(
+; CHECK-SAME: ptr [[A:%.*]], ptr [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP17:%.*]], label [[TMP4:%.*]], !prof [[PROF1]]
+; CHECK:       3:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       4:
+; CHECK-NEXT:    [[TMPVAR1:%.*]] = load <2 x i64>, ptr [[A]], align 16
+; CHECK-NEXT:    [[TMP5:%.*]] = ptrtoint ptr [[A]] to i64
+; CHECK-NEXT:    [[TMP6:%.*]] = xor i64 [[TMP5]], 193514046488576
+; CHECK-NEXT:    [[TMP7:%.*]] = inttoptr i64 [[TMP6]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <2 x i64>, ptr [[TMP7]], align 16
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i64 [[TMP2]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP2]], label [[TMP8:%.*]], label [[TMP9:%.*]], !prof [[PROF1]]
+; CHECK:       8:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       9:
+; CHECK-NEXT:    [[TMPVAR2:%.*]] = load <2 x i64>, ptr [[B]], align 16
+; CHECK-NEXT:    [[TMP10:%.*]] = ptrtoint ptr [[B]] to i64
+; CHECK-NEXT:    [[TMP11:%.*]] = xor i64 [[TMP10]], 193514046488576
+; CHECK-NEXT:    [[TMP12:%.*]] = inttoptr i64 [[TMP11]] to ptr
+; CHECK-NEXT:    [[_MSLD1:%.*]] = load <2 x i64>, ptr [[TMP12]], align 16
+; CHECK-NEXT:    [[TMP13:%.*]] = bitcast <2 x i64> [[_MSLD]] to i128
+; CHECK-NEXT:    [[_MSCMP3:%.*]] = icmp ne i128 [[TMP13]], 0
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast <2 x i64> [[_MSLD1]] to i128
+; CHECK-NEXT:    [[_MSCMP4:%.*]] = icmp ne i128 [[TMP14]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP3]], [[_MSCMP4]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP15:%.*]], label [[TMP16:%.*]], !prof [[PROF1]]
+; CHECK:       15:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       16:
+; CHECK-NEXT:    [[TMP3:%.*]] = call <2 x i32> @llvm.aarch64.neon.raddhn.v2i32(<2 x i64> [[TMPVAR1]], <2 x i64> [[TMPVAR2]])
+; CHECK-NEXT:    store <2 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x i32> [[TMP3]]
+;
+  %tmpvar1 = load <2 x i64>, ptr %A
+  %tmpvar2 = load <2 x i64>, ptr %B
+  %tmpvar3 = call <2 x i32> @llvm.aarch64.neon.raddhn.v2i32(<2 x i64> %tmpvar1, <2 x i64> %tmpvar2)
+  ret <2 x i32> %tmpvar3
+}
+
+define <16 x i8> @raddhn2_16b(<8 x i16> %a, <8 x i16> %b) nounwind #0 {
+; CHECK-LABEL: define <16 x i8> @raddhn2_16b(
+; CHECK-SAME: <8 x i16> [[A:%.*]], <8 x i16> [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <8 x i16> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP3]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <8 x i16> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP5:%.*]], label [[TMP6:%.*]], !prof [[PROF1]]
+; CHECK:       5:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       6:
+; CHECK-NEXT:    [[VRADDHN2_I:%.*]] = tail call <8 x i8> @llvm.aarch64.neon.raddhn.v8i8(<8 x i16> [[A]], <8 x i16> [[B]]) #[[ATTR3]]
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <8 x i16> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i128 [[TMP7]], 0
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <8 x i16> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP3:%.*]] = icmp ne i128 [[TMP8]], 0
+; CHECK-NEXT:    [[_MSOR4:%.*]] = or i1 [[_MSCMP2]], [[_MSCMP3]]
+; CHECK-NEXT:    br i1 [[_MSOR4]], label [[TMP9:%.*]], label [[TMP10:%.*]], !prof [[PROF1]]
+; CHECK:       9:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       10:
+; CHECK-NEXT:    [[VRADDHN_HIGH2_I:%.*]] = tail call <8 x i8> @llvm.aarch64.neon.raddhn.v8i8(<8 x i16> [[A]], <8 x i16> [[B]]) #[[ATTR3]]
+; CHECK-NEXT:    [[RES:%.*]] = shufflevector <8 x i8> [[VRADDHN2_I]], <8 x i8> [[VRADDHN_HIGH2_I]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; CHECK-NEXT:    store <16 x i8> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i8> [[RES]]
+;
+  %vraddhn2.i = tail call <8 x i8> @llvm.aarch64.neon.raddhn.v8i8(<8 x i16> %a, <8 x i16> %b) nounwind
+  %vraddhn_high2.i = tail call <8 x i8> @llvm.aarch64.neon.raddhn.v8i8(<8 x i16> %a, <8 x i16> %b) nounwind
+  %res = shufflevector <8 x i8> %vraddhn2.i, <8 x i8> %vraddhn_high2.i, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+  ret <16 x i8> %res
+}
+
+define <8 x i16> @raddhn2_8h(<4 x i32> %a, <4 x i32> %b) nounwind #0 {
+; CHECK-LABEL: define <8 x i16> @raddhn2_8h(
+; CHECK-SAME: <4 x i32> [[A:%.*]], <4 x i32> [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <4 x i32> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP3]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <4 x i32> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP5:%.*]], label [[TMP6:%.*]], !prof [[PROF1]]
+; CHECK:       5:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       6:
+; CHECK-NEXT:    [[VRADDHN2_I:%.*]] = tail call <4 x i16> @llvm.aarch64.neon.raddhn.v4i16(<4 x i32> [[A]], <4 x i32> [[B]]) #[[ATTR3]]
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <4 x i32> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i128 [[TMP7]], 0
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <4 x i32> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP3:%.*]] = icmp ne i128 [[TMP8]], 0
+; CHECK-NEXT:    [[_MSOR4:%.*]] = or i1 [[_MSCMP2]], [[_MSCMP3]]
+; CHECK-NEXT:    br i1 [[_MSOR4]], label [[TMP9:%.*]], label [[TMP10:%.*]], !prof [[PROF1]]
+; CHECK:       9:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       10:
+; CHECK-NEXT:    [[VRADDHN_HIGH3_I:%.*]] = tail call <4 x i16> @llvm.aarch64.neon.raddhn.v4i16(<4 x i32> [[A]], <4 x i32> [[B]]) #[[ATTR3]]
+; CHECK-NEXT:    [[RES:%.*]] = shufflevector <4 x i16> [[VRADDHN2_I]], <4 x i16> [[VRADDHN_HIGH3_I]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:    store <8 x i16> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i16> [[RES]]
+;
+  %vraddhn2.i = tail call <4 x i16> @llvm.aarch64.neon.raddhn.v4i16(<4 x i32> %a, <4 x i32> %b) nounwind
+  %vraddhn_high3.i = tail call <4 x i16> @llvm.aarch64.neon.raddhn.v4i16(<4 x i32> %a, <4 x i32> %b) nounwind
+  %res = shufflevector <4 x i16> %vraddhn2.i, <4 x i16> %vraddhn_high3.i, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+  ret <8 x i16> %res
+}
+
+define <4 x i32> @raddhn2_4s(<2 x i64> %a, <2 x i64> %b) nounwind #0 {
+; CHECK-LABEL: define <4 x i32> @raddhn2_4s(
+; CHECK-SAME: <2 x i64> [[A:%.*]], <2 x i64> [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <2 x i64> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP3]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <2 x i64> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP5:%.*]], label [[TMP6:%.*]], !prof [[PROF1]]
+; CHECK:       5:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       6:
+; CHECK-NEXT:    [[VRADDHN2_I:%.*]] = tail call <2 x i32> @llvm.aarch64.neon.raddhn.v2i32(<2 x i64> [[A]], <2 x i64> [[B]]) #[[ATTR3]]
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <2 x i64> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i128 [[TMP7]], 0
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <2 x i64> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP3:%.*]] = icmp ne i128 [[TMP8]], 0
+; CHECK-NEXT:    [[_MSOR4:%.*]] = or i1 [[_MSCMP2]], [[_MSCMP3]]
+; CHECK-NEXT:    br i1 [[_MSOR4]], label [[TMP9:%.*]], label [[TMP10:%.*]], !prof [[PROF1]]
+; CHECK:       9:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       10:
+; CHECK-NEXT:    [[VRADDHN_HIGH3_I:%.*]] = tail call <2 x i32> @llvm.aarch64.neon.raddhn.v2i32(<2 x i64> [[A]], <2 x i64> [[B]]) #[[ATTR3]]
+; CHECK-NEXT:    [[RES:%.*]] = shufflevector <2 x i32> [[VRADDHN2_I]], <2 x i32> [[VRADDHN_HIGH3_I]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    store <4 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i32> [[RES]]
+;
+  %vraddhn2.i = tail call <2 x i32> @llvm.aarch64.neon.raddhn.v2i32(<2 x i64> %a, <2 x i64> %b) nounwind
+  %vraddhn_high3.i = tail call <2 x i32> @llvm.aarch64.neon.raddhn.v2i32(<2 x i64> %a, <2 x i64> %b) nounwind
+  %res = shufflevector <2 x i32> %vraddhn2.i, <2 x i32> %vraddhn_high3.i, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  ret <4 x i32> %res
+}
+
+declare <2 x i32> @llvm.aarch64.neon.raddhn.v2i32(<2 x i64>, <2 x i64>) nounwind readnone
+declare <4 x i16> @llvm.aarch64.neon.raddhn.v4i16(<4 x i32>, <4 x i32>) nounwind readnone
+declare <8 x i8> @llvm.aarch64.neon.raddhn.v8i8(<8 x i16>, <8 x i16>) nounwind readnone
+
+define <8 x i16> @saddl8h(ptr %A, ptr %B) nounwind #0 {
+; CHECK-LABEL: define <8 x i16> @saddl8h(
+; CHECK-SAME: ptr [[A:%.*]], ptr [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP3:%.*]], label [[TMP4:%.*]], !prof [[PROF1]]
+; CHECK:       3:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       4:
+; CHECK-NEXT:    [[TMPVAR1:%.*]] = load <8 x i8>, ptr [[A]], align 8
+; CHECK-NEXT:    [[TMP5:%.*]] = ptrtoint ptr [[A]] to i64
+; CHECK-NEXT:    [[TMP6:%.*]] = xor i64 [[TMP5]], 193514046488576
+; CHECK-NEXT:    [[TMP7:%.*]] = inttoptr i64 [[TMP6]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <8 x i8>, ptr [[TMP7]], align 8
+; CHECK-NEXT:    [[_MSCMP4:%.*]] = icmp ne i64 [[TMP2]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP4]], label [[TMP8:%.*]], label [[TMP9:%.*]], !prof [[PROF1]]
+; CHECK:       8:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       9:
+; CHECK-NEXT:    [[TMPVAR2:%.*]] = load <8 x i8>, ptr [[B]], align 8
+; CHECK-NEXT:    [[TMP10:%.*]] = ptrtoint ptr [[B]] to i64
+; CHECK-NEXT:    [[TMP11:%.*]] = xor i64 [[TMP10]], 193514046488576
+; CHECK-NEXT:    [[TMP12:%.*]] = inttoptr i64 [[TMP11]] to ptr
+; CHECK-NEXT:    [[_MSLD1:%.*]] = load <8 x i8>, ptr [[TMP12]], align 8
+; CHECK-NEXT:    [[_MSPROP:%.*]] = sext <8 x i8> [[_MSLD]] to <8 x i16>
+; CHECK-NEXT:    [[TMPVAR3:%.*]] = sext <8 x i8> [[TMPVAR1]] to <8 x i16>
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = sext <8 x i8> [[_MSLD1]] to <8 x i16>
+; CHECK-NEXT:    [[TMPVAR4:%.*]] = sext <8 x i8> [[TMPVAR2]] to <8 x i16>
+; CHECK-NEXT:    [[_MSPROP3:%.*]] = or <8 x i16> [[_MSPROP]], [[_MSPROP2]]
+; CHECK-NEXT:    [[TMPVAR5:%.*]] = add <8 x i16> [[TMPVAR3]], [[TMPVAR4]]
+; CHECK-NEXT:    store <8 x i16> [[_MSPROP3]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i16> [[TMPVAR5]]
+;
+  %tmpvar1 = load <8 x i8>, ptr %A
+  %tmpvar2 = load <8 x i8>, ptr %B
+  %tmpvar3 = sext <8 x i8> %tmpvar1 to <8 x i16>
+  %tmpvar4 = sext <8 x i8> %tmpvar2 to <8 x i16>
+  %tmpvar5 = add <8 x i16> %tmpvar3, %tmpvar4
+  ret <8 x i16> %tmpvar5
+}
+
+define <4 x i32> @saddl4s(ptr %A, ptr %B) nounwind #0 {
+; CHECK-LABEL: define <4 x i32> @saddl4s(
+; CHECK-SAME: ptr [[A:%.*]], ptr [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP3:%.*]], label [[TMP4:%.*]], !prof [[PROF1]]
+; CHECK:       3:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       4:
+; CHECK-NEXT:    [[TMPVAR1:%.*]] = load <4 x i16>, ptr [[A]], align 8
+; CHECK-NEXT:    [[TMP5:%.*]] = ptrtoint ptr [[A]] to i64
+; CHECK-NEXT:    [[TMP6:%.*]] = xor i64 [[TMP5]], 193514046488576
+; CHECK-NEXT:    [[TMP7:%.*]] = inttoptr i64 [[TMP6]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <4 x i16>, ptr [[TMP7]], align 8
+; CHECK-NEXT:    [[_MSCMP4:%.*]] = icmp ne i64 [[TMP2]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP4]], label [[TMP8:%.*]], label [[TMP9:%.*]], !prof [[PROF1]]
+; CHECK:       8:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       9:
+; CHECK-NEXT:    [[TMPVAR2:%.*]] = load <4 x i16>, ptr [[B]], align 8
+; CHECK-NEXT:    [[TMP10:%.*]] = ptrtoint ptr [[B]] to i64
+; CHECK-NEXT:    [[TMP11:%.*]] = xor i64 [[TMP10]], 193514046488576
+; CHECK-NEXT:    [[TMP12:%.*]] = inttoptr i64 [[TMP11]] to ptr
+; CHECK-NEXT:    [[_MSLD1:%.*]] = load <4 x i16>, ptr [[TMP12]], align 8
+; CHECK-NEXT:    [[_MSPROP:%.*]] = sext <4 x i16> [[_MSLD]] to <4 x i32>
+; CHECK-NEXT:    [[TMPVAR3:%.*]] = sext <4 x i16> [[TMPVAR1]] to <4 x i32>
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = sext <4 x i16> [[_MSLD1]] to <4 x i32>
+; CHECK-NEXT:    [[TMPVAR4:%.*]] = sext <4 x i16> [[TMPVAR2]] to <4 x i32>
+; CHECK-NEXT:    [[_MSPROP3:%.*]] = or <4 x i32> [[_MSPROP]], [[_MSPROP2]]
+; CHECK-NEXT:    [[TMPVAR5:%.*]] = add <4 x i32> [[TMPVAR3]], [[TMPVAR4]]
+; CHECK-NEXT:    store <4 x i32> [[_MSPROP3]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i32> [[TMPVAR5]]
+;
+  %tmpvar1 = load <4 x i16>, ptr %A
+  %tmpvar2 = load <4 x i16>, ptr %B
+  %tmpvar3 = sext <4 x i16> %tmpvar1 to <4 x i32>
+  %tmpvar4 = sext <4 x i16> %tmpvar2 to <4 x i32>
+  %tmpvar5 = add <4 x i32> %tmpvar3, %tmpvar4
+  ret <4 x i32> %tmpvar5
+}
+
+define <2 x i64> @saddl2d(ptr %A, ptr %B) nounwind #0 {
+; CHECK-LABEL: define <2 x i64> @saddl2d(
+; CHECK-SAME: ptr [[A:%.*]], ptr [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP3:%.*]], label [[TMP4:%.*]], !prof [[PROF1]]
+; CHECK:       3:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       4:
+; CHECK-NEXT:    [[TMPVAR1:%.*]] = load <2 x i32>, ptr [[A]], align 8
+; CHECK-NEXT:    [[TMP5:%.*]] = ptrtoint ptr [[A]] to i64
+; CHECK-NEXT:    [[TMP6:%.*]] = xor i64 [[TMP5]], 193514046488576
+; CHECK-NEXT:    [[TMP7:%.*]] = inttoptr i64 [[TMP6]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <2 x i32>, ptr [[TMP7]], align 8
+; CHECK-NEXT:    [[_MSCMP4:%.*]] = icmp ne i64 [[TMP2]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP4]], label [[TMP8:%.*]], label [[TMP9:%.*]], !prof [[PROF1]]
+; CHECK:       8:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       9:
+; CHECK-NEXT:    [[TMPVAR2:%.*]] = load <2 x i32>, ptr [[B]], align 8
+; CHECK-NEXT:    [[TMP10:%.*]] = ptrtoint ptr [[B]] to i64
+; CHECK-NEXT:    [[TMP11:%.*]] = xor i64 [[TMP10]], 193514046488576
+; CHECK-NEXT:    [[TMP12:%.*]] = inttoptr i64 [[TMP11]] to ptr
+; CHECK-NEXT:    [[_MSLD1:%.*]] = load <2 x i32>, ptr [[TMP12]], align 8
+; CHECK-NEXT:    [[_MSPROP:%.*]] = sext <2 x i32> [[_MSLD]] to <2 x i64>
+; CHECK-NEXT:    [[TMPVAR3:%.*]] = sext <2 x i32> [[TMPVAR1]] to <2 x i64>
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = sext <2 x i32> [[_MSLD1]] to <2 x i64>
+; CHECK-NEXT:    [[TMPVAR4:%.*]] = sext <2 x i32> [[TMPVAR2]] to <2 x i64>
+; CHECK-NEXT:    [[_MSPROP3:%.*]] = or <2 x i64> [[_MSPROP]], [[_MSPROP2]]
+; CHECK-NEXT:    [[TMPVAR5:%.*]] = add <2 x i64> [[TMPVAR3]], [[TMPVAR4]]
+; CHECK-NEXT:    store <2 x i64> [[_MSPROP3]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x i64> [[TMPVAR5]]
+;
+  %tmpvar1 = load <2 x i32>, ptr %A
+  %tmpvar2 = load <2 x i32>, ptr %B
+  %tmpvar3 = sext <2 x i32> %tmpvar1 to <2 x i64>
+  %tmpvar4 = sext <2 x i32> %tmpvar2 to <2 x i64>
+  %tmpvar5 = add <2 x i64> %tmpvar3, %tmpvar4
+  ret <2 x i64> %tmpvar5
+}
+
+define <8 x i16> @saddl2_8h(<16 x i8> %a, <16 x i8> %b, <2 x i64> %param1, <2 x i64> %param2) nounwind  #0 {
+; CHECK-LABEL: define <8 x i16> @saddl2_8h(
+; CHECK-SAME: <16 x i8> [[A:%.*]], <16 x i8> [[B:%.*]], <2 x i64> [[PARAM1:%.*]], <2 x i64> [[PARAM2:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i8>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP7:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP8:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 48) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <16 x i8> [[TMP1]] to <2 x i64>
+; CHECK-NEXT:    [[TMP:%.*]] = bitcast <16 x i8> [[A]] to <2 x i64>
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <2 x i64> [[TMP3]], <2 x i64> [[TMP7]], <1 x i32> <i32 1>
+; CHECK-NEXT:    [[SHUFFLE_I_I_I:%.*]] = shufflevector <2 x i64> [[TMP]], <2 x i64> [[PARAM1]], <1 x i32> <i32 1>
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <1 x i64> [[_MSPROP]] to <8 x i8>
+; CHECK-NEXT:    [[TMPVAR1:%.*]] = bitcast <1 x i64> [[SHUFFLE_I_I_I]] to <8 x i8>
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = sext <8 x i8> [[TMP4]] to <8 x i16>
+; CHECK-NEXT:    [[VMOVL_I_I_I:%.*]] = sext <8 x i8> [[TMPVAR1]] to <8 x i16>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <16 x i8> [[TMP2]] to <2 x i64>
+; CHECK-NEXT:    [[TMPVAR2:%.*]] = bitcast <16 x i8> [[B]] to <2 x i64>
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = shufflevector <2 x i64> [[TMP5]], <2 x i64> [[TMP8]], <1 x i32> <i32 1>
+; CHECK-NEXT:    [[SHUFFLE_I_I4_I:%.*]] = shufflevector <2 x i64> [[TMPVAR2]], <2 x i64> [[PARAM2]], <1 x i32> <i32 1>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <1 x i64> [[_MSPROP2]] to <8 x i8>
+; CHECK-NEXT:    [[TMPVAR3:%.*]] = bitcast <1 x i64> [[SHUFFLE_I_I4_I]] to <8 x i8>
+; CHECK-NEXT:    [[_MSPROP3:%.*]] = sext <8 x i8> [[TMP6]] to <8 x i16>
+; CHECK-NEXT:    [[VMOVL_I_I5_I:%.*]] = sext <8 x i8> [[TMPVAR3]] to <8 x i16>
+; CHECK-NEXT:    [[_MSPROP4:%.*]] = or <8 x i16> [[_MSPROP1]], [[_MSPROP3]]
+; CHECK-NEXT:    [[ADD_I:%.*]] = add <8 x i16> [[VMOVL_I_I_I]], [[VMOVL_I_I5_I]]
+; CHECK-NEXT:    store <8 x i16> [[_MSPROP4]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i16> [[ADD_I]]
+;
+  %tmpvar = bitcast <16 x i8> %a to <2 x i64>
+  %shuffle.i.i.i = shufflevector <2 x i64> %tmpvar, <2 x i64> %param1, <1 x i32> <i32 1>
+  %tmpvar1 = bitcast <1 x i64> %shuffle.i.i.i to <8 x i8>
+  %vmovl.i.i.i = sext <8 x i8> %tmpvar1 to <8 x i16>
+  %tmpvar2 = bitcast <16 x i8> %b to <2 x i64>
+  %shuffle.i.i4.i = shufflevector <2 x i64> %tmpvar2, <2 x i64> %param2, <1 x i32> <i32 1>
+  %tmpvar3 = bitcast <1 x i64> %shuffle.i.i4.i to <8 x i8>
+  %vmovl.i.i5.i = sext <8 x i8> %tmpvar3 to <8 x i16>
+  %add.i = add <8 x i16> %vmovl.i.i.i, %vmovl.i.i5.i
+  ret <8 x i16> %add.i
+}
+
+define <4 x i32> @saddl2_4s(<8 x i16> %a, <8 x i16> %b, <2 x i64> %param1, <2 x i64> %param2) nounwind  #0 {
+; CHECK-LABEL: define <4 x i32> @saddl2_4s(
+; CHECK-SAME: <8 x i16> [[A:%.*]], <8 x i16> [[B:%.*]], <2 x i64> [[PARAM1:%.*]], <2 x i64> [[PARAM2:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP7:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP8:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 48) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <8 x i16> [[TMP1]] to <2 x i64>
+; CHECK-NEXT:    [[TMP:%.*]] = bitcast <8 x i16> [[A]] to <2 x i64>
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <2 x i64> [[TMP3]], <2 x i64> [[TMP7]], <1 x i32> <i32 1>
+; CHECK-NEXT:    [[SHUFFLE_I_I_I:%.*]] = shufflevector <2 x i64> [[TMP]], <2 x i64> [[PARAM1]], <1 x i32> <i32 1>
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <1 x i64> [[_MSPROP]] to <4 x i16>
+; CHECK-NEXT:    [[TMPVAR1:%.*]] = bitcast <1 x i64> [[SHUFFLE_I_I_I]] to <4 x i16>
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = sext <4 x i16> [[TMP4]] to <4 x i32>
+; CHECK-NEXT:    [[VMOVL_I_I_I:%.*]] = sext <4 x i16> [[TMPVAR1]] to <4 x i32>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <8 x i16> [[TMP2]] to <2 x i64>
+; CHECK-NEXT:    [[TMPVAR2:%.*]] = bitcast <8 x i16> [[B]] to <2 x i64>
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = shufflevector <2 x i64> [[TMP5]], <2 x i64> [[TMP8]], <1 x i32> <i32 1>
+; CHECK-NEXT:    [[SHUFFLE_I_I4_I:%.*]] = shufflevector <2 x i64> [[TMPVAR2]], <2 x i64> [[PARAM2]], <1 x i32> <i32 1>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <1 x i64> [[_MSPROP2]] to <4 x i16>
+; CHECK-NEXT:    [[TMPVAR3:%.*]] = bitcast <1 x i64> [[SHUFFLE_I_I4_I]] to <4 x i16>
+; CHECK-NEXT:    [[_MSPROP3:%.*]] = sext <4 x i16> [[TMP6]] to <4 x i32>
+; CHECK-NEXT:    [[VMOVL_I_I5_I:%.*]] = sext <4 x i16> [[TMPVAR3]] to <4 x i32>
+; CHECK-NEXT:    [[_MSPROP4:%.*]] = or <4 x i32> [[_MSPROP1]], [[_MSPROP3]]
+; CHECK-NEXT:    [[ADD_I:%.*]] = add <4 x i32> [[VMOVL_I_I_I]], [[VMOVL_I_I5_I]]
+; CHECK-NEXT:    store <4 x i32> [[_MSPROP4]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i32> [[ADD_I]]
+;
+  %tmpvar = bitcast <8 x i16> %a to <2 x i64>
+  %shuffle.i.i.i = shufflevector <2 x i64> %tmpvar, <2 x i64> %param1, <1 x i32> <i32 1>
+  %tmpvar1 = bitcast <1 x i64> %shuffle.i.i.i to <4 x i16>
+  %vmovl.i.i.i = sext <4 x i16> %tmpvar1 to <4 x i32>
+  %tmpvar2 = bitcast <8 x i16> %b to <2 x i64>
+  %shuffle.i.i4.i = shufflevector <2 x i64> %tmpvar2, <2 x i64> %param2, <1 x i32> <i32 1>
+  %tmpvar3 = bitcast <1 x i64> %shuffle.i.i4.i to <4 x i16>
+  %vmovl.i.i5.i = sext <4 x i16> %tmpvar3 to <4 x i32>
+  %add.i = add <4 x i32> %vmovl.i.i.i, %vmovl.i.i5.i
+  ret <4 x i32> %add.i
+}
+
+define <2 x i64> @saddl2_2d(<4 x i32> %a, <4 x i32> %b, <2 x i64> %param1, <2 x i64> %param2) nounwind  #0 {
+; CHECK-LABEL: define <2 x i64> @saddl2_2d(
+; CHECK-SAME: <4 x i32> [[A:%.*]], <4 x i32> [[B:%.*]], <2 x i64> [[PARAM1:%.*]], <2 x i64> [[PARAM2:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP7:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP8:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 48) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <4 x i32> [[TMP1]] to <2 x i64>
+; CHECK-NEXT:    [[TMP:%.*]] = bitcast <4 x i32> [[A]] to <2 x i64>
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <2 x i64> [[TMP3]], <2 x i64> [[TMP7]], <1 x i32> <i32 1>
+; CHECK-NEXT:    [[SHUFFLE_I_I_I:%.*]] = shufflevector <2 x i64> [[TMP]], <2 x i64> [[PARAM1]], <1 x i32> <i32 1>
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <1 x i64> [[_MSPROP]] to <2 x i32>
+; CHECK-NEXT:    [[TMPVAR1:%.*]] = bitcast <1 x i64> [[SHUFFLE_I_I_I]] to <2 x i32>
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = sext <2 x i32> [[TMP4]] to <2 x i64>
+; CHECK-NEXT:    [[VMOVL_I_I_I:%.*]] = sext <2 x i32> [[TMPVAR1]] to <2 x i64>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <4 x i32> [[TMP2]] to <2 x i64>
+; CHECK-NEXT:    [[TMPVAR2:%.*]] = bitcast <4 x i32> [[B]] to <2 x i64>
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = shufflevector <2 x i64> [[TMP5]], <2 x i64> [[TMP8]], <1 x i32> <i32 1>
+; CHECK-NEXT:    [[SHUFFLE_I_I4_I:%.*]] = shufflevector <2 x i64> [[TMPVAR2]], <2 x i64> [[PARAM2]], <1 x i32> <i32 1>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <1 x i64> [[_MSPROP2]] to <2 x i32>
+; CHECK-NEXT:    [[TMPVAR3:%.*]] = bitcast <1 x i64> [[SHUFFLE_I_I4_I]] to <2 x i32>
+; CHECK-NEXT:    [[_MSPROP3:%.*]] = sext <2 x i32> [[TMP6]] to <2 x i64>
+; CHECK-NEXT:    [[VMOVL_I_I5_I:%.*]] = sext <2 x i32> [[TMPVAR3]] to <2 x i64>
+; CHECK-NEXT:    [[_MSPROP4:%.*]] = or <2 x i64> [[_MSPROP1]], [[_MSPROP3]]
+; CHECK-NEXT:    [[ADD_I:%.*]] = add <2 x i64> [[VMOVL_I_I_I]], [[VMOVL_I_I5_I]]
+; CHECK-NEXT:    store <2 x i64> [[_MSPROP4]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x i64> [[ADD_I]]
+;
+  %tmpvar = bitcast <4 x i32> %a to <2 x i64>
+  %shuffle.i.i.i = shufflevector <2 x i64> %tmpvar, <2 x i64> %param1, <1 x i32> <i32 1>
+  %tmpvar1 = bitcast <1 x i64> %shuffle.i.i.i to <2 x i32>
+  %vmovl.i.i.i = sext <2 x i32> %tmpvar1 to <2 x i64>
+  %tmpvar2 = bitcast <4 x i32> %b to <2 x i64>
+  %shuffle.i.i4.i = shufflevector <2 x i64> %tmpvar2, <2 x i64> %param2, <1 x i32> <i32 1>
+  %tmpvar3 = bitcast <1 x i64> %shuffle.i.i4.i to <2 x i32>
+  %vmovl.i.i5.i = sext <2 x i32> %tmpvar3 to <2 x i64>
+  %add.i = add <2 x i64> %vmovl.i.i.i, %vmovl.i.i5.i
+  ret <2 x i64> %add.i
+}
+
+define <8 x i16> @uaddl8h(ptr %A, ptr %B) nounwind #0 {
+; CHECK-LABEL: define <8 x i16> @uaddl8h(
+; CHECK-SAME: ptr [[A:%.*]], ptr [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP3:%.*]], label [[TMP4:%.*]], !prof [[PROF1]]
+; CHECK:       3:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       4:
+; CHECK-NEXT:    [[TMPVAR1:%.*]] = load <8 x i8>, ptr [[A]], align 8
+; CHECK-NEXT:    [[TMP5:%.*]] = ptrtoint ptr [[A]] to i64
+; CHECK-NEXT:    [[TMP6:%.*]] = xor i64 [[TMP5]], 193514046488576
+; CHECK-NEXT:    [[TMP7:%.*]] = inttoptr i64 [[TMP6]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <8 x i8>, ptr [[TMP7]], align 8
+; CHECK-NEXT:    [[_MSCMP4:%.*]] = icmp ne i64 [[TMP2]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP4]], label [[TMP8:%.*]], label [[TMP9:%.*]], !prof [[PROF1]]
+; CHECK:       8:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       9:
+; CHECK-NEXT:    [[TMPVAR2:%.*]] = load <8 x i8>, ptr [[B]], align 8
+; CHECK-NEXT:    [[TMP10:%.*]] = ptrtoint ptr [[B]] to i64
+; CHECK-NEXT:    [[TMP11:%.*]] = xor i64 [[TMP10]], 193514046488576
+; CHECK-NEXT:    [[TMP12:%.*]] = inttoptr i64 [[TMP11]] to ptr
+; CHECK-NEXT:    [[_MSLD1:%.*]] = load <8 x i8>, ptr [[TMP12]], align 8
+; CHECK-NEXT:    [[_MSPROP:%.*]] = zext <8 x i8> [[_MSLD]] to <8 x i16>
+; CHECK-NEXT:    [[TMPVAR3:%.*]] = zext <8 x i8> [[TMPVAR1]] to <8 x i16>
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = zext <8 x i8> [[_MSLD1]] to <8 x i16>
+; CHECK-NEXT:    [[TMPVAR4:%.*]] = zext <8 x i8> [[TMPVAR2]] to <8 x i16>
+; CHECK-NEXT:    [[_MSPROP3:%.*]] = or <8 x i16> [[_MSPROP]], [[_MSPROP2]]
+; CHECK-NEXT:    [[TMPVAR5:%.*]] = add <8 x i16> [[TMPVAR3]], [[TMPVAR4]]
+; CHECK-NEXT:    store <8 x i16> [[_MSPROP3]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i16> [[TMPVAR5]]
+;
+  %tmpvar1 = load <8 x i8>, ptr %A
+  %tmpvar2 = load <8 x i8>, ptr %B
+  %tmpvar3 = zext <8 x i8> %tmpvar1 to <8 x i16>
+  %tmpvar4 = zext <8 x i8> %tmpvar2 to <8 x i16>
+  %tmpvar5 = add <8 x i16> %tmpvar3, %tmpvar4
+  ret <8 x i16> %tmpvar5
+}
+
+define <4 x i32> @uaddl4s(ptr %A, ptr %B) nounwind #0 {
+; CHECK-LABEL: define <4 x i32> @uaddl4s(
+; CHECK-SAME: ptr [[A:%.*]], ptr [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP3:%.*]], label [[TMP4:%.*]], !prof [[PROF1]]
+; CHECK:       3:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       4:
+; CHECK-NEXT:    [[TMPVAR1:%.*]] = load <4 x i16>, ptr [[A]], align 8
+; CHECK-NEXT:    [[TMP5:%.*]] = ptrtoint ptr [[A]] to i64
+; CHECK-NEXT:    [[TMP6:%.*]] = xor i64 [[TMP5]], 193514046488576
+; CHECK-NEXT:    [[TMP7:%.*]] = inttoptr i64 [[TMP6]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <4 x i16>, ptr [[TMP7]], align 8
+; CHECK-NEXT:    [[_MSCMP4:%.*]] = icmp ne i64 [[TMP2]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP4]], label [[TMP8:%.*]], label [[TMP9:%.*]], !prof [[PROF1]]
+; CHECK:       8:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       9:
+; CHECK-NEXT:    [[TMPVAR2:%.*]] = load <4 x i16>, ptr [[B]], align 8
+; CHECK-NEXT:    [[TMP10:%.*]] = ptrtoint ptr [[B]] to i64
+; CHECK-NEXT:    [[TMP11:%.*]] = xor i64 [[TMP10]], 193514046488576
+; CHECK-NEXT:    [[TMP12:%.*]] = inttoptr i64 [[TMP11]] to ptr
+; CHECK-NEXT:    [[_MSLD1:%.*]] = load <4 x i16>, ptr [[TMP12]], align 8
+; CHECK-NEXT:    [[_MSPROP:%.*]] = zext <4 x i16> [[_MSLD]] to <4 x i32>
+; CHECK-NEXT:    [[TMPVAR3:%.*]] = zext <4 x i16> [[TMPVAR1]] to <4 x i32>
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = zext <4 x i16> [[_MSLD1]] to <4 x i32>
+; CHECK-NEXT:    [[TMPVAR4:%.*]] = zext <4 x i16> [[TMPVAR2]] to <4 x i32>
+; CHECK-NEXT:    [[_MSPROP3:%.*]] = or <4 x i32> [[_MSPROP]], [[_MSPROP2]]
+; CHECK-NEXT:    [[TMPVAR5:%.*]] = add <4 x i32> [[TMPVAR3]], [[TMPVAR4]]
+; CHECK-NEXT:    store <4 x i32> [[_MSPROP3]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i32> [[TMPVAR5]]
+;
+  %tmpvar1 = load <4 x i16>, ptr %A
+  %tmpvar2 = load <4 x i16>, ptr %B
+  %tmpvar3 = zext <4 x i16> %tmpvar1 to <4 x i32>
+  %tmpvar4 = zext <4 x i16> %tmpvar2 to <4 x i32>
+  %tmpvar5 = add <4 x i32> %tmpvar3, %tmpvar4
+  ret <4 x i32> %tmpvar5
+}
+
+define <2 x i64> @uaddl2d(ptr %A, ptr %B) nounwind #0 {
+; CHECK-LABEL: define <2 x i64> @uaddl2d(
+; CHECK-SAME: ptr [[A:%.*]], ptr [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP3:%.*]], label [[TMP4:%.*]], !prof [[PROF1]]
+; CHECK:       3:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       4:
+; CHECK-NEXT:    [[TMPVAR1:%.*]] = load <2 x i32>, ptr [[A]], align 8
+; CHECK-NEXT:    [[TMP5:%.*]] = ptrtoint ptr [[A]] to i64
+; CHECK-NEXT:    [[TMP6:%.*]] = xor i64 [[TMP5]], 193514046488576
+; CHECK-NEXT:    [[TMP7:%.*]] = inttoptr i64 [[TMP6]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <2 x i32>, ptr [[TMP7]], align 8
+; CHECK-NEXT:    [[_MSCMP4:%.*]] = icmp ne i64 [[TMP2]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP4]], label [[TMP8:%.*]], label [[TMP9:%.*]], !prof [[PROF1]]
+; CHECK:       8:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       9:
+; CHECK-NEXT:    [[TMPVAR2:%.*]] = load <2 x i32>, ptr [[B]], align 8
+; CHECK-NEXT:    [[TMP10:%.*]] = ptrtoint ptr [[B]] to i64
+; CHECK-NEXT:    [[TMP11:%.*]] = xor i64 [[TMP10]], 193514046488576
+; CHECK-NEXT:    [[TMP12:%.*]] = inttoptr i64 [[TMP11]] to ptr
+; CHECK-NEXT:    [[_MSLD1:%.*]] = load <2 x i32>, ptr [[TMP12]], align 8
+; CHECK-NEXT:    [[_MSPROP:%.*]] = zext <2 x i32> [[_MSLD]] to <2 x i64>
+; CHECK-NEXT:    [[TMPVAR3:%.*]] = zext <2 x i32> [[TMPVAR1]] to <2 x i64>
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = zext <2 x i32> [[_MSLD1]] to <2 x i64>
+; CHECK-NEXT:    [[TMPVAR4:%.*]] = zext <2 x i32> [[TMPVAR2]] to <2 x i64>
+; CHECK-NEXT:    [[_MSPROP3:%.*]] = or <2 x i64> [[_MSPROP]], [[_MSPROP2]]
+; CHECK-NEXT:    [[TMPVAR5:%.*]] = add <2 x i64> [[TMPVAR3]], [[TMPVAR4]]
+; CHECK-NEXT:    store <2 x i64> [[_MSPROP3]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x i64> [[TMPVAR5]]
+;
+  %tmpvar1 = load <2 x i32>, ptr %A
+  %tmpvar2 = load <2 x i32>, ptr %B
+  %tmpvar3 = zext <2 x i32> %tmpvar1 to <2 x i64>
+  %tmpvar4 = zext <2 x i32> %tmpvar2 to <2 x i64>
+  %tmpvar5 = add <2 x i64> %tmpvar3, %tmpvar4
+  ret <2 x i64> %tmpvar5
+}
+
+
+define <8 x i16> @uaddl2_8h(<16 x i8> %a, <16 x i8> %b, <2 x i64> %param1, <2 x i64> %param2) nounwind  #0 {
+; CHECK-LABEL: define <8 x i16> @uaddl2_8h(
+; CHECK-SAME: <16 x i8> [[A:%.*]], <16 x i8> [[B:%.*]], <2 x i64> [[PARAM1:%.*]], <2 x i64> [[PARAM2:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i8>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP7:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP8:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 48) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <16 x i8> [[TMP1]] to <2 x i64>
+; CHECK-NEXT:    [[TMP:%.*]] = bitcast <16 x i8> [[A]] to <2 x i64>
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <2 x i64> [[TMP3]], <2 x i64> [[TMP7]], <1 x i32> <i32 1>
+; CHECK-NEXT:    [[SHUFFLE_I_I_I:%.*]] = shufflevector <2 x i64> [[TMP]], <2 x i64> [[PARAM1]], <1 x i32> <i32 1>
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <1 x i64> [[_MSPROP]] to <8 x i8>
+; CHECK-NEXT:    [[TMPVAR1:%.*]] = bitcast <1 x i64> [[SHUFFLE_I_I_I]] to <8 x i8>
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = zext <8 x i8> [[TMP4]] to <8 x i16>
+; CHECK-NEXT:    [[VMOVL_I_I_I:%.*]] = zext <8 x i8> [[TMPVAR1]] to <8 x i16>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <16 x i8> [[TMP2]] to <2 x i64>
+; CHECK-NEXT:    [[TMPVAR2:%.*]] = bitcast <16 x i8> [[B]] to <2 x i64>
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = shufflevector <2 x i64> [[TMP5]], <2 x i64> [[TMP8]], <1 x i32> <i32 1>
+; CHECK-NEXT:    [[SHUFFLE_I_I4_I:%.*]] = shufflevector <2 x i64> [[TMPVAR2]], <2 x i64> [[PARAM2]], <1 x i32> <i32 1>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <1 x i64> [[_MSPROP2]] to <8 x i8>
+; CHECK-NEXT:    [[TMPVAR3:%.*]] = bitcast <1 x i64> [[SHUFFLE_I_I4_I]] to <8 x i8>
+; CHECK-NEXT:    [[_MSPROP3:%.*]] = zext <8 x i8> [[TMP6]] to <8 x i16>
+; CHECK-NEXT:    [[VMOVL_I_I5_I:%.*]] = zext <8 x i8> [[TMPVAR3]] to <8 x i16>
+; CHECK-NEXT:    [[_MSPROP4:%.*]] = or <8 x i16> [[_MSPROP1]], [[_MSPROP3]]
+; CHECK-NEXT:    [[ADD_I:%.*]] = add <8 x i16> [[VMOVL_I_I_I]], [[VMOVL_I_I5_I]]
+; CHECK-NEXT:    store <8 x i16> [[_MSPROP4]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i16> [[ADD_I]]
+;
+  %tmpvar = bitcast <16 x i8> %a to <2 x i64>
+  %shuffle.i.i.i = shufflevector <2 x i64> %tmpvar, <2 x i64> %param1, <1 x i32> <i32 1>
+  %tmpvar1 = bitcast <1 x i64> %shuffle.i.i.i to <8 x i8>
+  %vmovl.i.i.i = zext <8 x i8> %tmpvar1 to <8 x i16>
+  %tmpvar2 = bitcast <16 x i8> %b to <2 x i64>
+  %shuffle.i.i4.i = shufflevector <2 x i64> %tmpvar2, <2 x i64> %param2, <1 x i32> <i32 1>
+  %tmpvar3 = bitcast <1 x i64> %shuffle.i.i4.i to <8 x i8>
+  %vmovl.i.i5.i = zext <8 x i8> %tmpvar3 to <8 x i16>
+  %add.i = add <8 x i16> %vmovl.i.i.i, %vmovl.i.i5.i
+  ret <8 x i16> %add.i
+}
+
+define <4 x i32> @uaddl2_4s(<8 x i16> %a, <8 x i16> %b, <2 x i64> %param1, <2 x i64> %param2) nounwind  #0 {
+; CHECK-LABEL: define <4 x i32> @uaddl2_4s(
+; CHECK-SAME: <8 x i16> [[A:%.*]], <8 x i16> [[B:%.*]], <2 x i64> [[PARAM1:%.*]], <2 x i64> [[PARAM2:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP7:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP8:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 48) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <8 x i16> [[TMP1]] to <2 x i64>
+; CHECK-NEXT:    [[TMP:%.*]] = bitcast <8 x i16> [[A]] to <2 x i64>
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <2 x i64> [[TMP3]], <2 x i64> [[TMP7]], <1 x i32> <i32 1>
+; CHECK-NEXT:    [[SHUFFLE_I_I_I:%.*]] = shufflevector <2 x i64> [[TMP]], <2 x i64> [[PARAM1]], <1 x i32> <i32 1>
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <1 x i64> [[_MSPROP]] to <4 x i16>
+; CHECK-NEXT:    [[TMPVAR1:%.*]] = bitcast <1 x i64> [[SHUFFLE_I_I_I]] to <4 x i16>
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = zext <4 x i16> [[TMP4]] to <4 x i32>
+; CHECK-NEXT:    [[VMOVL_I_I_I:%.*]] = zext <4 x i16> [[TMPVAR1]] to <4 x i32>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <8 x i16> [[TMP2]] to <2 x i64>
+; CHECK-NEXT:    [[TMPVAR2:%.*]] = bitcast <8 x i16> [[B]] to <2 x i64>
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = shufflevector <2 x i64> [[TMP5]], <2 x i64> [[TMP8]], <1 x i32> <i32 1>
+; CHECK-NEXT:    [[SHUFFLE_I_I4_I:%.*]] = shufflevector <2 x i64> [[TMPVAR2]], <2 x i64> [[PARAM2]], <1 x i32> <i32 1>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <1 x i64> [[_MSPROP2]] to <4 x i16>
+; CHECK-NEXT:    [[TMPVAR3:%.*]] = bitcast <1 x i64> [[SHUFFLE_I_I4_I]] to <4 x i16>
+; CHECK-NEXT:    [[_MSPROP3:%.*]] = zext <4 x i16> [[TMP6]] to <4 x i32>
+; CHECK-NEXT:    [[VMOVL_I_I5_I:%.*]] = zext <4 x i16> [[TMPVAR3]] to <4 x i32>
+; CHECK-NEXT:    [[_MSPROP4:%.*]] = or <4 x i32> [[_MSPROP1]], [[_MSPROP3]]
+; CHECK-NEXT:    [[ADD_I:%.*]] = add <4 x i32> [[VMOVL_I_I_I]], [[VMOVL_I_I5_I]]
+; CHECK-NEXT:    store <4 x i32> [[_MSPROP4]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i32> [[ADD_I]]
+;
+  %tmpvar = bitcast <8 x i16> %a to <2 x i64>
+  %shuffle.i.i.i = shufflevector <2 x i64> %tmpvar, <2 x i64> %param1, <1 x i32> <i32 1>
+  %tmpvar1 = bitcast <1 x i64> %shuffle.i.i.i to <4 x i16>
+  %vmovl.i.i.i = zext <4 x i16> %tmpvar1 to <4 x i32>
+  %tmpvar2 = bitcast <8 x i16> %b to <2 x i64>
+  %shuffle.i.i4.i = shufflevector <2 x i64> %tmpvar2, <2 x i64> %param2, <1 x i32> <i32 1>
+  %tmpvar3 = bitcast <1 x i64> %shuffle.i.i4.i to <4 x i16>
+  %vmovl.i.i5.i = zext <4 x i16> %tmpvar3 to <4 x i32>
+  %add.i = add <4 x i32> %vmovl.i.i.i, %vmovl.i.i5.i
+  ret <4 x i32> %add.i
+}
+
+define <2 x i64> @uaddl2_2d(<4 x i32> %a, <4 x i32> %b, <2 x i64> %param1, <2 x i64> %param2) nounwind  #0 {
+; CHECK-LABEL: define <2 x i64> @uaddl2_2d(
+; CHECK-SAME: <4 x i32> [[A:%.*]], <4 x i32> [[B:%.*]], <2 x i64> [[PARAM1:%.*]], <2 x i64> [[PARAM2:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP7:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP8:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 48) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <4 x i32> [[TMP1]] to <2 x i64>
+; CHECK-NEXT:    [[TMP:%.*]] = bitcast <4 x i32> [[A]] to <2 x i64>
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <2 x i64> [[TMP3]], <2 x i64> [[TMP7]], <1 x i32> <i32 1>
+; CHECK-NEXT:    [[SHUFFLE_I_I_I:%.*]] = shufflevector <2 x i64> [[TMP]], <2 x i64> [[PARAM1]], <1 x i32> <i32 1>
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <1 x i64> [[_MSPROP]] to <2 x i32>
+; CHECK-NEXT:    [[TMPVAR1:%.*]] = bitcast <1 x i64> [[SHUFFLE_I_I_I]] to <2 x i32>
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = zext <2 x i32> [[TMP4]] to <2 x i64>
+; CHECK-NEXT:    [[VMOVL_I_I_I:%.*]] = zext <2 x i32> [[TMPVAR1]] to <2 x i64>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <4 x i32> [[TMP2]] to <2 x i64>
+; CHECK-NEXT:    [[TMPVAR2:%.*]] = bitcast <4 x i32> [[B]] to <2 x i64>
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = shufflevector <2 x i64> [[TMP5]], <2 x i64> [[TMP8]], <1 x i32> <i32 1>
+; CHECK-NEXT:    [[SHUFFLE_I_I4_I:%.*]] = shufflevector <2 x i64> [[TMPVAR2]], <2 x i64> [[PARAM2]], <1 x i32> <i32 1>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <1 x i64> [[_MSPROP2]] to <2 x i32>
+; CHECK-NEXT:    [[TMPVAR3:%.*]] = bitcast <1 x i64> [[SHUFFLE_I_I4_I]] to <2 x i32>
+; CHECK-NEXT:    [[_MSPROP3:%.*]] = zext <2 x i32> [[TMP6]] to <2 x i64>
+; CHECK-NEXT:    [[VMOVL_I_I5_I:%.*]] = zext <2 x i32> [[TMPVAR3]] to <2 x i64>
+; CHECK-NEXT:    [[_MSPROP4:%.*]] = or <2 x i64> [[_MSPROP1]], [[_MSPROP3]]
+; CHECK-NEXT:    [[ADD_I:%.*]] = add <2 x i64> [[VMOVL_I_I_I]], [[VMOVL_I_I5_I]]
+; CHECK-NEXT:    store <2 x i64> [[_MSPROP4]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x i64> [[ADD_I]]
+;
+  %tmpvar = bitcast <4 x i32> %a to <2 x i64>
+  %shuffle.i.i.i = shufflevector <2 x i64> %tmpvar, <2 x i64> %param1, <1 x i32> <i32 1>
+  %tmpvar1 = bitcast <1 x i64> %shuffle.i.i.i to <2 x i32>
+  %vmovl.i.i.i = zext <2 x i32> %tmpvar1 to <2 x i64>
+  %tmpvar2 = bitcast <4 x i32> %b to <2 x i64>
+  %shuffle.i.i4.i = shufflevector <2 x i64> %tmpvar2, <2 x i64> %param2, <1 x i32> <i32 1>
+  %tmpvar3 = bitcast <1 x i64> %shuffle.i.i4.i to <2 x i32>
+  %vmovl.i.i5.i = zext <2 x i32> %tmpvar3 to <2 x i64>
+  %add.i = add <2 x i64> %vmovl.i.i.i, %vmovl.i.i5.i
+  ret <2 x i64> %add.i
+}
+
+define <8 x i16> @uaddw8h(ptr %A, ptr %B) nounwind #0 {
+; CHECK-LABEL: define <8 x i16> @uaddw8h(
+; CHECK-SAME: ptr [[A:%.*]], ptr [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP3:%.*]], label [[TMP13:%.*]], !prof [[PROF1]]
+; CHECK:       3:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       4:
+; CHECK-NEXT:    [[TMPVAR1:%.*]] = load <8 x i16>, ptr [[A]], align 16
+; CHECK-NEXT:    [[TMP5:%.*]] = ptrtoint ptr [[A]] to i64
+; CHECK-NEXT:    [[TMP6:%.*]] = xor i64 [[TMP5]], 193514046488576
+; CHECK-NEXT:    [[TMP7:%.*]] = inttoptr i64 [[TMP6]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <8 x i16>, ptr [[TMP7]], align 16
+; CHECK-NEXT:    [[_MSCMP3:%.*]] = icmp ne i64 [[TMP2]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP3]], label [[TMP8:%.*]], label [[TMP9:%.*]], !prof [[PROF1]]
+; CHECK:       8:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       9:
+; CHECK-NEXT:    [[TMPVAR2:%.*]] = load <8 x i8>, ptr [[B]], align 8
+; CHECK-NEXT:    [[TMP10:%.*]] = ptrtoint ptr [[B]] to i64
+; CHECK-NEXT:    [[TMP11:%.*]] = xor i64 [[TMP10]], 193514046488576
+; CHECK-NEXT:    [[TMP12:%.*]] = inttoptr i64 [[TMP11]] to ptr
+; CHECK-NEXT:    [[_MSLD1:%.*]] = load <8 x i8>, ptr [[TMP12]], align 8
+; CHECK-NEXT:    [[_MSPROP:%.*]] = zext <8 x i8> [[_MSLD1]] to <8 x i16>
+; CHECK-NEXT:    [[TMPVAR3:%.*]] = zext <8 x i8> [[TMPVAR2]] to <8 x i16>
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = or <8 x i16> [[_MSLD]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP4:%.*]] = add <8 x i16> [[TMPVAR1]], [[TMPVAR3]]
+; CHECK-NEXT:    store <8 x i16> [[_MSPROP2]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i16> [[TMP4]]
+;
+  %tmpvar1 = load <8 x i16>, ptr %A
+  %tmpvar2 = load <8 x i8>, ptr %B
+  %tmpvar3 = zext <8 x i8> %tmpvar2 to <8 x i16>
+  %tmpvar4 = add <8 x i16> %tmpvar1, %tmpvar3
+  ret <8 x i16> %tmpvar4
+}
+
+define <4 x i32> @uaddw4s(ptr %A, ptr %B) nounwind #0 {
+; CHECK-LABEL: define <4 x i32> @uaddw4s(
+; CHECK-SAME: ptr [[A:%.*]], ptr [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP3:%.*]], label [[TMP13:%.*]], !prof [[PROF1]]
+; CHECK:       3:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       4:
+; CHECK-NEXT:    [[TMPVAR1:%.*]] = load <4 x i32>, ptr [[A]], align 16
+; CHECK-NEXT:    [[TMP5:%.*]] = ptrtoint ptr [[A]] to i64
+; CHECK-NEXT:    [[TMP6:%.*]] = xor i64 [[TMP5]], 193514046488576
+; CHECK-NEXT:    [[TMP7:%.*]] = inttoptr i64 [[TMP6]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <4 x i32>, ptr [[TMP7]], align 16
+; CHECK-NEXT:    [[_MSCMP3:%.*]] = icmp ne i64 [[TMP2]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP3]], label [[TMP8:%.*]], label [[TMP9:%.*]], !prof [[PROF1]]
+; CHECK:       8:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       9:
+; CHECK-NEXT:    [[TMPVAR2:%.*]] = load <4 x i16>, ptr [[B]], align 8
+; CHECK-NEXT:    [[TMP10:%.*]] = ptrtoint ptr [[B]] to i64
+; CHECK-NEXT:    [[TMP11:%.*]] = xor i64 [[TMP10]], 193514046488576
+; CHECK-NEXT:    [[TMP12:%.*]] = inttoptr i64 [[TMP11]] to ptr
+; CHECK-NEXT:    [[_MSLD1:%.*]] = load <4 x i16>, ptr [[TMP12]], align 8
+; CHECK-NEXT:    [[_MSPROP:%.*]] = zext <4 x i16> [[_MSLD1]] to <4 x i32>
+; CHECK-NEXT:    [[TMPVAR3:%.*]] = zext <4 x i16> [[TMPVAR2]] to <4 x i32>
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = or <4 x i32> [[_MSLD]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP4:%.*]] = add <4 x i32> [[TMPVAR1]], [[TMPVAR3]]
+; CHECK-NEXT:    store <4 x i32> [[_MSPROP2]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i32> [[TMP4]]
+;
+  %tmpvar1 = load <4 x i32>, ptr %A
+  %tmpvar2 = load <4 x i16>, ptr %B
+  %tmpvar3 = zext <4 x i16> %tmpvar2 to <4 x i32>
+  %tmpvar4 = add <4 x i32> %tmpvar1, %tmpvar3
+  ret <4 x i32> %tmpvar4
+}
+
+define <2 x i64> @uaddw2d(ptr %A, ptr %B) nounwind #0 {
+; CHECK-LABEL: define <2 x i64> @uaddw2d(
+; CHECK-SAME: ptr [[A:%.*]], ptr [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP3:%.*]], label [[TMP13:%.*]], !prof [[PROF1]]
+; CHECK:       3:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       4:
+; CHECK-NEXT:    [[TMPVAR1:%.*]] = load <2 x i64>, ptr [[A]], align 16
+; CHECK-NEXT:    [[TMP5:%.*]] = ptrtoint ptr [[A]] to i64
+; CHECK-NEXT:    [[TMP6:%.*]] = xor i64 [[TMP5]], 193514046488576
+; CHECK-NEXT:    [[TMP7:%.*]] = inttoptr i64 [[TMP6]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <2 x i64>, ptr [[TMP7]], align 16
+; CHECK-NEXT:    [[_MSCMP3:%.*]] = icmp ne i64 [[TMP2]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP3]], label [[TMP8:%.*]], label [[TMP9:%.*]], !prof [[PROF1]]
+; CHECK:       8:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       9:
+; CHECK-NEXT:    [[TMPVAR2:%.*]] = load <2 x i32>, ptr [[B]], align 8
+; CHECK-NEXT:    [[TMP10:%.*]] = ptrtoint ptr [[B]] to i64
+; CHECK-NEXT:    [[TMP11:%.*]] = xor i64 [[TMP10]], 193514046488576
+; CHECK-NEXT:    [[TMP12:%.*]] = inttoptr i64 [[TMP11]] to ptr
+; CHECK-NEXT:    [[_MSLD1:%.*]] = load <2 x i32>, ptr [[TMP12]], align 8
+; CHECK-NEXT:    [[_MSPROP:%.*]] = zext <2 x i32> [[_MSLD1]] to <2 x i64>
+; CHECK-NEXT:    [[TMPVAR3:%.*]] = zext <2 x i32> [[TMPVAR2]] to <2 x i64>
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = or <2 x i64> [[_MSLD]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP4:%.*]] = add <2 x i64> [[TMPVAR1]], [[TMPVAR3]]
+; CHECK-NEXT:    store <2 x i64> [[_MSPROP2]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x i64> [[TMP4]]
+;
+  %tmpvar1 = load <2 x i64>, ptr %A
+  %tmpvar2 = load <2 x i32>, ptr %B
+  %tmpvar3 = zext <2 x i32> %tmpvar2 to <2 x i64>
+  %tmpvar4 = add <2 x i64> %tmpvar1, %tmpvar3
+  ret <2 x i64> %tmpvar4
+}
+
+define <8 x i16> @uaddw2_8h(ptr %A, ptr %B, <16 x i8> %param1) nounwind #0 {
+; CHECK-LABEL: define <8 x i16> @uaddw2_8h(
+; CHECK-SAME: ptr [[A:%.*]], ptr [[B:%.*]], <16 x i8> [[PARAM1:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <16 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP4:%.*]], label [[TMP8:%.*]], !prof [[PROF1]]
+; CHECK:       4:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       5:
+; CHECK-NEXT:    [[TMPVAR1:%.*]] = load <8 x i16>, ptr [[A]], align 16
+; CHECK-NEXT:    [[TMP5:%.*]] = ptrtoint ptr [[A]] to i64
+; CHECK-NEXT:    [[TMP6:%.*]] = xor i64 [[TMP5]], 193514046488576
+; CHECK-NEXT:    [[TMP7:%.*]] = inttoptr i64 [[TMP6]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <8 x i16>, ptr [[TMP7]], align 16
+; CHECK-NEXT:    [[_MSCMP4:%.*]] = icmp ne i64 [[TMP2]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP4]], label [[TMP9:%.*]], label [[TMP13:%.*]], !prof [[PROF1]]
+; CHECK:       9:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       10:
+; CHECK-NEXT:    [[TMPVAR2:%.*]] = load <16 x i8>, ptr [[B]], align 16
+; CHECK-NEXT:    [[TMP10:%.*]] = ptrtoint ptr [[B]] to i64
+; CHECK-NEXT:    [[TMP11:%.*]] = xor i64 [[TMP10]], 193514046488576
+; CHECK-NEXT:    [[TMP12:%.*]] = inttoptr i64 [[TMP11]] to ptr
+; CHECK-NEXT:    [[_MSLD1:%.*]] = load <16 x i8>, ptr [[TMP12]], align 16
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <16 x i8> [[_MSLD1]], <16 x i8> [[TMP3]], <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; CHECK-NEXT:    [[HIGH2:%.*]] = shufflevector <16 x i8> [[TMPVAR2]], <16 x i8> [[PARAM1]], <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = zext <8 x i8> [[_MSPROP]] to <8 x i16>
+; CHECK-NEXT:    [[EXT2:%.*]] = zext <8 x i8> [[HIGH2]] to <8 x i16>
+; CHECK-NEXT:    [[_MSPROP3:%.*]] = or <8 x i16> [[_MSLD]], [[_MSPROP2]]
+; CHECK-NEXT:    [[RES:%.*]] = add <8 x i16> [[TMPVAR1]], [[EXT2]]
+; CHECK-NEXT:    store <8 x i16> [[_MSPROP3]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i16> [[RES]]
+;
+  %tmpvar1 = load <8 x i16>, ptr %A
+
+  %tmpvar2 = load <16 x i8>, ptr %B
+  %high2 = shufflevector <16 x i8> %tmpvar2, <16 x i8> %param1, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+  %ext2 = zext <8 x i8> %high2 to <8 x i16>
+
+  %res = add <8 x i16> %tmpvar1, %ext2
+  ret <8 x i16> %res
+}
+
+define <4 x i32> @uaddw2_4s(ptr %A, ptr %B, <8 x i16> %param1) nounwind #0 {
+; CHECK-LABEL: define <4 x i32> @uaddw2_4s(
+; CHECK-SAME: ptr [[A:%.*]], ptr [[B:%.*]], <8 x i16> [[PARAM1:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP4:%.*]], label [[TMP8:%.*]], !prof [[PROF1]]
+; CHECK:       4:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       5:
+; CHECK-NEXT:    [[TMPVAR1:%.*]] = load <4 x i32>, ptr [[A]], align 16
+; CHECK-NEXT:    [[TMP5:%.*]] = ptrtoint ptr [[A]] to i64
+; CHECK-NEXT:    [[TMP6:%.*]] = xor i64 [[TMP5]], 193514046488576
+; CHECK-NEXT:    [[TMP7:%.*]] = inttoptr i64 [[TMP6]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <4 x i32>, ptr [[TMP7]], align 16
+; CHECK-NEXT:    [[_MSCMP4:%.*]] = icmp ne i64 [[TMP2]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP4]], label [[TMP9:%.*]], label [[TMP13:%.*]], !prof [[PROF1]]
+; CHECK:       9:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       10:
+; CHECK-NEXT:    [[TMPVAR2:%.*]] = load <8 x i16>, ptr [[B]], align 16
+; CHECK-NEXT:    [[TMP10:%.*]] = ptrtoint ptr [[B]] to i64
+; CHECK-NEXT:    [[TMP11:%.*]] = xor i64 [[TMP10]], 193514046488576
+; CHECK-NEXT:    [[TMP12:%.*]] = inttoptr i64 [[TMP11]] to ptr
+; CHECK-NEXT:    [[_MSLD1:%.*]] = load <8 x i16>, ptr [[TMP12]], align 16
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <8 x i16> [[_MSLD1]], <8 x i16> [[TMP3]], <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:    [[HIGH2:%.*]] = shufflevector <8 x i16> [[TMPVAR2]], <8 x i16> [[PARAM1]], <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = zext <4 x i16> [[_MSPROP]] to <4 x i32>
+; CHECK-NEXT:    [[EXT2:%.*]] = zext <4 x i16> [[HIGH2]] to <4 x i32>
+; CHECK-NEXT:    [[_MSPROP3:%.*]] = or <4 x i32> [[_MSLD]], [[_MSPROP2]]
+; CHECK-NEXT:    [[RES:%.*]] = add <4 x i32> [[TMPVAR1]], [[EXT2]]
+; CHECK-NEXT:    store <4 x i32> [[_MSPROP3]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i32> [[RES]]
+;
+  %tmpvar1 = load <4 x i32>, ptr %A
+
+  %tmpvar2 = load <8 x i16>, ptr %B
+  %high2 = shufflevector <8 x i16> %tmpvar2, <8 x i16> %param1, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+  %ext2 = zext <4 x i16> %high2 to <4 x i32>
+
+  %res = add <4 x i32> %tmpvar1, %ext2
+  ret <4 x i32> %res
+}
+
+define <2 x i64> @uaddw2_2d(ptr %A, ptr %B, <4 x i32> %param1) nounwind #0 {
+; CHECK-LABEL: define <2 x i64> @uaddw2_2d(
+; CHECK-SAME: ptr [[A:%.*]], ptr [[B:%.*]], <4 x i32> [[PARAM1:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP4:%.*]], label [[TMP8:%.*]], !prof [[PROF1]]
+; CHECK:       4:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       5:
+; CHECK-NEXT:    [[TMPVAR1:%.*]] = load <2 x i64>, ptr [[A]], align 16
+; CHECK-NEXT:    [[TMP5:%.*]] = ptrtoint ptr [[A]] to i64
+; CHECK-NEXT:    [[TMP6:%.*]] = xor i64 [[TMP5]], 193514046488576
+; CHECK-NEXT:    [[TMP7:%.*]] = inttoptr i64 [[TMP6]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <2 x i64>, ptr [[TMP7]], align 16
+; CHECK-NEXT:    [[_MSCMP4:%.*]] = icmp ne i64 [[TMP2]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP4]], label [[TMP9:%.*]], label [[TMP13:%.*]], !prof [[PROF1]]
+; CHECK:       9:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       10:
+; CHECK-NEXT:    [[TMPVAR2:%.*]] = load <4 x i32>, ptr [[B]], align 16
+; CHECK-NEXT:    [[TMP10:%.*]] = ptrtoint ptr [[B]] to i64
+; CHECK-NEXT:    [[TMP11:%.*]] = xor i64 [[TMP10]], 193514046488576
+; CHECK-NEXT:    [[TMP12:%.*]] = inttoptr i64 [[TMP11]] to ptr
+; CHECK-NEXT:    [[_MSLD1:%.*]] = load <4 x i32>, ptr [[TMP12]], align 16
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <4 x i32> [[_MSLD1]], <4 x i32> [[TMP3]], <2 x i32> <i32 2, i32 3>
+; CHECK-NEXT:    [[HIGH2:%.*]] = shufflevector <4 x i32> [[TMPVAR2]], <4 x i32> [[PARAM1]], <2 x i32> <i32 2, i32 3>
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = zext <2 x i32> [[_MSPROP]] to <2 x i64>
+; CHECK-NEXT:    [[EXT2:%.*]] = zext <2 x i32> [[HIGH2]] to <2 x i64>
+; CHECK-NEXT:    [[_MSPROP3:%.*]] = or <2 x i64> [[_MSLD]], [[_MSPROP2]]
+; CHECK-NEXT:    [[RES:%.*]] = add <2 x i64> [[TMPVAR1]], [[EXT2]]
+; CHECK-NEXT:    store <2 x i64> [[_MSPROP3]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x i64> [[RES]]
+;
+  %tmpvar1 = load <2 x i64>, ptr %A
+
+  %tmpvar2 = load <4 x i32>, ptr %B
+  %high2 = shufflevector <4 x i32> %tmpvar2, <4 x i32> %param1, <2 x i32> <i32 2, i32 3>
+  %ext2 = zext <2 x i32> %high2 to <2 x i64>
+
+  %res = add <2 x i64> %tmpvar1, %ext2
+  ret <2 x i64> %res
+}
+
+define <8 x i16> @saddw8h(ptr %A, ptr %B) nounwind #0 {
+; CHECK-LABEL: define <8 x i16> @saddw8h(
+; CHECK-SAME: ptr [[A:%.*]], ptr [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP3:%.*]], label [[TMP13:%.*]], !prof [[PROF1]]
+; CHECK:       3:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       4:
+; CHECK-NEXT:    [[TMPVAR1:%.*]] = load <8 x i16>, ptr [[A]], align 16
+; CHECK-NEXT:    [[TMP5:%.*]] = ptrtoint ptr [[A]] to i64
+; CHECK-NEXT:    [[TMP6:%.*]] = xor i64 [[TMP5]], 193514046488576
+; CHECK-NEXT:    [[TMP7:%.*]] = inttoptr i64 [[TMP6]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <8 x i16>, ptr [[TMP7]], align 16
+; CHECK-NEXT:    [[_MSCMP3:%.*]] = icmp ne i64 [[TMP2]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP3]], label [[TMP8:%.*]], label [[TMP9:%.*]], !prof [[PROF1]]
+; CHECK:       8:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       9:
+; CHECK-NEXT:    [[TMPVAR2:%.*]] = load <8 x i8>, ptr [[B]], align 8
+; CHECK-NEXT:    [[TMP10:%.*]] = ptrtoint ptr [[B]] to i64
+; CHECK-NEXT:    [[TMP11:%.*]] = xor i64 [[TMP10]], 193514046488576
+; CHECK-NEXT:    [[TMP12:%.*]] = inttoptr i64 [[TMP11]] to ptr
+; CHECK-NEXT:    [[_MSLD1:%.*]] = load <8 x i8>, ptr [[TMP12]], align 8
+; CHECK-NEXT:    [[_MSPROP:%.*]] = sext <8 x i8> [[_MSLD1]] to <8 x i16>
+; CHECK-NEXT:    [[TMPVAR3:%.*]] = sext <8 x i8> [[TMPVAR2]] to <8 x i16>
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = or <8 x i16> [[_MSLD]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP4:%.*]] = add <8 x i16> [[TMPVAR1]], [[TMPVAR3]]
+; CHECK-NEXT:    store <8 x i16> [[_MSPROP2]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i16> [[TMP4]]
+;
+  %tmpvar1 = load <8 x i16>, ptr %A
+  %tmpvar2 = load <8 x i8>, ptr %B
+  %tmpvar3 = sext <8 x i8> %tmpvar2 to <8 x i16>
+  %tmpvar4 = add <8 x i16> %tmpvar1, %tmpvar3
+  ret <8 x i16> %tmpvar4
+}
+
+define <4 x i32> @saddw4s(ptr %A, ptr %B) nounwind #0 {
+; CHECK-LABEL: define <4 x i32> @saddw4s(
+; CHECK-SAME: ptr [[A:%.*]], ptr [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP3:%.*]], label [[TMP13:%.*]], !prof [[PROF1]]
+; CHECK:       3:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       4:
+; CHECK-NEXT:    [[TMPVAR1:%.*]] = load <4 x i32>, ptr [[A]], align 16
+; CHECK-NEXT:    [[TMP5:%.*]] = ptrtoint ptr [[A]] to i64
+; CHECK-NEXT:    [[TMP6:%.*]] = xor i64 [[TMP5]], 193514046488576
+; CHECK-NEXT:    [[TMP7:%.*]] = inttoptr i64 [[TMP6]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <4 x i32>, ptr [[TMP7]], align 16
+; CHECK-NEXT:    [[_MSCMP3:%.*]] = icmp ne i64 [[TMP2]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP3]], label [[TMP8:%.*]], label [[TMP9:%.*]], !prof [[PROF1]]
+; CHECK:       8:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       9:
+; CHECK-NEXT:    [[TMPVAR2:%.*]] = load <4 x i16>, ptr [[B]], align 8
+; CHECK-NEXT:    [[TMP10:%.*]] = ptrtoint ptr [[B]] to i64
+; CHECK-NEXT:    [[TMP11:%.*]] = xor i64 [[TMP10]], 193514046488576
+; CHECK-NEXT:    [[TMP12:%.*]] = inttoptr i64 [[TMP11]] to ptr
+; CHECK-NEXT:    [[_MSLD1:%.*]] = load <4 x i16>, ptr [[TMP12]], align 8
+; CHECK-NEXT:    [[_MSPROP:%.*]] = sext <4 x i16> [[_MSLD1]] to <4 x i32>
+; CHECK-NEXT:    [[TMPVAR3:%.*]] = sext <4 x i16> [[TMPVAR2]] to <4 x i32>
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = or <4 x i32> [[_MSLD]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP4:%.*]] = add <4 x i32> [[TMPVAR1]], [[TMPVAR3]]
+; CHECK-NEXT:    store <4 x i32> [[_MSPROP2]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i32> [[TMP4]]
+;
+  %tmpvar1 = load <4 x i32>, ptr %A
+  %tmpvar2 = load <4 x i16>, ptr %B
+  %tmpvar3 = sext <4 x i16> %tmpvar2 to <4 x i32>
+  %tmpvar4 = add <4 x i32> %tmpvar1, %tmpvar3
+  ret <4 x i32> %tmpvar4
+}
+
+define <2 x i64> @saddw2d(ptr %A, ptr %B) nounwind #0 {
+; CHECK-LABEL: define <2 x i64> @saddw2d(
+; CHECK-SAME: ptr [[A:%.*]], ptr [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP3:%.*]], label [[TMP13:%.*]], !prof [[PROF1]]
+; CHECK:       3:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       4:
+; CHECK-NEXT:    [[TMPVAR1:%.*]] = load <2 x i64>, ptr [[A]], align 16
+; CHECK-NEXT:    [[TMP5:%.*]] = ptrtoint ptr [[A]] to i64
+; CHECK-NEXT:    [[TMP6:%.*]] = xor i64 [[TMP5]], 193514046488576
+; CHECK-NEXT:    [[TMP7:%.*]] = inttoptr i64 [[TMP6]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <2 x i64>, ptr [[TMP7]], align 16
+; CHECK-NEXT:    [[_MSCMP3:%.*]] = icmp ne i64 [[TMP2]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP3]], label [[TMP8:%.*]], label [[TMP9:%.*]], !prof [[PROF1]]
+; CHECK:       8:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       9:
+; CHECK-NEXT:    [[TMPVAR2:%.*]] = load <2 x i32>, ptr [[B]], align 8
+; CHECK-NEXT:    [[TMP10:%.*]] = ptrtoint ptr [[B]] to i64
+; CHECK-NEXT:    [[TMP11:%.*]] = xor i64 [[TMP10]], 193514046488576
+; CHECK-NEXT:    [[TMP12:%.*]] = inttoptr i64 [[TMP11]] to ptr
+; CHECK-NEXT:    [[_MSLD1:%.*]] = load <2 x i32>, ptr [[TMP12]], align 8
+; CHECK-NEXT:    [[_MSPROP:%.*]] = sext <2 x i32> [[_MSLD1]] to <2 x i64>
+; CHECK-NEXT:    [[TMPVAR3:%.*]] = sext <2 x i32> [[TMPVAR2]] to <2 x i64>
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = or <2 x i64> [[_MSLD]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP4:%.*]] = add <2 x i64> [[TMPVAR1]], [[TMPVAR3]]
+; CHECK-NEXT:    store <2 x i64> [[_MSPROP2]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x i64> [[TMP4]]
+;
+  %tmpvar1 = load <2 x i64>, ptr %A
+  %tmpvar2 = load <2 x i32>, ptr %B
+  %tmpvar3 = sext <2 x i32> %tmpvar2 to <2 x i64>
+  %tmpvar4 = add <2 x i64> %tmpvar1, %tmpvar3
+  ret <2 x i64> %tmpvar4
+}
+
+define <8 x i16> @saddw2_8h(ptr %A, ptr %B, <16 x i8> %param1) nounwind #0 {
+; CHECK-LABEL: define <8 x i16> @saddw2_8h(
+; CHECK-SAME: ptr [[A:%.*]], ptr [[B:%.*]], <16 x i8> [[PARAM1:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <16 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP4:%.*]], label [[TMP8:%.*]], !prof [[PROF1]]
+; CHECK:       4:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       5:
+; CHECK-NEXT:    [[TMPVAR1:%.*]] = load <8 x i16>, ptr [[A]], align 16
+; CHECK-NEXT:    [[TMP5:%.*]] = ptrtoint ptr [[A]] to i64
+; CHECK-NEXT:    [[TMP6:%.*]] = xor i64 [[TMP5]], 193514046488576
+; CHECK-NEXT:    [[TMP7:%.*]] = inttoptr i64 [[TMP6]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <8 x i16>, ptr [[TMP7]], align 16
+; CHECK-NEXT:    [[_MSCMP4:%.*]] = icmp ne i64 [[TMP2]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP4]], label [[TMP9:%.*]], label [[TMP13:%.*]], !prof [[PROF1]]
+; CHECK:       9:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       10:
+; CHECK-NEXT:    [[TMPVAR2:%.*]] = load <16 x i8>, ptr [[B]], align 16
+; CHECK-NEXT:    [[TMP10:%.*]] = ptrtoint ptr [[B]] to i64
+; CHECK-NEXT:    [[TMP11:%.*]] = xor i64 [[TMP10]], 193514046488576
+; CHECK-NEXT:    [[TMP12:%.*]] = inttoptr i64 [[TMP11]] to ptr
+; CHECK-NEXT:    [[_MSLD1:%.*]] = load <16 x i8>, ptr [[TMP12]], align 16
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <16 x i8> [[_MSLD1]], <16 x i8> [[TMP3]], <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; CHECK-NEXT:    [[HIGH2:%.*]] = shufflevector <16 x i8> [[TMPVAR2]], <16 x i8> [[PARAM1]], <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = sext <8 x i8> [[_MSPROP]] to <8 x i16>
+; CHECK-NEXT:    [[EXT2:%.*]] = sext <8 x i8> [[HIGH2]] to <8 x i16>
+; CHECK-NEXT:    [[_MSPROP3:%.*]] = or <8 x i16> [[_MSLD]], [[_MSPROP2]]
+; CHECK-NEXT:    [[RES:%.*]] = add <8 x i16> [[TMPVAR1]], [[EXT2]]
+; CHECK-NEXT:    store <8 x i16> [[_MSPROP3]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i16> [[RES]]
+;
+  %tmpvar1 = load <8 x i16>, ptr %A
+
+  %tmpvar2 = load <16 x i8>, ptr %B
+  %high2 = shufflevector <16 x i8> %tmpvar2, <16 x i8> %param1, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+  %ext2 = sext <8 x i8> %high2 to <8 x i16>
+
+  %res = add <8 x i16> %tmpvar1, %ext2
+  ret <8 x i16> %res
+}
+
+define <4 x i32> @saddw2_4s(ptr %A, ptr %B, <8 x i16> %param1) nounwind #0 {
+; CHECK-LABEL: define <4 x i32> @saddw2_4s(
+; CHECK-SAME: ptr [[A:%.*]], ptr [[B:%.*]], <8 x i16> [[PARAM1:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP4:%.*]], label [[TMP8:%.*]], !prof [[PROF1]]
+; CHECK:       4:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       5:
+; CHECK-NEXT:    [[TMPVAR1:%.*]] = load <4 x i32>, ptr [[A]], align 16
+; CHECK-NEXT:    [[TMP5:%.*]] = ptrtoint ptr [[A]] to i64
+; CHECK-NEXT:    [[TMP6:%.*]] = xor i64 [[TMP5]], 193514046488576
+; CHECK-NEXT:    [[TMP7:%.*]] = inttoptr i64 [[TMP6]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <4 x i32>, ptr [[TMP7]], align 16
+; CHECK-NEXT:    [[_MSCMP4:%.*]] = icmp ne i64 [[TMP2]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP4]], label [[TMP9:%.*]], label [[TMP13:%.*]], !prof [[PROF1]]
+; CHECK:       9:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       10:
+; CHECK-NEXT:    [[TMPVAR2:%.*]] = load <8 x i16>, ptr [[B]], align 16
+; CHECK-NEXT:    [[TMP10:%.*]] = ptrtoint ptr [[B]] to i64
+; CHECK-NEXT:    [[TMP11:%.*]] = xor i64 [[TMP10]], 193514046488576
+; CHECK-NEXT:    [[TMP12:%.*]] = inttoptr i64 [[TMP11]] to ptr
+; CHECK-NEXT:    [[_MSLD1:%.*]] = load <8 x i16>, ptr [[TMP12]], align 16
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <8 x i16> [[_MSLD1]], <8 x i16> [[TMP3]], <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:    [[HIGH2:%.*]] = shufflevector <8 x i16> [[TMPVAR2]], <8 x i16> [[PARAM1]], <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = sext <4 x i16> [[_MSPROP]] to <4 x i32>
+; CHECK-NEXT:    [[EXT2:%.*]] = sext <4 x i16> [[HIGH2]] to <4 x i32>
+; CHECK-NEXT:    [[_MSPROP3:%.*]] = or <4 x i32> [[_MSLD]], [[_MSPROP2]]
+; CHECK-NEXT:    [[RES:%.*]] = add <4 x i32> [[TMPVAR1]], [[EXT2]]
+; CHECK-NEXT:    store <4 x i32> [[_MSPROP3]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i32> [[RES]]
+;
+  %tmpvar1 = load <4 x i32>, ptr %A
+
+  %tmpvar2 = load <8 x i16>, ptr %B
+  %high2 = shufflevector <8 x i16> %tmpvar2, <8 x i16> %param1, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+  %ext2 = sext <4 x i16> %high2 to <4 x i32>
+
+  %res = add <4 x i32> %tmpvar1, %ext2
+  ret <4 x i32> %res
+}
+
+define <2 x i64> @saddw2_2d(ptr %A, ptr %B, <4 x i32> %param1) nounwind #0 {
+; CHECK-LABEL: define <2 x i64> @saddw2_2d(
+; CHECK-SAME: ptr [[A:%.*]], ptr [[B:%.*]], <4 x i32> [[PARAM1:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP4:%.*]], label [[TMP8:%.*]], !prof [[PROF1]]
+; CHECK:       4:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       5:
+; CHECK-NEXT:    [[TMPVAR1:%.*]] = load <2 x i64>, ptr [[A]], align 16
+; CHECK-NEXT:    [[TMP5:%.*]] = ptrtoint ptr [[A]] to i64
+; CHECK-NEXT:    [[TMP6:%.*]] = xor i64 [[TMP5]], 193514046488576
+; CHECK-NEXT:    [[TMP7:%.*]] = inttoptr i64 [[TMP6]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <2 x i64>, ptr [[TMP7]], align 16
+; CHECK-NEXT:    [[_MSCMP4:%.*]] = icmp ne i64 [[TMP2]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP4]], label [[TMP9:%.*]], label [[TMP13:%.*]], !prof [[PROF1]]
+; CHECK:       9:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       10:
+; CHECK-NEXT:    [[TMPVAR2:%.*]] = load <4 x i32>, ptr [[B]], align 16
+; CHECK-NEXT:    [[TMP10:%.*]] = ptrtoint ptr [[B]] to i64
+; CHECK-NEXT:    [[TMP11:%.*]] = xor i64 [[TMP10]], 193514046488576
+; CHECK-NEXT:    [[TMP12:%.*]] = inttoptr i64 [[TMP11]] to ptr
+; CHECK-NEXT:    [[_MSLD1:%.*]] = load <4 x i32>, ptr [[TMP12]], align 16
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <4 x i32> [[_MSLD1]], <4 x i32> [[TMP3]], <2 x i32> <i32 2, i32 3>
+; CHECK-NEXT:    [[HIGH2:%.*]] = shufflevector <4 x i32> [[TMPVAR2]], <4 x i32> [[PARAM1]], <2 x i32> <i32 2, i32 3>
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = sext <2 x i32> [[_MSPROP]] to <2 x i64>
+; CHECK-NEXT:    [[EXT2:%.*]] = sext <2 x i32> [[HIGH2]] to <2 x i64>
+; CHECK-NEXT:    [[_MSPROP3:%.*]] = or <2 x i64> [[_MSLD]], [[_MSPROP2]]
+; CHECK-NEXT:    [[RES:%.*]] = add <2 x i64> [[TMPVAR1]], [[EXT2]]
+; CHECK-NEXT:    store <2 x i64> [[_MSPROP3]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x i64> [[RES]]
+;
+  %tmpvar1 = load <2 x i64>, ptr %A
+
+  %tmpvar2 = load <4 x i32>, ptr %B
+  %high2 = shufflevector <4 x i32> %tmpvar2, <4 x i32> %param1, <2 x i32> <i32 2, i32 3>
+  %ext2 = sext <2 x i32> %high2 to <2 x i64>
+
+  %res = add <2 x i64> %tmpvar1, %ext2
+  ret <2 x i64> %res
+}
+
+define <4 x i16> @saddlp4h(ptr %A) nounwind #0 {
+; CHECK-LABEL: define <4 x i16> @saddlp4h(
+; CHECK-SAME: ptr [[A:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP2:%.*]], label [[TMP10:%.*]], !prof [[PROF1]]
+; CHECK:       2:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       3:
+; CHECK-NEXT:    [[TMPVAR1:%.*]] = load <8 x i8>, ptr [[A]], align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = ptrtoint ptr [[A]] to i64
+; CHECK-NEXT:    [[TMP5:%.*]] = xor i64 [[TMP4]], 193514046488576
+; CHECK-NEXT:    [[TMP6:%.*]] = inttoptr i64 [[TMP5]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <8 x i8>, ptr [[TMP6]], align 8
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <8 x i8> [[_MSLD]] to i64
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i64 [[TMP7]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP1]], label [[TMP8:%.*]], label [[TMP9:%.*]], !prof [[PROF1]]
+; CHECK:       8:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       9:
+; CHECK-NEXT:    [[TMP3:%.*]] = call <4 x i16> @llvm.aarch64.neon.saddlp.v4i16.v8i8(<8 x i8> [[TMPVAR1]])
+; CHECK-NEXT:    store <4 x i16> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i16> [[TMP3]]
+;
+  %tmpvar1 = load <8 x i8>, ptr %A
+  %tmpvar3 = call <4 x i16> @llvm.aarch64.neon.saddlp.v4i16.v8i8(<8 x i8> %tmpvar1)
+  ret <4 x i16> %tmpvar3
+}
+
+define <2 x i32> @saddlp2s(ptr %A) nounwind #0 {
+; CHECK-LABEL: define <2 x i32> @saddlp2s(
+; CHECK-SAME: ptr [[A:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP2:%.*]], label [[TMP10:%.*]], !prof [[PROF1]]
+; CHECK:       2:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       3:
+; CHECK-NEXT:    [[TMPVAR1:%.*]] = load <4 x i16>, ptr [[A]], align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = ptrtoint ptr [[A]] to i64
+; CHECK-NEXT:    [[TMP5:%.*]] = xor i64 [[TMP4]], 193514046488576
+; CHECK-NEXT:    [[TMP6:%.*]] = inttoptr i64 [[TMP5]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <4 x i16>, ptr [[TMP6]], align 8
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <4 x i16> [[_MSLD]] to i64
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i64 [[TMP7]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP1]], label [[TMP8:%.*]], label [[TMP9:%.*]], !prof [[PROF1]]
+; CHECK:       8:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       9:
+; CHECK-NEXT:    [[TMP3:%.*]] = call <2 x i32> @llvm.aarch64.neon.saddlp.v2i32.v4i16(<4 x i16> [[TMPVAR1]])
+; CHECK-NEXT:    store <2 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x i32> [[TMP3]]
+;
+  %tmpvar1 = load <4 x i16>, ptr %A
+  %tmpvar3 = call <2 x i32> @llvm.aarch64.neon.saddlp.v2i32.v4i16(<4 x i16> %tmpvar1)
+  ret <2 x i32> %tmpvar3
+}
+
+define <1 x i64> @saddlp1d(ptr %A) nounwind #0 {
+; CHECK-LABEL: define <1 x i64> @saddlp1d(
+; CHECK-SAME: ptr [[A:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP2:%.*]], label [[TMP10:%.*]], !prof [[PROF1]]
+; CHECK:       2:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       3:
+; CHECK-NEXT:    [[TMPVAR1:%.*]] = load <2 x i32>, ptr [[A]], align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = ptrtoint ptr [[A]] to i64
+; CHECK-NEXT:    [[TMP5:%.*]] = xor i64 [[TMP4]], 193514046488576
+; CHECK-NEXT:    [[TMP6:%.*]] = inttoptr i64 [[TMP5]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <2 x i32>, ptr [[TMP6]], align 8
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <2 x i32> [[_MSLD]] to i64
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i64 [[TMP7]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP1]], label [[TMP8:%.*]], label [[TMP9:%.*]], !prof [[PROF1]]
+; CHECK:       8:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       9:
+; CHECK-NEXT:    [[TMP3:%.*]] = call <1 x i64> @llvm.aarch64.neon.saddlp.v1i64.v2i32(<2 x i32> [[TMPVAR1]])
+; CHECK-NEXT:    store <1 x i64> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <1 x i64> [[TMP3]]
+;
+  %tmpvar1 = load <2 x i32>, ptr %A
+  %tmpvar3 = call <1 x i64> @llvm.aarch64.neon.saddlp.v1i64.v2i32(<2 x i32> %tmpvar1)
+  ret <1 x i64> %tmpvar3
+}
+
+define <8 x i16> @saddlp8h(ptr %A) nounwind #0 {
+; CHECK-LABEL: define <8 x i16> @saddlp8h(
+; CHECK-SAME: ptr [[A:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP2:%.*]], label [[TMP10:%.*]], !prof [[PROF1]]
+; CHECK:       2:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       3:
+; CHECK-NEXT:    [[TMPVAR1:%.*]] = load <16 x i8>, ptr [[A]], align 16
+; CHECK-NEXT:    [[TMP4:%.*]] = ptrtoint ptr [[A]] to i64
+; CHECK-NEXT:    [[TMP5:%.*]] = xor i64 [[TMP4]], 193514046488576
+; CHECK-NEXT:    [[TMP6:%.*]] = inttoptr i64 [[TMP5]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <16 x i8>, ptr [[TMP6]], align 16
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <16 x i8> [[_MSLD]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP7]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP1]], label [[TMP8:%.*]], label [[TMP9:%.*]], !prof [[PROF1]]
+; CHECK:       8:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       9:
+; CHECK-NEXT:    [[TMP3:%.*]] = call <8 x i16> @llvm.aarch64.neon.saddlp.v8i16.v16i8(<16 x i8> [[TMPVAR1]])
+; CHECK-NEXT:    store <8 x i16> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i16> [[TMP3]]
+;
+  %tmpvar1 = load <16 x i8>, ptr %A
+  %tmpvar3 = call <8 x i16> @llvm.aarch64.neon.saddlp.v8i16.v16i8(<16 x i8> %tmpvar1)
+  ret <8 x i16> %tmpvar3
+}
+
+define <4 x i32> @saddlp4s(ptr %A) nounwind #0 {
+; CHECK-LABEL: define <4 x i32> @saddlp4s(
+; CHECK-SAME: ptr [[A:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP2:%.*]], label [[TMP10:%.*]], !prof [[PROF1]]
+; CHECK:       2:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       3:
+; CHECK-NEXT:    [[TMPVAR1:%.*]] = load <8 x i16>, ptr [[A]], align 16
+; CHECK-NEXT:    [[TMP4:%.*]] = ptrtoint ptr [[A]] to i64
+; CHECK-NEXT:    [[TMP5:%.*]] = xor i64 [[TMP4]], 193514046488576
+; CHECK-NEXT:    [[TMP6:%.*]] = inttoptr i64 [[TMP5]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <8 x i16>, ptr [[TMP6]], align 16
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <8 x i16> [[_MSLD]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP7]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP1]], label [[TMP8:%.*]], label [[TMP9:%.*]], !prof [[PROF1]]
+; CHECK:       8:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       9:
+; CHECK-NEXT:    [[TMP3:%.*]] = call <4 x i32> @llvm.aarch64.neon.saddlp.v4i32.v8i16(<8 x i16> [[TMPVAR1]])
+; CHECK-NEXT:    store <4 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i32> [[TMP3]]
+;
+  %tmpvar1 = load <8 x i16>, ptr %A
+  %tmpvar3 = call <4 x i32> @llvm.aarch64.neon.saddlp.v4i32.v8i16(<8 x i16> %tmpvar1)
+  ret <4 x i32> %tmpvar3
+}
+
+define <2 x i64> @saddlp2d(ptr %A) nounwind #0 {
+; CHECK-LABEL: define <2 x i64> @saddlp2d(
+; CHECK-SAME: ptr [[A:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP2:%.*]], label [[TMP10:%.*]], !prof [[PROF1]]
+; CHECK:       2:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       3:
+; CHECK-NEXT:    [[TMPVAR1:%.*]] = load <4 x i32>, ptr [[A]], align 16
+; CHECK-NEXT:    [[TMP4:%.*]] = ptrtoint ptr [[A]] to i64
+; CHECK-NEXT:    [[TMP5:%.*]] = xor i64 [[TMP4]], 193514046488576
+; CHECK-NEXT:    [[TMP6:%.*]] = inttoptr i64 [[TMP5]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <4 x i32>, ptr [[TMP6]], align 16
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <4 x i32> [[_MSLD]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP7]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP1]], label [[TMP8:%.*]], label [[TMP9:%.*]], !prof [[PROF1]]
+; CHECK:       8:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       9:
+; CHECK-NEXT:    [[TMP3:%.*]] = call <2 x i64> @llvm.aarch64.neon.saddlp.v2i64.v4i32(<4 x i32> [[TMPVAR1]])
+; CHECK-NEXT:    store <2 x i64> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x i64> [[TMP3]]
+;
+  %tmpvar1 = load <4 x i32>, ptr %A
+  %tmpvar3 = call <2 x i64> @llvm.aarch64.neon.saddlp.v2i64.v4i32(<4 x i32> %tmpvar1)
+  ret <2 x i64> %tmpvar3
+}
+
+declare <4 x i16>  @llvm.aarch64.neon.saddlp.v4i16.v8i8(<8 x i8>) nounwind readnone
+declare <2 x i32> @llvm.aarch64.neon.saddlp.v2i32.v4i16(<4 x i16>) nounwind readnone
+declare <1 x i64> @llvm.aarch64.neon.saddlp.v1i64.v2i32(<2 x i32>) nounwind readnone
+
+declare <8 x i16>  @llvm.aarch64.neon.saddlp.v8i16.v16i8(<16 x i8>) nounwind readnone
+declare <4 x i32> @llvm.aarch64.neon.saddlp.v4i32.v8i16(<8 x i16>) nounwind readnone
+declare <2 x i64> @llvm.aarch64.neon.saddlp.v2i64.v4i32(<4 x i32>) nounwind readnone
+
+define <4 x i16> @uaddlp4h(ptr %A) nounwind #0 {
+; CHECK-LABEL: define <4 x i16> @uaddlp4h(
+; CHECK-SAME: ptr [[A:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP2:%.*]], label [[TMP10:%.*]], !prof [[PROF1]]
+; CHECK:       2:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       3:
+; CHECK-NEXT:    [[TMPVAR1:%.*]] = load <8 x i8>, ptr [[A]], align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = ptrtoint ptr [[A]] to i64
+; CHECK-NEXT:    [[TMP5:%.*]] = xor i64 [[TMP4]], 193514046488576
+; CHECK-NEXT:    [[TMP6:%.*]] = inttoptr i64 [[TMP5]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <8 x i8>, ptr [[TMP6]], align 8
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <8 x i8> [[_MSLD]] to i64
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i64 [[TMP7]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP1]], label [[TMP8:%.*]], label [[TMP9:%.*]], !prof [[PROF1]]
+; CHECK:       8:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       9:
+; CHECK-NEXT:    [[TMP3:%.*]] = call <4 x i16> @llvm.aarch64.neon.uaddlp.v4i16.v8i8(<8 x i8> [[TMPVAR1]])
+; CHECK-NEXT:    store <4 x i16> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i16> [[TMP3]]
+;
+  %tmpvar1 = load <8 x i8>, ptr %A
+  %tmpvar3 = call <4 x i16> @llvm.aarch64.neon.uaddlp.v4i16.v8i8(<8 x i8> %tmpvar1)
+  ret <4 x i16> %tmpvar3
+}
+
+define <2 x i32> @uaddlp2s(ptr %A) nounwind #0 {
+; CHECK-LABEL: define <2 x i32> @uaddlp2s(
+; CHECK-SAME: ptr [[A:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP2:%.*]], label [[TMP10:%.*]], !prof [[PROF1]]
+; CHECK:       2:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       3:
+; CHECK-NEXT:    [[TMPVAR1:%.*]] = load <4 x i16>, ptr [[A]], align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = ptrtoint ptr [[A]] to i64
+; CHECK-NEXT:    [[TMP5:%.*]] = xor i64 [[TMP4]], 193514046488576
+; CHECK-NEXT:    [[TMP6:%.*]] = inttoptr i64 [[TMP5]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <4 x i16>, ptr [[TMP6]], align 8
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <4 x i16> [[_MSLD]] to i64
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i64 [[TMP7]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP1]], label [[TMP8:%.*]], label [[TMP9:%.*]], !prof [[PROF1]]
+; CHECK:       8:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       9:
+; CHECK-NEXT:    [[TMP3:%.*]] = call <2 x i32> @llvm.aarch64.neon.uaddlp.v2i32.v4i16(<4 x i16> [[TMPVAR1]])
+; CHECK-NEXT:    store <2 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x i32> [[TMP3]]
+;
+  %tmpvar1 = load <4 x i16>, ptr %A
+  %tmpvar3 = call <2 x i32> @llvm.aarch64.neon.uaddlp.v2i32.v4i16(<4 x i16> %tmpvar1)
+  ret <2 x i32> %tmpvar3
+}
+
+define <1 x i64> @uaddlp1d(ptr %A) nounwind #0 {
+; CHECK-LABEL: define <1 x i64> @uaddlp1d(
+; CHECK-SAME: ptr [[A:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP2:%.*]], label [[TMP10:%.*]], !prof [[PROF1]]
+; CHECK:       2:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       3:
+; CHECK-NEXT:    [[TMPVAR1:%.*]] = load <2 x i32>, ptr [[A]], align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = ptrtoint ptr [[A]] to i64
+; CHECK-NEXT:    [[TMP5:%.*]] = xor i64 [[TMP4]], 193514046488576
+; CHECK-NEXT:    [[TMP6:%.*]] = inttoptr i64 [[TMP5]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <2 x i32>, ptr [[TMP6]], align 8
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <2 x i32> [[_MSLD]] to i64
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i64 [[TMP7]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP1]], label [[TMP8:%.*]], label [[TMP9:%.*]], !prof [[PROF1]]
+; CHECK:       8:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       9:
+; CHECK-NEXT:    [[TMP3:%.*]] = call <1 x i64> @llvm.aarch64.neon.uaddlp.v1i64.v2i32(<2 x i32> [[TMPVAR1]])
+; CHECK-NEXT:    store <1 x i64> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <1 x i64> [[TMP3]]
+;
+  %tmpvar1 = load <2 x i32>, ptr %A
+  %tmpvar3 = call <1 x i64> @llvm.aarch64.neon.uaddlp.v1i64.v2i32(<2 x i32> %tmpvar1)
+  ret <1 x i64> %tmpvar3
+}
+
+define <8 x i16> @uaddlp8h(ptr %A) nounwind #0 {
+; CHECK-LABEL: define <8 x i16> @uaddlp8h(
+; CHECK-SAME: ptr [[A:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP2:%.*]], label [[TMP10:%.*]], !prof [[PROF1]]
+; CHECK:       2:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       3:
+; CHECK-NEXT:    [[TMPVAR1:%.*]] = load <16 x i8>, ptr [[A]], align 16
+; CHECK-NEXT:    [[TMP4:%.*]] = ptrtoint ptr [[A]] to i64
+; CHECK-NEXT:    [[TMP5:%.*]] = xor i64 [[TMP4]], 193514046488576
+; CHECK-NEXT:    [[TMP6:%.*]] = inttoptr i64 [[TMP5]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <16 x i8>, ptr [[TMP6]], align 16
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <16 x i8> [[_MSLD]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP7]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP1]], label [[TMP8:%.*]], label [[TMP9:%.*]], !prof [[PROF1]]
+; CHECK:       8:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       9:
+; CHECK-NEXT:    [[TMP3:%.*]] = call <8 x i16> @llvm.aarch64.neon.uaddlp.v8i16.v16i8(<16 x i8> [[TMPVAR1]])
+; CHECK-NEXT:    store <8 x i16> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i16> [[TMP3]]
+;
+  %tmpvar1 = load <16 x i8>, ptr %A
+  %tmpvar3 = call <8 x i16> @llvm.aarch64.neon.uaddlp.v8i16.v16i8(<16 x i8> %tmpvar1)
+  ret <8 x i16> %tmpvar3
+}
+
+define <4 x i32> @uaddlp4s(ptr %A) nounwind #0 {
+; CHECK-LABEL: define <4 x i32> @uaddlp4s(
+; CHECK-SAME: ptr [[A:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP2:%.*]], label [[TMP10:%.*]], !prof [[PROF1]]
+; CHECK:       2:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       3:
+; CHECK-NEXT:    [[TMPVAR1:%.*]] = load <8 x i16>, ptr [[A]], align 16
+; CHECK-NEXT:    [[TMP4:%.*]] = ptrtoint ptr [[A]] to i64
+; CHECK-NEXT:    [[TMP5:%.*]] = xor i64 [[TMP4]], 193514046488576
+; CHECK-NEXT:    [[TMP6:%.*]] = inttoptr i64 [[TMP5]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <8 x i16>, ptr [[TMP6]], align 16
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <8 x i16> [[_MSLD]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP7]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP1]], label [[TMP8:%.*]], label [[TMP9:%.*]], !prof [[PROF1]]
+; CHECK:       8:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       9:
+; CHECK-NEXT:    [[TMP3:%.*]] = call <4 x i32> @llvm.aarch64.neon.uaddlp.v4i32.v8i16(<8 x i16> [[TMPVAR1]])
+; CHECK-NEXT:    store <4 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i32> [[TMP3]]
+;
+  %tmpvar1 = load <8 x i16>, ptr %A
+  %tmpvar3 = call <4 x i32> @llvm.aarch64.neon.uaddlp.v4i32.v8i16(<8 x i16> %tmpvar1)
+  ret <4 x i32> %tmpvar3
+}
+
+define <2 x i64> @uaddlp2d(ptr %A) nounwind #0 {
+; CHECK-LABEL: define <2 x i64> @uaddlp2d(
+; CHECK-SAME: ptr [[A:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP2:%.*]], label [[TMP10:%.*]], !prof [[PROF1]]
+; CHECK:       2:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       3:
+; CHECK-NEXT:    [[TMPVAR1:%.*]] = load <4 x i32>, ptr [[A]], align 16
+; CHECK-NEXT:    [[TMP4:%.*]] = ptrtoint ptr [[A]] to i64
+; CHECK-NEXT:    [[TMP5:%.*]] = xor i64 [[TMP4]], 193514046488576
+; CHECK-NEXT:    [[TMP6:%.*]] = inttoptr i64 [[TMP5]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <4 x i32>, ptr [[TMP6]], align 16
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <4 x i32> [[_MSLD]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP7]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP1]], label [[TMP8:%.*]], label [[TMP9:%.*]], !prof [[PROF1]]
+; CHECK:       8:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       9:
+; CHECK-NEXT:    [[TMP3:%.*]] = call <2 x i64> @llvm.aarch64.neon.uaddlp.v2i64.v4i32(<4 x i32> [[TMPVAR1]])
+; CHECK-NEXT:    store <2 x i64> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x i64> [[TMP3]]
+;
+  %tmpvar1 = load <4 x i32>, ptr %A
+  %tmpvar3 = call <2 x i64> @llvm.aarch64.neon.uaddlp.v2i64.v4i32(<4 x i32> %tmpvar1)
+  ret <2 x i64> %tmpvar3
+}
+
+declare <4 x i16>  @llvm.aarch64.neon.uaddlp.v4i16.v8i8(<8 x i8>) nounwind readnone
+declare <2 x i32> @llvm.aarch64.neon.uaddlp.v2i32.v4i16(<4 x i16>) nounwind readnone
+declare <1 x i64> @llvm.aarch64.neon.uaddlp.v1i64.v2i32(<2 x i32>) nounwind readnone
+
+declare <8 x i16>  @llvm.aarch64.neon.uaddlp.v8i16.v16i8(<16 x i8>) nounwind readnone
+declare <4 x i32> @llvm.aarch64.neon.uaddlp.v4i32.v8i16(<8 x i16>) nounwind readnone
+declare <2 x i64> @llvm.aarch64.neon.uaddlp.v2i64.v4i32(<4 x i32>) nounwind readnone
+
+define <4 x i16> @sadalp4h(ptr %A, ptr %B) nounwind #0 {
+; CHECK-LABEL: define <4 x i16> @sadalp4h(
+; CHECK-SAME: ptr [[A:%.*]], ptr [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP3:%.*]], label [[TMP16:%.*]], !prof [[PROF1]]
+; CHECK:       3:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       4:
+; CHECK-NEXT:    [[TMPVAR1:%.*]] = load <8 x i8>, ptr [[A]], align 8
+; CHECK-NEXT:    [[TMP5:%.*]] = ptrtoint ptr [[A]] to i64
+; CHECK-NEXT:    [[TMP6:%.*]] = xor i64 [[TMP5]], 193514046488576
+; CHECK-NEXT:    [[TMP7:%.*]] = inttoptr i64 [[TMP6]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <8 x i8>, ptr [[TMP7]], align 8
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <8 x i8> [[_MSLD]] to i64
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i64 [[TMP8]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP2]], label [[TMP9:%.*]], label [[TMP10:%.*]], !prof [[PROF1]]
+; CHECK:       9:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       10:
+; CHECK-NEXT:    [[TMPVAR3:%.*]] = call <4 x i16> @llvm.aarch64.neon.saddlp.v4i16.v8i8(<8 x i8> [[TMPVAR1]])
+; CHECK-NEXT:    [[_MSCMP3:%.*]] = icmp ne i64 [[TMP2]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP3]], label [[TMP11:%.*]], label [[TMP12:%.*]], !prof [[PROF1]]
+; CHECK:       11:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       12:
+; CHECK-NEXT:    [[TMP4:%.*]] = load <4 x i16>, ptr [[B]], align 8
+; CHECK-NEXT:    [[TMP13:%.*]] = ptrtoint ptr [[B]] to i64
+; CHECK-NEXT:    [[TMP14:%.*]] = xor i64 [[TMP13]], 193514046488576
+; CHECK-NEXT:    [[TMP15:%.*]] = inttoptr i64 [[TMP14]] to ptr
+; CHECK-NEXT:    [[_MSLD1:%.*]] = load <4 x i16>, ptr [[TMP15]], align 8
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <4 x i16> zeroinitializer, [[_MSLD1]]
+; CHECK-NEXT:    [[TMPVAR5:%.*]] = add <4 x i16> [[TMPVAR3]], [[TMP4]]
+; CHECK-NEXT:    store <4 x i16> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i16> [[TMPVAR5]]
+;
+  %tmpvar1 = load <8 x i8>, ptr %A
+  %tmpvar3 = call <4 x i16> @llvm.aarch64.neon.saddlp.v4i16.v8i8(<8 x i8> %tmpvar1)
+  %tmpvar4 = load <4 x i16>, ptr %B
+  %tmpvar5 = add <4 x i16> %tmpvar3, %tmpvar4
+  ret <4 x i16> %tmpvar5
+}
+
+define <2 x i32> @sadalp2s(ptr %A, ptr %B) nounwind #0 {
+; CHECK-LABEL: define <2 x i32> @sadalp2s(
+; CHECK-SAME: ptr [[A:%.*]], ptr [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP3:%.*]], label [[TMP16:%.*]], !prof [[PROF1]]
+; CHECK:       3:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       4:
+; CHECK-NEXT:    [[TMPVAR1:%.*]] = load <4 x i16>, ptr [[A]], align 8
+; CHECK-NEXT:    [[TMP5:%.*]] = ptrtoint ptr [[A]] to i64
+; CHECK-NEXT:    [[TMP6:%.*]] = xor i64 [[TMP5]], 193514046488576
+; CHECK-NEXT:    [[TMP7:%.*]] = inttoptr i64 [[TMP6]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <4 x i16>, ptr [[TMP7]], align 8
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <4 x i16> [[_MSLD]] to i64
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i64 [[TMP8]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP2]], label [[TMP9:%.*]], label [[TMP10:%.*]], !prof [[PROF1]]
+; CHECK:       9:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       10:
+; CHECK-NEXT:    [[TMPVAR3:%.*]] = call <2 x i32> @llvm.aarch64.neon.saddlp.v2i32.v4i16(<4 x i16> [[TMPVAR1]])
+; CHECK-NEXT:    [[_MSCMP3:%.*]] = icmp ne i64 [[TMP2]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP3]], label [[TMP11:%.*]], label [[TMP12:%.*]], !prof [[PROF1]]
+; CHECK:       11:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       12:
+; CHECK-NEXT:    [[TMP4:%.*]] = load <2 x i32>, ptr [[B]], align 8
+; CHECK-NEXT:    [[TMP13:%.*]] = ptrtoint ptr [[B]] to i64
+; CHECK-NEXT:    [[TMP14:%.*]] = xor i64 [[TMP13]], 193514046488576
+; CHECK-NEXT:    [[TMP15:%.*]] = inttoptr i64 [[TMP14]] to ptr
+; CHECK-NEXT:    [[_MSLD1:%.*]] = load <2 x i32>, ptr [[TMP15]], align 8
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <2 x i32> zeroinitializer, [[_MSLD1]]
+; CHECK-NEXT:    [[TMPVAR5:%.*]] = add <2 x i32> [[TMPVAR3]], [[TMP4]]
+; CHECK-NEXT:    store <2 x i32> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x i32> [[TMPVAR5]]
+;
+  %tmpvar1 = load <4 x i16>, ptr %A
+  %tmpvar3 = call <2 x i32> @llvm.aarch64.neon.saddlp.v2i32.v4i16(<4 x i16> %tmpvar1)
+  %tmpvar4 = load <2 x i32>, ptr %B
+  %tmpvar5 = add <2 x i32> %tmpvar3, %tmpvar4
+  ret <2 x i32> %tmpvar5
+}
+
+define <8 x i16> @sadalp8h(ptr %A, ptr %B) nounwind #0 {
+; CHECK-LABEL: define <8 x i16> @sadalp8h(
+; CHECK-SAME: ptr [[A:%.*]], ptr [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP3:%.*]], label [[TMP16:%.*]], !prof [[PROF1]]
+; CHECK:       3:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       4:
+; CHECK-NEXT:    [[TMPVAR1:%.*]] = load <16 x i8>, ptr [[A]], align 16
+; CHECK-NEXT:    [[TMP5:%.*]] = ptrtoint ptr [[A]] to i64
+; CHECK-NEXT:    [[TMP6:%.*]] = xor i64 [[TMP5]], 193514046488576
+; CHECK-NEXT:    [[TMP7:%.*]] = inttoptr i64 [[TMP6]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <16 x i8>, ptr [[TMP7]], align 16
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <16 x i8> [[_MSLD]] to i128
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i128 [[TMP8]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP2]], label [[TMP9:%.*]], label [[TMP10:%.*]], !prof [[PROF1]]
+; CHECK:       9:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       10:
+; CHECK-NEXT:    [[TMPVAR3:%.*]] = call <8 x i16> @llvm.aarch64.neon.saddlp.v8i16.v16i8(<16 x i8> [[TMPVAR1]])
+; CHECK-NEXT:    [[_MSCMP3:%.*]] = icmp ne i64 [[TMP2]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP3]], label [[TMP11:%.*]], label [[TMP12:%.*]], !prof [[PROF1]]
+; CHECK:       11:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       12:
+; CHECK-NEXT:    [[TMP4:%.*]] = load <8 x i16>, ptr [[B]], align 16
+; CHECK-NEXT:    [[TMP13:%.*]] = ptrtoint ptr [[B]] to i64
+; CHECK-NEXT:    [[TMP14:%.*]] = xor i64 [[TMP13]], 193514046488576
+; CHECK-NEXT:    [[TMP15:%.*]] = inttoptr i64 [[TMP14]] to ptr
+; CHECK-NEXT:    [[_MSLD1:%.*]] = load <8 x i16>, ptr [[TMP15]], align 16
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <8 x i16> zeroinitializer, [[_MSLD1]]
+; CHECK-NEXT:    [[TMPVAR5:%.*]] = add <8 x i16> [[TMPVAR3]], [[TMP4]]
+; CHECK-NEXT:    store <8 x i16> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i16> [[TMPVAR5]]
+;
+  %tmpvar1 = load <16 x i8>, ptr %A
+  %tmpvar3 = call <8 x i16> @llvm.aarch64.neon.saddlp.v8i16.v16i8(<16 x i8> %tmpvar1)
+  %tmpvar4 = load <8 x i16>, ptr %B
+  %tmpvar5 = add <8 x i16> %tmpvar3, %tmpvar4
+  ret <8 x i16> %tmpvar5
+}
+
+define <4 x i32> @sadalp4s(ptr %A, ptr %B) nounwind #0 {
+; CHECK-LABEL: define <4 x i32> @sadalp4s(
+; CHECK-SAME: ptr [[A:%.*]], ptr [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP3:%.*]], label [[TMP16:%.*]], !prof [[PROF1]]
+; CHECK:       3:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       4:
+; CHECK-NEXT:    [[TMPVAR1:%.*]] = load <8 x i16>, ptr [[A]], align 16
+; CHECK-NEXT:    [[TMP5:%.*]] = ptrtoint ptr [[A]] to i64
+; CHECK-NEXT:    [[TMP6:%.*]] = xor i64 [[TMP5]], 193514046488576
+; CHECK-NEXT:    [[TMP7:%.*]] = inttoptr i64 [[TMP6]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <8 x i16>, ptr [[TMP7]], align 16
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <8 x i16> [[_MSLD]] to i128
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i128 [[TMP8]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP2]], label [[TMP9:%.*]], label [[TMP10:%.*]], !prof [[PROF1]]
+; CHECK:       9:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       10:
+; CHECK-NEXT:    [[TMPVAR3:%.*]] = call <4 x i32> @llvm.aarch64.neon.saddlp.v4i32.v8i16(<8 x i16> [[TMPVAR1]])
+; CHECK-NEXT:    [[_MSCMP3:%.*]] = icmp ne i64 [[TMP2]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP3]], label [[TMP11:%.*]], label [[TMP12:%.*]], !prof [[PROF1]]
+; CHECK:       11:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       12:
+; CHECK-NEXT:    [[TMP4:%.*]] = load <4 x i32>, ptr [[B]], align 16
+; CHECK-NEXT:    [[TMP13:%.*]] = ptrtoint ptr [[B]] to i64
+; CHECK-NEXT:    [[TMP14:%.*]] = xor i64 [[TMP13]], 193514046488576
+; CHECK-NEXT:    [[TMP15:%.*]] = inttoptr i64 [[TMP14]] to ptr
+; CHECK-NEXT:    [[_MSLD1:%.*]] = load <4 x i32>, ptr [[TMP15]], align 16
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <4 x i32> zeroinitializer, [[_MSLD1]]
+; CHECK-NEXT:    [[TMPVAR5:%.*]] = add <4 x i32> [[TMPVAR3]], [[TMP4]]
+; CHECK-NEXT:    store <4 x i32> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i32> [[TMPVAR5]]
+;
+  %tmpvar1 = load <8 x i16>, ptr %A
+  %tmpvar3 = call <4 x i32> @llvm.aarch64.neon.saddlp.v4i32.v8i16(<8 x i16> %tmpvar1)
+  %tmpvar4 = load <4 x i32>, ptr %B
+  %tmpvar5 = add <4 x i32> %tmpvar3, %tmpvar4
+  ret <4 x i32> %tmpvar5
+}
+
+define <2 x i64> @sadalp2d(ptr %A, ptr %B) nounwind #0 {
+; CHECK-LABEL: define <2 x i64> @sadalp2d(
+; CHECK-SAME: ptr [[A:%.*]], ptr [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP3:%.*]], label [[TMP16:%.*]], !prof [[PROF1]]
+; CHECK:       3:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       4:
+; CHECK-NEXT:    [[TMPVAR1:%.*]] = load <4 x i32>, ptr [[A]], align 16
+; CHECK-NEXT:    [[TMP5:%.*]] = ptrtoint ptr [[A]] to i64
+; CHECK-NEXT:    [[TMP6:%.*]] = xor i64 [[TMP5]], 193514046488576
+; CHECK-NEXT:    [[TMP7:%.*]] = inttoptr i64 [[TMP6]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <4 x i32>, ptr [[TMP7]], align 16
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <4 x i32> [[_MSLD]] to i128
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i128 [[TMP8]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP2]], label [[TMP9:%.*]], label [[TMP10:%.*]], !prof [[PROF1]]
+; CHECK:       9:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       10:
+; CHECK-NEXT:    [[TMPVAR3:%.*]] = call <2 x i64> @llvm.aarch64.neon.saddlp.v2i64.v4i32(<4 x i32> [[TMPVAR1]])
+; CHECK-NEXT:    [[_MSCMP3:%.*]] = icmp ne i64 [[TMP2]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP3]], label [[TMP11:%.*]], label [[TMP12:%.*]], !prof [[PROF1]]
+; CHECK:       11:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       12:
+; CHECK-NEXT:    [[TMP4:%.*]] = load <2 x i64>, ptr [[B]], align 16
+; CHECK-NEXT:    [[TMP13:%.*]] = ptrtoint ptr [[B]] to i64
+; CHECK-NEXT:    [[TMP14:%.*]] = xor i64 [[TMP13]], 193514046488576
+; CHECK-NEXT:    [[TMP15:%.*]] = inttoptr i64 [[TMP14]] to ptr
+; CHECK-NEXT:    [[_MSLD1:%.*]] = load <2 x i64>, ptr [[TMP15]], align 16
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <2 x i64> zeroinitializer, [[_MSLD1]]
+; CHECK-NEXT:    [[TMPVAR5:%.*]] = add <2 x i64> [[TMPVAR3]], [[TMP4]]
+; CHECK-NEXT:    store <2 x i64> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x i64> [[TMPVAR5]]
+;
+  %tmpvar1 = load <4 x i32>, ptr %A
+  %tmpvar3 = call <2 x i64> @llvm.aarch64.neon.saddlp.v2i64.v4i32(<4 x i32> %tmpvar1)
+  %tmpvar4 = load <2 x i64>, ptr %B
+  %tmpvar5 = add <2 x i64> %tmpvar3, %tmpvar4
+  ret <2 x i64> %tmpvar5
+}
+
+define <4 x i16> @uadalp4h(ptr %A, ptr %B) nounwind #0 {
+; CHECK-LABEL: define <4 x i16> @uadalp4h(
+; CHECK-SAME: ptr [[A:%.*]], ptr [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP3:%.*]], label [[TMP16:%.*]], !prof [[PROF1]]
+; CHECK:       3:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       4:
+; CHECK-NEXT:    [[TMPVAR1:%.*]] = load <8 x i8>, ptr [[A]], align 8
+; CHECK-NEXT:    [[TMP5:%.*]] = ptrtoint ptr [[A]] to i64
+; CHECK-NEXT:    [[TMP6:%.*]] = xor i64 [[TMP5]], 193514046488576
+; CHECK-NEXT:    [[TMP7:%.*]] = inttoptr i64 [[TMP6]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <8 x i8>, ptr [[TMP7]], align 8
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <8 x i8> [[_MSLD]] to i64
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i64 [[TMP8]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP2]], label [[TMP9:%.*]], label [[TMP10:%.*]], !prof [[PROF1]]
+; CHECK:       9:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       10:
+; CHECK-NEXT:    [[TMPVAR3:%.*]] = call <4 x i16> @llvm.aarch64.neon.uaddlp.v4i16.v8i8(<8 x i8> [[TMPVAR1]])
+; CHECK-NEXT:    [[_MSCMP3:%.*]] = icmp ne i64 [[TMP2]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP3]], label [[TMP11:%.*]], label [[TMP12:%.*]], !prof [[PROF1]]
+; CHECK:       11:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       12:
+; CHECK-NEXT:    [[TMP4:%.*]] = load <4 x i16>, ptr [[B]], align 8
+; CHECK-NEXT:    [[TMP13:%.*]] = ptrtoint ptr [[B]] to i64
+; CHECK-NEXT:    [[TMP14:%.*]] = xor i64 [[TMP13]], 193514046488576
+; CHECK-NEXT:    [[TMP15:%.*]] = inttoptr i64 [[TMP14]] to ptr
+; CHECK-NEXT:    [[_MSLD1:%.*]] = load <4 x i16>, ptr [[TMP15]], align 8
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <4 x i16> zeroinitializer, [[_MSLD1]]
+; CHECK-NEXT:    [[TMPVAR5:%.*]] = add <4 x i16> [[TMPVAR3]], [[TMP4]]
+; CHECK-NEXT:    store <4 x i16> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i16> [[TMPVAR5]]
+;
+  %tmpvar1 = load <8 x i8>, ptr %A
+  %tmpvar3 = call <4 x i16> @llvm.aarch64.neon.uaddlp.v4i16.v8i8(<8 x i8> %tmpvar1)
+  %tmpvar4 = load <4 x i16>, ptr %B
+  %tmpvar5 = add <4 x i16> %tmpvar3, %tmpvar4
+  ret <4 x i16> %tmpvar5
+}
+
+define <2 x i32> @uadalp2s(ptr %A, ptr %B) nounwind #0 {
+; CHECK-LABEL: define <2 x i32> @uadalp2s(
+; CHECK-SAME: ptr [[A:%.*]], ptr [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP3:%.*]], label [[TMP16:%.*]], !prof [[PROF1]]
+; CHECK:       3:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       4:
+; CHECK-NEXT:    [[TMPVAR1:%.*]] = load <4 x i16>, ptr [[A]], align 8
+; CHECK-NEXT:    [[TMP5:%.*]] = ptrtoint ptr [[A]] to i64
+; CHECK-NEXT:    [[TMP6:%.*]] = xor i64 [[TMP5]], 193514046488576
+; CHECK-NEXT:    [[TMP7:%.*]] = inttoptr i64 [[TMP6]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <4 x i16>, ptr [[TMP7]], align 8
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <4 x i16> [[_MSLD]] to i64
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i64 [[TMP8]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP2]], label [[TMP9:%.*]], label [[TMP10:%.*]], !prof [[PROF1]]
+; CHECK:       9:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       10:
+; CHECK-NEXT:    [[TMPVAR3:%.*]] = call <2 x i32> @llvm.aarch64.neon.uaddlp.v2i32.v4i16(<4 x i16> [[TMPVAR1]])
+; CHECK-NEXT:    [[_MSCMP3:%.*]] = icmp ne i64 [[TMP2]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP3]], label [[TMP11:%.*]], label [[TMP12:%.*]], !prof [[PROF1]]
+; CHECK:       11:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       12:
+; CHECK-NEXT:    [[TMP4:%.*]] = load <2 x i32>, ptr [[B]], align 8
+; CHECK-NEXT:    [[TMP13:%.*]] = ptrtoint ptr [[B]] to i64
+; CHECK-NEXT:    [[TMP14:%.*]] = xor i64 [[TMP13]], 193514046488576
+; CHECK-NEXT:    [[TMP15:%.*]] = inttoptr i64 [[TMP14]] to ptr
+; CHECK-NEXT:    [[_MSLD1:%.*]] = load <2 x i32>, ptr [[TMP15]], align 8
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <2 x i32> zeroinitializer, [[_MSLD1]]
+; CHECK-NEXT:    [[TMPVAR5:%.*]] = add <2 x i32> [[TMPVAR3]], [[TMP4]]
+; CHECK-NEXT:    store <2 x i32> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x i32> [[TMPVAR5]]
+;
+  %tmpvar1 = load <4 x i16>, ptr %A
+  %tmpvar3 = call <2 x i32> @llvm.aarch64.neon.uaddlp.v2i32.v4i16(<4 x i16> %tmpvar1)
+  %tmpvar4 = load <2 x i32>, ptr %B
+  %tmpvar5 = add <2 x i32> %tmpvar3, %tmpvar4
+  ret <2 x i32> %tmpvar5
+}
+
+define <8 x i16> @uadalp8h(ptr %A, ptr %B) nounwind #0 {
+; CHECK-LABEL: define <8 x i16> @uadalp8h(
+; CHECK-SAME: ptr [[A:%.*]], ptr [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP3:%.*]], label [[TMP16:%.*]], !prof [[PROF1]]
+; CHECK:       3:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       4:
+; CHECK-NEXT:    [[TMPVAR1:%.*]] = load <16 x i8>, ptr [[A]], align 16
+; CHECK-NEXT:    [[TMP5:%.*]] = ptrtoint ptr [[A]] to i64
+; CHECK-NEXT:    [[TMP6:%.*]] = xor i64 [[TMP5]], 193514046488576
+; CHECK-NEXT:    [[TMP7:%.*]] = inttoptr i64 [[TMP6]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <16 x i8>, ptr [[TMP7]], align 16
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <16 x i8> [[_MSLD]] to i128
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i128 [[TMP8]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP2]], label [[TMP9:%.*]], label [[TMP10:%.*]], !prof [[PROF1]]
+; CHECK:       9:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       10:
+; CHECK-NEXT:    [[TMPVAR3:%.*]] = call <8 x i16> @llvm.aarch64.neon.uaddlp.v8i16.v16i8(<16 x i8> [[TMPVAR1]])
+; CHECK-NEXT:    [[_MSCMP3:%.*]] = icmp ne i64 [[TMP2]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP3]], label [[TMP11:%.*]], label [[TMP12:%.*]], !prof [[PROF1]]
+; CHECK:       11:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       12:
+; CHECK-NEXT:    [[TMP4:%.*]] = load <8 x i16>, ptr [[B]], align 16
+; CHECK-NEXT:    [[TMP13:%.*]] = ptrtoint ptr [[B]] to i64
+; CHECK-NEXT:    [[TMP14:%.*]] = xor i64 [[TMP13]], 193514046488576
+; CHECK-NEXT:    [[TMP15:%.*]] = inttoptr i64 [[TMP14]] to ptr
+; CHECK-NEXT:    [[_MSLD1:%.*]] = load <8 x i16>, ptr [[TMP15]], align 16
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <8 x i16> zeroinitializer, [[_MSLD1]]
+; CHECK-NEXT:    [[TMPVAR5:%.*]] = add <8 x i16> [[TMPVAR3]], [[TMP4]]
+; CHECK-NEXT:    store <8 x i16> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i16> [[TMPVAR5]]
+;
+  %tmpvar1 = load <16 x i8>, ptr %A
+  %tmpvar3 = call <8 x i16> @llvm.aarch64.neon.uaddlp.v8i16.v16i8(<16 x i8> %tmpvar1)
+  %tmpvar4 = load <8 x i16>, ptr %B
+  %tmpvar5 = add <8 x i16> %tmpvar3, %tmpvar4
+  ret <8 x i16> %tmpvar5
+}
+
+define <4 x i32> @uadalp4s(ptr %A, ptr %B) nounwind #0 {
+; CHECK-LABEL: define <4 x i32> @uadalp4s(
+; CHECK-SAME: ptr [[A:%.*]], ptr [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP3:%.*]], label [[TMP16:%.*]], !prof [[PROF1]]
+; CHECK:       3:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       4:
+; CHECK-NEXT:    [[TMPVAR1:%.*]] = load <8 x i16>, ptr [[A]], align 16
+; CHECK-NEXT:    [[TMP5:%.*]] = ptrtoint ptr [[A]] to i64
+; CHECK-NEXT:    [[TMP6:%.*]] = xor i64 [[TMP5]], 193514046488576
+; CHECK-NEXT:    [[TMP7:%.*]] = inttoptr i64 [[TMP6]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <8 x i16>, ptr [[TMP7]], align 16
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <8 x i16> [[_MSLD]] to i128
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i128 [[TMP8]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP2]], label [[TMP9:%.*]], label [[TMP10:%.*]], !prof [[PROF1]]
+; CHECK:       9:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       10:
+; CHECK-NEXT:    [[TMPVAR3:%.*]] = call <4 x i32> @llvm.aarch64.neon.uaddlp.v4i32.v8i16(<8 x i16> [[TMPVAR1]])
+; CHECK-NEXT:    [[_MSCMP3:%.*]] = icmp ne i64 [[TMP2]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP3]], label [[TMP11:%.*]], label [[TMP12:%.*]], !prof [[PROF1]]
+; CHECK:       11:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       12:
+; CHECK-NEXT:    [[TMP4:%.*]] = load <4 x i32>, ptr [[B]], align 16
+; CHECK-NEXT:    [[TMP13:%.*]] = ptrtoint ptr [[B]] to i64
+; CHECK-NEXT:    [[TMP14:%.*]] = xor i64 [[TMP13]], 193514046488576
+; CHECK-NEXT:    [[TMP15:%.*]] = inttoptr i64 [[TMP14]] to ptr
+; CHECK-NEXT:    [[_MSLD1:%.*]] = load <4 x i32>, ptr [[TMP15]], align 16
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <4 x i32> zeroinitializer, [[_MSLD1]]
+; CHECK-NEXT:    [[TMPVAR5:%.*]] = add <4 x i32> [[TMPVAR3]], [[TMP4]]
+; CHECK-NEXT:    store <4 x i32> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i32> [[TMPVAR5]]
+;
+  %tmpvar1 = load <8 x i16>, ptr %A
+  %tmpvar3 = call <4 x i32> @llvm.aarch64.neon.uaddlp.v4i32.v8i16(<8 x i16> %tmpvar1)
+  %tmpvar4 = load <4 x i32>, ptr %B
+  %tmpvar5 = add <4 x i32> %tmpvar3, %tmpvar4
+  ret <4 x i32> %tmpvar5
+}
+
+define <2 x i64> @uadalp2d(ptr %A, ptr %B) nounwind #0 {
+; CHECK-LABEL: define <2 x i64> @uadalp2d(
+; CHECK-SAME: ptr [[A:%.*]], ptr [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP3:%.*]], label [[TMP16:%.*]], !prof [[PROF1]]
+; CHECK:       3:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       4:
+; CHECK-NEXT:    [[TMPVAR1:%.*]] = load <4 x i32>, ptr [[A]], align 16
+; CHECK-NEXT:    [[TMP5:%.*]] = ptrtoint ptr [[A]] to i64
+; CHECK-NEXT:    [[TMP6:%.*]] = xor i64 [[TMP5]], 193514046488576
+; CHECK-NEXT:    [[TMP7:%.*]] = inttoptr i64 [[TMP6]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <4 x i32>, ptr [[TMP7]], align 16
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <4 x i32> [[_MSLD]] to i128
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i128 [[TMP8]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP2]], label [[TMP9:%.*]], label [[TMP10:%.*]], !prof [[PROF1]]
+; CHECK:       9:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       10:
+; CHECK-NEXT:    [[TMPVAR3:%.*]] = call <2 x i64> @llvm.aarch64.neon.uaddlp.v2i64.v4i32(<4 x i32> [[TMPVAR1]])
+; CHECK-NEXT:    [[_MSCMP3:%.*]] = icmp ne i64 [[TMP2]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP3]], label [[TMP11:%.*]], label [[TMP12:%.*]], !prof [[PROF1]]
+; CHECK:       11:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       12:
+; CHECK-NEXT:    [[TMP4:%.*]] = load <2 x i64>, ptr [[B]], align 16
+; CHECK-NEXT:    [[TMP13:%.*]] = ptrtoint ptr [[B]] to i64
+; CHECK-NEXT:    [[TMP14:%.*]] = xor i64 [[TMP13]], 193514046488576
+; CHECK-NEXT:    [[TMP15:%.*]] = inttoptr i64 [[TMP14]] to ptr
+; CHECK-NEXT:    [[_MSLD1:%.*]] = load <2 x i64>, ptr [[TMP15]], align 16
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <2 x i64> zeroinitializer, [[_MSLD1]]
+; CHECK-NEXT:    [[TMPVAR5:%.*]] = add <2 x i64> [[TMPVAR3]], [[TMP4]]
+; CHECK-NEXT:    store <2 x i64> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x i64> [[TMPVAR5]]
+;
+  %tmpvar1 = load <4 x i32>, ptr %A
+  %tmpvar3 = call <2 x i64> @llvm.aarch64.neon.uaddlp.v2i64.v4i32(<4 x i32> %tmpvar1)
+  %tmpvar4 = load <2 x i64>, ptr %B
+  %tmpvar5 = add <2 x i64> %tmpvar3, %tmpvar4
+  ret <2 x i64> %tmpvar5
+}
+
+define <8 x i8> @addp_8b(ptr %A, ptr %B) nounwind #0 {
+; CHECK-LABEL: define <8 x i8> @addp_8b(
+; CHECK-SAME: ptr [[A:%.*]], ptr [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP13:%.*]], label [[TMP4:%.*]], !prof [[PROF1]]
+; CHECK:       3:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       4:
+; CHECK-NEXT:    [[TMPVAR1:%.*]] = load <8 x i8>, ptr [[A]], align 8
+; CHECK-NEXT:    [[TMP5:%.*]] = ptrtoint ptr [[A]] to i64
+; CHECK-NEXT:    [[TMP6:%.*]] = xor i64 [[TMP5]], 193514046488576
+; CHECK-NEXT:    [[TMP7:%.*]] = inttoptr i64 [[TMP6]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <8 x i8>, ptr [[TMP7]], align 8
+; CHECK-NEXT:    [[_MSCMP3:%.*]] = icmp ne i64 [[TMP2]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP3]], label [[TMP8:%.*]], label [[TMP9:%.*]], !prof [[PROF1]]
+; CHECK:       8:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       9:
+; CHECK-NEXT:    [[TMPVAR2:%.*]] = load <8 x i8>, ptr [[B]], align 8
+; CHECK-NEXT:    [[TMP10:%.*]] = ptrtoint ptr [[B]] to i64
+; CHECK-NEXT:    [[TMP11:%.*]] = xor i64 [[TMP10]], 193514046488576
+; CHECK-NEXT:    [[TMP12:%.*]] = inttoptr i64 [[TMP11]] to ptr
+; CHECK-NEXT:    [[_MSLD1:%.*]] = load <8 x i8>, ptr [[TMP12]], align 8
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <8 x i8> [[_MSLD]], [[_MSLD1]]
+; CHECK-NEXT:    [[TMP3:%.*]] = call <8 x i8> @llvm.aarch64.neon.addp.v8i8(<8 x i8> [[TMPVAR1]], <8 x i8> [[TMPVAR2]])
+; CHECK-NEXT:    store <8 x i8> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i8> [[TMP3]]
+;
+  %tmpvar1 = load <8 x i8>, ptr %A
+  %tmpvar2 = load <8 x i8>, ptr %B
+  %tmpvar3 = call <8 x i8> @llvm.aarch64.neon.addp.v8i8(<8 x i8> %tmpvar1, <8 x i8> %tmpvar2)
+  ret <8 x i8> %tmpvar3
+}
+
+define <16 x i8> @addp_16b(ptr %A, ptr %B) nounwind #0 {
+; CHECK-LABEL: define <16 x i8> @addp_16b(
+; CHECK-SAME: ptr [[A:%.*]], ptr [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP13:%.*]], label [[TMP4:%.*]], !prof [[PROF1]]
+; CHECK:       3:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       4:
+; CHECK-NEXT:    [[TMPVAR1:%.*]] = load <16 x i8>, ptr [[A]], align 16
+; CHECK-NEXT:    [[TMP5:%.*]] = ptrtoint ptr [[A]] to i64
+; CHECK-NEXT:    [[TMP6:%.*]] = xor i64 [[TMP5]], 193514046488576
+; CHECK-NEXT:    [[TMP7:%.*]] = inttoptr i64 [[TMP6]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <16 x i8>, ptr [[TMP7]], align 16
+; CHECK-NEXT:    [[_MSCMP3:%.*]] = icmp ne i64 [[TMP2]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP3]], label [[TMP8:%.*]], label [[TMP9:%.*]], !prof [[PROF1]]
+; CHECK:       8:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       9:
+; CHECK-NEXT:    [[TMPVAR2:%.*]] = load <16 x i8>, ptr [[B]], align 16
+; CHECK-NEXT:    [[TMP10:%.*]] = ptrtoint ptr [[B]] to i64
+; CHECK-NEXT:    [[TMP11:%.*]] = xor i64 [[TMP10]], 193514046488576
+; CHECK-NEXT:    [[TMP12:%.*]] = inttoptr i64 [[TMP11]] to ptr
+; CHECK-NEXT:    [[_MSLD1:%.*]] = load <16 x i8>, ptr [[TMP12]], align 16
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <16 x i8> [[_MSLD]], [[_MSLD1]]
+; CHECK-NEXT:    [[TMP3:%.*]] = call <16 x i8> @llvm.aarch64.neon.addp.v16i8(<16 x i8> [[TMPVAR1]], <16 x i8> [[TMPVAR2]])
+; CHECK-NEXT:    store <16 x i8> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i8> [[TMP3]]
+;
+  %tmpvar1 = load <16 x i8>, ptr %A
+  %tmpvar2 = load <16 x i8>, ptr %B
+  %tmpvar3 = call <16 x i8> @llvm.aarch64.neon.addp.v16i8(<16 x i8> %tmpvar1, <16 x i8> %tmpvar2)
+  ret <16 x i8> %tmpvar3
+}
+
+define <4 x i16> @addp_4h(ptr %A, ptr %B) nounwind #0 {
+; CHECK-LABEL: define <4 x i16> @addp_4h(
+; CHECK-SAME: ptr [[A:%.*]], ptr [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP13:%.*]], label [[TMP4:%.*]], !prof [[PROF1]]
+; CHECK:       3:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       4:
+; CHECK-NEXT:    [[TMPVAR1:%.*]] = load <4 x i16>, ptr [[A]], align 8
+; CHECK-NEXT:    [[TMP5:%.*]] = ptrtoint ptr [[A]] to i64
+; CHECK-NEXT:    [[TMP6:%.*]] = xor i64 [[TMP5]], 193514046488576
+; CHECK-NEXT:    [[TMP7:%.*]] = inttoptr i64 [[TMP6]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <4 x i16>, ptr [[TMP7]], align 8
+; CHECK-NEXT:    [[_MSCMP3:%.*]] = icmp ne i64 [[TMP2]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP3]], label [[TMP8:%.*]], label [[TMP9:%.*]], !prof [[PROF1]]
+; CHECK:       8:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       9:
+; CHECK-NEXT:    [[TMPVAR2:%.*]] = load <4 x i16>, ptr [[B]], align 8
+; CHECK-NEXT:    [[TMP10:%.*]] = ptrtoint ptr [[B]] to i64
+; CHECK-NEXT:    [[TMP11:%.*]] = xor i64 [[TMP10]], 193514046488576
+; CHECK-NEXT:    [[TMP12:%.*]] = inttoptr i64 [[TMP11]] to ptr
+; CHECK-NEXT:    [[_MSLD1:%.*]] = load <4 x i16>, ptr [[TMP12]], align 8
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <4 x i16> [[_MSLD]], [[_MSLD1]]
+; CHECK-NEXT:    [[TMP3:%.*]] = call <4 x i16> @llvm.aarch64.neon.addp.v4i16(<4 x i16> [[TMPVAR1]], <4 x i16> [[TMPVAR2]])
+; CHECK-NEXT:    store <4 x i16> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i16> [[TMP3]]
+;
+  %tmpvar1 = load <4 x i16>, ptr %A
+  %tmpvar2 = load <4 x i16>, ptr %B
+  %tmpvar3 = call <4 x i16> @llvm.aarch64.neon.addp.v4i16(<4 x i16> %tmpvar1, <4 x i16> %tmpvar2)
+  ret <4 x i16> %tmpvar3
+}
+
+define <8 x i16> @addp_8h(ptr %A, ptr %B) nounwind #0 {
+; CHECK-LABEL: define <8 x i16> @addp_8h(
+; CHECK-SAME: ptr [[A:%.*]], ptr [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP13:%.*]], label [[TMP4:%.*]], !prof [[PROF1]]
+; CHECK:       3:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       4:
+; CHECK-NEXT:    [[TMPVAR1:%.*]] = load <8 x i16>, ptr [[A]], align 16
+; CHECK-NEXT:    [[TMP5:%.*]] = ptrtoint ptr [[A]] to i64
+; CHECK-NEXT:    [[TMP6:%.*]] = xor i64 [[TMP5]], 193514046488576
+; CHECK-NEXT:    [[TMP7:%.*]] = inttoptr i64 [[TMP6]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <8 x i16>, ptr [[TMP7]], align 16
+; CHECK-NEXT:    [[_MSCMP3:%.*]] = icmp ne i64 [[TMP2]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP3]], label [[TMP8:%.*]], label [[TMP9:%.*]], !prof [[PROF1]]
+; CHECK:       8:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       9:
+; CHECK-NEXT:    [[TMPVAR2:%.*]] = load <8 x i16>, ptr [[B]], align 16
+; CHECK-NEXT:    [[TMP10:%.*]] = ptrtoint ptr [[B]] to i64
+; CHECK-NEXT:    [[TMP11:%.*]] = xor i64 [[TMP10]], 193514046488576
+; CHECK-NEXT:    [[TMP12:%.*]] = inttoptr i64 [[TMP11]] to ptr
+; CHECK-NEXT:    [[_MSLD1:%.*]] = load <8 x i16>, ptr [[TMP12]], align 16
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <8 x i16> [[_MSLD]], [[_MSLD1]]
+; CHECK-NEXT:    [[TMP3:%.*]] = call <8 x i16> @llvm.aarch64.neon.addp.v8i16(<8 x i16> [[TMPVAR1]], <8 x i16> [[TMPVAR2]])
+; CHECK-NEXT:    store <8 x i16> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i16> [[TMP3]]
+;
+  %tmpvar1 = load <8 x i16>, ptr %A
+  %tmpvar2 = load <8 x i16>, ptr %B
+  %tmpvar3 = call <8 x i16> @llvm.aarch64.neon.addp.v8i16(<8 x i16> %tmpvar1, <8 x i16> %tmpvar2)
+  ret <8 x i16> %tmpvar3
+}
+
+define <2 x i32> @addp_2s(ptr %A, ptr %B) nounwind #0 {
+; CHECK-LABEL: define <2 x i32> @addp_2s(
+; CHECK-SAME: ptr [[A:%.*]], ptr [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP13:%.*]], label [[TMP4:%.*]], !prof [[PROF1]]
+; CHECK:       3:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       4:
+; CHECK-NEXT:    [[TMPVAR1:%.*]] = load <2 x i32>, ptr [[A]], align 8
+; CHECK-NEXT:    [[TMP5:%.*]] = ptrtoint ptr [[A]] to i64
+; CHECK-NEXT:    [[TMP6:%.*]] = xor i64 [[TMP5]], 193514046488576
+; CHECK-NEXT:    [[TMP7:%.*]] = inttoptr i64 [[TMP6]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <2 x i32>, ptr [[TMP7]], align 8
+; CHECK-NEXT:    [[_MSCMP3:%.*]] = icmp ne i64 [[TMP2]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP3]], label [[TMP8:%.*]], label [[TMP9:%.*]], !prof [[PROF1]]
+; CHECK:       8:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       9:
+; CHECK-NEXT:    [[TMPVAR2:%.*]] = load <2 x i32>, ptr [[B]], align 8
+; CHECK-NEXT:    [[TMP10:%.*]] = ptrtoint ptr [[B]] to i64
+; CHECK-NEXT:    [[TMP11:%.*]] = xor i64 [[TMP10]], 193514046488576
+; CHECK-NEXT:    [[TMP12:%.*]] = inttoptr i64 [[TMP11]] to ptr
+; CHECK-NEXT:    [[_MSLD1:%.*]] = load <2 x i32>, ptr [[TMP12]], align 8
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <2 x i32> [[_MSLD]], [[_MSLD1]]
+; CHECK-NEXT:    [[TMP3:%.*]] = call <2 x i32> @llvm.aarch64.neon.addp.v2i32(<2 x i32> [[TMPVAR1]], <2 x i32> [[TMPVAR2]])
+; CHECK-NEXT:    store <2 x i32> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x i32> [[TMP3]]
+;
+  %tmpvar1 = load <2 x i32>, ptr %A
+  %tmpvar2 = load <2 x i32>, ptr %B
+  %tmpvar3 = call <2 x i32> @llvm.aarch64.neon.addp.v2i32(<2 x i32> %tmpvar1, <2 x i32> %tmpvar2)
+  ret <2 x i32> %tmpvar3
+}
+
+define <4 x i32> @addp_4s(ptr %A, ptr %B) nounwind #0 {
+; CHECK-LABEL: define <4 x i32> @addp_4s(
+; CHECK-SAME: ptr [[A:%.*]], ptr [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP13:%.*]], label [[TMP4:%.*]], !prof [[PROF1]]
+; CHECK:       3:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       4:
+; CHECK-NEXT:    [[TMPVAR1:%.*]] = load <4 x i32>, ptr [[A]], align 16
+; CHECK-NEXT:    [[TMP5:%.*]] = ptrtoint ptr [[A]] to i64
+; CHECK-NEXT:    [[TMP6:%.*]] = xor i64 [[TMP5]], 193514046488576
+; CHECK-NEXT:    [[TMP7:%.*]] = inttoptr i64 [[TMP6]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <4 x i32>, ptr [[TMP7]], align 16
+; CHECK-NEXT:    [[_MSCMP3:%.*]] = icmp ne i64 [[TMP2]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP3]], label [[TMP8:%.*]], label [[TMP9:%.*]], !prof [[PROF1]]
+; CHECK:       8:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       9:
+; CHECK-NEXT:    [[TMPVAR2:%.*]] = load <4 x i32>, ptr [[B]], align 16
+; CHECK-NEXT:    [[TMP10:%.*]] = ptrtoint ptr [[B]] to i64
+; CHECK-NEXT:    [[TMP11:%.*]] = xor i64 [[TMP10]], 193514046488576
+; CHECK-NEXT:    [[TMP12:%.*]] = inttoptr i64 [[TMP11]] to ptr
+; CHECK-NEXT:    [[_MSLD1:%.*]] = load <4 x i32>, ptr [[TMP12]], align 16
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <4 x i32> [[_MSLD]], [[_MSLD1]]
+; CHECK-NEXT:    [[TMP3:%.*]] = call <4 x i32> @llvm.aarch64.neon.addp.v4i32(<4 x i32> [[TMPVAR1]], <4 x i32> [[TMPVAR2]])
+; CHECK-NEXT:    store <4 x i32> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i32> [[TMP3]]
+;
+  %tmpvar1 = load <4 x i32>, ptr %A
+  %tmpvar2 = load <4 x i32>, ptr %B
+  %tmpvar3 = call <4 x i32> @llvm.aarch64.neon.addp.v4i32(<4 x i32> %tmpvar1, <4 x i32> %tmpvar2)
+  ret <4 x i32> %tmpvar3
+}
+
+define <2 x i64> @addp_2d(ptr %A, ptr %B) nounwind #0 {
+; CHECK-LABEL: define <2 x i64> @addp_2d(
+; CHECK-SAME: ptr [[A:%.*]], ptr [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP13:%.*]], label [[TMP4:%.*]], !prof [[PROF1]]
+; CHECK:       3:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       4:
+; CHECK-NEXT:    [[TMPVAR1:%.*]] = load <2 x i64>, ptr [[A]], align 16
+; CHECK-NEXT:    [[TMP5:%.*]] = ptrtoint ptr [[A]] to i64
+; CHECK-NEXT:    [[TMP6:%.*]] = xor i64 [[TMP5]], 193514046488576
+; CHECK-NEXT:    [[TMP7:%.*]] = inttoptr i64 [[TMP6]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <2 x i64>, ptr [[TMP7]], align 16
+; CHECK-NEXT:    [[_MSCMP3:%.*]] = icmp ne i64 [[TMP2]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP3]], label [[TMP8:%.*]], label [[TMP9:%.*]], !prof [[PROF1]]
+; CHECK:       8:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       9:
+; CHECK-NEXT:    [[TMPVAR2:%.*]] = load <2 x i64>, ptr [[B]], align 16
+; CHECK-NEXT:    [[TMP10:%.*]] = ptrtoint ptr [[B]] to i64
+; CHECK-NEXT:    [[TMP11:%.*]] = xor i64 [[TMP10]], 193514046488576
+; CHECK-NEXT:    [[TMP12:%.*]] = inttoptr i64 [[TMP11]] to ptr
+; CHECK-NEXT:    [[_MSLD1:%.*]] = load <2 x i64>, ptr [[TMP12]], align 16
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <2 x i64> [[_MSLD]], [[_MSLD1]]
+; CHECK-NEXT:    [[TMP3:%.*]] = call <2 x i64> @llvm.aarch64.neon.addp.v2i64(<2 x i64> [[TMPVAR1]], <2 x i64> [[TMPVAR2]])
+; CHECK-NEXT:    store <2 x i64> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x i64> [[TMP3]]
+;
+  %tmpvar1 = load <2 x i64>, ptr %A
+  %tmpvar2 = load <2 x i64>, ptr %B
+  %tmpvar3 = call <2 x i64> @llvm.aarch64.neon.addp.v2i64(<2 x i64> %tmpvar1, <2 x i64> %tmpvar2)
+  ret <2 x i64> %tmpvar3
+}
+
+declare <8 x i8> @llvm.aarch64.neon.addp.v8i8(<8 x i8>, <8 x i8>) nounwind readnone
+declare <16 x i8> @llvm.aarch64.neon.addp.v16i8(<16 x i8>, <16 x i8>) nounwind readnone
+declare <4 x i16> @llvm.aarch64.neon.addp.v4i16(<4 x i16>, <4 x i16>) nounwind readnone
+declare <8 x i16> @llvm.aarch64.neon.addp.v8i16(<8 x i16>, <8 x i16>) nounwind readnone
+declare <2 x i32> @llvm.aarch64.neon.addp.v2i32(<2 x i32>, <2 x i32>) nounwind readnone
+declare <4 x i32> @llvm.aarch64.neon.addp.v4i32(<4 x i32>, <4 x i32>) nounwind readnone
+declare <2 x i64> @llvm.aarch64.neon.addp.v2i64(<2 x i64>, <2 x i64>) nounwind readnone
+
+define <2 x float> @faddp_2s(ptr %A, ptr %B) nounwind #0 {
+; CHECK-LABEL: define <2 x float> @faddp_2s(
+; CHECK-SAME: ptr [[A:%.*]], ptr [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP13:%.*]], label [[TMP4:%.*]], !prof [[PROF1]]
+; CHECK:       3:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       4:
+; CHECK-NEXT:    [[TMPVAR1:%.*]] = load <2 x float>, ptr [[A]], align 8
+; CHECK-NEXT:    [[TMP5:%.*]] = ptrtoint ptr [[A]] to i64
+; CHECK-NEXT:    [[TMP6:%.*]] = xor i64 [[TMP5]], 193514046488576
+; CHECK-NEXT:    [[TMP7:%.*]] = inttoptr i64 [[TMP6]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <2 x i32>, ptr [[TMP7]], align 8
+; CHECK-NEXT:    [[_MSCMP3:%.*]] = icmp ne i64 [[TMP2]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP3]], label [[TMP8:%.*]], label [[TMP9:%.*]], !prof [[PROF1]]
+; CHECK:       8:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       9:
+; CHECK-NEXT:    [[TMPVAR2:%.*]] = load <2 x float>, ptr [[B]], align 8
+; CHECK-NEXT:    [[TMP10:%.*]] = ptrtoint ptr [[B]] to i64
+; CHECK-NEXT:    [[TMP11:%.*]] = xor i64 [[TMP10]], 193514046488576
+; CHECK-NEXT:    [[TMP12:%.*]] = inttoptr i64 [[TMP11]] to ptr
+; CHECK-NEXT:    [[_MSLD1:%.*]] = load <2 x i32>, ptr [[TMP12]], align 8
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <2 x i32> [[_MSLD]], [[_MSLD1]]
+; CHECK-NEXT:    [[TMP3:%.*]] = call <2 x float> @llvm.aarch64.neon.faddp.v2f32(<2 x float> [[TMPVAR1]], <2 x float> [[TMPVAR2]])
+; CHECK-NEXT:    store <2 x i32> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x float> [[TMP3]]
+;
+  %tmpvar1 = load <2 x float>, ptr %A
+  %tmpvar2 = load <2 x float>, ptr %B
+  %tmpvar3 = call <2 x float> @llvm.aarch64.neon.faddp.v2f32(<2 x float> %tmpvar1, <2 x float> %tmpvar2)
+  ret <2 x float> %tmpvar3
+}
+
+define <4 x float> @faddp_4s(ptr %A, ptr %B) nounwind #0 {
+; CHECK-LABEL: define <4 x float> @faddp_4s(
+; CHECK-SAME: ptr [[A:%.*]], ptr [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP13:%.*]], label [[TMP4:%.*]], !prof [[PROF1]]
+; CHECK:       3:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       4:
+; CHECK-NEXT:    [[TMPVAR1:%.*]] = load <4 x float>, ptr [[A]], align 16
+; CHECK-NEXT:    [[TMP5:%.*]] = ptrtoint ptr [[A]] to i64
+; CHECK-NEXT:    [[TMP6:%.*]] = xor i64 [[TMP5]], 193514046488576
+; CHECK-NEXT:    [[TMP7:%.*]] = inttoptr i64 [[TMP6]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <4 x i32>, ptr [[TMP7]], align 16
+; CHECK-NEXT:    [[_MSCMP3:%.*]] = icmp ne i64 [[TMP2]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP3]], label [[TMP8:%.*]], label [[TMP9:%.*]], !prof [[PROF1]]
+; CHECK:       8:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       9:
+; CHECK-NEXT:    [[TMPVAR2:%.*]] = load <4 x float>, ptr [[B]], align 16
+; CHECK-NEXT:    [[TMP10:%.*]] = ptrtoint ptr [[B]] to i64
+; CHECK-NEXT:    [[TMP11:%.*]] = xor i64 [[TMP10]], 193514046488576
+; CHECK-NEXT:    [[TMP12:%.*]] = inttoptr i64 [[TMP11]] to ptr
+; CHECK-NEXT:    [[_MSLD1:%.*]] = load <4 x i32>, ptr [[TMP12]], align 16
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <4 x i32> [[_MSLD]], [[_MSLD1]]
+; CHECK-NEXT:    [[TMP3:%.*]] = call <4 x float> @llvm.aarch64.neon.faddp.v4f32(<4 x float> [[TMPVAR1]], <4 x float> [[TMPVAR2]])
+; CHECK-NEXT:    store <4 x i32> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x float> [[TMP3]]
+;
+  %tmpvar1 = load <4 x float>, ptr %A
+  %tmpvar2 = load <4 x float>, ptr %B
+  %tmpvar3 = call <4 x float> @llvm.aarch64.neon.faddp.v4f32(<4 x float> %tmpvar1, <4 x float> %tmpvar2)
+  ret <4 x float> %tmpvar3
+}
+
+define <2 x double> @faddp_2d(ptr %A, ptr %B) nounwind #0 {
+; CHECK-LABEL: define <2 x double> @faddp_2d(
+; CHECK-SAME: ptr [[A:%.*]], ptr [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP13:%.*]], label [[TMP4:%.*]], !prof [[PROF1]]
+; CHECK:       3:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       4:
+; CHECK-NEXT:    [[TMPVAR1:%.*]] = load <2 x double>, ptr [[A]], align 16
+; CHECK-NEXT:    [[TMP5:%.*]] = ptrtoint ptr [[A]] to i64
+; CHECK-NEXT:    [[TMP6:%.*]] = xor i64 [[TMP5]], 193514046488576
+; CHECK-NEXT:    [[TMP7:%.*]] = inttoptr i64 [[TMP6]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <2 x i64>, ptr [[TMP7]], align 16
+; CHECK-NEXT:    [[_MSCMP3:%.*]] = icmp ne i64 [[TMP2]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP3]], label [[TMP8:%.*]], label [[TMP9:%.*]], !prof [[PROF1]]
+; CHECK:       8:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       9:
+; CHECK-NEXT:    [[TMPVAR2:%.*]] = load <2 x double>, ptr [[B]], align 16
+; CHECK-NEXT:    [[TMP10:%.*]] = ptrtoint ptr [[B]] to i64
+; CHECK-NEXT:    [[TMP11:%.*]] = xor i64 [[TMP10]], 193514046488576
+; CHECK-NEXT:    [[TMP12:%.*]] = inttoptr i64 [[TMP11]] to ptr
+; CHECK-NEXT:    [[_MSLD1:%.*]] = load <2 x i64>, ptr [[TMP12]], align 16
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <2 x i64> [[_MSLD]], [[_MSLD1]]
+; CHECK-NEXT:    [[TMP3:%.*]] = call <2 x double> @llvm.aarch64.neon.faddp.v2f64(<2 x double> [[TMPVAR1]], <2 x double> [[TMPVAR2]])
+; CHECK-NEXT:    store <2 x i64> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x double> [[TMP3]]
+;
+  %tmpvar1 = load <2 x double>, ptr %A
+  %tmpvar2 = load <2 x double>, ptr %B
+  %tmpvar3 = call <2 x double> @llvm.aarch64.neon.faddp.v2f64(<2 x double> %tmpvar1, <2 x double> %tmpvar2)
+  ret <2 x double> %tmpvar3
+}
+
+declare <2 x float> @llvm.aarch64.neon.faddp.v2f32(<2 x float>, <2 x float>) nounwind readnone
+declare <4 x float> @llvm.aarch64.neon.faddp.v4f32(<4 x float>, <4 x float>) nounwind readnone
+declare <2 x double> @llvm.aarch64.neon.faddp.v2f64(<2 x double>, <2 x double>) nounwind readnone
+
+define <2 x i64> @uaddl_duprhs(<4 x i32> %lhs, i32 %rhs, <2 x i32> %param1, <4 x i32> %param2) #0 {
+; CHECK-LABEL: define <2 x i64> @uaddl_duprhs(
+; CHECK-SAME: <4 x i32> [[LHS:%.*]], i32 [[RHS:%.*]], <2 x i32> [[PARAM1:%.*]], <4 x i32> [[PARAM2:%.*]]) #[[ATTR2:[0-9]+]] {
+; CHECK-NEXT:    [[TMP3:%.*]] = load <2 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 24) to ptr), align 8
+; CHECK-NEXT:    [[TMP1:%.*]] = load i32, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = insertelement <2 x i32> [[TMP3]], i32 [[TMP1]], i32 0
+; CHECK-NEXT:    [[RHSVEC_TMP:%.*]] = insertelement <2 x i32> [[PARAM1]], i32 [[RHS]], i32 0
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = insertelement <2 x i32> [[_MSPROP]], i32 [[TMP1]], i32 1
+; CHECK-NEXT:    [[RHSVEC:%.*]] = insertelement <2 x i32> [[RHSVEC_TMP]], i32 [[RHS]], i32 1
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = shufflevector <4 x i32> [[TMP2]], <4 x i32> [[TMP4]], <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT:    [[LHS_HIGH:%.*]] = shufflevector <4 x i32> [[LHS]], <4 x i32> [[PARAM2]], <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT:    [[_MSPROP3:%.*]] = zext <2 x i32> [[_MSPROP2]] to <2 x i64>
+; CHECK-NEXT:    [[LHS_EXT:%.*]] = zext <2 x i32> [[LHS_HIGH]] to <2 x i64>
+; CHECK-NEXT:    [[_MSPROP4:%.*]] = zext <2 x i32> [[_MSPROP1]] to <2 x i64>
+; CHECK-NEXT:    [[RHS_EXT:%.*]] = zext <2 x i32> [[RHSVEC]] to <2 x i64>
+; CHECK-NEXT:    [[_MSPROP5:%.*]] = or <2 x i64> [[_MSPROP3]], [[_MSPROP4]]
+; CHECK-NEXT:    [[RES:%.*]] = add <2 x i64> [[LHS_EXT]], [[RHS_EXT]]
+; CHECK-NEXT:    store <2 x i64> [[_MSPROP5]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x i64> [[RES]]
+;
+  %rhsvec.tmp = insertelement <2 x i32> %param1, i32 %rhs, i32 0
+  %rhsvec = insertelement <2 x i32> %rhsvec.tmp, i32 %rhs, i32 1
+
+  %lhs.high = shufflevector <4 x i32> %lhs, <4 x i32> %param2, <2 x i32> <i32 0, i32 1>
+
+  %lhs.ext = zext <2 x i32> %lhs.high to <2 x i64>
+  %rhs.ext = zext <2 x i32> %rhsvec to <2 x i64>
+
+  %res = add <2 x i64> %lhs.ext, %rhs.ext
+  ret <2 x i64> %res
+}
+
+define <2 x i64> @uaddl2_duprhs(<4 x i32> %lhs, i32 %rhs, <2 x i32> %param1, <4 x i32> %param2) #0 {
+; CHECK-LABEL: define <2 x i64> @uaddl2_duprhs(
+; CHECK-SAME: <4 x i32> [[LHS:%.*]], i32 [[RHS:%.*]], <2 x i32> [[PARAM1:%.*]], <4 x i32> [[PARAM2:%.*]]) #[[ATTR2]] {
+; CHECK-NEXT:    [[TMP3:%.*]] = load <2 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 24) to ptr), align 8
+; CHECK-NEXT:    [[TMP1:%.*]] = load i32, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = insertelement <2 x i32> [[TMP3]], i32 [[TMP1]], i32 0
+; CHECK-NEXT:    [[RHSVEC_TMP:%.*]] = insertelement <2 x i32> [[PARAM1]], i32 [[RHS]], i32 0
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = insertelement <2 x i32> [[_MSPROP]], i32 [[TMP1]], i32 1
+; CHECK-NEXT:    [[RHSVEC:%.*]] = insertelement <2 x i32> [[RHSVEC_TMP]], i32 [[RHS]], i32 1
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = shufflevector <4 x i32> [[TMP2]], <4 x i32> [[TMP4]], <2 x i32> <i32 2, i32 3>
+; CHECK-NEXT:    [[LHS_HIGH:%.*]] = shufflevector <4 x i32> [[LHS]], <4 x i32> [[PARAM2]], <2 x i32> <i32 2, i32 3>
+; CHECK-NEXT:    [[_MSPROP3:%.*]] = zext <2 x i32> [[_MSPROP2]] to <2 x i64>
+; CHECK-NEXT:    [[LHS_EXT:%.*]] = zext <2 x i32> [[LHS_HIGH]] to <2 x i64>
+; CHECK-NEXT:    [[_MSPROP4:%.*]] = zext <2 x i32> [[_MSPROP1]] to <2 x i64>
+; CHECK-NEXT:    [[RHS_EXT:%.*]] = zext <2 x i32> [[RHSVEC]] to <2 x i64>
+; CHECK-NEXT:    [[_MSPROP5:%.*]] = or <2 x i64> [[_MSPROP3]], [[_MSPROP4]]
+; CHECK-NEXT:    [[RES:%.*]] = add <2 x i64> [[LHS_EXT]], [[RHS_EXT]]
+; CHECK-NEXT:    store <2 x i64> [[_MSPROP5]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x i64> [[RES]]
+;
+  %rhsvec.tmp = insertelement <2 x i32> %param1, i32 %rhs, i32 0
+  %rhsvec = insertelement <2 x i32> %rhsvec.tmp, i32 %rhs, i32 1
+
+  %lhs.high = shufflevector <4 x i32> %lhs, <4 x i32> %param2, <2 x i32> <i32 2, i32 3>
+
+  %lhs.ext = zext <2 x i32> %lhs.high to <2 x i64>
+  %rhs.ext = zext <2 x i32> %rhsvec to <2 x i64>
+
+  %res = add <2 x i64> %lhs.ext, %rhs.ext
+  ret <2 x i64> %res
+}
+
+define <2 x i64> @saddl_duplhs(i32 %lhs, <4 x i32> %rhs, <2 x i32> %param1, <4 x i32> %param2) #0 {
+; CHECK-LABEL: define <2 x i64> @saddl_duplhs(
+; CHECK-SAME: i32 [[LHS:%.*]], <4 x i32> [[RHS:%.*]], <2 x i32> [[PARAM1:%.*]], <4 x i32> [[PARAM2:%.*]]) #[[ATTR2]] {
+; CHECK-NEXT:    [[TMP3:%.*]] = load <2 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 24) to ptr), align 8
+; CHECK-NEXT:    [[TMP1:%.*]] = load i32, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = insertelement <2 x i32> [[TMP3]], i32 [[TMP1]], i32 0
+; CHECK-NEXT:    [[LHSVEC_TMP:%.*]] = insertelement <2 x i32> [[PARAM1]], i32 [[LHS]], i32 0
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = insertelement <2 x i32> [[_MSPROP]], i32 [[TMP1]], i32 1
+; CHECK-NEXT:    [[LHSVEC:%.*]] = insertelement <2 x i32> [[LHSVEC_TMP]], i32 [[LHS]], i32 1
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = shufflevector <4 x i32> [[TMP2]], <4 x i32> [[TMP4]], <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT:    [[RHS_HIGH:%.*]] = shufflevector <4 x i32> [[RHS]], <4 x i32> [[PARAM2]], <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT:    [[_MSPROP3:%.*]] = sext <2 x i32> [[_MSPROP1]] to <2 x i64>
+; CHECK-NEXT:    [[LHS_EXT:%.*]] = sext <2 x i32> [[LHSVEC]] to <2 x i64>
+; CHECK-NEXT:    [[_MSPROP4:%.*]] = sext <2 x i32> [[_MSPROP2]] to <2 x i64>
+; CHECK-NEXT:    [[RHS_EXT:%.*]] = sext <2 x i32> [[RHS_HIGH]] to <2 x i64>
+; CHECK-NEXT:    [[_MSPROP5:%.*]] = or <2 x i64> [[_MSPROP3]], [[_MSPROP4]]
+; CHECK-NEXT:    [[RES:%.*]] = add <2 x i64> [[LHS_EXT]], [[RHS_EXT]]
+; CHECK-NEXT:    store <2 x i64> [[_MSPROP5]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x i64> [[RES]]
+;
+  %lhsvec.tmp = insertelement <2 x i32> %param1, i32 %lhs, i32 0
+  %lhsvec = insertelement <2 x i32> %lhsvec.tmp, i32 %lhs, i32 1
+
+  %rhs.high = shufflevector <4 x i32> %rhs, <4 x i32> %param2, <2 x i32> <i32 0, i32 1>
+
+  %lhs.ext = sext <2 x i32> %lhsvec to <2 x i64>
+  %rhs.ext = sext <2 x i32> %rhs.high to <2 x i64>
+
+  %res = add <2 x i64> %lhs.ext, %rhs.ext
+  ret <2 x i64> %res
+}
+
+define <2 x i64> @saddl2_duplhs(i32 %lhs, <4 x i32> %rhs, <2 x i32> %param1, <4 x i32> %param2) #0 {
+; CHECK-LABEL: define <2 x i64> @saddl2_duplhs(
+; CHECK-SAME: i32 [[LHS:%.*]], <4 x i32> [[RHS:%.*]], <2 x i32> [[PARAM1:%.*]], <4 x i32> [[PARAM2:%.*]]) #[[ATTR2]] {
+; CHECK-NEXT:    [[TMP3:%.*]] = load <2 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 24) to ptr), align 8
+; CHECK-NEXT:    [[TMP1:%.*]] = load i32, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = insertelement <2 x i32> [[TMP3]], i32 [[TMP1]], i32 0
+; CHECK-NEXT:    [[LHSVEC_TMP:%.*]] = insertelement <2 x i32> [[PARAM1]], i32 [[LHS]], i32 0
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = insertelement <2 x i32> [[_MSPROP]], i32 [[TMP1]], i32 1
+; CHECK-NEXT:    [[LHSVEC:%.*]] = insertelement <2 x i32> [[LHSVEC_TMP]], i32 [[LHS]], i32 1
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = shufflevector <4 x i32> [[TMP2]], <4 x i32> [[TMP4]], <2 x i32> <i32 2, i32 3>
+; CHECK-NEXT:    [[RHS_HIGH:%.*]] = shufflevector <4 x i32> [[RHS]], <4 x i32> [[PARAM2]], <2 x i32> <i32 2, i32 3>
+; CHECK-NEXT:    [[_MSPROP3:%.*]] = sext <2 x i32> [[_MSPROP1]] to <2 x i64>
+; CHECK-NEXT:    [[LHS_EXT:%.*]] = sext <2 x i32> [[LHSVEC]] to <2 x i64>
+; CHECK-NEXT:    [[_MSPROP4:%.*]] = sext <2 x i32> [[_MSPROP2]] to <2 x i64>
+; CHECK-NEXT:    [[RHS_EXT:%.*]] = sext <2 x i32> [[RHS_HIGH]] to <2 x i64>
+; CHECK-NEXT:    [[_MSPROP5:%.*]] = or <2 x i64> [[_MSPROP3]], [[_MSPROP4]]
+; CHECK-NEXT:    [[RES:%.*]] = add <2 x i64> [[LHS_EXT]], [[RHS_EXT]]
+; CHECK-NEXT:    store <2 x i64> [[_MSPROP5]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x i64> [[RES]]
+;
+  %lhsvec.tmp = insertelement <2 x i32> %param1, i32 %lhs, i32 0
+  %lhsvec = insertelement <2 x i32> %lhsvec.tmp, i32 %lhs, i32 1
+
+  %rhs.high = shufflevector <4 x i32> %rhs, <4 x i32> %param2, <2 x i32> <i32 2, i32 3>
+
+  %lhs.ext = sext <2 x i32> %lhsvec to <2 x i64>
+  %rhs.ext = sext <2 x i32> %rhs.high to <2 x i64>
+
+  %res = add <2 x i64> %lhs.ext, %rhs.ext
+  ret <2 x i64> %res
+}
+
+define <2 x i64> @usubl_duprhs(<4 x i32> %lhs, i32 %rhs, <2 x i32> %param1, <4 x i32> %param2) #0 {
+; CHECK-LABEL: define <2 x i64> @usubl_duprhs(
+; CHECK-SAME: <4 x i32> [[LHS:%.*]], i32 [[RHS:%.*]], <2 x i32> [[PARAM1:%.*]], <4 x i32> [[PARAM2:%.*]]) #[[ATTR2]] {
+; CHECK-NEXT:    [[TMP3:%.*]] = load <2 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 24) to ptr), align 8
+; CHECK-NEXT:    [[TMP1:%.*]] = load i32, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = insertelement <2 x i32> [[TMP3]], i32 [[TMP1]], i32 0
+; CHECK-NEXT:    [[RHSVEC_TMP:%.*]] = insertelement <2 x i32> [[PARAM1]], i32 [[RHS]], i32 0
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = insertelement <2 x i32> [[_MSPROP]], i32 [[TMP1]], i32 1
+; CHECK-NEXT:    [[RHSVEC:%.*]] = insertelement <2 x i32> [[RHSVEC_TMP]], i32 [[RHS]], i32 1
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = shufflevector <4 x i32> [[TMP2]], <4 x i32> [[TMP4]], <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT:    [[LHS_HIGH:%.*]] = shufflevector <4 x i32> [[LHS]], <4 x i32> [[PARAM2]], <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT:    [[_MSPROP3:%.*]] = zext <2 x i32> [[_MSPROP2]] to <2 x i64>
+; CHECK-NEXT:    [[LHS_EXT:%.*]] = zext <2 x i32> [[LHS_HIGH]] to <2 x i64>
+; CHECK-NEXT:    [[_MSPROP4:%.*]] = zext <2 x i32> [[_MSPROP1]] to <2 x i64>
+; CHECK-NEXT:    [[RHS_EXT:%.*]] = zext <2 x i32> [[RHSVEC]] to <2 x i64>
+; CHECK-NEXT:    [[_MSPROP5:%.*]] = or <2 x i64> [[_MSPROP3]], [[_MSPROP4]]
+; CHECK-NEXT:    [[RES:%.*]] = sub <2 x i64> [[LHS_EXT]], [[RHS_EXT]]
+; CHECK-NEXT:    store <2 x i64> [[_MSPROP5]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x i64> [[RES]]
+;
+  %rhsvec.tmp = insertelement <2 x i32> %param1, i32 %rhs, i32 0
+  %rhsvec = insertelement <2 x i32> %rhsvec.tmp, i32 %rhs, i32 1
+
+  %lhs.high = shufflevector <4 x i32> %lhs, <4 x i32> %param2, <2 x i32> <i32 0, i32 1>
+
+  %lhs.ext = zext <2 x i32> %lhs.high to <2 x i64>
+  %rhs.ext = zext <2 x i32> %rhsvec to <2 x i64>
+
+  %res = sub <2 x i64> %lhs.ext, %rhs.ext
+  ret <2 x i64> %res
+}
+
+define <2 x i64> @usubl2_duprhs(<4 x i32> %lhs, i32 %rhs, <2 x i32> %param1, <4 x i32> %param2) #0 {
+; CHECK-LABEL: define <2 x i64> @usubl2_duprhs(
+; CHECK-SAME: <4 x i32> [[LHS:%.*]], i32 [[RHS:%.*]], <2 x i32> [[PARAM1:%.*]], <4 x i32> [[PARAM2:%.*]]) #[[ATTR2]] {
+; CHECK-NEXT:    [[TMP3:%.*]] = load <2 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 24) to ptr), align 8
+; CHECK-NEXT:    [[TMP1:%.*]] = load i32, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = insertelement <2 x i32> [[TMP3]], i32 [[TMP1]], i32 0
+; CHECK-NEXT:    [[RHSVEC_TMP:%.*]] = insertelement <2 x i32> [[PARAM1]], i32 [[RHS]], i32 0
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = insertelement <2 x i32> [[_MSPROP]], i32 [[TMP1]], i32 1
+; CHECK-NEXT:    [[RHSVEC:%.*]] = insertelement <2 x i32> [[RHSVEC_TMP]], i32 [[RHS]], i32 1
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = shufflevector <4 x i32> [[TMP2]], <4 x i32> [[TMP4]], <2 x i32> <i32 2, i32 3>
+; CHECK-NEXT:    [[LHS_HIGH:%.*]] = shufflevector <4 x i32> [[LHS]], <4 x i32> [[PARAM2]], <2 x i32> <i32 2, i32 3>
+; CHECK-NEXT:    [[_MSPROP3:%.*]] = zext <2 x i32> [[_MSPROP2]] to <2 x i64>
+; CHECK-NEXT:    [[LHS_EXT:%.*]] = zext <2 x i32> [[LHS_HIGH]] to <2 x i64>
+; CHECK-NEXT:    [[_MSPROP4:%.*]] = zext <2 x i32> [[_MSPROP1]] to <2 x i64>
+; CHECK-NEXT:    [[RHS_EXT:%.*]] = zext <2 x i32> [[RHSVEC]] to <2 x i64>
+; CHECK-NEXT:    [[_MSPROP5:%.*]] = or <2 x i64> [[_MSPROP3]], [[_MSPROP4]]
+; CHECK-NEXT:    [[RES:%.*]] = sub <2 x i64> [[LHS_EXT]], [[RHS_EXT]]
+; CHECK-NEXT:    store <2 x i64> [[_MSPROP5]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x i64> [[RES]]
+;
+  %rhsvec.tmp = insertelement <2 x i32> %param1, i32 %rhs, i32 0
+  %rhsvec = insertelement <2 x i32> %rhsvec.tmp, i32 %rhs, i32 1
+
+  %lhs.high = shufflevector <4 x i32> %lhs, <4 x i32> %param2, <2 x i32> <i32 2, i32 3>
+
+  %lhs.ext = zext <2 x i32> %lhs.high to <2 x i64>
+  %rhs.ext = zext <2 x i32> %rhsvec to <2 x i64>
+
+  %res = sub <2 x i64> %lhs.ext, %rhs.ext
+  ret <2 x i64> %res
+}
+
+define <2 x i64> @ssubl_duplhs(i32 %lhs, <4 x i32> %rhs, <2 x i32> %param1, <4 x i32> %param2) #0 {
+; CHECK-LABEL: define <2 x i64> @ssubl_duplhs(
+; CHECK-SAME: i32 [[LHS:%.*]], <4 x i32> [[RHS:%.*]], <2 x i32> [[PARAM1:%.*]], <4 x i32> [[PARAM2:%.*]]) #[[ATTR2]] {
+; CHECK-NEXT:    [[TMP3:%.*]] = load <2 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 24) to ptr), align 8
+; CHECK-NEXT:    [[TMP1:%.*]] = load i32, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = insertelement <2 x i32> [[TMP3]], i32 [[TMP1]], i32 0
+; CHECK-NEXT:    [[LHSVEC_TMP:%.*]] = insertelement <2 x i32> [[PARAM1]], i32 [[LHS]], i32 0
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = insertelement <2 x i32> [[_MSPROP]], i32 [[TMP1]], i32 1
+; CHECK-NEXT:    [[LHSVEC:%.*]] = insertelement <2 x i32> [[LHSVEC_TMP]], i32 [[LHS]], i32 1
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = shufflevector <4 x i32> [[TMP2]], <4 x i32> [[TMP4]], <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT:    [[RHS_HIGH:%.*]] = shufflevector <4 x i32> [[RHS]], <4 x i32> [[PARAM2]], <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT:    [[_MSPROP3:%.*]] = sext <2 x i32> [[_MSPROP1]] to <2 x i64>
+; CHECK-NEXT:    [[LHS_EXT:%.*]] = sext <2 x i32> [[LHSVEC]] to <2 x i64>
+; CHECK-NEXT:    [[_MSPROP4:%.*]] = sext <2 x i32> [[_MSPROP2]] to <2 x i64>
+; CHECK-NEXT:    [[RHS_EXT:%.*]] = sext <2 x i32> [[RHS_HIGH]] to <2 x i64>
+; CHECK-NEXT:    [[_MSPROP5:%.*]] = or <2 x i64> [[_MSPROP3]], [[_MSPROP4]]
+; CHECK-NEXT:    [[RES:%.*]] = sub <2 x i64> [[LHS_EXT]], [[RHS_EXT]]
+; CHECK-NEXT:    store <2 x i64> [[_MSPROP5]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x i64> [[RES]]
+;
+  %lhsvec.tmp = insertelement <2 x i32> %param1, i32 %lhs, i32 0
+  %lhsvec = insertelement <2 x i32> %lhsvec.tmp, i32 %lhs, i32 1
+
+  %rhs.high = shufflevector <4 x i32> %rhs, <4 x i32> %param2, <2 x i32> <i32 0, i32 1>
+
+  %lhs.ext = sext <2 x i32> %lhsvec to <2 x i64>
+  %rhs.ext = sext <2 x i32> %rhs.high to <2 x i64>
+
+  %res = sub <2 x i64> %lhs.ext, %rhs.ext
+  ret <2 x i64> %res
+}
+
+define <2 x i64> @ssubl2_duplhs(i32 %lhs, <4 x i32> %rhs, <2 x i32> %param1, <4 x i32> %param2) #0 {
+; CHECK-LABEL: define <2 x i64> @ssubl2_duplhs(
+; CHECK-SAME: i32 [[LHS:%.*]], <4 x i32> [[RHS:%.*]], <2 x i32> [[PARAM1:%.*]], <4 x i32> [[PARAM2:%.*]]) #[[ATTR2]] {
+; CHECK-NEXT:    [[TMP3:%.*]] = load <2 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 24) to ptr), align 8
+; CHECK-NEXT:    [[TMP1:%.*]] = load i32, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = insertelement <2 x i32> [[TMP3]], i32 [[TMP1]], i32 0
+; CHECK-NEXT:    [[LHSVEC_TMP:%.*]] = insertelement <2 x i32> [[PARAM1]], i32 [[LHS]], i32 0
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = insertelement <2 x i32> [[_MSPROP]], i32 [[TMP1]], i32 1
+; CHECK-NEXT:    [[LHSVEC:%.*]] = insertelement <2 x i32> [[LHSVEC_TMP]], i32 [[LHS]], i32 1
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = shufflevector <4 x i32> [[TMP2]], <4 x i32> [[TMP4]], <2 x i32> <i32 2, i32 3>
+; CHECK-NEXT:    [[RHS_HIGH:%.*]] = shufflevector <4 x i32> [[RHS]], <4 x i32> [[PARAM2]], <2 x i32> <i32 2, i32 3>
+; CHECK-NEXT:    [[_MSPROP3:%.*]] = sext <2 x i32> [[_MSPROP1]] to <2 x i64>
+; CHECK-NEXT:    [[LHS_EXT:%.*]] = sext <2 x i32> [[LHSVEC]] to <2 x i64>
+; CHECK-NEXT:    [[_MSPROP4:%.*]] = sext <2 x i32> [[_MSPROP2]] to <2 x i64>
+; CHECK-NEXT:    [[RHS_EXT:%.*]] = sext <2 x i32> [[RHS_HIGH]] to <2 x i64>
+; CHECK-NEXT:    [[_MSPROP5:%.*]] = or <2 x i64> [[_MSPROP3]], [[_MSPROP4]]
+; CHECK-NEXT:    [[RES:%.*]] = sub <2 x i64> [[LHS_EXT]], [[RHS_EXT]]
+; CHECK-NEXT:    store <2 x i64> [[_MSPROP5]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x i64> [[RES]]
+;
+  %lhsvec.tmp = insertelement <2 x i32> %param1, i32 %lhs, i32 0
+  %lhsvec = insertelement <2 x i32> %lhsvec.tmp, i32 %lhs, i32 1
+
+  %rhs.high = shufflevector <4 x i32> %rhs, <4 x i32> %param2, <2 x i32> <i32 2, i32 3>
+
+  %lhs.ext = sext <2 x i32> %lhsvec to <2 x i64>
+  %rhs.ext = sext <2 x i32> %rhs.high to <2 x i64>
+
+  %res = sub <2 x i64> %lhs.ext, %rhs.ext
+  ret <2 x i64> %res
+}
+
+define <8 x i8> @addhn8b_natural(ptr %A, ptr %B) nounwind #0 {
+; CHECK-LABEL: define <8 x i8> @addhn8b_natural(
+; CHECK-SAME: ptr [[A:%.*]], ptr [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP3:%.*]], label [[TMP4:%.*]], !prof [[PROF1]]
+; CHECK:       3:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       4:
+; CHECK-NEXT:    [[TMPVAR1:%.*]] = load <8 x i16>, ptr [[A]], align 16
+; CHECK-NEXT:    [[TMP5:%.*]] = ptrtoint ptr [[A]] to i64
+; CHECK-NEXT:    [[TMP6:%.*]] = xor i64 [[TMP5]], 193514046488576
+; CHECK-NEXT:    [[TMP7:%.*]] = inttoptr i64 [[TMP6]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <8 x i16>, ptr [[TMP7]], align 16
+; CHECK-NEXT:    [[_MSCMP3:%.*]] = icmp ne i64 [[TMP2]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP3]], label [[TMP8:%.*]], label [[TMP9:%.*]], !prof [[PROF1]]
+; CHECK:       8:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       9:
+; CHECK-NEXT:    [[TMPVAR2:%.*]] = load <8 x i16>, ptr [[B]], align 16
+; CHECK-NEXT:    [[TMP10:%.*]] = ptrtoint ptr [[B]] to i64
+; CHECK-NEXT:    [[TMP11:%.*]] = xor i64 [[TMP10]], 193514046488576
+; CHECK-NEXT:    [[TMP12:%.*]] = inttoptr i64 [[TMP11]] to ptr
+; CHECK-NEXT:    [[_MSLD1:%.*]] = load <8 x i16>, ptr [[TMP12]], align 16
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <8 x i16> [[_MSLD]], [[_MSLD1]]
+; CHECK-NEXT:    [[SUM:%.*]] = add <8 x i16> [[TMPVAR1]], [[TMPVAR2]]
+; CHECK-NEXT:    [[TMP13:%.*]] = lshr <8 x i16> [[_MSPROP]], splat (i16 8)
+; CHECK-NEXT:    [[TMP14:%.*]] = or <8 x i16> [[TMP13]], zeroinitializer
+; CHECK-NEXT:    [[HIGH_BITS:%.*]] = lshr <8 x i16> [[SUM]], splat (i16 8)
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = trunc <8 x i16> [[TMP14]] to <8 x i8>
+; CHECK-NEXT:    [[NARROWED:%.*]] = trunc <8 x i16> [[HIGH_BITS]] to <8 x i8>
+; CHECK-NEXT:    store <8 x i8> [[_MSPROP2]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i8> [[NARROWED]]
+;
+  %tmpvar1 = load <8 x i16>, ptr %A
+  %tmpvar2 = load <8 x i16>, ptr %B
+  %sum = add <8 x i16> %tmpvar1, %tmpvar2
+  %high_bits = lshr <8 x i16> %sum, <i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8>
+  %narrowed = trunc <8 x i16> %high_bits to <8 x i8>
+  ret <8 x i8> %narrowed
+}
+
+define <4 x i16> @addhn4h_natural(ptr %A, ptr %B) nounwind #0 {
+; CHECK-LABEL: define <4 x i16> @addhn4h_natural(
+; CHECK-SAME: ptr [[A:%.*]], ptr [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP3:%.*]], label [[TMP4:%.*]], !prof [[PROF1]]
+; CHECK:       3:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       4:
+; CHECK-NEXT:    [[TMPVAR1:%.*]] = load <4 x i32>, ptr [[A]], align 16
+; CHECK-NEXT:    [[TMP5:%.*]] = ptrtoint ptr [[A]] to i64
+; CHECK-NEXT:    [[TMP6:%.*]] = xor i64 [[TMP5]], 193514046488576
+; CHECK-NEXT:    [[TMP7:%.*]] = inttoptr i64 [[TMP6]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <4 x i32>, ptr [[TMP7]], align 16
+; CHECK-NEXT:    [[_MSCMP3:%.*]] = icmp ne i64 [[TMP2]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP3]], label [[TMP8:%.*]], label [[TMP9:%.*]], !prof [[PROF1]]
+; CHECK:       8:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       9:
+; CHECK-NEXT:    [[TMPVAR2:%.*]] = load <4 x i32>, ptr [[B]], align 16
+; CHECK-NEXT:    [[TMP10:%.*]] = ptrtoint ptr [[B]] to i64
+; CHECK-NEXT:    [[TMP11:%.*]] = xor i64 [[TMP10]], 193514046488576
+; CHECK-NEXT:    [[TMP12:%.*]] = inttoptr i64 [[TMP11]] to ptr
+; CHECK-NEXT:    [[_MSLD1:%.*]] = load <4 x i32>, ptr [[TMP12]], align 16
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <4 x i32> [[_MSLD]], [[_MSLD1]]
+; CHECK-NEXT:    [[SUM:%.*]] = add <4 x i32> [[TMPVAR1]], [[TMPVAR2]]
+; CHECK-NEXT:    [[TMP13:%.*]] = lshr <4 x i32> [[_MSPROP]], splat (i32 16)
+; CHECK-NEXT:    [[TMP14:%.*]] = or <4 x i32> [[TMP13]], zeroinitializer
+; CHECK-NEXT:    [[HIGH_BITS:%.*]] = lshr <4 x i32> [[SUM]], splat (i32 16)
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = trunc <4 x i32> [[TMP14]] to <4 x i16>
+; CHECK-NEXT:    [[NARROWED:%.*]] = trunc <4 x i32> [[HIGH_BITS]] to <4 x i16>
+; CHECK-NEXT:    store <4 x i16> [[_MSPROP2]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i16> [[NARROWED]]
+;
+  %tmpvar1 = load <4 x i32>, ptr %A
+  %tmpvar2 = load <4 x i32>, ptr %B
+  %sum = add <4 x i32> %tmpvar1, %tmpvar2
+  %high_bits = lshr <4 x i32> %sum, <i32 16, i32 16, i32 16, i32 16>
+  %narrowed = trunc <4 x i32> %high_bits to <4 x i16>
+  ret <4 x i16> %narrowed
+}
+
+define <2 x i32> @addhn2s_natural(ptr %A, ptr %B) nounwind #0 {
+; CHECK-LABEL: define <2 x i32> @addhn2s_natural(
+; CHECK-SAME: ptr [[A:%.*]], ptr [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP3:%.*]], label [[TMP4:%.*]], !prof [[PROF1]]
+; CHECK:       3:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       4:
+; CHECK-NEXT:    [[TMPVAR1:%.*]] = load <2 x i64>, ptr [[A]], align 16
+; CHECK-NEXT:    [[TMP5:%.*]] = ptrtoint ptr [[A]] to i64
+; CHECK-NEXT:    [[TMP6:%.*]] = xor i64 [[TMP5]], 193514046488576
+; CHECK-NEXT:    [[TMP7:%.*]] = inttoptr i64 [[TMP6]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <2 x i64>, ptr [[TMP7]], align 16
+; CHECK-NEXT:    [[_MSCMP3:%.*]] = icmp ne i64 [[TMP2]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP3]], label [[TMP8:%.*]], label [[TMP9:%.*]], !prof [[PROF1]]
+; CHECK:       8:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       9:
+; CHECK-NEXT:    [[TMPVAR2:%.*]] = load <2 x i64>, ptr [[B]], align 16
+; CHECK-NEXT:    [[TMP10:%.*]] = ptrtoint ptr [[B]] to i64
+; CHECK-NEXT:    [[TMP11:%.*]] = xor i64 [[TMP10]], 193514046488576
+; CHECK-NEXT:    [[TMP12:%.*]] = inttoptr i64 [[TMP11]] to ptr
+; CHECK-NEXT:    [[_MSLD1:%.*]] = load <2 x i64>, ptr [[TMP12]], align 16
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <2 x i64> [[_MSLD]], [[_MSLD1]]
+; CHECK-NEXT:    [[SUM:%.*]] = add <2 x i64> [[TMPVAR1]], [[TMPVAR2]]
+; CHECK-NEXT:    [[TMP13:%.*]] = lshr <2 x i64> [[_MSPROP]], splat (i64 32)
+; CHECK-NEXT:    [[TMP14:%.*]] = or <2 x i64> [[TMP13]], zeroinitializer
+; CHECK-NEXT:    [[HIGH_BITS:%.*]] = lshr <2 x i64> [[SUM]], splat (i64 32)
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = trunc <2 x i64> [[TMP14]] to <2 x i32>
+; CHECK-NEXT:    [[NARROWED:%.*]] = trunc <2 x i64> [[HIGH_BITS]] to <2 x i32>
+; CHECK-NEXT:    store <2 x i32> [[_MSPROP2]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x i32> [[NARROWED]]
+;
+  %tmpvar1 = load <2 x i64>, ptr %A
+  %tmpvar2 = load <2 x i64>, ptr %B
+  %sum = add <2 x i64> %tmpvar1, %tmpvar2
+  %high_bits = lshr <2 x i64> %sum, <i64 32, i64 32>
+  %narrowed = trunc <2 x i64> %high_bits to <2 x i32>
+  ret <2 x i32> %narrowed
+}
+
+define <16 x i8> @addhn2_16b_natural(<8 x i8> %low, ptr %A, ptr %B) nounwind #0 {
+; CHECK-LABEL: define <16 x i8> @addhn2_16b_natural(
+; CHECK-SAME: <8 x i8> [[LOW:%.*]], ptr [[A:%.*]], ptr [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i8>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP4:%.*]], label [[TMP5:%.*]], !prof [[PROF1]]
+; CHECK:       4:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       5:
+; CHECK-NEXT:    [[TMPVAR1:%.*]] = load <8 x i16>, ptr [[A]], align 16
+; CHECK-NEXT:    [[TMP6:%.*]] = ptrtoint ptr [[A]] to i64
+; CHECK-NEXT:    [[TMP7:%.*]] = xor i64 [[TMP6]], 193514046488576
+; CHECK-NEXT:    [[TMP8:%.*]] = inttoptr i64 [[TMP7]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <8 x i16>, ptr [[TMP8]], align 16
+; CHECK-NEXT:    [[_MSCMP4:%.*]] = icmp ne i64 [[TMP2]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP4]], label [[TMP9:%.*]], label [[TMP10:%.*]], !prof [[PROF1]]
+; CHECK:       9:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       10:
+; CHECK-NEXT:    [[TMPVAR2:%.*]] = load <8 x i16>, ptr [[B]], align 16
+; CHECK-NEXT:    [[TMP11:%.*]] = ptrtoint ptr [[B]] to i64
+; CHECK-NEXT:    [[TMP12:%.*]] = xor i64 [[TMP11]], 193514046488576
+; CHECK-NEXT:    [[TMP13:%.*]] = inttoptr i64 [[TMP12]] to ptr
+; CHECK-NEXT:    [[_MSLD1:%.*]] = load <8 x i16>, ptr [[TMP13]], align 16
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <8 x i16> [[_MSLD]], [[_MSLD1]]
+; CHECK-NEXT:    [[SUM:%.*]] = add <8 x i16> [[TMPVAR1]], [[TMPVAR2]]
+; CHECK-NEXT:    [[TMP14:%.*]] = lshr <8 x i16> [[_MSPROP]], splat (i16 8)
+; CHECK-NEXT:    [[TMP15:%.*]] = or <8 x i16> [[TMP14]], zeroinitializer
+; CHECK-NEXT:    [[HIGH_BITS:%.*]] = lshr <8 x i16> [[SUM]], splat (i16 8)
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = trunc <8 x i16> [[TMP15]] to <8 x i8>
+; CHECK-NEXT:    [[NARROWED:%.*]] = trunc <8 x i16> [[HIGH_BITS]] to <8 x i8>
+; CHECK-NEXT:    [[_MSPROP3:%.*]] = shufflevector <8 x i8> [[TMP3]], <8 x i8> [[_MSPROP2]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; CHECK-NEXT:    [[RES:%.*]] = shufflevector <8 x i8> [[LOW]], <8 x i8> [[NARROWED]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; CHECK-NEXT:    store <16 x i8> [[_MSPROP3]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i8> [[RES]]
+;
+  %tmpvar1 = load <8 x i16>, ptr %A
+  %tmpvar2 = load <8 x i16>, ptr %B
+  %sum = add <8 x i16> %tmpvar1, %tmpvar2
+  %high_bits = lshr <8 x i16> %sum, <i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8>
+  %narrowed = trunc <8 x i16> %high_bits to <8 x i8>
+  %res = shufflevector <8 x i8> %low, <8 x i8> %narrowed, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+  ret <16 x i8> %res
+}
+
+define <8 x i16> @addhn2_8h_natural(<4 x i16> %low, ptr %A, ptr %B) nounwind #0 {
+; CHECK-LABEL: define <8 x i16> @addhn2_8h_natural(
+; CHECK-SAME: <4 x i16> [[LOW:%.*]], ptr [[A:%.*]], ptr [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP4:%.*]], label [[TMP5:%.*]], !prof [[PROF1]]
+; CHECK:       4:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       5:
+; CHECK-NEXT:    [[TMPVAR1:%.*]] = load <4 x i32>, ptr [[A]], align 16
+; CHECK-NEXT:    [[TMP6:%.*]] = ptrtoint ptr [[A]] to i64
+; CHECK-NEXT:    [[TMP7:%.*]] = xor i64 [[TMP6]], 193514046488576
+; CHECK-NEXT:    [[TMP8:%.*]] = inttoptr i64 [[TMP7]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <4 x i32>, ptr [[TMP8]], align 16
+; CHECK-NEXT:    [[_MSCMP4:%.*]] = icmp ne i64 [[TMP2]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP4]], label [[TMP9:%.*]], label [[TMP10:%.*]], !prof [[PROF1]]
+; CHECK:       9:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       10:
+; CHECK-NEXT:    [[TMPVAR2:%.*]] = load <4 x i32>, ptr [[B]], align 16
+; CHECK-NEXT:    [[TMP11:%.*]] = ptrtoint ptr [[B]] to i64
+; CHECK-NEXT:    [[TMP12:%.*]] = xor i64 [[TMP11]], 193514046488576
+; CHECK-NEXT:    [[TMP13:%.*]] = inttoptr i64 [[TMP12]] to ptr
+; CHECK-NEXT:    [[_MSLD1:%.*]] = load <4 x i32>, ptr [[TMP13]], align 16
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <4 x i32> [[_MSLD]], [[_MSLD1]]
+; CHECK-NEXT:    [[SUM:%.*]] = add <4 x i32> [[TMPVAR1]], [[TMPVAR2]]
+; CHECK-NEXT:    [[TMP14:%.*]] = lshr <4 x i32> [[_MSPROP]], splat (i32 16)
+; CHECK-NEXT:    [[TMP15:%.*]] = or <4 x i32> [[TMP14]], zeroinitializer
+; CHECK-NEXT:    [[HIGH_BITS:%.*]] = lshr <4 x i32> [[SUM]], splat (i32 16)
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = trunc <4 x i32> [[TMP15]] to <4 x i16>
+; CHECK-NEXT:    [[NARROWED:%.*]] = trunc <4 x i32> [[HIGH_BITS]] to <4 x i16>
+; CHECK-NEXT:    [[_MSPROP3:%.*]] = shufflevector <4 x i16> [[TMP3]], <4 x i16> [[_MSPROP2]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:    [[RES:%.*]] = shufflevector <4 x i16> [[LOW]], <4 x i16> [[NARROWED]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:    store <8 x i16> [[_MSPROP3]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i16> [[RES]]
+;
+  %tmpvar1 = load <4 x i32>, ptr %A
+  %tmpvar2 = load <4 x i32>, ptr %B
+  %sum = add <4 x i32> %tmpvar1, %tmpvar2
+  %high_bits = lshr <4 x i32> %sum, <i32 16, i32 16, i32 16, i32 16>
+  %narrowed = trunc <4 x i32> %high_bits to <4 x i16>
+  %res = shufflevector <4 x i16> %low, <4 x i16> %narrowed, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+  ret <8 x i16> %res
+}
+
+define <4 x i32> @addhn2_4s_natural(<2 x i32> %low, ptr %A, ptr %B) nounwind #0 {
+; CHECK-LABEL: define <4 x i32> @addhn2_4s_natural(
+; CHECK-SAME: <2 x i32> [[LOW:%.*]], ptr [[A:%.*]], ptr [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <2 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP4:%.*]], label [[TMP5:%.*]], !prof [[PROF1]]
+; CHECK:       4:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       5:
+; CHECK-NEXT:    [[TMPVAR1:%.*]] = load <2 x i64>, ptr [[A]], align 16
+; CHECK-NEXT:    [[TMP6:%.*]] = ptrtoint ptr [[A]] to i64
+; CHECK-NEXT:    [[TMP7:%.*]] = xor i64 [[TMP6]], 193514046488576
+; CHECK-NEXT:    [[TMP8:%.*]] = inttoptr i64 [[TMP7]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <2 x i64>, ptr [[TMP8]], align 16
+; CHECK-NEXT:    [[_MSCMP4:%.*]] = icmp ne i64 [[TMP2]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP4]], label [[TMP9:%.*]], label [[TMP10:%.*]], !prof [[PROF1]]
+; CHECK:       9:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       10:
+; CHECK-NEXT:    [[TMPVAR2:%.*]] = load <2 x i64>, ptr [[B]], align 16
+; CHECK-NEXT:    [[TMP11:%.*]] = ptrtoint ptr [[B]] to i64
+; CHECK-NEXT:    [[TMP12:%.*]] = xor i64 [[TMP11]], 193514046488576
+; CHECK-NEXT:    [[TMP13:%.*]] = inttoptr i64 [[TMP12]] to ptr
+; CHECK-NEXT:    [[_MSLD1:%.*]] = load <2 x i64>, ptr [[TMP13]], align 16
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <2 x i64> [[_MSLD]], [[_MSLD1]]
+; CHECK-NEXT:    [[SUM:%.*]] = add <2 x i64> [[TMPVAR1]], [[TMPVAR2]]
+; CHECK-NEXT:    [[TMP14:%.*]] = lshr <2 x i64> [[_MSPROP]], splat (i64 32)
+; CHECK-NEXT:    [[TMP15:%.*]] = or <2 x i64> [[TMP14]], zeroinitializer
+; CHECK-NEXT:    [[HIGH_BITS:%.*]] = lshr <2 x i64> [[SUM]], splat (i64 32)
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = trunc <2 x i64> [[TMP15]] to <2 x i32>
+; CHECK-NEXT:    [[NARROWED:%.*]] = trunc <2 x i64> [[HIGH_BITS]] to <2 x i32>
+; CHECK-NEXT:    [[_MSPROP3:%.*]] = shufflevector <2 x i32> [[TMP3]], <2 x i32> [[_MSPROP2]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[RES:%.*]] = shufflevector <2 x i32> [[LOW]], <2 x i32> [[NARROWED]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    store <4 x i32> [[_MSPROP3]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i32> [[RES]]
+;
+  %tmpvar1 = load <2 x i64>, ptr %A
+  %tmpvar2 = load <2 x i64>, ptr %B
+  %sum = add <2 x i64> %tmpvar1, %tmpvar2
+  %high_bits = lshr <2 x i64> %sum, <i64 32, i64 32>
+  %narrowed = trunc <2 x i64> %high_bits to <2 x i32>
+  %res = shufflevector <2 x i32> %low, <2 x i32> %narrowed, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  ret <4 x i32> %res
+}
+
+define <4 x i32> @addhn_addhn2_4s(ptr %A, ptr %B, ptr %C, ptr %D) nounwind #0 {
+; CHECK-LABEL: define <4 x i32> @addhn_addhn2_4s(
+; CHECK-SAME: ptr [[A:%.*]], ptr [[B:%.*]], ptr [[C:%.*]], ptr [[D:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 24) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP5:%.*]], label [[TMP6:%.*]], !prof [[PROF1]]
+; CHECK:       5:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       6:
+; CHECK-NEXT:    [[TMPVAR1:%.*]] = load <2 x i64>, ptr [[A]], align 16
+; CHECK-NEXT:    [[TMP7:%.*]] = ptrtoint ptr [[A]] to i64
+; CHECK-NEXT:    [[TMP8:%.*]] = xor i64 [[TMP7]], 193514046488576
+; CHECK-NEXT:    [[TMP9:%.*]] = inttoptr i64 [[TMP8]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <2 x i64>, ptr [[TMP9]], align 16
+; CHECK-NEXT:    [[_MSCMP8:%.*]] = icmp ne i64 [[TMP2]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP8]], label [[TMP10:%.*]], label [[TMP11:%.*]], !prof [[PROF1]]
+; CHECK:       10:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       11:
+; CHECK-NEXT:    [[TMPVAR2:%.*]] = load <2 x i64>, ptr [[B]], align 16
+; CHECK-NEXT:    [[TMP12:%.*]] = ptrtoint ptr [[B]] to i64
+; CHECK-NEXT:    [[TMP13:%.*]] = xor i64 [[TMP12]], 193514046488576
+; CHECK-NEXT:    [[TMP14:%.*]] = inttoptr i64 [[TMP13]] to ptr
+; CHECK-NEXT:    [[_MSLD1:%.*]] = load <2 x i64>, ptr [[TMP14]], align 16
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <2 x i64> [[_MSLD]], [[_MSLD1]]
+; CHECK-NEXT:    [[SUM1:%.*]] = add <2 x i64> [[TMPVAR1]], [[TMPVAR2]]
+; CHECK-NEXT:    [[TMP15:%.*]] = lshr <2 x i64> [[_MSPROP]], splat (i64 32)
+; CHECK-NEXT:    [[TMP16:%.*]] = or <2 x i64> [[TMP15]], zeroinitializer
+; CHECK-NEXT:    [[LOW_BITS:%.*]] = lshr <2 x i64> [[SUM1]], splat (i64 32)
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = trunc <2 x i64> [[TMP16]] to <2 x i32>
+; CHECK-NEXT:    [[NARROWED1:%.*]] = trunc <2 x i64> [[LOW_BITS]] to <2 x i32>
+; CHECK-NEXT:    [[_MSCMP9:%.*]] = icmp ne i64 [[TMP3]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP9]], label [[TMP17:%.*]], label [[TMP18:%.*]], !prof [[PROF1]]
+; CHECK:       17:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       18:
+; CHECK-NEXT:    [[TMPVAR3:%.*]] = load <2 x i64>, ptr [[C]], align 16
+; CHECK-NEXT:    [[TMP19:%.*]] = ptrtoint ptr [[C]] to i64
+; CHECK-NEXT:    [[TMP20:%.*]] = xor i64 [[TMP19]], 193514046488576
+; CHECK-NEXT:    [[TMP21:%.*]] = inttoptr i64 [[TMP20]] to ptr
+; CHECK-NEXT:    [[_MSLD3:%.*]] = load <2 x i64>, ptr [[TMP21]], align 16
+; CHECK-NEXT:    [[_MSCMP10:%.*]] = icmp ne i64 [[TMP4]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP10]], label [[TMP22:%.*]], label [[TMP23:%.*]], !prof [[PROF1]]
+; CHECK:       22:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       23:
+; CHECK-NEXT:    [[TMPVAR4:%.*]] = load <2 x i64>, ptr [[D]], align 16
+; CHECK-NEXT:    [[TMP24:%.*]] = ptrtoint ptr [[D]] to i64
+; CHECK-NEXT:    [[TMP25:%.*]] = xor i64 [[TMP24]], 193514046488576
+; CHECK-NEXT:    [[TMP26:%.*]] = inttoptr i64 [[TMP25]] to ptr
+; CHECK-NEXT:    [[_MSLD4:%.*]] = load <2 x i64>, ptr [[TMP26]], align 16
+; CHECK-NEXT:    [[_MSPROP5:%.*]] = or <2 x i64> [[_MSLD3]], [[_MSLD4]]
+; CHECK-NEXT:    [[SUM2:%.*]] = add <2 x i64> [[TMPVAR3]], [[TMPVAR4]]
+; CHECK-NEXT:    [[TMP27:%.*]] = lshr <2 x i64> [[_MSPROP]], splat (i64 32)
+; CHECK-NEXT:    [[TMP28:%.*]] = or <2 x i64> [[TMP27]], zeroinitializer
+; CHECK-NEXT:    [[HIGH_BITS:%.*]] = lshr <2 x i64> [[SUM1]], splat (i64 32)
+; CHECK-NEXT:    [[_MSPROP6:%.*]] = trunc <2 x i64> [[TMP28]] to <2 x i32>
+; CHECK-NEXT:    [[NARROWED2:%.*]] = trunc <2 x i64> [[HIGH_BITS]] to <2 x i32>
+; CHECK-NEXT:    [[_MSPROP7:%.*]] = shufflevector <2 x i32> [[_MSPROP2]], <2 x i32> [[_MSPROP6]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[RES:%.*]] = shufflevector <2 x i32> [[NARROWED1]], <2 x i32> [[NARROWED2]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    store <4 x i32> [[_MSPROP7]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i32> [[RES]]
+;
+  %tmpvar1 = load <2 x i64>, ptr %A
+  %tmpvar2 = load <2 x i64>, ptr %B
+  %sum1 = add <2 x i64> %tmpvar1, %tmpvar2
+  %low_bits = lshr <2 x i64> %sum1, <i64 32, i64 32>
+  %narrowed1 = trunc <2 x i64> %low_bits to <2 x i32>
+  %tmpvar3 = load <2 x i64>, ptr %C
+  %tmpvar4 = load <2 x i64>, ptr %D
+  %sum2 = add <2 x i64> %tmpvar3, %tmpvar4
+  %high_bits = lshr <2 x i64> %sum1, <i64 32, i64 32>
+  %narrowed2 = trunc <2 x i64> %high_bits to <2 x i32>
+  %res = shufflevector <2 x i32> %narrowed1, <2 x i32> %narrowed2, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  ret <4 x i32> %res
+}
+
+define <8 x i8> @subhn8b_natural(ptr %A, ptr %B) nounwind #0 {
+; CHECK-LABEL: define <8 x i8> @subhn8b_natural(
+; CHECK-SAME: ptr [[A:%.*]], ptr [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP3:%.*]], label [[TMP4:%.*]], !prof [[PROF1]]
+; CHECK:       3:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       4:
+; CHECK-NEXT:    [[TMPVAR1:%.*]] = load <8 x i16>, ptr [[A]], align 16
+; CHECK-NEXT:    [[TMP5:%.*]] = ptrtoint ptr [[A]] to i64
+; CHECK-NEXT:    [[TMP6:%.*]] = xor i64 [[TMP5]], 193514046488576
+; CHECK-NEXT:    [[TMP7:%.*]] = inttoptr i64 [[TMP6]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <8 x i16>, ptr [[TMP7]], align 16
+; CHECK-NEXT:    [[_MSCMP3:%.*]] = icmp ne i64 [[TMP2]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP3]], label [[TMP8:%.*]], label [[TMP9:%.*]], !prof [[PROF1]]
+; CHECK:       8:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       9:
+; CHECK-NEXT:    [[TMPVAR2:%.*]] = load <8 x i16>, ptr [[B]], align 16
+; CHECK-NEXT:    [[TMP10:%.*]] = ptrtoint ptr [[B]] to i64
+; CHECK-NEXT:    [[TMP11:%.*]] = xor i64 [[TMP10]], 193514046488576
+; CHECK-NEXT:    [[TMP12:%.*]] = inttoptr i64 [[TMP11]] to ptr
+; CHECK-NEXT:    [[_MSLD1:%.*]] = load <8 x i16>, ptr [[TMP12]], align 16
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <8 x i16> [[_MSLD]], [[_MSLD1]]
+; CHECK-NEXT:    [[DIFF:%.*]] = sub <8 x i16> [[TMPVAR1]], [[TMPVAR2]]
+; CHECK-NEXT:    [[TMP13:%.*]] = lshr <8 x i16> [[_MSPROP]], splat (i16 8)
+; CHECK-NEXT:    [[TMP14:%.*]] = or <8 x i16> [[TMP13]], zeroinitializer
+; CHECK-NEXT:    [[HIGH_BITS:%.*]] = lshr <8 x i16> [[DIFF]], splat (i16 8)
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = trunc <8 x i16> [[TMP14]] to <8 x i8>
+; CHECK-NEXT:    [[NARROWED:%.*]] = trunc <8 x i16> [[HIGH_BITS]] to <8 x i8>
+; CHECK-NEXT:    store <8 x i8> [[_MSPROP2]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i8> [[NARROWED]]
+;
+  %tmpvar1 = load <8 x i16>, ptr %A
+  %tmpvar2 = load <8 x i16>, ptr %B
+  %diff = sub <8 x i16> %tmpvar1, %tmpvar2
+  %high_bits = lshr <8 x i16> %diff, <i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8>
+  %narrowed = trunc <8 x i16> %high_bits to <8 x i8>
+  ret <8 x i8> %narrowed
+}
+
+define <4 x i16> @subhn4h_natural(ptr %A, ptr %B) nounwind #0 {
+; CHECK-LABEL: define <4 x i16> @subhn4h_natural(
+; CHECK-SAME: ptr [[A:%.*]], ptr [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP3:%.*]], label [[TMP4:%.*]], !prof [[PROF1]]
+; CHECK:       3:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       4:
+; CHECK-NEXT:    [[TMPVAR1:%.*]] = load <4 x i32>, ptr [[A]], align 16
+; CHECK-NEXT:    [[TMP5:%.*]] = ptrtoint ptr [[A]] to i64
+; CHECK-NEXT:    [[TMP6:%.*]] = xor i64 [[TMP5]], 193514046488576
+; CHECK-NEXT:    [[TMP7:%.*]] = inttoptr i64 [[TMP6]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <4 x i32>, ptr [[TMP7]], align 16
+; CHECK-NEXT:    [[_MSCMP3:%.*]] = icmp ne i64 [[TMP2]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP3]], label [[TMP8:%.*]], label [[TMP9:%.*]], !prof [[PROF1]]
+; CHECK:       8:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       9:
+; CHECK-NEXT:    [[TMPVAR2:%.*]] = load <4 x i32>, ptr [[B]], align 16
+; CHECK-NEXT:    [[TMP10:%.*]] = ptrtoint ptr [[B]] to i64
+; CHECK-NEXT:    [[TMP11:%.*]] = xor i64 [[TMP10]], 193514046488576
+; CHECK-NEXT:    [[TMP12:%.*]] = inttoptr i64 [[TMP11]] to ptr
+; CHECK-NEXT:    [[_MSLD1:%.*]] = load <4 x i32>, ptr [[TMP12]], align 16
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <4 x i32> [[_MSLD]], [[_MSLD1]]
+; CHECK-NEXT:    [[DIFF:%.*]] = sub <4 x i32> [[TMPVAR1]], [[TMPVAR2]]
+; CHECK-NEXT:    [[TMP13:%.*]] = lshr <4 x i32> [[_MSPROP]], splat (i32 16)
+; CHECK-NEXT:    [[TMP14:%.*]] = or <4 x i32> [[TMP13]], zeroinitializer
+; CHECK-NEXT:    [[HIGH_BITS:%.*]] = lshr <4 x i32> [[DIFF]], splat (i32 16)
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = trunc <4 x i32> [[TMP14]] to <4 x i16>
+; CHECK-NEXT:    [[NARROWED:%.*]] = trunc <4 x i32> [[HIGH_BITS]] to <4 x i16>
+; CHECK-NEXT:    store <4 x i16> [[_MSPROP2]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i16> [[NARROWED]]
+;
+  %tmpvar1 = load <4 x i32>, ptr %A
+  %tmpvar2 = load <4 x i32>, ptr %B
+  %diff = sub <4 x i32> %tmpvar1, %tmpvar2
+  %high_bits = lshr <4 x i32> %diff, <i32 16, i32 16, i32 16, i32 16>
+  %narrowed = trunc <4 x i32> %high_bits to <4 x i16>
+  ret <4 x i16> %narrowed
+}
+
+define <2 x i32> @subhn2s_natural(ptr %A, ptr %B) nounwind #0 {
+; CHECK-LABEL: define <2 x i32> @subhn2s_natural(
+; CHECK-SAME: ptr [[A:%.*]], ptr [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP3:%.*]], label [[TMP4:%.*]], !prof [[PROF1]]
+; CHECK:       3:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       4:
+; CHECK-NEXT:    [[TMPVAR1:%.*]] = load <2 x i64>, ptr [[A]], align 16
+; CHECK-NEXT:    [[TMP5:%.*]] = ptrtoint ptr [[A]] to i64
+; CHECK-NEXT:    [[TMP6:%.*]] = xor i64 [[TMP5]], 193514046488576
+; CHECK-NEXT:    [[TMP7:%.*]] = inttoptr i64 [[TMP6]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <2 x i64>, ptr [[TMP7]], align 16
+; CHECK-NEXT:    [[_MSCMP3:%.*]] = icmp ne i64 [[TMP2]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP3]], label [[TMP8:%.*]], label [[TMP9:%.*]], !prof [[PROF1]]
+; CHECK:       8:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       9:
+; CHECK-NEXT:    [[TMPVAR2:%.*]] = load <2 x i64>, ptr [[B]], align 16
+; CHECK-NEXT:    [[TMP10:%.*]] = ptrtoint ptr [[B]] to i64
+; CHECK-NEXT:    [[TMP11:%.*]] = xor i64 [[TMP10]], 193514046488576
+; CHECK-NEXT:    [[TMP12:%.*]] = inttoptr i64 [[TMP11]] to ptr
+; CHECK-NEXT:    [[_MSLD1:%.*]] = load <2 x i64>, ptr [[TMP12]], align 16
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <2 x i64> [[_MSLD]], [[_MSLD1]]
+; CHECK-NEXT:    [[DIFF:%.*]] = sub <2 x i64> [[TMPVAR1]], [[TMPVAR2]]
+; CHECK-NEXT:    [[TMP13:%.*]] = lshr <2 x i64> [[_MSPROP]], splat (i64 32)
+; CHECK-NEXT:    [[TMP14:%.*]] = or <2 x i64> [[TMP13]], zeroinitializer
+; CHECK-NEXT:    [[HIGH_BITS:%.*]] = lshr <2 x i64> [[DIFF]], splat (i64 32)
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = trunc <2 x i64> [[TMP14]] to <2 x i32>
+; CHECK-NEXT:    [[NARROWED:%.*]] = trunc <2 x i64> [[HIGH_BITS]] to <2 x i32>
+; CHECK-NEXT:    store <2 x i32> [[_MSPROP2]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x i32> [[NARROWED]]
+;
+  %tmpvar1 = load <2 x i64>, ptr %A
+  %tmpvar2 = load <2 x i64>, ptr %B
+  %diff = sub <2 x i64> %tmpvar1, %tmpvar2
+  %high_bits = lshr <2 x i64> %diff, <i64 32, i64 32>
+  %narrowed = trunc <2 x i64> %high_bits to <2 x i32>
+  ret <2 x i32> %narrowed
+}
+
+define <16 x i8> @subhn2_16b_natural(<8 x i8> %low, ptr %A, ptr %B) nounwind #0 {
+; CHECK-LABEL: define <16 x i8> @subhn2_16b_natural(
+; CHECK-SAME: <8 x i8> [[LOW:%.*]], ptr [[A:%.*]], ptr [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i8>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP4:%.*]], label [[TMP5:%.*]], !prof [[PROF1]]
+; CHECK:       4:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       5:
+; CHECK-NEXT:    [[TMPVAR1:%.*]] = load <8 x i16>, ptr [[A]], align 16
+; CHECK-NEXT:    [[TMP6:%.*]] = ptrtoint ptr [[A]] to i64
+; CHECK-NEXT:    [[TMP7:%.*]] = xor i64 [[TMP6]], 193514046488576
+; CHECK-NEXT:    [[TMP8:%.*]] = inttoptr i64 [[TMP7]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <8 x i16>, ptr [[TMP8]], align 16
+; CHECK-NEXT:    [[_MSCMP4:%.*]] = icmp ne i64 [[TMP2]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP4]], label [[TMP9:%.*]], label [[TMP10:%.*]], !prof [[PROF1]]
+; CHECK:       9:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       10:
+; CHECK-NEXT:    [[TMPVAR2:%.*]] = load <8 x i16>, ptr [[B]], align 16
+; CHECK-NEXT:    [[TMP11:%.*]] = ptrtoint ptr [[B]] to i64
+; CHECK-NEXT:    [[TMP12:%.*]] = xor i64 [[TMP11]], 193514046488576
+; CHECK-NEXT:    [[TMP13:%.*]] = inttoptr i64 [[TMP12]] to ptr
+; CHECK-NEXT:    [[_MSLD1:%.*]] = load <8 x i16>, ptr [[TMP13]], align 16
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <8 x i16> [[_MSLD]], [[_MSLD1]]
+; CHECK-NEXT:    [[DIFF:%.*]] = sub <8 x i16> [[TMPVAR1]], [[TMPVAR2]]
+; CHECK-NEXT:    [[TMP14:%.*]] = lshr <8 x i16> [[_MSPROP]], splat (i16 8)
+; CHECK-NEXT:    [[TMP15:%.*]] = or <8 x i16> [[TMP14]], zeroinitializer
+; CHECK-NEXT:    [[HIGH_BITS:%.*]] = lshr <8 x i16> [[DIFF]], splat (i16 8)
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = trunc <8 x i16> [[TMP15]] to <8 x i8>
+; CHECK-NEXT:    [[NARROWED:%.*]] = trunc <8 x i16> [[HIGH_BITS]] to <8 x i8>
+; CHECK-NEXT:    [[_MSPROP3:%.*]] = shufflevector <8 x i8> [[TMP3]], <8 x i8> [[_MSPROP2]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; CHECK-NEXT:    [[RES:%.*]] = shufflevector <8 x i8> [[LOW]], <8 x i8> [[NARROWED]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; CHECK-NEXT:    store <16 x i8> [[_MSPROP3]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i8> [[RES]]
+;
+  %tmpvar1 = load <8 x i16>, ptr %A
+  %tmpvar2 = load <8 x i16>, ptr %B
+  %diff = sub <8 x i16> %tmpvar1, %tmpvar2
+  %high_bits = lshr <8 x i16> %diff, <i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8>
+  %narrowed = trunc <8 x i16> %high_bits to <8 x i8>
+  %res = shufflevector <8 x i8> %low, <8 x i8> %narrowed, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+  ret <16 x i8> %res
+}
+
+define <8 x i16> @subhn2_8h_natural(<4 x i16> %low, ptr %A, ptr %B) nounwind #0 {
+; CHECK-LABEL: define <8 x i16> @subhn2_8h_natural(
+; CHECK-SAME: <4 x i16> [[LOW:%.*]], ptr [[A:%.*]], ptr [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP4:%.*]], label [[TMP5:%.*]], !prof [[PROF1]]
+; CHECK:       4:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       5:
+; CHECK-NEXT:    [[TMPVAR1:%.*]] = load <4 x i32>, ptr [[A]], align 16
+; CHECK-NEXT:    [[TMP6:%.*]] = ptrtoint ptr [[A]] to i64
+; CHECK-NEXT:    [[TMP7:%.*]] = xor i64 [[TMP6]], 193514046488576
+; CHECK-NEXT:    [[TMP8:%.*]] = inttoptr i64 [[TMP7]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <4 x i32>, ptr [[TMP8]], align 16
+; CHECK-NEXT:    [[_MSCMP4:%.*]] = icmp ne i64 [[TMP2]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP4]], label [[TMP9:%.*]], label [[TMP10:%.*]], !prof [[PROF1]]
+; CHECK:       9:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       10:
+; CHECK-NEXT:    [[TMPVAR2:%.*]] = load <4 x i32>, ptr [[B]], align 16
+; CHECK-NEXT:    [[TMP11:%.*]] = ptrtoint ptr [[B]] to i64
+; CHECK-NEXT:    [[TMP12:%.*]] = xor i64 [[TMP11]], 193514046488576
+; CHECK-NEXT:    [[TMP13:%.*]] = inttoptr i64 [[TMP12]] to ptr
+; CHECK-NEXT:    [[_MSLD1:%.*]] = load <4 x i32>, ptr [[TMP13]], align 16
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <4 x i32> [[_MSLD]], [[_MSLD1]]
+; CHECK-NEXT:    [[DIFF:%.*]] = sub <4 x i32> [[TMPVAR1]], [[TMPVAR2]]
+; CHECK-NEXT:    [[TMP14:%.*]] = lshr <4 x i32> [[_MSPROP]], splat (i32 16)
+; CHECK-NEXT:    [[TMP15:%.*]] = or <4 x i32> [[TMP14]], zeroinitializer
+; CHECK-NEXT:    [[HIGH_BITS:%.*]] = lshr <4 x i32> [[DIFF]], splat (i32 16)
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = trunc <4 x i32> [[TMP15]] to <4 x i16>
+; CHECK-NEXT:    [[NARROWED:%.*]] = trunc <4 x i32> [[HIGH_BITS]] to <4 x i16>
+; CHECK-NEXT:    [[_MSPROP3:%.*]] = shufflevector <4 x i16> [[TMP3]], <4 x i16> [[_MSPROP2]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:    [[RES:%.*]] = shufflevector <4 x i16> [[LOW]], <4 x i16> [[NARROWED]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:    store <8 x i16> [[_MSPROP3]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i16> [[RES]]
+;
+  %tmpvar1 = load <4 x i32>, ptr %A
+  %tmpvar2 = load <4 x i32>, ptr %B
+  %diff = sub <4 x i32> %tmpvar1, %tmpvar2
+  %high_bits = lshr <4 x i32> %diff, <i32 16, i32 16, i32 16, i32 16>
+  %narrowed = trunc <4 x i32> %high_bits to <4 x i16>
+  %res = shufflevector <4 x i16> %low, <4 x i16> %narrowed, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+  ret <8 x i16> %res
+}
+
+define <4 x i32> @subhn2_4s_natural(<2 x i32> %low, ptr %A, ptr %B) nounwind #0 {
+; CHECK-LABEL: define <4 x i32> @subhn2_4s_natural(
+; CHECK-SAME: <2 x i32> [[LOW:%.*]], ptr [[A:%.*]], ptr [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <2 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP4:%.*]], label [[TMP5:%.*]], !prof [[PROF1]]
+; CHECK:       4:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       5:
+; CHECK-NEXT:    [[TMPVAR1:%.*]] = load <2 x i64>, ptr [[A]], align 16
+; CHECK-NEXT:    [[TMP6:%.*]] = ptrtoint ptr [[A]] to i64
+; CHECK-NEXT:    [[TMP7:%.*]] = xor i64 [[TMP6]], 193514046488576
+; CHECK-NEXT:    [[TMP8:%.*]] = inttoptr i64 [[TMP7]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <2 x i64>, ptr [[TMP8]], align 16
+; CHECK-NEXT:    [[_MSCMP4:%.*]] = icmp ne i64 [[TMP2]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP4]], label [[TMP9:%.*]], label [[TMP10:%.*]], !prof [[PROF1]]
+; CHECK:       9:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       10:
+; CHECK-NEXT:    [[TMPVAR2:%.*]] = load <2 x i64>, ptr [[B]], align 16
+; CHECK-NEXT:    [[TMP11:%.*]] = ptrtoint ptr [[B]] to i64
+; CHECK-NEXT:    [[TMP12:%.*]] = xor i64 [[TMP11]], 193514046488576
+; CHECK-NEXT:    [[TMP13:%.*]] = inttoptr i64 [[TMP12]] to ptr
+; CHECK-NEXT:    [[_MSLD1:%.*]] = load <2 x i64>, ptr [[TMP13]], align 16
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <2 x i64> [[_MSLD]], [[_MSLD1]]
+; CHECK-NEXT:    [[DIFF:%.*]] = sub <2 x i64> [[TMPVAR1]], [[TMPVAR2]]
+; CHECK-NEXT:    [[TMP14:%.*]] = lshr <2 x i64> [[_MSPROP]], splat (i64 32)
+; CHECK-NEXT:    [[TMP15:%.*]] = or <2 x i64> [[TMP14]], zeroinitializer
+; CHECK-NEXT:    [[HIGH_BITS:%.*]] = lshr <2 x i64> [[DIFF]], splat (i64 32)
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = trunc <2 x i64> [[TMP15]] to <2 x i32>
+; CHECK-NEXT:    [[NARROWED:%.*]] = trunc <2 x i64> [[HIGH_BITS]] to <2 x i32>
+; CHECK-NEXT:    [[_MSPROP3:%.*]] = shufflevector <2 x i32> [[TMP3]], <2 x i32> [[_MSPROP2]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[RES:%.*]] = shufflevector <2 x i32> [[LOW]], <2 x i32> [[NARROWED]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    store <4 x i32> [[_MSPROP3]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i32> [[RES]]
+;
+  %tmpvar1 = load <2 x i64>, ptr %A
+  %tmpvar2 = load <2 x i64>, ptr %B
+  %diff = sub <2 x i64> %tmpvar1, %tmpvar2
+  %high_bits = lshr <2 x i64> %diff, <i64 32, i64 32>
+  %narrowed = trunc <2 x i64> %high_bits to <2 x i32>
+  %res = shufflevector <2 x i32> %low, <2 x i32> %narrowed, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  ret <4 x i32> %res
+}
+
+attributes #0 = { sanitize_memory }
+;.
+; CHECK: [[PROF1]] = !{!"branch_weights", i32 1, i32 1048575}
+;.
diff --git a/llvm/test/Instrumentation/MemorySanitizer/AArch64/arm64-vaddlv.ll b/llvm/test/Instrumentation/MemorySanitizer/AArch64/arm64-vaddlv.ll
new file mode 100644
index 0000000000000..33a54080609fe
--- /dev/null
+++ b/llvm/test/Instrumentation/MemorySanitizer/AArch64/arm64-vaddlv.ll
@@ -0,0 +1,47 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 4
+; RUN: opt < %s -passes=msan -S | FileCheck %s
+;
+; Forked from llvm/test/CodeGen/AArch64/arm64-vaddlv.ll
+
+target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
+target triple = "aarch64--linux-android9001"
+
+define i64 @test_vaddlv_s32(<2 x i32> %a1) nounwind readnone #0 {
+; CHECK-LABEL: define i64 @test_vaddlv_s32(
+; CHECK-SAME: <2 x i32> [[A1:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load <2 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @llvm.vector.reduce.or.v2i32(<2 x i32> [[TMP0]])
+; CHECK-NEXT:    [[TMP2:%.*]] = zext i32 [[TMP1]] to i64
+; CHECK-NEXT:    [[VADDLV_I:%.*]] = tail call i64 @llvm.aarch64.neon.saddlv.i64.v2i32(<2 x i32> [[A1]]) #[[ATTR2:[0-9]+]]
+; CHECK-NEXT:    store i64 [[TMP2]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i64 [[VADDLV_I]]
+;
+entry:
+  %vaddlv.i = tail call i64 @llvm.aarch64.neon.saddlv.i64.v2i32(<2 x i32> %a1) nounwind
+  ret i64 %vaddlv.i
+}
+
+define i64 @test_vaddlv_u32(<2 x i32> %a1) nounwind readnone #0 {
+; CHECK-LABEL: define i64 @test_vaddlv_u32(
+; CHECK-SAME: <2 x i32> [[A1:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load <2 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @llvm.vector.reduce.or.v2i32(<2 x i32> [[TMP0]])
+; CHECK-NEXT:    [[TMP2:%.*]] = zext i32 [[TMP1]] to i64
+; CHECK-NEXT:    [[VADDLV_I:%.*]] = tail call i64 @llvm.aarch64.neon.uaddlv.i64.v2i32(<2 x i32> [[A1]]) #[[ATTR2]]
+; CHECK-NEXT:    store i64 [[TMP2]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i64 [[VADDLV_I]]
+;
+entry:
+  %vaddlv.i = tail call i64 @llvm.aarch64.neon.uaddlv.i64.v2i32(<2 x i32> %a1) nounwind
+  ret i64 %vaddlv.i
+}
+
+declare i64 @llvm.aarch64.neon.uaddlv.i64.v2i32(<2 x i32>) nounwind readnone
+
+declare i64 @llvm.aarch64.neon.saddlv.i64.v2i32(<2 x i32>) nounwind readnone
+
+attributes #0 = { sanitize_memory }
diff --git a/llvm/test/Instrumentation/MemorySanitizer/AArch64/arm64-vaddv.ll b/llvm/test/Instrumentation/MemorySanitizer/AArch64/arm64-vaddv.ll
new file mode 100644
index 0000000000000..3a2ecfefd209e
--- /dev/null
+++ b/llvm/test/Instrumentation/MemorySanitizer/AArch64/arm64-vaddv.ll
@@ -0,0 +1,727 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 4
+; RUN: opt < %s -passes=msan -S | FileCheck %s
+;
+; Forked from llvm/test/CodeGen/AArch64/arm64-vaddv.ll
+
+target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
+target triple = "aarch64--linux-android9001"
+
+define signext i8 @test_vaddv_s8(<8 x i8> %a1) #0 {
+; CHECK-LABEL: define signext i8 @test_vaddv_s8(
+; CHECK-SAME: <8 x i8> [[A1:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load <8 x i8>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP1:%.*]] = call i8 @llvm.vector.reduce.or.v8i8(<8 x i8> [[TMP0]])
+; CHECK-NEXT:    [[TMP2:%.*]] = zext i8 [[TMP1]] to i32
+; CHECK-NEXT:    [[VADDV_I:%.*]] = tail call i32 @llvm.aarch64.neon.saddv.i32.v8i8(<8 x i8> [[A1]])
+; CHECK-NEXT:    [[_MSPROP:%.*]] = trunc i32 [[TMP2]] to i8
+; CHECK-NEXT:    [[TMP4:%.*]] = trunc i32 [[VADDV_I]] to i8
+; CHECK-NEXT:    store i8 [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i8 [[TMP4]]
+;
+entry:
+  %vaddv.i = tail call i32 @llvm.aarch64.neon.saddv.i32.v8i8(<8 x i8> %a1)
+  %0 = trunc i32 %vaddv.i to i8
+  ret i8 %0
+}
+
+define <8 x i8> @test_vaddv_s8_used_by_laneop(<8 x i8> %a1, <8 x i8> %a2) #0 {
+; CHECK-LABEL: define <8 x i8> @test_vaddv_s8_used_by_laneop(
+; CHECK-SAME: <8 x i8> [[A1:%.*]], <8 x i8> [[A2:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load <8 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i8>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = call i8 @llvm.vector.reduce.or.v8i8(<8 x i8> [[TMP0]])
+; CHECK-NEXT:    [[TMP3:%.*]] = zext i8 [[TMP2]] to i32
+; CHECK-NEXT:    [[TMP5:%.*]] = tail call i32 @llvm.aarch64.neon.saddv.i32.v8i8(<8 x i8> [[A2]])
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = trunc i32 [[TMP3]] to i8
+; CHECK-NEXT:    [[TMP6:%.*]] = trunc i32 [[TMP5]] to i8
+; CHECK-NEXT:    [[_MSPROP:%.*]] = insertelement <8 x i8> [[TMP1]], i8 [[_MSPROP1]], i32 3
+; CHECK-NEXT:    [[TMP7:%.*]] = insertelement <8 x i8> [[A1]], i8 [[TMP6]], i32 3
+; CHECK-NEXT:    store <8 x i8> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i8> [[TMP7]]
+;
+entry:
+  %0 = tail call i32 @llvm.aarch64.neon.saddv.i32.v8i8(<8 x i8> %a2)
+  %1 = trunc i32 %0 to i8
+  %2 = insertelement <8 x i8> %a1, i8 %1, i32 3
+  ret <8 x i8> %2
+}
+
+define signext i16 @test_vaddv_s16(<4 x i16> %a1) #0 {
+; CHECK-LABEL: define signext i16 @test_vaddv_s16(
+; CHECK-SAME: <4 x i16> [[A1:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load <4 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP1:%.*]] = call i16 @llvm.vector.reduce.or.v4i16(<4 x i16> [[TMP0]])
+; CHECK-NEXT:    [[TMP2:%.*]] = zext i16 [[TMP1]] to i32
+; CHECK-NEXT:    [[VADDV_I:%.*]] = tail call i32 @llvm.aarch64.neon.saddv.i32.v4i16(<4 x i16> [[A1]])
+; CHECK-NEXT:    [[_MSPROP:%.*]] = trunc i32 [[TMP2]] to i16
+; CHECK-NEXT:    [[TMP4:%.*]] = trunc i32 [[VADDV_I]] to i16
+; CHECK-NEXT:    store i16 [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i16 [[TMP4]]
+;
+entry:
+  %vaddv.i = tail call i32 @llvm.aarch64.neon.saddv.i32.v4i16(<4 x i16> %a1)
+  %0 = trunc i32 %vaddv.i to i16
+  ret i16 %0
+}
+
+define <4 x i16> @test_vaddv_s16_used_by_laneop(<4 x i16> %a1, <4 x i16> %a2) #0 {
+; CHECK-LABEL: define <4 x i16> @test_vaddv_s16_used_by_laneop(
+; CHECK-SAME: <4 x i16> [[A1:%.*]], <4 x i16> [[A2:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load <4 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = call i16 @llvm.vector.reduce.or.v4i16(<4 x i16> [[TMP0]])
+; CHECK-NEXT:    [[TMP3:%.*]] = zext i16 [[TMP2]] to i32
+; CHECK-NEXT:    [[TMP5:%.*]] = tail call i32 @llvm.aarch64.neon.saddv.i32.v4i16(<4 x i16> [[A2]])
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = trunc i32 [[TMP3]] to i16
+; CHECK-NEXT:    [[TMP6:%.*]] = trunc i32 [[TMP5]] to i16
+; CHECK-NEXT:    [[_MSPROP:%.*]] = insertelement <4 x i16> [[TMP1]], i16 [[_MSPROP1]], i32 3
+; CHECK-NEXT:    [[TMP7:%.*]] = insertelement <4 x i16> [[A1]], i16 [[TMP6]], i32 3
+; CHECK-NEXT:    store <4 x i16> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i16> [[TMP7]]
+;
+entry:
+  %0 = tail call i32 @llvm.aarch64.neon.saddv.i32.v4i16(<4 x i16> %a2)
+  %1 = trunc i32 %0 to i16
+  %2 = insertelement <4 x i16> %a1, i16 %1, i32 3
+  ret <4 x i16> %2
+}
+
+define i32 @test_vaddv_s32(<2 x i32> %a1) #0 {
+; CHECK-LABEL: define i32 @test_vaddv_s32(
+; CHECK-SAME: <2 x i32> [[A1:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load <2 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @llvm.vector.reduce.or.v2i32(<2 x i32> [[TMP0]])
+; CHECK-NEXT:    [[VADDV_I:%.*]] = tail call i32 @llvm.aarch64.neon.saddv.i32.v2i32(<2 x i32> [[A1]])
+; CHECK-NEXT:    store i32 [[TMP1]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i32 [[VADDV_I]]
+;
+; 2 x i32 is not supported by the ISA, thus, this is a special case
+entry:
+  %vaddv.i = tail call i32 @llvm.aarch64.neon.saddv.i32.v2i32(<2 x i32> %a1)
+  ret i32 %vaddv.i
+}
+
+define <2 x i32> @test_vaddv_s32_used_by_laneop(<2 x i32> %a1, <2 x i32> %a2) #0 {
+; CHECK-LABEL: define <2 x i32> @test_vaddv_s32_used_by_laneop(
+; CHECK-SAME: <2 x i32> [[A1:%.*]], <2 x i32> [[A2:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load <2 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = call i32 @llvm.vector.reduce.or.v2i32(<2 x i32> [[TMP0]])
+; CHECK-NEXT:    [[TMP5:%.*]] = tail call i32 @llvm.aarch64.neon.saddv.i32.v2i32(<2 x i32> [[A2]])
+; CHECK-NEXT:    [[_MSPROP:%.*]] = insertelement <2 x i32> [[TMP1]], i32 [[TMP2]], i32 1
+; CHECK-NEXT:    [[TMP6:%.*]] = insertelement <2 x i32> [[A1]], i32 [[TMP5]], i32 1
+; CHECK-NEXT:    store <2 x i32> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x i32> [[TMP6]]
+;
+entry:
+  %0 = tail call i32 @llvm.aarch64.neon.saddv.i32.v2i32(<2 x i32> %a2)
+  %1 = insertelement <2 x i32> %a1, i32 %0, i32 1
+  ret <2 x i32> %1
+}
+
+define i64 @test_vaddv_s64(<2 x i64> %a1) #0 {
+; CHECK-LABEL: define i64 @test_vaddv_s64(
+; CHECK-SAME: <2 x i64> [[A1:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vector.reduce.or.v2i64(<2 x i64> [[TMP0]])
+; CHECK-NEXT:    [[VADDV_I:%.*]] = tail call i64 @llvm.aarch64.neon.saddv.i64.v2i64(<2 x i64> [[A1]])
+; CHECK-NEXT:    store i64 [[TMP1]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i64 [[VADDV_I]]
+;
+entry:
+  %vaddv.i = tail call i64 @llvm.aarch64.neon.saddv.i64.v2i64(<2 x i64> %a1)
+  ret i64 %vaddv.i
+}
+
+define <2 x i64> @test_vaddv_s64_used_by_laneop(<2 x i64> %a1, <2 x i64> %a2) #0 {
+; CHECK-LABEL: define <2 x i64> @test_vaddv_s64_used_by_laneop(
+; CHECK-SAME: <2 x i64> [[A1:%.*]], <2 x i64> [[A2:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vector.reduce.or.v2i64(<2 x i64> [[TMP0]])
+; CHECK-NEXT:    [[TMP5:%.*]] = tail call i64 @llvm.aarch64.neon.saddv.i64.v2i64(<2 x i64> [[A2]])
+; CHECK-NEXT:    [[_MSPROP:%.*]] = insertelement <2 x i64> [[TMP1]], i64 [[TMP2]], i64 1
+; CHECK-NEXT:    [[TMP6:%.*]] = insertelement <2 x i64> [[A1]], i64 [[TMP5]], i64 1
+; CHECK-NEXT:    store <2 x i64> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x i64> [[TMP6]]
+;
+entry:
+  %0 = tail call i64 @llvm.aarch64.neon.saddv.i64.v2i64(<2 x i64> %a2)
+  %1 = insertelement <2 x i64> %a1, i64 %0, i64 1
+  ret <2 x i64> %1
+}
+
+define zeroext i8 @test_vaddv_u8(<8 x i8> %a1) #0 {
+; CHECK-LABEL: define zeroext i8 @test_vaddv_u8(
+; CHECK-SAME: <8 x i8> [[A1:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load <8 x i8>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP1:%.*]] = call i8 @llvm.vector.reduce.or.v8i8(<8 x i8> [[TMP0]])
+; CHECK-NEXT:    [[TMP2:%.*]] = zext i8 [[TMP1]] to i32
+; CHECK-NEXT:    [[VADDV_I:%.*]] = tail call i32 @llvm.aarch64.neon.uaddv.i32.v8i8(<8 x i8> [[A1]])
+; CHECK-NEXT:    [[_MSPROP:%.*]] = trunc i32 [[TMP2]] to i8
+; CHECK-NEXT:    [[TMP4:%.*]] = trunc i32 [[VADDV_I]] to i8
+; CHECK-NEXT:    store i8 [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i8 [[TMP4]]
+;
+entry:
+  %vaddv.i = tail call i32 @llvm.aarch64.neon.uaddv.i32.v8i8(<8 x i8> %a1)
+  %0 = trunc i32 %vaddv.i to i8
+  ret i8 %0
+}
+
+define <8 x i8> @test_vaddv_u8_used_by_laneop(<8 x i8> %a1, <8 x i8> %a2) #0 {
+; CHECK-LABEL: define <8 x i8> @test_vaddv_u8_used_by_laneop(
+; CHECK-SAME: <8 x i8> [[A1:%.*]], <8 x i8> [[A2:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load <8 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i8>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = call i8 @llvm.vector.reduce.or.v8i8(<8 x i8> [[TMP0]])
+; CHECK-NEXT:    [[TMP3:%.*]] = zext i8 [[TMP2]] to i32
+; CHECK-NEXT:    [[TMP5:%.*]] = tail call i32 @llvm.aarch64.neon.uaddv.i32.v8i8(<8 x i8> [[A2]])
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = trunc i32 [[TMP3]] to i8
+; CHECK-NEXT:    [[TMP6:%.*]] = trunc i32 [[TMP5]] to i8
+; CHECK-NEXT:    [[_MSPROP:%.*]] = insertelement <8 x i8> [[TMP1]], i8 [[_MSPROP1]], i32 3
+; CHECK-NEXT:    [[TMP7:%.*]] = insertelement <8 x i8> [[A1]], i8 [[TMP6]], i32 3
+; CHECK-NEXT:    store <8 x i8> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i8> [[TMP7]]
+;
+entry:
+  %0 = tail call i32 @llvm.aarch64.neon.uaddv.i32.v8i8(<8 x i8> %a2)
+  %1 = trunc i32 %0 to i8
+  %2 = insertelement <8 x i8> %a1, i8 %1, i32 3
+  ret <8 x i8> %2
+}
+
+define i32 @test_vaddv_u8_masked(<8 x i8> %a1) #0 {
+; CHECK-LABEL: define i32 @test_vaddv_u8_masked(
+; CHECK-SAME: <8 x i8> [[A1:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load <8 x i8>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP1:%.*]] = call i8 @llvm.vector.reduce.or.v8i8(<8 x i8> [[TMP0]])
+; CHECK-NEXT:    [[TMP2:%.*]] = zext i8 [[TMP1]] to i32
+; CHECK-NEXT:    [[VADDV_I:%.*]] = tail call i32 @llvm.aarch64.neon.uaddv.i32.v8i8(<8 x i8> [[A1]])
+; CHECK-NEXT:    [[TMP3:%.*]] = and i32 [[TMP2]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = and i32 [[VADDV_I]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = and i32 [[TMP2]], 511
+; CHECK-NEXT:    [[TMP8:%.*]] = or i32 [[TMP3]], [[TMP4]]
+; CHECK-NEXT:    [[TMP6:%.*]] = or i32 [[TMP8]], [[TMP5]]
+; CHECK-NEXT:    [[TMP7:%.*]] = and i32 [[VADDV_I]], 511
+; CHECK-NEXT:    store i32 [[TMP6]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i32 [[TMP7]]
+;
+entry:
+  %vaddv.i = tail call i32 @llvm.aarch64.neon.uaddv.i32.v8i8(<8 x i8> %a1)
+  %0 = and i32 %vaddv.i, 511 ; 0x1ff
+  ret i32 %0
+}
+
+define zeroext i16 @test_vaddv_u16(<4 x i16> %a1) #0 {
+; CHECK-LABEL: define zeroext i16 @test_vaddv_u16(
+; CHECK-SAME: <4 x i16> [[A1:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load <4 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP1:%.*]] = call i16 @llvm.vector.reduce.or.v4i16(<4 x i16> [[TMP0]])
+; CHECK-NEXT:    [[TMP2:%.*]] = zext i16 [[TMP1]] to i32
+; CHECK-NEXT:    [[VADDV_I:%.*]] = tail call i32 @llvm.aarch64.neon.uaddv.i32.v4i16(<4 x i16> [[A1]])
+; CHECK-NEXT:    [[_MSPROP:%.*]] = trunc i32 [[TMP2]] to i16
+; CHECK-NEXT:    [[TMP4:%.*]] = trunc i32 [[VADDV_I]] to i16
+; CHECK-NEXT:    store i16 [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i16 [[TMP4]]
+;
+entry:
+  %vaddv.i = tail call i32 @llvm.aarch64.neon.uaddv.i32.v4i16(<4 x i16> %a1)
+  %0 = trunc i32 %vaddv.i to i16
+  ret i16 %0
+}
+
+define <4 x i16> @test_vaddv_u16_used_by_laneop(<4 x i16> %a1, <4 x i16> %a2) #0 {
+; CHECK-LABEL: define <4 x i16> @test_vaddv_u16_used_by_laneop(
+; CHECK-SAME: <4 x i16> [[A1:%.*]], <4 x i16> [[A2:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load <4 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = call i16 @llvm.vector.reduce.or.v4i16(<4 x i16> [[TMP0]])
+; CHECK-NEXT:    [[TMP3:%.*]] = zext i16 [[TMP2]] to i32
+; CHECK-NEXT:    [[TMP5:%.*]] = tail call i32 @llvm.aarch64.neon.uaddv.i32.v4i16(<4 x i16> [[A2]])
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = trunc i32 [[TMP3]] to i16
+; CHECK-NEXT:    [[TMP6:%.*]] = trunc i32 [[TMP5]] to i16
+; CHECK-NEXT:    [[_MSPROP:%.*]] = insertelement <4 x i16> [[TMP1]], i16 [[_MSPROP1]], i32 3
+; CHECK-NEXT:    [[TMP7:%.*]] = insertelement <4 x i16> [[A1]], i16 [[TMP6]], i32 3
+; CHECK-NEXT:    store <4 x i16> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i16> [[TMP7]]
+;
+entry:
+  %0 = tail call i32 @llvm.aarch64.neon.uaddv.i32.v4i16(<4 x i16> %a2)
+  %1 = trunc i32 %0 to i16
+  %2 = insertelement <4 x i16> %a1, i16 %1, i32 3
+  ret <4 x i16> %2
+}
+
+define i32 @test_vaddv_u16_masked(<4 x i16> %a1) #0 {
+; CHECK-LABEL: define i32 @test_vaddv_u16_masked(
+; CHECK-SAME: <4 x i16> [[A1:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load <4 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP1:%.*]] = call i16 @llvm.vector.reduce.or.v4i16(<4 x i16> [[TMP0]])
+; CHECK-NEXT:    [[TMP2:%.*]] = zext i16 [[TMP1]] to i32
+; CHECK-NEXT:    [[VADDV_I:%.*]] = tail call i32 @llvm.aarch64.neon.uaddv.i32.v4i16(<4 x i16> [[A1]])
+; CHECK-NEXT:    [[TMP3:%.*]] = and i32 [[TMP2]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = and i32 [[VADDV_I]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = and i32 [[TMP2]], 3276799
+; CHECK-NEXT:    [[TMP8:%.*]] = or i32 [[TMP3]], [[TMP4]]
+; CHECK-NEXT:    [[TMP6:%.*]] = or i32 [[TMP8]], [[TMP5]]
+; CHECK-NEXT:    [[TMP7:%.*]] = and i32 [[VADDV_I]], 3276799
+; CHECK-NEXT:    store i32 [[TMP6]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i32 [[TMP7]]
+;
+entry:
+  %vaddv.i = tail call i32 @llvm.aarch64.neon.uaddv.i32.v4i16(<4 x i16> %a1)
+  %0 = and i32 %vaddv.i, 3276799 ; 0x31ffff
+  ret i32 %0
+}
+
+define i32 @test_vaddv_u32(<2 x i32> %a1) #0 {
+; CHECK-LABEL: define i32 @test_vaddv_u32(
+; CHECK-SAME: <2 x i32> [[A1:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load <2 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @llvm.vector.reduce.or.v2i32(<2 x i32> [[TMP0]])
+; CHECK-NEXT:    [[VADDV_I:%.*]] = tail call i32 @llvm.aarch64.neon.uaddv.i32.v2i32(<2 x i32> [[A1]])
+; CHECK-NEXT:    store i32 [[TMP1]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i32 [[VADDV_I]]
+;
+; 2 x i32 is not supported by the ISA, thus, this is a special case
+entry:
+  %vaddv.i = tail call i32 @llvm.aarch64.neon.uaddv.i32.v2i32(<2 x i32> %a1)
+  ret i32 %vaddv.i
+}
+
+define <2 x i32> @test_vaddv_u32_used_by_laneop(<2 x i32> %a1, <2 x i32> %a2) #0 {
+; CHECK-LABEL: define <2 x i32> @test_vaddv_u32_used_by_laneop(
+; CHECK-SAME: <2 x i32> [[A1:%.*]], <2 x i32> [[A2:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load <2 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = call i32 @llvm.vector.reduce.or.v2i32(<2 x i32> [[TMP0]])
+; CHECK-NEXT:    [[TMP5:%.*]] = tail call i32 @llvm.aarch64.neon.uaddv.i32.v2i32(<2 x i32> [[A2]])
+; CHECK-NEXT:    [[_MSPROP:%.*]] = insertelement <2 x i32> [[TMP1]], i32 [[TMP2]], i32 1
+; CHECK-NEXT:    [[TMP6:%.*]] = insertelement <2 x i32> [[A1]], i32 [[TMP5]], i32 1
+; CHECK-NEXT:    store <2 x i32> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x i32> [[TMP6]]
+;
+entry:
+  %0 = tail call i32 @llvm.aarch64.neon.uaddv.i32.v2i32(<2 x i32> %a2)
+  %1 = insertelement <2 x i32> %a1, i32 %0, i32 1
+  ret <2 x i32> %1
+}
+
+define float @test_vaddv_f32(<2 x float> %a1) #0 {
+; CHECK-LABEL: define float @test_vaddv_f32(
+; CHECK-SAME: <2 x float> [[A1:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load <2 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @llvm.vector.reduce.or.v2i32(<2 x i32> [[TMP0]])
+; CHECK-NEXT:    [[VADDV_I:%.*]] = tail call float @llvm.aarch64.neon.faddv.f32.v2f32(<2 x float> [[A1]])
+; CHECK-NEXT:    store i32 [[TMP1]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret float [[VADDV_I]]
+;
+entry:
+  %vaddv.i = tail call float @llvm.aarch64.neon.faddv.f32.v2f32(<2 x float> %a1)
+  ret float %vaddv.i
+}
+
+define float @test_vaddv_v4f32(<4 x float> %a1) #0 {
+; CHECK-LABEL: define float @test_vaddv_v4f32(
+; CHECK-SAME: <4 x float> [[A1:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @llvm.vector.reduce.or.v4i32(<4 x i32> [[TMP0]])
+; CHECK-NEXT:    [[VADDV_I:%.*]] = tail call float @llvm.aarch64.neon.faddv.f32.v4f32(<4 x float> [[A1]])
+; CHECK-NEXT:    store i32 [[TMP1]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret float [[VADDV_I]]
+;
+entry:
+  %vaddv.i = tail call float @llvm.aarch64.neon.faddv.f32.v4f32(<4 x float> %a1)
+  ret float %vaddv.i
+}
+
+define double @test_vaddv_f64(<2 x double> %a1) #0 {
+; CHECK-LABEL: define double @test_vaddv_f64(
+; CHECK-SAME: <2 x double> [[A1:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vector.reduce.or.v2i64(<2 x i64> [[TMP0]])
+; CHECK-NEXT:    [[VADDV_I:%.*]] = tail call double @llvm.aarch64.neon.faddv.f64.v2f64(<2 x double> [[A1]])
+; CHECK-NEXT:    store i64 [[TMP1]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret double [[VADDV_I]]
+;
+entry:
+  %vaddv.i = tail call double @llvm.aarch64.neon.faddv.f64.v2f64(<2 x double> %a1)
+  ret double %vaddv.i
+}
+
+define i64 @test_vaddv_u64(<2 x i64> %a1) #0 {
+; CHECK-LABEL: define i64 @test_vaddv_u64(
+; CHECK-SAME: <2 x i64> [[A1:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vector.reduce.or.v2i64(<2 x i64> [[TMP0]])
+; CHECK-NEXT:    [[VADDV_I:%.*]] = tail call i64 @llvm.aarch64.neon.uaddv.i64.v2i64(<2 x i64> [[A1]])
+; CHECK-NEXT:    store i64 [[TMP1]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i64 [[VADDV_I]]
+;
+entry:
+  %vaddv.i = tail call i64 @llvm.aarch64.neon.uaddv.i64.v2i64(<2 x i64> %a1)
+  ret i64 %vaddv.i
+}
+
+define <2 x i64> @test_vaddv_u64_used_by_laneop(<2 x i64> %a1, <2 x i64> %a2) #0 {
+; CHECK-LABEL: define <2 x i64> @test_vaddv_u64_used_by_laneop(
+; CHECK-SAME: <2 x i64> [[A1:%.*]], <2 x i64> [[A2:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vector.reduce.or.v2i64(<2 x i64> [[TMP0]])
+; CHECK-NEXT:    [[TMP5:%.*]] = tail call i64 @llvm.aarch64.neon.uaddv.i64.v2i64(<2 x i64> [[A2]])
+; CHECK-NEXT:    [[_MSPROP:%.*]] = insertelement <2 x i64> [[TMP1]], i64 [[TMP2]], i64 1
+; CHECK-NEXT:    [[TMP6:%.*]] = insertelement <2 x i64> [[A1]], i64 [[TMP5]], i64 1
+; CHECK-NEXT:    store <2 x i64> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x i64> [[TMP6]]
+;
+entry:
+  %0 = tail call i64 @llvm.aarch64.neon.uaddv.i64.v2i64(<2 x i64> %a2)
+  %1 = insertelement <2 x i64> %a1, i64 %0, i64 1
+  ret <2 x i64> %1
+}
+
+define <1 x i64> @test_vaddv_u64_to_vec(<2 x i64> %a1, <1 x i64> %param1) #0 {
+; CHECK-LABEL: define <1 x i64> @test_vaddv_u64_to_vec(
+; CHECK-SAME: <2 x i64> [[A1:%.*]], <1 x i64> [[PARAM1:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <1 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vector.reduce.or.v2i64(<2 x i64> [[TMP0]])
+; CHECK-NEXT:    [[VADDV_I:%.*]] = tail call i64 @llvm.aarch64.neon.uaddv.i64.v2i64(<2 x i64> [[A1]])
+; CHECK-NEXT:    [[_MSPROP:%.*]] = insertelement <1 x i64> [[TMP2]], i64 [[TMP3]], i32 0
+; CHECK-NEXT:    [[VEC:%.*]] = insertelement <1 x i64> [[PARAM1]], i64 [[VADDV_I]], i32 0
+; CHECK-NEXT:    store <1 x i64> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <1 x i64> [[VEC]]
+;
+entry:
+  %vaddv.i = tail call i64 @llvm.aarch64.neon.uaddv.i64.v2i64(<2 x i64> %a1)
+  %vec = insertelement <1 x i64> %param1, i64 %vaddv.i, i32 0
+  ret <1 x i64> %vec
+}
+
+define signext i8 @test_vaddvq_s8(<16 x i8> %a1) #0 {
+; CHECK-LABEL: define signext i8 @test_vaddvq_s8(
+; CHECK-SAME: <16 x i8> [[A1:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load <16 x i8>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP1:%.*]] = call i8 @llvm.vector.reduce.or.v16i8(<16 x i8> [[TMP0]])
+; CHECK-NEXT:    [[TMP2:%.*]] = zext i8 [[TMP1]] to i32
+; CHECK-NEXT:    [[VADDV_I:%.*]] = tail call i32 @llvm.aarch64.neon.saddv.i32.v16i8(<16 x i8> [[A1]])
+; CHECK-NEXT:    [[_MSPROP:%.*]] = trunc i32 [[TMP2]] to i8
+; CHECK-NEXT:    [[TMP4:%.*]] = trunc i32 [[VADDV_I]] to i8
+; CHECK-NEXT:    store i8 [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i8 [[TMP4]]
+;
+entry:
+  %vaddv.i = tail call i32 @llvm.aarch64.neon.saddv.i32.v16i8(<16 x i8> %a1)
+  %0 = trunc i32 %vaddv.i to i8
+  ret i8 %0
+}
+
+define <16 x i8> @test_vaddvq_s8_used_by_laneop(<16 x i8> %a1, <16 x i8> %a2) #0 {
+; CHECK-LABEL: define <16 x i8> @test_vaddvq_s8_used_by_laneop(
+; CHECK-SAME: <16 x i8> [[A1:%.*]], <16 x i8> [[A2:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load <16 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i8>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = call i8 @llvm.vector.reduce.or.v16i8(<16 x i8> [[TMP0]])
+; CHECK-NEXT:    [[TMP3:%.*]] = zext i8 [[TMP2]] to i32
+; CHECK-NEXT:    [[TMP5:%.*]] = tail call i32 @llvm.aarch64.neon.saddv.i32.v16i8(<16 x i8> [[A2]])
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = trunc i32 [[TMP3]] to i8
+; CHECK-NEXT:    [[TMP6:%.*]] = trunc i32 [[TMP5]] to i8
+; CHECK-NEXT:    [[_MSPROP:%.*]] = insertelement <16 x i8> [[TMP1]], i8 [[_MSPROP1]], i32 3
+; CHECK-NEXT:    [[TMP7:%.*]] = insertelement <16 x i8> [[A1]], i8 [[TMP6]], i32 3
+; CHECK-NEXT:    store <16 x i8> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i8> [[TMP7]]
+;
+entry:
+  %0 = tail call i32 @llvm.aarch64.neon.saddv.i32.v16i8(<16 x i8> %a2)
+  %1 = trunc i32 %0 to i8
+  %2 = insertelement <16 x i8> %a1, i8 %1, i32 3
+  ret <16 x i8> %2
+}
+
+define signext i16 @test_vaddvq_s16(<8 x i16> %a1) #0 {
+; CHECK-LABEL: define signext i16 @test_vaddvq_s16(
+; CHECK-SAME: <8 x i16> [[A1:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load <8 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP1:%.*]] = call i16 @llvm.vector.reduce.or.v8i16(<8 x i16> [[TMP0]])
+; CHECK-NEXT:    [[TMP2:%.*]] = zext i16 [[TMP1]] to i32
+; CHECK-NEXT:    [[VADDV_I:%.*]] = tail call i32 @llvm.aarch64.neon.saddv.i32.v8i16(<8 x i16> [[A1]])
+; CHECK-NEXT:    [[_MSPROP:%.*]] = trunc i32 [[TMP2]] to i16
+; CHECK-NEXT:    [[TMP4:%.*]] = trunc i32 [[VADDV_I]] to i16
+; CHECK-NEXT:    store i16 [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i16 [[TMP4]]
+;
+entry:
+  %vaddv.i = tail call i32 @llvm.aarch64.neon.saddv.i32.v8i16(<8 x i16> %a1)
+  %0 = trunc i32 %vaddv.i to i16
+  ret i16 %0
+}
+
+define <8 x i16> @test_vaddvq_s16_used_by_laneop(<8 x i16> %a1, <8 x i16> %a2) #0 {
+; CHECK-LABEL: define <8 x i16> @test_vaddvq_s16_used_by_laneop(
+; CHECK-SAME: <8 x i16> [[A1:%.*]], <8 x i16> [[A2:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load <8 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = call i16 @llvm.vector.reduce.or.v8i16(<8 x i16> [[TMP0]])
+; CHECK-NEXT:    [[TMP3:%.*]] = zext i16 [[TMP2]] to i32
+; CHECK-NEXT:    [[TMP5:%.*]] = tail call i32 @llvm.aarch64.neon.saddv.i32.v8i16(<8 x i16> [[A2]])
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = trunc i32 [[TMP3]] to i16
+; CHECK-NEXT:    [[TMP6:%.*]] = trunc i32 [[TMP5]] to i16
+; CHECK-NEXT:    [[_MSPROP:%.*]] = insertelement <8 x i16> [[TMP1]], i16 [[_MSPROP1]], i32 3
+; CHECK-NEXT:    [[TMP7:%.*]] = insertelement <8 x i16> [[A1]], i16 [[TMP6]], i32 3
+; CHECK-NEXT:    store <8 x i16> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i16> [[TMP7]]
+;
+entry:
+  %0 = tail call i32 @llvm.aarch64.neon.saddv.i32.v8i16(<8 x i16> %a2)
+  %1 = trunc i32 %0 to i16
+  %2 = insertelement <8 x i16> %a1, i16 %1, i32 3
+  ret <8 x i16> %2
+}
+
+define i32 @test_vaddvq_s32(<4 x i32> %a1) #0 {
+; CHECK-LABEL: define i32 @test_vaddvq_s32(
+; CHECK-SAME: <4 x i32> [[A1:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @llvm.vector.reduce.or.v4i32(<4 x i32> [[TMP0]])
+; CHECK-NEXT:    [[VADDV_I:%.*]] = tail call i32 @llvm.aarch64.neon.saddv.i32.v4i32(<4 x i32> [[A1]])
+; CHECK-NEXT:    store i32 [[TMP1]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i32 [[VADDV_I]]
+;
+entry:
+  %vaddv.i = tail call i32 @llvm.aarch64.neon.saddv.i32.v4i32(<4 x i32> %a1)
+  ret i32 %vaddv.i
+}
+
+define <4 x i32> @test_vaddvq_s32_used_by_laneop(<4 x i32> %a1, <4 x i32> %a2) #0 {
+; CHECK-LABEL: define <4 x i32> @test_vaddvq_s32_used_by_laneop(
+; CHECK-SAME: <4 x i32> [[A1:%.*]], <4 x i32> [[A2:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = call i32 @llvm.vector.reduce.or.v4i32(<4 x i32> [[TMP0]])
+; CHECK-NEXT:    [[TMP5:%.*]] = tail call i32 @llvm.aarch64.neon.saddv.i32.v4i32(<4 x i32> [[A2]])
+; CHECK-NEXT:    [[_MSPROP:%.*]] = insertelement <4 x i32> [[TMP1]], i32 [[TMP2]], i32 3
+; CHECK-NEXT:    [[TMP6:%.*]] = insertelement <4 x i32> [[A1]], i32 [[TMP5]], i32 3
+; CHECK-NEXT:    store <4 x i32> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i32> [[TMP6]]
+;
+entry:
+  %0 = tail call i32 @llvm.aarch64.neon.saddv.i32.v4i32(<4 x i32> %a2)
+  %1 = insertelement <4 x i32> %a1, i32 %0, i32 3
+  ret <4 x i32> %1
+}
+
+define zeroext i8 @test_vaddvq_u8(<16 x i8> %a1) #0 {
+; CHECK-LABEL: define zeroext i8 @test_vaddvq_u8(
+; CHECK-SAME: <16 x i8> [[A1:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load <16 x i8>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP1:%.*]] = call i8 @llvm.vector.reduce.or.v16i8(<16 x i8> [[TMP0]])
+; CHECK-NEXT:    [[TMP2:%.*]] = zext i8 [[TMP1]] to i32
+; CHECK-NEXT:    [[VADDV_I:%.*]] = tail call i32 @llvm.aarch64.neon.uaddv.i32.v16i8(<16 x i8> [[A1]])
+; CHECK-NEXT:    [[_MSPROP:%.*]] = trunc i32 [[TMP2]] to i8
+; CHECK-NEXT:    [[TMP4:%.*]] = trunc i32 [[VADDV_I]] to i8
+; CHECK-NEXT:    store i8 [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i8 [[TMP4]]
+;
+entry:
+  %vaddv.i = tail call i32 @llvm.aarch64.neon.uaddv.i32.v16i8(<16 x i8> %a1)
+  %0 = trunc i32 %vaddv.i to i8
+  ret i8 %0
+}
+
+define <16 x i8> @test_vaddvq_u8_used_by_laneop(<16 x i8> %a1, <16 x i8> %a2) #0 {
+; CHECK-LABEL: define <16 x i8> @test_vaddvq_u8_used_by_laneop(
+; CHECK-SAME: <16 x i8> [[A1:%.*]], <16 x i8> [[A2:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load <16 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i8>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = call i8 @llvm.vector.reduce.or.v16i8(<16 x i8> [[TMP0]])
+; CHECK-NEXT:    [[TMP3:%.*]] = zext i8 [[TMP2]] to i32
+; CHECK-NEXT:    [[TMP5:%.*]] = tail call i32 @llvm.aarch64.neon.uaddv.i32.v16i8(<16 x i8> [[A2]])
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = trunc i32 [[TMP3]] to i8
+; CHECK-NEXT:    [[TMP6:%.*]] = trunc i32 [[TMP5]] to i8
+; CHECK-NEXT:    [[_MSPROP:%.*]] = insertelement <16 x i8> [[TMP1]], i8 [[_MSPROP1]], i32 3
+; CHECK-NEXT:    [[TMP7:%.*]] = insertelement <16 x i8> [[A1]], i8 [[TMP6]], i32 3
+; CHECK-NEXT:    store <16 x i8> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i8> [[TMP7]]
+;
+entry:
+  %0 = tail call i32 @llvm.aarch64.neon.uaddv.i32.v16i8(<16 x i8> %a2)
+  %1 = trunc i32 %0 to i8
+  %2 = insertelement <16 x i8> %a1, i8 %1, i32 3
+  ret <16 x i8> %2
+}
+
+define zeroext i16 @test_vaddvq_u16(<8 x i16> %a1) #0 {
+; CHECK-LABEL: define zeroext i16 @test_vaddvq_u16(
+; CHECK-SAME: <8 x i16> [[A1:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load <8 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP1:%.*]] = call i16 @llvm.vector.reduce.or.v8i16(<8 x i16> [[TMP0]])
+; CHECK-NEXT:    [[TMP2:%.*]] = zext i16 [[TMP1]] to i32
+; CHECK-NEXT:    [[VADDV_I:%.*]] = tail call i32 @llvm.aarch64.neon.uaddv.i32.v8i16(<8 x i16> [[A1]])
+; CHECK-NEXT:    [[_MSPROP:%.*]] = trunc i32 [[TMP2]] to i16
+; CHECK-NEXT:    [[TMP4:%.*]] = trunc i32 [[VADDV_I]] to i16
+; CHECK-NEXT:    store i16 [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i16 [[TMP4]]
+;
+entry:
+  %vaddv.i = tail call i32 @llvm.aarch64.neon.uaddv.i32.v8i16(<8 x i16> %a1)
+  %0 = trunc i32 %vaddv.i to i16
+  ret i16 %0
+}
+
+define <8 x i16> @test_vaddvq_u16_used_by_laneop(<8 x i16> %a1, <8 x i16> %a2) #0 {
+; CHECK-LABEL: define <8 x i16> @test_vaddvq_u16_used_by_laneop(
+; CHECK-SAME: <8 x i16> [[A1:%.*]], <8 x i16> [[A2:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load <8 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = call i16 @llvm.vector.reduce.or.v8i16(<8 x i16> [[TMP0]])
+; CHECK-NEXT:    [[TMP3:%.*]] = zext i16 [[TMP2]] to i32
+; CHECK-NEXT:    [[TMP5:%.*]] = tail call i32 @llvm.aarch64.neon.uaddv.i32.v8i16(<8 x i16> [[A2]])
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = trunc i32 [[TMP3]] to i16
+; CHECK-NEXT:    [[TMP6:%.*]] = trunc i32 [[TMP5]] to i16
+; CHECK-NEXT:    [[_MSPROP:%.*]] = insertelement <8 x i16> [[TMP1]], i16 [[_MSPROP1]], i32 3
+; CHECK-NEXT:    [[TMP7:%.*]] = insertelement <8 x i16> [[A1]], i16 [[TMP6]], i32 3
+; CHECK-NEXT:    store <8 x i16> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i16> [[TMP7]]
+;
+entry:
+  %0 = tail call i32 @llvm.aarch64.neon.uaddv.i32.v8i16(<8 x i16> %a2)
+  %1 = trunc i32 %0 to i16
+  %2 = insertelement <8 x i16> %a1, i16 %1, i32 3
+  ret <8 x i16> %2
+}
+
+define i32 @test_vaddvq_u32(<4 x i32> %a1) #0 {
+; CHECK-LABEL: define i32 @test_vaddvq_u32(
+; CHECK-SAME: <4 x i32> [[A1:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @llvm.vector.reduce.or.v4i32(<4 x i32> [[TMP0]])
+; CHECK-NEXT:    [[VADDV_I:%.*]] = tail call i32 @llvm.aarch64.neon.uaddv.i32.v4i32(<4 x i32> [[A1]])
+; CHECK-NEXT:    store i32 [[TMP1]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i32 [[VADDV_I]]
+;
+entry:
+  %vaddv.i = tail call i32 @llvm.aarch64.neon.uaddv.i32.v4i32(<4 x i32> %a1)
+  ret i32 %vaddv.i
+}
+
+define <4 x i32> @test_vaddvq_u32_used_by_laneop(<4 x i32> %a1, <4 x i32> %a2) #0 {
+; CHECK-LABEL: define <4 x i32> @test_vaddvq_u32_used_by_laneop(
+; CHECK-SAME: <4 x i32> [[A1:%.*]], <4 x i32> [[A2:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = call i32 @llvm.vector.reduce.or.v4i32(<4 x i32> [[TMP0]])
+; CHECK-NEXT:    [[TMP5:%.*]] = tail call i32 @llvm.aarch64.neon.uaddv.i32.v4i32(<4 x i32> [[A2]])
+; CHECK-NEXT:    [[_MSPROP:%.*]] = insertelement <4 x i32> [[TMP1]], i32 [[TMP2]], i32 3
+; CHECK-NEXT:    [[TMP6:%.*]] = insertelement <4 x i32> [[A1]], i32 [[TMP5]], i32 3
+; CHECK-NEXT:    store <4 x i32> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i32> [[TMP6]]
+;
+entry:
+  %0 = tail call i32 @llvm.aarch64.neon.uaddv.i32.v4i32(<4 x i32> %a2)
+  %1 = insertelement <4 x i32> %a1, i32 %0, i32 3
+  ret <4 x i32> %1
+}
+
+declare i32 @llvm.aarch64.neon.uaddv.i32.v4i32(<4 x i32>)
+
+declare i32 @llvm.aarch64.neon.uaddv.i32.v8i16(<8 x i16>)
+
+declare i32 @llvm.aarch64.neon.uaddv.i32.v16i8(<16 x i8>)
+
+declare i32 @llvm.aarch64.neon.saddv.i32.v4i32(<4 x i32>)
+
+declare i32 @llvm.aarch64.neon.saddv.i32.v8i16(<8 x i16>)
+
+declare i32 @llvm.aarch64.neon.saddv.i32.v16i8(<16 x i8>)
+
+declare i64 @llvm.aarch64.neon.uaddv.i64.v2i64(<2 x i64>)
+
+declare i32 @llvm.aarch64.neon.uaddv.i32.v2i32(<2 x i32>)
+
+declare i32 @llvm.aarch64.neon.uaddv.i32.v4i16(<4 x i16>)
+
+declare i32 @llvm.aarch64.neon.uaddv.i32.v8i8(<8 x i8>)
+
+declare i32 @llvm.aarch64.neon.saddv.i32.v2i32(<2 x i32>)
+
+declare i64 @llvm.aarch64.neon.saddv.i64.v2i64(<2 x i64>)
+
+declare i32 @llvm.aarch64.neon.saddv.i32.v4i16(<4 x i16>)
+
+declare i32 @llvm.aarch64.neon.saddv.i32.v8i8(<8 x i8>)
+
+declare float @llvm.aarch64.neon.faddv.f32.v2f32(<2 x float> %a1)
+declare float @llvm.aarch64.neon.faddv.f32.v4f32(<4 x float> %a1)
+declare double @llvm.aarch64.neon.faddv.f64.v2f64(<2 x double> %a1)
+
+attributes #0 = { sanitize_memory }
diff --git a/llvm/test/Instrumentation/MemorySanitizer/AArch64/arm64-vcvt.ll b/llvm/test/Instrumentation/MemorySanitizer/AArch64/arm64-vcvt.ll
new file mode 100644
index 0000000000000..2f49fde82c1af
--- /dev/null
+++ b/llvm/test/Instrumentation/MemorySanitizer/AArch64/arm64-vcvt.ll
@@ -0,0 +1,1644 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 4
+; RUN: opt < %s -passes=msan -S | FileCheck %s
+;
+; Suboptimally handled by visitInstruction: llvm.aarch64.neon.
+; - fcvtas, fcvtau
+; - fcvtms, fcvtmu
+; - fcvtns, fcvtnu
+; - fcvtps, fcvtpu
+; - fcvtzs, fcvtzu
+; - fcvtxn
+; - vcvtfp2fxs, vcvtfp2fxu
+; - vcvtfxs2fp, vcvtfxu2fp
+;
+; Forked from llvm/test/CodeGen/AArch64/arm64-vcvt.ll
+
+target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
+target triple = "aarch64--linux-android9001"
+
+define <2 x i32> @fcvtas_2s(<2 x float> %A) nounwind #0 {
+; CHECK-LABEL: define <2 x i32> @fcvtas_2s(
+; CHECK-SAME: <2 x float> [[A:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <2 x i32> [[TMP1]] to i64
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP2]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP3:%.*]], label [[TMP4:%.*]], !prof [[PROF1:![0-9]+]]
+; CHECK:       3:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR5:[0-9]+]]
+; CHECK-NEXT:    unreachable
+; CHECK:       4:
+; CHECK-NEXT:    [[TMPVAR3:%.*]] = call <2 x i32> @llvm.aarch64.neon.fcvtas.v2i32.v2f32(<2 x float> [[A]])
+; CHECK-NEXT:    store <2 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x i32> [[TMPVAR3]]
+;
+  %tmpvar3 = call <2 x i32> @llvm.aarch64.neon.fcvtas.v2i32.v2f32(<2 x float> %A)
+  ret <2 x i32> %tmpvar3
+}
+
+define <4 x i32> @fcvtas_4s(<4 x float> %A) nounwind #0 {
+; CHECK-LABEL: define <4 x i32> @fcvtas_4s(
+; CHECK-SAME: <4 x float> [[A:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <4 x i32> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP2]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP3:%.*]], label [[TMP4:%.*]], !prof [[PROF1]]
+; CHECK:       3:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR5]]
+; CHECK-NEXT:    unreachable
+; CHECK:       4:
+; CHECK-NEXT:    [[TMPVAR3:%.*]] = call <4 x i32> @llvm.aarch64.neon.fcvtas.v4i32.v4f32(<4 x float> [[A]])
+; CHECK-NEXT:    store <4 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i32> [[TMPVAR3]]
+;
+  %tmpvar3 = call <4 x i32> @llvm.aarch64.neon.fcvtas.v4i32.v4f32(<4 x float> %A)
+  ret <4 x i32> %tmpvar3
+}
+
+define <2 x i64> @fcvtas_2d(<2 x double> %A) nounwind #0 {
+; CHECK-LABEL: define <2 x i64> @fcvtas_2d(
+; CHECK-SAME: <2 x double> [[A:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <2 x i64> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP2]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP3:%.*]], label [[TMP4:%.*]], !prof [[PROF1]]
+; CHECK:       3:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR5]]
+; CHECK-NEXT:    unreachable
+; CHECK:       4:
+; CHECK-NEXT:    [[TMPVAR3:%.*]] = call <2 x i64> @llvm.aarch64.neon.fcvtas.v2i64.v2f64(<2 x double> [[A]])
+; CHECK-NEXT:    store <2 x i64> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x i64> [[TMPVAR3]]
+;
+  %tmpvar3 = call <2 x i64> @llvm.aarch64.neon.fcvtas.v2i64.v2f64(<2 x double> %A)
+  ret <2 x i64> %tmpvar3
+}
+
+define <1 x i64> @fcvtas_1d(<1 x double> %A) nounwind #0 {
+; CHECK-LABEL: define <1 x i64> @fcvtas_1d(
+; CHECK-SAME: <1 x double> [[A:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <1 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <1 x i64> [[TMP1]] to i64
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP2]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP3:%.*]], label [[TMP4:%.*]], !prof [[PROF1]]
+; CHECK:       3:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR5]]
+; CHECK-NEXT:    unreachable
+; CHECK:       4:
+; CHECK-NEXT:    [[TMPVAR3:%.*]] = call <1 x i64> @llvm.aarch64.neon.fcvtas.v1i64.v1f64(<1 x double> [[A]])
+; CHECK-NEXT:    store <1 x i64> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <1 x i64> [[TMPVAR3]]
+;
+  %tmpvar3 = call <1 x i64> @llvm.aarch64.neon.fcvtas.v1i64.v1f64(<1 x double> %A)
+  ret <1 x i64> %tmpvar3
+}
+
+declare <2 x i32> @llvm.aarch64.neon.fcvtas.v2i32.v2f32(<2 x float>) nounwind readnone
+declare <4 x i32> @llvm.aarch64.neon.fcvtas.v4i32.v4f32(<4 x float>) nounwind readnone
+declare <2 x i64> @llvm.aarch64.neon.fcvtas.v2i64.v2f64(<2 x double>) nounwind readnone
+declare <1 x i64> @llvm.aarch64.neon.fcvtas.v1i64.v1f64(<1 x double>) nounwind readnone
+
+define <2 x i32> @fcvtau_2s(<2 x float> %A) nounwind #0 {
+; CHECK-LABEL: define <2 x i32> @fcvtau_2s(
+; CHECK-SAME: <2 x float> [[A:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <2 x i32> [[TMP1]] to i64
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP2]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP3:%.*]], label [[TMP4:%.*]], !prof [[PROF1]]
+; CHECK:       3:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR5]]
+; CHECK-NEXT:    unreachable
+; CHECK:       4:
+; CHECK-NEXT:    [[TMPVAR3:%.*]] = call <2 x i32> @llvm.aarch64.neon.fcvtau.v2i32.v2f32(<2 x float> [[A]])
+; CHECK-NEXT:    store <2 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x i32> [[TMPVAR3]]
+;
+  %tmpvar3 = call <2 x i32> @llvm.aarch64.neon.fcvtau.v2i32.v2f32(<2 x float> %A)
+  ret <2 x i32> %tmpvar3
+}
+
+define <4 x i32> @fcvtau_4s(<4 x float> %A) nounwind #0 {
+; CHECK-LABEL: define <4 x i32> @fcvtau_4s(
+; CHECK-SAME: <4 x float> [[A:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <4 x i32> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP2]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP3:%.*]], label [[TMP4:%.*]], !prof [[PROF1]]
+; CHECK:       3:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR5]]
+; CHECK-NEXT:    unreachable
+; CHECK:       4:
+; CHECK-NEXT:    [[TMPVAR3:%.*]] = call <4 x i32> @llvm.aarch64.neon.fcvtau.v4i32.v4f32(<4 x float> [[A]])
+; CHECK-NEXT:    store <4 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i32> [[TMPVAR3]]
+;
+  %tmpvar3 = call <4 x i32> @llvm.aarch64.neon.fcvtau.v4i32.v4f32(<4 x float> %A)
+  ret <4 x i32> %tmpvar3
+}
+
+define <2 x i64> @fcvtau_2d(<2 x double> %A) nounwind #0 {
+; CHECK-LABEL: define <2 x i64> @fcvtau_2d(
+; CHECK-SAME: <2 x double> [[A:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <2 x i64> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP2]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP3:%.*]], label [[TMP4:%.*]], !prof [[PROF1]]
+; CHECK:       3:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR5]]
+; CHECK-NEXT:    unreachable
+; CHECK:       4:
+; CHECK-NEXT:    [[TMPVAR3:%.*]] = call <2 x i64> @llvm.aarch64.neon.fcvtau.v2i64.v2f64(<2 x double> [[A]])
+; CHECK-NEXT:    store <2 x i64> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x i64> [[TMPVAR3]]
+;
+  %tmpvar3 = call <2 x i64> @llvm.aarch64.neon.fcvtau.v2i64.v2f64(<2 x double> %A)
+  ret <2 x i64> %tmpvar3
+}
+
+define <1 x i64> @fcvtau_1d(<1 x double> %A) nounwind #0 {
+; CHECK-LABEL: define <1 x i64> @fcvtau_1d(
+; CHECK-SAME: <1 x double> [[A:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <1 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <1 x i64> [[TMP1]] to i64
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP2]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP3:%.*]], label [[TMP4:%.*]], !prof [[PROF1]]
+; CHECK:       3:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR5]]
+; CHECK-NEXT:    unreachable
+; CHECK:       4:
+; CHECK-NEXT:    [[TMPVAR3:%.*]] = call <1 x i64> @llvm.aarch64.neon.fcvtau.v1i64.v1f64(<1 x double> [[A]])
+; CHECK-NEXT:    store <1 x i64> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <1 x i64> [[TMPVAR3]]
+;
+  %tmpvar3 = call <1 x i64> @llvm.aarch64.neon.fcvtau.v1i64.v1f64(<1 x double> %A)
+  ret <1 x i64> %tmpvar3
+}
+
+declare <2 x i32> @llvm.aarch64.neon.fcvtau.v2i32.v2f32(<2 x float>) nounwind readnone
+declare <4 x i32> @llvm.aarch64.neon.fcvtau.v4i32.v4f32(<4 x float>) nounwind readnone
+declare <2 x i64> @llvm.aarch64.neon.fcvtau.v2i64.v2f64(<2 x double>) nounwind readnone
+declare <1 x i64> @llvm.aarch64.neon.fcvtau.v1i64.v1f64(<1 x double>) nounwind readnone
+
+define <2 x i32> @fcvtms_2s(<2 x float> %A) nounwind #0 {
+; CHECK-LABEL: define <2 x i32> @fcvtms_2s(
+; CHECK-SAME: <2 x float> [[A:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <2 x i32> [[TMP1]] to i64
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP2]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP3:%.*]], label [[TMP4:%.*]], !prof [[PROF1]]
+; CHECK:       3:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR5]]
+; CHECK-NEXT:    unreachable
+; CHECK:       4:
+; CHECK-NEXT:    [[TMPVAR3:%.*]] = call <2 x i32> @llvm.aarch64.neon.fcvtms.v2i32.v2f32(<2 x float> [[A]])
+; CHECK-NEXT:    store <2 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x i32> [[TMPVAR3]]
+;
+  %tmpvar3 = call <2 x i32> @llvm.aarch64.neon.fcvtms.v2i32.v2f32(<2 x float> %A)
+  ret <2 x i32> %tmpvar3
+}
+
+define <4 x i32> @fcvtms_4s(<4 x float> %A) nounwind #0 {
+; CHECK-LABEL: define <4 x i32> @fcvtms_4s(
+; CHECK-SAME: <4 x float> [[A:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <4 x i32> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP2]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP3:%.*]], label [[TMP4:%.*]], !prof [[PROF1]]
+; CHECK:       3:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR5]]
+; CHECK-NEXT:    unreachable
+; CHECK:       4:
+; CHECK-NEXT:    [[TMPVAR3:%.*]] = call <4 x i32> @llvm.aarch64.neon.fcvtms.v4i32.v4f32(<4 x float> [[A]])
+; CHECK-NEXT:    store <4 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i32> [[TMPVAR3]]
+;
+  %tmpvar3 = call <4 x i32> @llvm.aarch64.neon.fcvtms.v4i32.v4f32(<4 x float> %A)
+  ret <4 x i32> %tmpvar3
+}
+
+define <2 x i64> @fcvtms_2d(<2 x double> %A) nounwind #0 {
+; CHECK-LABEL: define <2 x i64> @fcvtms_2d(
+; CHECK-SAME: <2 x double> [[A:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <2 x i64> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP2]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP3:%.*]], label [[TMP4:%.*]], !prof [[PROF1]]
+; CHECK:       3:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR5]]
+; CHECK-NEXT:    unreachable
+; CHECK:       4:
+; CHECK-NEXT:    [[TMPVAR3:%.*]] = call <2 x i64> @llvm.aarch64.neon.fcvtms.v2i64.v2f64(<2 x double> [[A]])
+; CHECK-NEXT:    store <2 x i64> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x i64> [[TMPVAR3]]
+;
+  %tmpvar3 = call <2 x i64> @llvm.aarch64.neon.fcvtms.v2i64.v2f64(<2 x double> %A)
+  ret <2 x i64> %tmpvar3
+}
+
+define <1 x i64> @fcvtms_1d(<1 x double> %A) nounwind #0 {
+; CHECK-LABEL: define <1 x i64> @fcvtms_1d(
+; CHECK-SAME: <1 x double> [[A:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <1 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <1 x i64> [[TMP1]] to i64
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP2]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP3:%.*]], label [[TMP4:%.*]], !prof [[PROF1]]
+; CHECK:       3:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR5]]
+; CHECK-NEXT:    unreachable
+; CHECK:       4:
+; CHECK-NEXT:    [[TMPVAR3:%.*]] = call <1 x i64> @llvm.aarch64.neon.fcvtms.v1i64.v1f64(<1 x double> [[A]])
+; CHECK-NEXT:    store <1 x i64> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <1 x i64> [[TMPVAR3]]
+;
+  %tmpvar3 = call <1 x i64> @llvm.aarch64.neon.fcvtms.v1i64.v1f64(<1 x double> %A)
+  ret <1 x i64> %tmpvar3
+}
+
+declare <2 x i32> @llvm.aarch64.neon.fcvtms.v2i32.v2f32(<2 x float>) nounwind readnone
+declare <4 x i32> @llvm.aarch64.neon.fcvtms.v4i32.v4f32(<4 x float>) nounwind readnone
+declare <2 x i64> @llvm.aarch64.neon.fcvtms.v2i64.v2f64(<2 x double>) nounwind readnone
+declare <1 x i64> @llvm.aarch64.neon.fcvtms.v1i64.v1f64(<1 x double>) nounwind readnone
+
+define <2 x i32> @fcvtmu_2s(<2 x float> %A) nounwind #0 {
+; CHECK-LABEL: define <2 x i32> @fcvtmu_2s(
+; CHECK-SAME: <2 x float> [[A:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <2 x i32> [[TMP1]] to i64
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP2]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP3:%.*]], label [[TMP4:%.*]], !prof [[PROF1]]
+; CHECK:       3:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR5]]
+; CHECK-NEXT:    unreachable
+; CHECK:       4:
+; CHECK-NEXT:    [[TMPVAR3:%.*]] = call <2 x i32> @llvm.aarch64.neon.fcvtmu.v2i32.v2f32(<2 x float> [[A]])
+; CHECK-NEXT:    store <2 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x i32> [[TMPVAR3]]
+;
+  %tmpvar3 = call <2 x i32> @llvm.aarch64.neon.fcvtmu.v2i32.v2f32(<2 x float> %A)
+  ret <2 x i32> %tmpvar3
+}
+
+define <4 x i32> @fcvtmu_4s(<4 x float> %A) nounwind #0 {
+; CHECK-LABEL: define <4 x i32> @fcvtmu_4s(
+; CHECK-SAME: <4 x float> [[A:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <4 x i32> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP2]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP3:%.*]], label [[TMP4:%.*]], !prof [[PROF1]]
+; CHECK:       3:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR5]]
+; CHECK-NEXT:    unreachable
+; CHECK:       4:
+; CHECK-NEXT:    [[TMPVAR3:%.*]] = call <4 x i32> @llvm.aarch64.neon.fcvtmu.v4i32.v4f32(<4 x float> [[A]])
+; CHECK-NEXT:    store <4 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i32> [[TMPVAR3]]
+;
+  %tmpvar3 = call <4 x i32> @llvm.aarch64.neon.fcvtmu.v4i32.v4f32(<4 x float> %A)
+  ret <4 x i32> %tmpvar3
+}
+
+define <2 x i64> @fcvtmu_2d(<2 x double> %A) nounwind #0 {
+; CHECK-LABEL: define <2 x i64> @fcvtmu_2d(
+; CHECK-SAME: <2 x double> [[A:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <2 x i64> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP2]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP3:%.*]], label [[TMP4:%.*]], !prof [[PROF1]]
+; CHECK:       3:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR5]]
+; CHECK-NEXT:    unreachable
+; CHECK:       4:
+; CHECK-NEXT:    [[TMPVAR3:%.*]] = call <2 x i64> @llvm.aarch64.neon.fcvtmu.v2i64.v2f64(<2 x double> [[A]])
+; CHECK-NEXT:    store <2 x i64> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x i64> [[TMPVAR3]]
+;
+  %tmpvar3 = call <2 x i64> @llvm.aarch64.neon.fcvtmu.v2i64.v2f64(<2 x double> %A)
+  ret <2 x i64> %tmpvar3
+}
+
+define <1 x i64> @fcvtmu_1d(<1 x double> %A) nounwind #0 {
+; CHECK-LABEL: define <1 x i64> @fcvtmu_1d(
+; CHECK-SAME: <1 x double> [[A:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <1 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <1 x i64> [[TMP1]] to i64
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP2]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP3:%.*]], label [[TMP4:%.*]], !prof [[PROF1]]
+; CHECK:       3:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR5]]
+; CHECK-NEXT:    unreachable
+; CHECK:       4:
+; CHECK-NEXT:    [[TMPVAR3:%.*]] = call <1 x i64> @llvm.aarch64.neon.fcvtmu.v1i64.v1f64(<1 x double> [[A]])
+; CHECK-NEXT:    store <1 x i64> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <1 x i64> [[TMPVAR3]]
+;
+  %tmpvar3 = call <1 x i64> @llvm.aarch64.neon.fcvtmu.v1i64.v1f64(<1 x double> %A)
+  ret <1 x i64> %tmpvar3
+}
+
+declare <2 x i32> @llvm.aarch64.neon.fcvtmu.v2i32.v2f32(<2 x float>) nounwind readnone
+declare <4 x i32> @llvm.aarch64.neon.fcvtmu.v4i32.v4f32(<4 x float>) nounwind readnone
+declare <2 x i64> @llvm.aarch64.neon.fcvtmu.v2i64.v2f64(<2 x double>) nounwind readnone
+declare <1 x i64> @llvm.aarch64.neon.fcvtmu.v1i64.v1f64(<1 x double>) nounwind readnone
+
+define <2 x i32> @fcvtps_2s(<2 x float> %A) nounwind #0 {
+; CHECK-LABEL: define <2 x i32> @fcvtps_2s(
+; CHECK-SAME: <2 x float> [[A:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <2 x i32> [[TMP1]] to i64
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP2]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP3:%.*]], label [[TMP4:%.*]], !prof [[PROF1]]
+; CHECK:       3:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR5]]
+; CHECK-NEXT:    unreachable
+; CHECK:       4:
+; CHECK-NEXT:    [[TMPVAR3:%.*]] = call <2 x i32> @llvm.aarch64.neon.fcvtps.v2i32.v2f32(<2 x float> [[A]])
+; CHECK-NEXT:    store <2 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x i32> [[TMPVAR3]]
+;
+  %tmpvar3 = call <2 x i32> @llvm.aarch64.neon.fcvtps.v2i32.v2f32(<2 x float> %A)
+  ret <2 x i32> %tmpvar3
+}
+
+define <4 x i32> @fcvtps_4s(<4 x float> %A) nounwind #0 {
+; CHECK-LABEL: define <4 x i32> @fcvtps_4s(
+; CHECK-SAME: <4 x float> [[A:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <4 x i32> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP2]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP3:%.*]], label [[TMP4:%.*]], !prof [[PROF1]]
+; CHECK:       3:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR5]]
+; CHECK-NEXT:    unreachable
+; CHECK:       4:
+; CHECK-NEXT:    [[TMPVAR3:%.*]] = call <4 x i32> @llvm.aarch64.neon.fcvtps.v4i32.v4f32(<4 x float> [[A]])
+; CHECK-NEXT:    store <4 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i32> [[TMPVAR3]]
+;
+  %tmpvar3 = call <4 x i32> @llvm.aarch64.neon.fcvtps.v4i32.v4f32(<4 x float> %A)
+  ret <4 x i32> %tmpvar3
+}
+
+define <2 x i64> @fcvtps_2d(<2 x double> %A) nounwind #0 {
+; CHECK-LABEL: define <2 x i64> @fcvtps_2d(
+; CHECK-SAME: <2 x double> [[A:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <2 x i64> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP2]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP3:%.*]], label [[TMP4:%.*]], !prof [[PROF1]]
+; CHECK:       3:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR5]]
+; CHECK-NEXT:    unreachable
+; CHECK:       4:
+; CHECK-NEXT:    [[TMPVAR3:%.*]] = call <2 x i64> @llvm.aarch64.neon.fcvtps.v2i64.v2f64(<2 x double> [[A]])
+; CHECK-NEXT:    store <2 x i64> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x i64> [[TMPVAR3]]
+;
+  %tmpvar3 = call <2 x i64> @llvm.aarch64.neon.fcvtps.v2i64.v2f64(<2 x double> %A)
+  ret <2 x i64> %tmpvar3
+}
+
+define <1 x i64> @fcvtps_1d(<1 x double> %A) nounwind #0 {
+; CHECK-LABEL: define <1 x i64> @fcvtps_1d(
+; CHECK-SAME: <1 x double> [[A:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <1 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <1 x i64> [[TMP1]] to i64
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP2]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP3:%.*]], label [[TMP4:%.*]], !prof [[PROF1]]
+; CHECK:       3:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR5]]
+; CHECK-NEXT:    unreachable
+; CHECK:       4:
+; CHECK-NEXT:    [[TMPVAR3:%.*]] = call <1 x i64> @llvm.aarch64.neon.fcvtps.v1i64.v1f64(<1 x double> [[A]])
+; CHECK-NEXT:    store <1 x i64> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <1 x i64> [[TMPVAR3]]
+;
+  %tmpvar3 = call <1 x i64> @llvm.aarch64.neon.fcvtps.v1i64.v1f64(<1 x double> %A)
+  ret <1 x i64> %tmpvar3
+}
+
+declare <2 x i32> @llvm.aarch64.neon.fcvtps.v2i32.v2f32(<2 x float>) nounwind readnone
+declare <4 x i32> @llvm.aarch64.neon.fcvtps.v4i32.v4f32(<4 x float>) nounwind readnone
+declare <2 x i64> @llvm.aarch64.neon.fcvtps.v2i64.v2f64(<2 x double>) nounwind readnone
+declare <1 x i64> @llvm.aarch64.neon.fcvtps.v1i64.v1f64(<1 x double>) nounwind readnone
+
+define <2 x i32> @fcvtpu_2s(<2 x float> %A) nounwind #0 {
+; CHECK-LABEL: define <2 x i32> @fcvtpu_2s(
+; CHECK-SAME: <2 x float> [[A:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <2 x i32> [[TMP1]] to i64
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP2]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP3:%.*]], label [[TMP4:%.*]], !prof [[PROF1]]
+; CHECK:       3:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR5]]
+; CHECK-NEXT:    unreachable
+; CHECK:       4:
+; CHECK-NEXT:    [[TMPVAR3:%.*]] = call <2 x i32> @llvm.aarch64.neon.fcvtpu.v2i32.v2f32(<2 x float> [[A]])
+; CHECK-NEXT:    store <2 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x i32> [[TMPVAR3]]
+;
+  %tmpvar3 = call <2 x i32> @llvm.aarch64.neon.fcvtpu.v2i32.v2f32(<2 x float> %A)
+  ret <2 x i32> %tmpvar3
+}
+
+define <4 x i32> @fcvtpu_4s(<4 x float> %A) nounwind #0 {
+; CHECK-LABEL: define <4 x i32> @fcvtpu_4s(
+; CHECK-SAME: <4 x float> [[A:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <4 x i32> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP2]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP3:%.*]], label [[TMP4:%.*]], !prof [[PROF1]]
+; CHECK:       3:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR5]]
+; CHECK-NEXT:    unreachable
+; CHECK:       4:
+; CHECK-NEXT:    [[TMPVAR3:%.*]] = call <4 x i32> @llvm.aarch64.neon.fcvtpu.v4i32.v4f32(<4 x float> [[A]])
+; CHECK-NEXT:    store <4 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i32> [[TMPVAR3]]
+;
+  %tmpvar3 = call <4 x i32> @llvm.aarch64.neon.fcvtpu.v4i32.v4f32(<4 x float> %A)
+  ret <4 x i32> %tmpvar3
+}
+
+define <2 x i64> @fcvtpu_2d(<2 x double> %A) nounwind #0 {
+; CHECK-LABEL: define <2 x i64> @fcvtpu_2d(
+; CHECK-SAME: <2 x double> [[A:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <2 x i64> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP2]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP3:%.*]], label [[TMP4:%.*]], !prof [[PROF1]]
+; CHECK:       3:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR5]]
+; CHECK-NEXT:    unreachable
+; CHECK:       4:
+; CHECK-NEXT:    [[TMPVAR3:%.*]] = call <2 x i64> @llvm.aarch64.neon.fcvtpu.v2i64.v2f64(<2 x double> [[A]])
+; CHECK-NEXT:    store <2 x i64> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x i64> [[TMPVAR3]]
+;
+  %tmpvar3 = call <2 x i64> @llvm.aarch64.neon.fcvtpu.v2i64.v2f64(<2 x double> %A)
+  ret <2 x i64> %tmpvar3
+}
+
+define <1 x i64> @fcvtpu_1d(<1 x double> %A) nounwind #0 {
+; CHECK-LABEL: define <1 x i64> @fcvtpu_1d(
+; CHECK-SAME: <1 x double> [[A:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <1 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <1 x i64> [[TMP1]] to i64
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP2]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP3:%.*]], label [[TMP4:%.*]], !prof [[PROF1]]
+; CHECK:       3:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR5]]
+; CHECK-NEXT:    unreachable
+; CHECK:       4:
+; CHECK-NEXT:    [[TMPVAR3:%.*]] = call <1 x i64> @llvm.aarch64.neon.fcvtpu.v1i64.v1f64(<1 x double> [[A]])
+; CHECK-NEXT:    store <1 x i64> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <1 x i64> [[TMPVAR3]]
+;
+  %tmpvar3 = call <1 x i64> @llvm.aarch64.neon.fcvtpu.v1i64.v1f64(<1 x double> %A)
+  ret <1 x i64> %tmpvar3
+}
+
+declare <2 x i32> @llvm.aarch64.neon.fcvtpu.v2i32.v2f32(<2 x float>) nounwind readnone
+declare <4 x i32> @llvm.aarch64.neon.fcvtpu.v4i32.v4f32(<4 x float>) nounwind readnone
+declare <2 x i64> @llvm.aarch64.neon.fcvtpu.v2i64.v2f64(<2 x double>) nounwind readnone
+declare <1 x i64> @llvm.aarch64.neon.fcvtpu.v1i64.v1f64(<1 x double>) nounwind readnone
+
+define <2 x i32> @fcvtns_2s(<2 x float> %A) nounwind #0 {
+; CHECK-LABEL: define <2 x i32> @fcvtns_2s(
+; CHECK-SAME: <2 x float> [[A:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <2 x i32> [[TMP1]] to i64
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP2]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP3:%.*]], label [[TMP4:%.*]], !prof [[PROF1]]
+; CHECK:       3:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR5]]
+; CHECK-NEXT:    unreachable
+; CHECK:       4:
+; CHECK-NEXT:    [[TMPVAR3:%.*]] = call <2 x i32> @llvm.aarch64.neon.fcvtns.v2i32.v2f32(<2 x float> [[A]])
+; CHECK-NEXT:    store <2 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x i32> [[TMPVAR3]]
+;
+  %tmpvar3 = call <2 x i32> @llvm.aarch64.neon.fcvtns.v2i32.v2f32(<2 x float> %A)
+  ret <2 x i32> %tmpvar3
+}
+
+define <4 x i32> @fcvtns_4s(<4 x float> %A) nounwind #0 {
+; CHECK-LABEL: define <4 x i32> @fcvtns_4s(
+; CHECK-SAME: <4 x float> [[A:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <4 x i32> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP2]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP3:%.*]], label [[TMP4:%.*]], !prof [[PROF1]]
+; CHECK:       3:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR5]]
+; CHECK-NEXT:    unreachable
+; CHECK:       4:
+; CHECK-NEXT:    [[TMPVAR3:%.*]] = call <4 x i32> @llvm.aarch64.neon.fcvtns.v4i32.v4f32(<4 x float> [[A]])
+; CHECK-NEXT:    store <4 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i32> [[TMPVAR3]]
+;
+  %tmpvar3 = call <4 x i32> @llvm.aarch64.neon.fcvtns.v4i32.v4f32(<4 x float> %A)
+  ret <4 x i32> %tmpvar3
+}
+
+define <2 x i64> @fcvtns_2d(<2 x double> %A) nounwind #0 {
+; CHECK-LABEL: define <2 x i64> @fcvtns_2d(
+; CHECK-SAME: <2 x double> [[A:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <2 x i64> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP2]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP3:%.*]], label [[TMP4:%.*]], !prof [[PROF1]]
+; CHECK:       3:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR5]]
+; CHECK-NEXT:    unreachable
+; CHECK:       4:
+; CHECK-NEXT:    [[TMPVAR3:%.*]] = call <2 x i64> @llvm.aarch64.neon.fcvtns.v2i64.v2f64(<2 x double> [[A]])
+; CHECK-NEXT:    store <2 x i64> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x i64> [[TMPVAR3]]
+;
+  %tmpvar3 = call <2 x i64> @llvm.aarch64.neon.fcvtns.v2i64.v2f64(<2 x double> %A)
+  ret <2 x i64> %tmpvar3
+}
+
+define <1 x i64> @fcvtns_1d(<1 x double> %A) nounwind #0 {
+; CHECK-LABEL: define <1 x i64> @fcvtns_1d(
+; CHECK-SAME: <1 x double> [[A:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <1 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <1 x i64> [[TMP1]] to i64
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP2]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP3:%.*]], label [[TMP4:%.*]], !prof [[PROF1]]
+; CHECK:       3:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR5]]
+; CHECK-NEXT:    unreachable
+; CHECK:       4:
+; CHECK-NEXT:    [[TMPVAR3:%.*]] = call <1 x i64> @llvm.aarch64.neon.fcvtns.v1i64.v1f64(<1 x double> [[A]])
+; CHECK-NEXT:    store <1 x i64> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <1 x i64> [[TMPVAR3]]
+;
+  %tmpvar3 = call <1 x i64> @llvm.aarch64.neon.fcvtns.v1i64.v1f64(<1 x double> %A)
+  ret <1 x i64> %tmpvar3
+}
+
+declare <2 x i32> @llvm.aarch64.neon.fcvtns.v2i32.v2f32(<2 x float>) nounwind readnone
+declare <4 x i32> @llvm.aarch64.neon.fcvtns.v4i32.v4f32(<4 x float>) nounwind readnone
+declare <2 x i64> @llvm.aarch64.neon.fcvtns.v2i64.v2f64(<2 x double>) nounwind readnone
+declare <1 x i64> @llvm.aarch64.neon.fcvtns.v1i64.v1f64(<1 x double>) nounwind readnone
+
+define <2 x i32> @fcvtnu_2s(<2 x float> %A) nounwind #0 {
+; CHECK-LABEL: define <2 x i32> @fcvtnu_2s(
+; CHECK-SAME: <2 x float> [[A:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <2 x i32> [[TMP1]] to i64
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP2]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP3:%.*]], label [[TMP4:%.*]], !prof [[PROF1]]
+; CHECK:       3:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR5]]
+; CHECK-NEXT:    unreachable
+; CHECK:       4:
+; CHECK-NEXT:    [[TMPVAR3:%.*]] = call <2 x i32> @llvm.aarch64.neon.fcvtnu.v2i32.v2f32(<2 x float> [[A]])
+; CHECK-NEXT:    store <2 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x i32> [[TMPVAR3]]
+;
+  %tmpvar3 = call <2 x i32> @llvm.aarch64.neon.fcvtnu.v2i32.v2f32(<2 x float> %A)
+  ret <2 x i32> %tmpvar3
+}
+
+define <4 x i32> @fcvtnu_4s(<4 x float> %A) nounwind #0 {
+; CHECK-LABEL: define <4 x i32> @fcvtnu_4s(
+; CHECK-SAME: <4 x float> [[A:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <4 x i32> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP2]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP3:%.*]], label [[TMP4:%.*]], !prof [[PROF1]]
+; CHECK:       3:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR5]]
+; CHECK-NEXT:    unreachable
+; CHECK:       4:
+; CHECK-NEXT:    [[TMPVAR3:%.*]] = call <4 x i32> @llvm.aarch64.neon.fcvtnu.v4i32.v4f32(<4 x float> [[A]])
+; CHECK-NEXT:    store <4 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i32> [[TMPVAR3]]
+;
+  %tmpvar3 = call <4 x i32> @llvm.aarch64.neon.fcvtnu.v4i32.v4f32(<4 x float> %A)
+  ret <4 x i32> %tmpvar3
+}
+
+define <2 x i64> @fcvtnu_2d(<2 x double> %A) nounwind #0 {
+; CHECK-LABEL: define <2 x i64> @fcvtnu_2d(
+; CHECK-SAME: <2 x double> [[A:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <2 x i64> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP2]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP3:%.*]], label [[TMP4:%.*]], !prof [[PROF1]]
+; CHECK:       3:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR5]]
+; CHECK-NEXT:    unreachable
+; CHECK:       4:
+; CHECK-NEXT:    [[TMPVAR3:%.*]] = call <2 x i64> @llvm.aarch64.neon.fcvtnu.v2i64.v2f64(<2 x double> [[A]])
+; CHECK-NEXT:    store <2 x i64> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x i64> [[TMPVAR3]]
+;
+  %tmpvar3 = call <2 x i64> @llvm.aarch64.neon.fcvtnu.v2i64.v2f64(<2 x double> %A)
+  ret <2 x i64> %tmpvar3
+}
+
+define <1 x i64> @fcvtnu_1d(<1 x double> %A) nounwind #0 {
+; CHECK-LABEL: define <1 x i64> @fcvtnu_1d(
+; CHECK-SAME: <1 x double> [[A:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <1 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <1 x i64> [[TMP1]] to i64
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP2]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP3:%.*]], label [[TMP4:%.*]], !prof [[PROF1]]
+; CHECK:       3:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR5]]
+; CHECK-NEXT:    unreachable
+; CHECK:       4:
+; CHECK-NEXT:    [[TMPVAR3:%.*]] = call <1 x i64> @llvm.aarch64.neon.fcvtnu.v1i64.v1f64(<1 x double> [[A]])
+; CHECK-NEXT:    store <1 x i64> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <1 x i64> [[TMPVAR3]]
+;
+  %tmpvar3 = call <1 x i64> @llvm.aarch64.neon.fcvtnu.v1i64.v1f64(<1 x double> %A)
+  ret <1 x i64> %tmpvar3
+}
+
+declare <2 x i32> @llvm.aarch64.neon.fcvtnu.v2i32.v2f32(<2 x float>) nounwind readnone
+declare <4 x i32> @llvm.aarch64.neon.fcvtnu.v4i32.v4f32(<4 x float>) nounwind readnone
+declare <2 x i64> @llvm.aarch64.neon.fcvtnu.v2i64.v2f64(<2 x double>) nounwind readnone
+declare <1 x i64> @llvm.aarch64.neon.fcvtnu.v1i64.v1f64(<1 x double>) nounwind readnone
+
+define <2 x i32> @fcvtzs_2s(<2 x float> %A) nounwind #0 {
+; CHECK-LABEL: define <2 x i32> @fcvtzs_2s(
+; CHECK-SAME: <2 x float> [[A:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMPVAR3:%.*]] = fptosi <2 x float> [[A]] to <2 x i32>
+; CHECK-NEXT:    store <2 x i32> [[TMP1]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x i32> [[TMPVAR3]]
+;
+  %tmpvar3 = fptosi <2 x float> %A to <2 x i32>
+  ret <2 x i32> %tmpvar3
+}
+
+define <4 x i32> @fcvtzs_4s(<4 x float> %A) nounwind #0 {
+; CHECK-LABEL: define <4 x i32> @fcvtzs_4s(
+; CHECK-SAME: <4 x float> [[A:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMPVAR3:%.*]] = fptosi <4 x float> [[A]] to <4 x i32>
+; CHECK-NEXT:    store <4 x i32> [[TMP1]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i32> [[TMPVAR3]]
+;
+  %tmpvar3 = fptosi <4 x float> %A to <4 x i32>
+  ret <4 x i32> %tmpvar3
+}
+
+define <2 x i64> @fcvtzs_2d(<2 x double> %A) nounwind #0 {
+; CHECK-LABEL: define <2 x i64> @fcvtzs_2d(
+; CHECK-SAME: <2 x double> [[A:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMPVAR3:%.*]] = fptosi <2 x double> [[A]] to <2 x i64>
+; CHECK-NEXT:    store <2 x i64> [[TMP1]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x i64> [[TMPVAR3]]
+;
+  %tmpvar3 = fptosi <2 x double> %A to <2 x i64>
+  ret <2 x i64> %tmpvar3
+}
+
+define <1 x i64> @fcvtzs_1d(<1 x double> %A) nounwind #0 {
+; CHECK-LABEL: define <1 x i64> @fcvtzs_1d(
+; CHECK-SAME: <1 x double> [[A:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <1 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMPVAR3:%.*]] = fptosi <1 x double> [[A]] to <1 x i64>
+; CHECK-NEXT:    store <1 x i64> [[TMP1]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <1 x i64> [[TMPVAR3]]
+;
+  %tmpvar3 = fptosi <1 x double> %A to <1 x i64>
+  ret <1 x i64> %tmpvar3
+}
+
+define <2 x i32> @fcvtzs_2s_intrinsic(<2 x float> %A) nounwind #0 {
+; CHECK-LABEL: define <2 x i32> @fcvtzs_2s_intrinsic(
+; CHECK-SAME: <2 x float> [[A:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <2 x i32> [[TMP1]] to i64
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP2]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP3:%.*]], label [[TMP4:%.*]], !prof [[PROF1]]
+; CHECK:       3:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR5]]
+; CHECK-NEXT:    unreachable
+; CHECK:       4:
+; CHECK-NEXT:    [[TMPVAR3:%.*]] = call <2 x i32> @llvm.aarch64.neon.fcvtzs.v2i32.v2f32(<2 x float> [[A]])
+; CHECK-NEXT:    store <2 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x i32> [[TMPVAR3]]
+;
+  %tmpvar3 = call <2 x i32> @llvm.aarch64.neon.fcvtzs.v2i32.v2f32(<2 x float> %A)
+  ret <2 x i32> %tmpvar3
+}
+
+define <4 x i32> @fcvtzs_4s_intrinsic(<4 x float> %A) nounwind #0 {
+; CHECK-LABEL: define <4 x i32> @fcvtzs_4s_intrinsic(
+; CHECK-SAME: <4 x float> [[A:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <4 x i32> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP2]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP3:%.*]], label [[TMP4:%.*]], !prof [[PROF1]]
+; CHECK:       3:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR5]]
+; CHECK-NEXT:    unreachable
+; CHECK:       4:
+; CHECK-NEXT:    [[TMPVAR3:%.*]] = call <4 x i32> @llvm.aarch64.neon.fcvtzs.v4i32.v4f32(<4 x float> [[A]])
+; CHECK-NEXT:    store <4 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i32> [[TMPVAR3]]
+;
+  %tmpvar3 = call <4 x i32> @llvm.aarch64.neon.fcvtzs.v4i32.v4f32(<4 x float> %A)
+  ret <4 x i32> %tmpvar3
+}
+
+define <2 x i64> @fcvtzs_2d_intrinsic(<2 x double> %A) nounwind #0 {
+; CHECK-LABEL: define <2 x i64> @fcvtzs_2d_intrinsic(
+; CHECK-SAME: <2 x double> [[A:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <2 x i64> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP2]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP3:%.*]], label [[TMP4:%.*]], !prof [[PROF1]]
+; CHECK:       3:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR5]]
+; CHECK-NEXT:    unreachable
+; CHECK:       4:
+; CHECK-NEXT:    [[TMPVAR3:%.*]] = call <2 x i64> @llvm.aarch64.neon.fcvtzs.v2i64.v2f64(<2 x double> [[A]])
+; CHECK-NEXT:    store <2 x i64> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x i64> [[TMPVAR3]]
+;
+  %tmpvar3 = call <2 x i64> @llvm.aarch64.neon.fcvtzs.v2i64.v2f64(<2 x double> %A)
+  ret <2 x i64> %tmpvar3
+}
+
+define <1 x i64> @fcvtzs_1d_intrinsic(<1 x double> %A) nounwind #0 {
+; CHECK-LABEL: define <1 x i64> @fcvtzs_1d_intrinsic(
+; CHECK-SAME: <1 x double> [[A:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <1 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <1 x i64> [[TMP1]] to i64
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP2]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP3:%.*]], label [[TMP4:%.*]], !prof [[PROF1]]
+; CHECK:       3:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR5]]
+; CHECK-NEXT:    unreachable
+; CHECK:       4:
+; CHECK-NEXT:    [[TMPVAR3:%.*]] = call <1 x i64> @llvm.aarch64.neon.fcvtzs.v1i64.v1f64(<1 x double> [[A]])
+; CHECK-NEXT:    store <1 x i64> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <1 x i64> [[TMPVAR3]]
+;
+  %tmpvar3 = call <1 x i64> @llvm.aarch64.neon.fcvtzs.v1i64.v1f64(<1 x double> %A)
+  ret <1 x i64> %tmpvar3
+}
+
+declare <2 x i32> @llvm.aarch64.neon.fcvtzs.v2i32.v2f32(<2 x float>) nounwind readnone
+declare <4 x i32> @llvm.aarch64.neon.fcvtzs.v4i32.v4f32(<4 x float>) nounwind readnone
+declare <2 x i64> @llvm.aarch64.neon.fcvtzs.v2i64.v2f64(<2 x double>) nounwind readnone
+declare <1 x i64> @llvm.aarch64.neon.fcvtzs.v1i64.v1f64(<1 x double>) nounwind readnone
+
+define <2 x i32> @fcvtzu_2s(<2 x float> %A) nounwind #0 {
+; CHECK-LABEL: define <2 x i32> @fcvtzu_2s(
+; CHECK-SAME: <2 x float> [[A:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMPVAR3:%.*]] = fptoui <2 x float> [[A]] to <2 x i32>
+; CHECK-NEXT:    store <2 x i32> [[TMP1]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x i32> [[TMPVAR3]]
+;
+  %tmpvar3 = fptoui <2 x float> %A to <2 x i32>
+  ret <2 x i32> %tmpvar3
+}
+
+define <4 x i32> @fcvtzu_4s(<4 x float> %A) nounwind #0 {
+; CHECK-LABEL: define <4 x i32> @fcvtzu_4s(
+; CHECK-SAME: <4 x float> [[A:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMPVAR3:%.*]] = fptoui <4 x float> [[A]] to <4 x i32>
+; CHECK-NEXT:    store <4 x i32> [[TMP1]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i32> [[TMPVAR3]]
+;
+  %tmpvar3 = fptoui <4 x float> %A to <4 x i32>
+  ret <4 x i32> %tmpvar3
+}
+
+define <2 x i64> @fcvtzu_2d(<2 x double> %A) nounwind #0 {
+; CHECK-LABEL: define <2 x i64> @fcvtzu_2d(
+; CHECK-SAME: <2 x double> [[A:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMPVAR3:%.*]] = fptoui <2 x double> [[A]] to <2 x i64>
+; CHECK-NEXT:    store <2 x i64> [[TMP1]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x i64> [[TMPVAR3]]
+;
+  %tmpvar3 = fptoui <2 x double> %A to <2 x i64>
+  ret <2 x i64> %tmpvar3
+}
+
+define <1 x i64> @fcvtzu_1d(<1 x double> %A) nounwind #0 {
+; CHECK-LABEL: define <1 x i64> @fcvtzu_1d(
+; CHECK-SAME: <1 x double> [[A:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <1 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMPVAR3:%.*]] = fptoui <1 x double> [[A]] to <1 x i64>
+; CHECK-NEXT:    store <1 x i64> [[TMP1]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <1 x i64> [[TMPVAR3]]
+;
+  %tmpvar3 = fptoui <1 x double> %A to <1 x i64>
+  ret <1 x i64> %tmpvar3
+}
+
+define <2 x i32> @fcvtzu_2s_intrinsic(<2 x float> %A) nounwind #0 {
+; CHECK-LABEL: define <2 x i32> @fcvtzu_2s_intrinsic(
+; CHECK-SAME: <2 x float> [[A:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <2 x i32> [[TMP1]] to i64
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP2]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP3:%.*]], label [[TMP4:%.*]], !prof [[PROF1]]
+; CHECK:       3:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR5]]
+; CHECK-NEXT:    unreachable
+; CHECK:       4:
+; CHECK-NEXT:    [[TMPVAR3:%.*]] = call <2 x i32> @llvm.aarch64.neon.fcvtzu.v2i32.v2f32(<2 x float> [[A]])
+; CHECK-NEXT:    store <2 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x i32> [[TMPVAR3]]
+;
+  %tmpvar3 = call <2 x i32> @llvm.aarch64.neon.fcvtzu.v2i32.v2f32(<2 x float> %A)
+  ret <2 x i32> %tmpvar3
+}
+
+define <4 x i32> @fcvtzu_4s_intrinsic(<4 x float> %A) nounwind #0 {
+; CHECK-LABEL: define <4 x i32> @fcvtzu_4s_intrinsic(
+; CHECK-SAME: <4 x float> [[A:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <4 x i32> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP2]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP3:%.*]], label [[TMP4:%.*]], !prof [[PROF1]]
+; CHECK:       3:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR5]]
+; CHECK-NEXT:    unreachable
+; CHECK:       4:
+; CHECK-NEXT:    [[TMPVAR3:%.*]] = call <4 x i32> @llvm.aarch64.neon.fcvtzu.v4i32.v4f32(<4 x float> [[A]])
+; CHECK-NEXT:    store <4 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i32> [[TMPVAR3]]
+;
+  %tmpvar3 = call <4 x i32> @llvm.aarch64.neon.fcvtzu.v4i32.v4f32(<4 x float> %A)
+  ret <4 x i32> %tmpvar3
+}
+
+define <2 x i64> @fcvtzu_2d_intrinsic(<2 x double> %A) nounwind #0 {
+; CHECK-LABEL: define <2 x i64> @fcvtzu_2d_intrinsic(
+; CHECK-SAME: <2 x double> [[A:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <2 x i64> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP2]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP3:%.*]], label [[TMP4:%.*]], !prof [[PROF1]]
+; CHECK:       3:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR5]]
+; CHECK-NEXT:    unreachable
+; CHECK:       4:
+; CHECK-NEXT:    [[TMPVAR3:%.*]] = call <2 x i64> @llvm.aarch64.neon.fcvtzu.v2i64.v2f64(<2 x double> [[A]])
+; CHECK-NEXT:    store <2 x i64> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x i64> [[TMPVAR3]]
+;
+  %tmpvar3 = call <2 x i64> @llvm.aarch64.neon.fcvtzu.v2i64.v2f64(<2 x double> %A)
+  ret <2 x i64> %tmpvar3
+}
+
+define <1 x i64> @fcvtzu_1d_intrinsic(<1 x double> %A) nounwind #0 {
+; CHECK-LABEL: define <1 x i64> @fcvtzu_1d_intrinsic(
+; CHECK-SAME: <1 x double> [[A:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <1 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <1 x i64> [[TMP1]] to i64
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP2]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP3:%.*]], label [[TMP4:%.*]], !prof [[PROF1]]
+; CHECK:       3:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR5]]
+; CHECK-NEXT:    unreachable
+; CHECK:       4:
+; CHECK-NEXT:    [[TMPVAR3:%.*]] = call <1 x i64> @llvm.aarch64.neon.fcvtzu.v1i64.v1f64(<1 x double> [[A]])
+; CHECK-NEXT:    store <1 x i64> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <1 x i64> [[TMPVAR3]]
+;
+  %tmpvar3 = call <1 x i64> @llvm.aarch64.neon.fcvtzu.v1i64.v1f64(<1 x double> %A)
+  ret <1 x i64> %tmpvar3
+}
+
+declare <2 x i32> @llvm.aarch64.neon.fcvtzu.v2i32.v2f32(<2 x float>) nounwind readnone
+declare <4 x i32> @llvm.aarch64.neon.fcvtzu.v4i32.v4f32(<4 x float>) nounwind readnone
+declare <2 x i64> @llvm.aarch64.neon.fcvtzu.v2i64.v2f64(<2 x double>) nounwind readnone
+declare <1 x i64> @llvm.aarch64.neon.fcvtzu.v1i64.v1f64(<1 x double>) nounwind readnone
+
+define <2 x float> @frinta_2s(<2 x float> %A) nounwind #0 {
+; CHECK-LABEL: define <2 x float> @frinta_2s(
+; CHECK-SAME: <2 x float> [[A:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMPVAR3:%.*]] = call <2 x float> @llvm.round.v2f32(<2 x float> [[A]])
+; CHECK-NEXT:    store <2 x i32> [[TMP1]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x float> [[TMPVAR3]]
+;
+  %tmpvar3 = call <2 x float> @llvm.round.v2f32(<2 x float> %A)
+  ret <2 x float> %tmpvar3
+}
+
+define <4 x float> @frinta_4s(<4 x float> %A) nounwind #0 {
+; CHECK-LABEL: define <4 x float> @frinta_4s(
+; CHECK-SAME: <4 x float> [[A:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMPVAR3:%.*]] = call <4 x float> @llvm.round.v4f32(<4 x float> [[A]])
+; CHECK-NEXT:    store <4 x i32> [[TMP1]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x float> [[TMPVAR3]]
+;
+  %tmpvar3 = call <4 x float> @llvm.round.v4f32(<4 x float> %A)
+  ret <4 x float> %tmpvar3
+}
+
+define <2 x double> @frinta_2d(<2 x double> %A) nounwind #0 {
+; CHECK-LABEL: define <2 x double> @frinta_2d(
+; CHECK-SAME: <2 x double> [[A:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMPVAR3:%.*]] = call <2 x double> @llvm.round.v2f64(<2 x double> [[A]])
+; CHECK-NEXT:    store <2 x i64> [[TMP1]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x double> [[TMPVAR3]]
+;
+  %tmpvar3 = call <2 x double> @llvm.round.v2f64(<2 x double> %A)
+  ret <2 x double> %tmpvar3
+}
+
+declare <2 x float> @llvm.round.v2f32(<2 x float>) nounwind readnone
+declare <4 x float> @llvm.round.v4f32(<4 x float>) nounwind readnone
+declare <2 x double> @llvm.round.v2f64(<2 x double>) nounwind readnone
+
+define <2 x float> @frinti_2s(<2 x float> %A) nounwind #0 {
+; CHECK-LABEL: define <2 x float> @frinti_2s(
+; CHECK-SAME: <2 x float> [[A:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMPVAR3:%.*]] = call <2 x float> @llvm.nearbyint.v2f32(<2 x float> [[A]])
+; CHECK-NEXT:    store <2 x i32> [[TMP1]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x float> [[TMPVAR3]]
+;
+  %tmpvar3 = call <2 x float> @llvm.nearbyint.v2f32(<2 x float> %A)
+  ret <2 x float> %tmpvar3
+}
+
+define <4 x float> @frinti_4s(<4 x float> %A) nounwind #0 {
+; CHECK-LABEL: define <4 x float> @frinti_4s(
+; CHECK-SAME: <4 x float> [[A:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMPVAR3:%.*]] = call <4 x float> @llvm.nearbyint.v4f32(<4 x float> [[A]])
+; CHECK-NEXT:    store <4 x i32> [[TMP1]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x float> [[TMPVAR3]]
+;
+  %tmpvar3 = call <4 x float> @llvm.nearbyint.v4f32(<4 x float> %A)
+  ret <4 x float> %tmpvar3
+}
+
+define <2 x double> @frinti_2d(<2 x double> %A) nounwind #0 {
+; CHECK-LABEL: define <2 x double> @frinti_2d(
+; CHECK-SAME: <2 x double> [[A:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMPVAR3:%.*]] = call <2 x double> @llvm.nearbyint.v2f64(<2 x double> [[A]])
+; CHECK-NEXT:    store <2 x i64> [[TMP1]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x double> [[TMPVAR3]]
+;
+  %tmpvar3 = call <2 x double> @llvm.nearbyint.v2f64(<2 x double> %A)
+  ret <2 x double> %tmpvar3
+}
+
+declare <2 x float> @llvm.nearbyint.v2f32(<2 x float>) nounwind readnone
+declare <4 x float> @llvm.nearbyint.v4f32(<4 x float>) nounwind readnone
+declare <2 x double> @llvm.nearbyint.v2f64(<2 x double>) nounwind readnone
+
+define <2 x float> @frintm_2s(<2 x float> %A) nounwind #0 {
+; CHECK-LABEL: define <2 x float> @frintm_2s(
+; CHECK-SAME: <2 x float> [[A:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMPVAR3:%.*]] = call <2 x float> @llvm.floor.v2f32(<2 x float> [[A]])
+; CHECK-NEXT:    store <2 x i32> [[TMP1]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x float> [[TMPVAR3]]
+;
+  %tmpvar3 = call <2 x float> @llvm.floor.v2f32(<2 x float> %A)
+  ret <2 x float> %tmpvar3
+}
+
+define <4 x float> @frintm_4s(<4 x float> %A) nounwind #0 {
+; CHECK-LABEL: define <4 x float> @frintm_4s(
+; CHECK-SAME: <4 x float> [[A:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMPVAR3:%.*]] = call <4 x float> @llvm.floor.v4f32(<4 x float> [[A]])
+; CHECK-NEXT:    store <4 x i32> [[TMP1]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x float> [[TMPVAR3]]
+;
+  %tmpvar3 = call <4 x float> @llvm.floor.v4f32(<4 x float> %A)
+  ret <4 x float> %tmpvar3
+}
+
+define <2 x double> @frintm_2d(<2 x double> %A) nounwind #0 {
+; CHECK-LABEL: define <2 x double> @frintm_2d(
+; CHECK-SAME: <2 x double> [[A:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMPVAR3:%.*]] = call <2 x double> @llvm.floor.v2f64(<2 x double> [[A]])
+; CHECK-NEXT:    store <2 x i64> [[TMP1]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x double> [[TMPVAR3]]
+;
+  %tmpvar3 = call <2 x double> @llvm.floor.v2f64(<2 x double> %A)
+  ret <2 x double> %tmpvar3
+}
+
+declare <2 x float> @llvm.floor.v2f32(<2 x float>) nounwind readnone
+declare <4 x float> @llvm.floor.v4f32(<4 x float>) nounwind readnone
+declare <2 x double> @llvm.floor.v2f64(<2 x double>) nounwind readnone
+
+define <2 x float> @frintn_2s(<2 x float> %A) nounwind #0 {
+; CHECK-LABEL: define <2 x float> @frintn_2s(
+; CHECK-SAME: <2 x float> [[A:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMPVAR3:%.*]] = call <2 x float> @llvm.roundeven.v2f32(<2 x float> [[A]])
+; CHECK-NEXT:    store <2 x i32> [[TMP1]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x float> [[TMPVAR3]]
+;
+  %tmpvar3 = call <2 x float> @llvm.roundeven.v2f32(<2 x float> %A)
+  ret <2 x float> %tmpvar3
+}
+
+define <4 x float> @frintn_4s(<4 x float> %A) nounwind #0 {
+; CHECK-LABEL: define <4 x float> @frintn_4s(
+; CHECK-SAME: <4 x float> [[A:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMPVAR3:%.*]] = call <4 x float> @llvm.roundeven.v4f32(<4 x float> [[A]])
+; CHECK-NEXT:    store <4 x i32> [[TMP1]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x float> [[TMPVAR3]]
+;
+  %tmpvar3 = call <4 x float> @llvm.roundeven.v4f32(<4 x float> %A)
+  ret <4 x float> %tmpvar3
+}
+
+define <2 x double> @frintn_2d(<2 x double> %A) nounwind #0 {
+; CHECK-LABEL: define <2 x double> @frintn_2d(
+; CHECK-SAME: <2 x double> [[A:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMPVAR3:%.*]] = call <2 x double> @llvm.roundeven.v2f64(<2 x double> [[A]])
+; CHECK-NEXT:    store <2 x i64> [[TMP1]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x double> [[TMPVAR3]]
+;
+  %tmpvar3 = call <2 x double> @llvm.roundeven.v2f64(<2 x double> %A)
+  ret <2 x double> %tmpvar3
+}
+
+declare <2 x float> @llvm.roundeven.v2f32(<2 x float>) nounwind readnone
+declare <4 x float> @llvm.roundeven.v4f32(<4 x float>) nounwind readnone
+declare <2 x double> @llvm.roundeven.v2f64(<2 x double>) nounwind readnone
+
+define <2 x float> @frintp_2s(<2 x float> %A) nounwind #0 {
+; CHECK-LABEL: define <2 x float> @frintp_2s(
+; CHECK-SAME: <2 x float> [[A:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMPVAR3:%.*]] = call <2 x float> @llvm.ceil.v2f32(<2 x float> [[A]])
+; CHECK-NEXT:    store <2 x i32> [[TMP1]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x float> [[TMPVAR3]]
+;
+  %tmpvar3 = call <2 x float> @llvm.ceil.v2f32(<2 x float> %A)
+  ret <2 x float> %tmpvar3
+}
+
+define <4 x float> @frintp_4s(<4 x float> %A) nounwind #0 {
+; CHECK-LABEL: define <4 x float> @frintp_4s(
+; CHECK-SAME: <4 x float> [[A:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMPVAR3:%.*]] = call <4 x float> @llvm.ceil.v4f32(<4 x float> [[A]])
+; CHECK-NEXT:    store <4 x i32> [[TMP1]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x float> [[TMPVAR3]]
+;
+  %tmpvar3 = call <4 x float> @llvm.ceil.v4f32(<4 x float> %A)
+  ret <4 x float> %tmpvar3
+}
+
+define <2 x double> @frintp_2d(<2 x double> %A) nounwind #0 {
+; CHECK-LABEL: define <2 x double> @frintp_2d(
+; CHECK-SAME: <2 x double> [[A:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMPVAR3:%.*]] = call <2 x double> @llvm.ceil.v2f64(<2 x double> [[A]])
+; CHECK-NEXT:    store <2 x i64> [[TMP1]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x double> [[TMPVAR3]]
+;
+  %tmpvar3 = call <2 x double> @llvm.ceil.v2f64(<2 x double> %A)
+  ret <2 x double> %tmpvar3
+}
+
+declare <2 x float> @llvm.ceil.v2f32(<2 x float>) nounwind readnone
+declare <4 x float> @llvm.ceil.v4f32(<4 x float>) nounwind readnone
+declare <2 x double> @llvm.ceil.v2f64(<2 x double>) nounwind readnone
+
+define <2 x float> @frintx_2s(<2 x float> %A) nounwind #0 {
+; CHECK-LABEL: define <2 x float> @frintx_2s(
+; CHECK-SAME: <2 x float> [[A:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMPVAR3:%.*]] = call <2 x float> @llvm.rint.v2f32(<2 x float> [[A]])
+; CHECK-NEXT:    store <2 x i32> [[TMP1]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x float> [[TMPVAR3]]
+;
+  %tmpvar3 = call <2 x float> @llvm.rint.v2f32(<2 x float> %A)
+  ret <2 x float> %tmpvar3
+}
+
+define <4 x float> @frintx_4s(<4 x float> %A) nounwind #0 {
+; CHECK-LABEL: define <4 x float> @frintx_4s(
+; CHECK-SAME: <4 x float> [[A:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMPVAR3:%.*]] = call <4 x float> @llvm.rint.v4f32(<4 x float> [[A]])
+; CHECK-NEXT:    store <4 x i32> [[TMP1]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x float> [[TMPVAR3]]
+;
+  %tmpvar3 = call <4 x float> @llvm.rint.v4f32(<4 x float> %A)
+  ret <4 x float> %tmpvar3
+}
+
+define <2 x double> @frintx_2d(<2 x double> %A) nounwind #0 {
+; CHECK-LABEL: define <2 x double> @frintx_2d(
+; CHECK-SAME: <2 x double> [[A:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMPVAR3:%.*]] = call <2 x double> @llvm.rint.v2f64(<2 x double> [[A]])
+; CHECK-NEXT:    store <2 x i64> [[TMP1]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x double> [[TMPVAR3]]
+;
+  %tmpvar3 = call <2 x double> @llvm.rint.v2f64(<2 x double> %A)
+  ret <2 x double> %tmpvar3
+}
+
+declare <2 x float> @llvm.rint.v2f32(<2 x float>) nounwind readnone
+declare <4 x float> @llvm.rint.v4f32(<4 x float>) nounwind readnone
+declare <2 x double> @llvm.rint.v2f64(<2 x double>) nounwind readnone
+
+define <2 x float> @frintz_2s(<2 x float> %A) nounwind #0 {
+; CHECK-LABEL: define <2 x float> @frintz_2s(
+; CHECK-SAME: <2 x float> [[A:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMPVAR3:%.*]] = call <2 x float> @llvm.trunc.v2f32(<2 x float> [[A]])
+; CHECK-NEXT:    store <2 x i32> [[TMP1]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x float> [[TMPVAR3]]
+;
+  %tmpvar3 = call <2 x float> @llvm.trunc.v2f32(<2 x float> %A)
+  ret <2 x float> %tmpvar3
+}
+
+define <4 x float> @frintz_4s(<4 x float> %A) nounwind #0 {
+; CHECK-LABEL: define <4 x float> @frintz_4s(
+; CHECK-SAME: <4 x float> [[A:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMPVAR3:%.*]] = call <4 x float> @llvm.trunc.v4f32(<4 x float> [[A]])
+; CHECK-NEXT:    store <4 x i32> [[TMP1]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x float> [[TMPVAR3]]
+;
+  %tmpvar3 = call <4 x float> @llvm.trunc.v4f32(<4 x float> %A)
+  ret <4 x float> %tmpvar3
+}
+
+define <2 x double> @frintz_2d(<2 x double> %A) nounwind #0 {
+; CHECK-LABEL: define <2 x double> @frintz_2d(
+; CHECK-SAME: <2 x double> [[A:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMPVAR3:%.*]] = call <2 x double> @llvm.trunc.v2f64(<2 x double> [[A]])
+; CHECK-NEXT:    store <2 x i64> [[TMP1]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x double> [[TMPVAR3]]
+;
+  %tmpvar3 = call <2 x double> @llvm.trunc.v2f64(<2 x double> %A)
+  ret <2 x double> %tmpvar3
+}
+
+declare <2 x float> @llvm.trunc.v2f32(<2 x float>) nounwind readnone
+declare <4 x float> @llvm.trunc.v4f32(<4 x float>) nounwind readnone
+declare <2 x double> @llvm.trunc.v2f64(<2 x double>) nounwind readnone
+
+define <2 x float> @fcvtxn_2s(<2 x double> %A) nounwind #0 {
+; CHECK-LABEL: define <2 x float> @fcvtxn_2s(
+; CHECK-SAME: <2 x double> [[A:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <2 x i64> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP2]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP3:%.*]], label [[TMP4:%.*]], !prof [[PROF1]]
+; CHECK:       3:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR5]]
+; CHECK-NEXT:    unreachable
+; CHECK:       4:
+; CHECK-NEXT:    [[TMPVAR3:%.*]] = call <2 x float> @llvm.aarch64.neon.fcvtxn.v2f32.v2f64(<2 x double> [[A]])
+; CHECK-NEXT:    store <2 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x float> [[TMPVAR3]]
+;
+  %tmpvar3 = call <2 x float> @llvm.aarch64.neon.fcvtxn.v2f32.v2f64(<2 x double> %A)
+  ret <2 x float> %tmpvar3
+}
+
+define <4 x float> @fcvtxn_4s(<2 x float> %ret, <2 x double> %A) nounwind #0 {
+; CHECK-LABEL: define <4 x float> @fcvtxn_4s(
+; CHECK-SAME: <2 x float> [[RET:%.*]], <2 x double> [[A:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <2 x i64> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP3]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP4:%.*]], label [[TMP5:%.*]], !prof [[PROF1]]
+; CHECK:       4:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR5]]
+; CHECK-NEXT:    unreachable
+; CHECK:       5:
+; CHECK-NEXT:    [[TMPVAR3:%.*]] = call <2 x float> @llvm.aarch64.neon.fcvtxn.v2f32.v2f64(<2 x double> [[A]])
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <2 x i32> [[TMP2]], <2 x i32> zeroinitializer, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[RES:%.*]] = shufflevector <2 x float> [[RET]], <2 x float> [[TMPVAR3]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    store <4 x i32> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x float> [[RES]]
+;
+  %tmpvar3 = call <2 x float> @llvm.aarch64.neon.fcvtxn.v2f32.v2f64(<2 x double> %A)
+  %res = shufflevector <2 x float> %ret, <2 x float> %tmpvar3, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  ret <4 x float> %res
+}
+
+declare <2 x float> @llvm.aarch64.neon.fcvtxn.v2f32.v2f64(<2 x double>) nounwind readnone
+
+define <2 x i32> @fcvtzsc_2s(<2 x float> %A) nounwind #0 {
+; CHECK-LABEL: define <2 x i32> @fcvtzsc_2s(
+; CHECK-SAME: <2 x float> [[A:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <2 x i32> [[TMP1]] to i64
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP2]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP3:%.*]], label [[TMP4:%.*]], !prof [[PROF1]]
+; CHECK:       3:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR5]]
+; CHECK-NEXT:    unreachable
+; CHECK:       4:
+; CHECK-NEXT:    [[TMPVAR3:%.*]] = call <2 x i32> @llvm.aarch64.neon.vcvtfp2fxs.v2i32.v2f32(<2 x float> [[A]], i32 1)
+; CHECK-NEXT:    store <2 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x i32> [[TMPVAR3]]
+;
+  %tmpvar3 = call <2 x i32> @llvm.aarch64.neon.vcvtfp2fxs.v2i32.v2f32(<2 x float> %A, i32 1)
+  ret <2 x i32> %tmpvar3
+}
+
+define <4 x i32> @fcvtzsc_4s(<4 x float> %A) nounwind #0 {
+; CHECK-LABEL: define <4 x i32> @fcvtzsc_4s(
+; CHECK-SAME: <4 x float> [[A:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <4 x i32> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP2]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP3:%.*]], label [[TMP4:%.*]], !prof [[PROF1]]
+; CHECK:       3:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR5]]
+; CHECK-NEXT:    unreachable
+; CHECK:       4:
+; CHECK-NEXT:    [[TMPVAR3:%.*]] = call <4 x i32> @llvm.aarch64.neon.vcvtfp2fxs.v4i32.v4f32(<4 x float> [[A]], i32 1)
+; CHECK-NEXT:    store <4 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i32> [[TMPVAR3]]
+;
+  %tmpvar3 = call <4 x i32> @llvm.aarch64.neon.vcvtfp2fxs.v4i32.v4f32(<4 x float> %A, i32 1)
+  ret <4 x i32> %tmpvar3
+}
+
+define <2 x i64> @fcvtzsc_2d(<2 x double> %A) nounwind #0 {
+; CHECK-LABEL: define <2 x i64> @fcvtzsc_2d(
+; CHECK-SAME: <2 x double> [[A:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <2 x i64> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP2]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP3:%.*]], label [[TMP4:%.*]], !prof [[PROF1]]
+; CHECK:       3:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR5]]
+; CHECK-NEXT:    unreachable
+; CHECK:       4:
+; CHECK-NEXT:    [[TMPVAR3:%.*]] = call <2 x i64> @llvm.aarch64.neon.vcvtfp2fxs.v2i64.v2f64(<2 x double> [[A]], i32 1)
+; CHECK-NEXT:    store <2 x i64> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x i64> [[TMPVAR3]]
+;
+  %tmpvar3 = call <2 x i64> @llvm.aarch64.neon.vcvtfp2fxs.v2i64.v2f64(<2 x double> %A, i32 1)
+  ret <2 x i64> %tmpvar3
+}
+
+declare <2 x i32> @llvm.aarch64.neon.vcvtfp2fxs.v2i32.v2f32(<2 x float>, i32) nounwind readnone
+declare <4 x i32> @llvm.aarch64.neon.vcvtfp2fxs.v4i32.v4f32(<4 x float>, i32) nounwind readnone
+declare <2 x i64> @llvm.aarch64.neon.vcvtfp2fxs.v2i64.v2f64(<2 x double>, i32) nounwind readnone
+
+define <2 x i32> @fcvtzuc_2s(<2 x float> %A) nounwind #0 {
+; CHECK-LABEL: define <2 x i32> @fcvtzuc_2s(
+; CHECK-SAME: <2 x float> [[A:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <2 x i32> [[TMP1]] to i64
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP2]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP3:%.*]], label [[TMP4:%.*]], !prof [[PROF1]]
+; CHECK:       3:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR5]]
+; CHECK-NEXT:    unreachable
+; CHECK:       4:
+; CHECK-NEXT:    [[TMPVAR3:%.*]] = call <2 x i32> @llvm.aarch64.neon.vcvtfp2fxu.v2i32.v2f32(<2 x float> [[A]], i32 1)
+; CHECK-NEXT:    store <2 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x i32> [[TMPVAR3]]
+;
+  %tmpvar3 = call <2 x i32> @llvm.aarch64.neon.vcvtfp2fxu.v2i32.v2f32(<2 x float> %A, i32 1)
+  ret <2 x i32> %tmpvar3
+}
+
+define <4 x i32> @fcvtzuc_4s(<4 x float> %A) nounwind #0 {
+; CHECK-LABEL: define <4 x i32> @fcvtzuc_4s(
+; CHECK-SAME: <4 x float> [[A:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <4 x i32> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP2]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP3:%.*]], label [[TMP4:%.*]], !prof [[PROF1]]
+; CHECK:       3:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR5]]
+; CHECK-NEXT:    unreachable
+; CHECK:       4:
+; CHECK-NEXT:    [[TMPVAR3:%.*]] = call <4 x i32> @llvm.aarch64.neon.vcvtfp2fxu.v4i32.v4f32(<4 x float> [[A]], i32 1)
+; CHECK-NEXT:    store <4 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i32> [[TMPVAR3]]
+;
+  %tmpvar3 = call <4 x i32> @llvm.aarch64.neon.vcvtfp2fxu.v4i32.v4f32(<4 x float> %A, i32 1)
+  ret <4 x i32> %tmpvar3
+}
+
+define <2 x i64> @fcvtzuc_2d(<2 x double> %A) nounwind #0 {
+; CHECK-LABEL: define <2 x i64> @fcvtzuc_2d(
+; CHECK-SAME: <2 x double> [[A:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <2 x i64> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP2]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP3:%.*]], label [[TMP4:%.*]], !prof [[PROF1]]
+; CHECK:       3:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR5]]
+; CHECK-NEXT:    unreachable
+; CHECK:       4:
+; CHECK-NEXT:    [[TMPVAR3:%.*]] = call <2 x i64> @llvm.aarch64.neon.vcvtfp2fxu.v2i64.v2f64(<2 x double> [[A]], i32 1)
+; CHECK-NEXT:    store <2 x i64> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x i64> [[TMPVAR3]]
+;
+  %tmpvar3 = call <2 x i64> @llvm.aarch64.neon.vcvtfp2fxu.v2i64.v2f64(<2 x double> %A, i32 1)
+  ret <2 x i64> %tmpvar3
+}
+
+declare <2 x i32> @llvm.aarch64.neon.vcvtfp2fxu.v2i32.v2f32(<2 x float>, i32) nounwind readnone
+declare <4 x i32> @llvm.aarch64.neon.vcvtfp2fxu.v4i32.v4f32(<4 x float>, i32) nounwind readnone
+declare <2 x i64> @llvm.aarch64.neon.vcvtfp2fxu.v2i64.v2f64(<2 x double>, i32) nounwind readnone
+
+define <2 x float> @scvtf_2sc(<2 x i32> %A) nounwind #0 {
+; CHECK-LABEL: define <2 x float> @scvtf_2sc(
+; CHECK-SAME: <2 x i32> [[A:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <2 x i32> [[TMP1]] to i64
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP2]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP3:%.*]], label [[TMP4:%.*]], !prof [[PROF1]]
+; CHECK:       3:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR5]]
+; CHECK-NEXT:    unreachable
+; CHECK:       4:
+; CHECK-NEXT:    [[TMPVAR3:%.*]] = call <2 x float> @llvm.aarch64.neon.vcvtfxs2fp.v2f32.v2i32(<2 x i32> [[A]], i32 1)
+; CHECK-NEXT:    store <2 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x float> [[TMPVAR3]]
+;
+  %tmpvar3 = call <2 x float> @llvm.aarch64.neon.vcvtfxs2fp.v2f32.v2i32(<2 x i32> %A, i32 1)
+  ret <2 x float> %tmpvar3
+}
+
+define <4 x float> @scvtf_4sc(<4 x i32> %A) nounwind #0 {
+; CHECK-LABEL: define <4 x float> @scvtf_4sc(
+; CHECK-SAME: <4 x i32> [[A:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <4 x i32> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP2]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP3:%.*]], label [[TMP4:%.*]], !prof [[PROF1]]
+; CHECK:       3:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR5]]
+; CHECK-NEXT:    unreachable
+; CHECK:       4:
+; CHECK-NEXT:    [[TMPVAR3:%.*]] = call <4 x float> @llvm.aarch64.neon.vcvtfxs2fp.v4f32.v4i32(<4 x i32> [[A]], i32 1)
+; CHECK-NEXT:    store <4 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x float> [[TMPVAR3]]
+;
+  %tmpvar3 = call <4 x float> @llvm.aarch64.neon.vcvtfxs2fp.v4f32.v4i32(<4 x i32> %A, i32 1)
+  ret <4 x float> %tmpvar3
+}
+
+define <2 x double> @scvtf_2dc(<2 x i64> %A) nounwind #0 {
+; CHECK-LABEL: define <2 x double> @scvtf_2dc(
+; CHECK-SAME: <2 x i64> [[A:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <2 x i64> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP2]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP3:%.*]], label [[TMP4:%.*]], !prof [[PROF1]]
+; CHECK:       3:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR5]]
+; CHECK-NEXT:    unreachable
+; CHECK:       4:
+; CHECK-NEXT:    [[TMPVAR3:%.*]] = call <2 x double> @llvm.aarch64.neon.vcvtfxs2fp.v2f64.v2i64(<2 x i64> [[A]], i32 1)
+; CHECK-NEXT:    store <2 x i64> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x double> [[TMPVAR3]]
+;
+  %tmpvar3 = call <2 x double> @llvm.aarch64.neon.vcvtfxs2fp.v2f64.v2i64(<2 x i64> %A, i32 1)
+  ret <2 x double> %tmpvar3
+}
+
+declare <2 x float> @llvm.aarch64.neon.vcvtfxs2fp.v2f32.v2i32(<2 x i32>, i32) nounwind readnone
+declare <4 x float> @llvm.aarch64.neon.vcvtfxs2fp.v4f32.v4i32(<4 x i32>, i32) nounwind readnone
+declare <2 x double> @llvm.aarch64.neon.vcvtfxs2fp.v2f64.v2i64(<2 x i64>, i32) nounwind readnone
+
+define <2 x float> @ucvtf_2sc(<2 x i32> %A) nounwind #0 {
+; CHECK-LABEL: define <2 x float> @ucvtf_2sc(
+; CHECK-SAME: <2 x i32> [[A:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <2 x i32> [[TMP1]] to i64
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP2]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP3:%.*]], label [[TMP4:%.*]], !prof [[PROF1]]
+; CHECK:       3:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR5]]
+; CHECK-NEXT:    unreachable
+; CHECK:       4:
+; CHECK-NEXT:    [[TMPVAR3:%.*]] = call <2 x float> @llvm.aarch64.neon.vcvtfxu2fp.v2f32.v2i32(<2 x i32> [[A]], i32 1)
+; CHECK-NEXT:    store <2 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x float> [[TMPVAR3]]
+;
+  %tmpvar3 = call <2 x float> @llvm.aarch64.neon.vcvtfxu2fp.v2f32.v2i32(<2 x i32> %A, i32 1)
+  ret <2 x float> %tmpvar3
+}
+
+define <4 x float> @ucvtf_4sc(<4 x i32> %A) nounwind #0 {
+; CHECK-LABEL: define <4 x float> @ucvtf_4sc(
+; CHECK-SAME: <4 x i32> [[A:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <4 x i32> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP2]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP3:%.*]], label [[TMP4:%.*]], !prof [[PROF1]]
+; CHECK:       3:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR5]]
+; CHECK-NEXT:    unreachable
+; CHECK:       4:
+; CHECK-NEXT:    [[TMPVAR3:%.*]] = call <4 x float> @llvm.aarch64.neon.vcvtfxu2fp.v4f32.v4i32(<4 x i32> [[A]], i32 1)
+; CHECK-NEXT:    store <4 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x float> [[TMPVAR3]]
+;
+  %tmpvar3 = call <4 x float> @llvm.aarch64.neon.vcvtfxu2fp.v4f32.v4i32(<4 x i32> %A, i32 1)
+  ret <4 x float> %tmpvar3
+}
+
+define <2 x double> @ucvtf_2dc(<2 x i64> %A) nounwind #0 {
+; CHECK-LABEL: define <2 x double> @ucvtf_2dc(
+; CHECK-SAME: <2 x i64> [[A:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <2 x i64> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP2]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP3:%.*]], label [[TMP4:%.*]], !prof [[PROF1]]
+; CHECK:       3:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR5]]
+; CHECK-NEXT:    unreachable
+; CHECK:       4:
+; CHECK-NEXT:    [[TMPVAR3:%.*]] = call <2 x double> @llvm.aarch64.neon.vcvtfxu2fp.v2f64.v2i64(<2 x i64> [[A]], i32 1)
+; CHECK-NEXT:    store <2 x i64> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x double> [[TMPVAR3]]
+;
+  %tmpvar3 = call <2 x double> @llvm.aarch64.neon.vcvtfxu2fp.v2f64.v2i64(<2 x i64> %A, i32 1)
+  ret <2 x double> %tmpvar3
+}
+
+
+define void @autogen_SD28458(<8 x double> %val.f64, ptr %addr.f32) #0 {
+; CHECK-LABEL: define void @autogen_SD28458(
+; CHECK-SAME: <8 x double> [[VAL_F64:%.*]], ptr [[ADDR_F32:%.*]]) #[[ATTR3:[0-9]+]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = trunc <8 x i64> [[TMP1]] to <8 x i32>
+; CHECK-NEXT:    [[TR53:%.*]] = fptrunc <8 x double> [[VAL_F64]] to <8 x float>
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP2]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP4:%.*]], label [[TMP5:%.*]], !prof [[PROF1]]
+; CHECK:       4:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR5]]
+; CHECK-NEXT:    unreachable
+; CHECK:       5:
+; CHECK-NEXT:    [[TMP6:%.*]] = ptrtoint ptr [[ADDR_F32]] to i64
+; CHECK-NEXT:    [[TMP7:%.*]] = xor i64 [[TMP6]], 193514046488576
+; CHECK-NEXT:    [[TMP8:%.*]] = inttoptr i64 [[TMP7]] to ptr
+; CHECK-NEXT:    store <8 x i32> [[TMP3]], ptr [[TMP8]], align 32
+; CHECK-NEXT:    store <8 x float> [[TR53]], ptr [[ADDR_F32]], align 32
+; CHECK-NEXT:    ret void
+;
+  %Tr53 = fptrunc <8 x double> %val.f64 to <8 x float>
+  store <8 x float> %Tr53, ptr %addr.f32
+  ret void
+}
+
+define void @autogen_SD19225(ptr %addr.f64, ptr %addr.f32) #0 {
+; CHECK-LABEL: define void @autogen_SD19225(
+; CHECK-SAME: ptr [[ADDR_F64:%.*]], ptr [[ADDR_F32:%.*]]) #[[ATTR3]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP3:%.*]], label [[TMP4:%.*]], !prof [[PROF1]]
+; CHECK:       3:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR5]]
+; CHECK-NEXT:    unreachable
+; CHECK:       4:
+; CHECK-NEXT:    [[A:%.*]] = load <8 x float>, ptr [[ADDR_F32]], align 32
+; CHECK-NEXT:    [[TMP5:%.*]] = ptrtoint ptr [[ADDR_F32]] to i64
+; CHECK-NEXT:    [[TMP6:%.*]] = xor i64 [[TMP5]], 193514046488576
+; CHECK-NEXT:    [[TMP7:%.*]] = inttoptr i64 [[TMP6]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <8 x i32>, ptr [[TMP7]], align 32
+; CHECK-NEXT:    [[TMP8:%.*]] = zext <8 x i32> [[_MSLD]] to <8 x i64>
+; CHECK-NEXT:    [[TR53:%.*]] = fpext <8 x float> [[A]] to <8 x double>
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i64 [[TMP2]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP1]], label [[TMP9:%.*]], label [[TMP10:%.*]], !prof [[PROF1]]
+; CHECK:       9:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR5]]
+; CHECK-NEXT:    unreachable
+; CHECK:       10:
+; CHECK-NEXT:    [[TMP11:%.*]] = ptrtoint ptr [[ADDR_F64]] to i64
+; CHECK-NEXT:    [[TMP12:%.*]] = xor i64 [[TMP11]], 193514046488576
+; CHECK-NEXT:    [[TMP13:%.*]] = inttoptr i64 [[TMP12]] to ptr
+; CHECK-NEXT:    store <8 x i64> [[TMP8]], ptr [[TMP13]], align 64
+; CHECK-NEXT:    store <8 x double> [[TR53]], ptr [[ADDR_F64]], align 64
+; CHECK-NEXT:    ret void
+;
+  %A = load <8 x float>, ptr %addr.f32
+  %Tr53 = fpext <8 x float> %A to <8 x double>
+  store <8 x double> %Tr53, ptr %addr.f64
+  ret void
+}
+
+declare <2 x float> @llvm.aarch64.neon.vcvtfxu2fp.v2f32.v2i32(<2 x i32>, i32) nounwind readnone
+declare <4 x float> @llvm.aarch64.neon.vcvtfxu2fp.v4f32.v4i32(<4 x i32>, i32) nounwind readnone
+declare <2 x double> @llvm.aarch64.neon.vcvtfxu2fp.v2f64.v2i64(<2 x i64>, i32) nounwind readnone
+
+attributes #0 = { sanitize_memory }
+;.
+; CHECK: [[PROF1]] = !{!"branch_weights", i32 1, i32 1048575}
+;.
diff --git a/llvm/test/Instrumentation/MemorySanitizer/AArch64/arm64-vmovn.ll b/llvm/test/Instrumentation/MemorySanitizer/AArch64/arm64-vmovn.ll
new file mode 100644
index 0000000000000..5ecedf61871d8
--- /dev/null
+++ b/llvm/test/Instrumentation/MemorySanitizer/AArch64/arm64-vmovn.ll
@@ -0,0 +1,521 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt < %s -passes=msan -S | FileCheck %s
+;
+; Forked from llvm/test/CodeGen/AArch64/arm64-vmovn.ll
+;
+; Not correctly handled (by visitInstruction):
+; - llvm.aarch64.neon.sqxtn
+; - llvm.aarch64.neon.sqxtun
+; - llvm.aarch64.neon.uqxtn
+
+target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
+target triple = "aarch64--linux-android9001"
+
+define <8 x i8> @xtn8b(<8 x i16> %A) nounwind #0 {
+; CHECK-LABEL: define <8 x i8> @xtn8b(
+; CHECK-SAME: <8 x i16> [[A:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = trunc <8 x i16> [[TMP1]] to <8 x i8>
+; CHECK-NEXT:    [[TMP3:%.*]] = trunc <8 x i16> [[A]] to <8 x i8>
+; CHECK-NEXT:    store <8 x i8> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i8> [[TMP3]]
+;
+  %tmpvar3 = trunc <8 x i16> %A to <8 x i8>
+  ret <8 x i8> %tmpvar3
+}
+
+define <4 x i16> @xtn4h(<4 x i32> %A) nounwind #0 {
+; CHECK-LABEL: define <4 x i16> @xtn4h(
+; CHECK-SAME: <4 x i32> [[A:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = trunc <4 x i32> [[TMP1]] to <4 x i16>
+; CHECK-NEXT:    [[TMP3:%.*]] = trunc <4 x i32> [[A]] to <4 x i16>
+; CHECK-NEXT:    store <4 x i16> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i16> [[TMP3]]
+;
+  %tmpvar3 = trunc <4 x i32> %A to <4 x i16>
+  ret <4 x i16> %tmpvar3
+}
+
+define <2 x i32> @xtn2s(<2 x i64> %A) nounwind #0 {
+; CHECK-LABEL: define <2 x i32> @xtn2s(
+; CHECK-SAME: <2 x i64> [[A:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = trunc <2 x i64> [[TMP1]] to <2 x i32>
+; CHECK-NEXT:    [[TMP3:%.*]] = trunc <2 x i64> [[A]] to <2 x i32>
+; CHECK-NEXT:    store <2 x i32> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x i32> [[TMP3]]
+;
+  %tmpvar3 = trunc <2 x i64> %A to <2 x i32>
+  ret <2 x i32> %tmpvar3
+}
+
+define <16 x i8> @xtn2_16b(<8 x i8> %ret, <8 x i16> %A) nounwind #0 {
+; CHECK-LABEL: define <16 x i8> @xtn2_16b(
+; CHECK-SAME: <8 x i8> [[RET:%.*]], <8 x i16> [[A:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i8>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = trunc <8 x i16> [[TMP1]] to <8 x i8>
+; CHECK-NEXT:    [[TMP3:%.*]] = trunc <8 x i16> [[A]] to <8 x i8>
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = shufflevector <8 x i8> [[TMP2]], <8 x i8> [[_MSPROP]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; CHECK-NEXT:    [[RES:%.*]] = shufflevector <8 x i8> [[RET]], <8 x i8> [[TMP3]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; CHECK-NEXT:    store <16 x i8> [[_MSPROP1]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i8> [[RES]]
+;
+  %tmpvar3 = trunc <8 x i16> %A to <8 x i8>
+  %res = shufflevector <8 x i8> %ret, <8 x i8> %tmpvar3, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+  ret <16 x i8> %res
+}
+
+define <8 x i16> @xtn2_8h(<4 x i16> %ret, <4 x i32> %A) nounwind #0 {
+; CHECK-LABEL: define <8 x i16> @xtn2_8h(
+; CHECK-SAME: <4 x i16> [[RET:%.*]], <4 x i32> [[A:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = trunc <4 x i32> [[TMP1]] to <4 x i16>
+; CHECK-NEXT:    [[TMP3:%.*]] = trunc <4 x i32> [[A]] to <4 x i16>
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = shufflevector <4 x i16> [[TMP2]], <4 x i16> [[_MSPROP]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:    [[RES:%.*]] = shufflevector <4 x i16> [[RET]], <4 x i16> [[TMP3]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:    store <8 x i16> [[_MSPROP1]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i16> [[RES]]
+;
+  %tmpvar3 = trunc <4 x i32> %A to <4 x i16>
+  %res = shufflevector <4 x i16> %ret, <4 x i16> %tmpvar3, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+  ret <8 x i16> %res
+}
+
+define <4 x i32> @xtn2_4s(<2 x i32> %ret, <2 x i64> %A) nounwind #0 {
+; CHECK-LABEL: define <4 x i32> @xtn2_4s(
+; CHECK-SAME: <2 x i32> [[RET:%.*]], <2 x i64> [[A:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = trunc <2 x i64> [[TMP1]] to <2 x i32>
+; CHECK-NEXT:    [[TMP3:%.*]] = trunc <2 x i64> [[A]] to <2 x i32>
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = shufflevector <2 x i32> [[TMP2]], <2 x i32> [[_MSPROP]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[RES:%.*]] = shufflevector <2 x i32> [[RET]], <2 x i32> [[TMP3]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    store <4 x i32> [[_MSPROP1]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i32> [[RES]]
+;
+  %tmpvar3 = trunc <2 x i64> %A to <2 x i32>
+  %res = shufflevector <2 x i32> %ret, <2 x i32> %tmpvar3, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  ret <4 x i32> %res
+}
+
+define <8 x i8> @sqxtn8b(<8 x i16> %A) nounwind #0 {
+; CHECK-LABEL: define <8 x i8> @sqxtn8b(
+; CHECK-SAME: <8 x i16> [[A:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <8 x i16> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP2]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB3:.*]], label %[[BB4:.*]], !prof [[PROF1:![0-9]+]]
+; CHECK:       [[BB3]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR3:[0-9]+]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB4]]:
+; CHECK-NEXT:    [[TMP3:%.*]] = call <8 x i8> @llvm.aarch64.neon.sqxtn.v8i8(<8 x i16> [[A]])
+; CHECK-NEXT:    store <8 x i8> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i8> [[TMP3]]
+;
+  %tmpvar3 = call <8 x i8> @llvm.aarch64.neon.sqxtn.v8i8(<8 x i16> %A)
+  ret <8 x i8> %tmpvar3
+}
+
+define <4 x i16> @sqxtn4h(<4 x i32> %A) nounwind #0 {
+; CHECK-LABEL: define <4 x i16> @sqxtn4h(
+; CHECK-SAME: <4 x i32> [[A:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <4 x i32> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP2]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB3:.*]], label %[[BB4:.*]], !prof [[PROF1]]
+; CHECK:       [[BB3]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR3]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB4]]:
+; CHECK-NEXT:    [[TMP3:%.*]] = call <4 x i16> @llvm.aarch64.neon.sqxtn.v4i16(<4 x i32> [[A]])
+; CHECK-NEXT:    store <4 x i16> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i16> [[TMP3]]
+;
+  %tmpvar3 = call <4 x i16> @llvm.aarch64.neon.sqxtn.v4i16(<4 x i32> %A)
+  ret <4 x i16> %tmpvar3
+}
+
+define <2 x i32> @sqxtn2s(<2 x i64> %A) nounwind #0 {
+; CHECK-LABEL: define <2 x i32> @sqxtn2s(
+; CHECK-SAME: <2 x i64> [[A:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <2 x i64> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP2]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB3:.*]], label %[[BB4:.*]], !prof [[PROF1]]
+; CHECK:       [[BB3]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR3]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB4]]:
+; CHECK-NEXT:    [[TMP3:%.*]] = call <2 x i32> @llvm.aarch64.neon.sqxtn.v2i32(<2 x i64> [[A]])
+; CHECK-NEXT:    store <2 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x i32> [[TMP3]]
+;
+  %tmpvar3 = call <2 x i32> @llvm.aarch64.neon.sqxtn.v2i32(<2 x i64> %A)
+  ret <2 x i32> %tmpvar3
+}
+
+define <16 x i8> @sqxtn2_16b(<8 x i8> %ret, <8 x i16> %A) nounwind #0 {
+; CHECK-LABEL: define <16 x i8> @sqxtn2_16b(
+; CHECK-SAME: <8 x i8> [[RET:%.*]], <8 x i16> [[A:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i8>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <8 x i16> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP4]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB4:.*]], label %[[BB5:.*]], !prof [[PROF1]]
+; CHECK:       [[BB4]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR3]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB5]]:
+; CHECK-NEXT:    [[TMP3:%.*]] = call <8 x i8> @llvm.aarch64.neon.sqxtn.v8i8(<8 x i16> [[A]])
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <8 x i8> [[TMP2]], <8 x i8> zeroinitializer, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; CHECK-NEXT:    [[RES:%.*]] = shufflevector <8 x i8> [[RET]], <8 x i8> [[TMP3]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; CHECK-NEXT:    store <16 x i8> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i8> [[RES]]
+;
+  %tmpvar3 = call <8 x i8> @llvm.aarch64.neon.sqxtn.v8i8(<8 x i16> %A)
+  %res = shufflevector <8 x i8> %ret, <8 x i8> %tmpvar3, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+  ret <16 x i8> %res
+}
+
+define <8 x i16> @sqxtn2_8h(<4 x i16> %ret, <4 x i32> %A) nounwind #0 {
+; CHECK-LABEL: define <8 x i16> @sqxtn2_8h(
+; CHECK-SAME: <4 x i16> [[RET:%.*]], <4 x i32> [[A:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <4 x i32> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP4]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB4:.*]], label %[[BB5:.*]], !prof [[PROF1]]
+; CHECK:       [[BB4]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR3]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB5]]:
+; CHECK-NEXT:    [[TMP3:%.*]] = call <4 x i16> @llvm.aarch64.neon.sqxtn.v4i16(<4 x i32> [[A]])
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <4 x i16> [[TMP2]], <4 x i16> zeroinitializer, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:    [[RES:%.*]] = shufflevector <4 x i16> [[RET]], <4 x i16> [[TMP3]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:    store <8 x i16> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i16> [[RES]]
+;
+  %tmpvar3 = call <4 x i16> @llvm.aarch64.neon.sqxtn.v4i16(<4 x i32> %A)
+  %res = shufflevector <4 x i16> %ret, <4 x i16> %tmpvar3, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+  ret <8 x i16> %res
+}
+
+define <4 x i32> @sqxtn2_4s(<2 x i32> %ret, <2 x i64> %A) nounwind #0 {
+; CHECK-LABEL: define <4 x i32> @sqxtn2_4s(
+; CHECK-SAME: <2 x i32> [[RET:%.*]], <2 x i64> [[A:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <2 x i64> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP4]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB4:.*]], label %[[BB5:.*]], !prof [[PROF1]]
+; CHECK:       [[BB4]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR3]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB5]]:
+; CHECK-NEXT:    [[TMP3:%.*]] = call <2 x i32> @llvm.aarch64.neon.sqxtn.v2i32(<2 x i64> [[A]])
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <2 x i32> [[TMP2]], <2 x i32> zeroinitializer, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[RES:%.*]] = shufflevector <2 x i32> [[RET]], <2 x i32> [[TMP3]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    store <4 x i32> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i32> [[RES]]
+;
+  %tmpvar3 = call <2 x i32> @llvm.aarch64.neon.sqxtn.v2i32(<2 x i64> %A)
+  %res = shufflevector <2 x i32> %ret, <2 x i32> %tmpvar3, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  ret <4 x i32> %res
+}
+
+declare <8 x i8>  @llvm.aarch64.neon.sqxtn.v8i8(<8 x i16>) nounwind readnone
+declare <4 x i16> @llvm.aarch64.neon.sqxtn.v4i16(<4 x i32>) nounwind readnone
+declare <2 x i32> @llvm.aarch64.neon.sqxtn.v2i32(<2 x i64>) nounwind readnone
+
+define <8 x i8> @uqxtn8b(<8 x i16> %A) nounwind #0 {
+; CHECK-LABEL: define <8 x i8> @uqxtn8b(
+; CHECK-SAME: <8 x i16> [[A:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <8 x i16> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP2]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB3:.*]], label %[[BB4:.*]], !prof [[PROF1]]
+; CHECK:       [[BB3]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR3]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB4]]:
+; CHECK-NEXT:    [[TMP3:%.*]] = call <8 x i8> @llvm.aarch64.neon.uqxtn.v8i8(<8 x i16> [[A]])
+; CHECK-NEXT:    store <8 x i8> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i8> [[TMP3]]
+;
+  %tmpvar3 = call <8 x i8> @llvm.aarch64.neon.uqxtn.v8i8(<8 x i16> %A)
+  ret <8 x i8> %tmpvar3
+}
+
+define <4 x i16> @uqxtn4h(<4 x i32> %A) nounwind #0 {
+; CHECK-LABEL: define <4 x i16> @uqxtn4h(
+; CHECK-SAME: <4 x i32> [[A:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <4 x i32> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP2]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB3:.*]], label %[[BB4:.*]], !prof [[PROF1]]
+; CHECK:       [[BB3]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR3]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB4]]:
+; CHECK-NEXT:    [[TMP3:%.*]] = call <4 x i16> @llvm.aarch64.neon.uqxtn.v4i16(<4 x i32> [[A]])
+; CHECK-NEXT:    store <4 x i16> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i16> [[TMP3]]
+;
+  %tmpvar3 = call <4 x i16> @llvm.aarch64.neon.uqxtn.v4i16(<4 x i32> %A)
+  ret <4 x i16> %tmpvar3
+}
+
+define <2 x i32> @uqxtn2s(<2 x i64> %A) nounwind #0 {
+; CHECK-LABEL: define <2 x i32> @uqxtn2s(
+; CHECK-SAME: <2 x i64> [[A:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <2 x i64> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP2]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB3:.*]], label %[[BB4:.*]], !prof [[PROF1]]
+; CHECK:       [[BB3]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR3]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB4]]:
+; CHECK-NEXT:    [[TMP3:%.*]] = call <2 x i32> @llvm.aarch64.neon.uqxtn.v2i32(<2 x i64> [[A]])
+; CHECK-NEXT:    store <2 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x i32> [[TMP3]]
+;
+  %tmpvar3 = call <2 x i32> @llvm.aarch64.neon.uqxtn.v2i32(<2 x i64> %A)
+  ret <2 x i32> %tmpvar3
+}
+
+define <16 x i8> @uqxtn2_16b(<8 x i8> %ret, <8 x i16> %A) nounwind #0 {
+; CHECK-LABEL: define <16 x i8> @uqxtn2_16b(
+; CHECK-SAME: <8 x i8> [[RET:%.*]], <8 x i16> [[A:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i8>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <8 x i16> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP4]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB4:.*]], label %[[BB5:.*]], !prof [[PROF1]]
+; CHECK:       [[BB4]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR3]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB5]]:
+; CHECK-NEXT:    [[TMP3:%.*]] = call <8 x i8> @llvm.aarch64.neon.uqxtn.v8i8(<8 x i16> [[A]])
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <8 x i8> [[TMP2]], <8 x i8> zeroinitializer, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; CHECK-NEXT:    [[RES:%.*]] = shufflevector <8 x i8> [[RET]], <8 x i8> [[TMP3]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; CHECK-NEXT:    store <16 x i8> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i8> [[RES]]
+;
+  %tmpvar3 = call <8 x i8> @llvm.aarch64.neon.uqxtn.v8i8(<8 x i16> %A)
+  %res = shufflevector <8 x i8> %ret, <8 x i8> %tmpvar3, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+  ret <16 x i8> %res
+}
+
+define <8 x i16> @uqxtn2_8h(<4 x i16> %ret, <4 x i32> %A) nounwind #0 {
+; CHECK-LABEL: define <8 x i16> @uqxtn2_8h(
+; CHECK-SAME: <4 x i16> [[RET:%.*]], <4 x i32> [[A:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <4 x i32> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP4]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB4:.*]], label %[[BB5:.*]], !prof [[PROF1]]
+; CHECK:       [[BB4]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR3]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB5]]:
+; CHECK-NEXT:    [[TMP3:%.*]] = call <4 x i16> @llvm.aarch64.neon.uqxtn.v4i16(<4 x i32> [[A]])
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <4 x i16> [[TMP2]], <4 x i16> zeroinitializer, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:    [[RES:%.*]] = shufflevector <4 x i16> [[RET]], <4 x i16> [[TMP3]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:    store <8 x i16> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i16> [[RES]]
+;
+  %tmpvar3 = call <4 x i16> @llvm.aarch64.neon.uqxtn.v4i16(<4 x i32> %A)
+  %res = shufflevector <4 x i16> %ret, <4 x i16> %tmpvar3, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+  ret <8 x i16> %res
+}
+
+define <4 x i32> @uqxtn2_4s(<2 x i32> %ret, <2 x i64> %A) nounwind #0 {
+; CHECK-LABEL: define <4 x i32> @uqxtn2_4s(
+; CHECK-SAME: <2 x i32> [[RET:%.*]], <2 x i64> [[A:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <2 x i64> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP4]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB4:.*]], label %[[BB5:.*]], !prof [[PROF1]]
+; CHECK:       [[BB4]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR3]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB5]]:
+; CHECK-NEXT:    [[TMP3:%.*]] = call <2 x i32> @llvm.aarch64.neon.uqxtn.v2i32(<2 x i64> [[A]])
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <2 x i32> [[TMP2]], <2 x i32> zeroinitializer, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[RES:%.*]] = shufflevector <2 x i32> [[RET]], <2 x i32> [[TMP3]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    store <4 x i32> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i32> [[RES]]
+;
+  %tmpvar3 = call <2 x i32> @llvm.aarch64.neon.uqxtn.v2i32(<2 x i64> %A)
+  %res = shufflevector <2 x i32> %ret, <2 x i32> %tmpvar3, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  ret <4 x i32> %res
+}
+
+declare <8 x i8>  @llvm.aarch64.neon.uqxtn.v8i8(<8 x i16>) nounwind readnone
+declare <4 x i16> @llvm.aarch64.neon.uqxtn.v4i16(<4 x i32>) nounwind readnone
+declare <2 x i32> @llvm.aarch64.neon.uqxtn.v2i32(<2 x i64>) nounwind readnone
+
+define <8 x i8> @sqxtun8b(<8 x i16> %A) nounwind #0 {
+; CHECK-LABEL: define <8 x i8> @sqxtun8b(
+; CHECK-SAME: <8 x i16> [[A:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <8 x i16> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP2]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB3:.*]], label %[[BB4:.*]], !prof [[PROF1]]
+; CHECK:       [[BB3]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR3]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB4]]:
+; CHECK-NEXT:    [[TMP3:%.*]] = call <8 x i8> @llvm.aarch64.neon.sqxtun.v8i8(<8 x i16> [[A]])
+; CHECK-NEXT:    store <8 x i8> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i8> [[TMP3]]
+;
+  %tmpvar3 = call <8 x i8> @llvm.aarch64.neon.sqxtun.v8i8(<8 x i16> %A)
+  ret <8 x i8> %tmpvar3
+}
+
+define <4 x i16> @sqxtun4h(<4 x i32> %A) nounwind #0 {
+; CHECK-LABEL: define <4 x i16> @sqxtun4h(
+; CHECK-SAME: <4 x i32> [[A:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <4 x i32> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP2]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB3:.*]], label %[[BB4:.*]], !prof [[PROF1]]
+; CHECK:       [[BB3]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR3]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB4]]:
+; CHECK-NEXT:    [[TMP3:%.*]] = call <4 x i16> @llvm.aarch64.neon.sqxtun.v4i16(<4 x i32> [[A]])
+; CHECK-NEXT:    store <4 x i16> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i16> [[TMP3]]
+;
+  %tmpvar3 = call <4 x i16> @llvm.aarch64.neon.sqxtun.v4i16(<4 x i32> %A)
+  ret <4 x i16> %tmpvar3
+}
+
+define <2 x i32> @sqxtun2s(<2 x i64> %A) nounwind #0 {
+; CHECK-LABEL: define <2 x i32> @sqxtun2s(
+; CHECK-SAME: <2 x i64> [[A:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <2 x i64> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP2]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB3:.*]], label %[[BB4:.*]], !prof [[PROF1]]
+; CHECK:       [[BB3]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR3]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB4]]:
+; CHECK-NEXT:    [[TMP3:%.*]] = call <2 x i32> @llvm.aarch64.neon.sqxtun.v2i32(<2 x i64> [[A]])
+; CHECK-NEXT:    store <2 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x i32> [[TMP3]]
+;
+  %tmpvar3 = call <2 x i32> @llvm.aarch64.neon.sqxtun.v2i32(<2 x i64> %A)
+  ret <2 x i32> %tmpvar3
+}
+
+define <16 x i8> @sqxtun2_16b(<8 x i8> %ret, <8 x i16> %A) nounwind #0 {
+; CHECK-LABEL: define <16 x i8> @sqxtun2_16b(
+; CHECK-SAME: <8 x i8> [[RET:%.*]], <8 x i16> [[A:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i8>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <8 x i16> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP4]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB4:.*]], label %[[BB5:.*]], !prof [[PROF1]]
+; CHECK:       [[BB4]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR3]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB5]]:
+; CHECK-NEXT:    [[TMP3:%.*]] = call <8 x i8> @llvm.aarch64.neon.sqxtun.v8i8(<8 x i16> [[A]])
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <8 x i8> [[TMP2]], <8 x i8> zeroinitializer, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; CHECK-NEXT:    [[RES:%.*]] = shufflevector <8 x i8> [[RET]], <8 x i8> [[TMP3]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; CHECK-NEXT:    store <16 x i8> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i8> [[RES]]
+;
+  %tmpvar3 = call <8 x i8> @llvm.aarch64.neon.sqxtun.v8i8(<8 x i16> %A)
+  %res = shufflevector <8 x i8> %ret, <8 x i8> %tmpvar3, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+  ret <16 x i8> %res
+}
+
+define <8 x i16> @sqxtun2_8h(<4 x i16> %ret, <4 x i32> %A) nounwind #0 {
+; CHECK-LABEL: define <8 x i16> @sqxtun2_8h(
+; CHECK-SAME: <4 x i16> [[RET:%.*]], <4 x i32> [[A:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <4 x i32> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP4]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB4:.*]], label %[[BB5:.*]], !prof [[PROF1]]
+; CHECK:       [[BB4]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR3]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB5]]:
+; CHECK-NEXT:    [[TMP3:%.*]] = call <4 x i16> @llvm.aarch64.neon.sqxtun.v4i16(<4 x i32> [[A]])
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <4 x i16> [[TMP2]], <4 x i16> zeroinitializer, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:    [[RES:%.*]] = shufflevector <4 x i16> [[RET]], <4 x i16> [[TMP3]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:    store <8 x i16> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i16> [[RES]]
+;
+  %tmpvar3 = call <4 x i16> @llvm.aarch64.neon.sqxtun.v4i16(<4 x i32> %A)
+  %res = shufflevector <4 x i16> %ret, <4 x i16> %tmpvar3, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+  ret <8 x i16> %res
+}
+
+define <4 x i32> @sqxtun2_4s(<2 x i32> %ret, <2 x i64> %A) nounwind #0 {
+; CHECK-LABEL: define <4 x i32> @sqxtun2_4s(
+; CHECK-SAME: <2 x i32> [[RET:%.*]], <2 x i64> [[A:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <2 x i64> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP4]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB4:.*]], label %[[BB5:.*]], !prof [[PROF1]]
+; CHECK:       [[BB4]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR3]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB5]]:
+; CHECK-NEXT:    [[TMP3:%.*]] = call <2 x i32> @llvm.aarch64.neon.sqxtun.v2i32(<2 x i64> [[A]])
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <2 x i32> [[TMP2]], <2 x i32> zeroinitializer, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[RES:%.*]] = shufflevector <2 x i32> [[RET]], <2 x i32> [[TMP3]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    store <4 x i32> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i32> [[RES]]
+;
+  %tmpvar3 = call <2 x i32> @llvm.aarch64.neon.sqxtun.v2i32(<2 x i64> %A)
+  %res = shufflevector <2 x i32> %ret, <2 x i32> %tmpvar3, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  ret <4 x i32> %res
+}
+
+declare <8 x i8>  @llvm.aarch64.neon.sqxtun.v8i8(<8 x i16>) nounwind readnone
+declare <4 x i16> @llvm.aarch64.neon.sqxtun.v4i16(<4 x i32>) nounwind readnone
+declare <2 x i32> @llvm.aarch64.neon.sqxtun.v2i32(<2 x i64>) nounwind readnone
+
+attributes #0 = { sanitize_memory }
+;.
+; CHECK: [[PROF1]] = !{!"branch_weights", i32 1, i32 1048575}
+;.
diff --git a/llvm/test/Instrumentation/MemorySanitizer/AArch64/qshrn.ll b/llvm/test/Instrumentation/MemorySanitizer/AArch64/qshrn.ll
new file mode 100644
index 0000000000000..f35178a30dc14
--- /dev/null
+++ b/llvm/test/Instrumentation/MemorySanitizer/AArch64/qshrn.ll
@@ -0,0 +1,753 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 4
+; RUN: opt < %s -passes=msan -S | FileCheck %s
+;
+; Forked from llvm/test/CodeGen/AArch64/qshrn.ll
+;
+; Heuristically (but correctly) handled: llvm.smax, llvm.smin, llvm.umin
+; Incorrectly handled (handleUnknownInstruction): llvm.aarch64.neon.sqxtn, llvm.aarch64.neon.sqxtun, llvm.aarch64.neon.uqxtn
+
+target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
+target triple = "aarch64--linux-android9001"
+
+define <4 x i16> @NarrowAShrI32By5(<4 x i32> %x) #0 {
+; CHECK-LABEL: define <4 x i16> @NarrowAShrI32By5(
+; CHECK-SAME: <4 x i32> [[X:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = ashr <4 x i32> [[TMP1]], splat (i32 5)
+; CHECK-NEXT:    [[TMP3:%.*]] = or <4 x i32> [[TMP2]], zeroinitializer
+; CHECK-NEXT:    [[S:%.*]] = ashr <4 x i32> [[X]], splat (i32 5)
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <4 x i32> [[TMP3]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP4]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP5:%.*]], label [[TMP6:%.*]], !prof [[PROF1:![0-9]+]]
+; CHECK:       5:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4:[0-9]+]]
+; CHECK-NEXT:    unreachable
+; CHECK:       6:
+; CHECK-NEXT:    [[R:%.*]] = tail call <4 x i16> @llvm.aarch64.neon.sqxtn.v4i16(<4 x i32> [[S]])
+; CHECK-NEXT:    store <4 x i16> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i16> [[R]]
+;
+  %s = ashr <4 x i32> %x, <i32 5, i32 5, i32 5, i32 5>
+  %r = tail call <4 x i16> @llvm.aarch64.neon.sqxtn.v4i16(<4 x i32> %s)
+  ret <4 x i16> %r
+}
+
+define <4 x i16> @NarrowAShrU32By5(<4 x i32> %x) #0 {
+; CHECK-LABEL: define <4 x i16> @NarrowAShrU32By5(
+; CHECK-SAME: <4 x i32> [[X:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = ashr <4 x i32> [[TMP1]], splat (i32 5)
+; CHECK-NEXT:    [[TMP3:%.*]] = or <4 x i32> [[TMP2]], zeroinitializer
+; CHECK-NEXT:    [[S:%.*]] = ashr <4 x i32> [[X]], splat (i32 5)
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <4 x i32> [[TMP3]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP4]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP5:%.*]], label [[TMP6:%.*]], !prof [[PROF1]]
+; CHECK:       5:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       6:
+; CHECK-NEXT:    [[R:%.*]] = tail call <4 x i16> @llvm.aarch64.neon.uqxtn.v4i16(<4 x i32> [[S]])
+; CHECK-NEXT:    store <4 x i16> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i16> [[R]]
+;
+  %s = ashr <4 x i32> %x, <i32 5, i32 5, i32 5, i32 5>
+  %r = tail call <4 x i16> @llvm.aarch64.neon.uqxtn.v4i16(<4 x i32> %s)
+  ret <4 x i16> %r
+}
+
+define <4 x i16> @NarrowAShrI32By5ToU16(<4 x i32> %x) #0 {
+; CHECK-LABEL: define <4 x i16> @NarrowAShrI32By5ToU16(
+; CHECK-SAME: <4 x i32> [[X:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = ashr <4 x i32> [[TMP1]], splat (i32 5)
+; CHECK-NEXT:    [[TMP3:%.*]] = or <4 x i32> [[TMP2]], zeroinitializer
+; CHECK-NEXT:    [[S:%.*]] = ashr <4 x i32> [[X]], splat (i32 5)
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <4 x i32> [[TMP3]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP4]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP5:%.*]], label [[TMP6:%.*]], !prof [[PROF1]]
+; CHECK:       5:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       6:
+; CHECK-NEXT:    [[R:%.*]] = tail call <4 x i16> @llvm.aarch64.neon.sqxtun.v4i16(<4 x i32> [[S]])
+; CHECK-NEXT:    store <4 x i16> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i16> [[R]]
+;
+  %s = ashr <4 x i32> %x, <i32 5, i32 5, i32 5, i32 5>
+  %r = tail call <4 x i16> @llvm.aarch64.neon.sqxtun.v4i16(<4 x i32> %s)
+  ret <4 x i16> %r
+}
+
+define <4 x i16> @NarrowLShrI32By5(<4 x i32> %x) #0 {
+; CHECK-LABEL: define <4 x i16> @NarrowLShrI32By5(
+; CHECK-SAME: <4 x i32> [[X:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = lshr <4 x i32> [[TMP1]], splat (i32 5)
+; CHECK-NEXT:    [[TMP3:%.*]] = or <4 x i32> [[TMP2]], zeroinitializer
+; CHECK-NEXT:    [[S:%.*]] = lshr <4 x i32> [[X]], splat (i32 5)
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <4 x i32> [[TMP3]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP4]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP5:%.*]], label [[TMP6:%.*]], !prof [[PROF1]]
+; CHECK:       5:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       6:
+; CHECK-NEXT:    [[R:%.*]] = tail call <4 x i16> @llvm.aarch64.neon.sqxtn.v4i16(<4 x i32> [[S]])
+; CHECK-NEXT:    store <4 x i16> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i16> [[R]]
+;
+  %s = lshr <4 x i32> %x, <i32 5, i32 5, i32 5, i32 5>
+  %r = tail call <4 x i16> @llvm.aarch64.neon.sqxtn.v4i16(<4 x i32> %s)
+  ret <4 x i16> %r
+}
+
+define <4 x i16> @NarrowLShrU32By5(<4 x i32> %x) #0 {
+; CHECK-LABEL: define <4 x i16> @NarrowLShrU32By5(
+; CHECK-SAME: <4 x i32> [[X:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = lshr <4 x i32> [[TMP1]], splat (i32 5)
+; CHECK-NEXT:    [[TMP3:%.*]] = or <4 x i32> [[TMP2]], zeroinitializer
+; CHECK-NEXT:    [[S:%.*]] = lshr <4 x i32> [[X]], splat (i32 5)
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <4 x i32> [[TMP3]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP4]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP5:%.*]], label [[TMP6:%.*]], !prof [[PROF1]]
+; CHECK:       5:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       6:
+; CHECK-NEXT:    [[R:%.*]] = tail call <4 x i16> @llvm.aarch64.neon.uqxtn.v4i16(<4 x i32> [[S]])
+; CHECK-NEXT:    store <4 x i16> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i16> [[R]]
+;
+  %s = lshr <4 x i32> %x, <i32 5, i32 5, i32 5, i32 5>
+  %r = tail call <4 x i16> @llvm.aarch64.neon.uqxtn.v4i16(<4 x i32> %s)
+  ret <4 x i16> %r
+}
+
+define <4 x i16> @NarrowLShrI32By5ToU16(<4 x i32> %x) #0 {
+; CHECK-LABEL: define <4 x i16> @NarrowLShrI32By5ToU16(
+; CHECK-SAME: <4 x i32> [[X:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = lshr <4 x i32> [[TMP1]], splat (i32 5)
+; CHECK-NEXT:    [[TMP3:%.*]] = or <4 x i32> [[TMP2]], zeroinitializer
+; CHECK-NEXT:    [[S:%.*]] = lshr <4 x i32> [[X]], splat (i32 5)
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <4 x i32> [[TMP3]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP4]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP5:%.*]], label [[TMP6:%.*]], !prof [[PROF1]]
+; CHECK:       5:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       6:
+; CHECK-NEXT:    [[R:%.*]] = tail call <4 x i16> @llvm.aarch64.neon.sqxtun.v4i16(<4 x i32> [[S]])
+; CHECK-NEXT:    store <4 x i16> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i16> [[R]]
+;
+  %s = lshr <4 x i32> %x, <i32 5, i32 5, i32 5, i32 5>
+  %r = tail call <4 x i16> @llvm.aarch64.neon.sqxtun.v4i16(<4 x i32> %s)
+  ret <4 x i16> %r
+}
+
+
+define <2 x i32> @NarrowAShri64By5(<2 x i64> %x) #0 {
+; CHECK-LABEL: define <2 x i32> @NarrowAShri64By5(
+; CHECK-SAME: <2 x i64> [[X:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = ashr <2 x i64> [[TMP1]], splat (i64 5)
+; CHECK-NEXT:    [[TMP3:%.*]] = or <2 x i64> [[TMP2]], zeroinitializer
+; CHECK-NEXT:    [[S:%.*]] = ashr <2 x i64> [[X]], splat (i64 5)
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <2 x i64> [[TMP3]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP4]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP5:%.*]], label [[TMP6:%.*]], !prof [[PROF1]]
+; CHECK:       5:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       6:
+; CHECK-NEXT:    [[R:%.*]] = tail call <2 x i32> @llvm.aarch64.neon.sqxtn.v2i32(<2 x i64> [[S]])
+; CHECK-NEXT:    store <2 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x i32> [[R]]
+;
+  %s = ashr <2 x i64> %x, <i64 5, i64 5>
+  %r = tail call <2 x i32> @llvm.aarch64.neon.sqxtn.v2i32(<2 x i64> %s)
+  ret <2 x i32> %r
+}
+
+define <2 x i32> @NarrowAShrU64By5(<2 x i64> %x) #0 {
+; CHECK-LABEL: define <2 x i32> @NarrowAShrU64By5(
+; CHECK-SAME: <2 x i64> [[X:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = ashr <2 x i64> [[TMP1]], splat (i64 5)
+; CHECK-NEXT:    [[TMP3:%.*]] = or <2 x i64> [[TMP2]], zeroinitializer
+; CHECK-NEXT:    [[S:%.*]] = ashr <2 x i64> [[X]], splat (i64 5)
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <2 x i64> [[TMP3]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP4]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP5:%.*]], label [[TMP6:%.*]], !prof [[PROF1]]
+; CHECK:       5:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       6:
+; CHECK-NEXT:    [[R:%.*]] = tail call <2 x i32> @llvm.aarch64.neon.uqxtn.v2i32(<2 x i64> [[S]])
+; CHECK-NEXT:    store <2 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x i32> [[R]]
+;
+  %s = ashr <2 x i64> %x, <i64 5, i64 5>
+  %r = tail call <2 x i32> @llvm.aarch64.neon.uqxtn.v2i32(<2 x i64> %s)
+  ret <2 x i32> %r
+}
+
+define <2 x i32> @NarrowAShri64By5ToU32(<2 x i64> %x) #0 {
+; CHECK-LABEL: define <2 x i32> @NarrowAShri64By5ToU32(
+; CHECK-SAME: <2 x i64> [[X:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = ashr <2 x i64> [[TMP1]], splat (i64 5)
+; CHECK-NEXT:    [[TMP3:%.*]] = or <2 x i64> [[TMP2]], zeroinitializer
+; CHECK-NEXT:    [[S:%.*]] = ashr <2 x i64> [[X]], splat (i64 5)
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <2 x i64> [[TMP3]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP4]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP5:%.*]], label [[TMP6:%.*]], !prof [[PROF1]]
+; CHECK:       5:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       6:
+; CHECK-NEXT:    [[R:%.*]] = tail call <2 x i32> @llvm.aarch64.neon.sqxtun.v2i32(<2 x i64> [[S]])
+; CHECK-NEXT:    store <2 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x i32> [[R]]
+;
+  %s = ashr <2 x i64> %x, <i64 5, i64 5>
+  %r = tail call <2 x i32> @llvm.aarch64.neon.sqxtun.v2i32(<2 x i64> %s)
+  ret <2 x i32> %r
+}
+
+define <2 x i32> @NarrowLShri64By5(<2 x i64> %x) #0 {
+; CHECK-LABEL: define <2 x i32> @NarrowLShri64By5(
+; CHECK-SAME: <2 x i64> [[X:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = lshr <2 x i64> [[TMP1]], splat (i64 5)
+; CHECK-NEXT:    [[TMP3:%.*]] = or <2 x i64> [[TMP2]], zeroinitializer
+; CHECK-NEXT:    [[S:%.*]] = lshr <2 x i64> [[X]], splat (i64 5)
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <2 x i64> [[TMP3]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP4]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP5:%.*]], label [[TMP6:%.*]], !prof [[PROF1]]
+; CHECK:       5:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       6:
+; CHECK-NEXT:    [[R:%.*]] = tail call <2 x i32> @llvm.aarch64.neon.sqxtn.v2i32(<2 x i64> [[S]])
+; CHECK-NEXT:    store <2 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x i32> [[R]]
+;
+  %s = lshr <2 x i64> %x, <i64 5, i64 5>
+  %r = tail call <2 x i32> @llvm.aarch64.neon.sqxtn.v2i32(<2 x i64> %s)
+  ret <2 x i32> %r
+}
+
+define <2 x i32> @NarrowLShrU64By5(<2 x i64> %x) #0 {
+; CHECK-LABEL: define <2 x i32> @NarrowLShrU64By5(
+; CHECK-SAME: <2 x i64> [[X:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = lshr <2 x i64> [[TMP1]], splat (i64 5)
+; CHECK-NEXT:    [[TMP3:%.*]] = or <2 x i64> [[TMP2]], zeroinitializer
+; CHECK-NEXT:    [[S:%.*]] = lshr <2 x i64> [[X]], splat (i64 5)
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <2 x i64> [[TMP3]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP4]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP5:%.*]], label [[TMP6:%.*]], !prof [[PROF1]]
+; CHECK:       5:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       6:
+; CHECK-NEXT:    [[R:%.*]] = tail call <2 x i32> @llvm.aarch64.neon.uqxtn.v2i32(<2 x i64> [[S]])
+; CHECK-NEXT:    store <2 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x i32> [[R]]
+;
+  %s = lshr <2 x i64> %x, <i64 5, i64 5>
+  %r = tail call <2 x i32> @llvm.aarch64.neon.uqxtn.v2i32(<2 x i64> %s)
+  ret <2 x i32> %r
+}
+
+define <2 x i32> @NarrowLShri64By5ToU32(<2 x i64> %x) #0 {
+; CHECK-LABEL: define <2 x i32> @NarrowLShri64By5ToU32(
+; CHECK-SAME: <2 x i64> [[X:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = lshr <2 x i64> [[TMP1]], splat (i64 5)
+; CHECK-NEXT:    [[TMP3:%.*]] = or <2 x i64> [[TMP2]], zeroinitializer
+; CHECK-NEXT:    [[S:%.*]] = lshr <2 x i64> [[X]], splat (i64 5)
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <2 x i64> [[TMP3]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP4]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP5:%.*]], label [[TMP6:%.*]], !prof [[PROF1]]
+; CHECK:       5:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       6:
+; CHECK-NEXT:    [[R:%.*]] = tail call <2 x i32> @llvm.aarch64.neon.sqxtun.v2i32(<2 x i64> [[S]])
+; CHECK-NEXT:    store <2 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x i32> [[R]]
+;
+  %s = lshr <2 x i64> %x, <i64 5, i64 5>
+  %r = tail call <2 x i32> @llvm.aarch64.neon.sqxtun.v2i32(<2 x i64> %s)
+  ret <2 x i32> %r
+}
+
+
+define <8 x i8> @NarrowAShri16By5(<8 x i16> %x) #0 {
+; CHECK-LABEL: define <8 x i8> @NarrowAShri16By5(
+; CHECK-SAME: <8 x i16> [[X:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = ashr <8 x i16> [[TMP1]], splat (i16 5)
+; CHECK-NEXT:    [[TMP3:%.*]] = or <8 x i16> [[TMP2]], zeroinitializer
+; CHECK-NEXT:    [[S:%.*]] = ashr <8 x i16> [[X]], splat (i16 5)
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <8 x i16> [[TMP3]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP4]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP5:%.*]], label [[TMP6:%.*]], !prof [[PROF1]]
+; CHECK:       5:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       6:
+; CHECK-NEXT:    [[R:%.*]] = tail call <8 x i8> @llvm.aarch64.neon.sqxtn.v8i8(<8 x i16> [[S]])
+; CHECK-NEXT:    store <8 x i8> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i8> [[R]]
+;
+  %s = ashr <8 x i16> %x, <i16 5, i16 5, i16 5, i16 5, i16 5, i16 5, i16 5, i16 5>
+  %r = tail call <8 x i8> @llvm.aarch64.neon.sqxtn.v8i8(<8 x i16> %s)
+  ret <8 x i8> %r
+}
+
+define <8 x i8> @NarrowAShrU16By5(<8 x i16> %x) #0 {
+; CHECK-LABEL: define <8 x i8> @NarrowAShrU16By5(
+; CHECK-SAME: <8 x i16> [[X:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = ashr <8 x i16> [[TMP1]], splat (i16 5)
+; CHECK-NEXT:    [[TMP3:%.*]] = or <8 x i16> [[TMP2]], zeroinitializer
+; CHECK-NEXT:    [[S:%.*]] = ashr <8 x i16> [[X]], splat (i16 5)
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <8 x i16> [[TMP3]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP4]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP5:%.*]], label [[TMP6:%.*]], !prof [[PROF1]]
+; CHECK:       5:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       6:
+; CHECK-NEXT:    [[R:%.*]] = tail call <8 x i8> @llvm.aarch64.neon.uqxtn.v8i8(<8 x i16> [[S]])
+; CHECK-NEXT:    store <8 x i8> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i8> [[R]]
+;
+  %s = ashr <8 x i16> %x, <i16 5, i16 5, i16 5, i16 5, i16 5, i16 5, i16 5, i16 5>
+  %r = tail call <8 x i8> @llvm.aarch64.neon.uqxtn.v8i8(<8 x i16> %s)
+  ret <8 x i8> %r
+}
+
+define <8 x i8> @NarrowAShri16By5ToU8(<8 x i16> %x) #0 {
+; CHECK-LABEL: define <8 x i8> @NarrowAShri16By5ToU8(
+; CHECK-SAME: <8 x i16> [[X:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = ashr <8 x i16> [[TMP1]], splat (i16 5)
+; CHECK-NEXT:    [[TMP3:%.*]] = or <8 x i16> [[TMP2]], zeroinitializer
+; CHECK-NEXT:    [[S:%.*]] = ashr <8 x i16> [[X]], splat (i16 5)
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <8 x i16> [[TMP3]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP4]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP5:%.*]], label [[TMP6:%.*]], !prof [[PROF1]]
+; CHECK:       5:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       6:
+; CHECK-NEXT:    [[R:%.*]] = tail call <8 x i8> @llvm.aarch64.neon.sqxtun.v8i8(<8 x i16> [[S]])
+; CHECK-NEXT:    store <8 x i8> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i8> [[R]]
+;
+  %s = ashr <8 x i16> %x, <i16 5, i16 5, i16 5, i16 5, i16 5, i16 5, i16 5, i16 5>
+  %r = tail call <8 x i8> @llvm.aarch64.neon.sqxtun.v8i8(<8 x i16> %s)
+  ret <8 x i8> %r
+}
+
+define <8 x i8> @NarrowLShri16By5(<8 x i16> %x) #0 {
+; CHECK-LABEL: define <8 x i8> @NarrowLShri16By5(
+; CHECK-SAME: <8 x i16> [[X:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = lshr <8 x i16> [[TMP1]], splat (i16 5)
+; CHECK-NEXT:    [[TMP3:%.*]] = or <8 x i16> [[TMP2]], zeroinitializer
+; CHECK-NEXT:    [[S:%.*]] = lshr <8 x i16> [[X]], splat (i16 5)
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <8 x i16> [[TMP3]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP4]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP5:%.*]], label [[TMP6:%.*]], !prof [[PROF1]]
+; CHECK:       5:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       6:
+; CHECK-NEXT:    [[R:%.*]] = tail call <8 x i8> @llvm.aarch64.neon.sqxtn.v8i8(<8 x i16> [[S]])
+; CHECK-NEXT:    store <8 x i8> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i8> [[R]]
+;
+  %s = lshr <8 x i16> %x, <i16 5, i16 5, i16 5, i16 5, i16 5, i16 5, i16 5, i16 5>
+  %r = tail call <8 x i8> @llvm.aarch64.neon.sqxtn.v8i8(<8 x i16> %s)
+  ret <8 x i8> %r
+}
+
+define <8 x i8> @NarrowLShrU16By5(<8 x i16> %x) #0 {
+; CHECK-LABEL: define <8 x i8> @NarrowLShrU16By5(
+; CHECK-SAME: <8 x i16> [[X:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = lshr <8 x i16> [[TMP1]], splat (i16 5)
+; CHECK-NEXT:    [[TMP3:%.*]] = or <8 x i16> [[TMP2]], zeroinitializer
+; CHECK-NEXT:    [[S:%.*]] = lshr <8 x i16> [[X]], splat (i16 5)
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <8 x i16> [[TMP3]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP4]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP5:%.*]], label [[TMP6:%.*]], !prof [[PROF1]]
+; CHECK:       5:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       6:
+; CHECK-NEXT:    [[R:%.*]] = tail call <8 x i8> @llvm.aarch64.neon.uqxtn.v8i8(<8 x i16> [[S]])
+; CHECK-NEXT:    store <8 x i8> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i8> [[R]]
+;
+  %s = lshr <8 x i16> %x, <i16 5, i16 5, i16 5, i16 5, i16 5, i16 5, i16 5, i16 5>
+  %r = tail call <8 x i8> @llvm.aarch64.neon.uqxtn.v8i8(<8 x i16> %s)
+  ret <8 x i8> %r
+}
+
+define <8 x i8> @NarrowLShri16By5ToU8(<8 x i16> %x) #0 {
+; CHECK-LABEL: define <8 x i8> @NarrowLShri16By5ToU8(
+; CHECK-SAME: <8 x i16> [[X:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = lshr <8 x i16> [[TMP1]], splat (i16 5)
+; CHECK-NEXT:    [[TMP3:%.*]] = or <8 x i16> [[TMP2]], zeroinitializer
+; CHECK-NEXT:    [[S:%.*]] = lshr <8 x i16> [[X]], splat (i16 5)
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <8 x i16> [[TMP3]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP4]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP5:%.*]], label [[TMP6:%.*]], !prof [[PROF1]]
+; CHECK:       5:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       6:
+; CHECK-NEXT:    [[R:%.*]] = tail call <8 x i8> @llvm.aarch64.neon.sqxtun.v8i8(<8 x i16> [[S]])
+; CHECK-NEXT:    store <8 x i8> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i8> [[R]]
+;
+  %s = lshr <8 x i16> %x, <i16 5, i16 5, i16 5, i16 5, i16 5, i16 5, i16 5, i16 5>
+  %r = tail call <8 x i8> @llvm.aarch64.neon.sqxtun.v8i8(<8 x i16> %s)
+  ret <8 x i8> %r
+}
+
+
+
+
+
+define <4 x i16> @NarrowAShrI32By31(<4 x i32> %x) #0 {
+; CHECK-LABEL: define <4 x i16> @NarrowAShrI32By31(
+; CHECK-SAME: <4 x i32> [[X:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = ashr <4 x i32> [[TMP1]], splat (i32 16)
+; CHECK-NEXT:    [[TMP3:%.*]] = or <4 x i32> [[TMP2]], zeroinitializer
+; CHECK-NEXT:    [[S:%.*]] = ashr <4 x i32> [[X]], splat (i32 16)
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <4 x i32> [[TMP3]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP4]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP5:%.*]], label [[TMP6:%.*]], !prof [[PROF1]]
+; CHECK:       5:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       6:
+; CHECK-NEXT:    [[R:%.*]] = tail call <4 x i16> @llvm.aarch64.neon.sqxtn.v4i16(<4 x i32> [[S]])
+; CHECK-NEXT:    store <4 x i16> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i16> [[R]]
+;
+  %s = ashr <4 x i32> %x, <i32 16, i32 16, i32 16, i32 16>
+  %r = tail call <4 x i16> @llvm.aarch64.neon.sqxtn.v4i16(<4 x i32> %s)
+  ret <4 x i16> %r
+}
+
+define <4 x i16> @NarrowAShrI32By31ToU16(<4 x i32> %x) #0 {
+; CHECK-LABEL: define <4 x i16> @NarrowAShrI32By31ToU16(
+; CHECK-SAME: <4 x i32> [[X:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = ashr <4 x i32> [[TMP1]], splat (i32 16)
+; CHECK-NEXT:    [[TMP3:%.*]] = or <4 x i32> [[TMP2]], zeroinitializer
+; CHECK-NEXT:    [[S:%.*]] = ashr <4 x i32> [[X]], splat (i32 16)
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <4 x i32> [[TMP3]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP4]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP5:%.*]], label [[TMP6:%.*]], !prof [[PROF1]]
+; CHECK:       5:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       6:
+; CHECK-NEXT:    [[R:%.*]] = tail call <4 x i16> @llvm.aarch64.neon.sqxtun.v4i16(<4 x i32> [[S]])
+; CHECK-NEXT:    store <4 x i16> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i16> [[R]]
+;
+  %s = ashr <4 x i32> %x, <i32 16, i32 16, i32 16, i32 16>
+  %r = tail call <4 x i16> @llvm.aarch64.neon.sqxtun.v4i16(<4 x i32> %s)
+  ret <4 x i16> %r
+}
+
+define <4 x i16> @NarrowLShrU32By31(<4 x i32> %x) #0 {
+; CHECK-LABEL: define <4 x i16> @NarrowLShrU32By31(
+; CHECK-SAME: <4 x i32> [[X:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = lshr <4 x i32> [[TMP1]], splat (i32 16)
+; CHECK-NEXT:    [[TMP3:%.*]] = or <4 x i32> [[TMP2]], zeroinitializer
+; CHECK-NEXT:    [[S:%.*]] = lshr <4 x i32> [[X]], splat (i32 16)
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <4 x i32> [[TMP3]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP4]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP5:%.*]], label [[TMP6:%.*]], !prof [[PROF1]]
+; CHECK:       5:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       6:
+; CHECK-NEXT:    [[R:%.*]] = tail call <4 x i16> @llvm.aarch64.neon.uqxtn.v4i16(<4 x i32> [[S]])
+; CHECK-NEXT:    store <4 x i16> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i16> [[R]]
+;
+  %s = lshr <4 x i32> %x, <i32 16, i32 16, i32 16, i32 16>
+  %r = tail call <4 x i16> @llvm.aarch64.neon.uqxtn.v4i16(<4 x i32> %s)
+  ret <4 x i16> %r
+}
+
+
+define <16 x i8> @signed_minmax_v8i16_to_v16i8(<16 x i16> %x) #0 {
+; CHECK-LABEL: define <16 x i8> @signed_minmax_v8i16_to_v16i8(
+; CHECK-SAME: <16 x i16> [[X:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load <16 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP1:%.*]] = ashr <16 x i16> [[TMP0]], splat (i16 5)
+; CHECK-NEXT:    [[TMP2:%.*]] = or <16 x i16> [[TMP1]], zeroinitializer
+; CHECK-NEXT:    [[S:%.*]] = ashr <16 x i16> [[X]], splat (i16 5)
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <16 x i16> [[TMP2]], zeroinitializer
+; CHECK-NEXT:    [[MIN:%.*]] = call <16 x i16> @llvm.smin.v16i16(<16 x i16> [[S]], <16 x i16> splat (i16 127))
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <16 x i16> [[_MSPROP]], zeroinitializer
+; CHECK-NEXT:    [[MAX:%.*]] = call <16 x i16> @llvm.smax.v16i16(<16 x i16> [[MIN]], <16 x i16> splat (i16 -128))
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = trunc <16 x i16> [[_MSPROP1]] to <16 x i8>
+; CHECK-NEXT:    [[TRUNC:%.*]] = trunc <16 x i16> [[MAX]] to <16 x i8>
+; CHECK-NEXT:    store <16 x i8> [[_MSPROP2]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i8> [[TRUNC]]
+;
+entry:
+  %s = ashr <16 x i16> %x, <i16 5, i16 5, i16 5, i16 5, i16 5, i16 5, i16 5, i16 5, i16 5, i16 5, i16 5, i16 5, i16 5, i16 5, i16 5, i16 5>
+  %min = call <16 x i16> @llvm.smin.v8i16(<16 x i16> %s, <16 x i16> <i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127>)
+  %max = call <16 x i16> @llvm.smax.v8i16(<16 x i16> %min, <16 x i16> <i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128>)
+  %trunc = trunc <16 x i16> %max to <16 x i8>
+  ret <16 x i8> %trunc
+}
+
+define <16 x i8> @unsigned_minmax_v8i16_to_v16i8(<16 x i16> %x) #0 {
+; CHECK-LABEL: define <16 x i8> @unsigned_minmax_v8i16_to_v16i8(
+; CHECK-SAME: <16 x i16> [[X:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load <16 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP1:%.*]] = lshr <16 x i16> [[TMP0]], splat (i16 5)
+; CHECK-NEXT:    [[TMP2:%.*]] = or <16 x i16> [[TMP1]], zeroinitializer
+; CHECK-NEXT:    [[S:%.*]] = lshr <16 x i16> [[X]], splat (i16 5)
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <16 x i16> [[TMP2]], zeroinitializer
+; CHECK-NEXT:    [[MIN:%.*]] = call <16 x i16> @llvm.umin.v16i16(<16 x i16> [[S]], <16 x i16> splat (i16 255))
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = trunc <16 x i16> [[_MSPROP]] to <16 x i8>
+; CHECK-NEXT:    [[TRUNC:%.*]] = trunc <16 x i16> [[MIN]] to <16 x i8>
+; CHECK-NEXT:    store <16 x i8> [[_MSPROP1]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i8> [[TRUNC]]
+;
+entry:
+  %s = lshr <16 x i16> %x, <i16 5, i16 5, i16 5, i16 5, i16 5, i16 5, i16 5, i16 5, i16 5, i16 5, i16 5, i16 5, i16 5, i16 5, i16 5, i16 5>
+  %min = call <16 x i16> @llvm.umin.v8i16(<16 x i16> %s, <16 x i16> <i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255>)
+  %trunc = trunc <16 x i16> %min to <16 x i8>
+  ret <16 x i8> %trunc
+}
+
+define <16 x i8> @unsigned_signed_minmax_v8i16_to_v16i8(<16 x i16> %x) #0 {
+; CHECK-LABEL: define <16 x i8> @unsigned_signed_minmax_v8i16_to_v16i8(
+; CHECK-SAME: <16 x i16> [[X:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load <16 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP1:%.*]] = ashr <16 x i16> [[TMP0]], splat (i16 5)
+; CHECK-NEXT:    [[TMP2:%.*]] = or <16 x i16> [[TMP1]], zeroinitializer
+; CHECK-NEXT:    [[S:%.*]] = ashr <16 x i16> [[X]], splat (i16 5)
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <16 x i16> [[TMP2]], zeroinitializer
+; CHECK-NEXT:    [[MAX:%.*]] = call <16 x i16> @llvm.smax.v16i16(<16 x i16> [[S]], <16 x i16> zeroinitializer)
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <16 x i16> [[_MSPROP]], zeroinitializer
+; CHECK-NEXT:    [[MIN:%.*]] = call <16 x i16> @llvm.umin.v16i16(<16 x i16> [[MAX]], <16 x i16> splat (i16 255))
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = trunc <16 x i16> [[_MSPROP1]] to <16 x i8>
+; CHECK-NEXT:    [[TRUNC:%.*]] = trunc <16 x i16> [[MIN]] to <16 x i8>
+; CHECK-NEXT:    store <16 x i8> [[_MSPROP2]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i8> [[TRUNC]]
+;
+entry:
+  %s = ashr <16 x i16> %x, <i16 5, i16 5, i16 5, i16 5, i16 5, i16 5, i16 5, i16 5, i16 5, i16 5, i16 5, i16 5, i16 5, i16 5, i16 5, i16 5>
+  %max = call <16 x i16> @llvm.smax.v8i16(<16 x i16> %s, <16 x i16> <i16 0, i16 0, i16 0, i16 0, i16 0, i16 0, i16 0, i16 0, i16 0, i16 0, i16 0, i16 0, i16 0, i16 0, i16 0, i16 0>)
+  %min = call <16 x i16> @llvm.umin.v8i16(<16 x i16> %max, <16 x i16> <i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255>)
+  %trunc = trunc <16 x i16> %min to <16 x i8>
+  ret <16 x i8> %trunc
+}
+
+
+define <8 x i16> @signed_minmax_v4i32_to_v8i16(<8 x i32> %x) #0 {
+; CHECK-LABEL: define <8 x i16> @signed_minmax_v4i32_to_v8i16(
+; CHECK-SAME: <8 x i32> [[X:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load <8 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP1:%.*]] = ashr <8 x i32> [[TMP0]], splat (i32 5)
+; CHECK-NEXT:    [[TMP2:%.*]] = or <8 x i32> [[TMP1]], zeroinitializer
+; CHECK-NEXT:    [[S:%.*]] = ashr <8 x i32> [[X]], splat (i32 5)
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <8 x i32> [[TMP2]], zeroinitializer
+; CHECK-NEXT:    [[MIN:%.*]] = call <8 x i32> @llvm.smin.v8i32(<8 x i32> [[S]], <8 x i32> splat (i32 32767))
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <8 x i32> [[_MSPROP]], zeroinitializer
+; CHECK-NEXT:    [[MAX:%.*]] = call <8 x i32> @llvm.smax.v8i32(<8 x i32> [[MIN]], <8 x i32> splat (i32 -32768))
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = trunc <8 x i32> [[_MSPROP1]] to <8 x i16>
+; CHECK-NEXT:    [[TRUNC:%.*]] = trunc <8 x i32> [[MAX]] to <8 x i16>
+; CHECK-NEXT:    store <8 x i16> [[_MSPROP2]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i16> [[TRUNC]]
+;
+entry:
+  %s = ashr <8 x i32> %x, <i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5>
+  %min = call <8 x i32> @llvm.smin.v8i32(<8 x i32> %s, <8 x i32> <i32 32767, i32 32767, i32 32767, i32 32767, i32 32767, i32 32767, i32 32767, i32 32767>)
+  %max = call <8 x i32> @llvm.smax.v8i32(<8 x i32> %min, <8 x i32> <i32 -32768, i32 -32768, i32 -32768, i32 -32768, i32 -32768, i32 -32768, i32 -32768, i32 -32768>)
+  %trunc = trunc <8 x i32> %max to <8 x i16>
+  ret <8 x i16> %trunc
+}
+
+define <8 x i16> @unsigned_minmax_v4i32_to_v8i16(<8 x i32> %x) #0 {
+; CHECK-LABEL: define <8 x i16> @unsigned_minmax_v4i32_to_v8i16(
+; CHECK-SAME: <8 x i32> [[X:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load <8 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP1:%.*]] = lshr <8 x i32> [[TMP0]], splat (i32 5)
+; CHECK-NEXT:    [[TMP2:%.*]] = or <8 x i32> [[TMP1]], zeroinitializer
+; CHECK-NEXT:    [[S:%.*]] = lshr <8 x i32> [[X]], splat (i32 5)
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <8 x i32> [[TMP2]], zeroinitializer
+; CHECK-NEXT:    [[MIN:%.*]] = call <8 x i32> @llvm.umin.v8i32(<8 x i32> [[S]], <8 x i32> splat (i32 65535))
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = trunc <8 x i32> [[_MSPROP]] to <8 x i16>
+; CHECK-NEXT:    [[TRUNC:%.*]] = trunc <8 x i32> [[MIN]] to <8 x i16>
+; CHECK-NEXT:    store <8 x i16> [[_MSPROP1]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i16> [[TRUNC]]
+;
+entry:
+  %s = lshr <8 x i32> %x, <i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5>
+  %min = call <8 x i32> @llvm.umin.v8i32(<8 x i32> %s, <8 x i32> <i32 65535, i32 65535, i32 65535, i32 65535, i32 65535, i32 65535, i32 65535, i32 65535>)
+  %trunc = trunc <8 x i32> %min to <8 x i16>
+  ret <8 x i16> %trunc
+}
+
+define <8 x i16> @unsigned_signed_minmax_v4i32_to_v8i16(<8 x i32> %x) #0 {
+; CHECK-LABEL: define <8 x i16> @unsigned_signed_minmax_v4i32_to_v8i16(
+; CHECK-SAME: <8 x i32> [[X:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load <8 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP1:%.*]] = ashr <8 x i32> [[TMP0]], splat (i32 5)
+; CHECK-NEXT:    [[TMP2:%.*]] = or <8 x i32> [[TMP1]], zeroinitializer
+; CHECK-NEXT:    [[S:%.*]] = ashr <8 x i32> [[X]], splat (i32 5)
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <8 x i32> [[TMP2]], zeroinitializer
+; CHECK-NEXT:    [[MAX:%.*]] = call <8 x i32> @llvm.smax.v8i32(<8 x i32> [[S]], <8 x i32> zeroinitializer)
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <8 x i32> [[_MSPROP]], zeroinitializer
+; CHECK-NEXT:    [[MIN:%.*]] = call <8 x i32> @llvm.umin.v8i32(<8 x i32> [[MAX]], <8 x i32> splat (i32 65535))
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = trunc <8 x i32> [[_MSPROP1]] to <8 x i16>
+; CHECK-NEXT:    [[TRUNC:%.*]] = trunc <8 x i32> [[MIN]] to <8 x i16>
+; CHECK-NEXT:    store <8 x i16> [[_MSPROP2]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i16> [[TRUNC]]
+;
+entry:
+  %s = ashr <8 x i32> %x, <i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5>
+  %max = call <8 x i32> @llvm.smax.v8i32(<8 x i32> %s, <8 x i32> <i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0>)
+  %min = call <8 x i32> @llvm.umin.v8i32(<8 x i32> %max, <8 x i32> <i32 65535, i32 65535, i32 65535, i32 65535, i32 65535, i32 65535, i32 65535, i32 65535>)
+  %trunc = trunc <8 x i32> %min to <8 x i16>
+  ret <8 x i16> %trunc
+}
+
+
+define <4 x i32> @signed_minmax_v4i64_to_v8i32(<4 x i64> %x) #0 {
+; CHECK-LABEL: define <4 x i32> @signed_minmax_v4i64_to_v8i32(
+; CHECK-SAME: <4 x i64> [[X:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load <4 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP1:%.*]] = ashr <4 x i64> [[TMP0]], splat (i64 5)
+; CHECK-NEXT:    [[TMP2:%.*]] = or <4 x i64> [[TMP1]], zeroinitializer
+; CHECK-NEXT:    [[S:%.*]] = ashr <4 x i64> [[X]], splat (i64 5)
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <4 x i64> [[TMP2]], zeroinitializer
+; CHECK-NEXT:    [[MIN:%.*]] = call <4 x i64> @llvm.smin.v4i64(<4 x i64> [[S]], <4 x i64> splat (i64 2147483647))
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <4 x i64> [[_MSPROP]], zeroinitializer
+; CHECK-NEXT:    [[MAX:%.*]] = call <4 x i64> @llvm.smax.v4i64(<4 x i64> [[MIN]], <4 x i64> splat (i64 -2147483648))
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = trunc <4 x i64> [[_MSPROP1]] to <4 x i32>
+; CHECK-NEXT:    [[TRUNC:%.*]] = trunc <4 x i64> [[MAX]] to <4 x i32>
+; CHECK-NEXT:    store <4 x i32> [[_MSPROP2]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i32> [[TRUNC]]
+;
+entry:
+  %s = ashr <4 x i64> %x, <i64 5, i64 5, i64 5, i64 5>
+  %min = call <4 x i64> @llvm.smin.v8i64(<4 x i64> %s, <4 x i64> <i64 2147483647, i64 2147483647, i64 2147483647, i64 2147483647>)
+  %max = call <4 x i64> @llvm.smax.v8i64(<4 x i64> %min, <4 x i64> <i64 -2147483648, i64 -2147483648, i64 -2147483648, i64 -2147483648>)
+  %trunc = trunc <4 x i64> %max to <4 x i32>
+  ret <4 x i32> %trunc
+}
+
+define <4 x i32> @unsigned_minmax_v4i64_to_v8i32(<4 x i64> %x) #0 {
+; CHECK-LABEL: define <4 x i32> @unsigned_minmax_v4i64_to_v8i32(
+; CHECK-SAME: <4 x i64> [[X:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load <4 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP1:%.*]] = lshr <4 x i64> [[TMP0]], splat (i64 5)
+; CHECK-NEXT:    [[TMP2:%.*]] = or <4 x i64> [[TMP1]], zeroinitializer
+; CHECK-NEXT:    [[S:%.*]] = lshr <4 x i64> [[X]], splat (i64 5)
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <4 x i64> [[TMP2]], zeroinitializer
+; CHECK-NEXT:    [[MIN:%.*]] = call <4 x i64> @llvm.umin.v4i64(<4 x i64> [[S]], <4 x i64> splat (i64 4294967295))
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = trunc <4 x i64> [[_MSPROP]] to <4 x i32>
+; CHECK-NEXT:    [[TRUNC:%.*]] = trunc <4 x i64> [[MIN]] to <4 x i32>
+; CHECK-NEXT:    store <4 x i32> [[_MSPROP1]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i32> [[TRUNC]]
+;
+entry:
+  %s = lshr <4 x i64> %x, <i64 5, i64 5, i64 5, i64 5>
+  %min = call <4 x i64> @llvm.umin.v8i64(<4 x i64> %s, <4 x i64> <i64 4294967295, i64 4294967295, i64 4294967295, i64 4294967295>)
+  %trunc = trunc <4 x i64> %min to <4 x i32>
+  ret <4 x i32> %trunc
+}
+
+define <4 x i32> @unsigned_signed_minmax_v4i64_to_v8i32(<4 x i64> %x) #0 {
+; CHECK-LABEL: define <4 x i32> @unsigned_signed_minmax_v4i64_to_v8i32(
+; CHECK-SAME: <4 x i64> [[X:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load <4 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP1:%.*]] = ashr <4 x i64> [[TMP0]], splat (i64 5)
+; CHECK-NEXT:    [[TMP2:%.*]] = or <4 x i64> [[TMP1]], zeroinitializer
+; CHECK-NEXT:    [[S:%.*]] = ashr <4 x i64> [[X]], splat (i64 5)
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <4 x i64> [[TMP2]], zeroinitializer
+; CHECK-NEXT:    [[MAX:%.*]] = call <4 x i64> @llvm.smax.v4i64(<4 x i64> [[S]], <4 x i64> zeroinitializer)
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <4 x i64> [[_MSPROP]], zeroinitializer
+; CHECK-NEXT:    [[MIN:%.*]] = call <4 x i64> @llvm.umin.v4i64(<4 x i64> [[MAX]], <4 x i64> splat (i64 4294967295))
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = trunc <4 x i64> [[_MSPROP1]] to <4 x i32>
+; CHECK-NEXT:    [[TRUNC:%.*]] = trunc <4 x i64> [[MIN]] to <4 x i32>
+; CHECK-NEXT:    store <4 x i32> [[_MSPROP2]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i32> [[TRUNC]]
+;
+entry:
+  %s = ashr <4 x i64> %x, <i64 5, i64 5, i64 5, i64 5>
+  %max = call <4 x i64> @llvm.smax.v8i64(<4 x i64> %s, <4 x i64> <i64 0, i64 0, i64 0, i64 0>)
+  %min = call <4 x i64> @llvm.umin.v8i64(<4 x i64> %max, <4 x i64> <i64 4294967295, i64 4294967295, i64 4294967295, i64 4294967295>)
+  %trunc = trunc <4 x i64> %min to <4 x i32>
+  ret <4 x i32> %trunc
+}
+
+attributes #0 = { sanitize_memory }
+;.
+; CHECK: [[PROF1]] = !{!"branch_weights", i32 1, i32 1048575}
+;.
diff --git a/llvm/test/Instrumentation/MemorySanitizer/X86/avx-intrinsics-x86.ll b/llvm/test/Instrumentation/MemorySanitizer/X86/avx-intrinsics-x86.ll
index 7273e431a9c2a..26e9c39696f70 100644
--- a/llvm/test/Instrumentation/MemorySanitizer/X86/avx-intrinsics-x86.ll
+++ b/llvm/test/Instrumentation/MemorySanitizer/X86/avx-intrinsics-x86.ll
@@ -475,7 +475,7 @@ define <4 x double> @test_x86_avx_hsub_pd_256(<4 x double> %a0, <4 x double> %a1
 ; CHECK-NEXT:    call void @llvm.donothing()
 ; CHECK-NEXT:    [[A0:%.*]] = bitcast <4 x i64> [[TMP1]] to <4 x double>
 ; CHECK-NEXT:    [[A1:%.*]] = bitcast <4 x i64> [[TMP2]] to <4 x double>
-; CHECK-NEXT:    [[RES:%.*]] = call <4 x double> @llvm.x86.avx.hsub.pd.256(<4 x double> [[A0]], <4 x double> [[A1]])
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x double> @llvm.x86.avx.hadd.pd.256(<4 x double> [[A0]], <4 x double> [[A1]])
 ; CHECK-NEXT:    [[_MSPROP:%.*]] = bitcast <4 x double> [[RES]] to <4 x i64>
 ; CHECK-NEXT:    [[RES1:%.*]] = call <4 x double> @llvm.x86.avx.hsub.pd.256(<4 x double> [[A2:%.*]], <4 x double> [[A3:%.*]])
 ; CHECK-NEXT:    store <4 x i64> [[_MSPROP]], ptr @__msan_retval_tls, align 8
@@ -494,7 +494,7 @@ define <8 x float> @test_x86_avx_hsub_ps_256(<8 x float> %a0, <8 x float> %a1) #
 ; CHECK-NEXT:    call void @llvm.donothing()
 ; CHECK-NEXT:    [[A0:%.*]] = bitcast <8 x i32> [[TMP1]] to <8 x float>
 ; CHECK-NEXT:    [[A1:%.*]] = bitcast <8 x i32> [[TMP2]] to <8 x float>
-; CHECK-NEXT:    [[RES:%.*]] = call <8 x float> @llvm.x86.avx.hsub.ps.256(<8 x float> [[A0]], <8 x float> [[A1]])
+; CHECK-NEXT:    [[RES:%.*]] = call <8 x float> @llvm.x86.avx.hadd.ps.256(<8 x float> [[A0]], <8 x float> [[A1]])
 ; CHECK-NEXT:    [[_MSPROP:%.*]] = bitcast <8 x float> [[RES]] to <8 x i32>
 ; CHECK-NEXT:    [[RES1:%.*]] = call <8 x float> @llvm.x86.avx.hsub.ps.256(<8 x float> [[A2:%.*]], <8 x float> [[A3:%.*]])
 ; CHECK-NEXT:    store <8 x i32> [[_MSPROP]], ptr @__msan_retval_tls, align 8
@@ -532,20 +532,22 @@ declare <32 x i8> @llvm.x86.avx.ldu.dq.256(ptr) nounwind readonly
 
 define <2 x double> @test_x86_avx_maskload_pd(ptr %a0, <2 x i64> %mask) #0 {
 ; CHECK-LABEL: @test_x86_avx_maskload_pd(
-; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    [[TMP7:%.*]] = ptrtoint ptr [[A0:%.*]] to i64
+; CHECK-NEXT:    [[TMP10:%.*]] = xor i64 [[TMP7]], 87960930222080
+; CHECK-NEXT:    [[TMP4:%.*]] = inttoptr i64 [[TMP10]] to ptr
+; CHECK-NEXT:    [[TMP5:%.*]] = call <2 x double> @llvm.x86.avx.maskload.pd(ptr [[TMP4]], <2 x i64> [[MASK:%.*]])
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <2 x double> [[TMP5]] to <2 x i64>
 ; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <2 x i64> [[TMP2]] to i128
 ; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP3]], 0
-; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
-; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP4:%.*]], label [[TMP5:%.*]], !prof [[PROF1]]
-; CHECK:       4:
+; CHECK-NEXT:    br i1 [[_MSCMP1]], label [[TMP8:%.*]], label [[TMP9:%.*]], !prof [[PROF1]]
+; CHECK:       8:
 ; CHECK-NEXT:    call void @__msan_warning_noreturn()
 ; CHECK-NEXT:    unreachable
-; CHECK:       5:
-; CHECK-NEXT:    [[RES:%.*]] = call <2 x double> @llvm.x86.avx.maskload.pd(ptr [[A0:%.*]], <2 x i64> [[MASK:%.*]])
-; CHECK-NEXT:    store <2 x i64> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK:       9:
+; CHECK-NEXT:    [[RES:%.*]] = call <2 x double> @llvm.x86.avx.maskload.pd(ptr [[A0]], <2 x i64> [[MASK]])
+; CHECK-NEXT:    store <2 x i64> [[TMP6]], ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret <2 x double> [[RES]]
 ;
   %res = call <2 x double> @llvm.x86.avx.maskload.pd(ptr %a0, <2 x i64> %mask) ; <<2 x double>> [#uses=1]
@@ -556,20 +558,22 @@ declare <2 x double> @llvm.x86.avx.maskload.pd(ptr, <2 x i64>) nounwind readonly
 
 define <4 x double> @test_x86_avx_maskload_pd_256(ptr %a0, <4 x i64> %mask) #0 {
 ; CHECK-LABEL: @test_x86_avx_maskload_pd_256(
-; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    [[TMP7:%.*]] = ptrtoint ptr [[A0:%.*]] to i64
+; CHECK-NEXT:    [[TMP10:%.*]] = xor i64 [[TMP7]], 87960930222080
+; CHECK-NEXT:    [[TMP4:%.*]] = inttoptr i64 [[TMP10]] to ptr
+; CHECK-NEXT:    [[TMP5:%.*]] = call <4 x double> @llvm.x86.avx.maskload.pd.256(ptr [[TMP4]], <4 x i64> [[MASK:%.*]])
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <4 x double> [[TMP5]] to <4 x i64>
 ; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <4 x i64> [[TMP2]] to i256
 ; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i256 [[TMP3]], 0
-; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
-; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP4:%.*]], label [[TMP5:%.*]], !prof [[PROF1]]
-; CHECK:       4:
+; CHECK-NEXT:    br i1 [[_MSCMP1]], label [[TMP8:%.*]], label [[TMP9:%.*]], !prof [[PROF1]]
+; CHECK:       8:
 ; CHECK-NEXT:    call void @__msan_warning_noreturn()
 ; CHECK-NEXT:    unreachable
-; CHECK:       5:
-; CHECK-NEXT:    [[RES:%.*]] = call <4 x double> @llvm.x86.avx.maskload.pd.256(ptr [[A0:%.*]], <4 x i64> [[MASK:%.*]])
-; CHECK-NEXT:    store <4 x i64> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK:       9:
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x double> @llvm.x86.avx.maskload.pd.256(ptr [[A0]], <4 x i64> [[MASK]])
+; CHECK-NEXT:    store <4 x i64> [[TMP6]], ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret <4 x double> [[RES]]
 ;
   %res = call <4 x double> @llvm.x86.avx.maskload.pd.256(ptr %a0, <4 x i64> %mask) ; <<4 x double>> [#uses=1]
@@ -580,20 +584,22 @@ declare <4 x double> @llvm.x86.avx.maskload.pd.256(ptr, <4 x i64>) nounwind read
 
 define <4 x float> @test_x86_avx_maskload_ps(ptr %a0, <4 x i32> %mask) #0 {
 ; CHECK-LABEL: @test_x86_avx_maskload_ps(
-; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    [[TMP7:%.*]] = ptrtoint ptr [[A0:%.*]] to i64
+; CHECK-NEXT:    [[TMP10:%.*]] = xor i64 [[TMP7]], 87960930222080
+; CHECK-NEXT:    [[TMP4:%.*]] = inttoptr i64 [[TMP10]] to ptr
+; CHECK-NEXT:    [[TMP5:%.*]] = call <4 x float> @llvm.x86.avx.maskload.ps(ptr [[TMP4]], <4 x i32> [[MASK:%.*]])
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <4 x float> [[TMP5]] to <4 x i32>
 ; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <4 x i32> [[TMP2]] to i128
 ; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP3]], 0
-; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
-; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP4:%.*]], label [[TMP5:%.*]], !prof [[PROF1]]
-; CHECK:       4:
+; CHECK-NEXT:    br i1 [[_MSCMP1]], label [[TMP8:%.*]], label [[TMP9:%.*]], !prof [[PROF1]]
+; CHECK:       8:
 ; CHECK-NEXT:    call void @__msan_warning_noreturn()
 ; CHECK-NEXT:    unreachable
-; CHECK:       5:
-; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.x86.avx.maskload.ps(ptr [[A0:%.*]], <4 x i32> [[MASK:%.*]])
-; CHECK-NEXT:    store <4 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK:       9:
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.x86.avx.maskload.ps(ptr [[A0]], <4 x i32> [[MASK]])
+; CHECK-NEXT:    store <4 x i32> [[TMP6]], ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret <4 x float> [[RES]]
 ;
   %res = call <4 x float> @llvm.x86.avx.maskload.ps(ptr %a0, <4 x i32> %mask) ; <<4 x float>> [#uses=1]
@@ -604,20 +610,22 @@ declare <4 x float> @llvm.x86.avx.maskload.ps(ptr, <4 x i32>) nounwind readonly
 
 define <8 x float> @test_x86_avx_maskload_ps_256(ptr %a0, <8 x i32> %mask) #0 {
 ; CHECK-LABEL: @test_x86_avx_maskload_ps_256(
-; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    [[TMP7:%.*]] = ptrtoint ptr [[A0:%.*]] to i64
+; CHECK-NEXT:    [[TMP10:%.*]] = xor i64 [[TMP7]], 87960930222080
+; CHECK-NEXT:    [[TMP4:%.*]] = inttoptr i64 [[TMP10]] to ptr
+; CHECK-NEXT:    [[TMP5:%.*]] = call <8 x float> @llvm.x86.avx.maskload.ps.256(ptr [[TMP4]], <8 x i32> [[MASK:%.*]])
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <8 x float> [[TMP5]] to <8 x i32>
 ; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <8 x i32> [[TMP2]] to i256
 ; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i256 [[TMP3]], 0
-; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
-; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP4:%.*]], label [[TMP5:%.*]], !prof [[PROF1]]
-; CHECK:       4:
+; CHECK-NEXT:    br i1 [[_MSCMP1]], label [[TMP8:%.*]], label [[TMP9:%.*]], !prof [[PROF1]]
+; CHECK:       8:
 ; CHECK-NEXT:    call void @__msan_warning_noreturn()
 ; CHECK-NEXT:    unreachable
-; CHECK:       5:
-; CHECK-NEXT:    [[RES:%.*]] = call <8 x float> @llvm.x86.avx.maskload.ps.256(ptr [[A0:%.*]], <8 x i32> [[MASK:%.*]])
-; CHECK-NEXT:    store <8 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK:       9:
+; CHECK-NEXT:    [[RES:%.*]] = call <8 x float> @llvm.x86.avx.maskload.ps.256(ptr [[A0]], <8 x i32> [[MASK]])
+; CHECK-NEXT:    store <8 x i32> [[TMP6]], ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret <8 x float> [[RES]]
 ;
   %res = call <8 x float> @llvm.x86.avx.maskload.ps.256(ptr %a0, <8 x i32> %mask) ; <<8 x float>> [#uses=1]
@@ -628,23 +636,25 @@ declare <8 x float> @llvm.x86.avx.maskload.ps.256(ptr, <8 x i32>) nounwind reado
 
 define void @test_x86_avx_maskstore_pd(ptr %a0, <2 x i64> %mask, <2 x double> %a2) #0 {
 ; CHECK-LABEL: @test_x86_avx_maskstore_pd(
+; CHECK-NEXT:    [[TMP3:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 24) to ptr), align 8
 ; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
-; CHECK-NEXT:    [[TMP3:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 24) to ptr), align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = ptrtoint ptr [[A0:%.*]] to i64
+; CHECK-NEXT:    [[TMP8:%.*]] = xor i64 [[TMP4]], 87960930222080
+; CHECK-NEXT:    [[TMP6:%.*]] = inttoptr i64 [[TMP8]] to ptr
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <2 x i64> [[TMP3]] to <2 x double>
+; CHECK-NEXT:    call void @llvm.x86.avx.maskstore.pd(ptr [[TMP6]], <2 x i64> [[MASK:%.*]], <2 x double> [[TMP7]])
 ; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
-; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <2 x i64> [[TMP2]] to i128
-; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP4]], 0
-; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
-; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <2 x i64> [[TMP3]] to i128
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <2 x i64> [[TMP2]] to i128
 ; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i128 [[TMP5]], 0
-; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
-; CHECK-NEXT:    br i1 [[_MSOR3]], label [[TMP6:%.*]], label [[TMP7:%.*]], !prof [[PROF1]]
-; CHECK:       6:
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP9:%.*]], label [[TMP10:%.*]], !prof [[PROF1]]
+; CHECK:       9:
 ; CHECK-NEXT:    call void @__msan_warning_noreturn()
 ; CHECK-NEXT:    unreachable
-; CHECK:       7:
-; CHECK-NEXT:    call void @llvm.x86.avx.maskstore.pd(ptr [[A0:%.*]], <2 x i64> [[MASK:%.*]], <2 x double> [[A2:%.*]])
+; CHECK:       10:
+; CHECK-NEXT:    call void @llvm.x86.avx.maskstore.pd(ptr [[A0]], <2 x i64> [[MASK]], <2 x double> [[A2:%.*]])
 ; CHECK-NEXT:    ret void
 ;
   call void @llvm.x86.avx.maskstore.pd(ptr %a0, <2 x i64> %mask, <2 x double> %a2)
@@ -655,23 +665,25 @@ declare void @llvm.x86.avx.maskstore.pd(ptr, <2 x i64>, <2 x double>) nounwind
 
 define void @test_x86_avx_maskstore_pd_256(ptr %a0, <4 x i64> %mask, <4 x double> %a2) #0 {
 ; CHECK-LABEL: @test_x86_avx_maskstore_pd_256(
+; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 40) to ptr), align 8
 ; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
-; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 40) to ptr), align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = ptrtoint ptr [[A0:%.*]] to i64
+; CHECK-NEXT:    [[TMP8:%.*]] = xor i64 [[TMP4]], 87960930222080
+; CHECK-NEXT:    [[TMP6:%.*]] = inttoptr i64 [[TMP8]] to ptr
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <4 x i64> [[TMP3]] to <4 x double>
+; CHECK-NEXT:    call void @llvm.x86.avx.maskstore.pd.256(ptr [[TMP6]], <4 x i64> [[MASK:%.*]], <4 x double> [[TMP7]])
 ; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
-; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <4 x i64> [[TMP2]] to i256
-; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i256 [[TMP4]], 0
-; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
-; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <4 x i64> [[TMP3]] to i256
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <4 x i64> [[TMP2]] to i256
 ; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i256 [[TMP5]], 0
-; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
-; CHECK-NEXT:    br i1 [[_MSOR3]], label [[TMP6:%.*]], label [[TMP7:%.*]], !prof [[PROF1]]
-; CHECK:       6:
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP9:%.*]], label [[TMP10:%.*]], !prof [[PROF1]]
+; CHECK:       9:
 ; CHECK-NEXT:    call void @__msan_warning_noreturn()
 ; CHECK-NEXT:    unreachable
-; CHECK:       7:
-; CHECK-NEXT:    call void @llvm.x86.avx.maskstore.pd.256(ptr [[A0:%.*]], <4 x i64> [[MASK:%.*]], <4 x double> [[A2:%.*]])
+; CHECK:       10:
+; CHECK-NEXT:    call void @llvm.x86.avx.maskstore.pd.256(ptr [[A0]], <4 x i64> [[MASK]], <4 x double> [[A2:%.*]])
 ; CHECK-NEXT:    ret void
 ;
   call void @llvm.x86.avx.maskstore.pd.256(ptr %a0, <4 x i64> %mask, <4 x double> %a2)
@@ -682,23 +694,25 @@ declare void @llvm.x86.avx.maskstore.pd.256(ptr, <4 x i64>, <4 x double>) nounwi
 
 define void @test_x86_avx_maskstore_ps(ptr %a0, <4 x i32> %mask, <4 x float> %a2) #0 {
 ; CHECK-LABEL: @test_x86_avx_maskstore_ps(
+; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 24) to ptr), align 8
 ; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
-; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 24) to ptr), align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = ptrtoint ptr [[A0:%.*]] to i64
+; CHECK-NEXT:    [[TMP8:%.*]] = xor i64 [[TMP4]], 87960930222080
+; CHECK-NEXT:    [[TMP6:%.*]] = inttoptr i64 [[TMP8]] to ptr
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <4 x i32> [[TMP3]] to <4 x float>
+; CHECK-NEXT:    call void @llvm.x86.avx.maskstore.ps(ptr [[TMP6]], <4 x i32> [[MASK:%.*]], <4 x float> [[TMP7]])
 ; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
-; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <4 x i32> [[TMP2]] to i128
-; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP4]], 0
-; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
-; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <4 x i32> [[TMP3]] to i128
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <4 x i32> [[TMP2]] to i128
 ; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i128 [[TMP5]], 0
-; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
-; CHECK-NEXT:    br i1 [[_MSOR3]], label [[TMP6:%.*]], label [[TMP7:%.*]], !prof [[PROF1]]
-; CHECK:       6:
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP9:%.*]], label [[TMP10:%.*]], !prof [[PROF1]]
+; CHECK:       9:
 ; CHECK-NEXT:    call void @__msan_warning_noreturn()
 ; CHECK-NEXT:    unreachable
-; CHECK:       7:
-; CHECK-NEXT:    call void @llvm.x86.avx.maskstore.ps(ptr [[A0:%.*]], <4 x i32> [[MASK:%.*]], <4 x float> [[A2:%.*]])
+; CHECK:       10:
+; CHECK-NEXT:    call void @llvm.x86.avx.maskstore.ps(ptr [[A0]], <4 x i32> [[MASK]], <4 x float> [[A2:%.*]])
 ; CHECK-NEXT:    ret void
 ;
   call void @llvm.x86.avx.maskstore.ps(ptr %a0, <4 x i32> %mask, <4 x float> %a2)
@@ -709,23 +723,25 @@ declare void @llvm.x86.avx.maskstore.ps(ptr, <4 x i32>, <4 x float>) nounwind
 
 define void @test_x86_avx_maskstore_ps_256(ptr %a0, <8 x i32> %mask, <8 x float> %a2) #0 {
 ; CHECK-LABEL: @test_x86_avx_maskstore_ps_256(
+; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 40) to ptr), align 8
 ; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
-; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 40) to ptr), align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = ptrtoint ptr [[A0:%.*]] to i64
+; CHECK-NEXT:    [[TMP8:%.*]] = xor i64 [[TMP4]], 87960930222080
+; CHECK-NEXT:    [[TMP6:%.*]] = inttoptr i64 [[TMP8]] to ptr
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <8 x i32> [[TMP3]] to <8 x float>
+; CHECK-NEXT:    call void @llvm.x86.avx.maskstore.ps.256(ptr [[TMP6]], <8 x i32> [[MASK:%.*]], <8 x float> [[TMP7]])
 ; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
-; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <8 x i32> [[TMP2]] to i256
-; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i256 [[TMP4]], 0
-; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
-; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <8 x i32> [[TMP3]] to i256
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <8 x i32> [[TMP2]] to i256
 ; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i256 [[TMP5]], 0
-; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
-; CHECK-NEXT:    br i1 [[_MSOR3]], label [[TMP6:%.*]], label [[TMP7:%.*]], !prof [[PROF1]]
-; CHECK:       6:
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP9:%.*]], label [[TMP10:%.*]], !prof [[PROF1]]
+; CHECK:       9:
 ; CHECK-NEXT:    call void @__msan_warning_noreturn()
 ; CHECK-NEXT:    unreachable
-; CHECK:       7:
-; CHECK-NEXT:    call void @llvm.x86.avx.maskstore.ps.256(ptr [[A0:%.*]], <8 x i32> [[MASK:%.*]], <8 x float> [[A2:%.*]])
+; CHECK:       10:
+; CHECK-NEXT:    call void @llvm.x86.avx.maskstore.ps.256(ptr [[A0]], <8 x i32> [[MASK]], <8 x float> [[A2:%.*]])
 ; CHECK-NEXT:    ret void
 ;
   call void @llvm.x86.avx.maskstore.ps.256(ptr %a0, <8 x i32> %mask, <8 x float> %a2)
diff --git a/llvm/test/Instrumentation/MemorySanitizer/X86/avx2-intrinsics-x86.ll b/llvm/test/Instrumentation/MemorySanitizer/X86/avx2-intrinsics-x86.ll
index e10062142c046..5597a9c96611f 100644
--- a/llvm/test/Instrumentation/MemorySanitizer/X86/avx2-intrinsics-x86.ll
+++ b/llvm/test/Instrumentation/MemorySanitizer/X86/avx2-intrinsics-x86.ll
@@ -617,7 +617,7 @@ define <8 x i32> @test_x86_avx2_phsub_d(<8 x i32> %a0, <8 x i32> %a1) #0 {
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i32>, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[_MSPROP:%.*]] = call <8 x i32> @llvm.x86.avx2.phsub.d(<8 x i32> [[TMP1]], <8 x i32> [[TMP2]])
+; CHECK-NEXT:    [[_MSPROP:%.*]] = call <8 x i32> @llvm.x86.avx2.phadd.d(<8 x i32> [[TMP1]], <8 x i32> [[TMP2]])
 ; CHECK-NEXT:    [[RES:%.*]] = call <8 x i32> @llvm.x86.avx2.phsub.d(<8 x i32> [[A0:%.*]], <8 x i32> [[A1:%.*]])
 ; CHECK-NEXT:    store <8 x i32> [[_MSPROP]], ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret <8 x i32> [[RES]]
@@ -633,7 +633,7 @@ define <16 x i16> @test_x86_avx2_phsub_sw(<16 x i16> %a0, <16 x i16> %a1) #0 {
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i16>, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[_MSPROP:%.*]] = call <16 x i16> @llvm.x86.avx2.phsub.sw(<16 x i16> [[TMP1]], <16 x i16> [[TMP2]])
+; CHECK-NEXT:    [[_MSPROP:%.*]] = call <16 x i16> @llvm.x86.avx2.phadd.sw(<16 x i16> [[TMP1]], <16 x i16> [[TMP2]])
 ; CHECK-NEXT:    [[RES:%.*]] = call <16 x i16> @llvm.x86.avx2.phsub.sw(<16 x i16> [[A0:%.*]], <16 x i16> [[A1:%.*]])
 ; CHECK-NEXT:    store <16 x i16> [[_MSPROP]], ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret <16 x i16> [[RES]]
@@ -649,7 +649,7 @@ define <16 x i16> @test_x86_avx2_phsub_w(<16 x i16> %a0, <16 x i16> %a1) #0 {
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i16>, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[_MSPROP:%.*]] = call <16 x i16> @llvm.x86.avx2.phsub.w(<16 x i16> [[TMP1]], <16 x i16> [[TMP2]])
+; CHECK-NEXT:    [[_MSPROP:%.*]] = call <16 x i16> @llvm.x86.avx2.phadd.w(<16 x i16> [[TMP1]], <16 x i16> [[TMP2]])
 ; CHECK-NEXT:    [[RES:%.*]] = call <16 x i16> @llvm.x86.avx2.phsub.w(<16 x i16> [[A0:%.*]], <16 x i16> [[A1:%.*]])
 ; CHECK-NEXT:    store <16 x i16> [[_MSPROP]], ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret <16 x i16> [[RES]]
@@ -995,20 +995,21 @@ declare <8 x float> @llvm.x86.avx2.permps(<8 x float>, <8 x i32>) nounwind reado
 
 define <2 x i64> @test_x86_avx2_maskload_q(ptr %a0, <2 x i64> %a1) #0 {
 ; CHECK-LABEL: @test_x86_avx2_maskload_q(
-; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = ptrtoint ptr [[A0:%.*]] to i64
+; CHECK-NEXT:    [[TMP9:%.*]] = xor i64 [[TMP6]], 87960930222080
+; CHECK-NEXT:    [[TMP4:%.*]] = inttoptr i64 [[TMP9]] to ptr
+; CHECK-NEXT:    [[TMP5:%.*]] = call <2 x i64> @llvm.x86.avx2.maskload.q(ptr [[TMP4]], <2 x i64> [[A1:%.*]])
 ; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <2 x i64> [[TMP2]] to i128
 ; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP3]], 0
-; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
-; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP4:%.*]], label [[TMP5:%.*]], !prof [[PROF1]]
-; CHECK:       4:
+; CHECK-NEXT:    br i1 [[_MSCMP1]], label [[TMP7:%.*]], label [[TMP8:%.*]], !prof [[PROF1]]
+; CHECK:       7:
 ; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
 ; CHECK-NEXT:    unreachable
-; CHECK:       5:
-; CHECK-NEXT:    [[RES:%.*]] = call <2 x i64> @llvm.x86.avx2.maskload.q(ptr [[A0:%.*]], <2 x i64> [[A1:%.*]])
-; CHECK-NEXT:    store <2 x i64> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK:       8:
+; CHECK-NEXT:    [[RES:%.*]] = call <2 x i64> @llvm.x86.avx2.maskload.q(ptr [[A0]], <2 x i64> [[A1]])
+; CHECK-NEXT:    store <2 x i64> [[TMP5]], ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret <2 x i64> [[RES]]
 ;
   %res = call <2 x i64> @llvm.x86.avx2.maskload.q(ptr %a0, <2 x i64> %a1) ; <<2 x i64>> [#uses=1]
@@ -1019,20 +1020,21 @@ declare <2 x i64> @llvm.x86.avx2.maskload.q(ptr, <2 x i64>) nounwind readonly
 
 define <4 x i64> @test_x86_avx2_maskload_q_256(ptr %a0, <4 x i64> %a1) #0 {
 ; CHECK-LABEL: @test_x86_avx2_maskload_q_256(
-; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = ptrtoint ptr [[A0:%.*]] to i64
+; CHECK-NEXT:    [[TMP9:%.*]] = xor i64 [[TMP6]], 87960930222080
+; CHECK-NEXT:    [[TMP4:%.*]] = inttoptr i64 [[TMP9]] to ptr
+; CHECK-NEXT:    [[TMP5:%.*]] = call <4 x i64> @llvm.x86.avx2.maskload.q.256(ptr [[TMP4]], <4 x i64> [[A1:%.*]])
 ; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <4 x i64> [[TMP2]] to i256
 ; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i256 [[TMP3]], 0
-; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
-; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP4:%.*]], label [[TMP5:%.*]], !prof [[PROF1]]
-; CHECK:       4:
+; CHECK-NEXT:    br i1 [[_MSCMP1]], label [[TMP7:%.*]], label [[TMP8:%.*]], !prof [[PROF1]]
+; CHECK:       7:
 ; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
 ; CHECK-NEXT:    unreachable
-; CHECK:       5:
-; CHECK-NEXT:    [[RES:%.*]] = call <4 x i64> @llvm.x86.avx2.maskload.q.256(ptr [[A0:%.*]], <4 x i64> [[A1:%.*]])
-; CHECK-NEXT:    store <4 x i64> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK:       8:
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x i64> @llvm.x86.avx2.maskload.q.256(ptr [[A0]], <4 x i64> [[A1]])
+; CHECK-NEXT:    store <4 x i64> [[TMP5]], ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret <4 x i64> [[RES]]
 ;
   %res = call <4 x i64> @llvm.x86.avx2.maskload.q.256(ptr %a0, <4 x i64> %a1) ; <<4 x i64>> [#uses=1]
@@ -1043,20 +1045,21 @@ declare <4 x i64> @llvm.x86.avx2.maskload.q.256(ptr, <4 x i64>) nounwind readonl
 
 define <4 x i32> @test_x86_avx2_maskload_d(ptr %a0, <4 x i32> %a1) #0 {
 ; CHECK-LABEL: @test_x86_avx2_maskload_d(
-; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = ptrtoint ptr [[A0:%.*]] to i64
+; CHECK-NEXT:    [[TMP9:%.*]] = xor i64 [[TMP6]], 87960930222080
+; CHECK-NEXT:    [[TMP4:%.*]] = inttoptr i64 [[TMP9]] to ptr
+; CHECK-NEXT:    [[TMP5:%.*]] = call <4 x i32> @llvm.x86.avx2.maskload.d(ptr [[TMP4]], <4 x i32> [[A1:%.*]])
 ; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <4 x i32> [[TMP2]] to i128
 ; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP3]], 0
-; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
-; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP4:%.*]], label [[TMP5:%.*]], !prof [[PROF1]]
-; CHECK:       4:
+; CHECK-NEXT:    br i1 [[_MSCMP1]], label [[TMP7:%.*]], label [[TMP8:%.*]], !prof [[PROF1]]
+; CHECK:       7:
 ; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
 ; CHECK-NEXT:    unreachable
-; CHECK:       5:
-; CHECK-NEXT:    [[RES:%.*]] = call <4 x i32> @llvm.x86.avx2.maskload.d(ptr [[A0:%.*]], <4 x i32> [[A1:%.*]])
-; CHECK-NEXT:    store <4 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK:       8:
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x i32> @llvm.x86.avx2.maskload.d(ptr [[A0]], <4 x i32> [[A1]])
+; CHECK-NEXT:    store <4 x i32> [[TMP5]], ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret <4 x i32> [[RES]]
 ;
   %res = call <4 x i32> @llvm.x86.avx2.maskload.d(ptr %a0, <4 x i32> %a1) ; <<4 x i32>> [#uses=1]
@@ -1067,20 +1070,21 @@ declare <4 x i32> @llvm.x86.avx2.maskload.d(ptr, <4 x i32>) nounwind readonly
 
 define <8 x i32> @test_x86_avx2_maskload_d_256(ptr %a0, <8 x i32> %a1) #0 {
 ; CHECK-LABEL: @test_x86_avx2_maskload_d_256(
-; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = ptrtoint ptr [[A0:%.*]] to i64
+; CHECK-NEXT:    [[TMP9:%.*]] = xor i64 [[TMP6]], 87960930222080
+; CHECK-NEXT:    [[TMP4:%.*]] = inttoptr i64 [[TMP9]] to ptr
+; CHECK-NEXT:    [[TMP5:%.*]] = call <8 x i32> @llvm.x86.avx2.maskload.d.256(ptr [[TMP4]], <8 x i32> [[A1:%.*]])
 ; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <8 x i32> [[TMP2]] to i256
 ; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i256 [[TMP3]], 0
-; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
-; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP4:%.*]], label [[TMP5:%.*]], !prof [[PROF1]]
-; CHECK:       4:
+; CHECK-NEXT:    br i1 [[_MSCMP1]], label [[TMP7:%.*]], label [[TMP8:%.*]], !prof [[PROF1]]
+; CHECK:       7:
 ; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
 ; CHECK-NEXT:    unreachable
-; CHECK:       5:
-; CHECK-NEXT:    [[RES:%.*]] = call <8 x i32> @llvm.x86.avx2.maskload.d.256(ptr [[A0:%.*]], <8 x i32> [[A1:%.*]])
-; CHECK-NEXT:    store <8 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK:       8:
+; CHECK-NEXT:    [[RES:%.*]] = call <8 x i32> @llvm.x86.avx2.maskload.d.256(ptr [[A0]], <8 x i32> [[A1]])
+; CHECK-NEXT:    store <8 x i32> [[TMP5]], ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret <8 x i32> [[RES]]
 ;
   %res = call <8 x i32> @llvm.x86.avx2.maskload.d.256(ptr %a0, <8 x i32> %a1) ; <<8 x i32>> [#uses=1]
@@ -1091,23 +1095,24 @@ declare <8 x i32> @llvm.x86.avx2.maskload.d.256(ptr, <8 x i32>) nounwind readonl
 
 define void @test_x86_avx2_maskstore_q(ptr %a0, <2 x i64> %a1, <2 x i64> %a2) #0 {
 ; CHECK-LABEL: @test_x86_avx2_maskstore_q(
+; CHECK-NEXT:    [[TMP3:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 24) to ptr), align 8
 ; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
-; CHECK-NEXT:    [[TMP3:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 24) to ptr), align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = ptrtoint ptr [[A0:%.*]] to i64
+; CHECK-NEXT:    [[TMP7:%.*]] = xor i64 [[TMP4]], 87960930222080
+; CHECK-NEXT:    [[TMP6:%.*]] = inttoptr i64 [[TMP7]] to ptr
+; CHECK-NEXT:    call void @llvm.x86.avx2.maskstore.q(ptr [[TMP6]], <2 x i64> [[A1:%.*]], <2 x i64> [[TMP3]])
 ; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
-; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <2 x i64> [[TMP2]] to i128
-; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP4]], 0
-; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
-; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <2 x i64> [[TMP3]] to i128
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <2 x i64> [[TMP2]] to i128
 ; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i128 [[TMP5]], 0
-; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
-; CHECK-NEXT:    br i1 [[_MSOR3]], label [[TMP6:%.*]], label [[TMP7:%.*]], !prof [[PROF1]]
-; CHECK:       6:
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP8:%.*]], label [[TMP9:%.*]], !prof [[PROF1]]
+; CHECK:       8:
 ; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
 ; CHECK-NEXT:    unreachable
-; CHECK:       7:
-; CHECK-NEXT:    call void @llvm.x86.avx2.maskstore.q(ptr [[A0:%.*]], <2 x i64> [[A1:%.*]], <2 x i64> [[A2:%.*]])
+; CHECK:       9:
+; CHECK-NEXT:    call void @llvm.x86.avx2.maskstore.q(ptr [[A0]], <2 x i64> [[A1]], <2 x i64> [[A2:%.*]])
 ; CHECK-NEXT:    ret void
 ;
   call void @llvm.x86.avx2.maskstore.q(ptr %a0, <2 x i64> %a1, <2 x i64> %a2)
@@ -1118,23 +1123,24 @@ declare void @llvm.x86.avx2.maskstore.q(ptr, <2 x i64>, <2 x i64>) nounwind
 
 define void @test_x86_avx2_maskstore_q_256(ptr %a0, <4 x i64> %a1, <4 x i64> %a2) #0 {
 ; CHECK-LABEL: @test_x86_avx2_maskstore_q_256(
+; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 40) to ptr), align 8
 ; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
-; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 40) to ptr), align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = ptrtoint ptr [[A0:%.*]] to i64
+; CHECK-NEXT:    [[TMP7:%.*]] = xor i64 [[TMP4]], 87960930222080
+; CHECK-NEXT:    [[TMP6:%.*]] = inttoptr i64 [[TMP7]] to ptr
+; CHECK-NEXT:    call void @llvm.x86.avx2.maskstore.q.256(ptr [[TMP6]], <4 x i64> [[A1:%.*]], <4 x i64> [[TMP3]])
 ; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
-; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <4 x i64> [[TMP2]] to i256
-; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i256 [[TMP4]], 0
-; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
-; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <4 x i64> [[TMP3]] to i256
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <4 x i64> [[TMP2]] to i256
 ; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i256 [[TMP5]], 0
-; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
-; CHECK-NEXT:    br i1 [[_MSOR3]], label [[TMP6:%.*]], label [[TMP7:%.*]], !prof [[PROF1]]
-; CHECK:       6:
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP8:%.*]], label [[TMP9:%.*]], !prof [[PROF1]]
+; CHECK:       8:
 ; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
 ; CHECK-NEXT:    unreachable
-; CHECK:       7:
-; CHECK-NEXT:    call void @llvm.x86.avx2.maskstore.q.256(ptr [[A0:%.*]], <4 x i64> [[A1:%.*]], <4 x i64> [[A2:%.*]])
+; CHECK:       9:
+; CHECK-NEXT:    call void @llvm.x86.avx2.maskstore.q.256(ptr [[A0]], <4 x i64> [[A1]], <4 x i64> [[A2:%.*]])
 ; CHECK-NEXT:    ret void
 ;
   call void @llvm.x86.avx2.maskstore.q.256(ptr %a0, <4 x i64> %a1, <4 x i64> %a2)
@@ -1145,23 +1151,24 @@ declare void @llvm.x86.avx2.maskstore.q.256(ptr, <4 x i64>, <4 x i64>) nounwind
 
 define void @test_x86_avx2_maskstore_d(ptr %a0, <4 x i32> %a1, <4 x i32> %a2) #0 {
 ; CHECK-LABEL: @test_x86_avx2_maskstore_d(
+; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 24) to ptr), align 8
 ; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
-; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 24) to ptr), align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = ptrtoint ptr [[A0:%.*]] to i64
+; CHECK-NEXT:    [[TMP7:%.*]] = xor i64 [[TMP4]], 87960930222080
+; CHECK-NEXT:    [[TMP6:%.*]] = inttoptr i64 [[TMP7]] to ptr
+; CHECK-NEXT:    call void @llvm.x86.avx2.maskstore.d(ptr [[TMP6]], <4 x i32> [[A1:%.*]], <4 x i32> [[TMP3]])
 ; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
-; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <4 x i32> [[TMP2]] to i128
-; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP4]], 0
-; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
-; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <4 x i32> [[TMP3]] to i128
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <4 x i32> [[TMP2]] to i128
 ; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i128 [[TMP5]], 0
-; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
-; CHECK-NEXT:    br i1 [[_MSOR3]], label [[TMP6:%.*]], label [[TMP7:%.*]], !prof [[PROF1]]
-; CHECK:       6:
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP8:%.*]], label [[TMP9:%.*]], !prof [[PROF1]]
+; CHECK:       8:
 ; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
 ; CHECK-NEXT:    unreachable
-; CHECK:       7:
-; CHECK-NEXT:    call void @llvm.x86.avx2.maskstore.d(ptr [[A0:%.*]], <4 x i32> [[A1:%.*]], <4 x i32> [[A2:%.*]])
+; CHECK:       9:
+; CHECK-NEXT:    call void @llvm.x86.avx2.maskstore.d(ptr [[A0]], <4 x i32> [[A1]], <4 x i32> [[A2:%.*]])
 ; CHECK-NEXT:    ret void
 ;
   call void @llvm.x86.avx2.maskstore.d(ptr %a0, <4 x i32> %a1, <4 x i32> %a2)
@@ -1172,23 +1179,24 @@ declare void @llvm.x86.avx2.maskstore.d(ptr, <4 x i32>, <4 x i32>) nounwind
 
 define void @test_x86_avx2_maskstore_d_256(ptr %a0, <8 x i32> %a1, <8 x i32> %a2) #0 {
 ; CHECK-LABEL: @test_x86_avx2_maskstore_d_256(
+; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 40) to ptr), align 8
 ; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
-; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 40) to ptr), align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = ptrtoint ptr [[A0:%.*]] to i64
+; CHECK-NEXT:    [[TMP7:%.*]] = xor i64 [[TMP4]], 87960930222080
+; CHECK-NEXT:    [[TMP6:%.*]] = inttoptr i64 [[TMP7]] to ptr
+; CHECK-NEXT:    call void @llvm.x86.avx2.maskstore.d.256(ptr [[TMP6]], <8 x i32> [[A1:%.*]], <8 x i32> [[TMP3]])
 ; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
-; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <8 x i32> [[TMP2]] to i256
-; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i256 [[TMP4]], 0
-; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
-; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <8 x i32> [[TMP3]] to i256
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <8 x i32> [[TMP2]] to i256
 ; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i256 [[TMP5]], 0
-; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
-; CHECK-NEXT:    br i1 [[_MSOR3]], label [[TMP6:%.*]], label [[TMP7:%.*]], !prof [[PROF1]]
-; CHECK:       6:
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP8:%.*]], label [[TMP9:%.*]], !prof [[PROF1]]
+; CHECK:       8:
 ; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
 ; CHECK-NEXT:    unreachable
-; CHECK:       7:
-; CHECK-NEXT:    call void @llvm.x86.avx2.maskstore.d.256(ptr [[A0:%.*]], <8 x i32> [[A1:%.*]], <8 x i32> [[A2:%.*]])
+; CHECK:       9:
+; CHECK-NEXT:    call void @llvm.x86.avx2.maskstore.d.256(ptr [[A0]], <8 x i32> [[A1]], <8 x i32> [[A2:%.*]])
 ; CHECK-NEXT:    ret void
 ;
   call void @llvm.x86.avx2.maskstore.d.256(ptr %a0, <8 x i32> %a1, <8 x i32> %a2)
diff --git a/llvm/test/Instrumentation/MemorySanitizer/X86/avx512-intrinsics-upgrade.ll b/llvm/test/Instrumentation/MemorySanitizer/X86/avx512-intrinsics-upgrade.ll
new file mode 100644
index 0000000000000..2ce2c076f5ad1
--- /dev/null
+++ b/llvm/test/Instrumentation/MemorySanitizer/X86/avx512-intrinsics-upgrade.ll
@@ -0,0 +1,20021 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt %s -S -mattr=+avx512f -passes=msan 2>&1 | FileCheck %s
+;
+; Forked from llvm/test/CodeGen/X86/avx512-intrinsics-upgrade.ll
+
+target datalayout = "e-m:o-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+declare i16 @llvm.x86.avx512.kunpck.bw(i16, i16) nounwind readnone
+
+define i16 @unpckbw_test(i16 %a0, i16 %a1)  #0 {
+;
+; CHECK-LABEL: @unpckbw_test(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i16, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast i16 [[TMP1]] to <16 x i1>
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast i16 [[A0:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i16 [[TMP2]] to <16 x i1>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i16 [[A1:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <16 x i1> [[TMP3]], <16 x i1> [[TMP3]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:    [[TMP7:%.*]] = shufflevector <16 x i1> [[TMP4]], <16 x i1> [[TMP4]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = shufflevector <16 x i1> [[TMP5]], <16 x i1> [[TMP5]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:    [[TMP8:%.*]] = shufflevector <16 x i1> [[TMP6]], <16 x i1> [[TMP6]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = shufflevector <8 x i1> [[_MSPROP1]], <8 x i1> [[_MSPROP]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; CHECK-NEXT:    [[TMP9:%.*]] = shufflevector <8 x i1> [[TMP8]], <8 x i1> [[TMP7]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast <16 x i1> [[_MSPROP2]] to i16
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <16 x i1> [[TMP9]] to i16
+; CHECK-NEXT:    store i16 [[TMP10]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i16 [[TMP11]]
+;
+  %res = call i16 @llvm.x86.avx512.kunpck.bw(i16 %a0, i16 %a1)
+  ret i16 %res
+}
+
+define { <16 x i32>, <16 x i32>, <16 x i32> } @test_int_x86_avx512_mask_pbroadcastd_gpr_512(i32 %x0, <16 x i32> %x1, i16 %mask)  #0 {
+;
+; CHECK-LABEL: @test_int_x86_avx512_mask_pbroadcastd_gpr_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i32, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 72) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = insertelement <16 x i32> splat (i32 -1), i32 [[TMP1]], i64 0
+; CHECK-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <16 x i32> poison, i32 [[X0:%.*]], i64 0
+; CHECK-NEXT:    [[_MSPROP5:%.*]] = shufflevector <16 x i32> [[_MSPROP]], <16 x i32> splat (i32 -1), <16 x i32> zeroinitializer
+; CHECK-NEXT:    [[DOTSPLAT:%.*]] = shufflevector <16 x i32> [[DOTSPLATINSERT]], <16 x i32> poison, <16 x i32> zeroinitializer
+; CHECK-NEXT:    [[_MSPROP6:%.*]] = insertelement <16 x i32> splat (i32 -1), i32 [[TMP1]], i64 0
+; CHECK-NEXT:    [[DOTSPLATINSERT1:%.*]] = insertelement <16 x i32> poison, i32 [[X0]], i64 0
+; CHECK-NEXT:    [[_MSPROP7:%.*]] = shufflevector <16 x i32> [[_MSPROP6]], <16 x i32> splat (i32 -1), <16 x i32> zeroinitializer
+; CHECK-NEXT:    [[DOTSPLAT2:%.*]] = shufflevector <16 x i32> [[DOTSPLATINSERT1]], <16 x i32> poison, <16 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast i16 [[TMP2]] to <16 x i1>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i16 [[MASK:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP6:%.*]] = select <16 x i1> [[TMP5]], <16 x i32> [[_MSPROP7]], <16 x i32> [[TMP3]]
+; CHECK-NEXT:    [[TMP7:%.*]] = xor <16 x i32> [[DOTSPLAT2]], [[X1:%.*]]
+; CHECK-NEXT:    [[TMP8:%.*]] = or <16 x i32> [[TMP7]], [[_MSPROP7]]
+; CHECK-NEXT:    [[TMP9:%.*]] = or <16 x i32> [[TMP8]], [[TMP3]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP4]], <16 x i32> [[TMP9]], <16 x i32> [[TMP6]]
+; CHECK-NEXT:    [[TMP10:%.*]] = select <16 x i1> [[TMP5]], <16 x i32> [[DOTSPLAT2]], <16 x i32> [[X1]]
+; CHECK-NEXT:    [[_MSPROP8:%.*]] = insertelement <16 x i32> splat (i32 -1), i32 [[TMP1]], i64 0
+; CHECK-NEXT:    [[DOTSPLATINSERT3:%.*]] = insertelement <16 x i32> poison, i32 [[X0]], i64 0
+; CHECK-NEXT:    [[_MSPROP9:%.*]] = shufflevector <16 x i32> [[_MSPROP8]], <16 x i32> splat (i32 -1), <16 x i32> zeroinitializer
+; CHECK-NEXT:    [[DOTSPLAT4:%.*]] = shufflevector <16 x i32> [[DOTSPLATINSERT3]], <16 x i32> poison, <16 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast i16 [[TMP2]] to <16 x i1>
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast i16 [[MASK]] to <16 x i1>
+; CHECK-NEXT:    [[TMP13:%.*]] = select <16 x i1> [[TMP12]], <16 x i32> [[_MSPROP9]], <16 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP14:%.*]] = xor <16 x i32> [[DOTSPLAT4]], zeroinitializer
+; CHECK-NEXT:    [[TMP15:%.*]] = or <16 x i32> [[TMP14]], [[_MSPROP9]]
+; CHECK-NEXT:    [[TMP16:%.*]] = or <16 x i32> [[TMP15]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT10:%.*]] = select <16 x i1> [[TMP11]], <16 x i32> [[TMP16]], <16 x i32> [[TMP13]]
+; CHECK-NEXT:    [[TMP17:%.*]] = select <16 x i1> [[TMP12]], <16 x i32> [[DOTSPLAT4]], <16 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP18:%.*]] = insertvalue { <16 x i32>, <16 x i32>, <16 x i32> } { <16 x i32> splat (i32 -1), <16 x i32> splat (i32 -1), <16 x i32> splat (i32 -1) }, <16 x i32> [[_MSPROP5]], 0
+; CHECK-NEXT:    [[RES3:%.*]] = insertvalue { <16 x i32>, <16 x i32>, <16 x i32> } poison, <16 x i32> [[DOTSPLAT]], 0
+; CHECK-NEXT:    [[TMP19:%.*]] = insertvalue { <16 x i32>, <16 x i32>, <16 x i32> } [[TMP18]], <16 x i32> [[_MSPROP_SELECT]], 1
+; CHECK-NEXT:    [[RES4:%.*]] = insertvalue { <16 x i32>, <16 x i32>, <16 x i32> } [[RES3]], <16 x i32> [[TMP10]], 1
+; CHECK-NEXT:    [[TMP20:%.*]] = insertvalue { <16 x i32>, <16 x i32>, <16 x i32> } [[TMP19]], <16 x i32> [[_MSPROP_SELECT10]], 2
+; CHECK-NEXT:    [[RES5:%.*]] = insertvalue { <16 x i32>, <16 x i32>, <16 x i32> } [[RES4]], <16 x i32> [[TMP17]], 2
+; CHECK-NEXT:    store { <16 x i32>, <16 x i32>, <16 x i32> } [[TMP20]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret { <16 x i32>, <16 x i32>, <16 x i32> } [[RES5]]
+;
+  %res0 = call <16 x i32> @llvm.x86.avx512.mask.pbroadcast.d.gpr.512(i32 %x0, <16 x i32> %x1, i16 -1)
+  %res1 = call <16 x i32> @llvm.x86.avx512.mask.pbroadcast.d.gpr.512(i32 %x0, <16 x i32> %x1, i16 %mask)
+  %res2 = call <16 x i32> @llvm.x86.avx512.mask.pbroadcast.d.gpr.512(i32 %x0, <16 x i32> zeroinitializer, i16 %mask)
+  %res3 = insertvalue { <16 x i32>, <16 x i32>, <16 x i32> } poison, <16 x i32> %res0, 0
+  %res4 = insertvalue { <16 x i32>, <16 x i32>, <16 x i32> }  %res3, <16 x i32> %res1, 1
+  %res5 = insertvalue { <16 x i32>, <16 x i32>, <16 x i32> }  %res4, <16 x i32> %res2, 2
+  ret { <16 x i32>, <16 x i32>, <16 x i32> } %res5
+}
+declare <16 x i32> @llvm.x86.avx512.mask.pbroadcast.d.gpr.512(i32, <16 x i32>, i16)
+
+
+define { <8 x i64>, <8 x i64>, <8 x i64> } @test_int_x86_avx512_mask_pbroadcastq_gpr_512(i64 %x0, <8 x i64> %x1, i8 %mask)  #0 {
+;
+; CHECK-LABEL: @test_int_x86_avx512_mask_pbroadcastq_gpr_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 72) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = insertelement <8 x i64> splat (i64 -1), i64 [[TMP1]], i64 0
+; CHECK-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <8 x i64> poison, i64 [[X0:%.*]], i64 0
+; CHECK-NEXT:    [[_MSPROP5:%.*]] = shufflevector <8 x i64> [[_MSPROP]], <8 x i64> splat (i64 -1), <8 x i32> zeroinitializer
+; CHECK-NEXT:    [[DOTSPLAT:%.*]] = shufflevector <8 x i64> [[DOTSPLATINSERT]], <8 x i64> poison, <8 x i32> zeroinitializer
+; CHECK-NEXT:    [[_MSPROP6:%.*]] = insertelement <8 x i64> splat (i64 -1), i64 [[TMP1]], i64 0
+; CHECK-NEXT:    [[DOTSPLATINSERT1:%.*]] = insertelement <8 x i64> poison, i64 [[X0]], i64 0
+; CHECK-NEXT:    [[_MSPROP7:%.*]] = shufflevector <8 x i64> [[_MSPROP6]], <8 x i64> splat (i64 -1), <8 x i32> zeroinitializer
+; CHECK-NEXT:    [[DOTSPLAT2:%.*]] = shufflevector <8 x i64> [[DOTSPLATINSERT1]], <8 x i64> poison, <8 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast i8 [[TMP2]] to <8 x i1>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP6:%.*]] = select <8 x i1> [[TMP5]], <8 x i64> [[_MSPROP7]], <8 x i64> [[TMP3]]
+; CHECK-NEXT:    [[TMP7:%.*]] = xor <8 x i64> [[DOTSPLAT2]], [[X1:%.*]]
+; CHECK-NEXT:    [[TMP8:%.*]] = or <8 x i64> [[TMP7]], [[_MSPROP7]]
+; CHECK-NEXT:    [[TMP9:%.*]] = or <8 x i64> [[TMP8]], [[TMP3]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP4]], <8 x i64> [[TMP9]], <8 x i64> [[TMP6]]
+; CHECK-NEXT:    [[TMP10:%.*]] = select <8 x i1> [[TMP5]], <8 x i64> [[DOTSPLAT2]], <8 x i64> [[X1]]
+; CHECK-NEXT:    [[_MSPROP8:%.*]] = insertelement <8 x i64> splat (i64 -1), i64 [[TMP1]], i64 0
+; CHECK-NEXT:    [[DOTSPLATINSERT3:%.*]] = insertelement <8 x i64> poison, i64 [[X0]], i64 0
+; CHECK-NEXT:    [[_MSPROP9:%.*]] = shufflevector <8 x i64> [[_MSPROP8]], <8 x i64> splat (i64 -1), <8 x i32> zeroinitializer
+; CHECK-NEXT:    [[DOTSPLAT4:%.*]] = shufflevector <8 x i64> [[DOTSPLATINSERT3]], <8 x i64> poison, <8 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast i8 [[TMP2]] to <8 x i1>
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast i8 [[MASK]] to <8 x i1>
+; CHECK-NEXT:    [[TMP13:%.*]] = select <8 x i1> [[TMP12]], <8 x i64> [[_MSPROP9]], <8 x i64> zeroinitializer
+; CHECK-NEXT:    [[TMP14:%.*]] = xor <8 x i64> [[DOTSPLAT4]], zeroinitializer
+; CHECK-NEXT:    [[TMP15:%.*]] = or <8 x i64> [[TMP14]], [[_MSPROP9]]
+; CHECK-NEXT:    [[TMP16:%.*]] = or <8 x i64> [[TMP15]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT10:%.*]] = select <8 x i1> [[TMP11]], <8 x i64> [[TMP16]], <8 x i64> [[TMP13]]
+; CHECK-NEXT:    [[TMP17:%.*]] = select <8 x i1> [[TMP12]], <8 x i64> [[DOTSPLAT4]], <8 x i64> zeroinitializer
+; CHECK-NEXT:    [[TMP18:%.*]] = insertvalue { <8 x i64>, <8 x i64>, <8 x i64> } { <8 x i64> splat (i64 -1), <8 x i64> splat (i64 -1), <8 x i64> splat (i64 -1) }, <8 x i64> [[_MSPROP5]], 0
+; CHECK-NEXT:    [[RES3:%.*]] = insertvalue { <8 x i64>, <8 x i64>, <8 x i64> } poison, <8 x i64> [[DOTSPLAT]], 0
+; CHECK-NEXT:    [[TMP19:%.*]] = insertvalue { <8 x i64>, <8 x i64>, <8 x i64> } [[TMP18]], <8 x i64> [[_MSPROP_SELECT]], 1
+; CHECK-NEXT:    [[RES4:%.*]] = insertvalue { <8 x i64>, <8 x i64>, <8 x i64> } [[RES3]], <8 x i64> [[TMP10]], 1
+; CHECK-NEXT:    [[TMP20:%.*]] = insertvalue { <8 x i64>, <8 x i64>, <8 x i64> } [[TMP19]], <8 x i64> [[_MSPROP_SELECT10]], 2
+; CHECK-NEXT:    [[RES5:%.*]] = insertvalue { <8 x i64>, <8 x i64>, <8 x i64> } [[RES4]], <8 x i64> [[TMP17]], 2
+; CHECK-NEXT:    store { <8 x i64>, <8 x i64>, <8 x i64> } [[TMP20]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret { <8 x i64>, <8 x i64>, <8 x i64> } [[RES5]]
+;
+  %res0 = call <8 x i64> @llvm.x86.avx512.mask.pbroadcast.q.gpr.512(i64 %x0, <8 x i64> %x1,i8 -1)
+  %res1 = call <8 x i64> @llvm.x86.avx512.mask.pbroadcast.q.gpr.512(i64 %x0, <8 x i64> %x1,i8 %mask)
+  %res2 = call <8 x i64> @llvm.x86.avx512.mask.pbroadcast.q.gpr.512(i64 %x0, <8 x i64> zeroinitializer,i8 %mask)
+  %res3 = insertvalue { <8 x i64>, <8 x i64>, <8 x i64> } poison, <8 x i64> %res0, 0
+  %res4 = insertvalue { <8 x i64>, <8 x i64>, <8 x i64> }  %res3, <8 x i64> %res1, 1
+  %res5 = insertvalue { <8 x i64>, <8 x i64>, <8 x i64> }  %res4, <8 x i64> %res2, 2
+  ret { <8 x i64>, <8 x i64>, <8 x i64> } %res5
+}
+declare <8 x i64> @llvm.x86.avx512.mask.pbroadcast.q.gpr.512(i64, <8 x i64>, i8)
+
+
+declare <16 x float> @llvm.x86.avx512.mask.broadcast.ss.ps.512(<4 x float>, <16 x float>, i16) nounwind readonly
+
+define <16 x float> @test_x86_vbroadcast_ss_ps_512(<4 x float> %a0, <16 x float> %a1, <16 x float> %extra_param)  #0 {
+; CHECK-LABEL: @test_x86_vbroadcast_ss_ps_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <4 x i32> [[TMP1]], <4 x i32> splat (i32 -1), <16 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <4 x float> [[A0:%.*]], <4 x float> poison, <16 x i32> zeroinitializer
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x float> [[TMP2]]
+;
+  %res = call <16 x float> @llvm.x86.avx512.mask.broadcast.ss.ps.512(<4 x float> %a0, <16 x float> %extra_param, i16 -1)
+  ret <16 x float> %res
+}
+
+define <16 x float> @test_x86_mask_vbroadcast_ss_ps_512(<4 x float> %a0, <16 x float> %a1, i16 %mask )  #0 {
+;
+; CHECK-LABEL: @test_x86_mask_vbroadcast_ss_ps_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 80) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <4 x i32> [[TMP1]], <4 x i32> splat (i32 -1), <16 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <4 x float> [[A0:%.*]], <4 x float> poison, <16 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i16 [[TMP2]] to <16 x i1>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i16 [[MASK:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP7:%.*]] = select <16 x i1> [[TMP6]], <16 x i32> [[_MSPROP]], <16 x i32> [[TMP3]]
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <16 x float> [[TMP4]] to <16 x i32>
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <16 x float> [[A1:%.*]] to <16 x i32>
+; CHECK-NEXT:    [[TMP10:%.*]] = xor <16 x i32> [[TMP8]], [[TMP9]]
+; CHECK-NEXT:    [[TMP11:%.*]] = or <16 x i32> [[TMP10]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP12:%.*]] = or <16 x i32> [[TMP11]], [[TMP3]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP5]], <16 x i32> [[TMP12]], <16 x i32> [[TMP7]]
+; CHECK-NEXT:    [[TMP13:%.*]] = select <16 x i1> [[TMP6]], <16 x float> [[TMP4]], <16 x float> [[A1]]
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x float> [[TMP13]]
+;
+  %res = call <16 x float> @llvm.x86.avx512.mask.broadcast.ss.ps.512(<4 x float> %a0, <16 x float> %a1, i16 %mask)
+  ret <16 x float> %res
+}
+
+define <16 x float> @test_x86_maskz_vbroadcast_ss_ps_512(<4 x float> %a0, i16 %mask )  #0 {
+;
+; CHECK-LABEL: @test_x86_maskz_vbroadcast_ss_ps_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <4 x i32> [[TMP1]], <4 x i32> splat (i32 -1), <16 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <4 x float> [[A0:%.*]], <4 x float> poison, <16 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast i16 [[TMP2]] to <16 x i1>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i16 [[MASK:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP6:%.*]] = select <16 x i1> [[TMP5]], <16 x i32> [[_MSPROP]], <16 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <16 x float> [[TMP3]] to <16 x i32>
+; CHECK-NEXT:    [[TMP8:%.*]] = xor <16 x i32> [[TMP7]], zeroinitializer
+; CHECK-NEXT:    [[TMP9:%.*]] = or <16 x i32> [[TMP8]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP10:%.*]] = or <16 x i32> [[TMP9]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP4]], <16 x i32> [[TMP10]], <16 x i32> [[TMP6]]
+; CHECK-NEXT:    [[TMP11:%.*]] = select <16 x i1> [[TMP5]], <16 x float> [[TMP3]], <16 x float> zeroinitializer
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x float> [[TMP11]]
+;
+  %res = call <16 x float> @llvm.x86.avx512.mask.broadcast.ss.ps.512(<4 x float> %a0, <16 x float> zeroinitializer, i16 %mask)
+  ret <16 x float> %res
+}
+
+declare <8 x double> @llvm.x86.avx512.mask.broadcast.sd.pd.512(<2 x double>, <8 x double>, i8) nounwind readonly
+
+define <8 x double> @test_x86_vbroadcast_sd_pd_512(<2 x double> %a0, <8 x double> %a1, <8 x double> %extra_param)  #0 {
+; CHECK-LABEL: @test_x86_vbroadcast_sd_pd_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <2 x i64> [[TMP1]], <2 x i64> splat (i64 -1), <8 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <2 x double> [[A0:%.*]], <2 x double> poison, <8 x i32> zeroinitializer
+; CHECK-NEXT:    store <8 x i64> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x double> [[TMP2]]
+;
+  %res = call <8 x double> @llvm.x86.avx512.mask.broadcast.sd.pd.512(<2 x double> %a0, <8 x double> %extra_param, i8 -1)
+  ret <8 x double> %res
+}
+
+define <8 x double> @test_x86_mask_vbroadcast_sd_pd_512(<2 x double> %a0, <8 x double> %a1, i8 %mask )  #0 {
+;
+; CHECK-LABEL: @test_x86_mask_vbroadcast_sd_pd_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 80) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <2 x i64> [[TMP1]], <2 x i64> splat (i64 -1), <8 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <2 x double> [[A0:%.*]], <2 x double> poison, <8 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i8 [[TMP2]] to <8 x i1>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP7:%.*]] = select <8 x i1> [[TMP6]], <8 x i64> [[_MSPROP]], <8 x i64> [[TMP3]]
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <8 x double> [[TMP4]] to <8 x i64>
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <8 x double> [[A1:%.*]] to <8 x i64>
+; CHECK-NEXT:    [[TMP10:%.*]] = xor <8 x i64> [[TMP8]], [[TMP9]]
+; CHECK-NEXT:    [[TMP11:%.*]] = or <8 x i64> [[TMP10]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP12:%.*]] = or <8 x i64> [[TMP11]], [[TMP3]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP5]], <8 x i64> [[TMP12]], <8 x i64> [[TMP7]]
+; CHECK-NEXT:    [[TMP13:%.*]] = select <8 x i1> [[TMP6]], <8 x double> [[TMP4]], <8 x double> [[A1]]
+; CHECK-NEXT:    store <8 x i64> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x double> [[TMP13]]
+;
+  %res = call <8 x double> @llvm.x86.avx512.mask.broadcast.sd.pd.512(<2 x double> %a0, <8 x double> %a1, i8 %mask)
+  ret <8 x double> %res
+}
+
+define <8 x double> @test_x86_maskz_vbroadcast_sd_pd_512(<2 x double> %a0, i8 %mask )  #0 {
+;
+; CHECK-LABEL: @test_x86_maskz_vbroadcast_sd_pd_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <2 x i64> [[TMP1]], <2 x i64> splat (i64 -1), <8 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <2 x double> [[A0:%.*]], <2 x double> poison, <8 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast i8 [[TMP2]] to <8 x i1>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP6:%.*]] = select <8 x i1> [[TMP5]], <8 x i64> [[_MSPROP]], <8 x i64> zeroinitializer
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <8 x double> [[TMP3]] to <8 x i64>
+; CHECK-NEXT:    [[TMP8:%.*]] = xor <8 x i64> [[TMP7]], zeroinitializer
+; CHECK-NEXT:    [[TMP9:%.*]] = or <8 x i64> [[TMP8]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP10:%.*]] = or <8 x i64> [[TMP9]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP4]], <8 x i64> [[TMP10]], <8 x i64> [[TMP6]]
+; CHECK-NEXT:    [[TMP11:%.*]] = select <8 x i1> [[TMP5]], <8 x double> [[TMP3]], <8 x double> zeroinitializer
+; CHECK-NEXT:    store <8 x i64> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x double> [[TMP11]]
+;
+  %res = call <8 x double> @llvm.x86.avx512.mask.broadcast.sd.pd.512(<2 x double> %a0, <8 x double> zeroinitializer, i8 %mask)
+  ret <8 x double> %res
+}
+
+declare <16 x i32> @llvm.x86.avx512.pbroadcastd.512(<4 x i32>, <16 x i32>, i16)
+
+define <16 x i32>@test_int_x86_avx512_pbroadcastd_512(<4 x i32> %x0, <16 x i32> %x1)  #0 {
+; CHECK-LABEL: @test_int_x86_avx512_pbroadcastd_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <4 x i32> [[TMP1]], <4 x i32> splat (i32 -1), <16 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <4 x i32> [[X0:%.*]], <4 x i32> poison, <16 x i32> zeroinitializer
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i32> [[TMP2]]
+;
+  %res = call <16 x i32> @llvm.x86.avx512.pbroadcastd.512(<4 x i32> %x0, <16 x i32> %x1, i16 -1)
+  ret <16 x i32> %res
+}
+
+define <16 x i32>@test_int_x86_avx512_mask_pbroadcastd_512(<4 x i32> %x0, <16 x i32> %x1, i16 %mask)  #0 {
+;
+; CHECK-LABEL: @test_int_x86_avx512_mask_pbroadcastd_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 80) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <4 x i32> [[TMP1]], <4 x i32> splat (i32 -1), <16 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <4 x i32> [[X0:%.*]], <4 x i32> poison, <16 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i16 [[TMP2]] to <16 x i1>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i16 [[MASK:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP7:%.*]] = select <16 x i1> [[TMP6]], <16 x i32> [[_MSPROP]], <16 x i32> [[TMP3]]
+; CHECK-NEXT:    [[TMP8:%.*]] = xor <16 x i32> [[TMP4]], [[X1:%.*]]
+; CHECK-NEXT:    [[TMP9:%.*]] = or <16 x i32> [[TMP8]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP10:%.*]] = or <16 x i32> [[TMP9]], [[TMP3]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP5]], <16 x i32> [[TMP10]], <16 x i32> [[TMP7]]
+; CHECK-NEXT:    [[TMP11:%.*]] = select <16 x i1> [[TMP6]], <16 x i32> [[TMP4]], <16 x i32> [[X1]]
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i32> [[TMP11]]
+;
+  %res = call <16 x i32> @llvm.x86.avx512.pbroadcastd.512(<4 x i32> %x0, <16 x i32> %x1, i16 %mask)
+  ret <16 x i32> %res
+}
+
+define <16 x i32>@test_int_x86_avx512_maskz_pbroadcastd_512(<4 x i32> %x0, i16 %mask)  #0 {
+;
+; CHECK-LABEL: @test_int_x86_avx512_maskz_pbroadcastd_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <4 x i32> [[TMP1]], <4 x i32> splat (i32 -1), <16 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <4 x i32> [[X0:%.*]], <4 x i32> poison, <16 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast i16 [[TMP2]] to <16 x i1>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i16 [[MASK:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP6:%.*]] = select <16 x i1> [[TMP5]], <16 x i32> [[_MSPROP]], <16 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP7:%.*]] = xor <16 x i32> [[TMP3]], zeroinitializer
+; CHECK-NEXT:    [[TMP8:%.*]] = or <16 x i32> [[TMP7]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP9:%.*]] = or <16 x i32> [[TMP8]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP4]], <16 x i32> [[TMP9]], <16 x i32> [[TMP6]]
+; CHECK-NEXT:    [[TMP10:%.*]] = select <16 x i1> [[TMP5]], <16 x i32> [[TMP3]], <16 x i32> zeroinitializer
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i32> [[TMP10]]
+;
+  %res = call <16 x i32> @llvm.x86.avx512.pbroadcastd.512(<4 x i32> %x0, <16 x i32> zeroinitializer, i16 %mask)
+  ret <16 x i32> %res
+}
+
+declare <8 x i64> @llvm.x86.avx512.pbroadcastq.512(<2 x i64>, <8 x i64>, i8)
+
+define <8 x i64>@test_int_x86_avx512_pbroadcastq_512(<2 x i64> %x0, <8 x i64> %x1)  #0 {
+; CHECK-LABEL: @test_int_x86_avx512_pbroadcastq_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <2 x i64> [[TMP1]], <2 x i64> splat (i64 -1), <8 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <2 x i64> [[X0:%.*]], <2 x i64> poison, <8 x i32> zeroinitializer
+; CHECK-NEXT:    store <8 x i64> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i64> [[TMP2]]
+;
+  %res = call <8 x i64> @llvm.x86.avx512.pbroadcastq.512(<2 x i64> %x0, <8 x i64> %x1,i8 -1)
+  ret <8 x i64> %res
+}
+
+define <8 x i64>@test_int_x86_avx512_mask_pbroadcastq_512(<2 x i64> %x0, <8 x i64> %x1, i8 %mask)  #0 {
+;
+; CHECK-LABEL: @test_int_x86_avx512_mask_pbroadcastq_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 80) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <2 x i64> [[TMP1]], <2 x i64> splat (i64 -1), <8 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <2 x i64> [[X0:%.*]], <2 x i64> poison, <8 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i8 [[TMP2]] to <8 x i1>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP7:%.*]] = select <8 x i1> [[TMP6]], <8 x i64> [[_MSPROP]], <8 x i64> [[TMP3]]
+; CHECK-NEXT:    [[TMP8:%.*]] = xor <8 x i64> [[TMP4]], [[X1:%.*]]
+; CHECK-NEXT:    [[TMP9:%.*]] = or <8 x i64> [[TMP8]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP10:%.*]] = or <8 x i64> [[TMP9]], [[TMP3]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP5]], <8 x i64> [[TMP10]], <8 x i64> [[TMP7]]
+; CHECK-NEXT:    [[TMP11:%.*]] = select <8 x i1> [[TMP6]], <8 x i64> [[TMP4]], <8 x i64> [[X1]]
+; CHECK-NEXT:    store <8 x i64> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i64> [[TMP11]]
+;
+  %res = call <8 x i64> @llvm.x86.avx512.pbroadcastq.512(<2 x i64> %x0, <8 x i64> %x1,i8 %mask)
+  ret <8 x i64> %res
+}
+
+define <8 x i64>@test_int_x86_avx512_maskz_pbroadcastq_512(<2 x i64> %x0, i8 %mask)  #0 {
+;
+; CHECK-LABEL: @test_int_x86_avx512_maskz_pbroadcastq_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <2 x i64> [[TMP1]], <2 x i64> splat (i64 -1), <8 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <2 x i64> [[X0:%.*]], <2 x i64> poison, <8 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast i8 [[TMP2]] to <8 x i1>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP6:%.*]] = select <8 x i1> [[TMP5]], <8 x i64> [[_MSPROP]], <8 x i64> zeroinitializer
+; CHECK-NEXT:    [[TMP7:%.*]] = xor <8 x i64> [[TMP3]], zeroinitializer
+; CHECK-NEXT:    [[TMP8:%.*]] = or <8 x i64> [[TMP7]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP9:%.*]] = or <8 x i64> [[TMP8]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP4]], <8 x i64> [[TMP9]], <8 x i64> [[TMP6]]
+; CHECK-NEXT:    [[TMP10:%.*]] = select <8 x i1> [[TMP5]], <8 x i64> [[TMP3]], <8 x i64> zeroinitializer
+; CHECK-NEXT:    store <8 x i64> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i64> [[TMP10]]
+;
+  %res = call <8 x i64> @llvm.x86.avx512.pbroadcastq.512(<2 x i64> %x0, <8 x i64> zeroinitializer,i8 %mask)
+  ret <8 x i64> %res
+}
+
+declare <16 x float> @llvm.x86.avx512.mask.movsldup.512(<16 x float>, <16 x float>, i16)
+
+define <16 x float>@test_int_x86_avx512_movsldup_512(<16 x float> %x0, <16 x float> %x1)  #0 {
+; CHECK-LABEL: @test_int_x86_avx512_movsldup_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <16 x i32> [[TMP1]], <16 x i32> [[TMP1]], <16 x i32> <i32 0, i32 0, i32 2, i32 2, i32 4, i32 4, i32 6, i32 6, i32 8, i32 8, i32 10, i32 10, i32 12, i32 12, i32 14, i32 14>
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <16 x float> [[X0:%.*]], <16 x float> [[X0]], <16 x i32> <i32 0, i32 0, i32 2, i32 2, i32 4, i32 4, i32 6, i32 6, i32 8, i32 8, i32 10, i32 10, i32 12, i32 12, i32 14, i32 14>
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x float> [[TMP2]]
+;
+  %res = call <16 x float> @llvm.x86.avx512.mask.movsldup.512(<16 x float> %x0, <16 x float> %x1, i16 -1)
+  ret <16 x float> %res
+}
+
+define <16 x float>@test_int_x86_avx512_mask_movsldup_512(<16 x float> %x0, <16 x float> %x1, i16 %x2)  #0 {
+;
+; CHECK-LABEL: @test_int_x86_avx512_mask_movsldup_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <16 x i32> [[TMP1]], <16 x i32> [[TMP1]], <16 x i32> <i32 0, i32 0, i32 2, i32 2, i32 4, i32 4, i32 6, i32 6, i32 8, i32 8, i32 10, i32 10, i32 12, i32 12, i32 14, i32 14>
+; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <16 x float> [[X0:%.*]], <16 x float> [[X0]], <16 x i32> <i32 0, i32 0, i32 2, i32 2, i32 4, i32 4, i32 6, i32 6, i32 8, i32 8, i32 10, i32 10, i32 12, i32 12, i32 14, i32 14>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i16 [[TMP2]] to <16 x i1>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i16 [[X2:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP7:%.*]] = select <16 x i1> [[TMP6]], <16 x i32> [[_MSPROP]], <16 x i32> [[TMP3]]
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <16 x float> [[TMP4]] to <16 x i32>
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <16 x float> [[X1:%.*]] to <16 x i32>
+; CHECK-NEXT:    [[TMP10:%.*]] = xor <16 x i32> [[TMP8]], [[TMP9]]
+; CHECK-NEXT:    [[TMP11:%.*]] = or <16 x i32> [[TMP10]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP12:%.*]] = or <16 x i32> [[TMP11]], [[TMP3]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP5]], <16 x i32> [[TMP12]], <16 x i32> [[TMP7]]
+; CHECK-NEXT:    [[TMP13:%.*]] = select <16 x i1> [[TMP6]], <16 x float> [[TMP4]], <16 x float> [[X1]]
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x float> [[TMP13]]
+;
+  %res = call <16 x float> @llvm.x86.avx512.mask.movsldup.512(<16 x float> %x0, <16 x float> %x1, i16 %x2)
+  ret <16 x float> %res
+}
+
+define <16 x float>@test_int_x86_avx512_maskz_movsldup_512(<16 x float> %x0, i16 %x2)  #0 {
+;
+; CHECK-LABEL: @test_int_x86_avx512_maskz_movsldup_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <16 x i32> [[TMP1]], <16 x i32> [[TMP1]], <16 x i32> <i32 0, i32 0, i32 2, i32 2, i32 4, i32 4, i32 6, i32 6, i32 8, i32 8, i32 10, i32 10, i32 12, i32 12, i32 14, i32 14>
+; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <16 x float> [[X0:%.*]], <16 x float> [[X0]], <16 x i32> <i32 0, i32 0, i32 2, i32 2, i32 4, i32 4, i32 6, i32 6, i32 8, i32 8, i32 10, i32 10, i32 12, i32 12, i32 14, i32 14>
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast i16 [[TMP2]] to <16 x i1>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i16 [[X2:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP6:%.*]] = select <16 x i1> [[TMP5]], <16 x i32> [[_MSPROP]], <16 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <16 x float> [[TMP3]] to <16 x i32>
+; CHECK-NEXT:    [[TMP8:%.*]] = xor <16 x i32> [[TMP7]], zeroinitializer
+; CHECK-NEXT:    [[TMP9:%.*]] = or <16 x i32> [[TMP8]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP10:%.*]] = or <16 x i32> [[TMP9]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP4]], <16 x i32> [[TMP10]], <16 x i32> [[TMP6]]
+; CHECK-NEXT:    [[TMP11:%.*]] = select <16 x i1> [[TMP5]], <16 x float> [[TMP3]], <16 x float> zeroinitializer
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x float> [[TMP11]]
+;
+  %res = call <16 x float> @llvm.x86.avx512.mask.movsldup.512(<16 x float> %x0, <16 x float> zeroinitializer, i16 %x2)
+  ret <16 x float> %res
+}
+
+declare <16 x float> @llvm.x86.avx512.mask.movshdup.512(<16 x float>, <16 x float>, i16)
+
+define <16 x float>@test_int_x86_avx512_movshdup_512(<16 x float> %x0, <16 x float> %x1)  #0 {
+; CHECK-LABEL: @test_int_x86_avx512_movshdup_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <16 x i32> [[TMP1]], <16 x i32> [[TMP1]], <16 x i32> <i32 1, i32 1, i32 3, i32 3, i32 5, i32 5, i32 7, i32 7, i32 9, i32 9, i32 11, i32 11, i32 13, i32 13, i32 15, i32 15>
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <16 x float> [[X0:%.*]], <16 x float> [[X0]], <16 x i32> <i32 1, i32 1, i32 3, i32 3, i32 5, i32 5, i32 7, i32 7, i32 9, i32 9, i32 11, i32 11, i32 13, i32 13, i32 15, i32 15>
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x float> [[TMP2]]
+;
+  %res = call <16 x float> @llvm.x86.avx512.mask.movshdup.512(<16 x float> %x0, <16 x float> %x1, i16 -1)
+  ret <16 x float> %res
+}
+
+define <16 x float>@test_int_x86_avx512_mask_movshdup_512(<16 x float> %x0, <16 x float> %x1, i16 %x2)  #0 {
+;
+; CHECK-LABEL: @test_int_x86_avx512_mask_movshdup_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <16 x i32> [[TMP1]], <16 x i32> [[TMP1]], <16 x i32> <i32 1, i32 1, i32 3, i32 3, i32 5, i32 5, i32 7, i32 7, i32 9, i32 9, i32 11, i32 11, i32 13, i32 13, i32 15, i32 15>
+; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <16 x float> [[X0:%.*]], <16 x float> [[X0]], <16 x i32> <i32 1, i32 1, i32 3, i32 3, i32 5, i32 5, i32 7, i32 7, i32 9, i32 9, i32 11, i32 11, i32 13, i32 13, i32 15, i32 15>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i16 [[TMP2]] to <16 x i1>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i16 [[X2:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP7:%.*]] = select <16 x i1> [[TMP6]], <16 x i32> [[_MSPROP]], <16 x i32> [[TMP3]]
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <16 x float> [[TMP4]] to <16 x i32>
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <16 x float> [[X1:%.*]] to <16 x i32>
+; CHECK-NEXT:    [[TMP10:%.*]] = xor <16 x i32> [[TMP8]], [[TMP9]]
+; CHECK-NEXT:    [[TMP11:%.*]] = or <16 x i32> [[TMP10]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP12:%.*]] = or <16 x i32> [[TMP11]], [[TMP3]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP5]], <16 x i32> [[TMP12]], <16 x i32> [[TMP7]]
+; CHECK-NEXT:    [[TMP13:%.*]] = select <16 x i1> [[TMP6]], <16 x float> [[TMP4]], <16 x float> [[X1]]
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x float> [[TMP13]]
+;
+  %res = call <16 x float> @llvm.x86.avx512.mask.movshdup.512(<16 x float> %x0, <16 x float> %x1, i16 %x2)
+  ret <16 x float> %res
+}
+
+define <16 x float>@test_int_x86_avx512_maskz_movshdup_512(<16 x float> %x0, i16 %x2)  #0 {
+;
+; CHECK-LABEL: @test_int_x86_avx512_maskz_movshdup_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <16 x i32> [[TMP1]], <16 x i32> [[TMP1]], <16 x i32> <i32 1, i32 1, i32 3, i32 3, i32 5, i32 5, i32 7, i32 7, i32 9, i32 9, i32 11, i32 11, i32 13, i32 13, i32 15, i32 15>
+; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <16 x float> [[X0:%.*]], <16 x float> [[X0]], <16 x i32> <i32 1, i32 1, i32 3, i32 3, i32 5, i32 5, i32 7, i32 7, i32 9, i32 9, i32 11, i32 11, i32 13, i32 13, i32 15, i32 15>
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast i16 [[TMP2]] to <16 x i1>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i16 [[X2:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP6:%.*]] = select <16 x i1> [[TMP5]], <16 x i32> [[_MSPROP]], <16 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <16 x float> [[TMP3]] to <16 x i32>
+; CHECK-NEXT:    [[TMP8:%.*]] = xor <16 x i32> [[TMP7]], zeroinitializer
+; CHECK-NEXT:    [[TMP9:%.*]] = or <16 x i32> [[TMP8]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP10:%.*]] = or <16 x i32> [[TMP9]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP4]], <16 x i32> [[TMP10]], <16 x i32> [[TMP6]]
+; CHECK-NEXT:    [[TMP11:%.*]] = select <16 x i1> [[TMP5]], <16 x float> [[TMP3]], <16 x float> zeroinitializer
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x float> [[TMP11]]
+;
+  %res = call <16 x float> @llvm.x86.avx512.mask.movshdup.512(<16 x float> %x0, <16 x float> zeroinitializer, i16 %x2)
+  ret <16 x float> %res
+}
+
+declare <8 x double> @llvm.x86.avx512.mask.movddup.512(<8 x double>, <8 x double>, i8)
+
+define <8 x double>@test_int_x86_avx512_movddup_512(<8 x double> %x0, <8 x double> %x1)  #0 {
+; CHECK-LABEL: @test_int_x86_avx512_movddup_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <8 x i64> [[TMP1]], <8 x i64> [[TMP1]], <8 x i32> <i32 0, i32 0, i32 2, i32 2, i32 4, i32 4, i32 6, i32 6>
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <8 x double> [[X0:%.*]], <8 x double> [[X0]], <8 x i32> <i32 0, i32 0, i32 2, i32 2, i32 4, i32 4, i32 6, i32 6>
+; CHECK-NEXT:    store <8 x i64> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x double> [[TMP2]]
+;
+  %res = call <8 x double> @llvm.x86.avx512.mask.movddup.512(<8 x double> %x0, <8 x double> %x1, i8 -1)
+  ret <8 x double> %res
+}
+
+define <8 x double>@test_int_x86_avx512_mask_movddup_512(<8 x double> %x0, <8 x double> %x1, i8 %x2)  #0 {
+;
+; CHECK-LABEL: @test_int_x86_avx512_mask_movddup_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <8 x i64> [[TMP1]], <8 x i64> [[TMP1]], <8 x i32> <i32 0, i32 0, i32 2, i32 2, i32 4, i32 4, i32 6, i32 6>
+; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <8 x double> [[X0:%.*]], <8 x double> [[X0]], <8 x i32> <i32 0, i32 0, i32 2, i32 2, i32 4, i32 4, i32 6, i32 6>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i8 [[TMP2]] to <8 x i1>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i8 [[X2:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP7:%.*]] = select <8 x i1> [[TMP6]], <8 x i64> [[_MSPROP]], <8 x i64> [[TMP3]]
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <8 x double> [[TMP4]] to <8 x i64>
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <8 x double> [[X1:%.*]] to <8 x i64>
+; CHECK-NEXT:    [[TMP10:%.*]] = xor <8 x i64> [[TMP8]], [[TMP9]]
+; CHECK-NEXT:    [[TMP11:%.*]] = or <8 x i64> [[TMP10]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP12:%.*]] = or <8 x i64> [[TMP11]], [[TMP3]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP5]], <8 x i64> [[TMP12]], <8 x i64> [[TMP7]]
+; CHECK-NEXT:    [[TMP13:%.*]] = select <8 x i1> [[TMP6]], <8 x double> [[TMP4]], <8 x double> [[X1]]
+; CHECK-NEXT:    store <8 x i64> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x double> [[TMP13]]
+;
+  %res = call <8 x double> @llvm.x86.avx512.mask.movddup.512(<8 x double> %x0, <8 x double> %x1, i8 %x2)
+  ret <8 x double> %res
+}
+
+define <8 x double>@test_int_x86_avx512_maskz_movddup_512(<8 x double> %x0, i8 %x2)  #0 {
+;
+; CHECK-LABEL: @test_int_x86_avx512_maskz_movddup_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <8 x i64> [[TMP1]], <8 x i64> [[TMP1]], <8 x i32> <i32 0, i32 0, i32 2, i32 2, i32 4, i32 4, i32 6, i32 6>
+; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <8 x double> [[X0:%.*]], <8 x double> [[X0]], <8 x i32> <i32 0, i32 0, i32 2, i32 2, i32 4, i32 4, i32 6, i32 6>
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast i8 [[TMP2]] to <8 x i1>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i8 [[X2:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP6:%.*]] = select <8 x i1> [[TMP5]], <8 x i64> [[_MSPROP]], <8 x i64> zeroinitializer
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <8 x double> [[TMP3]] to <8 x i64>
+; CHECK-NEXT:    [[TMP8:%.*]] = xor <8 x i64> [[TMP7]], zeroinitializer
+; CHECK-NEXT:    [[TMP9:%.*]] = or <8 x i64> [[TMP8]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP10:%.*]] = or <8 x i64> [[TMP9]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP4]], <8 x i64> [[TMP10]], <8 x i64> [[TMP6]]
+; CHECK-NEXT:    [[TMP11:%.*]] = select <8 x i1> [[TMP5]], <8 x double> [[TMP3]], <8 x double> zeroinitializer
+; CHECK-NEXT:    store <8 x i64> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x double> [[TMP11]]
+;
+  %res = call <8 x double> @llvm.x86.avx512.mask.movddup.512(<8 x double> %x0, <8 x double> zeroinitializer, i8 %x2)
+  ret <8 x double> %res
+}
+
+declare <8 x double> @llvm.x86.avx512.mask.perm.df.512(<8 x double>, i32, <8 x double>, i8)
+
+define <8 x double>@test_int_x86_avx512_perm_df_512(<8 x double> %x0, <8 x double> %x2)  #0 {
+; CHECK-LABEL: @test_int_x86_avx512_perm_df_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <8 x i64> [[TMP1]], <8 x i64> [[TMP1]], <8 x i32> <i32 3, i32 0, i32 0, i32 0, i32 7, i32 4, i32 4, i32 4>
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <8 x double> [[X0:%.*]], <8 x double> [[X0]], <8 x i32> <i32 3, i32 0, i32 0, i32 0, i32 7, i32 4, i32 4, i32 4>
+; CHECK-NEXT:    store <8 x i64> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x double> [[TMP2]]
+;
+  %res = call <8 x double> @llvm.x86.avx512.mask.perm.df.512(<8 x double> %x0, i32 3, <8 x double> %x2, i8 -1)
+  ret <8 x double> %res
+}
+
+define <8 x double>@test_int_x86_avx512_mask_perm_df_512(<8 x double> %x0, <8 x double> %x2, i8 %x3)  #0 {
+;
+; CHECK-LABEL: @test_int_x86_avx512_mask_perm_df_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <8 x i64> [[TMP1]], <8 x i64> [[TMP1]], <8 x i32> <i32 3, i32 0, i32 0, i32 0, i32 7, i32 4, i32 4, i32 4>
+; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <8 x double> [[X0:%.*]], <8 x double> [[X0]], <8 x i32> <i32 3, i32 0, i32 0, i32 0, i32 7, i32 4, i32 4, i32 4>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i8 [[TMP2]] to <8 x i1>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i8 [[X3:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP7:%.*]] = select <8 x i1> [[TMP6]], <8 x i64> [[_MSPROP]], <8 x i64> [[TMP3]]
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <8 x double> [[TMP4]] to <8 x i64>
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <8 x double> [[X2:%.*]] to <8 x i64>
+; CHECK-NEXT:    [[TMP10:%.*]] = xor <8 x i64> [[TMP8]], [[TMP9]]
+; CHECK-NEXT:    [[TMP11:%.*]] = or <8 x i64> [[TMP10]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP12:%.*]] = or <8 x i64> [[TMP11]], [[TMP3]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP5]], <8 x i64> [[TMP12]], <8 x i64> [[TMP7]]
+; CHECK-NEXT:    [[TMP13:%.*]] = select <8 x i1> [[TMP6]], <8 x double> [[TMP4]], <8 x double> [[X2]]
+; CHECK-NEXT:    store <8 x i64> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x double> [[TMP13]]
+;
+  %res = call <8 x double> @llvm.x86.avx512.mask.perm.df.512(<8 x double> %x0, i32 3, <8 x double> %x2, i8 %x3)
+  ret <8 x double> %res
+}
+
+define <8 x double>@test_int_x86_avx512_maskz_perm_df_512(<8 x double> %x0, i8 %x3)  #0 {
+;
+; CHECK-LABEL: @test_int_x86_avx512_maskz_perm_df_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <8 x i64> [[TMP1]], <8 x i64> [[TMP1]], <8 x i32> <i32 3, i32 0, i32 0, i32 0, i32 7, i32 4, i32 4, i32 4>
+; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <8 x double> [[X0:%.*]], <8 x double> [[X0]], <8 x i32> <i32 3, i32 0, i32 0, i32 0, i32 7, i32 4, i32 4, i32 4>
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast i8 [[TMP2]] to <8 x i1>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i8 [[X3:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP6:%.*]] = select <8 x i1> [[TMP5]], <8 x i64> [[_MSPROP]], <8 x i64> zeroinitializer
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <8 x double> [[TMP3]] to <8 x i64>
+; CHECK-NEXT:    [[TMP8:%.*]] = xor <8 x i64> [[TMP7]], zeroinitializer
+; CHECK-NEXT:    [[TMP9:%.*]] = or <8 x i64> [[TMP8]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP10:%.*]] = or <8 x i64> [[TMP9]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP4]], <8 x i64> [[TMP10]], <8 x i64> [[TMP6]]
+; CHECK-NEXT:    [[TMP11:%.*]] = select <8 x i1> [[TMP5]], <8 x double> [[TMP3]], <8 x double> zeroinitializer
+; CHECK-NEXT:    store <8 x i64> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x double> [[TMP11]]
+;
+  %res = call <8 x double> @llvm.x86.avx512.mask.perm.df.512(<8 x double> %x0, i32 3, <8 x double> zeroinitializer, i8 %x3)
+  ret <8 x double> %res
+}
+
+declare <8 x i64> @llvm.x86.avx512.mask.perm.di.512(<8 x i64>, i32, <8 x i64>, i8)
+
+define <8 x i64>@test_int_x86_avx512_perm_di_512(<8 x i64> %x0, i32 %x1, <8 x i64> %x2)  #0 {
+; CHECK-LABEL: @test_int_x86_avx512_perm_di_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <8 x i64> [[TMP1]], <8 x i64> [[TMP1]], <8 x i32> <i32 3, i32 0, i32 0, i32 0, i32 7, i32 4, i32 4, i32 4>
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <8 x i64> [[X0:%.*]], <8 x i64> [[X0]], <8 x i32> <i32 3, i32 0, i32 0, i32 0, i32 7, i32 4, i32 4, i32 4>
+; CHECK-NEXT:    store <8 x i64> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i64> [[TMP2]]
+;
+  %res = call <8 x i64> @llvm.x86.avx512.mask.perm.di.512(<8 x i64> %x0, i32 3, <8 x i64> %x2, i8 -1)
+  ret <8 x i64> %res
+}
+
+define <8 x i64>@test_int_x86_avx512_mask_perm_di_512(<8 x i64> %x0, i32 %x1, <8 x i64> %x2, i8 %x3)  #0 {
+;
+; CHECK-LABEL: @test_int_x86_avx512_mask_perm_di_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 136) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 72) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <8 x i64> [[TMP1]], <8 x i64> [[TMP1]], <8 x i32> <i32 3, i32 0, i32 0, i32 0, i32 7, i32 4, i32 4, i32 4>
+; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <8 x i64> [[X0:%.*]], <8 x i64> [[X0]], <8 x i32> <i32 3, i32 0, i32 0, i32 0, i32 7, i32 4, i32 4, i32 4>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i8 [[TMP2]] to <8 x i1>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i8 [[X3:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP7:%.*]] = select <8 x i1> [[TMP6]], <8 x i64> [[_MSPROP]], <8 x i64> [[TMP3]]
+; CHECK-NEXT:    [[TMP8:%.*]] = xor <8 x i64> [[TMP4]], [[X2:%.*]]
+; CHECK-NEXT:    [[TMP9:%.*]] = or <8 x i64> [[TMP8]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP10:%.*]] = or <8 x i64> [[TMP9]], [[TMP3]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP5]], <8 x i64> [[TMP10]], <8 x i64> [[TMP7]]
+; CHECK-NEXT:    [[TMP11:%.*]] = select <8 x i1> [[TMP6]], <8 x i64> [[TMP4]], <8 x i64> [[X2]]
+; CHECK-NEXT:    store <8 x i64> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i64> [[TMP11]]
+;
+  %res = call <8 x i64> @llvm.x86.avx512.mask.perm.di.512(<8 x i64> %x0, i32 3, <8 x i64> %x2, i8 %x3)
+  ret <8 x i64> %res
+}
+
+define <8 x i64>@test_int_x86_avx512_maskz_perm_di_512(<8 x i64> %x0, i32 %x1, i8 %x3)  #0 {
+;
+; CHECK-LABEL: @test_int_x86_avx512_maskz_perm_di_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 72) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <8 x i64> [[TMP1]], <8 x i64> [[TMP1]], <8 x i32> <i32 3, i32 0, i32 0, i32 0, i32 7, i32 4, i32 4, i32 4>
+; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <8 x i64> [[X0:%.*]], <8 x i64> [[X0]], <8 x i32> <i32 3, i32 0, i32 0, i32 0, i32 7, i32 4, i32 4, i32 4>
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast i8 [[TMP2]] to <8 x i1>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i8 [[X3:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP6:%.*]] = select <8 x i1> [[TMP5]], <8 x i64> [[_MSPROP]], <8 x i64> zeroinitializer
+; CHECK-NEXT:    [[TMP7:%.*]] = xor <8 x i64> [[TMP3]], zeroinitializer
+; CHECK-NEXT:    [[TMP8:%.*]] = or <8 x i64> [[TMP7]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP9:%.*]] = or <8 x i64> [[TMP8]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP4]], <8 x i64> [[TMP9]], <8 x i64> [[TMP6]]
+; CHECK-NEXT:    [[TMP10:%.*]] = select <8 x i1> [[TMP5]], <8 x i64> [[TMP3]], <8 x i64> zeroinitializer
+; CHECK-NEXT:    store <8 x i64> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i64> [[TMP10]]
+;
+  %res = call <8 x i64> @llvm.x86.avx512.mask.perm.di.512(<8 x i64> %x0, i32 3, <8 x i64> zeroinitializer, i8 %x3)
+  ret <8 x i64> %res
+}
+
+define void @test_store1(<16 x float> %data, ptr %ptr, ptr %ptr2, i16 %mask)  #0 {
+;
+; CHECK-LABEL: @test_store1(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 80) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 72) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i16 [[TMP1]] to <16 x i1>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i16 [[MASK:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP7:%.*]] = ptrtoint ptr [[PTR:%.*]] to i64
+; CHECK-NEXT:    [[TMP8:%.*]] = xor i64 [[TMP7]], 87960930222080
+; CHECK-NEXT:    [[TMP9:%.*]] = inttoptr i64 [[TMP8]] to ptr
+; CHECK-NEXT:    call void @llvm.masked.store.v16i32.p0(<16 x i32> [[TMP2]], ptr [[TMP9]], i32 1, <16 x i1> [[TMP6]])
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP3]], 0
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast <16 x i1> [[TMP5]] to i16
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i16 [[TMP10]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP11:%.*]], label [[TMP12:%.*]], !prof [[PROF1:![0-9]+]]
+; CHECK:       11:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8:[0-9]+]]
+; CHECK-NEXT:    unreachable
+; CHECK:       12:
+; CHECK-NEXT:    call void @llvm.masked.store.v16f32.p0(<16 x float> [[DATA:%.*]], ptr [[PTR]], i32 1, <16 x i1> [[TMP6]])
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i64 [[TMP4]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP2]], label [[TMP13:%.*]], label [[TMP14:%.*]], !prof [[PROF1]]
+; CHECK:       13:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       14:
+; CHECK-NEXT:    [[TMP15:%.*]] = ptrtoint ptr [[PTR2:%.*]] to i64
+; CHECK-NEXT:    [[TMP16:%.*]] = xor i64 [[TMP15]], 87960930222080
+; CHECK-NEXT:    [[TMP17:%.*]] = inttoptr i64 [[TMP16]] to ptr
+; CHECK-NEXT:    store <16 x i32> [[TMP2]], ptr [[TMP17]], align 1
+; CHECK-NEXT:    store <16 x float> [[DATA]], ptr [[PTR2]], align 1
+; CHECK-NEXT:    ret void
+;
+  call void @llvm.x86.avx512.mask.storeu.ps.512(ptr %ptr, <16 x float> %data, i16 %mask)
+  call void @llvm.x86.avx512.mask.storeu.ps.512(ptr %ptr2, <16 x float> %data, i16 -1)
+  ret void
+}
+
+declare void @llvm.x86.avx512.mask.storeu.ps.512(ptr, <16 x float>, i16 )
+
+define void @test_store2(<8 x double> %data, ptr %ptr, ptr %ptr2, i8 %mask)  #0 {
+;
+; CHECK-LABEL: @test_store2(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 80) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 72) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i8 [[TMP1]] to <8 x i1>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP7:%.*]] = ptrtoint ptr [[PTR:%.*]] to i64
+; CHECK-NEXT:    [[TMP8:%.*]] = xor i64 [[TMP7]], 87960930222080
+; CHECK-NEXT:    [[TMP9:%.*]] = inttoptr i64 [[TMP8]] to ptr
+; CHECK-NEXT:    call void @llvm.masked.store.v8i64.p0(<8 x i64> [[TMP2]], ptr [[TMP9]], i32 1, <8 x i1> [[TMP6]])
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP3]], 0
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast <8 x i1> [[TMP5]] to i8
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i8 [[TMP10]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP11:%.*]], label [[TMP12:%.*]], !prof [[PROF1]]
+; CHECK:       11:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       12:
+; CHECK-NEXT:    call void @llvm.masked.store.v8f64.p0(<8 x double> [[DATA:%.*]], ptr [[PTR]], i32 1, <8 x i1> [[TMP6]])
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i64 [[TMP4]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP2]], label [[TMP13:%.*]], label [[TMP14:%.*]], !prof [[PROF1]]
+; CHECK:       13:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       14:
+; CHECK-NEXT:    [[TMP15:%.*]] = ptrtoint ptr [[PTR2:%.*]] to i64
+; CHECK-NEXT:    [[TMP16:%.*]] = xor i64 [[TMP15]], 87960930222080
+; CHECK-NEXT:    [[TMP17:%.*]] = inttoptr i64 [[TMP16]] to ptr
+; CHECK-NEXT:    store <8 x i64> [[TMP2]], ptr [[TMP17]], align 1
+; CHECK-NEXT:    store <8 x double> [[DATA]], ptr [[PTR2]], align 1
+; CHECK-NEXT:    ret void
+;
+  call void @llvm.x86.avx512.mask.storeu.pd.512(ptr %ptr, <8 x double> %data, i8 %mask)
+  call void @llvm.x86.avx512.mask.storeu.pd.512(ptr %ptr2, <8 x double> %data, i8 -1)
+  ret void
+}
+
+declare void @llvm.x86.avx512.mask.storeu.pd.512(ptr, <8 x double>, i8)
+
+define void @test_mask_store_aligned_ps(<16 x float> %data, ptr %ptr, ptr %ptr2, i16 %mask)  #0 {
+;
+; CHECK-LABEL: @test_mask_store_aligned_ps(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 80) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 72) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i16 [[TMP1]] to <16 x i1>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i16 [[MASK:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP7:%.*]] = ptrtoint ptr [[PTR:%.*]] to i64
+; CHECK-NEXT:    [[TMP8:%.*]] = xor i64 [[TMP7]], 87960930222080
+; CHECK-NEXT:    [[TMP9:%.*]] = inttoptr i64 [[TMP8]] to ptr
+; CHECK-NEXT:    call void @llvm.masked.store.v16i32.p0(<16 x i32> [[TMP2]], ptr [[TMP9]], i32 64, <16 x i1> [[TMP6]])
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP3]], 0
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast <16 x i1> [[TMP5]] to i16
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i16 [[TMP10]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP11:%.*]], label [[TMP12:%.*]], !prof [[PROF1]]
+; CHECK:       11:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       12:
+; CHECK-NEXT:    call void @llvm.masked.store.v16f32.p0(<16 x float> [[DATA:%.*]], ptr [[PTR]], i32 64, <16 x i1> [[TMP6]])
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i64 [[TMP4]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP2]], label [[TMP13:%.*]], label [[TMP14:%.*]], !prof [[PROF1]]
+; CHECK:       13:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       14:
+; CHECK-NEXT:    [[TMP15:%.*]] = ptrtoint ptr [[PTR2:%.*]] to i64
+; CHECK-NEXT:    [[TMP16:%.*]] = xor i64 [[TMP15]], 87960930222080
+; CHECK-NEXT:    [[TMP17:%.*]] = inttoptr i64 [[TMP16]] to ptr
+; CHECK-NEXT:    store <16 x i32> [[TMP2]], ptr [[TMP17]], align 64
+; CHECK-NEXT:    store <16 x float> [[DATA]], ptr [[PTR2]], align 64
+; CHECK-NEXT:    ret void
+;
+  call void @llvm.x86.avx512.mask.store.ps.512(ptr %ptr, <16 x float> %data, i16 %mask)
+  call void @llvm.x86.avx512.mask.store.ps.512(ptr %ptr2, <16 x float> %data, i16 -1)
+  ret void
+}
+
+declare void @llvm.x86.avx512.mask.store.ps.512(ptr, <16 x float>, i16 )
+
+define void @test_mask_store_aligned_pd(<8 x double> %data, ptr %ptr, ptr %ptr2, i8 %mask)  #0 {
+;
+; CHECK-LABEL: @test_mask_store_aligned_pd(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 80) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 72) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i8 [[TMP1]] to <8 x i1>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP7:%.*]] = ptrtoint ptr [[PTR:%.*]] to i64
+; CHECK-NEXT:    [[TMP8:%.*]] = xor i64 [[TMP7]], 87960930222080
+; CHECK-NEXT:    [[TMP9:%.*]] = inttoptr i64 [[TMP8]] to ptr
+; CHECK-NEXT:    call void @llvm.masked.store.v8i64.p0(<8 x i64> [[TMP2]], ptr [[TMP9]], i32 64, <8 x i1> [[TMP6]])
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP3]], 0
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast <8 x i1> [[TMP5]] to i8
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i8 [[TMP10]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP11:%.*]], label [[TMP12:%.*]], !prof [[PROF1]]
+; CHECK:       11:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       12:
+; CHECK-NEXT:    call void @llvm.masked.store.v8f64.p0(<8 x double> [[DATA:%.*]], ptr [[PTR]], i32 64, <8 x i1> [[TMP6]])
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i64 [[TMP4]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP2]], label [[TMP13:%.*]], label [[TMP14:%.*]], !prof [[PROF1]]
+; CHECK:       13:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       14:
+; CHECK-NEXT:    [[TMP15:%.*]] = ptrtoint ptr [[PTR2:%.*]] to i64
+; CHECK-NEXT:    [[TMP16:%.*]] = xor i64 [[TMP15]], 87960930222080
+; CHECK-NEXT:    [[TMP17:%.*]] = inttoptr i64 [[TMP16]] to ptr
+; CHECK-NEXT:    store <8 x i64> [[TMP2]], ptr [[TMP17]], align 64
+; CHECK-NEXT:    store <8 x double> [[DATA]], ptr [[PTR2]], align 64
+; CHECK-NEXT:    ret void
+;
+  call void @llvm.x86.avx512.mask.store.pd.512(ptr %ptr, <8 x double> %data, i8 %mask)
+  call void @llvm.x86.avx512.mask.store.pd.512(ptr %ptr2, <8 x double> %data, i8 -1)
+  ret void
+}
+
+declare void @llvm.x86.avx512.mask.store.pd.512(ptr, <8 x double>, i8)
+
+define void@test_int_x86_avx512_mask_storeu_q_512(ptr %ptr1, ptr %ptr2, <8 x i64> %x1, i8 %x2)  #0 {
+;
+; CHECK-LABEL: @test_int_x86_avx512_mask_storeu_q_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 80) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i8 [[TMP1]] to <8 x i1>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i8 [[X2:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP7:%.*]] = ptrtoint ptr [[PTR1:%.*]] to i64
+; CHECK-NEXT:    [[TMP8:%.*]] = xor i64 [[TMP7]], 87960930222080
+; CHECK-NEXT:    [[TMP9:%.*]] = inttoptr i64 [[TMP8]] to ptr
+; CHECK-NEXT:    call void @llvm.masked.store.v8i64.p0(<8 x i64> [[TMP2]], ptr [[TMP9]], i32 1, <8 x i1> [[TMP6]])
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP3]], 0
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast <8 x i1> [[TMP5]] to i8
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i8 [[TMP10]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP11:%.*]], label [[TMP12:%.*]], !prof [[PROF1]]
+; CHECK:       11:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       12:
+; CHECK-NEXT:    call void @llvm.masked.store.v8i64.p0(<8 x i64> [[X1:%.*]], ptr [[PTR1]], i32 1, <8 x i1> [[TMP6]])
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i64 [[TMP4]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP2]], label [[TMP13:%.*]], label [[TMP14:%.*]], !prof [[PROF1]]
+; CHECK:       13:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       14:
+; CHECK-NEXT:    [[TMP15:%.*]] = ptrtoint ptr [[PTR2:%.*]] to i64
+; CHECK-NEXT:    [[TMP16:%.*]] = xor i64 [[TMP15]], 87960930222080
+; CHECK-NEXT:    [[TMP17:%.*]] = inttoptr i64 [[TMP16]] to ptr
+; CHECK-NEXT:    store <8 x i64> [[TMP2]], ptr [[TMP17]], align 1
+; CHECK-NEXT:    store <8 x i64> [[X1]], ptr [[PTR2]], align 1
+; CHECK-NEXT:    ret void
+;
+  call void @llvm.x86.avx512.mask.storeu.q.512(ptr %ptr1, <8 x i64> %x1, i8 %x2)
+  call void @llvm.x86.avx512.mask.storeu.q.512(ptr %ptr2, <8 x i64> %x1, i8 -1)
+  ret void
+}
+
+declare void @llvm.x86.avx512.mask.storeu.q.512(ptr, <8 x i64>, i8)
+
+define void@test_int_x86_avx512_mask_storeu_d_512(ptr %ptr1, ptr %ptr2, <16 x i32> %x1, i16 %x2)  #0 {
+;
+; CHECK-LABEL: @test_int_x86_avx512_mask_storeu_d_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 80) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i16 [[TMP1]] to <16 x i1>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i16 [[X2:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP7:%.*]] = ptrtoint ptr [[PTR1:%.*]] to i64
+; CHECK-NEXT:    [[TMP8:%.*]] = xor i64 [[TMP7]], 87960930222080
+; CHECK-NEXT:    [[TMP9:%.*]] = inttoptr i64 [[TMP8]] to ptr
+; CHECK-NEXT:    call void @llvm.masked.store.v16i32.p0(<16 x i32> [[TMP2]], ptr [[TMP9]], i32 1, <16 x i1> [[TMP6]])
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP3]], 0
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast <16 x i1> [[TMP5]] to i16
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i16 [[TMP10]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP11:%.*]], label [[TMP12:%.*]], !prof [[PROF1]]
+; CHECK:       11:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       12:
+; CHECK-NEXT:    call void @llvm.masked.store.v16i32.p0(<16 x i32> [[X1:%.*]], ptr [[PTR1]], i32 1, <16 x i1> [[TMP6]])
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i64 [[TMP4]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP2]], label [[TMP13:%.*]], label [[TMP14:%.*]], !prof [[PROF1]]
+; CHECK:       13:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       14:
+; CHECK-NEXT:    [[TMP15:%.*]] = ptrtoint ptr [[PTR2:%.*]] to i64
+; CHECK-NEXT:    [[TMP16:%.*]] = xor i64 [[TMP15]], 87960930222080
+; CHECK-NEXT:    [[TMP17:%.*]] = inttoptr i64 [[TMP16]] to ptr
+; CHECK-NEXT:    store <16 x i32> [[TMP2]], ptr [[TMP17]], align 1
+; CHECK-NEXT:    store <16 x i32> [[X1]], ptr [[PTR2]], align 1
+; CHECK-NEXT:    ret void
+;
+  call void @llvm.x86.avx512.mask.storeu.d.512(ptr %ptr1, <16 x i32> %x1, i16 %x2)
+  call void @llvm.x86.avx512.mask.storeu.d.512(ptr %ptr2, <16 x i32> %x1, i16 -1)
+  ret void
+}
+
+declare void @llvm.x86.avx512.mask.storeu.d.512(ptr, <16 x i32>, i16)
+
+define void@test_int_x86_avx512_mask_store_q_512(ptr %ptr1, ptr %ptr2, <8 x i64> %x1, i8 %x2)  #0 {
+;
+; CHECK-LABEL: @test_int_x86_avx512_mask_store_q_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 80) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i8 [[TMP1]] to <8 x i1>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i8 [[X2:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP7:%.*]] = ptrtoint ptr [[PTR1:%.*]] to i64
+; CHECK-NEXT:    [[TMP8:%.*]] = xor i64 [[TMP7]], 87960930222080
+; CHECK-NEXT:    [[TMP9:%.*]] = inttoptr i64 [[TMP8]] to ptr
+; CHECK-NEXT:    call void @llvm.masked.store.v8i64.p0(<8 x i64> [[TMP2]], ptr [[TMP9]], i32 64, <8 x i1> [[TMP6]])
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP3]], 0
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast <8 x i1> [[TMP5]] to i8
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i8 [[TMP10]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP11:%.*]], label [[TMP12:%.*]], !prof [[PROF1]]
+; CHECK:       11:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       12:
+; CHECK-NEXT:    call void @llvm.masked.store.v8i64.p0(<8 x i64> [[X1:%.*]], ptr [[PTR1]], i32 64, <8 x i1> [[TMP6]])
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i64 [[TMP4]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP2]], label [[TMP13:%.*]], label [[TMP14:%.*]], !prof [[PROF1]]
+; CHECK:       13:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       14:
+; CHECK-NEXT:    [[TMP15:%.*]] = ptrtoint ptr [[PTR2:%.*]] to i64
+; CHECK-NEXT:    [[TMP16:%.*]] = xor i64 [[TMP15]], 87960930222080
+; CHECK-NEXT:    [[TMP17:%.*]] = inttoptr i64 [[TMP16]] to ptr
+; CHECK-NEXT:    store <8 x i64> [[TMP2]], ptr [[TMP17]], align 64
+; CHECK-NEXT:    store <8 x i64> [[X1]], ptr [[PTR2]], align 64
+; CHECK-NEXT:    ret void
+;
+  call void @llvm.x86.avx512.mask.store.q.512(ptr %ptr1, <8 x i64> %x1, i8 %x2)
+  call void @llvm.x86.avx512.mask.store.q.512(ptr %ptr2, <8 x i64> %x1, i8 -1)
+  ret void
+}
+
+declare void @llvm.x86.avx512.mask.store.q.512(ptr, <8 x i64>, i8)
+
+define void@test_int_x86_avx512_mask_store_d_512(ptr %ptr1, ptr %ptr2, <16 x i32> %x1, i16 %x2)  #0 {
+;
+; CHECK-LABEL: @test_int_x86_avx512_mask_store_d_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 80) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i16 [[TMP1]] to <16 x i1>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i16 [[X2:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP7:%.*]] = ptrtoint ptr [[PTR1:%.*]] to i64
+; CHECK-NEXT:    [[TMP8:%.*]] = xor i64 [[TMP7]], 87960930222080
+; CHECK-NEXT:    [[TMP9:%.*]] = inttoptr i64 [[TMP8]] to ptr
+; CHECK-NEXT:    call void @llvm.masked.store.v16i32.p0(<16 x i32> [[TMP2]], ptr [[TMP9]], i32 64, <16 x i1> [[TMP6]])
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP3]], 0
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast <16 x i1> [[TMP5]] to i16
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i16 [[TMP10]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP11:%.*]], label [[TMP12:%.*]], !prof [[PROF1]]
+; CHECK:       11:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       12:
+; CHECK-NEXT:    call void @llvm.masked.store.v16i32.p0(<16 x i32> [[X1:%.*]], ptr [[PTR1]], i32 64, <16 x i1> [[TMP6]])
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i64 [[TMP4]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP2]], label [[TMP13:%.*]], label [[TMP14:%.*]], !prof [[PROF1]]
+; CHECK:       13:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       14:
+; CHECK-NEXT:    [[TMP15:%.*]] = ptrtoint ptr [[PTR2:%.*]] to i64
+; CHECK-NEXT:    [[TMP16:%.*]] = xor i64 [[TMP15]], 87960930222080
+; CHECK-NEXT:    [[TMP17:%.*]] = inttoptr i64 [[TMP16]] to ptr
+; CHECK-NEXT:    store <16 x i32> [[TMP2]], ptr [[TMP17]], align 64
+; CHECK-NEXT:    store <16 x i32> [[X1]], ptr [[PTR2]], align 64
+; CHECK-NEXT:    ret void
+;
+  call void @llvm.x86.avx512.mask.store.d.512(ptr %ptr1, <16 x i32> %x1, i16 %x2)
+  call void @llvm.x86.avx512.mask.store.d.512(ptr %ptr2, <16 x i32> %x1, i16 -1)
+  ret void
+}
+
+declare void @llvm.x86.avx512.mask.store.d.512(ptr, <16 x i32>, i16)
+
+define <16 x float> @test_mask_load_aligned_ps(<16 x float> %data, ptr %ptr, i16 %mask)  #0 {
+;
+; CHECK-LABEL: @test_mask_load_aligned_ps(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 72) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP3:%.*]], label [[TMP4:%.*]], !prof [[PROF1]]
+; CHECK:       3:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       4:
+; CHECK-NEXT:    [[TMP5:%.*]] = load <16 x float>, ptr [[PTR:%.*]], align 64
+; CHECK-NEXT:    [[TMP6:%.*]] = ptrtoint ptr [[PTR]] to i64
+; CHECK-NEXT:    [[TMP7:%.*]] = xor i64 [[TMP6]], 87960930222080
+; CHECK-NEXT:    [[TMP8:%.*]] = inttoptr i64 [[TMP7]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <16 x i32>, ptr [[TMP8]], align 64
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast i16 [[TMP2]] to <16 x i1>
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i16 [[MASK:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP11:%.*]] = ptrtoint ptr [[PTR]] to i64
+; CHECK-NEXT:    [[TMP12:%.*]] = xor i64 [[TMP11]], 87960930222080
+; CHECK-NEXT:    [[TMP13:%.*]] = inttoptr i64 [[TMP12]] to ptr
+; CHECK-NEXT:    [[_MSMASKEDLD:%.*]] = call <16 x i32> @llvm.masked.load.v16i32.p0(ptr [[TMP13]], i32 64, <16 x i1> [[TMP10]], <16 x i32> [[_MSLD]])
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast <16 x i1> [[TMP9]] to i16
+; CHECK-NEXT:    [[_MSCMP3:%.*]] = icmp ne i16 [[TMP14]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP2]], [[_MSCMP3]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP15:%.*]], label [[TMP16:%.*]], !prof [[PROF1]]
+; CHECK:       15:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       16:
+; CHECK-NEXT:    [[TMP17:%.*]] = call <16 x float> @llvm.masked.load.v16f32.p0(ptr [[PTR]], i32 64, <16 x i1> [[TMP10]], <16 x float> [[TMP5]])
+; CHECK-NEXT:    [[TMP18:%.*]] = bitcast i16 [[TMP2]] to <16 x i1>
+; CHECK-NEXT:    [[TMP19:%.*]] = bitcast i16 [[MASK]] to <16 x i1>
+; CHECK-NEXT:    [[TMP20:%.*]] = ptrtoint ptr [[PTR]] to i64
+; CHECK-NEXT:    [[TMP21:%.*]] = xor i64 [[TMP20]], 87960930222080
+; CHECK-NEXT:    [[TMP22:%.*]] = inttoptr i64 [[TMP21]] to ptr
+; CHECK-NEXT:    [[_MSMASKEDLD1:%.*]] = call <16 x i32> @llvm.masked.load.v16i32.p0(ptr [[TMP22]], i32 64, <16 x i1> [[TMP19]], <16 x i32> zeroinitializer)
+; CHECK-NEXT:    [[_MSCMP4:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    [[TMP23:%.*]] = bitcast <16 x i1> [[TMP18]] to i16
+; CHECK-NEXT:    [[_MSCMP5:%.*]] = icmp ne i16 [[TMP23]], 0
+; CHECK-NEXT:    [[_MSOR6:%.*]] = or i1 [[_MSCMP4]], [[_MSCMP5]]
+; CHECK-NEXT:    br i1 [[_MSOR6]], label [[TMP24:%.*]], label [[TMP25:%.*]], !prof [[PROF1]]
+; CHECK:       24:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       25:
+; CHECK-NEXT:    [[TMP26:%.*]] = call <16 x float> @llvm.masked.load.v16f32.p0(ptr [[PTR]], i32 64, <16 x i1> [[TMP19]], <16 x float> zeroinitializer)
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <16 x i32> [[_MSMASKEDLD1]], [[_MSMASKEDLD]]
+; CHECK-NEXT:    [[RES4:%.*]] = fadd <16 x float> [[TMP26]], [[TMP17]]
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x float> [[RES4]]
+;
+  %res = call <16 x float> @llvm.x86.avx512.mask.load.ps.512(ptr %ptr, <16 x float> zeroinitializer, i16 -1)
+  %res1 = call <16 x float> @llvm.x86.avx512.mask.load.ps.512(ptr %ptr, <16 x float> %res, i16 %mask)
+  %res2 = call <16 x float> @llvm.x86.avx512.mask.load.ps.512(ptr %ptr, <16 x float> zeroinitializer, i16 %mask)
+  %res4 = fadd <16 x float> %res2, %res1
+  ret <16 x float> %res4
+}
+
+declare <16 x float> @llvm.x86.avx512.mask.load.ps.512(ptr, <16 x float>, i16)
+
+define <16 x float> @test_mask_load_unaligned_ps(<16 x float> %data, ptr %ptr, i16 %mask)  #0 {
+;
+; CHECK-LABEL: @test_mask_load_unaligned_ps(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 72) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP3:%.*]], label [[TMP4:%.*]], !prof [[PROF1]]
+; CHECK:       3:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       4:
+; CHECK-NEXT:    [[TMP5:%.*]] = load <16 x float>, ptr [[PTR:%.*]], align 1
+; CHECK-NEXT:    [[TMP6:%.*]] = ptrtoint ptr [[PTR]] to i64
+; CHECK-NEXT:    [[TMP7:%.*]] = xor i64 [[TMP6]], 87960930222080
+; CHECK-NEXT:    [[TMP8:%.*]] = inttoptr i64 [[TMP7]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <16 x i32>, ptr [[TMP8]], align 1
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast i16 [[TMP2]] to <16 x i1>
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i16 [[MASK:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP11:%.*]] = ptrtoint ptr [[PTR]] to i64
+; CHECK-NEXT:    [[TMP12:%.*]] = xor i64 [[TMP11]], 87960930222080
+; CHECK-NEXT:    [[TMP13:%.*]] = inttoptr i64 [[TMP12]] to ptr
+; CHECK-NEXT:    [[_MSMASKEDLD:%.*]] = call <16 x i32> @llvm.masked.load.v16i32.p0(ptr [[TMP13]], i32 1, <16 x i1> [[TMP10]], <16 x i32> [[_MSLD]])
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast <16 x i1> [[TMP9]] to i16
+; CHECK-NEXT:    [[_MSCMP3:%.*]] = icmp ne i16 [[TMP14]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP2]], [[_MSCMP3]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP15:%.*]], label [[TMP16:%.*]], !prof [[PROF1]]
+; CHECK:       15:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       16:
+; CHECK-NEXT:    [[TMP17:%.*]] = call <16 x float> @llvm.masked.load.v16f32.p0(ptr [[PTR]], i32 1, <16 x i1> [[TMP10]], <16 x float> [[TMP5]])
+; CHECK-NEXT:    [[TMP18:%.*]] = bitcast i16 [[TMP2]] to <16 x i1>
+; CHECK-NEXT:    [[TMP19:%.*]] = bitcast i16 [[MASK]] to <16 x i1>
+; CHECK-NEXT:    [[TMP20:%.*]] = ptrtoint ptr [[PTR]] to i64
+; CHECK-NEXT:    [[TMP21:%.*]] = xor i64 [[TMP20]], 87960930222080
+; CHECK-NEXT:    [[TMP22:%.*]] = inttoptr i64 [[TMP21]] to ptr
+; CHECK-NEXT:    [[_MSMASKEDLD1:%.*]] = call <16 x i32> @llvm.masked.load.v16i32.p0(ptr [[TMP22]], i32 1, <16 x i1> [[TMP19]], <16 x i32> zeroinitializer)
+; CHECK-NEXT:    [[_MSCMP4:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    [[TMP23:%.*]] = bitcast <16 x i1> [[TMP18]] to i16
+; CHECK-NEXT:    [[_MSCMP5:%.*]] = icmp ne i16 [[TMP23]], 0
+; CHECK-NEXT:    [[_MSOR6:%.*]] = or i1 [[_MSCMP4]], [[_MSCMP5]]
+; CHECK-NEXT:    br i1 [[_MSOR6]], label [[TMP24:%.*]], label [[TMP25:%.*]], !prof [[PROF1]]
+; CHECK:       24:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       25:
+; CHECK-NEXT:    [[TMP26:%.*]] = call <16 x float> @llvm.masked.load.v16f32.p0(ptr [[PTR]], i32 1, <16 x i1> [[TMP19]], <16 x float> zeroinitializer)
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <16 x i32> [[_MSMASKEDLD1]], [[_MSMASKEDLD]]
+; CHECK-NEXT:    [[RES4:%.*]] = fadd <16 x float> [[TMP26]], [[TMP17]]
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x float> [[RES4]]
+;
+  %res = call <16 x float> @llvm.x86.avx512.mask.loadu.ps.512(ptr %ptr, <16 x float> zeroinitializer, i16 -1)
+  %res1 = call <16 x float> @llvm.x86.avx512.mask.loadu.ps.512(ptr %ptr, <16 x float> %res, i16 %mask)
+  %res2 = call <16 x float> @llvm.x86.avx512.mask.loadu.ps.512(ptr %ptr, <16 x float> zeroinitializer, i16 %mask)
+  %res4 = fadd <16 x float> %res2, %res1
+  ret <16 x float> %res4
+}
+
+declare <16 x float> @llvm.x86.avx512.mask.loadu.ps.512(ptr, <16 x float>, i16)
+
+define <8 x double> @test_mask_load_aligned_pd(<8 x double> %data, ptr %ptr, i8 %mask)  #0 {
+;
+; CHECK-LABEL: @test_mask_load_aligned_pd(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 72) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP3:%.*]], label [[TMP4:%.*]], !prof [[PROF1]]
+; CHECK:       3:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       4:
+; CHECK-NEXT:    [[TMP5:%.*]] = load <8 x double>, ptr [[PTR:%.*]], align 64
+; CHECK-NEXT:    [[TMP6:%.*]] = ptrtoint ptr [[PTR]] to i64
+; CHECK-NEXT:    [[TMP7:%.*]] = xor i64 [[TMP6]], 87960930222080
+; CHECK-NEXT:    [[TMP8:%.*]] = inttoptr i64 [[TMP7]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <8 x i64>, ptr [[TMP8]], align 64
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast i8 [[TMP2]] to <8 x i1>
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP11:%.*]] = ptrtoint ptr [[PTR]] to i64
+; CHECK-NEXT:    [[TMP12:%.*]] = xor i64 [[TMP11]], 87960930222080
+; CHECK-NEXT:    [[TMP13:%.*]] = inttoptr i64 [[TMP12]] to ptr
+; CHECK-NEXT:    [[_MSMASKEDLD:%.*]] = call <8 x i64> @llvm.masked.load.v8i64.p0(ptr [[TMP13]], i32 64, <8 x i1> [[TMP10]], <8 x i64> [[_MSLD]])
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast <8 x i1> [[TMP9]] to i8
+; CHECK-NEXT:    [[_MSCMP3:%.*]] = icmp ne i8 [[TMP14]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP2]], [[_MSCMP3]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP15:%.*]], label [[TMP16:%.*]], !prof [[PROF1]]
+; CHECK:       15:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       16:
+; CHECK-NEXT:    [[TMP17:%.*]] = call <8 x double> @llvm.masked.load.v8f64.p0(ptr [[PTR]], i32 64, <8 x i1> [[TMP10]], <8 x double> [[TMP5]])
+; CHECK-NEXT:    [[TMP18:%.*]] = bitcast i8 [[TMP2]] to <8 x i1>
+; CHECK-NEXT:    [[TMP19:%.*]] = bitcast i8 [[MASK]] to <8 x i1>
+; CHECK-NEXT:    [[TMP20:%.*]] = ptrtoint ptr [[PTR]] to i64
+; CHECK-NEXT:    [[TMP21:%.*]] = xor i64 [[TMP20]], 87960930222080
+; CHECK-NEXT:    [[TMP22:%.*]] = inttoptr i64 [[TMP21]] to ptr
+; CHECK-NEXT:    [[_MSMASKEDLD1:%.*]] = call <8 x i64> @llvm.masked.load.v8i64.p0(ptr [[TMP22]], i32 64, <8 x i1> [[TMP19]], <8 x i64> zeroinitializer)
+; CHECK-NEXT:    [[_MSCMP4:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    [[TMP23:%.*]] = bitcast <8 x i1> [[TMP18]] to i8
+; CHECK-NEXT:    [[_MSCMP5:%.*]] = icmp ne i8 [[TMP23]], 0
+; CHECK-NEXT:    [[_MSOR6:%.*]] = or i1 [[_MSCMP4]], [[_MSCMP5]]
+; CHECK-NEXT:    br i1 [[_MSOR6]], label [[TMP24:%.*]], label [[TMP25:%.*]], !prof [[PROF1]]
+; CHECK:       24:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       25:
+; CHECK-NEXT:    [[TMP26:%.*]] = call <8 x double> @llvm.masked.load.v8f64.p0(ptr [[PTR]], i32 64, <8 x i1> [[TMP19]], <8 x double> zeroinitializer)
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <8 x i64> [[_MSMASKEDLD1]], [[_MSMASKEDLD]]
+; CHECK-NEXT:    [[RES4:%.*]] = fadd <8 x double> [[TMP26]], [[TMP17]]
+; CHECK-NEXT:    store <8 x i64> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x double> [[RES4]]
+;
+  %res = call <8 x double> @llvm.x86.avx512.mask.load.pd.512(ptr %ptr, <8 x double> zeroinitializer, i8 -1)
+  %res1 = call <8 x double> @llvm.x86.avx512.mask.load.pd.512(ptr %ptr, <8 x double> %res, i8 %mask)
+  %res2 = call <8 x double> @llvm.x86.avx512.mask.load.pd.512(ptr %ptr, <8 x double> zeroinitializer, i8 %mask)
+  %res4 = fadd <8 x double> %res2, %res1
+  ret <8 x double> %res4
+}
+
+declare <8 x double> @llvm.x86.avx512.mask.load.pd.512(ptr, <8 x double>, i8)
+
+define <8 x double> @test_mask_load_unaligned_pd(<8 x double> %data, ptr %ptr, i8 %mask)  #0 {
+;
+; CHECK-LABEL: @test_mask_load_unaligned_pd(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 72) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP3:%.*]], label [[TMP4:%.*]], !prof [[PROF1]]
+; CHECK:       3:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       4:
+; CHECK-NEXT:    [[TMP5:%.*]] = load <8 x double>, ptr [[PTR:%.*]], align 1
+; CHECK-NEXT:    [[TMP6:%.*]] = ptrtoint ptr [[PTR]] to i64
+; CHECK-NEXT:    [[TMP7:%.*]] = xor i64 [[TMP6]], 87960930222080
+; CHECK-NEXT:    [[TMP8:%.*]] = inttoptr i64 [[TMP7]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <8 x i64>, ptr [[TMP8]], align 1
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast i8 [[TMP2]] to <8 x i1>
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP11:%.*]] = ptrtoint ptr [[PTR]] to i64
+; CHECK-NEXT:    [[TMP12:%.*]] = xor i64 [[TMP11]], 87960930222080
+; CHECK-NEXT:    [[TMP13:%.*]] = inttoptr i64 [[TMP12]] to ptr
+; CHECK-NEXT:    [[_MSMASKEDLD:%.*]] = call <8 x i64> @llvm.masked.load.v8i64.p0(ptr [[TMP13]], i32 1, <8 x i1> [[TMP10]], <8 x i64> [[_MSLD]])
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast <8 x i1> [[TMP9]] to i8
+; CHECK-NEXT:    [[_MSCMP3:%.*]] = icmp ne i8 [[TMP14]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP2]], [[_MSCMP3]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP15:%.*]], label [[TMP16:%.*]], !prof [[PROF1]]
+; CHECK:       15:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       16:
+; CHECK-NEXT:    [[TMP17:%.*]] = call <8 x double> @llvm.masked.load.v8f64.p0(ptr [[PTR]], i32 1, <8 x i1> [[TMP10]], <8 x double> [[TMP5]])
+; CHECK-NEXT:    [[TMP18:%.*]] = bitcast i8 [[TMP2]] to <8 x i1>
+; CHECK-NEXT:    [[TMP19:%.*]] = bitcast i8 [[MASK]] to <8 x i1>
+; CHECK-NEXT:    [[TMP20:%.*]] = ptrtoint ptr [[PTR]] to i64
+; CHECK-NEXT:    [[TMP21:%.*]] = xor i64 [[TMP20]], 87960930222080
+; CHECK-NEXT:    [[TMP22:%.*]] = inttoptr i64 [[TMP21]] to ptr
+; CHECK-NEXT:    [[_MSMASKEDLD1:%.*]] = call <8 x i64> @llvm.masked.load.v8i64.p0(ptr [[TMP22]], i32 1, <8 x i1> [[TMP19]], <8 x i64> zeroinitializer)
+; CHECK-NEXT:    [[_MSCMP4:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    [[TMP23:%.*]] = bitcast <8 x i1> [[TMP18]] to i8
+; CHECK-NEXT:    [[_MSCMP5:%.*]] = icmp ne i8 [[TMP23]], 0
+; CHECK-NEXT:    [[_MSOR6:%.*]] = or i1 [[_MSCMP4]], [[_MSCMP5]]
+; CHECK-NEXT:    br i1 [[_MSOR6]], label [[TMP24:%.*]], label [[TMP25:%.*]], !prof [[PROF1]]
+; CHECK:       24:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       25:
+; CHECK-NEXT:    [[TMP26:%.*]] = call <8 x double> @llvm.masked.load.v8f64.p0(ptr [[PTR]], i32 1, <8 x i1> [[TMP19]], <8 x double> zeroinitializer)
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <8 x i64> [[_MSMASKEDLD1]], [[_MSMASKEDLD]]
+; CHECK-NEXT:    [[RES4:%.*]] = fadd <8 x double> [[TMP26]], [[TMP17]]
+; CHECK-NEXT:    store <8 x i64> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x double> [[RES4]]
+;
+  %res = call <8 x double> @llvm.x86.avx512.mask.loadu.pd.512(ptr %ptr, <8 x double> zeroinitializer, i8 -1)
+  %res1 = call <8 x double> @llvm.x86.avx512.mask.loadu.pd.512(ptr %ptr, <8 x double> %res, i8 %mask)
+  %res2 = call <8 x double> @llvm.x86.avx512.mask.loadu.pd.512(ptr %ptr, <8 x double> zeroinitializer, i8 %mask)
+  %res4 = fadd <8 x double> %res2, %res1
+  ret <8 x double> %res4
+}
+
+declare <8 x double> @llvm.x86.avx512.mask.loadu.pd.512(ptr, <8 x double>, i8)
+
+declare <16 x i32> @llvm.x86.avx512.mask.loadu.d.512(ptr, <16 x i32>, i16)
+
+define <16 x i32> @test_mask_load_unaligned_d(ptr %ptr, ptr %ptr2, <16 x i32> %data, i16 %mask)  #0 {
+;
+; CHECK-LABEL: @test_mask_load_unaligned_d(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 80) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP4:%.*]], label [[TMP5:%.*]], !prof [[PROF1]]
+; CHECK:       4:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       5:
+; CHECK-NEXT:    [[TMP6:%.*]] = load <16 x i32>, ptr [[PTR:%.*]], align 1
+; CHECK-NEXT:    [[TMP7:%.*]] = ptrtoint ptr [[PTR]] to i64
+; CHECK-NEXT:    [[TMP8:%.*]] = xor i64 [[TMP7]], 87960930222080
+; CHECK-NEXT:    [[TMP9:%.*]] = inttoptr i64 [[TMP8]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <16 x i32>, ptr [[TMP9]], align 1
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i16 [[TMP2]] to <16 x i1>
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast i16 [[MASK:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP12:%.*]] = ptrtoint ptr [[PTR2:%.*]] to i64
+; CHECK-NEXT:    [[TMP13:%.*]] = xor i64 [[TMP12]], 87960930222080
+; CHECK-NEXT:    [[TMP14:%.*]] = inttoptr i64 [[TMP13]] to ptr
+; CHECK-NEXT:    [[_MSMASKEDLD:%.*]] = call <16 x i32> @llvm.masked.load.v16i32.p0(ptr [[TMP14]], i32 1, <16 x i1> [[TMP11]], <16 x i32> [[_MSLD]])
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i64 [[TMP3]], 0
+; CHECK-NEXT:    [[TMP15:%.*]] = bitcast <16 x i1> [[TMP10]] to i16
+; CHECK-NEXT:    [[_MSCMP3:%.*]] = icmp ne i16 [[TMP15]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP2]], [[_MSCMP3]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP16:%.*]], label [[TMP17:%.*]], !prof [[PROF1]]
+; CHECK:       16:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       17:
+; CHECK-NEXT:    [[TMP18:%.*]] = call <16 x i32> @llvm.masked.load.v16i32.p0(ptr [[PTR2]], i32 1, <16 x i1> [[TMP11]], <16 x i32> [[TMP6]])
+; CHECK-NEXT:    [[TMP19:%.*]] = bitcast i16 [[TMP2]] to <16 x i1>
+; CHECK-NEXT:    [[TMP20:%.*]] = bitcast i16 [[MASK]] to <16 x i1>
+; CHECK-NEXT:    [[TMP21:%.*]] = ptrtoint ptr [[PTR]] to i64
+; CHECK-NEXT:    [[TMP22:%.*]] = xor i64 [[TMP21]], 87960930222080
+; CHECK-NEXT:    [[TMP23:%.*]] = inttoptr i64 [[TMP22]] to ptr
+; CHECK-NEXT:    [[_MSMASKEDLD1:%.*]] = call <16 x i32> @llvm.masked.load.v16i32.p0(ptr [[TMP23]], i32 1, <16 x i1> [[TMP20]], <16 x i32> zeroinitializer)
+; CHECK-NEXT:    [[_MSCMP4:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    [[TMP24:%.*]] = bitcast <16 x i1> [[TMP19]] to i16
+; CHECK-NEXT:    [[_MSCMP5:%.*]] = icmp ne i16 [[TMP24]], 0
+; CHECK-NEXT:    [[_MSOR6:%.*]] = or i1 [[_MSCMP4]], [[_MSCMP5]]
+; CHECK-NEXT:    br i1 [[_MSOR6]], label [[TMP25:%.*]], label [[TMP26:%.*]], !prof [[PROF1]]
+; CHECK:       25:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       26:
+; CHECK-NEXT:    [[TMP27:%.*]] = call <16 x i32> @llvm.masked.load.v16i32.p0(ptr [[PTR]], i32 1, <16 x i1> [[TMP20]], <16 x i32> zeroinitializer)
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <16 x i32> [[_MSMASKEDLD1]], [[_MSMASKEDLD]]
+; CHECK-NEXT:    [[RES4:%.*]] = add <16 x i32> [[TMP27]], [[TMP18]]
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i32> [[RES4]]
+;
+  %res = call <16 x i32> @llvm.x86.avx512.mask.loadu.d.512(ptr %ptr, <16 x i32> zeroinitializer, i16 -1)
+  %res1 = call <16 x i32> @llvm.x86.avx512.mask.loadu.d.512(ptr %ptr2, <16 x i32> %res, i16 %mask)
+  %res2 = call <16 x i32> @llvm.x86.avx512.mask.loadu.d.512(ptr %ptr, <16 x i32> zeroinitializer, i16 %mask)
+  %res4 = add <16 x i32> %res2, %res1
+  ret <16 x i32> %res4
+}
+
+declare <8 x i64> @llvm.x86.avx512.mask.loadu.q.512(ptr, <8 x i64>, i8)
+
+define <8 x i64> @test_mask_load_unaligned_q(ptr %ptr, ptr %ptr2, <8 x i64> %data, i8 %mask)  #0 {
+;
+; CHECK-LABEL: @test_mask_load_unaligned_q(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 80) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP4:%.*]], label [[TMP5:%.*]], !prof [[PROF1]]
+; CHECK:       4:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       5:
+; CHECK-NEXT:    [[TMP6:%.*]] = load <8 x i64>, ptr [[PTR:%.*]], align 1
+; CHECK-NEXT:    [[TMP7:%.*]] = ptrtoint ptr [[PTR]] to i64
+; CHECK-NEXT:    [[TMP8:%.*]] = xor i64 [[TMP7]], 87960930222080
+; CHECK-NEXT:    [[TMP9:%.*]] = inttoptr i64 [[TMP8]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <8 x i64>, ptr [[TMP9]], align 1
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i8 [[TMP2]] to <8 x i1>
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP12:%.*]] = ptrtoint ptr [[PTR2:%.*]] to i64
+; CHECK-NEXT:    [[TMP13:%.*]] = xor i64 [[TMP12]], 87960930222080
+; CHECK-NEXT:    [[TMP14:%.*]] = inttoptr i64 [[TMP13]] to ptr
+; CHECK-NEXT:    [[_MSMASKEDLD:%.*]] = call <8 x i64> @llvm.masked.load.v8i64.p0(ptr [[TMP14]], i32 1, <8 x i1> [[TMP11]], <8 x i64> [[_MSLD]])
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i64 [[TMP3]], 0
+; CHECK-NEXT:    [[TMP15:%.*]] = bitcast <8 x i1> [[TMP10]] to i8
+; CHECK-NEXT:    [[_MSCMP3:%.*]] = icmp ne i8 [[TMP15]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP2]], [[_MSCMP3]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP16:%.*]], label [[TMP17:%.*]], !prof [[PROF1]]
+; CHECK:       16:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       17:
+; CHECK-NEXT:    [[TMP18:%.*]] = call <8 x i64> @llvm.masked.load.v8i64.p0(ptr [[PTR2]], i32 1, <8 x i1> [[TMP11]], <8 x i64> [[TMP6]])
+; CHECK-NEXT:    [[TMP19:%.*]] = bitcast i8 [[TMP2]] to <8 x i1>
+; CHECK-NEXT:    [[TMP20:%.*]] = bitcast i8 [[MASK]] to <8 x i1>
+; CHECK-NEXT:    [[TMP21:%.*]] = ptrtoint ptr [[PTR]] to i64
+; CHECK-NEXT:    [[TMP22:%.*]] = xor i64 [[TMP21]], 87960930222080
+; CHECK-NEXT:    [[TMP23:%.*]] = inttoptr i64 [[TMP22]] to ptr
+; CHECK-NEXT:    [[_MSMASKEDLD1:%.*]] = call <8 x i64> @llvm.masked.load.v8i64.p0(ptr [[TMP23]], i32 1, <8 x i1> [[TMP20]], <8 x i64> zeroinitializer)
+; CHECK-NEXT:    [[_MSCMP4:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    [[TMP24:%.*]] = bitcast <8 x i1> [[TMP19]] to i8
+; CHECK-NEXT:    [[_MSCMP5:%.*]] = icmp ne i8 [[TMP24]], 0
+; CHECK-NEXT:    [[_MSOR6:%.*]] = or i1 [[_MSCMP4]], [[_MSCMP5]]
+; CHECK-NEXT:    br i1 [[_MSOR6]], label [[TMP25:%.*]], label [[TMP26:%.*]], !prof [[PROF1]]
+; CHECK:       25:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       26:
+; CHECK-NEXT:    [[TMP27:%.*]] = call <8 x i64> @llvm.masked.load.v8i64.p0(ptr [[PTR]], i32 1, <8 x i1> [[TMP20]], <8 x i64> zeroinitializer)
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <8 x i64> [[_MSMASKEDLD1]], [[_MSMASKEDLD]]
+; CHECK-NEXT:    [[RES4:%.*]] = add <8 x i64> [[TMP27]], [[TMP18]]
+; CHECK-NEXT:    store <8 x i64> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i64> [[RES4]]
+;
+  %res = call <8 x i64> @llvm.x86.avx512.mask.loadu.q.512(ptr %ptr, <8 x i64> zeroinitializer, i8 -1)
+  %res1 = call <8 x i64> @llvm.x86.avx512.mask.loadu.q.512(ptr %ptr2, <8 x i64> %res, i8 %mask)
+  %res2 = call <8 x i64> @llvm.x86.avx512.mask.loadu.q.512(ptr %ptr, <8 x i64> zeroinitializer, i8 %mask)
+  %res4 = add <8 x i64> %res2, %res1
+  ret <8 x i64> %res4
+}
+
+declare <16 x i32> @llvm.x86.avx512.mask.load.d.512(ptr, <16 x i32>, i16)
+
+define <16 x i32> @test_mask_load_aligned_d(<16 x i32> %data, ptr %ptr, i16 %mask)  #0 {
+;
+; CHECK-LABEL: @test_mask_load_aligned_d(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 72) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP3:%.*]], label [[TMP4:%.*]], !prof [[PROF1]]
+; CHECK:       3:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       4:
+; CHECK-NEXT:    [[TMP5:%.*]] = load <16 x i32>, ptr [[PTR:%.*]], align 64
+; CHECK-NEXT:    [[TMP6:%.*]] = ptrtoint ptr [[PTR]] to i64
+; CHECK-NEXT:    [[TMP7:%.*]] = xor i64 [[TMP6]], 87960930222080
+; CHECK-NEXT:    [[TMP8:%.*]] = inttoptr i64 [[TMP7]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <16 x i32>, ptr [[TMP8]], align 64
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast i16 [[TMP2]] to <16 x i1>
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i16 [[MASK:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP11:%.*]] = ptrtoint ptr [[PTR]] to i64
+; CHECK-NEXT:    [[TMP12:%.*]] = xor i64 [[TMP11]], 87960930222080
+; CHECK-NEXT:    [[TMP13:%.*]] = inttoptr i64 [[TMP12]] to ptr
+; CHECK-NEXT:    [[_MSMASKEDLD:%.*]] = call <16 x i32> @llvm.masked.load.v16i32.p0(ptr [[TMP13]], i32 64, <16 x i1> [[TMP10]], <16 x i32> [[_MSLD]])
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast <16 x i1> [[TMP9]] to i16
+; CHECK-NEXT:    [[_MSCMP3:%.*]] = icmp ne i16 [[TMP14]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP2]], [[_MSCMP3]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP15:%.*]], label [[TMP16:%.*]], !prof [[PROF1]]
+; CHECK:       15:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       16:
+; CHECK-NEXT:    [[TMP17:%.*]] = call <16 x i32> @llvm.masked.load.v16i32.p0(ptr [[PTR]], i32 64, <16 x i1> [[TMP10]], <16 x i32> [[TMP5]])
+; CHECK-NEXT:    [[TMP18:%.*]] = bitcast i16 [[TMP2]] to <16 x i1>
+; CHECK-NEXT:    [[TMP19:%.*]] = bitcast i16 [[MASK]] to <16 x i1>
+; CHECK-NEXT:    [[TMP20:%.*]] = ptrtoint ptr [[PTR]] to i64
+; CHECK-NEXT:    [[TMP21:%.*]] = xor i64 [[TMP20]], 87960930222080
+; CHECK-NEXT:    [[TMP22:%.*]] = inttoptr i64 [[TMP21]] to ptr
+; CHECK-NEXT:    [[_MSMASKEDLD1:%.*]] = call <16 x i32> @llvm.masked.load.v16i32.p0(ptr [[TMP22]], i32 64, <16 x i1> [[TMP19]], <16 x i32> zeroinitializer)
+; CHECK-NEXT:    [[_MSCMP4:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    [[TMP23:%.*]] = bitcast <16 x i1> [[TMP18]] to i16
+; CHECK-NEXT:    [[_MSCMP5:%.*]] = icmp ne i16 [[TMP23]], 0
+; CHECK-NEXT:    [[_MSOR6:%.*]] = or i1 [[_MSCMP4]], [[_MSCMP5]]
+; CHECK-NEXT:    br i1 [[_MSOR6]], label [[TMP24:%.*]], label [[TMP25:%.*]], !prof [[PROF1]]
+; CHECK:       24:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       25:
+; CHECK-NEXT:    [[TMP26:%.*]] = call <16 x i32> @llvm.masked.load.v16i32.p0(ptr [[PTR]], i32 64, <16 x i1> [[TMP19]], <16 x i32> zeroinitializer)
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <16 x i32> [[_MSMASKEDLD1]], [[_MSMASKEDLD]]
+; CHECK-NEXT:    [[RES4:%.*]] = add <16 x i32> [[TMP26]], [[TMP17]]
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i32> [[RES4]]
+;
+  %res = call <16 x i32> @llvm.x86.avx512.mask.load.d.512(ptr %ptr, <16 x i32> zeroinitializer, i16 -1)
+  %res1 = call <16 x i32> @llvm.x86.avx512.mask.load.d.512(ptr %ptr, <16 x i32> %res, i16 %mask)
+  %res2 = call <16 x i32> @llvm.x86.avx512.mask.load.d.512(ptr %ptr, <16 x i32> zeroinitializer, i16 %mask)
+  %res4 = add <16 x i32> %res2, %res1
+  ret <16 x i32> %res4
+}
+
+declare <8 x i64> @llvm.x86.avx512.mask.load.q.512(ptr, <8 x i64>, i8)
+
+define <8 x i64> @test_mask_load_aligned_q(<8 x i64> %data, ptr %ptr, i8 %mask)  #0 {
+;
+; CHECK-LABEL: @test_mask_load_aligned_q(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 72) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP3:%.*]], label [[TMP4:%.*]], !prof [[PROF1]]
+; CHECK:       3:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       4:
+; CHECK-NEXT:    [[TMP5:%.*]] = load <8 x i64>, ptr [[PTR:%.*]], align 64
+; CHECK-NEXT:    [[TMP6:%.*]] = ptrtoint ptr [[PTR]] to i64
+; CHECK-NEXT:    [[TMP7:%.*]] = xor i64 [[TMP6]], 87960930222080
+; CHECK-NEXT:    [[TMP8:%.*]] = inttoptr i64 [[TMP7]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <8 x i64>, ptr [[TMP8]], align 64
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast i8 [[TMP2]] to <8 x i1>
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP11:%.*]] = ptrtoint ptr [[PTR]] to i64
+; CHECK-NEXT:    [[TMP12:%.*]] = xor i64 [[TMP11]], 87960930222080
+; CHECK-NEXT:    [[TMP13:%.*]] = inttoptr i64 [[TMP12]] to ptr
+; CHECK-NEXT:    [[_MSMASKEDLD:%.*]] = call <8 x i64> @llvm.masked.load.v8i64.p0(ptr [[TMP13]], i32 64, <8 x i1> [[TMP10]], <8 x i64> [[_MSLD]])
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast <8 x i1> [[TMP9]] to i8
+; CHECK-NEXT:    [[_MSCMP3:%.*]] = icmp ne i8 [[TMP14]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP2]], [[_MSCMP3]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP15:%.*]], label [[TMP16:%.*]], !prof [[PROF1]]
+; CHECK:       15:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       16:
+; CHECK-NEXT:    [[TMP17:%.*]] = call <8 x i64> @llvm.masked.load.v8i64.p0(ptr [[PTR]], i32 64, <8 x i1> [[TMP10]], <8 x i64> [[TMP5]])
+; CHECK-NEXT:    [[TMP18:%.*]] = bitcast i8 [[TMP2]] to <8 x i1>
+; CHECK-NEXT:    [[TMP19:%.*]] = bitcast i8 [[MASK]] to <8 x i1>
+; CHECK-NEXT:    [[TMP20:%.*]] = ptrtoint ptr [[PTR]] to i64
+; CHECK-NEXT:    [[TMP21:%.*]] = xor i64 [[TMP20]], 87960930222080
+; CHECK-NEXT:    [[TMP22:%.*]] = inttoptr i64 [[TMP21]] to ptr
+; CHECK-NEXT:    [[_MSMASKEDLD1:%.*]] = call <8 x i64> @llvm.masked.load.v8i64.p0(ptr [[TMP22]], i32 64, <8 x i1> [[TMP19]], <8 x i64> zeroinitializer)
+; CHECK-NEXT:    [[_MSCMP4:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    [[TMP23:%.*]] = bitcast <8 x i1> [[TMP18]] to i8
+; CHECK-NEXT:    [[_MSCMP5:%.*]] = icmp ne i8 [[TMP23]], 0
+; CHECK-NEXT:    [[_MSOR6:%.*]] = or i1 [[_MSCMP4]], [[_MSCMP5]]
+; CHECK-NEXT:    br i1 [[_MSOR6]], label [[TMP24:%.*]], label [[TMP25:%.*]], !prof [[PROF1]]
+; CHECK:       24:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       25:
+; CHECK-NEXT:    [[TMP26:%.*]] = call <8 x i64> @llvm.masked.load.v8i64.p0(ptr [[PTR]], i32 64, <8 x i1> [[TMP19]], <8 x i64> zeroinitializer)
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <8 x i64> [[_MSMASKEDLD1]], [[_MSMASKEDLD]]
+; CHECK-NEXT:    [[RES4:%.*]] = add <8 x i64> [[TMP26]], [[TMP17]]
+; CHECK-NEXT:    store <8 x i64> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i64> [[RES4]]
+;
+  %res = call <8 x i64> @llvm.x86.avx512.mask.load.q.512(ptr %ptr, <8 x i64> zeroinitializer, i8 -1)
+  %res1 = call <8 x i64> @llvm.x86.avx512.mask.load.q.512(ptr %ptr, <8 x i64> %res, i8 %mask)
+  %res2 = call <8 x i64> @llvm.x86.avx512.mask.load.q.512(ptr %ptr, <8 x i64> zeroinitializer, i8 %mask)
+  %res4 = add <8 x i64> %res2, %res1
+  ret <8 x i64> %res4
+}
+
+declare <8 x double> @llvm.x86.avx512.mask.vpermil.pd.512(<8 x double>, i32, <8 x double>, i8)
+
+define <8 x double>@test_int_x86_avx512_vpermil_pd_512(<8 x double> %x0, <8 x double> %x2)  #0 {
+; CHECK-LABEL: @test_int_x86_avx512_vpermil_pd_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <8 x i64> [[TMP1]], <8 x i64> [[TMP1]], <8 x i32> <i32 0, i32 1, i32 3, i32 2, i32 5, i32 4, i32 6, i32 6>
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <8 x double> [[X0:%.*]], <8 x double> [[X0]], <8 x i32> <i32 0, i32 1, i32 3, i32 2, i32 5, i32 4, i32 6, i32 6>
+; CHECK-NEXT:    store <8 x i64> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x double> [[TMP2]]
+;
+  %res = call <8 x double> @llvm.x86.avx512.mask.vpermil.pd.512(<8 x double> %x0, i32 22, <8 x double> %x2, i8 -1)
+  ret <8 x double> %res
+}
+
+define <8 x double>@test_int_x86_avx512_mask_vpermil_pd_512(<8 x double> %x0, <8 x double> %x2, i8 %x3)  #0 {
+;
+; CHECK-LABEL: @test_int_x86_avx512_mask_vpermil_pd_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <8 x i64> [[TMP1]], <8 x i64> [[TMP1]], <8 x i32> <i32 0, i32 1, i32 3, i32 2, i32 5, i32 4, i32 6, i32 6>
+; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <8 x double> [[X0:%.*]], <8 x double> [[X0]], <8 x i32> <i32 0, i32 1, i32 3, i32 2, i32 5, i32 4, i32 6, i32 6>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i8 [[TMP2]] to <8 x i1>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i8 [[X3:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP7:%.*]] = select <8 x i1> [[TMP6]], <8 x i64> [[_MSPROP]], <8 x i64> [[TMP3]]
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <8 x double> [[TMP4]] to <8 x i64>
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <8 x double> [[X2:%.*]] to <8 x i64>
+; CHECK-NEXT:    [[TMP10:%.*]] = xor <8 x i64> [[TMP8]], [[TMP9]]
+; CHECK-NEXT:    [[TMP11:%.*]] = or <8 x i64> [[TMP10]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP12:%.*]] = or <8 x i64> [[TMP11]], [[TMP3]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP5]], <8 x i64> [[TMP12]], <8 x i64> [[TMP7]]
+; CHECK-NEXT:    [[TMP13:%.*]] = select <8 x i1> [[TMP6]], <8 x double> [[TMP4]], <8 x double> [[X2]]
+; CHECK-NEXT:    store <8 x i64> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x double> [[TMP13]]
+;
+  %res = call <8 x double> @llvm.x86.avx512.mask.vpermil.pd.512(<8 x double> %x0, i32 22, <8 x double> %x2, i8 %x3)
+  ret <8 x double> %res
+}
+
+define <8 x double>@test_int_x86_avx512_maskz_vpermil_pd_512(<8 x double> %x0, i8 %x3)  #0 {
+;
+; CHECK-LABEL: @test_int_x86_avx512_maskz_vpermil_pd_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <8 x i64> [[TMP1]], <8 x i64> [[TMP1]], <8 x i32> <i32 0, i32 1, i32 3, i32 2, i32 5, i32 4, i32 6, i32 6>
+; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <8 x double> [[X0:%.*]], <8 x double> [[X0]], <8 x i32> <i32 0, i32 1, i32 3, i32 2, i32 5, i32 4, i32 6, i32 6>
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast i8 [[TMP2]] to <8 x i1>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i8 [[X3:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP6:%.*]] = select <8 x i1> [[TMP5]], <8 x i64> [[_MSPROP]], <8 x i64> zeroinitializer
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <8 x double> [[TMP3]] to <8 x i64>
+; CHECK-NEXT:    [[TMP8:%.*]] = xor <8 x i64> [[TMP7]], zeroinitializer
+; CHECK-NEXT:    [[TMP9:%.*]] = or <8 x i64> [[TMP8]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP10:%.*]] = or <8 x i64> [[TMP9]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP4]], <8 x i64> [[TMP10]], <8 x i64> [[TMP6]]
+; CHECK-NEXT:    [[TMP11:%.*]] = select <8 x i1> [[TMP5]], <8 x double> [[TMP3]], <8 x double> zeroinitializer
+; CHECK-NEXT:    store <8 x i64> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x double> [[TMP11]]
+;
+  %res = call <8 x double> @llvm.x86.avx512.mask.vpermil.pd.512(<8 x double> %x0, i32 22, <8 x double> zeroinitializer, i8 %x3)
+  ret <8 x double> %res
+}
+
+declare <16 x float> @llvm.x86.avx512.mask.vpermil.ps.512(<16 x float>, i32, <16 x float>, i16)
+
+define <16 x float>@test_int_x86_avx512_vpermil_ps_512(<16 x float> %x0, <16 x float> %x2)  #0 {
+; CHECK-LABEL: @test_int_x86_avx512_vpermil_ps_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <16 x i32> [[TMP1]], <16 x i32> [[TMP1]], <16 x i32> <i32 2, i32 1, i32 1, i32 0, i32 6, i32 5, i32 5, i32 4, i32 10, i32 9, i32 9, i32 8, i32 14, i32 13, i32 13, i32 12>
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <16 x float> [[X0:%.*]], <16 x float> [[X0]], <16 x i32> <i32 2, i32 1, i32 1, i32 0, i32 6, i32 5, i32 5, i32 4, i32 10, i32 9, i32 9, i32 8, i32 14, i32 13, i32 13, i32 12>
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x float> [[TMP2]]
+;
+  %res = call <16 x float> @llvm.x86.avx512.mask.vpermil.ps.512(<16 x float> %x0, i32 22, <16 x float> %x2, i16 -1)
+  ret <16 x float> %res
+}
+
+define <16 x float>@test_int_x86_avx512_mask_vpermil_ps_512(<16 x float> %x0, <16 x float> %x2, i16 %x3)  #0 {
+;
+; CHECK-LABEL: @test_int_x86_avx512_mask_vpermil_ps_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <16 x i32> [[TMP1]], <16 x i32> [[TMP1]], <16 x i32> <i32 2, i32 1, i32 1, i32 0, i32 6, i32 5, i32 5, i32 4, i32 10, i32 9, i32 9, i32 8, i32 14, i32 13, i32 13, i32 12>
+; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <16 x float> [[X0:%.*]], <16 x float> [[X0]], <16 x i32> <i32 2, i32 1, i32 1, i32 0, i32 6, i32 5, i32 5, i32 4, i32 10, i32 9, i32 9, i32 8, i32 14, i32 13, i32 13, i32 12>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i16 [[TMP2]] to <16 x i1>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i16 [[X3:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP7:%.*]] = select <16 x i1> [[TMP6]], <16 x i32> [[_MSPROP]], <16 x i32> [[TMP3]]
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <16 x float> [[TMP4]] to <16 x i32>
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <16 x float> [[X2:%.*]] to <16 x i32>
+; CHECK-NEXT:    [[TMP10:%.*]] = xor <16 x i32> [[TMP8]], [[TMP9]]
+; CHECK-NEXT:    [[TMP11:%.*]] = or <16 x i32> [[TMP10]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP12:%.*]] = or <16 x i32> [[TMP11]], [[TMP3]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP5]], <16 x i32> [[TMP12]], <16 x i32> [[TMP7]]
+; CHECK-NEXT:    [[TMP13:%.*]] = select <16 x i1> [[TMP6]], <16 x float> [[TMP4]], <16 x float> [[X2]]
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x float> [[TMP13]]
+;
+  %res = call <16 x float> @llvm.x86.avx512.mask.vpermil.ps.512(<16 x float> %x0, i32 22, <16 x float> %x2, i16 %x3)
+  ret <16 x float> %res
+}
+
+define <16 x float>@test_int_x86_avx512_maskz_vpermil_ps_512(<16 x float> %x0, i16 %x3)  #0 {
+;
+; CHECK-LABEL: @test_int_x86_avx512_maskz_vpermil_ps_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <16 x i32> [[TMP1]], <16 x i32> [[TMP1]], <16 x i32> <i32 2, i32 1, i32 1, i32 0, i32 6, i32 5, i32 5, i32 4, i32 10, i32 9, i32 9, i32 8, i32 14, i32 13, i32 13, i32 12>
+; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <16 x float> [[X0:%.*]], <16 x float> [[X0]], <16 x i32> <i32 2, i32 1, i32 1, i32 0, i32 6, i32 5, i32 5, i32 4, i32 10, i32 9, i32 9, i32 8, i32 14, i32 13, i32 13, i32 12>
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast i16 [[TMP2]] to <16 x i1>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i16 [[X3:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP6:%.*]] = select <16 x i1> [[TMP5]], <16 x i32> [[_MSPROP]], <16 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <16 x float> [[TMP3]] to <16 x i32>
+; CHECK-NEXT:    [[TMP8:%.*]] = xor <16 x i32> [[TMP7]], zeroinitializer
+; CHECK-NEXT:    [[TMP9:%.*]] = or <16 x i32> [[TMP8]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP10:%.*]] = or <16 x i32> [[TMP9]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP4]], <16 x i32> [[TMP10]], <16 x i32> [[TMP6]]
+; CHECK-NEXT:    [[TMP11:%.*]] = select <16 x i1> [[TMP5]], <16 x float> [[TMP3]], <16 x float> zeroinitializer
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x float> [[TMP11]]
+;
+  %res = call <16 x float> @llvm.x86.avx512.mask.vpermil.ps.512(<16 x float> %x0, i32 22, <16 x float> zeroinitializer, i16 %x3)
+  ret <16 x float> %res
+}
+
+declare <16 x i32> @llvm.x86.avx512.mask.pshuf.d.512(<16 x i32>, i32, <16 x i32>, i16)
+
+define <16 x i32>@test_int_x86_avx512_pshuf_d_512(<16 x i32> %x0, i32 %x1, <16 x i32> %x2)  #0 {
+; CHECK-LABEL: @test_int_x86_avx512_pshuf_d_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <16 x i32> [[TMP1]], <16 x i32> [[TMP1]], <16 x i32> <i32 3, i32 0, i32 0, i32 0, i32 7, i32 4, i32 4, i32 4, i32 11, i32 8, i32 8, i32 8, i32 15, i32 12, i32 12, i32 12>
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <16 x i32> [[X0:%.*]], <16 x i32> [[X0]], <16 x i32> <i32 3, i32 0, i32 0, i32 0, i32 7, i32 4, i32 4, i32 4, i32 11, i32 8, i32 8, i32 8, i32 15, i32 12, i32 12, i32 12>
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i32> [[TMP2]]
+;
+  %res = call <16 x i32> @llvm.x86.avx512.mask.pshuf.d.512(<16 x i32> %x0, i32 3, <16 x i32> %x2, i16 -1)
+  ret <16 x i32> %res
+}
+
+define <16 x i32>@test_int_x86_avx512_mask_pshuf_d_512(<16 x i32> %x0, i32 %x1, <16 x i32> %x2, i16 %x3)  #0 {
+;
+; CHECK-LABEL: @test_int_x86_avx512_mask_pshuf_d_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 136) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 72) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <16 x i32> [[TMP1]], <16 x i32> [[TMP1]], <16 x i32> <i32 3, i32 0, i32 0, i32 0, i32 7, i32 4, i32 4, i32 4, i32 11, i32 8, i32 8, i32 8, i32 15, i32 12, i32 12, i32 12>
+; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <16 x i32> [[X0:%.*]], <16 x i32> [[X0]], <16 x i32> <i32 3, i32 0, i32 0, i32 0, i32 7, i32 4, i32 4, i32 4, i32 11, i32 8, i32 8, i32 8, i32 15, i32 12, i32 12, i32 12>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i16 [[TMP2]] to <16 x i1>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i16 [[X3:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP7:%.*]] = select <16 x i1> [[TMP6]], <16 x i32> [[_MSPROP]], <16 x i32> [[TMP3]]
+; CHECK-NEXT:    [[TMP8:%.*]] = xor <16 x i32> [[TMP4]], [[X2:%.*]]
+; CHECK-NEXT:    [[TMP9:%.*]] = or <16 x i32> [[TMP8]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP10:%.*]] = or <16 x i32> [[TMP9]], [[TMP3]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP5]], <16 x i32> [[TMP10]], <16 x i32> [[TMP7]]
+; CHECK-NEXT:    [[TMP11:%.*]] = select <16 x i1> [[TMP6]], <16 x i32> [[TMP4]], <16 x i32> [[X2]]
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i32> [[TMP11]]
+;
+  %res = call <16 x i32> @llvm.x86.avx512.mask.pshuf.d.512(<16 x i32> %x0, i32 3, <16 x i32> %x2, i16 %x3)
+  ret <16 x i32> %res
+}
+
+define <16 x i32>@test_int_x86_avx512_maskz_pshuf_d_512(<16 x i32> %x0, i32 %x1, i16 %x3)  #0 {
+;
+; CHECK-LABEL: @test_int_x86_avx512_maskz_pshuf_d_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 72) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <16 x i32> [[TMP1]], <16 x i32> [[TMP1]], <16 x i32> <i32 3, i32 0, i32 0, i32 0, i32 7, i32 4, i32 4, i32 4, i32 11, i32 8, i32 8, i32 8, i32 15, i32 12, i32 12, i32 12>
+; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <16 x i32> [[X0:%.*]], <16 x i32> [[X0]], <16 x i32> <i32 3, i32 0, i32 0, i32 0, i32 7, i32 4, i32 4, i32 4, i32 11, i32 8, i32 8, i32 8, i32 15, i32 12, i32 12, i32 12>
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast i16 [[TMP2]] to <16 x i1>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i16 [[X3:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP6:%.*]] = select <16 x i1> [[TMP5]], <16 x i32> [[_MSPROP]], <16 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP7:%.*]] = xor <16 x i32> [[TMP3]], zeroinitializer
+; CHECK-NEXT:    [[TMP8:%.*]] = or <16 x i32> [[TMP7]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP9:%.*]] = or <16 x i32> [[TMP8]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP4]], <16 x i32> [[TMP9]], <16 x i32> [[TMP6]]
+; CHECK-NEXT:    [[TMP10:%.*]] = select <16 x i1> [[TMP5]], <16 x i32> [[TMP3]], <16 x i32> zeroinitializer
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i32> [[TMP10]]
+;
+  %res = call <16 x i32> @llvm.x86.avx512.mask.pshuf.d.512(<16 x i32> %x0, i32 3, <16 x i32> zeroinitializer, i16 %x3)
+  ret <16 x i32> %res
+}
+
+define i16 @test_pcmpeq_d(<16 x i32> %a, <16 x i32> %b)  #0 {
+; CHECK-LABEL: @test_pcmpeq_d(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = xor <16 x i32> [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[TMP4:%.*]] = or <16 x i32> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp ne <16 x i32> [[TMP4]], zeroinitializer
+; CHECK-NEXT:    [[TMP6:%.*]] = xor <16 x i32> [[TMP4]], splat (i32 -1)
+; CHECK-NEXT:    [[TMP7:%.*]] = and <16 x i32> [[TMP6]], [[TMP3]]
+; CHECK-NEXT:    [[TMP8:%.*]] = icmp eq <16 x i32> [[TMP7]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_ICMP:%.*]] = and <16 x i1> [[TMP5]], [[TMP8]]
+; CHECK-NEXT:    [[TMP9:%.*]] = icmp eq <16 x i32> [[A]], [[B]]
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast <16 x i1> [[_MSPROP_ICMP]] to i16
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <16 x i1> [[TMP9]] to i16
+; CHECK-NEXT:    store i16 [[TMP10]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i16 [[TMP11]]
+;
+  %res = call i16 @llvm.x86.avx512.mask.pcmpeq.d.512(<16 x i32> %a, <16 x i32> %b, i16 -1)
+  ret i16 %res
+}
+
+define i16 @test_mask_pcmpeq_d(<16 x i32> %a, <16 x i32> %b, i16 %mask)  #0 {
+;
+; CHECK-LABEL: @test_mask_pcmpeq_d(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = xor <16 x i32> [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[TMP5:%.*]] = or <16 x i32> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP6:%.*]] = icmp ne <16 x i32> [[TMP5]], zeroinitializer
+; CHECK-NEXT:    [[TMP7:%.*]] = xor <16 x i32> [[TMP5]], splat (i32 -1)
+; CHECK-NEXT:    [[TMP8:%.*]] = and <16 x i32> [[TMP7]], [[TMP4]]
+; CHECK-NEXT:    [[TMP9:%.*]] = icmp eq <16 x i32> [[TMP8]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_ICMP:%.*]] = and <16 x i1> [[TMP6]], [[TMP9]]
+; CHECK-NEXT:    [[TMP10:%.*]] = icmp eq <16 x i32> [[A]], [[B]]
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast i16 [[TMP3]] to <16 x i1>
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast i16 [[MASK:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP13:%.*]] = and <16 x i1> [[_MSPROP_ICMP]], [[TMP11]]
+; CHECK-NEXT:    [[TMP14:%.*]] = and <16 x i1> [[TMP10]], [[TMP11]]
+; CHECK-NEXT:    [[TMP15:%.*]] = and <16 x i1> [[_MSPROP_ICMP]], [[TMP12]]
+; CHECK-NEXT:    [[TMP16:%.*]] = or <16 x i1> [[TMP13]], [[TMP14]]
+; CHECK-NEXT:    [[TMP17:%.*]] = or <16 x i1> [[TMP16]], [[TMP15]]
+; CHECK-NEXT:    [[TMP18:%.*]] = and <16 x i1> [[TMP10]], [[TMP12]]
+; CHECK-NEXT:    [[TMP19:%.*]] = bitcast <16 x i1> [[TMP17]] to i16
+; CHECK-NEXT:    [[TMP20:%.*]] = bitcast <16 x i1> [[TMP18]] to i16
+; CHECK-NEXT:    store i16 [[TMP19]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i16 [[TMP20]]
+;
+  %res = call i16 @llvm.x86.avx512.mask.pcmpeq.d.512(<16 x i32> %a, <16 x i32> %b, i16 %mask)
+  ret i16 %res
+}
+
+declare i16 @llvm.x86.avx512.mask.pcmpeq.d.512(<16 x i32>, <16 x i32>, i16)
+
+define i8 @test_pcmpeq_q(<8 x i64> %a, <8 x i64> %b)  #0 {
+; CHECK-LABEL: @test_pcmpeq_q(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = xor <8 x i64> [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[TMP4:%.*]] = or <8 x i64> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp ne <8 x i64> [[TMP4]], zeroinitializer
+; CHECK-NEXT:    [[TMP6:%.*]] = xor <8 x i64> [[TMP4]], splat (i64 -1)
+; CHECK-NEXT:    [[TMP7:%.*]] = and <8 x i64> [[TMP6]], [[TMP3]]
+; CHECK-NEXT:    [[TMP8:%.*]] = icmp eq <8 x i64> [[TMP7]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_ICMP:%.*]] = and <8 x i1> [[TMP5]], [[TMP8]]
+; CHECK-NEXT:    [[TMP9:%.*]] = icmp eq <8 x i64> [[A]], [[B]]
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast <8 x i1> [[_MSPROP_ICMP]] to i8
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <8 x i1> [[TMP9]] to i8
+; CHECK-NEXT:    store i8 [[TMP10]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i8 [[TMP11]]
+;
+  %res = call i8 @llvm.x86.avx512.mask.pcmpeq.q.512(<8 x i64> %a, <8 x i64> %b, i8 -1)
+  ret i8 %res
+}
+
+define i8 @test_mask_pcmpeq_q(<8 x i64> %a, <8 x i64> %b, i8 %mask)  #0 {
+;
+; CHECK-LABEL: @test_mask_pcmpeq_q(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = xor <8 x i64> [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[TMP5:%.*]] = or <8 x i64> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP6:%.*]] = icmp ne <8 x i64> [[TMP5]], zeroinitializer
+; CHECK-NEXT:    [[TMP7:%.*]] = xor <8 x i64> [[TMP5]], splat (i64 -1)
+; CHECK-NEXT:    [[TMP8:%.*]] = and <8 x i64> [[TMP7]], [[TMP4]]
+; CHECK-NEXT:    [[TMP9:%.*]] = icmp eq <8 x i64> [[TMP8]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_ICMP:%.*]] = and <8 x i1> [[TMP6]], [[TMP9]]
+; CHECK-NEXT:    [[TMP10:%.*]] = icmp eq <8 x i64> [[A]], [[B]]
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast i8 [[TMP3]] to <8 x i1>
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP13:%.*]] = and <8 x i1> [[_MSPROP_ICMP]], [[TMP11]]
+; CHECK-NEXT:    [[TMP14:%.*]] = and <8 x i1> [[TMP10]], [[TMP11]]
+; CHECK-NEXT:    [[TMP15:%.*]] = and <8 x i1> [[_MSPROP_ICMP]], [[TMP12]]
+; CHECK-NEXT:    [[TMP16:%.*]] = or <8 x i1> [[TMP13]], [[TMP14]]
+; CHECK-NEXT:    [[TMP17:%.*]] = or <8 x i1> [[TMP16]], [[TMP15]]
+; CHECK-NEXT:    [[TMP18:%.*]] = and <8 x i1> [[TMP10]], [[TMP12]]
+; CHECK-NEXT:    [[TMP19:%.*]] = bitcast <8 x i1> [[TMP17]] to i8
+; CHECK-NEXT:    [[TMP20:%.*]] = bitcast <8 x i1> [[TMP18]] to i8
+; CHECK-NEXT:    store i8 [[TMP19]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i8 [[TMP20]]
+;
+  %res = call i8 @llvm.x86.avx512.mask.pcmpeq.q.512(<8 x i64> %a, <8 x i64> %b, i8 %mask)
+  ret i8 %res
+}
+
+declare i8 @llvm.x86.avx512.mask.pcmpeq.q.512(<8 x i64>, <8 x i64>, i8)
+
+define i16 @test_pcmpgt_d(<16 x i32> %a, <16 x i32> %b)  #0 {
+; CHECK-LABEL: @test_pcmpgt_d(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = xor <16 x i32> [[A:%.*]], splat (i32 -2147483648)
+; CHECK-NEXT:    [[TMP4:%.*]] = xor <16 x i32> [[TMP1]], splat (i32 -1)
+; CHECK-NEXT:    [[TMP5:%.*]] = and <16 x i32> [[TMP3]], [[TMP4]]
+; CHECK-NEXT:    [[TMP6:%.*]] = or <16 x i32> [[TMP3]], [[TMP1]]
+; CHECK-NEXT:    [[TMP7:%.*]] = xor <16 x i32> [[B:%.*]], splat (i32 -2147483648)
+; CHECK-NEXT:    [[TMP8:%.*]] = xor <16 x i32> [[TMP2]], splat (i32 -1)
+; CHECK-NEXT:    [[TMP9:%.*]] = and <16 x i32> [[TMP7]], [[TMP8]]
+; CHECK-NEXT:    [[TMP10:%.*]] = or <16 x i32> [[TMP7]], [[TMP2]]
+; CHECK-NEXT:    [[TMP11:%.*]] = icmp ugt <16 x i32> [[TMP5]], [[TMP10]]
+; CHECK-NEXT:    [[TMP12:%.*]] = icmp ugt <16 x i32> [[TMP6]], [[TMP9]]
+; CHECK-NEXT:    [[TMP13:%.*]] = xor <16 x i1> [[TMP11]], [[TMP12]]
+; CHECK-NEXT:    [[TMP14:%.*]] = icmp sgt <16 x i32> [[A]], [[B]]
+; CHECK-NEXT:    [[TMP15:%.*]] = bitcast <16 x i1> [[TMP13]] to i16
+; CHECK-NEXT:    [[TMP16:%.*]] = bitcast <16 x i1> [[TMP14]] to i16
+; CHECK-NEXT:    store i16 [[TMP15]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i16 [[TMP16]]
+;
+  %res = call i16 @llvm.x86.avx512.mask.pcmpgt.d.512(<16 x i32> %a, <16 x i32> %b, i16 -1)
+  ret i16 %res
+}
+
+define i16 @test_mask_pcmpgt_d(<16 x i32> %a, <16 x i32> %b, i16 %mask)  #0 {
+;
+; CHECK-LABEL: @test_mask_pcmpgt_d(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = xor <16 x i32> [[A:%.*]], splat (i32 -2147483648)
+; CHECK-NEXT:    [[TMP5:%.*]] = xor <16 x i32> [[TMP1]], splat (i32 -1)
+; CHECK-NEXT:    [[TMP6:%.*]] = and <16 x i32> [[TMP4]], [[TMP5]]
+; CHECK-NEXT:    [[TMP7:%.*]] = or <16 x i32> [[TMP4]], [[TMP1]]
+; CHECK-NEXT:    [[TMP8:%.*]] = xor <16 x i32> [[B:%.*]], splat (i32 -2147483648)
+; CHECK-NEXT:    [[TMP9:%.*]] = xor <16 x i32> [[TMP2]], splat (i32 -1)
+; CHECK-NEXT:    [[TMP10:%.*]] = and <16 x i32> [[TMP8]], [[TMP9]]
+; CHECK-NEXT:    [[TMP11:%.*]] = or <16 x i32> [[TMP8]], [[TMP2]]
+; CHECK-NEXT:    [[TMP12:%.*]] = icmp ugt <16 x i32> [[TMP6]], [[TMP11]]
+; CHECK-NEXT:    [[TMP13:%.*]] = icmp ugt <16 x i32> [[TMP7]], [[TMP10]]
+; CHECK-NEXT:    [[TMP14:%.*]] = xor <16 x i1> [[TMP12]], [[TMP13]]
+; CHECK-NEXT:    [[TMP15:%.*]] = icmp sgt <16 x i32> [[A]], [[B]]
+; CHECK-NEXT:    [[TMP16:%.*]] = bitcast i16 [[TMP3]] to <16 x i1>
+; CHECK-NEXT:    [[TMP17:%.*]] = bitcast i16 [[MASK:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP18:%.*]] = and <16 x i1> [[TMP14]], [[TMP16]]
+; CHECK-NEXT:    [[TMP19:%.*]] = and <16 x i1> [[TMP15]], [[TMP16]]
+; CHECK-NEXT:    [[TMP20:%.*]] = and <16 x i1> [[TMP14]], [[TMP17]]
+; CHECK-NEXT:    [[TMP21:%.*]] = or <16 x i1> [[TMP18]], [[TMP19]]
+; CHECK-NEXT:    [[TMP22:%.*]] = or <16 x i1> [[TMP21]], [[TMP20]]
+; CHECK-NEXT:    [[TMP23:%.*]] = and <16 x i1> [[TMP15]], [[TMP17]]
+; CHECK-NEXT:    [[TMP24:%.*]] = bitcast <16 x i1> [[TMP22]] to i16
+; CHECK-NEXT:    [[TMP25:%.*]] = bitcast <16 x i1> [[TMP23]] to i16
+; CHECK-NEXT:    store i16 [[TMP24]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i16 [[TMP25]]
+;
+  %res = call i16 @llvm.x86.avx512.mask.pcmpgt.d.512(<16 x i32> %a, <16 x i32> %b, i16 %mask)
+  ret i16 %res
+}
+
+declare i16 @llvm.x86.avx512.mask.pcmpgt.d.512(<16 x i32>, <16 x i32>, i16)
+
+define i8 @test_pcmpgt_q(<8 x i64> %a, <8 x i64> %b)  #0 {
+; CHECK-LABEL: @test_pcmpgt_q(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = xor <8 x i64> [[A:%.*]], splat (i64 -9223372036854775808)
+; CHECK-NEXT:    [[TMP4:%.*]] = xor <8 x i64> [[TMP1]], splat (i64 -1)
+; CHECK-NEXT:    [[TMP5:%.*]] = and <8 x i64> [[TMP3]], [[TMP4]]
+; CHECK-NEXT:    [[TMP6:%.*]] = or <8 x i64> [[TMP3]], [[TMP1]]
+; CHECK-NEXT:    [[TMP7:%.*]] = xor <8 x i64> [[B:%.*]], splat (i64 -9223372036854775808)
+; CHECK-NEXT:    [[TMP8:%.*]] = xor <8 x i64> [[TMP2]], splat (i64 -1)
+; CHECK-NEXT:    [[TMP9:%.*]] = and <8 x i64> [[TMP7]], [[TMP8]]
+; CHECK-NEXT:    [[TMP10:%.*]] = or <8 x i64> [[TMP7]], [[TMP2]]
+; CHECK-NEXT:    [[TMP11:%.*]] = icmp ugt <8 x i64> [[TMP5]], [[TMP10]]
+; CHECK-NEXT:    [[TMP12:%.*]] = icmp ugt <8 x i64> [[TMP6]], [[TMP9]]
+; CHECK-NEXT:    [[TMP13:%.*]] = xor <8 x i1> [[TMP11]], [[TMP12]]
+; CHECK-NEXT:    [[TMP14:%.*]] = icmp sgt <8 x i64> [[A]], [[B]]
+; CHECK-NEXT:    [[TMP15:%.*]] = bitcast <8 x i1> [[TMP13]] to i8
+; CHECK-NEXT:    [[TMP16:%.*]] = bitcast <8 x i1> [[TMP14]] to i8
+; CHECK-NEXT:    store i8 [[TMP15]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i8 [[TMP16]]
+;
+  %res = call i8 @llvm.x86.avx512.mask.pcmpgt.q.512(<8 x i64> %a, <8 x i64> %b, i8 -1)
+  ret i8 %res
+}
+
+define i8 @test_mask_pcmpgt_q(<8 x i64> %a, <8 x i64> %b, i8 %mask)  #0 {
+;
+; CHECK-LABEL: @test_mask_pcmpgt_q(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = xor <8 x i64> [[A:%.*]], splat (i64 -9223372036854775808)
+; CHECK-NEXT:    [[TMP5:%.*]] = xor <8 x i64> [[TMP1]], splat (i64 -1)
+; CHECK-NEXT:    [[TMP6:%.*]] = and <8 x i64> [[TMP4]], [[TMP5]]
+; CHECK-NEXT:    [[TMP7:%.*]] = or <8 x i64> [[TMP4]], [[TMP1]]
+; CHECK-NEXT:    [[TMP8:%.*]] = xor <8 x i64> [[B:%.*]], splat (i64 -9223372036854775808)
+; CHECK-NEXT:    [[TMP9:%.*]] = xor <8 x i64> [[TMP2]], splat (i64 -1)
+; CHECK-NEXT:    [[TMP10:%.*]] = and <8 x i64> [[TMP8]], [[TMP9]]
+; CHECK-NEXT:    [[TMP11:%.*]] = or <8 x i64> [[TMP8]], [[TMP2]]
+; CHECK-NEXT:    [[TMP12:%.*]] = icmp ugt <8 x i64> [[TMP6]], [[TMP11]]
+; CHECK-NEXT:    [[TMP13:%.*]] = icmp ugt <8 x i64> [[TMP7]], [[TMP10]]
+; CHECK-NEXT:    [[TMP14:%.*]] = xor <8 x i1> [[TMP12]], [[TMP13]]
+; CHECK-NEXT:    [[TMP15:%.*]] = icmp sgt <8 x i64> [[A]], [[B]]
+; CHECK-NEXT:    [[TMP16:%.*]] = bitcast i8 [[TMP3]] to <8 x i1>
+; CHECK-NEXT:    [[TMP17:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP18:%.*]] = and <8 x i1> [[TMP14]], [[TMP16]]
+; CHECK-NEXT:    [[TMP19:%.*]] = and <8 x i1> [[TMP15]], [[TMP16]]
+; CHECK-NEXT:    [[TMP20:%.*]] = and <8 x i1> [[TMP14]], [[TMP17]]
+; CHECK-NEXT:    [[TMP21:%.*]] = or <8 x i1> [[TMP18]], [[TMP19]]
+; CHECK-NEXT:    [[TMP22:%.*]] = or <8 x i1> [[TMP21]], [[TMP20]]
+; CHECK-NEXT:    [[TMP23:%.*]] = and <8 x i1> [[TMP15]], [[TMP17]]
+; CHECK-NEXT:    [[TMP24:%.*]] = bitcast <8 x i1> [[TMP22]] to i8
+; CHECK-NEXT:    [[TMP25:%.*]] = bitcast <8 x i1> [[TMP23]] to i8
+; CHECK-NEXT:    store i8 [[TMP24]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i8 [[TMP25]]
+;
+  %res = call i8 @llvm.x86.avx512.mask.pcmpgt.q.512(<8 x i64> %a, <8 x i64> %b, i8 %mask)
+  ret i8 %res
+}
+
+declare i8 @llvm.x86.avx512.mask.pcmpgt.q.512(<8 x i64>, <8 x i64>, i8)
+
+declare <8 x double> @llvm.x86.avx512.mask.unpckh.pd.512(<8 x double>, <8 x double>, <8 x double>, i8)
+
+define <8 x double>@test_int_x86_avx512_unpckh_pd_512(<8 x double> %x0, <8 x double> %x1, <8 x double> %x2)  #0 {
+; CHECK-LABEL: @test_int_x86_avx512_unpckh_pd_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <8 x i64> [[TMP1]], <8 x i64> [[TMP2]], <8 x i32> <i32 1, i32 9, i32 3, i32 11, i32 5, i32 13, i32 7, i32 15>
+; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <8 x double> [[X0:%.*]], <8 x double> [[X1:%.*]], <8 x i32> <i32 1, i32 9, i32 3, i32 11, i32 5, i32 13, i32 7, i32 15>
+; CHECK-NEXT:    store <8 x i64> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x double> [[TMP3]]
+;
+  %res = call <8 x double> @llvm.x86.avx512.mask.unpckh.pd.512(<8 x double> %x0, <8 x double> %x1, <8 x double> %x2, i8 -1)
+  ret <8 x double> %res
+}
+
+define <8 x double>@test_int_x86_avx512_mask_unpckh_pd_512(<8 x double> %x0, <8 x double> %x1, <8 x double> %x2, i8 %x3)  #0 {
+;
+; CHECK-LABEL: @test_int_x86_avx512_mask_unpckh_pd_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <8 x i64> [[TMP1]], <8 x i64> [[TMP2]], <8 x i32> <i32 1, i32 9, i32 3, i32 11, i32 5, i32 13, i32 7, i32 15>
+; CHECK-NEXT:    [[TMP5:%.*]] = shufflevector <8 x double> [[X0:%.*]], <8 x double> [[X1:%.*]], <8 x i32> <i32 1, i32 9, i32 3, i32 11, i32 5, i32 13, i32 7, i32 15>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i8 [[TMP3]] to <8 x i1>
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast i8 [[X3:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP8:%.*]] = select <8 x i1> [[TMP7]], <8 x i64> [[_MSPROP]], <8 x i64> [[TMP4]]
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <8 x double> [[TMP5]] to <8 x i64>
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast <8 x double> [[X2:%.*]] to <8 x i64>
+; CHECK-NEXT:    [[TMP11:%.*]] = xor <8 x i64> [[TMP9]], [[TMP10]]
+; CHECK-NEXT:    [[TMP12:%.*]] = or <8 x i64> [[TMP11]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP13:%.*]] = or <8 x i64> [[TMP12]], [[TMP4]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP6]], <8 x i64> [[TMP13]], <8 x i64> [[TMP8]]
+; CHECK-NEXT:    [[TMP14:%.*]] = select <8 x i1> [[TMP7]], <8 x double> [[TMP5]], <8 x double> [[X2]]
+; CHECK-NEXT:    store <8 x i64> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x double> [[TMP14]]
+;
+  %res = call <8 x double> @llvm.x86.avx512.mask.unpckh.pd.512(<8 x double> %x0, <8 x double> %x1, <8 x double> %x2, i8 %x3)
+  ret <8 x double> %res
+}
+
+declare <16 x float> @llvm.x86.avx512.mask.unpckh.ps.512(<16 x float>, <16 x float>, <16 x float>, i16)
+
+define <16 x float>@test_int_x86_avx512_unpckh_ps_512(<16 x float> %x0, <16 x float> %x1, <16 x float> %x2)  #0 {
+; CHECK-LABEL: @test_int_x86_avx512_unpckh_ps_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <16 x i32> [[TMP1]], <16 x i32> [[TMP2]], <16 x i32> <i32 2, i32 18, i32 3, i32 19, i32 6, i32 22, i32 7, i32 23, i32 10, i32 26, i32 11, i32 27, i32 14, i32 30, i32 15, i32 31>
+; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <16 x float> [[X0:%.*]], <16 x float> [[X1:%.*]], <16 x i32> <i32 2, i32 18, i32 3, i32 19, i32 6, i32 22, i32 7, i32 23, i32 10, i32 26, i32 11, i32 27, i32 14, i32 30, i32 15, i32 31>
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x float> [[TMP3]]
+;
+  %res = call <16 x float> @llvm.x86.avx512.mask.unpckh.ps.512(<16 x float> %x0, <16 x float> %x1, <16 x float> %x2, i16 -1)
+  ret <16 x float> %res
+}
+
+define <16 x float>@test_int_x86_avx512_mask_unpckh_ps_512(<16 x float> %x0, <16 x float> %x1, <16 x float> %x2, i16 %x3)  #0 {
+;
+; CHECK-LABEL: @test_int_x86_avx512_mask_unpckh_ps_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <16 x i32> [[TMP1]], <16 x i32> [[TMP2]], <16 x i32> <i32 2, i32 18, i32 3, i32 19, i32 6, i32 22, i32 7, i32 23, i32 10, i32 26, i32 11, i32 27, i32 14, i32 30, i32 15, i32 31>
+; CHECK-NEXT:    [[TMP5:%.*]] = shufflevector <16 x float> [[X0:%.*]], <16 x float> [[X1:%.*]], <16 x i32> <i32 2, i32 18, i32 3, i32 19, i32 6, i32 22, i32 7, i32 23, i32 10, i32 26, i32 11, i32 27, i32 14, i32 30, i32 15, i32 31>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i16 [[TMP3]] to <16 x i1>
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast i16 [[X3:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP8:%.*]] = select <16 x i1> [[TMP7]], <16 x i32> [[_MSPROP]], <16 x i32> [[TMP4]]
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <16 x float> [[TMP5]] to <16 x i32>
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast <16 x float> [[X2:%.*]] to <16 x i32>
+; CHECK-NEXT:    [[TMP11:%.*]] = xor <16 x i32> [[TMP9]], [[TMP10]]
+; CHECK-NEXT:    [[TMP12:%.*]] = or <16 x i32> [[TMP11]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP13:%.*]] = or <16 x i32> [[TMP12]], [[TMP4]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP6]], <16 x i32> [[TMP13]], <16 x i32> [[TMP8]]
+; CHECK-NEXT:    [[TMP14:%.*]] = select <16 x i1> [[TMP7]], <16 x float> [[TMP5]], <16 x float> [[X2]]
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x float> [[TMP14]]
+;
+  %res = call <16 x float> @llvm.x86.avx512.mask.unpckh.ps.512(<16 x float> %x0, <16 x float> %x1, <16 x float> %x2, i16 %x3)
+  ret <16 x float> %res
+}
+
+declare <8 x double> @llvm.x86.avx512.mask.unpckl.pd.512(<8 x double>, <8 x double>, <8 x double>, i8)
+
+define <8 x double>@test_int_x86_avx512_unpckl_pd_512(<8 x double> %x0, <8 x double> %x1, <8 x double> %x2)  #0 {
+; CHECK-LABEL: @test_int_x86_avx512_unpckl_pd_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <8 x i64> [[TMP1]], <8 x i64> [[TMP2]], <8 x i32> <i32 0, i32 8, i32 2, i32 10, i32 4, i32 12, i32 6, i32 14>
+; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <8 x double> [[X0:%.*]], <8 x double> [[X1:%.*]], <8 x i32> <i32 0, i32 8, i32 2, i32 10, i32 4, i32 12, i32 6, i32 14>
+; CHECK-NEXT:    store <8 x i64> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x double> [[TMP3]]
+;
+  %res = call <8 x double> @llvm.x86.avx512.mask.unpckl.pd.512(<8 x double> %x0, <8 x double> %x1, <8 x double> %x2, i8 -1)
+  ret <8 x double> %res
+}
+
+define <8 x double>@test_int_x86_avx512_mask_unpckl_pd_512(<8 x double> %x0, <8 x double> %x1, <8 x double> %x2, i8 %x3)  #0 {
+;
+; CHECK-LABEL: @test_int_x86_avx512_mask_unpckl_pd_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <8 x i64> [[TMP1]], <8 x i64> [[TMP2]], <8 x i32> <i32 0, i32 8, i32 2, i32 10, i32 4, i32 12, i32 6, i32 14>
+; CHECK-NEXT:    [[TMP5:%.*]] = shufflevector <8 x double> [[X0:%.*]], <8 x double> [[X1:%.*]], <8 x i32> <i32 0, i32 8, i32 2, i32 10, i32 4, i32 12, i32 6, i32 14>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i8 [[TMP3]] to <8 x i1>
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast i8 [[X3:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP8:%.*]] = select <8 x i1> [[TMP7]], <8 x i64> [[_MSPROP]], <8 x i64> [[TMP4]]
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <8 x double> [[TMP5]] to <8 x i64>
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast <8 x double> [[X2:%.*]] to <8 x i64>
+; CHECK-NEXT:    [[TMP11:%.*]] = xor <8 x i64> [[TMP9]], [[TMP10]]
+; CHECK-NEXT:    [[TMP12:%.*]] = or <8 x i64> [[TMP11]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP13:%.*]] = or <8 x i64> [[TMP12]], [[TMP4]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP6]], <8 x i64> [[TMP13]], <8 x i64> [[TMP8]]
+; CHECK-NEXT:    [[TMP14:%.*]] = select <8 x i1> [[TMP7]], <8 x double> [[TMP5]], <8 x double> [[X2]]
+; CHECK-NEXT:    store <8 x i64> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x double> [[TMP14]]
+;
+  %res = call <8 x double> @llvm.x86.avx512.mask.unpckl.pd.512(<8 x double> %x0, <8 x double> %x1, <8 x double> %x2, i8 %x3)
+  ret <8 x double> %res
+}
+
+declare <16 x float> @llvm.x86.avx512.mask.unpckl.ps.512(<16 x float>, <16 x float>, <16 x float>, i16)
+
+define <16 x float>@test_int_x86_avx512_unpckl_ps_512(<16 x float> %x0, <16 x float> %x1, <16 x float> %x2)  #0 {
+; CHECK-LABEL: @test_int_x86_avx512_unpckl_ps_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <16 x i32> [[TMP1]], <16 x i32> [[TMP2]], <16 x i32> <i32 0, i32 16, i32 1, i32 17, i32 4, i32 20, i32 5, i32 21, i32 8, i32 24, i32 9, i32 25, i32 12, i32 28, i32 13, i32 29>
+; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <16 x float> [[X0:%.*]], <16 x float> [[X1:%.*]], <16 x i32> <i32 0, i32 16, i32 1, i32 17, i32 4, i32 20, i32 5, i32 21, i32 8, i32 24, i32 9, i32 25, i32 12, i32 28, i32 13, i32 29>
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x float> [[TMP3]]
+;
+  %res = call <16 x float> @llvm.x86.avx512.mask.unpckl.ps.512(<16 x float> %x0, <16 x float> %x1, <16 x float> %x2, i16 -1)
+  ret <16 x float> %res
+}
+
+define <16 x float>@test_int_x86_avx512_mask_unpckl_ps_512(<16 x float> %x0, <16 x float> %x1, <16 x float> %x2, i16 %x3)  #0 {
+;
+; CHECK-LABEL: @test_int_x86_avx512_mask_unpckl_ps_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <16 x i32> [[TMP1]], <16 x i32> [[TMP2]], <16 x i32> <i32 0, i32 16, i32 1, i32 17, i32 4, i32 20, i32 5, i32 21, i32 8, i32 24, i32 9, i32 25, i32 12, i32 28, i32 13, i32 29>
+; CHECK-NEXT:    [[TMP5:%.*]] = shufflevector <16 x float> [[X0:%.*]], <16 x float> [[X1:%.*]], <16 x i32> <i32 0, i32 16, i32 1, i32 17, i32 4, i32 20, i32 5, i32 21, i32 8, i32 24, i32 9, i32 25, i32 12, i32 28, i32 13, i32 29>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i16 [[TMP3]] to <16 x i1>
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast i16 [[X3:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP8:%.*]] = select <16 x i1> [[TMP7]], <16 x i32> [[_MSPROP]], <16 x i32> [[TMP4]]
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <16 x float> [[TMP5]] to <16 x i32>
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast <16 x float> [[X2:%.*]] to <16 x i32>
+; CHECK-NEXT:    [[TMP11:%.*]] = xor <16 x i32> [[TMP9]], [[TMP10]]
+; CHECK-NEXT:    [[TMP12:%.*]] = or <16 x i32> [[TMP11]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP13:%.*]] = or <16 x i32> [[TMP12]], [[TMP4]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP6]], <16 x i32> [[TMP13]], <16 x i32> [[TMP8]]
+; CHECK-NEXT:    [[TMP14:%.*]] = select <16 x i1> [[TMP7]], <16 x float> [[TMP5]], <16 x float> [[X2]]
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x float> [[TMP14]]
+;
+  %res = call <16 x float> @llvm.x86.avx512.mask.unpckl.ps.512(<16 x float> %x0, <16 x float> %x1, <16 x float> %x2, i16 %x3)
+  ret <16 x float> %res
+}
+
+declare <8 x i64> @llvm.x86.avx512.mask.punpcklqd.q.512(<8 x i64>, <8 x i64>, <8 x i64>, i8)
+
+define <8 x i64>@test_int_x86_avx512_punpcklqd_q_512(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> %x2)  #0 {
+; CHECK-LABEL: @test_int_x86_avx512_punpcklqd_q_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <8 x i64> [[TMP1]], <8 x i64> [[TMP2]], <8 x i32> <i32 0, i32 8, i32 2, i32 10, i32 4, i32 12, i32 6, i32 14>
+; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <8 x i64> [[X0:%.*]], <8 x i64> [[X1:%.*]], <8 x i32> <i32 0, i32 8, i32 2, i32 10, i32 4, i32 12, i32 6, i32 14>
+; CHECK-NEXT:    store <8 x i64> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i64> [[TMP3]]
+;
+  %res = call <8 x i64> @llvm.x86.avx512.mask.punpcklqd.q.512(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> %x2, i8 -1)
+  ret <8 x i64> %res
+}
+
+define <8 x i64>@test_int_x86_avx512_mask_punpcklqd_q_512(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> %x2, i8 %x3)  #0 {
+;
+; CHECK-LABEL: @test_int_x86_avx512_mask_punpcklqd_q_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <8 x i64> [[TMP1]], <8 x i64> [[TMP2]], <8 x i32> <i32 0, i32 8, i32 2, i32 10, i32 4, i32 12, i32 6, i32 14>
+; CHECK-NEXT:    [[TMP5:%.*]] = shufflevector <8 x i64> [[X0:%.*]], <8 x i64> [[X1:%.*]], <8 x i32> <i32 0, i32 8, i32 2, i32 10, i32 4, i32 12, i32 6, i32 14>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i8 [[TMP3]] to <8 x i1>
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast i8 [[X3:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP8:%.*]] = select <8 x i1> [[TMP7]], <8 x i64> [[_MSPROP]], <8 x i64> [[TMP4]]
+; CHECK-NEXT:    [[TMP9:%.*]] = xor <8 x i64> [[TMP5]], [[X2:%.*]]
+; CHECK-NEXT:    [[TMP10:%.*]] = or <8 x i64> [[TMP9]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP11:%.*]] = or <8 x i64> [[TMP10]], [[TMP4]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP6]], <8 x i64> [[TMP11]], <8 x i64> [[TMP8]]
+; CHECK-NEXT:    [[TMP12:%.*]] = select <8 x i1> [[TMP7]], <8 x i64> [[TMP5]], <8 x i64> [[X2]]
+; CHECK-NEXT:    store <8 x i64> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i64> [[TMP12]]
+;
+  %res = call <8 x i64> @llvm.x86.avx512.mask.punpcklqd.q.512(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> %x2, i8 %x3)
+  ret <8 x i64> %res
+}
+
+define <8 x i64>@test_int_x86_avx512_maskz_punpcklqd_q_512(<8 x i64> %x0, <8 x i64> %x1, i8 %x3)  #0 {
+;
+; CHECK-LABEL: @test_int_x86_avx512_maskz_punpcklqd_q_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <8 x i64> [[TMP1]], <8 x i64> [[TMP2]], <8 x i32> <i32 0, i32 8, i32 2, i32 10, i32 4, i32 12, i32 6, i32 14>
+; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <8 x i64> [[X0:%.*]], <8 x i64> [[X1:%.*]], <8 x i32> <i32 0, i32 8, i32 2, i32 10, i32 4, i32 12, i32 6, i32 14>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i8 [[TMP3]] to <8 x i1>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i8 [[X3:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP7:%.*]] = select <8 x i1> [[TMP6]], <8 x i64> [[_MSPROP]], <8 x i64> zeroinitializer
+; CHECK-NEXT:    [[TMP8:%.*]] = xor <8 x i64> [[TMP4]], zeroinitializer
+; CHECK-NEXT:    [[TMP9:%.*]] = or <8 x i64> [[TMP8]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP10:%.*]] = or <8 x i64> [[TMP9]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP5]], <8 x i64> [[TMP10]], <8 x i64> [[TMP7]]
+; CHECK-NEXT:    [[TMP11:%.*]] = select <8 x i1> [[TMP6]], <8 x i64> [[TMP4]], <8 x i64> zeroinitializer
+; CHECK-NEXT:    store <8 x i64> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i64> [[TMP11]]
+;
+  %res = call <8 x i64> @llvm.x86.avx512.mask.punpcklqd.q.512(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> zeroinitializer,i8 %x3)
+  ret <8 x i64> %res
+}
+
+declare <8 x i64> @llvm.x86.avx512.mask.punpckhqd.q.512(<8 x i64>, <8 x i64>, <8 x i64>, i8)
+
+define <8 x i64>@test_int_x86_avx512_punpckhqd_q_512(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> %x2)  #0 {
+; CHECK-LABEL: @test_int_x86_avx512_punpckhqd_q_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <8 x i64> [[TMP1]], <8 x i64> [[TMP2]], <8 x i32> <i32 1, i32 9, i32 3, i32 11, i32 5, i32 13, i32 7, i32 15>
+; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <8 x i64> [[X0:%.*]], <8 x i64> [[X1:%.*]], <8 x i32> <i32 1, i32 9, i32 3, i32 11, i32 5, i32 13, i32 7, i32 15>
+; CHECK-NEXT:    store <8 x i64> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i64> [[TMP3]]
+;
+  %res = call <8 x i64> @llvm.x86.avx512.mask.punpckhqd.q.512(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> %x2, i8 -1)
+  ret <8 x i64> %res
+}
+
+define <8 x i64>@test_int_x86_avx512_mask_punpckhqd_q_512(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> %x2, i8 %x3)  #0 {
+;
+; CHECK-LABEL: @test_int_x86_avx512_mask_punpckhqd_q_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <8 x i64> [[TMP1]], <8 x i64> [[TMP2]], <8 x i32> <i32 1, i32 9, i32 3, i32 11, i32 5, i32 13, i32 7, i32 15>
+; CHECK-NEXT:    [[TMP5:%.*]] = shufflevector <8 x i64> [[X0:%.*]], <8 x i64> [[X1:%.*]], <8 x i32> <i32 1, i32 9, i32 3, i32 11, i32 5, i32 13, i32 7, i32 15>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i8 [[TMP3]] to <8 x i1>
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast i8 [[X3:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP8:%.*]] = select <8 x i1> [[TMP7]], <8 x i64> [[_MSPROP]], <8 x i64> [[TMP4]]
+; CHECK-NEXT:    [[TMP9:%.*]] = xor <8 x i64> [[TMP5]], [[X2:%.*]]
+; CHECK-NEXT:    [[TMP10:%.*]] = or <8 x i64> [[TMP9]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP11:%.*]] = or <8 x i64> [[TMP10]], [[TMP4]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP6]], <8 x i64> [[TMP11]], <8 x i64> [[TMP8]]
+; CHECK-NEXT:    [[TMP12:%.*]] = select <8 x i1> [[TMP7]], <8 x i64> [[TMP5]], <8 x i64> [[X2]]
+; CHECK-NEXT:    store <8 x i64> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i64> [[TMP12]]
+;
+  %res = call <8 x i64> @llvm.x86.avx512.mask.punpckhqd.q.512(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> %x2, i8 %x3)
+  ret <8 x i64> %res
+}
+
+declare <16 x i32> @llvm.x86.avx512.mask.punpckhd.q.512(<16 x i32>, <16 x i32>, <16 x i32>, i16)
+
+define <16 x i32>@test_int_x86_avx512_punpckhd_q_512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x2)  #0 {
+; CHECK-LABEL: @test_int_x86_avx512_punpckhd_q_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <16 x i32> [[TMP1]], <16 x i32> [[TMP2]], <16 x i32> <i32 2, i32 18, i32 3, i32 19, i32 6, i32 22, i32 7, i32 23, i32 10, i32 26, i32 11, i32 27, i32 14, i32 30, i32 15, i32 31>
+; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <16 x i32> [[X0:%.*]], <16 x i32> [[X1:%.*]], <16 x i32> <i32 2, i32 18, i32 3, i32 19, i32 6, i32 22, i32 7, i32 23, i32 10, i32 26, i32 11, i32 27, i32 14, i32 30, i32 15, i32 31>
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i32> [[TMP3]]
+;
+  %res = call <16 x i32> @llvm.x86.avx512.mask.punpckhd.q.512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x2, i16 -1)
+  ret <16 x i32> %res
+}
+
+define <16 x i32>@test_int_x86_avx512_mask_punpckhd_q_512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x2, i16 %x3)  #0 {
+;
+; CHECK-LABEL: @test_int_x86_avx512_mask_punpckhd_q_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <16 x i32> [[TMP1]], <16 x i32> [[TMP2]], <16 x i32> <i32 2, i32 18, i32 3, i32 19, i32 6, i32 22, i32 7, i32 23, i32 10, i32 26, i32 11, i32 27, i32 14, i32 30, i32 15, i32 31>
+; CHECK-NEXT:    [[TMP5:%.*]] = shufflevector <16 x i32> [[X0:%.*]], <16 x i32> [[X1:%.*]], <16 x i32> <i32 2, i32 18, i32 3, i32 19, i32 6, i32 22, i32 7, i32 23, i32 10, i32 26, i32 11, i32 27, i32 14, i32 30, i32 15, i32 31>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i16 [[TMP3]] to <16 x i1>
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast i16 [[X3:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP8:%.*]] = select <16 x i1> [[TMP7]], <16 x i32> [[_MSPROP]], <16 x i32> [[TMP4]]
+; CHECK-NEXT:    [[TMP9:%.*]] = xor <16 x i32> [[TMP5]], [[X2:%.*]]
+; CHECK-NEXT:    [[TMP10:%.*]] = or <16 x i32> [[TMP9]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP11:%.*]] = or <16 x i32> [[TMP10]], [[TMP4]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP6]], <16 x i32> [[TMP11]], <16 x i32> [[TMP8]]
+; CHECK-NEXT:    [[TMP12:%.*]] = select <16 x i1> [[TMP7]], <16 x i32> [[TMP5]], <16 x i32> [[X2]]
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i32> [[TMP12]]
+;
+  %res = call <16 x i32> @llvm.x86.avx512.mask.punpckhd.q.512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x2, i16 %x3)
+  ret <16 x i32> %res
+}
+
+declare <16 x i32> @llvm.x86.avx512.mask.punpckld.q.512(<16 x i32>, <16 x i32>, <16 x i32>, i16)
+
+define <16 x i32>@test_int_x86_avx512_punpckld_q_512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x2)  #0 {
+; CHECK-LABEL: @test_int_x86_avx512_punpckld_q_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <16 x i32> [[TMP1]], <16 x i32> [[TMP2]], <16 x i32> <i32 0, i32 16, i32 1, i32 17, i32 4, i32 20, i32 5, i32 21, i32 8, i32 24, i32 9, i32 25, i32 12, i32 28, i32 13, i32 29>
+; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <16 x i32> [[X0:%.*]], <16 x i32> [[X1:%.*]], <16 x i32> <i32 0, i32 16, i32 1, i32 17, i32 4, i32 20, i32 5, i32 21, i32 8, i32 24, i32 9, i32 25, i32 12, i32 28, i32 13, i32 29>
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i32> [[TMP3]]
+;
+  %res = call <16 x i32> @llvm.x86.avx512.mask.punpckld.q.512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x2, i16 -1)
+  ret <16 x i32> %res
+}
+
+define <16 x i32>@test_int_x86_avx512_mask_punpckld_q_512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x2, i16 %x3)  #0 {
+;
+; CHECK-LABEL: @test_int_x86_avx512_mask_punpckld_q_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <16 x i32> [[TMP1]], <16 x i32> [[TMP2]], <16 x i32> <i32 0, i32 16, i32 1, i32 17, i32 4, i32 20, i32 5, i32 21, i32 8, i32 24, i32 9, i32 25, i32 12, i32 28, i32 13, i32 29>
+; CHECK-NEXT:    [[TMP5:%.*]] = shufflevector <16 x i32> [[X0:%.*]], <16 x i32> [[X1:%.*]], <16 x i32> <i32 0, i32 16, i32 1, i32 17, i32 4, i32 20, i32 5, i32 21, i32 8, i32 24, i32 9, i32 25, i32 12, i32 28, i32 13, i32 29>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i16 [[TMP3]] to <16 x i1>
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast i16 [[X3:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP8:%.*]] = select <16 x i1> [[TMP7]], <16 x i32> [[_MSPROP]], <16 x i32> [[TMP4]]
+; CHECK-NEXT:    [[TMP9:%.*]] = xor <16 x i32> [[TMP5]], [[X2:%.*]]
+; CHECK-NEXT:    [[TMP10:%.*]] = or <16 x i32> [[TMP9]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP11:%.*]] = or <16 x i32> [[TMP10]], [[TMP4]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP6]], <16 x i32> [[TMP11]], <16 x i32> [[TMP8]]
+; CHECK-NEXT:    [[TMP12:%.*]] = select <16 x i1> [[TMP7]], <16 x i32> [[TMP5]], <16 x i32> [[X2]]
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i32> [[TMP12]]
+;
+  %res = call <16 x i32> @llvm.x86.avx512.mask.punpckld.q.512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x2, i16 %x3)
+  ret <16 x i32> %res
+}
+
+define <16 x i32> @test_x86_avx512_pslli_d(<16 x i32> %a0)  #0 {
+; CHECK-LABEL: @test_x86_avx512_pslli_d(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = call <16 x i32> @llvm.x86.avx512.pslli.d.512(<16 x i32> [[TMP1]], i32 7)
+; CHECK-NEXT:    [[TMP3:%.*]] = or <16 x i32> [[TMP2]], zeroinitializer
+; CHECK-NEXT:    [[TMP4:%.*]] = call <16 x i32> @llvm.x86.avx512.pslli.d.512(<16 x i32> [[A0:%.*]], i32 7)
+; CHECK-NEXT:    store <16 x i32> [[TMP3]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i32> [[TMP4]]
+;
+  %res = call <16 x i32> @llvm.x86.avx512.mask.pslli.d(<16 x i32> %a0, i32 7, <16 x i32> zeroinitializer, i16 -1)
+  ret <16 x i32> %res
+}
+
+define <16 x i32> @test_x86_avx512_mask_pslli_d(<16 x i32> %a0, <16 x i32> %a1, i16 %mask)  #0 {
+;
+; CHECK-LABEL: @test_x86_avx512_mask_pslli_d(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = call <16 x i32> @llvm.x86.avx512.pslli.d.512(<16 x i32> [[TMP1]], i32 7)
+; CHECK-NEXT:    [[TMP5:%.*]] = or <16 x i32> [[TMP4]], zeroinitializer
+; CHECK-NEXT:    [[TMP6:%.*]] = call <16 x i32> @llvm.x86.avx512.pslli.d.512(<16 x i32> [[A0:%.*]], i32 7)
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast i16 [[TMP2]] to <16 x i1>
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast i16 [[MASK:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP9:%.*]] = select <16 x i1> [[TMP8]], <16 x i32> [[TMP5]], <16 x i32> [[TMP3]]
+; CHECK-NEXT:    [[TMP10:%.*]] = xor <16 x i32> [[TMP6]], [[A1:%.*]]
+; CHECK-NEXT:    [[TMP11:%.*]] = or <16 x i32> [[TMP10]], [[TMP5]]
+; CHECK-NEXT:    [[TMP12:%.*]] = or <16 x i32> [[TMP11]], [[TMP3]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP7]], <16 x i32> [[TMP12]], <16 x i32> [[TMP9]]
+; CHECK-NEXT:    [[TMP13:%.*]] = select <16 x i1> [[TMP8]], <16 x i32> [[TMP6]], <16 x i32> [[A1]]
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i32> [[TMP13]]
+;
+  %res = call <16 x i32> @llvm.x86.avx512.mask.pslli.d(<16 x i32> %a0, i32 7, <16 x i32> %a1, i16 %mask)
+  ret <16 x i32> %res
+}
+
+define <16 x i32> @test_x86_avx512_maskz_pslli_d(<16 x i32> %a0, i16 %mask)  #0 {
+;
+; CHECK-LABEL: @test_x86_avx512_maskz_pslli_d(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = call <16 x i32> @llvm.x86.avx512.pslli.d.512(<16 x i32> [[TMP1]], i32 7)
+; CHECK-NEXT:    [[TMP4:%.*]] = or <16 x i32> [[TMP3]], zeroinitializer
+; CHECK-NEXT:    [[TMP5:%.*]] = call <16 x i32> @llvm.x86.avx512.pslli.d.512(<16 x i32> [[A0:%.*]], i32 7)
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i16 [[TMP2]] to <16 x i1>
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast i16 [[MASK:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP8:%.*]] = select <16 x i1> [[TMP7]], <16 x i32> [[TMP4]], <16 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP9:%.*]] = xor <16 x i32> [[TMP5]], zeroinitializer
+; CHECK-NEXT:    [[TMP10:%.*]] = or <16 x i32> [[TMP9]], [[TMP4]]
+; CHECK-NEXT:    [[TMP11:%.*]] = or <16 x i32> [[TMP10]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP6]], <16 x i32> [[TMP11]], <16 x i32> [[TMP8]]
+; CHECK-NEXT:    [[TMP12:%.*]] = select <16 x i1> [[TMP7]], <16 x i32> [[TMP5]], <16 x i32> zeroinitializer
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i32> [[TMP12]]
+;
+  %res = call <16 x i32> @llvm.x86.avx512.mask.pslli.d(<16 x i32> %a0, i32 7, <16 x i32> zeroinitializer, i16 %mask)
+  ret <16 x i32> %res
+}
+
+declare <16 x i32> @llvm.x86.avx512.mask.pslli.d(<16 x i32>, i32, <16 x i32>, i16) nounwind readnone
+
+define <8 x i64> @test_x86_avx512_pslli_q(<8 x i64> %a0)  #0 {
+; CHECK-LABEL: @test_x86_avx512_pslli_q(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = call <8 x i64> @llvm.x86.avx512.pslli.q.512(<8 x i64> [[TMP1]], i32 7)
+; CHECK-NEXT:    [[TMP3:%.*]] = or <8 x i64> [[TMP2]], zeroinitializer
+; CHECK-NEXT:    [[TMP4:%.*]] = call <8 x i64> @llvm.x86.avx512.pslli.q.512(<8 x i64> [[A0:%.*]], i32 7)
+; CHECK-NEXT:    store <8 x i64> [[TMP3]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i64> [[TMP4]]
+;
+  %res = call <8 x i64> @llvm.x86.avx512.mask.pslli.q(<8 x i64> %a0, i32 7, <8 x i64> zeroinitializer, i8 -1)
+  ret <8 x i64> %res
+}
+
+define <8 x i64> @test_x86_avx512_mask_pslli_q(<8 x i64> %a0, <8 x i64> %a1, i8 %mask)  #0 {
+;
+; CHECK-LABEL: @test_x86_avx512_mask_pslli_q(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = call <8 x i64> @llvm.x86.avx512.pslli.q.512(<8 x i64> [[TMP1]], i32 7)
+; CHECK-NEXT:    [[TMP5:%.*]] = or <8 x i64> [[TMP4]], zeroinitializer
+; CHECK-NEXT:    [[TMP6:%.*]] = call <8 x i64> @llvm.x86.avx512.pslli.q.512(<8 x i64> [[A0:%.*]], i32 7)
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast i8 [[TMP2]] to <8 x i1>
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP9:%.*]] = select <8 x i1> [[TMP8]], <8 x i64> [[TMP5]], <8 x i64> [[TMP3]]
+; CHECK-NEXT:    [[TMP10:%.*]] = xor <8 x i64> [[TMP6]], [[A1:%.*]]
+; CHECK-NEXT:    [[TMP11:%.*]] = or <8 x i64> [[TMP10]], [[TMP5]]
+; CHECK-NEXT:    [[TMP12:%.*]] = or <8 x i64> [[TMP11]], [[TMP3]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP7]], <8 x i64> [[TMP12]], <8 x i64> [[TMP9]]
+; CHECK-NEXT:    [[TMP13:%.*]] = select <8 x i1> [[TMP8]], <8 x i64> [[TMP6]], <8 x i64> [[A1]]
+; CHECK-NEXT:    store <8 x i64> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i64> [[TMP13]]
+;
+  %res = call <8 x i64> @llvm.x86.avx512.mask.pslli.q(<8 x i64> %a0, i32 7, <8 x i64> %a1, i8 %mask)
+  ret <8 x i64> %res
+}
+
+define <8 x i64> @test_x86_avx512_maskz_pslli_q(<8 x i64> %a0, i8 %mask)  #0 {
+;
+; CHECK-LABEL: @test_x86_avx512_maskz_pslli_q(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = call <8 x i64> @llvm.x86.avx512.pslli.q.512(<8 x i64> [[TMP1]], i32 7)
+; CHECK-NEXT:    [[TMP4:%.*]] = or <8 x i64> [[TMP3]], zeroinitializer
+; CHECK-NEXT:    [[TMP5:%.*]] = call <8 x i64> @llvm.x86.avx512.pslli.q.512(<8 x i64> [[A0:%.*]], i32 7)
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i8 [[TMP2]] to <8 x i1>
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP8:%.*]] = select <8 x i1> [[TMP7]], <8 x i64> [[TMP4]], <8 x i64> zeroinitializer
+; CHECK-NEXT:    [[TMP9:%.*]] = xor <8 x i64> [[TMP5]], zeroinitializer
+; CHECK-NEXT:    [[TMP10:%.*]] = or <8 x i64> [[TMP9]], [[TMP4]]
+; CHECK-NEXT:    [[TMP11:%.*]] = or <8 x i64> [[TMP10]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP6]], <8 x i64> [[TMP11]], <8 x i64> [[TMP8]]
+; CHECK-NEXT:    [[TMP12:%.*]] = select <8 x i1> [[TMP7]], <8 x i64> [[TMP5]], <8 x i64> zeroinitializer
+; CHECK-NEXT:    store <8 x i64> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i64> [[TMP12]]
+;
+  %res = call <8 x i64> @llvm.x86.avx512.mask.pslli.q(<8 x i64> %a0, i32 7, <8 x i64> zeroinitializer, i8 %mask)
+  ret <8 x i64> %res
+}
+
+declare <8 x i64> @llvm.x86.avx512.mask.pslli.q(<8 x i64>, i32, <8 x i64>, i8) nounwind readnone
+
+define <16 x i32> @test_x86_avx512_psrli_d(<16 x i32> %a0)  #0 {
+; CHECK-LABEL: @test_x86_avx512_psrli_d(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = call <16 x i32> @llvm.x86.avx512.psrli.d.512(<16 x i32> [[TMP1]], i32 7)
+; CHECK-NEXT:    [[TMP3:%.*]] = or <16 x i32> [[TMP2]], zeroinitializer
+; CHECK-NEXT:    [[TMP4:%.*]] = call <16 x i32> @llvm.x86.avx512.psrli.d.512(<16 x i32> [[A0:%.*]], i32 7)
+; CHECK-NEXT:    store <16 x i32> [[TMP3]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i32> [[TMP4]]
+;
+  %res = call <16 x i32> @llvm.x86.avx512.mask.psrli.d(<16 x i32> %a0, i32 7, <16 x i32> zeroinitializer, i16 -1)
+  ret <16 x i32> %res
+}
+
+define <16 x i32> @test_x86_avx512_mask_psrli_d(<16 x i32> %a0, <16 x i32> %a1, i16 %mask)  #0 {
+;
+; CHECK-LABEL: @test_x86_avx512_mask_psrli_d(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = call <16 x i32> @llvm.x86.avx512.psrli.d.512(<16 x i32> [[TMP1]], i32 7)
+; CHECK-NEXT:    [[TMP5:%.*]] = or <16 x i32> [[TMP4]], zeroinitializer
+; CHECK-NEXT:    [[TMP6:%.*]] = call <16 x i32> @llvm.x86.avx512.psrli.d.512(<16 x i32> [[A0:%.*]], i32 7)
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast i16 [[TMP2]] to <16 x i1>
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast i16 [[MASK:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP9:%.*]] = select <16 x i1> [[TMP8]], <16 x i32> [[TMP5]], <16 x i32> [[TMP3]]
+; CHECK-NEXT:    [[TMP10:%.*]] = xor <16 x i32> [[TMP6]], [[A1:%.*]]
+; CHECK-NEXT:    [[TMP11:%.*]] = or <16 x i32> [[TMP10]], [[TMP5]]
+; CHECK-NEXT:    [[TMP12:%.*]] = or <16 x i32> [[TMP11]], [[TMP3]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP7]], <16 x i32> [[TMP12]], <16 x i32> [[TMP9]]
+; CHECK-NEXT:    [[TMP13:%.*]] = select <16 x i1> [[TMP8]], <16 x i32> [[TMP6]], <16 x i32> [[A1]]
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i32> [[TMP13]]
+;
+  %res = call <16 x i32> @llvm.x86.avx512.mask.psrli.d(<16 x i32> %a0, i32 7, <16 x i32> %a1, i16 %mask)
+  ret <16 x i32> %res
+}
+
+define <16 x i32> @test_x86_avx512_maskz_psrli_d(<16 x i32> %a0, i16 %mask)  #0 {
+;
+; CHECK-LABEL: @test_x86_avx512_maskz_psrli_d(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = call <16 x i32> @llvm.x86.avx512.psrli.d.512(<16 x i32> [[TMP1]], i32 7)
+; CHECK-NEXT:    [[TMP4:%.*]] = or <16 x i32> [[TMP3]], zeroinitializer
+; CHECK-NEXT:    [[TMP5:%.*]] = call <16 x i32> @llvm.x86.avx512.psrli.d.512(<16 x i32> [[A0:%.*]], i32 7)
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i16 [[TMP2]] to <16 x i1>
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast i16 [[MASK:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP8:%.*]] = select <16 x i1> [[TMP7]], <16 x i32> [[TMP4]], <16 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP9:%.*]] = xor <16 x i32> [[TMP5]], zeroinitializer
+; CHECK-NEXT:    [[TMP10:%.*]] = or <16 x i32> [[TMP9]], [[TMP4]]
+; CHECK-NEXT:    [[TMP11:%.*]] = or <16 x i32> [[TMP10]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP6]], <16 x i32> [[TMP11]], <16 x i32> [[TMP8]]
+; CHECK-NEXT:    [[TMP12:%.*]] = select <16 x i1> [[TMP7]], <16 x i32> [[TMP5]], <16 x i32> zeroinitializer
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i32> [[TMP12]]
+;
+  %res = call <16 x i32> @llvm.x86.avx512.mask.psrli.d(<16 x i32> %a0, i32 7, <16 x i32> zeroinitializer, i16 %mask)
+  ret <16 x i32> %res
+}
+
+declare <16 x i32> @llvm.x86.avx512.mask.psrli.d(<16 x i32>, i32, <16 x i32>, i16) nounwind readnone
+
+define <8 x i64> @test_x86_avx512_psrli_q(<8 x i64> %a0)  #0 {
+; CHECK-LABEL: @test_x86_avx512_psrli_q(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = call <8 x i64> @llvm.x86.avx512.psrli.q.512(<8 x i64> [[TMP1]], i32 7)
+; CHECK-NEXT:    [[TMP3:%.*]] = or <8 x i64> [[TMP2]], zeroinitializer
+; CHECK-NEXT:    [[TMP4:%.*]] = call <8 x i64> @llvm.x86.avx512.psrli.q.512(<8 x i64> [[A0:%.*]], i32 7)
+; CHECK-NEXT:    store <8 x i64> [[TMP3]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i64> [[TMP4]]
+;
+  %res = call <8 x i64> @llvm.x86.avx512.mask.psrli.q(<8 x i64> %a0, i32 7, <8 x i64> zeroinitializer, i8 -1)
+  ret <8 x i64> %res
+}
+
+define <8 x i64> @test_x86_avx512_mask_psrli_q(<8 x i64> %a0, <8 x i64> %a1, i8 %mask)  #0 {
+;
+; CHECK-LABEL: @test_x86_avx512_mask_psrli_q(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = call <8 x i64> @llvm.x86.avx512.psrli.q.512(<8 x i64> [[TMP1]], i32 7)
+; CHECK-NEXT:    [[TMP5:%.*]] = or <8 x i64> [[TMP4]], zeroinitializer
+; CHECK-NEXT:    [[TMP6:%.*]] = call <8 x i64> @llvm.x86.avx512.psrli.q.512(<8 x i64> [[A0:%.*]], i32 7)
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast i8 [[TMP2]] to <8 x i1>
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP9:%.*]] = select <8 x i1> [[TMP8]], <8 x i64> [[TMP5]], <8 x i64> [[TMP3]]
+; CHECK-NEXT:    [[TMP10:%.*]] = xor <8 x i64> [[TMP6]], [[A1:%.*]]
+; CHECK-NEXT:    [[TMP11:%.*]] = or <8 x i64> [[TMP10]], [[TMP5]]
+; CHECK-NEXT:    [[TMP12:%.*]] = or <8 x i64> [[TMP11]], [[TMP3]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP7]], <8 x i64> [[TMP12]], <8 x i64> [[TMP9]]
+; CHECK-NEXT:    [[TMP13:%.*]] = select <8 x i1> [[TMP8]], <8 x i64> [[TMP6]], <8 x i64> [[A1]]
+; CHECK-NEXT:    store <8 x i64> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i64> [[TMP13]]
+;
+  %res = call <8 x i64> @llvm.x86.avx512.mask.psrli.q(<8 x i64> %a0, i32 7, <8 x i64> %a1, i8 %mask)
+  ret <8 x i64> %res
+}
+
+define <8 x i64> @test_x86_avx512_maskz_psrli_q(<8 x i64> %a0, i8 %mask)  #0 {
+;
+; CHECK-LABEL: @test_x86_avx512_maskz_psrli_q(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = call <8 x i64> @llvm.x86.avx512.psrli.q.512(<8 x i64> [[TMP1]], i32 7)
+; CHECK-NEXT:    [[TMP4:%.*]] = or <8 x i64> [[TMP3]], zeroinitializer
+; CHECK-NEXT:    [[TMP5:%.*]] = call <8 x i64> @llvm.x86.avx512.psrli.q.512(<8 x i64> [[A0:%.*]], i32 7)
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i8 [[TMP2]] to <8 x i1>
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP8:%.*]] = select <8 x i1> [[TMP7]], <8 x i64> [[TMP4]], <8 x i64> zeroinitializer
+; CHECK-NEXT:    [[TMP9:%.*]] = xor <8 x i64> [[TMP5]], zeroinitializer
+; CHECK-NEXT:    [[TMP10:%.*]] = or <8 x i64> [[TMP9]], [[TMP4]]
+; CHECK-NEXT:    [[TMP11:%.*]] = or <8 x i64> [[TMP10]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP6]], <8 x i64> [[TMP11]], <8 x i64> [[TMP8]]
+; CHECK-NEXT:    [[TMP12:%.*]] = select <8 x i1> [[TMP7]], <8 x i64> [[TMP5]], <8 x i64> zeroinitializer
+; CHECK-NEXT:    store <8 x i64> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i64> [[TMP12]]
+;
+  %res = call <8 x i64> @llvm.x86.avx512.mask.psrli.q(<8 x i64> %a0, i32 7, <8 x i64> zeroinitializer, i8 %mask)
+  ret <8 x i64> %res
+}
+
+declare <8 x i64> @llvm.x86.avx512.mask.psrli.q(<8 x i64>, i32, <8 x i64>, i8) nounwind readnone
+
+define <16 x i32> @test_x86_avx512_psrai_d(<16 x i32> %a0)  #0 {
+; CHECK-LABEL: @test_x86_avx512_psrai_d(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = call <16 x i32> @llvm.x86.avx512.psrai.d.512(<16 x i32> [[TMP1]], i32 7)
+; CHECK-NEXT:    [[TMP3:%.*]] = or <16 x i32> [[TMP2]], zeroinitializer
+; CHECK-NEXT:    [[TMP4:%.*]] = call <16 x i32> @llvm.x86.avx512.psrai.d.512(<16 x i32> [[A0:%.*]], i32 7)
+; CHECK-NEXT:    store <16 x i32> [[TMP3]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i32> [[TMP4]]
+;
+  %res = call <16 x i32> @llvm.x86.avx512.mask.psrai.d(<16 x i32> %a0, i32 7, <16 x i32> zeroinitializer, i16 -1)
+  ret <16 x i32> %res
+}
+
+define <16 x i32> @test_x86_avx512_mask_psrai_d(<16 x i32> %a0, <16 x i32> %a1, i16 %mask)  #0 {
+;
+; CHECK-LABEL: @test_x86_avx512_mask_psrai_d(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = call <16 x i32> @llvm.x86.avx512.psrai.d.512(<16 x i32> [[TMP1]], i32 7)
+; CHECK-NEXT:    [[TMP5:%.*]] = or <16 x i32> [[TMP4]], zeroinitializer
+; CHECK-NEXT:    [[TMP6:%.*]] = call <16 x i32> @llvm.x86.avx512.psrai.d.512(<16 x i32> [[A0:%.*]], i32 7)
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast i16 [[TMP2]] to <16 x i1>
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast i16 [[MASK:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP9:%.*]] = select <16 x i1> [[TMP8]], <16 x i32> [[TMP5]], <16 x i32> [[TMP3]]
+; CHECK-NEXT:    [[TMP10:%.*]] = xor <16 x i32> [[TMP6]], [[A1:%.*]]
+; CHECK-NEXT:    [[TMP11:%.*]] = or <16 x i32> [[TMP10]], [[TMP5]]
+; CHECK-NEXT:    [[TMP12:%.*]] = or <16 x i32> [[TMP11]], [[TMP3]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP7]], <16 x i32> [[TMP12]], <16 x i32> [[TMP9]]
+; CHECK-NEXT:    [[TMP13:%.*]] = select <16 x i1> [[TMP8]], <16 x i32> [[TMP6]], <16 x i32> [[A1]]
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i32> [[TMP13]]
+;
+  %res = call <16 x i32> @llvm.x86.avx512.mask.psrai.d(<16 x i32> %a0, i32 7, <16 x i32> %a1, i16 %mask)
+  ret <16 x i32> %res
+}
+
+define <16 x i32> @test_x86_avx512_maskz_psrai_d(<16 x i32> %a0, i16 %mask)  #0 {
+;
+; CHECK-LABEL: @test_x86_avx512_maskz_psrai_d(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = call <16 x i32> @llvm.x86.avx512.psrai.d.512(<16 x i32> [[TMP1]], i32 7)
+; CHECK-NEXT:    [[TMP4:%.*]] = or <16 x i32> [[TMP3]], zeroinitializer
+; CHECK-NEXT:    [[TMP5:%.*]] = call <16 x i32> @llvm.x86.avx512.psrai.d.512(<16 x i32> [[A0:%.*]], i32 7)
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i16 [[TMP2]] to <16 x i1>
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast i16 [[MASK:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP8:%.*]] = select <16 x i1> [[TMP7]], <16 x i32> [[TMP4]], <16 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP9:%.*]] = xor <16 x i32> [[TMP5]], zeroinitializer
+; CHECK-NEXT:    [[TMP10:%.*]] = or <16 x i32> [[TMP9]], [[TMP4]]
+; CHECK-NEXT:    [[TMP11:%.*]] = or <16 x i32> [[TMP10]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP6]], <16 x i32> [[TMP11]], <16 x i32> [[TMP8]]
+; CHECK-NEXT:    [[TMP12:%.*]] = select <16 x i1> [[TMP7]], <16 x i32> [[TMP5]], <16 x i32> zeroinitializer
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i32> [[TMP12]]
+;
+  %res = call <16 x i32> @llvm.x86.avx512.mask.psrai.d(<16 x i32> %a0, i32 7, <16 x i32> zeroinitializer, i16 %mask)
+  ret <16 x i32> %res
+}
+
+declare <16 x i32> @llvm.x86.avx512.mask.psrai.d(<16 x i32>, i32, <16 x i32>, i16) nounwind readnone
+
+define <8 x i64> @test_x86_avx512_psrai_q(<8 x i64> %a0)  #0 {
+; CHECK-LABEL: @test_x86_avx512_psrai_q(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = call <8 x i64> @llvm.x86.avx512.psrai.q.512(<8 x i64> [[TMP1]], i32 7)
+; CHECK-NEXT:    [[TMP3:%.*]] = or <8 x i64> [[TMP2]], zeroinitializer
+; CHECK-NEXT:    [[TMP4:%.*]] = call <8 x i64> @llvm.x86.avx512.psrai.q.512(<8 x i64> [[A0:%.*]], i32 7)
+; CHECK-NEXT:    store <8 x i64> [[TMP3]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i64> [[TMP4]]
+;
+  %res = call <8 x i64> @llvm.x86.avx512.mask.psrai.q(<8 x i64> %a0, i32 7, <8 x i64> zeroinitializer, i8 -1)
+  ret <8 x i64> %res
+}
+
+define <8 x i64> @test_x86_avx512_mask_psrai_q(<8 x i64> %a0, <8 x i64> %a1, i8 %mask)  #0 {
+;
+; CHECK-LABEL: @test_x86_avx512_mask_psrai_q(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = call <8 x i64> @llvm.x86.avx512.psrai.q.512(<8 x i64> [[TMP1]], i32 7)
+; CHECK-NEXT:    [[TMP5:%.*]] = or <8 x i64> [[TMP4]], zeroinitializer
+; CHECK-NEXT:    [[TMP6:%.*]] = call <8 x i64> @llvm.x86.avx512.psrai.q.512(<8 x i64> [[A0:%.*]], i32 7)
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast i8 [[TMP2]] to <8 x i1>
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP9:%.*]] = select <8 x i1> [[TMP8]], <8 x i64> [[TMP5]], <8 x i64> [[TMP3]]
+; CHECK-NEXT:    [[TMP10:%.*]] = xor <8 x i64> [[TMP6]], [[A1:%.*]]
+; CHECK-NEXT:    [[TMP11:%.*]] = or <8 x i64> [[TMP10]], [[TMP5]]
+; CHECK-NEXT:    [[TMP12:%.*]] = or <8 x i64> [[TMP11]], [[TMP3]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP7]], <8 x i64> [[TMP12]], <8 x i64> [[TMP9]]
+; CHECK-NEXT:    [[TMP13:%.*]] = select <8 x i1> [[TMP8]], <8 x i64> [[TMP6]], <8 x i64> [[A1]]
+; CHECK-NEXT:    store <8 x i64> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i64> [[TMP13]]
+;
+  %res = call <8 x i64> @llvm.x86.avx512.mask.psrai.q(<8 x i64> %a0, i32 7, <8 x i64> %a1, i8 %mask)
+  ret <8 x i64> %res
+}
+
+define <8 x i64> @test_x86_avx512_maskz_psrai_q(<8 x i64> %a0, i8 %mask)  #0 {
+;
+; CHECK-LABEL: @test_x86_avx512_maskz_psrai_q(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = call <8 x i64> @llvm.x86.avx512.psrai.q.512(<8 x i64> [[TMP1]], i32 7)
+; CHECK-NEXT:    [[TMP4:%.*]] = or <8 x i64> [[TMP3]], zeroinitializer
+; CHECK-NEXT:    [[TMP5:%.*]] = call <8 x i64> @llvm.x86.avx512.psrai.q.512(<8 x i64> [[A0:%.*]], i32 7)
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i8 [[TMP2]] to <8 x i1>
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP8:%.*]] = select <8 x i1> [[TMP7]], <8 x i64> [[TMP4]], <8 x i64> zeroinitializer
+; CHECK-NEXT:    [[TMP9:%.*]] = xor <8 x i64> [[TMP5]], zeroinitializer
+; CHECK-NEXT:    [[TMP10:%.*]] = or <8 x i64> [[TMP9]], [[TMP4]]
+; CHECK-NEXT:    [[TMP11:%.*]] = or <8 x i64> [[TMP10]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP6]], <8 x i64> [[TMP11]], <8 x i64> [[TMP8]]
+; CHECK-NEXT:    [[TMP12:%.*]] = select <8 x i1> [[TMP7]], <8 x i64> [[TMP5]], <8 x i64> zeroinitializer
+; CHECK-NEXT:    store <8 x i64> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i64> [[TMP12]]
+;
+  %res = call <8 x i64> @llvm.x86.avx512.mask.psrai.q(<8 x i64> %a0, i32 7, <8 x i64> zeroinitializer, i8 %mask)
+  ret <8 x i64> %res
+}
+
+declare <8 x i64> @llvm.x86.avx512.mask.psrai.q(<8 x i64>, i32, <8 x i64>, i8) nounwind readnone
+
+declare void @llvm.x86.avx512.storent.q.512(ptr, <8 x i64>)
+
+define void@test_storent_q_512(<8 x i64> %data, ptr %ptr)  #0 {
+;
+; CHECK-LABEL: @test_storent_q_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP3:%.*]], label [[TMP4:%.*]], !prof [[PROF1]]
+; CHECK:       3:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       4:
+; CHECK-NEXT:    [[TMP5:%.*]] = ptrtoint ptr [[PTR:%.*]] to i64
+; CHECK-NEXT:    [[TMP6:%.*]] = xor i64 [[TMP5]], 87960930222080
+; CHECK-NEXT:    [[TMP7:%.*]] = inttoptr i64 [[TMP6]] to ptr
+; CHECK-NEXT:    store <8 x i64> [[TMP2]], ptr [[TMP7]], align 64
+; CHECK-NEXT:    store <8 x i64> [[DATA:%.*]], ptr [[PTR]], align 64, !nontemporal [[META2:![0-9]+]]
+; CHECK-NEXT:    ret void
+;
+  call void @llvm.x86.avx512.storent.q.512(ptr %ptr, <8 x i64> %data)
+  ret void
+}
+
+declare void @llvm.x86.avx512.storent.pd.512(ptr, <8 x double>)
+
+define void @test_storent_pd_512(<8 x double> %data, ptr %ptr)  #0 {
+;
+; CHECK-LABEL: @test_storent_pd_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP3:%.*]], label [[TMP4:%.*]], !prof [[PROF1]]
+; CHECK:       3:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       4:
+; CHECK-NEXT:    [[TMP5:%.*]] = ptrtoint ptr [[PTR:%.*]] to i64
+; CHECK-NEXT:    [[TMP6:%.*]] = xor i64 [[TMP5]], 87960930222080
+; CHECK-NEXT:    [[TMP7:%.*]] = inttoptr i64 [[TMP6]] to ptr
+; CHECK-NEXT:    store <8 x i64> [[TMP2]], ptr [[TMP7]], align 64
+; CHECK-NEXT:    store <8 x double> [[DATA:%.*]], ptr [[PTR]], align 64, !nontemporal [[META2]]
+; CHECK-NEXT:    ret void
+;
+  call void @llvm.x86.avx512.storent.pd.512(ptr %ptr, <8 x double> %data)
+  ret void
+}
+
+declare void @llvm.x86.avx512.storent.ps.512(ptr, <16 x float>)
+
+define void @test_storent_ps_512(<16 x float> %data, ptr %ptr)  #0 {
+;
+; CHECK-LABEL: @test_storent_ps_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP3:%.*]], label [[TMP4:%.*]], !prof [[PROF1]]
+; CHECK:       3:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       4:
+; CHECK-NEXT:    [[TMP5:%.*]] = ptrtoint ptr [[PTR:%.*]] to i64
+; CHECK-NEXT:    [[TMP6:%.*]] = xor i64 [[TMP5]], 87960930222080
+; CHECK-NEXT:    [[TMP7:%.*]] = inttoptr i64 [[TMP6]] to ptr
+; CHECK-NEXT:    store <16 x i32> [[TMP2]], ptr [[TMP7]], align 64
+; CHECK-NEXT:    store <16 x float> [[DATA:%.*]], ptr [[PTR]], align 64, !nontemporal [[META2]]
+; CHECK-NEXT:    ret void
+;
+  call void @llvm.x86.avx512.storent.ps.512(ptr %ptr, <16 x float> %data)
+  ret void
+}
+
+define <16 x i32> @test_xor_epi32(<16 x i32> %a, <16 x i32> %b)  #0 {
+; CHECK-LABEL: @test_xor_epi32(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <16 x i32> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP3:%.*]] = xor <16 x i32> [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i32> [[TMP3]]
+;
+  %res = call <16 x i32> @llvm.x86.avx512.mask.pxor.d.512(<16 x i32> %a,<16 x i32> %b, <16 x i32>zeroinitializer, i16 -1)
+  ret < 16 x i32> %res
+}
+
+define <16 x i32> @test_mask_xor_epi32(<16 x i32> %a,<16 x i32> %b, <16 x i32> %passThru, i16 %mask)  #0 {
+;
+; CHECK-LABEL: @test_mask_xor_epi32(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <16 x i32> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP5:%.*]] = xor <16 x i32> [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i16 [[TMP3]] to <16 x i1>
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast i16 [[MASK:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP8:%.*]] = select <16 x i1> [[TMP7]], <16 x i32> [[_MSPROP]], <16 x i32> [[TMP4]]
+; CHECK-NEXT:    [[TMP9:%.*]] = xor <16 x i32> [[TMP5]], [[PASSTHRU:%.*]]
+; CHECK-NEXT:    [[TMP10:%.*]] = or <16 x i32> [[TMP9]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP11:%.*]] = or <16 x i32> [[TMP10]], [[TMP4]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP6]], <16 x i32> [[TMP11]], <16 x i32> [[TMP8]]
+; CHECK-NEXT:    [[TMP12:%.*]] = select <16 x i1> [[TMP7]], <16 x i32> [[TMP5]], <16 x i32> [[PASSTHRU]]
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i32> [[TMP12]]
+;
+  %res = call <16 x i32> @llvm.x86.avx512.mask.pxor.d.512(<16 x i32> %a, <16 x i32> %b, <16 x i32> %passThru, i16 %mask)
+  ret < 16 x i32> %res
+}
+
+declare <16 x i32> @llvm.x86.avx512.mask.pxor.d.512(<16 x i32>, <16 x i32>, <16 x i32>, i16)
+
+define <16 x i32> @test_or_epi32(<16 x i32> %a, <16 x i32> %b)  #0 {
+; CHECK-LABEL: @test_or_epi32(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = xor <16 x i32> [[A:%.*]], splat (i32 -1)
+; CHECK-NEXT:    [[TMP4:%.*]] = xor <16 x i32> [[B:%.*]], splat (i32 -1)
+; CHECK-NEXT:    [[TMP5:%.*]] = and <16 x i32> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP6:%.*]] = and <16 x i32> [[TMP3]], [[TMP2]]
+; CHECK-NEXT:    [[TMP7:%.*]] = and <16 x i32> [[TMP1]], [[TMP4]]
+; CHECK-NEXT:    [[TMP8:%.*]] = or <16 x i32> [[TMP5]], [[TMP6]]
+; CHECK-NEXT:    [[TMP9:%.*]] = or <16 x i32> [[TMP8]], [[TMP7]]
+; CHECK-NEXT:    [[TMP10:%.*]] = or <16 x i32> [[A]], [[B]]
+; CHECK-NEXT:    store <16 x i32> [[TMP9]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i32> [[TMP10]]
+;
+  %res = call <16 x i32> @llvm.x86.avx512.mask.por.d.512(<16 x i32> %a,<16 x i32> %b, <16 x i32>zeroinitializer, i16 -1)
+  ret < 16 x i32> %res
+}
+
+define <16 x i32> @test_mask_or_epi32(<16 x i32> %a,<16 x i32> %b, <16 x i32> %passThru, i16 %mask)  #0 {
+;
+; CHECK-LABEL: @test_mask_or_epi32(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = xor <16 x i32> [[A:%.*]], splat (i32 -1)
+; CHECK-NEXT:    [[TMP6:%.*]] = xor <16 x i32> [[B:%.*]], splat (i32 -1)
+; CHECK-NEXT:    [[TMP7:%.*]] = and <16 x i32> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP8:%.*]] = and <16 x i32> [[TMP5]], [[TMP2]]
+; CHECK-NEXT:    [[TMP9:%.*]] = and <16 x i32> [[TMP1]], [[TMP6]]
+; CHECK-NEXT:    [[TMP10:%.*]] = or <16 x i32> [[TMP7]], [[TMP8]]
+; CHECK-NEXT:    [[TMP11:%.*]] = or <16 x i32> [[TMP10]], [[TMP9]]
+; CHECK-NEXT:    [[TMP12:%.*]] = or <16 x i32> [[A]], [[B]]
+; CHECK-NEXT:    [[TMP13:%.*]] = bitcast i16 [[TMP3]] to <16 x i1>
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast i16 [[MASK:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP15:%.*]] = select <16 x i1> [[TMP14]], <16 x i32> [[TMP11]], <16 x i32> [[TMP4]]
+; CHECK-NEXT:    [[TMP16:%.*]] = xor <16 x i32> [[TMP12]], [[PASSTHRU:%.*]]
+; CHECK-NEXT:    [[TMP17:%.*]] = or <16 x i32> [[TMP16]], [[TMP11]]
+; CHECK-NEXT:    [[TMP18:%.*]] = or <16 x i32> [[TMP17]], [[TMP4]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP13]], <16 x i32> [[TMP18]], <16 x i32> [[TMP15]]
+; CHECK-NEXT:    [[TMP19:%.*]] = select <16 x i1> [[TMP14]], <16 x i32> [[TMP12]], <16 x i32> [[PASSTHRU]]
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i32> [[TMP19]]
+;
+  %res = call <16 x i32> @llvm.x86.avx512.mask.por.d.512(<16 x i32> %a, <16 x i32> %b, <16 x i32> %passThru, i16 %mask)
+  ret < 16 x i32> %res
+}
+
+declare <16 x i32> @llvm.x86.avx512.mask.por.d.512(<16 x i32>, <16 x i32>, <16 x i32>, i16)
+
+define <16 x i32> @test_and_epi32(<16 x i32> %a, <16 x i32> %b)  #0 {
+; CHECK-LABEL: @test_and_epi32(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = and <16 x i32> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = and <16 x i32> [[A:%.*]], [[TMP2]]
+; CHECK-NEXT:    [[TMP5:%.*]] = and <16 x i32> [[TMP1]], [[B:%.*]]
+; CHECK-NEXT:    [[TMP6:%.*]] = or <16 x i32> [[TMP3]], [[TMP4]]
+; CHECK-NEXT:    [[TMP7:%.*]] = or <16 x i32> [[TMP6]], [[TMP5]]
+; CHECK-NEXT:    [[TMP8:%.*]] = and <16 x i32> [[A]], [[B]]
+; CHECK-NEXT:    store <16 x i32> [[TMP7]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i32> [[TMP8]]
+;
+  %res = call <16 x i32> @llvm.x86.avx512.mask.pand.d.512(<16 x i32> %a,<16 x i32> %b, <16 x i32>zeroinitializer, i16 -1)
+  ret < 16 x i32> %res
+}
+
+define <16 x i32> @test_mask_and_epi32(<16 x i32> %a,<16 x i32> %b, <16 x i32> %passThru, i16 %mask)  #0 {
+;
+; CHECK-LABEL: @test_mask_and_epi32(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = and <16 x i32> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP6:%.*]] = and <16 x i32> [[A:%.*]], [[TMP2]]
+; CHECK-NEXT:    [[TMP7:%.*]] = and <16 x i32> [[TMP1]], [[B:%.*]]
+; CHECK-NEXT:    [[TMP8:%.*]] = or <16 x i32> [[TMP5]], [[TMP6]]
+; CHECK-NEXT:    [[TMP9:%.*]] = or <16 x i32> [[TMP8]], [[TMP7]]
+; CHECK-NEXT:    [[TMP10:%.*]] = and <16 x i32> [[A]], [[B]]
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast i16 [[TMP3]] to <16 x i1>
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast i16 [[MASK:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP13:%.*]] = select <16 x i1> [[TMP12]], <16 x i32> [[TMP9]], <16 x i32> [[TMP4]]
+; CHECK-NEXT:    [[TMP14:%.*]] = xor <16 x i32> [[TMP10]], [[PASSTHRU:%.*]]
+; CHECK-NEXT:    [[TMP15:%.*]] = or <16 x i32> [[TMP14]], [[TMP9]]
+; CHECK-NEXT:    [[TMP16:%.*]] = or <16 x i32> [[TMP15]], [[TMP4]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP11]], <16 x i32> [[TMP16]], <16 x i32> [[TMP13]]
+; CHECK-NEXT:    [[TMP17:%.*]] = select <16 x i1> [[TMP12]], <16 x i32> [[TMP10]], <16 x i32> [[PASSTHRU]]
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i32> [[TMP17]]
+;
+  %res = call <16 x i32> @llvm.x86.avx512.mask.pand.d.512(<16 x i32> %a, <16 x i32> %b, <16 x i32> %passThru, i16 %mask)
+  ret < 16 x i32> %res
+}
+
+declare <16 x i32> @llvm.x86.avx512.mask.pand.d.512(<16 x i32>, <16 x i32>, <16 x i32>, i16)
+
+define <8 x i64> @test_xor_epi64(<8 x i64> %a, <8 x i64> %b)  #0 {
+; CHECK-LABEL: @test_xor_epi64(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <8 x i64> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP3:%.*]] = xor <8 x i64> [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    store <8 x i64> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i64> [[TMP3]]
+;
+  %res = call <8 x i64> @llvm.x86.avx512.mask.pxor.q.512(<8 x i64> %a,<8 x i64> %b, <8 x i64>zeroinitializer, i8 -1)
+  ret < 8 x i64> %res
+}
+
+define <8 x i64> @test_mask_xor_epi64(<8 x i64> %a,<8 x i64> %b, <8 x i64> %passThru, i8 %mask)  #0 {
+;
+; CHECK-LABEL: @test_mask_xor_epi64(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <8 x i64> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP5:%.*]] = xor <8 x i64> [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i8 [[TMP3]] to <8 x i1>
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP8:%.*]] = select <8 x i1> [[TMP7]], <8 x i64> [[_MSPROP]], <8 x i64> [[TMP4]]
+; CHECK-NEXT:    [[TMP9:%.*]] = xor <8 x i64> [[TMP5]], [[PASSTHRU:%.*]]
+; CHECK-NEXT:    [[TMP10:%.*]] = or <8 x i64> [[TMP9]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP11:%.*]] = or <8 x i64> [[TMP10]], [[TMP4]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP6]], <8 x i64> [[TMP11]], <8 x i64> [[TMP8]]
+; CHECK-NEXT:    [[TMP12:%.*]] = select <8 x i1> [[TMP7]], <8 x i64> [[TMP5]], <8 x i64> [[PASSTHRU]]
+; CHECK-NEXT:    store <8 x i64> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i64> [[TMP12]]
+;
+  %res = call <8 x i64> @llvm.x86.avx512.mask.pxor.q.512(<8 x i64> %a, <8 x i64> %b, <8 x i64> %passThru, i8 %mask)
+  ret < 8 x i64> %res
+}
+
+declare <8 x i64> @llvm.x86.avx512.mask.pxor.q.512(<8 x i64>, <8 x i64>, <8 x i64>, i8)
+
+define <8 x i64> @test_or_epi64(<8 x i64> %a, <8 x i64> %b)  #0 {
+; CHECK-LABEL: @test_or_epi64(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = xor <8 x i64> [[A:%.*]], splat (i64 -1)
+; CHECK-NEXT:    [[TMP4:%.*]] = xor <8 x i64> [[B:%.*]], splat (i64 -1)
+; CHECK-NEXT:    [[TMP5:%.*]] = and <8 x i64> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP6:%.*]] = and <8 x i64> [[TMP3]], [[TMP2]]
+; CHECK-NEXT:    [[TMP7:%.*]] = and <8 x i64> [[TMP1]], [[TMP4]]
+; CHECK-NEXT:    [[TMP8:%.*]] = or <8 x i64> [[TMP5]], [[TMP6]]
+; CHECK-NEXT:    [[TMP9:%.*]] = or <8 x i64> [[TMP8]], [[TMP7]]
+; CHECK-NEXT:    [[TMP10:%.*]] = or <8 x i64> [[A]], [[B]]
+; CHECK-NEXT:    store <8 x i64> [[TMP9]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i64> [[TMP10]]
+;
+  %res = call <8 x i64> @llvm.x86.avx512.mask.por.q.512(<8 x i64> %a,<8 x i64> %b, <8 x i64>zeroinitializer, i8 -1)
+  ret < 8 x i64> %res
+}
+
+define <8 x i64> @test_mask_or_epi64(<8 x i64> %a,<8 x i64> %b, <8 x i64> %passThru, i8 %mask)  #0 {
+;
+; CHECK-LABEL: @test_mask_or_epi64(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = xor <8 x i64> [[A:%.*]], splat (i64 -1)
+; CHECK-NEXT:    [[TMP6:%.*]] = xor <8 x i64> [[B:%.*]], splat (i64 -1)
+; CHECK-NEXT:    [[TMP7:%.*]] = and <8 x i64> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP8:%.*]] = and <8 x i64> [[TMP5]], [[TMP2]]
+; CHECK-NEXT:    [[TMP9:%.*]] = and <8 x i64> [[TMP1]], [[TMP6]]
+; CHECK-NEXT:    [[TMP10:%.*]] = or <8 x i64> [[TMP7]], [[TMP8]]
+; CHECK-NEXT:    [[TMP11:%.*]] = or <8 x i64> [[TMP10]], [[TMP9]]
+; CHECK-NEXT:    [[TMP12:%.*]] = or <8 x i64> [[A]], [[B]]
+; CHECK-NEXT:    [[TMP13:%.*]] = bitcast i8 [[TMP3]] to <8 x i1>
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP15:%.*]] = select <8 x i1> [[TMP14]], <8 x i64> [[TMP11]], <8 x i64> [[TMP4]]
+; CHECK-NEXT:    [[TMP16:%.*]] = xor <8 x i64> [[TMP12]], [[PASSTHRU:%.*]]
+; CHECK-NEXT:    [[TMP17:%.*]] = or <8 x i64> [[TMP16]], [[TMP11]]
+; CHECK-NEXT:    [[TMP18:%.*]] = or <8 x i64> [[TMP17]], [[TMP4]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP13]], <8 x i64> [[TMP18]], <8 x i64> [[TMP15]]
+; CHECK-NEXT:    [[TMP19:%.*]] = select <8 x i1> [[TMP14]], <8 x i64> [[TMP12]], <8 x i64> [[PASSTHRU]]
+; CHECK-NEXT:    store <8 x i64> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i64> [[TMP19]]
+;
+  %res = call <8 x i64> @llvm.x86.avx512.mask.por.q.512(<8 x i64> %a, <8 x i64> %b, <8 x i64> %passThru, i8 %mask)
+  ret < 8 x i64> %res
+}
+
+declare <8 x i64> @llvm.x86.avx512.mask.por.q.512(<8 x i64>, <8 x i64>, <8 x i64>, i8)
+
+define <8 x i64> @test_and_epi64(<8 x i64> %a, <8 x i64> %b)  #0 {
+; CHECK-LABEL: @test_and_epi64(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = and <8 x i64> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = and <8 x i64> [[A:%.*]], [[TMP2]]
+; CHECK-NEXT:    [[TMP5:%.*]] = and <8 x i64> [[TMP1]], [[B:%.*]]
+; CHECK-NEXT:    [[TMP6:%.*]] = or <8 x i64> [[TMP3]], [[TMP4]]
+; CHECK-NEXT:    [[TMP7:%.*]] = or <8 x i64> [[TMP6]], [[TMP5]]
+; CHECK-NEXT:    [[TMP8:%.*]] = and <8 x i64> [[A]], [[B]]
+; CHECK-NEXT:    store <8 x i64> [[TMP7]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i64> [[TMP8]]
+;
+  %res = call <8 x i64> @llvm.x86.avx512.mask.pand.q.512(<8 x i64> %a,<8 x i64> %b, <8 x i64>zeroinitializer, i8 -1)
+  ret < 8 x i64> %res
+}
+
+define <8 x i64> @test_mask_and_epi64(<8 x i64> %a,<8 x i64> %b, <8 x i64> %passThru, i8 %mask)  #0 {
+;
+; CHECK-LABEL: @test_mask_and_epi64(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = and <8 x i64> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP6:%.*]] = and <8 x i64> [[A:%.*]], [[TMP2]]
+; CHECK-NEXT:    [[TMP7:%.*]] = and <8 x i64> [[TMP1]], [[B:%.*]]
+; CHECK-NEXT:    [[TMP8:%.*]] = or <8 x i64> [[TMP5]], [[TMP6]]
+; CHECK-NEXT:    [[TMP9:%.*]] = or <8 x i64> [[TMP8]], [[TMP7]]
+; CHECK-NEXT:    [[TMP10:%.*]] = and <8 x i64> [[A]], [[B]]
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast i8 [[TMP3]] to <8 x i1>
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP13:%.*]] = select <8 x i1> [[TMP12]], <8 x i64> [[TMP9]], <8 x i64> [[TMP4]]
+; CHECK-NEXT:    [[TMP14:%.*]] = xor <8 x i64> [[TMP10]], [[PASSTHRU:%.*]]
+; CHECK-NEXT:    [[TMP15:%.*]] = or <8 x i64> [[TMP14]], [[TMP9]]
+; CHECK-NEXT:    [[TMP16:%.*]] = or <8 x i64> [[TMP15]], [[TMP4]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP11]], <8 x i64> [[TMP16]], <8 x i64> [[TMP13]]
+; CHECK-NEXT:    [[TMP17:%.*]] = select <8 x i1> [[TMP12]], <8 x i64> [[TMP10]], <8 x i64> [[PASSTHRU]]
+; CHECK-NEXT:    store <8 x i64> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i64> [[TMP17]]
+;
+  %res = call <8 x i64> @llvm.x86.avx512.mask.pand.q.512(<8 x i64> %a, <8 x i64> %b, <8 x i64> %passThru, i8 %mask)
+  ret < 8 x i64> %res
+}
+
+declare <8 x i64> @llvm.x86.avx512.mask.pand.q.512(<8 x i64>, <8 x i64>, <8 x i64>, i8)
+
+define <16 x i32> @test_mask_add_epi32_rr(<16 x i32> %a, <16 x i32> %b)  #0 {
+; CHECK-LABEL: @test_mask_add_epi32_rr(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <16 x i32> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP3:%.*]] = add <16 x i32> [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i32> [[TMP3]]
+;
+  %res = call <16 x i32> @llvm.x86.avx512.mask.padd.d.512(<16 x i32> %a, <16 x i32> %b, <16 x i32> zeroinitializer, i16 -1)
+  ret < 16 x i32> %res
+}
+
+define <16 x i32> @test_mask_add_epi32_rrk(<16 x i32> %a, <16 x i32> %b, <16 x i32> %passThru, i16 %mask)  #0 {
+;
+; CHECK-LABEL: @test_mask_add_epi32_rrk(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <16 x i32> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP5:%.*]] = add <16 x i32> [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i16 [[TMP3]] to <16 x i1>
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast i16 [[MASK:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP8:%.*]] = select <16 x i1> [[TMP7]], <16 x i32> [[_MSPROP]], <16 x i32> [[TMP4]]
+; CHECK-NEXT:    [[TMP9:%.*]] = xor <16 x i32> [[TMP5]], [[PASSTHRU:%.*]]
+; CHECK-NEXT:    [[TMP10:%.*]] = or <16 x i32> [[TMP9]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP11:%.*]] = or <16 x i32> [[TMP10]], [[TMP4]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP6]], <16 x i32> [[TMP11]], <16 x i32> [[TMP8]]
+; CHECK-NEXT:    [[TMP12:%.*]] = select <16 x i1> [[TMP7]], <16 x i32> [[TMP5]], <16 x i32> [[PASSTHRU]]
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i32> [[TMP12]]
+;
+  %res = call <16 x i32> @llvm.x86.avx512.mask.padd.d.512(<16 x i32> %a, <16 x i32> %b, <16 x i32> %passThru, i16 %mask)
+  ret < 16 x i32> %res
+}
+
+define <16 x i32> @test_mask_add_epi32_rrkz(<16 x i32> %a, <16 x i32> %b, i16 %mask)  #0 {
+;
+; CHECK-LABEL: @test_mask_add_epi32_rrkz(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <16 x i32> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = add <16 x i32> [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i16 [[TMP3]] to <16 x i1>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i16 [[MASK:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP7:%.*]] = select <16 x i1> [[TMP6]], <16 x i32> [[_MSPROP]], <16 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP8:%.*]] = xor <16 x i32> [[TMP4]], zeroinitializer
+; CHECK-NEXT:    [[TMP9:%.*]] = or <16 x i32> [[TMP8]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP10:%.*]] = or <16 x i32> [[TMP9]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP5]], <16 x i32> [[TMP10]], <16 x i32> [[TMP7]]
+; CHECK-NEXT:    [[TMP11:%.*]] = select <16 x i1> [[TMP6]], <16 x i32> [[TMP4]], <16 x i32> zeroinitializer
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i32> [[TMP11]]
+;
+  %res = call <16 x i32> @llvm.x86.avx512.mask.padd.d.512(<16 x i32> %a, <16 x i32> %b, <16 x i32> zeroinitializer, i16 %mask)
+  ret < 16 x i32> %res
+}
+
+define <16 x i32> @test_mask_add_epi32_rm(<16 x i32> %a, ptr %ptr_b)  #0 {
+;
+; CHECK-LABEL: @test_mask_add_epi32_rm(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP3:%.*]], label [[TMP4:%.*]], !prof [[PROF1]]
+; CHECK:       3:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       4:
+; CHECK-NEXT:    [[B:%.*]] = load <16 x i32>, ptr [[PTR_B:%.*]], align 64
+; CHECK-NEXT:    [[TMP5:%.*]] = ptrtoint ptr [[PTR_B]] to i64
+; CHECK-NEXT:    [[TMP6:%.*]] = xor i64 [[TMP5]], 87960930222080
+; CHECK-NEXT:    [[TMP7:%.*]] = inttoptr i64 [[TMP6]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <16 x i32>, ptr [[TMP7]], align 64
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <16 x i32> [[TMP2]], [[_MSLD]]
+; CHECK-NEXT:    [[TMP8:%.*]] = add <16 x i32> [[A:%.*]], [[B]]
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i32> [[TMP8]]
+;
+  %b = load <16 x i32>, ptr %ptr_b
+  %res = call <16 x i32> @llvm.x86.avx512.mask.padd.d.512(<16 x i32> %a, <16 x i32> %b, <16 x i32> zeroinitializer, i16 -1)
+  ret < 16 x i32> %res
+}
+
+define <16 x i32> @test_mask_add_epi32_rmk(<16 x i32> %a, ptr %ptr_b, <16 x i32> %passThru, i16 %mask)  #0 {
+;
+; CHECK-LABEL: @test_mask_add_epi32_rmk(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 136) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 72) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP5:%.*]], label [[TMP6:%.*]], !prof [[PROF1]]
+; CHECK:       5:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       6:
+; CHECK-NEXT:    [[B:%.*]] = load <16 x i32>, ptr [[PTR_B:%.*]], align 64
+; CHECK-NEXT:    [[TMP7:%.*]] = ptrtoint ptr [[PTR_B]] to i64
+; CHECK-NEXT:    [[TMP8:%.*]] = xor i64 [[TMP7]], 87960930222080
+; CHECK-NEXT:    [[TMP9:%.*]] = inttoptr i64 [[TMP8]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <16 x i32>, ptr [[TMP9]], align 64
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <16 x i32> [[TMP2]], [[_MSLD]]
+; CHECK-NEXT:    [[TMP10:%.*]] = add <16 x i32> [[A:%.*]], [[B]]
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast i16 [[TMP3]] to <16 x i1>
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast i16 [[MASK:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP13:%.*]] = select <16 x i1> [[TMP12]], <16 x i32> [[_MSPROP]], <16 x i32> [[TMP4]]
+; CHECK-NEXT:    [[TMP14:%.*]] = xor <16 x i32> [[TMP10]], [[PASSTHRU:%.*]]
+; CHECK-NEXT:    [[TMP15:%.*]] = or <16 x i32> [[TMP14]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP16:%.*]] = or <16 x i32> [[TMP15]], [[TMP4]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP11]], <16 x i32> [[TMP16]], <16 x i32> [[TMP13]]
+; CHECK-NEXT:    [[TMP17:%.*]] = select <16 x i1> [[TMP12]], <16 x i32> [[TMP10]], <16 x i32> [[PASSTHRU]]
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i32> [[TMP17]]
+;
+  %b = load <16 x i32>, ptr %ptr_b
+  %res = call <16 x i32> @llvm.x86.avx512.mask.padd.d.512(<16 x i32> %a, <16 x i32> %b, <16 x i32> %passThru, i16 %mask)
+  ret < 16 x i32> %res
+}
+
+define <16 x i32> @test_mask_add_epi32_rmkz(<16 x i32> %a, ptr %ptr_b, i16 %mask)  #0 {
+;
+; CHECK-LABEL: @test_mask_add_epi32_rmkz(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 72) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP4:%.*]], label [[TMP5:%.*]], !prof [[PROF1]]
+; CHECK:       4:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       5:
+; CHECK-NEXT:    [[B:%.*]] = load <16 x i32>, ptr [[PTR_B:%.*]], align 64
+; CHECK-NEXT:    [[TMP6:%.*]] = ptrtoint ptr [[PTR_B]] to i64
+; CHECK-NEXT:    [[TMP7:%.*]] = xor i64 [[TMP6]], 87960930222080
+; CHECK-NEXT:    [[TMP8:%.*]] = inttoptr i64 [[TMP7]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <16 x i32>, ptr [[TMP8]], align 64
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <16 x i32> [[TMP2]], [[_MSLD]]
+; CHECK-NEXT:    [[TMP9:%.*]] = add <16 x i32> [[A:%.*]], [[B]]
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i16 [[TMP3]] to <16 x i1>
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast i16 [[MASK:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP12:%.*]] = select <16 x i1> [[TMP11]], <16 x i32> [[_MSPROP]], <16 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP13:%.*]] = xor <16 x i32> [[TMP9]], zeroinitializer
+; CHECK-NEXT:    [[TMP14:%.*]] = or <16 x i32> [[TMP13]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP15:%.*]] = or <16 x i32> [[TMP14]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP10]], <16 x i32> [[TMP15]], <16 x i32> [[TMP12]]
+; CHECK-NEXT:    [[TMP16:%.*]] = select <16 x i1> [[TMP11]], <16 x i32> [[TMP9]], <16 x i32> zeroinitializer
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i32> [[TMP16]]
+;
+  %b = load <16 x i32>, ptr %ptr_b
+  %res = call <16 x i32> @llvm.x86.avx512.mask.padd.d.512(<16 x i32> %a, <16 x i32> %b, <16 x i32> zeroinitializer, i16 %mask)
+  ret < 16 x i32> %res
+}
+
+define <16 x i32> @test_mask_add_epi32_rmb(<16 x i32> %a, ptr %ptr_b, <16 x i32> %extra_param, <16 x i32> %extra_param2)  #0 {
+;
+; CHECK-LABEL: @test_mask_add_epi32_rmb(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 72) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 136) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP9:%.*]], label [[TMP10:%.*]], !prof [[PROF1]]
+; CHECK:       5:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       6:
+; CHECK-NEXT:    [[Q:%.*]] = load i32, ptr [[PTR_B:%.*]], align 4
+; CHECK-NEXT:    [[TMP5:%.*]] = ptrtoint ptr [[PTR_B]] to i64
+; CHECK-NEXT:    [[TMP6:%.*]] = xor i64 [[TMP5]], 87960930222080
+; CHECK-NEXT:    [[TMP7:%.*]] = inttoptr i64 [[TMP6]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load i32, ptr [[TMP7]], align 4
+; CHECK-NEXT:    [[_MSPROP:%.*]] = insertelement <16 x i32> [[TMP3]], i32 [[_MSLD]], i32 0
+; CHECK-NEXT:    [[VECINIT_I:%.*]] = insertelement <16 x i32> [[EXTRA_PARAM:%.*]], i32 [[Q]], i32 0
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = shufflevector <16 x i32> [[_MSPROP]], <16 x i32> [[TMP4]], <16 x i32> zeroinitializer
+; CHECK-NEXT:    [[B:%.*]] = shufflevector <16 x i32> [[VECINIT_I]], <16 x i32> [[EXTRA_PARAM2:%.*]], <16 x i32> zeroinitializer
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = or <16 x i32> [[TMP2]], [[_MSPROP1]]
+; CHECK-NEXT:    [[TMP8:%.*]] = add <16 x i32> [[A:%.*]], [[B]]
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP2]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i32> [[TMP8]]
+;
+  %q = load i32, ptr %ptr_b
+  %vecinit.i = insertelement <16 x i32> %extra_param, i32 %q, i32 0
+  %b = shufflevector <16 x i32> %vecinit.i, <16 x i32> %extra_param2, <16 x i32> zeroinitializer
+  %res = call <16 x i32> @llvm.x86.avx512.mask.padd.d.512(<16 x i32> %a, <16 x i32> %b, <16 x i32> zeroinitializer, i16 -1)
+  ret < 16 x i32> %res
+}
+
+define <16 x i32> @test_mask_add_epi32_rmbk(<16 x i32> %a, ptr %ptr_b, <16 x i32> %passThru, i16 %mask, <16 x i32> %extra_param, <16 x i32> %extra_param2)  #0 {
+;
+; CHECK-LABEL: @test_mask_add_epi32_rmbk(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP5:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 144) to ptr), align 8
+; CHECK-NEXT:    [[TMP6:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 208) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 136) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 72) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP18:%.*]], label [[TMP19:%.*]], !prof [[PROF1]]
+; CHECK:       7:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       8:
+; CHECK-NEXT:    [[Q:%.*]] = load i32, ptr [[PTR_B:%.*]], align 4
+; CHECK-NEXT:    [[TMP7:%.*]] = ptrtoint ptr [[PTR_B]] to i64
+; CHECK-NEXT:    [[TMP8:%.*]] = xor i64 [[TMP7]], 87960930222080
+; CHECK-NEXT:    [[TMP9:%.*]] = inttoptr i64 [[TMP8]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load i32, ptr [[TMP9]], align 4
+; CHECK-NEXT:    [[_MSPROP:%.*]] = insertelement <16 x i32> [[TMP5]], i32 [[_MSLD]], i32 0
+; CHECK-NEXT:    [[VECINIT_I:%.*]] = insertelement <16 x i32> [[EXTRA_PARAM:%.*]], i32 [[Q]], i32 0
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = shufflevector <16 x i32> [[_MSPROP]], <16 x i32> [[TMP6]], <16 x i32> zeroinitializer
+; CHECK-NEXT:    [[B:%.*]] = shufflevector <16 x i32> [[VECINIT_I]], <16 x i32> [[EXTRA_PARAM2:%.*]], <16 x i32> zeroinitializer
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = or <16 x i32> [[TMP2]], [[_MSPROP1]]
+; CHECK-NEXT:    [[TMP10:%.*]] = add <16 x i32> [[A:%.*]], [[B]]
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast i16 [[TMP3]] to <16 x i1>
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast i16 [[MASK:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP13:%.*]] = select <16 x i1> [[TMP12]], <16 x i32> [[_MSPROP2]], <16 x i32> [[TMP4]]
+; CHECK-NEXT:    [[TMP14:%.*]] = xor <16 x i32> [[TMP10]], [[PASSTHRU:%.*]]
+; CHECK-NEXT:    [[TMP15:%.*]] = or <16 x i32> [[TMP14]], [[_MSPROP2]]
+; CHECK-NEXT:    [[TMP16:%.*]] = or <16 x i32> [[TMP15]], [[TMP4]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP11]], <16 x i32> [[TMP16]], <16 x i32> [[TMP13]]
+; CHECK-NEXT:    [[TMP17:%.*]] = select <16 x i1> [[TMP12]], <16 x i32> [[TMP10]], <16 x i32> [[PASSTHRU]]
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i32> [[TMP17]]
+;
+  %q = load i32, ptr %ptr_b
+  %vecinit.i = insertelement <16 x i32> %extra_param, i32 %q, i32 0
+  %b = shufflevector <16 x i32> %vecinit.i, <16 x i32> %extra_param2, <16 x i32> zeroinitializer
+  %res = call <16 x i32> @llvm.x86.avx512.mask.padd.d.512(<16 x i32> %a, <16 x i32> %b, <16 x i32> %passThru, i16 %mask)
+  ret < 16 x i32> %res
+}
+
+define <16 x i32> @test_mask_add_epi32_rmbkz(<16 x i32> %a, ptr %ptr_b, i16 %mask, <16 x i32> %extra_param, <16 x i32> %extra_param2)  #0 {
+;
+; CHECK-LABEL: @test_mask_add_epi32_rmbkz(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 80) to ptr), align 8
+; CHECK-NEXT:    [[TMP5:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 144) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 72) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP17:%.*]], label [[TMP18:%.*]], !prof [[PROF1]]
+; CHECK:       6:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       7:
+; CHECK-NEXT:    [[Q:%.*]] = load i32, ptr [[PTR_B:%.*]], align 4
+; CHECK-NEXT:    [[TMP6:%.*]] = ptrtoint ptr [[PTR_B]] to i64
+; CHECK-NEXT:    [[TMP7:%.*]] = xor i64 [[TMP6]], 87960930222080
+; CHECK-NEXT:    [[TMP8:%.*]] = inttoptr i64 [[TMP7]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load i32, ptr [[TMP8]], align 4
+; CHECK-NEXT:    [[_MSPROP:%.*]] = insertelement <16 x i32> [[TMP4]], i32 [[_MSLD]], i32 0
+; CHECK-NEXT:    [[VECINIT_I:%.*]] = insertelement <16 x i32> [[EXTRA_PARAM:%.*]], i32 [[Q]], i32 0
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = shufflevector <16 x i32> [[_MSPROP]], <16 x i32> [[TMP5]], <16 x i32> zeroinitializer
+; CHECK-NEXT:    [[B:%.*]] = shufflevector <16 x i32> [[VECINIT_I]], <16 x i32> [[EXTRA_PARAM2:%.*]], <16 x i32> zeroinitializer
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = or <16 x i32> [[TMP2]], [[_MSPROP1]]
+; CHECK-NEXT:    [[TMP9:%.*]] = add <16 x i32> [[A:%.*]], [[B]]
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i16 [[TMP3]] to <16 x i1>
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast i16 [[MASK:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP12:%.*]] = select <16 x i1> [[TMP11]], <16 x i32> [[_MSPROP2]], <16 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP13:%.*]] = xor <16 x i32> [[TMP9]], zeroinitializer
+; CHECK-NEXT:    [[TMP14:%.*]] = or <16 x i32> [[TMP13]], [[_MSPROP2]]
+; CHECK-NEXT:    [[TMP15:%.*]] = or <16 x i32> [[TMP14]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP10]], <16 x i32> [[TMP15]], <16 x i32> [[TMP12]]
+; CHECK-NEXT:    [[TMP16:%.*]] = select <16 x i1> [[TMP11]], <16 x i32> [[TMP9]], <16 x i32> zeroinitializer
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i32> [[TMP16]]
+;
+  %q = load i32, ptr %ptr_b
+  %vecinit.i = insertelement <16 x i32> %extra_param, i32 %q, i32 0
+  %b = shufflevector <16 x i32> %vecinit.i, <16 x i32> %extra_param2, <16 x i32> zeroinitializer
+  %res = call <16 x i32> @llvm.x86.avx512.mask.padd.d.512(<16 x i32> %a, <16 x i32> %b, <16 x i32> zeroinitializer, i16 %mask)
+  ret < 16 x i32> %res
+}
+
+declare <16 x i32> @llvm.x86.avx512.mask.padd.d.512(<16 x i32>, <16 x i32>, <16 x i32>, i16)
+
+define <16 x i32> @test_mask_sub_epi32_rr(<16 x i32> %a, <16 x i32> %b)  #0 {
+; CHECK-LABEL: @test_mask_sub_epi32_rr(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <16 x i32> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP3:%.*]] = sub <16 x i32> [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i32> [[TMP3]]
+;
+  %res = call <16 x i32> @llvm.x86.avx512.mask.psub.d.512(<16 x i32> %a, <16 x i32> %b, <16 x i32> zeroinitializer, i16 -1)
+  ret < 16 x i32> %res
+}
+
+define <16 x i32> @test_mask_sub_epi32_rrk(<16 x i32> %a, <16 x i32> %b, <16 x i32> %passThru, i16 %mask)  #0 {
+;
+; CHECK-LABEL: @test_mask_sub_epi32_rrk(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <16 x i32> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP5:%.*]] = sub <16 x i32> [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i16 [[TMP3]] to <16 x i1>
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast i16 [[MASK:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP8:%.*]] = select <16 x i1> [[TMP7]], <16 x i32> [[_MSPROP]], <16 x i32> [[TMP4]]
+; CHECK-NEXT:    [[TMP9:%.*]] = xor <16 x i32> [[TMP5]], [[PASSTHRU:%.*]]
+; CHECK-NEXT:    [[TMP10:%.*]] = or <16 x i32> [[TMP9]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP11:%.*]] = or <16 x i32> [[TMP10]], [[TMP4]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP6]], <16 x i32> [[TMP11]], <16 x i32> [[TMP8]]
+; CHECK-NEXT:    [[TMP12:%.*]] = select <16 x i1> [[TMP7]], <16 x i32> [[TMP5]], <16 x i32> [[PASSTHRU]]
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i32> [[TMP12]]
+;
+  %res = call <16 x i32> @llvm.x86.avx512.mask.psub.d.512(<16 x i32> %a, <16 x i32> %b, <16 x i32> %passThru, i16 %mask)
+  ret < 16 x i32> %res
+}
+
+define <16 x i32> @test_mask_sub_epi32_rrkz(<16 x i32> %a, <16 x i32> %b, i16 %mask)  #0 {
+;
+; CHECK-LABEL: @test_mask_sub_epi32_rrkz(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <16 x i32> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = sub <16 x i32> [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i16 [[TMP3]] to <16 x i1>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i16 [[MASK:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP7:%.*]] = select <16 x i1> [[TMP6]], <16 x i32> [[_MSPROP]], <16 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP8:%.*]] = xor <16 x i32> [[TMP4]], zeroinitializer
+; CHECK-NEXT:    [[TMP9:%.*]] = or <16 x i32> [[TMP8]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP10:%.*]] = or <16 x i32> [[TMP9]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP5]], <16 x i32> [[TMP10]], <16 x i32> [[TMP7]]
+; CHECK-NEXT:    [[TMP11:%.*]] = select <16 x i1> [[TMP6]], <16 x i32> [[TMP4]], <16 x i32> zeroinitializer
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i32> [[TMP11]]
+;
+  %res = call <16 x i32> @llvm.x86.avx512.mask.psub.d.512(<16 x i32> %a, <16 x i32> %b, <16 x i32> zeroinitializer, i16 %mask)
+  ret < 16 x i32> %res
+}
+
+define <16 x i32> @test_mask_sub_epi32_rm(<16 x i32> %a, ptr %ptr_b)  #0 {
+;
+; CHECK-LABEL: @test_mask_sub_epi32_rm(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP3:%.*]], label [[TMP4:%.*]], !prof [[PROF1]]
+; CHECK:       3:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       4:
+; CHECK-NEXT:    [[B:%.*]] = load <16 x i32>, ptr [[PTR_B:%.*]], align 64
+; CHECK-NEXT:    [[TMP5:%.*]] = ptrtoint ptr [[PTR_B]] to i64
+; CHECK-NEXT:    [[TMP6:%.*]] = xor i64 [[TMP5]], 87960930222080
+; CHECK-NEXT:    [[TMP7:%.*]] = inttoptr i64 [[TMP6]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <16 x i32>, ptr [[TMP7]], align 64
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <16 x i32> [[TMP2]], [[_MSLD]]
+; CHECK-NEXT:    [[TMP8:%.*]] = sub <16 x i32> [[A:%.*]], [[B]]
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i32> [[TMP8]]
+;
+  %b = load <16 x i32>, ptr %ptr_b
+  %res = call <16 x i32> @llvm.x86.avx512.mask.psub.d.512(<16 x i32> %a, <16 x i32> %b, <16 x i32> zeroinitializer, i16 -1)
+  ret < 16 x i32> %res
+}
+
+define <16 x i32> @test_mask_sub_epi32_rmk(<16 x i32> %a, ptr %ptr_b, <16 x i32> %passThru, i16 %mask)  #0 {
+;
+; CHECK-LABEL: @test_mask_sub_epi32_rmk(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 136) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 72) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP5:%.*]], label [[TMP6:%.*]], !prof [[PROF1]]
+; CHECK:       5:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       6:
+; CHECK-NEXT:    [[B:%.*]] = load <16 x i32>, ptr [[PTR_B:%.*]], align 64
+; CHECK-NEXT:    [[TMP7:%.*]] = ptrtoint ptr [[PTR_B]] to i64
+; CHECK-NEXT:    [[TMP8:%.*]] = xor i64 [[TMP7]], 87960930222080
+; CHECK-NEXT:    [[TMP9:%.*]] = inttoptr i64 [[TMP8]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <16 x i32>, ptr [[TMP9]], align 64
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <16 x i32> [[TMP2]], [[_MSLD]]
+; CHECK-NEXT:    [[TMP10:%.*]] = sub <16 x i32> [[A:%.*]], [[B]]
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast i16 [[TMP3]] to <16 x i1>
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast i16 [[MASK:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP13:%.*]] = select <16 x i1> [[TMP12]], <16 x i32> [[_MSPROP]], <16 x i32> [[TMP4]]
+; CHECK-NEXT:    [[TMP14:%.*]] = xor <16 x i32> [[TMP10]], [[PASSTHRU:%.*]]
+; CHECK-NEXT:    [[TMP15:%.*]] = or <16 x i32> [[TMP14]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP16:%.*]] = or <16 x i32> [[TMP15]], [[TMP4]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP11]], <16 x i32> [[TMP16]], <16 x i32> [[TMP13]]
+; CHECK-NEXT:    [[TMP17:%.*]] = select <16 x i1> [[TMP12]], <16 x i32> [[TMP10]], <16 x i32> [[PASSTHRU]]
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i32> [[TMP17]]
+;
+  %b = load <16 x i32>, ptr %ptr_b
+  %res = call <16 x i32> @llvm.x86.avx512.mask.psub.d.512(<16 x i32> %a, <16 x i32> %b, <16 x i32> %passThru, i16 %mask)
+  ret < 16 x i32> %res
+}
+
+define <16 x i32> @test_mask_sub_epi32_rmkz(<16 x i32> %a, ptr %ptr_b, i16 %mask)  #0 {
+;
+; CHECK-LABEL: @test_mask_sub_epi32_rmkz(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 72) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP4:%.*]], label [[TMP5:%.*]], !prof [[PROF1]]
+; CHECK:       4:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       5:
+; CHECK-NEXT:    [[B:%.*]] = load <16 x i32>, ptr [[PTR_B:%.*]], align 64
+; CHECK-NEXT:    [[TMP6:%.*]] = ptrtoint ptr [[PTR_B]] to i64
+; CHECK-NEXT:    [[TMP7:%.*]] = xor i64 [[TMP6]], 87960930222080
+; CHECK-NEXT:    [[TMP8:%.*]] = inttoptr i64 [[TMP7]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <16 x i32>, ptr [[TMP8]], align 64
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <16 x i32> [[TMP2]], [[_MSLD]]
+; CHECK-NEXT:    [[TMP9:%.*]] = sub <16 x i32> [[A:%.*]], [[B]]
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i16 [[TMP3]] to <16 x i1>
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast i16 [[MASK:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP12:%.*]] = select <16 x i1> [[TMP11]], <16 x i32> [[_MSPROP]], <16 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP13:%.*]] = xor <16 x i32> [[TMP9]], zeroinitializer
+; CHECK-NEXT:    [[TMP14:%.*]] = or <16 x i32> [[TMP13]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP15:%.*]] = or <16 x i32> [[TMP14]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP10]], <16 x i32> [[TMP15]], <16 x i32> [[TMP12]]
+; CHECK-NEXT:    [[TMP16:%.*]] = select <16 x i1> [[TMP11]], <16 x i32> [[TMP9]], <16 x i32> zeroinitializer
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i32> [[TMP16]]
+;
+  %b = load <16 x i32>, ptr %ptr_b
+  %res = call <16 x i32> @llvm.x86.avx512.mask.psub.d.512(<16 x i32> %a, <16 x i32> %b, <16 x i32> zeroinitializer, i16 %mask)
+  ret < 16 x i32> %res
+}
+
+define <16 x i32> @test_mask_sub_epi32_rmb(<16 x i32> %a, ptr %ptr_b, <16 x i32> %extra_param, <16 x i32> %extra_param2)  #0 {
+;
+; CHECK-LABEL: @test_mask_sub_epi32_rmb(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 72) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 136) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP9:%.*]], label [[TMP10:%.*]], !prof [[PROF1]]
+; CHECK:       5:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       6:
+; CHECK-NEXT:    [[Q:%.*]] = load i32, ptr [[PTR_B:%.*]], align 4
+; CHECK-NEXT:    [[TMP5:%.*]] = ptrtoint ptr [[PTR_B]] to i64
+; CHECK-NEXT:    [[TMP6:%.*]] = xor i64 [[TMP5]], 87960930222080
+; CHECK-NEXT:    [[TMP7:%.*]] = inttoptr i64 [[TMP6]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load i32, ptr [[TMP7]], align 4
+; CHECK-NEXT:    [[_MSPROP:%.*]] = insertelement <16 x i32> [[TMP3]], i32 [[_MSLD]], i32 0
+; CHECK-NEXT:    [[VECINIT_I:%.*]] = insertelement <16 x i32> [[EXTRA_PARAM:%.*]], i32 [[Q]], i32 0
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = shufflevector <16 x i32> [[_MSPROP]], <16 x i32> [[TMP4]], <16 x i32> zeroinitializer
+; CHECK-NEXT:    [[B:%.*]] = shufflevector <16 x i32> [[VECINIT_I]], <16 x i32> [[EXTRA_PARAM2:%.*]], <16 x i32> zeroinitializer
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = or <16 x i32> [[TMP2]], [[_MSPROP1]]
+; CHECK-NEXT:    [[TMP8:%.*]] = sub <16 x i32> [[A:%.*]], [[B]]
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP2]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i32> [[TMP8]]
+;
+  %q = load i32, ptr %ptr_b
+  %vecinit.i = insertelement <16 x i32> %extra_param, i32 %q, i32 0
+  %b = shufflevector <16 x i32> %vecinit.i, <16 x i32> %extra_param2, <16 x i32> zeroinitializer
+  %res = call <16 x i32> @llvm.x86.avx512.mask.psub.d.512(<16 x i32> %a, <16 x i32> %b, <16 x i32> zeroinitializer, i16 -1)
+  ret < 16 x i32> %res
+}
+
+define <16 x i32> @test_mask_sub_epi32_rmbk(<16 x i32> %a, ptr %ptr_b, <16 x i32> %passThru, i16 %mask, <16 x i32> %extra_param, <16 x i32> %extra_param2)  #0 {
+;
+; CHECK-LABEL: @test_mask_sub_epi32_rmbk(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP5:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 144) to ptr), align 8
+; CHECK-NEXT:    [[TMP6:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 208) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 136) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 72) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP18:%.*]], label [[TMP19:%.*]], !prof [[PROF1]]
+; CHECK:       7:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       8:
+; CHECK-NEXT:    [[Q:%.*]] = load i32, ptr [[PTR_B:%.*]], align 4
+; CHECK-NEXT:    [[TMP7:%.*]] = ptrtoint ptr [[PTR_B]] to i64
+; CHECK-NEXT:    [[TMP8:%.*]] = xor i64 [[TMP7]], 87960930222080
+; CHECK-NEXT:    [[TMP9:%.*]] = inttoptr i64 [[TMP8]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load i32, ptr [[TMP9]], align 4
+; CHECK-NEXT:    [[_MSPROP:%.*]] = insertelement <16 x i32> [[TMP5]], i32 [[_MSLD]], i32 0
+; CHECK-NEXT:    [[VECINIT_I:%.*]] = insertelement <16 x i32> [[EXTRA_PARAM:%.*]], i32 [[Q]], i32 0
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = shufflevector <16 x i32> [[_MSPROP]], <16 x i32> [[TMP6]], <16 x i32> zeroinitializer
+; CHECK-NEXT:    [[B:%.*]] = shufflevector <16 x i32> [[VECINIT_I]], <16 x i32> [[EXTRA_PARAM2:%.*]], <16 x i32> zeroinitializer
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = or <16 x i32> [[TMP2]], [[_MSPROP1]]
+; CHECK-NEXT:    [[TMP10:%.*]] = sub <16 x i32> [[A:%.*]], [[B]]
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast i16 [[TMP3]] to <16 x i1>
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast i16 [[MASK:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP13:%.*]] = select <16 x i1> [[TMP12]], <16 x i32> [[_MSPROP2]], <16 x i32> [[TMP4]]
+; CHECK-NEXT:    [[TMP14:%.*]] = xor <16 x i32> [[TMP10]], [[PASSTHRU:%.*]]
+; CHECK-NEXT:    [[TMP15:%.*]] = or <16 x i32> [[TMP14]], [[_MSPROP2]]
+; CHECK-NEXT:    [[TMP16:%.*]] = or <16 x i32> [[TMP15]], [[TMP4]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP11]], <16 x i32> [[TMP16]], <16 x i32> [[TMP13]]
+; CHECK-NEXT:    [[TMP17:%.*]] = select <16 x i1> [[TMP12]], <16 x i32> [[TMP10]], <16 x i32> [[PASSTHRU]]
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i32> [[TMP17]]
+;
+  %q = load i32, ptr %ptr_b
+  %vecinit.i = insertelement <16 x i32> %extra_param, i32 %q, i32 0
+  %b = shufflevector <16 x i32> %vecinit.i, <16 x i32> %extra_param2, <16 x i32> zeroinitializer
+  %res = call <16 x i32> @llvm.x86.avx512.mask.psub.d.512(<16 x i32> %a, <16 x i32> %b, <16 x i32> %passThru, i16 %mask)
+  ret < 16 x i32> %res
+}
+
+define <16 x i32> @test_mask_sub_epi32_rmbkz(<16 x i32> %a, ptr %ptr_b, i16 %mask, <16 x i32> %extra_param)  #0 {
+;
+; CHECK-LABEL: @test_mask_sub_epi32_rmbkz(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 80) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 72) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP5:%.*]], label [[TMP17:%.*]], !prof [[PROF1]]
+; CHECK:       5:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       6:
+; CHECK-NEXT:    [[Q:%.*]] = load i32, ptr [[PTR_B:%.*]], align 4
+; CHECK-NEXT:    [[TMP6:%.*]] = ptrtoint ptr [[PTR_B]] to i64
+; CHECK-NEXT:    [[TMP7:%.*]] = xor i64 [[TMP6]], 87960930222080
+; CHECK-NEXT:    [[TMP8:%.*]] = inttoptr i64 [[TMP7]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load i32, ptr [[TMP8]], align 4
+; CHECK-NEXT:    [[_MSPROP:%.*]] = insertelement <16 x i32> [[TMP4]], i32 [[_MSLD]], i32 0
+; CHECK-NEXT:    [[VECINIT_I:%.*]] = insertelement <16 x i32> [[EXTRA_PARAM:%.*]], i32 [[Q]], i32 0
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = shufflevector <16 x i32> [[_MSPROP]], <16 x i32> [[TMP4]], <16 x i32> zeroinitializer
+; CHECK-NEXT:    [[B:%.*]] = shufflevector <16 x i32> [[VECINIT_I]], <16 x i32> [[EXTRA_PARAM]], <16 x i32> zeroinitializer
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = or <16 x i32> [[TMP2]], [[_MSPROP1]]
+; CHECK-NEXT:    [[TMP9:%.*]] = sub <16 x i32> [[A:%.*]], [[B]]
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i16 [[TMP3]] to <16 x i1>
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast i16 [[MASK:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP12:%.*]] = select <16 x i1> [[TMP11]], <16 x i32> [[_MSPROP2]], <16 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP13:%.*]] = xor <16 x i32> [[TMP9]], zeroinitializer
+; CHECK-NEXT:    [[TMP14:%.*]] = or <16 x i32> [[TMP13]], [[_MSPROP2]]
+; CHECK-NEXT:    [[TMP15:%.*]] = or <16 x i32> [[TMP14]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP10]], <16 x i32> [[TMP15]], <16 x i32> [[TMP12]]
+; CHECK-NEXT:    [[TMP16:%.*]] = select <16 x i1> [[TMP11]], <16 x i32> [[TMP9]], <16 x i32> zeroinitializer
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i32> [[TMP16]]
+;
+  %q = load i32, ptr %ptr_b
+  %vecinit.i = insertelement <16 x i32> %extra_param, i32 %q, i32 0
+  %b = shufflevector <16 x i32> %vecinit.i, <16 x i32> %extra_param, <16 x i32> zeroinitializer
+  %res = call <16 x i32> @llvm.x86.avx512.mask.psub.d.512(<16 x i32> %a, <16 x i32> %b, <16 x i32> zeroinitializer, i16 %mask)
+  ret < 16 x i32> %res
+}
+
+declare <16 x i32> @llvm.x86.avx512.mask.psub.d.512(<16 x i32>, <16 x i32>, <16 x i32>, i16)
+
+define <8 x i64> @test_mask_add_epi64_rr(<8 x i64> %a, <8 x i64> %b)  #0 {
+; CHECK-LABEL: @test_mask_add_epi64_rr(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <8 x i64> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP3:%.*]] = add <8 x i64> [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    store <8 x i64> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i64> [[TMP3]]
+;
+  %res = call <8 x i64> @llvm.x86.avx512.mask.padd.q.512(<8 x i64> %a, <8 x i64> %b, <8 x i64> zeroinitializer, i8 -1)
+  ret < 8 x i64> %res
+}
+
+define <8 x i64> @test_mask_add_epi64_rrk(<8 x i64> %a, <8 x i64> %b, <8 x i64> %passThru, i8 %mask)  #0 {
+;
+; CHECK-LABEL: @test_mask_add_epi64_rrk(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <8 x i64> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP5:%.*]] = add <8 x i64> [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i8 [[TMP3]] to <8 x i1>
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP8:%.*]] = select <8 x i1> [[TMP7]], <8 x i64> [[_MSPROP]], <8 x i64> [[TMP4]]
+; CHECK-NEXT:    [[TMP9:%.*]] = xor <8 x i64> [[TMP5]], [[PASSTHRU:%.*]]
+; CHECK-NEXT:    [[TMP10:%.*]] = or <8 x i64> [[TMP9]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP11:%.*]] = or <8 x i64> [[TMP10]], [[TMP4]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP6]], <8 x i64> [[TMP11]], <8 x i64> [[TMP8]]
+; CHECK-NEXT:    [[TMP12:%.*]] = select <8 x i1> [[TMP7]], <8 x i64> [[TMP5]], <8 x i64> [[PASSTHRU]]
+; CHECK-NEXT:    store <8 x i64> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i64> [[TMP12]]
+;
+  %res = call <8 x i64> @llvm.x86.avx512.mask.padd.q.512(<8 x i64> %a, <8 x i64> %b, <8 x i64> %passThru, i8 %mask)
+  ret < 8 x i64> %res
+}
+
+define <8 x i64> @test_mask_add_epi64_rrkz(<8 x i64> %a, <8 x i64> %b, i8 %mask)  #0 {
+;
+; CHECK-LABEL: @test_mask_add_epi64_rrkz(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <8 x i64> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = add <8 x i64> [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i8 [[TMP3]] to <8 x i1>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP7:%.*]] = select <8 x i1> [[TMP6]], <8 x i64> [[_MSPROP]], <8 x i64> zeroinitializer
+; CHECK-NEXT:    [[TMP8:%.*]] = xor <8 x i64> [[TMP4]], zeroinitializer
+; CHECK-NEXT:    [[TMP9:%.*]] = or <8 x i64> [[TMP8]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP10:%.*]] = or <8 x i64> [[TMP9]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP5]], <8 x i64> [[TMP10]], <8 x i64> [[TMP7]]
+; CHECK-NEXT:    [[TMP11:%.*]] = select <8 x i1> [[TMP6]], <8 x i64> [[TMP4]], <8 x i64> zeroinitializer
+; CHECK-NEXT:    store <8 x i64> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i64> [[TMP11]]
+;
+  %res = call <8 x i64> @llvm.x86.avx512.mask.padd.q.512(<8 x i64> %a, <8 x i64> %b, <8 x i64> zeroinitializer, i8 %mask)
+  ret < 8 x i64> %res
+}
+
+define <8 x i64> @test_mask_add_epi64_rm(<8 x i64> %a, ptr %ptr_b)  #0 {
+;
+; CHECK-LABEL: @test_mask_add_epi64_rm(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP3:%.*]], label [[TMP4:%.*]], !prof [[PROF1]]
+; CHECK:       3:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       4:
+; CHECK-NEXT:    [[B:%.*]] = load <8 x i64>, ptr [[PTR_B:%.*]], align 64
+; CHECK-NEXT:    [[TMP5:%.*]] = ptrtoint ptr [[PTR_B]] to i64
+; CHECK-NEXT:    [[TMP6:%.*]] = xor i64 [[TMP5]], 87960930222080
+; CHECK-NEXT:    [[TMP7:%.*]] = inttoptr i64 [[TMP6]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <8 x i64>, ptr [[TMP7]], align 64
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <8 x i64> [[TMP2]], [[_MSLD]]
+; CHECK-NEXT:    [[TMP8:%.*]] = add <8 x i64> [[A:%.*]], [[B]]
+; CHECK-NEXT:    store <8 x i64> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i64> [[TMP8]]
+;
+  %b = load <8 x i64>, ptr %ptr_b
+  %res = call <8 x i64> @llvm.x86.avx512.mask.padd.q.512(<8 x i64> %a, <8 x i64> %b, <8 x i64> zeroinitializer, i8 -1)
+  ret < 8 x i64> %res
+}
+
+define <8 x i64> @test_mask_add_epi64_rmk(<8 x i64> %a, ptr %ptr_b, <8 x i64> %passThru, i8 %mask)  #0 {
+;
+; CHECK-LABEL: @test_mask_add_epi64_rmk(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 136) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 72) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP5:%.*]], label [[TMP6:%.*]], !prof [[PROF1]]
+; CHECK:       5:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       6:
+; CHECK-NEXT:    [[B:%.*]] = load <8 x i64>, ptr [[PTR_B:%.*]], align 64
+; CHECK-NEXT:    [[TMP7:%.*]] = ptrtoint ptr [[PTR_B]] to i64
+; CHECK-NEXT:    [[TMP8:%.*]] = xor i64 [[TMP7]], 87960930222080
+; CHECK-NEXT:    [[TMP9:%.*]] = inttoptr i64 [[TMP8]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <8 x i64>, ptr [[TMP9]], align 64
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <8 x i64> [[TMP2]], [[_MSLD]]
+; CHECK-NEXT:    [[TMP10:%.*]] = add <8 x i64> [[A:%.*]], [[B]]
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast i8 [[TMP3]] to <8 x i1>
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP13:%.*]] = select <8 x i1> [[TMP12]], <8 x i64> [[_MSPROP]], <8 x i64> [[TMP4]]
+; CHECK-NEXT:    [[TMP14:%.*]] = xor <8 x i64> [[TMP10]], [[PASSTHRU:%.*]]
+; CHECK-NEXT:    [[TMP15:%.*]] = or <8 x i64> [[TMP14]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP16:%.*]] = or <8 x i64> [[TMP15]], [[TMP4]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP11]], <8 x i64> [[TMP16]], <8 x i64> [[TMP13]]
+; CHECK-NEXT:    [[TMP17:%.*]] = select <8 x i1> [[TMP12]], <8 x i64> [[TMP10]], <8 x i64> [[PASSTHRU]]
+; CHECK-NEXT:    store <8 x i64> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i64> [[TMP17]]
+;
+  %b = load <8 x i64>, ptr %ptr_b
+  %res = call <8 x i64> @llvm.x86.avx512.mask.padd.q.512(<8 x i64> %a, <8 x i64> %b, <8 x i64> %passThru, i8 %mask)
+  ret < 8 x i64> %res
+}
+
+define <8 x i64> @test_mask_add_epi64_rmkz(<8 x i64> %a, ptr %ptr_b, i8 %mask)  #0 {
+;
+; CHECK-LABEL: @test_mask_add_epi64_rmkz(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 72) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP4:%.*]], label [[TMP5:%.*]], !prof [[PROF1]]
+; CHECK:       4:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       5:
+; CHECK-NEXT:    [[B:%.*]] = load <8 x i64>, ptr [[PTR_B:%.*]], align 64
+; CHECK-NEXT:    [[TMP6:%.*]] = ptrtoint ptr [[PTR_B]] to i64
+; CHECK-NEXT:    [[TMP7:%.*]] = xor i64 [[TMP6]], 87960930222080
+; CHECK-NEXT:    [[TMP8:%.*]] = inttoptr i64 [[TMP7]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <8 x i64>, ptr [[TMP8]], align 64
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <8 x i64> [[TMP2]], [[_MSLD]]
+; CHECK-NEXT:    [[TMP9:%.*]] = add <8 x i64> [[A:%.*]], [[B]]
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i8 [[TMP3]] to <8 x i1>
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP12:%.*]] = select <8 x i1> [[TMP11]], <8 x i64> [[_MSPROP]], <8 x i64> zeroinitializer
+; CHECK-NEXT:    [[TMP13:%.*]] = xor <8 x i64> [[TMP9]], zeroinitializer
+; CHECK-NEXT:    [[TMP14:%.*]] = or <8 x i64> [[TMP13]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP15:%.*]] = or <8 x i64> [[TMP14]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP10]], <8 x i64> [[TMP15]], <8 x i64> [[TMP12]]
+; CHECK-NEXT:    [[TMP16:%.*]] = select <8 x i1> [[TMP11]], <8 x i64> [[TMP9]], <8 x i64> zeroinitializer
+; CHECK-NEXT:    store <8 x i64> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i64> [[TMP16]]
+;
+  %b = load <8 x i64>, ptr %ptr_b
+  %res = call <8 x i64> @llvm.x86.avx512.mask.padd.q.512(<8 x i64> %a, <8 x i64> %b, <8 x i64> zeroinitializer, i8 %mask)
+  ret < 8 x i64> %res
+}
+
+define <8 x i64> @test_mask_add_epi64_rmb(<8 x i64> %a, ptr %ptr_b, <8 x i64> %extra_param, <8 x i64> %extra_param2)  #0 {
+;
+; CHECK-LABEL: @test_mask_add_epi64_rmb(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 72) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 136) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP9:%.*]], label [[TMP10:%.*]], !prof [[PROF1]]
+; CHECK:       5:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       6:
+; CHECK-NEXT:    [[Q:%.*]] = load i64, ptr [[PTR_B:%.*]], align 8
+; CHECK-NEXT:    [[TMP5:%.*]] = ptrtoint ptr [[PTR_B]] to i64
+; CHECK-NEXT:    [[TMP6:%.*]] = xor i64 [[TMP5]], 87960930222080
+; CHECK-NEXT:    [[TMP7:%.*]] = inttoptr i64 [[TMP6]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load i64, ptr [[TMP7]], align 8
+; CHECK-NEXT:    [[_MSPROP:%.*]] = insertelement <8 x i64> [[TMP3]], i64 [[_MSLD]], i32 0
+; CHECK-NEXT:    [[VECINIT_I:%.*]] = insertelement <8 x i64> [[EXTRA_PARAM:%.*]], i64 [[Q]], i32 0
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = shufflevector <8 x i64> [[_MSPROP]], <8 x i64> [[TMP4]], <8 x i32> zeroinitializer
+; CHECK-NEXT:    [[B:%.*]] = shufflevector <8 x i64> [[VECINIT_I]], <8 x i64> [[EXTRA_PARAM2:%.*]], <8 x i32> zeroinitializer
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = or <8 x i64> [[TMP2]], [[_MSPROP1]]
+; CHECK-NEXT:    [[TMP8:%.*]] = add <8 x i64> [[A:%.*]], [[B]]
+; CHECK-NEXT:    store <8 x i64> [[_MSPROP2]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i64> [[TMP8]]
+;
+  %q = load i64, ptr %ptr_b
+  %vecinit.i = insertelement <8 x i64> %extra_param, i64 %q, i32 0
+  %b = shufflevector <8 x i64> %vecinit.i, <8 x i64> %extra_param2, <8 x i32> zeroinitializer
+  %res = call <8 x i64> @llvm.x86.avx512.mask.padd.q.512(<8 x i64> %a, <8 x i64> %b, <8 x i64> zeroinitializer, i8 -1)
+  ret < 8 x i64> %res
+}
+
+define <8 x i64> @test_mask_add_epi64_rmbk(<8 x i64> %a, ptr %ptr_b, <8 x i64> %passThru, i8 %mask, <8 x i64> %extra_param, <8 x i64> %extra_param2)  #0 {
+;
+; CHECK-LABEL: @test_mask_add_epi64_rmbk(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP5:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 144) to ptr), align 8
+; CHECK-NEXT:    [[TMP6:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 208) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 136) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 72) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP18:%.*]], label [[TMP19:%.*]], !prof [[PROF1]]
+; CHECK:       7:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       8:
+; CHECK-NEXT:    [[Q:%.*]] = load i64, ptr [[PTR_B:%.*]], align 8
+; CHECK-NEXT:    [[TMP7:%.*]] = ptrtoint ptr [[PTR_B]] to i64
+; CHECK-NEXT:    [[TMP8:%.*]] = xor i64 [[TMP7]], 87960930222080
+; CHECK-NEXT:    [[TMP9:%.*]] = inttoptr i64 [[TMP8]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load i64, ptr [[TMP9]], align 8
+; CHECK-NEXT:    [[_MSPROP:%.*]] = insertelement <8 x i64> [[TMP5]], i64 [[_MSLD]], i32 0
+; CHECK-NEXT:    [[VECINIT_I:%.*]] = insertelement <8 x i64> [[EXTRA_PARAM:%.*]], i64 [[Q]], i32 0
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = shufflevector <8 x i64> [[_MSPROP]], <8 x i64> [[TMP6]], <8 x i32> zeroinitializer
+; CHECK-NEXT:    [[B:%.*]] = shufflevector <8 x i64> [[VECINIT_I]], <8 x i64> [[EXTRA_PARAM2:%.*]], <8 x i32> zeroinitializer
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = or <8 x i64> [[TMP2]], [[_MSPROP1]]
+; CHECK-NEXT:    [[TMP10:%.*]] = add <8 x i64> [[A:%.*]], [[B]]
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast i8 [[TMP3]] to <8 x i1>
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP13:%.*]] = select <8 x i1> [[TMP12]], <8 x i64> [[_MSPROP2]], <8 x i64> [[TMP4]]
+; CHECK-NEXT:    [[TMP14:%.*]] = xor <8 x i64> [[TMP10]], [[PASSTHRU:%.*]]
+; CHECK-NEXT:    [[TMP15:%.*]] = or <8 x i64> [[TMP14]], [[_MSPROP2]]
+; CHECK-NEXT:    [[TMP16:%.*]] = or <8 x i64> [[TMP15]], [[TMP4]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP11]], <8 x i64> [[TMP16]], <8 x i64> [[TMP13]]
+; CHECK-NEXT:    [[TMP17:%.*]] = select <8 x i1> [[TMP12]], <8 x i64> [[TMP10]], <8 x i64> [[PASSTHRU]]
+; CHECK-NEXT:    store <8 x i64> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i64> [[TMP17]]
+;
+  %q = load i64, ptr %ptr_b
+  %vecinit.i = insertelement <8 x i64> %extra_param, i64 %q, i32 0
+  %b = shufflevector <8 x i64> %vecinit.i, <8 x i64> %extra_param2, <8 x i32> zeroinitializer
+  %res = call <8 x i64> @llvm.x86.avx512.mask.padd.q.512(<8 x i64> %a, <8 x i64> %b, <8 x i64> %passThru, i8 %mask)
+  ret < 8 x i64> %res
+}
+
+define <8 x i64> @test_mask_add_epi64_rmbkz(<8 x i64> %a, ptr %ptr_b, i8 %mask, <8 x i64> %extra_param, <8 x i64> %extra_param2)  #0 {
+;
+; CHECK-LABEL: @test_mask_add_epi64_rmbkz(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 80) to ptr), align 8
+; CHECK-NEXT:    [[TMP5:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 144) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 72) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP17:%.*]], label [[TMP18:%.*]], !prof [[PROF1]]
+; CHECK:       6:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       7:
+; CHECK-NEXT:    [[Q:%.*]] = load i64, ptr [[PTR_B:%.*]], align 8
+; CHECK-NEXT:    [[TMP6:%.*]] = ptrtoint ptr [[PTR_B]] to i64
+; CHECK-NEXT:    [[TMP7:%.*]] = xor i64 [[TMP6]], 87960930222080
+; CHECK-NEXT:    [[TMP8:%.*]] = inttoptr i64 [[TMP7]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load i64, ptr [[TMP8]], align 8
+; CHECK-NEXT:    [[_MSPROP:%.*]] = insertelement <8 x i64> [[TMP4]], i64 [[_MSLD]], i32 0
+; CHECK-NEXT:    [[VECINIT_I:%.*]] = insertelement <8 x i64> [[EXTRA_PARAM:%.*]], i64 [[Q]], i32 0
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = shufflevector <8 x i64> [[_MSPROP]], <8 x i64> [[TMP5]], <8 x i32> zeroinitializer
+; CHECK-NEXT:    [[B:%.*]] = shufflevector <8 x i64> [[VECINIT_I]], <8 x i64> [[EXTRA_PARAM2:%.*]], <8 x i32> zeroinitializer
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = or <8 x i64> [[TMP2]], [[_MSPROP1]]
+; CHECK-NEXT:    [[TMP9:%.*]] = add <8 x i64> [[A:%.*]], [[B]]
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i8 [[TMP3]] to <8 x i1>
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP12:%.*]] = select <8 x i1> [[TMP11]], <8 x i64> [[_MSPROP2]], <8 x i64> zeroinitializer
+; CHECK-NEXT:    [[TMP13:%.*]] = xor <8 x i64> [[TMP9]], zeroinitializer
+; CHECK-NEXT:    [[TMP14:%.*]] = or <8 x i64> [[TMP13]], [[_MSPROP2]]
+; CHECK-NEXT:    [[TMP15:%.*]] = or <8 x i64> [[TMP14]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP10]], <8 x i64> [[TMP15]], <8 x i64> [[TMP12]]
+; CHECK-NEXT:    [[TMP16:%.*]] = select <8 x i1> [[TMP11]], <8 x i64> [[TMP9]], <8 x i64> zeroinitializer
+; CHECK-NEXT:    store <8 x i64> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i64> [[TMP16]]
+;
+  %q = load i64, ptr %ptr_b
+  %vecinit.i = insertelement <8 x i64> %extra_param, i64 %q, i32 0
+  %b = shufflevector <8 x i64> %vecinit.i, <8 x i64> %extra_param2, <8 x i32> zeroinitializer
+  %res = call <8 x i64> @llvm.x86.avx512.mask.padd.q.512(<8 x i64> %a, <8 x i64> %b, <8 x i64> zeroinitializer, i8 %mask)
+  ret < 8 x i64> %res
+}
+
+declare <8 x i64> @llvm.x86.avx512.mask.padd.q.512(<8 x i64>, <8 x i64>, <8 x i64>, i8)
+
+define <8 x i64> @test_mask_sub_epi64_rr(<8 x i64> %a, <8 x i64> %b)  #0 {
+; CHECK-LABEL: @test_mask_sub_epi64_rr(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <8 x i64> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP3:%.*]] = sub <8 x i64> [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    store <8 x i64> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i64> [[TMP3]]
+;
+  %res = call <8 x i64> @llvm.x86.avx512.mask.psub.q.512(<8 x i64> %a, <8 x i64> %b, <8 x i64> zeroinitializer, i8 -1)
+  ret < 8 x i64> %res
+}
+
+define <8 x i64> @test_mask_sub_epi64_rrk(<8 x i64> %a, <8 x i64> %b, <8 x i64> %passThru, i8 %mask)  #0 {
+;
+; CHECK-LABEL: @test_mask_sub_epi64_rrk(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <8 x i64> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP5:%.*]] = sub <8 x i64> [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i8 [[TMP3]] to <8 x i1>
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP8:%.*]] = select <8 x i1> [[TMP7]], <8 x i64> [[_MSPROP]], <8 x i64> [[TMP4]]
+; CHECK-NEXT:    [[TMP9:%.*]] = xor <8 x i64> [[TMP5]], [[PASSTHRU:%.*]]
+; CHECK-NEXT:    [[TMP10:%.*]] = or <8 x i64> [[TMP9]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP11:%.*]] = or <8 x i64> [[TMP10]], [[TMP4]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP6]], <8 x i64> [[TMP11]], <8 x i64> [[TMP8]]
+; CHECK-NEXT:    [[TMP12:%.*]] = select <8 x i1> [[TMP7]], <8 x i64> [[TMP5]], <8 x i64> [[PASSTHRU]]
+; CHECK-NEXT:    store <8 x i64> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i64> [[TMP12]]
+;
+  %res = call <8 x i64> @llvm.x86.avx512.mask.psub.q.512(<8 x i64> %a, <8 x i64> %b, <8 x i64> %passThru, i8 %mask)
+  ret < 8 x i64> %res
+}
+
+define <8 x i64> @test_mask_sub_epi64_rrkz(<8 x i64> %a, <8 x i64> %b, i8 %mask)  #0 {
+;
+; CHECK-LABEL: @test_mask_sub_epi64_rrkz(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <8 x i64> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = sub <8 x i64> [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i8 [[TMP3]] to <8 x i1>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP7:%.*]] = select <8 x i1> [[TMP6]], <8 x i64> [[_MSPROP]], <8 x i64> zeroinitializer
+; CHECK-NEXT:    [[TMP8:%.*]] = xor <8 x i64> [[TMP4]], zeroinitializer
+; CHECK-NEXT:    [[TMP9:%.*]] = or <8 x i64> [[TMP8]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP10:%.*]] = or <8 x i64> [[TMP9]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP5]], <8 x i64> [[TMP10]], <8 x i64> [[TMP7]]
+; CHECK-NEXT:    [[TMP11:%.*]] = select <8 x i1> [[TMP6]], <8 x i64> [[TMP4]], <8 x i64> zeroinitializer
+; CHECK-NEXT:    store <8 x i64> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i64> [[TMP11]]
+;
+  %res = call <8 x i64> @llvm.x86.avx512.mask.psub.q.512(<8 x i64> %a, <8 x i64> %b, <8 x i64> zeroinitializer, i8 %mask)
+  ret < 8 x i64> %res
+}
+
+define <8 x i64> @test_mask_sub_epi64_rm(<8 x i64> %a, ptr %ptr_b)  #0 {
+;
+; CHECK-LABEL: @test_mask_sub_epi64_rm(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP3:%.*]], label [[TMP4:%.*]], !prof [[PROF1]]
+; CHECK:       3:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       4:
+; CHECK-NEXT:    [[B:%.*]] = load <8 x i64>, ptr [[PTR_B:%.*]], align 64
+; CHECK-NEXT:    [[TMP5:%.*]] = ptrtoint ptr [[PTR_B]] to i64
+; CHECK-NEXT:    [[TMP6:%.*]] = xor i64 [[TMP5]], 87960930222080
+; CHECK-NEXT:    [[TMP7:%.*]] = inttoptr i64 [[TMP6]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <8 x i64>, ptr [[TMP7]], align 64
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <8 x i64> [[TMP2]], [[_MSLD]]
+; CHECK-NEXT:    [[TMP8:%.*]] = sub <8 x i64> [[A:%.*]], [[B]]
+; CHECK-NEXT:    store <8 x i64> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i64> [[TMP8]]
+;
+  %b = load <8 x i64>, ptr %ptr_b
+  %res = call <8 x i64> @llvm.x86.avx512.mask.psub.q.512(<8 x i64> %a, <8 x i64> %b, <8 x i64> zeroinitializer, i8 -1)
+  ret < 8 x i64> %res
+}
+
+define <8 x i64> @test_mask_sub_epi64_rmk(<8 x i64> %a, ptr %ptr_b, <8 x i64> %passThru, i8 %mask)  #0 {
+;
+; CHECK-LABEL: @test_mask_sub_epi64_rmk(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 136) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 72) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP5:%.*]], label [[TMP6:%.*]], !prof [[PROF1]]
+; CHECK:       5:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       6:
+; CHECK-NEXT:    [[B:%.*]] = load <8 x i64>, ptr [[PTR_B:%.*]], align 64
+; CHECK-NEXT:    [[TMP7:%.*]] = ptrtoint ptr [[PTR_B]] to i64
+; CHECK-NEXT:    [[TMP8:%.*]] = xor i64 [[TMP7]], 87960930222080
+; CHECK-NEXT:    [[TMP9:%.*]] = inttoptr i64 [[TMP8]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <8 x i64>, ptr [[TMP9]], align 64
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <8 x i64> [[TMP2]], [[_MSLD]]
+; CHECK-NEXT:    [[TMP10:%.*]] = sub <8 x i64> [[A:%.*]], [[B]]
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast i8 [[TMP3]] to <8 x i1>
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP13:%.*]] = select <8 x i1> [[TMP12]], <8 x i64> [[_MSPROP]], <8 x i64> [[TMP4]]
+; CHECK-NEXT:    [[TMP14:%.*]] = xor <8 x i64> [[TMP10]], [[PASSTHRU:%.*]]
+; CHECK-NEXT:    [[TMP15:%.*]] = or <8 x i64> [[TMP14]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP16:%.*]] = or <8 x i64> [[TMP15]], [[TMP4]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP11]], <8 x i64> [[TMP16]], <8 x i64> [[TMP13]]
+; CHECK-NEXT:    [[TMP17:%.*]] = select <8 x i1> [[TMP12]], <8 x i64> [[TMP10]], <8 x i64> [[PASSTHRU]]
+; CHECK-NEXT:    store <8 x i64> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i64> [[TMP17]]
+;
+  %b = load <8 x i64>, ptr %ptr_b
+  %res = call <8 x i64> @llvm.x86.avx512.mask.psub.q.512(<8 x i64> %a, <8 x i64> %b, <8 x i64> %passThru, i8 %mask)
+  ret < 8 x i64> %res
+}
+
+define <8 x i64> @test_mask_sub_epi64_rmkz(<8 x i64> %a, ptr %ptr_b, i8 %mask)  #0 {
+;
+; CHECK-LABEL: @test_mask_sub_epi64_rmkz(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 72) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP4:%.*]], label [[TMP5:%.*]], !prof [[PROF1]]
+; CHECK:       4:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       5:
+; CHECK-NEXT:    [[B:%.*]] = load <8 x i64>, ptr [[PTR_B:%.*]], align 64
+; CHECK-NEXT:    [[TMP6:%.*]] = ptrtoint ptr [[PTR_B]] to i64
+; CHECK-NEXT:    [[TMP7:%.*]] = xor i64 [[TMP6]], 87960930222080
+; CHECK-NEXT:    [[TMP8:%.*]] = inttoptr i64 [[TMP7]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <8 x i64>, ptr [[TMP8]], align 64
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <8 x i64> [[TMP2]], [[_MSLD]]
+; CHECK-NEXT:    [[TMP9:%.*]] = sub <8 x i64> [[A:%.*]], [[B]]
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i8 [[TMP3]] to <8 x i1>
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP12:%.*]] = select <8 x i1> [[TMP11]], <8 x i64> [[_MSPROP]], <8 x i64> zeroinitializer
+; CHECK-NEXT:    [[TMP13:%.*]] = xor <8 x i64> [[TMP9]], zeroinitializer
+; CHECK-NEXT:    [[TMP14:%.*]] = or <8 x i64> [[TMP13]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP15:%.*]] = or <8 x i64> [[TMP14]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP10]], <8 x i64> [[TMP15]], <8 x i64> [[TMP12]]
+; CHECK-NEXT:    [[TMP16:%.*]] = select <8 x i1> [[TMP11]], <8 x i64> [[TMP9]], <8 x i64> zeroinitializer
+; CHECK-NEXT:    store <8 x i64> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i64> [[TMP16]]
+;
+  %b = load <8 x i64>, ptr %ptr_b
+  %res = call <8 x i64> @llvm.x86.avx512.mask.psub.q.512(<8 x i64> %a, <8 x i64> %b, <8 x i64> zeroinitializer, i8 %mask)
+  ret < 8 x i64> %res
+}
+
+define <8 x i64> @test_mask_sub_epi64_rmb(<8 x i64> %a, ptr %ptr_b, <8 x i64> %extra_param, <8 x i64> %extra_param2)  #0 {
+;
+; CHECK-LABEL: @test_mask_sub_epi64_rmb(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 72) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 136) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP9:%.*]], label [[TMP10:%.*]], !prof [[PROF1]]
+; CHECK:       5:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       6:
+; CHECK-NEXT:    [[Q:%.*]] = load i64, ptr [[PTR_B:%.*]], align 8
+; CHECK-NEXT:    [[TMP5:%.*]] = ptrtoint ptr [[PTR_B]] to i64
+; CHECK-NEXT:    [[TMP6:%.*]] = xor i64 [[TMP5]], 87960930222080
+; CHECK-NEXT:    [[TMP7:%.*]] = inttoptr i64 [[TMP6]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load i64, ptr [[TMP7]], align 8
+; CHECK-NEXT:    [[_MSPROP:%.*]] = insertelement <8 x i64> [[TMP3]], i64 [[_MSLD]], i32 0
+; CHECK-NEXT:    [[VECINIT_I:%.*]] = insertelement <8 x i64> [[EXTRA_PARAM:%.*]], i64 [[Q]], i32 0
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = shufflevector <8 x i64> [[_MSPROP]], <8 x i64> [[TMP4]], <8 x i32> zeroinitializer
+; CHECK-NEXT:    [[B:%.*]] = shufflevector <8 x i64> [[VECINIT_I]], <8 x i64> [[EXTRA_PARAM2:%.*]], <8 x i32> zeroinitializer
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = or <8 x i64> [[TMP2]], [[_MSPROP1]]
+; CHECK-NEXT:    [[TMP8:%.*]] = sub <8 x i64> [[A:%.*]], [[B]]
+; CHECK-NEXT:    store <8 x i64> [[_MSPROP2]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i64> [[TMP8]]
+;
+  %q = load i64, ptr %ptr_b
+  %vecinit.i = insertelement <8 x i64> %extra_param, i64 %q, i32 0
+  %b = shufflevector <8 x i64> %vecinit.i, <8 x i64> %extra_param2, <8 x i32> zeroinitializer
+  %res = call <8 x i64> @llvm.x86.avx512.mask.psub.q.512(<8 x i64> %a, <8 x i64> %b, <8 x i64> zeroinitializer, i8 -1)
+  ret < 8 x i64> %res
+}
+
+define <8 x i64> @test_mask_sub_epi64_rmbk(<8 x i64> %a, ptr %ptr_b, <8 x i64> %passThru, i8 %mask, <8 x i64> %extra_param, <8 x i64> %extra_param2)  #0 {
+;
+; CHECK-LABEL: @test_mask_sub_epi64_rmbk(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP5:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 144) to ptr), align 8
+; CHECK-NEXT:    [[TMP6:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 208) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 136) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 72) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP18:%.*]], label [[TMP19:%.*]], !prof [[PROF1]]
+; CHECK:       7:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       8:
+; CHECK-NEXT:    [[Q:%.*]] = load i64, ptr [[PTR_B:%.*]], align 8
+; CHECK-NEXT:    [[TMP7:%.*]] = ptrtoint ptr [[PTR_B]] to i64
+; CHECK-NEXT:    [[TMP8:%.*]] = xor i64 [[TMP7]], 87960930222080
+; CHECK-NEXT:    [[TMP9:%.*]] = inttoptr i64 [[TMP8]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load i64, ptr [[TMP9]], align 8
+; CHECK-NEXT:    [[_MSPROP:%.*]] = insertelement <8 x i64> [[TMP5]], i64 [[_MSLD]], i32 0
+; CHECK-NEXT:    [[VECINIT_I:%.*]] = insertelement <8 x i64> [[EXTRA_PARAM:%.*]], i64 [[Q]], i32 0
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = shufflevector <8 x i64> [[_MSPROP]], <8 x i64> [[TMP6]], <8 x i32> zeroinitializer
+; CHECK-NEXT:    [[B:%.*]] = shufflevector <8 x i64> [[VECINIT_I]], <8 x i64> [[EXTRA_PARAM2:%.*]], <8 x i32> zeroinitializer
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = or <8 x i64> [[TMP2]], [[_MSPROP1]]
+; CHECK-NEXT:    [[TMP10:%.*]] = sub <8 x i64> [[A:%.*]], [[B]]
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast i8 [[TMP3]] to <8 x i1>
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP13:%.*]] = select <8 x i1> [[TMP12]], <8 x i64> [[_MSPROP2]], <8 x i64> [[TMP4]]
+; CHECK-NEXT:    [[TMP14:%.*]] = xor <8 x i64> [[TMP10]], [[PASSTHRU:%.*]]
+; CHECK-NEXT:    [[TMP15:%.*]] = or <8 x i64> [[TMP14]], [[_MSPROP2]]
+; CHECK-NEXT:    [[TMP16:%.*]] = or <8 x i64> [[TMP15]], [[TMP4]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP11]], <8 x i64> [[TMP16]], <8 x i64> [[TMP13]]
+; CHECK-NEXT:    [[TMP17:%.*]] = select <8 x i1> [[TMP12]], <8 x i64> [[TMP10]], <8 x i64> [[PASSTHRU]]
+; CHECK-NEXT:    store <8 x i64> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i64> [[TMP17]]
+;
+  %q = load i64, ptr %ptr_b
+  %vecinit.i = insertelement <8 x i64> %extra_param, i64 %q, i32 0
+  %b = shufflevector <8 x i64> %vecinit.i, <8 x i64> %extra_param2, <8 x i32> zeroinitializer
+  %res = call <8 x i64> @llvm.x86.avx512.mask.psub.q.512(<8 x i64> %a, <8 x i64> %b, <8 x i64> %passThru, i8 %mask)
+  ret < 8 x i64> %res
+}
+
+define <8 x i64> @test_mask_sub_epi64_rmbkz(<8 x i64> %a, ptr %ptr_b, i8 %mask, <8 x i64> %extra_param, <8 x i64> %extra_param2)  #0 {
+;
+; CHECK-LABEL: @test_mask_sub_epi64_rmbkz(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 80) to ptr), align 8
+; CHECK-NEXT:    [[TMP5:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 144) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 72) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP17:%.*]], label [[TMP18:%.*]], !prof [[PROF1]]
+; CHECK:       6:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       7:
+; CHECK-NEXT:    [[Q:%.*]] = load i64, ptr [[PTR_B:%.*]], align 8
+; CHECK-NEXT:    [[TMP6:%.*]] = ptrtoint ptr [[PTR_B]] to i64
+; CHECK-NEXT:    [[TMP7:%.*]] = xor i64 [[TMP6]], 87960930222080
+; CHECK-NEXT:    [[TMP8:%.*]] = inttoptr i64 [[TMP7]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load i64, ptr [[TMP8]], align 8
+; CHECK-NEXT:    [[_MSPROP:%.*]] = insertelement <8 x i64> [[TMP4]], i64 [[_MSLD]], i32 0
+; CHECK-NEXT:    [[VECINIT_I:%.*]] = insertelement <8 x i64> [[EXTRA_PARAM:%.*]], i64 [[Q]], i32 0
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = shufflevector <8 x i64> [[_MSPROP]], <8 x i64> [[TMP5]], <8 x i32> zeroinitializer
+; CHECK-NEXT:    [[B:%.*]] = shufflevector <8 x i64> [[VECINIT_I]], <8 x i64> [[EXTRA_PARAM2:%.*]], <8 x i32> zeroinitializer
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = or <8 x i64> [[TMP2]], [[_MSPROP1]]
+; CHECK-NEXT:    [[TMP9:%.*]] = sub <8 x i64> [[A:%.*]], [[B]]
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i8 [[TMP3]] to <8 x i1>
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP12:%.*]] = select <8 x i1> [[TMP11]], <8 x i64> [[_MSPROP2]], <8 x i64> zeroinitializer
+; CHECK-NEXT:    [[TMP13:%.*]] = xor <8 x i64> [[TMP9]], zeroinitializer
+; CHECK-NEXT:    [[TMP14:%.*]] = or <8 x i64> [[TMP13]], [[_MSPROP2]]
+; CHECK-NEXT:    [[TMP15:%.*]] = or <8 x i64> [[TMP14]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP10]], <8 x i64> [[TMP15]], <8 x i64> [[TMP12]]
+; CHECK-NEXT:    [[TMP16:%.*]] = select <8 x i1> [[TMP11]], <8 x i64> [[TMP9]], <8 x i64> zeroinitializer
+; CHECK-NEXT:    store <8 x i64> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i64> [[TMP16]]
+;
+  %q = load i64, ptr %ptr_b
+  %vecinit.i = insertelement <8 x i64> %extra_param, i64 %q, i32 0
+  %b = shufflevector <8 x i64> %vecinit.i, <8 x i64> %extra_param2, <8 x i32> zeroinitializer
+  %res = call <8 x i64> @llvm.x86.avx512.mask.psub.q.512(<8 x i64> %a, <8 x i64> %b, <8 x i64> zeroinitializer, i8 %mask)
+  ret < 8 x i64> %res
+}
+
+declare <8 x i64> @llvm.x86.avx512.mask.psub.q.512(<8 x i64>, <8 x i64>, <8 x i64>, i8)
+
+define <16 x i32> @test_mask_mullo_epi32_rr_512(<16 x i32> %a, <16 x i32> %b)  #0 {
+; CHECK-LABEL: @test_mask_mullo_epi32_rr_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <16 x i32> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP3:%.*]] = mul <16 x i32> [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i32> [[TMP3]]
+;
+  %res = call <16 x i32> @llvm.x86.avx512.mask.pmull.d.512(<16 x i32> %a, <16 x i32> %b, <16 x i32> zeroinitializer, i16 -1)
+  ret <16 x i32> %res
+}
+
+define <16 x i32> @test_mask_mullo_epi32_rrk_512(<16 x i32> %a, <16 x i32> %b, <16 x i32> %passThru, i16 %mask)  #0 {
+;
+; CHECK-LABEL: @test_mask_mullo_epi32_rrk_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <16 x i32> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP5:%.*]] = mul <16 x i32> [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i16 [[TMP3]] to <16 x i1>
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast i16 [[MASK:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP8:%.*]] = select <16 x i1> [[TMP7]], <16 x i32> [[_MSPROP]], <16 x i32> [[TMP4]]
+; CHECK-NEXT:    [[TMP9:%.*]] = xor <16 x i32> [[TMP5]], [[PASSTHRU:%.*]]
+; CHECK-NEXT:    [[TMP10:%.*]] = or <16 x i32> [[TMP9]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP11:%.*]] = or <16 x i32> [[TMP10]], [[TMP4]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP6]], <16 x i32> [[TMP11]], <16 x i32> [[TMP8]]
+; CHECK-NEXT:    [[TMP12:%.*]] = select <16 x i1> [[TMP7]], <16 x i32> [[TMP5]], <16 x i32> [[PASSTHRU]]
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i32> [[TMP12]]
+;
+  %res = call <16 x i32> @llvm.x86.avx512.mask.pmull.d.512(<16 x i32> %a, <16 x i32> %b, <16 x i32> %passThru, i16 %mask)
+  ret < 16 x i32> %res
+}
+
+define <16 x i32> @test_mask_mullo_epi32_rrkz_512(<16 x i32> %a, <16 x i32> %b, i16 %mask)  #0 {
+;
+; CHECK-LABEL: @test_mask_mullo_epi32_rrkz_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <16 x i32> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = mul <16 x i32> [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i16 [[TMP3]] to <16 x i1>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i16 [[MASK:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP7:%.*]] = select <16 x i1> [[TMP6]], <16 x i32> [[_MSPROP]], <16 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP8:%.*]] = xor <16 x i32> [[TMP4]], zeroinitializer
+; CHECK-NEXT:    [[TMP9:%.*]] = or <16 x i32> [[TMP8]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP10:%.*]] = or <16 x i32> [[TMP9]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP5]], <16 x i32> [[TMP10]], <16 x i32> [[TMP7]]
+; CHECK-NEXT:    [[TMP11:%.*]] = select <16 x i1> [[TMP6]], <16 x i32> [[TMP4]], <16 x i32> zeroinitializer
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i32> [[TMP11]]
+;
+  %res = call <16 x i32> @llvm.x86.avx512.mask.pmull.d.512(<16 x i32> %a, <16 x i32> %b, <16 x i32> zeroinitializer, i16 %mask)
+  ret < 16 x i32> %res
+}
+
+define <16 x i32> @test_mask_mullo_epi32_rm_512(<16 x i32> %a, ptr %ptr_b)  #0 {
+;
+; CHECK-LABEL: @test_mask_mullo_epi32_rm_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP3:%.*]], label [[TMP4:%.*]], !prof [[PROF1]]
+; CHECK:       3:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       4:
+; CHECK-NEXT:    [[B:%.*]] = load <16 x i32>, ptr [[PTR_B:%.*]], align 64
+; CHECK-NEXT:    [[TMP5:%.*]] = ptrtoint ptr [[PTR_B]] to i64
+; CHECK-NEXT:    [[TMP6:%.*]] = xor i64 [[TMP5]], 87960930222080
+; CHECK-NEXT:    [[TMP7:%.*]] = inttoptr i64 [[TMP6]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <16 x i32>, ptr [[TMP7]], align 64
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <16 x i32> [[TMP2]], [[_MSLD]]
+; CHECK-NEXT:    [[TMP8:%.*]] = mul <16 x i32> [[A:%.*]], [[B]]
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i32> [[TMP8]]
+;
+  %b = load <16 x i32>, ptr %ptr_b
+  %res = call <16 x i32> @llvm.x86.avx512.mask.pmull.d.512(<16 x i32> %a, <16 x i32> %b, <16 x i32> zeroinitializer, i16 -1)
+  ret < 16 x i32> %res
+}
+
+define <16 x i32> @test_mask_mullo_epi32_rmk_512(<16 x i32> %a, ptr %ptr_b, <16 x i32> %passThru, i16 %mask)  #0 {
+;
+; CHECK-LABEL: @test_mask_mullo_epi32_rmk_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 136) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 72) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP5:%.*]], label [[TMP6:%.*]], !prof [[PROF1]]
+; CHECK:       5:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       6:
+; CHECK-NEXT:    [[B:%.*]] = load <16 x i32>, ptr [[PTR_B:%.*]], align 64
+; CHECK-NEXT:    [[TMP7:%.*]] = ptrtoint ptr [[PTR_B]] to i64
+; CHECK-NEXT:    [[TMP8:%.*]] = xor i64 [[TMP7]], 87960930222080
+; CHECK-NEXT:    [[TMP9:%.*]] = inttoptr i64 [[TMP8]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <16 x i32>, ptr [[TMP9]], align 64
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <16 x i32> [[TMP2]], [[_MSLD]]
+; CHECK-NEXT:    [[TMP10:%.*]] = mul <16 x i32> [[A:%.*]], [[B]]
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast i16 [[TMP3]] to <16 x i1>
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast i16 [[MASK:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP13:%.*]] = select <16 x i1> [[TMP12]], <16 x i32> [[_MSPROP]], <16 x i32> [[TMP4]]
+; CHECK-NEXT:    [[TMP14:%.*]] = xor <16 x i32> [[TMP10]], [[PASSTHRU:%.*]]
+; CHECK-NEXT:    [[TMP15:%.*]] = or <16 x i32> [[TMP14]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP16:%.*]] = or <16 x i32> [[TMP15]], [[TMP4]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP11]], <16 x i32> [[TMP16]], <16 x i32> [[TMP13]]
+; CHECK-NEXT:    [[TMP17:%.*]] = select <16 x i1> [[TMP12]], <16 x i32> [[TMP10]], <16 x i32> [[PASSTHRU]]
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i32> [[TMP17]]
+;
+  %b = load <16 x i32>, ptr %ptr_b
+  %res = call <16 x i32> @llvm.x86.avx512.mask.pmull.d.512(<16 x i32> %a, <16 x i32> %b, <16 x i32> %passThru, i16 %mask)
+  ret < 16 x i32> %res
+}
+
+define <16 x i32> @test_mask_mullo_epi32_rmkz_512(<16 x i32> %a, ptr %ptr_b, i16 %mask)  #0 {
+;
+; CHECK-LABEL: @test_mask_mullo_epi32_rmkz_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 72) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP4:%.*]], label [[TMP5:%.*]], !prof [[PROF1]]
+; CHECK:       4:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       5:
+; CHECK-NEXT:    [[B:%.*]] = load <16 x i32>, ptr [[PTR_B:%.*]], align 64
+; CHECK-NEXT:    [[TMP6:%.*]] = ptrtoint ptr [[PTR_B]] to i64
+; CHECK-NEXT:    [[TMP7:%.*]] = xor i64 [[TMP6]], 87960930222080
+; CHECK-NEXT:    [[TMP8:%.*]] = inttoptr i64 [[TMP7]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <16 x i32>, ptr [[TMP8]], align 64
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <16 x i32> [[TMP2]], [[_MSLD]]
+; CHECK-NEXT:    [[TMP9:%.*]] = mul <16 x i32> [[A:%.*]], [[B]]
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i16 [[TMP3]] to <16 x i1>
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast i16 [[MASK:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP12:%.*]] = select <16 x i1> [[TMP11]], <16 x i32> [[_MSPROP]], <16 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP13:%.*]] = xor <16 x i32> [[TMP9]], zeroinitializer
+; CHECK-NEXT:    [[TMP14:%.*]] = or <16 x i32> [[TMP13]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP15:%.*]] = or <16 x i32> [[TMP14]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP10]], <16 x i32> [[TMP15]], <16 x i32> [[TMP12]]
+; CHECK-NEXT:    [[TMP16:%.*]] = select <16 x i1> [[TMP11]], <16 x i32> [[TMP9]], <16 x i32> zeroinitializer
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i32> [[TMP16]]
+;
+  %b = load <16 x i32>, ptr %ptr_b
+  %res = call <16 x i32> @llvm.x86.avx512.mask.pmull.d.512(<16 x i32> %a, <16 x i32> %b, <16 x i32> zeroinitializer, i16 %mask)
+  ret < 16 x i32> %res
+}
+
+define <16 x i32> @test_mask_mullo_epi32_rmb_512(<16 x i32> %a, ptr %ptr_b, <16 x i32> %extra_param, <16 x i32> %extra_param2)  #0 {
+;
+; CHECK-LABEL: @test_mask_mullo_epi32_rmb_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 72) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 136) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP9:%.*]], label [[TMP10:%.*]], !prof [[PROF1]]
+; CHECK:       5:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       6:
+; CHECK-NEXT:    [[Q:%.*]] = load i32, ptr [[PTR_B:%.*]], align 4
+; CHECK-NEXT:    [[TMP5:%.*]] = ptrtoint ptr [[PTR_B]] to i64
+; CHECK-NEXT:    [[TMP6:%.*]] = xor i64 [[TMP5]], 87960930222080
+; CHECK-NEXT:    [[TMP7:%.*]] = inttoptr i64 [[TMP6]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load i32, ptr [[TMP7]], align 4
+; CHECK-NEXT:    [[_MSPROP:%.*]] = insertelement <16 x i32> [[TMP3]], i32 [[_MSLD]], i32 0
+; CHECK-NEXT:    [[VECINIT_I:%.*]] = insertelement <16 x i32> [[EXTRA_PARAM:%.*]], i32 [[Q]], i32 0
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = shufflevector <16 x i32> [[_MSPROP]], <16 x i32> [[TMP4]], <16 x i32> zeroinitializer
+; CHECK-NEXT:    [[B:%.*]] = shufflevector <16 x i32> [[VECINIT_I]], <16 x i32> [[EXTRA_PARAM2:%.*]], <16 x i32> zeroinitializer
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = or <16 x i32> [[TMP2]], [[_MSPROP1]]
+; CHECK-NEXT:    [[TMP8:%.*]] = mul <16 x i32> [[A:%.*]], [[B]]
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP2]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i32> [[TMP8]]
+;
+  %q = load i32, ptr %ptr_b
+  %vecinit.i = insertelement <16 x i32> %extra_param, i32 %q, i32 0
+  %b = shufflevector <16 x i32> %vecinit.i, <16 x i32> %extra_param2, <16 x i32> zeroinitializer
+  %res = call <16 x i32> @llvm.x86.avx512.mask.pmull.d.512(<16 x i32> %a, <16 x i32> %b, <16 x i32> zeroinitializer, i16 -1)
+  ret < 16 x i32> %res
+}
+
+define <16 x i32> @test_mask_mullo_epi32_rmbk_512(<16 x i32> %a, ptr %ptr_b, <16 x i32> %passThru, i16 %mask, <16 x i32> %extra_param, <16 x i32> %extra_param2)  #0 {
+;
+; CHECK-LABEL: @test_mask_mullo_epi32_rmbk_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP5:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 144) to ptr), align 8
+; CHECK-NEXT:    [[TMP6:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 208) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 136) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 72) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP18:%.*]], label [[TMP19:%.*]], !prof [[PROF1]]
+; CHECK:       7:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       8:
+; CHECK-NEXT:    [[Q:%.*]] = load i32, ptr [[PTR_B:%.*]], align 4
+; CHECK-NEXT:    [[TMP7:%.*]] = ptrtoint ptr [[PTR_B]] to i64
+; CHECK-NEXT:    [[TMP8:%.*]] = xor i64 [[TMP7]], 87960930222080
+; CHECK-NEXT:    [[TMP9:%.*]] = inttoptr i64 [[TMP8]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load i32, ptr [[TMP9]], align 4
+; CHECK-NEXT:    [[_MSPROP:%.*]] = insertelement <16 x i32> [[TMP5]], i32 [[_MSLD]], i32 0
+; CHECK-NEXT:    [[VECINIT_I:%.*]] = insertelement <16 x i32> [[EXTRA_PARAM:%.*]], i32 [[Q]], i32 0
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = shufflevector <16 x i32> [[_MSPROP]], <16 x i32> [[TMP6]], <16 x i32> zeroinitializer
+; CHECK-NEXT:    [[B:%.*]] = shufflevector <16 x i32> [[VECINIT_I]], <16 x i32> [[EXTRA_PARAM2:%.*]], <16 x i32> zeroinitializer
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = or <16 x i32> [[TMP2]], [[_MSPROP1]]
+; CHECK-NEXT:    [[TMP10:%.*]] = mul <16 x i32> [[A:%.*]], [[B]]
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast i16 [[TMP3]] to <16 x i1>
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast i16 [[MASK:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP13:%.*]] = select <16 x i1> [[TMP12]], <16 x i32> [[_MSPROP2]], <16 x i32> [[TMP4]]
+; CHECK-NEXT:    [[TMP14:%.*]] = xor <16 x i32> [[TMP10]], [[PASSTHRU:%.*]]
+; CHECK-NEXT:    [[TMP15:%.*]] = or <16 x i32> [[TMP14]], [[_MSPROP2]]
+; CHECK-NEXT:    [[TMP16:%.*]] = or <16 x i32> [[TMP15]], [[TMP4]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP11]], <16 x i32> [[TMP16]], <16 x i32> [[TMP13]]
+; CHECK-NEXT:    [[TMP17:%.*]] = select <16 x i1> [[TMP12]], <16 x i32> [[TMP10]], <16 x i32> [[PASSTHRU]]
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i32> [[TMP17]]
+;
+  %q = load i32, ptr %ptr_b
+  %vecinit.i = insertelement <16 x i32> %extra_param, i32 %q, i32 0
+  %b = shufflevector <16 x i32> %vecinit.i, <16 x i32> %extra_param2, <16 x i32> zeroinitializer
+  %res = call <16 x i32> @llvm.x86.avx512.mask.pmull.d.512(<16 x i32> %a, <16 x i32> %b, <16 x i32> %passThru, i16 %mask)
+  ret < 16 x i32> %res
+}
+
+define <16 x i32> @test_mask_mullo_epi32_rmbkz_512(<16 x i32> %a, ptr %ptr_b, i16 %mask, <16 x i32> %extra_param, <16 x i32> %extra_param2)  #0 {
+;
+; CHECK-LABEL: @test_mask_mullo_epi32_rmbkz_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 80) to ptr), align 8
+; CHECK-NEXT:    [[TMP5:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 144) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 72) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP17:%.*]], label [[TMP18:%.*]], !prof [[PROF1]]
+; CHECK:       6:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       7:
+; CHECK-NEXT:    [[Q:%.*]] = load i32, ptr [[PTR_B:%.*]], align 4
+; CHECK-NEXT:    [[TMP6:%.*]] = ptrtoint ptr [[PTR_B]] to i64
+; CHECK-NEXT:    [[TMP7:%.*]] = xor i64 [[TMP6]], 87960930222080
+; CHECK-NEXT:    [[TMP8:%.*]] = inttoptr i64 [[TMP7]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load i32, ptr [[TMP8]], align 4
+; CHECK-NEXT:    [[_MSPROP:%.*]] = insertelement <16 x i32> [[TMP4]], i32 [[_MSLD]], i32 0
+; CHECK-NEXT:    [[VECINIT_I:%.*]] = insertelement <16 x i32> [[EXTRA_PARAM:%.*]], i32 [[Q]], i32 0
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = shufflevector <16 x i32> [[_MSPROP]], <16 x i32> [[TMP5]], <16 x i32> zeroinitializer
+; CHECK-NEXT:    [[B:%.*]] = shufflevector <16 x i32> [[VECINIT_I]], <16 x i32> [[EXTRA_PARAM2:%.*]], <16 x i32> zeroinitializer
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = or <16 x i32> [[TMP2]], [[_MSPROP1]]
+; CHECK-NEXT:    [[TMP9:%.*]] = mul <16 x i32> [[A:%.*]], [[B]]
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i16 [[TMP3]] to <16 x i1>
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast i16 [[MASK:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP12:%.*]] = select <16 x i1> [[TMP11]], <16 x i32> [[_MSPROP2]], <16 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP13:%.*]] = xor <16 x i32> [[TMP9]], zeroinitializer
+; CHECK-NEXT:    [[TMP14:%.*]] = or <16 x i32> [[TMP13]], [[_MSPROP2]]
+; CHECK-NEXT:    [[TMP15:%.*]] = or <16 x i32> [[TMP14]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP10]], <16 x i32> [[TMP15]], <16 x i32> [[TMP12]]
+; CHECK-NEXT:    [[TMP16:%.*]] = select <16 x i1> [[TMP11]], <16 x i32> [[TMP9]], <16 x i32> zeroinitializer
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i32> [[TMP16]]
+;
+  %q = load i32, ptr %ptr_b
+  %vecinit.i = insertelement <16 x i32> %extra_param, i32 %q, i32 0
+  %b = shufflevector <16 x i32> %vecinit.i, <16 x i32> %extra_param2, <16 x i32> zeroinitializer
+  %res = call <16 x i32> @llvm.x86.avx512.mask.pmull.d.512(<16 x i32> %a, <16 x i32> %b, <16 x i32> zeroinitializer, i16 %mask)
+  ret < 16 x i32> %res
+}
+
+declare <16 x i32> @llvm.x86.avx512.mask.pmull.d.512(<16 x i32>, <16 x i32>, <16 x i32>, i16)
+
+
+declare <16 x float> @llvm.x86.avx512.mask.shuf.f32x4(<16 x float>, <16 x float>, i32, <16 x float>, i16)
+
+define <16 x float>@test_int_x86_avx512_shuf_f32x4(<16 x float> %x0, <16 x float> %x1, <16 x float> %x3)  #0 {
+; CHECK-LABEL: @test_int_x86_avx512_shuf_f32x4(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <16 x i32> [[TMP1]], <16 x i32> [[TMP2]], <16 x i32> <i32 8, i32 9, i32 10, i32 11, i32 4, i32 5, i32 6, i32 7, i32 20, i32 21, i32 22, i32 23, i32 16, i32 17, i32 18, i32 19>
+; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <16 x float> [[X0:%.*]], <16 x float> [[X1:%.*]], <16 x i32> <i32 8, i32 9, i32 10, i32 11, i32 4, i32 5, i32 6, i32 7, i32 20, i32 21, i32 22, i32 23, i32 16, i32 17, i32 18, i32 19>
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x float> [[TMP3]]
+;
+  %res = call <16 x float> @llvm.x86.avx512.mask.shuf.f32x4(<16 x float> %x0, <16 x float> %x1, i32 22, <16 x float> %x3, i16 -1)
+  ret <16 x float> %res
+}
+
+define <16 x float>@test_int_x86_avx512_mask_shuf_f32x4(<16 x float> %x0, <16 x float> %x1, <16 x float> %x3, i16 %x4)  #0 {
+;
+; CHECK-LABEL: @test_int_x86_avx512_mask_shuf_f32x4(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <16 x i32> [[TMP1]], <16 x i32> [[TMP2]], <16 x i32> <i32 8, i32 9, i32 10, i32 11, i32 4, i32 5, i32 6, i32 7, i32 20, i32 21, i32 22, i32 23, i32 16, i32 17, i32 18, i32 19>
+; CHECK-NEXT:    [[TMP5:%.*]] = shufflevector <16 x float> [[X0:%.*]], <16 x float> [[X1:%.*]], <16 x i32> <i32 8, i32 9, i32 10, i32 11, i32 4, i32 5, i32 6, i32 7, i32 20, i32 21, i32 22, i32 23, i32 16, i32 17, i32 18, i32 19>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i16 [[TMP3]] to <16 x i1>
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast i16 [[X4:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP8:%.*]] = select <16 x i1> [[TMP7]], <16 x i32> [[_MSPROP]], <16 x i32> [[TMP4]]
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <16 x float> [[TMP5]] to <16 x i32>
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast <16 x float> [[X3:%.*]] to <16 x i32>
+; CHECK-NEXT:    [[TMP11:%.*]] = xor <16 x i32> [[TMP9]], [[TMP10]]
+; CHECK-NEXT:    [[TMP12:%.*]] = or <16 x i32> [[TMP11]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP13:%.*]] = or <16 x i32> [[TMP12]], [[TMP4]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP6]], <16 x i32> [[TMP13]], <16 x i32> [[TMP8]]
+; CHECK-NEXT:    [[TMP14:%.*]] = select <16 x i1> [[TMP7]], <16 x float> [[TMP5]], <16 x float> [[X3]]
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x float> [[TMP14]]
+;
+  %res = call <16 x float> @llvm.x86.avx512.mask.shuf.f32x4(<16 x float> %x0, <16 x float> %x1, i32 22, <16 x float> %x3, i16 %x4)
+  ret <16 x float> %res
+}
+
+declare <8 x double> @llvm.x86.avx512.mask.shuf.f64x2(<8 x double>, <8 x double>, i32, <8 x double>, i8)
+
+define <8 x double>@test_int_x86_avx512_shuf_f64x2(<8 x double> %x0, <8 x double> %x1, <8 x double> %x3)  #0 {
+; CHECK-LABEL: @test_int_x86_avx512_shuf_f64x2(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <8 x i64> [[TMP1]], <8 x i64> [[TMP2]], <8 x i32> <i32 4, i32 5, i32 2, i32 3, i32 10, i32 11, i32 8, i32 9>
+; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <8 x double> [[X0:%.*]], <8 x double> [[X1:%.*]], <8 x i32> <i32 4, i32 5, i32 2, i32 3, i32 10, i32 11, i32 8, i32 9>
+; CHECK-NEXT:    store <8 x i64> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x double> [[TMP3]]
+;
+  %res = call <8 x double> @llvm.x86.avx512.mask.shuf.f64x2(<8 x double> %x0, <8 x double> %x1, i32 22, <8 x double> %x3, i8 -1)
+  ret <8 x double> %res
+}
+
+define <8 x double>@test_int_x86_avx512_mask_shuf_f64x2(<8 x double> %x0, <8 x double> %x1, <8 x double> %x3, i8 %x4)  #0 {
+;
+; CHECK-LABEL: @test_int_x86_avx512_mask_shuf_f64x2(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <8 x i64> [[TMP1]], <8 x i64> [[TMP2]], <8 x i32> <i32 4, i32 5, i32 2, i32 3, i32 10, i32 11, i32 8, i32 9>
+; CHECK-NEXT:    [[TMP5:%.*]] = shufflevector <8 x double> [[X0:%.*]], <8 x double> [[X1:%.*]], <8 x i32> <i32 4, i32 5, i32 2, i32 3, i32 10, i32 11, i32 8, i32 9>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i8 [[TMP3]] to <8 x i1>
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast i8 [[X4:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP8:%.*]] = select <8 x i1> [[TMP7]], <8 x i64> [[_MSPROP]], <8 x i64> [[TMP4]]
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <8 x double> [[TMP5]] to <8 x i64>
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast <8 x double> [[X3:%.*]] to <8 x i64>
+; CHECK-NEXT:    [[TMP11:%.*]] = xor <8 x i64> [[TMP9]], [[TMP10]]
+; CHECK-NEXT:    [[TMP12:%.*]] = or <8 x i64> [[TMP11]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP13:%.*]] = or <8 x i64> [[TMP12]], [[TMP4]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP6]], <8 x i64> [[TMP13]], <8 x i64> [[TMP8]]
+; CHECK-NEXT:    [[TMP14:%.*]] = select <8 x i1> [[TMP7]], <8 x double> [[TMP5]], <8 x double> [[X3]]
+; CHECK-NEXT:    store <8 x i64> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x double> [[TMP14]]
+;
+  %res = call <8 x double> @llvm.x86.avx512.mask.shuf.f64x2(<8 x double> %x0, <8 x double> %x1, i32 22, <8 x double> %x3, i8 %x4)
+  ret <8 x double> %res
+}
+
+define <8 x double>@test_int_x86_avx512_maskz_shuf_f64x2(<8 x double> %x0, <8 x double> %x1, i8 %x4)  #0 {
+;
+; CHECK-LABEL: @test_int_x86_avx512_maskz_shuf_f64x2(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <8 x i64> [[TMP1]], <8 x i64> [[TMP2]], <8 x i32> <i32 4, i32 5, i32 2, i32 3, i32 10, i32 11, i32 8, i32 9>
+; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <8 x double> [[X0:%.*]], <8 x double> [[X1:%.*]], <8 x i32> <i32 4, i32 5, i32 2, i32 3, i32 10, i32 11, i32 8, i32 9>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i8 [[TMP3]] to <8 x i1>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i8 [[X4:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP7:%.*]] = select <8 x i1> [[TMP6]], <8 x i64> [[_MSPROP]], <8 x i64> zeroinitializer
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <8 x double> [[TMP4]] to <8 x i64>
+; CHECK-NEXT:    [[TMP9:%.*]] = xor <8 x i64> [[TMP8]], zeroinitializer
+; CHECK-NEXT:    [[TMP10:%.*]] = or <8 x i64> [[TMP9]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP11:%.*]] = or <8 x i64> [[TMP10]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP5]], <8 x i64> [[TMP11]], <8 x i64> [[TMP7]]
+; CHECK-NEXT:    [[TMP12:%.*]] = select <8 x i1> [[TMP6]], <8 x double> [[TMP4]], <8 x double> zeroinitializer
+; CHECK-NEXT:    store <8 x i64> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x double> [[TMP12]]
+;
+  %res = call <8 x double> @llvm.x86.avx512.mask.shuf.f64x2(<8 x double> %x0, <8 x double> %x1, i32 22, <8 x double> zeroinitializer, i8 %x4)
+  ret <8 x double> %res
+}
+
+declare <16 x i32> @llvm.x86.avx512.mask.shuf.i32x4(<16 x i32>, <16 x i32>, i32, <16 x i32>, i16)
+
+define <16 x i32>@test_int_x86_avx512_shuf_i32x4(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x3)  #0 {
+; CHECK-LABEL: @test_int_x86_avx512_shuf_i32x4(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <16 x i32> [[TMP1]], <16 x i32> [[TMP2]], <16 x i32> <i32 8, i32 9, i32 10, i32 11, i32 4, i32 5, i32 6, i32 7, i32 20, i32 21, i32 22, i32 23, i32 16, i32 17, i32 18, i32 19>
+; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <16 x i32> [[X0:%.*]], <16 x i32> [[X1:%.*]], <16 x i32> <i32 8, i32 9, i32 10, i32 11, i32 4, i32 5, i32 6, i32 7, i32 20, i32 21, i32 22, i32 23, i32 16, i32 17, i32 18, i32 19>
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i32> [[TMP3]]
+;
+  %res = call <16 x i32> @llvm.x86.avx512.mask.shuf.i32x4(<16 x i32> %x0, <16 x i32> %x1, i32 22, <16 x i32> %x3, i16 -1)
+  ret <16 x i32> %res
+}
+
+define <16 x i32>@test_int_x86_avx512_mask_shuf_i32x4(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x3, i16 %x4)  #0 {
+;
+; CHECK-LABEL: @test_int_x86_avx512_mask_shuf_i32x4(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <16 x i32> [[TMP1]], <16 x i32> [[TMP2]], <16 x i32> <i32 8, i32 9, i32 10, i32 11, i32 4, i32 5, i32 6, i32 7, i32 20, i32 21, i32 22, i32 23, i32 16, i32 17, i32 18, i32 19>
+; CHECK-NEXT:    [[TMP5:%.*]] = shufflevector <16 x i32> [[X0:%.*]], <16 x i32> [[X1:%.*]], <16 x i32> <i32 8, i32 9, i32 10, i32 11, i32 4, i32 5, i32 6, i32 7, i32 20, i32 21, i32 22, i32 23, i32 16, i32 17, i32 18, i32 19>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i16 [[TMP3]] to <16 x i1>
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast i16 [[X4:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP8:%.*]] = select <16 x i1> [[TMP7]], <16 x i32> [[_MSPROP]], <16 x i32> [[TMP4]]
+; CHECK-NEXT:    [[TMP9:%.*]] = xor <16 x i32> [[TMP5]], [[X3:%.*]]
+; CHECK-NEXT:    [[TMP10:%.*]] = or <16 x i32> [[TMP9]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP11:%.*]] = or <16 x i32> [[TMP10]], [[TMP4]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP6]], <16 x i32> [[TMP11]], <16 x i32> [[TMP8]]
+; CHECK-NEXT:    [[TMP12:%.*]] = select <16 x i1> [[TMP7]], <16 x i32> [[TMP5]], <16 x i32> [[X3]]
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i32> [[TMP12]]
+;
+  %res = call <16 x i32> @llvm.x86.avx512.mask.shuf.i32x4(<16 x i32> %x0, <16 x i32> %x1, i32 22, <16 x i32> %x3, i16 %x4)
+  ret <16 x i32> %res
+}
+
+declare <8 x i64> @llvm.x86.avx512.mask.shuf.i64x2(<8 x i64>, <8 x i64>, i32, <8 x i64>, i8)
+
+define <8 x i64>@test_int_x86_avx512_shuf_i64x2(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> %x3)  #0 {
+; CHECK-LABEL: @test_int_x86_avx512_shuf_i64x2(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <8 x i64> [[TMP1]], <8 x i64> [[TMP2]], <8 x i32> <i32 4, i32 5, i32 2, i32 3, i32 10, i32 11, i32 8, i32 9>
+; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <8 x i64> [[X0:%.*]], <8 x i64> [[X1:%.*]], <8 x i32> <i32 4, i32 5, i32 2, i32 3, i32 10, i32 11, i32 8, i32 9>
+; CHECK-NEXT:    store <8 x i64> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i64> [[TMP3]]
+;
+  %res = call <8 x i64> @llvm.x86.avx512.mask.shuf.i64x2(<8 x i64> %x0, <8 x i64> %x1, i32 22, <8 x i64> %x3, i8 -1)
+  ret <8 x i64> %res
+}
+
+define <8 x i64>@test_int_x86_avx512_mask_shuf_i64x2(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> %x3, i8 %x4)  #0 {
+;
+; CHECK-LABEL: @test_int_x86_avx512_mask_shuf_i64x2(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <8 x i64> [[TMP1]], <8 x i64> [[TMP2]], <8 x i32> <i32 4, i32 5, i32 2, i32 3, i32 10, i32 11, i32 8, i32 9>
+; CHECK-NEXT:    [[TMP5:%.*]] = shufflevector <8 x i64> [[X0:%.*]], <8 x i64> [[X1:%.*]], <8 x i32> <i32 4, i32 5, i32 2, i32 3, i32 10, i32 11, i32 8, i32 9>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i8 [[TMP3]] to <8 x i1>
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast i8 [[X4:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP8:%.*]] = select <8 x i1> [[TMP7]], <8 x i64> [[_MSPROP]], <8 x i64> [[TMP4]]
+; CHECK-NEXT:    [[TMP9:%.*]] = xor <8 x i64> [[TMP5]], [[X3:%.*]]
+; CHECK-NEXT:    [[TMP10:%.*]] = or <8 x i64> [[TMP9]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP11:%.*]] = or <8 x i64> [[TMP10]], [[TMP4]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP6]], <8 x i64> [[TMP11]], <8 x i64> [[TMP8]]
+; CHECK-NEXT:    [[TMP12:%.*]] = select <8 x i1> [[TMP7]], <8 x i64> [[TMP5]], <8 x i64> [[X3]]
+; CHECK-NEXT:    store <8 x i64> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i64> [[TMP12]]
+;
+  %res = call <8 x i64> @llvm.x86.avx512.mask.shuf.i64x2(<8 x i64> %x0, <8 x i64> %x1, i32 22, <8 x i64> %x3, i8 %x4)
+  ret <8 x i64> %res
+}
+
+declare <8 x double> @llvm.x86.avx512.mask.shuf.pd.512(<8 x double>, <8 x double>, i32, <8 x double>, i8)
+
+define <8 x double>@test_int_x86_avx512_shuf_pd_512(<8 x double> %x0, <8 x double> %x1, <8 x double> %x3)  #0 {
+; CHECK-LABEL: @test_int_x86_avx512_shuf_pd_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <8 x i64> [[TMP1]], <8 x i64> [[TMP2]], <8 x i32> <i32 0, i32 9, i32 3, i32 10, i32 5, i32 12, i32 6, i32 14>
+; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <8 x double> [[X0:%.*]], <8 x double> [[X1:%.*]], <8 x i32> <i32 0, i32 9, i32 3, i32 10, i32 5, i32 12, i32 6, i32 14>
+; CHECK-NEXT:    store <8 x i64> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x double> [[TMP3]]
+;
+  %res = call <8 x double> @llvm.x86.avx512.mask.shuf.pd.512(<8 x double> %x0, <8 x double> %x1, i32 22, <8 x double> %x3, i8 -1)
+  ret <8 x double> %res
+}
+
+define <8 x double>@test_int_x86_avx512_mask_shuf_pd_512(<8 x double> %x0, <8 x double> %x1, <8 x double> %x3, i8 %x4)  #0 {
+;
+; CHECK-LABEL: @test_int_x86_avx512_mask_shuf_pd_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <8 x i64> [[TMP1]], <8 x i64> [[TMP2]], <8 x i32> <i32 0, i32 9, i32 3, i32 10, i32 5, i32 12, i32 6, i32 14>
+; CHECK-NEXT:    [[TMP5:%.*]] = shufflevector <8 x double> [[X0:%.*]], <8 x double> [[X1:%.*]], <8 x i32> <i32 0, i32 9, i32 3, i32 10, i32 5, i32 12, i32 6, i32 14>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i8 [[TMP3]] to <8 x i1>
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast i8 [[X4:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP8:%.*]] = select <8 x i1> [[TMP7]], <8 x i64> [[_MSPROP]], <8 x i64> [[TMP4]]
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <8 x double> [[TMP5]] to <8 x i64>
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast <8 x double> [[X3:%.*]] to <8 x i64>
+; CHECK-NEXT:    [[TMP11:%.*]] = xor <8 x i64> [[TMP9]], [[TMP10]]
+; CHECK-NEXT:    [[TMP12:%.*]] = or <8 x i64> [[TMP11]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP13:%.*]] = or <8 x i64> [[TMP12]], [[TMP4]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP6]], <8 x i64> [[TMP13]], <8 x i64> [[TMP8]]
+; CHECK-NEXT:    [[TMP14:%.*]] = select <8 x i1> [[TMP7]], <8 x double> [[TMP5]], <8 x double> [[X3]]
+; CHECK-NEXT:    store <8 x i64> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x double> [[TMP14]]
+;
+  %res = call <8 x double> @llvm.x86.avx512.mask.shuf.pd.512(<8 x double> %x0, <8 x double> %x1, i32 22, <8 x double> %x3, i8 %x4)
+  ret <8 x double> %res
+}
+
+define <8 x double>@test_int_x86_avx512_maskz_shuf_pd_512(<8 x double> %x0, <8 x double> %x1, i8 %x4)  #0 {
+;
+; CHECK-LABEL: @test_int_x86_avx512_maskz_shuf_pd_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <8 x i64> [[TMP1]], <8 x i64> [[TMP2]], <8 x i32> <i32 0, i32 9, i32 3, i32 10, i32 5, i32 12, i32 6, i32 14>
+; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <8 x double> [[X0:%.*]], <8 x double> [[X1:%.*]], <8 x i32> <i32 0, i32 9, i32 3, i32 10, i32 5, i32 12, i32 6, i32 14>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i8 [[TMP3]] to <8 x i1>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i8 [[X4:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP7:%.*]] = select <8 x i1> [[TMP6]], <8 x i64> [[_MSPROP]], <8 x i64> zeroinitializer
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <8 x double> [[TMP4]] to <8 x i64>
+; CHECK-NEXT:    [[TMP9:%.*]] = xor <8 x i64> [[TMP8]], zeroinitializer
+; CHECK-NEXT:    [[TMP10:%.*]] = or <8 x i64> [[TMP9]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP11:%.*]] = or <8 x i64> [[TMP10]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP5]], <8 x i64> [[TMP11]], <8 x i64> [[TMP7]]
+; CHECK-NEXT:    [[TMP12:%.*]] = select <8 x i1> [[TMP6]], <8 x double> [[TMP4]], <8 x double> zeroinitializer
+; CHECK-NEXT:    store <8 x i64> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x double> [[TMP12]]
+;
+  %res = call <8 x double> @llvm.x86.avx512.mask.shuf.pd.512(<8 x double> %x0, <8 x double> %x1, i32 22, <8 x double> zeroinitializer, i8 %x4)
+  ret <8 x double> %res
+}
+
+declare <16 x float> @llvm.x86.avx512.mask.shuf.ps.512(<16 x float>, <16 x float>, i32, <16 x float>, i16)
+
+define <16 x float>@test_int_x86_avx512_shuf_ps_512(<16 x float> %x0, <16 x float> %x1, <16 x float> %x3)  #0 {
+; CHECK-LABEL: @test_int_x86_avx512_shuf_ps_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <16 x i32> [[TMP1]], <16 x i32> [[TMP2]], <16 x i32> <i32 2, i32 1, i32 17, i32 16, i32 6, i32 5, i32 21, i32 20, i32 10, i32 9, i32 25, i32 24, i32 14, i32 13, i32 29, i32 28>
+; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <16 x float> [[X0:%.*]], <16 x float> [[X1:%.*]], <16 x i32> <i32 2, i32 1, i32 17, i32 16, i32 6, i32 5, i32 21, i32 20, i32 10, i32 9, i32 25, i32 24, i32 14, i32 13, i32 29, i32 28>
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x float> [[TMP3]]
+;
+  %res = call <16 x float> @llvm.x86.avx512.mask.shuf.ps.512(<16 x float> %x0, <16 x float> %x1, i32 22, <16 x float> %x3, i16 -1)
+  ret <16 x float> %res
+}
+
+define <16 x float>@test_int_x86_avx512_mask_shuf_ps_512(<16 x float> %x0, <16 x float> %x1, <16 x float> %x3, i16 %x4)  #0 {
+;
+; CHECK-LABEL: @test_int_x86_avx512_mask_shuf_ps_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <16 x i32> [[TMP1]], <16 x i32> [[TMP2]], <16 x i32> <i32 2, i32 1, i32 17, i32 16, i32 6, i32 5, i32 21, i32 20, i32 10, i32 9, i32 25, i32 24, i32 14, i32 13, i32 29, i32 28>
+; CHECK-NEXT:    [[TMP5:%.*]] = shufflevector <16 x float> [[X0:%.*]], <16 x float> [[X1:%.*]], <16 x i32> <i32 2, i32 1, i32 17, i32 16, i32 6, i32 5, i32 21, i32 20, i32 10, i32 9, i32 25, i32 24, i32 14, i32 13, i32 29, i32 28>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i16 [[TMP3]] to <16 x i1>
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast i16 [[X4:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP8:%.*]] = select <16 x i1> [[TMP7]], <16 x i32> [[_MSPROP]], <16 x i32> [[TMP4]]
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <16 x float> [[TMP5]] to <16 x i32>
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast <16 x float> [[X3:%.*]] to <16 x i32>
+; CHECK-NEXT:    [[TMP11:%.*]] = xor <16 x i32> [[TMP9]], [[TMP10]]
+; CHECK-NEXT:    [[TMP12:%.*]] = or <16 x i32> [[TMP11]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP13:%.*]] = or <16 x i32> [[TMP12]], [[TMP4]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP6]], <16 x i32> [[TMP13]], <16 x i32> [[TMP8]]
+; CHECK-NEXT:    [[TMP14:%.*]] = select <16 x i1> [[TMP7]], <16 x float> [[TMP5]], <16 x float> [[X3]]
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x float> [[TMP14]]
+;
+  %res = call <16 x float> @llvm.x86.avx512.mask.shuf.ps.512(<16 x float> %x0, <16 x float> %x1, i32 22, <16 x float> %x3, i16 %x4)
+  ret <16 x float> %res
+}
+
+declare <16 x i32> @llvm.x86.avx512.mask.pmaxs.d.512(<16 x i32>, <16 x i32>, <16 x i32>, i16)
+
+define <16 x i32>@test_int_x86_avx512_pmaxs_d_512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x2)  #0 {
+; CHECK-LABEL: @test_int_x86_avx512_pmaxs_d_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <16 x i32> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP3:%.*]] = call <16 x i32> @llvm.smax.v16i32(<16 x i32> [[X0:%.*]], <16 x i32> [[X1:%.*]])
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i32> [[TMP3]]
+;
+  %res = call <16 x i32> @llvm.x86.avx512.mask.pmaxs.d.512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x2, i16 -1)
+  ret <16 x i32> %res
+}
+
+define <16 x i32>@test_int_x86_avx512_mask_pmaxs_d_512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x2, i16 %x3)  #0 {
+;
+; CHECK-LABEL: @test_int_x86_avx512_mask_pmaxs_d_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <16 x i32> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP5:%.*]] = call <16 x i32> @llvm.smax.v16i32(<16 x i32> [[X0:%.*]], <16 x i32> [[X1:%.*]])
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i16 [[TMP3]] to <16 x i1>
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast i16 [[X3:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP8:%.*]] = select <16 x i1> [[TMP7]], <16 x i32> [[_MSPROP]], <16 x i32> [[TMP4]]
+; CHECK-NEXT:    [[TMP9:%.*]] = xor <16 x i32> [[TMP5]], [[X2:%.*]]
+; CHECK-NEXT:    [[TMP10:%.*]] = or <16 x i32> [[TMP9]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP11:%.*]] = or <16 x i32> [[TMP10]], [[TMP4]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP6]], <16 x i32> [[TMP11]], <16 x i32> [[TMP8]]
+; CHECK-NEXT:    [[TMP12:%.*]] = select <16 x i1> [[TMP7]], <16 x i32> [[TMP5]], <16 x i32> [[X2]]
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i32> [[TMP12]]
+;
+  %res = call <16 x i32> @llvm.x86.avx512.mask.pmaxs.d.512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x2, i16 %x3)
+  ret <16 x i32> %res
+}
+
+declare <8 x i64> @llvm.x86.avx512.mask.pmaxs.q.512(<8 x i64>, <8 x i64>, <8 x i64>, i8)
+
+define <8 x i64>@test_int_x86_avx512_pmaxs_q_512(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> %x2)  #0 {
+; CHECK-LABEL: @test_int_x86_avx512_pmaxs_q_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <8 x i64> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP3:%.*]] = call <8 x i64> @llvm.smax.v8i64(<8 x i64> [[X0:%.*]], <8 x i64> [[X1:%.*]])
+; CHECK-NEXT:    store <8 x i64> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i64> [[TMP3]]
+;
+  %res = call <8 x i64> @llvm.x86.avx512.mask.pmaxs.q.512(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> %x2, i8 -1)
+  ret <8 x i64> %res
+}
+
+define <8 x i64>@test_int_x86_avx512_mask_pmaxs_q_512(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> %x2, i8 %x3)  #0 {
+;
+; CHECK-LABEL: @test_int_x86_avx512_mask_pmaxs_q_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <8 x i64> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP5:%.*]] = call <8 x i64> @llvm.smax.v8i64(<8 x i64> [[X0:%.*]], <8 x i64> [[X1:%.*]])
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i8 [[TMP3]] to <8 x i1>
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast i8 [[X3:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP8:%.*]] = select <8 x i1> [[TMP7]], <8 x i64> [[_MSPROP]], <8 x i64> [[TMP4]]
+; CHECK-NEXT:    [[TMP9:%.*]] = xor <8 x i64> [[TMP5]], [[X2:%.*]]
+; CHECK-NEXT:    [[TMP10:%.*]] = or <8 x i64> [[TMP9]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP11:%.*]] = or <8 x i64> [[TMP10]], [[TMP4]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP6]], <8 x i64> [[TMP11]], <8 x i64> [[TMP8]]
+; CHECK-NEXT:    [[TMP12:%.*]] = select <8 x i1> [[TMP7]], <8 x i64> [[TMP5]], <8 x i64> [[X2]]
+; CHECK-NEXT:    store <8 x i64> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i64> [[TMP12]]
+;
+  %res = call <8 x i64> @llvm.x86.avx512.mask.pmaxs.q.512(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> %x2, i8 %x3)
+  ret <8 x i64> %res
+}
+
+declare <16 x i32> @llvm.x86.avx512.mask.pmaxu.d.512(<16 x i32>, <16 x i32>, <16 x i32>, i16)
+
+define <16 x i32>@test_int_x86_avx512_pmaxu_d_512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x2)  #0 {
+; CHECK-LABEL: @test_int_x86_avx512_pmaxu_d_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <16 x i32> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP3:%.*]] = call <16 x i32> @llvm.umax.v16i32(<16 x i32> [[X0:%.*]], <16 x i32> [[X1:%.*]])
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i32> [[TMP3]]
+;
+  %res = call <16 x i32> @llvm.x86.avx512.mask.pmaxu.d.512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x2, i16 -1)
+  ret <16 x i32> %res
+}
+
+define <16 x i32>@test_int_x86_avx512_mask_pmaxu_d_512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x2, i16 %x3)  #0 {
+;
+; CHECK-LABEL: @test_int_x86_avx512_mask_pmaxu_d_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <16 x i32> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP5:%.*]] = call <16 x i32> @llvm.umax.v16i32(<16 x i32> [[X0:%.*]], <16 x i32> [[X1:%.*]])
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i16 [[TMP3]] to <16 x i1>
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast i16 [[X3:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP8:%.*]] = select <16 x i1> [[TMP7]], <16 x i32> [[_MSPROP]], <16 x i32> [[TMP4]]
+; CHECK-NEXT:    [[TMP9:%.*]] = xor <16 x i32> [[TMP5]], [[X2:%.*]]
+; CHECK-NEXT:    [[TMP10:%.*]] = or <16 x i32> [[TMP9]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP11:%.*]] = or <16 x i32> [[TMP10]], [[TMP4]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP6]], <16 x i32> [[TMP11]], <16 x i32> [[TMP8]]
+; CHECK-NEXT:    [[TMP12:%.*]] = select <16 x i1> [[TMP7]], <16 x i32> [[TMP5]], <16 x i32> [[X2]]
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i32> [[TMP12]]
+;
+  %res = call <16 x i32> @llvm.x86.avx512.mask.pmaxu.d.512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x2, i16 %x3)
+  ret <16 x i32> %res
+}
+
+declare <8 x i64> @llvm.x86.avx512.mask.pmaxu.q.512(<8 x i64>, <8 x i64>, <8 x i64>, i8)
+
+define <8 x i64>@test_int_x86_avx512_pmaxu_q_512(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> %x2)  #0 {
+; CHECK-LABEL: @test_int_x86_avx512_pmaxu_q_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <8 x i64> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP3:%.*]] = call <8 x i64> @llvm.umax.v8i64(<8 x i64> [[X0:%.*]], <8 x i64> [[X1:%.*]])
+; CHECK-NEXT:    store <8 x i64> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i64> [[TMP3]]
+;
+  %res = call <8 x i64> @llvm.x86.avx512.mask.pmaxu.q.512(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> %x2, i8 -1)
+  ret <8 x i64> %res
+}
+
+define <8 x i64>@test_int_x86_avx512_mask_pmaxu_q_512(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> %x2, i8 %x3)  #0 {
+;
+; CHECK-LABEL: @test_int_x86_avx512_mask_pmaxu_q_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <8 x i64> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP5:%.*]] = call <8 x i64> @llvm.umax.v8i64(<8 x i64> [[X0:%.*]], <8 x i64> [[X1:%.*]])
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i8 [[TMP3]] to <8 x i1>
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast i8 [[X3:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP8:%.*]] = select <8 x i1> [[TMP7]], <8 x i64> [[_MSPROP]], <8 x i64> [[TMP4]]
+; CHECK-NEXT:    [[TMP9:%.*]] = xor <8 x i64> [[TMP5]], [[X2:%.*]]
+; CHECK-NEXT:    [[TMP10:%.*]] = or <8 x i64> [[TMP9]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP11:%.*]] = or <8 x i64> [[TMP10]], [[TMP4]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP6]], <8 x i64> [[TMP11]], <8 x i64> [[TMP8]]
+; CHECK-NEXT:    [[TMP12:%.*]] = select <8 x i1> [[TMP7]], <8 x i64> [[TMP5]], <8 x i64> [[X2]]
+; CHECK-NEXT:    store <8 x i64> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i64> [[TMP12]]
+;
+  %res = call <8 x i64> @llvm.x86.avx512.mask.pmaxu.q.512(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> %x2, i8 %x3)
+  ret <8 x i64> %res
+}
+
+declare <16 x i32> @llvm.x86.avx512.mask.pmins.d.512(<16 x i32>, <16 x i32>, <16 x i32>, i16)
+
+define <16 x i32>@test_int_x86_avx512_pmins_d_512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x2)  #0 {
+; CHECK-LABEL: @test_int_x86_avx512_pmins_d_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <16 x i32> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP3:%.*]] = call <16 x i32> @llvm.smin.v16i32(<16 x i32> [[X0:%.*]], <16 x i32> [[X1:%.*]])
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i32> [[TMP3]]
+;
+  %res = call <16 x i32> @llvm.x86.avx512.mask.pmins.d.512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x2, i16 -1)
+  ret <16 x i32> %res
+}
+
+define <16 x i32>@test_int_x86_avx512_mask_pmins_d_512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x2, i16 %x3)  #0 {
+;
+; CHECK-LABEL: @test_int_x86_avx512_mask_pmins_d_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <16 x i32> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP5:%.*]] = call <16 x i32> @llvm.smin.v16i32(<16 x i32> [[X0:%.*]], <16 x i32> [[X1:%.*]])
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i16 [[TMP3]] to <16 x i1>
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast i16 [[X3:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP8:%.*]] = select <16 x i1> [[TMP7]], <16 x i32> [[_MSPROP]], <16 x i32> [[TMP4]]
+; CHECK-NEXT:    [[TMP9:%.*]] = xor <16 x i32> [[TMP5]], [[X2:%.*]]
+; CHECK-NEXT:    [[TMP10:%.*]] = or <16 x i32> [[TMP9]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP11:%.*]] = or <16 x i32> [[TMP10]], [[TMP4]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP6]], <16 x i32> [[TMP11]], <16 x i32> [[TMP8]]
+; CHECK-NEXT:    [[TMP12:%.*]] = select <16 x i1> [[TMP7]], <16 x i32> [[TMP5]], <16 x i32> [[X2]]
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i32> [[TMP12]]
+;
+  %res = call <16 x i32> @llvm.x86.avx512.mask.pmins.d.512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x2, i16 %x3)
+  ret <16 x i32> %res
+}
+
+declare <8 x i64> @llvm.x86.avx512.mask.pmins.q.512(<8 x i64>, <8 x i64>, <8 x i64>, i8)
+
+define <8 x i64>@test_int_x86_avx512_pmins_q_512(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> %x2)  #0 {
+; CHECK-LABEL: @test_int_x86_avx512_pmins_q_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <8 x i64> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP3:%.*]] = call <8 x i64> @llvm.smin.v8i64(<8 x i64> [[X0:%.*]], <8 x i64> [[X1:%.*]])
+; CHECK-NEXT:    store <8 x i64> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i64> [[TMP3]]
+;
+  %res = call <8 x i64> @llvm.x86.avx512.mask.pmins.q.512(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> %x2, i8 -1)
+  ret <8 x i64> %res
+}
+
+define <8 x i64>@test_int_x86_avx512_mask_pmins_q_512(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> %x2, i8 %x3)  #0 {
+;
+; CHECK-LABEL: @test_int_x86_avx512_mask_pmins_q_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <8 x i64> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP5:%.*]] = call <8 x i64> @llvm.smin.v8i64(<8 x i64> [[X0:%.*]], <8 x i64> [[X1:%.*]])
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i8 [[TMP3]] to <8 x i1>
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast i8 [[X3:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP8:%.*]] = select <8 x i1> [[TMP7]], <8 x i64> [[_MSPROP]], <8 x i64> [[TMP4]]
+; CHECK-NEXT:    [[TMP9:%.*]] = xor <8 x i64> [[TMP5]], [[X2:%.*]]
+; CHECK-NEXT:    [[TMP10:%.*]] = or <8 x i64> [[TMP9]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP11:%.*]] = or <8 x i64> [[TMP10]], [[TMP4]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP6]], <8 x i64> [[TMP11]], <8 x i64> [[TMP8]]
+; CHECK-NEXT:    [[TMP12:%.*]] = select <8 x i1> [[TMP7]], <8 x i64> [[TMP5]], <8 x i64> [[X2]]
+; CHECK-NEXT:    store <8 x i64> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i64> [[TMP12]]
+;
+  %res = call <8 x i64> @llvm.x86.avx512.mask.pmins.q.512(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> %x2, i8 %x3)
+  ret <8 x i64> %res
+}
+
+declare <16 x i32> @llvm.x86.avx512.mask.pminu.d.512(<16 x i32>, <16 x i32>, <16 x i32>, i16)
+
+define <16 x i32>@test_int_x86_avx512_pminu_d_512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x2)  #0 {
+; CHECK-LABEL: @test_int_x86_avx512_pminu_d_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <16 x i32> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP3:%.*]] = call <16 x i32> @llvm.umin.v16i32(<16 x i32> [[X0:%.*]], <16 x i32> [[X1:%.*]])
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i32> [[TMP3]]
+;
+  %res = call <16 x i32> @llvm.x86.avx512.mask.pminu.d.512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x2, i16 -1)
+  ret <16 x i32> %res
+}
+
+define <16 x i32>@test_int_x86_avx512_mask_pminu_d_512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x2, i16 %x3)  #0 {
+;
+; CHECK-LABEL: @test_int_x86_avx512_mask_pminu_d_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <16 x i32> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP5:%.*]] = call <16 x i32> @llvm.umin.v16i32(<16 x i32> [[X0:%.*]], <16 x i32> [[X1:%.*]])
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i16 [[TMP3]] to <16 x i1>
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast i16 [[X3:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP8:%.*]] = select <16 x i1> [[TMP7]], <16 x i32> [[_MSPROP]], <16 x i32> [[TMP4]]
+; CHECK-NEXT:    [[TMP9:%.*]] = xor <16 x i32> [[TMP5]], [[X2:%.*]]
+; CHECK-NEXT:    [[TMP10:%.*]] = or <16 x i32> [[TMP9]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP11:%.*]] = or <16 x i32> [[TMP10]], [[TMP4]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP6]], <16 x i32> [[TMP11]], <16 x i32> [[TMP8]]
+; CHECK-NEXT:    [[TMP12:%.*]] = select <16 x i1> [[TMP7]], <16 x i32> [[TMP5]], <16 x i32> [[X2]]
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i32> [[TMP12]]
+;
+  %res = call <16 x i32> @llvm.x86.avx512.mask.pminu.d.512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x2, i16 %x3)
+  ret <16 x i32> %res
+}
+
+declare <8 x i64> @llvm.x86.avx512.mask.pminu.q.512(<8 x i64>, <8 x i64>, <8 x i64>, i8)
+
+define <8 x i64>@test_int_x86_avx512_pminu_q_512(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> %x2)  #0 {
+; CHECK-LABEL: @test_int_x86_avx512_pminu_q_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <8 x i64> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP3:%.*]] = call <8 x i64> @llvm.umin.v8i64(<8 x i64> [[X0:%.*]], <8 x i64> [[X1:%.*]])
+; CHECK-NEXT:    store <8 x i64> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i64> [[TMP3]]
+;
+  %res = call <8 x i64> @llvm.x86.avx512.mask.pminu.q.512(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> %x2, i8 -1)
+  ret <8 x i64> %res
+}
+
+define <8 x i64>@test_int_x86_avx512_mask_pminu_q_512(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> %x2, i8 %x3)  #0 {
+;
+; CHECK-LABEL: @test_int_x86_avx512_mask_pminu_q_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <8 x i64> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP5:%.*]] = call <8 x i64> @llvm.umin.v8i64(<8 x i64> [[X0:%.*]], <8 x i64> [[X1:%.*]])
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i8 [[TMP3]] to <8 x i1>
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast i8 [[X3:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP8:%.*]] = select <8 x i1> [[TMP7]], <8 x i64> [[_MSPROP]], <8 x i64> [[TMP4]]
+; CHECK-NEXT:    [[TMP9:%.*]] = xor <8 x i64> [[TMP5]], [[X2:%.*]]
+; CHECK-NEXT:    [[TMP10:%.*]] = or <8 x i64> [[TMP9]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP11:%.*]] = or <8 x i64> [[TMP10]], [[TMP4]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP6]], <8 x i64> [[TMP11]], <8 x i64> [[TMP8]]
+; CHECK-NEXT:    [[TMP12:%.*]] = select <8 x i1> [[TMP7]], <8 x i64> [[TMP5]], <8 x i64> [[X2]]
+; CHECK-NEXT:    store <8 x i64> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i64> [[TMP12]]
+;
+  %res = call <8 x i64> @llvm.x86.avx512.mask.pminu.q.512(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> %x2, i8 %x3)
+  ret <8 x i64> %res
+}
+
+define <4 x float> @test_mm_mask_move_ss(<4 x float> %__W, i8 zeroext %__U, <4 x float> %__A, <4 x float> %__B)  #0 {
+;
+; CHECK-LABEL: @test_mm_mask_move_ss(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 40) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 24) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = and i8 [[TMP0]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = and i8 [[__U:%.*]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = and i8 [[TMP0]], 1
+; CHECK-NEXT:    [[TMP7:%.*]] = or i8 [[TMP4]], [[TMP5]]
+; CHECK-NEXT:    [[TMP8:%.*]] = or i8 [[TMP7]], [[TMP6]]
+; CHECK-NEXT:    [[TMP9:%.*]] = and i8 [[__U]], 1
+; CHECK-NEXT:    [[TMP10:%.*]] = xor i8 [[TMP9]], 0
+; CHECK-NEXT:    [[TMP11:%.*]] = or i8 [[TMP8]], 0
+; CHECK-NEXT:    [[TMP12:%.*]] = icmp ne i8 [[TMP11]], 0
+; CHECK-NEXT:    [[TMP13:%.*]] = xor i8 [[TMP11]], -1
+; CHECK-NEXT:    [[TMP14:%.*]] = and i8 [[TMP13]], [[TMP10]]
+; CHECK-NEXT:    [[TMP15:%.*]] = icmp eq i8 [[TMP14]], 0
+; CHECK-NEXT:    [[_MSPROP_ICMP:%.*]] = and i1 [[TMP12]], [[TMP15]]
+; CHECK-NEXT:    [[TMP16:%.*]] = icmp ne i8 [[TMP9]], 0
+; CHECK-NEXT:    [[_MSPROP:%.*]] = extractelement <4 x i32> [[TMP1]], i64 0
+; CHECK-NEXT:    [[TMP17:%.*]] = extractelement <4 x float> [[__B:%.*]], i64 0
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = extractelement <4 x i32> [[TMP2]], i64 0
+; CHECK-NEXT:    [[TMP18:%.*]] = extractelement <4 x float> [[__W:%.*]], i64 0
+; CHECK-NEXT:    [[TMP19:%.*]] = select i1 [[TMP16]], i32 [[_MSPROP]], i32 [[_MSPROP1]]
+; CHECK-NEXT:    [[TMP20:%.*]] = bitcast float [[TMP17]] to i32
+; CHECK-NEXT:    [[TMP21:%.*]] = bitcast float [[TMP18]] to i32
+; CHECK-NEXT:    [[TMP22:%.*]] = xor i32 [[TMP20]], [[TMP21]]
+; CHECK-NEXT:    [[TMP23:%.*]] = or i32 [[TMP22]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP24:%.*]] = or i32 [[TMP23]], [[_MSPROP1]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select i1 [[_MSPROP_ICMP]], i32 [[TMP24]], i32 [[TMP19]]
+; CHECK-NEXT:    [[TMP25:%.*]] = select i1 [[TMP16]], float [[TMP17]], float [[TMP18]]
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = insertelement <4 x i32> [[TMP3]], i32 [[_MSPROP_SELECT]], i64 0
+; CHECK-NEXT:    [[TMP26:%.*]] = insertelement <4 x float> [[__A:%.*]], float [[TMP25]], i64 0
+; CHECK-NEXT:    store <4 x i32> [[_MSPROP2]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x float> [[TMP26]]
+;
+entry:
+  %res = call <4 x float> @llvm.x86.avx512.mask.move.ss(<4 x float> %__A, <4 x float> %__B, <4 x float> %__W, i8 %__U)
+  ret <4 x float> %res
+}
+
+
+define <4 x float> @test_mm_maskz_move_ss(i8 zeroext %__U, <4 x float> %__A, <4 x float> %__B)  #0 {
+;
+; CHECK-LABEL: @test_mm_maskz_move_ss(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i8, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 24) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = and i8 [[TMP0]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = and i8 [[__U:%.*]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = and i8 [[TMP0]], 1
+; CHECK-NEXT:    [[TMP6:%.*]] = or i8 [[TMP3]], [[TMP4]]
+; CHECK-NEXT:    [[TMP7:%.*]] = or i8 [[TMP6]], [[TMP5]]
+; CHECK-NEXT:    [[TMP8:%.*]] = and i8 [[__U]], 1
+; CHECK-NEXT:    [[TMP9:%.*]] = xor i8 [[TMP8]], 0
+; CHECK-NEXT:    [[TMP10:%.*]] = or i8 [[TMP7]], 0
+; CHECK-NEXT:    [[TMP11:%.*]] = icmp ne i8 [[TMP10]], 0
+; CHECK-NEXT:    [[TMP12:%.*]] = xor i8 [[TMP10]], -1
+; CHECK-NEXT:    [[TMP13:%.*]] = and i8 [[TMP12]], [[TMP9]]
+; CHECK-NEXT:    [[TMP14:%.*]] = icmp eq i8 [[TMP13]], 0
+; CHECK-NEXT:    [[_MSPROP_ICMP:%.*]] = and i1 [[TMP11]], [[TMP14]]
+; CHECK-NEXT:    [[TMP15:%.*]] = icmp ne i8 [[TMP8]], 0
+; CHECK-NEXT:    [[_MSPROP:%.*]] = extractelement <4 x i32> [[TMP1]], i64 0
+; CHECK-NEXT:    [[TMP16:%.*]] = extractelement <4 x float> [[__B:%.*]], i64 0
+; CHECK-NEXT:    [[TMP17:%.*]] = select i1 [[TMP15]], i32 [[_MSPROP]], i32 0
+; CHECK-NEXT:    [[TMP18:%.*]] = bitcast float [[TMP16]] to i32
+; CHECK-NEXT:    [[TMP19:%.*]] = xor i32 [[TMP18]], 0
+; CHECK-NEXT:    [[TMP20:%.*]] = or i32 [[TMP19]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP21:%.*]] = or i32 [[TMP20]], 0
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select i1 [[_MSPROP_ICMP]], i32 [[TMP21]], i32 [[TMP17]]
+; CHECK-NEXT:    [[TMP22:%.*]] = select i1 [[TMP15]], float [[TMP16]], float 0.000000e+00
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = insertelement <4 x i32> [[TMP2]], i32 [[_MSPROP_SELECT]], i64 0
+; CHECK-NEXT:    [[TMP23:%.*]] = insertelement <4 x float> [[__A:%.*]], float [[TMP22]], i64 0
+; CHECK-NEXT:    store <4 x i32> [[_MSPROP1]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x float> [[TMP23]]
+;
+entry:
+  %res = call <4 x float> @llvm.x86.avx512.mask.move.ss(<4 x float> %__A, <4 x float> %__B, <4 x float> zeroinitializer, i8 %__U)
+  ret <4 x float> %res
+}
+
+define <2 x double> @test_mm_mask_move_sd(<2 x double> %__W, i8 zeroext %__U, <2 x double> %__A, <2 x double> %__B)  #0 {
+;
+; CHECK-LABEL: @test_mm_mask_move_sd(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 40) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 24) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = and i8 [[TMP0]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = and i8 [[__U:%.*]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = and i8 [[TMP0]], 1
+; CHECK-NEXT:    [[TMP7:%.*]] = or i8 [[TMP4]], [[TMP5]]
+; CHECK-NEXT:    [[TMP8:%.*]] = or i8 [[TMP7]], [[TMP6]]
+; CHECK-NEXT:    [[TMP9:%.*]] = and i8 [[__U]], 1
+; CHECK-NEXT:    [[TMP10:%.*]] = xor i8 [[TMP9]], 0
+; CHECK-NEXT:    [[TMP11:%.*]] = or i8 [[TMP8]], 0
+; CHECK-NEXT:    [[TMP12:%.*]] = icmp ne i8 [[TMP11]], 0
+; CHECK-NEXT:    [[TMP13:%.*]] = xor i8 [[TMP11]], -1
+; CHECK-NEXT:    [[TMP14:%.*]] = and i8 [[TMP13]], [[TMP10]]
+; CHECK-NEXT:    [[TMP15:%.*]] = icmp eq i8 [[TMP14]], 0
+; CHECK-NEXT:    [[_MSPROP_ICMP:%.*]] = and i1 [[TMP12]], [[TMP15]]
+; CHECK-NEXT:    [[TMP16:%.*]] = icmp ne i8 [[TMP9]], 0
+; CHECK-NEXT:    [[_MSPROP:%.*]] = extractelement <2 x i64> [[TMP1]], i64 0
+; CHECK-NEXT:    [[TMP17:%.*]] = extractelement <2 x double> [[__B:%.*]], i64 0
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = extractelement <2 x i64> [[TMP2]], i64 0
+; CHECK-NEXT:    [[TMP18:%.*]] = extractelement <2 x double> [[__W:%.*]], i64 0
+; CHECK-NEXT:    [[TMP19:%.*]] = select i1 [[TMP16]], i64 [[_MSPROP]], i64 [[_MSPROP1]]
+; CHECK-NEXT:    [[TMP20:%.*]] = bitcast double [[TMP17]] to i64
+; CHECK-NEXT:    [[TMP21:%.*]] = bitcast double [[TMP18]] to i64
+; CHECK-NEXT:    [[TMP22:%.*]] = xor i64 [[TMP20]], [[TMP21]]
+; CHECK-NEXT:    [[TMP23:%.*]] = or i64 [[TMP22]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP24:%.*]] = or i64 [[TMP23]], [[_MSPROP1]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select i1 [[_MSPROP_ICMP]], i64 [[TMP24]], i64 [[TMP19]]
+; CHECK-NEXT:    [[TMP25:%.*]] = select i1 [[TMP16]], double [[TMP17]], double [[TMP18]]
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = insertelement <2 x i64> [[TMP3]], i64 [[_MSPROP_SELECT]], i64 0
+; CHECK-NEXT:    [[TMP26:%.*]] = insertelement <2 x double> [[__A:%.*]], double [[TMP25]], i64 0
+; CHECK-NEXT:    store <2 x i64> [[_MSPROP2]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x double> [[TMP26]]
+;
+entry:
+  %res = call <2 x double> @llvm.x86.avx512.mask.move.sd(<2 x double> %__A, <2 x double> %__B, <2 x double> %__W, i8 %__U)
+  ret <2 x double> %res
+}
+
+define <2 x double> @test_mm_maskz_move_sd(i8 zeroext %__U, <2 x double> %__A, <2 x double> %__B)  #0 {
+;
+; CHECK-LABEL: @test_mm_maskz_move_sd(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i8, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 24) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = and i8 [[TMP0]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = and i8 [[__U:%.*]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = and i8 [[TMP0]], 1
+; CHECK-NEXT:    [[TMP6:%.*]] = or i8 [[TMP3]], [[TMP4]]
+; CHECK-NEXT:    [[TMP7:%.*]] = or i8 [[TMP6]], [[TMP5]]
+; CHECK-NEXT:    [[TMP8:%.*]] = and i8 [[__U]], 1
+; CHECK-NEXT:    [[TMP9:%.*]] = xor i8 [[TMP8]], 0
+; CHECK-NEXT:    [[TMP10:%.*]] = or i8 [[TMP7]], 0
+; CHECK-NEXT:    [[TMP11:%.*]] = icmp ne i8 [[TMP10]], 0
+; CHECK-NEXT:    [[TMP12:%.*]] = xor i8 [[TMP10]], -1
+; CHECK-NEXT:    [[TMP13:%.*]] = and i8 [[TMP12]], [[TMP9]]
+; CHECK-NEXT:    [[TMP14:%.*]] = icmp eq i8 [[TMP13]], 0
+; CHECK-NEXT:    [[_MSPROP_ICMP:%.*]] = and i1 [[TMP11]], [[TMP14]]
+; CHECK-NEXT:    [[TMP15:%.*]] = icmp ne i8 [[TMP8]], 0
+; CHECK-NEXT:    [[_MSPROP:%.*]] = extractelement <2 x i64> [[TMP1]], i64 0
+; CHECK-NEXT:    [[TMP16:%.*]] = extractelement <2 x double> [[__B:%.*]], i64 0
+; CHECK-NEXT:    [[TMP17:%.*]] = select i1 [[TMP15]], i64 [[_MSPROP]], i64 0
+; CHECK-NEXT:    [[TMP18:%.*]] = bitcast double [[TMP16]] to i64
+; CHECK-NEXT:    [[TMP19:%.*]] = xor i64 [[TMP18]], 0
+; CHECK-NEXT:    [[TMP20:%.*]] = or i64 [[TMP19]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP21:%.*]] = or i64 [[TMP20]], 0
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select i1 [[_MSPROP_ICMP]], i64 [[TMP21]], i64 [[TMP17]]
+; CHECK-NEXT:    [[TMP22:%.*]] = select i1 [[TMP15]], double [[TMP16]], double 0.000000e+00
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = insertelement <2 x i64> [[TMP2]], i64 [[_MSPROP_SELECT]], i64 0
+; CHECK-NEXT:    [[TMP23:%.*]] = insertelement <2 x double> [[__A:%.*]], double [[TMP22]], i64 0
+; CHECK-NEXT:    store <2 x i64> [[_MSPROP1]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x double> [[TMP23]]
+;
+entry:
+  %res = call <2 x double> @llvm.x86.avx512.mask.move.sd(<2 x double> %__A, <2 x double> %__B, <2 x double> zeroinitializer, i8 %__U)
+  ret <2 x double> %res
+}
+
+declare <4 x float> @llvm.x86.avx512.mask.move.ss(<4 x float>, <4 x float>, <4 x float>, i8)
+declare <2 x double> @llvm.x86.avx512.mask.move.sd(<2 x double>, <2 x double>, <2 x double>, i8)
+
+declare <16 x i32> @llvm.x86.avx512.mask.pmovzxb.d.512(<16 x i8>, <16 x i32>, i16)
+
+define <16 x i32>@test_int_x86_avx512_pmovzxb_d_512(<16 x i8> %x0, <16 x i32> %x1)  #0 {
+; CHECK-LABEL: @test_int_x86_avx512_pmovzxb_d_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i8>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <16 x i8> [[TMP1]], <16 x i8> splat (i8 -1), <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <16 x i8> [[X0:%.*]], <16 x i8> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = zext <16 x i8> [[_MSPROP]] to <16 x i32>
+; CHECK-NEXT:    [[TMP3:%.*]] = zext <16 x i8> [[TMP2]] to <16 x i32>
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP1]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i32> [[TMP3]]
+;
+  %res = call <16 x i32> @llvm.x86.avx512.mask.pmovzxb.d.512(<16 x i8> %x0, <16 x i32> %x1, i16 -1)
+  ret <16 x i32> %res
+}
+
+define <16 x i32>@test_int_x86_avx512_mask_pmovzxb_d_512(<16 x i8> %x0, <16 x i32> %x1, i16 %x2)  #0 {
+;
+; CHECK-LABEL: @test_int_x86_avx512_mask_pmovzxb_d_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i8>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 80) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <16 x i8> [[TMP1]], <16 x i8> splat (i8 -1), <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <16 x i8> [[X0:%.*]], <16 x i8> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = zext <16 x i8> [[_MSPROP]] to <16 x i32>
+; CHECK-NEXT:    [[TMP5:%.*]] = zext <16 x i8> [[TMP4]] to <16 x i32>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i16 [[TMP2]] to <16 x i1>
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast i16 [[X2:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP8:%.*]] = select <16 x i1> [[TMP7]], <16 x i32> [[_MSPROP1]], <16 x i32> [[TMP3]]
+; CHECK-NEXT:    [[TMP9:%.*]] = xor <16 x i32> [[TMP5]], [[X1:%.*]]
+; CHECK-NEXT:    [[TMP10:%.*]] = or <16 x i32> [[TMP9]], [[_MSPROP1]]
+; CHECK-NEXT:    [[TMP11:%.*]] = or <16 x i32> [[TMP10]], [[TMP3]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP6]], <16 x i32> [[TMP11]], <16 x i32> [[TMP8]]
+; CHECK-NEXT:    [[TMP12:%.*]] = select <16 x i1> [[TMP7]], <16 x i32> [[TMP5]], <16 x i32> [[X1]]
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i32> [[TMP12]]
+;
+  %res = call <16 x i32> @llvm.x86.avx512.mask.pmovzxb.d.512(<16 x i8> %x0, <16 x i32> %x1, i16 %x2)
+  ret <16 x i32> %res
+}
+
+define <16 x i32>@test_int_x86_avx512_maskz_pmovzxb_d_512(<16 x i8> %x0, i16 %x2)  #0 {
+;
+; CHECK-LABEL: @test_int_x86_avx512_maskz_pmovzxb_d_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i8>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <16 x i8> [[TMP1]], <16 x i8> splat (i8 -1), <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <16 x i8> [[X0:%.*]], <16 x i8> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = zext <16 x i8> [[_MSPROP]] to <16 x i32>
+; CHECK-NEXT:    [[TMP4:%.*]] = zext <16 x i8> [[TMP3]] to <16 x i32>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i16 [[TMP2]] to <16 x i1>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i16 [[X2:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP7:%.*]] = select <16 x i1> [[TMP6]], <16 x i32> [[_MSPROP1]], <16 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP8:%.*]] = xor <16 x i32> [[TMP4]], zeroinitializer
+; CHECK-NEXT:    [[TMP9:%.*]] = or <16 x i32> [[TMP8]], [[_MSPROP1]]
+; CHECK-NEXT:    [[TMP10:%.*]] = or <16 x i32> [[TMP9]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP5]], <16 x i32> [[TMP10]], <16 x i32> [[TMP7]]
+; CHECK-NEXT:    [[TMP11:%.*]] = select <16 x i1> [[TMP6]], <16 x i32> [[TMP4]], <16 x i32> zeroinitializer
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i32> [[TMP11]]
+;
+  %res = call <16 x i32> @llvm.x86.avx512.mask.pmovzxb.d.512(<16 x i8> %x0, <16 x i32> zeroinitializer, i16 %x2)
+  ret <16 x i32> %res
+}
+
+declare <8 x i64> @llvm.x86.avx512.mask.pmovzxb.q.512(<16 x i8>, <8 x i64>, i8)
+
+define <8 x i64>@test_int_x86_avx512_pmovzxb_q_512(<16 x i8> %x0, <8 x i64> %x1)  #0 {
+; CHECK-LABEL: @test_int_x86_avx512_pmovzxb_q_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i8>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <16 x i8> [[TMP1]], <16 x i8> splat (i8 -1), <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <16 x i8> [[X0:%.*]], <16 x i8> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = zext <8 x i8> [[_MSPROP]] to <8 x i64>
+; CHECK-NEXT:    [[TMP3:%.*]] = zext <8 x i8> [[TMP2]] to <8 x i64>
+; CHECK-NEXT:    store <8 x i64> [[_MSPROP1]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i64> [[TMP3]]
+;
+  %res = call <8 x i64> @llvm.x86.avx512.mask.pmovzxb.q.512(<16 x i8> %x0, <8 x i64> %x1, i8 -1)
+  ret <8 x i64> %res
+}
+
+define <8 x i64>@test_int_x86_avx512_mask_pmovzxb_q_512(<16 x i8> %x0, <8 x i64> %x1, i8 %x2)  #0 {
+;
+; CHECK-LABEL: @test_int_x86_avx512_mask_pmovzxb_q_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i8>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 80) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <16 x i8> [[TMP1]], <16 x i8> splat (i8 -1), <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <16 x i8> [[X0:%.*]], <16 x i8> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = zext <8 x i8> [[_MSPROP]] to <8 x i64>
+; CHECK-NEXT:    [[TMP5:%.*]] = zext <8 x i8> [[TMP4]] to <8 x i64>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i8 [[TMP2]] to <8 x i1>
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast i8 [[X2:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP8:%.*]] = select <8 x i1> [[TMP7]], <8 x i64> [[_MSPROP1]], <8 x i64> [[TMP3]]
+; CHECK-NEXT:    [[TMP9:%.*]] = xor <8 x i64> [[TMP5]], [[X1:%.*]]
+; CHECK-NEXT:    [[TMP10:%.*]] = or <8 x i64> [[TMP9]], [[_MSPROP1]]
+; CHECK-NEXT:    [[TMP11:%.*]] = or <8 x i64> [[TMP10]], [[TMP3]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP6]], <8 x i64> [[TMP11]], <8 x i64> [[TMP8]]
+; CHECK-NEXT:    [[TMP12:%.*]] = select <8 x i1> [[TMP7]], <8 x i64> [[TMP5]], <8 x i64> [[X1]]
+; CHECK-NEXT:    store <8 x i64> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i64> [[TMP12]]
+;
+  %res = call <8 x i64> @llvm.x86.avx512.mask.pmovzxb.q.512(<16 x i8> %x0, <8 x i64> %x1, i8 %x2)
+  ret <8 x i64> %res
+}
+
+define <8 x i64>@test_int_x86_avx512_maskz_pmovzxb_q_512(<16 x i8> %x0, i8 %x2)  #0 {
+;
+; CHECK-LABEL: @test_int_x86_avx512_maskz_pmovzxb_q_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i8>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <16 x i8> [[TMP1]], <16 x i8> splat (i8 -1), <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <16 x i8> [[X0:%.*]], <16 x i8> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = zext <8 x i8> [[_MSPROP]] to <8 x i64>
+; CHECK-NEXT:    [[TMP4:%.*]] = zext <8 x i8> [[TMP3]] to <8 x i64>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i8 [[TMP2]] to <8 x i1>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i8 [[X2:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP7:%.*]] = select <8 x i1> [[TMP6]], <8 x i64> [[_MSPROP1]], <8 x i64> zeroinitializer
+; CHECK-NEXT:    [[TMP8:%.*]] = xor <8 x i64> [[TMP4]], zeroinitializer
+; CHECK-NEXT:    [[TMP9:%.*]] = or <8 x i64> [[TMP8]], [[_MSPROP1]]
+; CHECK-NEXT:    [[TMP10:%.*]] = or <8 x i64> [[TMP9]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP5]], <8 x i64> [[TMP10]], <8 x i64> [[TMP7]]
+; CHECK-NEXT:    [[TMP11:%.*]] = select <8 x i1> [[TMP6]], <8 x i64> [[TMP4]], <8 x i64> zeroinitializer
+; CHECK-NEXT:    store <8 x i64> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i64> [[TMP11]]
+;
+  %res = call <8 x i64> @llvm.x86.avx512.mask.pmovzxb.q.512(<16 x i8> %x0, <8 x i64> zeroinitializer, i8 %x2)
+  ret <8 x i64> %res
+}
+
+declare <8 x i64> @llvm.x86.avx512.mask.pmovzxd.q.512(<8 x i32>, <8 x i64>, i8)
+
+define <8 x i64>@test_int_x86_avx512_pmovzxd_q_512(<8 x i32> %x0, <8 x i64> %x1)  #0 {
+; CHECK-LABEL: @test_int_x86_avx512_pmovzxd_q_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <8 x i32> [[TMP1]], <8 x i32> splat (i32 -1), <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <8 x i32> [[X0:%.*]], <8 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = zext <8 x i32> [[_MSPROP]] to <8 x i64>
+; CHECK-NEXT:    [[TMP3:%.*]] = zext <8 x i32> [[TMP2]] to <8 x i64>
+; CHECK-NEXT:    store <8 x i64> [[_MSPROP1]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i64> [[TMP3]]
+;
+  %res = call <8 x i64> @llvm.x86.avx512.mask.pmovzxd.q.512(<8 x i32> %x0, <8 x i64> %x1, i8 -1)
+  ret <8 x i64> %res
+}
+
+define <8 x i64>@test_int_x86_avx512_mask_pmovzxd_q_512(<8 x i32> %x0, <8 x i64> %x1, i8 %x2)  #0 {
+;
+; CHECK-LABEL: @test_int_x86_avx512_mask_pmovzxd_q_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 96) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <8 x i32> [[TMP1]], <8 x i32> splat (i32 -1), <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <8 x i32> [[X0:%.*]], <8 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = zext <8 x i32> [[_MSPROP]] to <8 x i64>
+; CHECK-NEXT:    [[TMP5:%.*]] = zext <8 x i32> [[TMP4]] to <8 x i64>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i8 [[TMP2]] to <8 x i1>
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast i8 [[X2:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP8:%.*]] = select <8 x i1> [[TMP7]], <8 x i64> [[_MSPROP1]], <8 x i64> [[TMP3]]
+; CHECK-NEXT:    [[TMP9:%.*]] = xor <8 x i64> [[TMP5]], [[X1:%.*]]
+; CHECK-NEXT:    [[TMP10:%.*]] = or <8 x i64> [[TMP9]], [[_MSPROP1]]
+; CHECK-NEXT:    [[TMP11:%.*]] = or <8 x i64> [[TMP10]], [[TMP3]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP6]], <8 x i64> [[TMP11]], <8 x i64> [[TMP8]]
+; CHECK-NEXT:    [[TMP12:%.*]] = select <8 x i1> [[TMP7]], <8 x i64> [[TMP5]], <8 x i64> [[X1]]
+; CHECK-NEXT:    store <8 x i64> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i64> [[TMP12]]
+;
+  %res = call <8 x i64> @llvm.x86.avx512.mask.pmovzxd.q.512(<8 x i32> %x0, <8 x i64> %x1, i8 %x2)
+  ret <8 x i64> %res
+}
+
+define <8 x i64>@test_int_x86_avx512_maskz_pmovzxd_q_512(<8 x i32> %x0, i8 %x2)  #0 {
+;
+; CHECK-LABEL: @test_int_x86_avx512_maskz_pmovzxd_q_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <8 x i32> [[TMP1]], <8 x i32> splat (i32 -1), <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <8 x i32> [[X0:%.*]], <8 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = zext <8 x i32> [[_MSPROP]] to <8 x i64>
+; CHECK-NEXT:    [[TMP4:%.*]] = zext <8 x i32> [[TMP3]] to <8 x i64>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i8 [[TMP2]] to <8 x i1>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i8 [[X2:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP7:%.*]] = select <8 x i1> [[TMP6]], <8 x i64> [[_MSPROP1]], <8 x i64> zeroinitializer
+; CHECK-NEXT:    [[TMP8:%.*]] = xor <8 x i64> [[TMP4]], zeroinitializer
+; CHECK-NEXT:    [[TMP9:%.*]] = or <8 x i64> [[TMP8]], [[_MSPROP1]]
+; CHECK-NEXT:    [[TMP10:%.*]] = or <8 x i64> [[TMP9]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP5]], <8 x i64> [[TMP10]], <8 x i64> [[TMP7]]
+; CHECK-NEXT:    [[TMP11:%.*]] = select <8 x i1> [[TMP6]], <8 x i64> [[TMP4]], <8 x i64> zeroinitializer
+; CHECK-NEXT:    store <8 x i64> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i64> [[TMP11]]
+;
+  %res = call <8 x i64> @llvm.x86.avx512.mask.pmovzxd.q.512(<8 x i32> %x0, <8 x i64> zeroinitializer, i8 %x2)
+  ret <8 x i64> %res
+}
+
+declare <16 x i32> @llvm.x86.avx512.mask.pmovzxw.d.512(<16 x i16>, <16 x i32>, i16)
+
+define <16 x i32>@test_int_x86_avx512_pmovzxw_d_512(<16 x i16> %x0, <16 x i32> %x1)  #0 {
+; CHECK-LABEL: @test_int_x86_avx512_pmovzxw_d_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <16 x i16> [[TMP1]], <16 x i16> splat (i16 -1), <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <16 x i16> [[X0:%.*]], <16 x i16> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = zext <16 x i16> [[_MSPROP]] to <16 x i32>
+; CHECK-NEXT:    [[TMP3:%.*]] = zext <16 x i16> [[TMP2]] to <16 x i32>
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP1]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i32> [[TMP3]]
+;
+  %res = call <16 x i32> @llvm.x86.avx512.mask.pmovzxw.d.512(<16 x i16> %x0, <16 x i32> %x1, i16 -1)
+  ret <16 x i32> %res
+}
+
+define <16 x i32>@test_int_x86_avx512_mask_pmovzxw_d_512(<16 x i16> %x0, <16 x i32> %x1, i16 %x2)  #0 {
+;
+; CHECK-LABEL: @test_int_x86_avx512_mask_pmovzxw_d_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 96) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <16 x i16> [[TMP1]], <16 x i16> splat (i16 -1), <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <16 x i16> [[X0:%.*]], <16 x i16> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = zext <16 x i16> [[_MSPROP]] to <16 x i32>
+; CHECK-NEXT:    [[TMP5:%.*]] = zext <16 x i16> [[TMP4]] to <16 x i32>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i16 [[TMP2]] to <16 x i1>
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast i16 [[X2:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP8:%.*]] = select <16 x i1> [[TMP7]], <16 x i32> [[_MSPROP1]], <16 x i32> [[TMP3]]
+; CHECK-NEXT:    [[TMP9:%.*]] = xor <16 x i32> [[TMP5]], [[X1:%.*]]
+; CHECK-NEXT:    [[TMP10:%.*]] = or <16 x i32> [[TMP9]], [[_MSPROP1]]
+; CHECK-NEXT:    [[TMP11:%.*]] = or <16 x i32> [[TMP10]], [[TMP3]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP6]], <16 x i32> [[TMP11]], <16 x i32> [[TMP8]]
+; CHECK-NEXT:    [[TMP12:%.*]] = select <16 x i1> [[TMP7]], <16 x i32> [[TMP5]], <16 x i32> [[X1]]
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i32> [[TMP12]]
+;
+  %res = call <16 x i32> @llvm.x86.avx512.mask.pmovzxw.d.512(<16 x i16> %x0, <16 x i32> %x1, i16 %x2)
+  ret <16 x i32> %res
+}
+
+define <16 x i32>@test_int_x86_avx512_maskz_pmovzxw_d_512(<16 x i16> %x0, i16 %x2)  #0 {
+;
+; CHECK-LABEL: @test_int_x86_avx512_maskz_pmovzxw_d_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <16 x i16> [[TMP1]], <16 x i16> splat (i16 -1), <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <16 x i16> [[X0:%.*]], <16 x i16> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = zext <16 x i16> [[_MSPROP]] to <16 x i32>
+; CHECK-NEXT:    [[TMP4:%.*]] = zext <16 x i16> [[TMP3]] to <16 x i32>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i16 [[TMP2]] to <16 x i1>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i16 [[X2:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP7:%.*]] = select <16 x i1> [[TMP6]], <16 x i32> [[_MSPROP1]], <16 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP8:%.*]] = xor <16 x i32> [[TMP4]], zeroinitializer
+; CHECK-NEXT:    [[TMP9:%.*]] = or <16 x i32> [[TMP8]], [[_MSPROP1]]
+; CHECK-NEXT:    [[TMP10:%.*]] = or <16 x i32> [[TMP9]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP5]], <16 x i32> [[TMP10]], <16 x i32> [[TMP7]]
+; CHECK-NEXT:    [[TMP11:%.*]] = select <16 x i1> [[TMP6]], <16 x i32> [[TMP4]], <16 x i32> zeroinitializer
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i32> [[TMP11]]
+;
+  %res = call <16 x i32> @llvm.x86.avx512.mask.pmovzxw.d.512(<16 x i16> %x0, <16 x i32> zeroinitializer, i16 %x2)
+  ret <16 x i32> %res
+}
+
+declare <8 x i64> @llvm.x86.avx512.mask.pmovzxw.q.512(<8 x i16>, <8 x i64>, i8)
+
+define <8 x i64>@test_int_x86_avx512_pmovzxw_q_512(<8 x i16> %x0, <8 x i64> %x1)  #0 {
+; CHECK-LABEL: @test_int_x86_avx512_pmovzxw_q_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <8 x i16> [[TMP1]], <8 x i16> splat (i16 -1), <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <8 x i16> [[X0:%.*]], <8 x i16> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = zext <8 x i16> [[_MSPROP]] to <8 x i64>
+; CHECK-NEXT:    [[TMP3:%.*]] = zext <8 x i16> [[TMP2]] to <8 x i64>
+; CHECK-NEXT:    store <8 x i64> [[_MSPROP1]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i64> [[TMP3]]
+;
+  %res = call <8 x i64> @llvm.x86.avx512.mask.pmovzxw.q.512(<8 x i16> %x0, <8 x i64> %x1, i8 -1)
+  ret <8 x i64> %res
+}
+
+define <8 x i64>@test_int_x86_avx512_mask_pmovzxw_q_512(<8 x i16> %x0, <8 x i64> %x1, i8 %x2)  #0 {
+;
+; CHECK-LABEL: @test_int_x86_avx512_mask_pmovzxw_q_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 80) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <8 x i16> [[TMP1]], <8 x i16> splat (i16 -1), <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <8 x i16> [[X0:%.*]], <8 x i16> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = zext <8 x i16> [[_MSPROP]] to <8 x i64>
+; CHECK-NEXT:    [[TMP5:%.*]] = zext <8 x i16> [[TMP4]] to <8 x i64>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i8 [[TMP2]] to <8 x i1>
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast i8 [[X2:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP8:%.*]] = select <8 x i1> [[TMP7]], <8 x i64> [[_MSPROP1]], <8 x i64> [[TMP3]]
+; CHECK-NEXT:    [[TMP9:%.*]] = xor <8 x i64> [[TMP5]], [[X1:%.*]]
+; CHECK-NEXT:    [[TMP10:%.*]] = or <8 x i64> [[TMP9]], [[_MSPROP1]]
+; CHECK-NEXT:    [[TMP11:%.*]] = or <8 x i64> [[TMP10]], [[TMP3]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP6]], <8 x i64> [[TMP11]], <8 x i64> [[TMP8]]
+; CHECK-NEXT:    [[TMP12:%.*]] = select <8 x i1> [[TMP7]], <8 x i64> [[TMP5]], <8 x i64> [[X1]]
+; CHECK-NEXT:    store <8 x i64> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i64> [[TMP12]]
+;
+  %res = call <8 x i64> @llvm.x86.avx512.mask.pmovzxw.q.512(<8 x i16> %x0, <8 x i64> %x1, i8 %x2)
+  ret <8 x i64> %res
+}
+
+define <8 x i64>@test_int_x86_avx512_maskz_pmovzxw_q_512(<8 x i16> %x0, i8 %x2)  #0 {
+;
+; CHECK-LABEL: @test_int_x86_avx512_maskz_pmovzxw_q_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <8 x i16> [[TMP1]], <8 x i16> splat (i16 -1), <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <8 x i16> [[X0:%.*]], <8 x i16> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = zext <8 x i16> [[_MSPROP]] to <8 x i64>
+; CHECK-NEXT:    [[TMP4:%.*]] = zext <8 x i16> [[TMP3]] to <8 x i64>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i8 [[TMP2]] to <8 x i1>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i8 [[X2:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP7:%.*]] = select <8 x i1> [[TMP6]], <8 x i64> [[_MSPROP1]], <8 x i64> zeroinitializer
+; CHECK-NEXT:    [[TMP8:%.*]] = xor <8 x i64> [[TMP4]], zeroinitializer
+; CHECK-NEXT:    [[TMP9:%.*]] = or <8 x i64> [[TMP8]], [[_MSPROP1]]
+; CHECK-NEXT:    [[TMP10:%.*]] = or <8 x i64> [[TMP9]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP5]], <8 x i64> [[TMP10]], <8 x i64> [[TMP7]]
+; CHECK-NEXT:    [[TMP11:%.*]] = select <8 x i1> [[TMP6]], <8 x i64> [[TMP4]], <8 x i64> zeroinitializer
+; CHECK-NEXT:    store <8 x i64> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i64> [[TMP11]]
+;
+  %res = call <8 x i64> @llvm.x86.avx512.mask.pmovzxw.q.512(<8 x i16> %x0, <8 x i64> zeroinitializer, i8 %x2)
+  ret <8 x i64> %res
+}
+
+declare <16 x i32> @llvm.x86.avx512.mask.pmovsxb.d.512(<16 x i8>, <16 x i32>, i16)
+
+define <16 x i32>@test_int_x86_avx512_pmovsxb_d_512(<16 x i8> %x0, <16 x i32> %x1)  #0 {
+; CHECK-LABEL: @test_int_x86_avx512_pmovsxb_d_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i8>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <16 x i8> [[TMP1]], <16 x i8> splat (i8 -1), <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <16 x i8> [[X0:%.*]], <16 x i8> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = sext <16 x i8> [[_MSPROP]] to <16 x i32>
+; CHECK-NEXT:    [[TMP3:%.*]] = sext <16 x i8> [[TMP2]] to <16 x i32>
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP1]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i32> [[TMP3]]
+;
+  %res = call <16 x i32> @llvm.x86.avx512.mask.pmovsxb.d.512(<16 x i8> %x0, <16 x i32> %x1, i16 -1)
+  ret <16 x i32> %res
+}
+
+define <16 x i32>@test_int_x86_avx512_mask_pmovsxb_d_512(<16 x i8> %x0, <16 x i32> %x1, i16 %x2)  #0 {
+;
+; CHECK-LABEL: @test_int_x86_avx512_mask_pmovsxb_d_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i8>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 80) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <16 x i8> [[TMP1]], <16 x i8> splat (i8 -1), <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <16 x i8> [[X0:%.*]], <16 x i8> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = sext <16 x i8> [[_MSPROP]] to <16 x i32>
+; CHECK-NEXT:    [[TMP5:%.*]] = sext <16 x i8> [[TMP4]] to <16 x i32>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i16 [[TMP2]] to <16 x i1>
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast i16 [[X2:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP8:%.*]] = select <16 x i1> [[TMP7]], <16 x i32> [[_MSPROP1]], <16 x i32> [[TMP3]]
+; CHECK-NEXT:    [[TMP9:%.*]] = xor <16 x i32> [[TMP5]], [[X1:%.*]]
+; CHECK-NEXT:    [[TMP10:%.*]] = or <16 x i32> [[TMP9]], [[_MSPROP1]]
+; CHECK-NEXT:    [[TMP11:%.*]] = or <16 x i32> [[TMP10]], [[TMP3]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP6]], <16 x i32> [[TMP11]], <16 x i32> [[TMP8]]
+; CHECK-NEXT:    [[TMP12:%.*]] = select <16 x i1> [[TMP7]], <16 x i32> [[TMP5]], <16 x i32> [[X1]]
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i32> [[TMP12]]
+;
+  %res = call <16 x i32> @llvm.x86.avx512.mask.pmovsxb.d.512(<16 x i8> %x0, <16 x i32> %x1, i16 %x2)
+  ret <16 x i32> %res
+}
+
+define <16 x i32>@test_int_x86_avx512_maskz_pmovsxb_d_512(<16 x i8> %x0, i16 %x2)  #0 {
+;
+; CHECK-LABEL: @test_int_x86_avx512_maskz_pmovsxb_d_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i8>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <16 x i8> [[TMP1]], <16 x i8> splat (i8 -1), <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <16 x i8> [[X0:%.*]], <16 x i8> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = sext <16 x i8> [[_MSPROP]] to <16 x i32>
+; CHECK-NEXT:    [[TMP4:%.*]] = sext <16 x i8> [[TMP3]] to <16 x i32>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i16 [[TMP2]] to <16 x i1>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i16 [[X2:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP7:%.*]] = select <16 x i1> [[TMP6]], <16 x i32> [[_MSPROP1]], <16 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP8:%.*]] = xor <16 x i32> [[TMP4]], zeroinitializer
+; CHECK-NEXT:    [[TMP9:%.*]] = or <16 x i32> [[TMP8]], [[_MSPROP1]]
+; CHECK-NEXT:    [[TMP10:%.*]] = or <16 x i32> [[TMP9]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP5]], <16 x i32> [[TMP10]], <16 x i32> [[TMP7]]
+; CHECK-NEXT:    [[TMP11:%.*]] = select <16 x i1> [[TMP6]], <16 x i32> [[TMP4]], <16 x i32> zeroinitializer
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i32> [[TMP11]]
+;
+  %res = call <16 x i32> @llvm.x86.avx512.mask.pmovsxb.d.512(<16 x i8> %x0, <16 x i32> zeroinitializer, i16 %x2)
+  ret <16 x i32> %res
+}
+
+declare <8 x i64> @llvm.x86.avx512.mask.pmovsxb.q.512(<16 x i8>, <8 x i64>, i8)
+
+define <8 x i64>@test_int_x86_avx512_pmovsxb_q_512(<16 x i8> %x0, <8 x i64> %x1)  #0 {
+; CHECK-LABEL: @test_int_x86_avx512_pmovsxb_q_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i8>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <16 x i8> [[TMP1]], <16 x i8> splat (i8 -1), <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <16 x i8> [[X0:%.*]], <16 x i8> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = sext <8 x i8> [[_MSPROP]] to <8 x i64>
+; CHECK-NEXT:    [[TMP3:%.*]] = sext <8 x i8> [[TMP2]] to <8 x i64>
+; CHECK-NEXT:    store <8 x i64> [[_MSPROP1]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i64> [[TMP3]]
+;
+  %res = call <8 x i64> @llvm.x86.avx512.mask.pmovsxb.q.512(<16 x i8> %x0, <8 x i64> %x1, i8 -1)
+  ret <8 x i64> %res
+}
+
+define <8 x i64>@test_int_x86_avx512_mask_pmovsxb_q_512(<16 x i8> %x0, <8 x i64> %x1, i8 %x2)  #0 {
+;
+; CHECK-LABEL: @test_int_x86_avx512_mask_pmovsxb_q_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i8>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 80) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <16 x i8> [[TMP1]], <16 x i8> splat (i8 -1), <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <16 x i8> [[X0:%.*]], <16 x i8> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = sext <8 x i8> [[_MSPROP]] to <8 x i64>
+; CHECK-NEXT:    [[TMP5:%.*]] = sext <8 x i8> [[TMP4]] to <8 x i64>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i8 [[TMP2]] to <8 x i1>
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast i8 [[X2:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP8:%.*]] = select <8 x i1> [[TMP7]], <8 x i64> [[_MSPROP1]], <8 x i64> [[TMP3]]
+; CHECK-NEXT:    [[TMP9:%.*]] = xor <8 x i64> [[TMP5]], [[X1:%.*]]
+; CHECK-NEXT:    [[TMP10:%.*]] = or <8 x i64> [[TMP9]], [[_MSPROP1]]
+; CHECK-NEXT:    [[TMP11:%.*]] = or <8 x i64> [[TMP10]], [[TMP3]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP6]], <8 x i64> [[TMP11]], <8 x i64> [[TMP8]]
+; CHECK-NEXT:    [[TMP12:%.*]] = select <8 x i1> [[TMP7]], <8 x i64> [[TMP5]], <8 x i64> [[X1]]
+; CHECK-NEXT:    store <8 x i64> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i64> [[TMP12]]
+;
+  %res = call <8 x i64> @llvm.x86.avx512.mask.pmovsxb.q.512(<16 x i8> %x0, <8 x i64> %x1, i8 %x2)
+  ret <8 x i64> %res
+}
+
+define <8 x i64>@test_int_x86_avx512_maskz_pmovsxb_q_512(<16 x i8> %x0, i8 %x2)  #0 {
+;
+; CHECK-LABEL: @test_int_x86_avx512_maskz_pmovsxb_q_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i8>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <16 x i8> [[TMP1]], <16 x i8> splat (i8 -1), <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <16 x i8> [[X0:%.*]], <16 x i8> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = sext <8 x i8> [[_MSPROP]] to <8 x i64>
+; CHECK-NEXT:    [[TMP4:%.*]] = sext <8 x i8> [[TMP3]] to <8 x i64>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i8 [[TMP2]] to <8 x i1>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i8 [[X2:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP7:%.*]] = select <8 x i1> [[TMP6]], <8 x i64> [[_MSPROP1]], <8 x i64> zeroinitializer
+; CHECK-NEXT:    [[TMP8:%.*]] = xor <8 x i64> [[TMP4]], zeroinitializer
+; CHECK-NEXT:    [[TMP9:%.*]] = or <8 x i64> [[TMP8]], [[_MSPROP1]]
+; CHECK-NEXT:    [[TMP10:%.*]] = or <8 x i64> [[TMP9]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP5]], <8 x i64> [[TMP10]], <8 x i64> [[TMP7]]
+; CHECK-NEXT:    [[TMP11:%.*]] = select <8 x i1> [[TMP6]], <8 x i64> [[TMP4]], <8 x i64> zeroinitializer
+; CHECK-NEXT:    store <8 x i64> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i64> [[TMP11]]
+;
+  %res = call <8 x i64> @llvm.x86.avx512.mask.pmovsxb.q.512(<16 x i8> %x0, <8 x i64> zeroinitializer, i8 %x2)
+  ret <8 x i64> %res
+}
+
+declare <8 x i64> @llvm.x86.avx512.mask.pmovsxd.q.512(<8 x i32>, <8 x i64>, i8)
+
+define <8 x i64>@test_int_x86_avx512_pmovsxd_q_512(<8 x i32> %x0, <8 x i64> %x1)  #0 {
+; CHECK-LABEL: @test_int_x86_avx512_pmovsxd_q_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <8 x i32> [[TMP1]], <8 x i32> splat (i32 -1), <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <8 x i32> [[X0:%.*]], <8 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = sext <8 x i32> [[_MSPROP]] to <8 x i64>
+; CHECK-NEXT:    [[TMP3:%.*]] = sext <8 x i32> [[TMP2]] to <8 x i64>
+; CHECK-NEXT:    store <8 x i64> [[_MSPROP1]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i64> [[TMP3]]
+;
+  %res = call <8 x i64> @llvm.x86.avx512.mask.pmovsxd.q.512(<8 x i32> %x0, <8 x i64> %x1, i8 -1)
+  ret <8 x i64> %res
+}
+
+define <8 x i64>@test_int_x86_avx512_mask_pmovsxd_q_512(<8 x i32> %x0, <8 x i64> %x1, i8 %x2)  #0 {
+;
+; CHECK-LABEL: @test_int_x86_avx512_mask_pmovsxd_q_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 96) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <8 x i32> [[TMP1]], <8 x i32> splat (i32 -1), <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <8 x i32> [[X0:%.*]], <8 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = sext <8 x i32> [[_MSPROP]] to <8 x i64>
+; CHECK-NEXT:    [[TMP5:%.*]] = sext <8 x i32> [[TMP4]] to <8 x i64>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i8 [[TMP2]] to <8 x i1>
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast i8 [[X2:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP8:%.*]] = select <8 x i1> [[TMP7]], <8 x i64> [[_MSPROP1]], <8 x i64> [[TMP3]]
+; CHECK-NEXT:    [[TMP9:%.*]] = xor <8 x i64> [[TMP5]], [[X1:%.*]]
+; CHECK-NEXT:    [[TMP10:%.*]] = or <8 x i64> [[TMP9]], [[_MSPROP1]]
+; CHECK-NEXT:    [[TMP11:%.*]] = or <8 x i64> [[TMP10]], [[TMP3]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP6]], <8 x i64> [[TMP11]], <8 x i64> [[TMP8]]
+; CHECK-NEXT:    [[TMP12:%.*]] = select <8 x i1> [[TMP7]], <8 x i64> [[TMP5]], <8 x i64> [[X1]]
+; CHECK-NEXT:    store <8 x i64> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i64> [[TMP12]]
+;
+  %res = call <8 x i64> @llvm.x86.avx512.mask.pmovsxd.q.512(<8 x i32> %x0, <8 x i64> %x1, i8 %x2)
+  ret <8 x i64> %res
+}
+
+define <8 x i64>@test_int_x86_avx512_maskz_pmovsxd_q_512(<8 x i32> %x0, i8 %x2)  #0 {
+;
+; CHECK-LABEL: @test_int_x86_avx512_maskz_pmovsxd_q_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <8 x i32> [[TMP1]], <8 x i32> splat (i32 -1), <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <8 x i32> [[X0:%.*]], <8 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = sext <8 x i32> [[_MSPROP]] to <8 x i64>
+; CHECK-NEXT:    [[TMP4:%.*]] = sext <8 x i32> [[TMP3]] to <8 x i64>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i8 [[TMP2]] to <8 x i1>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i8 [[X2:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP7:%.*]] = select <8 x i1> [[TMP6]], <8 x i64> [[_MSPROP1]], <8 x i64> zeroinitializer
+; CHECK-NEXT:    [[TMP8:%.*]] = xor <8 x i64> [[TMP4]], zeroinitializer
+; CHECK-NEXT:    [[TMP9:%.*]] = or <8 x i64> [[TMP8]], [[_MSPROP1]]
+; CHECK-NEXT:    [[TMP10:%.*]] = or <8 x i64> [[TMP9]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP5]], <8 x i64> [[TMP10]], <8 x i64> [[TMP7]]
+; CHECK-NEXT:    [[TMP11:%.*]] = select <8 x i1> [[TMP6]], <8 x i64> [[TMP4]], <8 x i64> zeroinitializer
+; CHECK-NEXT:    store <8 x i64> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i64> [[TMP11]]
+;
+  %res = call <8 x i64> @llvm.x86.avx512.mask.pmovsxd.q.512(<8 x i32> %x0, <8 x i64> zeroinitializer, i8 %x2)
+  ret <8 x i64> %res
+}
+
+declare <16 x i32> @llvm.x86.avx512.mask.pmovsxw.d.512(<16 x i16>, <16 x i32>, i16)
+
+define <16 x i32>@test_int_x86_avx512_pmovsxw_d_512(<16 x i16> %x0, <16 x i32> %x1)  #0 {
+; CHECK-LABEL: @test_int_x86_avx512_pmovsxw_d_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <16 x i16> [[TMP1]], <16 x i16> splat (i16 -1), <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <16 x i16> [[X0:%.*]], <16 x i16> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = sext <16 x i16> [[_MSPROP]] to <16 x i32>
+; CHECK-NEXT:    [[TMP3:%.*]] = sext <16 x i16> [[TMP2]] to <16 x i32>
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP1]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i32> [[TMP3]]
+;
+  %res = call <16 x i32> @llvm.x86.avx512.mask.pmovsxw.d.512(<16 x i16> %x0, <16 x i32> %x1, i16 -1)
+  ret <16 x i32> %res
+}
+
+define <16 x i32>@test_int_x86_avx512_mask_pmovsxw_d_512(<16 x i16> %x0, <16 x i32> %x1, i16 %x2)  #0 {
+;
+; CHECK-LABEL: @test_int_x86_avx512_mask_pmovsxw_d_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 96) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <16 x i16> [[TMP1]], <16 x i16> splat (i16 -1), <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <16 x i16> [[X0:%.*]], <16 x i16> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = sext <16 x i16> [[_MSPROP]] to <16 x i32>
+; CHECK-NEXT:    [[TMP5:%.*]] = sext <16 x i16> [[TMP4]] to <16 x i32>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i16 [[TMP2]] to <16 x i1>
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast i16 [[X2:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP8:%.*]] = select <16 x i1> [[TMP7]], <16 x i32> [[_MSPROP1]], <16 x i32> [[TMP3]]
+; CHECK-NEXT:    [[TMP9:%.*]] = xor <16 x i32> [[TMP5]], [[X1:%.*]]
+; CHECK-NEXT:    [[TMP10:%.*]] = or <16 x i32> [[TMP9]], [[_MSPROP1]]
+; CHECK-NEXT:    [[TMP11:%.*]] = or <16 x i32> [[TMP10]], [[TMP3]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP6]], <16 x i32> [[TMP11]], <16 x i32> [[TMP8]]
+; CHECK-NEXT:    [[TMP12:%.*]] = select <16 x i1> [[TMP7]], <16 x i32> [[TMP5]], <16 x i32> [[X1]]
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i32> [[TMP12]]
+;
+  %res = call <16 x i32> @llvm.x86.avx512.mask.pmovsxw.d.512(<16 x i16> %x0, <16 x i32> %x1, i16 %x2)
+  ret <16 x i32> %res
+}
+
+define <16 x i32>@test_int_x86_avx512_maskz_pmovsxw_d_512(<16 x i16> %x0, i16 %x2)  #0 {
+;
+; CHECK-LABEL: @test_int_x86_avx512_maskz_pmovsxw_d_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <16 x i16> [[TMP1]], <16 x i16> splat (i16 -1), <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <16 x i16> [[X0:%.*]], <16 x i16> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = sext <16 x i16> [[_MSPROP]] to <16 x i32>
+; CHECK-NEXT:    [[TMP4:%.*]] = sext <16 x i16> [[TMP3]] to <16 x i32>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i16 [[TMP2]] to <16 x i1>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i16 [[X2:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP7:%.*]] = select <16 x i1> [[TMP6]], <16 x i32> [[_MSPROP1]], <16 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP8:%.*]] = xor <16 x i32> [[TMP4]], zeroinitializer
+; CHECK-NEXT:    [[TMP9:%.*]] = or <16 x i32> [[TMP8]], [[_MSPROP1]]
+; CHECK-NEXT:    [[TMP10:%.*]] = or <16 x i32> [[TMP9]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP5]], <16 x i32> [[TMP10]], <16 x i32> [[TMP7]]
+; CHECK-NEXT:    [[TMP11:%.*]] = select <16 x i1> [[TMP6]], <16 x i32> [[TMP4]], <16 x i32> zeroinitializer
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i32> [[TMP11]]
+;
+  %res = call <16 x i32> @llvm.x86.avx512.mask.pmovsxw.d.512(<16 x i16> %x0, <16 x i32> zeroinitializer, i16 %x2)
+  ret <16 x i32> %res
+}
+
+declare <8 x i64> @llvm.x86.avx512.mask.pmovsxw.q.512(<8 x i16>, <8 x i64>, i8)
+
+define <8 x i64>@test_int_x86_avx512_pmovsxw_q_512(<8 x i16> %x0, <8 x i64> %x1)  #0 {
+; CHECK-LABEL: @test_int_x86_avx512_pmovsxw_q_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <8 x i16> [[TMP1]], <8 x i16> splat (i16 -1), <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <8 x i16> [[X0:%.*]], <8 x i16> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = sext <8 x i16> [[_MSPROP]] to <8 x i64>
+; CHECK-NEXT:    [[TMP3:%.*]] = sext <8 x i16> [[TMP2]] to <8 x i64>
+; CHECK-NEXT:    store <8 x i64> [[_MSPROP1]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i64> [[TMP3]]
+;
+  %res = call <8 x i64> @llvm.x86.avx512.mask.pmovsxw.q.512(<8 x i16> %x0, <8 x i64> %x1, i8 -1)
+  ret <8 x i64> %res
+}
+
+define <8 x i64>@test_int_x86_avx512_mask_pmovsxw_q_512(<8 x i16> %x0, <8 x i64> %x1, i8 %x2)  #0 {
+;
+; CHECK-LABEL: @test_int_x86_avx512_mask_pmovsxw_q_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 80) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <8 x i16> [[TMP1]], <8 x i16> splat (i16 -1), <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <8 x i16> [[X0:%.*]], <8 x i16> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = sext <8 x i16> [[_MSPROP]] to <8 x i64>
+; CHECK-NEXT:    [[TMP5:%.*]] = sext <8 x i16> [[TMP4]] to <8 x i64>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i8 [[TMP2]] to <8 x i1>
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast i8 [[X2:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP8:%.*]] = select <8 x i1> [[TMP7]], <8 x i64> [[_MSPROP1]], <8 x i64> [[TMP3]]
+; CHECK-NEXT:    [[TMP9:%.*]] = xor <8 x i64> [[TMP5]], [[X1:%.*]]
+; CHECK-NEXT:    [[TMP10:%.*]] = or <8 x i64> [[TMP9]], [[_MSPROP1]]
+; CHECK-NEXT:    [[TMP11:%.*]] = or <8 x i64> [[TMP10]], [[TMP3]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP6]], <8 x i64> [[TMP11]], <8 x i64> [[TMP8]]
+; CHECK-NEXT:    [[TMP12:%.*]] = select <8 x i1> [[TMP7]], <8 x i64> [[TMP5]], <8 x i64> [[X1]]
+; CHECK-NEXT:    store <8 x i64> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i64> [[TMP12]]
+;
+  %res = call <8 x i64> @llvm.x86.avx512.mask.pmovsxw.q.512(<8 x i16> %x0, <8 x i64> %x1, i8 %x2)
+  ret <8 x i64> %res
+}
+
+define <8 x i64>@test_int_x86_avx512_maskz_pmovsxw_q_512(<8 x i16> %x0, i8 %x2)  #0 {
+;
+; CHECK-LABEL: @test_int_x86_avx512_maskz_pmovsxw_q_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <8 x i16> [[TMP1]], <8 x i16> splat (i16 -1), <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <8 x i16> [[X0:%.*]], <8 x i16> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = sext <8 x i16> [[_MSPROP]] to <8 x i64>
+; CHECK-NEXT:    [[TMP4:%.*]] = sext <8 x i16> [[TMP3]] to <8 x i64>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i8 [[TMP2]] to <8 x i1>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i8 [[X2:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP7:%.*]] = select <8 x i1> [[TMP6]], <8 x i64> [[_MSPROP1]], <8 x i64> zeroinitializer
+; CHECK-NEXT:    [[TMP8:%.*]] = xor <8 x i64> [[TMP4]], zeroinitializer
+; CHECK-NEXT:    [[TMP9:%.*]] = or <8 x i64> [[TMP8]], [[_MSPROP1]]
+; CHECK-NEXT:    [[TMP10:%.*]] = or <8 x i64> [[TMP9]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP5]], <8 x i64> [[TMP10]], <8 x i64> [[TMP7]]
+; CHECK-NEXT:    [[TMP11:%.*]] = select <8 x i1> [[TMP6]], <8 x i64> [[TMP4]], <8 x i64> zeroinitializer
+; CHECK-NEXT:    store <8 x i64> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i64> [[TMP11]]
+;
+  %res = call <8 x i64> @llvm.x86.avx512.mask.pmovsxw.q.512(<8 x i16> %x0, <8 x i64> zeroinitializer, i8 %x2)
+  ret <8 x i64> %res
+}
+
+declare <16 x i32> @llvm.x86.avx512.prolv.d.512(<16 x i32>, <16 x i32>)
+
+define <16 x i32>@test_int_x86_avx512_prolv_d_512(<16 x i32> %x0, <16 x i32> %x1)  #0 {
+; CHECK-LABEL: @test_int_x86_avx512_prolv_d_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp ne <16 x i32> [[TMP2]], zeroinitializer
+; CHECK-NEXT:    [[TMP4:%.*]] = sext <16 x i1> [[TMP3]] to <16 x i32>
+; CHECK-NEXT:    [[TMP5:%.*]] = call <16 x i32> @llvm.fshl.v16i32(<16 x i32> [[TMP1]], <16 x i32> [[TMP1]], <16 x i32> [[X1:%.*]])
+; CHECK-NEXT:    [[TMP6:%.*]] = or <16 x i32> [[TMP5]], [[TMP4]]
+; CHECK-NEXT:    [[TMP7:%.*]] = call <16 x i32> @llvm.fshl.v16i32(<16 x i32> [[X0:%.*]], <16 x i32> [[X0]], <16 x i32> [[X1]])
+; CHECK-NEXT:    store <16 x i32> [[TMP6]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i32> [[TMP7]]
+;
+  %1 = call <16 x i32> @llvm.x86.avx512.prolv.d.512(<16 x i32> %x0, <16 x i32> %x1)
+  ret <16 x i32> %1
+}
+
+define <16 x i32>@test_int_x86_avx512_mask_prolv_d_512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x2, i16 %x3)  #0 {
+;
+; CHECK-LABEL: @test_int_x86_avx512_mask_prolv_d_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp ne <16 x i32> [[TMP2]], zeroinitializer
+; CHECK-NEXT:    [[TMP6:%.*]] = sext <16 x i1> [[TMP5]] to <16 x i32>
+; CHECK-NEXT:    [[TMP7:%.*]] = call <16 x i32> @llvm.fshl.v16i32(<16 x i32> [[TMP1]], <16 x i32> [[TMP1]], <16 x i32> [[X1:%.*]])
+; CHECK-NEXT:    [[TMP8:%.*]] = or <16 x i32> [[TMP7]], [[TMP6]]
+; CHECK-NEXT:    [[TMP9:%.*]] = call <16 x i32> @llvm.fshl.v16i32(<16 x i32> [[X0:%.*]], <16 x i32> [[X0]], <16 x i32> [[X1]])
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i16 [[TMP3]] to <16 x i1>
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast i16 [[X3:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP12:%.*]] = select <16 x i1> [[TMP11]], <16 x i32> [[TMP8]], <16 x i32> [[TMP4]]
+; CHECK-NEXT:    [[TMP13:%.*]] = xor <16 x i32> [[TMP9]], [[X2:%.*]]
+; CHECK-NEXT:    [[TMP14:%.*]] = or <16 x i32> [[TMP13]], [[TMP8]]
+; CHECK-NEXT:    [[TMP15:%.*]] = or <16 x i32> [[TMP14]], [[TMP4]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP10]], <16 x i32> [[TMP15]], <16 x i32> [[TMP12]]
+; CHECK-NEXT:    [[TMP16:%.*]] = select <16 x i1> [[TMP11]], <16 x i32> [[TMP9]], <16 x i32> [[X2]]
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i32> [[TMP16]]
+;
+  %1 = call <16 x i32> @llvm.x86.avx512.prolv.d.512(<16 x i32> %x0, <16 x i32> %x1)
+  %2 = bitcast i16 %x3 to <16 x i1>
+  %3 = select <16 x i1> %2, <16 x i32> %1, <16 x i32> %x2
+  ret <16 x i32> %3
+}
+
+define <16 x i32>@test_int_x86_avx512_maskz_prolv_d_512(<16 x i32> %x0, <16 x i32> %x1, i16 %x3)  #0 {
+;
+; CHECK-LABEL: @test_int_x86_avx512_maskz_prolv_d_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = icmp ne <16 x i32> [[TMP2]], zeroinitializer
+; CHECK-NEXT:    [[TMP5:%.*]] = sext <16 x i1> [[TMP4]] to <16 x i32>
+; CHECK-NEXT:    [[TMP6:%.*]] = call <16 x i32> @llvm.fshl.v16i32(<16 x i32> [[TMP1]], <16 x i32> [[TMP1]], <16 x i32> [[X1:%.*]])
+; CHECK-NEXT:    [[TMP7:%.*]] = or <16 x i32> [[TMP6]], [[TMP5]]
+; CHECK-NEXT:    [[TMP8:%.*]] = call <16 x i32> @llvm.fshl.v16i32(<16 x i32> [[X0:%.*]], <16 x i32> [[X0]], <16 x i32> [[X1]])
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast i16 [[TMP3]] to <16 x i1>
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i16 [[X3:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP11:%.*]] = select <16 x i1> [[TMP10]], <16 x i32> [[TMP7]], <16 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP12:%.*]] = xor <16 x i32> [[TMP8]], zeroinitializer
+; CHECK-NEXT:    [[TMP13:%.*]] = or <16 x i32> [[TMP12]], [[TMP7]]
+; CHECK-NEXT:    [[TMP14:%.*]] = or <16 x i32> [[TMP13]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP9]], <16 x i32> [[TMP14]], <16 x i32> [[TMP11]]
+; CHECK-NEXT:    [[TMP15:%.*]] = select <16 x i1> [[TMP10]], <16 x i32> [[TMP8]], <16 x i32> zeroinitializer
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i32> [[TMP15]]
+;
+  %1 = call <16 x i32> @llvm.x86.avx512.prolv.d.512(<16 x i32> %x0, <16 x i32> %x1)
+  %2 = bitcast i16 %x3 to <16 x i1>
+  %3 = select <16 x i1> %2, <16 x i32> %1, <16 x i32> zeroinitializer
+  ret <16 x i32> %3
+}
+
+declare <8 x i64> @llvm.x86.avx512.prolv.q.512(<8 x i64>, <8 x i64>)
+
+define <8 x i64>@test_int_x86_avx512_prolv_q_512(<8 x i64> %x0, <8 x i64> %x1)  #0 {
+; CHECK-LABEL: @test_int_x86_avx512_prolv_q_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp ne <8 x i64> [[TMP2]], zeroinitializer
+; CHECK-NEXT:    [[TMP4:%.*]] = sext <8 x i1> [[TMP3]] to <8 x i64>
+; CHECK-NEXT:    [[TMP5:%.*]] = call <8 x i64> @llvm.fshl.v8i64(<8 x i64> [[TMP1]], <8 x i64> [[TMP1]], <8 x i64> [[X1:%.*]])
+; CHECK-NEXT:    [[TMP6:%.*]] = or <8 x i64> [[TMP5]], [[TMP4]]
+; CHECK-NEXT:    [[TMP7:%.*]] = call <8 x i64> @llvm.fshl.v8i64(<8 x i64> [[X0:%.*]], <8 x i64> [[X0]], <8 x i64> [[X1]])
+; CHECK-NEXT:    store <8 x i64> [[TMP6]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i64> [[TMP7]]
+;
+  %1 = call <8 x i64> @llvm.x86.avx512.prolv.q.512(<8 x i64> %x0, <8 x i64> %x1)
+  ret <8 x i64> %1
+}
+
+define <8 x i64>@test_int_x86_avx512_mask_prolv_q_512(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> %x2, i8 %x3)  #0 {
+;
+; CHECK-LABEL: @test_int_x86_avx512_mask_prolv_q_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp ne <8 x i64> [[TMP2]], zeroinitializer
+; CHECK-NEXT:    [[TMP6:%.*]] = sext <8 x i1> [[TMP5]] to <8 x i64>
+; CHECK-NEXT:    [[TMP7:%.*]] = call <8 x i64> @llvm.fshl.v8i64(<8 x i64> [[TMP1]], <8 x i64> [[TMP1]], <8 x i64> [[X1:%.*]])
+; CHECK-NEXT:    [[TMP8:%.*]] = or <8 x i64> [[TMP7]], [[TMP6]]
+; CHECK-NEXT:    [[TMP9:%.*]] = call <8 x i64> @llvm.fshl.v8i64(<8 x i64> [[X0:%.*]], <8 x i64> [[X0]], <8 x i64> [[X1]])
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i8 [[TMP3]] to <8 x i1>
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast i8 [[X3:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP12:%.*]] = select <8 x i1> [[TMP11]], <8 x i64> [[TMP8]], <8 x i64> [[TMP4]]
+; CHECK-NEXT:    [[TMP13:%.*]] = xor <8 x i64> [[TMP9]], [[X2:%.*]]
+; CHECK-NEXT:    [[TMP14:%.*]] = or <8 x i64> [[TMP13]], [[TMP8]]
+; CHECK-NEXT:    [[TMP15:%.*]] = or <8 x i64> [[TMP14]], [[TMP4]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP10]], <8 x i64> [[TMP15]], <8 x i64> [[TMP12]]
+; CHECK-NEXT:    [[TMP16:%.*]] = select <8 x i1> [[TMP11]], <8 x i64> [[TMP9]], <8 x i64> [[X2]]
+; CHECK-NEXT:    store <8 x i64> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i64> [[TMP16]]
+;
+  %1 = call <8 x i64> @llvm.x86.avx512.prolv.q.512(<8 x i64> %x0, <8 x i64> %x1)
+  %2 = bitcast i8 %x3 to <8 x i1>
+  %3 = select <8 x i1> %2, <8 x i64> %1, <8 x i64> %x2
+  ret <8 x i64> %3
+}
+
+define <8 x i64>@test_int_x86_avx512_maskz_prolv_q_512(<8 x i64> %x0, <8 x i64> %x1, i8 %x3)  #0 {
+;
+; CHECK-LABEL: @test_int_x86_avx512_maskz_prolv_q_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = icmp ne <8 x i64> [[TMP2]], zeroinitializer
+; CHECK-NEXT:    [[TMP5:%.*]] = sext <8 x i1> [[TMP4]] to <8 x i64>
+; CHECK-NEXT:    [[TMP6:%.*]] = call <8 x i64> @llvm.fshl.v8i64(<8 x i64> [[TMP1]], <8 x i64> [[TMP1]], <8 x i64> [[X1:%.*]])
+; CHECK-NEXT:    [[TMP7:%.*]] = or <8 x i64> [[TMP6]], [[TMP5]]
+; CHECK-NEXT:    [[TMP8:%.*]] = call <8 x i64> @llvm.fshl.v8i64(<8 x i64> [[X0:%.*]], <8 x i64> [[X0]], <8 x i64> [[X1]])
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast i8 [[TMP3]] to <8 x i1>
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i8 [[X3:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP11:%.*]] = select <8 x i1> [[TMP10]], <8 x i64> [[TMP7]], <8 x i64> zeroinitializer
+; CHECK-NEXT:    [[TMP12:%.*]] = xor <8 x i64> [[TMP8]], zeroinitializer
+; CHECK-NEXT:    [[TMP13:%.*]] = or <8 x i64> [[TMP12]], [[TMP7]]
+; CHECK-NEXT:    [[TMP14:%.*]] = or <8 x i64> [[TMP13]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP9]], <8 x i64> [[TMP14]], <8 x i64> [[TMP11]]
+; CHECK-NEXT:    [[TMP15:%.*]] = select <8 x i1> [[TMP10]], <8 x i64> [[TMP8]], <8 x i64> zeroinitializer
+; CHECK-NEXT:    store <8 x i64> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i64> [[TMP15]]
+;
+  %1 = call <8 x i64> @llvm.x86.avx512.prolv.q.512(<8 x i64> %x0, <8 x i64> %x1)
+  %2 = bitcast i8 %x3 to <8 x i1>
+  %3 = select <8 x i1> %2, <8 x i64> %1, <8 x i64> zeroinitializer
+  ret <8 x i64> %3
+}
+
+declare <16 x i32> @llvm.x86.avx512.prorv.d.512(<16 x i32>, <16 x i32>)
+
+define <16 x i32>@test_int_x86_avx512_prorv_d_512(<16 x i32> %x0, <16 x i32> %x1)  #0 {
+; CHECK-LABEL: @test_int_x86_avx512_prorv_d_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp ne <16 x i32> [[TMP2]], zeroinitializer
+; CHECK-NEXT:    [[TMP4:%.*]] = sext <16 x i1> [[TMP3]] to <16 x i32>
+; CHECK-NEXT:    [[TMP5:%.*]] = call <16 x i32> @llvm.fshr.v16i32(<16 x i32> [[TMP1]], <16 x i32> [[TMP1]], <16 x i32> [[X1:%.*]])
+; CHECK-NEXT:    [[TMP6:%.*]] = or <16 x i32> [[TMP5]], [[TMP4]]
+; CHECK-NEXT:    [[TMP7:%.*]] = call <16 x i32> @llvm.fshr.v16i32(<16 x i32> [[X0:%.*]], <16 x i32> [[X0]], <16 x i32> [[X1]])
+; CHECK-NEXT:    store <16 x i32> [[TMP6]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i32> [[TMP7]]
+;
+  %1 = call <16 x i32> @llvm.x86.avx512.prorv.d.512(<16 x i32> %x0, <16 x i32> %x1)
+  ret <16 x i32> %1
+}
+
+define <16 x i32>@test_int_x86_avx512_mask_prorv_d_512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x2, i16 %x3)  #0 {
+;
+; CHECK-LABEL: @test_int_x86_avx512_mask_prorv_d_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp ne <16 x i32> [[TMP2]], zeroinitializer
+; CHECK-NEXT:    [[TMP6:%.*]] = sext <16 x i1> [[TMP5]] to <16 x i32>
+; CHECK-NEXT:    [[TMP7:%.*]] = call <16 x i32> @llvm.fshr.v16i32(<16 x i32> [[TMP1]], <16 x i32> [[TMP1]], <16 x i32> [[X1:%.*]])
+; CHECK-NEXT:    [[TMP8:%.*]] = or <16 x i32> [[TMP7]], [[TMP6]]
+; CHECK-NEXT:    [[TMP9:%.*]] = call <16 x i32> @llvm.fshr.v16i32(<16 x i32> [[X0:%.*]], <16 x i32> [[X0]], <16 x i32> [[X1]])
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i16 [[TMP3]] to <16 x i1>
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast i16 [[X3:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP12:%.*]] = select <16 x i1> [[TMP11]], <16 x i32> [[TMP8]], <16 x i32> [[TMP4]]
+; CHECK-NEXT:    [[TMP13:%.*]] = xor <16 x i32> [[TMP9]], [[X2:%.*]]
+; CHECK-NEXT:    [[TMP14:%.*]] = or <16 x i32> [[TMP13]], [[TMP8]]
+; CHECK-NEXT:    [[TMP15:%.*]] = or <16 x i32> [[TMP14]], [[TMP4]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP10]], <16 x i32> [[TMP15]], <16 x i32> [[TMP12]]
+; CHECK-NEXT:    [[TMP16:%.*]] = select <16 x i1> [[TMP11]], <16 x i32> [[TMP9]], <16 x i32> [[X2]]
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i32> [[TMP16]]
+;
+  %1 = call <16 x i32> @llvm.x86.avx512.prorv.d.512(<16 x i32> %x0, <16 x i32> %x1)
+  %2 = bitcast i16 %x3 to <16 x i1>
+  %3 = select <16 x i1> %2, <16 x i32> %1, <16 x i32> %x2
+  ret <16 x i32> %3
+}
+
+define <16 x i32>@test_int_x86_avx512_maskz_prorv_d_512(<16 x i32> %x0, <16 x i32> %x1, i16 %x3)  #0 {
+;
+; CHECK-LABEL: @test_int_x86_avx512_maskz_prorv_d_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = icmp ne <16 x i32> [[TMP2]], zeroinitializer
+; CHECK-NEXT:    [[TMP5:%.*]] = sext <16 x i1> [[TMP4]] to <16 x i32>
+; CHECK-NEXT:    [[TMP6:%.*]] = call <16 x i32> @llvm.fshr.v16i32(<16 x i32> [[TMP1]], <16 x i32> [[TMP1]], <16 x i32> [[X1:%.*]])
+; CHECK-NEXT:    [[TMP7:%.*]] = or <16 x i32> [[TMP6]], [[TMP5]]
+; CHECK-NEXT:    [[TMP8:%.*]] = call <16 x i32> @llvm.fshr.v16i32(<16 x i32> [[X0:%.*]], <16 x i32> [[X0]], <16 x i32> [[X1]])
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast i16 [[TMP3]] to <16 x i1>
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i16 [[X3:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP11:%.*]] = select <16 x i1> [[TMP10]], <16 x i32> [[TMP7]], <16 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP12:%.*]] = xor <16 x i32> [[TMP8]], zeroinitializer
+; CHECK-NEXT:    [[TMP13:%.*]] = or <16 x i32> [[TMP12]], [[TMP7]]
+; CHECK-NEXT:    [[TMP14:%.*]] = or <16 x i32> [[TMP13]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP9]], <16 x i32> [[TMP14]], <16 x i32> [[TMP11]]
+; CHECK-NEXT:    [[TMP15:%.*]] = select <16 x i1> [[TMP10]], <16 x i32> [[TMP8]], <16 x i32> zeroinitializer
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i32> [[TMP15]]
+;
+  %1 = call <16 x i32> @llvm.x86.avx512.prorv.d.512(<16 x i32> %x0, <16 x i32> %x1)
+  %2 = bitcast i16 %x3 to <16 x i1>
+  %3 = select <16 x i1> %2, <16 x i32> %1, <16 x i32> zeroinitializer
+  ret <16 x i32> %3
+}
+
+declare <8 x i64> @llvm.x86.avx512.prorv.q.512(<8 x i64>, <8 x i64>)
+
+define <8 x i64>@test_int_x86_avx512_prorv_q_512(<8 x i64> %x0, <8 x i64> %x1)  #0 {
+; CHECK-LABEL: @test_int_x86_avx512_prorv_q_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp ne <8 x i64> [[TMP2]], zeroinitializer
+; CHECK-NEXT:    [[TMP4:%.*]] = sext <8 x i1> [[TMP3]] to <8 x i64>
+; CHECK-NEXT:    [[TMP5:%.*]] = call <8 x i64> @llvm.fshr.v8i64(<8 x i64> [[TMP1]], <8 x i64> [[TMP1]], <8 x i64> [[X1:%.*]])
+; CHECK-NEXT:    [[TMP6:%.*]] = or <8 x i64> [[TMP5]], [[TMP4]]
+; CHECK-NEXT:    [[TMP7:%.*]] = call <8 x i64> @llvm.fshr.v8i64(<8 x i64> [[X0:%.*]], <8 x i64> [[X0]], <8 x i64> [[X1]])
+; CHECK-NEXT:    store <8 x i64> [[TMP6]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i64> [[TMP7]]
+;
+  %1 = call <8 x i64> @llvm.x86.avx512.prorv.q.512(<8 x i64> %x0, <8 x i64> %x1)
+  ret <8 x i64> %1
+}
+
+define <8 x i64>@test_int_x86_avx512_mask_prorv_q_512(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> %x2, i8 %x3)  #0 {
+;
+; CHECK-LABEL: @test_int_x86_avx512_mask_prorv_q_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp ne <8 x i64> [[TMP2]], zeroinitializer
+; CHECK-NEXT:    [[TMP6:%.*]] = sext <8 x i1> [[TMP5]] to <8 x i64>
+; CHECK-NEXT:    [[TMP7:%.*]] = call <8 x i64> @llvm.fshr.v8i64(<8 x i64> [[TMP1]], <8 x i64> [[TMP1]], <8 x i64> [[X1:%.*]])
+; CHECK-NEXT:    [[TMP8:%.*]] = or <8 x i64> [[TMP7]], [[TMP6]]
+; CHECK-NEXT:    [[TMP9:%.*]] = call <8 x i64> @llvm.fshr.v8i64(<8 x i64> [[X0:%.*]], <8 x i64> [[X0]], <8 x i64> [[X1]])
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i8 [[TMP3]] to <8 x i1>
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast i8 [[X3:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP12:%.*]] = select <8 x i1> [[TMP11]], <8 x i64> [[TMP8]], <8 x i64> [[TMP4]]
+; CHECK-NEXT:    [[TMP13:%.*]] = xor <8 x i64> [[TMP9]], [[X2:%.*]]
+; CHECK-NEXT:    [[TMP14:%.*]] = or <8 x i64> [[TMP13]], [[TMP8]]
+; CHECK-NEXT:    [[TMP15:%.*]] = or <8 x i64> [[TMP14]], [[TMP4]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP10]], <8 x i64> [[TMP15]], <8 x i64> [[TMP12]]
+; CHECK-NEXT:    [[TMP16:%.*]] = select <8 x i1> [[TMP11]], <8 x i64> [[TMP9]], <8 x i64> [[X2]]
+; CHECK-NEXT:    store <8 x i64> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i64> [[TMP16]]
+;
+  %1 = call <8 x i64> @llvm.x86.avx512.prorv.q.512(<8 x i64> %x0, <8 x i64> %x1)
+  %2 = bitcast i8 %x3 to <8 x i1>
+  %3 = select <8 x i1> %2, <8 x i64> %1, <8 x i64> %x2
+  ret <8 x i64> %3
+}
+
+define <8 x i64>@test_int_x86_avx512_maskz_prorv_q_512(<8 x i64> %x0, <8 x i64> %x1, i8 %x3)  #0 {
+;
+; CHECK-LABEL: @test_int_x86_avx512_maskz_prorv_q_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = icmp ne <8 x i64> [[TMP2]], zeroinitializer
+; CHECK-NEXT:    [[TMP5:%.*]] = sext <8 x i1> [[TMP4]] to <8 x i64>
+; CHECK-NEXT:    [[TMP6:%.*]] = call <8 x i64> @llvm.fshr.v8i64(<8 x i64> [[TMP1]], <8 x i64> [[TMP1]], <8 x i64> [[X1:%.*]])
+; CHECK-NEXT:    [[TMP7:%.*]] = or <8 x i64> [[TMP6]], [[TMP5]]
+; CHECK-NEXT:    [[TMP8:%.*]] = call <8 x i64> @llvm.fshr.v8i64(<8 x i64> [[X0:%.*]], <8 x i64> [[X0]], <8 x i64> [[X1]])
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast i8 [[TMP3]] to <8 x i1>
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i8 [[X3:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP11:%.*]] = select <8 x i1> [[TMP10]], <8 x i64> [[TMP7]], <8 x i64> zeroinitializer
+; CHECK-NEXT:    [[TMP12:%.*]] = xor <8 x i64> [[TMP8]], zeroinitializer
+; CHECK-NEXT:    [[TMP13:%.*]] = or <8 x i64> [[TMP12]], [[TMP7]]
+; CHECK-NEXT:    [[TMP14:%.*]] = or <8 x i64> [[TMP13]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP9]], <8 x i64> [[TMP14]], <8 x i64> [[TMP11]]
+; CHECK-NEXT:    [[TMP15:%.*]] = select <8 x i1> [[TMP10]], <8 x i64> [[TMP8]], <8 x i64> zeroinitializer
+; CHECK-NEXT:    store <8 x i64> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i64> [[TMP15]]
+;
+  %1 = call <8 x i64> @llvm.x86.avx512.prorv.q.512(<8 x i64> %x0, <8 x i64> %x1)
+  %2 = bitcast i8 %x3 to <8 x i1>
+  %3 = select <8 x i1> %2, <8 x i64> %1, <8 x i64> zeroinitializer
+  ret <8 x i64> %3
+}
+
+declare <16 x i32> @llvm.x86.avx512.prol.d.512(<16 x i32>, i32)
+
+define { <16 x i32>, <16 x i32>, <16 x i32> } @test_int_x86_avx512_prol_d_512(<16 x i32> %x0, i32 %x1, <16 x i32> %x2, i16 %x3)  #0 {
+;
+; CHECK-LABEL: @test_int_x86_avx512_prol_d_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 136) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 72) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = call <16 x i32> @llvm.fshl.v16i32(<16 x i32> [[TMP1]], <16 x i32> [[TMP1]], <16 x i32> splat (i32 3))
+; CHECK-NEXT:    [[TMP5:%.*]] = or <16 x i32> [[TMP4]], zeroinitializer
+; CHECK-NEXT:    [[TMP6:%.*]] = call <16 x i32> @llvm.fshl.v16i32(<16 x i32> [[X0:%.*]], <16 x i32> [[X0]], <16 x i32> splat (i32 3))
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast i16 [[TMP2]] to <16 x i1>
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast i16 [[X3:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP9:%.*]] = select <16 x i1> [[TMP8]], <16 x i32> [[TMP5]], <16 x i32> [[TMP3]]
+; CHECK-NEXT:    [[TMP10:%.*]] = xor <16 x i32> [[TMP6]], [[X2:%.*]]
+; CHECK-NEXT:    [[TMP11:%.*]] = or <16 x i32> [[TMP10]], [[TMP5]]
+; CHECK-NEXT:    [[TMP12:%.*]] = or <16 x i32> [[TMP11]], [[TMP3]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP7]], <16 x i32> [[TMP12]], <16 x i32> [[TMP9]]
+; CHECK-NEXT:    [[TMP13:%.*]] = select <16 x i1> [[TMP8]], <16 x i32> [[TMP6]], <16 x i32> [[X2]]
+; CHECK-NEXT:    [[TMP14:%.*]] = call <16 x i32> @llvm.fshl.v16i32(<16 x i32> [[TMP1]], <16 x i32> [[TMP1]], <16 x i32> splat (i32 4))
+; CHECK-NEXT:    [[TMP15:%.*]] = or <16 x i32> [[TMP14]], zeroinitializer
+; CHECK-NEXT:    [[TMP16:%.*]] = call <16 x i32> @llvm.fshl.v16i32(<16 x i32> [[X0]], <16 x i32> [[X0]], <16 x i32> splat (i32 4))
+; CHECK-NEXT:    [[TMP17:%.*]] = bitcast i16 [[TMP2]] to <16 x i1>
+; CHECK-NEXT:    [[TMP18:%.*]] = bitcast i16 [[X3]] to <16 x i1>
+; CHECK-NEXT:    [[TMP19:%.*]] = select <16 x i1> [[TMP18]], <16 x i32> [[TMP15]], <16 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP20:%.*]] = xor <16 x i32> [[TMP16]], zeroinitializer
+; CHECK-NEXT:    [[TMP21:%.*]] = or <16 x i32> [[TMP20]], [[TMP15]]
+; CHECK-NEXT:    [[TMP22:%.*]] = or <16 x i32> [[TMP21]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT1:%.*]] = select <16 x i1> [[TMP17]], <16 x i32> [[TMP22]], <16 x i32> [[TMP19]]
+; CHECK-NEXT:    [[TMP23:%.*]] = select <16 x i1> [[TMP18]], <16 x i32> [[TMP16]], <16 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP24:%.*]] = call <16 x i32> @llvm.fshl.v16i32(<16 x i32> [[TMP1]], <16 x i32> [[TMP1]], <16 x i32> splat (i32 5))
+; CHECK-NEXT:    [[TMP25:%.*]] = or <16 x i32> [[TMP24]], zeroinitializer
+; CHECK-NEXT:    [[TMP26:%.*]] = call <16 x i32> @llvm.fshl.v16i32(<16 x i32> [[X0]], <16 x i32> [[X0]], <16 x i32> splat (i32 5))
+; CHECK-NEXT:    [[TMP27:%.*]] = insertvalue { <16 x i32>, <16 x i32>, <16 x i32> } { <16 x i32> splat (i32 -1), <16 x i32> splat (i32 -1), <16 x i32> splat (i32 -1) }, <16 x i32> [[_MSPROP_SELECT]], 0
+; CHECK-NEXT:    [[RES3:%.*]] = insertvalue { <16 x i32>, <16 x i32>, <16 x i32> } poison, <16 x i32> [[TMP13]], 0
+; CHECK-NEXT:    [[TMP28:%.*]] = insertvalue { <16 x i32>, <16 x i32>, <16 x i32> } [[TMP27]], <16 x i32> [[_MSPROP_SELECT1]], 1
+; CHECK-NEXT:    [[RES4:%.*]] = insertvalue { <16 x i32>, <16 x i32>, <16 x i32> } [[RES3]], <16 x i32> [[TMP23]], 1
+; CHECK-NEXT:    [[TMP29:%.*]] = insertvalue { <16 x i32>, <16 x i32>, <16 x i32> } [[TMP28]], <16 x i32> [[TMP25]], 2
+; CHECK-NEXT:    [[RES5:%.*]] = insertvalue { <16 x i32>, <16 x i32>, <16 x i32> } [[RES4]], <16 x i32> [[TMP26]], 2
+; CHECK-NEXT:    store { <16 x i32>, <16 x i32>, <16 x i32> } [[TMP29]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret { <16 x i32>, <16 x i32>, <16 x i32> } [[RES5]]
+;
+  %1 = call <16 x i32> @llvm.x86.avx512.prol.d.512(<16 x i32> %x0, i32 3)
+  %2 = bitcast i16 %x3 to <16 x i1>
+  %3 = select <16 x i1> %2, <16 x i32> %1, <16 x i32> %x2
+  %4 = call <16 x i32> @llvm.x86.avx512.prol.d.512(<16 x i32> %x0, i32 4)
+  %5 = bitcast i16 %x3 to <16 x i1>
+  %6 = select <16 x i1> %5, <16 x i32> %4, <16 x i32> zeroinitializer
+  %7 = call <16 x i32> @llvm.x86.avx512.prol.d.512(<16 x i32> %x0, i32 5)
+  %res3 = insertvalue { <16 x i32>, <16 x i32>, <16 x i32> } poison, <16 x i32> %3, 0
+  %res4 = insertvalue { <16 x i32>, <16 x i32>, <16 x i32> }  %res3, <16 x i32> %6, 1
+  %res5 = insertvalue { <16 x i32>, <16 x i32>, <16 x i32> }  %res4, <16 x i32> %7, 2
+  ret { <16 x i32>, <16 x i32>, <16 x i32> } %res5
+}
+
+declare <8 x i64> @llvm.x86.avx512.prol.q.512(<8 x i64>, i32)
+
+define { <8 x i64>, <8 x i64>, <8 x i64> } @test_int_x86_avx512_prol_q_512(<8 x i64> %x0, i32 %x1, <8 x i64> %x2, i8 %x3)  #0 {
+;
+; CHECK-LABEL: @test_int_x86_avx512_prol_q_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 136) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 72) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = call <8 x i64> @llvm.fshl.v8i64(<8 x i64> [[TMP1]], <8 x i64> [[TMP1]], <8 x i64> splat (i64 3))
+; CHECK-NEXT:    [[TMP5:%.*]] = or <8 x i64> [[TMP4]], zeroinitializer
+; CHECK-NEXT:    [[TMP6:%.*]] = call <8 x i64> @llvm.fshl.v8i64(<8 x i64> [[X0:%.*]], <8 x i64> [[X0]], <8 x i64> splat (i64 3))
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast i8 [[TMP2]] to <8 x i1>
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast i8 [[X3:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP9:%.*]] = select <8 x i1> [[TMP8]], <8 x i64> [[TMP5]], <8 x i64> [[TMP3]]
+; CHECK-NEXT:    [[TMP10:%.*]] = xor <8 x i64> [[TMP6]], [[X2:%.*]]
+; CHECK-NEXT:    [[TMP11:%.*]] = or <8 x i64> [[TMP10]], [[TMP5]]
+; CHECK-NEXT:    [[TMP12:%.*]] = or <8 x i64> [[TMP11]], [[TMP3]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP7]], <8 x i64> [[TMP12]], <8 x i64> [[TMP9]]
+; CHECK-NEXT:    [[TMP13:%.*]] = select <8 x i1> [[TMP8]], <8 x i64> [[TMP6]], <8 x i64> [[X2]]
+; CHECK-NEXT:    [[TMP14:%.*]] = call <8 x i64> @llvm.fshl.v8i64(<8 x i64> [[TMP1]], <8 x i64> [[TMP1]], <8 x i64> splat (i64 4))
+; CHECK-NEXT:    [[TMP15:%.*]] = or <8 x i64> [[TMP14]], zeroinitializer
+; CHECK-NEXT:    [[TMP16:%.*]] = call <8 x i64> @llvm.fshl.v8i64(<8 x i64> [[X0]], <8 x i64> [[X0]], <8 x i64> splat (i64 4))
+; CHECK-NEXT:    [[TMP17:%.*]] = bitcast i8 [[TMP2]] to <8 x i1>
+; CHECK-NEXT:    [[TMP18:%.*]] = bitcast i8 [[X3]] to <8 x i1>
+; CHECK-NEXT:    [[TMP19:%.*]] = select <8 x i1> [[TMP18]], <8 x i64> [[TMP15]], <8 x i64> zeroinitializer
+; CHECK-NEXT:    [[TMP20:%.*]] = xor <8 x i64> [[TMP16]], zeroinitializer
+; CHECK-NEXT:    [[TMP21:%.*]] = or <8 x i64> [[TMP20]], [[TMP15]]
+; CHECK-NEXT:    [[TMP22:%.*]] = or <8 x i64> [[TMP21]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT1:%.*]] = select <8 x i1> [[TMP17]], <8 x i64> [[TMP22]], <8 x i64> [[TMP19]]
+; CHECK-NEXT:    [[TMP23:%.*]] = select <8 x i1> [[TMP18]], <8 x i64> [[TMP16]], <8 x i64> zeroinitializer
+; CHECK-NEXT:    [[TMP24:%.*]] = call <8 x i64> @llvm.fshl.v8i64(<8 x i64> [[TMP1]], <8 x i64> [[TMP1]], <8 x i64> splat (i64 5))
+; CHECK-NEXT:    [[TMP25:%.*]] = or <8 x i64> [[TMP24]], zeroinitializer
+; CHECK-NEXT:    [[TMP26:%.*]] = call <8 x i64> @llvm.fshl.v8i64(<8 x i64> [[X0]], <8 x i64> [[X0]], <8 x i64> splat (i64 5))
+; CHECK-NEXT:    [[TMP27:%.*]] = insertvalue { <8 x i64>, <8 x i64>, <8 x i64> } { <8 x i64> splat (i64 -1), <8 x i64> splat (i64 -1), <8 x i64> splat (i64 -1) }, <8 x i64> [[_MSPROP_SELECT]], 0
+; CHECK-NEXT:    [[RES3:%.*]] = insertvalue { <8 x i64>, <8 x i64>, <8 x i64> } poison, <8 x i64> [[TMP13]], 0
+; CHECK-NEXT:    [[TMP28:%.*]] = insertvalue { <8 x i64>, <8 x i64>, <8 x i64> } [[TMP27]], <8 x i64> [[_MSPROP_SELECT1]], 1
+; CHECK-NEXT:    [[RES4:%.*]] = insertvalue { <8 x i64>, <8 x i64>, <8 x i64> } [[RES3]], <8 x i64> [[TMP23]], 1
+; CHECK-NEXT:    [[TMP29:%.*]] = insertvalue { <8 x i64>, <8 x i64>, <8 x i64> } [[TMP28]], <8 x i64> [[TMP25]], 2
+; CHECK-NEXT:    [[RES5:%.*]] = insertvalue { <8 x i64>, <8 x i64>, <8 x i64> } [[RES4]], <8 x i64> [[TMP26]], 2
+; CHECK-NEXT:    store { <8 x i64>, <8 x i64>, <8 x i64> } [[TMP29]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret { <8 x i64>, <8 x i64>, <8 x i64> } [[RES5]]
+;
+  %1 = call <8 x i64> @llvm.x86.avx512.prol.q.512(<8 x i64> %x0, i32 3)
+  %2 = bitcast i8 %x3 to <8 x i1>
+  %3 = select <8 x i1> %2, <8 x i64> %1, <8 x i64> %x2
+  %4 = call <8 x i64> @llvm.x86.avx512.prol.q.512(<8 x i64> %x0, i32 4)
+  %5 = bitcast i8 %x3 to <8 x i1>
+  %6 = select <8 x i1> %5, <8 x i64> %4, <8 x i64> zeroinitializer
+  %7 = call <8 x i64> @llvm.x86.avx512.prol.q.512(<8 x i64> %x0, i32 5)
+  %res3 = insertvalue { <8 x i64>, <8 x i64>, <8 x i64> } poison, <8 x i64> %3, 0
+  %res4 = insertvalue { <8 x i64>, <8 x i64>, <8 x i64> }  %res3, <8 x i64> %6, 1
+  %res5 = insertvalue { <8 x i64>, <8 x i64>, <8 x i64> }  %res4, <8 x i64> %7, 2
+  ret { <8 x i64>, <8 x i64>, <8 x i64> } %res5
+}
+
+declare <16 x i32> @llvm.x86.avx512.pror.d.512(<16 x i32>, i32)
+
+define { <16 x i32>, <16 x i32>, <16 x i32> } @test_int_x86_avx512_pror_d_512(<16 x i32> %x0, i32 %x1, <16 x i32> %x2, i16 %x3)  #0 {
+;
+; CHECK-LABEL: @test_int_x86_avx512_pror_d_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 136) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 72) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = call <16 x i32> @llvm.fshr.v16i32(<16 x i32> [[TMP1]], <16 x i32> [[TMP1]], <16 x i32> splat (i32 3))
+; CHECK-NEXT:    [[TMP5:%.*]] = or <16 x i32> [[TMP4]], zeroinitializer
+; CHECK-NEXT:    [[TMP6:%.*]] = call <16 x i32> @llvm.fshr.v16i32(<16 x i32> [[X0:%.*]], <16 x i32> [[X0]], <16 x i32> splat (i32 3))
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast i16 [[TMP2]] to <16 x i1>
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast i16 [[X3:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP9:%.*]] = select <16 x i1> [[TMP8]], <16 x i32> [[TMP5]], <16 x i32> [[TMP3]]
+; CHECK-NEXT:    [[TMP10:%.*]] = xor <16 x i32> [[TMP6]], [[X2:%.*]]
+; CHECK-NEXT:    [[TMP11:%.*]] = or <16 x i32> [[TMP10]], [[TMP5]]
+; CHECK-NEXT:    [[TMP12:%.*]] = or <16 x i32> [[TMP11]], [[TMP3]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP7]], <16 x i32> [[TMP12]], <16 x i32> [[TMP9]]
+; CHECK-NEXT:    [[TMP13:%.*]] = select <16 x i1> [[TMP8]], <16 x i32> [[TMP6]], <16 x i32> [[X2]]
+; CHECK-NEXT:    [[TMP14:%.*]] = call <16 x i32> @llvm.fshr.v16i32(<16 x i32> [[TMP1]], <16 x i32> [[TMP1]], <16 x i32> splat (i32 4))
+; CHECK-NEXT:    [[TMP15:%.*]] = or <16 x i32> [[TMP14]], zeroinitializer
+; CHECK-NEXT:    [[TMP16:%.*]] = call <16 x i32> @llvm.fshr.v16i32(<16 x i32> [[X0]], <16 x i32> [[X0]], <16 x i32> splat (i32 4))
+; CHECK-NEXT:    [[TMP17:%.*]] = bitcast i16 [[TMP2]] to <16 x i1>
+; CHECK-NEXT:    [[TMP18:%.*]] = bitcast i16 [[X3]] to <16 x i1>
+; CHECK-NEXT:    [[TMP19:%.*]] = select <16 x i1> [[TMP18]], <16 x i32> [[TMP15]], <16 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP20:%.*]] = xor <16 x i32> [[TMP16]], zeroinitializer
+; CHECK-NEXT:    [[TMP21:%.*]] = or <16 x i32> [[TMP20]], [[TMP15]]
+; CHECK-NEXT:    [[TMP22:%.*]] = or <16 x i32> [[TMP21]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT1:%.*]] = select <16 x i1> [[TMP17]], <16 x i32> [[TMP22]], <16 x i32> [[TMP19]]
+; CHECK-NEXT:    [[TMP23:%.*]] = select <16 x i1> [[TMP18]], <16 x i32> [[TMP16]], <16 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP24:%.*]] = call <16 x i32> @llvm.fshr.v16i32(<16 x i32> [[TMP1]], <16 x i32> [[TMP1]], <16 x i32> splat (i32 5))
+; CHECK-NEXT:    [[TMP25:%.*]] = or <16 x i32> [[TMP24]], zeroinitializer
+; CHECK-NEXT:    [[TMP26:%.*]] = call <16 x i32> @llvm.fshr.v16i32(<16 x i32> [[X0]], <16 x i32> [[X0]], <16 x i32> splat (i32 5))
+; CHECK-NEXT:    [[TMP27:%.*]] = insertvalue { <16 x i32>, <16 x i32>, <16 x i32> } { <16 x i32> splat (i32 -1), <16 x i32> splat (i32 -1), <16 x i32> splat (i32 -1) }, <16 x i32> [[_MSPROP_SELECT]], 0
+; CHECK-NEXT:    [[RES3:%.*]] = insertvalue { <16 x i32>, <16 x i32>, <16 x i32> } poison, <16 x i32> [[TMP13]], 0
+; CHECK-NEXT:    [[TMP28:%.*]] = insertvalue { <16 x i32>, <16 x i32>, <16 x i32> } [[TMP27]], <16 x i32> [[_MSPROP_SELECT1]], 1
+; CHECK-NEXT:    [[RES4:%.*]] = insertvalue { <16 x i32>, <16 x i32>, <16 x i32> } [[RES3]], <16 x i32> [[TMP23]], 1
+; CHECK-NEXT:    [[TMP29:%.*]] = insertvalue { <16 x i32>, <16 x i32>, <16 x i32> } [[TMP28]], <16 x i32> [[TMP25]], 2
+; CHECK-NEXT:    [[RES5:%.*]] = insertvalue { <16 x i32>, <16 x i32>, <16 x i32> } [[RES4]], <16 x i32> [[TMP26]], 2
+; CHECK-NEXT:    store { <16 x i32>, <16 x i32>, <16 x i32> } [[TMP29]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret { <16 x i32>, <16 x i32>, <16 x i32> } [[RES5]]
+;
+  %1 = call <16 x i32> @llvm.x86.avx512.pror.d.512(<16 x i32> %x0, i32 3)
+  %2 = bitcast i16 %x3 to <16 x i1>
+  %3 = select <16 x i1> %2, <16 x i32> %1, <16 x i32> %x2
+  %4 = call <16 x i32> @llvm.x86.avx512.pror.d.512(<16 x i32> %x0, i32 4)
+  %5 = bitcast i16 %x3 to <16 x i1>
+  %6 = select <16 x i1> %5, <16 x i32> %4, <16 x i32> zeroinitializer
+  %7 = call <16 x i32> @llvm.x86.avx512.pror.d.512(<16 x i32> %x0, i32 5)
+  %res3 = insertvalue { <16 x i32>, <16 x i32>, <16 x i32> } poison, <16 x i32> %3, 0
+  %res4 = insertvalue { <16 x i32>, <16 x i32>, <16 x i32> }  %res3, <16 x i32> %6, 1
+  %res5 = insertvalue { <16 x i32>, <16 x i32>, <16 x i32> }  %res4, <16 x i32> %7, 2
+  ret { <16 x i32>, <16 x i32>, <16 x i32> } %res5
+}
+
+declare <8 x i64> @llvm.x86.avx512.pror.q.512(<8 x i64>, i32)
+
+define { <8 x i64>, <8 x i64>, <8 x i64> } @test_int_x86_avx512_pror_q_512(<8 x i64> %x0, i32 %x1, <8 x i64> %x2, i8 %x3)  #0 {
+;
+; CHECK-LABEL: @test_int_x86_avx512_pror_q_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 136) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 72) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = call <8 x i64> @llvm.fshr.v8i64(<8 x i64> [[TMP1]], <8 x i64> [[TMP1]], <8 x i64> splat (i64 3))
+; CHECK-NEXT:    [[TMP5:%.*]] = or <8 x i64> [[TMP4]], zeroinitializer
+; CHECK-NEXT:    [[TMP6:%.*]] = call <8 x i64> @llvm.fshr.v8i64(<8 x i64> [[X0:%.*]], <8 x i64> [[X0]], <8 x i64> splat (i64 3))
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast i8 [[TMP2]] to <8 x i1>
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast i8 [[X3:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP9:%.*]] = select <8 x i1> [[TMP8]], <8 x i64> [[TMP5]], <8 x i64> [[TMP3]]
+; CHECK-NEXT:    [[TMP10:%.*]] = xor <8 x i64> [[TMP6]], [[X2:%.*]]
+; CHECK-NEXT:    [[TMP11:%.*]] = or <8 x i64> [[TMP10]], [[TMP5]]
+; CHECK-NEXT:    [[TMP12:%.*]] = or <8 x i64> [[TMP11]], [[TMP3]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP7]], <8 x i64> [[TMP12]], <8 x i64> [[TMP9]]
+; CHECK-NEXT:    [[TMP13:%.*]] = select <8 x i1> [[TMP8]], <8 x i64> [[TMP6]], <8 x i64> [[X2]]
+; CHECK-NEXT:    [[TMP14:%.*]] = call <8 x i64> @llvm.fshr.v8i64(<8 x i64> [[TMP1]], <8 x i64> [[TMP1]], <8 x i64> splat (i64 4))
+; CHECK-NEXT:    [[TMP15:%.*]] = or <8 x i64> [[TMP14]], zeroinitializer
+; CHECK-NEXT:    [[TMP16:%.*]] = call <8 x i64> @llvm.fshr.v8i64(<8 x i64> [[X0]], <8 x i64> [[X0]], <8 x i64> splat (i64 4))
+; CHECK-NEXT:    [[TMP17:%.*]] = bitcast i8 [[TMP2]] to <8 x i1>
+; CHECK-NEXT:    [[TMP18:%.*]] = bitcast i8 [[X3]] to <8 x i1>
+; CHECK-NEXT:    [[TMP19:%.*]] = select <8 x i1> [[TMP18]], <8 x i64> [[TMP15]], <8 x i64> zeroinitializer
+; CHECK-NEXT:    [[TMP20:%.*]] = xor <8 x i64> [[TMP16]], zeroinitializer
+; CHECK-NEXT:    [[TMP21:%.*]] = or <8 x i64> [[TMP20]], [[TMP15]]
+; CHECK-NEXT:    [[TMP22:%.*]] = or <8 x i64> [[TMP21]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT1:%.*]] = select <8 x i1> [[TMP17]], <8 x i64> [[TMP22]], <8 x i64> [[TMP19]]
+; CHECK-NEXT:    [[TMP23:%.*]] = select <8 x i1> [[TMP18]], <8 x i64> [[TMP16]], <8 x i64> zeroinitializer
+; CHECK-NEXT:    [[TMP24:%.*]] = call <8 x i64> @llvm.fshr.v8i64(<8 x i64> [[TMP1]], <8 x i64> [[TMP1]], <8 x i64> splat (i64 5))
+; CHECK-NEXT:    [[TMP25:%.*]] = or <8 x i64> [[TMP24]], zeroinitializer
+; CHECK-NEXT:    [[TMP26:%.*]] = call <8 x i64> @llvm.fshr.v8i64(<8 x i64> [[X0]], <8 x i64> [[X0]], <8 x i64> splat (i64 5))
+; CHECK-NEXT:    [[TMP27:%.*]] = insertvalue { <8 x i64>, <8 x i64>, <8 x i64> } { <8 x i64> splat (i64 -1), <8 x i64> splat (i64 -1), <8 x i64> splat (i64 -1) }, <8 x i64> [[_MSPROP_SELECT]], 0
+; CHECK-NEXT:    [[RES3:%.*]] = insertvalue { <8 x i64>, <8 x i64>, <8 x i64> } poison, <8 x i64> [[TMP13]], 0
+; CHECK-NEXT:    [[TMP28:%.*]] = insertvalue { <8 x i64>, <8 x i64>, <8 x i64> } [[TMP27]], <8 x i64> [[_MSPROP_SELECT1]], 1
+; CHECK-NEXT:    [[RES4:%.*]] = insertvalue { <8 x i64>, <8 x i64>, <8 x i64> } [[RES3]], <8 x i64> [[TMP23]], 1
+; CHECK-NEXT:    [[TMP29:%.*]] = insertvalue { <8 x i64>, <8 x i64>, <8 x i64> } [[TMP28]], <8 x i64> [[TMP25]], 2
+; CHECK-NEXT:    [[RES5:%.*]] = insertvalue { <8 x i64>, <8 x i64>, <8 x i64> } [[RES4]], <8 x i64> [[TMP26]], 2
+; CHECK-NEXT:    store { <8 x i64>, <8 x i64>, <8 x i64> } [[TMP29]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret { <8 x i64>, <8 x i64>, <8 x i64> } [[RES5]]
+;
+  %1 = call <8 x i64> @llvm.x86.avx512.pror.q.512(<8 x i64> %x0, i32 3)
+  %2 = bitcast i8 %x3 to <8 x i1>
+  %3 = select <8 x i1> %2, <8 x i64> %1, <8 x i64> %x2
+  %4 = call <8 x i64> @llvm.x86.avx512.pror.q.512(<8 x i64> %x0, i32 4)
+  %5 = bitcast i8 %x3 to <8 x i1>
+  %6 = select <8 x i1> %5, <8 x i64> %4, <8 x i64> zeroinitializer
+  %7 = call <8 x i64> @llvm.x86.avx512.pror.q.512(<8 x i64> %x0, i32 5)
+  %res3 = insertvalue { <8 x i64>, <8 x i64>, <8 x i64> } poison, <8 x i64> %3, 0
+  %res4 = insertvalue { <8 x i64>, <8 x i64>, <8 x i64> }  %res3, <8 x i64> %6, 1
+  %res5 = insertvalue { <8 x i64>, <8 x i64>, <8 x i64> }  %res4, <8 x i64> %7, 2
+  ret { <8 x i64>, <8 x i64>, <8 x i64> } %res5
+}
+
+declare <8 x i64> @llvm.x86.avx512.mask.psrl.qi.512(<8 x i64>, i32, <8 x i64>, i8)
+
+define { <8 x i64>, <8 x i64>, <8 x i64> } @test_int_x86_avx512_mask_psrl_qi_512(<8 x i64> %x0, i32 %x1, <8 x i64> %x2, i8 %x3)  #0 {
+;
+; CHECK-LABEL: @test_int_x86_avx512_mask_psrl_qi_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 136) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 72) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = call <8 x i64> @llvm.x86.avx512.psrli.q.512(<8 x i64> [[TMP1]], i32 4)
+; CHECK-NEXT:    [[TMP5:%.*]] = or <8 x i64> [[TMP4]], zeroinitializer
+; CHECK-NEXT:    [[TMP6:%.*]] = call <8 x i64> @llvm.x86.avx512.psrli.q.512(<8 x i64> [[X0:%.*]], i32 4)
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast i8 [[TMP2]] to <8 x i1>
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast i8 [[X3:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP9:%.*]] = select <8 x i1> [[TMP8]], <8 x i64> [[TMP5]], <8 x i64> [[TMP3]]
+; CHECK-NEXT:    [[TMP10:%.*]] = xor <8 x i64> [[TMP6]], [[X2:%.*]]
+; CHECK-NEXT:    [[TMP11:%.*]] = or <8 x i64> [[TMP10]], [[TMP5]]
+; CHECK-NEXT:    [[TMP12:%.*]] = or <8 x i64> [[TMP11]], [[TMP3]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP7]], <8 x i64> [[TMP12]], <8 x i64> [[TMP9]]
+; CHECK-NEXT:    [[TMP13:%.*]] = select <8 x i1> [[TMP8]], <8 x i64> [[TMP6]], <8 x i64> [[X2]]
+; CHECK-NEXT:    [[TMP14:%.*]] = call <8 x i64> @llvm.x86.avx512.psrli.q.512(<8 x i64> [[TMP1]], i32 5)
+; CHECK-NEXT:    [[TMP15:%.*]] = or <8 x i64> [[TMP14]], zeroinitializer
+; CHECK-NEXT:    [[TMP16:%.*]] = call <8 x i64> @llvm.x86.avx512.psrli.q.512(<8 x i64> [[X0]], i32 5)
+; CHECK-NEXT:    [[TMP17:%.*]] = call <8 x i64> @llvm.x86.avx512.psrli.q.512(<8 x i64> [[TMP1]], i32 6)
+; CHECK-NEXT:    [[TMP18:%.*]] = or <8 x i64> [[TMP17]], zeroinitializer
+; CHECK-NEXT:    [[TMP19:%.*]] = call <8 x i64> @llvm.x86.avx512.psrli.q.512(<8 x i64> [[X0]], i32 6)
+; CHECK-NEXT:    [[TMP20:%.*]] = bitcast i8 [[TMP2]] to <8 x i1>
+; CHECK-NEXT:    [[TMP21:%.*]] = bitcast i8 [[X3]] to <8 x i1>
+; CHECK-NEXT:    [[TMP22:%.*]] = select <8 x i1> [[TMP21]], <8 x i64> [[TMP18]], <8 x i64> zeroinitializer
+; CHECK-NEXT:    [[TMP23:%.*]] = xor <8 x i64> [[TMP19]], zeroinitializer
+; CHECK-NEXT:    [[TMP24:%.*]] = or <8 x i64> [[TMP23]], [[TMP18]]
+; CHECK-NEXT:    [[TMP25:%.*]] = or <8 x i64> [[TMP24]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT1:%.*]] = select <8 x i1> [[TMP20]], <8 x i64> [[TMP25]], <8 x i64> [[TMP22]]
+; CHECK-NEXT:    [[TMP26:%.*]] = select <8 x i1> [[TMP21]], <8 x i64> [[TMP19]], <8 x i64> zeroinitializer
+; CHECK-NEXT:    [[TMP27:%.*]] = insertvalue { <8 x i64>, <8 x i64>, <8 x i64> } { <8 x i64> splat (i64 -1), <8 x i64> splat (i64 -1), <8 x i64> splat (i64 -1) }, <8 x i64> [[_MSPROP_SELECT]], 0
+; CHECK-NEXT:    [[RES3:%.*]] = insertvalue { <8 x i64>, <8 x i64>, <8 x i64> } poison, <8 x i64> [[TMP13]], 0
+; CHECK-NEXT:    [[TMP28:%.*]] = insertvalue { <8 x i64>, <8 x i64>, <8 x i64> } [[TMP27]], <8 x i64> [[TMP15]], 1
+; CHECK-NEXT:    [[RES4:%.*]] = insertvalue { <8 x i64>, <8 x i64>, <8 x i64> } [[RES3]], <8 x i64> [[TMP16]], 1
+; CHECK-NEXT:    [[TMP29:%.*]] = insertvalue { <8 x i64>, <8 x i64>, <8 x i64> } [[TMP28]], <8 x i64> [[_MSPROP_SELECT1]], 2
+; CHECK-NEXT:    [[RES5:%.*]] = insertvalue { <8 x i64>, <8 x i64>, <8 x i64> } [[RES4]], <8 x i64> [[TMP26]], 2
+; CHECK-NEXT:    store { <8 x i64>, <8 x i64>, <8 x i64> } [[TMP29]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret { <8 x i64>, <8 x i64>, <8 x i64> } [[RES5]]
+;
+  %res0 = call <8 x i64> @llvm.x86.avx512.mask.psrl.qi.512(<8 x i64> %x0, i32 4, <8 x i64> %x2, i8 %x3)
+  %res1 = call <8 x i64> @llvm.x86.avx512.mask.psrl.qi.512(<8 x i64> %x0, i32 5, <8 x i64> %x2, i8 -1)
+  %res2 = call <8 x i64> @llvm.x86.avx512.mask.psrl.qi.512(<8 x i64> %x0, i32 6, <8 x i64> zeroinitializer, i8 %x3)
+  %res3 = insertvalue { <8 x i64>, <8 x i64>, <8 x i64> } poison, <8 x i64> %res0, 0
+  %res4 = insertvalue { <8 x i64>, <8 x i64>, <8 x i64> }  %res3, <8 x i64> %res1, 1
+  %res5 = insertvalue { <8 x i64>, <8 x i64>, <8 x i64> }  %res4, <8 x i64> %res2, 2
+  ret { <8 x i64>, <8 x i64>, <8 x i64> } %res5
+}
+
+declare <16 x i32>@llvm.x86.avx512.mask.psrl.di.512(<16 x i32>, i32, <16 x i32>, i16)
+
+define { <16 x i32>, <16 x i32>, <16 x i32> } @test_int_x86_avx512_mask_psrl_di_512(<16 x i32> %x0, i32 %x1, <16 x i32> %x2, i16 %x3)  #0 {
+;
+; CHECK-LABEL: @test_int_x86_avx512_mask_psrl_di_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 136) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 72) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = call <16 x i32> @llvm.x86.avx512.psrli.d.512(<16 x i32> [[TMP1]], i32 4)
+; CHECK-NEXT:    [[TMP5:%.*]] = or <16 x i32> [[TMP4]], zeroinitializer
+; CHECK-NEXT:    [[TMP6:%.*]] = call <16 x i32> @llvm.x86.avx512.psrli.d.512(<16 x i32> [[X0:%.*]], i32 4)
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast i16 [[TMP2]] to <16 x i1>
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast i16 [[X3:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP9:%.*]] = select <16 x i1> [[TMP8]], <16 x i32> [[TMP5]], <16 x i32> [[TMP3]]
+; CHECK-NEXT:    [[TMP10:%.*]] = xor <16 x i32> [[TMP6]], [[X2:%.*]]
+; CHECK-NEXT:    [[TMP11:%.*]] = or <16 x i32> [[TMP10]], [[TMP5]]
+; CHECK-NEXT:    [[TMP12:%.*]] = or <16 x i32> [[TMP11]], [[TMP3]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP7]], <16 x i32> [[TMP12]], <16 x i32> [[TMP9]]
+; CHECK-NEXT:    [[TMP13:%.*]] = select <16 x i1> [[TMP8]], <16 x i32> [[TMP6]], <16 x i32> [[X2]]
+; CHECK-NEXT:    [[TMP14:%.*]] = call <16 x i32> @llvm.x86.avx512.psrli.d.512(<16 x i32> [[TMP1]], i32 5)
+; CHECK-NEXT:    [[TMP15:%.*]] = or <16 x i32> [[TMP14]], zeroinitializer
+; CHECK-NEXT:    [[TMP16:%.*]] = call <16 x i32> @llvm.x86.avx512.psrli.d.512(<16 x i32> [[X0]], i32 5)
+; CHECK-NEXT:    [[TMP17:%.*]] = call <16 x i32> @llvm.x86.avx512.psrli.d.512(<16 x i32> [[TMP1]], i32 6)
+; CHECK-NEXT:    [[TMP18:%.*]] = or <16 x i32> [[TMP17]], zeroinitializer
+; CHECK-NEXT:    [[TMP19:%.*]] = call <16 x i32> @llvm.x86.avx512.psrli.d.512(<16 x i32> [[X0]], i32 6)
+; CHECK-NEXT:    [[TMP20:%.*]] = bitcast i16 [[TMP2]] to <16 x i1>
+; CHECK-NEXT:    [[TMP21:%.*]] = bitcast i16 [[X3]] to <16 x i1>
+; CHECK-NEXT:    [[TMP22:%.*]] = select <16 x i1> [[TMP21]], <16 x i32> [[TMP18]], <16 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP23:%.*]] = xor <16 x i32> [[TMP19]], zeroinitializer
+; CHECK-NEXT:    [[TMP24:%.*]] = or <16 x i32> [[TMP23]], [[TMP18]]
+; CHECK-NEXT:    [[TMP25:%.*]] = or <16 x i32> [[TMP24]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT1:%.*]] = select <16 x i1> [[TMP20]], <16 x i32> [[TMP25]], <16 x i32> [[TMP22]]
+; CHECK-NEXT:    [[TMP26:%.*]] = select <16 x i1> [[TMP21]], <16 x i32> [[TMP19]], <16 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP27:%.*]] = insertvalue { <16 x i32>, <16 x i32>, <16 x i32> } { <16 x i32> splat (i32 -1), <16 x i32> splat (i32 -1), <16 x i32> splat (i32 -1) }, <16 x i32> [[_MSPROP_SELECT]], 0
+; CHECK-NEXT:    [[RES3:%.*]] = insertvalue { <16 x i32>, <16 x i32>, <16 x i32> } poison, <16 x i32> [[TMP13]], 0
+; CHECK-NEXT:    [[TMP28:%.*]] = insertvalue { <16 x i32>, <16 x i32>, <16 x i32> } [[TMP27]], <16 x i32> [[TMP15]], 1
+; CHECK-NEXT:    [[RES4:%.*]] = insertvalue { <16 x i32>, <16 x i32>, <16 x i32> } [[RES3]], <16 x i32> [[TMP16]], 1
+; CHECK-NEXT:    [[TMP29:%.*]] = insertvalue { <16 x i32>, <16 x i32>, <16 x i32> } [[TMP28]], <16 x i32> [[_MSPROP_SELECT1]], 2
+; CHECK-NEXT:    [[RES5:%.*]] = insertvalue { <16 x i32>, <16 x i32>, <16 x i32> } [[RES4]], <16 x i32> [[TMP26]], 2
+; CHECK-NEXT:    store { <16 x i32>, <16 x i32>, <16 x i32> } [[TMP29]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret { <16 x i32>, <16 x i32>, <16 x i32> } [[RES5]]
+;
+  %res0 = call <16 x i32> @llvm.x86.avx512.mask.psrl.di.512(<16 x i32> %x0, i32 4, <16 x i32> %x2, i16 %x3)
+  %res1 = call <16 x i32> @llvm.x86.avx512.mask.psrl.di.512(<16 x i32> %x0, i32 5, <16 x i32> %x2, i16 -1)
+  %res2 = call <16 x i32> @llvm.x86.avx512.mask.psrl.di.512(<16 x i32> %x0, i32 6, <16 x i32> zeroinitializer, i16 %x3)
+  %res3 = insertvalue { <16 x i32>, <16 x i32>, <16 x i32> } poison, <16 x i32> %res0, 0
+  %res4 = insertvalue { <16 x i32>, <16 x i32>, <16 x i32> }  %res3, <16 x i32> %res1, 1
+  %res5 = insertvalue { <16 x i32>, <16 x i32>, <16 x i32> }  %res4, <16 x i32> %res2, 2
+  ret { <16 x i32>, <16 x i32>, <16 x i32> } %res5
+}
+
+declare <16 x i32> @llvm.x86.avx512.mask.psra.di.512(<16 x i32>, i32, <16 x i32>, i16)
+
+define { <16 x i32>, <16 x i32>, <16 x i32> } @test_int_x86_avx512_mask_psra_di_512(<16 x i32> %x0, i32 %x1, <16 x i32> %x2, i16 %x3)  #0 {
+;
+; CHECK-LABEL: @test_int_x86_avx512_mask_psra_di_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 136) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 72) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = call <16 x i32> @llvm.x86.avx512.psrai.d.512(<16 x i32> [[TMP1]], i32 3)
+; CHECK-NEXT:    [[TMP5:%.*]] = or <16 x i32> [[TMP4]], zeroinitializer
+; CHECK-NEXT:    [[TMP6:%.*]] = call <16 x i32> @llvm.x86.avx512.psrai.d.512(<16 x i32> [[X0:%.*]], i32 3)
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast i16 [[TMP2]] to <16 x i1>
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast i16 [[X3:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP9:%.*]] = select <16 x i1> [[TMP8]], <16 x i32> [[TMP5]], <16 x i32> [[TMP3]]
+; CHECK-NEXT:    [[TMP10:%.*]] = xor <16 x i32> [[TMP6]], [[X2:%.*]]
+; CHECK-NEXT:    [[TMP11:%.*]] = or <16 x i32> [[TMP10]], [[TMP5]]
+; CHECK-NEXT:    [[TMP12:%.*]] = or <16 x i32> [[TMP11]], [[TMP3]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP7]], <16 x i32> [[TMP12]], <16 x i32> [[TMP9]]
+; CHECK-NEXT:    [[TMP13:%.*]] = select <16 x i1> [[TMP8]], <16 x i32> [[TMP6]], <16 x i32> [[X2]]
+; CHECK-NEXT:    [[TMP14:%.*]] = call <16 x i32> @llvm.x86.avx512.psrai.d.512(<16 x i32> [[TMP1]], i32 4)
+; CHECK-NEXT:    [[TMP15:%.*]] = or <16 x i32> [[TMP14]], zeroinitializer
+; CHECK-NEXT:    [[TMP16:%.*]] = call <16 x i32> @llvm.x86.avx512.psrai.d.512(<16 x i32> [[X0]], i32 4)
+; CHECK-NEXT:    [[TMP17:%.*]] = bitcast i16 [[TMP2]] to <16 x i1>
+; CHECK-NEXT:    [[TMP18:%.*]] = bitcast i16 [[X3]] to <16 x i1>
+; CHECK-NEXT:    [[TMP19:%.*]] = select <16 x i1> [[TMP18]], <16 x i32> [[TMP15]], <16 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP20:%.*]] = xor <16 x i32> [[TMP16]], zeroinitializer
+; CHECK-NEXT:    [[TMP21:%.*]] = or <16 x i32> [[TMP20]], [[TMP15]]
+; CHECK-NEXT:    [[TMP22:%.*]] = or <16 x i32> [[TMP21]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT1:%.*]] = select <16 x i1> [[TMP17]], <16 x i32> [[TMP22]], <16 x i32> [[TMP19]]
+; CHECK-NEXT:    [[TMP23:%.*]] = select <16 x i1> [[TMP18]], <16 x i32> [[TMP16]], <16 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP24:%.*]] = call <16 x i32> @llvm.x86.avx512.psrai.d.512(<16 x i32> [[TMP1]], i32 5)
+; CHECK-NEXT:    [[TMP25:%.*]] = or <16 x i32> [[TMP24]], zeroinitializer
+; CHECK-NEXT:    [[TMP26:%.*]] = call <16 x i32> @llvm.x86.avx512.psrai.d.512(<16 x i32> [[X0]], i32 5)
+; CHECK-NEXT:    [[TMP27:%.*]] = insertvalue { <16 x i32>, <16 x i32>, <16 x i32> } { <16 x i32> splat (i32 -1), <16 x i32> splat (i32 -1), <16 x i32> splat (i32 -1) }, <16 x i32> [[_MSPROP_SELECT]], 0
+; CHECK-NEXT:    [[RES3:%.*]] = insertvalue { <16 x i32>, <16 x i32>, <16 x i32> } poison, <16 x i32> [[TMP13]], 0
+; CHECK-NEXT:    [[TMP28:%.*]] = insertvalue { <16 x i32>, <16 x i32>, <16 x i32> } [[TMP27]], <16 x i32> [[_MSPROP_SELECT1]], 1
+; CHECK-NEXT:    [[RES4:%.*]] = insertvalue { <16 x i32>, <16 x i32>, <16 x i32> } [[RES3]], <16 x i32> [[TMP23]], 1
+; CHECK-NEXT:    [[TMP29:%.*]] = insertvalue { <16 x i32>, <16 x i32>, <16 x i32> } [[TMP28]], <16 x i32> [[TMP25]], 2
+; CHECK-NEXT:    [[RES5:%.*]] = insertvalue { <16 x i32>, <16 x i32>, <16 x i32> } [[RES4]], <16 x i32> [[TMP26]], 2
+; CHECK-NEXT:    store { <16 x i32>, <16 x i32>, <16 x i32> } [[TMP29]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret { <16 x i32>, <16 x i32>, <16 x i32> } [[RES5]]
+;
+  %res0 = call <16 x i32> @llvm.x86.avx512.mask.psra.di.512(<16 x i32> %x0, i32 3, <16 x i32> %x2, i16 %x3)
+  %res1 = call <16 x i32> @llvm.x86.avx512.mask.psra.di.512(<16 x i32> %x0, i32 4, <16 x i32> zeroinitializer, i16 %x3)
+  %res2 = call <16 x i32> @llvm.x86.avx512.mask.psra.di.512(<16 x i32> %x0, i32 5, <16 x i32> %x2, i16 -1)
+  %res3 = insertvalue { <16 x i32>, <16 x i32>, <16 x i32> } poison, <16 x i32> %res0, 0
+  %res4 = insertvalue { <16 x i32>, <16 x i32>, <16 x i32> }  %res3, <16 x i32> %res1, 1
+  %res5 = insertvalue { <16 x i32>, <16 x i32>, <16 x i32> }  %res4, <16 x i32> %res2, 2
+  ret { <16 x i32>, <16 x i32>, <16 x i32> } %res5
+}
+
+declare <8 x i64> @llvm.x86.avx512.mask.psra.qi.512(<8 x i64>, i32, <8 x i64>, i8)
+
+define { <8 x i64>, <8 x i64>, <8 x i64> } @test_int_x86_avx512_mask_psra_qi_512(<8 x i64> %x0, i32 %x1, <8 x i64> %x2, i8 %x3)  #0 {
+;
+; CHECK-LABEL: @test_int_x86_avx512_mask_psra_qi_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 136) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 72) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = call <8 x i64> @llvm.x86.avx512.psrai.q.512(<8 x i64> [[TMP1]], i32 3)
+; CHECK-NEXT:    [[TMP5:%.*]] = or <8 x i64> [[TMP4]], zeroinitializer
+; CHECK-NEXT:    [[TMP6:%.*]] = call <8 x i64> @llvm.x86.avx512.psrai.q.512(<8 x i64> [[X0:%.*]], i32 3)
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast i8 [[TMP2]] to <8 x i1>
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast i8 [[X3:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP9:%.*]] = select <8 x i1> [[TMP8]], <8 x i64> [[TMP5]], <8 x i64> [[TMP3]]
+; CHECK-NEXT:    [[TMP10:%.*]] = xor <8 x i64> [[TMP6]], [[X2:%.*]]
+; CHECK-NEXT:    [[TMP11:%.*]] = or <8 x i64> [[TMP10]], [[TMP5]]
+; CHECK-NEXT:    [[TMP12:%.*]] = or <8 x i64> [[TMP11]], [[TMP3]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP7]], <8 x i64> [[TMP12]], <8 x i64> [[TMP9]]
+; CHECK-NEXT:    [[TMP13:%.*]] = select <8 x i1> [[TMP8]], <8 x i64> [[TMP6]], <8 x i64> [[X2]]
+; CHECK-NEXT:    [[TMP14:%.*]] = call <8 x i64> @llvm.x86.avx512.psrai.q.512(<8 x i64> [[TMP1]], i32 4)
+; CHECK-NEXT:    [[TMP15:%.*]] = or <8 x i64> [[TMP14]], zeroinitializer
+; CHECK-NEXT:    [[TMP16:%.*]] = call <8 x i64> @llvm.x86.avx512.psrai.q.512(<8 x i64> [[X0]], i32 4)
+; CHECK-NEXT:    [[TMP17:%.*]] = bitcast i8 [[TMP2]] to <8 x i1>
+; CHECK-NEXT:    [[TMP18:%.*]] = bitcast i8 [[X3]] to <8 x i1>
+; CHECK-NEXT:    [[TMP19:%.*]] = select <8 x i1> [[TMP18]], <8 x i64> [[TMP15]], <8 x i64> zeroinitializer
+; CHECK-NEXT:    [[TMP20:%.*]] = xor <8 x i64> [[TMP16]], zeroinitializer
+; CHECK-NEXT:    [[TMP21:%.*]] = or <8 x i64> [[TMP20]], [[TMP15]]
+; CHECK-NEXT:    [[TMP22:%.*]] = or <8 x i64> [[TMP21]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT1:%.*]] = select <8 x i1> [[TMP17]], <8 x i64> [[TMP22]], <8 x i64> [[TMP19]]
+; CHECK-NEXT:    [[TMP23:%.*]] = select <8 x i1> [[TMP18]], <8 x i64> [[TMP16]], <8 x i64> zeroinitializer
+; CHECK-NEXT:    [[TMP24:%.*]] = call <8 x i64> @llvm.x86.avx512.psrai.q.512(<8 x i64> [[TMP1]], i32 5)
+; CHECK-NEXT:    [[TMP25:%.*]] = or <8 x i64> [[TMP24]], zeroinitializer
+; CHECK-NEXT:    [[TMP26:%.*]] = call <8 x i64> @llvm.x86.avx512.psrai.q.512(<8 x i64> [[X0]], i32 5)
+; CHECK-NEXT:    [[TMP27:%.*]] = insertvalue { <8 x i64>, <8 x i64>, <8 x i64> } { <8 x i64> splat (i64 -1), <8 x i64> splat (i64 -1), <8 x i64> splat (i64 -1) }, <8 x i64> [[_MSPROP_SELECT]], 0
+; CHECK-NEXT:    [[RES3:%.*]] = insertvalue { <8 x i64>, <8 x i64>, <8 x i64> } poison, <8 x i64> [[TMP13]], 0
+; CHECK-NEXT:    [[TMP28:%.*]] = insertvalue { <8 x i64>, <8 x i64>, <8 x i64> } [[TMP27]], <8 x i64> [[_MSPROP_SELECT1]], 1
+; CHECK-NEXT:    [[RES4:%.*]] = insertvalue { <8 x i64>, <8 x i64>, <8 x i64> } [[RES3]], <8 x i64> [[TMP23]], 1
+; CHECK-NEXT:    [[TMP29:%.*]] = insertvalue { <8 x i64>, <8 x i64>, <8 x i64> } [[TMP28]], <8 x i64> [[TMP25]], 2
+; CHECK-NEXT:    [[RES5:%.*]] = insertvalue { <8 x i64>, <8 x i64>, <8 x i64> } [[RES4]], <8 x i64> [[TMP26]], 2
+; CHECK-NEXT:    store { <8 x i64>, <8 x i64>, <8 x i64> } [[TMP29]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret { <8 x i64>, <8 x i64>, <8 x i64> } [[RES5]]
+;
+  %res0 = call <8 x i64> @llvm.x86.avx512.mask.psra.qi.512(<8 x i64> %x0, i32 3, <8 x i64> %x2, i8 %x3)
+  %res1 = call <8 x i64> @llvm.x86.avx512.mask.psra.qi.512(<8 x i64> %x0, i32 4, <8 x i64> zeroinitializer, i8 %x3)
+  %res2 = call <8 x i64> @llvm.x86.avx512.mask.psra.qi.512(<8 x i64> %x0, i32 5, <8 x i64> %x2, i8 -1)
+  %res3 = insertvalue { <8 x i64>, <8 x i64>, <8 x i64> } poison, <8 x i64> %res0, 0
+  %res4 = insertvalue { <8 x i64>, <8 x i64>, <8 x i64> }  %res3, <8 x i64> %res1, 1
+  %res5 = insertvalue { <8 x i64>, <8 x i64>, <8 x i64> }  %res4, <8 x i64> %res2, 2
+  ret { <8 x i64>, <8 x i64>, <8 x i64> } %res5
+}
+
+declare <16 x i32> @llvm.x86.avx512.mask.psll.di.512(<16 x i32>, i32, <16 x i32>, i16)
+
+define { <16 x i32>, <16 x i32>, <16 x i32> } @test_int_x86_avx512_mask_psll_di_512(<16 x i32> %x0, i32 %x1, <16 x i32> %x2, i16 %x3)  #0 {
+;
+; CHECK-LABEL: @test_int_x86_avx512_mask_psll_di_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 136) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 72) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = call <16 x i32> @llvm.x86.avx512.pslli.d.512(<16 x i32> [[TMP1]], i32 3)
+; CHECK-NEXT:    [[TMP5:%.*]] = or <16 x i32> [[TMP4]], zeroinitializer
+; CHECK-NEXT:    [[TMP6:%.*]] = call <16 x i32> @llvm.x86.avx512.pslli.d.512(<16 x i32> [[X0:%.*]], i32 3)
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast i16 [[TMP2]] to <16 x i1>
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast i16 [[X3:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP9:%.*]] = select <16 x i1> [[TMP8]], <16 x i32> [[TMP5]], <16 x i32> [[TMP3]]
+; CHECK-NEXT:    [[TMP10:%.*]] = xor <16 x i32> [[TMP6]], [[X2:%.*]]
+; CHECK-NEXT:    [[TMP11:%.*]] = or <16 x i32> [[TMP10]], [[TMP5]]
+; CHECK-NEXT:    [[TMP12:%.*]] = or <16 x i32> [[TMP11]], [[TMP3]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP7]], <16 x i32> [[TMP12]], <16 x i32> [[TMP9]]
+; CHECK-NEXT:    [[TMP13:%.*]] = select <16 x i1> [[TMP8]], <16 x i32> [[TMP6]], <16 x i32> [[X2]]
+; CHECK-NEXT:    [[TMP14:%.*]] = call <16 x i32> @llvm.x86.avx512.pslli.d.512(<16 x i32> [[TMP1]], i32 4)
+; CHECK-NEXT:    [[TMP15:%.*]] = or <16 x i32> [[TMP14]], zeroinitializer
+; CHECK-NEXT:    [[TMP16:%.*]] = call <16 x i32> @llvm.x86.avx512.pslli.d.512(<16 x i32> [[X0]], i32 4)
+; CHECK-NEXT:    [[TMP17:%.*]] = bitcast i16 [[TMP2]] to <16 x i1>
+; CHECK-NEXT:    [[TMP18:%.*]] = bitcast i16 [[X3]] to <16 x i1>
+; CHECK-NEXT:    [[TMP19:%.*]] = select <16 x i1> [[TMP18]], <16 x i32> [[TMP15]], <16 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP20:%.*]] = xor <16 x i32> [[TMP16]], zeroinitializer
+; CHECK-NEXT:    [[TMP21:%.*]] = or <16 x i32> [[TMP20]], [[TMP15]]
+; CHECK-NEXT:    [[TMP22:%.*]] = or <16 x i32> [[TMP21]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT1:%.*]] = select <16 x i1> [[TMP17]], <16 x i32> [[TMP22]], <16 x i32> [[TMP19]]
+; CHECK-NEXT:    [[TMP23:%.*]] = select <16 x i1> [[TMP18]], <16 x i32> [[TMP16]], <16 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP24:%.*]] = call <16 x i32> @llvm.x86.avx512.pslli.d.512(<16 x i32> [[TMP1]], i32 5)
+; CHECK-NEXT:    [[TMP25:%.*]] = or <16 x i32> [[TMP24]], zeroinitializer
+; CHECK-NEXT:    [[TMP26:%.*]] = call <16 x i32> @llvm.x86.avx512.pslli.d.512(<16 x i32> [[X0]], i32 5)
+; CHECK-NEXT:    [[TMP27:%.*]] = insertvalue { <16 x i32>, <16 x i32>, <16 x i32> } { <16 x i32> splat (i32 -1), <16 x i32> splat (i32 -1), <16 x i32> splat (i32 -1) }, <16 x i32> [[_MSPROP_SELECT]], 0
+; CHECK-NEXT:    [[RES3:%.*]] = insertvalue { <16 x i32>, <16 x i32>, <16 x i32> } poison, <16 x i32> [[TMP13]], 0
+; CHECK-NEXT:    [[TMP28:%.*]] = insertvalue { <16 x i32>, <16 x i32>, <16 x i32> } [[TMP27]], <16 x i32> [[_MSPROP_SELECT1]], 1
+; CHECK-NEXT:    [[RES4:%.*]] = insertvalue { <16 x i32>, <16 x i32>, <16 x i32> } [[RES3]], <16 x i32> [[TMP23]], 1
+; CHECK-NEXT:    [[TMP29:%.*]] = insertvalue { <16 x i32>, <16 x i32>, <16 x i32> } [[TMP28]], <16 x i32> [[TMP25]], 2
+; CHECK-NEXT:    [[RES5:%.*]] = insertvalue { <16 x i32>, <16 x i32>, <16 x i32> } [[RES4]], <16 x i32> [[TMP26]], 2
+; CHECK-NEXT:    store { <16 x i32>, <16 x i32>, <16 x i32> } [[TMP29]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret { <16 x i32>, <16 x i32>, <16 x i32> } [[RES5]]
+;
+  %res0 = call <16 x i32> @llvm.x86.avx512.mask.psll.di.512(<16 x i32> %x0, i32 3, <16 x i32> %x2, i16 %x3)
+  %res1 = call <16 x i32> @llvm.x86.avx512.mask.psll.di.512(<16 x i32> %x0, i32 4, <16 x i32> zeroinitializer, i16 %x3)
+  %res2 = call <16 x i32> @llvm.x86.avx512.mask.psll.di.512(<16 x i32> %x0, i32 5, <16 x i32> %x2, i16 -1)
+  %res3 = insertvalue { <16 x i32>, <16 x i32>, <16 x i32> } poison, <16 x i32> %res0, 0
+  %res4 = insertvalue { <16 x i32>, <16 x i32>, <16 x i32> }  %res3, <16 x i32> %res1, 1
+  %res5 = insertvalue { <16 x i32>, <16 x i32>, <16 x i32> }  %res4, <16 x i32> %res2, 2
+  ret { <16 x i32>, <16 x i32>, <16 x i32> } %res5
+}
+
+declare <8 x i64> @llvm.x86.avx512.mask.psll.qi.512(<8 x i64>, i32, <8 x i64>, i8)
+
+define { <8 x i64>, <8 x i64>, <8 x i64> } @test_int_x86_avx512_mask_psll_qi_512(<8 x i64> %x0, i32 %x1, <8 x i64> %x2, i8 %x3)  #0 {
+;
+; CHECK-LABEL: @test_int_x86_avx512_mask_psll_qi_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 136) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 72) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = call <8 x i64> @llvm.x86.avx512.pslli.q.512(<8 x i64> [[TMP1]], i32 3)
+; CHECK-NEXT:    [[TMP5:%.*]] = or <8 x i64> [[TMP4]], zeroinitializer
+; CHECK-NEXT:    [[TMP6:%.*]] = call <8 x i64> @llvm.x86.avx512.pslli.q.512(<8 x i64> [[X0:%.*]], i32 3)
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast i8 [[TMP2]] to <8 x i1>
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast i8 [[X3:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP9:%.*]] = select <8 x i1> [[TMP8]], <8 x i64> [[TMP5]], <8 x i64> [[TMP3]]
+; CHECK-NEXT:    [[TMP10:%.*]] = xor <8 x i64> [[TMP6]], [[X2:%.*]]
+; CHECK-NEXT:    [[TMP11:%.*]] = or <8 x i64> [[TMP10]], [[TMP5]]
+; CHECK-NEXT:    [[TMP12:%.*]] = or <8 x i64> [[TMP11]], [[TMP3]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP7]], <8 x i64> [[TMP12]], <8 x i64> [[TMP9]]
+; CHECK-NEXT:    [[TMP13:%.*]] = select <8 x i1> [[TMP8]], <8 x i64> [[TMP6]], <8 x i64> [[X2]]
+; CHECK-NEXT:    [[TMP14:%.*]] = call <8 x i64> @llvm.x86.avx512.pslli.q.512(<8 x i64> [[TMP1]], i32 4)
+; CHECK-NEXT:    [[TMP15:%.*]] = or <8 x i64> [[TMP14]], zeroinitializer
+; CHECK-NEXT:    [[TMP16:%.*]] = call <8 x i64> @llvm.x86.avx512.pslli.q.512(<8 x i64> [[X0]], i32 4)
+; CHECK-NEXT:    [[TMP17:%.*]] = bitcast i8 [[TMP2]] to <8 x i1>
+; CHECK-NEXT:    [[TMP18:%.*]] = bitcast i8 [[X3]] to <8 x i1>
+; CHECK-NEXT:    [[TMP19:%.*]] = select <8 x i1> [[TMP18]], <8 x i64> [[TMP15]], <8 x i64> zeroinitializer
+; CHECK-NEXT:    [[TMP20:%.*]] = xor <8 x i64> [[TMP16]], zeroinitializer
+; CHECK-NEXT:    [[TMP21:%.*]] = or <8 x i64> [[TMP20]], [[TMP15]]
+; CHECK-NEXT:    [[TMP22:%.*]] = or <8 x i64> [[TMP21]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT1:%.*]] = select <8 x i1> [[TMP17]], <8 x i64> [[TMP22]], <8 x i64> [[TMP19]]
+; CHECK-NEXT:    [[TMP23:%.*]] = select <8 x i1> [[TMP18]], <8 x i64> [[TMP16]], <8 x i64> zeroinitializer
+; CHECK-NEXT:    [[TMP24:%.*]] = call <8 x i64> @llvm.x86.avx512.pslli.q.512(<8 x i64> [[TMP1]], i32 5)
+; CHECK-NEXT:    [[TMP25:%.*]] = or <8 x i64> [[TMP24]], zeroinitializer
+; CHECK-NEXT:    [[TMP26:%.*]] = call <8 x i64> @llvm.x86.avx512.pslli.q.512(<8 x i64> [[X0]], i32 5)
+; CHECK-NEXT:    [[TMP27:%.*]] = insertvalue { <8 x i64>, <8 x i64>, <8 x i64> } { <8 x i64> splat (i64 -1), <8 x i64> splat (i64 -1), <8 x i64> splat (i64 -1) }, <8 x i64> [[_MSPROP_SELECT]], 0
+; CHECK-NEXT:    [[RES3:%.*]] = insertvalue { <8 x i64>, <8 x i64>, <8 x i64> } poison, <8 x i64> [[TMP13]], 0
+; CHECK-NEXT:    [[TMP28:%.*]] = insertvalue { <8 x i64>, <8 x i64>, <8 x i64> } [[TMP27]], <8 x i64> [[_MSPROP_SELECT1]], 1
+; CHECK-NEXT:    [[RES4:%.*]] = insertvalue { <8 x i64>, <8 x i64>, <8 x i64> } [[RES3]], <8 x i64> [[TMP23]], 1
+; CHECK-NEXT:    [[TMP29:%.*]] = insertvalue { <8 x i64>, <8 x i64>, <8 x i64> } [[TMP28]], <8 x i64> [[TMP25]], 2
+; CHECK-NEXT:    [[RES5:%.*]] = insertvalue { <8 x i64>, <8 x i64>, <8 x i64> } [[RES4]], <8 x i64> [[TMP26]], 2
+; CHECK-NEXT:    store { <8 x i64>, <8 x i64>, <8 x i64> } [[TMP29]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret { <8 x i64>, <8 x i64>, <8 x i64> } [[RES5]]
+;
+  %res0 = call <8 x i64> @llvm.x86.avx512.mask.psll.qi.512(<8 x i64> %x0, i32 3, <8 x i64> %x2, i8 %x3)
+  %res1 = call <8 x i64> @llvm.x86.avx512.mask.psll.qi.512(<8 x i64> %x0, i32 4, <8 x i64> zeroinitializer, i8 %x3)
+  %res2 = call <8 x i64> @llvm.x86.avx512.mask.psll.qi.512(<8 x i64> %x0, i32 5, <8 x i64> %x2, i8 -1)
+  %res3 = insertvalue { <8 x i64>, <8 x i64>, <8 x i64> } poison, <8 x i64> %res0, 0
+  %res4 = insertvalue { <8 x i64>, <8 x i64>, <8 x i64> }  %res3, <8 x i64> %res1, 1
+  %res5 = insertvalue { <8 x i64>, <8 x i64>, <8 x i64> }  %res4, <8 x i64> %res2, 2
+  ret { <8 x i64>, <8 x i64>, <8 x i64> } %res5
+}
+
+define <16 x i32> @test_x86_avx512_psll_d(<16 x i32> %a0, <4 x i32> %a1)  #0 {
+; CHECK-LABEL: @test_x86_avx512_psll_d(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <4 x i32> [[TMP2]] to i128
+; CHECK-NEXT:    [[TMP4:%.*]] = trunc i128 [[TMP3]] to i64
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp ne i64 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = sext i1 [[TMP5]] to i512
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast i512 [[TMP6]] to <16 x i32>
+; CHECK-NEXT:    [[TMP8:%.*]] = call <16 x i32> @llvm.x86.avx512.psll.d.512(<16 x i32> [[TMP1]], <4 x i32> [[A1:%.*]])
+; CHECK-NEXT:    [[TMP9:%.*]] = or <16 x i32> [[TMP8]], [[TMP7]]
+; CHECK-NEXT:    [[TMP10:%.*]] = call <16 x i32> @llvm.x86.avx512.psll.d.512(<16 x i32> [[A0:%.*]], <4 x i32> [[A1]])
+; CHECK-NEXT:    store <16 x i32> [[TMP9]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i32> [[TMP10]]
+;
+  %res = call <16 x i32> @llvm.x86.avx512.mask.psll.d(<16 x i32> %a0, <4 x i32> %a1, <16 x i32> zeroinitializer, i16 -1)
+  ret <16 x i32> %res
+}
+
+define <16 x i32> @test_x86_avx512_mask_psll_d(<16 x i32> %a0, <4 x i32> %a1, <16 x i32> %a2, i16 %mask)  #0 {
+;
+; CHECK-LABEL: @test_x86_avx512_mask_psll_d(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 144) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 80) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <4 x i32> [[TMP2]] to i128
+; CHECK-NEXT:    [[TMP6:%.*]] = trunc i128 [[TMP5]] to i64
+; CHECK-NEXT:    [[TMP7:%.*]] = icmp ne i64 [[TMP6]], 0
+; CHECK-NEXT:    [[TMP8:%.*]] = sext i1 [[TMP7]] to i512
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast i512 [[TMP8]] to <16 x i32>
+; CHECK-NEXT:    [[TMP10:%.*]] = call <16 x i32> @llvm.x86.avx512.psll.d.512(<16 x i32> [[TMP1]], <4 x i32> [[A1:%.*]])
+; CHECK-NEXT:    [[TMP11:%.*]] = or <16 x i32> [[TMP10]], [[TMP9]]
+; CHECK-NEXT:    [[TMP12:%.*]] = call <16 x i32> @llvm.x86.avx512.psll.d.512(<16 x i32> [[A0:%.*]], <4 x i32> [[A1]])
+; CHECK-NEXT:    [[TMP13:%.*]] = bitcast i16 [[TMP3]] to <16 x i1>
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast i16 [[MASK:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP15:%.*]] = select <16 x i1> [[TMP14]], <16 x i32> [[TMP11]], <16 x i32> [[TMP4]]
+; CHECK-NEXT:    [[TMP16:%.*]] = xor <16 x i32> [[TMP12]], [[A2:%.*]]
+; CHECK-NEXT:    [[TMP17:%.*]] = or <16 x i32> [[TMP16]], [[TMP11]]
+; CHECK-NEXT:    [[TMP18:%.*]] = or <16 x i32> [[TMP17]], [[TMP4]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP13]], <16 x i32> [[TMP18]], <16 x i32> [[TMP15]]
+; CHECK-NEXT:    [[TMP19:%.*]] = select <16 x i1> [[TMP14]], <16 x i32> [[TMP12]], <16 x i32> [[A2]]
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i32> [[TMP19]]
+;
+  %res = call <16 x i32> @llvm.x86.avx512.mask.psll.d(<16 x i32> %a0, <4 x i32> %a1, <16 x i32> %a2, i16 %mask)
+  ret <16 x i32> %res
+}
+
+define <16 x i32> @test_x86_avx512_maskz_psll_d(<16 x i32> %a0, <4 x i32> %a1, i16 %mask)  #0 {
+;
+; CHECK-LABEL: @test_x86_avx512_maskz_psll_d(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 80) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <4 x i32> [[TMP2]] to i128
+; CHECK-NEXT:    [[TMP5:%.*]] = trunc i128 [[TMP4]] to i64
+; CHECK-NEXT:    [[TMP6:%.*]] = icmp ne i64 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP7:%.*]] = sext i1 [[TMP6]] to i512
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast i512 [[TMP7]] to <16 x i32>
+; CHECK-NEXT:    [[TMP9:%.*]] = call <16 x i32> @llvm.x86.avx512.psll.d.512(<16 x i32> [[TMP1]], <4 x i32> [[A1:%.*]])
+; CHECK-NEXT:    [[TMP10:%.*]] = or <16 x i32> [[TMP9]], [[TMP8]]
+; CHECK-NEXT:    [[TMP11:%.*]] = call <16 x i32> @llvm.x86.avx512.psll.d.512(<16 x i32> [[A0:%.*]], <4 x i32> [[A1]])
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast i16 [[TMP3]] to <16 x i1>
+; CHECK-NEXT:    [[TMP13:%.*]] = bitcast i16 [[MASK:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP14:%.*]] = select <16 x i1> [[TMP13]], <16 x i32> [[TMP10]], <16 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP15:%.*]] = xor <16 x i32> [[TMP11]], zeroinitializer
+; CHECK-NEXT:    [[TMP16:%.*]] = or <16 x i32> [[TMP15]], [[TMP10]]
+; CHECK-NEXT:    [[TMP17:%.*]] = or <16 x i32> [[TMP16]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP12]], <16 x i32> [[TMP17]], <16 x i32> [[TMP14]]
+; CHECK-NEXT:    [[TMP18:%.*]] = select <16 x i1> [[TMP13]], <16 x i32> [[TMP11]], <16 x i32> zeroinitializer
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i32> [[TMP18]]
+;
+  %res = call <16 x i32> @llvm.x86.avx512.mask.psll.d(<16 x i32> %a0, <4 x i32> %a1, <16 x i32> zeroinitializer, i16 %mask)
+  ret <16 x i32> %res
+}
+
+declare <16 x i32> @llvm.x86.avx512.mask.psll.d(<16 x i32>, <4 x i32>, <16 x i32>, i16) nounwind readnone
+
+define <8 x i64> @test_x86_avx512_psll_q(<8 x i64> %a0, <2 x i64> %a1)  #0 {
+; CHECK-LABEL: @test_x86_avx512_psll_q(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <2 x i64> [[TMP2]] to i128
+; CHECK-NEXT:    [[TMP4:%.*]] = trunc i128 [[TMP3]] to i64
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp ne i64 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = sext i1 [[TMP5]] to i512
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast i512 [[TMP6]] to <8 x i64>
+; CHECK-NEXT:    [[TMP8:%.*]] = call <8 x i64> @llvm.x86.avx512.psll.q.512(<8 x i64> [[TMP1]], <2 x i64> [[A1:%.*]])
+; CHECK-NEXT:    [[TMP9:%.*]] = or <8 x i64> [[TMP8]], [[TMP7]]
+; CHECK-NEXT:    [[TMP10:%.*]] = call <8 x i64> @llvm.x86.avx512.psll.q.512(<8 x i64> [[A0:%.*]], <2 x i64> [[A1]])
+; CHECK-NEXT:    store <8 x i64> [[TMP9]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i64> [[TMP10]]
+;
+  %res = call <8 x i64> @llvm.x86.avx512.mask.psll.q(<8 x i64> %a0, <2 x i64> %a1, <8 x i64> zeroinitializer, i8 -1)
+  ret <8 x i64> %res
+}
+
+define <8 x i64> @test_x86_avx512_mask_psll_q(<8 x i64> %a0, <2 x i64> %a1, <8 x i64> %a2, i8 %mask)  #0 {
+;
+; CHECK-LABEL: @test_x86_avx512_mask_psll_q(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 144) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 80) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <2 x i64> [[TMP2]] to i128
+; CHECK-NEXT:    [[TMP6:%.*]] = trunc i128 [[TMP5]] to i64
+; CHECK-NEXT:    [[TMP7:%.*]] = icmp ne i64 [[TMP6]], 0
+; CHECK-NEXT:    [[TMP8:%.*]] = sext i1 [[TMP7]] to i512
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast i512 [[TMP8]] to <8 x i64>
+; CHECK-NEXT:    [[TMP10:%.*]] = call <8 x i64> @llvm.x86.avx512.psll.q.512(<8 x i64> [[TMP1]], <2 x i64> [[A1:%.*]])
+; CHECK-NEXT:    [[TMP11:%.*]] = or <8 x i64> [[TMP10]], [[TMP9]]
+; CHECK-NEXT:    [[TMP12:%.*]] = call <8 x i64> @llvm.x86.avx512.psll.q.512(<8 x i64> [[A0:%.*]], <2 x i64> [[A1]])
+; CHECK-NEXT:    [[TMP13:%.*]] = bitcast i8 [[TMP3]] to <8 x i1>
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP15:%.*]] = select <8 x i1> [[TMP14]], <8 x i64> [[TMP11]], <8 x i64> [[TMP4]]
+; CHECK-NEXT:    [[TMP16:%.*]] = xor <8 x i64> [[TMP12]], [[A2:%.*]]
+; CHECK-NEXT:    [[TMP17:%.*]] = or <8 x i64> [[TMP16]], [[TMP11]]
+; CHECK-NEXT:    [[TMP18:%.*]] = or <8 x i64> [[TMP17]], [[TMP4]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP13]], <8 x i64> [[TMP18]], <8 x i64> [[TMP15]]
+; CHECK-NEXT:    [[TMP19:%.*]] = select <8 x i1> [[TMP14]], <8 x i64> [[TMP12]], <8 x i64> [[A2]]
+; CHECK-NEXT:    store <8 x i64> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i64> [[TMP19]]
+;
+  %res = call <8 x i64> @llvm.x86.avx512.mask.psll.q(<8 x i64> %a0, <2 x i64> %a1, <8 x i64> %a2, i8 %mask)
+  ret <8 x i64> %res
+}
+
+define <8 x i64> @test_x86_avx512_maskz_psll_q(<8 x i64> %a0, <2 x i64> %a1, i8 %mask)  #0 {
+;
+; CHECK-LABEL: @test_x86_avx512_maskz_psll_q(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 80) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <2 x i64> [[TMP2]] to i128
+; CHECK-NEXT:    [[TMP5:%.*]] = trunc i128 [[TMP4]] to i64
+; CHECK-NEXT:    [[TMP6:%.*]] = icmp ne i64 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP7:%.*]] = sext i1 [[TMP6]] to i512
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast i512 [[TMP7]] to <8 x i64>
+; CHECK-NEXT:    [[TMP9:%.*]] = call <8 x i64> @llvm.x86.avx512.psll.q.512(<8 x i64> [[TMP1]], <2 x i64> [[A1:%.*]])
+; CHECK-NEXT:    [[TMP10:%.*]] = or <8 x i64> [[TMP9]], [[TMP8]]
+; CHECK-NEXT:    [[TMP11:%.*]] = call <8 x i64> @llvm.x86.avx512.psll.q.512(<8 x i64> [[A0:%.*]], <2 x i64> [[A1]])
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast i8 [[TMP3]] to <8 x i1>
+; CHECK-NEXT:    [[TMP13:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP14:%.*]] = select <8 x i1> [[TMP13]], <8 x i64> [[TMP10]], <8 x i64> zeroinitializer
+; CHECK-NEXT:    [[TMP15:%.*]] = xor <8 x i64> [[TMP11]], zeroinitializer
+; CHECK-NEXT:    [[TMP16:%.*]] = or <8 x i64> [[TMP15]], [[TMP10]]
+; CHECK-NEXT:    [[TMP17:%.*]] = or <8 x i64> [[TMP16]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP12]], <8 x i64> [[TMP17]], <8 x i64> [[TMP14]]
+; CHECK-NEXT:    [[TMP18:%.*]] = select <8 x i1> [[TMP13]], <8 x i64> [[TMP11]], <8 x i64> zeroinitializer
+; CHECK-NEXT:    store <8 x i64> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i64> [[TMP18]]
+;
+  %res = call <8 x i64> @llvm.x86.avx512.mask.psll.q(<8 x i64> %a0, <2 x i64> %a1, <8 x i64> zeroinitializer, i8 %mask)
+  ret <8 x i64> %res
+}
+
+declare <8 x i64> @llvm.x86.avx512.mask.psll.q(<8 x i64>, <2 x i64>, <8 x i64>, i8) nounwind readnone
+
+define <16 x i32> @test_x86_avx512_psrl_d(<16 x i32> %a0, <4 x i32> %a1)  #0 {
+; CHECK-LABEL: @test_x86_avx512_psrl_d(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <4 x i32> [[TMP2]] to i128
+; CHECK-NEXT:    [[TMP4:%.*]] = trunc i128 [[TMP3]] to i64
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp ne i64 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = sext i1 [[TMP5]] to i512
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast i512 [[TMP6]] to <16 x i32>
+; CHECK-NEXT:    [[TMP8:%.*]] = call <16 x i32> @llvm.x86.avx512.psrl.d.512(<16 x i32> [[TMP1]], <4 x i32> [[A1:%.*]])
+; CHECK-NEXT:    [[TMP9:%.*]] = or <16 x i32> [[TMP8]], [[TMP7]]
+; CHECK-NEXT:    [[TMP10:%.*]] = call <16 x i32> @llvm.x86.avx512.psrl.d.512(<16 x i32> [[A0:%.*]], <4 x i32> [[A1]])
+; CHECK-NEXT:    store <16 x i32> [[TMP9]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i32> [[TMP10]]
+;
+  %res = call <16 x i32> @llvm.x86.avx512.mask.psrl.d(<16 x i32> %a0, <4 x i32> %a1, <16 x i32> zeroinitializer, i16 -1)
+  ret <16 x i32> %res
+}
+
+define <16 x i32> @test_x86_avx512_mask_psrl_d(<16 x i32> %a0, <4 x i32> %a1, <16 x i32> %a2, i16 %mask)  #0 {
+;
+; CHECK-LABEL: @test_x86_avx512_mask_psrl_d(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 144) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 80) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <4 x i32> [[TMP2]] to i128
+; CHECK-NEXT:    [[TMP6:%.*]] = trunc i128 [[TMP5]] to i64
+; CHECK-NEXT:    [[TMP7:%.*]] = icmp ne i64 [[TMP6]], 0
+; CHECK-NEXT:    [[TMP8:%.*]] = sext i1 [[TMP7]] to i512
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast i512 [[TMP8]] to <16 x i32>
+; CHECK-NEXT:    [[TMP10:%.*]] = call <16 x i32> @llvm.x86.avx512.psrl.d.512(<16 x i32> [[TMP1]], <4 x i32> [[A1:%.*]])
+; CHECK-NEXT:    [[TMP11:%.*]] = or <16 x i32> [[TMP10]], [[TMP9]]
+; CHECK-NEXT:    [[TMP12:%.*]] = call <16 x i32> @llvm.x86.avx512.psrl.d.512(<16 x i32> [[A0:%.*]], <4 x i32> [[A1]])
+; CHECK-NEXT:    [[TMP13:%.*]] = bitcast i16 [[TMP3]] to <16 x i1>
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast i16 [[MASK:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP15:%.*]] = select <16 x i1> [[TMP14]], <16 x i32> [[TMP11]], <16 x i32> [[TMP4]]
+; CHECK-NEXT:    [[TMP16:%.*]] = xor <16 x i32> [[TMP12]], [[A2:%.*]]
+; CHECK-NEXT:    [[TMP17:%.*]] = or <16 x i32> [[TMP16]], [[TMP11]]
+; CHECK-NEXT:    [[TMP18:%.*]] = or <16 x i32> [[TMP17]], [[TMP4]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP13]], <16 x i32> [[TMP18]], <16 x i32> [[TMP15]]
+; CHECK-NEXT:    [[TMP19:%.*]] = select <16 x i1> [[TMP14]], <16 x i32> [[TMP12]], <16 x i32> [[A2]]
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i32> [[TMP19]]
+;
+  %res = call <16 x i32> @llvm.x86.avx512.mask.psrl.d(<16 x i32> %a0, <4 x i32> %a1, <16 x i32> %a2, i16 %mask)
+  ret <16 x i32> %res
+}
+
+define <16 x i32> @test_x86_avx512_maskz_psrl_d(<16 x i32> %a0, <4 x i32> %a1, i16 %mask)  #0 {
+;
+; CHECK-LABEL: @test_x86_avx512_maskz_psrl_d(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 80) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <4 x i32> [[TMP2]] to i128
+; CHECK-NEXT:    [[TMP5:%.*]] = trunc i128 [[TMP4]] to i64
+; CHECK-NEXT:    [[TMP6:%.*]] = icmp ne i64 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP7:%.*]] = sext i1 [[TMP6]] to i512
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast i512 [[TMP7]] to <16 x i32>
+; CHECK-NEXT:    [[TMP9:%.*]] = call <16 x i32> @llvm.x86.avx512.psrl.d.512(<16 x i32> [[TMP1]], <4 x i32> [[A1:%.*]])
+; CHECK-NEXT:    [[TMP10:%.*]] = or <16 x i32> [[TMP9]], [[TMP8]]
+; CHECK-NEXT:    [[TMP11:%.*]] = call <16 x i32> @llvm.x86.avx512.psrl.d.512(<16 x i32> [[A0:%.*]], <4 x i32> [[A1]])
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast i16 [[TMP3]] to <16 x i1>
+; CHECK-NEXT:    [[TMP13:%.*]] = bitcast i16 [[MASK:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP14:%.*]] = select <16 x i1> [[TMP13]], <16 x i32> [[TMP10]], <16 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP15:%.*]] = xor <16 x i32> [[TMP11]], zeroinitializer
+; CHECK-NEXT:    [[TMP16:%.*]] = or <16 x i32> [[TMP15]], [[TMP10]]
+; CHECK-NEXT:    [[TMP17:%.*]] = or <16 x i32> [[TMP16]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP12]], <16 x i32> [[TMP17]], <16 x i32> [[TMP14]]
+; CHECK-NEXT:    [[TMP18:%.*]] = select <16 x i1> [[TMP13]], <16 x i32> [[TMP11]], <16 x i32> zeroinitializer
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i32> [[TMP18]]
+;
+  %res = call <16 x i32> @llvm.x86.avx512.mask.psrl.d(<16 x i32> %a0, <4 x i32> %a1, <16 x i32> zeroinitializer, i16 %mask)
+  ret <16 x i32> %res
+}
+
+declare <16 x i32> @llvm.x86.avx512.mask.psrl.d(<16 x i32>, <4 x i32>, <16 x i32>, i16) nounwind readnone
+
+define <8 x i64> @test_x86_avx512_psrl_q(<8 x i64> %a0, <2 x i64> %a1)  #0 {
+; CHECK-LABEL: @test_x86_avx512_psrl_q(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <2 x i64> [[TMP2]] to i128
+; CHECK-NEXT:    [[TMP4:%.*]] = trunc i128 [[TMP3]] to i64
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp ne i64 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = sext i1 [[TMP5]] to i512
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast i512 [[TMP6]] to <8 x i64>
+; CHECK-NEXT:    [[TMP8:%.*]] = call <8 x i64> @llvm.x86.avx512.psrl.q.512(<8 x i64> [[TMP1]], <2 x i64> [[A1:%.*]])
+; CHECK-NEXT:    [[TMP9:%.*]] = or <8 x i64> [[TMP8]], [[TMP7]]
+; CHECK-NEXT:    [[TMP10:%.*]] = call <8 x i64> @llvm.x86.avx512.psrl.q.512(<8 x i64> [[A0:%.*]], <2 x i64> [[A1]])
+; CHECK-NEXT:    store <8 x i64> [[TMP9]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i64> [[TMP10]]
+;
+  %res = call <8 x i64> @llvm.x86.avx512.mask.psrl.q(<8 x i64> %a0, <2 x i64> %a1, <8 x i64> zeroinitializer, i8 -1)
+  ret <8 x i64> %res
+}
+
+define <8 x i64> @test_x86_avx512_mask_psrl_q(<8 x i64> %a0, <2 x i64> %a1, <8 x i64> %a2, i8 %mask)  #0 {
+;
+; CHECK-LABEL: @test_x86_avx512_mask_psrl_q(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 144) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 80) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <2 x i64> [[TMP2]] to i128
+; CHECK-NEXT:    [[TMP6:%.*]] = trunc i128 [[TMP5]] to i64
+; CHECK-NEXT:    [[TMP7:%.*]] = icmp ne i64 [[TMP6]], 0
+; CHECK-NEXT:    [[TMP8:%.*]] = sext i1 [[TMP7]] to i512
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast i512 [[TMP8]] to <8 x i64>
+; CHECK-NEXT:    [[TMP10:%.*]] = call <8 x i64> @llvm.x86.avx512.psrl.q.512(<8 x i64> [[TMP1]], <2 x i64> [[A1:%.*]])
+; CHECK-NEXT:    [[TMP11:%.*]] = or <8 x i64> [[TMP10]], [[TMP9]]
+; CHECK-NEXT:    [[TMP12:%.*]] = call <8 x i64> @llvm.x86.avx512.psrl.q.512(<8 x i64> [[A0:%.*]], <2 x i64> [[A1]])
+; CHECK-NEXT:    [[TMP13:%.*]] = bitcast i8 [[TMP3]] to <8 x i1>
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP15:%.*]] = select <8 x i1> [[TMP14]], <8 x i64> [[TMP11]], <8 x i64> [[TMP4]]
+; CHECK-NEXT:    [[TMP16:%.*]] = xor <8 x i64> [[TMP12]], [[A2:%.*]]
+; CHECK-NEXT:    [[TMP17:%.*]] = or <8 x i64> [[TMP16]], [[TMP11]]
+; CHECK-NEXT:    [[TMP18:%.*]] = or <8 x i64> [[TMP17]], [[TMP4]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP13]], <8 x i64> [[TMP18]], <8 x i64> [[TMP15]]
+; CHECK-NEXT:    [[TMP19:%.*]] = select <8 x i1> [[TMP14]], <8 x i64> [[TMP12]], <8 x i64> [[A2]]
+; CHECK-NEXT:    store <8 x i64> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i64> [[TMP19]]
+;
+  %res = call <8 x i64> @llvm.x86.avx512.mask.psrl.q(<8 x i64> %a0, <2 x i64> %a1, <8 x i64> %a2, i8 %mask)
+  ret <8 x i64> %res
+}
+
+define <8 x i64> @test_x86_avx512_maskz_psrl_q(<8 x i64> %a0, <2 x i64> %a1, i8 %mask)  #0 {
+;
+; CHECK-LABEL: @test_x86_avx512_maskz_psrl_q(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 80) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <2 x i64> [[TMP2]] to i128
+; CHECK-NEXT:    [[TMP5:%.*]] = trunc i128 [[TMP4]] to i64
+; CHECK-NEXT:    [[TMP6:%.*]] = icmp ne i64 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP7:%.*]] = sext i1 [[TMP6]] to i512
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast i512 [[TMP7]] to <8 x i64>
+; CHECK-NEXT:    [[TMP9:%.*]] = call <8 x i64> @llvm.x86.avx512.psrl.q.512(<8 x i64> [[TMP1]], <2 x i64> [[A1:%.*]])
+; CHECK-NEXT:    [[TMP10:%.*]] = or <8 x i64> [[TMP9]], [[TMP8]]
+; CHECK-NEXT:    [[TMP11:%.*]] = call <8 x i64> @llvm.x86.avx512.psrl.q.512(<8 x i64> [[A0:%.*]], <2 x i64> [[A1]])
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast i8 [[TMP3]] to <8 x i1>
+; CHECK-NEXT:    [[TMP13:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP14:%.*]] = select <8 x i1> [[TMP13]], <8 x i64> [[TMP10]], <8 x i64> zeroinitializer
+; CHECK-NEXT:    [[TMP15:%.*]] = xor <8 x i64> [[TMP11]], zeroinitializer
+; CHECK-NEXT:    [[TMP16:%.*]] = or <8 x i64> [[TMP15]], [[TMP10]]
+; CHECK-NEXT:    [[TMP17:%.*]] = or <8 x i64> [[TMP16]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP12]], <8 x i64> [[TMP17]], <8 x i64> [[TMP14]]
+; CHECK-NEXT:    [[TMP18:%.*]] = select <8 x i1> [[TMP13]], <8 x i64> [[TMP11]], <8 x i64> zeroinitializer
+; CHECK-NEXT:    store <8 x i64> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i64> [[TMP18]]
+;
+  %res = call <8 x i64> @llvm.x86.avx512.mask.psrl.q(<8 x i64> %a0, <2 x i64> %a1, <8 x i64> zeroinitializer, i8 %mask)
+  ret <8 x i64> %res
+}
+
+declare <8 x i64> @llvm.x86.avx512.mask.psrl.q(<8 x i64>, <2 x i64>, <8 x i64>, i8) nounwind readnone
+
+define <16 x i32> @test_x86_avx512_psra_d(<16 x i32> %a0, <4 x i32> %a1)  #0 {
+; CHECK-LABEL: @test_x86_avx512_psra_d(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <4 x i32> [[TMP2]] to i128
+; CHECK-NEXT:    [[TMP4:%.*]] = trunc i128 [[TMP3]] to i64
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp ne i64 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = sext i1 [[TMP5]] to i512
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast i512 [[TMP6]] to <16 x i32>
+; CHECK-NEXT:    [[TMP8:%.*]] = call <16 x i32> @llvm.x86.avx512.psra.d.512(<16 x i32> [[TMP1]], <4 x i32> [[A1:%.*]])
+; CHECK-NEXT:    [[TMP9:%.*]] = or <16 x i32> [[TMP8]], [[TMP7]]
+; CHECK-NEXT:    [[TMP10:%.*]] = call <16 x i32> @llvm.x86.avx512.psra.d.512(<16 x i32> [[A0:%.*]], <4 x i32> [[A1]])
+; CHECK-NEXT:    store <16 x i32> [[TMP9]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i32> [[TMP10]]
+;
+  %res = call <16 x i32> @llvm.x86.avx512.mask.psra.d(<16 x i32> %a0, <4 x i32> %a1, <16 x i32> zeroinitializer, i16 -1)
+  ret <16 x i32> %res
+}
+
+define <16 x i32> @test_x86_avx512_mask_psra_d(<16 x i32> %a0, <4 x i32> %a1, <16 x i32> %a2, i16 %mask)  #0 {
+;
+; CHECK-LABEL: @test_x86_avx512_mask_psra_d(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 144) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 80) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <4 x i32> [[TMP2]] to i128
+; CHECK-NEXT:    [[TMP6:%.*]] = trunc i128 [[TMP5]] to i64
+; CHECK-NEXT:    [[TMP7:%.*]] = icmp ne i64 [[TMP6]], 0
+; CHECK-NEXT:    [[TMP8:%.*]] = sext i1 [[TMP7]] to i512
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast i512 [[TMP8]] to <16 x i32>
+; CHECK-NEXT:    [[TMP10:%.*]] = call <16 x i32> @llvm.x86.avx512.psra.d.512(<16 x i32> [[TMP1]], <4 x i32> [[A1:%.*]])
+; CHECK-NEXT:    [[TMP11:%.*]] = or <16 x i32> [[TMP10]], [[TMP9]]
+; CHECK-NEXT:    [[TMP12:%.*]] = call <16 x i32> @llvm.x86.avx512.psra.d.512(<16 x i32> [[A0:%.*]], <4 x i32> [[A1]])
+; CHECK-NEXT:    [[TMP13:%.*]] = bitcast i16 [[TMP3]] to <16 x i1>
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast i16 [[MASK:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP15:%.*]] = select <16 x i1> [[TMP14]], <16 x i32> [[TMP11]], <16 x i32> [[TMP4]]
+; CHECK-NEXT:    [[TMP16:%.*]] = xor <16 x i32> [[TMP12]], [[A2:%.*]]
+; CHECK-NEXT:    [[TMP17:%.*]] = or <16 x i32> [[TMP16]], [[TMP11]]
+; CHECK-NEXT:    [[TMP18:%.*]] = or <16 x i32> [[TMP17]], [[TMP4]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP13]], <16 x i32> [[TMP18]], <16 x i32> [[TMP15]]
+; CHECK-NEXT:    [[TMP19:%.*]] = select <16 x i1> [[TMP14]], <16 x i32> [[TMP12]], <16 x i32> [[A2]]
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i32> [[TMP19]]
+;
+  %res = call <16 x i32> @llvm.x86.avx512.mask.psra.d(<16 x i32> %a0, <4 x i32> %a1, <16 x i32> %a2, i16 %mask)
+  ret <16 x i32> %res
+}
+
+define <16 x i32> @test_x86_avx512_maskz_psra_d(<16 x i32> %a0, <4 x i32> %a1, i16 %mask)  #0 {
+;
+; CHECK-LABEL: @test_x86_avx512_maskz_psra_d(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 80) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <4 x i32> [[TMP2]] to i128
+; CHECK-NEXT:    [[TMP5:%.*]] = trunc i128 [[TMP4]] to i64
+; CHECK-NEXT:    [[TMP6:%.*]] = icmp ne i64 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP7:%.*]] = sext i1 [[TMP6]] to i512
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast i512 [[TMP7]] to <16 x i32>
+; CHECK-NEXT:    [[TMP9:%.*]] = call <16 x i32> @llvm.x86.avx512.psra.d.512(<16 x i32> [[TMP1]], <4 x i32> [[A1:%.*]])
+; CHECK-NEXT:    [[TMP10:%.*]] = or <16 x i32> [[TMP9]], [[TMP8]]
+; CHECK-NEXT:    [[TMP11:%.*]] = call <16 x i32> @llvm.x86.avx512.psra.d.512(<16 x i32> [[A0:%.*]], <4 x i32> [[A1]])
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast i16 [[TMP3]] to <16 x i1>
+; CHECK-NEXT:    [[TMP13:%.*]] = bitcast i16 [[MASK:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP14:%.*]] = select <16 x i1> [[TMP13]], <16 x i32> [[TMP10]], <16 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP15:%.*]] = xor <16 x i32> [[TMP11]], zeroinitializer
+; CHECK-NEXT:    [[TMP16:%.*]] = or <16 x i32> [[TMP15]], [[TMP10]]
+; CHECK-NEXT:    [[TMP17:%.*]] = or <16 x i32> [[TMP16]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP12]], <16 x i32> [[TMP17]], <16 x i32> [[TMP14]]
+; CHECK-NEXT:    [[TMP18:%.*]] = select <16 x i1> [[TMP13]], <16 x i32> [[TMP11]], <16 x i32> zeroinitializer
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i32> [[TMP18]]
+;
+  %res = call <16 x i32> @llvm.x86.avx512.mask.psra.d(<16 x i32> %a0, <4 x i32> %a1, <16 x i32> zeroinitializer, i16 %mask)
+  ret <16 x i32> %res
+}
+
+declare <16 x i32> @llvm.x86.avx512.mask.psra.d(<16 x i32>, <4 x i32>, <16 x i32>, i16) nounwind readnone
+
+define <8 x i64> @test_x86_avx512_psra_q(<8 x i64> %a0, <2 x i64> %a1)  #0 {
+; CHECK-LABEL: @test_x86_avx512_psra_q(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <2 x i64> [[TMP2]] to i128
+; CHECK-NEXT:    [[TMP4:%.*]] = trunc i128 [[TMP3]] to i64
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp ne i64 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = sext i1 [[TMP5]] to i512
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast i512 [[TMP6]] to <8 x i64>
+; CHECK-NEXT:    [[TMP8:%.*]] = call <8 x i64> @llvm.x86.avx512.psra.q.512(<8 x i64> [[TMP1]], <2 x i64> [[A1:%.*]])
+; CHECK-NEXT:    [[TMP9:%.*]] = or <8 x i64> [[TMP8]], [[TMP7]]
+; CHECK-NEXT:    [[TMP10:%.*]] = call <8 x i64> @llvm.x86.avx512.psra.q.512(<8 x i64> [[A0:%.*]], <2 x i64> [[A1]])
+; CHECK-NEXT:    store <8 x i64> [[TMP9]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i64> [[TMP10]]
+;
+  %res = call <8 x i64> @llvm.x86.avx512.mask.psra.q(<8 x i64> %a0, <2 x i64> %a1, <8 x i64> zeroinitializer, i8 -1)
+  ret <8 x i64> %res
+}
+
+define <8 x i64> @test_x86_avx512_mask_psra_q(<8 x i64> %a0, <2 x i64> %a1, <8 x i64> %a2, i8 %mask)  #0 {
+;
+; CHECK-LABEL: @test_x86_avx512_mask_psra_q(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 144) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 80) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <2 x i64> [[TMP2]] to i128
+; CHECK-NEXT:    [[TMP6:%.*]] = trunc i128 [[TMP5]] to i64
+; CHECK-NEXT:    [[TMP7:%.*]] = icmp ne i64 [[TMP6]], 0
+; CHECK-NEXT:    [[TMP8:%.*]] = sext i1 [[TMP7]] to i512
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast i512 [[TMP8]] to <8 x i64>
+; CHECK-NEXT:    [[TMP10:%.*]] = call <8 x i64> @llvm.x86.avx512.psra.q.512(<8 x i64> [[TMP1]], <2 x i64> [[A1:%.*]])
+; CHECK-NEXT:    [[TMP11:%.*]] = or <8 x i64> [[TMP10]], [[TMP9]]
+; CHECK-NEXT:    [[TMP12:%.*]] = call <8 x i64> @llvm.x86.avx512.psra.q.512(<8 x i64> [[A0:%.*]], <2 x i64> [[A1]])
+; CHECK-NEXT:    [[TMP13:%.*]] = bitcast i8 [[TMP3]] to <8 x i1>
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP15:%.*]] = select <8 x i1> [[TMP14]], <8 x i64> [[TMP11]], <8 x i64> [[TMP4]]
+; CHECK-NEXT:    [[TMP16:%.*]] = xor <8 x i64> [[TMP12]], [[A2:%.*]]
+; CHECK-NEXT:    [[TMP17:%.*]] = or <8 x i64> [[TMP16]], [[TMP11]]
+; CHECK-NEXT:    [[TMP18:%.*]] = or <8 x i64> [[TMP17]], [[TMP4]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP13]], <8 x i64> [[TMP18]], <8 x i64> [[TMP15]]
+; CHECK-NEXT:    [[TMP19:%.*]] = select <8 x i1> [[TMP14]], <8 x i64> [[TMP12]], <8 x i64> [[A2]]
+; CHECK-NEXT:    store <8 x i64> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i64> [[TMP19]]
+;
+  %res = call <8 x i64> @llvm.x86.avx512.mask.psra.q(<8 x i64> %a0, <2 x i64> %a1, <8 x i64> %a2, i8 %mask)
+  ret <8 x i64> %res
+}
+
+define <8 x i64> @test_x86_avx512_maskz_psra_q(<8 x i64> %a0, <2 x i64> %a1, i8 %mask)  #0 {
+;
+; CHECK-LABEL: @test_x86_avx512_maskz_psra_q(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 80) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <2 x i64> [[TMP2]] to i128
+; CHECK-NEXT:    [[TMP5:%.*]] = trunc i128 [[TMP4]] to i64
+; CHECK-NEXT:    [[TMP6:%.*]] = icmp ne i64 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP7:%.*]] = sext i1 [[TMP6]] to i512
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast i512 [[TMP7]] to <8 x i64>
+; CHECK-NEXT:    [[TMP9:%.*]] = call <8 x i64> @llvm.x86.avx512.psra.q.512(<8 x i64> [[TMP1]], <2 x i64> [[A1:%.*]])
+; CHECK-NEXT:    [[TMP10:%.*]] = or <8 x i64> [[TMP9]], [[TMP8]]
+; CHECK-NEXT:    [[TMP11:%.*]] = call <8 x i64> @llvm.x86.avx512.psra.q.512(<8 x i64> [[A0:%.*]], <2 x i64> [[A1]])
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast i8 [[TMP3]] to <8 x i1>
+; CHECK-NEXT:    [[TMP13:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP14:%.*]] = select <8 x i1> [[TMP13]], <8 x i64> [[TMP10]], <8 x i64> zeroinitializer
+; CHECK-NEXT:    [[TMP15:%.*]] = xor <8 x i64> [[TMP11]], zeroinitializer
+; CHECK-NEXT:    [[TMP16:%.*]] = or <8 x i64> [[TMP15]], [[TMP10]]
+; CHECK-NEXT:    [[TMP17:%.*]] = or <8 x i64> [[TMP16]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP12]], <8 x i64> [[TMP17]], <8 x i64> [[TMP14]]
+; CHECK-NEXT:    [[TMP18:%.*]] = select <8 x i1> [[TMP13]], <8 x i64> [[TMP11]], <8 x i64> zeroinitializer
+; CHECK-NEXT:    store <8 x i64> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i64> [[TMP18]]
+;
+  %res = call <8 x i64> @llvm.x86.avx512.mask.psra.q(<8 x i64> %a0, <2 x i64> %a1, <8 x i64> zeroinitializer, i8 %mask)
+  ret <8 x i64> %res
+}
+
+declare <8 x i64> @llvm.x86.avx512.mask.psra.q(<8 x i64>, <2 x i64>, <8 x i64>, i8) nounwind readnone
+
+define <16 x i32> @test_x86_avx512_psllv_d(<16 x i32> %a0, <16 x i32> %a1)  #0 {
+; CHECK-LABEL: @test_x86_avx512_psllv_d(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp ne <16 x i32> [[TMP2]], zeroinitializer
+; CHECK-NEXT:    [[TMP4:%.*]] = sext <16 x i1> [[TMP3]] to <16 x i32>
+; CHECK-NEXT:    [[TMP5:%.*]] = call <16 x i32> @llvm.x86.avx512.psllv.d.512(<16 x i32> [[TMP1]], <16 x i32> [[A1:%.*]])
+; CHECK-NEXT:    [[TMP6:%.*]] = or <16 x i32> [[TMP5]], [[TMP4]]
+; CHECK-NEXT:    [[TMP7:%.*]] = call <16 x i32> @llvm.x86.avx512.psllv.d.512(<16 x i32> [[A0:%.*]], <16 x i32> [[A1]])
+; CHECK-NEXT:    store <16 x i32> [[TMP6]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i32> [[TMP7]]
+;
+  %res = call <16 x i32> @llvm.x86.avx512.mask.psllv.d(<16 x i32> %a0, <16 x i32> %a1, <16 x i32> zeroinitializer, i16 -1)
+  ret <16 x i32> %res
+}
+
+define <16 x i32> @test_x86_avx512_mask_psllv_d(<16 x i32> %a0, <16 x i32> %a1, <16 x i32> %a2, i16 %mask)  #0 {
+;
+; CHECK-LABEL: @test_x86_avx512_mask_psllv_d(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp ne <16 x i32> [[TMP2]], zeroinitializer
+; CHECK-NEXT:    [[TMP6:%.*]] = sext <16 x i1> [[TMP5]] to <16 x i32>
+; CHECK-NEXT:    [[TMP7:%.*]] = call <16 x i32> @llvm.x86.avx512.psllv.d.512(<16 x i32> [[TMP1]], <16 x i32> [[A1:%.*]])
+; CHECK-NEXT:    [[TMP8:%.*]] = or <16 x i32> [[TMP7]], [[TMP6]]
+; CHECK-NEXT:    [[TMP9:%.*]] = call <16 x i32> @llvm.x86.avx512.psllv.d.512(<16 x i32> [[A0:%.*]], <16 x i32> [[A1]])
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i16 [[TMP3]] to <16 x i1>
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast i16 [[MASK:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP12:%.*]] = select <16 x i1> [[TMP11]], <16 x i32> [[TMP8]], <16 x i32> [[TMP4]]
+; CHECK-NEXT:    [[TMP13:%.*]] = xor <16 x i32> [[TMP9]], [[A2:%.*]]
+; CHECK-NEXT:    [[TMP14:%.*]] = or <16 x i32> [[TMP13]], [[TMP8]]
+; CHECK-NEXT:    [[TMP15:%.*]] = or <16 x i32> [[TMP14]], [[TMP4]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP10]], <16 x i32> [[TMP15]], <16 x i32> [[TMP12]]
+; CHECK-NEXT:    [[TMP16:%.*]] = select <16 x i1> [[TMP11]], <16 x i32> [[TMP9]], <16 x i32> [[A2]]
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i32> [[TMP16]]
+;
+  %res = call <16 x i32> @llvm.x86.avx512.mask.psllv.d(<16 x i32> %a0, <16 x i32> %a1, <16 x i32> %a2, i16 %mask)
+  ret <16 x i32> %res
+}
+
+define <16 x i32> @test_x86_avx512_maskz_psllv_d(<16 x i32> %a0, <16 x i32> %a1, i16 %mask)  #0 {
+;
+; CHECK-LABEL: @test_x86_avx512_maskz_psllv_d(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = icmp ne <16 x i32> [[TMP2]], zeroinitializer
+; CHECK-NEXT:    [[TMP5:%.*]] = sext <16 x i1> [[TMP4]] to <16 x i32>
+; CHECK-NEXT:    [[TMP6:%.*]] = call <16 x i32> @llvm.x86.avx512.psllv.d.512(<16 x i32> [[TMP1]], <16 x i32> [[A1:%.*]])
+; CHECK-NEXT:    [[TMP7:%.*]] = or <16 x i32> [[TMP6]], [[TMP5]]
+; CHECK-NEXT:    [[TMP8:%.*]] = call <16 x i32> @llvm.x86.avx512.psllv.d.512(<16 x i32> [[A0:%.*]], <16 x i32> [[A1]])
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast i16 [[TMP3]] to <16 x i1>
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i16 [[MASK:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP11:%.*]] = select <16 x i1> [[TMP10]], <16 x i32> [[TMP7]], <16 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP12:%.*]] = xor <16 x i32> [[TMP8]], zeroinitializer
+; CHECK-NEXT:    [[TMP13:%.*]] = or <16 x i32> [[TMP12]], [[TMP7]]
+; CHECK-NEXT:    [[TMP14:%.*]] = or <16 x i32> [[TMP13]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP9]], <16 x i32> [[TMP14]], <16 x i32> [[TMP11]]
+; CHECK-NEXT:    [[TMP15:%.*]] = select <16 x i1> [[TMP10]], <16 x i32> [[TMP8]], <16 x i32> zeroinitializer
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i32> [[TMP15]]
+;
+  %res = call <16 x i32> @llvm.x86.avx512.mask.psllv.d(<16 x i32> %a0, <16 x i32> %a1, <16 x i32> zeroinitializer, i16 %mask)
+  ret <16 x i32> %res
+}
+
+declare <16 x i32> @llvm.x86.avx512.mask.psllv.d(<16 x i32>, <16 x i32>, <16 x i32>, i16) nounwind readnone
+
+define <8 x i64> @test_x86_avx512_psllv_q(<8 x i64> %a0, <8 x i64> %a1)  #0 {
+; CHECK-LABEL: @test_x86_avx512_psllv_q(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp ne <8 x i64> [[TMP2]], zeroinitializer
+; CHECK-NEXT:    [[TMP4:%.*]] = sext <8 x i1> [[TMP3]] to <8 x i64>
+; CHECK-NEXT:    [[TMP5:%.*]] = call <8 x i64> @llvm.x86.avx512.psllv.q.512(<8 x i64> [[TMP1]], <8 x i64> [[A1:%.*]])
+; CHECK-NEXT:    [[TMP6:%.*]] = or <8 x i64> [[TMP5]], [[TMP4]]
+; CHECK-NEXT:    [[TMP7:%.*]] = call <8 x i64> @llvm.x86.avx512.psllv.q.512(<8 x i64> [[A0:%.*]], <8 x i64> [[A1]])
+; CHECK-NEXT:    store <8 x i64> [[TMP6]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i64> [[TMP7]]
+;
+  %res = call <8 x i64> @llvm.x86.avx512.mask.psllv.q(<8 x i64> %a0, <8 x i64> %a1, <8 x i64> zeroinitializer, i8 -1)
+  ret <8 x i64> %res
+}
+
+define <8 x i64> @test_x86_avx512_mask_psllv_q(<8 x i64> %a0, <8 x i64> %a1, <8 x i64> %a2, i8 %mask)  #0 {
+;
+; CHECK-LABEL: @test_x86_avx512_mask_psllv_q(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp ne <8 x i64> [[TMP2]], zeroinitializer
+; CHECK-NEXT:    [[TMP6:%.*]] = sext <8 x i1> [[TMP5]] to <8 x i64>
+; CHECK-NEXT:    [[TMP7:%.*]] = call <8 x i64> @llvm.x86.avx512.psllv.q.512(<8 x i64> [[TMP1]], <8 x i64> [[A1:%.*]])
+; CHECK-NEXT:    [[TMP8:%.*]] = or <8 x i64> [[TMP7]], [[TMP6]]
+; CHECK-NEXT:    [[TMP9:%.*]] = call <8 x i64> @llvm.x86.avx512.psllv.q.512(<8 x i64> [[A0:%.*]], <8 x i64> [[A1]])
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i8 [[TMP3]] to <8 x i1>
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP12:%.*]] = select <8 x i1> [[TMP11]], <8 x i64> [[TMP8]], <8 x i64> [[TMP4]]
+; CHECK-NEXT:    [[TMP13:%.*]] = xor <8 x i64> [[TMP9]], [[A2:%.*]]
+; CHECK-NEXT:    [[TMP14:%.*]] = or <8 x i64> [[TMP13]], [[TMP8]]
+; CHECK-NEXT:    [[TMP15:%.*]] = or <8 x i64> [[TMP14]], [[TMP4]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP10]], <8 x i64> [[TMP15]], <8 x i64> [[TMP12]]
+; CHECK-NEXT:    [[TMP16:%.*]] = select <8 x i1> [[TMP11]], <8 x i64> [[TMP9]], <8 x i64> [[A2]]
+; CHECK-NEXT:    store <8 x i64> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i64> [[TMP16]]
+;
+  %res = call <8 x i64> @llvm.x86.avx512.mask.psllv.q(<8 x i64> %a0, <8 x i64> %a1, <8 x i64> %a2, i8 %mask)
+  ret <8 x i64> %res
+}
+
+define <8 x i64> @test_x86_avx512_maskz_psllv_q(<8 x i64> %a0, <8 x i64> %a1, i8 %mask)  #0 {
+;
+; CHECK-LABEL: @test_x86_avx512_maskz_psllv_q(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = icmp ne <8 x i64> [[TMP2]], zeroinitializer
+; CHECK-NEXT:    [[TMP5:%.*]] = sext <8 x i1> [[TMP4]] to <8 x i64>
+; CHECK-NEXT:    [[TMP6:%.*]] = call <8 x i64> @llvm.x86.avx512.psllv.q.512(<8 x i64> [[TMP1]], <8 x i64> [[A1:%.*]])
+; CHECK-NEXT:    [[TMP7:%.*]] = or <8 x i64> [[TMP6]], [[TMP5]]
+; CHECK-NEXT:    [[TMP8:%.*]] = call <8 x i64> @llvm.x86.avx512.psllv.q.512(<8 x i64> [[A0:%.*]], <8 x i64> [[A1]])
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast i8 [[TMP3]] to <8 x i1>
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP11:%.*]] = select <8 x i1> [[TMP10]], <8 x i64> [[TMP7]], <8 x i64> zeroinitializer
+; CHECK-NEXT:    [[TMP12:%.*]] = xor <8 x i64> [[TMP8]], zeroinitializer
+; CHECK-NEXT:    [[TMP13:%.*]] = or <8 x i64> [[TMP12]], [[TMP7]]
+; CHECK-NEXT:    [[TMP14:%.*]] = or <8 x i64> [[TMP13]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP9]], <8 x i64> [[TMP14]], <8 x i64> [[TMP11]]
+; CHECK-NEXT:    [[TMP15:%.*]] = select <8 x i1> [[TMP10]], <8 x i64> [[TMP8]], <8 x i64> zeroinitializer
+; CHECK-NEXT:    store <8 x i64> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i64> [[TMP15]]
+;
+  %res = call <8 x i64> @llvm.x86.avx512.mask.psllv.q(<8 x i64> %a0, <8 x i64> %a1, <8 x i64> zeroinitializer, i8 %mask)
+  ret <8 x i64> %res
+}
+
+declare <8 x i64> @llvm.x86.avx512.mask.psllv.q(<8 x i64>, <8 x i64>, <8 x i64>, i8) nounwind readnone
+
+
+define <16 x i32> @test_x86_avx512_psrav_d(<16 x i32> %a0, <16 x i32> %a1)  #0 {
+; CHECK-LABEL: @test_x86_avx512_psrav_d(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp ne <16 x i32> [[TMP2]], zeroinitializer
+; CHECK-NEXT:    [[TMP4:%.*]] = sext <16 x i1> [[TMP3]] to <16 x i32>
+; CHECK-NEXT:    [[TMP5:%.*]] = call <16 x i32> @llvm.x86.avx512.psrav.d.512(<16 x i32> [[TMP1]], <16 x i32> [[A1:%.*]])
+; CHECK-NEXT:    [[TMP6:%.*]] = or <16 x i32> [[TMP5]], [[TMP4]]
+; CHECK-NEXT:    [[TMP7:%.*]] = call <16 x i32> @llvm.x86.avx512.psrav.d.512(<16 x i32> [[A0:%.*]], <16 x i32> [[A1]])
+; CHECK-NEXT:    store <16 x i32> [[TMP6]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i32> [[TMP7]]
+;
+  %res = call <16 x i32> @llvm.x86.avx512.mask.psrav.d(<16 x i32> %a0, <16 x i32> %a1, <16 x i32> zeroinitializer, i16 -1)
+  ret <16 x i32> %res
+}
+
+define <16 x i32> @test_x86_avx512_mask_psrav_d(<16 x i32> %a0, <16 x i32> %a1, <16 x i32> %a2, i16 %mask)  #0 {
+;
+; CHECK-LABEL: @test_x86_avx512_mask_psrav_d(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp ne <16 x i32> [[TMP2]], zeroinitializer
+; CHECK-NEXT:    [[TMP6:%.*]] = sext <16 x i1> [[TMP5]] to <16 x i32>
+; CHECK-NEXT:    [[TMP7:%.*]] = call <16 x i32> @llvm.x86.avx512.psrav.d.512(<16 x i32> [[TMP1]], <16 x i32> [[A1:%.*]])
+; CHECK-NEXT:    [[TMP8:%.*]] = or <16 x i32> [[TMP7]], [[TMP6]]
+; CHECK-NEXT:    [[TMP9:%.*]] = call <16 x i32> @llvm.x86.avx512.psrav.d.512(<16 x i32> [[A0:%.*]], <16 x i32> [[A1]])
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i16 [[TMP3]] to <16 x i1>
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast i16 [[MASK:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP12:%.*]] = select <16 x i1> [[TMP11]], <16 x i32> [[TMP8]], <16 x i32> [[TMP4]]
+; CHECK-NEXT:    [[TMP13:%.*]] = xor <16 x i32> [[TMP9]], [[A2:%.*]]
+; CHECK-NEXT:    [[TMP14:%.*]] = or <16 x i32> [[TMP13]], [[TMP8]]
+; CHECK-NEXT:    [[TMP15:%.*]] = or <16 x i32> [[TMP14]], [[TMP4]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP10]], <16 x i32> [[TMP15]], <16 x i32> [[TMP12]]
+; CHECK-NEXT:    [[TMP16:%.*]] = select <16 x i1> [[TMP11]], <16 x i32> [[TMP9]], <16 x i32> [[A2]]
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i32> [[TMP16]]
+;
+  %res = call <16 x i32> @llvm.x86.avx512.mask.psrav.d(<16 x i32> %a0, <16 x i32> %a1, <16 x i32> %a2, i16 %mask)
+  ret <16 x i32> %res
+}
+
+define <16 x i32> @test_x86_avx512_maskz_psrav_d(<16 x i32> %a0, <16 x i32> %a1, i16 %mask)  #0 {
+;
+; CHECK-LABEL: @test_x86_avx512_maskz_psrav_d(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = icmp ne <16 x i32> [[TMP2]], zeroinitializer
+; CHECK-NEXT:    [[TMP5:%.*]] = sext <16 x i1> [[TMP4]] to <16 x i32>
+; CHECK-NEXT:    [[TMP6:%.*]] = call <16 x i32> @llvm.x86.avx512.psrav.d.512(<16 x i32> [[TMP1]], <16 x i32> [[A1:%.*]])
+; CHECK-NEXT:    [[TMP7:%.*]] = or <16 x i32> [[TMP6]], [[TMP5]]
+; CHECK-NEXT:    [[TMP8:%.*]] = call <16 x i32> @llvm.x86.avx512.psrav.d.512(<16 x i32> [[A0:%.*]], <16 x i32> [[A1]])
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast i16 [[TMP3]] to <16 x i1>
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i16 [[MASK:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP11:%.*]] = select <16 x i1> [[TMP10]], <16 x i32> [[TMP7]], <16 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP12:%.*]] = xor <16 x i32> [[TMP8]], zeroinitializer
+; CHECK-NEXT:    [[TMP13:%.*]] = or <16 x i32> [[TMP12]], [[TMP7]]
+; CHECK-NEXT:    [[TMP14:%.*]] = or <16 x i32> [[TMP13]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP9]], <16 x i32> [[TMP14]], <16 x i32> [[TMP11]]
+; CHECK-NEXT:    [[TMP15:%.*]] = select <16 x i1> [[TMP10]], <16 x i32> [[TMP8]], <16 x i32> zeroinitializer
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i32> [[TMP15]]
+;
+  %res = call <16 x i32> @llvm.x86.avx512.mask.psrav.d(<16 x i32> %a0, <16 x i32> %a1, <16 x i32> zeroinitializer, i16 %mask)
+  ret <16 x i32> %res
+}
+
+declare <16 x i32> @llvm.x86.avx512.mask.psrav.d(<16 x i32>, <16 x i32>, <16 x i32>, i16) nounwind readnone
+
+define <8 x i64> @test_x86_avx512_psrav_q(<8 x i64> %a0, <8 x i64> %a1)  #0 {
+; CHECK-LABEL: @test_x86_avx512_psrav_q(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp ne <8 x i64> [[TMP2]], zeroinitializer
+; CHECK-NEXT:    [[TMP4:%.*]] = sext <8 x i1> [[TMP3]] to <8 x i64>
+; CHECK-NEXT:    [[TMP5:%.*]] = call <8 x i64> @llvm.x86.avx512.psrav.q.512(<8 x i64> [[TMP1]], <8 x i64> [[A1:%.*]])
+; CHECK-NEXT:    [[TMP6:%.*]] = or <8 x i64> [[TMP5]], [[TMP4]]
+; CHECK-NEXT:    [[TMP7:%.*]] = call <8 x i64> @llvm.x86.avx512.psrav.q.512(<8 x i64> [[A0:%.*]], <8 x i64> [[A1]])
+; CHECK-NEXT:    store <8 x i64> [[TMP6]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i64> [[TMP7]]
+;
+  %res = call <8 x i64> @llvm.x86.avx512.mask.psrav.q(<8 x i64> %a0, <8 x i64> %a1, <8 x i64> zeroinitializer, i8 -1)
+  ret <8 x i64> %res
+}
+
+define <8 x i64> @test_x86_avx512_mask_psrav_q(<8 x i64> %a0, <8 x i64> %a1, <8 x i64> %a2, i8 %mask)  #0 {
+;
+; CHECK-LABEL: @test_x86_avx512_mask_psrav_q(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp ne <8 x i64> [[TMP2]], zeroinitializer
+; CHECK-NEXT:    [[TMP6:%.*]] = sext <8 x i1> [[TMP5]] to <8 x i64>
+; CHECK-NEXT:    [[TMP7:%.*]] = call <8 x i64> @llvm.x86.avx512.psrav.q.512(<8 x i64> [[TMP1]], <8 x i64> [[A1:%.*]])
+; CHECK-NEXT:    [[TMP8:%.*]] = or <8 x i64> [[TMP7]], [[TMP6]]
+; CHECK-NEXT:    [[TMP9:%.*]] = call <8 x i64> @llvm.x86.avx512.psrav.q.512(<8 x i64> [[A0:%.*]], <8 x i64> [[A1]])
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i8 [[TMP3]] to <8 x i1>
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP12:%.*]] = select <8 x i1> [[TMP11]], <8 x i64> [[TMP8]], <8 x i64> [[TMP4]]
+; CHECK-NEXT:    [[TMP13:%.*]] = xor <8 x i64> [[TMP9]], [[A2:%.*]]
+; CHECK-NEXT:    [[TMP14:%.*]] = or <8 x i64> [[TMP13]], [[TMP8]]
+; CHECK-NEXT:    [[TMP15:%.*]] = or <8 x i64> [[TMP14]], [[TMP4]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP10]], <8 x i64> [[TMP15]], <8 x i64> [[TMP12]]
+; CHECK-NEXT:    [[TMP16:%.*]] = select <8 x i1> [[TMP11]], <8 x i64> [[TMP9]], <8 x i64> [[A2]]
+; CHECK-NEXT:    store <8 x i64> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i64> [[TMP16]]
+;
+  %res = call <8 x i64> @llvm.x86.avx512.mask.psrav.q(<8 x i64> %a0, <8 x i64> %a1, <8 x i64> %a2, i8 %mask)
+  ret <8 x i64> %res
+}
+
+define <8 x i64> @test_x86_avx512_maskz_psrav_q(<8 x i64> %a0, <8 x i64> %a1, i8 %mask)  #0 {
+;
+; CHECK-LABEL: @test_x86_avx512_maskz_psrav_q(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = icmp ne <8 x i64> [[TMP2]], zeroinitializer
+; CHECK-NEXT:    [[TMP5:%.*]] = sext <8 x i1> [[TMP4]] to <8 x i64>
+; CHECK-NEXT:    [[TMP6:%.*]] = call <8 x i64> @llvm.x86.avx512.psrav.q.512(<8 x i64> [[TMP1]], <8 x i64> [[A1:%.*]])
+; CHECK-NEXT:    [[TMP7:%.*]] = or <8 x i64> [[TMP6]], [[TMP5]]
+; CHECK-NEXT:    [[TMP8:%.*]] = call <8 x i64> @llvm.x86.avx512.psrav.q.512(<8 x i64> [[A0:%.*]], <8 x i64> [[A1]])
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast i8 [[TMP3]] to <8 x i1>
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP11:%.*]] = select <8 x i1> [[TMP10]], <8 x i64> [[TMP7]], <8 x i64> zeroinitializer
+; CHECK-NEXT:    [[TMP12:%.*]] = xor <8 x i64> [[TMP8]], zeroinitializer
+; CHECK-NEXT:    [[TMP13:%.*]] = or <8 x i64> [[TMP12]], [[TMP7]]
+; CHECK-NEXT:    [[TMP14:%.*]] = or <8 x i64> [[TMP13]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP9]], <8 x i64> [[TMP14]], <8 x i64> [[TMP11]]
+; CHECK-NEXT:    [[TMP15:%.*]] = select <8 x i1> [[TMP10]], <8 x i64> [[TMP8]], <8 x i64> zeroinitializer
+; CHECK-NEXT:    store <8 x i64> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i64> [[TMP15]]
+;
+  %res = call <8 x i64> @llvm.x86.avx512.mask.psrav.q(<8 x i64> %a0, <8 x i64> %a1, <8 x i64> zeroinitializer, i8 %mask)
+  ret <8 x i64> %res
+}
+
+declare <8 x i64> @llvm.x86.avx512.mask.psrav.q(<8 x i64>, <8 x i64>, <8 x i64>, i8) nounwind readnone
+
+define <16 x i32> @test_x86_avx512_psrlv_d(<16 x i32> %a0, <16 x i32> %a1)  #0 {
+; CHECK-LABEL: @test_x86_avx512_psrlv_d(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp ne <16 x i32> [[TMP2]], zeroinitializer
+; CHECK-NEXT:    [[TMP4:%.*]] = sext <16 x i1> [[TMP3]] to <16 x i32>
+; CHECK-NEXT:    [[TMP5:%.*]] = call <16 x i32> @llvm.x86.avx512.psrlv.d.512(<16 x i32> [[TMP1]], <16 x i32> [[A1:%.*]])
+; CHECK-NEXT:    [[TMP6:%.*]] = or <16 x i32> [[TMP5]], [[TMP4]]
+; CHECK-NEXT:    [[TMP7:%.*]] = call <16 x i32> @llvm.x86.avx512.psrlv.d.512(<16 x i32> [[A0:%.*]], <16 x i32> [[A1]])
+; CHECK-NEXT:    store <16 x i32> [[TMP6]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i32> [[TMP7]]
+;
+  %res = call <16 x i32> @llvm.x86.avx512.mask.psrlv.d(<16 x i32> %a0, <16 x i32> %a1, <16 x i32> zeroinitializer, i16 -1)
+  ret <16 x i32> %res
+}
+
+define <16 x i32> @test_x86_avx512_mask_psrlv_d(<16 x i32> %a0, <16 x i32> %a1, <16 x i32> %a2, i16 %mask)  #0 {
+;
+; CHECK-LABEL: @test_x86_avx512_mask_psrlv_d(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp ne <16 x i32> [[TMP2]], zeroinitializer
+; CHECK-NEXT:    [[TMP6:%.*]] = sext <16 x i1> [[TMP5]] to <16 x i32>
+; CHECK-NEXT:    [[TMP7:%.*]] = call <16 x i32> @llvm.x86.avx512.psrlv.d.512(<16 x i32> [[TMP1]], <16 x i32> [[A1:%.*]])
+; CHECK-NEXT:    [[TMP8:%.*]] = or <16 x i32> [[TMP7]], [[TMP6]]
+; CHECK-NEXT:    [[TMP9:%.*]] = call <16 x i32> @llvm.x86.avx512.psrlv.d.512(<16 x i32> [[A0:%.*]], <16 x i32> [[A1]])
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i16 [[TMP3]] to <16 x i1>
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast i16 [[MASK:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP12:%.*]] = select <16 x i1> [[TMP11]], <16 x i32> [[TMP8]], <16 x i32> [[TMP4]]
+; CHECK-NEXT:    [[TMP13:%.*]] = xor <16 x i32> [[TMP9]], [[A2:%.*]]
+; CHECK-NEXT:    [[TMP14:%.*]] = or <16 x i32> [[TMP13]], [[TMP8]]
+; CHECK-NEXT:    [[TMP15:%.*]] = or <16 x i32> [[TMP14]], [[TMP4]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP10]], <16 x i32> [[TMP15]], <16 x i32> [[TMP12]]
+; CHECK-NEXT:    [[TMP16:%.*]] = select <16 x i1> [[TMP11]], <16 x i32> [[TMP9]], <16 x i32> [[A2]]
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i32> [[TMP16]]
+;
+  %res = call <16 x i32> @llvm.x86.avx512.mask.psrlv.d(<16 x i32> %a0, <16 x i32> %a1, <16 x i32> %a2, i16 %mask)
+  ret <16 x i32> %res
+}
+
+define <16 x i32> @test_x86_avx512_maskz_psrlv_d(<16 x i32> %a0, <16 x i32> %a1, i16 %mask)  #0 {
+;
+; CHECK-LABEL: @test_x86_avx512_maskz_psrlv_d(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = icmp ne <16 x i32> [[TMP2]], zeroinitializer
+; CHECK-NEXT:    [[TMP5:%.*]] = sext <16 x i1> [[TMP4]] to <16 x i32>
+; CHECK-NEXT:    [[TMP6:%.*]] = call <16 x i32> @llvm.x86.avx512.psrlv.d.512(<16 x i32> [[TMP1]], <16 x i32> [[A1:%.*]])
+; CHECK-NEXT:    [[TMP7:%.*]] = or <16 x i32> [[TMP6]], [[TMP5]]
+; CHECK-NEXT:    [[TMP8:%.*]] = call <16 x i32> @llvm.x86.avx512.psrlv.d.512(<16 x i32> [[A0:%.*]], <16 x i32> [[A1]])
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast i16 [[TMP3]] to <16 x i1>
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i16 [[MASK:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP11:%.*]] = select <16 x i1> [[TMP10]], <16 x i32> [[TMP7]], <16 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP12:%.*]] = xor <16 x i32> [[TMP8]], zeroinitializer
+; CHECK-NEXT:    [[TMP13:%.*]] = or <16 x i32> [[TMP12]], [[TMP7]]
+; CHECK-NEXT:    [[TMP14:%.*]] = or <16 x i32> [[TMP13]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP9]], <16 x i32> [[TMP14]], <16 x i32> [[TMP11]]
+; CHECK-NEXT:    [[TMP15:%.*]] = select <16 x i1> [[TMP10]], <16 x i32> [[TMP8]], <16 x i32> zeroinitializer
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i32> [[TMP15]]
+;
+  %res = call <16 x i32> @llvm.x86.avx512.mask.psrlv.d(<16 x i32> %a0, <16 x i32> %a1, <16 x i32> zeroinitializer, i16 %mask)
+  ret <16 x i32> %res
+}
+
+declare <16 x i32> @llvm.x86.avx512.mask.psrlv.d(<16 x i32>, <16 x i32>, <16 x i32>, i16) nounwind readnone
+
+define <8 x i64> @test_x86_avx512_psrlv_q(<8 x i64> %a0, <8 x i64> %a1)  #0 {
+; CHECK-LABEL: @test_x86_avx512_psrlv_q(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp ne <8 x i64> [[TMP2]], zeroinitializer
+; CHECK-NEXT:    [[TMP4:%.*]] = sext <8 x i1> [[TMP3]] to <8 x i64>
+; CHECK-NEXT:    [[TMP5:%.*]] = call <8 x i64> @llvm.x86.avx512.psrlv.q.512(<8 x i64> [[TMP1]], <8 x i64> [[A1:%.*]])
+; CHECK-NEXT:    [[TMP6:%.*]] = or <8 x i64> [[TMP5]], [[TMP4]]
+; CHECK-NEXT:    [[TMP7:%.*]] = call <8 x i64> @llvm.x86.avx512.psrlv.q.512(<8 x i64> [[A0:%.*]], <8 x i64> [[A1]])
+; CHECK-NEXT:    store <8 x i64> [[TMP6]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i64> [[TMP7]]
+;
+  %res = call <8 x i64> @llvm.x86.avx512.mask.psrlv.q(<8 x i64> %a0, <8 x i64> %a1, <8 x i64> zeroinitializer, i8 -1)
+  ret <8 x i64> %res
+}
+
+define <8 x i64> @test_x86_avx512_mask_psrlv_q(<8 x i64> %a0, <8 x i64> %a1, <8 x i64> %a2, i8 %mask)  #0 {
+;
+; CHECK-LABEL: @test_x86_avx512_mask_psrlv_q(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp ne <8 x i64> [[TMP2]], zeroinitializer
+; CHECK-NEXT:    [[TMP6:%.*]] = sext <8 x i1> [[TMP5]] to <8 x i64>
+; CHECK-NEXT:    [[TMP7:%.*]] = call <8 x i64> @llvm.x86.avx512.psrlv.q.512(<8 x i64> [[TMP1]], <8 x i64> [[A1:%.*]])
+; CHECK-NEXT:    [[TMP8:%.*]] = or <8 x i64> [[TMP7]], [[TMP6]]
+; CHECK-NEXT:    [[TMP9:%.*]] = call <8 x i64> @llvm.x86.avx512.psrlv.q.512(<8 x i64> [[A0:%.*]], <8 x i64> [[A1]])
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i8 [[TMP3]] to <8 x i1>
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP12:%.*]] = select <8 x i1> [[TMP11]], <8 x i64> [[TMP8]], <8 x i64> [[TMP4]]
+; CHECK-NEXT:    [[TMP13:%.*]] = xor <8 x i64> [[TMP9]], [[A2:%.*]]
+; CHECK-NEXT:    [[TMP14:%.*]] = or <8 x i64> [[TMP13]], [[TMP8]]
+; CHECK-NEXT:    [[TMP15:%.*]] = or <8 x i64> [[TMP14]], [[TMP4]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP10]], <8 x i64> [[TMP15]], <8 x i64> [[TMP12]]
+; CHECK-NEXT:    [[TMP16:%.*]] = select <8 x i1> [[TMP11]], <8 x i64> [[TMP9]], <8 x i64> [[A2]]
+; CHECK-NEXT:    store <8 x i64> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i64> [[TMP16]]
+;
+  %res = call <8 x i64> @llvm.x86.avx512.mask.psrlv.q(<8 x i64> %a0, <8 x i64> %a1, <8 x i64> %a2, i8 %mask)
+  ret <8 x i64> %res
+}
+
+define <8 x i64> @test_x86_avx512_maskz_psrlv_q(<8 x i64> %a0, <8 x i64> %a1, i8 %mask)  #0 {
+;
+; CHECK-LABEL: @test_x86_avx512_maskz_psrlv_q(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = icmp ne <8 x i64> [[TMP2]], zeroinitializer
+; CHECK-NEXT:    [[TMP5:%.*]] = sext <8 x i1> [[TMP4]] to <8 x i64>
+; CHECK-NEXT:    [[TMP6:%.*]] = call <8 x i64> @llvm.x86.avx512.psrlv.q.512(<8 x i64> [[TMP1]], <8 x i64> [[A1:%.*]])
+; CHECK-NEXT:    [[TMP7:%.*]] = or <8 x i64> [[TMP6]], [[TMP5]]
+; CHECK-NEXT:    [[TMP8:%.*]] = call <8 x i64> @llvm.x86.avx512.psrlv.q.512(<8 x i64> [[A0:%.*]], <8 x i64> [[A1]])
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast i8 [[TMP3]] to <8 x i1>
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP11:%.*]] = select <8 x i1> [[TMP10]], <8 x i64> [[TMP7]], <8 x i64> zeroinitializer
+; CHECK-NEXT:    [[TMP12:%.*]] = xor <8 x i64> [[TMP8]], zeroinitializer
+; CHECK-NEXT:    [[TMP13:%.*]] = or <8 x i64> [[TMP12]], [[TMP7]]
+; CHECK-NEXT:    [[TMP14:%.*]] = or <8 x i64> [[TMP13]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP9]], <8 x i64> [[TMP14]], <8 x i64> [[TMP11]]
+; CHECK-NEXT:    [[TMP15:%.*]] = select <8 x i1> [[TMP10]], <8 x i64> [[TMP8]], <8 x i64> zeroinitializer
+; CHECK-NEXT:    store <8 x i64> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i64> [[TMP15]]
+;
+  %res = call <8 x i64> @llvm.x86.avx512.mask.psrlv.q(<8 x i64> %a0, <8 x i64> %a1, <8 x i64> zeroinitializer, i8 %mask)
+  ret <8 x i64> %res
+}
+
+declare <8 x i64> @llvm.x86.avx512.mask.psrlv.q(<8 x i64>, <8 x i64>, <8 x i64>, i8) nounwind readnone
+
+define <8 x i64> @test_x86_avx512_psrlv_q_memop(<8 x i64> %a0, ptr %ptr)  #0 {
+;
+; CHECK-LABEL: @test_x86_avx512_psrlv_q_memop(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP3:%.*]], label [[TMP4:%.*]], !prof [[PROF1]]
+; CHECK:       3:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       4:
+; CHECK-NEXT:    [[B:%.*]] = load <8 x i64>, ptr [[PTR:%.*]], align 64
+; CHECK-NEXT:    [[TMP5:%.*]] = ptrtoint ptr [[PTR]] to i64
+; CHECK-NEXT:    [[TMP6:%.*]] = xor i64 [[TMP5]], 87960930222080
+; CHECK-NEXT:    [[TMP7:%.*]] = inttoptr i64 [[TMP6]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <8 x i64>, ptr [[TMP7]], align 64
+; CHECK-NEXT:    [[TMP8:%.*]] = icmp ne <8 x i64> [[_MSLD]], zeroinitializer
+; CHECK-NEXT:    [[TMP9:%.*]] = sext <8 x i1> [[TMP8]] to <8 x i64>
+; CHECK-NEXT:    [[TMP10:%.*]] = call <8 x i64> @llvm.x86.avx512.psrlv.q.512(<8 x i64> [[TMP2]], <8 x i64> [[B]])
+; CHECK-NEXT:    [[TMP11:%.*]] = or <8 x i64> [[TMP10]], [[TMP9]]
+; CHECK-NEXT:    [[TMP12:%.*]] = call <8 x i64> @llvm.x86.avx512.psrlv.q.512(<8 x i64> [[A0:%.*]], <8 x i64> [[B]])
+; CHECK-NEXT:    store <8 x i64> [[TMP11]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i64> [[TMP12]]
+;
+  %b = load <8 x i64>, ptr %ptr
+  %res = call <8 x i64> @llvm.x86.avx512.mask.psrlv.q(<8 x i64> %a0, <8 x i64> %b, <8 x i64> zeroinitializer, i8 -1)
+  ret <8 x i64> %res
+}
+
+declare <8 x double> @llvm.x86.avx512.mask.cvtdq2pd.512(<8 x i32>, <8 x double>, i8)
+
+define <8 x double>@test_int_x86_avx512_cvt_dq2pd_512(<8 x i32> %x0, <8 x double> %x1)  #0 {
+; CHECK-LABEL: @test_int_x86_avx512_cvt_dq2pd_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = zext <8 x i32> [[TMP1]] to <8 x i64>
+; CHECK-NEXT:    [[CVT:%.*]] = sitofp <8 x i32> [[X0:%.*]] to <8 x double>
+; CHECK-NEXT:    store <8 x i64> [[TMP2]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x double> [[CVT]]
+;
+  %res = call <8 x double> @llvm.x86.avx512.mask.cvtdq2pd.512(<8 x i32> %x0, <8 x double> %x1, i8 -1)
+  ret <8 x double> %res
+}
+
+define <8 x double>@test_int_x86_avx512_mask_cvt_dq2pd_512(<8 x i32> %x0, <8 x double> %x1, i8 %x2)  #0 {
+;
+; CHECK-LABEL: @test_int_x86_avx512_mask_cvt_dq2pd_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 96) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = zext <8 x i32> [[TMP1]] to <8 x i64>
+; CHECK-NEXT:    [[CVT:%.*]] = sitofp <8 x i32> [[X0:%.*]] to <8 x double>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i8 [[TMP2]] to <8 x i1>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i8 [[X2:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP7:%.*]] = select <8 x i1> [[TMP6]], <8 x i64> [[TMP4]], <8 x i64> [[TMP3]]
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <8 x double> [[CVT]] to <8 x i64>
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <8 x double> [[X1:%.*]] to <8 x i64>
+; CHECK-NEXT:    [[TMP10:%.*]] = xor <8 x i64> [[TMP8]], [[TMP9]]
+; CHECK-NEXT:    [[TMP11:%.*]] = or <8 x i64> [[TMP10]], [[TMP4]]
+; CHECK-NEXT:    [[TMP12:%.*]] = or <8 x i64> [[TMP11]], [[TMP3]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP5]], <8 x i64> [[TMP12]], <8 x i64> [[TMP7]]
+; CHECK-NEXT:    [[TMP13:%.*]] = select <8 x i1> [[TMP6]], <8 x double> [[CVT]], <8 x double> [[X1]]
+; CHECK-NEXT:    store <8 x i64> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x double> [[TMP13]]
+;
+  %res = call <8 x double> @llvm.x86.avx512.mask.cvtdq2pd.512(<8 x i32> %x0, <8 x double> %x1, i8 %x2)
+  ret <8 x double> %res
+}
+
+declare <8 x double> @llvm.x86.avx512.mask.cvtudq2pd.512(<8 x i32>, <8 x double>, i8)
+
+define <8 x double>@test_int_x86_avx512_cvt_udq2pd_512(<8 x i32> %x0, <8 x double> %x1)  #0 {
+; CHECK-LABEL: @test_int_x86_avx512_cvt_udq2pd_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = zext <8 x i32> [[TMP1]] to <8 x i64>
+; CHECK-NEXT:    [[CVT:%.*]] = uitofp <8 x i32> [[X0:%.*]] to <8 x double>
+; CHECK-NEXT:    store <8 x i64> [[TMP2]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x double> [[CVT]]
+;
+  %res = call <8 x double> @llvm.x86.avx512.mask.cvtudq2pd.512(<8 x i32> %x0, <8 x double> %x1, i8 -1)
+  ret <8 x double> %res
+}
+
+define <8 x double>@test_int_x86_avx512_mask_cvt_udq2pd_512(<8 x i32> %x0, <8 x double> %x1, i8 %x2)  #0 {
+;
+; CHECK-LABEL: @test_int_x86_avx512_mask_cvt_udq2pd_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 96) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = zext <8 x i32> [[TMP1]] to <8 x i64>
+; CHECK-NEXT:    [[CVT:%.*]] = uitofp <8 x i32> [[X0:%.*]] to <8 x double>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i8 [[TMP2]] to <8 x i1>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i8 [[X2:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP7:%.*]] = select <8 x i1> [[TMP6]], <8 x i64> [[TMP4]], <8 x i64> [[TMP3]]
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <8 x double> [[CVT]] to <8 x i64>
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <8 x double> [[X1:%.*]] to <8 x i64>
+; CHECK-NEXT:    [[TMP10:%.*]] = xor <8 x i64> [[TMP8]], [[TMP9]]
+; CHECK-NEXT:    [[TMP11:%.*]] = or <8 x i64> [[TMP10]], [[TMP4]]
+; CHECK-NEXT:    [[TMP12:%.*]] = or <8 x i64> [[TMP11]], [[TMP3]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP5]], <8 x i64> [[TMP12]], <8 x i64> [[TMP7]]
+; CHECK-NEXT:    [[TMP13:%.*]] = select <8 x i1> [[TMP6]], <8 x double> [[CVT]], <8 x double> [[X1]]
+; CHECK-NEXT:    store <8 x i64> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x double> [[TMP13]]
+;
+  %res = call <8 x double> @llvm.x86.avx512.mask.cvtudq2pd.512(<8 x i32> %x0, <8 x double> %x1, i8 %x2)
+  ret <8 x double> %res
+}
+
+define <16 x float> @test_x86_vcvtph2ps_512(<16 x i16> %a0)  #0 {
+; CHECK-LABEL: @test_x86_vcvtph2ps_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <16 x i16> [[TMP1]] to i256
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i256 [[TMP2]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP3:%.*]], label [[TMP4:%.*]], !prof [[PROF1]]
+; CHECK:       3:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       4:
+; CHECK-NEXT:    [[RES:%.*]] = call <16 x float> @llvm.x86.avx512.mask.vcvtph2ps.512(<16 x i16> [[A0:%.*]], <16 x float> zeroinitializer, i16 -1, i32 4)
+; CHECK-NEXT:    store <16 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x float> [[RES]]
+;
+  %res = call <16 x float> @llvm.x86.avx512.mask.vcvtph2ps.512(<16 x i16> %a0, <16 x float> zeroinitializer, i16 -1, i32 4)
+  ret <16 x float> %res
+}
+
+define <16 x float> @test_x86_vcvtph2ps_512_sae(<16 x i16> %a0)  #0 {
+; CHECK-LABEL: @test_x86_vcvtph2ps_512_sae(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <16 x i16> [[TMP1]] to i256
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i256 [[TMP2]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP3:%.*]], label [[TMP4:%.*]], !prof [[PROF1]]
+; CHECK:       3:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       4:
+; CHECK-NEXT:    [[RES:%.*]] = call <16 x float> @llvm.x86.avx512.mask.vcvtph2ps.512(<16 x i16> [[A0:%.*]], <16 x float> zeroinitializer, i16 -1, i32 8)
+; CHECK-NEXT:    store <16 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x float> [[RES]]
+;
+  %res = call <16 x float> @llvm.x86.avx512.mask.vcvtph2ps.512(<16 x i16> %a0, <16 x float> zeroinitializer, i16 -1, i32 8)
+  ret <16 x float> %res
+}
+
+define <16 x float> @test_x86_vcvtph2ps_512_rrk(<16 x i16> %a0,<16 x float> %a1, i16 %mask)  #0 {
+;
+; CHECK-LABEL: @test_x86_vcvtph2ps_512_rrk(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 96) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <16 x i16> [[TMP1]] to i256
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i256 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i16 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR3]], label [[TMP6:%.*]], label [[TMP7:%.*]], !prof [[PROF1]]
+; CHECK:       6:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       7:
+; CHECK-NEXT:    [[RES:%.*]] = call <16 x float> @llvm.x86.avx512.mask.vcvtph2ps.512(<16 x i16> [[A0:%.*]], <16 x float> [[A1:%.*]], i16 [[MASK:%.*]], i32 4)
+; CHECK-NEXT:    store <16 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x float> [[RES]]
+;
+  %res = call <16 x float> @llvm.x86.avx512.mask.vcvtph2ps.512(<16 x i16> %a0, <16 x float> %a1, i16 %mask, i32 4)
+  ret <16 x float> %res
+}
+
+define <16 x float> @test_x86_vcvtph2ps_512_sae_rrkz(<16 x i16> %a0, i16 %mask)  #0 {
+;
+; CHECK-LABEL: @test_x86_vcvtph2ps_512_sae_rrkz(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <16 x i16> [[TMP1]] to i256
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i256 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i16 [[TMP2]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP4:%.*]], label [[TMP5:%.*]], !prof [[PROF1]]
+; CHECK:       4:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       5:
+; CHECK-NEXT:    [[RES:%.*]] = call <16 x float> @llvm.x86.avx512.mask.vcvtph2ps.512(<16 x i16> [[A0:%.*]], <16 x float> zeroinitializer, i16 [[MASK:%.*]], i32 8)
+; CHECK-NEXT:    store <16 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x float> [[RES]]
+;
+  %res = call <16 x float> @llvm.x86.avx512.mask.vcvtph2ps.512(<16 x i16> %a0, <16 x float> zeroinitializer, i16 %mask, i32 8)
+  ret <16 x float> %res
+}
+
+define <16 x float> @test_x86_vcvtph2ps_512_rrkz(<16 x i16> %a0, i16 %mask)  #0 {
+;
+; CHECK-LABEL: @test_x86_vcvtph2ps_512_rrkz(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <16 x i16> [[TMP1]] to i256
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i256 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i16 [[TMP2]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP4:%.*]], label [[TMP5:%.*]], !prof [[PROF1]]
+; CHECK:       4:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       5:
+; CHECK-NEXT:    [[RES:%.*]] = call <16 x float> @llvm.x86.avx512.mask.vcvtph2ps.512(<16 x i16> [[A0:%.*]], <16 x float> zeroinitializer, i16 [[MASK:%.*]], i32 4)
+; CHECK-NEXT:    store <16 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x float> [[RES]]
+;
+  %res = call <16 x float> @llvm.x86.avx512.mask.vcvtph2ps.512(<16 x i16> %a0, <16 x float> zeroinitializer, i16 %mask, i32 4)
+  ret <16 x float> %res
+}
+
+declare <16 x float> @llvm.x86.avx512.mask.vcvtph2ps.512(<16 x i16>, <16 x float>, i16, i32) nounwind readonly
+
+define <8 x i64> @test_valign_q(<8 x i64> %a, <8 x i64> %b)  #0 {
+; CHECK-LABEL: @test_valign_q(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <8 x i64> [[TMP1]], <8 x i64> [[TMP2]], <8 x i32> <i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9>
+; CHECK-NEXT:    [[PALIGNR:%.*]] = shufflevector <8 x i64> [[B:%.*]], <8 x i64> [[A:%.*]], <8 x i32> <i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9>
+; CHECK-NEXT:    store <8 x i64> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i64> [[PALIGNR]]
+;
+  %res = call <8 x i64> @llvm.x86.avx512.mask.valign.q.512(<8 x i64> %a, <8 x i64> %b, i32 2, <8 x i64> zeroinitializer, i8 -1)
+  ret <8 x i64> %res
+}
+
+define <8 x i64> @test_mask_valign_q(<8 x i64> %a, <8 x i64> %b, <8 x i64> %src, i8 %mask)  #0 {
+;
+; CHECK-LABEL: @test_mask_valign_q(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <8 x i64> [[TMP1]], <8 x i64> [[TMP2]], <8 x i32> <i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9>
+; CHECK-NEXT:    [[PALIGNR:%.*]] = shufflevector <8 x i64> [[B:%.*]], <8 x i64> [[A:%.*]], <8 x i32> <i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i8 [[TMP3]] to <8 x i1>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP7:%.*]] = select <8 x i1> [[TMP6]], <8 x i64> [[_MSPROP]], <8 x i64> [[TMP4]]
+; CHECK-NEXT:    [[TMP8:%.*]] = xor <8 x i64> [[PALIGNR]], [[SRC:%.*]]
+; CHECK-NEXT:    [[TMP9:%.*]] = or <8 x i64> [[TMP8]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP10:%.*]] = or <8 x i64> [[TMP9]], [[TMP4]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP5]], <8 x i64> [[TMP10]], <8 x i64> [[TMP7]]
+; CHECK-NEXT:    [[TMP11:%.*]] = select <8 x i1> [[TMP6]], <8 x i64> [[PALIGNR]], <8 x i64> [[SRC]]
+; CHECK-NEXT:    store <8 x i64> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i64> [[TMP11]]
+;
+  %res = call <8 x i64> @llvm.x86.avx512.mask.valign.q.512(<8 x i64> %a, <8 x i64> %b, i32 2, <8 x i64> %src, i8 %mask)
+  ret <8 x i64> %res
+}
+
+declare <8 x i64> @llvm.x86.avx512.mask.valign.q.512(<8 x i64>, <8 x i64>, i32, <8 x i64>, i8)
+
+define <16 x i32> @test_maskz_valign_d(<16 x i32> %a, <16 x i32> %b, i16 %mask)  #0 {
+;
+; CHECK-LABEL: @test_maskz_valign_d(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <16 x i32> [[TMP1]], <16 x i32> [[TMP2]], <16 x i32> <i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20>
+; CHECK-NEXT:    [[PALIGNR:%.*]] = shufflevector <16 x i32> [[B:%.*]], <16 x i32> [[A:%.*]], <16 x i32> <i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20>
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast i16 [[TMP3]] to <16 x i1>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i16 [[MASK:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP6:%.*]] = select <16 x i1> [[TMP5]], <16 x i32> [[_MSPROP]], <16 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP7:%.*]] = xor <16 x i32> [[PALIGNR]], zeroinitializer
+; CHECK-NEXT:    [[TMP8:%.*]] = or <16 x i32> [[TMP7]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP9:%.*]] = or <16 x i32> [[TMP8]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP4]], <16 x i32> [[TMP9]], <16 x i32> [[TMP6]]
+; CHECK-NEXT:    [[TMP10:%.*]] = select <16 x i1> [[TMP5]], <16 x i32> [[PALIGNR]], <16 x i32> zeroinitializer
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i32> [[TMP10]]
+;
+  %res = call <16 x i32> @llvm.x86.avx512.mask.valign.d.512(<16 x i32> %a, <16 x i32> %b, i32 5, <16 x i32> zeroinitializer, i16 %mask)
+  ret <16 x i32> %res
+}
+
+declare <16 x i32> @llvm.x86.avx512.mask.valign.d.512(<16 x i32>, <16 x i32>, i32, <16 x i32>, i16)
+
+declare <8 x double> @llvm.x86.avx512.mask.vpermilvar.pd.512(<8 x double>, <8 x i64>, <8 x double>, i8)
+
+define <8 x double>@test_int_x86_avx512_vpermilvar_pd_512(<8 x double> %x0, <8 x i64> %x1, <8 x double> %x2)  #0 {
+; CHECK-LABEL: @test_int_x86_avx512_vpermilvar_pd_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <8 x i64> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP3]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <8 x i64> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP5:%.*]], label [[TMP6:%.*]], !prof [[PROF1]]
+; CHECK:       5:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       6:
+; CHECK-NEXT:    [[TMP7:%.*]] = call <8 x double> @llvm.x86.avx512.vpermilvar.pd.512(<8 x double> [[X0:%.*]], <8 x i64> [[X1:%.*]])
+; CHECK-NEXT:    store <8 x i64> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x double> [[TMP7]]
+;
+  %res = call <8 x double> @llvm.x86.avx512.mask.vpermilvar.pd.512(<8 x double> %x0, <8 x i64> %x1, <8 x double> %x2, i8 -1)
+  ret <8 x double> %res
+}
+
+define <8 x double>@test_int_x86_avx512_mask_vpermilvar_pd_512(<8 x double> %x0, <8 x i64> %x1, <8 x double> %x2, i8 %x3)  #0 {
+;
+; CHECK-LABEL: @test_int_x86_avx512_mask_vpermilvar_pd_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <8 x i64> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <8 x i64> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP6]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP7:%.*]], label [[TMP8:%.*]], !prof [[PROF1]]
+; CHECK:       7:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       8:
+; CHECK-NEXT:    [[TMP9:%.*]] = call <8 x double> @llvm.x86.avx512.vpermilvar.pd.512(<8 x double> [[X0:%.*]], <8 x i64> [[X1:%.*]])
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i8 [[TMP3]] to <8 x i1>
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast i8 [[X3:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP12:%.*]] = select <8 x i1> [[TMP11]], <8 x i64> zeroinitializer, <8 x i64> [[TMP4]]
+; CHECK-NEXT:    [[TMP13:%.*]] = bitcast <8 x double> [[TMP9]] to <8 x i64>
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast <8 x double> [[X2:%.*]] to <8 x i64>
+; CHECK-NEXT:    [[TMP15:%.*]] = xor <8 x i64> [[TMP13]], [[TMP14]]
+; CHECK-NEXT:    [[TMP16:%.*]] = or <8 x i64> [[TMP15]], zeroinitializer
+; CHECK-NEXT:    [[TMP17:%.*]] = or <8 x i64> [[TMP16]], [[TMP4]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP10]], <8 x i64> [[TMP17]], <8 x i64> [[TMP12]]
+; CHECK-NEXT:    [[TMP18:%.*]] = select <8 x i1> [[TMP11]], <8 x double> [[TMP9]], <8 x double> [[X2]]
+; CHECK-NEXT:    store <8 x i64> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x double> [[TMP18]]
+;
+  %res = call <8 x double> @llvm.x86.avx512.mask.vpermilvar.pd.512(<8 x double> %x0, <8 x i64> %x1, <8 x double> %x2, i8 %x3)
+  ret <8 x double> %res
+}
+
+define <8 x double>@test_int_x86_avx512_maskz_vpermilvar_pd_512(<8 x double> %x0, <8 x i64> %x1, <8 x double> %x2, i8 %x3)  #0 {
+;
+; CHECK-LABEL: @test_int_x86_avx512_maskz_vpermilvar_pd_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <8 x i64> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <8 x i64> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP6:%.*]], label [[TMP7:%.*]], !prof [[PROF1]]
+; CHECK:       6:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       7:
+; CHECK-NEXT:    [[TMP8:%.*]] = call <8 x double> @llvm.x86.avx512.vpermilvar.pd.512(<8 x double> [[X0:%.*]], <8 x i64> [[X1:%.*]])
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast i8 [[TMP3]] to <8 x i1>
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i8 [[X3:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP11:%.*]] = select <8 x i1> [[TMP10]], <8 x i64> zeroinitializer, <8 x i64> zeroinitializer
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <8 x double> [[TMP8]] to <8 x i64>
+; CHECK-NEXT:    [[TMP13:%.*]] = xor <8 x i64> [[TMP12]], zeroinitializer
+; CHECK-NEXT:    [[TMP14:%.*]] = or <8 x i64> [[TMP13]], zeroinitializer
+; CHECK-NEXT:    [[TMP15:%.*]] = or <8 x i64> [[TMP14]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP9]], <8 x i64> [[TMP15]], <8 x i64> [[TMP11]]
+; CHECK-NEXT:    [[TMP16:%.*]] = select <8 x i1> [[TMP10]], <8 x double> [[TMP8]], <8 x double> zeroinitializer
+; CHECK-NEXT:    store <8 x i64> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x double> [[TMP16]]
+;
+  %res = call <8 x double> @llvm.x86.avx512.mask.vpermilvar.pd.512(<8 x double> %x0, <8 x i64> %x1, <8 x double> zeroinitializer, i8 %x3)
+  ret <8 x double> %res
+}
+
+declare <16 x float> @llvm.x86.avx512.mask.vpermilvar.ps.512(<16 x float>, <16 x i32>, <16 x float>, i16)
+
+define <16 x float>@test_int_x86_avx512_vpermilvar_ps_512(<16 x float> %x0, <16 x i32> %x1, <16 x float> %x2)  #0 {
+; CHECK-LABEL: @test_int_x86_avx512_vpermilvar_ps_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP3]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP5:%.*]], label [[TMP6:%.*]], !prof [[PROF1]]
+; CHECK:       5:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       6:
+; CHECK-NEXT:    [[TMP7:%.*]] = call <16 x float> @llvm.x86.avx512.vpermilvar.ps.512(<16 x float> [[X0:%.*]], <16 x i32> [[X1:%.*]])
+; CHECK-NEXT:    store <16 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x float> [[TMP7]]
+;
+  %res = call <16 x float> @llvm.x86.avx512.mask.vpermilvar.ps.512(<16 x float> %x0, <16 x i32> %x1, <16 x float> %x2, i16 -1)
+  ret <16 x float> %res
+}
+
+define <16 x float>@test_int_x86_avx512_mask_vpermilvar_ps_512(<16 x float> %x0, <16 x i32> %x1, <16 x float> %x2, i16 %x3)  #0 {
+;
+; CHECK-LABEL: @test_int_x86_avx512_mask_vpermilvar_ps_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP6]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP7:%.*]], label [[TMP8:%.*]], !prof [[PROF1]]
+; CHECK:       7:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       8:
+; CHECK-NEXT:    [[TMP9:%.*]] = call <16 x float> @llvm.x86.avx512.vpermilvar.ps.512(<16 x float> [[X0:%.*]], <16 x i32> [[X1:%.*]])
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i16 [[TMP3]] to <16 x i1>
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast i16 [[X3:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP12:%.*]] = select <16 x i1> [[TMP11]], <16 x i32> zeroinitializer, <16 x i32> [[TMP4]]
+; CHECK-NEXT:    [[TMP13:%.*]] = bitcast <16 x float> [[TMP9]] to <16 x i32>
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast <16 x float> [[X2:%.*]] to <16 x i32>
+; CHECK-NEXT:    [[TMP15:%.*]] = xor <16 x i32> [[TMP13]], [[TMP14]]
+; CHECK-NEXT:    [[TMP16:%.*]] = or <16 x i32> [[TMP15]], zeroinitializer
+; CHECK-NEXT:    [[TMP17:%.*]] = or <16 x i32> [[TMP16]], [[TMP4]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP10]], <16 x i32> [[TMP17]], <16 x i32> [[TMP12]]
+; CHECK-NEXT:    [[TMP18:%.*]] = select <16 x i1> [[TMP11]], <16 x float> [[TMP9]], <16 x float> [[X2]]
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x float> [[TMP18]]
+;
+  %res = call <16 x float> @llvm.x86.avx512.mask.vpermilvar.ps.512(<16 x float> %x0, <16 x i32> %x1, <16 x float> %x2, i16 %x3)
+  ret <16 x float> %res
+}
+
+
+define <16 x float>@test_int_x86_avx512_maskz_vpermilvar_ps_512(<16 x float> %x0, <16 x i32> %x1, i16 %x3)  #0 {
+;
+; CHECK-LABEL: @test_int_x86_avx512_maskz_vpermilvar_ps_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP6:%.*]], label [[TMP7:%.*]], !prof [[PROF1]]
+; CHECK:       6:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       7:
+; CHECK-NEXT:    [[TMP8:%.*]] = call <16 x float> @llvm.x86.avx512.vpermilvar.ps.512(<16 x float> [[X0:%.*]], <16 x i32> [[X1:%.*]])
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast i16 [[TMP3]] to <16 x i1>
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i16 [[X3:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP11:%.*]] = select <16 x i1> [[TMP10]], <16 x i32> zeroinitializer, <16 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <16 x float> [[TMP8]] to <16 x i32>
+; CHECK-NEXT:    [[TMP13:%.*]] = xor <16 x i32> [[TMP12]], zeroinitializer
+; CHECK-NEXT:    [[TMP14:%.*]] = or <16 x i32> [[TMP13]], zeroinitializer
+; CHECK-NEXT:    [[TMP15:%.*]] = or <16 x i32> [[TMP14]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP9]], <16 x i32> [[TMP15]], <16 x i32> [[TMP11]]
+; CHECK-NEXT:    [[TMP16:%.*]] = select <16 x i1> [[TMP10]], <16 x float> [[TMP8]], <16 x float> zeroinitializer
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x float> [[TMP16]]
+;
+  %res = call <16 x float> @llvm.x86.avx512.mask.vpermilvar.ps.512(<16 x float> %x0, <16 x i32> %x1, <16 x float> zeroinitializer, i16 %x3)
+  ret <16 x float> %res
+}
+
+; Test case to make sure we can print shuffle decode comments for constant pool loads.
+define <16 x float>@test_int_x86_avx512_mask_vpermilvar_ps_512_constant_pool(<16 x float> %x0, <16 x i32> %x1, <16 x float> %x2, i16 %x3)  #0 {
+;
+; CHECK-LABEL: @test_int_x86_avx512_mask_vpermilvar_ps_512_constant_pool(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP4]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP5:%.*]], label [[TMP6:%.*]], !prof [[PROF1]]
+; CHECK:       5:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       6:
+; CHECK-NEXT:    [[TMP7:%.*]] = call <16 x float> @llvm.x86.avx512.vpermilvar.ps.512(<16 x float> [[X0:%.*]], <16 x i32> <i32 2, i32 3, i32 0, i32 1, i32 3, i32 2, i32 1, i32 0, i32 1, i32 0, i32 3, i32 2, i32 0, i32 1, i32 2, i32 3>)
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast i16 [[TMP2]] to <16 x i1>
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast i16 [[X3:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP10:%.*]] = select <16 x i1> [[TMP9]], <16 x i32> zeroinitializer, <16 x i32> [[TMP3]]
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <16 x float> [[TMP7]] to <16 x i32>
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <16 x float> [[X2:%.*]] to <16 x i32>
+; CHECK-NEXT:    [[TMP13:%.*]] = xor <16 x i32> [[TMP11]], [[TMP12]]
+; CHECK-NEXT:    [[TMP14:%.*]] = or <16 x i32> [[TMP13]], zeroinitializer
+; CHECK-NEXT:    [[TMP15:%.*]] = or <16 x i32> [[TMP14]], [[TMP3]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP8]], <16 x i32> [[TMP15]], <16 x i32> [[TMP10]]
+; CHECK-NEXT:    [[TMP16:%.*]] = select <16 x i1> [[TMP9]], <16 x float> [[TMP7]], <16 x float> [[X2]]
+; CHECK-NEXT:    [[TMP17:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP3:%.*]] = icmp ne i512 [[TMP17]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP3]], label [[TMP18:%.*]], label [[TMP19:%.*]], !prof [[PROF1]]
+; CHECK:       18:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       19:
+; CHECK-NEXT:    [[TMP20:%.*]] = call <16 x float> @llvm.x86.avx512.vpermilvar.ps.512(<16 x float> [[X0]], <16 x i32> <i32 1, i32 0, i32 3, i32 2, i32 0, i32 1, i32 2, i32 3, i32 1, i32 0, i32 3, i32 2, i32 0, i32 1, i32 2, i32 3>)
+; CHECK-NEXT:    [[TMP21:%.*]] = bitcast i16 [[TMP2]] to <16 x i1>
+; CHECK-NEXT:    [[TMP22:%.*]] = bitcast i16 [[X3]] to <16 x i1>
+; CHECK-NEXT:    [[TMP23:%.*]] = select <16 x i1> [[TMP22]], <16 x i32> zeroinitializer, <16 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP24:%.*]] = bitcast <16 x float> [[TMP20]] to <16 x i32>
+; CHECK-NEXT:    [[TMP25:%.*]] = xor <16 x i32> [[TMP24]], zeroinitializer
+; CHECK-NEXT:    [[TMP26:%.*]] = or <16 x i32> [[TMP25]], zeroinitializer
+; CHECK-NEXT:    [[TMP27:%.*]] = or <16 x i32> [[TMP26]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT1:%.*]] = select <16 x i1> [[TMP21]], <16 x i32> [[TMP27]], <16 x i32> [[TMP23]]
+; CHECK-NEXT:    [[TMP28:%.*]] = select <16 x i1> [[TMP22]], <16 x float> [[TMP20]], <16 x float> zeroinitializer
+; CHECK-NEXT:    [[TMP29:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP4:%.*]] = icmp ne i512 [[TMP29]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP4]], label [[TMP30:%.*]], label [[TMP31:%.*]], !prof [[PROF1]]
+; CHECK:       30:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       31:
+; CHECK-NEXT:    [[TMP32:%.*]] = call <16 x float> @llvm.x86.avx512.vpermilvar.ps.512(<16 x float> [[X0]], <16 x i32> <i32 1, i32 0, i32 3, i32 2, i32 0, i32 1, i32 2, i32 3, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 1, i32 0>)
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <16 x i32> [[_MSPROP_SELECT]], [[_MSPROP_SELECT1]]
+; CHECK-NEXT:    [[RES3:%.*]] = fadd <16 x float> [[TMP16]], [[TMP28]]
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = or <16 x i32> zeroinitializer, [[_MSPROP]]
+; CHECK-NEXT:    [[RES4:%.*]] = fadd <16 x float> [[TMP32]], [[RES3]]
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP2]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x float> [[RES4]]
+;
+  %res = call <16 x float> @llvm.x86.avx512.mask.vpermilvar.ps.512(<16 x float> %x0, <16 x i32> <i32 2, i32 3, i32 0, i32 1, i32 3, i32 2, i32 1, i32 0, i32 1, i32 0, i32 3, i32 2, i32 0, i32 1, i32 2, i32 3>, <16 x float> %x2, i16 %x3)
+  %res1 = call <16 x float> @llvm.x86.avx512.mask.vpermilvar.ps.512(<16 x float> %x0, <16 x i32> <i32 1, i32 0, i32 3, i32 2, i32 0, i32 1, i32 2, i32 3, i32 1, i32 0, i32 3, i32 2, i32 0, i32 1, i32 2, i32 3>, <16 x float> zeroinitializer, i16 %x3)
+  %res2 = call <16 x float> @llvm.x86.avx512.mask.vpermilvar.ps.512(<16 x float> %x0, <16 x i32> <i32 1, i32 0, i32 3, i32 2, i32 0, i32 1, i32 2, i32 3, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 1, i32 0>, <16 x float> %x2, i16 -1)
+  %res3 = fadd <16 x float> %res, %res1
+  %res4 = fadd <16 x float> %res2, %res3
+  ret <16 x float> %res4
+}
+
+define <8 x i64> @test_mask_mul_epi32_rr(<16 x i32> %a, <16 x i32> %b)  #0 {
+; CHECK-LABEL: @test_mask_mul_epi32_rr(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <16 x i32> [[TMP1]] to <8 x i64>
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <16 x i32> [[A:%.*]] to <8 x i64>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <16 x i32> [[TMP2]] to <8 x i64>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <16 x i32> [[B:%.*]] to <8 x i64>
+; CHECK-NEXT:    [[TMP7:%.*]] = shl <8 x i64> [[TMP3]], splat (i64 32)
+; CHECK-NEXT:    [[TMP8:%.*]] = or <8 x i64> [[TMP7]], zeroinitializer
+; CHECK-NEXT:    [[TMP9:%.*]] = shl <8 x i64> [[TMP4]], splat (i64 32)
+; CHECK-NEXT:    [[TMP10:%.*]] = ashr <8 x i64> [[TMP8]], splat (i64 32)
+; CHECK-NEXT:    [[TMP11:%.*]] = or <8 x i64> [[TMP10]], zeroinitializer
+; CHECK-NEXT:    [[TMP12:%.*]] = ashr <8 x i64> [[TMP9]], splat (i64 32)
+; CHECK-NEXT:    [[TMP13:%.*]] = shl <8 x i64> [[TMP5]], splat (i64 32)
+; CHECK-NEXT:    [[TMP14:%.*]] = or <8 x i64> [[TMP13]], zeroinitializer
+; CHECK-NEXT:    [[TMP15:%.*]] = shl <8 x i64> [[TMP6]], splat (i64 32)
+; CHECK-NEXT:    [[TMP16:%.*]] = ashr <8 x i64> [[TMP14]], splat (i64 32)
+; CHECK-NEXT:    [[TMP17:%.*]] = or <8 x i64> [[TMP16]], zeroinitializer
+; CHECK-NEXT:    [[TMP18:%.*]] = ashr <8 x i64> [[TMP15]], splat (i64 32)
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <8 x i64> [[TMP11]], [[TMP17]]
+; CHECK-NEXT:    [[TMP19:%.*]] = mul <8 x i64> [[TMP12]], [[TMP18]]
+; CHECK-NEXT:    store <8 x i64> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i64> [[TMP19]]
+;
+  %res = call <8 x i64> @llvm.x86.avx512.mask.pmul.dq.512(<16 x i32> %a, <16 x i32> %b, <8 x i64> zeroinitializer, i8 -1)
+  ret < 8 x i64> %res
+}
+
+define <8 x i64> @test_mask_mul_epi32_rrk(<16 x i32> %a, <16 x i32> %b, <8 x i64> %passThru, i8 %mask)  #0 {
+;
+; CHECK-LABEL: @test_mask_mul_epi32_rrk(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <16 x i32> [[TMP1]] to <8 x i64>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <16 x i32> [[A:%.*]] to <8 x i64>
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <16 x i32> [[TMP2]] to <8 x i64>
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <16 x i32> [[B:%.*]] to <8 x i64>
+; CHECK-NEXT:    [[TMP9:%.*]] = shl <8 x i64> [[TMP5]], splat (i64 32)
+; CHECK-NEXT:    [[TMP10:%.*]] = or <8 x i64> [[TMP9]], zeroinitializer
+; CHECK-NEXT:    [[TMP11:%.*]] = shl <8 x i64> [[TMP6]], splat (i64 32)
+; CHECK-NEXT:    [[TMP12:%.*]] = ashr <8 x i64> [[TMP10]], splat (i64 32)
+; CHECK-NEXT:    [[TMP13:%.*]] = or <8 x i64> [[TMP12]], zeroinitializer
+; CHECK-NEXT:    [[TMP14:%.*]] = ashr <8 x i64> [[TMP11]], splat (i64 32)
+; CHECK-NEXT:    [[TMP15:%.*]] = shl <8 x i64> [[TMP7]], splat (i64 32)
+; CHECK-NEXT:    [[TMP16:%.*]] = or <8 x i64> [[TMP15]], zeroinitializer
+; CHECK-NEXT:    [[TMP17:%.*]] = shl <8 x i64> [[TMP8]], splat (i64 32)
+; CHECK-NEXT:    [[TMP18:%.*]] = ashr <8 x i64> [[TMP16]], splat (i64 32)
+; CHECK-NEXT:    [[TMP19:%.*]] = or <8 x i64> [[TMP18]], zeroinitializer
+; CHECK-NEXT:    [[TMP20:%.*]] = ashr <8 x i64> [[TMP17]], splat (i64 32)
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <8 x i64> [[TMP13]], [[TMP19]]
+; CHECK-NEXT:    [[TMP21:%.*]] = mul <8 x i64> [[TMP14]], [[TMP20]]
+; CHECK-NEXT:    [[TMP22:%.*]] = bitcast i8 [[TMP3]] to <8 x i1>
+; CHECK-NEXT:    [[TMP23:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP24:%.*]] = select <8 x i1> [[TMP23]], <8 x i64> [[_MSPROP]], <8 x i64> [[TMP4]]
+; CHECK-NEXT:    [[TMP25:%.*]] = xor <8 x i64> [[TMP21]], [[PASSTHRU:%.*]]
+; CHECK-NEXT:    [[TMP26:%.*]] = or <8 x i64> [[TMP25]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP27:%.*]] = or <8 x i64> [[TMP26]], [[TMP4]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP22]], <8 x i64> [[TMP27]], <8 x i64> [[TMP24]]
+; CHECK-NEXT:    [[TMP28:%.*]] = select <8 x i1> [[TMP23]], <8 x i64> [[TMP21]], <8 x i64> [[PASSTHRU]]
+; CHECK-NEXT:    store <8 x i64> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i64> [[TMP28]]
+;
+  %res = call <8 x i64> @llvm.x86.avx512.mask.pmul.dq.512(<16 x i32> %a, <16 x i32> %b, <8 x i64> %passThru, i8 %mask)
+  ret < 8 x i64> %res
+}
+
+define <8 x i64> @test_mask_mul_epi32_rrkz(<16 x i32> %a, <16 x i32> %b, i8 %mask)  #0 {
+;
+; CHECK-LABEL: @test_mask_mul_epi32_rrkz(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <16 x i32> [[TMP1]] to <8 x i64>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <16 x i32> [[A:%.*]] to <8 x i64>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <16 x i32> [[TMP2]] to <8 x i64>
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <16 x i32> [[B:%.*]] to <8 x i64>
+; CHECK-NEXT:    [[TMP8:%.*]] = shl <8 x i64> [[TMP4]], splat (i64 32)
+; CHECK-NEXT:    [[TMP9:%.*]] = or <8 x i64> [[TMP8]], zeroinitializer
+; CHECK-NEXT:    [[TMP10:%.*]] = shl <8 x i64> [[TMP5]], splat (i64 32)
+; CHECK-NEXT:    [[TMP11:%.*]] = ashr <8 x i64> [[TMP9]], splat (i64 32)
+; CHECK-NEXT:    [[TMP12:%.*]] = or <8 x i64> [[TMP11]], zeroinitializer
+; CHECK-NEXT:    [[TMP13:%.*]] = ashr <8 x i64> [[TMP10]], splat (i64 32)
+; CHECK-NEXT:    [[TMP14:%.*]] = shl <8 x i64> [[TMP6]], splat (i64 32)
+; CHECK-NEXT:    [[TMP15:%.*]] = or <8 x i64> [[TMP14]], zeroinitializer
+; CHECK-NEXT:    [[TMP16:%.*]] = shl <8 x i64> [[TMP7]], splat (i64 32)
+; CHECK-NEXT:    [[TMP17:%.*]] = ashr <8 x i64> [[TMP15]], splat (i64 32)
+; CHECK-NEXT:    [[TMP18:%.*]] = or <8 x i64> [[TMP17]], zeroinitializer
+; CHECK-NEXT:    [[TMP19:%.*]] = ashr <8 x i64> [[TMP16]], splat (i64 32)
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <8 x i64> [[TMP12]], [[TMP18]]
+; CHECK-NEXT:    [[TMP20:%.*]] = mul <8 x i64> [[TMP13]], [[TMP19]]
+; CHECK-NEXT:    [[TMP21:%.*]] = bitcast i8 [[TMP3]] to <8 x i1>
+; CHECK-NEXT:    [[TMP22:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP23:%.*]] = select <8 x i1> [[TMP22]], <8 x i64> [[_MSPROP]], <8 x i64> zeroinitializer
+; CHECK-NEXT:    [[TMP24:%.*]] = xor <8 x i64> [[TMP20]], zeroinitializer
+; CHECK-NEXT:    [[TMP25:%.*]] = or <8 x i64> [[TMP24]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP26:%.*]] = or <8 x i64> [[TMP25]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP21]], <8 x i64> [[TMP26]], <8 x i64> [[TMP23]]
+; CHECK-NEXT:    [[TMP27:%.*]] = select <8 x i1> [[TMP22]], <8 x i64> [[TMP20]], <8 x i64> zeroinitializer
+; CHECK-NEXT:    store <8 x i64> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i64> [[TMP27]]
+;
+  %res = call <8 x i64> @llvm.x86.avx512.mask.pmul.dq.512(<16 x i32> %a, <16 x i32> %b, <8 x i64> zeroinitializer, i8 %mask)
+  ret < 8 x i64> %res
+}
+
+define <8 x i64> @test_mask_mul_epi32_rm(<16 x i32> %a, ptr %ptr_b)  #0 {
+;
+; CHECK-LABEL: @test_mask_mul_epi32_rm(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP3:%.*]], label [[TMP4:%.*]], !prof [[PROF1]]
+; CHECK:       3:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       4:
+; CHECK-NEXT:    [[B:%.*]] = load <16 x i32>, ptr [[PTR_B:%.*]], align 64
+; CHECK-NEXT:    [[TMP5:%.*]] = ptrtoint ptr [[PTR_B]] to i64
+; CHECK-NEXT:    [[TMP6:%.*]] = xor i64 [[TMP5]], 87960930222080
+; CHECK-NEXT:    [[TMP7:%.*]] = inttoptr i64 [[TMP6]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <16 x i32>, ptr [[TMP7]], align 64
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <16 x i32> [[TMP2]] to <8 x i64>
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <16 x i32> [[A:%.*]] to <8 x i64>
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast <16 x i32> [[_MSLD]] to <8 x i64>
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <16 x i32> [[B]] to <8 x i64>
+; CHECK-NEXT:    [[TMP12:%.*]] = shl <8 x i64> [[TMP8]], splat (i64 32)
+; CHECK-NEXT:    [[TMP13:%.*]] = or <8 x i64> [[TMP12]], zeroinitializer
+; CHECK-NEXT:    [[TMP14:%.*]] = shl <8 x i64> [[TMP9]], splat (i64 32)
+; CHECK-NEXT:    [[TMP15:%.*]] = ashr <8 x i64> [[TMP13]], splat (i64 32)
+; CHECK-NEXT:    [[TMP16:%.*]] = or <8 x i64> [[TMP15]], zeroinitializer
+; CHECK-NEXT:    [[TMP17:%.*]] = ashr <8 x i64> [[TMP14]], splat (i64 32)
+; CHECK-NEXT:    [[TMP18:%.*]] = shl <8 x i64> [[TMP10]], splat (i64 32)
+; CHECK-NEXT:    [[TMP19:%.*]] = or <8 x i64> [[TMP18]], zeroinitializer
+; CHECK-NEXT:    [[TMP20:%.*]] = shl <8 x i64> [[TMP11]], splat (i64 32)
+; CHECK-NEXT:    [[TMP21:%.*]] = ashr <8 x i64> [[TMP19]], splat (i64 32)
+; CHECK-NEXT:    [[TMP22:%.*]] = or <8 x i64> [[TMP21]], zeroinitializer
+; CHECK-NEXT:    [[TMP23:%.*]] = ashr <8 x i64> [[TMP20]], splat (i64 32)
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <8 x i64> [[TMP16]], [[TMP22]]
+; CHECK-NEXT:    [[TMP24:%.*]] = mul <8 x i64> [[TMP17]], [[TMP23]]
+; CHECK-NEXT:    store <8 x i64> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i64> [[TMP24]]
+;
+  %b = load <16 x i32>, ptr %ptr_b
+  %res = call <8 x i64> @llvm.x86.avx512.mask.pmul.dq.512(<16 x i32> %a, <16 x i32> %b, <8 x i64> zeroinitializer, i8 -1)
+  ret < 8 x i64> %res
+}
+
+define <8 x i64> @test_mask_mul_epi32_rmk(<16 x i32> %a, ptr %ptr_b, <8 x i64> %passThru, i8 %mask)  #0 {
+;
+; CHECK-LABEL: @test_mask_mul_epi32_rmk(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 136) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 72) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP5:%.*]], label [[TMP6:%.*]], !prof [[PROF1]]
+; CHECK:       5:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       6:
+; CHECK-NEXT:    [[B:%.*]] = load <16 x i32>, ptr [[PTR_B:%.*]], align 64
+; CHECK-NEXT:    [[TMP7:%.*]] = ptrtoint ptr [[PTR_B]] to i64
+; CHECK-NEXT:    [[TMP8:%.*]] = xor i64 [[TMP7]], 87960930222080
+; CHECK-NEXT:    [[TMP9:%.*]] = inttoptr i64 [[TMP8]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <16 x i32>, ptr [[TMP9]], align 64
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast <16 x i32> [[TMP2]] to <8 x i64>
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <16 x i32> [[A:%.*]] to <8 x i64>
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <16 x i32> [[_MSLD]] to <8 x i64>
+; CHECK-NEXT:    [[TMP13:%.*]] = bitcast <16 x i32> [[B]] to <8 x i64>
+; CHECK-NEXT:    [[TMP14:%.*]] = shl <8 x i64> [[TMP10]], splat (i64 32)
+; CHECK-NEXT:    [[TMP15:%.*]] = or <8 x i64> [[TMP14]], zeroinitializer
+; CHECK-NEXT:    [[TMP16:%.*]] = shl <8 x i64> [[TMP11]], splat (i64 32)
+; CHECK-NEXT:    [[TMP17:%.*]] = ashr <8 x i64> [[TMP15]], splat (i64 32)
+; CHECK-NEXT:    [[TMP18:%.*]] = or <8 x i64> [[TMP17]], zeroinitializer
+; CHECK-NEXT:    [[TMP19:%.*]] = ashr <8 x i64> [[TMP16]], splat (i64 32)
+; CHECK-NEXT:    [[TMP20:%.*]] = shl <8 x i64> [[TMP12]], splat (i64 32)
+; CHECK-NEXT:    [[TMP21:%.*]] = or <8 x i64> [[TMP20]], zeroinitializer
+; CHECK-NEXT:    [[TMP22:%.*]] = shl <8 x i64> [[TMP13]], splat (i64 32)
+; CHECK-NEXT:    [[TMP23:%.*]] = ashr <8 x i64> [[TMP21]], splat (i64 32)
+; CHECK-NEXT:    [[TMP24:%.*]] = or <8 x i64> [[TMP23]], zeroinitializer
+; CHECK-NEXT:    [[TMP25:%.*]] = ashr <8 x i64> [[TMP22]], splat (i64 32)
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <8 x i64> [[TMP18]], [[TMP24]]
+; CHECK-NEXT:    [[TMP26:%.*]] = mul <8 x i64> [[TMP19]], [[TMP25]]
+; CHECK-NEXT:    [[TMP27:%.*]] = bitcast i8 [[TMP3]] to <8 x i1>
+; CHECK-NEXT:    [[TMP28:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP29:%.*]] = select <8 x i1> [[TMP28]], <8 x i64> [[_MSPROP]], <8 x i64> [[TMP4]]
+; CHECK-NEXT:    [[TMP30:%.*]] = xor <8 x i64> [[TMP26]], [[PASSTHRU:%.*]]
+; CHECK-NEXT:    [[TMP31:%.*]] = or <8 x i64> [[TMP30]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP32:%.*]] = or <8 x i64> [[TMP31]], [[TMP4]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP27]], <8 x i64> [[TMP32]], <8 x i64> [[TMP29]]
+; CHECK-NEXT:    [[TMP33:%.*]] = select <8 x i1> [[TMP28]], <8 x i64> [[TMP26]], <8 x i64> [[PASSTHRU]]
+; CHECK-NEXT:    store <8 x i64> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i64> [[TMP33]]
+;
+  %b = load <16 x i32>, ptr %ptr_b
+  %res = call <8 x i64> @llvm.x86.avx512.mask.pmul.dq.512(<16 x i32> %a, <16 x i32> %b, <8 x i64> %passThru, i8 %mask)
+  ret < 8 x i64> %res
+}
+
+define <8 x i64> @test_mask_mul_epi32_rmkz(<16 x i32> %a, ptr %ptr_b, i8 %mask)  #0 {
+;
+; CHECK-LABEL: @test_mask_mul_epi32_rmkz(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 72) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP4:%.*]], label [[TMP5:%.*]], !prof [[PROF1]]
+; CHECK:       4:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       5:
+; CHECK-NEXT:    [[B:%.*]] = load <16 x i32>, ptr [[PTR_B:%.*]], align 64
+; CHECK-NEXT:    [[TMP6:%.*]] = ptrtoint ptr [[PTR_B]] to i64
+; CHECK-NEXT:    [[TMP7:%.*]] = xor i64 [[TMP6]], 87960930222080
+; CHECK-NEXT:    [[TMP8:%.*]] = inttoptr i64 [[TMP7]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <16 x i32>, ptr [[TMP8]], align 64
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <16 x i32> [[TMP2]] to <8 x i64>
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast <16 x i32> [[A:%.*]] to <8 x i64>
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <16 x i32> [[_MSLD]] to <8 x i64>
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <16 x i32> [[B]] to <8 x i64>
+; CHECK-NEXT:    [[TMP13:%.*]] = shl <8 x i64> [[TMP9]], splat (i64 32)
+; CHECK-NEXT:    [[TMP14:%.*]] = or <8 x i64> [[TMP13]], zeroinitializer
+; CHECK-NEXT:    [[TMP15:%.*]] = shl <8 x i64> [[TMP10]], splat (i64 32)
+; CHECK-NEXT:    [[TMP16:%.*]] = ashr <8 x i64> [[TMP14]], splat (i64 32)
+; CHECK-NEXT:    [[TMP17:%.*]] = or <8 x i64> [[TMP16]], zeroinitializer
+; CHECK-NEXT:    [[TMP18:%.*]] = ashr <8 x i64> [[TMP15]], splat (i64 32)
+; CHECK-NEXT:    [[TMP19:%.*]] = shl <8 x i64> [[TMP11]], splat (i64 32)
+; CHECK-NEXT:    [[TMP20:%.*]] = or <8 x i64> [[TMP19]], zeroinitializer
+; CHECK-NEXT:    [[TMP21:%.*]] = shl <8 x i64> [[TMP12]], splat (i64 32)
+; CHECK-NEXT:    [[TMP22:%.*]] = ashr <8 x i64> [[TMP20]], splat (i64 32)
+; CHECK-NEXT:    [[TMP23:%.*]] = or <8 x i64> [[TMP22]], zeroinitializer
+; CHECK-NEXT:    [[TMP24:%.*]] = ashr <8 x i64> [[TMP21]], splat (i64 32)
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <8 x i64> [[TMP17]], [[TMP23]]
+; CHECK-NEXT:    [[TMP25:%.*]] = mul <8 x i64> [[TMP18]], [[TMP24]]
+; CHECK-NEXT:    [[TMP26:%.*]] = bitcast i8 [[TMP3]] to <8 x i1>
+; CHECK-NEXT:    [[TMP27:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP28:%.*]] = select <8 x i1> [[TMP27]], <8 x i64> [[_MSPROP]], <8 x i64> zeroinitializer
+; CHECK-NEXT:    [[TMP29:%.*]] = xor <8 x i64> [[TMP25]], zeroinitializer
+; CHECK-NEXT:    [[TMP30:%.*]] = or <8 x i64> [[TMP29]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP31:%.*]] = or <8 x i64> [[TMP30]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP26]], <8 x i64> [[TMP31]], <8 x i64> [[TMP28]]
+; CHECK-NEXT:    [[TMP32:%.*]] = select <8 x i1> [[TMP27]], <8 x i64> [[TMP25]], <8 x i64> zeroinitializer
+; CHECK-NEXT:    store <8 x i64> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i64> [[TMP32]]
+;
+  %b = load <16 x i32>, ptr %ptr_b
+  %res = call <8 x i64> @llvm.x86.avx512.mask.pmul.dq.512(<16 x i32> %a, <16 x i32> %b, <8 x i64> zeroinitializer, i8 %mask)
+  ret < 8 x i64> %res
+}
+
+define <8 x i64> @test_mask_mul_epi32_rmb(<16 x i32> %a, ptr %ptr_b, <8 x i64> %extra_param, <8 x i64> %extra_param2)  #0 {
+;
+; CHECK-LABEL: @test_mask_mul_epi32_rmb(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 72) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 136) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP26:%.*]], label [[TMP27:%.*]], !prof [[PROF1]]
+; CHECK:       5:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       6:
+; CHECK-NEXT:    [[Q:%.*]] = load i64, ptr [[PTR_B:%.*]], align 8
+; CHECK-NEXT:    [[TMP5:%.*]] = ptrtoint ptr [[PTR_B]] to i64
+; CHECK-NEXT:    [[TMP6:%.*]] = xor i64 [[TMP5]], 87960930222080
+; CHECK-NEXT:    [[TMP7:%.*]] = inttoptr i64 [[TMP6]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load i64, ptr [[TMP7]], align 8
+; CHECK-NEXT:    [[_MSPROP:%.*]] = insertelement <8 x i64> [[TMP3]], i64 [[_MSLD]], i32 0
+; CHECK-NEXT:    [[VECINIT_I:%.*]] = insertelement <8 x i64> [[EXTRA_PARAM:%.*]], i64 [[Q]], i32 0
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = shufflevector <8 x i64> [[_MSPROP]], <8 x i64> [[TMP4]], <8 x i32> zeroinitializer
+; CHECK-NEXT:    [[B64:%.*]] = shufflevector <8 x i64> [[VECINIT_I]], <8 x i64> [[EXTRA_PARAM2:%.*]], <8 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <8 x i64> [[_MSPROP1]] to <16 x i32>
+; CHECK-NEXT:    [[B:%.*]] = bitcast <8 x i64> [[B64]] to <16 x i32>
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <16 x i32> [[TMP2]] to <8 x i64>
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast <16 x i32> [[A:%.*]] to <8 x i64>
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <16 x i32> [[TMP8]] to <8 x i64>
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <16 x i32> [[B]] to <8 x i64>
+; CHECK-NEXT:    [[TMP13:%.*]] = shl <8 x i64> [[TMP9]], splat (i64 32)
+; CHECK-NEXT:    [[TMP14:%.*]] = or <8 x i64> [[TMP13]], zeroinitializer
+; CHECK-NEXT:    [[TMP15:%.*]] = shl <8 x i64> [[TMP10]], splat (i64 32)
+; CHECK-NEXT:    [[TMP16:%.*]] = ashr <8 x i64> [[TMP14]], splat (i64 32)
+; CHECK-NEXT:    [[TMP17:%.*]] = or <8 x i64> [[TMP16]], zeroinitializer
+; CHECK-NEXT:    [[TMP18:%.*]] = ashr <8 x i64> [[TMP15]], splat (i64 32)
+; CHECK-NEXT:    [[TMP19:%.*]] = shl <8 x i64> [[TMP11]], splat (i64 32)
+; CHECK-NEXT:    [[TMP20:%.*]] = or <8 x i64> [[TMP19]], zeroinitializer
+; CHECK-NEXT:    [[TMP21:%.*]] = shl <8 x i64> [[TMP12]], splat (i64 32)
+; CHECK-NEXT:    [[TMP22:%.*]] = ashr <8 x i64> [[TMP20]], splat (i64 32)
+; CHECK-NEXT:    [[TMP23:%.*]] = or <8 x i64> [[TMP22]], zeroinitializer
+; CHECK-NEXT:    [[TMP24:%.*]] = ashr <8 x i64> [[TMP21]], splat (i64 32)
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = or <8 x i64> [[TMP17]], [[TMP23]]
+; CHECK-NEXT:    [[TMP25:%.*]] = mul <8 x i64> [[TMP18]], [[TMP24]]
+; CHECK-NEXT:    store <8 x i64> [[_MSPROP2]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i64> [[TMP25]]
+;
+  %q = load i64, ptr %ptr_b
+  %vecinit.i = insertelement <8 x i64> %extra_param, i64 %q, i32 0
+  %b64 = shufflevector <8 x i64> %vecinit.i, <8 x i64> %extra_param2, <8 x i32> zeroinitializer
+  %b = bitcast <8 x i64> %b64 to <16 x i32>
+  %res = call <8 x i64> @llvm.x86.avx512.mask.pmul.dq.512(<16 x i32> %a, <16 x i32> %b, <8 x i64> zeroinitializer, i8 -1)
+  ret < 8 x i64> %res
+}
+
+define <8 x i64> @test_mask_mul_epi32_rmbk(<16 x i32> %a, ptr %ptr_b, <8 x i64> %passThru, i8 %mask, <8 x i64> %extra_param, <8 x i64> %extra_param2)  #0 {
+;
+; CHECK-LABEL: @test_mask_mul_epi32_rmbk(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP5:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 144) to ptr), align 8
+; CHECK-NEXT:    [[TMP6:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 208) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 136) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 72) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP35:%.*]], label [[TMP36:%.*]], !prof [[PROF1]]
+; CHECK:       7:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       8:
+; CHECK-NEXT:    [[Q:%.*]] = load i64, ptr [[PTR_B:%.*]], align 8
+; CHECK-NEXT:    [[TMP7:%.*]] = ptrtoint ptr [[PTR_B]] to i64
+; CHECK-NEXT:    [[TMP8:%.*]] = xor i64 [[TMP7]], 87960930222080
+; CHECK-NEXT:    [[TMP9:%.*]] = inttoptr i64 [[TMP8]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load i64, ptr [[TMP9]], align 8
+; CHECK-NEXT:    [[_MSPROP:%.*]] = insertelement <8 x i64> [[TMP5]], i64 [[_MSLD]], i32 0
+; CHECK-NEXT:    [[VECINIT_I:%.*]] = insertelement <8 x i64> [[EXTRA_PARAM:%.*]], i64 [[Q]], i32 0
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = shufflevector <8 x i64> [[_MSPROP]], <8 x i64> [[TMP6]], <8 x i32> zeroinitializer
+; CHECK-NEXT:    [[B64:%.*]] = shufflevector <8 x i64> [[VECINIT_I]], <8 x i64> [[EXTRA_PARAM2:%.*]], <8 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast <8 x i64> [[_MSPROP1]] to <16 x i32>
+; CHECK-NEXT:    [[B:%.*]] = bitcast <8 x i64> [[B64]] to <16 x i32>
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <16 x i32> [[TMP2]] to <8 x i64>
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <16 x i32> [[A:%.*]] to <8 x i64>
+; CHECK-NEXT:    [[TMP13:%.*]] = bitcast <16 x i32> [[TMP10]] to <8 x i64>
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast <16 x i32> [[B]] to <8 x i64>
+; CHECK-NEXT:    [[TMP15:%.*]] = shl <8 x i64> [[TMP11]], splat (i64 32)
+; CHECK-NEXT:    [[TMP16:%.*]] = or <8 x i64> [[TMP15]], zeroinitializer
+; CHECK-NEXT:    [[TMP17:%.*]] = shl <8 x i64> [[TMP12]], splat (i64 32)
+; CHECK-NEXT:    [[TMP18:%.*]] = ashr <8 x i64> [[TMP16]], splat (i64 32)
+; CHECK-NEXT:    [[TMP19:%.*]] = or <8 x i64> [[TMP18]], zeroinitializer
+; CHECK-NEXT:    [[TMP20:%.*]] = ashr <8 x i64> [[TMP17]], splat (i64 32)
+; CHECK-NEXT:    [[TMP21:%.*]] = shl <8 x i64> [[TMP13]], splat (i64 32)
+; CHECK-NEXT:    [[TMP22:%.*]] = or <8 x i64> [[TMP21]], zeroinitializer
+; CHECK-NEXT:    [[TMP23:%.*]] = shl <8 x i64> [[TMP14]], splat (i64 32)
+; CHECK-NEXT:    [[TMP24:%.*]] = ashr <8 x i64> [[TMP22]], splat (i64 32)
+; CHECK-NEXT:    [[TMP25:%.*]] = or <8 x i64> [[TMP24]], zeroinitializer
+; CHECK-NEXT:    [[TMP26:%.*]] = ashr <8 x i64> [[TMP23]], splat (i64 32)
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = or <8 x i64> [[TMP19]], [[TMP25]]
+; CHECK-NEXT:    [[TMP27:%.*]] = mul <8 x i64> [[TMP20]], [[TMP26]]
+; CHECK-NEXT:    [[TMP28:%.*]] = bitcast i8 [[TMP3]] to <8 x i1>
+; CHECK-NEXT:    [[TMP29:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP30:%.*]] = select <8 x i1> [[TMP29]], <8 x i64> [[_MSPROP2]], <8 x i64> [[TMP4]]
+; CHECK-NEXT:    [[TMP31:%.*]] = xor <8 x i64> [[TMP27]], [[PASSTHRU:%.*]]
+; CHECK-NEXT:    [[TMP32:%.*]] = or <8 x i64> [[TMP31]], [[_MSPROP2]]
+; CHECK-NEXT:    [[TMP33:%.*]] = or <8 x i64> [[TMP32]], [[TMP4]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP28]], <8 x i64> [[TMP33]], <8 x i64> [[TMP30]]
+; CHECK-NEXT:    [[TMP34:%.*]] = select <8 x i1> [[TMP29]], <8 x i64> [[TMP27]], <8 x i64> [[PASSTHRU]]
+; CHECK-NEXT:    store <8 x i64> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i64> [[TMP34]]
+;
+  %q = load i64, ptr %ptr_b
+  %vecinit.i = insertelement <8 x i64> %extra_param, i64 %q, i32 0
+  %b64 = shufflevector <8 x i64> %vecinit.i, <8 x i64> %extra_param2, <8 x i32> zeroinitializer
+  %b = bitcast <8 x i64> %b64 to <16 x i32>
+  %res = call <8 x i64> @llvm.x86.avx512.mask.pmul.dq.512(<16 x i32> %a, <16 x i32> %b, <8 x i64> %passThru, i8 %mask)
+  ret < 8 x i64> %res
+}
+
+define <8 x i64> @test_mask_mul_epi32_rmbk_buildvector(<16 x i32> %a, ptr %ptr_b, <8 x i64> %passThru, i8 %mask, <8 x i64> %extra_param)  #0 {
+;
+; CHECK-LABEL: @test_mask_mul_epi32_rmbk_buildvector(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP5:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 144) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 136) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 72) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP6:%.*]], label [[TMP35:%.*]], !prof [[PROF1]]
+; CHECK:       6:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       7:
+; CHECK-NEXT:    [[Q:%.*]] = load i64, ptr [[PTR_B:%.*]], align 8
+; CHECK-NEXT:    [[TMP7:%.*]] = ptrtoint ptr [[PTR_B]] to i64
+; CHECK-NEXT:    [[TMP8:%.*]] = xor i64 [[TMP7]], 87960930222080
+; CHECK-NEXT:    [[TMP9:%.*]] = inttoptr i64 [[TMP8]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load i64, ptr [[TMP9]], align 8
+; CHECK-NEXT:    [[_MSPROP:%.*]] = insertelement <8 x i64> [[TMP5]], i64 [[_MSLD]], i32 0
+; CHECK-NEXT:    [[VECINIT_I:%.*]] = insertelement <8 x i64> [[EXTRA_PARAM:%.*]], i64 [[Q]], i32 0
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = insertelement <8 x i64> [[_MSPROP]], i64 [[_MSLD]], i32 1
+; CHECK-NEXT:    [[VECINIT_I1:%.*]] = insertelement <8 x i64> [[VECINIT_I]], i64 [[Q]], i32 1
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = insertelement <8 x i64> [[_MSPROP1]], i64 [[_MSLD]], i32 2
+; CHECK-NEXT:    [[VECINIT_I2:%.*]] = insertelement <8 x i64> [[VECINIT_I1]], i64 [[Q]], i32 2
+; CHECK-NEXT:    [[_MSPROP3:%.*]] = insertelement <8 x i64> [[_MSPROP2]], i64 [[_MSLD]], i32 3
+; CHECK-NEXT:    [[VECINIT_I3:%.*]] = insertelement <8 x i64> [[VECINIT_I2]], i64 [[Q]], i32 3
+; CHECK-NEXT:    [[_MSPROP4:%.*]] = insertelement <8 x i64> [[_MSPROP3]], i64 [[_MSLD]], i32 4
+; CHECK-NEXT:    [[VECINIT_I4:%.*]] = insertelement <8 x i64> [[VECINIT_I3]], i64 [[Q]], i32 4
+; CHECK-NEXT:    [[_MSPROP5:%.*]] = insertelement <8 x i64> [[_MSPROP4]], i64 [[_MSLD]], i32 5
+; CHECK-NEXT:    [[VECINIT_I5:%.*]] = insertelement <8 x i64> [[VECINIT_I4]], i64 [[Q]], i32 5
+; CHECK-NEXT:    [[_MSPROP6:%.*]] = insertelement <8 x i64> [[_MSPROP5]], i64 [[_MSLD]], i32 6
+; CHECK-NEXT:    [[VECINIT_I6:%.*]] = insertelement <8 x i64> [[VECINIT_I5]], i64 [[Q]], i32 6
+; CHECK-NEXT:    [[_MSPROP7:%.*]] = insertelement <8 x i64> [[_MSPROP6]], i64 [[_MSLD]], i32 7
+; CHECK-NEXT:    [[B64:%.*]] = insertelement <8 x i64> [[VECINIT_I6]], i64 [[Q]], i32 7
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast <8 x i64> [[_MSPROP7]] to <16 x i32>
+; CHECK-NEXT:    [[B:%.*]] = bitcast <8 x i64> [[B64]] to <16 x i32>
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <16 x i32> [[TMP2]] to <8 x i64>
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <16 x i32> [[A:%.*]] to <8 x i64>
+; CHECK-NEXT:    [[TMP13:%.*]] = bitcast <16 x i32> [[TMP10]] to <8 x i64>
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast <16 x i32> [[B]] to <8 x i64>
+; CHECK-NEXT:    [[TMP15:%.*]] = shl <8 x i64> [[TMP11]], splat (i64 32)
+; CHECK-NEXT:    [[TMP16:%.*]] = or <8 x i64> [[TMP15]], zeroinitializer
+; CHECK-NEXT:    [[TMP17:%.*]] = shl <8 x i64> [[TMP12]], splat (i64 32)
+; CHECK-NEXT:    [[TMP18:%.*]] = ashr <8 x i64> [[TMP16]], splat (i64 32)
+; CHECK-NEXT:    [[TMP19:%.*]] = or <8 x i64> [[TMP18]], zeroinitializer
+; CHECK-NEXT:    [[TMP20:%.*]] = ashr <8 x i64> [[TMP17]], splat (i64 32)
+; CHECK-NEXT:    [[TMP21:%.*]] = shl <8 x i64> [[TMP13]], splat (i64 32)
+; CHECK-NEXT:    [[TMP22:%.*]] = or <8 x i64> [[TMP21]], zeroinitializer
+; CHECK-NEXT:    [[TMP23:%.*]] = shl <8 x i64> [[TMP14]], splat (i64 32)
+; CHECK-NEXT:    [[TMP24:%.*]] = ashr <8 x i64> [[TMP22]], splat (i64 32)
+; CHECK-NEXT:    [[TMP25:%.*]] = or <8 x i64> [[TMP24]], zeroinitializer
+; CHECK-NEXT:    [[TMP26:%.*]] = ashr <8 x i64> [[TMP23]], splat (i64 32)
+; CHECK-NEXT:    [[_MSPROP8:%.*]] = or <8 x i64> [[TMP19]], [[TMP25]]
+; CHECK-NEXT:    [[TMP27:%.*]] = mul <8 x i64> [[TMP20]], [[TMP26]]
+; CHECK-NEXT:    [[TMP28:%.*]] = bitcast i8 [[TMP3]] to <8 x i1>
+; CHECK-NEXT:    [[TMP29:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP30:%.*]] = select <8 x i1> [[TMP29]], <8 x i64> [[_MSPROP8]], <8 x i64> [[TMP4]]
+; CHECK-NEXT:    [[TMP31:%.*]] = xor <8 x i64> [[TMP27]], [[PASSTHRU:%.*]]
+; CHECK-NEXT:    [[TMP32:%.*]] = or <8 x i64> [[TMP31]], [[_MSPROP8]]
+; CHECK-NEXT:    [[TMP33:%.*]] = or <8 x i64> [[TMP32]], [[TMP4]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP28]], <8 x i64> [[TMP33]], <8 x i64> [[TMP30]]
+; CHECK-NEXT:    [[TMP34:%.*]] = select <8 x i1> [[TMP29]], <8 x i64> [[TMP27]], <8 x i64> [[PASSTHRU]]
+; CHECK-NEXT:    store <8 x i64> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i64> [[TMP34]]
+;
+  %q = load i64, ptr %ptr_b
+  %vecinit.i = insertelement < 8 x i64> %extra_param, i64 %q, i32 0
+  %vecinit.i1 = insertelement < 8 x i64> %vecinit.i, i64 %q, i32 1
+  %vecinit.i2 = insertelement < 8 x i64> %vecinit.i1, i64 %q, i32 2
+  %vecinit.i3 = insertelement < 8 x i64> %vecinit.i2, i64 %q, i32 3
+  %vecinit.i4 = insertelement < 8 x i64> %vecinit.i3, i64 %q, i32 4
+  %vecinit.i5 = insertelement < 8 x i64> %vecinit.i4, i64 %q, i32 5
+  %vecinit.i6 = insertelement < 8 x i64> %vecinit.i5, i64 %q, i32 6
+  %b64 = insertelement < 8 x i64> %vecinit.i6, i64 %q, i32 7
+  %b = bitcast <8 x i64> %b64 to <16 x i32>
+  %res = call <8 x i64> @llvm.x86.avx512.mask.pmul.dq.512(<16 x i32> %a, <16 x i32> %b, <8 x i64> %passThru, i8 %mask)
+  ret < 8 x i64> %res
+}
+
+define <8 x i64> @test_mask_mul_epi32_rmbkz(<16 x i32> %a, ptr %ptr_b, i8 %mask, <8 x i64> %extra_param, <8 x i64> %extra_param2)  #0 {
+;
+; CHECK-LABEL: @test_mask_mul_epi32_rmbkz(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 80) to ptr), align 8
+; CHECK-NEXT:    [[TMP5:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 144) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 72) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP34:%.*]], label [[TMP35:%.*]], !prof [[PROF1]]
+; CHECK:       6:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       7:
+; CHECK-NEXT:    [[Q:%.*]] = load i64, ptr [[PTR_B:%.*]], align 8
+; CHECK-NEXT:    [[TMP6:%.*]] = ptrtoint ptr [[PTR_B]] to i64
+; CHECK-NEXT:    [[TMP7:%.*]] = xor i64 [[TMP6]], 87960930222080
+; CHECK-NEXT:    [[TMP8:%.*]] = inttoptr i64 [[TMP7]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load i64, ptr [[TMP8]], align 8
+; CHECK-NEXT:    [[_MSPROP:%.*]] = insertelement <8 x i64> [[TMP4]], i64 [[_MSLD]], i32 0
+; CHECK-NEXT:    [[VECINIT_I:%.*]] = insertelement <8 x i64> [[EXTRA_PARAM:%.*]], i64 [[Q]], i32 0
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = shufflevector <8 x i64> [[_MSPROP]], <8 x i64> [[TMP5]], <8 x i32> zeroinitializer
+; CHECK-NEXT:    [[B64:%.*]] = shufflevector <8 x i64> [[VECINIT_I]], <8 x i64> [[EXTRA_PARAM2:%.*]], <8 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <8 x i64> [[_MSPROP1]] to <16 x i32>
+; CHECK-NEXT:    [[B:%.*]] = bitcast <8 x i64> [[B64]] to <16 x i32>
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast <16 x i32> [[TMP2]] to <8 x i64>
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <16 x i32> [[A:%.*]] to <8 x i64>
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <16 x i32> [[TMP9]] to <8 x i64>
+; CHECK-NEXT:    [[TMP13:%.*]] = bitcast <16 x i32> [[B]] to <8 x i64>
+; CHECK-NEXT:    [[TMP14:%.*]] = shl <8 x i64> [[TMP10]], splat (i64 32)
+; CHECK-NEXT:    [[TMP15:%.*]] = or <8 x i64> [[TMP14]], zeroinitializer
+; CHECK-NEXT:    [[TMP16:%.*]] = shl <8 x i64> [[TMP11]], splat (i64 32)
+; CHECK-NEXT:    [[TMP17:%.*]] = ashr <8 x i64> [[TMP15]], splat (i64 32)
+; CHECK-NEXT:    [[TMP18:%.*]] = or <8 x i64> [[TMP17]], zeroinitializer
+; CHECK-NEXT:    [[TMP19:%.*]] = ashr <8 x i64> [[TMP16]], splat (i64 32)
+; CHECK-NEXT:    [[TMP20:%.*]] = shl <8 x i64> [[TMP12]], splat (i64 32)
+; CHECK-NEXT:    [[TMP21:%.*]] = or <8 x i64> [[TMP20]], zeroinitializer
+; CHECK-NEXT:    [[TMP22:%.*]] = shl <8 x i64> [[TMP13]], splat (i64 32)
+; CHECK-NEXT:    [[TMP23:%.*]] = ashr <8 x i64> [[TMP21]], splat (i64 32)
+; CHECK-NEXT:    [[TMP24:%.*]] = or <8 x i64> [[TMP23]], zeroinitializer
+; CHECK-NEXT:    [[TMP25:%.*]] = ashr <8 x i64> [[TMP22]], splat (i64 32)
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = or <8 x i64> [[TMP18]], [[TMP24]]
+; CHECK-NEXT:    [[TMP26:%.*]] = mul <8 x i64> [[TMP19]], [[TMP25]]
+; CHECK-NEXT:    [[TMP27:%.*]] = bitcast i8 [[TMP3]] to <8 x i1>
+; CHECK-NEXT:    [[TMP28:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP29:%.*]] = select <8 x i1> [[TMP28]], <8 x i64> [[_MSPROP2]], <8 x i64> zeroinitializer
+; CHECK-NEXT:    [[TMP30:%.*]] = xor <8 x i64> [[TMP26]], zeroinitializer
+; CHECK-NEXT:    [[TMP31:%.*]] = or <8 x i64> [[TMP30]], [[_MSPROP2]]
+; CHECK-NEXT:    [[TMP32:%.*]] = or <8 x i64> [[TMP31]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP27]], <8 x i64> [[TMP32]], <8 x i64> [[TMP29]]
+; CHECK-NEXT:    [[TMP33:%.*]] = select <8 x i1> [[TMP28]], <8 x i64> [[TMP26]], <8 x i64> zeroinitializer
+; CHECK-NEXT:    store <8 x i64> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i64> [[TMP33]]
+;
+  %q = load i64, ptr %ptr_b
+  %vecinit.i = insertelement <8 x i64> %extra_param, i64 %q, i32 0
+  %b64 = shufflevector <8 x i64> %vecinit.i, <8 x i64> %extra_param2, <8 x i32> zeroinitializer
+  %b = bitcast <8 x i64> %b64 to <16 x i32>
+  %res = call <8 x i64> @llvm.x86.avx512.mask.pmul.dq.512(<16 x i32> %a, <16 x i32> %b, <8 x i64> zeroinitializer, i8 %mask)
+  ret < 8 x i64> %res
+}
+
+define <8 x i64> @test_mask_mul_epi32_rmbkz_buildvector(<16 x i32> %a, ptr %ptr_b, i8 %mask, <8 x i64> %extra_param)  #0 {
+;
+; CHECK-LABEL: @test_mask_mul_epi32_rmbkz_buildvector(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 80) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 72) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP5:%.*]], label [[TMP34:%.*]], !prof [[PROF1]]
+; CHECK:       5:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       6:
+; CHECK-NEXT:    [[Q:%.*]] = load i64, ptr [[PTR_B:%.*]], align 8
+; CHECK-NEXT:    [[TMP6:%.*]] = ptrtoint ptr [[PTR_B]] to i64
+; CHECK-NEXT:    [[TMP7:%.*]] = xor i64 [[TMP6]], 87960930222080
+; CHECK-NEXT:    [[TMP8:%.*]] = inttoptr i64 [[TMP7]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load i64, ptr [[TMP8]], align 8
+; CHECK-NEXT:    [[_MSPROP:%.*]] = insertelement <8 x i64> [[TMP4]], i64 [[_MSLD]], i32 0
+; CHECK-NEXT:    [[VECINIT_I:%.*]] = insertelement <8 x i64> [[EXTRA_PARAM:%.*]], i64 [[Q]], i32 0
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = insertelement <8 x i64> [[_MSPROP]], i64 [[_MSLD]], i32 1
+; CHECK-NEXT:    [[VECINIT_I1:%.*]] = insertelement <8 x i64> [[VECINIT_I]], i64 [[Q]], i32 1
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = insertelement <8 x i64> [[_MSPROP1]], i64 [[_MSLD]], i32 2
+; CHECK-NEXT:    [[VECINIT_I2:%.*]] = insertelement <8 x i64> [[VECINIT_I1]], i64 [[Q]], i32 2
+; CHECK-NEXT:    [[_MSPROP3:%.*]] = insertelement <8 x i64> [[_MSPROP2]], i64 [[_MSLD]], i32 3
+; CHECK-NEXT:    [[VECINIT_I3:%.*]] = insertelement <8 x i64> [[VECINIT_I2]], i64 [[Q]], i32 3
+; CHECK-NEXT:    [[_MSPROP4:%.*]] = insertelement <8 x i64> [[_MSPROP3]], i64 [[_MSLD]], i32 4
+; CHECK-NEXT:    [[VECINIT_I4:%.*]] = insertelement <8 x i64> [[VECINIT_I3]], i64 [[Q]], i32 4
+; CHECK-NEXT:    [[_MSPROP5:%.*]] = insertelement <8 x i64> [[_MSPROP4]], i64 [[_MSLD]], i32 5
+; CHECK-NEXT:    [[VECINIT_I5:%.*]] = insertelement <8 x i64> [[VECINIT_I4]], i64 [[Q]], i32 5
+; CHECK-NEXT:    [[_MSPROP6:%.*]] = insertelement <8 x i64> [[_MSPROP5]], i64 [[_MSLD]], i32 6
+; CHECK-NEXT:    [[VECINIT_I6:%.*]] = insertelement <8 x i64> [[VECINIT_I5]], i64 [[Q]], i32 6
+; CHECK-NEXT:    [[_MSPROP7:%.*]] = insertelement <8 x i64> [[_MSPROP6]], i64 [[_MSLD]], i32 7
+; CHECK-NEXT:    [[B64:%.*]] = insertelement <8 x i64> [[VECINIT_I6]], i64 [[Q]], i32 7
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <8 x i64> [[_MSPROP7]] to <16 x i32>
+; CHECK-NEXT:    [[B:%.*]] = bitcast <8 x i64> [[B64]] to <16 x i32>
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast <16 x i32> [[TMP2]] to <8 x i64>
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <16 x i32> [[A:%.*]] to <8 x i64>
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <16 x i32> [[TMP9]] to <8 x i64>
+; CHECK-NEXT:    [[TMP13:%.*]] = bitcast <16 x i32> [[B]] to <8 x i64>
+; CHECK-NEXT:    [[TMP14:%.*]] = shl <8 x i64> [[TMP10]], splat (i64 32)
+; CHECK-NEXT:    [[TMP15:%.*]] = or <8 x i64> [[TMP14]], zeroinitializer
+; CHECK-NEXT:    [[TMP16:%.*]] = shl <8 x i64> [[TMP11]], splat (i64 32)
+; CHECK-NEXT:    [[TMP17:%.*]] = ashr <8 x i64> [[TMP15]], splat (i64 32)
+; CHECK-NEXT:    [[TMP18:%.*]] = or <8 x i64> [[TMP17]], zeroinitializer
+; CHECK-NEXT:    [[TMP19:%.*]] = ashr <8 x i64> [[TMP16]], splat (i64 32)
+; CHECK-NEXT:    [[TMP20:%.*]] = shl <8 x i64> [[TMP12]], splat (i64 32)
+; CHECK-NEXT:    [[TMP21:%.*]] = or <8 x i64> [[TMP20]], zeroinitializer
+; CHECK-NEXT:    [[TMP22:%.*]] = shl <8 x i64> [[TMP13]], splat (i64 32)
+; CHECK-NEXT:    [[TMP23:%.*]] = ashr <8 x i64> [[TMP21]], splat (i64 32)
+; CHECK-NEXT:    [[TMP24:%.*]] = or <8 x i64> [[TMP23]], zeroinitializer
+; CHECK-NEXT:    [[TMP25:%.*]] = ashr <8 x i64> [[TMP22]], splat (i64 32)
+; CHECK-NEXT:    [[_MSPROP8:%.*]] = or <8 x i64> [[TMP18]], [[TMP24]]
+; CHECK-NEXT:    [[TMP26:%.*]] = mul <8 x i64> [[TMP19]], [[TMP25]]
+; CHECK-NEXT:    [[TMP27:%.*]] = bitcast i8 [[TMP3]] to <8 x i1>
+; CHECK-NEXT:    [[TMP28:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP29:%.*]] = select <8 x i1> [[TMP28]], <8 x i64> [[_MSPROP8]], <8 x i64> zeroinitializer
+; CHECK-NEXT:    [[TMP30:%.*]] = xor <8 x i64> [[TMP26]], zeroinitializer
+; CHECK-NEXT:    [[TMP31:%.*]] = or <8 x i64> [[TMP30]], [[_MSPROP8]]
+; CHECK-NEXT:    [[TMP32:%.*]] = or <8 x i64> [[TMP31]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP27]], <8 x i64> [[TMP32]], <8 x i64> [[TMP29]]
+; CHECK-NEXT:    [[TMP33:%.*]] = select <8 x i1> [[TMP28]], <8 x i64> [[TMP26]], <8 x i64> zeroinitializer
+; CHECK-NEXT:    store <8 x i64> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i64> [[TMP33]]
+;
+  %q = load i64, ptr %ptr_b
+  %vecinit.i = insertelement < 8 x i64> %extra_param, i64 %q, i32 0
+  %vecinit.i1 = insertelement < 8 x i64> %vecinit.i, i64 %q, i32 1
+  %vecinit.i2 = insertelement < 8 x i64> %vecinit.i1, i64 %q, i32 2
+  %vecinit.i3 = insertelement < 8 x i64> %vecinit.i2, i64 %q, i32 3
+  %vecinit.i4 = insertelement < 8 x i64> %vecinit.i3, i64 %q, i32 4
+  %vecinit.i5 = insertelement < 8 x i64> %vecinit.i4, i64 %q, i32 5
+  %vecinit.i6 = insertelement < 8 x i64> %vecinit.i5, i64 %q, i32 6
+  %b64 = insertelement < 8 x i64> %vecinit.i6, i64 %q, i32 7
+  %b = bitcast <8 x i64> %b64 to <16 x i32>
+  %res = call <8 x i64> @llvm.x86.avx512.mask.pmul.dq.512(<16 x i32> %a, <16 x i32> %b, <8 x i64> zeroinitializer, i8 %mask)
+  ret < 8 x i64> %res
+}
+
+declare <8 x i64> @llvm.x86.avx512.mask.pmul.dq.512(<16 x i32>, <16 x i32>, <8 x i64>, i8)
+
+define <8 x i64> @test_mask_mul_epu32_rr(<16 x i32> %a, <16 x i32> %b)  #0 {
+; CHECK-LABEL: @test_mask_mul_epu32_rr(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <16 x i32> [[TMP1]] to <8 x i64>
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <16 x i32> [[A:%.*]] to <8 x i64>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <16 x i32> [[TMP2]] to <8 x i64>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <16 x i32> [[B:%.*]] to <8 x i64>
+; CHECK-NEXT:    [[TMP7:%.*]] = and <8 x i64> [[TMP3]], zeroinitializer
+; CHECK-NEXT:    [[TMP8:%.*]] = and <8 x i64> [[TMP4]], zeroinitializer
+; CHECK-NEXT:    [[TMP9:%.*]] = and <8 x i64> [[TMP3]], splat (i64 4294967295)
+; CHECK-NEXT:    [[TMP10:%.*]] = or <8 x i64> [[TMP7]], [[TMP8]]
+; CHECK-NEXT:    [[TMP11:%.*]] = or <8 x i64> [[TMP10]], [[TMP9]]
+; CHECK-NEXT:    [[TMP12:%.*]] = and <8 x i64> [[TMP4]], splat (i64 4294967295)
+; CHECK-NEXT:    [[TMP13:%.*]] = and <8 x i64> [[TMP5]], zeroinitializer
+; CHECK-NEXT:    [[TMP14:%.*]] = and <8 x i64> [[TMP6]], zeroinitializer
+; CHECK-NEXT:    [[TMP15:%.*]] = and <8 x i64> [[TMP5]], splat (i64 4294967295)
+; CHECK-NEXT:    [[TMP16:%.*]] = or <8 x i64> [[TMP13]], [[TMP14]]
+; CHECK-NEXT:    [[TMP17:%.*]] = or <8 x i64> [[TMP16]], [[TMP15]]
+; CHECK-NEXT:    [[TMP18:%.*]] = and <8 x i64> [[TMP6]], splat (i64 4294967295)
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <8 x i64> [[TMP11]], [[TMP17]]
+; CHECK-NEXT:    [[TMP19:%.*]] = mul <8 x i64> [[TMP12]], [[TMP18]]
+; CHECK-NEXT:    store <8 x i64> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i64> [[TMP19]]
+;
+  %res = call <8 x i64> @llvm.x86.avx512.mask.pmulu.dq.512(<16 x i32> %a, <16 x i32> %b, <8 x i64> zeroinitializer, i8 -1)
+  ret < 8 x i64> %res
+}
+
+define <8 x i64> @test_mask_mul_epu32_rrk(<16 x i32> %a, <16 x i32> %b, <8 x i64> %passThru, i8 %mask)  #0 {
+;
+; CHECK-LABEL: @test_mask_mul_epu32_rrk(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <16 x i32> [[TMP1]] to <8 x i64>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <16 x i32> [[A:%.*]] to <8 x i64>
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <16 x i32> [[TMP2]] to <8 x i64>
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <16 x i32> [[B:%.*]] to <8 x i64>
+; CHECK-NEXT:    [[TMP9:%.*]] = and <8 x i64> [[TMP5]], zeroinitializer
+; CHECK-NEXT:    [[TMP10:%.*]] = and <8 x i64> [[TMP6]], zeroinitializer
+; CHECK-NEXT:    [[TMP11:%.*]] = and <8 x i64> [[TMP5]], splat (i64 4294967295)
+; CHECK-NEXT:    [[TMP12:%.*]] = or <8 x i64> [[TMP9]], [[TMP10]]
+; CHECK-NEXT:    [[TMP13:%.*]] = or <8 x i64> [[TMP12]], [[TMP11]]
+; CHECK-NEXT:    [[TMP14:%.*]] = and <8 x i64> [[TMP6]], splat (i64 4294967295)
+; CHECK-NEXT:    [[TMP15:%.*]] = and <8 x i64> [[TMP7]], zeroinitializer
+; CHECK-NEXT:    [[TMP16:%.*]] = and <8 x i64> [[TMP8]], zeroinitializer
+; CHECK-NEXT:    [[TMP17:%.*]] = and <8 x i64> [[TMP7]], splat (i64 4294967295)
+; CHECK-NEXT:    [[TMP18:%.*]] = or <8 x i64> [[TMP15]], [[TMP16]]
+; CHECK-NEXT:    [[TMP19:%.*]] = or <8 x i64> [[TMP18]], [[TMP17]]
+; CHECK-NEXT:    [[TMP20:%.*]] = and <8 x i64> [[TMP8]], splat (i64 4294967295)
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <8 x i64> [[TMP13]], [[TMP19]]
+; CHECK-NEXT:    [[TMP21:%.*]] = mul <8 x i64> [[TMP14]], [[TMP20]]
+; CHECK-NEXT:    [[TMP22:%.*]] = bitcast i8 [[TMP3]] to <8 x i1>
+; CHECK-NEXT:    [[TMP23:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP24:%.*]] = select <8 x i1> [[TMP23]], <8 x i64> [[_MSPROP]], <8 x i64> [[TMP4]]
+; CHECK-NEXT:    [[TMP25:%.*]] = xor <8 x i64> [[TMP21]], [[PASSTHRU:%.*]]
+; CHECK-NEXT:    [[TMP26:%.*]] = or <8 x i64> [[TMP25]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP27:%.*]] = or <8 x i64> [[TMP26]], [[TMP4]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP22]], <8 x i64> [[TMP27]], <8 x i64> [[TMP24]]
+; CHECK-NEXT:    [[TMP28:%.*]] = select <8 x i1> [[TMP23]], <8 x i64> [[TMP21]], <8 x i64> [[PASSTHRU]]
+; CHECK-NEXT:    store <8 x i64> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i64> [[TMP28]]
+;
+  %res = call <8 x i64> @llvm.x86.avx512.mask.pmulu.dq.512(<16 x i32> %a, <16 x i32> %b, <8 x i64> %passThru, i8 %mask)
+  ret < 8 x i64> %res
+}
+
+define <8 x i64> @test_mask_mul_epu32_rrkz(<16 x i32> %a, <16 x i32> %b, i8 %mask)  #0 {
+;
+; CHECK-LABEL: @test_mask_mul_epu32_rrkz(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <16 x i32> [[TMP1]] to <8 x i64>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <16 x i32> [[A:%.*]] to <8 x i64>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <16 x i32> [[TMP2]] to <8 x i64>
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <16 x i32> [[B:%.*]] to <8 x i64>
+; CHECK-NEXT:    [[TMP8:%.*]] = and <8 x i64> [[TMP4]], zeroinitializer
+; CHECK-NEXT:    [[TMP9:%.*]] = and <8 x i64> [[TMP5]], zeroinitializer
+; CHECK-NEXT:    [[TMP10:%.*]] = and <8 x i64> [[TMP4]], splat (i64 4294967295)
+; CHECK-NEXT:    [[TMP11:%.*]] = or <8 x i64> [[TMP8]], [[TMP9]]
+; CHECK-NEXT:    [[TMP12:%.*]] = or <8 x i64> [[TMP11]], [[TMP10]]
+; CHECK-NEXT:    [[TMP13:%.*]] = and <8 x i64> [[TMP5]], splat (i64 4294967295)
+; CHECK-NEXT:    [[TMP14:%.*]] = and <8 x i64> [[TMP6]], zeroinitializer
+; CHECK-NEXT:    [[TMP15:%.*]] = and <8 x i64> [[TMP7]], zeroinitializer
+; CHECK-NEXT:    [[TMP16:%.*]] = and <8 x i64> [[TMP6]], splat (i64 4294967295)
+; CHECK-NEXT:    [[TMP17:%.*]] = or <8 x i64> [[TMP14]], [[TMP15]]
+; CHECK-NEXT:    [[TMP18:%.*]] = or <8 x i64> [[TMP17]], [[TMP16]]
+; CHECK-NEXT:    [[TMP19:%.*]] = and <8 x i64> [[TMP7]], splat (i64 4294967295)
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <8 x i64> [[TMP12]], [[TMP18]]
+; CHECK-NEXT:    [[TMP20:%.*]] = mul <8 x i64> [[TMP13]], [[TMP19]]
+; CHECK-NEXT:    [[TMP21:%.*]] = bitcast i8 [[TMP3]] to <8 x i1>
+; CHECK-NEXT:    [[TMP22:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP23:%.*]] = select <8 x i1> [[TMP22]], <8 x i64> [[_MSPROP]], <8 x i64> zeroinitializer
+; CHECK-NEXT:    [[TMP24:%.*]] = xor <8 x i64> [[TMP20]], zeroinitializer
+; CHECK-NEXT:    [[TMP25:%.*]] = or <8 x i64> [[TMP24]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP26:%.*]] = or <8 x i64> [[TMP25]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP21]], <8 x i64> [[TMP26]], <8 x i64> [[TMP23]]
+; CHECK-NEXT:    [[TMP27:%.*]] = select <8 x i1> [[TMP22]], <8 x i64> [[TMP20]], <8 x i64> zeroinitializer
+; CHECK-NEXT:    store <8 x i64> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i64> [[TMP27]]
+;
+  %res = call <8 x i64> @llvm.x86.avx512.mask.pmulu.dq.512(<16 x i32> %a, <16 x i32> %b, <8 x i64> zeroinitializer, i8 %mask)
+  ret < 8 x i64> %res
+}
+
+define <8 x i64> @test_mask_mul_epu32_rm(<16 x i32> %a, ptr %ptr_b)  #0 {
+;
+; CHECK-LABEL: @test_mask_mul_epu32_rm(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP3:%.*]], label [[TMP4:%.*]], !prof [[PROF1]]
+; CHECK:       3:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       4:
+; CHECK-NEXT:    [[B:%.*]] = load <16 x i32>, ptr [[PTR_B:%.*]], align 64
+; CHECK-NEXT:    [[TMP5:%.*]] = ptrtoint ptr [[PTR_B]] to i64
+; CHECK-NEXT:    [[TMP6:%.*]] = xor i64 [[TMP5]], 87960930222080
+; CHECK-NEXT:    [[TMP7:%.*]] = inttoptr i64 [[TMP6]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <16 x i32>, ptr [[TMP7]], align 64
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <16 x i32> [[TMP2]] to <8 x i64>
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <16 x i32> [[A:%.*]] to <8 x i64>
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast <16 x i32> [[_MSLD]] to <8 x i64>
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <16 x i32> [[B]] to <8 x i64>
+; CHECK-NEXT:    [[TMP12:%.*]] = and <8 x i64> [[TMP8]], zeroinitializer
+; CHECK-NEXT:    [[TMP13:%.*]] = and <8 x i64> [[TMP9]], zeroinitializer
+; CHECK-NEXT:    [[TMP14:%.*]] = and <8 x i64> [[TMP8]], splat (i64 4294967295)
+; CHECK-NEXT:    [[TMP15:%.*]] = or <8 x i64> [[TMP12]], [[TMP13]]
+; CHECK-NEXT:    [[TMP16:%.*]] = or <8 x i64> [[TMP15]], [[TMP14]]
+; CHECK-NEXT:    [[TMP17:%.*]] = and <8 x i64> [[TMP9]], splat (i64 4294967295)
+; CHECK-NEXT:    [[TMP18:%.*]] = and <8 x i64> [[TMP10]], zeroinitializer
+; CHECK-NEXT:    [[TMP19:%.*]] = and <8 x i64> [[TMP11]], zeroinitializer
+; CHECK-NEXT:    [[TMP20:%.*]] = and <8 x i64> [[TMP10]], splat (i64 4294967295)
+; CHECK-NEXT:    [[TMP21:%.*]] = or <8 x i64> [[TMP18]], [[TMP19]]
+; CHECK-NEXT:    [[TMP22:%.*]] = or <8 x i64> [[TMP21]], [[TMP20]]
+; CHECK-NEXT:    [[TMP23:%.*]] = and <8 x i64> [[TMP11]], splat (i64 4294967295)
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <8 x i64> [[TMP16]], [[TMP22]]
+; CHECK-NEXT:    [[TMP24:%.*]] = mul <8 x i64> [[TMP17]], [[TMP23]]
+; CHECK-NEXT:    store <8 x i64> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i64> [[TMP24]]
+;
+  %b = load <16 x i32>, ptr %ptr_b
+  %res = call <8 x i64> @llvm.x86.avx512.mask.pmulu.dq.512(<16 x i32> %a, <16 x i32> %b, <8 x i64> zeroinitializer, i8 -1)
+  ret < 8 x i64> %res
+}
+
+define <8 x i64> @test_mask_mul_epu32_rmk(<16 x i32> %a, ptr %ptr_b, <8 x i64> %passThru, i8 %mask)  #0 {
+;
+; CHECK-LABEL: @test_mask_mul_epu32_rmk(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 136) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 72) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP5:%.*]], label [[TMP6:%.*]], !prof [[PROF1]]
+; CHECK:       5:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       6:
+; CHECK-NEXT:    [[B:%.*]] = load <16 x i32>, ptr [[PTR_B:%.*]], align 64
+; CHECK-NEXT:    [[TMP7:%.*]] = ptrtoint ptr [[PTR_B]] to i64
+; CHECK-NEXT:    [[TMP8:%.*]] = xor i64 [[TMP7]], 87960930222080
+; CHECK-NEXT:    [[TMP9:%.*]] = inttoptr i64 [[TMP8]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <16 x i32>, ptr [[TMP9]], align 64
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast <16 x i32> [[TMP2]] to <8 x i64>
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <16 x i32> [[A:%.*]] to <8 x i64>
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <16 x i32> [[_MSLD]] to <8 x i64>
+; CHECK-NEXT:    [[TMP13:%.*]] = bitcast <16 x i32> [[B]] to <8 x i64>
+; CHECK-NEXT:    [[TMP14:%.*]] = and <8 x i64> [[TMP10]], zeroinitializer
+; CHECK-NEXT:    [[TMP15:%.*]] = and <8 x i64> [[TMP11]], zeroinitializer
+; CHECK-NEXT:    [[TMP16:%.*]] = and <8 x i64> [[TMP10]], splat (i64 4294967295)
+; CHECK-NEXT:    [[TMP17:%.*]] = or <8 x i64> [[TMP14]], [[TMP15]]
+; CHECK-NEXT:    [[TMP18:%.*]] = or <8 x i64> [[TMP17]], [[TMP16]]
+; CHECK-NEXT:    [[TMP19:%.*]] = and <8 x i64> [[TMP11]], splat (i64 4294967295)
+; CHECK-NEXT:    [[TMP20:%.*]] = and <8 x i64> [[TMP12]], zeroinitializer
+; CHECK-NEXT:    [[TMP21:%.*]] = and <8 x i64> [[TMP13]], zeroinitializer
+; CHECK-NEXT:    [[TMP22:%.*]] = and <8 x i64> [[TMP12]], splat (i64 4294967295)
+; CHECK-NEXT:    [[TMP23:%.*]] = or <8 x i64> [[TMP20]], [[TMP21]]
+; CHECK-NEXT:    [[TMP24:%.*]] = or <8 x i64> [[TMP23]], [[TMP22]]
+; CHECK-NEXT:    [[TMP25:%.*]] = and <8 x i64> [[TMP13]], splat (i64 4294967295)
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <8 x i64> [[TMP18]], [[TMP24]]
+; CHECK-NEXT:    [[TMP26:%.*]] = mul <8 x i64> [[TMP19]], [[TMP25]]
+; CHECK-NEXT:    [[TMP27:%.*]] = bitcast i8 [[TMP3]] to <8 x i1>
+; CHECK-NEXT:    [[TMP28:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP29:%.*]] = select <8 x i1> [[TMP28]], <8 x i64> [[_MSPROP]], <8 x i64> [[TMP4]]
+; CHECK-NEXT:    [[TMP30:%.*]] = xor <8 x i64> [[TMP26]], [[PASSTHRU:%.*]]
+; CHECK-NEXT:    [[TMP31:%.*]] = or <8 x i64> [[TMP30]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP32:%.*]] = or <8 x i64> [[TMP31]], [[TMP4]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP27]], <8 x i64> [[TMP32]], <8 x i64> [[TMP29]]
+; CHECK-NEXT:    [[TMP33:%.*]] = select <8 x i1> [[TMP28]], <8 x i64> [[TMP26]], <8 x i64> [[PASSTHRU]]
+; CHECK-NEXT:    store <8 x i64> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i64> [[TMP33]]
+;
+  %b = load <16 x i32>, ptr %ptr_b
+  %res = call <8 x i64> @llvm.x86.avx512.mask.pmulu.dq.512(<16 x i32> %a, <16 x i32> %b, <8 x i64> %passThru, i8 %mask)
+  ret < 8 x i64> %res
+}
+
+define <8 x i64> @test_mask_mul_epu32_rmkz(<16 x i32> %a, ptr %ptr_b, i8 %mask)  #0 {
+;
+; CHECK-LABEL: @test_mask_mul_epu32_rmkz(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 72) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP4:%.*]], label [[TMP5:%.*]], !prof [[PROF1]]
+; CHECK:       4:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       5:
+; CHECK-NEXT:    [[B:%.*]] = load <16 x i32>, ptr [[PTR_B:%.*]], align 64
+; CHECK-NEXT:    [[TMP6:%.*]] = ptrtoint ptr [[PTR_B]] to i64
+; CHECK-NEXT:    [[TMP7:%.*]] = xor i64 [[TMP6]], 87960930222080
+; CHECK-NEXT:    [[TMP8:%.*]] = inttoptr i64 [[TMP7]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <16 x i32>, ptr [[TMP8]], align 64
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <16 x i32> [[TMP2]] to <8 x i64>
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast <16 x i32> [[A:%.*]] to <8 x i64>
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <16 x i32> [[_MSLD]] to <8 x i64>
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <16 x i32> [[B]] to <8 x i64>
+; CHECK-NEXT:    [[TMP13:%.*]] = and <8 x i64> [[TMP9]], zeroinitializer
+; CHECK-NEXT:    [[TMP14:%.*]] = and <8 x i64> [[TMP10]], zeroinitializer
+; CHECK-NEXT:    [[TMP15:%.*]] = and <8 x i64> [[TMP9]], splat (i64 4294967295)
+; CHECK-NEXT:    [[TMP16:%.*]] = or <8 x i64> [[TMP13]], [[TMP14]]
+; CHECK-NEXT:    [[TMP17:%.*]] = or <8 x i64> [[TMP16]], [[TMP15]]
+; CHECK-NEXT:    [[TMP18:%.*]] = and <8 x i64> [[TMP10]], splat (i64 4294967295)
+; CHECK-NEXT:    [[TMP19:%.*]] = and <8 x i64> [[TMP11]], zeroinitializer
+; CHECK-NEXT:    [[TMP20:%.*]] = and <8 x i64> [[TMP12]], zeroinitializer
+; CHECK-NEXT:    [[TMP21:%.*]] = and <8 x i64> [[TMP11]], splat (i64 4294967295)
+; CHECK-NEXT:    [[TMP22:%.*]] = or <8 x i64> [[TMP19]], [[TMP20]]
+; CHECK-NEXT:    [[TMP23:%.*]] = or <8 x i64> [[TMP22]], [[TMP21]]
+; CHECK-NEXT:    [[TMP24:%.*]] = and <8 x i64> [[TMP12]], splat (i64 4294967295)
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <8 x i64> [[TMP17]], [[TMP23]]
+; CHECK-NEXT:    [[TMP25:%.*]] = mul <8 x i64> [[TMP18]], [[TMP24]]
+; CHECK-NEXT:    [[TMP26:%.*]] = bitcast i8 [[TMP3]] to <8 x i1>
+; CHECK-NEXT:    [[TMP27:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP28:%.*]] = select <8 x i1> [[TMP27]], <8 x i64> [[_MSPROP]], <8 x i64> zeroinitializer
+; CHECK-NEXT:    [[TMP29:%.*]] = xor <8 x i64> [[TMP25]], zeroinitializer
+; CHECK-NEXT:    [[TMP30:%.*]] = or <8 x i64> [[TMP29]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP31:%.*]] = or <8 x i64> [[TMP30]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP26]], <8 x i64> [[TMP31]], <8 x i64> [[TMP28]]
+; CHECK-NEXT:    [[TMP32:%.*]] = select <8 x i1> [[TMP27]], <8 x i64> [[TMP25]], <8 x i64> zeroinitializer
+; CHECK-NEXT:    store <8 x i64> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i64> [[TMP32]]
+;
+  %b = load <16 x i32>, ptr %ptr_b
+  %res = call <8 x i64> @llvm.x86.avx512.mask.pmulu.dq.512(<16 x i32> %a, <16 x i32> %b, <8 x i64> zeroinitializer, i8 %mask)
+  ret < 8 x i64> %res
+}
+
+define <8 x i64> @test_mask_mul_epu32_rmb(<16 x i32> %a, ptr %ptr_b, <8 x i64> %extra_param, <8 x i64> %extra_param2)  #0 {
+;
+; CHECK-LABEL: @test_mask_mul_epu32_rmb(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 72) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 136) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP26:%.*]], label [[TMP27:%.*]], !prof [[PROF1]]
+; CHECK:       5:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       6:
+; CHECK-NEXT:    [[Q:%.*]] = load i64, ptr [[PTR_B:%.*]], align 8
+; CHECK-NEXT:    [[TMP5:%.*]] = ptrtoint ptr [[PTR_B]] to i64
+; CHECK-NEXT:    [[TMP6:%.*]] = xor i64 [[TMP5]], 87960930222080
+; CHECK-NEXT:    [[TMP7:%.*]] = inttoptr i64 [[TMP6]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load i64, ptr [[TMP7]], align 8
+; CHECK-NEXT:    [[_MSPROP:%.*]] = insertelement <8 x i64> [[TMP3]], i64 [[_MSLD]], i32 0
+; CHECK-NEXT:    [[VECINIT_I:%.*]] = insertelement <8 x i64> [[EXTRA_PARAM:%.*]], i64 [[Q]], i32 0
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = shufflevector <8 x i64> [[_MSPROP]], <8 x i64> [[TMP4]], <8 x i32> zeroinitializer
+; CHECK-NEXT:    [[B64:%.*]] = shufflevector <8 x i64> [[VECINIT_I]], <8 x i64> [[EXTRA_PARAM2:%.*]], <8 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <8 x i64> [[_MSPROP1]] to <16 x i32>
+; CHECK-NEXT:    [[B:%.*]] = bitcast <8 x i64> [[B64]] to <16 x i32>
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <16 x i32> [[TMP2]] to <8 x i64>
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast <16 x i32> [[A:%.*]] to <8 x i64>
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <16 x i32> [[TMP8]] to <8 x i64>
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <16 x i32> [[B]] to <8 x i64>
+; CHECK-NEXT:    [[TMP13:%.*]] = and <8 x i64> [[TMP9]], zeroinitializer
+; CHECK-NEXT:    [[TMP14:%.*]] = and <8 x i64> [[TMP10]], zeroinitializer
+; CHECK-NEXT:    [[TMP15:%.*]] = and <8 x i64> [[TMP9]], splat (i64 4294967295)
+; CHECK-NEXT:    [[TMP16:%.*]] = or <8 x i64> [[TMP13]], [[TMP14]]
+; CHECK-NEXT:    [[TMP17:%.*]] = or <8 x i64> [[TMP16]], [[TMP15]]
+; CHECK-NEXT:    [[TMP18:%.*]] = and <8 x i64> [[TMP10]], splat (i64 4294967295)
+; CHECK-NEXT:    [[TMP19:%.*]] = and <8 x i64> [[TMP11]], zeroinitializer
+; CHECK-NEXT:    [[TMP20:%.*]] = and <8 x i64> [[TMP12]], zeroinitializer
+; CHECK-NEXT:    [[TMP21:%.*]] = and <8 x i64> [[TMP11]], splat (i64 4294967295)
+; CHECK-NEXT:    [[TMP22:%.*]] = or <8 x i64> [[TMP19]], [[TMP20]]
+; CHECK-NEXT:    [[TMP23:%.*]] = or <8 x i64> [[TMP22]], [[TMP21]]
+; CHECK-NEXT:    [[TMP24:%.*]] = and <8 x i64> [[TMP12]], splat (i64 4294967295)
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = or <8 x i64> [[TMP17]], [[TMP23]]
+; CHECK-NEXT:    [[TMP25:%.*]] = mul <8 x i64> [[TMP18]], [[TMP24]]
+; CHECK-NEXT:    store <8 x i64> [[_MSPROP2]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i64> [[TMP25]]
+;
+  %q = load i64, ptr %ptr_b
+  %vecinit.i = insertelement <8 x i64> %extra_param, i64 %q, i32 0
+  %b64 = shufflevector <8 x i64> %vecinit.i, <8 x i64> %extra_param2, <8 x i32> zeroinitializer
+  %b = bitcast <8 x i64> %b64 to <16 x i32>
+  %res = call <8 x i64> @llvm.x86.avx512.mask.pmulu.dq.512(<16 x i32> %a, <16 x i32> %b, <8 x i64> zeroinitializer, i8 -1)
+  ret < 8 x i64> %res
+}
+
+define <8 x i64> @test_mask_mul_epu32_rmbk(<16 x i32> %a, ptr %ptr_b, <8 x i64> %passThru, i8 %mask, <8 x i64> %extra_param, <8 x i64> %extra_param2)  #0 {
+;
+; CHECK-LABEL: @test_mask_mul_epu32_rmbk(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP5:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 144) to ptr), align 8
+; CHECK-NEXT:    [[TMP6:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 208) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 136) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 72) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP35:%.*]], label [[TMP36:%.*]], !prof [[PROF1]]
+; CHECK:       7:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       8:
+; CHECK-NEXT:    [[Q:%.*]] = load i64, ptr [[PTR_B:%.*]], align 8
+; CHECK-NEXT:    [[TMP7:%.*]] = ptrtoint ptr [[PTR_B]] to i64
+; CHECK-NEXT:    [[TMP8:%.*]] = xor i64 [[TMP7]], 87960930222080
+; CHECK-NEXT:    [[TMP9:%.*]] = inttoptr i64 [[TMP8]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load i64, ptr [[TMP9]], align 8
+; CHECK-NEXT:    [[_MSPROP:%.*]] = insertelement <8 x i64> [[TMP5]], i64 [[_MSLD]], i32 0
+; CHECK-NEXT:    [[VECINIT_I:%.*]] = insertelement <8 x i64> [[EXTRA_PARAM:%.*]], i64 [[Q]], i32 0
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = shufflevector <8 x i64> [[_MSPROP]], <8 x i64> [[TMP6]], <8 x i32> zeroinitializer
+; CHECK-NEXT:    [[B64:%.*]] = shufflevector <8 x i64> [[VECINIT_I]], <8 x i64> [[EXTRA_PARAM2:%.*]], <8 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast <8 x i64> [[_MSPROP1]] to <16 x i32>
+; CHECK-NEXT:    [[B:%.*]] = bitcast <8 x i64> [[B64]] to <16 x i32>
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <16 x i32> [[TMP2]] to <8 x i64>
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <16 x i32> [[A:%.*]] to <8 x i64>
+; CHECK-NEXT:    [[TMP13:%.*]] = bitcast <16 x i32> [[TMP10]] to <8 x i64>
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast <16 x i32> [[B]] to <8 x i64>
+; CHECK-NEXT:    [[TMP15:%.*]] = and <8 x i64> [[TMP11]], zeroinitializer
+; CHECK-NEXT:    [[TMP16:%.*]] = and <8 x i64> [[TMP12]], zeroinitializer
+; CHECK-NEXT:    [[TMP17:%.*]] = and <8 x i64> [[TMP11]], splat (i64 4294967295)
+; CHECK-NEXT:    [[TMP18:%.*]] = or <8 x i64> [[TMP15]], [[TMP16]]
+; CHECK-NEXT:    [[TMP19:%.*]] = or <8 x i64> [[TMP18]], [[TMP17]]
+; CHECK-NEXT:    [[TMP20:%.*]] = and <8 x i64> [[TMP12]], splat (i64 4294967295)
+; CHECK-NEXT:    [[TMP21:%.*]] = and <8 x i64> [[TMP13]], zeroinitializer
+; CHECK-NEXT:    [[TMP22:%.*]] = and <8 x i64> [[TMP14]], zeroinitializer
+; CHECK-NEXT:    [[TMP23:%.*]] = and <8 x i64> [[TMP13]], splat (i64 4294967295)
+; CHECK-NEXT:    [[TMP24:%.*]] = or <8 x i64> [[TMP21]], [[TMP22]]
+; CHECK-NEXT:    [[TMP25:%.*]] = or <8 x i64> [[TMP24]], [[TMP23]]
+; CHECK-NEXT:    [[TMP26:%.*]] = and <8 x i64> [[TMP14]], splat (i64 4294967295)
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = or <8 x i64> [[TMP19]], [[TMP25]]
+; CHECK-NEXT:    [[TMP27:%.*]] = mul <8 x i64> [[TMP20]], [[TMP26]]
+; CHECK-NEXT:    [[TMP28:%.*]] = bitcast i8 [[TMP3]] to <8 x i1>
+; CHECK-NEXT:    [[TMP29:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP30:%.*]] = select <8 x i1> [[TMP29]], <8 x i64> [[_MSPROP2]], <8 x i64> [[TMP4]]
+; CHECK-NEXT:    [[TMP31:%.*]] = xor <8 x i64> [[TMP27]], [[PASSTHRU:%.*]]
+; CHECK-NEXT:    [[TMP32:%.*]] = or <8 x i64> [[TMP31]], [[_MSPROP2]]
+; CHECK-NEXT:    [[TMP33:%.*]] = or <8 x i64> [[TMP32]], [[TMP4]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP28]], <8 x i64> [[TMP33]], <8 x i64> [[TMP30]]
+; CHECK-NEXT:    [[TMP34:%.*]] = select <8 x i1> [[TMP29]], <8 x i64> [[TMP27]], <8 x i64> [[PASSTHRU]]
+; CHECK-NEXT:    store <8 x i64> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i64> [[TMP34]]
+;
+  %q = load i64, ptr %ptr_b
+  %vecinit.i = insertelement <8 x i64> %extra_param, i64 %q, i32 0
+  %b64 = shufflevector <8 x i64> %vecinit.i, <8 x i64> %extra_param2, <8 x i32> zeroinitializer
+  %b = bitcast <8 x i64> %b64 to <16 x i32>
+  %res = call <8 x i64> @llvm.x86.avx512.mask.pmulu.dq.512(<16 x i32> %a, <16 x i32> %b, <8 x i64> %passThru, i8 %mask)
+  ret < 8 x i64> %res
+}
+
+define <8 x i64> @test_mask_mul_epu32_rmbkz(<16 x i32> %a, ptr %ptr_b, i8 %mask, <8 x i64> %extra_param, <8 x i64> %extra_param2)  #0 {
+;
+; CHECK-LABEL: @test_mask_mul_epu32_rmbkz(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 80) to ptr), align 8
+; CHECK-NEXT:    [[TMP5:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 144) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 72) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP34:%.*]], label [[TMP35:%.*]], !prof [[PROF1]]
+; CHECK:       6:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       7:
+; CHECK-NEXT:    [[Q:%.*]] = load i64, ptr [[PTR_B:%.*]], align 8
+; CHECK-NEXT:    [[TMP6:%.*]] = ptrtoint ptr [[PTR_B]] to i64
+; CHECK-NEXT:    [[TMP7:%.*]] = xor i64 [[TMP6]], 87960930222080
+; CHECK-NEXT:    [[TMP8:%.*]] = inttoptr i64 [[TMP7]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load i64, ptr [[TMP8]], align 8
+; CHECK-NEXT:    [[_MSPROP:%.*]] = insertelement <8 x i64> [[TMP4]], i64 [[_MSLD]], i32 0
+; CHECK-NEXT:    [[VECINIT_I:%.*]] = insertelement <8 x i64> [[EXTRA_PARAM:%.*]], i64 [[Q]], i32 0
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = shufflevector <8 x i64> [[_MSPROP]], <8 x i64> [[TMP5]], <8 x i32> zeroinitializer
+; CHECK-NEXT:    [[B64:%.*]] = shufflevector <8 x i64> [[VECINIT_I]], <8 x i64> [[EXTRA_PARAM2:%.*]], <8 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <8 x i64> [[_MSPROP1]] to <16 x i32>
+; CHECK-NEXT:    [[B:%.*]] = bitcast <8 x i64> [[B64]] to <16 x i32>
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast <16 x i32> [[TMP2]] to <8 x i64>
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <16 x i32> [[A:%.*]] to <8 x i64>
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <16 x i32> [[TMP9]] to <8 x i64>
+; CHECK-NEXT:    [[TMP13:%.*]] = bitcast <16 x i32> [[B]] to <8 x i64>
+; CHECK-NEXT:    [[TMP14:%.*]] = and <8 x i64> [[TMP10]], zeroinitializer
+; CHECK-NEXT:    [[TMP15:%.*]] = and <8 x i64> [[TMP11]], zeroinitializer
+; CHECK-NEXT:    [[TMP16:%.*]] = and <8 x i64> [[TMP10]], splat (i64 4294967295)
+; CHECK-NEXT:    [[TMP17:%.*]] = or <8 x i64> [[TMP14]], [[TMP15]]
+; CHECK-NEXT:    [[TMP18:%.*]] = or <8 x i64> [[TMP17]], [[TMP16]]
+; CHECK-NEXT:    [[TMP19:%.*]] = and <8 x i64> [[TMP11]], splat (i64 4294967295)
+; CHECK-NEXT:    [[TMP20:%.*]] = and <8 x i64> [[TMP12]], zeroinitializer
+; CHECK-NEXT:    [[TMP21:%.*]] = and <8 x i64> [[TMP13]], zeroinitializer
+; CHECK-NEXT:    [[TMP22:%.*]] = and <8 x i64> [[TMP12]], splat (i64 4294967295)
+; CHECK-NEXT:    [[TMP23:%.*]] = or <8 x i64> [[TMP20]], [[TMP21]]
+; CHECK-NEXT:    [[TMP24:%.*]] = or <8 x i64> [[TMP23]], [[TMP22]]
+; CHECK-NEXT:    [[TMP25:%.*]] = and <8 x i64> [[TMP13]], splat (i64 4294967295)
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = or <8 x i64> [[TMP18]], [[TMP24]]
+; CHECK-NEXT:    [[TMP26:%.*]] = mul <8 x i64> [[TMP19]], [[TMP25]]
+; CHECK-NEXT:    [[TMP27:%.*]] = bitcast i8 [[TMP3]] to <8 x i1>
+; CHECK-NEXT:    [[TMP28:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP29:%.*]] = select <8 x i1> [[TMP28]], <8 x i64> [[_MSPROP2]], <8 x i64> zeroinitializer
+; CHECK-NEXT:    [[TMP30:%.*]] = xor <8 x i64> [[TMP26]], zeroinitializer
+; CHECK-NEXT:    [[TMP31:%.*]] = or <8 x i64> [[TMP30]], [[_MSPROP2]]
+; CHECK-NEXT:    [[TMP32:%.*]] = or <8 x i64> [[TMP31]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP27]], <8 x i64> [[TMP32]], <8 x i64> [[TMP29]]
+; CHECK-NEXT:    [[TMP33:%.*]] = select <8 x i1> [[TMP28]], <8 x i64> [[TMP26]], <8 x i64> zeroinitializer
+; CHECK-NEXT:    store <8 x i64> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i64> [[TMP33]]
+;
+  %q = load i64, ptr %ptr_b
+  %vecinit.i = insertelement <8 x i64> %extra_param, i64 %q, i32 0
+  %b64 = shufflevector <8 x i64> %vecinit.i, <8 x i64> %extra_param2, <8 x i32> zeroinitializer
+  %b = bitcast <8 x i64> %b64 to <16 x i32>
+  %res = call <8 x i64> @llvm.x86.avx512.mask.pmulu.dq.512(<16 x i32> %a, <16 x i32> %b, <8 x i64> zeroinitializer, i8 %mask)
+  ret < 8 x i64> %res
+}
+
+declare <8 x i64> @llvm.x86.avx512.mask.pmulu.dq.512(<16 x i32>, <16 x i32>, <8 x i64>, i8)
+
+define <4 x float> @test_mask_vextractf32x4(<4 x float> %b, <16 x float> %a, i8 %mask)  #0 {
+;
+; CHECK-LABEL: @test_mask_vextractf32x4(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 80) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <16 x i32> [[TMP1]], <16 x i32> [[TMP1]], <4 x i32> <i32 8, i32 9, i32 10, i32 11>
+; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <16 x float> [[A:%.*]], <16 x float> [[A]], <4 x i32> <i32 8, i32 9, i32 10, i32 11>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i8 [[TMP2]] to <8 x i1>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = shufflevector <8 x i1> [[TMP5]], <8 x i1> [[TMP5]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[EXTRACT:%.*]] = shufflevector <8 x i1> [[TMP6]], <8 x i1> [[TMP6]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[TMP7:%.*]] = select <4 x i1> [[EXTRACT]], <4 x i32> [[_MSPROP]], <4 x i32> [[TMP3]]
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <4 x float> [[TMP4]] to <4 x i32>
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <4 x float> [[B:%.*]] to <4 x i32>
+; CHECK-NEXT:    [[TMP10:%.*]] = xor <4 x i32> [[TMP8]], [[TMP9]]
+; CHECK-NEXT:    [[TMP11:%.*]] = or <4 x i32> [[TMP10]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP12:%.*]] = or <4 x i32> [[TMP11]], [[TMP3]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <4 x i1> [[_MSPROP1]], <4 x i32> [[TMP12]], <4 x i32> [[TMP7]]
+; CHECK-NEXT:    [[TMP13:%.*]] = select <4 x i1> [[EXTRACT]], <4 x float> [[TMP4]], <4 x float> [[B]]
+; CHECK-NEXT:    store <4 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x float> [[TMP13]]
+;
+  %res = call <4 x float> @llvm.x86.avx512.mask.vextractf32x4.512(<16 x float> %a, i32 2, <4 x float> %b, i8 %mask)
+  ret <4 x float> %res
+}
+
+declare <4 x float> @llvm.x86.avx512.mask.vextractf32x4.512(<16 x float>, i32, <4 x float>, i8)
+
+define <4 x i64> @test_mask_vextracti64x4(<4 x i64> %b, <8 x i64> %a, i8 %mask)  #0 {
+;
+; CHECK-LABEL: @test_mask_vextracti64x4(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 96) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <8 x i64> [[TMP1]], <8 x i64> [[TMP1]], <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <8 x i64> [[A:%.*]], <8 x i64> [[A]], <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i8 [[TMP2]] to <8 x i1>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = shufflevector <8 x i1> [[TMP5]], <8 x i1> [[TMP5]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[EXTRACT:%.*]] = shufflevector <8 x i1> [[TMP6]], <8 x i1> [[TMP6]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[TMP7:%.*]] = select <4 x i1> [[EXTRACT]], <4 x i64> [[_MSPROP]], <4 x i64> [[TMP3]]
+; CHECK-NEXT:    [[TMP8:%.*]] = xor <4 x i64> [[TMP4]], [[B:%.*]]
+; CHECK-NEXT:    [[TMP9:%.*]] = or <4 x i64> [[TMP8]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP10:%.*]] = or <4 x i64> [[TMP9]], [[TMP3]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <4 x i1> [[_MSPROP1]], <4 x i64> [[TMP10]], <4 x i64> [[TMP7]]
+; CHECK-NEXT:    [[TMP11:%.*]] = select <4 x i1> [[EXTRACT]], <4 x i64> [[TMP4]], <4 x i64> [[B]]
+; CHECK-NEXT:    store <4 x i64> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i64> [[TMP11]]
+;
+  %res = call <4 x i64> @llvm.x86.avx512.mask.vextracti64x4.512(<8 x i64> %a, i32 1, <4 x i64> %b, i8 %mask)
+  ret <4 x i64> %res
+}
+
+declare <4 x i64> @llvm.x86.avx512.mask.vextracti64x4.512(<8 x i64>, i32, <4 x i64>, i8)
+
+define <4 x i32> @test_maskz_vextracti32x4(<16 x i32> %a, i8 %mask)  #0 {
+;
+; CHECK-LABEL: @test_maskz_vextracti32x4(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <16 x i32> [[TMP1]], <16 x i32> [[TMP1]], <4 x i32> <i32 8, i32 9, i32 10, i32 11>
+; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <16 x i32> [[A:%.*]], <16 x i32> [[A]], <4 x i32> <i32 8, i32 9, i32 10, i32 11>
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast i8 [[TMP2]] to <8 x i1>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = shufflevector <8 x i1> [[TMP4]], <8 x i1> [[TMP4]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[EXTRACT:%.*]] = shufflevector <8 x i1> [[TMP5]], <8 x i1> [[TMP5]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[TMP6:%.*]] = select <4 x i1> [[EXTRACT]], <4 x i32> [[_MSPROP]], <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP7:%.*]] = xor <4 x i32> [[TMP3]], zeroinitializer
+; CHECK-NEXT:    [[TMP8:%.*]] = or <4 x i32> [[TMP7]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP9:%.*]] = or <4 x i32> [[TMP8]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <4 x i1> [[_MSPROP1]], <4 x i32> [[TMP9]], <4 x i32> [[TMP6]]
+; CHECK-NEXT:    [[TMP10:%.*]] = select <4 x i1> [[EXTRACT]], <4 x i32> [[TMP3]], <4 x i32> zeroinitializer
+; CHECK-NEXT:    store <4 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i32> [[TMP10]]
+;
+  %res = call <4 x i32> @llvm.x86.avx512.mask.vextracti32x4.512(<16 x i32> %a, i32 2, <4 x i32> zeroinitializer, i8 %mask)
+  ret <4 x i32> %res
+}
+
+declare <4 x i32> @llvm.x86.avx512.mask.vextracti32x4.512(<16 x i32>, i32, <4 x i32>, i8)
+
+define <4 x double> @test_vextractf64x4(<8 x double> %a)  #0 {
+; CHECK-LABEL: @test_vextractf64x4(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <8 x i64> [[TMP1]], <8 x i64> [[TMP1]], <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <8 x double> [[A:%.*]], <8 x double> [[A]], <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:    store <4 x i64> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x double> [[TMP2]]
+;
+  %res = call <4 x double> @llvm.x86.avx512.mask.vextractf64x4.512(<8 x double> %a, i32 1, <4 x double> zeroinitializer, i8 -1)
+  ret <4 x double> %res
+}
+
+declare <4 x double> @llvm.x86.avx512.mask.vextractf64x4.512(<8 x double>, i32, <4 x double>, i8)
+
+declare <16 x float> @llvm.x86.avx512.mask.insertf32x4.512(<16 x float>, <4 x float>, i32, <16 x float>, i16)
+
+define <16 x float>@test_int_x86_avx512_insertf32x4_512(<16 x float> %x0, <4 x float> %x1, <16 x float> %x3)  #0 {
+; CHECK-LABEL: @test_int_x86_avx512_insertf32x4_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <4 x i32> [[TMP1]], <4 x i32> splat (i32 -1), <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 4, i32 4, i32 4, i32 4, i32 4, i32 4, i32 4, i32 4, i32 4, i32 4, i32 4>
+; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <4 x float> [[X1:%.*]], <4 x float> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 4, i32 4, i32 4, i32 4, i32 4, i32 4, i32 4, i32 4, i32 4, i32 4, i32 4>
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = shufflevector <16 x i32> [[TMP2]], <16 x i32> [[_MSPROP]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 16, i32 17, i32 18, i32 19, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <16 x float> [[X0:%.*]], <16 x float> [[TMP3]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 16, i32 17, i32 18, i32 19, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP1]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x float> [[TMP4]]
+;
+  %res = call <16 x float> @llvm.x86.avx512.mask.insertf32x4.512(<16 x float> %x0, <4 x float> %x1, i32 1, <16 x float> %x3, i16 -1)
+  ret <16 x float> %res
+}
+
+define <16 x float>@test_int_x86_avx512_mask_insertf32x4_512(<16 x float> %x0, <4 x float> %x1, <16 x float> %x3, i16 %x4)  #0 {
+;
+; CHECK-LABEL: @test_int_x86_avx512_mask_insertf32x4_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 144) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 80) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <4 x i32> [[TMP1]], <4 x i32> splat (i32 -1), <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 4, i32 4, i32 4, i32 4, i32 4, i32 4, i32 4, i32 4, i32 4, i32 4, i32 4>
+; CHECK-NEXT:    [[TMP5:%.*]] = shufflevector <4 x float> [[X1:%.*]], <4 x float> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 4, i32 4, i32 4, i32 4, i32 4, i32 4, i32 4, i32 4, i32 4, i32 4, i32 4>
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = shufflevector <16 x i32> [[TMP2]], <16 x i32> [[_MSPROP]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 16, i32 17, i32 18, i32 19, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; CHECK-NEXT:    [[TMP6:%.*]] = shufflevector <16 x float> [[X0:%.*]], <16 x float> [[TMP5]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 16, i32 17, i32 18, i32 19, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast i16 [[TMP3]] to <16 x i1>
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast i16 [[X4:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP9:%.*]] = select <16 x i1> [[TMP8]], <16 x i32> [[_MSPROP1]], <16 x i32> [[TMP4]]
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast <16 x float> [[TMP6]] to <16 x i32>
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <16 x float> [[X3:%.*]] to <16 x i32>
+; CHECK-NEXT:    [[TMP12:%.*]] = xor <16 x i32> [[TMP10]], [[TMP11]]
+; CHECK-NEXT:    [[TMP13:%.*]] = or <16 x i32> [[TMP12]], [[_MSPROP1]]
+; CHECK-NEXT:    [[TMP14:%.*]] = or <16 x i32> [[TMP13]], [[TMP4]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP7]], <16 x i32> [[TMP14]], <16 x i32> [[TMP9]]
+; CHECK-NEXT:    [[TMP15:%.*]] = select <16 x i1> [[TMP8]], <16 x float> [[TMP6]], <16 x float> [[X3]]
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x float> [[TMP15]]
+;
+  %res = call <16 x float> @llvm.x86.avx512.mask.insertf32x4.512(<16 x float> %x0, <4 x float> %x1, i32 1, <16 x float> %x3, i16 %x4)
+  ret <16 x float> %res
+}
+
+define <16 x float>@test_int_x86_avx512_maskz_insertf32x4_512(<16 x float> %x0, <4 x float> %x1, i16 %x4)  #0 {
+;
+; CHECK-LABEL: @test_int_x86_avx512_maskz_insertf32x4_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 80) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <4 x i32> [[TMP1]], <4 x i32> splat (i32 -1), <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 4, i32 4, i32 4, i32 4, i32 4, i32 4, i32 4, i32 4, i32 4, i32 4, i32 4>
+; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <4 x float> [[X1:%.*]], <4 x float> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 4, i32 4, i32 4, i32 4, i32 4, i32 4, i32 4, i32 4, i32 4, i32 4, i32 4>
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = shufflevector <16 x i32> [[TMP2]], <16 x i32> [[_MSPROP]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 16, i32 17, i32 18, i32 19, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; CHECK-NEXT:    [[TMP5:%.*]] = shufflevector <16 x float> [[X0:%.*]], <16 x float> [[TMP4]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 16, i32 17, i32 18, i32 19, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i16 [[TMP3]] to <16 x i1>
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast i16 [[X4:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP8:%.*]] = select <16 x i1> [[TMP7]], <16 x i32> [[_MSPROP1]], <16 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <16 x float> [[TMP5]] to <16 x i32>
+; CHECK-NEXT:    [[TMP10:%.*]] = xor <16 x i32> [[TMP9]], zeroinitializer
+; CHECK-NEXT:    [[TMP11:%.*]] = or <16 x i32> [[TMP10]], [[_MSPROP1]]
+; CHECK-NEXT:    [[TMP12:%.*]] = or <16 x i32> [[TMP11]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP6]], <16 x i32> [[TMP12]], <16 x i32> [[TMP8]]
+; CHECK-NEXT:    [[TMP13:%.*]] = select <16 x i1> [[TMP7]], <16 x float> [[TMP5]], <16 x float> zeroinitializer
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x float> [[TMP13]]
+;
+  %res = call <16 x float> @llvm.x86.avx512.mask.insertf32x4.512(<16 x float> %x0, <4 x float> %x1, i32 1, <16 x float> zeroinitializer, i16 %x4)
+  ret <16 x float> %res
+}
+
+declare <16 x i32> @llvm.x86.avx512.mask.inserti32x4.512(<16 x i32>, <4 x i32>, i32, <16 x i32>, i16)
+
+define <16 x i32>@test_int_x86_avx512_inserti32x4_512(<16 x i32> %x0, <4 x i32> %x1, <16 x i32> %x3, i16 %x4)  #0 {
+; CHECK-LABEL: @test_int_x86_avx512_inserti32x4_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <4 x i32> [[TMP1]], <4 x i32> splat (i32 -1), <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 4, i32 4, i32 4, i32 4, i32 4, i32 4, i32 4, i32 4, i32 4, i32 4, i32 4>
+; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <4 x i32> [[X1:%.*]], <4 x i32> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 4, i32 4, i32 4, i32 4, i32 4, i32 4, i32 4, i32 4, i32 4, i32 4, i32 4>
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = shufflevector <16 x i32> [[TMP2]], <16 x i32> [[_MSPROP]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 16, i32 17, i32 18, i32 19, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <16 x i32> [[X0:%.*]], <16 x i32> [[TMP3]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 16, i32 17, i32 18, i32 19, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP1]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i32> [[TMP4]]
+;
+  %res = call <16 x i32> @llvm.x86.avx512.mask.inserti32x4.512(<16 x i32> %x0, <4 x i32> %x1, i32 1, <16 x i32> %x3, i16 -1)
+  ret <16 x i32> %res
+}
+
+define <16 x i32>@test_int_x86_avx512_mask_inserti32x4_512(<16 x i32> %x0, <4 x i32> %x1, <16 x i32> %x3, i16 %x4)  #0 {
+;
+; CHECK-LABEL: @test_int_x86_avx512_mask_inserti32x4_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 144) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 80) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <4 x i32> [[TMP1]], <4 x i32> splat (i32 -1), <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 4, i32 4, i32 4, i32 4, i32 4, i32 4, i32 4, i32 4, i32 4, i32 4, i32 4>
+; CHECK-NEXT:    [[TMP5:%.*]] = shufflevector <4 x i32> [[X1:%.*]], <4 x i32> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 4, i32 4, i32 4, i32 4, i32 4, i32 4, i32 4, i32 4, i32 4, i32 4, i32 4>
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = shufflevector <16 x i32> [[TMP2]], <16 x i32> [[_MSPROP]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 16, i32 17, i32 18, i32 19, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; CHECK-NEXT:    [[TMP6:%.*]] = shufflevector <16 x i32> [[X0:%.*]], <16 x i32> [[TMP5]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 16, i32 17, i32 18, i32 19, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast i16 [[TMP3]] to <16 x i1>
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast i16 [[X4:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP9:%.*]] = select <16 x i1> [[TMP8]], <16 x i32> [[_MSPROP1]], <16 x i32> [[TMP4]]
+; CHECK-NEXT:    [[TMP10:%.*]] = xor <16 x i32> [[TMP6]], [[X3:%.*]]
+; CHECK-NEXT:    [[TMP11:%.*]] = or <16 x i32> [[TMP10]], [[_MSPROP1]]
+; CHECK-NEXT:    [[TMP12:%.*]] = or <16 x i32> [[TMP11]], [[TMP4]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP7]], <16 x i32> [[TMP12]], <16 x i32> [[TMP9]]
+; CHECK-NEXT:    [[TMP13:%.*]] = select <16 x i1> [[TMP8]], <16 x i32> [[TMP6]], <16 x i32> [[X3]]
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i32> [[TMP13]]
+;
+  %res = call <16 x i32> @llvm.x86.avx512.mask.inserti32x4.512(<16 x i32> %x0, <4 x i32> %x1, i32 1, <16 x i32> %x3, i16 %x4)
+  ret <16 x i32> %res
+}
+
+define <16 x i32>@test_int_x86_avx512_maskz_inserti32x4_512(<16 x i32> %x0, <4 x i32> %x1, i16 %x4)  #0 {
+;
+; CHECK-LABEL: @test_int_x86_avx512_maskz_inserti32x4_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 80) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <4 x i32> [[TMP1]], <4 x i32> splat (i32 -1), <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 4, i32 4, i32 4, i32 4, i32 4, i32 4, i32 4, i32 4, i32 4, i32 4, i32 4>
+; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <4 x i32> [[X1:%.*]], <4 x i32> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 4, i32 4, i32 4, i32 4, i32 4, i32 4, i32 4, i32 4, i32 4, i32 4, i32 4>
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = shufflevector <16 x i32> [[TMP2]], <16 x i32> [[_MSPROP]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 16, i32 17, i32 18, i32 19, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; CHECK-NEXT:    [[TMP5:%.*]] = shufflevector <16 x i32> [[X0:%.*]], <16 x i32> [[TMP4]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 16, i32 17, i32 18, i32 19, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i16 [[TMP3]] to <16 x i1>
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast i16 [[X4:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP8:%.*]] = select <16 x i1> [[TMP7]], <16 x i32> [[_MSPROP1]], <16 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP9:%.*]] = xor <16 x i32> [[TMP5]], zeroinitializer
+; CHECK-NEXT:    [[TMP10:%.*]] = or <16 x i32> [[TMP9]], [[_MSPROP1]]
+; CHECK-NEXT:    [[TMP11:%.*]] = or <16 x i32> [[TMP10]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP6]], <16 x i32> [[TMP11]], <16 x i32> [[TMP8]]
+; CHECK-NEXT:    [[TMP12:%.*]] = select <16 x i1> [[TMP7]], <16 x i32> [[TMP5]], <16 x i32> zeroinitializer
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i32> [[TMP12]]
+;
+  %res = call <16 x i32> @llvm.x86.avx512.mask.inserti32x4.512(<16 x i32> %x0, <4 x i32> %x1, i32 1, <16 x i32> zeroinitializer, i16 %x4)
+  ret <16 x i32> %res
+}
+
+declare <8 x double> @llvm.x86.avx512.mask.insertf64x4.512(<8 x double>, <4 x double>, i32, <8 x double>, i8)
+
+define <8 x double>@test_int_x86_avx512_insertf64x4_512(<8 x double> %x0, <4 x double> %x1, <8 x double> %x3)  #0 {
+; CHECK-LABEL: @test_int_x86_avx512_insertf64x4_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <4 x i64> [[TMP1]], <4 x i64> splat (i64 -1), <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 4, i32 4, i32 4>
+; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <4 x double> [[X1:%.*]], <4 x double> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 4, i32 4, i32 4>
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = shufflevector <8 x i64> [[TMP2]], <8 x i64> [[_MSPROP]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
+; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <8 x double> [[X0:%.*]], <8 x double> [[TMP3]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
+; CHECK-NEXT:    store <8 x i64> [[_MSPROP1]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x double> [[TMP4]]
+;
+  %res = call <8 x double> @llvm.x86.avx512.mask.insertf64x4.512(<8 x double> %x0, <4 x double> %x1, i32 1, <8 x double> %x3, i8 -1)
+  ret <8 x double> %res
+}
+
+define <8 x double>@test_int_x86_avx512_mask_insertf64x4_512(<8 x double> %x0, <4 x double> %x1, <8 x double> %x3, i8 %x4)  #0 {
+;
+; CHECK-LABEL: @test_int_x86_avx512_mask_insertf64x4_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 160) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 96) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <4 x i64> [[TMP1]], <4 x i64> splat (i64 -1), <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 4, i32 4, i32 4>
+; CHECK-NEXT:    [[TMP5:%.*]] = shufflevector <4 x double> [[X1:%.*]], <4 x double> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 4, i32 4, i32 4>
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = shufflevector <8 x i64> [[TMP2]], <8 x i64> [[_MSPROP]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
+; CHECK-NEXT:    [[TMP6:%.*]] = shufflevector <8 x double> [[X0:%.*]], <8 x double> [[TMP5]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast i8 [[TMP3]] to <8 x i1>
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast i8 [[X4:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP9:%.*]] = select <8 x i1> [[TMP8]], <8 x i64> [[_MSPROP1]], <8 x i64> [[TMP4]]
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast <8 x double> [[TMP6]] to <8 x i64>
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <8 x double> [[X3:%.*]] to <8 x i64>
+; CHECK-NEXT:    [[TMP12:%.*]] = xor <8 x i64> [[TMP10]], [[TMP11]]
+; CHECK-NEXT:    [[TMP13:%.*]] = or <8 x i64> [[TMP12]], [[_MSPROP1]]
+; CHECK-NEXT:    [[TMP14:%.*]] = or <8 x i64> [[TMP13]], [[TMP4]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP7]], <8 x i64> [[TMP14]], <8 x i64> [[TMP9]]
+; CHECK-NEXT:    [[TMP15:%.*]] = select <8 x i1> [[TMP8]], <8 x double> [[TMP6]], <8 x double> [[X3]]
+; CHECK-NEXT:    store <8 x i64> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x double> [[TMP15]]
+;
+  %res = call <8 x double> @llvm.x86.avx512.mask.insertf64x4.512(<8 x double> %x0, <4 x double> %x1, i32 1, <8 x double> %x3, i8 %x4)
+  ret <8 x double> %res
+}
+
+define <8 x double>@test_int_x86_avx512_maskz_insertf64x4_512(<8 x double> %x0, <4 x double> %x1, i8 %x4)  #0 {
+;
+; CHECK-LABEL: @test_int_x86_avx512_maskz_insertf64x4_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 96) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <4 x i64> [[TMP1]], <4 x i64> splat (i64 -1), <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 4, i32 4, i32 4>
+; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <4 x double> [[X1:%.*]], <4 x double> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 4, i32 4, i32 4>
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = shufflevector <8 x i64> [[TMP2]], <8 x i64> [[_MSPROP]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
+; CHECK-NEXT:    [[TMP5:%.*]] = shufflevector <8 x double> [[X0:%.*]], <8 x double> [[TMP4]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i8 [[TMP3]] to <8 x i1>
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast i8 [[X4:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP8:%.*]] = select <8 x i1> [[TMP7]], <8 x i64> [[_MSPROP1]], <8 x i64> zeroinitializer
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <8 x double> [[TMP5]] to <8 x i64>
+; CHECK-NEXT:    [[TMP10:%.*]] = xor <8 x i64> [[TMP9]], zeroinitializer
+; CHECK-NEXT:    [[TMP11:%.*]] = or <8 x i64> [[TMP10]], [[_MSPROP1]]
+; CHECK-NEXT:    [[TMP12:%.*]] = or <8 x i64> [[TMP11]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP6]], <8 x i64> [[TMP12]], <8 x i64> [[TMP8]]
+; CHECK-NEXT:    [[TMP13:%.*]] = select <8 x i1> [[TMP7]], <8 x double> [[TMP5]], <8 x double> zeroinitializer
+; CHECK-NEXT:    store <8 x i64> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x double> [[TMP13]]
+;
+  %res = call <8 x double> @llvm.x86.avx512.mask.insertf64x4.512(<8 x double> %x0, <4 x double> %x1, i32 1, <8 x double> zeroinitializer, i8 %x4)
+  ret <8 x double> %res
+}
+
+declare <8 x i64> @llvm.x86.avx512.mask.inserti64x4.512(<8 x i64>, <4 x i64>, i32, <8 x i64>, i8)
+
+define <8 x i64>@test_int_x86_avx512_inserti64x4_512(<8 x i64> %x0, <4 x i64> %x1, <8 x i64> %x3)  #0 {
+; CHECK-LABEL: @test_int_x86_avx512_inserti64x4_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <4 x i64> [[TMP1]], <4 x i64> splat (i64 -1), <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 4, i32 4, i32 4>
+; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <4 x i64> [[X1:%.*]], <4 x i64> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 4, i32 4, i32 4>
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = shufflevector <8 x i64> [[TMP2]], <8 x i64> [[_MSPROP]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
+; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <8 x i64> [[X0:%.*]], <8 x i64> [[TMP3]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
+; CHECK-NEXT:    store <8 x i64> [[_MSPROP1]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i64> [[TMP4]]
+;
+  %res = call <8 x i64> @llvm.x86.avx512.mask.inserti64x4.512(<8 x i64> %x0, <4 x i64> %x1, i32 1, <8 x i64> %x3, i8 -1)
+  ret <8 x i64> %res
+}
+
+define <8 x i64>@test_int_x86_avx512_mask_inserti64x4_512(<8 x i64> %x0, <4 x i64> %x1, <8 x i64> %x3, i8 %x4)  #0 {
+;
+; CHECK-LABEL: @test_int_x86_avx512_mask_inserti64x4_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 160) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 96) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <4 x i64> [[TMP1]], <4 x i64> splat (i64 -1), <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 4, i32 4, i32 4>
+; CHECK-NEXT:    [[TMP5:%.*]] = shufflevector <4 x i64> [[X1:%.*]], <4 x i64> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 4, i32 4, i32 4>
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = shufflevector <8 x i64> [[TMP2]], <8 x i64> [[_MSPROP]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
+; CHECK-NEXT:    [[TMP6:%.*]] = shufflevector <8 x i64> [[X0:%.*]], <8 x i64> [[TMP5]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast i8 [[TMP3]] to <8 x i1>
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast i8 [[X4:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP9:%.*]] = select <8 x i1> [[TMP8]], <8 x i64> [[_MSPROP1]], <8 x i64> [[TMP4]]
+; CHECK-NEXT:    [[TMP10:%.*]] = xor <8 x i64> [[TMP6]], [[X3:%.*]]
+; CHECK-NEXT:    [[TMP11:%.*]] = or <8 x i64> [[TMP10]], [[_MSPROP1]]
+; CHECK-NEXT:    [[TMP12:%.*]] = or <8 x i64> [[TMP11]], [[TMP4]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP7]], <8 x i64> [[TMP12]], <8 x i64> [[TMP9]]
+; CHECK-NEXT:    [[TMP13:%.*]] = select <8 x i1> [[TMP8]], <8 x i64> [[TMP6]], <8 x i64> [[X3]]
+; CHECK-NEXT:    store <8 x i64> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i64> [[TMP13]]
+;
+  %res = call <8 x i64> @llvm.x86.avx512.mask.inserti64x4.512(<8 x i64> %x0, <4 x i64> %x1, i32 1, <8 x i64> %x3, i8 %x4)
+  ret <8 x i64> %res
+}
+
+define <8 x i64>@test_int_x86_avx512_maskz_inserti64x4_512(<8 x i64> %x0, <4 x i64> %x1, i8 %x4)  #0 {
+;
+; CHECK-LABEL: @test_int_x86_avx512_maskz_inserti64x4_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 96) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <4 x i64> [[TMP1]], <4 x i64> splat (i64 -1), <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 4, i32 4, i32 4>
+; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <4 x i64> [[X1:%.*]], <4 x i64> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 4, i32 4, i32 4>
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = shufflevector <8 x i64> [[TMP2]], <8 x i64> [[_MSPROP]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
+; CHECK-NEXT:    [[TMP5:%.*]] = shufflevector <8 x i64> [[X0:%.*]], <8 x i64> [[TMP4]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i8 [[TMP3]] to <8 x i1>
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast i8 [[X4:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP8:%.*]] = select <8 x i1> [[TMP7]], <8 x i64> [[_MSPROP1]], <8 x i64> zeroinitializer
+; CHECK-NEXT:    [[TMP9:%.*]] = xor <8 x i64> [[TMP5]], zeroinitializer
+; CHECK-NEXT:    [[TMP10:%.*]] = or <8 x i64> [[TMP9]], [[_MSPROP1]]
+; CHECK-NEXT:    [[TMP11:%.*]] = or <8 x i64> [[TMP10]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP6]], <8 x i64> [[TMP11]], <8 x i64> [[TMP8]]
+; CHECK-NEXT:    [[TMP12:%.*]] = select <8 x i1> [[TMP7]], <8 x i64> [[TMP5]], <8 x i64> zeroinitializer
+; CHECK-NEXT:    store <8 x i64> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i64> [[TMP12]]
+;
+  %res = call <8 x i64> @llvm.x86.avx512.mask.inserti64x4.512(<8 x i64> %x0, <4 x i64> %x1, i32 1, <8 x i64> zeroinitializer, i8 %x4)
+  ret <8 x i64> %res
+}
+
+define <8 x i64> @test_x86_avx512_movntdqa(ptr %a0)  #0 {
+;
+; CHECK-LABEL: @test_x86_avx512_movntdqa(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP2:%.*]], label [[TMP3:%.*]], !prof [[PROF1]]
+; CHECK:       2:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       3:
+; CHECK-NEXT:    [[TMP4:%.*]] = load <8 x i64>, ptr [[A0:%.*]], align 64, !nontemporal [[META2]]
+; CHECK-NEXT:    [[TMP5:%.*]] = ptrtoint ptr [[A0]] to i64
+; CHECK-NEXT:    [[TMP6:%.*]] = xor i64 [[TMP5]], 87960930222080
+; CHECK-NEXT:    [[TMP7:%.*]] = inttoptr i64 [[TMP6]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <8 x i64>, ptr [[TMP7]], align 64
+; CHECK-NEXT:    store <8 x i64> [[_MSLD]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i64> [[TMP4]]
+;
+  %res = call <8 x i64> @llvm.x86.avx512.movntdqa(ptr %a0)
+  ret <8 x i64> %res
+}
+
+declare <8 x i64> @llvm.x86.avx512.movntdqa(ptr) nounwind readonly
+
+define <8 x i16> @test_cmp_d_512(<16 x i32> %a0, <16 x i32> %a1, <8 x i16> %extra_param)  #0 {
+; CHECK-LABEL: @test_cmp_d_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP77:%.*]] = load <8 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = xor <16 x i32> [[A0:%.*]], [[A1:%.*]]
+; CHECK-NEXT:    [[TMP4:%.*]] = or <16 x i32> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp ne <16 x i32> [[TMP4]], zeroinitializer
+; CHECK-NEXT:    [[TMP6:%.*]] = xor <16 x i32> [[TMP4]], splat (i32 -1)
+; CHECK-NEXT:    [[TMP7:%.*]] = and <16 x i32> [[TMP6]], [[TMP3]]
+; CHECK-NEXT:    [[TMP8:%.*]] = icmp eq <16 x i32> [[TMP7]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_ICMP:%.*]] = and <16 x i1> [[TMP5]], [[TMP8]]
+; CHECK-NEXT:    [[TMP9:%.*]] = icmp eq <16 x i32> [[A0]], [[A1]]
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast <16 x i1> [[_MSPROP_ICMP]] to i16
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <16 x i1> [[TMP9]] to i16
+; CHECK-NEXT:    [[_MSPROP:%.*]] = insertelement <8 x i16> [[TMP77]], i16 [[TMP10]], i32 0
+; CHECK-NEXT:    [[VEC0:%.*]] = insertelement <8 x i16> [[EXTRA_PARAM:%.*]], i16 [[TMP11]], i32 0
+; CHECK-NEXT:    [[TMP12:%.*]] = xor <16 x i32> [[A0]], splat (i32 -2147483648)
+; CHECK-NEXT:    [[TMP13:%.*]] = xor <16 x i32> [[TMP1]], splat (i32 -1)
+; CHECK-NEXT:    [[TMP14:%.*]] = and <16 x i32> [[TMP12]], [[TMP13]]
+; CHECK-NEXT:    [[TMP15:%.*]] = or <16 x i32> [[TMP12]], [[TMP1]]
+; CHECK-NEXT:    [[TMP16:%.*]] = xor <16 x i32> [[A1]], splat (i32 -2147483648)
+; CHECK-NEXT:    [[TMP17:%.*]] = xor <16 x i32> [[TMP2]], splat (i32 -1)
+; CHECK-NEXT:    [[TMP18:%.*]] = and <16 x i32> [[TMP16]], [[TMP17]]
+; CHECK-NEXT:    [[TMP19:%.*]] = or <16 x i32> [[TMP16]], [[TMP2]]
+; CHECK-NEXT:    [[TMP20:%.*]] = icmp ult <16 x i32> [[TMP14]], [[TMP19]]
+; CHECK-NEXT:    [[TMP21:%.*]] = icmp ult <16 x i32> [[TMP15]], [[TMP18]]
+; CHECK-NEXT:    [[TMP22:%.*]] = xor <16 x i1> [[TMP20]], [[TMP21]]
+; CHECK-NEXT:    [[TMP23:%.*]] = icmp slt <16 x i32> [[A0]], [[A1]]
+; CHECK-NEXT:    [[TMP24:%.*]] = bitcast <16 x i1> [[TMP22]] to i16
+; CHECK-NEXT:    [[TMP25:%.*]] = bitcast <16 x i1> [[TMP23]] to i16
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = insertelement <8 x i16> [[_MSPROP]], i16 [[TMP24]], i32 1
+; CHECK-NEXT:    [[VEC1:%.*]] = insertelement <8 x i16> [[VEC0]], i16 [[TMP25]], i32 1
+; CHECK-NEXT:    [[TMP26:%.*]] = xor <16 x i32> [[A0]], splat (i32 -2147483648)
+; CHECK-NEXT:    [[TMP27:%.*]] = xor <16 x i32> [[TMP1]], splat (i32 -1)
+; CHECK-NEXT:    [[TMP28:%.*]] = and <16 x i32> [[TMP26]], [[TMP27]]
+; CHECK-NEXT:    [[TMP29:%.*]] = or <16 x i32> [[TMP26]], [[TMP1]]
+; CHECK-NEXT:    [[TMP30:%.*]] = xor <16 x i32> [[A1]], splat (i32 -2147483648)
+; CHECK-NEXT:    [[TMP31:%.*]] = xor <16 x i32> [[TMP2]], splat (i32 -1)
+; CHECK-NEXT:    [[TMP32:%.*]] = and <16 x i32> [[TMP30]], [[TMP31]]
+; CHECK-NEXT:    [[TMP33:%.*]] = or <16 x i32> [[TMP30]], [[TMP2]]
+; CHECK-NEXT:    [[TMP34:%.*]] = icmp ule <16 x i32> [[TMP28]], [[TMP33]]
+; CHECK-NEXT:    [[TMP35:%.*]] = icmp ule <16 x i32> [[TMP29]], [[TMP32]]
+; CHECK-NEXT:    [[TMP36:%.*]] = xor <16 x i1> [[TMP34]], [[TMP35]]
+; CHECK-NEXT:    [[TMP37:%.*]] = icmp sle <16 x i32> [[A0]], [[A1]]
+; CHECK-NEXT:    [[TMP38:%.*]] = bitcast <16 x i1> [[TMP36]] to i16
+; CHECK-NEXT:    [[TMP39:%.*]] = bitcast <16 x i1> [[TMP37]] to i16
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = insertelement <8 x i16> [[_MSPROP1]], i16 [[TMP38]], i32 2
+; CHECK-NEXT:    [[VEC2:%.*]] = insertelement <8 x i16> [[VEC1]], i16 [[TMP39]], i32 2
+; CHECK-NEXT:    [[_MSPROP3:%.*]] = insertelement <8 x i16> [[_MSPROP2]], i16 0, i32 3
+; CHECK-NEXT:    [[VEC3:%.*]] = insertelement <8 x i16> [[VEC2]], i16 0, i32 3
+; CHECK-NEXT:    [[TMP40:%.*]] = xor <16 x i32> [[A0]], [[A1]]
+; CHECK-NEXT:    [[TMP41:%.*]] = or <16 x i32> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP42:%.*]] = icmp ne <16 x i32> [[TMP41]], zeroinitializer
+; CHECK-NEXT:    [[TMP43:%.*]] = xor <16 x i32> [[TMP41]], splat (i32 -1)
+; CHECK-NEXT:    [[TMP44:%.*]] = and <16 x i32> [[TMP43]], [[TMP40]]
+; CHECK-NEXT:    [[TMP45:%.*]] = icmp eq <16 x i32> [[TMP44]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_ICMP4:%.*]] = and <16 x i1> [[TMP42]], [[TMP45]]
+; CHECK-NEXT:    [[TMP46:%.*]] = icmp ne <16 x i32> [[A0]], [[A1]]
+; CHECK-NEXT:    [[TMP47:%.*]] = bitcast <16 x i1> [[_MSPROP_ICMP4]] to i16
+; CHECK-NEXT:    [[TMP48:%.*]] = bitcast <16 x i1> [[TMP46]] to i16
+; CHECK-NEXT:    [[_MSPROP5:%.*]] = insertelement <8 x i16> [[_MSPROP3]], i16 [[TMP47]], i32 4
+; CHECK-NEXT:    [[VEC4:%.*]] = insertelement <8 x i16> [[VEC3]], i16 [[TMP48]], i32 4
+; CHECK-NEXT:    [[TMP49:%.*]] = xor <16 x i32> [[A0]], splat (i32 -2147483648)
+; CHECK-NEXT:    [[TMP50:%.*]] = xor <16 x i32> [[TMP1]], splat (i32 -1)
+; CHECK-NEXT:    [[TMP51:%.*]] = and <16 x i32> [[TMP49]], [[TMP50]]
+; CHECK-NEXT:    [[TMP52:%.*]] = or <16 x i32> [[TMP49]], [[TMP1]]
+; CHECK-NEXT:    [[TMP53:%.*]] = xor <16 x i32> [[A1]], splat (i32 -2147483648)
+; CHECK-NEXT:    [[TMP54:%.*]] = xor <16 x i32> [[TMP2]], splat (i32 -1)
+; CHECK-NEXT:    [[TMP55:%.*]] = and <16 x i32> [[TMP53]], [[TMP54]]
+; CHECK-NEXT:    [[TMP56:%.*]] = or <16 x i32> [[TMP53]], [[TMP2]]
+; CHECK-NEXT:    [[TMP57:%.*]] = icmp uge <16 x i32> [[TMP51]], [[TMP56]]
+; CHECK-NEXT:    [[TMP58:%.*]] = icmp uge <16 x i32> [[TMP52]], [[TMP55]]
+; CHECK-NEXT:    [[TMP59:%.*]] = xor <16 x i1> [[TMP57]], [[TMP58]]
+; CHECK-NEXT:    [[TMP60:%.*]] = icmp sge <16 x i32> [[A0]], [[A1]]
+; CHECK-NEXT:    [[TMP61:%.*]] = bitcast <16 x i1> [[TMP59]] to i16
+; CHECK-NEXT:    [[TMP62:%.*]] = bitcast <16 x i1> [[TMP60]] to i16
+; CHECK-NEXT:    [[_MSPROP6:%.*]] = insertelement <8 x i16> [[_MSPROP5]], i16 [[TMP61]], i32 5
+; CHECK-NEXT:    [[VEC5:%.*]] = insertelement <8 x i16> [[VEC4]], i16 [[TMP62]], i32 5
+; CHECK-NEXT:    [[TMP63:%.*]] = xor <16 x i32> [[A0]], splat (i32 -2147483648)
+; CHECK-NEXT:    [[TMP64:%.*]] = xor <16 x i32> [[TMP1]], splat (i32 -1)
+; CHECK-NEXT:    [[TMP65:%.*]] = and <16 x i32> [[TMP63]], [[TMP64]]
+; CHECK-NEXT:    [[TMP66:%.*]] = or <16 x i32> [[TMP63]], [[TMP1]]
+; CHECK-NEXT:    [[TMP67:%.*]] = xor <16 x i32> [[A1]], splat (i32 -2147483648)
+; CHECK-NEXT:    [[TMP68:%.*]] = xor <16 x i32> [[TMP2]], splat (i32 -1)
+; CHECK-NEXT:    [[TMP69:%.*]] = and <16 x i32> [[TMP67]], [[TMP68]]
+; CHECK-NEXT:    [[TMP70:%.*]] = or <16 x i32> [[TMP67]], [[TMP2]]
+; CHECK-NEXT:    [[TMP71:%.*]] = icmp ugt <16 x i32> [[TMP65]], [[TMP70]]
+; CHECK-NEXT:    [[TMP72:%.*]] = icmp ugt <16 x i32> [[TMP66]], [[TMP69]]
+; CHECK-NEXT:    [[TMP73:%.*]] = xor <16 x i1> [[TMP71]], [[TMP72]]
+; CHECK-NEXT:    [[TMP74:%.*]] = icmp sgt <16 x i32> [[A0]], [[A1]]
+; CHECK-NEXT:    [[TMP75:%.*]] = bitcast <16 x i1> [[TMP73]] to i16
+; CHECK-NEXT:    [[TMP76:%.*]] = bitcast <16 x i1> [[TMP74]] to i16
+; CHECK-NEXT:    [[_MSPROP7:%.*]] = insertelement <8 x i16> [[_MSPROP6]], i16 [[TMP75]], i32 6
+; CHECK-NEXT:    [[VEC6:%.*]] = insertelement <8 x i16> [[VEC5]], i16 [[TMP76]], i32 6
+; CHECK-NEXT:    [[_MSPROP8:%.*]] = insertelement <8 x i16> [[_MSPROP7]], i16 0, i32 7
+; CHECK-NEXT:    [[VEC7:%.*]] = insertelement <8 x i16> [[VEC6]], i16 -1, i32 7
+; CHECK-NEXT:    store <8 x i16> [[_MSPROP8]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i16> [[VEC7]]
+;
+  %res0 = call i16 @llvm.x86.avx512.mask.cmp.d.512(<16 x i32> %a0, <16 x i32> %a1, i32 0, i16 -1)
+  %vec0 = insertelement <8 x i16> %extra_param, i16 %res0, i32 0
+  %res1 = call i16 @llvm.x86.avx512.mask.cmp.d.512(<16 x i32> %a0, <16 x i32> %a1, i32 1, i16 -1)
+  %vec1 = insertelement <8 x i16> %vec0, i16 %res1, i32 1
+  %res2 = call i16 @llvm.x86.avx512.mask.cmp.d.512(<16 x i32> %a0, <16 x i32> %a1, i32 2, i16 -1)
+  %vec2 = insertelement <8 x i16> %vec1, i16 %res2, i32 2
+  %res3 = call i16 @llvm.x86.avx512.mask.cmp.d.512(<16 x i32> %a0, <16 x i32> %a1, i32 3, i16 -1)
+  %vec3 = insertelement <8 x i16> %vec2, i16 %res3, i32 3
+  %res4 = call i16 @llvm.x86.avx512.mask.cmp.d.512(<16 x i32> %a0, <16 x i32> %a1, i32 4, i16 -1)
+  %vec4 = insertelement <8 x i16> %vec3, i16 %res4, i32 4
+  %res5 = call i16 @llvm.x86.avx512.mask.cmp.d.512(<16 x i32> %a0, <16 x i32> %a1, i32 5, i16 -1)
+  %vec5 = insertelement <8 x i16> %vec4, i16 %res5, i32 5
+  %res6 = call i16 @llvm.x86.avx512.mask.cmp.d.512(<16 x i32> %a0, <16 x i32> %a1, i32 6, i16 -1)
+  %vec6 = insertelement <8 x i16> %vec5, i16 %res6, i32 6
+  %res7 = call i16 @llvm.x86.avx512.mask.cmp.d.512(<16 x i32> %a0, <16 x i32> %a1, i32 7, i16 -1)
+  %vec7 = insertelement <8 x i16> %vec6, i16 %res7, i32 7
+  ret <8 x i16> %vec7
+}
+
+define <8 x i16> @test_mask_cmp_d_512(<16 x i32> %a0, <16 x i32> %a1, i16 %mask, <8 x i16> %extra_param)  #0 {
+;
+; CHECK-LABEL: @test_mask_cmp_d_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    [[TMP146:%.*]] = load <8 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 136) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = xor <16 x i32> [[A0:%.*]], [[A1:%.*]]
+; CHECK-NEXT:    [[TMP5:%.*]] = or <16 x i32> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP6:%.*]] = icmp ne <16 x i32> [[TMP5]], zeroinitializer
+; CHECK-NEXT:    [[TMP7:%.*]] = xor <16 x i32> [[TMP5]], splat (i32 -1)
+; CHECK-NEXT:    [[TMP8:%.*]] = and <16 x i32> [[TMP7]], [[TMP4]]
+; CHECK-NEXT:    [[TMP9:%.*]] = icmp eq <16 x i32> [[TMP8]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_ICMP:%.*]] = and <16 x i1> [[TMP6]], [[TMP9]]
+; CHECK-NEXT:    [[TMP10:%.*]] = icmp eq <16 x i32> [[A0]], [[A1]]
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast i16 [[TMP3]] to <16 x i1>
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast i16 [[MASK:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP13:%.*]] = and <16 x i1> [[_MSPROP_ICMP]], [[TMP11]]
+; CHECK-NEXT:    [[TMP14:%.*]] = and <16 x i1> [[TMP10]], [[TMP11]]
+; CHECK-NEXT:    [[TMP15:%.*]] = and <16 x i1> [[_MSPROP_ICMP]], [[TMP12]]
+; CHECK-NEXT:    [[TMP16:%.*]] = or <16 x i1> [[TMP13]], [[TMP14]]
+; CHECK-NEXT:    [[TMP17:%.*]] = or <16 x i1> [[TMP16]], [[TMP15]]
+; CHECK-NEXT:    [[TMP18:%.*]] = and <16 x i1> [[TMP10]], [[TMP12]]
+; CHECK-NEXT:    [[TMP19:%.*]] = bitcast <16 x i1> [[TMP17]] to i16
+; CHECK-NEXT:    [[TMP20:%.*]] = bitcast <16 x i1> [[TMP18]] to i16
+; CHECK-NEXT:    [[_MSPROP:%.*]] = insertelement <8 x i16> [[TMP146]], i16 [[TMP19]], i32 0
+; CHECK-NEXT:    [[VEC0:%.*]] = insertelement <8 x i16> [[EXTRA_PARAM:%.*]], i16 [[TMP20]], i32 0
+; CHECK-NEXT:    [[TMP21:%.*]] = xor <16 x i32> [[A0]], splat (i32 -2147483648)
+; CHECK-NEXT:    [[TMP22:%.*]] = xor <16 x i32> [[TMP1]], splat (i32 -1)
+; CHECK-NEXT:    [[TMP23:%.*]] = and <16 x i32> [[TMP21]], [[TMP22]]
+; CHECK-NEXT:    [[TMP24:%.*]] = or <16 x i32> [[TMP21]], [[TMP1]]
+; CHECK-NEXT:    [[TMP25:%.*]] = xor <16 x i32> [[A1]], splat (i32 -2147483648)
+; CHECK-NEXT:    [[TMP26:%.*]] = xor <16 x i32> [[TMP2]], splat (i32 -1)
+; CHECK-NEXT:    [[TMP27:%.*]] = and <16 x i32> [[TMP25]], [[TMP26]]
+; CHECK-NEXT:    [[TMP28:%.*]] = or <16 x i32> [[TMP25]], [[TMP2]]
+; CHECK-NEXT:    [[TMP29:%.*]] = icmp ult <16 x i32> [[TMP23]], [[TMP28]]
+; CHECK-NEXT:    [[TMP30:%.*]] = icmp ult <16 x i32> [[TMP24]], [[TMP27]]
+; CHECK-NEXT:    [[TMP31:%.*]] = xor <16 x i1> [[TMP29]], [[TMP30]]
+; CHECK-NEXT:    [[TMP32:%.*]] = icmp slt <16 x i32> [[A0]], [[A1]]
+; CHECK-NEXT:    [[TMP33:%.*]] = bitcast i16 [[TMP3]] to <16 x i1>
+; CHECK-NEXT:    [[TMP34:%.*]] = bitcast i16 [[MASK]] to <16 x i1>
+; CHECK-NEXT:    [[TMP35:%.*]] = and <16 x i1> [[TMP31]], [[TMP33]]
+; CHECK-NEXT:    [[TMP36:%.*]] = and <16 x i1> [[TMP32]], [[TMP33]]
+; CHECK-NEXT:    [[TMP37:%.*]] = and <16 x i1> [[TMP31]], [[TMP34]]
+; CHECK-NEXT:    [[TMP38:%.*]] = or <16 x i1> [[TMP35]], [[TMP36]]
+; CHECK-NEXT:    [[TMP39:%.*]] = or <16 x i1> [[TMP38]], [[TMP37]]
+; CHECK-NEXT:    [[TMP40:%.*]] = and <16 x i1> [[TMP32]], [[TMP34]]
+; CHECK-NEXT:    [[TMP41:%.*]] = bitcast <16 x i1> [[TMP39]] to i16
+; CHECK-NEXT:    [[TMP42:%.*]] = bitcast <16 x i1> [[TMP40]] to i16
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = insertelement <8 x i16> [[_MSPROP]], i16 [[TMP41]], i32 1
+; CHECK-NEXT:    [[VEC1:%.*]] = insertelement <8 x i16> [[VEC0]], i16 [[TMP42]], i32 1
+; CHECK-NEXT:    [[TMP43:%.*]] = xor <16 x i32> [[A0]], splat (i32 -2147483648)
+; CHECK-NEXT:    [[TMP44:%.*]] = xor <16 x i32> [[TMP1]], splat (i32 -1)
+; CHECK-NEXT:    [[TMP45:%.*]] = and <16 x i32> [[TMP43]], [[TMP44]]
+; CHECK-NEXT:    [[TMP46:%.*]] = or <16 x i32> [[TMP43]], [[TMP1]]
+; CHECK-NEXT:    [[TMP47:%.*]] = xor <16 x i32> [[A1]], splat (i32 -2147483648)
+; CHECK-NEXT:    [[TMP48:%.*]] = xor <16 x i32> [[TMP2]], splat (i32 -1)
+; CHECK-NEXT:    [[TMP49:%.*]] = and <16 x i32> [[TMP47]], [[TMP48]]
+; CHECK-NEXT:    [[TMP50:%.*]] = or <16 x i32> [[TMP47]], [[TMP2]]
+; CHECK-NEXT:    [[TMP51:%.*]] = icmp ule <16 x i32> [[TMP45]], [[TMP50]]
+; CHECK-NEXT:    [[TMP52:%.*]] = icmp ule <16 x i32> [[TMP46]], [[TMP49]]
+; CHECK-NEXT:    [[TMP53:%.*]] = xor <16 x i1> [[TMP51]], [[TMP52]]
+; CHECK-NEXT:    [[TMP54:%.*]] = icmp sle <16 x i32> [[A0]], [[A1]]
+; CHECK-NEXT:    [[TMP55:%.*]] = bitcast i16 [[TMP3]] to <16 x i1>
+; CHECK-NEXT:    [[TMP56:%.*]] = bitcast i16 [[MASK]] to <16 x i1>
+; CHECK-NEXT:    [[TMP57:%.*]] = and <16 x i1> [[TMP53]], [[TMP55]]
+; CHECK-NEXT:    [[TMP58:%.*]] = and <16 x i1> [[TMP54]], [[TMP55]]
+; CHECK-NEXT:    [[TMP59:%.*]] = and <16 x i1> [[TMP53]], [[TMP56]]
+; CHECK-NEXT:    [[TMP60:%.*]] = or <16 x i1> [[TMP57]], [[TMP58]]
+; CHECK-NEXT:    [[TMP61:%.*]] = or <16 x i1> [[TMP60]], [[TMP59]]
+; CHECK-NEXT:    [[TMP62:%.*]] = and <16 x i1> [[TMP54]], [[TMP56]]
+; CHECK-NEXT:    [[TMP63:%.*]] = bitcast <16 x i1> [[TMP61]] to i16
+; CHECK-NEXT:    [[TMP64:%.*]] = bitcast <16 x i1> [[TMP62]] to i16
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = insertelement <8 x i16> [[_MSPROP1]], i16 [[TMP63]], i32 2
+; CHECK-NEXT:    [[VEC2:%.*]] = insertelement <8 x i16> [[VEC1]], i16 [[TMP64]], i32 2
+; CHECK-NEXT:    [[TMP65:%.*]] = bitcast i16 [[TMP3]] to <16 x i1>
+; CHECK-NEXT:    [[TMP66:%.*]] = bitcast i16 [[MASK]] to <16 x i1>
+; CHECK-NEXT:    [[TMP67:%.*]] = and <16 x i1> zeroinitializer, [[TMP65]]
+; CHECK-NEXT:    [[TMP68:%.*]] = and <16 x i1> zeroinitializer, [[TMP65]]
+; CHECK-NEXT:    [[TMP69:%.*]] = and <16 x i1> zeroinitializer, [[TMP66]]
+; CHECK-NEXT:    [[TMP70:%.*]] = or <16 x i1> [[TMP67]], [[TMP68]]
+; CHECK-NEXT:    [[TMP71:%.*]] = or <16 x i1> [[TMP70]], [[TMP69]]
+; CHECK-NEXT:    [[TMP72:%.*]] = and <16 x i1> zeroinitializer, [[TMP66]]
+; CHECK-NEXT:    [[TMP73:%.*]] = bitcast <16 x i1> [[TMP71]] to i16
+; CHECK-NEXT:    [[TMP74:%.*]] = bitcast <16 x i1> [[TMP72]] to i16
+; CHECK-NEXT:    [[_MSPROP3:%.*]] = insertelement <8 x i16> [[_MSPROP2]], i16 [[TMP73]], i32 3
+; CHECK-NEXT:    [[VEC3:%.*]] = insertelement <8 x i16> [[VEC2]], i16 [[TMP74]], i32 3
+; CHECK-NEXT:    [[TMP75:%.*]] = xor <16 x i32> [[A0]], [[A1]]
+; CHECK-NEXT:    [[TMP76:%.*]] = or <16 x i32> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP77:%.*]] = icmp ne <16 x i32> [[TMP76]], zeroinitializer
+; CHECK-NEXT:    [[TMP78:%.*]] = xor <16 x i32> [[TMP76]], splat (i32 -1)
+; CHECK-NEXT:    [[TMP79:%.*]] = and <16 x i32> [[TMP78]], [[TMP75]]
+; CHECK-NEXT:    [[TMP80:%.*]] = icmp eq <16 x i32> [[TMP79]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_ICMP4:%.*]] = and <16 x i1> [[TMP77]], [[TMP80]]
+; CHECK-NEXT:    [[TMP81:%.*]] = icmp ne <16 x i32> [[A0]], [[A1]]
+; CHECK-NEXT:    [[TMP82:%.*]] = bitcast i16 [[TMP3]] to <16 x i1>
+; CHECK-NEXT:    [[TMP83:%.*]] = bitcast i16 [[MASK]] to <16 x i1>
+; CHECK-NEXT:    [[TMP84:%.*]] = and <16 x i1> [[_MSPROP_ICMP4]], [[TMP82]]
+; CHECK-NEXT:    [[TMP85:%.*]] = and <16 x i1> [[TMP81]], [[TMP82]]
+; CHECK-NEXT:    [[TMP86:%.*]] = and <16 x i1> [[_MSPROP_ICMP4]], [[TMP83]]
+; CHECK-NEXT:    [[TMP87:%.*]] = or <16 x i1> [[TMP84]], [[TMP85]]
+; CHECK-NEXT:    [[TMP88:%.*]] = or <16 x i1> [[TMP87]], [[TMP86]]
+; CHECK-NEXT:    [[TMP89:%.*]] = and <16 x i1> [[TMP81]], [[TMP83]]
+; CHECK-NEXT:    [[TMP90:%.*]] = bitcast <16 x i1> [[TMP88]] to i16
+; CHECK-NEXT:    [[TMP91:%.*]] = bitcast <16 x i1> [[TMP89]] to i16
+; CHECK-NEXT:    [[_MSPROP5:%.*]] = insertelement <8 x i16> [[_MSPROP3]], i16 [[TMP90]], i32 4
+; CHECK-NEXT:    [[VEC4:%.*]] = insertelement <8 x i16> [[VEC3]], i16 [[TMP91]], i32 4
+; CHECK-NEXT:    [[TMP92:%.*]] = xor <16 x i32> [[A0]], splat (i32 -2147483648)
+; CHECK-NEXT:    [[TMP93:%.*]] = xor <16 x i32> [[TMP1]], splat (i32 -1)
+; CHECK-NEXT:    [[TMP94:%.*]] = and <16 x i32> [[TMP92]], [[TMP93]]
+; CHECK-NEXT:    [[TMP95:%.*]] = or <16 x i32> [[TMP92]], [[TMP1]]
+; CHECK-NEXT:    [[TMP96:%.*]] = xor <16 x i32> [[A1]], splat (i32 -2147483648)
+; CHECK-NEXT:    [[TMP97:%.*]] = xor <16 x i32> [[TMP2]], splat (i32 -1)
+; CHECK-NEXT:    [[TMP98:%.*]] = and <16 x i32> [[TMP96]], [[TMP97]]
+; CHECK-NEXT:    [[TMP99:%.*]] = or <16 x i32> [[TMP96]], [[TMP2]]
+; CHECK-NEXT:    [[TMP100:%.*]] = icmp uge <16 x i32> [[TMP94]], [[TMP99]]
+; CHECK-NEXT:    [[TMP101:%.*]] = icmp uge <16 x i32> [[TMP95]], [[TMP98]]
+; CHECK-NEXT:    [[TMP102:%.*]] = xor <16 x i1> [[TMP100]], [[TMP101]]
+; CHECK-NEXT:    [[TMP103:%.*]] = icmp sge <16 x i32> [[A0]], [[A1]]
+; CHECK-NEXT:    [[TMP104:%.*]] = bitcast i16 [[TMP3]] to <16 x i1>
+; CHECK-NEXT:    [[TMP105:%.*]] = bitcast i16 [[MASK]] to <16 x i1>
+; CHECK-NEXT:    [[TMP106:%.*]] = and <16 x i1> [[TMP102]], [[TMP104]]
+; CHECK-NEXT:    [[TMP107:%.*]] = and <16 x i1> [[TMP103]], [[TMP104]]
+; CHECK-NEXT:    [[TMP108:%.*]] = and <16 x i1> [[TMP102]], [[TMP105]]
+; CHECK-NEXT:    [[TMP109:%.*]] = or <16 x i1> [[TMP106]], [[TMP107]]
+; CHECK-NEXT:    [[TMP110:%.*]] = or <16 x i1> [[TMP109]], [[TMP108]]
+; CHECK-NEXT:    [[TMP111:%.*]] = and <16 x i1> [[TMP103]], [[TMP105]]
+; CHECK-NEXT:    [[TMP112:%.*]] = bitcast <16 x i1> [[TMP110]] to i16
+; CHECK-NEXT:    [[TMP113:%.*]] = bitcast <16 x i1> [[TMP111]] to i16
+; CHECK-NEXT:    [[_MSPROP6:%.*]] = insertelement <8 x i16> [[_MSPROP5]], i16 [[TMP112]], i32 5
+; CHECK-NEXT:    [[VEC5:%.*]] = insertelement <8 x i16> [[VEC4]], i16 [[TMP113]], i32 5
+; CHECK-NEXT:    [[TMP114:%.*]] = xor <16 x i32> [[A0]], splat (i32 -2147483648)
+; CHECK-NEXT:    [[TMP115:%.*]] = xor <16 x i32> [[TMP1]], splat (i32 -1)
+; CHECK-NEXT:    [[TMP116:%.*]] = and <16 x i32> [[TMP114]], [[TMP115]]
+; CHECK-NEXT:    [[TMP117:%.*]] = or <16 x i32> [[TMP114]], [[TMP1]]
+; CHECK-NEXT:    [[TMP118:%.*]] = xor <16 x i32> [[A1]], splat (i32 -2147483648)
+; CHECK-NEXT:    [[TMP119:%.*]] = xor <16 x i32> [[TMP2]], splat (i32 -1)
+; CHECK-NEXT:    [[TMP120:%.*]] = and <16 x i32> [[TMP118]], [[TMP119]]
+; CHECK-NEXT:    [[TMP121:%.*]] = or <16 x i32> [[TMP118]], [[TMP2]]
+; CHECK-NEXT:    [[TMP122:%.*]] = icmp ugt <16 x i32> [[TMP116]], [[TMP121]]
+; CHECK-NEXT:    [[TMP123:%.*]] = icmp ugt <16 x i32> [[TMP117]], [[TMP120]]
+; CHECK-NEXT:    [[TMP124:%.*]] = xor <16 x i1> [[TMP122]], [[TMP123]]
+; CHECK-NEXT:    [[TMP125:%.*]] = icmp sgt <16 x i32> [[A0]], [[A1]]
+; CHECK-NEXT:    [[TMP126:%.*]] = bitcast i16 [[TMP3]] to <16 x i1>
+; CHECK-NEXT:    [[TMP127:%.*]] = bitcast i16 [[MASK]] to <16 x i1>
+; CHECK-NEXT:    [[TMP128:%.*]] = and <16 x i1> [[TMP124]], [[TMP126]]
+; CHECK-NEXT:    [[TMP129:%.*]] = and <16 x i1> [[TMP125]], [[TMP126]]
+; CHECK-NEXT:    [[TMP130:%.*]] = and <16 x i1> [[TMP124]], [[TMP127]]
+; CHECK-NEXT:    [[TMP131:%.*]] = or <16 x i1> [[TMP128]], [[TMP129]]
+; CHECK-NEXT:    [[TMP132:%.*]] = or <16 x i1> [[TMP131]], [[TMP130]]
+; CHECK-NEXT:    [[TMP133:%.*]] = and <16 x i1> [[TMP125]], [[TMP127]]
+; CHECK-NEXT:    [[TMP134:%.*]] = bitcast <16 x i1> [[TMP132]] to i16
+; CHECK-NEXT:    [[TMP135:%.*]] = bitcast <16 x i1> [[TMP133]] to i16
+; CHECK-NEXT:    [[_MSPROP7:%.*]] = insertelement <8 x i16> [[_MSPROP6]], i16 [[TMP134]], i32 6
+; CHECK-NEXT:    [[VEC6:%.*]] = insertelement <8 x i16> [[VEC5]], i16 [[TMP135]], i32 6
+; CHECK-NEXT:    [[TMP136:%.*]] = bitcast i16 [[TMP3]] to <16 x i1>
+; CHECK-NEXT:    [[TMP137:%.*]] = bitcast i16 [[MASK]] to <16 x i1>
+; CHECK-NEXT:    [[TMP138:%.*]] = and <16 x i1> zeroinitializer, [[TMP136]]
+; CHECK-NEXT:    [[TMP139:%.*]] = and <16 x i1> splat (i1 true), [[TMP136]]
+; CHECK-NEXT:    [[TMP140:%.*]] = and <16 x i1> zeroinitializer, [[TMP137]]
+; CHECK-NEXT:    [[TMP141:%.*]] = or <16 x i1> [[TMP138]], [[TMP139]]
+; CHECK-NEXT:    [[TMP142:%.*]] = or <16 x i1> [[TMP141]], [[TMP140]]
+; CHECK-NEXT:    [[TMP143:%.*]] = and <16 x i1> splat (i1 true), [[TMP137]]
+; CHECK-NEXT:    [[TMP144:%.*]] = bitcast <16 x i1> [[TMP142]] to i16
+; CHECK-NEXT:    [[TMP145:%.*]] = bitcast <16 x i1> [[TMP143]] to i16
+; CHECK-NEXT:    [[_MSPROP8:%.*]] = insertelement <8 x i16> [[_MSPROP7]], i16 [[TMP144]], i32 7
+; CHECK-NEXT:    [[VEC7:%.*]] = insertelement <8 x i16> [[VEC6]], i16 [[TMP145]], i32 7
+; CHECK-NEXT:    store <8 x i16> [[_MSPROP8]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i16> [[VEC7]]
+;
+  %res0 = call i16 @llvm.x86.avx512.mask.cmp.d.512(<16 x i32> %a0, <16 x i32> %a1, i32 0, i16 %mask)
+  %vec0 = insertelement <8 x i16> %extra_param, i16 %res0, i32 0
+  %res1 = call i16 @llvm.x86.avx512.mask.cmp.d.512(<16 x i32> %a0, <16 x i32> %a1, i32 1, i16 %mask)
+  %vec1 = insertelement <8 x i16> %vec0, i16 %res1, i32 1
+  %res2 = call i16 @llvm.x86.avx512.mask.cmp.d.512(<16 x i32> %a0, <16 x i32> %a1, i32 2, i16 %mask)
+  %vec2 = insertelement <8 x i16> %vec1, i16 %res2, i32 2
+  %res3 = call i16 @llvm.x86.avx512.mask.cmp.d.512(<16 x i32> %a0, <16 x i32> %a1, i32 3, i16 %mask)
+  %vec3 = insertelement <8 x i16> %vec2, i16 %res3, i32 3
+  %res4 = call i16 @llvm.x86.avx512.mask.cmp.d.512(<16 x i32> %a0, <16 x i32> %a1, i32 4, i16 %mask)
+  %vec4 = insertelement <8 x i16> %vec3, i16 %res4, i32 4
+  %res5 = call i16 @llvm.x86.avx512.mask.cmp.d.512(<16 x i32> %a0, <16 x i32> %a1, i32 5, i16 %mask)
+  %vec5 = insertelement <8 x i16> %vec4, i16 %res5, i32 5
+  %res6 = call i16 @llvm.x86.avx512.mask.cmp.d.512(<16 x i32> %a0, <16 x i32> %a1, i32 6, i16 %mask)
+  %vec6 = insertelement <8 x i16> %vec5, i16 %res6, i32 6
+  %res7 = call i16 @llvm.x86.avx512.mask.cmp.d.512(<16 x i32> %a0, <16 x i32> %a1, i32 7, i16 %mask)
+  %vec7 = insertelement <8 x i16> %vec6, i16 %res7, i32 7
+  ret <8 x i16> %vec7
+}
+
+declare i16 @llvm.x86.avx512.mask.cmp.d.512(<16 x i32>, <16 x i32>, i32, i16) nounwind readnone
+
+define <8 x i16> @test_ucmp_d_512(<16 x i32> %a0, <16 x i32> %a1, <8 x i16> %extra_param)  #0 {
+; CHECK-LABEL: @test_ucmp_d_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP69:%.*]] = load <8 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = xor <16 x i32> [[A0:%.*]], [[A1:%.*]]
+; CHECK-NEXT:    [[TMP4:%.*]] = or <16 x i32> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp ne <16 x i32> [[TMP4]], zeroinitializer
+; CHECK-NEXT:    [[TMP6:%.*]] = xor <16 x i32> [[TMP4]], splat (i32 -1)
+; CHECK-NEXT:    [[TMP7:%.*]] = and <16 x i32> [[TMP6]], [[TMP3]]
+; CHECK-NEXT:    [[TMP8:%.*]] = icmp eq <16 x i32> [[TMP7]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_ICMP:%.*]] = and <16 x i1> [[TMP5]], [[TMP8]]
+; CHECK-NEXT:    [[TMP9:%.*]] = icmp eq <16 x i32> [[A0]], [[A1]]
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast <16 x i1> [[_MSPROP_ICMP]] to i16
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <16 x i1> [[TMP9]] to i16
+; CHECK-NEXT:    [[_MSPROP:%.*]] = insertelement <8 x i16> [[TMP69]], i16 [[TMP10]], i32 0
+; CHECK-NEXT:    [[VEC0:%.*]] = insertelement <8 x i16> [[EXTRA_PARAM:%.*]], i16 [[TMP11]], i32 0
+; CHECK-NEXT:    [[TMP12:%.*]] = xor <16 x i32> [[TMP1]], splat (i32 -1)
+; CHECK-NEXT:    [[TMP13:%.*]] = and <16 x i32> [[A0]], [[TMP12]]
+; CHECK-NEXT:    [[TMP14:%.*]] = or <16 x i32> [[A0]], [[TMP1]]
+; CHECK-NEXT:    [[TMP15:%.*]] = xor <16 x i32> [[TMP2]], splat (i32 -1)
+; CHECK-NEXT:    [[TMP16:%.*]] = and <16 x i32> [[A1]], [[TMP15]]
+; CHECK-NEXT:    [[TMP17:%.*]] = or <16 x i32> [[A1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP18:%.*]] = icmp ult <16 x i32> [[TMP13]], [[TMP17]]
+; CHECK-NEXT:    [[TMP19:%.*]] = icmp ult <16 x i32> [[TMP14]], [[TMP16]]
+; CHECK-NEXT:    [[TMP20:%.*]] = xor <16 x i1> [[TMP18]], [[TMP19]]
+; CHECK-NEXT:    [[TMP21:%.*]] = icmp ult <16 x i32> [[A0]], [[A1]]
+; CHECK-NEXT:    [[TMP22:%.*]] = bitcast <16 x i1> [[TMP20]] to i16
+; CHECK-NEXT:    [[TMP23:%.*]] = bitcast <16 x i1> [[TMP21]] to i16
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = insertelement <8 x i16> [[_MSPROP]], i16 [[TMP22]], i32 1
+; CHECK-NEXT:    [[VEC1:%.*]] = insertelement <8 x i16> [[VEC0]], i16 [[TMP23]], i32 1
+; CHECK-NEXT:    [[TMP24:%.*]] = xor <16 x i32> [[TMP1]], splat (i32 -1)
+; CHECK-NEXT:    [[TMP25:%.*]] = and <16 x i32> [[A0]], [[TMP24]]
+; CHECK-NEXT:    [[TMP26:%.*]] = or <16 x i32> [[A0]], [[TMP1]]
+; CHECK-NEXT:    [[TMP27:%.*]] = xor <16 x i32> [[TMP2]], splat (i32 -1)
+; CHECK-NEXT:    [[TMP28:%.*]] = and <16 x i32> [[A1]], [[TMP27]]
+; CHECK-NEXT:    [[TMP29:%.*]] = or <16 x i32> [[A1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP30:%.*]] = icmp ule <16 x i32> [[TMP25]], [[TMP29]]
+; CHECK-NEXT:    [[TMP31:%.*]] = icmp ule <16 x i32> [[TMP26]], [[TMP28]]
+; CHECK-NEXT:    [[TMP32:%.*]] = xor <16 x i1> [[TMP30]], [[TMP31]]
+; CHECK-NEXT:    [[TMP33:%.*]] = icmp ule <16 x i32> [[A0]], [[A1]]
+; CHECK-NEXT:    [[TMP34:%.*]] = bitcast <16 x i1> [[TMP32]] to i16
+; CHECK-NEXT:    [[TMP35:%.*]] = bitcast <16 x i1> [[TMP33]] to i16
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = insertelement <8 x i16> [[_MSPROP1]], i16 [[TMP34]], i32 2
+; CHECK-NEXT:    [[VEC2:%.*]] = insertelement <8 x i16> [[VEC1]], i16 [[TMP35]], i32 2
+; CHECK-NEXT:    [[_MSPROP3:%.*]] = insertelement <8 x i16> [[_MSPROP2]], i16 0, i32 3
+; CHECK-NEXT:    [[VEC3:%.*]] = insertelement <8 x i16> [[VEC2]], i16 0, i32 3
+; CHECK-NEXT:    [[TMP36:%.*]] = xor <16 x i32> [[A0]], [[A1]]
+; CHECK-NEXT:    [[TMP37:%.*]] = or <16 x i32> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP38:%.*]] = icmp ne <16 x i32> [[TMP37]], zeroinitializer
+; CHECK-NEXT:    [[TMP39:%.*]] = xor <16 x i32> [[TMP37]], splat (i32 -1)
+; CHECK-NEXT:    [[TMP40:%.*]] = and <16 x i32> [[TMP39]], [[TMP36]]
+; CHECK-NEXT:    [[TMP41:%.*]] = icmp eq <16 x i32> [[TMP40]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_ICMP4:%.*]] = and <16 x i1> [[TMP38]], [[TMP41]]
+; CHECK-NEXT:    [[TMP42:%.*]] = icmp ne <16 x i32> [[A0]], [[A1]]
+; CHECK-NEXT:    [[TMP43:%.*]] = bitcast <16 x i1> [[_MSPROP_ICMP4]] to i16
+; CHECK-NEXT:    [[TMP44:%.*]] = bitcast <16 x i1> [[TMP42]] to i16
+; CHECK-NEXT:    [[_MSPROP5:%.*]] = insertelement <8 x i16> [[_MSPROP3]], i16 [[TMP43]], i32 4
+; CHECK-NEXT:    [[VEC4:%.*]] = insertelement <8 x i16> [[VEC3]], i16 [[TMP44]], i32 4
+; CHECK-NEXT:    [[TMP45:%.*]] = xor <16 x i32> [[TMP1]], splat (i32 -1)
+; CHECK-NEXT:    [[TMP46:%.*]] = and <16 x i32> [[A0]], [[TMP45]]
+; CHECK-NEXT:    [[TMP47:%.*]] = or <16 x i32> [[A0]], [[TMP1]]
+; CHECK-NEXT:    [[TMP48:%.*]] = xor <16 x i32> [[TMP2]], splat (i32 -1)
+; CHECK-NEXT:    [[TMP49:%.*]] = and <16 x i32> [[A1]], [[TMP48]]
+; CHECK-NEXT:    [[TMP50:%.*]] = or <16 x i32> [[A1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP51:%.*]] = icmp uge <16 x i32> [[TMP46]], [[TMP50]]
+; CHECK-NEXT:    [[TMP52:%.*]] = icmp uge <16 x i32> [[TMP47]], [[TMP49]]
+; CHECK-NEXT:    [[TMP53:%.*]] = xor <16 x i1> [[TMP51]], [[TMP52]]
+; CHECK-NEXT:    [[TMP54:%.*]] = icmp uge <16 x i32> [[A0]], [[A1]]
+; CHECK-NEXT:    [[TMP55:%.*]] = bitcast <16 x i1> [[TMP53]] to i16
+; CHECK-NEXT:    [[TMP56:%.*]] = bitcast <16 x i1> [[TMP54]] to i16
+; CHECK-NEXT:    [[_MSPROP6:%.*]] = insertelement <8 x i16> [[_MSPROP5]], i16 [[TMP55]], i32 5
+; CHECK-NEXT:    [[VEC5:%.*]] = insertelement <8 x i16> [[VEC4]], i16 [[TMP56]], i32 5
+; CHECK-NEXT:    [[TMP57:%.*]] = xor <16 x i32> [[TMP1]], splat (i32 -1)
+; CHECK-NEXT:    [[TMP58:%.*]] = and <16 x i32> [[A0]], [[TMP57]]
+; CHECK-NEXT:    [[TMP59:%.*]] = or <16 x i32> [[A0]], [[TMP1]]
+; CHECK-NEXT:    [[TMP60:%.*]] = xor <16 x i32> [[TMP2]], splat (i32 -1)
+; CHECK-NEXT:    [[TMP61:%.*]] = and <16 x i32> [[A1]], [[TMP60]]
+; CHECK-NEXT:    [[TMP62:%.*]] = or <16 x i32> [[A1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP63:%.*]] = icmp ugt <16 x i32> [[TMP58]], [[TMP62]]
+; CHECK-NEXT:    [[TMP64:%.*]] = icmp ugt <16 x i32> [[TMP59]], [[TMP61]]
+; CHECK-NEXT:    [[TMP65:%.*]] = xor <16 x i1> [[TMP63]], [[TMP64]]
+; CHECK-NEXT:    [[TMP66:%.*]] = icmp ugt <16 x i32> [[A0]], [[A1]]
+; CHECK-NEXT:    [[TMP67:%.*]] = bitcast <16 x i1> [[TMP65]] to i16
+; CHECK-NEXT:    [[TMP68:%.*]] = bitcast <16 x i1> [[TMP66]] to i16
+; CHECK-NEXT:    [[_MSPROP7:%.*]] = insertelement <8 x i16> [[_MSPROP6]], i16 [[TMP67]], i32 6
+; CHECK-NEXT:    [[VEC6:%.*]] = insertelement <8 x i16> [[VEC5]], i16 [[TMP68]], i32 6
+; CHECK-NEXT:    [[_MSPROP8:%.*]] = insertelement <8 x i16> [[_MSPROP7]], i16 0, i32 7
+; CHECK-NEXT:    [[VEC7:%.*]] = insertelement <8 x i16> [[VEC6]], i16 -1, i32 7
+; CHECK-NEXT:    store <8 x i16> [[_MSPROP8]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i16> [[VEC7]]
+;
+  %res0 = call i16 @llvm.x86.avx512.mask.ucmp.d.512(<16 x i32> %a0, <16 x i32> %a1, i32 0, i16 -1)
+  %vec0 = insertelement <8 x i16> %extra_param, i16 %res0, i32 0
+  %res1 = call i16 @llvm.x86.avx512.mask.ucmp.d.512(<16 x i32> %a0, <16 x i32> %a1, i32 1, i16 -1)
+  %vec1 = insertelement <8 x i16> %vec0, i16 %res1, i32 1
+  %res2 = call i16 @llvm.x86.avx512.mask.ucmp.d.512(<16 x i32> %a0, <16 x i32> %a1, i32 2, i16 -1)
+  %vec2 = insertelement <8 x i16> %vec1, i16 %res2, i32 2
+  %res3 = call i16 @llvm.x86.avx512.mask.ucmp.d.512(<16 x i32> %a0, <16 x i32> %a1, i32 3, i16 -1)
+  %vec3 = insertelement <8 x i16> %vec2, i16 %res3, i32 3
+  %res4 = call i16 @llvm.x86.avx512.mask.ucmp.d.512(<16 x i32> %a0, <16 x i32> %a1, i32 4, i16 -1)
+  %vec4 = insertelement <8 x i16> %vec3, i16 %res4, i32 4
+  %res5 = call i16 @llvm.x86.avx512.mask.ucmp.d.512(<16 x i32> %a0, <16 x i32> %a1, i32 5, i16 -1)
+  %vec5 = insertelement <8 x i16> %vec4, i16 %res5, i32 5
+  %res6 = call i16 @llvm.x86.avx512.mask.ucmp.d.512(<16 x i32> %a0, <16 x i32> %a1, i32 6, i16 -1)
+  %vec6 = insertelement <8 x i16> %vec5, i16 %res6, i32 6
+  %res7 = call i16 @llvm.x86.avx512.mask.ucmp.d.512(<16 x i32> %a0, <16 x i32> %a1, i32 7, i16 -1)
+  %vec7 = insertelement <8 x i16> %vec6, i16 %res7, i32 7
+  ret <8 x i16> %vec7
+}
+
+define <8 x i16> @test_mask_ucmp_d_512(<16 x i32> %a0, <16 x i32> %a1, i16 %mask, <8 x i16> %extra_param)  #0 {
+;
+; CHECK-LABEL: @test_mask_ucmp_d_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    [[TMP138:%.*]] = load <8 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 136) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = xor <16 x i32> [[A0:%.*]], [[A1:%.*]]
+; CHECK-NEXT:    [[TMP5:%.*]] = or <16 x i32> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP6:%.*]] = icmp ne <16 x i32> [[TMP5]], zeroinitializer
+; CHECK-NEXT:    [[TMP7:%.*]] = xor <16 x i32> [[TMP5]], splat (i32 -1)
+; CHECK-NEXT:    [[TMP8:%.*]] = and <16 x i32> [[TMP7]], [[TMP4]]
+; CHECK-NEXT:    [[TMP9:%.*]] = icmp eq <16 x i32> [[TMP8]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_ICMP:%.*]] = and <16 x i1> [[TMP6]], [[TMP9]]
+; CHECK-NEXT:    [[TMP10:%.*]] = icmp eq <16 x i32> [[A0]], [[A1]]
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast i16 [[TMP3]] to <16 x i1>
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast i16 [[MASK:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP13:%.*]] = and <16 x i1> [[_MSPROP_ICMP]], [[TMP11]]
+; CHECK-NEXT:    [[TMP14:%.*]] = and <16 x i1> [[TMP10]], [[TMP11]]
+; CHECK-NEXT:    [[TMP15:%.*]] = and <16 x i1> [[_MSPROP_ICMP]], [[TMP12]]
+; CHECK-NEXT:    [[TMP16:%.*]] = or <16 x i1> [[TMP13]], [[TMP14]]
+; CHECK-NEXT:    [[TMP17:%.*]] = or <16 x i1> [[TMP16]], [[TMP15]]
+; CHECK-NEXT:    [[TMP18:%.*]] = and <16 x i1> [[TMP10]], [[TMP12]]
+; CHECK-NEXT:    [[TMP19:%.*]] = bitcast <16 x i1> [[TMP17]] to i16
+; CHECK-NEXT:    [[TMP20:%.*]] = bitcast <16 x i1> [[TMP18]] to i16
+; CHECK-NEXT:    [[_MSPROP:%.*]] = insertelement <8 x i16> [[TMP138]], i16 [[TMP19]], i32 0
+; CHECK-NEXT:    [[VEC0:%.*]] = insertelement <8 x i16> [[EXTRA_PARAM:%.*]], i16 [[TMP20]], i32 0
+; CHECK-NEXT:    [[TMP21:%.*]] = xor <16 x i32> [[TMP1]], splat (i32 -1)
+; CHECK-NEXT:    [[TMP22:%.*]] = and <16 x i32> [[A0]], [[TMP21]]
+; CHECK-NEXT:    [[TMP23:%.*]] = or <16 x i32> [[A0]], [[TMP1]]
+; CHECK-NEXT:    [[TMP24:%.*]] = xor <16 x i32> [[TMP2]], splat (i32 -1)
+; CHECK-NEXT:    [[TMP25:%.*]] = and <16 x i32> [[A1]], [[TMP24]]
+; CHECK-NEXT:    [[TMP26:%.*]] = or <16 x i32> [[A1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP27:%.*]] = icmp ult <16 x i32> [[TMP22]], [[TMP26]]
+; CHECK-NEXT:    [[TMP28:%.*]] = icmp ult <16 x i32> [[TMP23]], [[TMP25]]
+; CHECK-NEXT:    [[TMP29:%.*]] = xor <16 x i1> [[TMP27]], [[TMP28]]
+; CHECK-NEXT:    [[TMP30:%.*]] = icmp ult <16 x i32> [[A0]], [[A1]]
+; CHECK-NEXT:    [[TMP31:%.*]] = bitcast i16 [[TMP3]] to <16 x i1>
+; CHECK-NEXT:    [[TMP32:%.*]] = bitcast i16 [[MASK]] to <16 x i1>
+; CHECK-NEXT:    [[TMP33:%.*]] = and <16 x i1> [[TMP29]], [[TMP31]]
+; CHECK-NEXT:    [[TMP34:%.*]] = and <16 x i1> [[TMP30]], [[TMP31]]
+; CHECK-NEXT:    [[TMP35:%.*]] = and <16 x i1> [[TMP29]], [[TMP32]]
+; CHECK-NEXT:    [[TMP36:%.*]] = or <16 x i1> [[TMP33]], [[TMP34]]
+; CHECK-NEXT:    [[TMP37:%.*]] = or <16 x i1> [[TMP36]], [[TMP35]]
+; CHECK-NEXT:    [[TMP38:%.*]] = and <16 x i1> [[TMP30]], [[TMP32]]
+; CHECK-NEXT:    [[TMP39:%.*]] = bitcast <16 x i1> [[TMP37]] to i16
+; CHECK-NEXT:    [[TMP40:%.*]] = bitcast <16 x i1> [[TMP38]] to i16
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = insertelement <8 x i16> [[_MSPROP]], i16 [[TMP39]], i32 1
+; CHECK-NEXT:    [[VEC1:%.*]] = insertelement <8 x i16> [[VEC0]], i16 [[TMP40]], i32 1
+; CHECK-NEXT:    [[TMP41:%.*]] = xor <16 x i32> [[TMP1]], splat (i32 -1)
+; CHECK-NEXT:    [[TMP42:%.*]] = and <16 x i32> [[A0]], [[TMP41]]
+; CHECK-NEXT:    [[TMP43:%.*]] = or <16 x i32> [[A0]], [[TMP1]]
+; CHECK-NEXT:    [[TMP44:%.*]] = xor <16 x i32> [[TMP2]], splat (i32 -1)
+; CHECK-NEXT:    [[TMP45:%.*]] = and <16 x i32> [[A1]], [[TMP44]]
+; CHECK-NEXT:    [[TMP46:%.*]] = or <16 x i32> [[A1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP47:%.*]] = icmp ule <16 x i32> [[TMP42]], [[TMP46]]
+; CHECK-NEXT:    [[TMP48:%.*]] = icmp ule <16 x i32> [[TMP43]], [[TMP45]]
+; CHECK-NEXT:    [[TMP49:%.*]] = xor <16 x i1> [[TMP47]], [[TMP48]]
+; CHECK-NEXT:    [[TMP50:%.*]] = icmp ule <16 x i32> [[A0]], [[A1]]
+; CHECK-NEXT:    [[TMP51:%.*]] = bitcast i16 [[TMP3]] to <16 x i1>
+; CHECK-NEXT:    [[TMP52:%.*]] = bitcast i16 [[MASK]] to <16 x i1>
+; CHECK-NEXT:    [[TMP53:%.*]] = and <16 x i1> [[TMP49]], [[TMP51]]
+; CHECK-NEXT:    [[TMP54:%.*]] = and <16 x i1> [[TMP50]], [[TMP51]]
+; CHECK-NEXT:    [[TMP55:%.*]] = and <16 x i1> [[TMP49]], [[TMP52]]
+; CHECK-NEXT:    [[TMP56:%.*]] = or <16 x i1> [[TMP53]], [[TMP54]]
+; CHECK-NEXT:    [[TMP57:%.*]] = or <16 x i1> [[TMP56]], [[TMP55]]
+; CHECK-NEXT:    [[TMP58:%.*]] = and <16 x i1> [[TMP50]], [[TMP52]]
+; CHECK-NEXT:    [[TMP59:%.*]] = bitcast <16 x i1> [[TMP57]] to i16
+; CHECK-NEXT:    [[TMP60:%.*]] = bitcast <16 x i1> [[TMP58]] to i16
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = insertelement <8 x i16> [[_MSPROP1]], i16 [[TMP59]], i32 2
+; CHECK-NEXT:    [[VEC2:%.*]] = insertelement <8 x i16> [[VEC1]], i16 [[TMP60]], i32 2
+; CHECK-NEXT:    [[TMP61:%.*]] = bitcast i16 [[TMP3]] to <16 x i1>
+; CHECK-NEXT:    [[TMP62:%.*]] = bitcast i16 [[MASK]] to <16 x i1>
+; CHECK-NEXT:    [[TMP63:%.*]] = and <16 x i1> zeroinitializer, [[TMP61]]
+; CHECK-NEXT:    [[TMP64:%.*]] = and <16 x i1> zeroinitializer, [[TMP61]]
+; CHECK-NEXT:    [[TMP65:%.*]] = and <16 x i1> zeroinitializer, [[TMP62]]
+; CHECK-NEXT:    [[TMP66:%.*]] = or <16 x i1> [[TMP63]], [[TMP64]]
+; CHECK-NEXT:    [[TMP67:%.*]] = or <16 x i1> [[TMP66]], [[TMP65]]
+; CHECK-NEXT:    [[TMP68:%.*]] = and <16 x i1> zeroinitializer, [[TMP62]]
+; CHECK-NEXT:    [[TMP69:%.*]] = bitcast <16 x i1> [[TMP67]] to i16
+; CHECK-NEXT:    [[TMP70:%.*]] = bitcast <16 x i1> [[TMP68]] to i16
+; CHECK-NEXT:    [[_MSPROP3:%.*]] = insertelement <8 x i16> [[_MSPROP2]], i16 [[TMP69]], i32 3
+; CHECK-NEXT:    [[VEC3:%.*]] = insertelement <8 x i16> [[VEC2]], i16 [[TMP70]], i32 3
+; CHECK-NEXT:    [[TMP71:%.*]] = xor <16 x i32> [[A0]], [[A1]]
+; CHECK-NEXT:    [[TMP72:%.*]] = or <16 x i32> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP73:%.*]] = icmp ne <16 x i32> [[TMP72]], zeroinitializer
+; CHECK-NEXT:    [[TMP74:%.*]] = xor <16 x i32> [[TMP72]], splat (i32 -1)
+; CHECK-NEXT:    [[TMP75:%.*]] = and <16 x i32> [[TMP74]], [[TMP71]]
+; CHECK-NEXT:    [[TMP76:%.*]] = icmp eq <16 x i32> [[TMP75]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_ICMP4:%.*]] = and <16 x i1> [[TMP73]], [[TMP76]]
+; CHECK-NEXT:    [[TMP77:%.*]] = icmp ne <16 x i32> [[A0]], [[A1]]
+; CHECK-NEXT:    [[TMP78:%.*]] = bitcast i16 [[TMP3]] to <16 x i1>
+; CHECK-NEXT:    [[TMP79:%.*]] = bitcast i16 [[MASK]] to <16 x i1>
+; CHECK-NEXT:    [[TMP80:%.*]] = and <16 x i1> [[_MSPROP_ICMP4]], [[TMP78]]
+; CHECK-NEXT:    [[TMP81:%.*]] = and <16 x i1> [[TMP77]], [[TMP78]]
+; CHECK-NEXT:    [[TMP82:%.*]] = and <16 x i1> [[_MSPROP_ICMP4]], [[TMP79]]
+; CHECK-NEXT:    [[TMP83:%.*]] = or <16 x i1> [[TMP80]], [[TMP81]]
+; CHECK-NEXT:    [[TMP84:%.*]] = or <16 x i1> [[TMP83]], [[TMP82]]
+; CHECK-NEXT:    [[TMP85:%.*]] = and <16 x i1> [[TMP77]], [[TMP79]]
+; CHECK-NEXT:    [[TMP86:%.*]] = bitcast <16 x i1> [[TMP84]] to i16
+; CHECK-NEXT:    [[TMP87:%.*]] = bitcast <16 x i1> [[TMP85]] to i16
+; CHECK-NEXT:    [[_MSPROP5:%.*]] = insertelement <8 x i16> [[_MSPROP3]], i16 [[TMP86]], i32 4
+; CHECK-NEXT:    [[VEC4:%.*]] = insertelement <8 x i16> [[VEC3]], i16 [[TMP87]], i32 4
+; CHECK-NEXT:    [[TMP88:%.*]] = xor <16 x i32> [[TMP1]], splat (i32 -1)
+; CHECK-NEXT:    [[TMP89:%.*]] = and <16 x i32> [[A0]], [[TMP88]]
+; CHECK-NEXT:    [[TMP90:%.*]] = or <16 x i32> [[A0]], [[TMP1]]
+; CHECK-NEXT:    [[TMP91:%.*]] = xor <16 x i32> [[TMP2]], splat (i32 -1)
+; CHECK-NEXT:    [[TMP92:%.*]] = and <16 x i32> [[A1]], [[TMP91]]
+; CHECK-NEXT:    [[TMP93:%.*]] = or <16 x i32> [[A1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP94:%.*]] = icmp uge <16 x i32> [[TMP89]], [[TMP93]]
+; CHECK-NEXT:    [[TMP95:%.*]] = icmp uge <16 x i32> [[TMP90]], [[TMP92]]
+; CHECK-NEXT:    [[TMP96:%.*]] = xor <16 x i1> [[TMP94]], [[TMP95]]
+; CHECK-NEXT:    [[TMP97:%.*]] = icmp uge <16 x i32> [[A0]], [[A1]]
+; CHECK-NEXT:    [[TMP98:%.*]] = bitcast i16 [[TMP3]] to <16 x i1>
+; CHECK-NEXT:    [[TMP99:%.*]] = bitcast i16 [[MASK]] to <16 x i1>
+; CHECK-NEXT:    [[TMP100:%.*]] = and <16 x i1> [[TMP96]], [[TMP98]]
+; CHECK-NEXT:    [[TMP101:%.*]] = and <16 x i1> [[TMP97]], [[TMP98]]
+; CHECK-NEXT:    [[TMP102:%.*]] = and <16 x i1> [[TMP96]], [[TMP99]]
+; CHECK-NEXT:    [[TMP103:%.*]] = or <16 x i1> [[TMP100]], [[TMP101]]
+; CHECK-NEXT:    [[TMP104:%.*]] = or <16 x i1> [[TMP103]], [[TMP102]]
+; CHECK-NEXT:    [[TMP105:%.*]] = and <16 x i1> [[TMP97]], [[TMP99]]
+; CHECK-NEXT:    [[TMP106:%.*]] = bitcast <16 x i1> [[TMP104]] to i16
+; CHECK-NEXT:    [[TMP107:%.*]] = bitcast <16 x i1> [[TMP105]] to i16
+; CHECK-NEXT:    [[_MSPROP6:%.*]] = insertelement <8 x i16> [[_MSPROP5]], i16 [[TMP106]], i32 5
+; CHECK-NEXT:    [[VEC5:%.*]] = insertelement <8 x i16> [[VEC4]], i16 [[TMP107]], i32 5
+; CHECK-NEXT:    [[TMP108:%.*]] = xor <16 x i32> [[TMP1]], splat (i32 -1)
+; CHECK-NEXT:    [[TMP109:%.*]] = and <16 x i32> [[A0]], [[TMP108]]
+; CHECK-NEXT:    [[TMP110:%.*]] = or <16 x i32> [[A0]], [[TMP1]]
+; CHECK-NEXT:    [[TMP111:%.*]] = xor <16 x i32> [[TMP2]], splat (i32 -1)
+; CHECK-NEXT:    [[TMP112:%.*]] = and <16 x i32> [[A1]], [[TMP111]]
+; CHECK-NEXT:    [[TMP113:%.*]] = or <16 x i32> [[A1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP114:%.*]] = icmp ugt <16 x i32> [[TMP109]], [[TMP113]]
+; CHECK-NEXT:    [[TMP115:%.*]] = icmp ugt <16 x i32> [[TMP110]], [[TMP112]]
+; CHECK-NEXT:    [[TMP116:%.*]] = xor <16 x i1> [[TMP114]], [[TMP115]]
+; CHECK-NEXT:    [[TMP117:%.*]] = icmp ugt <16 x i32> [[A0]], [[A1]]
+; CHECK-NEXT:    [[TMP118:%.*]] = bitcast i16 [[TMP3]] to <16 x i1>
+; CHECK-NEXT:    [[TMP119:%.*]] = bitcast i16 [[MASK]] to <16 x i1>
+; CHECK-NEXT:    [[TMP120:%.*]] = and <16 x i1> [[TMP116]], [[TMP118]]
+; CHECK-NEXT:    [[TMP121:%.*]] = and <16 x i1> [[TMP117]], [[TMP118]]
+; CHECK-NEXT:    [[TMP122:%.*]] = and <16 x i1> [[TMP116]], [[TMP119]]
+; CHECK-NEXT:    [[TMP123:%.*]] = or <16 x i1> [[TMP120]], [[TMP121]]
+; CHECK-NEXT:    [[TMP124:%.*]] = or <16 x i1> [[TMP123]], [[TMP122]]
+; CHECK-NEXT:    [[TMP125:%.*]] = and <16 x i1> [[TMP117]], [[TMP119]]
+; CHECK-NEXT:    [[TMP126:%.*]] = bitcast <16 x i1> [[TMP124]] to i16
+; CHECK-NEXT:    [[TMP127:%.*]] = bitcast <16 x i1> [[TMP125]] to i16
+; CHECK-NEXT:    [[_MSPROP7:%.*]] = insertelement <8 x i16> [[_MSPROP6]], i16 [[TMP126]], i32 6
+; CHECK-NEXT:    [[VEC6:%.*]] = insertelement <8 x i16> [[VEC5]], i16 [[TMP127]], i32 6
+; CHECK-NEXT:    [[TMP128:%.*]] = bitcast i16 [[TMP3]] to <16 x i1>
+; CHECK-NEXT:    [[TMP129:%.*]] = bitcast i16 [[MASK]] to <16 x i1>
+; CHECK-NEXT:    [[TMP130:%.*]] = and <16 x i1> zeroinitializer, [[TMP128]]
+; CHECK-NEXT:    [[TMP131:%.*]] = and <16 x i1> splat (i1 true), [[TMP128]]
+; CHECK-NEXT:    [[TMP132:%.*]] = and <16 x i1> zeroinitializer, [[TMP129]]
+; CHECK-NEXT:    [[TMP133:%.*]] = or <16 x i1> [[TMP130]], [[TMP131]]
+; CHECK-NEXT:    [[TMP134:%.*]] = or <16 x i1> [[TMP133]], [[TMP132]]
+; CHECK-NEXT:    [[TMP135:%.*]] = and <16 x i1> splat (i1 true), [[TMP129]]
+; CHECK-NEXT:    [[TMP136:%.*]] = bitcast <16 x i1> [[TMP134]] to i16
+; CHECK-NEXT:    [[TMP137:%.*]] = bitcast <16 x i1> [[TMP135]] to i16
+; CHECK-NEXT:    [[_MSPROP8:%.*]] = insertelement <8 x i16> [[_MSPROP7]], i16 [[TMP136]], i32 7
+; CHECK-NEXT:    [[VEC7:%.*]] = insertelement <8 x i16> [[VEC6]], i16 [[TMP137]], i32 7
+; CHECK-NEXT:    store <8 x i16> [[_MSPROP8]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i16> [[VEC7]]
+;
+  %res0 = call i16 @llvm.x86.avx512.mask.ucmp.d.512(<16 x i32> %a0, <16 x i32> %a1, i32 0, i16 %mask)
+  %vec0 = insertelement <8 x i16> %extra_param, i16 %res0, i32 0
+  %res1 = call i16 @llvm.x86.avx512.mask.ucmp.d.512(<16 x i32> %a0, <16 x i32> %a1, i32 1, i16 %mask)
+  %vec1 = insertelement <8 x i16> %vec0, i16 %res1, i32 1
+  %res2 = call i16 @llvm.x86.avx512.mask.ucmp.d.512(<16 x i32> %a0, <16 x i32> %a1, i32 2, i16 %mask)
+  %vec2 = insertelement <8 x i16> %vec1, i16 %res2, i32 2
+  %res3 = call i16 @llvm.x86.avx512.mask.ucmp.d.512(<16 x i32> %a0, <16 x i32> %a1, i32 3, i16 %mask)
+  %vec3 = insertelement <8 x i16> %vec2, i16 %res3, i32 3
+  %res4 = call i16 @llvm.x86.avx512.mask.ucmp.d.512(<16 x i32> %a0, <16 x i32> %a1, i32 4, i16 %mask)
+  %vec4 = insertelement <8 x i16> %vec3, i16 %res4, i32 4
+  %res5 = call i16 @llvm.x86.avx512.mask.ucmp.d.512(<16 x i32> %a0, <16 x i32> %a1, i32 5, i16 %mask)
+  %vec5 = insertelement <8 x i16> %vec4, i16 %res5, i32 5
+  %res6 = call i16 @llvm.x86.avx512.mask.ucmp.d.512(<16 x i32> %a0, <16 x i32> %a1, i32 6, i16 %mask)
+  %vec6 = insertelement <8 x i16> %vec5, i16 %res6, i32 6
+  %res7 = call i16 @llvm.x86.avx512.mask.ucmp.d.512(<16 x i32> %a0, <16 x i32> %a1, i32 7, i16 %mask)
+  %vec7 = insertelement <8 x i16> %vec6, i16 %res7, i32 7
+  ret <8 x i16> %vec7
+}
+
+declare i16 @llvm.x86.avx512.mask.ucmp.d.512(<16 x i32>, <16 x i32>, i32, i16) nounwind readnone
+
+define <8 x i8> @test_cmp_q_512(<8 x i64> %a0, <8 x i64> %a1, <8 x i8> %extra_param)  #0 {
+; CHECK-LABEL: @test_cmp_q_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP77:%.*]] = load <8 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = xor <8 x i64> [[A0:%.*]], [[A1:%.*]]
+; CHECK-NEXT:    [[TMP4:%.*]] = or <8 x i64> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp ne <8 x i64> [[TMP4]], zeroinitializer
+; CHECK-NEXT:    [[TMP6:%.*]] = xor <8 x i64> [[TMP4]], splat (i64 -1)
+; CHECK-NEXT:    [[TMP7:%.*]] = and <8 x i64> [[TMP6]], [[TMP3]]
+; CHECK-NEXT:    [[TMP8:%.*]] = icmp eq <8 x i64> [[TMP7]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_ICMP:%.*]] = and <8 x i1> [[TMP5]], [[TMP8]]
+; CHECK-NEXT:    [[TMP9:%.*]] = icmp eq <8 x i64> [[A0]], [[A1]]
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast <8 x i1> [[_MSPROP_ICMP]] to i8
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <8 x i1> [[TMP9]] to i8
+; CHECK-NEXT:    [[_MSPROP:%.*]] = insertelement <8 x i8> [[TMP77]], i8 [[TMP10]], i32 0
+; CHECK-NEXT:    [[VEC0:%.*]] = insertelement <8 x i8> [[EXTRA_PARAM:%.*]], i8 [[TMP11]], i32 0
+; CHECK-NEXT:    [[TMP12:%.*]] = xor <8 x i64> [[A0]], splat (i64 -9223372036854775808)
+; CHECK-NEXT:    [[TMP13:%.*]] = xor <8 x i64> [[TMP1]], splat (i64 -1)
+; CHECK-NEXT:    [[TMP14:%.*]] = and <8 x i64> [[TMP12]], [[TMP13]]
+; CHECK-NEXT:    [[TMP15:%.*]] = or <8 x i64> [[TMP12]], [[TMP1]]
+; CHECK-NEXT:    [[TMP16:%.*]] = xor <8 x i64> [[A1]], splat (i64 -9223372036854775808)
+; CHECK-NEXT:    [[TMP17:%.*]] = xor <8 x i64> [[TMP2]], splat (i64 -1)
+; CHECK-NEXT:    [[TMP18:%.*]] = and <8 x i64> [[TMP16]], [[TMP17]]
+; CHECK-NEXT:    [[TMP19:%.*]] = or <8 x i64> [[TMP16]], [[TMP2]]
+; CHECK-NEXT:    [[TMP20:%.*]] = icmp ult <8 x i64> [[TMP14]], [[TMP19]]
+; CHECK-NEXT:    [[TMP21:%.*]] = icmp ult <8 x i64> [[TMP15]], [[TMP18]]
+; CHECK-NEXT:    [[TMP22:%.*]] = xor <8 x i1> [[TMP20]], [[TMP21]]
+; CHECK-NEXT:    [[TMP23:%.*]] = icmp slt <8 x i64> [[A0]], [[A1]]
+; CHECK-NEXT:    [[TMP24:%.*]] = bitcast <8 x i1> [[TMP22]] to i8
+; CHECK-NEXT:    [[TMP25:%.*]] = bitcast <8 x i1> [[TMP23]] to i8
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = insertelement <8 x i8> [[_MSPROP]], i8 [[TMP24]], i32 1
+; CHECK-NEXT:    [[VEC1:%.*]] = insertelement <8 x i8> [[VEC0]], i8 [[TMP25]], i32 1
+; CHECK-NEXT:    [[TMP26:%.*]] = xor <8 x i64> [[A0]], splat (i64 -9223372036854775808)
+; CHECK-NEXT:    [[TMP27:%.*]] = xor <8 x i64> [[TMP1]], splat (i64 -1)
+; CHECK-NEXT:    [[TMP28:%.*]] = and <8 x i64> [[TMP26]], [[TMP27]]
+; CHECK-NEXT:    [[TMP29:%.*]] = or <8 x i64> [[TMP26]], [[TMP1]]
+; CHECK-NEXT:    [[TMP30:%.*]] = xor <8 x i64> [[A1]], splat (i64 -9223372036854775808)
+; CHECK-NEXT:    [[TMP31:%.*]] = xor <8 x i64> [[TMP2]], splat (i64 -1)
+; CHECK-NEXT:    [[TMP32:%.*]] = and <8 x i64> [[TMP30]], [[TMP31]]
+; CHECK-NEXT:    [[TMP33:%.*]] = or <8 x i64> [[TMP30]], [[TMP2]]
+; CHECK-NEXT:    [[TMP34:%.*]] = icmp ule <8 x i64> [[TMP28]], [[TMP33]]
+; CHECK-NEXT:    [[TMP35:%.*]] = icmp ule <8 x i64> [[TMP29]], [[TMP32]]
+; CHECK-NEXT:    [[TMP36:%.*]] = xor <8 x i1> [[TMP34]], [[TMP35]]
+; CHECK-NEXT:    [[TMP37:%.*]] = icmp sle <8 x i64> [[A0]], [[A1]]
+; CHECK-NEXT:    [[TMP38:%.*]] = bitcast <8 x i1> [[TMP36]] to i8
+; CHECK-NEXT:    [[TMP39:%.*]] = bitcast <8 x i1> [[TMP37]] to i8
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = insertelement <8 x i8> [[_MSPROP1]], i8 [[TMP38]], i32 2
+; CHECK-NEXT:    [[VEC2:%.*]] = insertelement <8 x i8> [[VEC1]], i8 [[TMP39]], i32 2
+; CHECK-NEXT:    [[_MSPROP3:%.*]] = insertelement <8 x i8> [[_MSPROP2]], i8 0, i32 3
+; CHECK-NEXT:    [[VEC3:%.*]] = insertelement <8 x i8> [[VEC2]], i8 0, i32 3
+; CHECK-NEXT:    [[TMP40:%.*]] = xor <8 x i64> [[A0]], [[A1]]
+; CHECK-NEXT:    [[TMP41:%.*]] = or <8 x i64> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP42:%.*]] = icmp ne <8 x i64> [[TMP41]], zeroinitializer
+; CHECK-NEXT:    [[TMP43:%.*]] = xor <8 x i64> [[TMP41]], splat (i64 -1)
+; CHECK-NEXT:    [[TMP44:%.*]] = and <8 x i64> [[TMP43]], [[TMP40]]
+; CHECK-NEXT:    [[TMP45:%.*]] = icmp eq <8 x i64> [[TMP44]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_ICMP4:%.*]] = and <8 x i1> [[TMP42]], [[TMP45]]
+; CHECK-NEXT:    [[TMP46:%.*]] = icmp ne <8 x i64> [[A0]], [[A1]]
+; CHECK-NEXT:    [[TMP47:%.*]] = bitcast <8 x i1> [[_MSPROP_ICMP4]] to i8
+; CHECK-NEXT:    [[TMP48:%.*]] = bitcast <8 x i1> [[TMP46]] to i8
+; CHECK-NEXT:    [[_MSPROP5:%.*]] = insertelement <8 x i8> [[_MSPROP3]], i8 [[TMP47]], i32 4
+; CHECK-NEXT:    [[VEC4:%.*]] = insertelement <8 x i8> [[VEC3]], i8 [[TMP48]], i32 4
+; CHECK-NEXT:    [[TMP49:%.*]] = xor <8 x i64> [[A0]], splat (i64 -9223372036854775808)
+; CHECK-NEXT:    [[TMP50:%.*]] = xor <8 x i64> [[TMP1]], splat (i64 -1)
+; CHECK-NEXT:    [[TMP51:%.*]] = and <8 x i64> [[TMP49]], [[TMP50]]
+; CHECK-NEXT:    [[TMP52:%.*]] = or <8 x i64> [[TMP49]], [[TMP1]]
+; CHECK-NEXT:    [[TMP53:%.*]] = xor <8 x i64> [[A1]], splat (i64 -9223372036854775808)
+; CHECK-NEXT:    [[TMP54:%.*]] = xor <8 x i64> [[TMP2]], splat (i64 -1)
+; CHECK-NEXT:    [[TMP55:%.*]] = and <8 x i64> [[TMP53]], [[TMP54]]
+; CHECK-NEXT:    [[TMP56:%.*]] = or <8 x i64> [[TMP53]], [[TMP2]]
+; CHECK-NEXT:    [[TMP57:%.*]] = icmp uge <8 x i64> [[TMP51]], [[TMP56]]
+; CHECK-NEXT:    [[TMP58:%.*]] = icmp uge <8 x i64> [[TMP52]], [[TMP55]]
+; CHECK-NEXT:    [[TMP59:%.*]] = xor <8 x i1> [[TMP57]], [[TMP58]]
+; CHECK-NEXT:    [[TMP60:%.*]] = icmp sge <8 x i64> [[A0]], [[A1]]
+; CHECK-NEXT:    [[TMP61:%.*]] = bitcast <8 x i1> [[TMP59]] to i8
+; CHECK-NEXT:    [[TMP62:%.*]] = bitcast <8 x i1> [[TMP60]] to i8
+; CHECK-NEXT:    [[_MSPROP6:%.*]] = insertelement <8 x i8> [[_MSPROP5]], i8 [[TMP61]], i32 5
+; CHECK-NEXT:    [[VEC5:%.*]] = insertelement <8 x i8> [[VEC4]], i8 [[TMP62]], i32 5
+; CHECK-NEXT:    [[TMP63:%.*]] = xor <8 x i64> [[A0]], splat (i64 -9223372036854775808)
+; CHECK-NEXT:    [[TMP64:%.*]] = xor <8 x i64> [[TMP1]], splat (i64 -1)
+; CHECK-NEXT:    [[TMP65:%.*]] = and <8 x i64> [[TMP63]], [[TMP64]]
+; CHECK-NEXT:    [[TMP66:%.*]] = or <8 x i64> [[TMP63]], [[TMP1]]
+; CHECK-NEXT:    [[TMP67:%.*]] = xor <8 x i64> [[A1]], splat (i64 -9223372036854775808)
+; CHECK-NEXT:    [[TMP68:%.*]] = xor <8 x i64> [[TMP2]], splat (i64 -1)
+; CHECK-NEXT:    [[TMP69:%.*]] = and <8 x i64> [[TMP67]], [[TMP68]]
+; CHECK-NEXT:    [[TMP70:%.*]] = or <8 x i64> [[TMP67]], [[TMP2]]
+; CHECK-NEXT:    [[TMP71:%.*]] = icmp ugt <8 x i64> [[TMP65]], [[TMP70]]
+; CHECK-NEXT:    [[TMP72:%.*]] = icmp ugt <8 x i64> [[TMP66]], [[TMP69]]
+; CHECK-NEXT:    [[TMP73:%.*]] = xor <8 x i1> [[TMP71]], [[TMP72]]
+; CHECK-NEXT:    [[TMP74:%.*]] = icmp sgt <8 x i64> [[A0]], [[A1]]
+; CHECK-NEXT:    [[TMP75:%.*]] = bitcast <8 x i1> [[TMP73]] to i8
+; CHECK-NEXT:    [[TMP76:%.*]] = bitcast <8 x i1> [[TMP74]] to i8
+; CHECK-NEXT:    [[_MSPROP7:%.*]] = insertelement <8 x i8> [[_MSPROP6]], i8 [[TMP75]], i32 6
+; CHECK-NEXT:    [[VEC6:%.*]] = insertelement <8 x i8> [[VEC5]], i8 [[TMP76]], i32 6
+; CHECK-NEXT:    [[_MSPROP8:%.*]] = insertelement <8 x i8> [[_MSPROP7]], i8 0, i32 7
+; CHECK-NEXT:    [[VEC7:%.*]] = insertelement <8 x i8> [[VEC6]], i8 -1, i32 7
+; CHECK-NEXT:    store <8 x i8> [[_MSPROP8]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i8> [[VEC7]]
+;
+  %res0 = call i8 @llvm.x86.avx512.mask.cmp.q.512(<8 x i64> %a0, <8 x i64> %a1, i32 0, i8 -1)
+  %vec0 = insertelement <8 x i8> %extra_param, i8 %res0, i32 0
+  %res1 = call i8 @llvm.x86.avx512.mask.cmp.q.512(<8 x i64> %a0, <8 x i64> %a1, i32 1, i8 -1)
+  %vec1 = insertelement <8 x i8> %vec0, i8 %res1, i32 1
+  %res2 = call i8 @llvm.x86.avx512.mask.cmp.q.512(<8 x i64> %a0, <8 x i64> %a1, i32 2, i8 -1)
+  %vec2 = insertelement <8 x i8> %vec1, i8 %res2, i32 2
+  %res3 = call i8 @llvm.x86.avx512.mask.cmp.q.512(<8 x i64> %a0, <8 x i64> %a1, i32 3, i8 -1)
+  %vec3 = insertelement <8 x i8> %vec2, i8 %res3, i32 3
+  %res4 = call i8 @llvm.x86.avx512.mask.cmp.q.512(<8 x i64> %a0, <8 x i64> %a1, i32 4, i8 -1)
+  %vec4 = insertelement <8 x i8> %vec3, i8 %res4, i32 4
+  %res5 = call i8 @llvm.x86.avx512.mask.cmp.q.512(<8 x i64> %a0, <8 x i64> %a1, i32 5, i8 -1)
+  %vec5 = insertelement <8 x i8> %vec4, i8 %res5, i32 5
+  %res6 = call i8 @llvm.x86.avx512.mask.cmp.q.512(<8 x i64> %a0, <8 x i64> %a1, i32 6, i8 -1)
+  %vec6 = insertelement <8 x i8> %vec5, i8 %res6, i32 6
+  %res7 = call i8 @llvm.x86.avx512.mask.cmp.q.512(<8 x i64> %a0, <8 x i64> %a1, i32 7, i8 -1)
+  %vec7 = insertelement <8 x i8> %vec6, i8 %res7, i32 7
+  ret <8 x i8> %vec7
+}
+
+define <8 x i8> @test_mask_cmp_q_512(<8 x i64> %a0, <8 x i64> %a1, i8 %mask, <8 x i8> %extra_param)  #0 {
+;
+; CHECK-LABEL: @test_mask_cmp_q_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    [[TMP146:%.*]] = load <8 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 136) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = xor <8 x i64> [[A0:%.*]], [[A1:%.*]]
+; CHECK-NEXT:    [[TMP5:%.*]] = or <8 x i64> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP6:%.*]] = icmp ne <8 x i64> [[TMP5]], zeroinitializer
+; CHECK-NEXT:    [[TMP7:%.*]] = xor <8 x i64> [[TMP5]], splat (i64 -1)
+; CHECK-NEXT:    [[TMP8:%.*]] = and <8 x i64> [[TMP7]], [[TMP4]]
+; CHECK-NEXT:    [[TMP9:%.*]] = icmp eq <8 x i64> [[TMP8]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_ICMP:%.*]] = and <8 x i1> [[TMP6]], [[TMP9]]
+; CHECK-NEXT:    [[TMP10:%.*]] = icmp eq <8 x i64> [[A0]], [[A1]]
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast i8 [[TMP3]] to <8 x i1>
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP13:%.*]] = and <8 x i1> [[_MSPROP_ICMP]], [[TMP11]]
+; CHECK-NEXT:    [[TMP14:%.*]] = and <8 x i1> [[TMP10]], [[TMP11]]
+; CHECK-NEXT:    [[TMP15:%.*]] = and <8 x i1> [[_MSPROP_ICMP]], [[TMP12]]
+; CHECK-NEXT:    [[TMP16:%.*]] = or <8 x i1> [[TMP13]], [[TMP14]]
+; CHECK-NEXT:    [[TMP17:%.*]] = or <8 x i1> [[TMP16]], [[TMP15]]
+; CHECK-NEXT:    [[TMP18:%.*]] = and <8 x i1> [[TMP10]], [[TMP12]]
+; CHECK-NEXT:    [[TMP19:%.*]] = bitcast <8 x i1> [[TMP17]] to i8
+; CHECK-NEXT:    [[TMP20:%.*]] = bitcast <8 x i1> [[TMP18]] to i8
+; CHECK-NEXT:    [[_MSPROP:%.*]] = insertelement <8 x i8> [[TMP146]], i8 [[TMP19]], i32 0
+; CHECK-NEXT:    [[VEC0:%.*]] = insertelement <8 x i8> [[EXTRA_PARAM:%.*]], i8 [[TMP20]], i32 0
+; CHECK-NEXT:    [[TMP21:%.*]] = xor <8 x i64> [[A0]], splat (i64 -9223372036854775808)
+; CHECK-NEXT:    [[TMP22:%.*]] = xor <8 x i64> [[TMP1]], splat (i64 -1)
+; CHECK-NEXT:    [[TMP23:%.*]] = and <8 x i64> [[TMP21]], [[TMP22]]
+; CHECK-NEXT:    [[TMP24:%.*]] = or <8 x i64> [[TMP21]], [[TMP1]]
+; CHECK-NEXT:    [[TMP25:%.*]] = xor <8 x i64> [[A1]], splat (i64 -9223372036854775808)
+; CHECK-NEXT:    [[TMP26:%.*]] = xor <8 x i64> [[TMP2]], splat (i64 -1)
+; CHECK-NEXT:    [[TMP27:%.*]] = and <8 x i64> [[TMP25]], [[TMP26]]
+; CHECK-NEXT:    [[TMP28:%.*]] = or <8 x i64> [[TMP25]], [[TMP2]]
+; CHECK-NEXT:    [[TMP29:%.*]] = icmp ult <8 x i64> [[TMP23]], [[TMP28]]
+; CHECK-NEXT:    [[TMP30:%.*]] = icmp ult <8 x i64> [[TMP24]], [[TMP27]]
+; CHECK-NEXT:    [[TMP31:%.*]] = xor <8 x i1> [[TMP29]], [[TMP30]]
+; CHECK-NEXT:    [[TMP32:%.*]] = icmp slt <8 x i64> [[A0]], [[A1]]
+; CHECK-NEXT:    [[TMP33:%.*]] = bitcast i8 [[TMP3]] to <8 x i1>
+; CHECK-NEXT:    [[TMP34:%.*]] = bitcast i8 [[MASK]] to <8 x i1>
+; CHECK-NEXT:    [[TMP35:%.*]] = and <8 x i1> [[TMP31]], [[TMP33]]
+; CHECK-NEXT:    [[TMP36:%.*]] = and <8 x i1> [[TMP32]], [[TMP33]]
+; CHECK-NEXT:    [[TMP37:%.*]] = and <8 x i1> [[TMP31]], [[TMP34]]
+; CHECK-NEXT:    [[TMP38:%.*]] = or <8 x i1> [[TMP35]], [[TMP36]]
+; CHECK-NEXT:    [[TMP39:%.*]] = or <8 x i1> [[TMP38]], [[TMP37]]
+; CHECK-NEXT:    [[TMP40:%.*]] = and <8 x i1> [[TMP32]], [[TMP34]]
+; CHECK-NEXT:    [[TMP41:%.*]] = bitcast <8 x i1> [[TMP39]] to i8
+; CHECK-NEXT:    [[TMP42:%.*]] = bitcast <8 x i1> [[TMP40]] to i8
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = insertelement <8 x i8> [[_MSPROP]], i8 [[TMP41]], i32 1
+; CHECK-NEXT:    [[VEC1:%.*]] = insertelement <8 x i8> [[VEC0]], i8 [[TMP42]], i32 1
+; CHECK-NEXT:    [[TMP43:%.*]] = xor <8 x i64> [[A0]], splat (i64 -9223372036854775808)
+; CHECK-NEXT:    [[TMP44:%.*]] = xor <8 x i64> [[TMP1]], splat (i64 -1)
+; CHECK-NEXT:    [[TMP45:%.*]] = and <8 x i64> [[TMP43]], [[TMP44]]
+; CHECK-NEXT:    [[TMP46:%.*]] = or <8 x i64> [[TMP43]], [[TMP1]]
+; CHECK-NEXT:    [[TMP47:%.*]] = xor <8 x i64> [[A1]], splat (i64 -9223372036854775808)
+; CHECK-NEXT:    [[TMP48:%.*]] = xor <8 x i64> [[TMP2]], splat (i64 -1)
+; CHECK-NEXT:    [[TMP49:%.*]] = and <8 x i64> [[TMP47]], [[TMP48]]
+; CHECK-NEXT:    [[TMP50:%.*]] = or <8 x i64> [[TMP47]], [[TMP2]]
+; CHECK-NEXT:    [[TMP51:%.*]] = icmp ule <8 x i64> [[TMP45]], [[TMP50]]
+; CHECK-NEXT:    [[TMP52:%.*]] = icmp ule <8 x i64> [[TMP46]], [[TMP49]]
+; CHECK-NEXT:    [[TMP53:%.*]] = xor <8 x i1> [[TMP51]], [[TMP52]]
+; CHECK-NEXT:    [[TMP54:%.*]] = icmp sle <8 x i64> [[A0]], [[A1]]
+; CHECK-NEXT:    [[TMP55:%.*]] = bitcast i8 [[TMP3]] to <8 x i1>
+; CHECK-NEXT:    [[TMP56:%.*]] = bitcast i8 [[MASK]] to <8 x i1>
+; CHECK-NEXT:    [[TMP57:%.*]] = and <8 x i1> [[TMP53]], [[TMP55]]
+; CHECK-NEXT:    [[TMP58:%.*]] = and <8 x i1> [[TMP54]], [[TMP55]]
+; CHECK-NEXT:    [[TMP59:%.*]] = and <8 x i1> [[TMP53]], [[TMP56]]
+; CHECK-NEXT:    [[TMP60:%.*]] = or <8 x i1> [[TMP57]], [[TMP58]]
+; CHECK-NEXT:    [[TMP61:%.*]] = or <8 x i1> [[TMP60]], [[TMP59]]
+; CHECK-NEXT:    [[TMP62:%.*]] = and <8 x i1> [[TMP54]], [[TMP56]]
+; CHECK-NEXT:    [[TMP63:%.*]] = bitcast <8 x i1> [[TMP61]] to i8
+; CHECK-NEXT:    [[TMP64:%.*]] = bitcast <8 x i1> [[TMP62]] to i8
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = insertelement <8 x i8> [[_MSPROP1]], i8 [[TMP63]], i32 2
+; CHECK-NEXT:    [[VEC2:%.*]] = insertelement <8 x i8> [[VEC1]], i8 [[TMP64]], i32 2
+; CHECK-NEXT:    [[TMP65:%.*]] = bitcast i8 [[TMP3]] to <8 x i1>
+; CHECK-NEXT:    [[TMP66:%.*]] = bitcast i8 [[MASK]] to <8 x i1>
+; CHECK-NEXT:    [[TMP67:%.*]] = and <8 x i1> zeroinitializer, [[TMP65]]
+; CHECK-NEXT:    [[TMP68:%.*]] = and <8 x i1> zeroinitializer, [[TMP65]]
+; CHECK-NEXT:    [[TMP69:%.*]] = and <8 x i1> zeroinitializer, [[TMP66]]
+; CHECK-NEXT:    [[TMP70:%.*]] = or <8 x i1> [[TMP67]], [[TMP68]]
+; CHECK-NEXT:    [[TMP71:%.*]] = or <8 x i1> [[TMP70]], [[TMP69]]
+; CHECK-NEXT:    [[TMP72:%.*]] = and <8 x i1> zeroinitializer, [[TMP66]]
+; CHECK-NEXT:    [[TMP73:%.*]] = bitcast <8 x i1> [[TMP71]] to i8
+; CHECK-NEXT:    [[TMP74:%.*]] = bitcast <8 x i1> [[TMP72]] to i8
+; CHECK-NEXT:    [[_MSPROP3:%.*]] = insertelement <8 x i8> [[_MSPROP2]], i8 [[TMP73]], i32 3
+; CHECK-NEXT:    [[VEC3:%.*]] = insertelement <8 x i8> [[VEC2]], i8 [[TMP74]], i32 3
+; CHECK-NEXT:    [[TMP75:%.*]] = xor <8 x i64> [[A0]], [[A1]]
+; CHECK-NEXT:    [[TMP76:%.*]] = or <8 x i64> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP77:%.*]] = icmp ne <8 x i64> [[TMP76]], zeroinitializer
+; CHECK-NEXT:    [[TMP78:%.*]] = xor <8 x i64> [[TMP76]], splat (i64 -1)
+; CHECK-NEXT:    [[TMP79:%.*]] = and <8 x i64> [[TMP78]], [[TMP75]]
+; CHECK-NEXT:    [[TMP80:%.*]] = icmp eq <8 x i64> [[TMP79]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_ICMP4:%.*]] = and <8 x i1> [[TMP77]], [[TMP80]]
+; CHECK-NEXT:    [[TMP81:%.*]] = icmp ne <8 x i64> [[A0]], [[A1]]
+; CHECK-NEXT:    [[TMP82:%.*]] = bitcast i8 [[TMP3]] to <8 x i1>
+; CHECK-NEXT:    [[TMP83:%.*]] = bitcast i8 [[MASK]] to <8 x i1>
+; CHECK-NEXT:    [[TMP84:%.*]] = and <8 x i1> [[_MSPROP_ICMP4]], [[TMP82]]
+; CHECK-NEXT:    [[TMP85:%.*]] = and <8 x i1> [[TMP81]], [[TMP82]]
+; CHECK-NEXT:    [[TMP86:%.*]] = and <8 x i1> [[_MSPROP_ICMP4]], [[TMP83]]
+; CHECK-NEXT:    [[TMP87:%.*]] = or <8 x i1> [[TMP84]], [[TMP85]]
+; CHECK-NEXT:    [[TMP88:%.*]] = or <8 x i1> [[TMP87]], [[TMP86]]
+; CHECK-NEXT:    [[TMP89:%.*]] = and <8 x i1> [[TMP81]], [[TMP83]]
+; CHECK-NEXT:    [[TMP90:%.*]] = bitcast <8 x i1> [[TMP88]] to i8
+; CHECK-NEXT:    [[TMP91:%.*]] = bitcast <8 x i1> [[TMP89]] to i8
+; CHECK-NEXT:    [[_MSPROP5:%.*]] = insertelement <8 x i8> [[_MSPROP3]], i8 [[TMP90]], i32 4
+; CHECK-NEXT:    [[VEC4:%.*]] = insertelement <8 x i8> [[VEC3]], i8 [[TMP91]], i32 4
+; CHECK-NEXT:    [[TMP92:%.*]] = xor <8 x i64> [[A0]], splat (i64 -9223372036854775808)
+; CHECK-NEXT:    [[TMP93:%.*]] = xor <8 x i64> [[TMP1]], splat (i64 -1)
+; CHECK-NEXT:    [[TMP94:%.*]] = and <8 x i64> [[TMP92]], [[TMP93]]
+; CHECK-NEXT:    [[TMP95:%.*]] = or <8 x i64> [[TMP92]], [[TMP1]]
+; CHECK-NEXT:    [[TMP96:%.*]] = xor <8 x i64> [[A1]], splat (i64 -9223372036854775808)
+; CHECK-NEXT:    [[TMP97:%.*]] = xor <8 x i64> [[TMP2]], splat (i64 -1)
+; CHECK-NEXT:    [[TMP98:%.*]] = and <8 x i64> [[TMP96]], [[TMP97]]
+; CHECK-NEXT:    [[TMP99:%.*]] = or <8 x i64> [[TMP96]], [[TMP2]]
+; CHECK-NEXT:    [[TMP100:%.*]] = icmp uge <8 x i64> [[TMP94]], [[TMP99]]
+; CHECK-NEXT:    [[TMP101:%.*]] = icmp uge <8 x i64> [[TMP95]], [[TMP98]]
+; CHECK-NEXT:    [[TMP102:%.*]] = xor <8 x i1> [[TMP100]], [[TMP101]]
+; CHECK-NEXT:    [[TMP103:%.*]] = icmp sge <8 x i64> [[A0]], [[A1]]
+; CHECK-NEXT:    [[TMP104:%.*]] = bitcast i8 [[TMP3]] to <8 x i1>
+; CHECK-NEXT:    [[TMP105:%.*]] = bitcast i8 [[MASK]] to <8 x i1>
+; CHECK-NEXT:    [[TMP106:%.*]] = and <8 x i1> [[TMP102]], [[TMP104]]
+; CHECK-NEXT:    [[TMP107:%.*]] = and <8 x i1> [[TMP103]], [[TMP104]]
+; CHECK-NEXT:    [[TMP108:%.*]] = and <8 x i1> [[TMP102]], [[TMP105]]
+; CHECK-NEXT:    [[TMP109:%.*]] = or <8 x i1> [[TMP106]], [[TMP107]]
+; CHECK-NEXT:    [[TMP110:%.*]] = or <8 x i1> [[TMP109]], [[TMP108]]
+; CHECK-NEXT:    [[TMP111:%.*]] = and <8 x i1> [[TMP103]], [[TMP105]]
+; CHECK-NEXT:    [[TMP112:%.*]] = bitcast <8 x i1> [[TMP110]] to i8
+; CHECK-NEXT:    [[TMP113:%.*]] = bitcast <8 x i1> [[TMP111]] to i8
+; CHECK-NEXT:    [[_MSPROP6:%.*]] = insertelement <8 x i8> [[_MSPROP5]], i8 [[TMP112]], i32 5
+; CHECK-NEXT:    [[VEC5:%.*]] = insertelement <8 x i8> [[VEC4]], i8 [[TMP113]], i32 5
+; CHECK-NEXT:    [[TMP114:%.*]] = xor <8 x i64> [[A0]], splat (i64 -9223372036854775808)
+; CHECK-NEXT:    [[TMP115:%.*]] = xor <8 x i64> [[TMP1]], splat (i64 -1)
+; CHECK-NEXT:    [[TMP116:%.*]] = and <8 x i64> [[TMP114]], [[TMP115]]
+; CHECK-NEXT:    [[TMP117:%.*]] = or <8 x i64> [[TMP114]], [[TMP1]]
+; CHECK-NEXT:    [[TMP118:%.*]] = xor <8 x i64> [[A1]], splat (i64 -9223372036854775808)
+; CHECK-NEXT:    [[TMP119:%.*]] = xor <8 x i64> [[TMP2]], splat (i64 -1)
+; CHECK-NEXT:    [[TMP120:%.*]] = and <8 x i64> [[TMP118]], [[TMP119]]
+; CHECK-NEXT:    [[TMP121:%.*]] = or <8 x i64> [[TMP118]], [[TMP2]]
+; CHECK-NEXT:    [[TMP122:%.*]] = icmp ugt <8 x i64> [[TMP116]], [[TMP121]]
+; CHECK-NEXT:    [[TMP123:%.*]] = icmp ugt <8 x i64> [[TMP117]], [[TMP120]]
+; CHECK-NEXT:    [[TMP124:%.*]] = xor <8 x i1> [[TMP122]], [[TMP123]]
+; CHECK-NEXT:    [[TMP125:%.*]] = icmp sgt <8 x i64> [[A0]], [[A1]]
+; CHECK-NEXT:    [[TMP126:%.*]] = bitcast i8 [[TMP3]] to <8 x i1>
+; CHECK-NEXT:    [[TMP127:%.*]] = bitcast i8 [[MASK]] to <8 x i1>
+; CHECK-NEXT:    [[TMP128:%.*]] = and <8 x i1> [[TMP124]], [[TMP126]]
+; CHECK-NEXT:    [[TMP129:%.*]] = and <8 x i1> [[TMP125]], [[TMP126]]
+; CHECK-NEXT:    [[TMP130:%.*]] = and <8 x i1> [[TMP124]], [[TMP127]]
+; CHECK-NEXT:    [[TMP131:%.*]] = or <8 x i1> [[TMP128]], [[TMP129]]
+; CHECK-NEXT:    [[TMP132:%.*]] = or <8 x i1> [[TMP131]], [[TMP130]]
+; CHECK-NEXT:    [[TMP133:%.*]] = and <8 x i1> [[TMP125]], [[TMP127]]
+; CHECK-NEXT:    [[TMP134:%.*]] = bitcast <8 x i1> [[TMP132]] to i8
+; CHECK-NEXT:    [[TMP135:%.*]] = bitcast <8 x i1> [[TMP133]] to i8
+; CHECK-NEXT:    [[_MSPROP7:%.*]] = insertelement <8 x i8> [[_MSPROP6]], i8 [[TMP134]], i32 6
+; CHECK-NEXT:    [[VEC6:%.*]] = insertelement <8 x i8> [[VEC5]], i8 [[TMP135]], i32 6
+; CHECK-NEXT:    [[TMP136:%.*]] = bitcast i8 [[TMP3]] to <8 x i1>
+; CHECK-NEXT:    [[TMP137:%.*]] = bitcast i8 [[MASK]] to <8 x i1>
+; CHECK-NEXT:    [[TMP138:%.*]] = and <8 x i1> zeroinitializer, [[TMP136]]
+; CHECK-NEXT:    [[TMP139:%.*]] = and <8 x i1> splat (i1 true), [[TMP136]]
+; CHECK-NEXT:    [[TMP140:%.*]] = and <8 x i1> zeroinitializer, [[TMP137]]
+; CHECK-NEXT:    [[TMP141:%.*]] = or <8 x i1> [[TMP138]], [[TMP139]]
+; CHECK-NEXT:    [[TMP142:%.*]] = or <8 x i1> [[TMP141]], [[TMP140]]
+; CHECK-NEXT:    [[TMP143:%.*]] = and <8 x i1> splat (i1 true), [[TMP137]]
+; CHECK-NEXT:    [[TMP144:%.*]] = bitcast <8 x i1> [[TMP142]] to i8
+; CHECK-NEXT:    [[TMP145:%.*]] = bitcast <8 x i1> [[TMP143]] to i8
+; CHECK-NEXT:    [[_MSPROP8:%.*]] = insertelement <8 x i8> [[_MSPROP7]], i8 [[TMP144]], i32 7
+; CHECK-NEXT:    [[VEC7:%.*]] = insertelement <8 x i8> [[VEC6]], i8 [[TMP145]], i32 7
+; CHECK-NEXT:    store <8 x i8> [[_MSPROP8]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i8> [[VEC7]]
+;
+  %res0 = call i8 @llvm.x86.avx512.mask.cmp.q.512(<8 x i64> %a0, <8 x i64> %a1, i32 0, i8 %mask)
+  %vec0 = insertelement <8 x i8> %extra_param, i8 %res0, i32 0
+  %res1 = call i8 @llvm.x86.avx512.mask.cmp.q.512(<8 x i64> %a0, <8 x i64> %a1, i32 1, i8 %mask)
+  %vec1 = insertelement <8 x i8> %vec0, i8 %res1, i32 1
+  %res2 = call i8 @llvm.x86.avx512.mask.cmp.q.512(<8 x i64> %a0, <8 x i64> %a1, i32 2, i8 %mask)
+  %vec2 = insertelement <8 x i8> %vec1, i8 %res2, i32 2
+  %res3 = call i8 @llvm.x86.avx512.mask.cmp.q.512(<8 x i64> %a0, <8 x i64> %a1, i32 3, i8 %mask)
+  %vec3 = insertelement <8 x i8> %vec2, i8 %res3, i32 3
+  %res4 = call i8 @llvm.x86.avx512.mask.cmp.q.512(<8 x i64> %a0, <8 x i64> %a1, i32 4, i8 %mask)
+  %vec4 = insertelement <8 x i8> %vec3, i8 %res4, i32 4
+  %res5 = call i8 @llvm.x86.avx512.mask.cmp.q.512(<8 x i64> %a0, <8 x i64> %a1, i32 5, i8 %mask)
+  %vec5 = insertelement <8 x i8> %vec4, i8 %res5, i32 5
+  %res6 = call i8 @llvm.x86.avx512.mask.cmp.q.512(<8 x i64> %a0, <8 x i64> %a1, i32 6, i8 %mask)
+  %vec6 = insertelement <8 x i8> %vec5, i8 %res6, i32 6
+  %res7 = call i8 @llvm.x86.avx512.mask.cmp.q.512(<8 x i64> %a0, <8 x i64> %a1, i32 7, i8 %mask)
+  %vec7 = insertelement <8 x i8> %vec6, i8 %res7, i32 7
+  ret <8 x i8> %vec7
+}
+
+declare i8 @llvm.x86.avx512.mask.cmp.q.512(<8 x i64>, <8 x i64>, i32, i8) nounwind readnone
+
+define <8 x i8> @test_ucmp_q_512(<8 x i64> %a0, <8 x i64> %a1, <8 x i8> %extra_param)  #0 {
+; CHECK-LABEL: @test_ucmp_q_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP69:%.*]] = load <8 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = xor <8 x i64> [[A0:%.*]], [[A1:%.*]]
+; CHECK-NEXT:    [[TMP4:%.*]] = or <8 x i64> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp ne <8 x i64> [[TMP4]], zeroinitializer
+; CHECK-NEXT:    [[TMP6:%.*]] = xor <8 x i64> [[TMP4]], splat (i64 -1)
+; CHECK-NEXT:    [[TMP7:%.*]] = and <8 x i64> [[TMP6]], [[TMP3]]
+; CHECK-NEXT:    [[TMP8:%.*]] = icmp eq <8 x i64> [[TMP7]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_ICMP:%.*]] = and <8 x i1> [[TMP5]], [[TMP8]]
+; CHECK-NEXT:    [[TMP9:%.*]] = icmp eq <8 x i64> [[A0]], [[A1]]
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast <8 x i1> [[_MSPROP_ICMP]] to i8
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <8 x i1> [[TMP9]] to i8
+; CHECK-NEXT:    [[_MSPROP:%.*]] = insertelement <8 x i8> [[TMP69]], i8 [[TMP10]], i32 0
+; CHECK-NEXT:    [[VEC0:%.*]] = insertelement <8 x i8> [[EXTRA_PARAM:%.*]], i8 [[TMP11]], i32 0
+; CHECK-NEXT:    [[TMP12:%.*]] = xor <8 x i64> [[TMP1]], splat (i64 -1)
+; CHECK-NEXT:    [[TMP13:%.*]] = and <8 x i64> [[A0]], [[TMP12]]
+; CHECK-NEXT:    [[TMP14:%.*]] = or <8 x i64> [[A0]], [[TMP1]]
+; CHECK-NEXT:    [[TMP15:%.*]] = xor <8 x i64> [[TMP2]], splat (i64 -1)
+; CHECK-NEXT:    [[TMP16:%.*]] = and <8 x i64> [[A1]], [[TMP15]]
+; CHECK-NEXT:    [[TMP17:%.*]] = or <8 x i64> [[A1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP18:%.*]] = icmp ult <8 x i64> [[TMP13]], [[TMP17]]
+; CHECK-NEXT:    [[TMP19:%.*]] = icmp ult <8 x i64> [[TMP14]], [[TMP16]]
+; CHECK-NEXT:    [[TMP20:%.*]] = xor <8 x i1> [[TMP18]], [[TMP19]]
+; CHECK-NEXT:    [[TMP21:%.*]] = icmp ult <8 x i64> [[A0]], [[A1]]
+; CHECK-NEXT:    [[TMP22:%.*]] = bitcast <8 x i1> [[TMP20]] to i8
+; CHECK-NEXT:    [[TMP23:%.*]] = bitcast <8 x i1> [[TMP21]] to i8
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = insertelement <8 x i8> [[_MSPROP]], i8 [[TMP22]], i32 1
+; CHECK-NEXT:    [[VEC1:%.*]] = insertelement <8 x i8> [[VEC0]], i8 [[TMP23]], i32 1
+; CHECK-NEXT:    [[TMP24:%.*]] = xor <8 x i64> [[TMP1]], splat (i64 -1)
+; CHECK-NEXT:    [[TMP25:%.*]] = and <8 x i64> [[A0]], [[TMP24]]
+; CHECK-NEXT:    [[TMP26:%.*]] = or <8 x i64> [[A0]], [[TMP1]]
+; CHECK-NEXT:    [[TMP27:%.*]] = xor <8 x i64> [[TMP2]], splat (i64 -1)
+; CHECK-NEXT:    [[TMP28:%.*]] = and <8 x i64> [[A1]], [[TMP27]]
+; CHECK-NEXT:    [[TMP29:%.*]] = or <8 x i64> [[A1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP30:%.*]] = icmp ule <8 x i64> [[TMP25]], [[TMP29]]
+; CHECK-NEXT:    [[TMP31:%.*]] = icmp ule <8 x i64> [[TMP26]], [[TMP28]]
+; CHECK-NEXT:    [[TMP32:%.*]] = xor <8 x i1> [[TMP30]], [[TMP31]]
+; CHECK-NEXT:    [[TMP33:%.*]] = icmp ule <8 x i64> [[A0]], [[A1]]
+; CHECK-NEXT:    [[TMP34:%.*]] = bitcast <8 x i1> [[TMP32]] to i8
+; CHECK-NEXT:    [[TMP35:%.*]] = bitcast <8 x i1> [[TMP33]] to i8
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = insertelement <8 x i8> [[_MSPROP1]], i8 [[TMP34]], i32 2
+; CHECK-NEXT:    [[VEC2:%.*]] = insertelement <8 x i8> [[VEC1]], i8 [[TMP35]], i32 2
+; CHECK-NEXT:    [[_MSPROP3:%.*]] = insertelement <8 x i8> [[_MSPROP2]], i8 0, i32 3
+; CHECK-NEXT:    [[VEC3:%.*]] = insertelement <8 x i8> [[VEC2]], i8 0, i32 3
+; CHECK-NEXT:    [[TMP36:%.*]] = xor <8 x i64> [[A0]], [[A1]]
+; CHECK-NEXT:    [[TMP37:%.*]] = or <8 x i64> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP38:%.*]] = icmp ne <8 x i64> [[TMP37]], zeroinitializer
+; CHECK-NEXT:    [[TMP39:%.*]] = xor <8 x i64> [[TMP37]], splat (i64 -1)
+; CHECK-NEXT:    [[TMP40:%.*]] = and <8 x i64> [[TMP39]], [[TMP36]]
+; CHECK-NEXT:    [[TMP41:%.*]] = icmp eq <8 x i64> [[TMP40]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_ICMP4:%.*]] = and <8 x i1> [[TMP38]], [[TMP41]]
+; CHECK-NEXT:    [[TMP42:%.*]] = icmp ne <8 x i64> [[A0]], [[A1]]
+; CHECK-NEXT:    [[TMP43:%.*]] = bitcast <8 x i1> [[_MSPROP_ICMP4]] to i8
+; CHECK-NEXT:    [[TMP44:%.*]] = bitcast <8 x i1> [[TMP42]] to i8
+; CHECK-NEXT:    [[_MSPROP5:%.*]] = insertelement <8 x i8> [[_MSPROP3]], i8 [[TMP43]], i32 4
+; CHECK-NEXT:    [[VEC4:%.*]] = insertelement <8 x i8> [[VEC3]], i8 [[TMP44]], i32 4
+; CHECK-NEXT:    [[TMP45:%.*]] = xor <8 x i64> [[TMP1]], splat (i64 -1)
+; CHECK-NEXT:    [[TMP46:%.*]] = and <8 x i64> [[A0]], [[TMP45]]
+; CHECK-NEXT:    [[TMP47:%.*]] = or <8 x i64> [[A0]], [[TMP1]]
+; CHECK-NEXT:    [[TMP48:%.*]] = xor <8 x i64> [[TMP2]], splat (i64 -1)
+; CHECK-NEXT:    [[TMP49:%.*]] = and <8 x i64> [[A1]], [[TMP48]]
+; CHECK-NEXT:    [[TMP50:%.*]] = or <8 x i64> [[A1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP51:%.*]] = icmp uge <8 x i64> [[TMP46]], [[TMP50]]
+; CHECK-NEXT:    [[TMP52:%.*]] = icmp uge <8 x i64> [[TMP47]], [[TMP49]]
+; CHECK-NEXT:    [[TMP53:%.*]] = xor <8 x i1> [[TMP51]], [[TMP52]]
+; CHECK-NEXT:    [[TMP54:%.*]] = icmp uge <8 x i64> [[A0]], [[A1]]
+; CHECK-NEXT:    [[TMP55:%.*]] = bitcast <8 x i1> [[TMP53]] to i8
+; CHECK-NEXT:    [[TMP56:%.*]] = bitcast <8 x i1> [[TMP54]] to i8
+; CHECK-NEXT:    [[_MSPROP6:%.*]] = insertelement <8 x i8> [[_MSPROP5]], i8 [[TMP55]], i32 5
+; CHECK-NEXT:    [[VEC5:%.*]] = insertelement <8 x i8> [[VEC4]], i8 [[TMP56]], i32 5
+; CHECK-NEXT:    [[TMP57:%.*]] = xor <8 x i64> [[TMP1]], splat (i64 -1)
+; CHECK-NEXT:    [[TMP58:%.*]] = and <8 x i64> [[A0]], [[TMP57]]
+; CHECK-NEXT:    [[TMP59:%.*]] = or <8 x i64> [[A0]], [[TMP1]]
+; CHECK-NEXT:    [[TMP60:%.*]] = xor <8 x i64> [[TMP2]], splat (i64 -1)
+; CHECK-NEXT:    [[TMP61:%.*]] = and <8 x i64> [[A1]], [[TMP60]]
+; CHECK-NEXT:    [[TMP62:%.*]] = or <8 x i64> [[A1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP63:%.*]] = icmp ugt <8 x i64> [[TMP58]], [[TMP62]]
+; CHECK-NEXT:    [[TMP64:%.*]] = icmp ugt <8 x i64> [[TMP59]], [[TMP61]]
+; CHECK-NEXT:    [[TMP65:%.*]] = xor <8 x i1> [[TMP63]], [[TMP64]]
+; CHECK-NEXT:    [[TMP66:%.*]] = icmp ugt <8 x i64> [[A0]], [[A1]]
+; CHECK-NEXT:    [[TMP67:%.*]] = bitcast <8 x i1> [[TMP65]] to i8
+; CHECK-NEXT:    [[TMP68:%.*]] = bitcast <8 x i1> [[TMP66]] to i8
+; CHECK-NEXT:    [[_MSPROP7:%.*]] = insertelement <8 x i8> [[_MSPROP6]], i8 [[TMP67]], i32 6
+; CHECK-NEXT:    [[VEC6:%.*]] = insertelement <8 x i8> [[VEC5]], i8 [[TMP68]], i32 6
+; CHECK-NEXT:    [[_MSPROP8:%.*]] = insertelement <8 x i8> [[_MSPROP7]], i8 0, i32 7
+; CHECK-NEXT:    [[VEC7:%.*]] = insertelement <8 x i8> [[VEC6]], i8 -1, i32 7
+; CHECK-NEXT:    store <8 x i8> [[_MSPROP8]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i8> [[VEC7]]
+;
+  %res0 = call i8 @llvm.x86.avx512.mask.ucmp.q.512(<8 x i64> %a0, <8 x i64> %a1, i32 0, i8 -1)
+  %vec0 = insertelement <8 x i8> %extra_param, i8 %res0, i32 0
+  %res1 = call i8 @llvm.x86.avx512.mask.ucmp.q.512(<8 x i64> %a0, <8 x i64> %a1, i32 1, i8 -1)
+  %vec1 = insertelement <8 x i8> %vec0, i8 %res1, i32 1
+  %res2 = call i8 @llvm.x86.avx512.mask.ucmp.q.512(<8 x i64> %a0, <8 x i64> %a1, i32 2, i8 -1)
+  %vec2 = insertelement <8 x i8> %vec1, i8 %res2, i32 2
+  %res3 = call i8 @llvm.x86.avx512.mask.ucmp.q.512(<8 x i64> %a0, <8 x i64> %a1, i32 3, i8 -1)
+  %vec3 = insertelement <8 x i8> %vec2, i8 %res3, i32 3
+  %res4 = call i8 @llvm.x86.avx512.mask.ucmp.q.512(<8 x i64> %a0, <8 x i64> %a1, i32 4, i8 -1)
+  %vec4 = insertelement <8 x i8> %vec3, i8 %res4, i32 4
+  %res5 = call i8 @llvm.x86.avx512.mask.ucmp.q.512(<8 x i64> %a0, <8 x i64> %a1, i32 5, i8 -1)
+  %vec5 = insertelement <8 x i8> %vec4, i8 %res5, i32 5
+  %res6 = call i8 @llvm.x86.avx512.mask.ucmp.q.512(<8 x i64> %a0, <8 x i64> %a1, i32 6, i8 -1)
+  %vec6 = insertelement <8 x i8> %vec5, i8 %res6, i32 6
+  %res7 = call i8 @llvm.x86.avx512.mask.ucmp.q.512(<8 x i64> %a0, <8 x i64> %a1, i32 7, i8 -1)
+  %vec7 = insertelement <8 x i8> %vec6, i8 %res7, i32 7
+  ret <8 x i8> %vec7
+}
+
+define <8 x i8> @test_mask_ucmp_q_512(<8 x i64> %a0, <8 x i64> %a1, i8 %mask, <8 x i8> %extra_param)  #0 {
+;
+; CHECK-LABEL: @test_mask_ucmp_q_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    [[TMP138:%.*]] = load <8 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 136) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = xor <8 x i64> [[A0:%.*]], [[A1:%.*]]
+; CHECK-NEXT:    [[TMP5:%.*]] = or <8 x i64> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP6:%.*]] = icmp ne <8 x i64> [[TMP5]], zeroinitializer
+; CHECK-NEXT:    [[TMP7:%.*]] = xor <8 x i64> [[TMP5]], splat (i64 -1)
+; CHECK-NEXT:    [[TMP8:%.*]] = and <8 x i64> [[TMP7]], [[TMP4]]
+; CHECK-NEXT:    [[TMP9:%.*]] = icmp eq <8 x i64> [[TMP8]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_ICMP:%.*]] = and <8 x i1> [[TMP6]], [[TMP9]]
+; CHECK-NEXT:    [[TMP10:%.*]] = icmp eq <8 x i64> [[A0]], [[A1]]
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast i8 [[TMP3]] to <8 x i1>
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP13:%.*]] = and <8 x i1> [[_MSPROP_ICMP]], [[TMP11]]
+; CHECK-NEXT:    [[TMP14:%.*]] = and <8 x i1> [[TMP10]], [[TMP11]]
+; CHECK-NEXT:    [[TMP15:%.*]] = and <8 x i1> [[_MSPROP_ICMP]], [[TMP12]]
+; CHECK-NEXT:    [[TMP16:%.*]] = or <8 x i1> [[TMP13]], [[TMP14]]
+; CHECK-NEXT:    [[TMP17:%.*]] = or <8 x i1> [[TMP16]], [[TMP15]]
+; CHECK-NEXT:    [[TMP18:%.*]] = and <8 x i1> [[TMP10]], [[TMP12]]
+; CHECK-NEXT:    [[TMP19:%.*]] = bitcast <8 x i1> [[TMP17]] to i8
+; CHECK-NEXT:    [[TMP20:%.*]] = bitcast <8 x i1> [[TMP18]] to i8
+; CHECK-NEXT:    [[_MSPROP:%.*]] = insertelement <8 x i8> [[TMP138]], i8 [[TMP19]], i32 0
+; CHECK-NEXT:    [[VEC0:%.*]] = insertelement <8 x i8> [[EXTRA_PARAM:%.*]], i8 [[TMP20]], i32 0
+; CHECK-NEXT:    [[TMP21:%.*]] = xor <8 x i64> [[TMP1]], splat (i64 -1)
+; CHECK-NEXT:    [[TMP22:%.*]] = and <8 x i64> [[A0]], [[TMP21]]
+; CHECK-NEXT:    [[TMP23:%.*]] = or <8 x i64> [[A0]], [[TMP1]]
+; CHECK-NEXT:    [[TMP24:%.*]] = xor <8 x i64> [[TMP2]], splat (i64 -1)
+; CHECK-NEXT:    [[TMP25:%.*]] = and <8 x i64> [[A1]], [[TMP24]]
+; CHECK-NEXT:    [[TMP26:%.*]] = or <8 x i64> [[A1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP27:%.*]] = icmp ult <8 x i64> [[TMP22]], [[TMP26]]
+; CHECK-NEXT:    [[TMP28:%.*]] = icmp ult <8 x i64> [[TMP23]], [[TMP25]]
+; CHECK-NEXT:    [[TMP29:%.*]] = xor <8 x i1> [[TMP27]], [[TMP28]]
+; CHECK-NEXT:    [[TMP30:%.*]] = icmp ult <8 x i64> [[A0]], [[A1]]
+; CHECK-NEXT:    [[TMP31:%.*]] = bitcast i8 [[TMP3]] to <8 x i1>
+; CHECK-NEXT:    [[TMP32:%.*]] = bitcast i8 [[MASK]] to <8 x i1>
+; CHECK-NEXT:    [[TMP33:%.*]] = and <8 x i1> [[TMP29]], [[TMP31]]
+; CHECK-NEXT:    [[TMP34:%.*]] = and <8 x i1> [[TMP30]], [[TMP31]]
+; CHECK-NEXT:    [[TMP35:%.*]] = and <8 x i1> [[TMP29]], [[TMP32]]
+; CHECK-NEXT:    [[TMP36:%.*]] = or <8 x i1> [[TMP33]], [[TMP34]]
+; CHECK-NEXT:    [[TMP37:%.*]] = or <8 x i1> [[TMP36]], [[TMP35]]
+; CHECK-NEXT:    [[TMP38:%.*]] = and <8 x i1> [[TMP30]], [[TMP32]]
+; CHECK-NEXT:    [[TMP39:%.*]] = bitcast <8 x i1> [[TMP37]] to i8
+; CHECK-NEXT:    [[TMP40:%.*]] = bitcast <8 x i1> [[TMP38]] to i8
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = insertelement <8 x i8> [[_MSPROP]], i8 [[TMP39]], i32 1
+; CHECK-NEXT:    [[VEC1:%.*]] = insertelement <8 x i8> [[VEC0]], i8 [[TMP40]], i32 1
+; CHECK-NEXT:    [[TMP41:%.*]] = xor <8 x i64> [[TMP1]], splat (i64 -1)
+; CHECK-NEXT:    [[TMP42:%.*]] = and <8 x i64> [[A0]], [[TMP41]]
+; CHECK-NEXT:    [[TMP43:%.*]] = or <8 x i64> [[A0]], [[TMP1]]
+; CHECK-NEXT:    [[TMP44:%.*]] = xor <8 x i64> [[TMP2]], splat (i64 -1)
+; CHECK-NEXT:    [[TMP45:%.*]] = and <8 x i64> [[A1]], [[TMP44]]
+; CHECK-NEXT:    [[TMP46:%.*]] = or <8 x i64> [[A1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP47:%.*]] = icmp ule <8 x i64> [[TMP42]], [[TMP46]]
+; CHECK-NEXT:    [[TMP48:%.*]] = icmp ule <8 x i64> [[TMP43]], [[TMP45]]
+; CHECK-NEXT:    [[TMP49:%.*]] = xor <8 x i1> [[TMP47]], [[TMP48]]
+; CHECK-NEXT:    [[TMP50:%.*]] = icmp ule <8 x i64> [[A0]], [[A1]]
+; CHECK-NEXT:    [[TMP51:%.*]] = bitcast i8 [[TMP3]] to <8 x i1>
+; CHECK-NEXT:    [[TMP52:%.*]] = bitcast i8 [[MASK]] to <8 x i1>
+; CHECK-NEXT:    [[TMP53:%.*]] = and <8 x i1> [[TMP49]], [[TMP51]]
+; CHECK-NEXT:    [[TMP54:%.*]] = and <8 x i1> [[TMP50]], [[TMP51]]
+; CHECK-NEXT:    [[TMP55:%.*]] = and <8 x i1> [[TMP49]], [[TMP52]]
+; CHECK-NEXT:    [[TMP56:%.*]] = or <8 x i1> [[TMP53]], [[TMP54]]
+; CHECK-NEXT:    [[TMP57:%.*]] = or <8 x i1> [[TMP56]], [[TMP55]]
+; CHECK-NEXT:    [[TMP58:%.*]] = and <8 x i1> [[TMP50]], [[TMP52]]
+; CHECK-NEXT:    [[TMP59:%.*]] = bitcast <8 x i1> [[TMP57]] to i8
+; CHECK-NEXT:    [[TMP60:%.*]] = bitcast <8 x i1> [[TMP58]] to i8
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = insertelement <8 x i8> [[_MSPROP1]], i8 [[TMP59]], i32 2
+; CHECK-NEXT:    [[VEC2:%.*]] = insertelement <8 x i8> [[VEC1]], i8 [[TMP60]], i32 2
+; CHECK-NEXT:    [[TMP61:%.*]] = bitcast i8 [[TMP3]] to <8 x i1>
+; CHECK-NEXT:    [[TMP62:%.*]] = bitcast i8 [[MASK]] to <8 x i1>
+; CHECK-NEXT:    [[TMP63:%.*]] = and <8 x i1> zeroinitializer, [[TMP61]]
+; CHECK-NEXT:    [[TMP64:%.*]] = and <8 x i1> zeroinitializer, [[TMP61]]
+; CHECK-NEXT:    [[TMP65:%.*]] = and <8 x i1> zeroinitializer, [[TMP62]]
+; CHECK-NEXT:    [[TMP66:%.*]] = or <8 x i1> [[TMP63]], [[TMP64]]
+; CHECK-NEXT:    [[TMP67:%.*]] = or <8 x i1> [[TMP66]], [[TMP65]]
+; CHECK-NEXT:    [[TMP68:%.*]] = and <8 x i1> zeroinitializer, [[TMP62]]
+; CHECK-NEXT:    [[TMP69:%.*]] = bitcast <8 x i1> [[TMP67]] to i8
+; CHECK-NEXT:    [[TMP70:%.*]] = bitcast <8 x i1> [[TMP68]] to i8
+; CHECK-NEXT:    [[_MSPROP3:%.*]] = insertelement <8 x i8> [[_MSPROP2]], i8 [[TMP69]], i32 3
+; CHECK-NEXT:    [[VEC3:%.*]] = insertelement <8 x i8> [[VEC2]], i8 [[TMP70]], i32 3
+; CHECK-NEXT:    [[TMP71:%.*]] = xor <8 x i64> [[A0]], [[A1]]
+; CHECK-NEXT:    [[TMP72:%.*]] = or <8 x i64> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP73:%.*]] = icmp ne <8 x i64> [[TMP72]], zeroinitializer
+; CHECK-NEXT:    [[TMP74:%.*]] = xor <8 x i64> [[TMP72]], splat (i64 -1)
+; CHECK-NEXT:    [[TMP75:%.*]] = and <8 x i64> [[TMP74]], [[TMP71]]
+; CHECK-NEXT:    [[TMP76:%.*]] = icmp eq <8 x i64> [[TMP75]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_ICMP4:%.*]] = and <8 x i1> [[TMP73]], [[TMP76]]
+; CHECK-NEXT:    [[TMP77:%.*]] = icmp ne <8 x i64> [[A0]], [[A1]]
+; CHECK-NEXT:    [[TMP78:%.*]] = bitcast i8 [[TMP3]] to <8 x i1>
+; CHECK-NEXT:    [[TMP79:%.*]] = bitcast i8 [[MASK]] to <8 x i1>
+; CHECK-NEXT:    [[TMP80:%.*]] = and <8 x i1> [[_MSPROP_ICMP4]], [[TMP78]]
+; CHECK-NEXT:    [[TMP81:%.*]] = and <8 x i1> [[TMP77]], [[TMP78]]
+; CHECK-NEXT:    [[TMP82:%.*]] = and <8 x i1> [[_MSPROP_ICMP4]], [[TMP79]]
+; CHECK-NEXT:    [[TMP83:%.*]] = or <8 x i1> [[TMP80]], [[TMP81]]
+; CHECK-NEXT:    [[TMP84:%.*]] = or <8 x i1> [[TMP83]], [[TMP82]]
+; CHECK-NEXT:    [[TMP85:%.*]] = and <8 x i1> [[TMP77]], [[TMP79]]
+; CHECK-NEXT:    [[TMP86:%.*]] = bitcast <8 x i1> [[TMP84]] to i8
+; CHECK-NEXT:    [[TMP87:%.*]] = bitcast <8 x i1> [[TMP85]] to i8
+; CHECK-NEXT:    [[_MSPROP5:%.*]] = insertelement <8 x i8> [[_MSPROP3]], i8 [[TMP86]], i32 4
+; CHECK-NEXT:    [[VEC4:%.*]] = insertelement <8 x i8> [[VEC3]], i8 [[TMP87]], i32 4
+; CHECK-NEXT:    [[TMP88:%.*]] = xor <8 x i64> [[TMP1]], splat (i64 -1)
+; CHECK-NEXT:    [[TMP89:%.*]] = and <8 x i64> [[A0]], [[TMP88]]
+; CHECK-NEXT:    [[TMP90:%.*]] = or <8 x i64> [[A0]], [[TMP1]]
+; CHECK-NEXT:    [[TMP91:%.*]] = xor <8 x i64> [[TMP2]], splat (i64 -1)
+; CHECK-NEXT:    [[TMP92:%.*]] = and <8 x i64> [[A1]], [[TMP91]]
+; CHECK-NEXT:    [[TMP93:%.*]] = or <8 x i64> [[A1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP94:%.*]] = icmp uge <8 x i64> [[TMP89]], [[TMP93]]
+; CHECK-NEXT:    [[TMP95:%.*]] = icmp uge <8 x i64> [[TMP90]], [[TMP92]]
+; CHECK-NEXT:    [[TMP96:%.*]] = xor <8 x i1> [[TMP94]], [[TMP95]]
+; CHECK-NEXT:    [[TMP97:%.*]] = icmp uge <8 x i64> [[A0]], [[A1]]
+; CHECK-NEXT:    [[TMP98:%.*]] = bitcast i8 [[TMP3]] to <8 x i1>
+; CHECK-NEXT:    [[TMP99:%.*]] = bitcast i8 [[MASK]] to <8 x i1>
+; CHECK-NEXT:    [[TMP100:%.*]] = and <8 x i1> [[TMP96]], [[TMP98]]
+; CHECK-NEXT:    [[TMP101:%.*]] = and <8 x i1> [[TMP97]], [[TMP98]]
+; CHECK-NEXT:    [[TMP102:%.*]] = and <8 x i1> [[TMP96]], [[TMP99]]
+; CHECK-NEXT:    [[TMP103:%.*]] = or <8 x i1> [[TMP100]], [[TMP101]]
+; CHECK-NEXT:    [[TMP104:%.*]] = or <8 x i1> [[TMP103]], [[TMP102]]
+; CHECK-NEXT:    [[TMP105:%.*]] = and <8 x i1> [[TMP97]], [[TMP99]]
+; CHECK-NEXT:    [[TMP106:%.*]] = bitcast <8 x i1> [[TMP104]] to i8
+; CHECK-NEXT:    [[TMP107:%.*]] = bitcast <8 x i1> [[TMP105]] to i8
+; CHECK-NEXT:    [[_MSPROP6:%.*]] = insertelement <8 x i8> [[_MSPROP5]], i8 [[TMP106]], i32 5
+; CHECK-NEXT:    [[VEC5:%.*]] = insertelement <8 x i8> [[VEC4]], i8 [[TMP107]], i32 5
+; CHECK-NEXT:    [[TMP108:%.*]] = xor <8 x i64> [[TMP1]], splat (i64 -1)
+; CHECK-NEXT:    [[TMP109:%.*]] = and <8 x i64> [[A0]], [[TMP108]]
+; CHECK-NEXT:    [[TMP110:%.*]] = or <8 x i64> [[A0]], [[TMP1]]
+; CHECK-NEXT:    [[TMP111:%.*]] = xor <8 x i64> [[TMP2]], splat (i64 -1)
+; CHECK-NEXT:    [[TMP112:%.*]] = and <8 x i64> [[A1]], [[TMP111]]
+; CHECK-NEXT:    [[TMP113:%.*]] = or <8 x i64> [[A1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP114:%.*]] = icmp ugt <8 x i64> [[TMP109]], [[TMP113]]
+; CHECK-NEXT:    [[TMP115:%.*]] = icmp ugt <8 x i64> [[TMP110]], [[TMP112]]
+; CHECK-NEXT:    [[TMP116:%.*]] = xor <8 x i1> [[TMP114]], [[TMP115]]
+; CHECK-NEXT:    [[TMP117:%.*]] = icmp ugt <8 x i64> [[A0]], [[A1]]
+; CHECK-NEXT:    [[TMP118:%.*]] = bitcast i8 [[TMP3]] to <8 x i1>
+; CHECK-NEXT:    [[TMP119:%.*]] = bitcast i8 [[MASK]] to <8 x i1>
+; CHECK-NEXT:    [[TMP120:%.*]] = and <8 x i1> [[TMP116]], [[TMP118]]
+; CHECK-NEXT:    [[TMP121:%.*]] = and <8 x i1> [[TMP117]], [[TMP118]]
+; CHECK-NEXT:    [[TMP122:%.*]] = and <8 x i1> [[TMP116]], [[TMP119]]
+; CHECK-NEXT:    [[TMP123:%.*]] = or <8 x i1> [[TMP120]], [[TMP121]]
+; CHECK-NEXT:    [[TMP124:%.*]] = or <8 x i1> [[TMP123]], [[TMP122]]
+; CHECK-NEXT:    [[TMP125:%.*]] = and <8 x i1> [[TMP117]], [[TMP119]]
+; CHECK-NEXT:    [[TMP126:%.*]] = bitcast <8 x i1> [[TMP124]] to i8
+; CHECK-NEXT:    [[TMP127:%.*]] = bitcast <8 x i1> [[TMP125]] to i8
+; CHECK-NEXT:    [[_MSPROP7:%.*]] = insertelement <8 x i8> [[_MSPROP6]], i8 [[TMP126]], i32 6
+; CHECK-NEXT:    [[VEC6:%.*]] = insertelement <8 x i8> [[VEC5]], i8 [[TMP127]], i32 6
+; CHECK-NEXT:    [[TMP128:%.*]] = bitcast i8 [[TMP3]] to <8 x i1>
+; CHECK-NEXT:    [[TMP129:%.*]] = bitcast i8 [[MASK]] to <8 x i1>
+; CHECK-NEXT:    [[TMP130:%.*]] = and <8 x i1> zeroinitializer, [[TMP128]]
+; CHECK-NEXT:    [[TMP131:%.*]] = and <8 x i1> splat (i1 true), [[TMP128]]
+; CHECK-NEXT:    [[TMP132:%.*]] = and <8 x i1> zeroinitializer, [[TMP129]]
+; CHECK-NEXT:    [[TMP133:%.*]] = or <8 x i1> [[TMP130]], [[TMP131]]
+; CHECK-NEXT:    [[TMP134:%.*]] = or <8 x i1> [[TMP133]], [[TMP132]]
+; CHECK-NEXT:    [[TMP135:%.*]] = and <8 x i1> splat (i1 true), [[TMP129]]
+; CHECK-NEXT:    [[TMP136:%.*]] = bitcast <8 x i1> [[TMP134]] to i8
+; CHECK-NEXT:    [[TMP137:%.*]] = bitcast <8 x i1> [[TMP135]] to i8
+; CHECK-NEXT:    [[_MSPROP8:%.*]] = insertelement <8 x i8> [[_MSPROP7]], i8 [[TMP136]], i32 7
+; CHECK-NEXT:    [[VEC7:%.*]] = insertelement <8 x i8> [[VEC6]], i8 [[TMP137]], i32 7
+; CHECK-NEXT:    store <8 x i8> [[_MSPROP8]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i8> [[VEC7]]
+;
+  %res0 = call i8 @llvm.x86.avx512.mask.ucmp.q.512(<8 x i64> %a0, <8 x i64> %a1, i32 0, i8 %mask)
+  %vec0 = insertelement <8 x i8> %extra_param, i8 %res0, i32 0
+  %res1 = call i8 @llvm.x86.avx512.mask.ucmp.q.512(<8 x i64> %a0, <8 x i64> %a1, i32 1, i8 %mask)
+  %vec1 = insertelement <8 x i8> %vec0, i8 %res1, i32 1
+  %res2 = call i8 @llvm.x86.avx512.mask.ucmp.q.512(<8 x i64> %a0, <8 x i64> %a1, i32 2, i8 %mask)
+  %vec2 = insertelement <8 x i8> %vec1, i8 %res2, i32 2
+  %res3 = call i8 @llvm.x86.avx512.mask.ucmp.q.512(<8 x i64> %a0, <8 x i64> %a1, i32 3, i8 %mask)
+  %vec3 = insertelement <8 x i8> %vec2, i8 %res3, i32 3
+  %res4 = call i8 @llvm.x86.avx512.mask.ucmp.q.512(<8 x i64> %a0, <8 x i64> %a1, i32 4, i8 %mask)
+  %vec4 = insertelement <8 x i8> %vec3, i8 %res4, i32 4
+  %res5 = call i8 @llvm.x86.avx512.mask.ucmp.q.512(<8 x i64> %a0, <8 x i64> %a1, i32 5, i8 %mask)
+  %vec5 = insertelement <8 x i8> %vec4, i8 %res5, i32 5
+  %res6 = call i8 @llvm.x86.avx512.mask.ucmp.q.512(<8 x i64> %a0, <8 x i64> %a1, i32 6, i8 %mask)
+  %vec6 = insertelement <8 x i8> %vec5, i8 %res6, i32 6
+  %res7 = call i8 @llvm.x86.avx512.mask.ucmp.q.512(<8 x i64> %a0, <8 x i64> %a1, i32 7, i8 %mask)
+  %vec7 = insertelement <8 x i8> %vec6, i8 %res7, i32 7
+  ret <8 x i8> %vec7
+}
+
+declare i8 @llvm.x86.avx512.mask.ucmp.q.512(<8 x i64>, <8 x i64>, i32, i8) nounwind readnone
+
+declare <16 x float> @llvm.x86.avx512.mask.broadcastf32x4.512(<4 x float>, <16 x float>, i16)
+
+define <16 x float>@test_int_x86_avx512_mask_broadcastf32x4_512(<4 x float> %x0, <16 x float> %x2, i16 %mask)  #0 {
+;
+; CHECK-LABEL: @test_int_x86_avx512_mask_broadcastf32x4_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 80) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <4 x i32> [[TMP1]], <4 x i32> [[TMP1]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <4 x float> [[X0:%.*]], <4 x float> [[X0]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = shufflevector <4 x i32> [[TMP1]], <4 x i32> [[TMP1]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[TMP5:%.*]] = shufflevector <4 x float> [[X0]], <4 x float> [[X0]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i16 [[TMP2]] to <16 x i1>
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast i16 [[MASK:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP8:%.*]] = select <16 x i1> [[TMP7]], <16 x i32> [[_MSPROP1]], <16 x i32> [[TMP3]]
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <16 x float> [[TMP5]] to <16 x i32>
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast <16 x float> [[X2:%.*]] to <16 x i32>
+; CHECK-NEXT:    [[TMP11:%.*]] = xor <16 x i32> [[TMP9]], [[TMP10]]
+; CHECK-NEXT:    [[TMP12:%.*]] = or <16 x i32> [[TMP11]], [[_MSPROP1]]
+; CHECK-NEXT:    [[TMP13:%.*]] = or <16 x i32> [[TMP12]], [[TMP3]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP6]], <16 x i32> [[TMP13]], <16 x i32> [[TMP8]]
+; CHECK-NEXT:    [[TMP14:%.*]] = select <16 x i1> [[TMP7]], <16 x float> [[TMP5]], <16 x float> [[X2]]
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = shufflevector <4 x i32> [[TMP1]], <4 x i32> [[TMP1]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[TMP15:%.*]] = shufflevector <4 x float> [[X0]], <4 x float> [[X0]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[TMP16:%.*]] = bitcast i16 [[TMP2]] to <16 x i1>
+; CHECK-NEXT:    [[TMP17:%.*]] = bitcast i16 [[MASK]] to <16 x i1>
+; CHECK-NEXT:    [[TMP18:%.*]] = select <16 x i1> [[TMP17]], <16 x i32> [[_MSPROP2]], <16 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP19:%.*]] = bitcast <16 x float> [[TMP15]] to <16 x i32>
+; CHECK-NEXT:    [[TMP20:%.*]] = xor <16 x i32> [[TMP19]], zeroinitializer
+; CHECK-NEXT:    [[TMP21:%.*]] = or <16 x i32> [[TMP20]], [[_MSPROP2]]
+; CHECK-NEXT:    [[TMP22:%.*]] = or <16 x i32> [[TMP21]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT3:%.*]] = select <16 x i1> [[TMP16]], <16 x i32> [[TMP22]], <16 x i32> [[TMP18]]
+; CHECK-NEXT:    [[TMP23:%.*]] = select <16 x i1> [[TMP17]], <16 x float> [[TMP15]], <16 x float> zeroinitializer
+; CHECK-NEXT:    [[_MSPROP4:%.*]] = or <16 x i32> [[_MSPROP]], [[_MSPROP_SELECT]]
+; CHECK-NEXT:    [[RES4:%.*]] = fadd <16 x float> [[TMP4]], [[TMP14]]
+; CHECK-NEXT:    [[_MSPROP5:%.*]] = or <16 x i32> [[_MSPROP_SELECT3]], [[_MSPROP4]]
+; CHECK-NEXT:    [[RES5:%.*]] = fadd <16 x float> [[TMP23]], [[RES4]]
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP5]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x float> [[RES5]]
+;
+  %res1 = call <16 x float> @llvm.x86.avx512.mask.broadcastf32x4.512(<4 x float> %x0, <16 x float> %x2, i16 -1)
+  %res2 = call <16 x float> @llvm.x86.avx512.mask.broadcastf32x4.512(<4 x float> %x0, <16 x float> %x2, i16 %mask)
+  %res3 = call <16 x float> @llvm.x86.avx512.mask.broadcastf32x4.512(<4 x float> %x0, <16 x float> zeroinitializer, i16 %mask)
+  %res4 = fadd <16 x float> %res1, %res2
+  %res5 = fadd <16 x float> %res3, %res4
+  ret <16 x float> %res5
+}
+
+define <16 x float>@test_int_x86_avx512_mask_broadcastf32x4_512_load(ptr %x0ptr, <16 x float> %x2, i16 %mask)  #0 {
+;
+; CHECK-LABEL: @test_int_x86_avx512_mask_broadcastf32x4_512_load(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 72) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP4:%.*]], label [[TMP5:%.*]], !prof [[PROF1]]
+; CHECK:       4:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       5:
+; CHECK-NEXT:    [[X0:%.*]] = load <4 x float>, ptr [[X0PTR:%.*]], align 16
+; CHECK-NEXT:    [[TMP6:%.*]] = ptrtoint ptr [[X0PTR]] to i64
+; CHECK-NEXT:    [[TMP7:%.*]] = xor i64 [[TMP6]], 87960930222080
+; CHECK-NEXT:    [[TMP8:%.*]] = inttoptr i64 [[TMP7]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <4 x i32>, ptr [[TMP8]], align 16
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <4 x i32> [[_MSLD]], <4 x i32> [[_MSLD]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[TMP9:%.*]] = shufflevector <4 x float> [[X0]], <4 x float> [[X0]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i16 [[TMP2]] to <16 x i1>
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast i16 [[MASK:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP12:%.*]] = select <16 x i1> [[TMP11]], <16 x i32> [[_MSPROP]], <16 x i32> [[TMP3]]
+; CHECK-NEXT:    [[TMP13:%.*]] = bitcast <16 x float> [[TMP9]] to <16 x i32>
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast <16 x float> [[X2:%.*]] to <16 x i32>
+; CHECK-NEXT:    [[TMP15:%.*]] = xor <16 x i32> [[TMP13]], [[TMP14]]
+; CHECK-NEXT:    [[TMP16:%.*]] = or <16 x i32> [[TMP15]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP17:%.*]] = or <16 x i32> [[TMP16]], [[TMP3]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP10]], <16 x i32> [[TMP17]], <16 x i32> [[TMP12]]
+; CHECK-NEXT:    [[TMP18:%.*]] = select <16 x i1> [[TMP11]], <16 x float> [[TMP9]], <16 x float> [[X2]]
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x float> [[TMP18]]
+;
+  %x0 = load <4 x float>, ptr %x0ptr
+  %res = call <16 x float> @llvm.x86.avx512.mask.broadcastf32x4.512(<4 x float> %x0, <16 x float> %x2, i16 %mask)
+  ret <16 x float> %res
+}
+
+declare <8 x double> @llvm.x86.avx512.mask.broadcastf64x4.512(<4 x double>, <8 x double>, i8)
+
+define <8 x double>@test_int_x86_avx512_broadcastf64x4_512(<4 x double> %x0, <8 x double> %x2)  #0 {
+; CHECK-LABEL: @test_int_x86_avx512_broadcastf64x4_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <4 x i64> [[TMP1]], <4 x i64> [[TMP1]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <4 x double> [[X0:%.*]], <4 x double> [[X0]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    store <8 x i64> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x double> [[TMP2]]
+;
+  %res = call <8 x double> @llvm.x86.avx512.mask.broadcastf64x4.512(<4 x double> %x0, <8 x double> %x2, i8 -1)
+  ret <8 x double> %res
+}
+
+define <8 x double>@test_int_x86_avx512_mask_broadcastf64x4_512(<4 x double> %x0, <8 x double> %x2, i8 %mask)  #0 {
+;
+; CHECK-LABEL: @test_int_x86_avx512_mask_broadcastf64x4_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 96) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <4 x i64> [[TMP1]], <4 x i64> [[TMP1]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <4 x double> [[X0:%.*]], <4 x double> [[X0]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i8 [[TMP2]] to <8 x i1>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP7:%.*]] = select <8 x i1> [[TMP6]], <8 x i64> [[_MSPROP]], <8 x i64> [[TMP3]]
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <8 x double> [[TMP4]] to <8 x i64>
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <8 x double> [[X2:%.*]] to <8 x i64>
+; CHECK-NEXT:    [[TMP10:%.*]] = xor <8 x i64> [[TMP8]], [[TMP9]]
+; CHECK-NEXT:    [[TMP11:%.*]] = or <8 x i64> [[TMP10]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP12:%.*]] = or <8 x i64> [[TMP11]], [[TMP3]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP5]], <8 x i64> [[TMP12]], <8 x i64> [[TMP7]]
+; CHECK-NEXT:    [[TMP13:%.*]] = select <8 x i1> [[TMP6]], <8 x double> [[TMP4]], <8 x double> [[X2]]
+; CHECK-NEXT:    store <8 x i64> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x double> [[TMP13]]
+;
+  %res = call <8 x double> @llvm.x86.avx512.mask.broadcastf64x4.512(<4 x double> %x0, <8 x double> %x2, i8 %mask)
+  ret <8 x double> %res
+}
+
+define <8 x double>@test_int_x86_avx512_maskz_broadcastf64x4_512(<4 x double> %x0, i8 %mask)  #0 {
+;
+; CHECK-LABEL: @test_int_x86_avx512_maskz_broadcastf64x4_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <4 x i64> [[TMP1]], <4 x i64> [[TMP1]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <4 x double> [[X0:%.*]], <4 x double> [[X0]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast i8 [[TMP2]] to <8 x i1>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP6:%.*]] = select <8 x i1> [[TMP5]], <8 x i64> [[_MSPROP]], <8 x i64> zeroinitializer
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <8 x double> [[TMP3]] to <8 x i64>
+; CHECK-NEXT:    [[TMP8:%.*]] = xor <8 x i64> [[TMP7]], zeroinitializer
+; CHECK-NEXT:    [[TMP9:%.*]] = or <8 x i64> [[TMP8]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP10:%.*]] = or <8 x i64> [[TMP9]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP4]], <8 x i64> [[TMP10]], <8 x i64> [[TMP6]]
+; CHECK-NEXT:    [[TMP11:%.*]] = select <8 x i1> [[TMP5]], <8 x double> [[TMP3]], <8 x double> zeroinitializer
+; CHECK-NEXT:    store <8 x i64> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x double> [[TMP11]]
+;
+  %res = call <8 x double> @llvm.x86.avx512.mask.broadcastf64x4.512(<4 x double> %x0, <8 x double> zeroinitializer, i8 %mask)
+  ret <8 x double> %res
+}
+
+define <8 x double>@test_int_x86_avx512_mask_broadcastf64x4_512_load(ptr %x0ptr, <8 x double> %x2, i8 %mask)  #0 {
+;
+; CHECK-LABEL: @test_int_x86_avx512_mask_broadcastf64x4_512_load(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 72) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP4:%.*]], label [[TMP5:%.*]], !prof [[PROF1]]
+; CHECK:       4:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       5:
+; CHECK-NEXT:    [[X0:%.*]] = load <4 x double>, ptr [[X0PTR:%.*]], align 32
+; CHECK-NEXT:    [[TMP6:%.*]] = ptrtoint ptr [[X0PTR]] to i64
+; CHECK-NEXT:    [[TMP7:%.*]] = xor i64 [[TMP6]], 87960930222080
+; CHECK-NEXT:    [[TMP8:%.*]] = inttoptr i64 [[TMP7]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <4 x i64>, ptr [[TMP8]], align 32
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <4 x i64> [[_MSLD]], <4 x i64> [[_MSLD]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[TMP9:%.*]] = shufflevector <4 x double> [[X0]], <4 x double> [[X0]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i8 [[TMP2]] to <8 x i1>
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP12:%.*]] = select <8 x i1> [[TMP11]], <8 x i64> [[_MSPROP]], <8 x i64> [[TMP3]]
+; CHECK-NEXT:    [[TMP13:%.*]] = bitcast <8 x double> [[TMP9]] to <8 x i64>
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast <8 x double> [[X2:%.*]] to <8 x i64>
+; CHECK-NEXT:    [[TMP15:%.*]] = xor <8 x i64> [[TMP13]], [[TMP14]]
+; CHECK-NEXT:    [[TMP16:%.*]] = or <8 x i64> [[TMP15]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP17:%.*]] = or <8 x i64> [[TMP16]], [[TMP3]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP10]], <8 x i64> [[TMP17]], <8 x i64> [[TMP12]]
+; CHECK-NEXT:    [[TMP18:%.*]] = select <8 x i1> [[TMP11]], <8 x double> [[TMP9]], <8 x double> [[X2]]
+; CHECK-NEXT:    store <8 x i64> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x double> [[TMP18]]
+;
+  %x0 = load <4 x double>, ptr %x0ptr
+  %res = call <8 x double> @llvm.x86.avx512.mask.broadcastf64x4.512(<4 x double> %x0, <8 x double> %x2, i8 %mask)
+  ret <8 x double> %res
+}
+
+declare <16 x i32> @llvm.x86.avx512.mask.broadcasti32x4.512(<4 x i32>, <16 x i32>, i16)
+
+define { <16 x i32>, <16 x i32>, <16 x i32> } @test_int_x86_avx512_mask_broadcasti32x4_512(<4 x i32> %x0, <16 x i32> %x2, i16 %mask)  #0 {
+;
+; CHECK-LABEL: @test_int_x86_avx512_mask_broadcasti32x4_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 80) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <4 x i32> [[TMP1]], <4 x i32> [[TMP1]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <4 x i32> [[X0:%.*]], <4 x i32> [[X0]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = shufflevector <4 x i32> [[TMP1]], <4 x i32> [[TMP1]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[TMP5:%.*]] = shufflevector <4 x i32> [[X0]], <4 x i32> [[X0]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i16 [[TMP2]] to <16 x i1>
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast i16 [[MASK:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP8:%.*]] = select <16 x i1> [[TMP7]], <16 x i32> [[_MSPROP1]], <16 x i32> [[TMP3]]
+; CHECK-NEXT:    [[TMP9:%.*]] = xor <16 x i32> [[TMP5]], [[X2:%.*]]
+; CHECK-NEXT:    [[TMP10:%.*]] = or <16 x i32> [[TMP9]], [[_MSPROP1]]
+; CHECK-NEXT:    [[TMP11:%.*]] = or <16 x i32> [[TMP10]], [[TMP3]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP6]], <16 x i32> [[TMP11]], <16 x i32> [[TMP8]]
+; CHECK-NEXT:    [[TMP12:%.*]] = select <16 x i1> [[TMP7]], <16 x i32> [[TMP5]], <16 x i32> [[X2]]
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = shufflevector <4 x i32> [[TMP1]], <4 x i32> [[TMP1]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[TMP13:%.*]] = shufflevector <4 x i32> [[X0]], <4 x i32> [[X0]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast i16 [[TMP2]] to <16 x i1>
+; CHECK-NEXT:    [[TMP15:%.*]] = bitcast i16 [[MASK]] to <16 x i1>
+; CHECK-NEXT:    [[TMP16:%.*]] = select <16 x i1> [[TMP15]], <16 x i32> [[_MSPROP2]], <16 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP17:%.*]] = xor <16 x i32> [[TMP13]], zeroinitializer
+; CHECK-NEXT:    [[TMP18:%.*]] = or <16 x i32> [[TMP17]], [[_MSPROP2]]
+; CHECK-NEXT:    [[TMP19:%.*]] = or <16 x i32> [[TMP18]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT3:%.*]] = select <16 x i1> [[TMP14]], <16 x i32> [[TMP19]], <16 x i32> [[TMP16]]
+; CHECK-NEXT:    [[TMP20:%.*]] = select <16 x i1> [[TMP15]], <16 x i32> [[TMP13]], <16 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP21:%.*]] = insertvalue { <16 x i32>, <16 x i32>, <16 x i32> } { <16 x i32> splat (i32 -1), <16 x i32> splat (i32 -1), <16 x i32> splat (i32 -1) }, <16 x i32> [[_MSPROP]], 0
+; CHECK-NEXT:    [[RES3:%.*]] = insertvalue { <16 x i32>, <16 x i32>, <16 x i32> } poison, <16 x i32> [[TMP4]], 0
+; CHECK-NEXT:    [[TMP22:%.*]] = insertvalue { <16 x i32>, <16 x i32>, <16 x i32> } [[TMP21]], <16 x i32> [[_MSPROP_SELECT]], 1
+; CHECK-NEXT:    [[RES4:%.*]] = insertvalue { <16 x i32>, <16 x i32>, <16 x i32> } [[RES3]], <16 x i32> [[TMP12]], 1
+; CHECK-NEXT:    [[TMP23:%.*]] = insertvalue { <16 x i32>, <16 x i32>, <16 x i32> } [[TMP22]], <16 x i32> [[_MSPROP_SELECT3]], 2
+; CHECK-NEXT:    [[RES5:%.*]] = insertvalue { <16 x i32>, <16 x i32>, <16 x i32> } [[RES4]], <16 x i32> [[TMP20]], 2
+; CHECK-NEXT:    store { <16 x i32>, <16 x i32>, <16 x i32> } [[TMP23]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret { <16 x i32>, <16 x i32>, <16 x i32> } [[RES5]]
+;
+  %res0 = call <16 x i32> @llvm.x86.avx512.mask.broadcasti32x4.512(<4 x i32> %x0, <16 x i32> %x2, i16 -1)
+  %res1 = call <16 x i32> @llvm.x86.avx512.mask.broadcasti32x4.512(<4 x i32> %x0, <16 x i32> %x2, i16 %mask)
+  %res2 = call <16 x i32> @llvm.x86.avx512.mask.broadcasti32x4.512(<4 x i32> %x0, <16 x i32> zeroinitializer, i16 %mask)
+  %res3 = insertvalue { <16 x i32>, <16 x i32>, <16 x i32> } poison, <16 x i32> %res0, 0
+  %res4 = insertvalue { <16 x i32>, <16 x i32>, <16 x i32> }  %res3, <16 x i32> %res1, 1
+  %res5 = insertvalue { <16 x i32>, <16 x i32>, <16 x i32> }  %res4, <16 x i32> %res2, 2
+  ret { <16 x i32>, <16 x i32>, <16 x i32> } %res5
+}
+
+define <16 x i32>@test_int_x86_avx512_mask_broadcasti32x4_512_load(ptr %x0ptr, <16 x i32> %x2, i16 %mask)  #0 {
+;
+; CHECK-LABEL: @test_int_x86_avx512_mask_broadcasti32x4_512_load(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 72) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP4:%.*]], label [[TMP5:%.*]], !prof [[PROF1]]
+; CHECK:       4:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       5:
+; CHECK-NEXT:    [[X0:%.*]] = load <4 x i32>, ptr [[X0PTR:%.*]], align 16
+; CHECK-NEXT:    [[TMP6:%.*]] = ptrtoint ptr [[X0PTR]] to i64
+; CHECK-NEXT:    [[TMP7:%.*]] = xor i64 [[TMP6]], 87960930222080
+; CHECK-NEXT:    [[TMP8:%.*]] = inttoptr i64 [[TMP7]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <4 x i32>, ptr [[TMP8]], align 16
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <4 x i32> [[_MSLD]], <4 x i32> [[_MSLD]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[TMP9:%.*]] = shufflevector <4 x i32> [[X0]], <4 x i32> [[X0]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i16 [[TMP2]] to <16 x i1>
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast i16 [[MASK:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP12:%.*]] = select <16 x i1> [[TMP11]], <16 x i32> [[_MSPROP]], <16 x i32> [[TMP3]]
+; CHECK-NEXT:    [[TMP13:%.*]] = xor <16 x i32> [[TMP9]], [[X2:%.*]]
+; CHECK-NEXT:    [[TMP14:%.*]] = or <16 x i32> [[TMP13]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP15:%.*]] = or <16 x i32> [[TMP14]], [[TMP3]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP10]], <16 x i32> [[TMP15]], <16 x i32> [[TMP12]]
+; CHECK-NEXT:    [[TMP16:%.*]] = select <16 x i1> [[TMP11]], <16 x i32> [[TMP9]], <16 x i32> [[X2]]
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i32> [[TMP16]]
+;
+  %x0 = load <4 x i32>, ptr %x0ptr
+  %res = call <16 x i32> @llvm.x86.avx512.mask.broadcasti32x4.512(<4 x i32> %x0, <16 x i32> %x2, i16 %mask)
+  ret <16 x i32> %res
+}
+
+declare <8 x i64> @llvm.x86.avx512.mask.broadcasti64x4.512(<4 x i64>, <8 x i64>, i8)
+
+define <8 x i64>@test_int_x86_avx512_broadcasti64x4_512(<4 x i64> %x0, <8 x i64> %x2)  #0 {
+; CHECK-LABEL: @test_int_x86_avx512_broadcasti64x4_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <4 x i64> [[TMP1]], <4 x i64> [[TMP1]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <4 x i64> [[X0:%.*]], <4 x i64> [[X0]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    store <8 x i64> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i64> [[TMP2]]
+;
+  %res = call <8 x i64> @llvm.x86.avx512.mask.broadcasti64x4.512(<4 x i64> %x0, <8 x i64> %x2, i8 -1)
+  ret <8 x i64> %res
+}
+
+define <8 x i64>@test_int_x86_avx512_mask_broadcasti64x4_512(<4 x i64> %x0, <8 x i64> %x2, i8 %mask)  #0 {
+;
+; CHECK-LABEL: @test_int_x86_avx512_mask_broadcasti64x4_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 96) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <4 x i64> [[TMP1]], <4 x i64> [[TMP1]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <4 x i64> [[X0:%.*]], <4 x i64> [[X0]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i8 [[TMP2]] to <8 x i1>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP7:%.*]] = select <8 x i1> [[TMP6]], <8 x i64> [[_MSPROP]], <8 x i64> [[TMP3]]
+; CHECK-NEXT:    [[TMP8:%.*]] = xor <8 x i64> [[TMP4]], [[X2:%.*]]
+; CHECK-NEXT:    [[TMP9:%.*]] = or <8 x i64> [[TMP8]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP10:%.*]] = or <8 x i64> [[TMP9]], [[TMP3]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP5]], <8 x i64> [[TMP10]], <8 x i64> [[TMP7]]
+; CHECK-NEXT:    [[TMP11:%.*]] = select <8 x i1> [[TMP6]], <8 x i64> [[TMP4]], <8 x i64> [[X2]]
+; CHECK-NEXT:    store <8 x i64> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i64> [[TMP11]]
+;
+  %res = call <8 x i64> @llvm.x86.avx512.mask.broadcasti64x4.512(<4 x i64> %x0, <8 x i64> %x2, i8 %mask)
+  ret <8 x i64> %res
+}
+
+define <8 x i64>@test_int_x86_avx512_maskz_broadcasti64x4_512(<4 x i64> %x0, i8 %mask)  #0 {
+;
+; CHECK-LABEL: @test_int_x86_avx512_maskz_broadcasti64x4_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <4 x i64> [[TMP1]], <4 x i64> [[TMP1]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <4 x i64> [[X0:%.*]], <4 x i64> [[X0]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast i8 [[TMP2]] to <8 x i1>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP6:%.*]] = select <8 x i1> [[TMP5]], <8 x i64> [[_MSPROP]], <8 x i64> zeroinitializer
+; CHECK-NEXT:    [[TMP7:%.*]] = xor <8 x i64> [[TMP3]], zeroinitializer
+; CHECK-NEXT:    [[TMP8:%.*]] = or <8 x i64> [[TMP7]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP9:%.*]] = or <8 x i64> [[TMP8]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP4]], <8 x i64> [[TMP9]], <8 x i64> [[TMP6]]
+; CHECK-NEXT:    [[TMP10:%.*]] = select <8 x i1> [[TMP5]], <8 x i64> [[TMP3]], <8 x i64> zeroinitializer
+; CHECK-NEXT:    store <8 x i64> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i64> [[TMP10]]
+;
+  %res = call <8 x i64> @llvm.x86.avx512.mask.broadcasti64x4.512(<4 x i64> %x0, <8 x i64> zeroinitializer, i8 %mask)
+  ret <8 x i64> %res
+}
+
+define <8 x i64>@test_int_x86_avx512_mask_broadcasti64x4_512_load(ptr %x0ptr, <8 x i64> %x2, i8 %mask)  #0 {
+;
+; CHECK-LABEL: @test_int_x86_avx512_mask_broadcasti64x4_512_load(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 72) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP4:%.*]], label [[TMP5:%.*]], !prof [[PROF1]]
+; CHECK:       4:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       5:
+; CHECK-NEXT:    [[X0:%.*]] = load <4 x i64>, ptr [[X0PTR:%.*]], align 32
+; CHECK-NEXT:    [[TMP6:%.*]] = ptrtoint ptr [[X0PTR]] to i64
+; CHECK-NEXT:    [[TMP7:%.*]] = xor i64 [[TMP6]], 87960930222080
+; CHECK-NEXT:    [[TMP8:%.*]] = inttoptr i64 [[TMP7]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <4 x i64>, ptr [[TMP8]], align 32
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <4 x i64> [[_MSLD]], <4 x i64> [[_MSLD]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[TMP9:%.*]] = shufflevector <4 x i64> [[X0]], <4 x i64> [[X0]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i8 [[TMP2]] to <8 x i1>
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP12:%.*]] = select <8 x i1> [[TMP11]], <8 x i64> [[_MSPROP]], <8 x i64> [[TMP3]]
+; CHECK-NEXT:    [[TMP13:%.*]] = xor <8 x i64> [[TMP9]], [[X2:%.*]]
+; CHECK-NEXT:    [[TMP14:%.*]] = or <8 x i64> [[TMP13]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP15:%.*]] = or <8 x i64> [[TMP14]], [[TMP3]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP10]], <8 x i64> [[TMP15]], <8 x i64> [[TMP12]]
+; CHECK-NEXT:    [[TMP16:%.*]] = select <8 x i1> [[TMP11]], <8 x i64> [[TMP9]], <8 x i64> [[X2]]
+; CHECK-NEXT:    store <8 x i64> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i64> [[TMP16]]
+;
+  %x0 = load <4 x i64>, ptr %x0ptr
+  %res = call <8 x i64> @llvm.x86.avx512.mask.broadcasti64x4.512(<4 x i64> %x0, <8 x i64> %x2, i8 %mask)
+  ret <8 x i64> %res
+}
+
+declare <16 x i32> @llvm.x86.avx512.mask.pabs.d.512(<16 x i32>, <16 x i32>, i16)
+
+define <16 x i32>@test_int_x86_avx512_pabs_d_512(<16 x i32> %x0, <16 x i32> %x1)  #0 {
+; CHECK-LABEL: @test_int_x86_avx512_pabs_d_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = call <16 x i32> @llvm.abs.v16i32(<16 x i32> [[X0:%.*]], i1 false)
+; CHECK-NEXT:    store <16 x i32> [[TMP1]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i32> [[TMP2]]
+;
+  %res = call <16 x i32> @llvm.x86.avx512.mask.pabs.d.512(<16 x i32> %x0, <16 x i32> %x1, i16 -1)
+  ret <16 x i32> %res
+}
+
+define <16 x i32>@test_int_x86_avx512_mask_pabs_d_512(<16 x i32> %x0, <16 x i32> %x1, i16 %x2)  #0 {
+;
+; CHECK-LABEL: @test_int_x86_avx512_mask_pabs_d_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = call <16 x i32> @llvm.abs.v16i32(<16 x i32> [[X0:%.*]], i1 false)
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i16 [[TMP2]] to <16 x i1>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i16 [[X2:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP7:%.*]] = select <16 x i1> [[TMP6]], <16 x i32> [[TMP1]], <16 x i32> [[TMP3]]
+; CHECK-NEXT:    [[TMP8:%.*]] = xor <16 x i32> [[TMP4]], [[X1:%.*]]
+; CHECK-NEXT:    [[TMP9:%.*]] = or <16 x i32> [[TMP8]], [[TMP1]]
+; CHECK-NEXT:    [[TMP10:%.*]] = or <16 x i32> [[TMP9]], [[TMP3]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP5]], <16 x i32> [[TMP10]], <16 x i32> [[TMP7]]
+; CHECK-NEXT:    [[TMP11:%.*]] = select <16 x i1> [[TMP6]], <16 x i32> [[TMP4]], <16 x i32> [[X1]]
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i32> [[TMP11]]
+;
+  %res = call <16 x i32> @llvm.x86.avx512.mask.pabs.d.512(<16 x i32> %x0, <16 x i32> %x1, i16 %x2)
+  ret <16 x i32> %res
+}
+
+declare <8 x i64> @llvm.x86.avx512.mask.pabs.q.512(<8 x i64>, <8 x i64>, i8)
+
+define <8 x i64>@test_int_x86_avx512_pabs_q_512(<8 x i64> %x0, <8 x i64> %x1, i8 %x2)  #0 {
+; CHECK-LABEL: @test_int_x86_avx512_pabs_q_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = call <8 x i64> @llvm.abs.v8i64(<8 x i64> [[X0:%.*]], i1 false)
+; CHECK-NEXT:    store <8 x i64> [[TMP1]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i64> [[TMP2]]
+;
+  %res = call <8 x i64> @llvm.x86.avx512.mask.pabs.q.512(<8 x i64> %x0, <8 x i64> %x1, i8 -1)
+  ret <8 x i64> %res
+}
+
+define <8 x i64>@test_int_x86_avx512_mask_pabs_q_512(<8 x i64> %x0, <8 x i64> %x1, i8 %x2)  #0 {
+;
+; CHECK-LABEL: @test_int_x86_avx512_mask_pabs_q_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = call <8 x i64> @llvm.abs.v8i64(<8 x i64> [[X0:%.*]], i1 false)
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i8 [[TMP2]] to <8 x i1>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i8 [[X2:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP7:%.*]] = select <8 x i1> [[TMP6]], <8 x i64> [[TMP1]], <8 x i64> [[TMP3]]
+; CHECK-NEXT:    [[TMP8:%.*]] = xor <8 x i64> [[TMP4]], [[X1:%.*]]
+; CHECK-NEXT:    [[TMP9:%.*]] = or <8 x i64> [[TMP8]], [[TMP1]]
+; CHECK-NEXT:    [[TMP10:%.*]] = or <8 x i64> [[TMP9]], [[TMP3]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP5]], <8 x i64> [[TMP10]], <8 x i64> [[TMP7]]
+; CHECK-NEXT:    [[TMP11:%.*]] = select <8 x i1> [[TMP6]], <8 x i64> [[TMP4]], <8 x i64> [[X1]]
+; CHECK-NEXT:    store <8 x i64> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i64> [[TMP11]]
+;
+  %res = call <8 x i64> @llvm.x86.avx512.mask.pabs.q.512(<8 x i64> %x0, <8 x i64> %x1, i8 %x2)
+  ret <8 x i64> %res
+}
+
+define i8 @test_vptestmq(<8 x i64> %a0, <8 x i64> %a1, i8 %m)  #0 {
+;
+; CHECK-LABEL: @test_vptestmq(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = and <8 x i64> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP5:%.*]] = and <8 x i64> [[A0:%.*]], [[TMP2]]
+; CHECK-NEXT:    [[TMP6:%.*]] = and <8 x i64> [[TMP1]], [[A1:%.*]]
+; CHECK-NEXT:    [[TMP7:%.*]] = or <8 x i64> [[TMP4]], [[TMP5]]
+; CHECK-NEXT:    [[TMP8:%.*]] = or <8 x i64> [[TMP7]], [[TMP6]]
+; CHECK-NEXT:    [[TMP9:%.*]] = and <8 x i64> [[A0]], [[A1]]
+; CHECK-NEXT:    [[TMP10:%.*]] = xor <8 x i64> [[TMP9]], zeroinitializer
+; CHECK-NEXT:    [[TMP11:%.*]] = or <8 x i64> [[TMP8]], zeroinitializer
+; CHECK-NEXT:    [[TMP12:%.*]] = icmp ne <8 x i64> [[TMP11]], zeroinitializer
+; CHECK-NEXT:    [[TMP13:%.*]] = xor <8 x i64> [[TMP11]], splat (i64 -1)
+; CHECK-NEXT:    [[TMP14:%.*]] = and <8 x i64> [[TMP13]], [[TMP10]]
+; CHECK-NEXT:    [[TMP15:%.*]] = icmp eq <8 x i64> [[TMP14]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_ICMP:%.*]] = and <8 x i1> [[TMP12]], [[TMP15]]
+; CHECK-NEXT:    [[TMP16:%.*]] = icmp ne <8 x i64> [[TMP9]], zeroinitializer
+; CHECK-NEXT:    [[TMP17:%.*]] = bitcast <8 x i1> [[_MSPROP_ICMP]] to i8
+; CHECK-NEXT:    [[TMP18:%.*]] = bitcast <8 x i1> [[TMP16]] to i8
+; CHECK-NEXT:    [[TMP19:%.*]] = and <8 x i64> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP20:%.*]] = and <8 x i64> [[A0]], [[TMP2]]
+; CHECK-NEXT:    [[TMP21:%.*]] = and <8 x i64> [[TMP1]], [[A1]]
+; CHECK-NEXT:    [[TMP22:%.*]] = or <8 x i64> [[TMP19]], [[TMP20]]
+; CHECK-NEXT:    [[TMP23:%.*]] = or <8 x i64> [[TMP22]], [[TMP21]]
+; CHECK-NEXT:    [[TMP24:%.*]] = and <8 x i64> [[A0]], [[A1]]
+; CHECK-NEXT:    [[TMP25:%.*]] = xor <8 x i64> [[TMP24]], zeroinitializer
+; CHECK-NEXT:    [[TMP26:%.*]] = or <8 x i64> [[TMP23]], zeroinitializer
+; CHECK-NEXT:    [[TMP27:%.*]] = icmp ne <8 x i64> [[TMP26]], zeroinitializer
+; CHECK-NEXT:    [[TMP28:%.*]] = xor <8 x i64> [[TMP26]], splat (i64 -1)
+; CHECK-NEXT:    [[TMP29:%.*]] = and <8 x i64> [[TMP28]], [[TMP25]]
+; CHECK-NEXT:    [[TMP30:%.*]] = icmp eq <8 x i64> [[TMP29]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_ICMP1:%.*]] = and <8 x i1> [[TMP27]], [[TMP30]]
+; CHECK-NEXT:    [[TMP31:%.*]] = icmp ne <8 x i64> [[TMP24]], zeroinitializer
+; CHECK-NEXT:    [[TMP32:%.*]] = bitcast i8 [[TMP3]] to <8 x i1>
+; CHECK-NEXT:    [[TMP33:%.*]] = bitcast i8 [[M:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP34:%.*]] = and <8 x i1> [[_MSPROP_ICMP1]], [[TMP32]]
+; CHECK-NEXT:    [[TMP35:%.*]] = and <8 x i1> [[TMP31]], [[TMP32]]
+; CHECK-NEXT:    [[TMP36:%.*]] = and <8 x i1> [[_MSPROP_ICMP1]], [[TMP33]]
+; CHECK-NEXT:    [[TMP37:%.*]] = or <8 x i1> [[TMP34]], [[TMP35]]
+; CHECK-NEXT:    [[TMP38:%.*]] = or <8 x i1> [[TMP37]], [[TMP36]]
+; CHECK-NEXT:    [[TMP39:%.*]] = and <8 x i1> [[TMP31]], [[TMP33]]
+; CHECK-NEXT:    [[TMP40:%.*]] = bitcast <8 x i1> [[TMP38]] to i8
+; CHECK-NEXT:    [[TMP41:%.*]] = bitcast <8 x i1> [[TMP39]] to i8
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or i8 [[TMP40]], [[TMP17]]
+; CHECK-NEXT:    [[RES2:%.*]] = add i8 [[TMP41]], [[TMP18]]
+; CHECK-NEXT:    store i8 [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i8 [[RES2]]
+;
+  %res = call i8 @llvm.x86.avx512.ptestm.q.512(<8 x i64> %a0, <8 x i64> %a1, i8 -1)
+  %res1 = call i8 @llvm.x86.avx512.ptestm.q.512(<8 x i64> %a0, <8 x i64> %a1, i8 %m)
+  %res2 = add i8 %res1, %res
+  ret i8 %res2
+}
+declare i8 @llvm.x86.avx512.ptestm.q.512(<8 x i64>, <8 x i64>, i8)
+
+define i16 @test_vptestmd(<16 x i32> %a0, <16 x i32> %a1, i16 %m)  #0 {
+;
+; CHECK-LABEL: @test_vptestmd(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = and <16 x i32> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP5:%.*]] = and <16 x i32> [[A0:%.*]], [[TMP2]]
+; CHECK-NEXT:    [[TMP6:%.*]] = and <16 x i32> [[TMP1]], [[A1:%.*]]
+; CHECK-NEXT:    [[TMP7:%.*]] = or <16 x i32> [[TMP4]], [[TMP5]]
+; CHECK-NEXT:    [[TMP8:%.*]] = or <16 x i32> [[TMP7]], [[TMP6]]
+; CHECK-NEXT:    [[TMP9:%.*]] = and <16 x i32> [[A0]], [[A1]]
+; CHECK-NEXT:    [[TMP10:%.*]] = xor <16 x i32> [[TMP9]], zeroinitializer
+; CHECK-NEXT:    [[TMP11:%.*]] = or <16 x i32> [[TMP8]], zeroinitializer
+; CHECK-NEXT:    [[TMP12:%.*]] = icmp ne <16 x i32> [[TMP11]], zeroinitializer
+; CHECK-NEXT:    [[TMP13:%.*]] = xor <16 x i32> [[TMP11]], splat (i32 -1)
+; CHECK-NEXT:    [[TMP14:%.*]] = and <16 x i32> [[TMP13]], [[TMP10]]
+; CHECK-NEXT:    [[TMP15:%.*]] = icmp eq <16 x i32> [[TMP14]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_ICMP:%.*]] = and <16 x i1> [[TMP12]], [[TMP15]]
+; CHECK-NEXT:    [[TMP16:%.*]] = icmp ne <16 x i32> [[TMP9]], zeroinitializer
+; CHECK-NEXT:    [[TMP17:%.*]] = bitcast <16 x i1> [[_MSPROP_ICMP]] to i16
+; CHECK-NEXT:    [[TMP18:%.*]] = bitcast <16 x i1> [[TMP16]] to i16
+; CHECK-NEXT:    [[TMP19:%.*]] = and <16 x i32> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP20:%.*]] = and <16 x i32> [[A0]], [[TMP2]]
+; CHECK-NEXT:    [[TMP21:%.*]] = and <16 x i32> [[TMP1]], [[A1]]
+; CHECK-NEXT:    [[TMP22:%.*]] = or <16 x i32> [[TMP19]], [[TMP20]]
+; CHECK-NEXT:    [[TMP23:%.*]] = or <16 x i32> [[TMP22]], [[TMP21]]
+; CHECK-NEXT:    [[TMP24:%.*]] = and <16 x i32> [[A0]], [[A1]]
+; CHECK-NEXT:    [[TMP25:%.*]] = xor <16 x i32> [[TMP24]], zeroinitializer
+; CHECK-NEXT:    [[TMP26:%.*]] = or <16 x i32> [[TMP23]], zeroinitializer
+; CHECK-NEXT:    [[TMP27:%.*]] = icmp ne <16 x i32> [[TMP26]], zeroinitializer
+; CHECK-NEXT:    [[TMP28:%.*]] = xor <16 x i32> [[TMP26]], splat (i32 -1)
+; CHECK-NEXT:    [[TMP29:%.*]] = and <16 x i32> [[TMP28]], [[TMP25]]
+; CHECK-NEXT:    [[TMP30:%.*]] = icmp eq <16 x i32> [[TMP29]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_ICMP1:%.*]] = and <16 x i1> [[TMP27]], [[TMP30]]
+; CHECK-NEXT:    [[TMP31:%.*]] = icmp ne <16 x i32> [[TMP24]], zeroinitializer
+; CHECK-NEXT:    [[TMP32:%.*]] = bitcast i16 [[TMP3]] to <16 x i1>
+; CHECK-NEXT:    [[TMP33:%.*]] = bitcast i16 [[M:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP34:%.*]] = and <16 x i1> [[_MSPROP_ICMP1]], [[TMP32]]
+; CHECK-NEXT:    [[TMP35:%.*]] = and <16 x i1> [[TMP31]], [[TMP32]]
+; CHECK-NEXT:    [[TMP36:%.*]] = and <16 x i1> [[_MSPROP_ICMP1]], [[TMP33]]
+; CHECK-NEXT:    [[TMP37:%.*]] = or <16 x i1> [[TMP34]], [[TMP35]]
+; CHECK-NEXT:    [[TMP38:%.*]] = or <16 x i1> [[TMP37]], [[TMP36]]
+; CHECK-NEXT:    [[TMP39:%.*]] = and <16 x i1> [[TMP31]], [[TMP33]]
+; CHECK-NEXT:    [[TMP40:%.*]] = bitcast <16 x i1> [[TMP38]] to i16
+; CHECK-NEXT:    [[TMP41:%.*]] = bitcast <16 x i1> [[TMP39]] to i16
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or i16 [[TMP40]], [[TMP17]]
+; CHECK-NEXT:    [[RES2:%.*]] = add i16 [[TMP41]], [[TMP18]]
+; CHECK-NEXT:    store i16 [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i16 [[RES2]]
+;
+  %res = call i16 @llvm.x86.avx512.ptestm.d.512(<16 x i32> %a0, <16 x i32> %a1, i16 -1)
+  %res1 = call i16 @llvm.x86.avx512.ptestm.d.512(<16 x i32> %a0, <16 x i32> %a1, i16 %m)
+  %res2 = add i16 %res1, %res
+  ret i16 %res2
+}
+declare i16 @llvm.x86.avx512.ptestm.d.512(<16 x i32>, <16 x i32>, i16)
+
+declare i16 @llvm.x86.avx512.ptestnm.d.512(<16 x i32>, <16 x i32>, i16 %x2)
+
+define i16@test_int_x86_avx512_ptestnm_d_512(<16 x i32> %x0, <16 x i32> %x1, i16 %x2)  #0 {
+;
+; CHECK-LABEL: @test_int_x86_avx512_ptestnm_d_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = and <16 x i32> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP5:%.*]] = and <16 x i32> [[X0:%.*]], [[TMP2]]
+; CHECK-NEXT:    [[TMP6:%.*]] = and <16 x i32> [[TMP1]], [[X1:%.*]]
+; CHECK-NEXT:    [[TMP7:%.*]] = or <16 x i32> [[TMP4]], [[TMP5]]
+; CHECK-NEXT:    [[TMP8:%.*]] = or <16 x i32> [[TMP7]], [[TMP6]]
+; CHECK-NEXT:    [[TMP9:%.*]] = and <16 x i32> [[X0]], [[X1]]
+; CHECK-NEXT:    [[TMP10:%.*]] = xor <16 x i32> [[TMP9]], zeroinitializer
+; CHECK-NEXT:    [[TMP11:%.*]] = or <16 x i32> [[TMP8]], zeroinitializer
+; CHECK-NEXT:    [[TMP12:%.*]] = icmp ne <16 x i32> [[TMP11]], zeroinitializer
+; CHECK-NEXT:    [[TMP13:%.*]] = xor <16 x i32> [[TMP11]], splat (i32 -1)
+; CHECK-NEXT:    [[TMP14:%.*]] = and <16 x i32> [[TMP13]], [[TMP10]]
+; CHECK-NEXT:    [[TMP15:%.*]] = icmp eq <16 x i32> [[TMP14]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_ICMP:%.*]] = and <16 x i1> [[TMP12]], [[TMP15]]
+; CHECK-NEXT:    [[TMP16:%.*]] = icmp eq <16 x i32> [[TMP9]], zeroinitializer
+; CHECK-NEXT:    [[TMP17:%.*]] = bitcast i16 [[TMP3]] to <16 x i1>
+; CHECK-NEXT:    [[TMP18:%.*]] = bitcast i16 [[X2:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP19:%.*]] = and <16 x i1> [[_MSPROP_ICMP]], [[TMP17]]
+; CHECK-NEXT:    [[TMP20:%.*]] = and <16 x i1> [[TMP16]], [[TMP17]]
+; CHECK-NEXT:    [[TMP21:%.*]] = and <16 x i1> [[_MSPROP_ICMP]], [[TMP18]]
+; CHECK-NEXT:    [[TMP22:%.*]] = or <16 x i1> [[TMP19]], [[TMP20]]
+; CHECK-NEXT:    [[TMP23:%.*]] = or <16 x i1> [[TMP22]], [[TMP21]]
+; CHECK-NEXT:    [[TMP24:%.*]] = and <16 x i1> [[TMP16]], [[TMP18]]
+; CHECK-NEXT:    [[TMP25:%.*]] = bitcast <16 x i1> [[TMP23]] to i16
+; CHECK-NEXT:    [[TMP26:%.*]] = bitcast <16 x i1> [[TMP24]] to i16
+; CHECK-NEXT:    [[TMP27:%.*]] = and <16 x i32> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP28:%.*]] = and <16 x i32> [[X0]], [[TMP2]]
+; CHECK-NEXT:    [[TMP29:%.*]] = and <16 x i32> [[TMP1]], [[X1]]
+; CHECK-NEXT:    [[TMP30:%.*]] = or <16 x i32> [[TMP27]], [[TMP28]]
+; CHECK-NEXT:    [[TMP31:%.*]] = or <16 x i32> [[TMP30]], [[TMP29]]
+; CHECK-NEXT:    [[TMP32:%.*]] = and <16 x i32> [[X0]], [[X1]]
+; CHECK-NEXT:    [[TMP33:%.*]] = xor <16 x i32> [[TMP32]], zeroinitializer
+; CHECK-NEXT:    [[TMP34:%.*]] = or <16 x i32> [[TMP31]], zeroinitializer
+; CHECK-NEXT:    [[TMP35:%.*]] = icmp ne <16 x i32> [[TMP34]], zeroinitializer
+; CHECK-NEXT:    [[TMP36:%.*]] = xor <16 x i32> [[TMP34]], splat (i32 -1)
+; CHECK-NEXT:    [[TMP37:%.*]] = and <16 x i32> [[TMP36]], [[TMP33]]
+; CHECK-NEXT:    [[TMP38:%.*]] = icmp eq <16 x i32> [[TMP37]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_ICMP1:%.*]] = and <16 x i1> [[TMP35]], [[TMP38]]
+; CHECK-NEXT:    [[TMP39:%.*]] = icmp eq <16 x i32> [[TMP32]], zeroinitializer
+; CHECK-NEXT:    [[TMP40:%.*]] = bitcast <16 x i1> [[_MSPROP_ICMP1]] to i16
+; CHECK-NEXT:    [[TMP41:%.*]] = bitcast <16 x i1> [[TMP39]] to i16
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or i16 [[TMP25]], [[TMP40]]
+; CHECK-NEXT:    [[RES2:%.*]] = add i16 [[TMP26]], [[TMP41]]
+; CHECK-NEXT:    store i16 [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i16 [[RES2]]
+;
+  %res = call i16 @llvm.x86.avx512.ptestnm.d.512(<16 x i32> %x0, <16 x i32> %x1, i16 %x2)
+  %res1 = call i16 @llvm.x86.avx512.ptestnm.d.512(<16 x i32> %x0, <16 x i32> %x1, i16-1)
+  %res2 = add i16 %res, %res1
+  ret i16 %res2
+}
+
+declare i8 @llvm.x86.avx512.ptestnm.q.512(<8 x i64>, <8 x i64>, i8 %x2)
+
+define i8@test_int_x86_avx512_ptestnm_q_512(<8 x i64> %x0, <8 x i64> %x1, i8 %x2)  #0 {
+;
+; CHECK-LABEL: @test_int_x86_avx512_ptestnm_q_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = and <8 x i64> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP5:%.*]] = and <8 x i64> [[X0:%.*]], [[TMP2]]
+; CHECK-NEXT:    [[TMP6:%.*]] = and <8 x i64> [[TMP1]], [[X1:%.*]]
+; CHECK-NEXT:    [[TMP7:%.*]] = or <8 x i64> [[TMP4]], [[TMP5]]
+; CHECK-NEXT:    [[TMP8:%.*]] = or <8 x i64> [[TMP7]], [[TMP6]]
+; CHECK-NEXT:    [[TMP9:%.*]] = and <8 x i64> [[X0]], [[X1]]
+; CHECK-NEXT:    [[TMP10:%.*]] = xor <8 x i64> [[TMP9]], zeroinitializer
+; CHECK-NEXT:    [[TMP11:%.*]] = or <8 x i64> [[TMP8]], zeroinitializer
+; CHECK-NEXT:    [[TMP12:%.*]] = icmp ne <8 x i64> [[TMP11]], zeroinitializer
+; CHECK-NEXT:    [[TMP13:%.*]] = xor <8 x i64> [[TMP11]], splat (i64 -1)
+; CHECK-NEXT:    [[TMP14:%.*]] = and <8 x i64> [[TMP13]], [[TMP10]]
+; CHECK-NEXT:    [[TMP15:%.*]] = icmp eq <8 x i64> [[TMP14]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_ICMP:%.*]] = and <8 x i1> [[TMP12]], [[TMP15]]
+; CHECK-NEXT:    [[TMP16:%.*]] = icmp eq <8 x i64> [[TMP9]], zeroinitializer
+; CHECK-NEXT:    [[TMP17:%.*]] = bitcast i8 [[TMP3]] to <8 x i1>
+; CHECK-NEXT:    [[TMP18:%.*]] = bitcast i8 [[X2:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP19:%.*]] = and <8 x i1> [[_MSPROP_ICMP]], [[TMP17]]
+; CHECK-NEXT:    [[TMP20:%.*]] = and <8 x i1> [[TMP16]], [[TMP17]]
+; CHECK-NEXT:    [[TMP21:%.*]] = and <8 x i1> [[_MSPROP_ICMP]], [[TMP18]]
+; CHECK-NEXT:    [[TMP22:%.*]] = or <8 x i1> [[TMP19]], [[TMP20]]
+; CHECK-NEXT:    [[TMP23:%.*]] = or <8 x i1> [[TMP22]], [[TMP21]]
+; CHECK-NEXT:    [[TMP24:%.*]] = and <8 x i1> [[TMP16]], [[TMP18]]
+; CHECK-NEXT:    [[TMP25:%.*]] = bitcast <8 x i1> [[TMP23]] to i8
+; CHECK-NEXT:    [[TMP26:%.*]] = bitcast <8 x i1> [[TMP24]] to i8
+; CHECK-NEXT:    [[TMP27:%.*]] = and <8 x i64> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP28:%.*]] = and <8 x i64> [[X0]], [[TMP2]]
+; CHECK-NEXT:    [[TMP29:%.*]] = and <8 x i64> [[TMP1]], [[X1]]
+; CHECK-NEXT:    [[TMP30:%.*]] = or <8 x i64> [[TMP27]], [[TMP28]]
+; CHECK-NEXT:    [[TMP31:%.*]] = or <8 x i64> [[TMP30]], [[TMP29]]
+; CHECK-NEXT:    [[TMP32:%.*]] = and <8 x i64> [[X0]], [[X1]]
+; CHECK-NEXT:    [[TMP33:%.*]] = xor <8 x i64> [[TMP32]], zeroinitializer
+; CHECK-NEXT:    [[TMP34:%.*]] = or <8 x i64> [[TMP31]], zeroinitializer
+; CHECK-NEXT:    [[TMP35:%.*]] = icmp ne <8 x i64> [[TMP34]], zeroinitializer
+; CHECK-NEXT:    [[TMP36:%.*]] = xor <8 x i64> [[TMP34]], splat (i64 -1)
+; CHECK-NEXT:    [[TMP37:%.*]] = and <8 x i64> [[TMP36]], [[TMP33]]
+; CHECK-NEXT:    [[TMP38:%.*]] = icmp eq <8 x i64> [[TMP37]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_ICMP1:%.*]] = and <8 x i1> [[TMP35]], [[TMP38]]
+; CHECK-NEXT:    [[TMP39:%.*]] = icmp eq <8 x i64> [[TMP32]], zeroinitializer
+; CHECK-NEXT:    [[TMP40:%.*]] = bitcast <8 x i1> [[_MSPROP_ICMP1]] to i8
+; CHECK-NEXT:    [[TMP41:%.*]] = bitcast <8 x i1> [[TMP39]] to i8
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or i8 [[TMP25]], [[TMP40]]
+; CHECK-NEXT:    [[RES2:%.*]] = add i8 [[TMP26]], [[TMP41]]
+; CHECK-NEXT:    store i8 [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i8 [[RES2]]
+;
+  %res = call i8 @llvm.x86.avx512.ptestnm.q.512(<8 x i64> %x0, <8 x i64> %x1, i8 %x2)
+  %res1 = call i8 @llvm.x86.avx512.ptestnm.q.512(<8 x i64> %x0, <8 x i64> %x1, i8-1)
+  %res2 = add i8 %res, %res1
+  ret i8 %res2
+}
+
+declare i16 @llvm.x86.avx512.kand.w(i16, i16) nounwind readnone
+define i16 @test_kand(i16 %a0, i16 %a1)  #0 {
+;
+; CHECK-LABEL: @test_kand(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i16, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast i16 [[TMP1]] to <16 x i1>
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast i16 [[A0:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP5:%.*]] = and <16 x i1> [[TMP3]], zeroinitializer
+; CHECK-NEXT:    [[TMP6:%.*]] = and <16 x i1> [[TMP4]], zeroinitializer
+; CHECK-NEXT:    [[TMP7:%.*]] = and <16 x i1> [[TMP3]], bitcast (<1 x i16> splat (i16 8) to <16 x i1>)
+; CHECK-NEXT:    [[TMP8:%.*]] = or <16 x i1> [[TMP5]], [[TMP6]]
+; CHECK-NEXT:    [[TMP9:%.*]] = or <16 x i1> [[TMP8]], [[TMP7]]
+; CHECK-NEXT:    [[TMP10:%.*]] = and <16 x i1> [[TMP4]], bitcast (<1 x i16> splat (i16 8) to <16 x i1>)
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <16 x i1> [[TMP9]] to i16
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <16 x i1> [[TMP10]] to i16
+; CHECK-NEXT:    [[TMP13:%.*]] = bitcast i16 [[TMP11]] to <16 x i1>
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast i16 [[TMP12]] to <16 x i1>
+; CHECK-NEXT:    [[TMP15:%.*]] = bitcast i16 [[TMP2]] to <16 x i1>
+; CHECK-NEXT:    [[TMP16:%.*]] = bitcast i16 [[A1:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP17:%.*]] = and <16 x i1> [[TMP13]], [[TMP15]]
+; CHECK-NEXT:    [[TMP18:%.*]] = and <16 x i1> [[TMP14]], [[TMP15]]
+; CHECK-NEXT:    [[TMP19:%.*]] = and <16 x i1> [[TMP13]], [[TMP16]]
+; CHECK-NEXT:    [[TMP20:%.*]] = or <16 x i1> [[TMP17]], [[TMP18]]
+; CHECK-NEXT:    [[TMP21:%.*]] = or <16 x i1> [[TMP20]], [[TMP19]]
+; CHECK-NEXT:    [[TMP22:%.*]] = and <16 x i1> [[TMP14]], [[TMP16]]
+; CHECK-NEXT:    [[TMP23:%.*]] = bitcast <16 x i1> [[TMP21]] to i16
+; CHECK-NEXT:    [[TMP24:%.*]] = bitcast <16 x i1> [[TMP22]] to i16
+; CHECK-NEXT:    store i16 [[TMP23]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i16 [[TMP24]]
+;
+  %t1 = call i16 @llvm.x86.avx512.kand.w(i16 %a0, i16 8)
+  %t2 = call i16 @llvm.x86.avx512.kand.w(i16 %t1, i16 %a1)
+  ret i16 %t2
+}
+
+declare i16 @llvm.x86.avx512.kandn.w(i16, i16) nounwind readnone
+define i16 @test_kandn(i16 %a0, i16 %a1)  #0 {
+;
+; CHECK-LABEL: @test_kandn(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i16, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast i16 [[TMP1]] to <16 x i1>
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast i16 [[A0:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <16 x i1> [[TMP3]], zeroinitializer
+; CHECK-NEXT:    [[TMP5:%.*]] = xor <16 x i1> [[TMP4]], splat (i1 true)
+; CHECK-NEXT:    [[TMP6:%.*]] = and <16 x i1> [[_MSPROP]], zeroinitializer
+; CHECK-NEXT:    [[TMP7:%.*]] = and <16 x i1> [[TMP5]], zeroinitializer
+; CHECK-NEXT:    [[TMP8:%.*]] = and <16 x i1> [[_MSPROP]], bitcast (<1 x i16> splat (i16 8) to <16 x i1>)
+; CHECK-NEXT:    [[TMP9:%.*]] = or <16 x i1> [[TMP6]], [[TMP7]]
+; CHECK-NEXT:    [[TMP10:%.*]] = or <16 x i1> [[TMP9]], [[TMP8]]
+; CHECK-NEXT:    [[TMP11:%.*]] = and <16 x i1> [[TMP5]], bitcast (<1 x i16> splat (i16 8) to <16 x i1>)
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <16 x i1> [[TMP10]] to i16
+; CHECK-NEXT:    [[TMP13:%.*]] = bitcast <16 x i1> [[TMP11]] to i16
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast i16 [[TMP12]] to <16 x i1>
+; CHECK-NEXT:    [[TMP15:%.*]] = bitcast i16 [[TMP13]] to <16 x i1>
+; CHECK-NEXT:    [[TMP16:%.*]] = bitcast i16 [[TMP2]] to <16 x i1>
+; CHECK-NEXT:    [[TMP17:%.*]] = bitcast i16 [[A1:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <16 x i1> [[TMP14]], zeroinitializer
+; CHECK-NEXT:    [[TMP18:%.*]] = xor <16 x i1> [[TMP15]], splat (i1 true)
+; CHECK-NEXT:    [[TMP19:%.*]] = and <16 x i1> [[_MSPROP1]], [[TMP16]]
+; CHECK-NEXT:    [[TMP20:%.*]] = and <16 x i1> [[TMP18]], [[TMP16]]
+; CHECK-NEXT:    [[TMP21:%.*]] = and <16 x i1> [[_MSPROP1]], [[TMP17]]
+; CHECK-NEXT:    [[TMP22:%.*]] = or <16 x i1> [[TMP19]], [[TMP20]]
+; CHECK-NEXT:    [[TMP23:%.*]] = or <16 x i1> [[TMP22]], [[TMP21]]
+; CHECK-NEXT:    [[TMP24:%.*]] = and <16 x i1> [[TMP18]], [[TMP17]]
+; CHECK-NEXT:    [[TMP25:%.*]] = bitcast <16 x i1> [[TMP23]] to i16
+; CHECK-NEXT:    [[TMP26:%.*]] = bitcast <16 x i1> [[TMP24]] to i16
+; CHECK-NEXT:    store i16 [[TMP25]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i16 [[TMP26]]
+;
+  %t1 = call i16 @llvm.x86.avx512.kandn.w(i16 %a0, i16 8)
+  %t2 = call i16 @llvm.x86.avx512.kandn.w(i16 %t1, i16 %a1)
+  ret i16 %t2
+}
+
+declare i16 @llvm.x86.avx512.knot.w(i16) nounwind readnone
+define i16 @test_knot(i16 %a0)  #0 {
+;
+; CHECK-LABEL: @test_knot(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i16, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i16 [[TMP1]] to <16 x i1>
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast i16 [[A0:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <16 x i1> [[TMP2]], zeroinitializer
+; CHECK-NEXT:    [[TMP4:%.*]] = xor <16 x i1> [[TMP3]], splat (i1 true)
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <16 x i1> [[_MSPROP]] to i16
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <16 x i1> [[TMP4]] to i16
+; CHECK-NEXT:    store i16 [[TMP5]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i16 [[TMP6]]
+;
+  %res = call i16 @llvm.x86.avx512.knot.w(i16 %a0)
+  ret i16 %res
+}
+
+declare i16 @llvm.x86.avx512.kor.w(i16, i16) nounwind readnone
+define i16 @test_kor(i16 %a0, i16 %a1)  #0 {
+;
+; CHECK-LABEL: @test_kor(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i16, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast i16 [[TMP1]] to <16 x i1>
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast i16 [[A0:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP5:%.*]] = xor <16 x i1> [[TMP4]], splat (i1 true)
+; CHECK-NEXT:    [[TMP6:%.*]] = and <16 x i1> [[TMP3]], zeroinitializer
+; CHECK-NEXT:    [[TMP7:%.*]] = and <16 x i1> [[TMP5]], zeroinitializer
+; CHECK-NEXT:    [[TMP8:%.*]] = and <16 x i1> [[TMP3]], <i1 xor (i1 extractelement (<16 x i1> bitcast (<1 x i16> splat (i16 8) to <16 x i1>), i32 0), i1 true), i1 xor (i1 extractelement (<16 x i1> bitcast (<1 x i16> splat (i16 8) to <16 x i1>), i32 1), i1 true), i1 xor (i1 extractelement (<16 x i1> bitcast (<1 x i16> splat (i16 8) to <16 x i1>), i32 2), i1 true), i1 xor (i1 extractelement (<16 x i1> bitcast (<1 x i16> splat (i16 8) to <16 x i1>), i32 3), i1 true), i1 xor (i1 extractelement (<16 x i1> bitcast (<1 x i16> splat (i16 8) to <16 x i1>), i32 4), i1 true), i1 xor (i1 extractelement (<16 x i1> bitcast (<1 x i16> splat (i16 8) to <16 x i1>), i32 5), i1 true), i1 xor (i1 extractelement (<16 x i1> bitcast (<1 x i16> splat (i16 8) to <16 x i1>), i32 6), i1 true), i1 xor (i1 extractelement (<16 x i1> bitcast (<1 x i16> splat (i16 8) to <16 x i1>), i32 7), i1 true), i1 xor (i1 extractelement (<16 x i1> bitcast (<1 x i16> splat (i16 8) to <16 x i1>), i32 8), i1 true), i1 xor (i1 extractelement (<16 x i1> bitcast (<1 x i16> splat (i16 8) to <16 x i1>), i32 9), i1 true), i1 xor (i1 extractelement (<16 x i1> bitcast (<1 x i16> splat (i16 8) to <16 x i1>), i32 10), i1 true), i1 xor (i1 extractelement (<16 x i1> bitcast (<1 x i16> splat (i16 8) to <16 x i1>), i32 11), i1 true), i1 xor (i1 extractelement (<16 x i1> bitcast (<1 x i16> splat (i16 8) to <16 x i1>), i32 12), i1 true), i1 xor (i1 extractelement (<16 x i1> bitcast (<1 x i16> splat (i16 8) to <16 x i1>), i32 13), i1 true), i1 xor (i1 extractelement (<16 x i1> bitcast (<1 x i16> splat (i16 8) to <16 x i1>), i32 14), i1 true), i1 xor (i1 extractelement (<16 x i1> bitcast (<1 x i16> splat (i16 8) to <16 x i1>), i32 15), i1 true)>
+; CHECK-NEXT:    [[TMP9:%.*]] = or <16 x i1> [[TMP6]], [[TMP7]]
+; CHECK-NEXT:    [[TMP10:%.*]] = or <16 x i1> [[TMP9]], [[TMP8]]
+; CHECK-NEXT:    [[TMP11:%.*]] = or <16 x i1> [[TMP4]], bitcast (<1 x i16> splat (i16 8) to <16 x i1>)
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <16 x i1> [[TMP10]] to i16
+; CHECK-NEXT:    [[TMP13:%.*]] = bitcast <16 x i1> [[TMP11]] to i16
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast i16 [[TMP12]] to <16 x i1>
+; CHECK-NEXT:    [[TMP15:%.*]] = bitcast i16 [[TMP13]] to <16 x i1>
+; CHECK-NEXT:    [[TMP16:%.*]] = bitcast i16 [[TMP2]] to <16 x i1>
+; CHECK-NEXT:    [[TMP17:%.*]] = bitcast i16 [[A1:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP18:%.*]] = xor <16 x i1> [[TMP15]], splat (i1 true)
+; CHECK-NEXT:    [[TMP19:%.*]] = xor <16 x i1> [[TMP17]], splat (i1 true)
+; CHECK-NEXT:    [[TMP20:%.*]] = and <16 x i1> [[TMP14]], [[TMP16]]
+; CHECK-NEXT:    [[TMP21:%.*]] = and <16 x i1> [[TMP18]], [[TMP16]]
+; CHECK-NEXT:    [[TMP22:%.*]] = and <16 x i1> [[TMP14]], [[TMP19]]
+; CHECK-NEXT:    [[TMP23:%.*]] = or <16 x i1> [[TMP20]], [[TMP21]]
+; CHECK-NEXT:    [[TMP24:%.*]] = or <16 x i1> [[TMP23]], [[TMP22]]
+; CHECK-NEXT:    [[TMP25:%.*]] = or <16 x i1> [[TMP15]], [[TMP17]]
+; CHECK-NEXT:    [[TMP26:%.*]] = bitcast <16 x i1> [[TMP24]] to i16
+; CHECK-NEXT:    [[TMP27:%.*]] = bitcast <16 x i1> [[TMP25]] to i16
+; CHECK-NEXT:    store i16 [[TMP26]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i16 [[TMP27]]
+;
+  %t1 = call i16 @llvm.x86.avx512.kor.w(i16 %a0, i16 8)
+  %t2 = call i16 @llvm.x86.avx512.kor.w(i16 %t1, i16 %a1)
+  ret i16 %t2
+}
+
+declare i16 @llvm.x86.avx512.kxnor.w(i16, i16) nounwind readnone
+; TODO: the two kxnor instructions here a no op and should be elimintaed,
+; probably by FoldConstantArithmetic in SelectionDAG.
+define i16 @test_kxnor(i16 %a0, i16 %a1)  #0 {
+;
+; CHECK-LABEL: @test_kxnor(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i16, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast i16 [[TMP1]] to <16 x i1>
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast i16 [[A0:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <16 x i1> [[TMP3]], zeroinitializer
+; CHECK-NEXT:    [[TMP5:%.*]] = xor <16 x i1> [[TMP4]], splat (i1 true)
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <16 x i1> [[_MSPROP]], zeroinitializer
+; CHECK-NEXT:    [[TMP6:%.*]] = xor <16 x i1> [[TMP5]], bitcast (<1 x i16> splat (i16 8) to <16 x i1>)
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <16 x i1> [[_MSPROP1]] to i16
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <16 x i1> [[TMP6]] to i16
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast i16 [[TMP7]] to <16 x i1>
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i16 [[TMP8]] to <16 x i1>
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast i16 [[TMP2]] to <16 x i1>
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast i16 [[A1:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = or <16 x i1> [[TMP9]], zeroinitializer
+; CHECK-NEXT:    [[TMP13:%.*]] = xor <16 x i1> [[TMP10]], splat (i1 true)
+; CHECK-NEXT:    [[_MSPROP3:%.*]] = or <16 x i1> [[_MSPROP2]], [[TMP11]]
+; CHECK-NEXT:    [[TMP14:%.*]] = xor <16 x i1> [[TMP13]], [[TMP12]]
+; CHECK-NEXT:    [[TMP15:%.*]] = bitcast <16 x i1> [[_MSPROP3]] to i16
+; CHECK-NEXT:    [[TMP16:%.*]] = bitcast <16 x i1> [[TMP14]] to i16
+; CHECK-NEXT:    store i16 [[TMP15]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i16 [[TMP16]]
+;
+  %t1 = call i16 @llvm.x86.avx512.kxnor.w(i16 %a0, i16 8)
+  %t2 = call i16 @llvm.x86.avx512.kxnor.w(i16 %t1, i16 %a1)
+  ret i16 %t2
+}
+
+declare i16 @llvm.x86.avx512.kxor.w(i16, i16) nounwind readnone
+define i16 @test_kxor(i16 %a0, i16 %a1)  #0 {
+;
+; CHECK-LABEL: @test_kxor(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i16, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast i16 [[TMP1]] to <16 x i1>
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast i16 [[A0:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <16 x i1> [[TMP3]], zeroinitializer
+; CHECK-NEXT:    [[TMP5:%.*]] = xor <16 x i1> [[TMP4]], bitcast (<1 x i16> splat (i16 8) to <16 x i1>)
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <16 x i1> [[_MSPROP]] to i16
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <16 x i1> [[TMP5]] to i16
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast i16 [[TMP6]] to <16 x i1>
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast i16 [[TMP7]] to <16 x i1>
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i16 [[TMP2]] to <16 x i1>
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast i16 [[A1:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <16 x i1> [[TMP8]], [[TMP10]]
+; CHECK-NEXT:    [[TMP12:%.*]] = xor <16 x i1> [[TMP9]], [[TMP11]]
+; CHECK-NEXT:    [[TMP13:%.*]] = bitcast <16 x i1> [[_MSPROP1]] to i16
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast <16 x i1> [[TMP12]] to i16
+; CHECK-NEXT:    store i16 [[TMP13]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i16 [[TMP14]]
+;
+  %t1 = call i16 @llvm.x86.avx512.kxor.w(i16 %a0, i16 8)
+  %t2 = call i16 @llvm.x86.avx512.kxor.w(i16 %t1, i16 %a1)
+  ret i16 %t2
+}
+
+declare i32 @llvm.x86.avx512.kortestz.w(i16, i16) nounwind readnone
+define i32 @test_kortestz(<8 x i64> %A, <8 x i64> %B, <8 x i64> %C, <8 x i64> %D)  #0 {
+; CHECK-LABEL: @test_kortestz(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <8 x i64> [[TMP0]] to <16 x i32>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <8 x i64> [[A:%.*]] to <16 x i32>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <8 x i64> [[TMP1]] to <16 x i32>
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <8 x i64> [[B:%.*]] to <16 x i32>
+; CHECK-NEXT:    [[TMP8:%.*]] = xor <16 x i32> [[TMP5]], [[TMP7]]
+; CHECK-NEXT:    [[TMP9:%.*]] = or <16 x i32> [[TMP4]], [[TMP6]]
+; CHECK-NEXT:    [[TMP10:%.*]] = icmp ne <16 x i32> [[TMP9]], zeroinitializer
+; CHECK-NEXT:    [[TMP11:%.*]] = xor <16 x i32> [[TMP9]], splat (i32 -1)
+; CHECK-NEXT:    [[TMP12:%.*]] = and <16 x i32> [[TMP11]], [[TMP8]]
+; CHECK-NEXT:    [[TMP13:%.*]] = icmp eq <16 x i32> [[TMP12]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_ICMP:%.*]] = and <16 x i1> [[TMP10]], [[TMP13]]
+; CHECK-NEXT:    [[TMP14:%.*]] = icmp ne <16 x i32> [[TMP5]], [[TMP7]]
+; CHECK-NEXT:    [[TMP15:%.*]] = bitcast <8 x i64> [[TMP2]] to <16 x i32>
+; CHECK-NEXT:    [[TMP16:%.*]] = bitcast <8 x i64> [[C:%.*]] to <16 x i32>
+; CHECK-NEXT:    [[TMP17:%.*]] = bitcast <8 x i64> [[TMP3]] to <16 x i32>
+; CHECK-NEXT:    [[TMP18:%.*]] = bitcast <8 x i64> [[D:%.*]] to <16 x i32>
+; CHECK-NEXT:    [[TMP19:%.*]] = xor <16 x i32> [[TMP16]], [[TMP18]]
+; CHECK-NEXT:    [[TMP20:%.*]] = or <16 x i32> [[TMP15]], [[TMP17]]
+; CHECK-NEXT:    [[TMP21:%.*]] = icmp ne <16 x i32> [[TMP20]], zeroinitializer
+; CHECK-NEXT:    [[TMP22:%.*]] = xor <16 x i32> [[TMP20]], splat (i32 -1)
+; CHECK-NEXT:    [[TMP23:%.*]] = and <16 x i32> [[TMP22]], [[TMP19]]
+; CHECK-NEXT:    [[TMP24:%.*]] = icmp eq <16 x i32> [[TMP23]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_ICMP1:%.*]] = and <16 x i1> [[TMP21]], [[TMP24]]
+; CHECK-NEXT:    [[TMP25:%.*]] = icmp ne <16 x i32> [[TMP16]], [[TMP18]]
+; CHECK-NEXT:    [[TMP26:%.*]] = bitcast <16 x i1> [[_MSPROP_ICMP]] to i16
+; CHECK-NEXT:    [[TMP27:%.*]] = bitcast <16 x i1> [[TMP14]] to i16
+; CHECK-NEXT:    [[TMP28:%.*]] = bitcast <16 x i1> [[_MSPROP_ICMP1]] to i16
+; CHECK-NEXT:    [[TMP29:%.*]] = bitcast <16 x i1> [[TMP25]] to i16
+; CHECK-NEXT:    [[TMP30:%.*]] = bitcast i16 [[TMP26]] to <16 x i1>
+; CHECK-NEXT:    [[TMP31:%.*]] = bitcast i16 [[TMP27]] to <16 x i1>
+; CHECK-NEXT:    [[TMP32:%.*]] = bitcast i16 [[TMP28]] to <16 x i1>
+; CHECK-NEXT:    [[TMP33:%.*]] = bitcast i16 [[TMP29]] to <16 x i1>
+; CHECK-NEXT:    [[TMP34:%.*]] = xor <16 x i1> [[TMP31]], splat (i1 true)
+; CHECK-NEXT:    [[TMP35:%.*]] = xor <16 x i1> [[TMP33]], splat (i1 true)
+; CHECK-NEXT:    [[TMP36:%.*]] = and <16 x i1> [[TMP30]], [[TMP32]]
+; CHECK-NEXT:    [[TMP37:%.*]] = and <16 x i1> [[TMP34]], [[TMP32]]
+; CHECK-NEXT:    [[TMP38:%.*]] = and <16 x i1> [[TMP30]], [[TMP35]]
+; CHECK-NEXT:    [[TMP39:%.*]] = or <16 x i1> [[TMP36]], [[TMP37]]
+; CHECK-NEXT:    [[TMP40:%.*]] = or <16 x i1> [[TMP39]], [[TMP38]]
+; CHECK-NEXT:    [[TMP41:%.*]] = or <16 x i1> [[TMP31]], [[TMP33]]
+; CHECK-NEXT:    [[TMP42:%.*]] = bitcast <16 x i1> [[TMP40]] to i16
+; CHECK-NEXT:    [[TMP43:%.*]] = bitcast <16 x i1> [[TMP41]] to i16
+; CHECK-NEXT:    [[TMP44:%.*]] = xor i16 [[TMP43]], 0
+; CHECK-NEXT:    [[TMP45:%.*]] = or i16 [[TMP42]], 0
+; CHECK-NEXT:    [[TMP46:%.*]] = icmp ne i16 [[TMP45]], 0
+; CHECK-NEXT:    [[TMP47:%.*]] = xor i16 [[TMP45]], -1
+; CHECK-NEXT:    [[TMP48:%.*]] = and i16 [[TMP47]], [[TMP44]]
+; CHECK-NEXT:    [[TMP49:%.*]] = icmp eq i16 [[TMP48]], 0
+; CHECK-NEXT:    [[_MSPROP_ICMP2:%.*]] = and i1 [[TMP46]], [[TMP49]]
+; CHECK-NEXT:    [[TMP50:%.*]] = icmp eq i16 [[TMP43]], 0
+; CHECK-NEXT:    [[_MSPROP:%.*]] = zext i1 [[_MSPROP_ICMP2]] to i32
+; CHECK-NEXT:    [[TMP51:%.*]] = zext i1 [[TMP50]] to i32
+; CHECK-NEXT:    store i32 [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i32 [[TMP51]]
+;
+entry:
+  %0 = bitcast <8 x i64> %A to <16 x i32>
+  %1 = bitcast <8 x i64> %B to <16 x i32>
+  %2 = icmp ne <16 x i32> %0, %1
+  %3 = bitcast <8 x i64> %C to <16 x i32>
+  %4 = bitcast <8 x i64> %D to <16 x i32>
+  %5 = icmp ne <16 x i32> %3, %4
+  %6 = bitcast <16 x i1> %2 to i16
+  %7 = bitcast <16 x i1> %5 to i16
+  %res = call i32 @llvm.x86.avx512.kortestz.w(i16 %6, i16 %7)
+  ret i32 %res
+}
+
+declare i32 @llvm.x86.avx512.kortestc.w(i16, i16) nounwind readnone
+define i32 @test_kortestc(<8 x i64> %A, <8 x i64> %B, <8 x i64> %C, <8 x i64> %D)  #0 {
+; CHECK-LABEL: @test_kortestc(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <8 x i64> [[TMP0]] to <16 x i32>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <8 x i64> [[A:%.*]] to <16 x i32>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <8 x i64> [[TMP1]] to <16 x i32>
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <8 x i64> [[B:%.*]] to <16 x i32>
+; CHECK-NEXT:    [[TMP8:%.*]] = xor <16 x i32> [[TMP5]], [[TMP7]]
+; CHECK-NEXT:    [[TMP9:%.*]] = or <16 x i32> [[TMP4]], [[TMP6]]
+; CHECK-NEXT:    [[TMP10:%.*]] = icmp ne <16 x i32> [[TMP9]], zeroinitializer
+; CHECK-NEXT:    [[TMP11:%.*]] = xor <16 x i32> [[TMP9]], splat (i32 -1)
+; CHECK-NEXT:    [[TMP12:%.*]] = and <16 x i32> [[TMP11]], [[TMP8]]
+; CHECK-NEXT:    [[TMP13:%.*]] = icmp eq <16 x i32> [[TMP12]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_ICMP:%.*]] = and <16 x i1> [[TMP10]], [[TMP13]]
+; CHECK-NEXT:    [[TMP14:%.*]] = icmp ne <16 x i32> [[TMP5]], [[TMP7]]
+; CHECK-NEXT:    [[TMP15:%.*]] = bitcast <8 x i64> [[TMP2]] to <16 x i32>
+; CHECK-NEXT:    [[TMP16:%.*]] = bitcast <8 x i64> [[C:%.*]] to <16 x i32>
+; CHECK-NEXT:    [[TMP17:%.*]] = bitcast <8 x i64> [[TMP3]] to <16 x i32>
+; CHECK-NEXT:    [[TMP18:%.*]] = bitcast <8 x i64> [[D:%.*]] to <16 x i32>
+; CHECK-NEXT:    [[TMP19:%.*]] = xor <16 x i32> [[TMP16]], [[TMP18]]
+; CHECK-NEXT:    [[TMP20:%.*]] = or <16 x i32> [[TMP15]], [[TMP17]]
+; CHECK-NEXT:    [[TMP21:%.*]] = icmp ne <16 x i32> [[TMP20]], zeroinitializer
+; CHECK-NEXT:    [[TMP22:%.*]] = xor <16 x i32> [[TMP20]], splat (i32 -1)
+; CHECK-NEXT:    [[TMP23:%.*]] = and <16 x i32> [[TMP22]], [[TMP19]]
+; CHECK-NEXT:    [[TMP24:%.*]] = icmp eq <16 x i32> [[TMP23]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_ICMP1:%.*]] = and <16 x i1> [[TMP21]], [[TMP24]]
+; CHECK-NEXT:    [[TMP25:%.*]] = icmp ne <16 x i32> [[TMP16]], [[TMP18]]
+; CHECK-NEXT:    [[TMP26:%.*]] = bitcast <16 x i1> [[_MSPROP_ICMP]] to i16
+; CHECK-NEXT:    [[TMP27:%.*]] = bitcast <16 x i1> [[TMP14]] to i16
+; CHECK-NEXT:    [[TMP28:%.*]] = bitcast <16 x i1> [[_MSPROP_ICMP1]] to i16
+; CHECK-NEXT:    [[TMP29:%.*]] = bitcast <16 x i1> [[TMP25]] to i16
+; CHECK-NEXT:    [[TMP30:%.*]] = bitcast i16 [[TMP26]] to <16 x i1>
+; CHECK-NEXT:    [[TMP31:%.*]] = bitcast i16 [[TMP27]] to <16 x i1>
+; CHECK-NEXT:    [[TMP32:%.*]] = bitcast i16 [[TMP28]] to <16 x i1>
+; CHECK-NEXT:    [[TMP33:%.*]] = bitcast i16 [[TMP29]] to <16 x i1>
+; CHECK-NEXT:    [[TMP34:%.*]] = xor <16 x i1> [[TMP31]], splat (i1 true)
+; CHECK-NEXT:    [[TMP35:%.*]] = xor <16 x i1> [[TMP33]], splat (i1 true)
+; CHECK-NEXT:    [[TMP36:%.*]] = and <16 x i1> [[TMP30]], [[TMP32]]
+; CHECK-NEXT:    [[TMP37:%.*]] = and <16 x i1> [[TMP34]], [[TMP32]]
+; CHECK-NEXT:    [[TMP38:%.*]] = and <16 x i1> [[TMP30]], [[TMP35]]
+; CHECK-NEXT:    [[TMP39:%.*]] = or <16 x i1> [[TMP36]], [[TMP37]]
+; CHECK-NEXT:    [[TMP40:%.*]] = or <16 x i1> [[TMP39]], [[TMP38]]
+; CHECK-NEXT:    [[TMP41:%.*]] = or <16 x i1> [[TMP31]], [[TMP33]]
+; CHECK-NEXT:    [[TMP42:%.*]] = bitcast <16 x i1> [[TMP40]] to i16
+; CHECK-NEXT:    [[TMP43:%.*]] = bitcast <16 x i1> [[TMP41]] to i16
+; CHECK-NEXT:    [[TMP44:%.*]] = xor i16 [[TMP43]], 0
+; CHECK-NEXT:    [[TMP45:%.*]] = or i16 [[TMP42]], 0
+; CHECK-NEXT:    [[TMP46:%.*]] = icmp ne i16 [[TMP45]], 0
+; CHECK-NEXT:    [[TMP47:%.*]] = xor i16 [[TMP45]], -1
+; CHECK-NEXT:    [[TMP48:%.*]] = and i16 [[TMP47]], [[TMP44]]
+; CHECK-NEXT:    [[TMP49:%.*]] = icmp eq i16 [[TMP48]], 0
+; CHECK-NEXT:    [[_MSPROP_ICMP2:%.*]] = and i1 [[TMP46]], [[TMP49]]
+; CHECK-NEXT:    [[TMP50:%.*]] = icmp eq i16 [[TMP43]], 0
+; CHECK-NEXT:    [[_MSPROP:%.*]] = zext i1 [[_MSPROP_ICMP2]] to i32
+; CHECK-NEXT:    [[TMP51:%.*]] = zext i1 [[TMP50]] to i32
+; CHECK-NEXT:    store i32 [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i32 [[TMP51]]
+;
+entry:
+  %0 = bitcast <8 x i64> %A to <16 x i32>
+  %1 = bitcast <8 x i64> %B to <16 x i32>
+  %2 = icmp ne <16 x i32> %0, %1
+  %3 = bitcast <8 x i64> %C to <16 x i32>
+  %4 = bitcast <8 x i64> %D to <16 x i32>
+  %5 = icmp ne <16 x i32> %3, %4
+  %6 = bitcast <16 x i1> %2 to i16
+  %7 = bitcast <16 x i1> %5 to i16
+  %res = call i32 @llvm.x86.avx512.kortestz.w(i16 %6, i16 %7)
+  ret i32 %res
+}
+
+define i16 @test_cmpps(<16 x float> %a, <16 x float> %b)  #0 {
+; CHECK-LABEL: @test_cmpps(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP3]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP5:%.*]], label [[TMP6:%.*]], !prof [[PROF1]]
+; CHECK:       5:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       6:
+; CHECK-NEXT:    [[RES:%.*]] = call <16 x i1> @llvm.x86.avx512.mask.cmp.ps.512(<16 x float> [[A:%.*]], <16 x float> [[B:%.*]], i32 2, <16 x i1> splat (i1 true), i32 8)
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <16 x i1> [[RES]] to i16
+; CHECK-NEXT:    store i16 0, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i16 [[TMP7]]
+;
+  %res = call i16 @llvm.x86.avx512.mask.cmp.ps.512(<16 x float> %a, <16 x float> %b, i32 2, i16 -1, i32 8)
+  ret i16 %res
+}
+declare i16 @llvm.x86.avx512.mask.cmp.ps.512(<16 x float> , <16 x float> , i32, i16, i32)
+
+define i8 @test_cmppd(<8 x double> %a, <8 x double> %b)  #0 {
+; CHECK-LABEL: @test_cmppd(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <8 x i64> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP3]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <8 x i64> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP5:%.*]], label [[TMP6:%.*]], !prof [[PROF1]]
+; CHECK:       5:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       6:
+; CHECK-NEXT:    [[RES:%.*]] = call <8 x i1> @llvm.x86.avx512.mask.cmp.pd.512(<8 x double> [[A:%.*]], <8 x double> [[B:%.*]], i32 4, <8 x i1> splat (i1 true), i32 4)
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <8 x i1> [[RES]] to i8
+; CHECK-NEXT:    store i8 0, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i8 [[TMP7]]
+;
+  %res = call i8 @llvm.x86.avx512.mask.cmp.pd.512(<8 x double> %a, <8 x double> %b, i32 4, i8 -1, i32 4)
+  ret i8 %res
+}
+declare i8 @llvm.x86.avx512.mask.cmp.pd.512(<8 x double> , <8 x double> , i32, i8, i32)
+
+define <8 x i64> @test_mul_epi32_rr(<16 x i32> %a, <16 x i32> %b)  #0 {
+; CHECK-LABEL: @test_mul_epi32_rr(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <16 x i32> [[TMP1]] to <8 x i64>
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <16 x i32> [[A:%.*]] to <8 x i64>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <16 x i32> [[TMP2]] to <8 x i64>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <16 x i32> [[B:%.*]] to <8 x i64>
+; CHECK-NEXT:    [[TMP7:%.*]] = shl <8 x i64> [[TMP3]], splat (i64 32)
+; CHECK-NEXT:    [[TMP8:%.*]] = or <8 x i64> [[TMP7]], zeroinitializer
+; CHECK-NEXT:    [[TMP9:%.*]] = shl <8 x i64> [[TMP4]], splat (i64 32)
+; CHECK-NEXT:    [[TMP10:%.*]] = ashr <8 x i64> [[TMP8]], splat (i64 32)
+; CHECK-NEXT:    [[TMP11:%.*]] = or <8 x i64> [[TMP10]], zeroinitializer
+; CHECK-NEXT:    [[TMP12:%.*]] = ashr <8 x i64> [[TMP9]], splat (i64 32)
+; CHECK-NEXT:    [[TMP13:%.*]] = shl <8 x i64> [[TMP5]], splat (i64 32)
+; CHECK-NEXT:    [[TMP14:%.*]] = or <8 x i64> [[TMP13]], zeroinitializer
+; CHECK-NEXT:    [[TMP15:%.*]] = shl <8 x i64> [[TMP6]], splat (i64 32)
+; CHECK-NEXT:    [[TMP16:%.*]] = ashr <8 x i64> [[TMP14]], splat (i64 32)
+; CHECK-NEXT:    [[TMP17:%.*]] = or <8 x i64> [[TMP16]], zeroinitializer
+; CHECK-NEXT:    [[TMP18:%.*]] = ashr <8 x i64> [[TMP15]], splat (i64 32)
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <8 x i64> [[TMP11]], [[TMP17]]
+; CHECK-NEXT:    [[TMP19:%.*]] = mul <8 x i64> [[TMP12]], [[TMP18]]
+; CHECK-NEXT:    store <8 x i64> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i64> [[TMP19]]
+;
+  %res = call <8 x i64> @llvm.x86.avx512.pmul.dq.512(<16 x i32> %a, <16 x i32> %b)
+  ret < 8 x i64> %res
+}
+
+define <8 x i64> @test_mul_epi32_rrk(<16 x i32> %a, <16 x i32> %b, <8 x i64> %passThru, i8 %mask)  #0 {
+;
+; CHECK-LABEL: @test_mul_epi32_rrk(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <16 x i32> [[TMP1]] to <8 x i64>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <16 x i32> [[A:%.*]] to <8 x i64>
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <16 x i32> [[TMP2]] to <8 x i64>
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <16 x i32> [[B:%.*]] to <8 x i64>
+; CHECK-NEXT:    [[TMP9:%.*]] = shl <8 x i64> [[TMP5]], splat (i64 32)
+; CHECK-NEXT:    [[TMP10:%.*]] = or <8 x i64> [[TMP9]], zeroinitializer
+; CHECK-NEXT:    [[TMP11:%.*]] = shl <8 x i64> [[TMP6]], splat (i64 32)
+; CHECK-NEXT:    [[TMP12:%.*]] = ashr <8 x i64> [[TMP10]], splat (i64 32)
+; CHECK-NEXT:    [[TMP13:%.*]] = or <8 x i64> [[TMP12]], zeroinitializer
+; CHECK-NEXT:    [[TMP14:%.*]] = ashr <8 x i64> [[TMP11]], splat (i64 32)
+; CHECK-NEXT:    [[TMP15:%.*]] = shl <8 x i64> [[TMP7]], splat (i64 32)
+; CHECK-NEXT:    [[TMP16:%.*]] = or <8 x i64> [[TMP15]], zeroinitializer
+; CHECK-NEXT:    [[TMP17:%.*]] = shl <8 x i64> [[TMP8]], splat (i64 32)
+; CHECK-NEXT:    [[TMP18:%.*]] = ashr <8 x i64> [[TMP16]], splat (i64 32)
+; CHECK-NEXT:    [[TMP19:%.*]] = or <8 x i64> [[TMP18]], zeroinitializer
+; CHECK-NEXT:    [[TMP20:%.*]] = ashr <8 x i64> [[TMP17]], splat (i64 32)
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <8 x i64> [[TMP13]], [[TMP19]]
+; CHECK-NEXT:    [[TMP21:%.*]] = mul <8 x i64> [[TMP14]], [[TMP20]]
+; CHECK-NEXT:    [[TMP22:%.*]] = bitcast i8 [[TMP3]] to <8 x i1>
+; CHECK-NEXT:    [[MASK_CAST:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP23:%.*]] = select <8 x i1> [[MASK_CAST]], <8 x i64> [[_MSPROP]], <8 x i64> [[TMP4]]
+; CHECK-NEXT:    [[TMP24:%.*]] = xor <8 x i64> [[TMP21]], [[PASSTHRU:%.*]]
+; CHECK-NEXT:    [[TMP25:%.*]] = or <8 x i64> [[TMP24]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP26:%.*]] = or <8 x i64> [[TMP25]], [[TMP4]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP22]], <8 x i64> [[TMP26]], <8 x i64> [[TMP23]]
+; CHECK-NEXT:    [[RES:%.*]] = select <8 x i1> [[MASK_CAST]], <8 x i64> [[TMP21]], <8 x i64> [[PASSTHRU]]
+; CHECK-NEXT:    store <8 x i64> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i64> [[RES]]
+;
+  %mul = call <8 x i64> @llvm.x86.avx512.pmul.dq.512(<16 x i32> %a, <16 x i32> %b)
+  %mask.cast = bitcast i8 %mask to <8 x i1>
+  %res = select <8 x i1> %mask.cast, <8 x i64> %mul, <8 x i64> %passThru
+  ret < 8 x i64> %res
+}
+
+define <8 x i64> @test_mul_epi32_rrkz(<16 x i32> %a, <16 x i32> %b, i8 %mask)  #0 {
+;
+; CHECK-LABEL: @test_mul_epi32_rrkz(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <16 x i32> [[TMP1]] to <8 x i64>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <16 x i32> [[A:%.*]] to <8 x i64>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <16 x i32> [[TMP2]] to <8 x i64>
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <16 x i32> [[B:%.*]] to <8 x i64>
+; CHECK-NEXT:    [[TMP8:%.*]] = shl <8 x i64> [[TMP4]], splat (i64 32)
+; CHECK-NEXT:    [[TMP9:%.*]] = or <8 x i64> [[TMP8]], zeroinitializer
+; CHECK-NEXT:    [[TMP10:%.*]] = shl <8 x i64> [[TMP5]], splat (i64 32)
+; CHECK-NEXT:    [[TMP11:%.*]] = ashr <8 x i64> [[TMP9]], splat (i64 32)
+; CHECK-NEXT:    [[TMP12:%.*]] = or <8 x i64> [[TMP11]], zeroinitializer
+; CHECK-NEXT:    [[TMP13:%.*]] = ashr <8 x i64> [[TMP10]], splat (i64 32)
+; CHECK-NEXT:    [[TMP14:%.*]] = shl <8 x i64> [[TMP6]], splat (i64 32)
+; CHECK-NEXT:    [[TMP15:%.*]] = or <8 x i64> [[TMP14]], zeroinitializer
+; CHECK-NEXT:    [[TMP16:%.*]] = shl <8 x i64> [[TMP7]], splat (i64 32)
+; CHECK-NEXT:    [[TMP17:%.*]] = ashr <8 x i64> [[TMP15]], splat (i64 32)
+; CHECK-NEXT:    [[TMP18:%.*]] = or <8 x i64> [[TMP17]], zeroinitializer
+; CHECK-NEXT:    [[TMP19:%.*]] = ashr <8 x i64> [[TMP16]], splat (i64 32)
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <8 x i64> [[TMP12]], [[TMP18]]
+; CHECK-NEXT:    [[TMP20:%.*]] = mul <8 x i64> [[TMP13]], [[TMP19]]
+; CHECK-NEXT:    [[TMP21:%.*]] = bitcast i8 [[TMP3]] to <8 x i1>
+; CHECK-NEXT:    [[MASK_CAST:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP22:%.*]] = select <8 x i1> [[MASK_CAST]], <8 x i64> [[_MSPROP]], <8 x i64> zeroinitializer
+; CHECK-NEXT:    [[TMP23:%.*]] = xor <8 x i64> [[TMP20]], zeroinitializer
+; CHECK-NEXT:    [[TMP24:%.*]] = or <8 x i64> [[TMP23]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP25:%.*]] = or <8 x i64> [[TMP24]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP21]], <8 x i64> [[TMP25]], <8 x i64> [[TMP22]]
+; CHECK-NEXT:    [[RES:%.*]] = select <8 x i1> [[MASK_CAST]], <8 x i64> [[TMP20]], <8 x i64> zeroinitializer
+; CHECK-NEXT:    store <8 x i64> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i64> [[RES]]
+;
+  %mul = call <8 x i64> @llvm.x86.avx512.pmul.dq.512(<16 x i32> %a, <16 x i32> %b)
+  %mask.cast = bitcast i8 %mask to <8 x i1>
+  %res = select <8 x i1> %mask.cast, <8 x i64> %mul, <8 x i64> zeroinitializer
+  ret < 8 x i64> %res
+}
+
+define <8 x i64> @test_mul_epi32_rm(<16 x i32> %a, ptr %ptr_b)  #0 {
+;
+; CHECK-LABEL: @test_mul_epi32_rm(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP3:%.*]], label [[TMP4:%.*]], !prof [[PROF1]]
+; CHECK:       3:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       4:
+; CHECK-NEXT:    [[B:%.*]] = load <16 x i32>, ptr [[PTR_B:%.*]], align 64
+; CHECK-NEXT:    [[TMP5:%.*]] = ptrtoint ptr [[PTR_B]] to i64
+; CHECK-NEXT:    [[TMP6:%.*]] = xor i64 [[TMP5]], 87960930222080
+; CHECK-NEXT:    [[TMP7:%.*]] = inttoptr i64 [[TMP6]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <16 x i32>, ptr [[TMP7]], align 64
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <16 x i32> [[TMP2]] to <8 x i64>
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <16 x i32> [[A:%.*]] to <8 x i64>
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast <16 x i32> [[_MSLD]] to <8 x i64>
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <16 x i32> [[B]] to <8 x i64>
+; CHECK-NEXT:    [[TMP12:%.*]] = shl <8 x i64> [[TMP8]], splat (i64 32)
+; CHECK-NEXT:    [[TMP13:%.*]] = or <8 x i64> [[TMP12]], zeroinitializer
+; CHECK-NEXT:    [[TMP14:%.*]] = shl <8 x i64> [[TMP9]], splat (i64 32)
+; CHECK-NEXT:    [[TMP15:%.*]] = ashr <8 x i64> [[TMP13]], splat (i64 32)
+; CHECK-NEXT:    [[TMP16:%.*]] = or <8 x i64> [[TMP15]], zeroinitializer
+; CHECK-NEXT:    [[TMP17:%.*]] = ashr <8 x i64> [[TMP14]], splat (i64 32)
+; CHECK-NEXT:    [[TMP18:%.*]] = shl <8 x i64> [[TMP10]], splat (i64 32)
+; CHECK-NEXT:    [[TMP19:%.*]] = or <8 x i64> [[TMP18]], zeroinitializer
+; CHECK-NEXT:    [[TMP20:%.*]] = shl <8 x i64> [[TMP11]], splat (i64 32)
+; CHECK-NEXT:    [[TMP21:%.*]] = ashr <8 x i64> [[TMP19]], splat (i64 32)
+; CHECK-NEXT:    [[TMP22:%.*]] = or <8 x i64> [[TMP21]], zeroinitializer
+; CHECK-NEXT:    [[TMP23:%.*]] = ashr <8 x i64> [[TMP20]], splat (i64 32)
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <8 x i64> [[TMP16]], [[TMP22]]
+; CHECK-NEXT:    [[TMP24:%.*]] = mul <8 x i64> [[TMP17]], [[TMP23]]
+; CHECK-NEXT:    store <8 x i64> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i64> [[TMP24]]
+;
+  %b = load <16 x i32>, ptr %ptr_b
+  %res = call <8 x i64> @llvm.x86.avx512.pmul.dq.512(<16 x i32> %a, <16 x i32> %b)
+  ret < 8 x i64> %res
+}
+
+define <8 x i64> @test_mul_epi32_rmk(<16 x i32> %a, ptr %ptr_b, <8 x i64> %passThru, i8 %mask)  #0 {
+;
+; CHECK-LABEL: @test_mul_epi32_rmk(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 136) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 72) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP5:%.*]], label [[TMP6:%.*]], !prof [[PROF1]]
+; CHECK:       5:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       6:
+; CHECK-NEXT:    [[B:%.*]] = load <16 x i32>, ptr [[PTR_B:%.*]], align 64
+; CHECK-NEXT:    [[TMP7:%.*]] = ptrtoint ptr [[PTR_B]] to i64
+; CHECK-NEXT:    [[TMP8:%.*]] = xor i64 [[TMP7]], 87960930222080
+; CHECK-NEXT:    [[TMP9:%.*]] = inttoptr i64 [[TMP8]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <16 x i32>, ptr [[TMP9]], align 64
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast <16 x i32> [[TMP2]] to <8 x i64>
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <16 x i32> [[A:%.*]] to <8 x i64>
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <16 x i32> [[_MSLD]] to <8 x i64>
+; CHECK-NEXT:    [[TMP13:%.*]] = bitcast <16 x i32> [[B]] to <8 x i64>
+; CHECK-NEXT:    [[TMP14:%.*]] = shl <8 x i64> [[TMP10]], splat (i64 32)
+; CHECK-NEXT:    [[TMP15:%.*]] = or <8 x i64> [[TMP14]], zeroinitializer
+; CHECK-NEXT:    [[TMP16:%.*]] = shl <8 x i64> [[TMP11]], splat (i64 32)
+; CHECK-NEXT:    [[TMP17:%.*]] = ashr <8 x i64> [[TMP15]], splat (i64 32)
+; CHECK-NEXT:    [[TMP18:%.*]] = or <8 x i64> [[TMP17]], zeroinitializer
+; CHECK-NEXT:    [[TMP19:%.*]] = ashr <8 x i64> [[TMP16]], splat (i64 32)
+; CHECK-NEXT:    [[TMP20:%.*]] = shl <8 x i64> [[TMP12]], splat (i64 32)
+; CHECK-NEXT:    [[TMP21:%.*]] = or <8 x i64> [[TMP20]], zeroinitializer
+; CHECK-NEXT:    [[TMP22:%.*]] = shl <8 x i64> [[TMP13]], splat (i64 32)
+; CHECK-NEXT:    [[TMP23:%.*]] = ashr <8 x i64> [[TMP21]], splat (i64 32)
+; CHECK-NEXT:    [[TMP24:%.*]] = or <8 x i64> [[TMP23]], zeroinitializer
+; CHECK-NEXT:    [[TMP25:%.*]] = ashr <8 x i64> [[TMP22]], splat (i64 32)
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <8 x i64> [[TMP18]], [[TMP24]]
+; CHECK-NEXT:    [[TMP26:%.*]] = mul <8 x i64> [[TMP19]], [[TMP25]]
+; CHECK-NEXT:    [[TMP27:%.*]] = bitcast i8 [[TMP3]] to <8 x i1>
+; CHECK-NEXT:    [[MASK_CAST:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP28:%.*]] = select <8 x i1> [[MASK_CAST]], <8 x i64> [[_MSPROP]], <8 x i64> [[TMP4]]
+; CHECK-NEXT:    [[TMP29:%.*]] = xor <8 x i64> [[TMP26]], [[PASSTHRU:%.*]]
+; CHECK-NEXT:    [[TMP30:%.*]] = or <8 x i64> [[TMP29]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP31:%.*]] = or <8 x i64> [[TMP30]], [[TMP4]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP27]], <8 x i64> [[TMP31]], <8 x i64> [[TMP28]]
+; CHECK-NEXT:    [[RES:%.*]] = select <8 x i1> [[MASK_CAST]], <8 x i64> [[TMP26]], <8 x i64> [[PASSTHRU]]
+; CHECK-NEXT:    store <8 x i64> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i64> [[RES]]
+;
+  %b = load <16 x i32>, ptr %ptr_b
+  %mul = call <8 x i64> @llvm.x86.avx512.pmul.dq.512(<16 x i32> %a, <16 x i32> %b)
+  %mask.cast = bitcast i8 %mask to <8 x i1>
+  %res = select <8 x i1> %mask.cast, <8 x i64> %mul, <8 x i64> %passThru
+  ret < 8 x i64> %res
+}
+
+define <8 x i64> @test_mul_epi32_rmkz(<16 x i32> %a, ptr %ptr_b, i8 %mask)  #0 {
+;
+; CHECK-LABEL: @test_mul_epi32_rmkz(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 72) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP4:%.*]], label [[TMP5:%.*]], !prof [[PROF1]]
+; CHECK:       4:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       5:
+; CHECK-NEXT:    [[B:%.*]] = load <16 x i32>, ptr [[PTR_B:%.*]], align 64
+; CHECK-NEXT:    [[TMP6:%.*]] = ptrtoint ptr [[PTR_B]] to i64
+; CHECK-NEXT:    [[TMP7:%.*]] = xor i64 [[TMP6]], 87960930222080
+; CHECK-NEXT:    [[TMP8:%.*]] = inttoptr i64 [[TMP7]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <16 x i32>, ptr [[TMP8]], align 64
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <16 x i32> [[TMP2]] to <8 x i64>
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast <16 x i32> [[A:%.*]] to <8 x i64>
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <16 x i32> [[_MSLD]] to <8 x i64>
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <16 x i32> [[B]] to <8 x i64>
+; CHECK-NEXT:    [[TMP13:%.*]] = shl <8 x i64> [[TMP9]], splat (i64 32)
+; CHECK-NEXT:    [[TMP14:%.*]] = or <8 x i64> [[TMP13]], zeroinitializer
+; CHECK-NEXT:    [[TMP15:%.*]] = shl <8 x i64> [[TMP10]], splat (i64 32)
+; CHECK-NEXT:    [[TMP16:%.*]] = ashr <8 x i64> [[TMP14]], splat (i64 32)
+; CHECK-NEXT:    [[TMP17:%.*]] = or <8 x i64> [[TMP16]], zeroinitializer
+; CHECK-NEXT:    [[TMP18:%.*]] = ashr <8 x i64> [[TMP15]], splat (i64 32)
+; CHECK-NEXT:    [[TMP19:%.*]] = shl <8 x i64> [[TMP11]], splat (i64 32)
+; CHECK-NEXT:    [[TMP20:%.*]] = or <8 x i64> [[TMP19]], zeroinitializer
+; CHECK-NEXT:    [[TMP21:%.*]] = shl <8 x i64> [[TMP12]], splat (i64 32)
+; CHECK-NEXT:    [[TMP22:%.*]] = ashr <8 x i64> [[TMP20]], splat (i64 32)
+; CHECK-NEXT:    [[TMP23:%.*]] = or <8 x i64> [[TMP22]], zeroinitializer
+; CHECK-NEXT:    [[TMP24:%.*]] = ashr <8 x i64> [[TMP21]], splat (i64 32)
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <8 x i64> [[TMP17]], [[TMP23]]
+; CHECK-NEXT:    [[TMP25:%.*]] = mul <8 x i64> [[TMP18]], [[TMP24]]
+; CHECK-NEXT:    [[TMP26:%.*]] = bitcast i8 [[TMP3]] to <8 x i1>
+; CHECK-NEXT:    [[MASK_CAST:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP27:%.*]] = select <8 x i1> [[MASK_CAST]], <8 x i64> [[_MSPROP]], <8 x i64> zeroinitializer
+; CHECK-NEXT:    [[TMP28:%.*]] = xor <8 x i64> [[TMP25]], zeroinitializer
+; CHECK-NEXT:    [[TMP29:%.*]] = or <8 x i64> [[TMP28]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP30:%.*]] = or <8 x i64> [[TMP29]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP26]], <8 x i64> [[TMP30]], <8 x i64> [[TMP27]]
+; CHECK-NEXT:    [[RES:%.*]] = select <8 x i1> [[MASK_CAST]], <8 x i64> [[TMP25]], <8 x i64> zeroinitializer
+; CHECK-NEXT:    store <8 x i64> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i64> [[RES]]
+;
+  %b = load <16 x i32>, ptr %ptr_b
+  %mul = call <8 x i64> @llvm.x86.avx512.pmul.dq.512(<16 x i32> %a, <16 x i32> %b)
+  %mask.cast = bitcast i8 %mask to <8 x i1>
+  %res = select <8 x i1> %mask.cast, <8 x i64> %mul, <8 x i64> zeroinitializer
+  ret < 8 x i64> %res
+}
+
+define <8 x i64> @test_mul_epi32_rmb(<16 x i32> %a, ptr %ptr_b, <8 x i64> %extra_param)  #0 {
+;
+; CHECK-LABEL: @test_mul_epi32_rmb(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 72) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP4:%.*]], label [[TMP26:%.*]], !prof [[PROF1]]
+; CHECK:       4:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       5:
+; CHECK-NEXT:    [[Q:%.*]] = load i64, ptr [[PTR_B:%.*]], align 8
+; CHECK-NEXT:    [[TMP5:%.*]] = ptrtoint ptr [[PTR_B]] to i64
+; CHECK-NEXT:    [[TMP6:%.*]] = xor i64 [[TMP5]], 87960930222080
+; CHECK-NEXT:    [[TMP7:%.*]] = inttoptr i64 [[TMP6]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load i64, ptr [[TMP7]], align 8
+; CHECK-NEXT:    [[_MSPROP:%.*]] = insertelement <8 x i64> [[TMP3]], i64 [[_MSLD]], i32 0
+; CHECK-NEXT:    [[VECINIT_I:%.*]] = insertelement <8 x i64> [[EXTRA_PARAM:%.*]], i64 [[Q]], i32 0
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = shufflevector <8 x i64> [[_MSPROP]], <8 x i64> [[TMP3]], <8 x i32> zeroinitializer
+; CHECK-NEXT:    [[B64:%.*]] = shufflevector <8 x i64> [[VECINIT_I]], <8 x i64> [[EXTRA_PARAM]], <8 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <8 x i64> [[_MSPROP1]] to <16 x i32>
+; CHECK-NEXT:    [[B:%.*]] = bitcast <8 x i64> [[B64]] to <16 x i32>
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <16 x i32> [[TMP2]] to <8 x i64>
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast <16 x i32> [[A:%.*]] to <8 x i64>
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <16 x i32> [[TMP8]] to <8 x i64>
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <16 x i32> [[B]] to <8 x i64>
+; CHECK-NEXT:    [[TMP13:%.*]] = shl <8 x i64> [[TMP9]], splat (i64 32)
+; CHECK-NEXT:    [[TMP14:%.*]] = or <8 x i64> [[TMP13]], zeroinitializer
+; CHECK-NEXT:    [[TMP15:%.*]] = shl <8 x i64> [[TMP10]], splat (i64 32)
+; CHECK-NEXT:    [[TMP16:%.*]] = ashr <8 x i64> [[TMP14]], splat (i64 32)
+; CHECK-NEXT:    [[TMP17:%.*]] = or <8 x i64> [[TMP16]], zeroinitializer
+; CHECK-NEXT:    [[TMP18:%.*]] = ashr <8 x i64> [[TMP15]], splat (i64 32)
+; CHECK-NEXT:    [[TMP19:%.*]] = shl <8 x i64> [[TMP11]], splat (i64 32)
+; CHECK-NEXT:    [[TMP20:%.*]] = or <8 x i64> [[TMP19]], zeroinitializer
+; CHECK-NEXT:    [[TMP21:%.*]] = shl <8 x i64> [[TMP12]], splat (i64 32)
+; CHECK-NEXT:    [[TMP22:%.*]] = ashr <8 x i64> [[TMP20]], splat (i64 32)
+; CHECK-NEXT:    [[TMP23:%.*]] = or <8 x i64> [[TMP22]], zeroinitializer
+; CHECK-NEXT:    [[TMP24:%.*]] = ashr <8 x i64> [[TMP21]], splat (i64 32)
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = or <8 x i64> [[TMP17]], [[TMP23]]
+; CHECK-NEXT:    [[TMP25:%.*]] = mul <8 x i64> [[TMP18]], [[TMP24]]
+; CHECK-NEXT:    store <8 x i64> [[_MSPROP2]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i64> [[TMP25]]
+;
+  %q = load i64, ptr %ptr_b
+  %vecinit.i = insertelement <8 x i64> %extra_param, i64 %q, i32 0
+  %b64 = shufflevector <8 x i64> %vecinit.i, <8 x i64> %extra_param, <8 x i32> zeroinitializer
+  %b = bitcast <8 x i64> %b64 to <16 x i32>
+  %res = call <8 x i64> @llvm.x86.avx512.pmul.dq.512(<16 x i32> %a, <16 x i32> %b)
+  ret < 8 x i64> %res
+}
+
+define <8 x i64> @test_mul_epi32_rmbk(<16 x i32> %a, ptr %ptr_b, <8 x i64> %passThru, i8 %mask, <8 x i64> %extra_param)  #0 {
+;
+; CHECK-LABEL: @test_mul_epi32_rmbk(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP5:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 144) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 136) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 72) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP6:%.*]], label [[TMP33:%.*]], !prof [[PROF1]]
+; CHECK:       6:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       7:
+; CHECK-NEXT:    [[Q:%.*]] = load i64, ptr [[PTR_B:%.*]], align 8
+; CHECK-NEXT:    [[TMP7:%.*]] = ptrtoint ptr [[PTR_B]] to i64
+; CHECK-NEXT:    [[TMP8:%.*]] = xor i64 [[TMP7]], 87960930222080
+; CHECK-NEXT:    [[TMP9:%.*]] = inttoptr i64 [[TMP8]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load i64, ptr [[TMP9]], align 8
+; CHECK-NEXT:    [[_MSPROP:%.*]] = insertelement <8 x i64> [[TMP5]], i64 [[_MSLD]], i32 0
+; CHECK-NEXT:    [[VECINIT_I:%.*]] = insertelement <8 x i64> [[EXTRA_PARAM:%.*]], i64 [[Q]], i32 0
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = shufflevector <8 x i64> [[_MSPROP]], <8 x i64> [[TMP5]], <8 x i32> zeroinitializer
+; CHECK-NEXT:    [[B64:%.*]] = shufflevector <8 x i64> [[VECINIT_I]], <8 x i64> [[EXTRA_PARAM]], <8 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast <8 x i64> [[_MSPROP1]] to <16 x i32>
+; CHECK-NEXT:    [[B:%.*]] = bitcast <8 x i64> [[B64]] to <16 x i32>
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <16 x i32> [[TMP2]] to <8 x i64>
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <16 x i32> [[A:%.*]] to <8 x i64>
+; CHECK-NEXT:    [[TMP13:%.*]] = bitcast <16 x i32> [[TMP10]] to <8 x i64>
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast <16 x i32> [[B]] to <8 x i64>
+; CHECK-NEXT:    [[TMP15:%.*]] = shl <8 x i64> [[TMP11]], splat (i64 32)
+; CHECK-NEXT:    [[TMP16:%.*]] = or <8 x i64> [[TMP15]], zeroinitializer
+; CHECK-NEXT:    [[TMP17:%.*]] = shl <8 x i64> [[TMP12]], splat (i64 32)
+; CHECK-NEXT:    [[TMP18:%.*]] = ashr <8 x i64> [[TMP16]], splat (i64 32)
+; CHECK-NEXT:    [[TMP19:%.*]] = or <8 x i64> [[TMP18]], zeroinitializer
+; CHECK-NEXT:    [[TMP20:%.*]] = ashr <8 x i64> [[TMP17]], splat (i64 32)
+; CHECK-NEXT:    [[TMP21:%.*]] = shl <8 x i64> [[TMP13]], splat (i64 32)
+; CHECK-NEXT:    [[TMP22:%.*]] = or <8 x i64> [[TMP21]], zeroinitializer
+; CHECK-NEXT:    [[TMP23:%.*]] = shl <8 x i64> [[TMP14]], splat (i64 32)
+; CHECK-NEXT:    [[TMP24:%.*]] = ashr <8 x i64> [[TMP22]], splat (i64 32)
+; CHECK-NEXT:    [[TMP25:%.*]] = or <8 x i64> [[TMP24]], zeroinitializer
+; CHECK-NEXT:    [[TMP26:%.*]] = ashr <8 x i64> [[TMP23]], splat (i64 32)
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = or <8 x i64> [[TMP19]], [[TMP25]]
+; CHECK-NEXT:    [[TMP27:%.*]] = mul <8 x i64> [[TMP20]], [[TMP26]]
+; CHECK-NEXT:    [[TMP28:%.*]] = bitcast i8 [[TMP3]] to <8 x i1>
+; CHECK-NEXT:    [[MASK_CAST:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP29:%.*]] = select <8 x i1> [[MASK_CAST]], <8 x i64> [[_MSPROP2]], <8 x i64> [[TMP4]]
+; CHECK-NEXT:    [[TMP30:%.*]] = xor <8 x i64> [[TMP27]], [[PASSTHRU:%.*]]
+; CHECK-NEXT:    [[TMP31:%.*]] = or <8 x i64> [[TMP30]], [[_MSPROP2]]
+; CHECK-NEXT:    [[TMP32:%.*]] = or <8 x i64> [[TMP31]], [[TMP4]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP28]], <8 x i64> [[TMP32]], <8 x i64> [[TMP29]]
+; CHECK-NEXT:    [[RES:%.*]] = select <8 x i1> [[MASK_CAST]], <8 x i64> [[TMP27]], <8 x i64> [[PASSTHRU]]
+; CHECK-NEXT:    store <8 x i64> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i64> [[RES]]
+;
+  %q = load i64, ptr %ptr_b
+  %vecinit.i = insertelement <8 x i64> %extra_param, i64 %q, i32 0
+  %b64 = shufflevector <8 x i64> %vecinit.i, <8 x i64> %extra_param, <8 x i32> zeroinitializer
+  %b = bitcast <8 x i64> %b64 to <16 x i32>
+  %mul = call <8 x i64> @llvm.x86.avx512.pmul.dq.512(<16 x i32> %a, <16 x i32> %b)
+  %mask.cast = bitcast i8 %mask to <8 x i1>
+  %res = select <8 x i1> %mask.cast, <8 x i64> %mul, <8 x i64> %passThru
+  ret < 8 x i64> %res
+}
+
+define <8 x i64> @test_mul_epi32_rmbkz(<16 x i32> %a, ptr %ptr_b, i8 %mask, <8 x i64> %extra_param)  #0 {
+;
+; CHECK-LABEL: @test_mul_epi32_rmbkz(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 80) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 72) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP5:%.*]], label [[TMP32:%.*]], !prof [[PROF1]]
+; CHECK:       5:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       6:
+; CHECK-NEXT:    [[Q:%.*]] = load i64, ptr [[PTR_B:%.*]], align 8
+; CHECK-NEXT:    [[TMP6:%.*]] = ptrtoint ptr [[PTR_B]] to i64
+; CHECK-NEXT:    [[TMP7:%.*]] = xor i64 [[TMP6]], 87960930222080
+; CHECK-NEXT:    [[TMP8:%.*]] = inttoptr i64 [[TMP7]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load i64, ptr [[TMP8]], align 8
+; CHECK-NEXT:    [[_MSPROP:%.*]] = insertelement <8 x i64> [[TMP4]], i64 [[_MSLD]], i32 0
+; CHECK-NEXT:    [[VECINIT_I:%.*]] = insertelement <8 x i64> [[EXTRA_PARAM:%.*]], i64 [[Q]], i32 0
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = shufflevector <8 x i64> [[_MSPROP]], <8 x i64> [[TMP4]], <8 x i32> zeroinitializer
+; CHECK-NEXT:    [[B64:%.*]] = shufflevector <8 x i64> [[VECINIT_I]], <8 x i64> [[EXTRA_PARAM]], <8 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <8 x i64> [[_MSPROP1]] to <16 x i32>
+; CHECK-NEXT:    [[B:%.*]] = bitcast <8 x i64> [[B64]] to <16 x i32>
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast <16 x i32> [[TMP2]] to <8 x i64>
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <16 x i32> [[A:%.*]] to <8 x i64>
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <16 x i32> [[TMP9]] to <8 x i64>
+; CHECK-NEXT:    [[TMP13:%.*]] = bitcast <16 x i32> [[B]] to <8 x i64>
+; CHECK-NEXT:    [[TMP14:%.*]] = shl <8 x i64> [[TMP10]], splat (i64 32)
+; CHECK-NEXT:    [[TMP15:%.*]] = or <8 x i64> [[TMP14]], zeroinitializer
+; CHECK-NEXT:    [[TMP16:%.*]] = shl <8 x i64> [[TMP11]], splat (i64 32)
+; CHECK-NEXT:    [[TMP17:%.*]] = ashr <8 x i64> [[TMP15]], splat (i64 32)
+; CHECK-NEXT:    [[TMP18:%.*]] = or <8 x i64> [[TMP17]], zeroinitializer
+; CHECK-NEXT:    [[TMP19:%.*]] = ashr <8 x i64> [[TMP16]], splat (i64 32)
+; CHECK-NEXT:    [[TMP20:%.*]] = shl <8 x i64> [[TMP12]], splat (i64 32)
+; CHECK-NEXT:    [[TMP21:%.*]] = or <8 x i64> [[TMP20]], zeroinitializer
+; CHECK-NEXT:    [[TMP22:%.*]] = shl <8 x i64> [[TMP13]], splat (i64 32)
+; CHECK-NEXT:    [[TMP23:%.*]] = ashr <8 x i64> [[TMP21]], splat (i64 32)
+; CHECK-NEXT:    [[TMP24:%.*]] = or <8 x i64> [[TMP23]], zeroinitializer
+; CHECK-NEXT:    [[TMP25:%.*]] = ashr <8 x i64> [[TMP22]], splat (i64 32)
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = or <8 x i64> [[TMP18]], [[TMP24]]
+; CHECK-NEXT:    [[TMP26:%.*]] = mul <8 x i64> [[TMP19]], [[TMP25]]
+; CHECK-NEXT:    [[TMP27:%.*]] = bitcast i8 [[TMP3]] to <8 x i1>
+; CHECK-NEXT:    [[MASK_CAST:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP28:%.*]] = select <8 x i1> [[MASK_CAST]], <8 x i64> [[_MSPROP2]], <8 x i64> zeroinitializer
+; CHECK-NEXT:    [[TMP29:%.*]] = xor <8 x i64> [[TMP26]], zeroinitializer
+; CHECK-NEXT:    [[TMP30:%.*]] = or <8 x i64> [[TMP29]], [[_MSPROP2]]
+; CHECK-NEXT:    [[TMP31:%.*]] = or <8 x i64> [[TMP30]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP27]], <8 x i64> [[TMP31]], <8 x i64> [[TMP28]]
+; CHECK-NEXT:    [[RES:%.*]] = select <8 x i1> [[MASK_CAST]], <8 x i64> [[TMP26]], <8 x i64> zeroinitializer
+; CHECK-NEXT:    store <8 x i64> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i64> [[RES]]
+;
+  %q = load i64, ptr %ptr_b
+  %vecinit.i = insertelement <8 x i64> %extra_param, i64 %q, i32 0
+  %b64 = shufflevector <8 x i64> %vecinit.i, <8 x i64> %extra_param, <8 x i32> zeroinitializer
+  %b = bitcast <8 x i64> %b64 to <16 x i32>
+  %mul = call <8 x i64> @llvm.x86.avx512.pmul.dq.512(<16 x i32> %a, <16 x i32> %b)
+  %mask.cast = bitcast i8 %mask to <8 x i1>
+  %res = select <8 x i1> %mask.cast, <8 x i64> %mul, <8 x i64> zeroinitializer
+  ret < 8 x i64> %res
+}
+
+declare <8 x i64> @llvm.x86.avx512.pmul.dq.512(<16 x i32>, <16 x i32>)
+
+define <8 x i64> @test_mul_epu32_rr(<16 x i32> %a, <16 x i32> %b)  #0 {
+; CHECK-LABEL: @test_mul_epu32_rr(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <16 x i32> [[TMP1]] to <8 x i64>
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <16 x i32> [[A:%.*]] to <8 x i64>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <16 x i32> [[TMP2]] to <8 x i64>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <16 x i32> [[B:%.*]] to <8 x i64>
+; CHECK-NEXT:    [[TMP7:%.*]] = and <8 x i64> [[TMP3]], zeroinitializer
+; CHECK-NEXT:    [[TMP8:%.*]] = and <8 x i64> [[TMP4]], zeroinitializer
+; CHECK-NEXT:    [[TMP9:%.*]] = and <8 x i64> [[TMP3]], splat (i64 4294967295)
+; CHECK-NEXT:    [[TMP10:%.*]] = or <8 x i64> [[TMP7]], [[TMP8]]
+; CHECK-NEXT:    [[TMP11:%.*]] = or <8 x i64> [[TMP10]], [[TMP9]]
+; CHECK-NEXT:    [[TMP12:%.*]] = and <8 x i64> [[TMP4]], splat (i64 4294967295)
+; CHECK-NEXT:    [[TMP13:%.*]] = and <8 x i64> [[TMP5]], zeroinitializer
+; CHECK-NEXT:    [[TMP14:%.*]] = and <8 x i64> [[TMP6]], zeroinitializer
+; CHECK-NEXT:    [[TMP15:%.*]] = and <8 x i64> [[TMP5]], splat (i64 4294967295)
+; CHECK-NEXT:    [[TMP16:%.*]] = or <8 x i64> [[TMP13]], [[TMP14]]
+; CHECK-NEXT:    [[TMP17:%.*]] = or <8 x i64> [[TMP16]], [[TMP15]]
+; CHECK-NEXT:    [[TMP18:%.*]] = and <8 x i64> [[TMP6]], splat (i64 4294967295)
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <8 x i64> [[TMP11]], [[TMP17]]
+; CHECK-NEXT:    [[TMP19:%.*]] = mul <8 x i64> [[TMP12]], [[TMP18]]
+; CHECK-NEXT:    store <8 x i64> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i64> [[TMP19]]
+;
+  %res = call <8 x i64> @llvm.x86.avx512.pmulu.dq.512(<16 x i32> %a, <16 x i32> %b)
+  ret < 8 x i64> %res
+}
+
+define <8 x i64> @test_mul_epu32_rrk(<16 x i32> %a, <16 x i32> %b, <8 x i64> %passThru, i8 %mask)  #0 {
+;
+; CHECK-LABEL: @test_mul_epu32_rrk(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <16 x i32> [[TMP1]] to <8 x i64>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <16 x i32> [[A:%.*]] to <8 x i64>
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <16 x i32> [[TMP2]] to <8 x i64>
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <16 x i32> [[B:%.*]] to <8 x i64>
+; CHECK-NEXT:    [[TMP9:%.*]] = and <8 x i64> [[TMP5]], zeroinitializer
+; CHECK-NEXT:    [[TMP10:%.*]] = and <8 x i64> [[TMP6]], zeroinitializer
+; CHECK-NEXT:    [[TMP11:%.*]] = and <8 x i64> [[TMP5]], splat (i64 4294967295)
+; CHECK-NEXT:    [[TMP12:%.*]] = or <8 x i64> [[TMP9]], [[TMP10]]
+; CHECK-NEXT:    [[TMP13:%.*]] = or <8 x i64> [[TMP12]], [[TMP11]]
+; CHECK-NEXT:    [[TMP14:%.*]] = and <8 x i64> [[TMP6]], splat (i64 4294967295)
+; CHECK-NEXT:    [[TMP15:%.*]] = and <8 x i64> [[TMP7]], zeroinitializer
+; CHECK-NEXT:    [[TMP16:%.*]] = and <8 x i64> [[TMP8]], zeroinitializer
+; CHECK-NEXT:    [[TMP17:%.*]] = and <8 x i64> [[TMP7]], splat (i64 4294967295)
+; CHECK-NEXT:    [[TMP18:%.*]] = or <8 x i64> [[TMP15]], [[TMP16]]
+; CHECK-NEXT:    [[TMP19:%.*]] = or <8 x i64> [[TMP18]], [[TMP17]]
+; CHECK-NEXT:    [[TMP20:%.*]] = and <8 x i64> [[TMP8]], splat (i64 4294967295)
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <8 x i64> [[TMP13]], [[TMP19]]
+; CHECK-NEXT:    [[TMP21:%.*]] = mul <8 x i64> [[TMP14]], [[TMP20]]
+; CHECK-NEXT:    [[TMP22:%.*]] = bitcast i8 [[TMP3]] to <8 x i1>
+; CHECK-NEXT:    [[MASK_CAST:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP23:%.*]] = select <8 x i1> [[MASK_CAST]], <8 x i64> [[_MSPROP]], <8 x i64> [[TMP4]]
+; CHECK-NEXT:    [[TMP24:%.*]] = xor <8 x i64> [[TMP21]], [[PASSTHRU:%.*]]
+; CHECK-NEXT:    [[TMP25:%.*]] = or <8 x i64> [[TMP24]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP26:%.*]] = or <8 x i64> [[TMP25]], [[TMP4]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP22]], <8 x i64> [[TMP26]], <8 x i64> [[TMP23]]
+; CHECK-NEXT:    [[RES:%.*]] = select <8 x i1> [[MASK_CAST]], <8 x i64> [[TMP21]], <8 x i64> [[PASSTHRU]]
+; CHECK-NEXT:    store <8 x i64> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i64> [[RES]]
+;
+  %mul = call <8 x i64> @llvm.x86.avx512.pmulu.dq.512(<16 x i32> %a, <16 x i32> %b)
+  %mask.cast = bitcast i8 %mask to <8 x i1>
+  %res = select <8 x i1> %mask.cast, <8 x i64> %mul, <8 x i64> %passThru
+  ret < 8 x i64> %res
+}
+
+define <8 x i64> @test_mul_epu32_rrkz(<16 x i32> %a, <16 x i32> %b, i8 %mask)  #0 {
+;
+; CHECK-LABEL: @test_mul_epu32_rrkz(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <16 x i32> [[TMP1]] to <8 x i64>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <16 x i32> [[A:%.*]] to <8 x i64>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <16 x i32> [[TMP2]] to <8 x i64>
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <16 x i32> [[B:%.*]] to <8 x i64>
+; CHECK-NEXT:    [[TMP8:%.*]] = and <8 x i64> [[TMP4]], zeroinitializer
+; CHECK-NEXT:    [[TMP9:%.*]] = and <8 x i64> [[TMP5]], zeroinitializer
+; CHECK-NEXT:    [[TMP10:%.*]] = and <8 x i64> [[TMP4]], splat (i64 4294967295)
+; CHECK-NEXT:    [[TMP11:%.*]] = or <8 x i64> [[TMP8]], [[TMP9]]
+; CHECK-NEXT:    [[TMP12:%.*]] = or <8 x i64> [[TMP11]], [[TMP10]]
+; CHECK-NEXT:    [[TMP13:%.*]] = and <8 x i64> [[TMP5]], splat (i64 4294967295)
+; CHECK-NEXT:    [[TMP14:%.*]] = and <8 x i64> [[TMP6]], zeroinitializer
+; CHECK-NEXT:    [[TMP15:%.*]] = and <8 x i64> [[TMP7]], zeroinitializer
+; CHECK-NEXT:    [[TMP16:%.*]] = and <8 x i64> [[TMP6]], splat (i64 4294967295)
+; CHECK-NEXT:    [[TMP17:%.*]] = or <8 x i64> [[TMP14]], [[TMP15]]
+; CHECK-NEXT:    [[TMP18:%.*]] = or <8 x i64> [[TMP17]], [[TMP16]]
+; CHECK-NEXT:    [[TMP19:%.*]] = and <8 x i64> [[TMP7]], splat (i64 4294967295)
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <8 x i64> [[TMP12]], [[TMP18]]
+; CHECK-NEXT:    [[TMP20:%.*]] = mul <8 x i64> [[TMP13]], [[TMP19]]
+; CHECK-NEXT:    [[TMP21:%.*]] = bitcast i8 [[TMP3]] to <8 x i1>
+; CHECK-NEXT:    [[MASK_CAST:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP22:%.*]] = select <8 x i1> [[MASK_CAST]], <8 x i64> [[_MSPROP]], <8 x i64> zeroinitializer
+; CHECK-NEXT:    [[TMP23:%.*]] = xor <8 x i64> [[TMP20]], zeroinitializer
+; CHECK-NEXT:    [[TMP24:%.*]] = or <8 x i64> [[TMP23]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP25:%.*]] = or <8 x i64> [[TMP24]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP21]], <8 x i64> [[TMP25]], <8 x i64> [[TMP22]]
+; CHECK-NEXT:    [[RES:%.*]] = select <8 x i1> [[MASK_CAST]], <8 x i64> [[TMP20]], <8 x i64> zeroinitializer
+; CHECK-NEXT:    store <8 x i64> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i64> [[RES]]
+;
+  %mul = call <8 x i64> @llvm.x86.avx512.pmulu.dq.512(<16 x i32> %a, <16 x i32> %b)
+  %mask.cast = bitcast i8 %mask to <8 x i1>
+  %res = select <8 x i1> %mask.cast, <8 x i64> %mul, <8 x i64> zeroinitializer
+  ret < 8 x i64> %res
+}
+
+define <8 x i64> @test_mul_epu32_rm(<16 x i32> %a, ptr %ptr_b)  #0 {
+;
+; CHECK-LABEL: @test_mul_epu32_rm(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP3:%.*]], label [[TMP4:%.*]], !prof [[PROF1]]
+; CHECK:       3:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       4:
+; CHECK-NEXT:    [[B:%.*]] = load <16 x i32>, ptr [[PTR_B:%.*]], align 64
+; CHECK-NEXT:    [[TMP5:%.*]] = ptrtoint ptr [[PTR_B]] to i64
+; CHECK-NEXT:    [[TMP6:%.*]] = xor i64 [[TMP5]], 87960930222080
+; CHECK-NEXT:    [[TMP7:%.*]] = inttoptr i64 [[TMP6]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <16 x i32>, ptr [[TMP7]], align 64
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <16 x i32> [[TMP2]] to <8 x i64>
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <16 x i32> [[A:%.*]] to <8 x i64>
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast <16 x i32> [[_MSLD]] to <8 x i64>
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <16 x i32> [[B]] to <8 x i64>
+; CHECK-NEXT:    [[TMP12:%.*]] = and <8 x i64> [[TMP8]], zeroinitializer
+; CHECK-NEXT:    [[TMP13:%.*]] = and <8 x i64> [[TMP9]], zeroinitializer
+; CHECK-NEXT:    [[TMP14:%.*]] = and <8 x i64> [[TMP8]], splat (i64 4294967295)
+; CHECK-NEXT:    [[TMP15:%.*]] = or <8 x i64> [[TMP12]], [[TMP13]]
+; CHECK-NEXT:    [[TMP16:%.*]] = or <8 x i64> [[TMP15]], [[TMP14]]
+; CHECK-NEXT:    [[TMP17:%.*]] = and <8 x i64> [[TMP9]], splat (i64 4294967295)
+; CHECK-NEXT:    [[TMP18:%.*]] = and <8 x i64> [[TMP10]], zeroinitializer
+; CHECK-NEXT:    [[TMP19:%.*]] = and <8 x i64> [[TMP11]], zeroinitializer
+; CHECK-NEXT:    [[TMP20:%.*]] = and <8 x i64> [[TMP10]], splat (i64 4294967295)
+; CHECK-NEXT:    [[TMP21:%.*]] = or <8 x i64> [[TMP18]], [[TMP19]]
+; CHECK-NEXT:    [[TMP22:%.*]] = or <8 x i64> [[TMP21]], [[TMP20]]
+; CHECK-NEXT:    [[TMP23:%.*]] = and <8 x i64> [[TMP11]], splat (i64 4294967295)
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <8 x i64> [[TMP16]], [[TMP22]]
+; CHECK-NEXT:    [[TMP24:%.*]] = mul <8 x i64> [[TMP17]], [[TMP23]]
+; CHECK-NEXT:    store <8 x i64> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i64> [[TMP24]]
+;
+  %b = load <16 x i32>, ptr %ptr_b
+  %res = call <8 x i64> @llvm.x86.avx512.pmulu.dq.512(<16 x i32> %a, <16 x i32> %b)
+  ret < 8 x i64> %res
+}
+
+define <8 x i64> @test_mul_epu32_rmk(<16 x i32> %a, ptr %ptr_b, <8 x i64> %passThru, i8 %mask)  #0 {
+;
+; CHECK-LABEL: @test_mul_epu32_rmk(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 136) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 72) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP5:%.*]], label [[TMP6:%.*]], !prof [[PROF1]]
+; CHECK:       5:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       6:
+; CHECK-NEXT:    [[B:%.*]] = load <16 x i32>, ptr [[PTR_B:%.*]], align 64
+; CHECK-NEXT:    [[TMP7:%.*]] = ptrtoint ptr [[PTR_B]] to i64
+; CHECK-NEXT:    [[TMP8:%.*]] = xor i64 [[TMP7]], 87960930222080
+; CHECK-NEXT:    [[TMP9:%.*]] = inttoptr i64 [[TMP8]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <16 x i32>, ptr [[TMP9]], align 64
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast <16 x i32> [[TMP2]] to <8 x i64>
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <16 x i32> [[A:%.*]] to <8 x i64>
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <16 x i32> [[_MSLD]] to <8 x i64>
+; CHECK-NEXT:    [[TMP13:%.*]] = bitcast <16 x i32> [[B]] to <8 x i64>
+; CHECK-NEXT:    [[TMP14:%.*]] = and <8 x i64> [[TMP10]], zeroinitializer
+; CHECK-NEXT:    [[TMP15:%.*]] = and <8 x i64> [[TMP11]], zeroinitializer
+; CHECK-NEXT:    [[TMP16:%.*]] = and <8 x i64> [[TMP10]], splat (i64 4294967295)
+; CHECK-NEXT:    [[TMP17:%.*]] = or <8 x i64> [[TMP14]], [[TMP15]]
+; CHECK-NEXT:    [[TMP18:%.*]] = or <8 x i64> [[TMP17]], [[TMP16]]
+; CHECK-NEXT:    [[TMP19:%.*]] = and <8 x i64> [[TMP11]], splat (i64 4294967295)
+; CHECK-NEXT:    [[TMP20:%.*]] = and <8 x i64> [[TMP12]], zeroinitializer
+; CHECK-NEXT:    [[TMP21:%.*]] = and <8 x i64> [[TMP13]], zeroinitializer
+; CHECK-NEXT:    [[TMP22:%.*]] = and <8 x i64> [[TMP12]], splat (i64 4294967295)
+; CHECK-NEXT:    [[TMP23:%.*]] = or <8 x i64> [[TMP20]], [[TMP21]]
+; CHECK-NEXT:    [[TMP24:%.*]] = or <8 x i64> [[TMP23]], [[TMP22]]
+; CHECK-NEXT:    [[TMP25:%.*]] = and <8 x i64> [[TMP13]], splat (i64 4294967295)
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <8 x i64> [[TMP18]], [[TMP24]]
+; CHECK-NEXT:    [[TMP26:%.*]] = mul <8 x i64> [[TMP19]], [[TMP25]]
+; CHECK-NEXT:    [[TMP27:%.*]] = bitcast i8 [[TMP3]] to <8 x i1>
+; CHECK-NEXT:    [[MASK_CAST:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP28:%.*]] = select <8 x i1> [[MASK_CAST]], <8 x i64> [[_MSPROP]], <8 x i64> [[TMP4]]
+; CHECK-NEXT:    [[TMP29:%.*]] = xor <8 x i64> [[TMP26]], [[PASSTHRU:%.*]]
+; CHECK-NEXT:    [[TMP30:%.*]] = or <8 x i64> [[TMP29]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP31:%.*]] = or <8 x i64> [[TMP30]], [[TMP4]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP27]], <8 x i64> [[TMP31]], <8 x i64> [[TMP28]]
+; CHECK-NEXT:    [[RES:%.*]] = select <8 x i1> [[MASK_CAST]], <8 x i64> [[TMP26]], <8 x i64> [[PASSTHRU]]
+; CHECK-NEXT:    store <8 x i64> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i64> [[RES]]
+;
+  %b = load <16 x i32>, ptr %ptr_b
+  %mul = call <8 x i64> @llvm.x86.avx512.pmulu.dq.512(<16 x i32> %a, <16 x i32> %b)
+  %mask.cast = bitcast i8 %mask to <8 x i1>
+  %res = select <8 x i1> %mask.cast, <8 x i64> %mul, <8 x i64> %passThru
+  ret < 8 x i64> %res
+}
+
+define <8 x i64> @test_mul_epu32_rmkz(<16 x i32> %a, ptr %ptr_b, i8 %mask)  #0 {
+;
+; CHECK-LABEL: @test_mul_epu32_rmkz(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 72) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP4:%.*]], label [[TMP5:%.*]], !prof [[PROF1]]
+; CHECK:       4:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       5:
+; CHECK-NEXT:    [[B:%.*]] = load <16 x i32>, ptr [[PTR_B:%.*]], align 64
+; CHECK-NEXT:    [[TMP6:%.*]] = ptrtoint ptr [[PTR_B]] to i64
+; CHECK-NEXT:    [[TMP7:%.*]] = xor i64 [[TMP6]], 87960930222080
+; CHECK-NEXT:    [[TMP8:%.*]] = inttoptr i64 [[TMP7]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <16 x i32>, ptr [[TMP8]], align 64
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <16 x i32> [[TMP2]] to <8 x i64>
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast <16 x i32> [[A:%.*]] to <8 x i64>
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <16 x i32> [[_MSLD]] to <8 x i64>
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <16 x i32> [[B]] to <8 x i64>
+; CHECK-NEXT:    [[TMP13:%.*]] = and <8 x i64> [[TMP9]], zeroinitializer
+; CHECK-NEXT:    [[TMP14:%.*]] = and <8 x i64> [[TMP10]], zeroinitializer
+; CHECK-NEXT:    [[TMP15:%.*]] = and <8 x i64> [[TMP9]], splat (i64 4294967295)
+; CHECK-NEXT:    [[TMP16:%.*]] = or <8 x i64> [[TMP13]], [[TMP14]]
+; CHECK-NEXT:    [[TMP17:%.*]] = or <8 x i64> [[TMP16]], [[TMP15]]
+; CHECK-NEXT:    [[TMP18:%.*]] = and <8 x i64> [[TMP10]], splat (i64 4294967295)
+; CHECK-NEXT:    [[TMP19:%.*]] = and <8 x i64> [[TMP11]], zeroinitializer
+; CHECK-NEXT:    [[TMP20:%.*]] = and <8 x i64> [[TMP12]], zeroinitializer
+; CHECK-NEXT:    [[TMP21:%.*]] = and <8 x i64> [[TMP11]], splat (i64 4294967295)
+; CHECK-NEXT:    [[TMP22:%.*]] = or <8 x i64> [[TMP19]], [[TMP20]]
+; CHECK-NEXT:    [[TMP23:%.*]] = or <8 x i64> [[TMP22]], [[TMP21]]
+; CHECK-NEXT:    [[TMP24:%.*]] = and <8 x i64> [[TMP12]], splat (i64 4294967295)
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <8 x i64> [[TMP17]], [[TMP23]]
+; CHECK-NEXT:    [[TMP25:%.*]] = mul <8 x i64> [[TMP18]], [[TMP24]]
+; CHECK-NEXT:    [[TMP26:%.*]] = bitcast i8 [[TMP3]] to <8 x i1>
+; CHECK-NEXT:    [[MASK_CAST:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP27:%.*]] = select <8 x i1> [[MASK_CAST]], <8 x i64> [[_MSPROP]], <8 x i64> zeroinitializer
+; CHECK-NEXT:    [[TMP28:%.*]] = xor <8 x i64> [[TMP25]], zeroinitializer
+; CHECK-NEXT:    [[TMP29:%.*]] = or <8 x i64> [[TMP28]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP30:%.*]] = or <8 x i64> [[TMP29]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP26]], <8 x i64> [[TMP30]], <8 x i64> [[TMP27]]
+; CHECK-NEXT:    [[RES:%.*]] = select <8 x i1> [[MASK_CAST]], <8 x i64> [[TMP25]], <8 x i64> zeroinitializer
+; CHECK-NEXT:    store <8 x i64> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i64> [[RES]]
+;
+  %b = load <16 x i32>, ptr %ptr_b
+  %mul = call <8 x i64> @llvm.x86.avx512.pmulu.dq.512(<16 x i32> %a, <16 x i32> %b)
+  %mask.cast = bitcast i8 %mask to <8 x i1>
+  %res = select <8 x i1> %mask.cast, <8 x i64> %mul, <8 x i64> zeroinitializer
+  ret < 8 x i64> %res
+}
+
+define <8 x i64> @test_mul_epu32_rmb(<16 x i32> %a, ptr %ptr_b, <8 x i64> %extra_param)  #0 {
+;
+; CHECK-LABEL: @test_mul_epu32_rmb(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 72) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP4:%.*]], label [[TMP26:%.*]], !prof [[PROF1]]
+; CHECK:       4:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       5:
+; CHECK-NEXT:    [[Q:%.*]] = load i64, ptr [[PTR_B:%.*]], align 8
+; CHECK-NEXT:    [[TMP5:%.*]] = ptrtoint ptr [[PTR_B]] to i64
+; CHECK-NEXT:    [[TMP6:%.*]] = xor i64 [[TMP5]], 87960930222080
+; CHECK-NEXT:    [[TMP7:%.*]] = inttoptr i64 [[TMP6]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load i64, ptr [[TMP7]], align 8
+; CHECK-NEXT:    [[_MSPROP:%.*]] = insertelement <8 x i64> [[TMP3]], i64 [[_MSLD]], i32 0
+; CHECK-NEXT:    [[VECINIT_I:%.*]] = insertelement <8 x i64> [[EXTRA_PARAM:%.*]], i64 [[Q]], i32 0
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = shufflevector <8 x i64> [[_MSPROP]], <8 x i64> [[TMP3]], <8 x i32> zeroinitializer
+; CHECK-NEXT:    [[B64:%.*]] = shufflevector <8 x i64> [[VECINIT_I]], <8 x i64> [[EXTRA_PARAM]], <8 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <8 x i64> [[_MSPROP1]] to <16 x i32>
+; CHECK-NEXT:    [[B:%.*]] = bitcast <8 x i64> [[B64]] to <16 x i32>
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <16 x i32> [[TMP2]] to <8 x i64>
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast <16 x i32> [[A:%.*]] to <8 x i64>
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <16 x i32> [[TMP8]] to <8 x i64>
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <16 x i32> [[B]] to <8 x i64>
+; CHECK-NEXT:    [[TMP13:%.*]] = and <8 x i64> [[TMP9]], zeroinitializer
+; CHECK-NEXT:    [[TMP14:%.*]] = and <8 x i64> [[TMP10]], zeroinitializer
+; CHECK-NEXT:    [[TMP15:%.*]] = and <8 x i64> [[TMP9]], splat (i64 4294967295)
+; CHECK-NEXT:    [[TMP16:%.*]] = or <8 x i64> [[TMP13]], [[TMP14]]
+; CHECK-NEXT:    [[TMP17:%.*]] = or <8 x i64> [[TMP16]], [[TMP15]]
+; CHECK-NEXT:    [[TMP18:%.*]] = and <8 x i64> [[TMP10]], splat (i64 4294967295)
+; CHECK-NEXT:    [[TMP19:%.*]] = and <8 x i64> [[TMP11]], zeroinitializer
+; CHECK-NEXT:    [[TMP20:%.*]] = and <8 x i64> [[TMP12]], zeroinitializer
+; CHECK-NEXT:    [[TMP21:%.*]] = and <8 x i64> [[TMP11]], splat (i64 4294967295)
+; CHECK-NEXT:    [[TMP22:%.*]] = or <8 x i64> [[TMP19]], [[TMP20]]
+; CHECK-NEXT:    [[TMP23:%.*]] = or <8 x i64> [[TMP22]], [[TMP21]]
+; CHECK-NEXT:    [[TMP24:%.*]] = and <8 x i64> [[TMP12]], splat (i64 4294967295)
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = or <8 x i64> [[TMP17]], [[TMP23]]
+; CHECK-NEXT:    [[TMP25:%.*]] = mul <8 x i64> [[TMP18]], [[TMP24]]
+; CHECK-NEXT:    store <8 x i64> [[_MSPROP2]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i64> [[TMP25]]
+;
+  %q = load i64, ptr %ptr_b
+  %vecinit.i = insertelement <8 x i64> %extra_param, i64 %q, i32 0
+  %b64 = shufflevector <8 x i64> %vecinit.i, <8 x i64> %extra_param, <8 x i32> zeroinitializer
+  %b = bitcast <8 x i64> %b64 to <16 x i32>
+  %res = call <8 x i64> @llvm.x86.avx512.pmulu.dq.512(<16 x i32> %a, <16 x i32> %b)
+  ret < 8 x i64> %res
+}
+
+define <8 x i64> @test_mul_epu32_rmbk(<16 x i32> %a, ptr %ptr_b, <8 x i64> %passThru, i8 %mask, <8 x i64> %extra_param)  #0 {
+;
+; CHECK-LABEL: @test_mul_epu32_rmbk(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP5:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 144) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 136) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 72) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP6:%.*]], label [[TMP33:%.*]], !prof [[PROF1]]
+; CHECK:       6:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       7:
+; CHECK-NEXT:    [[Q:%.*]] = load i64, ptr [[PTR_B:%.*]], align 8
+; CHECK-NEXT:    [[TMP7:%.*]] = ptrtoint ptr [[PTR_B]] to i64
+; CHECK-NEXT:    [[TMP8:%.*]] = xor i64 [[TMP7]], 87960930222080
+; CHECK-NEXT:    [[TMP9:%.*]] = inttoptr i64 [[TMP8]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load i64, ptr [[TMP9]], align 8
+; CHECK-NEXT:    [[_MSPROP:%.*]] = insertelement <8 x i64> [[TMP5]], i64 [[_MSLD]], i32 0
+; CHECK-NEXT:    [[VECINIT_I:%.*]] = insertelement <8 x i64> [[EXTRA_PARAM:%.*]], i64 [[Q]], i32 0
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = shufflevector <8 x i64> [[_MSPROP]], <8 x i64> [[TMP5]], <8 x i32> zeroinitializer
+; CHECK-NEXT:    [[B64:%.*]] = shufflevector <8 x i64> [[VECINIT_I]], <8 x i64> [[EXTRA_PARAM]], <8 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast <8 x i64> [[_MSPROP1]] to <16 x i32>
+; CHECK-NEXT:    [[B:%.*]] = bitcast <8 x i64> [[B64]] to <16 x i32>
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <16 x i32> [[TMP2]] to <8 x i64>
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <16 x i32> [[A:%.*]] to <8 x i64>
+; CHECK-NEXT:    [[TMP13:%.*]] = bitcast <16 x i32> [[TMP10]] to <8 x i64>
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast <16 x i32> [[B]] to <8 x i64>
+; CHECK-NEXT:    [[TMP15:%.*]] = and <8 x i64> [[TMP11]], zeroinitializer
+; CHECK-NEXT:    [[TMP16:%.*]] = and <8 x i64> [[TMP12]], zeroinitializer
+; CHECK-NEXT:    [[TMP17:%.*]] = and <8 x i64> [[TMP11]], splat (i64 4294967295)
+; CHECK-NEXT:    [[TMP18:%.*]] = or <8 x i64> [[TMP15]], [[TMP16]]
+; CHECK-NEXT:    [[TMP19:%.*]] = or <8 x i64> [[TMP18]], [[TMP17]]
+; CHECK-NEXT:    [[TMP20:%.*]] = and <8 x i64> [[TMP12]], splat (i64 4294967295)
+; CHECK-NEXT:    [[TMP21:%.*]] = and <8 x i64> [[TMP13]], zeroinitializer
+; CHECK-NEXT:    [[TMP22:%.*]] = and <8 x i64> [[TMP14]], zeroinitializer
+; CHECK-NEXT:    [[TMP23:%.*]] = and <8 x i64> [[TMP13]], splat (i64 4294967295)
+; CHECK-NEXT:    [[TMP24:%.*]] = or <8 x i64> [[TMP21]], [[TMP22]]
+; CHECK-NEXT:    [[TMP25:%.*]] = or <8 x i64> [[TMP24]], [[TMP23]]
+; CHECK-NEXT:    [[TMP26:%.*]] = and <8 x i64> [[TMP14]], splat (i64 4294967295)
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = or <8 x i64> [[TMP19]], [[TMP25]]
+; CHECK-NEXT:    [[TMP27:%.*]] = mul <8 x i64> [[TMP20]], [[TMP26]]
+; CHECK-NEXT:    [[TMP28:%.*]] = bitcast i8 [[TMP3]] to <8 x i1>
+; CHECK-NEXT:    [[MASK_CAST:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP29:%.*]] = select <8 x i1> [[MASK_CAST]], <8 x i64> [[_MSPROP2]], <8 x i64> [[TMP4]]
+; CHECK-NEXT:    [[TMP30:%.*]] = xor <8 x i64> [[TMP27]], [[PASSTHRU:%.*]]
+; CHECK-NEXT:    [[TMP31:%.*]] = or <8 x i64> [[TMP30]], [[_MSPROP2]]
+; CHECK-NEXT:    [[TMP32:%.*]] = or <8 x i64> [[TMP31]], [[TMP4]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP28]], <8 x i64> [[TMP32]], <8 x i64> [[TMP29]]
+; CHECK-NEXT:    [[RES:%.*]] = select <8 x i1> [[MASK_CAST]], <8 x i64> [[TMP27]], <8 x i64> [[PASSTHRU]]
+; CHECK-NEXT:    store <8 x i64> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i64> [[RES]]
+;
+  %q = load i64, ptr %ptr_b
+  %vecinit.i = insertelement <8 x i64> %extra_param, i64 %q, i32 0
+  %b64 = shufflevector <8 x i64> %vecinit.i, <8 x i64> %extra_param, <8 x i32> zeroinitializer
+  %b = bitcast <8 x i64> %b64 to <16 x i32>
+  %mul = call <8 x i64> @llvm.x86.avx512.pmulu.dq.512(<16 x i32> %a, <16 x i32> %b)
+  %mask.cast = bitcast i8 %mask to <8 x i1>
+  %res = select <8 x i1> %mask.cast, <8 x i64> %mul, <8 x i64> %passThru
+  ret < 8 x i64> %res
+}
+
+define <8 x i64> @test_mul_epu32_rmbkz(<16 x i32> %a, ptr %ptr_b, i8 %mask, <8 x i64> %extra_param)  #0 {
+;
+; CHECK-LABEL: @test_mul_epu32_rmbkz(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 80) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 72) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP5:%.*]], label [[TMP32:%.*]], !prof [[PROF1]]
+; CHECK:       5:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       6:
+; CHECK-NEXT:    [[Q:%.*]] = load i64, ptr [[PTR_B:%.*]], align 8
+; CHECK-NEXT:    [[TMP6:%.*]] = ptrtoint ptr [[PTR_B]] to i64
+; CHECK-NEXT:    [[TMP7:%.*]] = xor i64 [[TMP6]], 87960930222080
+; CHECK-NEXT:    [[TMP8:%.*]] = inttoptr i64 [[TMP7]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load i64, ptr [[TMP8]], align 8
+; CHECK-NEXT:    [[_MSPROP:%.*]] = insertelement <8 x i64> [[TMP4]], i64 [[_MSLD]], i32 0
+; CHECK-NEXT:    [[VECINIT_I:%.*]] = insertelement <8 x i64> [[EXTRA_PARAM:%.*]], i64 [[Q]], i32 0
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = shufflevector <8 x i64> [[_MSPROP]], <8 x i64> [[TMP4]], <8 x i32> zeroinitializer
+; CHECK-NEXT:    [[B64:%.*]] = shufflevector <8 x i64> [[VECINIT_I]], <8 x i64> [[EXTRA_PARAM]], <8 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <8 x i64> [[_MSPROP1]] to <16 x i32>
+; CHECK-NEXT:    [[B:%.*]] = bitcast <8 x i64> [[B64]] to <16 x i32>
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast <16 x i32> [[TMP2]] to <8 x i64>
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <16 x i32> [[A:%.*]] to <8 x i64>
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <16 x i32> [[TMP9]] to <8 x i64>
+; CHECK-NEXT:    [[TMP13:%.*]] = bitcast <16 x i32> [[B]] to <8 x i64>
+; CHECK-NEXT:    [[TMP14:%.*]] = and <8 x i64> [[TMP10]], zeroinitializer
+; CHECK-NEXT:    [[TMP15:%.*]] = and <8 x i64> [[TMP11]], zeroinitializer
+; CHECK-NEXT:    [[TMP16:%.*]] = and <8 x i64> [[TMP10]], splat (i64 4294967295)
+; CHECK-NEXT:    [[TMP17:%.*]] = or <8 x i64> [[TMP14]], [[TMP15]]
+; CHECK-NEXT:    [[TMP18:%.*]] = or <8 x i64> [[TMP17]], [[TMP16]]
+; CHECK-NEXT:    [[TMP19:%.*]] = and <8 x i64> [[TMP11]], splat (i64 4294967295)
+; CHECK-NEXT:    [[TMP20:%.*]] = and <8 x i64> [[TMP12]], zeroinitializer
+; CHECK-NEXT:    [[TMP21:%.*]] = and <8 x i64> [[TMP13]], zeroinitializer
+; CHECK-NEXT:    [[TMP22:%.*]] = and <8 x i64> [[TMP12]], splat (i64 4294967295)
+; CHECK-NEXT:    [[TMP23:%.*]] = or <8 x i64> [[TMP20]], [[TMP21]]
+; CHECK-NEXT:    [[TMP24:%.*]] = or <8 x i64> [[TMP23]], [[TMP22]]
+; CHECK-NEXT:    [[TMP25:%.*]] = and <8 x i64> [[TMP13]], splat (i64 4294967295)
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = or <8 x i64> [[TMP18]], [[TMP24]]
+; CHECK-NEXT:    [[TMP26:%.*]] = mul <8 x i64> [[TMP19]], [[TMP25]]
+; CHECK-NEXT:    [[TMP27:%.*]] = bitcast i8 [[TMP3]] to <8 x i1>
+; CHECK-NEXT:    [[MASK_CAST:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP28:%.*]] = select <8 x i1> [[MASK_CAST]], <8 x i64> [[_MSPROP2]], <8 x i64> zeroinitializer
+; CHECK-NEXT:    [[TMP29:%.*]] = xor <8 x i64> [[TMP26]], zeroinitializer
+; CHECK-NEXT:    [[TMP30:%.*]] = or <8 x i64> [[TMP29]], [[_MSPROP2]]
+; CHECK-NEXT:    [[TMP31:%.*]] = or <8 x i64> [[TMP30]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP27]], <8 x i64> [[TMP31]], <8 x i64> [[TMP28]]
+; CHECK-NEXT:    [[RES:%.*]] = select <8 x i1> [[MASK_CAST]], <8 x i64> [[TMP26]], <8 x i64> zeroinitializer
+; CHECK-NEXT:    store <8 x i64> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i64> [[RES]]
+;
+  %q = load i64, ptr %ptr_b
+  %vecinit.i = insertelement <8 x i64> %extra_param, i64 %q, i32 0
+  %b64 = shufflevector <8 x i64> %vecinit.i, <8 x i64> %extra_param, <8 x i32> zeroinitializer
+  %b = bitcast <8 x i64> %b64 to <16 x i32>
+  %mul = call <8 x i64> @llvm.x86.avx512.pmulu.dq.512(<16 x i32> %a, <16 x i32> %b)
+  %mask.cast = bitcast i8 %mask to <8 x i1>
+  %res = select <8 x i1> %mask.cast, <8 x i64> %mul, <8 x i64> zeroinitializer
+  ret < 8 x i64> %res
+}
+
+declare <8 x i64> @llvm.x86.avx512.pmulu.dq.512(<16 x i32>, <16 x i32>)
+
+define <2 x double> @test_x86_avx512_mm_cvtu32_sd(<2 x double> %a, i32 %b)
+;
+; CHECK-LABEL: @test_x86_avx512_mm_cvtu32_sd(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i32, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = zext i32 [[TMP1]] to i64
+; CHECK-NEXT:    [[TMP4:%.*]] = uitofp i32 [[B:%.*]] to double
+; CHECK-NEXT:    [[_MSPROP:%.*]] = insertelement <2 x i64> [[TMP2]], i64 [[TMP3]], i64 0
+; CHECK-NEXT:    [[TMP5:%.*]] = insertelement <2 x double> [[A:%.*]], double [[TMP4]], i64 0
+; CHECK-NEXT:    store <2 x i64> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x double> [[TMP5]]
+;
+  #0 {
+  %res = call <2 x double> @llvm.x86.avx512.cvtusi2sd(<2 x double> %a, i32 %b) ; <<<2 x double>> [#uses=1]
+  ret <2 x double> %res
+}
+declare <2 x double> @llvm.x86.avx512.cvtusi2sd(<2 x double>, i32) nounwind readnone
+
+define <16 x float> @test_x86_vbroadcast_ss_512(ptr %a0)  #0 {
+;
+; CHECK-LABEL: @test_x86_vbroadcast_ss_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP2:%.*]], label [[TMP3:%.*]], !prof [[PROF1]]
+; CHECK:       2:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       3:
+; CHECK-NEXT:    [[TMP4:%.*]] = load float, ptr [[A0:%.*]], align 4
+; CHECK-NEXT:    [[TMP5:%.*]] = ptrtoint ptr [[A0]] to i64
+; CHECK-NEXT:    [[TMP6:%.*]] = xor i64 [[TMP5]], 87960930222080
+; CHECK-NEXT:    [[TMP7:%.*]] = inttoptr i64 [[TMP6]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load i32, ptr [[TMP7]], align 4
+; CHECK-NEXT:    [[_MSPROP:%.*]] = insertelement <16 x i32> splat (i32 -1), i32 [[_MSLD]], i32 0
+; CHECK-NEXT:    [[TMP8:%.*]] = insertelement <16 x float> poison, float [[TMP4]], i32 0
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = insertelement <16 x i32> [[_MSPROP]], i32 [[_MSLD]], i32 1
+; CHECK-NEXT:    [[TMP9:%.*]] = insertelement <16 x float> [[TMP8]], float [[TMP4]], i32 1
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = insertelement <16 x i32> [[_MSPROP1]], i32 [[_MSLD]], i32 2
+; CHECK-NEXT:    [[TMP10:%.*]] = insertelement <16 x float> [[TMP9]], float [[TMP4]], i32 2
+; CHECK-NEXT:    [[_MSPROP3:%.*]] = insertelement <16 x i32> [[_MSPROP2]], i32 [[_MSLD]], i32 3
+; CHECK-NEXT:    [[TMP11:%.*]] = insertelement <16 x float> [[TMP10]], float [[TMP4]], i32 3
+; CHECK-NEXT:    [[_MSPROP4:%.*]] = insertelement <16 x i32> [[_MSPROP3]], i32 [[_MSLD]], i32 4
+; CHECK-NEXT:    [[TMP12:%.*]] = insertelement <16 x float> [[TMP11]], float [[TMP4]], i32 4
+; CHECK-NEXT:    [[_MSPROP5:%.*]] = insertelement <16 x i32> [[_MSPROP4]], i32 [[_MSLD]], i32 5
+; CHECK-NEXT:    [[TMP13:%.*]] = insertelement <16 x float> [[TMP12]], float [[TMP4]], i32 5
+; CHECK-NEXT:    [[_MSPROP6:%.*]] = insertelement <16 x i32> [[_MSPROP5]], i32 [[_MSLD]], i32 6
+; CHECK-NEXT:    [[TMP14:%.*]] = insertelement <16 x float> [[TMP13]], float [[TMP4]], i32 6
+; CHECK-NEXT:    [[_MSPROP7:%.*]] = insertelement <16 x i32> [[_MSPROP6]], i32 [[_MSLD]], i32 7
+; CHECK-NEXT:    [[TMP15:%.*]] = insertelement <16 x float> [[TMP14]], float [[TMP4]], i32 7
+; CHECK-NEXT:    [[_MSPROP8:%.*]] = insertelement <16 x i32> [[_MSPROP7]], i32 [[_MSLD]], i32 8
+; CHECK-NEXT:    [[TMP16:%.*]] = insertelement <16 x float> [[TMP15]], float [[TMP4]], i32 8
+; CHECK-NEXT:    [[_MSPROP9:%.*]] = insertelement <16 x i32> [[_MSPROP8]], i32 [[_MSLD]], i32 9
+; CHECK-NEXT:    [[TMP17:%.*]] = insertelement <16 x float> [[TMP16]], float [[TMP4]], i32 9
+; CHECK-NEXT:    [[_MSPROP10:%.*]] = insertelement <16 x i32> [[_MSPROP9]], i32 [[_MSLD]], i32 10
+; CHECK-NEXT:    [[TMP18:%.*]] = insertelement <16 x float> [[TMP17]], float [[TMP4]], i32 10
+; CHECK-NEXT:    [[_MSPROP11:%.*]] = insertelement <16 x i32> [[_MSPROP10]], i32 [[_MSLD]], i32 11
+; CHECK-NEXT:    [[TMP19:%.*]] = insertelement <16 x float> [[TMP18]], float [[TMP4]], i32 11
+; CHECK-NEXT:    [[_MSPROP12:%.*]] = insertelement <16 x i32> [[_MSPROP11]], i32 [[_MSLD]], i32 12
+; CHECK-NEXT:    [[TMP20:%.*]] = insertelement <16 x float> [[TMP19]], float [[TMP4]], i32 12
+; CHECK-NEXT:    [[_MSPROP13:%.*]] = insertelement <16 x i32> [[_MSPROP12]], i32 [[_MSLD]], i32 13
+; CHECK-NEXT:    [[TMP21:%.*]] = insertelement <16 x float> [[TMP20]], float [[TMP4]], i32 13
+; CHECK-NEXT:    [[_MSPROP14:%.*]] = insertelement <16 x i32> [[_MSPROP13]], i32 [[_MSLD]], i32 14
+; CHECK-NEXT:    [[TMP22:%.*]] = insertelement <16 x float> [[TMP21]], float [[TMP4]], i32 14
+; CHECK-NEXT:    [[_MSPROP15:%.*]] = insertelement <16 x i32> [[_MSPROP14]], i32 [[_MSLD]], i32 15
+; CHECK-NEXT:    [[TMP23:%.*]] = insertelement <16 x float> [[TMP22]], float [[TMP4]], i32 15
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP15]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x float> [[TMP23]]
+;
+  %res = call <16 x float> @llvm.x86.avx512.vbroadcast.ss.512(ptr %a0) ; <<16 x float>> [#uses=1]
+  ret <16 x float> %res
+}
+declare <16 x float> @llvm.x86.avx512.vbroadcast.ss.512(ptr) nounwind readonly
+
+define <8 x double> @test_x86_vbroadcast_sd_512(ptr %a0)  #0 {
+;
+; CHECK-LABEL: @test_x86_vbroadcast_sd_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP2:%.*]], label [[TMP3:%.*]], !prof [[PROF1]]
+; CHECK:       2:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       3:
+; CHECK-NEXT:    [[TMP4:%.*]] = load double, ptr [[A0:%.*]], align 8
+; CHECK-NEXT:    [[TMP5:%.*]] = ptrtoint ptr [[A0]] to i64
+; CHECK-NEXT:    [[TMP6:%.*]] = xor i64 [[TMP5]], 87960930222080
+; CHECK-NEXT:    [[TMP7:%.*]] = inttoptr i64 [[TMP6]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load i64, ptr [[TMP7]], align 8
+; CHECK-NEXT:    [[_MSPROP:%.*]] = insertelement <8 x i64> splat (i64 -1), i64 [[_MSLD]], i32 0
+; CHECK-NEXT:    [[TMP8:%.*]] = insertelement <8 x double> poison, double [[TMP4]], i32 0
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = insertelement <8 x i64> [[_MSPROP]], i64 [[_MSLD]], i32 1
+; CHECK-NEXT:    [[TMP9:%.*]] = insertelement <8 x double> [[TMP8]], double [[TMP4]], i32 1
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = insertelement <8 x i64> [[_MSPROP1]], i64 [[_MSLD]], i32 2
+; CHECK-NEXT:    [[TMP10:%.*]] = insertelement <8 x double> [[TMP9]], double [[TMP4]], i32 2
+; CHECK-NEXT:    [[_MSPROP3:%.*]] = insertelement <8 x i64> [[_MSPROP2]], i64 [[_MSLD]], i32 3
+; CHECK-NEXT:    [[TMP11:%.*]] = insertelement <8 x double> [[TMP10]], double [[TMP4]], i32 3
+; CHECK-NEXT:    [[_MSPROP4:%.*]] = insertelement <8 x i64> [[_MSPROP3]], i64 [[_MSLD]], i32 4
+; CHECK-NEXT:    [[TMP12:%.*]] = insertelement <8 x double> [[TMP11]], double [[TMP4]], i32 4
+; CHECK-NEXT:    [[_MSPROP5:%.*]] = insertelement <8 x i64> [[_MSPROP4]], i64 [[_MSLD]], i32 5
+; CHECK-NEXT:    [[TMP13:%.*]] = insertelement <8 x double> [[TMP12]], double [[TMP4]], i32 5
+; CHECK-NEXT:    [[_MSPROP6:%.*]] = insertelement <8 x i64> [[_MSPROP5]], i64 [[_MSLD]], i32 6
+; CHECK-NEXT:    [[TMP14:%.*]] = insertelement <8 x double> [[TMP13]], double [[TMP4]], i32 6
+; CHECK-NEXT:    [[_MSPROP7:%.*]] = insertelement <8 x i64> [[_MSPROP6]], i64 [[_MSLD]], i32 7
+; CHECK-NEXT:    [[TMP15:%.*]] = insertelement <8 x double> [[TMP14]], double [[TMP4]], i32 7
+; CHECK-NEXT:    store <8 x i64> [[_MSPROP7]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x double> [[TMP15]]
+;
+  %res = call <8 x double> @llvm.x86.avx512.vbroadcast.sd.512(ptr %a0) ; <<8 x double>> [#uses=1]
+  ret <8 x double> %res
+}
+declare <8 x double> @llvm.x86.avx512.vbroadcast.sd.512(ptr) nounwind readonly
+
+declare <8 x double> @llvm.x86.avx512.mask.permvar.df.512(<8 x double>, <8 x i64>, <8 x double>, i8)
+
+define <8 x double>@test_int_x86_avx512_permvar_df_512(<8 x double> %x0, <8 x i64> %x1, <8 x double> %x2)  #0 {
+; CHECK-LABEL: @test_int_x86_avx512_permvar_df_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <8 x i64> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP3]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <8 x i64> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP5:%.*]], label [[TMP6:%.*]], !prof [[PROF1]]
+; CHECK:       5:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       6:
+; CHECK-NEXT:    [[TMP7:%.*]] = call <8 x double> @llvm.x86.avx512.permvar.df.512(<8 x double> [[X0:%.*]], <8 x i64> [[X1:%.*]])
+; CHECK-NEXT:    store <8 x i64> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x double> [[TMP7]]
+;
+  %res = call <8 x double> @llvm.x86.avx512.mask.permvar.df.512(<8 x double> %x0, <8 x i64> %x1, <8 x double> %x2, i8 -1)
+  ret <8 x double> %res
+}
+
+define <8 x double>@test_int_x86_avx512_mask_permvar_df_512(<8 x double> %x0, <8 x i64> %x1, <8 x double> %x2, i8 %x3)  #0 {
+;
+; CHECK-LABEL: @test_int_x86_avx512_mask_permvar_df_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <8 x i64> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <8 x i64> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP6]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP7:%.*]], label [[TMP8:%.*]], !prof [[PROF1]]
+; CHECK:       7:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       8:
+; CHECK-NEXT:    [[TMP9:%.*]] = call <8 x double> @llvm.x86.avx512.permvar.df.512(<8 x double> [[X0:%.*]], <8 x i64> [[X1:%.*]])
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i8 [[TMP3]] to <8 x i1>
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast i8 [[X3:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP12:%.*]] = select <8 x i1> [[TMP11]], <8 x i64> zeroinitializer, <8 x i64> [[TMP4]]
+; CHECK-NEXT:    [[TMP13:%.*]] = bitcast <8 x double> [[TMP9]] to <8 x i64>
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast <8 x double> [[X2:%.*]] to <8 x i64>
+; CHECK-NEXT:    [[TMP15:%.*]] = xor <8 x i64> [[TMP13]], [[TMP14]]
+; CHECK-NEXT:    [[TMP16:%.*]] = or <8 x i64> [[TMP15]], zeroinitializer
+; CHECK-NEXT:    [[TMP17:%.*]] = or <8 x i64> [[TMP16]], [[TMP4]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP10]], <8 x i64> [[TMP17]], <8 x i64> [[TMP12]]
+; CHECK-NEXT:    [[TMP18:%.*]] = select <8 x i1> [[TMP11]], <8 x double> [[TMP9]], <8 x double> [[X2]]
+; CHECK-NEXT:    store <8 x i64> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x double> [[TMP18]]
+;
+  %res = call <8 x double> @llvm.x86.avx512.mask.permvar.df.512(<8 x double> %x0, <8 x i64> %x1, <8 x double> %x2, i8 %x3)
+  ret <8 x double> %res
+}
+
+define <8 x double>@test_int_x86_avx512_maskz_permvar_df_512(<8 x double> %x0, <8 x i64> %x1, i8 %x3)  #0 {
+;
+; CHECK-LABEL: @test_int_x86_avx512_maskz_permvar_df_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <8 x i64> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <8 x i64> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP6:%.*]], label [[TMP7:%.*]], !prof [[PROF1]]
+; CHECK:       6:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       7:
+; CHECK-NEXT:    [[TMP8:%.*]] = call <8 x double> @llvm.x86.avx512.permvar.df.512(<8 x double> [[X0:%.*]], <8 x i64> [[X1:%.*]])
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast i8 [[TMP3]] to <8 x i1>
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i8 [[X3:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP11:%.*]] = select <8 x i1> [[TMP10]], <8 x i64> zeroinitializer, <8 x i64> zeroinitializer
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <8 x double> [[TMP8]] to <8 x i64>
+; CHECK-NEXT:    [[TMP13:%.*]] = xor <8 x i64> [[TMP12]], zeroinitializer
+; CHECK-NEXT:    [[TMP14:%.*]] = or <8 x i64> [[TMP13]], zeroinitializer
+; CHECK-NEXT:    [[TMP15:%.*]] = or <8 x i64> [[TMP14]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP9]], <8 x i64> [[TMP15]], <8 x i64> [[TMP11]]
+; CHECK-NEXT:    [[TMP16:%.*]] = select <8 x i1> [[TMP10]], <8 x double> [[TMP8]], <8 x double> zeroinitializer
+; CHECK-NEXT:    store <8 x i64> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x double> [[TMP16]]
+;
+  %res = call <8 x double> @llvm.x86.avx512.mask.permvar.df.512(<8 x double> %x0, <8 x i64> %x1, <8 x double> zeroinitializer, i8 %x3)
+  ret <8 x double> %res
+}
+
+declare <8 x i64> @llvm.x86.avx512.mask.permvar.di.512(<8 x i64>, <8 x i64>, <8 x i64>, i8)
+
+define <8 x i64>@test_int_x86_avx512_permvar_di_512(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> %x2)  #0 {
+; CHECK-LABEL: @test_int_x86_avx512_permvar_di_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <8 x i64> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP3:%.*]] = call <8 x i64> @llvm.x86.avx512.permvar.di.512(<8 x i64> [[X0:%.*]], <8 x i64> [[X1:%.*]])
+; CHECK-NEXT:    store <8 x i64> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i64> [[TMP3]]
+;
+  %res = call <8 x i64> @llvm.x86.avx512.mask.permvar.di.512(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> %x2, i8 -1)
+  ret <8 x i64> %res
+}
+
+define <8 x i64>@test_int_x86_avx512_mask_permvar_di_512(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> %x2, i8 %x3)  #0 {
+;
+; CHECK-LABEL: @test_int_x86_avx512_mask_permvar_di_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <8 x i64> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP5:%.*]] = call <8 x i64> @llvm.x86.avx512.permvar.di.512(<8 x i64> [[X0:%.*]], <8 x i64> [[X1:%.*]])
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i8 [[TMP3]] to <8 x i1>
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast i8 [[X3:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP8:%.*]] = select <8 x i1> [[TMP7]], <8 x i64> [[_MSPROP]], <8 x i64> [[TMP4]]
+; CHECK-NEXT:    [[TMP9:%.*]] = xor <8 x i64> [[TMP5]], [[X2:%.*]]
+; CHECK-NEXT:    [[TMP10:%.*]] = or <8 x i64> [[TMP9]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP11:%.*]] = or <8 x i64> [[TMP10]], [[TMP4]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP6]], <8 x i64> [[TMP11]], <8 x i64> [[TMP8]]
+; CHECK-NEXT:    [[TMP12:%.*]] = select <8 x i1> [[TMP7]], <8 x i64> [[TMP5]], <8 x i64> [[X2]]
+; CHECK-NEXT:    store <8 x i64> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i64> [[TMP12]]
+;
+  %res = call <8 x i64> @llvm.x86.avx512.mask.permvar.di.512(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> %x2, i8 %x3)
+  ret <8 x i64> %res
+}
+
+define <8 x i64>@test_int_x86_avx512_maskz_permvar_di_512(<8 x i64> %x0, <8 x i64> %x1, i8 %x3)  #0 {
+;
+; CHECK-LABEL: @test_int_x86_avx512_maskz_permvar_di_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <8 x i64> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = call <8 x i64> @llvm.x86.avx512.permvar.di.512(<8 x i64> [[X0:%.*]], <8 x i64> [[X1:%.*]])
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i8 [[TMP3]] to <8 x i1>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i8 [[X3:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP7:%.*]] = select <8 x i1> [[TMP6]], <8 x i64> [[_MSPROP]], <8 x i64> zeroinitializer
+; CHECK-NEXT:    [[TMP8:%.*]] = xor <8 x i64> [[TMP4]], zeroinitializer
+; CHECK-NEXT:    [[TMP9:%.*]] = or <8 x i64> [[TMP8]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP10:%.*]] = or <8 x i64> [[TMP9]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP5]], <8 x i64> [[TMP10]], <8 x i64> [[TMP7]]
+; CHECK-NEXT:    [[TMP11:%.*]] = select <8 x i1> [[TMP6]], <8 x i64> [[TMP4]], <8 x i64> zeroinitializer
+; CHECK-NEXT:    store <8 x i64> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i64> [[TMP11]]
+;
+  %res = call <8 x i64> @llvm.x86.avx512.mask.permvar.di.512(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> zeroinitializer, i8 %x3)
+  ret <8 x i64> %res
+}
+
+declare <16 x float> @llvm.x86.avx512.mask.permvar.sf.512(<16 x float>, <16 x i32>, <16 x float>, i16)
+
+define <16 x float>@test_int_x86_avx512_permvar_sf_512(<16 x float> %x0, <16 x i32> %x1, <16 x float> %x2)  #0 {
+; CHECK-LABEL: @test_int_x86_avx512_permvar_sf_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP3]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP5:%.*]], label [[TMP6:%.*]], !prof [[PROF1]]
+; CHECK:       5:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       6:
+; CHECK-NEXT:    [[TMP7:%.*]] = call <16 x float> @llvm.x86.avx512.permvar.sf.512(<16 x float> [[X0:%.*]], <16 x i32> [[X1:%.*]])
+; CHECK-NEXT:    store <16 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x float> [[TMP7]]
+;
+  %res = call <16 x float> @llvm.x86.avx512.mask.permvar.sf.512(<16 x float> %x0, <16 x i32> %x1, <16 x float> %x2, i16 -1)
+  ret <16 x float> %res
+}
+
+define <16 x float>@test_int_x86_avx512_mask_permvar_sf_512(<16 x float> %x0, <16 x i32> %x1, <16 x float> %x2, i16 %x3)  #0 {
+;
+; CHECK-LABEL: @test_int_x86_avx512_mask_permvar_sf_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP6]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP7:%.*]], label [[TMP8:%.*]], !prof [[PROF1]]
+; CHECK:       7:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       8:
+; CHECK-NEXT:    [[TMP9:%.*]] = call <16 x float> @llvm.x86.avx512.permvar.sf.512(<16 x float> [[X0:%.*]], <16 x i32> [[X1:%.*]])
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i16 [[TMP3]] to <16 x i1>
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast i16 [[X3:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP12:%.*]] = select <16 x i1> [[TMP11]], <16 x i32> zeroinitializer, <16 x i32> [[TMP4]]
+; CHECK-NEXT:    [[TMP13:%.*]] = bitcast <16 x float> [[TMP9]] to <16 x i32>
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast <16 x float> [[X2:%.*]] to <16 x i32>
+; CHECK-NEXT:    [[TMP15:%.*]] = xor <16 x i32> [[TMP13]], [[TMP14]]
+; CHECK-NEXT:    [[TMP16:%.*]] = or <16 x i32> [[TMP15]], zeroinitializer
+; CHECK-NEXT:    [[TMP17:%.*]] = or <16 x i32> [[TMP16]], [[TMP4]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP10]], <16 x i32> [[TMP17]], <16 x i32> [[TMP12]]
+; CHECK-NEXT:    [[TMP18:%.*]] = select <16 x i1> [[TMP11]], <16 x float> [[TMP9]], <16 x float> [[X2]]
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x float> [[TMP18]]
+;
+  %res = call <16 x float> @llvm.x86.avx512.mask.permvar.sf.512(<16 x float> %x0, <16 x i32> %x1, <16 x float> %x2, i16 %x3)
+  ret <16 x float> %res
+}
+
+define <16 x float>@test_int_x86_avx512_maskz_permvar_sf_512(<16 x float> %x0, <16 x i32> %x1, i16 %x3)  #0 {
+;
+; CHECK-LABEL: @test_int_x86_avx512_maskz_permvar_sf_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP6:%.*]], label [[TMP7:%.*]], !prof [[PROF1]]
+; CHECK:       6:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       7:
+; CHECK-NEXT:    [[TMP8:%.*]] = call <16 x float> @llvm.x86.avx512.permvar.sf.512(<16 x float> [[X0:%.*]], <16 x i32> [[X1:%.*]])
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast i16 [[TMP3]] to <16 x i1>
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i16 [[X3:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP11:%.*]] = select <16 x i1> [[TMP10]], <16 x i32> zeroinitializer, <16 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <16 x float> [[TMP8]] to <16 x i32>
+; CHECK-NEXT:    [[TMP13:%.*]] = xor <16 x i32> [[TMP12]], zeroinitializer
+; CHECK-NEXT:    [[TMP14:%.*]] = or <16 x i32> [[TMP13]], zeroinitializer
+; CHECK-NEXT:    [[TMP15:%.*]] = or <16 x i32> [[TMP14]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP9]], <16 x i32> [[TMP15]], <16 x i32> [[TMP11]]
+; CHECK-NEXT:    [[TMP16:%.*]] = select <16 x i1> [[TMP10]], <16 x float> [[TMP8]], <16 x float> zeroinitializer
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x float> [[TMP16]]
+;
+  %res = call <16 x float> @llvm.x86.avx512.mask.permvar.sf.512(<16 x float> %x0, <16 x i32> %x1, <16 x float> zeroinitializer, i16 %x3)
+  ret <16 x float> %res
+}
+
+declare <16 x i32> @llvm.x86.avx512.mask.permvar.si.512(<16 x i32>, <16 x i32>, <16 x i32>, i16)
+
+define <16 x i32>@test_int_x86_avx512_permvar_si_512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x2)  #0 {
+; CHECK-LABEL: @test_int_x86_avx512_permvar_si_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <16 x i32> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP3:%.*]] = call <16 x i32> @llvm.x86.avx512.permvar.si.512(<16 x i32> [[X0:%.*]], <16 x i32> [[X1:%.*]])
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i32> [[TMP3]]
+;
+  %res = call <16 x i32> @llvm.x86.avx512.mask.permvar.si.512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x2, i16 -1)
+  ret <16 x i32> %res
+}
+
+define <16 x i32>@test_int_x86_avx512_mask_permvar_si_512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x2, i16 %x3)  #0 {
+;
+; CHECK-LABEL: @test_int_x86_avx512_mask_permvar_si_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <16 x i32> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP5:%.*]] = call <16 x i32> @llvm.x86.avx512.permvar.si.512(<16 x i32> [[X0:%.*]], <16 x i32> [[X1:%.*]])
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i16 [[TMP3]] to <16 x i1>
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast i16 [[X3:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP8:%.*]] = select <16 x i1> [[TMP7]], <16 x i32> [[_MSPROP]], <16 x i32> [[TMP4]]
+; CHECK-NEXT:    [[TMP9:%.*]] = xor <16 x i32> [[TMP5]], [[X2:%.*]]
+; CHECK-NEXT:    [[TMP10:%.*]] = or <16 x i32> [[TMP9]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP11:%.*]] = or <16 x i32> [[TMP10]], [[TMP4]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP6]], <16 x i32> [[TMP11]], <16 x i32> [[TMP8]]
+; CHECK-NEXT:    [[TMP12:%.*]] = select <16 x i1> [[TMP7]], <16 x i32> [[TMP5]], <16 x i32> [[X2]]
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i32> [[TMP12]]
+;
+  %res = call <16 x i32> @llvm.x86.avx512.mask.permvar.si.512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x2, i16 %x3)
+  ret <16 x i32> %res
+}
+
+define <16 x i32>@test_int_x86_avx512_maskz_permvar_si_512(<16 x i32> %x0, <16 x i32> %x1, i16 %x3)  #0 {
+;
+; CHECK-LABEL: @test_int_x86_avx512_maskz_permvar_si_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <16 x i32> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = call <16 x i32> @llvm.x86.avx512.permvar.si.512(<16 x i32> [[X0:%.*]], <16 x i32> [[X1:%.*]])
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i16 [[TMP3]] to <16 x i1>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i16 [[X3:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP7:%.*]] = select <16 x i1> [[TMP6]], <16 x i32> [[_MSPROP]], <16 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP8:%.*]] = xor <16 x i32> [[TMP4]], zeroinitializer
+; CHECK-NEXT:    [[TMP9:%.*]] = or <16 x i32> [[TMP8]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP10:%.*]] = or <16 x i32> [[TMP9]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP5]], <16 x i32> [[TMP10]], <16 x i32> [[TMP7]]
+; CHECK-NEXT:    [[TMP11:%.*]] = select <16 x i1> [[TMP6]], <16 x i32> [[TMP4]], <16 x i32> zeroinitializer
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i32> [[TMP11]]
+;
+  %res = call <16 x i32> @llvm.x86.avx512.mask.permvar.si.512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> zeroinitializer, i16 %x3)
+  ret <16 x i32> %res
+}
+
+declare <16 x i32> @llvm.x86.avx512.mask.pternlog.d.512(<16 x i32>, <16 x i32>, <16 x i32>, i32, i16)
+
+define <16 x i32>@test_int_x86_avx512_pternlog_d_512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x2)  #0 {
+; CHECK-LABEL: @test_int_x86_avx512_pternlog_d_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <16 x i32> [[TMP3]] to i512
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i512 [[TMP6]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR3]], label [[TMP7:%.*]], label [[TMP8:%.*]], !prof [[PROF1]]
+; CHECK:       7:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       8:
+; CHECK-NEXT:    [[TMP9:%.*]] = call <16 x i32> @llvm.x86.avx512.pternlog.d.512(<16 x i32> [[X0:%.*]], <16 x i32> [[X1:%.*]], <16 x i32> [[X2:%.*]], i32 33)
+; CHECK-NEXT:    store <16 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i32> [[TMP9]]
+;
+  %res = call <16 x i32> @llvm.x86.avx512.mask.pternlog.d.512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x2, i32 33, i16 -1)
+  ret <16 x i32> %res
+}
+
+define <16 x i32>@test_int_x86_avx512_mask_pternlog_d_512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x2, i16 %x4)  #0 {
+;
+; CHECK-LABEL: @test_int_x86_avx512_mask_pternlog_d_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP6]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <16 x i32> [[TMP3]] to i512
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i512 [[TMP7]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR3]], label [[TMP8:%.*]], label [[TMP9:%.*]], !prof [[PROF1]]
+; CHECK:       8:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       9:
+; CHECK-NEXT:    [[TMP10:%.*]] = call <16 x i32> @llvm.x86.avx512.pternlog.d.512(<16 x i32> [[X0:%.*]], <16 x i32> [[X1:%.*]], <16 x i32> [[X2:%.*]], i32 33)
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast i16 [[TMP4]] to <16 x i1>
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast i16 [[X4:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP13:%.*]] = select <16 x i1> [[TMP12]], <16 x i32> zeroinitializer, <16 x i32> [[TMP1]]
+; CHECK-NEXT:    [[TMP14:%.*]] = xor <16 x i32> [[TMP10]], [[X0]]
+; CHECK-NEXT:    [[TMP15:%.*]] = or <16 x i32> [[TMP14]], zeroinitializer
+; CHECK-NEXT:    [[TMP16:%.*]] = or <16 x i32> [[TMP15]], [[TMP1]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP11]], <16 x i32> [[TMP16]], <16 x i32> [[TMP13]]
+; CHECK-NEXT:    [[TMP17:%.*]] = select <16 x i1> [[TMP12]], <16 x i32> [[TMP10]], <16 x i32> [[X0]]
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i32> [[TMP17]]
+;
+  %res = call <16 x i32> @llvm.x86.avx512.mask.pternlog.d.512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x2, i32 33, i16 %x4)
+  ret <16 x i32> %res
+}
+
+declare <16 x i32> @llvm.x86.avx512.maskz.pternlog.d.512(<16 x i32>, <16 x i32>, <16 x i32>, i32, i16)
+
+define <16 x i32>@test_int_x86_avx512_maskz_pternlog_d_512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x2, i16 %x4)  #0 {
+;
+; CHECK-LABEL: @test_int_x86_avx512_maskz_pternlog_d_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP6]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <16 x i32> [[TMP3]] to i512
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i512 [[TMP7]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR3]], label [[TMP8:%.*]], label [[TMP9:%.*]], !prof [[PROF1]]
+; CHECK:       8:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       9:
+; CHECK-NEXT:    [[TMP10:%.*]] = call <16 x i32> @llvm.x86.avx512.pternlog.d.512(<16 x i32> [[X0:%.*]], <16 x i32> [[X1:%.*]], <16 x i32> [[X2:%.*]], i32 33)
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast i16 [[TMP4]] to <16 x i1>
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast i16 [[X4:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP13:%.*]] = select <16 x i1> [[TMP12]], <16 x i32> zeroinitializer, <16 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP14:%.*]] = xor <16 x i32> [[TMP10]], zeroinitializer
+; CHECK-NEXT:    [[TMP15:%.*]] = or <16 x i32> [[TMP14]], zeroinitializer
+; CHECK-NEXT:    [[TMP16:%.*]] = or <16 x i32> [[TMP15]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP11]], <16 x i32> [[TMP16]], <16 x i32> [[TMP13]]
+; CHECK-NEXT:    [[TMP17:%.*]] = select <16 x i1> [[TMP12]], <16 x i32> [[TMP10]], <16 x i32> zeroinitializer
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i32> [[TMP17]]
+;
+  %res = call <16 x i32> @llvm.x86.avx512.maskz.pternlog.d.512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x2, i32 33, i16 %x4)
+  ret <16 x i32> %res
+}
+
+declare <8 x i64> @llvm.x86.avx512.mask.pternlog.q.512(<8 x i64>, <8 x i64>, <8 x i64>, i32, i8)
+
+define <8 x i64>@test_int_x86_avx512_pternlog_q_512(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> %x2)  #0 {
+; CHECK-LABEL: @test_int_x86_avx512_pternlog_q_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <8 x i64> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <8 x i64> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <8 x i64> [[TMP3]] to i512
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i512 [[TMP6]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR3]], label [[TMP7:%.*]], label [[TMP8:%.*]], !prof [[PROF1]]
+; CHECK:       7:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       8:
+; CHECK-NEXT:    [[TMP9:%.*]] = call <8 x i64> @llvm.x86.avx512.pternlog.q.512(<8 x i64> [[X0:%.*]], <8 x i64> [[X1:%.*]], <8 x i64> [[X2:%.*]], i32 33)
+; CHECK-NEXT:    store <8 x i64> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i64> [[TMP9]]
+;
+  %res = call <8 x i64> @llvm.x86.avx512.mask.pternlog.q.512(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> %x2, i32 33, i8 -1)
+  ret <8 x i64> %res
+}
+
+define <8 x i64>@test_int_x86_avx512_mask_pternlog_q_512(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> %x2, i8 %x4)  #0 {
+;
+; CHECK-LABEL: @test_int_x86_avx512_mask_pternlog_q_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <8 x i64> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <8 x i64> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP6]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <8 x i64> [[TMP3]] to i512
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i512 [[TMP7]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR3]], label [[TMP8:%.*]], label [[TMP9:%.*]], !prof [[PROF1]]
+; CHECK:       8:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       9:
+; CHECK-NEXT:    [[TMP10:%.*]] = call <8 x i64> @llvm.x86.avx512.pternlog.q.512(<8 x i64> [[X0:%.*]], <8 x i64> [[X1:%.*]], <8 x i64> [[X2:%.*]], i32 33)
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast i8 [[TMP4]] to <8 x i1>
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast i8 [[X4:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP13:%.*]] = select <8 x i1> [[TMP12]], <8 x i64> zeroinitializer, <8 x i64> [[TMP1]]
+; CHECK-NEXT:    [[TMP14:%.*]] = xor <8 x i64> [[TMP10]], [[X0]]
+; CHECK-NEXT:    [[TMP15:%.*]] = or <8 x i64> [[TMP14]], zeroinitializer
+; CHECK-NEXT:    [[TMP16:%.*]] = or <8 x i64> [[TMP15]], [[TMP1]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP11]], <8 x i64> [[TMP16]], <8 x i64> [[TMP13]]
+; CHECK-NEXT:    [[TMP17:%.*]] = select <8 x i1> [[TMP12]], <8 x i64> [[TMP10]], <8 x i64> [[X0]]
+; CHECK-NEXT:    store <8 x i64> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i64> [[TMP17]]
+;
+  %res = call <8 x i64> @llvm.x86.avx512.mask.pternlog.q.512(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> %x2, i32 33, i8 %x4)
+  ret <8 x i64> %res
+}
+
+declare <8 x i64> @llvm.x86.avx512.maskz.pternlog.q.512(<8 x i64>, <8 x i64>, <8 x i64>, i32, i8)
+
+define <8 x i64>@test_int_x86_avx512_maskz_pternlog_q_512(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> %x2, i8 %x4)  #0 {
+;
+; CHECK-LABEL: @test_int_x86_avx512_maskz_pternlog_q_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <8 x i64> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <8 x i64> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP6]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <8 x i64> [[TMP3]] to i512
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i512 [[TMP7]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR3]], label [[TMP8:%.*]], label [[TMP9:%.*]], !prof [[PROF1]]
+; CHECK:       8:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       9:
+; CHECK-NEXT:    [[TMP10:%.*]] = call <8 x i64> @llvm.x86.avx512.pternlog.q.512(<8 x i64> [[X0:%.*]], <8 x i64> [[X1:%.*]], <8 x i64> [[X2:%.*]], i32 33)
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast i8 [[TMP4]] to <8 x i1>
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast i8 [[X4:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP13:%.*]] = select <8 x i1> [[TMP12]], <8 x i64> zeroinitializer, <8 x i64> zeroinitializer
+; CHECK-NEXT:    [[TMP14:%.*]] = xor <8 x i64> [[TMP10]], zeroinitializer
+; CHECK-NEXT:    [[TMP15:%.*]] = or <8 x i64> [[TMP14]], zeroinitializer
+; CHECK-NEXT:    [[TMP16:%.*]] = or <8 x i64> [[TMP15]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP11]], <8 x i64> [[TMP16]], <8 x i64> [[TMP13]]
+; CHECK-NEXT:    [[TMP17:%.*]] = select <8 x i1> [[TMP12]], <8 x i64> [[TMP10]], <8 x i64> zeroinitializer
+; CHECK-NEXT:    store <8 x i64> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i64> [[TMP17]]
+;
+  %res = call <8 x i64> @llvm.x86.avx512.maskz.pternlog.q.512(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> %x2, i32 33, i8 %x4)
+  ret <8 x i64> %res
+}
+
+declare <16 x i32> @llvm.x86.avx512.mask.vpermi2var.d.512(<16 x i32>, <16 x i32>, <16 x i32>, i16)
+
+define <16 x i32>@test_int_x86_avx512_vpermi2var_d_512(<16 x i32> %x0, <16 x i32> %x1, ptr %x2p, <16 x i32> %x4)  #0 {
+; CHECK-LABEL: @test_int_x86_avx512_vpermi2var_d_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 136) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP5:%.*]], label [[TMP6:%.*]], !prof [[PROF1]]
+; CHECK:       5:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       6:
+; CHECK-NEXT:    [[X2:%.*]] = load <16 x i32>, ptr [[X2P:%.*]], align 64
+; CHECK-NEXT:    [[TMP7:%.*]] = ptrtoint ptr [[X2P]] to i64
+; CHECK-NEXT:    [[TMP8:%.*]] = xor i64 [[TMP7]], 87960930222080
+; CHECK-NEXT:    [[TMP9:%.*]] = inttoptr i64 [[TMP8]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <16 x i32>, ptr [[TMP9]], align 64
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <16 x i32> [[TMP2]], [[TMP3]]
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <16 x i32> [[_MSPROP]], [[TMP4]]
+; CHECK-NEXT:    [[TMP10:%.*]] = call <16 x i32> @llvm.x86.avx512.vpermi2var.d.512(<16 x i32> [[X0:%.*]], <16 x i32> [[X1:%.*]], <16 x i32> [[X4:%.*]])
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP1]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i32> [[TMP10]]
+;
+  %x2 = load <16 x i32>, ptr %x2p
+  %res = call <16 x i32> @llvm.x86.avx512.mask.vpermi2var.d.512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x4, i16 -1)
+  ret <16 x i32> %res
+}
+
+define <16 x i32>@test_int_x86_avx512_mask_vpermi2var_d_512(<16 x i32> %x0, <16 x i32> %x1, ptr %x2p, <16 x i32> %x4, i16 %x3)  #0 {
+;
+; CHECK-LABEL: @test_int_x86_avx512_mask_vpermi2var_d_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 200) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP5:%.*]], label [[TMP6:%.*]], !prof [[PROF1]]
+; CHECK:       5:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       6:
+; CHECK-NEXT:    [[X2:%.*]] = load <16 x i32>, ptr [[X2P:%.*]], align 64
+; CHECK-NEXT:    [[TMP7:%.*]] = ptrtoint ptr [[X2P]] to i64
+; CHECK-NEXT:    [[TMP8:%.*]] = xor i64 [[TMP7]], 87960930222080
+; CHECK-NEXT:    [[TMP9:%.*]] = inttoptr i64 [[TMP8]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <16 x i32>, ptr [[TMP9]], align 64
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <16 x i32> [[TMP2]], [[TMP3]]
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <16 x i32> [[_MSPROP]], [[_MSLD]]
+; CHECK-NEXT:    [[TMP10:%.*]] = call <16 x i32> @llvm.x86.avx512.vpermi2var.d.512(<16 x i32> [[X0:%.*]], <16 x i32> [[X1:%.*]], <16 x i32> [[X2]])
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast i16 [[TMP4]] to <16 x i1>
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast i16 [[X3:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP13:%.*]] = select <16 x i1> [[TMP12]], <16 x i32> [[_MSPROP1]], <16 x i32> [[TMP3]]
+; CHECK-NEXT:    [[TMP14:%.*]] = xor <16 x i32> [[TMP10]], [[X1]]
+; CHECK-NEXT:    [[TMP15:%.*]] = or <16 x i32> [[TMP14]], [[_MSPROP1]]
+; CHECK-NEXT:    [[TMP16:%.*]] = or <16 x i32> [[TMP15]], [[TMP3]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP11]], <16 x i32> [[TMP16]], <16 x i32> [[TMP13]]
+; CHECK-NEXT:    [[TMP17:%.*]] = select <16 x i1> [[TMP12]], <16 x i32> [[TMP10]], <16 x i32> [[X1]]
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i32> [[TMP17]]
+;
+  %x2 = load <16 x i32>, ptr %x2p
+  %res = call <16 x i32> @llvm.x86.avx512.mask.vpermi2var.d.512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x2, i16 %x3)
+  ret <16 x i32> %res
+}
+
+declare <8 x double> @llvm.x86.avx512.mask.vpermi2var.pd.512(<8 x double>, <8 x i64>, <8 x double>, i8)
+
+define <8 x double>@test_int_x86_avx512_vpermi2var_pd_512(<8 x double> %x0, <8 x i64> %x1, <8 x double> %x2)  #0 {
+; CHECK-LABEL: @test_int_x86_avx512_vpermi2var_pd_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <8 x i64> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <8 x i64> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <8 x i64> [[TMP3]] to i512
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i512 [[TMP6]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR3]], label [[TMP7:%.*]], label [[TMP8:%.*]], !prof [[PROF1]]
+; CHECK:       7:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       8:
+; CHECK-NEXT:    [[TMP9:%.*]] = call <8 x double> @llvm.x86.avx512.vpermi2var.pd.512(<8 x double> [[X0:%.*]], <8 x i64> [[X1:%.*]], <8 x double> [[X2:%.*]])
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast <8 x i64> [[X1]] to <8 x double>
+; CHECK-NEXT:    store <8 x i64> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x double> [[TMP9]]
+;
+  %res = call <8 x double> @llvm.x86.avx512.mask.vpermi2var.pd.512(<8 x double> %x0, <8 x i64> %x1, <8 x double> %x2, i8 -1)
+  ret <8 x double> %res
+}
+
+define <8 x double>@test_int_x86_avx512_mask_vpermi2var_pd_512(<8 x double> %x0, <8 x i64> %x1, <8 x double> %x2, i8 %x3)  #0 {
+;
+; CHECK-LABEL: @test_int_x86_avx512_mask_vpermi2var_pd_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <8 x i64> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <8 x i64> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP6]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <8 x i64> [[TMP3]] to i512
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i512 [[TMP7]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR3]], label [[TMP8:%.*]], label [[TMP9:%.*]], !prof [[PROF1]]
+; CHECK:       8:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       9:
+; CHECK-NEXT:    [[TMP10:%.*]] = call <8 x double> @llvm.x86.avx512.vpermi2var.pd.512(<8 x double> [[X0:%.*]], <8 x i64> [[X1:%.*]], <8 x double> [[X2:%.*]])
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <8 x i64> [[X1]] to <8 x double>
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast i8 [[TMP4]] to <8 x i1>
+; CHECK-NEXT:    [[TMP13:%.*]] = bitcast i8 [[X3:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP14:%.*]] = select <8 x i1> [[TMP13]], <8 x i64> zeroinitializer, <8 x i64> [[TMP2]]
+; CHECK-NEXT:    [[TMP15:%.*]] = bitcast <8 x double> [[TMP10]] to <8 x i64>
+; CHECK-NEXT:    [[TMP16:%.*]] = bitcast <8 x double> [[TMP11]] to <8 x i64>
+; CHECK-NEXT:    [[TMP17:%.*]] = xor <8 x i64> [[TMP15]], [[TMP16]]
+; CHECK-NEXT:    [[TMP18:%.*]] = or <8 x i64> [[TMP17]], zeroinitializer
+; CHECK-NEXT:    [[TMP19:%.*]] = or <8 x i64> [[TMP18]], [[TMP2]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP12]], <8 x i64> [[TMP19]], <8 x i64> [[TMP14]]
+; CHECK-NEXT:    [[TMP20:%.*]] = select <8 x i1> [[TMP13]], <8 x double> [[TMP10]], <8 x double> [[TMP11]]
+; CHECK-NEXT:    store <8 x i64> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x double> [[TMP20]]
+;
+  %res = call <8 x double> @llvm.x86.avx512.mask.vpermi2var.pd.512(<8 x double> %x0, <8 x i64> %x1, <8 x double> %x2, i8 %x3)
+  ret <8 x double> %res
+}
+
+declare <16 x float> @llvm.x86.avx512.mask.vpermi2var.ps.512(<16 x float>, <16 x i32>, <16 x float>, i16)
+
+define <16 x float>@test_int_x86_avx512_vpermi2var_ps_512(<16 x float> %x0, <16 x i32> %x1, <16 x float> %x2)  #0 {
+; CHECK-LABEL: @test_int_x86_avx512_vpermi2var_ps_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <16 x i32> [[TMP3]] to i512
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i512 [[TMP6]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR3]], label [[TMP7:%.*]], label [[TMP8:%.*]], !prof [[PROF1]]
+; CHECK:       7:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       8:
+; CHECK-NEXT:    [[TMP9:%.*]] = call <16 x float> @llvm.x86.avx512.vpermi2var.ps.512(<16 x float> [[X0:%.*]], <16 x i32> [[X1:%.*]], <16 x float> [[X2:%.*]])
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast <16 x i32> [[X1]] to <16 x float>
+; CHECK-NEXT:    store <16 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x float> [[TMP9]]
+;
+  %res = call <16 x float> @llvm.x86.avx512.mask.vpermi2var.ps.512(<16 x float> %x0, <16 x i32> %x1, <16 x float> %x2, i16 -1)
+  ret <16 x float> %res
+}
+
+define <16 x float>@test_int_x86_avx512_mask_vpermi2var_ps_512(<16 x float> %x0, <16 x i32> %x1, <16 x float> %x2, i16 %x3)  #0 {
+;
+; CHECK-LABEL: @test_int_x86_avx512_mask_vpermi2var_ps_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP6]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <16 x i32> [[TMP3]] to i512
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i512 [[TMP7]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR3]], label [[TMP8:%.*]], label [[TMP9:%.*]], !prof [[PROF1]]
+; CHECK:       8:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       9:
+; CHECK-NEXT:    [[TMP10:%.*]] = call <16 x float> @llvm.x86.avx512.vpermi2var.ps.512(<16 x float> [[X0:%.*]], <16 x i32> [[X1:%.*]], <16 x float> [[X2:%.*]])
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <16 x i32> [[X1]] to <16 x float>
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast i16 [[TMP4]] to <16 x i1>
+; CHECK-NEXT:    [[TMP13:%.*]] = bitcast i16 [[X3:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP14:%.*]] = select <16 x i1> [[TMP13]], <16 x i32> zeroinitializer, <16 x i32> [[TMP2]]
+; CHECK-NEXT:    [[TMP15:%.*]] = bitcast <16 x float> [[TMP10]] to <16 x i32>
+; CHECK-NEXT:    [[TMP16:%.*]] = bitcast <16 x float> [[TMP11]] to <16 x i32>
+; CHECK-NEXT:    [[TMP17:%.*]] = xor <16 x i32> [[TMP15]], [[TMP16]]
+; CHECK-NEXT:    [[TMP18:%.*]] = or <16 x i32> [[TMP17]], zeroinitializer
+; CHECK-NEXT:    [[TMP19:%.*]] = or <16 x i32> [[TMP18]], [[TMP2]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP12]], <16 x i32> [[TMP19]], <16 x i32> [[TMP14]]
+; CHECK-NEXT:    [[TMP20:%.*]] = select <16 x i1> [[TMP13]], <16 x float> [[TMP10]], <16 x float> [[TMP11]]
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x float> [[TMP20]]
+;
+  %res = call <16 x float> @llvm.x86.avx512.mask.vpermi2var.ps.512(<16 x float> %x0, <16 x i32> %x1, <16 x float> %x2, i16 %x3)
+  ret <16 x float> %res
+}
+
+declare <8 x i64> @llvm.x86.avx512.mask.vpermi2var.q.512(<8 x i64>, <8 x i64>, <8 x i64>, i8)
+
+define <8 x i64>@test_int_x86_avx512_vpermi2var_q_512(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> %x2)  #0 {
+; CHECK-LABEL: @test_int_x86_avx512_vpermi2var_q_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <8 x i64> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <8 x i64> [[_MSPROP]], [[TMP3]]
+; CHECK-NEXT:    [[TMP4:%.*]] = call <8 x i64> @llvm.x86.avx512.vpermi2var.q.512(<8 x i64> [[X0:%.*]], <8 x i64> [[X1:%.*]], <8 x i64> [[X2:%.*]])
+; CHECK-NEXT:    store <8 x i64> [[_MSPROP1]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i64> [[TMP4]]
+;
+  %res = call <8 x i64> @llvm.x86.avx512.mask.vpermi2var.q.512(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> %x2, i8 -1)
+  ret <8 x i64> %res
+}
+
+define <8 x i64>@test_int_x86_avx512_mask_vpermi2var_q_512(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> %x2, i8 %x3)  #0 {
+;
+; CHECK-LABEL: @test_int_x86_avx512_mask_vpermi2var_q_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <8 x i64> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <8 x i64> [[_MSPROP]], [[TMP3]]
+; CHECK-NEXT:    [[TMP5:%.*]] = call <8 x i64> @llvm.x86.avx512.vpermi2var.q.512(<8 x i64> [[X0:%.*]], <8 x i64> [[X1:%.*]], <8 x i64> [[X2:%.*]])
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i8 [[TMP4]] to <8 x i1>
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast i8 [[X3:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP8:%.*]] = select <8 x i1> [[TMP7]], <8 x i64> [[_MSPROP1]], <8 x i64> [[TMP2]]
+; CHECK-NEXT:    [[TMP9:%.*]] = xor <8 x i64> [[TMP5]], [[X1]]
+; CHECK-NEXT:    [[TMP10:%.*]] = or <8 x i64> [[TMP9]], [[_MSPROP1]]
+; CHECK-NEXT:    [[TMP11:%.*]] = or <8 x i64> [[TMP10]], [[TMP2]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP6]], <8 x i64> [[TMP11]], <8 x i64> [[TMP8]]
+; CHECK-NEXT:    [[TMP12:%.*]] = select <8 x i1> [[TMP7]], <8 x i64> [[TMP5]], <8 x i64> [[X1]]
+; CHECK-NEXT:    store <8 x i64> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i64> [[TMP12]]
+;
+  %res = call <8 x i64> @llvm.x86.avx512.mask.vpermi2var.q.512(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> %x2, i8 %x3)
+  ret <8 x i64> %res
+}
+
+declare <16 x i32> @llvm.x86.avx512.maskz.vpermt2var.d.512(<16 x i32>, <16 x i32>, <16 x i32>, i16)
+
+define <16 x i32>@test_int_x86_avx512_maskz_vpermt2var_d_512(<16 x i32> %x0, <16 x i32> %x1, ptr %x2p, i16 %x3)  #0 {
+;
+; CHECK-LABEL: @test_int_x86_avx512_maskz_vpermt2var_d_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 136) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP5:%.*]], label [[TMP6:%.*]], !prof [[PROF1]]
+; CHECK:       5:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       6:
+; CHECK-NEXT:    [[X2:%.*]] = load <16 x i32>, ptr [[X2P:%.*]], align 64
+; CHECK-NEXT:    [[TMP7:%.*]] = ptrtoint ptr [[X2P]] to i64
+; CHECK-NEXT:    [[TMP8:%.*]] = xor i64 [[TMP7]], 87960930222080
+; CHECK-NEXT:    [[TMP9:%.*]] = inttoptr i64 [[TMP8]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <16 x i32>, ptr [[TMP9]], align 64
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <16 x i32> [[TMP2]], [[TMP3]]
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <16 x i32> [[_MSPROP]], [[_MSLD]]
+; CHECK-NEXT:    [[TMP10:%.*]] = call <16 x i32> @llvm.x86.avx512.vpermi2var.d.512(<16 x i32> [[X1:%.*]], <16 x i32> [[X0:%.*]], <16 x i32> [[X2]])
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast i16 [[TMP4]] to <16 x i1>
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast i16 [[X3:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP13:%.*]] = select <16 x i1> [[TMP12]], <16 x i32> [[_MSPROP1]], <16 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP14:%.*]] = xor <16 x i32> [[TMP10]], zeroinitializer
+; CHECK-NEXT:    [[TMP15:%.*]] = or <16 x i32> [[TMP14]], [[_MSPROP1]]
+; CHECK-NEXT:    [[TMP16:%.*]] = or <16 x i32> [[TMP15]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP11]], <16 x i32> [[TMP16]], <16 x i32> [[TMP13]]
+; CHECK-NEXT:    [[TMP17:%.*]] = select <16 x i1> [[TMP12]], <16 x i32> [[TMP10]], <16 x i32> zeroinitializer
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i32> [[TMP17]]
+;
+  %x2 = load <16 x i32>, ptr %x2p
+  %res = call <16 x i32> @llvm.x86.avx512.maskz.vpermt2var.d.512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x2, i16 %x3)
+  ret <16 x i32> %res
+}
+
+declare <8 x double> @llvm.x86.avx512.maskz.vpermt2var.pd.512(<8 x i64>, <8 x double>, <8 x double>, i8)
+
+define <8 x double>@test_int_x86_avx512_maskz_vpermt2var_pd_512(<8 x i64> %x0, <8 x double> %x1, ptr %x2ptr, i8 %x3, <8 x double> %extra_param)  #0 {
+;
+; CHECK-LABEL: @test_int_x86_avx512_maskz_vpermt2var_pd_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    [[TMP5:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 144) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 136) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP6:%.*]], label [[TMP13:%.*]], !prof [[PROF1]]
+; CHECK:       6:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       7:
+; CHECK-NEXT:    [[X2S:%.*]] = load double, ptr [[X2PTR:%.*]], align 8
+; CHECK-NEXT:    [[TMP7:%.*]] = ptrtoint ptr [[X2PTR]] to i64
+; CHECK-NEXT:    [[TMP8:%.*]] = xor i64 [[TMP7]], 87960930222080
+; CHECK-NEXT:    [[TMP9:%.*]] = inttoptr i64 [[TMP8]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load i64, ptr [[TMP9]], align 8
+; CHECK-NEXT:    [[_MSPROP:%.*]] = insertelement <8 x i64> [[TMP5]], i64 [[_MSLD]], i32 0
+; CHECK-NEXT:    [[X2INS:%.*]] = insertelement <8 x double> [[EXTRA_PARAM:%.*]], double [[X2S]], i32 0
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = shufflevector <8 x i64> [[_MSPROP]], <8 x i64> [[TMP5]], <8 x i32> zeroinitializer
+; CHECK-NEXT:    [[X2:%.*]] = shufflevector <8 x double> [[X2INS]], <8 x double> [[EXTRA_PARAM]], <8 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast <8 x i64> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i512 [[TMP10]], 0
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <8 x i64> [[TMP3]] to i512
+; CHECK-NEXT:    [[_MSCMP3:%.*]] = icmp ne i512 [[TMP11]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP2]], [[_MSCMP3]]
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <8 x i64> [[_MSPROP1]] to i512
+; CHECK-NEXT:    [[_MSCMP4:%.*]] = icmp ne i512 [[TMP12]], 0
+; CHECK-NEXT:    [[_MSOR5:%.*]] = or i1 [[_MSOR]], [[_MSCMP4]]
+; CHECK-NEXT:    br i1 [[_MSOR5]], label [[TMP14:%.*]], label [[TMP24:%.*]], !prof [[PROF1]]
+; CHECK:       14:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       15:
+; CHECK-NEXT:    [[TMP15:%.*]] = call <8 x double> @llvm.x86.avx512.vpermi2var.pd.512(<8 x double> [[X1:%.*]], <8 x i64> [[X0:%.*]], <8 x double> [[X2]])
+; CHECK-NEXT:    [[TMP16:%.*]] = bitcast i8 [[TMP4]] to <8 x i1>
+; CHECK-NEXT:    [[TMP17:%.*]] = bitcast i8 [[X3:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP18:%.*]] = select <8 x i1> [[TMP17]], <8 x i64> zeroinitializer, <8 x i64> zeroinitializer
+; CHECK-NEXT:    [[TMP19:%.*]] = bitcast <8 x double> [[TMP15]] to <8 x i64>
+; CHECK-NEXT:    [[TMP20:%.*]] = xor <8 x i64> [[TMP19]], zeroinitializer
+; CHECK-NEXT:    [[TMP21:%.*]] = or <8 x i64> [[TMP20]], zeroinitializer
+; CHECK-NEXT:    [[TMP22:%.*]] = or <8 x i64> [[TMP21]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP16]], <8 x i64> [[TMP22]], <8 x i64> [[TMP18]]
+; CHECK-NEXT:    [[TMP23:%.*]] = select <8 x i1> [[TMP17]], <8 x double> [[TMP15]], <8 x double> zeroinitializer
+; CHECK-NEXT:    store <8 x i64> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x double> [[TMP23]]
+;
+  %x2s = load double, ptr %x2ptr
+  %x2ins = insertelement <8 x double> %extra_param, double %x2s, i32 0
+  %x2 = shufflevector <8 x double> %x2ins, <8 x double> %extra_param, <8 x i32> zeroinitializer
+  %res = call <8 x double> @llvm.x86.avx512.maskz.vpermt2var.pd.512(<8 x i64> %x0, <8 x double> %x1, <8 x double> %x2, i8 %x3)
+  ret <8 x double> %res
+}
+
+declare <16 x float> @llvm.x86.avx512.maskz.vpermt2var.ps.512(<16 x i32>, <16 x float>, <16 x float>, i16)
+
+define <16 x float>@test_int_x86_avx512_maskz_vpermt2var_ps_512(<16 x i32> %x0, <16 x float> %x1, <16 x float> %x2, i16 %x3)  #0 {
+;
+; CHECK-LABEL: @test_int_x86_avx512_maskz_vpermt2var_ps_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP6]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <16 x i32> [[TMP3]] to i512
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i512 [[TMP7]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR3]], label [[TMP8:%.*]], label [[TMP9:%.*]], !prof [[PROF1]]
+; CHECK:       8:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       9:
+; CHECK-NEXT:    [[TMP10:%.*]] = call <16 x float> @llvm.x86.avx512.vpermi2var.ps.512(<16 x float> [[X1:%.*]], <16 x i32> [[X0:%.*]], <16 x float> [[X2:%.*]])
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast i16 [[TMP4]] to <16 x i1>
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast i16 [[X3:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP13:%.*]] = select <16 x i1> [[TMP12]], <16 x i32> zeroinitializer, <16 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast <16 x float> [[TMP10]] to <16 x i32>
+; CHECK-NEXT:    [[TMP15:%.*]] = xor <16 x i32> [[TMP14]], zeroinitializer
+; CHECK-NEXT:    [[TMP16:%.*]] = or <16 x i32> [[TMP15]], zeroinitializer
+; CHECK-NEXT:    [[TMP17:%.*]] = or <16 x i32> [[TMP16]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP11]], <16 x i32> [[TMP17]], <16 x i32> [[TMP13]]
+; CHECK-NEXT:    [[TMP18:%.*]] = select <16 x i1> [[TMP12]], <16 x float> [[TMP10]], <16 x float> zeroinitializer
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x float> [[TMP18]]
+;
+  %res = call <16 x float> @llvm.x86.avx512.maskz.vpermt2var.ps.512(<16 x i32> %x0, <16 x float> %x1, <16 x float> %x2, i16 %x3)
+  ret <16 x float> %res
+}
+
+
+declare <8 x i64> @llvm.x86.avx512.maskz.vpermt2var.q.512(<8 x i64>, <8 x i64>, <8 x i64>, i8)
+
+define <8 x i64>@test_int_x86_avx512_maskz_vpermt2var_q_512(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> %x2, i8 %x3)  #0 {
+;
+; CHECK-LABEL: @test_int_x86_avx512_maskz_vpermt2var_q_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <8 x i64> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <8 x i64> [[_MSPROP]], [[TMP3]]
+; CHECK-NEXT:    [[TMP5:%.*]] = call <8 x i64> @llvm.x86.avx512.vpermi2var.q.512(<8 x i64> [[X1:%.*]], <8 x i64> [[X0:%.*]], <8 x i64> [[X2:%.*]])
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i8 [[TMP4]] to <8 x i1>
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast i8 [[X3:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP8:%.*]] = select <8 x i1> [[TMP7]], <8 x i64> [[_MSPROP1]], <8 x i64> zeroinitializer
+; CHECK-NEXT:    [[TMP9:%.*]] = xor <8 x i64> [[TMP5]], zeroinitializer
+; CHECK-NEXT:    [[TMP10:%.*]] = or <8 x i64> [[TMP9]], [[_MSPROP1]]
+; CHECK-NEXT:    [[TMP11:%.*]] = or <8 x i64> [[TMP10]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP6]], <8 x i64> [[TMP11]], <8 x i64> [[TMP8]]
+; CHECK-NEXT:    [[TMP12:%.*]] = select <8 x i1> [[TMP7]], <8 x i64> [[TMP5]], <8 x i64> zeroinitializer
+; CHECK-NEXT:    store <8 x i64> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i64> [[TMP12]]
+;
+  %res = call <8 x i64> @llvm.x86.avx512.maskz.vpermt2var.q.512(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> %x2, i8 %x3)
+  ret <8 x i64> %res
+}
+
+declare <16 x i32> @llvm.x86.avx512.mask.vpermt2var.d.512(<16 x i32>, <16 x i32>, <16 x i32>, i16)
+
+define <16 x i32>@test_int_x86_avx512_vpermt2var_d_512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x2)  #0 {
+; CHECK-LABEL: @test_int_x86_avx512_vpermt2var_d_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <16 x i32> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <16 x i32> [[_MSPROP]], [[TMP3]]
+; CHECK-NEXT:    [[TMP4:%.*]] = call <16 x i32> @llvm.x86.avx512.vpermi2var.d.512(<16 x i32> [[X1:%.*]], <16 x i32> [[X0:%.*]], <16 x i32> [[X2:%.*]])
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP1]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i32> [[TMP4]]
+;
+  %res = call <16 x i32> @llvm.x86.avx512.mask.vpermt2var.d.512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x2, i16 -1)
+  ret <16 x i32> %res
+}
+
+define <16 x i32>@test_int_x86_avx512_mask_vpermt2var_d_512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x2, i16 %x3)  #0 {
+;
+; CHECK-LABEL: @test_int_x86_avx512_mask_vpermt2var_d_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <16 x i32> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <16 x i32> [[_MSPROP]], [[TMP3]]
+; CHECK-NEXT:    [[TMP5:%.*]] = call <16 x i32> @llvm.x86.avx512.vpermi2var.d.512(<16 x i32> [[X1:%.*]], <16 x i32> [[X0:%.*]], <16 x i32> [[X2:%.*]])
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i16 [[TMP4]] to <16 x i1>
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast i16 [[X3:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP8:%.*]] = select <16 x i1> [[TMP7]], <16 x i32> [[_MSPROP1]], <16 x i32> [[TMP1]]
+; CHECK-NEXT:    [[TMP9:%.*]] = xor <16 x i32> [[TMP5]], [[X1]]
+; CHECK-NEXT:    [[TMP10:%.*]] = or <16 x i32> [[TMP9]], [[_MSPROP1]]
+; CHECK-NEXT:    [[TMP11:%.*]] = or <16 x i32> [[TMP10]], [[TMP1]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP6]], <16 x i32> [[TMP11]], <16 x i32> [[TMP8]]
+; CHECK-NEXT:    [[TMP12:%.*]] = select <16 x i1> [[TMP7]], <16 x i32> [[TMP5]], <16 x i32> [[X1]]
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i32> [[TMP12]]
+;
+  %res = call <16 x i32> @llvm.x86.avx512.mask.vpermt2var.d.512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x2, i16 %x3)
+  ret <16 x i32> %res
+}
+
+declare <16 x float> @llvm.x86.avx512.mask.sub.ps.512(<16 x float>, <16 x float>, <16 x float>, i16, i32)
+declare <16 x float> @llvm.x86.avx512.mask.mul.ps.512(<16 x float>, <16 x float>, <16 x float>, i16, i32)
+declare <8 x double> @llvm.x86.avx512.mask.mul.pd.512(<8 x double>, <8 x double>, <8 x double>, i8, i32)
+
+define <16 x float> @test_vsubps_rn(<16 x float> %a0, <16 x float> %a1)  #0 {
+; CHECK-LABEL: @test_vsubps_rn(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP3]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP5:%.*]], label [[TMP6:%.*]], !prof [[PROF1]]
+; CHECK:       5:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       6:
+; CHECK-NEXT:    [[TMP7:%.*]] = call <16 x float> @llvm.x86.avx512.sub.ps.512(<16 x float> [[A0:%.*]], <16 x float> [[A1:%.*]], i32 8)
+; CHECK-NEXT:    store <16 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x float> [[TMP7]]
+;
+  %res = call <16 x float> @llvm.x86.avx512.mask.sub.ps.512(<16 x float> %a0, <16 x float> %a1,
+  <16 x float> zeroinitializer, i16 -1, i32 8)
+  ret <16 x float> %res
+}
+
+define <16 x float> @test_vsubps_rd(<16 x float> %a0, <16 x float> %a1)  #0 {
+; CHECK-LABEL: @test_vsubps_rd(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP3]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP5:%.*]], label [[TMP6:%.*]], !prof [[PROF1]]
+; CHECK:       5:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       6:
+; CHECK-NEXT:    [[TMP7:%.*]] = call <16 x float> @llvm.x86.avx512.sub.ps.512(<16 x float> [[A0:%.*]], <16 x float> [[A1:%.*]], i32 9)
+; CHECK-NEXT:    store <16 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x float> [[TMP7]]
+;
+  %res = call <16 x float> @llvm.x86.avx512.mask.sub.ps.512(<16 x float> %a0, <16 x float> %a1,
+  <16 x float> zeroinitializer, i16 -1, i32 9)
+  ret <16 x float> %res
+}
+
+define <16 x float> @test_vsubps_ru(<16 x float> %a0, <16 x float> %a1)  #0 {
+; CHECK-LABEL: @test_vsubps_ru(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP3]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP5:%.*]], label [[TMP6:%.*]], !prof [[PROF1]]
+; CHECK:       5:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       6:
+; CHECK-NEXT:    [[TMP7:%.*]] = call <16 x float> @llvm.x86.avx512.sub.ps.512(<16 x float> [[A0:%.*]], <16 x float> [[A1:%.*]], i32 10)
+; CHECK-NEXT:    store <16 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x float> [[TMP7]]
+;
+  %res = call <16 x float> @llvm.x86.avx512.mask.sub.ps.512(<16 x float> %a0, <16 x float> %a1,
+  <16 x float> zeroinitializer, i16 -1, i32 10)
+  ret <16 x float> %res
+}
+
+define <16 x float> @test_vsubps_rz(<16 x float> %a0, <16 x float> %a1)  #0 {
+; CHECK-LABEL: @test_vsubps_rz(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP3]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP5:%.*]], label [[TMP6:%.*]], !prof [[PROF1]]
+; CHECK:       5:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       6:
+; CHECK-NEXT:    [[TMP7:%.*]] = call <16 x float> @llvm.x86.avx512.sub.ps.512(<16 x float> [[A0:%.*]], <16 x float> [[A1:%.*]], i32 11)
+; CHECK-NEXT:    store <16 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x float> [[TMP7]]
+;
+  %res = call <16 x float> @llvm.x86.avx512.mask.sub.ps.512(<16 x float> %a0, <16 x float> %a1,
+  <16 x float> zeroinitializer, i16 -1, i32 11)
+  ret <16 x float> %res
+}
+
+define <16 x float> @test_vmulps_rn(<16 x float> %a0, <16 x float> %a1)  #0 {
+; CHECK-LABEL: @test_vmulps_rn(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP3]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP5:%.*]], label [[TMP6:%.*]], !prof [[PROF1]]
+; CHECK:       5:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       6:
+; CHECK-NEXT:    [[TMP7:%.*]] = call <16 x float> @llvm.x86.avx512.mul.ps.512(<16 x float> [[A0:%.*]], <16 x float> [[A1:%.*]], i32 8)
+; CHECK-NEXT:    store <16 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x float> [[TMP7]]
+;
+  %res = call <16 x float> @llvm.x86.avx512.mask.mul.ps.512(<16 x float> %a0, <16 x float> %a1,
+  <16 x float> zeroinitializer, i16 -1, i32 8)
+  ret <16 x float> %res
+}
+
+define <16 x float> @test_vmulps_rd(<16 x float> %a0, <16 x float> %a1)  #0 {
+; CHECK-LABEL: @test_vmulps_rd(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP3]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP5:%.*]], label [[TMP6:%.*]], !prof [[PROF1]]
+; CHECK:       5:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       6:
+; CHECK-NEXT:    [[TMP7:%.*]] = call <16 x float> @llvm.x86.avx512.mul.ps.512(<16 x float> [[A0:%.*]], <16 x float> [[A1:%.*]], i32 9)
+; CHECK-NEXT:    store <16 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x float> [[TMP7]]
+;
+  %res = call <16 x float> @llvm.x86.avx512.mask.mul.ps.512(<16 x float> %a0, <16 x float> %a1,
+  <16 x float> zeroinitializer, i16 -1, i32 9)
+  ret <16 x float> %res
+}
+
+define <16 x float> @test_vmulps_ru(<16 x float> %a0, <16 x float> %a1)  #0 {
+; CHECK-LABEL: @test_vmulps_ru(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP3]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP5:%.*]], label [[TMP6:%.*]], !prof [[PROF1]]
+; CHECK:       5:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       6:
+; CHECK-NEXT:    [[TMP7:%.*]] = call <16 x float> @llvm.x86.avx512.mul.ps.512(<16 x float> [[A0:%.*]], <16 x float> [[A1:%.*]], i32 10)
+; CHECK-NEXT:    store <16 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x float> [[TMP7]]
+;
+  %res = call <16 x float> @llvm.x86.avx512.mask.mul.ps.512(<16 x float> %a0, <16 x float> %a1,
+  <16 x float> zeroinitializer, i16 -1, i32 10)
+  ret <16 x float> %res
+}
+
+define <16 x float> @test_vmulps_rz(<16 x float> %a0, <16 x float> %a1)  #0 {
+; CHECK-LABEL: @test_vmulps_rz(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP3]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP5:%.*]], label [[TMP6:%.*]], !prof [[PROF1]]
+; CHECK:       5:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       6:
+; CHECK-NEXT:    [[TMP7:%.*]] = call <16 x float> @llvm.x86.avx512.mul.ps.512(<16 x float> [[A0:%.*]], <16 x float> [[A1:%.*]], i32 11)
+; CHECK-NEXT:    store <16 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x float> [[TMP7]]
+;
+  %res = call <16 x float> @llvm.x86.avx512.mask.mul.ps.512(<16 x float> %a0, <16 x float> %a1,
+  <16 x float> zeroinitializer, i16 -1, i32 11)
+  ret <16 x float> %res
+}
+
+;; mask float
+define <16 x float> @test_vmulps_mask_rn(<16 x float> %a0, <16 x float> %a1, i16 %mask)  #0 {
+;
+; CHECK-LABEL: @test_vmulps_mask_rn(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP6:%.*]], label [[TMP7:%.*]], !prof [[PROF1]]
+; CHECK:       6:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       7:
+; CHECK-NEXT:    [[TMP8:%.*]] = call <16 x float> @llvm.x86.avx512.mul.ps.512(<16 x float> [[A0:%.*]], <16 x float> [[A1:%.*]], i32 8)
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast i16 [[TMP3]] to <16 x i1>
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i16 [[MASK:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP11:%.*]] = select <16 x i1> [[TMP10]], <16 x i32> zeroinitializer, <16 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <16 x float> [[TMP8]] to <16 x i32>
+; CHECK-NEXT:    [[TMP13:%.*]] = xor <16 x i32> [[TMP12]], zeroinitializer
+; CHECK-NEXT:    [[TMP14:%.*]] = or <16 x i32> [[TMP13]], zeroinitializer
+; CHECK-NEXT:    [[TMP15:%.*]] = or <16 x i32> [[TMP14]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP9]], <16 x i32> [[TMP15]], <16 x i32> [[TMP11]]
+; CHECK-NEXT:    [[TMP16:%.*]] = select <16 x i1> [[TMP10]], <16 x float> [[TMP8]], <16 x float> zeroinitializer
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x float> [[TMP16]]
+;
+  %res = call <16 x float> @llvm.x86.avx512.mask.mul.ps.512(<16 x float> %a0, <16 x float> %a1,
+  <16 x float> zeroinitializer, i16 %mask, i32 8)
+  ret <16 x float> %res
+}
+
+define <16 x float> @test_vmulps_mask_rd(<16 x float> %a0, <16 x float> %a1, i16 %mask)  #0 {
+;
+; CHECK-LABEL: @test_vmulps_mask_rd(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP6:%.*]], label [[TMP7:%.*]], !prof [[PROF1]]
+; CHECK:       6:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       7:
+; CHECK-NEXT:    [[TMP8:%.*]] = call <16 x float> @llvm.x86.avx512.mul.ps.512(<16 x float> [[A0:%.*]], <16 x float> [[A1:%.*]], i32 9)
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast i16 [[TMP3]] to <16 x i1>
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i16 [[MASK:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP11:%.*]] = select <16 x i1> [[TMP10]], <16 x i32> zeroinitializer, <16 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <16 x float> [[TMP8]] to <16 x i32>
+; CHECK-NEXT:    [[TMP13:%.*]] = xor <16 x i32> [[TMP12]], zeroinitializer
+; CHECK-NEXT:    [[TMP14:%.*]] = or <16 x i32> [[TMP13]], zeroinitializer
+; CHECK-NEXT:    [[TMP15:%.*]] = or <16 x i32> [[TMP14]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP9]], <16 x i32> [[TMP15]], <16 x i32> [[TMP11]]
+; CHECK-NEXT:    [[TMP16:%.*]] = select <16 x i1> [[TMP10]], <16 x float> [[TMP8]], <16 x float> zeroinitializer
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x float> [[TMP16]]
+;
+  %res = call <16 x float> @llvm.x86.avx512.mask.mul.ps.512(<16 x float> %a0, <16 x float> %a1,
+  <16 x float> zeroinitializer, i16 %mask, i32 9)
+  ret <16 x float> %res
+}
+
+define <16 x float> @test_vmulps_mask_ru(<16 x float> %a0, <16 x float> %a1, i16 %mask)  #0 {
+;
+; CHECK-LABEL: @test_vmulps_mask_ru(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP6:%.*]], label [[TMP7:%.*]], !prof [[PROF1]]
+; CHECK:       6:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       7:
+; CHECK-NEXT:    [[TMP8:%.*]] = call <16 x float> @llvm.x86.avx512.mul.ps.512(<16 x float> [[A0:%.*]], <16 x float> [[A1:%.*]], i32 10)
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast i16 [[TMP3]] to <16 x i1>
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i16 [[MASK:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP11:%.*]] = select <16 x i1> [[TMP10]], <16 x i32> zeroinitializer, <16 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <16 x float> [[TMP8]] to <16 x i32>
+; CHECK-NEXT:    [[TMP13:%.*]] = xor <16 x i32> [[TMP12]], zeroinitializer
+; CHECK-NEXT:    [[TMP14:%.*]] = or <16 x i32> [[TMP13]], zeroinitializer
+; CHECK-NEXT:    [[TMP15:%.*]] = or <16 x i32> [[TMP14]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP9]], <16 x i32> [[TMP15]], <16 x i32> [[TMP11]]
+; CHECK-NEXT:    [[TMP16:%.*]] = select <16 x i1> [[TMP10]], <16 x float> [[TMP8]], <16 x float> zeroinitializer
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x float> [[TMP16]]
+;
+  %res = call <16 x float> @llvm.x86.avx512.mask.mul.ps.512(<16 x float> %a0, <16 x float> %a1,
+  <16 x float> zeroinitializer, i16 %mask, i32 10)
+  ret <16 x float> %res
+}
+
+define <16 x float> @test_vmulps_mask_rz(<16 x float> %a0, <16 x float> %a1, i16 %mask)  #0 {
+;
+; CHECK-LABEL: @test_vmulps_mask_rz(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP6:%.*]], label [[TMP7:%.*]], !prof [[PROF1]]
+; CHECK:       6:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       7:
+; CHECK-NEXT:    [[TMP8:%.*]] = call <16 x float> @llvm.x86.avx512.mul.ps.512(<16 x float> [[A0:%.*]], <16 x float> [[A1:%.*]], i32 11)
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast i16 [[TMP3]] to <16 x i1>
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i16 [[MASK:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP11:%.*]] = select <16 x i1> [[TMP10]], <16 x i32> zeroinitializer, <16 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <16 x float> [[TMP8]] to <16 x i32>
+; CHECK-NEXT:    [[TMP13:%.*]] = xor <16 x i32> [[TMP12]], zeroinitializer
+; CHECK-NEXT:    [[TMP14:%.*]] = or <16 x i32> [[TMP13]], zeroinitializer
+; CHECK-NEXT:    [[TMP15:%.*]] = or <16 x i32> [[TMP14]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP9]], <16 x i32> [[TMP15]], <16 x i32> [[TMP11]]
+; CHECK-NEXT:    [[TMP16:%.*]] = select <16 x i1> [[TMP10]], <16 x float> [[TMP8]], <16 x float> zeroinitializer
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x float> [[TMP16]]
+;
+  %res = call <16 x float> @llvm.x86.avx512.mask.mul.ps.512(<16 x float> %a0, <16 x float> %a1,
+  <16 x float> zeroinitializer, i16 %mask, i32 11)
+  ret <16 x float> %res
+}
+
+;; With Passthru value
+define <16 x float> @test_vmulps_mask_passthru_rn(<16 x float> %a0, <16 x float> %a1, <16 x float> %passthru, i16 %mask)  #0 {
+;
+; CHECK-LABEL: @test_vmulps_mask_passthru_rn(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP6]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP7:%.*]], label [[TMP8:%.*]], !prof [[PROF1]]
+; CHECK:       7:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       8:
+; CHECK-NEXT:    [[TMP9:%.*]] = call <16 x float> @llvm.x86.avx512.mul.ps.512(<16 x float> [[A0:%.*]], <16 x float> [[A1:%.*]], i32 8)
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i16 [[TMP3]] to <16 x i1>
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast i16 [[MASK:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP12:%.*]] = select <16 x i1> [[TMP11]], <16 x i32> zeroinitializer, <16 x i32> [[TMP4]]
+; CHECK-NEXT:    [[TMP13:%.*]] = bitcast <16 x float> [[TMP9]] to <16 x i32>
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast <16 x float> [[PASSTHRU:%.*]] to <16 x i32>
+; CHECK-NEXT:    [[TMP15:%.*]] = xor <16 x i32> [[TMP13]], [[TMP14]]
+; CHECK-NEXT:    [[TMP16:%.*]] = or <16 x i32> [[TMP15]], zeroinitializer
+; CHECK-NEXT:    [[TMP17:%.*]] = or <16 x i32> [[TMP16]], [[TMP4]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP10]], <16 x i32> [[TMP17]], <16 x i32> [[TMP12]]
+; CHECK-NEXT:    [[TMP18:%.*]] = select <16 x i1> [[TMP11]], <16 x float> [[TMP9]], <16 x float> [[PASSTHRU]]
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x float> [[TMP18]]
+;
+  %res = call <16 x float> @llvm.x86.avx512.mask.mul.ps.512(<16 x float> %a0, <16 x float> %a1,
+  <16 x float> %passthru, i16 %mask, i32 8)
+  ret <16 x float> %res
+}
+
+define <16 x float> @test_vmulps_mask_passthru_rd(<16 x float> %a0, <16 x float> %a1, <16 x float> %passthru, i16 %mask)  #0 {
+;
+; CHECK-LABEL: @test_vmulps_mask_passthru_rd(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP6]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP7:%.*]], label [[TMP8:%.*]], !prof [[PROF1]]
+; CHECK:       7:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       8:
+; CHECK-NEXT:    [[TMP9:%.*]] = call <16 x float> @llvm.x86.avx512.mul.ps.512(<16 x float> [[A0:%.*]], <16 x float> [[A1:%.*]], i32 9)
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i16 [[TMP3]] to <16 x i1>
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast i16 [[MASK:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP12:%.*]] = select <16 x i1> [[TMP11]], <16 x i32> zeroinitializer, <16 x i32> [[TMP4]]
+; CHECK-NEXT:    [[TMP13:%.*]] = bitcast <16 x float> [[TMP9]] to <16 x i32>
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast <16 x float> [[PASSTHRU:%.*]] to <16 x i32>
+; CHECK-NEXT:    [[TMP15:%.*]] = xor <16 x i32> [[TMP13]], [[TMP14]]
+; CHECK-NEXT:    [[TMP16:%.*]] = or <16 x i32> [[TMP15]], zeroinitializer
+; CHECK-NEXT:    [[TMP17:%.*]] = or <16 x i32> [[TMP16]], [[TMP4]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP10]], <16 x i32> [[TMP17]], <16 x i32> [[TMP12]]
+; CHECK-NEXT:    [[TMP18:%.*]] = select <16 x i1> [[TMP11]], <16 x float> [[TMP9]], <16 x float> [[PASSTHRU]]
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x float> [[TMP18]]
+;
+  %res = call <16 x float> @llvm.x86.avx512.mask.mul.ps.512(<16 x float> %a0, <16 x float> %a1,
+  <16 x float> %passthru, i16 %mask, i32 9)
+  ret <16 x float> %res
+}
+
+define <16 x float> @test_vmulps_mask_passthru_ru(<16 x float> %a0, <16 x float> %a1, <16 x float> %passthru, i16 %mask)  #0 {
+;
+; CHECK-LABEL: @test_vmulps_mask_passthru_ru(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP6]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP7:%.*]], label [[TMP8:%.*]], !prof [[PROF1]]
+; CHECK:       7:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       8:
+; CHECK-NEXT:    [[TMP9:%.*]] = call <16 x float> @llvm.x86.avx512.mul.ps.512(<16 x float> [[A0:%.*]], <16 x float> [[A1:%.*]], i32 10)
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i16 [[TMP3]] to <16 x i1>
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast i16 [[MASK:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP12:%.*]] = select <16 x i1> [[TMP11]], <16 x i32> zeroinitializer, <16 x i32> [[TMP4]]
+; CHECK-NEXT:    [[TMP13:%.*]] = bitcast <16 x float> [[TMP9]] to <16 x i32>
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast <16 x float> [[PASSTHRU:%.*]] to <16 x i32>
+; CHECK-NEXT:    [[TMP15:%.*]] = xor <16 x i32> [[TMP13]], [[TMP14]]
+; CHECK-NEXT:    [[TMP16:%.*]] = or <16 x i32> [[TMP15]], zeroinitializer
+; CHECK-NEXT:    [[TMP17:%.*]] = or <16 x i32> [[TMP16]], [[TMP4]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP10]], <16 x i32> [[TMP17]], <16 x i32> [[TMP12]]
+; CHECK-NEXT:    [[TMP18:%.*]] = select <16 x i1> [[TMP11]], <16 x float> [[TMP9]], <16 x float> [[PASSTHRU]]
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x float> [[TMP18]]
+;
+  %res = call <16 x float> @llvm.x86.avx512.mask.mul.ps.512(<16 x float> %a0, <16 x float> %a1,
+  <16 x float> %passthru, i16 %mask, i32 10)
+  ret <16 x float> %res
+}
+
+define <16 x float> @test_vmulps_mask_passthru_rz(<16 x float> %a0, <16 x float> %a1, <16 x float> %passthru, i16 %mask)  #0 {
+;
+; CHECK-LABEL: @test_vmulps_mask_passthru_rz(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP6]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP7:%.*]], label [[TMP8:%.*]], !prof [[PROF1]]
+; CHECK:       7:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       8:
+; CHECK-NEXT:    [[TMP9:%.*]] = call <16 x float> @llvm.x86.avx512.mul.ps.512(<16 x float> [[A0:%.*]], <16 x float> [[A1:%.*]], i32 11)
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i16 [[TMP3]] to <16 x i1>
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast i16 [[MASK:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP12:%.*]] = select <16 x i1> [[TMP11]], <16 x i32> zeroinitializer, <16 x i32> [[TMP4]]
+; CHECK-NEXT:    [[TMP13:%.*]] = bitcast <16 x float> [[TMP9]] to <16 x i32>
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast <16 x float> [[PASSTHRU:%.*]] to <16 x i32>
+; CHECK-NEXT:    [[TMP15:%.*]] = xor <16 x i32> [[TMP13]], [[TMP14]]
+; CHECK-NEXT:    [[TMP16:%.*]] = or <16 x i32> [[TMP15]], zeroinitializer
+; CHECK-NEXT:    [[TMP17:%.*]] = or <16 x i32> [[TMP16]], [[TMP4]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP10]], <16 x i32> [[TMP17]], <16 x i32> [[TMP12]]
+; CHECK-NEXT:    [[TMP18:%.*]] = select <16 x i1> [[TMP11]], <16 x float> [[TMP9]], <16 x float> [[PASSTHRU]]
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x float> [[TMP18]]
+;
+  %res = call <16 x float> @llvm.x86.avx512.mask.mul.ps.512(<16 x float> %a0, <16 x float> %a1,
+  <16 x float> %passthru, i16 %mask, i32 11)
+  ret <16 x float> %res
+}
+
+;; mask double
+define <8 x double> @test_vmulpd_mask_rn(<8 x double> %a0, <8 x double> %a1, i8 %mask)  #0 {
+;
+; CHECK-LABEL: @test_vmulpd_mask_rn(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <8 x i64> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <8 x i64> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP6:%.*]], label [[TMP7:%.*]], !prof [[PROF1]]
+; CHECK:       6:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       7:
+; CHECK-NEXT:    [[TMP8:%.*]] = call <8 x double> @llvm.x86.avx512.mul.pd.512(<8 x double> [[A0:%.*]], <8 x double> [[A1:%.*]], i32 8)
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast i8 [[TMP3]] to <8 x i1>
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP11:%.*]] = select <8 x i1> [[TMP10]], <8 x i64> zeroinitializer, <8 x i64> zeroinitializer
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <8 x double> [[TMP8]] to <8 x i64>
+; CHECK-NEXT:    [[TMP13:%.*]] = xor <8 x i64> [[TMP12]], zeroinitializer
+; CHECK-NEXT:    [[TMP14:%.*]] = or <8 x i64> [[TMP13]], zeroinitializer
+; CHECK-NEXT:    [[TMP15:%.*]] = or <8 x i64> [[TMP14]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP9]], <8 x i64> [[TMP15]], <8 x i64> [[TMP11]]
+; CHECK-NEXT:    [[TMP16:%.*]] = select <8 x i1> [[TMP10]], <8 x double> [[TMP8]], <8 x double> zeroinitializer
+; CHECK-NEXT:    store <8 x i64> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x double> [[TMP16]]
+;
+  %res = call <8 x double> @llvm.x86.avx512.mask.mul.pd.512(<8 x double> %a0, <8 x double> %a1,
+  <8 x double> zeroinitializer, i8 %mask, i32 8)
+  ret <8 x double> %res
+}
+
+define <8 x double> @test_vmulpd_mask_rd(<8 x double> %a0, <8 x double> %a1, i8 %mask)  #0 {
+;
+; CHECK-LABEL: @test_vmulpd_mask_rd(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <8 x i64> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <8 x i64> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP6:%.*]], label [[TMP7:%.*]], !prof [[PROF1]]
+; CHECK:       6:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       7:
+; CHECK-NEXT:    [[TMP8:%.*]] = call <8 x double> @llvm.x86.avx512.mul.pd.512(<8 x double> [[A0:%.*]], <8 x double> [[A1:%.*]], i32 9)
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast i8 [[TMP3]] to <8 x i1>
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP11:%.*]] = select <8 x i1> [[TMP10]], <8 x i64> zeroinitializer, <8 x i64> zeroinitializer
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <8 x double> [[TMP8]] to <8 x i64>
+; CHECK-NEXT:    [[TMP13:%.*]] = xor <8 x i64> [[TMP12]], zeroinitializer
+; CHECK-NEXT:    [[TMP14:%.*]] = or <8 x i64> [[TMP13]], zeroinitializer
+; CHECK-NEXT:    [[TMP15:%.*]] = or <8 x i64> [[TMP14]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP9]], <8 x i64> [[TMP15]], <8 x i64> [[TMP11]]
+; CHECK-NEXT:    [[TMP16:%.*]] = select <8 x i1> [[TMP10]], <8 x double> [[TMP8]], <8 x double> zeroinitializer
+; CHECK-NEXT:    store <8 x i64> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x double> [[TMP16]]
+;
+  %res = call <8 x double> @llvm.x86.avx512.mask.mul.pd.512(<8 x double> %a0, <8 x double> %a1,
+  <8 x double> zeroinitializer, i8 %mask, i32 9)
+  ret <8 x double> %res
+}
+
+define <8 x double> @test_vmulpd_mask_ru(<8 x double> %a0, <8 x double> %a1, i8 %mask)  #0 {
+;
+; CHECK-LABEL: @test_vmulpd_mask_ru(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <8 x i64> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <8 x i64> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP6:%.*]], label [[TMP7:%.*]], !prof [[PROF1]]
+; CHECK:       6:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       7:
+; CHECK-NEXT:    [[TMP8:%.*]] = call <8 x double> @llvm.x86.avx512.mul.pd.512(<8 x double> [[A0:%.*]], <8 x double> [[A1:%.*]], i32 10)
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast i8 [[TMP3]] to <8 x i1>
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP11:%.*]] = select <8 x i1> [[TMP10]], <8 x i64> zeroinitializer, <8 x i64> zeroinitializer
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <8 x double> [[TMP8]] to <8 x i64>
+; CHECK-NEXT:    [[TMP13:%.*]] = xor <8 x i64> [[TMP12]], zeroinitializer
+; CHECK-NEXT:    [[TMP14:%.*]] = or <8 x i64> [[TMP13]], zeroinitializer
+; CHECK-NEXT:    [[TMP15:%.*]] = or <8 x i64> [[TMP14]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP9]], <8 x i64> [[TMP15]], <8 x i64> [[TMP11]]
+; CHECK-NEXT:    [[TMP16:%.*]] = select <8 x i1> [[TMP10]], <8 x double> [[TMP8]], <8 x double> zeroinitializer
+; CHECK-NEXT:    store <8 x i64> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x double> [[TMP16]]
+;
+  %res = call <8 x double> @llvm.x86.avx512.mask.mul.pd.512(<8 x double> %a0, <8 x double> %a1,
+  <8 x double> zeroinitializer, i8 %mask, i32 10)
+  ret <8 x double> %res
+}
+
+define <8 x double> @test_vmulpd_mask_rz(<8 x double> %a0, <8 x double> %a1, i8 %mask)  #0 {
+;
+; CHECK-LABEL: @test_vmulpd_mask_rz(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <8 x i64> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <8 x i64> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP6:%.*]], label [[TMP7:%.*]], !prof [[PROF1]]
+; CHECK:       6:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       7:
+; CHECK-NEXT:    [[TMP8:%.*]] = call <8 x double> @llvm.x86.avx512.mul.pd.512(<8 x double> [[A0:%.*]], <8 x double> [[A1:%.*]], i32 11)
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast i8 [[TMP3]] to <8 x i1>
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP11:%.*]] = select <8 x i1> [[TMP10]], <8 x i64> zeroinitializer, <8 x i64> zeroinitializer
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <8 x double> [[TMP8]] to <8 x i64>
+; CHECK-NEXT:    [[TMP13:%.*]] = xor <8 x i64> [[TMP12]], zeroinitializer
+; CHECK-NEXT:    [[TMP14:%.*]] = or <8 x i64> [[TMP13]], zeroinitializer
+; CHECK-NEXT:    [[TMP15:%.*]] = or <8 x i64> [[TMP14]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP9]], <8 x i64> [[TMP15]], <8 x i64> [[TMP11]]
+; CHECK-NEXT:    [[TMP16:%.*]] = select <8 x i1> [[TMP10]], <8 x double> [[TMP8]], <8 x double> zeroinitializer
+; CHECK-NEXT:    store <8 x i64> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x double> [[TMP16]]
+;
+  %res = call <8 x double> @llvm.x86.avx512.mask.mul.pd.512(<8 x double> %a0, <8 x double> %a1,
+  <8 x double> zeroinitializer, i8 %mask, i32 11)
+  ret <8 x double> %res
+}
+
+define <16 x float> @test_mm512_maskz_add_round_ps_rn_sae(<16 x float> %a0, <16 x float> %a1, i16 %mask)  #0 {
+;
+; CHECK-LABEL: @test_mm512_maskz_add_round_ps_rn_sae(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP6:%.*]], label [[TMP7:%.*]], !prof [[PROF1]]
+; CHECK:       6:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       7:
+; CHECK-NEXT:    [[TMP8:%.*]] = call <16 x float> @llvm.x86.avx512.add.ps.512(<16 x float> [[A0:%.*]], <16 x float> [[A1:%.*]], i32 8)
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast i16 [[TMP3]] to <16 x i1>
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i16 [[MASK:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP11:%.*]] = select <16 x i1> [[TMP10]], <16 x i32> zeroinitializer, <16 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <16 x float> [[TMP8]] to <16 x i32>
+; CHECK-NEXT:    [[TMP13:%.*]] = xor <16 x i32> [[TMP12]], zeroinitializer
+; CHECK-NEXT:    [[TMP14:%.*]] = or <16 x i32> [[TMP13]], zeroinitializer
+; CHECK-NEXT:    [[TMP15:%.*]] = or <16 x i32> [[TMP14]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP9]], <16 x i32> [[TMP15]], <16 x i32> [[TMP11]]
+; CHECK-NEXT:    [[TMP16:%.*]] = select <16 x i1> [[TMP10]], <16 x float> [[TMP8]], <16 x float> zeroinitializer
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x float> [[TMP16]]
+;
+  %res = call <16 x float> @llvm.x86.avx512.mask.add.ps.512(<16 x float> %a0, <16 x float> %a1, <16 x float>zeroinitializer, i16 %mask, i32 8)
+  ret <16 x float> %res
+}
+define <16 x float> @test_mm512_maskz_add_round_ps_rd_sae(<16 x float> %a0, <16 x float> %a1, i16 %mask)  #0 {
+;
+; CHECK-LABEL: @test_mm512_maskz_add_round_ps_rd_sae(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP6:%.*]], label [[TMP7:%.*]], !prof [[PROF1]]
+; CHECK:       6:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       7:
+; CHECK-NEXT:    [[TMP8:%.*]] = call <16 x float> @llvm.x86.avx512.add.ps.512(<16 x float> [[A0:%.*]], <16 x float> [[A1:%.*]], i32 9)
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast i16 [[TMP3]] to <16 x i1>
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i16 [[MASK:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP11:%.*]] = select <16 x i1> [[TMP10]], <16 x i32> zeroinitializer, <16 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <16 x float> [[TMP8]] to <16 x i32>
+; CHECK-NEXT:    [[TMP13:%.*]] = xor <16 x i32> [[TMP12]], zeroinitializer
+; CHECK-NEXT:    [[TMP14:%.*]] = or <16 x i32> [[TMP13]], zeroinitializer
+; CHECK-NEXT:    [[TMP15:%.*]] = or <16 x i32> [[TMP14]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP9]], <16 x i32> [[TMP15]], <16 x i32> [[TMP11]]
+; CHECK-NEXT:    [[TMP16:%.*]] = select <16 x i1> [[TMP10]], <16 x float> [[TMP8]], <16 x float> zeroinitializer
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x float> [[TMP16]]
+;
+  %res = call <16 x float> @llvm.x86.avx512.mask.add.ps.512(<16 x float> %a0, <16 x float> %a1, <16 x float>zeroinitializer, i16 %mask, i32 9)
+  ret <16 x float> %res
+}
+define <16 x float> @test_mm512_maskz_add_round_ps_ru_sae(<16 x float> %a0, <16 x float> %a1, i16 %mask)  #0 {
+;
+; CHECK-LABEL: @test_mm512_maskz_add_round_ps_ru_sae(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP6:%.*]], label [[TMP7:%.*]], !prof [[PROF1]]
+; CHECK:       6:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       7:
+; CHECK-NEXT:    [[TMP8:%.*]] = call <16 x float> @llvm.x86.avx512.add.ps.512(<16 x float> [[A0:%.*]], <16 x float> [[A1:%.*]], i32 10)
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast i16 [[TMP3]] to <16 x i1>
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i16 [[MASK:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP11:%.*]] = select <16 x i1> [[TMP10]], <16 x i32> zeroinitializer, <16 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <16 x float> [[TMP8]] to <16 x i32>
+; CHECK-NEXT:    [[TMP13:%.*]] = xor <16 x i32> [[TMP12]], zeroinitializer
+; CHECK-NEXT:    [[TMP14:%.*]] = or <16 x i32> [[TMP13]], zeroinitializer
+; CHECK-NEXT:    [[TMP15:%.*]] = or <16 x i32> [[TMP14]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP9]], <16 x i32> [[TMP15]], <16 x i32> [[TMP11]]
+; CHECK-NEXT:    [[TMP16:%.*]] = select <16 x i1> [[TMP10]], <16 x float> [[TMP8]], <16 x float> zeroinitializer
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x float> [[TMP16]]
+;
+  %res = call <16 x float> @llvm.x86.avx512.mask.add.ps.512(<16 x float> %a0, <16 x float> %a1, <16 x float>zeroinitializer, i16 %mask, i32 10)
+  ret <16 x float> %res
+}
+
+define <16 x float> @test_mm512_maskz_add_round_ps_rz_sae(<16 x float> %a0, <16 x float> %a1, i16 %mask)  #0 {
+;
+; CHECK-LABEL: @test_mm512_maskz_add_round_ps_rz_sae(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP6:%.*]], label [[TMP7:%.*]], !prof [[PROF1]]
+; CHECK:       6:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       7:
+; CHECK-NEXT:    [[TMP8:%.*]] = call <16 x float> @llvm.x86.avx512.add.ps.512(<16 x float> [[A0:%.*]], <16 x float> [[A1:%.*]], i32 11)
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast i16 [[TMP3]] to <16 x i1>
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i16 [[MASK:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP11:%.*]] = select <16 x i1> [[TMP10]], <16 x i32> zeroinitializer, <16 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <16 x float> [[TMP8]] to <16 x i32>
+; CHECK-NEXT:    [[TMP13:%.*]] = xor <16 x i32> [[TMP12]], zeroinitializer
+; CHECK-NEXT:    [[TMP14:%.*]] = or <16 x i32> [[TMP13]], zeroinitializer
+; CHECK-NEXT:    [[TMP15:%.*]] = or <16 x i32> [[TMP14]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP9]], <16 x i32> [[TMP15]], <16 x i32> [[TMP11]]
+; CHECK-NEXT:    [[TMP16:%.*]] = select <16 x i1> [[TMP10]], <16 x float> [[TMP8]], <16 x float> zeroinitializer
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x float> [[TMP16]]
+;
+  %res = call <16 x float> @llvm.x86.avx512.mask.add.ps.512(<16 x float> %a0, <16 x float> %a1, <16 x float>zeroinitializer, i16 %mask, i32 11)
+  ret <16 x float> %res
+}
+
+
+define <16 x float> @test_mm512_maskz_add_round_ps_current(<16 x float> %a0, <16 x float> %a1, i16 %mask)  #0 {
+;
+; CHECK-LABEL: @test_mm512_maskz_add_round_ps_current(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP6:%.*]], label [[TMP7:%.*]], !prof [[PROF1]]
+; CHECK:       6:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       7:
+; CHECK-NEXT:    [[TMP8:%.*]] = call <16 x float> @llvm.x86.avx512.add.ps.512(<16 x float> [[A0:%.*]], <16 x float> [[A1:%.*]], i32 4)
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast i16 [[TMP3]] to <16 x i1>
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i16 [[MASK:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP11:%.*]] = select <16 x i1> [[TMP10]], <16 x i32> zeroinitializer, <16 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <16 x float> [[TMP8]] to <16 x i32>
+; CHECK-NEXT:    [[TMP13:%.*]] = xor <16 x i32> [[TMP12]], zeroinitializer
+; CHECK-NEXT:    [[TMP14:%.*]] = or <16 x i32> [[TMP13]], zeroinitializer
+; CHECK-NEXT:    [[TMP15:%.*]] = or <16 x i32> [[TMP14]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP9]], <16 x i32> [[TMP15]], <16 x i32> [[TMP11]]
+; CHECK-NEXT:    [[TMP16:%.*]] = select <16 x i1> [[TMP10]], <16 x float> [[TMP8]], <16 x float> zeroinitializer
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x float> [[TMP16]]
+;
+  %res = call <16 x float> @llvm.x86.avx512.mask.add.ps.512(<16 x float> %a0, <16 x float> %a1, <16 x float>zeroinitializer, i16 %mask, i32 4)
+  ret <16 x float> %res
+}
+
+define <16 x float> @test_mm512_mask_add_round_ps_rn_sae(<16 x float> %a0, <16 x float> %a1, <16 x float> %src, i16 %mask)  #0 {
+;
+; CHECK-LABEL: @test_mm512_mask_add_round_ps_rn_sae(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP6]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP7:%.*]], label [[TMP8:%.*]], !prof [[PROF1]]
+; CHECK:       7:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       8:
+; CHECK-NEXT:    [[TMP9:%.*]] = call <16 x float> @llvm.x86.avx512.add.ps.512(<16 x float> [[A0:%.*]], <16 x float> [[A1:%.*]], i32 8)
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i16 [[TMP3]] to <16 x i1>
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast i16 [[MASK:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP12:%.*]] = select <16 x i1> [[TMP11]], <16 x i32> zeroinitializer, <16 x i32> [[TMP4]]
+; CHECK-NEXT:    [[TMP13:%.*]] = bitcast <16 x float> [[TMP9]] to <16 x i32>
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast <16 x float> [[SRC:%.*]] to <16 x i32>
+; CHECK-NEXT:    [[TMP15:%.*]] = xor <16 x i32> [[TMP13]], [[TMP14]]
+; CHECK-NEXT:    [[TMP16:%.*]] = or <16 x i32> [[TMP15]], zeroinitializer
+; CHECK-NEXT:    [[TMP17:%.*]] = or <16 x i32> [[TMP16]], [[TMP4]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP10]], <16 x i32> [[TMP17]], <16 x i32> [[TMP12]]
+; CHECK-NEXT:    [[TMP18:%.*]] = select <16 x i1> [[TMP11]], <16 x float> [[TMP9]], <16 x float> [[SRC]]
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x float> [[TMP18]]
+;
+  %res = call <16 x float> @llvm.x86.avx512.mask.add.ps.512(<16 x float> %a0, <16 x float> %a1, <16 x float> %src, i16 %mask, i32 8)
+  ret <16 x float> %res
+}
+define <16 x float> @test_mm512_mask_add_round_ps_rd_sae(<16 x float> %a0, <16 x float> %a1, <16 x float> %src, i16 %mask)  #0 {
+;
+; CHECK-LABEL: @test_mm512_mask_add_round_ps_rd_sae(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP6]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP7:%.*]], label [[TMP8:%.*]], !prof [[PROF1]]
+; CHECK:       7:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       8:
+; CHECK-NEXT:    [[TMP9:%.*]] = call <16 x float> @llvm.x86.avx512.add.ps.512(<16 x float> [[A0:%.*]], <16 x float> [[A1:%.*]], i32 9)
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i16 [[TMP3]] to <16 x i1>
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast i16 [[MASK:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP12:%.*]] = select <16 x i1> [[TMP11]], <16 x i32> zeroinitializer, <16 x i32> [[TMP4]]
+; CHECK-NEXT:    [[TMP13:%.*]] = bitcast <16 x float> [[TMP9]] to <16 x i32>
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast <16 x float> [[SRC:%.*]] to <16 x i32>
+; CHECK-NEXT:    [[TMP15:%.*]] = xor <16 x i32> [[TMP13]], [[TMP14]]
+; CHECK-NEXT:    [[TMP16:%.*]] = or <16 x i32> [[TMP15]], zeroinitializer
+; CHECK-NEXT:    [[TMP17:%.*]] = or <16 x i32> [[TMP16]], [[TMP4]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP10]], <16 x i32> [[TMP17]], <16 x i32> [[TMP12]]
+; CHECK-NEXT:    [[TMP18:%.*]] = select <16 x i1> [[TMP11]], <16 x float> [[TMP9]], <16 x float> [[SRC]]
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x float> [[TMP18]]
+;
+  %res = call <16 x float> @llvm.x86.avx512.mask.add.ps.512(<16 x float> %a0, <16 x float> %a1, <16 x float> %src, i16 %mask, i32 9)
+  ret <16 x float> %res
+}
+define <16 x float> @test_mm512_mask_add_round_ps_ru_sae(<16 x float> %a0, <16 x float> %a1, <16 x float> %src, i16 %mask)  #0 {
+;
+; CHECK-LABEL: @test_mm512_mask_add_round_ps_ru_sae(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP6]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP7:%.*]], label [[TMP8:%.*]], !prof [[PROF1]]
+; CHECK:       7:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       8:
+; CHECK-NEXT:    [[TMP9:%.*]] = call <16 x float> @llvm.x86.avx512.add.ps.512(<16 x float> [[A0:%.*]], <16 x float> [[A1:%.*]], i32 10)
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i16 [[TMP3]] to <16 x i1>
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast i16 [[MASK:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP12:%.*]] = select <16 x i1> [[TMP11]], <16 x i32> zeroinitializer, <16 x i32> [[TMP4]]
+; CHECK-NEXT:    [[TMP13:%.*]] = bitcast <16 x float> [[TMP9]] to <16 x i32>
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast <16 x float> [[SRC:%.*]] to <16 x i32>
+; CHECK-NEXT:    [[TMP15:%.*]] = xor <16 x i32> [[TMP13]], [[TMP14]]
+; CHECK-NEXT:    [[TMP16:%.*]] = or <16 x i32> [[TMP15]], zeroinitializer
+; CHECK-NEXT:    [[TMP17:%.*]] = or <16 x i32> [[TMP16]], [[TMP4]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP10]], <16 x i32> [[TMP17]], <16 x i32> [[TMP12]]
+; CHECK-NEXT:    [[TMP18:%.*]] = select <16 x i1> [[TMP11]], <16 x float> [[TMP9]], <16 x float> [[SRC]]
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x float> [[TMP18]]
+;
+  %res = call <16 x float> @llvm.x86.avx512.mask.add.ps.512(<16 x float> %a0, <16 x float> %a1, <16 x float> %src, i16 %mask, i32 10)
+  ret <16 x float> %res
+}
+
+define <16 x float> @test_mm512_mask_add_round_ps_rz_sae(<16 x float> %a0, <16 x float> %a1, <16 x float> %src, i16 %mask)  #0 {
+;
+; CHECK-LABEL: @test_mm512_mask_add_round_ps_rz_sae(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP6]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP7:%.*]], label [[TMP8:%.*]], !prof [[PROF1]]
+; CHECK:       7:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       8:
+; CHECK-NEXT:    [[TMP9:%.*]] = call <16 x float> @llvm.x86.avx512.add.ps.512(<16 x float> [[A0:%.*]], <16 x float> [[A1:%.*]], i32 11)
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i16 [[TMP3]] to <16 x i1>
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast i16 [[MASK:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP12:%.*]] = select <16 x i1> [[TMP11]], <16 x i32> zeroinitializer, <16 x i32> [[TMP4]]
+; CHECK-NEXT:    [[TMP13:%.*]] = bitcast <16 x float> [[TMP9]] to <16 x i32>
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast <16 x float> [[SRC:%.*]] to <16 x i32>
+; CHECK-NEXT:    [[TMP15:%.*]] = xor <16 x i32> [[TMP13]], [[TMP14]]
+; CHECK-NEXT:    [[TMP16:%.*]] = or <16 x i32> [[TMP15]], zeroinitializer
+; CHECK-NEXT:    [[TMP17:%.*]] = or <16 x i32> [[TMP16]], [[TMP4]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP10]], <16 x i32> [[TMP17]], <16 x i32> [[TMP12]]
+; CHECK-NEXT:    [[TMP18:%.*]] = select <16 x i1> [[TMP11]], <16 x float> [[TMP9]], <16 x float> [[SRC]]
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x float> [[TMP18]]
+;
+  %res = call <16 x float> @llvm.x86.avx512.mask.add.ps.512(<16 x float> %a0, <16 x float> %a1, <16 x float> %src, i16 %mask, i32 11)
+  ret <16 x float> %res
+}
+
+
+define <16 x float> @test_mm512_mask_add_round_ps_current(<16 x float> %a0, <16 x float> %a1, <16 x float> %src, i16 %mask)  #0 {
+;
+; CHECK-LABEL: @test_mm512_mask_add_round_ps_current(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP6]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP7:%.*]], label [[TMP8:%.*]], !prof [[PROF1]]
+; CHECK:       7:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       8:
+; CHECK-NEXT:    [[TMP9:%.*]] = call <16 x float> @llvm.x86.avx512.add.ps.512(<16 x float> [[A0:%.*]], <16 x float> [[A1:%.*]], i32 4)
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i16 [[TMP3]] to <16 x i1>
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast i16 [[MASK:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP12:%.*]] = select <16 x i1> [[TMP11]], <16 x i32> zeroinitializer, <16 x i32> [[TMP4]]
+; CHECK-NEXT:    [[TMP13:%.*]] = bitcast <16 x float> [[TMP9]] to <16 x i32>
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast <16 x float> [[SRC:%.*]] to <16 x i32>
+; CHECK-NEXT:    [[TMP15:%.*]] = xor <16 x i32> [[TMP13]], [[TMP14]]
+; CHECK-NEXT:    [[TMP16:%.*]] = or <16 x i32> [[TMP15]], zeroinitializer
+; CHECK-NEXT:    [[TMP17:%.*]] = or <16 x i32> [[TMP16]], [[TMP4]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP10]], <16 x i32> [[TMP17]], <16 x i32> [[TMP12]]
+; CHECK-NEXT:    [[TMP18:%.*]] = select <16 x i1> [[TMP11]], <16 x float> [[TMP9]], <16 x float> [[SRC]]
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x float> [[TMP18]]
+;
+  %res = call <16 x float> @llvm.x86.avx512.mask.add.ps.512(<16 x float> %a0, <16 x float> %a1, <16 x float> %src, i16 %mask, i32 4)
+  ret <16 x float> %res
+}
+
+
+define <16 x float> @test_mm512_add_round_ps_rn_sae(<16 x float> %a0, <16 x float> %a1, i16 %mask)  #0 {
+; CHECK-LABEL: @test_mm512_add_round_ps_rn_sae(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP3]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP5:%.*]], label [[TMP6:%.*]], !prof [[PROF1]]
+; CHECK:       5:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       6:
+; CHECK-NEXT:    [[TMP7:%.*]] = call <16 x float> @llvm.x86.avx512.add.ps.512(<16 x float> [[A0:%.*]], <16 x float> [[A1:%.*]], i32 8)
+; CHECK-NEXT:    store <16 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x float> [[TMP7]]
+;
+  %res = call <16 x float> @llvm.x86.avx512.mask.add.ps.512(<16 x float> %a0, <16 x float> %a1, <16 x float>zeroinitializer, i16 -1, i32 8)
+  ret <16 x float> %res
+}
+define <16 x float> @test_mm512_add_round_ps_rd_sae(<16 x float> %a0, <16 x float> %a1, i16 %mask)  #0 {
+; CHECK-LABEL: @test_mm512_add_round_ps_rd_sae(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP3]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP5:%.*]], label [[TMP6:%.*]], !prof [[PROF1]]
+; CHECK:       5:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       6:
+; CHECK-NEXT:    [[TMP7:%.*]] = call <16 x float> @llvm.x86.avx512.add.ps.512(<16 x float> [[A0:%.*]], <16 x float> [[A1:%.*]], i32 9)
+; CHECK-NEXT:    store <16 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x float> [[TMP7]]
+;
+  %res = call <16 x float> @llvm.x86.avx512.mask.add.ps.512(<16 x float> %a0, <16 x float> %a1, <16 x float>zeroinitializer, i16 -1, i32 9)
+  ret <16 x float> %res
+}
+define <16 x float> @test_mm512_add_round_ps_ru_sae(<16 x float> %a0, <16 x float> %a1, i16 %mask)  #0 {
+; CHECK-LABEL: @test_mm512_add_round_ps_ru_sae(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP3]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP5:%.*]], label [[TMP6:%.*]], !prof [[PROF1]]
+; CHECK:       5:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       6:
+; CHECK-NEXT:    [[TMP7:%.*]] = call <16 x float> @llvm.x86.avx512.add.ps.512(<16 x float> [[A0:%.*]], <16 x float> [[A1:%.*]], i32 10)
+; CHECK-NEXT:    store <16 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x float> [[TMP7]]
+;
+  %res = call <16 x float> @llvm.x86.avx512.mask.add.ps.512(<16 x float> %a0, <16 x float> %a1, <16 x float>zeroinitializer, i16 -1, i32 10)
+  ret <16 x float> %res
+}
+
+define <16 x float> @test_mm512_add_round_ps_rz_sae(<16 x float> %a0, <16 x float> %a1, i16 %mask)  #0 {
+; CHECK-LABEL: @test_mm512_add_round_ps_rz_sae(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP3]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP5:%.*]], label [[TMP6:%.*]], !prof [[PROF1]]
+; CHECK:       5:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       6:
+; CHECK-NEXT:    [[TMP7:%.*]] = call <16 x float> @llvm.x86.avx512.add.ps.512(<16 x float> [[A0:%.*]], <16 x float> [[A1:%.*]], i32 11)
+; CHECK-NEXT:    store <16 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x float> [[TMP7]]
+;
+  %res = call <16 x float> @llvm.x86.avx512.mask.add.ps.512(<16 x float> %a0, <16 x float> %a1, <16 x float>zeroinitializer, i16 -1, i32 11)
+  ret <16 x float> %res
+}
+
+define <16 x float> @test_mm512_add_round_ps_current(<16 x float> %a0, <16 x float> %a1, i16 %mask)  #0 {
+; CHECK-LABEL: @test_mm512_add_round_ps_current(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP3]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP5:%.*]], label [[TMP6:%.*]], !prof [[PROF1]]
+; CHECK:       5:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       6:
+; CHECK-NEXT:    [[TMP7:%.*]] = call <16 x float> @llvm.x86.avx512.add.ps.512(<16 x float> [[A0:%.*]], <16 x float> [[A1:%.*]], i32 4)
+; CHECK-NEXT:    store <16 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x float> [[TMP7]]
+;
+  %res = call <16 x float> @llvm.x86.avx512.mask.add.ps.512(<16 x float> %a0, <16 x float> %a1, <16 x float>zeroinitializer, i16 -1, i32 4)
+  ret <16 x float> %res
+}
+declare <16 x float> @llvm.x86.avx512.mask.add.ps.512(<16 x float>, <16 x float>, <16 x float>, i16, i32)
+
+define <16 x float> @test_mm512_mask_sub_round_ps_rn_sae(<16 x float> %a0, <16 x float> %a1, <16 x float> %src, i16 %mask)  #0 {
+;
+; CHECK-LABEL: @test_mm512_mask_sub_round_ps_rn_sae(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP6]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP7:%.*]], label [[TMP8:%.*]], !prof [[PROF1]]
+; CHECK:       7:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       8:
+; CHECK-NEXT:    [[TMP9:%.*]] = call <16 x float> @llvm.x86.avx512.sub.ps.512(<16 x float> [[A0:%.*]], <16 x float> [[A1:%.*]], i32 8)
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i16 [[TMP3]] to <16 x i1>
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast i16 [[MASK:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP12:%.*]] = select <16 x i1> [[TMP11]], <16 x i32> zeroinitializer, <16 x i32> [[TMP4]]
+; CHECK-NEXT:    [[TMP13:%.*]] = bitcast <16 x float> [[TMP9]] to <16 x i32>
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast <16 x float> [[SRC:%.*]] to <16 x i32>
+; CHECK-NEXT:    [[TMP15:%.*]] = xor <16 x i32> [[TMP13]], [[TMP14]]
+; CHECK-NEXT:    [[TMP16:%.*]] = or <16 x i32> [[TMP15]], zeroinitializer
+; CHECK-NEXT:    [[TMP17:%.*]] = or <16 x i32> [[TMP16]], [[TMP4]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP10]], <16 x i32> [[TMP17]], <16 x i32> [[TMP12]]
+; CHECK-NEXT:    [[TMP18:%.*]] = select <16 x i1> [[TMP11]], <16 x float> [[TMP9]], <16 x float> [[SRC]]
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x float> [[TMP18]]
+;
+  %res = call <16 x float> @llvm.x86.avx512.mask.sub.ps.512(<16 x float> %a0, <16 x float> %a1, <16 x float> %src, i16 %mask, i32 8)
+  ret <16 x float> %res
+}
+define <16 x float> @test_mm512_mask_sub_round_ps_rd_sae(<16 x float> %a0, <16 x float> %a1, <16 x float> %src, i16 %mask)  #0 {
+;
+; CHECK-LABEL: @test_mm512_mask_sub_round_ps_rd_sae(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP6]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP7:%.*]], label [[TMP8:%.*]], !prof [[PROF1]]
+; CHECK:       7:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       8:
+; CHECK-NEXT:    [[TMP9:%.*]] = call <16 x float> @llvm.x86.avx512.sub.ps.512(<16 x float> [[A0:%.*]], <16 x float> [[A1:%.*]], i32 9)
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i16 [[TMP3]] to <16 x i1>
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast i16 [[MASK:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP12:%.*]] = select <16 x i1> [[TMP11]], <16 x i32> zeroinitializer, <16 x i32> [[TMP4]]
+; CHECK-NEXT:    [[TMP13:%.*]] = bitcast <16 x float> [[TMP9]] to <16 x i32>
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast <16 x float> [[SRC:%.*]] to <16 x i32>
+; CHECK-NEXT:    [[TMP15:%.*]] = xor <16 x i32> [[TMP13]], [[TMP14]]
+; CHECK-NEXT:    [[TMP16:%.*]] = or <16 x i32> [[TMP15]], zeroinitializer
+; CHECK-NEXT:    [[TMP17:%.*]] = or <16 x i32> [[TMP16]], [[TMP4]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP10]], <16 x i32> [[TMP17]], <16 x i32> [[TMP12]]
+; CHECK-NEXT:    [[TMP18:%.*]] = select <16 x i1> [[TMP11]], <16 x float> [[TMP9]], <16 x float> [[SRC]]
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x float> [[TMP18]]
+;
+  %res = call <16 x float> @llvm.x86.avx512.mask.sub.ps.512(<16 x float> %a0, <16 x float> %a1, <16 x float> %src, i16 %mask, i32 9)
+  ret <16 x float> %res
+}
+define <16 x float> @test_mm512_mask_sub_round_ps_ru_sae(<16 x float> %a0, <16 x float> %a1, <16 x float> %src, i16 %mask)  #0 {
+;
+; CHECK-LABEL: @test_mm512_mask_sub_round_ps_ru_sae(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP6]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP7:%.*]], label [[TMP8:%.*]], !prof [[PROF1]]
+; CHECK:       7:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       8:
+; CHECK-NEXT:    [[TMP9:%.*]] = call <16 x float> @llvm.x86.avx512.sub.ps.512(<16 x float> [[A0:%.*]], <16 x float> [[A1:%.*]], i32 10)
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i16 [[TMP3]] to <16 x i1>
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast i16 [[MASK:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP12:%.*]] = select <16 x i1> [[TMP11]], <16 x i32> zeroinitializer, <16 x i32> [[TMP4]]
+; CHECK-NEXT:    [[TMP13:%.*]] = bitcast <16 x float> [[TMP9]] to <16 x i32>
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast <16 x float> [[SRC:%.*]] to <16 x i32>
+; CHECK-NEXT:    [[TMP15:%.*]] = xor <16 x i32> [[TMP13]], [[TMP14]]
+; CHECK-NEXT:    [[TMP16:%.*]] = or <16 x i32> [[TMP15]], zeroinitializer
+; CHECK-NEXT:    [[TMP17:%.*]] = or <16 x i32> [[TMP16]], [[TMP4]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP10]], <16 x i32> [[TMP17]], <16 x i32> [[TMP12]]
+; CHECK-NEXT:    [[TMP18:%.*]] = select <16 x i1> [[TMP11]], <16 x float> [[TMP9]], <16 x float> [[SRC]]
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x float> [[TMP18]]
+;
+  %res = call <16 x float> @llvm.x86.avx512.mask.sub.ps.512(<16 x float> %a0, <16 x float> %a1, <16 x float> %src, i16 %mask, i32 10)
+  ret <16 x float> %res
+}
+
+define <16 x float> @test_mm512_mask_sub_round_ps_rz_sae(<16 x float> %a0, <16 x float> %a1, <16 x float> %src, i16 %mask)  #0 {
+;
+; CHECK-LABEL: @test_mm512_mask_sub_round_ps_rz_sae(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP6]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP7:%.*]], label [[TMP8:%.*]], !prof [[PROF1]]
+; CHECK:       7:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       8:
+; CHECK-NEXT:    [[TMP9:%.*]] = call <16 x float> @llvm.x86.avx512.sub.ps.512(<16 x float> [[A0:%.*]], <16 x float> [[A1:%.*]], i32 11)
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i16 [[TMP3]] to <16 x i1>
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast i16 [[MASK:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP12:%.*]] = select <16 x i1> [[TMP11]], <16 x i32> zeroinitializer, <16 x i32> [[TMP4]]
+; CHECK-NEXT:    [[TMP13:%.*]] = bitcast <16 x float> [[TMP9]] to <16 x i32>
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast <16 x float> [[SRC:%.*]] to <16 x i32>
+; CHECK-NEXT:    [[TMP15:%.*]] = xor <16 x i32> [[TMP13]], [[TMP14]]
+; CHECK-NEXT:    [[TMP16:%.*]] = or <16 x i32> [[TMP15]], zeroinitializer
+; CHECK-NEXT:    [[TMP17:%.*]] = or <16 x i32> [[TMP16]], [[TMP4]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP10]], <16 x i32> [[TMP17]], <16 x i32> [[TMP12]]
+; CHECK-NEXT:    [[TMP18:%.*]] = select <16 x i1> [[TMP11]], <16 x float> [[TMP9]], <16 x float> [[SRC]]
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x float> [[TMP18]]
+;
+  %res = call <16 x float> @llvm.x86.avx512.mask.sub.ps.512(<16 x float> %a0, <16 x float> %a1, <16 x float> %src, i16 %mask, i32 11)
+  ret <16 x float> %res
+}
+
+
+define <16 x float> @test_mm512_mask_sub_round_ps_current(<16 x float> %a0, <16 x float> %a1, <16 x float> %src, i16 %mask)  #0 {
+;
+; CHECK-LABEL: @test_mm512_mask_sub_round_ps_current(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP6]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP7:%.*]], label [[TMP8:%.*]], !prof [[PROF1]]
+; CHECK:       7:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       8:
+; CHECK-NEXT:    [[TMP9:%.*]] = call <16 x float> @llvm.x86.avx512.sub.ps.512(<16 x float> [[A0:%.*]], <16 x float> [[A1:%.*]], i32 4)
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i16 [[TMP3]] to <16 x i1>
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast i16 [[MASK:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP12:%.*]] = select <16 x i1> [[TMP11]], <16 x i32> zeroinitializer, <16 x i32> [[TMP4]]
+; CHECK-NEXT:    [[TMP13:%.*]] = bitcast <16 x float> [[TMP9]] to <16 x i32>
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast <16 x float> [[SRC:%.*]] to <16 x i32>
+; CHECK-NEXT:    [[TMP15:%.*]] = xor <16 x i32> [[TMP13]], [[TMP14]]
+; CHECK-NEXT:    [[TMP16:%.*]] = or <16 x i32> [[TMP15]], zeroinitializer
+; CHECK-NEXT:    [[TMP17:%.*]] = or <16 x i32> [[TMP16]], [[TMP4]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP10]], <16 x i32> [[TMP17]], <16 x i32> [[TMP12]]
+; CHECK-NEXT:    [[TMP18:%.*]] = select <16 x i1> [[TMP11]], <16 x float> [[TMP9]], <16 x float> [[SRC]]
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x float> [[TMP18]]
+;
+  %res = call <16 x float> @llvm.x86.avx512.mask.sub.ps.512(<16 x float> %a0, <16 x float> %a1, <16 x float> %src, i16 %mask, i32 4)
+  ret <16 x float> %res
+}
+
+define <16 x float> @test_mm512_sub_round_ps_rn_sae(<16 x float> %a0, <16 x float> %a1, i16 %mask)  #0 {
+; CHECK-LABEL: @test_mm512_sub_round_ps_rn_sae(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP3]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP5:%.*]], label [[TMP6:%.*]], !prof [[PROF1]]
+; CHECK:       5:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       6:
+; CHECK-NEXT:    [[TMP7:%.*]] = call <16 x float> @llvm.x86.avx512.sub.ps.512(<16 x float> [[A0:%.*]], <16 x float> [[A1:%.*]], i32 8)
+; CHECK-NEXT:    store <16 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x float> [[TMP7]]
+;
+  %res = call <16 x float> @llvm.x86.avx512.mask.sub.ps.512(<16 x float> %a0, <16 x float> %a1, <16 x float>zeroinitializer, i16 -1, i32 8)
+  ret <16 x float> %res
+}
+define <16 x float> @test_mm512_sub_round_ps_rd_sae(<16 x float> %a0, <16 x float> %a1, i16 %mask)  #0 {
+; CHECK-LABEL: @test_mm512_sub_round_ps_rd_sae(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP3]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP5:%.*]], label [[TMP6:%.*]], !prof [[PROF1]]
+; CHECK:       5:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       6:
+; CHECK-NEXT:    [[TMP7:%.*]] = call <16 x float> @llvm.x86.avx512.sub.ps.512(<16 x float> [[A0:%.*]], <16 x float> [[A1:%.*]], i32 9)
+; CHECK-NEXT:    store <16 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x float> [[TMP7]]
+;
+  %res = call <16 x float> @llvm.x86.avx512.mask.sub.ps.512(<16 x float> %a0, <16 x float> %a1, <16 x float>zeroinitializer, i16 -1, i32 9)
+  ret <16 x float> %res
+}
+define <16 x float> @test_mm512_sub_round_ps_ru_sae(<16 x float> %a0, <16 x float> %a1, i16 %mask)  #0 {
+; CHECK-LABEL: @test_mm512_sub_round_ps_ru_sae(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP3]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP5:%.*]], label [[TMP6:%.*]], !prof [[PROF1]]
+; CHECK:       5:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       6:
+; CHECK-NEXT:    [[TMP7:%.*]] = call <16 x float> @llvm.x86.avx512.sub.ps.512(<16 x float> [[A0:%.*]], <16 x float> [[A1:%.*]], i32 10)
+; CHECK-NEXT:    store <16 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x float> [[TMP7]]
+;
+  %res = call <16 x float> @llvm.x86.avx512.mask.sub.ps.512(<16 x float> %a0, <16 x float> %a1, <16 x float>zeroinitializer, i16 -1, i32 10)
+  ret <16 x float> %res
+}
+
+define <16 x float> @test_mm512_sub_round_ps_rz_sae(<16 x float> %a0, <16 x float> %a1, i16 %mask)  #0 {
+; CHECK-LABEL: @test_mm512_sub_round_ps_rz_sae(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP3]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP5:%.*]], label [[TMP6:%.*]], !prof [[PROF1]]
+; CHECK:       5:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       6:
+; CHECK-NEXT:    [[TMP7:%.*]] = call <16 x float> @llvm.x86.avx512.sub.ps.512(<16 x float> [[A0:%.*]], <16 x float> [[A1:%.*]], i32 11)
+; CHECK-NEXT:    store <16 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x float> [[TMP7]]
+;
+  %res = call <16 x float> @llvm.x86.avx512.mask.sub.ps.512(<16 x float> %a0, <16 x float> %a1, <16 x float>zeroinitializer, i16 -1, i32 11)
+  ret <16 x float> %res
+}
+
+define <16 x float> @test_mm512_sub_round_ps_current(<16 x float> %a0, <16 x float> %a1, i16 %mask)  #0 {
+; CHECK-LABEL: @test_mm512_sub_round_ps_current(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP3]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP5:%.*]], label [[TMP6:%.*]], !prof [[PROF1]]
+; CHECK:       5:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       6:
+; CHECK-NEXT:    [[TMP7:%.*]] = call <16 x float> @llvm.x86.avx512.sub.ps.512(<16 x float> [[A0:%.*]], <16 x float> [[A1:%.*]], i32 4)
+; CHECK-NEXT:    store <16 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x float> [[TMP7]]
+;
+  %res = call <16 x float> @llvm.x86.avx512.mask.sub.ps.512(<16 x float> %a0, <16 x float> %a1, <16 x float>zeroinitializer, i16 -1, i32 4)
+  ret <16 x float> %res
+}
+
+define <16 x float> @test_mm512_maskz_div_round_ps_rn_sae(<16 x float> %a0, <16 x float> %a1, i16 %mask)  #0 {
+;
+; CHECK-LABEL: @test_mm512_maskz_div_round_ps_rn_sae(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP6:%.*]], label [[TMP7:%.*]], !prof [[PROF1]]
+; CHECK:       6:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       7:
+; CHECK-NEXT:    [[TMP8:%.*]] = call <16 x float> @llvm.x86.avx512.div.ps.512(<16 x float> [[A0:%.*]], <16 x float> [[A1:%.*]], i32 8)
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast i16 [[TMP3]] to <16 x i1>
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i16 [[MASK:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP11:%.*]] = select <16 x i1> [[TMP10]], <16 x i32> zeroinitializer, <16 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <16 x float> [[TMP8]] to <16 x i32>
+; CHECK-NEXT:    [[TMP13:%.*]] = xor <16 x i32> [[TMP12]], zeroinitializer
+; CHECK-NEXT:    [[TMP14:%.*]] = or <16 x i32> [[TMP13]], zeroinitializer
+; CHECK-NEXT:    [[TMP15:%.*]] = or <16 x i32> [[TMP14]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP9]], <16 x i32> [[TMP15]], <16 x i32> [[TMP11]]
+; CHECK-NEXT:    [[TMP16:%.*]] = select <16 x i1> [[TMP10]], <16 x float> [[TMP8]], <16 x float> zeroinitializer
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x float> [[TMP16]]
+;
+  %res = call <16 x float> @llvm.x86.avx512.mask.div.ps.512(<16 x float> %a0, <16 x float> %a1, <16 x float>zeroinitializer, i16 %mask, i32 8)
+  ret <16 x float> %res
+}
+define <16 x float> @test_mm512_maskz_div_round_ps_rd_sae(<16 x float> %a0, <16 x float> %a1, i16 %mask)  #0 {
+;
+; CHECK-LABEL: @test_mm512_maskz_div_round_ps_rd_sae(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP6:%.*]], label [[TMP7:%.*]], !prof [[PROF1]]
+; CHECK:       6:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       7:
+; CHECK-NEXT:    [[TMP8:%.*]] = call <16 x float> @llvm.x86.avx512.div.ps.512(<16 x float> [[A0:%.*]], <16 x float> [[A1:%.*]], i32 9)
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast i16 [[TMP3]] to <16 x i1>
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i16 [[MASK:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP11:%.*]] = select <16 x i1> [[TMP10]], <16 x i32> zeroinitializer, <16 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <16 x float> [[TMP8]] to <16 x i32>
+; CHECK-NEXT:    [[TMP13:%.*]] = xor <16 x i32> [[TMP12]], zeroinitializer
+; CHECK-NEXT:    [[TMP14:%.*]] = or <16 x i32> [[TMP13]], zeroinitializer
+; CHECK-NEXT:    [[TMP15:%.*]] = or <16 x i32> [[TMP14]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP9]], <16 x i32> [[TMP15]], <16 x i32> [[TMP11]]
+; CHECK-NEXT:    [[TMP16:%.*]] = select <16 x i1> [[TMP10]], <16 x float> [[TMP8]], <16 x float> zeroinitializer
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x float> [[TMP16]]
+;
+  %res = call <16 x float> @llvm.x86.avx512.mask.div.ps.512(<16 x float> %a0, <16 x float> %a1, <16 x float>zeroinitializer, i16 %mask, i32 9)
+  ret <16 x float> %res
+}
+define <16 x float> @test_mm512_maskz_div_round_ps_ru_sae(<16 x float> %a0, <16 x float> %a1, i16 %mask)  #0 {
+;
+; CHECK-LABEL: @test_mm512_maskz_div_round_ps_ru_sae(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP6:%.*]], label [[TMP7:%.*]], !prof [[PROF1]]
+; CHECK:       6:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       7:
+; CHECK-NEXT:    [[TMP8:%.*]] = call <16 x float> @llvm.x86.avx512.div.ps.512(<16 x float> [[A0:%.*]], <16 x float> [[A1:%.*]], i32 10)
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast i16 [[TMP3]] to <16 x i1>
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i16 [[MASK:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP11:%.*]] = select <16 x i1> [[TMP10]], <16 x i32> zeroinitializer, <16 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <16 x float> [[TMP8]] to <16 x i32>
+; CHECK-NEXT:    [[TMP13:%.*]] = xor <16 x i32> [[TMP12]], zeroinitializer
+; CHECK-NEXT:    [[TMP14:%.*]] = or <16 x i32> [[TMP13]], zeroinitializer
+; CHECK-NEXT:    [[TMP15:%.*]] = or <16 x i32> [[TMP14]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP9]], <16 x i32> [[TMP15]], <16 x i32> [[TMP11]]
+; CHECK-NEXT:    [[TMP16:%.*]] = select <16 x i1> [[TMP10]], <16 x float> [[TMP8]], <16 x float> zeroinitializer
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x float> [[TMP16]]
+;
+  %res = call <16 x float> @llvm.x86.avx512.mask.div.ps.512(<16 x float> %a0, <16 x float> %a1, <16 x float>zeroinitializer, i16 %mask, i32 10)
+  ret <16 x float> %res
+}
+
+define <16 x float> @test_mm512_maskz_div_round_ps_rz_sae(<16 x float> %a0, <16 x float> %a1, i16 %mask)  #0 {
+;
+; CHECK-LABEL: @test_mm512_maskz_div_round_ps_rz_sae(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP6:%.*]], label [[TMP7:%.*]], !prof [[PROF1]]
+; CHECK:       6:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       7:
+; CHECK-NEXT:    [[TMP8:%.*]] = call <16 x float> @llvm.x86.avx512.div.ps.512(<16 x float> [[A0:%.*]], <16 x float> [[A1:%.*]], i32 11)
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast i16 [[TMP3]] to <16 x i1>
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i16 [[MASK:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP11:%.*]] = select <16 x i1> [[TMP10]], <16 x i32> zeroinitializer, <16 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <16 x float> [[TMP8]] to <16 x i32>
+; CHECK-NEXT:    [[TMP13:%.*]] = xor <16 x i32> [[TMP12]], zeroinitializer
+; CHECK-NEXT:    [[TMP14:%.*]] = or <16 x i32> [[TMP13]], zeroinitializer
+; CHECK-NEXT:    [[TMP15:%.*]] = or <16 x i32> [[TMP14]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP9]], <16 x i32> [[TMP15]], <16 x i32> [[TMP11]]
+; CHECK-NEXT:    [[TMP16:%.*]] = select <16 x i1> [[TMP10]], <16 x float> [[TMP8]], <16 x float> zeroinitializer
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x float> [[TMP16]]
+;
+  %res = call <16 x float> @llvm.x86.avx512.mask.div.ps.512(<16 x float> %a0, <16 x float> %a1, <16 x float>zeroinitializer, i16 %mask, i32 11)
+  ret <16 x float> %res
+}
+
+
+define <16 x float> @test_mm512_maskz_div_round_ps_current(<16 x float> %a0, <16 x float> %a1, i16 %mask)  #0 {
+;
+; CHECK-LABEL: @test_mm512_maskz_div_round_ps_current(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP6:%.*]], label [[TMP7:%.*]], !prof [[PROF1]]
+; CHECK:       6:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       7:
+; CHECK-NEXT:    [[TMP8:%.*]] = call <16 x float> @llvm.x86.avx512.div.ps.512(<16 x float> [[A0:%.*]], <16 x float> [[A1:%.*]], i32 4)
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast i16 [[TMP3]] to <16 x i1>
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i16 [[MASK:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP11:%.*]] = select <16 x i1> [[TMP10]], <16 x i32> zeroinitializer, <16 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <16 x float> [[TMP8]] to <16 x i32>
+; CHECK-NEXT:    [[TMP13:%.*]] = xor <16 x i32> [[TMP12]], zeroinitializer
+; CHECK-NEXT:    [[TMP14:%.*]] = or <16 x i32> [[TMP13]], zeroinitializer
+; CHECK-NEXT:    [[TMP15:%.*]] = or <16 x i32> [[TMP14]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP9]], <16 x i32> [[TMP15]], <16 x i32> [[TMP11]]
+; CHECK-NEXT:    [[TMP16:%.*]] = select <16 x i1> [[TMP10]], <16 x float> [[TMP8]], <16 x float> zeroinitializer
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x float> [[TMP16]]
+;
+  %res = call <16 x float> @llvm.x86.avx512.mask.div.ps.512(<16 x float> %a0, <16 x float> %a1, <16 x float>zeroinitializer, i16 %mask, i32 4)
+  ret <16 x float> %res
+}
+
+define <16 x float> @test_mm512_mask_div_round_ps_rn_sae(<16 x float> %a0, <16 x float> %a1, <16 x float> %src, i16 %mask)  #0 {
+;
+; CHECK-LABEL: @test_mm512_mask_div_round_ps_rn_sae(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP6]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP7:%.*]], label [[TMP8:%.*]], !prof [[PROF1]]
+; CHECK:       7:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       8:
+; CHECK-NEXT:    [[TMP9:%.*]] = call <16 x float> @llvm.x86.avx512.div.ps.512(<16 x float> [[A0:%.*]], <16 x float> [[A1:%.*]], i32 8)
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i16 [[TMP3]] to <16 x i1>
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast i16 [[MASK:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP12:%.*]] = select <16 x i1> [[TMP11]], <16 x i32> zeroinitializer, <16 x i32> [[TMP4]]
+; CHECK-NEXT:    [[TMP13:%.*]] = bitcast <16 x float> [[TMP9]] to <16 x i32>
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast <16 x float> [[SRC:%.*]] to <16 x i32>
+; CHECK-NEXT:    [[TMP15:%.*]] = xor <16 x i32> [[TMP13]], [[TMP14]]
+; CHECK-NEXT:    [[TMP16:%.*]] = or <16 x i32> [[TMP15]], zeroinitializer
+; CHECK-NEXT:    [[TMP17:%.*]] = or <16 x i32> [[TMP16]], [[TMP4]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP10]], <16 x i32> [[TMP17]], <16 x i32> [[TMP12]]
+; CHECK-NEXT:    [[TMP18:%.*]] = select <16 x i1> [[TMP11]], <16 x float> [[TMP9]], <16 x float> [[SRC]]
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x float> [[TMP18]]
+;
+  %res = call <16 x float> @llvm.x86.avx512.mask.div.ps.512(<16 x float> %a0, <16 x float> %a1, <16 x float> %src, i16 %mask, i32 8)
+  ret <16 x float> %res
+}
+define <16 x float> @test_mm512_mask_div_round_ps_rd_sae(<16 x float> %a0, <16 x float> %a1, <16 x float> %src, i16 %mask)  #0 {
+;
+; CHECK-LABEL: @test_mm512_mask_div_round_ps_rd_sae(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP6]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP7:%.*]], label [[TMP8:%.*]], !prof [[PROF1]]
+; CHECK:       7:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       8:
+; CHECK-NEXT:    [[TMP9:%.*]] = call <16 x float> @llvm.x86.avx512.div.ps.512(<16 x float> [[A0:%.*]], <16 x float> [[A1:%.*]], i32 9)
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i16 [[TMP3]] to <16 x i1>
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast i16 [[MASK:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP12:%.*]] = select <16 x i1> [[TMP11]], <16 x i32> zeroinitializer, <16 x i32> [[TMP4]]
+; CHECK-NEXT:    [[TMP13:%.*]] = bitcast <16 x float> [[TMP9]] to <16 x i32>
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast <16 x float> [[SRC:%.*]] to <16 x i32>
+; CHECK-NEXT:    [[TMP15:%.*]] = xor <16 x i32> [[TMP13]], [[TMP14]]
+; CHECK-NEXT:    [[TMP16:%.*]] = or <16 x i32> [[TMP15]], zeroinitializer
+; CHECK-NEXT:    [[TMP17:%.*]] = or <16 x i32> [[TMP16]], [[TMP4]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP10]], <16 x i32> [[TMP17]], <16 x i32> [[TMP12]]
+; CHECK-NEXT:    [[TMP18:%.*]] = select <16 x i1> [[TMP11]], <16 x float> [[TMP9]], <16 x float> [[SRC]]
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x float> [[TMP18]]
+;
+  %res = call <16 x float> @llvm.x86.avx512.mask.div.ps.512(<16 x float> %a0, <16 x float> %a1, <16 x float> %src, i16 %mask, i32 9)
+  ret <16 x float> %res
+}
+define <16 x float> @test_mm512_mask_div_round_ps_ru_sae(<16 x float> %a0, <16 x float> %a1, <16 x float> %src, i16 %mask)  #0 {
+;
+; CHECK-LABEL: @test_mm512_mask_div_round_ps_ru_sae(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP6]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP7:%.*]], label [[TMP8:%.*]], !prof [[PROF1]]
+; CHECK:       7:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       8:
+; CHECK-NEXT:    [[TMP9:%.*]] = call <16 x float> @llvm.x86.avx512.div.ps.512(<16 x float> [[A0:%.*]], <16 x float> [[A1:%.*]], i32 10)
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i16 [[TMP3]] to <16 x i1>
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast i16 [[MASK:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP12:%.*]] = select <16 x i1> [[TMP11]], <16 x i32> zeroinitializer, <16 x i32> [[TMP4]]
+; CHECK-NEXT:    [[TMP13:%.*]] = bitcast <16 x float> [[TMP9]] to <16 x i32>
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast <16 x float> [[SRC:%.*]] to <16 x i32>
+; CHECK-NEXT:    [[TMP15:%.*]] = xor <16 x i32> [[TMP13]], [[TMP14]]
+; CHECK-NEXT:    [[TMP16:%.*]] = or <16 x i32> [[TMP15]], zeroinitializer
+; CHECK-NEXT:    [[TMP17:%.*]] = or <16 x i32> [[TMP16]], [[TMP4]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP10]], <16 x i32> [[TMP17]], <16 x i32> [[TMP12]]
+; CHECK-NEXT:    [[TMP18:%.*]] = select <16 x i1> [[TMP11]], <16 x float> [[TMP9]], <16 x float> [[SRC]]
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x float> [[TMP18]]
+;
+  %res = call <16 x float> @llvm.x86.avx512.mask.div.ps.512(<16 x float> %a0, <16 x float> %a1, <16 x float> %src, i16 %mask, i32 10)
+  ret <16 x float> %res
+}
+
+define <16 x float> @test_mm512_mask_div_round_ps_rz_sae(<16 x float> %a0, <16 x float> %a1, <16 x float> %src, i16 %mask)  #0 {
+;
+; CHECK-LABEL: @test_mm512_mask_div_round_ps_rz_sae(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP6]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP7:%.*]], label [[TMP8:%.*]], !prof [[PROF1]]
+; CHECK:       7:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       8:
+; CHECK-NEXT:    [[TMP9:%.*]] = call <16 x float> @llvm.x86.avx512.div.ps.512(<16 x float> [[A0:%.*]], <16 x float> [[A1:%.*]], i32 11)
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i16 [[TMP3]] to <16 x i1>
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast i16 [[MASK:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP12:%.*]] = select <16 x i1> [[TMP11]], <16 x i32> zeroinitializer, <16 x i32> [[TMP4]]
+; CHECK-NEXT:    [[TMP13:%.*]] = bitcast <16 x float> [[TMP9]] to <16 x i32>
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast <16 x float> [[SRC:%.*]] to <16 x i32>
+; CHECK-NEXT:    [[TMP15:%.*]] = xor <16 x i32> [[TMP13]], [[TMP14]]
+; CHECK-NEXT:    [[TMP16:%.*]] = or <16 x i32> [[TMP15]], zeroinitializer
+; CHECK-NEXT:    [[TMP17:%.*]] = or <16 x i32> [[TMP16]], [[TMP4]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP10]], <16 x i32> [[TMP17]], <16 x i32> [[TMP12]]
+; CHECK-NEXT:    [[TMP18:%.*]] = select <16 x i1> [[TMP11]], <16 x float> [[TMP9]], <16 x float> [[SRC]]
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x float> [[TMP18]]
+;
+  %res = call <16 x float> @llvm.x86.avx512.mask.div.ps.512(<16 x float> %a0, <16 x float> %a1, <16 x float> %src, i16 %mask, i32 11)
+  ret <16 x float> %res
+}
+
+
+define <16 x float> @test_mm512_mask_div_round_ps_current(<16 x float> %a0, <16 x float> %a1, <16 x float> %src, i16 %mask)  #0 {
+;
+; CHECK-LABEL: @test_mm512_mask_div_round_ps_current(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP6]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP7:%.*]], label [[TMP8:%.*]], !prof [[PROF1]]
+; CHECK:       7:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       8:
+; CHECK-NEXT:    [[TMP9:%.*]] = call <16 x float> @llvm.x86.avx512.div.ps.512(<16 x float> [[A0:%.*]], <16 x float> [[A1:%.*]], i32 4)
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i16 [[TMP3]] to <16 x i1>
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast i16 [[MASK:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP12:%.*]] = select <16 x i1> [[TMP11]], <16 x i32> zeroinitializer, <16 x i32> [[TMP4]]
+; CHECK-NEXT:    [[TMP13:%.*]] = bitcast <16 x float> [[TMP9]] to <16 x i32>
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast <16 x float> [[SRC:%.*]] to <16 x i32>
+; CHECK-NEXT:    [[TMP15:%.*]] = xor <16 x i32> [[TMP13]], [[TMP14]]
+; CHECK-NEXT:    [[TMP16:%.*]] = or <16 x i32> [[TMP15]], zeroinitializer
+; CHECK-NEXT:    [[TMP17:%.*]] = or <16 x i32> [[TMP16]], [[TMP4]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP10]], <16 x i32> [[TMP17]], <16 x i32> [[TMP12]]
+; CHECK-NEXT:    [[TMP18:%.*]] = select <16 x i1> [[TMP11]], <16 x float> [[TMP9]], <16 x float> [[SRC]]
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x float> [[TMP18]]
+;
+  %res = call <16 x float> @llvm.x86.avx512.mask.div.ps.512(<16 x float> %a0, <16 x float> %a1, <16 x float> %src, i16 %mask, i32 4)
+  ret <16 x float> %res
+}
+
+
+define <16 x float> @test_mm512_div_round_ps_rn_sae(<16 x float> %a0, <16 x float> %a1, i16 %mask)  #0 {
+; CHECK-LABEL: @test_mm512_div_round_ps_rn_sae(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP3]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP5:%.*]], label [[TMP6:%.*]], !prof [[PROF1]]
+; CHECK:       5:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       6:
+; CHECK-NEXT:    [[TMP7:%.*]] = call <16 x float> @llvm.x86.avx512.div.ps.512(<16 x float> [[A0:%.*]], <16 x float> [[A1:%.*]], i32 8)
+; CHECK-NEXT:    store <16 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x float> [[TMP7]]
+;
+  %res = call <16 x float> @llvm.x86.avx512.mask.div.ps.512(<16 x float> %a0, <16 x float> %a1, <16 x float>zeroinitializer, i16 -1, i32 8)
+  ret <16 x float> %res
+}
+define <16 x float> @test_mm512_div_round_ps_rd_sae(<16 x float> %a0, <16 x float> %a1, i16 %mask)  #0 {
+; CHECK-LABEL: @test_mm512_div_round_ps_rd_sae(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP3]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP5:%.*]], label [[TMP6:%.*]], !prof [[PROF1]]
+; CHECK:       5:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       6:
+; CHECK-NEXT:    [[TMP7:%.*]] = call <16 x float> @llvm.x86.avx512.div.ps.512(<16 x float> [[A0:%.*]], <16 x float> [[A1:%.*]], i32 9)
+; CHECK-NEXT:    store <16 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x float> [[TMP7]]
+;
+  %res = call <16 x float> @llvm.x86.avx512.mask.div.ps.512(<16 x float> %a0, <16 x float> %a1, <16 x float>zeroinitializer, i16 -1, i32 9)
+  ret <16 x float> %res
+}
+define <16 x float> @test_mm512_div_round_ps_ru_sae(<16 x float> %a0, <16 x float> %a1, i16 %mask)  #0 {
+; CHECK-LABEL: @test_mm512_div_round_ps_ru_sae(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP3]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP5:%.*]], label [[TMP6:%.*]], !prof [[PROF1]]
+; CHECK:       5:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       6:
+; CHECK-NEXT:    [[TMP7:%.*]] = call <16 x float> @llvm.x86.avx512.div.ps.512(<16 x float> [[A0:%.*]], <16 x float> [[A1:%.*]], i32 10)
+; CHECK-NEXT:    store <16 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x float> [[TMP7]]
+;
+  %res = call <16 x float> @llvm.x86.avx512.mask.div.ps.512(<16 x float> %a0, <16 x float> %a1, <16 x float>zeroinitializer, i16 -1, i32 10)
+  ret <16 x float> %res
+}
+
+define <16 x float> @test_mm512_div_round_ps_rz_sae(<16 x float> %a0, <16 x float> %a1, i16 %mask)  #0 {
+; CHECK-LABEL: @test_mm512_div_round_ps_rz_sae(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP3]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP5:%.*]], label [[TMP6:%.*]], !prof [[PROF1]]
+; CHECK:       5:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       6:
+; CHECK-NEXT:    [[TMP7:%.*]] = call <16 x float> @llvm.x86.avx512.div.ps.512(<16 x float> [[A0:%.*]], <16 x float> [[A1:%.*]], i32 11)
+; CHECK-NEXT:    store <16 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x float> [[TMP7]]
+;
+  %res = call <16 x float> @llvm.x86.avx512.mask.div.ps.512(<16 x float> %a0, <16 x float> %a1, <16 x float>zeroinitializer, i16 -1, i32 11)
+  ret <16 x float> %res
+}
+
+define <16 x float> @test_mm512_div_round_ps_current(<16 x float> %a0, <16 x float> %a1, i16 %mask)  #0 {
+; CHECK-LABEL: @test_mm512_div_round_ps_current(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP3]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP5:%.*]], label [[TMP6:%.*]], !prof [[PROF1]]
+; CHECK:       5:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       6:
+; CHECK-NEXT:    [[TMP7:%.*]] = call <16 x float> @llvm.x86.avx512.div.ps.512(<16 x float> [[A0:%.*]], <16 x float> [[A1:%.*]], i32 4)
+; CHECK-NEXT:    store <16 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x float> [[TMP7]]
+;
+  %res = call <16 x float> @llvm.x86.avx512.mask.div.ps.512(<16 x float> %a0, <16 x float> %a1, <16 x float>zeroinitializer, i16 -1, i32 4)
+  ret <16 x float> %res
+}
+declare <16 x float> @llvm.x86.avx512.mask.div.ps.512(<16 x float>, <16 x float>, <16 x float>, i16, i32)
+
+define void @test_mask_compress_store_pd_512(ptr %addr, <8 x double> %data, i8 %mask)  #0 {
+;
+; CHECK-LABEL: @test_mask_compress_store_pd_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 72) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast i8 [[TMP1]] to <8 x i1>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP6:%.*]] = ptrtoint ptr [[ADDR:%.*]] to i64
+; CHECK-NEXT:    [[TMP7:%.*]] = xor i64 [[TMP6]], 87960930222080
+; CHECK-NEXT:    [[TMP8:%.*]] = inttoptr i64 [[TMP7]] to ptr
+; CHECK-NEXT:    call void @llvm.masked.compressstore.v8i64(<8 x i64> [[TMP3]], ptr [[TMP8]], <8 x i1> [[TMP5]])
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP2]], 0
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <8 x i1> [[TMP4]] to i8
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i8 [[TMP9]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP10:%.*]], label [[TMP11:%.*]], !prof [[PROF1]]
+; CHECK:       10:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       11:
+; CHECK-NEXT:    call void @llvm.masked.compressstore.v8f64(<8 x double> [[DATA:%.*]], ptr [[ADDR]], <8 x i1> [[TMP5]])
+; CHECK-NEXT:    ret void
+;
+  call void @llvm.x86.avx512.mask.compress.store.pd.512(ptr %addr, <8 x double> %data, i8 %mask)
+  ret void
+}
+
+declare void @llvm.x86.avx512.mask.compress.store.pd.512(ptr %addr, <8 x double> %data, i8 %mask)
+
+define void @test_compress_store_pd_512(ptr %addr, <8 x double> %data)  #0 {
+;
+; CHECK-LABEL: @test_compress_store_pd_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = ptrtoint ptr [[ADDR:%.*]] to i64
+; CHECK-NEXT:    [[TMP4:%.*]] = xor i64 [[TMP3]], 87960930222080
+; CHECK-NEXT:    [[TMP5:%.*]] = inttoptr i64 [[TMP4]] to ptr
+; CHECK-NEXT:    call void @llvm.masked.compressstore.v8i64(<8 x i64> [[TMP2]], ptr [[TMP5]], <8 x i1> splat (i1 true))
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP6:%.*]], label [[TMP7:%.*]], !prof [[PROF1]]
+; CHECK:       6:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       7:
+; CHECK-NEXT:    call void @llvm.masked.compressstore.v8f64(<8 x double> [[DATA:%.*]], ptr [[ADDR]], <8 x i1> splat (i1 true))
+; CHECK-NEXT:    ret void
+;
+  call void @llvm.x86.avx512.mask.compress.store.pd.512(ptr %addr, <8 x double> %data, i8 -1)
+  ret void
+}
+
+define void @test_mask_compress_store_ps_512(ptr %addr, <16 x float> %data, i16 %mask)  #0 {
+;
+; CHECK-LABEL: @test_mask_compress_store_ps_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 72) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast i16 [[TMP1]] to <16 x i1>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i16 [[MASK:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP6:%.*]] = ptrtoint ptr [[ADDR:%.*]] to i64
+; CHECK-NEXT:    [[TMP7:%.*]] = xor i64 [[TMP6]], 87960930222080
+; CHECK-NEXT:    [[TMP8:%.*]] = inttoptr i64 [[TMP7]] to ptr
+; CHECK-NEXT:    call void @llvm.masked.compressstore.v16i32(<16 x i32> [[TMP3]], ptr [[TMP8]], <16 x i1> [[TMP5]])
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP2]], 0
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <16 x i1> [[TMP4]] to i16
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i16 [[TMP9]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP10:%.*]], label [[TMP11:%.*]], !prof [[PROF1]]
+; CHECK:       10:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       11:
+; CHECK-NEXT:    call void @llvm.masked.compressstore.v16f32(<16 x float> [[DATA:%.*]], ptr [[ADDR]], <16 x i1> [[TMP5]])
+; CHECK-NEXT:    ret void
+;
+  call void @llvm.x86.avx512.mask.compress.store.ps.512(ptr %addr, <16 x float> %data, i16 %mask)
+  ret void
+}
+
+declare void @llvm.x86.avx512.mask.compress.store.ps.512(ptr %addr, <16 x float> %data, i16 %mask)
+
+define void @test_compress_store_ps_512(ptr %addr, <16 x float> %data)  #0 {
+;
+; CHECK-LABEL: @test_compress_store_ps_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = ptrtoint ptr [[ADDR:%.*]] to i64
+; CHECK-NEXT:    [[TMP4:%.*]] = xor i64 [[TMP3]], 87960930222080
+; CHECK-NEXT:    [[TMP5:%.*]] = inttoptr i64 [[TMP4]] to ptr
+; CHECK-NEXT:    call void @llvm.masked.compressstore.v16i32(<16 x i32> [[TMP2]], ptr [[TMP5]], <16 x i1> splat (i1 true))
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP6:%.*]], label [[TMP7:%.*]], !prof [[PROF1]]
+; CHECK:       6:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       7:
+; CHECK-NEXT:    call void @llvm.masked.compressstore.v16f32(<16 x float> [[DATA:%.*]], ptr [[ADDR]], <16 x i1> splat (i1 true))
+; CHECK-NEXT:    ret void
+;
+  call void @llvm.x86.avx512.mask.compress.store.ps.512(ptr %addr, <16 x float> %data, i16 -1)
+  ret void
+}
+
+define void @test_mask_compress_store_q_512(ptr %addr, <8 x i64> %data, i8 %mask)  #0 {
+;
+; CHECK-LABEL: @test_mask_compress_store_q_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 72) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast i8 [[TMP1]] to <8 x i1>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP6:%.*]] = ptrtoint ptr [[ADDR:%.*]] to i64
+; CHECK-NEXT:    [[TMP7:%.*]] = xor i64 [[TMP6]], 87960930222080
+; CHECK-NEXT:    [[TMP8:%.*]] = inttoptr i64 [[TMP7]] to ptr
+; CHECK-NEXT:    call void @llvm.masked.compressstore.v8i64(<8 x i64> [[TMP3]], ptr [[TMP8]], <8 x i1> [[TMP5]])
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP2]], 0
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <8 x i1> [[TMP4]] to i8
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i8 [[TMP9]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP10:%.*]], label [[TMP11:%.*]], !prof [[PROF1]]
+; CHECK:       10:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       11:
+; CHECK-NEXT:    call void @llvm.masked.compressstore.v8i64(<8 x i64> [[DATA:%.*]], ptr [[ADDR]], <8 x i1> [[TMP5]])
+; CHECK-NEXT:    ret void
+;
+  call void @llvm.x86.avx512.mask.compress.store.q.512(ptr %addr, <8 x i64> %data, i8 %mask)
+  ret void
+}
+
+declare void @llvm.x86.avx512.mask.compress.store.q.512(ptr %addr, <8 x i64> %data, i8 %mask)
+
+define void @test_compress_store_q_512(ptr %addr, <8 x i64> %data)  #0 {
+;
+; CHECK-LABEL: @test_compress_store_q_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = ptrtoint ptr [[ADDR:%.*]] to i64
+; CHECK-NEXT:    [[TMP4:%.*]] = xor i64 [[TMP3]], 87960930222080
+; CHECK-NEXT:    [[TMP5:%.*]] = inttoptr i64 [[TMP4]] to ptr
+; CHECK-NEXT:    call void @llvm.masked.compressstore.v8i64(<8 x i64> [[TMP2]], ptr [[TMP5]], <8 x i1> splat (i1 true))
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP6:%.*]], label [[TMP7:%.*]], !prof [[PROF1]]
+; CHECK:       6:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       7:
+; CHECK-NEXT:    call void @llvm.masked.compressstore.v8i64(<8 x i64> [[DATA:%.*]], ptr [[ADDR]], <8 x i1> splat (i1 true))
+; CHECK-NEXT:    ret void
+;
+  call void @llvm.x86.avx512.mask.compress.store.q.512(ptr %addr, <8 x i64> %data, i8 -1)
+  ret void
+}
+
+define void @test_mask_compress_store_d_512(ptr %addr, <16 x i32> %data, i16 %mask)  #0 {
+;
+; CHECK-LABEL: @test_mask_compress_store_d_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 72) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast i16 [[TMP1]] to <16 x i1>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i16 [[MASK:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP6:%.*]] = ptrtoint ptr [[ADDR:%.*]] to i64
+; CHECK-NEXT:    [[TMP7:%.*]] = xor i64 [[TMP6]], 87960930222080
+; CHECK-NEXT:    [[TMP8:%.*]] = inttoptr i64 [[TMP7]] to ptr
+; CHECK-NEXT:    call void @llvm.masked.compressstore.v16i32(<16 x i32> [[TMP3]], ptr [[TMP8]], <16 x i1> [[TMP5]])
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP2]], 0
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <16 x i1> [[TMP4]] to i16
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i16 [[TMP9]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP10:%.*]], label [[TMP11:%.*]], !prof [[PROF1]]
+; CHECK:       10:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       11:
+; CHECK-NEXT:    call void @llvm.masked.compressstore.v16i32(<16 x i32> [[DATA:%.*]], ptr [[ADDR]], <16 x i1> [[TMP5]])
+; CHECK-NEXT:    ret void
+;
+  call void @llvm.x86.avx512.mask.compress.store.d.512(ptr %addr, <16 x i32> %data, i16 %mask)
+  ret void
+}
+
+declare void @llvm.x86.avx512.mask.compress.store.d.512(ptr %addr, <16 x i32> %data, i16 %mask)
+
+define void @test_compress_store_d_512(ptr %addr, <16 x i32> %data)  #0 {
+;
+; CHECK-LABEL: @test_compress_store_d_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = ptrtoint ptr [[ADDR:%.*]] to i64
+; CHECK-NEXT:    [[TMP4:%.*]] = xor i64 [[TMP3]], 87960930222080
+; CHECK-NEXT:    [[TMP5:%.*]] = inttoptr i64 [[TMP4]] to ptr
+; CHECK-NEXT:    call void @llvm.masked.compressstore.v16i32(<16 x i32> [[TMP2]], ptr [[TMP5]], <16 x i1> splat (i1 true))
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP6:%.*]], label [[TMP7:%.*]], !prof [[PROF1]]
+; CHECK:       6:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       7:
+; CHECK-NEXT:    call void @llvm.masked.compressstore.v16i32(<16 x i32> [[DATA:%.*]], ptr [[ADDR]], <16 x i1> splat (i1 true))
+; CHECK-NEXT:    ret void
+;
+  call void @llvm.x86.avx512.mask.compress.store.d.512(ptr %addr, <16 x i32> %data, i16 -1)
+  ret void
+}
+
+define <8 x double> @test_mask_expand_load_pd_512(ptr %addr, <8 x double> %data, i8 %mask)  #0 {
+;
+; CHECK-LABEL: @test_mask_expand_load_pd_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 72) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast i8 [[TMP1]] to <8 x i1>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP6:%.*]] = ptrtoint ptr [[ADDR:%.*]] to i64
+; CHECK-NEXT:    [[TMP7:%.*]] = xor i64 [[TMP6]], 87960930222080
+; CHECK-NEXT:    [[TMP8:%.*]] = inttoptr i64 [[TMP7]] to ptr
+; CHECK-NEXT:    [[_MSMASKEDEXPLOAD:%.*]] = call <8 x i64> @llvm.masked.expandload.v8i64(ptr [[TMP8]], <8 x i1> [[TMP5]], <8 x i64> [[TMP3]])
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP2]], 0
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <8 x i1> [[TMP4]] to i8
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i8 [[TMP9]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP10:%.*]], label [[TMP11:%.*]], !prof [[PROF1]]
+; CHECK:       10:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       11:
+; CHECK-NEXT:    [[TMP12:%.*]] = call <8 x double> @llvm.masked.expandload.v8f64(ptr [[ADDR]], <8 x i1> [[TMP5]], <8 x double> [[DATA:%.*]])
+; CHECK-NEXT:    store <8 x i64> [[_MSMASKEDEXPLOAD]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x double> [[TMP12]]
+;
+  %res = call <8 x double> @llvm.x86.avx512.mask.expand.load.pd.512(ptr %addr, <8 x double> %data, i8 %mask)
+  ret <8 x double> %res
+}
+
+define <8 x double> @test_maskz_expand_load_pd_512(ptr %addr, i8 %mask)  #0 {
+;
+; CHECK-LABEL: @test_maskz_expand_load_pd_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast i8 [[TMP1]] to <8 x i1>
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP5:%.*]] = ptrtoint ptr [[ADDR:%.*]] to i64
+; CHECK-NEXT:    [[TMP6:%.*]] = xor i64 [[TMP5]], 87960930222080
+; CHECK-NEXT:    [[TMP7:%.*]] = inttoptr i64 [[TMP6]] to ptr
+; CHECK-NEXT:    [[_MSMASKEDEXPLOAD:%.*]] = call <8 x i64> @llvm.masked.expandload.v8i64(ptr [[TMP7]], <8 x i1> [[TMP4]], <8 x i64> zeroinitializer)
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP2]], 0
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <8 x i1> [[TMP3]] to i8
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i8 [[TMP8]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP9:%.*]], label [[TMP10:%.*]], !prof [[PROF1]]
+; CHECK:       9:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       10:
+; CHECK-NEXT:    [[TMP11:%.*]] = call <8 x double> @llvm.masked.expandload.v8f64(ptr [[ADDR]], <8 x i1> [[TMP4]], <8 x double> zeroinitializer)
+; CHECK-NEXT:    store <8 x i64> [[_MSMASKEDEXPLOAD]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x double> [[TMP11]]
+;
+  %res = call <8 x double> @llvm.x86.avx512.mask.expand.load.pd.512(ptr %addr, <8 x double> zeroinitializer, i8 %mask)
+  ret <8 x double> %res
+}
+
+declare <8 x double> @llvm.x86.avx512.mask.expand.load.pd.512(ptr %addr, <8 x double> %data, i8 %mask)
+
+define <8 x double> @test_expand_load_pd_512(ptr %addr, <8 x double> %data)  #0 {
+;
+; CHECK-LABEL: @test_expand_load_pd_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = ptrtoint ptr [[ADDR:%.*]] to i64
+; CHECK-NEXT:    [[TMP4:%.*]] = xor i64 [[TMP3]], 87960930222080
+; CHECK-NEXT:    [[TMP5:%.*]] = inttoptr i64 [[TMP4]] to ptr
+; CHECK-NEXT:    [[_MSMASKEDEXPLOAD:%.*]] = call <8 x i64> @llvm.masked.expandload.v8i64(ptr [[TMP5]], <8 x i1> splat (i1 true), <8 x i64> [[TMP2]])
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP6:%.*]], label [[TMP7:%.*]], !prof [[PROF1]]
+; CHECK:       6:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       7:
+; CHECK-NEXT:    [[TMP8:%.*]] = call <8 x double> @llvm.masked.expandload.v8f64(ptr [[ADDR]], <8 x i1> splat (i1 true), <8 x double> [[DATA:%.*]])
+; CHECK-NEXT:    store <8 x i64> [[_MSMASKEDEXPLOAD]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x double> [[TMP8]]
+;
+  %res = call <8 x double> @llvm.x86.avx512.mask.expand.load.pd.512(ptr %addr, <8 x double> %data, i8 -1)
+  ret <8 x double> %res
+}
+
+; Make sure we don't crash if you pass 0 to the mask.
+define <8 x double> @test_zero_mask_expand_load_pd_512(ptr %addr, <8 x double> %data, i8 %mask)  #0 {
+; CHECK-LABEL: @test_zero_mask_expand_load_pd_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = ptrtoint ptr [[ADDR:%.*]] to i64
+; CHECK-NEXT:    [[TMP4:%.*]] = xor i64 [[TMP3]], 87960930222080
+; CHECK-NEXT:    [[TMP5:%.*]] = inttoptr i64 [[TMP4]] to ptr
+; CHECK-NEXT:    [[_MSMASKEDEXPLOAD:%.*]] = call <8 x i64> @llvm.masked.expandload.v8i64(ptr [[TMP5]], <8 x i1> zeroinitializer, <8 x i64> [[TMP2]])
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP6:%.*]], label [[TMP7:%.*]], !prof [[PROF1]]
+; CHECK:       6:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       7:
+; CHECK-NEXT:    [[TMP8:%.*]] = call <8 x double> @llvm.masked.expandload.v8f64(ptr [[ADDR]], <8 x i1> zeroinitializer, <8 x double> [[DATA:%.*]])
+; CHECK-NEXT:    store <8 x i64> [[_MSMASKEDEXPLOAD]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x double> [[TMP8]]
+;
+  %res = call <8 x double> @llvm.x86.avx512.mask.expand.load.pd.512(ptr %addr, <8 x double> %data, i8 0)
+  ret <8 x double> %res
+}
+
+define <16 x float> @test_mask_expand_load_ps_512(ptr %addr, <16 x float> %data, i16 %mask)  #0 {
+;
+; CHECK-LABEL: @test_mask_expand_load_ps_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 72) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast i16 [[TMP1]] to <16 x i1>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i16 [[MASK:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP6:%.*]] = ptrtoint ptr [[ADDR:%.*]] to i64
+; CHECK-NEXT:    [[TMP7:%.*]] = xor i64 [[TMP6]], 87960930222080
+; CHECK-NEXT:    [[TMP8:%.*]] = inttoptr i64 [[TMP7]] to ptr
+; CHECK-NEXT:    [[_MSMASKEDEXPLOAD:%.*]] = call <16 x i32> @llvm.masked.expandload.v16i32(ptr [[TMP8]], <16 x i1> [[TMP5]], <16 x i32> [[TMP3]])
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP2]], 0
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <16 x i1> [[TMP4]] to i16
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i16 [[TMP9]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP10:%.*]], label [[TMP11:%.*]], !prof [[PROF1]]
+; CHECK:       10:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       11:
+; CHECK-NEXT:    [[TMP12:%.*]] = call <16 x float> @llvm.masked.expandload.v16f32(ptr [[ADDR]], <16 x i1> [[TMP5]], <16 x float> [[DATA:%.*]])
+; CHECK-NEXT:    store <16 x i32> [[_MSMASKEDEXPLOAD]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x float> [[TMP12]]
+;
+  %res = call <16 x float> @llvm.x86.avx512.mask.expand.load.ps.512(ptr %addr, <16 x float> %data, i16 %mask)
+  ret <16 x float> %res
+}
+
+define <16 x float> @test_maskz_expand_load_ps_512(ptr %addr, i16 %mask)  #0 {
+;
+; CHECK-LABEL: @test_maskz_expand_load_ps_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast i16 [[TMP1]] to <16 x i1>
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast i16 [[MASK:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP5:%.*]] = ptrtoint ptr [[ADDR:%.*]] to i64
+; CHECK-NEXT:    [[TMP6:%.*]] = xor i64 [[TMP5]], 87960930222080
+; CHECK-NEXT:    [[TMP7:%.*]] = inttoptr i64 [[TMP6]] to ptr
+; CHECK-NEXT:    [[_MSMASKEDEXPLOAD:%.*]] = call <16 x i32> @llvm.masked.expandload.v16i32(ptr [[TMP7]], <16 x i1> [[TMP4]], <16 x i32> zeroinitializer)
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP2]], 0
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <16 x i1> [[TMP3]] to i16
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i16 [[TMP8]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP9:%.*]], label [[TMP10:%.*]], !prof [[PROF1]]
+; CHECK:       9:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       10:
+; CHECK-NEXT:    [[TMP11:%.*]] = call <16 x float> @llvm.masked.expandload.v16f32(ptr [[ADDR]], <16 x i1> [[TMP4]], <16 x float> zeroinitializer)
+; CHECK-NEXT:    store <16 x i32> [[_MSMASKEDEXPLOAD]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x float> [[TMP11]]
+;
+  %res = call <16 x float> @llvm.x86.avx512.mask.expand.load.ps.512(ptr %addr, <16 x float> zeroinitializer, i16 %mask)
+  ret <16 x float> %res
+}
+
+declare <16 x float> @llvm.x86.avx512.mask.expand.load.ps.512(ptr %addr, <16 x float> %data, i16 %mask)
+
+define <16 x float> @test_expand_load_ps_512(ptr %addr, <16 x float> %data)  #0 {
+;
+; CHECK-LABEL: @test_expand_load_ps_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = ptrtoint ptr [[ADDR:%.*]] to i64
+; CHECK-NEXT:    [[TMP4:%.*]] = xor i64 [[TMP3]], 87960930222080
+; CHECK-NEXT:    [[TMP5:%.*]] = inttoptr i64 [[TMP4]] to ptr
+; CHECK-NEXT:    [[_MSMASKEDEXPLOAD:%.*]] = call <16 x i32> @llvm.masked.expandload.v16i32(ptr [[TMP5]], <16 x i1> splat (i1 true), <16 x i32> [[TMP2]])
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP6:%.*]], label [[TMP7:%.*]], !prof [[PROF1]]
+; CHECK:       6:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       7:
+; CHECK-NEXT:    [[TMP8:%.*]] = call <16 x float> @llvm.masked.expandload.v16f32(ptr [[ADDR]], <16 x i1> splat (i1 true), <16 x float> [[DATA:%.*]])
+; CHECK-NEXT:    store <16 x i32> [[_MSMASKEDEXPLOAD]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x float> [[TMP8]]
+;
+  %res = call <16 x float> @llvm.x86.avx512.mask.expand.load.ps.512(ptr %addr, <16 x float> %data, i16 -1)
+  ret <16 x float> %res
+}
+
+define <8 x i64> @test_mask_expand_load_q_512(ptr %addr, <8 x i64> %data, i8 %mask)  #0 {
+;
+; CHECK-LABEL: @test_mask_expand_load_q_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 72) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast i8 [[TMP1]] to <8 x i1>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP6:%.*]] = ptrtoint ptr [[ADDR:%.*]] to i64
+; CHECK-NEXT:    [[TMP7:%.*]] = xor i64 [[TMP6]], 87960930222080
+; CHECK-NEXT:    [[TMP8:%.*]] = inttoptr i64 [[TMP7]] to ptr
+; CHECK-NEXT:    [[_MSMASKEDEXPLOAD:%.*]] = call <8 x i64> @llvm.masked.expandload.v8i64(ptr [[TMP8]], <8 x i1> [[TMP5]], <8 x i64> [[TMP3]])
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP2]], 0
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <8 x i1> [[TMP4]] to i8
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i8 [[TMP9]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP10:%.*]], label [[TMP11:%.*]], !prof [[PROF1]]
+; CHECK:       10:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       11:
+; CHECK-NEXT:    [[TMP12:%.*]] = call <8 x i64> @llvm.masked.expandload.v8i64(ptr [[ADDR]], <8 x i1> [[TMP5]], <8 x i64> [[DATA:%.*]])
+; CHECK-NEXT:    store <8 x i64> [[_MSMASKEDEXPLOAD]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i64> [[TMP12]]
+;
+  %res = call <8 x i64> @llvm.x86.avx512.mask.expand.load.q.512(ptr %addr, <8 x i64> %data, i8 %mask)
+  ret <8 x i64> %res
+}
+
+define <8 x i64> @test_maskz_expand_load_q_512(ptr %addr, i8 %mask)  #0 {
+;
+; CHECK-LABEL: @test_maskz_expand_load_q_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast i8 [[TMP1]] to <8 x i1>
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP5:%.*]] = ptrtoint ptr [[ADDR:%.*]] to i64
+; CHECK-NEXT:    [[TMP6:%.*]] = xor i64 [[TMP5]], 87960930222080
+; CHECK-NEXT:    [[TMP7:%.*]] = inttoptr i64 [[TMP6]] to ptr
+; CHECK-NEXT:    [[_MSMASKEDEXPLOAD:%.*]] = call <8 x i64> @llvm.masked.expandload.v8i64(ptr [[TMP7]], <8 x i1> [[TMP4]], <8 x i64> zeroinitializer)
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP2]], 0
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <8 x i1> [[TMP3]] to i8
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i8 [[TMP8]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP9:%.*]], label [[TMP10:%.*]], !prof [[PROF1]]
+; CHECK:       9:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       10:
+; CHECK-NEXT:    [[TMP11:%.*]] = call <8 x i64> @llvm.masked.expandload.v8i64(ptr [[ADDR]], <8 x i1> [[TMP4]], <8 x i64> zeroinitializer)
+; CHECK-NEXT:    store <8 x i64> [[_MSMASKEDEXPLOAD]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i64> [[TMP11]]
+;
+  %res = call <8 x i64> @llvm.x86.avx512.mask.expand.load.q.512(ptr %addr, <8 x i64> zeroinitializer, i8 %mask)
+  ret <8 x i64> %res
+}
+
+declare <8 x i64> @llvm.x86.avx512.mask.expand.load.q.512(ptr %addr, <8 x i64> %data, i8 %mask)
+
+define <8 x i64> @test_expand_load_q_512(ptr %addr, <8 x i64> %data)  #0 {
+;
+; CHECK-LABEL: @test_expand_load_q_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = ptrtoint ptr [[ADDR:%.*]] to i64
+; CHECK-NEXT:    [[TMP4:%.*]] = xor i64 [[TMP3]], 87960930222080
+; CHECK-NEXT:    [[TMP5:%.*]] = inttoptr i64 [[TMP4]] to ptr
+; CHECK-NEXT:    [[_MSMASKEDEXPLOAD:%.*]] = call <8 x i64> @llvm.masked.expandload.v8i64(ptr [[TMP5]], <8 x i1> splat (i1 true), <8 x i64> [[TMP2]])
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP6:%.*]], label [[TMP7:%.*]], !prof [[PROF1]]
+; CHECK:       6:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       7:
+; CHECK-NEXT:    [[TMP8:%.*]] = call <8 x i64> @llvm.masked.expandload.v8i64(ptr [[ADDR]], <8 x i1> splat (i1 true), <8 x i64> [[DATA:%.*]])
+; CHECK-NEXT:    store <8 x i64> [[_MSMASKEDEXPLOAD]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i64> [[TMP8]]
+;
+  %res = call <8 x i64> @llvm.x86.avx512.mask.expand.load.q.512(ptr %addr, <8 x i64> %data, i8 -1)
+  ret <8 x i64> %res
+}
+
+define <16 x i32> @test_mask_expand_load_d_512(ptr %addr, <16 x i32> %data, i16 %mask)  #0 {
+;
+; CHECK-LABEL: @test_mask_expand_load_d_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 72) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast i16 [[TMP1]] to <16 x i1>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i16 [[MASK:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP6:%.*]] = ptrtoint ptr [[ADDR:%.*]] to i64
+; CHECK-NEXT:    [[TMP7:%.*]] = xor i64 [[TMP6]], 87960930222080
+; CHECK-NEXT:    [[TMP8:%.*]] = inttoptr i64 [[TMP7]] to ptr
+; CHECK-NEXT:    [[_MSMASKEDEXPLOAD:%.*]] = call <16 x i32> @llvm.masked.expandload.v16i32(ptr [[TMP8]], <16 x i1> [[TMP5]], <16 x i32> [[TMP3]])
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP2]], 0
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <16 x i1> [[TMP4]] to i16
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i16 [[TMP9]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP10:%.*]], label [[TMP11:%.*]], !prof [[PROF1]]
+; CHECK:       10:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       11:
+; CHECK-NEXT:    [[TMP12:%.*]] = call <16 x i32> @llvm.masked.expandload.v16i32(ptr [[ADDR]], <16 x i1> [[TMP5]], <16 x i32> [[DATA:%.*]])
+; CHECK-NEXT:    store <16 x i32> [[_MSMASKEDEXPLOAD]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i32> [[TMP12]]
+;
+  %res = call <16 x i32> @llvm.x86.avx512.mask.expand.load.d.512(ptr %addr, <16 x i32> %data, i16 %mask)
+  ret <16 x i32> %res
+}
+
+define <16 x i32> @test_maskz_expand_load_d_512(ptr %addr, i16 %mask)  #0 {
+;
+; CHECK-LABEL: @test_maskz_expand_load_d_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast i16 [[TMP1]] to <16 x i1>
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast i16 [[MASK:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP5:%.*]] = ptrtoint ptr [[ADDR:%.*]] to i64
+; CHECK-NEXT:    [[TMP6:%.*]] = xor i64 [[TMP5]], 87960930222080
+; CHECK-NEXT:    [[TMP7:%.*]] = inttoptr i64 [[TMP6]] to ptr
+; CHECK-NEXT:    [[_MSMASKEDEXPLOAD:%.*]] = call <16 x i32> @llvm.masked.expandload.v16i32(ptr [[TMP7]], <16 x i1> [[TMP4]], <16 x i32> zeroinitializer)
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP2]], 0
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <16 x i1> [[TMP3]] to i16
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i16 [[TMP8]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP9:%.*]], label [[TMP10:%.*]], !prof [[PROF1]]
+; CHECK:       9:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       10:
+; CHECK-NEXT:    [[TMP11:%.*]] = call <16 x i32> @llvm.masked.expandload.v16i32(ptr [[ADDR]], <16 x i1> [[TMP4]], <16 x i32> zeroinitializer)
+; CHECK-NEXT:    store <16 x i32> [[_MSMASKEDEXPLOAD]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i32> [[TMP11]]
+;
+  %res = call <16 x i32> @llvm.x86.avx512.mask.expand.load.d.512(ptr %addr, <16 x i32> zeroinitializer, i16 %mask)
+  ret <16 x i32> %res
+}
+
+declare <16 x i32> @llvm.x86.avx512.mask.expand.load.d.512(ptr %addr, <16 x i32> %data, i16 %mask)
+
+define <16 x i32> @test_expand_load_d_512(ptr %addr, <16 x i32> %data)  #0 {
+;
+; CHECK-LABEL: @test_expand_load_d_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = ptrtoint ptr [[ADDR:%.*]] to i64
+; CHECK-NEXT:    [[TMP4:%.*]] = xor i64 [[TMP3]], 87960930222080
+; CHECK-NEXT:    [[TMP5:%.*]] = inttoptr i64 [[TMP4]] to ptr
+; CHECK-NEXT:    [[_MSMASKEDEXPLOAD:%.*]] = call <16 x i32> @llvm.masked.expandload.v16i32(ptr [[TMP5]], <16 x i1> splat (i1 true), <16 x i32> [[TMP2]])
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP6:%.*]], label [[TMP7:%.*]], !prof [[PROF1]]
+; CHECK:       6:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       7:
+; CHECK-NEXT:    [[TMP8:%.*]] = call <16 x i32> @llvm.masked.expandload.v16i32(ptr [[ADDR]], <16 x i1> splat (i1 true), <16 x i32> [[DATA:%.*]])
+; CHECK-NEXT:    store <16 x i32> [[_MSMASKEDEXPLOAD]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i32> [[TMP8]]
+;
+  %res = call <16 x i32> @llvm.x86.avx512.mask.expand.load.d.512(ptr %addr, <16 x i32> %data, i16 -1)
+  ret <16 x i32> %res
+}
+
+define <16 x float> @test_mm512_maskz_min_round_ps_sae(<16 x float> %a0, <16 x float> %a1, i16 %mask)  #0 {
+;
+; CHECK-LABEL: @test_mm512_maskz_min_round_ps_sae(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <16 x i32> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <16 x i32> [[_MSPROP]], zeroinitializer
+; CHECK-NEXT:    [[TMP8:%.*]] = call <16 x float> @llvm.x86.avx512.min.ps.512(<16 x float> [[A0:%.*]], <16 x float> [[A1:%.*]], i32 8)
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast i16 [[TMP3]] to <16 x i1>
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i16 [[MASK:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP11:%.*]] = select <16 x i1> [[TMP10]], <16 x i32> [[_MSPROP1]], <16 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <16 x float> [[TMP8]] to <16 x i32>
+; CHECK-NEXT:    [[TMP13:%.*]] = xor <16 x i32> [[TMP12]], zeroinitializer
+; CHECK-NEXT:    [[TMP14:%.*]] = or <16 x i32> [[TMP13]], [[_MSPROP1]]
+; CHECK-NEXT:    [[TMP15:%.*]] = or <16 x i32> [[TMP14]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP9]], <16 x i32> [[TMP15]], <16 x i32> [[TMP11]]
+; CHECK-NEXT:    [[TMP16:%.*]] = select <16 x i1> [[TMP10]], <16 x float> [[TMP8]], <16 x float> zeroinitializer
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x float> [[TMP16]]
+;
+  %res = call <16 x float> @llvm.x86.avx512.mask.min.ps.512(<16 x float> %a0, <16 x float> %a1, <16 x float>zeroinitializer, i16 %mask, i32 8)
+  ret <16 x float> %res
+}
+
+define <16 x float> @test_mm512_maskz_min_round_ps_current(<16 x float> %a0, <16 x float> %a1, i16 %mask)  #0 {
+;
+; CHECK-LABEL: @test_mm512_maskz_min_round_ps_current(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <16 x i32> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <16 x i32> [[_MSPROP]], zeroinitializer
+; CHECK-NEXT:    [[TMP8:%.*]] = call <16 x float> @llvm.x86.avx512.min.ps.512(<16 x float> [[A0:%.*]], <16 x float> [[A1:%.*]], i32 4)
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast i16 [[TMP3]] to <16 x i1>
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i16 [[MASK:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP11:%.*]] = select <16 x i1> [[TMP10]], <16 x i32> [[_MSPROP1]], <16 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <16 x float> [[TMP8]] to <16 x i32>
+; CHECK-NEXT:    [[TMP13:%.*]] = xor <16 x i32> [[TMP12]], zeroinitializer
+; CHECK-NEXT:    [[TMP14:%.*]] = or <16 x i32> [[TMP13]], [[_MSPROP1]]
+; CHECK-NEXT:    [[TMP15:%.*]] = or <16 x i32> [[TMP14]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP9]], <16 x i32> [[TMP15]], <16 x i32> [[TMP11]]
+; CHECK-NEXT:    [[TMP16:%.*]] = select <16 x i1> [[TMP10]], <16 x float> [[TMP8]], <16 x float> zeroinitializer
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x float> [[TMP16]]
+;
+  %res = call <16 x float> @llvm.x86.avx512.mask.min.ps.512(<16 x float> %a0, <16 x float> %a1, <16 x float>zeroinitializer, i16 %mask, i32 4)
+  ret <16 x float> %res
+}
+
+define <16 x float> @test_mm512_mask_min_round_ps_sae(<16 x float> %a0, <16 x float> %a1, <16 x float> %src, i16 %mask)  #0 {
+;
+; CHECK-LABEL: @test_mm512_mask_min_round_ps_sae(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <16 x i32> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <16 x i32> [[_MSPROP]], zeroinitializer
+; CHECK-NEXT:    [[TMP9:%.*]] = call <16 x float> @llvm.x86.avx512.min.ps.512(<16 x float> [[A0:%.*]], <16 x float> [[A1:%.*]], i32 8)
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i16 [[TMP3]] to <16 x i1>
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast i16 [[MASK:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP12:%.*]] = select <16 x i1> [[TMP11]], <16 x i32> [[_MSPROP1]], <16 x i32> [[TMP4]]
+; CHECK-NEXT:    [[TMP13:%.*]] = bitcast <16 x float> [[TMP9]] to <16 x i32>
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast <16 x float> [[SRC:%.*]] to <16 x i32>
+; CHECK-NEXT:    [[TMP15:%.*]] = xor <16 x i32> [[TMP13]], [[TMP14]]
+; CHECK-NEXT:    [[TMP16:%.*]] = or <16 x i32> [[TMP15]], [[_MSPROP1]]
+; CHECK-NEXT:    [[TMP17:%.*]] = or <16 x i32> [[TMP16]], [[TMP4]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP10]], <16 x i32> [[TMP17]], <16 x i32> [[TMP12]]
+; CHECK-NEXT:    [[TMP18:%.*]] = select <16 x i1> [[TMP11]], <16 x float> [[TMP9]], <16 x float> [[SRC]]
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x float> [[TMP18]]
+;
+  %res = call <16 x float> @llvm.x86.avx512.mask.min.ps.512(<16 x float> %a0, <16 x float> %a1, <16 x float> %src, i16 %mask, i32 8)
+  ret <16 x float> %res
+}
+
+define <16 x float> @test_mm512_mask_min_round_ps_current(<16 x float> %a0, <16 x float> %a1, <16 x float> %src, i16 %mask)  #0 {
+;
+; CHECK-LABEL: @test_mm512_mask_min_round_ps_current(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <16 x i32> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <16 x i32> [[_MSPROP]], zeroinitializer
+; CHECK-NEXT:    [[TMP9:%.*]] = call <16 x float> @llvm.x86.avx512.min.ps.512(<16 x float> [[A0:%.*]], <16 x float> [[A1:%.*]], i32 4)
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i16 [[TMP3]] to <16 x i1>
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast i16 [[MASK:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP12:%.*]] = select <16 x i1> [[TMP11]], <16 x i32> [[_MSPROP1]], <16 x i32> [[TMP4]]
+; CHECK-NEXT:    [[TMP13:%.*]] = bitcast <16 x float> [[TMP9]] to <16 x i32>
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast <16 x float> [[SRC:%.*]] to <16 x i32>
+; CHECK-NEXT:    [[TMP15:%.*]] = xor <16 x i32> [[TMP13]], [[TMP14]]
+; CHECK-NEXT:    [[TMP16:%.*]] = or <16 x i32> [[TMP15]], [[_MSPROP1]]
+; CHECK-NEXT:    [[TMP17:%.*]] = or <16 x i32> [[TMP16]], [[TMP4]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP10]], <16 x i32> [[TMP17]], <16 x i32> [[TMP12]]
+; CHECK-NEXT:    [[TMP18:%.*]] = select <16 x i1> [[TMP11]], <16 x float> [[TMP9]], <16 x float> [[SRC]]
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x float> [[TMP18]]
+;
+  %res = call <16 x float> @llvm.x86.avx512.mask.min.ps.512(<16 x float> %a0, <16 x float> %a1, <16 x float> %src, i16 %mask, i32 4)
+  ret <16 x float> %res
+}
+
+define <16 x float> @test_mm512_min_round_ps_sae(<16 x float> %a0, <16 x float> %a1, i16 %mask)  #0 {
+; CHECK-LABEL: @test_mm512_min_round_ps_sae(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <16 x i32> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <16 x i32> [[_MSPROP]], zeroinitializer
+; CHECK-NEXT:    [[TMP7:%.*]] = call <16 x float> @llvm.x86.avx512.min.ps.512(<16 x float> [[A0:%.*]], <16 x float> [[A1:%.*]], i32 8)
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP1]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x float> [[TMP7]]
+;
+  %res = call <16 x float> @llvm.x86.avx512.mask.min.ps.512(<16 x float> %a0, <16 x float> %a1, <16 x float>zeroinitializer, i16 -1, i32 8)
+  ret <16 x float> %res
+}
+
+define <16 x float> @test_mm512_min_round_ps_current(<16 x float> %a0, <16 x float> %a1, i16 %mask)  #0 {
+; CHECK-LABEL: @test_mm512_min_round_ps_current(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <16 x i32> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <16 x i32> [[_MSPROP]], zeroinitializer
+; CHECK-NEXT:    [[TMP7:%.*]] = call <16 x float> @llvm.x86.avx512.min.ps.512(<16 x float> [[A0:%.*]], <16 x float> [[A1:%.*]], i32 4)
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP1]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x float> [[TMP7]]
+;
+  %res = call <16 x float> @llvm.x86.avx512.mask.min.ps.512(<16 x float> %a0, <16 x float> %a1, <16 x float>zeroinitializer, i16 -1, i32 4)
+  ret <16 x float> %res
+}
+declare <16 x float> @llvm.x86.avx512.mask.min.ps.512(<16 x float>, <16 x float>, <16 x float>, i16, i32)
+
+define <16 x float> @test_mm512_maskz_max_round_ps_sae(<16 x float> %a0, <16 x float> %a1, i16 %mask)  #0 {
+;
+; CHECK-LABEL: @test_mm512_maskz_max_round_ps_sae(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <16 x i32> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <16 x i32> [[_MSPROP]], zeroinitializer
+; CHECK-NEXT:    [[TMP8:%.*]] = call <16 x float> @llvm.x86.avx512.max.ps.512(<16 x float> [[A0:%.*]], <16 x float> [[A1:%.*]], i32 8)
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast i16 [[TMP3]] to <16 x i1>
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i16 [[MASK:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP11:%.*]] = select <16 x i1> [[TMP10]], <16 x i32> [[_MSPROP1]], <16 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <16 x float> [[TMP8]] to <16 x i32>
+; CHECK-NEXT:    [[TMP13:%.*]] = xor <16 x i32> [[TMP12]], zeroinitializer
+; CHECK-NEXT:    [[TMP14:%.*]] = or <16 x i32> [[TMP13]], [[_MSPROP1]]
+; CHECK-NEXT:    [[TMP15:%.*]] = or <16 x i32> [[TMP14]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP9]], <16 x i32> [[TMP15]], <16 x i32> [[TMP11]]
+; CHECK-NEXT:    [[TMP16:%.*]] = select <16 x i1> [[TMP10]], <16 x float> [[TMP8]], <16 x float> zeroinitializer
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x float> [[TMP16]]
+;
+  %res = call <16 x float> @llvm.x86.avx512.mask.max.ps.512(<16 x float> %a0, <16 x float> %a1, <16 x float>zeroinitializer, i16 %mask, i32 8)
+  ret <16 x float> %res
+}
+
+define <16 x float> @test_mm512_maskz_max_round_ps_current(<16 x float> %a0, <16 x float> %a1, i16 %mask)  #0 {
+;
+; CHECK-LABEL: @test_mm512_maskz_max_round_ps_current(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <16 x i32> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <16 x i32> [[_MSPROP]], zeroinitializer
+; CHECK-NEXT:    [[TMP8:%.*]] = call <16 x float> @llvm.x86.avx512.max.ps.512(<16 x float> [[A0:%.*]], <16 x float> [[A1:%.*]], i32 4)
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast i16 [[TMP3]] to <16 x i1>
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i16 [[MASK:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP11:%.*]] = select <16 x i1> [[TMP10]], <16 x i32> [[_MSPROP1]], <16 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <16 x float> [[TMP8]] to <16 x i32>
+; CHECK-NEXT:    [[TMP13:%.*]] = xor <16 x i32> [[TMP12]], zeroinitializer
+; CHECK-NEXT:    [[TMP14:%.*]] = or <16 x i32> [[TMP13]], [[_MSPROP1]]
+; CHECK-NEXT:    [[TMP15:%.*]] = or <16 x i32> [[TMP14]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP9]], <16 x i32> [[TMP15]], <16 x i32> [[TMP11]]
+; CHECK-NEXT:    [[TMP16:%.*]] = select <16 x i1> [[TMP10]], <16 x float> [[TMP8]], <16 x float> zeroinitializer
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x float> [[TMP16]]
+;
+  %res = call <16 x float> @llvm.x86.avx512.mask.max.ps.512(<16 x float> %a0, <16 x float> %a1, <16 x float>zeroinitializer, i16 %mask, i32 4)
+  ret <16 x float> %res
+}
+
+define <16 x float> @test_mm512_mask_max_round_ps_sae(<16 x float> %a0, <16 x float> %a1, <16 x float> %src, i16 %mask)  #0 {
+;
+; CHECK-LABEL: @test_mm512_mask_max_round_ps_sae(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <16 x i32> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <16 x i32> [[_MSPROP]], zeroinitializer
+; CHECK-NEXT:    [[TMP9:%.*]] = call <16 x float> @llvm.x86.avx512.max.ps.512(<16 x float> [[A0:%.*]], <16 x float> [[A1:%.*]], i32 8)
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i16 [[TMP3]] to <16 x i1>
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast i16 [[MASK:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP12:%.*]] = select <16 x i1> [[TMP11]], <16 x i32> [[_MSPROP1]], <16 x i32> [[TMP4]]
+; CHECK-NEXT:    [[TMP13:%.*]] = bitcast <16 x float> [[TMP9]] to <16 x i32>
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast <16 x float> [[SRC:%.*]] to <16 x i32>
+; CHECK-NEXT:    [[TMP15:%.*]] = xor <16 x i32> [[TMP13]], [[TMP14]]
+; CHECK-NEXT:    [[TMP16:%.*]] = or <16 x i32> [[TMP15]], [[_MSPROP1]]
+; CHECK-NEXT:    [[TMP17:%.*]] = or <16 x i32> [[TMP16]], [[TMP4]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP10]], <16 x i32> [[TMP17]], <16 x i32> [[TMP12]]
+; CHECK-NEXT:    [[TMP18:%.*]] = select <16 x i1> [[TMP11]], <16 x float> [[TMP9]], <16 x float> [[SRC]]
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x float> [[TMP18]]
+;
+  %res = call <16 x float> @llvm.x86.avx512.mask.max.ps.512(<16 x float> %a0, <16 x float> %a1, <16 x float> %src, i16 %mask, i32 8)
+  ret <16 x float> %res
+}
+
+define <16 x float> @test_mm512_mask_max_round_ps_current(<16 x float> %a0, <16 x float> %a1, <16 x float> %src, i16 %mask)  #0 {
+;
+; CHECK-LABEL: @test_mm512_mask_max_round_ps_current(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <16 x i32> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <16 x i32> [[_MSPROP]], zeroinitializer
+; CHECK-NEXT:    [[TMP9:%.*]] = call <16 x float> @llvm.x86.avx512.max.ps.512(<16 x float> [[A0:%.*]], <16 x float> [[A1:%.*]], i32 4)
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i16 [[TMP3]] to <16 x i1>
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast i16 [[MASK:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP12:%.*]] = select <16 x i1> [[TMP11]], <16 x i32> [[_MSPROP1]], <16 x i32> [[TMP4]]
+; CHECK-NEXT:    [[TMP13:%.*]] = bitcast <16 x float> [[TMP9]] to <16 x i32>
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast <16 x float> [[SRC:%.*]] to <16 x i32>
+; CHECK-NEXT:    [[TMP15:%.*]] = xor <16 x i32> [[TMP13]], [[TMP14]]
+; CHECK-NEXT:    [[TMP16:%.*]] = or <16 x i32> [[TMP15]], [[_MSPROP1]]
+; CHECK-NEXT:    [[TMP17:%.*]] = or <16 x i32> [[TMP16]], [[TMP4]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP10]], <16 x i32> [[TMP17]], <16 x i32> [[TMP12]]
+; CHECK-NEXT:    [[TMP18:%.*]] = select <16 x i1> [[TMP11]], <16 x float> [[TMP9]], <16 x float> [[SRC]]
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x float> [[TMP18]]
+;
+  %res = call <16 x float> @llvm.x86.avx512.mask.max.ps.512(<16 x float> %a0, <16 x float> %a1, <16 x float> %src, i16 %mask, i32 4)
+  ret <16 x float> %res
+}
+
+define <16 x float> @test_mm512_max_round_ps_sae(<16 x float> %a0, <16 x float> %a1, i16 %mask)  #0 {
+; CHECK-LABEL: @test_mm512_max_round_ps_sae(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <16 x i32> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <16 x i32> [[_MSPROP]], zeroinitializer
+; CHECK-NEXT:    [[TMP7:%.*]] = call <16 x float> @llvm.x86.avx512.max.ps.512(<16 x float> [[A0:%.*]], <16 x float> [[A1:%.*]], i32 8)
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP1]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x float> [[TMP7]]
+;
+  %res = call <16 x float> @llvm.x86.avx512.mask.max.ps.512(<16 x float> %a0, <16 x float> %a1, <16 x float>zeroinitializer, i16 -1, i32 8)
+  ret <16 x float> %res
+}
+
+define <16 x float> @test_mm512_max_round_ps_current(<16 x float> %a0, <16 x float> %a1, i16 %mask)  #0 {
+; CHECK-LABEL: @test_mm512_max_round_ps_current(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <16 x i32> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <16 x i32> [[_MSPROP]], zeroinitializer
+; CHECK-NEXT:    [[TMP7:%.*]] = call <16 x float> @llvm.x86.avx512.max.ps.512(<16 x float> [[A0:%.*]], <16 x float> [[A1:%.*]], i32 4)
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP1]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x float> [[TMP7]]
+;
+  %res = call <16 x float> @llvm.x86.avx512.mask.max.ps.512(<16 x float> %a0, <16 x float> %a1, <16 x float>zeroinitializer, i16 -1, i32 4)
+  ret <16 x float> %res
+}
+declare <16 x float> @llvm.x86.avx512.mask.max.ps.512(<16 x float>, <16 x float>, <16 x float>, i16, i32)
+
+define <8 x double> @test_sqrt_pd_512(<8 x double> %a0, <8 x double> %extra_param)  #0 {
+; CHECK-LABEL: @test_sqrt_pd_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = call <8 x double> @llvm.sqrt.v8f64(<8 x double> [[A0:%.*]])
+; CHECK-NEXT:    store <8 x i64> [[TMP1]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x double> [[TMP2]]
+;
+  %res = call <8 x double> @llvm.x86.avx512.mask.sqrt.pd.512(<8 x double> %a0,  <8 x double> %extra_param, i8 -1, i32 4)
+  ret <8 x double> %res
+}
+define <8 x double> @test_mask_sqrt_pd_512(<8 x double> %a0, <8 x double> %passthru, i8 %mask)  #0 {
+;
+; CHECK-LABEL: @test_mask_sqrt_pd_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = call <8 x double> @llvm.sqrt.v8f64(<8 x double> [[A0:%.*]])
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i8 [[TMP2]] to <8 x i1>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP7:%.*]] = select <8 x i1> [[TMP6]], <8 x i64> [[TMP1]], <8 x i64> [[TMP3]]
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <8 x double> [[TMP4]] to <8 x i64>
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <8 x double> [[PASSTHRU:%.*]] to <8 x i64>
+; CHECK-NEXT:    [[TMP10:%.*]] = xor <8 x i64> [[TMP8]], [[TMP9]]
+; CHECK-NEXT:    [[TMP11:%.*]] = or <8 x i64> [[TMP10]], [[TMP1]]
+; CHECK-NEXT:    [[TMP12:%.*]] = or <8 x i64> [[TMP11]], [[TMP3]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP5]], <8 x i64> [[TMP12]], <8 x i64> [[TMP7]]
+; CHECK-NEXT:    [[TMP13:%.*]] = select <8 x i1> [[TMP6]], <8 x double> [[TMP4]], <8 x double> [[PASSTHRU]]
+; CHECK-NEXT:    store <8 x i64> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x double> [[TMP13]]
+;
+  %res = call <8 x double> @llvm.x86.avx512.mask.sqrt.pd.512(<8 x double> %a0,  <8 x double> %passthru, i8 %mask, i32 4)
+  ret <8 x double> %res
+}
+define <8 x double> @test_maskz_sqrt_pd_512(<8 x double> %a0, i8 %mask)  #0 {
+;
+; CHECK-LABEL: @test_maskz_sqrt_pd_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = call <8 x double> @llvm.sqrt.v8f64(<8 x double> [[A0:%.*]])
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast i8 [[TMP2]] to <8 x i1>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP6:%.*]] = select <8 x i1> [[TMP5]], <8 x i64> [[TMP1]], <8 x i64> zeroinitializer
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <8 x double> [[TMP3]] to <8 x i64>
+; CHECK-NEXT:    [[TMP8:%.*]] = xor <8 x i64> [[TMP7]], zeroinitializer
+; CHECK-NEXT:    [[TMP9:%.*]] = or <8 x i64> [[TMP8]], [[TMP1]]
+; CHECK-NEXT:    [[TMP10:%.*]] = or <8 x i64> [[TMP9]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP4]], <8 x i64> [[TMP10]], <8 x i64> [[TMP6]]
+; CHECK-NEXT:    [[TMP11:%.*]] = select <8 x i1> [[TMP5]], <8 x double> [[TMP3]], <8 x double> zeroinitializer
+; CHECK-NEXT:    store <8 x i64> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x double> [[TMP11]]
+;
+  %res = call <8 x double> @llvm.x86.avx512.mask.sqrt.pd.512(<8 x double> %a0,  <8 x double> zeroinitializer, i8 %mask, i32 4)
+  ret <8 x double> %res
+}
+define <8 x double> @test_sqrt_round_pd_512(<8 x double> %a0, <8 x double> %extra_param)  #0 {
+; CHECK-LABEL: @test_sqrt_round_pd_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <8 x i64> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP2]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP3:%.*]], label [[TMP4:%.*]], !prof [[PROF1]]
+; CHECK:       3:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       4:
+; CHECK-NEXT:    [[TMP5:%.*]] = call <8 x double> @llvm.x86.avx512.sqrt.pd.512(<8 x double> [[A0:%.*]], i32 11)
+; CHECK-NEXT:    store <8 x i64> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x double> [[TMP5]]
+;
+  %res = call <8 x double> @llvm.x86.avx512.mask.sqrt.pd.512(<8 x double> %a0,  <8 x double> %extra_param, i8 -1, i32 11)
+  ret <8 x double> %res
+}
+define <8 x double> @test_mask_sqrt_round_pd_512(<8 x double> %a0, <8 x double> %passthru, i8 %mask)  #0 {
+;
+; CHECK-LABEL: @test_mask_sqrt_round_pd_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <8 x i64> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP4]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP5:%.*]], label [[TMP6:%.*]], !prof [[PROF1]]
+; CHECK:       5:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       6:
+; CHECK-NEXT:    [[TMP7:%.*]] = call <8 x double> @llvm.x86.avx512.sqrt.pd.512(<8 x double> [[A0:%.*]], i32 11)
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast i8 [[TMP2]] to <8 x i1>
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP10:%.*]] = select <8 x i1> [[TMP9]], <8 x i64> zeroinitializer, <8 x i64> [[TMP3]]
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <8 x double> [[TMP7]] to <8 x i64>
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <8 x double> [[PASSTHRU:%.*]] to <8 x i64>
+; CHECK-NEXT:    [[TMP13:%.*]] = xor <8 x i64> [[TMP11]], [[TMP12]]
+; CHECK-NEXT:    [[TMP14:%.*]] = or <8 x i64> [[TMP13]], zeroinitializer
+; CHECK-NEXT:    [[TMP15:%.*]] = or <8 x i64> [[TMP14]], [[TMP3]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP8]], <8 x i64> [[TMP15]], <8 x i64> [[TMP10]]
+; CHECK-NEXT:    [[TMP16:%.*]] = select <8 x i1> [[TMP9]], <8 x double> [[TMP7]], <8 x double> [[PASSTHRU]]
+; CHECK-NEXT:    store <8 x i64> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x double> [[TMP16]]
+;
+  %res = call <8 x double> @llvm.x86.avx512.mask.sqrt.pd.512(<8 x double> %a0,  <8 x double> %passthru, i8 %mask, i32 11)
+  ret <8 x double> %res
+}
+define <8 x double> @test_maskz_sqrt_round_pd_512(<8 x double> %a0, i8 %mask)  #0 {
+;
+; CHECK-LABEL: @test_maskz_sqrt_round_pd_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <8 x i64> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP3]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP4:%.*]], label [[TMP5:%.*]], !prof [[PROF1]]
+; CHECK:       4:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       5:
+; CHECK-NEXT:    [[TMP6:%.*]] = call <8 x double> @llvm.x86.avx512.sqrt.pd.512(<8 x double> [[A0:%.*]], i32 11)
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast i8 [[TMP2]] to <8 x i1>
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP9:%.*]] = select <8 x i1> [[TMP8]], <8 x i64> zeroinitializer, <8 x i64> zeroinitializer
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast <8 x double> [[TMP6]] to <8 x i64>
+; CHECK-NEXT:    [[TMP11:%.*]] = xor <8 x i64> [[TMP10]], zeroinitializer
+; CHECK-NEXT:    [[TMP12:%.*]] = or <8 x i64> [[TMP11]], zeroinitializer
+; CHECK-NEXT:    [[TMP13:%.*]] = or <8 x i64> [[TMP12]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP7]], <8 x i64> [[TMP13]], <8 x i64> [[TMP9]]
+; CHECK-NEXT:    [[TMP14:%.*]] = select <8 x i1> [[TMP8]], <8 x double> [[TMP6]], <8 x double> zeroinitializer
+; CHECK-NEXT:    store <8 x i64> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x double> [[TMP14]]
+;
+  %res = call <8 x double> @llvm.x86.avx512.mask.sqrt.pd.512(<8 x double> %a0,  <8 x double> zeroinitializer, i8 %mask, i32 11)
+  ret <8 x double> %res
+}
+declare <8 x double> @llvm.x86.avx512.mask.sqrt.pd.512(<8 x double>, <8 x double>, i8, i32) nounwind readnone
+
+define <16 x float> @test_sqrt_ps_512(<16 x float> %a0, <16 x float> %extra_param)  #0 {
+; CHECK-LABEL: @test_sqrt_ps_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = call <16 x float> @llvm.sqrt.v16f32(<16 x float> [[A0:%.*]])
+; CHECK-NEXT:    store <16 x i32> [[TMP1]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x float> [[TMP2]]
+;
+  %res = call <16 x float> @llvm.x86.avx512.mask.sqrt.ps.512(<16 x float> %a0, <16 x float> %extra_param, i16 -1, i32 4)
+  ret <16 x float> %res
+}
+define <16 x float> @test_mask_sqrt_ps_512(<16 x float> %a0, <16 x float> %passthru, i16 %mask)  #0 {
+;
+; CHECK-LABEL: @test_mask_sqrt_ps_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = call <16 x float> @llvm.sqrt.v16f32(<16 x float> [[A0:%.*]])
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i16 [[TMP2]] to <16 x i1>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i16 [[MASK:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP7:%.*]] = select <16 x i1> [[TMP6]], <16 x i32> [[TMP1]], <16 x i32> [[TMP3]]
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <16 x float> [[TMP4]] to <16 x i32>
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <16 x float> [[PASSTHRU:%.*]] to <16 x i32>
+; CHECK-NEXT:    [[TMP10:%.*]] = xor <16 x i32> [[TMP8]], [[TMP9]]
+; CHECK-NEXT:    [[TMP11:%.*]] = or <16 x i32> [[TMP10]], [[TMP1]]
+; CHECK-NEXT:    [[TMP12:%.*]] = or <16 x i32> [[TMP11]], [[TMP3]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP5]], <16 x i32> [[TMP12]], <16 x i32> [[TMP7]]
+; CHECK-NEXT:    [[TMP13:%.*]] = select <16 x i1> [[TMP6]], <16 x float> [[TMP4]], <16 x float> [[PASSTHRU]]
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x float> [[TMP13]]
+;
+  %res = call <16 x float> @llvm.x86.avx512.mask.sqrt.ps.512(<16 x float> %a0, <16 x float> %passthru, i16 %mask, i32 4)
+  ret <16 x float> %res
+}
+define <16 x float> @test_maskz_sqrt_ps_512(<16 x float> %a0, i16 %mask)  #0 {
+;
+; CHECK-LABEL: @test_maskz_sqrt_ps_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = call <16 x float> @llvm.sqrt.v16f32(<16 x float> [[A0:%.*]])
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast i16 [[TMP2]] to <16 x i1>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i16 [[MASK:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP6:%.*]] = select <16 x i1> [[TMP5]], <16 x i32> [[TMP1]], <16 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <16 x float> [[TMP3]] to <16 x i32>
+; CHECK-NEXT:    [[TMP8:%.*]] = xor <16 x i32> [[TMP7]], zeroinitializer
+; CHECK-NEXT:    [[TMP9:%.*]] = or <16 x i32> [[TMP8]], [[TMP1]]
+; CHECK-NEXT:    [[TMP10:%.*]] = or <16 x i32> [[TMP9]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP4]], <16 x i32> [[TMP10]], <16 x i32> [[TMP6]]
+; CHECK-NEXT:    [[TMP11:%.*]] = select <16 x i1> [[TMP5]], <16 x float> [[TMP3]], <16 x float> zeroinitializer
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x float> [[TMP11]]
+;
+  %res = call <16 x float> @llvm.x86.avx512.mask.sqrt.ps.512(<16 x float> %a0, <16 x float> zeroinitializer, i16 %mask, i32 4)
+  ret <16 x float> %res
+}
+define <16 x float> @test_sqrt_round_ps_512(<16 x float> %a0)  #0 {
+; CHECK-LABEL: @test_sqrt_round_ps_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP2]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP3:%.*]], label [[TMP4:%.*]], !prof [[PROF1]]
+; CHECK:       3:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       4:
+; CHECK-NEXT:    [[TMP5:%.*]] = call <16 x float> @llvm.x86.avx512.sqrt.ps.512(<16 x float> [[A0:%.*]], i32 11)
+; CHECK-NEXT:    store <16 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x float> [[TMP5]]
+;
+  %res = call <16 x float> @llvm.x86.avx512.mask.sqrt.ps.512(<16 x float> %a0, <16 x float> zeroinitializer, i16 -1, i32 11)
+  ret <16 x float> %res
+}
+define <16 x float> @test_mask_sqrt_round_ps_512(<16 x float> %a0, <16 x float> %passthru, i16 %mask)  #0 {
+;
+; CHECK-LABEL: @test_mask_sqrt_round_ps_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP4]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP5:%.*]], label [[TMP6:%.*]], !prof [[PROF1]]
+; CHECK:       5:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       6:
+; CHECK-NEXT:    [[TMP7:%.*]] = call <16 x float> @llvm.x86.avx512.sqrt.ps.512(<16 x float> [[A0:%.*]], i32 11)
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast i16 [[TMP2]] to <16 x i1>
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast i16 [[MASK:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP10:%.*]] = select <16 x i1> [[TMP9]], <16 x i32> zeroinitializer, <16 x i32> [[TMP3]]
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <16 x float> [[TMP7]] to <16 x i32>
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <16 x float> [[PASSTHRU:%.*]] to <16 x i32>
+; CHECK-NEXT:    [[TMP13:%.*]] = xor <16 x i32> [[TMP11]], [[TMP12]]
+; CHECK-NEXT:    [[TMP14:%.*]] = or <16 x i32> [[TMP13]], zeroinitializer
+; CHECK-NEXT:    [[TMP15:%.*]] = or <16 x i32> [[TMP14]], [[TMP3]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP8]], <16 x i32> [[TMP15]], <16 x i32> [[TMP10]]
+; CHECK-NEXT:    [[TMP16:%.*]] = select <16 x i1> [[TMP9]], <16 x float> [[TMP7]], <16 x float> [[PASSTHRU]]
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x float> [[TMP16]]
+;
+  %res = call <16 x float> @llvm.x86.avx512.mask.sqrt.ps.512(<16 x float> %a0, <16 x float> %passthru, i16 %mask, i32 11)
+  ret <16 x float> %res
+}
+define <16 x float> @test_maskz_sqrt_round_ps_512(<16 x float> %a0, i16 %mask)  #0 {
+;
+; CHECK-LABEL: @test_maskz_sqrt_round_ps_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP3]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP4:%.*]], label [[TMP5:%.*]], !prof [[PROF1]]
+; CHECK:       4:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       5:
+; CHECK-NEXT:    [[TMP6:%.*]] = call <16 x float> @llvm.x86.avx512.sqrt.ps.512(<16 x float> [[A0:%.*]], i32 11)
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast i16 [[TMP2]] to <16 x i1>
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast i16 [[MASK:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP9:%.*]] = select <16 x i1> [[TMP8]], <16 x i32> zeroinitializer, <16 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast <16 x float> [[TMP6]] to <16 x i32>
+; CHECK-NEXT:    [[TMP11:%.*]] = xor <16 x i32> [[TMP10]], zeroinitializer
+; CHECK-NEXT:    [[TMP12:%.*]] = or <16 x i32> [[TMP11]], zeroinitializer
+; CHECK-NEXT:    [[TMP13:%.*]] = or <16 x i32> [[TMP12]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP7]], <16 x i32> [[TMP13]], <16 x i32> [[TMP9]]
+; CHECK-NEXT:    [[TMP14:%.*]] = select <16 x i1> [[TMP8]], <16 x float> [[TMP6]], <16 x float> zeroinitializer
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x float> [[TMP14]]
+;
+  %res = call <16 x float> @llvm.x86.avx512.mask.sqrt.ps.512(<16 x float> %a0, <16 x float> zeroinitializer, i16 %mask, i32 11)
+  ret <16 x float> %res
+}
+declare <16 x float> @llvm.x86.avx512.mask.sqrt.ps.512(<16 x float>, <16 x float>, i16, i32) nounwind readnone
+
+declare <16 x i32> @llvm.x86.avx512.mask.prolv.d.512(<16 x i32>, <16 x i32>, <16 x i32>, i16)
+
+define <16 x i32>@test_int_x86_avx512_prolv_d_512_old(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x2)  #0 {
+; CHECK-LABEL: @test_int_x86_avx512_prolv_d_512_old(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp ne <16 x i32> [[TMP2]], zeroinitializer
+; CHECK-NEXT:    [[TMP4:%.*]] = sext <16 x i1> [[TMP3]] to <16 x i32>
+; CHECK-NEXT:    [[TMP5:%.*]] = call <16 x i32> @llvm.fshl.v16i32(<16 x i32> [[TMP1]], <16 x i32> [[TMP1]], <16 x i32> [[X1:%.*]])
+; CHECK-NEXT:    [[TMP6:%.*]] = or <16 x i32> [[TMP5]], [[TMP4]]
+; CHECK-NEXT:    [[TMP7:%.*]] = call <16 x i32> @llvm.fshl.v16i32(<16 x i32> [[X0:%.*]], <16 x i32> [[X0]], <16 x i32> [[X1]])
+; CHECK-NEXT:    store <16 x i32> [[TMP6]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i32> [[TMP7]]
+;
+  %res = call <16 x i32> @llvm.x86.avx512.mask.prolv.d.512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x2, i16 -1)
+  ret <16 x i32> %res
+}
+
+define <16 x i32>@test_int_x86_avx512_mask_prolv_d_512_old(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x2, i16 %x3)  #0 {
+;
+; CHECK-LABEL: @test_int_x86_avx512_mask_prolv_d_512_old(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp ne <16 x i32> [[TMP2]], zeroinitializer
+; CHECK-NEXT:    [[TMP6:%.*]] = sext <16 x i1> [[TMP5]] to <16 x i32>
+; CHECK-NEXT:    [[TMP7:%.*]] = call <16 x i32> @llvm.fshl.v16i32(<16 x i32> [[TMP1]], <16 x i32> [[TMP1]], <16 x i32> [[X1:%.*]])
+; CHECK-NEXT:    [[TMP8:%.*]] = or <16 x i32> [[TMP7]], [[TMP6]]
+; CHECK-NEXT:    [[TMP9:%.*]] = call <16 x i32> @llvm.fshl.v16i32(<16 x i32> [[X0:%.*]], <16 x i32> [[X0]], <16 x i32> [[X1]])
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i16 [[TMP3]] to <16 x i1>
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast i16 [[X3:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP12:%.*]] = select <16 x i1> [[TMP11]], <16 x i32> [[TMP8]], <16 x i32> [[TMP4]]
+; CHECK-NEXT:    [[TMP13:%.*]] = xor <16 x i32> [[TMP9]], [[X2:%.*]]
+; CHECK-NEXT:    [[TMP14:%.*]] = or <16 x i32> [[TMP13]], [[TMP8]]
+; CHECK-NEXT:    [[TMP15:%.*]] = or <16 x i32> [[TMP14]], [[TMP4]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP10]], <16 x i32> [[TMP15]], <16 x i32> [[TMP12]]
+; CHECK-NEXT:    [[TMP16:%.*]] = select <16 x i1> [[TMP11]], <16 x i32> [[TMP9]], <16 x i32> [[X2]]
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i32> [[TMP16]]
+;
+  %res = call <16 x i32> @llvm.x86.avx512.mask.prolv.d.512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x2, i16 %x3)
+  ret <16 x i32> %res
+}
+
+define <16 x i32>@test_int_x86_avx512_maskz_prolv_d_512_old(<16 x i32> %x0, <16 x i32> %x1, i16 %x3)  #0 {
+;
+; CHECK-LABEL: @test_int_x86_avx512_maskz_prolv_d_512_old(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = icmp ne <16 x i32> [[TMP2]], zeroinitializer
+; CHECK-NEXT:    [[TMP5:%.*]] = sext <16 x i1> [[TMP4]] to <16 x i32>
+; CHECK-NEXT:    [[TMP6:%.*]] = call <16 x i32> @llvm.fshl.v16i32(<16 x i32> [[TMP1]], <16 x i32> [[TMP1]], <16 x i32> [[X1:%.*]])
+; CHECK-NEXT:    [[TMP7:%.*]] = or <16 x i32> [[TMP6]], [[TMP5]]
+; CHECK-NEXT:    [[TMP8:%.*]] = call <16 x i32> @llvm.fshl.v16i32(<16 x i32> [[X0:%.*]], <16 x i32> [[X0]], <16 x i32> [[X1]])
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast i16 [[TMP3]] to <16 x i1>
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i16 [[X3:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP11:%.*]] = select <16 x i1> [[TMP10]], <16 x i32> [[TMP7]], <16 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP12:%.*]] = xor <16 x i32> [[TMP8]], zeroinitializer
+; CHECK-NEXT:    [[TMP13:%.*]] = or <16 x i32> [[TMP12]], [[TMP7]]
+; CHECK-NEXT:    [[TMP14:%.*]] = or <16 x i32> [[TMP13]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP9]], <16 x i32> [[TMP14]], <16 x i32> [[TMP11]]
+; CHECK-NEXT:    [[TMP15:%.*]] = select <16 x i1> [[TMP10]], <16 x i32> [[TMP8]], <16 x i32> zeroinitializer
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i32> [[TMP15]]
+;
+  %res = call <16 x i32> @llvm.x86.avx512.mask.prolv.d.512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> zeroinitializer, i16 %x3)
+  ret <16 x i32> %res
+}
+
+declare <8 x i64> @llvm.x86.avx512.mask.prolv.q.512(<8 x i64>, <8 x i64>, <8 x i64>, i8)
+
+define <8 x i64>@test_int_x86_avx512_prolv_q_512_old(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> %x2)  #0 {
+; CHECK-LABEL: @test_int_x86_avx512_prolv_q_512_old(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp ne <8 x i64> [[TMP2]], zeroinitializer
+; CHECK-NEXT:    [[TMP4:%.*]] = sext <8 x i1> [[TMP3]] to <8 x i64>
+; CHECK-NEXT:    [[TMP5:%.*]] = call <8 x i64> @llvm.fshl.v8i64(<8 x i64> [[TMP1]], <8 x i64> [[TMP1]], <8 x i64> [[X1:%.*]])
+; CHECK-NEXT:    [[TMP6:%.*]] = or <8 x i64> [[TMP5]], [[TMP4]]
+; CHECK-NEXT:    [[TMP7:%.*]] = call <8 x i64> @llvm.fshl.v8i64(<8 x i64> [[X0:%.*]], <8 x i64> [[X0]], <8 x i64> [[X1]])
+; CHECK-NEXT:    store <8 x i64> [[TMP6]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i64> [[TMP7]]
+;
+  %res = call <8 x i64> @llvm.x86.avx512.mask.prolv.q.512(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> %x2, i8 -1)
+  ret <8 x i64> %res
+}
+
+define <8 x i64>@test_int_x86_avx512_mask_prolv_q_512_old(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> %x2, i8 %x3)  #0 {
+;
+; CHECK-LABEL: @test_int_x86_avx512_mask_prolv_q_512_old(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp ne <8 x i64> [[TMP2]], zeroinitializer
+; CHECK-NEXT:    [[TMP6:%.*]] = sext <8 x i1> [[TMP5]] to <8 x i64>
+; CHECK-NEXT:    [[TMP7:%.*]] = call <8 x i64> @llvm.fshl.v8i64(<8 x i64> [[TMP1]], <8 x i64> [[TMP1]], <8 x i64> [[X1:%.*]])
+; CHECK-NEXT:    [[TMP8:%.*]] = or <8 x i64> [[TMP7]], [[TMP6]]
+; CHECK-NEXT:    [[TMP9:%.*]] = call <8 x i64> @llvm.fshl.v8i64(<8 x i64> [[X0:%.*]], <8 x i64> [[X0]], <8 x i64> [[X1]])
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i8 [[TMP3]] to <8 x i1>
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast i8 [[X3:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP12:%.*]] = select <8 x i1> [[TMP11]], <8 x i64> [[TMP8]], <8 x i64> [[TMP4]]
+; CHECK-NEXT:    [[TMP13:%.*]] = xor <8 x i64> [[TMP9]], [[X2:%.*]]
+; CHECK-NEXT:    [[TMP14:%.*]] = or <8 x i64> [[TMP13]], [[TMP8]]
+; CHECK-NEXT:    [[TMP15:%.*]] = or <8 x i64> [[TMP14]], [[TMP4]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP10]], <8 x i64> [[TMP15]], <8 x i64> [[TMP12]]
+; CHECK-NEXT:    [[TMP16:%.*]] = select <8 x i1> [[TMP11]], <8 x i64> [[TMP9]], <8 x i64> [[X2]]
+; CHECK-NEXT:    store <8 x i64> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i64> [[TMP16]]
+;
+  %res = call <8 x i64> @llvm.x86.avx512.mask.prolv.q.512(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> %x2, i8 %x3)
+  ret <8 x i64> %res
+}
+
+define <8 x i64>@test_int_x86_avx512_maskz_prolv_q_512_old(<8 x i64> %x0, <8 x i64> %x1, i8 %x3)  #0 {
+;
+; CHECK-LABEL: @test_int_x86_avx512_maskz_prolv_q_512_old(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = icmp ne <8 x i64> [[TMP2]], zeroinitializer
+; CHECK-NEXT:    [[TMP5:%.*]] = sext <8 x i1> [[TMP4]] to <8 x i64>
+; CHECK-NEXT:    [[TMP6:%.*]] = call <8 x i64> @llvm.fshl.v8i64(<8 x i64> [[TMP1]], <8 x i64> [[TMP1]], <8 x i64> [[X1:%.*]])
+; CHECK-NEXT:    [[TMP7:%.*]] = or <8 x i64> [[TMP6]], [[TMP5]]
+; CHECK-NEXT:    [[TMP8:%.*]] = call <8 x i64> @llvm.fshl.v8i64(<8 x i64> [[X0:%.*]], <8 x i64> [[X0]], <8 x i64> [[X1]])
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast i8 [[TMP3]] to <8 x i1>
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i8 [[X3:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP11:%.*]] = select <8 x i1> [[TMP10]], <8 x i64> [[TMP7]], <8 x i64> zeroinitializer
+; CHECK-NEXT:    [[TMP12:%.*]] = xor <8 x i64> [[TMP8]], zeroinitializer
+; CHECK-NEXT:    [[TMP13:%.*]] = or <8 x i64> [[TMP12]], [[TMP7]]
+; CHECK-NEXT:    [[TMP14:%.*]] = or <8 x i64> [[TMP13]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP9]], <8 x i64> [[TMP14]], <8 x i64> [[TMP11]]
+; CHECK-NEXT:    [[TMP15:%.*]] = select <8 x i1> [[TMP10]], <8 x i64> [[TMP8]], <8 x i64> zeroinitializer
+; CHECK-NEXT:    store <8 x i64> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i64> [[TMP15]]
+;
+  %res = call <8 x i64> @llvm.x86.avx512.mask.prolv.q.512(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> zeroinitializer, i8 %x3)
+  ret <8 x i64> %res
+}
+
+declare <16 x i32> @llvm.x86.avx512.mask.prorv.d.512(<16 x i32>, <16 x i32>, <16 x i32>, i16)
+
+define <16 x i32>@test_int_x86_avx512_prorv_d_512_old(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x2)  #0 {
+; CHECK-LABEL: @test_int_x86_avx512_prorv_d_512_old(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp ne <16 x i32> [[TMP2]], zeroinitializer
+; CHECK-NEXT:    [[TMP4:%.*]] = sext <16 x i1> [[TMP3]] to <16 x i32>
+; CHECK-NEXT:    [[TMP5:%.*]] = call <16 x i32> @llvm.fshr.v16i32(<16 x i32> [[TMP1]], <16 x i32> [[TMP1]], <16 x i32> [[X1:%.*]])
+; CHECK-NEXT:    [[TMP6:%.*]] = or <16 x i32> [[TMP5]], [[TMP4]]
+; CHECK-NEXT:    [[TMP7:%.*]] = call <16 x i32> @llvm.fshr.v16i32(<16 x i32> [[X0:%.*]], <16 x i32> [[X0]], <16 x i32> [[X1]])
+; CHECK-NEXT:    store <16 x i32> [[TMP6]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i32> [[TMP7]]
+;
+  %res = call <16 x i32> @llvm.x86.avx512.mask.prorv.d.512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x2, i16 -1)
+  ret <16 x i32> %res
+}
+
+define <16 x i32>@test_int_x86_avx512_mask_prorv_d_512_old(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x2, i16 %x3)  #0 {
+;
+; CHECK-LABEL: @test_int_x86_avx512_mask_prorv_d_512_old(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp ne <16 x i32> [[TMP2]], zeroinitializer
+; CHECK-NEXT:    [[TMP6:%.*]] = sext <16 x i1> [[TMP5]] to <16 x i32>
+; CHECK-NEXT:    [[TMP7:%.*]] = call <16 x i32> @llvm.fshr.v16i32(<16 x i32> [[TMP1]], <16 x i32> [[TMP1]], <16 x i32> [[X1:%.*]])
+; CHECK-NEXT:    [[TMP8:%.*]] = or <16 x i32> [[TMP7]], [[TMP6]]
+; CHECK-NEXT:    [[TMP9:%.*]] = call <16 x i32> @llvm.fshr.v16i32(<16 x i32> [[X0:%.*]], <16 x i32> [[X0]], <16 x i32> [[X1]])
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i16 [[TMP3]] to <16 x i1>
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast i16 [[X3:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP12:%.*]] = select <16 x i1> [[TMP11]], <16 x i32> [[TMP8]], <16 x i32> [[TMP4]]
+; CHECK-NEXT:    [[TMP13:%.*]] = xor <16 x i32> [[TMP9]], [[X2:%.*]]
+; CHECK-NEXT:    [[TMP14:%.*]] = or <16 x i32> [[TMP13]], [[TMP8]]
+; CHECK-NEXT:    [[TMP15:%.*]] = or <16 x i32> [[TMP14]], [[TMP4]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP10]], <16 x i32> [[TMP15]], <16 x i32> [[TMP12]]
+; CHECK-NEXT:    [[TMP16:%.*]] = select <16 x i1> [[TMP11]], <16 x i32> [[TMP9]], <16 x i32> [[X2]]
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i32> [[TMP16]]
+;
+  %res = call <16 x i32> @llvm.x86.avx512.mask.prorv.d.512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x2, i16 %x3)
+  ret <16 x i32> %res
+}
+
+define <16 x i32>@test_int_x86_avx512_maskz_prorv_d_512_old(<16 x i32> %x0, <16 x i32> %x1, i16 %x3)  #0 {
+;
+; CHECK-LABEL: @test_int_x86_avx512_maskz_prorv_d_512_old(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = icmp ne <16 x i32> [[TMP2]], zeroinitializer
+; CHECK-NEXT:    [[TMP5:%.*]] = sext <16 x i1> [[TMP4]] to <16 x i32>
+; CHECK-NEXT:    [[TMP6:%.*]] = call <16 x i32> @llvm.fshr.v16i32(<16 x i32> [[TMP1]], <16 x i32> [[TMP1]], <16 x i32> [[X1:%.*]])
+; CHECK-NEXT:    [[TMP7:%.*]] = or <16 x i32> [[TMP6]], [[TMP5]]
+; CHECK-NEXT:    [[TMP8:%.*]] = call <16 x i32> @llvm.fshr.v16i32(<16 x i32> [[X0:%.*]], <16 x i32> [[X0]], <16 x i32> [[X1]])
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast i16 [[TMP3]] to <16 x i1>
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i16 [[X3:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP11:%.*]] = select <16 x i1> [[TMP10]], <16 x i32> [[TMP7]], <16 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP12:%.*]] = xor <16 x i32> [[TMP8]], zeroinitializer
+; CHECK-NEXT:    [[TMP13:%.*]] = or <16 x i32> [[TMP12]], [[TMP7]]
+; CHECK-NEXT:    [[TMP14:%.*]] = or <16 x i32> [[TMP13]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP9]], <16 x i32> [[TMP14]], <16 x i32> [[TMP11]]
+; CHECK-NEXT:    [[TMP15:%.*]] = select <16 x i1> [[TMP10]], <16 x i32> [[TMP8]], <16 x i32> zeroinitializer
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i32> [[TMP15]]
+;
+  %res = call <16 x i32> @llvm.x86.avx512.mask.prorv.d.512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> zeroinitializer, i16 %x3)
+  ret <16 x i32> %res
+}
+
+declare <8 x i64> @llvm.x86.avx512.mask.prorv.q.512(<8 x i64>, <8 x i64>, <8 x i64>, i8)
+
+define <8 x i64>@test_int_x86_avx512_prorv_q_512_old(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> %x2)  #0 {
+; CHECK-LABEL: @test_int_x86_avx512_prorv_q_512_old(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp ne <8 x i64> [[TMP2]], zeroinitializer
+; CHECK-NEXT:    [[TMP4:%.*]] = sext <8 x i1> [[TMP3]] to <8 x i64>
+; CHECK-NEXT:    [[TMP5:%.*]] = call <8 x i64> @llvm.fshr.v8i64(<8 x i64> [[TMP1]], <8 x i64> [[TMP1]], <8 x i64> [[X1:%.*]])
+; CHECK-NEXT:    [[TMP6:%.*]] = or <8 x i64> [[TMP5]], [[TMP4]]
+; CHECK-NEXT:    [[TMP7:%.*]] = call <8 x i64> @llvm.fshr.v8i64(<8 x i64> [[X0:%.*]], <8 x i64> [[X0]], <8 x i64> [[X1]])
+; CHECK-NEXT:    store <8 x i64> [[TMP6]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i64> [[TMP7]]
+;
+  %res = call <8 x i64> @llvm.x86.avx512.mask.prorv.q.512(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> %x2, i8 -1)
+  ret <8 x i64> %res
+}
+
+define <8 x i64>@test_int_x86_avx512_mask_prorv_q_512_old(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> %x2, i8 %x3)  #0 {
+;
+; CHECK-LABEL: @test_int_x86_avx512_mask_prorv_q_512_old(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp ne <8 x i64> [[TMP2]], zeroinitializer
+; CHECK-NEXT:    [[TMP6:%.*]] = sext <8 x i1> [[TMP5]] to <8 x i64>
+; CHECK-NEXT:    [[TMP7:%.*]] = call <8 x i64> @llvm.fshr.v8i64(<8 x i64> [[TMP1]], <8 x i64> [[TMP1]], <8 x i64> [[X1:%.*]])
+; CHECK-NEXT:    [[TMP8:%.*]] = or <8 x i64> [[TMP7]], [[TMP6]]
+; CHECK-NEXT:    [[TMP9:%.*]] = call <8 x i64> @llvm.fshr.v8i64(<8 x i64> [[X0:%.*]], <8 x i64> [[X0]], <8 x i64> [[X1]])
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i8 [[TMP3]] to <8 x i1>
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast i8 [[X3:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP12:%.*]] = select <8 x i1> [[TMP11]], <8 x i64> [[TMP8]], <8 x i64> [[TMP4]]
+; CHECK-NEXT:    [[TMP13:%.*]] = xor <8 x i64> [[TMP9]], [[X2:%.*]]
+; CHECK-NEXT:    [[TMP14:%.*]] = or <8 x i64> [[TMP13]], [[TMP8]]
+; CHECK-NEXT:    [[TMP15:%.*]] = or <8 x i64> [[TMP14]], [[TMP4]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP10]], <8 x i64> [[TMP15]], <8 x i64> [[TMP12]]
+; CHECK-NEXT:    [[TMP16:%.*]] = select <8 x i1> [[TMP11]], <8 x i64> [[TMP9]], <8 x i64> [[X2]]
+; CHECK-NEXT:    store <8 x i64> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i64> [[TMP16]]
+;
+  %res = call <8 x i64> @llvm.x86.avx512.mask.prorv.q.512(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> %x2, i8 %x3)
+  ret <8 x i64> %res
+}
+
+define <8 x i64>@test_int_x86_avx512_maskz_prorv_q_512_old(<8 x i64> %x0, <8 x i64> %x1, i8 %x3)  #0 {
+;
+; CHECK-LABEL: @test_int_x86_avx512_maskz_prorv_q_512_old(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = icmp ne <8 x i64> [[TMP2]], zeroinitializer
+; CHECK-NEXT:    [[TMP5:%.*]] = sext <8 x i1> [[TMP4]] to <8 x i64>
+; CHECK-NEXT:    [[TMP6:%.*]] = call <8 x i64> @llvm.fshr.v8i64(<8 x i64> [[TMP1]], <8 x i64> [[TMP1]], <8 x i64> [[X1:%.*]])
+; CHECK-NEXT:    [[TMP7:%.*]] = or <8 x i64> [[TMP6]], [[TMP5]]
+; CHECK-NEXT:    [[TMP8:%.*]] = call <8 x i64> @llvm.fshr.v8i64(<8 x i64> [[X0:%.*]], <8 x i64> [[X0]], <8 x i64> [[X1]])
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast i8 [[TMP3]] to <8 x i1>
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i8 [[X3:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP11:%.*]] = select <8 x i1> [[TMP10]], <8 x i64> [[TMP7]], <8 x i64> zeroinitializer
+; CHECK-NEXT:    [[TMP12:%.*]] = xor <8 x i64> [[TMP8]], zeroinitializer
+; CHECK-NEXT:    [[TMP13:%.*]] = or <8 x i64> [[TMP12]], [[TMP7]]
+; CHECK-NEXT:    [[TMP14:%.*]] = or <8 x i64> [[TMP13]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP9]], <8 x i64> [[TMP14]], <8 x i64> [[TMP11]]
+; CHECK-NEXT:    [[TMP15:%.*]] = select <8 x i1> [[TMP10]], <8 x i64> [[TMP8]], <8 x i64> zeroinitializer
+; CHECK-NEXT:    store <8 x i64> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i64> [[TMP15]]
+;
+  %res = call <8 x i64> @llvm.x86.avx512.mask.prorv.q.512(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> zeroinitializer, i8 %x3)
+  ret <8 x i64> %res
+}
+
+declare <16 x i32> @llvm.x86.avx512.mask.prol.d.512(<16 x i32>, i32, <16 x i32>, i16)
+
+define { <16 x i32>, <16 x i32>, <16 x i32> } @test_int_x86_avx512_mask_prol_d_512(<16 x i32> %x0, i32 %x1, <16 x i32> %x2, i16 %x3)  #0 {
+;
+; CHECK-LABEL: @test_int_x86_avx512_mask_prol_d_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 136) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 72) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = call <16 x i32> @llvm.fshl.v16i32(<16 x i32> [[TMP1]], <16 x i32> [[TMP1]], <16 x i32> splat (i32 3))
+; CHECK-NEXT:    [[TMP5:%.*]] = or <16 x i32> [[TMP4]], zeroinitializer
+; CHECK-NEXT:    [[TMP6:%.*]] = call <16 x i32> @llvm.fshl.v16i32(<16 x i32> [[X0:%.*]], <16 x i32> [[X0]], <16 x i32> splat (i32 3))
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast i16 [[TMP2]] to <16 x i1>
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast i16 [[X3:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP9:%.*]] = select <16 x i1> [[TMP8]], <16 x i32> [[TMP5]], <16 x i32> [[TMP3]]
+; CHECK-NEXT:    [[TMP10:%.*]] = xor <16 x i32> [[TMP6]], [[X2:%.*]]
+; CHECK-NEXT:    [[TMP11:%.*]] = or <16 x i32> [[TMP10]], [[TMP5]]
+; CHECK-NEXT:    [[TMP12:%.*]] = or <16 x i32> [[TMP11]], [[TMP3]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP7]], <16 x i32> [[TMP12]], <16 x i32> [[TMP9]]
+; CHECK-NEXT:    [[TMP13:%.*]] = select <16 x i1> [[TMP8]], <16 x i32> [[TMP6]], <16 x i32> [[X2]]
+; CHECK-NEXT:    [[TMP14:%.*]] = call <16 x i32> @llvm.fshl.v16i32(<16 x i32> [[TMP1]], <16 x i32> [[TMP1]], <16 x i32> splat (i32 4))
+; CHECK-NEXT:    [[TMP15:%.*]] = or <16 x i32> [[TMP14]], zeroinitializer
+; CHECK-NEXT:    [[TMP16:%.*]] = call <16 x i32> @llvm.fshl.v16i32(<16 x i32> [[X0]], <16 x i32> [[X0]], <16 x i32> splat (i32 4))
+; CHECK-NEXT:    [[TMP17:%.*]] = bitcast i16 [[TMP2]] to <16 x i1>
+; CHECK-NEXT:    [[TMP18:%.*]] = bitcast i16 [[X3]] to <16 x i1>
+; CHECK-NEXT:    [[TMP19:%.*]] = select <16 x i1> [[TMP18]], <16 x i32> [[TMP15]], <16 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP20:%.*]] = xor <16 x i32> [[TMP16]], zeroinitializer
+; CHECK-NEXT:    [[TMP21:%.*]] = or <16 x i32> [[TMP20]], [[TMP15]]
+; CHECK-NEXT:    [[TMP22:%.*]] = or <16 x i32> [[TMP21]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT1:%.*]] = select <16 x i1> [[TMP17]], <16 x i32> [[TMP22]], <16 x i32> [[TMP19]]
+; CHECK-NEXT:    [[TMP23:%.*]] = select <16 x i1> [[TMP18]], <16 x i32> [[TMP16]], <16 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP24:%.*]] = call <16 x i32> @llvm.fshl.v16i32(<16 x i32> [[TMP1]], <16 x i32> [[TMP1]], <16 x i32> splat (i32 5))
+; CHECK-NEXT:    [[TMP25:%.*]] = or <16 x i32> [[TMP24]], zeroinitializer
+; CHECK-NEXT:    [[TMP26:%.*]] = call <16 x i32> @llvm.fshl.v16i32(<16 x i32> [[X0]], <16 x i32> [[X0]], <16 x i32> splat (i32 5))
+; CHECK-NEXT:    [[TMP27:%.*]] = insertvalue { <16 x i32>, <16 x i32>, <16 x i32> } { <16 x i32> splat (i32 -1), <16 x i32> splat (i32 -1), <16 x i32> splat (i32 -1) }, <16 x i32> [[_MSPROP_SELECT]], 0
+; CHECK-NEXT:    [[RES3:%.*]] = insertvalue { <16 x i32>, <16 x i32>, <16 x i32> } poison, <16 x i32> [[TMP13]], 0
+; CHECK-NEXT:    [[TMP28:%.*]] = insertvalue { <16 x i32>, <16 x i32>, <16 x i32> } [[TMP27]], <16 x i32> [[_MSPROP_SELECT1]], 1
+; CHECK-NEXT:    [[RES4:%.*]] = insertvalue { <16 x i32>, <16 x i32>, <16 x i32> } [[RES3]], <16 x i32> [[TMP23]], 1
+; CHECK-NEXT:    [[TMP29:%.*]] = insertvalue { <16 x i32>, <16 x i32>, <16 x i32> } [[TMP28]], <16 x i32> [[TMP25]], 2
+; CHECK-NEXT:    [[RES5:%.*]] = insertvalue { <16 x i32>, <16 x i32>, <16 x i32> } [[RES4]], <16 x i32> [[TMP26]], 2
+; CHECK-NEXT:    store { <16 x i32>, <16 x i32>, <16 x i32> } [[TMP29]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret { <16 x i32>, <16 x i32>, <16 x i32> } [[RES5]]
+;
+  %res0 = call <16 x i32> @llvm.x86.avx512.mask.prol.d.512(<16 x i32> %x0, i32 3, <16 x i32> %x2, i16 %x3)
+  %res1 = call <16 x i32> @llvm.x86.avx512.mask.prol.d.512(<16 x i32> %x0, i32 4, <16 x i32> zeroinitializer, i16 %x3)
+  %res2 = call <16 x i32> @llvm.x86.avx512.mask.prol.d.512(<16 x i32> %x0, i32 5, <16 x i32> %x2, i16 -1)
+  %res3 = insertvalue { <16 x i32>, <16 x i32>, <16 x i32> } poison, <16 x i32> %res0, 0
+  %res4 = insertvalue { <16 x i32>, <16 x i32>, <16 x i32> }  %res3, <16 x i32> %res1, 1
+  %res5 = insertvalue { <16 x i32>, <16 x i32>, <16 x i32> }  %res4, <16 x i32> %res2, 2
+  ret { <16 x i32>, <16 x i32>, <16 x i32> } %res5
+}
+
+declare <8 x i64> @llvm.x86.avx512.mask.prol.q.512(<8 x i64>, i32, <8 x i64>, i8)
+
+define { <8 x i64>, <8 x i64>, <8 x i64> } @test_int_x86_avx512_mask_prol_q_512(<8 x i64> %x0, i32 %x1, <8 x i64> %x2, i8 %x3)  #0 {
+;
+; CHECK-LABEL: @test_int_x86_avx512_mask_prol_q_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 136) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 72) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = call <8 x i64> @llvm.fshl.v8i64(<8 x i64> [[TMP1]], <8 x i64> [[TMP1]], <8 x i64> splat (i64 3))
+; CHECK-NEXT:    [[TMP5:%.*]] = or <8 x i64> [[TMP4]], zeroinitializer
+; CHECK-NEXT:    [[TMP6:%.*]] = call <8 x i64> @llvm.fshl.v8i64(<8 x i64> [[X0:%.*]], <8 x i64> [[X0]], <8 x i64> splat (i64 3))
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast i8 [[TMP2]] to <8 x i1>
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast i8 [[X3:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP9:%.*]] = select <8 x i1> [[TMP8]], <8 x i64> [[TMP5]], <8 x i64> [[TMP3]]
+; CHECK-NEXT:    [[TMP10:%.*]] = xor <8 x i64> [[TMP6]], [[X2:%.*]]
+; CHECK-NEXT:    [[TMP11:%.*]] = or <8 x i64> [[TMP10]], [[TMP5]]
+; CHECK-NEXT:    [[TMP12:%.*]] = or <8 x i64> [[TMP11]], [[TMP3]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP7]], <8 x i64> [[TMP12]], <8 x i64> [[TMP9]]
+; CHECK-NEXT:    [[TMP13:%.*]] = select <8 x i1> [[TMP8]], <8 x i64> [[TMP6]], <8 x i64> [[X2]]
+; CHECK-NEXT:    [[TMP14:%.*]] = call <8 x i64> @llvm.fshl.v8i64(<8 x i64> [[TMP1]], <8 x i64> [[TMP1]], <8 x i64> splat (i64 4))
+; CHECK-NEXT:    [[TMP15:%.*]] = or <8 x i64> [[TMP14]], zeroinitializer
+; CHECK-NEXT:    [[TMP16:%.*]] = call <8 x i64> @llvm.fshl.v8i64(<8 x i64> [[X0]], <8 x i64> [[X0]], <8 x i64> splat (i64 4))
+; CHECK-NEXT:    [[TMP17:%.*]] = bitcast i8 [[TMP2]] to <8 x i1>
+; CHECK-NEXT:    [[TMP18:%.*]] = bitcast i8 [[X3]] to <8 x i1>
+; CHECK-NEXT:    [[TMP19:%.*]] = select <8 x i1> [[TMP18]], <8 x i64> [[TMP15]], <8 x i64> zeroinitializer
+; CHECK-NEXT:    [[TMP20:%.*]] = xor <8 x i64> [[TMP16]], zeroinitializer
+; CHECK-NEXT:    [[TMP21:%.*]] = or <8 x i64> [[TMP20]], [[TMP15]]
+; CHECK-NEXT:    [[TMP22:%.*]] = or <8 x i64> [[TMP21]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT1:%.*]] = select <8 x i1> [[TMP17]], <8 x i64> [[TMP22]], <8 x i64> [[TMP19]]
+; CHECK-NEXT:    [[TMP23:%.*]] = select <8 x i1> [[TMP18]], <8 x i64> [[TMP16]], <8 x i64> zeroinitializer
+; CHECK-NEXT:    [[TMP24:%.*]] = call <8 x i64> @llvm.fshl.v8i64(<8 x i64> [[TMP1]], <8 x i64> [[TMP1]], <8 x i64> splat (i64 5))
+; CHECK-NEXT:    [[TMP25:%.*]] = or <8 x i64> [[TMP24]], zeroinitializer
+; CHECK-NEXT:    [[TMP26:%.*]] = call <8 x i64> @llvm.fshl.v8i64(<8 x i64> [[X0]], <8 x i64> [[X0]], <8 x i64> splat (i64 5))
+; CHECK-NEXT:    [[TMP27:%.*]] = insertvalue { <8 x i64>, <8 x i64>, <8 x i64> } { <8 x i64> splat (i64 -1), <8 x i64> splat (i64 -1), <8 x i64> splat (i64 -1) }, <8 x i64> [[_MSPROP_SELECT]], 0
+; CHECK-NEXT:    [[RES3:%.*]] = insertvalue { <8 x i64>, <8 x i64>, <8 x i64> } poison, <8 x i64> [[TMP13]], 0
+; CHECK-NEXT:    [[TMP28:%.*]] = insertvalue { <8 x i64>, <8 x i64>, <8 x i64> } [[TMP27]], <8 x i64> [[_MSPROP_SELECT1]], 1
+; CHECK-NEXT:    [[RES4:%.*]] = insertvalue { <8 x i64>, <8 x i64>, <8 x i64> } [[RES3]], <8 x i64> [[TMP23]], 1
+; CHECK-NEXT:    [[TMP29:%.*]] = insertvalue { <8 x i64>, <8 x i64>, <8 x i64> } [[TMP28]], <8 x i64> [[TMP25]], 2
+; CHECK-NEXT:    [[RES5:%.*]] = insertvalue { <8 x i64>, <8 x i64>, <8 x i64> } [[RES4]], <8 x i64> [[TMP26]], 2
+; CHECK-NEXT:    store { <8 x i64>, <8 x i64>, <8 x i64> } [[TMP29]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret { <8 x i64>, <8 x i64>, <8 x i64> } [[RES5]]
+;
+  %res0 = call <8 x i64> @llvm.x86.avx512.mask.prol.q.512(<8 x i64> %x0, i32 3, <8 x i64> %x2, i8 %x3)
+  %res1 = call <8 x i64> @llvm.x86.avx512.mask.prol.q.512(<8 x i64> %x0, i32 4, <8 x i64> zeroinitializer, i8 %x3)
+  %res2 = call <8 x i64> @llvm.x86.avx512.mask.prol.q.512(<8 x i64> %x0, i32 5, <8 x i64> %x2, i8 -1)
+  %res3 = insertvalue { <8 x i64>, <8 x i64>, <8 x i64> } poison, <8 x i64> %res0, 0
+  %res4 = insertvalue { <8 x i64>, <8 x i64>, <8 x i64> }  %res3, <8 x i64> %res1, 1
+  %res5 = insertvalue { <8 x i64>, <8 x i64>, <8 x i64> }  %res4, <8 x i64> %res2, 2
+  ret { <8 x i64>, <8 x i64>, <8 x i64> } %res5
+}
+
+declare <16 x i32> @llvm.x86.avx512.mask.pror.d.512(<16 x i32>, i32, <16 x i32>, i16)
+
+define { <16 x i32>, <16 x i32>, <16 x i32> } @test_int_x86_avx512_mask_pror_d_512(<16 x i32> %x0, i32 %x1, <16 x i32> %x2, i16 %x3)  #0 {
+;
+; CHECK-LABEL: @test_int_x86_avx512_mask_pror_d_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 136) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 72) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = call <16 x i32> @llvm.fshr.v16i32(<16 x i32> [[TMP1]], <16 x i32> [[TMP1]], <16 x i32> splat (i32 3))
+; CHECK-NEXT:    [[TMP5:%.*]] = or <16 x i32> [[TMP4]], zeroinitializer
+; CHECK-NEXT:    [[TMP6:%.*]] = call <16 x i32> @llvm.fshr.v16i32(<16 x i32> [[X0:%.*]], <16 x i32> [[X0]], <16 x i32> splat (i32 3))
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast i16 [[TMP2]] to <16 x i1>
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast i16 [[X3:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP9:%.*]] = select <16 x i1> [[TMP8]], <16 x i32> [[TMP5]], <16 x i32> [[TMP3]]
+; CHECK-NEXT:    [[TMP10:%.*]] = xor <16 x i32> [[TMP6]], [[X2:%.*]]
+; CHECK-NEXT:    [[TMP11:%.*]] = or <16 x i32> [[TMP10]], [[TMP5]]
+; CHECK-NEXT:    [[TMP12:%.*]] = or <16 x i32> [[TMP11]], [[TMP3]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP7]], <16 x i32> [[TMP12]], <16 x i32> [[TMP9]]
+; CHECK-NEXT:    [[TMP13:%.*]] = select <16 x i1> [[TMP8]], <16 x i32> [[TMP6]], <16 x i32> [[X2]]
+; CHECK-NEXT:    [[TMP14:%.*]] = call <16 x i32> @llvm.fshr.v16i32(<16 x i32> [[TMP1]], <16 x i32> [[TMP1]], <16 x i32> splat (i32 4))
+; CHECK-NEXT:    [[TMP15:%.*]] = or <16 x i32> [[TMP14]], zeroinitializer
+; CHECK-NEXT:    [[TMP16:%.*]] = call <16 x i32> @llvm.fshr.v16i32(<16 x i32> [[X0]], <16 x i32> [[X0]], <16 x i32> splat (i32 4))
+; CHECK-NEXT:    [[TMP17:%.*]] = bitcast i16 [[TMP2]] to <16 x i1>
+; CHECK-NEXT:    [[TMP18:%.*]] = bitcast i16 [[X3]] to <16 x i1>
+; CHECK-NEXT:    [[TMP19:%.*]] = select <16 x i1> [[TMP18]], <16 x i32> [[TMP15]], <16 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP20:%.*]] = xor <16 x i32> [[TMP16]], zeroinitializer
+; CHECK-NEXT:    [[TMP21:%.*]] = or <16 x i32> [[TMP20]], [[TMP15]]
+; CHECK-NEXT:    [[TMP22:%.*]] = or <16 x i32> [[TMP21]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT1:%.*]] = select <16 x i1> [[TMP17]], <16 x i32> [[TMP22]], <16 x i32> [[TMP19]]
+; CHECK-NEXT:    [[TMP23:%.*]] = select <16 x i1> [[TMP18]], <16 x i32> [[TMP16]], <16 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP24:%.*]] = call <16 x i32> @llvm.fshr.v16i32(<16 x i32> [[TMP1]], <16 x i32> [[TMP1]], <16 x i32> splat (i32 5))
+; CHECK-NEXT:    [[TMP25:%.*]] = or <16 x i32> [[TMP24]], zeroinitializer
+; CHECK-NEXT:    [[TMP26:%.*]] = call <16 x i32> @llvm.fshr.v16i32(<16 x i32> [[X0]], <16 x i32> [[X0]], <16 x i32> splat (i32 5))
+; CHECK-NEXT:    [[TMP27:%.*]] = insertvalue { <16 x i32>, <16 x i32>, <16 x i32> } { <16 x i32> splat (i32 -1), <16 x i32> splat (i32 -1), <16 x i32> splat (i32 -1) }, <16 x i32> [[_MSPROP_SELECT]], 0
+; CHECK-NEXT:    [[RES3:%.*]] = insertvalue { <16 x i32>, <16 x i32>, <16 x i32> } poison, <16 x i32> [[TMP13]], 0
+; CHECK-NEXT:    [[TMP28:%.*]] = insertvalue { <16 x i32>, <16 x i32>, <16 x i32> } [[TMP27]], <16 x i32> [[_MSPROP_SELECT1]], 1
+; CHECK-NEXT:    [[RES4:%.*]] = insertvalue { <16 x i32>, <16 x i32>, <16 x i32> } [[RES3]], <16 x i32> [[TMP23]], 1
+; CHECK-NEXT:    [[TMP29:%.*]] = insertvalue { <16 x i32>, <16 x i32>, <16 x i32> } [[TMP28]], <16 x i32> [[TMP25]], 2
+; CHECK-NEXT:    [[RES5:%.*]] = insertvalue { <16 x i32>, <16 x i32>, <16 x i32> } [[RES4]], <16 x i32> [[TMP26]], 2
+; CHECK-NEXT:    store { <16 x i32>, <16 x i32>, <16 x i32> } [[TMP29]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret { <16 x i32>, <16 x i32>, <16 x i32> } [[RES5]]
+;
+  %res0 = call <16 x i32> @llvm.x86.avx512.mask.pror.d.512(<16 x i32> %x0, i32 3, <16 x i32> %x2, i16 %x3)
+  %res1 = call <16 x i32> @llvm.x86.avx512.mask.pror.d.512(<16 x i32> %x0, i32 4, <16 x i32> zeroinitializer, i16 %x3)
+  %res2 = call <16 x i32> @llvm.x86.avx512.mask.pror.d.512(<16 x i32> %x0, i32 5, <16 x i32> %x2, i16 -1)
+  %res3 = insertvalue { <16 x i32>, <16 x i32>, <16 x i32> } poison, <16 x i32> %res0, 0
+  %res4 = insertvalue { <16 x i32>, <16 x i32>, <16 x i32> }  %res3, <16 x i32> %res1, 1
+  %res5 = insertvalue { <16 x i32>, <16 x i32>, <16 x i32> }  %res4, <16 x i32> %res2, 2
+  ret { <16 x i32>, <16 x i32>, <16 x i32> } %res5
+}
+
+declare <8 x i64> @llvm.x86.avx512.mask.pror.q.512(<8 x i64>, i32, <8 x i64>, i8)
+
+define { <8 x i64>, <8 x i64>, <8 x i64> } @test_int_x86_avx512_mask_pror_q_512(<8 x i64> %x0, i32 %x1, <8 x i64> %x2, i8 %x3)  #0 {
+;
+; CHECK-LABEL: @test_int_x86_avx512_mask_pror_q_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 136) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 72) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = call <8 x i64> @llvm.fshr.v8i64(<8 x i64> [[TMP1]], <8 x i64> [[TMP1]], <8 x i64> splat (i64 3))
+; CHECK-NEXT:    [[TMP5:%.*]] = or <8 x i64> [[TMP4]], zeroinitializer
+; CHECK-NEXT:    [[TMP6:%.*]] = call <8 x i64> @llvm.fshr.v8i64(<8 x i64> [[X0:%.*]], <8 x i64> [[X0]], <8 x i64> splat (i64 3))
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast i8 [[TMP2]] to <8 x i1>
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast i8 [[X3:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP9:%.*]] = select <8 x i1> [[TMP8]], <8 x i64> [[TMP5]], <8 x i64> [[TMP3]]
+; CHECK-NEXT:    [[TMP10:%.*]] = xor <8 x i64> [[TMP6]], [[X2:%.*]]
+; CHECK-NEXT:    [[TMP11:%.*]] = or <8 x i64> [[TMP10]], [[TMP5]]
+; CHECK-NEXT:    [[TMP12:%.*]] = or <8 x i64> [[TMP11]], [[TMP3]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP7]], <8 x i64> [[TMP12]], <8 x i64> [[TMP9]]
+; CHECK-NEXT:    [[TMP13:%.*]] = select <8 x i1> [[TMP8]], <8 x i64> [[TMP6]], <8 x i64> [[X2]]
+; CHECK-NEXT:    [[TMP14:%.*]] = call <8 x i64> @llvm.fshr.v8i64(<8 x i64> [[TMP1]], <8 x i64> [[TMP1]], <8 x i64> splat (i64 4))
+; CHECK-NEXT:    [[TMP15:%.*]] = or <8 x i64> [[TMP14]], zeroinitializer
+; CHECK-NEXT:    [[TMP16:%.*]] = call <8 x i64> @llvm.fshr.v8i64(<8 x i64> [[X0]], <8 x i64> [[X0]], <8 x i64> splat (i64 4))
+; CHECK-NEXT:    [[TMP17:%.*]] = bitcast i8 [[TMP2]] to <8 x i1>
+; CHECK-NEXT:    [[TMP18:%.*]] = bitcast i8 [[X3]] to <8 x i1>
+; CHECK-NEXT:    [[TMP19:%.*]] = select <8 x i1> [[TMP18]], <8 x i64> [[TMP15]], <8 x i64> zeroinitializer
+; CHECK-NEXT:    [[TMP20:%.*]] = xor <8 x i64> [[TMP16]], zeroinitializer
+; CHECK-NEXT:    [[TMP21:%.*]] = or <8 x i64> [[TMP20]], [[TMP15]]
+; CHECK-NEXT:    [[TMP22:%.*]] = or <8 x i64> [[TMP21]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT1:%.*]] = select <8 x i1> [[TMP17]], <8 x i64> [[TMP22]], <8 x i64> [[TMP19]]
+; CHECK-NEXT:    [[TMP23:%.*]] = select <8 x i1> [[TMP18]], <8 x i64> [[TMP16]], <8 x i64> zeroinitializer
+; CHECK-NEXT:    [[TMP24:%.*]] = call <8 x i64> @llvm.fshr.v8i64(<8 x i64> [[TMP1]], <8 x i64> [[TMP1]], <8 x i64> splat (i64 5))
+; CHECK-NEXT:    [[TMP25:%.*]] = or <8 x i64> [[TMP24]], zeroinitializer
+; CHECK-NEXT:    [[TMP26:%.*]] = call <8 x i64> @llvm.fshr.v8i64(<8 x i64> [[X0]], <8 x i64> [[X0]], <8 x i64> splat (i64 5))
+; CHECK-NEXT:    [[TMP27:%.*]] = insertvalue { <8 x i64>, <8 x i64>, <8 x i64> } { <8 x i64> splat (i64 -1), <8 x i64> splat (i64 -1), <8 x i64> splat (i64 -1) }, <8 x i64> [[_MSPROP_SELECT]], 0
+; CHECK-NEXT:    [[RES3:%.*]] = insertvalue { <8 x i64>, <8 x i64>, <8 x i64> } poison, <8 x i64> [[TMP13]], 0
+; CHECK-NEXT:    [[TMP28:%.*]] = insertvalue { <8 x i64>, <8 x i64>, <8 x i64> } [[TMP27]], <8 x i64> [[_MSPROP_SELECT1]], 1
+; CHECK-NEXT:    [[RES4:%.*]] = insertvalue { <8 x i64>, <8 x i64>, <8 x i64> } [[RES3]], <8 x i64> [[TMP23]], 1
+; CHECK-NEXT:    [[TMP29:%.*]] = insertvalue { <8 x i64>, <8 x i64>, <8 x i64> } [[TMP28]], <8 x i64> [[TMP25]], 2
+; CHECK-NEXT:    [[RES5:%.*]] = insertvalue { <8 x i64>, <8 x i64>, <8 x i64> } [[RES4]], <8 x i64> [[TMP26]], 2
+; CHECK-NEXT:    store { <8 x i64>, <8 x i64>, <8 x i64> } [[TMP29]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret { <8 x i64>, <8 x i64>, <8 x i64> } [[RES5]]
+;
+  %res0 = call <8 x i64> @llvm.x86.avx512.mask.pror.q.512(<8 x i64> %x0, i32 3, <8 x i64> %x2, i8 %x3)
+  %res1 = call <8 x i64> @llvm.x86.avx512.mask.pror.q.512(<8 x i64> %x0, i32 4, <8 x i64> zeroinitializer, i8 %x3)
+  %res2 = call <8 x i64> @llvm.x86.avx512.mask.pror.q.512(<8 x i64> %x0, i32 5, <8 x i64> %x2, i8 -1)
+  %res3 = insertvalue { <8 x i64>, <8 x i64>, <8 x i64> } poison, <8 x i64> %res0, 0
+  %res4 = insertvalue { <8 x i64>, <8 x i64>, <8 x i64> }  %res3, <8 x i64> %res1, 1
+  %res5 = insertvalue { <8 x i64>, <8 x i64>, <8 x i64> }  %res4, <8 x i64> %res2, 2
+  ret { <8 x i64>, <8 x i64>, <8 x i64> } %res5
+}
+
+declare <2 x double> @llvm.x86.avx512.mask.vfmadd.sd(<2 x double>, <2 x double>, <2 x double>, i8, i32)
+
+define <2 x double>@test_int_x86_avx512_mask_vfmadd_sd(<2 x double> %x0, <2 x double> %x1, <2 x double> %x2, i8 %x3,i32 %x4 ) #0 {
+;
+; CHECK-LABEL: @test_int_x86_avx512_mask_vfmadd_sd(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 48) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = extractelement <2 x i64> [[TMP1]], i64 0
+; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <2 x double> [[X0:%.*]], i64 0
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = extractelement <2 x i64> [[TMP2]], i64 0
+; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <2 x double> [[X1:%.*]], i64 0
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = extractelement <2 x i64> [[TMP3]], i64 0
+; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <2 x double> [[X2:%.*]], i64 0
+; CHECK-NEXT:    [[_MSPROP3:%.*]] = or i64 [[_MSPROP]], [[_MSPROP1]]
+; CHECK-NEXT:    [[_MSPROP4:%.*]] = or i64 [[_MSPROP3]], [[_MSPROP2]]
+; CHECK-NEXT:    [[TMP8:%.*]] = call double @llvm.fma.f64(double [[TMP5]], double [[TMP6]], double [[TMP7]])
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast i8 [[TMP4]] to <8 x i1>
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i8 [[X3:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[_MSPROP5:%.*]] = extractelement <8 x i1> [[TMP9]], i64 0
+; CHECK-NEXT:    [[TMP11:%.*]] = extractelement <8 x i1> [[TMP10]], i64 0
+; CHECK-NEXT:    [[TMP12:%.*]] = select i1 [[TMP11]], i64 [[_MSPROP4]], i64 [[_MSPROP]]
+; CHECK-NEXT:    [[TMP13:%.*]] = bitcast double [[TMP8]] to i64
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast double [[TMP5]] to i64
+; CHECK-NEXT:    [[TMP15:%.*]] = xor i64 [[TMP13]], [[TMP14]]
+; CHECK-NEXT:    [[TMP16:%.*]] = or i64 [[TMP15]], [[_MSPROP4]]
+; CHECK-NEXT:    [[TMP17:%.*]] = or i64 [[TMP16]], [[_MSPROP]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select i1 [[_MSPROP5]], i64 [[TMP17]], i64 [[TMP12]]
+; CHECK-NEXT:    [[TMP18:%.*]] = select i1 [[TMP11]], double [[TMP8]], double [[TMP5]]
+; CHECK-NEXT:    [[_MSPROP6:%.*]] = insertelement <2 x i64> [[TMP1]], i64 [[_MSPROP_SELECT]], i64 0
+; CHECK-NEXT:    [[TMP19:%.*]] = insertelement <2 x double> [[X0]], double [[TMP18]], i64 0
+; CHECK-NEXT:    [[_MSPROP7:%.*]] = extractelement <2 x i64> [[TMP1]], i64 0
+; CHECK-NEXT:    [[TMP20:%.*]] = extractelement <2 x double> [[X0]], i64 0
+; CHECK-NEXT:    [[_MSPROP8:%.*]] = extractelement <2 x i64> [[TMP2]], i64 0
+; CHECK-NEXT:    [[TMP21:%.*]] = extractelement <2 x double> [[X1]], i64 0
+; CHECK-NEXT:    [[_MSPROP9:%.*]] = extractelement <2 x i64> [[TMP3]], i64 0
+; CHECK-NEXT:    [[TMP22:%.*]] = extractelement <2 x double> [[X2]], i64 0
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[_MSPROP7]], 0
+; CHECK-NEXT:    [[_MSCMP19:%.*]] = icmp ne i64 [[_MSPROP8]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP19]]
+; CHECK-NEXT:    [[_MSCMP20:%.*]] = icmp ne i64 [[_MSPROP9]], 0
+; CHECK-NEXT:    [[_MSOR21:%.*]] = or i1 [[_MSOR]], [[_MSCMP20]]
+; CHECK-NEXT:    br i1 [[_MSOR21]], label [[TMP23:%.*]], label [[TMP24:%.*]], !prof [[PROF1]]
+; CHECK:       23:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       24:
+; CHECK-NEXT:    [[TMP25:%.*]] = call double @llvm.x86.avx512.vfmadd.f64(double [[TMP20]], double [[TMP21]], double [[TMP22]], i32 11)
+; CHECK-NEXT:    [[_MSPROP10:%.*]] = insertelement <2 x i64> [[TMP1]], i64 0, i64 0
+; CHECK-NEXT:    [[TMP26:%.*]] = insertelement <2 x double> [[X0]], double [[TMP25]], i64 0
+; CHECK-NEXT:    [[_MSPROP11:%.*]] = extractelement <2 x i64> [[TMP1]], i64 0
+; CHECK-NEXT:    [[TMP27:%.*]] = extractelement <2 x double> [[X0]], i64 0
+; CHECK-NEXT:    [[_MSPROP12:%.*]] = extractelement <2 x i64> [[TMP2]], i64 0
+; CHECK-NEXT:    [[TMP28:%.*]] = extractelement <2 x double> [[X1]], i64 0
+; CHECK-NEXT:    [[_MSPROP13:%.*]] = extractelement <2 x i64> [[TMP3]], i64 0
+; CHECK-NEXT:    [[TMP29:%.*]] = extractelement <2 x double> [[X2]], i64 0
+; CHECK-NEXT:    [[_MSCMP22:%.*]] = icmp ne i64 [[_MSPROP11]], 0
+; CHECK-NEXT:    [[_MSCMP23:%.*]] = icmp ne i64 [[_MSPROP12]], 0
+; CHECK-NEXT:    [[_MSOR24:%.*]] = or i1 [[_MSCMP22]], [[_MSCMP23]]
+; CHECK-NEXT:    [[_MSCMP25:%.*]] = icmp ne i64 [[_MSPROP13]], 0
+; CHECK-NEXT:    [[_MSOR26:%.*]] = or i1 [[_MSOR24]], [[_MSCMP25]]
+; CHECK-NEXT:    br i1 [[_MSOR26]], label [[TMP30:%.*]], label [[TMP31:%.*]], !prof [[PROF1]]
+; CHECK:       30:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       31:
+; CHECK-NEXT:    [[TMP32:%.*]] = call double @llvm.x86.avx512.vfmadd.f64(double [[TMP27]], double [[TMP28]], double [[TMP29]], i32 10)
+; CHECK-NEXT:    [[TMP33:%.*]] = bitcast i8 [[TMP4]] to <8 x i1>
+; CHECK-NEXT:    [[TMP34:%.*]] = bitcast i8 [[X3]] to <8 x i1>
+; CHECK-NEXT:    [[_MSPROP14:%.*]] = extractelement <8 x i1> [[TMP33]], i64 0
+; CHECK-NEXT:    [[TMP35:%.*]] = extractelement <8 x i1> [[TMP34]], i64 0
+; CHECK-NEXT:    [[TMP36:%.*]] = select i1 [[TMP35]], i64 0, i64 [[_MSPROP11]]
+; CHECK-NEXT:    [[TMP37:%.*]] = bitcast double [[TMP32]] to i64
+; CHECK-NEXT:    [[TMP38:%.*]] = bitcast double [[TMP27]] to i64
+; CHECK-NEXT:    [[TMP39:%.*]] = xor i64 [[TMP37]], [[TMP38]]
+; CHECK-NEXT:    [[TMP40:%.*]] = or i64 [[TMP39]], 0
+; CHECK-NEXT:    [[TMP41:%.*]] = or i64 [[TMP40]], [[_MSPROP11]]
+; CHECK-NEXT:    [[_MSPROP_SELECT15:%.*]] = select i1 [[_MSPROP14]], i64 [[TMP41]], i64 [[TMP36]]
+; CHECK-NEXT:    [[TMP42:%.*]] = select i1 [[TMP35]], double [[TMP32]], double [[TMP27]]
+; CHECK-NEXT:    [[_MSPROP16:%.*]] = insertelement <2 x i64> [[TMP1]], i64 [[_MSPROP_SELECT15]], i64 0
+; CHECK-NEXT:    [[TMP43:%.*]] = insertelement <2 x double> [[X0]], double [[TMP42]], i64 0
+; CHECK-NEXT:    [[_MSPROP17:%.*]] = or <2 x i64> [[_MSPROP6]], [[_MSPROP10]]
+; CHECK-NEXT:    [[RES3:%.*]] = fadd <2 x double> [[TMP19]], [[TMP26]]
+; CHECK-NEXT:    [[_MSPROP18:%.*]] = or <2 x i64> [[_MSPROP16]], [[_MSPROP17]]
+; CHECK-NEXT:    [[RES4:%.*]] = fadd <2 x double> [[TMP43]], [[RES3]]
+; CHECK-NEXT:    store <2 x i64> [[_MSPROP18]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x double> [[RES4]]
+;
+  %res = call <2 x double> @llvm.x86.avx512.mask.vfmadd.sd(<2 x double> %x0, <2 x double> %x1, <2 x double> %x2, i8 %x3, i32 4)
+  %res1 = call <2 x double> @llvm.x86.avx512.mask.vfmadd.sd(<2 x double> %x0, <2 x double> %x1, <2 x double> %x2, i8 -1, i32 11)
+  %res2 = call <2 x double> @llvm.x86.avx512.mask.vfmadd.sd(<2 x double> %x0, <2 x double> %x1, <2 x double> %x2, i8 %x3, i32 10)
+  %res3 = fadd <2 x double> %res, %res1
+  %res4 = fadd <2 x double> %res2, %res3
+  ret <2 x double> %res4
+}
+
+declare <4 x float> @llvm.x86.avx512.mask.vfmadd.ss(<4 x float>, <4 x float>, <4 x float>, i8, i32)
+
+define <4 x float>@test_int_x86_avx512_mask_vfmadd_ss(<4 x float> %x0, <4 x float> %x1, <4 x float> %x2, i8 %x3,i32 %x4 ) #0 {
+;
+; CHECK-LABEL: @test_int_x86_avx512_mask_vfmadd_ss(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 48) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = extractelement <4 x i32> [[TMP1]], i64 0
+; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <4 x float> [[X0:%.*]], i64 0
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = extractelement <4 x i32> [[TMP2]], i64 0
+; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <4 x float> [[X1:%.*]], i64 0
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = extractelement <4 x i32> [[TMP3]], i64 0
+; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <4 x float> [[X2:%.*]], i64 0
+; CHECK-NEXT:    [[_MSPROP3:%.*]] = or i32 [[_MSPROP]], [[_MSPROP1]]
+; CHECK-NEXT:    [[_MSPROP4:%.*]] = or i32 [[_MSPROP3]], [[_MSPROP2]]
+; CHECK-NEXT:    [[TMP8:%.*]] = call float @llvm.fma.f32(float [[TMP5]], float [[TMP6]], float [[TMP7]])
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast i8 [[TMP4]] to <8 x i1>
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i8 [[X3:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[_MSPROP5:%.*]] = extractelement <8 x i1> [[TMP9]], i64 0
+; CHECK-NEXT:    [[TMP11:%.*]] = extractelement <8 x i1> [[TMP10]], i64 0
+; CHECK-NEXT:    [[TMP12:%.*]] = select i1 [[TMP11]], i32 [[_MSPROP4]], i32 [[_MSPROP]]
+; CHECK-NEXT:    [[TMP13:%.*]] = bitcast float [[TMP8]] to i32
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast float [[TMP5]] to i32
+; CHECK-NEXT:    [[TMP15:%.*]] = xor i32 [[TMP13]], [[TMP14]]
+; CHECK-NEXT:    [[TMP16:%.*]] = or i32 [[TMP15]], [[_MSPROP4]]
+; CHECK-NEXT:    [[TMP17:%.*]] = or i32 [[TMP16]], [[_MSPROP]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select i1 [[_MSPROP5]], i32 [[TMP17]], i32 [[TMP12]]
+; CHECK-NEXT:    [[TMP18:%.*]] = select i1 [[TMP11]], float [[TMP8]], float [[TMP5]]
+; CHECK-NEXT:    [[_MSPROP6:%.*]] = insertelement <4 x i32> [[TMP1]], i32 [[_MSPROP_SELECT]], i64 0
+; CHECK-NEXT:    [[TMP19:%.*]] = insertelement <4 x float> [[X0]], float [[TMP18]], i64 0
+; CHECK-NEXT:    [[_MSPROP7:%.*]] = extractelement <4 x i32> [[TMP1]], i64 0
+; CHECK-NEXT:    [[TMP20:%.*]] = extractelement <4 x float> [[X0]], i64 0
+; CHECK-NEXT:    [[_MSPROP8:%.*]] = extractelement <4 x i32> [[TMP2]], i64 0
+; CHECK-NEXT:    [[TMP21:%.*]] = extractelement <4 x float> [[X1]], i64 0
+; CHECK-NEXT:    [[_MSPROP9:%.*]] = extractelement <4 x i32> [[TMP3]], i64 0
+; CHECK-NEXT:    [[TMP22:%.*]] = extractelement <4 x float> [[X2]], i64 0
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i32 [[_MSPROP7]], 0
+; CHECK-NEXT:    [[_MSCMP19:%.*]] = icmp ne i32 [[_MSPROP8]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP19]]
+; CHECK-NEXT:    [[_MSCMP20:%.*]] = icmp ne i32 [[_MSPROP9]], 0
+; CHECK-NEXT:    [[_MSOR21:%.*]] = or i1 [[_MSOR]], [[_MSCMP20]]
+; CHECK-NEXT:    br i1 [[_MSOR21]], label [[TMP23:%.*]], label [[TMP24:%.*]], !prof [[PROF1]]
+; CHECK:       23:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       24:
+; CHECK-NEXT:    [[TMP25:%.*]] = call float @llvm.x86.avx512.vfmadd.f32(float [[TMP20]], float [[TMP21]], float [[TMP22]], i32 11)
+; CHECK-NEXT:    [[_MSPROP10:%.*]] = insertelement <4 x i32> [[TMP1]], i32 0, i64 0
+; CHECK-NEXT:    [[TMP26:%.*]] = insertelement <4 x float> [[X0]], float [[TMP25]], i64 0
+; CHECK-NEXT:    [[_MSPROP11:%.*]] = extractelement <4 x i32> [[TMP1]], i64 0
+; CHECK-NEXT:    [[TMP27:%.*]] = extractelement <4 x float> [[X0]], i64 0
+; CHECK-NEXT:    [[_MSPROP12:%.*]] = extractelement <4 x i32> [[TMP2]], i64 0
+; CHECK-NEXT:    [[TMP28:%.*]] = extractelement <4 x float> [[X1]], i64 0
+; CHECK-NEXT:    [[_MSPROP13:%.*]] = extractelement <4 x i32> [[TMP3]], i64 0
+; CHECK-NEXT:    [[TMP29:%.*]] = extractelement <4 x float> [[X2]], i64 0
+; CHECK-NEXT:    [[_MSCMP22:%.*]] = icmp ne i32 [[_MSPROP11]], 0
+; CHECK-NEXT:    [[_MSCMP23:%.*]] = icmp ne i32 [[_MSPROP12]], 0
+; CHECK-NEXT:    [[_MSOR24:%.*]] = or i1 [[_MSCMP22]], [[_MSCMP23]]
+; CHECK-NEXT:    [[_MSCMP25:%.*]] = icmp ne i32 [[_MSPROP13]], 0
+; CHECK-NEXT:    [[_MSOR26:%.*]] = or i1 [[_MSOR24]], [[_MSCMP25]]
+; CHECK-NEXT:    br i1 [[_MSOR26]], label [[TMP30:%.*]], label [[TMP31:%.*]], !prof [[PROF1]]
+; CHECK:       30:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       31:
+; CHECK-NEXT:    [[TMP32:%.*]] = call float @llvm.x86.avx512.vfmadd.f32(float [[TMP27]], float [[TMP28]], float [[TMP29]], i32 10)
+; CHECK-NEXT:    [[TMP33:%.*]] = bitcast i8 [[TMP4]] to <8 x i1>
+; CHECK-NEXT:    [[TMP34:%.*]] = bitcast i8 [[X3]] to <8 x i1>
+; CHECK-NEXT:    [[_MSPROP14:%.*]] = extractelement <8 x i1> [[TMP33]], i64 0
+; CHECK-NEXT:    [[TMP35:%.*]] = extractelement <8 x i1> [[TMP34]], i64 0
+; CHECK-NEXT:    [[TMP36:%.*]] = select i1 [[TMP35]], i32 0, i32 [[_MSPROP11]]
+; CHECK-NEXT:    [[TMP37:%.*]] = bitcast float [[TMP32]] to i32
+; CHECK-NEXT:    [[TMP38:%.*]] = bitcast float [[TMP27]] to i32
+; CHECK-NEXT:    [[TMP39:%.*]] = xor i32 [[TMP37]], [[TMP38]]
+; CHECK-NEXT:    [[TMP40:%.*]] = or i32 [[TMP39]], 0
+; CHECK-NEXT:    [[TMP41:%.*]] = or i32 [[TMP40]], [[_MSPROP11]]
+; CHECK-NEXT:    [[_MSPROP_SELECT15:%.*]] = select i1 [[_MSPROP14]], i32 [[TMP41]], i32 [[TMP36]]
+; CHECK-NEXT:    [[TMP42:%.*]] = select i1 [[TMP35]], float [[TMP32]], float [[TMP27]]
+; CHECK-NEXT:    [[_MSPROP16:%.*]] = insertelement <4 x i32> [[TMP1]], i32 [[_MSPROP_SELECT15]], i64 0
+; CHECK-NEXT:    [[TMP43:%.*]] = insertelement <4 x float> [[X0]], float [[TMP42]], i64 0
+; CHECK-NEXT:    [[_MSPROP17:%.*]] = or <4 x i32> [[_MSPROP6]], [[_MSPROP10]]
+; CHECK-NEXT:    [[RES3:%.*]] = fadd <4 x float> [[TMP19]], [[TMP26]]
+; CHECK-NEXT:    [[_MSPROP18:%.*]] = or <4 x i32> [[_MSPROP16]], [[_MSPROP17]]
+; CHECK-NEXT:    [[RES4:%.*]] = fadd <4 x float> [[TMP43]], [[RES3]]
+; CHECK-NEXT:    store <4 x i32> [[_MSPROP18]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x float> [[RES4]]
+;
+  %res = call <4 x float> @llvm.x86.avx512.mask.vfmadd.ss(<4 x float> %x0, <4 x float> %x1, <4 x float> %x2, i8 %x3, i32 4)
+  %res1 = call <4 x float> @llvm.x86.avx512.mask.vfmadd.ss(<4 x float> %x0, <4 x float> %x1, <4 x float> %x2, i8 -1, i32 11)
+  %res2 = call <4 x float> @llvm.x86.avx512.mask.vfmadd.ss(<4 x float> %x0, <4 x float> %x1, <4 x float> %x2, i8 %x3, i32 10)
+  %res3 = fadd <4 x float> %res, %res1
+  %res4 = fadd <4 x float> %res2, %res3
+  ret <4 x float> %res4
+}
+
+declare <2 x double> @llvm.x86.avx512.maskz.vfmadd.sd(<2 x double>, <2 x double>, <2 x double>, i8, i32)
+
+define <2 x double>@test_int_x86_avx512_maskz_vfmadd_sd(<2 x double> %x0, <2 x double> %x1, <2 x double> %x2, i8 %x3,i32 %x4 ) #0 {
+;
+; CHECK-LABEL: @test_int_x86_avx512_maskz_vfmadd_sd(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 48) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = extractelement <2 x i64> [[TMP1]], i64 0
+; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <2 x double> [[X0:%.*]], i64 0
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = extractelement <2 x i64> [[TMP2]], i64 0
+; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <2 x double> [[X1:%.*]], i64 0
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = extractelement <2 x i64> [[TMP3]], i64 0
+; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <2 x double> [[X2:%.*]], i64 0
+; CHECK-NEXT:    [[_MSPROP3:%.*]] = or i64 [[_MSPROP]], [[_MSPROP1]]
+; CHECK-NEXT:    [[_MSPROP4:%.*]] = or i64 [[_MSPROP3]], [[_MSPROP2]]
+; CHECK-NEXT:    [[TMP8:%.*]] = call double @llvm.fma.f64(double [[TMP5]], double [[TMP6]], double [[TMP7]])
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast i8 [[TMP4]] to <8 x i1>
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i8 [[X3:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[_MSPROP5:%.*]] = extractelement <8 x i1> [[TMP9]], i64 0
+; CHECK-NEXT:    [[TMP11:%.*]] = extractelement <8 x i1> [[TMP10]], i64 0
+; CHECK-NEXT:    [[TMP12:%.*]] = select i1 [[TMP11]], i64 [[_MSPROP4]], i64 0
+; CHECK-NEXT:    [[TMP13:%.*]] = bitcast double [[TMP8]] to i64
+; CHECK-NEXT:    [[TMP14:%.*]] = xor i64 [[TMP13]], 0
+; CHECK-NEXT:    [[TMP15:%.*]] = or i64 [[TMP14]], [[_MSPROP4]]
+; CHECK-NEXT:    [[TMP16:%.*]] = or i64 [[TMP15]], 0
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select i1 [[_MSPROP5]], i64 [[TMP16]], i64 [[TMP12]]
+; CHECK-NEXT:    [[TMP17:%.*]] = select i1 [[TMP11]], double [[TMP8]], double 0.000000e+00
+; CHECK-NEXT:    [[_MSPROP6:%.*]] = insertelement <2 x i64> [[TMP1]], i64 [[_MSPROP_SELECT]], i64 0
+; CHECK-NEXT:    [[TMP18:%.*]] = insertelement <2 x double> [[X0]], double [[TMP17]], i64 0
+; CHECK-NEXT:    [[_MSPROP7:%.*]] = extractelement <2 x i64> [[TMP1]], i64 0
+; CHECK-NEXT:    [[TMP19:%.*]] = extractelement <2 x double> [[X0]], i64 0
+; CHECK-NEXT:    [[_MSPROP8:%.*]] = extractelement <2 x i64> [[TMP2]], i64 0
+; CHECK-NEXT:    [[TMP20:%.*]] = extractelement <2 x double> [[X1]], i64 0
+; CHECK-NEXT:    [[_MSPROP9:%.*]] = extractelement <2 x i64> [[TMP3]], i64 0
+; CHECK-NEXT:    [[TMP21:%.*]] = extractelement <2 x double> [[X2]], i64 0
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[_MSPROP7]], 0
+; CHECK-NEXT:    [[_MSCMP14:%.*]] = icmp ne i64 [[_MSPROP8]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP14]]
+; CHECK-NEXT:    [[_MSCMP15:%.*]] = icmp ne i64 [[_MSPROP9]], 0
+; CHECK-NEXT:    [[_MSOR16:%.*]] = or i1 [[_MSOR]], [[_MSCMP15]]
+; CHECK-NEXT:    br i1 [[_MSOR16]], label [[TMP22:%.*]], label [[TMP23:%.*]], !prof [[PROF1]]
+; CHECK:       22:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       23:
+; CHECK-NEXT:    [[TMP24:%.*]] = call double @llvm.x86.avx512.vfmadd.f64(double [[TMP19]], double [[TMP20]], double [[TMP21]], i32 11)
+; CHECK-NEXT:    [[TMP25:%.*]] = bitcast i8 [[TMP4]] to <8 x i1>
+; CHECK-NEXT:    [[TMP26:%.*]] = bitcast i8 [[X3]] to <8 x i1>
+; CHECK-NEXT:    [[_MSPROP10:%.*]] = extractelement <8 x i1> [[TMP25]], i64 0
+; CHECK-NEXT:    [[TMP27:%.*]] = extractelement <8 x i1> [[TMP26]], i64 0
+; CHECK-NEXT:    [[TMP28:%.*]] = select i1 [[TMP27]], i64 0, i64 0
+; CHECK-NEXT:    [[TMP29:%.*]] = bitcast double [[TMP24]] to i64
+; CHECK-NEXT:    [[TMP30:%.*]] = xor i64 [[TMP29]], 0
+; CHECK-NEXT:    [[TMP31:%.*]] = or i64 [[TMP30]], 0
+; CHECK-NEXT:    [[TMP32:%.*]] = or i64 [[TMP31]], 0
+; CHECK-NEXT:    [[_MSPROP_SELECT11:%.*]] = select i1 [[_MSPROP10]], i64 [[TMP32]], i64 [[TMP28]]
+; CHECK-NEXT:    [[TMP33:%.*]] = select i1 [[TMP27]], double [[TMP24]], double 0.000000e+00
+; CHECK-NEXT:    [[_MSPROP12:%.*]] = insertelement <2 x i64> [[TMP1]], i64 [[_MSPROP_SELECT11]], i64 0
+; CHECK-NEXT:    [[TMP34:%.*]] = insertelement <2 x double> [[X0]], double [[TMP33]], i64 0
+; CHECK-NEXT:    [[_MSPROP13:%.*]] = or <2 x i64> [[_MSPROP6]], [[_MSPROP12]]
+; CHECK-NEXT:    [[RES2:%.*]] = fadd <2 x double> [[TMP18]], [[TMP34]]
+; CHECK-NEXT:    store <2 x i64> [[_MSPROP13]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x double> [[RES2]]
+;
+  %res = call <2 x double> @llvm.x86.avx512.maskz.vfmadd.sd(<2 x double> %x0, <2 x double> %x1, <2 x double> %x2, i8 %x3, i32 4)
+  %res1 = call <2 x double> @llvm.x86.avx512.maskz.vfmadd.sd(<2 x double> %x0, <2 x double> %x1, <2 x double> %x2, i8 %x3, i32 11)
+  %res2 = fadd <2 x double> %res, %res1
+  ret <2 x double> %res2
+}
+
+declare <4 x float> @llvm.x86.avx512.maskz.vfmadd.ss(<4 x float>, <4 x float>, <4 x float>, i8, i32)
+
+define <4 x float>@test_int_x86_avx512_maskz_vfmadd_ss(<4 x float> %x0, <4 x float> %x1, <4 x float> %x2, i8 %x3,i32 %x4 ) #0 {
+;
+; CHECK-LABEL: @test_int_x86_avx512_maskz_vfmadd_ss(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 48) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = extractelement <4 x i32> [[TMP1]], i64 0
+; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <4 x float> [[X0:%.*]], i64 0
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = extractelement <4 x i32> [[TMP2]], i64 0
+; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <4 x float> [[X1:%.*]], i64 0
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = extractelement <4 x i32> [[TMP3]], i64 0
+; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <4 x float> [[X2:%.*]], i64 0
+; CHECK-NEXT:    [[_MSPROP3:%.*]] = or i32 [[_MSPROP]], [[_MSPROP1]]
+; CHECK-NEXT:    [[_MSPROP4:%.*]] = or i32 [[_MSPROP3]], [[_MSPROP2]]
+; CHECK-NEXT:    [[TMP8:%.*]] = call float @llvm.fma.f32(float [[TMP5]], float [[TMP6]], float [[TMP7]])
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast i8 [[TMP4]] to <8 x i1>
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i8 [[X3:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[_MSPROP5:%.*]] = extractelement <8 x i1> [[TMP9]], i64 0
+; CHECK-NEXT:    [[TMP11:%.*]] = extractelement <8 x i1> [[TMP10]], i64 0
+; CHECK-NEXT:    [[TMP12:%.*]] = select i1 [[TMP11]], i32 [[_MSPROP4]], i32 0
+; CHECK-NEXT:    [[TMP13:%.*]] = bitcast float [[TMP8]] to i32
+; CHECK-NEXT:    [[TMP14:%.*]] = xor i32 [[TMP13]], 0
+; CHECK-NEXT:    [[TMP15:%.*]] = or i32 [[TMP14]], [[_MSPROP4]]
+; CHECK-NEXT:    [[TMP16:%.*]] = or i32 [[TMP15]], 0
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select i1 [[_MSPROP5]], i32 [[TMP16]], i32 [[TMP12]]
+; CHECK-NEXT:    [[TMP17:%.*]] = select i1 [[TMP11]], float [[TMP8]], float 0.000000e+00
+; CHECK-NEXT:    [[_MSPROP6:%.*]] = insertelement <4 x i32> [[TMP1]], i32 [[_MSPROP_SELECT]], i64 0
+; CHECK-NEXT:    [[TMP18:%.*]] = insertelement <4 x float> [[X0]], float [[TMP17]], i64 0
+; CHECK-NEXT:    [[_MSPROP7:%.*]] = extractelement <4 x i32> [[TMP1]], i64 0
+; CHECK-NEXT:    [[TMP19:%.*]] = extractelement <4 x float> [[X0]], i64 0
+; CHECK-NEXT:    [[_MSPROP8:%.*]] = extractelement <4 x i32> [[TMP2]], i64 0
+; CHECK-NEXT:    [[TMP20:%.*]] = extractelement <4 x float> [[X1]], i64 0
+; CHECK-NEXT:    [[_MSPROP9:%.*]] = extractelement <4 x i32> [[TMP3]], i64 0
+; CHECK-NEXT:    [[TMP21:%.*]] = extractelement <4 x float> [[X2]], i64 0
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i32 [[_MSPROP7]], 0
+; CHECK-NEXT:    [[_MSCMP14:%.*]] = icmp ne i32 [[_MSPROP8]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP14]]
+; CHECK-NEXT:    [[_MSCMP15:%.*]] = icmp ne i32 [[_MSPROP9]], 0
+; CHECK-NEXT:    [[_MSOR16:%.*]] = or i1 [[_MSOR]], [[_MSCMP15]]
+; CHECK-NEXT:    br i1 [[_MSOR16]], label [[TMP22:%.*]], label [[TMP23:%.*]], !prof [[PROF1]]
+; CHECK:       22:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       23:
+; CHECK-NEXT:    [[TMP24:%.*]] = call float @llvm.x86.avx512.vfmadd.f32(float [[TMP19]], float [[TMP20]], float [[TMP21]], i32 11)
+; CHECK-NEXT:    [[TMP25:%.*]] = bitcast i8 [[TMP4]] to <8 x i1>
+; CHECK-NEXT:    [[TMP26:%.*]] = bitcast i8 [[X3]] to <8 x i1>
+; CHECK-NEXT:    [[_MSPROP10:%.*]] = extractelement <8 x i1> [[TMP25]], i64 0
+; CHECK-NEXT:    [[TMP27:%.*]] = extractelement <8 x i1> [[TMP26]], i64 0
+; CHECK-NEXT:    [[TMP28:%.*]] = select i1 [[TMP27]], i32 0, i32 0
+; CHECK-NEXT:    [[TMP29:%.*]] = bitcast float [[TMP24]] to i32
+; CHECK-NEXT:    [[TMP30:%.*]] = xor i32 [[TMP29]], 0
+; CHECK-NEXT:    [[TMP31:%.*]] = or i32 [[TMP30]], 0
+; CHECK-NEXT:    [[TMP32:%.*]] = or i32 [[TMP31]], 0
+; CHECK-NEXT:    [[_MSPROP_SELECT11:%.*]] = select i1 [[_MSPROP10]], i32 [[TMP32]], i32 [[TMP28]]
+; CHECK-NEXT:    [[TMP33:%.*]] = select i1 [[TMP27]], float [[TMP24]], float 0.000000e+00
+; CHECK-NEXT:    [[_MSPROP12:%.*]] = insertelement <4 x i32> [[TMP1]], i32 [[_MSPROP_SELECT11]], i64 0
+; CHECK-NEXT:    [[TMP34:%.*]] = insertelement <4 x float> [[X0]], float [[TMP33]], i64 0
+; CHECK-NEXT:    [[_MSPROP13:%.*]] = or <4 x i32> [[_MSPROP6]], [[_MSPROP12]]
+; CHECK-NEXT:    [[RES2:%.*]] = fadd <4 x float> [[TMP18]], [[TMP34]]
+; CHECK-NEXT:    store <4 x i32> [[_MSPROP6]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x float> [[TMP18]]
+;
+  %res = call <4 x float> @llvm.x86.avx512.maskz.vfmadd.ss(<4 x float> %x0, <4 x float> %x1, <4 x float> %x2, i8 %x3, i32 4)
+  %res1 = call <4 x float> @llvm.x86.avx512.maskz.vfmadd.ss(<4 x float> %x0, <4 x float> %x1, <4 x float> %x2, i8 %x3, i32 11)
+  %res2 = fadd <4 x float> %res, %res1
+  ret <4 x float> %res
+}
+declare <2 x double> @llvm.x86.avx512.mask3.vfmadd.sd(<2 x double>, <2 x double>, <2 x double>, i8, i32)
+
+define <2 x double>@test_int_x86_avx512_mask3_vfmadd_sd(<2 x double> %x0, <2 x double> %x1, <2 x double> %x2, i8 %x3,i32 %x4 ) #0 {
+;
+; CHECK-LABEL: @test_int_x86_avx512_mask3_vfmadd_sd(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 48) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = extractelement <2 x i64> [[TMP1]], i64 0
+; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <2 x double> [[X0:%.*]], i64 0
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = extractelement <2 x i64> [[TMP2]], i64 0
+; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <2 x double> [[X1:%.*]], i64 0
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = extractelement <2 x i64> [[TMP3]], i64 0
+; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <2 x double> [[X2:%.*]], i64 0
+; CHECK-NEXT:    [[_MSPROP3:%.*]] = or i64 [[_MSPROP]], [[_MSPROP1]]
+; CHECK-NEXT:    [[_MSPROP4:%.*]] = or i64 [[_MSPROP3]], [[_MSPROP2]]
+; CHECK-NEXT:    [[TMP8:%.*]] = call double @llvm.fma.f64(double [[TMP5]], double [[TMP6]], double [[TMP7]])
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast i8 [[TMP4]] to <8 x i1>
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i8 [[X3:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[_MSPROP5:%.*]] = extractelement <8 x i1> [[TMP9]], i64 0
+; CHECK-NEXT:    [[TMP11:%.*]] = extractelement <8 x i1> [[TMP10]], i64 0
+; CHECK-NEXT:    [[TMP12:%.*]] = select i1 [[TMP11]], i64 [[_MSPROP4]], i64 [[_MSPROP2]]
+; CHECK-NEXT:    [[TMP13:%.*]] = bitcast double [[TMP8]] to i64
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast double [[TMP7]] to i64
+; CHECK-NEXT:    [[TMP15:%.*]] = xor i64 [[TMP13]], [[TMP14]]
+; CHECK-NEXT:    [[TMP16:%.*]] = or i64 [[TMP15]], [[_MSPROP4]]
+; CHECK-NEXT:    [[TMP17:%.*]] = or i64 [[TMP16]], [[_MSPROP2]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select i1 [[_MSPROP5]], i64 [[TMP17]], i64 [[TMP12]]
+; CHECK-NEXT:    [[TMP18:%.*]] = select i1 [[TMP11]], double [[TMP8]], double [[TMP7]]
+; CHECK-NEXT:    [[_MSPROP6:%.*]] = insertelement <2 x i64> [[TMP3]], i64 [[_MSPROP_SELECT]], i64 0
+; CHECK-NEXT:    [[TMP19:%.*]] = insertelement <2 x double> [[X2]], double [[TMP18]], i64 0
+; CHECK-NEXT:    [[_MSPROP7:%.*]] = extractelement <2 x i64> [[TMP1]], i64 0
+; CHECK-NEXT:    [[TMP20:%.*]] = extractelement <2 x double> [[X0]], i64 0
+; CHECK-NEXT:    [[_MSPROP8:%.*]] = extractelement <2 x i64> [[TMP2]], i64 0
+; CHECK-NEXT:    [[TMP21:%.*]] = extractelement <2 x double> [[X1]], i64 0
+; CHECK-NEXT:    [[_MSPROP9:%.*]] = extractelement <2 x i64> [[TMP3]], i64 0
+; CHECK-NEXT:    [[TMP22:%.*]] = extractelement <2 x double> [[X2]], i64 0
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[_MSPROP7]], 0
+; CHECK-NEXT:    [[_MSCMP19:%.*]] = icmp ne i64 [[_MSPROP8]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP19]]
+; CHECK-NEXT:    [[_MSCMP20:%.*]] = icmp ne i64 [[_MSPROP9]], 0
+; CHECK-NEXT:    [[_MSOR21:%.*]] = or i1 [[_MSOR]], [[_MSCMP20]]
+; CHECK-NEXT:    br i1 [[_MSOR21]], label [[TMP23:%.*]], label [[TMP24:%.*]], !prof [[PROF1]]
+; CHECK:       23:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       24:
+; CHECK-NEXT:    [[TMP25:%.*]] = call double @llvm.x86.avx512.vfmadd.f64(double [[TMP20]], double [[TMP21]], double [[TMP22]], i32 11)
+; CHECK-NEXT:    [[_MSPROP10:%.*]] = insertelement <2 x i64> [[TMP3]], i64 0, i64 0
+; CHECK-NEXT:    [[TMP26:%.*]] = insertelement <2 x double> [[X2]], double [[TMP25]], i64 0
+; CHECK-NEXT:    [[_MSPROP11:%.*]] = extractelement <2 x i64> [[TMP1]], i64 0
+; CHECK-NEXT:    [[TMP27:%.*]] = extractelement <2 x double> [[X0]], i64 0
+; CHECK-NEXT:    [[_MSPROP12:%.*]] = extractelement <2 x i64> [[TMP2]], i64 0
+; CHECK-NEXT:    [[TMP28:%.*]] = extractelement <2 x double> [[X1]], i64 0
+; CHECK-NEXT:    [[_MSPROP13:%.*]] = extractelement <2 x i64> [[TMP3]], i64 0
+; CHECK-NEXT:    [[TMP29:%.*]] = extractelement <2 x double> [[X2]], i64 0
+; CHECK-NEXT:    [[_MSCMP22:%.*]] = icmp ne i64 [[_MSPROP11]], 0
+; CHECK-NEXT:    [[_MSCMP23:%.*]] = icmp ne i64 [[_MSPROP12]], 0
+; CHECK-NEXT:    [[_MSOR24:%.*]] = or i1 [[_MSCMP22]], [[_MSCMP23]]
+; CHECK-NEXT:    [[_MSCMP25:%.*]] = icmp ne i64 [[_MSPROP13]], 0
+; CHECK-NEXT:    [[_MSOR26:%.*]] = or i1 [[_MSOR24]], [[_MSCMP25]]
+; CHECK-NEXT:    br i1 [[_MSOR26]], label [[TMP30:%.*]], label [[TMP31:%.*]], !prof [[PROF1]]
+; CHECK:       30:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       31:
+; CHECK-NEXT:    [[TMP32:%.*]] = call double @llvm.x86.avx512.vfmadd.f64(double [[TMP27]], double [[TMP28]], double [[TMP29]], i32 10)
+; CHECK-NEXT:    [[TMP33:%.*]] = bitcast i8 [[TMP4]] to <8 x i1>
+; CHECK-NEXT:    [[TMP34:%.*]] = bitcast i8 [[X3]] to <8 x i1>
+; CHECK-NEXT:    [[_MSPROP14:%.*]] = extractelement <8 x i1> [[TMP33]], i64 0
+; CHECK-NEXT:    [[TMP35:%.*]] = extractelement <8 x i1> [[TMP34]], i64 0
+; CHECK-NEXT:    [[TMP36:%.*]] = select i1 [[TMP35]], i64 0, i64 [[_MSPROP13]]
+; CHECK-NEXT:    [[TMP37:%.*]] = bitcast double [[TMP32]] to i64
+; CHECK-NEXT:    [[TMP38:%.*]] = bitcast double [[TMP29]] to i64
+; CHECK-NEXT:    [[TMP39:%.*]] = xor i64 [[TMP37]], [[TMP38]]
+; CHECK-NEXT:    [[TMP40:%.*]] = or i64 [[TMP39]], 0
+; CHECK-NEXT:    [[TMP41:%.*]] = or i64 [[TMP40]], [[_MSPROP13]]
+; CHECK-NEXT:    [[_MSPROP_SELECT15:%.*]] = select i1 [[_MSPROP14]], i64 [[TMP41]], i64 [[TMP36]]
+; CHECK-NEXT:    [[TMP42:%.*]] = select i1 [[TMP35]], double [[TMP32]], double [[TMP29]]
+; CHECK-NEXT:    [[_MSPROP16:%.*]] = insertelement <2 x i64> [[TMP3]], i64 [[_MSPROP_SELECT15]], i64 0
+; CHECK-NEXT:    [[TMP43:%.*]] = insertelement <2 x double> [[X2]], double [[TMP42]], i64 0
+; CHECK-NEXT:    [[_MSPROP17:%.*]] = or <2 x i64> [[_MSPROP6]], [[_MSPROP10]]
+; CHECK-NEXT:    [[RES3:%.*]] = fadd <2 x double> [[TMP19]], [[TMP26]]
+; CHECK-NEXT:    [[_MSPROP18:%.*]] = or <2 x i64> [[_MSPROP16]], [[_MSPROP17]]
+; CHECK-NEXT:    [[RES4:%.*]] = fadd <2 x double> [[TMP43]], [[RES3]]
+; CHECK-NEXT:    store <2 x i64> [[_MSPROP18]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x double> [[RES4]]
+;
+  %res = call <2 x double> @llvm.x86.avx512.mask3.vfmadd.sd(<2 x double> %x0, <2 x double> %x1, <2 x double> %x2, i8 %x3, i32 4)
+  %res1 = call <2 x double> @llvm.x86.avx512.mask3.vfmadd.sd(<2 x double> %x0, <2 x double> %x1, <2 x double> %x2, i8 -1, i32 11)
+  %res2 = call <2 x double> @llvm.x86.avx512.mask3.vfmadd.sd(<2 x double> %x0, <2 x double> %x1, <2 x double> %x2, i8 %x3, i32 10)
+  %res3 = fadd <2 x double> %res, %res1
+  %res4 = fadd <2 x double> %res2, %res3
+  ret <2 x double> %res4
+}
+
+declare <4 x float> @llvm.x86.avx512.mask3.vfmadd.ss(<4 x float>, <4 x float>, <4 x float>, i8, i32)
+
+define <4 x float>@test_int_x86_avx512_mask3_vfmadd_ss(<4 x float> %x0, <4 x float> %x1, <4 x float> %x2, i8 %x3,i32 %x4 ) #0 {
+;
+; CHECK-LABEL: @test_int_x86_avx512_mask3_vfmadd_ss(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 48) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = extractelement <4 x i32> [[TMP1]], i64 0
+; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <4 x float> [[X0:%.*]], i64 0
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = extractelement <4 x i32> [[TMP2]], i64 0
+; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <4 x float> [[X1:%.*]], i64 0
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = extractelement <4 x i32> [[TMP3]], i64 0
+; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <4 x float> [[X2:%.*]], i64 0
+; CHECK-NEXT:    [[_MSPROP3:%.*]] = or i32 [[_MSPROP]], [[_MSPROP1]]
+; CHECK-NEXT:    [[_MSPROP4:%.*]] = or i32 [[_MSPROP3]], [[_MSPROP2]]
+; CHECK-NEXT:    [[TMP8:%.*]] = call float @llvm.fma.f32(float [[TMP5]], float [[TMP6]], float [[TMP7]])
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast i8 [[TMP4]] to <8 x i1>
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i8 [[X3:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[_MSPROP5:%.*]] = extractelement <8 x i1> [[TMP9]], i64 0
+; CHECK-NEXT:    [[TMP11:%.*]] = extractelement <8 x i1> [[TMP10]], i64 0
+; CHECK-NEXT:    [[TMP12:%.*]] = select i1 [[TMP11]], i32 [[_MSPROP4]], i32 [[_MSPROP2]]
+; CHECK-NEXT:    [[TMP13:%.*]] = bitcast float [[TMP8]] to i32
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast float [[TMP7]] to i32
+; CHECK-NEXT:    [[TMP15:%.*]] = xor i32 [[TMP13]], [[TMP14]]
+; CHECK-NEXT:    [[TMP16:%.*]] = or i32 [[TMP15]], [[_MSPROP4]]
+; CHECK-NEXT:    [[TMP17:%.*]] = or i32 [[TMP16]], [[_MSPROP2]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select i1 [[_MSPROP5]], i32 [[TMP17]], i32 [[TMP12]]
+; CHECK-NEXT:    [[TMP18:%.*]] = select i1 [[TMP11]], float [[TMP8]], float [[TMP7]]
+; CHECK-NEXT:    [[_MSPROP6:%.*]] = insertelement <4 x i32> [[TMP3]], i32 [[_MSPROP_SELECT]], i64 0
+; CHECK-NEXT:    [[TMP19:%.*]] = insertelement <4 x float> [[X2]], float [[TMP18]], i64 0
+; CHECK-NEXT:    [[_MSPROP7:%.*]] = extractelement <4 x i32> [[TMP1]], i64 0
+; CHECK-NEXT:    [[TMP20:%.*]] = extractelement <4 x float> [[X0]], i64 0
+; CHECK-NEXT:    [[_MSPROP8:%.*]] = extractelement <4 x i32> [[TMP2]], i64 0
+; CHECK-NEXT:    [[TMP21:%.*]] = extractelement <4 x float> [[X1]], i64 0
+; CHECK-NEXT:    [[_MSPROP9:%.*]] = extractelement <4 x i32> [[TMP3]], i64 0
+; CHECK-NEXT:    [[TMP22:%.*]] = extractelement <4 x float> [[X2]], i64 0
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i32 [[_MSPROP7]], 0
+; CHECK-NEXT:    [[_MSCMP19:%.*]] = icmp ne i32 [[_MSPROP8]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP19]]
+; CHECK-NEXT:    [[_MSCMP20:%.*]] = icmp ne i32 [[_MSPROP9]], 0
+; CHECK-NEXT:    [[_MSOR21:%.*]] = or i1 [[_MSOR]], [[_MSCMP20]]
+; CHECK-NEXT:    br i1 [[_MSOR21]], label [[TMP23:%.*]], label [[TMP24:%.*]], !prof [[PROF1]]
+; CHECK:       23:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       24:
+; CHECK-NEXT:    [[TMP25:%.*]] = call float @llvm.x86.avx512.vfmadd.f32(float [[TMP20]], float [[TMP21]], float [[TMP22]], i32 11)
+; CHECK-NEXT:    [[_MSPROP10:%.*]] = insertelement <4 x i32> [[TMP3]], i32 0, i64 0
+; CHECK-NEXT:    [[TMP26:%.*]] = insertelement <4 x float> [[X2]], float [[TMP25]], i64 0
+; CHECK-NEXT:    [[_MSPROP11:%.*]] = extractelement <4 x i32> [[TMP1]], i64 0
+; CHECK-NEXT:    [[TMP27:%.*]] = extractelement <4 x float> [[X0]], i64 0
+; CHECK-NEXT:    [[_MSPROP12:%.*]] = extractelement <4 x i32> [[TMP2]], i64 0
+; CHECK-NEXT:    [[TMP28:%.*]] = extractelement <4 x float> [[X1]], i64 0
+; CHECK-NEXT:    [[_MSPROP13:%.*]] = extractelement <4 x i32> [[TMP3]], i64 0
+; CHECK-NEXT:    [[TMP29:%.*]] = extractelement <4 x float> [[X2]], i64 0
+; CHECK-NEXT:    [[_MSCMP22:%.*]] = icmp ne i32 [[_MSPROP11]], 0
+; CHECK-NEXT:    [[_MSCMP23:%.*]] = icmp ne i32 [[_MSPROP12]], 0
+; CHECK-NEXT:    [[_MSOR24:%.*]] = or i1 [[_MSCMP22]], [[_MSCMP23]]
+; CHECK-NEXT:    [[_MSCMP25:%.*]] = icmp ne i32 [[_MSPROP13]], 0
+; CHECK-NEXT:    [[_MSOR26:%.*]] = or i1 [[_MSOR24]], [[_MSCMP25]]
+; CHECK-NEXT:    br i1 [[_MSOR26]], label [[TMP30:%.*]], label [[TMP31:%.*]], !prof [[PROF1]]
+; CHECK:       30:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       31:
+; CHECK-NEXT:    [[TMP32:%.*]] = call float @llvm.x86.avx512.vfmadd.f32(float [[TMP27]], float [[TMP28]], float [[TMP29]], i32 10)
+; CHECK-NEXT:    [[TMP33:%.*]] = bitcast i8 [[TMP4]] to <8 x i1>
+; CHECK-NEXT:    [[TMP34:%.*]] = bitcast i8 [[X3]] to <8 x i1>
+; CHECK-NEXT:    [[_MSPROP14:%.*]] = extractelement <8 x i1> [[TMP33]], i64 0
+; CHECK-NEXT:    [[TMP35:%.*]] = extractelement <8 x i1> [[TMP34]], i64 0
+; CHECK-NEXT:    [[TMP36:%.*]] = select i1 [[TMP35]], i32 0, i32 [[_MSPROP13]]
+; CHECK-NEXT:    [[TMP37:%.*]] = bitcast float [[TMP32]] to i32
+; CHECK-NEXT:    [[TMP38:%.*]] = bitcast float [[TMP29]] to i32
+; CHECK-NEXT:    [[TMP39:%.*]] = xor i32 [[TMP37]], [[TMP38]]
+; CHECK-NEXT:    [[TMP40:%.*]] = or i32 [[TMP39]], 0
+; CHECK-NEXT:    [[TMP41:%.*]] = or i32 [[TMP40]], [[_MSPROP13]]
+; CHECK-NEXT:    [[_MSPROP_SELECT15:%.*]] = select i1 [[_MSPROP14]], i32 [[TMP41]], i32 [[TMP36]]
+; CHECK-NEXT:    [[TMP42:%.*]] = select i1 [[TMP35]], float [[TMP32]], float [[TMP29]]
+; CHECK-NEXT:    [[_MSPROP16:%.*]] = insertelement <4 x i32> [[TMP3]], i32 [[_MSPROP_SELECT15]], i64 0
+; CHECK-NEXT:    [[TMP43:%.*]] = insertelement <4 x float> [[X2]], float [[TMP42]], i64 0
+; CHECK-NEXT:    [[_MSPROP17:%.*]] = or <4 x i32> [[_MSPROP6]], [[_MSPROP10]]
+; CHECK-NEXT:    [[RES3:%.*]] = fadd <4 x float> [[TMP19]], [[TMP26]]
+; CHECK-NEXT:    [[_MSPROP18:%.*]] = or <4 x i32> [[_MSPROP16]], [[_MSPROP17]]
+; CHECK-NEXT:    [[RES4:%.*]] = fadd <4 x float> [[TMP43]], [[RES3]]
+; CHECK-NEXT:    store <4 x i32> [[_MSPROP18]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x float> [[RES4]]
+;
+  %res = call <4 x float> @llvm.x86.avx512.mask3.vfmadd.ss(<4 x float> %x0, <4 x float> %x1, <4 x float> %x2, i8 %x3, i32 4)
+  %res1 = call <4 x float> @llvm.x86.avx512.mask3.vfmadd.ss(<4 x float> %x0, <4 x float> %x1, <4 x float> %x2, i8 -1, i32 11)
+  %res2 = call <4 x float> @llvm.x86.avx512.mask3.vfmadd.ss(<4 x float> %x0, <4 x float> %x1, <4 x float> %x2, i8 %x3, i32 10)
+  %res3 = fadd <4 x float> %res, %res1
+  %res4 = fadd <4 x float> %res2, %res3
+  ret <4 x float> %res4
+}
+
+define void @fmadd_ss_mask_memfold(ptr %a, ptr %b, i8 %c, <4 x float> %extra_param)  #0 {
+;
+; CHECK-LABEL: @fmadd_ss_mask_memfold(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 24) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP5:%.*]], label [[TMP9:%.*]], !prof [[PROF1]]
+; CHECK:       5:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       6:
+; CHECK-NEXT:    [[A_VAL:%.*]] = load float, ptr [[A:%.*]], align 4
+; CHECK-NEXT:    [[TMP6:%.*]] = ptrtoint ptr [[A]] to i64
+; CHECK-NEXT:    [[TMP7:%.*]] = xor i64 [[TMP6]], 87960930222080
+; CHECK-NEXT:    [[TMP8:%.*]] = inttoptr i64 [[TMP7]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load i32, ptr [[TMP8]], align 4
+; CHECK-NEXT:    [[_MSPROP:%.*]] = insertelement <4 x i32> [[TMP4]], i32 [[_MSLD]], i32 0
+; CHECK-NEXT:    [[AV0:%.*]] = insertelement <4 x float> [[EXTRA_PARAM:%.*]], float [[A_VAL]], i32 0
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = insertelement <4 x i32> [[_MSPROP]], i32 0, i32 1
+; CHECK-NEXT:    [[AV1:%.*]] = insertelement <4 x float> [[AV0]], float 0.000000e+00, i32 1
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = insertelement <4 x i32> [[_MSPROP1]], i32 0, i32 2
+; CHECK-NEXT:    [[AV2:%.*]] = insertelement <4 x float> [[AV1]], float 0.000000e+00, i32 2
+; CHECK-NEXT:    [[_MSPROP3:%.*]] = insertelement <4 x i32> [[_MSPROP2]], i32 0, i32 3
+; CHECK-NEXT:    [[AV:%.*]] = insertelement <4 x float> [[AV2]], float 0.000000e+00, i32 3
+; CHECK-NEXT:    [[_MSCMP17:%.*]] = icmp ne i64 [[TMP2]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP17]], label [[TMP10:%.*]], label [[TMP29:%.*]], !prof [[PROF1]]
+; CHECK:       10:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       11:
+; CHECK-NEXT:    [[B_VAL:%.*]] = load float, ptr [[B:%.*]], align 4
+; CHECK-NEXT:    [[TMP11:%.*]] = ptrtoint ptr [[B]] to i64
+; CHECK-NEXT:    [[TMP12:%.*]] = xor i64 [[TMP11]], 87960930222080
+; CHECK-NEXT:    [[TMP13:%.*]] = inttoptr i64 [[TMP12]] to ptr
+; CHECK-NEXT:    [[_MSLD4:%.*]] = load i32, ptr [[TMP13]], align 4
+; CHECK-NEXT:    [[_MSPROP5:%.*]] = insertelement <4 x i32> [[TMP4]], i32 [[_MSLD4]], i32 0
+; CHECK-NEXT:    [[BV0:%.*]] = insertelement <4 x float> [[EXTRA_PARAM]], float [[B_VAL]], i32 0
+; CHECK-NEXT:    [[_MSPROP6:%.*]] = insertelement <4 x i32> [[_MSPROP5]], i32 0, i32 1
+; CHECK-NEXT:    [[BV1:%.*]] = insertelement <4 x float> [[BV0]], float 0.000000e+00, i32 1
+; CHECK-NEXT:    [[_MSPROP7:%.*]] = insertelement <4 x i32> [[_MSPROP6]], i32 0, i32 2
+; CHECK-NEXT:    [[BV2:%.*]] = insertelement <4 x float> [[BV1]], float 0.000000e+00, i32 2
+; CHECK-NEXT:    [[_MSPROP8:%.*]] = insertelement <4 x i32> [[_MSPROP7]], i32 0, i32 3
+; CHECK-NEXT:    [[BV:%.*]] = insertelement <4 x float> [[BV2]], float 0.000000e+00, i32 3
+; CHECK-NEXT:    [[_MSPROP9:%.*]] = extractelement <4 x i32> [[_MSPROP3]], i64 0
+; CHECK-NEXT:    [[TMP14:%.*]] = extractelement <4 x float> [[AV]], i64 0
+; CHECK-NEXT:    [[_MSPROP10:%.*]] = extractelement <4 x i32> [[_MSPROP8]], i64 0
+; CHECK-NEXT:    [[TMP15:%.*]] = extractelement <4 x float> [[BV]], i64 0
+; CHECK-NEXT:    [[_MSPROP11:%.*]] = extractelement <4 x i32> [[_MSPROP3]], i64 0
+; CHECK-NEXT:    [[TMP16:%.*]] = extractelement <4 x float> [[AV]], i64 0
+; CHECK-NEXT:    [[_MSPROP12:%.*]] = or i32 [[_MSPROP9]], [[_MSPROP10]]
+; CHECK-NEXT:    [[_MSPROP13:%.*]] = or i32 [[_MSPROP12]], [[_MSPROP11]]
+; CHECK-NEXT:    [[TMP17:%.*]] = call float @llvm.fma.f32(float [[TMP14]], float [[TMP15]], float [[TMP16]])
+; CHECK-NEXT:    [[TMP18:%.*]] = bitcast i8 [[TMP3]] to <8 x i1>
+; CHECK-NEXT:    [[TMP19:%.*]] = bitcast i8 [[C:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[_MSPROP14:%.*]] = extractelement <8 x i1> [[TMP18]], i64 0
+; CHECK-NEXT:    [[TMP20:%.*]] = extractelement <8 x i1> [[TMP19]], i64 0
+; CHECK-NEXT:    [[TMP21:%.*]] = select i1 [[TMP20]], i32 [[_MSPROP13]], i32 [[_MSPROP9]]
+; CHECK-NEXT:    [[TMP22:%.*]] = bitcast float [[TMP17]] to i32
+; CHECK-NEXT:    [[TMP23:%.*]] = bitcast float [[TMP14]] to i32
+; CHECK-NEXT:    [[TMP24:%.*]] = xor i32 [[TMP22]], [[TMP23]]
+; CHECK-NEXT:    [[TMP25:%.*]] = or i32 [[TMP24]], [[_MSPROP13]]
+; CHECK-NEXT:    [[TMP26:%.*]] = or i32 [[TMP25]], [[_MSPROP9]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select i1 [[_MSPROP14]], i32 [[TMP26]], i32 [[TMP21]]
+; CHECK-NEXT:    [[TMP27:%.*]] = select i1 [[TMP20]], float [[TMP17]], float [[TMP14]]
+; CHECK-NEXT:    [[_MSPROP15:%.*]] = insertelement <4 x i32> [[_MSPROP3]], i32 [[_MSPROP_SELECT]], i64 0
+; CHECK-NEXT:    [[TMP28:%.*]] = insertelement <4 x float> [[AV]], float [[TMP27]], i64 0
+; CHECK-NEXT:    [[_MSPROP16:%.*]] = extractelement <4 x i32> [[_MSPROP15]], i32 0
+; CHECK-NEXT:    [[SR:%.*]] = extractelement <4 x float> [[TMP28]], i32 0
+; CHECK-NEXT:    [[_MSCMP18:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP18]], label [[TMP30:%.*]], label [[TMP34:%.*]], !prof [[PROF1]]
+; CHECK:       30:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       31:
+; CHECK-NEXT:    [[TMP31:%.*]] = ptrtoint ptr [[A]] to i64
+; CHECK-NEXT:    [[TMP32:%.*]] = xor i64 [[TMP31]], 87960930222080
+; CHECK-NEXT:    [[TMP33:%.*]] = inttoptr i64 [[TMP32]] to ptr
+; CHECK-NEXT:    store i32 [[_MSPROP16]], ptr [[TMP33]], align 4
+; CHECK-NEXT:    store float [[SR]], ptr [[A]], align 4
+; CHECK-NEXT:    ret void
+;
+  %a.val = load float, ptr %a
+  %av0 = insertelement <4 x float> %extra_param, float %a.val, i32 0
+  %av1 = insertelement <4 x float> %av0, float 0.000000e+00, i32 1
+  %av2 = insertelement <4 x float> %av1, float 0.000000e+00, i32 2
+  %av  = insertelement <4 x float> %av2, float 0.000000e+00, i32 3
+
+  %b.val = load float, ptr %b
+  %bv0 = insertelement <4 x float> %extra_param, float %b.val, i32 0
+  %bv1 = insertelement <4 x float> %bv0, float 0.000000e+00, i32 1
+  %bv2 = insertelement <4 x float> %bv1, float 0.000000e+00, i32 2
+  %bv  = insertelement <4 x float> %bv2, float 0.000000e+00, i32 3
+
+  %vr = call <4 x float> @llvm.x86.avx512.mask.vfmadd.ss(<4 x float> %av, <4 x float> %bv, <4 x float> %av, i8 %c, i32 4)
+
+  %sr = extractelement <4 x float> %vr, i32 0
+  store float %sr, ptr %a
+  ret void
+}
+
+define void @fmadd_ss_maskz_memfold(ptr %a, ptr %b, i8 %c, <4 x float> %extra_param)  #0 {
+;
+; CHECK-LABEL: @fmadd_ss_maskz_memfold(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 24) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP5:%.*]], label [[TMP9:%.*]], !prof [[PROF1]]
+; CHECK:       5:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       6:
+; CHECK-NEXT:    [[A_VAL:%.*]] = load float, ptr [[A:%.*]], align 4
+; CHECK-NEXT:    [[TMP6:%.*]] = ptrtoint ptr [[A]] to i64
+; CHECK-NEXT:    [[TMP7:%.*]] = xor i64 [[TMP6]], 87960930222080
+; CHECK-NEXT:    [[TMP8:%.*]] = inttoptr i64 [[TMP7]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load i32, ptr [[TMP8]], align 4
+; CHECK-NEXT:    [[_MSPROP:%.*]] = insertelement <4 x i32> [[TMP4]], i32 [[_MSLD]], i32 0
+; CHECK-NEXT:    [[AV0:%.*]] = insertelement <4 x float> [[EXTRA_PARAM:%.*]], float [[A_VAL]], i32 0
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = insertelement <4 x i32> [[_MSPROP]], i32 0, i32 1
+; CHECK-NEXT:    [[AV1:%.*]] = insertelement <4 x float> [[AV0]], float 0.000000e+00, i32 1
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = insertelement <4 x i32> [[_MSPROP1]], i32 0, i32 2
+; CHECK-NEXT:    [[AV2:%.*]] = insertelement <4 x float> [[AV1]], float 0.000000e+00, i32 2
+; CHECK-NEXT:    [[_MSPROP3:%.*]] = insertelement <4 x i32> [[_MSPROP2]], i32 0, i32 3
+; CHECK-NEXT:    [[AV:%.*]] = insertelement <4 x float> [[AV2]], float 0.000000e+00, i32 3
+; CHECK-NEXT:    [[_MSCMP17:%.*]] = icmp ne i64 [[TMP2]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP17]], label [[TMP10:%.*]], label [[TMP28:%.*]], !prof [[PROF1]]
+; CHECK:       10:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       11:
+; CHECK-NEXT:    [[B_VAL:%.*]] = load float, ptr [[B:%.*]], align 4
+; CHECK-NEXT:    [[TMP11:%.*]] = ptrtoint ptr [[B]] to i64
+; CHECK-NEXT:    [[TMP12:%.*]] = xor i64 [[TMP11]], 87960930222080
+; CHECK-NEXT:    [[TMP13:%.*]] = inttoptr i64 [[TMP12]] to ptr
+; CHECK-NEXT:    [[_MSLD4:%.*]] = load i32, ptr [[TMP13]], align 4
+; CHECK-NEXT:    [[_MSPROP5:%.*]] = insertelement <4 x i32> [[TMP4]], i32 [[_MSLD4]], i32 0
+; CHECK-NEXT:    [[BV0:%.*]] = insertelement <4 x float> [[EXTRA_PARAM]], float [[B_VAL]], i32 0
+; CHECK-NEXT:    [[_MSPROP6:%.*]] = insertelement <4 x i32> [[_MSPROP5]], i32 0, i32 1
+; CHECK-NEXT:    [[BV1:%.*]] = insertelement <4 x float> [[BV0]], float 0.000000e+00, i32 1
+; CHECK-NEXT:    [[_MSPROP7:%.*]] = insertelement <4 x i32> [[_MSPROP6]], i32 0, i32 2
+; CHECK-NEXT:    [[BV2:%.*]] = insertelement <4 x float> [[BV1]], float 0.000000e+00, i32 2
+; CHECK-NEXT:    [[_MSPROP8:%.*]] = insertelement <4 x i32> [[_MSPROP7]], i32 0, i32 3
+; CHECK-NEXT:    [[BV:%.*]] = insertelement <4 x float> [[BV2]], float 0.000000e+00, i32 3
+; CHECK-NEXT:    [[_MSPROP9:%.*]] = extractelement <4 x i32> [[_MSPROP3]], i64 0
+; CHECK-NEXT:    [[TMP14:%.*]] = extractelement <4 x float> [[AV]], i64 0
+; CHECK-NEXT:    [[_MSPROP10:%.*]] = extractelement <4 x i32> [[_MSPROP8]], i64 0
+; CHECK-NEXT:    [[TMP15:%.*]] = extractelement <4 x float> [[BV]], i64 0
+; CHECK-NEXT:    [[_MSPROP11:%.*]] = extractelement <4 x i32> [[_MSPROP3]], i64 0
+; CHECK-NEXT:    [[TMP16:%.*]] = extractelement <4 x float> [[AV]], i64 0
+; CHECK-NEXT:    [[_MSPROP12:%.*]] = or i32 [[_MSPROP9]], [[_MSPROP10]]
+; CHECK-NEXT:    [[_MSPROP13:%.*]] = or i32 [[_MSPROP12]], [[_MSPROP11]]
+; CHECK-NEXT:    [[TMP17:%.*]] = call float @llvm.fma.f32(float [[TMP14]], float [[TMP15]], float [[TMP16]])
+; CHECK-NEXT:    [[TMP18:%.*]] = bitcast i8 [[TMP3]] to <8 x i1>
+; CHECK-NEXT:    [[TMP19:%.*]] = bitcast i8 [[C:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[_MSPROP14:%.*]] = extractelement <8 x i1> [[TMP18]], i64 0
+; CHECK-NEXT:    [[TMP20:%.*]] = extractelement <8 x i1> [[TMP19]], i64 0
+; CHECK-NEXT:    [[TMP21:%.*]] = select i1 [[TMP20]], i32 [[_MSPROP13]], i32 0
+; CHECK-NEXT:    [[TMP22:%.*]] = bitcast float [[TMP17]] to i32
+; CHECK-NEXT:    [[TMP23:%.*]] = xor i32 [[TMP22]], 0
+; CHECK-NEXT:    [[TMP24:%.*]] = or i32 [[TMP23]], [[_MSPROP13]]
+; CHECK-NEXT:    [[TMP25:%.*]] = or i32 [[TMP24]], 0
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select i1 [[_MSPROP14]], i32 [[TMP25]], i32 [[TMP21]]
+; CHECK-NEXT:    [[TMP26:%.*]] = select i1 [[TMP20]], float [[TMP17]], float 0.000000e+00
+; CHECK-NEXT:    [[_MSPROP15:%.*]] = insertelement <4 x i32> [[_MSPROP3]], i32 [[_MSPROP_SELECT]], i64 0
+; CHECK-NEXT:    [[TMP27:%.*]] = insertelement <4 x float> [[AV]], float [[TMP26]], i64 0
+; CHECK-NEXT:    [[_MSPROP16:%.*]] = extractelement <4 x i32> [[_MSPROP15]], i32 0
+; CHECK-NEXT:    [[SR:%.*]] = extractelement <4 x float> [[TMP27]], i32 0
+; CHECK-NEXT:    [[_MSCMP18:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP18]], label [[TMP29:%.*]], label [[TMP33:%.*]], !prof [[PROF1]]
+; CHECK:       29:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       30:
+; CHECK-NEXT:    [[TMP30:%.*]] = ptrtoint ptr [[A]] to i64
+; CHECK-NEXT:    [[TMP31:%.*]] = xor i64 [[TMP30]], 87960930222080
+; CHECK-NEXT:    [[TMP32:%.*]] = inttoptr i64 [[TMP31]] to ptr
+; CHECK-NEXT:    store i32 [[_MSPROP16]], ptr [[TMP32]], align 4
+; CHECK-NEXT:    store float [[SR]], ptr [[A]], align 4
+; CHECK-NEXT:    ret void
+;
+  %a.val = load float, ptr %a
+  %av0 = insertelement <4 x float> %extra_param, float %a.val, i32 0
+  %av1 = insertelement <4 x float> %av0, float 0.000000e+00, i32 1
+  %av2 = insertelement <4 x float> %av1, float 0.000000e+00, i32 2
+  %av  = insertelement <4 x float> %av2, float 0.000000e+00, i32 3
+
+  %b.val = load float, ptr %b
+  %bv0 = insertelement <4 x float> %extra_param, float %b.val, i32 0
+  %bv1 = insertelement <4 x float> %bv0, float 0.000000e+00, i32 1
+  %bv2 = insertelement <4 x float> %bv1, float 0.000000e+00, i32 2
+  %bv  = insertelement <4 x float> %bv2, float 0.000000e+00, i32 3
+
+  %vr = call <4 x float> @llvm.x86.avx512.maskz.vfmadd.ss(<4 x float> %av, <4 x float> %bv, <4 x float> %av, i8 %c, i32 4)
+
+  %sr = extractelement <4 x float> %vr, i32 0
+  store float %sr, ptr %a
+  ret void
+}
+
+define void @fmadd_sd_mask_memfold(ptr %a, ptr %b, i8 %c, <2 x double> %extra_param)  #0 {
+;
+; CHECK-LABEL: @fmadd_sd_mask_memfold(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 24) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP5:%.*]], label [[TMP9:%.*]], !prof [[PROF1]]
+; CHECK:       5:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       6:
+; CHECK-NEXT:    [[A_VAL:%.*]] = load double, ptr [[A:%.*]], align 8
+; CHECK-NEXT:    [[TMP6:%.*]] = ptrtoint ptr [[A]] to i64
+; CHECK-NEXT:    [[TMP7:%.*]] = xor i64 [[TMP6]], 87960930222080
+; CHECK-NEXT:    [[TMP8:%.*]] = inttoptr i64 [[TMP7]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load i64, ptr [[TMP8]], align 8
+; CHECK-NEXT:    [[_MSPROP:%.*]] = insertelement <2 x i64> [[TMP4]], i64 [[_MSLD]], i32 0
+; CHECK-NEXT:    [[AV0:%.*]] = insertelement <2 x double> [[EXTRA_PARAM:%.*]], double [[A_VAL]], i32 0
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = insertelement <2 x i64> [[_MSPROP]], i64 0, i32 1
+; CHECK-NEXT:    [[AV:%.*]] = insertelement <2 x double> [[AV0]], double 0.000000e+00, i32 1
+; CHECK-NEXT:    [[_MSCMP13:%.*]] = icmp ne i64 [[TMP2]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP13]], label [[TMP10:%.*]], label [[TMP29:%.*]], !prof [[PROF1]]
+; CHECK:       10:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       11:
+; CHECK-NEXT:    [[B_VAL:%.*]] = load double, ptr [[B:%.*]], align 8
+; CHECK-NEXT:    [[TMP11:%.*]] = ptrtoint ptr [[B]] to i64
+; CHECK-NEXT:    [[TMP12:%.*]] = xor i64 [[TMP11]], 87960930222080
+; CHECK-NEXT:    [[TMP13:%.*]] = inttoptr i64 [[TMP12]] to ptr
+; CHECK-NEXT:    [[_MSLD2:%.*]] = load i64, ptr [[TMP13]], align 8
+; CHECK-NEXT:    [[_MSPROP3:%.*]] = insertelement <2 x i64> [[TMP4]], i64 [[_MSLD2]], i32 0
+; CHECK-NEXT:    [[BV0:%.*]] = insertelement <2 x double> [[EXTRA_PARAM]], double [[B_VAL]], i32 0
+; CHECK-NEXT:    [[_MSPROP4:%.*]] = insertelement <2 x i64> [[_MSPROP3]], i64 0, i32 1
+; CHECK-NEXT:    [[BV:%.*]] = insertelement <2 x double> [[BV0]], double 0.000000e+00, i32 1
+; CHECK-NEXT:    [[_MSPROP5:%.*]] = extractelement <2 x i64> [[_MSPROP1]], i64 0
+; CHECK-NEXT:    [[TMP14:%.*]] = extractelement <2 x double> [[AV]], i64 0
+; CHECK-NEXT:    [[_MSPROP6:%.*]] = extractelement <2 x i64> [[_MSPROP4]], i64 0
+; CHECK-NEXT:    [[TMP15:%.*]] = extractelement <2 x double> [[BV]], i64 0
+; CHECK-NEXT:    [[_MSPROP7:%.*]] = extractelement <2 x i64> [[_MSPROP1]], i64 0
+; CHECK-NEXT:    [[TMP16:%.*]] = extractelement <2 x double> [[AV]], i64 0
+; CHECK-NEXT:    [[_MSPROP8:%.*]] = or i64 [[_MSPROP5]], [[_MSPROP6]]
+; CHECK-NEXT:    [[_MSPROP9:%.*]] = or i64 [[_MSPROP8]], [[_MSPROP7]]
+; CHECK-NEXT:    [[TMP17:%.*]] = call double @llvm.fma.f64(double [[TMP14]], double [[TMP15]], double [[TMP16]])
+; CHECK-NEXT:    [[TMP18:%.*]] = bitcast i8 [[TMP3]] to <8 x i1>
+; CHECK-NEXT:    [[TMP19:%.*]] = bitcast i8 [[C:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[_MSPROP10:%.*]] = extractelement <8 x i1> [[TMP18]], i64 0
+; CHECK-NEXT:    [[TMP20:%.*]] = extractelement <8 x i1> [[TMP19]], i64 0
+; CHECK-NEXT:    [[TMP21:%.*]] = select i1 [[TMP20]], i64 [[_MSPROP9]], i64 [[_MSPROP5]]
+; CHECK-NEXT:    [[TMP22:%.*]] = bitcast double [[TMP17]] to i64
+; CHECK-NEXT:    [[TMP23:%.*]] = bitcast double [[TMP14]] to i64
+; CHECK-NEXT:    [[TMP24:%.*]] = xor i64 [[TMP22]], [[TMP23]]
+; CHECK-NEXT:    [[TMP25:%.*]] = or i64 [[TMP24]], [[_MSPROP9]]
+; CHECK-NEXT:    [[TMP26:%.*]] = or i64 [[TMP25]], [[_MSPROP5]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select i1 [[_MSPROP10]], i64 [[TMP26]], i64 [[TMP21]]
+; CHECK-NEXT:    [[TMP27:%.*]] = select i1 [[TMP20]], double [[TMP17]], double [[TMP14]]
+; CHECK-NEXT:    [[_MSPROP11:%.*]] = insertelement <2 x i64> [[_MSPROP1]], i64 [[_MSPROP_SELECT]], i64 0
+; CHECK-NEXT:    [[TMP28:%.*]] = insertelement <2 x double> [[AV]], double [[TMP27]], i64 0
+; CHECK-NEXT:    [[_MSPROP12:%.*]] = extractelement <2 x i64> [[_MSPROP11]], i32 0
+; CHECK-NEXT:    [[SR:%.*]] = extractelement <2 x double> [[TMP28]], i32 0
+; CHECK-NEXT:    [[_MSCMP14:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP14]], label [[TMP30:%.*]], label [[TMP34:%.*]], !prof [[PROF1]]
+; CHECK:       30:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       31:
+; CHECK-NEXT:    [[TMP31:%.*]] = ptrtoint ptr [[A]] to i64
+; CHECK-NEXT:    [[TMP32:%.*]] = xor i64 [[TMP31]], 87960930222080
+; CHECK-NEXT:    [[TMP33:%.*]] = inttoptr i64 [[TMP32]] to ptr
+; CHECK-NEXT:    store i64 [[_MSPROP12]], ptr [[TMP33]], align 8
+; CHECK-NEXT:    store double [[SR]], ptr [[A]], align 8
+; CHECK-NEXT:    ret void
+;
+  %a.val = load double, ptr %a
+  %av0 = insertelement <2 x double> %extra_param, double %a.val, i32 0
+  %av = insertelement <2 x double> %av0, double 0.000000e+00, i32 1
+
+  %b.val = load double, ptr %b
+  %bv0 = insertelement <2 x double> %extra_param, double %b.val, i32 0
+  %bv = insertelement <2 x double> %bv0, double 0.000000e+00, i32 1
+
+  %vr = call <2 x double> @llvm.x86.avx512.mask.vfmadd.sd(<2 x double> %av, <2 x double> %bv, <2 x double> %av, i8 %c, i32 4)
+
+  %sr = extractelement <2 x double> %vr, i32 0
+  store double %sr, ptr %a
+  ret void
+}
+
+define void @fmadd_sd_maskz_memfold(ptr %a, ptr %b, i8 %c, <2 x double> %extra_param)  #0 {
+;
+; CHECK-LABEL: @fmadd_sd_maskz_memfold(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 24) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP5:%.*]], label [[TMP9:%.*]], !prof [[PROF1]]
+; CHECK:       5:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       6:
+; CHECK-NEXT:    [[A_VAL:%.*]] = load double, ptr [[A:%.*]], align 8
+; CHECK-NEXT:    [[TMP6:%.*]] = ptrtoint ptr [[A]] to i64
+; CHECK-NEXT:    [[TMP7:%.*]] = xor i64 [[TMP6]], 87960930222080
+; CHECK-NEXT:    [[TMP8:%.*]] = inttoptr i64 [[TMP7]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load i64, ptr [[TMP8]], align 8
+; CHECK-NEXT:    [[_MSPROP:%.*]] = insertelement <2 x i64> [[TMP4]], i64 [[_MSLD]], i32 0
+; CHECK-NEXT:    [[AV0:%.*]] = insertelement <2 x double> [[EXTRA_PARAM:%.*]], double [[A_VAL]], i32 0
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = insertelement <2 x i64> [[_MSPROP]], i64 0, i32 1
+; CHECK-NEXT:    [[AV:%.*]] = insertelement <2 x double> [[AV0]], double 0.000000e+00, i32 1
+; CHECK-NEXT:    [[_MSCMP13:%.*]] = icmp ne i64 [[TMP2]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP13]], label [[TMP10:%.*]], label [[TMP28:%.*]], !prof [[PROF1]]
+; CHECK:       10:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       11:
+; CHECK-NEXT:    [[B_VAL:%.*]] = load double, ptr [[B:%.*]], align 8
+; CHECK-NEXT:    [[TMP11:%.*]] = ptrtoint ptr [[B]] to i64
+; CHECK-NEXT:    [[TMP12:%.*]] = xor i64 [[TMP11]], 87960930222080
+; CHECK-NEXT:    [[TMP13:%.*]] = inttoptr i64 [[TMP12]] to ptr
+; CHECK-NEXT:    [[_MSLD2:%.*]] = load i64, ptr [[TMP13]], align 8
+; CHECK-NEXT:    [[_MSPROP3:%.*]] = insertelement <2 x i64> [[TMP4]], i64 [[_MSLD2]], i32 0
+; CHECK-NEXT:    [[BV0:%.*]] = insertelement <2 x double> [[EXTRA_PARAM]], double [[B_VAL]], i32 0
+; CHECK-NEXT:    [[_MSPROP4:%.*]] = insertelement <2 x i64> [[_MSPROP3]], i64 0, i32 1
+; CHECK-NEXT:    [[BV:%.*]] = insertelement <2 x double> [[BV0]], double 0.000000e+00, i32 1
+; CHECK-NEXT:    [[_MSPROP5:%.*]] = extractelement <2 x i64> [[_MSPROP1]], i64 0
+; CHECK-NEXT:    [[TMP14:%.*]] = extractelement <2 x double> [[AV]], i64 0
+; CHECK-NEXT:    [[_MSPROP6:%.*]] = extractelement <2 x i64> [[_MSPROP4]], i64 0
+; CHECK-NEXT:    [[TMP15:%.*]] = extractelement <2 x double> [[BV]], i64 0
+; CHECK-NEXT:    [[_MSPROP7:%.*]] = extractelement <2 x i64> [[_MSPROP1]], i64 0
+; CHECK-NEXT:    [[TMP16:%.*]] = extractelement <2 x double> [[AV]], i64 0
+; CHECK-NEXT:    [[_MSPROP8:%.*]] = or i64 [[_MSPROP5]], [[_MSPROP6]]
+; CHECK-NEXT:    [[_MSPROP9:%.*]] = or i64 [[_MSPROP8]], [[_MSPROP7]]
+; CHECK-NEXT:    [[TMP17:%.*]] = call double @llvm.fma.f64(double [[TMP14]], double [[TMP15]], double [[TMP16]])
+; CHECK-NEXT:    [[TMP18:%.*]] = bitcast i8 [[TMP3]] to <8 x i1>
+; CHECK-NEXT:    [[TMP19:%.*]] = bitcast i8 [[C:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[_MSPROP10:%.*]] = extractelement <8 x i1> [[TMP18]], i64 0
+; CHECK-NEXT:    [[TMP20:%.*]] = extractelement <8 x i1> [[TMP19]], i64 0
+; CHECK-NEXT:    [[TMP21:%.*]] = select i1 [[TMP20]], i64 [[_MSPROP9]], i64 0
+; CHECK-NEXT:    [[TMP22:%.*]] = bitcast double [[TMP17]] to i64
+; CHECK-NEXT:    [[TMP23:%.*]] = xor i64 [[TMP22]], 0
+; CHECK-NEXT:    [[TMP24:%.*]] = or i64 [[TMP23]], [[_MSPROP9]]
+; CHECK-NEXT:    [[TMP25:%.*]] = or i64 [[TMP24]], 0
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select i1 [[_MSPROP10]], i64 [[TMP25]], i64 [[TMP21]]
+; CHECK-NEXT:    [[TMP26:%.*]] = select i1 [[TMP20]], double [[TMP17]], double 0.000000e+00
+; CHECK-NEXT:    [[_MSPROP11:%.*]] = insertelement <2 x i64> [[_MSPROP1]], i64 [[_MSPROP_SELECT]], i64 0
+; CHECK-NEXT:    [[TMP27:%.*]] = insertelement <2 x double> [[AV]], double [[TMP26]], i64 0
+; CHECK-NEXT:    [[_MSPROP12:%.*]] = extractelement <2 x i64> [[_MSPROP11]], i32 0
+; CHECK-NEXT:    [[SR:%.*]] = extractelement <2 x double> [[TMP27]], i32 0
+; CHECK-NEXT:    [[_MSCMP14:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP14]], label [[TMP29:%.*]], label [[TMP33:%.*]], !prof [[PROF1]]
+; CHECK:       29:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       30:
+; CHECK-NEXT:    [[TMP30:%.*]] = ptrtoint ptr [[A]] to i64
+; CHECK-NEXT:    [[TMP31:%.*]] = xor i64 [[TMP30]], 87960930222080
+; CHECK-NEXT:    [[TMP32:%.*]] = inttoptr i64 [[TMP31]] to ptr
+; CHECK-NEXT:    store i64 [[_MSPROP12]], ptr [[TMP32]], align 8
+; CHECK-NEXT:    store double [[SR]], ptr [[A]], align 8
+; CHECK-NEXT:    ret void
+;
+  %a.val = load double, ptr %a
+  %av0 = insertelement <2 x double> %extra_param, double %a.val, i32 0
+  %av = insertelement <2 x double> %av0, double 0.000000e+00, i32 1
+
+  %b.val = load double, ptr %b
+  %bv0 = insertelement <2 x double> %extra_param, double %b.val, i32 0
+  %bv = insertelement <2 x double> %bv0, double 0.000000e+00, i32 1
+
+  %vr = call <2 x double> @llvm.x86.avx512.maskz.vfmadd.sd(<2 x double> %av, <2 x double> %bv, <2 x double> %av, i8 %c, i32 4)
+
+  %sr = extractelement <2 x double> %vr, i32 0
+  store double %sr, ptr %a
+  ret void
+}
+
+declare <2 x double> @llvm.x86.avx512.mask3.vfmsub.sd(<2 x double>, <2 x double>, <2 x double>, i8, i32)
+
+define <2 x double>@test_int_x86_avx512_mask3_vfmsub_sd(<2 x double> %x0, <2 x double> %x1, <2 x double> %x2, i8 %x3,i32 %x4 ) #0 {
+;
+; CHECK-LABEL: @test_int_x86_avx512_mask3_vfmsub_sd(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 48) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = fneg <2 x double> [[X2:%.*]]
+; CHECK-NEXT:    [[_MSPROP:%.*]] = extractelement <2 x i64> [[TMP2]], i64 0
+; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <2 x double> [[X0:%.*]], i64 0
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = extractelement <2 x i64> [[TMP3]], i64 0
+; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <2 x double> [[X1:%.*]], i64 0
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = extractelement <2 x i64> [[TMP1]], i64 0
+; CHECK-NEXT:    [[TMP8:%.*]] = extractelement <2 x double> [[TMP5]], i64 0
+; CHECK-NEXT:    [[_MSPROP3:%.*]] = or i64 [[_MSPROP]], [[_MSPROP1]]
+; CHECK-NEXT:    [[_MSPROP4:%.*]] = or i64 [[_MSPROP3]], [[_MSPROP2]]
+; CHECK-NEXT:    [[TMP9:%.*]] = call double @llvm.fma.f64(double [[TMP6]], double [[TMP7]], double [[TMP8]])
+; CHECK-NEXT:    [[_MSPROP5:%.*]] = extractelement <2 x i64> [[TMP1]], i64 0
+; CHECK-NEXT:    [[TMP10:%.*]] = extractelement <2 x double> [[X2]], i64 0
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast i8 [[TMP4]] to <8 x i1>
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast i8 [[X3:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[_MSPROP6:%.*]] = extractelement <8 x i1> [[TMP11]], i64 0
+; CHECK-NEXT:    [[TMP13:%.*]] = extractelement <8 x i1> [[TMP12]], i64 0
+; CHECK-NEXT:    [[TMP14:%.*]] = select i1 [[TMP13]], i64 [[_MSPROP4]], i64 [[_MSPROP5]]
+; CHECK-NEXT:    [[TMP15:%.*]] = bitcast double [[TMP9]] to i64
+; CHECK-NEXT:    [[TMP16:%.*]] = bitcast double [[TMP10]] to i64
+; CHECK-NEXT:    [[TMP17:%.*]] = xor i64 [[TMP15]], [[TMP16]]
+; CHECK-NEXT:    [[TMP18:%.*]] = or i64 [[TMP17]], [[_MSPROP4]]
+; CHECK-NEXT:    [[TMP19:%.*]] = or i64 [[TMP18]], [[_MSPROP5]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select i1 [[_MSPROP6]], i64 [[TMP19]], i64 [[TMP14]]
+; CHECK-NEXT:    [[TMP20:%.*]] = select i1 [[TMP13]], double [[TMP9]], double [[TMP10]]
+; CHECK-NEXT:    [[_MSPROP7:%.*]] = insertelement <2 x i64> [[TMP1]], i64 [[_MSPROP_SELECT]], i64 0
+; CHECK-NEXT:    [[TMP21:%.*]] = insertelement <2 x double> [[X2]], double [[TMP20]], i64 0
+; CHECK-NEXT:    [[TMP22:%.*]] = fneg <2 x double> [[X2]]
+; CHECK-NEXT:    [[_MSPROP8:%.*]] = extractelement <2 x i64> [[TMP2]], i64 0
+; CHECK-NEXT:    [[TMP23:%.*]] = extractelement <2 x double> [[X0]], i64 0
+; CHECK-NEXT:    [[_MSPROP9:%.*]] = extractelement <2 x i64> [[TMP3]], i64 0
+; CHECK-NEXT:    [[TMP24:%.*]] = extractelement <2 x double> [[X1]], i64 0
+; CHECK-NEXT:    [[_MSPROP10:%.*]] = extractelement <2 x i64> [[TMP1]], i64 0
+; CHECK-NEXT:    [[TMP25:%.*]] = extractelement <2 x double> [[TMP22]], i64 0
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[_MSPROP8]], 0
+; CHECK-NEXT:    [[_MSCMP22:%.*]] = icmp ne i64 [[_MSPROP9]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP22]]
+; CHECK-NEXT:    [[_MSCMP23:%.*]] = icmp ne i64 [[_MSPROP10]], 0
+; CHECK-NEXT:    [[_MSOR24:%.*]] = or i1 [[_MSOR]], [[_MSCMP23]]
+; CHECK-NEXT:    br i1 [[_MSOR24]], label [[TMP26:%.*]], label [[TMP27:%.*]], !prof [[PROF1]]
+; CHECK:       26:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       27:
+; CHECK-NEXT:    [[TMP28:%.*]] = call double @llvm.x86.avx512.vfmadd.f64(double [[TMP23]], double [[TMP24]], double [[TMP25]], i32 11)
+; CHECK-NEXT:    [[_MSPROP11:%.*]] = extractelement <2 x i64> [[TMP1]], i64 0
+; CHECK-NEXT:    [[TMP29:%.*]] = extractelement <2 x double> [[X2]], i64 0
+; CHECK-NEXT:    [[_MSPROP12:%.*]] = insertelement <2 x i64> [[TMP1]], i64 0, i64 0
+; CHECK-NEXT:    [[TMP30:%.*]] = insertelement <2 x double> [[X2]], double [[TMP28]], i64 0
+; CHECK-NEXT:    [[TMP31:%.*]] = fneg <2 x double> [[X2]]
+; CHECK-NEXT:    [[_MSPROP13:%.*]] = extractelement <2 x i64> [[TMP2]], i64 0
+; CHECK-NEXT:    [[TMP32:%.*]] = extractelement <2 x double> [[X0]], i64 0
+; CHECK-NEXT:    [[_MSPROP14:%.*]] = extractelement <2 x i64> [[TMP3]], i64 0
+; CHECK-NEXT:    [[TMP33:%.*]] = extractelement <2 x double> [[X1]], i64 0
+; CHECK-NEXT:    [[_MSPROP15:%.*]] = extractelement <2 x i64> [[TMP1]], i64 0
+; CHECK-NEXT:    [[TMP34:%.*]] = extractelement <2 x double> [[TMP31]], i64 0
+; CHECK-NEXT:    [[_MSCMP25:%.*]] = icmp ne i64 [[_MSPROP13]], 0
+; CHECK-NEXT:    [[_MSCMP26:%.*]] = icmp ne i64 [[_MSPROP14]], 0
+; CHECK-NEXT:    [[_MSOR27:%.*]] = or i1 [[_MSCMP25]], [[_MSCMP26]]
+; CHECK-NEXT:    [[_MSCMP28:%.*]] = icmp ne i64 [[_MSPROP15]], 0
+; CHECK-NEXT:    [[_MSOR29:%.*]] = or i1 [[_MSOR27]], [[_MSCMP28]]
+; CHECK-NEXT:    br i1 [[_MSOR29]], label [[TMP35:%.*]], label [[TMP36:%.*]], !prof [[PROF1]]
+; CHECK:       35:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       36:
+; CHECK-NEXT:    [[TMP37:%.*]] = call double @llvm.x86.avx512.vfmadd.f64(double [[TMP32]], double [[TMP33]], double [[TMP34]], i32 10)
+; CHECK-NEXT:    [[_MSPROP16:%.*]] = extractelement <2 x i64> [[TMP1]], i64 0
+; CHECK-NEXT:    [[TMP38:%.*]] = extractelement <2 x double> [[X2]], i64 0
+; CHECK-NEXT:    [[TMP39:%.*]] = bitcast i8 [[TMP4]] to <8 x i1>
+; CHECK-NEXT:    [[TMP40:%.*]] = bitcast i8 [[X3]] to <8 x i1>
+; CHECK-NEXT:    [[_MSPROP17:%.*]] = extractelement <8 x i1> [[TMP39]], i64 0
+; CHECK-NEXT:    [[TMP41:%.*]] = extractelement <8 x i1> [[TMP40]], i64 0
+; CHECK-NEXT:    [[TMP42:%.*]] = select i1 [[TMP41]], i64 0, i64 [[_MSPROP16]]
+; CHECK-NEXT:    [[TMP43:%.*]] = bitcast double [[TMP37]] to i64
+; CHECK-NEXT:    [[TMP44:%.*]] = bitcast double [[TMP38]] to i64
+; CHECK-NEXT:    [[TMP45:%.*]] = xor i64 [[TMP43]], [[TMP44]]
+; CHECK-NEXT:    [[TMP46:%.*]] = or i64 [[TMP45]], 0
+; CHECK-NEXT:    [[TMP47:%.*]] = or i64 [[TMP46]], [[_MSPROP16]]
+; CHECK-NEXT:    [[_MSPROP_SELECT18:%.*]] = select i1 [[_MSPROP17]], i64 [[TMP47]], i64 [[TMP42]]
+; CHECK-NEXT:    [[TMP48:%.*]] = select i1 [[TMP41]], double [[TMP37]], double [[TMP38]]
+; CHECK-NEXT:    [[_MSPROP19:%.*]] = insertelement <2 x i64> [[TMP1]], i64 [[_MSPROP_SELECT18]], i64 0
+; CHECK-NEXT:    [[TMP49:%.*]] = insertelement <2 x double> [[X2]], double [[TMP48]], i64 0
+; CHECK-NEXT:    [[_MSPROP20:%.*]] = or <2 x i64> [[_MSPROP7]], [[_MSPROP12]]
+; CHECK-NEXT:    [[RES3:%.*]] = fadd <2 x double> [[TMP21]], [[TMP30]]
+; CHECK-NEXT:    [[_MSPROP21:%.*]] = or <2 x i64> [[_MSPROP19]], [[_MSPROP20]]
+; CHECK-NEXT:    [[RES4:%.*]] = fadd <2 x double> [[TMP49]], [[RES3]]
+; CHECK-NEXT:    store <2 x i64> [[_MSPROP21]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x double> [[RES4]]
+;
+  %res = call <2 x double> @llvm.x86.avx512.mask3.vfmsub.sd(<2 x double> %x0, <2 x double> %x1, <2 x double> %x2, i8 %x3, i32 4)
+  %res1 = call <2 x double> @llvm.x86.avx512.mask3.vfmsub.sd(<2 x double> %x0, <2 x double> %x1, <2 x double> %x2, i8 -1, i32 11)
+  %res2 = call <2 x double> @llvm.x86.avx512.mask3.vfmsub.sd(<2 x double> %x0, <2 x double> %x1, <2 x double> %x2, i8 %x3, i32 10)
+  %res3 = fadd <2 x double> %res, %res1
+  %res4 = fadd <2 x double> %res2, %res3
+  ret <2 x double> %res4
+}
+
+declare <4 x float> @llvm.x86.avx512.mask3.vfmsub.ss(<4 x float>, <4 x float>, <4 x float>, i8, i32)
+
+define <4 x float>@test_int_x86_avx512_mask3_vfmsub_ss(<4 x float> %x0, <4 x float> %x1, <4 x float> %x2, i8 %x3,i32 %x4 ) #0 {
+;
+; CHECK-LABEL: @test_int_x86_avx512_mask3_vfmsub_ss(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 48) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = fneg <4 x float> [[X2:%.*]]
+; CHECK-NEXT:    [[_MSPROP:%.*]] = extractelement <4 x i32> [[TMP2]], i64 0
+; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <4 x float> [[X0:%.*]], i64 0
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = extractelement <4 x i32> [[TMP3]], i64 0
+; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <4 x float> [[X1:%.*]], i64 0
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = extractelement <4 x i32> [[TMP1]], i64 0
+; CHECK-NEXT:    [[TMP8:%.*]] = extractelement <4 x float> [[TMP5]], i64 0
+; CHECK-NEXT:    [[_MSPROP3:%.*]] = or i32 [[_MSPROP]], [[_MSPROP1]]
+; CHECK-NEXT:    [[_MSPROP4:%.*]] = or i32 [[_MSPROP3]], [[_MSPROP2]]
+; CHECK-NEXT:    [[TMP9:%.*]] = call float @llvm.fma.f32(float [[TMP6]], float [[TMP7]], float [[TMP8]])
+; CHECK-NEXT:    [[_MSPROP5:%.*]] = extractelement <4 x i32> [[TMP1]], i64 0
+; CHECK-NEXT:    [[TMP10:%.*]] = extractelement <4 x float> [[X2]], i64 0
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast i8 [[TMP4]] to <8 x i1>
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast i8 [[X3:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[_MSPROP6:%.*]] = extractelement <8 x i1> [[TMP11]], i64 0
+; CHECK-NEXT:    [[TMP13:%.*]] = extractelement <8 x i1> [[TMP12]], i64 0
+; CHECK-NEXT:    [[TMP14:%.*]] = select i1 [[TMP13]], i32 [[_MSPROP4]], i32 [[_MSPROP5]]
+; CHECK-NEXT:    [[TMP15:%.*]] = bitcast float [[TMP9]] to i32
+; CHECK-NEXT:    [[TMP16:%.*]] = bitcast float [[TMP10]] to i32
+; CHECK-NEXT:    [[TMP17:%.*]] = xor i32 [[TMP15]], [[TMP16]]
+; CHECK-NEXT:    [[TMP18:%.*]] = or i32 [[TMP17]], [[_MSPROP4]]
+; CHECK-NEXT:    [[TMP19:%.*]] = or i32 [[TMP18]], [[_MSPROP5]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select i1 [[_MSPROP6]], i32 [[TMP19]], i32 [[TMP14]]
+; CHECK-NEXT:    [[TMP20:%.*]] = select i1 [[TMP13]], float [[TMP9]], float [[TMP10]]
+; CHECK-NEXT:    [[_MSPROP7:%.*]] = insertelement <4 x i32> [[TMP1]], i32 [[_MSPROP_SELECT]], i64 0
+; CHECK-NEXT:    [[TMP21:%.*]] = insertelement <4 x float> [[X2]], float [[TMP20]], i64 0
+; CHECK-NEXT:    [[TMP22:%.*]] = fneg <4 x float> [[X2]]
+; CHECK-NEXT:    [[_MSPROP8:%.*]] = extractelement <4 x i32> [[TMP2]], i64 0
+; CHECK-NEXT:    [[TMP23:%.*]] = extractelement <4 x float> [[X0]], i64 0
+; CHECK-NEXT:    [[_MSPROP9:%.*]] = extractelement <4 x i32> [[TMP3]], i64 0
+; CHECK-NEXT:    [[TMP24:%.*]] = extractelement <4 x float> [[X1]], i64 0
+; CHECK-NEXT:    [[_MSPROP10:%.*]] = extractelement <4 x i32> [[TMP1]], i64 0
+; CHECK-NEXT:    [[TMP25:%.*]] = extractelement <4 x float> [[TMP22]], i64 0
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i32 [[_MSPROP8]], 0
+; CHECK-NEXT:    [[_MSCMP22:%.*]] = icmp ne i32 [[_MSPROP9]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP22]]
+; CHECK-NEXT:    [[_MSCMP23:%.*]] = icmp ne i32 [[_MSPROP10]], 0
+; CHECK-NEXT:    [[_MSOR24:%.*]] = or i1 [[_MSOR]], [[_MSCMP23]]
+; CHECK-NEXT:    br i1 [[_MSOR24]], label [[TMP26:%.*]], label [[TMP27:%.*]], !prof [[PROF1]]
+; CHECK:       26:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       27:
+; CHECK-NEXT:    [[TMP28:%.*]] = call float @llvm.x86.avx512.vfmadd.f32(float [[TMP23]], float [[TMP24]], float [[TMP25]], i32 11)
+; CHECK-NEXT:    [[_MSPROP11:%.*]] = extractelement <4 x i32> [[TMP1]], i64 0
+; CHECK-NEXT:    [[TMP29:%.*]] = extractelement <4 x float> [[X2]], i64 0
+; CHECK-NEXT:    [[_MSPROP12:%.*]] = insertelement <4 x i32> [[TMP1]], i32 0, i64 0
+; CHECK-NEXT:    [[TMP30:%.*]] = insertelement <4 x float> [[X2]], float [[TMP28]], i64 0
+; CHECK-NEXT:    [[TMP31:%.*]] = fneg <4 x float> [[X2]]
+; CHECK-NEXT:    [[_MSPROP13:%.*]] = extractelement <4 x i32> [[TMP2]], i64 0
+; CHECK-NEXT:    [[TMP32:%.*]] = extractelement <4 x float> [[X0]], i64 0
+; CHECK-NEXT:    [[_MSPROP14:%.*]] = extractelement <4 x i32> [[TMP3]], i64 0
+; CHECK-NEXT:    [[TMP33:%.*]] = extractelement <4 x float> [[X1]], i64 0
+; CHECK-NEXT:    [[_MSPROP15:%.*]] = extractelement <4 x i32> [[TMP1]], i64 0
+; CHECK-NEXT:    [[TMP34:%.*]] = extractelement <4 x float> [[TMP31]], i64 0
+; CHECK-NEXT:    [[_MSCMP25:%.*]] = icmp ne i32 [[_MSPROP13]], 0
+; CHECK-NEXT:    [[_MSCMP26:%.*]] = icmp ne i32 [[_MSPROP14]], 0
+; CHECK-NEXT:    [[_MSOR27:%.*]] = or i1 [[_MSCMP25]], [[_MSCMP26]]
+; CHECK-NEXT:    [[_MSCMP28:%.*]] = icmp ne i32 [[_MSPROP15]], 0
+; CHECK-NEXT:    [[_MSOR29:%.*]] = or i1 [[_MSOR27]], [[_MSCMP28]]
+; CHECK-NEXT:    br i1 [[_MSOR29]], label [[TMP35:%.*]], label [[TMP36:%.*]], !prof [[PROF1]]
+; CHECK:       35:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       36:
+; CHECK-NEXT:    [[TMP37:%.*]] = call float @llvm.x86.avx512.vfmadd.f32(float [[TMP32]], float [[TMP33]], float [[TMP34]], i32 10)
+; CHECK-NEXT:    [[_MSPROP16:%.*]] = extractelement <4 x i32> [[TMP1]], i64 0
+; CHECK-NEXT:    [[TMP38:%.*]] = extractelement <4 x float> [[X2]], i64 0
+; CHECK-NEXT:    [[TMP39:%.*]] = bitcast i8 [[TMP4]] to <8 x i1>
+; CHECK-NEXT:    [[TMP40:%.*]] = bitcast i8 [[X3]] to <8 x i1>
+; CHECK-NEXT:    [[_MSPROP17:%.*]] = extractelement <8 x i1> [[TMP39]], i64 0
+; CHECK-NEXT:    [[TMP41:%.*]] = extractelement <8 x i1> [[TMP40]], i64 0
+; CHECK-NEXT:    [[TMP42:%.*]] = select i1 [[TMP41]], i32 0, i32 [[_MSPROP16]]
+; CHECK-NEXT:    [[TMP43:%.*]] = bitcast float [[TMP37]] to i32
+; CHECK-NEXT:    [[TMP44:%.*]] = bitcast float [[TMP38]] to i32
+; CHECK-NEXT:    [[TMP45:%.*]] = xor i32 [[TMP43]], [[TMP44]]
+; CHECK-NEXT:    [[TMP46:%.*]] = or i32 [[TMP45]], 0
+; CHECK-NEXT:    [[TMP47:%.*]] = or i32 [[TMP46]], [[_MSPROP16]]
+; CHECK-NEXT:    [[_MSPROP_SELECT18:%.*]] = select i1 [[_MSPROP17]], i32 [[TMP47]], i32 [[TMP42]]
+; CHECK-NEXT:    [[TMP48:%.*]] = select i1 [[TMP41]], float [[TMP37]], float [[TMP38]]
+; CHECK-NEXT:    [[_MSPROP19:%.*]] = insertelement <4 x i32> [[TMP1]], i32 [[_MSPROP_SELECT18]], i64 0
+; CHECK-NEXT:    [[TMP49:%.*]] = insertelement <4 x float> [[X2]], float [[TMP48]], i64 0
+; CHECK-NEXT:    [[_MSPROP20:%.*]] = or <4 x i32> [[_MSPROP7]], [[_MSPROP12]]
+; CHECK-NEXT:    [[RES3:%.*]] = fadd <4 x float> [[TMP21]], [[TMP30]]
+; CHECK-NEXT:    [[_MSPROP21:%.*]] = or <4 x i32> [[_MSPROP19]], [[_MSPROP20]]
+; CHECK-NEXT:    [[RES4:%.*]] = fadd <4 x float> [[TMP49]], [[RES3]]
+; CHECK-NEXT:    store <4 x i32> [[_MSPROP21]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x float> [[RES4]]
+;
+  %res = call <4 x float> @llvm.x86.avx512.mask3.vfmsub.ss(<4 x float> %x0, <4 x float> %x1, <4 x float> %x2, i8 %x3, i32 4)
+  %res1 = call <4 x float> @llvm.x86.avx512.mask3.vfmsub.ss(<4 x float> %x0, <4 x float> %x1, <4 x float> %x2, i8 -1, i32 11)
+  %res2 = call <4 x float> @llvm.x86.avx512.mask3.vfmsub.ss(<4 x float> %x0, <4 x float> %x1, <4 x float> %x2, i8 %x3, i32 10)
+  %res3 = fadd <4 x float> %res, %res1
+  %res4 = fadd <4 x float> %res2, %res3
+  ret <4 x float> %res4
+}
+
+declare <2 x double> @llvm.x86.avx512.mask3.vfnmsub.sd(<2 x double>, <2 x double>, <2 x double>, i8, i32)
+
+define <2 x double>@test_int_x86_avx512_mask3_vfnmsub_sd(<2 x double> %x0, <2 x double> %x1, <2 x double> %x2, i8 %x3,i32 %x4 ) #0 {
+;
+; CHECK-LABEL: @test_int_x86_avx512_mask3_vfnmsub_sd(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 48) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = fneg <2 x double> [[X0:%.*]]
+; CHECK-NEXT:    [[TMP6:%.*]] = fneg <2 x double> [[X2:%.*]]
+; CHECK-NEXT:    [[_MSPROP:%.*]] = extractelement <2 x i64> [[TMP1]], i64 0
+; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <2 x double> [[TMP5]], i64 0
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = extractelement <2 x i64> [[TMP3]], i64 0
+; CHECK-NEXT:    [[TMP8:%.*]] = extractelement <2 x double> [[X1:%.*]], i64 0
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = extractelement <2 x i64> [[TMP2]], i64 0
+; CHECK-NEXT:    [[TMP9:%.*]] = extractelement <2 x double> [[TMP6]], i64 0
+; CHECK-NEXT:    [[_MSPROP3:%.*]] = or i64 [[_MSPROP]], [[_MSPROP1]]
+; CHECK-NEXT:    [[_MSPROP4:%.*]] = or i64 [[_MSPROP3]], [[_MSPROP2]]
+; CHECK-NEXT:    [[TMP10:%.*]] = call double @llvm.fma.f64(double [[TMP7]], double [[TMP8]], double [[TMP9]])
+; CHECK-NEXT:    [[_MSPROP5:%.*]] = extractelement <2 x i64> [[TMP2]], i64 0
+; CHECK-NEXT:    [[TMP11:%.*]] = extractelement <2 x double> [[X2]], i64 0
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast i8 [[TMP4]] to <8 x i1>
+; CHECK-NEXT:    [[TMP13:%.*]] = bitcast i8 [[X3:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[_MSPROP6:%.*]] = extractelement <8 x i1> [[TMP12]], i64 0
+; CHECK-NEXT:    [[TMP14:%.*]] = extractelement <8 x i1> [[TMP13]], i64 0
+; CHECK-NEXT:    [[TMP15:%.*]] = select i1 [[TMP14]], i64 [[_MSPROP4]], i64 [[_MSPROP5]]
+; CHECK-NEXT:    [[TMP16:%.*]] = bitcast double [[TMP10]] to i64
+; CHECK-NEXT:    [[TMP17:%.*]] = bitcast double [[TMP11]] to i64
+; CHECK-NEXT:    [[TMP18:%.*]] = xor i64 [[TMP16]], [[TMP17]]
+; CHECK-NEXT:    [[TMP19:%.*]] = or i64 [[TMP18]], [[_MSPROP4]]
+; CHECK-NEXT:    [[TMP20:%.*]] = or i64 [[TMP19]], [[_MSPROP5]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select i1 [[_MSPROP6]], i64 [[TMP20]], i64 [[TMP15]]
+; CHECK-NEXT:    [[TMP21:%.*]] = select i1 [[TMP14]], double [[TMP10]], double [[TMP11]]
+; CHECK-NEXT:    [[_MSPROP7:%.*]] = insertelement <2 x i64> [[TMP2]], i64 [[_MSPROP_SELECT]], i64 0
+; CHECK-NEXT:    [[TMP22:%.*]] = insertelement <2 x double> [[X2]], double [[TMP21]], i64 0
+; CHECK-NEXT:    [[TMP23:%.*]] = fneg <2 x double> [[X0]]
+; CHECK-NEXT:    [[TMP24:%.*]] = fneg <2 x double> [[X2]]
+; CHECK-NEXT:    [[_MSPROP8:%.*]] = extractelement <2 x i64> [[TMP1]], i64 0
+; CHECK-NEXT:    [[TMP25:%.*]] = extractelement <2 x double> [[TMP23]], i64 0
+; CHECK-NEXT:    [[_MSPROP9:%.*]] = extractelement <2 x i64> [[TMP3]], i64 0
+; CHECK-NEXT:    [[TMP26:%.*]] = extractelement <2 x double> [[X1]], i64 0
+; CHECK-NEXT:    [[_MSPROP10:%.*]] = extractelement <2 x i64> [[TMP2]], i64 0
+; CHECK-NEXT:    [[TMP27:%.*]] = extractelement <2 x double> [[TMP24]], i64 0
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[_MSPROP8]], 0
+; CHECK-NEXT:    [[_MSCMP22:%.*]] = icmp ne i64 [[_MSPROP9]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP22]]
+; CHECK-NEXT:    [[_MSCMP23:%.*]] = icmp ne i64 [[_MSPROP10]], 0
+; CHECK-NEXT:    [[_MSOR24:%.*]] = or i1 [[_MSOR]], [[_MSCMP23]]
+; CHECK-NEXT:    br i1 [[_MSOR24]], label [[TMP28:%.*]], label [[TMP29:%.*]], !prof [[PROF1]]
+; CHECK:       28:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       29:
+; CHECK-NEXT:    [[TMP30:%.*]] = call double @llvm.x86.avx512.vfmadd.f64(double [[TMP25]], double [[TMP26]], double [[TMP27]], i32 11)
+; CHECK-NEXT:    [[_MSPROP11:%.*]] = extractelement <2 x i64> [[TMP2]], i64 0
+; CHECK-NEXT:    [[TMP31:%.*]] = extractelement <2 x double> [[X2]], i64 0
+; CHECK-NEXT:    [[_MSPROP12:%.*]] = insertelement <2 x i64> [[TMP2]], i64 0, i64 0
+; CHECK-NEXT:    [[TMP32:%.*]] = insertelement <2 x double> [[X2]], double [[TMP30]], i64 0
+; CHECK-NEXT:    [[TMP33:%.*]] = fneg <2 x double> [[X0]]
+; CHECK-NEXT:    [[TMP34:%.*]] = fneg <2 x double> [[X2]]
+; CHECK-NEXT:    [[_MSPROP13:%.*]] = extractelement <2 x i64> [[TMP1]], i64 0
+; CHECK-NEXT:    [[TMP35:%.*]] = extractelement <2 x double> [[TMP33]], i64 0
+; CHECK-NEXT:    [[_MSPROP14:%.*]] = extractelement <2 x i64> [[TMP3]], i64 0
+; CHECK-NEXT:    [[TMP36:%.*]] = extractelement <2 x double> [[X1]], i64 0
+; CHECK-NEXT:    [[_MSPROP15:%.*]] = extractelement <2 x i64> [[TMP2]], i64 0
+; CHECK-NEXT:    [[TMP37:%.*]] = extractelement <2 x double> [[TMP34]], i64 0
+; CHECK-NEXT:    [[_MSCMP25:%.*]] = icmp ne i64 [[_MSPROP13]], 0
+; CHECK-NEXT:    [[_MSCMP26:%.*]] = icmp ne i64 [[_MSPROP14]], 0
+; CHECK-NEXT:    [[_MSOR27:%.*]] = or i1 [[_MSCMP25]], [[_MSCMP26]]
+; CHECK-NEXT:    [[_MSCMP28:%.*]] = icmp ne i64 [[_MSPROP15]], 0
+; CHECK-NEXT:    [[_MSOR29:%.*]] = or i1 [[_MSOR27]], [[_MSCMP28]]
+; CHECK-NEXT:    br i1 [[_MSOR29]], label [[TMP38:%.*]], label [[TMP39:%.*]], !prof [[PROF1]]
+; CHECK:       38:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       39:
+; CHECK-NEXT:    [[TMP40:%.*]] = call double @llvm.x86.avx512.vfmadd.f64(double [[TMP35]], double [[TMP36]], double [[TMP37]], i32 10)
+; CHECK-NEXT:    [[_MSPROP16:%.*]] = extractelement <2 x i64> [[TMP2]], i64 0
+; CHECK-NEXT:    [[TMP41:%.*]] = extractelement <2 x double> [[X2]], i64 0
+; CHECK-NEXT:    [[TMP42:%.*]] = bitcast i8 [[TMP4]] to <8 x i1>
+; CHECK-NEXT:    [[TMP43:%.*]] = bitcast i8 [[X3]] to <8 x i1>
+; CHECK-NEXT:    [[_MSPROP17:%.*]] = extractelement <8 x i1> [[TMP42]], i64 0
+; CHECK-NEXT:    [[TMP44:%.*]] = extractelement <8 x i1> [[TMP43]], i64 0
+; CHECK-NEXT:    [[TMP45:%.*]] = select i1 [[TMP44]], i64 0, i64 [[_MSPROP16]]
+; CHECK-NEXT:    [[TMP46:%.*]] = bitcast double [[TMP40]] to i64
+; CHECK-NEXT:    [[TMP47:%.*]] = bitcast double [[TMP41]] to i64
+; CHECK-NEXT:    [[TMP48:%.*]] = xor i64 [[TMP46]], [[TMP47]]
+; CHECK-NEXT:    [[TMP49:%.*]] = or i64 [[TMP48]], 0
+; CHECK-NEXT:    [[TMP50:%.*]] = or i64 [[TMP49]], [[_MSPROP16]]
+; CHECK-NEXT:    [[_MSPROP_SELECT18:%.*]] = select i1 [[_MSPROP17]], i64 [[TMP50]], i64 [[TMP45]]
+; CHECK-NEXT:    [[TMP51:%.*]] = select i1 [[TMP44]], double [[TMP40]], double [[TMP41]]
+; CHECK-NEXT:    [[_MSPROP19:%.*]] = insertelement <2 x i64> [[TMP2]], i64 [[_MSPROP_SELECT18]], i64 0
+; CHECK-NEXT:    [[TMP52:%.*]] = insertelement <2 x double> [[X2]], double [[TMP51]], i64 0
+; CHECK-NEXT:    [[_MSPROP20:%.*]] = or <2 x i64> [[_MSPROP7]], [[_MSPROP12]]
+; CHECK-NEXT:    [[RES3:%.*]] = fadd <2 x double> [[TMP22]], [[TMP32]]
+; CHECK-NEXT:    [[_MSPROP21:%.*]] = or <2 x i64> [[_MSPROP19]], [[_MSPROP20]]
+; CHECK-NEXT:    [[RES4:%.*]] = fadd <2 x double> [[TMP52]], [[RES3]]
+; CHECK-NEXT:    store <2 x i64> [[_MSPROP21]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x double> [[RES4]]
+;
+  %res = call <2 x double> @llvm.x86.avx512.mask3.vfnmsub.sd(<2 x double> %x0, <2 x double> %x1, <2 x double> %x2, i8 %x3, i32 4)
+  %res1 = call <2 x double> @llvm.x86.avx512.mask3.vfnmsub.sd(<2 x double> %x0, <2 x double> %x1, <2 x double> %x2, i8 -1, i32 11)
+  %res2 = call <2 x double> @llvm.x86.avx512.mask3.vfnmsub.sd(<2 x double> %x0, <2 x double> %x1, <2 x double> %x2, i8 %x3, i32 10)
+  %res3 = fadd <2 x double> %res, %res1
+  %res4 = fadd <2 x double> %res2, %res3
+  ret <2 x double> %res4
+}
+
+declare <4 x float> @llvm.x86.avx512.mask3.vfnmsub.ss(<4 x float>, <4 x float>, <4 x float>, i8, i32)
+
+define <4 x float>@test_int_x86_avx512_mask3_vfnmsub_ss(<4 x float> %x0, <4 x float> %x1, <4 x float> %x2, i8 %x3,i32 %x4 ) #0 {
+;
+; CHECK-LABEL: @test_int_x86_avx512_mask3_vfnmsub_ss(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 48) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = fneg <4 x float> [[X0:%.*]]
+; CHECK-NEXT:    [[TMP6:%.*]] = fneg <4 x float> [[X2:%.*]]
+; CHECK-NEXT:    [[_MSPROP:%.*]] = extractelement <4 x i32> [[TMP1]], i64 0
+; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <4 x float> [[TMP5]], i64 0
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = extractelement <4 x i32> [[TMP3]], i64 0
+; CHECK-NEXT:    [[TMP8:%.*]] = extractelement <4 x float> [[X1:%.*]], i64 0
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = extractelement <4 x i32> [[TMP2]], i64 0
+; CHECK-NEXT:    [[TMP9:%.*]] = extractelement <4 x float> [[TMP6]], i64 0
+; CHECK-NEXT:    [[_MSPROP3:%.*]] = or i32 [[_MSPROP]], [[_MSPROP1]]
+; CHECK-NEXT:    [[_MSPROP4:%.*]] = or i32 [[_MSPROP3]], [[_MSPROP2]]
+; CHECK-NEXT:    [[TMP10:%.*]] = call float @llvm.fma.f32(float [[TMP7]], float [[TMP8]], float [[TMP9]])
+; CHECK-NEXT:    [[_MSPROP5:%.*]] = extractelement <4 x i32> [[TMP2]], i64 0
+; CHECK-NEXT:    [[TMP11:%.*]] = extractelement <4 x float> [[X2]], i64 0
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast i8 [[TMP4]] to <8 x i1>
+; CHECK-NEXT:    [[TMP13:%.*]] = bitcast i8 [[X3:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[_MSPROP6:%.*]] = extractelement <8 x i1> [[TMP12]], i64 0
+; CHECK-NEXT:    [[TMP14:%.*]] = extractelement <8 x i1> [[TMP13]], i64 0
+; CHECK-NEXT:    [[TMP15:%.*]] = select i1 [[TMP14]], i32 [[_MSPROP4]], i32 [[_MSPROP5]]
+; CHECK-NEXT:    [[TMP16:%.*]] = bitcast float [[TMP10]] to i32
+; CHECK-NEXT:    [[TMP17:%.*]] = bitcast float [[TMP11]] to i32
+; CHECK-NEXT:    [[TMP18:%.*]] = xor i32 [[TMP16]], [[TMP17]]
+; CHECK-NEXT:    [[TMP19:%.*]] = or i32 [[TMP18]], [[_MSPROP4]]
+; CHECK-NEXT:    [[TMP20:%.*]] = or i32 [[TMP19]], [[_MSPROP5]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select i1 [[_MSPROP6]], i32 [[TMP20]], i32 [[TMP15]]
+; CHECK-NEXT:    [[TMP21:%.*]] = select i1 [[TMP14]], float [[TMP10]], float [[TMP11]]
+; CHECK-NEXT:    [[_MSPROP7:%.*]] = insertelement <4 x i32> [[TMP2]], i32 [[_MSPROP_SELECT]], i64 0
+; CHECK-NEXT:    [[TMP22:%.*]] = insertelement <4 x float> [[X2]], float [[TMP21]], i64 0
+; CHECK-NEXT:    [[TMP23:%.*]] = fneg <4 x float> [[X0]]
+; CHECK-NEXT:    [[TMP24:%.*]] = fneg <4 x float> [[X2]]
+; CHECK-NEXT:    [[_MSPROP8:%.*]] = extractelement <4 x i32> [[TMP1]], i64 0
+; CHECK-NEXT:    [[TMP25:%.*]] = extractelement <4 x float> [[TMP23]], i64 0
+; CHECK-NEXT:    [[_MSPROP9:%.*]] = extractelement <4 x i32> [[TMP3]], i64 0
+; CHECK-NEXT:    [[TMP26:%.*]] = extractelement <4 x float> [[X1]], i64 0
+; CHECK-NEXT:    [[_MSPROP10:%.*]] = extractelement <4 x i32> [[TMP2]], i64 0
+; CHECK-NEXT:    [[TMP27:%.*]] = extractelement <4 x float> [[TMP24]], i64 0
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i32 [[_MSPROP8]], 0
+; CHECK-NEXT:    [[_MSCMP22:%.*]] = icmp ne i32 [[_MSPROP9]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP22]]
+; CHECK-NEXT:    [[_MSCMP23:%.*]] = icmp ne i32 [[_MSPROP10]], 0
+; CHECK-NEXT:    [[_MSOR24:%.*]] = or i1 [[_MSOR]], [[_MSCMP23]]
+; CHECK-NEXT:    br i1 [[_MSOR24]], label [[TMP28:%.*]], label [[TMP29:%.*]], !prof [[PROF1]]
+; CHECK:       28:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       29:
+; CHECK-NEXT:    [[TMP30:%.*]] = call float @llvm.x86.avx512.vfmadd.f32(float [[TMP25]], float [[TMP26]], float [[TMP27]], i32 11)
+; CHECK-NEXT:    [[_MSPROP11:%.*]] = extractelement <4 x i32> [[TMP2]], i64 0
+; CHECK-NEXT:    [[TMP31:%.*]] = extractelement <4 x float> [[X2]], i64 0
+; CHECK-NEXT:    [[_MSPROP12:%.*]] = insertelement <4 x i32> [[TMP2]], i32 0, i64 0
+; CHECK-NEXT:    [[TMP32:%.*]] = insertelement <4 x float> [[X2]], float [[TMP30]], i64 0
+; CHECK-NEXT:    [[TMP33:%.*]] = fneg <4 x float> [[X0]]
+; CHECK-NEXT:    [[TMP34:%.*]] = fneg <4 x float> [[X2]]
+; CHECK-NEXT:    [[_MSPROP13:%.*]] = extractelement <4 x i32> [[TMP1]], i64 0
+; CHECK-NEXT:    [[TMP35:%.*]] = extractelement <4 x float> [[TMP33]], i64 0
+; CHECK-NEXT:    [[_MSPROP14:%.*]] = extractelement <4 x i32> [[TMP3]], i64 0
+; CHECK-NEXT:    [[TMP36:%.*]] = extractelement <4 x float> [[X1]], i64 0
+; CHECK-NEXT:    [[_MSPROP15:%.*]] = extractelement <4 x i32> [[TMP2]], i64 0
+; CHECK-NEXT:    [[TMP37:%.*]] = extractelement <4 x float> [[TMP34]], i64 0
+; CHECK-NEXT:    [[_MSCMP25:%.*]] = icmp ne i32 [[_MSPROP13]], 0
+; CHECK-NEXT:    [[_MSCMP26:%.*]] = icmp ne i32 [[_MSPROP14]], 0
+; CHECK-NEXT:    [[_MSOR27:%.*]] = or i1 [[_MSCMP25]], [[_MSCMP26]]
+; CHECK-NEXT:    [[_MSCMP28:%.*]] = icmp ne i32 [[_MSPROP15]], 0
+; CHECK-NEXT:    [[_MSOR29:%.*]] = or i1 [[_MSOR27]], [[_MSCMP28]]
+; CHECK-NEXT:    br i1 [[_MSOR29]], label [[TMP38:%.*]], label [[TMP39:%.*]], !prof [[PROF1]]
+; CHECK:       38:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       39:
+; CHECK-NEXT:    [[TMP40:%.*]] = call float @llvm.x86.avx512.vfmadd.f32(float [[TMP35]], float [[TMP36]], float [[TMP37]], i32 10)
+; CHECK-NEXT:    [[_MSPROP16:%.*]] = extractelement <4 x i32> [[TMP2]], i64 0
+; CHECK-NEXT:    [[TMP41:%.*]] = extractelement <4 x float> [[X2]], i64 0
+; CHECK-NEXT:    [[TMP42:%.*]] = bitcast i8 [[TMP4]] to <8 x i1>
+; CHECK-NEXT:    [[TMP43:%.*]] = bitcast i8 [[X3]] to <8 x i1>
+; CHECK-NEXT:    [[_MSPROP17:%.*]] = extractelement <8 x i1> [[TMP42]], i64 0
+; CHECK-NEXT:    [[TMP44:%.*]] = extractelement <8 x i1> [[TMP43]], i64 0
+; CHECK-NEXT:    [[TMP45:%.*]] = select i1 [[TMP44]], i32 0, i32 [[_MSPROP16]]
+; CHECK-NEXT:    [[TMP46:%.*]] = bitcast float [[TMP40]] to i32
+; CHECK-NEXT:    [[TMP47:%.*]] = bitcast float [[TMP41]] to i32
+; CHECK-NEXT:    [[TMP48:%.*]] = xor i32 [[TMP46]], [[TMP47]]
+; CHECK-NEXT:    [[TMP49:%.*]] = or i32 [[TMP48]], 0
+; CHECK-NEXT:    [[TMP50:%.*]] = or i32 [[TMP49]], [[_MSPROP16]]
+; CHECK-NEXT:    [[_MSPROP_SELECT18:%.*]] = select i1 [[_MSPROP17]], i32 [[TMP50]], i32 [[TMP45]]
+; CHECK-NEXT:    [[TMP51:%.*]] = select i1 [[TMP44]], float [[TMP40]], float [[TMP41]]
+; CHECK-NEXT:    [[_MSPROP19:%.*]] = insertelement <4 x i32> [[TMP2]], i32 [[_MSPROP_SELECT18]], i64 0
+; CHECK-NEXT:    [[TMP52:%.*]] = insertelement <4 x float> [[X2]], float [[TMP51]], i64 0
+; CHECK-NEXT:    [[_MSPROP20:%.*]] = or <4 x i32> [[_MSPROP7]], [[_MSPROP12]]
+; CHECK-NEXT:    [[RES3:%.*]] = fadd <4 x float> [[TMP22]], [[TMP32]]
+; CHECK-NEXT:    [[_MSPROP21:%.*]] = or <4 x i32> [[_MSPROP19]], [[_MSPROP20]]
+; CHECK-NEXT:    [[RES4:%.*]] = fadd <4 x float> [[TMP52]], [[RES3]]
+; CHECK-NEXT:    store <4 x i32> [[_MSPROP21]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x float> [[RES4]]
+;
+  %res = call <4 x float> @llvm.x86.avx512.mask3.vfnmsub.ss(<4 x float> %x0, <4 x float> %x1, <4 x float> %x2, i8 %x3, i32 4)
+  %res1 = call <4 x float> @llvm.x86.avx512.mask3.vfnmsub.ss(<4 x float> %x0, <4 x float> %x1, <4 x float> %x2, i8 -1, i32 11)
+  %res2 = call <4 x float> @llvm.x86.avx512.mask3.vfnmsub.ss(<4 x float> %x0, <4 x float> %x1, <4 x float> %x2, i8 %x3, i32 10)
+  %res3 = fadd <4 x float> %res, %res1
+  %res4 = fadd <4 x float> %res2, %res3
+  ret <4 x float> %res4
+}
+
+define <4 x float>@test_int_x86_avx512_mask3_vfmadd_ss_rm(<4 x float> %x0, <4 x float> %x1, ptr%ptr_b ,i8 %x3,i32 %x4, <4 x float> %extra_param)  #0 {
+;
+; CHECK-LABEL: @test_int_x86_avx512_mask3_vfmadd_ss_rm(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP5:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 56) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 40) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP6:%.*]], label [[TMP25:%.*]], !prof [[PROF1]]
+; CHECK:       6:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       7:
+; CHECK-NEXT:    [[Q:%.*]] = load float, ptr [[PTR_B:%.*]], align 4
+; CHECK-NEXT:    [[TMP7:%.*]] = ptrtoint ptr [[PTR_B]] to i64
+; CHECK-NEXT:    [[TMP8:%.*]] = xor i64 [[TMP7]], 87960930222080
+; CHECK-NEXT:    [[TMP9:%.*]] = inttoptr i64 [[TMP8]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load i32, ptr [[TMP9]], align 4
+; CHECK-NEXT:    [[_MSPROP:%.*]] = insertelement <4 x i32> [[TMP5]], i32 [[_MSLD]], i32 0
+; CHECK-NEXT:    [[VECINIT_I:%.*]] = insertelement <4 x float> [[EXTRA_PARAM:%.*]], float [[Q]], i32 0
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = extractelement <4 x i32> [[TMP2]], i64 0
+; CHECK-NEXT:    [[TMP10:%.*]] = extractelement <4 x float> [[X0:%.*]], i64 0
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = extractelement <4 x i32> [[_MSPROP]], i64 0
+; CHECK-NEXT:    [[TMP11:%.*]] = extractelement <4 x float> [[VECINIT_I]], i64 0
+; CHECK-NEXT:    [[_MSPROP3:%.*]] = extractelement <4 x i32> [[TMP3]], i64 0
+; CHECK-NEXT:    [[TMP12:%.*]] = extractelement <4 x float> [[X1:%.*]], i64 0
+; CHECK-NEXT:    [[_MSPROP4:%.*]] = or i32 [[_MSPROP1]], [[_MSPROP2]]
+; CHECK-NEXT:    [[_MSPROP5:%.*]] = or i32 [[_MSPROP4]], [[_MSPROP3]]
+; CHECK-NEXT:    [[TMP13:%.*]] = call float @llvm.fma.f32(float [[TMP10]], float [[TMP11]], float [[TMP12]])
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast i8 [[TMP4]] to <8 x i1>
+; CHECK-NEXT:    [[TMP15:%.*]] = bitcast i8 [[X3:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[_MSPROP6:%.*]] = extractelement <8 x i1> [[TMP14]], i64 0
+; CHECK-NEXT:    [[TMP16:%.*]] = extractelement <8 x i1> [[TMP15]], i64 0
+; CHECK-NEXT:    [[TMP17:%.*]] = select i1 [[TMP16]], i32 [[_MSPROP5]], i32 [[_MSPROP3]]
+; CHECK-NEXT:    [[TMP18:%.*]] = bitcast float [[TMP13]] to i32
+; CHECK-NEXT:    [[TMP19:%.*]] = bitcast float [[TMP12]] to i32
+; CHECK-NEXT:    [[TMP20:%.*]] = xor i32 [[TMP18]], [[TMP19]]
+; CHECK-NEXT:    [[TMP21:%.*]] = or i32 [[TMP20]], [[_MSPROP5]]
+; CHECK-NEXT:    [[TMP22:%.*]] = or i32 [[TMP21]], [[_MSPROP3]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select i1 [[_MSPROP6]], i32 [[TMP22]], i32 [[TMP17]]
+; CHECK-NEXT:    [[TMP23:%.*]] = select i1 [[TMP16]], float [[TMP13]], float [[TMP12]]
+; CHECK-NEXT:    [[_MSPROP7:%.*]] = insertelement <4 x i32> [[TMP3]], i32 [[_MSPROP_SELECT]], i64 0
+; CHECK-NEXT:    [[TMP24:%.*]] = insertelement <4 x float> [[X1]], float [[TMP23]], i64 0
+; CHECK-NEXT:    store <4 x i32> [[_MSPROP7]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x float> [[TMP24]]
+;
+  %q = load float, ptr %ptr_b
+  %vecinit.i = insertelement <4 x float> %extra_param, float %q, i32 0
+  %res = call <4 x float> @llvm.x86.avx512.mask3.vfmadd.ss(<4 x float> %x0, <4 x float> %vecinit.i, <4 x float> %x1, i8 %x3, i32 4)
+  ret < 4 x float> %res
+}
+
+define <4 x float>@test_int_x86_avx512_mask_vfmadd_ss_rm(<4 x float> %x0, <4 x float> %x1,ptr%ptr_b ,i8 %x3,i32 %x4, <4 x float> %extra_param)  #0 {
+;
+; CHECK-LABEL: @test_int_x86_avx512_mask_vfmadd_ss_rm(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP5:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 56) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 40) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP6:%.*]], label [[TMP25:%.*]], !prof [[PROF1]]
+; CHECK:       6:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       7:
+; CHECK-NEXT:    [[Q:%.*]] = load float, ptr [[PTR_B:%.*]], align 4
+; CHECK-NEXT:    [[TMP7:%.*]] = ptrtoint ptr [[PTR_B]] to i64
+; CHECK-NEXT:    [[TMP8:%.*]] = xor i64 [[TMP7]], 87960930222080
+; CHECK-NEXT:    [[TMP9:%.*]] = inttoptr i64 [[TMP8]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load i32, ptr [[TMP9]], align 4
+; CHECK-NEXT:    [[_MSPROP:%.*]] = insertelement <4 x i32> [[TMP5]], i32 [[_MSLD]], i32 0
+; CHECK-NEXT:    [[VECINIT_I:%.*]] = insertelement <4 x float> [[EXTRA_PARAM:%.*]], float [[Q]], i32 0
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = extractelement <4 x i32> [[TMP2]], i64 0
+; CHECK-NEXT:    [[TMP10:%.*]] = extractelement <4 x float> [[X0:%.*]], i64 0
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = extractelement <4 x i32> [[_MSPROP]], i64 0
+; CHECK-NEXT:    [[TMP11:%.*]] = extractelement <4 x float> [[VECINIT_I]], i64 0
+; CHECK-NEXT:    [[_MSPROP3:%.*]] = extractelement <4 x i32> [[TMP3]], i64 0
+; CHECK-NEXT:    [[TMP12:%.*]] = extractelement <4 x float> [[X1:%.*]], i64 0
+; CHECK-NEXT:    [[_MSPROP4:%.*]] = or i32 [[_MSPROP1]], [[_MSPROP2]]
+; CHECK-NEXT:    [[_MSPROP5:%.*]] = or i32 [[_MSPROP4]], [[_MSPROP3]]
+; CHECK-NEXT:    [[TMP13:%.*]] = call float @llvm.fma.f32(float [[TMP10]], float [[TMP11]], float [[TMP12]])
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast i8 [[TMP4]] to <8 x i1>
+; CHECK-NEXT:    [[TMP15:%.*]] = bitcast i8 [[X3:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[_MSPROP6:%.*]] = extractelement <8 x i1> [[TMP14]], i64 0
+; CHECK-NEXT:    [[TMP16:%.*]] = extractelement <8 x i1> [[TMP15]], i64 0
+; CHECK-NEXT:    [[TMP17:%.*]] = select i1 [[TMP16]], i32 [[_MSPROP5]], i32 [[_MSPROP1]]
+; CHECK-NEXT:    [[TMP18:%.*]] = bitcast float [[TMP13]] to i32
+; CHECK-NEXT:    [[TMP19:%.*]] = bitcast float [[TMP10]] to i32
+; CHECK-NEXT:    [[TMP20:%.*]] = xor i32 [[TMP18]], [[TMP19]]
+; CHECK-NEXT:    [[TMP21:%.*]] = or i32 [[TMP20]], [[_MSPROP5]]
+; CHECK-NEXT:    [[TMP22:%.*]] = or i32 [[TMP21]], [[_MSPROP1]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select i1 [[_MSPROP6]], i32 [[TMP22]], i32 [[TMP17]]
+; CHECK-NEXT:    [[TMP23:%.*]] = select i1 [[TMP16]], float [[TMP13]], float [[TMP10]]
+; CHECK-NEXT:    [[_MSPROP7:%.*]] = insertelement <4 x i32> [[TMP2]], i32 [[_MSPROP_SELECT]], i64 0
+; CHECK-NEXT:    [[TMP24:%.*]] = insertelement <4 x float> [[X0]], float [[TMP23]], i64 0
+; CHECK-NEXT:    store <4 x i32> [[_MSPROP7]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x float> [[TMP24]]
+;
+  %q = load float, ptr %ptr_b
+  %vecinit.i = insertelement <4 x float> %extra_param, float %q, i32 0
+  %res = call <4 x float> @llvm.x86.avx512.mask.vfmadd.ss(<4 x float> %x0,<4 x float> %vecinit.i, <4 x float> %x1,  i8 %x3, i32 4)
+  ret < 4 x float> %res
+}
+
+
+define <4 x float>@test_int_x86_avx512_maskz_vfmadd_ss_rm(<4 x float> %x0, <4 x float> %x1,ptr%ptr_b ,i8 %x3,i32 %x4, <4 x float> %extra_param)  #0 {
+; CHECK-LABEL: @test_int_x86_avx512_maskz_vfmadd_ss_rm(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 56) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP5:%.*]], label [[TMP20:%.*]], !prof [[PROF1]]
+; CHECK:       5:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       6:
+; CHECK-NEXT:    [[Q:%.*]] = load float, ptr [[PTR_B:%.*]], align 4
+; CHECK-NEXT:    [[TMP6:%.*]] = ptrtoint ptr [[PTR_B]] to i64
+; CHECK-NEXT:    [[TMP7:%.*]] = xor i64 [[TMP6]], 87960930222080
+; CHECK-NEXT:    [[TMP8:%.*]] = inttoptr i64 [[TMP7]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load i32, ptr [[TMP8]], align 4
+; CHECK-NEXT:    [[_MSPROP:%.*]] = insertelement <4 x i32> [[TMP4]], i32 [[_MSLD]], i32 0
+; CHECK-NEXT:    [[VECINIT_I:%.*]] = insertelement <4 x float> [[EXTRA_PARAM:%.*]], float [[Q]], i32 0
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = extractelement <4 x i32> [[TMP2]], i64 0
+; CHECK-NEXT:    [[TMP9:%.*]] = extractelement <4 x float> [[X0:%.*]], i64 0
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = extractelement <4 x i32> [[TMP3]], i64 0
+; CHECK-NEXT:    [[TMP10:%.*]] = extractelement <4 x float> [[X1:%.*]], i64 0
+; CHECK-NEXT:    [[_MSPROP3:%.*]] = extractelement <4 x i32> [[_MSPROP]], i64 0
+; CHECK-NEXT:    [[TMP11:%.*]] = extractelement <4 x float> [[VECINIT_I]], i64 0
+; CHECK-NEXT:    [[_MSPROP4:%.*]] = or i32 [[_MSPROP1]], [[_MSPROP2]]
+; CHECK-NEXT:    [[_MSPROP5:%.*]] = or i32 [[_MSPROP4]], [[_MSPROP3]]
+; CHECK-NEXT:    [[TMP12:%.*]] = call float @llvm.fma.f32(float [[TMP9]], float [[TMP10]], float [[TMP11]])
+; CHECK-NEXT:    [[TMP13:%.*]] = select i1 false, i32 [[_MSPROP5]], i32 0
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast float [[TMP12]] to i32
+; CHECK-NEXT:    [[TMP15:%.*]] = xor i32 [[TMP14]], 0
+; CHECK-NEXT:    [[TMP16:%.*]] = or i32 [[TMP15]], [[_MSPROP5]]
+; CHECK-NEXT:    [[TMP17:%.*]] = or i32 [[TMP16]], 0
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select i1 false, i32 [[TMP17]], i32 [[TMP13]]
+; CHECK-NEXT:    [[TMP18:%.*]] = select i1 false, float [[TMP12]], float 0.000000e+00
+; CHECK-NEXT:    [[_MSPROP6:%.*]] = insertelement <4 x i32> [[TMP2]], i32 [[_MSPROP_SELECT]], i64 0
+; CHECK-NEXT:    [[TMP19:%.*]] = insertelement <4 x float> [[X0]], float [[TMP18]], i64 0
+; CHECK-NEXT:    store <4 x i32> [[_MSPROP6]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x float> [[TMP19]]
+;
+  %q = load float, ptr %ptr_b
+  %vecinit.i = insertelement <4 x float> %extra_param, float %q, i32 0
+  %res = call <4 x float> @llvm.x86.avx512.maskz.vfmadd.ss(<4 x float> %x0, <4 x float> %x1, <4 x float> %vecinit.i, i8 0, i32 4)
+  ret < 4 x float> %res
+}
+
+declare <8 x i32> @llvm.x86.avx512.mask.pmov.qd.512(<8 x i64>, <8 x i32>, i8)
+
+define <8 x i32>@test_int_x86_avx512_pmov_qd_512(<8 x i64> %x0, <8 x i32> %x1)  #0 {
+; CHECK-LABEL: @test_int_x86_avx512_pmov_qd_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = trunc <8 x i64> [[TMP1]] to <8 x i32>
+; CHECK-NEXT:    [[TMP2:%.*]] = trunc <8 x i64> [[X0:%.*]] to <8 x i32>
+; CHECK-NEXT:    store <8 x i32> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i32> [[TMP2]]
+;
+  %res = call <8 x i32> @llvm.x86.avx512.mask.pmov.qd.512(<8 x i64> %x0, <8 x i32> %x1, i8 -1)
+  ret <8 x i32> %res
+}
+
+define <8 x i32>@test_int_x86_avx512_mask_pmov_qd_512(<8 x i64> %x0, <8 x i32> %x1, i8 %x2)  #0 {
+;
+; CHECK-LABEL: @test_int_x86_avx512_mask_pmov_qd_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 96) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = trunc <8 x i64> [[TMP1]] to <8 x i32>
+; CHECK-NEXT:    [[TMP4:%.*]] = trunc <8 x i64> [[X0:%.*]] to <8 x i32>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i8 [[TMP2]] to <8 x i1>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i8 [[X2:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP7:%.*]] = select <8 x i1> [[TMP6]], <8 x i32> [[_MSPROP]], <8 x i32> [[TMP3]]
+; CHECK-NEXT:    [[TMP8:%.*]] = xor <8 x i32> [[TMP4]], [[X1:%.*]]
+; CHECK-NEXT:    [[TMP9:%.*]] = or <8 x i32> [[TMP8]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP10:%.*]] = or <8 x i32> [[TMP9]], [[TMP3]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP5]], <8 x i32> [[TMP10]], <8 x i32> [[TMP7]]
+; CHECK-NEXT:    [[TMP11:%.*]] = select <8 x i1> [[TMP6]], <8 x i32> [[TMP4]], <8 x i32> [[X1]]
+; CHECK-NEXT:    store <8 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i32> [[TMP11]]
+;
+  %res = call <8 x i32> @llvm.x86.avx512.mask.pmov.qd.512(<8 x i64> %x0, <8 x i32> %x1, i8 %x2)
+  ret <8 x i32> %res
+}
+
+define <8 x i32>@test_int_x86_avx512_maskz_pmov_qd_512(<8 x i64> %x0, i8 %x2)  #0 {
+;
+; CHECK-LABEL: @test_int_x86_avx512_maskz_pmov_qd_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = trunc <8 x i64> [[TMP1]] to <8 x i32>
+; CHECK-NEXT:    [[TMP3:%.*]] = trunc <8 x i64> [[X0:%.*]] to <8 x i32>
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast i8 [[TMP2]] to <8 x i1>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i8 [[X2:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP6:%.*]] = select <8 x i1> [[TMP5]], <8 x i32> [[_MSPROP]], <8 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP7:%.*]] = xor <8 x i32> [[TMP3]], zeroinitializer
+; CHECK-NEXT:    [[TMP8:%.*]] = or <8 x i32> [[TMP7]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP9:%.*]] = or <8 x i32> [[TMP8]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP4]], <8 x i32> [[TMP9]], <8 x i32> [[TMP6]]
+; CHECK-NEXT:    [[TMP10:%.*]] = select <8 x i1> [[TMP5]], <8 x i32> [[TMP3]], <8 x i32> zeroinitializer
+; CHECK-NEXT:    store <8 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i32> [[TMP10]]
+;
+  %res = call <8 x i32> @llvm.x86.avx512.mask.pmov.qd.512(<8 x i64> %x0, <8 x i32> zeroinitializer, i8 %x2)
+  ret <8 x i32> %res
+}
+
+declare <16 x float> @llvm.x86.avx512.mask.cvtdq2ps.512(<16 x i32>, <16 x float>, i16, i32)
+
+define <16 x float> @test_int_x86_avx512_mask_cvt_dq2ps_512(<16 x i32> %x0, <16 x float> %x1, i16 %x2)  #0 {
+;
+; CHECK-LABEL: @test_int_x86_avx512_mask_cvt_dq2ps_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[CVT:%.*]] = sitofp <16 x i32> [[X0:%.*]] to <16 x float>
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast i16 [[TMP2]] to <16 x i1>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i16 [[X2:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP6:%.*]] = select <16 x i1> [[TMP5]], <16 x i32> [[TMP1]], <16 x i32> [[TMP3]]
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <16 x float> [[CVT]] to <16 x i32>
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <16 x float> [[X1:%.*]] to <16 x i32>
+; CHECK-NEXT:    [[TMP9:%.*]] = xor <16 x i32> [[TMP7]], [[TMP8]]
+; CHECK-NEXT:    [[TMP10:%.*]] = or <16 x i32> [[TMP9]], [[TMP1]]
+; CHECK-NEXT:    [[TMP11:%.*]] = or <16 x i32> [[TMP10]], [[TMP3]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP4]], <16 x i32> [[TMP11]], <16 x i32> [[TMP6]]
+; CHECK-NEXT:    [[TMP12:%.*]] = select <16 x i1> [[TMP5]], <16 x float> [[CVT]], <16 x float> [[X1]]
+; CHECK-NEXT:    [[TMP13:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP13]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP14:%.*]], label [[TMP15:%.*]], !prof [[PROF1]]
+; CHECK:       14:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       15:
+; CHECK-NEXT:    [[TMP16:%.*]] = call <16 x float> @llvm.x86.avx512.sitofp.round.v16f32.v16i32(<16 x i32> [[X0]], i32 8)
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <16 x i32> [[_MSPROP_SELECT]], zeroinitializer
+; CHECK-NEXT:    [[RES2:%.*]] = fadd <16 x float> [[TMP12]], [[TMP16]]
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x float> [[RES2]]
+;
+  %res = call <16 x float> @llvm.x86.avx512.mask.cvtdq2ps.512(<16 x i32> %x0, <16 x float> %x1, i16 %x2, i32 4)
+  %res1 = call <16 x float> @llvm.x86.avx512.mask.cvtdq2ps.512(<16 x i32> %x0, <16 x float> %x1, i16 -1, i32 8)
+  %res2 = fadd <16 x float> %res, %res1
+  ret <16 x float> %res2
+}
+
+declare <16 x float> @llvm.x86.avx512.mask.cvtudq2ps.512(<16 x i32>, <16 x float>, i16, i32)
+
+define <16 x float> @test_int_x86_avx512_mask_cvt_udq2ps_512(<16 x i32> %x0, <16 x float> %x1, i16 %x2)  #0 {
+;
+; CHECK-LABEL: @test_int_x86_avx512_mask_cvt_udq2ps_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[CVT:%.*]] = uitofp <16 x i32> [[X0:%.*]] to <16 x float>
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast i16 [[TMP2]] to <16 x i1>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i16 [[X2:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP6:%.*]] = select <16 x i1> [[TMP5]], <16 x i32> [[TMP1]], <16 x i32> [[TMP3]]
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <16 x float> [[CVT]] to <16 x i32>
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <16 x float> [[X1:%.*]] to <16 x i32>
+; CHECK-NEXT:    [[TMP9:%.*]] = xor <16 x i32> [[TMP7]], [[TMP8]]
+; CHECK-NEXT:    [[TMP10:%.*]] = or <16 x i32> [[TMP9]], [[TMP1]]
+; CHECK-NEXT:    [[TMP11:%.*]] = or <16 x i32> [[TMP10]], [[TMP3]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP4]], <16 x i32> [[TMP11]], <16 x i32> [[TMP6]]
+; CHECK-NEXT:    [[TMP12:%.*]] = select <16 x i1> [[TMP5]], <16 x float> [[CVT]], <16 x float> [[X1]]
+; CHECK-NEXT:    [[TMP13:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP13]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP14:%.*]], label [[TMP15:%.*]], !prof [[PROF1]]
+; CHECK:       14:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       15:
+; CHECK-NEXT:    [[TMP16:%.*]] = call <16 x float> @llvm.x86.avx512.uitofp.round.v16f32.v16i32(<16 x i32> [[X0]], i32 8)
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <16 x i32> [[_MSPROP_SELECT]], zeroinitializer
+; CHECK-NEXT:    [[RES2:%.*]] = fadd <16 x float> [[TMP12]], [[TMP16]]
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x float> [[RES2]]
+;
+  %res = call <16 x float> @llvm.x86.avx512.mask.cvtudq2ps.512(<16 x i32> %x0, <16 x float> %x1, i16 %x2, i32 4)
+  %res1 = call <16 x float> @llvm.x86.avx512.mask.cvtudq2ps.512(<16 x i32> %x0, <16 x float> %x1, i16 -1, i32 8)
+  %res2 = fadd <16 x float> %res, %res1
+  ret <16 x float> %res2
+}
+
+define <8 x double> @test_mask_compress_pd_512(<8 x double> %data, <8 x double> %passthru, i8 %mask)  #0 {
+;
+; CHECK-LABEL: @test_mask_compress_pd_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast i8 [[TMP1]] to <8 x i1>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <8 x i64> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP6]], 0
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <8 x i64> [[TMP3]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP7]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <8 x i1> [[TMP4]] to i8
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i8 [[TMP8]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR3]], label [[TMP9:%.*]], label [[TMP10:%.*]], !prof [[PROF1]]
+; CHECK:       9:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       10:
+; CHECK-NEXT:    [[TMP11:%.*]] = call <8 x double> @llvm.x86.avx512.mask.compress.v8f64(<8 x double> [[DATA:%.*]], <8 x double> [[PASSTHRU:%.*]], <8 x i1> [[TMP5]])
+; CHECK-NEXT:    store <8 x i64> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x double> [[TMP11]]
+;
+  %res = call <8 x double> @llvm.x86.avx512.mask.compress.pd.512(<8 x double> %data, <8 x double> %passthru, i8 %mask)
+  ret <8 x double> %res
+}
+
+define <8 x double> @test_maskz_compress_pd_512(<8 x double> %data, i8 %mask)  #0 {
+;
+; CHECK-LABEL: @test_maskz_compress_pd_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast i8 [[TMP1]] to <8 x i1>
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <8 x i64> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <8 x i1> [[TMP3]] to i8
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i8 [[TMP6]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP7:%.*]], label [[TMP8:%.*]], !prof [[PROF1]]
+; CHECK:       7:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       8:
+; CHECK-NEXT:    [[TMP9:%.*]] = call <8 x double> @llvm.x86.avx512.mask.compress.v8f64(<8 x double> [[DATA:%.*]], <8 x double> zeroinitializer, <8 x i1> [[TMP4]])
+; CHECK-NEXT:    store <8 x i64> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x double> [[TMP9]]
+;
+  %res = call <8 x double> @llvm.x86.avx512.mask.compress.pd.512(<8 x double> %data, <8 x double> zeroinitializer, i8 %mask)
+  ret <8 x double> %res
+}
+
+define <8 x double> @test_compress_pd_512(<8 x double> %data, <8 x double> %extra_param)  #0 {
+; CHECK-LABEL: @test_compress_pd_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP7:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <8 x i64> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP3]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <8 x i64> [[TMP7]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP5:%.*]], label [[TMP6:%.*]], !prof [[PROF1]]
+; CHECK:       5:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       6:
+; CHECK-NEXT:    [[TMP2:%.*]] = call <8 x double> @llvm.x86.avx512.mask.compress.v8f64(<8 x double> [[DATA:%.*]], <8 x double> [[EXTRA_PARAM:%.*]], <8 x i1> splat (i1 true))
+; CHECK-NEXT:    store <8 x i64> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x double> [[TMP2]]
+;
+  %res = call <8 x double> @llvm.x86.avx512.mask.compress.pd.512(<8 x double> %data, <8 x double> %extra_param, i8 -1)
+  ret <8 x double> %res
+}
+
+declare <8 x double> @llvm.x86.avx512.mask.compress.pd.512(<8 x double> %data, <8 x double> %src0, i8 %mask)
+
+define <16 x float> @test_mask_compress_ps_512(<16 x float> %data, <16 x float> %passthru, i16 %mask)  #0 {
+;
+; CHECK-LABEL: @test_mask_compress_ps_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast i16 [[TMP1]] to <16 x i1>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i16 [[MASK:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP6]], 0
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <16 x i32> [[TMP3]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP7]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <16 x i1> [[TMP4]] to i16
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i16 [[TMP8]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR3]], label [[TMP9:%.*]], label [[TMP10:%.*]], !prof [[PROF1]]
+; CHECK:       9:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       10:
+; CHECK-NEXT:    [[TMP11:%.*]] = call <16 x float> @llvm.x86.avx512.mask.compress.v16f32(<16 x float> [[DATA:%.*]], <16 x float> [[PASSTHRU:%.*]], <16 x i1> [[TMP5]])
+; CHECK-NEXT:    store <16 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x float> [[TMP11]]
+;
+  %res = call <16 x float> @llvm.x86.avx512.mask.compress.ps.512(<16 x float> %data, <16 x float> %passthru, i16 %mask)
+  ret <16 x float> %res
+}
+
+define <16 x float> @test_maskz_compress_ps_512(<16 x float> %data, i16 %mask)  #0 {
+;
+; CHECK-LABEL: @test_maskz_compress_ps_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast i16 [[TMP1]] to <16 x i1>
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast i16 [[MASK:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <16 x i1> [[TMP3]] to i16
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i16 [[TMP6]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP7:%.*]], label [[TMP8:%.*]], !prof [[PROF1]]
+; CHECK:       7:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       8:
+; CHECK-NEXT:    [[TMP9:%.*]] = call <16 x float> @llvm.x86.avx512.mask.compress.v16f32(<16 x float> [[DATA:%.*]], <16 x float> zeroinitializer, <16 x i1> [[TMP4]])
+; CHECK-NEXT:    store <16 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x float> [[TMP9]]
+;
+  %res = call <16 x float> @llvm.x86.avx512.mask.compress.ps.512(<16 x float> %data, <16 x float> zeroinitializer, i16 %mask)
+  ret <16 x float> %res
+}
+
+define <16 x float> @test_compress_ps_512(<16 x float> %data, <16 x float> %extra_param)  #0 {
+; CHECK-LABEL: @test_compress_ps_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP7:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP3]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <16 x i32> [[TMP7]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP5:%.*]], label [[TMP6:%.*]], !prof [[PROF1]]
+; CHECK:       5:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       6:
+; CHECK-NEXT:    [[TMP2:%.*]] = call <16 x float> @llvm.x86.avx512.mask.compress.v16f32(<16 x float> [[DATA:%.*]], <16 x float> [[EXTRA_PARAM:%.*]], <16 x i1> splat (i1 true))
+; CHECK-NEXT:    store <16 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x float> [[TMP2]]
+;
+  %res = call <16 x float> @llvm.x86.avx512.mask.compress.ps.512(<16 x float> %data, <16 x float> %extra_param, i16 -1)
+  ret <16 x float> %res
+}
+
+declare <16 x float> @llvm.x86.avx512.mask.compress.ps.512(<16 x float> %data, <16 x float> %src0, i16 %mask)
+
+define <8 x i64> @test_mask_compress_q_512(<8 x i64> %data, <8 x i64> %passthru, i8 %mask)  #0 {
+;
+; CHECK-LABEL: @test_mask_compress_q_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast i8 [[TMP1]] to <8 x i1>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <8 x i64> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP6]], 0
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <8 x i64> [[TMP3]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP7]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <8 x i1> [[TMP4]] to i8
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i8 [[TMP8]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR3]], label [[TMP9:%.*]], label [[TMP10:%.*]], !prof [[PROF1]]
+; CHECK:       9:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       10:
+; CHECK-NEXT:    [[TMP11:%.*]] = call <8 x i64> @llvm.x86.avx512.mask.compress.v8i64(<8 x i64> [[DATA:%.*]], <8 x i64> [[PASSTHRU:%.*]], <8 x i1> [[TMP5]])
+; CHECK-NEXT:    store <8 x i64> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i64> [[TMP11]]
+;
+  %res = call <8 x i64> @llvm.x86.avx512.mask.compress.q.512(<8 x i64> %data, <8 x i64> %passthru, i8 %mask)
+  ret <8 x i64> %res
+}
+
+define <8 x i64> @test_maskz_compress_q_512(<8 x i64> %data, i8 %mask)  #0 {
+;
+; CHECK-LABEL: @test_maskz_compress_q_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast i8 [[TMP1]] to <8 x i1>
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <8 x i64> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <8 x i1> [[TMP3]] to i8
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i8 [[TMP6]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP7:%.*]], label [[TMP8:%.*]], !prof [[PROF1]]
+; CHECK:       7:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       8:
+; CHECK-NEXT:    [[TMP9:%.*]] = call <8 x i64> @llvm.x86.avx512.mask.compress.v8i64(<8 x i64> [[DATA:%.*]], <8 x i64> zeroinitializer, <8 x i1> [[TMP4]])
+; CHECK-NEXT:    store <8 x i64> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i64> [[TMP9]]
+;
+  %res = call <8 x i64> @llvm.x86.avx512.mask.compress.q.512(<8 x i64> %data, <8 x i64> zeroinitializer, i8 %mask)
+  ret <8 x i64> %res
+}
+
+define <8 x i64> @test_compress_q_512(<8 x i64> %data, <8 x i64> %extra_param)  #0 {
+; CHECK-LABEL: @test_compress_q_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP7:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <8 x i64> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP3]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <8 x i64> [[TMP7]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP5:%.*]], label [[TMP6:%.*]], !prof [[PROF1]]
+; CHECK:       5:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       6:
+; CHECK-NEXT:    [[TMP2:%.*]] = call <8 x i64> @llvm.x86.avx512.mask.compress.v8i64(<8 x i64> [[DATA:%.*]], <8 x i64> [[EXTRA_PARAM:%.*]], <8 x i1> splat (i1 true))
+; CHECK-NEXT:    store <8 x i64> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i64> [[TMP2]]
+;
+  %res = call <8 x i64> @llvm.x86.avx512.mask.compress.q.512(<8 x i64> %data, <8 x i64> %extra_param, i8 -1)
+  ret <8 x i64> %res
+}
+
+declare <8 x i64> @llvm.x86.avx512.mask.compress.q.512(<8 x i64> %data, <8 x i64> %src0, i8 %mask)
+
+define <16 x i32> @test_mask_compress_d_512(<16 x i32> %data, <16 x i32> %passthru, i16 %mask)  #0 {
+;
+; CHECK-LABEL: @test_mask_compress_d_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast i16 [[TMP1]] to <16 x i1>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i16 [[MASK:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP6]], 0
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <16 x i32> [[TMP3]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP7]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <16 x i1> [[TMP4]] to i16
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i16 [[TMP8]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR3]], label [[TMP9:%.*]], label [[TMP10:%.*]], !prof [[PROF1]]
+; CHECK:       9:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       10:
+; CHECK-NEXT:    [[TMP11:%.*]] = call <16 x i32> @llvm.x86.avx512.mask.compress.v16i32(<16 x i32> [[DATA:%.*]], <16 x i32> [[PASSTHRU:%.*]], <16 x i1> [[TMP5]])
+; CHECK-NEXT:    store <16 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i32> [[TMP11]]
+;
+  %res = call <16 x i32> @llvm.x86.avx512.mask.compress.d.512(<16 x i32> %data, <16 x i32> %passthru, i16 %mask)
+  ret <16 x i32> %res
+}
+
+define <16 x i32> @test_maskz_compress_d_512(<16 x i32> %data, i16 %mask)  #0 {
+;
+; CHECK-LABEL: @test_maskz_compress_d_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast i16 [[TMP1]] to <16 x i1>
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast i16 [[MASK:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <16 x i1> [[TMP3]] to i16
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i16 [[TMP6]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP7:%.*]], label [[TMP8:%.*]], !prof [[PROF1]]
+; CHECK:       7:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       8:
+; CHECK-NEXT:    [[TMP9:%.*]] = call <16 x i32> @llvm.x86.avx512.mask.compress.v16i32(<16 x i32> [[DATA:%.*]], <16 x i32> zeroinitializer, <16 x i1> [[TMP4]])
+; CHECK-NEXT:    store <16 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i32> [[TMP9]]
+;
+  %res = call <16 x i32> @llvm.x86.avx512.mask.compress.d.512(<16 x i32> %data, <16 x i32> zeroinitializer, i16 %mask)
+  ret <16 x i32> %res
+}
+
+define <16 x i32> @test_compress_d_512(<16 x i32> %data, <16 x i32> %extra_param)  #0 {
+; CHECK-LABEL: @test_compress_d_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP7:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP3]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <16 x i32> [[TMP7]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP5:%.*]], label [[TMP6:%.*]], !prof [[PROF1]]
+; CHECK:       5:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       6:
+; CHECK-NEXT:    [[TMP2:%.*]] = call <16 x i32> @llvm.x86.avx512.mask.compress.v16i32(<16 x i32> [[DATA:%.*]], <16 x i32> [[EXTRA_PARAM:%.*]], <16 x i1> splat (i1 true))
+; CHECK-NEXT:    store <16 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i32> [[TMP2]]
+;
+  %res = call <16 x i32> @llvm.x86.avx512.mask.compress.d.512(<16 x i32> %data, <16 x i32> %extra_param, i16 -1)
+  ret <16 x i32> %res
+}
+
+declare <16 x i32> @llvm.x86.avx512.mask.compress.d.512(<16 x i32> %data, <16 x i32> %src0, i16 %mask)
+
+define <8 x double> @test_expand_pd_512(<8 x double> %data, <8 x double> %extra_param)  #0 {
+; CHECK-LABEL: @test_expand_pd_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP7:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <8 x i64> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP3]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <8 x i64> [[TMP7]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP5:%.*]], label [[TMP6:%.*]], !prof [[PROF1]]
+; CHECK:       5:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       6:
+; CHECK-NEXT:    [[TMP2:%.*]] = call <8 x double> @llvm.x86.avx512.mask.expand.v8f64(<8 x double> [[DATA:%.*]], <8 x double> [[EXTRA_PARAM:%.*]], <8 x i1> splat (i1 true))
+; CHECK-NEXT:    store <8 x i64> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x double> [[TMP2]]
+;
+  %res = call <8 x double> @llvm.x86.avx512.mask.expand.pd.512(<8 x double> %data, <8 x double> %extra_param, i8 -1)
+  ret <8 x double> %res
+}
+
+define <8 x double> @test_mask_expand_pd_512(<8 x double> %data, <8 x double> %passthru, i8 %mask)  #0 {
+;
+; CHECK-LABEL: @test_mask_expand_pd_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast i8 [[TMP1]] to <8 x i1>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <8 x i64> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP6]], 0
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <8 x i64> [[TMP3]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP7]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <8 x i1> [[TMP4]] to i8
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i8 [[TMP8]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR3]], label [[TMP9:%.*]], label [[TMP10:%.*]], !prof [[PROF1]]
+; CHECK:       9:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       10:
+; CHECK-NEXT:    [[TMP11:%.*]] = call <8 x double> @llvm.x86.avx512.mask.expand.v8f64(<8 x double> [[DATA:%.*]], <8 x double> [[PASSTHRU:%.*]], <8 x i1> [[TMP5]])
+; CHECK-NEXT:    store <8 x i64> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x double> [[TMP11]]
+;
+  %res = call <8 x double> @llvm.x86.avx512.mask.expand.pd.512(<8 x double> %data, <8 x double> %passthru, i8 %mask)
+  ret <8 x double> %res
+}
+
+define <8 x double> @test_maskz_expand_pd_512(<8 x double> %data, i8 %mask)  #0 {
+;
+; CHECK-LABEL: @test_maskz_expand_pd_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast i8 [[TMP1]] to <8 x i1>
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <8 x i64> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <8 x i1> [[TMP3]] to i8
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i8 [[TMP6]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP7:%.*]], label [[TMP8:%.*]], !prof [[PROF1]]
+; CHECK:       7:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       8:
+; CHECK-NEXT:    [[TMP9:%.*]] = call <8 x double> @llvm.x86.avx512.mask.expand.v8f64(<8 x double> [[DATA:%.*]], <8 x double> zeroinitializer, <8 x i1> [[TMP4]])
+; CHECK-NEXT:    store <8 x i64> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x double> [[TMP9]]
+;
+  %res = call <8 x double> @llvm.x86.avx512.mask.expand.pd.512(<8 x double> %data, <8 x double> zeroinitializer, i8 %mask)
+  ret <8 x double> %res
+}
+
+declare <8 x double> @llvm.x86.avx512.mask.expand.pd.512(<8 x double> %data, <8 x double> %src0, i8 %mask)
+
+define <16 x float> @test_expand_ps_512(<16 x float> %data, <16 x float> %extra_param)  #0 {
+; CHECK-LABEL: @test_expand_ps_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP7:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP3]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <16 x i32> [[TMP7]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP5:%.*]], label [[TMP6:%.*]], !prof [[PROF1]]
+; CHECK:       5:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       6:
+; CHECK-NEXT:    [[TMP2:%.*]] = call <16 x float> @llvm.x86.avx512.mask.expand.v16f32(<16 x float> [[DATA:%.*]], <16 x float> [[EXTRA_PARAM:%.*]], <16 x i1> splat (i1 true))
+; CHECK-NEXT:    store <16 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x float> [[TMP2]]
+;
+  %res = call <16 x float> @llvm.x86.avx512.mask.expand.ps.512(<16 x float> %data, <16 x float> %extra_param, i16 -1)
+  ret <16 x float> %res
+}
+
+define <16 x float> @test_mask_expand_ps_512(<16 x float> %data, <16 x float> %passthru, i16 %mask)  #0 {
+;
+; CHECK-LABEL: @test_mask_expand_ps_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast i16 [[TMP1]] to <16 x i1>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i16 [[MASK:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP6]], 0
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <16 x i32> [[TMP3]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP7]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <16 x i1> [[TMP4]] to i16
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i16 [[TMP8]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR3]], label [[TMP9:%.*]], label [[TMP10:%.*]], !prof [[PROF1]]
+; CHECK:       9:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       10:
+; CHECK-NEXT:    [[TMP11:%.*]] = call <16 x float> @llvm.x86.avx512.mask.expand.v16f32(<16 x float> [[DATA:%.*]], <16 x float> [[PASSTHRU:%.*]], <16 x i1> [[TMP5]])
+; CHECK-NEXT:    store <16 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x float> [[TMP11]]
+;
+  %res = call <16 x float> @llvm.x86.avx512.mask.expand.ps.512(<16 x float> %data, <16 x float> %passthru, i16 %mask)
+  ret <16 x float> %res
+}
+
+define <16 x float> @test_maskz_expand_ps_512(<16 x float> %data, i16 %mask)  #0 {
+;
+; CHECK-LABEL: @test_maskz_expand_ps_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast i16 [[TMP1]] to <16 x i1>
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast i16 [[MASK:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <16 x i1> [[TMP3]] to i16
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i16 [[TMP6]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP7:%.*]], label [[TMP8:%.*]], !prof [[PROF1]]
+; CHECK:       7:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       8:
+; CHECK-NEXT:    [[TMP9:%.*]] = call <16 x float> @llvm.x86.avx512.mask.expand.v16f32(<16 x float> [[DATA:%.*]], <16 x float> zeroinitializer, <16 x i1> [[TMP4]])
+; CHECK-NEXT:    store <16 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x float> [[TMP9]]
+;
+  %res = call <16 x float> @llvm.x86.avx512.mask.expand.ps.512(<16 x float> %data, <16 x float> zeroinitializer, i16 %mask)
+  ret <16 x float> %res
+}
+
+declare <16 x float> @llvm.x86.avx512.mask.expand.ps.512(<16 x float> %data, <16 x float> %src0, i16 %mask)
+
+define <8 x i64> @test_expand_q_512(<8 x i64> %data, <8 x i64> %extra_param)  #0 {
+; CHECK-LABEL: @test_expand_q_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP7:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <8 x i64> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP3]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <8 x i64> [[TMP7]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP5:%.*]], label [[TMP6:%.*]], !prof [[PROF1]]
+; CHECK:       5:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       6:
+; CHECK-NEXT:    [[TMP2:%.*]] = call <8 x i64> @llvm.x86.avx512.mask.expand.v8i64(<8 x i64> [[DATA:%.*]], <8 x i64> [[EXTRA_PARAM:%.*]], <8 x i1> splat (i1 true))
+; CHECK-NEXT:    store <8 x i64> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i64> [[TMP2]]
+;
+  %res = call <8 x i64> @llvm.x86.avx512.mask.expand.q.512(<8 x i64> %data, <8 x i64> %extra_param, i8 -1)
+  ret <8 x i64> %res
+}
+
+define <8 x i64> @test_mask_expand_q_512(<8 x i64> %data, <8 x i64> %passthru, i8 %mask)  #0 {
+;
+; CHECK-LABEL: @test_mask_expand_q_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast i8 [[TMP1]] to <8 x i1>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <8 x i64> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP6]], 0
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <8 x i64> [[TMP3]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP7]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <8 x i1> [[TMP4]] to i8
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i8 [[TMP8]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR3]], label [[TMP9:%.*]], label [[TMP10:%.*]], !prof [[PROF1]]
+; CHECK:       9:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       10:
+; CHECK-NEXT:    [[TMP11:%.*]] = call <8 x i64> @llvm.x86.avx512.mask.expand.v8i64(<8 x i64> [[DATA:%.*]], <8 x i64> [[PASSTHRU:%.*]], <8 x i1> [[TMP5]])
+; CHECK-NEXT:    store <8 x i64> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i64> [[TMP11]]
+;
+  %res = call <8 x i64> @llvm.x86.avx512.mask.expand.q.512(<8 x i64> %data, <8 x i64> %passthru, i8 %mask)
+  ret <8 x i64> %res
+}
+
+define <8 x i64> @test_maskz_expand_q_512(<8 x i64> %data, i8 %mask)  #0 {
+;
+; CHECK-LABEL: @test_maskz_expand_q_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast i8 [[TMP1]] to <8 x i1>
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <8 x i64> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <8 x i1> [[TMP3]] to i8
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i8 [[TMP6]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP7:%.*]], label [[TMP8:%.*]], !prof [[PROF1]]
+; CHECK:       7:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       8:
+; CHECK-NEXT:    [[TMP9:%.*]] = call <8 x i64> @llvm.x86.avx512.mask.expand.v8i64(<8 x i64> [[DATA:%.*]], <8 x i64> zeroinitializer, <8 x i1> [[TMP4]])
+; CHECK-NEXT:    store <8 x i64> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i64> [[TMP9]]
+;
+  %res = call <8 x i64> @llvm.x86.avx512.mask.expand.q.512(<8 x i64> %data, <8 x i64> zeroinitializer, i8 %mask)
+  ret <8 x i64> %res
+}
+
+declare <8 x i64> @llvm.x86.avx512.mask.expand.q.512(<8 x i64> %data, <8 x i64> %src0, i8 %mask)
+
+define <16 x i32> @test_expand_d_512(<16 x i32> %data, <16 x i32> %extra_param)  #0 {
+; CHECK-LABEL: @test_expand_d_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP7:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP3]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <16 x i32> [[TMP7]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP5:%.*]], label [[TMP6:%.*]], !prof [[PROF1]]
+; CHECK:       5:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       6:
+; CHECK-NEXT:    [[TMP2:%.*]] = call <16 x i32> @llvm.x86.avx512.mask.expand.v16i32(<16 x i32> [[DATA:%.*]], <16 x i32> [[EXTRA_PARAM:%.*]], <16 x i1> splat (i1 true))
+; CHECK-NEXT:    store <16 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i32> [[TMP2]]
+;
+  %res = call <16 x i32> @llvm.x86.avx512.mask.expand.d.512(<16 x i32> %data, <16 x i32> %extra_param, i16 -1)
+  ret <16 x i32> %res
+}
+
+define <16 x i32> @test_mask_expand_d_512(<16 x i32> %data, <16 x i32> %passthru, i16 %mask)  #0 {
+;
+; CHECK-LABEL: @test_mask_expand_d_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast i16 [[TMP1]] to <16 x i1>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i16 [[MASK:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP6]], 0
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <16 x i32> [[TMP3]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP7]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <16 x i1> [[TMP4]] to i16
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i16 [[TMP8]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR3]], label [[TMP9:%.*]], label [[TMP10:%.*]], !prof [[PROF1]]
+; CHECK:       9:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       10:
+; CHECK-NEXT:    [[TMP11:%.*]] = call <16 x i32> @llvm.x86.avx512.mask.expand.v16i32(<16 x i32> [[DATA:%.*]], <16 x i32> [[PASSTHRU:%.*]], <16 x i1> [[TMP5]])
+; CHECK-NEXT:    store <16 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i32> [[TMP11]]
+;
+  %res = call <16 x i32> @llvm.x86.avx512.mask.expand.d.512(<16 x i32> %data, <16 x i32> %passthru, i16 %mask)
+  ret <16 x i32> %res
+}
+
+define <16 x i32> @test_maskz_expand_d_512(<16 x i32> %data, i16 %mask)  #0 {
+;
+; CHECK-LABEL: @test_maskz_expand_d_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast i16 [[TMP1]] to <16 x i1>
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast i16 [[MASK:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <16 x i1> [[TMP3]] to i16
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i16 [[TMP6]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP7:%.*]], label [[TMP8:%.*]], !prof [[PROF1]]
+; CHECK:       7:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       8:
+; CHECK-NEXT:    [[TMP9:%.*]] = call <16 x i32> @llvm.x86.avx512.mask.expand.v16i32(<16 x i32> [[DATA:%.*]], <16 x i32> zeroinitializer, <16 x i1> [[TMP4]])
+; CHECK-NEXT:    store <16 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i32> [[TMP9]]
+;
+  %res = call <16 x i32> @llvm.x86.avx512.mask.expand.d.512(<16 x i32> %data, <16 x i32> zeroinitializer, i16 %mask)
+  ret <16 x i32> %res
+}
+
+declare <16 x i32> @llvm.x86.avx512.mask.expand.d.512(<16 x i32> %data, <16 x i32> %src0, i16 %mask)
+
+define <16 x float> @test_cmp_512(<16 x float> %a, <16 x float> %b, <16 x float> %c, <16 x float> %d, ptr %p)  #0 {
+;
+; CHECK-LABEL: @test_cmp_512(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 256) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <16 x i32> [[TMP0]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP6]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP7:%.*]], label [[TMP8:%.*]], !prof [[PROF1]]
+; CHECK:       7:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       8:
+; CHECK-NEXT:    [[TMP9:%.*]] = call <16 x i1> @llvm.x86.avx512.mask.cmp.ps.512(<16 x float> [[A:%.*]], <16 x float> [[B:%.*]], i32 1, <16 x i1> splat (i1 true), i32 8)
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i512 [[TMP10]], 0
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <16 x i32> [[TMP3]] to i512
+; CHECK-NEXT:    [[_MSCMP3:%.*]] = icmp ne i512 [[TMP11]], 0
+; CHECK-NEXT:    [[_MSOR4:%.*]] = or i1 [[_MSCMP2]], [[_MSCMP3]]
+; CHECK-NEXT:    br i1 [[_MSOR4]], label [[TMP12:%.*]], label [[TMP13:%.*]], !prof [[PROF1]]
+; CHECK:       12:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       13:
+; CHECK-NEXT:    [[TMP14:%.*]] = call <16 x i1> @llvm.x86.avx512.mask.cmp.ps.512(<16 x float> [[C:%.*]], <16 x float> [[D:%.*]], i32 1, <16 x i1> splat (i1 true), i32 4)
+; CHECK-NEXT:    [[_MSCMP5:%.*]] = icmp ne i64 [[TMP4]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP5]], label [[TMP15:%.*]], label [[TMP16:%.*]], !prof [[PROF1]]
+; CHECK:       15:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       16:
+; CHECK-NEXT:    [[TMP17:%.*]] = load <16 x float>, ptr [[P:%.*]], align 64
+; CHECK-NEXT:    [[TMP18:%.*]] = ptrtoint ptr [[P]] to i64
+; CHECK-NEXT:    [[TMP19:%.*]] = xor i64 [[TMP18]], 87960930222080
+; CHECK-NEXT:    [[TMP20:%.*]] = inttoptr i64 [[TMP19]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <16 x i32>, ptr [[TMP20]], align 64
+; CHECK-NEXT:    [[TMP21:%.*]] = xor <16 x i1> [[TMP9]], [[TMP14]]
+; CHECK-NEXT:    [[TMP22:%.*]] = select <16 x i1> [[TMP21]], <16 x i32> zeroinitializer, <16 x i32> [[_MSLD]]
+; CHECK-NEXT:    [[TMP23:%.*]] = bitcast <16 x float> [[TMP17]] to <16 x i32>
+; CHECK-NEXT:    [[TMP24:%.*]] = xor <16 x i32> zeroinitializer, [[TMP23]]
+; CHECK-NEXT:    [[TMP25:%.*]] = or <16 x i32> [[TMP24]], zeroinitializer
+; CHECK-NEXT:    [[TMP26:%.*]] = or <16 x i32> [[TMP25]], [[_MSLD]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> zeroinitializer, <16 x i32> [[TMP26]], <16 x i32> [[TMP22]]
+; CHECK-NEXT:    [[TMP27:%.*]] = select <16 x i1> [[TMP21]], <16 x float> zeroinitializer, <16 x float> [[TMP17]]
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x float> [[TMP27]]
+;
+  entry:
+  %0 = tail call <16 x i1> @llvm.x86.avx512.cmp.ps.512(<16 x float> %a, <16 x float> %b, i32 1, i32 8)
+  %1 = tail call <16 x i1> @llvm.x86.avx512.cmp.ps.512(<16 x float> %c, <16 x float> %d, i32 1, i32 4)
+  %2 = load <16 x float>, ptr %p
+  %3 = xor <16 x i1> %0, %1
+  %4 = select <16 x i1> %3, <16 x float> zeroinitializer, <16 x float> %2
+  ret <16 x float> %4
+}
+
+declare <16 x i1> @llvm.x86.avx512.cmp.ps.512(<16 x float>, <16 x float>, i32, i32)
+
+attributes #0 = { sanitize_memory }
diff --git a/llvm/test/Instrumentation/MemorySanitizer/X86/avx512-intrinsics.ll b/llvm/test/Instrumentation/MemorySanitizer/X86/avx512-intrinsics.ll
new file mode 100644
index 0000000000000..d6fa08cd61f5e
--- /dev/null
+++ b/llvm/test/Instrumentation/MemorySanitizer/X86/avx512-intrinsics.ll
@@ -0,0 +1,13743 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt %s -S -mattr=+avx512f -passes=msan 2>&1 | FileCheck %s
+;
+; Forked from llvm/test/CodeGen/X86/avx512-intrinsics.ll
+
+target datalayout = "e-m:o-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+define <8 x double> @test_mask_compress_pd_512(<8 x double> %data, <8 x double> %passthru, i8 %mask) #0 {
+; CHECK-LABEL: @test_mask_compress_pd_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast i8 [[TMP1]] to <8 x i1>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <8 x i64> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP6]], 0
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <8 x i64> [[TMP3]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP7]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <8 x i1> [[TMP4]] to i8
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i8 [[TMP8]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR3]], label [[TMP9:%.*]], label [[TMP10:%.*]], !prof [[PROF1:![0-9]+]]
+; CHECK:       9:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10:[0-9]+]]
+; CHECK-NEXT:    unreachable
+; CHECK:       10:
+; CHECK-NEXT:    [[TMP11:%.*]] = call <8 x double> @llvm.x86.avx512.mask.compress.v8f64(<8 x double> [[DATA:%.*]], <8 x double> [[PASSTHRU:%.*]], <8 x i1> [[TMP5]])
+; CHECK-NEXT:    store <8 x i64> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x double> [[TMP11]]
+;
+  %1 = bitcast i8 %mask to <8 x i1>
+  %2 = call <8 x double> @llvm.x86.avx512.mask.compress.v8f64(<8 x double> %data, <8 x double> %passthru, <8 x i1> %1)
+  ret <8 x double> %2
+}
+
+define <8 x double> @test_maskz_compress_pd_512(<8 x double> %data, i8 %mask) #0 {
+; CHECK-LABEL: @test_maskz_compress_pd_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast i8 [[TMP1]] to <8 x i1>
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <8 x i64> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <8 x i1> [[TMP3]] to i8
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i8 [[TMP6]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP7:%.*]], label [[TMP8:%.*]], !prof [[PROF1]]
+; CHECK:       7:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       8:
+; CHECK-NEXT:    [[TMP9:%.*]] = call <8 x double> @llvm.x86.avx512.mask.compress.v8f64(<8 x double> [[DATA:%.*]], <8 x double> zeroinitializer, <8 x i1> [[TMP4]])
+; CHECK-NEXT:    store <8 x i64> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x double> [[TMP9]]
+;
+  %1 = bitcast i8 %mask to <8 x i1>
+  %2 = call <8 x double> @llvm.x86.avx512.mask.compress.v8f64(<8 x double> %data, <8 x double> zeroinitializer, <8 x i1> %1)
+  ret <8 x double> %2
+}
+
+define <8 x double> @test_compress_pd_512(<8 x double> %data, <8 x double> %extra_param) #0 {
+; CHECK-LABEL: @test_compress_pd_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP7:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <8 x i64> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP3]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <8 x i64> [[TMP7]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP5:%.*]], label [[TMP6:%.*]], !prof [[PROF1]]
+; CHECK:       5:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       6:
+; CHECK-NEXT:    [[TMP2:%.*]] = call <8 x double> @llvm.x86.avx512.mask.compress.v8f64(<8 x double> [[DATA:%.*]], <8 x double> [[EXTRA_PARAM:%.*]], <8 x i1> splat (i1 true))
+; CHECK-NEXT:    store <8 x i64> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x double> [[TMP2]]
+;
+  %1 = call <8 x double> @llvm.x86.avx512.mask.compress.v8f64(<8 x double> %data, <8 x double> %extra_param, <8 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>)
+  ret <8 x double> %1
+}
+
+define <16 x float> @test_mask_compress_ps_512(<16 x float> %data, <16 x float> %passthru, i16 %mask) #0 {
+; CHECK-LABEL: @test_mask_compress_ps_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast i16 [[TMP1]] to <16 x i1>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i16 [[MASK:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP6]], 0
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <16 x i32> [[TMP3]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP7]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <16 x i1> [[TMP4]] to i16
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i16 [[TMP8]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR3]], label [[TMP9:%.*]], label [[TMP10:%.*]], !prof [[PROF1]]
+; CHECK:       9:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       10:
+; CHECK-NEXT:    [[TMP11:%.*]] = call <16 x float> @llvm.x86.avx512.mask.compress.v16f32(<16 x float> [[DATA:%.*]], <16 x float> [[PASSTHRU:%.*]], <16 x i1> [[TMP5]])
+; CHECK-NEXT:    store <16 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x float> [[TMP11]]
+;
+  %1 = bitcast i16 %mask to <16 x i1>
+  %2 = call <16 x float> @llvm.x86.avx512.mask.compress.v16f32(<16 x float> %data, <16 x float> %passthru, <16 x i1> %1)
+  ret <16 x float> %2
+}
+
+define <16 x float> @test_maskz_compress_ps_512(<16 x float> %data, i16 %mask) #0 {
+; CHECK-LABEL: @test_maskz_compress_ps_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast i16 [[TMP1]] to <16 x i1>
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast i16 [[MASK:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <16 x i1> [[TMP3]] to i16
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i16 [[TMP6]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP7:%.*]], label [[TMP8:%.*]], !prof [[PROF1]]
+; CHECK:       7:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       8:
+; CHECK-NEXT:    [[TMP9:%.*]] = call <16 x float> @llvm.x86.avx512.mask.compress.v16f32(<16 x float> [[DATA:%.*]], <16 x float> zeroinitializer, <16 x i1> [[TMP4]])
+; CHECK-NEXT:    store <16 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x float> [[TMP9]]
+;
+  %1 = bitcast i16 %mask to <16 x i1>
+  %2 = call <16 x float> @llvm.x86.avx512.mask.compress.v16f32(<16 x float> %data, <16 x float> zeroinitializer, <16 x i1> %1)
+  ret <16 x float> %2
+}
+
+define <16 x float> @test_compress_ps_512(<16 x float> %data, <16 x float> %extra_param) #0 {
+; CHECK-LABEL: @test_compress_ps_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP7:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP3]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <16 x i32> [[TMP7]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP5:%.*]], label [[TMP6:%.*]], !prof [[PROF1]]
+; CHECK:       5:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       6:
+; CHECK-NEXT:    [[TMP2:%.*]] = call <16 x float> @llvm.x86.avx512.mask.compress.v16f32(<16 x float> [[DATA:%.*]], <16 x float> [[EXTRA_PARAM:%.*]], <16 x i1> splat (i1 true))
+; CHECK-NEXT:    store <16 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x float> [[TMP2]]
+;
+  %1 = call <16 x float> @llvm.x86.avx512.mask.compress.v16f32(<16 x float> %data, <16 x float> %extra_param, <16 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>)
+  ret <16 x float> %1
+}
+
+define <8 x i64> @test_mask_compress_q_512(<8 x i64> %data, <8 x i64> %passthru, i8 %mask) #0 {
+; CHECK-LABEL: @test_mask_compress_q_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast i8 [[TMP1]] to <8 x i1>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <8 x i64> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP6]], 0
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <8 x i64> [[TMP3]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP7]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <8 x i1> [[TMP4]] to i8
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i8 [[TMP8]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR3]], label [[TMP9:%.*]], label [[TMP10:%.*]], !prof [[PROF1]]
+; CHECK:       9:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       10:
+; CHECK-NEXT:    [[TMP11:%.*]] = call <8 x i64> @llvm.x86.avx512.mask.compress.v8i64(<8 x i64> [[DATA:%.*]], <8 x i64> [[PASSTHRU:%.*]], <8 x i1> [[TMP5]])
+; CHECK-NEXT:    store <8 x i64> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i64> [[TMP11]]
+;
+  %1 = bitcast i8 %mask to <8 x i1>
+  %2 = call <8 x i64> @llvm.x86.avx512.mask.compress.v8i64(<8 x i64> %data, <8 x i64> %passthru, <8 x i1> %1)
+  ret <8 x i64> %2
+}
+
+define <8 x i64> @test_maskz_compress_q_512(<8 x i64> %data, i8 %mask) #0 {
+; CHECK-LABEL: @test_maskz_compress_q_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast i8 [[TMP1]] to <8 x i1>
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <8 x i64> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <8 x i1> [[TMP3]] to i8
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i8 [[TMP6]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP7:%.*]], label [[TMP8:%.*]], !prof [[PROF1]]
+; CHECK:       7:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       8:
+; CHECK-NEXT:    [[TMP9:%.*]] = call <8 x i64> @llvm.x86.avx512.mask.compress.v8i64(<8 x i64> [[DATA:%.*]], <8 x i64> zeroinitializer, <8 x i1> [[TMP4]])
+; CHECK-NEXT:    store <8 x i64> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i64> [[TMP9]]
+;
+  %1 = bitcast i8 %mask to <8 x i1>
+  %2 = call <8 x i64> @llvm.x86.avx512.mask.compress.v8i64(<8 x i64> %data, <8 x i64> zeroinitializer, <8 x i1> %1)
+  ret <8 x i64> %2
+}
+
+define <8 x i64> @test_compress_q_512(<8 x i64> %data, <8 x i64> %extra_param) #0 {
+; CHECK-LABEL: @test_compress_q_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP7:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <8 x i64> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP3]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <8 x i64> [[TMP7]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP5:%.*]], label [[TMP6:%.*]], !prof [[PROF1]]
+; CHECK:       5:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       6:
+; CHECK-NEXT:    [[TMP2:%.*]] = call <8 x i64> @llvm.x86.avx512.mask.compress.v8i64(<8 x i64> [[DATA:%.*]], <8 x i64> [[EXTRA_PARAM:%.*]], <8 x i1> splat (i1 true))
+; CHECK-NEXT:    store <8 x i64> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i64> [[TMP2]]
+;
+  %1 = call <8 x i64> @llvm.x86.avx512.mask.compress.v8i64(<8 x i64> %data, <8 x i64> %extra_param, <8 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>)
+  ret <8 x i64> %1
+}
+
+define <16 x i32> @test_mask_compress_d_512(<16 x i32> %data, <16 x i32> %passthru, i16 %mask) #0 {
+; CHECK-LABEL: @test_mask_compress_d_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast i16 [[TMP1]] to <16 x i1>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i16 [[MASK:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP6]], 0
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <16 x i32> [[TMP3]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP7]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <16 x i1> [[TMP4]] to i16
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i16 [[TMP8]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR3]], label [[TMP9:%.*]], label [[TMP10:%.*]], !prof [[PROF1]]
+; CHECK:       9:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       10:
+; CHECK-NEXT:    [[TMP11:%.*]] = call <16 x i32> @llvm.x86.avx512.mask.compress.v16i32(<16 x i32> [[DATA:%.*]], <16 x i32> [[PASSTHRU:%.*]], <16 x i1> [[TMP5]])
+; CHECK-NEXT:    store <16 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i32> [[TMP11]]
+;
+  %1 = bitcast i16 %mask to <16 x i1>
+  %2 = call <16 x i32> @llvm.x86.avx512.mask.compress.v16i32(<16 x i32> %data, <16 x i32> %passthru, <16 x i1> %1)
+  ret <16 x i32> %2
+}
+
+define <16 x i32> @test_maskz_compress_d_512(<16 x i32> %data, i16 %mask) #0 {
+; CHECK-LABEL: @test_maskz_compress_d_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast i16 [[TMP1]] to <16 x i1>
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast i16 [[MASK:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <16 x i1> [[TMP3]] to i16
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i16 [[TMP6]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP7:%.*]], label [[TMP8:%.*]], !prof [[PROF1]]
+; CHECK:       7:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       8:
+; CHECK-NEXT:    [[TMP9:%.*]] = call <16 x i32> @llvm.x86.avx512.mask.compress.v16i32(<16 x i32> [[DATA:%.*]], <16 x i32> zeroinitializer, <16 x i1> [[TMP4]])
+; CHECK-NEXT:    store <16 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i32> [[TMP9]]
+;
+  %1 = bitcast i16 %mask to <16 x i1>
+  %2 = call <16 x i32> @llvm.x86.avx512.mask.compress.v16i32(<16 x i32> %data, <16 x i32> zeroinitializer, <16 x i1> %1)
+  ret <16 x i32> %2
+}
+
+define <16 x i32> @test_compress_d_512(<16 x i32> %data, <16 x i32> %extra_param) #0 {
+; CHECK-LABEL: @test_compress_d_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP7:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP3]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <16 x i32> [[TMP7]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP5:%.*]], label [[TMP6:%.*]], !prof [[PROF1]]
+; CHECK:       5:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       6:
+; CHECK-NEXT:    [[TMP2:%.*]] = call <16 x i32> @llvm.x86.avx512.mask.compress.v16i32(<16 x i32> [[DATA:%.*]], <16 x i32> [[EXTRA_PARAM:%.*]], <16 x i1> splat (i1 true))
+; CHECK-NEXT:    store <16 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i32> [[TMP2]]
+;
+  %1 = call <16 x i32> @llvm.x86.avx512.mask.compress.v16i32(<16 x i32> %data, <16 x i32> %extra_param, <16 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>)
+  ret <16 x i32> %1
+}
+
+define <8 x double> @test_expand_pd_512(<8 x double> %data, <8 x double> %extra_param) #0 {
+; CHECK-LABEL: @test_expand_pd_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP7:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <8 x i64> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP3]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <8 x i64> [[TMP7]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP5:%.*]], label [[TMP6:%.*]], !prof [[PROF1]]
+; CHECK:       5:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       6:
+; CHECK-NEXT:    [[TMP2:%.*]] = call <8 x double> @llvm.x86.avx512.mask.expand.v8f64(<8 x double> [[DATA:%.*]], <8 x double> [[EXTRA_PARAM:%.*]], <8 x i1> splat (i1 true))
+; CHECK-NEXT:    store <8 x i64> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x double> [[TMP2]]
+;
+  %1 = call <8 x double> @llvm.x86.avx512.mask.expand.v8f64(<8 x double> %data, <8 x double> %extra_param, <8 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>)
+  ret <8 x double> %1
+}
+
+define <8 x double> @test_mask_expand_pd_512(<8 x double> %data, <8 x double> %passthru, i8 %mask) #0 {
+; CHECK-LABEL: @test_mask_expand_pd_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast i8 [[TMP1]] to <8 x i1>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <8 x i64> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP6]], 0
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <8 x i64> [[TMP3]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP7]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <8 x i1> [[TMP4]] to i8
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i8 [[TMP8]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR3]], label [[TMP9:%.*]], label [[TMP10:%.*]], !prof [[PROF1]]
+; CHECK:       9:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       10:
+; CHECK-NEXT:    [[TMP11:%.*]] = call <8 x double> @llvm.x86.avx512.mask.expand.v8f64(<8 x double> [[DATA:%.*]], <8 x double> [[PASSTHRU:%.*]], <8 x i1> [[TMP5]])
+; CHECK-NEXT:    store <8 x i64> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x double> [[TMP11]]
+;
+  %1 = bitcast i8 %mask to <8 x i1>
+  %2 = call <8 x double> @llvm.x86.avx512.mask.expand.v8f64(<8 x double> %data, <8 x double> %passthru, <8 x i1> %1)
+  ret <8 x double> %2
+}
+
+define <8 x double> @test_maskz_expand_pd_512(<8 x double> %data, i8 %mask) #0 {
+; CHECK-LABEL: @test_maskz_expand_pd_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast i8 [[TMP1]] to <8 x i1>
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <8 x i64> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <8 x i1> [[TMP3]] to i8
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i8 [[TMP6]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP7:%.*]], label [[TMP8:%.*]], !prof [[PROF1]]
+; CHECK:       7:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       8:
+; CHECK-NEXT:    [[TMP9:%.*]] = call <8 x double> @llvm.x86.avx512.mask.expand.v8f64(<8 x double> [[DATA:%.*]], <8 x double> zeroinitializer, <8 x i1> [[TMP4]])
+; CHECK-NEXT:    store <8 x i64> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x double> [[TMP9]]
+;
+  %1 = bitcast i8 %mask to <8 x i1>
+  %2 = call <8 x double> @llvm.x86.avx512.mask.expand.v8f64(<8 x double> %data, <8 x double> zeroinitializer, <8 x i1> %1)
+  ret <8 x double> %2
+}
+
+define <16 x float> @test_expand_ps_512(<16 x float> %data, <16 x float> %extra_param) #0 {
+; CHECK-LABEL: @test_expand_ps_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP7:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP3]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <16 x i32> [[TMP7]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP5:%.*]], label [[TMP6:%.*]], !prof [[PROF1]]
+; CHECK:       5:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       6:
+; CHECK-NEXT:    [[TMP2:%.*]] = call <16 x float> @llvm.x86.avx512.mask.expand.v16f32(<16 x float> [[DATA:%.*]], <16 x float> [[EXTRA_PARAM:%.*]], <16 x i1> splat (i1 true))
+; CHECK-NEXT:    store <16 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x float> [[TMP2]]
+;
+  %1 = call <16 x float> @llvm.x86.avx512.mask.expand.v16f32(<16 x float> %data, <16 x float> %extra_param, <16 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>)
+  ret <16 x float> %1
+}
+
+define <16 x float> @test_mask_expand_ps_512(<16 x float> %data, <16 x float> %passthru, i16 %mask) #0 {
+; CHECK-LABEL: @test_mask_expand_ps_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast i16 [[TMP1]] to <16 x i1>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i16 [[MASK:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP6]], 0
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <16 x i32> [[TMP3]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP7]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <16 x i1> [[TMP4]] to i16
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i16 [[TMP8]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR3]], label [[TMP9:%.*]], label [[TMP10:%.*]], !prof [[PROF1]]
+; CHECK:       9:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       10:
+; CHECK-NEXT:    [[TMP11:%.*]] = call <16 x float> @llvm.x86.avx512.mask.expand.v16f32(<16 x float> [[DATA:%.*]], <16 x float> [[PASSTHRU:%.*]], <16 x i1> [[TMP5]])
+; CHECK-NEXT:    store <16 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x float> [[TMP11]]
+;
+  %1 = bitcast i16 %mask to <16 x i1>
+  %2 = call <16 x float> @llvm.x86.avx512.mask.expand.v16f32(<16 x float> %data, <16 x float> %passthru, <16 x i1> %1)
+  ret <16 x float> %2
+}
+
+define <16 x float> @test_maskz_expand_ps_512(<16 x float> %data, i16 %mask) #0 {
+; CHECK-LABEL: @test_maskz_expand_ps_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast i16 [[TMP1]] to <16 x i1>
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast i16 [[MASK:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <16 x i1> [[TMP3]] to i16
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i16 [[TMP6]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP7:%.*]], label [[TMP8:%.*]], !prof [[PROF1]]
+; CHECK:       7:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       8:
+; CHECK-NEXT:    [[TMP9:%.*]] = call <16 x float> @llvm.x86.avx512.mask.expand.v16f32(<16 x float> [[DATA:%.*]], <16 x float> zeroinitializer, <16 x i1> [[TMP4]])
+; CHECK-NEXT:    store <16 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x float> [[TMP9]]
+;
+  %1 = bitcast i16 %mask to <16 x i1>
+  %2 = call <16 x float> @llvm.x86.avx512.mask.expand.v16f32(<16 x float> %data, <16 x float> zeroinitializer, <16 x i1> %1)
+  ret <16 x float> %2
+}
+
+define <8 x i64> @test_expand_q_512(<8 x i64> %data, <8 x i64> %extra_param) #0 {
+; CHECK-LABEL: @test_expand_q_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP7:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <8 x i64> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP3]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <8 x i64> [[TMP7]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP5:%.*]], label [[TMP6:%.*]], !prof [[PROF1]]
+; CHECK:       5:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       6:
+; CHECK-NEXT:    [[TMP2:%.*]] = call <8 x i64> @llvm.x86.avx512.mask.expand.v8i64(<8 x i64> [[DATA:%.*]], <8 x i64> [[EXTRA_PARAM:%.*]], <8 x i1> splat (i1 true))
+; CHECK-NEXT:    store <8 x i64> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i64> [[TMP2]]
+;
+  %1 = call <8 x i64> @llvm.x86.avx512.mask.expand.v8i64(<8 x i64> %data, <8 x i64> %extra_param, <8 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>)
+  ret <8 x i64> %1
+}
+
+define <8 x i64> @test_mask_expand_q_512(<8 x i64> %data, <8 x i64> %passthru, i8 %mask) #0 {
+; CHECK-LABEL: @test_mask_expand_q_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast i8 [[TMP1]] to <8 x i1>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <8 x i64> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP6]], 0
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <8 x i64> [[TMP3]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP7]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <8 x i1> [[TMP4]] to i8
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i8 [[TMP8]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR3]], label [[TMP9:%.*]], label [[TMP10:%.*]], !prof [[PROF1]]
+; CHECK:       9:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       10:
+; CHECK-NEXT:    [[TMP11:%.*]] = call <8 x i64> @llvm.x86.avx512.mask.expand.v8i64(<8 x i64> [[DATA:%.*]], <8 x i64> [[PASSTHRU:%.*]], <8 x i1> [[TMP5]])
+; CHECK-NEXT:    store <8 x i64> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i64> [[TMP11]]
+;
+  %1 = bitcast i8 %mask to <8 x i1>
+  %2 = call <8 x i64> @llvm.x86.avx512.mask.expand.v8i64(<8 x i64> %data, <8 x i64> %passthru, <8 x i1> %1)
+  ret <8 x i64> %2
+}
+
+define <8 x i64> @test_maskz_expand_q_512(<8 x i64> %data, i8 %mask) #0 {
+; CHECK-LABEL: @test_maskz_expand_q_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast i8 [[TMP1]] to <8 x i1>
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <8 x i64> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <8 x i1> [[TMP3]] to i8
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i8 [[TMP6]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP7:%.*]], label [[TMP8:%.*]], !prof [[PROF1]]
+; CHECK:       7:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       8:
+; CHECK-NEXT:    [[TMP9:%.*]] = call <8 x i64> @llvm.x86.avx512.mask.expand.v8i64(<8 x i64> [[DATA:%.*]], <8 x i64> zeroinitializer, <8 x i1> [[TMP4]])
+; CHECK-NEXT:    store <8 x i64> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i64> [[TMP9]]
+;
+  %1 = bitcast i8 %mask to <8 x i1>
+  %2 = call <8 x i64> @llvm.x86.avx512.mask.expand.v8i64(<8 x i64> %data, <8 x i64> zeroinitializer, <8 x i1> %1)
+  ret <8 x i64> %2
+}
+
+define <16 x i32> @test_expand_d_512(<16 x i32> %data, <16 x i32> %extra_param) #0 {
+; CHECK-LABEL: @test_expand_d_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP7:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP3]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <16 x i32> [[TMP7]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP5:%.*]], label [[TMP6:%.*]], !prof [[PROF1]]
+; CHECK:       5:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       6:
+; CHECK-NEXT:    [[TMP2:%.*]] = call <16 x i32> @llvm.x86.avx512.mask.expand.v16i32(<16 x i32> [[DATA:%.*]], <16 x i32> [[EXTRA_PARAM:%.*]], <16 x i1> splat (i1 true))
+; CHECK-NEXT:    store <16 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i32> [[TMP2]]
+;
+  %1 = call <16 x i32> @llvm.x86.avx512.mask.expand.v16i32(<16 x i32> %data, <16 x i32> %extra_param, <16 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>)
+  ret <16 x i32> %1
+}
+
+define <16 x i32> @test_mask_expand_d_512(<16 x i32> %data, <16 x i32> %passthru, i16 %mask) #0 {
+; CHECK-LABEL: @test_mask_expand_d_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast i16 [[TMP1]] to <16 x i1>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i16 [[MASK:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP6]], 0
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <16 x i32> [[TMP3]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP7]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <16 x i1> [[TMP4]] to i16
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i16 [[TMP8]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR3]], label [[TMP9:%.*]], label [[TMP10:%.*]], !prof [[PROF1]]
+; CHECK:       9:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       10:
+; CHECK-NEXT:    [[TMP11:%.*]] = call <16 x i32> @llvm.x86.avx512.mask.expand.v16i32(<16 x i32> [[DATA:%.*]], <16 x i32> [[PASSTHRU:%.*]], <16 x i1> [[TMP5]])
+; CHECK-NEXT:    store <16 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i32> [[TMP11]]
+;
+  %1 = bitcast i16 %mask to <16 x i1>
+  %2 = call <16 x i32> @llvm.x86.avx512.mask.expand.v16i32(<16 x i32> %data, <16 x i32> %passthru, <16 x i1> %1)
+  ret <16 x i32> %2
+}
+
+define <16 x i32> @test_maskz_expand_d_512(<16 x i32> %data, i16 %mask) #0 {
+; CHECK-LABEL: @test_maskz_expand_d_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast i16 [[TMP1]] to <16 x i1>
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast i16 [[MASK:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <16 x i1> [[TMP3]] to i16
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i16 [[TMP6]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP7:%.*]], label [[TMP8:%.*]], !prof [[PROF1]]
+; CHECK:       7:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       8:
+; CHECK-NEXT:    [[TMP9:%.*]] = call <16 x i32> @llvm.x86.avx512.mask.expand.v16i32(<16 x i32> [[DATA:%.*]], <16 x i32> zeroinitializer, <16 x i1> [[TMP4]])
+; CHECK-NEXT:    store <16 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i32> [[TMP9]]
+;
+  %1 = bitcast i16 %mask to <16 x i1>
+  %2 = call <16 x i32> @llvm.x86.avx512.mask.expand.v16i32(<16 x i32> %data, <16 x i32> zeroinitializer, <16 x i1> %1)
+  ret <16 x i32> %2
+}
+
+define <16 x float> @test_rcp_ps_512(<16 x float> %a0) #0 {
+; CHECK-LABEL: @test_rcp_ps_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP2]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP3:%.*]], label [[TMP4:%.*]], !prof [[PROF1]]
+; CHECK:       3:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       4:
+; CHECK-NEXT:    [[RES:%.*]] = call <16 x float> @llvm.x86.avx512.rcp14.ps.512(<16 x float> [[A0:%.*]], <16 x float> zeroinitializer, i16 -1)
+; CHECK-NEXT:    store <16 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x float> [[RES]]
+;
+  %res = call <16 x float> @llvm.x86.avx512.rcp14.ps.512(<16 x float> %a0, <16 x float> zeroinitializer, i16 -1) ; <<16 x float>> [#uses=1]
+  ret <16 x float> %res
+}
+declare <16 x float> @llvm.x86.avx512.rcp14.ps.512(<16 x float>, <16 x float>, i16) nounwind readnone
+
+define <8 x double> @test_rcp_pd_512(<8 x double> %a0) #0 {
+; CHECK-LABEL: @test_rcp_pd_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <8 x i64> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP2]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP3:%.*]], label [[TMP4:%.*]], !prof [[PROF1]]
+; CHECK:       3:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       4:
+; CHECK-NEXT:    [[RES:%.*]] = call <8 x double> @llvm.x86.avx512.rcp14.pd.512(<8 x double> [[A0:%.*]], <8 x double> zeroinitializer, i8 -1)
+; CHECK-NEXT:    store <8 x i64> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x double> [[RES]]
+;
+  %res = call <8 x double> @llvm.x86.avx512.rcp14.pd.512(<8 x double> %a0, <8 x double> zeroinitializer, i8 -1) ; <<8 x double>> [#uses=1]
+  ret <8 x double> %res
+}
+declare <8 x double> @llvm.x86.avx512.rcp14.pd.512(<8 x double>, <8 x double>, i8) nounwind readnone
+
+declare <2 x double> @llvm.x86.avx512.mask.rndscale.sd(<2 x double>, <2 x double>, <2 x double>, i8, i32, i32)
+
+define <2 x double> @test_rndscale_sd(<2 x double> %a, <2 x double> %b, <2 x double> %extra_param) #0 {
+; CHECK-LABEL: @test_rndscale_sd(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP5:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <2 x i64> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP3]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <2 x i64> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <2 x i64> [[TMP5]] to i128
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i128 [[TMP6]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR3]], label [[TMP7:%.*]], label [[TMP8:%.*]], !prof [[PROF1]]
+; CHECK:       7:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       8:
+; CHECK-NEXT:    [[RES:%.*]] = call <2 x double> @llvm.x86.avx512.mask.rndscale.sd(<2 x double> [[A:%.*]], <2 x double> [[B:%.*]], <2 x double> [[EXTRA_PARAM:%.*]], i8 -1, i32 11, i32 4)
+; CHECK-NEXT:    store <2 x i64> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x double> [[RES]]
+;
+  %res = call <2 x double> @llvm.x86.avx512.mask.rndscale.sd(<2 x double> %a, <2 x double> %b, <2 x double> %extra_param, i8 -1, i32 11, i32 4)
+  ret <2 x double>%res
+}
+
+define <2 x double> @test_rndscale_sd_mask(<2 x double> %a, <2 x double> %b, <2 x double> %c, i8 %mask) #0 {
+; CHECK-LABEL: @test_rndscale_sd_mask(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 48) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <2 x i64> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <2 x i64> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP6]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <2 x i64> [[TMP3]] to i128
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i128 [[TMP7]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    [[_MSCMP4:%.*]] = icmp ne i8 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR5:%.*]] = or i1 [[_MSOR3]], [[_MSCMP4]]
+; CHECK-NEXT:    br i1 [[_MSOR5]], label [[TMP8:%.*]], label [[TMP9:%.*]], !prof [[PROF1]]
+; CHECK:       8:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       9:
+; CHECK-NEXT:    [[RES:%.*]] = call <2 x double> @llvm.x86.avx512.mask.rndscale.sd(<2 x double> [[A:%.*]], <2 x double> [[B:%.*]], <2 x double> [[C:%.*]], i8 [[MASK:%.*]], i32 11, i32 4)
+; CHECK-NEXT:    store <2 x i64> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x double> [[RES]]
+;
+  %res = call <2 x double> @llvm.x86.avx512.mask.rndscale.sd(<2 x double> %a, <2 x double> %b, <2 x double> %c, i8 %mask, i32 11, i32 4)
+  ret <2 x double>%res
+}
+
+define <2 x double> @test_rndscale_sd_mask_load(<2 x double> %a, ptr %bptr, <2 x double> %c, i8 %mask) #0 {
+; CHECK-LABEL: @test_rndscale_sd_mask_load(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 24) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 40) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP5:%.*]], label [[TMP6:%.*]], !prof [[PROF1]]
+; CHECK:       5:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       6:
+; CHECK-NEXT:    [[B:%.*]] = load <2 x double>, ptr [[BPTR:%.*]], align 16
+; CHECK-NEXT:    [[TMP7:%.*]] = ptrtoint ptr [[BPTR]] to i64
+; CHECK-NEXT:    [[TMP8:%.*]] = xor i64 [[TMP7]], 87960930222080
+; CHECK-NEXT:    [[TMP9:%.*]] = inttoptr i64 [[TMP8]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <2 x i64>, ptr [[TMP9]], align 16
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast <2 x i64> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP10]], 0
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <2 x i64> [[_MSLD]] to i128
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i128 [[TMP11]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP1]], [[_MSCMP2]]
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <2 x i64> [[TMP3]] to i128
+; CHECK-NEXT:    [[_MSCMP3:%.*]] = icmp ne i128 [[TMP12]], 0
+; CHECK-NEXT:    [[_MSOR4:%.*]] = or i1 [[_MSOR]], [[_MSCMP3]]
+; CHECK-NEXT:    [[_MSCMP5:%.*]] = icmp ne i8 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR6:%.*]] = or i1 [[_MSOR4]], [[_MSCMP5]]
+; CHECK-NEXT:    br i1 [[_MSOR6]], label [[TMP13:%.*]], label [[TMP14:%.*]], !prof [[PROF1]]
+; CHECK:       13:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       14:
+; CHECK-NEXT:    [[RES:%.*]] = call <2 x double> @llvm.x86.avx512.mask.rndscale.sd(<2 x double> [[A:%.*]], <2 x double> [[B]], <2 x double> [[C:%.*]], i8 [[MASK:%.*]], i32 11, i32 4)
+; CHECK-NEXT:    store <2 x i64> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x double> [[RES]]
+;
+  %b = load <2 x double>, ptr %bptr
+  %res = call <2 x double> @llvm.x86.avx512.mask.rndscale.sd(<2 x double> %a, <2 x double> %b, <2 x double> %c, i8 %mask, i32 11, i32 4)
+  ret <2 x double>%res
+}
+
+define <2 x double> @test_rndscale_sd_maskz(<2 x double> %a, <2 x double> %b, i8 %mask) #0 {
+; CHECK-LABEL: @test_rndscale_sd_maskz(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <2 x i64> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <2 x i64> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i8 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR3]], label [[TMP6:%.*]], label [[TMP7:%.*]], !prof [[PROF1]]
+; CHECK:       6:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       7:
+; CHECK-NEXT:    [[RES:%.*]] = call <2 x double> @llvm.x86.avx512.mask.rndscale.sd(<2 x double> [[A:%.*]], <2 x double> [[B:%.*]], <2 x double> zeroinitializer, i8 [[MASK:%.*]], i32 11, i32 4)
+; CHECK-NEXT:    store <2 x i64> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x double> [[RES]]
+;
+  %res = call <2 x double> @llvm.x86.avx512.mask.rndscale.sd(<2 x double> %a, <2 x double> %b, <2 x double> zeroinitializer, i8 %mask, i32 11, i32 4)
+  ret <2 x double>%res
+}
+
+declare <4 x float> @llvm.x86.avx512.mask.rndscale.ss(<4 x float>, <4 x float>, <4 x float>, i8, i32, i32)
+
+define <4 x float> @test_rndscale_ss(<4 x float> %a, <4 x float> %b, <4 x float> %extra_param) #0 {
+; CHECK-LABEL: @test_rndscale_ss(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP5:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <4 x i32> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP3]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <4 x i32> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <4 x i32> [[TMP5]] to i128
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i128 [[TMP6]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR3]], label [[TMP7:%.*]], label [[TMP8:%.*]], !prof [[PROF1]]
+; CHECK:       7:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       8:
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.x86.avx512.mask.rndscale.ss(<4 x float> [[A:%.*]], <4 x float> [[B:%.*]], <4 x float> [[EXTRA_PARAM:%.*]], i8 -1, i32 11, i32 4)
+; CHECK-NEXT:    store <4 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x float> [[RES]]
+;
+  %res = call <4 x float> @llvm.x86.avx512.mask.rndscale.ss(<4 x float> %a, <4 x float> %b, <4 x float> %extra_param, i8 -1, i32 11, i32 4)
+  ret <4 x float>%res
+}
+
+define <4 x float> @test_rndscale_ss_load(<4 x float> %a, ptr %bptr, <4 x float> %extra_param) #0 {
+; CHECK-LABEL: @test_rndscale_ss_load(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 24) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP4:%.*]], label [[TMP10:%.*]], !prof [[PROF1]]
+; CHECK:       4:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       5:
+; CHECK-NEXT:    [[B:%.*]] = load <4 x float>, ptr [[BPTR:%.*]], align 16
+; CHECK-NEXT:    [[TMP5:%.*]] = ptrtoint ptr [[BPTR]] to i64
+; CHECK-NEXT:    [[TMP6:%.*]] = xor i64 [[TMP5]], 87960930222080
+; CHECK-NEXT:    [[TMP7:%.*]] = inttoptr i64 [[TMP6]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <4 x i32>, ptr [[TMP7]], align 16
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <4 x i32> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP8]], 0
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <4 x i32> [[_MSLD]] to i128
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i128 [[TMP9]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP1]], [[_MSCMP2]]
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <4 x i32> [[TMP3]] to i128
+; CHECK-NEXT:    [[_MSCMP3:%.*]] = icmp ne i128 [[TMP11]], 0
+; CHECK-NEXT:    [[_MSOR4:%.*]] = or i1 [[_MSOR]], [[_MSCMP3]]
+; CHECK-NEXT:    br i1 [[_MSOR4]], label [[TMP12:%.*]], label [[TMP13:%.*]], !prof [[PROF1]]
+; CHECK:       12:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       13:
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.x86.avx512.mask.rndscale.ss(<4 x float> [[A:%.*]], <4 x float> [[B]], <4 x float> [[EXTRA_PARAM:%.*]], i8 -1, i32 11, i32 4)
+; CHECK-NEXT:    store <4 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x float> [[RES]]
+;
+  %b = load <4 x float>, ptr %bptr
+  %res = call <4 x float> @llvm.x86.avx512.mask.rndscale.ss(<4 x float> %a, <4 x float> %b, <4 x float> %extra_param, i8 -1, i32 11, i32 4)
+  ret <4 x float>%res
+}
+
+define <4 x float> @test_rndscale_ss_mask(<4 x float> %a, <4 x float> %b, <4 x float> %c, i8 %mask) #0 {
+; CHECK-LABEL: @test_rndscale_ss_mask(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 48) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <4 x i32> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <4 x i32> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP6]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <4 x i32> [[TMP3]] to i128
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i128 [[TMP7]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    [[_MSCMP4:%.*]] = icmp ne i8 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR5:%.*]] = or i1 [[_MSOR3]], [[_MSCMP4]]
+; CHECK-NEXT:    br i1 [[_MSOR5]], label [[TMP8:%.*]], label [[TMP9:%.*]], !prof [[PROF1]]
+; CHECK:       8:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       9:
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.x86.avx512.mask.rndscale.ss(<4 x float> [[A:%.*]], <4 x float> [[B:%.*]], <4 x float> [[C:%.*]], i8 [[MASK:%.*]], i32 11, i32 4)
+; CHECK-NEXT:    store <4 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x float> [[RES]]
+;
+  %res = call <4 x float> @llvm.x86.avx512.mask.rndscale.ss(<4 x float> %a, <4 x float> %b, <4 x float> %c, i8 %mask, i32 11, i32 4)
+  ret <4 x float>%res
+}
+
+define <4 x float> @test_rndscale_ss_maskz(<4 x float> %a, <4 x float> %b, i8 %mask) #0 {
+; CHECK-LABEL: @test_rndscale_ss_maskz(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <4 x i32> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <4 x i32> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i8 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR3]], label [[TMP6:%.*]], label [[TMP7:%.*]], !prof [[PROF1]]
+; CHECK:       6:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       7:
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.x86.avx512.mask.rndscale.ss(<4 x float> [[A:%.*]], <4 x float> [[B:%.*]], <4 x float> zeroinitializer, i8 [[MASK:%.*]], i32 11, i32 4)
+; CHECK-NEXT:    store <4 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x float> [[RES]]
+;
+  %res = call <4 x float> @llvm.x86.avx512.mask.rndscale.ss(<4 x float> %a, <4 x float> %b, <4 x float> zeroinitializer, i8 %mask, i32 11, i32 4)
+  ret <4 x float>%res
+}
+
+declare <8 x double> @llvm.x86.avx512.mask.rndscale.pd.512(<8 x double>, i32, <8 x double>, i8, i32)
+
+define <8 x double> @test7(<8 x double> %a) #0 {
+; CHECK-LABEL: @test7(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <8 x i64> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP2]], 0
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <8 x i64> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP4:%.*]], label [[TMP5:%.*]], !prof [[PROF1]]
+; CHECK:       4:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       5:
+; CHECK-NEXT:    [[RES:%.*]] = call <8 x double> @llvm.x86.avx512.mask.rndscale.pd.512(<8 x double> [[A:%.*]], i32 11, <8 x double> [[A]], i8 -1, i32 4)
+; CHECK-NEXT:    store <8 x i64> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x double> [[RES]]
+;
+  %res = call <8 x double> @llvm.x86.avx512.mask.rndscale.pd.512(<8 x double> %a, i32 11, <8 x double> %a, i8 -1, i32 4)
+  ret <8 x double>%res
+}
+
+declare <16 x float> @llvm.x86.avx512.mask.rndscale.ps.512(<16 x float>, i32, <16 x float>, i16, i32)
+
+define <16 x float> @test8(<16 x float> %a) #0 {
+; CHECK-LABEL: @test8(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP2]], 0
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP4:%.*]], label [[TMP5:%.*]], !prof [[PROF1]]
+; CHECK:       4:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       5:
+; CHECK-NEXT:    [[RES:%.*]] = call <16 x float> @llvm.x86.avx512.mask.rndscale.ps.512(<16 x float> [[A:%.*]], i32 11, <16 x float> [[A]], i16 -1, i32 4)
+; CHECK-NEXT:    store <16 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x float> [[RES]]
+;
+  %res = call <16 x float> @llvm.x86.avx512.mask.rndscale.ps.512(<16 x float> %a, i32 11, <16 x float> %a, i16 -1, i32 4)
+  ret <16 x float>%res
+}
+
+define <16 x float> @test_rsqrt_ps_512(<16 x float> %a0) #0 {
+; CHECK-LABEL: @test_rsqrt_ps_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP2]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP3:%.*]], label [[TMP4:%.*]], !prof [[PROF1]]
+; CHECK:       3:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       4:
+; CHECK-NEXT:    [[RES:%.*]] = call <16 x float> @llvm.x86.avx512.rsqrt14.ps.512(<16 x float> [[A0:%.*]], <16 x float> zeroinitializer, i16 -1)
+; CHECK-NEXT:    store <16 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x float> [[RES]]
+;
+  %res = call <16 x float> @llvm.x86.avx512.rsqrt14.ps.512(<16 x float> %a0, <16 x float> zeroinitializer, i16 -1) ; <<16 x float>> [#uses=1]
+  ret <16 x float> %res
+}
+declare <16 x float> @llvm.x86.avx512.rsqrt14.ps.512(<16 x float>, <16 x float>, i16) nounwind readnone
+
+define <8 x double> @test_sqrt_pd_512(<8 x double> %a0) #0 {
+; CHECK-LABEL: @test_sqrt_pd_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = call <8 x double> @llvm.sqrt.v8f64(<8 x double> [[A0:%.*]])
+; CHECK-NEXT:    store <8 x i64> [[TMP1]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x double> [[TMP2]]
+;
+  %1 = call <8 x double> @llvm.sqrt.v8f64(<8 x double> %a0)
+  ret <8 x double> %1
+}
+
+define <8 x double> @test_mask_sqrt_pd_512(<8 x double> %a0, <8 x double> %passthru, i8 %mask) #0 {
+; CHECK-LABEL: @test_mask_sqrt_pd_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = call <8 x double> @llvm.sqrt.v8f64(<8 x double> [[A0:%.*]])
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i8 [[TMP2]] to <8 x i1>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP7:%.*]] = select <8 x i1> [[TMP6]], <8 x i64> [[TMP1]], <8 x i64> [[TMP3]]
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <8 x double> [[TMP4]] to <8 x i64>
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <8 x double> [[PASSTHRU:%.*]] to <8 x i64>
+; CHECK-NEXT:    [[TMP10:%.*]] = xor <8 x i64> [[TMP8]], [[TMP9]]
+; CHECK-NEXT:    [[TMP11:%.*]] = or <8 x i64> [[TMP10]], [[TMP1]]
+; CHECK-NEXT:    [[TMP12:%.*]] = or <8 x i64> [[TMP11]], [[TMP3]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP5]], <8 x i64> [[TMP12]], <8 x i64> [[TMP7]]
+; CHECK-NEXT:    [[TMP13:%.*]] = select <8 x i1> [[TMP6]], <8 x double> [[TMP4]], <8 x double> [[PASSTHRU]]
+; CHECK-NEXT:    store <8 x i64> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x double> [[TMP13]]
+;
+  %1 = call <8 x double> @llvm.sqrt.v8f64(<8 x double> %a0)
+  %2 = bitcast i8 %mask to <8 x i1>
+  %3 = select <8 x i1> %2, <8 x double> %1, <8 x double> %passthru
+  ret <8 x double> %3
+}
+
+define <8 x double> @test_maskz_sqrt_pd_512(<8 x double> %a0, i8 %mask) #0 {
+; CHECK-LABEL: @test_maskz_sqrt_pd_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = call <8 x double> @llvm.sqrt.v8f64(<8 x double> [[A0:%.*]])
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast i8 [[TMP2]] to <8 x i1>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP6:%.*]] = select <8 x i1> [[TMP5]], <8 x i64> [[TMP1]], <8 x i64> zeroinitializer
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <8 x double> [[TMP3]] to <8 x i64>
+; CHECK-NEXT:    [[TMP8:%.*]] = xor <8 x i64> [[TMP7]], zeroinitializer
+; CHECK-NEXT:    [[TMP9:%.*]] = or <8 x i64> [[TMP8]], [[TMP1]]
+; CHECK-NEXT:    [[TMP10:%.*]] = or <8 x i64> [[TMP9]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP4]], <8 x i64> [[TMP10]], <8 x i64> [[TMP6]]
+; CHECK-NEXT:    [[TMP11:%.*]] = select <8 x i1> [[TMP5]], <8 x double> [[TMP3]], <8 x double> zeroinitializer
+; CHECK-NEXT:    store <8 x i64> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x double> [[TMP11]]
+;
+  %1 = call <8 x double> @llvm.sqrt.v8f64(<8 x double> %a0)
+  %2 = bitcast i8 %mask to <8 x i1>
+  %3 = select <8 x i1> %2, <8 x double> %1, <8 x double> zeroinitializer
+  ret <8 x double> %3
+}
+declare <8 x double> @llvm.sqrt.v8f64(<8 x double>)
+
+define <8 x double> @test_sqrt_round_pd_512(<8 x double> %a0) #0 {
+; CHECK-LABEL: @test_sqrt_round_pd_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <8 x i64> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP2]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP3:%.*]], label [[TMP4:%.*]], !prof [[PROF1]]
+; CHECK:       3:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       4:
+; CHECK-NEXT:    [[TMP5:%.*]] = call <8 x double> @llvm.x86.avx512.sqrt.pd.512(<8 x double> [[A0:%.*]], i32 11)
+; CHECK-NEXT:    store <8 x i64> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x double> [[TMP5]]
+;
+  %1 = call <8 x double> @llvm.x86.avx512.sqrt.pd.512(<8 x double> %a0, i32 11)
+  ret <8 x double> %1
+}
+
+define <8 x double> @test_mask_sqrt_round_pd_512(<8 x double> %a0, <8 x double> %passthru, i8 %mask) #0 {
+; CHECK-LABEL: @test_mask_sqrt_round_pd_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <8 x i64> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP4]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP5:%.*]], label [[TMP6:%.*]], !prof [[PROF1]]
+; CHECK:       5:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       6:
+; CHECK-NEXT:    [[TMP7:%.*]] = call <8 x double> @llvm.x86.avx512.sqrt.pd.512(<8 x double> [[A0:%.*]], i32 11)
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast i8 [[TMP2]] to <8 x i1>
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP10:%.*]] = select <8 x i1> [[TMP9]], <8 x i64> zeroinitializer, <8 x i64> [[TMP3]]
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <8 x double> [[TMP7]] to <8 x i64>
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <8 x double> [[PASSTHRU:%.*]] to <8 x i64>
+; CHECK-NEXT:    [[TMP13:%.*]] = xor <8 x i64> [[TMP11]], [[TMP12]]
+; CHECK-NEXT:    [[TMP14:%.*]] = or <8 x i64> [[TMP13]], zeroinitializer
+; CHECK-NEXT:    [[TMP15:%.*]] = or <8 x i64> [[TMP14]], [[TMP3]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP8]], <8 x i64> [[TMP15]], <8 x i64> [[TMP10]]
+; CHECK-NEXT:    [[TMP16:%.*]] = select <8 x i1> [[TMP9]], <8 x double> [[TMP7]], <8 x double> [[PASSTHRU]]
+; CHECK-NEXT:    store <8 x i64> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x double> [[TMP16]]
+;
+  %1 = call <8 x double> @llvm.x86.avx512.sqrt.pd.512(<8 x double> %a0, i32 11)
+  %2 = bitcast i8 %mask to <8 x i1>
+  %3 = select <8 x i1> %2, <8 x double> %1, <8 x double> %passthru
+  ret <8 x double> %3
+}
+
+define <8 x double> @test_maskz_sqrt_round_pd_512(<8 x double> %a0, i8 %mask) #0 {
+; CHECK-LABEL: @test_maskz_sqrt_round_pd_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <8 x i64> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP3]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP4:%.*]], label [[TMP5:%.*]], !prof [[PROF1]]
+; CHECK:       4:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       5:
+; CHECK-NEXT:    [[TMP6:%.*]] = call <8 x double> @llvm.x86.avx512.sqrt.pd.512(<8 x double> [[A0:%.*]], i32 11)
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast i8 [[TMP2]] to <8 x i1>
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP9:%.*]] = select <8 x i1> [[TMP8]], <8 x i64> zeroinitializer, <8 x i64> zeroinitializer
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast <8 x double> [[TMP6]] to <8 x i64>
+; CHECK-NEXT:    [[TMP11:%.*]] = xor <8 x i64> [[TMP10]], zeroinitializer
+; CHECK-NEXT:    [[TMP12:%.*]] = or <8 x i64> [[TMP11]], zeroinitializer
+; CHECK-NEXT:    [[TMP13:%.*]] = or <8 x i64> [[TMP12]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP7]], <8 x i64> [[TMP13]], <8 x i64> [[TMP9]]
+; CHECK-NEXT:    [[TMP14:%.*]] = select <8 x i1> [[TMP8]], <8 x double> [[TMP6]], <8 x double> zeroinitializer
+; CHECK-NEXT:    store <8 x i64> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x double> [[TMP14]]
+;
+  %1 = call <8 x double> @llvm.x86.avx512.sqrt.pd.512(<8 x double> %a0, i32 11)
+  %2 = bitcast i8 %mask to <8 x i1>
+  %3 = select <8 x i1> %2, <8 x double> %1, <8 x double> zeroinitializer
+  ret <8 x double> %3
+}
+declare <8 x double> @llvm.x86.avx512.sqrt.pd.512(<8 x double>, i32) nounwind readnone
+
+define <16 x float> @test_sqrt_ps_512(<16 x float> %a0) #0 {
+; CHECK-LABEL: @test_sqrt_ps_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = call <16 x float> @llvm.sqrt.v16f32(<16 x float> [[A0:%.*]])
+; CHECK-NEXT:    store <16 x i32> [[TMP1]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x float> [[TMP2]]
+;
+  %1 = call <16 x float> @llvm.sqrt.v16f32(<16 x float> %a0)
+  ret <16 x float> %1
+}
+
+define <16 x float> @test_mask_sqrt_ps_512(<16 x float> %a0, <16 x float> %passthru, i16 %mask) #0 {
+; CHECK-LABEL: @test_mask_sqrt_ps_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = call <16 x float> @llvm.sqrt.v16f32(<16 x float> [[A0:%.*]])
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i16 [[TMP2]] to <16 x i1>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i16 [[MASK:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP7:%.*]] = select <16 x i1> [[TMP6]], <16 x i32> [[TMP1]], <16 x i32> [[TMP3]]
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <16 x float> [[TMP4]] to <16 x i32>
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <16 x float> [[PASSTHRU:%.*]] to <16 x i32>
+; CHECK-NEXT:    [[TMP10:%.*]] = xor <16 x i32> [[TMP8]], [[TMP9]]
+; CHECK-NEXT:    [[TMP11:%.*]] = or <16 x i32> [[TMP10]], [[TMP1]]
+; CHECK-NEXT:    [[TMP12:%.*]] = or <16 x i32> [[TMP11]], [[TMP3]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP5]], <16 x i32> [[TMP12]], <16 x i32> [[TMP7]]
+; CHECK-NEXT:    [[TMP13:%.*]] = select <16 x i1> [[TMP6]], <16 x float> [[TMP4]], <16 x float> [[PASSTHRU]]
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x float> [[TMP13]]
+;
+  %1 = call <16 x float> @llvm.sqrt.v16f32(<16 x float> %a0)
+  %2 = bitcast i16 %mask to <16 x i1>
+  %3 = select <16 x i1> %2, <16 x float> %1, <16 x float> %passthru
+  ret <16 x float> %3
+}
+
+define <16 x float> @test_maskz_sqrt_ps_512(<16 x float> %a0, i16 %mask) #0 {
+; CHECK-LABEL: @test_maskz_sqrt_ps_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = call <16 x float> @llvm.sqrt.v16f32(<16 x float> [[A0:%.*]])
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast i16 [[TMP2]] to <16 x i1>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i16 [[MASK:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP6:%.*]] = select <16 x i1> [[TMP5]], <16 x i32> [[TMP1]], <16 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <16 x float> [[TMP3]] to <16 x i32>
+; CHECK-NEXT:    [[TMP8:%.*]] = xor <16 x i32> [[TMP7]], zeroinitializer
+; CHECK-NEXT:    [[TMP9:%.*]] = or <16 x i32> [[TMP8]], [[TMP1]]
+; CHECK-NEXT:    [[TMP10:%.*]] = or <16 x i32> [[TMP9]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP4]], <16 x i32> [[TMP10]], <16 x i32> [[TMP6]]
+; CHECK-NEXT:    [[TMP11:%.*]] = select <16 x i1> [[TMP5]], <16 x float> [[TMP3]], <16 x float> zeroinitializer
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x float> [[TMP11]]
+;
+  %1 = call <16 x float> @llvm.sqrt.v16f32(<16 x float> %a0)
+  %2 = bitcast i16 %mask to <16 x i1>
+  %3 = select <16 x i1> %2, <16 x float> %1, <16 x float> zeroinitializer
+  ret <16 x float> %3
+}
+declare <16 x float> @llvm.sqrt.v16f32(<16 x float>)
+
+define <16 x float> @test_sqrt_round_ps_512(<16 x float> %a0) #0 {
+; CHECK-LABEL: @test_sqrt_round_ps_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP2]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP3:%.*]], label [[TMP4:%.*]], !prof [[PROF1]]
+; CHECK:       3:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       4:
+; CHECK-NEXT:    [[TMP5:%.*]] = call <16 x float> @llvm.x86.avx512.sqrt.ps.512(<16 x float> [[A0:%.*]], i32 11)
+; CHECK-NEXT:    store <16 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x float> [[TMP5]]
+;
+  %1 = call <16 x float> @llvm.x86.avx512.sqrt.ps.512(<16 x float> %a0, i32 11)
+  ret <16 x float> %1
+}
+
+define <16 x float> @test_mask_sqrt_round_ps_512(<16 x float> %a0, <16 x float> %passthru, i16 %mask) #0 {
+; CHECK-LABEL: @test_mask_sqrt_round_ps_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP4]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP5:%.*]], label [[TMP6:%.*]], !prof [[PROF1]]
+; CHECK:       5:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       6:
+; CHECK-NEXT:    [[TMP7:%.*]] = call <16 x float> @llvm.x86.avx512.sqrt.ps.512(<16 x float> [[A0:%.*]], i32 11)
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast i16 [[TMP2]] to <16 x i1>
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast i16 [[MASK:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP10:%.*]] = select <16 x i1> [[TMP9]], <16 x i32> zeroinitializer, <16 x i32> [[TMP3]]
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <16 x float> [[TMP7]] to <16 x i32>
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <16 x float> [[PASSTHRU:%.*]] to <16 x i32>
+; CHECK-NEXT:    [[TMP13:%.*]] = xor <16 x i32> [[TMP11]], [[TMP12]]
+; CHECK-NEXT:    [[TMP14:%.*]] = or <16 x i32> [[TMP13]], zeroinitializer
+; CHECK-NEXT:    [[TMP15:%.*]] = or <16 x i32> [[TMP14]], [[TMP3]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP8]], <16 x i32> [[TMP15]], <16 x i32> [[TMP10]]
+; CHECK-NEXT:    [[TMP16:%.*]] = select <16 x i1> [[TMP9]], <16 x float> [[TMP7]], <16 x float> [[PASSTHRU]]
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x float> [[TMP16]]
+;
+  %1 = call <16 x float> @llvm.x86.avx512.sqrt.ps.512(<16 x float> %a0, i32 11)
+  %2 = bitcast i16 %mask to <16 x i1>
+  %3 = select <16 x i1> %2, <16 x float> %1, <16 x float> %passthru
+  ret <16 x float> %3
+}
+
+define <16 x float> @test_maskz_sqrt_round_ps_512(<16 x float> %a0, i16 %mask) #0 {
+; CHECK-LABEL: @test_maskz_sqrt_round_ps_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP3]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP4:%.*]], label [[TMP5:%.*]], !prof [[PROF1]]
+; CHECK:       4:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       5:
+; CHECK-NEXT:    [[TMP6:%.*]] = call <16 x float> @llvm.x86.avx512.sqrt.ps.512(<16 x float> [[A0:%.*]], i32 11)
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast i16 [[TMP2]] to <16 x i1>
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast i16 [[MASK:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP9:%.*]] = select <16 x i1> [[TMP8]], <16 x i32> zeroinitializer, <16 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast <16 x float> [[TMP6]] to <16 x i32>
+; CHECK-NEXT:    [[TMP11:%.*]] = xor <16 x i32> [[TMP10]], zeroinitializer
+; CHECK-NEXT:    [[TMP12:%.*]] = or <16 x i32> [[TMP11]], zeroinitializer
+; CHECK-NEXT:    [[TMP13:%.*]] = or <16 x i32> [[TMP12]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP7]], <16 x i32> [[TMP13]], <16 x i32> [[TMP9]]
+; CHECK-NEXT:    [[TMP14:%.*]] = select <16 x i1> [[TMP8]], <16 x float> [[TMP6]], <16 x float> zeroinitializer
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x float> [[TMP14]]
+;
+  %1 = call <16 x float> @llvm.x86.avx512.sqrt.ps.512(<16 x float> %a0, i32 11)
+  %2 = bitcast i16 %mask to <16 x i1>
+  %3 = select <16 x i1> %2, <16 x float> %1, <16 x float> zeroinitializer
+  ret <16 x float> %3
+}
+declare <16 x float> @llvm.x86.avx512.sqrt.ps.512(<16 x float>, i32) nounwind readnone
+
+define <8 x double> @test_getexp_pd_512(<8 x double> %a0) #0 {
+; CHECK-LABEL: @test_getexp_pd_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <8 x i64> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP2]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP3:%.*]], label [[TMP4:%.*]], !prof [[PROF1]]
+; CHECK:       3:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       4:
+; CHECK-NEXT:    [[RES:%.*]] = call <8 x double> @llvm.x86.avx512.mask.getexp.pd.512(<8 x double> [[A0:%.*]], <8 x double> zeroinitializer, i8 -1, i32 4)
+; CHECK-NEXT:    store <8 x i64> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x double> [[RES]]
+;
+  %res = call <8 x double> @llvm.x86.avx512.mask.getexp.pd.512(<8 x double> %a0,  <8 x double> zeroinitializer, i8 -1, i32 4)
+  ret <8 x double> %res
+}
+define <8 x double> @test_getexp_round_pd_512(<8 x double> %a0) #0 {
+; CHECK-LABEL: @test_getexp_round_pd_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <8 x i64> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP2]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP3:%.*]], label [[TMP4:%.*]], !prof [[PROF1]]
+; CHECK:       3:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       4:
+; CHECK-NEXT:    [[RES:%.*]] = call <8 x double> @llvm.x86.avx512.mask.getexp.pd.512(<8 x double> [[A0:%.*]], <8 x double> zeroinitializer, i8 -1, i32 12)
+; CHECK-NEXT:    store <8 x i64> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x double> [[RES]]
+;
+  %res = call <8 x double> @llvm.x86.avx512.mask.getexp.pd.512(<8 x double> %a0,  <8 x double> zeroinitializer, i8 -1, i32 12)
+  ret <8 x double> %res
+}
+declare <8 x double> @llvm.x86.avx512.mask.getexp.pd.512(<8 x double>, <8 x double>, i8, i32) nounwind readnone
+
+define <16 x float> @test_getexp_ps_512(<16 x float> %a0) #0 {
+; CHECK-LABEL: @test_getexp_ps_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP2]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP3:%.*]], label [[TMP4:%.*]], !prof [[PROF1]]
+; CHECK:       3:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       4:
+; CHECK-NEXT:    [[RES:%.*]] = call <16 x float> @llvm.x86.avx512.mask.getexp.ps.512(<16 x float> [[A0:%.*]], <16 x float> zeroinitializer, i16 -1, i32 4)
+; CHECK-NEXT:    store <16 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x float> [[RES]]
+;
+  %res = call <16 x float> @llvm.x86.avx512.mask.getexp.ps.512(<16 x float> %a0, <16 x float> zeroinitializer, i16 -1, i32 4)
+  ret <16 x float> %res
+}
+
+define <16 x float> @test_getexp_round_ps_512(<16 x float> %a0) #0 {
+; CHECK-LABEL: @test_getexp_round_ps_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP2]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP3:%.*]], label [[TMP4:%.*]], !prof [[PROF1]]
+; CHECK:       3:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       4:
+; CHECK-NEXT:    [[RES:%.*]] = call <16 x float> @llvm.x86.avx512.mask.getexp.ps.512(<16 x float> [[A0:%.*]], <16 x float> zeroinitializer, i16 -1, i32 8)
+; CHECK-NEXT:    store <16 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x float> [[RES]]
+;
+  %res = call <16 x float> @llvm.x86.avx512.mask.getexp.ps.512(<16 x float> %a0, <16 x float> zeroinitializer, i16 -1, i32 8)
+  ret <16 x float> %res
+}
+declare <16 x float> @llvm.x86.avx512.mask.getexp.ps.512(<16 x float>, <16 x float>, i16, i32) nounwind readnone
+
+declare <4 x float> @llvm.x86.avx512.mask.sqrt.ss(<4 x float>, <4 x float>, <4 x float>, i8, i32) nounwind readnone
+
+define <4 x float> @test_sqrt_ss(<4 x float> %a0, <4 x float> %a1, <4 x float> %a2, i8 %mask) #0 {
+; CHECK-LABEL: @test_sqrt_ss(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 48) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <4 x i32> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <4 x i32> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP6]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <4 x i32> [[TMP3]] to i128
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i128 [[TMP7]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    [[_MSCMP4:%.*]] = icmp ne i8 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR5:%.*]] = or i1 [[_MSOR3]], [[_MSCMP4]]
+; CHECK-NEXT:    br i1 [[_MSOR5]], label [[TMP8:%.*]], label [[TMP9:%.*]], !prof [[PROF1]]
+; CHECK:       8:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       9:
+; CHECK-NEXT:    [[RES0:%.*]] = call <4 x float> @llvm.x86.avx512.mask.sqrt.ss(<4 x float> [[A0:%.*]], <4 x float> [[A1:%.*]], <4 x float> [[A2:%.*]], i8 [[MASK:%.*]], i32 4)
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast <4 x i32> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP6:%.*]] = icmp ne i128 [[TMP10]], 0
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <4 x i32> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP7:%.*]] = icmp ne i128 [[TMP11]], 0
+; CHECK-NEXT:    [[_MSOR8:%.*]] = or i1 [[_MSCMP6]], [[_MSCMP7]]
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <4 x i32> [[TMP3]] to i128
+; CHECK-NEXT:    [[_MSCMP9:%.*]] = icmp ne i128 [[TMP12]], 0
+; CHECK-NEXT:    [[_MSOR10:%.*]] = or i1 [[_MSOR8]], [[_MSCMP9]]
+; CHECK-NEXT:    [[_MSCMP11:%.*]] = icmp ne i8 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR12:%.*]] = or i1 [[_MSOR10]], [[_MSCMP11]]
+; CHECK-NEXT:    br i1 [[_MSOR12]], label [[TMP13:%.*]], label [[TMP14:%.*]], !prof [[PROF1]]
+; CHECK:       13:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       14:
+; CHECK-NEXT:    [[RES1:%.*]] = call <4 x float> @llvm.x86.avx512.mask.sqrt.ss(<4 x float> [[A0]], <4 x float> [[A1]], <4 x float> [[A2]], i8 [[MASK]], i32 9)
+; CHECK-NEXT:    [[TMP15:%.*]] = bitcast <4 x i32> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP13:%.*]] = icmp ne i128 [[TMP15]], 0
+; CHECK-NEXT:    [[TMP16:%.*]] = bitcast <4 x i32> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP14:%.*]] = icmp ne i128 [[TMP16]], 0
+; CHECK-NEXT:    [[_MSOR15:%.*]] = or i1 [[_MSCMP13]], [[_MSCMP14]]
+; CHECK-NEXT:    [[_MSCMP16:%.*]] = icmp ne i8 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR17:%.*]] = or i1 [[_MSOR15]], [[_MSCMP16]]
+; CHECK-NEXT:    br i1 [[_MSOR17]], label [[TMP17:%.*]], label [[TMP18:%.*]], !prof [[PROF1]]
+; CHECK:       17:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       18:
+; CHECK-NEXT:    [[RES2:%.*]] = call <4 x float> @llvm.x86.avx512.mask.sqrt.ss(<4 x float> [[A0]], <4 x float> [[A1]], <4 x float> zeroinitializer, i8 [[MASK]], i32 10)
+; CHECK-NEXT:    [[TMP19:%.*]] = bitcast <4 x i32> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP18:%.*]] = icmp ne i128 [[TMP19]], 0
+; CHECK-NEXT:    [[TMP20:%.*]] = bitcast <4 x i32> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP19:%.*]] = icmp ne i128 [[TMP20]], 0
+; CHECK-NEXT:    [[_MSOR20:%.*]] = or i1 [[_MSCMP18]], [[_MSCMP19]]
+; CHECK-NEXT:    br i1 [[_MSOR20]], label [[TMP21:%.*]], label [[TMP22:%.*]], !prof [[PROF1]]
+; CHECK:       21:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       22:
+; CHECK-NEXT:    [[RES3:%.*]] = call <4 x float> @llvm.x86.avx512.mask.sqrt.ss(<4 x float> [[A0]], <4 x float> [[A1]], <4 x float> zeroinitializer, i8 -1, i32 11)
+; CHECK-NEXT:    [[RES_1:%.*]] = fadd <4 x float> [[RES0]], [[RES1]]
+; CHECK-NEXT:    [[RES_2:%.*]] = fadd <4 x float> [[RES2]], [[RES3]]
+; CHECK-NEXT:    [[RES:%.*]] = fadd <4 x float> [[RES_1]], [[RES_2]]
+; CHECK-NEXT:    store <4 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x float> [[RES]]
+;
+  %res0 = call <4 x float> @llvm.x86.avx512.mask.sqrt.ss(<4 x float>%a0, <4 x float> %a1, <4 x float> %a2, i8 %mask, i32 4)
+  %res1 = call <4 x float> @llvm.x86.avx512.mask.sqrt.ss(<4 x float>%a0, <4 x float> %a1, <4 x float> %a2, i8 %mask, i32 9)
+  %res2 = call <4 x float> @llvm.x86.avx512.mask.sqrt.ss(<4 x float>%a0, <4 x float> %a1, <4 x float> zeroinitializer, i8 %mask, i32 10)
+  %res3 = call <4 x float> @llvm.x86.avx512.mask.sqrt.ss(<4 x float>%a0, <4 x float> %a1, <4 x float> zeroinitializer, i8 -1, i32 11)
+
+  %res.1 = fadd <4 x float> %res0, %res1
+  %res.2 = fadd <4 x float> %res2, %res3
+  %res   = fadd <4 x float> %res.1, %res.2
+  ret <4 x float> %res
+}
+
+declare <2 x double> @llvm.x86.avx512.mask.sqrt.sd(<2 x double>, <2 x double>, <2 x double>, i8, i32) nounwind readnone
+
+define <2 x double> @test_sqrt_sd(<2 x double> %a0, <2 x double> %a1, <2 x double> %a2, i8 %mask) #0 {
+; CHECK-LABEL: @test_sqrt_sd(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 48) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <2 x i64> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <2 x i64> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP6]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <2 x i64> [[TMP3]] to i128
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i128 [[TMP7]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    [[_MSCMP4:%.*]] = icmp ne i8 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR5:%.*]] = or i1 [[_MSOR3]], [[_MSCMP4]]
+; CHECK-NEXT:    br i1 [[_MSOR5]], label [[TMP8:%.*]], label [[TMP9:%.*]], !prof [[PROF1]]
+; CHECK:       8:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       9:
+; CHECK-NEXT:    [[RES0:%.*]] = call <2 x double> @llvm.x86.avx512.mask.sqrt.sd(<2 x double> [[A0:%.*]], <2 x double> [[A1:%.*]], <2 x double> [[A2:%.*]], i8 [[MASK:%.*]], i32 4)
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast <2 x i64> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP6:%.*]] = icmp ne i128 [[TMP10]], 0
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <2 x i64> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP7:%.*]] = icmp ne i128 [[TMP11]], 0
+; CHECK-NEXT:    [[_MSOR8:%.*]] = or i1 [[_MSCMP6]], [[_MSCMP7]]
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <2 x i64> [[TMP3]] to i128
+; CHECK-NEXT:    [[_MSCMP9:%.*]] = icmp ne i128 [[TMP12]], 0
+; CHECK-NEXT:    [[_MSOR10:%.*]] = or i1 [[_MSOR8]], [[_MSCMP9]]
+; CHECK-NEXT:    [[_MSCMP11:%.*]] = icmp ne i8 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR12:%.*]] = or i1 [[_MSOR10]], [[_MSCMP11]]
+; CHECK-NEXT:    br i1 [[_MSOR12]], label [[TMP13:%.*]], label [[TMP14:%.*]], !prof [[PROF1]]
+; CHECK:       13:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       14:
+; CHECK-NEXT:    [[RES1:%.*]] = call <2 x double> @llvm.x86.avx512.mask.sqrt.sd(<2 x double> [[A0]], <2 x double> [[A1]], <2 x double> [[A2]], i8 [[MASK]], i32 9)
+; CHECK-NEXT:    [[TMP15:%.*]] = bitcast <2 x i64> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP13:%.*]] = icmp ne i128 [[TMP15]], 0
+; CHECK-NEXT:    [[TMP16:%.*]] = bitcast <2 x i64> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP14:%.*]] = icmp ne i128 [[TMP16]], 0
+; CHECK-NEXT:    [[_MSOR15:%.*]] = or i1 [[_MSCMP13]], [[_MSCMP14]]
+; CHECK-NEXT:    [[_MSCMP16:%.*]] = icmp ne i8 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR17:%.*]] = or i1 [[_MSOR15]], [[_MSCMP16]]
+; CHECK-NEXT:    br i1 [[_MSOR17]], label [[TMP17:%.*]], label [[TMP18:%.*]], !prof [[PROF1]]
+; CHECK:       17:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       18:
+; CHECK-NEXT:    [[RES2:%.*]] = call <2 x double> @llvm.x86.avx512.mask.sqrt.sd(<2 x double> [[A0]], <2 x double> [[A1]], <2 x double> zeroinitializer, i8 [[MASK]], i32 10)
+; CHECK-NEXT:    [[TMP19:%.*]] = bitcast <2 x i64> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP18:%.*]] = icmp ne i128 [[TMP19]], 0
+; CHECK-NEXT:    [[TMP20:%.*]] = bitcast <2 x i64> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP19:%.*]] = icmp ne i128 [[TMP20]], 0
+; CHECK-NEXT:    [[_MSOR20:%.*]] = or i1 [[_MSCMP18]], [[_MSCMP19]]
+; CHECK-NEXT:    br i1 [[_MSOR20]], label [[TMP21:%.*]], label [[TMP22:%.*]], !prof [[PROF1]]
+; CHECK:       21:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       22:
+; CHECK-NEXT:    [[RES3:%.*]] = call <2 x double> @llvm.x86.avx512.mask.sqrt.sd(<2 x double> [[A0]], <2 x double> [[A1]], <2 x double> zeroinitializer, i8 -1, i32 11)
+; CHECK-NEXT:    [[RES_1:%.*]] = fadd <2 x double> [[RES0]], [[RES1]]
+; CHECK-NEXT:    [[RES_2:%.*]] = fadd <2 x double> [[RES2]], [[RES3]]
+; CHECK-NEXT:    [[RES:%.*]] = fadd <2 x double> [[RES_1]], [[RES_2]]
+; CHECK-NEXT:    store <2 x i64> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x double> [[RES]]
+;
+  %res0 = call <2 x double> @llvm.x86.avx512.mask.sqrt.sd(<2 x double>%a0, <2 x double> %a1, <2 x double> %a2, i8 %mask, i32 4)
+  %res1 = call <2 x double> @llvm.x86.avx512.mask.sqrt.sd(<2 x double>%a0, <2 x double> %a1, <2 x double> %a2, i8 %mask, i32 9)
+  %res2 = call <2 x double> @llvm.x86.avx512.mask.sqrt.sd(<2 x double>%a0, <2 x double> %a1, <2 x double> zeroinitializer, i8 %mask, i32 10)
+  %res3 = call <2 x double> @llvm.x86.avx512.mask.sqrt.sd(<2 x double>%a0, <2 x double> %a1, <2 x double> zeroinitializer, i8 -1, i32 11)
+
+  %res.1 = fadd <2 x double> %res0, %res1
+  %res.2 = fadd <2 x double> %res2, %res3
+  %res   = fadd <2 x double> %res.1, %res.2
+  ret <2 x double> %res
+}
+
+define i32 @test_x86_avx512_cvttsd2usi(<2 x double> %a0) #0 {
+; CHECK-LABEL: @test_x86_avx512_cvttsd2usi(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <2 x i64> [[TMP1]], i32 0
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP2]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP3:%.*]], label [[TMP4:%.*]], !prof [[PROF1]]
+; CHECK:       3:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       4:
+; CHECK-NEXT:    [[RES0:%.*]] = call i32 @llvm.x86.avx512.cvttsd2usi(<2 x double> [[A0:%.*]], i32 4)
+; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <2 x i64> [[TMP1]], i32 0
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i64 [[TMP5]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP1]], label [[TMP6:%.*]], label [[TMP7:%.*]], !prof [[PROF1]]
+; CHECK:       6:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       7:
+; CHECK-NEXT:    [[RES1:%.*]] = call i32 @llvm.x86.avx512.cvttsd2usi(<2 x double> [[A0]], i32 8)
+; CHECK-NEXT:    [[RES2:%.*]] = add i32 [[RES0]], [[RES1]]
+; CHECK-NEXT:    store i32 0, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i32 [[RES2]]
+;
+  %res0 = call i32 @llvm.x86.avx512.cvttsd2usi(<2 x double> %a0, i32 4) ;
+  %res1 = call i32 @llvm.x86.avx512.cvttsd2usi(<2 x double> %a0, i32 8) ;
+  %res2 = add i32 %res0, %res1
+  ret i32 %res2
+}
+declare i32 @llvm.x86.avx512.cvttsd2usi(<2 x double>, i32) nounwind readnone
+
+define i32 @test_x86_avx512_cvttsd2si(<2 x double> %a0) #0 {
+; CHECK-LABEL: @test_x86_avx512_cvttsd2si(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <2 x i64> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP2]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP3:%.*]], label [[TMP4:%.*]], !prof [[PROF1]]
+; CHECK:       3:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       4:
+; CHECK-NEXT:    [[RES0:%.*]] = call i32 @llvm.x86.avx512.cvttsd2si(<2 x double> [[A0:%.*]], i32 4)
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <2 x i64> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP5]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP1]], label [[TMP6:%.*]], label [[TMP7:%.*]], !prof [[PROF1]]
+; CHECK:       6:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       7:
+; CHECK-NEXT:    [[RES1:%.*]] = call i32 @llvm.x86.avx512.cvttsd2si(<2 x double> [[A0]], i32 8)
+; CHECK-NEXT:    [[RES2:%.*]] = add i32 [[RES0]], [[RES1]]
+; CHECK-NEXT:    store i32 0, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i32 [[RES2]]
+;
+  %res0 = call i32 @llvm.x86.avx512.cvttsd2si(<2 x double> %a0, i32 4) ;
+  %res1 = call i32 @llvm.x86.avx512.cvttsd2si(<2 x double> %a0, i32 8) ;
+  %res2 = add i32 %res0, %res1
+  ret i32 %res2
+}
+declare i32 @llvm.x86.avx512.cvttsd2si(<2 x double>, i32) nounwind readnone
+
+define i32 @test_x86_avx512_cvttss2si(<4 x float> %a0) #0 {
+; CHECK-LABEL: @test_x86_avx512_cvttss2si(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <4 x i32> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP2]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP3:%.*]], label [[TMP4:%.*]], !prof [[PROF1]]
+; CHECK:       3:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       4:
+; CHECK-NEXT:    [[RES0:%.*]] = call i32 @llvm.x86.avx512.cvttss2si(<4 x float> [[A0:%.*]], i32 8)
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <4 x i32> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP5]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP1]], label [[TMP6:%.*]], label [[TMP7:%.*]], !prof [[PROF1]]
+; CHECK:       6:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       7:
+; CHECK-NEXT:    [[RES1:%.*]] = call i32 @llvm.x86.avx512.cvttss2si(<4 x float> [[A0]], i32 4)
+; CHECK-NEXT:    [[RES2:%.*]] = add i32 [[RES0]], [[RES1]]
+; CHECK-NEXT:    store i32 0, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i32 [[RES2]]
+;
+  %res0 = call i32 @llvm.x86.avx512.cvttss2si(<4 x float> %a0, i32 8) ;
+  %res1 = call i32 @llvm.x86.avx512.cvttss2si(<4 x float> %a0, i32 4) ;
+  %res2 = add i32 %res0, %res1
+  ret i32 %res2
+}
+declare i32 @llvm.x86.avx512.cvttss2si(<4 x float>, i32) nounwind readnone
+
+define i32 @test_x86_avx512_cvttss2si_load(ptr %a0) #0 {
+; CHECK-LABEL: @test_x86_avx512_cvttss2si_load(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP2:%.*]], label [[TMP3:%.*]], !prof [[PROF1]]
+; CHECK:       2:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       3:
+; CHECK-NEXT:    [[A1:%.*]] = load <4 x float>, ptr [[A0:%.*]], align 16
+; CHECK-NEXT:    [[TMP4:%.*]] = ptrtoint ptr [[A0]] to i64
+; CHECK-NEXT:    [[TMP5:%.*]] = xor i64 [[TMP4]], 87960930222080
+; CHECK-NEXT:    [[TMP6:%.*]] = inttoptr i64 [[TMP5]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <4 x i32>, ptr [[TMP6]], align 16
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <4 x i32> [[_MSLD]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP7]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP1]], label [[TMP8:%.*]], label [[TMP9:%.*]], !prof [[PROF1]]
+; CHECK:       8:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       9:
+; CHECK-NEXT:    [[RES:%.*]] = call i32 @llvm.x86.avx512.cvttss2si(<4 x float> [[A1]], i32 4)
+; CHECK-NEXT:    store i32 0, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i32 [[RES]]
+;
+  %a1 = load <4 x float>, ptr %a0
+  %res = call i32 @llvm.x86.avx512.cvttss2si(<4 x float> %a1, i32 4) ;
+  ret i32 %res
+}
+
+define i32 @test_x86_avx512_cvttss2usi(<4 x float> %a0) #0 {
+; CHECK-LABEL: @test_x86_avx512_cvttss2usi(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <4 x i32> [[TMP1]], i32 0
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i32 [[TMP2]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP3:%.*]], label [[TMP4:%.*]], !prof [[PROF1]]
+; CHECK:       3:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       4:
+; CHECK-NEXT:    [[RES0:%.*]] = call i32 @llvm.x86.avx512.cvttss2usi(<4 x float> [[A0:%.*]], i32 8)
+; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <4 x i32> [[TMP1]], i32 0
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i32 [[TMP5]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP1]], label [[TMP6:%.*]], label [[TMP7:%.*]], !prof [[PROF1]]
+; CHECK:       6:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       7:
+; CHECK-NEXT:    [[RES1:%.*]] = call i32 @llvm.x86.avx512.cvttss2usi(<4 x float> [[A0]], i32 4)
+; CHECK-NEXT:    [[RES2:%.*]] = add i32 [[RES0]], [[RES1]]
+; CHECK-NEXT:    store i32 0, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i32 [[RES2]]
+;
+  %res0 = call i32 @llvm.x86.avx512.cvttss2usi(<4 x float> %a0, i32 8) ;
+  %res1 = call i32 @llvm.x86.avx512.cvttss2usi(<4 x float> %a0, i32 4) ;
+  %res2 = add i32 %res0, %res1
+  ret i32 %res2
+}
+declare i32 @llvm.x86.avx512.cvttss2usi(<4 x float>, i32) nounwind readnone
+
+define i32 @test_x86_avx512_cvtsd2usi32(<2 x double> %a0) #0 {
+; CHECK-LABEL: @test_x86_avx512_cvtsd2usi32(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <2 x i64> [[TMP1]], i32 0
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP2]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP3:%.*]], label [[TMP4:%.*]], !prof [[PROF1]]
+; CHECK:       3:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       4:
+; CHECK-NEXT:    [[RES:%.*]] = call i32 @llvm.x86.avx512.vcvtsd2usi32(<2 x double> [[A0:%.*]], i32 4)
+; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <2 x i64> [[TMP1]], i32 0
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i64 [[TMP5]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP1]], label [[TMP6:%.*]], label [[TMP7:%.*]], !prof [[PROF1]]
+; CHECK:       6:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       7:
+; CHECK-NEXT:    [[RES1:%.*]] = call i32 @llvm.x86.avx512.vcvtsd2usi32(<2 x double> [[A0]], i32 11)
+; CHECK-NEXT:    [[TMP8:%.*]] = extractelement <2 x i64> [[TMP1]], i32 0
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i64 [[TMP8]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP2]], label [[TMP9:%.*]], label [[TMP10:%.*]], !prof [[PROF1]]
+; CHECK:       9:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       10:
+; CHECK-NEXT:    [[RES2:%.*]] = call i32 @llvm.x86.avx512.vcvtsd2usi32(<2 x double> [[A0]], i32 9)
+; CHECK-NEXT:    [[RES3:%.*]] = add i32 [[RES]], [[RES1]]
+; CHECK-NEXT:    [[RES4:%.*]] = add i32 [[RES3]], [[RES2]]
+; CHECK-NEXT:    store i32 0, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i32 [[RES4]]
+;
+  %res = call i32 @llvm.x86.avx512.vcvtsd2usi32(<2 x double> %a0, i32 4)
+  %res1 = call i32 @llvm.x86.avx512.vcvtsd2usi32(<2 x double> %a0, i32 11)
+  %res2 = call i32 @llvm.x86.avx512.vcvtsd2usi32(<2 x double> %a0, i32 9)
+  %res3 = add i32 %res, %res1
+  %res4 = add i32 %res3, %res2
+  ret i32 %res4
+}
+declare i32 @llvm.x86.avx512.vcvtsd2usi32(<2 x double>, i32) nounwind readnone
+
+define i32 @test_x86_avx512_cvtsd2si32(<2 x double> %a0) #0 {
+; CHECK-LABEL: @test_x86_avx512_cvtsd2si32(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <2 x i64> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP2]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP3:%.*]], label [[TMP4:%.*]], !prof [[PROF1]]
+; CHECK:       3:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       4:
+; CHECK-NEXT:    [[RES:%.*]] = call i32 @llvm.x86.avx512.vcvtsd2si32(<2 x double> [[A0:%.*]], i32 4)
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <2 x i64> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP5]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP1]], label [[TMP6:%.*]], label [[TMP7:%.*]], !prof [[PROF1]]
+; CHECK:       6:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       7:
+; CHECK-NEXT:    [[RES1:%.*]] = call i32 @llvm.x86.avx512.vcvtsd2si32(<2 x double> [[A0]], i32 11)
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <2 x i64> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i128 [[TMP8]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP2]], label [[TMP9:%.*]], label [[TMP10:%.*]], !prof [[PROF1]]
+; CHECK:       9:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       10:
+; CHECK-NEXT:    [[RES2:%.*]] = call i32 @llvm.x86.avx512.vcvtsd2si32(<2 x double> [[A0]], i32 9)
+; CHECK-NEXT:    [[RES3:%.*]] = add i32 [[RES]], [[RES1]]
+; CHECK-NEXT:    [[RES4:%.*]] = add i32 [[RES3]], [[RES2]]
+; CHECK-NEXT:    store i32 0, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i32 [[RES4]]
+;
+  %res = call i32 @llvm.x86.avx512.vcvtsd2si32(<2 x double> %a0, i32 4)
+  %res1 = call i32 @llvm.x86.avx512.vcvtsd2si32(<2 x double> %a0, i32 11)
+  %res2 = call i32 @llvm.x86.avx512.vcvtsd2si32(<2 x double> %a0, i32 9)
+  %res3 = add i32 %res, %res1
+  %res4 = add i32 %res3, %res2
+  ret i32 %res4
+}
+declare i32 @llvm.x86.avx512.vcvtsd2si32(<2 x double>, i32) nounwind readnone
+
+define i32 @test_x86_avx512_cvtss2usi32(<4 x float> %a0) #0 {
+; CHECK-LABEL: @test_x86_avx512_cvtss2usi32(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <4 x i32> [[TMP1]], i32 0
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i32 [[TMP2]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP3:%.*]], label [[TMP4:%.*]], !prof [[PROF1]]
+; CHECK:       3:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       4:
+; CHECK-NEXT:    [[RES:%.*]] = call i32 @llvm.x86.avx512.vcvtss2usi32(<4 x float> [[A0:%.*]], i32 4)
+; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <4 x i32> [[TMP1]], i32 0
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i32 [[TMP5]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP1]], label [[TMP6:%.*]], label [[TMP7:%.*]], !prof [[PROF1]]
+; CHECK:       6:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       7:
+; CHECK-NEXT:    [[RES1:%.*]] = call i32 @llvm.x86.avx512.vcvtss2usi32(<4 x float> [[A0]], i32 11)
+; CHECK-NEXT:    [[TMP8:%.*]] = extractelement <4 x i32> [[TMP1]], i32 0
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i32 [[TMP8]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP2]], label [[TMP9:%.*]], label [[TMP10:%.*]], !prof [[PROF1]]
+; CHECK:       9:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       10:
+; CHECK-NEXT:    [[RES2:%.*]] = call i32 @llvm.x86.avx512.vcvtss2usi32(<4 x float> [[A0]], i32 9)
+; CHECK-NEXT:    [[RES3:%.*]] = add i32 [[RES]], [[RES1]]
+; CHECK-NEXT:    [[RES4:%.*]] = add i32 [[RES3]], [[RES2]]
+; CHECK-NEXT:    store i32 0, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i32 [[RES4]]
+;
+  %res = call i32 @llvm.x86.avx512.vcvtss2usi32(<4 x float> %a0, i32 4)
+  %res1 = call i32 @llvm.x86.avx512.vcvtss2usi32(<4 x float> %a0, i32 11)
+  %res2 = call i32 @llvm.x86.avx512.vcvtss2usi32(<4 x float> %a0, i32 9)
+  %res3 = add i32 %res, %res1
+  %res4 = add i32 %res3, %res2
+  ret i32 %res4
+}
+declare i32 @llvm.x86.avx512.vcvtss2usi32(<4 x float>, i32) nounwind readnone
+
+define i32 @test_x86_avx512_cvtss2si32(<4 x float> %a0) #0 {
+; CHECK-LABEL: @test_x86_avx512_cvtss2si32(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <4 x i32> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP2]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP3:%.*]], label [[TMP4:%.*]], !prof [[PROF1]]
+; CHECK:       3:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       4:
+; CHECK-NEXT:    [[RES:%.*]] = call i32 @llvm.x86.avx512.vcvtss2si32(<4 x float> [[A0:%.*]], i32 4)
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <4 x i32> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP5]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP1]], label [[TMP6:%.*]], label [[TMP7:%.*]], !prof [[PROF1]]
+; CHECK:       6:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       7:
+; CHECK-NEXT:    [[RES1:%.*]] = call i32 @llvm.x86.avx512.vcvtss2si32(<4 x float> [[A0]], i32 11)
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <4 x i32> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i128 [[TMP8]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP2]], label [[TMP9:%.*]], label [[TMP10:%.*]], !prof [[PROF1]]
+; CHECK:       9:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       10:
+; CHECK-NEXT:    [[RES2:%.*]] = call i32 @llvm.x86.avx512.vcvtss2si32(<4 x float> [[A0]], i32 9)
+; CHECK-NEXT:    [[RES3:%.*]] = add i32 [[RES]], [[RES1]]
+; CHECK-NEXT:    [[RES4:%.*]] = add i32 [[RES3]], [[RES2]]
+; CHECK-NEXT:    store i32 0, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i32 [[RES4]]
+;
+  %res = call i32 @llvm.x86.avx512.vcvtss2si32(<4 x float> %a0, i32 4)
+  %res1 = call i32 @llvm.x86.avx512.vcvtss2si32(<4 x float> %a0, i32 11)
+  %res2 = call i32 @llvm.x86.avx512.vcvtss2si32(<4 x float> %a0, i32 9)
+  %res3 = add i32 %res, %res1
+  %res4 = add i32 %res3, %res2
+  ret i32 %res4
+}
+declare i32 @llvm.x86.avx512.vcvtss2si32(<4 x float>, i32) nounwind readnone
+
+define <16 x i16> @test_x86_vcvtps2ph_256(<16 x float> %a0, <16 x i16> %src, i16 %mask, ptr %dst) #0 {
+; CHECK-LABEL: @test_x86_vcvtps2ph_256(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 96) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <16 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 104) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP5]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP6:%.*]], label [[TMP7:%.*]], !prof [[PROF1]]
+; CHECK:       6:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       7:
+; CHECK-NEXT:    [[RES1:%.*]] = call <16 x i16> @llvm.x86.avx512.mask.vcvtps2ph.512(<16 x float> [[A0:%.*]], i32 2, <16 x i16> zeroinitializer, i16 -1)
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP8]], 0
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i16 [[TMP2]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP1]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP9:%.*]], label [[TMP10:%.*]], !prof [[PROF1]]
+; CHECK:       9:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       10:
+; CHECK-NEXT:    [[RES2:%.*]] = call <16 x i16> @llvm.x86.avx512.mask.vcvtps2ph.512(<16 x float> [[A0]], i32 11, <16 x i16> zeroinitializer, i16 [[MASK:%.*]])
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP3:%.*]] = icmp ne i512 [[TMP11]], 0
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <16 x i16> [[TMP3]] to i256
+; CHECK-NEXT:    [[_MSCMP4:%.*]] = icmp ne i256 [[TMP12]], 0
+; CHECK-NEXT:    [[_MSOR5:%.*]] = or i1 [[_MSCMP3]], [[_MSCMP4]]
+; CHECK-NEXT:    [[_MSCMP6:%.*]] = icmp ne i16 [[TMP2]], 0
+; CHECK-NEXT:    [[_MSOR7:%.*]] = or i1 [[_MSOR5]], [[_MSCMP6]]
+; CHECK-NEXT:    br i1 [[_MSOR7]], label [[TMP13:%.*]], label [[TMP14:%.*]], !prof [[PROF1]]
+; CHECK:       13:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       14:
+; CHECK-NEXT:    [[RES3:%.*]] = call <16 x i16> @llvm.x86.avx512.mask.vcvtps2ph.512(<16 x float> [[A0]], i32 12, <16 x i16> [[SRC:%.*]], i16 [[MASK]])
+; CHECK-NEXT:    [[_MSCMP8:%.*]] = icmp ne i64 [[TMP4]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP8]], label [[TMP15:%.*]], label [[TMP16:%.*]], !prof [[PROF1]]
+; CHECK:       15:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       16:
+; CHECK-NEXT:    [[TMP17:%.*]] = ptrtoint ptr [[DST:%.*]] to i64
+; CHECK-NEXT:    [[TMP18:%.*]] = xor i64 [[TMP17]], 87960930222080
+; CHECK-NEXT:    [[TMP19:%.*]] = inttoptr i64 [[TMP18]] to ptr
+; CHECK-NEXT:    store <16 x i16> zeroinitializer, ptr [[TMP19]], align 32
+; CHECK-NEXT:    store <16 x i16> [[RES1]], ptr [[DST]], align 32
+; CHECK-NEXT:    [[RES:%.*]] = add <16 x i16> [[RES2]], [[RES3]]
+; CHECK-NEXT:    store <16 x i16> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i16> [[RES]]
+;
+  %res1 = call <16 x i16> @llvm.x86.avx512.mask.vcvtps2ph.512(<16 x float> %a0, i32 2, <16 x i16> zeroinitializer, i16 -1)
+  %res2 = call <16 x i16> @llvm.x86.avx512.mask.vcvtps2ph.512(<16 x float> %a0, i32 11, <16 x i16> zeroinitializer, i16 %mask)
+  %res3 = call <16 x i16> @llvm.x86.avx512.mask.vcvtps2ph.512(<16 x float> %a0, i32 12, <16 x i16> %src, i16 %mask)
+  store <16 x i16> %res1, ptr %dst
+  %res  = add <16 x i16> %res2, %res3
+  ret <16 x i16> %res
+}
+
+declare <16 x i16> @llvm.x86.avx512.mask.vcvtps2ph.512(<16 x float>, i32, <16 x i16>, i16) nounwind readonly
+
+define i16 @test_cmpps(<16 x float> %a, <16 x float> %b) #0 {
+; CHECK-LABEL: @test_cmpps(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP3]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP5:%.*]], label [[TMP6:%.*]], !prof [[PROF1]]
+; CHECK:       5:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       6:
+; CHECK-NEXT:    [[RES:%.*]] = call <16 x i1> @llvm.x86.avx512.mask.cmp.ps.512(<16 x float> [[A:%.*]], <16 x float> [[B:%.*]], i32 2, <16 x i1> splat (i1 true), i32 8)
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <16 x i1> [[RES]] to i16
+; CHECK-NEXT:    store i16 0, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i16 [[TMP7]]
+;
+  %res = call <16 x i1> @llvm.x86.avx512.mask.cmp.ps.512(<16 x float> %a, <16 x float> %b, i32 2, <16 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, i32 8)
+  %1 = bitcast <16 x i1> %res to i16
+  ret i16 %1
+}
+declare <16 x i1> @llvm.x86.avx512.mask.cmp.ps.512(<16 x float>, <16 x float>, i32, <16 x i1>, i32)
+
+define i8 @test_cmppd(<8 x double> %a, <8 x double> %b) #0 {
+; CHECK-LABEL: @test_cmppd(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <8 x i64> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP3]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <8 x i64> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP5:%.*]], label [[TMP6:%.*]], !prof [[PROF1]]
+; CHECK:       5:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       6:
+; CHECK-NEXT:    [[RES:%.*]] = call <8 x i1> @llvm.x86.avx512.mask.cmp.pd.512(<8 x double> [[A:%.*]], <8 x double> [[B:%.*]], i32 4, <8 x i1> splat (i1 true), i32 4)
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <8 x i1> [[RES]] to i8
+; CHECK-NEXT:    store i8 0, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i8 [[TMP7]]
+;
+  %res = call <8 x i1> @llvm.x86.avx512.mask.cmp.pd.512(<8 x double> %a, <8 x double> %b, i32 4, <8 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, i32 4)
+  %1 = bitcast <8 x i1> %res to i8
+  ret i8 %1
+}
+declare <8 x i1> @llvm.x86.avx512.mask.cmp.pd.512(<8 x double>, <8 x double>, i32, <8 x i1>, i32)
+
+
+  ; fp min - max
+define <8 x double> @test_vmaxpd(<8 x double> %a0, <8 x double> %a1) #0 {
+; CHECK-LABEL: @test_vmaxpd(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <8 x i64> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <8 x i64> [[_MSPROP]], zeroinitializer
+; CHECK-NEXT:    [[TMP7:%.*]] = call <8 x double> @llvm.x86.avx512.max.pd.512(<8 x double> [[A0:%.*]], <8 x double> [[A1:%.*]], i32 4)
+; CHECK-NEXT:    store <8 x i64> [[_MSPROP1]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x double> [[TMP7]]
+;
+  %1 = call <8 x double> @llvm.x86.avx512.max.pd.512(<8 x double> %a0, <8 x double> %a1, i32 4)
+  ret <8 x double> %1
+}
+declare <8 x double> @llvm.x86.avx512.max.pd.512(<8 x double>, <8 x double>, i32)
+
+define <8 x double> @test_vminpd(<8 x double> %a0, <8 x double> %a1) #0 {
+; CHECK-LABEL: @test_vminpd(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <8 x i64> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <8 x i64> [[_MSPROP]], zeroinitializer
+; CHECK-NEXT:    [[TMP7:%.*]] = call <8 x double> @llvm.x86.avx512.min.pd.512(<8 x double> [[A0:%.*]], <8 x double> [[A1:%.*]], i32 4)
+; CHECK-NEXT:    store <8 x i64> [[_MSPROP1]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x double> [[TMP7]]
+;
+  %1 = call <8 x double> @llvm.x86.avx512.min.pd.512(<8 x double> %a0, <8 x double> %a1, i32 4)
+  ret <8 x double> %1
+}
+declare <8 x double> @llvm.x86.avx512.min.pd.512(<8 x double>, <8 x double>, i32)
+
+define void @test_mask_store_ss(ptr %ptr, <4 x float> %data, i8 %mask) #0 {
+; CHECK-LABEL: @test_mask_store_ss(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 24) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = and i8 [[TMP1]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = and i8 [[MASK:%.*]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = and i8 [[TMP1]], 1
+; CHECK-NEXT:    [[TMP7:%.*]] = or i8 [[TMP4]], [[TMP5]]
+; CHECK-NEXT:    [[TMP8:%.*]] = or i8 [[TMP7]], [[TMP6]]
+; CHECK-NEXT:    [[TMP9:%.*]] = and i8 [[MASK]], 1
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i8 [[TMP8]] to <8 x i1>
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast i8 [[TMP9]] to <8 x i1>
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <8 x i1> [[TMP10]], <8 x i1> [[TMP10]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[EXTRACT:%.*]] = shufflevector <8 x i1> [[TMP11]], <8 x i1> [[TMP11]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[TMP12:%.*]] = ptrtoint ptr [[PTR:%.*]] to i64
+; CHECK-NEXT:    [[TMP13:%.*]] = xor i64 [[TMP12]], 87960930222080
+; CHECK-NEXT:    [[TMP14:%.*]] = inttoptr i64 [[TMP13]] to ptr
+; CHECK-NEXT:    call void @llvm.masked.store.v4i32.p0(<4 x i32> [[TMP2]], ptr [[TMP14]], i32 1, <4 x i1> [[EXTRACT]])
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP3]], 0
+; CHECK-NEXT:    [[TMP15:%.*]] = bitcast <4 x i1> [[_MSPROP]] to i4
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i4 [[TMP15]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP16:%.*]], label [[TMP17:%.*]], !prof [[PROF1]]
+; CHECK:       16:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       17:
+; CHECK-NEXT:    call void @llvm.masked.store.v4f32.p0(<4 x float> [[DATA:%.*]], ptr [[PTR]], i32 1, <4 x i1> [[EXTRACT]])
+; CHECK-NEXT:    ret void
+;
+  %1 = and i8 %mask, 1
+  %2 = bitcast i8 %1 to <8 x i1>
+  %extract = shufflevector <8 x i1> %2, <8 x i1> %2, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  call void @llvm.masked.store.v4f32.p0(<4 x float> %data, ptr %ptr, i32 1, <4 x i1> %extract)
+  ret void
+}
+declare void @llvm.masked.store.v4f32.p0(<4 x float>, ptr, i32, <4 x i1>) #1
+
+
+declare <16 x float> @llvm.x86.avx512.sub.ps.512(<16 x float>, <16 x float>, i32)
+declare <16 x float> @llvm.x86.avx512.mul.ps.512(<16 x float>, <16 x float>, i32)
+declare <8 x double> @llvm.x86.avx512.mul.pd.512(<8 x double>, <8 x double>, i32)
+
+define <16 x float> @test_vsubps_rn(<16 x float> %a0, <16 x float> %a1) #0 {
+; CHECK-LABEL: @test_vsubps_rn(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP3]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP5:%.*]], label [[TMP6:%.*]], !prof [[PROF1]]
+; CHECK:       5:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       6:
+; CHECK-NEXT:    [[TMP7:%.*]] = call <16 x float> @llvm.x86.avx512.sub.ps.512(<16 x float> [[A0:%.*]], <16 x float> [[A1:%.*]], i32 8)
+; CHECK-NEXT:    store <16 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x float> [[TMP7]]
+;
+  %1 = call <16 x float> @llvm.x86.avx512.sub.ps.512(<16 x float> %a0, <16 x float> %a1, i32 8)
+  ret <16 x float> %1
+}
+
+define <16 x float> @test_vsubps_rd(<16 x float> %a0, <16 x float> %a1) #0 {
+; CHECK-LABEL: @test_vsubps_rd(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP3]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP5:%.*]], label [[TMP6:%.*]], !prof [[PROF1]]
+; CHECK:       5:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       6:
+; CHECK-NEXT:    [[TMP7:%.*]] = call <16 x float> @llvm.x86.avx512.sub.ps.512(<16 x float> [[A0:%.*]], <16 x float> [[A1:%.*]], i32 9)
+; CHECK-NEXT:    store <16 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x float> [[TMP7]]
+;
+  %1 = call <16 x float> @llvm.x86.avx512.sub.ps.512(<16 x float> %a0, <16 x float> %a1, i32 9)
+  ret <16 x float> %1
+}
+
+define <16 x float> @test_vsubps_ru(<16 x float> %a0, <16 x float> %a1) #0 {
+; CHECK-LABEL: @test_vsubps_ru(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP3]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP5:%.*]], label [[TMP6:%.*]], !prof [[PROF1]]
+; CHECK:       5:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       6:
+; CHECK-NEXT:    [[TMP7:%.*]] = call <16 x float> @llvm.x86.avx512.sub.ps.512(<16 x float> [[A0:%.*]], <16 x float> [[A1:%.*]], i32 10)
+; CHECK-NEXT:    store <16 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x float> [[TMP7]]
+;
+  %1 = call <16 x float> @llvm.x86.avx512.sub.ps.512(<16 x float> %a0, <16 x float> %a1, i32 10)
+  ret <16 x float> %1
+}
+
+define <16 x float> @test_vsubps_rz(<16 x float> %a0, <16 x float> %a1) #0 {
+; CHECK-LABEL: @test_vsubps_rz(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP3]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP5:%.*]], label [[TMP6:%.*]], !prof [[PROF1]]
+; CHECK:       5:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       6:
+; CHECK-NEXT:    [[TMP7:%.*]] = call <16 x float> @llvm.x86.avx512.sub.ps.512(<16 x float> [[A0:%.*]], <16 x float> [[A1:%.*]], i32 11)
+; CHECK-NEXT:    store <16 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x float> [[TMP7]]
+;
+  %1 = call <16 x float> @llvm.x86.avx512.sub.ps.512(<16 x float> %a0, <16 x float> %a1, i32 11)
+  ret <16 x float> %1
+}
+
+define <16 x float> @test_vmulps_rn(<16 x float> %a0, <16 x float> %a1) #0 {
+; CHECK-LABEL: @test_vmulps_rn(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP3]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP5:%.*]], label [[TMP6:%.*]], !prof [[PROF1]]
+; CHECK:       5:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       6:
+; CHECK-NEXT:    [[TMP7:%.*]] = call <16 x float> @llvm.x86.avx512.mul.ps.512(<16 x float> [[A0:%.*]], <16 x float> [[A1:%.*]], i32 8)
+; CHECK-NEXT:    store <16 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x float> [[TMP7]]
+;
+  %1 = call <16 x float> @llvm.x86.avx512.mul.ps.512(<16 x float> %a0, <16 x float> %a1, i32 8)
+  ret <16 x float> %1
+}
+
+define <16 x float> @test_vmulps_rd(<16 x float> %a0, <16 x float> %a1) #0 {
+; CHECK-LABEL: @test_vmulps_rd(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP3]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP5:%.*]], label [[TMP6:%.*]], !prof [[PROF1]]
+; CHECK:       5:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       6:
+; CHECK-NEXT:    [[TMP7:%.*]] = call <16 x float> @llvm.x86.avx512.mul.ps.512(<16 x float> [[A0:%.*]], <16 x float> [[A1:%.*]], i32 9)
+; CHECK-NEXT:    store <16 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x float> [[TMP7]]
+;
+  %1 = call <16 x float> @llvm.x86.avx512.mul.ps.512(<16 x float> %a0, <16 x float> %a1, i32 9)
+  ret <16 x float> %1
+}
+
+define <16 x float> @test_vmulps_ru(<16 x float> %a0, <16 x float> %a1) #0 {
+; CHECK-LABEL: @test_vmulps_ru(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP3]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP5:%.*]], label [[TMP6:%.*]], !prof [[PROF1]]
+; CHECK:       5:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       6:
+; CHECK-NEXT:    [[TMP7:%.*]] = call <16 x float> @llvm.x86.avx512.mul.ps.512(<16 x float> [[A0:%.*]], <16 x float> [[A1:%.*]], i32 10)
+; CHECK-NEXT:    store <16 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x float> [[TMP7]]
+;
+  %1 = call <16 x float> @llvm.x86.avx512.mul.ps.512(<16 x float> %a0, <16 x float> %a1, i32 10)
+  ret <16 x float> %1
+}
+
+define <16 x float> @test_vmulps_rz(<16 x float> %a0, <16 x float> %a1) #0 {
+; CHECK-LABEL: @test_vmulps_rz(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP3]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP5:%.*]], label [[TMP6:%.*]], !prof [[PROF1]]
+; CHECK:       5:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       6:
+; CHECK-NEXT:    [[TMP7:%.*]] = call <16 x float> @llvm.x86.avx512.mul.ps.512(<16 x float> [[A0:%.*]], <16 x float> [[A1:%.*]], i32 11)
+; CHECK-NEXT:    store <16 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x float> [[TMP7]]
+;
+  %1 = call <16 x float> @llvm.x86.avx512.mul.ps.512(<16 x float> %a0, <16 x float> %a1, i32 11)
+  ret <16 x float> %1
+}
+
+define <16 x float> @test_vmulps_mask_rn(<16 x float> %a0, <16 x float> %a1, i16 %mask) #0 {
+; CHECK-LABEL: @test_vmulps_mask_rn(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP6:%.*]], label [[TMP7:%.*]], !prof [[PROF1]]
+; CHECK:       6:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       7:
+; CHECK-NEXT:    [[TMP8:%.*]] = call <16 x float> @llvm.x86.avx512.mul.ps.512(<16 x float> [[A0:%.*]], <16 x float> [[A1:%.*]], i32 8)
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast i16 [[TMP3]] to <16 x i1>
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i16 [[MASK:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP11:%.*]] = select <16 x i1> [[TMP10]], <16 x i32> zeroinitializer, <16 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <16 x float> [[TMP8]] to <16 x i32>
+; CHECK-NEXT:    [[TMP13:%.*]] = xor <16 x i32> [[TMP12]], zeroinitializer
+; CHECK-NEXT:    [[TMP14:%.*]] = or <16 x i32> [[TMP13]], zeroinitializer
+; CHECK-NEXT:    [[TMP15:%.*]] = or <16 x i32> [[TMP14]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP9]], <16 x i32> [[TMP15]], <16 x i32> [[TMP11]]
+; CHECK-NEXT:    [[TMP16:%.*]] = select <16 x i1> [[TMP10]], <16 x float> [[TMP8]], <16 x float> zeroinitializer
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x float> [[TMP16]]
+;
+  %1 = call <16 x float> @llvm.x86.avx512.mul.ps.512(<16 x float> %a0, <16 x float> %a1, i32 8)
+  %2 = bitcast i16 %mask to <16 x i1>
+  %3 = select <16 x i1> %2, <16 x float> %1, <16 x float> zeroinitializer
+  ret <16 x float> %3
+}
+
+define <16 x float> @test_vmulps_mask_rd(<16 x float> %a0, <16 x float> %a1, i16 %mask) #0 {
+; CHECK-LABEL: @test_vmulps_mask_rd(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP6:%.*]], label [[TMP7:%.*]], !prof [[PROF1]]
+; CHECK:       6:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       7:
+; CHECK-NEXT:    [[TMP8:%.*]] = call <16 x float> @llvm.x86.avx512.mul.ps.512(<16 x float> [[A0:%.*]], <16 x float> [[A1:%.*]], i32 9)
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast i16 [[TMP3]] to <16 x i1>
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i16 [[MASK:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP11:%.*]] = select <16 x i1> [[TMP10]], <16 x i32> zeroinitializer, <16 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <16 x float> [[TMP8]] to <16 x i32>
+; CHECK-NEXT:    [[TMP13:%.*]] = xor <16 x i32> [[TMP12]], zeroinitializer
+; CHECK-NEXT:    [[TMP14:%.*]] = or <16 x i32> [[TMP13]], zeroinitializer
+; CHECK-NEXT:    [[TMP15:%.*]] = or <16 x i32> [[TMP14]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP9]], <16 x i32> [[TMP15]], <16 x i32> [[TMP11]]
+; CHECK-NEXT:    [[TMP16:%.*]] = select <16 x i1> [[TMP10]], <16 x float> [[TMP8]], <16 x float> zeroinitializer
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x float> [[TMP16]]
+;
+  %1 = call <16 x float> @llvm.x86.avx512.mul.ps.512(<16 x float> %a0, <16 x float> %a1, i32 9)
+  %2 = bitcast i16 %mask to <16 x i1>
+  %3 = select <16 x i1> %2, <16 x float> %1, <16 x float> zeroinitializer
+  ret <16 x float> %3
+}
+
+define <16 x float> @test_vmulps_mask_ru(<16 x float> %a0, <16 x float> %a1, i16 %mask) #0 {
+; CHECK-LABEL: @test_vmulps_mask_ru(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP6:%.*]], label [[TMP7:%.*]], !prof [[PROF1]]
+; CHECK:       6:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       7:
+; CHECK-NEXT:    [[TMP8:%.*]] = call <16 x float> @llvm.x86.avx512.mul.ps.512(<16 x float> [[A0:%.*]], <16 x float> [[A1:%.*]], i32 10)
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast i16 [[TMP3]] to <16 x i1>
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i16 [[MASK:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP11:%.*]] = select <16 x i1> [[TMP10]], <16 x i32> zeroinitializer, <16 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <16 x float> [[TMP8]] to <16 x i32>
+; CHECK-NEXT:    [[TMP13:%.*]] = xor <16 x i32> [[TMP12]], zeroinitializer
+; CHECK-NEXT:    [[TMP14:%.*]] = or <16 x i32> [[TMP13]], zeroinitializer
+; CHECK-NEXT:    [[TMP15:%.*]] = or <16 x i32> [[TMP14]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP9]], <16 x i32> [[TMP15]], <16 x i32> [[TMP11]]
+; CHECK-NEXT:    [[TMP16:%.*]] = select <16 x i1> [[TMP10]], <16 x float> [[TMP8]], <16 x float> zeroinitializer
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x float> [[TMP16]]
+;
+  %1 = call <16 x float> @llvm.x86.avx512.mul.ps.512(<16 x float> %a0, <16 x float> %a1, i32 10)
+  %2 = bitcast i16 %mask to <16 x i1>
+  %3 = select <16 x i1> %2, <16 x float> %1, <16 x float> zeroinitializer
+  ret <16 x float> %3
+}
+
+define <16 x float> @test_vmulps_mask_rz(<16 x float> %a0, <16 x float> %a1, i16 %mask) #0 {
+; CHECK-LABEL: @test_vmulps_mask_rz(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP6:%.*]], label [[TMP7:%.*]], !prof [[PROF1]]
+; CHECK:       6:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       7:
+; CHECK-NEXT:    [[TMP8:%.*]] = call <16 x float> @llvm.x86.avx512.mul.ps.512(<16 x float> [[A0:%.*]], <16 x float> [[A1:%.*]], i32 11)
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast i16 [[TMP3]] to <16 x i1>
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i16 [[MASK:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP11:%.*]] = select <16 x i1> [[TMP10]], <16 x i32> zeroinitializer, <16 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <16 x float> [[TMP8]] to <16 x i32>
+; CHECK-NEXT:    [[TMP13:%.*]] = xor <16 x i32> [[TMP12]], zeroinitializer
+; CHECK-NEXT:    [[TMP14:%.*]] = or <16 x i32> [[TMP13]], zeroinitializer
+; CHECK-NEXT:    [[TMP15:%.*]] = or <16 x i32> [[TMP14]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP9]], <16 x i32> [[TMP15]], <16 x i32> [[TMP11]]
+; CHECK-NEXT:    [[TMP16:%.*]] = select <16 x i1> [[TMP10]], <16 x float> [[TMP8]], <16 x float> zeroinitializer
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x float> [[TMP16]]
+;
+  %1 = call <16 x float> @llvm.x86.avx512.mul.ps.512(<16 x float> %a0, <16 x float> %a1, i32 11)
+  %2 = bitcast i16 %mask to <16 x i1>
+  %3 = select <16 x i1> %2, <16 x float> %1, <16 x float> zeroinitializer
+  ret <16 x float> %3
+}
+
+define <16 x float> @test_vmulps_mask_passthru_rn(<16 x float> %a0, <16 x float> %a1, <16 x float> %passthru, i16 %mask) #0 {
+; CHECK-LABEL: @test_vmulps_mask_passthru_rn(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP6]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP7:%.*]], label [[TMP8:%.*]], !prof [[PROF1]]
+; CHECK:       7:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       8:
+; CHECK-NEXT:    [[TMP9:%.*]] = call <16 x float> @llvm.x86.avx512.mul.ps.512(<16 x float> [[A0:%.*]], <16 x float> [[A1:%.*]], i32 8)
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i16 [[TMP3]] to <16 x i1>
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast i16 [[MASK:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP12:%.*]] = select <16 x i1> [[TMP11]], <16 x i32> zeroinitializer, <16 x i32> [[TMP4]]
+; CHECK-NEXT:    [[TMP13:%.*]] = bitcast <16 x float> [[TMP9]] to <16 x i32>
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast <16 x float> [[PASSTHRU:%.*]] to <16 x i32>
+; CHECK-NEXT:    [[TMP15:%.*]] = xor <16 x i32> [[TMP13]], [[TMP14]]
+; CHECK-NEXT:    [[TMP16:%.*]] = or <16 x i32> [[TMP15]], zeroinitializer
+; CHECK-NEXT:    [[TMP17:%.*]] = or <16 x i32> [[TMP16]], [[TMP4]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP10]], <16 x i32> [[TMP17]], <16 x i32> [[TMP12]]
+; CHECK-NEXT:    [[TMP18:%.*]] = select <16 x i1> [[TMP11]], <16 x float> [[TMP9]], <16 x float> [[PASSTHRU]]
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x float> [[TMP18]]
+;
+  %1 = call <16 x float> @llvm.x86.avx512.mul.ps.512(<16 x float> %a0, <16 x float> %a1, i32 8)
+  %2 = bitcast i16 %mask to <16 x i1>
+  %3 = select <16 x i1> %2, <16 x float> %1, <16 x float> %passthru
+  ret <16 x float> %3
+}
+
+define <16 x float> @test_vmulps_mask_passthru_rd(<16 x float> %a0, <16 x float> %a1, <16 x float> %passthru, i16 %mask) #0 {
+; CHECK-LABEL: @test_vmulps_mask_passthru_rd(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP6]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP7:%.*]], label [[TMP8:%.*]], !prof [[PROF1]]
+; CHECK:       7:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       8:
+; CHECK-NEXT:    [[TMP9:%.*]] = call <16 x float> @llvm.x86.avx512.mul.ps.512(<16 x float> [[A0:%.*]], <16 x float> [[A1:%.*]], i32 9)
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i16 [[TMP3]] to <16 x i1>
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast i16 [[MASK:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP12:%.*]] = select <16 x i1> [[TMP11]], <16 x i32> zeroinitializer, <16 x i32> [[TMP4]]
+; CHECK-NEXT:    [[TMP13:%.*]] = bitcast <16 x float> [[TMP9]] to <16 x i32>
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast <16 x float> [[PASSTHRU:%.*]] to <16 x i32>
+; CHECK-NEXT:    [[TMP15:%.*]] = xor <16 x i32> [[TMP13]], [[TMP14]]
+; CHECK-NEXT:    [[TMP16:%.*]] = or <16 x i32> [[TMP15]], zeroinitializer
+; CHECK-NEXT:    [[TMP17:%.*]] = or <16 x i32> [[TMP16]], [[TMP4]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP10]], <16 x i32> [[TMP17]], <16 x i32> [[TMP12]]
+; CHECK-NEXT:    [[TMP18:%.*]] = select <16 x i1> [[TMP11]], <16 x float> [[TMP9]], <16 x float> [[PASSTHRU]]
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x float> [[TMP18]]
+;
+  %1 = call <16 x float> @llvm.x86.avx512.mul.ps.512(<16 x float> %a0, <16 x float> %a1, i32 9)
+  %2 = bitcast i16 %mask to <16 x i1>
+  %3 = select <16 x i1> %2, <16 x float> %1, <16 x float> %passthru
+  ret <16 x float> %3
+}
+
+define <16 x float> @test_vmulps_mask_passthru_ru(<16 x float> %a0, <16 x float> %a1, <16 x float> %passthru, i16 %mask) #0 {
+; CHECK-LABEL: @test_vmulps_mask_passthru_ru(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP6]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP7:%.*]], label [[TMP8:%.*]], !prof [[PROF1]]
+; CHECK:       7:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       8:
+; CHECK-NEXT:    [[TMP9:%.*]] = call <16 x float> @llvm.x86.avx512.mul.ps.512(<16 x float> [[A0:%.*]], <16 x float> [[A1:%.*]], i32 10)
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i16 [[TMP3]] to <16 x i1>
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast i16 [[MASK:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP12:%.*]] = select <16 x i1> [[TMP11]], <16 x i32> zeroinitializer, <16 x i32> [[TMP4]]
+; CHECK-NEXT:    [[TMP13:%.*]] = bitcast <16 x float> [[TMP9]] to <16 x i32>
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast <16 x float> [[PASSTHRU:%.*]] to <16 x i32>
+; CHECK-NEXT:    [[TMP15:%.*]] = xor <16 x i32> [[TMP13]], [[TMP14]]
+; CHECK-NEXT:    [[TMP16:%.*]] = or <16 x i32> [[TMP15]], zeroinitializer
+; CHECK-NEXT:    [[TMP17:%.*]] = or <16 x i32> [[TMP16]], [[TMP4]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP10]], <16 x i32> [[TMP17]], <16 x i32> [[TMP12]]
+; CHECK-NEXT:    [[TMP18:%.*]] = select <16 x i1> [[TMP11]], <16 x float> [[TMP9]], <16 x float> [[PASSTHRU]]
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x float> [[TMP18]]
+;
+  %1 = call <16 x float> @llvm.x86.avx512.mul.ps.512(<16 x float> %a0, <16 x float> %a1, i32 10)
+  %2 = bitcast i16 %mask to <16 x i1>
+  %3 = select <16 x i1> %2, <16 x float> %1, <16 x float> %passthru
+  ret <16 x float> %3
+}
+
+define <16 x float> @test_vmulps_mask_passthru_rz(<16 x float> %a0, <16 x float> %a1, <16 x float> %passthru, i16 %mask) #0 {
+; CHECK-LABEL: @test_vmulps_mask_passthru_rz(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP6]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP7:%.*]], label [[TMP8:%.*]], !prof [[PROF1]]
+; CHECK:       7:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       8:
+; CHECK-NEXT:    [[TMP9:%.*]] = call <16 x float> @llvm.x86.avx512.mul.ps.512(<16 x float> [[A0:%.*]], <16 x float> [[A1:%.*]], i32 11)
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i16 [[TMP3]] to <16 x i1>
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast i16 [[MASK:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP12:%.*]] = select <16 x i1> [[TMP11]], <16 x i32> zeroinitializer, <16 x i32> [[TMP4]]
+; CHECK-NEXT:    [[TMP13:%.*]] = bitcast <16 x float> [[TMP9]] to <16 x i32>
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast <16 x float> [[PASSTHRU:%.*]] to <16 x i32>
+; CHECK-NEXT:    [[TMP15:%.*]] = xor <16 x i32> [[TMP13]], [[TMP14]]
+; CHECK-NEXT:    [[TMP16:%.*]] = or <16 x i32> [[TMP15]], zeroinitializer
+; CHECK-NEXT:    [[TMP17:%.*]] = or <16 x i32> [[TMP16]], [[TMP4]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP10]], <16 x i32> [[TMP17]], <16 x i32> [[TMP12]]
+; CHECK-NEXT:    [[TMP18:%.*]] = select <16 x i1> [[TMP11]], <16 x float> [[TMP9]], <16 x float> [[PASSTHRU]]
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x float> [[TMP18]]
+;
+  %1 = call <16 x float> @llvm.x86.avx512.mul.ps.512(<16 x float> %a0, <16 x float> %a1, i32 11)
+  %2 = bitcast i16 %mask to <16 x i1>
+  %3 = select <16 x i1> %2, <16 x float> %1, <16 x float> %passthru
+  ret <16 x float> %3
+}
+
+define <8 x double> @test_vmulpd_mask_rn(<8 x double> %a0, <8 x double> %a1, i8 %mask) #0 {
+; CHECK-LABEL: @test_vmulpd_mask_rn(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <8 x i64> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <8 x i64> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP6:%.*]], label [[TMP7:%.*]], !prof [[PROF1]]
+; CHECK:       6:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       7:
+; CHECK-NEXT:    [[TMP8:%.*]] = call <8 x double> @llvm.x86.avx512.mul.pd.512(<8 x double> [[A0:%.*]], <8 x double> [[A1:%.*]], i32 8)
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast i8 [[TMP3]] to <8 x i1>
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP11:%.*]] = select <8 x i1> [[TMP10]], <8 x i64> zeroinitializer, <8 x i64> zeroinitializer
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <8 x double> [[TMP8]] to <8 x i64>
+; CHECK-NEXT:    [[TMP13:%.*]] = xor <8 x i64> [[TMP12]], zeroinitializer
+; CHECK-NEXT:    [[TMP14:%.*]] = or <8 x i64> [[TMP13]], zeroinitializer
+; CHECK-NEXT:    [[TMP15:%.*]] = or <8 x i64> [[TMP14]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP9]], <8 x i64> [[TMP15]], <8 x i64> [[TMP11]]
+; CHECK-NEXT:    [[TMP16:%.*]] = select <8 x i1> [[TMP10]], <8 x double> [[TMP8]], <8 x double> zeroinitializer
+; CHECK-NEXT:    store <8 x i64> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x double> [[TMP16]]
+;
+  %1 = call <8 x double> @llvm.x86.avx512.mul.pd.512(<8 x double> %a0, <8 x double> %a1, i32 8)
+  %2 = bitcast i8 %mask to <8 x i1>
+  %3 = select <8 x i1> %2, <8 x double> %1, <8 x double> zeroinitializer
+  ret <8 x double> %3
+}
+
+define <8 x double> @test_vmulpd_mask_rd(<8 x double> %a0, <8 x double> %a1, i8 %mask) #0 {
+; CHECK-LABEL: @test_vmulpd_mask_rd(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <8 x i64> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <8 x i64> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP6:%.*]], label [[TMP7:%.*]], !prof [[PROF1]]
+; CHECK:       6:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       7:
+; CHECK-NEXT:    [[TMP8:%.*]] = call <8 x double> @llvm.x86.avx512.mul.pd.512(<8 x double> [[A0:%.*]], <8 x double> [[A1:%.*]], i32 9)
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast i8 [[TMP3]] to <8 x i1>
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP11:%.*]] = select <8 x i1> [[TMP10]], <8 x i64> zeroinitializer, <8 x i64> zeroinitializer
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <8 x double> [[TMP8]] to <8 x i64>
+; CHECK-NEXT:    [[TMP13:%.*]] = xor <8 x i64> [[TMP12]], zeroinitializer
+; CHECK-NEXT:    [[TMP14:%.*]] = or <8 x i64> [[TMP13]], zeroinitializer
+; CHECK-NEXT:    [[TMP15:%.*]] = or <8 x i64> [[TMP14]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP9]], <8 x i64> [[TMP15]], <8 x i64> [[TMP11]]
+; CHECK-NEXT:    [[TMP16:%.*]] = select <8 x i1> [[TMP10]], <8 x double> [[TMP8]], <8 x double> zeroinitializer
+; CHECK-NEXT:    store <8 x i64> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x double> [[TMP16]]
+;
+  %1 = call <8 x double> @llvm.x86.avx512.mul.pd.512(<8 x double> %a0, <8 x double> %a1, i32 9)
+  %2 = bitcast i8 %mask to <8 x i1>
+  %3 = select <8 x i1> %2, <8 x double> %1, <8 x double> zeroinitializer
+  ret <8 x double> %3
+}
+
+define <8 x double> @test_vmulpd_mask_ru(<8 x double> %a0, <8 x double> %a1, i8 %mask) #0 {
+; CHECK-LABEL: @test_vmulpd_mask_ru(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <8 x i64> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <8 x i64> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP6:%.*]], label [[TMP7:%.*]], !prof [[PROF1]]
+; CHECK:       6:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       7:
+; CHECK-NEXT:    [[TMP8:%.*]] = call <8 x double> @llvm.x86.avx512.mul.pd.512(<8 x double> [[A0:%.*]], <8 x double> [[A1:%.*]], i32 10)
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast i8 [[TMP3]] to <8 x i1>
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP11:%.*]] = select <8 x i1> [[TMP10]], <8 x i64> zeroinitializer, <8 x i64> zeroinitializer
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <8 x double> [[TMP8]] to <8 x i64>
+; CHECK-NEXT:    [[TMP13:%.*]] = xor <8 x i64> [[TMP12]], zeroinitializer
+; CHECK-NEXT:    [[TMP14:%.*]] = or <8 x i64> [[TMP13]], zeroinitializer
+; CHECK-NEXT:    [[TMP15:%.*]] = or <8 x i64> [[TMP14]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP9]], <8 x i64> [[TMP15]], <8 x i64> [[TMP11]]
+; CHECK-NEXT:    [[TMP16:%.*]] = select <8 x i1> [[TMP10]], <8 x double> [[TMP8]], <8 x double> zeroinitializer
+; CHECK-NEXT:    store <8 x i64> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x double> [[TMP16]]
+;
+  %1 = call <8 x double> @llvm.x86.avx512.mul.pd.512(<8 x double> %a0, <8 x double> %a1, i32 10)
+  %2 = bitcast i8 %mask to <8 x i1>
+  %3 = select <8 x i1> %2, <8 x double> %1, <8 x double> zeroinitializer
+  ret <8 x double> %3
+}
+
+define <8 x double> @test_vmulpd_mask_rz(<8 x double> %a0, <8 x double> %a1, i8 %mask) #0 {
+; CHECK-LABEL: @test_vmulpd_mask_rz(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <8 x i64> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <8 x i64> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP6:%.*]], label [[TMP7:%.*]], !prof [[PROF1]]
+; CHECK:       6:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       7:
+; CHECK-NEXT:    [[TMP8:%.*]] = call <8 x double> @llvm.x86.avx512.mul.pd.512(<8 x double> [[A0:%.*]], <8 x double> [[A1:%.*]], i32 11)
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast i8 [[TMP3]] to <8 x i1>
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP11:%.*]] = select <8 x i1> [[TMP10]], <8 x i64> zeroinitializer, <8 x i64> zeroinitializer
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <8 x double> [[TMP8]] to <8 x i64>
+; CHECK-NEXT:    [[TMP13:%.*]] = xor <8 x i64> [[TMP12]], zeroinitializer
+; CHECK-NEXT:    [[TMP14:%.*]] = or <8 x i64> [[TMP13]], zeroinitializer
+; CHECK-NEXT:    [[TMP15:%.*]] = or <8 x i64> [[TMP14]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP9]], <8 x i64> [[TMP15]], <8 x i64> [[TMP11]]
+; CHECK-NEXT:    [[TMP16:%.*]] = select <8 x i1> [[TMP10]], <8 x double> [[TMP8]], <8 x double> zeroinitializer
+; CHECK-NEXT:    store <8 x i64> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x double> [[TMP16]]
+;
+  %1 = call <8 x double> @llvm.x86.avx512.mul.pd.512(<8 x double> %a0, <8 x double> %a1, i32 11)
+  %2 = bitcast i8 %mask to <8 x i1>
+  %3 = select <8 x i1> %2, <8 x double> %1, <8 x double> zeroinitializer
+  ret <8 x double> %3
+}
+
+define <16 x float> @test_mm512_maskz_add_round_ps_rn_sae(<16 x float> %a0, <16 x float> %a1, i16 %mask) #0 {
+; CHECK-LABEL: @test_mm512_maskz_add_round_ps_rn_sae(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP6:%.*]], label [[TMP7:%.*]], !prof [[PROF1]]
+; CHECK:       6:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       7:
+; CHECK-NEXT:    [[TMP8:%.*]] = call <16 x float> @llvm.x86.avx512.add.ps.512(<16 x float> [[A0:%.*]], <16 x float> [[A1:%.*]], i32 8)
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast i16 [[TMP3]] to <16 x i1>
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i16 [[MASK:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP11:%.*]] = select <16 x i1> [[TMP10]], <16 x i32> zeroinitializer, <16 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <16 x float> [[TMP8]] to <16 x i32>
+; CHECK-NEXT:    [[TMP13:%.*]] = xor <16 x i32> [[TMP12]], zeroinitializer
+; CHECK-NEXT:    [[TMP14:%.*]] = or <16 x i32> [[TMP13]], zeroinitializer
+; CHECK-NEXT:    [[TMP15:%.*]] = or <16 x i32> [[TMP14]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP9]], <16 x i32> [[TMP15]], <16 x i32> [[TMP11]]
+; CHECK-NEXT:    [[TMP16:%.*]] = select <16 x i1> [[TMP10]], <16 x float> [[TMP8]], <16 x float> zeroinitializer
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x float> [[TMP16]]
+;
+  %1 = call <16 x float> @llvm.x86.avx512.add.ps.512(<16 x float> %a0, <16 x float> %a1, i32 8)
+  %2 = bitcast i16 %mask to <16 x i1>
+  %3 = select <16 x i1> %2, <16 x float> %1, <16 x float> zeroinitializer
+  ret <16 x float> %3
+}
+
+define <16 x float> @test_mm512_maskz_add_round_ps_rd_sae(<16 x float> %a0, <16 x float> %a1, i16 %mask) #0 {
+; CHECK-LABEL: @test_mm512_maskz_add_round_ps_rd_sae(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP6:%.*]], label [[TMP7:%.*]], !prof [[PROF1]]
+; CHECK:       6:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       7:
+; CHECK-NEXT:    [[TMP8:%.*]] = call <16 x float> @llvm.x86.avx512.add.ps.512(<16 x float> [[A0:%.*]], <16 x float> [[A1:%.*]], i32 9)
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast i16 [[TMP3]] to <16 x i1>
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i16 [[MASK:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP11:%.*]] = select <16 x i1> [[TMP10]], <16 x i32> zeroinitializer, <16 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <16 x float> [[TMP8]] to <16 x i32>
+; CHECK-NEXT:    [[TMP13:%.*]] = xor <16 x i32> [[TMP12]], zeroinitializer
+; CHECK-NEXT:    [[TMP14:%.*]] = or <16 x i32> [[TMP13]], zeroinitializer
+; CHECK-NEXT:    [[TMP15:%.*]] = or <16 x i32> [[TMP14]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP9]], <16 x i32> [[TMP15]], <16 x i32> [[TMP11]]
+; CHECK-NEXT:    [[TMP16:%.*]] = select <16 x i1> [[TMP10]], <16 x float> [[TMP8]], <16 x float> zeroinitializer
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x float> [[TMP16]]
+;
+  %1 = call <16 x float> @llvm.x86.avx512.add.ps.512(<16 x float> %a0, <16 x float> %a1, i32 9)
+  %2 = bitcast i16 %mask to <16 x i1>
+  %3 = select <16 x i1> %2, <16 x float> %1, <16 x float> zeroinitializer
+  ret <16 x float> %3
+}
+
+define <16 x float> @test_mm512_maskz_add_round_ps_ru_sae(<16 x float> %a0, <16 x float> %a1, i16 %mask) #0 {
+; CHECK-LABEL: @test_mm512_maskz_add_round_ps_ru_sae(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP6:%.*]], label [[TMP7:%.*]], !prof [[PROF1]]
+; CHECK:       6:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       7:
+; CHECK-NEXT:    [[TMP8:%.*]] = call <16 x float> @llvm.x86.avx512.add.ps.512(<16 x float> [[A0:%.*]], <16 x float> [[A1:%.*]], i32 10)
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast i16 [[TMP3]] to <16 x i1>
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i16 [[MASK:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP11:%.*]] = select <16 x i1> [[TMP10]], <16 x i32> zeroinitializer, <16 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <16 x float> [[TMP8]] to <16 x i32>
+; CHECK-NEXT:    [[TMP13:%.*]] = xor <16 x i32> [[TMP12]], zeroinitializer
+; CHECK-NEXT:    [[TMP14:%.*]] = or <16 x i32> [[TMP13]], zeroinitializer
+; CHECK-NEXT:    [[TMP15:%.*]] = or <16 x i32> [[TMP14]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP9]], <16 x i32> [[TMP15]], <16 x i32> [[TMP11]]
+; CHECK-NEXT:    [[TMP16:%.*]] = select <16 x i1> [[TMP10]], <16 x float> [[TMP8]], <16 x float> zeroinitializer
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x float> [[TMP16]]
+;
+  %1 = call <16 x float> @llvm.x86.avx512.add.ps.512(<16 x float> %a0, <16 x float> %a1, i32 10)
+  %2 = bitcast i16 %mask to <16 x i1>
+  %3 = select <16 x i1> %2, <16 x float> %1, <16 x float> zeroinitializer
+  ret <16 x float> %3
+}
+
+define <16 x float> @test_mm512_maskz_add_round_ps_rz_sae(<16 x float> %a0, <16 x float> %a1, i16 %mask) #0 {
+; CHECK-LABEL: @test_mm512_maskz_add_round_ps_rz_sae(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP6:%.*]], label [[TMP7:%.*]], !prof [[PROF1]]
+; CHECK:       6:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       7:
+; CHECK-NEXT:    [[TMP8:%.*]] = call <16 x float> @llvm.x86.avx512.add.ps.512(<16 x float> [[A0:%.*]], <16 x float> [[A1:%.*]], i32 11)
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast i16 [[TMP3]] to <16 x i1>
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i16 [[MASK:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP11:%.*]] = select <16 x i1> [[TMP10]], <16 x i32> zeroinitializer, <16 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <16 x float> [[TMP8]] to <16 x i32>
+; CHECK-NEXT:    [[TMP13:%.*]] = xor <16 x i32> [[TMP12]], zeroinitializer
+; CHECK-NEXT:    [[TMP14:%.*]] = or <16 x i32> [[TMP13]], zeroinitializer
+; CHECK-NEXT:    [[TMP15:%.*]] = or <16 x i32> [[TMP14]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP9]], <16 x i32> [[TMP15]], <16 x i32> [[TMP11]]
+; CHECK-NEXT:    [[TMP16:%.*]] = select <16 x i1> [[TMP10]], <16 x float> [[TMP8]], <16 x float> zeroinitializer
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x float> [[TMP16]]
+;
+  %1 = call <16 x float> @llvm.x86.avx512.add.ps.512(<16 x float> %a0, <16 x float> %a1, i32 11)
+  %2 = bitcast i16 %mask to <16 x i1>
+  %3 = select <16 x i1> %2, <16 x float> %1, <16 x float> zeroinitializer
+  ret <16 x float> %3
+}
+
+define <16 x float> @test_mm512_maskz_add_round_ps_current(<16 x float> %a0, <16 x float> %a1, i16 %mask) #0 {
+; CHECK-LABEL: @test_mm512_maskz_add_round_ps_current(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP6:%.*]], label [[TMP7:%.*]], !prof [[PROF1]]
+; CHECK:       6:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       7:
+; CHECK-NEXT:    [[TMP8:%.*]] = call <16 x float> @llvm.x86.avx512.add.ps.512(<16 x float> [[A0:%.*]], <16 x float> [[A1:%.*]], i32 4)
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast i16 [[TMP3]] to <16 x i1>
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i16 [[MASK:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP11:%.*]] = select <16 x i1> [[TMP10]], <16 x i32> zeroinitializer, <16 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <16 x float> [[TMP8]] to <16 x i32>
+; CHECK-NEXT:    [[TMP13:%.*]] = xor <16 x i32> [[TMP12]], zeroinitializer
+; CHECK-NEXT:    [[TMP14:%.*]] = or <16 x i32> [[TMP13]], zeroinitializer
+; CHECK-NEXT:    [[TMP15:%.*]] = or <16 x i32> [[TMP14]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP9]], <16 x i32> [[TMP15]], <16 x i32> [[TMP11]]
+; CHECK-NEXT:    [[TMP16:%.*]] = select <16 x i1> [[TMP10]], <16 x float> [[TMP8]], <16 x float> zeroinitializer
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x float> [[TMP16]]
+;
+  %1 = call <16 x float> @llvm.x86.avx512.add.ps.512(<16 x float> %a0, <16 x float> %a1, i32 4)
+  %2 = bitcast i16 %mask to <16 x i1>
+  %3 = select <16 x i1> %2, <16 x float> %1, <16 x float> zeroinitializer
+  ret <16 x float> %3
+}
+
+define <16 x float> @test_mm512_mask_add_round_ps_rn_sae(<16 x float> %a0, <16 x float> %a1, <16 x float> %src, i16 %mask) #0 {
+; CHECK-LABEL: @test_mm512_mask_add_round_ps_rn_sae(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP6]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP7:%.*]], label [[TMP8:%.*]], !prof [[PROF1]]
+; CHECK:       7:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       8:
+; CHECK-NEXT:    [[TMP9:%.*]] = call <16 x float> @llvm.x86.avx512.add.ps.512(<16 x float> [[A0:%.*]], <16 x float> [[A1:%.*]], i32 8)
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i16 [[TMP3]] to <16 x i1>
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast i16 [[MASK:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP12:%.*]] = select <16 x i1> [[TMP11]], <16 x i32> zeroinitializer, <16 x i32> [[TMP4]]
+; CHECK-NEXT:    [[TMP13:%.*]] = bitcast <16 x float> [[TMP9]] to <16 x i32>
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast <16 x float> [[SRC:%.*]] to <16 x i32>
+; CHECK-NEXT:    [[TMP15:%.*]] = xor <16 x i32> [[TMP13]], [[TMP14]]
+; CHECK-NEXT:    [[TMP16:%.*]] = or <16 x i32> [[TMP15]], zeroinitializer
+; CHECK-NEXT:    [[TMP17:%.*]] = or <16 x i32> [[TMP16]], [[TMP4]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP10]], <16 x i32> [[TMP17]], <16 x i32> [[TMP12]]
+; CHECK-NEXT:    [[TMP18:%.*]] = select <16 x i1> [[TMP11]], <16 x float> [[TMP9]], <16 x float> [[SRC]]
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x float> [[TMP18]]
+;
+  %1 = call <16 x float> @llvm.x86.avx512.add.ps.512(<16 x float> %a0, <16 x float> %a1, i32 8)
+  %2 = bitcast i16 %mask to <16 x i1>
+  %3 = select <16 x i1> %2, <16 x float> %1, <16 x float> %src
+  ret <16 x float> %3
+}
+
+define <16 x float> @test_mm512_mask_add_round_ps_rd_sae(<16 x float> %a0, <16 x float> %a1, <16 x float> %src, i16 %mask) #0 {
+; CHECK-LABEL: @test_mm512_mask_add_round_ps_rd_sae(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP6]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP7:%.*]], label [[TMP8:%.*]], !prof [[PROF1]]
+; CHECK:       7:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       8:
+; CHECK-NEXT:    [[TMP9:%.*]] = call <16 x float> @llvm.x86.avx512.add.ps.512(<16 x float> [[A0:%.*]], <16 x float> [[A1:%.*]], i32 9)
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i16 [[TMP3]] to <16 x i1>
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast i16 [[MASK:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP12:%.*]] = select <16 x i1> [[TMP11]], <16 x i32> zeroinitializer, <16 x i32> [[TMP4]]
+; CHECK-NEXT:    [[TMP13:%.*]] = bitcast <16 x float> [[TMP9]] to <16 x i32>
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast <16 x float> [[SRC:%.*]] to <16 x i32>
+; CHECK-NEXT:    [[TMP15:%.*]] = xor <16 x i32> [[TMP13]], [[TMP14]]
+; CHECK-NEXT:    [[TMP16:%.*]] = or <16 x i32> [[TMP15]], zeroinitializer
+; CHECK-NEXT:    [[TMP17:%.*]] = or <16 x i32> [[TMP16]], [[TMP4]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP10]], <16 x i32> [[TMP17]], <16 x i32> [[TMP12]]
+; CHECK-NEXT:    [[TMP18:%.*]] = select <16 x i1> [[TMP11]], <16 x float> [[TMP9]], <16 x float> [[SRC]]
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x float> [[TMP18]]
+;
+  %1 = call <16 x float> @llvm.x86.avx512.add.ps.512(<16 x float> %a0, <16 x float> %a1, i32 9)
+  %2 = bitcast i16 %mask to <16 x i1>
+  %3 = select <16 x i1> %2, <16 x float> %1, <16 x float> %src
+  ret <16 x float> %3
+}
+
+define <16 x float> @test_mm512_mask_add_round_ps_ru_sae(<16 x float> %a0, <16 x float> %a1, <16 x float> %src, i16 %mask) #0 {
+; CHECK-LABEL: @test_mm512_mask_add_round_ps_ru_sae(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP6]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP7:%.*]], label [[TMP8:%.*]], !prof [[PROF1]]
+; CHECK:       7:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       8:
+; CHECK-NEXT:    [[TMP9:%.*]] = call <16 x float> @llvm.x86.avx512.add.ps.512(<16 x float> [[A0:%.*]], <16 x float> [[A1:%.*]], i32 10)
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i16 [[TMP3]] to <16 x i1>
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast i16 [[MASK:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP12:%.*]] = select <16 x i1> [[TMP11]], <16 x i32> zeroinitializer, <16 x i32> [[TMP4]]
+; CHECK-NEXT:    [[TMP13:%.*]] = bitcast <16 x float> [[TMP9]] to <16 x i32>
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast <16 x float> [[SRC:%.*]] to <16 x i32>
+; CHECK-NEXT:    [[TMP15:%.*]] = xor <16 x i32> [[TMP13]], [[TMP14]]
+; CHECK-NEXT:    [[TMP16:%.*]] = or <16 x i32> [[TMP15]], zeroinitializer
+; CHECK-NEXT:    [[TMP17:%.*]] = or <16 x i32> [[TMP16]], [[TMP4]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP10]], <16 x i32> [[TMP17]], <16 x i32> [[TMP12]]
+; CHECK-NEXT:    [[TMP18:%.*]] = select <16 x i1> [[TMP11]], <16 x float> [[TMP9]], <16 x float> [[SRC]]
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x float> [[TMP18]]
+;
+  %1 = call <16 x float> @llvm.x86.avx512.add.ps.512(<16 x float> %a0, <16 x float> %a1, i32 10)
+  %2 = bitcast i16 %mask to <16 x i1>
+  %3 = select <16 x i1> %2, <16 x float> %1, <16 x float> %src
+  ret <16 x float> %3
+}
+
+define <16 x float> @test_mm512_mask_add_round_ps_rz_sae(<16 x float> %a0, <16 x float> %a1, <16 x float> %src, i16 %mask) #0 {
+; CHECK-LABEL: @test_mm512_mask_add_round_ps_rz_sae(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP6]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP7:%.*]], label [[TMP8:%.*]], !prof [[PROF1]]
+; CHECK:       7:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       8:
+; CHECK-NEXT:    [[TMP9:%.*]] = call <16 x float> @llvm.x86.avx512.add.ps.512(<16 x float> [[A0:%.*]], <16 x float> [[A1:%.*]], i32 11)
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i16 [[TMP3]] to <16 x i1>
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast i16 [[MASK:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP12:%.*]] = select <16 x i1> [[TMP11]], <16 x i32> zeroinitializer, <16 x i32> [[TMP4]]
+; CHECK-NEXT:    [[TMP13:%.*]] = bitcast <16 x float> [[TMP9]] to <16 x i32>
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast <16 x float> [[SRC:%.*]] to <16 x i32>
+; CHECK-NEXT:    [[TMP15:%.*]] = xor <16 x i32> [[TMP13]], [[TMP14]]
+; CHECK-NEXT:    [[TMP16:%.*]] = or <16 x i32> [[TMP15]], zeroinitializer
+; CHECK-NEXT:    [[TMP17:%.*]] = or <16 x i32> [[TMP16]], [[TMP4]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP10]], <16 x i32> [[TMP17]], <16 x i32> [[TMP12]]
+; CHECK-NEXT:    [[TMP18:%.*]] = select <16 x i1> [[TMP11]], <16 x float> [[TMP9]], <16 x float> [[SRC]]
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x float> [[TMP18]]
+;
+  %1 = call <16 x float> @llvm.x86.avx512.add.ps.512(<16 x float> %a0, <16 x float> %a1, i32 11)
+  %2 = bitcast i16 %mask to <16 x i1>
+  %3 = select <16 x i1> %2, <16 x float> %1, <16 x float> %src
+  ret <16 x float> %3
+}
+
+define <16 x float> @test_mm512_mask_add_round_ps_current(<16 x float> %a0, <16 x float> %a1, <16 x float> %src, i16 %mask) #0 {
+; CHECK-LABEL: @test_mm512_mask_add_round_ps_current(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP6]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP7:%.*]], label [[TMP8:%.*]], !prof [[PROF1]]
+; CHECK:       7:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       8:
+; CHECK-NEXT:    [[TMP9:%.*]] = call <16 x float> @llvm.x86.avx512.add.ps.512(<16 x float> [[A0:%.*]], <16 x float> [[A1:%.*]], i32 4)
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i16 [[TMP3]] to <16 x i1>
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast i16 [[MASK:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP12:%.*]] = select <16 x i1> [[TMP11]], <16 x i32> zeroinitializer, <16 x i32> [[TMP4]]
+; CHECK-NEXT:    [[TMP13:%.*]] = bitcast <16 x float> [[TMP9]] to <16 x i32>
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast <16 x float> [[SRC:%.*]] to <16 x i32>
+; CHECK-NEXT:    [[TMP15:%.*]] = xor <16 x i32> [[TMP13]], [[TMP14]]
+; CHECK-NEXT:    [[TMP16:%.*]] = or <16 x i32> [[TMP15]], zeroinitializer
+; CHECK-NEXT:    [[TMP17:%.*]] = or <16 x i32> [[TMP16]], [[TMP4]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP10]], <16 x i32> [[TMP17]], <16 x i32> [[TMP12]]
+; CHECK-NEXT:    [[TMP18:%.*]] = select <16 x i1> [[TMP11]], <16 x float> [[TMP9]], <16 x float> [[SRC]]
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x float> [[TMP18]]
+;
+  %1 = call <16 x float> @llvm.x86.avx512.add.ps.512(<16 x float> %a0, <16 x float> %a1, i32 4)
+  %2 = bitcast i16 %mask to <16 x i1>
+  %3 = select <16 x i1> %2, <16 x float> %1, <16 x float> %src
+  ret <16 x float> %3
+}
+
+define <16 x float> @test_mm512_add_round_ps_rn_sae(<16 x float> %a0, <16 x float> %a1, i16 %mask) #0 {
+; CHECK-LABEL: @test_mm512_add_round_ps_rn_sae(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP3]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP5:%.*]], label [[TMP6:%.*]], !prof [[PROF1]]
+; CHECK:       5:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       6:
+; CHECK-NEXT:    [[TMP7:%.*]] = call <16 x float> @llvm.x86.avx512.add.ps.512(<16 x float> [[A0:%.*]], <16 x float> [[A1:%.*]], i32 8)
+; CHECK-NEXT:    store <16 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x float> [[TMP7]]
+;
+  %1 = call <16 x float> @llvm.x86.avx512.add.ps.512(<16 x float> %a0, <16 x float> %a1, i32 8)
+  ret <16 x float> %1
+}
+
+define <16 x float> @test_mm512_add_round_ps_rd_sae(<16 x float> %a0, <16 x float> %a1, i16 %mask) #0 {
+; CHECK-LABEL: @test_mm512_add_round_ps_rd_sae(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP3]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP5:%.*]], label [[TMP6:%.*]], !prof [[PROF1]]
+; CHECK:       5:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       6:
+; CHECK-NEXT:    [[TMP7:%.*]] = call <16 x float> @llvm.x86.avx512.add.ps.512(<16 x float> [[A0:%.*]], <16 x float> [[A1:%.*]], i32 9)
+; CHECK-NEXT:    store <16 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x float> [[TMP7]]
+;
+  %1 = call <16 x float> @llvm.x86.avx512.add.ps.512(<16 x float> %a0, <16 x float> %a1, i32 9)
+  ret <16 x float> %1
+}
+
+define <16 x float> @test_mm512_add_round_ps_ru_sae(<16 x float> %a0, <16 x float> %a1, i16 %mask) #0 {
+; CHECK-LABEL: @test_mm512_add_round_ps_ru_sae(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP3]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP5:%.*]], label [[TMP6:%.*]], !prof [[PROF1]]
+; CHECK:       5:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       6:
+; CHECK-NEXT:    [[TMP7:%.*]] = call <16 x float> @llvm.x86.avx512.add.ps.512(<16 x float> [[A0:%.*]], <16 x float> [[A1:%.*]], i32 10)
+; CHECK-NEXT:    store <16 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x float> [[TMP7]]
+;
+  %1 = call <16 x float> @llvm.x86.avx512.add.ps.512(<16 x float> %a0, <16 x float> %a1, i32 10)
+  ret <16 x float> %1
+}
+
+define <16 x float> @test_mm512_add_round_ps_rz_sae(<16 x float> %a0, <16 x float> %a1, i16 %mask) #0 {
+; CHECK-LABEL: @test_mm512_add_round_ps_rz_sae(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP3]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP5:%.*]], label [[TMP6:%.*]], !prof [[PROF1]]
+; CHECK:       5:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       6:
+; CHECK-NEXT:    [[TMP7:%.*]] = call <16 x float> @llvm.x86.avx512.add.ps.512(<16 x float> [[A0:%.*]], <16 x float> [[A1:%.*]], i32 11)
+; CHECK-NEXT:    store <16 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x float> [[TMP7]]
+;
+  %1 = call <16 x float> @llvm.x86.avx512.add.ps.512(<16 x float> %a0, <16 x float> %a1, i32 11)
+  ret <16 x float> %1
+}
+
+define <16 x float> @test_mm512_add_round_ps_current(<16 x float> %a0, <16 x float> %a1, i16 %mask) #0 {
+; CHECK-LABEL: @test_mm512_add_round_ps_current(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP3]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP5:%.*]], label [[TMP6:%.*]], !prof [[PROF1]]
+; CHECK:       5:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       6:
+; CHECK-NEXT:    [[TMP7:%.*]] = call <16 x float> @llvm.x86.avx512.add.ps.512(<16 x float> [[A0:%.*]], <16 x float> [[A1:%.*]], i32 4)
+; CHECK-NEXT:    store <16 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x float> [[TMP7]]
+;
+  %1 = call <16 x float> @llvm.x86.avx512.add.ps.512(<16 x float> %a0, <16 x float> %a1, i32 4)
+  ret <16 x float> %1
+}
+declare <16 x float> @llvm.x86.avx512.add.ps.512(<16 x float>, <16 x float>, i32)
+
+define <16 x float> @test_mm512_mask_sub_round_ps_rn_sae(<16 x float> %a0, <16 x float> %a1, <16 x float> %src, i16 %mask) #0 {
+; CHECK-LABEL: @test_mm512_mask_sub_round_ps_rn_sae(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP6]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP7:%.*]], label [[TMP8:%.*]], !prof [[PROF1]]
+; CHECK:       7:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       8:
+; CHECK-NEXT:    [[TMP9:%.*]] = call <16 x float> @llvm.x86.avx512.sub.ps.512(<16 x float> [[A0:%.*]], <16 x float> [[A1:%.*]], i32 8)
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i16 [[TMP3]] to <16 x i1>
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast i16 [[MASK:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP12:%.*]] = select <16 x i1> [[TMP11]], <16 x i32> zeroinitializer, <16 x i32> [[TMP4]]
+; CHECK-NEXT:    [[TMP13:%.*]] = bitcast <16 x float> [[TMP9]] to <16 x i32>
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast <16 x float> [[SRC:%.*]] to <16 x i32>
+; CHECK-NEXT:    [[TMP15:%.*]] = xor <16 x i32> [[TMP13]], [[TMP14]]
+; CHECK-NEXT:    [[TMP16:%.*]] = or <16 x i32> [[TMP15]], zeroinitializer
+; CHECK-NEXT:    [[TMP17:%.*]] = or <16 x i32> [[TMP16]], [[TMP4]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP10]], <16 x i32> [[TMP17]], <16 x i32> [[TMP12]]
+; CHECK-NEXT:    [[TMP18:%.*]] = select <16 x i1> [[TMP11]], <16 x float> [[TMP9]], <16 x float> [[SRC]]
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x float> [[TMP18]]
+;
+  %1 = call <16 x float> @llvm.x86.avx512.sub.ps.512(<16 x float> %a0, <16 x float> %a1, i32 8)
+  %2 = bitcast i16 %mask to <16 x i1>
+  %3 = select <16 x i1> %2, <16 x float> %1, <16 x float> %src
+  ret <16 x float> %3
+}
+
+define <16 x float> @test_mm512_mask_sub_round_ps_rd_sae(<16 x float> %a0, <16 x float> %a1, <16 x float> %src, i16 %mask) #0 {
+; CHECK-LABEL: @test_mm512_mask_sub_round_ps_rd_sae(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP6]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP7:%.*]], label [[TMP8:%.*]], !prof [[PROF1]]
+; CHECK:       7:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       8:
+; CHECK-NEXT:    [[TMP9:%.*]] = call <16 x float> @llvm.x86.avx512.sub.ps.512(<16 x float> [[A0:%.*]], <16 x float> [[A1:%.*]], i32 9)
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i16 [[TMP3]] to <16 x i1>
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast i16 [[MASK:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP12:%.*]] = select <16 x i1> [[TMP11]], <16 x i32> zeroinitializer, <16 x i32> [[TMP4]]
+; CHECK-NEXT:    [[TMP13:%.*]] = bitcast <16 x float> [[TMP9]] to <16 x i32>
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast <16 x float> [[SRC:%.*]] to <16 x i32>
+; CHECK-NEXT:    [[TMP15:%.*]] = xor <16 x i32> [[TMP13]], [[TMP14]]
+; CHECK-NEXT:    [[TMP16:%.*]] = or <16 x i32> [[TMP15]], zeroinitializer
+; CHECK-NEXT:    [[TMP17:%.*]] = or <16 x i32> [[TMP16]], [[TMP4]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP10]], <16 x i32> [[TMP17]], <16 x i32> [[TMP12]]
+; CHECK-NEXT:    [[TMP18:%.*]] = select <16 x i1> [[TMP11]], <16 x float> [[TMP9]], <16 x float> [[SRC]]
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x float> [[TMP18]]
+;
+  %1 = call <16 x float> @llvm.x86.avx512.sub.ps.512(<16 x float> %a0, <16 x float> %a1, i32 9)
+  %2 = bitcast i16 %mask to <16 x i1>
+  %3 = select <16 x i1> %2, <16 x float> %1, <16 x float> %src
+  ret <16 x float> %3
+}
+
+define <16 x float> @test_mm512_mask_sub_round_ps_ru_sae(<16 x float> %a0, <16 x float> %a1, <16 x float> %src, i16 %mask) #0 {
+; CHECK-LABEL: @test_mm512_mask_sub_round_ps_ru_sae(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP6]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP7:%.*]], label [[TMP8:%.*]], !prof [[PROF1]]
+; CHECK:       7:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       8:
+; CHECK-NEXT:    [[TMP9:%.*]] = call <16 x float> @llvm.x86.avx512.sub.ps.512(<16 x float> [[A0:%.*]], <16 x float> [[A1:%.*]], i32 10)
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i16 [[TMP3]] to <16 x i1>
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast i16 [[MASK:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP12:%.*]] = select <16 x i1> [[TMP11]], <16 x i32> zeroinitializer, <16 x i32> [[TMP4]]
+; CHECK-NEXT:    [[TMP13:%.*]] = bitcast <16 x float> [[TMP9]] to <16 x i32>
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast <16 x float> [[SRC:%.*]] to <16 x i32>
+; CHECK-NEXT:    [[TMP15:%.*]] = xor <16 x i32> [[TMP13]], [[TMP14]]
+; CHECK-NEXT:    [[TMP16:%.*]] = or <16 x i32> [[TMP15]], zeroinitializer
+; CHECK-NEXT:    [[TMP17:%.*]] = or <16 x i32> [[TMP16]], [[TMP4]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP10]], <16 x i32> [[TMP17]], <16 x i32> [[TMP12]]
+; CHECK-NEXT:    [[TMP18:%.*]] = select <16 x i1> [[TMP11]], <16 x float> [[TMP9]], <16 x float> [[SRC]]
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x float> [[TMP18]]
+;
+  %1 = call <16 x float> @llvm.x86.avx512.sub.ps.512(<16 x float> %a0, <16 x float> %a1, i32 10)
+  %2 = bitcast i16 %mask to <16 x i1>
+  %3 = select <16 x i1> %2, <16 x float> %1, <16 x float> %src
+  ret <16 x float> %3
+}
+
+define <16 x float> @test_mm512_mask_sub_round_ps_rz_sae(<16 x float> %a0, <16 x float> %a1, <16 x float> %src, i16 %mask) #0 {
+; CHECK-LABEL: @test_mm512_mask_sub_round_ps_rz_sae(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP6]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP7:%.*]], label [[TMP8:%.*]], !prof [[PROF1]]
+; CHECK:       7:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       8:
+; CHECK-NEXT:    [[TMP9:%.*]] = call <16 x float> @llvm.x86.avx512.sub.ps.512(<16 x float> [[A0:%.*]], <16 x float> [[A1:%.*]], i32 11)
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i16 [[TMP3]] to <16 x i1>
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast i16 [[MASK:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP12:%.*]] = select <16 x i1> [[TMP11]], <16 x i32> zeroinitializer, <16 x i32> [[TMP4]]
+; CHECK-NEXT:    [[TMP13:%.*]] = bitcast <16 x float> [[TMP9]] to <16 x i32>
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast <16 x float> [[SRC:%.*]] to <16 x i32>
+; CHECK-NEXT:    [[TMP15:%.*]] = xor <16 x i32> [[TMP13]], [[TMP14]]
+; CHECK-NEXT:    [[TMP16:%.*]] = or <16 x i32> [[TMP15]], zeroinitializer
+; CHECK-NEXT:    [[TMP17:%.*]] = or <16 x i32> [[TMP16]], [[TMP4]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP10]], <16 x i32> [[TMP17]], <16 x i32> [[TMP12]]
+; CHECK-NEXT:    [[TMP18:%.*]] = select <16 x i1> [[TMP11]], <16 x float> [[TMP9]], <16 x float> [[SRC]]
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x float> [[TMP18]]
+;
+  %1 = call <16 x float> @llvm.x86.avx512.sub.ps.512(<16 x float> %a0, <16 x float> %a1, i32 11)
+  %2 = bitcast i16 %mask to <16 x i1>
+  %3 = select <16 x i1> %2, <16 x float> %1, <16 x float> %src
+  ret <16 x float> %3
+}
+
+define <16 x float> @test_mm512_mask_sub_round_ps_current(<16 x float> %a0, <16 x float> %a1, <16 x float> %src, i16 %mask) #0 {
+; CHECK-LABEL: @test_mm512_mask_sub_round_ps_current(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP6]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP7:%.*]], label [[TMP8:%.*]], !prof [[PROF1]]
+; CHECK:       7:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       8:
+; CHECK-NEXT:    [[TMP9:%.*]] = call <16 x float> @llvm.x86.avx512.sub.ps.512(<16 x float> [[A0:%.*]], <16 x float> [[A1:%.*]], i32 4)
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i16 [[TMP3]] to <16 x i1>
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast i16 [[MASK:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP12:%.*]] = select <16 x i1> [[TMP11]], <16 x i32> zeroinitializer, <16 x i32> [[TMP4]]
+; CHECK-NEXT:    [[TMP13:%.*]] = bitcast <16 x float> [[TMP9]] to <16 x i32>
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast <16 x float> [[SRC:%.*]] to <16 x i32>
+; CHECK-NEXT:    [[TMP15:%.*]] = xor <16 x i32> [[TMP13]], [[TMP14]]
+; CHECK-NEXT:    [[TMP16:%.*]] = or <16 x i32> [[TMP15]], zeroinitializer
+; CHECK-NEXT:    [[TMP17:%.*]] = or <16 x i32> [[TMP16]], [[TMP4]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP10]], <16 x i32> [[TMP17]], <16 x i32> [[TMP12]]
+; CHECK-NEXT:    [[TMP18:%.*]] = select <16 x i1> [[TMP11]], <16 x float> [[TMP9]], <16 x float> [[SRC]]
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x float> [[TMP18]]
+;
+  %1 = call <16 x float> @llvm.x86.avx512.sub.ps.512(<16 x float> %a0, <16 x float> %a1, i32 4)
+  %2 = bitcast i16 %mask to <16 x i1>
+  %3 = select <16 x i1> %2, <16 x float> %1, <16 x float> %src
+  ret <16 x float> %3
+}
+
+define <16 x float> @test_mm512_sub_round_ps_rn_sae(<16 x float> %a0, <16 x float> %a1, i16 %mask) #0 {
+; CHECK-LABEL: @test_mm512_sub_round_ps_rn_sae(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP3]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP5:%.*]], label [[TMP6:%.*]], !prof [[PROF1]]
+; CHECK:       5:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       6:
+; CHECK-NEXT:    [[TMP7:%.*]] = call <16 x float> @llvm.x86.avx512.sub.ps.512(<16 x float> [[A0:%.*]], <16 x float> [[A1:%.*]], i32 8)
+; CHECK-NEXT:    store <16 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x float> [[TMP7]]
+;
+  %1 = call <16 x float> @llvm.x86.avx512.sub.ps.512(<16 x float> %a0, <16 x float> %a1, i32 8)
+  ret <16 x float> %1
+}
+
+define <16 x float> @test_mm512_sub_round_ps_rd_sae(<16 x float> %a0, <16 x float> %a1, i16 %mask) #0 {
+; CHECK-LABEL: @test_mm512_sub_round_ps_rd_sae(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP3]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP5:%.*]], label [[TMP6:%.*]], !prof [[PROF1]]
+; CHECK:       5:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       6:
+; CHECK-NEXT:    [[TMP7:%.*]] = call <16 x float> @llvm.x86.avx512.sub.ps.512(<16 x float> [[A0:%.*]], <16 x float> [[A1:%.*]], i32 9)
+; CHECK-NEXT:    store <16 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x float> [[TMP7]]
+;
+  %1 = call <16 x float> @llvm.x86.avx512.sub.ps.512(<16 x float> %a0, <16 x float> %a1, i32 9)
+  ret <16 x float> %1
+}
+
+define <16 x float> @test_mm512_sub_round_ps_ru_sae(<16 x float> %a0, <16 x float> %a1, i16 %mask) #0 {
+; CHECK-LABEL: @test_mm512_sub_round_ps_ru_sae(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP3]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP5:%.*]], label [[TMP6:%.*]], !prof [[PROF1]]
+; CHECK:       5:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       6:
+; CHECK-NEXT:    [[TMP7:%.*]] = call <16 x float> @llvm.x86.avx512.sub.ps.512(<16 x float> [[A0:%.*]], <16 x float> [[A1:%.*]], i32 10)
+; CHECK-NEXT:    store <16 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x float> [[TMP7]]
+;
+  %1 = call <16 x float> @llvm.x86.avx512.sub.ps.512(<16 x float> %a0, <16 x float> %a1, i32 10)
+  ret <16 x float> %1
+}
+
+define <16 x float> @test_mm512_sub_round_ps_rz_sae(<16 x float> %a0, <16 x float> %a1, i16 %mask) #0 {
+; CHECK-LABEL: @test_mm512_sub_round_ps_rz_sae(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP3]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP5:%.*]], label [[TMP6:%.*]], !prof [[PROF1]]
+; CHECK:       5:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       6:
+; CHECK-NEXT:    [[TMP7:%.*]] = call <16 x float> @llvm.x86.avx512.sub.ps.512(<16 x float> [[A0:%.*]], <16 x float> [[A1:%.*]], i32 11)
+; CHECK-NEXT:    store <16 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x float> [[TMP7]]
+;
+  %1 = call <16 x float> @llvm.x86.avx512.sub.ps.512(<16 x float> %a0, <16 x float> %a1, i32 11)
+  ret <16 x float> %1
+}
+
+define <16 x float> @test_mm512_sub_round_ps_current(<16 x float> %a0, <16 x float> %a1, i16 %mask) #0 {
+; CHECK-LABEL: @test_mm512_sub_round_ps_current(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP3]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP5:%.*]], label [[TMP6:%.*]], !prof [[PROF1]]
+; CHECK:       5:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       6:
+; CHECK-NEXT:    [[TMP7:%.*]] = call <16 x float> @llvm.x86.avx512.sub.ps.512(<16 x float> [[A0:%.*]], <16 x float> [[A1:%.*]], i32 4)
+; CHECK-NEXT:    store <16 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x float> [[TMP7]]
+;
+  %1 = call <16 x float> @llvm.x86.avx512.sub.ps.512(<16 x float> %a0, <16 x float> %a1, i32 4)
+  ret <16 x float> %1
+}
+
+define <16 x float> @test_mm512_maskz_div_round_ps_rn_sae(<16 x float> %a0, <16 x float> %a1, i16 %mask) #0 {
+; CHECK-LABEL: @test_mm512_maskz_div_round_ps_rn_sae(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP6:%.*]], label [[TMP7:%.*]], !prof [[PROF1]]
+; CHECK:       6:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       7:
+; CHECK-NEXT:    [[TMP8:%.*]] = call <16 x float> @llvm.x86.avx512.div.ps.512(<16 x float> [[A0:%.*]], <16 x float> [[A1:%.*]], i32 8)
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast i16 [[TMP3]] to <16 x i1>
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i16 [[MASK:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP11:%.*]] = select <16 x i1> [[TMP10]], <16 x i32> zeroinitializer, <16 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <16 x float> [[TMP8]] to <16 x i32>
+; CHECK-NEXT:    [[TMP13:%.*]] = xor <16 x i32> [[TMP12]], zeroinitializer
+; CHECK-NEXT:    [[TMP14:%.*]] = or <16 x i32> [[TMP13]], zeroinitializer
+; CHECK-NEXT:    [[TMP15:%.*]] = or <16 x i32> [[TMP14]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP9]], <16 x i32> [[TMP15]], <16 x i32> [[TMP11]]
+; CHECK-NEXT:    [[TMP16:%.*]] = select <16 x i1> [[TMP10]], <16 x float> [[TMP8]], <16 x float> zeroinitializer
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x float> [[TMP16]]
+;
+  %1 = call <16 x float> @llvm.x86.avx512.div.ps.512(<16 x float> %a0, <16 x float> %a1, i32 8)
+  %2 = bitcast i16 %mask to <16 x i1>
+  %3 = select <16 x i1> %2, <16 x float> %1, <16 x float> zeroinitializer
+  ret <16 x float> %3
+}
+
+define <16 x float> @test_mm512_maskz_div_round_ps_rd_sae(<16 x float> %a0, <16 x float> %a1, i16 %mask) #0 {
+; CHECK-LABEL: @test_mm512_maskz_div_round_ps_rd_sae(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP6:%.*]], label [[TMP7:%.*]], !prof [[PROF1]]
+; CHECK:       6:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       7:
+; CHECK-NEXT:    [[TMP8:%.*]] = call <16 x float> @llvm.x86.avx512.div.ps.512(<16 x float> [[A0:%.*]], <16 x float> [[A1:%.*]], i32 9)
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast i16 [[TMP3]] to <16 x i1>
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i16 [[MASK:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP11:%.*]] = select <16 x i1> [[TMP10]], <16 x i32> zeroinitializer, <16 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <16 x float> [[TMP8]] to <16 x i32>
+; CHECK-NEXT:    [[TMP13:%.*]] = xor <16 x i32> [[TMP12]], zeroinitializer
+; CHECK-NEXT:    [[TMP14:%.*]] = or <16 x i32> [[TMP13]], zeroinitializer
+; CHECK-NEXT:    [[TMP15:%.*]] = or <16 x i32> [[TMP14]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP9]], <16 x i32> [[TMP15]], <16 x i32> [[TMP11]]
+; CHECK-NEXT:    [[TMP16:%.*]] = select <16 x i1> [[TMP10]], <16 x float> [[TMP8]], <16 x float> zeroinitializer
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x float> [[TMP16]]
+;
+  %1 = call <16 x float> @llvm.x86.avx512.div.ps.512(<16 x float> %a0, <16 x float> %a1, i32 9)
+  %2 = bitcast i16 %mask to <16 x i1>
+  %3 = select <16 x i1> %2, <16 x float> %1, <16 x float> zeroinitializer
+  ret <16 x float> %3
+}
+
+define <16 x float> @test_mm512_maskz_div_round_ps_ru_sae(<16 x float> %a0, <16 x float> %a1, i16 %mask) #0 {
+; CHECK-LABEL: @test_mm512_maskz_div_round_ps_ru_sae(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP6:%.*]], label [[TMP7:%.*]], !prof [[PROF1]]
+; CHECK:       6:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       7:
+; CHECK-NEXT:    [[TMP8:%.*]] = call <16 x float> @llvm.x86.avx512.div.ps.512(<16 x float> [[A0:%.*]], <16 x float> [[A1:%.*]], i32 10)
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast i16 [[TMP3]] to <16 x i1>
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i16 [[MASK:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP11:%.*]] = select <16 x i1> [[TMP10]], <16 x i32> zeroinitializer, <16 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <16 x float> [[TMP8]] to <16 x i32>
+; CHECK-NEXT:    [[TMP13:%.*]] = xor <16 x i32> [[TMP12]], zeroinitializer
+; CHECK-NEXT:    [[TMP14:%.*]] = or <16 x i32> [[TMP13]], zeroinitializer
+; CHECK-NEXT:    [[TMP15:%.*]] = or <16 x i32> [[TMP14]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP9]], <16 x i32> [[TMP15]], <16 x i32> [[TMP11]]
+; CHECK-NEXT:    [[TMP16:%.*]] = select <16 x i1> [[TMP10]], <16 x float> [[TMP8]], <16 x float> zeroinitializer
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x float> [[TMP16]]
+;
+  %1 = call <16 x float> @llvm.x86.avx512.div.ps.512(<16 x float> %a0, <16 x float> %a1, i32 10)
+  %2 = bitcast i16 %mask to <16 x i1>
+  %3 = select <16 x i1> %2, <16 x float> %1, <16 x float> zeroinitializer
+  ret <16 x float> %3
+}
+
+define <16 x float> @test_mm512_maskz_div_round_ps_rz_sae(<16 x float> %a0, <16 x float> %a1, i16 %mask) #0 {
+; CHECK-LABEL: @test_mm512_maskz_div_round_ps_rz_sae(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP6:%.*]], label [[TMP7:%.*]], !prof [[PROF1]]
+; CHECK:       6:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       7:
+; CHECK-NEXT:    [[TMP8:%.*]] = call <16 x float> @llvm.x86.avx512.div.ps.512(<16 x float> [[A0:%.*]], <16 x float> [[A1:%.*]], i32 11)
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast i16 [[TMP3]] to <16 x i1>
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i16 [[MASK:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP11:%.*]] = select <16 x i1> [[TMP10]], <16 x i32> zeroinitializer, <16 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <16 x float> [[TMP8]] to <16 x i32>
+; CHECK-NEXT:    [[TMP13:%.*]] = xor <16 x i32> [[TMP12]], zeroinitializer
+; CHECK-NEXT:    [[TMP14:%.*]] = or <16 x i32> [[TMP13]], zeroinitializer
+; CHECK-NEXT:    [[TMP15:%.*]] = or <16 x i32> [[TMP14]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP9]], <16 x i32> [[TMP15]], <16 x i32> [[TMP11]]
+; CHECK-NEXT:    [[TMP16:%.*]] = select <16 x i1> [[TMP10]], <16 x float> [[TMP8]], <16 x float> zeroinitializer
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x float> [[TMP16]]
+;
+  %1 = call <16 x float> @llvm.x86.avx512.div.ps.512(<16 x float> %a0, <16 x float> %a1, i32 11)
+  %2 = bitcast i16 %mask to <16 x i1>
+  %3 = select <16 x i1> %2, <16 x float> %1, <16 x float> zeroinitializer
+  ret <16 x float> %3
+}
+
+define <16 x float> @test_mm512_maskz_div_round_ps_current(<16 x float> %a0, <16 x float> %a1, i16 %mask) #0 {
+; CHECK-LABEL: @test_mm512_maskz_div_round_ps_current(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP6:%.*]], label [[TMP7:%.*]], !prof [[PROF1]]
+; CHECK:       6:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       7:
+; CHECK-NEXT:    [[TMP8:%.*]] = call <16 x float> @llvm.x86.avx512.div.ps.512(<16 x float> [[A0:%.*]], <16 x float> [[A1:%.*]], i32 4)
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast i16 [[TMP3]] to <16 x i1>
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i16 [[MASK:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP11:%.*]] = select <16 x i1> [[TMP10]], <16 x i32> zeroinitializer, <16 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <16 x float> [[TMP8]] to <16 x i32>
+; CHECK-NEXT:    [[TMP13:%.*]] = xor <16 x i32> [[TMP12]], zeroinitializer
+; CHECK-NEXT:    [[TMP14:%.*]] = or <16 x i32> [[TMP13]], zeroinitializer
+; CHECK-NEXT:    [[TMP15:%.*]] = or <16 x i32> [[TMP14]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP9]], <16 x i32> [[TMP15]], <16 x i32> [[TMP11]]
+; CHECK-NEXT:    [[TMP16:%.*]] = select <16 x i1> [[TMP10]], <16 x float> [[TMP8]], <16 x float> zeroinitializer
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x float> [[TMP16]]
+;
+  %1 = call <16 x float> @llvm.x86.avx512.div.ps.512(<16 x float> %a0, <16 x float> %a1, i32 4)
+  %2 = bitcast i16 %mask to <16 x i1>
+  %3 = select <16 x i1> %2, <16 x float> %1, <16 x float> zeroinitializer
+  ret <16 x float> %3
+}
+
+define <16 x float> @test_mm512_mask_div_round_ps_rn_sae(<16 x float> %a0, <16 x float> %a1, <16 x float> %src, i16 %mask) #0 {
+; CHECK-LABEL: @test_mm512_mask_div_round_ps_rn_sae(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP6]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP7:%.*]], label [[TMP8:%.*]], !prof [[PROF1]]
+; CHECK:       7:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       8:
+; CHECK-NEXT:    [[TMP9:%.*]] = call <16 x float> @llvm.x86.avx512.div.ps.512(<16 x float> [[A0:%.*]], <16 x float> [[A1:%.*]], i32 8)
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i16 [[TMP3]] to <16 x i1>
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast i16 [[MASK:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP12:%.*]] = select <16 x i1> [[TMP11]], <16 x i32> zeroinitializer, <16 x i32> [[TMP4]]
+; CHECK-NEXT:    [[TMP13:%.*]] = bitcast <16 x float> [[TMP9]] to <16 x i32>
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast <16 x float> [[SRC:%.*]] to <16 x i32>
+; CHECK-NEXT:    [[TMP15:%.*]] = xor <16 x i32> [[TMP13]], [[TMP14]]
+; CHECK-NEXT:    [[TMP16:%.*]] = or <16 x i32> [[TMP15]], zeroinitializer
+; CHECK-NEXT:    [[TMP17:%.*]] = or <16 x i32> [[TMP16]], [[TMP4]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP10]], <16 x i32> [[TMP17]], <16 x i32> [[TMP12]]
+; CHECK-NEXT:    [[TMP18:%.*]] = select <16 x i1> [[TMP11]], <16 x float> [[TMP9]], <16 x float> [[SRC]]
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x float> [[TMP18]]
+;
+  %1 = call <16 x float> @llvm.x86.avx512.div.ps.512(<16 x float> %a0, <16 x float> %a1, i32 8)
+  %2 = bitcast i16 %mask to <16 x i1>
+  %3 = select <16 x i1> %2, <16 x float> %1, <16 x float> %src
+  ret <16 x float> %3
+}
+
+define <16 x float> @test_mm512_mask_div_round_ps_rd_sae(<16 x float> %a0, <16 x float> %a1, <16 x float> %src, i16 %mask) #0 {
+; CHECK-LABEL: @test_mm512_mask_div_round_ps_rd_sae(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP6]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP7:%.*]], label [[TMP8:%.*]], !prof [[PROF1]]
+; CHECK:       7:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       8:
+; CHECK-NEXT:    [[TMP9:%.*]] = call <16 x float> @llvm.x86.avx512.div.ps.512(<16 x float> [[A0:%.*]], <16 x float> [[A1:%.*]], i32 9)
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i16 [[TMP3]] to <16 x i1>
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast i16 [[MASK:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP12:%.*]] = select <16 x i1> [[TMP11]], <16 x i32> zeroinitializer, <16 x i32> [[TMP4]]
+; CHECK-NEXT:    [[TMP13:%.*]] = bitcast <16 x float> [[TMP9]] to <16 x i32>
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast <16 x float> [[SRC:%.*]] to <16 x i32>
+; CHECK-NEXT:    [[TMP15:%.*]] = xor <16 x i32> [[TMP13]], [[TMP14]]
+; CHECK-NEXT:    [[TMP16:%.*]] = or <16 x i32> [[TMP15]], zeroinitializer
+; CHECK-NEXT:    [[TMP17:%.*]] = or <16 x i32> [[TMP16]], [[TMP4]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP10]], <16 x i32> [[TMP17]], <16 x i32> [[TMP12]]
+; CHECK-NEXT:    [[TMP18:%.*]] = select <16 x i1> [[TMP11]], <16 x float> [[TMP9]], <16 x float> [[SRC]]
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x float> [[TMP18]]
+;
+  %1 = call <16 x float> @llvm.x86.avx512.div.ps.512(<16 x float> %a0, <16 x float> %a1, i32 9)
+  %2 = bitcast i16 %mask to <16 x i1>
+  %3 = select <16 x i1> %2, <16 x float> %1, <16 x float> %src
+  ret <16 x float> %3
+}
+
+define <16 x float> @test_mm512_mask_div_round_ps_ru_sae(<16 x float> %a0, <16 x float> %a1, <16 x float> %src, i16 %mask) #0 {
+; CHECK-LABEL: @test_mm512_mask_div_round_ps_ru_sae(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP6]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP7:%.*]], label [[TMP8:%.*]], !prof [[PROF1]]
+; CHECK:       7:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       8:
+; CHECK-NEXT:    [[TMP9:%.*]] = call <16 x float> @llvm.x86.avx512.div.ps.512(<16 x float> [[A0:%.*]], <16 x float> [[A1:%.*]], i32 10)
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i16 [[TMP3]] to <16 x i1>
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast i16 [[MASK:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP12:%.*]] = select <16 x i1> [[TMP11]], <16 x i32> zeroinitializer, <16 x i32> [[TMP4]]
+; CHECK-NEXT:    [[TMP13:%.*]] = bitcast <16 x float> [[TMP9]] to <16 x i32>
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast <16 x float> [[SRC:%.*]] to <16 x i32>
+; CHECK-NEXT:    [[TMP15:%.*]] = xor <16 x i32> [[TMP13]], [[TMP14]]
+; CHECK-NEXT:    [[TMP16:%.*]] = or <16 x i32> [[TMP15]], zeroinitializer
+; CHECK-NEXT:    [[TMP17:%.*]] = or <16 x i32> [[TMP16]], [[TMP4]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP10]], <16 x i32> [[TMP17]], <16 x i32> [[TMP12]]
+; CHECK-NEXT:    [[TMP18:%.*]] = select <16 x i1> [[TMP11]], <16 x float> [[TMP9]], <16 x float> [[SRC]]
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x float> [[TMP18]]
+;
+  %1 = call <16 x float> @llvm.x86.avx512.div.ps.512(<16 x float> %a0, <16 x float> %a1, i32 10)
+  %2 = bitcast i16 %mask to <16 x i1>
+  %3 = select <16 x i1> %2, <16 x float> %1, <16 x float> %src
+  ret <16 x float> %3
+}
+
+define <16 x float> @test_mm512_mask_div_round_ps_rz_sae(<16 x float> %a0, <16 x float> %a1, <16 x float> %src, i16 %mask) #0 {
+; CHECK-LABEL: @test_mm512_mask_div_round_ps_rz_sae(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP6]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP7:%.*]], label [[TMP8:%.*]], !prof [[PROF1]]
+; CHECK:       7:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       8:
+; CHECK-NEXT:    [[TMP9:%.*]] = call <16 x float> @llvm.x86.avx512.div.ps.512(<16 x float> [[A0:%.*]], <16 x float> [[A1:%.*]], i32 11)
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i16 [[TMP3]] to <16 x i1>
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast i16 [[MASK:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP12:%.*]] = select <16 x i1> [[TMP11]], <16 x i32> zeroinitializer, <16 x i32> [[TMP4]]
+; CHECK-NEXT:    [[TMP13:%.*]] = bitcast <16 x float> [[TMP9]] to <16 x i32>
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast <16 x float> [[SRC:%.*]] to <16 x i32>
+; CHECK-NEXT:    [[TMP15:%.*]] = xor <16 x i32> [[TMP13]], [[TMP14]]
+; CHECK-NEXT:    [[TMP16:%.*]] = or <16 x i32> [[TMP15]], zeroinitializer
+; CHECK-NEXT:    [[TMP17:%.*]] = or <16 x i32> [[TMP16]], [[TMP4]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP10]], <16 x i32> [[TMP17]], <16 x i32> [[TMP12]]
+; CHECK-NEXT:    [[TMP18:%.*]] = select <16 x i1> [[TMP11]], <16 x float> [[TMP9]], <16 x float> [[SRC]]
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x float> [[TMP18]]
+;
+  %1 = call <16 x float> @llvm.x86.avx512.div.ps.512(<16 x float> %a0, <16 x float> %a1, i32 11)
+  %2 = bitcast i16 %mask to <16 x i1>
+  %3 = select <16 x i1> %2, <16 x float> %1, <16 x float> %src
+  ret <16 x float> %3
+}
+
+define <16 x float> @test_mm512_mask_div_round_ps_current(<16 x float> %a0, <16 x float> %a1, <16 x float> %src, i16 %mask) #0 {
+; CHECK-LABEL: @test_mm512_mask_div_round_ps_current(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP6]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP7:%.*]], label [[TMP8:%.*]], !prof [[PROF1]]
+; CHECK:       7:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       8:
+; CHECK-NEXT:    [[TMP9:%.*]] = call <16 x float> @llvm.x86.avx512.div.ps.512(<16 x float> [[A0:%.*]], <16 x float> [[A1:%.*]], i32 4)
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i16 [[TMP3]] to <16 x i1>
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast i16 [[MASK:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP12:%.*]] = select <16 x i1> [[TMP11]], <16 x i32> zeroinitializer, <16 x i32> [[TMP4]]
+; CHECK-NEXT:    [[TMP13:%.*]] = bitcast <16 x float> [[TMP9]] to <16 x i32>
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast <16 x float> [[SRC:%.*]] to <16 x i32>
+; CHECK-NEXT:    [[TMP15:%.*]] = xor <16 x i32> [[TMP13]], [[TMP14]]
+; CHECK-NEXT:    [[TMP16:%.*]] = or <16 x i32> [[TMP15]], zeroinitializer
+; CHECK-NEXT:    [[TMP17:%.*]] = or <16 x i32> [[TMP16]], [[TMP4]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP10]], <16 x i32> [[TMP17]], <16 x i32> [[TMP12]]
+; CHECK-NEXT:    [[TMP18:%.*]] = select <16 x i1> [[TMP11]], <16 x float> [[TMP9]], <16 x float> [[SRC]]
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x float> [[TMP18]]
+;
+  %1 = call <16 x float> @llvm.x86.avx512.div.ps.512(<16 x float> %a0, <16 x float> %a1, i32 4)
+  %2 = bitcast i16 %mask to <16 x i1>
+  %3 = select <16 x i1> %2, <16 x float> %1, <16 x float> %src
+  ret <16 x float> %3
+}
+
+define <16 x float> @test_mm512_div_round_ps_rn_sae(<16 x float> %a0, <16 x float> %a1, i16 %mask) #0 {
+; CHECK-LABEL: @test_mm512_div_round_ps_rn_sae(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP3]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP5:%.*]], label [[TMP6:%.*]], !prof [[PROF1]]
+; CHECK:       5:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       6:
+; CHECK-NEXT:    [[TMP7:%.*]] = call <16 x float> @llvm.x86.avx512.div.ps.512(<16 x float> [[A0:%.*]], <16 x float> [[A1:%.*]], i32 8)
+; CHECK-NEXT:    store <16 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x float> [[TMP7]]
+;
+  %1 = call <16 x float> @llvm.x86.avx512.div.ps.512(<16 x float> %a0, <16 x float> %a1, i32 8)
+  ret <16 x float> %1
+}
+
+define <16 x float> @test_mm512_div_round_ps_rd_sae(<16 x float> %a0, <16 x float> %a1, i16 %mask) #0 {
+; CHECK-LABEL: @test_mm512_div_round_ps_rd_sae(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP3]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP5:%.*]], label [[TMP6:%.*]], !prof [[PROF1]]
+; CHECK:       5:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       6:
+; CHECK-NEXT:    [[TMP7:%.*]] = call <16 x float> @llvm.x86.avx512.div.ps.512(<16 x float> [[A0:%.*]], <16 x float> [[A1:%.*]], i32 9)
+; CHECK-NEXT:    store <16 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x float> [[TMP7]]
+;
+  %1 = call <16 x float> @llvm.x86.avx512.div.ps.512(<16 x float> %a0, <16 x float> %a1, i32 9)
+  ret <16 x float> %1
+}
+
+define <16 x float> @test_mm512_div_round_ps_ru_sae(<16 x float> %a0, <16 x float> %a1, i16 %mask) #0 {
+; CHECK-LABEL: @test_mm512_div_round_ps_ru_sae(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP3]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP5:%.*]], label [[TMP6:%.*]], !prof [[PROF1]]
+; CHECK:       5:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       6:
+; CHECK-NEXT:    [[TMP7:%.*]] = call <16 x float> @llvm.x86.avx512.div.ps.512(<16 x float> [[A0:%.*]], <16 x float> [[A1:%.*]], i32 10)
+; CHECK-NEXT:    store <16 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x float> [[TMP7]]
+;
+  %1 = call <16 x float> @llvm.x86.avx512.div.ps.512(<16 x float> %a0, <16 x float> %a1, i32 10)
+  ret <16 x float> %1
+}
+
+define <16 x float> @test_mm512_div_round_ps_rz_sae(<16 x float> %a0, <16 x float> %a1, i16 %mask) #0 {
+; CHECK-LABEL: @test_mm512_div_round_ps_rz_sae(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP3]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP5:%.*]], label [[TMP6:%.*]], !prof [[PROF1]]
+; CHECK:       5:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       6:
+; CHECK-NEXT:    [[TMP7:%.*]] = call <16 x float> @llvm.x86.avx512.div.ps.512(<16 x float> [[A0:%.*]], <16 x float> [[A1:%.*]], i32 11)
+; CHECK-NEXT:    store <16 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x float> [[TMP7]]
+;
+  %1 = call <16 x float> @llvm.x86.avx512.div.ps.512(<16 x float> %a0, <16 x float> %a1, i32 11)
+  ret <16 x float> %1
+}
+
+define <16 x float> @test_mm512_div_round_ps_current(<16 x float> %a0, <16 x float> %a1, i16 %mask) #0 {
+; CHECK-LABEL: @test_mm512_div_round_ps_current(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP3]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP5:%.*]], label [[TMP6:%.*]], !prof [[PROF1]]
+; CHECK:       5:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       6:
+; CHECK-NEXT:    [[TMP7:%.*]] = call <16 x float> @llvm.x86.avx512.div.ps.512(<16 x float> [[A0:%.*]], <16 x float> [[A1:%.*]], i32 4)
+; CHECK-NEXT:    store <16 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x float> [[TMP7]]
+;
+  %1 = call <16 x float> @llvm.x86.avx512.div.ps.512(<16 x float> %a0, <16 x float> %a1, i32 4)
+  ret <16 x float> %1
+}
+declare <16 x float> @llvm.x86.avx512.div.ps.512(<16 x float>, <16 x float>, i32)
+
+define <16 x float> @test_mm512_maskz_min_round_ps_sae(<16 x float> %a0, <16 x float> %a1, i16 %mask) #0 {
+; CHECK-LABEL: @test_mm512_maskz_min_round_ps_sae(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <16 x i32> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <16 x i32> [[_MSPROP]], zeroinitializer
+; CHECK-NEXT:    [[TMP8:%.*]] = call <16 x float> @llvm.x86.avx512.min.ps.512(<16 x float> [[A0:%.*]], <16 x float> [[A1:%.*]], i32 8)
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast i16 [[TMP3]] to <16 x i1>
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i16 [[MASK:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP11:%.*]] = select <16 x i1> [[TMP10]], <16 x i32> [[_MSPROP1]], <16 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <16 x float> [[TMP8]] to <16 x i32>
+; CHECK-NEXT:    [[TMP13:%.*]] = xor <16 x i32> [[TMP12]], zeroinitializer
+; CHECK-NEXT:    [[TMP14:%.*]] = or <16 x i32> [[TMP13]], [[_MSPROP1]]
+; CHECK-NEXT:    [[TMP15:%.*]] = or <16 x i32> [[TMP14]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP9]], <16 x i32> [[TMP15]], <16 x i32> [[TMP11]]
+; CHECK-NEXT:    [[TMP16:%.*]] = select <16 x i1> [[TMP10]], <16 x float> [[TMP8]], <16 x float> zeroinitializer
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x float> [[TMP16]]
+;
+  %1 = call <16 x float> @llvm.x86.avx512.min.ps.512(<16 x float> %a0, <16 x float> %a1, i32 8)
+  %2 = bitcast i16 %mask to <16 x i1>
+  %3 = select <16 x i1> %2, <16 x float> %1, <16 x float> zeroinitializer
+  ret <16 x float> %3
+}
+
+define <16 x float> @test_mm512_maskz_min_round_ps_current(<16 x float> %a0, <16 x float> %a1, i16 %mask) #0 {
+; CHECK-LABEL: @test_mm512_maskz_min_round_ps_current(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <16 x i32> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <16 x i32> [[_MSPROP]], zeroinitializer
+; CHECK-NEXT:    [[TMP8:%.*]] = call <16 x float> @llvm.x86.avx512.min.ps.512(<16 x float> [[A0:%.*]], <16 x float> [[A1:%.*]], i32 4)
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast i16 [[TMP3]] to <16 x i1>
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i16 [[MASK:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP11:%.*]] = select <16 x i1> [[TMP10]], <16 x i32> [[_MSPROP1]], <16 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <16 x float> [[TMP8]] to <16 x i32>
+; CHECK-NEXT:    [[TMP13:%.*]] = xor <16 x i32> [[TMP12]], zeroinitializer
+; CHECK-NEXT:    [[TMP14:%.*]] = or <16 x i32> [[TMP13]], [[_MSPROP1]]
+; CHECK-NEXT:    [[TMP15:%.*]] = or <16 x i32> [[TMP14]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP9]], <16 x i32> [[TMP15]], <16 x i32> [[TMP11]]
+; CHECK-NEXT:    [[TMP16:%.*]] = select <16 x i1> [[TMP10]], <16 x float> [[TMP8]], <16 x float> zeroinitializer
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x float> [[TMP16]]
+;
+  %1 = call <16 x float> @llvm.x86.avx512.min.ps.512(<16 x float> %a0, <16 x float> %a1, i32 4)
+  %2 = bitcast i16 %mask to <16 x i1>
+  %3 = select <16 x i1> %2, <16 x float> %1, <16 x float> zeroinitializer
+  ret <16 x float> %3
+}
+
+define <16 x float> @test_mm512_mask_min_round_ps_sae(<16 x float> %a0, <16 x float> %a1, <16 x float> %src, i16 %mask) #0 {
+; CHECK-LABEL: @test_mm512_mask_min_round_ps_sae(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <16 x i32> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <16 x i32> [[_MSPROP]], zeroinitializer
+; CHECK-NEXT:    [[TMP9:%.*]] = call <16 x float> @llvm.x86.avx512.min.ps.512(<16 x float> [[A0:%.*]], <16 x float> [[A1:%.*]], i32 8)
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i16 [[TMP3]] to <16 x i1>
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast i16 [[MASK:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP12:%.*]] = select <16 x i1> [[TMP11]], <16 x i32> [[_MSPROP1]], <16 x i32> [[TMP4]]
+; CHECK-NEXT:    [[TMP13:%.*]] = bitcast <16 x float> [[TMP9]] to <16 x i32>
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast <16 x float> [[SRC:%.*]] to <16 x i32>
+; CHECK-NEXT:    [[TMP15:%.*]] = xor <16 x i32> [[TMP13]], [[TMP14]]
+; CHECK-NEXT:    [[TMP16:%.*]] = or <16 x i32> [[TMP15]], [[_MSPROP1]]
+; CHECK-NEXT:    [[TMP17:%.*]] = or <16 x i32> [[TMP16]], [[TMP4]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP10]], <16 x i32> [[TMP17]], <16 x i32> [[TMP12]]
+; CHECK-NEXT:    [[TMP18:%.*]] = select <16 x i1> [[TMP11]], <16 x float> [[TMP9]], <16 x float> [[SRC]]
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x float> [[TMP18]]
+;
+  %1 = call <16 x float> @llvm.x86.avx512.min.ps.512(<16 x float> %a0, <16 x float> %a1, i32 8)
+  %2 = bitcast i16 %mask to <16 x i1>
+  %3 = select <16 x i1> %2, <16 x float> %1, <16 x float> %src
+  ret <16 x float> %3
+}
+
+define <16 x float> @test_mm512_mask_min_round_ps_current(<16 x float> %a0, <16 x float> %a1, <16 x float> %src, i16 %mask) #0 {
+; CHECK-LABEL: @test_mm512_mask_min_round_ps_current(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <16 x i32> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <16 x i32> [[_MSPROP]], zeroinitializer
+; CHECK-NEXT:    [[TMP9:%.*]] = call <16 x float> @llvm.x86.avx512.min.ps.512(<16 x float> [[A0:%.*]], <16 x float> [[A1:%.*]], i32 4)
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i16 [[TMP3]] to <16 x i1>
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast i16 [[MASK:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP12:%.*]] = select <16 x i1> [[TMP11]], <16 x i32> [[_MSPROP1]], <16 x i32> [[TMP4]]
+; CHECK-NEXT:    [[TMP13:%.*]] = bitcast <16 x float> [[TMP9]] to <16 x i32>
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast <16 x float> [[SRC:%.*]] to <16 x i32>
+; CHECK-NEXT:    [[TMP15:%.*]] = xor <16 x i32> [[TMP13]], [[TMP14]]
+; CHECK-NEXT:    [[TMP16:%.*]] = or <16 x i32> [[TMP15]], [[_MSPROP1]]
+; CHECK-NEXT:    [[TMP17:%.*]] = or <16 x i32> [[TMP16]], [[TMP4]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP10]], <16 x i32> [[TMP17]], <16 x i32> [[TMP12]]
+; CHECK-NEXT:    [[TMP18:%.*]] = select <16 x i1> [[TMP11]], <16 x float> [[TMP9]], <16 x float> [[SRC]]
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x float> [[TMP18]]
+;
+  %1 = call <16 x float> @llvm.x86.avx512.min.ps.512(<16 x float> %a0, <16 x float> %a1, i32 4)
+  %2 = bitcast i16 %mask to <16 x i1>
+  %3 = select <16 x i1> %2, <16 x float> %1, <16 x float> %src
+  ret <16 x float> %3
+}
+
+define <16 x float> @test_mm512_min_round_ps_sae(<16 x float> %a0, <16 x float> %a1, i16 %mask) #0 {
+; CHECK-LABEL: @test_mm512_min_round_ps_sae(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <16 x i32> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <16 x i32> [[_MSPROP]], zeroinitializer
+; CHECK-NEXT:    [[TMP7:%.*]] = call <16 x float> @llvm.x86.avx512.min.ps.512(<16 x float> [[A0:%.*]], <16 x float> [[A1:%.*]], i32 8)
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP1]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x float> [[TMP7]]
+;
+  %1 = call <16 x float> @llvm.x86.avx512.min.ps.512(<16 x float> %a0, <16 x float> %a1, i32 8)
+  ret <16 x float> %1
+}
+
+define <16 x float> @test_mm512_min_round_ps_current(<16 x float> %a0, <16 x float> %a1, i16 %mask) #0 {
+; CHECK-LABEL: @test_mm512_min_round_ps_current(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <16 x i32> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <16 x i32> [[_MSPROP]], zeroinitializer
+; CHECK-NEXT:    [[TMP7:%.*]] = call <16 x float> @llvm.x86.avx512.min.ps.512(<16 x float> [[A0:%.*]], <16 x float> [[A1:%.*]], i32 4)
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP1]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x float> [[TMP7]]
+;
+  %1 = call <16 x float> @llvm.x86.avx512.min.ps.512(<16 x float> %a0, <16 x float> %a1, i32 4)
+  ret <16 x float> %1
+}
+declare <16 x float> @llvm.x86.avx512.min.ps.512(<16 x float>, <16 x float>, i32)
+
+define <16 x float> @test_mm512_maskz_max_round_ps_sae(<16 x float> %a0, <16 x float> %a1, i16 %mask) #0 {
+; CHECK-LABEL: @test_mm512_maskz_max_round_ps_sae(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <16 x i32> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <16 x i32> [[_MSPROP]], zeroinitializer
+; CHECK-NEXT:    [[TMP8:%.*]] = call <16 x float> @llvm.x86.avx512.max.ps.512(<16 x float> [[A0:%.*]], <16 x float> [[A1:%.*]], i32 8)
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast i16 [[TMP3]] to <16 x i1>
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i16 [[MASK:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP11:%.*]] = select <16 x i1> [[TMP10]], <16 x i32> [[_MSPROP1]], <16 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <16 x float> [[TMP8]] to <16 x i32>
+; CHECK-NEXT:    [[TMP13:%.*]] = xor <16 x i32> [[TMP12]], zeroinitializer
+; CHECK-NEXT:    [[TMP14:%.*]] = or <16 x i32> [[TMP13]], [[_MSPROP1]]
+; CHECK-NEXT:    [[TMP15:%.*]] = or <16 x i32> [[TMP14]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP9]], <16 x i32> [[TMP15]], <16 x i32> [[TMP11]]
+; CHECK-NEXT:    [[TMP16:%.*]] = select <16 x i1> [[TMP10]], <16 x float> [[TMP8]], <16 x float> zeroinitializer
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x float> [[TMP16]]
+;
+  %1 = call <16 x float> @llvm.x86.avx512.max.ps.512(<16 x float> %a0, <16 x float> %a1, i32 8)
+  %2 = bitcast i16 %mask to <16 x i1>
+  %3 = select <16 x i1> %2, <16 x float> %1, <16 x float> zeroinitializer
+  ret <16 x float> %3
+}
+
+define <16 x float> @test_mm512_maskz_max_round_ps_current(<16 x float> %a0, <16 x float> %a1, i16 %mask) #0 {
+; CHECK-LABEL: @test_mm512_maskz_max_round_ps_current(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <16 x i32> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <16 x i32> [[_MSPROP]], zeroinitializer
+; CHECK-NEXT:    [[TMP8:%.*]] = call <16 x float> @llvm.x86.avx512.max.ps.512(<16 x float> [[A0:%.*]], <16 x float> [[A1:%.*]], i32 4)
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast i16 [[TMP3]] to <16 x i1>
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i16 [[MASK:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP11:%.*]] = select <16 x i1> [[TMP10]], <16 x i32> [[_MSPROP1]], <16 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <16 x float> [[TMP8]] to <16 x i32>
+; CHECK-NEXT:    [[TMP13:%.*]] = xor <16 x i32> [[TMP12]], zeroinitializer
+; CHECK-NEXT:    [[TMP14:%.*]] = or <16 x i32> [[TMP13]], [[_MSPROP1]]
+; CHECK-NEXT:    [[TMP15:%.*]] = or <16 x i32> [[TMP14]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP9]], <16 x i32> [[TMP15]], <16 x i32> [[TMP11]]
+; CHECK-NEXT:    [[TMP16:%.*]] = select <16 x i1> [[TMP10]], <16 x float> [[TMP8]], <16 x float> zeroinitializer
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x float> [[TMP16]]
+;
+  %1 = call <16 x float> @llvm.x86.avx512.max.ps.512(<16 x float> %a0, <16 x float> %a1, i32 4)
+  %2 = bitcast i16 %mask to <16 x i1>
+  %3 = select <16 x i1> %2, <16 x float> %1, <16 x float> zeroinitializer
+  ret <16 x float> %3
+}
+
+define <16 x float> @test_mm512_mask_max_round_ps_sae(<16 x float> %a0, <16 x float> %a1, <16 x float> %src, i16 %mask) #0 {
+; CHECK-LABEL: @test_mm512_mask_max_round_ps_sae(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <16 x i32> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <16 x i32> [[_MSPROP]], zeroinitializer
+; CHECK-NEXT:    [[TMP9:%.*]] = call <16 x float> @llvm.x86.avx512.max.ps.512(<16 x float> [[A0:%.*]], <16 x float> [[A1:%.*]], i32 8)
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i16 [[TMP3]] to <16 x i1>
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast i16 [[MASK:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP12:%.*]] = select <16 x i1> [[TMP11]], <16 x i32> [[_MSPROP1]], <16 x i32> [[TMP4]]
+; CHECK-NEXT:    [[TMP13:%.*]] = bitcast <16 x float> [[TMP9]] to <16 x i32>
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast <16 x float> [[SRC:%.*]] to <16 x i32>
+; CHECK-NEXT:    [[TMP15:%.*]] = xor <16 x i32> [[TMP13]], [[TMP14]]
+; CHECK-NEXT:    [[TMP16:%.*]] = or <16 x i32> [[TMP15]], [[_MSPROP1]]
+; CHECK-NEXT:    [[TMP17:%.*]] = or <16 x i32> [[TMP16]], [[TMP4]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP10]], <16 x i32> [[TMP17]], <16 x i32> [[TMP12]]
+; CHECK-NEXT:    [[TMP18:%.*]] = select <16 x i1> [[TMP11]], <16 x float> [[TMP9]], <16 x float> [[SRC]]
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x float> [[TMP18]]
+;
+  %1 = call <16 x float> @llvm.x86.avx512.max.ps.512(<16 x float> %a0, <16 x float> %a1, i32 8)
+  %2 = bitcast i16 %mask to <16 x i1>
+  %3 = select <16 x i1> %2, <16 x float> %1, <16 x float> %src
+  ret <16 x float> %3
+}
+
+define <16 x float> @test_mm512_mask_max_round_ps_current(<16 x float> %a0, <16 x float> %a1, <16 x float> %src, i16 %mask) #0 {
+; CHECK-LABEL: @test_mm512_mask_max_round_ps_current(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <16 x i32> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <16 x i32> [[_MSPROP]], zeroinitializer
+; CHECK-NEXT:    [[TMP9:%.*]] = call <16 x float> @llvm.x86.avx512.max.ps.512(<16 x float> [[A0:%.*]], <16 x float> [[A1:%.*]], i32 4)
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i16 [[TMP3]] to <16 x i1>
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast i16 [[MASK:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP12:%.*]] = select <16 x i1> [[TMP11]], <16 x i32> [[_MSPROP1]], <16 x i32> [[TMP4]]
+; CHECK-NEXT:    [[TMP13:%.*]] = bitcast <16 x float> [[TMP9]] to <16 x i32>
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast <16 x float> [[SRC:%.*]] to <16 x i32>
+; CHECK-NEXT:    [[TMP15:%.*]] = xor <16 x i32> [[TMP13]], [[TMP14]]
+; CHECK-NEXT:    [[TMP16:%.*]] = or <16 x i32> [[TMP15]], [[_MSPROP1]]
+; CHECK-NEXT:    [[TMP17:%.*]] = or <16 x i32> [[TMP16]], [[TMP4]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP10]], <16 x i32> [[TMP17]], <16 x i32> [[TMP12]]
+; CHECK-NEXT:    [[TMP18:%.*]] = select <16 x i1> [[TMP11]], <16 x float> [[TMP9]], <16 x float> [[SRC]]
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x float> [[TMP18]]
+;
+  %1 = call <16 x float> @llvm.x86.avx512.max.ps.512(<16 x float> %a0, <16 x float> %a1, i32 4)
+  %2 = bitcast i16 %mask to <16 x i1>
+  %3 = select <16 x i1> %2, <16 x float> %1, <16 x float> %src
+  ret <16 x float> %3
+}
+
+define <16 x float> @test_mm512_max_round_ps_sae(<16 x float> %a0, <16 x float> %a1, i16 %mask) #0 {
+; CHECK-LABEL: @test_mm512_max_round_ps_sae(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <16 x i32> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <16 x i32> [[_MSPROP]], zeroinitializer
+; CHECK-NEXT:    [[TMP7:%.*]] = call <16 x float> @llvm.x86.avx512.max.ps.512(<16 x float> [[A0:%.*]], <16 x float> [[A1:%.*]], i32 8)
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP1]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x float> [[TMP7]]
+;
+  %1 = call <16 x float> @llvm.x86.avx512.max.ps.512(<16 x float> %a0, <16 x float> %a1, i32 8)
+  ret <16 x float> %1
+}
+
+define <16 x float> @test_mm512_max_round_ps_current(<16 x float> %a0, <16 x float> %a1, i16 %mask) #0 {
+; CHECK-LABEL: @test_mm512_max_round_ps_current(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <16 x i32> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <16 x i32> [[_MSPROP]], zeroinitializer
+; CHECK-NEXT:    [[TMP7:%.*]] = call <16 x float> @llvm.x86.avx512.max.ps.512(<16 x float> [[A0:%.*]], <16 x float> [[A1:%.*]], i32 4)
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP1]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x float> [[TMP7]]
+;
+  %1 = call <16 x float> @llvm.x86.avx512.max.ps.512(<16 x float> %a0, <16 x float> %a1, i32 4)
+  ret <16 x float> %1
+}
+declare <16 x float> @llvm.x86.avx512.max.ps.512(<16 x float>, <16 x float>, i32)
+
+declare <4 x float> @llvm.x86.avx512.mask.add.ss.round(<4 x float>, <4 x float>, <4 x float>, i8, i32) nounwind readnone
+
+define <4 x float> @test_mask_add_ss_rn(<4 x float> %a0, <4 x float> %a1, <4 x float> %a2, i8 %mask) #0 {
+; CHECK-LABEL: @test_mask_add_ss_rn(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 48) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <4 x i32> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <4 x i32> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP6]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <4 x i32> [[TMP3]] to i128
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i128 [[TMP7]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    [[_MSCMP4:%.*]] = icmp ne i8 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR5:%.*]] = or i1 [[_MSOR3]], [[_MSCMP4]]
+; CHECK-NEXT:    br i1 [[_MSOR5]], label [[TMP8:%.*]], label [[TMP9:%.*]], !prof [[PROF1]]
+; CHECK:       8:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       9:
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.x86.avx512.mask.add.ss.round(<4 x float> [[A0:%.*]], <4 x float> [[A1:%.*]], <4 x float> [[A2:%.*]], i8 [[MASK:%.*]], i32 8)
+; CHECK-NEXT:    store <4 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x float> [[RES]]
+;
+  %res = call <4 x float> @llvm.x86.avx512.mask.add.ss.round(<4 x float>%a0, <4 x float> %a1, <4 x float> %a2, i8 %mask, i32 8)
+  ret <4 x float> %res
+}
+
+define <4 x float> @test_mask_add_ss_rd(<4 x float> %a0, <4 x float> %a1, <4 x float> %a2, i8 %mask) #0 {
+; CHECK-LABEL: @test_mask_add_ss_rd(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 48) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <4 x i32> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <4 x i32> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP6]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <4 x i32> [[TMP3]] to i128
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i128 [[TMP7]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    [[_MSCMP4:%.*]] = icmp ne i8 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR5:%.*]] = or i1 [[_MSOR3]], [[_MSCMP4]]
+; CHECK-NEXT:    br i1 [[_MSOR5]], label [[TMP8:%.*]], label [[TMP9:%.*]], !prof [[PROF1]]
+; CHECK:       8:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       9:
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.x86.avx512.mask.add.ss.round(<4 x float> [[A0:%.*]], <4 x float> [[A1:%.*]], <4 x float> [[A2:%.*]], i8 [[MASK:%.*]], i32 9)
+; CHECK-NEXT:    store <4 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x float> [[RES]]
+;
+  %res = call <4 x float> @llvm.x86.avx512.mask.add.ss.round(<4 x float>%a0, <4 x float> %a1, <4 x float> %a2, i8 %mask, i32 9)
+  ret <4 x float> %res
+}
+
+define <4 x float> @test_mask_add_ss_ru(<4 x float> %a0, <4 x float> %a1, <4 x float> %a2, i8 %mask) #0 {
+; CHECK-LABEL: @test_mask_add_ss_ru(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 48) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <4 x i32> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <4 x i32> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP6]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <4 x i32> [[TMP3]] to i128
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i128 [[TMP7]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    [[_MSCMP4:%.*]] = icmp ne i8 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR5:%.*]] = or i1 [[_MSOR3]], [[_MSCMP4]]
+; CHECK-NEXT:    br i1 [[_MSOR5]], label [[TMP8:%.*]], label [[TMP9:%.*]], !prof [[PROF1]]
+; CHECK:       8:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       9:
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.x86.avx512.mask.add.ss.round(<4 x float> [[A0:%.*]], <4 x float> [[A1:%.*]], <4 x float> [[A2:%.*]], i8 [[MASK:%.*]], i32 10)
+; CHECK-NEXT:    store <4 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x float> [[RES]]
+;
+  %res = call <4 x float> @llvm.x86.avx512.mask.add.ss.round(<4 x float>%a0, <4 x float> %a1, <4 x float> %a2, i8 %mask, i32 10)
+  ret <4 x float> %res
+}
+
+define <4 x float> @test_mask_add_ss_rz(<4 x float> %a0, <4 x float> %a1, <4 x float> %a2, i8 %mask) #0 {
+; CHECK-LABEL: @test_mask_add_ss_rz(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 48) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <4 x i32> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <4 x i32> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP6]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <4 x i32> [[TMP3]] to i128
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i128 [[TMP7]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    [[_MSCMP4:%.*]] = icmp ne i8 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR5:%.*]] = or i1 [[_MSOR3]], [[_MSCMP4]]
+; CHECK-NEXT:    br i1 [[_MSOR5]], label [[TMP8:%.*]], label [[TMP9:%.*]], !prof [[PROF1]]
+; CHECK:       8:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       9:
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.x86.avx512.mask.add.ss.round(<4 x float> [[A0:%.*]], <4 x float> [[A1:%.*]], <4 x float> [[A2:%.*]], i8 [[MASK:%.*]], i32 11)
+; CHECK-NEXT:    store <4 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x float> [[RES]]
+;
+  %res = call <4 x float> @llvm.x86.avx512.mask.add.ss.round(<4 x float>%a0, <4 x float> %a1, <4 x float> %a2, i8 %mask, i32 11)
+  ret <4 x float> %res
+}
+
+define <4 x float> @test_mask_add_ss_current(<4 x float> %a0, <4 x float> %a1, <4 x float> %a2, i8 %mask) #0 {
+; CHECK-LABEL: @test_mask_add_ss_current(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 48) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <4 x i32> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <4 x i32> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP6]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <4 x i32> [[TMP3]] to i128
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i128 [[TMP7]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    [[_MSCMP4:%.*]] = icmp ne i8 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR5:%.*]] = or i1 [[_MSOR3]], [[_MSCMP4]]
+; CHECK-NEXT:    br i1 [[_MSOR5]], label [[TMP8:%.*]], label [[TMP9:%.*]], !prof [[PROF1]]
+; CHECK:       8:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       9:
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.x86.avx512.mask.add.ss.round(<4 x float> [[A0:%.*]], <4 x float> [[A1:%.*]], <4 x float> [[A2:%.*]], i8 [[MASK:%.*]], i32 4)
+; CHECK-NEXT:    store <4 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x float> [[RES]]
+;
+  %res = call <4 x float> @llvm.x86.avx512.mask.add.ss.round(<4 x float>%a0, <4 x float> %a1, <4 x float> %a2, i8 %mask, i32 4)
+  ret <4 x float> %res
+}
+
+define <4 x float> @test_maskz_add_ss_rn(<4 x float> %a0, <4 x float> %a1, i8 %mask) #0 {
+; CHECK-LABEL: @test_maskz_add_ss_rn(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <4 x i32> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <4 x i32> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i8 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR3]], label [[TMP6:%.*]], label [[TMP7:%.*]], !prof [[PROF1]]
+; CHECK:       6:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       7:
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.x86.avx512.mask.add.ss.round(<4 x float> [[A0:%.*]], <4 x float> [[A1:%.*]], <4 x float> zeroinitializer, i8 [[MASK:%.*]], i32 8)
+; CHECK-NEXT:    store <4 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x float> [[RES]]
+;
+  %res = call <4 x float> @llvm.x86.avx512.mask.add.ss.round(<4 x float>%a0, <4 x float> %a1, <4 x float> zeroinitializer, i8 %mask, i32 8)
+  ret <4 x float> %res
+}
+
+define <4 x float> @test_add_ss_rn(<4 x float> %a0, <4 x float> %a1) #0 {
+; CHECK-LABEL: @test_add_ss_rn(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <4 x i32> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP3]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <4 x i32> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP5:%.*]], label [[TMP6:%.*]], !prof [[PROF1]]
+; CHECK:       5:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       6:
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.x86.avx512.mask.add.ss.round(<4 x float> [[A0:%.*]], <4 x float> [[A1:%.*]], <4 x float> zeroinitializer, i8 -1, i32 8)
+; CHECK-NEXT:    store <4 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x float> [[RES]]
+;
+  %res = call <4 x float> @llvm.x86.avx512.mask.add.ss.round(<4 x float>%a0, <4 x float> %a1, <4 x float> zeroinitializer, i8 -1, i32 8)
+  ret <4 x float> %res
+}
+
+define <4 x float> @test_mask_add_ss_current_memfold(<4 x float> %a0, ptr %a1, <4 x float> %a2, i8 %mask, <4 x float> %extra_param) #0 {
+; CHECK-LABEL: @test_mask_add_ss_current_memfold(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP5:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 48) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 24) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 40) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP6:%.*]], label [[TMP13:%.*]], !prof [[PROF1]]
+; CHECK:       6:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       7:
+; CHECK-NEXT:    [[A1_VAL:%.*]] = load float, ptr [[A1:%.*]], align 4
+; CHECK-NEXT:    [[TMP7:%.*]] = ptrtoint ptr [[A1]] to i64
+; CHECK-NEXT:    [[TMP8:%.*]] = xor i64 [[TMP7]], 87960930222080
+; CHECK-NEXT:    [[TMP9:%.*]] = inttoptr i64 [[TMP8]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load i32, ptr [[TMP9]], align 4
+; CHECK-NEXT:    [[_MSPROP:%.*]] = insertelement <4 x i32> [[TMP5]], i32 [[_MSLD]], i32 0
+; CHECK-NEXT:    [[A1V0:%.*]] = insertelement <4 x float> [[EXTRA_PARAM:%.*]], float [[A1_VAL]], i32 0
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = insertelement <4 x i32> [[_MSPROP]], i32 0, i32 1
+; CHECK-NEXT:    [[A1V1:%.*]] = insertelement <4 x float> [[A1V0]], float 0.000000e+00, i32 1
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = insertelement <4 x i32> [[_MSPROP1]], i32 0, i32 2
+; CHECK-NEXT:    [[A1V2:%.*]] = insertelement <4 x float> [[A1V1]], float 0.000000e+00, i32 2
+; CHECK-NEXT:    [[_MSPROP3:%.*]] = insertelement <4 x i32> [[_MSPROP2]], i32 0, i32 3
+; CHECK-NEXT:    [[A1V:%.*]] = insertelement <4 x float> [[A1V2]], float 0.000000e+00, i32 3
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast <4 x i32> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP4:%.*]] = icmp ne i128 [[TMP10]], 0
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <4 x i32> [[_MSPROP3]] to i128
+; CHECK-NEXT:    [[_MSCMP5:%.*]] = icmp ne i128 [[TMP11]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP4]], [[_MSCMP5]]
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <4 x i32> [[TMP3]] to i128
+; CHECK-NEXT:    [[_MSCMP6:%.*]] = icmp ne i128 [[TMP12]], 0
+; CHECK-NEXT:    [[_MSOR7:%.*]] = or i1 [[_MSOR]], [[_MSCMP6]]
+; CHECK-NEXT:    [[_MSCMP8:%.*]] = icmp ne i8 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR9:%.*]] = or i1 [[_MSOR7]], [[_MSCMP8]]
+; CHECK-NEXT:    br i1 [[_MSOR9]], label [[TMP14:%.*]], label [[TMP15:%.*]], !prof [[PROF1]]
+; CHECK:       14:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       15:
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.x86.avx512.mask.add.ss.round(<4 x float> [[A0:%.*]], <4 x float> [[A1V]], <4 x float> [[A2:%.*]], i8 [[MASK:%.*]], i32 4)
+; CHECK-NEXT:    store <4 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x float> [[RES]]
+;
+  %a1.val = load float, ptr %a1
+  %a1v0 = insertelement <4 x float> %extra_param, float %a1.val, i32 0
+  %a1v1 = insertelement <4 x float> %a1v0, float 0.000000e+00, i32 1
+  %a1v2 = insertelement <4 x float> %a1v1, float 0.000000e+00, i32 2
+  %a1v  = insertelement <4 x float> %a1v2, float 0.000000e+00, i32 3
+  %res = call <4 x float> @llvm.x86.avx512.mask.add.ss.round(<4 x float>%a0, <4 x float> %a1v, <4 x float> %a2, i8 %mask, i32 4)
+  ret <4 x float> %res
+}
+
+define <4 x float> @test_maskz_add_ss_current_memfold(<4 x float> %a0, ptr %a1, i8 %mask, <4 x float> %extra_param) #0 {
+; CHECK-LABEL: @test_maskz_add_ss_current_memfold(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 24) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP5:%.*]], label [[TMP11:%.*]], !prof [[PROF1]]
+; CHECK:       5:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       6:
+; CHECK-NEXT:    [[A1_VAL:%.*]] = load float, ptr [[A1:%.*]], align 4
+; CHECK-NEXT:    [[TMP6:%.*]] = ptrtoint ptr [[A1]] to i64
+; CHECK-NEXT:    [[TMP7:%.*]] = xor i64 [[TMP6]], 87960930222080
+; CHECK-NEXT:    [[TMP8:%.*]] = inttoptr i64 [[TMP7]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load i32, ptr [[TMP8]], align 4
+; CHECK-NEXT:    [[_MSPROP:%.*]] = insertelement <4 x i32> [[TMP4]], i32 [[_MSLD]], i32 0
+; CHECK-NEXT:    [[A1V0:%.*]] = insertelement <4 x float> [[EXTRA_PARAM:%.*]], float [[A1_VAL]], i32 0
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = insertelement <4 x i32> [[_MSPROP]], i32 0, i32 1
+; CHECK-NEXT:    [[A1V1:%.*]] = insertelement <4 x float> [[A1V0]], float 0.000000e+00, i32 1
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = insertelement <4 x i32> [[_MSPROP1]], i32 0, i32 2
+; CHECK-NEXT:    [[A1V2:%.*]] = insertelement <4 x float> [[A1V1]], float 0.000000e+00, i32 2
+; CHECK-NEXT:    [[_MSPROP3:%.*]] = insertelement <4 x i32> [[_MSPROP2]], i32 0, i32 3
+; CHECK-NEXT:    [[A1V:%.*]] = insertelement <4 x float> [[A1V2]], float 0.000000e+00, i32 3
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <4 x i32> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP4:%.*]] = icmp ne i128 [[TMP9]], 0
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast <4 x i32> [[_MSPROP3]] to i128
+; CHECK-NEXT:    [[_MSCMP5:%.*]] = icmp ne i128 [[TMP10]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP4]], [[_MSCMP5]]
+; CHECK-NEXT:    [[_MSCMP6:%.*]] = icmp ne i8 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSOR7:%.*]] = or i1 [[_MSOR]], [[_MSCMP6]]
+; CHECK-NEXT:    br i1 [[_MSOR7]], label [[TMP12:%.*]], label [[TMP13:%.*]], !prof [[PROF1]]
+; CHECK:       12:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       13:
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.x86.avx512.mask.add.ss.round(<4 x float> [[A0:%.*]], <4 x float> [[A1V]], <4 x float> zeroinitializer, i8 [[MASK:%.*]], i32 4)
+; CHECK-NEXT:    store <4 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x float> [[RES]]
+;
+  %a1.val = load float, ptr %a1
+  %a1v0 = insertelement <4 x float> %extra_param, float %a1.val, i32 0
+  %a1v1 = insertelement <4 x float> %a1v0, float 0.000000e+00, i32 1
+  %a1v2 = insertelement <4 x float> %a1v1, float 0.000000e+00, i32 2
+  %a1v  = insertelement <4 x float> %a1v2, float 0.000000e+00, i32 3
+  %res = call <4 x float> @llvm.x86.avx512.mask.add.ss.round(<4 x float>%a0, <4 x float> %a1v, <4 x float> zeroinitializer, i8 %mask, i32 4)
+  ret <4 x float> %res
+}
+
+declare <2 x double> @llvm.x86.avx512.mask.add.sd.round(<2 x double>, <2 x double>, <2 x double>, i8, i32) nounwind readnone
+
+define <2 x double> @test_mask_add_sd_rn(<2 x double> %a0, <2 x double> %a1, <2 x double> %a2, i8 %mask) #0 {
+; CHECK-LABEL: @test_mask_add_sd_rn(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 48) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <2 x i64> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <2 x i64> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP6]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <2 x i64> [[TMP3]] to i128
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i128 [[TMP7]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    [[_MSCMP4:%.*]] = icmp ne i8 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR5:%.*]] = or i1 [[_MSOR3]], [[_MSCMP4]]
+; CHECK-NEXT:    br i1 [[_MSOR5]], label [[TMP8:%.*]], label [[TMP9:%.*]], !prof [[PROF1]]
+; CHECK:       8:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       9:
+; CHECK-NEXT:    [[RES:%.*]] = call <2 x double> @llvm.x86.avx512.mask.add.sd.round(<2 x double> [[A0:%.*]], <2 x double> [[A1:%.*]], <2 x double> [[A2:%.*]], i8 [[MASK:%.*]], i32 8)
+; CHECK-NEXT:    store <2 x i64> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x double> [[RES]]
+;
+  %res = call <2 x double> @llvm.x86.avx512.mask.add.sd.round(<2 x double>%a0, <2 x double> %a1, <2 x double> %a2, i8 %mask, i32 8)
+  ret <2 x double> %res
+}
+
+define <2 x double> @test_mask_add_sd_rd(<2 x double> %a0, <2 x double> %a1, <2 x double> %a2, i8 %mask) #0 {
+; CHECK-LABEL: @test_mask_add_sd_rd(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 48) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <2 x i64> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <2 x i64> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP6]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <2 x i64> [[TMP3]] to i128
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i128 [[TMP7]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    [[_MSCMP4:%.*]] = icmp ne i8 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR5:%.*]] = or i1 [[_MSOR3]], [[_MSCMP4]]
+; CHECK-NEXT:    br i1 [[_MSOR5]], label [[TMP8:%.*]], label [[TMP9:%.*]], !prof [[PROF1]]
+; CHECK:       8:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       9:
+; CHECK-NEXT:    [[RES:%.*]] = call <2 x double> @llvm.x86.avx512.mask.add.sd.round(<2 x double> [[A0:%.*]], <2 x double> [[A1:%.*]], <2 x double> [[A2:%.*]], i8 [[MASK:%.*]], i32 9)
+; CHECK-NEXT:    store <2 x i64> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x double> [[RES]]
+;
+  %res = call <2 x double> @llvm.x86.avx512.mask.add.sd.round(<2 x double>%a0, <2 x double> %a1, <2 x double> %a2, i8 %mask, i32 9)
+  ret <2 x double> %res
+}
+
+define <2 x double> @test_mask_add_sd_ru(<2 x double> %a0, <2 x double> %a1, <2 x double> %a2, i8 %mask) #0 {
+; CHECK-LABEL: @test_mask_add_sd_ru(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 48) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <2 x i64> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <2 x i64> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP6]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <2 x i64> [[TMP3]] to i128
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i128 [[TMP7]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    [[_MSCMP4:%.*]] = icmp ne i8 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR5:%.*]] = or i1 [[_MSOR3]], [[_MSCMP4]]
+; CHECK-NEXT:    br i1 [[_MSOR5]], label [[TMP8:%.*]], label [[TMP9:%.*]], !prof [[PROF1]]
+; CHECK:       8:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       9:
+; CHECK-NEXT:    [[RES:%.*]] = call <2 x double> @llvm.x86.avx512.mask.add.sd.round(<2 x double> [[A0:%.*]], <2 x double> [[A1:%.*]], <2 x double> [[A2:%.*]], i8 [[MASK:%.*]], i32 10)
+; CHECK-NEXT:    store <2 x i64> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x double> [[RES]]
+;
+  %res = call <2 x double> @llvm.x86.avx512.mask.add.sd.round(<2 x double>%a0, <2 x double> %a1, <2 x double> %a2, i8 %mask, i32 10)
+  ret <2 x double> %res
+}
+
+define <2 x double> @test_mask_add_sd_rz(<2 x double> %a0, <2 x double> %a1, <2 x double> %a2, i8 %mask) #0 {
+; CHECK-LABEL: @test_mask_add_sd_rz(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 48) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <2 x i64> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <2 x i64> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP6]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <2 x i64> [[TMP3]] to i128
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i128 [[TMP7]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    [[_MSCMP4:%.*]] = icmp ne i8 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR5:%.*]] = or i1 [[_MSOR3]], [[_MSCMP4]]
+; CHECK-NEXT:    br i1 [[_MSOR5]], label [[TMP8:%.*]], label [[TMP9:%.*]], !prof [[PROF1]]
+; CHECK:       8:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       9:
+; CHECK-NEXT:    [[RES:%.*]] = call <2 x double> @llvm.x86.avx512.mask.add.sd.round(<2 x double> [[A0:%.*]], <2 x double> [[A1:%.*]], <2 x double> [[A2:%.*]], i8 [[MASK:%.*]], i32 11)
+; CHECK-NEXT:    store <2 x i64> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x double> [[RES]]
+;
+  %res = call <2 x double> @llvm.x86.avx512.mask.add.sd.round(<2 x double>%a0, <2 x double> %a1, <2 x double> %a2, i8 %mask, i32 11)
+  ret <2 x double> %res
+}
+
+define <2 x double> @test_mask_add_sd_current(<2 x double> %a0, <2 x double> %a1, <2 x double> %a2, i8 %mask) #0 {
+; CHECK-LABEL: @test_mask_add_sd_current(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 48) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <2 x i64> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <2 x i64> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP6]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <2 x i64> [[TMP3]] to i128
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i128 [[TMP7]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    [[_MSCMP4:%.*]] = icmp ne i8 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR5:%.*]] = or i1 [[_MSOR3]], [[_MSCMP4]]
+; CHECK-NEXT:    br i1 [[_MSOR5]], label [[TMP8:%.*]], label [[TMP9:%.*]], !prof [[PROF1]]
+; CHECK:       8:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       9:
+; CHECK-NEXT:    [[RES:%.*]] = call <2 x double> @llvm.x86.avx512.mask.add.sd.round(<2 x double> [[A0:%.*]], <2 x double> [[A1:%.*]], <2 x double> [[A2:%.*]], i8 [[MASK:%.*]], i32 4)
+; CHECK-NEXT:    store <2 x i64> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x double> [[RES]]
+;
+  %res = call <2 x double> @llvm.x86.avx512.mask.add.sd.round(<2 x double>%a0, <2 x double> %a1, <2 x double> %a2, i8 %mask, i32 4)
+  ret <2 x double> %res
+}
+
+define <2 x double> @test_maskz_add_sd_rn(<2 x double> %a0, <2 x double> %a1, i8 %mask) #0 {
+; CHECK-LABEL: @test_maskz_add_sd_rn(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <2 x i64> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <2 x i64> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i8 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR3]], label [[TMP6:%.*]], label [[TMP7:%.*]], !prof [[PROF1]]
+; CHECK:       6:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       7:
+; CHECK-NEXT:    [[RES:%.*]] = call <2 x double> @llvm.x86.avx512.mask.add.sd.round(<2 x double> [[A0:%.*]], <2 x double> [[A1:%.*]], <2 x double> zeroinitializer, i8 [[MASK:%.*]], i32 8)
+; CHECK-NEXT:    store <2 x i64> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x double> [[RES]]
+;
+  %res = call <2 x double> @llvm.x86.avx512.mask.add.sd.round(<2 x double>%a0, <2 x double> %a1, <2 x double> zeroinitializer, i8 %mask, i32 8)
+  ret <2 x double> %res
+}
+
+define <2 x double> @test_add_sd_rn(<2 x double> %a0, <2 x double> %a1) #0 {
+; CHECK-LABEL: @test_add_sd_rn(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <2 x i64> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP3]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <2 x i64> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP5:%.*]], label [[TMP6:%.*]], !prof [[PROF1]]
+; CHECK:       5:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       6:
+; CHECK-NEXT:    [[RES:%.*]] = call <2 x double> @llvm.x86.avx512.mask.add.sd.round(<2 x double> [[A0:%.*]], <2 x double> [[A1:%.*]], <2 x double> zeroinitializer, i8 -1, i32 8)
+; CHECK-NEXT:    store <2 x i64> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x double> [[RES]]
+;
+  %res = call <2 x double> @llvm.x86.avx512.mask.add.sd.round(<2 x double>%a0, <2 x double> %a1, <2 x double> zeroinitializer, i8 -1, i32 8)
+  ret <2 x double> %res
+}
+
+define <2 x double> @test_mask_add_sd_current_memfold(<2 x double> %a0, ptr %a1, <2 x double> %a2, i8 %mask, <2 x double> %extra_param) #0 {
+; CHECK-LABEL: @test_mask_add_sd_current_memfold(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP5:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 48) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 24) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 40) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP6:%.*]], label [[TMP13:%.*]], !prof [[PROF1]]
+; CHECK:       6:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       7:
+; CHECK-NEXT:    [[A1_VAL:%.*]] = load double, ptr [[A1:%.*]], align 8
+; CHECK-NEXT:    [[TMP7:%.*]] = ptrtoint ptr [[A1]] to i64
+; CHECK-NEXT:    [[TMP8:%.*]] = xor i64 [[TMP7]], 87960930222080
+; CHECK-NEXT:    [[TMP9:%.*]] = inttoptr i64 [[TMP8]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load i64, ptr [[TMP9]], align 8
+; CHECK-NEXT:    [[_MSPROP:%.*]] = insertelement <2 x i64> [[TMP5]], i64 [[_MSLD]], i32 0
+; CHECK-NEXT:    [[A1V0:%.*]] = insertelement <2 x double> [[EXTRA_PARAM:%.*]], double [[A1_VAL]], i32 0
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = insertelement <2 x i64> [[_MSPROP]], i64 0, i32 1
+; CHECK-NEXT:    [[A1V:%.*]] = insertelement <2 x double> [[A1V0]], double 0.000000e+00, i32 1
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast <2 x i64> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i128 [[TMP10]], 0
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <2 x i64> [[_MSPROP1]] to i128
+; CHECK-NEXT:    [[_MSCMP3:%.*]] = icmp ne i128 [[TMP11]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP2]], [[_MSCMP3]]
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <2 x i64> [[TMP3]] to i128
+; CHECK-NEXT:    [[_MSCMP4:%.*]] = icmp ne i128 [[TMP12]], 0
+; CHECK-NEXT:    [[_MSOR5:%.*]] = or i1 [[_MSOR]], [[_MSCMP4]]
+; CHECK-NEXT:    [[_MSCMP6:%.*]] = icmp ne i8 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR7:%.*]] = or i1 [[_MSOR5]], [[_MSCMP6]]
+; CHECK-NEXT:    br i1 [[_MSOR7]], label [[TMP14:%.*]], label [[TMP15:%.*]], !prof [[PROF1]]
+; CHECK:       14:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       15:
+; CHECK-NEXT:    [[RES:%.*]] = call <2 x double> @llvm.x86.avx512.mask.add.sd.round(<2 x double> [[A0:%.*]], <2 x double> [[A1V]], <2 x double> [[A2:%.*]], i8 [[MASK:%.*]], i32 4)
+; CHECK-NEXT:    store <2 x i64> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x double> [[RES]]
+;
+  %a1.val = load double, ptr %a1
+  %a1v0 = insertelement <2 x double> %extra_param, double %a1.val, i32 0
+  %a1v = insertelement <2 x double> %a1v0, double 0.000000e+00, i32 1
+  %res = call <2 x double> @llvm.x86.avx512.mask.add.sd.round(<2 x double>%a0, <2 x double> %a1v, <2 x double> %a2, i8 %mask, i32 4)
+  ret <2 x double> %res
+}
+
+define <2 x double> @test_maskz_add_sd_current_memfold(<2 x double> %a0, ptr %a1, i8 %mask, <2 x double> %extra_param) #0 {
+; CHECK-LABEL: @test_maskz_add_sd_current_memfold(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 24) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP5:%.*]], label [[TMP11:%.*]], !prof [[PROF1]]
+; CHECK:       5:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       6:
+; CHECK-NEXT:    [[A1_VAL:%.*]] = load double, ptr [[A1:%.*]], align 8
+; CHECK-NEXT:    [[TMP6:%.*]] = ptrtoint ptr [[A1]] to i64
+; CHECK-NEXT:    [[TMP7:%.*]] = xor i64 [[TMP6]], 87960930222080
+; CHECK-NEXT:    [[TMP8:%.*]] = inttoptr i64 [[TMP7]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load i64, ptr [[TMP8]], align 8
+; CHECK-NEXT:    [[_MSPROP:%.*]] = insertelement <2 x i64> [[TMP4]], i64 [[_MSLD]], i32 0
+; CHECK-NEXT:    [[A1V0:%.*]] = insertelement <2 x double> [[EXTRA_PARAM:%.*]], double [[A1_VAL]], i32 0
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = insertelement <2 x i64> [[_MSPROP]], i64 0, i32 1
+; CHECK-NEXT:    [[A1V:%.*]] = insertelement <2 x double> [[A1V0]], double 0.000000e+00, i32 1
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <2 x i64> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i128 [[TMP9]], 0
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast <2 x i64> [[_MSPROP1]] to i128
+; CHECK-NEXT:    [[_MSCMP3:%.*]] = icmp ne i128 [[TMP10]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP2]], [[_MSCMP3]]
+; CHECK-NEXT:    [[_MSCMP4:%.*]] = icmp ne i8 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSOR5:%.*]] = or i1 [[_MSOR]], [[_MSCMP4]]
+; CHECK-NEXT:    br i1 [[_MSOR5]], label [[TMP12:%.*]], label [[TMP13:%.*]], !prof [[PROF1]]
+; CHECK:       12:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       13:
+; CHECK-NEXT:    [[RES:%.*]] = call <2 x double> @llvm.x86.avx512.mask.add.sd.round(<2 x double> [[A0:%.*]], <2 x double> [[A1V]], <2 x double> zeroinitializer, i8 [[MASK:%.*]], i32 4)
+; CHECK-NEXT:    store <2 x i64> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x double> [[RES]]
+;
+  %a1.val = load double, ptr %a1
+  %a1v0 = insertelement <2 x double> %extra_param, double %a1.val, i32 0
+  %a1v = insertelement <2 x double> %a1v0, double 0.000000e+00, i32 1
+  %res = call <2 x double> @llvm.x86.avx512.mask.add.sd.round(<2 x double>%a0, <2 x double> %a1v, <2 x double> zeroinitializer, i8 %mask, i32 4)
+  ret <2 x double> %res
+}
+
+declare <4 x float> @llvm.x86.avx512.mask.max.ss.round(<4 x float>, <4 x float>, <4 x float>, i8, i32) nounwind readnone
+
+define <4 x float> @test_mask_max_ss_sae(<4 x float> %a0, <4 x float> %a1, <4 x float> %a2, i8 %mask) #0 {
+; CHECK-LABEL: @test_mask_max_ss_sae(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 48) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <4 x i32> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <4 x i32> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP6]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <4 x i32> [[TMP3]] to i128
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i128 [[TMP7]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    [[_MSCMP4:%.*]] = icmp ne i8 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR5:%.*]] = or i1 [[_MSOR3]], [[_MSCMP4]]
+; CHECK-NEXT:    br i1 [[_MSOR5]], label [[TMP8:%.*]], label [[TMP9:%.*]], !prof [[PROF1]]
+; CHECK:       8:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       9:
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.x86.avx512.mask.max.ss.round(<4 x float> [[A0:%.*]], <4 x float> [[A1:%.*]], <4 x float> [[A2:%.*]], i8 [[MASK:%.*]], i32 8)
+; CHECK-NEXT:    store <4 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x float> [[RES]]
+;
+  %res = call <4 x float> @llvm.x86.avx512.mask.max.ss.round(<4 x float>%a0, <4 x float> %a1, <4 x float> %a2, i8 %mask, i32 8)
+  ret <4 x float> %res
+}
+
+define <4 x float> @test_maskz_max_ss_sae(<4 x float> %a0, <4 x float> %a1, i8 %mask) #0 {
+; CHECK-LABEL: @test_maskz_max_ss_sae(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <4 x i32> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <4 x i32> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i8 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR3]], label [[TMP6:%.*]], label [[TMP7:%.*]], !prof [[PROF1]]
+; CHECK:       6:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       7:
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.x86.avx512.mask.max.ss.round(<4 x float> [[A0:%.*]], <4 x float> [[A1:%.*]], <4 x float> zeroinitializer, i8 [[MASK:%.*]], i32 8)
+; CHECK-NEXT:    store <4 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x float> [[RES]]
+;
+  %res = call <4 x float> @llvm.x86.avx512.mask.max.ss.round(<4 x float>%a0, <4 x float> %a1, <4 x float> zeroinitializer, i8 %mask, i32 8)
+  ret <4 x float> %res
+}
+
+define <4 x float> @test_max_ss_sae(<4 x float> %a0, <4 x float> %a1) #0 {
+; CHECK-LABEL: @test_max_ss_sae(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <4 x i32> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP3]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <4 x i32> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP5:%.*]], label [[TMP6:%.*]], !prof [[PROF1]]
+; CHECK:       5:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       6:
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.x86.avx512.mask.max.ss.round(<4 x float> [[A0:%.*]], <4 x float> [[A1:%.*]], <4 x float> zeroinitializer, i8 -1, i32 8)
+; CHECK-NEXT:    store <4 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x float> [[RES]]
+;
+  %res = call <4 x float> @llvm.x86.avx512.mask.max.ss.round(<4 x float>%a0, <4 x float> %a1, <4 x float> zeroinitializer, i8 -1, i32 8)
+  ret <4 x float> %res
+}
+
+define <4 x float> @test_mask_max_ss(<4 x float> %a0, <4 x float> %a1, <4 x float> %a2, i8 %mask) #0 {
+; CHECK-LABEL: @test_mask_max_ss(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 48) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <4 x i32> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <4 x i32> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP6]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <4 x i32> [[TMP3]] to i128
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i128 [[TMP7]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    [[_MSCMP4:%.*]] = icmp ne i8 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR5:%.*]] = or i1 [[_MSOR3]], [[_MSCMP4]]
+; CHECK-NEXT:    br i1 [[_MSOR5]], label [[TMP8:%.*]], label [[TMP9:%.*]], !prof [[PROF1]]
+; CHECK:       8:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       9:
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.x86.avx512.mask.max.ss.round(<4 x float> [[A0:%.*]], <4 x float> [[A1:%.*]], <4 x float> [[A2:%.*]], i8 [[MASK:%.*]], i32 4)
+; CHECK-NEXT:    store <4 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x float> [[RES]]
+;
+  %res = call <4 x float> @llvm.x86.avx512.mask.max.ss.round(<4 x float>%a0, <4 x float> %a1, <4 x float> %a2, i8 %mask, i32 4)
+  ret <4 x float> %res
+}
+
+define <4 x float> @test_maskz_max_ss(<4 x float> %a0, <4 x float> %a1, i8 %mask) #0 {
+; CHECK-LABEL: @test_maskz_max_ss(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <4 x i32> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <4 x i32> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i8 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR3]], label [[TMP6:%.*]], label [[TMP7:%.*]], !prof [[PROF1]]
+; CHECK:       6:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       7:
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.x86.avx512.mask.max.ss.round(<4 x float> [[A0:%.*]], <4 x float> [[A1:%.*]], <4 x float> zeroinitializer, i8 [[MASK:%.*]], i32 4)
+; CHECK-NEXT:    store <4 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x float> [[RES]]
+;
+  %res = call <4 x float> @llvm.x86.avx512.mask.max.ss.round(<4 x float>%a0, <4 x float> %a1, <4 x float> zeroinitializer, i8 %mask, i32 4)
+  ret <4 x float> %res
+}
+
+define <4 x float> @test_max_ss(<4 x float> %a0, <4 x float> %a1) #0 {
+; CHECK-LABEL: @test_max_ss(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <4 x i32> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP3]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <4 x i32> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP5:%.*]], label [[TMP6:%.*]], !prof [[PROF1]]
+; CHECK:       5:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       6:
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.x86.avx512.mask.max.ss.round(<4 x float> [[A0:%.*]], <4 x float> [[A1:%.*]], <4 x float> zeroinitializer, i8 -1, i32 4)
+; CHECK-NEXT:    store <4 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x float> [[RES]]
+;
+  %res = call <4 x float> @llvm.x86.avx512.mask.max.ss.round(<4 x float>%a0, <4 x float> %a1, <4 x float> zeroinitializer, i8 -1, i32 4)
+  ret <4 x float> %res
+}
+
+define <4 x float> @test_mask_max_ss_memfold(<4 x float> %a0, ptr %a1, <4 x float> %a2, i8 %mask, <4 x float> %extra_param) #0 {
+; CHECK-LABEL: @test_mask_max_ss_memfold(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP5:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 48) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 24) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 40) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP6:%.*]], label [[TMP13:%.*]], !prof [[PROF1]]
+; CHECK:       6:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       7:
+; CHECK-NEXT:    [[A1_VAL:%.*]] = load float, ptr [[A1:%.*]], align 4
+; CHECK-NEXT:    [[TMP7:%.*]] = ptrtoint ptr [[A1]] to i64
+; CHECK-NEXT:    [[TMP8:%.*]] = xor i64 [[TMP7]], 87960930222080
+; CHECK-NEXT:    [[TMP9:%.*]] = inttoptr i64 [[TMP8]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load i32, ptr [[TMP9]], align 4
+; CHECK-NEXT:    [[_MSPROP:%.*]] = insertelement <4 x i32> [[TMP5]], i32 [[_MSLD]], i32 0
+; CHECK-NEXT:    [[A1V0:%.*]] = insertelement <4 x float> [[EXTRA_PARAM:%.*]], float [[A1_VAL]], i32 0
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = insertelement <4 x i32> [[_MSPROP]], i32 0, i32 1
+; CHECK-NEXT:    [[A1V1:%.*]] = insertelement <4 x float> [[A1V0]], float 0.000000e+00, i32 1
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = insertelement <4 x i32> [[_MSPROP1]], i32 0, i32 2
+; CHECK-NEXT:    [[A1V2:%.*]] = insertelement <4 x float> [[A1V1]], float 0.000000e+00, i32 2
+; CHECK-NEXT:    [[_MSPROP3:%.*]] = insertelement <4 x i32> [[_MSPROP2]], i32 0, i32 3
+; CHECK-NEXT:    [[A1V:%.*]] = insertelement <4 x float> [[A1V2]], float 0.000000e+00, i32 3
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast <4 x i32> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP4:%.*]] = icmp ne i128 [[TMP10]], 0
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <4 x i32> [[_MSPROP3]] to i128
+; CHECK-NEXT:    [[_MSCMP5:%.*]] = icmp ne i128 [[TMP11]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP4]], [[_MSCMP5]]
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <4 x i32> [[TMP3]] to i128
+; CHECK-NEXT:    [[_MSCMP6:%.*]] = icmp ne i128 [[TMP12]], 0
+; CHECK-NEXT:    [[_MSOR7:%.*]] = or i1 [[_MSOR]], [[_MSCMP6]]
+; CHECK-NEXT:    [[_MSCMP8:%.*]] = icmp ne i8 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR9:%.*]] = or i1 [[_MSOR7]], [[_MSCMP8]]
+; CHECK-NEXT:    br i1 [[_MSOR9]], label [[TMP14:%.*]], label [[TMP15:%.*]], !prof [[PROF1]]
+; CHECK:       14:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       15:
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.x86.avx512.mask.max.ss.round(<4 x float> [[A0:%.*]], <4 x float> [[A1V]], <4 x float> [[A2:%.*]], i8 [[MASK:%.*]], i32 4)
+; CHECK-NEXT:    store <4 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x float> [[RES]]
+;
+  %a1.val = load float, ptr %a1
+  %a1v0 = insertelement <4 x float> %extra_param, float %a1.val, i32 0
+  %a1v1 = insertelement <4 x float> %a1v0, float 0.000000e+00, i32 1
+  %a1v2 = insertelement <4 x float> %a1v1, float 0.000000e+00, i32 2
+  %a1v  = insertelement <4 x float> %a1v2, float 0.000000e+00, i32 3
+  %res = call <4 x float> @llvm.x86.avx512.mask.max.ss.round(<4 x float>%a0, <4 x float> %a1v, <4 x float> %a2, i8 %mask, i32 4)
+  ret <4 x float> %res
+}
+
+define <4 x float> @test_maskz_max_ss_memfold(<4 x float> %a0, ptr %a1, i8 %mask, <4 x float> %extra_param) #0 {
+; CHECK-LABEL: @test_maskz_max_ss_memfold(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 24) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP5:%.*]], label [[TMP11:%.*]], !prof [[PROF1]]
+; CHECK:       5:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       6:
+; CHECK-NEXT:    [[A1_VAL:%.*]] = load float, ptr [[A1:%.*]], align 4
+; CHECK-NEXT:    [[TMP6:%.*]] = ptrtoint ptr [[A1]] to i64
+; CHECK-NEXT:    [[TMP7:%.*]] = xor i64 [[TMP6]], 87960930222080
+; CHECK-NEXT:    [[TMP8:%.*]] = inttoptr i64 [[TMP7]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load i32, ptr [[TMP8]], align 4
+; CHECK-NEXT:    [[_MSPROP:%.*]] = insertelement <4 x i32> [[TMP4]], i32 [[_MSLD]], i32 0
+; CHECK-NEXT:    [[A1V0:%.*]] = insertelement <4 x float> [[EXTRA_PARAM:%.*]], float [[A1_VAL]], i32 0
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = insertelement <4 x i32> [[_MSPROP]], i32 0, i32 1
+; CHECK-NEXT:    [[A1V1:%.*]] = insertelement <4 x float> [[A1V0]], float 0.000000e+00, i32 1
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = insertelement <4 x i32> [[_MSPROP1]], i32 0, i32 2
+; CHECK-NEXT:    [[A1V2:%.*]] = insertelement <4 x float> [[A1V1]], float 0.000000e+00, i32 2
+; CHECK-NEXT:    [[_MSPROP3:%.*]] = insertelement <4 x i32> [[_MSPROP2]], i32 0, i32 3
+; CHECK-NEXT:    [[A1V:%.*]] = insertelement <4 x float> [[A1V2]], float 0.000000e+00, i32 3
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <4 x i32> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP4:%.*]] = icmp ne i128 [[TMP9]], 0
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast <4 x i32> [[_MSPROP3]] to i128
+; CHECK-NEXT:    [[_MSCMP5:%.*]] = icmp ne i128 [[TMP10]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP4]], [[_MSCMP5]]
+; CHECK-NEXT:    [[_MSCMP6:%.*]] = icmp ne i8 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSOR7:%.*]] = or i1 [[_MSOR]], [[_MSCMP6]]
+; CHECK-NEXT:    br i1 [[_MSOR7]], label [[TMP12:%.*]], label [[TMP13:%.*]], !prof [[PROF1]]
+; CHECK:       12:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       13:
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.x86.avx512.mask.max.ss.round(<4 x float> [[A0:%.*]], <4 x float> [[A1V]], <4 x float> zeroinitializer, i8 [[MASK:%.*]], i32 4)
+; CHECK-NEXT:    store <4 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x float> [[RES]]
+;
+  %a1.val = load float, ptr %a1
+  %a1v0 = insertelement <4 x float> %extra_param, float %a1.val, i32 0
+  %a1v1 = insertelement <4 x float> %a1v0, float 0.000000e+00, i32 1
+  %a1v2 = insertelement <4 x float> %a1v1, float 0.000000e+00, i32 2
+  %a1v  = insertelement <4 x float> %a1v2, float 0.000000e+00, i32 3
+  %res = call <4 x float> @llvm.x86.avx512.mask.max.ss.round(<4 x float>%a0, <4 x float> %a1v, <4 x float> zeroinitializer, i8 %mask, i32 4)
+  ret <4 x float> %res
+}
+declare <2 x double> @llvm.x86.avx512.mask.max.sd.round(<2 x double>, <2 x double>, <2 x double>, i8, i32) nounwind readnone
+
+define <2 x double> @test_mask_max_sd_sae(<2 x double> %a0, <2 x double> %a1, <2 x double> %a2, i8 %mask) #0 {
+; CHECK-LABEL: @test_mask_max_sd_sae(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 48) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <2 x i64> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <2 x i64> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP6]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <2 x i64> [[TMP3]] to i128
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i128 [[TMP7]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    [[_MSCMP4:%.*]] = icmp ne i8 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR5:%.*]] = or i1 [[_MSOR3]], [[_MSCMP4]]
+; CHECK-NEXT:    br i1 [[_MSOR5]], label [[TMP8:%.*]], label [[TMP9:%.*]], !prof [[PROF1]]
+; CHECK:       8:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       9:
+; CHECK-NEXT:    [[RES:%.*]] = call <2 x double> @llvm.x86.avx512.mask.max.sd.round(<2 x double> [[A0:%.*]], <2 x double> [[A1:%.*]], <2 x double> [[A2:%.*]], i8 [[MASK:%.*]], i32 8)
+; CHECK-NEXT:    store <2 x i64> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x double> [[RES]]
+;
+  %res = call <2 x double> @llvm.x86.avx512.mask.max.sd.round(<2 x double>%a0, <2 x double> %a1, <2 x double> %a2, i8 %mask, i32 8)
+  ret <2 x double> %res
+}
+
+define <2 x double> @test_maskz_max_sd_sae(<2 x double> %a0, <2 x double> %a1, i8 %mask) #0 {
+; CHECK-LABEL: @test_maskz_max_sd_sae(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <2 x i64> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <2 x i64> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i8 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR3]], label [[TMP6:%.*]], label [[TMP7:%.*]], !prof [[PROF1]]
+; CHECK:       6:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       7:
+; CHECK-NEXT:    [[RES:%.*]] = call <2 x double> @llvm.x86.avx512.mask.max.sd.round(<2 x double> [[A0:%.*]], <2 x double> [[A1:%.*]], <2 x double> zeroinitializer, i8 [[MASK:%.*]], i32 8)
+; CHECK-NEXT:    store <2 x i64> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x double> [[RES]]
+;
+  %res = call <2 x double> @llvm.x86.avx512.mask.max.sd.round(<2 x double>%a0, <2 x double> %a1, <2 x double> zeroinitializer, i8 %mask, i32 8)
+  ret <2 x double> %res
+}
+
+define <2 x double> @test_max_sd_sae(<2 x double> %a0, <2 x double> %a1) #0 {
+; CHECK-LABEL: @test_max_sd_sae(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <2 x i64> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP3]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <2 x i64> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP5:%.*]], label [[TMP6:%.*]], !prof [[PROF1]]
+; CHECK:       5:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       6:
+; CHECK-NEXT:    [[RES:%.*]] = call <2 x double> @llvm.x86.avx512.mask.max.sd.round(<2 x double> [[A0:%.*]], <2 x double> [[A1:%.*]], <2 x double> zeroinitializer, i8 -1, i32 8)
+; CHECK-NEXT:    store <2 x i64> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x double> [[RES]]
+;
+  %res = call <2 x double> @llvm.x86.avx512.mask.max.sd.round(<2 x double>%a0, <2 x double> %a1, <2 x double> zeroinitializer, i8 -1, i32 8)
+  ret <2 x double> %res
+}
+
+define <2 x double> @test_mask_max_sd(<2 x double> %a0, <2 x double> %a1, <2 x double> %a2, i8 %mask) #0 {
+; CHECK-LABEL: @test_mask_max_sd(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 48) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <2 x i64> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <2 x i64> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP6]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <2 x i64> [[TMP3]] to i128
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i128 [[TMP7]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    [[_MSCMP4:%.*]] = icmp ne i8 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR5:%.*]] = or i1 [[_MSOR3]], [[_MSCMP4]]
+; CHECK-NEXT:    br i1 [[_MSOR5]], label [[TMP8:%.*]], label [[TMP9:%.*]], !prof [[PROF1]]
+; CHECK:       8:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       9:
+; CHECK-NEXT:    [[RES:%.*]] = call <2 x double> @llvm.x86.avx512.mask.max.sd.round(<2 x double> [[A0:%.*]], <2 x double> [[A1:%.*]], <2 x double> [[A2:%.*]], i8 [[MASK:%.*]], i32 4)
+; CHECK-NEXT:    store <2 x i64> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x double> [[RES]]
+;
+  %res = call <2 x double> @llvm.x86.avx512.mask.max.sd.round(<2 x double>%a0, <2 x double> %a1, <2 x double> %a2, i8 %mask, i32 4)
+  ret <2 x double> %res
+}
+
+define <2 x double> @test_maskz_max_sd(<2 x double> %a0, <2 x double> %a1, i8 %mask) #0 {
+; CHECK-LABEL: @test_maskz_max_sd(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <2 x i64> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <2 x i64> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i8 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR3]], label [[TMP6:%.*]], label [[TMP7:%.*]], !prof [[PROF1]]
+; CHECK:       6:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       7:
+; CHECK-NEXT:    [[RES:%.*]] = call <2 x double> @llvm.x86.avx512.mask.max.sd.round(<2 x double> [[A0:%.*]], <2 x double> [[A1:%.*]], <2 x double> zeroinitializer, i8 [[MASK:%.*]], i32 4)
+; CHECK-NEXT:    store <2 x i64> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x double> [[RES]]
+;
+  %res = call <2 x double> @llvm.x86.avx512.mask.max.sd.round(<2 x double>%a0, <2 x double> %a1, <2 x double> zeroinitializer, i8 %mask, i32 4)
+  ret <2 x double> %res
+}
+
+define <2 x double> @test_max_sd(<2 x double> %a0, <2 x double> %a1) #0 {
+; CHECK-LABEL: @test_max_sd(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <2 x i64> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP3]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <2 x i64> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP5:%.*]], label [[TMP6:%.*]], !prof [[PROF1]]
+; CHECK:       5:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       6:
+; CHECK-NEXT:    [[RES:%.*]] = call <2 x double> @llvm.x86.avx512.mask.max.sd.round(<2 x double> [[A0:%.*]], <2 x double> [[A1:%.*]], <2 x double> zeroinitializer, i8 -1, i32 4)
+; CHECK-NEXT:    store <2 x i64> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x double> [[RES]]
+;
+  %res = call <2 x double> @llvm.x86.avx512.mask.max.sd.round(<2 x double>%a0, <2 x double> %a1, <2 x double> zeroinitializer, i8 -1, i32 4)
+  ret <2 x double> %res
+}
+
+define <2 x double> @test_mask_max_sd_memfold(<2 x double> %a0, ptr %a1, <2 x double> %a2, i8 %mask, <2 x double> %extra_param) #0 {
+; CHECK-LABEL: @test_mask_max_sd_memfold(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP5:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 48) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 24) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 40) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP6:%.*]], label [[TMP13:%.*]], !prof [[PROF1]]
+; CHECK:       6:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       7:
+; CHECK-NEXT:    [[A1_VAL:%.*]] = load double, ptr [[A1:%.*]], align 8
+; CHECK-NEXT:    [[TMP7:%.*]] = ptrtoint ptr [[A1]] to i64
+; CHECK-NEXT:    [[TMP8:%.*]] = xor i64 [[TMP7]], 87960930222080
+; CHECK-NEXT:    [[TMP9:%.*]] = inttoptr i64 [[TMP8]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load i64, ptr [[TMP9]], align 8
+; CHECK-NEXT:    [[_MSPROP:%.*]] = insertelement <2 x i64> [[TMP5]], i64 [[_MSLD]], i32 0
+; CHECK-NEXT:    [[A1V0:%.*]] = insertelement <2 x double> [[EXTRA_PARAM:%.*]], double [[A1_VAL]], i32 0
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = insertelement <2 x i64> [[_MSPROP]], i64 0, i32 1
+; CHECK-NEXT:    [[A1V:%.*]] = insertelement <2 x double> [[A1V0]], double 0.000000e+00, i32 1
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast <2 x i64> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i128 [[TMP10]], 0
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <2 x i64> [[_MSPROP1]] to i128
+; CHECK-NEXT:    [[_MSCMP3:%.*]] = icmp ne i128 [[TMP11]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP2]], [[_MSCMP3]]
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <2 x i64> [[TMP3]] to i128
+; CHECK-NEXT:    [[_MSCMP4:%.*]] = icmp ne i128 [[TMP12]], 0
+; CHECK-NEXT:    [[_MSOR5:%.*]] = or i1 [[_MSOR]], [[_MSCMP4]]
+; CHECK-NEXT:    [[_MSCMP6:%.*]] = icmp ne i8 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR7:%.*]] = or i1 [[_MSOR5]], [[_MSCMP6]]
+; CHECK-NEXT:    br i1 [[_MSOR7]], label [[TMP14:%.*]], label [[TMP15:%.*]], !prof [[PROF1]]
+; CHECK:       14:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       15:
+; CHECK-NEXT:    [[RES:%.*]] = call <2 x double> @llvm.x86.avx512.mask.max.sd.round(<2 x double> [[A0:%.*]], <2 x double> [[A1V]], <2 x double> [[A2:%.*]], i8 [[MASK:%.*]], i32 4)
+; CHECK-NEXT:    store <2 x i64> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x double> [[RES]]
+;
+  %a1.val = load double, ptr %a1
+  %a1v0 = insertelement <2 x double> %extra_param, double %a1.val, i32 0
+  %a1v = insertelement <2 x double> %a1v0, double 0.000000e+00, i32 1
+  %res = call <2 x double> @llvm.x86.avx512.mask.max.sd.round(<2 x double>%a0, <2 x double> %a1v, <2 x double> %a2, i8 %mask, i32 4)
+  ret <2 x double> %res
+}
+
+define <2 x double> @test_maskz_max_sd_memfold(<2 x double> %a0, ptr %a1, i8 %mask, <2 x double> %extra_param) #0 {
+; CHECK-LABEL: @test_maskz_max_sd_memfold(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 24) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP5:%.*]], label [[TMP11:%.*]], !prof [[PROF1]]
+; CHECK:       5:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       6:
+; CHECK-NEXT:    [[A1_VAL:%.*]] = load double, ptr [[A1:%.*]], align 8
+; CHECK-NEXT:    [[TMP6:%.*]] = ptrtoint ptr [[A1]] to i64
+; CHECK-NEXT:    [[TMP7:%.*]] = xor i64 [[TMP6]], 87960930222080
+; CHECK-NEXT:    [[TMP8:%.*]] = inttoptr i64 [[TMP7]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load i64, ptr [[TMP8]], align 8
+; CHECK-NEXT:    [[_MSPROP:%.*]] = insertelement <2 x i64> [[TMP4]], i64 [[_MSLD]], i32 0
+; CHECK-NEXT:    [[A1V0:%.*]] = insertelement <2 x double> [[EXTRA_PARAM:%.*]], double [[A1_VAL]], i32 0
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = insertelement <2 x i64> [[_MSPROP]], i64 0, i32 1
+; CHECK-NEXT:    [[A1V:%.*]] = insertelement <2 x double> [[A1V0]], double 0.000000e+00, i32 1
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <2 x i64> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i128 [[TMP9]], 0
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast <2 x i64> [[_MSPROP1]] to i128
+; CHECK-NEXT:    [[_MSCMP3:%.*]] = icmp ne i128 [[TMP10]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP2]], [[_MSCMP3]]
+; CHECK-NEXT:    [[_MSCMP4:%.*]] = icmp ne i8 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSOR5:%.*]] = or i1 [[_MSOR]], [[_MSCMP4]]
+; CHECK-NEXT:    br i1 [[_MSOR5]], label [[TMP12:%.*]], label [[TMP13:%.*]], !prof [[PROF1]]
+; CHECK:       12:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       13:
+; CHECK-NEXT:    [[RES:%.*]] = call <2 x double> @llvm.x86.avx512.mask.max.sd.round(<2 x double> [[A0:%.*]], <2 x double> [[A1V]], <2 x double> zeroinitializer, i8 [[MASK:%.*]], i32 4)
+; CHECK-NEXT:    store <2 x i64> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x double> [[RES]]
+;
+  %a1.val = load double, ptr %a1
+  %a1v0 = insertelement <2 x double> %extra_param, double %a1.val, i32 0
+  %a1v = insertelement <2 x double> %a1v0, double 0.000000e+00, i32 1
+  %res = call <2 x double> @llvm.x86.avx512.mask.max.sd.round(<2 x double>%a0, <2 x double> %a1v, <2 x double> zeroinitializer, i8 %mask, i32 4)
+  ret <2 x double> %res
+}
+
+define <4 x float> @test_x86_avx512_cvtsi2ss32(<4 x float> %a, i32 %b) #0 {
+; CHECK-LABEL: @test_x86_avx512_cvtsi2ss32(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i32, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <4 x i32> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i32 [[TMP2]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP4:%.*]], label [[TMP5:%.*]], !prof [[PROF1]]
+; CHECK:       4:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       5:
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.x86.avx512.cvtsi2ss32(<4 x float> [[A:%.*]], i32 [[B:%.*]], i32 11)
+; CHECK-NEXT:    store <4 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x float> [[RES]]
+;
+  %res = call <4 x float> @llvm.x86.avx512.cvtsi2ss32(<4 x float> %a, i32 %b, i32 11) ; <<<4 x float>> [#uses=1]
+  ret <4 x float> %res
+}
+declare <4 x float> @llvm.x86.avx512.cvtsi2ss32(<4 x float>, i32, i32) nounwind readnone
+
+define <4 x float> @test_x86_avx512__mm_cvt_roundu32_ss (<4 x float> %a, i32 %b) #0 {
+; CHECK-LABEL: @test_x86_avx512__mm_cvt_roundu32_ss(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i32, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = insertelement <4 x i32> [[TMP2]], i32 0, i32 0
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i32 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP4:%.*]], label [[TMP5:%.*]], !prof [[PROF1]]
+; CHECK:       4:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       5:
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.x86.avx512.cvtusi2ss(<4 x float> [[A:%.*]], i32 [[B:%.*]], i32 9)
+; CHECK-NEXT:    store <4 x i32> [[TMP3]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x float> [[RES]]
+;
+  %res = call <4 x float> @llvm.x86.avx512.cvtusi2ss(<4 x float> %a, i32 %b, i32 9) ; <<<4 x float>> [#uses=1]
+  ret <4 x float> %res
+}
+
+define <4 x float> @test_x86_avx512__mm_cvt_roundu32_ss_mem(<4 x float> %a, ptr %ptr) #0 {
+; CHECK-LABEL: @test_x86_avx512__mm_cvt_roundu32_ss_mem(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP3:%.*]], label [[TMP4:%.*]], !prof [[PROF1]]
+; CHECK:       3:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       4:
+; CHECK-NEXT:    [[B:%.*]] = load i32, ptr [[PTR:%.*]], align 4
+; CHECK-NEXT:    [[TMP5:%.*]] = ptrtoint ptr [[PTR]] to i64
+; CHECK-NEXT:    [[TMP6:%.*]] = xor i64 [[TMP5]], 87960930222080
+; CHECK-NEXT:    [[TMP7:%.*]] = inttoptr i64 [[TMP6]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load i32, ptr [[TMP7]], align 4
+; CHECK-NEXT:    [[TMP8:%.*]] = insertelement <4 x i32> [[TMP2]], i32 0, i32 0
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i32 [[_MSLD]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP1]], label [[TMP9:%.*]], label [[TMP10:%.*]], !prof [[PROF1]]
+; CHECK:       9:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       10:
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.x86.avx512.cvtusi2ss(<4 x float> [[A:%.*]], i32 [[B]], i32 9)
+; CHECK-NEXT:    store <4 x i32> [[TMP8]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x float> [[RES]]
+;
+  %b = load i32, ptr %ptr
+  %res = call <4 x float> @llvm.x86.avx512.cvtusi2ss(<4 x float> %a, i32 %b, i32 9) ; <<<4 x float>> [#uses=1]
+  ret <4 x float> %res
+}
+
+define <4 x float> @test_x86_avx512__mm_cvtu32_ss(<4 x float> %a, i32 %b) #0 {
+; CHECK-LABEL: @test_x86_avx512__mm_cvtu32_ss(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i32, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = insertelement <4 x i32> [[TMP2]], i32 0, i32 0
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i32 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP4:%.*]], label [[TMP5:%.*]], !prof [[PROF1]]
+; CHECK:       4:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       5:
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.x86.avx512.cvtusi2ss(<4 x float> [[A:%.*]], i32 [[B:%.*]], i32 4)
+; CHECK-NEXT:    store <4 x i32> [[TMP3]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x float> [[RES]]
+;
+  %res = call <4 x float> @llvm.x86.avx512.cvtusi2ss(<4 x float> %a, i32 %b, i32 4) ; <<<4 x float>> [#uses=1]
+  ret <4 x float> %res
+}
+
+define <4 x float> @test_x86_avx512__mm_cvtu32_ss_mem(<4 x float> %a, ptr %ptr) #0 {
+; CHECK-LABEL: @test_x86_avx512__mm_cvtu32_ss_mem(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP3:%.*]], label [[TMP4:%.*]], !prof [[PROF1]]
+; CHECK:       3:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       4:
+; CHECK-NEXT:    [[B:%.*]] = load i32, ptr [[PTR:%.*]], align 4
+; CHECK-NEXT:    [[TMP5:%.*]] = ptrtoint ptr [[PTR]] to i64
+; CHECK-NEXT:    [[TMP6:%.*]] = xor i64 [[TMP5]], 87960930222080
+; CHECK-NEXT:    [[TMP7:%.*]] = inttoptr i64 [[TMP6]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load i32, ptr [[TMP7]], align 4
+; CHECK-NEXT:    [[TMP8:%.*]] = insertelement <4 x i32> [[TMP2]], i32 0, i32 0
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i32 [[_MSLD]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP1]], label [[TMP9:%.*]], label [[TMP10:%.*]], !prof [[PROF1]]
+; CHECK:       9:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       10:
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.x86.avx512.cvtusi2ss(<4 x float> [[A:%.*]], i32 [[B]], i32 4)
+; CHECK-NEXT:    store <4 x i32> [[TMP8]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x float> [[RES]]
+;
+  %b = load i32, ptr %ptr
+  %res = call <4 x float> @llvm.x86.avx512.cvtusi2ss(<4 x float> %a, i32 %b, i32 4) ; <<<4 x float>> [#uses=1]
+  ret <4 x float> %res
+}
+declare <4 x float> @llvm.x86.avx512.cvtusi2ss(<4 x float>, i32, i32) nounwind readnone
+
+declare <16 x i32> @llvm.x86.avx512.vpermi2var.d.512(<16 x i32>, <16 x i32>, <16 x i32>)
+
+define <16 x i32>@test_int_x86_avx512_vpermi2var_d_512(<16 x i32> %x0, <16 x i32> %x1, ptr %x2p) #0 {
+; CHECK-LABEL: @test_int_x86_avx512_vpermi2var_d_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP4:%.*]], label [[TMP5:%.*]], !prof [[PROF1]]
+; CHECK:       4:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       5:
+; CHECK-NEXT:    [[X2:%.*]] = load <16 x i32>, ptr [[X2P:%.*]], align 64
+; CHECK-NEXT:    [[TMP6:%.*]] = ptrtoint ptr [[X2P]] to i64
+; CHECK-NEXT:    [[TMP7:%.*]] = xor i64 [[TMP6]], 87960930222080
+; CHECK-NEXT:    [[TMP8:%.*]] = inttoptr i64 [[TMP7]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <16 x i32>, ptr [[TMP8]], align 64
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <16 x i32> [[TMP2]], [[TMP3]]
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <16 x i32> [[_MSPROP]], [[_MSLD]]
+; CHECK-NEXT:    [[TMP9:%.*]] = call <16 x i32> @llvm.x86.avx512.vpermi2var.d.512(<16 x i32> [[X0:%.*]], <16 x i32> [[X1:%.*]], <16 x i32> [[X2]])
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP1]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i32> [[TMP9]]
+;
+  %x2 = load <16 x i32>, ptr %x2p
+  %1 = call <16 x i32> @llvm.x86.avx512.vpermi2var.d.512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x2)
+  ret <16 x i32> %1
+}
+
+define <16 x i32>@test_int_x86_avx512_mask_vpermi2var_d_512(<16 x i32> %x0, <16 x i32> %x1, ptr %x2p, i16 %x3) #0 {
+; CHECK-LABEL: @test_int_x86_avx512_mask_vpermi2var_d_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 136) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP5:%.*]], label [[TMP6:%.*]], !prof [[PROF1]]
+; CHECK:       5:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       6:
+; CHECK-NEXT:    [[X2:%.*]] = load <16 x i32>, ptr [[X2P:%.*]], align 64
+; CHECK-NEXT:    [[TMP7:%.*]] = ptrtoint ptr [[X2P]] to i64
+; CHECK-NEXT:    [[TMP8:%.*]] = xor i64 [[TMP7]], 87960930222080
+; CHECK-NEXT:    [[TMP9:%.*]] = inttoptr i64 [[TMP8]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <16 x i32>, ptr [[TMP9]], align 64
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <16 x i32> [[TMP2]], [[TMP3]]
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <16 x i32> [[_MSPROP]], [[_MSLD]]
+; CHECK-NEXT:    [[TMP10:%.*]] = call <16 x i32> @llvm.x86.avx512.vpermi2var.d.512(<16 x i32> [[X0:%.*]], <16 x i32> [[X1:%.*]], <16 x i32> [[X2]])
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast i16 [[TMP4]] to <16 x i1>
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast i16 [[X3:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP13:%.*]] = select <16 x i1> [[TMP12]], <16 x i32> [[_MSPROP1]], <16 x i32> [[TMP3]]
+; CHECK-NEXT:    [[TMP14:%.*]] = xor <16 x i32> [[TMP10]], [[X1]]
+; CHECK-NEXT:    [[TMP15:%.*]] = or <16 x i32> [[TMP14]], [[_MSPROP1]]
+; CHECK-NEXT:    [[TMP16:%.*]] = or <16 x i32> [[TMP15]], [[TMP3]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP11]], <16 x i32> [[TMP16]], <16 x i32> [[TMP13]]
+; CHECK-NEXT:    [[TMP17:%.*]] = select <16 x i1> [[TMP12]], <16 x i32> [[TMP10]], <16 x i32> [[X1]]
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i32> [[TMP17]]
+;
+  %x2 = load <16 x i32>, ptr %x2p
+  %1 = call <16 x i32> @llvm.x86.avx512.vpermi2var.d.512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x2)
+  %2 = bitcast i16 %x3 to <16 x i1>
+  %3 = select <16 x i1> %2, <16 x i32> %1, <16 x i32> %x1
+  ret <16 x i32> %3
+}
+
+declare <8 x double> @llvm.x86.avx512.vpermi2var.pd.512(<8 x double>, <8 x i64>, <8 x double>)
+
+define <8 x double>@test_int_x86_avx512_vpermi2var_pd_512(<8 x double> %x0, <8 x i64> %x1, <8 x double> %x2) #0 {
+; CHECK-LABEL: @test_int_x86_avx512_vpermi2var_pd_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <8 x i64> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <8 x i64> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <8 x i64> [[TMP3]] to i512
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i512 [[TMP6]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR3]], label [[TMP7:%.*]], label [[TMP8:%.*]], !prof [[PROF1]]
+; CHECK:       7:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       8:
+; CHECK-NEXT:    [[TMP9:%.*]] = call <8 x double> @llvm.x86.avx512.vpermi2var.pd.512(<8 x double> [[X0:%.*]], <8 x i64> [[X1:%.*]], <8 x double> [[X2:%.*]])
+; CHECK-NEXT:    store <8 x i64> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x double> [[TMP9]]
+;
+  %1 = call <8 x double> @llvm.x86.avx512.vpermi2var.pd.512(<8 x double> %x0, <8 x i64> %x1, <8 x double> %x2)
+  ret <8 x double> %1
+}
+
+define <8 x double>@test_int_x86_avx512_mask_vpermi2var_pd_512(<8 x double> %x0, <8 x i64> %x1, <8 x double> %x2, i8 %x3) #0 {
+; CHECK-LABEL: @test_int_x86_avx512_mask_vpermi2var_pd_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <8 x i64> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <8 x i64> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP6]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <8 x i64> [[TMP3]] to i512
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i512 [[TMP7]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR3]], label [[TMP8:%.*]], label [[TMP9:%.*]], !prof [[PROF1]]
+; CHECK:       8:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       9:
+; CHECK-NEXT:    [[TMP10:%.*]] = call <8 x double> @llvm.x86.avx512.vpermi2var.pd.512(<8 x double> [[X0:%.*]], <8 x i64> [[X1:%.*]], <8 x double> [[X2:%.*]])
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <8 x i64> [[X1]] to <8 x double>
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast i8 [[TMP4]] to <8 x i1>
+; CHECK-NEXT:    [[TMP13:%.*]] = bitcast i8 [[X3:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP14:%.*]] = select <8 x i1> [[TMP13]], <8 x i64> zeroinitializer, <8 x i64> [[TMP2]]
+; CHECK-NEXT:    [[TMP15:%.*]] = bitcast <8 x double> [[TMP10]] to <8 x i64>
+; CHECK-NEXT:    [[TMP16:%.*]] = bitcast <8 x double> [[TMP11]] to <8 x i64>
+; CHECK-NEXT:    [[TMP17:%.*]] = xor <8 x i64> [[TMP15]], [[TMP16]]
+; CHECK-NEXT:    [[TMP18:%.*]] = or <8 x i64> [[TMP17]], zeroinitializer
+; CHECK-NEXT:    [[TMP19:%.*]] = or <8 x i64> [[TMP18]], [[TMP2]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP12]], <8 x i64> [[TMP19]], <8 x i64> [[TMP14]]
+; CHECK-NEXT:    [[TMP20:%.*]] = select <8 x i1> [[TMP13]], <8 x double> [[TMP10]], <8 x double> [[TMP11]]
+; CHECK-NEXT:    store <8 x i64> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x double> [[TMP20]]
+;
+  %1 = call <8 x double> @llvm.x86.avx512.vpermi2var.pd.512(<8 x double> %x0, <8 x i64> %x1, <8 x double> %x2)
+  %2 = bitcast <8 x i64> %x1 to <8 x double>
+  %3 = bitcast i8 %x3 to <8 x i1>
+  %4 = select <8 x i1> %3, <8 x double> %1, <8 x double> %2
+  ret <8 x double> %4
+}
+
+declare <16 x float> @llvm.x86.avx512.vpermi2var.ps.512(<16 x float>, <16 x i32>, <16 x float>)
+
+define <16 x float>@test_int_x86_avx512_vpermi2var_ps_512(<16 x float> %x0, <16 x i32> %x1, <16 x float> %x2) #0 {
+; CHECK-LABEL: @test_int_x86_avx512_vpermi2var_ps_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <16 x i32> [[TMP3]] to i512
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i512 [[TMP6]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR3]], label [[TMP7:%.*]], label [[TMP8:%.*]], !prof [[PROF1]]
+; CHECK:       7:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       8:
+; CHECK-NEXT:    [[TMP9:%.*]] = call <16 x float> @llvm.x86.avx512.vpermi2var.ps.512(<16 x float> [[X0:%.*]], <16 x i32> [[X1:%.*]], <16 x float> [[X2:%.*]])
+; CHECK-NEXT:    store <16 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x float> [[TMP9]]
+;
+  %1 = call <16 x float> @llvm.x86.avx512.vpermi2var.ps.512(<16 x float> %x0, <16 x i32> %x1, <16 x float> %x2)
+  ret <16 x float> %1
+}
+
+define <16 x float>@test_int_x86_avx512_mask_vpermi2var_ps_512(<16 x float> %x0, <16 x i32> %x1, <16 x float> %x2, i16 %x3) #0 {
+; CHECK-LABEL: @test_int_x86_avx512_mask_vpermi2var_ps_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP6]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <16 x i32> [[TMP3]] to i512
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i512 [[TMP7]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR3]], label [[TMP8:%.*]], label [[TMP9:%.*]], !prof [[PROF1]]
+; CHECK:       8:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       9:
+; CHECK-NEXT:    [[TMP10:%.*]] = call <16 x float> @llvm.x86.avx512.vpermi2var.ps.512(<16 x float> [[X0:%.*]], <16 x i32> [[X1:%.*]], <16 x float> [[X2:%.*]])
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <16 x i32> [[X1]] to <16 x float>
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast i16 [[TMP4]] to <16 x i1>
+; CHECK-NEXT:    [[TMP13:%.*]] = bitcast i16 [[X3:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP14:%.*]] = select <16 x i1> [[TMP13]], <16 x i32> zeroinitializer, <16 x i32> [[TMP2]]
+; CHECK-NEXT:    [[TMP15:%.*]] = bitcast <16 x float> [[TMP10]] to <16 x i32>
+; CHECK-NEXT:    [[TMP16:%.*]] = bitcast <16 x float> [[TMP11]] to <16 x i32>
+; CHECK-NEXT:    [[TMP17:%.*]] = xor <16 x i32> [[TMP15]], [[TMP16]]
+; CHECK-NEXT:    [[TMP18:%.*]] = or <16 x i32> [[TMP17]], zeroinitializer
+; CHECK-NEXT:    [[TMP19:%.*]] = or <16 x i32> [[TMP18]], [[TMP2]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP12]], <16 x i32> [[TMP19]], <16 x i32> [[TMP14]]
+; CHECK-NEXT:    [[TMP20:%.*]] = select <16 x i1> [[TMP13]], <16 x float> [[TMP10]], <16 x float> [[TMP11]]
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x float> [[TMP20]]
+;
+  %1 = call <16 x float> @llvm.x86.avx512.vpermi2var.ps.512(<16 x float> %x0, <16 x i32> %x1, <16 x float> %x2)
+  %2 = bitcast <16 x i32> %x1 to <16 x float>
+  %3 = bitcast i16 %x3 to <16 x i1>
+  %4 = select <16 x i1> %3, <16 x float> %1, <16 x float> %2
+  ret <16 x float> %4
+}
+
+declare <8 x i64> @llvm.x86.avx512.vpermi2var.q.512(<8 x i64>, <8 x i64>, <8 x i64>)
+
+define <8 x i64>@test_int_x86_avx512_vpermi2var_q_512(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> %x2) #0 {
+; CHECK-LABEL: @test_int_x86_avx512_vpermi2var_q_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <8 x i64> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <8 x i64> [[_MSPROP]], [[TMP3]]
+; CHECK-NEXT:    [[TMP4:%.*]] = call <8 x i64> @llvm.x86.avx512.vpermi2var.q.512(<8 x i64> [[X0:%.*]], <8 x i64> [[X1:%.*]], <8 x i64> [[X2:%.*]])
+; CHECK-NEXT:    store <8 x i64> [[_MSPROP1]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i64> [[TMP4]]
+;
+  %1 = call <8 x i64> @llvm.x86.avx512.vpermi2var.q.512(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> %x2)
+  ret <8 x i64> %1
+}
+
+define <8 x i64>@test_int_x86_avx512_mask_vpermi2var_q_512(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> %x2, i8 %x3) #0 {
+; CHECK-LABEL: @test_int_x86_avx512_mask_vpermi2var_q_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <8 x i64> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <8 x i64> [[_MSPROP]], [[TMP3]]
+; CHECK-NEXT:    [[TMP5:%.*]] = call <8 x i64> @llvm.x86.avx512.vpermi2var.q.512(<8 x i64> [[X0:%.*]], <8 x i64> [[X1:%.*]], <8 x i64> [[X2:%.*]])
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i8 [[TMP4]] to <8 x i1>
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast i8 [[X3:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP8:%.*]] = select <8 x i1> [[TMP7]], <8 x i64> [[_MSPROP1]], <8 x i64> [[TMP2]]
+; CHECK-NEXT:    [[TMP9:%.*]] = xor <8 x i64> [[TMP5]], [[X1]]
+; CHECK-NEXT:    [[TMP10:%.*]] = or <8 x i64> [[TMP9]], [[_MSPROP1]]
+; CHECK-NEXT:    [[TMP11:%.*]] = or <8 x i64> [[TMP10]], [[TMP2]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP6]], <8 x i64> [[TMP11]], <8 x i64> [[TMP8]]
+; CHECK-NEXT:    [[TMP12:%.*]] = select <8 x i1> [[TMP7]], <8 x i64> [[TMP5]], <8 x i64> [[X1]]
+; CHECK-NEXT:    store <8 x i64> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i64> [[TMP12]]
+;
+  %1 = call <8 x i64> @llvm.x86.avx512.vpermi2var.q.512(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> %x2)
+  %2 = bitcast i8 %x3 to <8 x i1>
+  %3 = select <8 x i1> %2, <8 x i64> %1, <8 x i64> %x1
+  ret <8 x i64> %3
+}
+
+define <16 x i32>@test_int_x86_avx512_maskz_vpermt2var_d_512(<16 x i32> %x0, <16 x i32> %x1, ptr %x2p, i16 %x3) #0 {
+; CHECK-LABEL: @test_int_x86_avx512_maskz_vpermt2var_d_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 136) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP5:%.*]], label [[TMP6:%.*]], !prof [[PROF1]]
+; CHECK:       5:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       6:
+; CHECK-NEXT:    [[X2:%.*]] = load <16 x i32>, ptr [[X2P:%.*]], align 64
+; CHECK-NEXT:    [[TMP7:%.*]] = ptrtoint ptr [[X2P]] to i64
+; CHECK-NEXT:    [[TMP8:%.*]] = xor i64 [[TMP7]], 87960930222080
+; CHECK-NEXT:    [[TMP9:%.*]] = inttoptr i64 [[TMP8]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <16 x i32>, ptr [[TMP9]], align 64
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <16 x i32> [[TMP2]], [[TMP3]]
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <16 x i32> [[_MSPROP]], [[_MSLD]]
+; CHECK-NEXT:    [[TMP10:%.*]] = call <16 x i32> @llvm.x86.avx512.vpermi2var.d.512(<16 x i32> [[X1:%.*]], <16 x i32> [[X0:%.*]], <16 x i32> [[X2]])
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast i16 [[TMP4]] to <16 x i1>
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast i16 [[X3:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP13:%.*]] = select <16 x i1> [[TMP12]], <16 x i32> [[_MSPROP1]], <16 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP14:%.*]] = xor <16 x i32> [[TMP10]], zeroinitializer
+; CHECK-NEXT:    [[TMP15:%.*]] = or <16 x i32> [[TMP14]], [[_MSPROP1]]
+; CHECK-NEXT:    [[TMP16:%.*]] = or <16 x i32> [[TMP15]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP11]], <16 x i32> [[TMP16]], <16 x i32> [[TMP13]]
+; CHECK-NEXT:    [[TMP17:%.*]] = select <16 x i1> [[TMP12]], <16 x i32> [[TMP10]], <16 x i32> zeroinitializer
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i32> [[TMP17]]
+;
+  %x2 = load <16 x i32>, ptr %x2p
+  %1 = call <16 x i32> @llvm.x86.avx512.vpermi2var.d.512(<16 x i32> %x1, <16 x i32> %x0, <16 x i32> %x2)
+  %2 = bitcast i16 %x3 to <16 x i1>
+  %3 = select <16 x i1> %2, <16 x i32> %1, <16 x i32> zeroinitializer
+  ret <16 x i32> %3
+}
+
+define <8 x double>@test_int_x86_avx512_maskz_vpermt2var_pd_512(<8 x i64> %x0, <8 x double> %x1, ptr %x2ptr, i8 %x3, <8 x double> %extra_param, <8 x double> %extra_param2) #0 {
+; CHECK-LABEL: @test_int_x86_avx512_maskz_vpermt2var_pd_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    [[TMP5:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 144) to ptr), align 8
+; CHECK-NEXT:    [[TMP6:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 208) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 136) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP13:%.*]], label [[TMP14:%.*]], !prof [[PROF1]]
+; CHECK:       7:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       8:
+; CHECK-NEXT:    [[X2S:%.*]] = load double, ptr [[X2PTR:%.*]], align 8
+; CHECK-NEXT:    [[TMP7:%.*]] = ptrtoint ptr [[X2PTR]] to i64
+; CHECK-NEXT:    [[TMP8:%.*]] = xor i64 [[TMP7]], 87960930222080
+; CHECK-NEXT:    [[TMP9:%.*]] = inttoptr i64 [[TMP8]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load i64, ptr [[TMP9]], align 8
+; CHECK-NEXT:    [[_MSPROP:%.*]] = insertelement <8 x i64> [[TMP5]], i64 [[_MSLD]], i32 0
+; CHECK-NEXT:    [[X2INS:%.*]] = insertelement <8 x double> [[EXTRA_PARAM:%.*]], double [[X2S]], i32 0
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = shufflevector <8 x i64> [[_MSPROP]], <8 x i64> [[TMP6]], <8 x i32> zeroinitializer
+; CHECK-NEXT:    [[X2:%.*]] = shufflevector <8 x double> [[X2INS]], <8 x double> [[EXTRA_PARAM2:%.*]], <8 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast <8 x i64> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i512 [[TMP10]], 0
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <8 x i64> [[TMP3]] to i512
+; CHECK-NEXT:    [[_MSCMP3:%.*]] = icmp ne i512 [[TMP11]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP2]], [[_MSCMP3]]
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <8 x i64> [[_MSPROP1]] to i512
+; CHECK-NEXT:    [[_MSCMP4:%.*]] = icmp ne i512 [[TMP12]], 0
+; CHECK-NEXT:    [[_MSOR5:%.*]] = or i1 [[_MSOR]], [[_MSCMP4]]
+; CHECK-NEXT:    br i1 [[_MSOR5]], label [[TMP24:%.*]], label [[TMP25:%.*]], !prof [[PROF1]]
+; CHECK:       15:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       16:
+; CHECK-NEXT:    [[TMP15:%.*]] = call <8 x double> @llvm.x86.avx512.vpermi2var.pd.512(<8 x double> [[X1:%.*]], <8 x i64> [[X0:%.*]], <8 x double> [[X2]])
+; CHECK-NEXT:    [[TMP16:%.*]] = bitcast i8 [[TMP4]] to <8 x i1>
+; CHECK-NEXT:    [[TMP17:%.*]] = bitcast i8 [[X3:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP18:%.*]] = select <8 x i1> [[TMP17]], <8 x i64> zeroinitializer, <8 x i64> zeroinitializer
+; CHECK-NEXT:    [[TMP19:%.*]] = bitcast <8 x double> [[TMP15]] to <8 x i64>
+; CHECK-NEXT:    [[TMP20:%.*]] = xor <8 x i64> [[TMP19]], zeroinitializer
+; CHECK-NEXT:    [[TMP21:%.*]] = or <8 x i64> [[TMP20]], zeroinitializer
+; CHECK-NEXT:    [[TMP22:%.*]] = or <8 x i64> [[TMP21]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP16]], <8 x i64> [[TMP22]], <8 x i64> [[TMP18]]
+; CHECK-NEXT:    [[TMP23:%.*]] = select <8 x i1> [[TMP17]], <8 x double> [[TMP15]], <8 x double> zeroinitializer
+; CHECK-NEXT:    store <8 x i64> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x double> [[TMP23]]
+;
+  %x2s = load double, ptr %x2ptr
+  %x2ins = insertelement <8 x double> %extra_param, double %x2s, i32 0
+  %x2 = shufflevector <8 x double> %x2ins, <8 x double> %extra_param2, <8 x i32> zeroinitializer
+  %1 = call <8 x double> @llvm.x86.avx512.vpermi2var.pd.512(<8 x double> %x1, <8 x i64> %x0, <8 x double> %x2)
+  %2 = bitcast i8 %x3 to <8 x i1>
+  %3 = select <8 x i1> %2, <8 x double> %1, <8 x double> zeroinitializer
+  ret <8 x double> %3
+}
+
+define <16 x float>@test_int_x86_avx512_maskz_vpermt2var_ps_512(<16 x i32> %x0, <16 x float> %x1, <16 x float> %x2, i16 %x3) #0 {
+; CHECK-LABEL: @test_int_x86_avx512_maskz_vpermt2var_ps_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP6]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <16 x i32> [[TMP3]] to i512
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i512 [[TMP7]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR3]], label [[TMP8:%.*]], label [[TMP9:%.*]], !prof [[PROF1]]
+; CHECK:       8:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       9:
+; CHECK-NEXT:    [[TMP10:%.*]] = call <16 x float> @llvm.x86.avx512.vpermi2var.ps.512(<16 x float> [[X1:%.*]], <16 x i32> [[X0:%.*]], <16 x float> [[X2:%.*]])
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast i16 [[TMP4]] to <16 x i1>
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast i16 [[X3:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP13:%.*]] = select <16 x i1> [[TMP12]], <16 x i32> zeroinitializer, <16 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast <16 x float> [[TMP10]] to <16 x i32>
+; CHECK-NEXT:    [[TMP15:%.*]] = xor <16 x i32> [[TMP14]], zeroinitializer
+; CHECK-NEXT:    [[TMP16:%.*]] = or <16 x i32> [[TMP15]], zeroinitializer
+; CHECK-NEXT:    [[TMP17:%.*]] = or <16 x i32> [[TMP16]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP11]], <16 x i32> [[TMP17]], <16 x i32> [[TMP13]]
+; CHECK-NEXT:    [[TMP18:%.*]] = select <16 x i1> [[TMP12]], <16 x float> [[TMP10]], <16 x float> zeroinitializer
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x float> [[TMP18]]
+;
+  %1 = call <16 x float> @llvm.x86.avx512.vpermi2var.ps.512(<16 x float> %x1, <16 x i32> %x0, <16 x float> %x2)
+  %2 = bitcast i16 %x3 to <16 x i1>
+  %3 = select <16 x i1> %2, <16 x float> %1, <16 x float> zeroinitializer
+  ret <16 x float> %3
+}
+
+define <8 x i64>@test_int_x86_avx512_maskz_vpermt2var_q_512(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> %x2, i8 %x3) #0 {
+; CHECK-LABEL: @test_int_x86_avx512_maskz_vpermt2var_q_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <8 x i64> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <8 x i64> [[_MSPROP]], [[TMP3]]
+; CHECK-NEXT:    [[TMP5:%.*]] = call <8 x i64> @llvm.x86.avx512.vpermi2var.q.512(<8 x i64> [[X1:%.*]], <8 x i64> [[X0:%.*]], <8 x i64> [[X2:%.*]])
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i8 [[TMP4]] to <8 x i1>
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast i8 [[X3:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP8:%.*]] = select <8 x i1> [[TMP7]], <8 x i64> [[_MSPROP1]], <8 x i64> zeroinitializer
+; CHECK-NEXT:    [[TMP9:%.*]] = xor <8 x i64> [[TMP5]], zeroinitializer
+; CHECK-NEXT:    [[TMP10:%.*]] = or <8 x i64> [[TMP9]], [[_MSPROP1]]
+; CHECK-NEXT:    [[TMP11:%.*]] = or <8 x i64> [[TMP10]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP6]], <8 x i64> [[TMP11]], <8 x i64> [[TMP8]]
+; CHECK-NEXT:    [[TMP12:%.*]] = select <8 x i1> [[TMP7]], <8 x i64> [[TMP5]], <8 x i64> zeroinitializer
+; CHECK-NEXT:    store <8 x i64> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i64> [[TMP12]]
+;
+  %1 = call <8 x i64> @llvm.x86.avx512.vpermi2var.q.512(<8 x i64> %x1, <8 x i64> %x0, <8 x i64> %x2)
+  %2 = bitcast i8 %x3 to <8 x i1>
+  %3 = select <8 x i1> %2, <8 x i64> %1, <8 x i64> zeroinitializer
+  ret <8 x i64> %3
+}
+
+define <16 x i32>@test_int_x86_avx512_vpermt2var_d_512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x2) #0 {
+; CHECK-LABEL: @test_int_x86_avx512_vpermt2var_d_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <16 x i32> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <16 x i32> [[_MSPROP]], [[TMP3]]
+; CHECK-NEXT:    [[TMP4:%.*]] = call <16 x i32> @llvm.x86.avx512.vpermi2var.d.512(<16 x i32> [[X1:%.*]], <16 x i32> [[X0:%.*]], <16 x i32> [[X2:%.*]])
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP1]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i32> [[TMP4]]
+;
+  %1 = call <16 x i32> @llvm.x86.avx512.vpermi2var.d.512(<16 x i32> %x1, <16 x i32> %x0, <16 x i32> %x2)
+  ret <16 x i32> %1
+}
+
+define <16 x i32>@test_int_x86_avx512_mask_vpermt2var_d_512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x2, i16 %x3) #0 {
+; CHECK-LABEL: @test_int_x86_avx512_mask_vpermt2var_d_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <16 x i32> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <16 x i32> [[_MSPROP]], [[TMP3]]
+; CHECK-NEXT:    [[TMP5:%.*]] = call <16 x i32> @llvm.x86.avx512.vpermi2var.d.512(<16 x i32> [[X1:%.*]], <16 x i32> [[X0:%.*]], <16 x i32> [[X2:%.*]])
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i16 [[TMP4]] to <16 x i1>
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast i16 [[X3:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP8:%.*]] = select <16 x i1> [[TMP7]], <16 x i32> [[_MSPROP1]], <16 x i32> [[TMP1]]
+; CHECK-NEXT:    [[TMP9:%.*]] = xor <16 x i32> [[TMP5]], [[X1]]
+; CHECK-NEXT:    [[TMP10:%.*]] = or <16 x i32> [[TMP9]], [[_MSPROP1]]
+; CHECK-NEXT:    [[TMP11:%.*]] = or <16 x i32> [[TMP10]], [[TMP1]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP6]], <16 x i32> [[TMP11]], <16 x i32> [[TMP8]]
+; CHECK-NEXT:    [[TMP12:%.*]] = select <16 x i1> [[TMP7]], <16 x i32> [[TMP5]], <16 x i32> [[X1]]
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i32> [[TMP12]]
+;
+  %1 = call <16 x i32> @llvm.x86.avx512.vpermi2var.d.512(<16 x i32> %x1, <16 x i32> %x0, <16 x i32> %x2)
+  %2 = bitcast i16 %x3 to <16 x i1>
+  %3 = select <16 x i1> %2, <16 x i32> %1, <16 x i32> %x1
+  ret <16 x i32> %3
+}
+
+declare <8 x double> @llvm.x86.avx512.mask.scalef.pd.512(<8 x double>, <8 x double>, <8 x double>, i8, i32)
+define <8 x double>@test_int_x86_avx512_mask_scalef_pd_512(<8 x double> %x0, <8 x double> %x1, <8 x double> %x2, i8 %x3) #0 {
+; CHECK-LABEL: @test_int_x86_avx512_mask_scalef_pd_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <8 x i64> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <8 x i64> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP6]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <8 x i64> [[TMP3]] to i512
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i512 [[TMP7]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    [[_MSCMP4:%.*]] = icmp ne i8 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR5:%.*]] = or i1 [[_MSOR3]], [[_MSCMP4]]
+; CHECK-NEXT:    br i1 [[_MSOR5]], label [[TMP8:%.*]], label [[TMP9:%.*]], !prof [[PROF1]]
+; CHECK:       8:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       9:
+; CHECK-NEXT:    [[RES:%.*]] = call <8 x double> @llvm.x86.avx512.mask.scalef.pd.512(<8 x double> [[X0:%.*]], <8 x double> [[X1:%.*]], <8 x double> [[X2:%.*]], i8 [[X3:%.*]], i32 11)
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast <8 x i64> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP6:%.*]] = icmp ne i512 [[TMP10]], 0
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <8 x i64> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP7:%.*]] = icmp ne i512 [[TMP11]], 0
+; CHECK-NEXT:    [[_MSOR8:%.*]] = or i1 [[_MSCMP6]], [[_MSCMP7]]
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <8 x i64> [[TMP3]] to i512
+; CHECK-NEXT:    [[_MSCMP9:%.*]] = icmp ne i512 [[TMP12]], 0
+; CHECK-NEXT:    [[_MSOR10:%.*]] = or i1 [[_MSOR8]], [[_MSCMP9]]
+; CHECK-NEXT:    br i1 [[_MSOR10]], label [[TMP13:%.*]], label [[TMP14:%.*]], !prof [[PROF1]]
+; CHECK:       13:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       14:
+; CHECK-NEXT:    [[RES1:%.*]] = call <8 x double> @llvm.x86.avx512.mask.scalef.pd.512(<8 x double> [[X0]], <8 x double> [[X1]], <8 x double> [[X2]], i8 -1, i32 8)
+; CHECK-NEXT:    [[RES2:%.*]] = fadd <8 x double> [[RES]], [[RES1]]
+; CHECK-NEXT:    store <8 x i64> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x double> [[RES2]]
+;
+  %res = call <8 x double> @llvm.x86.avx512.mask.scalef.pd.512(<8 x double> %x0, <8 x double> %x1, <8 x double> %x2, i8 %x3, i32 11)
+  %res1 = call <8 x double> @llvm.x86.avx512.mask.scalef.pd.512(<8 x double> %x0, <8 x double> %x1, <8 x double> %x2, i8 -1, i32 8)
+  %res2 = fadd <8 x double> %res, %res1
+  ret <8 x double> %res2
+}
+
+declare <16 x float> @llvm.x86.avx512.mask.scalef.ps.512(<16 x float>, <16 x float>, <16 x float>, i16, i32)
+define <16 x float>@test_int_x86_avx512_mask_scalef_ps_512(<16 x float> %x0, <16 x float> %x1, <16 x float> %x2, i16 %x3) #0 {
+; CHECK-LABEL: @test_int_x86_avx512_mask_scalef_ps_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP6]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <16 x i32> [[TMP3]] to i512
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i512 [[TMP7]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    [[_MSCMP4:%.*]] = icmp ne i16 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR5:%.*]] = or i1 [[_MSOR3]], [[_MSCMP4]]
+; CHECK-NEXT:    br i1 [[_MSOR5]], label [[TMP8:%.*]], label [[TMP9:%.*]], !prof [[PROF1]]
+; CHECK:       8:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       9:
+; CHECK-NEXT:    [[RES:%.*]] = call <16 x float> @llvm.x86.avx512.mask.scalef.ps.512(<16 x float> [[X0:%.*]], <16 x float> [[X1:%.*]], <16 x float> [[X2:%.*]], i16 [[X3:%.*]], i32 10)
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP6:%.*]] = icmp ne i512 [[TMP10]], 0
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP7:%.*]] = icmp ne i512 [[TMP11]], 0
+; CHECK-NEXT:    [[_MSOR8:%.*]] = or i1 [[_MSCMP6]], [[_MSCMP7]]
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <16 x i32> [[TMP3]] to i512
+; CHECK-NEXT:    [[_MSCMP9:%.*]] = icmp ne i512 [[TMP12]], 0
+; CHECK-NEXT:    [[_MSOR10:%.*]] = or i1 [[_MSOR8]], [[_MSCMP9]]
+; CHECK-NEXT:    br i1 [[_MSOR10]], label [[TMP13:%.*]], label [[TMP14:%.*]], !prof [[PROF1]]
+; CHECK:       13:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       14:
+; CHECK-NEXT:    [[RES1:%.*]] = call <16 x float> @llvm.x86.avx512.mask.scalef.ps.512(<16 x float> [[X0]], <16 x float> [[X1]], <16 x float> [[X2]], i16 -1, i32 8)
+; CHECK-NEXT:    [[RES2:%.*]] = fadd <16 x float> [[RES]], [[RES1]]
+; CHECK-NEXT:    store <16 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x float> [[RES2]]
+;
+  %res = call <16 x float> @llvm.x86.avx512.mask.scalef.ps.512(<16 x float> %x0, <16 x float> %x1, <16 x float> %x2, i16 %x3, i32 10)
+  %res1 = call <16 x float> @llvm.x86.avx512.mask.scalef.ps.512(<16 x float> %x0, <16 x float> %x1, <16 x float> %x2, i16 -1, i32 8)
+  %res2 = fadd <16 x float> %res, %res1
+  ret <16 x float> %res2
+}
+
+declare <16 x i8> @llvm.x86.avx512.mask.pmov.qb.512(<8 x i64>, <16 x i8>, i8)
+
+define <16 x i8>@test_int_x86_avx512_mask_pmov_qb_512(<8 x i64> %x0, <16 x i8> %x1, i8 %x2) #0 {
+; CHECK-LABEL: @test_int_x86_avx512_mask_pmov_qb_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 80) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <8 x i64> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <16 x i8> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP6:%.*]], label [[TMP7:%.*]], !prof [[PROF1]]
+; CHECK:       6:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       7:
+; CHECK-NEXT:    [[RES0:%.*]] = call <16 x i8> @llvm.x86.avx512.mask.pmov.qb.512(<8 x i64> [[X0:%.*]], <16 x i8> [[X1:%.*]], i8 -1)
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <8 x i64> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i512 [[TMP8]], 0
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <16 x i8> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP3:%.*]] = icmp ne i128 [[TMP9]], 0
+; CHECK-NEXT:    [[_MSOR4:%.*]] = or i1 [[_MSCMP2]], [[_MSCMP3]]
+; CHECK-NEXT:    [[_MSCMP5:%.*]] = icmp ne i8 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSOR6:%.*]] = or i1 [[_MSOR4]], [[_MSCMP5]]
+; CHECK-NEXT:    br i1 [[_MSOR6]], label [[TMP10:%.*]], label [[TMP11:%.*]], !prof [[PROF1]]
+; CHECK:       10:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       11:
+; CHECK-NEXT:    [[RES1:%.*]] = call <16 x i8> @llvm.x86.avx512.mask.pmov.qb.512(<8 x i64> [[X0]], <16 x i8> [[X1]], i8 [[X2:%.*]])
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <8 x i64> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP7:%.*]] = icmp ne i512 [[TMP12]], 0
+; CHECK-NEXT:    [[_MSCMP8:%.*]] = icmp ne i8 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSOR9:%.*]] = or i1 [[_MSCMP7]], [[_MSCMP8]]
+; CHECK-NEXT:    br i1 [[_MSOR9]], label [[TMP13:%.*]], label [[TMP14:%.*]], !prof [[PROF1]]
+; CHECK:       13:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       14:
+; CHECK-NEXT:    [[RES2:%.*]] = call <16 x i8> @llvm.x86.avx512.mask.pmov.qb.512(<8 x i64> [[X0]], <16 x i8> zeroinitializer, i8 [[X2]])
+; CHECK-NEXT:    [[RES3:%.*]] = add <16 x i8> [[RES0]], [[RES1]]
+; CHECK-NEXT:    [[RES4:%.*]] = add <16 x i8> [[RES3]], [[RES2]]
+; CHECK-NEXT:    store <16 x i8> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i8> [[RES4]]
+;
+  %res0 = call <16 x i8> @llvm.x86.avx512.mask.pmov.qb.512(<8 x i64> %x0, <16 x i8> %x1, i8 -1)
+  %res1 = call <16 x i8> @llvm.x86.avx512.mask.pmov.qb.512(<8 x i64> %x0, <16 x i8> %x1, i8 %x2)
+  %res2 = call <16 x i8> @llvm.x86.avx512.mask.pmov.qb.512(<8 x i64> %x0, <16 x i8> zeroinitializer, i8 %x2)
+  %res3 = add <16 x i8> %res0, %res1
+  %res4 = add <16 x i8> %res3, %res2
+  ret <16 x i8> %res4
+}
+
+declare void @llvm.x86.avx512.mask.pmov.qb.mem.512(ptr %ptr, <8 x i64>, i8)
+
+define void @test_int_x86_avx512_mask_pmov_qb_mem_512(ptr %ptr, <8 x i64> %x1, i8 %x2) #0 {
+; CHECK-LABEL: @test_int_x86_avx512_mask_pmov_qb_mem_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 72) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <8 x i64> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP5:%.*]], label [[TMP6:%.*]], !prof [[PROF1]]
+; CHECK:       5:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       6:
+; CHECK-NEXT:    call void @llvm.x86.avx512.mask.pmov.qb.mem.512(ptr [[PTR:%.*]], <8 x i64> [[X1:%.*]], i8 -1)
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <8 x i64> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP3:%.*]] = icmp ne i512 [[TMP7]], 0
+; CHECK-NEXT:    [[_MSOR4:%.*]] = or i1 [[_MSCMP2]], [[_MSCMP3]]
+; CHECK-NEXT:    [[_MSCMP5:%.*]] = icmp ne i8 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSOR6:%.*]] = or i1 [[_MSOR4]], [[_MSCMP5]]
+; CHECK-NEXT:    br i1 [[_MSOR6]], label [[TMP8:%.*]], label [[TMP9:%.*]], !prof [[PROF1]]
+; CHECK:       8:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       9:
+; CHECK-NEXT:    call void @llvm.x86.avx512.mask.pmov.qb.mem.512(ptr [[PTR]], <8 x i64> [[X1]], i8 [[X2:%.*]])
+; CHECK-NEXT:    ret void
+;
+  call void @llvm.x86.avx512.mask.pmov.qb.mem.512(ptr %ptr, <8 x i64> %x1, i8 -1)
+  call void @llvm.x86.avx512.mask.pmov.qb.mem.512(ptr %ptr, <8 x i64> %x1, i8 %x2)
+  ret void
+}
+
+declare <16 x i8> @llvm.x86.avx512.mask.pmovs.qb.512(<8 x i64>, <16 x i8>, i8)
+
+define <16 x i8>@test_int_x86_avx512_mask_pmovs_qb_512(<8 x i64> %x0, <16 x i8> %x1, i8 %x2) #0 {
+; CHECK-LABEL: @test_int_x86_avx512_mask_pmovs_qb_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 80) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <8 x i64> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <16 x i8> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP6:%.*]], label [[TMP7:%.*]], !prof [[PROF1]]
+; CHECK:       6:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       7:
+; CHECK-NEXT:    [[RES0:%.*]] = call <16 x i8> @llvm.x86.avx512.mask.pmovs.qb.512(<8 x i64> [[X0:%.*]], <16 x i8> [[X1:%.*]], i8 -1)
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <8 x i64> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i512 [[TMP8]], 0
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <16 x i8> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP3:%.*]] = icmp ne i128 [[TMP9]], 0
+; CHECK-NEXT:    [[_MSOR4:%.*]] = or i1 [[_MSCMP2]], [[_MSCMP3]]
+; CHECK-NEXT:    [[_MSCMP5:%.*]] = icmp ne i8 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSOR6:%.*]] = or i1 [[_MSOR4]], [[_MSCMP5]]
+; CHECK-NEXT:    br i1 [[_MSOR6]], label [[TMP10:%.*]], label [[TMP11:%.*]], !prof [[PROF1]]
+; CHECK:       10:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       11:
+; CHECK-NEXT:    [[RES1:%.*]] = call <16 x i8> @llvm.x86.avx512.mask.pmovs.qb.512(<8 x i64> [[X0]], <16 x i8> [[X1]], i8 [[X2:%.*]])
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <8 x i64> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP7:%.*]] = icmp ne i512 [[TMP12]], 0
+; CHECK-NEXT:    [[_MSCMP8:%.*]] = icmp ne i8 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSOR9:%.*]] = or i1 [[_MSCMP7]], [[_MSCMP8]]
+; CHECK-NEXT:    br i1 [[_MSOR9]], label [[TMP13:%.*]], label [[TMP14:%.*]], !prof [[PROF1]]
+; CHECK:       13:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       14:
+; CHECK-NEXT:    [[RES2:%.*]] = call <16 x i8> @llvm.x86.avx512.mask.pmovs.qb.512(<8 x i64> [[X0]], <16 x i8> zeroinitializer, i8 [[X2]])
+; CHECK-NEXT:    [[RES3:%.*]] = add <16 x i8> [[RES0]], [[RES1]]
+; CHECK-NEXT:    [[RES4:%.*]] = add <16 x i8> [[RES3]], [[RES2]]
+; CHECK-NEXT:    store <16 x i8> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i8> [[RES4]]
+;
+  %res0 = call <16 x i8> @llvm.x86.avx512.mask.pmovs.qb.512(<8 x i64> %x0, <16 x i8> %x1, i8 -1)
+  %res1 = call <16 x i8> @llvm.x86.avx512.mask.pmovs.qb.512(<8 x i64> %x0, <16 x i8> %x1, i8 %x2)
+  %res2 = call <16 x i8> @llvm.x86.avx512.mask.pmovs.qb.512(<8 x i64> %x0, <16 x i8> zeroinitializer, i8 %x2)
+  %res3 = add <16 x i8> %res0, %res1
+  %res4 = add <16 x i8> %res3, %res2
+  ret <16 x i8> %res4
+}
+
+declare void @llvm.x86.avx512.mask.pmovs.qb.mem.512(ptr %ptr, <8 x i64>, i8)
+
+define void @test_int_x86_avx512_mask_pmovs_qb_mem_512(ptr %ptr, <8 x i64> %x1, i8 %x2) #0 {
+; CHECK-LABEL: @test_int_x86_avx512_mask_pmovs_qb_mem_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 72) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <8 x i64> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP5:%.*]], label [[TMP6:%.*]], !prof [[PROF1]]
+; CHECK:       5:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       6:
+; CHECK-NEXT:    call void @llvm.x86.avx512.mask.pmovs.qb.mem.512(ptr [[PTR:%.*]], <8 x i64> [[X1:%.*]], i8 -1)
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <8 x i64> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP3:%.*]] = icmp ne i512 [[TMP7]], 0
+; CHECK-NEXT:    [[_MSOR4:%.*]] = or i1 [[_MSCMP2]], [[_MSCMP3]]
+; CHECK-NEXT:    [[_MSCMP5:%.*]] = icmp ne i8 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSOR6:%.*]] = or i1 [[_MSOR4]], [[_MSCMP5]]
+; CHECK-NEXT:    br i1 [[_MSOR6]], label [[TMP8:%.*]], label [[TMP9:%.*]], !prof [[PROF1]]
+; CHECK:       8:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       9:
+; CHECK-NEXT:    call void @llvm.x86.avx512.mask.pmovs.qb.mem.512(ptr [[PTR]], <8 x i64> [[X1]], i8 [[X2:%.*]])
+; CHECK-NEXT:    ret void
+;
+  call void @llvm.x86.avx512.mask.pmovs.qb.mem.512(ptr %ptr, <8 x i64> %x1, i8 -1)
+  call void @llvm.x86.avx512.mask.pmovs.qb.mem.512(ptr %ptr, <8 x i64> %x1, i8 %x2)
+  ret void
+}
+
+declare <16 x i8> @llvm.x86.avx512.mask.pmovus.qb.512(<8 x i64>, <16 x i8>, i8)
+
+define <16 x i8>@test_int_x86_avx512_mask_pmovus_qb_512(<8 x i64> %x0, <16 x i8> %x1, i8 %x2) #0 {
+; CHECK-LABEL: @test_int_x86_avx512_mask_pmovus_qb_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 80) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <8 x i64> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <16 x i8> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP6:%.*]], label [[TMP7:%.*]], !prof [[PROF1]]
+; CHECK:       6:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       7:
+; CHECK-NEXT:    [[RES0:%.*]] = call <16 x i8> @llvm.x86.avx512.mask.pmovus.qb.512(<8 x i64> [[X0:%.*]], <16 x i8> [[X1:%.*]], i8 -1)
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <8 x i64> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i512 [[TMP8]], 0
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <16 x i8> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP3:%.*]] = icmp ne i128 [[TMP9]], 0
+; CHECK-NEXT:    [[_MSOR4:%.*]] = or i1 [[_MSCMP2]], [[_MSCMP3]]
+; CHECK-NEXT:    [[_MSCMP5:%.*]] = icmp ne i8 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSOR6:%.*]] = or i1 [[_MSOR4]], [[_MSCMP5]]
+; CHECK-NEXT:    br i1 [[_MSOR6]], label [[TMP10:%.*]], label [[TMP11:%.*]], !prof [[PROF1]]
+; CHECK:       10:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       11:
+; CHECK-NEXT:    [[RES1:%.*]] = call <16 x i8> @llvm.x86.avx512.mask.pmovus.qb.512(<8 x i64> [[X0]], <16 x i8> [[X1]], i8 [[X2:%.*]])
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <8 x i64> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP7:%.*]] = icmp ne i512 [[TMP12]], 0
+; CHECK-NEXT:    [[_MSCMP8:%.*]] = icmp ne i8 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSOR9:%.*]] = or i1 [[_MSCMP7]], [[_MSCMP8]]
+; CHECK-NEXT:    br i1 [[_MSOR9]], label [[TMP13:%.*]], label [[TMP14:%.*]], !prof [[PROF1]]
+; CHECK:       13:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       14:
+; CHECK-NEXT:    [[RES2:%.*]] = call <16 x i8> @llvm.x86.avx512.mask.pmovus.qb.512(<8 x i64> [[X0]], <16 x i8> zeroinitializer, i8 [[X2]])
+; CHECK-NEXT:    [[RES3:%.*]] = add <16 x i8> [[RES0]], [[RES1]]
+; CHECK-NEXT:    [[RES4:%.*]] = add <16 x i8> [[RES3]], [[RES2]]
+; CHECK-NEXT:    store <16 x i8> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i8> [[RES4]]
+;
+  %res0 = call <16 x i8> @llvm.x86.avx512.mask.pmovus.qb.512(<8 x i64> %x0, <16 x i8> %x1, i8 -1)
+  %res1 = call <16 x i8> @llvm.x86.avx512.mask.pmovus.qb.512(<8 x i64> %x0, <16 x i8> %x1, i8 %x2)
+  %res2 = call <16 x i8> @llvm.x86.avx512.mask.pmovus.qb.512(<8 x i64> %x0, <16 x i8> zeroinitializer, i8 %x2)
+  %res3 = add <16 x i8> %res0, %res1
+  %res4 = add <16 x i8> %res3, %res2
+  ret <16 x i8> %res4
+}
+
+declare void @llvm.x86.avx512.mask.pmovus.qb.mem.512(ptr %ptr, <8 x i64>, i8)
+
+define void @test_int_x86_avx512_mask_pmovus_qb_mem_512(ptr %ptr, <8 x i64> %x1, i8 %x2) #0 {
+; CHECK-LABEL: @test_int_x86_avx512_mask_pmovus_qb_mem_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 72) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <8 x i64> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP5:%.*]], label [[TMP6:%.*]], !prof [[PROF1]]
+; CHECK:       5:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       6:
+; CHECK-NEXT:    call void @llvm.x86.avx512.mask.pmovus.qb.mem.512(ptr [[PTR:%.*]], <8 x i64> [[X1:%.*]], i8 -1)
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <8 x i64> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP3:%.*]] = icmp ne i512 [[TMP7]], 0
+; CHECK-NEXT:    [[_MSOR4:%.*]] = or i1 [[_MSCMP2]], [[_MSCMP3]]
+; CHECK-NEXT:    [[_MSCMP5:%.*]] = icmp ne i8 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSOR6:%.*]] = or i1 [[_MSOR4]], [[_MSCMP5]]
+; CHECK-NEXT:    br i1 [[_MSOR6]], label [[TMP8:%.*]], label [[TMP9:%.*]], !prof [[PROF1]]
+; CHECK:       8:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       9:
+; CHECK-NEXT:    call void @llvm.x86.avx512.mask.pmovus.qb.mem.512(ptr [[PTR]], <8 x i64> [[X1]], i8 [[X2:%.*]])
+; CHECK-NEXT:    ret void
+;
+  call void @llvm.x86.avx512.mask.pmovus.qb.mem.512(ptr %ptr, <8 x i64> %x1, i8 -1)
+  call void @llvm.x86.avx512.mask.pmovus.qb.mem.512(ptr %ptr, <8 x i64> %x1, i8 %x2)
+  ret void
+}
+
+declare <8 x i16> @llvm.x86.avx512.mask.pmov.qw.512(<8 x i64>, <8 x i16>, i8)
+
+define <8 x i16>@test_int_x86_avx512_mask_pmov_qw_512(<8 x i64> %x0, <8 x i16> %x1, i8 %x2) #0 {
+; CHECK-LABEL: @test_int_x86_avx512_mask_pmov_qw_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 80) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <8 x i64> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <8 x i16> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP6:%.*]], label [[TMP7:%.*]], !prof [[PROF1]]
+; CHECK:       6:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       7:
+; CHECK-NEXT:    [[RES0:%.*]] = call <8 x i16> @llvm.x86.avx512.mask.pmov.qw.512(<8 x i64> [[X0:%.*]], <8 x i16> [[X1:%.*]], i8 -1)
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <8 x i64> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i512 [[TMP8]], 0
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <8 x i16> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP3:%.*]] = icmp ne i128 [[TMP9]], 0
+; CHECK-NEXT:    [[_MSOR4:%.*]] = or i1 [[_MSCMP2]], [[_MSCMP3]]
+; CHECK-NEXT:    [[_MSCMP5:%.*]] = icmp ne i8 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSOR6:%.*]] = or i1 [[_MSOR4]], [[_MSCMP5]]
+; CHECK-NEXT:    br i1 [[_MSOR6]], label [[TMP10:%.*]], label [[TMP11:%.*]], !prof [[PROF1]]
+; CHECK:       10:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       11:
+; CHECK-NEXT:    [[RES1:%.*]] = call <8 x i16> @llvm.x86.avx512.mask.pmov.qw.512(<8 x i64> [[X0]], <8 x i16> [[X1]], i8 [[X2:%.*]])
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <8 x i64> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP7:%.*]] = icmp ne i512 [[TMP12]], 0
+; CHECK-NEXT:    [[_MSCMP8:%.*]] = icmp ne i8 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSOR9:%.*]] = or i1 [[_MSCMP7]], [[_MSCMP8]]
+; CHECK-NEXT:    br i1 [[_MSOR9]], label [[TMP13:%.*]], label [[TMP14:%.*]], !prof [[PROF1]]
+; CHECK:       13:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       14:
+; CHECK-NEXT:    [[RES2:%.*]] = call <8 x i16> @llvm.x86.avx512.mask.pmov.qw.512(<8 x i64> [[X0]], <8 x i16> zeroinitializer, i8 [[X2]])
+; CHECK-NEXT:    [[RES3:%.*]] = add <8 x i16> [[RES0]], [[RES1]]
+; CHECK-NEXT:    [[RES4:%.*]] = add <8 x i16> [[RES3]], [[RES2]]
+; CHECK-NEXT:    store <8 x i16> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i16> [[RES4]]
+;
+  %res0 = call <8 x i16> @llvm.x86.avx512.mask.pmov.qw.512(<8 x i64> %x0, <8 x i16> %x1, i8 -1)
+  %res1 = call <8 x i16> @llvm.x86.avx512.mask.pmov.qw.512(<8 x i64> %x0, <8 x i16> %x1, i8 %x2)
+  %res2 = call <8 x i16> @llvm.x86.avx512.mask.pmov.qw.512(<8 x i64> %x0, <8 x i16> zeroinitializer, i8 %x2)
+  %res3 = add <8 x i16> %res0, %res1
+  %res4 = add <8 x i16> %res3, %res2
+  ret <8 x i16> %res4
+}
+
+declare void @llvm.x86.avx512.mask.pmov.qw.mem.512(ptr %ptr, <8 x i64>, i8)
+
+define void @test_int_x86_avx512_mask_pmov_qw_mem_512(ptr %ptr, <8 x i64> %x1, i8 %x2) #0 {
+; CHECK-LABEL: @test_int_x86_avx512_mask_pmov_qw_mem_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 72) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <8 x i64> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP5:%.*]], label [[TMP6:%.*]], !prof [[PROF1]]
+; CHECK:       5:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       6:
+; CHECK-NEXT:    call void @llvm.x86.avx512.mask.pmov.qw.mem.512(ptr [[PTR:%.*]], <8 x i64> [[X1:%.*]], i8 -1)
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <8 x i64> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP3:%.*]] = icmp ne i512 [[TMP7]], 0
+; CHECK-NEXT:    [[_MSOR4:%.*]] = or i1 [[_MSCMP2]], [[_MSCMP3]]
+; CHECK-NEXT:    [[_MSCMP5:%.*]] = icmp ne i8 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSOR6:%.*]] = or i1 [[_MSOR4]], [[_MSCMP5]]
+; CHECK-NEXT:    br i1 [[_MSOR6]], label [[TMP8:%.*]], label [[TMP9:%.*]], !prof [[PROF1]]
+; CHECK:       8:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       9:
+; CHECK-NEXT:    call void @llvm.x86.avx512.mask.pmov.qw.mem.512(ptr [[PTR]], <8 x i64> [[X1]], i8 [[X2:%.*]])
+; CHECK-NEXT:    ret void
+;
+  call void @llvm.x86.avx512.mask.pmov.qw.mem.512(ptr %ptr, <8 x i64> %x1, i8 -1)
+  call void @llvm.x86.avx512.mask.pmov.qw.mem.512(ptr %ptr, <8 x i64> %x1, i8 %x2)
+  ret void
+}
+
+declare <8 x i16> @llvm.x86.avx512.mask.pmovs.qw.512(<8 x i64>, <8 x i16>, i8)
+
+define <8 x i16>@test_int_x86_avx512_mask_pmovs_qw_512(<8 x i64> %x0, <8 x i16> %x1, i8 %x2) #0 {
+; CHECK-LABEL: @test_int_x86_avx512_mask_pmovs_qw_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 80) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <8 x i64> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <8 x i16> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP6:%.*]], label [[TMP7:%.*]], !prof [[PROF1]]
+; CHECK:       6:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       7:
+; CHECK-NEXT:    [[RES0:%.*]] = call <8 x i16> @llvm.x86.avx512.mask.pmovs.qw.512(<8 x i64> [[X0:%.*]], <8 x i16> [[X1:%.*]], i8 -1)
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <8 x i64> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i512 [[TMP8]], 0
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <8 x i16> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP3:%.*]] = icmp ne i128 [[TMP9]], 0
+; CHECK-NEXT:    [[_MSOR4:%.*]] = or i1 [[_MSCMP2]], [[_MSCMP3]]
+; CHECK-NEXT:    [[_MSCMP5:%.*]] = icmp ne i8 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSOR6:%.*]] = or i1 [[_MSOR4]], [[_MSCMP5]]
+; CHECK-NEXT:    br i1 [[_MSOR6]], label [[TMP10:%.*]], label [[TMP11:%.*]], !prof [[PROF1]]
+; CHECK:       10:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       11:
+; CHECK-NEXT:    [[RES1:%.*]] = call <8 x i16> @llvm.x86.avx512.mask.pmovs.qw.512(<8 x i64> [[X0]], <8 x i16> [[X1]], i8 [[X2:%.*]])
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <8 x i64> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP7:%.*]] = icmp ne i512 [[TMP12]], 0
+; CHECK-NEXT:    [[_MSCMP8:%.*]] = icmp ne i8 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSOR9:%.*]] = or i1 [[_MSCMP7]], [[_MSCMP8]]
+; CHECK-NEXT:    br i1 [[_MSOR9]], label [[TMP13:%.*]], label [[TMP14:%.*]], !prof [[PROF1]]
+; CHECK:       13:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       14:
+; CHECK-NEXT:    [[RES2:%.*]] = call <8 x i16> @llvm.x86.avx512.mask.pmovs.qw.512(<8 x i64> [[X0]], <8 x i16> zeroinitializer, i8 [[X2]])
+; CHECK-NEXT:    [[RES3:%.*]] = add <8 x i16> [[RES0]], [[RES1]]
+; CHECK-NEXT:    [[RES4:%.*]] = add <8 x i16> [[RES3]], [[RES2]]
+; CHECK-NEXT:    store <8 x i16> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i16> [[RES4]]
+;
+  %res0 = call <8 x i16> @llvm.x86.avx512.mask.pmovs.qw.512(<8 x i64> %x0, <8 x i16> %x1, i8 -1)
+  %res1 = call <8 x i16> @llvm.x86.avx512.mask.pmovs.qw.512(<8 x i64> %x0, <8 x i16> %x1, i8 %x2)
+  %res2 = call <8 x i16> @llvm.x86.avx512.mask.pmovs.qw.512(<8 x i64> %x0, <8 x i16> zeroinitializer, i8 %x2)
+  %res3 = add <8 x i16> %res0, %res1
+  %res4 = add <8 x i16> %res3, %res2
+  ret <8 x i16> %res4
+}
+
+declare void @llvm.x86.avx512.mask.pmovs.qw.mem.512(ptr %ptr, <8 x i64>, i8)
+
+define void @test_int_x86_avx512_mask_pmovs_qw_mem_512(ptr %ptr, <8 x i64> %x1, i8 %x2) #0 {
+; CHECK-LABEL: @test_int_x86_avx512_mask_pmovs_qw_mem_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 72) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <8 x i64> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP5:%.*]], label [[TMP6:%.*]], !prof [[PROF1]]
+; CHECK:       5:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       6:
+; CHECK-NEXT:    call void @llvm.x86.avx512.mask.pmovs.qw.mem.512(ptr [[PTR:%.*]], <8 x i64> [[X1:%.*]], i8 -1)
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <8 x i64> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP3:%.*]] = icmp ne i512 [[TMP7]], 0
+; CHECK-NEXT:    [[_MSOR4:%.*]] = or i1 [[_MSCMP2]], [[_MSCMP3]]
+; CHECK-NEXT:    [[_MSCMP5:%.*]] = icmp ne i8 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSOR6:%.*]] = or i1 [[_MSOR4]], [[_MSCMP5]]
+; CHECK-NEXT:    br i1 [[_MSOR6]], label [[TMP8:%.*]], label [[TMP9:%.*]], !prof [[PROF1]]
+; CHECK:       8:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       9:
+; CHECK-NEXT:    call void @llvm.x86.avx512.mask.pmovs.qw.mem.512(ptr [[PTR]], <8 x i64> [[X1]], i8 [[X2:%.*]])
+; CHECK-NEXT:    ret void
+;
+  call void @llvm.x86.avx512.mask.pmovs.qw.mem.512(ptr %ptr, <8 x i64> %x1, i8 -1)
+  call void @llvm.x86.avx512.mask.pmovs.qw.mem.512(ptr %ptr, <8 x i64> %x1, i8 %x2)
+  ret void
+}
+
+declare <8 x i16> @llvm.x86.avx512.mask.pmovus.qw.512(<8 x i64>, <8 x i16>, i8)
+
+define <8 x i16>@test_int_x86_avx512_mask_pmovus_qw_512(<8 x i64> %x0, <8 x i16> %x1, i8 %x2) #0 {
+; CHECK-LABEL: @test_int_x86_avx512_mask_pmovus_qw_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 80) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <8 x i64> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <8 x i16> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP6:%.*]], label [[TMP7:%.*]], !prof [[PROF1]]
+; CHECK:       6:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       7:
+; CHECK-NEXT:    [[RES0:%.*]] = call <8 x i16> @llvm.x86.avx512.mask.pmovus.qw.512(<8 x i64> [[X0:%.*]], <8 x i16> [[X1:%.*]], i8 -1)
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <8 x i64> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i512 [[TMP8]], 0
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <8 x i16> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP3:%.*]] = icmp ne i128 [[TMP9]], 0
+; CHECK-NEXT:    [[_MSOR4:%.*]] = or i1 [[_MSCMP2]], [[_MSCMP3]]
+; CHECK-NEXT:    [[_MSCMP5:%.*]] = icmp ne i8 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSOR6:%.*]] = or i1 [[_MSOR4]], [[_MSCMP5]]
+; CHECK-NEXT:    br i1 [[_MSOR6]], label [[TMP10:%.*]], label [[TMP11:%.*]], !prof [[PROF1]]
+; CHECK:       10:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       11:
+; CHECK-NEXT:    [[RES1:%.*]] = call <8 x i16> @llvm.x86.avx512.mask.pmovus.qw.512(<8 x i64> [[X0]], <8 x i16> [[X1]], i8 [[X2:%.*]])
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <8 x i64> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP7:%.*]] = icmp ne i512 [[TMP12]], 0
+; CHECK-NEXT:    [[_MSCMP8:%.*]] = icmp ne i8 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSOR9:%.*]] = or i1 [[_MSCMP7]], [[_MSCMP8]]
+; CHECK-NEXT:    br i1 [[_MSOR9]], label [[TMP13:%.*]], label [[TMP14:%.*]], !prof [[PROF1]]
+; CHECK:       13:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       14:
+; CHECK-NEXT:    [[RES2:%.*]] = call <8 x i16> @llvm.x86.avx512.mask.pmovus.qw.512(<8 x i64> [[X0]], <8 x i16> zeroinitializer, i8 [[X2]])
+; CHECK-NEXT:    [[RES3:%.*]] = add <8 x i16> [[RES0]], [[RES1]]
+; CHECK-NEXT:    [[RES4:%.*]] = add <8 x i16> [[RES3]], [[RES2]]
+; CHECK-NEXT:    store <8 x i16> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i16> [[RES4]]
+;
+  %res0 = call <8 x i16> @llvm.x86.avx512.mask.pmovus.qw.512(<8 x i64> %x0, <8 x i16> %x1, i8 -1)
+  %res1 = call <8 x i16> @llvm.x86.avx512.mask.pmovus.qw.512(<8 x i64> %x0, <8 x i16> %x1, i8 %x2)
+  %res2 = call <8 x i16> @llvm.x86.avx512.mask.pmovus.qw.512(<8 x i64> %x0, <8 x i16> zeroinitializer, i8 %x2)
+  %res3 = add <8 x i16> %res0, %res1
+  %res4 = add <8 x i16> %res3, %res2
+  ret <8 x i16> %res4
+}
+
+declare void @llvm.x86.avx512.mask.pmovus.qw.mem.512(ptr %ptr, <8 x i64>, i8)
+
+define void @test_int_x86_avx512_mask_pmovus_qw_mem_512(ptr %ptr, <8 x i64> %x1, i8 %x2) #0 {
+; CHECK-LABEL: @test_int_x86_avx512_mask_pmovus_qw_mem_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 72) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <8 x i64> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP5:%.*]], label [[TMP6:%.*]], !prof [[PROF1]]
+; CHECK:       5:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       6:
+; CHECK-NEXT:    call void @llvm.x86.avx512.mask.pmovus.qw.mem.512(ptr [[PTR:%.*]], <8 x i64> [[X1:%.*]], i8 -1)
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <8 x i64> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP3:%.*]] = icmp ne i512 [[TMP7]], 0
+; CHECK-NEXT:    [[_MSOR4:%.*]] = or i1 [[_MSCMP2]], [[_MSCMP3]]
+; CHECK-NEXT:    [[_MSCMP5:%.*]] = icmp ne i8 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSOR6:%.*]] = or i1 [[_MSOR4]], [[_MSCMP5]]
+; CHECK-NEXT:    br i1 [[_MSOR6]], label [[TMP8:%.*]], label [[TMP9:%.*]], !prof [[PROF1]]
+; CHECK:       8:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       9:
+; CHECK-NEXT:    call void @llvm.x86.avx512.mask.pmovus.qw.mem.512(ptr [[PTR]], <8 x i64> [[X1]], i8 [[X2:%.*]])
+; CHECK-NEXT:    ret void
+;
+  call void @llvm.x86.avx512.mask.pmovus.qw.mem.512(ptr %ptr, <8 x i64> %x1, i8 -1)
+  call void @llvm.x86.avx512.mask.pmovus.qw.mem.512(ptr %ptr, <8 x i64> %x1, i8 %x2)
+  ret void
+}
+
+define <8 x i32>@test_int_x86_avx512_pmov_qd_512(<8 x i64> %x0, <8 x i32> %x1) #0 {
+; CHECK-LABEL: @test_int_x86_avx512_pmov_qd_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = trunc <8 x i64> [[TMP1]] to <8 x i32>
+; CHECK-NEXT:    [[TMP2:%.*]] = trunc <8 x i64> [[X0:%.*]] to <8 x i32>
+; CHECK-NEXT:    store <8 x i32> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i32> [[TMP2]]
+;
+  %1 = trunc <8 x i64> %x0 to <8 x i32>
+  ret <8 x i32> %1
+}
+
+define <8 x i32>@test_int_x86_avx512_mask_pmov_qd_512(<8 x i64> %x0, <8 x i32> %x1, i8 %x2) #0 {
+; CHECK-LABEL: @test_int_x86_avx512_mask_pmov_qd_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 96) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = trunc <8 x i64> [[TMP1]] to <8 x i32>
+; CHECK-NEXT:    [[TMP4:%.*]] = trunc <8 x i64> [[X0:%.*]] to <8 x i32>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i8 [[TMP2]] to <8 x i1>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i8 [[X2:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP7:%.*]] = select <8 x i1> [[TMP6]], <8 x i32> [[_MSPROP]], <8 x i32> [[TMP3]]
+; CHECK-NEXT:    [[TMP8:%.*]] = xor <8 x i32> [[TMP4]], [[X1:%.*]]
+; CHECK-NEXT:    [[TMP9:%.*]] = or <8 x i32> [[TMP8]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP10:%.*]] = or <8 x i32> [[TMP9]], [[TMP3]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP5]], <8 x i32> [[TMP10]], <8 x i32> [[TMP7]]
+; CHECK-NEXT:    [[TMP11:%.*]] = select <8 x i1> [[TMP6]], <8 x i32> [[TMP4]], <8 x i32> [[X1]]
+; CHECK-NEXT:    store <8 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i32> [[TMP11]]
+;
+  %1 = trunc <8 x i64> %x0 to <8 x i32>
+  %2 = bitcast i8 %x2 to <8 x i1>
+  %3 = select <8 x i1> %2, <8 x i32> %1, <8 x i32> %x1
+  ret <8 x i32> %3
+}
+
+define <8 x i32>@test_int_x86_avx512_maskz_pmov_qd_512(<8 x i64> %x0,  i8 %x2) #0 {
+; CHECK-LABEL: @test_int_x86_avx512_maskz_pmov_qd_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = trunc <8 x i64> [[TMP1]] to <8 x i32>
+; CHECK-NEXT:    [[TMP3:%.*]] = trunc <8 x i64> [[X0:%.*]] to <8 x i32>
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast i8 [[TMP2]] to <8 x i1>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i8 [[X2:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP6:%.*]] = select <8 x i1> [[TMP5]], <8 x i32> [[_MSPROP]], <8 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP7:%.*]] = xor <8 x i32> [[TMP3]], zeroinitializer
+; CHECK-NEXT:    [[TMP8:%.*]] = or <8 x i32> [[TMP7]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP9:%.*]] = or <8 x i32> [[TMP8]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP4]], <8 x i32> [[TMP9]], <8 x i32> [[TMP6]]
+; CHECK-NEXT:    [[TMP10:%.*]] = select <8 x i1> [[TMP5]], <8 x i32> [[TMP3]], <8 x i32> zeroinitializer
+; CHECK-NEXT:    store <8 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i32> [[TMP10]]
+;
+  %1 = trunc <8 x i64> %x0 to <8 x i32>
+  %2 = bitcast i8 %x2 to <8 x i1>
+  %3 = select <8 x i1> %2, <8 x i32> %1, <8 x i32> zeroinitializer
+  ret <8 x i32> %3
+}
+
+declare void @llvm.x86.avx512.mask.pmov.qd.mem.512(ptr %ptr, <8 x i64>, i8)
+
+define void @test_int_x86_avx512_mask_pmov_qd_mem_512(ptr %ptr, <8 x i64> %x1, i8 %x2) #0 {
+; CHECK-LABEL: @test_int_x86_avx512_mask_pmov_qd_mem_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 72) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <8 x i64> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP5:%.*]], label [[TMP6:%.*]], !prof [[PROF1]]
+; CHECK:       5:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       6:
+; CHECK-NEXT:    call void @llvm.x86.avx512.mask.pmov.qd.mem.512(ptr [[PTR:%.*]], <8 x i64> [[X1:%.*]], i8 -1)
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <8 x i64> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP3:%.*]] = icmp ne i512 [[TMP7]], 0
+; CHECK-NEXT:    [[_MSOR4:%.*]] = or i1 [[_MSCMP2]], [[_MSCMP3]]
+; CHECK-NEXT:    [[_MSCMP5:%.*]] = icmp ne i8 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSOR6:%.*]] = or i1 [[_MSOR4]], [[_MSCMP5]]
+; CHECK-NEXT:    br i1 [[_MSOR6]], label [[TMP8:%.*]], label [[TMP9:%.*]], !prof [[PROF1]]
+; CHECK:       8:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       9:
+; CHECK-NEXT:    call void @llvm.x86.avx512.mask.pmov.qd.mem.512(ptr [[PTR]], <8 x i64> [[X1]], i8 [[X2:%.*]])
+; CHECK-NEXT:    ret void
+;
+  call void @llvm.x86.avx512.mask.pmov.qd.mem.512(ptr %ptr, <8 x i64> %x1, i8 -1)
+  call void @llvm.x86.avx512.mask.pmov.qd.mem.512(ptr %ptr, <8 x i64> %x1, i8 %x2)
+  ret void
+}
+
+declare <8 x i32> @llvm.x86.avx512.mask.pmovs.qd.512(<8 x i64>, <8 x i32>, i8)
+
+define <8 x i32>@test_int_x86_avx512_pmovs_qd_512(<8 x i64> %x0, <8 x i32> %x1) #0 {
+; CHECK-LABEL: @test_int_x86_avx512_pmovs_qd_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <8 x i64> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP3]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <8 x i32> [[TMP2]] to i256
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i256 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP5:%.*]], label [[TMP6:%.*]], !prof [[PROF1]]
+; CHECK:       5:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       6:
+; CHECK-NEXT:    [[RES:%.*]] = call <8 x i32> @llvm.x86.avx512.mask.pmovs.qd.512(<8 x i64> [[X0:%.*]], <8 x i32> [[X1:%.*]], i8 -1)
+; CHECK-NEXT:    store <8 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i32> [[RES]]
+;
+  %res = call <8 x i32> @llvm.x86.avx512.mask.pmovs.qd.512(<8 x i64> %x0, <8 x i32> %x1, i8 -1)
+  ret <8 x i32> %res
+}
+
+define <8 x i32>@test_int_x86_avx512_mask_pmovs_qd_512(<8 x i64> %x0, <8 x i32> %x1, i8 %x2) #0 {
+; CHECK-LABEL: @test_int_x86_avx512_mask_pmovs_qd_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 96) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <8 x i64> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <8 x i32> [[TMP2]] to i256
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i256 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i8 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR3]], label [[TMP6:%.*]], label [[TMP7:%.*]], !prof [[PROF1]]
+; CHECK:       6:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       7:
+; CHECK-NEXT:    [[RES:%.*]] = call <8 x i32> @llvm.x86.avx512.mask.pmovs.qd.512(<8 x i64> [[X0:%.*]], <8 x i32> [[X1:%.*]], i8 [[X2:%.*]])
+; CHECK-NEXT:    store <8 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i32> [[RES]]
+;
+  %res = call <8 x i32> @llvm.x86.avx512.mask.pmovs.qd.512(<8 x i64> %x0, <8 x i32> %x1, i8 %x2)
+  ret <8 x i32> %res
+}
+
+define <8 x i32>@test_int_x86_avx512_maskz_pmovs_qd_512(<8 x i64> %x0, i8 %x2) #0 {
+; CHECK-LABEL: @test_int_x86_avx512_maskz_pmovs_qd_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <8 x i64> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i8 [[TMP2]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP4:%.*]], label [[TMP5:%.*]], !prof [[PROF1]]
+; CHECK:       4:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       5:
+; CHECK-NEXT:    [[RES:%.*]] = call <8 x i32> @llvm.x86.avx512.mask.pmovs.qd.512(<8 x i64> [[X0:%.*]], <8 x i32> zeroinitializer, i8 [[X2:%.*]])
+; CHECK-NEXT:    store <8 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i32> [[RES]]
+;
+  %res = call <8 x i32> @llvm.x86.avx512.mask.pmovs.qd.512(<8 x i64> %x0, <8 x i32> zeroinitializer, i8 %x2)
+  ret <8 x i32> %res
+}
+
+declare void @llvm.x86.avx512.mask.pmovs.qd.mem.512(ptr %ptr, <8 x i64>, i8)
+
+define void @test_int_x86_avx512_mask_pmovs_qd_mem_512(ptr %ptr, <8 x i64> %x1, i8 %x2) #0 {
+; CHECK-LABEL: @test_int_x86_avx512_mask_pmovs_qd_mem_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 72) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <8 x i64> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP5:%.*]], label [[TMP6:%.*]], !prof [[PROF1]]
+; CHECK:       5:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       6:
+; CHECK-NEXT:    call void @llvm.x86.avx512.mask.pmovs.qd.mem.512(ptr [[PTR:%.*]], <8 x i64> [[X1:%.*]], i8 -1)
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <8 x i64> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP3:%.*]] = icmp ne i512 [[TMP7]], 0
+; CHECK-NEXT:    [[_MSOR4:%.*]] = or i1 [[_MSCMP2]], [[_MSCMP3]]
+; CHECK-NEXT:    [[_MSCMP5:%.*]] = icmp ne i8 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSOR6:%.*]] = or i1 [[_MSOR4]], [[_MSCMP5]]
+; CHECK-NEXT:    br i1 [[_MSOR6]], label [[TMP8:%.*]], label [[TMP9:%.*]], !prof [[PROF1]]
+; CHECK:       8:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       9:
+; CHECK-NEXT:    call void @llvm.x86.avx512.mask.pmovs.qd.mem.512(ptr [[PTR]], <8 x i64> [[X1]], i8 [[X2:%.*]])
+; CHECK-NEXT:    ret void
+;
+  call void @llvm.x86.avx512.mask.pmovs.qd.mem.512(ptr %ptr, <8 x i64> %x1, i8 -1)
+  call void @llvm.x86.avx512.mask.pmovs.qd.mem.512(ptr %ptr, <8 x i64> %x1, i8 %x2)
+  ret void
+}
+
+declare <8 x i32> @llvm.x86.avx512.mask.pmovus.qd.512(<8 x i64>, <8 x i32>, i8)
+
+define <8 x i32>@test_int_x86_avx512_pmovus_qd_512(<8 x i64> %x0, <8 x i32> %x1) #0 {
+; CHECK-LABEL: @test_int_x86_avx512_pmovus_qd_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <8 x i64> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP3]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <8 x i32> [[TMP2]] to i256
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i256 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP5:%.*]], label [[TMP6:%.*]], !prof [[PROF1]]
+; CHECK:       5:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       6:
+; CHECK-NEXT:    [[RES:%.*]] = call <8 x i32> @llvm.x86.avx512.mask.pmovus.qd.512(<8 x i64> [[X0:%.*]], <8 x i32> [[X1:%.*]], i8 -1)
+; CHECK-NEXT:    store <8 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i32> [[RES]]
+;
+  %res = call <8 x i32> @llvm.x86.avx512.mask.pmovus.qd.512(<8 x i64> %x0, <8 x i32> %x1, i8 -1)
+  ret <8 x i32> %res
+}
+
+define <8 x i32>@test_int_x86_avx512_mask_pmovus_qd_512(<8 x i64> %x0, <8 x i32> %x1, i8 %x2) #0 {
+; CHECK-LABEL: @test_int_x86_avx512_mask_pmovus_qd_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 96) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <8 x i64> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <8 x i32> [[TMP2]] to i256
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i256 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i8 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR3]], label [[TMP6:%.*]], label [[TMP7:%.*]], !prof [[PROF1]]
+; CHECK:       6:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       7:
+; CHECK-NEXT:    [[RES:%.*]] = call <8 x i32> @llvm.x86.avx512.mask.pmovus.qd.512(<8 x i64> [[X0:%.*]], <8 x i32> [[X1:%.*]], i8 [[X2:%.*]])
+; CHECK-NEXT:    store <8 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i32> [[RES]]
+;
+  %res = call <8 x i32> @llvm.x86.avx512.mask.pmovus.qd.512(<8 x i64> %x0, <8 x i32> %x1, i8 %x2)
+  ret <8 x i32> %res
+}
+
+define <8 x i32>@test_int_x86_avx512_maskz_pmovus_qd_512(<8 x i64> %x0, i8 %x2) #0 {
+; CHECK-LABEL: @test_int_x86_avx512_maskz_pmovus_qd_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <8 x i64> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i8 [[TMP2]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP4:%.*]], label [[TMP5:%.*]], !prof [[PROF1]]
+; CHECK:       4:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       5:
+; CHECK-NEXT:    [[RES:%.*]] = call <8 x i32> @llvm.x86.avx512.mask.pmovus.qd.512(<8 x i64> [[X0:%.*]], <8 x i32> zeroinitializer, i8 [[X2:%.*]])
+; CHECK-NEXT:    store <8 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i32> [[RES]]
+;
+  %res = call <8 x i32> @llvm.x86.avx512.mask.pmovus.qd.512(<8 x i64> %x0, <8 x i32> zeroinitializer, i8 %x2)
+  ret <8 x i32> %res
+}
+
+declare void @llvm.x86.avx512.mask.pmovus.qd.mem.512(ptr %ptr, <8 x i64>, i8)
+
+define void @test_int_x86_avx512_mask_pmovus_qd_mem_512(ptr %ptr, <8 x i64> %x1, i8 %x2) #0 {
+; CHECK-LABEL: @test_int_x86_avx512_mask_pmovus_qd_mem_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 72) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <8 x i64> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP5:%.*]], label [[TMP6:%.*]], !prof [[PROF1]]
+; CHECK:       5:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       6:
+; CHECK-NEXT:    call void @llvm.x86.avx512.mask.pmovus.qd.mem.512(ptr [[PTR:%.*]], <8 x i64> [[X1:%.*]], i8 -1)
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <8 x i64> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP3:%.*]] = icmp ne i512 [[TMP7]], 0
+; CHECK-NEXT:    [[_MSOR4:%.*]] = or i1 [[_MSCMP2]], [[_MSCMP3]]
+; CHECK-NEXT:    [[_MSCMP5:%.*]] = icmp ne i8 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSOR6:%.*]] = or i1 [[_MSOR4]], [[_MSCMP5]]
+; CHECK-NEXT:    br i1 [[_MSOR6]], label [[TMP8:%.*]], label [[TMP9:%.*]], !prof [[PROF1]]
+; CHECK:       8:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       9:
+; CHECK-NEXT:    call void @llvm.x86.avx512.mask.pmovus.qd.mem.512(ptr [[PTR]], <8 x i64> [[X1]], i8 [[X2:%.*]])
+; CHECK-NEXT:    ret void
+;
+  call void @llvm.x86.avx512.mask.pmovus.qd.mem.512(ptr %ptr, <8 x i64> %x1, i8 -1)
+  call void @llvm.x86.avx512.mask.pmovus.qd.mem.512(ptr %ptr, <8 x i64> %x1, i8 %x2)
+  ret void
+}
+
+declare <16 x i8> @llvm.x86.avx512.mask.pmov.db.512(<16 x i32>, <16 x i8>, i16)
+
+define <16 x i8>@test_int_x86_avx512_mask_pmov_db_512(<16 x i32> %x0, <16 x i8> %x1, i16 %x2) #0 {
+; CHECK-LABEL: @test_int_x86_avx512_mask_pmov_db_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 80) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <16 x i8> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP6:%.*]], label [[TMP7:%.*]], !prof [[PROF1]]
+; CHECK:       6:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       7:
+; CHECK-NEXT:    [[RES0:%.*]] = call <16 x i8> @llvm.x86.avx512.mask.pmov.db.512(<16 x i32> [[X0:%.*]], <16 x i8> [[X1:%.*]], i16 -1)
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i512 [[TMP8]], 0
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <16 x i8> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP3:%.*]] = icmp ne i128 [[TMP9]], 0
+; CHECK-NEXT:    [[_MSOR4:%.*]] = or i1 [[_MSCMP2]], [[_MSCMP3]]
+; CHECK-NEXT:    [[_MSCMP5:%.*]] = icmp ne i16 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSOR6:%.*]] = or i1 [[_MSOR4]], [[_MSCMP5]]
+; CHECK-NEXT:    br i1 [[_MSOR6]], label [[TMP10:%.*]], label [[TMP11:%.*]], !prof [[PROF1]]
+; CHECK:       10:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       11:
+; CHECK-NEXT:    [[RES1:%.*]] = call <16 x i8> @llvm.x86.avx512.mask.pmov.db.512(<16 x i32> [[X0]], <16 x i8> [[X1]], i16 [[X2:%.*]])
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP7:%.*]] = icmp ne i512 [[TMP12]], 0
+; CHECK-NEXT:    [[_MSCMP8:%.*]] = icmp ne i16 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSOR9:%.*]] = or i1 [[_MSCMP7]], [[_MSCMP8]]
+; CHECK-NEXT:    br i1 [[_MSOR9]], label [[TMP13:%.*]], label [[TMP14:%.*]], !prof [[PROF1]]
+; CHECK:       13:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       14:
+; CHECK-NEXT:    [[RES2:%.*]] = call <16 x i8> @llvm.x86.avx512.mask.pmov.db.512(<16 x i32> [[X0]], <16 x i8> zeroinitializer, i16 [[X2]])
+; CHECK-NEXT:    [[RES3:%.*]] = add <16 x i8> [[RES0]], [[RES1]]
+; CHECK-NEXT:    [[RES4:%.*]] = add <16 x i8> [[RES3]], [[RES2]]
+; CHECK-NEXT:    store <16 x i8> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i8> [[RES4]]
+;
+  %res0 = call <16 x i8> @llvm.x86.avx512.mask.pmov.db.512(<16 x i32> %x0, <16 x i8> %x1, i16 -1)
+  %res1 = call <16 x i8> @llvm.x86.avx512.mask.pmov.db.512(<16 x i32> %x0, <16 x i8> %x1, i16 %x2)
+  %res2 = call <16 x i8> @llvm.x86.avx512.mask.pmov.db.512(<16 x i32> %x0, <16 x i8> zeroinitializer, i16 %x2)
+  %res3 = add <16 x i8> %res0, %res1
+  %res4 = add <16 x i8> %res3, %res2
+  ret <16 x i8> %res4
+}
+
+declare void @llvm.x86.avx512.mask.pmov.db.mem.512(ptr %ptr, <16 x i32>, i16)
+
+define void @test_int_x86_avx512_mask_pmov_db_mem_512(ptr %ptr, <16 x i32> %x1, i16 %x2) #0 {
+; CHECK-LABEL: @test_int_x86_avx512_mask_pmov_db_mem_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 72) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP5:%.*]], label [[TMP6:%.*]], !prof [[PROF1]]
+; CHECK:       5:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       6:
+; CHECK-NEXT:    call void @llvm.x86.avx512.mask.pmov.db.mem.512(ptr [[PTR:%.*]], <16 x i32> [[X1:%.*]], i16 -1)
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP3:%.*]] = icmp ne i512 [[TMP7]], 0
+; CHECK-NEXT:    [[_MSOR4:%.*]] = or i1 [[_MSCMP2]], [[_MSCMP3]]
+; CHECK-NEXT:    [[_MSCMP5:%.*]] = icmp ne i16 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSOR6:%.*]] = or i1 [[_MSOR4]], [[_MSCMP5]]
+; CHECK-NEXT:    br i1 [[_MSOR6]], label [[TMP8:%.*]], label [[TMP9:%.*]], !prof [[PROF1]]
+; CHECK:       8:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       9:
+; CHECK-NEXT:    call void @llvm.x86.avx512.mask.pmov.db.mem.512(ptr [[PTR]], <16 x i32> [[X1]], i16 [[X2:%.*]])
+; CHECK-NEXT:    ret void
+;
+  call void @llvm.x86.avx512.mask.pmov.db.mem.512(ptr %ptr, <16 x i32> %x1, i16 -1)
+  call void @llvm.x86.avx512.mask.pmov.db.mem.512(ptr %ptr, <16 x i32> %x1, i16 %x2)
+  ret void
+}
+
+declare <16 x i8> @llvm.x86.avx512.mask.pmovs.db.512(<16 x i32>, <16 x i8>, i16)
+
+define <16 x i8>@test_int_x86_avx512_mask_pmovs_db_512(<16 x i32> %x0, <16 x i8> %x1, i16 %x2) #0 {
+; CHECK-LABEL: @test_int_x86_avx512_mask_pmovs_db_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 80) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <16 x i8> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP6:%.*]], label [[TMP7:%.*]], !prof [[PROF1]]
+; CHECK:       6:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       7:
+; CHECK-NEXT:    [[RES0:%.*]] = call <16 x i8> @llvm.x86.avx512.mask.pmovs.db.512(<16 x i32> [[X0:%.*]], <16 x i8> [[X1:%.*]], i16 -1)
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i512 [[TMP8]], 0
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <16 x i8> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP3:%.*]] = icmp ne i128 [[TMP9]], 0
+; CHECK-NEXT:    [[_MSOR4:%.*]] = or i1 [[_MSCMP2]], [[_MSCMP3]]
+; CHECK-NEXT:    [[_MSCMP5:%.*]] = icmp ne i16 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSOR6:%.*]] = or i1 [[_MSOR4]], [[_MSCMP5]]
+; CHECK-NEXT:    br i1 [[_MSOR6]], label [[TMP10:%.*]], label [[TMP11:%.*]], !prof [[PROF1]]
+; CHECK:       10:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       11:
+; CHECK-NEXT:    [[RES1:%.*]] = call <16 x i8> @llvm.x86.avx512.mask.pmovs.db.512(<16 x i32> [[X0]], <16 x i8> [[X1]], i16 [[X2:%.*]])
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP7:%.*]] = icmp ne i512 [[TMP12]], 0
+; CHECK-NEXT:    [[_MSCMP8:%.*]] = icmp ne i16 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSOR9:%.*]] = or i1 [[_MSCMP7]], [[_MSCMP8]]
+; CHECK-NEXT:    br i1 [[_MSOR9]], label [[TMP13:%.*]], label [[TMP14:%.*]], !prof [[PROF1]]
+; CHECK:       13:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       14:
+; CHECK-NEXT:    [[RES2:%.*]] = call <16 x i8> @llvm.x86.avx512.mask.pmovs.db.512(<16 x i32> [[X0]], <16 x i8> zeroinitializer, i16 [[X2]])
+; CHECK-NEXT:    [[RES3:%.*]] = add <16 x i8> [[RES0]], [[RES1]]
+; CHECK-NEXT:    [[RES4:%.*]] = add <16 x i8> [[RES3]], [[RES2]]
+; CHECK-NEXT:    store <16 x i8> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i8> [[RES4]]
+;
+  %res0 = call <16 x i8> @llvm.x86.avx512.mask.pmovs.db.512(<16 x i32> %x0, <16 x i8> %x1, i16 -1)
+  %res1 = call <16 x i8> @llvm.x86.avx512.mask.pmovs.db.512(<16 x i32> %x0, <16 x i8> %x1, i16 %x2)
+  %res2 = call <16 x i8> @llvm.x86.avx512.mask.pmovs.db.512(<16 x i32> %x0, <16 x i8> zeroinitializer, i16 %x2)
+  %res3 = add <16 x i8> %res0, %res1
+  %res4 = add <16 x i8> %res3, %res2
+  ret <16 x i8> %res4
+}
+
+declare void @llvm.x86.avx512.mask.pmovs.db.mem.512(ptr %ptr, <16 x i32>, i16)
+
+define void @test_int_x86_avx512_mask_pmovs_db_mem_512(ptr %ptr, <16 x i32> %x1, i16 %x2) #0 {
+; CHECK-LABEL: @test_int_x86_avx512_mask_pmovs_db_mem_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 72) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP5:%.*]], label [[TMP6:%.*]], !prof [[PROF1]]
+; CHECK:       5:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       6:
+; CHECK-NEXT:    call void @llvm.x86.avx512.mask.pmovs.db.mem.512(ptr [[PTR:%.*]], <16 x i32> [[X1:%.*]], i16 -1)
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP3:%.*]] = icmp ne i512 [[TMP7]], 0
+; CHECK-NEXT:    [[_MSOR4:%.*]] = or i1 [[_MSCMP2]], [[_MSCMP3]]
+; CHECK-NEXT:    [[_MSCMP5:%.*]] = icmp ne i16 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSOR6:%.*]] = or i1 [[_MSOR4]], [[_MSCMP5]]
+; CHECK-NEXT:    br i1 [[_MSOR6]], label [[TMP8:%.*]], label [[TMP9:%.*]], !prof [[PROF1]]
+; CHECK:       8:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       9:
+; CHECK-NEXT:    call void @llvm.x86.avx512.mask.pmovs.db.mem.512(ptr [[PTR]], <16 x i32> [[X1]], i16 [[X2:%.*]])
+; CHECK-NEXT:    ret void
+;
+  call void @llvm.x86.avx512.mask.pmovs.db.mem.512(ptr %ptr, <16 x i32> %x1, i16 -1)
+  call void @llvm.x86.avx512.mask.pmovs.db.mem.512(ptr %ptr, <16 x i32> %x1, i16 %x2)
+  ret void
+}
+
+declare <16 x i8> @llvm.x86.avx512.mask.pmovus.db.512(<16 x i32>, <16 x i8>, i16)
+
+define <16 x i8>@test_int_x86_avx512_mask_pmovus_db_512(<16 x i32> %x0, <16 x i8> %x1, i16 %x2) #0 {
+; CHECK-LABEL: @test_int_x86_avx512_mask_pmovus_db_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 80) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <16 x i8> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP6:%.*]], label [[TMP7:%.*]], !prof [[PROF1]]
+; CHECK:       6:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       7:
+; CHECK-NEXT:    [[RES0:%.*]] = call <16 x i8> @llvm.x86.avx512.mask.pmovus.db.512(<16 x i32> [[X0:%.*]], <16 x i8> [[X1:%.*]], i16 -1)
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i512 [[TMP8]], 0
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <16 x i8> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP3:%.*]] = icmp ne i128 [[TMP9]], 0
+; CHECK-NEXT:    [[_MSOR4:%.*]] = or i1 [[_MSCMP2]], [[_MSCMP3]]
+; CHECK-NEXT:    [[_MSCMP5:%.*]] = icmp ne i16 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSOR6:%.*]] = or i1 [[_MSOR4]], [[_MSCMP5]]
+; CHECK-NEXT:    br i1 [[_MSOR6]], label [[TMP10:%.*]], label [[TMP11:%.*]], !prof [[PROF1]]
+; CHECK:       10:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       11:
+; CHECK-NEXT:    [[RES1:%.*]] = call <16 x i8> @llvm.x86.avx512.mask.pmovus.db.512(<16 x i32> [[X0]], <16 x i8> [[X1]], i16 [[X2:%.*]])
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP7:%.*]] = icmp ne i512 [[TMP12]], 0
+; CHECK-NEXT:    [[_MSCMP8:%.*]] = icmp ne i16 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSOR9:%.*]] = or i1 [[_MSCMP7]], [[_MSCMP8]]
+; CHECK-NEXT:    br i1 [[_MSOR9]], label [[TMP13:%.*]], label [[TMP14:%.*]], !prof [[PROF1]]
+; CHECK:       13:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       14:
+; CHECK-NEXT:    [[RES2:%.*]] = call <16 x i8> @llvm.x86.avx512.mask.pmovus.db.512(<16 x i32> [[X0]], <16 x i8> zeroinitializer, i16 [[X2]])
+; CHECK-NEXT:    [[RES3:%.*]] = add <16 x i8> [[RES0]], [[RES1]]
+; CHECK-NEXT:    [[RES4:%.*]] = add <16 x i8> [[RES3]], [[RES2]]
+; CHECK-NEXT:    store <16 x i8> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i8> [[RES4]]
+;
+  %res0 = call <16 x i8> @llvm.x86.avx512.mask.pmovus.db.512(<16 x i32> %x0, <16 x i8> %x1, i16 -1)
+  %res1 = call <16 x i8> @llvm.x86.avx512.mask.pmovus.db.512(<16 x i32> %x0, <16 x i8> %x1, i16 %x2)
+  %res2 = call <16 x i8> @llvm.x86.avx512.mask.pmovus.db.512(<16 x i32> %x0, <16 x i8> zeroinitializer, i16 %x2)
+  %res3 = add <16 x i8> %res0, %res1
+  %res4 = add <16 x i8> %res3, %res2
+  ret <16 x i8> %res4
+}
+
+declare void @llvm.x86.avx512.mask.pmovus.db.mem.512(ptr %ptr, <16 x i32>, i16)
+
+define void @test_int_x86_avx512_mask_pmovus_db_mem_512(ptr %ptr, <16 x i32> %x1, i16 %x2) #0 {
+; CHECK-LABEL: @test_int_x86_avx512_mask_pmovus_db_mem_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 72) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP5:%.*]], label [[TMP6:%.*]], !prof [[PROF1]]
+; CHECK:       5:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       6:
+; CHECK-NEXT:    call void @llvm.x86.avx512.mask.pmovus.db.mem.512(ptr [[PTR:%.*]], <16 x i32> [[X1:%.*]], i16 -1)
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP3:%.*]] = icmp ne i512 [[TMP7]], 0
+; CHECK-NEXT:    [[_MSOR4:%.*]] = or i1 [[_MSCMP2]], [[_MSCMP3]]
+; CHECK-NEXT:    [[_MSCMP5:%.*]] = icmp ne i16 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSOR6:%.*]] = or i1 [[_MSOR4]], [[_MSCMP5]]
+; CHECK-NEXT:    br i1 [[_MSOR6]], label [[TMP8:%.*]], label [[TMP9:%.*]], !prof [[PROF1]]
+; CHECK:       8:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       9:
+; CHECK-NEXT:    call void @llvm.x86.avx512.mask.pmovus.db.mem.512(ptr [[PTR]], <16 x i32> [[X1]], i16 [[X2:%.*]])
+; CHECK-NEXT:    ret void
+;
+  call void @llvm.x86.avx512.mask.pmovus.db.mem.512(ptr %ptr, <16 x i32> %x1, i16 -1)
+  call void @llvm.x86.avx512.mask.pmovus.db.mem.512(ptr %ptr, <16 x i32> %x1, i16 %x2)
+  ret void
+}
+
+declare <16 x i16> @llvm.x86.avx512.mask.pmov.dw.512(<16 x i32>, <16 x i16>, i16)
+
+define <16 x i16>@test_int_x86_avx512_mask_pmov_dw_512(<16 x i32> %x0, <16 x i16> %x1, i16 %x2) #0 {
+; CHECK-LABEL: @test_int_x86_avx512_mask_pmov_dw_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 96) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <16 x i16> [[TMP2]] to i256
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i256 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP6:%.*]], label [[TMP7:%.*]], !prof [[PROF1]]
+; CHECK:       6:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       7:
+; CHECK-NEXT:    [[RES0:%.*]] = call <16 x i16> @llvm.x86.avx512.mask.pmov.dw.512(<16 x i32> [[X0:%.*]], <16 x i16> [[X1:%.*]], i16 -1)
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i512 [[TMP8]], 0
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <16 x i16> [[TMP2]] to i256
+; CHECK-NEXT:    [[_MSCMP3:%.*]] = icmp ne i256 [[TMP9]], 0
+; CHECK-NEXT:    [[_MSOR4:%.*]] = or i1 [[_MSCMP2]], [[_MSCMP3]]
+; CHECK-NEXT:    [[_MSCMP5:%.*]] = icmp ne i16 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSOR6:%.*]] = or i1 [[_MSOR4]], [[_MSCMP5]]
+; CHECK-NEXT:    br i1 [[_MSOR6]], label [[TMP10:%.*]], label [[TMP11:%.*]], !prof [[PROF1]]
+; CHECK:       10:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       11:
+; CHECK-NEXT:    [[RES1:%.*]] = call <16 x i16> @llvm.x86.avx512.mask.pmov.dw.512(<16 x i32> [[X0]], <16 x i16> [[X1]], i16 [[X2:%.*]])
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP7:%.*]] = icmp ne i512 [[TMP12]], 0
+; CHECK-NEXT:    [[_MSCMP8:%.*]] = icmp ne i16 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSOR9:%.*]] = or i1 [[_MSCMP7]], [[_MSCMP8]]
+; CHECK-NEXT:    br i1 [[_MSOR9]], label [[TMP13:%.*]], label [[TMP14:%.*]], !prof [[PROF1]]
+; CHECK:       13:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       14:
+; CHECK-NEXT:    [[RES2:%.*]] = call <16 x i16> @llvm.x86.avx512.mask.pmov.dw.512(<16 x i32> [[X0]], <16 x i16> zeroinitializer, i16 [[X2]])
+; CHECK-NEXT:    [[RES3:%.*]] = add <16 x i16> [[RES0]], [[RES1]]
+; CHECK-NEXT:    [[RES4:%.*]] = add <16 x i16> [[RES3]], [[RES2]]
+; CHECK-NEXT:    store <16 x i16> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i16> [[RES4]]
+;
+  %res0 = call <16 x i16> @llvm.x86.avx512.mask.pmov.dw.512(<16 x i32> %x0, <16 x i16> %x1, i16 -1)
+  %res1 = call <16 x i16> @llvm.x86.avx512.mask.pmov.dw.512(<16 x i32> %x0, <16 x i16> %x1, i16 %x2)
+  %res2 = call <16 x i16> @llvm.x86.avx512.mask.pmov.dw.512(<16 x i32> %x0, <16 x i16> zeroinitializer, i16 %x2)
+  %res3 = add <16 x i16> %res0, %res1
+  %res4 = add <16 x i16> %res3, %res2
+  ret <16 x i16> %res4
+}
+
+declare void @llvm.x86.avx512.mask.pmov.dw.mem.512(ptr %ptr, <16 x i32>, i16)
+
+define void @test_int_x86_avx512_mask_pmov_dw_mem_512(ptr %ptr, <16 x i32> %x1, i16 %x2) #0 {
+; CHECK-LABEL: @test_int_x86_avx512_mask_pmov_dw_mem_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 72) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP5:%.*]], label [[TMP6:%.*]], !prof [[PROF1]]
+; CHECK:       5:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       6:
+; CHECK-NEXT:    call void @llvm.x86.avx512.mask.pmov.dw.mem.512(ptr [[PTR:%.*]], <16 x i32> [[X1:%.*]], i16 -1)
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP3:%.*]] = icmp ne i512 [[TMP7]], 0
+; CHECK-NEXT:    [[_MSOR4:%.*]] = or i1 [[_MSCMP2]], [[_MSCMP3]]
+; CHECK-NEXT:    [[_MSCMP5:%.*]] = icmp ne i16 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSOR6:%.*]] = or i1 [[_MSOR4]], [[_MSCMP5]]
+; CHECK-NEXT:    br i1 [[_MSOR6]], label [[TMP8:%.*]], label [[TMP9:%.*]], !prof [[PROF1]]
+; CHECK:       8:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       9:
+; CHECK-NEXT:    call void @llvm.x86.avx512.mask.pmov.dw.mem.512(ptr [[PTR]], <16 x i32> [[X1]], i16 [[X2:%.*]])
+; CHECK-NEXT:    ret void
+;
+  call void @llvm.x86.avx512.mask.pmov.dw.mem.512(ptr %ptr, <16 x i32> %x1, i16 -1)
+  call void @llvm.x86.avx512.mask.pmov.dw.mem.512(ptr %ptr, <16 x i32> %x1, i16 %x2)
+  ret void
+}
+
+declare <16 x i16> @llvm.x86.avx512.mask.pmovs.dw.512(<16 x i32>, <16 x i16>, i16)
+
+define <16 x i16>@test_int_x86_avx512_mask_pmovs_dw_512(<16 x i32> %x0, <16 x i16> %x1, i16 %x2) #0 {
+; CHECK-LABEL: @test_int_x86_avx512_mask_pmovs_dw_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 96) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <16 x i16> [[TMP2]] to i256
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i256 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP6:%.*]], label [[TMP7:%.*]], !prof [[PROF1]]
+; CHECK:       6:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       7:
+; CHECK-NEXT:    [[RES0:%.*]] = call <16 x i16> @llvm.x86.avx512.mask.pmovs.dw.512(<16 x i32> [[X0:%.*]], <16 x i16> [[X1:%.*]], i16 -1)
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i512 [[TMP8]], 0
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <16 x i16> [[TMP2]] to i256
+; CHECK-NEXT:    [[_MSCMP3:%.*]] = icmp ne i256 [[TMP9]], 0
+; CHECK-NEXT:    [[_MSOR4:%.*]] = or i1 [[_MSCMP2]], [[_MSCMP3]]
+; CHECK-NEXT:    [[_MSCMP5:%.*]] = icmp ne i16 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSOR6:%.*]] = or i1 [[_MSOR4]], [[_MSCMP5]]
+; CHECK-NEXT:    br i1 [[_MSOR6]], label [[TMP10:%.*]], label [[TMP11:%.*]], !prof [[PROF1]]
+; CHECK:       10:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       11:
+; CHECK-NEXT:    [[RES1:%.*]] = call <16 x i16> @llvm.x86.avx512.mask.pmovs.dw.512(<16 x i32> [[X0]], <16 x i16> [[X1]], i16 [[X2:%.*]])
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP7:%.*]] = icmp ne i512 [[TMP12]], 0
+; CHECK-NEXT:    [[_MSCMP8:%.*]] = icmp ne i16 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSOR9:%.*]] = or i1 [[_MSCMP7]], [[_MSCMP8]]
+; CHECK-NEXT:    br i1 [[_MSOR9]], label [[TMP13:%.*]], label [[TMP14:%.*]], !prof [[PROF1]]
+; CHECK:       13:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       14:
+; CHECK-NEXT:    [[RES2:%.*]] = call <16 x i16> @llvm.x86.avx512.mask.pmovs.dw.512(<16 x i32> [[X0]], <16 x i16> zeroinitializer, i16 [[X2]])
+; CHECK-NEXT:    [[RES3:%.*]] = add <16 x i16> [[RES0]], [[RES1]]
+; CHECK-NEXT:    [[RES4:%.*]] = add <16 x i16> [[RES3]], [[RES2]]
+; CHECK-NEXT:    store <16 x i16> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i16> [[RES4]]
+;
+  %res0 = call <16 x i16> @llvm.x86.avx512.mask.pmovs.dw.512(<16 x i32> %x0, <16 x i16> %x1, i16 -1)
+  %res1 = call <16 x i16> @llvm.x86.avx512.mask.pmovs.dw.512(<16 x i32> %x0, <16 x i16> %x1, i16 %x2)
+  %res2 = call <16 x i16> @llvm.x86.avx512.mask.pmovs.dw.512(<16 x i32> %x0, <16 x i16> zeroinitializer, i16 %x2)
+  %res3 = add <16 x i16> %res0, %res1
+  %res4 = add <16 x i16> %res3, %res2
+  ret <16 x i16> %res4
+}
+
+declare void @llvm.x86.avx512.mask.pmovs.dw.mem.512(ptr %ptr, <16 x i32>, i16)
+
+define void @test_int_x86_avx512_mask_pmovs_dw_mem_512(ptr %ptr, <16 x i32> %x1, i16 %x2) #0 {
+; CHECK-LABEL: @test_int_x86_avx512_mask_pmovs_dw_mem_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 72) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP5:%.*]], label [[TMP6:%.*]], !prof [[PROF1]]
+; CHECK:       5:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       6:
+; CHECK-NEXT:    call void @llvm.x86.avx512.mask.pmovs.dw.mem.512(ptr [[PTR:%.*]], <16 x i32> [[X1:%.*]], i16 -1)
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP3:%.*]] = icmp ne i512 [[TMP7]], 0
+; CHECK-NEXT:    [[_MSOR4:%.*]] = or i1 [[_MSCMP2]], [[_MSCMP3]]
+; CHECK-NEXT:    [[_MSCMP5:%.*]] = icmp ne i16 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSOR6:%.*]] = or i1 [[_MSOR4]], [[_MSCMP5]]
+; CHECK-NEXT:    br i1 [[_MSOR6]], label [[TMP8:%.*]], label [[TMP9:%.*]], !prof [[PROF1]]
+; CHECK:       8:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       9:
+; CHECK-NEXT:    call void @llvm.x86.avx512.mask.pmovs.dw.mem.512(ptr [[PTR]], <16 x i32> [[X1]], i16 [[X2:%.*]])
+; CHECK-NEXT:    ret void
+;
+  call void @llvm.x86.avx512.mask.pmovs.dw.mem.512(ptr %ptr, <16 x i32> %x1, i16 -1)
+  call void @llvm.x86.avx512.mask.pmovs.dw.mem.512(ptr %ptr, <16 x i32> %x1, i16 %x2)
+  ret void
+}
+
+declare <16 x i16> @llvm.x86.avx512.mask.pmovus.dw.512(<16 x i32>, <16 x i16>, i16)
+
+define <16 x i16>@test_int_x86_avx512_mask_pmovus_dw_512(<16 x i32> %x0, <16 x i16> %x1, i16 %x2) #0 {
+; CHECK-LABEL: @test_int_x86_avx512_mask_pmovus_dw_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 96) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <16 x i16> [[TMP2]] to i256
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i256 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP6:%.*]], label [[TMP7:%.*]], !prof [[PROF1]]
+; CHECK:       6:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       7:
+; CHECK-NEXT:    [[RES0:%.*]] = call <16 x i16> @llvm.x86.avx512.mask.pmovus.dw.512(<16 x i32> [[X0:%.*]], <16 x i16> [[X1:%.*]], i16 -1)
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i512 [[TMP8]], 0
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <16 x i16> [[TMP2]] to i256
+; CHECK-NEXT:    [[_MSCMP3:%.*]] = icmp ne i256 [[TMP9]], 0
+; CHECK-NEXT:    [[_MSOR4:%.*]] = or i1 [[_MSCMP2]], [[_MSCMP3]]
+; CHECK-NEXT:    [[_MSCMP5:%.*]] = icmp ne i16 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSOR6:%.*]] = or i1 [[_MSOR4]], [[_MSCMP5]]
+; CHECK-NEXT:    br i1 [[_MSOR6]], label [[TMP10:%.*]], label [[TMP11:%.*]], !prof [[PROF1]]
+; CHECK:       10:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       11:
+; CHECK-NEXT:    [[RES1:%.*]] = call <16 x i16> @llvm.x86.avx512.mask.pmovus.dw.512(<16 x i32> [[X0]], <16 x i16> [[X1]], i16 [[X2:%.*]])
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP7:%.*]] = icmp ne i512 [[TMP12]], 0
+; CHECK-NEXT:    [[_MSCMP8:%.*]] = icmp ne i16 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSOR9:%.*]] = or i1 [[_MSCMP7]], [[_MSCMP8]]
+; CHECK-NEXT:    br i1 [[_MSOR9]], label [[TMP13:%.*]], label [[TMP14:%.*]], !prof [[PROF1]]
+; CHECK:       13:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       14:
+; CHECK-NEXT:    [[RES2:%.*]] = call <16 x i16> @llvm.x86.avx512.mask.pmovus.dw.512(<16 x i32> [[X0]], <16 x i16> zeroinitializer, i16 [[X2]])
+; CHECK-NEXT:    [[RES3:%.*]] = add <16 x i16> [[RES0]], [[RES1]]
+; CHECK-NEXT:    [[RES4:%.*]] = add <16 x i16> [[RES3]], [[RES2]]
+; CHECK-NEXT:    store <16 x i16> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i16> [[RES4]]
+;
+  %res0 = call <16 x i16> @llvm.x86.avx512.mask.pmovus.dw.512(<16 x i32> %x0, <16 x i16> %x1, i16 -1)
+  %res1 = call <16 x i16> @llvm.x86.avx512.mask.pmovus.dw.512(<16 x i32> %x0, <16 x i16> %x1, i16 %x2)
+  %res2 = call <16 x i16> @llvm.x86.avx512.mask.pmovus.dw.512(<16 x i32> %x0, <16 x i16> zeroinitializer, i16 %x2)
+  %res3 = add <16 x i16> %res0, %res1
+  %res4 = add <16 x i16> %res3, %res2
+  ret <16 x i16> %res4
+}
+
+declare void @llvm.x86.avx512.mask.pmovus.dw.mem.512(ptr %ptr, <16 x i32>, i16)
+
+define void @test_int_x86_avx512_mask_pmovus_dw_mem_512(ptr %ptr, <16 x i32> %x1, i16 %x2) #0 {
+; CHECK-LABEL: @test_int_x86_avx512_mask_pmovus_dw_mem_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 72) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP5:%.*]], label [[TMP6:%.*]], !prof [[PROF1]]
+; CHECK:       5:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       6:
+; CHECK-NEXT:    call void @llvm.x86.avx512.mask.pmovus.dw.mem.512(ptr [[PTR:%.*]], <16 x i32> [[X1:%.*]], i16 -1)
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP3:%.*]] = icmp ne i512 [[TMP7]], 0
+; CHECK-NEXT:    [[_MSOR4:%.*]] = or i1 [[_MSCMP2]], [[_MSCMP3]]
+; CHECK-NEXT:    [[_MSCMP5:%.*]] = icmp ne i16 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSOR6:%.*]] = or i1 [[_MSOR4]], [[_MSCMP5]]
+; CHECK-NEXT:    br i1 [[_MSOR6]], label [[TMP8:%.*]], label [[TMP9:%.*]], !prof [[PROF1]]
+; CHECK:       8:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       9:
+; CHECK-NEXT:    call void @llvm.x86.avx512.mask.pmovus.dw.mem.512(ptr [[PTR]], <16 x i32> [[X1]], i16 [[X2:%.*]])
+; CHECK-NEXT:    ret void
+;
+  call void @llvm.x86.avx512.mask.pmovus.dw.mem.512(ptr %ptr, <16 x i32> %x1, i16 -1)
+  call void @llvm.x86.avx512.mask.pmovus.dw.mem.512(ptr %ptr, <16 x i32> %x1, i16 %x2)
+  ret void
+}
+
+declare <16 x float> @llvm.x86.avx512.sitofp.round.v16f32.v16i32(<16 x i32>, i32)
+
+define <16 x float>@test_int_x86_avx512_mask_cvt_dq2ps_512(<16 x i32> %x0, <16 x float> %x1, i16 %x2) #0 {
+; CHECK-LABEL: @test_int_x86_avx512_mask_cvt_dq2ps_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[CVT:%.*]] = sitofp <16 x i32> [[X0:%.*]] to <16 x float>
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast i16 [[TMP2]] to <16 x i1>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i16 [[X2:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP6:%.*]] = select <16 x i1> [[TMP5]], <16 x i32> [[TMP1]], <16 x i32> [[TMP3]]
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <16 x float> [[CVT]] to <16 x i32>
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <16 x float> [[X1:%.*]] to <16 x i32>
+; CHECK-NEXT:    [[TMP9:%.*]] = xor <16 x i32> [[TMP7]], [[TMP8]]
+; CHECK-NEXT:    [[TMP10:%.*]] = or <16 x i32> [[TMP9]], [[TMP1]]
+; CHECK-NEXT:    [[TMP11:%.*]] = or <16 x i32> [[TMP10]], [[TMP3]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP4]], <16 x i32> [[TMP11]], <16 x i32> [[TMP6]]
+; CHECK-NEXT:    [[TMP12:%.*]] = select <16 x i1> [[TMP5]], <16 x float> [[CVT]], <16 x float> [[X1]]
+; CHECK-NEXT:    [[TMP13:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP13]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP14:%.*]], label [[TMP15:%.*]], !prof [[PROF1]]
+; CHECK:       14:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       15:
+; CHECK-NEXT:    [[TMP16:%.*]] = call <16 x float> @llvm.x86.avx512.sitofp.round.v16f32.v16i32(<16 x i32> [[X0]], i32 8)
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <16 x i32> [[_MSPROP_SELECT]], zeroinitializer
+; CHECK-NEXT:    [[RES2:%.*]] = fadd <16 x float> [[TMP12]], [[TMP16]]
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x float> [[RES2]]
+;
+  %cvt = sitofp <16 x i32> %x0 to <16 x float>
+  %1 = bitcast i16 %x2 to <16 x i1>
+  %2 = select <16 x i1> %1, <16 x float> %cvt, <16 x float> %x1
+  %3 = call <16 x float> @llvm.x86.avx512.sitofp.round.v16f32.v16i32(<16 x i32> %x0, i32 8)
+  %res2 = fadd <16 x float> %2, %3
+  ret <16 x float> %res2
+}
+
+declare <8 x i32> @llvm.x86.avx512.mask.cvtpd2dq.512(<8 x double>, <8 x i32>, i8, i32)
+
+define <8 x i32>@test_int_x86_avx512_mask_cvt_pd2dq_512(<8 x double> %x0, <8 x i32> %x1, i8 %x2) #0 {
+; CHECK-LABEL: @test_int_x86_avx512_mask_cvt_pd2dq_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 96) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <8 x i64> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <8 x i32> [[TMP2]] to i256
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i256 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i8 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR3]], label [[TMP6:%.*]], label [[TMP7:%.*]], !prof [[PROF1]]
+; CHECK:       6:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       7:
+; CHECK-NEXT:    [[RES:%.*]] = call <8 x i32> @llvm.x86.avx512.mask.cvtpd2dq.512(<8 x double> [[X0:%.*]], <8 x i32> [[X1:%.*]], i8 [[X2:%.*]], i32 4)
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <8 x i64> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP4:%.*]] = icmp ne i512 [[TMP8]], 0
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <8 x i32> [[TMP2]] to i256
+; CHECK-NEXT:    [[_MSCMP5:%.*]] = icmp ne i256 [[TMP9]], 0
+; CHECK-NEXT:    [[_MSOR6:%.*]] = or i1 [[_MSCMP4]], [[_MSCMP5]]
+; CHECK-NEXT:    br i1 [[_MSOR6]], label [[TMP10:%.*]], label [[TMP11:%.*]], !prof [[PROF1]]
+; CHECK:       10:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       11:
+; CHECK-NEXT:    [[RES1:%.*]] = call <8 x i32> @llvm.x86.avx512.mask.cvtpd2dq.512(<8 x double> [[X0]], <8 x i32> [[X1]], i8 -1, i32 8)
+; CHECK-NEXT:    [[RES2:%.*]] = add <8 x i32> [[RES]], [[RES1]]
+; CHECK-NEXT:    store <8 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i32> [[RES2]]
+;
+  %res = call <8 x i32> @llvm.x86.avx512.mask.cvtpd2dq.512(<8 x double> %x0, <8 x i32> %x1, i8 %x2, i32 4)
+  %res1 = call <8 x i32> @llvm.x86.avx512.mask.cvtpd2dq.512(<8 x double> %x0, <8 x i32> %x1, i8 -1, i32 8)
+  %res2 = add <8 x i32> %res, %res1
+  ret <8 x i32> %res2
+}
+
+declare <8 x float> @llvm.x86.avx512.mask.cvtpd2ps.512(<8 x double>, <8 x float>, i8, i32)
+
+define <8 x float>@test_int_x86_avx512_mask_cvt_pd2ps_512(<8 x double> %x0, <8 x float> %x1, i8 %x2) #0 {
+; CHECK-LABEL: @test_int_x86_avx512_mask_cvt_pd2ps_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 96) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <8 x i64> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <8 x i32> [[TMP2]] to i256
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i256 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i8 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR3]], label [[TMP6:%.*]], label [[TMP7:%.*]], !prof [[PROF1]]
+; CHECK:       6:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       7:
+; CHECK-NEXT:    [[RES:%.*]] = call <8 x float> @llvm.x86.avx512.mask.cvtpd2ps.512(<8 x double> [[X0:%.*]], <8 x float> [[X1:%.*]], i8 [[X2:%.*]], i32 4)
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <8 x i64> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP4:%.*]] = icmp ne i512 [[TMP8]], 0
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <8 x i32> [[TMP2]] to i256
+; CHECK-NEXT:    [[_MSCMP5:%.*]] = icmp ne i256 [[TMP9]], 0
+; CHECK-NEXT:    [[_MSOR6:%.*]] = or i1 [[_MSCMP4]], [[_MSCMP5]]
+; CHECK-NEXT:    br i1 [[_MSOR6]], label [[TMP10:%.*]], label [[TMP11:%.*]], !prof [[PROF1]]
+; CHECK:       10:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       11:
+; CHECK-NEXT:    [[RES1:%.*]] = call <8 x float> @llvm.x86.avx512.mask.cvtpd2ps.512(<8 x double> [[X0]], <8 x float> [[X1]], i8 -1, i32 10)
+; CHECK-NEXT:    [[RES2:%.*]] = fadd <8 x float> [[RES]], [[RES1]]
+; CHECK-NEXT:    store <8 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x float> [[RES2]]
+;
+  %res = call <8 x float> @llvm.x86.avx512.mask.cvtpd2ps.512(<8 x double> %x0, <8 x float> %x1, i8 %x2, i32 4)
+  %res1 = call <8 x float> @llvm.x86.avx512.mask.cvtpd2ps.512(<8 x double> %x0, <8 x float> %x1, i8 -1, i32 10)
+  %res2 = fadd <8 x float> %res, %res1
+  ret <8 x float> %res2
+}
+
+declare <8 x i32> @llvm.x86.avx512.mask.cvtpd2udq.512(<8 x double>, <8 x i32>, i8, i32)
+
+define <8 x i32>@test_int_x86_avx512_mask_cvt_pd2udq_512(<8 x double> %x0, <8 x i32> %x1, i8 %x2) #0 {
+; CHECK-LABEL: @test_int_x86_avx512_mask_cvt_pd2udq_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 96) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <8 x i64> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <8 x i32> [[TMP2]] to i256
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i256 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i8 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR3]], label [[TMP6:%.*]], label [[TMP7:%.*]], !prof [[PROF1]]
+; CHECK:       6:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       7:
+; CHECK-NEXT:    [[RES:%.*]] = call <8 x i32> @llvm.x86.avx512.mask.cvtpd2udq.512(<8 x double> [[X0:%.*]], <8 x i32> [[X1:%.*]], i8 [[X2:%.*]], i32 10)
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <8 x i64> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP4:%.*]] = icmp ne i512 [[TMP8]], 0
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <8 x i32> [[TMP2]] to i256
+; CHECK-NEXT:    [[_MSCMP5:%.*]] = icmp ne i256 [[TMP9]], 0
+; CHECK-NEXT:    [[_MSOR6:%.*]] = or i1 [[_MSCMP4]], [[_MSCMP5]]
+; CHECK-NEXT:    br i1 [[_MSOR6]], label [[TMP10:%.*]], label [[TMP11:%.*]], !prof [[PROF1]]
+; CHECK:       10:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       11:
+; CHECK-NEXT:    [[RES1:%.*]] = call <8 x i32> @llvm.x86.avx512.mask.cvtpd2udq.512(<8 x double> [[X0]], <8 x i32> [[X1]], i8 -1, i32 8)
+; CHECK-NEXT:    [[RES2:%.*]] = add <8 x i32> [[RES]], [[RES1]]
+; CHECK-NEXT:    store <8 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i32> [[RES2]]
+;
+  %res = call <8 x i32> @llvm.x86.avx512.mask.cvtpd2udq.512(<8 x double> %x0, <8 x i32> %x1, i8 %x2, i32 10)
+  %res1 = call <8 x i32> @llvm.x86.avx512.mask.cvtpd2udq.512(<8 x double> %x0, <8 x i32> %x1, i8 -1, i32 8)
+  %res2 = add <8 x i32> %res, %res1
+  ret <8 x i32> %res2
+}
+
+declare <16 x i32> @llvm.x86.avx512.mask.cvtps2dq.512(<16 x float>, <16 x i32>, i16, i32)
+
+define <16 x i32>@test_int_x86_avx512_mask_cvt_ps2dq_512(<16 x float> %x0, <16 x i32> %x1, i16 %x2) #0 {
+; CHECK-LABEL: @test_int_x86_avx512_mask_cvt_ps2dq_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i16 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR3]], label [[TMP6:%.*]], label [[TMP7:%.*]], !prof [[PROF1]]
+; CHECK:       6:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       7:
+; CHECK-NEXT:    [[RES:%.*]] = call <16 x i32> @llvm.x86.avx512.mask.cvtps2dq.512(<16 x float> [[X0:%.*]], <16 x i32> [[X1:%.*]], i16 [[X2:%.*]], i32 10)
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP4:%.*]] = icmp ne i512 [[TMP8]], 0
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP5:%.*]] = icmp ne i512 [[TMP9]], 0
+; CHECK-NEXT:    [[_MSOR6:%.*]] = or i1 [[_MSCMP4]], [[_MSCMP5]]
+; CHECK-NEXT:    br i1 [[_MSOR6]], label [[TMP10:%.*]], label [[TMP11:%.*]], !prof [[PROF1]]
+; CHECK:       10:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       11:
+; CHECK-NEXT:    [[RES1:%.*]] = call <16 x i32> @llvm.x86.avx512.mask.cvtps2dq.512(<16 x float> [[X0]], <16 x i32> [[X1]], i16 -1, i32 8)
+; CHECK-NEXT:    [[RES2:%.*]] = add <16 x i32> [[RES]], [[RES1]]
+; CHECK-NEXT:    store <16 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i32> [[RES2]]
+;
+  %res = call <16 x i32> @llvm.x86.avx512.mask.cvtps2dq.512(<16 x float> %x0, <16 x i32> %x1, i16 %x2, i32 10)
+  %res1 = call <16 x i32> @llvm.x86.avx512.mask.cvtps2dq.512(<16 x float> %x0, <16 x i32> %x1, i16 -1, i32 8)
+  %res2 = add <16 x i32> %res, %res1
+  ret <16 x i32> %res2
+}
+
+declare <8 x double> @llvm.x86.avx512.mask.cvtps2pd.512(<8 x float>, <8 x double>, i8, i32)
+
+define <8 x double>@test_int_x86_avx512_mask_cvt_ps2pd_512(<8 x float> %x0, <8 x double> %x1, i8 %x2) #0 {
+; CHECK-LABEL: @test_int_x86_avx512_mask_cvt_ps2pd_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 96) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <8 x i32> [[TMP1]] to i256
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i256 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <8 x i64> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i8 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR3]], label [[TMP6:%.*]], label [[TMP7:%.*]], !prof [[PROF1]]
+; CHECK:       6:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       7:
+; CHECK-NEXT:    [[RES:%.*]] = call <8 x double> @llvm.x86.avx512.mask.cvtps2pd.512(<8 x float> [[X0:%.*]], <8 x double> [[X1:%.*]], i8 [[X2:%.*]], i32 4)
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <8 x i32> [[TMP1]] to i256
+; CHECK-NEXT:    [[_MSCMP4:%.*]] = icmp ne i256 [[TMP8]], 0
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <8 x i64> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP5:%.*]] = icmp ne i512 [[TMP9]], 0
+; CHECK-NEXT:    [[_MSOR6:%.*]] = or i1 [[_MSCMP4]], [[_MSCMP5]]
+; CHECK-NEXT:    br i1 [[_MSOR6]], label [[TMP10:%.*]], label [[TMP11:%.*]], !prof [[PROF1]]
+; CHECK:       10:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       11:
+; CHECK-NEXT:    [[RES1:%.*]] = call <8 x double> @llvm.x86.avx512.mask.cvtps2pd.512(<8 x float> [[X0]], <8 x double> [[X1]], i8 -1, i32 8)
+; CHECK-NEXT:    [[RES2:%.*]] = fadd <8 x double> [[RES]], [[RES1]]
+; CHECK-NEXT:    store <8 x i64> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x double> [[RES2]]
+;
+  %res = call <8 x double> @llvm.x86.avx512.mask.cvtps2pd.512(<8 x float> %x0, <8 x double> %x1, i8 %x2, i32 4)
+  %res1 = call <8 x double> @llvm.x86.avx512.mask.cvtps2pd.512(<8 x float> %x0, <8 x double> %x1, i8 -1, i32 8)
+  %res2 = fadd <8 x double> %res, %res1
+  ret <8 x double> %res2
+}
+
+declare <16 x i32> @llvm.x86.avx512.mask.cvtps2udq.512(<16 x float>, <16 x i32>, i16, i32)
+
+define <16 x i32>@test_int_x86_avx512_mask_cvt_ps2udq_512(<16 x float> %x0, <16 x i32> %x1, i16 %x2) #0 {
+; CHECK-LABEL: @test_int_x86_avx512_mask_cvt_ps2udq_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i16 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR3]], label [[TMP6:%.*]], label [[TMP7:%.*]], !prof [[PROF1]]
+; CHECK:       6:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       7:
+; CHECK-NEXT:    [[RES:%.*]] = call <16 x i32> @llvm.x86.avx512.mask.cvtps2udq.512(<16 x float> [[X0:%.*]], <16 x i32> [[X1:%.*]], i16 [[X2:%.*]], i32 10)
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP4:%.*]] = icmp ne i512 [[TMP8]], 0
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP5:%.*]] = icmp ne i512 [[TMP9]], 0
+; CHECK-NEXT:    [[_MSOR6:%.*]] = or i1 [[_MSCMP4]], [[_MSCMP5]]
+; CHECK-NEXT:    br i1 [[_MSOR6]], label [[TMP10:%.*]], label [[TMP11:%.*]], !prof [[PROF1]]
+; CHECK:       10:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       11:
+; CHECK-NEXT:    [[RES1:%.*]] = call <16 x i32> @llvm.x86.avx512.mask.cvtps2udq.512(<16 x float> [[X0]], <16 x i32> [[X1]], i16 -1, i32 8)
+; CHECK-NEXT:    [[RES2:%.*]] = add <16 x i32> [[RES]], [[RES1]]
+; CHECK-NEXT:    store <16 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i32> [[RES2]]
+;
+  %res = call <16 x i32> @llvm.x86.avx512.mask.cvtps2udq.512(<16 x float> %x0, <16 x i32> %x1, i16 %x2, i32 10)
+  %res1 = call <16 x i32> @llvm.x86.avx512.mask.cvtps2udq.512(<16 x float> %x0, <16 x i32> %x1, i16 -1, i32 8)
+  %res2 = add <16 x i32> %res, %res1
+  ret <16 x i32> %res2
+}
+
+declare <8 x i32> @llvm.x86.avx512.mask.cvttpd2dq.512(<8 x double>, <8 x i32>, i8, i32)
+
+define <8 x i32>@test_int_x86_avx512_mask_cvtt_pd2dq_512(<8 x double> %x0, <8 x i32> %x1, i8 %x2) #0 {
+; CHECK-LABEL: @test_int_x86_avx512_mask_cvtt_pd2dq_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 96) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <8 x i64> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <8 x i32> [[TMP2]] to i256
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i256 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i8 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR3]], label [[TMP6:%.*]], label [[TMP7:%.*]], !prof [[PROF1]]
+; CHECK:       6:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       7:
+; CHECK-NEXT:    [[RES:%.*]] = call <8 x i32> @llvm.x86.avx512.mask.cvttpd2dq.512(<8 x double> [[X0:%.*]], <8 x i32> [[X1:%.*]], i8 [[X2:%.*]], i32 4)
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <8 x i64> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP4:%.*]] = icmp ne i512 [[TMP8]], 0
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <8 x i32> [[TMP2]] to i256
+; CHECK-NEXT:    [[_MSCMP5:%.*]] = icmp ne i256 [[TMP9]], 0
+; CHECK-NEXT:    [[_MSOR6:%.*]] = or i1 [[_MSCMP4]], [[_MSCMP5]]
+; CHECK-NEXT:    br i1 [[_MSOR6]], label [[TMP10:%.*]], label [[TMP11:%.*]], !prof [[PROF1]]
+; CHECK:       10:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       11:
+; CHECK-NEXT:    [[RES1:%.*]] = call <8 x i32> @llvm.x86.avx512.mask.cvttpd2dq.512(<8 x double> [[X0]], <8 x i32> [[X1]], i8 -1, i32 8)
+; CHECK-NEXT:    [[RES2:%.*]] = add <8 x i32> [[RES]], [[RES1]]
+; CHECK-NEXT:    store <8 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i32> [[RES2]]
+;
+  %res = call <8 x i32> @llvm.x86.avx512.mask.cvttpd2dq.512(<8 x double> %x0, <8 x i32> %x1, i8 %x2, i32 4)
+  %res1 = call <8 x i32> @llvm.x86.avx512.mask.cvttpd2dq.512(<8 x double> %x0, <8 x i32> %x1, i8 -1, i32 8)
+  %res2 = add <8 x i32> %res, %res1
+  ret <8 x i32> %res2
+}
+
+declare <16 x float> @llvm.x86.avx512.uitofp.round.v16f32.v16i32(<16 x i32>, i32)
+
+define <16 x float>@test_int_x86_avx512_mask_cvt_udq2ps_512(<16 x i32> %x0, <16 x float> %x1, i16 %x2) #0 {
+; CHECK-LABEL: @test_int_x86_avx512_mask_cvt_udq2ps_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[CVT:%.*]] = uitofp <16 x i32> [[X0:%.*]] to <16 x float>
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast i16 [[TMP2]] to <16 x i1>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i16 [[X2:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP6:%.*]] = select <16 x i1> [[TMP5]], <16 x i32> [[TMP1]], <16 x i32> [[TMP3]]
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <16 x float> [[CVT]] to <16 x i32>
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <16 x float> [[X1:%.*]] to <16 x i32>
+; CHECK-NEXT:    [[TMP9:%.*]] = xor <16 x i32> [[TMP7]], [[TMP8]]
+; CHECK-NEXT:    [[TMP10:%.*]] = or <16 x i32> [[TMP9]], [[TMP1]]
+; CHECK-NEXT:    [[TMP11:%.*]] = or <16 x i32> [[TMP10]], [[TMP3]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP4]], <16 x i32> [[TMP11]], <16 x i32> [[TMP6]]
+; CHECK-NEXT:    [[TMP12:%.*]] = select <16 x i1> [[TMP5]], <16 x float> [[CVT]], <16 x float> [[X1]]
+; CHECK-NEXT:    [[TMP13:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP13]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP14:%.*]], label [[TMP15:%.*]], !prof [[PROF1]]
+; CHECK:       14:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       15:
+; CHECK-NEXT:    [[TMP16:%.*]] = call <16 x float> @llvm.x86.avx512.uitofp.round.v16f32.v16i32(<16 x i32> [[X0]], i32 8)
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <16 x i32> [[_MSPROP_SELECT]], zeroinitializer
+; CHECK-NEXT:    [[RES2:%.*]] = fadd <16 x float> [[TMP12]], [[TMP16]]
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x float> [[RES2]]
+;
+  %cvt = uitofp <16 x i32> %x0 to <16 x float>
+  %1 = bitcast i16 %x2 to <16 x i1>
+  %2 = select <16 x i1> %1, <16 x float> %cvt, <16 x float> %x1
+  %3 = call <16 x float> @llvm.x86.avx512.uitofp.round.v16f32.v16i32(<16 x i32> %x0, i32 8)
+  %res2 = fadd <16 x float> %2, %3
+  ret <16 x float> %res2
+}
+
+declare <8 x i32> @llvm.x86.avx512.mask.cvttpd2udq.512(<8 x double>, <8 x i32>, i8, i32)
+
+define <8 x i32>@test_int_x86_avx512_mask_cvtt_pd2udq_512(<8 x double> %x0, <8 x i32> %x1, i8 %x2) #0 {
+; CHECK-LABEL: @test_int_x86_avx512_mask_cvtt_pd2udq_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 96) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <8 x i64> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <8 x i32> [[TMP2]] to i256
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i256 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i8 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR3]], label [[TMP6:%.*]], label [[TMP7:%.*]], !prof [[PROF1]]
+; CHECK:       6:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       7:
+; CHECK-NEXT:    [[RES:%.*]] = call <8 x i32> @llvm.x86.avx512.mask.cvttpd2udq.512(<8 x double> [[X0:%.*]], <8 x i32> [[X1:%.*]], i8 [[X2:%.*]], i32 4)
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <8 x i64> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP4:%.*]] = icmp ne i512 [[TMP8]], 0
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <8 x i32> [[TMP2]] to i256
+; CHECK-NEXT:    [[_MSCMP5:%.*]] = icmp ne i256 [[TMP9]], 0
+; CHECK-NEXT:    [[_MSOR6:%.*]] = or i1 [[_MSCMP4]], [[_MSCMP5]]
+; CHECK-NEXT:    br i1 [[_MSOR6]], label [[TMP10:%.*]], label [[TMP11:%.*]], !prof [[PROF1]]
+; CHECK:       10:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       11:
+; CHECK-NEXT:    [[RES1:%.*]] = call <8 x i32> @llvm.x86.avx512.mask.cvttpd2udq.512(<8 x double> [[X0]], <8 x i32> [[X1]], i8 -1, i32 8)
+; CHECK-NEXT:    [[RES2:%.*]] = add <8 x i32> [[RES]], [[RES1]]
+; CHECK-NEXT:    store <8 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i32> [[RES2]]
+;
+  %res = call <8 x i32> @llvm.x86.avx512.mask.cvttpd2udq.512(<8 x double> %x0, <8 x i32> %x1, i8 %x2, i32 4)
+  %res1 = call <8 x i32> @llvm.x86.avx512.mask.cvttpd2udq.512(<8 x double> %x0, <8 x i32> %x1, i8 -1, i32 8)
+  %res2 = add <8 x i32> %res, %res1
+  ret <8 x i32> %res2
+}
+
+declare <16 x i32> @llvm.x86.avx512.mask.cvttps2dq.512(<16 x float>, <16 x i32>, i16, i32)
+
+define <16 x i32>@test_int_x86_avx512_mask_cvtt_ps2dq_512(<16 x float> %x0, <16 x i32> %x1, i16 %x2) #0 {
+; CHECK-LABEL: @test_int_x86_avx512_mask_cvtt_ps2dq_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i16 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR3]], label [[TMP6:%.*]], label [[TMP7:%.*]], !prof [[PROF1]]
+; CHECK:       6:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       7:
+; CHECK-NEXT:    [[RES:%.*]] = call <16 x i32> @llvm.x86.avx512.mask.cvttps2dq.512(<16 x float> [[X0:%.*]], <16 x i32> [[X1:%.*]], i16 [[X2:%.*]], i32 4)
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP4:%.*]] = icmp ne i512 [[TMP8]], 0
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP5:%.*]] = icmp ne i512 [[TMP9]], 0
+; CHECK-NEXT:    [[_MSOR6:%.*]] = or i1 [[_MSCMP4]], [[_MSCMP5]]
+; CHECK-NEXT:    br i1 [[_MSOR6]], label [[TMP10:%.*]], label [[TMP11:%.*]], !prof [[PROF1]]
+; CHECK:       10:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       11:
+; CHECK-NEXT:    [[RES1:%.*]] = call <16 x i32> @llvm.x86.avx512.mask.cvttps2dq.512(<16 x float> [[X0]], <16 x i32> [[X1]], i16 -1, i32 8)
+; CHECK-NEXT:    [[RES2:%.*]] = add <16 x i32> [[RES]], [[RES1]]
+; CHECK-NEXT:    store <16 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i32> [[RES2]]
+;
+  %res = call <16 x i32> @llvm.x86.avx512.mask.cvttps2dq.512(<16 x float> %x0, <16 x i32> %x1, i16 %x2, i32 4)
+  %res1 = call <16 x i32> @llvm.x86.avx512.mask.cvttps2dq.512(<16 x float> %x0, <16 x i32> %x1, i16 -1, i32 8)
+  %res2 = add <16 x i32> %res, %res1
+  ret <16 x i32> %res2
+}
+
+declare <16 x i32> @llvm.x86.avx512.mask.cvttps2udq.512(<16 x float>, <16 x i32>, i16, i32)
+
+define <16 x i32>@test_int_x86_avx512_mask_cvtt_ps2udq_512(<16 x float> %x0, <16 x i32> %x1, i16 %x2) #0 {
+; CHECK-LABEL: @test_int_x86_avx512_mask_cvtt_ps2udq_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i16 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR3]], label [[TMP6:%.*]], label [[TMP7:%.*]], !prof [[PROF1]]
+; CHECK:       6:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       7:
+; CHECK-NEXT:    [[RES:%.*]] = call <16 x i32> @llvm.x86.avx512.mask.cvttps2udq.512(<16 x float> [[X0:%.*]], <16 x i32> [[X1:%.*]], i16 [[X2:%.*]], i32 4)
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP4:%.*]] = icmp ne i512 [[TMP8]], 0
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP5:%.*]] = icmp ne i512 [[TMP9]], 0
+; CHECK-NEXT:    [[_MSOR6:%.*]] = or i1 [[_MSCMP4]], [[_MSCMP5]]
+; CHECK-NEXT:    br i1 [[_MSOR6]], label [[TMP10:%.*]], label [[TMP11:%.*]], !prof [[PROF1]]
+; CHECK:       10:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       11:
+; CHECK-NEXT:    [[RES1:%.*]] = call <16 x i32> @llvm.x86.avx512.mask.cvttps2udq.512(<16 x float> [[X0]], <16 x i32> [[X1]], i16 -1, i32 8)
+; CHECK-NEXT:    [[RES2:%.*]] = add <16 x i32> [[RES]], [[RES1]]
+; CHECK-NEXT:    store <16 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i32> [[RES2]]
+;
+  %res = call <16 x i32> @llvm.x86.avx512.mask.cvttps2udq.512(<16 x float> %x0, <16 x i32> %x1, i16 %x2, i32 4)
+  %res1 = call <16 x i32> @llvm.x86.avx512.mask.cvttps2udq.512(<16 x float> %x0, <16 x i32> %x1, i16 -1, i32 8)
+  %res2 = add <16 x i32> %res, %res1
+  ret <16 x i32> %res2
+}
+
+declare <4 x float> @llvm.x86.avx512.mask.getexp.ss(<4 x float>, <4 x float>, <4 x float>, i8, i32) nounwind readnone
+
+define <4 x float> @test_getexp_ss(<4 x float> %a0, <4 x float> %a1) #0 {
+; CHECK-LABEL: @test_getexp_ss(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <4 x i32> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP3]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <4 x i32> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP5:%.*]], label [[TMP6:%.*]], !prof [[PROF1]]
+; CHECK:       5:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       6:
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.x86.avx512.mask.getexp.ss(<4 x float> [[A0:%.*]], <4 x float> [[A1:%.*]], <4 x float> zeroinitializer, i8 -1, i32 8)
+; CHECK-NEXT:    store <4 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x float> [[RES]]
+;
+  %res = call <4 x float> @llvm.x86.avx512.mask.getexp.ss(<4 x float>%a0, <4 x float> %a1, <4 x float> zeroinitializer, i8 -1, i32 8)
+  ret <4 x float> %res
+}
+
+define <4 x float> @test_mask_getexp_ss(<4 x float> %a0, <4 x float> %a1, <4 x float> %a2, i8 %mask) #0 {
+; CHECK-LABEL: @test_mask_getexp_ss(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 48) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <4 x i32> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <4 x i32> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP6]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <4 x i32> [[TMP3]] to i128
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i128 [[TMP7]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    [[_MSCMP4:%.*]] = icmp ne i8 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR5:%.*]] = or i1 [[_MSOR3]], [[_MSCMP4]]
+; CHECK-NEXT:    br i1 [[_MSOR5]], label [[TMP8:%.*]], label [[TMP9:%.*]], !prof [[PROF1]]
+; CHECK:       8:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       9:
+; CHECK-NEXT:    [[RES0:%.*]] = call <4 x float> @llvm.x86.avx512.mask.getexp.ss(<4 x float> [[A0:%.*]], <4 x float> [[A1:%.*]], <4 x float> [[A2:%.*]], i8 [[MASK:%.*]], i32 4)
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast <4 x i32> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP6:%.*]] = icmp ne i128 [[TMP10]], 0
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <4 x i32> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP7:%.*]] = icmp ne i128 [[TMP11]], 0
+; CHECK-NEXT:    [[_MSOR8:%.*]] = or i1 [[_MSCMP6]], [[_MSCMP7]]
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <4 x i32> [[TMP3]] to i128
+; CHECK-NEXT:    [[_MSCMP9:%.*]] = icmp ne i128 [[TMP12]], 0
+; CHECK-NEXT:    [[_MSOR10:%.*]] = or i1 [[_MSOR8]], [[_MSCMP9]]
+; CHECK-NEXT:    [[_MSCMP11:%.*]] = icmp ne i8 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR12:%.*]] = or i1 [[_MSOR10]], [[_MSCMP11]]
+; CHECK-NEXT:    br i1 [[_MSOR12]], label [[TMP13:%.*]], label [[TMP14:%.*]], !prof [[PROF1]]
+; CHECK:       13:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       14:
+; CHECK-NEXT:    [[RES1:%.*]] = call <4 x float> @llvm.x86.avx512.mask.getexp.ss(<4 x float> [[A0]], <4 x float> [[A1]], <4 x float> [[A2]], i8 [[MASK]], i32 8)
+; CHECK-NEXT:    [[RES_1:%.*]] = fadd <4 x float> [[RES0]], [[RES1]]
+; CHECK-NEXT:    store <4 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x float> [[RES_1]]
+;
+  %res0 = call <4 x float> @llvm.x86.avx512.mask.getexp.ss(<4 x float>%a0, <4 x float> %a1, <4 x float> %a2, i8 %mask, i32 4)
+  %res1 = call <4 x float> @llvm.x86.avx512.mask.getexp.ss(<4 x float>%a0, <4 x float> %a1, <4 x float> %a2, i8 %mask, i32 8)
+  %res.1 = fadd <4 x float> %res0, %res1
+  ret <4 x float> %res.1
+}
+
+define <4 x float> @test_maskz_getexp_ss(<4 x float> %a0, <4 x float> %a1, i8 %mask) #0 {
+; CHECK-LABEL: @test_maskz_getexp_ss(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <4 x i32> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <4 x i32> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i8 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR3]], label [[TMP6:%.*]], label [[TMP7:%.*]], !prof [[PROF1]]
+; CHECK:       6:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       7:
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.x86.avx512.mask.getexp.ss(<4 x float> [[A0:%.*]], <4 x float> [[A1:%.*]], <4 x float> zeroinitializer, i8 [[MASK:%.*]], i32 8)
+; CHECK-NEXT:    store <4 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x float> [[RES]]
+;
+  %res = call <4 x float> @llvm.x86.avx512.mask.getexp.ss(<4 x float>%a0, <4 x float> %a1, <4 x float> zeroinitializer, i8 %mask, i32 8)
+  ret <4 x float> %res
+}
+
+declare <2 x double> @llvm.x86.avx512.mask.getexp.sd(<2 x double>, <2 x double>, <2 x double>, i8, i32) nounwind readnone
+
+define <2 x double> @test_getexp_sd(<2 x double> %a0, <2 x double> %a1) #0 {
+; CHECK-LABEL: @test_getexp_sd(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <2 x i64> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP3]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <2 x i64> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP5:%.*]], label [[TMP6:%.*]], !prof [[PROF1]]
+; CHECK:       5:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       6:
+; CHECK-NEXT:    [[RES:%.*]] = call <2 x double> @llvm.x86.avx512.mask.getexp.sd(<2 x double> [[A0:%.*]], <2 x double> [[A1:%.*]], <2 x double> zeroinitializer, i8 -1, i32 4)
+; CHECK-NEXT:    store <2 x i64> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x double> [[RES]]
+;
+  %res = call <2 x double> @llvm.x86.avx512.mask.getexp.sd(<2 x double>%a0, <2 x double> %a1, <2 x double> zeroinitializer, i8 -1, i32 4)
+  ret <2 x double> %res
+}
+
+define <2 x double> @test_mask_getexp_sd(<2 x double> %a0, <2 x double> %a1, <2 x double> %a2, i8 %mask) #0 {
+; CHECK-LABEL: @test_mask_getexp_sd(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 48) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <2 x i64> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <2 x i64> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP6]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <2 x i64> [[TMP3]] to i128
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i128 [[TMP7]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    [[_MSCMP4:%.*]] = icmp ne i8 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR5:%.*]] = or i1 [[_MSOR3]], [[_MSCMP4]]
+; CHECK-NEXT:    br i1 [[_MSOR5]], label [[TMP8:%.*]], label [[TMP9:%.*]], !prof [[PROF1]]
+; CHECK:       8:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       9:
+; CHECK-NEXT:    [[RES0:%.*]] = call <2 x double> @llvm.x86.avx512.mask.getexp.sd(<2 x double> [[A0:%.*]], <2 x double> [[A1:%.*]], <2 x double> [[A2:%.*]], i8 [[MASK:%.*]], i32 4)
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast <2 x i64> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP6:%.*]] = icmp ne i128 [[TMP10]], 0
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <2 x i64> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP7:%.*]] = icmp ne i128 [[TMP11]], 0
+; CHECK-NEXT:    [[_MSOR8:%.*]] = or i1 [[_MSCMP6]], [[_MSCMP7]]
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <2 x i64> [[TMP3]] to i128
+; CHECK-NEXT:    [[_MSCMP9:%.*]] = icmp ne i128 [[TMP12]], 0
+; CHECK-NEXT:    [[_MSOR10:%.*]] = or i1 [[_MSOR8]], [[_MSCMP9]]
+; CHECK-NEXT:    [[_MSCMP11:%.*]] = icmp ne i8 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR12:%.*]] = or i1 [[_MSOR10]], [[_MSCMP11]]
+; CHECK-NEXT:    br i1 [[_MSOR12]], label [[TMP13:%.*]], label [[TMP14:%.*]], !prof [[PROF1]]
+; CHECK:       13:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       14:
+; CHECK-NEXT:    [[RES1:%.*]] = call <2 x double> @llvm.x86.avx512.mask.getexp.sd(<2 x double> [[A0]], <2 x double> [[A1]], <2 x double> [[A2]], i8 [[MASK]], i32 8)
+; CHECK-NEXT:    [[RES_1:%.*]] = fadd <2 x double> [[RES0]], [[RES1]]
+; CHECK-NEXT:    store <2 x i64> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x double> [[RES_1]]
+;
+  %res0 = call <2 x double> @llvm.x86.avx512.mask.getexp.sd(<2 x double>%a0, <2 x double> %a1, <2 x double> %a2, i8 %mask, i32 4)
+  %res1 = call <2 x double> @llvm.x86.avx512.mask.getexp.sd(<2 x double>%a0, <2 x double> %a1, <2 x double> %a2, i8 %mask, i32 8)
+  %res.1 = fadd <2 x double> %res0, %res1
+  ret <2 x double> %res.1
+}
+
+define <2 x double> @test_maskz_getexp_sd(<2 x double> %a0, <2 x double> %a1, i8 %mask) #0 {
+; CHECK-LABEL: @test_maskz_getexp_sd(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <2 x i64> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <2 x i64> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i8 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR3]], label [[TMP6:%.*]], label [[TMP7:%.*]], !prof [[PROF1]]
+; CHECK:       6:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       7:
+; CHECK-NEXT:    [[RES:%.*]] = call <2 x double> @llvm.x86.avx512.mask.getexp.sd(<2 x double> [[A0:%.*]], <2 x double> [[A1:%.*]], <2 x double> zeroinitializer, i8 [[MASK:%.*]], i32 8)
+; CHECK-NEXT:    store <2 x i64> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x double> [[RES]]
+;
+  %res = call <2 x double> @llvm.x86.avx512.mask.getexp.sd(<2 x double>%a0, <2 x double> %a1, <2 x double> zeroinitializer, i8 %mask, i32 8)
+  ret <2 x double> %res
+}
+
+declare i8 @llvm.x86.avx512.mask.cmp.sd(<2 x double>, <2 x double>, i32, i8, i32)
+
+define i8@test_int_x86_avx512_mask_cmp_sd(<2 x double> %x0, <2 x double> %x1, i8 %x3, i32 %x4) #0 {
+; CHECK-LABEL: @test_int_x86_avx512_mask_cmp_sd(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <2 x i64> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <2 x i64> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i8 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR3]], label [[TMP6:%.*]], label [[TMP7:%.*]], !prof [[PROF1]]
+; CHECK:       6:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       7:
+; CHECK-NEXT:    [[RES4:%.*]] = call i8 @llvm.x86.avx512.mask.cmp.sd(<2 x double> [[X0:%.*]], <2 x double> [[X1:%.*]], i32 5, i8 [[X3:%.*]], i32 8)
+; CHECK-NEXT:    store i8 0, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i8 [[RES4]]
+;
+  %res4 = call i8 @llvm.x86.avx512.mask.cmp.sd(<2 x double> %x0, <2 x double> %x1, i32 5, i8 %x3, i32 8)
+  ret i8 %res4
+}
+
+define i8@test_int_x86_avx512_mask_cmp_sd_all(<2 x double> %x0, <2 x double> %x1, i8 %x3, i32 %x4) #0 {
+; CHECK-LABEL: @test_int_x86_avx512_mask_cmp_sd_all(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <2 x i64> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <2 x i64> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP6:%.*]], label [[TMP7:%.*]], !prof [[PROF1]]
+; CHECK:       6:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       7:
+; CHECK-NEXT:    [[RES1:%.*]] = call i8 @llvm.x86.avx512.mask.cmp.sd(<2 x double> [[X0:%.*]], <2 x double> [[X1:%.*]], i32 2, i8 -1, i32 4)
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <2 x i64> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i128 [[TMP8]], 0
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <2 x i64> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP3:%.*]] = icmp ne i128 [[TMP9]], 0
+; CHECK-NEXT:    [[_MSOR4:%.*]] = or i1 [[_MSCMP2]], [[_MSCMP3]]
+; CHECK-NEXT:    br i1 [[_MSOR4]], label [[TMP10:%.*]], label [[TMP11:%.*]], !prof [[PROF1]]
+; CHECK:       10:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       11:
+; CHECK-NEXT:    [[RES2:%.*]] = call i8 @llvm.x86.avx512.mask.cmp.sd(<2 x double> [[X0]], <2 x double> [[X1]], i32 3, i8 -1, i32 8)
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <2 x i64> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP5:%.*]] = icmp ne i128 [[TMP12]], 0
+; CHECK-NEXT:    [[TMP13:%.*]] = bitcast <2 x i64> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP6:%.*]] = icmp ne i128 [[TMP13]], 0
+; CHECK-NEXT:    [[_MSOR7:%.*]] = or i1 [[_MSCMP5]], [[_MSCMP6]]
+; CHECK-NEXT:    [[_MSCMP8:%.*]] = icmp ne i8 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSOR9:%.*]] = or i1 [[_MSOR7]], [[_MSCMP8]]
+; CHECK-NEXT:    br i1 [[_MSOR9]], label [[TMP14:%.*]], label [[TMP15:%.*]], !prof [[PROF1]]
+; CHECK:       14:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       15:
+; CHECK-NEXT:    [[RES3:%.*]] = call i8 @llvm.x86.avx512.mask.cmp.sd(<2 x double> [[X0]], <2 x double> [[X1]], i32 4, i8 [[X3:%.*]], i32 4)
+; CHECK-NEXT:    [[TMP16:%.*]] = bitcast <2 x i64> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP10:%.*]] = icmp ne i128 [[TMP16]], 0
+; CHECK-NEXT:    [[TMP17:%.*]] = bitcast <2 x i64> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP11:%.*]] = icmp ne i128 [[TMP17]], 0
+; CHECK-NEXT:    [[_MSOR12:%.*]] = or i1 [[_MSCMP10]], [[_MSCMP11]]
+; CHECK-NEXT:    [[_MSCMP13:%.*]] = icmp ne i8 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSOR14:%.*]] = or i1 [[_MSOR12]], [[_MSCMP13]]
+; CHECK-NEXT:    br i1 [[_MSOR14]], label [[TMP18:%.*]], label [[TMP19:%.*]], !prof [[PROF1]]
+; CHECK:       18:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       19:
+; CHECK-NEXT:    [[RES4:%.*]] = call i8 @llvm.x86.avx512.mask.cmp.sd(<2 x double> [[X0]], <2 x double> [[X1]], i32 5, i8 [[X3]], i32 8)
+; CHECK-NEXT:    [[TMP20:%.*]] = xor i8 [[RES1]], -1
+; CHECK-NEXT:    [[TMP21:%.*]] = xor i8 [[RES2]], -1
+; CHECK-NEXT:    [[TMP22:%.*]] = and i8 [[TMP20]], 0
+; CHECK-NEXT:    [[TMP23:%.*]] = and i8 0, [[TMP21]]
+; CHECK-NEXT:    [[TMP24:%.*]] = or i8 0, [[TMP22]]
+; CHECK-NEXT:    [[TMP25:%.*]] = or i8 [[TMP24]], [[TMP23]]
+; CHECK-NEXT:    [[RES11:%.*]] = or i8 [[RES1]], [[RES2]]
+; CHECK-NEXT:    [[TMP26:%.*]] = xor i8 [[RES3]], -1
+; CHECK-NEXT:    [[TMP27:%.*]] = xor i8 [[RES4]], -1
+; CHECK-NEXT:    [[TMP28:%.*]] = and i8 [[TMP26]], 0
+; CHECK-NEXT:    [[TMP29:%.*]] = and i8 0, [[TMP27]]
+; CHECK-NEXT:    [[TMP30:%.*]] = or i8 0, [[TMP28]]
+; CHECK-NEXT:    [[TMP31:%.*]] = or i8 [[TMP30]], [[TMP29]]
+; CHECK-NEXT:    [[RES12:%.*]] = or i8 [[RES3]], [[RES4]]
+; CHECK-NEXT:    [[TMP32:%.*]] = xor i8 [[RES11]], -1
+; CHECK-NEXT:    [[TMP33:%.*]] = xor i8 [[RES12]], -1
+; CHECK-NEXT:    [[TMP34:%.*]] = and i8 [[TMP25]], [[TMP31]]
+; CHECK-NEXT:    [[TMP35:%.*]] = and i8 [[TMP32]], [[TMP31]]
+; CHECK-NEXT:    [[TMP36:%.*]] = and i8 [[TMP25]], [[TMP33]]
+; CHECK-NEXT:    [[TMP37:%.*]] = or i8 [[TMP34]], [[TMP35]]
+; CHECK-NEXT:    [[TMP38:%.*]] = or i8 [[TMP37]], [[TMP36]]
+; CHECK-NEXT:    [[RES13:%.*]] = or i8 [[RES11]], [[RES12]]
+; CHECK-NEXT:    store i8 [[TMP38]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i8 [[RES13]]
+;
+  %res1 = call i8 @llvm.x86.avx512.mask.cmp.sd(<2 x double> %x0, <2 x double> %x1, i32 2, i8 -1, i32 4)
+  %res2 = call i8 @llvm.x86.avx512.mask.cmp.sd(<2 x double> %x0, <2 x double> %x1, i32 3, i8 -1, i32 8)
+  %res3 = call i8 @llvm.x86.avx512.mask.cmp.sd(<2 x double> %x0, <2 x double> %x1, i32 4, i8 %x3, i32 4)
+  %res4 = call i8 @llvm.x86.avx512.mask.cmp.sd(<2 x double> %x0, <2 x double> %x1, i32 5, i8 %x3, i32 8)
+
+  %res11 = or i8 %res1, %res2
+  %res12 = or i8 %res3, %res4
+  %res13 = or i8 %res11, %res12
+  ret i8 %res13
+}
+
+declare i8 @llvm.x86.avx512.mask.cmp.ss(<4 x float>, <4 x float>, i32, i8, i32)
+
+define i8@test_int_x86_avx512_mask_cmp_ss(<4 x float> %x0, <4 x float> %x1, i8 %x3, i32 %x4) #0 {
+; CHECK-LABEL: @test_int_x86_avx512_mask_cmp_ss(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <4 x i32> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <4 x i32> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i8 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR3]], label [[TMP6:%.*]], label [[TMP7:%.*]], !prof [[PROF1]]
+; CHECK:       6:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       7:
+; CHECK-NEXT:    [[RES2:%.*]] = call i8 @llvm.x86.avx512.mask.cmp.ss(<4 x float> [[X0:%.*]], <4 x float> [[X1:%.*]], i32 3, i8 [[X3:%.*]], i32 4)
+; CHECK-NEXT:    store i8 0, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i8 [[RES2]]
+;
+  %res2 = call i8 @llvm.x86.avx512.mask.cmp.ss(<4 x float> %x0, <4 x float> %x1, i32 3, i8 %x3, i32 4)
+  ret i8 %res2
+}
+
+
+define i8@test_int_x86_avx512_mask_cmp_ss_all(<4 x float> %x0, <4 x float> %x1, i8 %x3, i32 %x4) #0 {
+; CHECK-LABEL: @test_int_x86_avx512_mask_cmp_ss_all(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <4 x i32> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <4 x i32> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP6:%.*]], label [[TMP7:%.*]], !prof [[PROF1]]
+; CHECK:       6:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       7:
+; CHECK-NEXT:    [[RES1:%.*]] = call i8 @llvm.x86.avx512.mask.cmp.ss(<4 x float> [[X0:%.*]], <4 x float> [[X1:%.*]], i32 2, i8 -1, i32 4)
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <4 x i32> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i128 [[TMP8]], 0
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <4 x i32> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP3:%.*]] = icmp ne i128 [[TMP9]], 0
+; CHECK-NEXT:    [[_MSOR4:%.*]] = or i1 [[_MSCMP2]], [[_MSCMP3]]
+; CHECK-NEXT:    br i1 [[_MSOR4]], label [[TMP10:%.*]], label [[TMP11:%.*]], !prof [[PROF1]]
+; CHECK:       10:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       11:
+; CHECK-NEXT:    [[RES2:%.*]] = call i8 @llvm.x86.avx512.mask.cmp.ss(<4 x float> [[X0]], <4 x float> [[X1]], i32 3, i8 -1, i32 8)
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <4 x i32> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP5:%.*]] = icmp ne i128 [[TMP12]], 0
+; CHECK-NEXT:    [[TMP13:%.*]] = bitcast <4 x i32> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP6:%.*]] = icmp ne i128 [[TMP13]], 0
+; CHECK-NEXT:    [[_MSOR7:%.*]] = or i1 [[_MSCMP5]], [[_MSCMP6]]
+; CHECK-NEXT:    [[_MSCMP8:%.*]] = icmp ne i8 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSOR9:%.*]] = or i1 [[_MSOR7]], [[_MSCMP8]]
+; CHECK-NEXT:    br i1 [[_MSOR9]], label [[TMP14:%.*]], label [[TMP15:%.*]], !prof [[PROF1]]
+; CHECK:       14:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       15:
+; CHECK-NEXT:    [[RES3:%.*]] = call i8 @llvm.x86.avx512.mask.cmp.ss(<4 x float> [[X0]], <4 x float> [[X1]], i32 4, i8 [[X3:%.*]], i32 4)
+; CHECK-NEXT:    [[TMP16:%.*]] = bitcast <4 x i32> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP10:%.*]] = icmp ne i128 [[TMP16]], 0
+; CHECK-NEXT:    [[TMP17:%.*]] = bitcast <4 x i32> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP11:%.*]] = icmp ne i128 [[TMP17]], 0
+; CHECK-NEXT:    [[_MSOR12:%.*]] = or i1 [[_MSCMP10]], [[_MSCMP11]]
+; CHECK-NEXT:    [[_MSCMP13:%.*]] = icmp ne i8 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSOR14:%.*]] = or i1 [[_MSOR12]], [[_MSCMP13]]
+; CHECK-NEXT:    br i1 [[_MSOR14]], label [[TMP18:%.*]], label [[TMP19:%.*]], !prof [[PROF1]]
+; CHECK:       18:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       19:
+; CHECK-NEXT:    [[RES4:%.*]] = call i8 @llvm.x86.avx512.mask.cmp.ss(<4 x float> [[X0]], <4 x float> [[X1]], i32 5, i8 [[X3]], i32 8)
+; CHECK-NEXT:    [[TMP20:%.*]] = and i8 [[RES1]], 0
+; CHECK-NEXT:    [[TMP21:%.*]] = and i8 0, [[RES2]]
+; CHECK-NEXT:    [[TMP22:%.*]] = or i8 0, [[TMP20]]
+; CHECK-NEXT:    [[TMP23:%.*]] = or i8 [[TMP22]], [[TMP21]]
+; CHECK-NEXT:    [[RES11:%.*]] = and i8 [[RES1]], [[RES2]]
+; CHECK-NEXT:    [[TMP24:%.*]] = and i8 [[RES3]], 0
+; CHECK-NEXT:    [[TMP25:%.*]] = and i8 0, [[RES4]]
+; CHECK-NEXT:    [[TMP26:%.*]] = or i8 0, [[TMP24]]
+; CHECK-NEXT:    [[TMP27:%.*]] = or i8 [[TMP26]], [[TMP25]]
+; CHECK-NEXT:    [[RES12:%.*]] = and i8 [[RES3]], [[RES4]]
+; CHECK-NEXT:    [[TMP28:%.*]] = and i8 [[TMP23]], [[TMP27]]
+; CHECK-NEXT:    [[TMP29:%.*]] = and i8 [[RES11]], [[TMP27]]
+; CHECK-NEXT:    [[TMP30:%.*]] = and i8 [[TMP23]], [[RES12]]
+; CHECK-NEXT:    [[TMP31:%.*]] = or i8 [[TMP28]], [[TMP29]]
+; CHECK-NEXT:    [[TMP32:%.*]] = or i8 [[TMP31]], [[TMP30]]
+; CHECK-NEXT:    [[RES13:%.*]] = and i8 [[RES11]], [[RES12]]
+; CHECK-NEXT:    store i8 [[TMP32]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i8 [[RES13]]
+;
+  %res1 = call i8 @llvm.x86.avx512.mask.cmp.ss(<4 x float> %x0, <4 x float> %x1, i32 2, i8 -1, i32 4)
+  %res2 = call i8 @llvm.x86.avx512.mask.cmp.ss(<4 x float> %x0, <4 x float> %x1, i32 3, i8 -1, i32 8)
+  %res3 = call i8 @llvm.x86.avx512.mask.cmp.ss(<4 x float> %x0, <4 x float> %x1, i32 4, i8 %x3, i32 4)
+  %res4 = call i8 @llvm.x86.avx512.mask.cmp.ss(<4 x float> %x0, <4 x float> %x1, i32 5, i8 %x3, i32 8)
+
+  %res11 = and i8 %res1, %res2
+  %res12 = and i8 %res3, %res4
+  %res13 = and i8 %res11, %res12
+  ret i8 %res13
+}
+
+declare <8 x double> @llvm.x86.avx512.mask.getmant.pd.512(<8 x double>, i32, <8 x double>, i8, i32)
+
+define <8 x double>@test_int_x86_avx512_mask_getmant_pd_512(<8 x double> %x0, <8 x double> %x2, i8 %x3) #0 {
+; CHECK-LABEL: @test_int_x86_avx512_mask_getmant_pd_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <8 x i64> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <8 x i64> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i8 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR3]], label [[TMP6:%.*]], label [[TMP7:%.*]], !prof [[PROF1]]
+; CHECK:       6:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       7:
+; CHECK-NEXT:    [[RES:%.*]] = call <8 x double> @llvm.x86.avx512.mask.getmant.pd.512(<8 x double> [[X0:%.*]], i32 11, <8 x double> [[X2:%.*]], i8 [[X3:%.*]], i32 4)
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <8 x i64> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP4:%.*]] = icmp ne i512 [[TMP8]], 0
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <8 x i64> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP5:%.*]] = icmp ne i512 [[TMP9]], 0
+; CHECK-NEXT:    [[_MSOR6:%.*]] = or i1 [[_MSCMP4]], [[_MSCMP5]]
+; CHECK-NEXT:    br i1 [[_MSOR6]], label [[TMP10:%.*]], label [[TMP11:%.*]], !prof [[PROF1]]
+; CHECK:       10:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       11:
+; CHECK-NEXT:    [[RES1:%.*]] = call <8 x double> @llvm.x86.avx512.mask.getmant.pd.512(<8 x double> [[X0]], i32 11, <8 x double> [[X2]], i8 -1, i32 8)
+; CHECK-NEXT:    [[RES2:%.*]] = fadd <8 x double> [[RES]], [[RES1]]
+; CHECK-NEXT:    store <8 x i64> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x double> [[RES2]]
+;
+  %res = call <8 x double> @llvm.x86.avx512.mask.getmant.pd.512(<8 x double> %x0, i32 11, <8 x double> %x2, i8 %x3, i32 4)
+  %res1 = call <8 x double> @llvm.x86.avx512.mask.getmant.pd.512(<8 x double> %x0, i32 11, <8 x double> %x2, i8 -1, i32 8)
+  %res2 = fadd <8 x double> %res, %res1
+  ret <8 x double> %res2
+}
+
+declare <16 x float> @llvm.x86.avx512.mask.getmant.ps.512(<16 x float>, i32, <16 x float>, i16, i32)
+
+define <16 x float>@test_int_x86_avx512_mask_getmant_ps_512(<16 x float> %x0, <16 x float> %x2, i16 %x3) #0 {
+; CHECK-LABEL: @test_int_x86_avx512_mask_getmant_ps_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i16 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR3]], label [[TMP6:%.*]], label [[TMP7:%.*]], !prof [[PROF1]]
+; CHECK:       6:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       7:
+; CHECK-NEXT:    [[RES:%.*]] = call <16 x float> @llvm.x86.avx512.mask.getmant.ps.512(<16 x float> [[X0:%.*]], i32 11, <16 x float> [[X2:%.*]], i16 [[X3:%.*]], i32 4)
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP4:%.*]] = icmp ne i512 [[TMP8]], 0
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP5:%.*]] = icmp ne i512 [[TMP9]], 0
+; CHECK-NEXT:    [[_MSOR6:%.*]] = or i1 [[_MSCMP4]], [[_MSCMP5]]
+; CHECK-NEXT:    br i1 [[_MSOR6]], label [[TMP10:%.*]], label [[TMP11:%.*]], !prof [[PROF1]]
+; CHECK:       10:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       11:
+; CHECK-NEXT:    [[RES1:%.*]] = call <16 x float> @llvm.x86.avx512.mask.getmant.ps.512(<16 x float> [[X0]], i32 11, <16 x float> [[X2]], i16 -1, i32 8)
+; CHECK-NEXT:    [[RES2:%.*]] = fadd <16 x float> [[RES]], [[RES1]]
+; CHECK-NEXT:    store <16 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x float> [[RES2]]
+;
+  %res = call <16 x float> @llvm.x86.avx512.mask.getmant.ps.512(<16 x float> %x0, i32 11, <16 x float> %x2, i16 %x3, i32 4)
+  %res1 = call <16 x float> @llvm.x86.avx512.mask.getmant.ps.512(<16 x float> %x0, i32 11, <16 x float> %x2, i16 -1, i32 8)
+  %res2 = fadd <16 x float> %res, %res1
+  ret <16 x float> %res2
+}
+
+declare <2 x double> @llvm.x86.avx512.mask.getmant.sd(<2 x double>, <2 x double>, i32, <2 x double>, i8, i32)
+
+define <2 x double>@test_int_x86_avx512_mask_getmant_sd(<2 x double> %x0, <2 x double> %x1, <2 x double> %x2, i8 %x3) #0 {
+; CHECK-LABEL: @test_int_x86_avx512_mask_getmant_sd(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 48) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <2 x i64> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <2 x i64> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP6]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <2 x i64> [[TMP3]] to i128
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i128 [[TMP7]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    [[_MSCMP4:%.*]] = icmp ne i8 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR5:%.*]] = or i1 [[_MSOR3]], [[_MSCMP4]]
+; CHECK-NEXT:    br i1 [[_MSOR5]], label [[TMP8:%.*]], label [[TMP9:%.*]], !prof [[PROF1]]
+; CHECK:       8:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       9:
+; CHECK-NEXT:    [[RES:%.*]] = call <2 x double> @llvm.x86.avx512.mask.getmant.sd(<2 x double> [[X0:%.*]], <2 x double> [[X1:%.*]], i32 11, <2 x double> [[X2:%.*]], i8 [[X3:%.*]], i32 4)
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast <2 x i64> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP6:%.*]] = icmp ne i128 [[TMP10]], 0
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <2 x i64> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP7:%.*]] = icmp ne i128 [[TMP11]], 0
+; CHECK-NEXT:    [[_MSOR8:%.*]] = or i1 [[_MSCMP6]], [[_MSCMP7]]
+; CHECK-NEXT:    [[_MSCMP9:%.*]] = icmp ne i8 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR10:%.*]] = or i1 [[_MSOR8]], [[_MSCMP9]]
+; CHECK-NEXT:    br i1 [[_MSOR10]], label [[TMP12:%.*]], label [[TMP13:%.*]], !prof [[PROF1]]
+; CHECK:       12:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       13:
+; CHECK-NEXT:    [[RES1:%.*]] = call <2 x double> @llvm.x86.avx512.mask.getmant.sd(<2 x double> [[X0]], <2 x double> [[X1]], i32 12, <2 x double> zeroinitializer, i8 [[X3]], i32 4)
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast <2 x i64> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP11:%.*]] = icmp ne i128 [[TMP14]], 0
+; CHECK-NEXT:    [[TMP15:%.*]] = bitcast <2 x i64> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP12:%.*]] = icmp ne i128 [[TMP15]], 0
+; CHECK-NEXT:    [[_MSOR13:%.*]] = or i1 [[_MSCMP11]], [[_MSCMP12]]
+; CHECK-NEXT:    [[TMP16:%.*]] = bitcast <2 x i64> [[TMP3]] to i128
+; CHECK-NEXT:    [[_MSCMP14:%.*]] = icmp ne i128 [[TMP16]], 0
+; CHECK-NEXT:    [[_MSOR15:%.*]] = or i1 [[_MSOR13]], [[_MSCMP14]]
+; CHECK-NEXT:    [[_MSCMP16:%.*]] = icmp ne i8 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR17:%.*]] = or i1 [[_MSOR15]], [[_MSCMP16]]
+; CHECK-NEXT:    br i1 [[_MSOR17]], label [[TMP17:%.*]], label [[TMP18:%.*]], !prof [[PROF1]]
+; CHECK:       17:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       18:
+; CHECK-NEXT:    [[RES2:%.*]] = call <2 x double> @llvm.x86.avx512.mask.getmant.sd(<2 x double> [[X0]], <2 x double> [[X1]], i32 13, <2 x double> [[X2]], i8 [[X3]], i32 8)
+; CHECK-NEXT:    [[TMP19:%.*]] = bitcast <2 x i64> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP18:%.*]] = icmp ne i128 [[TMP19]], 0
+; CHECK-NEXT:    [[TMP20:%.*]] = bitcast <2 x i64> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP19:%.*]] = icmp ne i128 [[TMP20]], 0
+; CHECK-NEXT:    [[_MSOR20:%.*]] = or i1 [[_MSCMP18]], [[_MSCMP19]]
+; CHECK-NEXT:    [[TMP21:%.*]] = bitcast <2 x i64> [[TMP3]] to i128
+; CHECK-NEXT:    [[_MSCMP21:%.*]] = icmp ne i128 [[TMP21]], 0
+; CHECK-NEXT:    [[_MSOR22:%.*]] = or i1 [[_MSOR20]], [[_MSCMP21]]
+; CHECK-NEXT:    br i1 [[_MSOR22]], label [[TMP22:%.*]], label [[TMP23:%.*]], !prof [[PROF1]]
+; CHECK:       22:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       23:
+; CHECK-NEXT:    [[RES3:%.*]] = call <2 x double> @llvm.x86.avx512.mask.getmant.sd(<2 x double> [[X0]], <2 x double> [[X1]], i32 14, <2 x double> [[X2]], i8 -1, i32 4)
+; CHECK-NEXT:    [[RES11:%.*]] = fadd <2 x double> [[RES]], [[RES1]]
+; CHECK-NEXT:    [[RES12:%.*]] = fadd <2 x double> [[RES2]], [[RES3]]
+; CHECK-NEXT:    [[RES13:%.*]] = fadd <2 x double> [[RES11]], [[RES12]]
+; CHECK-NEXT:    store <2 x i64> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x double> [[RES13]]
+;
+  %res  = call <2 x double> @llvm.x86.avx512.mask.getmant.sd(<2 x double> %x0, <2 x double> %x1, i32 11, <2 x double> %x2, i8 %x3, i32 4)
+  %res1 = call <2 x double> @llvm.x86.avx512.mask.getmant.sd(<2 x double> %x0, <2 x double> %x1, i32 12, <2 x double> zeroinitializer, i8 %x3, i32 4)
+  %res2 = call <2 x double> @llvm.x86.avx512.mask.getmant.sd(<2 x double> %x0, <2 x double> %x1, i32 13, <2 x double> %x2, i8 %x3, i32 8)
+  %res3 = call <2 x double> @llvm.x86.avx512.mask.getmant.sd(<2 x double> %x0, <2 x double> %x1, i32 14, <2 x double> %x2, i8 -1, i32 4)
+  %res11 = fadd <2 x double> %res, %res1
+  %res12 = fadd <2 x double> %res2, %res3
+  %res13 = fadd <2 x double> %res11, %res12
+  ret <2 x double> %res13
+}
+
+declare <4 x float> @llvm.x86.avx512.mask.getmant.ss(<4 x float>, <4 x float>, i32, <4 x float>, i8, i32)
+
+define <4 x float>@test_int_x86_avx512_mask_getmant_ss(<4 x float> %x0, <4 x float> %x1, <4 x float> %x2, i8 %x3) #0 {
+; CHECK-LABEL: @test_int_x86_avx512_mask_getmant_ss(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 48) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <4 x i32> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <4 x i32> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP6]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <4 x i32> [[TMP3]] to i128
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i128 [[TMP7]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    [[_MSCMP4:%.*]] = icmp ne i8 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR5:%.*]] = or i1 [[_MSOR3]], [[_MSCMP4]]
+; CHECK-NEXT:    br i1 [[_MSOR5]], label [[TMP8:%.*]], label [[TMP9:%.*]], !prof [[PROF1]]
+; CHECK:       8:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       9:
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.x86.avx512.mask.getmant.ss(<4 x float> [[X0:%.*]], <4 x float> [[X1:%.*]], i32 11, <4 x float> [[X2:%.*]], i8 [[X3:%.*]], i32 4)
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast <4 x i32> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP6:%.*]] = icmp ne i128 [[TMP10]], 0
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <4 x i32> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP7:%.*]] = icmp ne i128 [[TMP11]], 0
+; CHECK-NEXT:    [[_MSOR8:%.*]] = or i1 [[_MSCMP6]], [[_MSCMP7]]
+; CHECK-NEXT:    [[_MSCMP9:%.*]] = icmp ne i8 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR10:%.*]] = or i1 [[_MSOR8]], [[_MSCMP9]]
+; CHECK-NEXT:    br i1 [[_MSOR10]], label [[TMP12:%.*]], label [[TMP13:%.*]], !prof [[PROF1]]
+; CHECK:       12:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       13:
+; CHECK-NEXT:    [[RES1:%.*]] = call <4 x float> @llvm.x86.avx512.mask.getmant.ss(<4 x float> [[X0]], <4 x float> [[X1]], i32 12, <4 x float> zeroinitializer, i8 [[X3]], i32 4)
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast <4 x i32> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP11:%.*]] = icmp ne i128 [[TMP14]], 0
+; CHECK-NEXT:    [[TMP15:%.*]] = bitcast <4 x i32> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP12:%.*]] = icmp ne i128 [[TMP15]], 0
+; CHECK-NEXT:    [[_MSOR13:%.*]] = or i1 [[_MSCMP11]], [[_MSCMP12]]
+; CHECK-NEXT:    [[TMP16:%.*]] = bitcast <4 x i32> [[TMP3]] to i128
+; CHECK-NEXT:    [[_MSCMP14:%.*]] = icmp ne i128 [[TMP16]], 0
+; CHECK-NEXT:    [[_MSOR15:%.*]] = or i1 [[_MSOR13]], [[_MSCMP14]]
+; CHECK-NEXT:    br i1 [[_MSOR15]], label [[TMP17:%.*]], label [[TMP18:%.*]], !prof [[PROF1]]
+; CHECK:       17:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       18:
+; CHECK-NEXT:    [[RES2:%.*]] = call <4 x float> @llvm.x86.avx512.mask.getmant.ss(<4 x float> [[X0]], <4 x float> [[X1]], i32 13, <4 x float> [[X2]], i8 -1, i32 8)
+; CHECK-NEXT:    [[TMP19:%.*]] = bitcast <4 x i32> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP16:%.*]] = icmp ne i128 [[TMP19]], 0
+; CHECK-NEXT:    [[TMP20:%.*]] = bitcast <4 x i32> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP17:%.*]] = icmp ne i128 [[TMP20]], 0
+; CHECK-NEXT:    [[_MSOR18:%.*]] = or i1 [[_MSCMP16]], [[_MSCMP17]]
+; CHECK-NEXT:    [[TMP21:%.*]] = bitcast <4 x i32> [[TMP3]] to i128
+; CHECK-NEXT:    [[_MSCMP19:%.*]] = icmp ne i128 [[TMP21]], 0
+; CHECK-NEXT:    [[_MSOR20:%.*]] = or i1 [[_MSOR18]], [[_MSCMP19]]
+; CHECK-NEXT:    br i1 [[_MSOR20]], label [[TMP22:%.*]], label [[TMP23:%.*]], !prof [[PROF1]]
+; CHECK:       22:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       23:
+; CHECK-NEXT:    [[RES3:%.*]] = call <4 x float> @llvm.x86.avx512.mask.getmant.ss(<4 x float> [[X0]], <4 x float> [[X1]], i32 14, <4 x float> [[X2]], i8 -1, i32 4)
+; CHECK-NEXT:    [[RES11:%.*]] = fadd <4 x float> [[RES]], [[RES1]]
+; CHECK-NEXT:    [[RES12:%.*]] = fadd <4 x float> [[RES2]], [[RES3]]
+; CHECK-NEXT:    [[RES13:%.*]] = fadd <4 x float> [[RES11]], [[RES12]]
+; CHECK-NEXT:    store <4 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x float> [[RES13]]
+;
+  %res  = call <4 x float> @llvm.x86.avx512.mask.getmant.ss(<4 x float> %x0, <4 x float> %x1, i32 11, <4 x float> %x2, i8 %x3, i32 4)
+  %res1 = call <4 x float> @llvm.x86.avx512.mask.getmant.ss(<4 x float> %x0, <4 x float> %x1, i32 12, <4 x float> zeroinitializer, i8 %x3, i32 4)
+  %res2 = call <4 x float> @llvm.x86.avx512.mask.getmant.ss(<4 x float> %x0, <4 x float> %x1, i32 13, <4 x float> %x2, i8 -1, i32 8)
+  %res3 = call <4 x float> @llvm.x86.avx512.mask.getmant.ss(<4 x float> %x0, <4 x float> %x1, i32 14, <4 x float> %x2, i8 -1, i32 4)
+  %res11 = fadd <4 x float> %res, %res1
+  %res12 = fadd <4 x float> %res2, %res3
+  %res13 = fadd <4 x float> %res11, %res12
+  ret <4 x float> %res13
+}
+
+define <4 x float> @test_int_x86_avx512_mask_getmant_ss_load(<4 x float> %x0, ptr %x1p, <4 x float> %extra_param) #0 {
+; CHECK-LABEL: @test_int_x86_avx512_mask_getmant_ss_load(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 24) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP4:%.*]], label [[TMP10:%.*]], !prof [[PROF1]]
+; CHECK:       4:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       5:
+; CHECK-NEXT:    [[X1:%.*]] = load <4 x float>, ptr [[X1P:%.*]], align 16
+; CHECK-NEXT:    [[TMP5:%.*]] = ptrtoint ptr [[X1P]] to i64
+; CHECK-NEXT:    [[TMP6:%.*]] = xor i64 [[TMP5]], 87960930222080
+; CHECK-NEXT:    [[TMP7:%.*]] = inttoptr i64 [[TMP6]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <4 x i32>, ptr [[TMP7]], align 16
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <4 x i32> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP8]], 0
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <4 x i32> [[_MSLD]] to i128
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i128 [[TMP9]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP1]], [[_MSCMP2]]
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <4 x i32> [[TMP3]] to i128
+; CHECK-NEXT:    [[_MSCMP3:%.*]] = icmp ne i128 [[TMP11]], 0
+; CHECK-NEXT:    [[_MSOR4:%.*]] = or i1 [[_MSOR]], [[_MSCMP3]]
+; CHECK-NEXT:    br i1 [[_MSOR4]], label [[TMP12:%.*]], label [[TMP13:%.*]], !prof [[PROF1]]
+; CHECK:       12:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       13:
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.x86.avx512.mask.getmant.ss(<4 x float> [[X0:%.*]], <4 x float> [[X1]], i32 11, <4 x float> [[EXTRA_PARAM:%.*]], i8 -1, i32 4)
+; CHECK-NEXT:    store <4 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x float> [[RES]]
+;
+  %x1 = load <4 x float>, ptr %x1p
+  %res = call <4 x float> @llvm.x86.avx512.mask.getmant.ss(<4 x float> %x0, <4 x float> %x1, i32 11, <4 x float> %extra_param, i8 -1, i32 4)
+  ret <4 x float> %res
+}
+
+declare <8 x double> @llvm.x86.avx512.vpermilvar.pd.512(<8 x double>, <8 x i64>)
+
+define <8 x double>@test_int_x86_avx512_vpermilvar_pd_512(<8 x double> %x0, <8 x i64> %x1) #0 {
+; CHECK-LABEL: @test_int_x86_avx512_vpermilvar_pd_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <8 x i64> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP3]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <8 x i64> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP5:%.*]], label [[TMP6:%.*]], !prof [[PROF1]]
+; CHECK:       5:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       6:
+; CHECK-NEXT:    [[RES:%.*]] = call <8 x double> @llvm.x86.avx512.vpermilvar.pd.512(<8 x double> [[X0:%.*]], <8 x i64> [[X1:%.*]])
+; CHECK-NEXT:    store <8 x i64> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x double> [[RES]]
+;
+  %res = call <8 x double> @llvm.x86.avx512.vpermilvar.pd.512(<8 x double> %x0, <8 x i64> %x1)
+  ret <8 x double> %res
+}
+
+define <8 x double>@test_int_x86_avx512_vpermilvar_pd_512_mask(<8 x double> %x0, <8 x i64> %x1, <8 x double> %x2, i8 %mask) #0 {
+; CHECK-LABEL: @test_int_x86_avx512_vpermilvar_pd_512_mask(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <8 x i64> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <8 x i64> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP6]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP7:%.*]], label [[TMP8:%.*]], !prof [[PROF1]]
+; CHECK:       7:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       8:
+; CHECK-NEXT:    [[RES:%.*]] = call <8 x double> @llvm.x86.avx512.vpermilvar.pd.512(<8 x double> [[X0:%.*]], <8 x i64> [[X1:%.*]])
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast i8 [[TMP3]] to <8 x i1>
+; CHECK-NEXT:    [[MASK_CAST:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP10:%.*]] = select <8 x i1> [[MASK_CAST]], <8 x i64> zeroinitializer, <8 x i64> [[TMP4]]
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <8 x double> [[RES]] to <8 x i64>
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <8 x double> [[X2:%.*]] to <8 x i64>
+; CHECK-NEXT:    [[TMP13:%.*]] = xor <8 x i64> [[TMP11]], [[TMP12]]
+; CHECK-NEXT:    [[TMP14:%.*]] = or <8 x i64> [[TMP13]], zeroinitializer
+; CHECK-NEXT:    [[TMP15:%.*]] = or <8 x i64> [[TMP14]], [[TMP4]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP9]], <8 x i64> [[TMP15]], <8 x i64> [[TMP10]]
+; CHECK-NEXT:    [[RES2:%.*]] = select <8 x i1> [[MASK_CAST]], <8 x double> [[RES]], <8 x double> [[X2]]
+; CHECK-NEXT:    store <8 x i64> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x double> [[RES2]]
+;
+  %res = call <8 x double> @llvm.x86.avx512.vpermilvar.pd.512(<8 x double> %x0, <8 x i64> %x1)
+  %mask.cast = bitcast i8 %mask to <8 x i1>
+  %res2 = select <8 x i1> %mask.cast, <8 x double> %res, <8 x double> %x2
+  ret <8 x double> %res2
+}
+
+define <8 x double>@test_int_x86_avx512_vpermilvar_pd_512_maskz(<8 x double> %x0, <8 x i64> %x1, i8 %mask) #0 {
+; CHECK-LABEL: @test_int_x86_avx512_vpermilvar_pd_512_maskz(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <8 x i64> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <8 x i64> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP6:%.*]], label [[TMP7:%.*]], !prof [[PROF1]]
+; CHECK:       6:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       7:
+; CHECK-NEXT:    [[RES:%.*]] = call <8 x double> @llvm.x86.avx512.vpermilvar.pd.512(<8 x double> [[X0:%.*]], <8 x i64> [[X1:%.*]])
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast i8 [[TMP3]] to <8 x i1>
+; CHECK-NEXT:    [[MASK_CAST:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP9:%.*]] = select <8 x i1> [[MASK_CAST]], <8 x i64> zeroinitializer, <8 x i64> zeroinitializer
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast <8 x double> [[RES]] to <8 x i64>
+; CHECK-NEXT:    [[TMP11:%.*]] = xor <8 x i64> [[TMP10]], zeroinitializer
+; CHECK-NEXT:    [[TMP12:%.*]] = or <8 x i64> [[TMP11]], zeroinitializer
+; CHECK-NEXT:    [[TMP13:%.*]] = or <8 x i64> [[TMP12]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP8]], <8 x i64> [[TMP13]], <8 x i64> [[TMP9]]
+; CHECK-NEXT:    [[RES2:%.*]] = select <8 x i1> [[MASK_CAST]], <8 x double> [[RES]], <8 x double> zeroinitializer
+; CHECK-NEXT:    store <8 x i64> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x double> [[RES2]]
+;
+  %res = call <8 x double> @llvm.x86.avx512.vpermilvar.pd.512(<8 x double> %x0, <8 x i64> %x1)
+  %mask.cast = bitcast i8 %mask to <8 x i1>
+  %res2 = select <8 x i1> %mask.cast, <8 x double> %res, <8 x double> zeroinitializer
+  ret <8 x double> %res2
+}
+
+declare <16 x float> @llvm.x86.avx512.vpermilvar.ps.512(<16 x float>, <16 x i32>)
+
+define <16 x float>@test_int_x86_avx512_vpermilvar_ps_512(<16 x float> %x0, <16 x i32> %x1) #0 {
+; CHECK-LABEL: @test_int_x86_avx512_vpermilvar_ps_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP3]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP5:%.*]], label [[TMP6:%.*]], !prof [[PROF1]]
+; CHECK:       5:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       6:
+; CHECK-NEXT:    [[RES:%.*]] = call <16 x float> @llvm.x86.avx512.vpermilvar.ps.512(<16 x float> [[X0:%.*]], <16 x i32> [[X1:%.*]])
+; CHECK-NEXT:    store <16 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x float> [[RES]]
+;
+  %res = call <16 x float> @llvm.x86.avx512.vpermilvar.ps.512(<16 x float> %x0, <16 x i32> %x1)
+  ret <16 x float> %res
+}
+
+define <16 x float>@test_int_x86_avx512_vpermilvar_ps_512_mask(<16 x float> %x0, <16 x i32> %x1, <16 x float> %x2, i16 %mask) #0 {
+; CHECK-LABEL: @test_int_x86_avx512_vpermilvar_ps_512_mask(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP6]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP7:%.*]], label [[TMP8:%.*]], !prof [[PROF1]]
+; CHECK:       7:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       8:
+; CHECK-NEXT:    [[RES:%.*]] = call <16 x float> @llvm.x86.avx512.vpermilvar.ps.512(<16 x float> [[X0:%.*]], <16 x i32> [[X1:%.*]])
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast i16 [[TMP3]] to <16 x i1>
+; CHECK-NEXT:    [[MASK_CAST:%.*]] = bitcast i16 [[MASK:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP10:%.*]] = select <16 x i1> [[MASK_CAST]], <16 x i32> zeroinitializer, <16 x i32> [[TMP4]]
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <16 x float> [[RES]] to <16 x i32>
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <16 x float> [[X2:%.*]] to <16 x i32>
+; CHECK-NEXT:    [[TMP13:%.*]] = xor <16 x i32> [[TMP11]], [[TMP12]]
+; CHECK-NEXT:    [[TMP14:%.*]] = or <16 x i32> [[TMP13]], zeroinitializer
+; CHECK-NEXT:    [[TMP15:%.*]] = or <16 x i32> [[TMP14]], [[TMP4]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP9]], <16 x i32> [[TMP15]], <16 x i32> [[TMP10]]
+; CHECK-NEXT:    [[RES2:%.*]] = select <16 x i1> [[MASK_CAST]], <16 x float> [[RES]], <16 x float> [[X2]]
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x float> [[RES2]]
+;
+  %res = call <16 x float> @llvm.x86.avx512.vpermilvar.ps.512(<16 x float> %x0, <16 x i32> %x1)
+  %mask.cast = bitcast i16 %mask to <16 x i1>
+  %res2 = select <16 x i1> %mask.cast, <16 x float> %res, <16 x float> %x2
+  ret <16 x float> %res2
+}
+
+define <16 x float>@test_int_x86_avx512_vpermilvar_ps_512_maskz(<16 x float> %x0, <16 x i32> %x1, i16 %mask) #0 {
+; CHECK-LABEL: @test_int_x86_avx512_vpermilvar_ps_512_maskz(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP6:%.*]], label [[TMP7:%.*]], !prof [[PROF1]]
+; CHECK:       6:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       7:
+; CHECK-NEXT:    [[RES:%.*]] = call <16 x float> @llvm.x86.avx512.vpermilvar.ps.512(<16 x float> [[X0:%.*]], <16 x i32> [[X1:%.*]])
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast i16 [[TMP3]] to <16 x i1>
+; CHECK-NEXT:    [[MASK_CAST:%.*]] = bitcast i16 [[MASK:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP9:%.*]] = select <16 x i1> [[MASK_CAST]], <16 x i32> zeroinitializer, <16 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast <16 x float> [[RES]] to <16 x i32>
+; CHECK-NEXT:    [[TMP11:%.*]] = xor <16 x i32> [[TMP10]], zeroinitializer
+; CHECK-NEXT:    [[TMP12:%.*]] = or <16 x i32> [[TMP11]], zeroinitializer
+; CHECK-NEXT:    [[TMP13:%.*]] = or <16 x i32> [[TMP12]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP8]], <16 x i32> [[TMP13]], <16 x i32> [[TMP9]]
+; CHECK-NEXT:    [[RES2:%.*]] = select <16 x i1> [[MASK_CAST]], <16 x float> [[RES]], <16 x float> zeroinitializer
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x float> [[RES2]]
+;
+  %res = call <16 x float> @llvm.x86.avx512.vpermilvar.ps.512(<16 x float> %x0, <16 x i32> %x1)
+  %mask.cast = bitcast i16 %mask to <16 x i1>
+  %res2 = select <16 x i1> %mask.cast, <16 x float> %res, <16 x float> zeroinitializer
+  ret <16 x float> %res2
+}
+
+define <16 x float>@test_int_x86_avx512_vpermilvar_ps_512_constant_pool(<16 x float> %x0, <16 x i32> %x1) #0 {
+; CHECK-LABEL: @test_int_x86_avx512_vpermilvar_ps_512_constant_pool(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP2]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP3:%.*]], label [[TMP4:%.*]], !prof [[PROF1]]
+; CHECK:       3:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       4:
+; CHECK-NEXT:    [[RES:%.*]] = call <16 x float> @llvm.x86.avx512.vpermilvar.ps.512(<16 x float> [[X0:%.*]], <16 x i32> <i32 1, i32 0, i32 3, i32 2, i32 0, i32 1, i32 2, i32 3, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 1, i32 0>)
+; CHECK-NEXT:    store <16 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x float> [[RES]]
+;
+  %res = call <16 x float> @llvm.x86.avx512.vpermilvar.ps.512(<16 x float> %x0, <16 x i32> <i32 1, i32 0, i32 3, i32 2, i32 0, i32 1, i32 2, i32 3, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 1, i32 0>)
+  ret <16 x float> %res
+}
+
+define <16 x float>@test_int_x86_avx512_vpermilvar_ps_512_constant_pool_mask(<16 x float> %x0, <16 x i32> %x1, <16 x float> %x2, i16 %mask) #0 {
+; CHECK-LABEL: @test_int_x86_avx512_vpermilvar_ps_512_constant_pool_mask(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP4]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP5:%.*]], label [[TMP6:%.*]], !prof [[PROF1]]
+; CHECK:       5:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       6:
+; CHECK-NEXT:    [[RES:%.*]] = call <16 x float> @llvm.x86.avx512.vpermilvar.ps.512(<16 x float> [[X0:%.*]], <16 x i32> <i32 1, i32 0, i32 3, i32 2, i32 0, i32 1, i32 2, i32 3, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 1, i32 0>)
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast i16 [[TMP2]] to <16 x i1>
+; CHECK-NEXT:    [[MASK_CAST:%.*]] = bitcast i16 [[MASK:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP8:%.*]] = select <16 x i1> [[MASK_CAST]], <16 x i32> zeroinitializer, <16 x i32> [[TMP3]]
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <16 x float> [[RES]] to <16 x i32>
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast <16 x float> [[X2:%.*]] to <16 x i32>
+; CHECK-NEXT:    [[TMP11:%.*]] = xor <16 x i32> [[TMP9]], [[TMP10]]
+; CHECK-NEXT:    [[TMP12:%.*]] = or <16 x i32> [[TMP11]], zeroinitializer
+; CHECK-NEXT:    [[TMP13:%.*]] = or <16 x i32> [[TMP12]], [[TMP3]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP7]], <16 x i32> [[TMP13]], <16 x i32> [[TMP8]]
+; CHECK-NEXT:    [[RES2:%.*]] = select <16 x i1> [[MASK_CAST]], <16 x float> [[RES]], <16 x float> [[X2]]
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x float> [[RES2]]
+;
+  %res = call <16 x float> @llvm.x86.avx512.vpermilvar.ps.512(<16 x float> %x0, <16 x i32> <i32 1, i32 0, i32 3, i32 2, i32 0, i32 1, i32 2, i32 3, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 1, i32 0>)
+  %mask.cast = bitcast i16 %mask to <16 x i1>
+  %res2 = select <16 x i1> %mask.cast, <16 x float> %res, <16 x float> %x2
+  ret <16 x float> %res2
+}
+
+define <16 x float>@test_int_x86_avx512_vpermilvar_ps_512_constant_pool_maskz(<16 x float> %x0, <16 x i32> %x1, i16 %mask) #0 {
+; CHECK-LABEL: @test_int_x86_avx512_vpermilvar_ps_512_constant_pool_maskz(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP3]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP4:%.*]], label [[TMP5:%.*]], !prof [[PROF1]]
+; CHECK:       4:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       5:
+; CHECK-NEXT:    [[RES:%.*]] = call <16 x float> @llvm.x86.avx512.vpermilvar.ps.512(<16 x float> [[X0:%.*]], <16 x i32> <i32 1, i32 0, i32 3, i32 2, i32 0, i32 1, i32 2, i32 3, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 1, i32 0>)
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i16 [[TMP2]] to <16 x i1>
+; CHECK-NEXT:    [[MASK_CAST:%.*]] = bitcast i16 [[MASK:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP7:%.*]] = select <16 x i1> [[MASK_CAST]], <16 x i32> zeroinitializer, <16 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <16 x float> [[RES]] to <16 x i32>
+; CHECK-NEXT:    [[TMP9:%.*]] = xor <16 x i32> [[TMP8]], zeroinitializer
+; CHECK-NEXT:    [[TMP10:%.*]] = or <16 x i32> [[TMP9]], zeroinitializer
+; CHECK-NEXT:    [[TMP11:%.*]] = or <16 x i32> [[TMP10]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP6]], <16 x i32> [[TMP11]], <16 x i32> [[TMP7]]
+; CHECK-NEXT:    [[RES2:%.*]] = select <16 x i1> [[MASK_CAST]], <16 x float> [[RES]], <16 x float> zeroinitializer
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x float> [[RES2]]
+;
+  %res = call <16 x float> @llvm.x86.avx512.vpermilvar.ps.512(<16 x float> %x0, <16 x i32> <i32 1, i32 0, i32 3, i32 2, i32 0, i32 1, i32 2, i32 3, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 1, i32 0>)
+  %mask.cast = bitcast i16 %mask to <16 x i1>
+  %res2 = select <16 x i1> %mask.cast, <16 x float> %res, <16 x float> zeroinitializer
+  ret <16 x float> %res2
+}
+
+declare <2 x double> @llvm.x86.avx512.mask.cvtss2sd.round(<2 x double>, <4 x float>, <2 x double>, i8, i32)
+
+define <2 x double>@test_int_x86_avx512_mask_cvt_ss2sd_round(<2 x double> %x0,<4 x float> %x1, <2 x double> %x2, i8 %x3) #0 {
+; CHECK-LABEL: @test_int_x86_avx512_mask_cvt_ss2sd_round(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 48) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <2 x i64> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <4 x i32> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP6]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <2 x i64> [[TMP3]] to i128
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i128 [[TMP7]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    [[_MSCMP4:%.*]] = icmp ne i8 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR5:%.*]] = or i1 [[_MSOR3]], [[_MSCMP4]]
+; CHECK-NEXT:    br i1 [[_MSOR5]], label [[TMP8:%.*]], label [[TMP9:%.*]], !prof [[PROF1]]
+; CHECK:       8:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       9:
+; CHECK-NEXT:    [[RES:%.*]] = call <2 x double> @llvm.x86.avx512.mask.cvtss2sd.round(<2 x double> [[X0:%.*]], <4 x float> [[X1:%.*]], <2 x double> [[X2:%.*]], i8 [[X3:%.*]], i32 4)
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast <2 x i64> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP6:%.*]] = icmp ne i128 [[TMP10]], 0
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <4 x i32> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP7:%.*]] = icmp ne i128 [[TMP11]], 0
+; CHECK-NEXT:    [[_MSOR8:%.*]] = or i1 [[_MSCMP6]], [[_MSCMP7]]
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <2 x i64> [[TMP3]] to i128
+; CHECK-NEXT:    [[_MSCMP9:%.*]] = icmp ne i128 [[TMP12]], 0
+; CHECK-NEXT:    [[_MSOR10:%.*]] = or i1 [[_MSOR8]], [[_MSCMP9]]
+; CHECK-NEXT:    br i1 [[_MSOR10]], label [[TMP13:%.*]], label [[TMP14:%.*]], !prof [[PROF1]]
+; CHECK:       13:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       14:
+; CHECK-NEXT:    [[RES1:%.*]] = call <2 x double> @llvm.x86.avx512.mask.cvtss2sd.round(<2 x double> [[X0]], <4 x float> [[X1]], <2 x double> [[X2]], i8 -1, i32 8)
+; CHECK-NEXT:    [[RES2:%.*]] = fadd <2 x double> [[RES]], [[RES1]]
+; CHECK-NEXT:    store <2 x i64> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x double> [[RES2]]
+;
+  %res = call <2 x double> @llvm.x86.avx512.mask.cvtss2sd.round(<2 x double> %x0, <4 x float> %x1, <2 x double> %x2, i8 %x3, i32 4)
+  %res1 = call <2 x double> @llvm.x86.avx512.mask.cvtss2sd.round(<2 x double> %x0, <4 x float> %x1, <2 x double> %x2, i8 -1, i32 8)
+  %res2 = fadd <2 x double> %res, %res1
+  ret <2 x double> %res2
+}
+
+declare <4 x float> @llvm.x86.avx512.mask.cvtsd2ss.round(<4 x float>, <2 x double>, <4 x float>, i8, i32)
+
+define <4 x float>@test_int_x86_avx512_mask_cvt_sd2ss_round(<4 x float> %x0,<2 x double> %x1, <4 x float> %x2, i8 %x3) #0 {
+; CHECK-LABEL: @test_int_x86_avx512_mask_cvt_sd2ss_round(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 48) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <4 x i32> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <2 x i64> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP6]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <4 x i32> [[TMP3]] to i128
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i128 [[TMP7]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    [[_MSCMP4:%.*]] = icmp ne i8 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR5:%.*]] = or i1 [[_MSOR3]], [[_MSCMP4]]
+; CHECK-NEXT:    br i1 [[_MSOR5]], label [[TMP8:%.*]], label [[TMP9:%.*]], !prof [[PROF1]]
+; CHECK:       8:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       9:
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.x86.avx512.mask.cvtsd2ss.round(<4 x float> [[X0:%.*]], <2 x double> [[X1:%.*]], <4 x float> [[X2:%.*]], i8 [[X3:%.*]], i32 11)
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast <4 x i32> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP6:%.*]] = icmp ne i128 [[TMP10]], 0
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <2 x i64> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP7:%.*]] = icmp ne i128 [[TMP11]], 0
+; CHECK-NEXT:    [[_MSOR8:%.*]] = or i1 [[_MSCMP6]], [[_MSCMP7]]
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <4 x i32> [[TMP3]] to i128
+; CHECK-NEXT:    [[_MSCMP9:%.*]] = icmp ne i128 [[TMP12]], 0
+; CHECK-NEXT:    [[_MSOR10:%.*]] = or i1 [[_MSOR8]], [[_MSCMP9]]
+; CHECK-NEXT:    br i1 [[_MSOR10]], label [[TMP13:%.*]], label [[TMP14:%.*]], !prof [[PROF1]]
+; CHECK:       13:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       14:
+; CHECK-NEXT:    [[RES1:%.*]] = call <4 x float> @llvm.x86.avx512.mask.cvtsd2ss.round(<4 x float> [[X0]], <2 x double> [[X1]], <4 x float> [[X2]], i8 -1, i32 8)
+; CHECK-NEXT:    [[RES2:%.*]] = fadd <4 x float> [[RES]], [[RES1]]
+; CHECK-NEXT:    store <4 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x float> [[RES2]]
+;
+  %res = call <4 x float> @llvm.x86.avx512.mask.cvtsd2ss.round(<4 x float> %x0, <2 x double> %x1, <4 x float> %x2, i8 %x3, i32 11)
+  %res1 = call <4 x float> @llvm.x86.avx512.mask.cvtsd2ss.round(<4 x float> %x0, <2 x double> %x1, <4 x float> %x2, i8 -1, i32 8)
+  %res2 = fadd <4 x float> %res, %res1
+  ret <4 x float> %res2
+}
+
+declare <16 x i32> @llvm.x86.avx512.pternlog.d.512(<16 x i32>, <16 x i32>, <16 x i32>, i32)
+
+define <16 x i32>@test_int_x86_avx512_pternlog_d_512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x2) #0 {
+; CHECK-LABEL: @test_int_x86_avx512_pternlog_d_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <16 x i32> [[TMP3]] to i512
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i512 [[TMP6]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR3]], label [[TMP7:%.*]], label [[TMP8:%.*]], !prof [[PROF1]]
+; CHECK:       7:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       8:
+; CHECK-NEXT:    [[TMP9:%.*]] = call <16 x i32> @llvm.x86.avx512.pternlog.d.512(<16 x i32> [[X0:%.*]], <16 x i32> [[X1:%.*]], <16 x i32> [[X2:%.*]], i32 33)
+; CHECK-NEXT:    store <16 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i32> [[TMP9]]
+;
+  %1 = call <16 x i32> @llvm.x86.avx512.pternlog.d.512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x2, i32 33)
+  ret <16 x i32> %1
+}
+
+define <16 x i32>@test_int_x86_avx512_mask_pternlog_d_512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x2, i16 %x4) #0 {
+; CHECK-LABEL: @test_int_x86_avx512_mask_pternlog_d_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP6]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <16 x i32> [[TMP3]] to i512
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i512 [[TMP7]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR3]], label [[TMP8:%.*]], label [[TMP9:%.*]], !prof [[PROF1]]
+; CHECK:       8:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       9:
+; CHECK-NEXT:    [[TMP10:%.*]] = call <16 x i32> @llvm.x86.avx512.pternlog.d.512(<16 x i32> [[X0:%.*]], <16 x i32> [[X1:%.*]], <16 x i32> [[X2:%.*]], i32 33)
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast i16 [[TMP4]] to <16 x i1>
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast i16 [[X4:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP13:%.*]] = select <16 x i1> [[TMP12]], <16 x i32> zeroinitializer, <16 x i32> [[TMP1]]
+; CHECK-NEXT:    [[TMP14:%.*]] = xor <16 x i32> [[TMP10]], [[X0]]
+; CHECK-NEXT:    [[TMP15:%.*]] = or <16 x i32> [[TMP14]], zeroinitializer
+; CHECK-NEXT:    [[TMP16:%.*]] = or <16 x i32> [[TMP15]], [[TMP1]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP11]], <16 x i32> [[TMP16]], <16 x i32> [[TMP13]]
+; CHECK-NEXT:    [[TMP17:%.*]] = select <16 x i1> [[TMP12]], <16 x i32> [[TMP10]], <16 x i32> [[X0]]
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i32> [[TMP17]]
+;
+  %1 = call <16 x i32> @llvm.x86.avx512.pternlog.d.512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x2, i32 33)
+  %2 = bitcast i16 %x4 to <16 x i1>
+  %3 = select <16 x i1> %2, <16 x i32> %1, <16 x i32> %x0
+  ret <16 x i32> %3
+}
+
+define <16 x i32>@test_int_x86_avx512_maskz_pternlog_d_512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x2, i16 %x4) #0 {
+; CHECK-LABEL: @test_int_x86_avx512_maskz_pternlog_d_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP6]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <16 x i32> [[TMP3]] to i512
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i512 [[TMP7]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR3]], label [[TMP8:%.*]], label [[TMP9:%.*]], !prof [[PROF1]]
+; CHECK:       8:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       9:
+; CHECK-NEXT:    [[TMP10:%.*]] = call <16 x i32> @llvm.x86.avx512.pternlog.d.512(<16 x i32> [[X0:%.*]], <16 x i32> [[X1:%.*]], <16 x i32> [[X2:%.*]], i32 33)
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast i16 [[TMP4]] to <16 x i1>
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast i16 [[X4:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP13:%.*]] = select <16 x i1> [[TMP12]], <16 x i32> zeroinitializer, <16 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP14:%.*]] = xor <16 x i32> [[TMP10]], zeroinitializer
+; CHECK-NEXT:    [[TMP15:%.*]] = or <16 x i32> [[TMP14]], zeroinitializer
+; CHECK-NEXT:    [[TMP16:%.*]] = or <16 x i32> [[TMP15]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP11]], <16 x i32> [[TMP16]], <16 x i32> [[TMP13]]
+; CHECK-NEXT:    [[TMP17:%.*]] = select <16 x i1> [[TMP12]], <16 x i32> [[TMP10]], <16 x i32> zeroinitializer
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i32> [[TMP17]]
+;
+  %1 = call <16 x i32> @llvm.x86.avx512.pternlog.d.512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x2, i32 33)
+  %2 = bitcast i16 %x4 to <16 x i1>
+  %3 = select <16 x i1> %2, <16 x i32> %1, <16 x i32> zeroinitializer
+  ret <16 x i32> %3
+}
+
+declare <8 x i64> @llvm.x86.avx512.pternlog.q.512(<8 x i64>, <8 x i64>, <8 x i64>, i32)
+
+define <8 x i64>@test_int_x86_avx512_pternlog_q_512(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> %x2) #0 {
+; CHECK-LABEL: @test_int_x86_avx512_pternlog_q_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <8 x i64> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <8 x i64> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <8 x i64> [[TMP3]] to i512
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i512 [[TMP6]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR3]], label [[TMP7:%.*]], label [[TMP8:%.*]], !prof [[PROF1]]
+; CHECK:       7:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       8:
+; CHECK-NEXT:    [[TMP9:%.*]] = call <8 x i64> @llvm.x86.avx512.pternlog.q.512(<8 x i64> [[X0:%.*]], <8 x i64> [[X1:%.*]], <8 x i64> [[X2:%.*]], i32 33)
+; CHECK-NEXT:    store <8 x i64> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i64> [[TMP9]]
+;
+  %1 = call <8 x i64> @llvm.x86.avx512.pternlog.q.512(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> %x2, i32 33)
+  ret <8 x i64> %1
+}
+
+define <8 x i64>@test_int_x86_avx512_mask_pternlog_q_512(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> %x2, i8 %x4) #0 {
+; CHECK-LABEL: @test_int_x86_avx512_mask_pternlog_q_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <8 x i64> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <8 x i64> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP6]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <8 x i64> [[TMP3]] to i512
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i512 [[TMP7]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR3]], label [[TMP8:%.*]], label [[TMP9:%.*]], !prof [[PROF1]]
+; CHECK:       8:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       9:
+; CHECK-NEXT:    [[TMP10:%.*]] = call <8 x i64> @llvm.x86.avx512.pternlog.q.512(<8 x i64> [[X0:%.*]], <8 x i64> [[X1:%.*]], <8 x i64> [[X2:%.*]], i32 33)
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast i8 [[TMP4]] to <8 x i1>
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast i8 [[X4:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP13:%.*]] = select <8 x i1> [[TMP12]], <8 x i64> zeroinitializer, <8 x i64> [[TMP1]]
+; CHECK-NEXT:    [[TMP14:%.*]] = xor <8 x i64> [[TMP10]], [[X0]]
+; CHECK-NEXT:    [[TMP15:%.*]] = or <8 x i64> [[TMP14]], zeroinitializer
+; CHECK-NEXT:    [[TMP16:%.*]] = or <8 x i64> [[TMP15]], [[TMP1]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP11]], <8 x i64> [[TMP16]], <8 x i64> [[TMP13]]
+; CHECK-NEXT:    [[TMP17:%.*]] = select <8 x i1> [[TMP12]], <8 x i64> [[TMP10]], <8 x i64> [[X0]]
+; CHECK-NEXT:    store <8 x i64> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i64> [[TMP17]]
+;
+  %1 = call <8 x i64> @llvm.x86.avx512.pternlog.q.512(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> %x2, i32 33)
+  %2 = bitcast i8 %x4 to <8 x i1>
+  %3 = select <8 x i1> %2, <8 x i64> %1, <8 x i64> %x0
+  ret <8 x i64> %3
+}
+
+define <8 x i64>@test_int_x86_avx512_maskz_pternlog_q_512(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> %x2, i8 %x4) #0 {
+; CHECK-LABEL: @test_int_x86_avx512_maskz_pternlog_q_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <8 x i64> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <8 x i64> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP6]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <8 x i64> [[TMP3]] to i512
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i512 [[TMP7]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR3]], label [[TMP8:%.*]], label [[TMP9:%.*]], !prof [[PROF1]]
+; CHECK:       8:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       9:
+; CHECK-NEXT:    [[TMP10:%.*]] = call <8 x i64> @llvm.x86.avx512.pternlog.q.512(<8 x i64> [[X0:%.*]], <8 x i64> [[X1:%.*]], <8 x i64> [[X2:%.*]], i32 33)
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast i8 [[TMP4]] to <8 x i1>
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast i8 [[X4:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP13:%.*]] = select <8 x i1> [[TMP12]], <8 x i64> zeroinitializer, <8 x i64> zeroinitializer
+; CHECK-NEXT:    [[TMP14:%.*]] = xor <8 x i64> [[TMP10]], zeroinitializer
+; CHECK-NEXT:    [[TMP15:%.*]] = or <8 x i64> [[TMP14]], zeroinitializer
+; CHECK-NEXT:    [[TMP16:%.*]] = or <8 x i64> [[TMP15]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP11]], <8 x i64> [[TMP16]], <8 x i64> [[TMP13]]
+; CHECK-NEXT:    [[TMP17:%.*]] = select <8 x i1> [[TMP12]], <8 x i64> [[TMP10]], <8 x i64> zeroinitializer
+; CHECK-NEXT:    store <8 x i64> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i64> [[TMP17]]
+;
+  %1 = call <8 x i64> @llvm.x86.avx512.pternlog.q.512(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> %x2, i32 33)
+  %2 = bitcast i8 %x4 to <8 x i1>
+  %3 = select <8 x i1> %2, <8 x i64> %1, <8 x i64> zeroinitializer
+  ret <8 x i64> %3
+}
+
+define i32 @test_x86_avx512_comi_sd_eq_sae(<2 x double> %a0, <2 x double> %a1) #0 {
+; CHECK-LABEL: @test_x86_avx512_comi_sd_eq_sae(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <2 x i64> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP3]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <2 x i64> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP5:%.*]], label [[TMP6:%.*]], !prof [[PROF1]]
+; CHECK:       5:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       6:
+; CHECK-NEXT:    [[RES:%.*]] = call i32 @llvm.x86.avx512.vcomi.sd(<2 x double> [[A0:%.*]], <2 x double> [[A1:%.*]], i32 0, i32 8)
+; CHECK-NEXT:    store i32 0, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i32 [[RES]]
+;
+  %res = call i32 @llvm.x86.avx512.vcomi.sd(<2 x double> %a0, <2 x double> %a1, i32 0, i32 8)
+  ret i32 %res
+}
+
+define i32 @test_x86_avx512_ucomi_sd_eq_sae(<2 x double> %a0, <2 x double> %a1) #0 {
+; CHECK-LABEL: @test_x86_avx512_ucomi_sd_eq_sae(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <2 x i64> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP3]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <2 x i64> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP5:%.*]], label [[TMP6:%.*]], !prof [[PROF1]]
+; CHECK:       5:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       6:
+; CHECK-NEXT:    [[RES:%.*]] = call i32 @llvm.x86.avx512.vcomi.sd(<2 x double> [[A0:%.*]], <2 x double> [[A1:%.*]], i32 8, i32 8)
+; CHECK-NEXT:    store i32 0, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i32 [[RES]]
+;
+  %res = call i32 @llvm.x86.avx512.vcomi.sd(<2 x double> %a0, <2 x double> %a1, i32 8, i32 8)
+  ret i32 %res
+}
+
+define i32 @test_x86_avx512_comi_sd_eq(<2 x double> %a0, <2 x double> %a1) #0 {
+; CHECK-LABEL: @test_x86_avx512_comi_sd_eq(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <2 x i64> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP3]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <2 x i64> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP5:%.*]], label [[TMP6:%.*]], !prof [[PROF1]]
+; CHECK:       5:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       6:
+; CHECK-NEXT:    [[RES:%.*]] = call i32 @llvm.x86.avx512.vcomi.sd(<2 x double> [[A0:%.*]], <2 x double> [[A1:%.*]], i32 0, i32 4)
+; CHECK-NEXT:    store i32 0, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i32 [[RES]]
+;
+  %res = call i32 @llvm.x86.avx512.vcomi.sd(<2 x double> %a0, <2 x double> %a1, i32 0, i32 4)
+  ret i32 %res
+}
+
+define i32 @test_x86_avx512_ucomi_sd_eq(<2 x double> %a0, <2 x double> %a1) #0 {
+; CHECK-LABEL: @test_x86_avx512_ucomi_sd_eq(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <2 x i64> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP3]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <2 x i64> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP5:%.*]], label [[TMP6:%.*]], !prof [[PROF1]]
+; CHECK:       5:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       6:
+; CHECK-NEXT:    [[RES:%.*]] = call i32 @llvm.x86.avx512.vcomi.sd(<2 x double> [[A0:%.*]], <2 x double> [[A1:%.*]], i32 8, i32 4)
+; CHECK-NEXT:    store i32 0, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i32 [[RES]]
+;
+  %res = call i32 @llvm.x86.avx512.vcomi.sd(<2 x double> %a0, <2 x double> %a1, i32 8, i32 4)
+  ret i32 %res
+}
+
+define i32 @test_x86_avx512_comi_sd_lt_sae(<2 x double> %a0, <2 x double> %a1) #0 {
+; CHECK-LABEL: @test_x86_avx512_comi_sd_lt_sae(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <2 x i64> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP3]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <2 x i64> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP5:%.*]], label [[TMP6:%.*]], !prof [[PROF1]]
+; CHECK:       5:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       6:
+; CHECK-NEXT:    [[RES:%.*]] = call i32 @llvm.x86.avx512.vcomi.sd(<2 x double> [[A0:%.*]], <2 x double> [[A1:%.*]], i32 1, i32 8)
+; CHECK-NEXT:    store i32 0, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i32 [[RES]]
+;
+  %res = call i32 @llvm.x86.avx512.vcomi.sd(<2 x double> %a0, <2 x double> %a1, i32 1, i32 8)
+  ret i32 %res
+}
+
+define i32 @test_x86_avx512_ucomi_sd_lt_sae(<2 x double> %a0, <2 x double> %a1) #0 {
+; CHECK-LABEL: @test_x86_avx512_ucomi_sd_lt_sae(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <2 x i64> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP3]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <2 x i64> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP5:%.*]], label [[TMP6:%.*]], !prof [[PROF1]]
+; CHECK:       5:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       6:
+; CHECK-NEXT:    [[RES:%.*]] = call i32 @llvm.x86.avx512.vcomi.sd(<2 x double> [[A0:%.*]], <2 x double> [[A1:%.*]], i32 9, i32 8)
+; CHECK-NEXT:    store i32 0, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i32 [[RES]]
+;
+  %res = call i32 @llvm.x86.avx512.vcomi.sd(<2 x double> %a0, <2 x double> %a1, i32 9, i32 8)
+  ret i32 %res
+}
+
+define i32 @test_x86_avx512_comi_sd_lt(<2 x double> %a0, <2 x double> %a1) #0 {
+; CHECK-LABEL: @test_x86_avx512_comi_sd_lt(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <2 x i64> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP3]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <2 x i64> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP5:%.*]], label [[TMP6:%.*]], !prof [[PROF1]]
+; CHECK:       5:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       6:
+; CHECK-NEXT:    [[RES:%.*]] = call i32 @llvm.x86.avx512.vcomi.sd(<2 x double> [[A0:%.*]], <2 x double> [[A1:%.*]], i32 1, i32 4)
+; CHECK-NEXT:    store i32 0, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i32 [[RES]]
+;
+  %res = call i32 @llvm.x86.avx512.vcomi.sd(<2 x double> %a0, <2 x double> %a1, i32 1, i32 4)
+  ret i32 %res
+}
+
+define i32 @test_x86_avx512_ucomi_sd_lt(<2 x double> %a0, <2 x double> %a1) #0 {
+; CHECK-LABEL: @test_x86_avx512_ucomi_sd_lt(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <2 x i64> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP3]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <2 x i64> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP5:%.*]], label [[TMP6:%.*]], !prof [[PROF1]]
+; CHECK:       5:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       6:
+; CHECK-NEXT:    [[RES:%.*]] = call i32 @llvm.x86.avx512.vcomi.sd(<2 x double> [[A0:%.*]], <2 x double> [[A1:%.*]], i32 9, i32 4)
+; CHECK-NEXT:    store i32 0, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i32 [[RES]]
+;
+  %res = call i32 @llvm.x86.avx512.vcomi.sd(<2 x double> %a0, <2 x double> %a1, i32 9, i32 4)
+  ret i32 %res
+}
+
+declare i32 @llvm.x86.avx512.vcomi.sd(<2 x double>, <2 x double>, i32, i32)
+
+define i32 @test_x86_avx512_ucomi_ss_lt(<4 x float> %a0, <4 x float> %a1) #0 {
+; CHECK-LABEL: @test_x86_avx512_ucomi_ss_lt(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <4 x i32> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP3]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <4 x i32> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP5:%.*]], label [[TMP6:%.*]], !prof [[PROF1]]
+; CHECK:       5:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       6:
+; CHECK-NEXT:    [[RES:%.*]] = call i32 @llvm.x86.avx512.vcomi.ss(<4 x float> [[A0:%.*]], <4 x float> [[A1:%.*]], i32 9, i32 4)
+; CHECK-NEXT:    store i32 0, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i32 [[RES]]
+;
+  %res = call i32 @llvm.x86.avx512.vcomi.ss(<4 x float> %a0, <4 x float> %a1, i32 9, i32 4)
+  ret i32 %res
+}
+
+declare i32 @llvm.x86.avx512.vcomi.ss(<4 x float>, <4 x float>, i32, i32)
+
+declare <8 x double> @llvm.x86.avx512.permvar.df.512(<8 x double>, <8 x i64>)
+
+define <8 x double>@test_int_x86_avx512_permvar_df_512(<8 x double> %x0, <8 x i64> %x1) #0 {
+; CHECK-LABEL: @test_int_x86_avx512_permvar_df_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <8 x i64> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP3]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <8 x i64> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP5:%.*]], label [[TMP6:%.*]], !prof [[PROF1]]
+; CHECK:       5:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       6:
+; CHECK-NEXT:    [[TMP7:%.*]] = call <8 x double> @llvm.x86.avx512.permvar.df.512(<8 x double> [[X0:%.*]], <8 x i64> [[X1:%.*]])
+; CHECK-NEXT:    store <8 x i64> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x double> [[TMP7]]
+;
+  %1 = call <8 x double> @llvm.x86.avx512.permvar.df.512(<8 x double> %x0, <8 x i64> %x1)
+  ret <8 x double> %1
+}
+
+define <8 x double>@test_int_x86_avx512_mask_permvar_df_512(<8 x double> %x0, <8 x i64> %x1, <8 x double> %x2, i8 %x3) #0 {
+; CHECK-LABEL: @test_int_x86_avx512_mask_permvar_df_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <8 x i64> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <8 x i64> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP6]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP7:%.*]], label [[TMP8:%.*]], !prof [[PROF1]]
+; CHECK:       7:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       8:
+; CHECK-NEXT:    [[TMP9:%.*]] = call <8 x double> @llvm.x86.avx512.permvar.df.512(<8 x double> [[X0:%.*]], <8 x i64> [[X1:%.*]])
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i8 [[TMP3]] to <8 x i1>
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast i8 [[X3:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP12:%.*]] = select <8 x i1> [[TMP11]], <8 x i64> zeroinitializer, <8 x i64> [[TMP4]]
+; CHECK-NEXT:    [[TMP13:%.*]] = bitcast <8 x double> [[TMP9]] to <8 x i64>
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast <8 x double> [[X2:%.*]] to <8 x i64>
+; CHECK-NEXT:    [[TMP15:%.*]] = xor <8 x i64> [[TMP13]], [[TMP14]]
+; CHECK-NEXT:    [[TMP16:%.*]] = or <8 x i64> [[TMP15]], zeroinitializer
+; CHECK-NEXT:    [[TMP17:%.*]] = or <8 x i64> [[TMP16]], [[TMP4]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP10]], <8 x i64> [[TMP17]], <8 x i64> [[TMP12]]
+; CHECK-NEXT:    [[TMP18:%.*]] = select <8 x i1> [[TMP11]], <8 x double> [[TMP9]], <8 x double> [[X2]]
+; CHECK-NEXT:    store <8 x i64> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x double> [[TMP18]]
+;
+  %1 = call <8 x double> @llvm.x86.avx512.permvar.df.512(<8 x double> %x0, <8 x i64> %x1)
+  %2 = bitcast i8 %x3 to <8 x i1>
+  %3 = select <8 x i1> %2, <8 x double> %1, <8 x double> %x2
+  ret <8 x double> %3
+}
+
+define <8 x double>@test_int_x86_avx512_maskz_permvar_df_512(<8 x double> %x0, <8 x i64> %x1, i8 %x3) #0 {
+; CHECK-LABEL: @test_int_x86_avx512_maskz_permvar_df_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <8 x i64> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <8 x i64> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP6:%.*]], label [[TMP7:%.*]], !prof [[PROF1]]
+; CHECK:       6:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       7:
+; CHECK-NEXT:    [[TMP8:%.*]] = call <8 x double> @llvm.x86.avx512.permvar.df.512(<8 x double> [[X0:%.*]], <8 x i64> [[X1:%.*]])
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast i8 [[TMP3]] to <8 x i1>
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i8 [[X3:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP11:%.*]] = select <8 x i1> [[TMP10]], <8 x i64> zeroinitializer, <8 x i64> zeroinitializer
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <8 x double> [[TMP8]] to <8 x i64>
+; CHECK-NEXT:    [[TMP13:%.*]] = xor <8 x i64> [[TMP12]], zeroinitializer
+; CHECK-NEXT:    [[TMP14:%.*]] = or <8 x i64> [[TMP13]], zeroinitializer
+; CHECK-NEXT:    [[TMP15:%.*]] = or <8 x i64> [[TMP14]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP9]], <8 x i64> [[TMP15]], <8 x i64> [[TMP11]]
+; CHECK-NEXT:    [[TMP16:%.*]] = select <8 x i1> [[TMP10]], <8 x double> [[TMP8]], <8 x double> zeroinitializer
+; CHECK-NEXT:    store <8 x i64> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x double> [[TMP16]]
+;
+  %1 = call <8 x double> @llvm.x86.avx512.permvar.df.512(<8 x double> %x0, <8 x i64> %x1)
+  %2 = bitcast i8 %x3 to <8 x i1>
+  %3 = select <8 x i1> %2, <8 x double> %1, <8 x double> zeroinitializer
+  ret <8 x double> %3
+}
+
+declare <8 x i64> @llvm.x86.avx512.permvar.di.512(<8 x i64>, <8 x i64>)
+
+define <8 x i64>@test_int_x86_avx512_permvar_di_512(<8 x i64> %x0, <8 x i64> %x1) #0 {
+; CHECK-LABEL: @test_int_x86_avx512_permvar_di_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <8 x i64> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP3:%.*]] = call <8 x i64> @llvm.x86.avx512.permvar.di.512(<8 x i64> [[X0:%.*]], <8 x i64> [[X1:%.*]])
+; CHECK-NEXT:    store <8 x i64> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i64> [[TMP3]]
+;
+  %1 = call <8 x i64> @llvm.x86.avx512.permvar.di.512(<8 x i64> %x0, <8 x i64> %x1)
+  ret <8 x i64> %1
+}
+
+define <8 x i64>@test_int_x86_avx512_mask_permvar_di_512(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> %x2, i8 %x3) #0 {
+; CHECK-LABEL: @test_int_x86_avx512_mask_permvar_di_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <8 x i64> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP5:%.*]] = call <8 x i64> @llvm.x86.avx512.permvar.di.512(<8 x i64> [[X0:%.*]], <8 x i64> [[X1:%.*]])
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i8 [[TMP3]] to <8 x i1>
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast i8 [[X3:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP8:%.*]] = select <8 x i1> [[TMP7]], <8 x i64> [[_MSPROP]], <8 x i64> [[TMP4]]
+; CHECK-NEXT:    [[TMP9:%.*]] = xor <8 x i64> [[TMP5]], [[X2:%.*]]
+; CHECK-NEXT:    [[TMP10:%.*]] = or <8 x i64> [[TMP9]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP11:%.*]] = or <8 x i64> [[TMP10]], [[TMP4]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP6]], <8 x i64> [[TMP11]], <8 x i64> [[TMP8]]
+; CHECK-NEXT:    [[TMP12:%.*]] = select <8 x i1> [[TMP7]], <8 x i64> [[TMP5]], <8 x i64> [[X2]]
+; CHECK-NEXT:    store <8 x i64> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i64> [[TMP12]]
+;
+  %1 = call <8 x i64> @llvm.x86.avx512.permvar.di.512(<8 x i64> %x0, <8 x i64> %x1)
+  %2 = bitcast i8 %x3 to <8 x i1>
+  %3 = select <8 x i1> %2, <8 x i64> %1, <8 x i64> %x2
+  ret <8 x i64> %3
+}
+
+define <8 x i64>@test_int_x86_avx512_maskz_permvar_di_512(<8 x i64> %x0, <8 x i64> %x1, i8 %x3) #0 {
+; CHECK-LABEL: @test_int_x86_avx512_maskz_permvar_di_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <8 x i64> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = call <8 x i64> @llvm.x86.avx512.permvar.di.512(<8 x i64> [[X0:%.*]], <8 x i64> [[X1:%.*]])
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i8 [[TMP3]] to <8 x i1>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i8 [[X3:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP7:%.*]] = select <8 x i1> [[TMP6]], <8 x i64> [[_MSPROP]], <8 x i64> zeroinitializer
+; CHECK-NEXT:    [[TMP8:%.*]] = xor <8 x i64> [[TMP4]], zeroinitializer
+; CHECK-NEXT:    [[TMP9:%.*]] = or <8 x i64> [[TMP8]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP10:%.*]] = or <8 x i64> [[TMP9]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP5]], <8 x i64> [[TMP10]], <8 x i64> [[TMP7]]
+; CHECK-NEXT:    [[TMP11:%.*]] = select <8 x i1> [[TMP6]], <8 x i64> [[TMP4]], <8 x i64> zeroinitializer
+; CHECK-NEXT:    store <8 x i64> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i64> [[TMP11]]
+;
+  %1 = call <8 x i64> @llvm.x86.avx512.permvar.di.512(<8 x i64> %x0, <8 x i64> %x1)
+  %2 = bitcast i8 %x3 to <8 x i1>
+  %3 = select <8 x i1> %2, <8 x i64> %1, <8 x i64> zeroinitializer
+  ret <8 x i64> %3
+}
+
+declare <16 x float> @llvm.x86.avx512.permvar.sf.512(<16 x float>, <16 x i32>)
+
+define <16 x float>@test_int_x86_avx512_permvar_sf_512(<16 x float> %x0, <16 x i32> %x1) #0 {
+; CHECK-LABEL: @test_int_x86_avx512_permvar_sf_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP3]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP5:%.*]], label [[TMP6:%.*]], !prof [[PROF1]]
+; CHECK:       5:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       6:
+; CHECK-NEXT:    [[TMP7:%.*]] = call <16 x float> @llvm.x86.avx512.permvar.sf.512(<16 x float> [[X0:%.*]], <16 x i32> [[X1:%.*]])
+; CHECK-NEXT:    store <16 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x float> [[TMP7]]
+;
+  %1 = call <16 x float> @llvm.x86.avx512.permvar.sf.512(<16 x float> %x0, <16 x i32> %x1)
+  ret <16 x float> %1
+}
+
+define <16 x float>@test_int_x86_avx512_mask_permvar_sf_512(<16 x float> %x0, <16 x i32> %x1, <16 x float> %x2, i16 %x3) #0 {
+; CHECK-LABEL: @test_int_x86_avx512_mask_permvar_sf_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP6]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP7:%.*]], label [[TMP8:%.*]], !prof [[PROF1]]
+; CHECK:       7:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       8:
+; CHECK-NEXT:    [[TMP9:%.*]] = call <16 x float> @llvm.x86.avx512.permvar.sf.512(<16 x float> [[X0:%.*]], <16 x i32> [[X1:%.*]])
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i16 [[TMP3]] to <16 x i1>
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast i16 [[X3:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP12:%.*]] = select <16 x i1> [[TMP11]], <16 x i32> zeroinitializer, <16 x i32> [[TMP4]]
+; CHECK-NEXT:    [[TMP13:%.*]] = bitcast <16 x float> [[TMP9]] to <16 x i32>
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast <16 x float> [[X2:%.*]] to <16 x i32>
+; CHECK-NEXT:    [[TMP15:%.*]] = xor <16 x i32> [[TMP13]], [[TMP14]]
+; CHECK-NEXT:    [[TMP16:%.*]] = or <16 x i32> [[TMP15]], zeroinitializer
+; CHECK-NEXT:    [[TMP17:%.*]] = or <16 x i32> [[TMP16]], [[TMP4]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP10]], <16 x i32> [[TMP17]], <16 x i32> [[TMP12]]
+; CHECK-NEXT:    [[TMP18:%.*]] = select <16 x i1> [[TMP11]], <16 x float> [[TMP9]], <16 x float> [[X2]]
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x float> [[TMP18]]
+;
+  %1 = call <16 x float> @llvm.x86.avx512.permvar.sf.512(<16 x float> %x0, <16 x i32> %x1)
+  %2 = bitcast i16 %x3 to <16 x i1>
+  %3 = select <16 x i1> %2, <16 x float> %1, <16 x float> %x2
+  ret <16 x float> %3
+}
+
+define <16 x float>@test_int_x86_avx512_maskz_permvar_sf_512(<16 x float> %x0, <16 x i32> %x1, i16 %x3) #0 {
+; CHECK-LABEL: @test_int_x86_avx512_maskz_permvar_sf_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP6:%.*]], label [[TMP7:%.*]], !prof [[PROF1]]
+; CHECK:       6:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       7:
+; CHECK-NEXT:    [[TMP8:%.*]] = call <16 x float> @llvm.x86.avx512.permvar.sf.512(<16 x float> [[X0:%.*]], <16 x i32> [[X1:%.*]])
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast i16 [[TMP3]] to <16 x i1>
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i16 [[X3:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP11:%.*]] = select <16 x i1> [[TMP10]], <16 x i32> zeroinitializer, <16 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <16 x float> [[TMP8]] to <16 x i32>
+; CHECK-NEXT:    [[TMP13:%.*]] = xor <16 x i32> [[TMP12]], zeroinitializer
+; CHECK-NEXT:    [[TMP14:%.*]] = or <16 x i32> [[TMP13]], zeroinitializer
+; CHECK-NEXT:    [[TMP15:%.*]] = or <16 x i32> [[TMP14]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP9]], <16 x i32> [[TMP15]], <16 x i32> [[TMP11]]
+; CHECK-NEXT:    [[TMP16:%.*]] = select <16 x i1> [[TMP10]], <16 x float> [[TMP8]], <16 x float> zeroinitializer
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x float> [[TMP16]]
+;
+  %1 = call <16 x float> @llvm.x86.avx512.permvar.sf.512(<16 x float> %x0, <16 x i32> %x1)
+  %2 = bitcast i16 %x3 to <16 x i1>
+  %3 = select <16 x i1> %2, <16 x float> %1, <16 x float> zeroinitializer
+  ret <16 x float> %3
+}
+
+declare <16 x i32> @llvm.x86.avx512.permvar.si.512(<16 x i32>, <16 x i32>)
+
+define <16 x i32>@test_int_x86_avx512_permvar_si_512(<16 x i32> %x0, <16 x i32> %x1) #0 {
+; CHECK-LABEL: @test_int_x86_avx512_permvar_si_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <16 x i32> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP3:%.*]] = call <16 x i32> @llvm.x86.avx512.permvar.si.512(<16 x i32> [[X0:%.*]], <16 x i32> [[X1:%.*]])
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i32> [[TMP3]]
+;
+  %1 = call <16 x i32> @llvm.x86.avx512.permvar.si.512(<16 x i32> %x0, <16 x i32> %x1)
+  ret <16 x i32> %1
+}
+
+define <16 x i32>@test_int_x86_avx512_mask_permvar_si_512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x2, i16 %x3) #0 {
+; CHECK-LABEL: @test_int_x86_avx512_mask_permvar_si_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <16 x i32> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP5:%.*]] = call <16 x i32> @llvm.x86.avx512.permvar.si.512(<16 x i32> [[X0:%.*]], <16 x i32> [[X1:%.*]])
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i16 [[TMP3]] to <16 x i1>
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast i16 [[X3:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP8:%.*]] = select <16 x i1> [[TMP7]], <16 x i32> [[_MSPROP]], <16 x i32> [[TMP4]]
+; CHECK-NEXT:    [[TMP9:%.*]] = xor <16 x i32> [[TMP5]], [[X2:%.*]]
+; CHECK-NEXT:    [[TMP10:%.*]] = or <16 x i32> [[TMP9]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP11:%.*]] = or <16 x i32> [[TMP10]], [[TMP4]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP6]], <16 x i32> [[TMP11]], <16 x i32> [[TMP8]]
+; CHECK-NEXT:    [[TMP12:%.*]] = select <16 x i1> [[TMP7]], <16 x i32> [[TMP5]], <16 x i32> [[X2]]
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i32> [[TMP12]]
+;
+  %1 = call <16 x i32> @llvm.x86.avx512.permvar.si.512(<16 x i32> %x0, <16 x i32> %x1)
+  %2 = bitcast i16 %x3 to <16 x i1>
+  %3 = select <16 x i1> %2, <16 x i32> %1, <16 x i32> %x2
+  ret <16 x i32> %3
+}
+
+define <16 x i32>@test_int_x86_avx512_maskz_permvar_si_512(<16 x i32> %x0, <16 x i32> %x1, i16 %x3) #0 {
+; CHECK-LABEL: @test_int_x86_avx512_maskz_permvar_si_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <16 x i32> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = call <16 x i32> @llvm.x86.avx512.permvar.si.512(<16 x i32> [[X0:%.*]], <16 x i32> [[X1:%.*]])
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i16 [[TMP3]] to <16 x i1>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i16 [[X3:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP7:%.*]] = select <16 x i1> [[TMP6]], <16 x i32> [[_MSPROP]], <16 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP8:%.*]] = xor <16 x i32> [[TMP4]], zeroinitializer
+; CHECK-NEXT:    [[TMP9:%.*]] = or <16 x i32> [[TMP8]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP10:%.*]] = or <16 x i32> [[TMP9]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP5]], <16 x i32> [[TMP10]], <16 x i32> [[TMP7]]
+; CHECK-NEXT:    [[TMP11:%.*]] = select <16 x i1> [[TMP6]], <16 x i32> [[TMP4]], <16 x i32> zeroinitializer
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i32> [[TMP11]]
+;
+  %1 = call <16 x i32> @llvm.x86.avx512.permvar.si.512(<16 x i32> %x0, <16 x i32> %x1)
+  %2 = bitcast i16 %x3 to <16 x i1>
+  %3 = select <16 x i1> %2, <16 x i32> %1, <16 x i32> zeroinitializer
+  ret <16 x i32> %3
+}
+
+declare <8 x double> @llvm.x86.avx512.mask.fixupimm.pd.512(<8 x double>, <8 x double>, <8 x i64>, i32, i8, i32)
+
+define <8 x double>@test_int_x86_avx512_mask_fixupimm_pd_512(<8 x double> %x0, <8 x double> %x1, <8 x i64> %x2, i8 %x4) #0 {
+; CHECK-LABEL: @test_int_x86_avx512_mask_fixupimm_pd_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <8 x i64> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <8 x i64> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP6]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <8 x i64> [[TMP3]] to i512
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i512 [[TMP7]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    [[_MSCMP4:%.*]] = icmp ne i8 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR5:%.*]] = or i1 [[_MSOR3]], [[_MSCMP4]]
+; CHECK-NEXT:    br i1 [[_MSOR5]], label [[TMP8:%.*]], label [[TMP9:%.*]], !prof [[PROF1]]
+; CHECK:       8:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       9:
+; CHECK-NEXT:    [[RES:%.*]] = call <8 x double> @llvm.x86.avx512.mask.fixupimm.pd.512(<8 x double> [[X0:%.*]], <8 x double> [[X1:%.*]], <8 x i64> [[X2:%.*]], i32 4, i8 [[X4:%.*]], i32 4)
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast <8 x i64> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP6:%.*]] = icmp ne i512 [[TMP10]], 0
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <8 x i64> [[TMP3]] to i512
+; CHECK-NEXT:    [[_MSCMP7:%.*]] = icmp ne i512 [[TMP11]], 0
+; CHECK-NEXT:    [[_MSOR8:%.*]] = or i1 [[_MSCMP6]], [[_MSCMP7]]
+; CHECK-NEXT:    [[_MSCMP9:%.*]] = icmp ne i8 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR10:%.*]] = or i1 [[_MSOR8]], [[_MSCMP9]]
+; CHECK-NEXT:    br i1 [[_MSOR10]], label [[TMP12:%.*]], label [[TMP13:%.*]], !prof [[PROF1]]
+; CHECK:       12:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       13:
+; CHECK-NEXT:    [[RES1:%.*]] = call <8 x double> @llvm.x86.avx512.mask.fixupimm.pd.512(<8 x double> zeroinitializer, <8 x double> [[X1]], <8 x i64> [[X2]], i32 5, i8 [[X4]], i32 4)
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast <8 x i64> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP11:%.*]] = icmp ne i512 [[TMP14]], 0
+; CHECK-NEXT:    [[TMP15:%.*]] = bitcast <8 x i64> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP12:%.*]] = icmp ne i512 [[TMP15]], 0
+; CHECK-NEXT:    [[_MSOR13:%.*]] = or i1 [[_MSCMP11]], [[_MSCMP12]]
+; CHECK-NEXT:    [[TMP16:%.*]] = bitcast <8 x i64> [[TMP3]] to i512
+; CHECK-NEXT:    [[_MSCMP14:%.*]] = icmp ne i512 [[TMP16]], 0
+; CHECK-NEXT:    [[_MSOR15:%.*]] = or i1 [[_MSOR13]], [[_MSCMP14]]
+; CHECK-NEXT:    br i1 [[_MSOR15]], label [[TMP17:%.*]], label [[TMP18:%.*]], !prof [[PROF1]]
+; CHECK:       17:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       18:
+; CHECK-NEXT:    [[RES2:%.*]] = call <8 x double> @llvm.x86.avx512.mask.fixupimm.pd.512(<8 x double> [[X0]], <8 x double> [[X1]], <8 x i64> [[X2]], i32 3, i8 -1, i32 8)
+; CHECK-NEXT:    [[RES3:%.*]] = fadd <8 x double> [[RES]], [[RES1]]
+; CHECK-NEXT:    [[RES4:%.*]] = fadd <8 x double> [[RES3]], [[RES2]]
+; CHECK-NEXT:    store <8 x i64> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x double> [[RES4]]
+;
+  %res = call <8 x double> @llvm.x86.avx512.mask.fixupimm.pd.512(<8 x double> %x0, <8 x double> %x1, <8 x i64> %x2, i32 4, i8 %x4, i32 4)
+  %res1 = call <8 x double> @llvm.x86.avx512.mask.fixupimm.pd.512(<8 x double> zeroinitializer, <8 x double> %x1, <8 x i64> %x2, i32 5, i8 %x4, i32 4)
+  %res2 = call <8 x double> @llvm.x86.avx512.mask.fixupimm.pd.512(<8 x double> %x0, <8 x double> %x1, <8 x i64> %x2, i32 3, i8 -1, i32 8)
+  %res3 = fadd <8 x double> %res, %res1
+  %res4 = fadd <8 x double> %res3, %res2
+  ret <8 x double> %res4
+}
+
+define <8 x double>@test_int_x86_avx512_mask_fixupimm_pd_512_load(<8 x double> %x0, <8 x double> %x1, ptr %x2ptr) #0 {
+; CHECK-LABEL: @test_int_x86_avx512_mask_fixupimm_pd_512_load(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP4:%.*]], label [[TMP5:%.*]], !prof [[PROF1]]
+; CHECK:       4:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       5:
+; CHECK-NEXT:    [[X2:%.*]] = load <8 x i64>, ptr [[X2PTR:%.*]], align 64
+; CHECK-NEXT:    [[TMP6:%.*]] = ptrtoint ptr [[X2PTR]] to i64
+; CHECK-NEXT:    [[TMP7:%.*]] = xor i64 [[TMP6]], 87960930222080
+; CHECK-NEXT:    [[TMP8:%.*]] = inttoptr i64 [[TMP7]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <8 x i64>, ptr [[TMP8]], align 64
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <8 x i64> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP9]], 0
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast <8 x i64> [[TMP3]] to i512
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i512 [[TMP10]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP1]], [[_MSCMP2]]
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <8 x i64> [[_MSLD]] to i512
+; CHECK-NEXT:    [[_MSCMP3:%.*]] = icmp ne i512 [[TMP11]], 0
+; CHECK-NEXT:    [[_MSOR4:%.*]] = or i1 [[_MSOR]], [[_MSCMP3]]
+; CHECK-NEXT:    br i1 [[_MSOR4]], label [[TMP12:%.*]], label [[TMP13:%.*]], !prof [[PROF1]]
+; CHECK:       12:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       13:
+; CHECK-NEXT:    [[RES:%.*]] = call <8 x double> @llvm.x86.avx512.mask.fixupimm.pd.512(<8 x double> [[X0:%.*]], <8 x double> [[X1:%.*]], <8 x i64> [[X2]], i32 3, i8 -1, i32 4)
+; CHECK-NEXT:    store <8 x i64> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x double> [[RES]]
+;
+  %x2 = load <8 x i64>, ptr %x2ptr
+  %res = call <8 x double> @llvm.x86.avx512.mask.fixupimm.pd.512(<8 x double> %x0, <8 x double> %x1, <8 x i64> %x2, i32 3, i8 -1, i32 4)
+  ret <8 x double> %res
+}
+
+declare <8 x double> @llvm.x86.avx512.maskz.fixupimm.pd.512(<8 x double>, <8 x double>, <8 x i64>, i32, i8, i32)
+
+define <8 x double>@test_int_x86_avx512_maskz_fixupimm_pd_512(<8 x double> %x0, <8 x double> %x1, <8 x i64> %x2, i8 %x4) #0 {
+; CHECK-LABEL: @test_int_x86_avx512_maskz_fixupimm_pd_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <8 x i64> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <8 x i64> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP6]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <8 x i64> [[TMP3]] to i512
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i512 [[TMP7]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    [[_MSCMP4:%.*]] = icmp ne i8 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR5:%.*]] = or i1 [[_MSOR3]], [[_MSCMP4]]
+; CHECK-NEXT:    br i1 [[_MSOR5]], label [[TMP8:%.*]], label [[TMP9:%.*]], !prof [[PROF1]]
+; CHECK:       8:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       9:
+; CHECK-NEXT:    [[RES:%.*]] = call <8 x double> @llvm.x86.avx512.maskz.fixupimm.pd.512(<8 x double> [[X0:%.*]], <8 x double> [[X1:%.*]], <8 x i64> [[X2:%.*]], i32 3, i8 [[X4:%.*]], i32 4)
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast <8 x i64> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP6:%.*]] = icmp ne i512 [[TMP10]], 0
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <8 x i64> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP7:%.*]] = icmp ne i512 [[TMP11]], 0
+; CHECK-NEXT:    [[_MSOR8:%.*]] = or i1 [[_MSCMP6]], [[_MSCMP7]]
+; CHECK-NEXT:    [[_MSCMP9:%.*]] = icmp ne i8 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR10:%.*]] = or i1 [[_MSOR8]], [[_MSCMP9]]
+; CHECK-NEXT:    br i1 [[_MSOR10]], label [[TMP12:%.*]], label [[TMP13:%.*]], !prof [[PROF1]]
+; CHECK:       12:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       13:
+; CHECK-NEXT:    [[RES1:%.*]] = call <8 x double> @llvm.x86.avx512.maskz.fixupimm.pd.512(<8 x double> [[X0]], <8 x double> [[X1]], <8 x i64> zeroinitializer, i32 5, i8 [[X4]], i32 4)
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast <8 x i64> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP11:%.*]] = icmp ne i512 [[TMP14]], 0
+; CHECK-NEXT:    [[TMP15:%.*]] = bitcast <8 x i64> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP12:%.*]] = icmp ne i512 [[TMP15]], 0
+; CHECK-NEXT:    [[_MSOR13:%.*]] = or i1 [[_MSCMP11]], [[_MSCMP12]]
+; CHECK-NEXT:    [[TMP16:%.*]] = bitcast <8 x i64> [[TMP3]] to i512
+; CHECK-NEXT:    [[_MSCMP14:%.*]] = icmp ne i512 [[TMP16]], 0
+; CHECK-NEXT:    [[_MSOR15:%.*]] = or i1 [[_MSOR13]], [[_MSCMP14]]
+; CHECK-NEXT:    br i1 [[_MSOR15]], label [[TMP17:%.*]], label [[TMP18:%.*]], !prof [[PROF1]]
+; CHECK:       17:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       18:
+; CHECK-NEXT:    [[RES2:%.*]] = call <8 x double> @llvm.x86.avx512.maskz.fixupimm.pd.512(<8 x double> [[X0]], <8 x double> [[X1]], <8 x i64> [[X2]], i32 2, i8 -1, i32 8)
+; CHECK-NEXT:    [[RES3:%.*]] = fadd <8 x double> [[RES]], [[RES1]]
+; CHECK-NEXT:    [[RES4:%.*]] = fadd <8 x double> [[RES3]], [[RES2]]
+; CHECK-NEXT:    store <8 x i64> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x double> [[RES4]]
+;
+  %res = call <8 x double> @llvm.x86.avx512.maskz.fixupimm.pd.512(<8 x double> %x0, <8 x double> %x1, <8 x i64> %x2, i32 3, i8 %x4, i32 4)
+  %res1 = call <8 x double> @llvm.x86.avx512.maskz.fixupimm.pd.512(<8 x double> %x0, <8 x double> %x1, <8 x i64> zeroinitializer, i32 5, i8 %x4, i32 4)
+  %res2 = call <8 x double> @llvm.x86.avx512.maskz.fixupimm.pd.512(<8 x double> %x0, <8 x double> %x1, <8 x i64> %x2, i32 2, i8 -1, i32 8)
+  %res3 = fadd <8 x double> %res, %res1
+  %res4 = fadd <8 x double> %res3, %res2
+  ret <8 x double> %res4
+}
+
+declare <4 x float> @llvm.x86.avx512.mask.fixupimm.ss(<4 x float>, <4 x float>, <4 x i32>, i32, i8, i32)
+
+define <4 x float>@test_int_x86_avx512_mask_fixupimm_ss(<4 x float> %x0, <4 x float> %x1, <4 x i32> %x2, i8 %x4) #0 {
+; CHECK-LABEL: @test_int_x86_avx512_mask_fixupimm_ss(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 48) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <4 x i32> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <4 x i32> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP6]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <4 x i32> [[TMP3]] to i128
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i128 [[TMP7]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    [[_MSCMP4:%.*]] = icmp ne i8 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR5:%.*]] = or i1 [[_MSOR3]], [[_MSCMP4]]
+; CHECK-NEXT:    br i1 [[_MSOR5]], label [[TMP8:%.*]], label [[TMP9:%.*]], !prof [[PROF1]]
+; CHECK:       8:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       9:
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.x86.avx512.mask.fixupimm.ss(<4 x float> [[X0:%.*]], <4 x float> [[X1:%.*]], <4 x i32> [[X2:%.*]], i32 5, i8 [[X4:%.*]], i32 4)
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast <4 x i32> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP6:%.*]] = icmp ne i128 [[TMP10]], 0
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <4 x i32> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP7:%.*]] = icmp ne i128 [[TMP11]], 0
+; CHECK-NEXT:    [[_MSOR8:%.*]] = or i1 [[_MSCMP6]], [[_MSCMP7]]
+; CHECK-NEXT:    [[_MSCMP9:%.*]] = icmp ne i8 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR10:%.*]] = or i1 [[_MSOR8]], [[_MSCMP9]]
+; CHECK-NEXT:    br i1 [[_MSOR10]], label [[TMP12:%.*]], label [[TMP13:%.*]], !prof [[PROF1]]
+; CHECK:       12:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       13:
+; CHECK-NEXT:    [[RES1:%.*]] = call <4 x float> @llvm.x86.avx512.mask.fixupimm.ss(<4 x float> [[X0]], <4 x float> [[X1]], <4 x i32> zeroinitializer, i32 5, i8 [[X4]], i32 4)
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast <4 x i32> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP11:%.*]] = icmp ne i128 [[TMP14]], 0
+; CHECK-NEXT:    [[TMP15:%.*]] = bitcast <4 x i32> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP12:%.*]] = icmp ne i128 [[TMP15]], 0
+; CHECK-NEXT:    [[_MSOR13:%.*]] = or i1 [[_MSCMP11]], [[_MSCMP12]]
+; CHECK-NEXT:    [[TMP16:%.*]] = bitcast <4 x i32> [[TMP3]] to i128
+; CHECK-NEXT:    [[_MSCMP14:%.*]] = icmp ne i128 [[TMP16]], 0
+; CHECK-NEXT:    [[_MSOR15:%.*]] = or i1 [[_MSOR13]], [[_MSCMP14]]
+; CHECK-NEXT:    br i1 [[_MSOR15]], label [[TMP17:%.*]], label [[TMP18:%.*]], !prof [[PROF1]]
+; CHECK:       17:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       18:
+; CHECK-NEXT:    [[RES2:%.*]] = call <4 x float> @llvm.x86.avx512.mask.fixupimm.ss(<4 x float> [[X0]], <4 x float> [[X1]], <4 x i32> [[X2]], i32 5, i8 -1, i32 8)
+; CHECK-NEXT:    [[RES3:%.*]] = fadd <4 x float> [[RES]], [[RES1]]
+; CHECK-NEXT:    [[RES4:%.*]] = fadd <4 x float> [[RES3]], [[RES2]]
+; CHECK-NEXT:    store <4 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x float> [[RES4]]
+;
+  %res = call <4 x float> @llvm.x86.avx512.mask.fixupimm.ss(<4 x float> %x0, <4 x float> %x1, <4 x i32> %x2, i32 5, i8 %x4, i32 4)
+  %res1 = call <4 x float> @llvm.x86.avx512.mask.fixupimm.ss(<4 x float> %x0, <4 x float> %x1, <4 x i32> zeroinitializer, i32 5, i8 %x4, i32 4)
+  %res2 = call <4 x float> @llvm.x86.avx512.mask.fixupimm.ss(<4 x float> %x0, <4 x float> %x1, <4 x i32> %x2, i32 5, i8 -1, i32 8)
+  %res3 = fadd <4 x float> %res, %res1
+  %res4 = fadd <4 x float> %res3, %res2
+  ret <4 x float> %res4
+}
+
+declare <4 x float> @llvm.x86.avx512.maskz.fixupimm.ss(<4 x float>, <4 x float>, <4 x i32>, i32, i8, i32)
+
+define <4 x float>@test_int_x86_avx512_maskz_fixupimm_ss(<4 x float> %x0, <4 x float> %x1, <4 x i32> %x2, i8 %x4) #0 {
+; CHECK-LABEL: @test_int_x86_avx512_maskz_fixupimm_ss(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 48) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <4 x i32> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <4 x i32> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP6]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <4 x i32> [[TMP3]] to i128
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i128 [[TMP7]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    [[_MSCMP4:%.*]] = icmp ne i8 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR5:%.*]] = or i1 [[_MSOR3]], [[_MSCMP4]]
+; CHECK-NEXT:    br i1 [[_MSOR5]], label [[TMP8:%.*]], label [[TMP9:%.*]], !prof [[PROF1]]
+; CHECK:       8:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       9:
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.x86.avx512.maskz.fixupimm.ss(<4 x float> [[X0:%.*]], <4 x float> [[X1:%.*]], <4 x i32> [[X2:%.*]], i32 5, i8 [[X4:%.*]], i32 4)
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast <4 x i32> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP6:%.*]] = icmp ne i128 [[TMP10]], 0
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <4 x i32> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP7:%.*]] = icmp ne i128 [[TMP11]], 0
+; CHECK-NEXT:    [[_MSOR8:%.*]] = or i1 [[_MSCMP6]], [[_MSCMP7]]
+; CHECK-NEXT:    [[_MSCMP9:%.*]] = icmp ne i8 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR10:%.*]] = or i1 [[_MSOR8]], [[_MSCMP9]]
+; CHECK-NEXT:    br i1 [[_MSOR10]], label [[TMP12:%.*]], label [[TMP13:%.*]], !prof [[PROF1]]
+; CHECK:       12:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       13:
+; CHECK-NEXT:    [[RES1:%.*]] = call <4 x float> @llvm.x86.avx512.maskz.fixupimm.ss(<4 x float> [[X0]], <4 x float> [[X1]], <4 x i32> zeroinitializer, i32 5, i8 [[X4]], i32 8)
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast <4 x i32> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP11:%.*]] = icmp ne i128 [[TMP14]], 0
+; CHECK-NEXT:    [[TMP15:%.*]] = bitcast <4 x i32> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP12:%.*]] = icmp ne i128 [[TMP15]], 0
+; CHECK-NEXT:    [[_MSOR13:%.*]] = or i1 [[_MSCMP11]], [[_MSCMP12]]
+; CHECK-NEXT:    [[TMP16:%.*]] = bitcast <4 x i32> [[TMP3]] to i128
+; CHECK-NEXT:    [[_MSCMP14:%.*]] = icmp ne i128 [[TMP16]], 0
+; CHECK-NEXT:    [[_MSOR15:%.*]] = or i1 [[_MSOR13]], [[_MSCMP14]]
+; CHECK-NEXT:    br i1 [[_MSOR15]], label [[TMP17:%.*]], label [[TMP18:%.*]], !prof [[PROF1]]
+; CHECK:       17:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       18:
+; CHECK-NEXT:    [[RES2:%.*]] = call <4 x float> @llvm.x86.avx512.maskz.fixupimm.ss(<4 x float> [[X0]], <4 x float> [[X1]], <4 x i32> [[X2]], i32 6, i8 -1, i32 4)
+; CHECK-NEXT:    [[RES3:%.*]] = fadd <4 x float> [[RES]], [[RES1]]
+; CHECK-NEXT:    [[RES4:%.*]] = fadd <4 x float> [[RES3]], [[RES2]]
+; CHECK-NEXT:    store <4 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x float> [[RES4]]
+;
+  %res = call <4 x float> @llvm.x86.avx512.maskz.fixupimm.ss(<4 x float> %x0, <4 x float> %x1, <4 x i32> %x2, i32 5, i8 %x4, i32 4)
+  %res1 = call <4 x float> @llvm.x86.avx512.maskz.fixupimm.ss(<4 x float> %x0, <4 x float> %x1, <4 x i32> zeroinitializer, i32 5, i8 %x4, i32 8)
+  %res2 = call <4 x float> @llvm.x86.avx512.maskz.fixupimm.ss(<4 x float> %x0, <4 x float> %x1, <4 x i32> %x2, i32 6, i8 -1, i32 4)
+  %res3 = fadd <4 x float> %res, %res1
+  %res4 = fadd <4 x float> %res3, %res2
+  ret <4 x float> %res4
+}
+
+declare <16 x float> @llvm.x86.avx512.mask.fixupimm.ps.512(<16 x float>, <16 x float>, <16 x i32>, i32, i16, i32)
+
+define <16 x float>@test_int_x86_avx512_mask_fixupimm_ps_512(<16 x float> %x0, <16 x float> %x1, <16 x i32> %x2, i16 %x4) #0 {
+; CHECK-LABEL: @test_int_x86_avx512_mask_fixupimm_ps_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP6]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <16 x i32> [[TMP3]] to i512
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i512 [[TMP7]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    [[_MSCMP4:%.*]] = icmp ne i16 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR5:%.*]] = or i1 [[_MSOR3]], [[_MSCMP4]]
+; CHECK-NEXT:    br i1 [[_MSOR5]], label [[TMP8:%.*]], label [[TMP9:%.*]], !prof [[PROF1]]
+; CHECK:       8:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       9:
+; CHECK-NEXT:    [[RES:%.*]] = call <16 x float> @llvm.x86.avx512.mask.fixupimm.ps.512(<16 x float> [[X0:%.*]], <16 x float> [[X1:%.*]], <16 x i32> [[X2:%.*]], i32 5, i16 [[X4:%.*]], i32 4)
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP6:%.*]] = icmp ne i512 [[TMP10]], 0
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP7:%.*]] = icmp ne i512 [[TMP11]], 0
+; CHECK-NEXT:    [[_MSOR8:%.*]] = or i1 [[_MSCMP6]], [[_MSCMP7]]
+; CHECK-NEXT:    [[_MSCMP9:%.*]] = icmp ne i16 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR10:%.*]] = or i1 [[_MSOR8]], [[_MSCMP9]]
+; CHECK-NEXT:    br i1 [[_MSOR10]], label [[TMP12:%.*]], label [[TMP13:%.*]], !prof [[PROF1]]
+; CHECK:       12:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       13:
+; CHECK-NEXT:    [[RES1:%.*]] = call <16 x float> @llvm.x86.avx512.mask.fixupimm.ps.512(<16 x float> [[X0]], <16 x float> [[X1]], <16 x i32> zeroinitializer, i32 5, i16 [[X4]], i32 4)
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP11:%.*]] = icmp ne i512 [[TMP14]], 0
+; CHECK-NEXT:    [[TMP15:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP12:%.*]] = icmp ne i512 [[TMP15]], 0
+; CHECK-NEXT:    [[_MSOR13:%.*]] = or i1 [[_MSCMP11]], [[_MSCMP12]]
+; CHECK-NEXT:    [[TMP16:%.*]] = bitcast <16 x i32> [[TMP3]] to i512
+; CHECK-NEXT:    [[_MSCMP14:%.*]] = icmp ne i512 [[TMP16]], 0
+; CHECK-NEXT:    [[_MSOR15:%.*]] = or i1 [[_MSOR13]], [[_MSCMP14]]
+; CHECK-NEXT:    br i1 [[_MSOR15]], label [[TMP17:%.*]], label [[TMP18:%.*]], !prof [[PROF1]]
+; CHECK:       17:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       18:
+; CHECK-NEXT:    [[RES2:%.*]] = call <16 x float> @llvm.x86.avx512.mask.fixupimm.ps.512(<16 x float> [[X0]], <16 x float> [[X1]], <16 x i32> [[X2]], i32 5, i16 -1, i32 8)
+; CHECK-NEXT:    [[RES3:%.*]] = fadd <16 x float> [[RES]], [[RES1]]
+; CHECK-NEXT:    [[RES4:%.*]] = fadd <16 x float> [[RES3]], [[RES2]]
+; CHECK-NEXT:    store <16 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x float> [[RES4]]
+;
+  %res = call <16 x float> @llvm.x86.avx512.mask.fixupimm.ps.512(<16 x float> %x0, <16 x float> %x1, <16 x i32> %x2, i32 5, i16 %x4, i32 4)
+  %res1 = call <16 x float> @llvm.x86.avx512.mask.fixupimm.ps.512(<16 x float> %x0, <16 x float> %x1, <16 x i32> zeroinitializer, i32 5, i16 %x4, i32 4)
+  %res2 = call <16 x float> @llvm.x86.avx512.mask.fixupimm.ps.512(<16 x float> %x0, <16 x float> %x1, <16 x i32> %x2, i32 5, i16 -1, i32 8)
+  %res3 = fadd <16 x float> %res, %res1
+  %res4 = fadd <16 x float> %res3, %res2
+  ret <16 x float> %res4
+}
+
+define <16 x float>@test_int_x86_avx512_mask_fixupimm_ps_512_load(<16 x float> %x0, <16 x float> %x1, ptr %x2ptr) #0 {
+; CHECK-LABEL: @test_int_x86_avx512_mask_fixupimm_ps_512_load(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP4:%.*]], label [[TMP5:%.*]], !prof [[PROF1]]
+; CHECK:       4:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       5:
+; CHECK-NEXT:    [[X2:%.*]] = load <16 x i32>, ptr [[X2PTR:%.*]], align 64
+; CHECK-NEXT:    [[TMP6:%.*]] = ptrtoint ptr [[X2PTR]] to i64
+; CHECK-NEXT:    [[TMP7:%.*]] = xor i64 [[TMP6]], 87960930222080
+; CHECK-NEXT:    [[TMP8:%.*]] = inttoptr i64 [[TMP7]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <16 x i32>, ptr [[TMP8]], align 64
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP9]], 0
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast <16 x i32> [[TMP3]] to i512
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i512 [[TMP10]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP1]], [[_MSCMP2]]
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <16 x i32> [[_MSLD]] to i512
+; CHECK-NEXT:    [[_MSCMP3:%.*]] = icmp ne i512 [[TMP11]], 0
+; CHECK-NEXT:    [[_MSOR4:%.*]] = or i1 [[_MSOR]], [[_MSCMP3]]
+; CHECK-NEXT:    br i1 [[_MSOR4]], label [[TMP12:%.*]], label [[TMP13:%.*]], !prof [[PROF1]]
+; CHECK:       12:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       13:
+; CHECK-NEXT:    [[RES:%.*]] = call <16 x float> @llvm.x86.avx512.mask.fixupimm.ps.512(<16 x float> [[X0:%.*]], <16 x float> [[X1:%.*]], <16 x i32> [[X2]], i32 5, i16 -1, i32 4)
+; CHECK-NEXT:    store <16 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x float> [[RES]]
+;
+  %x2 = load <16 x i32>, ptr %x2ptr
+  %res = call <16 x float> @llvm.x86.avx512.mask.fixupimm.ps.512(<16 x float> %x0, <16 x float> %x1, <16 x i32> %x2, i32 5, i16 -1, i32 4)
+  ret <16 x float> %res
+}
+
+declare <16 x float> @llvm.x86.avx512.maskz.fixupimm.ps.512(<16 x float>, <16 x float>, <16 x i32>, i32, i16, i32)
+
+define <16 x float>@test_int_x86_avx512_maskz_fixupimm_ps_512(<16 x float> %x0, <16 x float> %x1, <16 x i32> %x2, i16 %x4) #0 {
+; CHECK-LABEL: @test_int_x86_avx512_maskz_fixupimm_ps_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP6]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <16 x i32> [[TMP3]] to i512
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i512 [[TMP7]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    [[_MSCMP4:%.*]] = icmp ne i16 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR5:%.*]] = or i1 [[_MSOR3]], [[_MSCMP4]]
+; CHECK-NEXT:    br i1 [[_MSOR5]], label [[TMP8:%.*]], label [[TMP9:%.*]], !prof [[PROF1]]
+; CHECK:       8:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       9:
+; CHECK-NEXT:    [[RES:%.*]] = call <16 x float> @llvm.x86.avx512.maskz.fixupimm.ps.512(<16 x float> [[X0:%.*]], <16 x float> [[X1:%.*]], <16 x i32> [[X2:%.*]], i32 5, i16 [[X4:%.*]], i32 4)
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP6:%.*]] = icmp ne i512 [[TMP10]], 0
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP7:%.*]] = icmp ne i512 [[TMP11]], 0
+; CHECK-NEXT:    [[_MSOR8:%.*]] = or i1 [[_MSCMP6]], [[_MSCMP7]]
+; CHECK-NEXT:    [[_MSCMP9:%.*]] = icmp ne i16 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR10:%.*]] = or i1 [[_MSOR8]], [[_MSCMP9]]
+; CHECK-NEXT:    br i1 [[_MSOR10]], label [[TMP12:%.*]], label [[TMP13:%.*]], !prof [[PROF1]]
+; CHECK:       12:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       13:
+; CHECK-NEXT:    [[RES1:%.*]] = call <16 x float> @llvm.x86.avx512.maskz.fixupimm.ps.512(<16 x float> [[X0]], <16 x float> [[X1]], <16 x i32> zeroinitializer, i32 6, i16 [[X4]], i32 8)
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP11:%.*]] = icmp ne i512 [[TMP14]], 0
+; CHECK-NEXT:    [[TMP15:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP12:%.*]] = icmp ne i512 [[TMP15]], 0
+; CHECK-NEXT:    [[_MSOR13:%.*]] = or i1 [[_MSCMP11]], [[_MSCMP12]]
+; CHECK-NEXT:    [[TMP16:%.*]] = bitcast <16 x i32> [[TMP3]] to i512
+; CHECK-NEXT:    [[_MSCMP14:%.*]] = icmp ne i512 [[TMP16]], 0
+; CHECK-NEXT:    [[_MSOR15:%.*]] = or i1 [[_MSOR13]], [[_MSCMP14]]
+; CHECK-NEXT:    br i1 [[_MSOR15]], label [[TMP17:%.*]], label [[TMP18:%.*]], !prof [[PROF1]]
+; CHECK:       17:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       18:
+; CHECK-NEXT:    [[RES2:%.*]] = call <16 x float> @llvm.x86.avx512.maskz.fixupimm.ps.512(<16 x float> [[X0]], <16 x float> [[X1]], <16 x i32> [[X2]], i32 7, i16 -1, i32 4)
+; CHECK-NEXT:    [[RES3:%.*]] = fadd <16 x float> [[RES]], [[RES1]]
+; CHECK-NEXT:    [[RES4:%.*]] = fadd <16 x float> [[RES3]], [[RES2]]
+; CHECK-NEXT:    store <16 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x float> [[RES4]]
+;
+  %res = call <16 x float> @llvm.x86.avx512.maskz.fixupimm.ps.512(<16 x float> %x0, <16 x float> %x1, <16 x i32> %x2, i32 5, i16 %x4, i32 4)
+  %res1 = call <16 x float> @llvm.x86.avx512.maskz.fixupimm.ps.512(<16 x float> %x0, <16 x float> %x1, <16 x i32> zeroinitializer, i32 6, i16 %x4, i32 8)
+  %res2 = call <16 x float> @llvm.x86.avx512.maskz.fixupimm.ps.512(<16 x float> %x0, <16 x float> %x1, <16 x i32> %x2, i32 7, i16 -1, i32 4)
+  %res3 = fadd <16 x float> %res, %res1
+  %res4 = fadd <16 x float> %res3, %res2
+  ret <16 x float> %res4
+}
+
+declare <2 x double> @llvm.x86.avx512.mask.fixupimm.sd(<2 x double>, <2 x double>, <2 x i64>, i32, i8, i32)
+
+define <2 x double>@test_int_x86_avx512_mask_fixupimm_sd(<2 x double> %x0, <2 x double> %x1, <2 x i64> %x2, i8 %x4) #0 {
+; CHECK-LABEL: @test_int_x86_avx512_mask_fixupimm_sd(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 48) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <2 x i64> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <2 x i64> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP6]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <2 x i64> [[TMP3]] to i128
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i128 [[TMP7]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    [[_MSCMP4:%.*]] = icmp ne i8 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR5:%.*]] = or i1 [[_MSOR3]], [[_MSCMP4]]
+; CHECK-NEXT:    br i1 [[_MSOR5]], label [[TMP8:%.*]], label [[TMP9:%.*]], !prof [[PROF1]]
+; CHECK:       8:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       9:
+; CHECK-NEXT:    [[RES:%.*]] = call <2 x double> @llvm.x86.avx512.mask.fixupimm.sd(<2 x double> [[X0:%.*]], <2 x double> [[X1:%.*]], <2 x i64> [[X2:%.*]], i32 5, i8 [[X4:%.*]], i32 4)
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast <2 x i64> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP6:%.*]] = icmp ne i128 [[TMP10]], 0
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <2 x i64> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP7:%.*]] = icmp ne i128 [[TMP11]], 0
+; CHECK-NEXT:    [[_MSOR8:%.*]] = or i1 [[_MSCMP6]], [[_MSCMP7]]
+; CHECK-NEXT:    [[_MSCMP9:%.*]] = icmp ne i8 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR10:%.*]] = or i1 [[_MSOR8]], [[_MSCMP9]]
+; CHECK-NEXT:    br i1 [[_MSOR10]], label [[TMP12:%.*]], label [[TMP13:%.*]], !prof [[PROF1]]
+; CHECK:       12:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       13:
+; CHECK-NEXT:    [[RES1:%.*]] = call <2 x double> @llvm.x86.avx512.mask.fixupimm.sd(<2 x double> [[X0]], <2 x double> [[X1]], <2 x i64> zeroinitializer, i32 5, i8 [[X4]], i32 8)
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast <2 x i64> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP11:%.*]] = icmp ne i128 [[TMP14]], 0
+; CHECK-NEXT:    [[TMP15:%.*]] = bitcast <2 x i64> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP12:%.*]] = icmp ne i128 [[TMP15]], 0
+; CHECK-NEXT:    [[_MSOR13:%.*]] = or i1 [[_MSCMP11]], [[_MSCMP12]]
+; CHECK-NEXT:    [[TMP16:%.*]] = bitcast <2 x i64> [[TMP3]] to i128
+; CHECK-NEXT:    [[_MSCMP14:%.*]] = icmp ne i128 [[TMP16]], 0
+; CHECK-NEXT:    [[_MSOR15:%.*]] = or i1 [[_MSOR13]], [[_MSCMP14]]
+; CHECK-NEXT:    br i1 [[_MSOR15]], label [[TMP17:%.*]], label [[TMP18:%.*]], !prof [[PROF1]]
+; CHECK:       17:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       18:
+; CHECK-NEXT:    [[RES2:%.*]] = call <2 x double> @llvm.x86.avx512.mask.fixupimm.sd(<2 x double> [[X0]], <2 x double> [[X1]], <2 x i64> [[X2]], i32 6, i8 -1, i32 4)
+; CHECK-NEXT:    [[RES3:%.*]] = fadd <2 x double> [[RES]], [[RES1]]
+; CHECK-NEXT:    [[RES4:%.*]] = fadd <2 x double> [[RES3]], [[RES2]]
+; CHECK-NEXT:    store <2 x i64> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x double> [[RES4]]
+;
+  %res = call <2 x double> @llvm.x86.avx512.mask.fixupimm.sd(<2 x double> %x0, <2 x double> %x1, <2 x i64> %x2, i32 5, i8 %x4, i32 4)
+  %res1 = call <2 x double> @llvm.x86.avx512.mask.fixupimm.sd(<2 x double> %x0, <2 x double> %x1, <2 x i64> zeroinitializer, i32 5, i8 %x4, i32 8)
+  %res2 = call <2 x double> @llvm.x86.avx512.mask.fixupimm.sd(<2 x double> %x0, <2 x double> %x1, <2 x i64> %x2, i32 6, i8 -1, i32 4)
+  %res3 = fadd <2 x double> %res, %res1
+  %res4 = fadd <2 x double> %res3, %res2
+  ret <2 x double> %res4
+}
+
+declare <2 x double> @llvm.x86.avx512.maskz.fixupimm.sd(<2 x double>, <2 x double>, <2 x i64>, i32, i8, i32)
+
+define <2 x double>@test_int_x86_avx512_maskz_fixupimm_sd(<2 x double> %x0, <2 x double> %x1, <2 x i64> %x2, i8 %x4) #0 {
+; CHECK-LABEL: @test_int_x86_avx512_maskz_fixupimm_sd(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 48) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <2 x i64> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <2 x i64> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP6]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <2 x i64> [[TMP3]] to i128
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i128 [[TMP7]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    [[_MSCMP4:%.*]] = icmp ne i8 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR5:%.*]] = or i1 [[_MSOR3]], [[_MSCMP4]]
+; CHECK-NEXT:    br i1 [[_MSOR5]], label [[TMP8:%.*]], label [[TMP9:%.*]], !prof [[PROF1]]
+; CHECK:       8:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       9:
+; CHECK-NEXT:    [[RES:%.*]] = call <2 x double> @llvm.x86.avx512.maskz.fixupimm.sd(<2 x double> [[X0:%.*]], <2 x double> [[X1:%.*]], <2 x i64> [[X2:%.*]], i32 5, i8 [[X4:%.*]], i32 4)
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast <2 x i64> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP6:%.*]] = icmp ne i128 [[TMP10]], 0
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <2 x i64> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP7:%.*]] = icmp ne i128 [[TMP11]], 0
+; CHECK-NEXT:    [[_MSOR8:%.*]] = or i1 [[_MSCMP6]], [[_MSCMP7]]
+; CHECK-NEXT:    [[_MSCMP9:%.*]] = icmp ne i8 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR10:%.*]] = or i1 [[_MSOR8]], [[_MSCMP9]]
+; CHECK-NEXT:    br i1 [[_MSOR10]], label [[TMP12:%.*]], label [[TMP13:%.*]], !prof [[PROF1]]
+; CHECK:       12:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       13:
+; CHECK-NEXT:    [[RES1:%.*]] = call <2 x double> @llvm.x86.avx512.maskz.fixupimm.sd(<2 x double> [[X0]], <2 x double> [[X1]], <2 x i64> zeroinitializer, i32 5, i8 [[X4]], i32 8)
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast <2 x i64> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP11:%.*]] = icmp ne i128 [[TMP14]], 0
+; CHECK-NEXT:    [[TMP15:%.*]] = bitcast <2 x i64> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP12:%.*]] = icmp ne i128 [[TMP15]], 0
+; CHECK-NEXT:    [[_MSOR13:%.*]] = or i1 [[_MSCMP11]], [[_MSCMP12]]
+; CHECK-NEXT:    [[TMP16:%.*]] = bitcast <2 x i64> [[TMP3]] to i128
+; CHECK-NEXT:    [[_MSCMP14:%.*]] = icmp ne i128 [[TMP16]], 0
+; CHECK-NEXT:    [[_MSOR15:%.*]] = or i1 [[_MSOR13]], [[_MSCMP14]]
+; CHECK-NEXT:    [[_MSCMP16:%.*]] = icmp ne i8 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR17:%.*]] = or i1 [[_MSOR15]], [[_MSCMP16]]
+; CHECK-NEXT:    br i1 [[_MSOR17]], label [[TMP17:%.*]], label [[TMP18:%.*]], !prof [[PROF1]]
+; CHECK:       17:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       18:
+; CHECK-NEXT:    [[RES2:%.*]] = call <2 x double> @llvm.x86.avx512.maskz.fixupimm.sd(<2 x double> [[X0]], <2 x double> [[X1]], <2 x i64> [[X2]], i32 5, i8 [[X4]], i32 8)
+; CHECK-NEXT:    [[RES3:%.*]] = fadd <2 x double> [[RES]], [[RES1]]
+; CHECK-NEXT:    [[RES4:%.*]] = fadd <2 x double> [[RES3]], [[RES2]]
+; CHECK-NEXT:    store <2 x i64> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x double> [[RES4]]
+;
+  %res = call <2 x double> @llvm.x86.avx512.maskz.fixupimm.sd(<2 x double> %x0, <2 x double> %x1, <2 x i64> %x2, i32 5, i8 %x4, i32 4)
+  %res1 = call <2 x double> @llvm.x86.avx512.maskz.fixupimm.sd(<2 x double> %x0, <2 x double> %x1, <2 x i64> zeroinitializer, i32 5, i8 %x4, i32 8)
+  %res2 = call <2 x double> @llvm.x86.avx512.maskz.fixupimm.sd(<2 x double> %x0, <2 x double> %x1, <2 x i64> %x2, i32 5, i8 %x4, i32 8)
+  %res3 = fadd <2 x double> %res, %res1
+  %res4 = fadd <2 x double> %res3, %res2
+  ret <2 x double> %res4
+}
+
+declare double @llvm.fma.f64(double, double, double) #1
+declare double @llvm.x86.avx512.vfmadd.f64(double, double, double, i32) #0
+
+define <2 x double> @test_int_x86_avx512_mask_vfmadd_sd(<2 x double> %x0, <2 x double> %x1, <2 x double> %x2, i8 %x3, i32 %x4) #0 {
+; CHECK-LABEL: @test_int_x86_avx512_mask_vfmadd_sd(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 48) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = extractelement <2 x i64> [[TMP1]], i64 0
+; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <2 x double> [[X0:%.*]], i64 0
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = extractelement <2 x i64> [[TMP2]], i64 0
+; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <2 x double> [[X1:%.*]], i64 0
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = extractelement <2 x i64> [[TMP3]], i64 0
+; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <2 x double> [[X2:%.*]], i64 0
+; CHECK-NEXT:    [[_MSPROP3:%.*]] = or i64 [[_MSPROP]], [[_MSPROP1]]
+; CHECK-NEXT:    [[_MSPROP4:%.*]] = or i64 [[_MSPROP3]], [[_MSPROP2]]
+; CHECK-NEXT:    [[TMP8:%.*]] = call double @llvm.fma.f64(double [[TMP5]], double [[TMP6]], double [[TMP7]])
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast i8 [[TMP4]] to <8 x i1>
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i8 [[X3:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[_MSPROP5:%.*]] = extractelement <8 x i1> [[TMP9]], i64 0
+; CHECK-NEXT:    [[TMP11:%.*]] = extractelement <8 x i1> [[TMP10]], i64 0
+; CHECK-NEXT:    [[TMP12:%.*]] = select i1 [[TMP11]], i64 [[_MSPROP4]], i64 [[_MSPROP]]
+; CHECK-NEXT:    [[TMP13:%.*]] = bitcast double [[TMP8]] to i64
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast double [[TMP5]] to i64
+; CHECK-NEXT:    [[TMP15:%.*]] = xor i64 [[TMP13]], [[TMP14]]
+; CHECK-NEXT:    [[TMP16:%.*]] = or i64 [[TMP15]], [[_MSPROP4]]
+; CHECK-NEXT:    [[TMP17:%.*]] = or i64 [[TMP16]], [[_MSPROP]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select i1 [[_MSPROP5]], i64 [[TMP17]], i64 [[TMP12]]
+; CHECK-NEXT:    [[TMP18:%.*]] = select i1 [[TMP11]], double [[TMP8]], double [[TMP5]]
+; CHECK-NEXT:    [[_MSPROP6:%.*]] = insertelement <2 x i64> [[TMP1]], i64 [[_MSPROP_SELECT]], i64 0
+; CHECK-NEXT:    [[TMP19:%.*]] = insertelement <2 x double> [[X0]], double [[TMP18]], i64 0
+; CHECK-NEXT:    [[_MSPROP7:%.*]] = extractelement <2 x i64> [[TMP1]], i64 0
+; CHECK-NEXT:    [[TMP20:%.*]] = extractelement <2 x double> [[X0]], i64 0
+; CHECK-NEXT:    [[_MSPROP8:%.*]] = extractelement <2 x i64> [[TMP2]], i64 0
+; CHECK-NEXT:    [[TMP21:%.*]] = extractelement <2 x double> [[X1]], i64 0
+; CHECK-NEXT:    [[_MSPROP9:%.*]] = extractelement <2 x i64> [[TMP3]], i64 0
+; CHECK-NEXT:    [[TMP22:%.*]] = extractelement <2 x double> [[X2]], i64 0
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[_MSPROP7]], 0
+; CHECK-NEXT:    [[_MSCMP19:%.*]] = icmp ne i64 [[_MSPROP8]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP19]]
+; CHECK-NEXT:    [[_MSCMP20:%.*]] = icmp ne i64 [[_MSPROP9]], 0
+; CHECK-NEXT:    [[_MSOR21:%.*]] = or i1 [[_MSOR]], [[_MSCMP20]]
+; CHECK-NEXT:    br i1 [[_MSOR21]], label [[TMP23:%.*]], label [[TMP24:%.*]], !prof [[PROF1]]
+; CHECK:       23:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       24:
+; CHECK-NEXT:    [[TMP25:%.*]] = call double @llvm.x86.avx512.vfmadd.f64(double [[TMP20]], double [[TMP21]], double [[TMP22]], i32 11)
+; CHECK-NEXT:    [[_MSPROP10:%.*]] = insertelement <2 x i64> [[TMP1]], i64 0, i64 0
+; CHECK-NEXT:    [[TMP26:%.*]] = insertelement <2 x double> [[X0]], double [[TMP25]], i64 0
+; CHECK-NEXT:    [[_MSPROP11:%.*]] = extractelement <2 x i64> [[TMP1]], i64 0
+; CHECK-NEXT:    [[TMP27:%.*]] = extractelement <2 x double> [[X0]], i64 0
+; CHECK-NEXT:    [[_MSPROP12:%.*]] = extractelement <2 x i64> [[TMP2]], i64 0
+; CHECK-NEXT:    [[TMP28:%.*]] = extractelement <2 x double> [[X1]], i64 0
+; CHECK-NEXT:    [[_MSPROP13:%.*]] = extractelement <2 x i64> [[TMP3]], i64 0
+; CHECK-NEXT:    [[TMP29:%.*]] = extractelement <2 x double> [[X2]], i64 0
+; CHECK-NEXT:    [[_MSCMP22:%.*]] = icmp ne i64 [[_MSPROP11]], 0
+; CHECK-NEXT:    [[_MSCMP23:%.*]] = icmp ne i64 [[_MSPROP12]], 0
+; CHECK-NEXT:    [[_MSOR24:%.*]] = or i1 [[_MSCMP22]], [[_MSCMP23]]
+; CHECK-NEXT:    [[_MSCMP25:%.*]] = icmp ne i64 [[_MSPROP13]], 0
+; CHECK-NEXT:    [[_MSOR26:%.*]] = or i1 [[_MSOR24]], [[_MSCMP25]]
+; CHECK-NEXT:    br i1 [[_MSOR26]], label [[TMP30:%.*]], label [[TMP31:%.*]], !prof [[PROF1]]
+; CHECK:       30:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       31:
+; CHECK-NEXT:    [[TMP32:%.*]] = call double @llvm.x86.avx512.vfmadd.f64(double [[TMP27]], double [[TMP28]], double [[TMP29]], i32 10)
+; CHECK-NEXT:    [[TMP33:%.*]] = bitcast i8 [[TMP4]] to <8 x i1>
+; CHECK-NEXT:    [[TMP34:%.*]] = bitcast i8 [[X3]] to <8 x i1>
+; CHECK-NEXT:    [[_MSPROP14:%.*]] = extractelement <8 x i1> [[TMP33]], i64 0
+; CHECK-NEXT:    [[TMP35:%.*]] = extractelement <8 x i1> [[TMP34]], i64 0
+; CHECK-NEXT:    [[TMP36:%.*]] = select i1 [[TMP35]], i64 0, i64 [[_MSPROP11]]
+; CHECK-NEXT:    [[TMP37:%.*]] = bitcast double [[TMP32]] to i64
+; CHECK-NEXT:    [[TMP38:%.*]] = bitcast double [[TMP27]] to i64
+; CHECK-NEXT:    [[TMP39:%.*]] = xor i64 [[TMP37]], [[TMP38]]
+; CHECK-NEXT:    [[TMP40:%.*]] = or i64 [[TMP39]], 0
+; CHECK-NEXT:    [[TMP41:%.*]] = or i64 [[TMP40]], [[_MSPROP11]]
+; CHECK-NEXT:    [[_MSPROP_SELECT15:%.*]] = select i1 [[_MSPROP14]], i64 [[TMP41]], i64 [[TMP36]]
+; CHECK-NEXT:    [[TMP42:%.*]] = select i1 [[TMP35]], double [[TMP32]], double [[TMP27]]
+; CHECK-NEXT:    [[_MSPROP16:%.*]] = insertelement <2 x i64> [[TMP1]], i64 [[_MSPROP_SELECT15]], i64 0
+; CHECK-NEXT:    [[TMP43:%.*]] = insertelement <2 x double> [[X0]], double [[TMP42]], i64 0
+; CHECK-NEXT:    [[_MSPROP17:%.*]] = or <2 x i64> [[_MSPROP6]], [[_MSPROP10]]
+; CHECK-NEXT:    [[RES3:%.*]] = fadd <2 x double> [[TMP19]], [[TMP26]]
+; CHECK-NEXT:    [[_MSPROP18:%.*]] = or <2 x i64> [[_MSPROP16]], [[_MSPROP17]]
+; CHECK-NEXT:    [[RES4:%.*]] = fadd <2 x double> [[TMP43]], [[RES3]]
+; CHECK-NEXT:    store <2 x i64> [[_MSPROP18]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x double> [[RES4]]
+;
+  %1 = extractelement <2 x double> %x0, i64 0
+  %2 = extractelement <2 x double> %x1, i64 0
+  %3 = extractelement <2 x double> %x2, i64 0
+  %4 = call double @llvm.fma.f64(double %1, double %2, double %3)
+  %5 = bitcast i8 %x3 to <8 x i1>
+  %6 = extractelement <8 x i1> %5, i64 0
+  %7 = select i1 %6, double %4, double %1
+  %8 = insertelement <2 x double> %x0, double %7, i64 0
+  %9 = extractelement <2 x double> %x0, i64 0
+  %10 = extractelement <2 x double> %x1, i64 0
+  %11 = extractelement <2 x double> %x2, i64 0
+  %12 = call double @llvm.x86.avx512.vfmadd.f64(double %9, double %10, double %11, i32 11)
+  %13 = insertelement <2 x double> %x0, double %12, i64 0
+  %14 = extractelement <2 x double> %x0, i64 0
+  %15 = extractelement <2 x double> %x1, i64 0
+  %16 = extractelement <2 x double> %x2, i64 0
+  %17 = call double @llvm.x86.avx512.vfmadd.f64(double %14, double %15, double %16, i32 10)
+  %18 = bitcast i8 %x3 to <8 x i1>
+  %19 = extractelement <8 x i1> %18, i64 0
+  %20 = select i1 %19, double %17, double %14
+  %21 = insertelement <2 x double> %x0, double %20, i64 0
+  %res3 = fadd <2 x double> %8, %13
+  %res4 = fadd <2 x double> %21, %res3
+  ret <2 x double> %res4
+}
+
+define <4 x float> @test_int_x86_avx512_mask_vfmadd_ss(<4 x float> %x0, <4 x float> %x1, <4 x float> %x2, i8 %x3, i32 %x4) #0 {
+; CHECK-LABEL: @test_int_x86_avx512_mask_vfmadd_ss(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 48) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = extractelement <4 x i32> [[TMP1]], i64 0
+; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <4 x float> [[X0:%.*]], i64 0
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = extractelement <4 x i32> [[TMP2]], i64 0
+; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <4 x float> [[X1:%.*]], i64 0
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = extractelement <4 x i32> [[TMP3]], i64 0
+; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <4 x float> [[X2:%.*]], i64 0
+; CHECK-NEXT:    [[_MSPROP3:%.*]] = or i32 [[_MSPROP]], [[_MSPROP1]]
+; CHECK-NEXT:    [[_MSPROP4:%.*]] = or i32 [[_MSPROP3]], [[_MSPROP2]]
+; CHECK-NEXT:    [[TMP8:%.*]] = call float @llvm.fma.f32(float [[TMP5]], float [[TMP6]], float [[TMP7]])
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast i8 [[TMP4]] to <8 x i1>
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i8 [[X3:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[_MSPROP5:%.*]] = extractelement <8 x i1> [[TMP9]], i64 0
+; CHECK-NEXT:    [[TMP11:%.*]] = extractelement <8 x i1> [[TMP10]], i64 0
+; CHECK-NEXT:    [[TMP12:%.*]] = select i1 [[TMP11]], i32 [[_MSPROP4]], i32 [[_MSPROP]]
+; CHECK-NEXT:    [[TMP13:%.*]] = bitcast float [[TMP8]] to i32
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast float [[TMP5]] to i32
+; CHECK-NEXT:    [[TMP15:%.*]] = xor i32 [[TMP13]], [[TMP14]]
+; CHECK-NEXT:    [[TMP16:%.*]] = or i32 [[TMP15]], [[_MSPROP4]]
+; CHECK-NEXT:    [[TMP17:%.*]] = or i32 [[TMP16]], [[_MSPROP]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select i1 [[_MSPROP5]], i32 [[TMP17]], i32 [[TMP12]]
+; CHECK-NEXT:    [[TMP18:%.*]] = select i1 [[TMP11]], float [[TMP8]], float [[TMP5]]
+; CHECK-NEXT:    [[_MSPROP6:%.*]] = insertelement <4 x i32> [[TMP1]], i32 [[_MSPROP_SELECT]], i64 0
+; CHECK-NEXT:    [[TMP19:%.*]] = insertelement <4 x float> [[X0]], float [[TMP18]], i64 0
+; CHECK-NEXT:    [[_MSPROP7:%.*]] = extractelement <4 x i32> [[TMP1]], i64 0
+; CHECK-NEXT:    [[TMP20:%.*]] = extractelement <4 x float> [[X0]], i64 0
+; CHECK-NEXT:    [[_MSPROP8:%.*]] = extractelement <4 x i32> [[TMP2]], i64 0
+; CHECK-NEXT:    [[TMP21:%.*]] = extractelement <4 x float> [[X1]], i64 0
+; CHECK-NEXT:    [[_MSPROP9:%.*]] = extractelement <4 x i32> [[TMP3]], i64 0
+; CHECK-NEXT:    [[TMP22:%.*]] = extractelement <4 x float> [[X2]], i64 0
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i32 [[_MSPROP7]], 0
+; CHECK-NEXT:    [[_MSCMP19:%.*]] = icmp ne i32 [[_MSPROP8]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP19]]
+; CHECK-NEXT:    [[_MSCMP20:%.*]] = icmp ne i32 [[_MSPROP9]], 0
+; CHECK-NEXT:    [[_MSOR21:%.*]] = or i1 [[_MSOR]], [[_MSCMP20]]
+; CHECK-NEXT:    br i1 [[_MSOR21]], label [[TMP23:%.*]], label [[TMP24:%.*]], !prof [[PROF1]]
+; CHECK:       23:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       24:
+; CHECK-NEXT:    [[TMP25:%.*]] = call float @llvm.x86.avx512.vfmadd.f32(float [[TMP20]], float [[TMP21]], float [[TMP22]], i32 11)
+; CHECK-NEXT:    [[_MSPROP10:%.*]] = insertelement <4 x i32> [[TMP1]], i32 0, i64 0
+; CHECK-NEXT:    [[TMP26:%.*]] = insertelement <4 x float> [[X0]], float [[TMP25]], i64 0
+; CHECK-NEXT:    [[_MSPROP11:%.*]] = extractelement <4 x i32> [[TMP1]], i64 0
+; CHECK-NEXT:    [[TMP27:%.*]] = extractelement <4 x float> [[X0]], i64 0
+; CHECK-NEXT:    [[_MSPROP12:%.*]] = extractelement <4 x i32> [[TMP2]], i64 0
+; CHECK-NEXT:    [[TMP28:%.*]] = extractelement <4 x float> [[X1]], i64 0
+; CHECK-NEXT:    [[_MSPROP13:%.*]] = extractelement <4 x i32> [[TMP3]], i64 0
+; CHECK-NEXT:    [[TMP29:%.*]] = extractelement <4 x float> [[X2]], i64 0
+; CHECK-NEXT:    [[_MSCMP22:%.*]] = icmp ne i32 [[_MSPROP11]], 0
+; CHECK-NEXT:    [[_MSCMP23:%.*]] = icmp ne i32 [[_MSPROP12]], 0
+; CHECK-NEXT:    [[_MSOR24:%.*]] = or i1 [[_MSCMP22]], [[_MSCMP23]]
+; CHECK-NEXT:    [[_MSCMP25:%.*]] = icmp ne i32 [[_MSPROP13]], 0
+; CHECK-NEXT:    [[_MSOR26:%.*]] = or i1 [[_MSOR24]], [[_MSCMP25]]
+; CHECK-NEXT:    br i1 [[_MSOR26]], label [[TMP30:%.*]], label [[TMP31:%.*]], !prof [[PROF1]]
+; CHECK:       30:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       31:
+; CHECK-NEXT:    [[TMP32:%.*]] = call float @llvm.x86.avx512.vfmadd.f32(float [[TMP27]], float [[TMP28]], float [[TMP29]], i32 10)
+; CHECK-NEXT:    [[TMP33:%.*]] = bitcast i8 [[TMP4]] to <8 x i1>
+; CHECK-NEXT:    [[TMP34:%.*]] = bitcast i8 [[X3]] to <8 x i1>
+; CHECK-NEXT:    [[_MSPROP14:%.*]] = extractelement <8 x i1> [[TMP33]], i64 0
+; CHECK-NEXT:    [[TMP35:%.*]] = extractelement <8 x i1> [[TMP34]], i64 0
+; CHECK-NEXT:    [[TMP36:%.*]] = select i1 [[TMP35]], i32 0, i32 [[_MSPROP11]]
+; CHECK-NEXT:    [[TMP37:%.*]] = bitcast float [[TMP32]] to i32
+; CHECK-NEXT:    [[TMP38:%.*]] = bitcast float [[TMP27]] to i32
+; CHECK-NEXT:    [[TMP39:%.*]] = xor i32 [[TMP37]], [[TMP38]]
+; CHECK-NEXT:    [[TMP40:%.*]] = or i32 [[TMP39]], 0
+; CHECK-NEXT:    [[TMP41:%.*]] = or i32 [[TMP40]], [[_MSPROP11]]
+; CHECK-NEXT:    [[_MSPROP_SELECT15:%.*]] = select i1 [[_MSPROP14]], i32 [[TMP41]], i32 [[TMP36]]
+; CHECK-NEXT:    [[TMP42:%.*]] = select i1 [[TMP35]], float [[TMP32]], float [[TMP27]]
+; CHECK-NEXT:    [[_MSPROP16:%.*]] = insertelement <4 x i32> [[TMP1]], i32 [[_MSPROP_SELECT15]], i64 0
+; CHECK-NEXT:    [[TMP43:%.*]] = insertelement <4 x float> [[X0]], float [[TMP42]], i64 0
+; CHECK-NEXT:    [[_MSPROP17:%.*]] = or <4 x i32> [[_MSPROP6]], [[_MSPROP10]]
+; CHECK-NEXT:    [[RES3:%.*]] = fadd <4 x float> [[TMP19]], [[TMP26]]
+; CHECK-NEXT:    [[_MSPROP18:%.*]] = or <4 x i32> [[_MSPROP16]], [[_MSPROP17]]
+; CHECK-NEXT:    [[RES4:%.*]] = fadd <4 x float> [[TMP43]], [[RES3]]
+; CHECK-NEXT:    store <4 x i32> [[_MSPROP18]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x float> [[RES4]]
+;
+  %1 = extractelement <4 x float> %x0, i64 0
+  %2 = extractelement <4 x float> %x1, i64 0
+  %3 = extractelement <4 x float> %x2, i64 0
+  %4 = call float @llvm.fma.f32(float %1, float %2, float %3)
+  %5 = bitcast i8 %x3 to <8 x i1>
+  %6 = extractelement <8 x i1> %5, i64 0
+  %7 = select i1 %6, float %4, float %1
+  %8 = insertelement <4 x float> %x0, float %7, i64 0
+  %9 = extractelement <4 x float> %x0, i64 0
+  %10 = extractelement <4 x float> %x1, i64 0
+  %11 = extractelement <4 x float> %x2, i64 0
+  %12 = call float @llvm.x86.avx512.vfmadd.f32(float %9, float %10, float %11, i32 11)
+  %13 = insertelement <4 x float> %x0, float %12, i64 0
+  %14 = extractelement <4 x float> %x0, i64 0
+  %15 = extractelement <4 x float> %x1, i64 0
+  %16 = extractelement <4 x float> %x2, i64 0
+  %17 = call float @llvm.x86.avx512.vfmadd.f32(float %14, float %15, float %16, i32 10)
+  %18 = bitcast i8 %x3 to <8 x i1>
+  %19 = extractelement <8 x i1> %18, i64 0
+  %20 = select i1 %19, float %17, float %14
+  %21 = insertelement <4 x float> %x0, float %20, i64 0
+  %res3 = fadd <4 x float> %8, %13
+  %res4 = fadd <4 x float> %21, %res3
+  ret <4 x float> %res4
+}
+
+define <2 x double>@test_int_x86_avx512_maskz_vfmadd_sd(<2 x double> %x0, <2 x double> %x1, <2 x double> %x2, i8 %x3,i32 %x4 ){
+; CHECK-LABEL: @test_int_x86_avx512_maskz_vfmadd_sd(
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP1:%.*]] = extractelement <2 x double> [[X0:%.*]], i64 0
+; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <2 x double> [[X1:%.*]], i64 0
+; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <2 x double> [[X2:%.*]], i64 0
+; CHECK-NEXT:    [[TMP4:%.*]] = call double @llvm.fma.f64(double [[TMP1]], double [[TMP2]], double [[TMP3]])
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i8 [[X3:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <8 x i1> [[TMP5]], i64 0
+; CHECK-NEXT:    [[TMP7:%.*]] = select i1 [[TMP6]], i64 0, i64 0
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast double [[TMP4]] to i64
+; CHECK-NEXT:    [[TMP9:%.*]] = xor i64 [[TMP8]], 0
+; CHECK-NEXT:    [[TMP10:%.*]] = or i64 [[TMP9]], 0
+; CHECK-NEXT:    [[TMP11:%.*]] = or i64 [[TMP10]], 0
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select i1 false, i64 [[TMP11]], i64 [[TMP7]]
+; CHECK-NEXT:    [[TMP12:%.*]] = select i1 [[TMP6]], double [[TMP4]], double 0.000000e+00
+; CHECK-NEXT:    [[TMP13:%.*]] = insertelement <2 x double> [[X0]], double [[TMP12]], i64 0
+; CHECK-NEXT:    [[TMP14:%.*]] = extractelement <2 x double> [[X0]], i64 0
+; CHECK-NEXT:    [[TMP15:%.*]] = extractelement <2 x double> [[X1]], i64 0
+; CHECK-NEXT:    [[TMP16:%.*]] = extractelement <2 x double> [[X2]], i64 0
+; CHECK-NEXT:    [[TMP17:%.*]] = call double @llvm.x86.avx512.vfmadd.f64(double [[TMP14]], double [[TMP15]], double [[TMP16]], i32 11)
+; CHECK-NEXT:    [[TMP18:%.*]] = bitcast i8 [[X3]] to <8 x i1>
+; CHECK-NEXT:    [[TMP19:%.*]] = extractelement <8 x i1> [[TMP18]], i64 0
+; CHECK-NEXT:    [[TMP20:%.*]] = select i1 [[TMP19]], i64 0, i64 0
+; CHECK-NEXT:    [[TMP21:%.*]] = bitcast double [[TMP17]] to i64
+; CHECK-NEXT:    [[TMP22:%.*]] = xor i64 [[TMP21]], 0
+; CHECK-NEXT:    [[TMP23:%.*]] = or i64 [[TMP22]], 0
+; CHECK-NEXT:    [[TMP24:%.*]] = or i64 [[TMP23]], 0
+; CHECK-NEXT:    [[_MSPROP_SELECT1:%.*]] = select i1 false, i64 [[TMP24]], i64 [[TMP20]]
+; CHECK-NEXT:    [[TMP25:%.*]] = select i1 [[TMP19]], double [[TMP17]], double 0.000000e+00
+; CHECK-NEXT:    [[TMP26:%.*]] = insertelement <2 x double> [[X0]], double [[TMP25]], i64 0
+; CHECK-NEXT:    [[RES2:%.*]] = fadd <2 x double> [[TMP13]], [[TMP26]]
+; CHECK-NEXT:    store <2 x i64> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x double> [[RES2]]
+;
+  %1 = extractelement <2 x double> %x0, i64 0
+  %2 = extractelement <2 x double> %x1, i64 0
+  %3 = extractelement <2 x double> %x2, i64 0
+  %4 = call double @llvm.fma.f64(double %1, double %2, double %3)
+  %5 = bitcast i8 %x3 to <8 x i1>
+  %6 = extractelement <8 x i1> %5, i64 0
+  %7 = select i1 %6, double %4, double 0.000000e+00
+  %8 = insertelement <2 x double> %x0, double %7, i64 0
+  %9 = extractelement <2 x double> %x0, i64 0
+  %10 = extractelement <2 x double> %x1, i64 0
+  %11 = extractelement <2 x double> %x2, i64 0
+  %12 = call double @llvm.x86.avx512.vfmadd.f64(double %9, double %10, double %11, i32 11)
+  %13 = bitcast i8 %x3 to <8 x i1>
+  %14 = extractelement <8 x i1> %13, i64 0
+  %15 = select i1 %14, double %12, double 0.000000e+00
+  %16 = insertelement <2 x double> %x0, double %15, i64 0
+  %res2 = fadd <2 x double> %8, %16
+  ret <2 x double> %res2
+}
+
+declare float @llvm.fma.f32(float, float, float) #1
+declare float @llvm.x86.avx512.vfmadd.f32(float, float, float, i32) #0
+
+define <4 x float>@test_int_x86_avx512_maskz_vfmadd_ss(<4 x float> %x0, <4 x float> %x1, <4 x float> %x2, i8 %x3,i32 %x4 ){
+; CHECK-LABEL: @test_int_x86_avx512_maskz_vfmadd_ss(
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP1:%.*]] = extractelement <4 x float> [[X0:%.*]], i64 0
+; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <4 x float> [[X1:%.*]], i64 0
+; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <4 x float> [[X2:%.*]], i64 0
+; CHECK-NEXT:    [[TMP4:%.*]] = call float @llvm.fma.f32(float [[TMP1]], float [[TMP2]], float [[TMP3]])
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i8 [[X3:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <8 x i1> [[TMP5]], i64 0
+; CHECK-NEXT:    [[TMP7:%.*]] = select i1 [[TMP6]], i32 0, i32 0
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast float [[TMP4]] to i32
+; CHECK-NEXT:    [[TMP9:%.*]] = xor i32 [[TMP8]], 0
+; CHECK-NEXT:    [[TMP10:%.*]] = or i32 [[TMP9]], 0
+; CHECK-NEXT:    [[TMP11:%.*]] = or i32 [[TMP10]], 0
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select i1 false, i32 [[TMP11]], i32 [[TMP7]]
+; CHECK-NEXT:    [[TMP12:%.*]] = select i1 [[TMP6]], float [[TMP4]], float 0.000000e+00
+; CHECK-NEXT:    [[TMP13:%.*]] = insertelement <4 x float> [[X0]], float [[TMP12]], i64 0
+; CHECK-NEXT:    [[TMP14:%.*]] = extractelement <4 x float> [[X0]], i64 0
+; CHECK-NEXT:    [[TMP15:%.*]] = extractelement <4 x float> [[X1]], i64 0
+; CHECK-NEXT:    [[TMP16:%.*]] = extractelement <4 x float> [[X2]], i64 0
+; CHECK-NEXT:    [[TMP17:%.*]] = call float @llvm.x86.avx512.vfmadd.f32(float [[TMP14]], float [[TMP15]], float [[TMP16]], i32 11)
+; CHECK-NEXT:    [[TMP18:%.*]] = bitcast i8 [[X3]] to <8 x i1>
+; CHECK-NEXT:    [[TMP19:%.*]] = extractelement <8 x i1> [[TMP18]], i64 0
+; CHECK-NEXT:    [[TMP20:%.*]] = select i1 [[TMP19]], i32 0, i32 0
+; CHECK-NEXT:    [[TMP21:%.*]] = bitcast float [[TMP17]] to i32
+; CHECK-NEXT:    [[TMP22:%.*]] = xor i32 [[TMP21]], 0
+; CHECK-NEXT:    [[TMP23:%.*]] = or i32 [[TMP22]], 0
+; CHECK-NEXT:    [[TMP24:%.*]] = or i32 [[TMP23]], 0
+; CHECK-NEXT:    [[_MSPROP_SELECT1:%.*]] = select i1 false, i32 [[TMP24]], i32 [[TMP20]]
+; CHECK-NEXT:    [[TMP25:%.*]] = select i1 [[TMP19]], float [[TMP17]], float 0.000000e+00
+; CHECK-NEXT:    [[TMP26:%.*]] = insertelement <4 x float> [[X0]], float [[TMP25]], i64 0
+; CHECK-NEXT:    [[RES2:%.*]] = fadd <4 x float> [[TMP13]], [[TMP26]]
+; CHECK-NEXT:    store <4 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x float> [[RES2]]
+;
+  %1 = extractelement <4 x float> %x0, i64 0
+  %2 = extractelement <4 x float> %x1, i64 0
+  %3 = extractelement <4 x float> %x2, i64 0
+  %4 = call float @llvm.fma.f32(float %1, float %2, float %3)
+  %5 = bitcast i8 %x3 to <8 x i1>
+  %6 = extractelement <8 x i1> %5, i64 0
+  %7 = select i1 %6, float %4, float 0.000000e+00
+  %8 = insertelement <4 x float> %x0, float %7, i64 0
+  %9 = extractelement <4 x float> %x0, i64 0
+  %10 = extractelement <4 x float> %x1, i64 0
+  %11 = extractelement <4 x float> %x2, i64 0
+  %12 = call float @llvm.x86.avx512.vfmadd.f32(float %9, float %10, float %11, i32 11)
+  %13 = bitcast i8 %x3 to <8 x i1>
+  %14 = extractelement <8 x i1> %13, i64 0
+  %15 = select i1 %14, float %12, float 0.000000e+00
+  %16 = insertelement <4 x float> %x0, float %15, i64 0
+  %res2 = fadd <4 x float> %8, %16
+  ret <4 x float> %res2
+}
+
+define <4 x float> @test_int_x86_avx512_maskz_vfmadd_ss_load0(i8 zeroext %0, ptr nocapture readonly %1, float %2, float %3) #0 {
+; CHECK-LABEL: @test_int_x86_avx512_maskz_vfmadd_ss_load0(
+; CHECK-NEXT:    [[TMP5:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP6:%.*]] = load i32, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP7:%.*]] = load i32, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 24) to ptr), align 8
+; CHECK-NEXT:    [[TMP8:%.*]] = load i8, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP5]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP9:%.*]], label [[TMP10:%.*]], !prof [[PROF1]]
+; CHECK:       9:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       10:
+; CHECK-NEXT:    [[TMP11:%.*]] = load <4 x float>, ptr [[TMP1:%.*]], align 16
+; CHECK-NEXT:    [[TMP12:%.*]] = ptrtoint ptr [[TMP1]] to i64
+; CHECK-NEXT:    [[TMP13:%.*]] = xor i64 [[TMP12]], 87960930222080
+; CHECK-NEXT:    [[TMP14:%.*]] = inttoptr i64 [[TMP13]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <4 x i32>, ptr [[TMP14]], align 16
+; CHECK-NEXT:    [[_MSPROP:%.*]] = extractelement <4 x i32> [[_MSLD]], i64 0
+; CHECK-NEXT:    [[TMP15:%.*]] = extractelement <4 x float> [[TMP11]], i64 0
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or i32 [[_MSPROP]], [[TMP6]]
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = or i32 [[_MSPROP1]], [[TMP7]]
+; CHECK-NEXT:    [[TMP16:%.*]] = tail call float @llvm.fma.f32(float [[TMP15]], float [[TMP2:%.*]], float [[TMP3:%.*]])
+; CHECK-NEXT:    [[TMP17:%.*]] = bitcast i8 [[TMP8]] to <8 x i1>
+; CHECK-NEXT:    [[TMP18:%.*]] = bitcast i8 [[TMP0:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[_MSPROP3:%.*]] = extractelement <8 x i1> [[TMP17]], i64 0
+; CHECK-NEXT:    [[TMP19:%.*]] = extractelement <8 x i1> [[TMP18]], i64 0
+; CHECK-NEXT:    [[TMP20:%.*]] = select i1 [[TMP19]], i32 [[_MSPROP2]], i32 0
+; CHECK-NEXT:    [[TMP21:%.*]] = bitcast float [[TMP16]] to i32
+; CHECK-NEXT:    [[TMP22:%.*]] = xor i32 [[TMP21]], 0
+; CHECK-NEXT:    [[TMP23:%.*]] = or i32 [[TMP22]], [[_MSPROP2]]
+; CHECK-NEXT:    [[TMP24:%.*]] = or i32 [[TMP23]], 0
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select i1 [[_MSPROP3]], i32 [[TMP24]], i32 [[TMP20]]
+; CHECK-NEXT:    [[TMP25:%.*]] = select i1 [[TMP19]], float [[TMP16]], float 0.000000e+00
+; CHECK-NEXT:    [[_MSPROP4:%.*]] = insertelement <4 x i32> [[_MSLD]], i32 [[_MSPROP_SELECT]], i64 0
+; CHECK-NEXT:    [[TMP26:%.*]] = insertelement <4 x float> [[TMP11]], float [[TMP25]], i64 0
+; CHECK-NEXT:    store <4 x i32> [[_MSPROP4]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x float> [[TMP26]]
+;
+  %5 = load <4 x float>, ptr %1, align 16
+  %6 = extractelement <4 x float> %5, i64 0
+  %7 = tail call float @llvm.fma.f32(float %6, float %2, float %3) #2
+  %8 = bitcast i8 %0 to <8 x i1>
+  %9 = extractelement <8 x i1> %8, i64 0
+  %10 = select i1 %9, float %7, float 0.000000e+00
+  %11 = insertelement <4 x float> %5, float %10, i64 0
+  ret <4 x float> %11
+}
+
+define <2 x double> @test_int_x86_avx512_mask3_vfmadd_sd(<2 x double> %x0, <2 x double> %x1, <2 x double> %x2, i8 %x3, i32 %x4) #0 {
+; CHECK-LABEL: @test_int_x86_avx512_mask3_vfmadd_sd(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 48) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = extractelement <2 x i64> [[TMP1]], i64 0
+; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <2 x double> [[X0:%.*]], i64 0
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = extractelement <2 x i64> [[TMP2]], i64 0
+; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <2 x double> [[X1:%.*]], i64 0
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = extractelement <2 x i64> [[TMP3]], i64 0
+; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <2 x double> [[X2:%.*]], i64 0
+; CHECK-NEXT:    [[_MSPROP3:%.*]] = or i64 [[_MSPROP]], [[_MSPROP1]]
+; CHECK-NEXT:    [[_MSPROP4:%.*]] = or i64 [[_MSPROP3]], [[_MSPROP2]]
+; CHECK-NEXT:    [[TMP8:%.*]] = call double @llvm.fma.f64(double [[TMP5]], double [[TMP6]], double [[TMP7]])
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast i8 [[TMP4]] to <8 x i1>
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i8 [[X3:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[_MSPROP5:%.*]] = extractelement <8 x i1> [[TMP9]], i64 0
+; CHECK-NEXT:    [[TMP11:%.*]] = extractelement <8 x i1> [[TMP10]], i64 0
+; CHECK-NEXT:    [[TMP12:%.*]] = select i1 [[TMP11]], i64 [[_MSPROP4]], i64 [[_MSPROP2]]
+; CHECK-NEXT:    [[TMP13:%.*]] = bitcast double [[TMP8]] to i64
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast double [[TMP7]] to i64
+; CHECK-NEXT:    [[TMP15:%.*]] = xor i64 [[TMP13]], [[TMP14]]
+; CHECK-NEXT:    [[TMP16:%.*]] = or i64 [[TMP15]], [[_MSPROP4]]
+; CHECK-NEXT:    [[TMP17:%.*]] = or i64 [[TMP16]], [[_MSPROP2]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select i1 [[_MSPROP5]], i64 [[TMP17]], i64 [[TMP12]]
+; CHECK-NEXT:    [[TMP18:%.*]] = select i1 [[TMP11]], double [[TMP8]], double [[TMP7]]
+; CHECK-NEXT:    [[_MSPROP6:%.*]] = insertelement <2 x i64> [[TMP3]], i64 [[_MSPROP_SELECT]], i64 0
+; CHECK-NEXT:    [[TMP19:%.*]] = insertelement <2 x double> [[X2]], double [[TMP18]], i64 0
+; CHECK-NEXT:    [[_MSPROP7:%.*]] = extractelement <2 x i64> [[TMP1]], i64 0
+; CHECK-NEXT:    [[TMP20:%.*]] = extractelement <2 x double> [[X0]], i64 0
+; CHECK-NEXT:    [[_MSPROP8:%.*]] = extractelement <2 x i64> [[TMP2]], i64 0
+; CHECK-NEXT:    [[TMP21:%.*]] = extractelement <2 x double> [[X1]], i64 0
+; CHECK-NEXT:    [[_MSPROP9:%.*]] = extractelement <2 x i64> [[TMP3]], i64 0
+; CHECK-NEXT:    [[TMP22:%.*]] = extractelement <2 x double> [[X2]], i64 0
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[_MSPROP7]], 0
+; CHECK-NEXT:    [[_MSCMP19:%.*]] = icmp ne i64 [[_MSPROP8]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP19]]
+; CHECK-NEXT:    [[_MSCMP20:%.*]] = icmp ne i64 [[_MSPROP9]], 0
+; CHECK-NEXT:    [[_MSOR21:%.*]] = or i1 [[_MSOR]], [[_MSCMP20]]
+; CHECK-NEXT:    br i1 [[_MSOR21]], label [[TMP23:%.*]], label [[TMP24:%.*]], !prof [[PROF1]]
+; CHECK:       23:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       24:
+; CHECK-NEXT:    [[TMP25:%.*]] = call double @llvm.x86.avx512.vfmadd.f64(double [[TMP20]], double [[TMP21]], double [[TMP22]], i32 11)
+; CHECK-NEXT:    [[_MSPROP10:%.*]] = insertelement <2 x i64> [[TMP3]], i64 0, i64 0
+; CHECK-NEXT:    [[TMP26:%.*]] = insertelement <2 x double> [[X2]], double [[TMP25]], i64 0
+; CHECK-NEXT:    [[_MSPROP11:%.*]] = extractelement <2 x i64> [[TMP1]], i64 0
+; CHECK-NEXT:    [[TMP27:%.*]] = extractelement <2 x double> [[X0]], i64 0
+; CHECK-NEXT:    [[_MSPROP12:%.*]] = extractelement <2 x i64> [[TMP2]], i64 0
+; CHECK-NEXT:    [[TMP28:%.*]] = extractelement <2 x double> [[X1]], i64 0
+; CHECK-NEXT:    [[_MSPROP13:%.*]] = extractelement <2 x i64> [[TMP3]], i64 0
+; CHECK-NEXT:    [[TMP29:%.*]] = extractelement <2 x double> [[X2]], i64 0
+; CHECK-NEXT:    [[_MSCMP22:%.*]] = icmp ne i64 [[_MSPROP11]], 0
+; CHECK-NEXT:    [[_MSCMP23:%.*]] = icmp ne i64 [[_MSPROP12]], 0
+; CHECK-NEXT:    [[_MSOR24:%.*]] = or i1 [[_MSCMP22]], [[_MSCMP23]]
+; CHECK-NEXT:    [[_MSCMP25:%.*]] = icmp ne i64 [[_MSPROP13]], 0
+; CHECK-NEXT:    [[_MSOR26:%.*]] = or i1 [[_MSOR24]], [[_MSCMP25]]
+; CHECK-NEXT:    br i1 [[_MSOR26]], label [[TMP30:%.*]], label [[TMP31:%.*]], !prof [[PROF1]]
+; CHECK:       30:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       31:
+; CHECK-NEXT:    [[TMP32:%.*]] = call double @llvm.x86.avx512.vfmadd.f64(double [[TMP27]], double [[TMP28]], double [[TMP29]], i32 10)
+; CHECK-NEXT:    [[TMP33:%.*]] = bitcast i8 [[TMP4]] to <8 x i1>
+; CHECK-NEXT:    [[TMP34:%.*]] = bitcast i8 [[X3]] to <8 x i1>
+; CHECK-NEXT:    [[_MSPROP14:%.*]] = extractelement <8 x i1> [[TMP33]], i64 0
+; CHECK-NEXT:    [[TMP35:%.*]] = extractelement <8 x i1> [[TMP34]], i64 0
+; CHECK-NEXT:    [[TMP36:%.*]] = select i1 [[TMP35]], i64 0, i64 [[_MSPROP13]]
+; CHECK-NEXT:    [[TMP37:%.*]] = bitcast double [[TMP32]] to i64
+; CHECK-NEXT:    [[TMP38:%.*]] = bitcast double [[TMP29]] to i64
+; CHECK-NEXT:    [[TMP39:%.*]] = xor i64 [[TMP37]], [[TMP38]]
+; CHECK-NEXT:    [[TMP40:%.*]] = or i64 [[TMP39]], 0
+; CHECK-NEXT:    [[TMP41:%.*]] = or i64 [[TMP40]], [[_MSPROP13]]
+; CHECK-NEXT:    [[_MSPROP_SELECT15:%.*]] = select i1 [[_MSPROP14]], i64 [[TMP41]], i64 [[TMP36]]
+; CHECK-NEXT:    [[TMP42:%.*]] = select i1 [[TMP35]], double [[TMP32]], double [[TMP29]]
+; CHECK-NEXT:    [[_MSPROP16:%.*]] = insertelement <2 x i64> [[TMP3]], i64 [[_MSPROP_SELECT15]], i64 0
+; CHECK-NEXT:    [[TMP43:%.*]] = insertelement <2 x double> [[X2]], double [[TMP42]], i64 0
+; CHECK-NEXT:    [[_MSPROP17:%.*]] = or <2 x i64> [[_MSPROP6]], [[_MSPROP10]]
+; CHECK-NEXT:    [[RES3:%.*]] = fadd <2 x double> [[TMP19]], [[TMP26]]
+; CHECK-NEXT:    [[_MSPROP18:%.*]] = or <2 x i64> [[_MSPROP16]], [[_MSPROP17]]
+; CHECK-NEXT:    [[RES4:%.*]] = fadd <2 x double> [[TMP43]], [[RES3]]
+; CHECK-NEXT:    store <2 x i64> [[_MSPROP18]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x double> [[RES4]]
+;
+  %1 = extractelement <2 x double> %x0, i64 0
+  %2 = extractelement <2 x double> %x1, i64 0
+  %3 = extractelement <2 x double> %x2, i64 0
+  %4 = call double @llvm.fma.f64(double %1, double %2, double %3)
+  %5 = bitcast i8 %x3 to <8 x i1>
+  %6 = extractelement <8 x i1> %5, i64 0
+  %7 = select i1 %6, double %4, double %3
+  %8 = insertelement <2 x double> %x2, double %7, i64 0
+  %9 = extractelement <2 x double> %x0, i64 0
+  %10 = extractelement <2 x double> %x1, i64 0
+  %11 = extractelement <2 x double> %x2, i64 0
+  %12 = call double @llvm.x86.avx512.vfmadd.f64(double %9, double %10, double %11, i32 11)
+  %13 = insertelement <2 x double> %x2, double %12, i64 0
+  %14 = extractelement <2 x double> %x0, i64 0
+  %15 = extractelement <2 x double> %x1, i64 0
+  %16 = extractelement <2 x double> %x2, i64 0
+  %17 = call double @llvm.x86.avx512.vfmadd.f64(double %14, double %15, double %16, i32 10)
+  %18 = bitcast i8 %x3 to <8 x i1>
+  %19 = extractelement <8 x i1> %18, i64 0
+  %20 = select i1 %19, double %17, double %16
+  %21 = insertelement <2 x double> %x2, double %20, i64 0
+  %res3 = fadd <2 x double> %8, %13
+  %res4 = fadd <2 x double> %21, %res3
+  ret <2 x double> %res4
+}
+
+define <4 x float> @test_int_x86_avx512_mask3_vfmadd_ss(<4 x float> %x0, <4 x float> %x1, <4 x float> %x2, i8 %x3, i32 %x4) #0 {
+; CHECK-LABEL: @test_int_x86_avx512_mask3_vfmadd_ss(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 48) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = extractelement <4 x i32> [[TMP1]], i64 0
+; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <4 x float> [[X0:%.*]], i64 0
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = extractelement <4 x i32> [[TMP2]], i64 0
+; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <4 x float> [[X1:%.*]], i64 0
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = extractelement <4 x i32> [[TMP3]], i64 0
+; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <4 x float> [[X2:%.*]], i64 0
+; CHECK-NEXT:    [[_MSPROP3:%.*]] = or i32 [[_MSPROP]], [[_MSPROP1]]
+; CHECK-NEXT:    [[_MSPROP4:%.*]] = or i32 [[_MSPROP3]], [[_MSPROP2]]
+; CHECK-NEXT:    [[TMP8:%.*]] = call float @llvm.fma.f32(float [[TMP5]], float [[TMP6]], float [[TMP7]])
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast i8 [[TMP4]] to <8 x i1>
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i8 [[X3:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[_MSPROP5:%.*]] = extractelement <8 x i1> [[TMP9]], i64 0
+; CHECK-NEXT:    [[TMP11:%.*]] = extractelement <8 x i1> [[TMP10]], i64 0
+; CHECK-NEXT:    [[TMP12:%.*]] = select i1 [[TMP11]], i32 [[_MSPROP4]], i32 [[_MSPROP2]]
+; CHECK-NEXT:    [[TMP13:%.*]] = bitcast float [[TMP8]] to i32
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast float [[TMP7]] to i32
+; CHECK-NEXT:    [[TMP15:%.*]] = xor i32 [[TMP13]], [[TMP14]]
+; CHECK-NEXT:    [[TMP16:%.*]] = or i32 [[TMP15]], [[_MSPROP4]]
+; CHECK-NEXT:    [[TMP17:%.*]] = or i32 [[TMP16]], [[_MSPROP2]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select i1 [[_MSPROP5]], i32 [[TMP17]], i32 [[TMP12]]
+; CHECK-NEXT:    [[TMP18:%.*]] = select i1 [[TMP11]], float [[TMP8]], float [[TMP7]]
+; CHECK-NEXT:    [[_MSPROP6:%.*]] = insertelement <4 x i32> [[TMP3]], i32 [[_MSPROP_SELECT]], i64 0
+; CHECK-NEXT:    [[TMP19:%.*]] = insertelement <4 x float> [[X2]], float [[TMP18]], i64 0
+; CHECK-NEXT:    [[_MSPROP7:%.*]] = extractelement <4 x i32> [[TMP1]], i64 0
+; CHECK-NEXT:    [[TMP20:%.*]] = extractelement <4 x float> [[X0]], i64 0
+; CHECK-NEXT:    [[_MSPROP8:%.*]] = extractelement <4 x i32> [[TMP2]], i64 0
+; CHECK-NEXT:    [[TMP21:%.*]] = extractelement <4 x float> [[X1]], i64 0
+; CHECK-NEXT:    [[_MSPROP9:%.*]] = extractelement <4 x i32> [[TMP3]], i64 0
+; CHECK-NEXT:    [[TMP22:%.*]] = extractelement <4 x float> [[X2]], i64 0
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i32 [[_MSPROP7]], 0
+; CHECK-NEXT:    [[_MSCMP19:%.*]] = icmp ne i32 [[_MSPROP8]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP19]]
+; CHECK-NEXT:    [[_MSCMP20:%.*]] = icmp ne i32 [[_MSPROP9]], 0
+; CHECK-NEXT:    [[_MSOR21:%.*]] = or i1 [[_MSOR]], [[_MSCMP20]]
+; CHECK-NEXT:    br i1 [[_MSOR21]], label [[TMP23:%.*]], label [[TMP24:%.*]], !prof [[PROF1]]
+; CHECK:       23:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       24:
+; CHECK-NEXT:    [[TMP25:%.*]] = call float @llvm.x86.avx512.vfmadd.f32(float [[TMP20]], float [[TMP21]], float [[TMP22]], i32 11)
+; CHECK-NEXT:    [[_MSPROP10:%.*]] = insertelement <4 x i32> [[TMP3]], i32 0, i64 0
+; CHECK-NEXT:    [[TMP26:%.*]] = insertelement <4 x float> [[X2]], float [[TMP25]], i64 0
+; CHECK-NEXT:    [[_MSPROP11:%.*]] = extractelement <4 x i32> [[TMP1]], i64 0
+; CHECK-NEXT:    [[TMP27:%.*]] = extractelement <4 x float> [[X0]], i64 0
+; CHECK-NEXT:    [[_MSPROP12:%.*]] = extractelement <4 x i32> [[TMP2]], i64 0
+; CHECK-NEXT:    [[TMP28:%.*]] = extractelement <4 x float> [[X1]], i64 0
+; CHECK-NEXT:    [[_MSPROP13:%.*]] = extractelement <4 x i32> [[TMP3]], i64 0
+; CHECK-NEXT:    [[TMP29:%.*]] = extractelement <4 x float> [[X2]], i64 0
+; CHECK-NEXT:    [[_MSCMP22:%.*]] = icmp ne i32 [[_MSPROP11]], 0
+; CHECK-NEXT:    [[_MSCMP23:%.*]] = icmp ne i32 [[_MSPROP12]], 0
+; CHECK-NEXT:    [[_MSOR24:%.*]] = or i1 [[_MSCMP22]], [[_MSCMP23]]
+; CHECK-NEXT:    [[_MSCMP25:%.*]] = icmp ne i32 [[_MSPROP13]], 0
+; CHECK-NEXT:    [[_MSOR26:%.*]] = or i1 [[_MSOR24]], [[_MSCMP25]]
+; CHECK-NEXT:    br i1 [[_MSOR26]], label [[TMP30:%.*]], label [[TMP31:%.*]], !prof [[PROF1]]
+; CHECK:       30:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       31:
+; CHECK-NEXT:    [[TMP32:%.*]] = call float @llvm.x86.avx512.vfmadd.f32(float [[TMP27]], float [[TMP28]], float [[TMP29]], i32 10)
+; CHECK-NEXT:    [[TMP33:%.*]] = bitcast i8 [[TMP4]] to <8 x i1>
+; CHECK-NEXT:    [[TMP34:%.*]] = bitcast i8 [[X3]] to <8 x i1>
+; CHECK-NEXT:    [[_MSPROP14:%.*]] = extractelement <8 x i1> [[TMP33]], i64 0
+; CHECK-NEXT:    [[TMP35:%.*]] = extractelement <8 x i1> [[TMP34]], i64 0
+; CHECK-NEXT:    [[TMP36:%.*]] = select i1 [[TMP35]], i32 0, i32 [[_MSPROP13]]
+; CHECK-NEXT:    [[TMP37:%.*]] = bitcast float [[TMP32]] to i32
+; CHECK-NEXT:    [[TMP38:%.*]] = bitcast float [[TMP29]] to i32
+; CHECK-NEXT:    [[TMP39:%.*]] = xor i32 [[TMP37]], [[TMP38]]
+; CHECK-NEXT:    [[TMP40:%.*]] = or i32 [[TMP39]], 0
+; CHECK-NEXT:    [[TMP41:%.*]] = or i32 [[TMP40]], [[_MSPROP13]]
+; CHECK-NEXT:    [[_MSPROP_SELECT15:%.*]] = select i1 [[_MSPROP14]], i32 [[TMP41]], i32 [[TMP36]]
+; CHECK-NEXT:    [[TMP42:%.*]] = select i1 [[TMP35]], float [[TMP32]], float [[TMP29]]
+; CHECK-NEXT:    [[_MSPROP16:%.*]] = insertelement <4 x i32> [[TMP3]], i32 [[_MSPROP_SELECT15]], i64 0
+; CHECK-NEXT:    [[TMP43:%.*]] = insertelement <4 x float> [[X2]], float [[TMP42]], i64 0
+; CHECK-NEXT:    [[_MSPROP17:%.*]] = or <4 x i32> [[_MSPROP6]], [[_MSPROP10]]
+; CHECK-NEXT:    [[RES3:%.*]] = fadd <4 x float> [[TMP19]], [[TMP26]]
+; CHECK-NEXT:    [[_MSPROP18:%.*]] = or <4 x i32> [[_MSPROP16]], [[_MSPROP17]]
+; CHECK-NEXT:    [[RES4:%.*]] = fadd <4 x float> [[TMP43]], [[RES3]]
+; CHECK-NEXT:    store <4 x i32> [[_MSPROP18]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x float> [[RES4]]
+;
+  %1 = extractelement <4 x float> %x0, i64 0
+  %2 = extractelement <4 x float> %x1, i64 0
+  %3 = extractelement <4 x float> %x2, i64 0
+  %4 = call float @llvm.fma.f32(float %1, float %2, float %3)
+  %5 = bitcast i8 %x3 to <8 x i1>
+  %6 = extractelement <8 x i1> %5, i64 0
+  %7 = select i1 %6, float %4, float %3
+  %8 = insertelement <4 x float> %x2, float %7, i64 0
+  %9 = extractelement <4 x float> %x0, i64 0
+  %10 = extractelement <4 x float> %x1, i64 0
+  %11 = extractelement <4 x float> %x2, i64 0
+  %12 = call float @llvm.x86.avx512.vfmadd.f32(float %9, float %10, float %11, i32 11)
+  %13 = insertelement <4 x float> %x2, float %12, i64 0
+  %14 = extractelement <4 x float> %x0, i64 0
+  %15 = extractelement <4 x float> %x1, i64 0
+  %16 = extractelement <4 x float> %x2, i64 0
+  %17 = call float @llvm.x86.avx512.vfmadd.f32(float %14, float %15, float %16, i32 10)
+  %18 = bitcast i8 %x3 to <8 x i1>
+  %19 = extractelement <8 x i1> %18, i64 0
+  %20 = select i1 %19, float %17, float %16
+  %21 = insertelement <4 x float> %x2, float %20, i64 0
+  %res3 = fadd <4 x float> %8, %13
+  %res4 = fadd <4 x float> %21, %res3
+  ret <4 x float> %res4
+}
+
+define void @fmadd_ss_mask_memfold(ptr %a, ptr %b, i8 %c, <4 x float> %extra_param, <4 x float> %extra_param2) #0 {
+; CHECK-LABEL: @fmadd_ss_mask_memfold(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 24) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP5:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 40) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP9:%.*]], label [[TMP10:%.*]], !prof [[PROF1]]
+; CHECK:       6:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       7:
+; CHECK-NEXT:    [[A_VAL:%.*]] = load float, ptr [[A:%.*]], align 4
+; CHECK-NEXT:    [[TMP6:%.*]] = ptrtoint ptr [[A]] to i64
+; CHECK-NEXT:    [[TMP7:%.*]] = xor i64 [[TMP6]], 87960930222080
+; CHECK-NEXT:    [[TMP8:%.*]] = inttoptr i64 [[TMP7]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load i32, ptr [[TMP8]], align 4
+; CHECK-NEXT:    [[_MSPROP:%.*]] = insertelement <4 x i32> [[TMP4]], i32 [[_MSLD]], i32 0
+; CHECK-NEXT:    [[AV0:%.*]] = insertelement <4 x float> [[EXTRA_PARAM:%.*]], float [[A_VAL]], i32 0
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = insertelement <4 x i32> [[_MSPROP]], i32 0, i32 1
+; CHECK-NEXT:    [[AV1:%.*]] = insertelement <4 x float> [[AV0]], float 0.000000e+00, i32 1
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = insertelement <4 x i32> [[_MSPROP1]], i32 0, i32 2
+; CHECK-NEXT:    [[AV2:%.*]] = insertelement <4 x float> [[AV1]], float 0.000000e+00, i32 2
+; CHECK-NEXT:    [[_MSPROP3:%.*]] = insertelement <4 x i32> [[_MSPROP2]], i32 0, i32 3
+; CHECK-NEXT:    [[AV:%.*]] = insertelement <4 x float> [[AV2]], float 0.000000e+00, i32 3
+; CHECK-NEXT:    [[_MSCMP17:%.*]] = icmp ne i64 [[TMP2]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP17]], label [[TMP29:%.*]], label [[TMP30:%.*]], !prof [[PROF1]]
+; CHECK:       11:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       12:
+; CHECK-NEXT:    [[B_VAL:%.*]] = load float, ptr [[B:%.*]], align 4
+; CHECK-NEXT:    [[TMP11:%.*]] = ptrtoint ptr [[B]] to i64
+; CHECK-NEXT:    [[TMP12:%.*]] = xor i64 [[TMP11]], 87960930222080
+; CHECK-NEXT:    [[TMP13:%.*]] = inttoptr i64 [[TMP12]] to ptr
+; CHECK-NEXT:    [[_MSLD4:%.*]] = load i32, ptr [[TMP13]], align 4
+; CHECK-NEXT:    [[_MSPROP5:%.*]] = insertelement <4 x i32> [[TMP5]], i32 [[_MSLD4]], i32 0
+; CHECK-NEXT:    [[BV0:%.*]] = insertelement <4 x float> [[EXTRA_PARAM2:%.*]], float [[B_VAL]], i32 0
+; CHECK-NEXT:    [[_MSPROP6:%.*]] = insertelement <4 x i32> [[_MSPROP5]], i32 0, i32 1
+; CHECK-NEXT:    [[BV1:%.*]] = insertelement <4 x float> [[BV0]], float 0.000000e+00, i32 1
+; CHECK-NEXT:    [[_MSPROP7:%.*]] = insertelement <4 x i32> [[_MSPROP6]], i32 0, i32 2
+; CHECK-NEXT:    [[BV2:%.*]] = insertelement <4 x float> [[BV1]], float 0.000000e+00, i32 2
+; CHECK-NEXT:    [[_MSPROP8:%.*]] = insertelement <4 x i32> [[_MSPROP7]], i32 0, i32 3
+; CHECK-NEXT:    [[BV:%.*]] = insertelement <4 x float> [[BV2]], float 0.000000e+00, i32 3
+; CHECK-NEXT:    [[_MSPROP9:%.*]] = extractelement <4 x i32> [[_MSPROP3]], i64 0
+; CHECK-NEXT:    [[TMP14:%.*]] = extractelement <4 x float> [[AV]], i64 0
+; CHECK-NEXT:    [[_MSPROP10:%.*]] = extractelement <4 x i32> [[_MSPROP8]], i64 0
+; CHECK-NEXT:    [[TMP15:%.*]] = extractelement <4 x float> [[BV]], i64 0
+; CHECK-NEXT:    [[_MSPROP11:%.*]] = extractelement <4 x i32> [[_MSPROP3]], i64 0
+; CHECK-NEXT:    [[TMP16:%.*]] = extractelement <4 x float> [[AV]], i64 0
+; CHECK-NEXT:    [[_MSPROP12:%.*]] = or i32 [[_MSPROP9]], [[_MSPROP10]]
+; CHECK-NEXT:    [[_MSPROP13:%.*]] = or i32 [[_MSPROP12]], [[_MSPROP11]]
+; CHECK-NEXT:    [[TMP17:%.*]] = call float @llvm.fma.f32(float [[TMP14]], float [[TMP15]], float [[TMP16]])
+; CHECK-NEXT:    [[TMP18:%.*]] = bitcast i8 [[TMP3]] to <8 x i1>
+; CHECK-NEXT:    [[TMP19:%.*]] = bitcast i8 [[C:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[_MSPROP14:%.*]] = extractelement <8 x i1> [[TMP18]], i64 0
+; CHECK-NEXT:    [[TMP20:%.*]] = extractelement <8 x i1> [[TMP19]], i64 0
+; CHECK-NEXT:    [[TMP21:%.*]] = select i1 [[TMP20]], i32 [[_MSPROP13]], i32 [[_MSPROP9]]
+; CHECK-NEXT:    [[TMP22:%.*]] = bitcast float [[TMP17]] to i32
+; CHECK-NEXT:    [[TMP23:%.*]] = bitcast float [[TMP14]] to i32
+; CHECK-NEXT:    [[TMP24:%.*]] = xor i32 [[TMP22]], [[TMP23]]
+; CHECK-NEXT:    [[TMP25:%.*]] = or i32 [[TMP24]], [[_MSPROP13]]
+; CHECK-NEXT:    [[TMP26:%.*]] = or i32 [[TMP25]], [[_MSPROP9]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select i1 [[_MSPROP14]], i32 [[TMP26]], i32 [[TMP21]]
+; CHECK-NEXT:    [[TMP27:%.*]] = select i1 [[TMP20]], float [[TMP17]], float [[TMP14]]
+; CHECK-NEXT:    [[_MSPROP15:%.*]] = insertelement <4 x i32> [[_MSPROP3]], i32 [[_MSPROP_SELECT]], i64 0
+; CHECK-NEXT:    [[TMP28:%.*]] = insertelement <4 x float> [[AV]], float [[TMP27]], i64 0
+; CHECK-NEXT:    [[_MSPROP16:%.*]] = extractelement <4 x i32> [[_MSPROP15]], i32 0
+; CHECK-NEXT:    [[SR:%.*]] = extractelement <4 x float> [[TMP28]], i32 0
+; CHECK-NEXT:    [[_MSCMP18:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP18]], label [[TMP34:%.*]], label [[TMP35:%.*]], !prof [[PROF1]]
+; CHECK:       31:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       32:
+; CHECK-NEXT:    [[TMP31:%.*]] = ptrtoint ptr [[A]] to i64
+; CHECK-NEXT:    [[TMP32:%.*]] = xor i64 [[TMP31]], 87960930222080
+; CHECK-NEXT:    [[TMP33:%.*]] = inttoptr i64 [[TMP32]] to ptr
+; CHECK-NEXT:    store i32 [[_MSPROP16]], ptr [[TMP33]], align 4
+; CHECK-NEXT:    store float [[SR]], ptr [[A]], align 4
+; CHECK-NEXT:    ret void
+;
+  %a.val = load float, ptr %a
+  %av0 = insertelement <4 x float> %extra_param, float %a.val, i32 0
+  %av1 = insertelement <4 x float> %av0, float 0.000000e+00, i32 1
+  %av2 = insertelement <4 x float> %av1, float 0.000000e+00, i32 2
+  %av  = insertelement <4 x float> %av2, float 0.000000e+00, i32 3
+
+  %b.val = load float, ptr %b
+  %bv0 = insertelement <4 x float> %extra_param2, float %b.val, i32 0
+  %bv1 = insertelement <4 x float> %bv0, float 0.000000e+00, i32 1
+  %bv2 = insertelement <4 x float> %bv1, float 0.000000e+00, i32 2
+  %bv =  insertelement <4 x float> %bv2, float 0.000000e+00, i32 3
+  %1 = extractelement <4 x float> %av, i64 0
+  %2 = extractelement <4 x float> %bv, i64 0
+  %3 = extractelement <4 x float> %av, i64 0
+  %4 = call float @llvm.fma.f32(float %1, float %2, float %3)
+  %5 = bitcast i8 %c to <8 x i1>
+  %6 = extractelement <8 x i1> %5, i64 0
+  %7 = select i1 %6, float %4, float %1
+  %8 = insertelement <4 x float> %av, float %7, i64 0
+  %sr = extractelement <4 x float> %8, i32 0
+  store float %sr, ptr %a
+  ret void
+}
+
+define void @fmadd_ss_maskz_memfold(ptr %a, ptr %b, i8 %c, <4 x float> %extra_param, <4 x float> %extra_param2) #0 {
+; CHECK-LABEL: @fmadd_ss_maskz_memfold(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 24) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP5:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 40) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP9:%.*]], label [[TMP10:%.*]], !prof [[PROF1]]
+; CHECK:       6:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       7:
+; CHECK-NEXT:    [[A_VAL:%.*]] = load float, ptr [[A:%.*]], align 4
+; CHECK-NEXT:    [[TMP6:%.*]] = ptrtoint ptr [[A]] to i64
+; CHECK-NEXT:    [[TMP7:%.*]] = xor i64 [[TMP6]], 87960930222080
+; CHECK-NEXT:    [[TMP8:%.*]] = inttoptr i64 [[TMP7]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load i32, ptr [[TMP8]], align 4
+; CHECK-NEXT:    [[_MSPROP:%.*]] = insertelement <4 x i32> [[TMP4]], i32 [[_MSLD]], i32 0
+; CHECK-NEXT:    [[AV0:%.*]] = insertelement <4 x float> [[EXTRA_PARAM:%.*]], float [[A_VAL]], i32 0
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = insertelement <4 x i32> [[_MSPROP]], i32 0, i32 1
+; CHECK-NEXT:    [[AV1:%.*]] = insertelement <4 x float> [[AV0]], float 0.000000e+00, i32 1
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = insertelement <4 x i32> [[_MSPROP1]], i32 0, i32 2
+; CHECK-NEXT:    [[AV2:%.*]] = insertelement <4 x float> [[AV1]], float 0.000000e+00, i32 2
+; CHECK-NEXT:    [[_MSPROP3:%.*]] = insertelement <4 x i32> [[_MSPROP2]], i32 0, i32 3
+; CHECK-NEXT:    [[AV:%.*]] = insertelement <4 x float> [[AV2]], float 0.000000e+00, i32 3
+; CHECK-NEXT:    [[_MSCMP17:%.*]] = icmp ne i64 [[TMP2]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP17]], label [[TMP28:%.*]], label [[TMP29:%.*]], !prof [[PROF1]]
+; CHECK:       11:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       12:
+; CHECK-NEXT:    [[B_VAL:%.*]] = load float, ptr [[B:%.*]], align 4
+; CHECK-NEXT:    [[TMP11:%.*]] = ptrtoint ptr [[B]] to i64
+; CHECK-NEXT:    [[TMP12:%.*]] = xor i64 [[TMP11]], 87960930222080
+; CHECK-NEXT:    [[TMP13:%.*]] = inttoptr i64 [[TMP12]] to ptr
+; CHECK-NEXT:    [[_MSLD4:%.*]] = load i32, ptr [[TMP13]], align 4
+; CHECK-NEXT:    [[_MSPROP5:%.*]] = insertelement <4 x i32> [[TMP5]], i32 [[_MSLD4]], i32 0
+; CHECK-NEXT:    [[BV0:%.*]] = insertelement <4 x float> [[EXTRA_PARAM2:%.*]], float [[B_VAL]], i32 0
+; CHECK-NEXT:    [[_MSPROP6:%.*]] = insertelement <4 x i32> [[_MSPROP5]], i32 0, i32 1
+; CHECK-NEXT:    [[BV1:%.*]] = insertelement <4 x float> [[BV0]], float 0.000000e+00, i32 1
+; CHECK-NEXT:    [[_MSPROP7:%.*]] = insertelement <4 x i32> [[_MSPROP6]], i32 0, i32 2
+; CHECK-NEXT:    [[BV2:%.*]] = insertelement <4 x float> [[BV1]], float 0.000000e+00, i32 2
+; CHECK-NEXT:    [[_MSPROP8:%.*]] = insertelement <4 x i32> [[_MSPROP7]], i32 0, i32 3
+; CHECK-NEXT:    [[BV:%.*]] = insertelement <4 x float> [[BV2]], float 0.000000e+00, i32 3
+; CHECK-NEXT:    [[_MSPROP9:%.*]] = extractelement <4 x i32> [[_MSPROP3]], i64 0
+; CHECK-NEXT:    [[TMP14:%.*]] = extractelement <4 x float> [[AV]], i64 0
+; CHECK-NEXT:    [[_MSPROP10:%.*]] = extractelement <4 x i32> [[_MSPROP8]], i64 0
+; CHECK-NEXT:    [[TMP15:%.*]] = extractelement <4 x float> [[BV]], i64 0
+; CHECK-NEXT:    [[_MSPROP11:%.*]] = extractelement <4 x i32> [[_MSPROP3]], i64 0
+; CHECK-NEXT:    [[TMP16:%.*]] = extractelement <4 x float> [[AV]], i64 0
+; CHECK-NEXT:    [[_MSPROP12:%.*]] = or i32 [[_MSPROP9]], [[_MSPROP10]]
+; CHECK-NEXT:    [[_MSPROP13:%.*]] = or i32 [[_MSPROP12]], [[_MSPROP11]]
+; CHECK-NEXT:    [[TMP17:%.*]] = call float @llvm.fma.f32(float [[TMP14]], float [[TMP15]], float [[TMP16]])
+; CHECK-NEXT:    [[TMP18:%.*]] = bitcast i8 [[TMP3]] to <8 x i1>
+; CHECK-NEXT:    [[TMP19:%.*]] = bitcast i8 [[C:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[_MSPROP14:%.*]] = extractelement <8 x i1> [[TMP18]], i64 0
+; CHECK-NEXT:    [[TMP20:%.*]] = extractelement <8 x i1> [[TMP19]], i64 0
+; CHECK-NEXT:    [[TMP21:%.*]] = select i1 [[TMP20]], i32 [[_MSPROP13]], i32 0
+; CHECK-NEXT:    [[TMP22:%.*]] = bitcast float [[TMP17]] to i32
+; CHECK-NEXT:    [[TMP23:%.*]] = xor i32 [[TMP22]], 0
+; CHECK-NEXT:    [[TMP24:%.*]] = or i32 [[TMP23]], [[_MSPROP13]]
+; CHECK-NEXT:    [[TMP25:%.*]] = or i32 [[TMP24]], 0
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select i1 [[_MSPROP14]], i32 [[TMP25]], i32 [[TMP21]]
+; CHECK-NEXT:    [[TMP26:%.*]] = select i1 [[TMP20]], float [[TMP17]], float 0.000000e+00
+; CHECK-NEXT:    [[_MSPROP15:%.*]] = insertelement <4 x i32> [[_MSPROP3]], i32 [[_MSPROP_SELECT]], i64 0
+; CHECK-NEXT:    [[TMP27:%.*]] = insertelement <4 x float> [[AV]], float [[TMP26]], i64 0
+; CHECK-NEXT:    [[_MSPROP16:%.*]] = extractelement <4 x i32> [[_MSPROP15]], i32 0
+; CHECK-NEXT:    [[SR:%.*]] = extractelement <4 x float> [[TMP27]], i32 0
+; CHECK-NEXT:    [[_MSCMP18:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP18]], label [[TMP33:%.*]], label [[TMP34:%.*]], !prof [[PROF1]]
+; CHECK:       30:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       31:
+; CHECK-NEXT:    [[TMP30:%.*]] = ptrtoint ptr [[A]] to i64
+; CHECK-NEXT:    [[TMP31:%.*]] = xor i64 [[TMP30]], 87960930222080
+; CHECK-NEXT:    [[TMP32:%.*]] = inttoptr i64 [[TMP31]] to ptr
+; CHECK-NEXT:    store i32 [[_MSPROP16]], ptr [[TMP32]], align 4
+; CHECK-NEXT:    store float [[SR]], ptr [[A]], align 4
+; CHECK-NEXT:    ret void
+;
+  %a.val = load float, ptr %a
+  %av0 = insertelement <4 x float> %extra_param, float %a.val, i32 0
+  %av1 = insertelement <4 x float> %av0, float 0.000000e+00, i32 1
+  %av2 = insertelement <4 x float> %av1, float 0.000000e+00, i32 2
+  %av  = insertelement <4 x float> %av2, float 0.000000e+00, i32 3
+
+  %b.val = load float, ptr %b
+  %bv0 = insertelement <4 x float> %extra_param2, float %b.val, i32 0
+  %bv1 = insertelement <4 x float> %bv0, float 0.000000e+00, i32 1
+  %bv2 = insertelement <4 x float> %bv1, float 0.000000e+00, i32 2
+  %bv  = insertelement <4 x float> %bv2, float 0.000000e+00, i32 3
+  %1 = extractelement <4 x float> %av, i64 0
+  %2 = extractelement <4 x float> %bv, i64 0
+  %3 = extractelement <4 x float> %av, i64 0
+  %4 = call float @llvm.fma.f32(float %1, float %2, float %3)
+  %5 = bitcast i8 %c to <8 x i1>
+  %6 = extractelement <8 x i1> %5, i64 0
+  %7 = select i1 %6, float %4, float 0.000000e+00
+  %8 = insertelement <4 x float> %av, float %7, i64 0
+  %sr = extractelement <4 x float> %8, i32 0
+  store float %sr, ptr %a
+  ret void
+}
+
+define void @fmadd_sd_mask_memfold(ptr %a, ptr %b, i8 %c, <2 x double> %extra_param, <2 x double> %extra_param2) #0 {
+; CHECK-LABEL: @fmadd_sd_mask_memfold(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 24) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP5:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 40) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP9:%.*]], label [[TMP10:%.*]], !prof [[PROF1]]
+; CHECK:       6:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       7:
+; CHECK-NEXT:    [[A_VAL:%.*]] = load double, ptr [[A:%.*]], align 8
+; CHECK-NEXT:    [[TMP6:%.*]] = ptrtoint ptr [[A]] to i64
+; CHECK-NEXT:    [[TMP7:%.*]] = xor i64 [[TMP6]], 87960930222080
+; CHECK-NEXT:    [[TMP8:%.*]] = inttoptr i64 [[TMP7]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load i64, ptr [[TMP8]], align 8
+; CHECK-NEXT:    [[_MSPROP:%.*]] = insertelement <2 x i64> [[TMP4]], i64 [[_MSLD]], i32 0
+; CHECK-NEXT:    [[AV0:%.*]] = insertelement <2 x double> [[EXTRA_PARAM:%.*]], double [[A_VAL]], i32 0
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = insertelement <2 x i64> [[_MSPROP]], i64 0, i32 1
+; CHECK-NEXT:    [[AV:%.*]] = insertelement <2 x double> [[AV0]], double 0.000000e+00, i32 1
+; CHECK-NEXT:    [[_MSCMP13:%.*]] = icmp ne i64 [[TMP2]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP13]], label [[TMP29:%.*]], label [[TMP30:%.*]], !prof [[PROF1]]
+; CHECK:       11:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       12:
+; CHECK-NEXT:    [[B_VAL:%.*]] = load double, ptr [[B:%.*]], align 8
+; CHECK-NEXT:    [[TMP11:%.*]] = ptrtoint ptr [[B]] to i64
+; CHECK-NEXT:    [[TMP12:%.*]] = xor i64 [[TMP11]], 87960930222080
+; CHECK-NEXT:    [[TMP13:%.*]] = inttoptr i64 [[TMP12]] to ptr
+; CHECK-NEXT:    [[_MSLD2:%.*]] = load i64, ptr [[TMP13]], align 8
+; CHECK-NEXT:    [[_MSPROP3:%.*]] = insertelement <2 x i64> [[TMP5]], i64 [[_MSLD2]], i32 0
+; CHECK-NEXT:    [[BV0:%.*]] = insertelement <2 x double> [[EXTRA_PARAM2:%.*]], double [[B_VAL]], i32 0
+; CHECK-NEXT:    [[_MSPROP4:%.*]] = insertelement <2 x i64> [[_MSPROP3]], i64 0, i32 1
+; CHECK-NEXT:    [[BV:%.*]] = insertelement <2 x double> [[BV0]], double 0.000000e+00, i32 1
+; CHECK-NEXT:    [[_MSPROP5:%.*]] = extractelement <2 x i64> [[_MSPROP1]], i64 0
+; CHECK-NEXT:    [[TMP14:%.*]] = extractelement <2 x double> [[AV]], i64 0
+; CHECK-NEXT:    [[_MSPROP6:%.*]] = extractelement <2 x i64> [[_MSPROP4]], i64 0
+; CHECK-NEXT:    [[TMP15:%.*]] = extractelement <2 x double> [[BV]], i64 0
+; CHECK-NEXT:    [[_MSPROP7:%.*]] = extractelement <2 x i64> [[_MSPROP1]], i64 0
+; CHECK-NEXT:    [[TMP16:%.*]] = extractelement <2 x double> [[AV]], i64 0
+; CHECK-NEXT:    [[_MSPROP8:%.*]] = or i64 [[_MSPROP5]], [[_MSPROP6]]
+; CHECK-NEXT:    [[_MSPROP9:%.*]] = or i64 [[_MSPROP8]], [[_MSPROP7]]
+; CHECK-NEXT:    [[TMP17:%.*]] = call double @llvm.fma.f64(double [[TMP14]], double [[TMP15]], double [[TMP16]])
+; CHECK-NEXT:    [[TMP18:%.*]] = bitcast i8 [[TMP3]] to <8 x i1>
+; CHECK-NEXT:    [[TMP19:%.*]] = bitcast i8 [[C:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[_MSPROP10:%.*]] = extractelement <8 x i1> [[TMP18]], i64 0
+; CHECK-NEXT:    [[TMP20:%.*]] = extractelement <8 x i1> [[TMP19]], i64 0
+; CHECK-NEXT:    [[TMP21:%.*]] = select i1 [[TMP20]], i64 [[_MSPROP9]], i64 [[_MSPROP5]]
+; CHECK-NEXT:    [[TMP22:%.*]] = bitcast double [[TMP17]] to i64
+; CHECK-NEXT:    [[TMP23:%.*]] = bitcast double [[TMP14]] to i64
+; CHECK-NEXT:    [[TMP24:%.*]] = xor i64 [[TMP22]], [[TMP23]]
+; CHECK-NEXT:    [[TMP25:%.*]] = or i64 [[TMP24]], [[_MSPROP9]]
+; CHECK-NEXT:    [[TMP26:%.*]] = or i64 [[TMP25]], [[_MSPROP5]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select i1 [[_MSPROP10]], i64 [[TMP26]], i64 [[TMP21]]
+; CHECK-NEXT:    [[TMP27:%.*]] = select i1 [[TMP20]], double [[TMP17]], double [[TMP14]]
+; CHECK-NEXT:    [[_MSPROP11:%.*]] = insertelement <2 x i64> [[_MSPROP1]], i64 [[_MSPROP_SELECT]], i64 0
+; CHECK-NEXT:    [[TMP28:%.*]] = insertelement <2 x double> [[AV]], double [[TMP27]], i64 0
+; CHECK-NEXT:    [[_MSPROP12:%.*]] = extractelement <2 x i64> [[_MSPROP11]], i32 0
+; CHECK-NEXT:    [[SR:%.*]] = extractelement <2 x double> [[TMP28]], i32 0
+; CHECK-NEXT:    [[_MSCMP14:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP14]], label [[TMP34:%.*]], label [[TMP35:%.*]], !prof [[PROF1]]
+; CHECK:       31:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       32:
+; CHECK-NEXT:    [[TMP31:%.*]] = ptrtoint ptr [[A]] to i64
+; CHECK-NEXT:    [[TMP32:%.*]] = xor i64 [[TMP31]], 87960930222080
+; CHECK-NEXT:    [[TMP33:%.*]] = inttoptr i64 [[TMP32]] to ptr
+; CHECK-NEXT:    store i64 [[_MSPROP12]], ptr [[TMP33]], align 8
+; CHECK-NEXT:    store double [[SR]], ptr [[A]], align 8
+; CHECK-NEXT:    ret void
+;
+  %a.val = load double, ptr %a
+  %av0 = insertelement <2 x double> %extra_param, double %a.val, i32 0
+  %av = insertelement <2 x double> %av0, double 0.000000e+00, i32 1
+
+  %b.val = load double, ptr %b
+  %bv0 = insertelement <2 x double> %extra_param2, double %b.val, i32 0
+  %bv = insertelement <2 x double> %bv0, double 0.000000e+00, i32 1
+  %1 = extractelement <2 x double> %av, i64 0
+  %2 = extractelement <2 x double> %bv, i64 0
+  %3 = extractelement <2 x double> %av, i64 0
+  %4 = call double @llvm.fma.f64(double %1, double %2, double %3)
+  %5 = bitcast i8 %c to <8 x i1>
+  %6 = extractelement <8 x i1> %5, i64 0
+  %7 = select i1 %6, double %4, double %1
+  %8 = insertelement <2 x double> %av, double %7, i64 0
+  %sr = extractelement <2 x double> %8, i32 0
+  store double %sr, ptr %a
+  ret void
+}
+
+define void @fmadd_sd_maskz_memfold(ptr %a, ptr %b, i8 %c, <2x double> %extra_param, <2x double> %extra_param2) #0 {
+; CHECK-LABEL: @fmadd_sd_maskz_memfold(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 24) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP5:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 40) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP9:%.*]], label [[TMP10:%.*]], !prof [[PROF1]]
+; CHECK:       6:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       7:
+; CHECK-NEXT:    [[A_VAL:%.*]] = load double, ptr [[A:%.*]], align 8
+; CHECK-NEXT:    [[TMP6:%.*]] = ptrtoint ptr [[A]] to i64
+; CHECK-NEXT:    [[TMP7:%.*]] = xor i64 [[TMP6]], 87960930222080
+; CHECK-NEXT:    [[TMP8:%.*]] = inttoptr i64 [[TMP7]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load i64, ptr [[TMP8]], align 8
+; CHECK-NEXT:    [[_MSPROP:%.*]] = insertelement <2 x i64> [[TMP4]], i64 [[_MSLD]], i32 0
+; CHECK-NEXT:    [[AV0:%.*]] = insertelement <2 x double> [[EXTRA_PARAM:%.*]], double [[A_VAL]], i32 0
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = insertelement <2 x i64> [[_MSPROP]], i64 0, i32 1
+; CHECK-NEXT:    [[AV:%.*]] = insertelement <2 x double> [[AV0]], double 0.000000e+00, i32 1
+; CHECK-NEXT:    [[_MSCMP13:%.*]] = icmp ne i64 [[TMP2]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP13]], label [[TMP28:%.*]], label [[TMP29:%.*]], !prof [[PROF1]]
+; CHECK:       11:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       12:
+; CHECK-NEXT:    [[B_VAL:%.*]] = load double, ptr [[B:%.*]], align 8
+; CHECK-NEXT:    [[TMP11:%.*]] = ptrtoint ptr [[B]] to i64
+; CHECK-NEXT:    [[TMP12:%.*]] = xor i64 [[TMP11]], 87960930222080
+; CHECK-NEXT:    [[TMP13:%.*]] = inttoptr i64 [[TMP12]] to ptr
+; CHECK-NEXT:    [[_MSLD2:%.*]] = load i64, ptr [[TMP13]], align 8
+; CHECK-NEXT:    [[_MSPROP3:%.*]] = insertelement <2 x i64> [[TMP5]], i64 [[_MSLD2]], i32 0
+; CHECK-NEXT:    [[BV0:%.*]] = insertelement <2 x double> [[EXTRA_PARAM2:%.*]], double [[B_VAL]], i32 0
+; CHECK-NEXT:    [[_MSPROP4:%.*]] = insertelement <2 x i64> [[_MSPROP3]], i64 0, i32 1
+; CHECK-NEXT:    [[BV:%.*]] = insertelement <2 x double> [[BV0]], double 0.000000e+00, i32 1
+; CHECK-NEXT:    [[_MSPROP5:%.*]] = extractelement <2 x i64> [[_MSPROP1]], i64 0
+; CHECK-NEXT:    [[TMP14:%.*]] = extractelement <2 x double> [[AV]], i64 0
+; CHECK-NEXT:    [[_MSPROP6:%.*]] = extractelement <2 x i64> [[_MSPROP4]], i64 0
+; CHECK-NEXT:    [[TMP15:%.*]] = extractelement <2 x double> [[BV]], i64 0
+; CHECK-NEXT:    [[_MSPROP7:%.*]] = extractelement <2 x i64> [[_MSPROP1]], i64 0
+; CHECK-NEXT:    [[TMP16:%.*]] = extractelement <2 x double> [[AV]], i64 0
+; CHECK-NEXT:    [[_MSPROP8:%.*]] = or i64 [[_MSPROP5]], [[_MSPROP6]]
+; CHECK-NEXT:    [[_MSPROP9:%.*]] = or i64 [[_MSPROP8]], [[_MSPROP7]]
+; CHECK-NEXT:    [[TMP17:%.*]] = call double @llvm.fma.f64(double [[TMP14]], double [[TMP15]], double [[TMP16]])
+; CHECK-NEXT:    [[TMP18:%.*]] = bitcast i8 [[TMP3]] to <8 x i1>
+; CHECK-NEXT:    [[TMP19:%.*]] = bitcast i8 [[C:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[_MSPROP10:%.*]] = extractelement <8 x i1> [[TMP18]], i64 0
+; CHECK-NEXT:    [[TMP20:%.*]] = extractelement <8 x i1> [[TMP19]], i64 0
+; CHECK-NEXT:    [[TMP21:%.*]] = select i1 [[TMP20]], i64 [[_MSPROP9]], i64 0
+; CHECK-NEXT:    [[TMP22:%.*]] = bitcast double [[TMP17]] to i64
+; CHECK-NEXT:    [[TMP23:%.*]] = xor i64 [[TMP22]], 0
+; CHECK-NEXT:    [[TMP24:%.*]] = or i64 [[TMP23]], [[_MSPROP9]]
+; CHECK-NEXT:    [[TMP25:%.*]] = or i64 [[TMP24]], 0
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select i1 [[_MSPROP10]], i64 [[TMP25]], i64 [[TMP21]]
+; CHECK-NEXT:    [[TMP26:%.*]] = select i1 [[TMP20]], double [[TMP17]], double 0.000000e+00
+; CHECK-NEXT:    [[_MSPROP11:%.*]] = insertelement <2 x i64> [[_MSPROP1]], i64 [[_MSPROP_SELECT]], i64 0
+; CHECK-NEXT:    [[TMP27:%.*]] = insertelement <2 x double> [[AV]], double [[TMP26]], i64 0
+; CHECK-NEXT:    [[_MSPROP12:%.*]] = extractelement <2 x i64> [[_MSPROP11]], i32 0
+; CHECK-NEXT:    [[SR:%.*]] = extractelement <2 x double> [[TMP27]], i32 0
+; CHECK-NEXT:    [[_MSCMP14:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP14]], label [[TMP33:%.*]], label [[TMP34:%.*]], !prof [[PROF1]]
+; CHECK:       30:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       31:
+; CHECK-NEXT:    [[TMP30:%.*]] = ptrtoint ptr [[A]] to i64
+; CHECK-NEXT:    [[TMP31:%.*]] = xor i64 [[TMP30]], 87960930222080
+; CHECK-NEXT:    [[TMP32:%.*]] = inttoptr i64 [[TMP31]] to ptr
+; CHECK-NEXT:    store i64 [[_MSPROP12]], ptr [[TMP32]], align 8
+; CHECK-NEXT:    store double [[SR]], ptr [[A]], align 8
+; CHECK-NEXT:    ret void
+;
+  %a.val = load double, ptr %a
+  %av0 = insertelement <2 x double> %extra_param, double %a.val, i32 0
+  %av = insertelement <2 x double> %av0, double 0.000000e+00, i32 1
+
+  %b.val = load double, ptr %b
+  %bv0 = insertelement <2 x double> %extra_param2, double %b.val, i32 0
+  %bv = insertelement <2 x double> %bv0, double 0.000000e+00, i32 1
+  %1 = extractelement <2 x double> %av, i64 0
+  %2 = extractelement <2 x double> %bv, i64 0
+  %3 = extractelement <2 x double> %av, i64 0
+  %4 = call double @llvm.fma.f64(double %1, double %2, double %3)
+  %5 = bitcast i8 %c to <8 x i1>
+  %6 = extractelement <8 x i1> %5, i64 0
+  %7 = select i1 %6, double %4, double 0.000000e+00
+  %8 = insertelement <2 x double> %av, double %7, i64 0
+  %sr = extractelement <2 x double> %8, i32 0
+  store double %sr, ptr %a
+  ret void
+}
+
+define <2 x double> @test_int_x86_avx512_mask3_vfmsub_sd(<2 x double> %x0, <2 x double> %x1, <2 x double> %x2, i8 %x3, i32 %x4) #0 {
+; CHECK-LABEL: @test_int_x86_avx512_mask3_vfmsub_sd(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 48) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = fneg <2 x double> [[X2:%.*]]
+; CHECK-NEXT:    [[_MSPROP:%.*]] = extractelement <2 x i64> [[TMP2]], i64 0
+; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <2 x double> [[X0:%.*]], i64 0
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = extractelement <2 x i64> [[TMP3]], i64 0
+; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <2 x double> [[X1:%.*]], i64 0
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = extractelement <2 x i64> [[TMP1]], i64 0
+; CHECK-NEXT:    [[TMP8:%.*]] = extractelement <2 x double> [[TMP5]], i64 0
+; CHECK-NEXT:    [[_MSPROP3:%.*]] = or i64 [[_MSPROP]], [[_MSPROP1]]
+; CHECK-NEXT:    [[_MSPROP4:%.*]] = or i64 [[_MSPROP3]], [[_MSPROP2]]
+; CHECK-NEXT:    [[TMP9:%.*]] = call double @llvm.fma.f64(double [[TMP6]], double [[TMP7]], double [[TMP8]])
+; CHECK-NEXT:    [[_MSPROP5:%.*]] = extractelement <2 x i64> [[TMP1]], i64 0
+; CHECK-NEXT:    [[TMP10:%.*]] = extractelement <2 x double> [[X2]], i64 0
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast i8 [[TMP4]] to <8 x i1>
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast i8 [[X3:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[_MSPROP6:%.*]] = extractelement <8 x i1> [[TMP11]], i64 0
+; CHECK-NEXT:    [[TMP13:%.*]] = extractelement <8 x i1> [[TMP12]], i64 0
+; CHECK-NEXT:    [[TMP14:%.*]] = select i1 [[TMP13]], i64 [[_MSPROP4]], i64 [[_MSPROP5]]
+; CHECK-NEXT:    [[TMP15:%.*]] = bitcast double [[TMP9]] to i64
+; CHECK-NEXT:    [[TMP16:%.*]] = bitcast double [[TMP10]] to i64
+; CHECK-NEXT:    [[TMP17:%.*]] = xor i64 [[TMP15]], [[TMP16]]
+; CHECK-NEXT:    [[TMP18:%.*]] = or i64 [[TMP17]], [[_MSPROP4]]
+; CHECK-NEXT:    [[TMP19:%.*]] = or i64 [[TMP18]], [[_MSPROP5]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select i1 [[_MSPROP6]], i64 [[TMP19]], i64 [[TMP14]]
+; CHECK-NEXT:    [[TMP20:%.*]] = select i1 [[TMP13]], double [[TMP9]], double [[TMP10]]
+; CHECK-NEXT:    [[_MSPROP7:%.*]] = insertelement <2 x i64> [[TMP1]], i64 [[_MSPROP_SELECT]], i64 0
+; CHECK-NEXT:    [[TMP21:%.*]] = insertelement <2 x double> [[X2]], double [[TMP20]], i64 0
+; CHECK-NEXT:    [[TMP22:%.*]] = fneg <2 x double> [[X2]]
+; CHECK-NEXT:    [[_MSPROP8:%.*]] = extractelement <2 x i64> [[TMP2]], i64 0
+; CHECK-NEXT:    [[TMP23:%.*]] = extractelement <2 x double> [[X0]], i64 0
+; CHECK-NEXT:    [[_MSPROP9:%.*]] = extractelement <2 x i64> [[TMP3]], i64 0
+; CHECK-NEXT:    [[TMP24:%.*]] = extractelement <2 x double> [[X1]], i64 0
+; CHECK-NEXT:    [[_MSPROP10:%.*]] = extractelement <2 x i64> [[TMP1]], i64 0
+; CHECK-NEXT:    [[TMP25:%.*]] = extractelement <2 x double> [[TMP22]], i64 0
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[_MSPROP8]], 0
+; CHECK-NEXT:    [[_MSCMP22:%.*]] = icmp ne i64 [[_MSPROP9]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP22]]
+; CHECK-NEXT:    [[_MSCMP23:%.*]] = icmp ne i64 [[_MSPROP10]], 0
+; CHECK-NEXT:    [[_MSOR24:%.*]] = or i1 [[_MSOR]], [[_MSCMP23]]
+; CHECK-NEXT:    br i1 [[_MSOR24]], label [[TMP26:%.*]], label [[TMP27:%.*]], !prof [[PROF1]]
+; CHECK:       26:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       27:
+; CHECK-NEXT:    [[TMP28:%.*]] = call double @llvm.x86.avx512.vfmadd.f64(double [[TMP23]], double [[TMP24]], double [[TMP25]], i32 11)
+; CHECK-NEXT:    [[_MSPROP11:%.*]] = extractelement <2 x i64> [[TMP1]], i64 0
+; CHECK-NEXT:    [[TMP29:%.*]] = extractelement <2 x double> [[X2]], i64 0
+; CHECK-NEXT:    [[_MSPROP12:%.*]] = insertelement <2 x i64> [[TMP1]], i64 0, i64 0
+; CHECK-NEXT:    [[TMP30:%.*]] = insertelement <2 x double> [[X2]], double [[TMP28]], i64 0
+; CHECK-NEXT:    [[TMP31:%.*]] = fneg <2 x double> [[X2]]
+; CHECK-NEXT:    [[_MSPROP13:%.*]] = extractelement <2 x i64> [[TMP2]], i64 0
+; CHECK-NEXT:    [[TMP32:%.*]] = extractelement <2 x double> [[X0]], i64 0
+; CHECK-NEXT:    [[_MSPROP14:%.*]] = extractelement <2 x i64> [[TMP3]], i64 0
+; CHECK-NEXT:    [[TMP33:%.*]] = extractelement <2 x double> [[X1]], i64 0
+; CHECK-NEXT:    [[_MSPROP15:%.*]] = extractelement <2 x i64> [[TMP1]], i64 0
+; CHECK-NEXT:    [[TMP34:%.*]] = extractelement <2 x double> [[TMP31]], i64 0
+; CHECK-NEXT:    [[_MSCMP25:%.*]] = icmp ne i64 [[_MSPROP13]], 0
+; CHECK-NEXT:    [[_MSCMP26:%.*]] = icmp ne i64 [[_MSPROP14]], 0
+; CHECK-NEXT:    [[_MSOR27:%.*]] = or i1 [[_MSCMP25]], [[_MSCMP26]]
+; CHECK-NEXT:    [[_MSCMP28:%.*]] = icmp ne i64 [[_MSPROP15]], 0
+; CHECK-NEXT:    [[_MSOR29:%.*]] = or i1 [[_MSOR27]], [[_MSCMP28]]
+; CHECK-NEXT:    br i1 [[_MSOR29]], label [[TMP35:%.*]], label [[TMP36:%.*]], !prof [[PROF1]]
+; CHECK:       35:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       36:
+; CHECK-NEXT:    [[TMP37:%.*]] = call double @llvm.x86.avx512.vfmadd.f64(double [[TMP32]], double [[TMP33]], double [[TMP34]], i32 10)
+; CHECK-NEXT:    [[_MSPROP16:%.*]] = extractelement <2 x i64> [[TMP1]], i64 0
+; CHECK-NEXT:    [[TMP38:%.*]] = extractelement <2 x double> [[X2]], i64 0
+; CHECK-NEXT:    [[TMP39:%.*]] = bitcast i8 [[TMP4]] to <8 x i1>
+; CHECK-NEXT:    [[TMP40:%.*]] = bitcast i8 [[X3]] to <8 x i1>
+; CHECK-NEXT:    [[_MSPROP17:%.*]] = extractelement <8 x i1> [[TMP39]], i64 0
+; CHECK-NEXT:    [[TMP41:%.*]] = extractelement <8 x i1> [[TMP40]], i64 0
+; CHECK-NEXT:    [[TMP42:%.*]] = select i1 [[TMP41]], i64 0, i64 [[_MSPROP16]]
+; CHECK-NEXT:    [[TMP43:%.*]] = bitcast double [[TMP37]] to i64
+; CHECK-NEXT:    [[TMP44:%.*]] = bitcast double [[TMP38]] to i64
+; CHECK-NEXT:    [[TMP45:%.*]] = xor i64 [[TMP43]], [[TMP44]]
+; CHECK-NEXT:    [[TMP46:%.*]] = or i64 [[TMP45]], 0
+; CHECK-NEXT:    [[TMP47:%.*]] = or i64 [[TMP46]], [[_MSPROP16]]
+; CHECK-NEXT:    [[_MSPROP_SELECT18:%.*]] = select i1 [[_MSPROP17]], i64 [[TMP47]], i64 [[TMP42]]
+; CHECK-NEXT:    [[TMP48:%.*]] = select i1 [[TMP41]], double [[TMP37]], double [[TMP38]]
+; CHECK-NEXT:    [[_MSPROP19:%.*]] = insertelement <2 x i64> [[TMP1]], i64 [[_MSPROP_SELECT18]], i64 0
+; CHECK-NEXT:    [[TMP49:%.*]] = insertelement <2 x double> [[X2]], double [[TMP48]], i64 0
+; CHECK-NEXT:    [[_MSPROP20:%.*]] = or <2 x i64> [[_MSPROP7]], [[_MSPROP12]]
+; CHECK-NEXT:    [[RES3:%.*]] = fadd <2 x double> [[TMP21]], [[TMP30]]
+; CHECK-NEXT:    [[_MSPROP21:%.*]] = or <2 x i64> [[_MSPROP19]], [[_MSPROP20]]
+; CHECK-NEXT:    [[RES4:%.*]] = fadd <2 x double> [[TMP49]], [[RES3]]
+; CHECK-NEXT:    store <2 x i64> [[_MSPROP21]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x double> [[RES4]]
+;
+  %1 = fneg <2 x double> %x2
+  %2 = extractelement <2 x double> %x0, i64 0
+  %3 = extractelement <2 x double> %x1, i64 0
+  %4 = extractelement <2 x double> %1, i64 0
+  %5 = call double @llvm.fma.f64(double %2, double %3, double %4)
+  %6 = extractelement <2 x double> %x2, i64 0
+  %7 = bitcast i8 %x3 to <8 x i1>
+  %8 = extractelement <8 x i1> %7, i64 0
+  %9 = select i1 %8, double %5, double %6
+  %10 = insertelement <2 x double> %x2, double %9, i64 0
+  %11 = fneg <2 x double> %x2
+  %12 = extractelement <2 x double> %x0, i64 0
+  %13 = extractelement <2 x double> %x1, i64 0
+  %14 = extractelement <2 x double> %11, i64 0
+  %15 = call double @llvm.x86.avx512.vfmadd.f64(double %12, double %13, double %14, i32 11)
+  %16 = extractelement <2 x double> %x2, i64 0
+  %17 = insertelement <2 x double> %x2, double %15, i64 0
+  %18 = fneg <2 x double> %x2
+  %19 = extractelement <2 x double> %x0, i64 0
+  %20 = extractelement <2 x double> %x1, i64 0
+  %21 = extractelement <2 x double> %18, i64 0
+  %22 = call double @llvm.x86.avx512.vfmadd.f64(double %19, double %20, double %21, i32 10)
+  %23 = extractelement <2 x double> %x2, i64 0
+  %24 = bitcast i8 %x3 to <8 x i1>
+  %25 = extractelement <8 x i1> %24, i64 0
+  %26 = select i1 %25, double %22, double %23
+  %27 = insertelement <2 x double> %x2, double %26, i64 0
+  %res3 = fadd <2 x double> %10, %17
+  %res4 = fadd <2 x double> %27, %res3
+  ret <2 x double> %res4
+}
+
+define <4 x float> @test_int_x86_avx512_mask3_vfmsub_ss(<4 x float> %x0, <4 x float> %x1, <4 x float> %x2, i8 %x3, i32 %x4) #0 {
+; CHECK-LABEL: @test_int_x86_avx512_mask3_vfmsub_ss(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 48) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = fneg <4 x float> [[X2:%.*]]
+; CHECK-NEXT:    [[_MSPROP:%.*]] = extractelement <4 x i32> [[TMP2]], i64 0
+; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <4 x float> [[X0:%.*]], i64 0
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = extractelement <4 x i32> [[TMP3]], i64 0
+; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <4 x float> [[X1:%.*]], i64 0
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = extractelement <4 x i32> [[TMP1]], i64 0
+; CHECK-NEXT:    [[TMP8:%.*]] = extractelement <4 x float> [[TMP5]], i64 0
+; CHECK-NEXT:    [[_MSPROP3:%.*]] = or i32 [[_MSPROP]], [[_MSPROP1]]
+; CHECK-NEXT:    [[_MSPROP4:%.*]] = or i32 [[_MSPROP3]], [[_MSPROP2]]
+; CHECK-NEXT:    [[TMP9:%.*]] = call float @llvm.fma.f32(float [[TMP6]], float [[TMP7]], float [[TMP8]])
+; CHECK-NEXT:    [[_MSPROP5:%.*]] = extractelement <4 x i32> [[TMP1]], i64 0
+; CHECK-NEXT:    [[TMP10:%.*]] = extractelement <4 x float> [[X2]], i64 0
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast i8 [[TMP4]] to <8 x i1>
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast i8 [[X3:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[_MSPROP6:%.*]] = extractelement <8 x i1> [[TMP11]], i64 0
+; CHECK-NEXT:    [[TMP13:%.*]] = extractelement <8 x i1> [[TMP12]], i64 0
+; CHECK-NEXT:    [[TMP14:%.*]] = select i1 [[TMP13]], i32 [[_MSPROP4]], i32 [[_MSPROP5]]
+; CHECK-NEXT:    [[TMP15:%.*]] = bitcast float [[TMP9]] to i32
+; CHECK-NEXT:    [[TMP16:%.*]] = bitcast float [[TMP10]] to i32
+; CHECK-NEXT:    [[TMP17:%.*]] = xor i32 [[TMP15]], [[TMP16]]
+; CHECK-NEXT:    [[TMP18:%.*]] = or i32 [[TMP17]], [[_MSPROP4]]
+; CHECK-NEXT:    [[TMP19:%.*]] = or i32 [[TMP18]], [[_MSPROP5]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select i1 [[_MSPROP6]], i32 [[TMP19]], i32 [[TMP14]]
+; CHECK-NEXT:    [[TMP20:%.*]] = select i1 [[TMP13]], float [[TMP9]], float [[TMP10]]
+; CHECK-NEXT:    [[_MSPROP7:%.*]] = insertelement <4 x i32> [[TMP1]], i32 [[_MSPROP_SELECT]], i64 0
+; CHECK-NEXT:    [[TMP21:%.*]] = insertelement <4 x float> [[X2]], float [[TMP20]], i64 0
+; CHECK-NEXT:    [[TMP22:%.*]] = fneg <4 x float> [[X2]]
+; CHECK-NEXT:    [[_MSPROP8:%.*]] = extractelement <4 x i32> [[TMP2]], i64 0
+; CHECK-NEXT:    [[TMP23:%.*]] = extractelement <4 x float> [[X0]], i64 0
+; CHECK-NEXT:    [[_MSPROP9:%.*]] = extractelement <4 x i32> [[TMP3]], i64 0
+; CHECK-NEXT:    [[TMP24:%.*]] = extractelement <4 x float> [[X1]], i64 0
+; CHECK-NEXT:    [[_MSPROP10:%.*]] = extractelement <4 x i32> [[TMP1]], i64 0
+; CHECK-NEXT:    [[TMP25:%.*]] = extractelement <4 x float> [[TMP22]], i64 0
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i32 [[_MSPROP8]], 0
+; CHECK-NEXT:    [[_MSCMP22:%.*]] = icmp ne i32 [[_MSPROP9]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP22]]
+; CHECK-NEXT:    [[_MSCMP23:%.*]] = icmp ne i32 [[_MSPROP10]], 0
+; CHECK-NEXT:    [[_MSOR24:%.*]] = or i1 [[_MSOR]], [[_MSCMP23]]
+; CHECK-NEXT:    br i1 [[_MSOR24]], label [[TMP26:%.*]], label [[TMP27:%.*]], !prof [[PROF1]]
+; CHECK:       26:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       27:
+; CHECK-NEXT:    [[TMP28:%.*]] = call float @llvm.x86.avx512.vfmadd.f32(float [[TMP23]], float [[TMP24]], float [[TMP25]], i32 11)
+; CHECK-NEXT:    [[_MSPROP11:%.*]] = extractelement <4 x i32> [[TMP1]], i64 0
+; CHECK-NEXT:    [[TMP29:%.*]] = extractelement <4 x float> [[X2]], i64 0
+; CHECK-NEXT:    [[_MSPROP12:%.*]] = insertelement <4 x i32> [[TMP1]], i32 0, i64 0
+; CHECK-NEXT:    [[TMP30:%.*]] = insertelement <4 x float> [[X2]], float [[TMP28]], i64 0
+; CHECK-NEXT:    [[TMP31:%.*]] = fneg <4 x float> [[X2]]
+; CHECK-NEXT:    [[_MSPROP13:%.*]] = extractelement <4 x i32> [[TMP2]], i64 0
+; CHECK-NEXT:    [[TMP32:%.*]] = extractelement <4 x float> [[X0]], i64 0
+; CHECK-NEXT:    [[_MSPROP14:%.*]] = extractelement <4 x i32> [[TMP3]], i64 0
+; CHECK-NEXT:    [[TMP33:%.*]] = extractelement <4 x float> [[X1]], i64 0
+; CHECK-NEXT:    [[_MSPROP15:%.*]] = extractelement <4 x i32> [[TMP1]], i64 0
+; CHECK-NEXT:    [[TMP34:%.*]] = extractelement <4 x float> [[TMP31]], i64 0
+; CHECK-NEXT:    [[_MSCMP25:%.*]] = icmp ne i32 [[_MSPROP13]], 0
+; CHECK-NEXT:    [[_MSCMP26:%.*]] = icmp ne i32 [[_MSPROP14]], 0
+; CHECK-NEXT:    [[_MSOR27:%.*]] = or i1 [[_MSCMP25]], [[_MSCMP26]]
+; CHECK-NEXT:    [[_MSCMP28:%.*]] = icmp ne i32 [[_MSPROP15]], 0
+; CHECK-NEXT:    [[_MSOR29:%.*]] = or i1 [[_MSOR27]], [[_MSCMP28]]
+; CHECK-NEXT:    br i1 [[_MSOR29]], label [[TMP35:%.*]], label [[TMP36:%.*]], !prof [[PROF1]]
+; CHECK:       35:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       36:
+; CHECK-NEXT:    [[TMP37:%.*]] = call float @llvm.x86.avx512.vfmadd.f32(float [[TMP32]], float [[TMP33]], float [[TMP34]], i32 10)
+; CHECK-NEXT:    [[_MSPROP16:%.*]] = extractelement <4 x i32> [[TMP1]], i64 0
+; CHECK-NEXT:    [[TMP38:%.*]] = extractelement <4 x float> [[X2]], i64 0
+; CHECK-NEXT:    [[TMP39:%.*]] = bitcast i8 [[TMP4]] to <8 x i1>
+; CHECK-NEXT:    [[TMP40:%.*]] = bitcast i8 [[X3]] to <8 x i1>
+; CHECK-NEXT:    [[_MSPROP17:%.*]] = extractelement <8 x i1> [[TMP39]], i64 0
+; CHECK-NEXT:    [[TMP41:%.*]] = extractelement <8 x i1> [[TMP40]], i64 0
+; CHECK-NEXT:    [[TMP42:%.*]] = select i1 [[TMP41]], i32 0, i32 [[_MSPROP16]]
+; CHECK-NEXT:    [[TMP43:%.*]] = bitcast float [[TMP37]] to i32
+; CHECK-NEXT:    [[TMP44:%.*]] = bitcast float [[TMP38]] to i32
+; CHECK-NEXT:    [[TMP45:%.*]] = xor i32 [[TMP43]], [[TMP44]]
+; CHECK-NEXT:    [[TMP46:%.*]] = or i32 [[TMP45]], 0
+; CHECK-NEXT:    [[TMP47:%.*]] = or i32 [[TMP46]], [[_MSPROP16]]
+; CHECK-NEXT:    [[_MSPROP_SELECT18:%.*]] = select i1 [[_MSPROP17]], i32 [[TMP47]], i32 [[TMP42]]
+; CHECK-NEXT:    [[TMP48:%.*]] = select i1 [[TMP41]], float [[TMP37]], float [[TMP38]]
+; CHECK-NEXT:    [[_MSPROP19:%.*]] = insertelement <4 x i32> [[TMP1]], i32 [[_MSPROP_SELECT18]], i64 0
+; CHECK-NEXT:    [[TMP49:%.*]] = insertelement <4 x float> [[X2]], float [[TMP48]], i64 0
+; CHECK-NEXT:    [[_MSPROP20:%.*]] = or <4 x i32> [[_MSPROP7]], [[_MSPROP12]]
+; CHECK-NEXT:    [[RES3:%.*]] = fadd <4 x float> [[TMP21]], [[TMP30]]
+; CHECK-NEXT:    [[_MSPROP21:%.*]] = or <4 x i32> [[_MSPROP19]], [[_MSPROP20]]
+; CHECK-NEXT:    [[RES4:%.*]] = fadd <4 x float> [[TMP49]], [[RES3]]
+; CHECK-NEXT:    store <4 x i32> [[_MSPROP21]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x float> [[RES4]]
+;
+  %1 = fneg <4 x float> %x2
+  %2 = extractelement <4 x float> %x0, i64 0
+  %3 = extractelement <4 x float> %x1, i64 0
+  %4 = extractelement <4 x float> %1, i64 0
+  %5 = call float @llvm.fma.f32(float %2, float %3, float %4)
+  %6 = extractelement <4 x float> %x2, i64 0
+  %7 = bitcast i8 %x3 to <8 x i1>
+  %8 = extractelement <8 x i1> %7, i64 0
+  %9 = select i1 %8, float %5, float %6
+  %10 = insertelement <4 x float> %x2, float %9, i64 0
+  %11 = fneg <4 x float> %x2
+  %12 = extractelement <4 x float> %x0, i64 0
+  %13 = extractelement <4 x float> %x1, i64 0
+  %14 = extractelement <4 x float> %11, i64 0
+  %15 = call float @llvm.x86.avx512.vfmadd.f32(float %12, float %13, float %14, i32 11)
+  %16 = extractelement <4 x float> %x2, i64 0
+  %17 = insertelement <4 x float> %x2, float %15, i64 0
+  %18 = fneg <4 x float> %x2
+  %19 = extractelement <4 x float> %x0, i64 0
+  %20 = extractelement <4 x float> %x1, i64 0
+  %21 = extractelement <4 x float> %18, i64 0
+  %22 = call float @llvm.x86.avx512.vfmadd.f32(float %19, float %20, float %21, i32 10)
+  %23 = extractelement <4 x float> %x2, i64 0
+  %24 = bitcast i8 %x3 to <8 x i1>
+  %25 = extractelement <8 x i1> %24, i64 0
+  %26 = select i1 %25, float %22, float %23
+  %27 = insertelement <4 x float> %x2, float %26, i64 0
+  %res3 = fadd <4 x float> %10, %17
+  %res4 = fadd <4 x float> %27, %res3
+  ret <4 x float> %res4
+}
+
+define <2 x double> @test_int_x86_avx512_mask3_vfnmsub_sd(<2 x double> %x0, <2 x double> %x1, <2 x double> %x2, i8 %x3, i32 %x4) #0 {
+; CHECK-LABEL: @test_int_x86_avx512_mask3_vfnmsub_sd(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 48) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = fneg <2 x double> [[X0:%.*]]
+; CHECK-NEXT:    [[TMP6:%.*]] = fneg <2 x double> [[X2:%.*]]
+; CHECK-NEXT:    [[_MSPROP:%.*]] = extractelement <2 x i64> [[TMP1]], i64 0
+; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <2 x double> [[TMP5]], i64 0
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = extractelement <2 x i64> [[TMP3]], i64 0
+; CHECK-NEXT:    [[TMP8:%.*]] = extractelement <2 x double> [[X1:%.*]], i64 0
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = extractelement <2 x i64> [[TMP2]], i64 0
+; CHECK-NEXT:    [[TMP9:%.*]] = extractelement <2 x double> [[TMP6]], i64 0
+; CHECK-NEXT:    [[_MSPROP3:%.*]] = or i64 [[_MSPROP]], [[_MSPROP1]]
+; CHECK-NEXT:    [[_MSPROP4:%.*]] = or i64 [[_MSPROP3]], [[_MSPROP2]]
+; CHECK-NEXT:    [[TMP10:%.*]] = call double @llvm.fma.f64(double [[TMP7]], double [[TMP8]], double [[TMP9]])
+; CHECK-NEXT:    [[_MSPROP5:%.*]] = extractelement <2 x i64> [[TMP2]], i64 0
+; CHECK-NEXT:    [[TMP11:%.*]] = extractelement <2 x double> [[X2]], i64 0
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast i8 [[TMP4]] to <8 x i1>
+; CHECK-NEXT:    [[TMP13:%.*]] = bitcast i8 [[X3:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[_MSPROP6:%.*]] = extractelement <8 x i1> [[TMP12]], i64 0
+; CHECK-NEXT:    [[TMP14:%.*]] = extractelement <8 x i1> [[TMP13]], i64 0
+; CHECK-NEXT:    [[TMP15:%.*]] = select i1 [[TMP14]], i64 [[_MSPROP4]], i64 [[_MSPROP5]]
+; CHECK-NEXT:    [[TMP16:%.*]] = bitcast double [[TMP10]] to i64
+; CHECK-NEXT:    [[TMP17:%.*]] = bitcast double [[TMP11]] to i64
+; CHECK-NEXT:    [[TMP18:%.*]] = xor i64 [[TMP16]], [[TMP17]]
+; CHECK-NEXT:    [[TMP19:%.*]] = or i64 [[TMP18]], [[_MSPROP4]]
+; CHECK-NEXT:    [[TMP20:%.*]] = or i64 [[TMP19]], [[_MSPROP5]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select i1 [[_MSPROP6]], i64 [[TMP20]], i64 [[TMP15]]
+; CHECK-NEXT:    [[TMP21:%.*]] = select i1 [[TMP14]], double [[TMP10]], double [[TMP11]]
+; CHECK-NEXT:    [[_MSPROP7:%.*]] = insertelement <2 x i64> [[TMP2]], i64 [[_MSPROP_SELECT]], i64 0
+; CHECK-NEXT:    [[TMP22:%.*]] = insertelement <2 x double> [[X2]], double [[TMP21]], i64 0
+; CHECK-NEXT:    [[TMP23:%.*]] = fneg <2 x double> [[X0]]
+; CHECK-NEXT:    [[TMP24:%.*]] = fneg <2 x double> [[X2]]
+; CHECK-NEXT:    [[_MSPROP8:%.*]] = extractelement <2 x i64> [[TMP1]], i64 0
+; CHECK-NEXT:    [[TMP25:%.*]] = extractelement <2 x double> [[TMP23]], i64 0
+; CHECK-NEXT:    [[_MSPROP9:%.*]] = extractelement <2 x i64> [[TMP3]], i64 0
+; CHECK-NEXT:    [[TMP26:%.*]] = extractelement <2 x double> [[X1]], i64 0
+; CHECK-NEXT:    [[_MSPROP10:%.*]] = extractelement <2 x i64> [[TMP2]], i64 0
+; CHECK-NEXT:    [[TMP27:%.*]] = extractelement <2 x double> [[TMP24]], i64 0
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[_MSPROP8]], 0
+; CHECK-NEXT:    [[_MSCMP22:%.*]] = icmp ne i64 [[_MSPROP9]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP22]]
+; CHECK-NEXT:    [[_MSCMP23:%.*]] = icmp ne i64 [[_MSPROP10]], 0
+; CHECK-NEXT:    [[_MSOR24:%.*]] = or i1 [[_MSOR]], [[_MSCMP23]]
+; CHECK-NEXT:    br i1 [[_MSOR24]], label [[TMP28:%.*]], label [[TMP29:%.*]], !prof [[PROF1]]
+; CHECK:       28:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       29:
+; CHECK-NEXT:    [[TMP30:%.*]] = call double @llvm.x86.avx512.vfmadd.f64(double [[TMP25]], double [[TMP26]], double [[TMP27]], i32 11)
+; CHECK-NEXT:    [[_MSPROP11:%.*]] = extractelement <2 x i64> [[TMP2]], i64 0
+; CHECK-NEXT:    [[TMP31:%.*]] = extractelement <2 x double> [[X2]], i64 0
+; CHECK-NEXT:    [[_MSPROP12:%.*]] = insertelement <2 x i64> [[TMP2]], i64 0, i64 0
+; CHECK-NEXT:    [[TMP32:%.*]] = insertelement <2 x double> [[X2]], double [[TMP30]], i64 0
+; CHECK-NEXT:    [[TMP33:%.*]] = fneg <2 x double> [[X0]]
+; CHECK-NEXT:    [[TMP34:%.*]] = fneg <2 x double> [[X2]]
+; CHECK-NEXT:    [[_MSPROP13:%.*]] = extractelement <2 x i64> [[TMP1]], i64 0
+; CHECK-NEXT:    [[TMP35:%.*]] = extractelement <2 x double> [[TMP33]], i64 0
+; CHECK-NEXT:    [[_MSPROP14:%.*]] = extractelement <2 x i64> [[TMP3]], i64 0
+; CHECK-NEXT:    [[TMP36:%.*]] = extractelement <2 x double> [[X1]], i64 0
+; CHECK-NEXT:    [[_MSPROP15:%.*]] = extractelement <2 x i64> [[TMP2]], i64 0
+; CHECK-NEXT:    [[TMP37:%.*]] = extractelement <2 x double> [[TMP34]], i64 0
+; CHECK-NEXT:    [[_MSCMP25:%.*]] = icmp ne i64 [[_MSPROP13]], 0
+; CHECK-NEXT:    [[_MSCMP26:%.*]] = icmp ne i64 [[_MSPROP14]], 0
+; CHECK-NEXT:    [[_MSOR27:%.*]] = or i1 [[_MSCMP25]], [[_MSCMP26]]
+; CHECK-NEXT:    [[_MSCMP28:%.*]] = icmp ne i64 [[_MSPROP15]], 0
+; CHECK-NEXT:    [[_MSOR29:%.*]] = or i1 [[_MSOR27]], [[_MSCMP28]]
+; CHECK-NEXT:    br i1 [[_MSOR29]], label [[TMP38:%.*]], label [[TMP39:%.*]], !prof [[PROF1]]
+; CHECK:       38:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       39:
+; CHECK-NEXT:    [[TMP40:%.*]] = call double @llvm.x86.avx512.vfmadd.f64(double [[TMP35]], double [[TMP36]], double [[TMP37]], i32 10)
+; CHECK-NEXT:    [[_MSPROP16:%.*]] = extractelement <2 x i64> [[TMP2]], i64 0
+; CHECK-NEXT:    [[TMP41:%.*]] = extractelement <2 x double> [[X2]], i64 0
+; CHECK-NEXT:    [[TMP42:%.*]] = bitcast i8 [[TMP4]] to <8 x i1>
+; CHECK-NEXT:    [[TMP43:%.*]] = bitcast i8 [[X3]] to <8 x i1>
+; CHECK-NEXT:    [[_MSPROP17:%.*]] = extractelement <8 x i1> [[TMP42]], i64 0
+; CHECK-NEXT:    [[TMP44:%.*]] = extractelement <8 x i1> [[TMP43]], i64 0
+; CHECK-NEXT:    [[TMP45:%.*]] = select i1 [[TMP44]], i64 0, i64 [[_MSPROP16]]
+; CHECK-NEXT:    [[TMP46:%.*]] = bitcast double [[TMP40]] to i64
+; CHECK-NEXT:    [[TMP47:%.*]] = bitcast double [[TMP41]] to i64
+; CHECK-NEXT:    [[TMP48:%.*]] = xor i64 [[TMP46]], [[TMP47]]
+; CHECK-NEXT:    [[TMP49:%.*]] = or i64 [[TMP48]], 0
+; CHECK-NEXT:    [[TMP50:%.*]] = or i64 [[TMP49]], [[_MSPROP16]]
+; CHECK-NEXT:    [[_MSPROP_SELECT18:%.*]] = select i1 [[_MSPROP17]], i64 [[TMP50]], i64 [[TMP45]]
+; CHECK-NEXT:    [[TMP51:%.*]] = select i1 [[TMP44]], double [[TMP40]], double [[TMP41]]
+; CHECK-NEXT:    [[_MSPROP19:%.*]] = insertelement <2 x i64> [[TMP2]], i64 [[_MSPROP_SELECT18]], i64 0
+; CHECK-NEXT:    [[TMP52:%.*]] = insertelement <2 x double> [[X2]], double [[TMP51]], i64 0
+; CHECK-NEXT:    [[_MSPROP20:%.*]] = or <2 x i64> [[_MSPROP7]], [[_MSPROP12]]
+; CHECK-NEXT:    [[RES3:%.*]] = fadd <2 x double> [[TMP22]], [[TMP32]]
+; CHECK-NEXT:    [[_MSPROP21:%.*]] = or <2 x i64> [[_MSPROP19]], [[_MSPROP20]]
+; CHECK-NEXT:    [[RES4:%.*]] = fadd <2 x double> [[TMP52]], [[RES3]]
+; CHECK-NEXT:    store <2 x i64> [[_MSPROP21]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x double> [[RES4]]
+;
+  %1 = fneg <2 x double> %x0
+  %2 = fneg <2 x double> %x2
+  %3 = extractelement <2 x double> %1, i64 0
+  %4 = extractelement <2 x double> %x1, i64 0
+  %5 = extractelement <2 x double> %2, i64 0
+  %6 = call double @llvm.fma.f64(double %3, double %4, double %5)
+  %7 = extractelement <2 x double> %x2, i64 0
+  %8 = bitcast i8 %x3 to <8 x i1>
+  %9 = extractelement <8 x i1> %8, i64 0
+  %10 = select i1 %9, double %6, double %7
+  %11 = insertelement <2 x double> %x2, double %10, i64 0
+  %12 = fneg <2 x double> %x0
+  %13 = fneg <2 x double> %x2
+  %14 = extractelement <2 x double> %12, i64 0
+  %15 = extractelement <2 x double> %x1, i64 0
+  %16 = extractelement <2 x double> %13, i64 0
+  %17 = call double @llvm.x86.avx512.vfmadd.f64(double %14, double %15, double %16, i32 11)
+  %18 = extractelement <2 x double> %x2, i64 0
+  %19 = insertelement <2 x double> %x2, double %17, i64 0
+  %20 = fneg <2 x double> %x0
+  %21 = fneg <2 x double> %x2
+  %22 = extractelement <2 x double> %20, i64 0
+  %23 = extractelement <2 x double> %x1, i64 0
+  %24 = extractelement <2 x double> %21, i64 0
+  %25 = call double @llvm.x86.avx512.vfmadd.f64(double %22, double %23, double %24, i32 10)
+  %26 = extractelement <2 x double> %x2, i64 0
+  %27 = bitcast i8 %x3 to <8 x i1>
+  %28 = extractelement <8 x i1> %27, i64 0
+  %29 = select i1 %28, double %25, double %26
+  %30 = insertelement <2 x double> %x2, double %29, i64 0
+  %res3 = fadd <2 x double> %11, %19
+  %res4 = fadd <2 x double> %30, %res3
+  ret <2 x double> %res4
+}
+
+define <4 x float> @test_int_x86_avx512_mask3_vfnmsub_ss(<4 x float> %x0, <4 x float> %x1, <4 x float> %x2, i8 %x3, i32 %x4) #0 {
+; CHECK-LABEL: @test_int_x86_avx512_mask3_vfnmsub_ss(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 48) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = fneg <4 x float> [[X0:%.*]]
+; CHECK-NEXT:    [[TMP6:%.*]] = fneg <4 x float> [[X2:%.*]]
+; CHECK-NEXT:    [[_MSPROP:%.*]] = extractelement <4 x i32> [[TMP1]], i64 0
+; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <4 x float> [[TMP5]], i64 0
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = extractelement <4 x i32> [[TMP3]], i64 0
+; CHECK-NEXT:    [[TMP8:%.*]] = extractelement <4 x float> [[X1:%.*]], i64 0
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = extractelement <4 x i32> [[TMP2]], i64 0
+; CHECK-NEXT:    [[TMP9:%.*]] = extractelement <4 x float> [[TMP6]], i64 0
+; CHECK-NEXT:    [[_MSPROP3:%.*]] = or i32 [[_MSPROP]], [[_MSPROP1]]
+; CHECK-NEXT:    [[_MSPROP4:%.*]] = or i32 [[_MSPROP3]], [[_MSPROP2]]
+; CHECK-NEXT:    [[TMP10:%.*]] = call float @llvm.fma.f32(float [[TMP7]], float [[TMP8]], float [[TMP9]])
+; CHECK-NEXT:    [[_MSPROP5:%.*]] = extractelement <4 x i32> [[TMP2]], i64 0
+; CHECK-NEXT:    [[TMP11:%.*]] = extractelement <4 x float> [[X2]], i64 0
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast i8 [[TMP4]] to <8 x i1>
+; CHECK-NEXT:    [[TMP13:%.*]] = bitcast i8 [[X3:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[_MSPROP6:%.*]] = extractelement <8 x i1> [[TMP12]], i64 0
+; CHECK-NEXT:    [[TMP14:%.*]] = extractelement <8 x i1> [[TMP13]], i64 0
+; CHECK-NEXT:    [[TMP15:%.*]] = select i1 [[TMP14]], i32 [[_MSPROP4]], i32 [[_MSPROP5]]
+; CHECK-NEXT:    [[TMP16:%.*]] = bitcast float [[TMP10]] to i32
+; CHECK-NEXT:    [[TMP17:%.*]] = bitcast float [[TMP11]] to i32
+; CHECK-NEXT:    [[TMP18:%.*]] = xor i32 [[TMP16]], [[TMP17]]
+; CHECK-NEXT:    [[TMP19:%.*]] = or i32 [[TMP18]], [[_MSPROP4]]
+; CHECK-NEXT:    [[TMP20:%.*]] = or i32 [[TMP19]], [[_MSPROP5]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select i1 [[_MSPROP6]], i32 [[TMP20]], i32 [[TMP15]]
+; CHECK-NEXT:    [[TMP21:%.*]] = select i1 [[TMP14]], float [[TMP10]], float [[TMP11]]
+; CHECK-NEXT:    [[_MSPROP7:%.*]] = insertelement <4 x i32> [[TMP2]], i32 [[_MSPROP_SELECT]], i64 0
+; CHECK-NEXT:    [[TMP22:%.*]] = insertelement <4 x float> [[X2]], float [[TMP21]], i64 0
+; CHECK-NEXT:    [[TMP23:%.*]] = fneg <4 x float> [[X0]]
+; CHECK-NEXT:    [[TMP24:%.*]] = fneg <4 x float> [[X2]]
+; CHECK-NEXT:    [[_MSPROP8:%.*]] = extractelement <4 x i32> [[TMP1]], i64 0
+; CHECK-NEXT:    [[TMP25:%.*]] = extractelement <4 x float> [[TMP23]], i64 0
+; CHECK-NEXT:    [[_MSPROP9:%.*]] = extractelement <4 x i32> [[TMP3]], i64 0
+; CHECK-NEXT:    [[TMP26:%.*]] = extractelement <4 x float> [[X1]], i64 0
+; CHECK-NEXT:    [[_MSPROP10:%.*]] = extractelement <4 x i32> [[TMP2]], i64 0
+; CHECK-NEXT:    [[TMP27:%.*]] = extractelement <4 x float> [[TMP24]], i64 0
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i32 [[_MSPROP8]], 0
+; CHECK-NEXT:    [[_MSCMP22:%.*]] = icmp ne i32 [[_MSPROP9]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP22]]
+; CHECK-NEXT:    [[_MSCMP23:%.*]] = icmp ne i32 [[_MSPROP10]], 0
+; CHECK-NEXT:    [[_MSOR24:%.*]] = or i1 [[_MSOR]], [[_MSCMP23]]
+; CHECK-NEXT:    br i1 [[_MSOR24]], label [[TMP28:%.*]], label [[TMP29:%.*]], !prof [[PROF1]]
+; CHECK:       28:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       29:
+; CHECK-NEXT:    [[TMP30:%.*]] = call float @llvm.x86.avx512.vfmadd.f32(float [[TMP25]], float [[TMP26]], float [[TMP27]], i32 11)
+; CHECK-NEXT:    [[_MSPROP11:%.*]] = extractelement <4 x i32> [[TMP2]], i64 0
+; CHECK-NEXT:    [[TMP31:%.*]] = extractelement <4 x float> [[X2]], i64 0
+; CHECK-NEXT:    [[_MSPROP12:%.*]] = insertelement <4 x i32> [[TMP2]], i32 0, i64 0
+; CHECK-NEXT:    [[TMP32:%.*]] = insertelement <4 x float> [[X2]], float [[TMP30]], i64 0
+; CHECK-NEXT:    [[TMP33:%.*]] = fneg <4 x float> [[X0]]
+; CHECK-NEXT:    [[TMP34:%.*]] = fneg <4 x float> [[X2]]
+; CHECK-NEXT:    [[_MSPROP13:%.*]] = extractelement <4 x i32> [[TMP1]], i64 0
+; CHECK-NEXT:    [[TMP35:%.*]] = extractelement <4 x float> [[TMP33]], i64 0
+; CHECK-NEXT:    [[_MSPROP14:%.*]] = extractelement <4 x i32> [[TMP3]], i64 0
+; CHECK-NEXT:    [[TMP36:%.*]] = extractelement <4 x float> [[X1]], i64 0
+; CHECK-NEXT:    [[_MSPROP15:%.*]] = extractelement <4 x i32> [[TMP2]], i64 0
+; CHECK-NEXT:    [[TMP37:%.*]] = extractelement <4 x float> [[TMP34]], i64 0
+; CHECK-NEXT:    [[_MSCMP25:%.*]] = icmp ne i32 [[_MSPROP13]], 0
+; CHECK-NEXT:    [[_MSCMP26:%.*]] = icmp ne i32 [[_MSPROP14]], 0
+; CHECK-NEXT:    [[_MSOR27:%.*]] = or i1 [[_MSCMP25]], [[_MSCMP26]]
+; CHECK-NEXT:    [[_MSCMP28:%.*]] = icmp ne i32 [[_MSPROP15]], 0
+; CHECK-NEXT:    [[_MSOR29:%.*]] = or i1 [[_MSOR27]], [[_MSCMP28]]
+; CHECK-NEXT:    br i1 [[_MSOR29]], label [[TMP38:%.*]], label [[TMP39:%.*]], !prof [[PROF1]]
+; CHECK:       38:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       39:
+; CHECK-NEXT:    [[TMP40:%.*]] = call float @llvm.x86.avx512.vfmadd.f32(float [[TMP35]], float [[TMP36]], float [[TMP37]], i32 10)
+; CHECK-NEXT:    [[_MSPROP16:%.*]] = extractelement <4 x i32> [[TMP2]], i64 0
+; CHECK-NEXT:    [[TMP41:%.*]] = extractelement <4 x float> [[X2]], i64 0
+; CHECK-NEXT:    [[TMP42:%.*]] = bitcast i8 [[TMP4]] to <8 x i1>
+; CHECK-NEXT:    [[TMP43:%.*]] = bitcast i8 [[X3]] to <8 x i1>
+; CHECK-NEXT:    [[_MSPROP17:%.*]] = extractelement <8 x i1> [[TMP42]], i64 0
+; CHECK-NEXT:    [[TMP44:%.*]] = extractelement <8 x i1> [[TMP43]], i64 0
+; CHECK-NEXT:    [[TMP45:%.*]] = select i1 [[TMP44]], i32 0, i32 [[_MSPROP16]]
+; CHECK-NEXT:    [[TMP46:%.*]] = bitcast float [[TMP40]] to i32
+; CHECK-NEXT:    [[TMP47:%.*]] = bitcast float [[TMP41]] to i32
+; CHECK-NEXT:    [[TMP48:%.*]] = xor i32 [[TMP46]], [[TMP47]]
+; CHECK-NEXT:    [[TMP49:%.*]] = or i32 [[TMP48]], 0
+; CHECK-NEXT:    [[TMP50:%.*]] = or i32 [[TMP49]], [[_MSPROP16]]
+; CHECK-NEXT:    [[_MSPROP_SELECT18:%.*]] = select i1 [[_MSPROP17]], i32 [[TMP50]], i32 [[TMP45]]
+; CHECK-NEXT:    [[TMP51:%.*]] = select i1 [[TMP44]], float [[TMP40]], float [[TMP41]]
+; CHECK-NEXT:    [[_MSPROP19:%.*]] = insertelement <4 x i32> [[TMP2]], i32 [[_MSPROP_SELECT18]], i64 0
+; CHECK-NEXT:    [[TMP52:%.*]] = insertelement <4 x float> [[X2]], float [[TMP51]], i64 0
+; CHECK-NEXT:    [[_MSPROP20:%.*]] = or <4 x i32> [[_MSPROP7]], [[_MSPROP12]]
+; CHECK-NEXT:    [[RES3:%.*]] = fadd <4 x float> [[TMP22]], [[TMP32]]
+; CHECK-NEXT:    [[_MSPROP21:%.*]] = or <4 x i32> [[_MSPROP19]], [[_MSPROP20]]
+; CHECK-NEXT:    [[RES4:%.*]] = fadd <4 x float> [[TMP52]], [[RES3]]
+; CHECK-NEXT:    store <4 x i32> [[_MSPROP21]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x float> [[RES4]]
+;
+  %1 = fneg <4 x float> %x0
+  %2 = fneg <4 x float> %x2
+  %3 = extractelement <4 x float> %1, i64 0
+  %4 = extractelement <4 x float> %x1, i64 0
+  %5 = extractelement <4 x float> %2, i64 0
+  %6 = call float @llvm.fma.f32(float %3, float %4, float %5)
+  %7 = extractelement <4 x float> %x2, i64 0
+  %8 = bitcast i8 %x3 to <8 x i1>
+  %9 = extractelement <8 x i1> %8, i64 0
+  %10 = select i1 %9, float %6, float %7
+  %11 = insertelement <4 x float> %x2, float %10, i64 0
+  %12 = fneg <4 x float> %x0
+  %13 = fneg <4 x float> %x2
+  %14 = extractelement <4 x float> %12, i64 0
+  %15 = extractelement <4 x float> %x1, i64 0
+  %16 = extractelement <4 x float> %13, i64 0
+  %17 = call float @llvm.x86.avx512.vfmadd.f32(float %14, float %15, float %16, i32 11)
+  %18 = extractelement <4 x float> %x2, i64 0
+  %19 = insertelement <4 x float> %x2, float %17, i64 0
+  %20 = fneg <4 x float> %x0
+  %21 = fneg <4 x float> %x2
+  %22 = extractelement <4 x float> %20, i64 0
+  %23 = extractelement <4 x float> %x1, i64 0
+  %24 = extractelement <4 x float> %21, i64 0
+  %25 = call float @llvm.x86.avx512.vfmadd.f32(float %22, float %23, float %24, i32 10)
+  %26 = extractelement <4 x float> %x2, i64 0
+  %27 = bitcast i8 %x3 to <8 x i1>
+  %28 = extractelement <8 x i1> %27, i64 0
+  %29 = select i1 %28, float %25, float %26
+  %30 = insertelement <4 x float> %x2, float %29, i64 0
+  %res3 = fadd <4 x float> %11, %19
+  %res4 = fadd <4 x float> %30, %res3
+  ret <4 x float> %res4
+}
+
+define <4 x float>@test_int_x86_avx512_mask3_vfmadd_ss_rm(<4 x float> %x0, <4 x float> %x1, ptr%ptr_b ,i8 %x3,i32 %x4, <4 x float> %extra_param) #0 {
+; CHECK-LABEL: @test_int_x86_avx512_mask3_vfmadd_ss_rm(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP5:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 56) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 40) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP6:%.*]], label [[TMP25:%.*]], !prof [[PROF1]]
+; CHECK:       6:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       7:
+; CHECK-NEXT:    [[Q:%.*]] = load float, ptr [[PTR_B:%.*]], align 4
+; CHECK-NEXT:    [[TMP7:%.*]] = ptrtoint ptr [[PTR_B]] to i64
+; CHECK-NEXT:    [[TMP8:%.*]] = xor i64 [[TMP7]], 87960930222080
+; CHECK-NEXT:    [[TMP9:%.*]] = inttoptr i64 [[TMP8]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load i32, ptr [[TMP9]], align 4
+; CHECK-NEXT:    [[_MSPROP:%.*]] = insertelement <4 x i32> [[TMP5]], i32 [[_MSLD]], i32 0
+; CHECK-NEXT:    [[VECINIT_I:%.*]] = insertelement <4 x float> [[EXTRA_PARAM:%.*]], float [[Q]], i32 0
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = extractelement <4 x i32> [[TMP2]], i64 0
+; CHECK-NEXT:    [[TMP10:%.*]] = extractelement <4 x float> [[X0:%.*]], i64 0
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = extractelement <4 x i32> [[_MSPROP]], i64 0
+; CHECK-NEXT:    [[TMP11:%.*]] = extractelement <4 x float> [[VECINIT_I]], i64 0
+; CHECK-NEXT:    [[_MSPROP3:%.*]] = extractelement <4 x i32> [[TMP3]], i64 0
+; CHECK-NEXT:    [[TMP12:%.*]] = extractelement <4 x float> [[X1:%.*]], i64 0
+; CHECK-NEXT:    [[_MSPROP4:%.*]] = or i32 [[_MSPROP1]], [[_MSPROP2]]
+; CHECK-NEXT:    [[_MSPROP5:%.*]] = or i32 [[_MSPROP4]], [[_MSPROP3]]
+; CHECK-NEXT:    [[TMP13:%.*]] = call float @llvm.fma.f32(float [[TMP10]], float [[TMP11]], float [[TMP12]])
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast i8 [[TMP4]] to <8 x i1>
+; CHECK-NEXT:    [[TMP15:%.*]] = bitcast i8 [[X3:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[_MSPROP6:%.*]] = extractelement <8 x i1> [[TMP14]], i64 0
+; CHECK-NEXT:    [[TMP16:%.*]] = extractelement <8 x i1> [[TMP15]], i64 0
+; CHECK-NEXT:    [[TMP17:%.*]] = select i1 [[TMP16]], i32 [[_MSPROP5]], i32 [[_MSPROP3]]
+; CHECK-NEXT:    [[TMP18:%.*]] = bitcast float [[TMP13]] to i32
+; CHECK-NEXT:    [[TMP19:%.*]] = bitcast float [[TMP12]] to i32
+; CHECK-NEXT:    [[TMP20:%.*]] = xor i32 [[TMP18]], [[TMP19]]
+; CHECK-NEXT:    [[TMP21:%.*]] = or i32 [[TMP20]], [[_MSPROP5]]
+; CHECK-NEXT:    [[TMP22:%.*]] = or i32 [[TMP21]], [[_MSPROP3]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select i1 [[_MSPROP6]], i32 [[TMP22]], i32 [[TMP17]]
+; CHECK-NEXT:    [[TMP23:%.*]] = select i1 [[TMP16]], float [[TMP13]], float [[TMP12]]
+; CHECK-NEXT:    [[_MSPROP7:%.*]] = insertelement <4 x i32> [[TMP3]], i32 [[_MSPROP_SELECT]], i64 0
+; CHECK-NEXT:    [[TMP24:%.*]] = insertelement <4 x float> [[X1]], float [[TMP23]], i64 0
+; CHECK-NEXT:    store <4 x i32> [[_MSPROP7]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x float> [[TMP24]]
+;
+  %q = load float, ptr %ptr_b
+  %vecinit.i = insertelement <4 x float> %extra_param, float %q, i32 0
+  %1 = extractelement <4 x float> %x0, i64 0
+  %2 = extractelement <4 x float> %vecinit.i, i64 0
+  %3 = extractelement <4 x float> %x1, i64 0
+  %4 = call float @llvm.fma.f32(float %1, float %2, float %3)
+  %5 = bitcast i8 %x3 to <8 x i1>
+  %6 = extractelement <8 x i1> %5, i64 0
+  %7 = select i1 %6, float %4, float %3
+  %8 = insertelement <4 x float> %x1, float %7, i64 0
+  ret <4 x float> %8
+}
+
+define <4 x float>@test_int_x86_avx512_mask_vfmadd_ss_rm(<4 x float> %x0, <4 x float> %x1,ptr%ptr_b ,i8 %x3,i32 %x4, <4 x float> %extra_param) #0 {
+; CHECK-LABEL: @test_int_x86_avx512_mask_vfmadd_ss_rm(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP5:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 56) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 40) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP6:%.*]], label [[TMP25:%.*]], !prof [[PROF1]]
+; CHECK:       6:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       7:
+; CHECK-NEXT:    [[Q:%.*]] = load float, ptr [[PTR_B:%.*]], align 4
+; CHECK-NEXT:    [[TMP7:%.*]] = ptrtoint ptr [[PTR_B]] to i64
+; CHECK-NEXT:    [[TMP8:%.*]] = xor i64 [[TMP7]], 87960930222080
+; CHECK-NEXT:    [[TMP9:%.*]] = inttoptr i64 [[TMP8]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load i32, ptr [[TMP9]], align 4
+; CHECK-NEXT:    [[_MSPROP:%.*]] = insertelement <4 x i32> [[TMP5]], i32 [[_MSLD]], i32 0
+; CHECK-NEXT:    [[VECINIT_I:%.*]] = insertelement <4 x float> [[EXTRA_PARAM:%.*]], float [[Q]], i32 0
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = extractelement <4 x i32> [[TMP2]], i64 0
+; CHECK-NEXT:    [[TMP10:%.*]] = extractelement <4 x float> [[X0:%.*]], i64 0
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = extractelement <4 x i32> [[_MSPROP]], i64 0
+; CHECK-NEXT:    [[TMP11:%.*]] = extractelement <4 x float> [[VECINIT_I]], i64 0
+; CHECK-NEXT:    [[_MSPROP3:%.*]] = extractelement <4 x i32> [[TMP3]], i64 0
+; CHECK-NEXT:    [[TMP12:%.*]] = extractelement <4 x float> [[X1:%.*]], i64 0
+; CHECK-NEXT:    [[_MSPROP4:%.*]] = or i32 [[_MSPROP1]], [[_MSPROP2]]
+; CHECK-NEXT:    [[_MSPROP5:%.*]] = or i32 [[_MSPROP4]], [[_MSPROP3]]
+; CHECK-NEXT:    [[TMP13:%.*]] = call float @llvm.fma.f32(float [[TMP10]], float [[TMP11]], float [[TMP12]])
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast i8 [[TMP4]] to <8 x i1>
+; CHECK-NEXT:    [[TMP15:%.*]] = bitcast i8 [[X3:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[_MSPROP6:%.*]] = extractelement <8 x i1> [[TMP14]], i64 0
+; CHECK-NEXT:    [[TMP16:%.*]] = extractelement <8 x i1> [[TMP15]], i64 0
+; CHECK-NEXT:    [[TMP17:%.*]] = select i1 [[TMP16]], i32 [[_MSPROP5]], i32 [[_MSPROP1]]
+; CHECK-NEXT:    [[TMP18:%.*]] = bitcast float [[TMP13]] to i32
+; CHECK-NEXT:    [[TMP19:%.*]] = bitcast float [[TMP10]] to i32
+; CHECK-NEXT:    [[TMP20:%.*]] = xor i32 [[TMP18]], [[TMP19]]
+; CHECK-NEXT:    [[TMP21:%.*]] = or i32 [[TMP20]], [[_MSPROP5]]
+; CHECK-NEXT:    [[TMP22:%.*]] = or i32 [[TMP21]], [[_MSPROP1]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select i1 [[_MSPROP6]], i32 [[TMP22]], i32 [[TMP17]]
+; CHECK-NEXT:    [[TMP23:%.*]] = select i1 [[TMP16]], float [[TMP13]], float [[TMP10]]
+; CHECK-NEXT:    [[_MSPROP7:%.*]] = insertelement <4 x i32> [[TMP2]], i32 [[_MSPROP_SELECT]], i64 0
+; CHECK-NEXT:    [[TMP24:%.*]] = insertelement <4 x float> [[X0]], float [[TMP23]], i64 0
+; CHECK-NEXT:    store <4 x i32> [[_MSPROP7]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x float> [[TMP24]]
+;
+  %q = load float, ptr %ptr_b
+  %vecinit.i = insertelement <4 x float> %extra_param, float %q, i32 0
+  %1 = extractelement <4 x float> %x0, i64 0
+  %2 = extractelement <4 x float> %vecinit.i, i64 0
+  %3 = extractelement <4 x float> %x1, i64 0
+  %4 = call float @llvm.fma.f32(float %1, float %2, float %3)
+  %5 = bitcast i8 %x3 to <8 x i1>
+  %6 = extractelement <8 x i1> %5, i64 0
+  %7 = select i1 %6, float %4, float %1
+  %8 = insertelement <4 x float> %x0, float %7, i64 0
+  ret <4 x float> %8
+}
+
+
+define <4 x float>@test_int_x86_avx512_maskz_vfmadd_ss_rm(<4 x float> %x0, <4 x float> %x1,ptr%ptr_b ,i8 %x3,i32 %x4, <4 x float> %extra_param) #0 {
+; CHECK-LABEL: @test_int_x86_avx512_maskz_vfmadd_ss_rm(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 56) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP5:%.*]], label [[TMP20:%.*]], !prof [[PROF1]]
+; CHECK:       5:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       6:
+; CHECK-NEXT:    [[Q:%.*]] = load float, ptr [[PTR_B:%.*]], align 4
+; CHECK-NEXT:    [[TMP6:%.*]] = ptrtoint ptr [[PTR_B]] to i64
+; CHECK-NEXT:    [[TMP7:%.*]] = xor i64 [[TMP6]], 87960930222080
+; CHECK-NEXT:    [[TMP8:%.*]] = inttoptr i64 [[TMP7]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load i32, ptr [[TMP8]], align 4
+; CHECK-NEXT:    [[_MSPROP:%.*]] = insertelement <4 x i32> [[TMP4]], i32 [[_MSLD]], i32 0
+; CHECK-NEXT:    [[VECINIT_I:%.*]] = insertelement <4 x float> [[EXTRA_PARAM:%.*]], float [[Q]], i32 0
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = extractelement <4 x i32> [[TMP2]], i64 0
+; CHECK-NEXT:    [[TMP9:%.*]] = extractelement <4 x float> [[X0:%.*]], i64 0
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = extractelement <4 x i32> [[TMP3]], i64 0
+; CHECK-NEXT:    [[TMP10:%.*]] = extractelement <4 x float> [[X1:%.*]], i64 0
+; CHECK-NEXT:    [[_MSPROP3:%.*]] = extractelement <4 x i32> [[_MSPROP]], i64 0
+; CHECK-NEXT:    [[TMP11:%.*]] = extractelement <4 x float> [[VECINIT_I]], i64 0
+; CHECK-NEXT:    [[_MSPROP4:%.*]] = or i32 [[_MSPROP1]], [[_MSPROP2]]
+; CHECK-NEXT:    [[_MSPROP5:%.*]] = or i32 [[_MSPROP4]], [[_MSPROP3]]
+; CHECK-NEXT:    [[TMP12:%.*]] = call float @llvm.fma.f32(float [[TMP9]], float [[TMP10]], float [[TMP11]])
+; CHECK-NEXT:    [[TMP13:%.*]] = select i1 false, i32 [[_MSPROP5]], i32 0
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast float [[TMP12]] to i32
+; CHECK-NEXT:    [[TMP15:%.*]] = xor i32 [[TMP14]], 0
+; CHECK-NEXT:    [[TMP16:%.*]] = or i32 [[TMP15]], [[_MSPROP5]]
+; CHECK-NEXT:    [[TMP17:%.*]] = or i32 [[TMP16]], 0
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select i1 false, i32 [[TMP17]], i32 [[TMP13]]
+; CHECK-NEXT:    [[TMP18:%.*]] = select i1 false, float [[TMP12]], float 0.000000e+00
+; CHECK-NEXT:    [[_MSPROP6:%.*]] = insertelement <4 x i32> [[TMP2]], i32 [[_MSPROP_SELECT]], i64 0
+; CHECK-NEXT:    [[TMP19:%.*]] = insertelement <4 x float> [[X0]], float [[TMP18]], i64 0
+; CHECK-NEXT:    store <4 x i32> [[_MSPROP6]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x float> [[TMP19]]
+;
+  %q = load float, ptr %ptr_b
+  %vecinit.i = insertelement <4 x float> %extra_param, float %q, i32 0
+  %1 = extractelement <4 x float> %x0, i64 0
+  %2 = extractelement <4 x float> %x1, i64 0
+  %3 = extractelement <4 x float> %vecinit.i, i64 0
+  %4 = call float @llvm.fma.f32(float %1, float %2, float %3)
+  %5 = select i1 false, float %4, float 0.000000e+00
+  %6 = insertelement <4 x float> %x0, float %5, i64 0
+  ret <4 x float> %6
+}
+
+define <16 x i32> @test_x86_avx512_psll_d_512(<16 x i32> %a0, <4 x i32> %a1) #0 {
+; CHECK-LABEL: @test_x86_avx512_psll_d_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <4 x i32> [[TMP2]] to i128
+; CHECK-NEXT:    [[TMP4:%.*]] = trunc i128 [[TMP3]] to i64
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp ne i64 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = sext i1 [[TMP5]] to i512
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast i512 [[TMP6]] to <16 x i32>
+; CHECK-NEXT:    [[TMP8:%.*]] = call <16 x i32> @llvm.x86.avx512.psll.d.512(<16 x i32> [[TMP1]], <4 x i32> [[A1:%.*]])
+; CHECK-NEXT:    [[TMP9:%.*]] = or <16 x i32> [[TMP8]], [[TMP7]]
+; CHECK-NEXT:    [[RES:%.*]] = call <16 x i32> @llvm.x86.avx512.psll.d.512(<16 x i32> [[A0:%.*]], <4 x i32> [[A1]])
+; CHECK-NEXT:    store <16 x i32> [[TMP9]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i32> [[RES]]
+;
+  %res = call <16 x i32> @llvm.x86.avx512.psll.d.512(<16 x i32> %a0, <4 x i32> %a1) ; <<16 x i32>> [#uses=1]
+  ret <16 x i32> %res
+}
+define <16 x i32> @test_x86_avx512_mask_psll_d_512(<16 x i32> %a0, <4 x i32> %a1, <16 x i32> %passthru, i16 %mask) #0 {
+; CHECK-LABEL: @test_x86_avx512_mask_psll_d_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 144) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 80) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <4 x i32> [[TMP2]] to i128
+; CHECK-NEXT:    [[TMP6:%.*]] = trunc i128 [[TMP5]] to i64
+; CHECK-NEXT:    [[TMP7:%.*]] = icmp ne i64 [[TMP6]], 0
+; CHECK-NEXT:    [[TMP8:%.*]] = sext i1 [[TMP7]] to i512
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast i512 [[TMP8]] to <16 x i32>
+; CHECK-NEXT:    [[TMP10:%.*]] = call <16 x i32> @llvm.x86.avx512.psll.d.512(<16 x i32> [[TMP1]], <4 x i32> [[A1:%.*]])
+; CHECK-NEXT:    [[TMP11:%.*]] = or <16 x i32> [[TMP10]], [[TMP9]]
+; CHECK-NEXT:    [[RES:%.*]] = call <16 x i32> @llvm.x86.avx512.psll.d.512(<16 x i32> [[A0:%.*]], <4 x i32> [[A1]])
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast i16 [[TMP3]] to <16 x i1>
+; CHECK-NEXT:    [[MASK_CAST:%.*]] = bitcast i16 [[MASK:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP13:%.*]] = select <16 x i1> [[MASK_CAST]], <16 x i32> [[TMP11]], <16 x i32> [[TMP4]]
+; CHECK-NEXT:    [[TMP14:%.*]] = xor <16 x i32> [[RES]], [[PASSTHRU:%.*]]
+; CHECK-NEXT:    [[TMP15:%.*]] = or <16 x i32> [[TMP14]], [[TMP11]]
+; CHECK-NEXT:    [[TMP16:%.*]] = or <16 x i32> [[TMP15]], [[TMP4]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP12]], <16 x i32> [[TMP16]], <16 x i32> [[TMP13]]
+; CHECK-NEXT:    [[RES2:%.*]] = select <16 x i1> [[MASK_CAST]], <16 x i32> [[RES]], <16 x i32> [[PASSTHRU]]
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i32> [[RES2]]
+;
+  %res = call <16 x i32> @llvm.x86.avx512.psll.d.512(<16 x i32> %a0, <4 x i32> %a1) ; <<16 x i32>> [#uses=1]
+  %mask.cast = bitcast i16 %mask to <16 x i1>
+  %res2 = select <16 x i1> %mask.cast, <16 x i32> %res, <16 x i32> %passthru
+  ret <16 x i32> %res2
+}
+define <16 x i32> @test_x86_avx512_maskz_psll_d_512(<16 x i32> %a0, <4 x i32> %a1, i16 %mask) #0 {
+; CHECK-LABEL: @test_x86_avx512_maskz_psll_d_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 80) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <4 x i32> [[TMP2]] to i128
+; CHECK-NEXT:    [[TMP5:%.*]] = trunc i128 [[TMP4]] to i64
+; CHECK-NEXT:    [[TMP6:%.*]] = icmp ne i64 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP7:%.*]] = sext i1 [[TMP6]] to i512
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast i512 [[TMP7]] to <16 x i32>
+; CHECK-NEXT:    [[TMP9:%.*]] = call <16 x i32> @llvm.x86.avx512.psll.d.512(<16 x i32> [[TMP1]], <4 x i32> [[A1:%.*]])
+; CHECK-NEXT:    [[TMP10:%.*]] = or <16 x i32> [[TMP9]], [[TMP8]]
+; CHECK-NEXT:    [[RES:%.*]] = call <16 x i32> @llvm.x86.avx512.psll.d.512(<16 x i32> [[A0:%.*]], <4 x i32> [[A1]])
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast i16 [[TMP3]] to <16 x i1>
+; CHECK-NEXT:    [[MASK_CAST:%.*]] = bitcast i16 [[MASK:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP12:%.*]] = select <16 x i1> [[MASK_CAST]], <16 x i32> [[TMP10]], <16 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP13:%.*]] = xor <16 x i32> [[RES]], zeroinitializer
+; CHECK-NEXT:    [[TMP14:%.*]] = or <16 x i32> [[TMP13]], [[TMP10]]
+; CHECK-NEXT:    [[TMP15:%.*]] = or <16 x i32> [[TMP14]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP11]], <16 x i32> [[TMP15]], <16 x i32> [[TMP12]]
+; CHECK-NEXT:    [[RES2:%.*]] = select <16 x i1> [[MASK_CAST]], <16 x i32> [[RES]], <16 x i32> zeroinitializer
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i32> [[RES2]]
+;
+  %res = call <16 x i32> @llvm.x86.avx512.psll.d.512(<16 x i32> %a0, <4 x i32> %a1) ; <<16 x i32>> [#uses=1]
+  %mask.cast = bitcast i16 %mask to <16 x i1>
+  %res2 = select <16 x i1> %mask.cast, <16 x i32> %res, <16 x i32> zeroinitializer
+  ret <16 x i32> %res2
+}
+declare <16 x i32> @llvm.x86.avx512.psll.d.512(<16 x i32>, <4 x i32>) nounwind readnone
+
+
+define <8 x i64> @test_x86_avx512_psll_q_512(<8 x i64> %a0, <2 x i64> %a1) #0 {
+; CHECK-LABEL: @test_x86_avx512_psll_q_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <2 x i64> [[TMP2]] to i128
+; CHECK-NEXT:    [[TMP4:%.*]] = trunc i128 [[TMP3]] to i64
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp ne i64 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = sext i1 [[TMP5]] to i512
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast i512 [[TMP6]] to <8 x i64>
+; CHECK-NEXT:    [[TMP8:%.*]] = call <8 x i64> @llvm.x86.avx512.psll.q.512(<8 x i64> [[TMP1]], <2 x i64> [[A1:%.*]])
+; CHECK-NEXT:    [[TMP9:%.*]] = or <8 x i64> [[TMP8]], [[TMP7]]
+; CHECK-NEXT:    [[RES:%.*]] = call <8 x i64> @llvm.x86.avx512.psll.q.512(<8 x i64> [[A0:%.*]], <2 x i64> [[A1]])
+; CHECK-NEXT:    store <8 x i64> [[TMP9]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i64> [[RES]]
+;
+  %res = call <8 x i64> @llvm.x86.avx512.psll.q.512(<8 x i64> %a0, <2 x i64> %a1) ; <<8 x i64>> [#uses=1]
+  ret <8 x i64> %res
+}
+define <8 x i64> @test_x86_avx512_mask_psll_q_512(<8 x i64> %a0, <2 x i64> %a1, <8 x i64> %passthru, i8 %mask) #0 {
+; CHECK-LABEL: @test_x86_avx512_mask_psll_q_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 144) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 80) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <2 x i64> [[TMP2]] to i128
+; CHECK-NEXT:    [[TMP6:%.*]] = trunc i128 [[TMP5]] to i64
+; CHECK-NEXT:    [[TMP7:%.*]] = icmp ne i64 [[TMP6]], 0
+; CHECK-NEXT:    [[TMP8:%.*]] = sext i1 [[TMP7]] to i512
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast i512 [[TMP8]] to <8 x i64>
+; CHECK-NEXT:    [[TMP10:%.*]] = call <8 x i64> @llvm.x86.avx512.psll.q.512(<8 x i64> [[TMP1]], <2 x i64> [[A1:%.*]])
+; CHECK-NEXT:    [[TMP11:%.*]] = or <8 x i64> [[TMP10]], [[TMP9]]
+; CHECK-NEXT:    [[RES:%.*]] = call <8 x i64> @llvm.x86.avx512.psll.q.512(<8 x i64> [[A0:%.*]], <2 x i64> [[A1]])
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast i8 [[TMP3]] to <8 x i1>
+; CHECK-NEXT:    [[MASK_CAST:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP13:%.*]] = select <8 x i1> [[MASK_CAST]], <8 x i64> [[TMP11]], <8 x i64> [[TMP4]]
+; CHECK-NEXT:    [[TMP14:%.*]] = xor <8 x i64> [[RES]], [[PASSTHRU:%.*]]
+; CHECK-NEXT:    [[TMP15:%.*]] = or <8 x i64> [[TMP14]], [[TMP11]]
+; CHECK-NEXT:    [[TMP16:%.*]] = or <8 x i64> [[TMP15]], [[TMP4]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP12]], <8 x i64> [[TMP16]], <8 x i64> [[TMP13]]
+; CHECK-NEXT:    [[RES2:%.*]] = select <8 x i1> [[MASK_CAST]], <8 x i64> [[RES]], <8 x i64> [[PASSTHRU]]
+; CHECK-NEXT:    store <8 x i64> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i64> [[RES2]]
+;
+  %res = call <8 x i64> @llvm.x86.avx512.psll.q.512(<8 x i64> %a0, <2 x i64> %a1) ; <<8 x i64>> [#uses=1]
+  %mask.cast = bitcast i8 %mask to <8 x i1>
+  %res2 = select <8 x i1> %mask.cast, <8 x i64> %res, <8 x i64> %passthru
+  ret <8 x i64> %res2
+}
+define <8 x i64> @test_x86_avx512_maskz_psll_q_512(<8 x i64> %a0, <2 x i64> %a1, i8 %mask) #0 {
+; CHECK-LABEL: @test_x86_avx512_maskz_psll_q_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 80) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <2 x i64> [[TMP2]] to i128
+; CHECK-NEXT:    [[TMP5:%.*]] = trunc i128 [[TMP4]] to i64
+; CHECK-NEXT:    [[TMP6:%.*]] = icmp ne i64 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP7:%.*]] = sext i1 [[TMP6]] to i512
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast i512 [[TMP7]] to <8 x i64>
+; CHECK-NEXT:    [[TMP9:%.*]] = call <8 x i64> @llvm.x86.avx512.psll.q.512(<8 x i64> [[TMP1]], <2 x i64> [[A1:%.*]])
+; CHECK-NEXT:    [[TMP10:%.*]] = or <8 x i64> [[TMP9]], [[TMP8]]
+; CHECK-NEXT:    [[RES:%.*]] = call <8 x i64> @llvm.x86.avx512.psll.q.512(<8 x i64> [[A0:%.*]], <2 x i64> [[A1]])
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast i8 [[TMP3]] to <8 x i1>
+; CHECK-NEXT:    [[MASK_CAST:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP12:%.*]] = select <8 x i1> [[MASK_CAST]], <8 x i64> [[TMP10]], <8 x i64> zeroinitializer
+; CHECK-NEXT:    [[TMP13:%.*]] = xor <8 x i64> [[RES]], zeroinitializer
+; CHECK-NEXT:    [[TMP14:%.*]] = or <8 x i64> [[TMP13]], [[TMP10]]
+; CHECK-NEXT:    [[TMP15:%.*]] = or <8 x i64> [[TMP14]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP11]], <8 x i64> [[TMP15]], <8 x i64> [[TMP12]]
+; CHECK-NEXT:    [[RES2:%.*]] = select <8 x i1> [[MASK_CAST]], <8 x i64> [[RES]], <8 x i64> zeroinitializer
+; CHECK-NEXT:    store <8 x i64> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i64> [[RES2]]
+;
+  %res = call <8 x i64> @llvm.x86.avx512.psll.q.512(<8 x i64> %a0, <2 x i64> %a1) ; <<8 x i64>> [#uses=1]
+  %mask.cast = bitcast i8 %mask to <8 x i1>
+  %res2 = select <8 x i1> %mask.cast, <8 x i64> %res, <8 x i64> zeroinitializer
+  ret <8 x i64> %res2
+}
+declare <8 x i64> @llvm.x86.avx512.psll.q.512(<8 x i64>, <2 x i64>) nounwind readnone
+
+
+define <16 x i32> @test_x86_avx512_pslli_d_512(<16 x i32> %a0) #0 {
+; CHECK-LABEL: @test_x86_avx512_pslli_d_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = call <16 x i32> @llvm.x86.avx512.pslli.d.512(<16 x i32> [[TMP1]], i32 7)
+; CHECK-NEXT:    [[TMP3:%.*]] = or <16 x i32> [[TMP2]], zeroinitializer
+; CHECK-NEXT:    [[RES:%.*]] = call <16 x i32> @llvm.x86.avx512.pslli.d.512(<16 x i32> [[A0:%.*]], i32 7)
+; CHECK-NEXT:    store <16 x i32> [[TMP3]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i32> [[RES]]
+;
+  %res = call <16 x i32> @llvm.x86.avx512.pslli.d.512(<16 x i32> %a0, i32 7) ; <<16 x i32>> [#uses=1]
+  ret <16 x i32> %res
+}
+define <16 x i32> @test_x86_avx512_mask_pslli_d_512(<16 x i32> %a0, <16 x i32> %passthru, i16 %mask) #0 {
+; CHECK-LABEL: @test_x86_avx512_mask_pslli_d_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = call <16 x i32> @llvm.x86.avx512.pslli.d.512(<16 x i32> [[TMP1]], i32 7)
+; CHECK-NEXT:    [[TMP5:%.*]] = or <16 x i32> [[TMP4]], zeroinitializer
+; CHECK-NEXT:    [[RES:%.*]] = call <16 x i32> @llvm.x86.avx512.pslli.d.512(<16 x i32> [[A0:%.*]], i32 7)
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i16 [[TMP2]] to <16 x i1>
+; CHECK-NEXT:    [[MASK_CAST:%.*]] = bitcast i16 [[MASK:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP7:%.*]] = select <16 x i1> [[MASK_CAST]], <16 x i32> [[TMP5]], <16 x i32> [[TMP3]]
+; CHECK-NEXT:    [[TMP8:%.*]] = xor <16 x i32> [[RES]], [[PASSTHRU:%.*]]
+; CHECK-NEXT:    [[TMP9:%.*]] = or <16 x i32> [[TMP8]], [[TMP5]]
+; CHECK-NEXT:    [[TMP10:%.*]] = or <16 x i32> [[TMP9]], [[TMP3]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP6]], <16 x i32> [[TMP10]], <16 x i32> [[TMP7]]
+; CHECK-NEXT:    [[RES2:%.*]] = select <16 x i1> [[MASK_CAST]], <16 x i32> [[RES]], <16 x i32> [[PASSTHRU]]
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i32> [[RES2]]
+;
+  %res = call <16 x i32> @llvm.x86.avx512.pslli.d.512(<16 x i32> %a0, i32 7) ; <<16 x i32>> [#uses=1]
+  %mask.cast = bitcast i16 %mask to <16 x i1>
+  %res2 = select <16 x i1> %mask.cast, <16 x i32> %res, <16 x i32> %passthru
+  ret <16 x i32> %res2
+}
+define <16 x i32> @test_x86_avx512_maskz_pslli_d_512(<16 x i32> %a0, i16 %mask) #0 {
+; CHECK-LABEL: @test_x86_avx512_maskz_pslli_d_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = call <16 x i32> @llvm.x86.avx512.pslli.d.512(<16 x i32> [[TMP1]], i32 7)
+; CHECK-NEXT:    [[TMP4:%.*]] = or <16 x i32> [[TMP3]], zeroinitializer
+; CHECK-NEXT:    [[RES:%.*]] = call <16 x i32> @llvm.x86.avx512.pslli.d.512(<16 x i32> [[A0:%.*]], i32 7)
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i16 [[TMP2]] to <16 x i1>
+; CHECK-NEXT:    [[MASK_CAST:%.*]] = bitcast i16 [[MASK:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP6:%.*]] = select <16 x i1> [[MASK_CAST]], <16 x i32> [[TMP4]], <16 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP7:%.*]] = xor <16 x i32> [[RES]], zeroinitializer
+; CHECK-NEXT:    [[TMP8:%.*]] = or <16 x i32> [[TMP7]], [[TMP4]]
+; CHECK-NEXT:    [[TMP9:%.*]] = or <16 x i32> [[TMP8]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP5]], <16 x i32> [[TMP9]], <16 x i32> [[TMP6]]
+; CHECK-NEXT:    [[RES2:%.*]] = select <16 x i1> [[MASK_CAST]], <16 x i32> [[RES]], <16 x i32> zeroinitializer
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i32> [[RES2]]
+;
+  %res = call <16 x i32> @llvm.x86.avx512.pslli.d.512(<16 x i32> %a0, i32 7) ; <<16 x i32>> [#uses=1]
+  %mask.cast = bitcast i16 %mask to <16 x i1>
+  %res2 = select <16 x i1> %mask.cast, <16 x i32> %res, <16 x i32> zeroinitializer
+  ret <16 x i32> %res2
+}
+declare <16 x i32> @llvm.x86.avx512.pslli.d.512(<16 x i32>, i32) nounwind readnone
+
+
+define <8 x i64> @test_x86_avx512_pslli_q_512(<8 x i64> %a0) #0 {
+; CHECK-LABEL: @test_x86_avx512_pslli_q_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = call <8 x i64> @llvm.x86.avx512.pslli.q.512(<8 x i64> [[TMP1]], i32 7)
+; CHECK-NEXT:    [[TMP3:%.*]] = or <8 x i64> [[TMP2]], zeroinitializer
+; CHECK-NEXT:    [[RES:%.*]] = call <8 x i64> @llvm.x86.avx512.pslli.q.512(<8 x i64> [[A0:%.*]], i32 7)
+; CHECK-NEXT:    store <8 x i64> [[TMP3]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i64> [[RES]]
+;
+  %res = call <8 x i64> @llvm.x86.avx512.pslli.q.512(<8 x i64> %a0, i32 7) ; <<8 x i64>> [#uses=1]
+  ret <8 x i64> %res
+}
+define <8 x i64> @test_x86_avx512_mask_pslli_q_512(<8 x i64> %a0, <8 x i64> %passthru, i8 %mask) #0 {
+; CHECK-LABEL: @test_x86_avx512_mask_pslli_q_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = call <8 x i64> @llvm.x86.avx512.pslli.q.512(<8 x i64> [[TMP1]], i32 7)
+; CHECK-NEXT:    [[TMP5:%.*]] = or <8 x i64> [[TMP4]], zeroinitializer
+; CHECK-NEXT:    [[RES:%.*]] = call <8 x i64> @llvm.x86.avx512.pslli.q.512(<8 x i64> [[A0:%.*]], i32 7)
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i8 [[TMP2]] to <8 x i1>
+; CHECK-NEXT:    [[MASK_CAST:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP7:%.*]] = select <8 x i1> [[MASK_CAST]], <8 x i64> [[TMP5]], <8 x i64> [[TMP3]]
+; CHECK-NEXT:    [[TMP8:%.*]] = xor <8 x i64> [[RES]], [[PASSTHRU:%.*]]
+; CHECK-NEXT:    [[TMP9:%.*]] = or <8 x i64> [[TMP8]], [[TMP5]]
+; CHECK-NEXT:    [[TMP10:%.*]] = or <8 x i64> [[TMP9]], [[TMP3]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP6]], <8 x i64> [[TMP10]], <8 x i64> [[TMP7]]
+; CHECK-NEXT:    [[RES2:%.*]] = select <8 x i1> [[MASK_CAST]], <8 x i64> [[RES]], <8 x i64> [[PASSTHRU]]
+; CHECK-NEXT:    store <8 x i64> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i64> [[RES2]]
+;
+  %res = call <8 x i64> @llvm.x86.avx512.pslli.q.512(<8 x i64> %a0, i32 7) ; <<8 x i64>> [#uses=1]
+  %mask.cast = bitcast i8 %mask to <8 x i1>
+  %res2 = select <8 x i1> %mask.cast, <8 x i64> %res, <8 x i64> %passthru
+  ret <8 x i64> %res2
+}
+define <8 x i64> @test_x86_avx512_maskz_pslli_q_512(<8 x i64> %a0, <8 x i64> %passthru, i8 %mask) #0 {
+; CHECK-LABEL: @test_x86_avx512_maskz_pslli_q_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = call <8 x i64> @llvm.x86.avx512.pslli.q.512(<8 x i64> [[TMP1]], i32 7)
+; CHECK-NEXT:    [[TMP4:%.*]] = or <8 x i64> [[TMP3]], zeroinitializer
+; CHECK-NEXT:    [[RES:%.*]] = call <8 x i64> @llvm.x86.avx512.pslli.q.512(<8 x i64> [[A0:%.*]], i32 7)
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i8 [[TMP2]] to <8 x i1>
+; CHECK-NEXT:    [[MASK_CAST:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP6:%.*]] = select <8 x i1> [[MASK_CAST]], <8 x i64> [[TMP4]], <8 x i64> zeroinitializer
+; CHECK-NEXT:    [[TMP7:%.*]] = xor <8 x i64> [[RES]], zeroinitializer
+; CHECK-NEXT:    [[TMP8:%.*]] = or <8 x i64> [[TMP7]], [[TMP4]]
+; CHECK-NEXT:    [[TMP9:%.*]] = or <8 x i64> [[TMP8]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP5]], <8 x i64> [[TMP9]], <8 x i64> [[TMP6]]
+; CHECK-NEXT:    [[RES2:%.*]] = select <8 x i1> [[MASK_CAST]], <8 x i64> [[RES]], <8 x i64> zeroinitializer
+; CHECK-NEXT:    store <8 x i64> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i64> [[RES2]]
+;
+  %res = call <8 x i64> @llvm.x86.avx512.pslli.q.512(<8 x i64> %a0, i32 7) ; <<8 x i64>> [#uses=1]
+  %mask.cast = bitcast i8 %mask to <8 x i1>
+  %res2 = select <8 x i1> %mask.cast, <8 x i64> %res, <8 x i64> zeroinitializer
+  ret <8 x i64> %res2
+}
+declare <8 x i64> @llvm.x86.avx512.pslli.q.512(<8 x i64>, i32) nounwind readnone
+
+
+define <8 x i64> @test_x86_avx512_psra_q_512(<8 x i64> %a0, <2 x i64> %a1) #0 {
+; CHECK-LABEL: @test_x86_avx512_psra_q_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <2 x i64> [[TMP2]] to i128
+; CHECK-NEXT:    [[TMP4:%.*]] = trunc i128 [[TMP3]] to i64
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp ne i64 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = sext i1 [[TMP5]] to i512
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast i512 [[TMP6]] to <8 x i64>
+; CHECK-NEXT:    [[TMP8:%.*]] = call <8 x i64> @llvm.x86.avx512.psra.q.512(<8 x i64> [[TMP1]], <2 x i64> [[A1:%.*]])
+; CHECK-NEXT:    [[TMP9:%.*]] = or <8 x i64> [[TMP8]], [[TMP7]]
+; CHECK-NEXT:    [[RES:%.*]] = call <8 x i64> @llvm.x86.avx512.psra.q.512(<8 x i64> [[A0:%.*]], <2 x i64> [[A1]])
+; CHECK-NEXT:    store <8 x i64> [[TMP9]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i64> [[RES]]
+;
+  %res = call <8 x i64> @llvm.x86.avx512.psra.q.512(<8 x i64> %a0, <2 x i64> %a1) ; <<8 x i64>> [#uses=1]
+  ret <8 x i64> %res
+}
+define <8 x i64> @test_x86_avx512_mask_psra_q_512(<8 x i64> %a0, <2 x i64> %a1, <8 x i64> %passthru, i8 %mask) #0 {
+; CHECK-LABEL: @test_x86_avx512_mask_psra_q_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 144) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 80) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <2 x i64> [[TMP2]] to i128
+; CHECK-NEXT:    [[TMP6:%.*]] = trunc i128 [[TMP5]] to i64
+; CHECK-NEXT:    [[TMP7:%.*]] = icmp ne i64 [[TMP6]], 0
+; CHECK-NEXT:    [[TMP8:%.*]] = sext i1 [[TMP7]] to i512
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast i512 [[TMP8]] to <8 x i64>
+; CHECK-NEXT:    [[TMP10:%.*]] = call <8 x i64> @llvm.x86.avx512.psra.q.512(<8 x i64> [[TMP1]], <2 x i64> [[A1:%.*]])
+; CHECK-NEXT:    [[TMP11:%.*]] = or <8 x i64> [[TMP10]], [[TMP9]]
+; CHECK-NEXT:    [[RES:%.*]] = call <8 x i64> @llvm.x86.avx512.psra.q.512(<8 x i64> [[A0:%.*]], <2 x i64> [[A1]])
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast i8 [[TMP3]] to <8 x i1>
+; CHECK-NEXT:    [[MASK_CAST:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP13:%.*]] = select <8 x i1> [[MASK_CAST]], <8 x i64> [[TMP11]], <8 x i64> [[TMP4]]
+; CHECK-NEXT:    [[TMP14:%.*]] = xor <8 x i64> [[RES]], [[PASSTHRU:%.*]]
+; CHECK-NEXT:    [[TMP15:%.*]] = or <8 x i64> [[TMP14]], [[TMP11]]
+; CHECK-NEXT:    [[TMP16:%.*]] = or <8 x i64> [[TMP15]], [[TMP4]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP12]], <8 x i64> [[TMP16]], <8 x i64> [[TMP13]]
+; CHECK-NEXT:    [[RES2:%.*]] = select <8 x i1> [[MASK_CAST]], <8 x i64> [[RES]], <8 x i64> [[PASSTHRU]]
+; CHECK-NEXT:    store <8 x i64> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i64> [[RES2]]
+;
+  %res = call <8 x i64> @llvm.x86.avx512.psra.q.512(<8 x i64> %a0, <2 x i64> %a1) ; <<8 x i64>> [#uses=1]
+  %mask.cast = bitcast i8 %mask to <8 x i1>
+  %res2 = select <8 x i1> %mask.cast, <8 x i64> %res, <8 x i64> %passthru
+  ret <8 x i64> %res2
+}
+define <8 x i64> @test_x86_avx512_maskz_psra_q_512(<8 x i64> %a0, <2 x i64> %a1, i8 %mask) #0 {
+; CHECK-LABEL: @test_x86_avx512_maskz_psra_q_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 80) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <2 x i64> [[TMP2]] to i128
+; CHECK-NEXT:    [[TMP5:%.*]] = trunc i128 [[TMP4]] to i64
+; CHECK-NEXT:    [[TMP6:%.*]] = icmp ne i64 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP7:%.*]] = sext i1 [[TMP6]] to i512
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast i512 [[TMP7]] to <8 x i64>
+; CHECK-NEXT:    [[TMP9:%.*]] = call <8 x i64> @llvm.x86.avx512.psra.q.512(<8 x i64> [[TMP1]], <2 x i64> [[A1:%.*]])
+; CHECK-NEXT:    [[TMP10:%.*]] = or <8 x i64> [[TMP9]], [[TMP8]]
+; CHECK-NEXT:    [[RES:%.*]] = call <8 x i64> @llvm.x86.avx512.psra.q.512(<8 x i64> [[A0:%.*]], <2 x i64> [[A1]])
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast i8 [[TMP3]] to <8 x i1>
+; CHECK-NEXT:    [[MASK_CAST:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP12:%.*]] = select <8 x i1> [[MASK_CAST]], <8 x i64> [[TMP10]], <8 x i64> zeroinitializer
+; CHECK-NEXT:    [[TMP13:%.*]] = xor <8 x i64> [[RES]], zeroinitializer
+; CHECK-NEXT:    [[TMP14:%.*]] = or <8 x i64> [[TMP13]], [[TMP10]]
+; CHECK-NEXT:    [[TMP15:%.*]] = or <8 x i64> [[TMP14]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP11]], <8 x i64> [[TMP15]], <8 x i64> [[TMP12]]
+; CHECK-NEXT:    [[RES2:%.*]] = select <8 x i1> [[MASK_CAST]], <8 x i64> [[RES]], <8 x i64> zeroinitializer
+; CHECK-NEXT:    store <8 x i64> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i64> [[RES2]]
+;
+  %res = call <8 x i64> @llvm.x86.avx512.psra.q.512(<8 x i64> %a0, <2 x i64> %a1) ; <<8 x i64>> [#uses=1]
+  %mask.cast = bitcast i8 %mask to <8 x i1>
+  %res2 = select <8 x i1> %mask.cast, <8 x i64> %res, <8 x i64> zeroinitializer
+  ret <8 x i64> %res2
+}
+declare <8 x i64> @llvm.x86.avx512.psra.q.512(<8 x i64>, <2 x i64>) nounwind readnone
+
+
+define <16 x i32> @test_x86_avx512_psra_d_512(<16 x i32> %a0, <4 x i32> %a1) #0 {
+; CHECK-LABEL: @test_x86_avx512_psra_d_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <4 x i32> [[TMP2]] to i128
+; CHECK-NEXT:    [[TMP4:%.*]] = trunc i128 [[TMP3]] to i64
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp ne i64 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = sext i1 [[TMP5]] to i512
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast i512 [[TMP6]] to <16 x i32>
+; CHECK-NEXT:    [[TMP8:%.*]] = call <16 x i32> @llvm.x86.avx512.psra.d.512(<16 x i32> [[TMP1]], <4 x i32> [[A1:%.*]])
+; CHECK-NEXT:    [[TMP9:%.*]] = or <16 x i32> [[TMP8]], [[TMP7]]
+; CHECK-NEXT:    [[RES:%.*]] = call <16 x i32> @llvm.x86.avx512.psra.d.512(<16 x i32> [[A0:%.*]], <4 x i32> [[A1]])
+; CHECK-NEXT:    store <16 x i32> [[TMP9]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i32> [[RES]]
+;
+  %res = call <16 x i32> @llvm.x86.avx512.psra.d.512(<16 x i32> %a0, <4 x i32> %a1) ; <<16 x i32>> [#uses=1]
+  ret <16 x i32> %res
+}
+define <16 x i32> @test_x86_avx512_mask_psra_d_512(<16 x i32> %a0, <4 x i32> %a1, <16 x i32> %passthru, i16 %mask) #0 {
+; CHECK-LABEL: @test_x86_avx512_mask_psra_d_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 144) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 80) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <4 x i32> [[TMP2]] to i128
+; CHECK-NEXT:    [[TMP6:%.*]] = trunc i128 [[TMP5]] to i64
+; CHECK-NEXT:    [[TMP7:%.*]] = icmp ne i64 [[TMP6]], 0
+; CHECK-NEXT:    [[TMP8:%.*]] = sext i1 [[TMP7]] to i512
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast i512 [[TMP8]] to <16 x i32>
+; CHECK-NEXT:    [[TMP10:%.*]] = call <16 x i32> @llvm.x86.avx512.psra.d.512(<16 x i32> [[TMP1]], <4 x i32> [[A1:%.*]])
+; CHECK-NEXT:    [[TMP11:%.*]] = or <16 x i32> [[TMP10]], [[TMP9]]
+; CHECK-NEXT:    [[RES:%.*]] = call <16 x i32> @llvm.x86.avx512.psra.d.512(<16 x i32> [[A0:%.*]], <4 x i32> [[A1]])
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast i16 [[TMP3]] to <16 x i1>
+; CHECK-NEXT:    [[MASK_CAST:%.*]] = bitcast i16 [[MASK:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP13:%.*]] = select <16 x i1> [[MASK_CAST]], <16 x i32> [[TMP11]], <16 x i32> [[TMP4]]
+; CHECK-NEXT:    [[TMP14:%.*]] = xor <16 x i32> [[RES]], [[PASSTHRU:%.*]]
+; CHECK-NEXT:    [[TMP15:%.*]] = or <16 x i32> [[TMP14]], [[TMP11]]
+; CHECK-NEXT:    [[TMP16:%.*]] = or <16 x i32> [[TMP15]], [[TMP4]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP12]], <16 x i32> [[TMP16]], <16 x i32> [[TMP13]]
+; CHECK-NEXT:    [[RES2:%.*]] = select <16 x i1> [[MASK_CAST]], <16 x i32> [[RES]], <16 x i32> [[PASSTHRU]]
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i32> [[RES2]]
+;
+  %res = call <16 x i32> @llvm.x86.avx512.psra.d.512(<16 x i32> %a0, <4 x i32> %a1) ; <<16 x i32>> [#uses=1]
+  %mask.cast = bitcast i16 %mask to <16 x i1>
+  %res2 = select <16 x i1> %mask.cast, <16 x i32> %res, <16 x i32> %passthru
+  ret <16 x i32> %res2
+}
+define <16 x i32> @test_x86_avx512_maskz_psra_d_512(<16 x i32> %a0, <4 x i32> %a1, i16 %mask) #0 {
+; CHECK-LABEL: @test_x86_avx512_maskz_psra_d_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 80) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <4 x i32> [[TMP2]] to i128
+; CHECK-NEXT:    [[TMP5:%.*]] = trunc i128 [[TMP4]] to i64
+; CHECK-NEXT:    [[TMP6:%.*]] = icmp ne i64 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP7:%.*]] = sext i1 [[TMP6]] to i512
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast i512 [[TMP7]] to <16 x i32>
+; CHECK-NEXT:    [[TMP9:%.*]] = call <16 x i32> @llvm.x86.avx512.psra.d.512(<16 x i32> [[TMP1]], <4 x i32> [[A1:%.*]])
+; CHECK-NEXT:    [[TMP10:%.*]] = or <16 x i32> [[TMP9]], [[TMP8]]
+; CHECK-NEXT:    [[RES:%.*]] = call <16 x i32> @llvm.x86.avx512.psra.d.512(<16 x i32> [[A0:%.*]], <4 x i32> [[A1]])
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast i16 [[TMP3]] to <16 x i1>
+; CHECK-NEXT:    [[MASK_CAST:%.*]] = bitcast i16 [[MASK:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP12:%.*]] = select <16 x i1> [[MASK_CAST]], <16 x i32> [[TMP10]], <16 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP13:%.*]] = xor <16 x i32> [[RES]], zeroinitializer
+; CHECK-NEXT:    [[TMP14:%.*]] = or <16 x i32> [[TMP13]], [[TMP10]]
+; CHECK-NEXT:    [[TMP15:%.*]] = or <16 x i32> [[TMP14]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP11]], <16 x i32> [[TMP15]], <16 x i32> [[TMP12]]
+; CHECK-NEXT:    [[RES2:%.*]] = select <16 x i1> [[MASK_CAST]], <16 x i32> [[RES]], <16 x i32> zeroinitializer
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i32> [[RES2]]
+;
+  %res = call <16 x i32> @llvm.x86.avx512.psra.d.512(<16 x i32> %a0, <4 x i32> %a1) ; <<16 x i32>> [#uses=1]
+  %mask.cast = bitcast i16 %mask to <16 x i1>
+  %res2 = select <16 x i1> %mask.cast, <16 x i32> %res, <16 x i32> zeroinitializer
+  ret <16 x i32> %res2
+}
+declare <16 x i32> @llvm.x86.avx512.psra.d.512(<16 x i32>, <4 x i32>) nounwind readnone
+
+
+
+define <8 x i64> @test_x86_avx512_psrai_q_512(<8 x i64> %a0) #0 {
+; CHECK-LABEL: @test_x86_avx512_psrai_q_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = call <8 x i64> @llvm.x86.avx512.psrai.q.512(<8 x i64> [[TMP1]], i32 7)
+; CHECK-NEXT:    [[TMP3:%.*]] = or <8 x i64> [[TMP2]], zeroinitializer
+; CHECK-NEXT:    [[RES:%.*]] = call <8 x i64> @llvm.x86.avx512.psrai.q.512(<8 x i64> [[A0:%.*]], i32 7)
+; CHECK-NEXT:    store <8 x i64> [[TMP3]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i64> [[RES]]
+;
+  %res = call <8 x i64> @llvm.x86.avx512.psrai.q.512(<8 x i64> %a0, i32 7) ; <<8 x i64>> [#uses=1]
+  ret <8 x i64> %res
+}
+define <8 x i64> @test_x86_avx512_mask_psrai_q_512(<8 x i64> %a0, <8 x i64> %passthru, i8 %mask) #0 {
+; CHECK-LABEL: @test_x86_avx512_mask_psrai_q_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = call <8 x i64> @llvm.x86.avx512.psrai.q.512(<8 x i64> [[TMP1]], i32 7)
+; CHECK-NEXT:    [[TMP5:%.*]] = or <8 x i64> [[TMP4]], zeroinitializer
+; CHECK-NEXT:    [[RES:%.*]] = call <8 x i64> @llvm.x86.avx512.psrai.q.512(<8 x i64> [[A0:%.*]], i32 7)
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i8 [[TMP2]] to <8 x i1>
+; CHECK-NEXT:    [[MASK_CAST:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP7:%.*]] = select <8 x i1> [[MASK_CAST]], <8 x i64> [[TMP5]], <8 x i64> [[TMP3]]
+; CHECK-NEXT:    [[TMP8:%.*]] = xor <8 x i64> [[RES]], [[PASSTHRU:%.*]]
+; CHECK-NEXT:    [[TMP9:%.*]] = or <8 x i64> [[TMP8]], [[TMP5]]
+; CHECK-NEXT:    [[TMP10:%.*]] = or <8 x i64> [[TMP9]], [[TMP3]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP6]], <8 x i64> [[TMP10]], <8 x i64> [[TMP7]]
+; CHECK-NEXT:    [[RES2:%.*]] = select <8 x i1> [[MASK_CAST]], <8 x i64> [[RES]], <8 x i64> [[PASSTHRU]]
+; CHECK-NEXT:    store <8 x i64> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i64> [[RES2]]
+;
+  %res = call <8 x i64> @llvm.x86.avx512.psrai.q.512(<8 x i64> %a0, i32 7) ; <<8 x i64>> [#uses=1]
+  %mask.cast = bitcast i8 %mask to <8 x i1>
+  %res2 = select <8 x i1> %mask.cast, <8 x i64> %res, <8 x i64> %passthru
+  ret <8 x i64> %res2
+}
+define <8 x i64> @test_x86_avx512_maskz_psrai_q_512(<8 x i64> %a0, i8 %mask) #0 {
+; CHECK-LABEL: @test_x86_avx512_maskz_psrai_q_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = call <8 x i64> @llvm.x86.avx512.psrai.q.512(<8 x i64> [[TMP1]], i32 7)
+; CHECK-NEXT:    [[TMP4:%.*]] = or <8 x i64> [[TMP3]], zeroinitializer
+; CHECK-NEXT:    [[RES:%.*]] = call <8 x i64> @llvm.x86.avx512.psrai.q.512(<8 x i64> [[A0:%.*]], i32 7)
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i8 [[TMP2]] to <8 x i1>
+; CHECK-NEXT:    [[MASK_CAST:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP6:%.*]] = select <8 x i1> [[MASK_CAST]], <8 x i64> [[TMP4]], <8 x i64> zeroinitializer
+; CHECK-NEXT:    [[TMP7:%.*]] = xor <8 x i64> [[RES]], zeroinitializer
+; CHECK-NEXT:    [[TMP8:%.*]] = or <8 x i64> [[TMP7]], [[TMP4]]
+; CHECK-NEXT:    [[TMP9:%.*]] = or <8 x i64> [[TMP8]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP5]], <8 x i64> [[TMP9]], <8 x i64> [[TMP6]]
+; CHECK-NEXT:    [[RES2:%.*]] = select <8 x i1> [[MASK_CAST]], <8 x i64> [[RES]], <8 x i64> zeroinitializer
+; CHECK-NEXT:    store <8 x i64> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i64> [[RES2]]
+;
+  %res = call <8 x i64> @llvm.x86.avx512.psrai.q.512(<8 x i64> %a0, i32 7) ; <<8 x i64>> [#uses=1]
+  %mask.cast = bitcast i8 %mask to <8 x i1>
+  %res2 = select <8 x i1> %mask.cast, <8 x i64> %res, <8 x i64> zeroinitializer
+  ret <8 x i64> %res2
+}
+declare <8 x i64> @llvm.x86.avx512.psrai.q.512(<8 x i64>, i32) nounwind readnone
+
+
+define <16 x i32> @test_x86_avx512_psrai_d_512(<16 x i32> %a0) #0 {
+; CHECK-LABEL: @test_x86_avx512_psrai_d_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = call <16 x i32> @llvm.x86.avx512.psrai.d.512(<16 x i32> [[TMP1]], i32 7)
+; CHECK-NEXT:    [[TMP3:%.*]] = or <16 x i32> [[TMP2]], zeroinitializer
+; CHECK-NEXT:    [[RES:%.*]] = call <16 x i32> @llvm.x86.avx512.psrai.d.512(<16 x i32> [[A0:%.*]], i32 7)
+; CHECK-NEXT:    store <16 x i32> [[TMP3]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i32> [[RES]]
+;
+  %res = call <16 x i32> @llvm.x86.avx512.psrai.d.512(<16 x i32> %a0, i32 7) ; <<16 x i32>> [#uses=1]
+  ret <16 x i32> %res
+}
+define <16 x i32> @test_x86_avx512_mask_psrai_d_512(<16 x i32> %a0, <16 x i32> %passthru, i16 %mask) #0 {
+; CHECK-LABEL: @test_x86_avx512_mask_psrai_d_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = call <16 x i32> @llvm.x86.avx512.psrai.d.512(<16 x i32> [[TMP1]], i32 7)
+; CHECK-NEXT:    [[TMP5:%.*]] = or <16 x i32> [[TMP4]], zeroinitializer
+; CHECK-NEXT:    [[RES:%.*]] = call <16 x i32> @llvm.x86.avx512.psrai.d.512(<16 x i32> [[A0:%.*]], i32 7)
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i16 [[TMP2]] to <16 x i1>
+; CHECK-NEXT:    [[MASK_CAST:%.*]] = bitcast i16 [[MASK:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP7:%.*]] = select <16 x i1> [[MASK_CAST]], <16 x i32> [[TMP5]], <16 x i32> [[TMP3]]
+; CHECK-NEXT:    [[TMP8:%.*]] = xor <16 x i32> [[RES]], [[PASSTHRU:%.*]]
+; CHECK-NEXT:    [[TMP9:%.*]] = or <16 x i32> [[TMP8]], [[TMP5]]
+; CHECK-NEXT:    [[TMP10:%.*]] = or <16 x i32> [[TMP9]], [[TMP3]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP6]], <16 x i32> [[TMP10]], <16 x i32> [[TMP7]]
+; CHECK-NEXT:    [[RES2:%.*]] = select <16 x i1> [[MASK_CAST]], <16 x i32> [[RES]], <16 x i32> [[PASSTHRU]]
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i32> [[RES2]]
+;
+  %res = call <16 x i32> @llvm.x86.avx512.psrai.d.512(<16 x i32> %a0, i32 7) ; <<16 x i32>> [#uses=1]
+  %mask.cast = bitcast i16 %mask to <16 x i1>
+  %res2 = select <16 x i1> %mask.cast, <16 x i32> %res, <16 x i32> %passthru
+  ret <16 x i32> %res2
+}
+define <16 x i32> @test_x86_avx512_maskz_psrai_d_512(<16 x i32> %a0, i16 %mask) #0 {
+; CHECK-LABEL: @test_x86_avx512_maskz_psrai_d_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = call <16 x i32> @llvm.x86.avx512.psrai.d.512(<16 x i32> [[TMP1]], i32 7)
+; CHECK-NEXT:    [[TMP4:%.*]] = or <16 x i32> [[TMP3]], zeroinitializer
+; CHECK-NEXT:    [[RES:%.*]] = call <16 x i32> @llvm.x86.avx512.psrai.d.512(<16 x i32> [[A0:%.*]], i32 7)
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i16 [[TMP2]] to <16 x i1>
+; CHECK-NEXT:    [[MASK_CAST:%.*]] = bitcast i16 [[MASK:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP6:%.*]] = select <16 x i1> [[MASK_CAST]], <16 x i32> [[TMP4]], <16 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP7:%.*]] = xor <16 x i32> [[RES]], zeroinitializer
+; CHECK-NEXT:    [[TMP8:%.*]] = or <16 x i32> [[TMP7]], [[TMP4]]
+; CHECK-NEXT:    [[TMP9:%.*]] = or <16 x i32> [[TMP8]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP5]], <16 x i32> [[TMP9]], <16 x i32> [[TMP6]]
+; CHECK-NEXT:    [[RES2:%.*]] = select <16 x i1> [[MASK_CAST]], <16 x i32> [[RES]], <16 x i32> zeroinitializer
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i32> [[RES2]]
+;
+  %res = call <16 x i32> @llvm.x86.avx512.psrai.d.512(<16 x i32> %a0, i32 7) ; <<16 x i32>> [#uses=1]
+  %mask.cast = bitcast i16 %mask to <16 x i1>
+  %res2 = select <16 x i1> %mask.cast, <16 x i32> %res, <16 x i32> zeroinitializer
+  ret <16 x i32> %res2
+}
+declare <16 x i32> @llvm.x86.avx512.psrai.d.512(<16 x i32>, i32) nounwind readnone
+
+
+
+define <16 x i32> @test_x86_avx512_psrl_d_512(<16 x i32> %a0, <4 x i32> %a1) #0 {
+; CHECK-LABEL: @test_x86_avx512_psrl_d_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <4 x i32> [[TMP2]] to i128
+; CHECK-NEXT:    [[TMP4:%.*]] = trunc i128 [[TMP3]] to i64
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp ne i64 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = sext i1 [[TMP5]] to i512
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast i512 [[TMP6]] to <16 x i32>
+; CHECK-NEXT:    [[TMP8:%.*]] = call <16 x i32> @llvm.x86.avx512.psrl.d.512(<16 x i32> [[TMP1]], <4 x i32> [[A1:%.*]])
+; CHECK-NEXT:    [[TMP9:%.*]] = or <16 x i32> [[TMP8]], [[TMP7]]
+; CHECK-NEXT:    [[RES:%.*]] = call <16 x i32> @llvm.x86.avx512.psrl.d.512(<16 x i32> [[A0:%.*]], <4 x i32> [[A1]])
+; CHECK-NEXT:    store <16 x i32> [[TMP9]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i32> [[RES]]
+;
+  %res = call <16 x i32> @llvm.x86.avx512.psrl.d.512(<16 x i32> %a0, <4 x i32> %a1) ; <<16 x i32>> [#uses=1]
+  ret <16 x i32> %res
+}
+define <16 x i32> @test_x86_avx512_mask_psrl_d_512(<16 x i32> %a0, <4 x i32> %a1, <16 x i32> %passthru, i16 %mask) #0 {
+; CHECK-LABEL: @test_x86_avx512_mask_psrl_d_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 144) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 80) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <4 x i32> [[TMP2]] to i128
+; CHECK-NEXT:    [[TMP6:%.*]] = trunc i128 [[TMP5]] to i64
+; CHECK-NEXT:    [[TMP7:%.*]] = icmp ne i64 [[TMP6]], 0
+; CHECK-NEXT:    [[TMP8:%.*]] = sext i1 [[TMP7]] to i512
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast i512 [[TMP8]] to <16 x i32>
+; CHECK-NEXT:    [[TMP10:%.*]] = call <16 x i32> @llvm.x86.avx512.psrl.d.512(<16 x i32> [[TMP1]], <4 x i32> [[A1:%.*]])
+; CHECK-NEXT:    [[TMP11:%.*]] = or <16 x i32> [[TMP10]], [[TMP9]]
+; CHECK-NEXT:    [[RES:%.*]] = call <16 x i32> @llvm.x86.avx512.psrl.d.512(<16 x i32> [[A0:%.*]], <4 x i32> [[A1]])
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast i16 [[TMP3]] to <16 x i1>
+; CHECK-NEXT:    [[MASK_CAST:%.*]] = bitcast i16 [[MASK:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP13:%.*]] = select <16 x i1> [[MASK_CAST]], <16 x i32> [[TMP11]], <16 x i32> [[TMP4]]
+; CHECK-NEXT:    [[TMP14:%.*]] = xor <16 x i32> [[RES]], [[PASSTHRU:%.*]]
+; CHECK-NEXT:    [[TMP15:%.*]] = or <16 x i32> [[TMP14]], [[TMP11]]
+; CHECK-NEXT:    [[TMP16:%.*]] = or <16 x i32> [[TMP15]], [[TMP4]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP12]], <16 x i32> [[TMP16]], <16 x i32> [[TMP13]]
+; CHECK-NEXT:    [[RES2:%.*]] = select <16 x i1> [[MASK_CAST]], <16 x i32> [[RES]], <16 x i32> [[PASSTHRU]]
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i32> [[RES2]]
+;
+  %res = call <16 x i32> @llvm.x86.avx512.psrl.d.512(<16 x i32> %a0, <4 x i32> %a1) ; <<16 x i32>> [#uses=1]
+  %mask.cast = bitcast i16 %mask to <16 x i1>
+  %res2 = select <16 x i1> %mask.cast, <16 x i32> %res, <16 x i32> %passthru
+  ret <16 x i32> %res2
+}
+define <16 x i32> @test_x86_avx512_maskz_psrl_d_512(<16 x i32> %a0, <4 x i32> %a1, i16 %mask) #0 {
+; CHECK-LABEL: @test_x86_avx512_maskz_psrl_d_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 80) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <4 x i32> [[TMP2]] to i128
+; CHECK-NEXT:    [[TMP5:%.*]] = trunc i128 [[TMP4]] to i64
+; CHECK-NEXT:    [[TMP6:%.*]] = icmp ne i64 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP7:%.*]] = sext i1 [[TMP6]] to i512
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast i512 [[TMP7]] to <16 x i32>
+; CHECK-NEXT:    [[TMP9:%.*]] = call <16 x i32> @llvm.x86.avx512.psrl.d.512(<16 x i32> [[TMP1]], <4 x i32> [[A1:%.*]])
+; CHECK-NEXT:    [[TMP10:%.*]] = or <16 x i32> [[TMP9]], [[TMP8]]
+; CHECK-NEXT:    [[RES:%.*]] = call <16 x i32> @llvm.x86.avx512.psrl.d.512(<16 x i32> [[A0:%.*]], <4 x i32> [[A1]])
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast i16 [[TMP3]] to <16 x i1>
+; CHECK-NEXT:    [[MASK_CAST:%.*]] = bitcast i16 [[MASK:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP12:%.*]] = select <16 x i1> [[MASK_CAST]], <16 x i32> [[TMP10]], <16 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP13:%.*]] = xor <16 x i32> [[RES]], zeroinitializer
+; CHECK-NEXT:    [[TMP14:%.*]] = or <16 x i32> [[TMP13]], [[TMP10]]
+; CHECK-NEXT:    [[TMP15:%.*]] = or <16 x i32> [[TMP14]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP11]], <16 x i32> [[TMP15]], <16 x i32> [[TMP12]]
+; CHECK-NEXT:    [[RES2:%.*]] = select <16 x i1> [[MASK_CAST]], <16 x i32> [[RES]], <16 x i32> zeroinitializer
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i32> [[RES2]]
+;
+  %res = call <16 x i32> @llvm.x86.avx512.psrl.d.512(<16 x i32> %a0, <4 x i32> %a1) ; <<16 x i32>> [#uses=1]
+  %mask.cast = bitcast i16 %mask to <16 x i1>
+  %res2 = select <16 x i1> %mask.cast, <16 x i32> %res, <16 x i32> zeroinitializer
+  ret <16 x i32> %res2
+}
+declare <16 x i32> @llvm.x86.avx512.psrl.d.512(<16 x i32>, <4 x i32>) nounwind readnone
+
+
+define <8 x i64> @test_x86_avx512_psrl_q_512(<8 x i64> %a0, <2 x i64> %a1) #0 {
+; CHECK-LABEL: @test_x86_avx512_psrl_q_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <2 x i64> [[TMP2]] to i128
+; CHECK-NEXT:    [[TMP4:%.*]] = trunc i128 [[TMP3]] to i64
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp ne i64 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = sext i1 [[TMP5]] to i512
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast i512 [[TMP6]] to <8 x i64>
+; CHECK-NEXT:    [[TMP8:%.*]] = call <8 x i64> @llvm.x86.avx512.psrl.q.512(<8 x i64> [[TMP1]], <2 x i64> [[A1:%.*]])
+; CHECK-NEXT:    [[TMP9:%.*]] = or <8 x i64> [[TMP8]], [[TMP7]]
+; CHECK-NEXT:    [[RES:%.*]] = call <8 x i64> @llvm.x86.avx512.psrl.q.512(<8 x i64> [[A0:%.*]], <2 x i64> [[A1]])
+; CHECK-NEXT:    store <8 x i64> [[TMP9]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i64> [[RES]]
+;
+  %res = call <8 x i64> @llvm.x86.avx512.psrl.q.512(<8 x i64> %a0, <2 x i64> %a1) ; <<8 x i64>> [#uses=1]
+  ret <8 x i64> %res
+}
+define <8 x i64> @test_x86_avx512_mask_psrl_q_512(<8 x i64> %a0, <2 x i64> %a1, <8 x i64> %passthru, i8 %mask) #0 {
+; CHECK-LABEL: @test_x86_avx512_mask_psrl_q_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 144) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 80) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <2 x i64> [[TMP2]] to i128
+; CHECK-NEXT:    [[TMP6:%.*]] = trunc i128 [[TMP5]] to i64
+; CHECK-NEXT:    [[TMP7:%.*]] = icmp ne i64 [[TMP6]], 0
+; CHECK-NEXT:    [[TMP8:%.*]] = sext i1 [[TMP7]] to i512
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast i512 [[TMP8]] to <8 x i64>
+; CHECK-NEXT:    [[TMP10:%.*]] = call <8 x i64> @llvm.x86.avx512.psrl.q.512(<8 x i64> [[TMP1]], <2 x i64> [[A1:%.*]])
+; CHECK-NEXT:    [[TMP11:%.*]] = or <8 x i64> [[TMP10]], [[TMP9]]
+; CHECK-NEXT:    [[RES:%.*]] = call <8 x i64> @llvm.x86.avx512.psrl.q.512(<8 x i64> [[A0:%.*]], <2 x i64> [[A1]])
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast i8 [[TMP3]] to <8 x i1>
+; CHECK-NEXT:    [[MASK_CAST:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP13:%.*]] = select <8 x i1> [[MASK_CAST]], <8 x i64> [[TMP11]], <8 x i64> [[TMP4]]
+; CHECK-NEXT:    [[TMP14:%.*]] = xor <8 x i64> [[RES]], [[PASSTHRU:%.*]]
+; CHECK-NEXT:    [[TMP15:%.*]] = or <8 x i64> [[TMP14]], [[TMP11]]
+; CHECK-NEXT:    [[TMP16:%.*]] = or <8 x i64> [[TMP15]], [[TMP4]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP12]], <8 x i64> [[TMP16]], <8 x i64> [[TMP13]]
+; CHECK-NEXT:    [[RES2:%.*]] = select <8 x i1> [[MASK_CAST]], <8 x i64> [[RES]], <8 x i64> [[PASSTHRU]]
+; CHECK-NEXT:    store <8 x i64> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i64> [[RES2]]
+;
+  %res = call <8 x i64> @llvm.x86.avx512.psrl.q.512(<8 x i64> %a0, <2 x i64> %a1) ; <<8 x i64>> [#uses=1]
+  %mask.cast = bitcast i8 %mask to <8 x i1>
+  %res2 = select <8 x i1> %mask.cast, <8 x i64> %res, <8 x i64> %passthru
+  ret <8 x i64> %res2
+}
+define <8 x i64> @test_x86_avx512_maskz_psrl_q_512(<8 x i64> %a0, <2 x i64> %a1, i8 %mask) #0 {
+; CHECK-LABEL: @test_x86_avx512_maskz_psrl_q_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 80) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <2 x i64> [[TMP2]] to i128
+; CHECK-NEXT:    [[TMP5:%.*]] = trunc i128 [[TMP4]] to i64
+; CHECK-NEXT:    [[TMP6:%.*]] = icmp ne i64 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP7:%.*]] = sext i1 [[TMP6]] to i512
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast i512 [[TMP7]] to <8 x i64>
+; CHECK-NEXT:    [[TMP9:%.*]] = call <8 x i64> @llvm.x86.avx512.psrl.q.512(<8 x i64> [[TMP1]], <2 x i64> [[A1:%.*]])
+; CHECK-NEXT:    [[TMP10:%.*]] = or <8 x i64> [[TMP9]], [[TMP8]]
+; CHECK-NEXT:    [[RES:%.*]] = call <8 x i64> @llvm.x86.avx512.psrl.q.512(<8 x i64> [[A0:%.*]], <2 x i64> [[A1]])
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast i8 [[TMP3]] to <8 x i1>
+; CHECK-NEXT:    [[MASK_CAST:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP12:%.*]] = select <8 x i1> [[MASK_CAST]], <8 x i64> [[TMP10]], <8 x i64> zeroinitializer
+; CHECK-NEXT:    [[TMP13:%.*]] = xor <8 x i64> [[RES]], zeroinitializer
+; CHECK-NEXT:    [[TMP14:%.*]] = or <8 x i64> [[TMP13]], [[TMP10]]
+; CHECK-NEXT:    [[TMP15:%.*]] = or <8 x i64> [[TMP14]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP11]], <8 x i64> [[TMP15]], <8 x i64> [[TMP12]]
+; CHECK-NEXT:    [[RES2:%.*]] = select <8 x i1> [[MASK_CAST]], <8 x i64> [[RES]], <8 x i64> zeroinitializer
+; CHECK-NEXT:    store <8 x i64> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i64> [[RES2]]
+;
+  %res = call <8 x i64> @llvm.x86.avx512.psrl.q.512(<8 x i64> %a0, <2 x i64> %a1) ; <<8 x i64>> [#uses=1]
+  %mask.cast = bitcast i8 %mask to <8 x i1>
+  %res2 = select <8 x i1> %mask.cast, <8 x i64> %res, <8 x i64> zeroinitializer
+  ret <8 x i64> %res2
+}
+declare <8 x i64> @llvm.x86.avx512.psrl.q.512(<8 x i64>, <2 x i64>) nounwind readnone
+
+
+define <16 x i32> @test_x86_avx512_psrli_d_512(<16 x i32> %a0) #0 {
+; CHECK-LABEL: @test_x86_avx512_psrli_d_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = call <16 x i32> @llvm.x86.avx512.psrli.d.512(<16 x i32> [[TMP1]], i32 7)
+; CHECK-NEXT:    [[TMP3:%.*]] = or <16 x i32> [[TMP2]], zeroinitializer
+; CHECK-NEXT:    [[RES:%.*]] = call <16 x i32> @llvm.x86.avx512.psrli.d.512(<16 x i32> [[A0:%.*]], i32 7)
+; CHECK-NEXT:    store <16 x i32> [[TMP3]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i32> [[RES]]
+;
+  %res = call <16 x i32> @llvm.x86.avx512.psrli.d.512(<16 x i32> %a0, i32 7) ; <<16 x i32>> [#uses=1]
+  ret <16 x i32> %res
+}
+define <16 x i32> @test_x86_avx512_mask_psrli_d_512(<16 x i32> %a0, <16 x i32> %passthru, i16 %mask) #0 {
+; CHECK-LABEL: @test_x86_avx512_mask_psrli_d_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = call <16 x i32> @llvm.x86.avx512.psrli.d.512(<16 x i32> [[TMP1]], i32 7)
+; CHECK-NEXT:    [[TMP5:%.*]] = or <16 x i32> [[TMP4]], zeroinitializer
+; CHECK-NEXT:    [[RES:%.*]] = call <16 x i32> @llvm.x86.avx512.psrli.d.512(<16 x i32> [[A0:%.*]], i32 7)
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i16 [[TMP2]] to <16 x i1>
+; CHECK-NEXT:    [[MASK_CAST:%.*]] = bitcast i16 [[MASK:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP7:%.*]] = select <16 x i1> [[MASK_CAST]], <16 x i32> [[TMP5]], <16 x i32> [[TMP3]]
+; CHECK-NEXT:    [[TMP8:%.*]] = xor <16 x i32> [[RES]], [[PASSTHRU:%.*]]
+; CHECK-NEXT:    [[TMP9:%.*]] = or <16 x i32> [[TMP8]], [[TMP5]]
+; CHECK-NEXT:    [[TMP10:%.*]] = or <16 x i32> [[TMP9]], [[TMP3]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP6]], <16 x i32> [[TMP10]], <16 x i32> [[TMP7]]
+; CHECK-NEXT:    [[RES2:%.*]] = select <16 x i1> [[MASK_CAST]], <16 x i32> [[RES]], <16 x i32> [[PASSTHRU]]
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i32> [[RES2]]
+;
+  %res = call <16 x i32> @llvm.x86.avx512.psrli.d.512(<16 x i32> %a0, i32 7) ; <<16 x i32>> [#uses=1]
+  %mask.cast = bitcast i16 %mask to <16 x i1>
+  %res2 = select <16 x i1> %mask.cast, <16 x i32> %res, <16 x i32> %passthru
+  ret <16 x i32> %res2
+}
+define <16 x i32> @test_x86_avx512_maskz_psrli_d_512(<16 x i32> %a0, i16 %mask) #0 {
+; CHECK-LABEL: @test_x86_avx512_maskz_psrli_d_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = call <16 x i32> @llvm.x86.avx512.psrli.d.512(<16 x i32> [[TMP1]], i32 7)
+; CHECK-NEXT:    [[TMP4:%.*]] = or <16 x i32> [[TMP3]], zeroinitializer
+; CHECK-NEXT:    [[RES:%.*]] = call <16 x i32> @llvm.x86.avx512.psrli.d.512(<16 x i32> [[A0:%.*]], i32 7)
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i16 [[TMP2]] to <16 x i1>
+; CHECK-NEXT:    [[MASK_CAST:%.*]] = bitcast i16 [[MASK:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP6:%.*]] = select <16 x i1> [[MASK_CAST]], <16 x i32> [[TMP4]], <16 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP7:%.*]] = xor <16 x i32> [[RES]], zeroinitializer
+; CHECK-NEXT:    [[TMP8:%.*]] = or <16 x i32> [[TMP7]], [[TMP4]]
+; CHECK-NEXT:    [[TMP9:%.*]] = or <16 x i32> [[TMP8]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP5]], <16 x i32> [[TMP9]], <16 x i32> [[TMP6]]
+; CHECK-NEXT:    [[RES2:%.*]] = select <16 x i1> [[MASK_CAST]], <16 x i32> [[RES]], <16 x i32> zeroinitializer
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i32> [[RES2]]
+;
+  %res = call <16 x i32> @llvm.x86.avx512.psrli.d.512(<16 x i32> %a0, i32 7) ; <<16 x i32>> [#uses=1]
+  %mask.cast = bitcast i16 %mask to <16 x i1>
+  %res2 = select <16 x i1> %mask.cast, <16 x i32> %res, <16 x i32> zeroinitializer
+  ret <16 x i32> %res2
+}
+declare <16 x i32> @llvm.x86.avx512.psrli.d.512(<16 x i32>, i32) nounwind readnone
+
+
+define <8 x i64> @test_x86_avx512_psrli_q_512(<8 x i64> %a0) #0 {
+; CHECK-LABEL: @test_x86_avx512_psrli_q_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = call <8 x i64> @llvm.x86.avx512.psrli.q.512(<8 x i64> [[TMP1]], i32 7)
+; CHECK-NEXT:    [[TMP3:%.*]] = or <8 x i64> [[TMP2]], zeroinitializer
+; CHECK-NEXT:    [[RES:%.*]] = call <8 x i64> @llvm.x86.avx512.psrli.q.512(<8 x i64> [[A0:%.*]], i32 7)
+; CHECK-NEXT:    store <8 x i64> [[TMP3]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i64> [[RES]]
+;
+  %res = call <8 x i64> @llvm.x86.avx512.psrli.q.512(<8 x i64> %a0, i32 7) ; <<8 x i64>> [#uses=1]
+  ret <8 x i64> %res
+}
+define <8 x i64> @test_x86_avx512_mask_psrli_q_512(<8 x i64> %a0, <8 x i64> %passthru, i8 %mask) #0 {
+; CHECK-LABEL: @test_x86_avx512_mask_psrli_q_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = call <8 x i64> @llvm.x86.avx512.psrli.q.512(<8 x i64> [[TMP1]], i32 7)
+; CHECK-NEXT:    [[TMP5:%.*]] = or <8 x i64> [[TMP4]], zeroinitializer
+; CHECK-NEXT:    [[RES:%.*]] = call <8 x i64> @llvm.x86.avx512.psrli.q.512(<8 x i64> [[A0:%.*]], i32 7)
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i8 [[TMP2]] to <8 x i1>
+; CHECK-NEXT:    [[MASK_CAST:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP7:%.*]] = select <8 x i1> [[MASK_CAST]], <8 x i64> [[TMP5]], <8 x i64> [[TMP3]]
+; CHECK-NEXT:    [[TMP8:%.*]] = xor <8 x i64> [[RES]], [[PASSTHRU:%.*]]
+; CHECK-NEXT:    [[TMP9:%.*]] = or <8 x i64> [[TMP8]], [[TMP5]]
+; CHECK-NEXT:    [[TMP10:%.*]] = or <8 x i64> [[TMP9]], [[TMP3]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP6]], <8 x i64> [[TMP10]], <8 x i64> [[TMP7]]
+; CHECK-NEXT:    [[RES2:%.*]] = select <8 x i1> [[MASK_CAST]], <8 x i64> [[RES]], <8 x i64> [[PASSTHRU]]
+; CHECK-NEXT:    store <8 x i64> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i64> [[RES2]]
+;
+  %res = call <8 x i64> @llvm.x86.avx512.psrli.q.512(<8 x i64> %a0, i32 7) ; <<8 x i64>> [#uses=1]
+  %mask.cast = bitcast i8 %mask to <8 x i1>
+  %res2 = select <8 x i1> %mask.cast, <8 x i64> %res, <8 x i64> %passthru
+  ret <8 x i64> %res2
+}
+define <8 x i64> @test_x86_avx512_maskz_psrli_q_512(<8 x i64> %a0, i8 %mask) #0 {
+; CHECK-LABEL: @test_x86_avx512_maskz_psrli_q_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = call <8 x i64> @llvm.x86.avx512.psrli.q.512(<8 x i64> [[TMP1]], i32 7)
+; CHECK-NEXT:    [[TMP4:%.*]] = or <8 x i64> [[TMP3]], zeroinitializer
+; CHECK-NEXT:    [[RES:%.*]] = call <8 x i64> @llvm.x86.avx512.psrli.q.512(<8 x i64> [[A0:%.*]], i32 7)
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i8 [[TMP2]] to <8 x i1>
+; CHECK-NEXT:    [[MASK_CAST:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP6:%.*]] = select <8 x i1> [[MASK_CAST]], <8 x i64> [[TMP4]], <8 x i64> zeroinitializer
+; CHECK-NEXT:    [[TMP7:%.*]] = xor <8 x i64> [[RES]], zeroinitializer
+; CHECK-NEXT:    [[TMP8:%.*]] = or <8 x i64> [[TMP7]], [[TMP4]]
+; CHECK-NEXT:    [[TMP9:%.*]] = or <8 x i64> [[TMP8]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP5]], <8 x i64> [[TMP9]], <8 x i64> [[TMP6]]
+; CHECK-NEXT:    [[RES2:%.*]] = select <8 x i1> [[MASK_CAST]], <8 x i64> [[RES]], <8 x i64> zeroinitializer
+; CHECK-NEXT:    store <8 x i64> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i64> [[RES2]]
+;
+  %res = call <8 x i64> @llvm.x86.avx512.psrli.q.512(<8 x i64> %a0, i32 7) ; <<8 x i64>> [#uses=1]
+  %mask.cast = bitcast i8 %mask to <8 x i1>
+  %res2 = select <8 x i1> %mask.cast, <8 x i64> %res, <8 x i64> zeroinitializer
+  ret <8 x i64> %res2
+}
+declare <8 x i64> @llvm.x86.avx512.psrli.q.512(<8 x i64>, i32) nounwind readnone
+
+define <16 x i32> @test_x86_avx512_psllv_d_512(<16 x i32> %a0, <16 x i32> %a1) #0 {
+; CHECK-LABEL: @test_x86_avx512_psllv_d_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp ne <16 x i32> [[TMP2]], zeroinitializer
+; CHECK-NEXT:    [[TMP4:%.*]] = sext <16 x i1> [[TMP3]] to <16 x i32>
+; CHECK-NEXT:    [[TMP5:%.*]] = call <16 x i32> @llvm.x86.avx512.psllv.d.512(<16 x i32> [[TMP1]], <16 x i32> [[A1:%.*]])
+; CHECK-NEXT:    [[TMP6:%.*]] = or <16 x i32> [[TMP5]], [[TMP4]]
+; CHECK-NEXT:    [[RES:%.*]] = call <16 x i32> @llvm.x86.avx512.psllv.d.512(<16 x i32> [[A0:%.*]], <16 x i32> [[A1]])
+; CHECK-NEXT:    store <16 x i32> [[TMP6]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i32> [[RES]]
+;
+  %res = call <16 x i32> @llvm.x86.avx512.psllv.d.512(<16 x i32> %a0, <16 x i32> %a1)
+  ret <16 x i32> %res
+}
+
+define <16 x i32> @test_x86_avx512_psllv_d_512_const() #0 {
+; CHECK-LABEL: @test_x86_avx512_psllv_d_512_const(
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP1:%.*]] = call <16 x i32> @llvm.x86.avx512.psllv.d.512(<16 x i32> zeroinitializer, <16 x i32> <i32 1, i32 0, i32 33, i32 -1, i32 2, i32 0, i32 34, i32 -2, i32 3, i32 0, i32 35, i32 -1, i32 4, i32 0, i32 36, i32 -3>)
+; CHECK-NEXT:    [[TMP2:%.*]] = or <16 x i32> [[TMP1]], zeroinitializer
+; CHECK-NEXT:    [[RES0:%.*]] = call <16 x i32> @llvm.x86.avx512.psllv.d.512(<16 x i32> <i32 2, i32 9, i32 0, i32 -1, i32 3, i32 7, i32 -1, i32 0, i32 4, i32 5, i32 -2, i32 0, i32 5, i32 3, i32 -3, i32 0>, <16 x i32> <i32 1, i32 0, i32 33, i32 -1, i32 2, i32 0, i32 34, i32 -2, i32 3, i32 0, i32 35, i32 -1, i32 4, i32 0, i32 36, i32 -3>)
+; CHECK-NEXT:    [[TMP3:%.*]] = call <16 x i32> @llvm.x86.avx512.psllv.d.512(<16 x i32> zeroinitializer, <16 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 -1>)
+; CHECK-NEXT:    [[TMP4:%.*]] = or <16 x i32> [[TMP3]], zeroinitializer
+; CHECK-NEXT:    [[RES1:%.*]] = call <16 x i32> @llvm.x86.avx512.psllv.d.512(<16 x i32> <i32 4, i32 4, i32 4, i32 4, i32 4, i32 4, i32 4, i32 4, i32 4, i32 4, i32 4, i32 4, i32 4, i32 4, i32 4, i32 -1>, <16 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 -1>)
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <16 x i32> [[TMP2]], [[TMP4]]
+; CHECK-NEXT:    [[RES2:%.*]] = add <16 x i32> [[RES0]], [[RES1]]
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i32> [[RES2]]
+;
+  %res0 = call <16 x i32> @llvm.x86.avx512.psllv.d.512(<16 x i32> <i32 2, i32 9, i32 0, i32 -1, i32 3, i32 7, i32 -1, i32 0, i32 4, i32 5, i32 -2, i32 0, i32 5, i32 3, i32 -3, i32 0>, <16 x i32> <i32 1, i32 0, i32 33, i32 -1,i32 2, i32 0, i32 34, i32 -2, i32 3, i32 0, i32 35, i32 -1, i32 4, i32 0, i32 36, i32 -3>)
+  %res1 = call <16 x i32> @llvm.x86.avx512.psllv.d.512(<16 x i32> <i32 4, i32 4, i32 4, i32 4, i32 4, i32 4, i32 4, i32 4, i32 4, i32 4, i32 4, i32 4, i32 4, i32 4, i32 4, i32 -1>, <16 x i32> <i32 1, i32 1, i32 1,  i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1,  i32 1, i32 1, i32 1, i32 1, i32 -1>)
+  %res2 = add <16 x i32> %res0, %res1
+  ret <16 x i32> %res2
+}
+
+define <16 x i32> @test_x86_avx512_mask_psllv_d_512(<16 x i32> %a0, <16 x i32> %a1, <16 x i32> %a2, i16 %mask) #0 {
+; CHECK-LABEL: @test_x86_avx512_mask_psllv_d_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp ne <16 x i32> [[TMP2]], zeroinitializer
+; CHECK-NEXT:    [[TMP6:%.*]] = sext <16 x i1> [[TMP5]] to <16 x i32>
+; CHECK-NEXT:    [[TMP7:%.*]] = call <16 x i32> @llvm.x86.avx512.psllv.d.512(<16 x i32> [[TMP1]], <16 x i32> [[A1:%.*]])
+; CHECK-NEXT:    [[TMP8:%.*]] = or <16 x i32> [[TMP7]], [[TMP6]]
+; CHECK-NEXT:    [[RES:%.*]] = call <16 x i32> @llvm.x86.avx512.psllv.d.512(<16 x i32> [[A0:%.*]], <16 x i32> [[A1]])
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast i16 [[TMP3]] to <16 x i1>
+; CHECK-NEXT:    [[MASK_CAST:%.*]] = bitcast i16 [[MASK:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP10:%.*]] = select <16 x i1> [[MASK_CAST]], <16 x i32> [[TMP8]], <16 x i32> [[TMP4]]
+; CHECK-NEXT:    [[TMP11:%.*]] = xor <16 x i32> [[RES]], [[A2:%.*]]
+; CHECK-NEXT:    [[TMP12:%.*]] = or <16 x i32> [[TMP11]], [[TMP8]]
+; CHECK-NEXT:    [[TMP13:%.*]] = or <16 x i32> [[TMP12]], [[TMP4]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP9]], <16 x i32> [[TMP13]], <16 x i32> [[TMP10]]
+; CHECK-NEXT:    [[RES2:%.*]] = select <16 x i1> [[MASK_CAST]], <16 x i32> [[RES]], <16 x i32> [[A2]]
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i32> [[RES2]]
+;
+  %res = call <16 x i32> @llvm.x86.avx512.psllv.d.512(<16 x i32> %a0, <16 x i32> %a1)
+  %mask.cast = bitcast i16 %mask to <16 x i1>
+  %res2 = select <16 x i1> %mask.cast, <16 x i32> %res, <16 x i32> %a2
+  ret <16 x i32> %res2
+}
+
+define <16 x i32> @test_x86_avx512_maskz_psllv_d_512(<16 x i32> %a0, <16 x i32> %a1, i16 %mask) #0 {
+; CHECK-LABEL: @test_x86_avx512_maskz_psllv_d_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = icmp ne <16 x i32> [[TMP2]], zeroinitializer
+; CHECK-NEXT:    [[TMP5:%.*]] = sext <16 x i1> [[TMP4]] to <16 x i32>
+; CHECK-NEXT:    [[TMP6:%.*]] = call <16 x i32> @llvm.x86.avx512.psllv.d.512(<16 x i32> [[TMP1]], <16 x i32> [[A1:%.*]])
+; CHECK-NEXT:    [[TMP7:%.*]] = or <16 x i32> [[TMP6]], [[TMP5]]
+; CHECK-NEXT:    [[RES:%.*]] = call <16 x i32> @llvm.x86.avx512.psllv.d.512(<16 x i32> [[A0:%.*]], <16 x i32> [[A1]])
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast i16 [[TMP3]] to <16 x i1>
+; CHECK-NEXT:    [[MASK_CAST:%.*]] = bitcast i16 [[MASK:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP9:%.*]] = select <16 x i1> [[MASK_CAST]], <16 x i32> [[TMP7]], <16 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP10:%.*]] = xor <16 x i32> [[RES]], zeroinitializer
+; CHECK-NEXT:    [[TMP11:%.*]] = or <16 x i32> [[TMP10]], [[TMP7]]
+; CHECK-NEXT:    [[TMP12:%.*]] = or <16 x i32> [[TMP11]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP8]], <16 x i32> [[TMP12]], <16 x i32> [[TMP9]]
+; CHECK-NEXT:    [[RES2:%.*]] = select <16 x i1> [[MASK_CAST]], <16 x i32> [[RES]], <16 x i32> zeroinitializer
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i32> [[RES2]]
+;
+  %res = call <16 x i32> @llvm.x86.avx512.psllv.d.512(<16 x i32> %a0, <16 x i32> %a1)
+  %mask.cast = bitcast i16 %mask to <16 x i1>
+  %res2 = select <16 x i1> %mask.cast, <16 x i32> %res, <16 x i32> zeroinitializer
+  ret <16 x i32> %res2
+}
+
+declare <16 x i32> @llvm.x86.avx512.psllv.d.512(<16 x i32>, <16 x i32>) nounwind readnone
+
+define <8 x i64> @test_x86_avx512_psllv_q_512(<8 x i64> %a0, <8 x i64> %a1) #0 {
+; CHECK-LABEL: @test_x86_avx512_psllv_q_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp ne <8 x i64> [[TMP2]], zeroinitializer
+; CHECK-NEXT:    [[TMP4:%.*]] = sext <8 x i1> [[TMP3]] to <8 x i64>
+; CHECK-NEXT:    [[TMP5:%.*]] = call <8 x i64> @llvm.x86.avx512.psllv.q.512(<8 x i64> [[TMP1]], <8 x i64> [[A1:%.*]])
+; CHECK-NEXT:    [[TMP6:%.*]] = or <8 x i64> [[TMP5]], [[TMP4]]
+; CHECK-NEXT:    [[RES:%.*]] = call <8 x i64> @llvm.x86.avx512.psllv.q.512(<8 x i64> [[A0:%.*]], <8 x i64> [[A1]])
+; CHECK-NEXT:    store <8 x i64> [[TMP6]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i64> [[RES]]
+;
+  %res = call <8 x i64> @llvm.x86.avx512.psllv.q.512(<8 x i64> %a0, <8 x i64> %a1)
+  ret <8 x i64> %res
+}
+
+define <8 x i64> @test_x86_avx512_psllv_q_512_const() #0 {
+; CHECK-LABEL: @test_x86_avx512_psllv_q_512_const(
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP1:%.*]] = call <8 x i64> @llvm.x86.avx512.psllv.q.512(<8 x i64> zeroinitializer, <8 x i64> <i64 1, i64 0, i64 33, i64 -1, i64 2, i64 0, i64 34, i64 -2>)
+; CHECK-NEXT:    [[TMP2:%.*]] = or <8 x i64> [[TMP1]], zeroinitializer
+; CHECK-NEXT:    [[RES0:%.*]] = call <8 x i64> @llvm.x86.avx512.psllv.q.512(<8 x i64> <i64 2, i64 9, i64 0, i64 -1, i64 3, i64 7, i64 -1, i64 0>, <8 x i64> <i64 1, i64 0, i64 33, i64 -1, i64 2, i64 0, i64 34, i64 -2>)
+; CHECK-NEXT:    [[TMP3:%.*]] = call <8 x i64> @llvm.x86.avx512.psllv.q.512(<8 x i64> zeroinitializer, <8 x i64> <i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 -1>)
+; CHECK-NEXT:    [[TMP4:%.*]] = or <8 x i64> [[TMP3]], zeroinitializer
+; CHECK-NEXT:    [[RES1:%.*]] = call <8 x i64> @llvm.x86.avx512.psllv.q.512(<8 x i64> <i64 4, i64 4, i64 4, i64 4, i64 4, i64 4, i64 4, i64 -1>, <8 x i64> <i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 -1>)
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <8 x i64> [[TMP2]], [[TMP4]]
+; CHECK-NEXT:    [[RES2:%.*]] = add <8 x i64> [[RES0]], [[RES1]]
+; CHECK-NEXT:    store <8 x i64> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i64> [[RES2]]
+;
+  %res0 = call <8 x i64> @llvm.x86.avx512.psllv.q.512(<8 x i64> <i64 2, i64 9, i64 0, i64 -1, i64 3, i64 7, i64 -1, i64 0>, <8 x i64> <i64 1, i64 0, i64 33, i64 -1,i64 2, i64 0, i64 34, i64 -2>)
+  %res1 = call <8 x i64> @llvm.x86.avx512.psllv.q.512(<8 x i64> <i64 4, i64 4, i64 4, i64 4, i64 4, i64 4, i64 4, i64 -1>, <8 x i64> <i64 1, i64 1, i64 1, i64 1,  i64 1, i64 1, i64 1, i64 -1>)
+  %res2 = add <8 x i64> %res0, %res1
+  ret <8 x i64> %res2
+}
+
+define <8 x i64> @test_x86_avx512_mask_psllv_q_512(<8 x i64> %a0, <8 x i64> %a1, <8 x i64> %a2, i8 %mask) #0 {
+; CHECK-LABEL: @test_x86_avx512_mask_psllv_q_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp ne <8 x i64> [[TMP2]], zeroinitializer
+; CHECK-NEXT:    [[TMP6:%.*]] = sext <8 x i1> [[TMP5]] to <8 x i64>
+; CHECK-NEXT:    [[TMP7:%.*]] = call <8 x i64> @llvm.x86.avx512.psllv.q.512(<8 x i64> [[TMP1]], <8 x i64> [[A1:%.*]])
+; CHECK-NEXT:    [[TMP8:%.*]] = or <8 x i64> [[TMP7]], [[TMP6]]
+; CHECK-NEXT:    [[RES:%.*]] = call <8 x i64> @llvm.x86.avx512.psllv.q.512(<8 x i64> [[A0:%.*]], <8 x i64> [[A1]])
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast i8 [[TMP3]] to <8 x i1>
+; CHECK-NEXT:    [[MASK_CAST:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP10:%.*]] = select <8 x i1> [[MASK_CAST]], <8 x i64> [[TMP8]], <8 x i64> [[TMP4]]
+; CHECK-NEXT:    [[TMP11:%.*]] = xor <8 x i64> [[RES]], [[A2:%.*]]
+; CHECK-NEXT:    [[TMP12:%.*]] = or <8 x i64> [[TMP11]], [[TMP8]]
+; CHECK-NEXT:    [[TMP13:%.*]] = or <8 x i64> [[TMP12]], [[TMP4]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP9]], <8 x i64> [[TMP13]], <8 x i64> [[TMP10]]
+; CHECK-NEXT:    [[RES2:%.*]] = select <8 x i1> [[MASK_CAST]], <8 x i64> [[RES]], <8 x i64> [[A2]]
+; CHECK-NEXT:    store <8 x i64> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i64> [[RES2]]
+;
+  %res = call <8 x i64> @llvm.x86.avx512.psllv.q.512(<8 x i64> %a0, <8 x i64> %a1)
+  %mask.cast = bitcast i8 %mask to <8 x i1>
+  %res2 = select <8 x i1> %mask.cast, <8 x i64> %res, <8 x i64> %a2
+  ret <8 x i64> %res2
+}
+
+define <8 x i64> @test_x86_avx512_maskz_psllv_q_512(<8 x i64> %a0, <8 x i64> %a1, i8 %mask) #0 {
+; CHECK-LABEL: @test_x86_avx512_maskz_psllv_q_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = icmp ne <8 x i64> [[TMP2]], zeroinitializer
+; CHECK-NEXT:    [[TMP5:%.*]] = sext <8 x i1> [[TMP4]] to <8 x i64>
+; CHECK-NEXT:    [[TMP6:%.*]] = call <8 x i64> @llvm.x86.avx512.psllv.q.512(<8 x i64> [[TMP1]], <8 x i64> [[A1:%.*]])
+; CHECK-NEXT:    [[TMP7:%.*]] = or <8 x i64> [[TMP6]], [[TMP5]]
+; CHECK-NEXT:    [[RES:%.*]] = call <8 x i64> @llvm.x86.avx512.psllv.q.512(<8 x i64> [[A0:%.*]], <8 x i64> [[A1]])
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast i8 [[TMP3]] to <8 x i1>
+; CHECK-NEXT:    [[MASK_CAST:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP9:%.*]] = select <8 x i1> [[MASK_CAST]], <8 x i64> [[TMP7]], <8 x i64> zeroinitializer
+; CHECK-NEXT:    [[TMP10:%.*]] = xor <8 x i64> [[RES]], zeroinitializer
+; CHECK-NEXT:    [[TMP11:%.*]] = or <8 x i64> [[TMP10]], [[TMP7]]
+; CHECK-NEXT:    [[TMP12:%.*]] = or <8 x i64> [[TMP11]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP8]], <8 x i64> [[TMP12]], <8 x i64> [[TMP9]]
+; CHECK-NEXT:    [[RES2:%.*]] = select <8 x i1> [[MASK_CAST]], <8 x i64> [[RES]], <8 x i64> zeroinitializer
+; CHECK-NEXT:    store <8 x i64> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i64> [[RES2]]
+;
+  %res = call <8 x i64> @llvm.x86.avx512.psllv.q.512(<8 x i64> %a0, <8 x i64> %a1)
+  %mask.cast = bitcast i8 %mask to <8 x i1>
+  %res2 = select <8 x i1> %mask.cast, <8 x i64> %res, <8 x i64> zeroinitializer
+  ret <8 x i64> %res2
+}
+
+declare <8 x i64> @llvm.x86.avx512.psllv.q.512(<8 x i64>, <8 x i64>) nounwind readnone
+
+define <16 x i32> @test_x86_avx512_psrav_d_512(<16 x i32> %a0, <16 x i32> %a1) #0 {
+; CHECK-LABEL: @test_x86_avx512_psrav_d_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp ne <16 x i32> [[TMP2]], zeroinitializer
+; CHECK-NEXT:    [[TMP4:%.*]] = sext <16 x i1> [[TMP3]] to <16 x i32>
+; CHECK-NEXT:    [[TMP5:%.*]] = call <16 x i32> @llvm.x86.avx512.psrav.d.512(<16 x i32> [[TMP1]], <16 x i32> [[A1:%.*]])
+; CHECK-NEXT:    [[TMP6:%.*]] = or <16 x i32> [[TMP5]], [[TMP4]]
+; CHECK-NEXT:    [[RES:%.*]] = call <16 x i32> @llvm.x86.avx512.psrav.d.512(<16 x i32> [[A0:%.*]], <16 x i32> [[A1]])
+; CHECK-NEXT:    store <16 x i32> [[TMP6]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i32> [[RES]]
+;
+  %res = call <16 x i32> @llvm.x86.avx512.psrav.d.512(<16 x i32> %a0, <16 x i32> %a1)
+  ret <16 x i32> %res
+}
+
+define <16 x i32> @test_x86_avx512_mask_psrav_d_512(<16 x i32> %a0, <16 x i32> %a1, <16 x i32> %a2, i16 %mask) #0 {
+; CHECK-LABEL: @test_x86_avx512_mask_psrav_d_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp ne <16 x i32> [[TMP2]], zeroinitializer
+; CHECK-NEXT:    [[TMP6:%.*]] = sext <16 x i1> [[TMP5]] to <16 x i32>
+; CHECK-NEXT:    [[TMP7:%.*]] = call <16 x i32> @llvm.x86.avx512.psrav.d.512(<16 x i32> [[TMP1]], <16 x i32> [[A1:%.*]])
+; CHECK-NEXT:    [[TMP8:%.*]] = or <16 x i32> [[TMP7]], [[TMP6]]
+; CHECK-NEXT:    [[RES:%.*]] = call <16 x i32> @llvm.x86.avx512.psrav.d.512(<16 x i32> [[A0:%.*]], <16 x i32> [[A1]])
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast i16 [[TMP3]] to <16 x i1>
+; CHECK-NEXT:    [[MASK_CAST:%.*]] = bitcast i16 [[MASK:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP10:%.*]] = select <16 x i1> [[MASK_CAST]], <16 x i32> [[TMP8]], <16 x i32> [[TMP4]]
+; CHECK-NEXT:    [[TMP11:%.*]] = xor <16 x i32> [[RES]], [[A2:%.*]]
+; CHECK-NEXT:    [[TMP12:%.*]] = or <16 x i32> [[TMP11]], [[TMP8]]
+; CHECK-NEXT:    [[TMP13:%.*]] = or <16 x i32> [[TMP12]], [[TMP4]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP9]], <16 x i32> [[TMP13]], <16 x i32> [[TMP10]]
+; CHECK-NEXT:    [[RES2:%.*]] = select <16 x i1> [[MASK_CAST]], <16 x i32> [[RES]], <16 x i32> [[A2]]
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i32> [[RES2]]
+;
+  %res = call <16 x i32> @llvm.x86.avx512.psrav.d.512(<16 x i32> %a0, <16 x i32> %a1)
+  %mask.cast = bitcast i16 %mask to <16 x i1>
+  %res2 = select <16 x i1> %mask.cast, <16 x i32> %res, <16 x i32> %a2
+  ret <16 x i32> %res2
+}
+
+define <16 x i32> @test_x86_avx512_maskz_psrav_d_512(<16 x i32> %a0, <16 x i32> %a1, i16 %mask) #0 {
+; CHECK-LABEL: @test_x86_avx512_maskz_psrav_d_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = icmp ne <16 x i32> [[TMP2]], zeroinitializer
+; CHECK-NEXT:    [[TMP5:%.*]] = sext <16 x i1> [[TMP4]] to <16 x i32>
+; CHECK-NEXT:    [[TMP6:%.*]] = call <16 x i32> @llvm.x86.avx512.psrav.d.512(<16 x i32> [[TMP1]], <16 x i32> [[A1:%.*]])
+; CHECK-NEXT:    [[TMP7:%.*]] = or <16 x i32> [[TMP6]], [[TMP5]]
+; CHECK-NEXT:    [[RES:%.*]] = call <16 x i32> @llvm.x86.avx512.psrav.d.512(<16 x i32> [[A0:%.*]], <16 x i32> [[A1]])
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast i16 [[TMP3]] to <16 x i1>
+; CHECK-NEXT:    [[MASK_CAST:%.*]] = bitcast i16 [[MASK:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP9:%.*]] = select <16 x i1> [[MASK_CAST]], <16 x i32> [[TMP7]], <16 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP10:%.*]] = xor <16 x i32> [[RES]], zeroinitializer
+; CHECK-NEXT:    [[TMP11:%.*]] = or <16 x i32> [[TMP10]], [[TMP7]]
+; CHECK-NEXT:    [[TMP12:%.*]] = or <16 x i32> [[TMP11]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP8]], <16 x i32> [[TMP12]], <16 x i32> [[TMP9]]
+; CHECK-NEXT:    [[RES2:%.*]] = select <16 x i1> [[MASK_CAST]], <16 x i32> [[RES]], <16 x i32> zeroinitializer
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i32> [[RES2]]
+;
+  %res = call <16 x i32> @llvm.x86.avx512.psrav.d.512(<16 x i32> %a0, <16 x i32> %a1)
+  %mask.cast = bitcast i16 %mask to <16 x i1>
+  %res2 = select <16 x i1> %mask.cast, <16 x i32> %res, <16 x i32> zeroinitializer
+  ret <16 x i32> %res2
+}
+
+declare <16 x i32> @llvm.x86.avx512.psrav.d.512(<16 x i32>, <16 x i32>) nounwind readnone
+
+define <8 x i64> @test_x86_avx512_psrav_q_512(<8 x i64> %a0, <8 x i64> %a1) #0 {
+; CHECK-LABEL: @test_x86_avx512_psrav_q_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp ne <8 x i64> [[TMP2]], zeroinitializer
+; CHECK-NEXT:    [[TMP4:%.*]] = sext <8 x i1> [[TMP3]] to <8 x i64>
+; CHECK-NEXT:    [[TMP5:%.*]] = call <8 x i64> @llvm.x86.avx512.psrav.q.512(<8 x i64> [[TMP1]], <8 x i64> [[A1:%.*]])
+; CHECK-NEXT:    [[TMP6:%.*]] = or <8 x i64> [[TMP5]], [[TMP4]]
+; CHECK-NEXT:    [[RES:%.*]] = call <8 x i64> @llvm.x86.avx512.psrav.q.512(<8 x i64> [[A0:%.*]], <8 x i64> [[A1]])
+; CHECK-NEXT:    store <8 x i64> [[TMP6]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i64> [[RES]]
+;
+  %res = call <8 x i64> @llvm.x86.avx512.psrav.q.512(<8 x i64> %a0, <8 x i64> %a1)
+  ret <8 x i64> %res
+}
+
+define <8 x i64> @test_x86_avx512_mask_psrav_q_512(<8 x i64> %a0, <8 x i64> %a1, <8 x i64> %a2, i8 %mask) #0 {
+; CHECK-LABEL: @test_x86_avx512_mask_psrav_q_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp ne <8 x i64> [[TMP2]], zeroinitializer
+; CHECK-NEXT:    [[TMP6:%.*]] = sext <8 x i1> [[TMP5]] to <8 x i64>
+; CHECK-NEXT:    [[TMP7:%.*]] = call <8 x i64> @llvm.x86.avx512.psrav.q.512(<8 x i64> [[TMP1]], <8 x i64> [[A1:%.*]])
+; CHECK-NEXT:    [[TMP8:%.*]] = or <8 x i64> [[TMP7]], [[TMP6]]
+; CHECK-NEXT:    [[RES:%.*]] = call <8 x i64> @llvm.x86.avx512.psrav.q.512(<8 x i64> [[A0:%.*]], <8 x i64> [[A1]])
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast i8 [[TMP3]] to <8 x i1>
+; CHECK-NEXT:    [[MASK_CAST:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP10:%.*]] = select <8 x i1> [[MASK_CAST]], <8 x i64> [[TMP8]], <8 x i64> [[TMP4]]
+; CHECK-NEXT:    [[TMP11:%.*]] = xor <8 x i64> [[RES]], [[A2:%.*]]
+; CHECK-NEXT:    [[TMP12:%.*]] = or <8 x i64> [[TMP11]], [[TMP8]]
+; CHECK-NEXT:    [[TMP13:%.*]] = or <8 x i64> [[TMP12]], [[TMP4]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP9]], <8 x i64> [[TMP13]], <8 x i64> [[TMP10]]
+; CHECK-NEXT:    [[RES2:%.*]] = select <8 x i1> [[MASK_CAST]], <8 x i64> [[RES]], <8 x i64> [[A2]]
+; CHECK-NEXT:    store <8 x i64> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i64> [[RES2]]
+;
+  %res = call <8 x i64> @llvm.x86.avx512.psrav.q.512(<8 x i64> %a0, <8 x i64> %a1)
+  %mask.cast = bitcast i8 %mask to <8 x i1>
+  %res2 = select <8 x i1> %mask.cast, <8 x i64> %res, <8 x i64> %a2
+  ret <8 x i64> %res2
+}
+
+define <8 x i64> @test_x86_avx512_maskz_psrav_q_512(<8 x i64> %a0, <8 x i64> %a1, i8 %mask) #0 {
+; CHECK-LABEL: @test_x86_avx512_maskz_psrav_q_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = icmp ne <8 x i64> [[TMP2]], zeroinitializer
+; CHECK-NEXT:    [[TMP5:%.*]] = sext <8 x i1> [[TMP4]] to <8 x i64>
+; CHECK-NEXT:    [[TMP6:%.*]] = call <8 x i64> @llvm.x86.avx512.psrav.q.512(<8 x i64> [[TMP1]], <8 x i64> [[A1:%.*]])
+; CHECK-NEXT:    [[TMP7:%.*]] = or <8 x i64> [[TMP6]], [[TMP5]]
+; CHECK-NEXT:    [[RES:%.*]] = call <8 x i64> @llvm.x86.avx512.psrav.q.512(<8 x i64> [[A0:%.*]], <8 x i64> [[A1]])
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast i8 [[TMP3]] to <8 x i1>
+; CHECK-NEXT:    [[MASK_CAST:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP9:%.*]] = select <8 x i1> [[MASK_CAST]], <8 x i64> [[TMP7]], <8 x i64> zeroinitializer
+; CHECK-NEXT:    [[TMP10:%.*]] = xor <8 x i64> [[RES]], zeroinitializer
+; CHECK-NEXT:    [[TMP11:%.*]] = or <8 x i64> [[TMP10]], [[TMP7]]
+; CHECK-NEXT:    [[TMP12:%.*]] = or <8 x i64> [[TMP11]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP8]], <8 x i64> [[TMP12]], <8 x i64> [[TMP9]]
+; CHECK-NEXT:    [[RES2:%.*]] = select <8 x i1> [[MASK_CAST]], <8 x i64> [[RES]], <8 x i64> zeroinitializer
+; CHECK-NEXT:    store <8 x i64> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i64> [[RES2]]
+;
+  %res = call <8 x i64> @llvm.x86.avx512.psrav.q.512(<8 x i64> %a0, <8 x i64> %a1)
+  %mask.cast = bitcast i8 %mask to <8 x i1>
+  %res2 = select <8 x i1> %mask.cast, <8 x i64> %res, <8 x i64> zeroinitializer
+  ret <8 x i64> %res2
+}
+
+declare <8 x i64> @llvm.x86.avx512.psrav.q.512(<8 x i64>, <8 x i64>) nounwind readnone
+
+define <16 x i32> @test_x86_avx512_psrlv_d_512(<16 x i32> %a0, <16 x i32> %a1) #0 {
+; CHECK-LABEL: @test_x86_avx512_psrlv_d_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp ne <16 x i32> [[TMP2]], zeroinitializer
+; CHECK-NEXT:    [[TMP4:%.*]] = sext <16 x i1> [[TMP3]] to <16 x i32>
+; CHECK-NEXT:    [[TMP5:%.*]] = call <16 x i32> @llvm.x86.avx512.psrlv.d.512(<16 x i32> [[TMP1]], <16 x i32> [[A1:%.*]])
+; CHECK-NEXT:    [[TMP6:%.*]] = or <16 x i32> [[TMP5]], [[TMP4]]
+; CHECK-NEXT:    [[RES:%.*]] = call <16 x i32> @llvm.x86.avx512.psrlv.d.512(<16 x i32> [[A0:%.*]], <16 x i32> [[A1]])
+; CHECK-NEXT:    store <16 x i32> [[TMP6]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i32> [[RES]]
+;
+  %res = call <16 x i32> @llvm.x86.avx512.psrlv.d.512(<16 x i32> %a0, <16 x i32> %a1)
+  ret <16 x i32> %res
+}
+
+define <16 x i32> @test_x86_avx512_psrlv_d_512_const() #0 {
+; CHECK-LABEL: @test_x86_avx512_psrlv_d_512_const(
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP1:%.*]] = call <16 x i32> @llvm.x86.avx512.psrlv.d.512(<16 x i32> zeroinitializer, <16 x i32> <i32 1, i32 0, i32 33, i32 -1, i32 2, i32 0, i32 34, i32 -2, i32 3, i32 0, i32 35, i32 -1, i32 4, i32 0, i32 36, i32 -3>)
+; CHECK-NEXT:    [[TMP2:%.*]] = or <16 x i32> [[TMP1]], zeroinitializer
+; CHECK-NEXT:    [[RES0:%.*]] = call <16 x i32> @llvm.x86.avx512.psrlv.d.512(<16 x i32> <i32 2, i32 9, i32 0, i32 -1, i32 3, i32 7, i32 -1, i32 0, i32 4, i32 5, i32 -2, i32 0, i32 5, i32 3, i32 -3, i32 0>, <16 x i32> <i32 1, i32 0, i32 33, i32 -1, i32 2, i32 0, i32 34, i32 -2, i32 3, i32 0, i32 35, i32 -1, i32 4, i32 0, i32 36, i32 -3>)
+; CHECK-NEXT:    [[TMP3:%.*]] = call <16 x i32> @llvm.x86.avx512.psrlv.d.512(<16 x i32> zeroinitializer, <16 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 -1>)
+; CHECK-NEXT:    [[TMP4:%.*]] = or <16 x i32> [[TMP3]], zeroinitializer
+; CHECK-NEXT:    [[RES1:%.*]] = call <16 x i32> @llvm.x86.avx512.psrlv.d.512(<16 x i32> <i32 4, i32 4, i32 4, i32 4, i32 4, i32 4, i32 4, i32 4, i32 4, i32 4, i32 4, i32 4, i32 4, i32 4, i32 4, i32 -1>, <16 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 -1>)
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <16 x i32> [[TMP2]], [[TMP4]]
+; CHECK-NEXT:    [[RES2:%.*]] = add <16 x i32> [[RES0]], [[RES1]]
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i32> [[RES2]]
+;
+  %res0 = call <16 x i32> @llvm.x86.avx512.psrlv.d.512(<16 x i32> <i32 2, i32 9, i32 0, i32 -1, i32 3, i32 7, i32 -1, i32 0, i32 4, i32 5, i32 -2, i32 0, i32 5, i32 3, i32 -3, i32 0>, <16 x i32> <i32 1, i32 0, i32 33, i32 -1,i32 2, i32 0, i32 34, i32 -2, i32 3, i32 0, i32 35, i32 -1, i32 4, i32 0, i32 36, i32 -3>)
+  %res1 = call <16 x i32> @llvm.x86.avx512.psrlv.d.512(<16 x i32> <i32 4, i32 4, i32 4, i32 4, i32 4, i32 4, i32 4, i32 4, i32 4, i32 4, i32 4, i32 4, i32 4, i32 4, i32 4, i32 -1>, <16 x i32> <i32 1, i32 1, i32 1,  i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1,  i32 1, i32 1, i32 1, i32 1, i32 -1  >)
+  %res2 = add <16 x i32> %res0, %res1
+  ret <16 x i32> %res2
+}
+
+define <16 x i32> @test_x86_avx512_mask_psrlv_d_512(<16 x i32> %a0, <16 x i32> %a1, <16 x i32> %a2, i16 %mask) #0 {
+; CHECK-LABEL: @test_x86_avx512_mask_psrlv_d_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp ne <16 x i32> [[TMP2]], zeroinitializer
+; CHECK-NEXT:    [[TMP6:%.*]] = sext <16 x i1> [[TMP5]] to <16 x i32>
+; CHECK-NEXT:    [[TMP7:%.*]] = call <16 x i32> @llvm.x86.avx512.psrlv.d.512(<16 x i32> [[TMP1]], <16 x i32> [[A1:%.*]])
+; CHECK-NEXT:    [[TMP8:%.*]] = or <16 x i32> [[TMP7]], [[TMP6]]
+; CHECK-NEXT:    [[RES:%.*]] = call <16 x i32> @llvm.x86.avx512.psrlv.d.512(<16 x i32> [[A0:%.*]], <16 x i32> [[A1]])
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast i16 [[TMP3]] to <16 x i1>
+; CHECK-NEXT:    [[MASK_CAST:%.*]] = bitcast i16 [[MASK:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP10:%.*]] = select <16 x i1> [[MASK_CAST]], <16 x i32> [[TMP8]], <16 x i32> [[TMP4]]
+; CHECK-NEXT:    [[TMP11:%.*]] = xor <16 x i32> [[RES]], [[A2:%.*]]
+; CHECK-NEXT:    [[TMP12:%.*]] = or <16 x i32> [[TMP11]], [[TMP8]]
+; CHECK-NEXT:    [[TMP13:%.*]] = or <16 x i32> [[TMP12]], [[TMP4]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP9]], <16 x i32> [[TMP13]], <16 x i32> [[TMP10]]
+; CHECK-NEXT:    [[RES2:%.*]] = select <16 x i1> [[MASK_CAST]], <16 x i32> [[RES]], <16 x i32> [[A2]]
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i32> [[RES2]]
+;
+  %res = call <16 x i32> @llvm.x86.avx512.psrlv.d.512(<16 x i32> %a0, <16 x i32> %a1)
+  %mask.cast = bitcast i16 %mask to <16 x i1>
+  %res2 = select <16 x i1> %mask.cast, <16 x i32> %res, <16 x i32> %a2
+  ret <16 x i32> %res2
+}
+
+define <16 x i32> @test_x86_avx512_maskz_psrlv_d_512(<16 x i32> %a0, <16 x i32> %a1, i16 %mask) #0 {
+; CHECK-LABEL: @test_x86_avx512_maskz_psrlv_d_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = icmp ne <16 x i32> [[TMP2]], zeroinitializer
+; CHECK-NEXT:    [[TMP5:%.*]] = sext <16 x i1> [[TMP4]] to <16 x i32>
+; CHECK-NEXT:    [[TMP6:%.*]] = call <16 x i32> @llvm.x86.avx512.psrlv.d.512(<16 x i32> [[TMP1]], <16 x i32> [[A1:%.*]])
+; CHECK-NEXT:    [[TMP7:%.*]] = or <16 x i32> [[TMP6]], [[TMP5]]
+; CHECK-NEXT:    [[RES:%.*]] = call <16 x i32> @llvm.x86.avx512.psrlv.d.512(<16 x i32> [[A0:%.*]], <16 x i32> [[A1]])
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast i16 [[TMP3]] to <16 x i1>
+; CHECK-NEXT:    [[MASK_CAST:%.*]] = bitcast i16 [[MASK:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP9:%.*]] = select <16 x i1> [[MASK_CAST]], <16 x i32> [[TMP7]], <16 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP10:%.*]] = xor <16 x i32> [[RES]], zeroinitializer
+; CHECK-NEXT:    [[TMP11:%.*]] = or <16 x i32> [[TMP10]], [[TMP7]]
+; CHECK-NEXT:    [[TMP12:%.*]] = or <16 x i32> [[TMP11]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP8]], <16 x i32> [[TMP12]], <16 x i32> [[TMP9]]
+; CHECK-NEXT:    [[RES2:%.*]] = select <16 x i1> [[MASK_CAST]], <16 x i32> [[RES]], <16 x i32> zeroinitializer
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i32> [[RES2]]
+;
+  %res = call <16 x i32> @llvm.x86.avx512.psrlv.d.512(<16 x i32> %a0, <16 x i32> %a1)
+  %mask.cast = bitcast i16 %mask to <16 x i1>
+  %res2 = select <16 x i1> %mask.cast, <16 x i32> %res, <16 x i32> zeroinitializer
+  ret <16 x i32> %res2
+}
+
+declare <16 x i32> @llvm.x86.avx512.psrlv.d.512(<16 x i32>, <16 x i32>) nounwind readnone
+
+define <8 x i64> @test_x86_avx512_psrlv_q_512(<8 x i64> %a0, <8 x i64> %a1) #0 {
+; CHECK-LABEL: @test_x86_avx512_psrlv_q_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp ne <8 x i64> [[TMP2]], zeroinitializer
+; CHECK-NEXT:    [[TMP4:%.*]] = sext <8 x i1> [[TMP3]] to <8 x i64>
+; CHECK-NEXT:    [[TMP5:%.*]] = call <8 x i64> @llvm.x86.avx512.psrlv.q.512(<8 x i64> [[TMP1]], <8 x i64> [[A1:%.*]])
+; CHECK-NEXT:    [[TMP6:%.*]] = or <8 x i64> [[TMP5]], [[TMP4]]
+; CHECK-NEXT:    [[RES:%.*]] = call <8 x i64> @llvm.x86.avx512.psrlv.q.512(<8 x i64> [[A0:%.*]], <8 x i64> [[A1]])
+; CHECK-NEXT:    store <8 x i64> [[TMP6]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i64> [[RES]]
+;
+  %res = call <8 x i64> @llvm.x86.avx512.psrlv.q.512(<8 x i64> %a0, <8 x i64> %a1)
+  ret <8 x i64> %res
+}
+
+define <8 x i64> @test_x86_avx512_psrlv_q_512_const() #0 {
+; CHECK-LABEL: @test_x86_avx512_psrlv_q_512_const(
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP1:%.*]] = call <8 x i64> @llvm.x86.avx512.psrlv.q.512(<8 x i64> zeroinitializer, <8 x i64> <i64 1, i64 0, i64 33, i64 -1, i64 2, i64 0, i64 34, i64 -2>)
+; CHECK-NEXT:    [[TMP2:%.*]] = or <8 x i64> [[TMP1]], zeroinitializer
+; CHECK-NEXT:    [[RES0:%.*]] = call <8 x i64> @llvm.x86.avx512.psrlv.q.512(<8 x i64> <i64 2, i64 9, i64 0, i64 -1, i64 3, i64 7, i64 -1, i64 0>, <8 x i64> <i64 1, i64 0, i64 33, i64 -1, i64 2, i64 0, i64 34, i64 -2>)
+; CHECK-NEXT:    [[TMP3:%.*]] = call <8 x i64> @llvm.x86.avx512.psrlv.q.512(<8 x i64> zeroinitializer, <8 x i64> <i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 -1>)
+; CHECK-NEXT:    [[TMP4:%.*]] = or <8 x i64> [[TMP3]], zeroinitializer
+; CHECK-NEXT:    [[RES1:%.*]] = call <8 x i64> @llvm.x86.avx512.psrlv.q.512(<8 x i64> <i64 4, i64 4, i64 4, i64 4, i64 4, i64 4, i64 4, i64 -1>, <8 x i64> <i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 -1>)
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <8 x i64> [[TMP2]], [[TMP4]]
+; CHECK-NEXT:    [[RES2:%.*]] = add <8 x i64> [[RES0]], [[RES1]]
+; CHECK-NEXT:    store <8 x i64> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i64> [[RES2]]
+;
+  %res0 = call <8 x i64> @llvm.x86.avx512.psrlv.q.512(<8 x i64> <i64 2, i64 9, i64 0, i64 -1, i64 3, i64 7, i64 -1, i64 0>, <8 x i64> <i64 1, i64 0, i64 33, i64 -1,i64 2, i64 0, i64 34, i64 -2>)
+  %res1 = call <8 x i64> @llvm.x86.avx512.psrlv.q.512(<8 x i64> <i64 4, i64 4, i64 4, i64 4, i64 4, i64 4, i64 4, i64 -1>, <8 x i64> <i64 1, i64 1, i64 1, i64 1,  i64 1, i64 1, i64 1, i64 -1>)
+  %res2 = add <8 x i64> %res0, %res1
+  ret <8 x i64> %res2
+}
+
+define <8 x i64> @test_x86_avx512_mask_psrlv_q_512(<8 x i64> %a0, <8 x i64> %a1, <8 x i64> %a2, i8 %mask) #0 {
+; CHECK-LABEL: @test_x86_avx512_mask_psrlv_q_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp ne <8 x i64> [[TMP2]], zeroinitializer
+; CHECK-NEXT:    [[TMP6:%.*]] = sext <8 x i1> [[TMP5]] to <8 x i64>
+; CHECK-NEXT:    [[TMP7:%.*]] = call <8 x i64> @llvm.x86.avx512.psrlv.q.512(<8 x i64> [[TMP1]], <8 x i64> [[A1:%.*]])
+; CHECK-NEXT:    [[TMP8:%.*]] = or <8 x i64> [[TMP7]], [[TMP6]]
+; CHECK-NEXT:    [[RES:%.*]] = call <8 x i64> @llvm.x86.avx512.psrlv.q.512(<8 x i64> [[A0:%.*]], <8 x i64> [[A1]])
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast i8 [[TMP3]] to <8 x i1>
+; CHECK-NEXT:    [[MASK_CAST:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP10:%.*]] = select <8 x i1> [[MASK_CAST]], <8 x i64> [[TMP8]], <8 x i64> [[TMP4]]
+; CHECK-NEXT:    [[TMP11:%.*]] = xor <8 x i64> [[RES]], [[A2:%.*]]
+; CHECK-NEXT:    [[TMP12:%.*]] = or <8 x i64> [[TMP11]], [[TMP8]]
+; CHECK-NEXT:    [[TMP13:%.*]] = or <8 x i64> [[TMP12]], [[TMP4]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP9]], <8 x i64> [[TMP13]], <8 x i64> [[TMP10]]
+; CHECK-NEXT:    [[RES2:%.*]] = select <8 x i1> [[MASK_CAST]], <8 x i64> [[RES]], <8 x i64> [[A2]]
+; CHECK-NEXT:    store <8 x i64> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i64> [[RES2]]
+;
+  %res = call <8 x i64> @llvm.x86.avx512.psrlv.q.512(<8 x i64> %a0, <8 x i64> %a1)
+  %mask.cast = bitcast i8 %mask to <8 x i1>
+  %res2 = select <8 x i1> %mask.cast, <8 x i64> %res, <8 x i64> %a2
+  ret <8 x i64> %res2
+}
+
+define <8 x i64> @test_x86_avx512_maskz_psrlv_q_512(<8 x i64> %a0, <8 x i64> %a1, i8 %mask) #0 {
+; CHECK-LABEL: @test_x86_avx512_maskz_psrlv_q_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = icmp ne <8 x i64> [[TMP2]], zeroinitializer
+; CHECK-NEXT:    [[TMP5:%.*]] = sext <8 x i1> [[TMP4]] to <8 x i64>
+; CHECK-NEXT:    [[TMP6:%.*]] = call <8 x i64> @llvm.x86.avx512.psrlv.q.512(<8 x i64> [[TMP1]], <8 x i64> [[A1:%.*]])
+; CHECK-NEXT:    [[TMP7:%.*]] = or <8 x i64> [[TMP6]], [[TMP5]]
+; CHECK-NEXT:    [[RES:%.*]] = call <8 x i64> @llvm.x86.avx512.psrlv.q.512(<8 x i64> [[A0:%.*]], <8 x i64> [[A1]])
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast i8 [[TMP3]] to <8 x i1>
+; CHECK-NEXT:    [[MASK_CAST:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP9:%.*]] = select <8 x i1> [[MASK_CAST]], <8 x i64> [[TMP7]], <8 x i64> zeroinitializer
+; CHECK-NEXT:    [[TMP10:%.*]] = xor <8 x i64> [[RES]], zeroinitializer
+; CHECK-NEXT:    [[TMP11:%.*]] = or <8 x i64> [[TMP10]], [[TMP7]]
+; CHECK-NEXT:    [[TMP12:%.*]] = or <8 x i64> [[TMP11]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP8]], <8 x i64> [[TMP12]], <8 x i64> [[TMP9]]
+; CHECK-NEXT:    [[RES2:%.*]] = select <8 x i1> [[MASK_CAST]], <8 x i64> [[RES]], <8 x i64> zeroinitializer
+; CHECK-NEXT:    store <8 x i64> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i64> [[RES2]]
+;
+  %res = call <8 x i64> @llvm.x86.avx512.psrlv.q.512(<8 x i64> %a0, <8 x i64> %a1)
+  %mask.cast = bitcast i8 %mask to <8 x i1>
+  %res2 = select <8 x i1> %mask.cast, <8 x i64> %res, <8 x i64> zeroinitializer
+  ret <8 x i64> %res2
+}
+
+declare <8 x i64> @llvm.x86.avx512.psrlv.q.512(<8 x i64>, <8 x i64>) nounwind readnone
+
+
+define <8 x double> @test_mm256_castpd128_pd256_freeze(<2 x double> %a0) nounwind #0 {
+; CHECK-LABEL: @test_mm256_castpd128_pd256_freeze(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[A1:%.*]] = freeze <2 x double> poison
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <2 x i64> [[TMP1]], <2 x i64> zeroinitializer, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 2, i32 3, i32 2, i32 3>
+; CHECK-NEXT:    [[RES:%.*]] = shufflevector <2 x double> [[A0:%.*]], <2 x double> [[A1]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 2, i32 3, i32 2, i32 3>
+; CHECK-NEXT:    store <8 x i64> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x double> [[RES]]
+;
+  %a1 = freeze <2 x double> poison
+  %res = shufflevector <2 x double> %a0, <2 x double> %a1, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 2, i32 3, i32 2, i32 3>
+  ret <8 x double> %res
+}
+
+
+define <8 x double> @test_mm256_castpd256_pd256_freeze(<4 x double> %a0) nounwind #0 {
+; CHECK-LABEL: @test_mm256_castpd256_pd256_freeze(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[A1:%.*]] = freeze <4 x double> poison
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <4 x i64> [[TMP1]], <4 x i64> zeroinitializer, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:    [[RES:%.*]] = shufflevector <4 x double> [[A0:%.*]], <4 x double> [[A1]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:    store <8 x i64> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x double> [[RES]]
+;
+  %a1 = freeze <4 x double> poison
+  %res = shufflevector <4 x double> %a0, <4 x double> %a1, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+  ret <8 x double> %res
+}
+
+
+define <16 x float> @test_mm256_castps128_ps512_freeze(<4 x float> %a0) nounwind #0 {
+; CHECK-LABEL: @test_mm256_castps128_ps512_freeze(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[A1:%.*]] = freeze <4 x float> poison
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <4 x i32> [[TMP1]], <4 x i32> zeroinitializer, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 4, i32 5, i32 6, i32 7, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:    [[RES:%.*]] = shufflevector <4 x float> [[A0:%.*]], <4 x float> [[A1]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 4, i32 5, i32 6, i32 7, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x float> [[RES]]
+;
+  %a1 = freeze <4 x float> poison
+  %res = shufflevector <4 x float> %a0, <4 x float> %a1, <16x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 4, i32 5, i32 6, i32 7, i32 4, i32 5, i32 6, i32 7>
+  ret <16 x float> %res
+}
+
+
+define <16 x float> @test_mm256_castps256_ps512_freeze(<8 x float> %a0) nounwind #0 {
+; CHECK-LABEL: @test_mm256_castps256_ps512_freeze(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[A1:%.*]] = freeze <8 x float> poison
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <8 x i32> [[TMP1]], <8 x i32> zeroinitializer, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; CHECK-NEXT:    [[RES:%.*]] = shufflevector <8 x float> [[A0:%.*]], <8 x float> [[A1]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x float> [[RES]]
+;
+  %a1 = freeze <8 x float> poison
+  %res = shufflevector <8 x float> %a0, <8 x float> %a1, <16x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+  ret <16 x float> %res
+}
+
+
+define <8 x i64> @test_mm512_castsi128_si512_freeze(<2 x i64> %a0) nounwind #0 {
+; CHECK-LABEL: @test_mm512_castsi128_si512_freeze(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[A1:%.*]] = freeze <2 x i64> poison
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <2 x i64> [[TMP1]], <2 x i64> zeroinitializer, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 2, i32 3, i32 2, i32 3>
+; CHECK-NEXT:    [[RES:%.*]] = shufflevector <2 x i64> [[A0:%.*]], <2 x i64> [[A1]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 2, i32 3, i32 2, i32 3>
+; CHECK-NEXT:    store <8 x i64> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i64> [[RES]]
+;
+  %a1 = freeze <2 x i64> poison
+  %res = shufflevector <2 x i64> %a0, <2 x i64> %a1, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 2, i32 3, i32 2, i32 3>
+  ret <8 x i64> %res
+}
+
+
+define <8 x i64> @test_mm512_castsi256_si512_pd256_freeze(<4 x i64> %a0) nounwind #0 {
+; CHECK-LABEL: @test_mm512_castsi256_si512_pd256_freeze(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[A1:%.*]] = freeze <4 x i64> poison
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <4 x i64> [[TMP1]], <4 x i64> zeroinitializer, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:    [[RES:%.*]] = shufflevector <4 x i64> [[A0:%.*]], <4 x i64> [[A1]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:    store <8 x i64> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i64> [[RES]]
+;
+  %a1 = freeze <4 x i64> poison
+  %res = shufflevector <4 x i64> %a0, <4 x i64> %a1, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+  ret <8 x i64> %res
+}
+
+
+define <16 x float> @bad_mask_transition(<8 x double> %a, <8 x double> %b, <8 x double> %c, <8 x double> %d, <16 x float> %e, <16 x float> %f, <16 x i1> %extra_param, <16 x i1> %extra_param2) #0 {
+; CHECK-LABEL: @bad_mask_transition(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
+; CHECK-NEXT:    [[TMP8:%.*]] = load <16 x i1>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 384) to ptr), align 8
+; CHECK-NEXT:    [[TMP9:%.*]] = load <16 x i1>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 392) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 320) to ptr), align 8
+; CHECK-NEXT:    [[TMP5:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 256) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <8 x i64> [[TMP0]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP6]], 0
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <8 x i64> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP7]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP14:%.*]], label [[TMP15:%.*]], !prof [[PROF1]]
+; CHECK:       10:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       11:
+; CHECK-NEXT:    [[TMP10:%.*]] = call <8 x i1> @llvm.x86.avx512.mask.cmp.pd.512(<8 x double> [[A:%.*]], <8 x double> [[B:%.*]], i32 17, <8 x i1> splat (i1 true), i32 4)
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <8 x i1> [[TMP10]] to i8
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <8 x i64> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i512 [[TMP12]], 0
+; CHECK-NEXT:    [[TMP13:%.*]] = bitcast <8 x i64> [[TMP3]] to i512
+; CHECK-NEXT:    [[_MSCMP3:%.*]] = icmp ne i512 [[TMP13]], 0
+; CHECK-NEXT:    [[_MSOR4:%.*]] = or i1 [[_MSCMP2]], [[_MSCMP3]]
+; CHECK-NEXT:    br i1 [[_MSOR4]], label [[TMP30:%.*]], label [[TMP34:%.*]], !prof [[PROF1]]
+; CHECK:       16:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       17:
+; CHECK-NEXT:    [[TMP16:%.*]] = call <8 x i1> @llvm.x86.avx512.mask.cmp.pd.512(<8 x double> [[C:%.*]], <8 x double> [[D:%.*]], i32 17, <8 x i1> splat (i1 true), i32 4)
+; CHECK-NEXT:    [[TMP17:%.*]] = bitcast <8 x i1> [[TMP16]] to i8
+; CHECK-NEXT:    [[CONV:%.*]] = zext i8 [[TMP11]] to i16
+; CHECK-NEXT:    [[CONV2:%.*]] = zext i8 [[TMP17]] to i16
+; CHECK-NEXT:    [[TMP18:%.*]] = bitcast i16 [[CONV]] to <16 x i1>
+; CHECK-NEXT:    [[TMP19:%.*]] = bitcast i16 [[CONV2]] to <16 x i1>
+; CHECK-NEXT:    [[TMP20:%.*]] = shufflevector <16 x i1> zeroinitializer, <16 x i1> [[TMP8]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:    [[TMP31:%.*]] = shufflevector <16 x i1> [[TMP18]], <16 x i1> [[EXTRA_PARAM:%.*]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:    [[TMP21:%.*]] = shufflevector <16 x i1> zeroinitializer, <16 x i1> [[TMP9]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:    [[TMP32:%.*]] = shufflevector <16 x i1> [[TMP19]], <16 x i1> [[EXTRA_PARAM2:%.*]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:    [[TMP22:%.*]] = shufflevector <8 x i1> [[TMP20]], <8 x i1> [[TMP21]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; CHECK-NEXT:    [[TMP23:%.*]] = shufflevector <8 x i1> [[TMP31]], <8 x i1> [[TMP32]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; CHECK-NEXT:    [[TMP33:%.*]] = select <16 x i1> [[TMP23]], <16 x i32> [[TMP4]], <16 x i32> [[TMP5]]
+; CHECK-NEXT:    [[TMP24:%.*]] = bitcast <16 x float> [[F:%.*]] to <16 x i32>
+; CHECK-NEXT:    [[TMP25:%.*]] = bitcast <16 x float> [[E:%.*]] to <16 x i32>
+; CHECK-NEXT:    [[TMP26:%.*]] = xor <16 x i32> [[TMP24]], [[TMP25]]
+; CHECK-NEXT:    [[TMP27:%.*]] = or <16 x i32> [[TMP26]], [[TMP4]]
+; CHECK-NEXT:    [[TMP28:%.*]] = or <16 x i32> [[TMP27]], [[TMP5]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP22]], <16 x i32> [[TMP28]], <16 x i32> [[TMP33]]
+; CHECK-NEXT:    [[TMP29:%.*]] = select <16 x i1> [[TMP23]], <16 x float> [[F]], <16 x float> [[E]]
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x float> [[TMP29]]
+;
+entry:
+  %0 = call <8 x i1> @llvm.x86.avx512.mask.cmp.pd.512(<8 x double> %a, <8 x double> %b, i32 17, <8 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, i32 4)
+  %1 = bitcast <8 x i1> %0 to i8
+  %2 = call <8 x i1> @llvm.x86.avx512.mask.cmp.pd.512(<8 x double> %c, <8 x double> %d, i32 17, <8 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, i32 4)
+  %3 = bitcast <8 x i1> %2 to i8
+  %conv = zext i8 %1 to i16
+  %conv2 = zext i8 %3 to i16
+  %4 = bitcast i16 %conv to <16 x i1>
+  %5 = bitcast i16 %conv2 to <16 x i1>
+  %6 = shufflevector <16 x i1> %4, <16 x i1> %extra_param, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+  %7 = shufflevector <16 x i1> %5, <16 x i1> %extra_param2, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+  %8 = shufflevector <8 x i1> %6, <8 x i1> %7, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+  %9 = select <16 x i1> %8, <16 x float> %f, <16 x float> %e
+  ret <16 x float> %9
+}
+
+define <16 x float> @bad_mask_transition_2(<8 x double> %a, <8 x double> %b, <8 x double> %c, <8 x double> %d, <16 x float> %e, <16 x float> %f) #0 {
+; CHECK-LABEL: @bad_mask_transition_2(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 320) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 256) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <8 x i64> [[TMP0]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <8 x i64> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP6:%.*]], label [[TMP7:%.*]], !prof [[PROF1]]
+; CHECK:       6:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       7:
+; CHECK-NEXT:    [[TMP8:%.*]] = call <8 x i1> @llvm.x86.avx512.mask.cmp.pd.512(<8 x double> [[A:%.*]], <8 x double> [[B:%.*]], i32 17, <8 x i1> splat (i1 true), i32 4)
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <8 x i1> [[TMP8]] to i8
+; CHECK-NEXT:    [[CONV:%.*]] = zext i8 [[TMP9]] to i16
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i16 [[CONV]] to <16 x i1>
+; CHECK-NEXT:    [[TMP11:%.*]] = select <16 x i1> [[TMP10]], <16 x i32> [[TMP2]], <16 x i32> [[TMP3]]
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <16 x float> [[F:%.*]] to <16 x i32>
+; CHECK-NEXT:    [[TMP13:%.*]] = bitcast <16 x float> [[E:%.*]] to <16 x i32>
+; CHECK-NEXT:    [[TMP14:%.*]] = xor <16 x i32> [[TMP12]], [[TMP13]]
+; CHECK-NEXT:    [[TMP15:%.*]] = or <16 x i32> [[TMP14]], [[TMP2]]
+; CHECK-NEXT:    [[TMP16:%.*]] = or <16 x i32> [[TMP15]], [[TMP3]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> zeroinitializer, <16 x i32> [[TMP16]], <16 x i32> [[TMP11]]
+; CHECK-NEXT:    [[TMP17:%.*]] = select <16 x i1> [[TMP10]], <16 x float> [[F]], <16 x float> [[E]]
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x float> [[TMP17]]
+;
+entry:
+  %0 = call <8 x i1> @llvm.x86.avx512.mask.cmp.pd.512(<8 x double> %a, <8 x double> %b, i32 17, <8 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, i32 4)
+  %1 = bitcast <8 x i1> %0 to i8
+  %conv = zext i8 %1 to i16
+  %2 = bitcast i16 %conv to <16 x i1>
+  %3 = select <16 x i1> %2, <16 x float> %f, <16 x float> %e
+  ret <16 x float> %3
+}
+
+declare <8 x double> @llvm.x86.avx512.mask.compress.v8f64(<8 x double>, <8 x double>, <8 x i1>)
+declare <16 x float> @llvm.x86.avx512.mask.compress.v16f32(<16 x float>, <16 x float>, <16 x i1>)
+declare <8 x i64> @llvm.x86.avx512.mask.compress.v8i64(<8 x i64>, <8 x i64>, <8 x i1>)
+declare <16 x i32> @llvm.x86.avx512.mask.compress.v16i32(<16 x i32>, <16 x i32>, <16 x i1>)
+declare <8 x double> @llvm.x86.avx512.mask.expand.v8f64(<8 x double>, <8 x double>, <8 x i1>)
+declare <16 x float> @llvm.x86.avx512.mask.expand.v16f32(<16 x float>, <16 x float>, <16 x i1>)
+declare <8 x i64> @llvm.x86.avx512.mask.expand.v8i64(<8 x i64>, <8 x i64>, <8 x i1>)
+declare <16 x i32> @llvm.x86.avx512.mask.expand.v16i32(<16 x i32>, <16 x i32>, <16 x i1>)
+
+attributes #0 = { sanitize_memory }
diff --git a/llvm/test/Instrumentation/MemorySanitizer/X86/mmx-intrinsics.ll b/llvm/test/Instrumentation/MemorySanitizer/X86/mmx-intrinsics.ll
index 61c90d0fb80d4..2b6c6ff2e2b92 100644
--- a/llvm/test/Instrumentation/MemorySanitizer/X86/mmx-intrinsics.ll
+++ b/llvm/test/Instrumentation/MemorySanitizer/X86/mmx-intrinsics.ll
@@ -3339,7 +3339,7 @@ define i64 @test6(<1 x i64> %a, <1 x i64> %b) #0 {
 ; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <4 x i16> [[TMP1]] to <1 x i64>
 ; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <4 x i16> [[TMP12]] to <1 x i64>
 ; CHECK-NEXT:    [[TMP17:%.*]] = bitcast <4 x i16> [[TMP0]] to <1 x i64>
-; CHECK-NEXT:    [[_MSPROP:%.*]] = call <1 x i64> @llvm.x86.ssse3.phsub.sw(<1 x i64> [[TMP16]], <1 x i64> [[TMP8]])
+; CHECK-NEXT:    [[_MSPROP:%.*]] = call <1 x i64> @llvm.x86.ssse3.phadd.sw(<1 x i64> [[TMP16]], <1 x i64> [[TMP8]])
 ; CHECK-NEXT:    [[TMP18:%.*]] = tail call <1 x i64> @llvm.x86.ssse3.phsub.sw(<1 x i64> [[TMP2]], <1 x i64> [[TMP17]]) #[[ATTR5]]
 ; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <1 x i64> [[_MSPROP]] to <4 x i16>
 ; CHECK-NEXT:    [[TMP19:%.*]] = bitcast <1 x i64> [[TMP18]] to <4 x i16>
@@ -3379,7 +3379,7 @@ define i64 @test5(<1 x i64> %a, <1 x i64> %b) #0 {
 ; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <2 x i32> [[TMP1]] to <1 x i64>
 ; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <2 x i32> [[TMP12]] to <1 x i64>
 ; CHECK-NEXT:    [[TMP17:%.*]] = bitcast <2 x i32> [[TMP0]] to <1 x i64>
-; CHECK-NEXT:    [[_MSPROP:%.*]] = call <1 x i64> @llvm.x86.ssse3.phsub.d(<1 x i64> [[TMP16]], <1 x i64> [[TMP8]])
+; CHECK-NEXT:    [[_MSPROP:%.*]] = call <1 x i64> @llvm.x86.ssse3.phadd.d(<1 x i64> [[TMP16]], <1 x i64> [[TMP8]])
 ; CHECK-NEXT:    [[TMP18:%.*]] = tail call <1 x i64> @llvm.x86.ssse3.phsub.d(<1 x i64> [[TMP2]], <1 x i64> [[TMP17]]) #[[ATTR5]]
 ; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <1 x i64> [[_MSPROP]] to <2 x i32>
 ; CHECK-NEXT:    [[TMP19:%.*]] = bitcast <1 x i64> [[TMP18]] to <2 x i32>
@@ -3419,7 +3419,7 @@ define i64 @test4(<1 x i64> %a, <1 x i64> %b) #0 {
 ; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <4 x i16> [[TMP1]] to <1 x i64>
 ; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <4 x i16> [[TMP12]] to <1 x i64>
 ; CHECK-NEXT:    [[TMP17:%.*]] = bitcast <4 x i16> [[TMP0]] to <1 x i64>
-; CHECK-NEXT:    [[_MSPROP:%.*]] = call <1 x i64> @llvm.x86.ssse3.phsub.w(<1 x i64> [[TMP16]], <1 x i64> [[TMP8]])
+; CHECK-NEXT:    [[_MSPROP:%.*]] = call <1 x i64> @llvm.x86.ssse3.phadd.w(<1 x i64> [[TMP16]], <1 x i64> [[TMP8]])
 ; CHECK-NEXT:    [[TMP18:%.*]] = tail call <1 x i64> @llvm.x86.ssse3.phsub.w(<1 x i64> [[TMP2]], <1 x i64> [[TMP17]]) #[[ATTR5]]
 ; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <1 x i64> [[_MSPROP]] to <4 x i16>
 ; CHECK-NEXT:    [[TMP19:%.*]] = bitcast <1 x i64> [[TMP18]] to <4 x i16>
diff --git a/llvm/test/Instrumentation/MemorySanitizer/bitreverse.ll b/llvm/test/Instrumentation/MemorySanitizer/bitreverse.ll
new file mode 100644
index 0000000000000..f1b6181289967
--- /dev/null
+++ b/llvm/test/Instrumentation/MemorySanitizer/bitreverse.ll
@@ -0,0 +1,231 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 4
+; RUN: opt < %s -passes=msan -S | FileCheck %s
+;
+; Forked from llvm/test/CodeGen/X86/bitreverse.ll
+
+target datalayout = "e-m:o-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+declare <2 x i16> @llvm.bitreverse.v2i16(<2 x i16>) readnone
+
+define <2 x i16> @test_bitreverse_v2i16(<2 x i16> %a) nounwind #0 {
+; CHECK-LABEL: define <2 x i16> @test_bitreverse_v2i16(
+; CHECK-SAME: <2 x i16> [[A:%.*]]) #[[ATTR1:[0-9]+]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = call <2 x i16> @llvm.bitreverse.v2i16(<2 x i16> [[TMP1]])
+; CHECK-NEXT:    [[B:%.*]] = call <2 x i16> @llvm.bitreverse.v2i16(<2 x i16> [[A]])
+; CHECK-NEXT:    store <2 x i16> [[TMP2]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x i16> [[B]]
+;
+  %b = call <2 x i16> @llvm.bitreverse.v2i16(<2 x i16> %a)
+  ret <2 x i16> %b
+}
+
+declare i64 @llvm.bitreverse.i64(i64) readnone
+
+define i64 @test_bitreverse_i64(i64 %a) nounwind #0 {
+; CHECK-LABEL: define i64 @test_bitreverse_i64(
+; CHECK-SAME: i64 [[A:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.bitreverse.i64(i64 [[TMP1]])
+; CHECK-NEXT:    [[B:%.*]] = call i64 @llvm.bitreverse.i64(i64 [[A]])
+; CHECK-NEXT:    store i64 [[TMP2]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i64 [[B]]
+;
+  %b = call i64 @llvm.bitreverse.i64(i64 %a)
+  ret i64 %b
+}
+
+declare i32 @llvm.bitreverse.i32(i32) readnone
+
+define i32 @test_bitreverse_i32(i32 %a) nounwind #0 {
+; CHECK-LABEL: define i32 @test_bitreverse_i32(
+; CHECK-SAME: i32 [[A:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i32, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = call i32 @llvm.bitreverse.i32(i32 [[TMP1]])
+; CHECK-NEXT:    [[B:%.*]] = call i32 @llvm.bitreverse.i32(i32 [[A]])
+; CHECK-NEXT:    store i32 [[TMP2]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i32 [[B]]
+;
+  %b = call i32 @llvm.bitreverse.i32(i32 %a)
+  ret i32 %b
+}
+
+declare i24 @llvm.bitreverse.i24(i24) readnone
+
+define i24 @test_bitreverse_i24(i24 %a) nounwind #0 {
+; CHECK-LABEL: define i24 @test_bitreverse_i24(
+; CHECK-SAME: i24 [[A:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i24, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = call i24 @llvm.bitreverse.i24(i24 [[TMP1]])
+; CHECK-NEXT:    [[B:%.*]] = call i24 @llvm.bitreverse.i24(i24 [[A]])
+; CHECK-NEXT:    store i24 [[TMP2]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i24 [[B]]
+;
+  %b = call i24 @llvm.bitreverse.i24(i24 %a)
+  ret i24 %b
+}
+
+declare i16 @llvm.bitreverse.i16(i16) readnone
+
+define i16 @test_bitreverse_i16(i16 %a) nounwind #0 {
+; CHECK-LABEL: define i16 @test_bitreverse_i16(
+; CHECK-SAME: i16 [[A:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i16, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = call i16 @llvm.bitreverse.i16(i16 [[TMP1]])
+; CHECK-NEXT:    [[B:%.*]] = call i16 @llvm.bitreverse.i16(i16 [[A]])
+; CHECK-NEXT:    store i16 [[TMP2]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i16 [[B]]
+;
+  %b = call i16 @llvm.bitreverse.i16(i16 %a)
+  ret i16 %b
+}
+
+declare i8 @llvm.bitreverse.i8(i8) readnone
+
+define i8 @test_bitreverse_i8(i8 %a) #0 {
+; CHECK-LABEL: define i8 @test_bitreverse_i8(
+; CHECK-SAME: i8 [[A:%.*]]) #[[ATTR2:[0-9]+]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i8, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = call i8 @llvm.bitreverse.i8(i8 [[TMP1]])
+; CHECK-NEXT:    [[B:%.*]] = call i8 @llvm.bitreverse.i8(i8 [[A]])
+; CHECK-NEXT:    store i8 [[TMP2]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i8 [[B]]
+;
+  %b = call i8 @llvm.bitreverse.i8(i8 %a)
+  ret i8 %b
+}
+
+declare i4 @llvm.bitreverse.i4(i4) readnone
+
+define i4 @test_bitreverse_i4(i4 %a) #0 {
+; CHECK-LABEL: define i4 @test_bitreverse_i4(
+; CHECK-SAME: i4 [[A:%.*]]) #[[ATTR2]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i4, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = call i4 @llvm.bitreverse.i4(i4 [[TMP1]])
+; CHECK-NEXT:    [[B:%.*]] = call i4 @llvm.bitreverse.i4(i4 [[A]])
+; CHECK-NEXT:    store i4 [[TMP2]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i4 [[B]]
+;
+  %b = call i4 @llvm.bitreverse.i4(i4 %a)
+  ret i4 %b
+}
+
+; These tests check that bitreverse(constant) calls are folded
+
+define <2 x i16> @fold_v2i16() #0 {
+; CHECK-LABEL: define <2 x i16> @fold_v2i16(
+; CHECK-SAME: ) #[[ATTR2]] {
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP1:%.*]] = call <2 x i16> @llvm.bitreverse.v2i16(<2 x i16> zeroinitializer)
+; CHECK-NEXT:    [[B:%.*]] = call <2 x i16> @llvm.bitreverse.v2i16(<2 x i16> <i16 15, i16 3840>)
+; CHECK-NEXT:    store <2 x i16> [[TMP1]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x i16> [[B]]
+;
+  %b = call <2 x i16> @llvm.bitreverse.v2i16(<2 x i16> <i16 15, i16 3840>)
+  ret <2 x i16> %b
+}
+
+define i24 @fold_i24() #0 {
+; CHECK-LABEL: define i24 @fold_i24(
+; CHECK-SAME: ) #[[ATTR2]] {
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP1:%.*]] = call i24 @llvm.bitreverse.i24(i24 0)
+; CHECK-NEXT:    [[B:%.*]] = call i24 @llvm.bitreverse.i24(i24 4096)
+; CHECK-NEXT:    store i24 [[TMP1]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i24 [[B]]
+;
+  %b = call i24 @llvm.bitreverse.i24(i24 4096)
+  ret i24 %b
+}
+
+define i8 @fold_i8() #0 {
+; CHECK-LABEL: define i8 @fold_i8(
+; CHECK-SAME: ) #[[ATTR2]] {
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP1:%.*]] = call i8 @llvm.bitreverse.i8(i8 0)
+; CHECK-NEXT:    [[B:%.*]] = call i8 @llvm.bitreverse.i8(i8 15)
+; CHECK-NEXT:    store i8 [[TMP1]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i8 [[B]]
+;
+  %b = call i8 @llvm.bitreverse.i8(i8 15)
+  ret i8 %b
+}
+
+define i4 @fold_i4() #0 {
+; CHECK-LABEL: define i4 @fold_i4(
+; CHECK-SAME: ) #[[ATTR2]] {
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP1:%.*]] = call i4 @llvm.bitreverse.i4(i4 0)
+; CHECK-NEXT:    [[B:%.*]] = call i4 @llvm.bitreverse.i4(i4 -8)
+; CHECK-NEXT:    store i4 [[TMP1]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i4 [[B]]
+;
+  %b = call i4 @llvm.bitreverse.i4(i4 8)
+  ret i4 %b
+}
+
+; These tests check that bitreverse(bitreverse()) calls are removed
+
+define i8 @identity_i8(i8 %a) #0 {
+; CHECK-LABEL: define i8 @identity_i8(
+; CHECK-SAME: i8 [[A:%.*]]) #[[ATTR2]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i8, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = call i8 @llvm.bitreverse.i8(i8 [[TMP1]])
+; CHECK-NEXT:    [[B:%.*]] = call i8 @llvm.bitreverse.i8(i8 [[A]])
+; CHECK-NEXT:    [[TMP3:%.*]] = call i8 @llvm.bitreverse.i8(i8 [[TMP2]])
+; CHECK-NEXT:    [[C:%.*]] = call i8 @llvm.bitreverse.i8(i8 [[B]])
+; CHECK-NEXT:    store i8 [[TMP3]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i8 [[C]]
+;
+  %b = call i8 @llvm.bitreverse.i8(i8 %a)
+  %c = call i8 @llvm.bitreverse.i8(i8 %b)
+  ret i8 %c
+}
+
+define <2 x i16> @identity_v2i16(<2 x i16> %a) #0 {
+; CHECK-LABEL: define <2 x i16> @identity_v2i16(
+; CHECK-SAME: <2 x i16> [[A:%.*]]) #[[ATTR2]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = call <2 x i16> @llvm.bitreverse.v2i16(<2 x i16> [[TMP1]])
+; CHECK-NEXT:    [[B:%.*]] = call <2 x i16> @llvm.bitreverse.v2i16(<2 x i16> [[A]])
+; CHECK-NEXT:    [[TMP3:%.*]] = call <2 x i16> @llvm.bitreverse.v2i16(<2 x i16> [[TMP2]])
+; CHECK-NEXT:    [[C:%.*]] = call <2 x i16> @llvm.bitreverse.v2i16(<2 x i16> [[B]])
+; CHECK-NEXT:    store <2 x i16> [[TMP3]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x i16> [[C]]
+;
+  %b = call <2 x i16> @llvm.bitreverse.v2i16(<2 x i16> %a)
+  %c = call <2 x i16> @llvm.bitreverse.v2i16(<2 x i16> %b)
+  ret <2 x i16> %c
+}
+
+; Undef tests omitted
+
+; Make sure we don't assert during type legalization promoting a large
+; bitreverse due to the need for a large shift that won't fit in the i8 returned
+; from getShiftAmountTy.
+define i528 @large_promotion(i528 %A) nounwind #0 {
+; CHECK-LABEL: define i528 @large_promotion(
+; CHECK-SAME: i528 [[A:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i528, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = call i528 @llvm.bitreverse.i528(i528 [[TMP1]])
+; CHECK-NEXT:    [[Z:%.*]] = call i528 @llvm.bitreverse.i528(i528 [[A]])
+; CHECK-NEXT:    store i528 [[TMP2]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i528 [[Z]]
+;
+  %Z = call i528 @llvm.bitreverse.i528(i528 %A)
+  ret i528 %Z
+}
+declare i528 @llvm.bitreverse.i528(i528)
+
+attributes #0 = { sanitize_memory }
diff --git a/llvm/test/Instrumentation/MemorySanitizer/i386/avx-intrinsics-i386.ll b/llvm/test/Instrumentation/MemorySanitizer/i386/avx-intrinsics-i386.ll
index 68337d6d962db..08dd27ffcaaf1 100644
--- a/llvm/test/Instrumentation/MemorySanitizer/i386/avx-intrinsics-i386.ll
+++ b/llvm/test/Instrumentation/MemorySanitizer/i386/avx-intrinsics-i386.ll
@@ -491,7 +491,7 @@ define <4 x double> @test_x86_avx_hsub_pd_256(<4 x double> %a0, <4 x double> %a1
 ; CHECK-NEXT:    call void @llvm.donothing()
 ; CHECK-NEXT:    [[A0:%.*]] = bitcast <4 x i64> [[TMP1]] to <4 x double>
 ; CHECK-NEXT:    [[A1:%.*]] = bitcast <4 x i64> [[TMP2]] to <4 x double>
-; CHECK-NEXT:    [[RES:%.*]] = call <4 x double> @llvm.x86.avx.hsub.pd.256(<4 x double> [[A0]], <4 x double> [[A1]])
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x double> @llvm.x86.avx.hadd.pd.256(<4 x double> [[A0]], <4 x double> [[A1]])
 ; CHECK-NEXT:    [[_MSPROP:%.*]] = bitcast <4 x double> [[RES]] to <4 x i64>
 ; CHECK-NEXT:    [[RES1:%.*]] = call <4 x double> @llvm.x86.avx.hsub.pd.256(<4 x double> [[A2:%.*]], <4 x double> [[A3:%.*]])
 ; CHECK-NEXT:    store <4 x i64> [[_MSPROP]], ptr @__msan_retval_tls, align 8
@@ -511,7 +511,7 @@ define <8 x float> @test_x86_avx_hsub_ps_256(<8 x float> %a0, <8 x float> %a1) #
 ; CHECK-NEXT:    call void @llvm.donothing()
 ; CHECK-NEXT:    [[A0:%.*]] = bitcast <8 x i32> [[TMP1]] to <8 x float>
 ; CHECK-NEXT:    [[A1:%.*]] = bitcast <8 x i32> [[TMP2]] to <8 x float>
-; CHECK-NEXT:    [[RES:%.*]] = call <8 x float> @llvm.x86.avx.hsub.ps.256(<8 x float> [[A0]], <8 x float> [[A1]])
+; CHECK-NEXT:    [[RES:%.*]] = call <8 x float> @llvm.x86.avx.hadd.ps.256(<8 x float> [[A0]], <8 x float> [[A1]])
 ; CHECK-NEXT:    [[_MSPROP:%.*]] = bitcast <8 x float> [[RES]] to <8 x i32>
 ; CHECK-NEXT:    [[RES1:%.*]] = call <8 x float> @llvm.x86.avx.hsub.ps.256(<8 x float> [[A2:%.*]], <8 x float> [[A3:%.*]])
 ; CHECK-NEXT:    store <8 x i32> [[_MSPROP]], ptr @__msan_retval_tls, align 8
@@ -550,21 +550,23 @@ declare <32 x i8> @llvm.x86.avx.ldu.dq.256(ptr) nounwind readonly
 
 define <2 x double> @test_x86_avx_maskload_pd(ptr %a0, <2 x i64> %mask) #0 {
 ; CHECK-LABEL: @test_x86_avx_maskload_pd(
-; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
 ; CHECK-NEXT:    [[TMP5:%.*]] = load i64, ptr @__msan_va_arg_overflow_size_tls, align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    [[TMP8:%.*]] = ptrtoint ptr [[A0:%.*]] to i64
+; CHECK-NEXT:    [[TMP4:%.*]] = and i64 [[TMP8]], -2147483649
+; CHECK-NEXT:    [[TMP11:%.*]] = inttoptr i64 [[TMP4]] to ptr
+; CHECK-NEXT:    [[TMP6:%.*]] = call <2 x double> @llvm.x86.avx.maskload.pd(ptr [[TMP11]], <2 x i64> [[MASK:%.*]])
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <2 x double> [[TMP6]] to <2 x i64>
 ; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <2 x i64> [[TMP2]] to i128
 ; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP3]], 0
-; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
-; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP7:%.*]], label [[TMP6:%.*]], !prof [[PROF1]]
-; CHECK:       5:
+; CHECK-NEXT:    br i1 [[_MSCMP1]], label [[TMP9:%.*]], label [[TMP10:%.*]], !prof [[PROF1]]
+; CHECK:       9:
 ; CHECK-NEXT:    call void @__msan_warning_noreturn()
 ; CHECK-NEXT:    unreachable
-; CHECK:       6:
-; CHECK-NEXT:    [[RES:%.*]] = call <2 x double> @llvm.x86.avx.maskload.pd(ptr [[A0:%.*]], <2 x i64> [[MASK:%.*]])
-; CHECK-NEXT:    store <2 x i64> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK:       10:
+; CHECK-NEXT:    [[RES:%.*]] = call <2 x double> @llvm.x86.avx.maskload.pd(ptr [[A0]], <2 x i64> [[MASK]])
+; CHECK-NEXT:    store <2 x i64> [[TMP7]], ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret <2 x double> [[RES]]
 ;
   %res = call <2 x double> @llvm.x86.avx.maskload.pd(ptr %a0, <2 x i64> %mask) ; <<2 x double>> [#uses=1]
@@ -575,21 +577,23 @@ declare <2 x double> @llvm.x86.avx.maskload.pd(ptr, <2 x i64>) nounwind readonly
 
 define <4 x double> @test_x86_avx_maskload_pd_256(ptr %a0, <4 x i64> %mask) #0 {
 ; CHECK-LABEL: @test_x86_avx_maskload_pd_256(
-; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
 ; CHECK-NEXT:    [[TMP5:%.*]] = load i64, ptr @__msan_va_arg_overflow_size_tls, align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    [[TMP8:%.*]] = ptrtoint ptr [[A0:%.*]] to i64
+; CHECK-NEXT:    [[TMP4:%.*]] = and i64 [[TMP8]], -2147483649
+; CHECK-NEXT:    [[TMP11:%.*]] = inttoptr i64 [[TMP4]] to ptr
+; CHECK-NEXT:    [[TMP6:%.*]] = call <4 x double> @llvm.x86.avx.maskload.pd.256(ptr [[TMP11]], <4 x i64> [[MASK:%.*]])
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <4 x double> [[TMP6]] to <4 x i64>
 ; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <4 x i64> [[TMP2]] to i256
 ; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i256 [[TMP3]], 0
-; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
-; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP7:%.*]], label [[TMP6:%.*]], !prof [[PROF1]]
-; CHECK:       5:
+; CHECK-NEXT:    br i1 [[_MSCMP1]], label [[TMP9:%.*]], label [[TMP10:%.*]], !prof [[PROF1]]
+; CHECK:       9:
 ; CHECK-NEXT:    call void @__msan_warning_noreturn()
 ; CHECK-NEXT:    unreachable
-; CHECK:       6:
-; CHECK-NEXT:    [[RES:%.*]] = call <4 x double> @llvm.x86.avx.maskload.pd.256(ptr [[A0:%.*]], <4 x i64> [[MASK:%.*]])
-; CHECK-NEXT:    store <4 x i64> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK:       10:
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x double> @llvm.x86.avx.maskload.pd.256(ptr [[A0]], <4 x i64> [[MASK]])
+; CHECK-NEXT:    store <4 x i64> [[TMP7]], ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret <4 x double> [[RES]]
 ;
   %res = call <4 x double> @llvm.x86.avx.maskload.pd.256(ptr %a0, <4 x i64> %mask) ; <<4 x double>> [#uses=1]
@@ -600,21 +604,23 @@ declare <4 x double> @llvm.x86.avx.maskload.pd.256(ptr, <4 x i64>) nounwind read
 
 define <4 x float> @test_x86_avx_maskload_ps(ptr %a0, <4 x i32> %mask) #0 {
 ; CHECK-LABEL: @test_x86_avx_maskload_ps(
-; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
 ; CHECK-NEXT:    [[TMP5:%.*]] = load i64, ptr @__msan_va_arg_overflow_size_tls, align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    [[TMP8:%.*]] = ptrtoint ptr [[A0:%.*]] to i64
+; CHECK-NEXT:    [[TMP4:%.*]] = and i64 [[TMP8]], -2147483649
+; CHECK-NEXT:    [[TMP11:%.*]] = inttoptr i64 [[TMP4]] to ptr
+; CHECK-NEXT:    [[TMP6:%.*]] = call <4 x float> @llvm.x86.avx.maskload.ps(ptr [[TMP11]], <4 x i32> [[MASK:%.*]])
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <4 x float> [[TMP6]] to <4 x i32>
 ; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <4 x i32> [[TMP2]] to i128
 ; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP3]], 0
-; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
-; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP7:%.*]], label [[TMP6:%.*]], !prof [[PROF1]]
-; CHECK:       5:
+; CHECK-NEXT:    br i1 [[_MSCMP1]], label [[TMP9:%.*]], label [[TMP10:%.*]], !prof [[PROF1]]
+; CHECK:       9:
 ; CHECK-NEXT:    call void @__msan_warning_noreturn()
 ; CHECK-NEXT:    unreachable
-; CHECK:       6:
-; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.x86.avx.maskload.ps(ptr [[A0:%.*]], <4 x i32> [[MASK:%.*]])
-; CHECK-NEXT:    store <4 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK:       10:
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.x86.avx.maskload.ps(ptr [[A0]], <4 x i32> [[MASK]])
+; CHECK-NEXT:    store <4 x i32> [[TMP7]], ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret <4 x float> [[RES]]
 ;
   %res = call <4 x float> @llvm.x86.avx.maskload.ps(ptr %a0, <4 x i32> %mask) ; <<4 x float>> [#uses=1]
@@ -625,21 +631,23 @@ declare <4 x float> @llvm.x86.avx.maskload.ps(ptr, <4 x i32>) nounwind readonly
 
 define <8 x float> @test_x86_avx_maskload_ps_256(ptr %a0, <8 x i32> %mask) #0 {
 ; CHECK-LABEL: @test_x86_avx_maskload_ps_256(
-; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
 ; CHECK-NEXT:    [[TMP5:%.*]] = load i64, ptr @__msan_va_arg_overflow_size_tls, align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    [[TMP8:%.*]] = ptrtoint ptr [[A0:%.*]] to i64
+; CHECK-NEXT:    [[TMP4:%.*]] = and i64 [[TMP8]], -2147483649
+; CHECK-NEXT:    [[TMP11:%.*]] = inttoptr i64 [[TMP4]] to ptr
+; CHECK-NEXT:    [[TMP6:%.*]] = call <8 x float> @llvm.x86.avx.maskload.ps.256(ptr [[TMP11]], <8 x i32> [[MASK:%.*]])
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <8 x float> [[TMP6]] to <8 x i32>
 ; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <8 x i32> [[TMP2]] to i256
 ; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i256 [[TMP3]], 0
-; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
-; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP7:%.*]], label [[TMP6:%.*]], !prof [[PROF1]]
-; CHECK:       5:
+; CHECK-NEXT:    br i1 [[_MSCMP1]], label [[TMP9:%.*]], label [[TMP10:%.*]], !prof [[PROF1]]
+; CHECK:       9:
 ; CHECK-NEXT:    call void @__msan_warning_noreturn()
 ; CHECK-NEXT:    unreachable
-; CHECK:       6:
-; CHECK-NEXT:    [[RES:%.*]] = call <8 x float> @llvm.x86.avx.maskload.ps.256(ptr [[A0:%.*]], <8 x i32> [[MASK:%.*]])
-; CHECK-NEXT:    store <8 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK:       10:
+; CHECK-NEXT:    [[RES:%.*]] = call <8 x float> @llvm.x86.avx.maskload.ps.256(ptr [[A0]], <8 x i32> [[MASK]])
+; CHECK-NEXT:    store <8 x i32> [[TMP7]], ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret <8 x float> [[RES]]
 ;
   %res = call <8 x float> @llvm.x86.avx.maskload.ps.256(ptr %a0, <8 x i32> %mask) ; <<8 x float>> [#uses=1]
@@ -650,24 +658,26 @@ declare <8 x float> @llvm.x86.avx.maskload.ps.256(ptr, <8 x i32>) nounwind reado
 
 define void @test_x86_avx_maskstore_pd(ptr %a0, <2 x i64> %mask, <2 x double> %a2) #0 {
 ; CHECK-LABEL: @test_x86_avx_maskstore_pd(
+; CHECK-NEXT:    [[TMP3:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 24) to ptr), align 8
 ; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
-; CHECK-NEXT:    [[TMP3:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 24) to ptr), align 8
 ; CHECK-NEXT:    [[TMP6:%.*]] = load i64, ptr @__msan_va_arg_overflow_size_tls, align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP9:%.*]] = ptrtoint ptr [[A0:%.*]] to i64
+; CHECK-NEXT:    [[TMP12:%.*]] = and i64 [[TMP9]], -2147483649
+; CHECK-NEXT:    [[TMP7:%.*]] = inttoptr i64 [[TMP12]] to ptr
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <2 x i64> [[TMP3]] to <2 x double>
+; CHECK-NEXT:    call void @llvm.x86.avx.maskstore.pd(ptr [[TMP7]], <2 x i64> [[MASK:%.*]], <2 x double> [[TMP8]])
 ; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
-; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <2 x i64> [[TMP2]] to i128
-; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP4]], 0
-; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
-; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <2 x i64> [[TMP3]] to i128
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <2 x i64> [[TMP2]] to i128
 ; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i128 [[TMP5]], 0
-; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
-; CHECK-NEXT:    br i1 [[_MSOR3]], label [[TMP7:%.*]], label [[TMP8:%.*]], !prof [[PROF1]]
-; CHECK:       7:
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP10:%.*]], label [[TMP11:%.*]], !prof [[PROF1]]
+; CHECK:       10:
 ; CHECK-NEXT:    call void @__msan_warning_noreturn()
 ; CHECK-NEXT:    unreachable
-; CHECK:       8:
-; CHECK-NEXT:    call void @llvm.x86.avx.maskstore.pd(ptr [[A0:%.*]], <2 x i64> [[MASK:%.*]], <2 x double> [[A2:%.*]])
+; CHECK:       11:
+; CHECK-NEXT:    call void @llvm.x86.avx.maskstore.pd(ptr [[A0]], <2 x i64> [[MASK]], <2 x double> [[A2:%.*]])
 ; CHECK-NEXT:    ret void
 ;
   call void @llvm.x86.avx.maskstore.pd(ptr %a0, <2 x i64> %mask, <2 x double> %a2)
@@ -678,24 +688,26 @@ declare void @llvm.x86.avx.maskstore.pd(ptr, <2 x i64>, <2 x double>) nounwind
 
 define void @test_x86_avx_maskstore_pd_256(ptr %a0, <4 x i64> %mask, <4 x double> %a2) #0 {
 ; CHECK-LABEL: @test_x86_avx_maskstore_pd_256(
+; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 40) to ptr), align 8
 ; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
-; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 40) to ptr), align 8
 ; CHECK-NEXT:    [[TMP6:%.*]] = load i64, ptr @__msan_va_arg_overflow_size_tls, align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP9:%.*]] = ptrtoint ptr [[A0:%.*]] to i64
+; CHECK-NEXT:    [[TMP12:%.*]] = and i64 [[TMP9]], -2147483649
+; CHECK-NEXT:    [[TMP7:%.*]] = inttoptr i64 [[TMP12]] to ptr
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <4 x i64> [[TMP3]] to <4 x double>
+; CHECK-NEXT:    call void @llvm.x86.avx.maskstore.pd.256(ptr [[TMP7]], <4 x i64> [[MASK:%.*]], <4 x double> [[TMP8]])
 ; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
-; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <4 x i64> [[TMP2]] to i256
-; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i256 [[TMP4]], 0
-; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
-; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <4 x i64> [[TMP3]] to i256
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <4 x i64> [[TMP2]] to i256
 ; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i256 [[TMP5]], 0
-; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
-; CHECK-NEXT:    br i1 [[_MSOR3]], label [[TMP7:%.*]], label [[TMP8:%.*]], !prof [[PROF1]]
-; CHECK:       7:
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP10:%.*]], label [[TMP11:%.*]], !prof [[PROF1]]
+; CHECK:       10:
 ; CHECK-NEXT:    call void @__msan_warning_noreturn()
 ; CHECK-NEXT:    unreachable
-; CHECK:       8:
-; CHECK-NEXT:    call void @llvm.x86.avx.maskstore.pd.256(ptr [[A0:%.*]], <4 x i64> [[MASK:%.*]], <4 x double> [[A2:%.*]])
+; CHECK:       11:
+; CHECK-NEXT:    call void @llvm.x86.avx.maskstore.pd.256(ptr [[A0]], <4 x i64> [[MASK]], <4 x double> [[A2:%.*]])
 ; CHECK-NEXT:    ret void
 ;
   call void @llvm.x86.avx.maskstore.pd.256(ptr %a0, <4 x i64> %mask, <4 x double> %a2)
@@ -706,24 +718,26 @@ declare void @llvm.x86.avx.maskstore.pd.256(ptr, <4 x i64>, <4 x double>) nounwi
 
 define void @test_x86_avx_maskstore_ps(ptr %a0, <4 x i32> %mask, <4 x float> %a2) #0 {
 ; CHECK-LABEL: @test_x86_avx_maskstore_ps(
+; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 24) to ptr), align 8
 ; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
-; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 24) to ptr), align 8
 ; CHECK-NEXT:    [[TMP6:%.*]] = load i64, ptr @__msan_va_arg_overflow_size_tls, align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP9:%.*]] = ptrtoint ptr [[A0:%.*]] to i64
+; CHECK-NEXT:    [[TMP12:%.*]] = and i64 [[TMP9]], -2147483649
+; CHECK-NEXT:    [[TMP7:%.*]] = inttoptr i64 [[TMP12]] to ptr
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <4 x i32> [[TMP3]] to <4 x float>
+; CHECK-NEXT:    call void @llvm.x86.avx.maskstore.ps(ptr [[TMP7]], <4 x i32> [[MASK:%.*]], <4 x float> [[TMP8]])
 ; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
-; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <4 x i32> [[TMP2]] to i128
-; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP4]], 0
-; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
-; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <4 x i32> [[TMP3]] to i128
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <4 x i32> [[TMP2]] to i128
 ; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i128 [[TMP5]], 0
-; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
-; CHECK-NEXT:    br i1 [[_MSOR3]], label [[TMP7:%.*]], label [[TMP8:%.*]], !prof [[PROF1]]
-; CHECK:       7:
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP10:%.*]], label [[TMP11:%.*]], !prof [[PROF1]]
+; CHECK:       10:
 ; CHECK-NEXT:    call void @__msan_warning_noreturn()
 ; CHECK-NEXT:    unreachable
-; CHECK:       8:
-; CHECK-NEXT:    call void @llvm.x86.avx.maskstore.ps(ptr [[A0:%.*]], <4 x i32> [[MASK:%.*]], <4 x float> [[A2:%.*]])
+; CHECK:       11:
+; CHECK-NEXT:    call void @llvm.x86.avx.maskstore.ps(ptr [[A0]], <4 x i32> [[MASK]], <4 x float> [[A2:%.*]])
 ; CHECK-NEXT:    ret void
 ;
   call void @llvm.x86.avx.maskstore.ps(ptr %a0, <4 x i32> %mask, <4 x float> %a2)
@@ -734,24 +748,26 @@ declare void @llvm.x86.avx.maskstore.ps(ptr, <4 x i32>, <4 x float>) nounwind
 
 define void @test_x86_avx_maskstore_ps_256(ptr %a0, <8 x i32> %mask, <8 x float> %a2) #0 {
 ; CHECK-LABEL: @test_x86_avx_maskstore_ps_256(
+; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 40) to ptr), align 8
 ; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
-; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 40) to ptr), align 8
 ; CHECK-NEXT:    [[TMP6:%.*]] = load i64, ptr @__msan_va_arg_overflow_size_tls, align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP9:%.*]] = ptrtoint ptr [[A0:%.*]] to i64
+; CHECK-NEXT:    [[TMP12:%.*]] = and i64 [[TMP9]], -2147483649
+; CHECK-NEXT:    [[TMP7:%.*]] = inttoptr i64 [[TMP12]] to ptr
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <8 x i32> [[TMP3]] to <8 x float>
+; CHECK-NEXT:    call void @llvm.x86.avx.maskstore.ps.256(ptr [[TMP7]], <8 x i32> [[MASK:%.*]], <8 x float> [[TMP8]])
 ; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
-; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <8 x i32> [[TMP2]] to i256
-; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i256 [[TMP4]], 0
-; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
-; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <8 x i32> [[TMP3]] to i256
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <8 x i32> [[TMP2]] to i256
 ; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i256 [[TMP5]], 0
-; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
-; CHECK-NEXT:    br i1 [[_MSOR3]], label [[TMP7:%.*]], label [[TMP8:%.*]], !prof [[PROF1]]
-; CHECK:       7:
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP10:%.*]], label [[TMP11:%.*]], !prof [[PROF1]]
+; CHECK:       10:
 ; CHECK-NEXT:    call void @__msan_warning_noreturn()
 ; CHECK-NEXT:    unreachable
-; CHECK:       8:
-; CHECK-NEXT:    call void @llvm.x86.avx.maskstore.ps.256(ptr [[A0:%.*]], <8 x i32> [[MASK:%.*]], <8 x float> [[A2:%.*]])
+; CHECK:       11:
+; CHECK-NEXT:    call void @llvm.x86.avx.maskstore.ps.256(ptr [[A0]], <8 x i32> [[MASK]], <8 x float> [[A2:%.*]])
 ; CHECK-NEXT:    ret void
 ;
   call void @llvm.x86.avx.maskstore.ps.256(ptr %a0, <8 x i32> %mask, <8 x float> %a2)
diff --git a/llvm/test/Instrumentation/MemorySanitizer/i386/avx2-intrinsics-i386.ll b/llvm/test/Instrumentation/MemorySanitizer/i386/avx2-intrinsics-i386.ll
index 29e2931d2ca48..109166975eee1 100644
--- a/llvm/test/Instrumentation/MemorySanitizer/i386/avx2-intrinsics-i386.ll
+++ b/llvm/test/Instrumentation/MemorySanitizer/i386/avx2-intrinsics-i386.ll
@@ -651,7 +651,7 @@ define <8 x i32> @test_x86_avx2_phsub_d(<8 x i32> %a0, <8 x i32> %a1) #0 {
 ; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
 ; CHECK-NEXT:    [[TMP3:%.*]] = load i64, ptr @__msan_va_arg_overflow_size_tls, align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[_MSPROP:%.*]] = call <8 x i32> @llvm.x86.avx2.phsub.d(<8 x i32> [[TMP1]], <8 x i32> [[TMP2]])
+; CHECK-NEXT:    [[_MSPROP:%.*]] = call <8 x i32> @llvm.x86.avx2.phadd.d(<8 x i32> [[TMP1]], <8 x i32> [[TMP2]])
 ; CHECK-NEXT:    [[RES:%.*]] = call <8 x i32> @llvm.x86.avx2.phsub.d(<8 x i32> [[A0:%.*]], <8 x i32> [[A1:%.*]])
 ; CHECK-NEXT:    store <8 x i32> [[_MSPROP]], ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret <8 x i32> [[RES]]
@@ -668,7 +668,7 @@ define <16 x i16> @test_x86_avx2_phsub_sw(<16 x i16> %a0, <16 x i16> %a1) #0 {
 ; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
 ; CHECK-NEXT:    [[TMP3:%.*]] = load i64, ptr @__msan_va_arg_overflow_size_tls, align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[_MSPROP:%.*]] = call <16 x i16> @llvm.x86.avx2.phsub.sw(<16 x i16> [[TMP1]], <16 x i16> [[TMP2]])
+; CHECK-NEXT:    [[_MSPROP:%.*]] = call <16 x i16> @llvm.x86.avx2.phadd.sw(<16 x i16> [[TMP1]], <16 x i16> [[TMP2]])
 ; CHECK-NEXT:    [[RES:%.*]] = call <16 x i16> @llvm.x86.avx2.phsub.sw(<16 x i16> [[A0:%.*]], <16 x i16> [[A1:%.*]])
 ; CHECK-NEXT:    store <16 x i16> [[_MSPROP]], ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret <16 x i16> [[RES]]
@@ -685,7 +685,7 @@ define <16 x i16> @test_x86_avx2_phsub_w(<16 x i16> %a0, <16 x i16> %a1) #0 {
 ; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
 ; CHECK-NEXT:    [[TMP3:%.*]] = load i64, ptr @__msan_va_arg_overflow_size_tls, align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[_MSPROP:%.*]] = call <16 x i16> @llvm.x86.avx2.phsub.w(<16 x i16> [[TMP1]], <16 x i16> [[TMP2]])
+; CHECK-NEXT:    [[_MSPROP:%.*]] = call <16 x i16> @llvm.x86.avx2.phadd.w(<16 x i16> [[TMP1]], <16 x i16> [[TMP2]])
 ; CHECK-NEXT:    [[RES:%.*]] = call <16 x i16> @llvm.x86.avx2.phsub.w(<16 x i16> [[A0:%.*]], <16 x i16> [[A1:%.*]])
 ; CHECK-NEXT:    store <16 x i16> [[_MSPROP]], ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret <16 x i16> [[RES]]
@@ -1048,21 +1048,22 @@ declare <8 x float> @llvm.x86.avx2.permps(<8 x float>, <8 x i32>) nounwind reado
 
 define <2 x i64> @test_x86_avx2_maskload_q(ptr %a0, <2 x i64> %a1) #0 {
 ; CHECK-LABEL: @test_x86_avx2_maskload_q(
-; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
 ; CHECK-NEXT:    [[TMP5:%.*]] = load i64, ptr @__msan_va_arg_overflow_size_tls, align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    [[TMP7:%.*]] = ptrtoint ptr [[A0:%.*]] to i64
+; CHECK-NEXT:    [[TMP4:%.*]] = and i64 [[TMP7]], -2147483649
+; CHECK-NEXT:    [[TMP10:%.*]] = inttoptr i64 [[TMP4]] to ptr
+; CHECK-NEXT:    [[TMP6:%.*]] = call <2 x i64> @llvm.x86.avx2.maskload.q(ptr [[TMP10]], <2 x i64> [[A1:%.*]])
 ; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <2 x i64> [[TMP2]] to i128
 ; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP3]], 0
-; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
-; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP7:%.*]], label [[TMP6:%.*]], !prof [[PROF1]]
-; CHECK:       5:
+; CHECK-NEXT:    br i1 [[_MSCMP1]], label [[TMP8:%.*]], label [[TMP9:%.*]], !prof [[PROF1]]
+; CHECK:       8:
 ; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
 ; CHECK-NEXT:    unreachable
-; CHECK:       6:
-; CHECK-NEXT:    [[RES:%.*]] = call <2 x i64> @llvm.x86.avx2.maskload.q(ptr [[A0:%.*]], <2 x i64> [[A1:%.*]])
-; CHECK-NEXT:    store <2 x i64> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK:       9:
+; CHECK-NEXT:    [[RES:%.*]] = call <2 x i64> @llvm.x86.avx2.maskload.q(ptr [[A0]], <2 x i64> [[A1]])
+; CHECK-NEXT:    store <2 x i64> [[TMP6]], ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret <2 x i64> [[RES]]
 ;
   %res = call <2 x i64> @llvm.x86.avx2.maskload.q(ptr %a0, <2 x i64> %a1) ; <<2 x i64>> [#uses=1]
@@ -1073,21 +1074,22 @@ declare <2 x i64> @llvm.x86.avx2.maskload.q(ptr, <2 x i64>) nounwind readonly
 
 define <4 x i64> @test_x86_avx2_maskload_q_256(ptr %a0, <4 x i64> %a1) #0 {
 ; CHECK-LABEL: @test_x86_avx2_maskload_q_256(
-; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
 ; CHECK-NEXT:    [[TMP5:%.*]] = load i64, ptr @__msan_va_arg_overflow_size_tls, align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    [[TMP7:%.*]] = ptrtoint ptr [[A0:%.*]] to i64
+; CHECK-NEXT:    [[TMP4:%.*]] = and i64 [[TMP7]], -2147483649
+; CHECK-NEXT:    [[TMP10:%.*]] = inttoptr i64 [[TMP4]] to ptr
+; CHECK-NEXT:    [[TMP6:%.*]] = call <4 x i64> @llvm.x86.avx2.maskload.q.256(ptr [[TMP10]], <4 x i64> [[A1:%.*]])
 ; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <4 x i64> [[TMP2]] to i256
 ; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i256 [[TMP3]], 0
-; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
-; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP7:%.*]], label [[TMP6:%.*]], !prof [[PROF1]]
-; CHECK:       5:
+; CHECK-NEXT:    br i1 [[_MSCMP1]], label [[TMP8:%.*]], label [[TMP9:%.*]], !prof [[PROF1]]
+; CHECK:       8:
 ; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
 ; CHECK-NEXT:    unreachable
-; CHECK:       6:
-; CHECK-NEXT:    [[RES:%.*]] = call <4 x i64> @llvm.x86.avx2.maskload.q.256(ptr [[A0:%.*]], <4 x i64> [[A1:%.*]])
-; CHECK-NEXT:    store <4 x i64> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK:       9:
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x i64> @llvm.x86.avx2.maskload.q.256(ptr [[A0]], <4 x i64> [[A1]])
+; CHECK-NEXT:    store <4 x i64> [[TMP6]], ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret <4 x i64> [[RES]]
 ;
   %res = call <4 x i64> @llvm.x86.avx2.maskload.q.256(ptr %a0, <4 x i64> %a1) ; <<4 x i64>> [#uses=1]
@@ -1098,21 +1100,22 @@ declare <4 x i64> @llvm.x86.avx2.maskload.q.256(ptr, <4 x i64>) nounwind readonl
 
 define <4 x i32> @test_x86_avx2_maskload_d(ptr %a0, <4 x i32> %a1) #0 {
 ; CHECK-LABEL: @test_x86_avx2_maskload_d(
-; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
 ; CHECK-NEXT:    [[TMP5:%.*]] = load i64, ptr @__msan_va_arg_overflow_size_tls, align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    [[TMP7:%.*]] = ptrtoint ptr [[A0:%.*]] to i64
+; CHECK-NEXT:    [[TMP4:%.*]] = and i64 [[TMP7]], -2147483649
+; CHECK-NEXT:    [[TMP10:%.*]] = inttoptr i64 [[TMP4]] to ptr
+; CHECK-NEXT:    [[TMP6:%.*]] = call <4 x i32> @llvm.x86.avx2.maskload.d(ptr [[TMP10]], <4 x i32> [[A1:%.*]])
 ; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <4 x i32> [[TMP2]] to i128
 ; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP3]], 0
-; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
-; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP7:%.*]], label [[TMP6:%.*]], !prof [[PROF1]]
-; CHECK:       5:
+; CHECK-NEXT:    br i1 [[_MSCMP1]], label [[TMP8:%.*]], label [[TMP9:%.*]], !prof [[PROF1]]
+; CHECK:       8:
 ; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
 ; CHECK-NEXT:    unreachable
-; CHECK:       6:
-; CHECK-NEXT:    [[RES:%.*]] = call <4 x i32> @llvm.x86.avx2.maskload.d(ptr [[A0:%.*]], <4 x i32> [[A1:%.*]])
-; CHECK-NEXT:    store <4 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK:       9:
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x i32> @llvm.x86.avx2.maskload.d(ptr [[A0]], <4 x i32> [[A1]])
+; CHECK-NEXT:    store <4 x i32> [[TMP6]], ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret <4 x i32> [[RES]]
 ;
   %res = call <4 x i32> @llvm.x86.avx2.maskload.d(ptr %a0, <4 x i32> %a1) ; <<4 x i32>> [#uses=1]
@@ -1123,21 +1126,22 @@ declare <4 x i32> @llvm.x86.avx2.maskload.d(ptr, <4 x i32>) nounwind readonly
 
 define <8 x i32> @test_x86_avx2_maskload_d_256(ptr %a0, <8 x i32> %a1) #0 {
 ; CHECK-LABEL: @test_x86_avx2_maskload_d_256(
-; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
 ; CHECK-NEXT:    [[TMP5:%.*]] = load i64, ptr @__msan_va_arg_overflow_size_tls, align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    [[TMP7:%.*]] = ptrtoint ptr [[A0:%.*]] to i64
+; CHECK-NEXT:    [[TMP4:%.*]] = and i64 [[TMP7]], -2147483649
+; CHECK-NEXT:    [[TMP10:%.*]] = inttoptr i64 [[TMP4]] to ptr
+; CHECK-NEXT:    [[TMP6:%.*]] = call <8 x i32> @llvm.x86.avx2.maskload.d.256(ptr [[TMP10]], <8 x i32> [[A1:%.*]])
 ; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <8 x i32> [[TMP2]] to i256
 ; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i256 [[TMP3]], 0
-; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
-; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP7:%.*]], label [[TMP6:%.*]], !prof [[PROF1]]
-; CHECK:       5:
+; CHECK-NEXT:    br i1 [[_MSCMP1]], label [[TMP8:%.*]], label [[TMP9:%.*]], !prof [[PROF1]]
+; CHECK:       8:
 ; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
 ; CHECK-NEXT:    unreachable
-; CHECK:       6:
-; CHECK-NEXT:    [[RES:%.*]] = call <8 x i32> @llvm.x86.avx2.maskload.d.256(ptr [[A0:%.*]], <8 x i32> [[A1:%.*]])
-; CHECK-NEXT:    store <8 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK:       9:
+; CHECK-NEXT:    [[RES:%.*]] = call <8 x i32> @llvm.x86.avx2.maskload.d.256(ptr [[A0]], <8 x i32> [[A1]])
+; CHECK-NEXT:    store <8 x i32> [[TMP6]], ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret <8 x i32> [[RES]]
 ;
   %res = call <8 x i32> @llvm.x86.avx2.maskload.d.256(ptr %a0, <8 x i32> %a1) ; <<8 x i32>> [#uses=1]
@@ -1148,24 +1152,25 @@ declare <8 x i32> @llvm.x86.avx2.maskload.d.256(ptr, <8 x i32>) nounwind readonl
 
 define void @test_x86_avx2_maskstore_q(ptr %a0, <2 x i64> %a1, <2 x i64> %a2) #0 {
 ; CHECK-LABEL: @test_x86_avx2_maskstore_q(
+; CHECK-NEXT:    [[TMP3:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 24) to ptr), align 8
 ; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
-; CHECK-NEXT:    [[TMP3:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 24) to ptr), align 8
 ; CHECK-NEXT:    [[TMP6:%.*]] = load i64, ptr @__msan_va_arg_overflow_size_tls, align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP8:%.*]] = ptrtoint ptr [[A0:%.*]] to i64
+; CHECK-NEXT:    [[TMP11:%.*]] = and i64 [[TMP8]], -2147483649
+; CHECK-NEXT:    [[TMP7:%.*]] = inttoptr i64 [[TMP11]] to ptr
+; CHECK-NEXT:    call void @llvm.x86.avx2.maskstore.q(ptr [[TMP7]], <2 x i64> [[A1:%.*]], <2 x i64> [[TMP3]])
 ; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
-; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <2 x i64> [[TMP2]] to i128
-; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP4]], 0
-; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
-; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <2 x i64> [[TMP3]] to i128
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <2 x i64> [[TMP2]] to i128
 ; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i128 [[TMP5]], 0
-; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
-; CHECK-NEXT:    br i1 [[_MSOR3]], label [[TMP7:%.*]], label [[TMP8:%.*]], !prof [[PROF1]]
-; CHECK:       7:
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP9:%.*]], label [[TMP10:%.*]], !prof [[PROF1]]
+; CHECK:       9:
 ; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
 ; CHECK-NEXT:    unreachable
-; CHECK:       8:
-; CHECK-NEXT:    call void @llvm.x86.avx2.maskstore.q(ptr [[A0:%.*]], <2 x i64> [[A1:%.*]], <2 x i64> [[A2:%.*]])
+; CHECK:       10:
+; CHECK-NEXT:    call void @llvm.x86.avx2.maskstore.q(ptr [[A0]], <2 x i64> [[A1]], <2 x i64> [[A2:%.*]])
 ; CHECK-NEXT:    ret void
 ;
   call void @llvm.x86.avx2.maskstore.q(ptr %a0, <2 x i64> %a1, <2 x i64> %a2)
@@ -1176,24 +1181,25 @@ declare void @llvm.x86.avx2.maskstore.q(ptr, <2 x i64>, <2 x i64>) nounwind
 
 define void @test_x86_avx2_maskstore_q_256(ptr %a0, <4 x i64> %a1, <4 x i64> %a2) #0 {
 ; CHECK-LABEL: @test_x86_avx2_maskstore_q_256(
+; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 40) to ptr), align 8
 ; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
-; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 40) to ptr), align 8
 ; CHECK-NEXT:    [[TMP6:%.*]] = load i64, ptr @__msan_va_arg_overflow_size_tls, align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP8:%.*]] = ptrtoint ptr [[A0:%.*]] to i64
+; CHECK-NEXT:    [[TMP11:%.*]] = and i64 [[TMP8]], -2147483649
+; CHECK-NEXT:    [[TMP7:%.*]] = inttoptr i64 [[TMP11]] to ptr
+; CHECK-NEXT:    call void @llvm.x86.avx2.maskstore.q.256(ptr [[TMP7]], <4 x i64> [[A1:%.*]], <4 x i64> [[TMP3]])
 ; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
-; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <4 x i64> [[TMP2]] to i256
-; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i256 [[TMP4]], 0
-; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
-; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <4 x i64> [[TMP3]] to i256
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <4 x i64> [[TMP2]] to i256
 ; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i256 [[TMP5]], 0
-; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
-; CHECK-NEXT:    br i1 [[_MSOR3]], label [[TMP7:%.*]], label [[TMP8:%.*]], !prof [[PROF1]]
-; CHECK:       7:
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP9:%.*]], label [[TMP10:%.*]], !prof [[PROF1]]
+; CHECK:       9:
 ; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
 ; CHECK-NEXT:    unreachable
-; CHECK:       8:
-; CHECK-NEXT:    call void @llvm.x86.avx2.maskstore.q.256(ptr [[A0:%.*]], <4 x i64> [[A1:%.*]], <4 x i64> [[A2:%.*]])
+; CHECK:       10:
+; CHECK-NEXT:    call void @llvm.x86.avx2.maskstore.q.256(ptr [[A0]], <4 x i64> [[A1]], <4 x i64> [[A2:%.*]])
 ; CHECK-NEXT:    ret void
 ;
   call void @llvm.x86.avx2.maskstore.q.256(ptr %a0, <4 x i64> %a1, <4 x i64> %a2)
@@ -1204,24 +1210,25 @@ declare void @llvm.x86.avx2.maskstore.q.256(ptr, <4 x i64>, <4 x i64>) nounwind
 
 define void @test_x86_avx2_maskstore_d(ptr %a0, <4 x i32> %a1, <4 x i32> %a2) #0 {
 ; CHECK-LABEL: @test_x86_avx2_maskstore_d(
+; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 24) to ptr), align 8
 ; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
-; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 24) to ptr), align 8
 ; CHECK-NEXT:    [[TMP6:%.*]] = load i64, ptr @__msan_va_arg_overflow_size_tls, align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP8:%.*]] = ptrtoint ptr [[A0:%.*]] to i64
+; CHECK-NEXT:    [[TMP11:%.*]] = and i64 [[TMP8]], -2147483649
+; CHECK-NEXT:    [[TMP7:%.*]] = inttoptr i64 [[TMP11]] to ptr
+; CHECK-NEXT:    call void @llvm.x86.avx2.maskstore.d(ptr [[TMP7]], <4 x i32> [[A1:%.*]], <4 x i32> [[TMP3]])
 ; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
-; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <4 x i32> [[TMP2]] to i128
-; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP4]], 0
-; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
-; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <4 x i32> [[TMP3]] to i128
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <4 x i32> [[TMP2]] to i128
 ; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i128 [[TMP5]], 0
-; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
-; CHECK-NEXT:    br i1 [[_MSOR3]], label [[TMP7:%.*]], label [[TMP8:%.*]], !prof [[PROF1]]
-; CHECK:       7:
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP9:%.*]], label [[TMP10:%.*]], !prof [[PROF1]]
+; CHECK:       9:
 ; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
 ; CHECK-NEXT:    unreachable
-; CHECK:       8:
-; CHECK-NEXT:    call void @llvm.x86.avx2.maskstore.d(ptr [[A0:%.*]], <4 x i32> [[A1:%.*]], <4 x i32> [[A2:%.*]])
+; CHECK:       10:
+; CHECK-NEXT:    call void @llvm.x86.avx2.maskstore.d(ptr [[A0]], <4 x i32> [[A1]], <4 x i32> [[A2:%.*]])
 ; CHECK-NEXT:    ret void
 ;
   call void @llvm.x86.avx2.maskstore.d(ptr %a0, <4 x i32> %a1, <4 x i32> %a2)
@@ -1232,24 +1239,25 @@ declare void @llvm.x86.avx2.maskstore.d(ptr, <4 x i32>, <4 x i32>) nounwind
 
 define void @test_x86_avx2_maskstore_d_256(ptr %a0, <8 x i32> %a1, <8 x i32> %a2) #0 {
 ; CHECK-LABEL: @test_x86_avx2_maskstore_d_256(
+; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 40) to ptr), align 8
 ; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
-; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 40) to ptr), align 8
 ; CHECK-NEXT:    [[TMP6:%.*]] = load i64, ptr @__msan_va_arg_overflow_size_tls, align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP8:%.*]] = ptrtoint ptr [[A0:%.*]] to i64
+; CHECK-NEXT:    [[TMP11:%.*]] = and i64 [[TMP8]], -2147483649
+; CHECK-NEXT:    [[TMP7:%.*]] = inttoptr i64 [[TMP11]] to ptr
+; CHECK-NEXT:    call void @llvm.x86.avx2.maskstore.d.256(ptr [[TMP7]], <8 x i32> [[A1:%.*]], <8 x i32> [[TMP3]])
 ; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
-; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <8 x i32> [[TMP2]] to i256
-; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i256 [[TMP4]], 0
-; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
-; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <8 x i32> [[TMP3]] to i256
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <8 x i32> [[TMP2]] to i256
 ; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i256 [[TMP5]], 0
-; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
-; CHECK-NEXT:    br i1 [[_MSOR3]], label [[TMP7:%.*]], label [[TMP8:%.*]], !prof [[PROF1]]
-; CHECK:       7:
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP9:%.*]], label [[TMP10:%.*]], !prof [[PROF1]]
+; CHECK:       9:
 ; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
 ; CHECK-NEXT:    unreachable
-; CHECK:       8:
-; CHECK-NEXT:    call void @llvm.x86.avx2.maskstore.d.256(ptr [[A0:%.*]], <8 x i32> [[A1:%.*]], <8 x i32> [[A2:%.*]])
+; CHECK:       10:
+; CHECK-NEXT:    call void @llvm.x86.avx2.maskstore.d.256(ptr [[A0]], <8 x i32> [[A1]], <8 x i32> [[A2:%.*]])
 ; CHECK-NEXT:    ret void
 ;
   call void @llvm.x86.avx2.maskstore.d.256(ptr %a0, <8 x i32> %a1, <8 x i32> %a2)
diff --git a/llvm/test/Instrumentation/MemorySanitizer/i386/mmx-intrinsics.ll b/llvm/test/Instrumentation/MemorySanitizer/i386/mmx-intrinsics.ll
index f5ef8a9837b70..15bd1755d479b 100644
--- a/llvm/test/Instrumentation/MemorySanitizer/i386/mmx-intrinsics.ll
+++ b/llvm/test/Instrumentation/MemorySanitizer/i386/mmx-intrinsics.ll
@@ -3426,7 +3426,7 @@ define i64 @test6(<1 x i64> %a, <1 x i64> %b) #0 {
 ; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <4 x i16> [[TMP1]] to <1 x i64>
 ; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <4 x i16> [[TMP12]] to <1 x i64>
 ; CHECK-NEXT:    [[TMP17:%.*]] = bitcast <4 x i16> [[TMP0]] to <1 x i64>
-; CHECK-NEXT:    [[_MSPROP:%.*]] = call <1 x i64> @llvm.x86.ssse3.phsub.sw(<1 x i64> [[TMP16]], <1 x i64> [[TMP8]])
+; CHECK-NEXT:    [[_MSPROP:%.*]] = call <1 x i64> @llvm.x86.ssse3.phadd.sw(<1 x i64> [[TMP16]], <1 x i64> [[TMP8]])
 ; CHECK-NEXT:    [[TMP18:%.*]] = tail call <1 x i64> @llvm.x86.ssse3.phsub.sw(<1 x i64> [[TMP2]], <1 x i64> [[TMP17]]) #[[ATTR5]]
 ; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <1 x i64> [[_MSPROP]] to <4 x i16>
 ; CHECK-NEXT:    [[TMP19:%.*]] = bitcast <1 x i64> [[TMP18]] to <4 x i16>
@@ -3467,7 +3467,7 @@ define i64 @test5(<1 x i64> %a, <1 x i64> %b) #0 {
 ; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <2 x i32> [[TMP1]] to <1 x i64>
 ; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <2 x i32> [[TMP12]] to <1 x i64>
 ; CHECK-NEXT:    [[TMP17:%.*]] = bitcast <2 x i32> [[TMP0]] to <1 x i64>
-; CHECK-NEXT:    [[_MSPROP:%.*]] = call <1 x i64> @llvm.x86.ssse3.phsub.d(<1 x i64> [[TMP16]], <1 x i64> [[TMP8]])
+; CHECK-NEXT:    [[_MSPROP:%.*]] = call <1 x i64> @llvm.x86.ssse3.phadd.d(<1 x i64> [[TMP16]], <1 x i64> [[TMP8]])
 ; CHECK-NEXT:    [[TMP18:%.*]] = tail call <1 x i64> @llvm.x86.ssse3.phsub.d(<1 x i64> [[TMP2]], <1 x i64> [[TMP17]]) #[[ATTR5]]
 ; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <1 x i64> [[_MSPROP]] to <2 x i32>
 ; CHECK-NEXT:    [[TMP19:%.*]] = bitcast <1 x i64> [[TMP18]] to <2 x i32>
@@ -3508,7 +3508,7 @@ define i64 @test4(<1 x i64> %a, <1 x i64> %b) #0 {
 ; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <4 x i16> [[TMP1]] to <1 x i64>
 ; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <4 x i16> [[TMP12]] to <1 x i64>
 ; CHECK-NEXT:    [[TMP17:%.*]] = bitcast <4 x i16> [[TMP0]] to <1 x i64>
-; CHECK-NEXT:    [[_MSPROP:%.*]] = call <1 x i64> @llvm.x86.ssse3.phsub.w(<1 x i64> [[TMP16]], <1 x i64> [[TMP8]])
+; CHECK-NEXT:    [[_MSPROP:%.*]] = call <1 x i64> @llvm.x86.ssse3.phadd.w(<1 x i64> [[TMP16]], <1 x i64> [[TMP8]])
 ; CHECK-NEXT:    [[TMP18:%.*]] = tail call <1 x i64> @llvm.x86.ssse3.phsub.w(<1 x i64> [[TMP2]], <1 x i64> [[TMP17]]) #[[ATTR5]]
 ; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <1 x i64> [[_MSPROP]] to <4 x i16>
 ; CHECK-NEXT:    [[TMP19:%.*]] = bitcast <1 x i64> [[TMP18]] to <4 x i16>
diff --git a/llvm/test/Instrumentation/MemorySanitizer/msan_basic.ll b/llvm/test/Instrumentation/MemorySanitizer/msan_basic.ll
index 3ce200e2222df..b4feb1ec57224 100644
--- a/llvm/test/Instrumentation/MemorySanitizer/msan_basic.ll
+++ b/llvm/test/Instrumentation/MemorySanitizer/msan_basic.ll
@@ -16,7 +16,7 @@ target triple = "x86_64-unknown-linux-gnu"
 
 define void @Store(ptr nocapture %p, i32 %x) nounwind uwtable sanitize_memory {
 ; CHECK-LABEL: define void @Store(
-; CHECK-SAME: ptr nocapture [[P:%.*]], i32 [[X:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-SAME: ptr captures(none) [[P:%.*]], i32 [[X:%.*]]) #[[ATTR0:[0-9]+]] {
 ; CHECK-NEXT:  [[ENTRY:.*:]]
 ; CHECK-NEXT:    [[TMP0:%.*]] = load i32, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
@@ -28,7 +28,7 @@ define void @Store(ptr nocapture %p, i32 %x) nounwind uwtable sanitize_memory {
 ; CHECK-NEXT:    ret void
 ;
 ; ORIGIN-LABEL: define void @Store(
-; ORIGIN-SAME: ptr nocapture [[P:%.*]], i32 [[X:%.*]]) #[[ATTR0:[0-9]+]] {
+; ORIGIN-SAME: ptr captures(none) [[P:%.*]], i32 [[X:%.*]]) #[[ATTR0:[0-9]+]] {
 ; ORIGIN-NEXT:  [[ENTRY:.*:]]
 ; ORIGIN-NEXT:    [[TMP0:%.*]] = load i32, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
 ; ORIGIN-NEXT:    [[TMP1:%.*]] = load i32, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_origin_tls to i64), i64 8) to ptr), align 4
@@ -49,7 +49,7 @@ define void @Store(ptr nocapture %p, i32 %x) nounwind uwtable sanitize_memory {
 ; ORIGIN-NEXT:    ret void
 ;
 ; CALLS-LABEL: define void @Store(
-; CALLS-SAME: ptr nocapture [[P:%.*]], i32 [[X:%.*]]) #[[ATTR0:[0-9]+]] {
+; CALLS-SAME: ptr captures(none) [[P:%.*]], i32 [[X:%.*]]) #[[ATTR0:[0-9]+]] {
 ; CALLS-NEXT:  [[ENTRY:.*:]]
 ; CALLS-NEXT:    [[TMP0:%.*]] = load i64, ptr @__msan_param_tls, align 8
 ; CALLS-NEXT:    [[TMP1:%.*]] = load i32, ptr @__msan_param_origin_tls, align 4
@@ -78,7 +78,7 @@ entry:
 
 define void @AlignedStore(ptr nocapture %p, i32 %x) nounwind uwtable sanitize_memory {
 ; CHECK-LABEL: define void @AlignedStore(
-; CHECK-SAME: ptr nocapture [[P:%.*]], i32 [[X:%.*]]) #[[ATTR0]] {
+; CHECK-SAME: ptr captures(none) [[P:%.*]], i32 [[X:%.*]]) #[[ATTR0]] {
 ; CHECK-NEXT:  [[ENTRY:.*:]]
 ; CHECK-NEXT:    [[TMP0:%.*]] = load i32, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
@@ -90,7 +90,7 @@ define void @AlignedStore(ptr nocapture %p, i32 %x) nounwind uwtable sanitize_me
 ; CHECK-NEXT:    ret void
 ;
 ; ORIGIN-LABEL: define void @AlignedStore(
-; ORIGIN-SAME: ptr nocapture [[P:%.*]], i32 [[X:%.*]]) #[[ATTR0]] {
+; ORIGIN-SAME: ptr captures(none) [[P:%.*]], i32 [[X:%.*]]) #[[ATTR0]] {
 ; ORIGIN-NEXT:  [[ENTRY:.*:]]
 ; ORIGIN-NEXT:    [[TMP0:%.*]] = load i32, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
 ; ORIGIN-NEXT:    [[TMP1:%.*]] = load i32, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_origin_tls to i64), i64 8) to ptr), align 4
@@ -114,7 +114,7 @@ define void @AlignedStore(ptr nocapture %p, i32 %x) nounwind uwtable sanitize_me
 ; ORIGIN-NEXT:    ret void
 ;
 ; CALLS-LABEL: define void @AlignedStore(
-; CALLS-SAME: ptr nocapture [[P:%.*]], i32 [[X:%.*]]) #[[ATTR0]] {
+; CALLS-SAME: ptr captures(none) [[P:%.*]], i32 [[X:%.*]]) #[[ATTR0]] {
 ; CALLS-NEXT:  [[ENTRY:.*:]]
 ; CALLS-NEXT:    [[TMP0:%.*]] = load i64, ptr @__msan_param_tls, align 8
 ; CALLS-NEXT:    [[TMP1:%.*]] = load i32, ptr @__msan_param_origin_tls, align 4
@@ -140,7 +140,7 @@ entry:
 ; load followed by cmp: check that we load the shadow and call __msan_warning_with_origin.
 define void @LoadAndCmp(ptr nocapture %a) nounwind uwtable sanitize_memory {
 ; CHECK-LABEL: define void @LoadAndCmp(
-; CHECK-SAME: ptr nocapture [[A:%.*]]) #[[ATTR0]] {
+; CHECK-SAME: ptr captures(none) [[A:%.*]]) #[[ATTR0]] {
 ; CHECK-NEXT:  [[ENTRY:.*:]]
 ; CHECK-NEXT:    call void @llvm.donothing()
 ; CHECK-NEXT:    [[TMP0:%.*]] = load i32, ptr [[A]], align 4
@@ -170,7 +170,7 @@ define void @LoadAndCmp(ptr nocapture %a) nounwind uwtable sanitize_memory {
 ; CHECK-NEXT:    ret void
 ;
 ; ORIGIN-LABEL: define void @LoadAndCmp(
-; ORIGIN-SAME: ptr nocapture [[A:%.*]]) #[[ATTR0]] {
+; ORIGIN-SAME: ptr captures(none) [[A:%.*]]) #[[ATTR0]] {
 ; ORIGIN-NEXT:  [[ENTRY:.*:]]
 ; ORIGIN-NEXT:    call void @llvm.donothing()
 ; ORIGIN-NEXT:    [[TMP0:%.*]] = load i32, ptr [[A]], align 4
@@ -203,7 +203,7 @@ define void @LoadAndCmp(ptr nocapture %a) nounwind uwtable sanitize_memory {
 ; ORIGIN-NEXT:    ret void
 ;
 ; CALLS-LABEL: define void @LoadAndCmp(
-; CALLS-SAME: ptr nocapture [[A:%.*]]) #[[ATTR0]] {
+; CALLS-SAME: ptr captures(none) [[A:%.*]]) #[[ATTR0]] {
 ; CALLS-NEXT:  [[ENTRY:.*:]]
 ; CALLS-NEXT:    [[TMP0:%.*]] = load i64, ptr @__msan_param_tls, align 8
 ; CALLS-NEXT:    [[TMP1:%.*]] = load i32, ptr @__msan_param_origin_tls, align 4
@@ -283,7 +283,7 @@ entry:
 ; Check that we get the shadow for the retval.
 define void @CopyRetVal(ptr nocapture %a) nounwind uwtable sanitize_memory {
 ; CHECK-LABEL: define void @CopyRetVal(
-; CHECK-SAME: ptr nocapture [[A:%.*]]) #[[ATTR0]] {
+; CHECK-SAME: ptr captures(none) [[A:%.*]]) #[[ATTR0]] {
 ; CHECK-NEXT:  [[ENTRY:.*:]]
 ; CHECK-NEXT:    call void @llvm.donothing()
 ; CHECK-NEXT:    store i32 0, ptr @__msan_retval_tls, align 8
@@ -297,7 +297,7 @@ define void @CopyRetVal(ptr nocapture %a) nounwind uwtable sanitize_memory {
 ; CHECK-NEXT:    ret void
 ;
 ; ORIGIN-LABEL: define void @CopyRetVal(
-; ORIGIN-SAME: ptr nocapture [[A:%.*]]) #[[ATTR0]] {
+; ORIGIN-SAME: ptr captures(none) [[A:%.*]]) #[[ATTR0]] {
 ; ORIGIN-NEXT:  [[ENTRY:.*:]]
 ; ORIGIN-NEXT:    call void @llvm.donothing()
 ; ORIGIN-NEXT:    store i32 0, ptr @__msan_retval_tls, align 8
@@ -320,7 +320,7 @@ define void @CopyRetVal(ptr nocapture %a) nounwind uwtable sanitize_memory {
 ; ORIGIN-NEXT:    ret void
 ;
 ; CALLS-LABEL: define void @CopyRetVal(
-; CALLS-SAME: ptr nocapture [[A:%.*]]) #[[ATTR0]] {
+; CALLS-SAME: ptr captures(none) [[A:%.*]]) #[[ATTR0]] {
 ; CALLS-NEXT:  [[ENTRY:.*:]]
 ; CALLS-NEXT:    [[TMP0:%.*]] = load i64, ptr @__msan_param_tls, align 8
 ; CALLS-NEXT:    [[TMP1:%.*]] = load i32, ptr @__msan_param_origin_tls, align 4
@@ -351,7 +351,7 @@ entry:
 ; Check that we generate PHIs for shadow.
 define void @FuncWithPhi(ptr nocapture %a, ptr %b, ptr nocapture %c) nounwind uwtable sanitize_memory {
 ; CHECK-LABEL: define void @FuncWithPhi(
-; CHECK-SAME: ptr nocapture [[A:%.*]], ptr [[B:%.*]], ptr nocapture [[C:%.*]]) #[[ATTR0]] {
+; CHECK-SAME: ptr captures(none) [[A:%.*]], ptr [[B:%.*]], ptr captures(none) [[C:%.*]]) #[[ATTR0]] {
 ; CHECK-NEXT:  [[ENTRY:.*:]]
 ; CHECK-NEXT:    [[TMP0:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
@@ -395,7 +395,7 @@ define void @FuncWithPhi(ptr nocapture %a, ptr %b, ptr nocapture %c) nounwind uw
 ; CHECK-NEXT:    ret void
 ;
 ; ORIGIN-LABEL: define void @FuncWithPhi(
-; ORIGIN-SAME: ptr nocapture [[A:%.*]], ptr [[B:%.*]], ptr nocapture [[C:%.*]]) #[[ATTR0]] {
+; ORIGIN-SAME: ptr captures(none) [[A:%.*]], ptr [[B:%.*]], ptr captures(none) [[C:%.*]]) #[[ATTR0]] {
 ; ORIGIN-NEXT:  [[ENTRY:.*:]]
 ; ORIGIN-NEXT:    [[TMP0:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
 ; ORIGIN-NEXT:    [[TMP1:%.*]] = load i32, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_origin_tls to i64), i64 8) to ptr), align 4
@@ -455,7 +455,7 @@ define void @FuncWithPhi(ptr nocapture %a, ptr %b, ptr nocapture %c) nounwind uw
 ; ORIGIN-NEXT:    ret void
 ;
 ; CALLS-LABEL: define void @FuncWithPhi(
-; CALLS-SAME: ptr nocapture [[A:%.*]], ptr [[B:%.*]], ptr nocapture [[C:%.*]]) #[[ATTR0]] {
+; CALLS-SAME: ptr captures(none) [[A:%.*]], ptr [[B:%.*]], ptr captures(none) [[C:%.*]]) #[[ATTR0]] {
 ; CALLS-NEXT:  [[ENTRY:.*:]]
 ; CALLS-NEXT:    [[TMP0:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
 ; CALLS-NEXT:    [[TMP1:%.*]] = load i32, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_origin_tls to i64), i64 8) to ptr), align 4
@@ -534,7 +534,7 @@ entry:
 ; Compute shadow for "x << 10"
 define void @ShlConst(ptr nocapture %x) nounwind uwtable sanitize_memory {
 ; CHECK-LABEL: define void @ShlConst(
-; CHECK-SAME: ptr nocapture [[X:%.*]]) #[[ATTR0]] {
+; CHECK-SAME: ptr captures(none) [[X:%.*]]) #[[ATTR0]] {
 ; CHECK-NEXT:  [[ENTRY:.*:]]
 ; CHECK-NEXT:    call void @llvm.donothing()
 ; CHECK-NEXT:    [[TMP0:%.*]] = load i32, ptr [[X]], align 4
@@ -553,7 +553,7 @@ define void @ShlConst(ptr nocapture %x) nounwind uwtable sanitize_memory {
 ; CHECK-NEXT:    ret void
 ;
 ; ORIGIN-LABEL: define void @ShlConst(
-; ORIGIN-SAME: ptr nocapture [[X:%.*]]) #[[ATTR0]] {
+; ORIGIN-SAME: ptr captures(none) [[X:%.*]]) #[[ATTR0]] {
 ; ORIGIN-NEXT:  [[ENTRY:.*:]]
 ; ORIGIN-NEXT:    call void @llvm.donothing()
 ; ORIGIN-NEXT:    [[TMP0:%.*]] = load i32, ptr [[X]], align 4
@@ -583,7 +583,7 @@ define void @ShlConst(ptr nocapture %x) nounwind uwtable sanitize_memory {
 ; ORIGIN-NEXT:    ret void
 ;
 ; CALLS-LABEL: define void @ShlConst(
-; CALLS-SAME: ptr nocapture [[X:%.*]]) #[[ATTR0]] {
+; CALLS-SAME: ptr captures(none) [[X:%.*]]) #[[ATTR0]] {
 ; CALLS-NEXT:  [[ENTRY:.*:]]
 ; CALLS-NEXT:    [[TMP0:%.*]] = load i64, ptr @__msan_param_tls, align 8
 ; CALLS-NEXT:    [[TMP1:%.*]] = load i32, ptr @__msan_param_origin_tls, align 4
@@ -622,7 +622,7 @@ entry:
 ; Compute shadow for "10 << x": it should have 'sext i1'.
 define void @ShlNonConst(ptr nocapture %x) nounwind uwtable sanitize_memory {
 ; CHECK-LABEL: define void @ShlNonConst(
-; CHECK-SAME: ptr nocapture [[X:%.*]]) #[[ATTR0]] {
+; CHECK-SAME: ptr captures(none) [[X:%.*]]) #[[ATTR0]] {
 ; CHECK-NEXT:  [[ENTRY:.*:]]
 ; CHECK-NEXT:    call void @llvm.donothing()
 ; CHECK-NEXT:    [[TMP0:%.*]] = load i32, ptr [[X]], align 4
@@ -643,7 +643,7 @@ define void @ShlNonConst(ptr nocapture %x) nounwind uwtable sanitize_memory {
 ; CHECK-NEXT:    ret void
 ;
 ; ORIGIN-LABEL: define void @ShlNonConst(
-; ORIGIN-SAME: ptr nocapture [[X:%.*]]) #[[ATTR0]] {
+; ORIGIN-SAME: ptr captures(none) [[X:%.*]]) #[[ATTR0]] {
 ; ORIGIN-NEXT:  [[ENTRY:.*:]]
 ; ORIGIN-NEXT:    call void @llvm.donothing()
 ; ORIGIN-NEXT:    [[TMP0:%.*]] = load i32, ptr [[X]], align 4
@@ -677,7 +677,7 @@ define void @ShlNonConst(ptr nocapture %x) nounwind uwtable sanitize_memory {
 ; ORIGIN-NEXT:    ret void
 ;
 ; CALLS-LABEL: define void @ShlNonConst(
-; CALLS-SAME: ptr nocapture [[X:%.*]]) #[[ATTR0]] {
+; CALLS-SAME: ptr captures(none) [[X:%.*]]) #[[ATTR0]] {
 ; CALLS-NEXT:  [[ENTRY:.*:]]
 ; CALLS-NEXT:    [[TMP0:%.*]] = load i64, ptr @__msan_param_tls, align 8
 ; CALLS-NEXT:    [[TMP1:%.*]] = load i32, ptr @__msan_param_origin_tls, align 4
@@ -720,7 +720,7 @@ entry:
 ; SExt
 define void @SExt(ptr nocapture %a, ptr nocapture %b) nounwind uwtable sanitize_memory {
 ; CHECK-LABEL: define void @SExt(
-; CHECK-SAME: ptr nocapture [[A:%.*]], ptr nocapture [[B:%.*]]) #[[ATTR0]] {
+; CHECK-SAME: ptr captures(none) [[A:%.*]], ptr captures(none) [[B:%.*]]) #[[ATTR0]] {
 ; CHECK-NEXT:  [[ENTRY:.*:]]
 ; CHECK-NEXT:    call void @llvm.donothing()
 ; CHECK-NEXT:    [[TMP0:%.*]] = load i16, ptr [[B]], align 2
@@ -738,7 +738,7 @@ define void @SExt(ptr nocapture %a, ptr nocapture %b) nounwind uwtable sanitize_
 ; CHECK-NEXT:    ret void
 ;
 ; ORIGIN-LABEL: define void @SExt(
-; ORIGIN-SAME: ptr nocapture [[A:%.*]], ptr nocapture [[B:%.*]]) #[[ATTR0]] {
+; ORIGIN-SAME: ptr captures(none) [[A:%.*]], ptr captures(none) [[B:%.*]]) #[[ATTR0]] {
 ; ORIGIN-NEXT:  [[ENTRY:.*:]]
 ; ORIGIN-NEXT:    call void @llvm.donothing()
 ; ORIGIN-NEXT:    [[TMP0:%.*]] = load i16, ptr [[B]], align 2
@@ -768,7 +768,7 @@ define void @SExt(ptr nocapture %a, ptr nocapture %b) nounwind uwtable sanitize_
 ; ORIGIN-NEXT:    ret void
 ;
 ; CALLS-LABEL: define void @SExt(
-; CALLS-SAME: ptr nocapture [[A:%.*]], ptr nocapture [[B:%.*]]) #[[ATTR0]] {
+; CALLS-SAME: ptr captures(none) [[A:%.*]], ptr captures(none) [[B:%.*]]) #[[ATTR0]] {
 ; CALLS-NEXT:  [[ENTRY:.*:]]
 ; CALLS-NEXT:    [[TMP0:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
 ; CALLS-NEXT:    [[TMP1:%.*]] = load i32, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_origin_tls to i64), i64 8) to ptr), align 4
@@ -810,21 +810,21 @@ entry:
 ; memset
 define void @MemSet(ptr nocapture %x) nounwind uwtable sanitize_memory {
 ; CHECK-LABEL: define void @MemSet(
-; CHECK-SAME: ptr nocapture [[X:%.*]]) #[[ATTR0]] {
+; CHECK-SAME: ptr captures(none) [[X:%.*]]) #[[ATTR0]] {
 ; CHECK-NEXT:  [[ENTRY:.*:]]
 ; CHECK-NEXT:    call void @llvm.donothing()
 ; CHECK-NEXT:    [[TMP0:%.*]] = call ptr @__msan_memset(ptr [[X]], i32 42, i64 10)
 ; CHECK-NEXT:    ret void
 ;
 ; ORIGIN-LABEL: define void @MemSet(
-; ORIGIN-SAME: ptr nocapture [[X:%.*]]) #[[ATTR0]] {
+; ORIGIN-SAME: ptr captures(none) [[X:%.*]]) #[[ATTR0]] {
 ; ORIGIN-NEXT:  [[ENTRY:.*:]]
 ; ORIGIN-NEXT:    call void @llvm.donothing()
 ; ORIGIN-NEXT:    [[TMP0:%.*]] = call ptr @__msan_memset(ptr [[X]], i32 42, i64 10)
 ; ORIGIN-NEXT:    ret void
 ;
 ; CALLS-LABEL: define void @MemSet(
-; CALLS-SAME: ptr nocapture [[X:%.*]]) #[[ATTR0]] {
+; CALLS-SAME: ptr captures(none) [[X:%.*]]) #[[ATTR0]] {
 ; CALLS-NEXT:  [[ENTRY:.*:]]
 ; CALLS-NEXT:    call void @llvm.donothing()
 ; CALLS-NEXT:    [[TMP0:%.*]] = call ptr @__msan_memset(ptr [[X]], i32 42, i64 10)
@@ -842,7 +842,7 @@ declare void @llvm.memset.p0.i64(ptr nocapture, i8, i64, i1) nounwind
 ; memcpy
 define void @MemCpy(ptr nocapture %x, ptr nocapture %y) nounwind uwtable sanitize_memory {
 ; CHECK-LABEL: define void @MemCpy(
-; CHECK-SAME: ptr nocapture [[X:%.*]], ptr nocapture [[Y:%.*]]) #[[ATTR0]] {
+; CHECK-SAME: ptr captures(none) [[X:%.*]], ptr captures(none) [[Y:%.*]]) #[[ATTR0]] {
 ; CHECK-NEXT:  [[ENTRY:.*:]]
 ; CHECK-NEXT:    [[TMP0:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
@@ -850,7 +850,7 @@ define void @MemCpy(ptr nocapture %x, ptr nocapture %y) nounwind uwtable sanitiz
 ; CHECK-NEXT:    ret void
 ;
 ; ORIGIN-LABEL: define void @MemCpy(
-; ORIGIN-SAME: ptr nocapture [[X:%.*]], ptr nocapture [[Y:%.*]]) #[[ATTR0]] {
+; ORIGIN-SAME: ptr captures(none) [[X:%.*]], ptr captures(none) [[Y:%.*]]) #[[ATTR0]] {
 ; ORIGIN-NEXT:  [[ENTRY:.*:]]
 ; ORIGIN-NEXT:    [[TMP0:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
 ; ORIGIN-NEXT:    [[TMP1:%.*]] = load i32, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_origin_tls to i64), i64 8) to ptr), align 4
@@ -859,7 +859,7 @@ define void @MemCpy(ptr nocapture %x, ptr nocapture %y) nounwind uwtable sanitiz
 ; ORIGIN-NEXT:    ret void
 ;
 ; CALLS-LABEL: define void @MemCpy(
-; CALLS-SAME: ptr nocapture [[X:%.*]], ptr nocapture [[Y:%.*]]) #[[ATTR0]] {
+; CALLS-SAME: ptr captures(none) [[X:%.*]], ptr captures(none) [[Y:%.*]]) #[[ATTR0]] {
 ; CALLS-NEXT:  [[ENTRY:.*:]]
 ; CALLS-NEXT:    [[TMP0:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
 ; CALLS-NEXT:    [[TMP1:%.*]] = load i32, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_origin_tls to i64), i64 8) to ptr), align 4
@@ -878,21 +878,21 @@ declare void @llvm.memcpy.p0.p0.i64(ptr nocapture, ptr nocapture, i64, i1) nounw
 ; memset.inline
 define void @MemSetInline(ptr nocapture %x) nounwind uwtable sanitize_memory {
 ; CHECK-LABEL: define void @MemSetInline(
-; CHECK-SAME: ptr nocapture [[X:%.*]]) #[[ATTR0]] {
+; CHECK-SAME: ptr captures(none) [[X:%.*]]) #[[ATTR0]] {
 ; CHECK-NEXT:  [[ENTRY:.*:]]
 ; CHECK-NEXT:    call void @llvm.donothing()
 ; CHECK-NEXT:    [[TMP0:%.*]] = call ptr @__msan_memset(ptr [[X]], i32 42, i64 10)
 ; CHECK-NEXT:    ret void
 ;
 ; ORIGIN-LABEL: define void @MemSetInline(
-; ORIGIN-SAME: ptr nocapture [[X:%.*]]) #[[ATTR0]] {
+; ORIGIN-SAME: ptr captures(none) [[X:%.*]]) #[[ATTR0]] {
 ; ORIGIN-NEXT:  [[ENTRY:.*:]]
 ; ORIGIN-NEXT:    call void @llvm.donothing()
 ; ORIGIN-NEXT:    [[TMP0:%.*]] = call ptr @__msan_memset(ptr [[X]], i32 42, i64 10)
 ; ORIGIN-NEXT:    ret void
 ;
 ; CALLS-LABEL: define void @MemSetInline(
-; CALLS-SAME: ptr nocapture [[X:%.*]]) #[[ATTR0]] {
+; CALLS-SAME: ptr captures(none) [[X:%.*]]) #[[ATTR0]] {
 ; CALLS-NEXT:  [[ENTRY:.*:]]
 ; CALLS-NEXT:    call void @llvm.donothing()
 ; CALLS-NEXT:    [[TMP0:%.*]] = call ptr @__msan_memset(ptr [[X]], i32 42, i64 10)
@@ -909,7 +909,7 @@ declare void @llvm.memset.inline.p0.i64(ptr nocapture, i8, i64, i1) nounwind
 ; memcpy.inline
 define void @MemCpyInline(ptr nocapture %x, ptr nocapture %y) nounwind uwtable sanitize_memory {
 ; CHECK-LABEL: define void @MemCpyInline(
-; CHECK-SAME: ptr nocapture [[X:%.*]], ptr nocapture [[Y:%.*]]) #[[ATTR0]] {
+; CHECK-SAME: ptr captures(none) [[X:%.*]], ptr captures(none) [[Y:%.*]]) #[[ATTR0]] {
 ; CHECK-NEXT:  [[ENTRY:.*:]]
 ; CHECK-NEXT:    [[TMP0:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
@@ -917,7 +917,7 @@ define void @MemCpyInline(ptr nocapture %x, ptr nocapture %y) nounwind uwtable s
 ; CHECK-NEXT:    ret void
 ;
 ; ORIGIN-LABEL: define void @MemCpyInline(
-; ORIGIN-SAME: ptr nocapture [[X:%.*]], ptr nocapture [[Y:%.*]]) #[[ATTR0]] {
+; ORIGIN-SAME: ptr captures(none) [[X:%.*]], ptr captures(none) [[Y:%.*]]) #[[ATTR0]] {
 ; ORIGIN-NEXT:  [[ENTRY:.*:]]
 ; ORIGIN-NEXT:    [[TMP0:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
 ; ORIGIN-NEXT:    [[TMP1:%.*]] = load i32, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_origin_tls to i64), i64 8) to ptr), align 4
@@ -926,7 +926,7 @@ define void @MemCpyInline(ptr nocapture %x, ptr nocapture %y) nounwind uwtable s
 ; ORIGIN-NEXT:    ret void
 ;
 ; CALLS-LABEL: define void @MemCpyInline(
-; CALLS-SAME: ptr nocapture [[X:%.*]], ptr nocapture [[Y:%.*]]) #[[ATTR0]] {
+; CALLS-SAME: ptr captures(none) [[X:%.*]], ptr captures(none) [[Y:%.*]]) #[[ATTR0]] {
 ; CALLS-NEXT:  [[ENTRY:.*:]]
 ; CALLS-NEXT:    [[TMP0:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
 ; CALLS-NEXT:    [[TMP1:%.*]] = load i32, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_origin_tls to i64), i64 8) to ptr), align 4
@@ -945,7 +945,7 @@ declare void @llvm.memcpy.inline.p0.p0.i64(ptr nocapture, ptr nocapture, i64, i1
 ; memmove is lowered to a call
 define void @MemMove(ptr nocapture %x, ptr nocapture %y) nounwind uwtable sanitize_memory {
 ; CHECK-LABEL: define void @MemMove(
-; CHECK-SAME: ptr nocapture [[X:%.*]], ptr nocapture [[Y:%.*]]) #[[ATTR0]] {
+; CHECK-SAME: ptr captures(none) [[X:%.*]], ptr captures(none) [[Y:%.*]]) #[[ATTR0]] {
 ; CHECK-NEXT:  [[ENTRY:.*:]]
 ; CHECK-NEXT:    [[TMP0:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
@@ -953,7 +953,7 @@ define void @MemMove(ptr nocapture %x, ptr nocapture %y) nounwind uwtable saniti
 ; CHECK-NEXT:    ret void
 ;
 ; ORIGIN-LABEL: define void @MemMove(
-; ORIGIN-SAME: ptr nocapture [[X:%.*]], ptr nocapture [[Y:%.*]]) #[[ATTR0]] {
+; ORIGIN-SAME: ptr captures(none) [[X:%.*]], ptr captures(none) [[Y:%.*]]) #[[ATTR0]] {
 ; ORIGIN-NEXT:  [[ENTRY:.*:]]
 ; ORIGIN-NEXT:    [[TMP0:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
 ; ORIGIN-NEXT:    [[TMP1:%.*]] = load i32, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_origin_tls to i64), i64 8) to ptr), align 4
@@ -962,7 +962,7 @@ define void @MemMove(ptr nocapture %x, ptr nocapture %y) nounwind uwtable saniti
 ; ORIGIN-NEXT:    ret void
 ;
 ; CALLS-LABEL: define void @MemMove(
-; CALLS-SAME: ptr nocapture [[X:%.*]], ptr nocapture [[Y:%.*]]) #[[ATTR0]] {
+; CALLS-SAME: ptr captures(none) [[X:%.*]], ptr captures(none) [[Y:%.*]]) #[[ATTR0]] {
 ; CALLS-NEXT:  [[ENTRY:.*:]]
 ; CALLS-NEXT:    [[TMP0:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
 ; CALLS-NEXT:    [[TMP1:%.*]] = load i32, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_origin_tls to i64), i64 8) to ptr), align 4
@@ -989,19 +989,19 @@ declare void @llvm.memcpy.element.unordered.atomic.p0.p0.i64(ptr nocapture write
 
 define void @atomic_memcpy(ptr nocapture %x, ptr nocapture %y) nounwind {
 ; CHECK-LABEL: define void @atomic_memcpy(
-; CHECK-SAME: ptr nocapture [[X:%.*]], ptr nocapture [[Y:%.*]]) #[[ATTR5]] {
+; CHECK-SAME: ptr captures(none) [[X:%.*]], ptr captures(none) [[Y:%.*]]) #[[ATTR5]] {
 ; CHECK-NEXT:    call void @llvm.donothing()
 ; CHECK-NEXT:    call void @llvm.memcpy.element.unordered.atomic.p0.p0.i64(ptr align 1 [[X]], ptr align 2 [[Y]], i64 16, i32 1)
 ; CHECK-NEXT:    ret void
 ;
 ; ORIGIN-LABEL: define void @atomic_memcpy(
-; ORIGIN-SAME: ptr nocapture [[X:%.*]], ptr nocapture [[Y:%.*]]) #[[ATTR5]] {
+; ORIGIN-SAME: ptr captures(none) [[X:%.*]], ptr captures(none) [[Y:%.*]]) #[[ATTR5]] {
 ; ORIGIN-NEXT:    call void @llvm.donothing()
 ; ORIGIN-NEXT:    call void @llvm.memcpy.element.unordered.atomic.p0.p0.i64(ptr align 1 [[X]], ptr align 2 [[Y]], i64 16, i32 1)
 ; ORIGIN-NEXT:    ret void
 ;
 ; CALLS-LABEL: define void @atomic_memcpy(
-; CALLS-SAME: ptr nocapture [[X:%.*]], ptr nocapture [[Y:%.*]]) #[[ATTR5]] {
+; CALLS-SAME: ptr captures(none) [[X:%.*]], ptr captures(none) [[Y:%.*]]) #[[ATTR5]] {
 ; CALLS-NEXT:    call void @llvm.donothing()
 ; CALLS-NEXT:    call void @llvm.memcpy.element.unordered.atomic.p0.p0.i64(ptr align 1 [[X]], ptr align 2 [[Y]], i64 16, i32 1)
 ; CALLS-NEXT:    ret void
@@ -1012,19 +1012,19 @@ define void @atomic_memcpy(ptr nocapture %x, ptr nocapture %y) nounwind {
 
 define void @atomic_memmove(ptr nocapture %x, ptr nocapture %y) nounwind {
 ; CHECK-LABEL: define void @atomic_memmove(
-; CHECK-SAME: ptr nocapture [[X:%.*]], ptr nocapture [[Y:%.*]]) #[[ATTR5]] {
+; CHECK-SAME: ptr captures(none) [[X:%.*]], ptr captures(none) [[Y:%.*]]) #[[ATTR5]] {
 ; CHECK-NEXT:    call void @llvm.donothing()
 ; CHECK-NEXT:    call void @llvm.memmove.element.unordered.atomic.p0.p0.i64(ptr align 1 [[X]], ptr align 2 [[Y]], i64 16, i32 1)
 ; CHECK-NEXT:    ret void
 ;
 ; ORIGIN-LABEL: define void @atomic_memmove(
-; ORIGIN-SAME: ptr nocapture [[X:%.*]], ptr nocapture [[Y:%.*]]) #[[ATTR5]] {
+; ORIGIN-SAME: ptr captures(none) [[X:%.*]], ptr captures(none) [[Y:%.*]]) #[[ATTR5]] {
 ; ORIGIN-NEXT:    call void @llvm.donothing()
 ; ORIGIN-NEXT:    call void @llvm.memmove.element.unordered.atomic.p0.p0.i64(ptr align 1 [[X]], ptr align 2 [[Y]], i64 16, i32 1)
 ; ORIGIN-NEXT:    ret void
 ;
 ; CALLS-LABEL: define void @atomic_memmove(
-; CALLS-SAME: ptr nocapture [[X:%.*]], ptr nocapture [[Y:%.*]]) #[[ATTR5]] {
+; CALLS-SAME: ptr captures(none) [[X:%.*]], ptr captures(none) [[Y:%.*]]) #[[ATTR5]] {
 ; CALLS-NEXT:    call void @llvm.donothing()
 ; CALLS-NEXT:    call void @llvm.memmove.element.unordered.atomic.p0.p0.i64(ptr align 1 [[X]], ptr align 2 [[Y]], i64 16, i32 1)
 ; CALLS-NEXT:    ret void
@@ -1035,19 +1035,19 @@ define void @atomic_memmove(ptr nocapture %x, ptr nocapture %y) nounwind {
 
 define void @atomic_memset(ptr nocapture %x) nounwind {
 ; CHECK-LABEL: define void @atomic_memset(
-; CHECK-SAME: ptr nocapture [[X:%.*]]) #[[ATTR5]] {
+; CHECK-SAME: ptr captures(none) [[X:%.*]]) #[[ATTR5]] {
 ; CHECK-NEXT:    call void @llvm.donothing()
 ; CHECK-NEXT:    call void @llvm.memset.element.unordered.atomic.p0.i64(ptr align 1 [[X]], i8 88, i64 16, i32 1)
 ; CHECK-NEXT:    ret void
 ;
 ; ORIGIN-LABEL: define void @atomic_memset(
-; ORIGIN-SAME: ptr nocapture [[X:%.*]]) #[[ATTR5]] {
+; ORIGIN-SAME: ptr captures(none) [[X:%.*]]) #[[ATTR5]] {
 ; ORIGIN-NEXT:    call void @llvm.donothing()
 ; ORIGIN-NEXT:    call void @llvm.memset.element.unordered.atomic.p0.i64(ptr align 1 [[X]], i8 88, i64 16, i32 1)
 ; ORIGIN-NEXT:    ret void
 ;
 ; CALLS-LABEL: define void @atomic_memset(
-; CALLS-SAME: ptr nocapture [[X:%.*]]) #[[ATTR5]] {
+; CALLS-SAME: ptr captures(none) [[X:%.*]]) #[[ATTR5]] {
 ; CALLS-NEXT:    call void @llvm.donothing()
 ; CALLS-NEXT:    call void @llvm.memset.element.unordered.atomic.p0.i64(ptr align 1 [[X]], i8 88, i64 16, i32 1)
 ; CALLS-NEXT:    ret void
@@ -2946,7 +2946,7 @@ entry:
 
 define void @VolatileStore(ptr nocapture %p, i32 %x) nounwind uwtable sanitize_memory {
 ; CHECK-LABEL: define void @VolatileStore(
-; CHECK-SAME: ptr nocapture [[P:%.*]], i32 [[X:%.*]]) #[[ATTR0]] {
+; CHECK-SAME: ptr captures(none) [[P:%.*]], i32 [[X:%.*]]) #[[ATTR0]] {
 ; CHECK-NEXT:  [[ENTRY:.*:]]
 ; CHECK-NEXT:    [[TMP0:%.*]] = load i32, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
@@ -2958,7 +2958,7 @@ define void @VolatileStore(ptr nocapture %p, i32 %x) nounwind uwtable sanitize_m
 ; CHECK-NEXT:    ret void
 ;
 ; ORIGIN-LABEL: define void @VolatileStore(
-; ORIGIN-SAME: ptr nocapture [[P:%.*]], i32 [[X:%.*]]) #[[ATTR0]] {
+; ORIGIN-SAME: ptr captures(none) [[P:%.*]], i32 [[X:%.*]]) #[[ATTR0]] {
 ; ORIGIN-NEXT:  [[ENTRY:.*:]]
 ; ORIGIN-NEXT:    [[TMP0:%.*]] = load i32, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
 ; ORIGIN-NEXT:    [[TMP1:%.*]] = load i32, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_origin_tls to i64), i64 8) to ptr), align 4
@@ -2979,7 +2979,7 @@ define void @VolatileStore(ptr nocapture %p, i32 %x) nounwind uwtable sanitize_m
 ; ORIGIN-NEXT:    ret void
 ;
 ; CALLS-LABEL: define void @VolatileStore(
-; CALLS-SAME: ptr nocapture [[P:%.*]], i32 [[X:%.*]]) #[[ATTR0]] {
+; CALLS-SAME: ptr captures(none) [[P:%.*]], i32 [[X:%.*]]) #[[ATTR0]] {
 ; CALLS-NEXT:  [[ENTRY:.*:]]
 ; CALLS-NEXT:    [[TMP0:%.*]] = load i64, ptr @__msan_param_tls, align 8
 ; CALLS-NEXT:    [[TMP1:%.*]] = load i32, ptr @__msan_param_origin_tls, align 4
@@ -3278,7 +3278,7 @@ cond.end:                                         ; preds = %cond.false, %cond.t
 
 define i32 @NoSanitizeMemoryParamTLS(ptr nocapture readonly %x) {
 ; CHECK-LABEL: define i32 @NoSanitizeMemoryParamTLS(
-; CHECK-SAME: ptr nocapture readonly [[X:%.*]]) {
+; CHECK-SAME: ptr readonly captures(none) [[X:%.*]]) {
 ; CHECK-NEXT:  [[ENTRY:.*:]]
 ; CHECK-NEXT:    call void @llvm.donothing()
 ; CHECK-NEXT:    [[TMP0:%.*]] = load i32, ptr [[X]], align 4
@@ -3290,7 +3290,7 @@ define i32 @NoSanitizeMemoryParamTLS(ptr nocapture readonly %x) {
 ; CHECK-NEXT:    ret i32 [[CALL]]
 ;
 ; ORIGIN-LABEL: define i32 @NoSanitizeMemoryParamTLS(
-; ORIGIN-SAME: ptr nocapture readonly [[X:%.*]]) {
+; ORIGIN-SAME: ptr readonly captures(none) [[X:%.*]]) {
 ; ORIGIN-NEXT:  [[ENTRY:.*:]]
 ; ORIGIN-NEXT:    call void @llvm.donothing()
 ; ORIGIN-NEXT:    [[TMP0:%.*]] = load i32, ptr [[X]], align 4
@@ -3304,7 +3304,7 @@ define i32 @NoSanitizeMemoryParamTLS(ptr nocapture readonly %x) {
 ; ORIGIN-NEXT:    ret i32 [[CALL]]
 ;
 ; CALLS-LABEL: define i32 @NoSanitizeMemoryParamTLS(
-; CALLS-SAME: ptr nocapture readonly [[X:%.*]]) {
+; CALLS-SAME: ptr readonly captures(none) [[X:%.*]]) {
 ; CALLS-NEXT:  [[ENTRY:.*:]]
 ; CALLS-NEXT:    call void @llvm.donothing()
 ; CALLS-NEXT:    [[TMP0:%.*]] = load i32, ptr [[X]], align 4
@@ -3434,7 +3434,7 @@ declare void @VAArgStructFn(i32 %guard, ...)
 
 define void @VAArgStruct(ptr nocapture %s) sanitize_memory {
 ; CHECK-LABEL: define void @VAArgStruct(
-; CHECK-SAME: ptr nocapture [[S:%.*]]) #[[ATTR6]] {
+; CHECK-SAME: ptr captures(none) [[S:%.*]]) #[[ATTR6]] {
 ; CHECK-NEXT:  [[ENTRY:.*:]]
 ; CHECK-NEXT:    [[TMP0:%.*]] = load i64, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
@@ -3479,7 +3479,7 @@ define void @VAArgStruct(ptr nocapture %s) sanitize_memory {
 ; CHECK-NEXT:    ret void
 ;
 ; ORIGIN-LABEL: define void @VAArgStruct(
-; ORIGIN-SAME: ptr nocapture [[S:%.*]]) #[[ATTR6]] {
+; ORIGIN-SAME: ptr captures(none) [[S:%.*]]) #[[ATTR6]] {
 ; ORIGIN-NEXT:  [[ENTRY:.*:]]
 ; ORIGIN-NEXT:    [[TMP0:%.*]] = load i64, ptr @__msan_param_tls, align 8
 ; ORIGIN-NEXT:    [[TMP1:%.*]] = load i32, ptr @__msan_param_origin_tls, align 4
@@ -3562,7 +3562,7 @@ define void @VAArgStruct(ptr nocapture %s) sanitize_memory {
 ; ORIGIN-NEXT:    ret void
 ;
 ; CALLS-LABEL: define void @VAArgStruct(
-; CALLS-SAME: ptr nocapture [[S:%.*]]) #[[ATTR6]] {
+; CALLS-SAME: ptr captures(none) [[S:%.*]]) #[[ATTR6]] {
 ; CALLS-NEXT:  [[ENTRY:.*:]]
 ; CALLS-NEXT:    [[TMP0:%.*]] = load i64, ptr @__msan_param_tls, align 8
 ; CALLS-NEXT:    [[TMP1:%.*]] = load i32, ptr @__msan_param_origin_tls, align 4
@@ -3661,7 +3661,7 @@ entry:
 ; The register save area is only 48 bytes instead of 176.
 define void @VAArgStructNoSSE(ptr nocapture %s) sanitize_memory #0 {
 ; CHECK-LABEL: define void @VAArgStructNoSSE(
-; CHECK-SAME: ptr nocapture [[S:%.*]]) #[[ATTR9:[0-9]+]] {
+; CHECK-SAME: ptr captures(none) [[S:%.*]]) #[[ATTR9:[0-9]+]] {
 ; CHECK-NEXT:  [[ENTRY:.*:]]
 ; CHECK-NEXT:    [[TMP0:%.*]] = load i64, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
@@ -3706,7 +3706,7 @@ define void @VAArgStructNoSSE(ptr nocapture %s) sanitize_memory #0 {
 ; CHECK-NEXT:    ret void
 ;
 ; ORIGIN-LABEL: define void @VAArgStructNoSSE(
-; ORIGIN-SAME: ptr nocapture [[S:%.*]]) #[[ATTR9:[0-9]+]] {
+; ORIGIN-SAME: ptr captures(none) [[S:%.*]]) #[[ATTR9:[0-9]+]] {
 ; ORIGIN-NEXT:  [[ENTRY:.*:]]
 ; ORIGIN-NEXT:    [[TMP0:%.*]] = load i64, ptr @__msan_param_tls, align 8
 ; ORIGIN-NEXT:    [[TMP1:%.*]] = load i32, ptr @__msan_param_origin_tls, align 4
@@ -3789,7 +3789,7 @@ define void @VAArgStructNoSSE(ptr nocapture %s) sanitize_memory #0 {
 ; ORIGIN-NEXT:    ret void
 ;
 ; CALLS-LABEL: define void @VAArgStructNoSSE(
-; CALLS-SAME: ptr nocapture [[S:%.*]]) #[[ATTR9:[0-9]+]] {
+; CALLS-SAME: ptr captures(none) [[S:%.*]]) #[[ATTR9:[0-9]+]] {
 ; CALLS-NEXT:  [[ENTRY:.*:]]
 ; CALLS-NEXT:    [[TMP0:%.*]] = load i64, ptr @__msan_param_tls, align 8
 ; CALLS-NEXT:    [[TMP1:%.*]] = load i32, ptr @__msan_param_origin_tls, align 4
diff --git a/llvm/test/Instrumentation/MemorySanitizer/scmp.ll b/llvm/test/Instrumentation/MemorySanitizer/scmp.ll
new file mode 100644
index 0000000000000..89c5b283b2510
--- /dev/null
+++ b/llvm/test/Instrumentation/MemorySanitizer/scmp.ll
@@ -0,0 +1,492 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 4
+; RUN: opt < %s -passes=msan -S | FileCheck %s
+;
+; llvm.scmp is correctly handled heuristically when each parameter is the same
+; type as the return type e.g.,
+;   call i8 @llvm.scmp.i8.i8(i8 %x, i8 %y)
+; but handled incorrectly by visitInstruction when the return type is different
+; e.g.,
+;   call i8 @llvm.scmp.i8.i62(i62 %x, i62 %y)
+;   call <4 x i8> @llvm.scmp.v4i8.v4i32(<4 x i32> %x, <4 x i32> %y)
+;
+; Forked from llvm/test/CodeGen/X86/scmp.ll
+
+target datalayout = "e-m:o-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+define i8 @scmp.8.8(i8 %x, i8 %y) nounwind #0 {
+; CHECK-LABEL: define i8 @scmp.8.8(
+; CHECK-SAME: i8 [[X:%.*]], i8 [[Y:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i8, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or i8 [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP3:%.*]] = call i8 @llvm.scmp.i8.i8(i8 [[X]], i8 [[Y]])
+; CHECK-NEXT:    store i8 [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i8 [[TMP3]]
+;
+  %1 = call i8 @llvm.scmp(i8 %x, i8 %y)
+  ret i8 %1
+}
+
+define i8 @scmp.8.16(i16 %x, i16 %y) nounwind #0 {
+; CHECK-LABEL: define i8 @scmp.8.16(
+; CHECK-SAME: i16 [[X:%.*]], i16 [[Y:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i16, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i16 [[TMP1]], 0
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i16 [[TMP2]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP3:%.*]], label [[TMP4:%.*]], !prof [[PROF1:![0-9]+]]
+; CHECK:       3:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4:[0-9]+]]
+; CHECK-NEXT:    unreachable
+; CHECK:       4:
+; CHECK-NEXT:    [[TMP5:%.*]] = call i8 @llvm.scmp.i8.i16(i16 [[X]], i16 [[Y]])
+; CHECK-NEXT:    store i8 0, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i8 [[TMP5]]
+;
+  %1 = call i8 @llvm.scmp(i16 %x, i16 %y)
+  ret i8 %1
+}
+
+define i8 @scmp.8.32(i32 %x, i32 %y) nounwind #0 {
+; CHECK-LABEL: define i8 @scmp.8.32(
+; CHECK-SAME: i32 [[X:%.*]], i32 [[Y:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i32, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i32, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i32 [[TMP1]], 0
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i32 [[TMP2]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP3:%.*]], label [[TMP4:%.*]], !prof [[PROF1]]
+; CHECK:       3:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       4:
+; CHECK-NEXT:    [[TMP5:%.*]] = call i8 @llvm.scmp.i8.i32(i32 [[X]], i32 [[Y]])
+; CHECK-NEXT:    store i8 0, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i8 [[TMP5]]
+;
+  %1 = call i8 @llvm.scmp(i32 %x, i32 %y)
+  ret i8 %1
+}
+
+define i8 @scmp.8.64(i64 %x, i64 %y) nounwind #0 {
+; CHECK-LABEL: define i8 @scmp.8.64(
+; CHECK-SAME: i64 [[X:%.*]], i64 [[Y:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i64 [[TMP2]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP3:%.*]], label [[TMP4:%.*]], !prof [[PROF1]]
+; CHECK:       3:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       4:
+; CHECK-NEXT:    [[TMP5:%.*]] = call i8 @llvm.scmp.i8.i64(i64 [[X]], i64 [[Y]])
+; CHECK-NEXT:    store i8 0, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i8 [[TMP5]]
+;
+  %1 = call i8 @llvm.scmp(i64 %x, i64 %y)
+  ret i8 %1
+}
+
+define i8 @scmp.8.128(i128 %x, i128 %y) nounwind #0 {
+; CHECK-LABEL: define i8 @scmp.8.128(
+; CHECK-SAME: i128 [[X:%.*]], i128 [[Y:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i128, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i128, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP1]], 0
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP2]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP3:%.*]], label [[TMP4:%.*]], !prof [[PROF1]]
+; CHECK:       3:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       4:
+; CHECK-NEXT:    [[TMP5:%.*]] = call i8 @llvm.scmp.i8.i128(i128 [[X]], i128 [[Y]])
+; CHECK-NEXT:    store i8 0, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i8 [[TMP5]]
+;
+  %1 = call i8 @llvm.scmp(i128 %x, i128 %y)
+  ret i8 %1
+}
+
+define i32 @scmp.32.32(i32 %x, i32 %y) nounwind #0 {
+; CHECK-LABEL: define i32 @scmp.32.32(
+; CHECK-SAME: i32 [[X:%.*]], i32 [[Y:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i32, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i32, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or i32 [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP3:%.*]] = call i32 @llvm.scmp.i32.i32(i32 [[X]], i32 [[Y]])
+; CHECK-NEXT:    store i32 [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i32 [[TMP3]]
+;
+  %1 = call i32 @llvm.scmp(i32 %x, i32 %y)
+  ret i32 %1
+}
+
+define i32 @scmp.32.64(i64 %x, i64 %y) nounwind #0 {
+; CHECK-LABEL: define i32 @scmp.32.64(
+; CHECK-SAME: i64 [[X:%.*]], i64 [[Y:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i64 [[TMP2]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP3:%.*]], label [[TMP4:%.*]], !prof [[PROF1]]
+; CHECK:       3:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       4:
+; CHECK-NEXT:    [[TMP5:%.*]] = call i32 @llvm.scmp.i32.i64(i64 [[X]], i64 [[Y]])
+; CHECK-NEXT:    store i32 0, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i32 [[TMP5]]
+;
+  %1 = call i32 @llvm.scmp(i64 %x, i64 %y)
+  ret i32 %1
+}
+
+define i64 @scmp.64.64(i64 %x, i64 %y) nounwind #0 {
+; CHECK-LABEL: define i64 @scmp.64.64(
+; CHECK-SAME: i64 [[X:%.*]], i64 [[Y:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or i64 [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP3:%.*]] = call i64 @llvm.scmp.i64.i64(i64 [[X]], i64 [[Y]])
+; CHECK-NEXT:    store i64 [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i64 [[TMP3]]
+;
+  %1 = call i64 @llvm.scmp(i64 %x, i64 %y)
+  ret i64 %1
+}
+
+define i4 @scmp_narrow_result(i32 %x, i32 %y) nounwind #0 {
+; CHECK-LABEL: define i4 @scmp_narrow_result(
+; CHECK-SAME: i32 [[X:%.*]], i32 [[Y:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i32, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i32, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i32 [[TMP1]], 0
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i32 [[TMP2]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP3:%.*]], label [[TMP4:%.*]], !prof [[PROF1]]
+; CHECK:       3:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       4:
+; CHECK-NEXT:    [[TMP5:%.*]] = call i4 @llvm.scmp.i4.i32(i32 [[X]], i32 [[Y]])
+; CHECK-NEXT:    store i4 0, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i4 [[TMP5]]
+;
+  %1 = call i4 @llvm.scmp(i32 %x, i32 %y)
+  ret i4 %1
+}
+
+define i8 @scmp_narrow_op(i62 %x, i62 %y) nounwind #0 {
+; CHECK-LABEL: define i8 @scmp_narrow_op(
+; CHECK-SAME: i62 [[X:%.*]], i62 [[Y:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i62, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i62, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i62 [[TMP1]], 0
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i62 [[TMP2]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP3:%.*]], label [[TMP4:%.*]], !prof [[PROF1]]
+; CHECK:       3:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       4:
+; CHECK-NEXT:    [[TMP5:%.*]] = call i8 @llvm.scmp.i8.i62(i62 [[X]], i62 [[Y]])
+; CHECK-NEXT:    store i8 0, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i8 [[TMP5]]
+;
+  %1 = call i8 @llvm.scmp(i62 %x, i62 %y)
+  ret i8 %1
+}
+
+define i141 @scmp_wide_result(i32 %x, i32 %y) nounwind #0 {
+; CHECK-LABEL: define i141 @scmp_wide_result(
+; CHECK-SAME: i32 [[X:%.*]], i32 [[Y:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i32, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i32, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i32 [[TMP1]], 0
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i32 [[TMP2]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP3:%.*]], label [[TMP4:%.*]], !prof [[PROF1]]
+; CHECK:       3:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       4:
+; CHECK-NEXT:    [[TMP5:%.*]] = call i141 @llvm.scmp.i141.i32(i32 [[X]], i32 [[Y]])
+; CHECK-NEXT:    store i141 0, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i141 [[TMP5]]
+;
+  %1 = call i141 @llvm.scmp(i32 %x, i32 %y)
+  ret i141 %1
+}
+
+define i8 @scmp_wide_op(i109 %x, i109 %y) nounwind #0 {
+; CHECK-LABEL: define i8 @scmp_wide_op(
+; CHECK-SAME: i109 [[X:%.*]], i109 [[Y:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i109, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i109, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i109 [[TMP1]], 0
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i109 [[TMP2]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP3:%.*]], label [[TMP4:%.*]], !prof [[PROF1]]
+; CHECK:       3:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       4:
+; CHECK-NEXT:    [[TMP5:%.*]] = call i8 @llvm.scmp.i8.i109(i109 [[X]], i109 [[Y]])
+; CHECK-NEXT:    store i8 0, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i8 [[TMP5]]
+;
+  %1 = call i8 @llvm.scmp(i109 %x, i109 %y)
+  ret i8 %1
+}
+
+define i41 @scmp_uncommon_types(i7 %x, i7 %y) nounwind #0 {
+; CHECK-LABEL: define i41 @scmp_uncommon_types(
+; CHECK-SAME: i7 [[X:%.*]], i7 [[Y:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i7, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i7, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i7 [[TMP1]], 0
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i7 [[TMP2]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP3:%.*]], label [[TMP4:%.*]], !prof [[PROF1]]
+; CHECK:       3:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       4:
+; CHECK-NEXT:    [[TMP5:%.*]] = call i41 @llvm.scmp.i41.i7(i7 [[X]], i7 [[Y]])
+; CHECK-NEXT:    store i41 0, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i41 [[TMP5]]
+;
+  %1 = call i41 @llvm.scmp(i7 %x, i7 %y)
+  ret i41 %1
+}
+
+define <4 x i32> @scmp_normal_vectors(<4 x i32> %x, <4 x i32> %y) nounwind #0 {
+; CHECK-LABEL: define <4 x i32> @scmp_normal_vectors(
+; CHECK-SAME: <4 x i32> [[X:%.*]], <4 x i32> [[Y:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <4 x i32> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP3:%.*]] = call <4 x i32> @llvm.scmp.v4i32.v4i32(<4 x i32> [[X]], <4 x i32> [[Y]])
+; CHECK-NEXT:    store <4 x i32> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i32> [[TMP3]]
+;
+  %1 = call <4 x i32> @llvm.scmp(<4 x i32> %x, <4 x i32> %y)
+  ret <4 x i32> %1
+}
+
+define <4 x i8> @scmp_narrow_vec_result(<4 x i32> %x, <4 x i32> %y) nounwind #0 {
+; CHECK-LABEL: define <4 x i8> @scmp_narrow_vec_result(
+; CHECK-SAME: <4 x i32> [[X:%.*]], <4 x i32> [[Y:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <4 x i32> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP3]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <4 x i32> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP5:%.*]], label [[TMP6:%.*]], !prof [[PROF1]]
+; CHECK:       5:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       6:
+; CHECK-NEXT:    [[TMP7:%.*]] = call <4 x i8> @llvm.scmp.v4i8.v4i32(<4 x i32> [[X]], <4 x i32> [[Y]])
+; CHECK-NEXT:    store <4 x i8> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i8> [[TMP7]]
+;
+  %1 = call <4 x i8> @llvm.scmp(<4 x i32> %x, <4 x i32> %y)
+  ret <4 x i8> %1
+}
+
+define <4 x i32> @scmp_narrow_vec_op(<4 x i8> %x, <4 x i8> %y) nounwind #0 {
+; CHECK-LABEL: define <4 x i32> @scmp_narrow_vec_op(
+; CHECK-SAME: <4 x i8> [[X:%.*]], <4 x i8> [[Y:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i8>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <4 x i8> [[TMP1]] to i32
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i32 [[TMP3]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <4 x i8> [[TMP2]] to i32
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i32 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP5:%.*]], label [[TMP6:%.*]], !prof [[PROF1]]
+; CHECK:       5:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       6:
+; CHECK-NEXT:    [[TMP7:%.*]] = call <4 x i32> @llvm.scmp.v4i32.v4i8(<4 x i8> [[X]], <4 x i8> [[Y]])
+; CHECK-NEXT:    store <4 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i32> [[TMP7]]
+;
+  %1 = call <4 x i32> @llvm.scmp(<4 x i8> %x, <4 x i8> %y)
+  ret <4 x i32> %1
+}
+
+define <16 x i32> @scmp_wide_vec_result(<16 x i8> %x, <16 x i8> %y) nounwind #0 {
+; CHECK-LABEL: define <16 x i32> @scmp_wide_vec_result(
+; CHECK-SAME: <16 x i8> [[X:%.*]], <16 x i8> [[Y:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i8>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <16 x i8> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP3]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <16 x i8> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP5:%.*]], label [[TMP6:%.*]], !prof [[PROF1]]
+; CHECK:       5:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       6:
+; CHECK-NEXT:    [[TMP7:%.*]] = call <16 x i32> @llvm.scmp.v16i32.v16i8(<16 x i8> [[X]], <16 x i8> [[Y]])
+; CHECK-NEXT:    store <16 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i32> [[TMP7]]
+;
+  %1 = call <16 x i32> @llvm.scmp(<16 x i8> %x, <16 x i8> %y)
+  ret <16 x i32> %1
+}
+
+define <16 x i8> @scmp_wide_vec_op(<16 x i64> %x, <16 x i64> %y) nounwind #0 {
+; CHECK-LABEL: define <16 x i8> @scmp_wide_vec_op(
+; CHECK-SAME: <16 x i64> [[X:%.*]], <16 x i64> [[Y:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <16 x i64> [[TMP1]] to i1024
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i1024 [[TMP3]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <16 x i64> [[TMP2]] to i1024
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i1024 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP5:%.*]], label [[TMP6:%.*]], !prof [[PROF1]]
+; CHECK:       5:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       6:
+; CHECK-NEXT:    [[TMP7:%.*]] = call <16 x i8> @llvm.scmp.v16i8.v16i64(<16 x i64> [[X]], <16 x i64> [[Y]])
+; CHECK-NEXT:    store <16 x i8> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i8> [[TMP7]]
+;
+  %1 = call <16 x i8> @llvm.scmp(<16 x i64> %x, <16 x i64> %y)
+  ret <16 x i8> %1
+}
+
+define <7 x i117> @scmp_uncommon_vectors(<7 x i7> %x, <7 x i7> %y) nounwind #0 {
+; CHECK-LABEL: define <7 x i117> @scmp_uncommon_vectors(
+; CHECK-SAME: <7 x i7> [[X:%.*]], <7 x i7> [[Y:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <7 x i7>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <7 x i7>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <7 x i7> [[TMP1]] to i49
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i49 [[TMP3]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <7 x i7> [[TMP2]] to i49
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i49 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP5:%.*]], label [[TMP6:%.*]], !prof [[PROF1]]
+; CHECK:       5:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       6:
+; CHECK-NEXT:    [[TMP7:%.*]] = call <7 x i117> @llvm.scmp.v7i117.v7i7(<7 x i7> [[X]], <7 x i7> [[Y]])
+; CHECK-NEXT:    store <7 x i117> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <7 x i117> [[TMP7]]
+;
+  %1 = call <7 x i117> @llvm.scmp(<7 x i7> %x, <7 x i7> %y)
+  ret <7 x i117> %1
+}
+
+define <1 x i3> @scmp_scalarize(<1 x i33> %x, <1 x i33> %y) nounwind #0 {
+; CHECK-LABEL: define <1 x i3> @scmp_scalarize(
+; CHECK-SAME: <1 x i33> [[X:%.*]], <1 x i33> [[Y:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <1 x i33>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <1 x i33>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <1 x i33> [[TMP1]] to i33
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i33 [[TMP3]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <1 x i33> [[TMP2]] to i33
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i33 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP5:%.*]], label [[TMP6:%.*]], !prof [[PROF1]]
+; CHECK:       5:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       6:
+; CHECK-NEXT:    [[TMP7:%.*]] = call <1 x i3> @llvm.scmp.v1i3.v1i33(<1 x i33> [[X]], <1 x i33> [[Y]])
+; CHECK-NEXT:    store <1 x i3> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <1 x i3> [[TMP7]]
+;
+  %1 = call <1 x i3> @llvm.scmp(<1 x i33> %x, <1 x i33> %y)
+  ret <1 x i3> %1
+}
+
+define <2 x i8> @scmp_bool_operands(<2 x i1> %x, <2 x i1> %y) nounwind #0 {
+; CHECK-LABEL: define <2 x i8> @scmp_bool_operands(
+; CHECK-SAME: <2 x i1> [[X:%.*]], <2 x i1> [[Y:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i1>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i1>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <2 x i1> [[TMP1]] to i2
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i2 [[TMP3]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <2 x i1> [[TMP2]] to i2
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i2 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP5:%.*]], label [[TMP6:%.*]], !prof [[PROF1]]
+; CHECK:       5:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       6:
+; CHECK-NEXT:    [[TMP7:%.*]] = call <2 x i8> @llvm.scmp.v2i8.v2i1(<2 x i1> [[X]], <2 x i1> [[Y]])
+; CHECK-NEXT:    store <2 x i8> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x i8> [[TMP7]]
+;
+  %1 = call <2 x i8> @llvm.scmp(<2 x i1> %x, <2 x i1> %y)
+  ret <2 x i8> %1
+}
+
+define <2 x i16> @scmp_ret_wider_than_operands(<2 x i8> %x, <2 x i8> %y) nounwind #0 {
+; CHECK-LABEL: define <2 x i16> @scmp_ret_wider_than_operands(
+; CHECK-SAME: <2 x i8> [[X:%.*]], <2 x i8> [[Y:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i8>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <2 x i8> [[TMP1]] to i16
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i16 [[TMP3]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <2 x i8> [[TMP2]] to i16
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i16 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP5:%.*]], label [[TMP6:%.*]], !prof [[PROF1]]
+; CHECK:       5:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       6:
+; CHECK-NEXT:    [[TMP7:%.*]] = call <2 x i16> @llvm.scmp.v2i16.v2i8(<2 x i8> [[X]], <2 x i8> [[Y]])
+; CHECK-NEXT:    store <2 x i16> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x i16> [[TMP7]]
+;
+  %1 = call <2 x i16> @llvm.scmp(<2 x i8> %x, <2 x i8> %y)
+  ret <2 x i16> %1
+}
+
+attributes #0 = { sanitize_memory }
+;.
+; CHECK: [[PROF1]] = !{!"branch_weights", i32 1, i32 1048575}
+;.
diff --git a/llvm/test/Instrumentation/MemorySanitizer/ucmp.ll b/llvm/test/Instrumentation/MemorySanitizer/ucmp.ll
new file mode 100644
index 0000000000000..5e0a248afe748
--- /dev/null
+++ b/llvm/test/Instrumentation/MemorySanitizer/ucmp.ll
@@ -0,0 +1,420 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 4
+; RUN: opt < %s -passes=msan -S | FileCheck %s
+;
+; llvm.ucmp is correctly handled heuristically when each parameter is the same
+; type as the return type e.g.,
+;   call i8 @llvm.ucmp.i8.i8(i8 %x, i8 %y)
+; but handled incorrectly by visitInstruction when the return type is different  
+; e.g.,
+;   call i8 @llvm.ucmp.i8.i62(i62 %x, i62 %y)
+;   call <4 x i8> @llvm.ucmp.v4i8.v4i32(<4 x i32> %x, <4 x i32> %y)
+
+; Forked from llvm/test/CodeGen/X86/ucmp.ll
+
+target datalayout = "e-m:o-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+define i8 @ucmp.8.8(i8 %x, i8 %y) nounwind #0 {
+; CHECK-LABEL: define i8 @ucmp.8.8(
+; CHECK-SAME: i8 [[X:%.*]], i8 [[Y:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i8, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or i8 [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP3:%.*]] = call i8 @llvm.ucmp.i8.i8(i8 [[X]], i8 [[Y]])
+; CHECK-NEXT:    store i8 [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i8 [[TMP3]]
+;
+  %1 = call i8 @llvm.ucmp(i8 %x, i8 %y)
+  ret i8 %1
+}
+
+define i8 @ucmp.8.16(i16 %x, i16 %y) nounwind #0 {
+; CHECK-LABEL: define i8 @ucmp.8.16(
+; CHECK-SAME: i16 [[X:%.*]], i16 [[Y:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i16, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i16 [[TMP1]], 0
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i16 [[TMP2]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP3:%.*]], label [[TMP4:%.*]], !prof [[PROF1:![0-9]+]]
+; CHECK:       3:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4:[0-9]+]]
+; CHECK-NEXT:    unreachable
+; CHECK:       4:
+; CHECK-NEXT:    [[TMP5:%.*]] = call i8 @llvm.ucmp.i8.i16(i16 [[X]], i16 [[Y]])
+; CHECK-NEXT:    store i8 0, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i8 [[TMP5]]
+;
+  %1 = call i8 @llvm.ucmp(i16 %x, i16 %y)
+  ret i8 %1
+}
+
+define i8 @ucmp.8.32(i32 %x, i32 %y) nounwind #0 {
+; CHECK-LABEL: define i8 @ucmp.8.32(
+; CHECK-SAME: i32 [[X:%.*]], i32 [[Y:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i32, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i32, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i32 [[TMP1]], 0
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i32 [[TMP2]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP3:%.*]], label [[TMP4:%.*]], !prof [[PROF1]]
+; CHECK:       3:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       4:
+; CHECK-NEXT:    [[TMP5:%.*]] = call i8 @llvm.ucmp.i8.i32(i32 [[X]], i32 [[Y]])
+; CHECK-NEXT:    store i8 0, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i8 [[TMP5]]
+;
+  %1 = call i8 @llvm.ucmp(i32 %x, i32 %y)
+  ret i8 %1
+}
+
+define i8 @ucmp.8.64(i64 %x, i64 %y) nounwind #0 {
+; CHECK-LABEL: define i8 @ucmp.8.64(
+; CHECK-SAME: i64 [[X:%.*]], i64 [[Y:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i64 [[TMP2]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP3:%.*]], label [[TMP4:%.*]], !prof [[PROF1]]
+; CHECK:       3:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       4:
+; CHECK-NEXT:    [[TMP5:%.*]] = call i8 @llvm.ucmp.i8.i64(i64 [[X]], i64 [[Y]])
+; CHECK-NEXT:    store i8 0, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i8 [[TMP5]]
+;
+  %1 = call i8 @llvm.ucmp(i64 %x, i64 %y)
+  ret i8 %1
+}
+
+define i8 @ucmp.8.128(i128 %x, i128 %y) nounwind #0 {
+; CHECK-LABEL: define i8 @ucmp.8.128(
+; CHECK-SAME: i128 [[X:%.*]], i128 [[Y:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i128, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i128, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP1]], 0
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP2]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP3:%.*]], label [[TMP4:%.*]], !prof [[PROF1]]
+; CHECK:       3:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       4:
+; CHECK-NEXT:    [[TMP5:%.*]] = call i8 @llvm.ucmp.i8.i128(i128 [[X]], i128 [[Y]])
+; CHECK-NEXT:    store i8 0, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i8 [[TMP5]]
+;
+  %1 = call i8 @llvm.ucmp(i128 %x, i128 %y)
+  ret i8 %1
+}
+
+define i32 @ucmp.32.32(i32 %x, i32 %y) nounwind #0 {
+; CHECK-LABEL: define i32 @ucmp.32.32(
+; CHECK-SAME: i32 [[X:%.*]], i32 [[Y:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i32, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i32, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or i32 [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP3:%.*]] = call i32 @llvm.ucmp.i32.i32(i32 [[X]], i32 [[Y]])
+; CHECK-NEXT:    store i32 [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i32 [[TMP3]]
+;
+  %1 = call i32 @llvm.ucmp(i32 %x, i32 %y)
+  ret i32 %1
+}
+
+define i32 @ucmp.32.64(i64 %x, i64 %y) nounwind #0 {
+; CHECK-LABEL: define i32 @ucmp.32.64(
+; CHECK-SAME: i64 [[X:%.*]], i64 [[Y:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i64 [[TMP2]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP3:%.*]], label [[TMP4:%.*]], !prof [[PROF1]]
+; CHECK:       3:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       4:
+; CHECK-NEXT:    [[TMP5:%.*]] = call i32 @llvm.ucmp.i32.i64(i64 [[X]], i64 [[Y]])
+; CHECK-NEXT:    store i32 0, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i32 [[TMP5]]
+;
+  %1 = call i32 @llvm.ucmp(i64 %x, i64 %y)
+  ret i32 %1
+}
+
+define i64 @ucmp.64.64(i64 %x, i64 %y) nounwind #0 {
+; CHECK-LABEL: define i64 @ucmp.64.64(
+; CHECK-SAME: i64 [[X:%.*]], i64 [[Y:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or i64 [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP3:%.*]] = call i64 @llvm.ucmp.i64.i64(i64 [[X]], i64 [[Y]])
+; CHECK-NEXT:    store i64 [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i64 [[TMP3]]
+;
+  %1 = call i64 @llvm.ucmp(i64 %x, i64 %y)
+  ret i64 %1
+}
+
+define i4 @ucmp_narrow_result(i32 %x, i32 %y) nounwind #0 {
+; CHECK-LABEL: define i4 @ucmp_narrow_result(
+; CHECK-SAME: i32 [[X:%.*]], i32 [[Y:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i32, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i32, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i32 [[TMP1]], 0
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i32 [[TMP2]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP3:%.*]], label [[TMP4:%.*]], !prof [[PROF1]]
+; CHECK:       3:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       4:
+; CHECK-NEXT:    [[TMP5:%.*]] = call i4 @llvm.ucmp.i4.i32(i32 [[X]], i32 [[Y]])
+; CHECK-NEXT:    store i4 0, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i4 [[TMP5]]
+;
+  %1 = call i4 @llvm.ucmp(i32 %x, i32 %y)
+  ret i4 %1
+}
+
+define i8 @ucmp_narrow_op(i62 %x, i62 %y) nounwind #0 {
+; CHECK-LABEL: define i8 @ucmp_narrow_op(
+; CHECK-SAME: i62 [[X:%.*]], i62 [[Y:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i62, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i62, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i62 [[TMP1]], 0
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i62 [[TMP2]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP3:%.*]], label [[TMP4:%.*]], !prof [[PROF1]]
+; CHECK:       3:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       4:
+; CHECK-NEXT:    [[TMP5:%.*]] = call i8 @llvm.ucmp.i8.i62(i62 [[X]], i62 [[Y]])
+; CHECK-NEXT:    store i8 0, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i8 [[TMP5]]
+;
+  %1 = call i8 @llvm.ucmp(i62 %x, i62 %y)
+  ret i8 %1
+}
+
+define i141 @ucmp_wide_result(i32 %x, i32 %y) nounwind #0 {
+; CHECK-LABEL: define i141 @ucmp_wide_result(
+; CHECK-SAME: i32 [[X:%.*]], i32 [[Y:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i32, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i32, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i32 [[TMP1]], 0
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i32 [[TMP2]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP3:%.*]], label [[TMP4:%.*]], !prof [[PROF1]]
+; CHECK:       3:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       4:
+; CHECK-NEXT:    [[TMP5:%.*]] = call i141 @llvm.ucmp.i141.i32(i32 [[X]], i32 [[Y]])
+; CHECK-NEXT:    store i141 0, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i141 [[TMP5]]
+;
+  %1 = call i141 @llvm.ucmp(i32 %x, i32 %y)
+  ret i141 %1
+}
+
+define i8 @ucmp_wide_op(i109 %x, i109 %y) nounwind #0 {
+; CHECK-LABEL: define i8 @ucmp_wide_op(
+; CHECK-SAME: i109 [[X:%.*]], i109 [[Y:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i109, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i109, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i109 [[TMP1]], 0
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i109 [[TMP2]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP3:%.*]], label [[TMP4:%.*]], !prof [[PROF1]]
+; CHECK:       3:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       4:
+; CHECK-NEXT:    [[TMP5:%.*]] = call i8 @llvm.ucmp.i8.i109(i109 [[X]], i109 [[Y]])
+; CHECK-NEXT:    store i8 0, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i8 [[TMP5]]
+;
+  %1 = call i8 @llvm.ucmp(i109 %x, i109 %y)
+  ret i8 %1
+}
+
+define i41 @ucmp_uncommon_types(i7 %x, i7 %y) nounwind #0 {
+; CHECK-LABEL: define i41 @ucmp_uncommon_types(
+; CHECK-SAME: i7 [[X:%.*]], i7 [[Y:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i7, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i7, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i7 [[TMP1]], 0
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i7 [[TMP2]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP3:%.*]], label [[TMP4:%.*]], !prof [[PROF1]]
+; CHECK:       3:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       4:
+; CHECK-NEXT:    [[TMP5:%.*]] = call i41 @llvm.ucmp.i41.i7(i7 [[X]], i7 [[Y]])
+; CHECK-NEXT:    store i41 0, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i41 [[TMP5]]
+;
+  %1 = call i41 @llvm.ucmp(i7 %x, i7 %y)
+  ret i41 %1
+}
+
+define <4 x i32> @ucmp_normal_vectors(<4 x i32> %x, <4 x i32> %y) nounwind #0 {
+; CHECK-LABEL: define <4 x i32> @ucmp_normal_vectors(
+; CHECK-SAME: <4 x i32> [[X:%.*]], <4 x i32> [[Y:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <4 x i32> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP3:%.*]] = call <4 x i32> @llvm.ucmp.v4i32.v4i32(<4 x i32> [[X]], <4 x i32> [[Y]])
+; CHECK-NEXT:    store <4 x i32> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i32> [[TMP3]]
+;
+  %1 = call <4 x i32> @llvm.ucmp(<4 x i32> %x, <4 x i32> %y)
+  ret <4 x i32> %1
+}
+
+define <4 x i8> @ucmp_narrow_vec_result(<4 x i32> %x, <4 x i32> %y) nounwind #0 {
+; CHECK-LABEL: define <4 x i8> @ucmp_narrow_vec_result(
+; CHECK-SAME: <4 x i32> [[X:%.*]], <4 x i32> [[Y:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <4 x i32> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP3]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <4 x i32> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP5:%.*]], label [[TMP6:%.*]], !prof [[PROF1]]
+; CHECK:       5:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       6:
+; CHECK-NEXT:    [[TMP7:%.*]] = call <4 x i8> @llvm.ucmp.v4i8.v4i32(<4 x i32> [[X]], <4 x i32> [[Y]])
+; CHECK-NEXT:    store <4 x i8> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i8> [[TMP7]]
+;
+  %1 = call <4 x i8> @llvm.ucmp(<4 x i32> %x, <4 x i32> %y)
+  ret <4 x i8> %1
+}
+
+define <4 x i32> @ucmp_narrow_vec_op(<4 x i8> %x, <4 x i8> %y) nounwind #0 {
+; CHECK-LABEL: define <4 x i32> @ucmp_narrow_vec_op(
+; CHECK-SAME: <4 x i8> [[X:%.*]], <4 x i8> [[Y:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i8>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <4 x i8> [[TMP1]] to i32
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i32 [[TMP3]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <4 x i8> [[TMP2]] to i32
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i32 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP5:%.*]], label [[TMP6:%.*]], !prof [[PROF1]]
+; CHECK:       5:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       6:
+; CHECK-NEXT:    [[TMP7:%.*]] = call <4 x i32> @llvm.ucmp.v4i32.v4i8(<4 x i8> [[X]], <4 x i8> [[Y]])
+; CHECK-NEXT:    store <4 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i32> [[TMP7]]
+;
+  %1 = call <4 x i32> @llvm.ucmp(<4 x i8> %x, <4 x i8> %y)
+  ret <4 x i32> %1
+}
+
+define <16 x i32> @ucmp_wide_vec_result(<16 x i8> %x, <16 x i8> %y) nounwind #0 {
+; CHECK-LABEL: define <16 x i32> @ucmp_wide_vec_result(
+; CHECK-SAME: <16 x i8> [[X:%.*]], <16 x i8> [[Y:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i8>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <16 x i8> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP3]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <16 x i8> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP5:%.*]], label [[TMP6:%.*]], !prof [[PROF1]]
+; CHECK:       5:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       6:
+; CHECK-NEXT:    [[TMP7:%.*]] = call <16 x i32> @llvm.ucmp.v16i32.v16i8(<16 x i8> [[X]], <16 x i8> [[Y]])
+; CHECK-NEXT:    store <16 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i32> [[TMP7]]
+;
+  %1 = call <16 x i32> @llvm.ucmp(<16 x i8> %x, <16 x i8> %y)
+  ret <16 x i32> %1
+}
+
+define <16 x i8> @ucmp_wide_vec_op(<16 x i32> %x, <16 x i32> %y) nounwind #0 {
+; CHECK-LABEL: define <16 x i8> @ucmp_wide_vec_op(
+; CHECK-SAME: <16 x i32> [[X:%.*]], <16 x i32> [[Y:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP3]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP5:%.*]], label [[TMP6:%.*]], !prof [[PROF1]]
+; CHECK:       5:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       6:
+; CHECK-NEXT:    [[TMP7:%.*]] = call <16 x i8> @llvm.ucmp.v16i8.v16i32(<16 x i32> [[X]], <16 x i32> [[Y]])
+; CHECK-NEXT:    store <16 x i8> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i8> [[TMP7]]
+;
+  %1 = call <16 x i8> @llvm.ucmp(<16 x i32> %x, <16 x i32> %y)
+  ret <16 x i8> %1
+}
+
+define <17 x i2> @ucmp_uncommon_vectors(<17 x i71> %x, <17 x i71> %y) nounwind #0 {
+; CHECK-LABEL: define <17 x i2> @ucmp_uncommon_vectors(
+; CHECK-SAME: <17 x i71> [[X:%.*]], <17 x i71> [[Y:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <17 x i71>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <17 x i71>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 256) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <17 x i71> [[TMP1]] to i1207
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i1207 [[TMP3]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <17 x i71> [[TMP2]] to i1207
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i1207 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP5:%.*]], label [[TMP6:%.*]], !prof [[PROF1]]
+; CHECK:       5:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       6:
+; CHECK-NEXT:    [[TMP7:%.*]] = call <17 x i2> @llvm.ucmp.v17i2.v17i71(<17 x i71> [[X]], <17 x i71> [[Y]])
+; CHECK-NEXT:    store <17 x i2> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <17 x i2> [[TMP7]]
+;
+  %1 = call <17 x i2> @llvm.ucmp(<17 x i71> %x, <17 x i71> %y)
+  ret <17 x i2> %1
+}
+
+attributes #0 = { sanitize_memory }
+;.
+; CHECK: [[PROF1]] = !{!"branch_weights", i32 1, i32 1048575}
+;.
diff --git a/llvm/test/Instrumentation/MemorySanitizer/vector-reduce-fadd.ll b/llvm/test/Instrumentation/MemorySanitizer/vector-reduce-fadd.ll
new file mode 100644
index 0000000000000..5da4c7357b6ad
--- /dev/null
+++ b/llvm/test/Instrumentation/MemorySanitizer/vector-reduce-fadd.ll
@@ -0,0 +1,293 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 4
+; RUN: opt < %s -passes=msan -S | FileCheck %s
+;
+; Forked from llvm/test/CodeGen/X86/vector-reduce-fadd.ll
+;
+; Currently handled incorrectly by visitInstruction:
+; - llvm.vector.reduce.fadd
+
+target datalayout = "e-m:o-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+define float @test_v2f32(float %a0, <2 x float> %a1) #0 {
+; CHECK-LABEL: define float @test_v2f32(
+; CHECK-SAME: float [[A0:%.*]], <2 x float> [[A1:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i32, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = call i32 @llvm.vector.reduce.or.v2i32(<2 x i32> [[TMP2]])
+; CHECK-NEXT:    [[TMP4:%.*]] = or i32 [[TMP1]], [[TMP3]]
+; CHECK-NEXT:    [[TMP6:%.*]] = call float @llvm.vector.reduce.fadd.v2f32(float [[A0]], <2 x float> [[A1]])
+; CHECK-NEXT:    store i32 [[TMP4]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret float [[TMP6]]
+;
+  %1 = call float @llvm.vector.reduce.fadd.f32.v2f32(float %a0, <2 x float> %a1)
+  ret float %1
+}
+
+define float @test_v4f32(float %a0, <4 x float> %a1) #0 {
+; CHECK-LABEL: define float @test_v4f32(
+; CHECK-SAME: float [[A0:%.*]], <4 x float> [[A1:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i32, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = call i32 @llvm.vector.reduce.or.v4i32(<4 x i32> [[TMP2]])
+; CHECK-NEXT:    [[TMP4:%.*]] = or i32 [[TMP1]], [[TMP3]]
+; CHECK-NEXT:    [[TMP6:%.*]] = call float @llvm.vector.reduce.fadd.v4f32(float [[A0]], <4 x float> [[A1]])
+; CHECK-NEXT:    store i32 [[TMP4]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret float [[TMP6]]
+;
+  %1 = call float @llvm.vector.reduce.fadd.f32.v4f32(float %a0, <4 x float> %a1)
+  ret float %1
+}
+
+define float @test_v8f32(float %a0, <8 x float> %a1) #0 {
+; CHECK-LABEL: define float @test_v8f32(
+; CHECK-SAME: float [[A0:%.*]], <8 x float> [[A1:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i32, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = call i32 @llvm.vector.reduce.or.v8i32(<8 x i32> [[TMP2]])
+; CHECK-NEXT:    [[TMP4:%.*]] = or i32 [[TMP1]], [[TMP3]]
+; CHECK-NEXT:    [[TMP6:%.*]] = call float @llvm.vector.reduce.fadd.v8f32(float [[A0]], <8 x float> [[A1]])
+; CHECK-NEXT:    store i32 [[TMP4]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret float [[TMP6]]
+;
+  %1 = call float @llvm.vector.reduce.fadd.f32.v8f32(float %a0, <8 x float> %a1)
+  ret float %1
+}
+
+define float @test_v16f32(float %a0, <16 x float> %a1) #0 {
+; CHECK-LABEL: define float @test_v16f32(
+; CHECK-SAME: float [[A0:%.*]], <16 x float> [[A1:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i32, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = call i32 @llvm.vector.reduce.or.v16i32(<16 x i32> [[TMP2]])
+; CHECK-NEXT:    [[TMP4:%.*]] = or i32 [[TMP1]], [[TMP3]]
+; CHECK-NEXT:    [[TMP6:%.*]] = call float @llvm.vector.reduce.fadd.v16f32(float [[A0]], <16 x float> [[A1]])
+; CHECK-NEXT:    store i32 [[TMP4]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret float [[TMP6]]
+;
+  %1 = call float @llvm.vector.reduce.fadd.f32.v16f32(float %a0, <16 x float> %a1)
+  ret float %1
+}
+
+
+define float @test_v2f32_zero(<2 x float> %a0) #0 {
+; CHECK-LABEL: define float @test_v2f32_zero(
+; CHECK-SAME: <2 x float> [[A0:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = call i32 @llvm.vector.reduce.or.v2i32(<2 x i32> [[TMP1]])
+; CHECK-NEXT:    [[TMP3:%.*]] = or i32 0, [[TMP2]]
+; CHECK-NEXT:    [[TMP5:%.*]] = call float @llvm.vector.reduce.fadd.v2f32(float -0.000000e+00, <2 x float> [[A0]])
+; CHECK-NEXT:    store i32 [[TMP3]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret float [[TMP5]]
+;
+  %1 = call float @llvm.vector.reduce.fadd.f32.v2f32(float -0.0, <2 x float> %a0)
+  ret float %1
+}
+
+define float @test_v4f32_zero(<4 x float> %a0) #0 {
+; CHECK-LABEL: define float @test_v4f32_zero(
+; CHECK-SAME: <4 x float> [[A0:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = call i32 @llvm.vector.reduce.or.v4i32(<4 x i32> [[TMP1]])
+; CHECK-NEXT:    [[TMP3:%.*]] = or i32 0, [[TMP2]]
+; CHECK-NEXT:    [[TMP5:%.*]] = call float @llvm.vector.reduce.fadd.v4f32(float -0.000000e+00, <4 x float> [[A0]])
+; CHECK-NEXT:    store i32 [[TMP3]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret float [[TMP5]]
+;
+  %1 = call float @llvm.vector.reduce.fadd.f32.v4f32(float -0.0, <4 x float> %a0)
+  ret float %1
+}
+
+define float @test_v8f32_zero(<8 x float> %a0) #0 {
+; CHECK-LABEL: define float @test_v8f32_zero(
+; CHECK-SAME: <8 x float> [[A0:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = call i32 @llvm.vector.reduce.or.v8i32(<8 x i32> [[TMP1]])
+; CHECK-NEXT:    [[TMP3:%.*]] = or i32 0, [[TMP2]]
+; CHECK-NEXT:    [[TMP5:%.*]] = call float @llvm.vector.reduce.fadd.v8f32(float -0.000000e+00, <8 x float> [[A0]])
+; CHECK-NEXT:    store i32 [[TMP3]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret float [[TMP5]]
+;
+  %1 = call float @llvm.vector.reduce.fadd.f32.v8f32(float -0.0, <8 x float> %a0)
+  ret float %1
+}
+
+define float @test_v16f32_zero(<16 x float> %a0) #0 {
+; CHECK-LABEL: define float @test_v16f32_zero(
+; CHECK-SAME: <16 x float> [[A0:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = call i32 @llvm.vector.reduce.or.v16i32(<16 x i32> [[TMP1]])
+; CHECK-NEXT:    [[TMP3:%.*]] = or i32 0, [[TMP2]]
+; CHECK-NEXT:    [[TMP5:%.*]] = call float @llvm.vector.reduce.fadd.v16f32(float -0.000000e+00, <16 x float> [[A0]])
+; CHECK-NEXT:    store i32 [[TMP3]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret float [[TMP5]]
+;
+  %1 = call float @llvm.vector.reduce.fadd.f32.v16f32(float -0.0, <16 x float> %a0)
+  ret float %1
+}
+
+define double @test_v2f64(double %a0, <2 x double> %a1) #0 {
+; CHECK-LABEL: define double @test_v2f64(
+; CHECK-SAME: double [[A0:%.*]], <2 x double> [[A1:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vector.reduce.or.v2i64(<2 x i64> [[TMP2]])
+; CHECK-NEXT:    [[TMP4:%.*]] = or i64 [[TMP1]], [[TMP3]]
+; CHECK-NEXT:    [[TMP6:%.*]] = call double @llvm.vector.reduce.fadd.v2f64(double [[A0]], <2 x double> [[A1]])
+; CHECK-NEXT:    store i64 [[TMP4]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret double [[TMP6]]
+;
+  %1 = call double @llvm.vector.reduce.fadd.f64.v2f64(double %a0, <2 x double> %a1)
+  ret double %1
+}
+
+define double @test_v4f64(double %a0, <4 x double> %a1) #0 {
+; CHECK-LABEL: define double @test_v4f64(
+; CHECK-SAME: double [[A0:%.*]], <4 x double> [[A1:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vector.reduce.or.v4i64(<4 x i64> [[TMP2]])
+; CHECK-NEXT:    [[TMP4:%.*]] = or i64 [[TMP1]], [[TMP3]]
+; CHECK-NEXT:    [[TMP6:%.*]] = call double @llvm.vector.reduce.fadd.v4f64(double [[A0]], <4 x double> [[A1]])
+; CHECK-NEXT:    store i64 [[TMP4]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret double [[TMP6]]
+;
+  %1 = call double @llvm.vector.reduce.fadd.f64.v4f64(double %a0, <4 x double> %a1)
+  ret double %1
+}
+
+define double @test_v8f64(double %a0, <8 x double> %a1) #0 {
+; CHECK-LABEL: define double @test_v8f64(
+; CHECK-SAME: double [[A0:%.*]], <8 x double> [[A1:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vector.reduce.or.v8i64(<8 x i64> [[TMP2]])
+; CHECK-NEXT:    [[TMP4:%.*]] = or i64 [[TMP1]], [[TMP3]]
+; CHECK-NEXT:    [[TMP6:%.*]] = call double @llvm.vector.reduce.fadd.v8f64(double [[A0]], <8 x double> [[A1]])
+; CHECK-NEXT:    store i64 [[TMP4]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret double [[TMP6]]
+;
+  %1 = call double @llvm.vector.reduce.fadd.f64.v8f64(double %a0, <8 x double> %a1)
+  ret double %1
+}
+
+define double @test_v16f64(double %a0, <16 x double> %a1) #0 {
+; CHECK-LABEL: define double @test_v16f64(
+; CHECK-SAME: double [[A0:%.*]], <16 x double> [[A1:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vector.reduce.or.v16i64(<16 x i64> [[TMP2]])
+; CHECK-NEXT:    [[TMP4:%.*]] = or i64 [[TMP1]], [[TMP3]]
+; CHECK-NEXT:    [[TMP6:%.*]] = call double @llvm.vector.reduce.fadd.v16f64(double [[A0]], <16 x double> [[A1]])
+; CHECK-NEXT:    store i64 [[TMP4]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret double [[TMP6]]
+;
+  %1 = call double @llvm.vector.reduce.fadd.f64.v16f64(double %a0, <16 x double> %a1)
+  ret double %1
+}
+
+
+define double @test_v2f64_zero(<2 x double> %a0) #0 {
+; CHECK-LABEL: define double @test_v2f64_zero(
+; CHECK-SAME: <2 x double> [[A0:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vector.reduce.or.v2i64(<2 x i64> [[TMP1]])
+; CHECK-NEXT:    [[TMP3:%.*]] = or i64 0, [[TMP2]]
+; CHECK-NEXT:    [[TMP5:%.*]] = call double @llvm.vector.reduce.fadd.v2f64(double -0.000000e+00, <2 x double> [[A0]])
+; CHECK-NEXT:    store i64 [[TMP3]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret double [[TMP5]]
+;
+  %1 = call double @llvm.vector.reduce.fadd.f64.v2f64(double -0.0, <2 x double> %a0)
+  ret double %1
+}
+
+define double @test_v4f64_zero(<4 x double> %a0) #0 {
+; CHECK-LABEL: define double @test_v4f64_zero(
+; CHECK-SAME: <4 x double> [[A0:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vector.reduce.or.v4i64(<4 x i64> [[TMP1]])
+; CHECK-NEXT:    [[TMP3:%.*]] = or i64 0, [[TMP2]]
+; CHECK-NEXT:    [[TMP5:%.*]] = call double @llvm.vector.reduce.fadd.v4f64(double -0.000000e+00, <4 x double> [[A0]])
+; CHECK-NEXT:    store i64 [[TMP3]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret double [[TMP5]]
+;
+  %1 = call double @llvm.vector.reduce.fadd.f64.v4f64(double -0.0, <4 x double> %a0)
+  ret double %1
+}
+
+define double @test_v8f64_zero(<8 x double> %a0) #0 {
+; CHECK-LABEL: define double @test_v8f64_zero(
+; CHECK-SAME: <8 x double> [[A0:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vector.reduce.or.v8i64(<8 x i64> [[TMP1]])
+; CHECK-NEXT:    [[TMP3:%.*]] = or i64 0, [[TMP2]]
+; CHECK-NEXT:    [[TMP5:%.*]] = call double @llvm.vector.reduce.fadd.v8f64(double -0.000000e+00, <8 x double> [[A0]])
+; CHECK-NEXT:    store i64 [[TMP3]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret double [[TMP5]]
+;
+  %1 = call double @llvm.vector.reduce.fadd.f64.v8f64(double -0.0, <8 x double> %a0)
+  ret double %1
+}
+
+define double @test_v16f64_zero(<16 x double> %a0) #0 {
+; CHECK-LABEL: define double @test_v16f64_zero(
+; CHECK-SAME: <16 x double> [[A0:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vector.reduce.or.v16i64(<16 x i64> [[TMP1]])
+; CHECK-NEXT:    [[TMP3:%.*]] = or i64 0, [[TMP2]]
+; CHECK-NEXT:    [[TMP5:%.*]] = call double @llvm.vector.reduce.fadd.v16f64(double -0.000000e+00, <16 x double> [[A0]])
+; CHECK-NEXT:    store i64 [[TMP3]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret double [[TMP5]]
+;
+  %1 = call double @llvm.vector.reduce.fadd.f64.v16f64(double -0.0, <16 x double> %a0)
+  ret double %1
+}
+
+define float @PR64627() #0 {
+; CHECK-LABEL: define float @PR64627(
+; CHECK-SAME: ) #[[ATTR0]] {
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast i5 0 to <5 x i1>
+; CHECK-NEXT:    [[TMP2:%.*]] = select <5 x i1> [[TMP1]], <5 x i32> zeroinitializer, <5 x i32> zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <5 x i1> zeroinitializer, <5 x i32> splat (i32 1065353216), <5 x i32> [[TMP2]]
+; CHECK-NEXT:    [[TMP3:%.*]] = select <5 x i1> [[TMP1]], <5 x float> zeroinitializer, <5 x float> splat (float 1.000000e+00)
+; CHECK-NEXT:    [[TMP4:%.*]] = call i32 @llvm.vector.reduce.or.v5i32(<5 x i32> [[_MSPROP_SELECT]])
+; CHECK-NEXT:    [[TMP5:%.*]] = or i32 0, [[TMP4]]
+; CHECK-NEXT:    [[TMP7:%.*]] = call float @llvm.vector.reduce.fadd.v5f32(float -0.000000e+00, <5 x float> [[TMP3]])
+; CHECK-NEXT:    store i32 [[TMP5]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret float [[TMP7]]
+;
+  %1 = bitcast i5 0 to <5 x i1>
+  %2 = select <5 x i1> %1, <5 x float> zeroinitializer, <5 x float> <float 1.0, float 1.0, float 1.0, float 1.0, float 1.0>
+  %3 = call float @llvm.vector.reduce.fadd.v5f32(float -0.0, <5 x float> %2)
+  ret float %3
+}
+declare float @llvm.vector.reduce.fadd.v5f32(float, <5 x float>)
+
+declare float @llvm.vector.reduce.fadd.f32.v2f32(float, <2 x float>)
+declare float @llvm.vector.reduce.fadd.f32.v4f32(float, <4 x float>)
+declare float @llvm.vector.reduce.fadd.f32.v8f32(float, <8 x float>)
+declare float @llvm.vector.reduce.fadd.f32.v16f32(float, <16 x float>)
+
+declare double @llvm.vector.reduce.fadd.f64.v2f64(double, <2 x double>)
+declare double @llvm.vector.reduce.fadd.f64.v4f64(double, <4 x double>)
+declare double @llvm.vector.reduce.fadd.f64.v8f64(double, <8 x double>)
+declare double @llvm.vector.reduce.fadd.f64.v16f64(double, <16 x double>)
+
+attributes #0 = { sanitize_memory }
diff --git a/llvm/test/Instrumentation/MemorySanitizer/vector-reduce-fmul.ll b/llvm/test/Instrumentation/MemorySanitizer/vector-reduce-fmul.ll
new file mode 100644
index 0000000000000..0c1c4edc4367f
--- /dev/null
+++ b/llvm/test/Instrumentation/MemorySanitizer/vector-reduce-fmul.ll
@@ -0,0 +1,272 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 4
+; RUN: opt < %s -passes=msan -S | FileCheck %s
+;
+; Forked from llvm/test/CodeGen/X86/vector-reduce-fmul.ll
+;
+; Currently handled incorrectly by visitInstruction:
+; - llvm.vector.reduce.fmul
+
+target datalayout = "e-m:o-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+define float @test_v2f32(float %a0, <2 x float> %a1) #0 {
+; CHECK-LABEL: define float @test_v2f32(
+; CHECK-SAME: float [[A0:%.*]], <2 x float> [[A1:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i32, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = call i32 @llvm.vector.reduce.or.v2i32(<2 x i32> [[TMP2]])
+; CHECK-NEXT:    [[TMP4:%.*]] = or i32 [[TMP1]], [[TMP3]]
+; CHECK-NEXT:    [[TMP6:%.*]] = call float @llvm.vector.reduce.fmul.v2f32(float [[A0]], <2 x float> [[A1]])
+; CHECK-NEXT:    store i32 [[TMP4]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret float [[TMP6]]
+;
+  %1 = call float @llvm.vector.reduce.fmul.f32.v2f32(float %a0, <2 x float> %a1)
+  ret float %1
+}
+
+define float @test_v4f32(float %a0, <4 x float> %a1) #0 {
+; CHECK-LABEL: define float @test_v4f32(
+; CHECK-SAME: float [[A0:%.*]], <4 x float> [[A1:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i32, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = call i32 @llvm.vector.reduce.or.v4i32(<4 x i32> [[TMP2]])
+; CHECK-NEXT:    [[TMP4:%.*]] = or i32 [[TMP1]], [[TMP3]]
+; CHECK-NEXT:    [[TMP6:%.*]] = call float @llvm.vector.reduce.fmul.v4f32(float [[A0]], <4 x float> [[A1]])
+; CHECK-NEXT:    store i32 [[TMP4]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret float [[TMP6]]
+;
+  %1 = call float @llvm.vector.reduce.fmul.f32.v4f32(float %a0, <4 x float> %a1)
+  ret float %1
+}
+
+define float @test_v8f32(float %a0, <8 x float> %a1) #0 {
+; CHECK-LABEL: define float @test_v8f32(
+; CHECK-SAME: float [[A0:%.*]], <8 x float> [[A1:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i32, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = call i32 @llvm.vector.reduce.or.v8i32(<8 x i32> [[TMP2]])
+; CHECK-NEXT:    [[TMP4:%.*]] = or i32 [[TMP1]], [[TMP3]]
+; CHECK-NEXT:    [[TMP6:%.*]] = call float @llvm.vector.reduce.fmul.v8f32(float [[A0]], <8 x float> [[A1]])
+; CHECK-NEXT:    store i32 [[TMP4]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret float [[TMP6]]
+;
+  %1 = call float @llvm.vector.reduce.fmul.f32.v8f32(float %a0, <8 x float> %a1)
+  ret float %1
+}
+
+define float @test_v16f32(float %a0, <16 x float> %a1) #0 {
+; CHECK-LABEL: define float @test_v16f32(
+; CHECK-SAME: float [[A0:%.*]], <16 x float> [[A1:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i32, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = call i32 @llvm.vector.reduce.or.v16i32(<16 x i32> [[TMP2]])
+; CHECK-NEXT:    [[TMP4:%.*]] = or i32 [[TMP1]], [[TMP3]]
+; CHECK-NEXT:    [[TMP6:%.*]] = call float @llvm.vector.reduce.fmul.v16f32(float [[A0]], <16 x float> [[A1]])
+; CHECK-NEXT:    store i32 [[TMP4]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret float [[TMP6]]
+;
+  %1 = call float @llvm.vector.reduce.fmul.f32.v16f32(float %a0, <16 x float> %a1)
+  ret float %1
+}
+
+
+define float @test_v2f32_one(<2 x float> %a0) #0 {
+; CHECK-LABEL: define float @test_v2f32_one(
+; CHECK-SAME: <2 x float> [[A0:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = call i32 @llvm.vector.reduce.or.v2i32(<2 x i32> [[TMP1]])
+; CHECK-NEXT:    [[TMP3:%.*]] = or i32 0, [[TMP2]]
+; CHECK-NEXT:    [[TMP5:%.*]] = call float @llvm.vector.reduce.fmul.v2f32(float 1.000000e+00, <2 x float> [[A0]])
+; CHECK-NEXT:    store i32 [[TMP3]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret float [[TMP5]]
+;
+  %1 = call float @llvm.vector.reduce.fmul.f32.v2f32(float 1.0, <2 x float> %a0)
+  ret float %1
+}
+
+define float @test_v4f32_one(<4 x float> %a0) #0 {
+; CHECK-LABEL: define float @test_v4f32_one(
+; CHECK-SAME: <4 x float> [[A0:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = call i32 @llvm.vector.reduce.or.v4i32(<4 x i32> [[TMP1]])
+; CHECK-NEXT:    [[TMP3:%.*]] = or i32 0, [[TMP2]]
+; CHECK-NEXT:    [[TMP5:%.*]] = call float @llvm.vector.reduce.fmul.v4f32(float 1.000000e+00, <4 x float> [[A0]])
+; CHECK-NEXT:    store i32 [[TMP3]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret float [[TMP5]]
+;
+  %1 = call float @llvm.vector.reduce.fmul.f32.v4f32(float 1.0, <4 x float> %a0)
+  ret float %1
+}
+
+define float @test_v8f32_one(<8 x float> %a0) #0 {
+; CHECK-LABEL: define float @test_v8f32_one(
+; CHECK-SAME: <8 x float> [[A0:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = call i32 @llvm.vector.reduce.or.v8i32(<8 x i32> [[TMP1]])
+; CHECK-NEXT:    [[TMP3:%.*]] = or i32 0, [[TMP2]]
+; CHECK-NEXT:    [[TMP5:%.*]] = call float @llvm.vector.reduce.fmul.v8f32(float 1.000000e+00, <8 x float> [[A0]])
+; CHECK-NEXT:    store i32 [[TMP3]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret float [[TMP5]]
+;
+  %1 = call float @llvm.vector.reduce.fmul.f32.v8f32(float 1.0, <8 x float> %a0)
+  ret float %1
+}
+
+define float @test_v16f32_one(<16 x float> %a0) #0 {
+; CHECK-LABEL: define float @test_v16f32_one(
+; CHECK-SAME: <16 x float> [[A0:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = call i32 @llvm.vector.reduce.or.v16i32(<16 x i32> [[TMP1]])
+; CHECK-NEXT:    [[TMP3:%.*]] = or i32 0, [[TMP2]]
+; CHECK-NEXT:    [[TMP5:%.*]] = call float @llvm.vector.reduce.fmul.v16f32(float 1.000000e+00, <16 x float> [[A0]])
+; CHECK-NEXT:    store i32 [[TMP3]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret float [[TMP5]]
+;
+  %1 = call float @llvm.vector.reduce.fmul.f32.v16f32(float 1.0, <16 x float> %a0)
+  ret float %1
+}
+
+define double @test_v2f64(double %a0, <2 x double> %a1) #0 {
+; CHECK-LABEL: define double @test_v2f64(
+; CHECK-SAME: double [[A0:%.*]], <2 x double> [[A1:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vector.reduce.or.v2i64(<2 x i64> [[TMP2]])
+; CHECK-NEXT:    [[TMP4:%.*]] = or i64 [[TMP1]], [[TMP3]]
+; CHECK-NEXT:    [[TMP6:%.*]] = call double @llvm.vector.reduce.fmul.v2f64(double [[A0]], <2 x double> [[A1]])
+; CHECK-NEXT:    store i64 [[TMP4]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret double [[TMP6]]
+;
+  %1 = call double @llvm.vector.reduce.fmul.f64.v2f64(double %a0, <2 x double> %a1)
+  ret double %1
+}
+
+define double @test_v4f64(double %a0, <4 x double> %a1) #0 {
+; CHECK-LABEL: define double @test_v4f64(
+; CHECK-SAME: double [[A0:%.*]], <4 x double> [[A1:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vector.reduce.or.v4i64(<4 x i64> [[TMP2]])
+; CHECK-NEXT:    [[TMP4:%.*]] = or i64 [[TMP1]], [[TMP3]]
+; CHECK-NEXT:    [[TMP6:%.*]] = call double @llvm.vector.reduce.fmul.v4f64(double [[A0]], <4 x double> [[A1]])
+; CHECK-NEXT:    store i64 [[TMP4]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret double [[TMP6]]
+;
+  %1 = call double @llvm.vector.reduce.fmul.f64.v4f64(double %a0, <4 x double> %a1)
+  ret double %1
+}
+
+define double @test_v8f64(double %a0, <8 x double> %a1) #0 {
+; CHECK-LABEL: define double @test_v8f64(
+; CHECK-SAME: double [[A0:%.*]], <8 x double> [[A1:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vector.reduce.or.v8i64(<8 x i64> [[TMP2]])
+; CHECK-NEXT:    [[TMP4:%.*]] = or i64 [[TMP1]], [[TMP3]]
+; CHECK-NEXT:    [[TMP6:%.*]] = call double @llvm.vector.reduce.fmul.v8f64(double [[A0]], <8 x double> [[A1]])
+; CHECK-NEXT:    store i64 [[TMP4]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret double [[TMP6]]
+;
+  %1 = call double @llvm.vector.reduce.fmul.f64.v8f64(double %a0, <8 x double> %a1)
+  ret double %1
+}
+
+define double @test_v16f64(double %a0, <16 x double> %a1) #0 {
+; CHECK-LABEL: define double @test_v16f64(
+; CHECK-SAME: double [[A0:%.*]], <16 x double> [[A1:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vector.reduce.or.v16i64(<16 x i64> [[TMP2]])
+; CHECK-NEXT:    [[TMP4:%.*]] = or i64 [[TMP1]], [[TMP3]]
+; CHECK-NEXT:    [[TMP6:%.*]] = call double @llvm.vector.reduce.fmul.v16f64(double [[A0]], <16 x double> [[A1]])
+; CHECK-NEXT:    store i64 [[TMP4]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret double [[TMP6]]
+;
+  %1 = call double @llvm.vector.reduce.fmul.f64.v16f64(double %a0, <16 x double> %a1)
+  ret double %1
+}
+
+
+define double @test_v2f64_one(<2 x double> %a0) #0 {
+; CHECK-LABEL: define double @test_v2f64_one(
+; CHECK-SAME: <2 x double> [[A0:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vector.reduce.or.v2i64(<2 x i64> [[TMP1]])
+; CHECK-NEXT:    [[TMP3:%.*]] = or i64 0, [[TMP2]]
+; CHECK-NEXT:    [[TMP5:%.*]] = call double @llvm.vector.reduce.fmul.v2f64(double 1.000000e+00, <2 x double> [[A0]])
+; CHECK-NEXT:    store i64 [[TMP3]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret double [[TMP5]]
+;
+  %1 = call double @llvm.vector.reduce.fmul.f64.v2f64(double 1.0, <2 x double> %a0)
+  ret double %1
+}
+
+define double @test_v4f64_one(<4 x double> %a0) #0 {
+; CHECK-LABEL: define double @test_v4f64_one(
+; CHECK-SAME: <4 x double> [[A0:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vector.reduce.or.v4i64(<4 x i64> [[TMP1]])
+; CHECK-NEXT:    [[TMP3:%.*]] = or i64 0, [[TMP2]]
+; CHECK-NEXT:    [[TMP5:%.*]] = call double @llvm.vector.reduce.fmul.v4f64(double 1.000000e+00, <4 x double> [[A0]])
+; CHECK-NEXT:    store i64 [[TMP3]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret double [[TMP5]]
+;
+  %1 = call double @llvm.vector.reduce.fmul.f64.v4f64(double 1.0, <4 x double> %a0)
+  ret double %1
+}
+
+define double @test_v8f64_one(<8 x double> %a0) #0 {
+; CHECK-LABEL: define double @test_v8f64_one(
+; CHECK-SAME: <8 x double> [[A0:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vector.reduce.or.v8i64(<8 x i64> [[TMP1]])
+; CHECK-NEXT:    [[TMP3:%.*]] = or i64 0, [[TMP2]]
+; CHECK-NEXT:    [[TMP5:%.*]] = call double @llvm.vector.reduce.fmul.v8f64(double 1.000000e+00, <8 x double> [[A0]])
+; CHECK-NEXT:    store i64 [[TMP3]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret double [[TMP5]]
+;
+  %1 = call double @llvm.vector.reduce.fmul.f64.v8f64(double 1.0, <8 x double> %a0)
+  ret double %1
+}
+
+define double @test_v16f64_one(<16 x double> %a0) #0 {
+; CHECK-LABEL: define double @test_v16f64_one(
+; CHECK-SAME: <16 x double> [[A0:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vector.reduce.or.v16i64(<16 x i64> [[TMP1]])
+; CHECK-NEXT:    [[TMP3:%.*]] = or i64 0, [[TMP2]]
+; CHECK-NEXT:    [[TMP5:%.*]] = call double @llvm.vector.reduce.fmul.v16f64(double 1.000000e+00, <16 x double> [[A0]])
+; CHECK-NEXT:    store i64 [[TMP3]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret double [[TMP5]]
+;
+  %1 = call double @llvm.vector.reduce.fmul.f64.v16f64(double 1.0, <16 x double> %a0)
+  ret double %1
+}
+
+declare float @llvm.vector.reduce.fmul.f32.v2f32(float, <2 x float>)
+declare float @llvm.vector.reduce.fmul.f32.v4f32(float, <4 x float>)
+declare float @llvm.vector.reduce.fmul.f32.v8f32(float, <8 x float>)
+declare float @llvm.vector.reduce.fmul.f32.v16f32(float, <16 x float>)
+
+declare double @llvm.vector.reduce.fmul.f64.v2f64(double, <2 x double>)
+declare double @llvm.vector.reduce.fmul.f64.v4f64(double, <4 x double>)
+declare double @llvm.vector.reduce.fmul.f64.v8f64(double, <8 x double>)
+declare double @llvm.vector.reduce.fmul.f64.v16f64(double, <16 x double>)
+
+attributes #0 = { sanitize_memory }
diff --git a/llvm/test/Instrumentation/SanitizerCoverage/gep-tracing.ll b/llvm/test/Instrumentation/SanitizerCoverage/gep-tracing.ll
index 17bed67fdcfc7..8827d885df7c1 100644
--- a/llvm/test/Instrumentation/SanitizerCoverage/gep-tracing.ll
+++ b/llvm/test/Instrumentation/SanitizerCoverage/gep-tracing.ll
@@ -4,7 +4,7 @@
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
-define void @gep_1(ptr nocapture %a, i32 %i)  {
+define void @gep_1(ptr captures(none) %a, i32 %i)  {
 entry:
   %idxprom = sext i32 %i to i64
   %arrayidx = getelementptr inbounds i32, ptr %a, i64 %idxprom
@@ -12,12 +12,12 @@ entry:
   ret void
 }
 
-; CHECK-LABEL: define void @gep_1(ptr nocapture %a, i32 %i)
+; CHECK-LABEL: define void @gep_1(ptr captures(none) %a, i32 %i)
 ; CHECK:   call void @__sanitizer_cov_trace_gep(i64 %idxprom)
 ; CHECK: ret void
 
 
-define void @gep_2(ptr nocapture %a, i32 %i, i32 %j) {
+define void @gep_2(ptr captures(none) %a, i32 %i, i32 %j) {
 entry:
   %idxprom = sext i32 %j to i64
   %idxprom1 = sext i32 %i to i64
@@ -26,7 +26,7 @@ entry:
   ret void
 }
 
-; CHECK-LABEL: define void @gep_2(ptr nocapture %a, i32 %i, i32 %j)
+; CHECK-LABEL: define void @gep_2(ptr captures(none) %a, i32 %i, i32 %j)
 ; CHECK: call void @__sanitizer_cov_trace_gep(i64 %idxprom1)
 ; CHECK: call void @__sanitizer_cov_trace_gep(i64 %idxprom)
 ; CHECK: ret void
diff --git a/llvm/test/LTO/X86/mix-opaque-typed.ll b/llvm/test/LTO/X86/mix-opaque-typed.ll
index 4fb60b99c41f3..79977e7ec8e3d 100644
--- a/llvm/test/LTO/X86/mix-opaque-typed.ll
+++ b/llvm/test/LTO/X86/mix-opaque-typed.ll
@@ -10,7 +10,7 @@ target triple = "x86_64-unknown-linux-gnu"
 declare i64 @foo(i64* %p);
 
 define i64 @call_foo(i64* %p) {
-  ; CHECK-LABEL: define i64 @call_foo(ptr nocapture readonly %p) local_unnamed_addr #0 {
+  ; CHECK-LABEL: define i64 @call_foo(ptr readonly captures(none) %p) local_unnamed_addr #0 {
   ; CHECK-NEXT: %t.i = load i64, ptr %p, align 8
   %t = call i64 @foo(i64* %p)
   ret i64 %t
diff --git a/llvm/test/MC/AArch64/basic-a64-instructions.s b/llvm/test/MC/AArch64/basic-a64-instructions.s
index 0ae23d672e4a3..14ac11f581a55 100644
--- a/llvm/test/MC/AArch64/basic-a64-instructions.s
+++ b/llvm/test/MC/AArch64/basic-a64-instructions.s
@@ -3347,6 +3347,11 @@ _func:
 // CHECK: mov      x2, #5299989643264             // encoding: [0x42,0x9a,0xc0,0xd2]
 // CHECK: movk     xzr, #{{4321|0x10e1}}, lsl #48 // encoding: [0x3f,0x1c,0xe2,0xf2]
 
+	.equ equvalue, 0x0001
+        movk x1, equvalue, lsl 16
+// CHECK: .set equvalue, 1
+// CHECK-NEXT: movk x1, #1, lsl #16 // encoding: [0x21,0x00,0xa0,0xf2]
+
         movz x2, #:abs_g0:sym
         movk w3, #:abs_g0_nc:sym
 
diff --git a/llvm/test/MC/AMDGPU/gfx11_asm_opsel_err.s b/llvm/test/MC/AMDGPU/gfx11_asm_opsel_err.s
new file mode 100644
index 0000000000000..c555405d042d0
--- /dev/null
+++ b/llvm/test/MC/AMDGPU/gfx11_asm_opsel_err.s
@@ -0,0 +1,11 @@
+// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize32,+real-true16 -show-encoding %s 2>&1 | FileCheck --check-prefixes=GFX11 %s
+// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize64,+real-true16 -show-encoding %s 2>&1 | FileCheck --check-prefixes=GFX11 %s
+
+
+// op_sel hi on vgpr but not on op_sel operand
+v_add_f16 v0.h, v1.h, v2.h op_sel:[0,1,1]
+// GFX11: op_sel operand conflicts with 16-bit operand suffix
+
+// op_sel hi on op_sel operand but not on vgpr
+v_add_f16 v0.h, v1.l, v2.h op_sel:[1,1,1]
+// GFX11: op_sel operand conflicts with 16-bit operand suffix
diff --git a/llvm/test/MC/AMDGPU/gfx11_asm_vop3.s b/llvm/test/MC/AMDGPU/gfx11_asm_vop3.s
index 40e3fbda47787..857a1359b00d9 100644
--- a/llvm/test/MC/AMDGPU/gfx11_asm_vop3.s
+++ b/llvm/test/MC/AMDGPU/gfx11_asm_vop3.s
@@ -461,11 +461,11 @@ v_alignbyte_b32 v5, s1, v255, s3
 v_alignbyte_b32 v5, s105, s105, s105
 // GFX11: v_alignbyte_b32 v5, s105, s105, s105    ; encoding: [0x05,0x00,0x17,0xd6,0x69,0xd2,0xa4,0x01]
 
-v_alignbyte_b32 v5, vcc_lo, ttmp15, v3
-// GFX11: v_alignbyte_b32 v5, vcc_lo, ttmp15, v3  ; encoding: [0x05,0x00,0x17,0xd6,0x6a,0xf6,0x0c,0x04]
+v_alignbyte_b32 v5, vcc_lo, ttmp15, v3.l
+// GFX11: v_alignbyte_b32 v5, vcc_lo, ttmp15, v3.l ; encoding: [0x05,0x00,0x17,0xd6,0x6a,0xf6,0x0c,0x04]
 
-v_alignbyte_b32 v5, vcc_hi, 0xaf123456, v255
-// GFX11: v_alignbyte_b32 v5, vcc_hi, 0xaf123456, v255 ; encoding: [0x05,0x00,0x17,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf]
+v_alignbyte_b32 v5, vcc_hi, 0xaf123456, v255.l
+// GFX11: v_alignbyte_b32 v5, vcc_hi, 0xaf123456, v255.l ; encoding: [0x05,0x00,0x17,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf]
 
 v_alignbyte_b32 v5, ttmp15, src_scc, ttmp15
 // GFX11: v_alignbyte_b32 v5, ttmp15, src_scc, ttmp15 ; encoding: [0x05,0x00,0x17,0xd6,0x7b,0xfa,0xed,0x01]
@@ -494,6 +494,9 @@ v_alignbyte_b32 v5, src_scc, vcc_lo, -1
 v_alignbyte_b32 v255, 0xaf123456, vcc_hi, null
 // GFX11: v_alignbyte_b32 v255, 0xaf123456, vcc_hi, null ; encoding: [0xff,0x00,0x17,0xd6,0xff,0xd6,0xf0,0x01,0x56,0x34,0x12,0xaf]
 
+v_alignbyte_b32 v5, vcc_hi, 0xaf123456, v255.h
+// GFX11: v_alignbyte_b32 v5, vcc_hi, 0xaf123456, v255.h op_sel:[0,0,1,0] ; encoding: [0x05,0x20,0x17,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf]
+
 v_and_b16 v5.l, v1.l, v2.l
 // GFX11: v_and_b16 v5.l, v1.l, v2.l              ; encoding: [0x05,0x00,0x62,0xd7,0x01,0x05,0x02,0x00]
 
@@ -2906,11 +2909,11 @@ v_mad_i16 v5.l, -1, exec_hi, src_scc
 v_mad_i16 v5.l, src_scc, vcc_lo, -1
 // GFX11: v_mad_i16 v5.l, src_scc, vcc_lo, -1     ; encoding: [0x05,0x00,0x53,0xd6,0xfd,0xd4,0x04,0x03]
 
-v_mad_i32_i16 v5, v1, v2, v3
-// GFX11: v_mad_i32_i16 v5, v1, v2, v3            ; encoding: [0x05,0x00,0x5a,0xd6,0x01,0x05,0x0e,0x04]
+v_mad_i32_i16 v5, v1.l, v2.l, v3
+// GFX11: v_mad_i32_i16 v5, v1.l, v2.l, v3        ; encoding: [0x05,0x00,0x5a,0xd6,0x01,0x05,0x0e,0x04]
 
-v_mad_i32_i16 v5, v255, v255, s3
-// GFX11: v_mad_i32_i16 v5, v255, v255, s3        ; encoding: [0x05,0x00,0x5a,0xd6,0xff,0xff,0x0f,0x00]
+v_mad_i32_i16 v5, v255.l, v255.l, s3
+// GFX11: v_mad_i32_i16 v5, v255.l, v255.l, s3    ; encoding: [0x05,0x00,0x5a,0xd6,0xff,0xff,0x0f,0x00]
 
 v_mad_i32_i16 v5, s1, s2, v255
 // GFX11: v_mad_i32_i16 v5, s1, s2, v255          ; encoding: [0x05,0x00,0x5a,0xd6,0x01,0x04,0xfc,0x07]
@@ -2951,6 +2954,18 @@ v_mad_i32_i16 v5, src_scc, vcc_lo, src_scc op_sel:[1,0,0,0]
 v_mad_i32_i16 v255, 0xfe0b, vcc_hi, 0.5 op_sel:[0,1,0,0] clamp
 // GFX11: v_mad_i32_i16 v255, 0xfe0b, vcc_hi, 0.5 op_sel:[0,1,0,0] clamp ; encoding: [0xff,0x90,0x5a,0xd6,0xff,0xd6,0xc0,0x03,0x0b,0xfe,0x00,0x00]
 
+v_mad_i32_i16 v5, v1.h, v2.l, v3
+// GFX11: v_mad_i32_i16 v5, v1.h, v2.l, v3 op_sel:[1,0,0,0] ; encoding: [0x05,0x08,0x5a,0xd6,0x01,0x05,0x0e,0x04]
+
+v_mad_i32_i16 v5, v255.l, v255.h, s3
+// GFX11: v_mad_i32_i16 v5, v255.l, v255.h, s3 op_sel:[0,1,0,0] ; encoding: [0x05,0x10,0x5a,0xd6,0xff,0xff,0x0f,0x00]
+
+v_mad_i32_i16 v5, src_scc, vcc_lo, src_scc
+// GFX11: v_mad_i32_i16 v5, src_scc, vcc_lo, src_scc ; encoding: [0x05,0x00,0x5a,0xd6,0xfd,0xd4,0xf4,0x03]
+
+v_mad_i32_i16 v255, 0xfe0b, vcc_hi, 0.5 clamp
+// GFX11: v_mad_i32_i16 v255, 0xfe0b, vcc_hi, 0.5 clamp ; encoding: [0xff,0x80,0x5a,0xd6,0xff,0xd6,0xc0,0x03,0x0b,0xfe,0x00,0x00]
+
 v_mad_i32_i24 v5, v1, v2, s3
 // GFX11: v_mad_i32_i24 v5, v1, v2, s3            ; encoding: [0x05,0x00,0x0a,0xd6,0x01,0x05,0x0e,0x00]
 
@@ -3134,11 +3149,11 @@ v_mad_u16 v5.l, -1, exec_hi, src_scc
 v_mad_u16 v5.l, src_scc, vcc_lo, -1
 // GFX11: v_mad_u16 v5.l, src_scc, vcc_lo, -1     ; encoding: [0x05,0x00,0x41,0xd6,0xfd,0xd4,0x04,0x03]
 
-v_mad_u32_u16 v5, v1, v2, v3
-// GFX11: v_mad_u32_u16 v5, v1, v2, v3            ; encoding: [0x05,0x00,0x59,0xd6,0x01,0x05,0x0e,0x04]
+v_mad_u32_u16 v5, v1.l, v2.l, v3
+// GFX11: v_mad_u32_u16 v5, v1.l, v2.l, v3        ; encoding: [0x05,0x00,0x59,0xd6,0x01,0x05,0x0e,0x04]
 
-v_mad_u32_u16 v5, v255, v255, s3
-// GFX11: v_mad_u32_u16 v5, v255, v255, s3        ; encoding: [0x05,0x00,0x59,0xd6,0xff,0xff,0x0f,0x00]
+v_mad_u32_u16 v5, v255.l, v255.l, s3
+// GFX11: v_mad_u32_u16 v5, v255.l, v255.l, s3    ; encoding: [0x05,0x00,0x59,0xd6,0xff,0xff,0x0f,0x00]
 
 v_mad_u32_u16 v5, s1, s2, v255
 // GFX11: v_mad_u32_u16 v5, s1, s2, v255          ; encoding: [0x05,0x00,0x59,0xd6,0x01,0x04,0xfc,0x07]
@@ -3179,6 +3194,18 @@ v_mad_u32_u16 v5, src_scc, vcc_lo, src_scc op_sel:[1,0,0,0]
 v_mad_u32_u16 v255, 0xfe0b, vcc_hi, 0.5 op_sel:[0,1,0,0] clamp
 // GFX11: v_mad_u32_u16 v255, 0xfe0b, vcc_hi, 0.5 op_sel:[0,1,0,0] clamp ; encoding: [0xff,0x90,0x59,0xd6,0xff,0xd6,0xc0,0x03,0x0b,0xfe,0x00,0x00]
 
+v_mad_u32_u16 v5, v1.h, v2.l, v3
+// GFX11: v_mad_u32_u16 v5, v1.h, v2.l, v3 op_sel:[1,0,0,0] ; encoding: [0x05,0x08,0x59,0xd6,0x01,0x05,0x0e,0x04]
+
+v_mad_u32_u16 v5, v255.l, v255.h, s3
+// GFX11: v_mad_u32_u16 v5, v255.l, v255.h, s3 op_sel:[0,1,0,0] ; encoding: [0x05,0x10,0x59,0xd6,0xff,0xff,0x0f,0x00]
+
+v_mad_u32_u16 v5, src_scc, vcc_lo, src_scc
+// GFX11: v_mad_u32_u16 v5, src_scc, vcc_lo, src_scc ; encoding: [0x05,0x00,0x59,0xd6,0xfd,0xd4,0xf4,0x03]
+
+v_mad_u32_u16 v255, 0xfe0b, vcc_hi, 0.5 clamp
+// GFX11: v_mad_u32_u16 v255, 0xfe0b, vcc_hi, 0.5 clamp ; encoding: [0xff,0x80,0x59,0xd6,0xff,0xd6,0xc0,0x03,0x0b,0xfe,0x00,0x00]
+
 v_mad_u32_u24 v5, v1, v2, s3
 // GFX11: v_mad_u32_u24 v5, v1, v2, s3            ; encoding: [0x05,0x00,0x0b,0xd6,0x01,0x05,0x0e,0x00]
 
diff --git a/llvm/test/MC/AMDGPU/gfx11_asm_vop3_dpp16.s b/llvm/test/MC/AMDGPU/gfx11_asm_vop3_dpp16.s
index 2bff644605ff6..1864996b26028 100644
--- a/llvm/test/MC/AMDGPU/gfx11_asm_vop3_dpp16.s
+++ b/llvm/test/MC/AMDGPU/gfx11_asm_vop3_dpp16.s
@@ -363,22 +363,22 @@ v_alignbit_b32_e64_dpp v5, v1, v2, -1 row_xmask:0 row_mask:0x1 bank_mask:0x3 bou
 v_alignbit_b32_e64_dpp v255, v255, v255, src_scc row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
 // GFX11: v_alignbit_b32_e64_dpp v255, v255, v255, src_scc row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xff,0x00,0x16,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x05,0x30]
 
-v_alignbyte_b32_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0]
-// GFX11: v_alignbyte_b32_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x17,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
+v_alignbyte_b32_e64_dpp v5, v1, v2, v3.l quad_perm:[3,2,1,0]
+// GFX11: v_alignbyte_b32_e64_dpp v5, v1, v2, v3.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x17,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 
-v_alignbyte_b32_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3]
-// GFX11: v_alignbyte_b32_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x17,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
+v_alignbyte_b32_e64_dpp v5, v1, v2, v3.l quad_perm:[0,1,2,3]
+// GFX11: v_alignbyte_b32_e64_dpp v5, v1, v2, v3.l quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x17,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
 
-v_alignbyte_b32_e64_dpp v5, v1, v2, v3 row_mirror
-// GFX11: v_alignbyte_b32_e64_dpp v5, v1, v2, v3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x17,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
+v_alignbyte_b32_e64_dpp v5, v1, v2, v3.l row_mirror row_mask:0xf bank_mask:0xf
+// GFX11: v_alignbyte_b32_e64_dpp v5, v1, v2, v3.l row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x17,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
 
-v_alignbyte_b32_e64_dpp v5, v1, v2, v3 row_half_mirror
-// GFX11: v_alignbyte_b32_e64_dpp v5, v1, v2, v3 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x17,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x41,0x01,0xff]
+v_alignbyte_b32_e64_dpp v5, v1, v2, v3.l row_half_mirror row_mask:0xf bank_mask:0xf
+// GFX11: v_alignbyte_b32_e64_dpp v5, v1, v2, v3.l row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x17,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x41,0x01,0xff]
 
-v_alignbyte_b32_e64_dpp v5, v1, v2, v255 row_shl:1
-// GFX11: v_alignbyte_b32_e64_dpp v5, v1, v2, v255 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x17,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x01,0x01,0xff]
+v_alignbyte_b32_e64_dpp v5, v1, v2, v255.l row_shl:1 row_mask:0xf bank_mask:0xf
+// GFX11: v_alignbyte_b32_e64_dpp v5, v1, v2, v255.l row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x17,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x01,0x01,0xff]
 
-v_alignbyte_b32_e64_dpp v5, v1, v2, s105 row_shl:15
+v_alignbyte_b32_e64_dpp v5, v1, v2, s105 row_shl:15 row_mask:0xf bank_mask:0xf
 // GFX11: v_alignbyte_b32_e64_dpp v5, v1, v2, s105 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x17,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x0f,0x01,0xff]
 
 v_alignbyte_b32_e64_dpp v5, v1, v2, vcc_hi row_shr:1
@@ -387,7 +387,7 @@ v_alignbyte_b32_e64_dpp v5, v1, v2, vcc_hi row_shr:1
 v_alignbyte_b32_e64_dpp v5, v1, v2, vcc_lo row_shr:15
 // GFX11: v_alignbyte_b32_e64_dpp v5, v1, v2, vcc_lo row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x17,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x1f,0x01,0xff]
 
-v_alignbyte_b32_e64_dpp v5, v1, v2, ttmp15 row_ror:1
+v_alignbyte_b32_e64_dpp v5, v1, v2, ttmp15 row_ror:1 row_mask:0xf bank_mask:0xf
 // GFX11: v_alignbyte_b32_e64_dpp v5, v1, v2, ttmp15 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x17,0xd6,0xfa,0x04,0xee,0x01,0x01,0x21,0x01,0xff]
 
 v_alignbyte_b32_e64_dpp v5, v1, v2, exec_hi row_ror:15
@@ -405,6 +405,24 @@ v_alignbyte_b32_e64_dpp v5, v1, v2, -1 row_xmask:0 row_mask:0x1 bank_mask:0x3 bo
 v_alignbyte_b32_e64_dpp v255, v255, v255, src_scc row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
 // GFX11: v_alignbyte_b32_e64_dpp v255, v255, v255, src_scc row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xff,0x00,0x17,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x05,0x30]
 
+v_alignbyte_b32_e64_dpp v5, v1, v2, v255.l row_mirror
+// GFX11: v_alignbyte_b32_e64_dpp v5, v1, v2, v255.l row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x17,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x40,0x01,0xff]
+
+v_alignbyte_b32_e64_dpp v5, v1, v2, s3 row_half_mirror
+// GFX11: v_alignbyte_b32_e64_dpp v5, v1, v2, s3 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x17,0xd6,0xfa,0x04,0x0e,0x00,0x01,0x41,0x01,0xff]
+
+v_alignbyte_b32_e64_dpp v5, v1, v2, s105 row_shl:1
+// GFX11: v_alignbyte_b32_e64_dpp v5, v1, v2, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x17,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
+
+v_alignbyte_b32_e64_dpp v5, v1, v2, ttmp15 row_shl:15
+// GFX11: v_alignbyte_b32_e64_dpp v5, v1, v2, ttmp15 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x17,0xd6,0xfa,0x04,0xee,0x01,0x01,0x0f,0x01,0xff]
+
+v_alignbyte_b32_e64_dpp v5, v1, v2, m0 row_ror:1
+// GFX11: v_alignbyte_b32_e64_dpp v5, v1, v2, m0 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x17,0xd6,0xfa,0x04,0xf6,0x01,0x01,0x21,0x01,0xff]
+
+v_alignbyte_b32_e64_dpp v5, v1, v2, v255.h row_mirror
+// GFX11: v_alignbyte_b32_e64_dpp v5, v1, v2, v255.h op_sel:[0,0,1,0] row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x20,0x17,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x40,0x01,0xff]
+
 v_and_b16_e64_dpp v5.l, v1.l, v2.l quad_perm:[3,2,1,0]
 // GFX11: v_and_b16_e64_dpp v5.l, v1.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x62,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
 
@@ -2000,47 +2018,68 @@ v_mad_i16_e64_dpp v5.l, v1.l, v2.h, -1 row_xmask:0 row_mask:0x1 bank_mask:0x3 bo
 v_mad_i16_e64_dpp v255.h, v255.l, v255.l, src_scc clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
 // GFX11: v_mad_i16_e64_dpp v255.h, v255.l, v255.l, src_scc op_sel:[0,0,0,1] clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xff,0xc0,0x53,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x05,0x30]
 
-v_mad_i32_i16_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0]
-// GFX11: v_mad_i32_i16_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5a,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
+v_mad_i32_i16_e64_dpp v5, v1.l, v2.l, v3 quad_perm:[3,2,1,0]
+// GFX11: v_mad_i32_i16_e64_dpp v5, v1.l, v2.l, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5a,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
+
+v_mad_i32_i16_e64_dpp v5, v1.l, v2.l, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf
+// GFX11: v_mad_i32_i16_e64_dpp v5, v1.l, v2.l, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5a,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
+
+v_mad_i32_i16_e64_dpp v5, v1.l, v2.l, v3 row_mirror row_mask:0xf bank_mask:0xf
+// GFX11: v_mad_i32_i16_e64_dpp v5, v1.l, v2.l, v3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5a,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
+
+v_mad_i32_i16_e64_dpp v5, v1.l, v2.l, v255 row_half_mirror row_mask:0xf bank_mask:0xf
+// GFX11: v_mad_i32_i16_e64_dpp v5, v1.l, v2.l, v255 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5a,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff]
+
+v_mad_i32_i16_e64_dpp v5, v1.l, v2.l, s105 row_shl:1 row_mask:0xf bank_mask:0xf
+// GFX11: v_mad_i32_i16_e64_dpp v5, v1.l, v2.l, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5a,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
 
-v_mad_i32_i16_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3]
-// GFX11: v_mad_i32_i16_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5a,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
+v_mad_i32_i16_e64_dpp v5, v1.l, v2.l, vcc_hi row_shl:15
+// GFX11: v_mad_i32_i16_e64_dpp v5, v1.l, v2.l, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5a,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
 
-v_mad_i32_i16_e64_dpp v5, v1, v2, v3 row_mirror
-// GFX11: v_mad_i32_i16_e64_dpp v5, v1, v2, v3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5a,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
+v_mad_i32_i16_e64_dpp v5, v1.l, v2.l, vcc_lo row_shr:1
+// GFX11: v_mad_i32_i16_e64_dpp v5, v1.l, v2.l, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5a,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
 
-v_mad_i32_i16_e64_dpp v5, v1, v2, v255 row_half_mirror
-// GFX11: v_mad_i32_i16_e64_dpp v5, v1, v2, v255 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5a,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff]
+v_mad_i32_i16_e64_dpp v5, v1.l, v2.l, ttmp15 row_shr:15 row_mask:0xf bank_mask:0xf
+// GFX11: v_mad_i32_i16_e64_dpp v5, v1.l, v2.l, ttmp15 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5a,0xd6,0xfa,0x04,0xee,0x01,0x01,0x1f,0x01,0xff]
 
-v_mad_i32_i16_e64_dpp v5, v1, v2, s105 row_shl:1
-// GFX11: v_mad_i32_i16_e64_dpp v5, v1, v2, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5a,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
+v_mad_i32_i16_e64_dpp v5, v1.l, v2.l, exec_hi row_ror:1
+// GFX11: v_mad_i32_i16_e64_dpp v5, v1.l, v2.l, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5a,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x21,0x01,0xff]
 
-v_mad_i32_i16_e64_dpp v5, v1, v2, vcc_hi row_shl:15
-// GFX11: v_mad_i32_i16_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5a,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
+v_mad_i32_i16_e64_dpp v5, v1.l, v2.l, exec_lo row_ror:15
+// GFX11: v_mad_i32_i16_e64_dpp v5, v1.l, v2.l, exec_lo row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5a,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x2f,0x01,0xff]
 
-v_mad_i32_i16_e64_dpp v5, v1, v2, vcc_lo row_shr:1
-// GFX11: v_mad_i32_i16_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5a,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
+v_mad_i32_i16_e64_dpp v5, v1.l, v2.l, null row_share:0 row_mask:0xf bank_mask:0xf
+// GFX11: v_mad_i32_i16_e64_dpp v5, v1.l, v2.l, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5a,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x50,0x01,0xff]
 
-v_mad_i32_i16_e64_dpp v5, v1, v2, ttmp15 row_shr:15
-// GFX11: v_mad_i32_i16_e64_dpp v5, v1, v2, ttmp15 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5a,0xd6,0xfa,0x04,0xee,0x01,0x01,0x1f,0x01,0xff]
+v_mad_i32_i16_e64_dpp v5, v1.l, v2.l, -1 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: v_mad_i32_i16_e64_dpp v5, v1.l, v2.l, -1 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x5a,0xd6,0xfa,0x04,0x06,0x03,0x01,0x5f,0x01,0x01]
 
-v_mad_i32_i16_e64_dpp v5, v1, v2, exec_hi row_ror:1
-// GFX11: v_mad_i32_i16_e64_dpp v5, v1, v2, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5a,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x21,0x01,0xff]
+v_mad_i32_i16_e64_dpp v5, v1.l, v2.l, 0.5 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1
+// GFX11: v_mad_i32_i16_e64_dpp v5, v1.l, v2.l, 0.5 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x05,0x00,0x5a,0xd6,0xfa,0x04,0xc2,0x03,0x01,0x60,0x09,0x13]
 
-v_mad_i32_i16_e64_dpp v5, v1, v2, exec_lo row_ror:15
-// GFX11: v_mad_i32_i16_e64_dpp v5, v1, v2, exec_lo row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5a,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x2f,0x01,0xff]
+v_mad_i32_i16_e64_dpp v255, v255.l, v255.l, src_scc clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1
+// GFX11: v_mad_i32_i16_e64_dpp v255, v255.l, v255.l, src_scc clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xff,0x80,0x5a,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x05,0x30]
 
-v_mad_i32_i16_e64_dpp v5, v1, v2, null row_share:0 row_mask:0xf bank_mask:0xf
-// GFX11: v_mad_i32_i16_e64_dpp v5, v1, v2, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5a,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x50,0x01,0xff]
+v_mad_i32_i16_e64_dpp v5, v1.l, v2.l, v255 quad_perm:[0,1,2,3]
+// GFX11: v_mad_i32_i16_e64_dpp v5, v1.l, v2.l, v255 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5a,0xd6,0xfa,0x04,0xfe,0x07,0x01,0xe4,0x00,0xff]
 
-v_mad_i32_i16_e64_dpp v5, v1, v2, -1 row_share:15 row_mask:0x0 bank_mask:0x1
-// GFX11: v_mad_i32_i16_e64_dpp v5, v1, v2, -1 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x5a,0xd6,0xfa,0x04,0x06,0x03,0x01,0x5f,0x01,0x01]
+v_mad_i32_i16_e64_dpp v5, v1.l, v2.l, s3 row_mirror
+// GFX11: v_mad_i32_i16_e64_dpp v5, v1.l, v2.l, s3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5a,0xd6,0xfa,0x04,0x0e,0x00,0x01,0x40,0x01,0xff]
 
-v_mad_i32_i16_e64_dpp v5, v1, v2, 0.5 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
-// GFX11: v_mad_i32_i16_e64_dpp v5, v1, v2, 0.5 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x05,0x00,0x5a,0xd6,0xfa,0x04,0xc2,0x03,0x01,0x60,0x09,0x13]
+v_mad_i32_i16_e64_dpp v5, v1.l, v2.l, s105 row_half_mirror
+// GFX11: v_mad_i32_i16_e64_dpp v5, v1.l, v2.l, s105 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5a,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x41,0x01,0xff]
 
-v_mad_i32_i16_e64_dpp v255, v255, v255, src_scc clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
-// GFX11: v_mad_i32_i16_e64_dpp v255, v255, v255, src_scc clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xff,0x80,0x5a,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x05,0x30]
+v_mad_i32_i16_e64_dpp v5, v1.l, v2.l, ttmp15 row_shl:1
+// GFX11: v_mad_i32_i16_e64_dpp v5, v1.l, v2.l, ttmp15 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5a,0xd6,0xfa,0x04,0xee,0x01,0x01,0x01,0x01,0xff]
+
+v_mad_i32_i16_e64_dpp v5, v1.l, v2.l, m0 row_shr:15
+// GFX11: v_mad_i32_i16_e64_dpp v5, v1.l, v2.l, m0 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5a,0xd6,0xfa,0x04,0xf6,0x01,0x01,0x1f,0x01,0xff]
+
+v_mad_i32_i16_e64_dpp v5, v1.h, v2.l, 0.5 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: v_mad_i32_i16_e64_dpp v5, v1.h, v2.l, 0.5 op_sel:[1,0,0,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x05,0x08,0x5a,0xd6,0xfa,0x04,0xc2,0x03,0x01,0x60,0x09,0x13]
+
+v_mad_i32_i16_e64_dpp v255, v255.l, v255.h, src_scc clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: v_mad_i32_i16_e64_dpp v255, v255.l, v255.h, src_scc op_sel:[0,1,0,0] clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xff,0x90,0x5a,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x05,0x30]
 
 v_mad_i32_i24_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0]
 // GFX11: v_mad_i32_i24_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0a,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
@@ -2153,47 +2192,68 @@ v_mad_u16_e64_dpp v5.l, v1.l, v2.h, -1 row_xmask:0 row_mask:0x1 bank_mask:0x3 bo
 v_mad_u16_e64_dpp v255.h, v255.l, v255.l, src_scc clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
 // GFX11: v_mad_u16_e64_dpp v255.h, v255.l, v255.l, src_scc op_sel:[0,0,0,1] clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xff,0xc0,0x41,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x05,0x30]
 
-v_mad_u32_u16_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0]
-// GFX11: v_mad_u32_u16_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x59,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
+v_mad_u32_u16_e64_dpp v5, v1.l, v2.l, v3 quad_perm:[3,2,1,0]
+// GFX11: v_mad_u32_u16_e64_dpp v5, v1.l, v2.l, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x59,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
+
+v_mad_u32_u16_e64_dpp v5, v1.l, v2.l, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf
+// GFX11: v_mad_u32_u16_e64_dpp v5, v1.l, v2.l, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x59,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
+
+v_mad_u32_u16_e64_dpp v5, v1.l, v2.l, v3 row_mirror row_mask:0xf bank_mask:0xf
+// GFX11: v_mad_u32_u16_e64_dpp v5, v1.l, v2.l, v3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x59,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
+
+v_mad_u32_u16_e64_dpp v5, v1.l, v2.l, v255 row_half_mirror row_mask:0xf bank_mask:0xf
+// GFX11: v_mad_u32_u16_e64_dpp v5, v1.l, v2.l, v255 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x59,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff]
+
+v_mad_u32_u16_e64_dpp v5, v1.l, v2.l, s105 row_shl:1 row_mask:0xf bank_mask:0xf
+// GFX11: v_mad_u32_u16_e64_dpp v5, v1.l, v2.l, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x59,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
+
+v_mad_u32_u16_e64_dpp v5, v1.l, v2.l, vcc_hi row_shl:15
+// GFX11: v_mad_u32_u16_e64_dpp v5, v1.l, v2.l, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x59,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
+
+v_mad_u32_u16_e64_dpp v5, v1.l, v2.l, vcc_lo row_shr:1
+// GFX11: v_mad_u32_u16_e64_dpp v5, v1.l, v2.l, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x59,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
+
+v_mad_u32_u16_e64_dpp v5, v1.l, v2.l, ttmp15 row_shr:15 row_mask:0xf bank_mask:0xf
+// GFX11: v_mad_u32_u16_e64_dpp v5, v1.l, v2.l, ttmp15 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x59,0xd6,0xfa,0x04,0xee,0x01,0x01,0x1f,0x01,0xff]
 
-v_mad_u32_u16_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3]
-// GFX11: v_mad_u32_u16_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x59,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
+v_mad_u32_u16_e64_dpp v5, v1.l, v2.l, exec_hi row_ror:1
+// GFX11: v_mad_u32_u16_e64_dpp v5, v1.l, v2.l, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x59,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x21,0x01,0xff]
 
-v_mad_u32_u16_e64_dpp v5, v1, v2, v3 row_mirror
-// GFX11: v_mad_u32_u16_e64_dpp v5, v1, v2, v3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x59,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
+v_mad_u32_u16_e64_dpp v5, v1.l, v2.l, exec_lo row_ror:15
+// GFX11: v_mad_u32_u16_e64_dpp v5, v1.l, v2.l, exec_lo row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x59,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x2f,0x01,0xff]
 
-v_mad_u32_u16_e64_dpp v5, v1, v2, v255 row_half_mirror
-// GFX11: v_mad_u32_u16_e64_dpp v5, v1, v2, v255 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x59,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff]
+v_mad_u32_u16_e64_dpp v5, v1.l, v2.l, null row_share:0 row_mask:0xf bank_mask:0xf
+// GFX11: v_mad_u32_u16_e64_dpp v5, v1.l, v2.l, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x59,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x50,0x01,0xff]
 
-v_mad_u32_u16_e64_dpp v5, v1, v2, s105 row_shl:1
-// GFX11: v_mad_u32_u16_e64_dpp v5, v1, v2, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x59,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
+v_mad_u32_u16_e64_dpp v5, v1.l, v2.l, -1 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: v_mad_u32_u16_e64_dpp v5, v1.l, v2.l, -1 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x59,0xd6,0xfa,0x04,0x06,0x03,0x01,0x5f,0x01,0x01]
 
-v_mad_u32_u16_e64_dpp v5, v1, v2, vcc_hi row_shl:15
-// GFX11: v_mad_u32_u16_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x59,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
+v_mad_u32_u16_e64_dpp v5, v1.l, v2.l, 0.5 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1
+// GFX11: v_mad_u32_u16_e64_dpp v5, v1.l, v2.l, 0.5 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x05,0x00,0x59,0xd6,0xfa,0x04,0xc2,0x03,0x01,0x60,0x09,0x13]
 
-v_mad_u32_u16_e64_dpp v5, v1, v2, vcc_lo row_shr:1
-// GFX11: v_mad_u32_u16_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x59,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
+v_mad_u32_u16_e64_dpp v255, v255.l, v255.l, src_scc clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1
+// GFX11: v_mad_u32_u16_e64_dpp v255, v255.l, v255.l, src_scc clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xff,0x80,0x59,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x05,0x30]
 
-v_mad_u32_u16_e64_dpp v5, v1, v2, ttmp15 row_shr:15
-// GFX11: v_mad_u32_u16_e64_dpp v5, v1, v2, ttmp15 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x59,0xd6,0xfa,0x04,0xee,0x01,0x01,0x1f,0x01,0xff]
+v_mad_u32_u16_e64_dpp v5, v1.l, v2.l, v255 quad_perm:[0,1,2,3]
+// GFX11: v_mad_u32_u16_e64_dpp v5, v1.l, v2.l, v255 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x59,0xd6,0xfa,0x04,0xfe,0x07,0x01,0xe4,0x00,0xff]
 
-v_mad_u32_u16_e64_dpp v5, v1, v2, exec_hi row_ror:1
-// GFX11: v_mad_u32_u16_e64_dpp v5, v1, v2, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x59,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x21,0x01,0xff]
+v_mad_u32_u16_e64_dpp v5, v1.l, v2.l, s3 row_mirror
+// GFX11: v_mad_u32_u16_e64_dpp v5, v1.l, v2.l, s3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x59,0xd6,0xfa,0x04,0x0e,0x00,0x01,0x40,0x01,0xff]
 
-v_mad_u32_u16_e64_dpp v5, v1, v2, exec_lo row_ror:15
-// GFX11: v_mad_u32_u16_e64_dpp v5, v1, v2, exec_lo row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x59,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x2f,0x01,0xff]
+v_mad_u32_u16_e64_dpp v5, v1.l, v2.l, s105 row_half_mirror
+// GFX11: v_mad_u32_u16_e64_dpp v5, v1.l, v2.l, s105 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x59,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x41,0x01,0xff]
 
-v_mad_u32_u16_e64_dpp v5, v1, v2, null row_share:0 row_mask:0xf bank_mask:0xf
-// GFX11: v_mad_u32_u16_e64_dpp v5, v1, v2, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x59,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x50,0x01,0xff]
+v_mad_u32_u16_e64_dpp v5, v1.l, v2.l, ttmp15 row_shl:1
+// GFX11: v_mad_u32_u16_e64_dpp v5, v1.l, v2.l, ttmp15 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x59,0xd6,0xfa,0x04,0xee,0x01,0x01,0x01,0x01,0xff]
 
-v_mad_u32_u16_e64_dpp v5, v1, v2, -1 row_share:15 row_mask:0x0 bank_mask:0x1
-// GFX11: v_mad_u32_u16_e64_dpp v5, v1, v2, -1 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x59,0xd6,0xfa,0x04,0x06,0x03,0x01,0x5f,0x01,0x01]
+v_mad_u32_u16_e64_dpp v5, v1.l, v2.l, m0 row_shr:15
+// GFX11: v_mad_u32_u16_e64_dpp v5, v1.l, v2.l, m0 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x59,0xd6,0xfa,0x04,0xf6,0x01,0x01,0x1f,0x01,0xff]
 
-v_mad_u32_u16_e64_dpp v5, v1, v2, 0.5 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
-// GFX11: v_mad_u32_u16_e64_dpp v5, v1, v2, 0.5 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x05,0x00,0x59,0xd6,0xfa,0x04,0xc2,0x03,0x01,0x60,0x09,0x13]
+v_mad_u32_u16_e64_dpp v5, v1.h, v2.l, 0.5 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: v_mad_u32_u16_e64_dpp v5, v1.h, v2.l, 0.5 op_sel:[1,0,0,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x05,0x08,0x59,0xd6,0xfa,0x04,0xc2,0x03,0x01,0x60,0x09,0x13]
 
-v_mad_u32_u16_e64_dpp v255, v255, v255, src_scc clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
-// GFX11: v_mad_u32_u16_e64_dpp v255, v255, v255, src_scc clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xff,0x80,0x59,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x05,0x30]
+v_mad_u32_u16_e64_dpp v255, v255.l, v255.h, src_scc clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: v_mad_u32_u16_e64_dpp v255, v255.l, v255.h, src_scc op_sel:[0,1,0,0] clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xff,0x90,0x59,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x05,0x30]
 
 v_mad_u32_u24_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0]
 // GFX11: v_mad_u32_u24_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0b,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
@@ -5016,11 +5076,11 @@ v_mad_i16_e64_dpp v5.l, v1.l, v2.l, -1 op_sel:[0,0,1,0] row_xmask:0 row_mask:0x1
 v_mad_i16_e64_dpp v255.h, v255.l, v255.l, src_scc op_sel:[0,0,0,1] clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1
 // GFX11: v_mad_i16_e64_dpp v255.h, v255.l, v255.l, src_scc op_sel:[0,0,0,1] clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0xc0,0x53,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30]
 
-v_mad_i32_i16_e64_dpp v5, v1, v2, 0.5 op_sel:[1,0,0,0] row_xmask:0 row_mask:0x1 bank_mask:0x3
-// GFX11: v_mad_i32_i16_e64_dpp v5, v1, v2, 0.5 op_sel:[1,0,0,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x08,0x5a,0xd6,0xfa,0x04,0xc2,0x03,0x01,0x60,0x01,0x13]
+v_mad_i32_i16_e64_dpp v5, v1.h, v2.l, 0.5 op_sel:[1,0,0,0] row_xmask:0 row_mask:0x1 bank_mask:0x3
+// GFX11: v_mad_i32_i16_e64_dpp v5, v1.h, v2.l, 0.5 op_sel:[1,0,0,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x08,0x5a,0xd6,0xfa,0x04,0xc2,0x03,0x01,0x60,0x01,0x13]
 
-v_mad_i32_i16_e64_dpp v255, v255, v255, src_scc op_sel:[0,1,0,0] clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1
-// GFX11: v_mad_i32_i16_e64_dpp v255, v255, v255, src_scc op_sel:[0,1,0,0] clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x90,0x5a,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30]
+v_mad_i32_i16_e64_dpp v255, v255.l, v255.h, src_scc op_sel:[0,1,0,0] clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1
+// GFX11: v_mad_i32_i16_e64_dpp v255, v255.l, v255.h, src_scc op_sel:[0,1,0,0] clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x90,0x5a,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30]
 
 v_mad_u16_e64_dpp v5.h, v1.h, v2.h, exec_hi op_sel:[1,1,1,1] row_ror:15 row_mask:0xf bank_mask:0xf
 // GFX11: v_mad_u16_e64_dpp v5.h, v1.h, v2.h, exec_hi op_sel:[1,1,1,1] row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x78,0x41,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x2f,0x01,0xff]
@@ -5037,11 +5097,11 @@ v_mad_u16_e64_dpp v5.l, v1.l, v2.l, -1 op_sel:[0,0,1,0] row_xmask:0 row_mask:0x1
 v_mad_u16_e64_dpp v255.h, v255.l, v255.l, src_scc op_sel:[0,0,0,1] clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1
 // GFX11: v_mad_u16_e64_dpp v255.h, v255.l, v255.l, src_scc op_sel:[0,0,0,1] clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0xc0,0x41,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30]
 
-v_mad_u32_u16_e64_dpp v5, v1, v2, 0.5 op_sel:[1,0,0,0] row_xmask:0 row_mask:0x1 bank_mask:0x3
-// GFX11: v_mad_u32_u16_e64_dpp v5, v1, v2, 0.5 op_sel:[1,0,0,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x08,0x59,0xd6,0xfa,0x04,0xc2,0x03,0x01,0x60,0x01,0x13]
+v_mad_u32_u16_e64_dpp v5, v1.h, v2.l, 0.5 op_sel:[1,0,0,0] row_xmask:0 row_mask:0x1 bank_mask:0x3
+// GFX11: v_mad_u32_u16_e64_dpp v5, v1.h, v2.l, 0.5 op_sel:[1,0,0,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x08,0x59,0xd6,0xfa,0x04,0xc2,0x03,0x01,0x60,0x01,0x13]
 
-v_mad_u32_u16_e64_dpp v255, v255, v255, src_scc op_sel:[0,1,0,0] clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1
-// GFX11: v_mad_u32_u16_e64_dpp v255, v255, v255, src_scc op_sel:[0,1,0,0] clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x90,0x59,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30]
+v_mad_u32_u16_e64_dpp v255, v255.l, v255.h, src_scc op_sel:[0,1,0,0] clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1
+// GFX11: v_mad_u32_u16_e64_dpp v255, v255.l, v255.h, src_scc op_sel:[0,1,0,0] clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x90,0x59,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30]
 
 v_max3_f16_e64_dpp v5.h, -v1.h, v2.h, |exec_lo| op_sel:[1,1,1,1] row_ror:15 row_mask:0xf bank_mask:0xf
 // GFX11: v_max3_f16_e64_dpp v5.h, -v1.h, v2.h, |exec_lo| op_sel:[1,1,1,1] row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x7c,0x4c,0xd6,0xfa,0x04,0xfa,0x21,0x01,0x2f,0x01,0xff]
diff --git a/llvm/test/MC/AMDGPU/gfx11_asm_vop3_dpp16_from_vopcx.s b/llvm/test/MC/AMDGPU/gfx11_asm_vop3_dpp16_from_vopcx.s
index 80264a4a791bb..6864ce20499cb 100644
--- a/llvm/test/MC/AMDGPU/gfx11_asm_vop3_dpp16_from_vopcx.s
+++ b/llvm/test/MC/AMDGPU/gfx11_asm_vop3_dpp16_from_vopcx.s
@@ -96,47 +96,56 @@ v_cmpx_class_f32_e64_dpp v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctr
 v_cmpx_class_f32_e64_dpp -|v255|, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
 // GFX11: v_cmpx_class_f32_e64_dpp -|v255|, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0x7e,0x01,0xfe,0xd4,0xfa,0xfe,0x03,0x20,0xff,0x6f,0x05,0x30]
 
-v_cmpx_eq_f16_e64_dpp v1, v2 quad_perm:[3,2,1,0]
-// GFX11: v_cmpx_eq_f16_e64_dpp v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x82,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+v_cmpx_eq_f16_e64_dpp v1.l, v2.l quad_perm:[3,2,1,0]
+// GFX11: v_cmpx_eq_f16_e64_dpp v1.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x82,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
 
-v_cmpx_eq_f16_e64_dpp v1, v2 quad_perm:[0,1,2,3]
-// GFX11: v_cmpx_eq_f16_e64_dpp v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x82,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+v_cmpx_eq_f16_e64_dpp v1.l, v2.l quad_perm:[0,1,2,3]
+// GFX11: v_cmpx_eq_f16_e64_dpp v1.l, v2.l quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x82,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
 
-v_cmpx_eq_f16_e64_dpp v1, v2 row_mirror
-// GFX11: v_cmpx_eq_f16_e64_dpp v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x82,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+v_cmpx_eq_f16_e64_dpp v1.l, v2.l row_mirror
+// GFX11: v_cmpx_eq_f16_e64_dpp v1.l, v2.l row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x82,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
 
-v_cmpx_eq_f16_e64_dpp v1, v2 row_half_mirror
-// GFX11: v_cmpx_eq_f16_e64_dpp v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x82,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+v_cmpx_eq_f16_e64_dpp v1.l, v2.l row_half_mirror
+// GFX11: v_cmpx_eq_f16_e64_dpp v1.l, v2.l row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x82,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
 
-v_cmpx_eq_f16_e64_dpp v1, v2 row_shl:1
-// GFX11: v_cmpx_eq_f16_e64_dpp v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x82,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+v_cmpx_eq_f16_e64_dpp v1.l, v2.l row_shl:1
+// GFX11: v_cmpx_eq_f16_e64_dpp v1.l, v2.l row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x82,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
 
-v_cmpx_eq_f16_e64_dpp v1, v2 row_shl:15
-// GFX11: v_cmpx_eq_f16_e64_dpp v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x82,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+v_cmpx_eq_f16_e64_dpp v1.l, v2.l row_shl:15
+// GFX11: v_cmpx_eq_f16_e64_dpp v1.l, v2.l row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x82,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
 
-v_cmpx_eq_f16_e64_dpp v1, v2 row_shr:1
-// GFX11: v_cmpx_eq_f16_e64_dpp v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x82,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+v_cmpx_eq_f16_e64_dpp v1.l, v2.l row_shr:1
+// GFX11: v_cmpx_eq_f16_e64_dpp v1.l, v2.l row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x82,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
 
-v_cmpx_eq_f16_e64_dpp v1, v2 row_shr:15
-// GFX11: v_cmpx_eq_f16_e64_dpp v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x82,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+v_cmpx_eq_f16_e64_dpp v1.l, v2.l row_shr:15
+// GFX11: v_cmpx_eq_f16_e64_dpp v1.l, v2.l row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x82,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
 
-v_cmpx_eq_f16_e64_dpp v1, v2 row_ror:1
-// GFX11: v_cmpx_eq_f16_e64_dpp v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x82,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+v_cmpx_eq_f16_e64_dpp v1.l, v2.l row_ror:1
+// GFX11: v_cmpx_eq_f16_e64_dpp v1.l, v2.l row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x82,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
 
-v_cmpx_eq_f16_e64_dpp v1, v2 row_ror:15
-// GFX11: v_cmpx_eq_f16_e64_dpp v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x82,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+v_cmpx_eq_f16_e64_dpp v1.l, v2.l row_ror:15
+// GFX11: v_cmpx_eq_f16_e64_dpp v1.l, v2.l row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x82,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
 
-v_cmpx_eq_f16_e64_dpp v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
-// GFX11: v_cmpx_eq_f16_e64_dpp v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x82,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+v_cmpx_eq_f16_e64_dpp v1.l, v2.l row_share:0 row_mask:0xf bank_mask:0xf
+// GFX11: v_cmpx_eq_f16_e64_dpp v1.l, v2.l row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x82,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 
-v_cmpx_eq_f16_e64_dpp |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1
-// GFX11: v_cmpx_eq_f16_e64_dpp |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x01,0x82,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+v_cmpx_eq_f16_e64_dpp |v1.l|, -v2.l row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: v_cmpx_eq_f16_e64_dpp |v1.l|, -v2.l row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x01,0x82,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
 
-v_cmpx_eq_f16_e64_dpp -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
-// GFX11: v_cmpx_eq_f16_e64_dpp -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7e,0x02,0x82,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x09,0x13]
+v_cmpx_eq_f16_e64_dpp -v1.l, |v2.l| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: v_cmpx_eq_f16_e64_dpp -v1.l, |v2.l| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7e,0x02,0x82,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x09,0x13]
 
-v_cmpx_eq_f16_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
-// GFX11: v_cmpx_eq_f16_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0x7e,0x83,0x82,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x05,0x30]
+v_cmpx_eq_f16_e64_dpp -|v255.l|, -|v255.l| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: v_cmpx_eq_f16_e64_dpp -|v255.l|, -|v255.l| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0x7e,0x83,0x82,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x05,0x30]
+
+v_cmpx_eq_f16_e64_dpp |v1.h|, -v2.h row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: v_cmpx_eq_f16_e64_dpp |v1.h|, -v2.h op_sel:[1,1] row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x19,0x82,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+
+v_cmpx_eq_f16_e64_dpp -v1.h, |v2.l| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: v_cmpx_eq_f16_e64_dpp -v1.h, |v2.l| op_sel:[1,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7e,0x0a,0x82,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x09,0x13]
+
+v_cmpx_eq_f16_e64_dpp -|v255.l|, -|v255.h| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: v_cmpx_eq_f16_e64_dpp -|v255.l|, -|v255.h| op_sel:[0,1] clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0x7e,0x93,0x82,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x05,0x30]
 
 v_cmpx_eq_f32_e64_dpp v1, v2 quad_perm:[3,2,1,0]
 // GFX11: v_cmpx_eq_f32_e64_dpp v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x92,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
@@ -366,47 +375,56 @@ v_cmpx_eq_u32_e64_dpp v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1
 v_cmpx_eq_u32_e64_dpp v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
 // GFX11: v_cmpx_eq_u32_e64_dpp v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0x7e,0x00,0xca,0xd4,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x05,0x30]
 
-v_cmpx_f_f16_e64_dpp v1, v2 quad_perm:[3,2,1,0]
-// GFX11: v_cmpx_f_f16_e64_dpp v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x80,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+v_cmpx_f_f16_e64_dpp v1.l, v2.l quad_perm:[3,2,1,0]
+// GFX11: v_cmpx_f_f16_e64_dpp v1.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x80,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+
+v_cmpx_f_f16_e64_dpp v1.l, v2.l quad_perm:[0,1,2,3]
+// GFX11: v_cmpx_f_f16_e64_dpp v1.l, v2.l quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x80,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+
+v_cmpx_f_f16_e64_dpp v1.l, v2.l row_mirror
+// GFX11: v_cmpx_f_f16_e64_dpp v1.l, v2.l row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x80,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+
+v_cmpx_f_f16_e64_dpp v1.l, v2.l row_half_mirror
+// GFX11: v_cmpx_f_f16_e64_dpp v1.l, v2.l row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x80,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
 
-v_cmpx_f_f16_e64_dpp v1, v2 quad_perm:[0,1,2,3]
-// GFX11: v_cmpx_f_f16_e64_dpp v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x80,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+v_cmpx_f_f16_e64_dpp v1.l, v2.l row_shl:1
+// GFX11: v_cmpx_f_f16_e64_dpp v1.l, v2.l row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x80,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
 
-v_cmpx_f_f16_e64_dpp v1, v2 row_mirror
-// GFX11: v_cmpx_f_f16_e64_dpp v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x80,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+v_cmpx_f_f16_e64_dpp v1.l, v2.l row_shl:15
+// GFX11: v_cmpx_f_f16_e64_dpp v1.l, v2.l row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x80,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
 
-v_cmpx_f_f16_e64_dpp v1, v2 row_half_mirror
-// GFX11: v_cmpx_f_f16_e64_dpp v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x80,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+v_cmpx_f_f16_e64_dpp v1.l, v2.l row_shr:1
+// GFX11: v_cmpx_f_f16_e64_dpp v1.l, v2.l row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x80,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
 
-v_cmpx_f_f16_e64_dpp v1, v2 row_shl:1
-// GFX11: v_cmpx_f_f16_e64_dpp v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x80,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+v_cmpx_f_f16_e64_dpp v1.l, v2.l row_shr:15
+// GFX11: v_cmpx_f_f16_e64_dpp v1.l, v2.l row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x80,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
 
-v_cmpx_f_f16_e64_dpp v1, v2 row_shl:15
-// GFX11: v_cmpx_f_f16_e64_dpp v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x80,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+v_cmpx_f_f16_e64_dpp v1.l, v2.l row_ror:1
+// GFX11: v_cmpx_f_f16_e64_dpp v1.l, v2.l row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x80,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
 
-v_cmpx_f_f16_e64_dpp v1, v2 row_shr:1
-// GFX11: v_cmpx_f_f16_e64_dpp v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x80,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+v_cmpx_f_f16_e64_dpp v1.l, v2.l row_ror:15
+// GFX11: v_cmpx_f_f16_e64_dpp v1.l, v2.l row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x80,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
 
-v_cmpx_f_f16_e64_dpp v1, v2 row_shr:15
-// GFX11: v_cmpx_f_f16_e64_dpp v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x80,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+v_cmpx_f_f16_e64_dpp v1.l, v2.l row_share:0 row_mask:0xf bank_mask:0xf
+// GFX11: v_cmpx_f_f16_e64_dpp v1.l, v2.l row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x80,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 
-v_cmpx_f_f16_e64_dpp v1, v2 row_ror:1
-// GFX11: v_cmpx_f_f16_e64_dpp v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x80,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+v_cmpx_f_f16_e64_dpp |v1.l|, -v2.l row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: v_cmpx_f_f16_e64_dpp |v1.l|, -v2.l row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x01,0x80,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
 
-v_cmpx_f_f16_e64_dpp v1, v2 row_ror:15
-// GFX11: v_cmpx_f_f16_e64_dpp v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x80,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+v_cmpx_f_f16_e64_dpp -v1.l, |v2.l| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: v_cmpx_f_f16_e64_dpp -v1.l, |v2.l| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7e,0x02,0x80,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x09,0x13]
 
-v_cmpx_f_f16_e64_dpp v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
-// GFX11: v_cmpx_f_f16_e64_dpp v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x80,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+v_cmpx_f_f16_e64_dpp -|v255.l|, -|v255.l| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: v_cmpx_f_f16_e64_dpp -|v255.l|, -|v255.l| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0x7e,0x83,0x80,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x05,0x30]
 
-v_cmpx_f_f16_e64_dpp |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1
-// GFX11: v_cmpx_f_f16_e64_dpp |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x01,0x80,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+v_cmpx_f_f16_e64_dpp |v1.h|, -v2.h row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: v_cmpx_f_f16_e64_dpp |v1.h|, -v2.h op_sel:[1,1] row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x19,0x80,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
 
-v_cmpx_f_f16_e64_dpp -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
-// GFX11: v_cmpx_f_f16_e64_dpp -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7e,0x02,0x80,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x09,0x13]
+v_cmpx_f_f16_e64_dpp -v1.h, |v2.l| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: v_cmpx_f_f16_e64_dpp -v1.h, |v2.l| op_sel:[1,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7e,0x0a,0x80,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x09,0x13]
 
-v_cmpx_f_f16_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
-// GFX11: v_cmpx_f_f16_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0x7e,0x83,0x80,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x05,0x30]
+v_cmpx_f_f16_e64_dpp -|v255.l|, -|v255.h| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: v_cmpx_f_f16_e64_dpp -|v255.l|, -|v255.h| op_sel:[0,1] clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0x7e,0x93,0x80,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x05,0x30]
 
 v_cmpx_f_f32_e64_dpp v1, v2 quad_perm:[3,2,1,0]
 // GFX11: v_cmpx_f_f32_e64_dpp v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x90,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
@@ -534,47 +552,56 @@ v_cmpx_f_u32_e64_dpp v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1
 v_cmpx_f_u32_e64_dpp v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
 // GFX11: v_cmpx_f_u32_e64_dpp v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0x7e,0x00,0xc8,0xd4,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x05,0x30]
 
-v_cmpx_ge_f16_e64_dpp v1, v2 quad_perm:[3,2,1,0]
-// GFX11: v_cmpx_ge_f16_e64_dpp v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x86,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+v_cmpx_ge_f16_e64_dpp v1.l, v2.l quad_perm:[3,2,1,0]
+// GFX11: v_cmpx_ge_f16_e64_dpp v1.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x86,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
 
-v_cmpx_ge_f16_e64_dpp v1, v2 quad_perm:[0,1,2,3]
-// GFX11: v_cmpx_ge_f16_e64_dpp v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x86,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+v_cmpx_ge_f16_e64_dpp v1.l, v2.l quad_perm:[0,1,2,3]
+// GFX11: v_cmpx_ge_f16_e64_dpp v1.l, v2.l quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x86,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
 
-v_cmpx_ge_f16_e64_dpp v1, v2 row_mirror
-// GFX11: v_cmpx_ge_f16_e64_dpp v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x86,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+v_cmpx_ge_f16_e64_dpp v1.l, v2.l row_mirror
+// GFX11: v_cmpx_ge_f16_e64_dpp v1.l, v2.l row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x86,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
 
-v_cmpx_ge_f16_e64_dpp v1, v2 row_half_mirror
-// GFX11: v_cmpx_ge_f16_e64_dpp v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x86,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+v_cmpx_ge_f16_e64_dpp v1.l, v2.l row_half_mirror
+// GFX11: v_cmpx_ge_f16_e64_dpp v1.l, v2.l row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x86,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
 
-v_cmpx_ge_f16_e64_dpp v1, v2 row_shl:1
-// GFX11: v_cmpx_ge_f16_e64_dpp v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x86,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+v_cmpx_ge_f16_e64_dpp v1.l, v2.l row_shl:1
+// GFX11: v_cmpx_ge_f16_e64_dpp v1.l, v2.l row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x86,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
 
-v_cmpx_ge_f16_e64_dpp v1, v2 row_shl:15
-// GFX11: v_cmpx_ge_f16_e64_dpp v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x86,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+v_cmpx_ge_f16_e64_dpp v1.l, v2.l row_shl:15
+// GFX11: v_cmpx_ge_f16_e64_dpp v1.l, v2.l row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x86,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
 
-v_cmpx_ge_f16_e64_dpp v1, v2 row_shr:1
-// GFX11: v_cmpx_ge_f16_e64_dpp v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x86,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+v_cmpx_ge_f16_e64_dpp v1.l, v2.l row_shr:1
+// GFX11: v_cmpx_ge_f16_e64_dpp v1.l, v2.l row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x86,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
 
-v_cmpx_ge_f16_e64_dpp v1, v2 row_shr:15
-// GFX11: v_cmpx_ge_f16_e64_dpp v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x86,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+v_cmpx_ge_f16_e64_dpp v1.l, v2.l row_shr:15
+// GFX11: v_cmpx_ge_f16_e64_dpp v1.l, v2.l row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x86,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
 
-v_cmpx_ge_f16_e64_dpp v1, v2 row_ror:1
-// GFX11: v_cmpx_ge_f16_e64_dpp v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x86,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+v_cmpx_ge_f16_e64_dpp v1.l, v2.l row_ror:1
+// GFX11: v_cmpx_ge_f16_e64_dpp v1.l, v2.l row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x86,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
 
-v_cmpx_ge_f16_e64_dpp v1, v2 row_ror:15
-// GFX11: v_cmpx_ge_f16_e64_dpp v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x86,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+v_cmpx_ge_f16_e64_dpp v1.l, v2.l row_ror:15
+// GFX11: v_cmpx_ge_f16_e64_dpp v1.l, v2.l row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x86,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
 
-v_cmpx_ge_f16_e64_dpp v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
-// GFX11: v_cmpx_ge_f16_e64_dpp v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x86,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+v_cmpx_ge_f16_e64_dpp v1.l, v2.l row_share:0 row_mask:0xf bank_mask:0xf
+// GFX11: v_cmpx_ge_f16_e64_dpp v1.l, v2.l row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x86,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 
-v_cmpx_ge_f16_e64_dpp |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1
-// GFX11: v_cmpx_ge_f16_e64_dpp |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x01,0x86,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+v_cmpx_ge_f16_e64_dpp |v1.l|, -v2.l row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: v_cmpx_ge_f16_e64_dpp |v1.l|, -v2.l row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x01,0x86,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
 
-v_cmpx_ge_f16_e64_dpp -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
-// GFX11: v_cmpx_ge_f16_e64_dpp -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7e,0x02,0x86,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x09,0x13]
+v_cmpx_ge_f16_e64_dpp -v1.l, |v2.l| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: v_cmpx_ge_f16_e64_dpp -v1.l, |v2.l| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7e,0x02,0x86,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x09,0x13]
 
-v_cmpx_ge_f16_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
-// GFX11: v_cmpx_ge_f16_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0x7e,0x83,0x86,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x05,0x30]
+v_cmpx_ge_f16_e64_dpp -|v255.l|, -|v255.l| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: v_cmpx_ge_f16_e64_dpp -|v255.l|, -|v255.l| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0x7e,0x83,0x86,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x05,0x30]
+
+v_cmpx_ge_f16_e64_dpp |v1.h|, -v2.h row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: v_cmpx_ge_f16_e64_dpp |v1.h|, -v2.h op_sel:[1,1] row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x19,0x86,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+
+v_cmpx_ge_f16_e64_dpp -v1.h, |v2.l| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: v_cmpx_ge_f16_e64_dpp -v1.h, |v2.l| op_sel:[1,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7e,0x0a,0x86,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x09,0x13]
+
+v_cmpx_ge_f16_e64_dpp -|v255.l|, -|v255.h| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: v_cmpx_ge_f16_e64_dpp -|v255.l|, -|v255.h| op_sel:[0,1] clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0x7e,0x93,0x86,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x05,0x30]
 
 v_cmpx_ge_f32_e64_dpp v1, v2 quad_perm:[3,2,1,0]
 // GFX11: v_cmpx_ge_f32_e64_dpp v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x96,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
@@ -804,47 +831,56 @@ v_cmpx_ge_u32_e64_dpp v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1
 v_cmpx_ge_u32_e64_dpp v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
 // GFX11: v_cmpx_ge_u32_e64_dpp v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0x7e,0x00,0xce,0xd4,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x05,0x30]
 
-v_cmpx_gt_f16_e64_dpp v1, v2 quad_perm:[3,2,1,0]
-// GFX11: v_cmpx_gt_f16_e64_dpp v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x84,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+v_cmpx_gt_f16_e64_dpp v1.l, v2.l quad_perm:[3,2,1,0]
+// GFX11: v_cmpx_gt_f16_e64_dpp v1.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x84,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+
+v_cmpx_gt_f16_e64_dpp v1.l, v2.l quad_perm:[0,1,2,3]
+// GFX11: v_cmpx_gt_f16_e64_dpp v1.l, v2.l quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x84,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+
+v_cmpx_gt_f16_e64_dpp v1.l, v2.l row_mirror
+// GFX11: v_cmpx_gt_f16_e64_dpp v1.l, v2.l row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x84,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+
+v_cmpx_gt_f16_e64_dpp v1.l, v2.l row_half_mirror
+// GFX11: v_cmpx_gt_f16_e64_dpp v1.l, v2.l row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x84,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
 
-v_cmpx_gt_f16_e64_dpp v1, v2 quad_perm:[0,1,2,3]
-// GFX11: v_cmpx_gt_f16_e64_dpp v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x84,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+v_cmpx_gt_f16_e64_dpp v1.l, v2.l row_shl:1
+// GFX11: v_cmpx_gt_f16_e64_dpp v1.l, v2.l row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x84,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
 
-v_cmpx_gt_f16_e64_dpp v1, v2 row_mirror
-// GFX11: v_cmpx_gt_f16_e64_dpp v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x84,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+v_cmpx_gt_f16_e64_dpp v1.l, v2.l row_shl:15
+// GFX11: v_cmpx_gt_f16_e64_dpp v1.l, v2.l row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x84,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
 
-v_cmpx_gt_f16_e64_dpp v1, v2 row_half_mirror
-// GFX11: v_cmpx_gt_f16_e64_dpp v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x84,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+v_cmpx_gt_f16_e64_dpp v1.l, v2.l row_shr:1
+// GFX11: v_cmpx_gt_f16_e64_dpp v1.l, v2.l row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x84,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
 
-v_cmpx_gt_f16_e64_dpp v1, v2 row_shl:1
-// GFX11: v_cmpx_gt_f16_e64_dpp v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x84,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+v_cmpx_gt_f16_e64_dpp v1.l, v2.l row_shr:15
+// GFX11: v_cmpx_gt_f16_e64_dpp v1.l, v2.l row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x84,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
 
-v_cmpx_gt_f16_e64_dpp v1, v2 row_shl:15
-// GFX11: v_cmpx_gt_f16_e64_dpp v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x84,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+v_cmpx_gt_f16_e64_dpp v1.l, v2.l row_ror:1
+// GFX11: v_cmpx_gt_f16_e64_dpp v1.l, v2.l row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x84,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
 
-v_cmpx_gt_f16_e64_dpp v1, v2 row_shr:1
-// GFX11: v_cmpx_gt_f16_e64_dpp v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x84,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+v_cmpx_gt_f16_e64_dpp v1.l, v2.l row_ror:15
+// GFX11: v_cmpx_gt_f16_e64_dpp v1.l, v2.l row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x84,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
 
-v_cmpx_gt_f16_e64_dpp v1, v2 row_shr:15
-// GFX11: v_cmpx_gt_f16_e64_dpp v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x84,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+v_cmpx_gt_f16_e64_dpp v1.l, v2.l row_share:0 row_mask:0xf bank_mask:0xf
+// GFX11: v_cmpx_gt_f16_e64_dpp v1.l, v2.l row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x84,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 
-v_cmpx_gt_f16_e64_dpp v1, v2 row_ror:1
-// GFX11: v_cmpx_gt_f16_e64_dpp v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x84,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+v_cmpx_gt_f16_e64_dpp |v1.l|, -v2.l row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: v_cmpx_gt_f16_e64_dpp |v1.l|, -v2.l row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x01,0x84,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
 
-v_cmpx_gt_f16_e64_dpp v1, v2 row_ror:15
-// GFX11: v_cmpx_gt_f16_e64_dpp v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x84,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+v_cmpx_gt_f16_e64_dpp -v1.l, |v2.l| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: v_cmpx_gt_f16_e64_dpp -v1.l, |v2.l| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7e,0x02,0x84,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x09,0x13]
 
-v_cmpx_gt_f16_e64_dpp v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
-// GFX11: v_cmpx_gt_f16_e64_dpp v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x84,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+v_cmpx_gt_f16_e64_dpp -|v255.l|, -|v255.l| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: v_cmpx_gt_f16_e64_dpp -|v255.l|, -|v255.l| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0x7e,0x83,0x84,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x05,0x30]
 
-v_cmpx_gt_f16_e64_dpp |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1
-// GFX11: v_cmpx_gt_f16_e64_dpp |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x01,0x84,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+v_cmpx_gt_f16_e64_dpp |v1.h|, -v2.h row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: v_cmpx_gt_f16_e64_dpp |v1.h|, -v2.h op_sel:[1,1] row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x19,0x84,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
 
-v_cmpx_gt_f16_e64_dpp -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
-// GFX11: v_cmpx_gt_f16_e64_dpp -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7e,0x02,0x84,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x09,0x13]
+v_cmpx_gt_f16_e64_dpp -v1.h, |v2.l| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: v_cmpx_gt_f16_e64_dpp -v1.h, |v2.l| op_sel:[1,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7e,0x0a,0x84,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x09,0x13]
 
-v_cmpx_gt_f16_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
-// GFX11: v_cmpx_gt_f16_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0x7e,0x83,0x84,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x05,0x30]
+v_cmpx_gt_f16_e64_dpp -|v255.l|, -|v255.h| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: v_cmpx_gt_f16_e64_dpp -|v255.l|, -|v255.h| op_sel:[0,1] clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0x7e,0x93,0x84,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x05,0x30]
 
 v_cmpx_gt_f32_e64_dpp v1, v2 quad_perm:[3,2,1,0]
 // GFX11: v_cmpx_gt_f32_e64_dpp v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x94,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
@@ -1074,47 +1110,56 @@ v_cmpx_gt_u32_e64_dpp v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1
 v_cmpx_gt_u32_e64_dpp v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
 // GFX11: v_cmpx_gt_u32_e64_dpp v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0x7e,0x00,0xcc,0xd4,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x05,0x30]
 
-v_cmpx_le_f16_e64_dpp v1, v2 quad_perm:[3,2,1,0]
-// GFX11: v_cmpx_le_f16_e64_dpp v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x83,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+v_cmpx_le_f16_e64_dpp v1.l, v2.l quad_perm:[3,2,1,0]
+// GFX11: v_cmpx_le_f16_e64_dpp v1.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x83,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
 
-v_cmpx_le_f16_e64_dpp v1, v2 quad_perm:[0,1,2,3]
-// GFX11: v_cmpx_le_f16_e64_dpp v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x83,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+v_cmpx_le_f16_e64_dpp v1.l, v2.l quad_perm:[0,1,2,3]
+// GFX11: v_cmpx_le_f16_e64_dpp v1.l, v2.l quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x83,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
 
-v_cmpx_le_f16_e64_dpp v1, v2 row_mirror
-// GFX11: v_cmpx_le_f16_e64_dpp v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x83,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+v_cmpx_le_f16_e64_dpp v1.l, v2.l row_mirror
+// GFX11: v_cmpx_le_f16_e64_dpp v1.l, v2.l row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x83,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
 
-v_cmpx_le_f16_e64_dpp v1, v2 row_half_mirror
-// GFX11: v_cmpx_le_f16_e64_dpp v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x83,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+v_cmpx_le_f16_e64_dpp v1.l, v2.l row_half_mirror
+// GFX11: v_cmpx_le_f16_e64_dpp v1.l, v2.l row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x83,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
 
-v_cmpx_le_f16_e64_dpp v1, v2 row_shl:1
-// GFX11: v_cmpx_le_f16_e64_dpp v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x83,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+v_cmpx_le_f16_e64_dpp v1.l, v2.l row_shl:1
+// GFX11: v_cmpx_le_f16_e64_dpp v1.l, v2.l row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x83,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
 
-v_cmpx_le_f16_e64_dpp v1, v2 row_shl:15
-// GFX11: v_cmpx_le_f16_e64_dpp v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x83,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+v_cmpx_le_f16_e64_dpp v1.l, v2.l row_shl:15
+// GFX11: v_cmpx_le_f16_e64_dpp v1.l, v2.l row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x83,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
 
-v_cmpx_le_f16_e64_dpp v1, v2 row_shr:1
-// GFX11: v_cmpx_le_f16_e64_dpp v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x83,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+v_cmpx_le_f16_e64_dpp v1.l, v2.l row_shr:1
+// GFX11: v_cmpx_le_f16_e64_dpp v1.l, v2.l row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x83,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
 
-v_cmpx_le_f16_e64_dpp v1, v2 row_shr:15
-// GFX11: v_cmpx_le_f16_e64_dpp v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x83,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+v_cmpx_le_f16_e64_dpp v1.l, v2.l row_shr:15
+// GFX11: v_cmpx_le_f16_e64_dpp v1.l, v2.l row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x83,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
 
-v_cmpx_le_f16_e64_dpp v1, v2 row_ror:1
-// GFX11: v_cmpx_le_f16_e64_dpp v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x83,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+v_cmpx_le_f16_e64_dpp v1.l, v2.l row_ror:1
+// GFX11: v_cmpx_le_f16_e64_dpp v1.l, v2.l row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x83,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
 
-v_cmpx_le_f16_e64_dpp v1, v2 row_ror:15
-// GFX11: v_cmpx_le_f16_e64_dpp v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x83,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+v_cmpx_le_f16_e64_dpp v1.l, v2.l row_ror:15
+// GFX11: v_cmpx_le_f16_e64_dpp v1.l, v2.l row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x83,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
 
-v_cmpx_le_f16_e64_dpp v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
-// GFX11: v_cmpx_le_f16_e64_dpp v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x83,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+v_cmpx_le_f16_e64_dpp v1.l, v2.l row_share:0 row_mask:0xf bank_mask:0xf
+// GFX11: v_cmpx_le_f16_e64_dpp v1.l, v2.l row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x83,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 
-v_cmpx_le_f16_e64_dpp |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1
-// GFX11: v_cmpx_le_f16_e64_dpp |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x01,0x83,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+v_cmpx_le_f16_e64_dpp |v1.l|, -v2.l row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: v_cmpx_le_f16_e64_dpp |v1.l|, -v2.l row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x01,0x83,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
 
-v_cmpx_le_f16_e64_dpp -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
-// GFX11: v_cmpx_le_f16_e64_dpp -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7e,0x02,0x83,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x09,0x13]
+v_cmpx_le_f16_e64_dpp -v1.l, |v2.l| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: v_cmpx_le_f16_e64_dpp -v1.l, |v2.l| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7e,0x02,0x83,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x09,0x13]
 
-v_cmpx_le_f16_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
-// GFX11: v_cmpx_le_f16_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0x7e,0x83,0x83,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x05,0x30]
+v_cmpx_le_f16_e64_dpp -|v255.l|, -|v255.l| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: v_cmpx_le_f16_e64_dpp -|v255.l|, -|v255.l| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0x7e,0x83,0x83,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x05,0x30]
+
+v_cmpx_le_f16_e64_dpp |v1.h|, -v2.h row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: v_cmpx_le_f16_e64_dpp |v1.h|, -v2.h op_sel:[1,1] row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x19,0x83,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+
+v_cmpx_le_f16_e64_dpp -v1.h, |v2.l| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: v_cmpx_le_f16_e64_dpp -v1.h, |v2.l| op_sel:[1,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7e,0x0a,0x83,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x09,0x13]
+
+v_cmpx_le_f16_e64_dpp -|v255.l|, -|v255.h| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: v_cmpx_le_f16_e64_dpp -|v255.l|, -|v255.h| op_sel:[0,1] clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0x7e,0x93,0x83,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x05,0x30]
 
 v_cmpx_le_f32_e64_dpp v1, v2 quad_perm:[3,2,1,0]
 // GFX11: v_cmpx_le_f32_e64_dpp v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x93,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
@@ -1344,47 +1389,56 @@ v_cmpx_le_u32_e64_dpp v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1
 v_cmpx_le_u32_e64_dpp v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
 // GFX11: v_cmpx_le_u32_e64_dpp v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0x7e,0x00,0xcb,0xd4,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x05,0x30]
 
-v_cmpx_lg_f16_e64_dpp v1, v2 quad_perm:[3,2,1,0]
-// GFX11: v_cmpx_lg_f16_e64_dpp v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x85,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+v_cmpx_lg_f16_e64_dpp v1.l, v2.l quad_perm:[3,2,1,0]
+// GFX11: v_cmpx_lg_f16_e64_dpp v1.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x85,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+
+v_cmpx_lg_f16_e64_dpp v1.l, v2.l quad_perm:[0,1,2,3]
+// GFX11: v_cmpx_lg_f16_e64_dpp v1.l, v2.l quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x85,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+
+v_cmpx_lg_f16_e64_dpp v1.l, v2.l row_mirror
+// GFX11: v_cmpx_lg_f16_e64_dpp v1.l, v2.l row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x85,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+
+v_cmpx_lg_f16_e64_dpp v1.l, v2.l row_half_mirror
+// GFX11: v_cmpx_lg_f16_e64_dpp v1.l, v2.l row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x85,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
 
-v_cmpx_lg_f16_e64_dpp v1, v2 quad_perm:[0,1,2,3]
-// GFX11: v_cmpx_lg_f16_e64_dpp v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x85,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+v_cmpx_lg_f16_e64_dpp v1.l, v2.l row_shl:1
+// GFX11: v_cmpx_lg_f16_e64_dpp v1.l, v2.l row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x85,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
 
-v_cmpx_lg_f16_e64_dpp v1, v2 row_mirror
-// GFX11: v_cmpx_lg_f16_e64_dpp v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x85,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+v_cmpx_lg_f16_e64_dpp v1.l, v2.l row_shl:15
+// GFX11: v_cmpx_lg_f16_e64_dpp v1.l, v2.l row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x85,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
 
-v_cmpx_lg_f16_e64_dpp v1, v2 row_half_mirror
-// GFX11: v_cmpx_lg_f16_e64_dpp v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x85,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+v_cmpx_lg_f16_e64_dpp v1.l, v2.l row_shr:1
+// GFX11: v_cmpx_lg_f16_e64_dpp v1.l, v2.l row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x85,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
 
-v_cmpx_lg_f16_e64_dpp v1, v2 row_shl:1
-// GFX11: v_cmpx_lg_f16_e64_dpp v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x85,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+v_cmpx_lg_f16_e64_dpp v1.l, v2.l row_shr:15
+// GFX11: v_cmpx_lg_f16_e64_dpp v1.l, v2.l row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x85,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
 
-v_cmpx_lg_f16_e64_dpp v1, v2 row_shl:15
-// GFX11: v_cmpx_lg_f16_e64_dpp v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x85,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+v_cmpx_lg_f16_e64_dpp v1.l, v2.l row_ror:1
+// GFX11: v_cmpx_lg_f16_e64_dpp v1.l, v2.l row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x85,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
 
-v_cmpx_lg_f16_e64_dpp v1, v2 row_shr:1
-// GFX11: v_cmpx_lg_f16_e64_dpp v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x85,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+v_cmpx_lg_f16_e64_dpp v1.l, v2.l row_ror:15
+// GFX11: v_cmpx_lg_f16_e64_dpp v1.l, v2.l row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x85,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
 
-v_cmpx_lg_f16_e64_dpp v1, v2 row_shr:15
-// GFX11: v_cmpx_lg_f16_e64_dpp v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x85,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+v_cmpx_lg_f16_e64_dpp v1.l, v2.l row_share:0 row_mask:0xf bank_mask:0xf
+// GFX11: v_cmpx_lg_f16_e64_dpp v1.l, v2.l row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x85,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 
-v_cmpx_lg_f16_e64_dpp v1, v2 row_ror:1
-// GFX11: v_cmpx_lg_f16_e64_dpp v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x85,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+v_cmpx_lg_f16_e64_dpp |v1.l|, -v2.l row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: v_cmpx_lg_f16_e64_dpp |v1.l|, -v2.l row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x01,0x85,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
 
-v_cmpx_lg_f16_e64_dpp v1, v2 row_ror:15
-// GFX11: v_cmpx_lg_f16_e64_dpp v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x85,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+v_cmpx_lg_f16_e64_dpp -v1.l, |v2.l| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: v_cmpx_lg_f16_e64_dpp -v1.l, |v2.l| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7e,0x02,0x85,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x09,0x13]
 
-v_cmpx_lg_f16_e64_dpp v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
-// GFX11: v_cmpx_lg_f16_e64_dpp v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x85,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+v_cmpx_lg_f16_e64_dpp -|v255.l|, -|v255.l| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: v_cmpx_lg_f16_e64_dpp -|v255.l|, -|v255.l| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0x7e,0x83,0x85,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x05,0x30]
 
-v_cmpx_lg_f16_e64_dpp |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1
-// GFX11: v_cmpx_lg_f16_e64_dpp |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x01,0x85,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+v_cmpx_lg_f16_e64_dpp |v1.h|, -v2.h row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: v_cmpx_lg_f16_e64_dpp |v1.h|, -v2.h op_sel:[1,1] row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x19,0x85,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
 
-v_cmpx_lg_f16_e64_dpp -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
-// GFX11: v_cmpx_lg_f16_e64_dpp -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7e,0x02,0x85,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x09,0x13]
+v_cmpx_lg_f16_e64_dpp -v1.h, |v2.l| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: v_cmpx_lg_f16_e64_dpp -v1.h, |v2.l| op_sel:[1,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7e,0x0a,0x85,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x09,0x13]
 
-v_cmpx_lg_f16_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
-// GFX11: v_cmpx_lg_f16_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0x7e,0x83,0x85,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x05,0x30]
+v_cmpx_lg_f16_e64_dpp -|v255.l|, -|v255.h| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: v_cmpx_lg_f16_e64_dpp -|v255.l|, -|v255.h| op_sel:[0,1] clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0x7e,0x93,0x85,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x05,0x30]
 
 v_cmpx_lg_f32_e64_dpp v1, v2 quad_perm:[3,2,1,0]
 // GFX11: v_cmpx_lg_f32_e64_dpp v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x95,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
@@ -1893,47 +1947,56 @@ v_cmpx_ne_u32_e64_dpp v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1
 v_cmpx_ne_u32_e64_dpp v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
 // GFX11: v_cmpx_ne_u32_e64_dpp v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0x7e,0x00,0xcd,0xd4,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x05,0x30]
 
-v_cmpx_neq_f16_e64_dpp v1, v2 quad_perm:[3,2,1,0]
-// GFX11: v_cmpx_neq_f16_e64_dpp v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+v_cmpx_neq_f16_e64_dpp v1.l, v2.l quad_perm:[3,2,1,0]
+// GFX11: v_cmpx_neq_f16_e64_dpp v1.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
 
-v_cmpx_neq_f16_e64_dpp v1, v2 quad_perm:[0,1,2,3]
-// GFX11: v_cmpx_neq_f16_e64_dpp v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8d,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+v_cmpx_neq_f16_e64_dpp v1.l, v2.l quad_perm:[0,1,2,3]
+// GFX11: v_cmpx_neq_f16_e64_dpp v1.l, v2.l quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8d,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
 
-v_cmpx_neq_f16_e64_dpp v1, v2 row_mirror
-// GFX11: v_cmpx_neq_f16_e64_dpp v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+v_cmpx_neq_f16_e64_dpp v1.l, v2.l row_mirror
+// GFX11: v_cmpx_neq_f16_e64_dpp v1.l, v2.l row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
 
-v_cmpx_neq_f16_e64_dpp v1, v2 row_half_mirror
-// GFX11: v_cmpx_neq_f16_e64_dpp v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+v_cmpx_neq_f16_e64_dpp v1.l, v2.l row_half_mirror
+// GFX11: v_cmpx_neq_f16_e64_dpp v1.l, v2.l row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
 
-v_cmpx_neq_f16_e64_dpp v1, v2 row_shl:1
-// GFX11: v_cmpx_neq_f16_e64_dpp v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+v_cmpx_neq_f16_e64_dpp v1.l, v2.l row_shl:1
+// GFX11: v_cmpx_neq_f16_e64_dpp v1.l, v2.l row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
 
-v_cmpx_neq_f16_e64_dpp v1, v2 row_shl:15
-// GFX11: v_cmpx_neq_f16_e64_dpp v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+v_cmpx_neq_f16_e64_dpp v1.l, v2.l row_shl:15
+// GFX11: v_cmpx_neq_f16_e64_dpp v1.l, v2.l row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
 
-v_cmpx_neq_f16_e64_dpp v1, v2 row_shr:1
-// GFX11: v_cmpx_neq_f16_e64_dpp v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+v_cmpx_neq_f16_e64_dpp v1.l, v2.l row_shr:1
+// GFX11: v_cmpx_neq_f16_e64_dpp v1.l, v2.l row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
 
-v_cmpx_neq_f16_e64_dpp v1, v2 row_shr:15
-// GFX11: v_cmpx_neq_f16_e64_dpp v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+v_cmpx_neq_f16_e64_dpp v1.l, v2.l row_shr:15
+// GFX11: v_cmpx_neq_f16_e64_dpp v1.l, v2.l row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
 
-v_cmpx_neq_f16_e64_dpp v1, v2 row_ror:1
-// GFX11: v_cmpx_neq_f16_e64_dpp v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+v_cmpx_neq_f16_e64_dpp v1.l, v2.l row_ror:1
+// GFX11: v_cmpx_neq_f16_e64_dpp v1.l, v2.l row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
 
-v_cmpx_neq_f16_e64_dpp v1, v2 row_ror:15
-// GFX11: v_cmpx_neq_f16_e64_dpp v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+v_cmpx_neq_f16_e64_dpp v1.l, v2.l row_ror:15
+// GFX11: v_cmpx_neq_f16_e64_dpp v1.l, v2.l row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
 
-v_cmpx_neq_f16_e64_dpp v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
-// GFX11: v_cmpx_neq_f16_e64_dpp v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+v_cmpx_neq_f16_e64_dpp v1.l, v2.l row_share:0 row_mask:0xf bank_mask:0xf
+// GFX11: v_cmpx_neq_f16_e64_dpp v1.l, v2.l row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 
-v_cmpx_neq_f16_e64_dpp |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1
-// GFX11: v_cmpx_neq_f16_e64_dpp |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x01,0x8d,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+v_cmpx_neq_f16_e64_dpp |v1.l|, -v2.l row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: v_cmpx_neq_f16_e64_dpp |v1.l|, -v2.l row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x01,0x8d,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
 
-v_cmpx_neq_f16_e64_dpp -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
-// GFX11: v_cmpx_neq_f16_e64_dpp -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7e,0x02,0x8d,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x09,0x13]
+v_cmpx_neq_f16_e64_dpp -v1.l, |v2.l| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: v_cmpx_neq_f16_e64_dpp -v1.l, |v2.l| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7e,0x02,0x8d,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x09,0x13]
 
-v_cmpx_neq_f16_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
-// GFX11: v_cmpx_neq_f16_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0x7e,0x83,0x8d,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x05,0x30]
+v_cmpx_neq_f16_e64_dpp -|v255.l|, -|v255.l| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: v_cmpx_neq_f16_e64_dpp -|v255.l|, -|v255.l| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0x7e,0x83,0x8d,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x05,0x30]
+
+v_cmpx_neq_f16_e64_dpp |v1.h|, -v2.h row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: v_cmpx_neq_f16_e64_dpp |v1.h|, -v2.h op_sel:[1,1] row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x19,0x8d,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+
+v_cmpx_neq_f16_e64_dpp -v1.h, |v2.l| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: v_cmpx_neq_f16_e64_dpp -v1.h, |v2.l| op_sel:[1,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7e,0x0a,0x8d,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x09,0x13]
+
+v_cmpx_neq_f16_e64_dpp -|v255.l|, -|v255.h| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: v_cmpx_neq_f16_e64_dpp -|v255.l|, -|v255.h| op_sel:[0,1] clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0x7e,0x93,0x8d,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x05,0x30]
 
 v_cmpx_neq_f32_e64_dpp v1, v2 quad_perm:[3,2,1,0]
 // GFX11: v_cmpx_neq_f32_e64_dpp v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x9d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
@@ -1977,47 +2040,56 @@ v_cmpx_neq_f32_e64_dpp -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ct
 v_cmpx_neq_f32_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
 // GFX11: v_cmpx_neq_f32_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0x7e,0x83,0x9d,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x05,0x30]
 
-v_cmpx_nge_f16_e64_dpp v1, v2 quad_perm:[3,2,1,0]
-// GFX11: v_cmpx_nge_f16_e64_dpp v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x89,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+v_cmpx_nge_f16_e64_dpp v1.l, v2.l quad_perm:[3,2,1,0]
+// GFX11: v_cmpx_nge_f16_e64_dpp v1.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x89,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+
+v_cmpx_nge_f16_e64_dpp v1.l, v2.l quad_perm:[0,1,2,3]
+// GFX11: v_cmpx_nge_f16_e64_dpp v1.l, v2.l quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x89,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+
+v_cmpx_nge_f16_e64_dpp v1.l, v2.l row_mirror
+// GFX11: v_cmpx_nge_f16_e64_dpp v1.l, v2.l row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x89,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
 
-v_cmpx_nge_f16_e64_dpp v1, v2 quad_perm:[0,1,2,3]
-// GFX11: v_cmpx_nge_f16_e64_dpp v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x89,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+v_cmpx_nge_f16_e64_dpp v1.l, v2.l row_half_mirror
+// GFX11: v_cmpx_nge_f16_e64_dpp v1.l, v2.l row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x89,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
 
-v_cmpx_nge_f16_e64_dpp v1, v2 row_mirror
-// GFX11: v_cmpx_nge_f16_e64_dpp v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x89,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+v_cmpx_nge_f16_e64_dpp v1.l, v2.l row_shl:1
+// GFX11: v_cmpx_nge_f16_e64_dpp v1.l, v2.l row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x89,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
 
-v_cmpx_nge_f16_e64_dpp v1, v2 row_half_mirror
-// GFX11: v_cmpx_nge_f16_e64_dpp v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x89,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+v_cmpx_nge_f16_e64_dpp v1.l, v2.l row_shl:15
+// GFX11: v_cmpx_nge_f16_e64_dpp v1.l, v2.l row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x89,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
 
-v_cmpx_nge_f16_e64_dpp v1, v2 row_shl:1
-// GFX11: v_cmpx_nge_f16_e64_dpp v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x89,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+v_cmpx_nge_f16_e64_dpp v1.l, v2.l row_shr:1
+// GFX11: v_cmpx_nge_f16_e64_dpp v1.l, v2.l row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x89,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
 
-v_cmpx_nge_f16_e64_dpp v1, v2 row_shl:15
-// GFX11: v_cmpx_nge_f16_e64_dpp v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x89,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+v_cmpx_nge_f16_e64_dpp v1.l, v2.l row_shr:15
+// GFX11: v_cmpx_nge_f16_e64_dpp v1.l, v2.l row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x89,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
 
-v_cmpx_nge_f16_e64_dpp v1, v2 row_shr:1
-// GFX11: v_cmpx_nge_f16_e64_dpp v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x89,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+v_cmpx_nge_f16_e64_dpp v1.l, v2.l row_ror:1
+// GFX11: v_cmpx_nge_f16_e64_dpp v1.l, v2.l row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x89,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
 
-v_cmpx_nge_f16_e64_dpp v1, v2 row_shr:15
-// GFX11: v_cmpx_nge_f16_e64_dpp v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x89,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+v_cmpx_nge_f16_e64_dpp v1.l, v2.l row_ror:15
+// GFX11: v_cmpx_nge_f16_e64_dpp v1.l, v2.l row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x89,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
 
-v_cmpx_nge_f16_e64_dpp v1, v2 row_ror:1
-// GFX11: v_cmpx_nge_f16_e64_dpp v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x89,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+v_cmpx_nge_f16_e64_dpp v1.l, v2.l row_share:0 row_mask:0xf bank_mask:0xf
+// GFX11: v_cmpx_nge_f16_e64_dpp v1.l, v2.l row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x89,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 
-v_cmpx_nge_f16_e64_dpp v1, v2 row_ror:15
-// GFX11: v_cmpx_nge_f16_e64_dpp v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x89,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+v_cmpx_nge_f16_e64_dpp |v1.l|, -v2.l row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: v_cmpx_nge_f16_e64_dpp |v1.l|, -v2.l row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x01,0x89,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
 
-v_cmpx_nge_f16_e64_dpp v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
-// GFX11: v_cmpx_nge_f16_e64_dpp v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x89,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+v_cmpx_nge_f16_e64_dpp -v1.l, |v2.l| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: v_cmpx_nge_f16_e64_dpp -v1.l, |v2.l| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7e,0x02,0x89,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x09,0x13]
 
-v_cmpx_nge_f16_e64_dpp |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1
-// GFX11: v_cmpx_nge_f16_e64_dpp |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x01,0x89,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+v_cmpx_nge_f16_e64_dpp -|v255.l|, -|v255.l| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: v_cmpx_nge_f16_e64_dpp -|v255.l|, -|v255.l| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0x7e,0x83,0x89,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x05,0x30]
 
-v_cmpx_nge_f16_e64_dpp -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
-// GFX11: v_cmpx_nge_f16_e64_dpp -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7e,0x02,0x89,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x09,0x13]
+v_cmpx_nge_f16_e64_dpp |v1.h|, -v2.h row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: v_cmpx_nge_f16_e64_dpp |v1.h|, -v2.h op_sel:[1,1] row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x19,0x89,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
 
-v_cmpx_nge_f16_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
-// GFX11: v_cmpx_nge_f16_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0x7e,0x83,0x89,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x05,0x30]
+v_cmpx_nge_f16_e64_dpp -v1.h, |v2.l| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: v_cmpx_nge_f16_e64_dpp -v1.h, |v2.l| op_sel:[1,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7e,0x0a,0x89,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x09,0x13]
+
+v_cmpx_nge_f16_e64_dpp -|v255.l|, -|v255.h| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: v_cmpx_nge_f16_e64_dpp -|v255.l|, -|v255.h| op_sel:[0,1] clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0x7e,0x93,0x89,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x05,0x30]
 
 v_cmpx_nge_f32_e64_dpp v1, v2 quad_perm:[3,2,1,0]
 // GFX11: v_cmpx_nge_f32_e64_dpp v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x99,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
@@ -2061,47 +2133,56 @@ v_cmpx_nge_f32_e64_dpp -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ct
 v_cmpx_nge_f32_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
 // GFX11: v_cmpx_nge_f32_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0x7e,0x83,0x99,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x05,0x30]
 
-v_cmpx_ngt_f16_e64_dpp v1, v2 quad_perm:[3,2,1,0]
-// GFX11: v_cmpx_ngt_f16_e64_dpp v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8b,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+v_cmpx_ngt_f16_e64_dpp v1.l, v2.l quad_perm:[3,2,1,0]
+// GFX11: v_cmpx_ngt_f16_e64_dpp v1.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8b,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+
+v_cmpx_ngt_f16_e64_dpp v1.l, v2.l quad_perm:[0,1,2,3]
+// GFX11: v_cmpx_ngt_f16_e64_dpp v1.l, v2.l quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8b,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+
+v_cmpx_ngt_f16_e64_dpp v1.l, v2.l row_mirror
+// GFX11: v_cmpx_ngt_f16_e64_dpp v1.l, v2.l row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8b,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
 
-v_cmpx_ngt_f16_e64_dpp v1, v2 quad_perm:[0,1,2,3]
-// GFX11: v_cmpx_ngt_f16_e64_dpp v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8b,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+v_cmpx_ngt_f16_e64_dpp v1.l, v2.l row_half_mirror
+// GFX11: v_cmpx_ngt_f16_e64_dpp v1.l, v2.l row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8b,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
 
-v_cmpx_ngt_f16_e64_dpp v1, v2 row_mirror
-// GFX11: v_cmpx_ngt_f16_e64_dpp v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8b,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+v_cmpx_ngt_f16_e64_dpp v1.l, v2.l row_shl:1
+// GFX11: v_cmpx_ngt_f16_e64_dpp v1.l, v2.l row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8b,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
 
-v_cmpx_ngt_f16_e64_dpp v1, v2 row_half_mirror
-// GFX11: v_cmpx_ngt_f16_e64_dpp v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8b,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+v_cmpx_ngt_f16_e64_dpp v1.l, v2.l row_shl:15
+// GFX11: v_cmpx_ngt_f16_e64_dpp v1.l, v2.l row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8b,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
 
-v_cmpx_ngt_f16_e64_dpp v1, v2 row_shl:1
-// GFX11: v_cmpx_ngt_f16_e64_dpp v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8b,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+v_cmpx_ngt_f16_e64_dpp v1.l, v2.l row_shr:1
+// GFX11: v_cmpx_ngt_f16_e64_dpp v1.l, v2.l row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8b,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
 
-v_cmpx_ngt_f16_e64_dpp v1, v2 row_shl:15
-// GFX11: v_cmpx_ngt_f16_e64_dpp v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8b,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+v_cmpx_ngt_f16_e64_dpp v1.l, v2.l row_shr:15
+// GFX11: v_cmpx_ngt_f16_e64_dpp v1.l, v2.l row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8b,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
 
-v_cmpx_ngt_f16_e64_dpp v1, v2 row_shr:1
-// GFX11: v_cmpx_ngt_f16_e64_dpp v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8b,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+v_cmpx_ngt_f16_e64_dpp v1.l, v2.l row_ror:1
+// GFX11: v_cmpx_ngt_f16_e64_dpp v1.l, v2.l row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8b,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
 
-v_cmpx_ngt_f16_e64_dpp v1, v2 row_shr:15
-// GFX11: v_cmpx_ngt_f16_e64_dpp v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8b,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+v_cmpx_ngt_f16_e64_dpp v1.l, v2.l row_ror:15
+// GFX11: v_cmpx_ngt_f16_e64_dpp v1.l, v2.l row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8b,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
 
-v_cmpx_ngt_f16_e64_dpp v1, v2 row_ror:1
-// GFX11: v_cmpx_ngt_f16_e64_dpp v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8b,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+v_cmpx_ngt_f16_e64_dpp v1.l, v2.l row_share:0 row_mask:0xf bank_mask:0xf
+// GFX11: v_cmpx_ngt_f16_e64_dpp v1.l, v2.l row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8b,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 
-v_cmpx_ngt_f16_e64_dpp v1, v2 row_ror:15
-// GFX11: v_cmpx_ngt_f16_e64_dpp v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8b,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+v_cmpx_ngt_f16_e64_dpp |v1.l|, -v2.l row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: v_cmpx_ngt_f16_e64_dpp |v1.l|, -v2.l row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x01,0x8b,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
 
-v_cmpx_ngt_f16_e64_dpp v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
-// GFX11: v_cmpx_ngt_f16_e64_dpp v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8b,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+v_cmpx_ngt_f16_e64_dpp -v1.l, |v2.l| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: v_cmpx_ngt_f16_e64_dpp -v1.l, |v2.l| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7e,0x02,0x8b,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x09,0x13]
 
-v_cmpx_ngt_f16_e64_dpp |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1
-// GFX11: v_cmpx_ngt_f16_e64_dpp |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x01,0x8b,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+v_cmpx_ngt_f16_e64_dpp -|v255.l|, -|v255.l| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: v_cmpx_ngt_f16_e64_dpp -|v255.l|, -|v255.l| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0x7e,0x83,0x8b,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x05,0x30]
 
-v_cmpx_ngt_f16_e64_dpp -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
-// GFX11: v_cmpx_ngt_f16_e64_dpp -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7e,0x02,0x8b,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x09,0x13]
+v_cmpx_ngt_f16_e64_dpp |v1.h|, -v2.h row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: v_cmpx_ngt_f16_e64_dpp |v1.h|, -v2.h op_sel:[1,1] row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x19,0x8b,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
 
-v_cmpx_ngt_f16_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
-// GFX11: v_cmpx_ngt_f16_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0x7e,0x83,0x8b,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x05,0x30]
+v_cmpx_ngt_f16_e64_dpp -v1.h, |v2.l| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: v_cmpx_ngt_f16_e64_dpp -v1.h, |v2.l| op_sel:[1,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7e,0x0a,0x8b,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x09,0x13]
+
+v_cmpx_ngt_f16_e64_dpp -|v255.l|, -|v255.h| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: v_cmpx_ngt_f16_e64_dpp -|v255.l|, -|v255.h| op_sel:[0,1] clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0x7e,0x93,0x8b,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x05,0x30]
 
 v_cmpx_ngt_f32_e64_dpp v1, v2 quad_perm:[3,2,1,0]
 // GFX11: v_cmpx_ngt_f32_e64_dpp v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x9b,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
@@ -2145,47 +2226,56 @@ v_cmpx_ngt_f32_e64_dpp -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ct
 v_cmpx_ngt_f32_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
 // GFX11: v_cmpx_ngt_f32_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0x7e,0x83,0x9b,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x05,0x30]
 
-v_cmpx_nle_f16_e64_dpp v1, v2 quad_perm:[3,2,1,0]
-// GFX11: v_cmpx_nle_f16_e64_dpp v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8c,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+v_cmpx_nle_f16_e64_dpp v1.l, v2.l quad_perm:[3,2,1,0]
+// GFX11: v_cmpx_nle_f16_e64_dpp v1.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8c,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+
+v_cmpx_nle_f16_e64_dpp v1.l, v2.l quad_perm:[0,1,2,3]
+// GFX11: v_cmpx_nle_f16_e64_dpp v1.l, v2.l quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8c,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+
+v_cmpx_nle_f16_e64_dpp v1.l, v2.l row_mirror
+// GFX11: v_cmpx_nle_f16_e64_dpp v1.l, v2.l row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8c,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
 
-v_cmpx_nle_f16_e64_dpp v1, v2 quad_perm:[0,1,2,3]
-// GFX11: v_cmpx_nle_f16_e64_dpp v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8c,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+v_cmpx_nle_f16_e64_dpp v1.l, v2.l row_half_mirror
+// GFX11: v_cmpx_nle_f16_e64_dpp v1.l, v2.l row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8c,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
 
-v_cmpx_nle_f16_e64_dpp v1, v2 row_mirror
-// GFX11: v_cmpx_nle_f16_e64_dpp v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8c,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+v_cmpx_nle_f16_e64_dpp v1.l, v2.l row_shl:1
+// GFX11: v_cmpx_nle_f16_e64_dpp v1.l, v2.l row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8c,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
 
-v_cmpx_nle_f16_e64_dpp v1, v2 row_half_mirror
-// GFX11: v_cmpx_nle_f16_e64_dpp v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8c,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+v_cmpx_nle_f16_e64_dpp v1.l, v2.l row_shl:15
+// GFX11: v_cmpx_nle_f16_e64_dpp v1.l, v2.l row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8c,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
 
-v_cmpx_nle_f16_e64_dpp v1, v2 row_shl:1
-// GFX11: v_cmpx_nle_f16_e64_dpp v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8c,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+v_cmpx_nle_f16_e64_dpp v1.l, v2.l row_shr:1
+// GFX11: v_cmpx_nle_f16_e64_dpp v1.l, v2.l row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8c,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
 
-v_cmpx_nle_f16_e64_dpp v1, v2 row_shl:15
-// GFX11: v_cmpx_nle_f16_e64_dpp v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8c,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+v_cmpx_nle_f16_e64_dpp v1.l, v2.l row_shr:15
+// GFX11: v_cmpx_nle_f16_e64_dpp v1.l, v2.l row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8c,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
 
-v_cmpx_nle_f16_e64_dpp v1, v2 row_shr:1
-// GFX11: v_cmpx_nle_f16_e64_dpp v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8c,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+v_cmpx_nle_f16_e64_dpp v1.l, v2.l row_ror:1
+// GFX11: v_cmpx_nle_f16_e64_dpp v1.l, v2.l row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8c,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
 
-v_cmpx_nle_f16_e64_dpp v1, v2 row_shr:15
-// GFX11: v_cmpx_nle_f16_e64_dpp v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8c,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+v_cmpx_nle_f16_e64_dpp v1.l, v2.l row_ror:15
+// GFX11: v_cmpx_nle_f16_e64_dpp v1.l, v2.l row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8c,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
 
-v_cmpx_nle_f16_e64_dpp v1, v2 row_ror:1
-// GFX11: v_cmpx_nle_f16_e64_dpp v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8c,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+v_cmpx_nle_f16_e64_dpp v1.l, v2.l row_share:0 row_mask:0xf bank_mask:0xf
+// GFX11: v_cmpx_nle_f16_e64_dpp v1.l, v2.l row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8c,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 
-v_cmpx_nle_f16_e64_dpp v1, v2 row_ror:15
-// GFX11: v_cmpx_nle_f16_e64_dpp v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8c,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+v_cmpx_nle_f16_e64_dpp |v1.l|, -v2.l row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: v_cmpx_nle_f16_e64_dpp |v1.l|, -v2.l row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x01,0x8c,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
 
-v_cmpx_nle_f16_e64_dpp v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
-// GFX11: v_cmpx_nle_f16_e64_dpp v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8c,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+v_cmpx_nle_f16_e64_dpp -v1.l, |v2.l| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: v_cmpx_nle_f16_e64_dpp -v1.l, |v2.l| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7e,0x02,0x8c,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x09,0x13]
 
-v_cmpx_nle_f16_e64_dpp |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1
-// GFX11: v_cmpx_nle_f16_e64_dpp |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x01,0x8c,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+v_cmpx_nle_f16_e64_dpp -|v255.l|, -|v255.l| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: v_cmpx_nle_f16_e64_dpp -|v255.l|, -|v255.l| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0x7e,0x83,0x8c,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x05,0x30]
 
-v_cmpx_nle_f16_e64_dpp -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
-// GFX11: v_cmpx_nle_f16_e64_dpp -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7e,0x02,0x8c,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x09,0x13]
+v_cmpx_nle_f16_e64_dpp |v1.h|, -v2.h row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: v_cmpx_nle_f16_e64_dpp |v1.h|, -v2.h op_sel:[1,1] row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x19,0x8c,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
 
-v_cmpx_nle_f16_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
-// GFX11: v_cmpx_nle_f16_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0x7e,0x83,0x8c,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x05,0x30]
+v_cmpx_nle_f16_e64_dpp -v1.h, |v2.l| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: v_cmpx_nle_f16_e64_dpp -v1.h, |v2.l| op_sel:[1,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7e,0x0a,0x8c,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x09,0x13]
+
+v_cmpx_nle_f16_e64_dpp -|v255.l|, -|v255.h| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: v_cmpx_nle_f16_e64_dpp -|v255.l|, -|v255.h| op_sel:[0,1] clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0x7e,0x93,0x8c,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x05,0x30]
 
 v_cmpx_nle_f32_e64_dpp v1, v2 quad_perm:[3,2,1,0]
 // GFX11: v_cmpx_nle_f32_e64_dpp v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x9c,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
@@ -2229,47 +2319,56 @@ v_cmpx_nle_f32_e64_dpp -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ct
 v_cmpx_nle_f32_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
 // GFX11: v_cmpx_nle_f32_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0x7e,0x83,0x9c,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x05,0x30]
 
-v_cmpx_nlg_f16_e64_dpp v1, v2 quad_perm:[3,2,1,0]
-// GFX11: v_cmpx_nlg_f16_e64_dpp v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8a,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+v_cmpx_nlg_f16_e64_dpp v1.l, v2.l quad_perm:[3,2,1,0]
+// GFX11: v_cmpx_nlg_f16_e64_dpp v1.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8a,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+
+v_cmpx_nlg_f16_e64_dpp v1.l, v2.l quad_perm:[0,1,2,3]
+// GFX11: v_cmpx_nlg_f16_e64_dpp v1.l, v2.l quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8a,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+
+v_cmpx_nlg_f16_e64_dpp v1.l, v2.l row_mirror
+// GFX11: v_cmpx_nlg_f16_e64_dpp v1.l, v2.l row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8a,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
 
-v_cmpx_nlg_f16_e64_dpp v1, v2 quad_perm:[0,1,2,3]
-// GFX11: v_cmpx_nlg_f16_e64_dpp v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8a,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+v_cmpx_nlg_f16_e64_dpp v1.l, v2.l row_half_mirror
+// GFX11: v_cmpx_nlg_f16_e64_dpp v1.l, v2.l row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8a,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
 
-v_cmpx_nlg_f16_e64_dpp v1, v2 row_mirror
-// GFX11: v_cmpx_nlg_f16_e64_dpp v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8a,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+v_cmpx_nlg_f16_e64_dpp v1.l, v2.l row_shl:1
+// GFX11: v_cmpx_nlg_f16_e64_dpp v1.l, v2.l row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8a,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
 
-v_cmpx_nlg_f16_e64_dpp v1, v2 row_half_mirror
-// GFX11: v_cmpx_nlg_f16_e64_dpp v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8a,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+v_cmpx_nlg_f16_e64_dpp v1.l, v2.l row_shl:15
+// GFX11: v_cmpx_nlg_f16_e64_dpp v1.l, v2.l row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8a,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
 
-v_cmpx_nlg_f16_e64_dpp v1, v2 row_shl:1
-// GFX11: v_cmpx_nlg_f16_e64_dpp v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8a,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+v_cmpx_nlg_f16_e64_dpp v1.l, v2.l row_shr:1
+// GFX11: v_cmpx_nlg_f16_e64_dpp v1.l, v2.l row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8a,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
 
-v_cmpx_nlg_f16_e64_dpp v1, v2 row_shl:15
-// GFX11: v_cmpx_nlg_f16_e64_dpp v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8a,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+v_cmpx_nlg_f16_e64_dpp v1.l, v2.l row_shr:15
+// GFX11: v_cmpx_nlg_f16_e64_dpp v1.l, v2.l row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8a,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
 
-v_cmpx_nlg_f16_e64_dpp v1, v2 row_shr:1
-// GFX11: v_cmpx_nlg_f16_e64_dpp v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8a,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+v_cmpx_nlg_f16_e64_dpp v1.l, v2.l row_ror:1
+// GFX11: v_cmpx_nlg_f16_e64_dpp v1.l, v2.l row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8a,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
 
-v_cmpx_nlg_f16_e64_dpp v1, v2 row_shr:15
-// GFX11: v_cmpx_nlg_f16_e64_dpp v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8a,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+v_cmpx_nlg_f16_e64_dpp v1.l, v2.l row_ror:15
+// GFX11: v_cmpx_nlg_f16_e64_dpp v1.l, v2.l row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8a,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
 
-v_cmpx_nlg_f16_e64_dpp v1, v2 row_ror:1
-// GFX11: v_cmpx_nlg_f16_e64_dpp v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8a,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+v_cmpx_nlg_f16_e64_dpp v1.l, v2.l row_share:0 row_mask:0xf bank_mask:0xf
+// GFX11: v_cmpx_nlg_f16_e64_dpp v1.l, v2.l row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8a,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 
-v_cmpx_nlg_f16_e64_dpp v1, v2 row_ror:15
-// GFX11: v_cmpx_nlg_f16_e64_dpp v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8a,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+v_cmpx_nlg_f16_e64_dpp |v1.l|, -v2.l row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: v_cmpx_nlg_f16_e64_dpp |v1.l|, -v2.l row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x01,0x8a,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
 
-v_cmpx_nlg_f16_e64_dpp v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
-// GFX11: v_cmpx_nlg_f16_e64_dpp v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8a,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+v_cmpx_nlg_f16_e64_dpp -v1.l, |v2.l| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: v_cmpx_nlg_f16_e64_dpp -v1.l, |v2.l| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7e,0x02,0x8a,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x09,0x13]
 
-v_cmpx_nlg_f16_e64_dpp |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1
-// GFX11: v_cmpx_nlg_f16_e64_dpp |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x01,0x8a,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+v_cmpx_nlg_f16_e64_dpp -|v255.l|, -|v255.l| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: v_cmpx_nlg_f16_e64_dpp -|v255.l|, -|v255.l| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0x7e,0x83,0x8a,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x05,0x30]
 
-v_cmpx_nlg_f16_e64_dpp -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
-// GFX11: v_cmpx_nlg_f16_e64_dpp -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7e,0x02,0x8a,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x09,0x13]
+v_cmpx_nlg_f16_e64_dpp |v1.h|, -v2.h row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: v_cmpx_nlg_f16_e64_dpp |v1.h|, -v2.h op_sel:[1,1] row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x19,0x8a,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
 
-v_cmpx_nlg_f16_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
-// GFX11: v_cmpx_nlg_f16_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0x7e,0x83,0x8a,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x05,0x30]
+v_cmpx_nlg_f16_e64_dpp -v1.h, |v2.l| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: v_cmpx_nlg_f16_e64_dpp -v1.h, |v2.l| op_sel:[1,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7e,0x0a,0x8a,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x09,0x13]
+
+v_cmpx_nlg_f16_e64_dpp -|v255.l|, -|v255.h| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: v_cmpx_nlg_f16_e64_dpp -|v255.l|, -|v255.h| op_sel:[0,1] clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0x7e,0x93,0x8a,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x05,0x30]
 
 v_cmpx_nlg_f32_e64_dpp v1, v2 quad_perm:[3,2,1,0]
 // GFX11: v_cmpx_nlg_f32_e64_dpp v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x9a,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
@@ -2313,47 +2412,56 @@ v_cmpx_nlg_f32_e64_dpp -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ct
 v_cmpx_nlg_f32_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
 // GFX11: v_cmpx_nlg_f32_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0x7e,0x83,0x9a,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x05,0x30]
 
-v_cmpx_nlt_f16_e64_dpp v1, v2 quad_perm:[3,2,1,0]
-// GFX11: v_cmpx_nlt_f16_e64_dpp v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8e,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+v_cmpx_nlt_f16_e64_dpp v1.l, v2.l quad_perm:[3,2,1,0]
+// GFX11: v_cmpx_nlt_f16_e64_dpp v1.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8e,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+
+v_cmpx_nlt_f16_e64_dpp v1.l, v2.l quad_perm:[0,1,2,3]
+// GFX11: v_cmpx_nlt_f16_e64_dpp v1.l, v2.l quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8e,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
 
-v_cmpx_nlt_f16_e64_dpp v1, v2 quad_perm:[0,1,2,3]
-// GFX11: v_cmpx_nlt_f16_e64_dpp v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8e,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+v_cmpx_nlt_f16_e64_dpp v1.l, v2.l row_mirror
+// GFX11: v_cmpx_nlt_f16_e64_dpp v1.l, v2.l row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8e,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
 
-v_cmpx_nlt_f16_e64_dpp v1, v2 row_mirror
-// GFX11: v_cmpx_nlt_f16_e64_dpp v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8e,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+v_cmpx_nlt_f16_e64_dpp v1.l, v2.l row_half_mirror
+// GFX11: v_cmpx_nlt_f16_e64_dpp v1.l, v2.l row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8e,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
 
-v_cmpx_nlt_f16_e64_dpp v1, v2 row_half_mirror
-// GFX11: v_cmpx_nlt_f16_e64_dpp v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8e,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+v_cmpx_nlt_f16_e64_dpp v1.l, v2.l row_shl:1
+// GFX11: v_cmpx_nlt_f16_e64_dpp v1.l, v2.l row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8e,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
 
-v_cmpx_nlt_f16_e64_dpp v1, v2 row_shl:1
-// GFX11: v_cmpx_nlt_f16_e64_dpp v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8e,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+v_cmpx_nlt_f16_e64_dpp v1.l, v2.l row_shl:15
+// GFX11: v_cmpx_nlt_f16_e64_dpp v1.l, v2.l row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8e,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
 
-v_cmpx_nlt_f16_e64_dpp v1, v2 row_shl:15
-// GFX11: v_cmpx_nlt_f16_e64_dpp v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8e,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+v_cmpx_nlt_f16_e64_dpp v1.l, v2.l row_shr:1
+// GFX11: v_cmpx_nlt_f16_e64_dpp v1.l, v2.l row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8e,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
 
-v_cmpx_nlt_f16_e64_dpp v1, v2 row_shr:1
-// GFX11: v_cmpx_nlt_f16_e64_dpp v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8e,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+v_cmpx_nlt_f16_e64_dpp v1.l, v2.l row_shr:15
+// GFX11: v_cmpx_nlt_f16_e64_dpp v1.l, v2.l row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8e,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
 
-v_cmpx_nlt_f16_e64_dpp v1, v2 row_shr:15
-// GFX11: v_cmpx_nlt_f16_e64_dpp v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8e,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+v_cmpx_nlt_f16_e64_dpp v1.l, v2.l row_ror:1
+// GFX11: v_cmpx_nlt_f16_e64_dpp v1.l, v2.l row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8e,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
 
-v_cmpx_nlt_f16_e64_dpp v1, v2 row_ror:1
-// GFX11: v_cmpx_nlt_f16_e64_dpp v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8e,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+v_cmpx_nlt_f16_e64_dpp v1.l, v2.l row_ror:15
+// GFX11: v_cmpx_nlt_f16_e64_dpp v1.l, v2.l row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8e,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
 
-v_cmpx_nlt_f16_e64_dpp v1, v2 row_ror:15
-// GFX11: v_cmpx_nlt_f16_e64_dpp v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8e,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+v_cmpx_nlt_f16_e64_dpp v1.l, v2.l row_share:0 row_mask:0xf bank_mask:0xf
+// GFX11: v_cmpx_nlt_f16_e64_dpp v1.l, v2.l row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8e,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 
-v_cmpx_nlt_f16_e64_dpp v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
-// GFX11: v_cmpx_nlt_f16_e64_dpp v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8e,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+v_cmpx_nlt_f16_e64_dpp |v1.l|, -v2.l row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: v_cmpx_nlt_f16_e64_dpp |v1.l|, -v2.l row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x01,0x8e,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
 
-v_cmpx_nlt_f16_e64_dpp |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1
-// GFX11: v_cmpx_nlt_f16_e64_dpp |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x01,0x8e,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+v_cmpx_nlt_f16_e64_dpp -v1.l, |v2.l| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: v_cmpx_nlt_f16_e64_dpp -v1.l, |v2.l| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7e,0x02,0x8e,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x09,0x13]
 
-v_cmpx_nlt_f16_e64_dpp -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
-// GFX11: v_cmpx_nlt_f16_e64_dpp -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7e,0x02,0x8e,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x09,0x13]
+v_cmpx_nlt_f16_e64_dpp -|v255.l|, -|v255.l| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: v_cmpx_nlt_f16_e64_dpp -|v255.l|, -|v255.l| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0x7e,0x83,0x8e,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x05,0x30]
 
-v_cmpx_nlt_f16_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
-// GFX11: v_cmpx_nlt_f16_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0x7e,0x83,0x8e,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x05,0x30]
+v_cmpx_nlt_f16_e64_dpp |v1.h|, -v2.h row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: v_cmpx_nlt_f16_e64_dpp |v1.h|, -v2.h op_sel:[1,1] row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x19,0x8e,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+
+v_cmpx_nlt_f16_e64_dpp -v1.h, |v2.l| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: v_cmpx_nlt_f16_e64_dpp -v1.h, |v2.l| op_sel:[1,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7e,0x0a,0x8e,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x09,0x13]
+
+v_cmpx_nlt_f16_e64_dpp -|v255.l|, -|v255.h| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: v_cmpx_nlt_f16_e64_dpp -|v255.l|, -|v255.h| op_sel:[0,1] clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0x7e,0x93,0x8e,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x05,0x30]
 
 v_cmpx_nlt_f32_e64_dpp v1, v2 quad_perm:[3,2,1,0]
 // GFX11: v_cmpx_nlt_f32_e64_dpp v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x9e,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
@@ -2397,47 +2505,56 @@ v_cmpx_nlt_f32_e64_dpp -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ct
 v_cmpx_nlt_f32_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
 // GFX11: v_cmpx_nlt_f32_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0x7e,0x83,0x9e,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x05,0x30]
 
-v_cmpx_o_f16_e64_dpp v1, v2 quad_perm:[3,2,1,0]
-// GFX11: v_cmpx_o_f16_e64_dpp v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x87,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+v_cmpx_o_f16_e64_dpp v1.l, v2.l quad_perm:[3,2,1,0]
+// GFX11: v_cmpx_o_f16_e64_dpp v1.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x87,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+
+v_cmpx_o_f16_e64_dpp v1.l, v2.l quad_perm:[0,1,2,3]
+// GFX11: v_cmpx_o_f16_e64_dpp v1.l, v2.l quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x87,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
 
-v_cmpx_o_f16_e64_dpp v1, v2 quad_perm:[0,1,2,3]
-// GFX11: v_cmpx_o_f16_e64_dpp v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x87,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+v_cmpx_o_f16_e64_dpp v1.l, v2.l row_mirror
+// GFX11: v_cmpx_o_f16_e64_dpp v1.l, v2.l row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x87,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
 
-v_cmpx_o_f16_e64_dpp v1, v2 row_mirror
-// GFX11: v_cmpx_o_f16_e64_dpp v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x87,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+v_cmpx_o_f16_e64_dpp v1.l, v2.l row_half_mirror
+// GFX11: v_cmpx_o_f16_e64_dpp v1.l, v2.l row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x87,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
 
-v_cmpx_o_f16_e64_dpp v1, v2 row_half_mirror
-// GFX11: v_cmpx_o_f16_e64_dpp v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x87,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+v_cmpx_o_f16_e64_dpp v1.l, v2.l row_shl:1
+// GFX11: v_cmpx_o_f16_e64_dpp v1.l, v2.l row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x87,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
 
-v_cmpx_o_f16_e64_dpp v1, v2 row_shl:1
-// GFX11: v_cmpx_o_f16_e64_dpp v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x87,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+v_cmpx_o_f16_e64_dpp v1.l, v2.l row_shl:15
+// GFX11: v_cmpx_o_f16_e64_dpp v1.l, v2.l row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x87,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
 
-v_cmpx_o_f16_e64_dpp v1, v2 row_shl:15
-// GFX11: v_cmpx_o_f16_e64_dpp v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x87,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+v_cmpx_o_f16_e64_dpp v1.l, v2.l row_shr:1
+// GFX11: v_cmpx_o_f16_e64_dpp v1.l, v2.l row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x87,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
 
-v_cmpx_o_f16_e64_dpp v1, v2 row_shr:1
-// GFX11: v_cmpx_o_f16_e64_dpp v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x87,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+v_cmpx_o_f16_e64_dpp v1.l, v2.l row_shr:15
+// GFX11: v_cmpx_o_f16_e64_dpp v1.l, v2.l row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x87,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
 
-v_cmpx_o_f16_e64_dpp v1, v2 row_shr:15
-// GFX11: v_cmpx_o_f16_e64_dpp v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x87,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+v_cmpx_o_f16_e64_dpp v1.l, v2.l row_ror:1
+// GFX11: v_cmpx_o_f16_e64_dpp v1.l, v2.l row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x87,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
 
-v_cmpx_o_f16_e64_dpp v1, v2 row_ror:1
-// GFX11: v_cmpx_o_f16_e64_dpp v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x87,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+v_cmpx_o_f16_e64_dpp v1.l, v2.l row_ror:15
+// GFX11: v_cmpx_o_f16_e64_dpp v1.l, v2.l row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x87,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
 
-v_cmpx_o_f16_e64_dpp v1, v2 row_ror:15
-// GFX11: v_cmpx_o_f16_e64_dpp v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x87,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+v_cmpx_o_f16_e64_dpp v1.l, v2.l row_share:0 row_mask:0xf bank_mask:0xf
+// GFX11: v_cmpx_o_f16_e64_dpp v1.l, v2.l row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x87,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 
-v_cmpx_o_f16_e64_dpp v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
-// GFX11: v_cmpx_o_f16_e64_dpp v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x87,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+v_cmpx_o_f16_e64_dpp |v1.l|, -v2.l row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: v_cmpx_o_f16_e64_dpp |v1.l|, -v2.l row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x01,0x87,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
 
-v_cmpx_o_f16_e64_dpp |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1
-// GFX11: v_cmpx_o_f16_e64_dpp |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x01,0x87,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+v_cmpx_o_f16_e64_dpp -v1.l, |v2.l| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: v_cmpx_o_f16_e64_dpp -v1.l, |v2.l| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7e,0x02,0x87,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x09,0x13]
 
-v_cmpx_o_f16_e64_dpp -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
-// GFX11: v_cmpx_o_f16_e64_dpp -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7e,0x02,0x87,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x09,0x13]
+v_cmpx_o_f16_e64_dpp -|v255.l|, -|v255.l| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: v_cmpx_o_f16_e64_dpp -|v255.l|, -|v255.l| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0x7e,0x83,0x87,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x05,0x30]
 
-v_cmpx_o_f16_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
-// GFX11: v_cmpx_o_f16_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0x7e,0x83,0x87,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x05,0x30]
+v_cmpx_o_f16_e64_dpp |v1.h|, -v2.h row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: v_cmpx_o_f16_e64_dpp |v1.h|, -v2.h op_sel:[1,1] row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x19,0x87,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+
+v_cmpx_o_f16_e64_dpp -v1.h, |v2.l| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: v_cmpx_o_f16_e64_dpp -v1.h, |v2.l| op_sel:[1,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7e,0x0a,0x87,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x09,0x13]
+
+v_cmpx_o_f16_e64_dpp -|v255.l|, -|v255.h| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: v_cmpx_o_f16_e64_dpp -|v255.l|, -|v255.h| op_sel:[0,1] clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0x7e,0x93,0x87,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x05,0x30]
 
 v_cmpx_o_f32_e64_dpp v1, v2 quad_perm:[3,2,1,0]
 // GFX11: v_cmpx_o_f32_e64_dpp v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x97,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
@@ -2481,47 +2598,47 @@ v_cmpx_o_f32_e64_dpp -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl
 v_cmpx_o_f32_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
 // GFX11: v_cmpx_o_f32_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0x7e,0x83,0x97,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x05,0x30]
 
-v_cmpx_t_f16_e64_dpp v1, v2 quad_perm:[3,2,1,0]
-// GFX11: v_cmpx_t_f16_e64_dpp v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8f,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+v_cmpx_t_f16_e64_dpp v1.l, v2.l quad_perm:[3,2,1,0]
+// GFX11: v_cmpx_t_f16_e64_dpp v1.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8f,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
 
-v_cmpx_t_f16_e64_dpp v1, v2 quad_perm:[0,1,2,3]
-// GFX11: v_cmpx_t_f16_e64_dpp v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8f,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+v_cmpx_t_f16_e64_dpp v1.l, v2.l quad_perm:[0,1,2,3]
+// GFX11: v_cmpx_t_f16_e64_dpp v1.l, v2.l quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8f,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
 
-v_cmpx_t_f16_e64_dpp v1, v2 row_mirror
-// GFX11: v_cmpx_t_f16_e64_dpp v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8f,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+v_cmpx_t_f16_e64_dpp v1.l, v2.l row_mirror
+// GFX11: v_cmpx_t_f16_e64_dpp v1.l, v2.l row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8f,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
 
-v_cmpx_t_f16_e64_dpp v1, v2 row_half_mirror
-// GFX11: v_cmpx_t_f16_e64_dpp v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8f,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+v_cmpx_t_f16_e64_dpp v1.l, v2.l row_half_mirror
+// GFX11: v_cmpx_t_f16_e64_dpp v1.l, v2.l row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8f,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
 
-v_cmpx_t_f16_e64_dpp v1, v2 row_shl:1
-// GFX11: v_cmpx_t_f16_e64_dpp v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8f,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+v_cmpx_t_f16_e64_dpp v1.l, v2.l row_shl:1
+// GFX11: v_cmpx_t_f16_e64_dpp v1.l, v2.l row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8f,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
 
-v_cmpx_t_f16_e64_dpp v1, v2 row_shl:15
-// GFX11: v_cmpx_t_f16_e64_dpp v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8f,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+v_cmpx_t_f16_e64_dpp v1.l, v2.l row_shl:15
+// GFX11: v_cmpx_t_f16_e64_dpp v1.l, v2.l row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8f,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
 
-v_cmpx_t_f16_e64_dpp v1, v2 row_shr:1
-// GFX11: v_cmpx_t_f16_e64_dpp v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8f,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+v_cmpx_t_f16_e64_dpp v1.l, v2.l row_shr:1
+// GFX11: v_cmpx_t_f16_e64_dpp v1.l, v2.l row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8f,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
 
-v_cmpx_t_f16_e64_dpp v1, v2 row_shr:15
-// GFX11: v_cmpx_t_f16_e64_dpp v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8f,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+v_cmpx_t_f16_e64_dpp v1.l, v2.l row_shr:15
+// GFX11: v_cmpx_t_f16_e64_dpp v1.l, v2.l row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8f,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
 
-v_cmpx_t_f16_e64_dpp v1, v2 row_ror:1
-// GFX11: v_cmpx_t_f16_e64_dpp v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8f,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+v_cmpx_t_f16_e64_dpp v1.l, v2.l row_ror:1
+// GFX11: v_cmpx_t_f16_e64_dpp v1.l, v2.l row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8f,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
 
-v_cmpx_t_f16_e64_dpp v1, v2 row_ror:15
-// GFX11: v_cmpx_t_f16_e64_dpp v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8f,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+v_cmpx_t_f16_e64_dpp v1.l, v2.l row_ror:15
+// GFX11: v_cmpx_t_f16_e64_dpp v1.l, v2.l row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8f,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
 
-v_cmpx_t_f16_e64_dpp v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
-// GFX11: v_cmpx_t_f16_e64_dpp v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8f,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+v_cmpx_t_f16_e64_dpp v1.l, v2.l row_share:0 row_mask:0xf bank_mask:0xf
+// GFX11: v_cmpx_t_f16_e64_dpp v1.l, v2.l row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8f,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 
-v_cmpx_t_f16_e64_dpp |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1
-// GFX11: v_cmpx_t_f16_e64_dpp |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x01,0x8f,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+v_cmpx_t_f16_e64_dpp |v1.l|, -v2.l row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: v_cmpx_t_f16_e64_dpp |v1.l|, -v2.l row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x01,0x8f,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
 
-v_cmpx_t_f16_e64_dpp -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
-// GFX11: v_cmpx_t_f16_e64_dpp -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7e,0x02,0x8f,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x09,0x13]
+v_cmpx_t_f16_e64_dpp -v1.l, |v2.l| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: v_cmpx_t_f16_e64_dpp -v1.l, |v2.l| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7e,0x02,0x8f,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x09,0x13]
 
-v_cmpx_t_f16_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
-// GFX11: v_cmpx_t_f16_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0x7e,0x83,0x8f,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x05,0x30]
+v_cmpx_t_f16_e64_dpp -|v255.l|, -|v255.l| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: v_cmpx_t_f16_e64_dpp -|v255.l|, -|v255.l| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0x7e,0x83,0x8f,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x05,0x30]
 
 v_cmpx_t_f32_e64_dpp v1, v2 quad_perm:[3,2,1,0]
 // GFX11: v_cmpx_t_f32_e64_dpp v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x9f,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
@@ -2649,47 +2766,56 @@ v_cmpx_t_u32_e64_dpp v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1
 v_cmpx_t_u32_e64_dpp v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
 // GFX11: v_cmpx_t_u32_e64_dpp v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0x7e,0x00,0xcf,0xd4,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x05,0x30]
 
-v_cmpx_tru_f16_e64_dpp v1, v2 quad_perm:[3,2,1,0]
-// GFX11: v_cmpx_t_f16_e64_dpp v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8f,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+v_cmpx_tru_f16_e64_dpp v1.l, v2.l quad_perm:[3,2,1,0]
+// GFX11: v_cmpx_t_f16_e64_dpp v1.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8f,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+
+v_cmpx_tru_f16_e64_dpp v1.l, v2.l quad_perm:[0,1,2,3]
+// GFX11: v_cmpx_t_f16_e64_dpp v1.l, v2.l quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8f,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
 
-v_cmpx_tru_f16_e64_dpp v1, v2 quad_perm:[0,1,2,3]
-// GFX11: v_cmpx_t_f16_e64_dpp v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8f,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+v_cmpx_tru_f16_e64_dpp v1.l, v2.l row_mirror
+// GFX11: v_cmpx_t_f16_e64_dpp v1.l, v2.l row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8f,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
 
-v_cmpx_tru_f16_e64_dpp v1, v2 row_mirror
-// GFX11: v_cmpx_t_f16_e64_dpp v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8f,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+v_cmpx_tru_f16_e64_dpp v1.l, v2.l row_half_mirror
+// GFX11: v_cmpx_t_f16_e64_dpp v1.l, v2.l row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8f,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
 
-v_cmpx_tru_f16_e64_dpp v1, v2 row_half_mirror
-// GFX11: v_cmpx_t_f16_e64_dpp v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8f,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+v_cmpx_tru_f16_e64_dpp v1.l, v2.l row_shl:1
+// GFX11: v_cmpx_t_f16_e64_dpp v1.l, v2.l row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8f,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
 
-v_cmpx_tru_f16_e64_dpp v1, v2 row_shl:1
-// GFX11: v_cmpx_t_f16_e64_dpp v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8f,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+v_cmpx_tru_f16_e64_dpp v1.l, v2.l row_shl:15
+// GFX11: v_cmpx_t_f16_e64_dpp v1.l, v2.l row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8f,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
 
-v_cmpx_tru_f16_e64_dpp v1, v2 row_shl:15
-// GFX11: v_cmpx_t_f16_e64_dpp v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8f,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+v_cmpx_tru_f16_e64_dpp v1.l, v2.l row_shr:1
+// GFX11: v_cmpx_t_f16_e64_dpp v1.l, v2.l row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8f,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
 
-v_cmpx_tru_f16_e64_dpp v1, v2 row_shr:1
-// GFX11: v_cmpx_t_f16_e64_dpp v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8f,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+v_cmpx_tru_f16_e64_dpp v1.l, v2.l row_shr:15
+// GFX11: v_cmpx_t_f16_e64_dpp v1.l, v2.l row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8f,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
 
-v_cmpx_tru_f16_e64_dpp v1, v2 row_shr:15
-// GFX11: v_cmpx_t_f16_e64_dpp v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8f,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+v_cmpx_tru_f16_e64_dpp v1.l, v2.l row_ror:1
+// GFX11: v_cmpx_t_f16_e64_dpp v1.l, v2.l row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8f,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
 
-v_cmpx_tru_f16_e64_dpp v1, v2 row_ror:1
-// GFX11: v_cmpx_t_f16_e64_dpp v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8f,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+v_cmpx_tru_f16_e64_dpp v1.l, v2.l row_ror:15
+// GFX11: v_cmpx_t_f16_e64_dpp v1.l, v2.l row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8f,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
 
-v_cmpx_tru_f16_e64_dpp v1, v2 row_ror:15
-// GFX11: v_cmpx_t_f16_e64_dpp v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8f,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+v_cmpx_tru_f16_e64_dpp v1.l, v2.l row_share:0 row_mask:0xf bank_mask:0xf
+// GFX11: v_cmpx_t_f16_e64_dpp v1.l, v2.l row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8f,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 
-v_cmpx_tru_f16_e64_dpp v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
-// GFX11: v_cmpx_t_f16_e64_dpp v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8f,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+v_cmpx_tru_f16_e64_dpp |v1.l|, -v2.l row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: v_cmpx_t_f16_e64_dpp |v1.l|, -v2.l row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x01,0x8f,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
 
-v_cmpx_tru_f16_e64_dpp |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1
-// GFX11: v_cmpx_t_f16_e64_dpp |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x01,0x8f,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+v_cmpx_tru_f16_e64_dpp -v1.l, |v2.l| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: v_cmpx_t_f16_e64_dpp -v1.l, |v2.l| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7e,0x02,0x8f,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x09,0x13]
 
-v_cmpx_tru_f16_e64_dpp -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
-// GFX11: v_cmpx_t_f16_e64_dpp -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7e,0x02,0x8f,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x09,0x13]
+v_cmpx_tru_f16_e64_dpp -|v255.l|, -|v255.l| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: v_cmpx_t_f16_e64_dpp -|v255.l|, -|v255.l| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0x7e,0x83,0x8f,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x05,0x30]
 
-v_cmpx_tru_f16_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
-// GFX11: v_cmpx_t_f16_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0x7e,0x83,0x8f,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x05,0x30]
+v_cmpx_t_f16_e64_dpp |v1.h|, -v2.h row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: v_cmpx_t_f16_e64_dpp |v1.h|, -v2.h op_sel:[1,1] row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x19,0x8f,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+
+v_cmpx_t_f16_e64_dpp -v1.h, |v2.l| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: v_cmpx_t_f16_e64_dpp -v1.h, |v2.l| op_sel:[1,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7e,0x0a,0x8f,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x09,0x13]
+
+v_cmpx_t_f16_e64_dpp -|v255.l|, -|v255.h| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: v_cmpx_t_f16_e64_dpp -|v255.l|, -|v255.h| op_sel:[0,1] clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0x7e,0x93,0x8f,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x05,0x30]
 
 v_cmpx_tru_f32_e64_dpp v1, v2 quad_perm:[3,2,1,0]
 // GFX11: v_cmpx_t_f32_e64_dpp v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x9f,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
@@ -2733,47 +2859,56 @@ v_cmpx_tru_f32_e64_dpp -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ct
 v_cmpx_tru_f32_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
 // GFX11: v_cmpx_t_f32_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0x7e,0x83,0x9f,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x05,0x30]
 
-v_cmpx_u_f16_e64_dpp v1, v2 quad_perm:[3,2,1,0]
-// GFX11: v_cmpx_u_f16_e64_dpp v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x88,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+v_cmpx_u_f16_e64_dpp v1.l, v2.l quad_perm:[3,2,1,0]
+// GFX11: v_cmpx_u_f16_e64_dpp v1.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x88,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+
+v_cmpx_u_f16_e64_dpp v1.l, v2.l quad_perm:[0,1,2,3]
+// GFX11: v_cmpx_u_f16_e64_dpp v1.l, v2.l quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x88,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+
+v_cmpx_u_f16_e64_dpp v1.l, v2.l row_mirror
+// GFX11: v_cmpx_u_f16_e64_dpp v1.l, v2.l row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x88,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+
+v_cmpx_u_f16_e64_dpp v1.l, v2.l row_half_mirror
+// GFX11: v_cmpx_u_f16_e64_dpp v1.l, v2.l row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x88,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
 
-v_cmpx_u_f16_e64_dpp v1, v2 quad_perm:[0,1,2,3]
-// GFX11: v_cmpx_u_f16_e64_dpp v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x88,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+v_cmpx_u_f16_e64_dpp v1.l, v2.l row_shl:1
+// GFX11: v_cmpx_u_f16_e64_dpp v1.l, v2.l row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x88,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
 
-v_cmpx_u_f16_e64_dpp v1, v2 row_mirror
-// GFX11: v_cmpx_u_f16_e64_dpp v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x88,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+v_cmpx_u_f16_e64_dpp v1.l, v2.l row_shl:15
+// GFX11: v_cmpx_u_f16_e64_dpp v1.l, v2.l row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x88,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
 
-v_cmpx_u_f16_e64_dpp v1, v2 row_half_mirror
-// GFX11: v_cmpx_u_f16_e64_dpp v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x88,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+v_cmpx_u_f16_e64_dpp v1.l, v2.l row_shr:1
+// GFX11: v_cmpx_u_f16_e64_dpp v1.l, v2.l row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x88,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
 
-v_cmpx_u_f16_e64_dpp v1, v2 row_shl:1
-// GFX11: v_cmpx_u_f16_e64_dpp v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x88,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+v_cmpx_u_f16_e64_dpp v1.l, v2.l row_shr:15
+// GFX11: v_cmpx_u_f16_e64_dpp v1.l, v2.l row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x88,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
 
-v_cmpx_u_f16_e64_dpp v1, v2 row_shl:15
-// GFX11: v_cmpx_u_f16_e64_dpp v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x88,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+v_cmpx_u_f16_e64_dpp v1.l, v2.l row_ror:1
+// GFX11: v_cmpx_u_f16_e64_dpp v1.l, v2.l row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x88,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
 
-v_cmpx_u_f16_e64_dpp v1, v2 row_shr:1
-// GFX11: v_cmpx_u_f16_e64_dpp v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x88,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+v_cmpx_u_f16_e64_dpp v1.l, v2.l row_ror:15
+// GFX11: v_cmpx_u_f16_e64_dpp v1.l, v2.l row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x88,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
 
-v_cmpx_u_f16_e64_dpp v1, v2 row_shr:15
-// GFX11: v_cmpx_u_f16_e64_dpp v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x88,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+v_cmpx_u_f16_e64_dpp v1.l, v2.l row_share:0 row_mask:0xf bank_mask:0xf
+// GFX11: v_cmpx_u_f16_e64_dpp v1.l, v2.l row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x88,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 
-v_cmpx_u_f16_e64_dpp v1, v2 row_ror:1
-// GFX11: v_cmpx_u_f16_e64_dpp v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x88,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+v_cmpx_u_f16_e64_dpp |v1.l|, -v2.l row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: v_cmpx_u_f16_e64_dpp |v1.l|, -v2.l row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x01,0x88,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
 
-v_cmpx_u_f16_e64_dpp v1, v2 row_ror:15
-// GFX11: v_cmpx_u_f16_e64_dpp v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x88,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+v_cmpx_u_f16_e64_dpp -v1.l, |v2.l| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: v_cmpx_u_f16_e64_dpp -v1.l, |v2.l| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7e,0x02,0x88,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x09,0x13]
 
-v_cmpx_u_f16_e64_dpp v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
-// GFX11: v_cmpx_u_f16_e64_dpp v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x88,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+v_cmpx_u_f16_e64_dpp -|v255.l|, -|v255.l| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: v_cmpx_u_f16_e64_dpp -|v255.l|, -|v255.l| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0x7e,0x83,0x88,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x05,0x30]
 
-v_cmpx_u_f16_e64_dpp |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1
-// GFX11: v_cmpx_u_f16_e64_dpp |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x01,0x88,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+v_cmpx_u_f16_e64_dpp |v1.h|, -v2.h row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: v_cmpx_u_f16_e64_dpp |v1.h|, -v2.h op_sel:[1,1] row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x19,0x88,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
 
-v_cmpx_u_f16_e64_dpp -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
-// GFX11: v_cmpx_u_f16_e64_dpp -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7e,0x02,0x88,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x09,0x13]
+v_cmpx_u_f16_e64_dpp -v1.h, |v2.l| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: v_cmpx_u_f16_e64_dpp -v1.h, |v2.l| op_sel:[1,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7e,0x0a,0x88,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x09,0x13]
 
-v_cmpx_u_f16_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
-// GFX11: v_cmpx_u_f16_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0x7e,0x83,0x88,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x05,0x30]
+v_cmpx_u_f16_e64_dpp -|v255.l|, -|v255.h| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: v_cmpx_u_f16_e64_dpp -|v255.l|, -|v255.h| op_sel:[0,1] clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0x7e,0x93,0x88,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x05,0x30]
 
 v_cmpx_u_f32_e64_dpp v1, v2 quad_perm:[3,2,1,0]
 // GFX11: v_cmpx_u_f32_e64_dpp v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x98,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
diff --git a/llvm/test/MC/AMDGPU/gfx11_asm_vop3_dpp8.s b/llvm/test/MC/AMDGPU/gfx11_asm_vop3_dpp8.s
index 2f9b5efca9e17..300e5ef22f5ae 100644
--- a/llvm/test/MC/AMDGPU/gfx11_asm_vop3_dpp8.s
+++ b/llvm/test/MC/AMDGPU/gfx11_asm_vop3_dpp8.s
@@ -187,11 +187,11 @@ v_alignbit_b32_e64_dpp v5, v1, v2, -1 dpp8:[7,6,5,4,3,2,1,0] fi:1
 v_alignbit_b32_e64_dpp v255, v255, v255, src_scc dpp8:[0,0,0,0,0,0,0,0] fi:0
 // GFX11: v_alignbit_b32_e64_dpp v255, v255, v255, src_scc dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x00,0x16,0xd6,0xe9,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00]
 
-v_alignbyte_b32_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_alignbyte_b32_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x17,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
+v_alignbyte_b32_e64_dpp v5, v1, v2, v3.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_alignbyte_b32_e64_dpp v5, v1, v2, v3.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x17,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 
-v_alignbyte_b32_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_alignbyte_b32_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x17,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
+v_alignbyte_b32_e64_dpp v5, v1, v2, v255.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_alignbyte_b32_e64_dpp v5, v1, v2, v255.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x17,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 
 v_alignbyte_b32_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0]
 // GFX11: v_alignbyte_b32_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x17,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
@@ -220,6 +220,15 @@ v_alignbyte_b32_e64_dpp v5, v1, v2, -1 dpp8:[7,6,5,4,3,2,1,0] fi:1
 v_alignbyte_b32_e64_dpp v255, v255, v255, src_scc dpp8:[0,0,0,0,0,0,0,0] fi:0
 // GFX11: v_alignbyte_b32_e64_dpp v255, v255, v255, src_scc dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x00,0x17,0xd6,0xe9,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00]
 
+v_alignbyte_b32_e64_dpp v5, v1, v2, s3 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_alignbyte_b32_e64_dpp v5, v1, v2, s3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x17,0xd6,0xe9,0x04,0x0e,0x00,0x01,0x77,0x39,0x05]
+
+v_alignbyte_b32_e64_dpp v5, v1, v2, m0 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_alignbyte_b32_e64_dpp v5, v1, v2, m0 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x17,0xd6,0xe9,0x04,0xf6,0x01,0x01,0x77,0x39,0x05]
+
+v_alignbyte_b32_e64_dpp v5, v1, v2, v255.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_alignbyte_b32_e64_dpp v5, v1, v2, v255.h op_sel:[0,0,1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x20,0x17,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
+
 v_and_b16_e64_dpp v5.l, v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0]
 // GFX11: v_and_b16_e64_dpp v5.l, v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x62,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 
@@ -1162,41 +1171,53 @@ v_mad_i16_e64_dpp v5.l, v1.l, v2.h, -1 dpp8:[7,6,5,4,3,2,1,0] fi:1
 v_mad_i16_e64_dpp v255.h, v255.l, v255.l, src_scc clamp dpp8:[0,0,0,0,0,0,0,0] fi:0
 // GFX11: v_mad_i16_e64_dpp v255.h, v255.l, v255.l, src_scc op_sel:[0,0,0,1] clamp dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0xc0,0x53,0xd6,0xe9,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00]
 
-v_mad_i32_i16_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_mad_i32_i16_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x5a,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
+v_mad_i32_i16_e64_dpp v5, v1.l, v2.l, v3 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_mad_i32_i16_e64_dpp v5, v1.l, v2.l, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x5a,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
+
+v_mad_i32_i16_e64_dpp v5, v1.l, v2.l, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_mad_i32_i16_e64_dpp v5, v1.l, v2.l, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x5a,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
+
+v_mad_i32_i16_e64_dpp v5, v1.l, v2.l, s105 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_mad_i32_i16_e64_dpp v5, v1.l, v2.l, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x5a,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
+
+v_mad_i32_i16_e64_dpp v5, v1.l, v2.l, vcc_hi dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_mad_i32_i16_e64_dpp v5, v1.l, v2.l, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x5a,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 
-v_mad_i32_i16_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_mad_i32_i16_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x5a,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
+v_mad_i32_i16_e64_dpp v5, v1.l, v2.l, vcc_lo dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_mad_i32_i16_e64_dpp v5, v1.l, v2.l, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x5a,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 
-v_mad_i32_i16_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_mad_i32_i16_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x5a,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
+v_mad_i32_i16_e64_dpp v5, v1.l, v2.l, ttmp15 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_mad_i32_i16_e64_dpp v5, v1.l, v2.l, ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x5a,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05]
 
-v_mad_i32_i16_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_mad_i32_i16_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x5a,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
+v_mad_i32_i16_e64_dpp v5, v1.l, v2.l, exec_hi dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_mad_i32_i16_e64_dpp v5, v1.l, v2.l, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x5a,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05]
 
-v_mad_i32_i16_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_mad_i32_i16_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x5a,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
+v_mad_i32_i16_e64_dpp v5, v1.l, v2.l, exec_lo dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_mad_i32_i16_e64_dpp v5, v1.l, v2.l, exec_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x5a,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05]
 
-v_mad_i32_i16_e64_dpp v5, v1, v2, ttmp15 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_mad_i32_i16_e64_dpp v5, v1, v2, ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x5a,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05]
+v_mad_i32_i16_e64_dpp v5, v1.l, v2.l, null dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_mad_i32_i16_e64_dpp v5, v1.l, v2.l, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x5a,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05]
 
-v_mad_i32_i16_e64_dpp v5, v1, v2, exec_hi dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_mad_i32_i16_e64_dpp v5, v1, v2, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x5a,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05]
+v_mad_i32_i16_e64_dpp v5, v1.l, v2.l, -1 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_mad_i32_i16_e64_dpp v5, v1.l, v2.l, -1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x5a,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05]
 
-v_mad_i32_i16_e64_dpp v5, v1, v2, exec_lo dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_mad_i32_i16_e64_dpp v5, v1, v2, exec_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x5a,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05]
+v_mad_i32_i16_e64_dpp v5, v1.l, v2.l, 0.5 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: v_mad_i32_i16_e64_dpp v5, v1.l, v2.l, 0.5 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x05,0x00,0x5a,0xd6,0xea,0x04,0xc2,0x03,0x01,0x77,0x39,0x05]
 
-v_mad_i32_i16_e64_dpp v5, v1, v2, null dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_mad_i32_i16_e64_dpp v5, v1, v2, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x5a,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05]
+v_mad_i32_i16_e64_dpp v255, v255.l, v255.l, src_scc clamp dpp8:[0,0,0,0,0,0,0,0]
+// GFX11: v_mad_i32_i16_e64_dpp v255, v255.l, v255.l, src_scc clamp dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x80,0x5a,0xd6,0xe9,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00]
 
-v_mad_i32_i16_e64_dpp v5, v1, v2, -1 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_mad_i32_i16_e64_dpp v5, v1, v2, -1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x5a,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05]
+v_mad_i32_i16_e64_dpp v5, v1.l, v2.l, s3 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_mad_i32_i16_e64_dpp v5, v1.l, v2.l, s3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x5a,0xd6,0xe9,0x04,0x0e,0x00,0x01,0x77,0x39,0x05]
 
-v_mad_i32_i16_e64_dpp v5, v1, v2, 0.5 dpp8:[7,6,5,4,3,2,1,0] fi:1
-// GFX11: v_mad_i32_i16_e64_dpp v5, v1, v2, 0.5 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x05,0x00,0x5a,0xd6,0xea,0x04,0xc2,0x03,0x01,0x77,0x39,0x05]
+v_mad_i32_i16_e64_dpp v5, v1.l, v2.l, m0 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_mad_i32_i16_e64_dpp v5, v1.l, v2.l, m0 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x5a,0xd6,0xe9,0x04,0xf6,0x01,0x01,0x77,0x39,0x05]
 
-v_mad_i32_i16_e64_dpp v255, v255, v255, src_scc clamp dpp8:[0,0,0,0,0,0,0,0] fi:0
-// GFX11: v_mad_i32_i16_e64_dpp v255, v255, v255, src_scc clamp dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x80,0x5a,0xd6,0xe9,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00]
+v_mad_i32_i16_e64_dpp v5, v1.h, v2.l, 0.5 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: v_mad_i32_i16_e64_dpp v5, v1.h, v2.l, 0.5 op_sel:[1,0,0,0] dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x05,0x08,0x5a,0xd6,0xea,0x04,0xc2,0x03,0x01,0x77,0x39,0x05]
+
+v_mad_i32_i16_e64_dpp v255, v255.l, v255.h, src_scc clamp dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: v_mad_i32_i16_e64_dpp v255, v255.l, v255.h, src_scc op_sel:[0,1,0,0] clamp dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x90,0x5a,0xd6,0xe9,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00]
 
 v_mad_i32_i24_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0]
 // GFX11: v_mad_i32_i24_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0a,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
@@ -1288,41 +1309,53 @@ v_mad_u16_e64_dpp v5.l, v1.l, v2.h, -1 dpp8:[7,6,5,4,3,2,1,0] fi:1
 v_mad_u16_e64_dpp v255.h, v255.l, v255.l, src_scc clamp dpp8:[0,0,0,0,0,0,0,0] fi:0
 // GFX11: v_mad_u16_e64_dpp v255.h, v255.l, v255.l, src_scc op_sel:[0,0,0,1] clamp dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0xc0,0x41,0xd6,0xe9,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00]
 
-v_mad_u32_u16_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_mad_u32_u16_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x59,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
+v_mad_u32_u16_e64_dpp v5, v1.l, v2.l, v3 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_mad_u32_u16_e64_dpp v5, v1.l, v2.l, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x59,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
+
+v_mad_u32_u16_e64_dpp v5, v1.l, v2.l, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_mad_u32_u16_e64_dpp v5, v1.l, v2.l, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x59,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
+
+v_mad_u32_u16_e64_dpp v5, v1.l, v2.l, s105 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_mad_u32_u16_e64_dpp v5, v1.l, v2.l, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x59,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
+
+v_mad_u32_u16_e64_dpp v5, v1.l, v2.l, vcc_hi dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_mad_u32_u16_e64_dpp v5, v1.l, v2.l, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x59,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
+
+v_mad_u32_u16_e64_dpp v5, v1.l, v2.l, vcc_lo dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_mad_u32_u16_e64_dpp v5, v1.l, v2.l, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x59,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 
-v_mad_u32_u16_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_mad_u32_u16_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x59,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
+v_mad_u32_u16_e64_dpp v5, v1.l, v2.l, ttmp15 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_mad_u32_u16_e64_dpp v5, v1.l, v2.l, ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x59,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05]
 
-v_mad_u32_u16_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_mad_u32_u16_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x59,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
+v_mad_u32_u16_e64_dpp v5, v1.l, v2.l, exec_hi dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_mad_u32_u16_e64_dpp v5, v1.l, v2.l, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x59,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05]
 
-v_mad_u32_u16_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_mad_u32_u16_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x59,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
+v_mad_u32_u16_e64_dpp v5, v1.l, v2.l, exec_lo dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_mad_u32_u16_e64_dpp v5, v1.l, v2.l, exec_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x59,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05]
 
-v_mad_u32_u16_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_mad_u32_u16_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x59,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
+v_mad_u32_u16_e64_dpp v5, v1.l, v2.l, null dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_mad_u32_u16_e64_dpp v5, v1.l, v2.l, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x59,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05]
 
-v_mad_u32_u16_e64_dpp v5, v1, v2, ttmp15 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_mad_u32_u16_e64_dpp v5, v1, v2, ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x59,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05]
+v_mad_u32_u16_e64_dpp v5, v1.l, v2.l, -1 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_mad_u32_u16_e64_dpp v5, v1.l, v2.l, -1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x59,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05]
 
-v_mad_u32_u16_e64_dpp v5, v1, v2, exec_hi dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_mad_u32_u16_e64_dpp v5, v1, v2, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x59,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05]
+v_mad_u32_u16_e64_dpp v5, v1.l, v2.l, 0.5 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: v_mad_u32_u16_e64_dpp v5, v1.l, v2.l, 0.5 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x05,0x00,0x59,0xd6,0xea,0x04,0xc2,0x03,0x01,0x77,0x39,0x05]
 
-v_mad_u32_u16_e64_dpp v5, v1, v2, exec_lo dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_mad_u32_u16_e64_dpp v5, v1, v2, exec_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x59,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05]
+v_mad_u32_u16_e64_dpp v255, v255.l, v255.l, src_scc clamp dpp8:[0,0,0,0,0,0,0,0]
+// GFX11: v_mad_u32_u16_e64_dpp v255, v255.l, v255.l, src_scc clamp dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x80,0x59,0xd6,0xe9,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00]
 
-v_mad_u32_u16_e64_dpp v5, v1, v2, null dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_mad_u32_u16_e64_dpp v5, v1, v2, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x59,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05]
+v_mad_u32_u16_e64_dpp v5, v1.l, v2.l, s3 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_mad_u32_u16_e64_dpp v5, v1.l, v2.l, s3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x59,0xd6,0xe9,0x04,0x0e,0x00,0x01,0x77,0x39,0x05]
 
-v_mad_u32_u16_e64_dpp v5, v1, v2, -1 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_mad_u32_u16_e64_dpp v5, v1, v2, -1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x59,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05]
+v_mad_u32_u16_e64_dpp v5, v1.l, v2.l, m0 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_mad_u32_u16_e64_dpp v5, v1.l, v2.l, m0 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x59,0xd6,0xe9,0x04,0xf6,0x01,0x01,0x77,0x39,0x05]
 
-v_mad_u32_u16_e64_dpp v5, v1, v2, 0.5 dpp8:[7,6,5,4,3,2,1,0] fi:1
-// GFX11: v_mad_u32_u16_e64_dpp v5, v1, v2, 0.5 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x05,0x00,0x59,0xd6,0xea,0x04,0xc2,0x03,0x01,0x77,0x39,0x05]
+v_mad_u32_u16_e64_dpp v5, v1.h, v2.l, 0.5 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: v_mad_u32_u16_e64_dpp v5, v1.h, v2.l, 0.5 op_sel:[1,0,0,0] dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x05,0x08,0x59,0xd6,0xea,0x04,0xc2,0x03,0x01,0x77,0x39,0x05]
 
-v_mad_u32_u16_e64_dpp v255, v255, v255, src_scc clamp dpp8:[0,0,0,0,0,0,0,0] fi:0
-// GFX11: v_mad_u32_u16_e64_dpp v255, v255, v255, src_scc clamp dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x80,0x59,0xd6,0xe9,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00]
+v_mad_u32_u16_e64_dpp v255, v255.l, v255.h, src_scc clamp dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: v_mad_u32_u16_e64_dpp v255, v255.l, v255.h, src_scc op_sel:[0,1,0,0] clamp dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x90,0x59,0xd6,0xe9,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00]
 
 v_mad_u32_u24_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0]
 // GFX11: v_mad_u32_u24_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0b,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
@@ -3298,11 +3331,11 @@ v_mad_i16_e64_dpp v5.l, v1.l, v2.l, -1 op_sel:[0,0,1,0] dpp8:[7,6,5,4,3,2,1,0]
 v_mad_i16_e64_dpp v255.h, v255.l, v255.l, src_scc op_sel:[0,0,0,1] clamp dpp8:[0,0,0,0,0,0,0,0] fi:1
 // GFX11: v_mad_i16_e64_dpp v255.h, v255.l, v255.l, src_scc op_sel:[0,0,0,1] clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0xc0,0x53,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00]
 
-v_mad_i32_i16_e64_dpp v5, v1, v2, 0.5 op_sel:[1,0,0,0] dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_mad_i32_i16_e64_dpp v5, v1, v2, 0.5 op_sel:[1,0,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x08,0x5a,0xd6,0xe9,0x04,0xc2,0x03,0x01,0x77,0x39,0x05]
+v_mad_i32_i16_e64_dpp v5, v1.h, v2.l, 0.5 op_sel:[1,0,0,0] dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_mad_i32_i16_e64_dpp v5, v1.h, v2.l, 0.5 op_sel:[1,0,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x08,0x5a,0xd6,0xe9,0x04,0xc2,0x03,0x01,0x77,0x39,0x05]
 
-v_mad_i32_i16_e64_dpp v255, v255, v255, src_scc op_sel:[0,1,0,0] clamp dpp8:[0,0,0,0,0,0,0,0] fi:1
-// GFX11: v_mad_i32_i16_e64_dpp v255, v255, v255, src_scc op_sel:[0,1,0,0] clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x90,0x5a,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00]
+v_mad_i32_i16_e64_dpp v255, v255.l, v255.h, src_scc op_sel:[0,1,0,0] clamp dpp8:[0,0,0,0,0,0,0,0] fi:1
+// GFX11: v_mad_i32_i16_e64_dpp v255, v255.l, v255.h, src_scc op_sel:[0,1,0,0] clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x90,0x5a,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00]
 
 v_mad_u16_e64_dpp v5.h, v1.h, v2.h, exec_hi op_sel:[1,1,1,1] dpp8:[7,6,5,4,3,2,1,0]
 // GFX11: v_mad_u16_e64_dpp v5.h, v1.h, v2.h, exec_hi op_sel:[1,1,1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x78,0x41,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05]
@@ -3319,11 +3352,11 @@ v_mad_u16_e64_dpp v5.l, v1.l, v2.l, -1 op_sel:[0,0,1,0] dpp8:[7,6,5,4,3,2,1,0]
 v_mad_u16_e64_dpp v255.h, v255.l, v255.l, src_scc op_sel:[0,0,0,1] clamp dpp8:[0,0,0,0,0,0,0,0] fi:1
 // GFX11: v_mad_u16_e64_dpp v255.h, v255.l, v255.l, src_scc op_sel:[0,0,0,1] clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0xc0,0x41,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00]
 
-v_mad_u32_u16_e64_dpp v5, v1, v2, 0.5 op_sel:[1,0,0,0] dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_mad_u32_u16_e64_dpp v5, v1, v2, 0.5 op_sel:[1,0,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x08,0x59,0xd6,0xe9,0x04,0xc2,0x03,0x01,0x77,0x39,0x05]
+v_mad_u32_u16_e64_dpp v5, v1.h, v2.l, 0.5 op_sel:[1,0,0,0] dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_mad_u32_u16_e64_dpp v5, v1.h, v2.l, 0.5 op_sel:[1,0,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x08,0x59,0xd6,0xe9,0x04,0xc2,0x03,0x01,0x77,0x39,0x05]
 
-v_mad_u32_u16_e64_dpp v255, v255, v255, src_scc op_sel:[0,1,0,0] clamp dpp8:[0,0,0,0,0,0,0,0] fi:1
-// GFX11: v_mad_u32_u16_e64_dpp v255, v255, v255, src_scc op_sel:[0,1,0,0] clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x90,0x59,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00]
+v_mad_u32_u16_e64_dpp v255, v255.l, v255.h, src_scc op_sel:[0,1,0,0] clamp dpp8:[0,0,0,0,0,0,0,0] fi:1
+// GFX11: v_mad_u32_u16_e64_dpp v255, v255.l, v255.h, src_scc op_sel:[0,1,0,0] clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x90,0x59,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00]
 
 v_max3_f16_e64_dpp v5.h, -v1.h, v2.h, |exec_lo| op_sel:[1,1,1,1] dpp8:[7,6,5,4,3,2,1,0]
 // GFX11: v_max3_f16_e64_dpp v5.h, -v1.h, v2.h, |exec_lo| op_sel:[1,1,1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x7c,0x4c,0xd6,0xe9,0x04,0xfa,0x21,0x01,0x77,0x39,0x05]
diff --git a/llvm/test/MC/AMDGPU/gfx11_asm_vop3_dpp8_from_vopcx.s b/llvm/test/MC/AMDGPU/gfx11_asm_vop3_dpp8_from_vopcx.s
index 119e4826b3277..0e36812c78dc1 100644
--- a/llvm/test/MC/AMDGPU/gfx11_asm_vop3_dpp8_from_vopcx.s
+++ b/llvm/test/MC/AMDGPU/gfx11_asm_vop3_dpp8_from_vopcx.s
@@ -29,17 +29,26 @@ v_cmpx_class_f32_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
 v_cmpx_class_f32_e64_dpp -|v255|, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
 // GFX11: v_cmpx_class_f32_e64_dpp -|v255|, v255 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0x7e,0x01,0xfe,0xd4,0xe9,0xfe,0x03,0x20,0xff,0x00,0x00,0x00]
 
-v_cmpx_eq_f16_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmpx_eq_f16_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x82,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+v_cmpx_eq_f16_e64_dpp v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_eq_f16_e64_dpp v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x82,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 
-v_cmpx_eq_f16_e64_dpp |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmpx_eq_f16_e64_dpp |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x01,0x82,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+v_cmpx_eq_f16_e64_dpp |v1.l|, -v2.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_eq_f16_e64_dpp |v1.l|, -v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x01,0x82,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
 
-v_cmpx_eq_f16_e64_dpp -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1
-// GFX11: v_cmpx_eq_f16_e64_dpp -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7e,0x02,0x82,0xd4,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+v_cmpx_eq_f16_e64_dpp -v1.l, |v2.l| dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: v_cmpx_eq_f16_e64_dpp -v1.l, |v2.l| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7e,0x02,0x82,0xd4,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
 
-v_cmpx_eq_f16_e64_dpp -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] fi:0
-// GFX11: v_cmpx_eq_f16_e64_dpp -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0x7e,0x83,0x82,0xd4,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+v_cmpx_eq_f16_e64_dpp -|v255.l|, -|v255.l| clamp dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: v_cmpx_eq_f16_e64_dpp -|v255.l|, -|v255.l| clamp dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0x7e,0x83,0x82,0xd4,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+
+v_cmpx_eq_f16_e64_dpp |v1.h|, -v2.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_eq_f16_e64_dpp |v1.h|, -v2.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x19,0x82,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+
+v_cmpx_eq_f16_e64_dpp -v1.h, |v2.l| dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: v_cmpx_eq_f16_e64_dpp -v1.h, |v2.l| op_sel:[1,0] dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7e,0x0a,0x82,0xd4,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+
+v_cmpx_eq_f16_e64_dpp -|v255.l|, -|v255.h| clamp dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: v_cmpx_eq_f16_e64_dpp -|v255.l|, -|v255.h| op_sel:[0,1] clamp dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0x7e,0x93,0x82,0xd4,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
 
 v_cmpx_eq_f32_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0]
 // GFX11: v_cmpx_eq_f32_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x92,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
@@ -107,17 +116,26 @@ v_cmpx_eq_u32_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
 v_cmpx_eq_u32_e64_dpp v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
 // GFX11: v_cmpx_eq_u32_e64_dpp v255, v255 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0x7e,0x00,0xca,0xd4,0xe9,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 
-v_cmpx_f_f16_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmpx_f_f16_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x80,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+v_cmpx_f_f16_e64_dpp v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_f_f16_e64_dpp v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x80,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+
+v_cmpx_f_f16_e64_dpp |v1.l|, -v2.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_f_f16_e64_dpp |v1.l|, -v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x01,0x80,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+
+v_cmpx_f_f16_e64_dpp -v1.l, |v2.l| dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: v_cmpx_f_f16_e64_dpp -v1.l, |v2.l| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7e,0x02,0x80,0xd4,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+
+v_cmpx_f_f16_e64_dpp -|v255.l|, -|v255.l| clamp dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: v_cmpx_f_f16_e64_dpp -|v255.l|, -|v255.l| clamp dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0x7e,0x83,0x80,0xd4,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
 
-v_cmpx_f_f16_e64_dpp |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmpx_f_f16_e64_dpp |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x01,0x80,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+v_cmpx_f_f16_e64_dpp |v1.h|, -v2.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_f_f16_e64_dpp |v1.h|, -v2.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x19,0x80,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
 
-v_cmpx_f_f16_e64_dpp -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1
-// GFX11: v_cmpx_f_f16_e64_dpp -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7e,0x02,0x80,0xd4,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+v_cmpx_f_f16_e64_dpp -v1.h, |v2.l| dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: v_cmpx_f_f16_e64_dpp -v1.h, |v2.l| op_sel:[1,0] dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7e,0x0a,0x80,0xd4,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
 
-v_cmpx_f_f16_e64_dpp -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] fi:0
-// GFX11: v_cmpx_f_f16_e64_dpp -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0x7e,0x83,0x80,0xd4,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+v_cmpx_f_f16_e64_dpp -|v255.l|, -|v255.h| clamp dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: v_cmpx_f_f16_e64_dpp -|v255.l|, -|v255.h| op_sel:[0,1] clamp dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0x7e,0x93,0x80,0xd4,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
 
 v_cmpx_f_f32_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0]
 // GFX11: v_cmpx_f_f32_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x90,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
@@ -149,17 +167,26 @@ v_cmpx_f_u32_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
 v_cmpx_f_u32_e64_dpp v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
 // GFX11: v_cmpx_f_u32_e64_dpp v255, v255 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0x7e,0x00,0xc8,0xd4,0xe9,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 
-v_cmpx_ge_f16_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmpx_ge_f16_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x86,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+v_cmpx_ge_f16_e64_dpp v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_ge_f16_e64_dpp v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x86,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 
-v_cmpx_ge_f16_e64_dpp |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmpx_ge_f16_e64_dpp |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x01,0x86,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+v_cmpx_ge_f16_e64_dpp |v1.l|, -v2.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_ge_f16_e64_dpp |v1.l|, -v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x01,0x86,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
 
-v_cmpx_ge_f16_e64_dpp -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1
-// GFX11: v_cmpx_ge_f16_e64_dpp -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7e,0x02,0x86,0xd4,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+v_cmpx_ge_f16_e64_dpp -v1.l, |v2.l| dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: v_cmpx_ge_f16_e64_dpp -v1.l, |v2.l| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7e,0x02,0x86,0xd4,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
 
-v_cmpx_ge_f16_e64_dpp -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] fi:0
-// GFX11: v_cmpx_ge_f16_e64_dpp -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0x7e,0x83,0x86,0xd4,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+v_cmpx_ge_f16_e64_dpp -|v255.l|, -|v255.l| clamp dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: v_cmpx_ge_f16_e64_dpp -|v255.l|, -|v255.l| clamp dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0x7e,0x83,0x86,0xd4,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+
+v_cmpx_ge_f16_e64_dpp |v1.h|, -v2.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_ge_f16_e64_dpp |v1.h|, -v2.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x19,0x86,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+
+v_cmpx_ge_f16_e64_dpp -v1.h, |v2.l| dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: v_cmpx_ge_f16_e64_dpp -v1.h, |v2.l| op_sel:[1,0] dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7e,0x0a,0x86,0xd4,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+
+v_cmpx_ge_f16_e64_dpp -|v255.l|, -|v255.h| clamp dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: v_cmpx_ge_f16_e64_dpp -|v255.l|, -|v255.h| op_sel:[0,1] clamp dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0x7e,0x93,0x86,0xd4,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
 
 v_cmpx_ge_f32_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0]
 // GFX11: v_cmpx_ge_f32_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x96,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
@@ -227,17 +254,26 @@ v_cmpx_ge_u32_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
 v_cmpx_ge_u32_e64_dpp v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
 // GFX11: v_cmpx_ge_u32_e64_dpp v255, v255 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0x7e,0x00,0xce,0xd4,0xe9,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 
-v_cmpx_gt_f16_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmpx_gt_f16_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x84,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+v_cmpx_gt_f16_e64_dpp v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_gt_f16_e64_dpp v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x84,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+
+v_cmpx_gt_f16_e64_dpp |v1.l|, -v2.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_gt_f16_e64_dpp |v1.l|, -v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x01,0x84,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+
+v_cmpx_gt_f16_e64_dpp -v1.l, |v2.l| dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: v_cmpx_gt_f16_e64_dpp -v1.l, |v2.l| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7e,0x02,0x84,0xd4,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+
+v_cmpx_gt_f16_e64_dpp -|v255.l|, -|v255.l| clamp dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: v_cmpx_gt_f16_e64_dpp -|v255.l|, -|v255.l| clamp dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0x7e,0x83,0x84,0xd4,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
 
-v_cmpx_gt_f16_e64_dpp |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmpx_gt_f16_e64_dpp |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x01,0x84,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+v_cmpx_gt_f16_e64_dpp |v1.h|, -v2.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_gt_f16_e64_dpp |v1.h|, -v2.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x19,0x84,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
 
-v_cmpx_gt_f16_e64_dpp -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1
-// GFX11: v_cmpx_gt_f16_e64_dpp -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7e,0x02,0x84,0xd4,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+v_cmpx_gt_f16_e64_dpp -v1.h, |v2.l| dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: v_cmpx_gt_f16_e64_dpp -v1.h, |v2.l| op_sel:[1,0] dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7e,0x0a,0x84,0xd4,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
 
-v_cmpx_gt_f16_e64_dpp -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] fi:0
-// GFX11: v_cmpx_gt_f16_e64_dpp -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0x7e,0x83,0x84,0xd4,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+v_cmpx_gt_f16_e64_dpp -|v255.l|, -|v255.h| clamp dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: v_cmpx_gt_f16_e64_dpp -|v255.l|, -|v255.h| op_sel:[0,1] clamp dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0x7e,0x93,0x84,0xd4,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
 
 v_cmpx_gt_f32_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0]
 // GFX11: v_cmpx_gt_f32_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x94,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
@@ -305,17 +341,26 @@ v_cmpx_gt_u32_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
 v_cmpx_gt_u32_e64_dpp v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
 // GFX11: v_cmpx_gt_u32_e64_dpp v255, v255 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0x7e,0x00,0xcc,0xd4,0xe9,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 
-v_cmpx_le_f16_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmpx_le_f16_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x83,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+v_cmpx_le_f16_e64_dpp v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_le_f16_e64_dpp v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x83,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 
-v_cmpx_le_f16_e64_dpp |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmpx_le_f16_e64_dpp |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x01,0x83,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+v_cmpx_le_f16_e64_dpp |v1.l|, -v2.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_le_f16_e64_dpp |v1.l|, -v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x01,0x83,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
 
-v_cmpx_le_f16_e64_dpp -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1
-// GFX11: v_cmpx_le_f16_e64_dpp -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7e,0x02,0x83,0xd4,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+v_cmpx_le_f16_e64_dpp -v1.l, |v2.l| dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: v_cmpx_le_f16_e64_dpp -v1.l, |v2.l| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7e,0x02,0x83,0xd4,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
 
-v_cmpx_le_f16_e64_dpp -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] fi:0
-// GFX11: v_cmpx_le_f16_e64_dpp -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0x7e,0x83,0x83,0xd4,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+v_cmpx_le_f16_e64_dpp -|v255.l|, -|v255.l| clamp dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: v_cmpx_le_f16_e64_dpp -|v255.l|, -|v255.l| clamp dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0x7e,0x83,0x83,0xd4,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+
+v_cmpx_le_f16_e64_dpp |v1.h|, -v2.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_le_f16_e64_dpp |v1.h|, -v2.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x19,0x83,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+
+v_cmpx_le_f16_e64_dpp -v1.h, |v2.l| dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: v_cmpx_le_f16_e64_dpp -v1.h, |v2.l| op_sel:[1,0] dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7e,0x0a,0x83,0xd4,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+
+v_cmpx_le_f16_e64_dpp -|v255.l|, -|v255.h| clamp dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: v_cmpx_le_f16_e64_dpp -|v255.l|, -|v255.h| op_sel:[0,1] clamp dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0x7e,0x93,0x83,0xd4,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
 
 v_cmpx_le_f32_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0]
 // GFX11: v_cmpx_le_f32_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x93,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
@@ -383,17 +428,26 @@ v_cmpx_le_u32_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
 v_cmpx_le_u32_e64_dpp v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
 // GFX11: v_cmpx_le_u32_e64_dpp v255, v255 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0x7e,0x00,0xcb,0xd4,0xe9,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 
-v_cmpx_lg_f16_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmpx_lg_f16_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x85,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+v_cmpx_lg_f16_e64_dpp v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_lg_f16_e64_dpp v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x85,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+
+v_cmpx_lg_f16_e64_dpp |v1.l|, -v2.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_lg_f16_e64_dpp |v1.l|, -v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x01,0x85,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+
+v_cmpx_lg_f16_e64_dpp -v1.l, |v2.l| dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: v_cmpx_lg_f16_e64_dpp -v1.l, |v2.l| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7e,0x02,0x85,0xd4,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+
+v_cmpx_lg_f16_e64_dpp -|v255.l|, -|v255.l| clamp dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: v_cmpx_lg_f16_e64_dpp -|v255.l|, -|v255.l| clamp dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0x7e,0x83,0x85,0xd4,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
 
-v_cmpx_lg_f16_e64_dpp |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmpx_lg_f16_e64_dpp |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x01,0x85,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+v_cmpx_lg_f16_e64_dpp |v1.h|, -v2.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_lg_f16_e64_dpp |v1.h|, -v2.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x19,0x85,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
 
-v_cmpx_lg_f16_e64_dpp -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1
-// GFX11: v_cmpx_lg_f16_e64_dpp -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7e,0x02,0x85,0xd4,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+v_cmpx_lg_f16_e64_dpp -v1.h, |v2.l| dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: v_cmpx_lg_f16_e64_dpp -v1.h, |v2.l| op_sel:[1,0] dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7e,0x0a,0x85,0xd4,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
 
-v_cmpx_lg_f16_e64_dpp -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] fi:0
-// GFX11: v_cmpx_lg_f16_e64_dpp -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0x7e,0x83,0x85,0xd4,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+v_cmpx_lg_f16_e64_dpp -|v255.l|, -|v255.h| clamp dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: v_cmpx_lg_f16_e64_dpp -|v255.l|, -|v255.h| op_sel:[0,1] clamp dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0x7e,0x93,0x85,0xd4,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
 
 v_cmpx_lg_f32_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0]
 // GFX11: v_cmpx_lg_f32_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x95,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
@@ -548,17 +602,26 @@ v_cmpx_ne_u32_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
 v_cmpx_ne_u32_e64_dpp v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
 // GFX11: v_cmpx_ne_u32_e64_dpp v255, v255 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0x7e,0x00,0xcd,0xd4,0xe9,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 
-v_cmpx_neq_f16_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmpx_neq_f16_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x8d,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+v_cmpx_neq_f16_e64_dpp v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_neq_f16_e64_dpp v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x8d,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 
-v_cmpx_neq_f16_e64_dpp |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmpx_neq_f16_e64_dpp |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x01,0x8d,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+v_cmpx_neq_f16_e64_dpp |v1.l|, -v2.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_neq_f16_e64_dpp |v1.l|, -v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x01,0x8d,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
 
-v_cmpx_neq_f16_e64_dpp -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1
-// GFX11: v_cmpx_neq_f16_e64_dpp -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7e,0x02,0x8d,0xd4,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+v_cmpx_neq_f16_e64_dpp -v1.l, |v2.l| dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: v_cmpx_neq_f16_e64_dpp -v1.l, |v2.l| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7e,0x02,0x8d,0xd4,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
 
-v_cmpx_neq_f16_e64_dpp -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] fi:0
-// GFX11: v_cmpx_neq_f16_e64_dpp -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0x7e,0x83,0x8d,0xd4,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+v_cmpx_neq_f16_e64_dpp -|v255.l|, -|v255.l| clamp dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: v_cmpx_neq_f16_e64_dpp -|v255.l|, -|v255.l| clamp dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0x7e,0x83,0x8d,0xd4,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+
+v_cmpx_neq_f16_e64_dpp |v1.h|, -v2.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_neq_f16_e64_dpp |v1.h|, -v2.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x19,0x8d,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+
+v_cmpx_neq_f16_e64_dpp -v1.h, |v2.l| dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: v_cmpx_neq_f16_e64_dpp -v1.h, |v2.l| op_sel:[1,0] dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7e,0x0a,0x8d,0xd4,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+
+v_cmpx_neq_f16_e64_dpp -|v255.l|, -|v255.h| clamp dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: v_cmpx_neq_f16_e64_dpp -|v255.l|, -|v255.h| op_sel:[0,1] clamp dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0x7e,0x93,0x8d,0xd4,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
 
 v_cmpx_neq_f32_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0]
 // GFX11: v_cmpx_neq_f32_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x9d,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
@@ -572,17 +635,26 @@ v_cmpx_neq_f32_e64_dpp -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1
 v_cmpx_neq_f32_e64_dpp -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] fi:0
 // GFX11: v_cmpx_neq_f32_e64_dpp -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0x7e,0x83,0x9d,0xd4,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
 
-v_cmpx_nge_f16_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmpx_nge_f16_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x89,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+v_cmpx_nge_f16_e64_dpp v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_nge_f16_e64_dpp v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x89,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+
+v_cmpx_nge_f16_e64_dpp |v1.l|, -v2.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_nge_f16_e64_dpp |v1.l|, -v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x01,0x89,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+
+v_cmpx_nge_f16_e64_dpp -v1.l, |v2.l| dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: v_cmpx_nge_f16_e64_dpp -v1.l, |v2.l| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7e,0x02,0x89,0xd4,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
 
-v_cmpx_nge_f16_e64_dpp |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmpx_nge_f16_e64_dpp |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x01,0x89,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+v_cmpx_nge_f16_e64_dpp -|v255.l|, -|v255.l| clamp dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: v_cmpx_nge_f16_e64_dpp -|v255.l|, -|v255.l| clamp dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0x7e,0x83,0x89,0xd4,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
 
-v_cmpx_nge_f16_e64_dpp -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1
-// GFX11: v_cmpx_nge_f16_e64_dpp -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7e,0x02,0x89,0xd4,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+v_cmpx_nge_f16_e64_dpp |v1.h|, -v2.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_nge_f16_e64_dpp |v1.h|, -v2.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x19,0x89,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
 
-v_cmpx_nge_f16_e64_dpp -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] fi:0
-// GFX11: v_cmpx_nge_f16_e64_dpp -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0x7e,0x83,0x89,0xd4,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+v_cmpx_nge_f16_e64_dpp -v1.h, |v2.l| dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: v_cmpx_nge_f16_e64_dpp -v1.h, |v2.l| op_sel:[1,0] dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7e,0x0a,0x89,0xd4,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+
+v_cmpx_nge_f16_e64_dpp -|v255.l|, -|v255.h| clamp dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: v_cmpx_nge_f16_e64_dpp -|v255.l|, -|v255.h| op_sel:[0,1] clamp dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0x7e,0x93,0x89,0xd4,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
 
 v_cmpx_nge_f32_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0]
 // GFX11: v_cmpx_nge_f32_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x99,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
@@ -596,17 +668,26 @@ v_cmpx_nge_f32_e64_dpp -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1
 v_cmpx_nge_f32_e64_dpp -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] fi:0
 // GFX11: v_cmpx_nge_f32_e64_dpp -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0x7e,0x83,0x99,0xd4,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
 
-v_cmpx_ngt_f16_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmpx_ngt_f16_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x8b,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+v_cmpx_ngt_f16_e64_dpp v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_ngt_f16_e64_dpp v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x8b,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+
+v_cmpx_ngt_f16_e64_dpp |v1.l|, -v2.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_ngt_f16_e64_dpp |v1.l|, -v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x01,0x8b,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+
+v_cmpx_ngt_f16_e64_dpp -v1.l, |v2.l| dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: v_cmpx_ngt_f16_e64_dpp -v1.l, |v2.l| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7e,0x02,0x8b,0xd4,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
 
-v_cmpx_ngt_f16_e64_dpp |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmpx_ngt_f16_e64_dpp |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x01,0x8b,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+v_cmpx_ngt_f16_e64_dpp -|v255.l|, -|v255.l| clamp dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: v_cmpx_ngt_f16_e64_dpp -|v255.l|, -|v255.l| clamp dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0x7e,0x83,0x8b,0xd4,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
 
-v_cmpx_ngt_f16_e64_dpp -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1
-// GFX11: v_cmpx_ngt_f16_e64_dpp -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7e,0x02,0x8b,0xd4,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+v_cmpx_ngt_f16_e64_dpp |v1.h|, -v2.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_ngt_f16_e64_dpp |v1.h|, -v2.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x19,0x8b,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
 
-v_cmpx_ngt_f16_e64_dpp -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] fi:0
-// GFX11: v_cmpx_ngt_f16_e64_dpp -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0x7e,0x83,0x8b,0xd4,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+v_cmpx_ngt_f16_e64_dpp -v1.h, |v2.l| dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: v_cmpx_ngt_f16_e64_dpp -v1.h, |v2.l| op_sel:[1,0] dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7e,0x0a,0x8b,0xd4,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+
+v_cmpx_ngt_f16_e64_dpp -|v255.l|, -|v255.h| clamp dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: v_cmpx_ngt_f16_e64_dpp -|v255.l|, -|v255.h| op_sel:[0,1] clamp dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0x7e,0x93,0x8b,0xd4,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
 
 v_cmpx_ngt_f32_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0]
 // GFX11: v_cmpx_ngt_f32_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x9b,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
@@ -620,17 +701,26 @@ v_cmpx_ngt_f32_e64_dpp -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1
 v_cmpx_ngt_f32_e64_dpp -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] fi:0
 // GFX11: v_cmpx_ngt_f32_e64_dpp -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0x7e,0x83,0x9b,0xd4,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
 
-v_cmpx_nle_f16_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmpx_nle_f16_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x8c,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+v_cmpx_nle_f16_e64_dpp v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_nle_f16_e64_dpp v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x8c,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+
+v_cmpx_nle_f16_e64_dpp |v1.l|, -v2.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_nle_f16_e64_dpp |v1.l|, -v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x01,0x8c,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+
+v_cmpx_nle_f16_e64_dpp -v1.l, |v2.l| dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: v_cmpx_nle_f16_e64_dpp -v1.l, |v2.l| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7e,0x02,0x8c,0xd4,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
 
-v_cmpx_nle_f16_e64_dpp |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmpx_nle_f16_e64_dpp |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x01,0x8c,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+v_cmpx_nle_f16_e64_dpp -|v255.l|, -|v255.l| clamp dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: v_cmpx_nle_f16_e64_dpp -|v255.l|, -|v255.l| clamp dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0x7e,0x83,0x8c,0xd4,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
 
-v_cmpx_nle_f16_e64_dpp -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1
-// GFX11: v_cmpx_nle_f16_e64_dpp -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7e,0x02,0x8c,0xd4,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+v_cmpx_nle_f16_e64_dpp |v1.h|, -v2.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_nle_f16_e64_dpp |v1.h|, -v2.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x19,0x8c,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
 
-v_cmpx_nle_f16_e64_dpp -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] fi:0
-// GFX11: v_cmpx_nle_f16_e64_dpp -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0x7e,0x83,0x8c,0xd4,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+v_cmpx_nle_f16_e64_dpp -v1.h, |v2.l| dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: v_cmpx_nle_f16_e64_dpp -v1.h, |v2.l| op_sel:[1,0] dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7e,0x0a,0x8c,0xd4,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+
+v_cmpx_nle_f16_e64_dpp -|v255.l|, -|v255.h| clamp dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: v_cmpx_nle_f16_e64_dpp -|v255.l|, -|v255.h| op_sel:[0,1] clamp dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0x7e,0x93,0x8c,0xd4,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
 
 v_cmpx_nle_f32_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0]
 // GFX11: v_cmpx_nle_f32_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x9c,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
@@ -644,17 +734,26 @@ v_cmpx_nle_f32_e64_dpp -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1
 v_cmpx_nle_f32_e64_dpp -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] fi:0
 // GFX11: v_cmpx_nle_f32_e64_dpp -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0x7e,0x83,0x9c,0xd4,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
 
-v_cmpx_nlg_f16_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmpx_nlg_f16_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x8a,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+v_cmpx_nlg_f16_e64_dpp v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_nlg_f16_e64_dpp v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x8a,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+
+v_cmpx_nlg_f16_e64_dpp |v1.l|, -v2.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_nlg_f16_e64_dpp |v1.l|, -v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x01,0x8a,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+
+v_cmpx_nlg_f16_e64_dpp -v1.l, |v2.l| dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: v_cmpx_nlg_f16_e64_dpp -v1.l, |v2.l| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7e,0x02,0x8a,0xd4,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
 
-v_cmpx_nlg_f16_e64_dpp |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmpx_nlg_f16_e64_dpp |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x01,0x8a,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+v_cmpx_nlg_f16_e64_dpp -|v255.l|, -|v255.l| clamp dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: v_cmpx_nlg_f16_e64_dpp -|v255.l|, -|v255.l| clamp dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0x7e,0x83,0x8a,0xd4,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
 
-v_cmpx_nlg_f16_e64_dpp -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1
-// GFX11: v_cmpx_nlg_f16_e64_dpp -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7e,0x02,0x8a,0xd4,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+v_cmpx_nlg_f16_e64_dpp |v1.h|, -v2.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_nlg_f16_e64_dpp |v1.h|, -v2.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x19,0x8a,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
 
-v_cmpx_nlg_f16_e64_dpp -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] fi:0
-// GFX11: v_cmpx_nlg_f16_e64_dpp -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0x7e,0x83,0x8a,0xd4,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+v_cmpx_nlg_f16_e64_dpp -v1.h, |v2.l| dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: v_cmpx_nlg_f16_e64_dpp -v1.h, |v2.l| op_sel:[1,0] dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7e,0x0a,0x8a,0xd4,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+
+v_cmpx_nlg_f16_e64_dpp -|v255.l|, -|v255.h| clamp dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: v_cmpx_nlg_f16_e64_dpp -|v255.l|, -|v255.h| op_sel:[0,1] clamp dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0x7e,0x93,0x8a,0xd4,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
 
 v_cmpx_nlg_f32_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0]
 // GFX11: v_cmpx_nlg_f32_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x9a,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
@@ -668,17 +767,26 @@ v_cmpx_nlg_f32_e64_dpp -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1
 v_cmpx_nlg_f32_e64_dpp -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] fi:0
 // GFX11: v_cmpx_nlg_f32_e64_dpp -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0x7e,0x83,0x9a,0xd4,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
 
-v_cmpx_nlt_f16_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmpx_nlt_f16_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x8e,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+v_cmpx_nlt_f16_e64_dpp v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_nlt_f16_e64_dpp v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x8e,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+
+v_cmpx_nlt_f16_e64_dpp |v1.l|, -v2.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_nlt_f16_e64_dpp |v1.l|, -v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x01,0x8e,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
 
-v_cmpx_nlt_f16_e64_dpp |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmpx_nlt_f16_e64_dpp |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x01,0x8e,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+v_cmpx_nlt_f16_e64_dpp -v1.l, |v2.l| dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: v_cmpx_nlt_f16_e64_dpp -v1.l, |v2.l| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7e,0x02,0x8e,0xd4,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
 
-v_cmpx_nlt_f16_e64_dpp -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1
-// GFX11: v_cmpx_nlt_f16_e64_dpp -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7e,0x02,0x8e,0xd4,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+v_cmpx_nlt_f16_e64_dpp -|v255.l|, -|v255.l| clamp dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: v_cmpx_nlt_f16_e64_dpp -|v255.l|, -|v255.l| clamp dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0x7e,0x83,0x8e,0xd4,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
 
-v_cmpx_nlt_f16_e64_dpp -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] fi:0
-// GFX11: v_cmpx_nlt_f16_e64_dpp -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0x7e,0x83,0x8e,0xd4,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+v_cmpx_nlt_f16_e64_dpp |v1.h|, -v2.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_nlt_f16_e64_dpp |v1.h|, -v2.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x19,0x8e,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+
+v_cmpx_nlt_f16_e64_dpp -v1.h, |v2.l| dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: v_cmpx_nlt_f16_e64_dpp -v1.h, |v2.l| op_sel:[1,0] dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7e,0x0a,0x8e,0xd4,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+
+v_cmpx_nlt_f16_e64_dpp -|v255.l|, -|v255.h| clamp dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: v_cmpx_nlt_f16_e64_dpp -|v255.l|, -|v255.h| op_sel:[0,1] clamp dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0x7e,0x93,0x8e,0xd4,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
 
 v_cmpx_nlt_f32_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0]
 // GFX11: v_cmpx_nlt_f32_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x9e,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
@@ -692,17 +800,26 @@ v_cmpx_nlt_f32_e64_dpp -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1
 v_cmpx_nlt_f32_e64_dpp -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] fi:0
 // GFX11: v_cmpx_nlt_f32_e64_dpp -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0x7e,0x83,0x9e,0xd4,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
 
-v_cmpx_o_f16_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmpx_o_f16_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x87,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+v_cmpx_o_f16_e64_dpp v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_o_f16_e64_dpp v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x87,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+
+v_cmpx_o_f16_e64_dpp |v1.l|, -v2.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_o_f16_e64_dpp |v1.l|, -v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x01,0x87,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
 
-v_cmpx_o_f16_e64_dpp |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmpx_o_f16_e64_dpp |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x01,0x87,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+v_cmpx_o_f16_e64_dpp -v1.l, |v2.l| dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: v_cmpx_o_f16_e64_dpp -v1.l, |v2.l| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7e,0x02,0x87,0xd4,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
 
-v_cmpx_o_f16_e64_dpp -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1
-// GFX11: v_cmpx_o_f16_e64_dpp -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7e,0x02,0x87,0xd4,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+v_cmpx_o_f16_e64_dpp -|v255.l|, -|v255.l| clamp dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: v_cmpx_o_f16_e64_dpp -|v255.l|, -|v255.l| clamp dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0x7e,0x83,0x87,0xd4,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
 
-v_cmpx_o_f16_e64_dpp -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] fi:0
-// GFX11: v_cmpx_o_f16_e64_dpp -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0x7e,0x83,0x87,0xd4,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+v_cmpx_o_f16_e64_dpp |v1.h|, -v2.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_o_f16_e64_dpp |v1.h|, -v2.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x19,0x87,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+
+v_cmpx_o_f16_e64_dpp -v1.h, |v2.l| dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: v_cmpx_o_f16_e64_dpp -v1.h, |v2.l| op_sel:[1,0] dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7e,0x0a,0x87,0xd4,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+
+v_cmpx_o_f16_e64_dpp -|v255.l|, -|v255.h| clamp dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: v_cmpx_o_f16_e64_dpp -|v255.l|, -|v255.h| op_sel:[0,1] clamp dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0x7e,0x93,0x87,0xd4,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
 
 v_cmpx_o_f32_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0]
 // GFX11: v_cmpx_o_f32_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x97,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
@@ -716,17 +833,17 @@ v_cmpx_o_f32_e64_dpp -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1
 v_cmpx_o_f32_e64_dpp -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] fi:0
 // GFX11: v_cmpx_o_f32_e64_dpp -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0x7e,0x83,0x97,0xd4,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
 
-v_cmpx_t_f16_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmpx_t_f16_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x8f,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+v_cmpx_t_f16_e64_dpp v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_t_f16_e64_dpp v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x8f,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 
-v_cmpx_t_f16_e64_dpp |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmpx_t_f16_e64_dpp |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x01,0x8f,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+v_cmpx_t_f16_e64_dpp |v1.l|, -v2.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_t_f16_e64_dpp |v1.l|, -v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x01,0x8f,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
 
-v_cmpx_t_f16_e64_dpp -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1
-// GFX11: v_cmpx_t_f16_e64_dpp -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7e,0x02,0x8f,0xd4,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+v_cmpx_t_f16_e64_dpp -v1.l, |v2.l| dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: v_cmpx_t_f16_e64_dpp -v1.l, |v2.l| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7e,0x02,0x8f,0xd4,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
 
-v_cmpx_t_f16_e64_dpp -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] fi:0
-// GFX11: v_cmpx_t_f16_e64_dpp -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0x7e,0x83,0x8f,0xd4,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+v_cmpx_t_f16_e64_dpp -|v255.l|, -|v255.l| clamp dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: v_cmpx_t_f16_e64_dpp -|v255.l|, -|v255.l| clamp dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0x7e,0x83,0x8f,0xd4,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
 
 v_cmpx_t_f32_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0]
 // GFX11: v_cmpx_t_f32_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x9f,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
@@ -758,17 +875,26 @@ v_cmpx_t_u32_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
 v_cmpx_t_u32_e64_dpp v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
 // GFX11: v_cmpx_t_u32_e64_dpp v255, v255 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0x7e,0x00,0xcf,0xd4,0xe9,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 
-v_cmpx_tru_f16_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmpx_t_f16_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x8f,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+v_cmpx_tru_f16_e64_dpp v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_t_f16_e64_dpp v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x8f,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+
+v_cmpx_tru_f16_e64_dpp |v1.l|, -v2.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_t_f16_e64_dpp |v1.l|, -v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x01,0x8f,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
 
-v_cmpx_tru_f16_e64_dpp |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmpx_t_f16_e64_dpp |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x01,0x8f,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+v_cmpx_tru_f16_e64_dpp -v1.l, |v2.l| dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: v_cmpx_t_f16_e64_dpp -v1.l, |v2.l| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7e,0x02,0x8f,0xd4,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
 
-v_cmpx_tru_f16_e64_dpp -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1
-// GFX11: v_cmpx_t_f16_e64_dpp -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7e,0x02,0x8f,0xd4,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+v_cmpx_tru_f16_e64_dpp -|v255.l|, -|v255.l| clamp dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: v_cmpx_t_f16_e64_dpp -|v255.l|, -|v255.l| clamp dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0x7e,0x83,0x8f,0xd4,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
 
-v_cmpx_tru_f16_e64_dpp -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] fi:0
-// GFX11: v_cmpx_t_f16_e64_dpp -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0x7e,0x83,0x8f,0xd4,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+v_cmpx_t_f16_e64_dpp |v1.h|, -v2.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_t_f16_e64_dpp |v1.h|, -v2.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x19,0x8f,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+
+v_cmpx_t_f16_e64_dpp -v1.h, |v2.l| dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: v_cmpx_t_f16_e64_dpp -v1.h, |v2.l| op_sel:[1,0] dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7e,0x0a,0x8f,0xd4,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+
+v_cmpx_t_f16_e64_dpp -|v255.l|, -|v255.h| clamp dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: v_cmpx_t_f16_e64_dpp -|v255.l|, -|v255.h| op_sel:[0,1] clamp dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0x7e,0x93,0x8f,0xd4,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
 
 v_cmpx_tru_f32_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0]
 // GFX11: v_cmpx_t_f32_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x9f,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
@@ -782,17 +908,26 @@ v_cmpx_tru_f32_e64_dpp -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1
 v_cmpx_tru_f32_e64_dpp -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] fi:0
 // GFX11: v_cmpx_t_f32_e64_dpp -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0x7e,0x83,0x9f,0xd4,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
 
-v_cmpx_u_f16_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmpx_u_f16_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x88,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+v_cmpx_u_f16_e64_dpp v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_u_f16_e64_dpp v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x88,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+
+v_cmpx_u_f16_e64_dpp |v1.l|, -v2.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_u_f16_e64_dpp |v1.l|, -v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x01,0x88,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+
+v_cmpx_u_f16_e64_dpp -v1.l, |v2.l| dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: v_cmpx_u_f16_e64_dpp -v1.l, |v2.l| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7e,0x02,0x88,0xd4,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+
+v_cmpx_u_f16_e64_dpp -|v255.l|, -|v255.l| clamp dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: v_cmpx_u_f16_e64_dpp -|v255.l|, -|v255.l| clamp dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0x7e,0x83,0x88,0xd4,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
 
-v_cmpx_u_f16_e64_dpp |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmpx_u_f16_e64_dpp |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x01,0x88,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+v_cmpx_u_f16_e64_dpp |v1.h|, -v2.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_u_f16_e64_dpp |v1.h|, -v2.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x19,0x88,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
 
-v_cmpx_u_f16_e64_dpp -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1
-// GFX11: v_cmpx_u_f16_e64_dpp -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7e,0x02,0x88,0xd4,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+v_cmpx_u_f16_e64_dpp -v1.h, |v2.l| dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: v_cmpx_u_f16_e64_dpp -v1.h, |v2.l| op_sel:[1,0] dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7e,0x0a,0x88,0xd4,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
 
-v_cmpx_u_f16_e64_dpp -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] fi:0
-// GFX11: v_cmpx_u_f16_e64_dpp -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0x7e,0x83,0x88,0xd4,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+v_cmpx_u_f16_e64_dpp -|v255.l|, -|v255.h| clamp dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: v_cmpx_u_f16_e64_dpp -|v255.l|, -|v255.h| op_sel:[0,1] clamp dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0x7e,0x93,0x88,0xd4,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
 
 v_cmpx_u_f32_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0]
 // GFX11: v_cmpx_u_f32_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x98,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
diff --git a/llvm/test/MC/AMDGPU/gfx11_asm_vop3_from_vopc.s b/llvm/test/MC/AMDGPU/gfx11_asm_vop3_from_vopc.s
index d1e55d105c0b5..8ab6c93adf874 100644
--- a/llvm/test/MC/AMDGPU/gfx11_asm_vop3_from_vopc.s
+++ b/llvm/test/MC/AMDGPU/gfx11_asm_vop3_from_vopc.s
@@ -128,19 +128,19 @@ v_cmp_class_f16_e64 vcc_lo, 0.5, m0
 // W64-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
 
 v_cmp_class_f16_e64 s5, v255.h, v2.l
-// W32: v_cmp_class_f16_e64 s5, v255.h, v2.l    ; encoding: [0x05,0x08,0x7d,0xd4,0xff,0x05,0x02,0x00]
+// W32: v_cmp_class_f16_e64 s5, v255.h, v2.l op_sel:[1,0,0] ; encoding: [0x05,0x08,0x7d,0xd4,0xff,0x05,0x02,0x00]
 // W64-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
 
 v_cmp_class_f16_e64 s5, s105, v255.h
-// W32: v_cmp_class_f16_e64 s5, s105, v255.h    ; encoding: [0x05,0x10,0x7d,0xd4,0x69,0xfe,0x03,0x00]
+// W32: v_cmp_class_f16_e64 s5, s105, v255.h op_sel:[0,1,0] ; encoding: [0x05,0x10,0x7d,0xd4,0x69,0xfe,0x03,0x00]
 // W64-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
 
 v_cmp_class_f16_e64 s[10:11], v255.h, v2.l
-// W64: v_cmp_class_f16_e64 s[10:11], v255.h, v2.l ; encoding: [0x0a,0x08,0x7d,0xd4,0xff,0x05,0x02,0x00]
+// W64: v_cmp_class_f16_e64 s[10:11], v255.h, v2.l op_sel:[1,0,0] ; encoding: [0x0a,0x08,0x7d,0xd4,0xff,0x05,0x02,0x00]
 // W32-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
 
 v_cmp_class_f16_e64 s[10:11], s105, v255.h
-// W64: v_cmp_class_f16_e64 s[10:11], s105, v255.h ; encoding: [0x0a,0x10,0x7d,0xd4,0x69,0xfe,0x03,0x00]
+// W64: v_cmp_class_f16_e64 s[10:11], s105, v255.h op_sel:[0,1,0] ; encoding: [0x0a,0x10,0x7d,0xd4,0x69,0xfe,0x03,0x00]
 // W32-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
 
 v_cmp_class_f32_e64 s5, v1, v2
diff --git a/llvm/test/MC/AMDGPU/gfx11_asm_vop3_from_vopcx.s b/llvm/test/MC/AMDGPU/gfx11_asm_vop3_from_vopcx.s
index 1614f00e1f07e..ed397bd1c2188 100644
--- a/llvm/test/MC/AMDGPU/gfx11_asm_vop3_from_vopcx.s
+++ b/llvm/test/MC/AMDGPU/gfx11_asm_vop3_from_vopcx.s
@@ -51,13 +51,13 @@ v_cmpx_class_f16_e64 v1.l, 0.5
 // GFX11: v_cmpx_class_f16_e64 v1.l, 0.5          ; encoding: [0x7e,0x00,0xfd,0xd4,0x01,0xe1,0x01,0x00]
 
 v_cmpx_class_f16_e64 v1.h, v2.h
-// GFX11: v_cmpx_class_f16_e64 v1.h, v2.h         ; encoding: [0x7e,0x18,0xfd,0xd4,0x01,0x05,0x02,0x00]
+// GFX11: v_cmpx_class_f16_e64 v1.h, v2.h op_sel:[1,1,0] ; encoding: [0x7e,0x18,0xfd,0xd4,0x01,0x05,0x02,0x00]
 
 v_cmpx_class_f16_e64 v255.h, v2.l
-// GFX11: v_cmpx_class_f16_e64 v255.h, v2.l       ; encoding: [0x7e,0x08,0xfd,0xd4,0xff,0x05,0x02,0x00]
+// GFX11: v_cmpx_class_f16_e64 v255.h, v2.l op_sel:[1,0,0] ; encoding: [0x7e,0x08,0xfd,0xd4,0xff,0x05,0x02,0x00]
 
 v_cmpx_class_f16_e64 s105, v255.h
-// GFX11: v_cmpx_class_f16_e64 s105, v255.h       ; encoding: [0x7e,0x10,0xfd,0xd4,0x69,0xfe,0x03,0x00]
+// GFX11: v_cmpx_class_f16_e64 s105, v255.h op_sel:[0,1,0] ; encoding: [0x7e,0x10,0xfd,0xd4,0x69,0xfe,0x03,0x00]
 
 v_cmpx_class_f32_e64 v1, v2
 // GFX11: v_cmpx_class_f32_e64 v1, v2             ; encoding: [0x7e,0x00,0xfe,0xd4,0x01,0x05,0x02,0x00]
@@ -149,11 +149,11 @@ v_cmpx_class_f64_e64 -|src_scc|, src_scc
 v_cmpx_class_f64_e64 0xaf123456, 0xaf123456
 // GFX11: v_cmpx_class_f64_e64 0xaf123456, 0xaf123456 ; encoding: [0x7e,0x00,0xff,0xd4,0xff,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
 
-v_cmpx_eq_f16_e64 v1, v2
-// GFX11: v_cmpx_eq_f16_e64 v1, v2                ; encoding: [0x7e,0x00,0x82,0xd4,0x01,0x05,0x02,0x00]
+v_cmpx_eq_f16_e64 v1.l, v2.l
+// GFX11: v_cmpx_eq_f16_e64 v1.l, v2.l            ; encoding: [0x7e,0x00,0x82,0xd4,0x01,0x05,0x02,0x00]
 
-v_cmpx_eq_f16_e64 v255, v255
-// GFX11: v_cmpx_eq_f16_e64 v255, v255            ; encoding: [0x7e,0x00,0x82,0xd4,0xff,0xff,0x03,0x00]
+v_cmpx_eq_f16_e64 v255.l, v255.l
+// GFX11: v_cmpx_eq_f16_e64 v255.l, v255.l        ; encoding: [0x7e,0x00,0x82,0xd4,0xff,0xff,0x03,0x00]
 
 v_cmpx_eq_f16_e64 s1, s2
 // GFX11: v_cmpx_eq_f16_e64 s1, s2                ; encoding: [0x7e,0x00,0x82,0xd4,0x01,0x04,0x00,0x00]
@@ -194,6 +194,12 @@ v_cmpx_eq_f16_e64 -src_scc, |vcc_lo|
 v_cmpx_eq_f16_e64 -|0xfe0b|, -|vcc_hi| clamp
 // GFX11: v_cmpx_eq_f16_e64 -|0xfe0b|, -|vcc_hi| clamp ; encoding: [0x7e,0x83,0x82,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00]
 
+v_cmpx_eq_f16_e64 v1.h, v2.l
+// GFX11: v_cmpx_eq_f16_e64 v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x7e,0x08,0x82,0xd4,0x01,0x05,0x02,0x00]
+
+v_cmpx_eq_f16_e64 v255.l, v255.h
+// GFX11: v_cmpx_eq_f16_e64 v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x7e,0x10,0x82,0xd4,0xff,0xff,0x03,0x00]
+
 v_cmpx_eq_f32_e64 v1, v2
 // GFX11: v_cmpx_eq_f32_e64 v1, v2                ; encoding: [0x7e,0x00,0x92,0xd4,0x01,0x05,0x02,0x00]
 
@@ -321,10 +327,10 @@ v_cmpx_eq_i16_e64 0xfe0b, vcc_hi
 // GFX11: v_cmpx_eq_i16_e64 0xfe0b, vcc_hi        ; encoding: [0x7e,0x00,0xb2,0xd4,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
 
 v_cmpx_eq_i16_e64 v1.h, v2.l
-// GFX11: v_cmpx_eq_i16_e64 v1.h, v2.l            ; encoding: [0x7e,0x08,0xb2,0xd4,0x01,0x05,0x02,0x00]
+// GFX11: v_cmpx_eq_i16_e64 v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x7e,0x08,0xb2,0xd4,0x01,0x05,0x02,0x00]
 
 v_cmpx_eq_i16_e64 v255.l, v255.h
-// GFX11: v_cmpx_eq_i16_e64 v255.l, v255.h        ; encoding: [0x7e,0x10,0xb2,0xd4,0xff,0xff,0x03,0x00]
+// GFX11: v_cmpx_eq_i16_e64 v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x7e,0x10,0xb2,0xd4,0xff,0xff,0x03,0x00]
 
 v_cmpx_eq_i32_e64 v1, v2
 // GFX11: v_cmpx_eq_i32_e64 v1, v2                ; encoding: [0x7e,0x00,0xc2,0xd4,0x01,0x05,0x02,0x00]
@@ -453,10 +459,10 @@ v_cmpx_eq_u16_e64 0xfe0b, vcc_hi
 // GFX11: v_cmpx_eq_u16_e64 0xfe0b, vcc_hi        ; encoding: [0x7e,0x00,0xba,0xd4,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
 
 v_cmpx_eq_u16_e64 v1.h, v2.l
-// GFX11: v_cmpx_eq_u16_e64 v1.h, v2.l            ; encoding: [0x7e,0x08,0xba,0xd4,0x01,0x05,0x02,0x00]
+// GFX11: v_cmpx_eq_u16_e64 v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x7e,0x08,0xba,0xd4,0x01,0x05,0x02,0x00]
 
 v_cmpx_eq_u16_e64 v255.l, v255.h
-// GFX11: v_cmpx_eq_u16_e64 v255.l, v255.h        ; encoding: [0x7e,0x10,0xba,0xd4,0xff,0xff,0x03,0x00]
+// GFX11: v_cmpx_eq_u16_e64 v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x7e,0x10,0xba,0xd4,0xff,0xff,0x03,0x00]
 
 v_cmpx_eq_u32_e64 v1, v2
 // GFX11: v_cmpx_eq_u32_e64 v1, v2                ; encoding: [0x7e,0x00,0xca,0xd4,0x01,0x05,0x02,0x00]
@@ -539,11 +545,11 @@ v_cmpx_eq_u64_e64 src_scc, exec
 v_cmpx_eq_u64_e64 0xaf123456, vcc
 // GFX11: v_cmpx_eq_u64_e64 0xaf123456, vcc       ; encoding: [0x7e,0x00,0xda,0xd4,0xff,0xd4,0x00,0x00,0x56,0x34,0x12,0xaf]
 
-v_cmpx_f_f16_e64 v1, v2
-// GFX11: v_cmpx_f_f16_e64 v1, v2                 ; encoding: [0x7e,0x00,0x80,0xd4,0x01,0x05,0x02,0x00]
+v_cmpx_f_f16_e64 v1.l, v2.l
+// GFX11: v_cmpx_f_f16_e64 v1.l, v2.l             ; encoding: [0x7e,0x00,0x80,0xd4,0x01,0x05,0x02,0x00]
 
-v_cmpx_f_f16_e64 v255, v255
-// GFX11: v_cmpx_f_f16_e64 v255, v255             ; encoding: [0x7e,0x00,0x80,0xd4,0xff,0xff,0x03,0x00]
+v_cmpx_f_f16_e64 v255.l, v255.l
+// GFX11: v_cmpx_f_f16_e64 v255.l, v255.l         ; encoding: [0x7e,0x00,0x80,0xd4,0xff,0xff,0x03,0x00]
 
 v_cmpx_f_f16_e64 s1, s2
 // GFX11: v_cmpx_f_f16_e64 s1, s2                 ; encoding: [0x7e,0x00,0x80,0xd4,0x01,0x04,0x00,0x00]
@@ -584,6 +590,12 @@ v_cmpx_f_f16_e64 -src_scc, |vcc_lo|
 v_cmpx_f_f16_e64 -|0xfe0b|, -|vcc_hi| clamp
 // GFX11: v_cmpx_f_f16_e64 -|0xfe0b|, -|vcc_hi| clamp ; encoding: [0x7e,0x83,0x80,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00]
 
+v_cmpx_f_f16_e64 v1.h, v2.l
+// GFX11: v_cmpx_f_f16_e64 v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x7e,0x08,0x80,0xd4,0x01,0x05,0x02,0x00]
+
+v_cmpx_f_f16_e64 v255.l, v255.h
+// GFX11: v_cmpx_f_f16_e64 v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x7e,0x10,0x80,0xd4,0xff,0xff,0x03,0x00]
+
 v_cmpx_f_f32_e64 v1, v2
 // GFX11: v_cmpx_f_f32_e64 v1, v2                 ; encoding: [0x7e,0x00,0x90,0xd4,0x01,0x05,0x02,0x00]
 
@@ -827,11 +839,11 @@ v_cmpx_f_u64_e64 src_scc, exec
 v_cmpx_f_u64_e64 0xaf123456, vcc
 // GFX11: v_cmpx_f_u64_e64 0xaf123456, vcc        ; encoding: [0x7e,0x00,0xd8,0xd4,0xff,0xd4,0x00,0x00,0x56,0x34,0x12,0xaf]
 
-v_cmpx_ge_f16_e64 v1, v2
-// GFX11: v_cmpx_ge_f16_e64 v1, v2                ; encoding: [0x7e,0x00,0x86,0xd4,0x01,0x05,0x02,0x00]
+v_cmpx_ge_f16_e64 v1.l, v2.l
+// GFX11: v_cmpx_ge_f16_e64 v1.l, v2.l            ; encoding: [0x7e,0x00,0x86,0xd4,0x01,0x05,0x02,0x00]
 
-v_cmpx_ge_f16_e64 v255, v255
-// GFX11: v_cmpx_ge_f16_e64 v255, v255            ; encoding: [0x7e,0x00,0x86,0xd4,0xff,0xff,0x03,0x00]
+v_cmpx_ge_f16_e64 v255.l, v255.l
+// GFX11: v_cmpx_ge_f16_e64 v255.l, v255.l        ; encoding: [0x7e,0x00,0x86,0xd4,0xff,0xff,0x03,0x00]
 
 v_cmpx_ge_f16_e64 s1, s2
 // GFX11: v_cmpx_ge_f16_e64 s1, s2                ; encoding: [0x7e,0x00,0x86,0xd4,0x01,0x04,0x00,0x00]
@@ -872,6 +884,12 @@ v_cmpx_ge_f16_e64 -src_scc, |vcc_lo|
 v_cmpx_ge_f16_e64 -|0xfe0b|, -|vcc_hi| clamp
 // GFX11: v_cmpx_ge_f16_e64 -|0xfe0b|, -|vcc_hi| clamp ; encoding: [0x7e,0x83,0x86,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00]
 
+v_cmpx_ge_f16_e64 v1.h, v2.l
+// GFX11: v_cmpx_ge_f16_e64 v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x7e,0x08,0x86,0xd4,0x01,0x05,0x02,0x00]
+
+v_cmpx_ge_f16_e64 v255.l, v255.h
+// GFX11: v_cmpx_ge_f16_e64 v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x7e,0x10,0x86,0xd4,0xff,0xff,0x03,0x00]
+
 v_cmpx_ge_f32_e64 v1, v2
 // GFX11: v_cmpx_ge_f32_e64 v1, v2                ; encoding: [0x7e,0x00,0x96,0xd4,0x01,0x05,0x02,0x00]
 
@@ -999,10 +1017,10 @@ v_cmpx_ge_i16_e64 0xfe0b, vcc_hi
 // GFX11: v_cmpx_ge_i16_e64 0xfe0b, vcc_hi        ; encoding: [0x7e,0x00,0xb6,0xd4,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
 
 v_cmpx_ge_i16_e64 v1.h, v2.l
-// GFX11: v_cmpx_ge_i16_e64 v1.h, v2.l            ; encoding: [0x7e,0x08,0xb6,0xd4,0x01,0x05,0x02,0x00]
+// GFX11: v_cmpx_ge_i16_e64 v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x7e,0x08,0xb6,0xd4,0x01,0x05,0x02,0x00]
 
 v_cmpx_ge_i16_e64 v255.l, v255.h
-// GFX11: v_cmpx_ge_i16_e64 v255.l, v255.h        ; encoding: [0x7e,0x10,0xb6,0xd4,0xff,0xff,0x03,0x00]
+// GFX11: v_cmpx_ge_i16_e64 v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x7e,0x10,0xb6,0xd4,0xff,0xff,0x03,0x00]
 
 v_cmpx_ge_i32_e64 v1, v2
 // GFX11: v_cmpx_ge_i32_e64 v1, v2                ; encoding: [0x7e,0x00,0xc6,0xd4,0x01,0x05,0x02,0x00]
@@ -1131,10 +1149,10 @@ v_cmpx_ge_u16_e64 0xfe0b, vcc_hi
 // GFX11: v_cmpx_ge_u16_e64 0xfe0b, vcc_hi        ; encoding: [0x7e,0x00,0xbe,0xd4,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
 
 v_cmpx_ge_u16_e64 v1.h, v2.l
-// GFX11: v_cmpx_ge_u16_e64 v1.h, v2.l            ; encoding: [0x7e,0x08,0xbe,0xd4,0x01,0x05,0x02,0x00]
+// GFX11: v_cmpx_ge_u16_e64 v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x7e,0x08,0xbe,0xd4,0x01,0x05,0x02,0x00]
 
 v_cmpx_ge_u16_e64 v255.l, v255.h
-// GFX11: v_cmpx_ge_u16_e64 v255.l, v255.h        ; encoding: [0x7e,0x10,0xbe,0xd4,0xff,0xff,0x03,0x00]
+// GFX11: v_cmpx_ge_u16_e64 v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x7e,0x10,0xbe,0xd4,0xff,0xff,0x03,0x00]
 
 v_cmpx_ge_u32_e64 v1, v2
 // GFX11: v_cmpx_ge_u32_e64 v1, v2                ; encoding: [0x7e,0x00,0xce,0xd4,0x01,0x05,0x02,0x00]
@@ -1217,11 +1235,11 @@ v_cmpx_ge_u64_e64 src_scc, exec
 v_cmpx_ge_u64_e64 0xaf123456, vcc
 // GFX11: v_cmpx_ge_u64_e64 0xaf123456, vcc       ; encoding: [0x7e,0x00,0xde,0xd4,0xff,0xd4,0x00,0x00,0x56,0x34,0x12,0xaf]
 
-v_cmpx_gt_f16_e64 v1, v2
-// GFX11: v_cmpx_gt_f16_e64 v1, v2                ; encoding: [0x7e,0x00,0x84,0xd4,0x01,0x05,0x02,0x00]
+v_cmpx_gt_f16_e64 v1.l, v2.l
+// GFX11: v_cmpx_gt_f16_e64 v1.l, v2.l            ; encoding: [0x7e,0x00,0x84,0xd4,0x01,0x05,0x02,0x00]
 
-v_cmpx_gt_f16_e64 v255, v255
-// GFX11: v_cmpx_gt_f16_e64 v255, v255            ; encoding: [0x7e,0x00,0x84,0xd4,0xff,0xff,0x03,0x00]
+v_cmpx_gt_f16_e64 v255.l, v255.l
+// GFX11: v_cmpx_gt_f16_e64 v255.l, v255.l        ; encoding: [0x7e,0x00,0x84,0xd4,0xff,0xff,0x03,0x00]
 
 v_cmpx_gt_f16_e64 s1, s2
 // GFX11: v_cmpx_gt_f16_e64 s1, s2                ; encoding: [0x7e,0x00,0x84,0xd4,0x01,0x04,0x00,0x00]
@@ -1262,6 +1280,12 @@ v_cmpx_gt_f16_e64 -src_scc, |vcc_lo|
 v_cmpx_gt_f16_e64 -|0xfe0b|, -|vcc_hi| clamp
 // GFX11: v_cmpx_gt_f16_e64 -|0xfe0b|, -|vcc_hi| clamp ; encoding: [0x7e,0x83,0x84,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00]
 
+v_cmpx_gt_f16_e64 v1.h, v2.l
+// GFX11: v_cmpx_gt_f16_e64 v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x7e,0x08,0x84,0xd4,0x01,0x05,0x02,0x00]
+
+v_cmpx_gt_f16_e64 v255.l, v255.h
+// GFX11: v_cmpx_gt_f16_e64 v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x7e,0x10,0x84,0xd4,0xff,0xff,0x03,0x00]
+
 v_cmpx_gt_f32_e64 v1, v2
 // GFX11: v_cmpx_gt_f32_e64 v1, v2                ; encoding: [0x7e,0x00,0x94,0xd4,0x01,0x05,0x02,0x00]
 
@@ -1389,10 +1413,10 @@ v_cmpx_gt_i16_e64 0xfe0b, vcc_hi
 // GFX11: v_cmpx_gt_i16_e64 0xfe0b, vcc_hi        ; encoding: [0x7e,0x00,0xb4,0xd4,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
 
 v_cmpx_gt_i16_e64 v1.h, v2.l
-// GFX11: v_cmpx_gt_i16_e64 v1.h, v2.l            ; encoding: [0x7e,0x08,0xb4,0xd4,0x01,0x05,0x02,0x00]
+// GFX11: v_cmpx_gt_i16_e64 v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x7e,0x08,0xb4,0xd4,0x01,0x05,0x02,0x00]
 
 v_cmpx_gt_i16_e64 v255.l, v255.h
-// GFX11: v_cmpx_gt_i16_e64 v255.l, v255.h        ; encoding: [0x7e,0x10,0xb4,0xd4,0xff,0xff,0x03,0x00]
+// GFX11: v_cmpx_gt_i16_e64 v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x7e,0x10,0xb4,0xd4,0xff,0xff,0x03,0x00]
 
 v_cmpx_gt_i32_e64 v1, v2
 // GFX11: v_cmpx_gt_i32_e64 v1, v2                ; encoding: [0x7e,0x00,0xc4,0xd4,0x01,0x05,0x02,0x00]
@@ -1521,10 +1545,10 @@ v_cmpx_gt_u16_e64 0xfe0b, vcc_hi
 // GFX11: v_cmpx_gt_u16_e64 0xfe0b, vcc_hi        ; encoding: [0x7e,0x00,0xbc,0xd4,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
 
 v_cmpx_gt_u16_e64 v1.h, v2.l
-// GFX11: v_cmpx_gt_u16_e64 v1.h, v2.l            ; encoding: [0x7e,0x08,0xbc,0xd4,0x01,0x05,0x02,0x00]
+// GFX11: v_cmpx_gt_u16_e64 v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x7e,0x08,0xbc,0xd4,0x01,0x05,0x02,0x00]
 
 v_cmpx_gt_u16_e64 v255.l, v255.h
-// GFX11: v_cmpx_gt_u16_e64 v255.l, v255.h        ; encoding: [0x7e,0x10,0xbc,0xd4,0xff,0xff,0x03,0x00]
+// GFX11: v_cmpx_gt_u16_e64 v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x7e,0x10,0xbc,0xd4,0xff,0xff,0x03,0x00]
 
 v_cmpx_gt_u32_e64 v1, v2
 // GFX11: v_cmpx_gt_u32_e64 v1, v2                ; encoding: [0x7e,0x00,0xcc,0xd4,0x01,0x05,0x02,0x00]
@@ -1607,11 +1631,11 @@ v_cmpx_gt_u64_e64 src_scc, exec
 v_cmpx_gt_u64_e64 0xaf123456, vcc
 // GFX11: v_cmpx_gt_u64_e64 0xaf123456, vcc       ; encoding: [0x7e,0x00,0xdc,0xd4,0xff,0xd4,0x00,0x00,0x56,0x34,0x12,0xaf]
 
-v_cmpx_le_f16_e64 v1, v2
-// GFX11: v_cmpx_le_f16_e64 v1, v2                ; encoding: [0x7e,0x00,0x83,0xd4,0x01,0x05,0x02,0x00]
+v_cmpx_le_f16_e64 v1.l, v2.l
+// GFX11: v_cmpx_le_f16_e64 v1.l, v2.l            ; encoding: [0x7e,0x00,0x83,0xd4,0x01,0x05,0x02,0x00]
 
-v_cmpx_le_f16_e64 v255, v255
-// GFX11: v_cmpx_le_f16_e64 v255, v255            ; encoding: [0x7e,0x00,0x83,0xd4,0xff,0xff,0x03,0x00]
+v_cmpx_le_f16_e64 v255.l, v255.l
+// GFX11: v_cmpx_le_f16_e64 v255.l, v255.l        ; encoding: [0x7e,0x00,0x83,0xd4,0xff,0xff,0x03,0x00]
 
 v_cmpx_le_f16_e64 s1, s2
 // GFX11: v_cmpx_le_f16_e64 s1, s2                ; encoding: [0x7e,0x00,0x83,0xd4,0x01,0x04,0x00,0x00]
@@ -1652,6 +1676,12 @@ v_cmpx_le_f16_e64 -src_scc, |vcc_lo|
 v_cmpx_le_f16_e64 -|0xfe0b|, -|vcc_hi| clamp
 // GFX11: v_cmpx_le_f16_e64 -|0xfe0b|, -|vcc_hi| clamp ; encoding: [0x7e,0x83,0x83,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00]
 
+v_cmpx_le_f16_e64 v1.h, v2.l
+// GFX11: v_cmpx_le_f16_e64 v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x7e,0x08,0x83,0xd4,0x01,0x05,0x02,0x00]
+
+v_cmpx_le_f16_e64 v255.l, v255.h
+// GFX11: v_cmpx_le_f16_e64 v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x7e,0x10,0x83,0xd4,0xff,0xff,0x03,0x00]
+
 v_cmpx_le_f32_e64 v1, v2
 // GFX11: v_cmpx_le_f32_e64 v1, v2                ; encoding: [0x7e,0x00,0x93,0xd4,0x01,0x05,0x02,0x00]
 
@@ -1779,10 +1809,10 @@ v_cmpx_le_i16_e64 0xfe0b, vcc_hi
 // GFX11: v_cmpx_le_i16_e64 0xfe0b, vcc_hi        ; encoding: [0x7e,0x00,0xb3,0xd4,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
 
 v_cmpx_le_i16_e64 v1.h, v2.l
-// GFX11: v_cmpx_le_i16_e64 v1.h, v2.l            ; encoding: [0x7e,0x08,0xb3,0xd4,0x01,0x05,0x02,0x00]
+// GFX11: v_cmpx_le_i16_e64 v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x7e,0x08,0xb3,0xd4,0x01,0x05,0x02,0x00]
 
 v_cmpx_le_i16_e64 v255.l, v255.h
-// GFX11: v_cmpx_le_i16_e64 v255.l, v255.h        ; encoding: [0x7e,0x10,0xb3,0xd4,0xff,0xff,0x03,0x00]
+// GFX11: v_cmpx_le_i16_e64 v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x7e,0x10,0xb3,0xd4,0xff,0xff,0x03,0x00]
 
 v_cmpx_le_i32_e64 v1, v2
 // GFX11: v_cmpx_le_i32_e64 v1, v2                ; encoding: [0x7e,0x00,0xc3,0xd4,0x01,0x05,0x02,0x00]
@@ -1911,10 +1941,10 @@ v_cmpx_le_u16_e64 0xfe0b, vcc_hi
 // GFX11: v_cmpx_le_u16_e64 0xfe0b, vcc_hi        ; encoding: [0x7e,0x00,0xbb,0xd4,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
 
 v_cmpx_le_u16_e64 v1.h, v2.l
-// GFX11: v_cmpx_le_u16_e64 v1.h, v2.l            ; encoding: [0x7e,0x08,0xbb,0xd4,0x01,0x05,0x02,0x00]
+// GFX11: v_cmpx_le_u16_e64 v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x7e,0x08,0xbb,0xd4,0x01,0x05,0x02,0x00]
 
 v_cmpx_le_u16_e64 v255.l, v255.h
-// GFX11: v_cmpx_le_u16_e64 v255.l, v255.h        ; encoding: [0x7e,0x10,0xbb,0xd4,0xff,0xff,0x03,0x00]
+// GFX11: v_cmpx_le_u16_e64 v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x7e,0x10,0xbb,0xd4,0xff,0xff,0x03,0x00]
 
 v_cmpx_le_u32_e64 v1, v2
 // GFX11: v_cmpx_le_u32_e64 v1, v2                ; encoding: [0x7e,0x00,0xcb,0xd4,0x01,0x05,0x02,0x00]
@@ -1997,11 +2027,11 @@ v_cmpx_le_u64_e64 src_scc, exec
 v_cmpx_le_u64_e64 0xaf123456, vcc
 // GFX11: v_cmpx_le_u64_e64 0xaf123456, vcc       ; encoding: [0x7e,0x00,0xdb,0xd4,0xff,0xd4,0x00,0x00,0x56,0x34,0x12,0xaf]
 
-v_cmpx_lg_f16_e64 v1, v2
-// GFX11: v_cmpx_lg_f16_e64 v1, v2                ; encoding: [0x7e,0x00,0x85,0xd4,0x01,0x05,0x02,0x00]
+v_cmpx_lg_f16_e64 v1.l, v2.l
+// GFX11: v_cmpx_lg_f16_e64 v1.l, v2.l            ; encoding: [0x7e,0x00,0x85,0xd4,0x01,0x05,0x02,0x00]
 
-v_cmpx_lg_f16_e64 v255, v255
-// GFX11: v_cmpx_lg_f16_e64 v255, v255            ; encoding: [0x7e,0x00,0x85,0xd4,0xff,0xff,0x03,0x00]
+v_cmpx_lg_f16_e64 v255.l, v255.l
+// GFX11: v_cmpx_lg_f16_e64 v255.l, v255.l        ; encoding: [0x7e,0x00,0x85,0xd4,0xff,0xff,0x03,0x00]
 
 v_cmpx_lg_f16_e64 s1, s2
 // GFX11: v_cmpx_lg_f16_e64 s1, s2                ; encoding: [0x7e,0x00,0x85,0xd4,0x01,0x04,0x00,0x00]
@@ -2042,6 +2072,12 @@ v_cmpx_lg_f16_e64 -src_scc, |vcc_lo|
 v_cmpx_lg_f16_e64 -|0xfe0b|, -|vcc_hi| clamp
 // GFX11: v_cmpx_lg_f16_e64 -|0xfe0b|, -|vcc_hi| clamp ; encoding: [0x7e,0x83,0x85,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00]
 
+v_cmpx_lg_f16_e64 v1.h, v2.l
+// GFX11: v_cmpx_lg_f16_e64 v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x7e,0x08,0x85,0xd4,0x01,0x05,0x02,0x00]
+
+v_cmpx_lg_f16_e64 v255.l, v255.h
+// GFX11: v_cmpx_lg_f16_e64 v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x7e,0x10,0x85,0xd4,0xff,0xff,0x03,0x00]
+
 v_cmpx_lg_f32_e64 v1, v2
 // GFX11: v_cmpx_lg_f32_e64 v1, v2                ; encoding: [0x7e,0x00,0x95,0xd4,0x01,0x05,0x02,0x00]
 
@@ -2169,10 +2205,10 @@ v_cmpx_lt_f16_e64 -|0xfe0b|, -|vcc_hi| clamp
 // GFX11: v_cmpx_lt_f16_e64 -|0xfe0b|, -|vcc_hi| clamp ; encoding: [0x7e,0x83,0x81,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00]
 
 v_cmpx_lt_f16_e64 v1.h, v2.l
-// GFX11: v_cmpx_lt_f16_e64 v1.h, v2.l            ; encoding: [0x7e,0x08,0x81,0xd4,0x01,0x05,0x02,0x00]
+// GFX11: v_cmpx_lt_f16_e64 v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x7e,0x08,0x81,0xd4,0x01,0x05,0x02,0x00]
 
 v_cmpx_lt_f16_e64 v255.l, v255.h
-// GFX11: v_cmpx_lt_f16_e64 v255.l, v255.h        ; encoding: [0x7e,0x10,0x81,0xd4,0xff,0xff,0x03,0x00]
+// GFX11: v_cmpx_lt_f16_e64 v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x7e,0x10,0x81,0xd4,0xff,0xff,0x03,0x00]
 
 v_cmpx_lt_f32_e64 v1, v2
 // GFX11: v_cmpx_lt_f32_e64 v1, v2                ; encoding: [0x7e,0x00,0x91,0xd4,0x01,0x05,0x02,0x00]
@@ -2301,10 +2337,10 @@ v_cmpx_lt_i16_e64 0xfe0b, vcc_hi
 // GFX11: v_cmpx_lt_i16_e64 0xfe0b, vcc_hi        ; encoding: [0x7e,0x00,0xb1,0xd4,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
 
 v_cmpx_lt_i16_e64 v1.h, v2.l
-// GFX11: v_cmpx_lt_i16_e64 v1.h, v2.l            ; encoding: [0x7e,0x08,0xb1,0xd4,0x01,0x05,0x02,0x00]
+// GFX11: v_cmpx_lt_i16_e64 v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x7e,0x08,0xb1,0xd4,0x01,0x05,0x02,0x00]
 
 v_cmpx_lt_i16_e64 v255.l, v255.h
-// GFX11: v_cmpx_lt_i16_e64 v255.l, v255.h        ; encoding: [0x7e,0x10,0xb1,0xd4,0xff,0xff,0x03,0x00]
+// GFX11: v_cmpx_lt_i16_e64 v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x7e,0x10,0xb1,0xd4,0xff,0xff,0x03,0x00]
 
 v_cmpx_lt_i32_e64 v1, v2
 // GFX11: v_cmpx_lt_i32_e64 v1, v2                ; encoding: [0x7e,0x00,0xc1,0xd4,0x01,0x05,0x02,0x00]
@@ -2433,10 +2469,10 @@ v_cmpx_lt_u16_e64 0xfe0b, vcc_hi
 // GFX11: v_cmpx_lt_u16_e64 0xfe0b, vcc_hi        ; encoding: [0x7e,0x00,0xb9,0xd4,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
 
 v_cmpx_lt_u16_e64 v1.h, v2.l
-// GFX11: v_cmpx_lt_u16_e64 v1.h, v2.l            ; encoding: [0x7e,0x08,0xb9,0xd4,0x01,0x05,0x02,0x00]
+// GFX11: v_cmpx_lt_u16_e64 v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x7e,0x08,0xb9,0xd4,0x01,0x05,0x02,0x00]
 
 v_cmpx_lt_u16_e64 v255.l, v255.h
-// GFX11: v_cmpx_lt_u16_e64 v255.l, v255.h        ; encoding: [0x7e,0x10,0xb9,0xd4,0xff,0xff,0x03,0x00]
+// GFX11: v_cmpx_lt_u16_e64 v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x7e,0x10,0xb9,0xd4,0xff,0xff,0x03,0x00]
 
 v_cmpx_lt_u32_e64 v1, v2
 // GFX11: v_cmpx_lt_u32_e64 v1, v2                ; encoding: [0x7e,0x00,0xc9,0xd4,0x01,0x05,0x02,0x00]
@@ -2565,10 +2601,10 @@ v_cmpx_ne_i16_e64 0xfe0b, vcc_hi
 // GFX11: v_cmpx_ne_i16_e64 0xfe0b, vcc_hi        ; encoding: [0x7e,0x00,0xb5,0xd4,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
 
 v_cmpx_ne_i16_e64 v1.h, v2.l
-// GFX11: v_cmpx_ne_i16_e64 v1.h, v2.l            ; encoding: [0x7e,0x08,0xb5,0xd4,0x01,0x05,0x02,0x00]
+// GFX11: v_cmpx_ne_i16_e64 v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x7e,0x08,0xb5,0xd4,0x01,0x05,0x02,0x00]
 
 v_cmpx_ne_i16_e64 v255.l, v255.h
-// GFX11: v_cmpx_ne_i16_e64 v255.l, v255.h        ; encoding: [0x7e,0x10,0xb5,0xd4,0xff,0xff,0x03,0x00]
+// GFX11: v_cmpx_ne_i16_e64 v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x7e,0x10,0xb5,0xd4,0xff,0xff,0x03,0x00]
 
 v_cmpx_ne_i32_e64 v1, v2
 // GFX11: v_cmpx_ne_i32_e64 v1, v2                ; encoding: [0x7e,0x00,0xc5,0xd4,0x01,0x05,0x02,0x00]
@@ -2697,10 +2733,10 @@ v_cmpx_ne_u16_e64 0xfe0b, vcc_hi
 // GFX11: v_cmpx_ne_u16_e64 0xfe0b, vcc_hi        ; encoding: [0x7e,0x00,0xbd,0xd4,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
 
 v_cmpx_ne_u16_e64 v1.h, v2.l
-// GFX11: v_cmpx_ne_u16_e64 v1.h, v2.l            ; encoding: [0x7e,0x08,0xbd,0xd4,0x01,0x05,0x02,0x00]
+// GFX11: v_cmpx_ne_u16_e64 v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x7e,0x08,0xbd,0xd4,0x01,0x05,0x02,0x00]
 
 v_cmpx_ne_u16_e64 v255.l, v255.h
-// GFX11: v_cmpx_ne_u16_e64 v255.l, v255.h        ; encoding: [0x7e,0x10,0xbd,0xd4,0xff,0xff,0x03,0x00]
+// GFX11: v_cmpx_ne_u16_e64 v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x7e,0x10,0xbd,0xd4,0xff,0xff,0x03,0x00]
 
 v_cmpx_ne_u32_e64 v1, v2
 // GFX11: v_cmpx_ne_u32_e64 v1, v2                ; encoding: [0x7e,0x00,0xcd,0xd4,0x01,0x05,0x02,0x00]
@@ -2783,11 +2819,11 @@ v_cmpx_ne_u64_e64 src_scc, exec
 v_cmpx_ne_u64_e64 0xaf123456, vcc
 // GFX11: v_cmpx_ne_u64_e64 0xaf123456, vcc       ; encoding: [0x7e,0x00,0xdd,0xd4,0xff,0xd4,0x00,0x00,0x56,0x34,0x12,0xaf]
 
-v_cmpx_neq_f16_e64 v1, v2
-// GFX11: v_cmpx_neq_f16_e64 v1, v2               ; encoding: [0x7e,0x00,0x8d,0xd4,0x01,0x05,0x02,0x00]
+v_cmpx_neq_f16_e64 v1.l, v2.l
+// GFX11: v_cmpx_neq_f16_e64 v1.l, v2.l           ; encoding: [0x7e,0x00,0x8d,0xd4,0x01,0x05,0x02,0x00]
 
-v_cmpx_neq_f16_e64 v255, v255
-// GFX11: v_cmpx_neq_f16_e64 v255, v255           ; encoding: [0x7e,0x00,0x8d,0xd4,0xff,0xff,0x03,0x00]
+v_cmpx_neq_f16_e64 v255.l, v255.l
+// GFX11: v_cmpx_neq_f16_e64 v255.l, v255.l       ; encoding: [0x7e,0x00,0x8d,0xd4,0xff,0xff,0x03,0x00]
 
 v_cmpx_neq_f16_e64 s1, s2
 // GFX11: v_cmpx_neq_f16_e64 s1, s2               ; encoding: [0x7e,0x00,0x8d,0xd4,0x01,0x04,0x00,0x00]
@@ -2828,6 +2864,12 @@ v_cmpx_neq_f16_e64 -src_scc, |vcc_lo|
 v_cmpx_neq_f16_e64 -|0xfe0b|, -|vcc_hi| clamp
 // GFX11: v_cmpx_neq_f16_e64 -|0xfe0b|, -|vcc_hi| clamp ; encoding: [0x7e,0x83,0x8d,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00]
 
+v_cmpx_neq_f16_e64 v1.h, v2.l
+// GFX11: v_cmpx_neq_f16_e64 v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x7e,0x08,0x8d,0xd4,0x01,0x05,0x02,0x00]
+
+v_cmpx_neq_f16_e64 v255.l, v255.h
+// GFX11: v_cmpx_neq_f16_e64 v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x7e,0x10,0x8d,0xd4,0xff,0xff,0x03,0x00]
+
 v_cmpx_neq_f32_e64 v1, v2
 // GFX11: v_cmpx_neq_f32_e64 v1, v2               ; encoding: [0x7e,0x00,0x9d,0xd4,0x01,0x05,0x02,0x00]
 
@@ -2909,11 +2951,11 @@ v_cmpx_neq_f64_e64 -|src_scc|, -|exec|
 v_cmpx_neq_f64_e64 0xaf123456, -|vcc| clamp
 // GFX11: v_cmpx_neq_f64_e64 0xaf123456, -|vcc| clamp ; encoding: [0x7e,0x82,0xad,0xd4,0xff,0xd4,0x00,0x40,0x56,0x34,0x12,0xaf]
 
-v_cmpx_nge_f16_e64 v1, v2
-// GFX11: v_cmpx_nge_f16_e64 v1, v2               ; encoding: [0x7e,0x00,0x89,0xd4,0x01,0x05,0x02,0x00]
+v_cmpx_nge_f16_e64 v1.l, v2.l
+// GFX11: v_cmpx_nge_f16_e64 v1.l, v2.l           ; encoding: [0x7e,0x00,0x89,0xd4,0x01,0x05,0x02,0x00]
 
-v_cmpx_nge_f16_e64 v255, v255
-// GFX11: v_cmpx_nge_f16_e64 v255, v255           ; encoding: [0x7e,0x00,0x89,0xd4,0xff,0xff,0x03,0x00]
+v_cmpx_nge_f16_e64 v255.l, v255.l
+// GFX11: v_cmpx_nge_f16_e64 v255.l, v255.l       ; encoding: [0x7e,0x00,0x89,0xd4,0xff,0xff,0x03,0x00]
 
 v_cmpx_nge_f16_e64 s1, s2
 // GFX11: v_cmpx_nge_f16_e64 s1, s2               ; encoding: [0x7e,0x00,0x89,0xd4,0x01,0x04,0x00,0x00]
@@ -2954,6 +2996,12 @@ v_cmpx_nge_f16_e64 -src_scc, |vcc_lo|
 v_cmpx_nge_f16_e64 -|0xfe0b|, -|vcc_hi| clamp
 // GFX11: v_cmpx_nge_f16_e64 -|0xfe0b|, -|vcc_hi| clamp ; encoding: [0x7e,0x83,0x89,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00]
 
+v_cmpx_nge_f16_e64 v1.h, v2.l
+// GFX11: v_cmpx_nge_f16_e64 v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x7e,0x08,0x89,0xd4,0x01,0x05,0x02,0x00]
+
+v_cmpx_nge_f16_e64 v255.l, v255.h
+// GFX11: v_cmpx_nge_f16_e64 v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x7e,0x10,0x89,0xd4,0xff,0xff,0x03,0x00]
+
 v_cmpx_nge_f32_e64 v1, v2
 // GFX11: v_cmpx_nge_f32_e64 v1, v2               ; encoding: [0x7e,0x00,0x99,0xd4,0x01,0x05,0x02,0x00]
 
@@ -3035,11 +3083,11 @@ v_cmpx_nge_f64_e64 -|src_scc|, -|exec|
 v_cmpx_nge_f64_e64 0xaf123456, -|vcc| clamp
 // GFX11: v_cmpx_nge_f64_e64 0xaf123456, -|vcc| clamp ; encoding: [0x7e,0x82,0xa9,0xd4,0xff,0xd4,0x00,0x40,0x56,0x34,0x12,0xaf]
 
-v_cmpx_ngt_f16_e64 v1, v2
-// GFX11: v_cmpx_ngt_f16_e64 v1, v2               ; encoding: [0x7e,0x00,0x8b,0xd4,0x01,0x05,0x02,0x00]
+v_cmpx_ngt_f16_e64 v1.l, v2.l
+// GFX11: v_cmpx_ngt_f16_e64 v1.l, v2.l           ; encoding: [0x7e,0x00,0x8b,0xd4,0x01,0x05,0x02,0x00]
 
-v_cmpx_ngt_f16_e64 v255, v255
-// GFX11: v_cmpx_ngt_f16_e64 v255, v255           ; encoding: [0x7e,0x00,0x8b,0xd4,0xff,0xff,0x03,0x00]
+v_cmpx_ngt_f16_e64 v255.l, v255.l
+// GFX11: v_cmpx_ngt_f16_e64 v255.l, v255.l       ; encoding: [0x7e,0x00,0x8b,0xd4,0xff,0xff,0x03,0x00]
 
 v_cmpx_ngt_f16_e64 s1, s2
 // GFX11: v_cmpx_ngt_f16_e64 s1, s2               ; encoding: [0x7e,0x00,0x8b,0xd4,0x01,0x04,0x00,0x00]
@@ -3080,6 +3128,12 @@ v_cmpx_ngt_f16_e64 -src_scc, |vcc_lo|
 v_cmpx_ngt_f16_e64 -|0xfe0b|, -|vcc_hi| clamp
 // GFX11: v_cmpx_ngt_f16_e64 -|0xfe0b|, -|vcc_hi| clamp ; encoding: [0x7e,0x83,0x8b,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00]
 
+v_cmpx_ngt_f16_e64 v1.h, v2.l
+// GFX11: v_cmpx_ngt_f16_e64 v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x7e,0x08,0x8b,0xd4,0x01,0x05,0x02,0x00]
+
+v_cmpx_ngt_f16_e64 v255.l, v255.h
+// GFX11: v_cmpx_ngt_f16_e64 v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x7e,0x10,0x8b,0xd4,0xff,0xff,0x03,0x00]
+
 v_cmpx_ngt_f32_e64 v1, v2
 // GFX11: v_cmpx_ngt_f32_e64 v1, v2               ; encoding: [0x7e,0x00,0x9b,0xd4,0x01,0x05,0x02,0x00]
 
@@ -3161,11 +3215,11 @@ v_cmpx_ngt_f64_e64 -|src_scc|, -|exec|
 v_cmpx_ngt_f64_e64 0xaf123456, -|vcc| clamp
 // GFX11: v_cmpx_ngt_f64_e64 0xaf123456, -|vcc| clamp ; encoding: [0x7e,0x82,0xab,0xd4,0xff,0xd4,0x00,0x40,0x56,0x34,0x12,0xaf]
 
-v_cmpx_nle_f16_e64 v1, v2
-// GFX11: v_cmpx_nle_f16_e64 v1, v2               ; encoding: [0x7e,0x00,0x8c,0xd4,0x01,0x05,0x02,0x00]
+v_cmpx_nle_f16_e64 v1.l, v2.l
+// GFX11: v_cmpx_nle_f16_e64 v1.l, v2.l           ; encoding: [0x7e,0x00,0x8c,0xd4,0x01,0x05,0x02,0x00]
 
-v_cmpx_nle_f16_e64 v255, v255
-// GFX11: v_cmpx_nle_f16_e64 v255, v255           ; encoding: [0x7e,0x00,0x8c,0xd4,0xff,0xff,0x03,0x00]
+v_cmpx_nle_f16_e64 v255.l, v255.l
+// GFX11: v_cmpx_nle_f16_e64 v255.l, v255.l       ; encoding: [0x7e,0x00,0x8c,0xd4,0xff,0xff,0x03,0x00]
 
 v_cmpx_nle_f16_e64 s1, s2
 // GFX11: v_cmpx_nle_f16_e64 s1, s2               ; encoding: [0x7e,0x00,0x8c,0xd4,0x01,0x04,0x00,0x00]
@@ -3206,6 +3260,12 @@ v_cmpx_nle_f16_e64 -src_scc, |vcc_lo|
 v_cmpx_nle_f16_e64 -|0xfe0b|, -|vcc_hi| clamp
 // GFX11: v_cmpx_nle_f16_e64 -|0xfe0b|, -|vcc_hi| clamp ; encoding: [0x7e,0x83,0x8c,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00]
 
+v_cmpx_nle_f16_e64 v1.h, v2.l
+// GFX11: v_cmpx_nle_f16_e64 v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x7e,0x08,0x8c,0xd4,0x01,0x05,0x02,0x00]
+
+v_cmpx_nle_f16_e64 v255.l, v255.h
+// GFX11: v_cmpx_nle_f16_e64 v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x7e,0x10,0x8c,0xd4,0xff,0xff,0x03,0x00]
+
 v_cmpx_nle_f32_e64 v1, v2
 // GFX11: v_cmpx_nle_f32_e64 v1, v2               ; encoding: [0x7e,0x00,0x9c,0xd4,0x01,0x05,0x02,0x00]
 
@@ -3287,11 +3347,11 @@ v_cmpx_nle_f64_e64 -|src_scc|, -|exec|
 v_cmpx_nle_f64_e64 0xaf123456, -|vcc| clamp
 // GFX11: v_cmpx_nle_f64_e64 0xaf123456, -|vcc| clamp ; encoding: [0x7e,0x82,0xac,0xd4,0xff,0xd4,0x00,0x40,0x56,0x34,0x12,0xaf]
 
-v_cmpx_nlg_f16_e64 v1, v2
-// GFX11: v_cmpx_nlg_f16_e64 v1, v2               ; encoding: [0x7e,0x00,0x8a,0xd4,0x01,0x05,0x02,0x00]
+v_cmpx_nlg_f16_e64 v1.l, v2.l
+// GFX11: v_cmpx_nlg_f16_e64 v1.l, v2.l           ; encoding: [0x7e,0x00,0x8a,0xd4,0x01,0x05,0x02,0x00]
 
-v_cmpx_nlg_f16_e64 v255, v255
-// GFX11: v_cmpx_nlg_f16_e64 v255, v255           ; encoding: [0x7e,0x00,0x8a,0xd4,0xff,0xff,0x03,0x00]
+v_cmpx_nlg_f16_e64 v255.l, v255.l
+// GFX11: v_cmpx_nlg_f16_e64 v255.l, v255.l       ; encoding: [0x7e,0x00,0x8a,0xd4,0xff,0xff,0x03,0x00]
 
 v_cmpx_nlg_f16_e64 s1, s2
 // GFX11: v_cmpx_nlg_f16_e64 s1, s2               ; encoding: [0x7e,0x00,0x8a,0xd4,0x01,0x04,0x00,0x00]
@@ -3332,6 +3392,12 @@ v_cmpx_nlg_f16_e64 -src_scc, |vcc_lo|
 v_cmpx_nlg_f16_e64 -|0xfe0b|, -|vcc_hi| clamp
 // GFX11: v_cmpx_nlg_f16_e64 -|0xfe0b|, -|vcc_hi| clamp ; encoding: [0x7e,0x83,0x8a,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00]
 
+v_cmpx_nlg_f16_e64 v1.h, v2.l
+// GFX11: v_cmpx_nlg_f16_e64 v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x7e,0x08,0x8a,0xd4,0x01,0x05,0x02,0x00]
+
+v_cmpx_nlg_f16_e64 v255.l, v255.h
+// GFX11: v_cmpx_nlg_f16_e64 v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x7e,0x10,0x8a,0xd4,0xff,0xff,0x03,0x00]
+
 v_cmpx_nlg_f32_e64 v1, v2
 // GFX11: v_cmpx_nlg_f32_e64 v1, v2               ; encoding: [0x7e,0x00,0x9a,0xd4,0x01,0x05,0x02,0x00]
 
@@ -3413,11 +3479,11 @@ v_cmpx_nlg_f64_e64 -|src_scc|, -|exec|
 v_cmpx_nlg_f64_e64 0xaf123456, -|vcc| clamp
 // GFX11: v_cmpx_nlg_f64_e64 0xaf123456, -|vcc| clamp ; encoding: [0x7e,0x82,0xaa,0xd4,0xff,0xd4,0x00,0x40,0x56,0x34,0x12,0xaf]
 
-v_cmpx_nlt_f16_e64 v1, v2
-// GFX11: v_cmpx_nlt_f16_e64 v1, v2               ; encoding: [0x7e,0x00,0x8e,0xd4,0x01,0x05,0x02,0x00]
+v_cmpx_nlt_f16_e64 v1.l, v2.l
+// GFX11: v_cmpx_nlt_f16_e64 v1.l, v2.l           ; encoding: [0x7e,0x00,0x8e,0xd4,0x01,0x05,0x02,0x00]
 
-v_cmpx_nlt_f16_e64 v255, v255
-// GFX11: v_cmpx_nlt_f16_e64 v255, v255           ; encoding: [0x7e,0x00,0x8e,0xd4,0xff,0xff,0x03,0x00]
+v_cmpx_nlt_f16_e64 v255.l, v255.l
+// GFX11: v_cmpx_nlt_f16_e64 v255.l, v255.l       ; encoding: [0x7e,0x00,0x8e,0xd4,0xff,0xff,0x03,0x00]
 
 v_cmpx_nlt_f16_e64 s1, s2
 // GFX11: v_cmpx_nlt_f16_e64 s1, s2               ; encoding: [0x7e,0x00,0x8e,0xd4,0x01,0x04,0x00,0x00]
@@ -3458,6 +3524,12 @@ v_cmpx_nlt_f16_e64 -src_scc, |vcc_lo|
 v_cmpx_nlt_f16_e64 -|0xfe0b|, -|vcc_hi| clamp
 // GFX11: v_cmpx_nlt_f16_e64 -|0xfe0b|, -|vcc_hi| clamp ; encoding: [0x7e,0x83,0x8e,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00]
 
+v_cmpx_nlt_f16_e64 v1.h, v2.l
+// GFX11: v_cmpx_nlt_f16_e64 v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x7e,0x08,0x8e,0xd4,0x01,0x05,0x02,0x00]
+
+v_cmpx_nlt_f16_e64 v255.l, v255.h
+// GFX11: v_cmpx_nlt_f16_e64 v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x7e,0x10,0x8e,0xd4,0xff,0xff,0x03,0x00]
+
 v_cmpx_nlt_f32_e64 v1, v2
 // GFX11: v_cmpx_nlt_f32_e64 v1, v2               ; encoding: [0x7e,0x00,0x9e,0xd4,0x01,0x05,0x02,0x00]
 
@@ -3539,11 +3611,11 @@ v_cmpx_nlt_f64_e64 -|src_scc|, -|exec|
 v_cmpx_nlt_f64_e64 0xaf123456, -|vcc| clamp
 // GFX11: v_cmpx_nlt_f64_e64 0xaf123456, -|vcc| clamp ; encoding: [0x7e,0x82,0xae,0xd4,0xff,0xd4,0x00,0x40,0x56,0x34,0x12,0xaf]
 
-v_cmpx_o_f16_e64 v1, v2
-// GFX11: v_cmpx_o_f16_e64 v1, v2                 ; encoding: [0x7e,0x00,0x87,0xd4,0x01,0x05,0x02,0x00]
+v_cmpx_o_f16_e64 v1.l, v2.l
+// GFX11: v_cmpx_o_f16_e64 v1.l, v2.l             ; encoding: [0x7e,0x00,0x87,0xd4,0x01,0x05,0x02,0x00]
 
-v_cmpx_o_f16_e64 v255, v255
-// GFX11: v_cmpx_o_f16_e64 v255, v255             ; encoding: [0x7e,0x00,0x87,0xd4,0xff,0xff,0x03,0x00]
+v_cmpx_o_f16_e64 v255.l, v255.l
+// GFX11: v_cmpx_o_f16_e64 v255.l, v255.l         ; encoding: [0x7e,0x00,0x87,0xd4,0xff,0xff,0x03,0x00]
 
 v_cmpx_o_f16_e64 s1, s2
 // GFX11: v_cmpx_o_f16_e64 s1, s2                 ; encoding: [0x7e,0x00,0x87,0xd4,0x01,0x04,0x00,0x00]
@@ -3584,6 +3656,12 @@ v_cmpx_o_f16_e64 -src_scc, |vcc_lo|
 v_cmpx_o_f16_e64 -|0xfe0b|, -|vcc_hi| clamp
 // GFX11: v_cmpx_o_f16_e64 -|0xfe0b|, -|vcc_hi| clamp ; encoding: [0x7e,0x83,0x87,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00]
 
+v_cmpx_o_f16_e64 v1.h, v2.l
+// GFX11: v_cmpx_o_f16_e64 v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x7e,0x08,0x87,0xd4,0x01,0x05,0x02,0x00]
+
+v_cmpx_o_f16_e64 v255.l, v255.h
+// GFX11: v_cmpx_o_f16_e64 v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x7e,0x10,0x87,0xd4,0xff,0xff,0x03,0x00]
+
 v_cmpx_o_f32_e64 v1, v2
 // GFX11: v_cmpx_o_f32_e64 v1, v2                 ; encoding: [0x7e,0x00,0x97,0xd4,0x01,0x05,0x02,0x00]
 
@@ -3665,11 +3743,11 @@ v_cmpx_o_f64_e64 -|src_scc|, -|exec|
 v_cmpx_o_f64_e64 0xaf123456, -|vcc| clamp
 // GFX11: v_cmpx_o_f64_e64 0xaf123456, -|vcc| clamp ; encoding: [0x7e,0x82,0xa7,0xd4,0xff,0xd4,0x00,0x40,0x56,0x34,0x12,0xaf]
 
-v_cmpx_t_f16_e64 v1, v2
-// GFX11: v_cmpx_t_f16_e64 v1, v2                 ; encoding: [0x7e,0x00,0x8f,0xd4,0x01,0x05,0x02,0x00]
+v_cmpx_t_f16_e64 v1.l, v2.l
+// GFX11: v_cmpx_t_f16_e64 v1.l, v2.l             ; encoding: [0x7e,0x00,0x8f,0xd4,0x01,0x05,0x02,0x00]
 
-v_cmpx_t_f16_e64 v255, v255
-// GFX11: v_cmpx_t_f16_e64 v255, v255             ; encoding: [0x7e,0x00,0x8f,0xd4,0xff,0xff,0x03,0x00]
+v_cmpx_t_f16_e64 v255.l, v255.l
+// GFX11: v_cmpx_t_f16_e64 v255.l, v255.l         ; encoding: [0x7e,0x00,0x8f,0xd4,0xff,0xff,0x03,0x00]
 
 v_cmpx_t_f16_e64 s1, s2
 // GFX11: v_cmpx_t_f16_e64 s1, s2                 ; encoding: [0x7e,0x00,0x8f,0xd4,0x01,0x04,0x00,0x00]
@@ -3953,11 +4031,11 @@ v_cmpx_t_u64_e64 src_scc, exec
 v_cmpx_t_u64_e64 0xaf123456, vcc
 // GFX11: v_cmpx_t_u64_e64 0xaf123456, vcc        ; encoding: [0x7e,0x00,0xdf,0xd4,0xff,0xd4,0x00,0x00,0x56,0x34,0x12,0xaf]
 
-v_cmpx_tru_f16_e64 v1, v2
-// GFX11: v_cmpx_t_f16_e64 v1, v2                 ; encoding: [0x7e,0x00,0x8f,0xd4,0x01,0x05,0x02,0x00]
+v_cmpx_tru_f16_e64 v1.l, v2.l
+// GFX11: v_cmpx_t_f16_e64 v1.l, v2.l             ; encoding: [0x7e,0x00,0x8f,0xd4,0x01,0x05,0x02,0x00]
 
-v_cmpx_tru_f16_e64 v255, v255
-// GFX11: v_cmpx_t_f16_e64 v255, v255             ; encoding: [0x7e,0x00,0x8f,0xd4,0xff,0xff,0x03,0x00]
+v_cmpx_tru_f16_e64 v255.l, v255.l
+// GFX11: v_cmpx_t_f16_e64 v255.l, v255.l         ; encoding: [0x7e,0x00,0x8f,0xd4,0xff,0xff,0x03,0x00]
 
 v_cmpx_tru_f16_e64 s1, s2
 // GFX11: v_cmpx_t_f16_e64 s1, s2                 ; encoding: [0x7e,0x00,0x8f,0xd4,0x01,0x04,0x00,0x00]
@@ -3998,6 +4076,12 @@ v_cmpx_tru_f16_e64 -src_scc, |vcc_lo|
 v_cmpx_tru_f16_e64 -|0xfe0b|, -|vcc_hi| clamp
 // GFX11: v_cmpx_t_f16_e64 -|0xfe0b|, -|vcc_hi| clamp ; encoding: [0x7e,0x83,0x8f,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00]
 
+v_cmpx_t_f16_e64 v1.h, v2.l
+// GFX11: v_cmpx_t_f16_e64 v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x7e,0x08,0x8f,0xd4,0x01,0x05,0x02,0x00]
+
+v_cmpx_t_f16_e64 v255.l, v255.h
+// GFX11: v_cmpx_t_f16_e64 v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x7e,0x10,0x8f,0xd4,0xff,0xff,0x03,0x00]
+
 v_cmpx_tru_f32_e64 v1, v2
 // GFX11: v_cmpx_t_f32_e64 v1, v2                 ; encoding: [0x7e,0x00,0x9f,0xd4,0x01,0x05,0x02,0x00]
 
@@ -4079,11 +4163,11 @@ v_cmpx_tru_f64_e64 -|src_scc|, -|exec|
 v_cmpx_tru_f64_e64 0xaf123456, -|vcc| clamp
 // GFX11: v_cmpx_t_f64_e64 0xaf123456, -|vcc| clamp ; encoding: [0x7e,0x82,0xaf,0xd4,0xff,0xd4,0x00,0x40,0x56,0x34,0x12,0xaf]
 
-v_cmpx_u_f16_e64 v1, v2
-// GFX11: v_cmpx_u_f16_e64 v1, v2                 ; encoding: [0x7e,0x00,0x88,0xd4,0x01,0x05,0x02,0x00]
+v_cmpx_u_f16_e64 v1.l, v2.l
+// GFX11: v_cmpx_u_f16_e64 v1.l, v2.l             ; encoding: [0x7e,0x00,0x88,0xd4,0x01,0x05,0x02,0x00]
 
-v_cmpx_u_f16_e64 v255, v255
-// GFX11: v_cmpx_u_f16_e64 v255, v255             ; encoding: [0x7e,0x00,0x88,0xd4,0xff,0xff,0x03,0x00]
+v_cmpx_u_f16_e64 v255.l, v255.l
+// GFX11: v_cmpx_u_f16_e64 v255.l, v255.l         ; encoding: [0x7e,0x00,0x88,0xd4,0xff,0xff,0x03,0x00]
 
 v_cmpx_u_f16_e64 s1, s2
 // GFX11: v_cmpx_u_f16_e64 s1, s2                 ; encoding: [0x7e,0x00,0x88,0xd4,0x01,0x04,0x00,0x00]
@@ -4124,6 +4208,12 @@ v_cmpx_u_f16_e64 -src_scc, |vcc_lo|
 v_cmpx_u_f16_e64 -|0xfe0b|, -|vcc_hi| clamp
 // GFX11: v_cmpx_u_f16_e64 -|0xfe0b|, -|vcc_hi| clamp ; encoding: [0x7e,0x83,0x88,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00]
 
+v_cmpx_u_f16_e64 v1.h, v2.l
+// GFX11: v_cmpx_u_f16_e64 v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x7e,0x08,0x88,0xd4,0x01,0x05,0x02,0x00]
+
+v_cmpx_u_f16_e64 v255.l, v255.h
+// GFX11: v_cmpx_u_f16_e64 v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x7e,0x10,0x88,0xd4,0xff,0xff,0x03,0x00]
+
 v_cmpx_u_f32_e64 v1, v2
 // GFX11: v_cmpx_u_f32_e64 v1, v2                 ; encoding: [0x7e,0x00,0x98,0xd4,0x01,0x05,0x02,0x00]
 
diff --git a/llvm/test/MC/AMDGPU/gfx11_asm_vopc_t16_promote.s b/llvm/test/MC/AMDGPU/gfx11_asm_vopc_t16_promote.s
index d3eba050d6be7..e821a3bf68f28 100644
--- a/llvm/test/MC/AMDGPU/gfx11_asm_vopc_t16_promote.s
+++ b/llvm/test/MC/AMDGPU/gfx11_asm_vopc_t16_promote.s
@@ -2,7 +2,7 @@
 // RUN: llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize64,+real-true16 -show-encoding %s 2>&1 | FileCheck --check-prefix=GFX11 %s
 
 v_cmp_class_f16 vcc, v1.h, v255.h
-// GFX11: v_cmp_class_f16_e64 vcc, v1.h, v255.h   ; encoding: [0x6a,0x18,0x7d,0xd4,0x01,0xff,0x03,0x00]
+// GFX11: v_cmp_class_f16_e64 vcc, v1.h, v255.h op_sel:[1,1,0] ; encoding: [0x6a,0x18,0x7d,0xd4,0x01,0xff,0x03,0x00]
 
 v_cmp_class_f16 vcc, v1.h, v255.h dpp8:[7,6,5,4,3,2,1,0]
 // GFX11: v_cmp_class_f16_e64_dpp vcc, v1.h, v255.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x18,0x7d,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
@@ -20,10 +20,10 @@ v_cmp_class_f16 vcc, v1.l, v255.l quad_perm:[3,2,1,0]
 // GFX11: v_cmp_class_f16_e64_dpp vcc, v1.l, v255.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x7d,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
 
 v_cmp_class_f16 vcc, v127.h, v255.h
-// GFX11: v_cmp_class_f16_e64 vcc, v127.h, v255.h ; encoding: [0x6a,0x18,0x7d,0xd4,0x7f,0xff,0x03,0x00]
+// GFX11: v_cmp_class_f16_e64 vcc, v127.h, v255.h op_sel:[1,1,0] ; encoding: [0x6a,0x18,0x7d,0xd4,0x7f,0xff,0x03,0x00]
 
 v_cmp_class_f16 vcc, v127.h, v255.h
-// GFX11: v_cmp_class_f16_e64 vcc, v127.h, v255.h ; encoding: [0x6a,0x18,0x7d,0xd4,0x7f,0xff,0x03,0x00]
+// GFX11: v_cmp_class_f16_e64 vcc, v127.h, v255.h op_sel:[1,1,0] ; encoding: [0x6a,0x18,0x7d,0xd4,0x7f,0xff,0x03,0x00]
 
 v_cmp_class_f16 vcc, v127.h, v255.h dpp8:[7,6,5,4,3,2,1,0]
 // GFX11: v_cmp_class_f16_e64_dpp vcc, v127.h, v255.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x18,0x7d,0xd4,0xe9,0xfe,0x03,0x00,0x7f,0x77,0x39,0x05]
@@ -56,10 +56,10 @@ v_cmp_class_f16 vcc, v127.l, v255.l quad_perm:[3,2,1,0]
 // GFX11: v_cmp_class_f16_e64_dpp vcc, v127.l, v255.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x7d,0xd4,0xfa,0xfe,0x03,0x00,0x7f,0x1b,0x00,0xff]
 
 v_cmp_class_f16 vcc, v128.h, v2.h
-// GFX11: v_cmp_class_f16_e64 vcc, v128.h, v2.h   ; encoding: [0x6a,0x18,0x7d,0xd4,0x80,0x05,0x02,0x00]
+// GFX11: v_cmp_class_f16_e64 vcc, v128.h, v2.h op_sel:[1,1,0] ; encoding: [0x6a,0x18,0x7d,0xd4,0x80,0x05,0x02,0x00]
 
 v_cmp_class_f16 vcc, v128.h, v2.h
-// GFX11: v_cmp_class_f16_e64 vcc, v128.h, v2.h   ; encoding: [0x6a,0x18,0x7d,0xd4,0x80,0x05,0x02,0x00]
+// GFX11: v_cmp_class_f16_e64 vcc, v128.h, v2.h op_sel:[1,1,0] ; encoding: [0x6a,0x18,0x7d,0xd4,0x80,0x05,0x02,0x00]
 
 v_cmp_class_f16 vcc, v128.h, v2.h dpp8:[7,6,5,4,3,2,1,0]
 // GFX11: v_cmp_class_f16_e64_dpp vcc, v128.h, v2.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x18,0x7d,0xd4,0xe9,0x04,0x02,0x00,0x80,0x77,0x39,0x05]
@@ -92,10 +92,10 @@ v_cmp_class_f16 vcc, v128.l, v2.l quad_perm:[3,2,1,0]
 // GFX11: v_cmp_class_f16_e64_dpp vcc, v128.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x7d,0xd4,0xfa,0x04,0x02,0x00,0x80,0x1b,0x00,0xff]
 
 v_cmp_class_f16 vcc, vcc_hi, v255.h
-// GFX11: v_cmp_class_f16_e64 vcc, vcc_hi, v255.h ; encoding: [0x6a,0x10,0x7d,0xd4,0x6b,0xfe,0x03,0x00]
+// GFX11: v_cmp_class_f16_e64 vcc, vcc_hi, v255.h op_sel:[0,1,0] ; encoding: [0x6a,0x10,0x7d,0xd4,0x6b,0xfe,0x03,0x00]
 
 v_cmp_class_f16 vcc, vcc_hi, v255.h
-// GFX11: v_cmp_class_f16_e64 vcc, vcc_hi, v255.h ; encoding: [0x6a,0x10,0x7d,0xd4,0x6b,0xfe,0x03,0x00]
+// GFX11: v_cmp_class_f16_e64 vcc, vcc_hi, v255.h op_sel:[0,1,0] ; encoding: [0x6a,0x10,0x7d,0xd4,0x6b,0xfe,0x03,0x00]
 
 v_cmp_class_f16 vcc, vcc_hi, v255.l
 // GFX11: v_cmp_class_f16_e64 vcc, vcc_hi, v255.l ; encoding: [0x6a,0x00,0x7d,0xd4,0x6b,0xfe,0x03,0x00]
@@ -104,10 +104,10 @@ v_cmp_class_f16 vcc, vcc_hi, v255.l
 // GFX11: v_cmp_class_f16_e64 vcc, vcc_hi, v255.l ; encoding: [0x6a,0x00,0x7d,0xd4,0x6b,0xfe,0x03,0x00]
 
 v_cmp_class_f16 vcc, vcc_lo, v255.h
-// GFX11: v_cmp_class_f16_e64 vcc, vcc_lo, v255.h ; encoding: [0x6a,0x10,0x7d,0xd4,0x6a,0xfe,0x03,0x00]
+// GFX11: v_cmp_class_f16_e64 vcc, vcc_lo, v255.h op_sel:[0,1,0] ; encoding: [0x6a,0x10,0x7d,0xd4,0x6a,0xfe,0x03,0x00]
 
 v_cmp_class_f16 vcc, vcc_lo, v255.h
-// GFX11: v_cmp_class_f16_e64 vcc, vcc_lo, v255.h ; encoding: [0x6a,0x10,0x7d,0xd4,0x6a,0xfe,0x03,0x00]
+// GFX11: v_cmp_class_f16_e64 vcc, vcc_lo, v255.h op_sel:[0,1,0] ; encoding: [0x6a,0x10,0x7d,0xd4,0x6a,0xfe,0x03,0x00]
 
 v_cmp_class_f16 vcc, vcc_lo, v255.l
 // GFX11: v_cmp_class_f16_e64 vcc, vcc_lo, v255.l ; encoding: [0x6a,0x00,0x7d,0xd4,0x6a,0xfe,0x03,0x00]
diff --git a/llvm/test/MC/AMDGPU/gfx11_asm_vopcx.s b/llvm/test/MC/AMDGPU/gfx11_asm_vopcx.s
index cdad89321d89a..98aba2b960ad9 100644
--- a/llvm/test/MC/AMDGPU/gfx11_asm_vopcx.s
+++ b/llvm/test/MC/AMDGPU/gfx11_asm_vopcx.s
@@ -143,50 +143,65 @@ v_cmpx_class_f64 src_scc, v2
 v_cmpx_class_f64 0xaf123456, v255
 // GFX11: v_cmpx_class_f64_e32 0xaf123456, v255   ; encoding: [0xff,0xfe,0xff,0x7d,0x56,0x34,0x12,0xaf]
 
-v_cmpx_eq_f16 v1, v2
-// GFX11: v_cmpx_eq_f16_e32 v1, v2                ; encoding: [0x01,0x05,0x04,0x7d]
+v_cmpx_eq_f16 v1.l, v2.l
+// GFX11: v_cmpx_eq_f16_e32 v1.l, v2.l            ; encoding: [0x01,0x05,0x04,0x7d]
 
-v_cmpx_eq_f16 v127, v2
-// GFX11: v_cmpx_eq_f16_e32 v127, v2              ; encoding: [0x7f,0x05,0x04,0x7d]
+v_cmpx_eq_f16 v127.l, v2.l
+// GFX11: v_cmpx_eq_f16_e32 v127.l, v2.l          ; encoding: [0x7f,0x05,0x04,0x7d]
 
-v_cmpx_eq_f16 s1, v2
-// GFX11: v_cmpx_eq_f16_e32 s1, v2                ; encoding: [0x01,0x04,0x04,0x7d]
+v_cmpx_eq_f16 s1, v2.l
+// GFX11: v_cmpx_eq_f16_e32 s1, v2.l              ; encoding: [0x01,0x04,0x04,0x7d]
 
-v_cmpx_eq_f16 s105, v2
-// GFX11: v_cmpx_eq_f16_e32 s105, v2              ; encoding: [0x69,0x04,0x04,0x7d]
+v_cmpx_eq_f16 s105, v2.l
+// GFX11: v_cmpx_eq_f16_e32 s105, v2.l            ; encoding: [0x69,0x04,0x04,0x7d]
 
-v_cmpx_eq_f16 vcc_lo, v2
-// GFX11: v_cmpx_eq_f16_e32 vcc_lo, v2            ; encoding: [0x6a,0x04,0x04,0x7d]
+v_cmpx_eq_f16 vcc_lo, v2.l
+// GFX11: v_cmpx_eq_f16_e32 vcc_lo, v2.l          ; encoding: [0x6a,0x04,0x04,0x7d]
 
-v_cmpx_eq_f16 vcc_hi, v2
-// GFX11: v_cmpx_eq_f16_e32 vcc_hi, v2            ; encoding: [0x6b,0x04,0x04,0x7d]
+v_cmpx_eq_f16 vcc_hi, v2.l
+// GFX11: v_cmpx_eq_f16_e32 vcc_hi, v2.l          ; encoding: [0x6b,0x04,0x04,0x7d]
 
-v_cmpx_eq_f16 ttmp15, v2
-// GFX11: v_cmpx_eq_f16_e32 ttmp15, v2            ; encoding: [0x7b,0x04,0x04,0x7d]
+v_cmpx_eq_f16 ttmp15, v2.l
+// GFX11: v_cmpx_eq_f16_e32 ttmp15, v2.l          ; encoding: [0x7b,0x04,0x04,0x7d]
 
-v_cmpx_eq_f16 m0, v2
-// GFX11: v_cmpx_eq_f16_e32 m0, v2                ; encoding: [0x7d,0x04,0x04,0x7d]
+v_cmpx_eq_f16 m0, v2.l
+// GFX11: v_cmpx_eq_f16_e32 m0, v2.l              ; encoding: [0x7d,0x04,0x04,0x7d]
 
-v_cmpx_eq_f16 exec_lo, v2
-// GFX11: v_cmpx_eq_f16_e32 exec_lo, v2           ; encoding: [0x7e,0x04,0x04,0x7d]
+v_cmpx_eq_f16 exec_lo, v2.l
+// GFX11: v_cmpx_eq_f16_e32 exec_lo, v2.l         ; encoding: [0x7e,0x04,0x04,0x7d]
 
-v_cmpx_eq_f16 exec_hi, v2
-// GFX11: v_cmpx_eq_f16_e32 exec_hi, v2           ; encoding: [0x7f,0x04,0x04,0x7d]
+v_cmpx_eq_f16 exec_hi, v2.l
+// GFX11: v_cmpx_eq_f16_e32 exec_hi, v2.l         ; encoding: [0x7f,0x04,0x04,0x7d]
 
-v_cmpx_eq_f16 null, v2
-// GFX11: v_cmpx_eq_f16_e32 null, v2              ; encoding: [0x7c,0x04,0x04,0x7d]
+v_cmpx_eq_f16 null, v2.l
+// GFX11: v_cmpx_eq_f16_e32 null, v2.l            ; encoding: [0x7c,0x04,0x04,0x7d]
 
-v_cmpx_eq_f16 -1, v2
-// GFX11: v_cmpx_eq_f16_e32 -1, v2                ; encoding: [0xc1,0x04,0x04,0x7d]
+v_cmpx_eq_f16 -1, v2.l
+// GFX11: v_cmpx_eq_f16_e32 -1, v2.l              ; encoding: [0xc1,0x04,0x04,0x7d]
 
-v_cmpx_eq_f16 0.5, v2
-// GFX11: v_cmpx_eq_f16_e32 0.5, v2               ; encoding: [0xf0,0x04,0x04,0x7d]
+v_cmpx_eq_f16 0.5, v2.l
+// GFX11: v_cmpx_eq_f16_e32 0.5, v2.l             ; encoding: [0xf0,0x04,0x04,0x7d]
 
-v_cmpx_eq_f16 src_scc, v2
-// GFX11: v_cmpx_eq_f16_e32 src_scc, v2           ; encoding: [0xfd,0x04,0x04,0x7d]
+v_cmpx_eq_f16 src_scc, v2.l
+// GFX11: v_cmpx_eq_f16_e32 src_scc, v2.l         ; encoding: [0xfd,0x04,0x04,0x7d]
 
-v_cmpx_eq_f16 0xfe0b, v127
-// GFX11: v_cmpx_eq_f16_e32 0xfe0b, v127          ; encoding: [0xff,0xfe,0x04,0x7d,0x0b,0xfe,0x00,0x00]
+v_cmpx_eq_f16 0xfe0b, v127.l
+// GFX11: v_cmpx_eq_f16_e32 0xfe0b, v127.l        ; encoding: [0xff,0xfe,0x04,0x7d,0x0b,0xfe,0x00,0x00]
+
+v_cmpx_eq_f16 v1.h, v2.l
+// GFX11: v_cmpx_eq_f16_e32 v1.h, v2.l            ; encoding: [0x81,0x05,0x04,0x7d]
+
+v_cmpx_eq_f16 v127.h, v2.l
+// GFX11: v_cmpx_eq_f16_e32 v127.h, v2.l          ; encoding: [0xff,0x05,0x04,0x7d]
+
+v_cmpx_eq_f16 0.5, v127.l
+// GFX11: v_cmpx_eq_f16_e32 0.5, v127.l           ; encoding: [0xf0,0xfe,0x04,0x7d]
+
+v_cmpx_eq_f16 src_scc, v2.h
+// GFX11: v_cmpx_eq_f16_e32 src_scc, v2.h         ; encoding: [0xfd,0x04,0x05,0x7d]
+
+v_cmpx_eq_f16 0xfe0b, v127.h
+// GFX11: v_cmpx_eq_f16_e32 0xfe0b, v127.h        ; encoding: [0xff,0xfe,0x05,0x7d,0x0b,0xfe,0x00,0x00]
 
 v_cmpx_eq_f32 v1, v2
 // GFX11: v_cmpx_eq_f32_e32 v1, v2                ; encoding: [0x01,0x05,0x24,0x7d]
@@ -551,50 +566,65 @@ v_cmpx_eq_u64 src_scc, v[2:3]
 v_cmpx_eq_u64 0xaf123456, v[254:255]
 // GFX11: v_cmpx_eq_u64_e32 0xaf123456, v[254:255] ; encoding: [0xff,0xfc,0xb5,0x7d,0x56,0x34,0x12,0xaf]
 
-v_cmpx_f_f16 v1, v2
-// GFX11: v_cmpx_f_f16_e32 v1, v2                 ; encoding: [0x01,0x05,0x00,0x7d]
+v_cmpx_f_f16 v1.l, v2.l
+// GFX11: v_cmpx_f_f16_e32 v1.l, v2.l             ; encoding: [0x01,0x05,0x00,0x7d]
+
+v_cmpx_f_f16 v127.l, v2.l
+// GFX11: v_cmpx_f_f16_e32 v127.l, v2.l           ; encoding: [0x7f,0x05,0x00,0x7d]
 
-v_cmpx_f_f16 v127, v2
-// GFX11: v_cmpx_f_f16_e32 v127, v2               ; encoding: [0x7f,0x05,0x00,0x7d]
+v_cmpx_f_f16 s1, v2.l
+// GFX11: v_cmpx_f_f16_e32 s1, v2.l               ; encoding: [0x01,0x04,0x00,0x7d]
 
-v_cmpx_f_f16 s1, v2
-// GFX11: v_cmpx_f_f16_e32 s1, v2                 ; encoding: [0x01,0x04,0x00,0x7d]
+v_cmpx_f_f16 s105, v2.l
+// GFX11: v_cmpx_f_f16_e32 s105, v2.l             ; encoding: [0x69,0x04,0x00,0x7d]
 
-v_cmpx_f_f16 s105, v2
-// GFX11: v_cmpx_f_f16_e32 s105, v2               ; encoding: [0x69,0x04,0x00,0x7d]
+v_cmpx_f_f16 vcc_lo, v2.l
+// GFX11: v_cmpx_f_f16_e32 vcc_lo, v2.l           ; encoding: [0x6a,0x04,0x00,0x7d]
 
-v_cmpx_f_f16 vcc_lo, v2
-// GFX11: v_cmpx_f_f16_e32 vcc_lo, v2             ; encoding: [0x6a,0x04,0x00,0x7d]
+v_cmpx_f_f16 vcc_hi, v2.l
+// GFX11: v_cmpx_f_f16_e32 vcc_hi, v2.l           ; encoding: [0x6b,0x04,0x00,0x7d]
 
-v_cmpx_f_f16 vcc_hi, v2
-// GFX11: v_cmpx_f_f16_e32 vcc_hi, v2             ; encoding: [0x6b,0x04,0x00,0x7d]
+v_cmpx_f_f16 ttmp15, v2.l
+// GFX11: v_cmpx_f_f16_e32 ttmp15, v2.l           ; encoding: [0x7b,0x04,0x00,0x7d]
 
-v_cmpx_f_f16 ttmp15, v2
-// GFX11: v_cmpx_f_f16_e32 ttmp15, v2             ; encoding: [0x7b,0x04,0x00,0x7d]
+v_cmpx_f_f16 m0, v2.l
+// GFX11: v_cmpx_f_f16_e32 m0, v2.l               ; encoding: [0x7d,0x04,0x00,0x7d]
 
-v_cmpx_f_f16 m0, v2
-// GFX11: v_cmpx_f_f16_e32 m0, v2                 ; encoding: [0x7d,0x04,0x00,0x7d]
+v_cmpx_f_f16 exec_lo, v2.l
+// GFX11: v_cmpx_f_f16_e32 exec_lo, v2.l          ; encoding: [0x7e,0x04,0x00,0x7d]
 
-v_cmpx_f_f16 exec_lo, v2
-// GFX11: v_cmpx_f_f16_e32 exec_lo, v2            ; encoding: [0x7e,0x04,0x00,0x7d]
+v_cmpx_f_f16 exec_hi, v2.l
+// GFX11: v_cmpx_f_f16_e32 exec_hi, v2.l          ; encoding: [0x7f,0x04,0x00,0x7d]
 
-v_cmpx_f_f16 exec_hi, v2
-// GFX11: v_cmpx_f_f16_e32 exec_hi, v2            ; encoding: [0x7f,0x04,0x00,0x7d]
+v_cmpx_f_f16 null, v2.l
+// GFX11: v_cmpx_f_f16_e32 null, v2.l             ; encoding: [0x7c,0x04,0x00,0x7d]
 
-v_cmpx_f_f16 null, v2
-// GFX11: v_cmpx_f_f16_e32 null, v2               ; encoding: [0x7c,0x04,0x00,0x7d]
+v_cmpx_f_f16 -1, v2.l
+// GFX11: v_cmpx_f_f16_e32 -1, v2.l               ; encoding: [0xc1,0x04,0x00,0x7d]
 
-v_cmpx_f_f16 -1, v2
-// GFX11: v_cmpx_f_f16_e32 -1, v2                 ; encoding: [0xc1,0x04,0x00,0x7d]
+v_cmpx_f_f16 0.5, v2.l
+// GFX11: v_cmpx_f_f16_e32 0.5, v2.l              ; encoding: [0xf0,0x04,0x00,0x7d]
 
-v_cmpx_f_f16 0.5, v2
-// GFX11: v_cmpx_f_f16_e32 0.5, v2                ; encoding: [0xf0,0x04,0x00,0x7d]
+v_cmpx_f_f16 src_scc, v2.l
+// GFX11: v_cmpx_f_f16_e32 src_scc, v2.l          ; encoding: [0xfd,0x04,0x00,0x7d]
 
-v_cmpx_f_f16 src_scc, v2
-// GFX11: v_cmpx_f_f16_e32 src_scc, v2            ; encoding: [0xfd,0x04,0x00,0x7d]
+v_cmpx_f_f16 0xfe0b, v127.l
+// GFX11: v_cmpx_f_f16_e32 0xfe0b, v127.l         ; encoding: [0xff,0xfe,0x00,0x7d,0x0b,0xfe,0x00,0x00]
 
-v_cmpx_f_f16 0xfe0b, v127
-// GFX11: v_cmpx_f_f16_e32 0xfe0b, v127           ; encoding: [0xff,0xfe,0x00,0x7d,0x0b,0xfe,0x00,0x00]
+v_cmpx_f_f16 v1.h, v2.l
+// GFX11: v_cmpx_f_f16_e32 v1.h, v2.l             ; encoding: [0x81,0x05,0x00,0x7d]
+
+v_cmpx_f_f16 v127.h, v2.l
+// GFX11: v_cmpx_f_f16_e32 v127.h, v2.l           ; encoding: [0xff,0x05,0x00,0x7d]
+
+v_cmpx_f_f16 0.5, v127.l
+// GFX11: v_cmpx_f_f16_e32 0.5, v127.l            ; encoding: [0xf0,0xfe,0x00,0x7d]
+
+v_cmpx_f_f16 src_scc, v2.h
+// GFX11: v_cmpx_f_f16_e32 src_scc, v2.h          ; encoding: [0xfd,0x04,0x01,0x7d]
+
+v_cmpx_f_f16 0xfe0b, v127.h
+// GFX11: v_cmpx_f_f16_e32 0xfe0b, v127.h         ; encoding: [0xff,0xfe,0x01,0x7d,0x0b,0xfe,0x00,0x00]
 
 v_cmpx_f_f32 v1, v2
 // GFX11: v_cmpx_f_f32_e32 v1, v2                 ; encoding: [0x01,0x05,0x20,0x7d]
@@ -839,50 +869,65 @@ v_cmpx_f_u64 src_scc, v[2:3]
 v_cmpx_f_u64 0xaf123456, v[254:255]
 // GFX11: v_cmpx_f_u64_e32 0xaf123456, v[254:255] ; encoding: [0xff,0xfc,0xb1,0x7d,0x56,0x34,0x12,0xaf]
 
-v_cmpx_ge_f16 v1, v2
-// GFX11: v_cmpx_ge_f16_e32 v1, v2                ; encoding: [0x01,0x05,0x0c,0x7d]
+v_cmpx_ge_f16 v1.l, v2.l
+// GFX11: v_cmpx_ge_f16_e32 v1.l, v2.l            ; encoding: [0x01,0x05,0x0c,0x7d]
+
+v_cmpx_ge_f16 v127.l, v2.l
+// GFX11: v_cmpx_ge_f16_e32 v127.l, v2.l          ; encoding: [0x7f,0x05,0x0c,0x7d]
 
-v_cmpx_ge_f16 v127, v2
-// GFX11: v_cmpx_ge_f16_e32 v127, v2              ; encoding: [0x7f,0x05,0x0c,0x7d]
+v_cmpx_ge_f16 s1, v2.l
+// GFX11: v_cmpx_ge_f16_e32 s1, v2.l              ; encoding: [0x01,0x04,0x0c,0x7d]
 
-v_cmpx_ge_f16 s1, v2
-// GFX11: v_cmpx_ge_f16_e32 s1, v2                ; encoding: [0x01,0x04,0x0c,0x7d]
+v_cmpx_ge_f16 s105, v2.l
+// GFX11: v_cmpx_ge_f16_e32 s105, v2.l            ; encoding: [0x69,0x04,0x0c,0x7d]
 
-v_cmpx_ge_f16 s105, v2
-// GFX11: v_cmpx_ge_f16_e32 s105, v2              ; encoding: [0x69,0x04,0x0c,0x7d]
+v_cmpx_ge_f16 vcc_lo, v2.l
+// GFX11: v_cmpx_ge_f16_e32 vcc_lo, v2.l          ; encoding: [0x6a,0x04,0x0c,0x7d]
 
-v_cmpx_ge_f16 vcc_lo, v2
-// GFX11: v_cmpx_ge_f16_e32 vcc_lo, v2            ; encoding: [0x6a,0x04,0x0c,0x7d]
+v_cmpx_ge_f16 vcc_hi, v2.l
+// GFX11: v_cmpx_ge_f16_e32 vcc_hi, v2.l          ; encoding: [0x6b,0x04,0x0c,0x7d]
 
-v_cmpx_ge_f16 vcc_hi, v2
-// GFX11: v_cmpx_ge_f16_e32 vcc_hi, v2            ; encoding: [0x6b,0x04,0x0c,0x7d]
+v_cmpx_ge_f16 ttmp15, v2.l
+// GFX11: v_cmpx_ge_f16_e32 ttmp15, v2.l          ; encoding: [0x7b,0x04,0x0c,0x7d]
 
-v_cmpx_ge_f16 ttmp15, v2
-// GFX11: v_cmpx_ge_f16_e32 ttmp15, v2            ; encoding: [0x7b,0x04,0x0c,0x7d]
+v_cmpx_ge_f16 m0, v2.l
+// GFX11: v_cmpx_ge_f16_e32 m0, v2.l              ; encoding: [0x7d,0x04,0x0c,0x7d]
 
-v_cmpx_ge_f16 m0, v2
-// GFX11: v_cmpx_ge_f16_e32 m0, v2                ; encoding: [0x7d,0x04,0x0c,0x7d]
+v_cmpx_ge_f16 exec_lo, v2.l
+// GFX11: v_cmpx_ge_f16_e32 exec_lo, v2.l         ; encoding: [0x7e,0x04,0x0c,0x7d]
 
-v_cmpx_ge_f16 exec_lo, v2
-// GFX11: v_cmpx_ge_f16_e32 exec_lo, v2           ; encoding: [0x7e,0x04,0x0c,0x7d]
+v_cmpx_ge_f16 exec_hi, v2.l
+// GFX11: v_cmpx_ge_f16_e32 exec_hi, v2.l         ; encoding: [0x7f,0x04,0x0c,0x7d]
 
-v_cmpx_ge_f16 exec_hi, v2
-// GFX11: v_cmpx_ge_f16_e32 exec_hi, v2           ; encoding: [0x7f,0x04,0x0c,0x7d]
+v_cmpx_ge_f16 null, v2.l
+// GFX11: v_cmpx_ge_f16_e32 null, v2.l            ; encoding: [0x7c,0x04,0x0c,0x7d]
 
-v_cmpx_ge_f16 null, v2
-// GFX11: v_cmpx_ge_f16_e32 null, v2              ; encoding: [0x7c,0x04,0x0c,0x7d]
+v_cmpx_ge_f16 -1, v2.l
+// GFX11: v_cmpx_ge_f16_e32 -1, v2.l              ; encoding: [0xc1,0x04,0x0c,0x7d]
 
-v_cmpx_ge_f16 -1, v2
-// GFX11: v_cmpx_ge_f16_e32 -1, v2                ; encoding: [0xc1,0x04,0x0c,0x7d]
+v_cmpx_ge_f16 0.5, v2.l
+// GFX11: v_cmpx_ge_f16_e32 0.5, v2.l             ; encoding: [0xf0,0x04,0x0c,0x7d]
 
-v_cmpx_ge_f16 0.5, v2
-// GFX11: v_cmpx_ge_f16_e32 0.5, v2               ; encoding: [0xf0,0x04,0x0c,0x7d]
+v_cmpx_ge_f16 src_scc, v2.l
+// GFX11: v_cmpx_ge_f16_e32 src_scc, v2.l         ; encoding: [0xfd,0x04,0x0c,0x7d]
 
-v_cmpx_ge_f16 src_scc, v2
-// GFX11: v_cmpx_ge_f16_e32 src_scc, v2           ; encoding: [0xfd,0x04,0x0c,0x7d]
+v_cmpx_ge_f16 0xfe0b, v127.l
+// GFX11: v_cmpx_ge_f16_e32 0xfe0b, v127.l        ; encoding: [0xff,0xfe,0x0c,0x7d,0x0b,0xfe,0x00,0x00]
 
-v_cmpx_ge_f16 0xfe0b, v127
-// GFX11: v_cmpx_ge_f16_e32 0xfe0b, v127          ; encoding: [0xff,0xfe,0x0c,0x7d,0x0b,0xfe,0x00,0x00]
+v_cmpx_ge_f16 v1.h, v2.l
+// GFX11: v_cmpx_ge_f16_e32 v1.h, v2.l            ; encoding: [0x81,0x05,0x0c,0x7d]
+
+v_cmpx_ge_f16 v127.h, v2.l
+// GFX11: v_cmpx_ge_f16_e32 v127.h, v2.l          ; encoding: [0xff,0x05,0x0c,0x7d]
+
+v_cmpx_ge_f16 0.5, v127.l
+// GFX11: v_cmpx_ge_f16_e32 0.5, v127.l           ; encoding: [0xf0,0xfe,0x0c,0x7d]
+
+v_cmpx_ge_f16 src_scc, v2.h
+// GFX11: v_cmpx_ge_f16_e32 src_scc, v2.h         ; encoding: [0xfd,0x04,0x0d,0x7d]
+
+v_cmpx_ge_f16 0xfe0b, v127.h
+// GFX11: v_cmpx_ge_f16_e32 0xfe0b, v127.h        ; encoding: [0xff,0xfe,0x0d,0x7d,0x0b,0xfe,0x00,0x00]
 
 v_cmpx_ge_f32 v1, v2
 // GFX11: v_cmpx_ge_f32_e32 v1, v2                ; encoding: [0x01,0x05,0x2c,0x7d]
@@ -1247,50 +1292,65 @@ v_cmpx_ge_u64 src_scc, v[2:3]
 v_cmpx_ge_u64 0xaf123456, v[254:255]
 // GFX11: v_cmpx_ge_u64_e32 0xaf123456, v[254:255] ; encoding: [0xff,0xfc,0xbd,0x7d,0x56,0x34,0x12,0xaf]
 
-v_cmpx_gt_f16 v1, v2
-// GFX11: v_cmpx_gt_f16_e32 v1, v2                ; encoding: [0x01,0x05,0x08,0x7d]
+v_cmpx_gt_f16 v1.l, v2.l
+// GFX11: v_cmpx_gt_f16_e32 v1.l, v2.l            ; encoding: [0x01,0x05,0x08,0x7d]
+
+v_cmpx_gt_f16 v127.l, v2.l
+// GFX11: v_cmpx_gt_f16_e32 v127.l, v2.l          ; encoding: [0x7f,0x05,0x08,0x7d]
 
-v_cmpx_gt_f16 v127, v2
-// GFX11: v_cmpx_gt_f16_e32 v127, v2              ; encoding: [0x7f,0x05,0x08,0x7d]
+v_cmpx_gt_f16 s1, v2.l
+// GFX11: v_cmpx_gt_f16_e32 s1, v2.l              ; encoding: [0x01,0x04,0x08,0x7d]
 
-v_cmpx_gt_f16 s1, v2
-// GFX11: v_cmpx_gt_f16_e32 s1, v2                ; encoding: [0x01,0x04,0x08,0x7d]
+v_cmpx_gt_f16 s105, v2.l
+// GFX11: v_cmpx_gt_f16_e32 s105, v2.l            ; encoding: [0x69,0x04,0x08,0x7d]
 
-v_cmpx_gt_f16 s105, v2
-// GFX11: v_cmpx_gt_f16_e32 s105, v2              ; encoding: [0x69,0x04,0x08,0x7d]
+v_cmpx_gt_f16 vcc_lo, v2.l
+// GFX11: v_cmpx_gt_f16_e32 vcc_lo, v2.l          ; encoding: [0x6a,0x04,0x08,0x7d]
 
-v_cmpx_gt_f16 vcc_lo, v2
-// GFX11: v_cmpx_gt_f16_e32 vcc_lo, v2            ; encoding: [0x6a,0x04,0x08,0x7d]
+v_cmpx_gt_f16 vcc_hi, v2.l
+// GFX11: v_cmpx_gt_f16_e32 vcc_hi, v2.l          ; encoding: [0x6b,0x04,0x08,0x7d]
 
-v_cmpx_gt_f16 vcc_hi, v2
-// GFX11: v_cmpx_gt_f16_e32 vcc_hi, v2            ; encoding: [0x6b,0x04,0x08,0x7d]
+v_cmpx_gt_f16 ttmp15, v2.l
+// GFX11: v_cmpx_gt_f16_e32 ttmp15, v2.l          ; encoding: [0x7b,0x04,0x08,0x7d]
 
-v_cmpx_gt_f16 ttmp15, v2
-// GFX11: v_cmpx_gt_f16_e32 ttmp15, v2            ; encoding: [0x7b,0x04,0x08,0x7d]
+v_cmpx_gt_f16 m0, v2.l
+// GFX11: v_cmpx_gt_f16_e32 m0, v2.l              ; encoding: [0x7d,0x04,0x08,0x7d]
 
-v_cmpx_gt_f16 m0, v2
-// GFX11: v_cmpx_gt_f16_e32 m0, v2                ; encoding: [0x7d,0x04,0x08,0x7d]
+v_cmpx_gt_f16 exec_lo, v2.l
+// GFX11: v_cmpx_gt_f16_e32 exec_lo, v2.l         ; encoding: [0x7e,0x04,0x08,0x7d]
 
-v_cmpx_gt_f16 exec_lo, v2
-// GFX11: v_cmpx_gt_f16_e32 exec_lo, v2           ; encoding: [0x7e,0x04,0x08,0x7d]
+v_cmpx_gt_f16 exec_hi, v2.l
+// GFX11: v_cmpx_gt_f16_e32 exec_hi, v2.l         ; encoding: [0x7f,0x04,0x08,0x7d]
 
-v_cmpx_gt_f16 exec_hi, v2
-// GFX11: v_cmpx_gt_f16_e32 exec_hi, v2           ; encoding: [0x7f,0x04,0x08,0x7d]
+v_cmpx_gt_f16 null, v2.l
+// GFX11: v_cmpx_gt_f16_e32 null, v2.l            ; encoding: [0x7c,0x04,0x08,0x7d]
 
-v_cmpx_gt_f16 null, v2
-// GFX11: v_cmpx_gt_f16_e32 null, v2              ; encoding: [0x7c,0x04,0x08,0x7d]
+v_cmpx_gt_f16 -1, v2.l
+// GFX11: v_cmpx_gt_f16_e32 -1, v2.l              ; encoding: [0xc1,0x04,0x08,0x7d]
 
-v_cmpx_gt_f16 -1, v2
-// GFX11: v_cmpx_gt_f16_e32 -1, v2                ; encoding: [0xc1,0x04,0x08,0x7d]
+v_cmpx_gt_f16 0.5, v2.l
+// GFX11: v_cmpx_gt_f16_e32 0.5, v2.l             ; encoding: [0xf0,0x04,0x08,0x7d]
 
-v_cmpx_gt_f16 0.5, v2
-// GFX11: v_cmpx_gt_f16_e32 0.5, v2               ; encoding: [0xf0,0x04,0x08,0x7d]
+v_cmpx_gt_f16 src_scc, v2.l
+// GFX11: v_cmpx_gt_f16_e32 src_scc, v2.l         ; encoding: [0xfd,0x04,0x08,0x7d]
 
-v_cmpx_gt_f16 src_scc, v2
-// GFX11: v_cmpx_gt_f16_e32 src_scc, v2           ; encoding: [0xfd,0x04,0x08,0x7d]
+v_cmpx_gt_f16 0xfe0b, v127.l
+// GFX11: v_cmpx_gt_f16_e32 0xfe0b, v127.l        ; encoding: [0xff,0xfe,0x08,0x7d,0x0b,0xfe,0x00,0x00]
 
-v_cmpx_gt_f16 0xfe0b, v127
-// GFX11: v_cmpx_gt_f16_e32 0xfe0b, v127          ; encoding: [0xff,0xfe,0x08,0x7d,0x0b,0xfe,0x00,0x00]
+v_cmpx_gt_f16 v1.h, v2.l
+// GFX11: v_cmpx_gt_f16_e32 v1.h, v2.l            ; encoding: [0x81,0x05,0x08,0x7d]
+
+v_cmpx_gt_f16 v127.h, v2.l
+// GFX11: v_cmpx_gt_f16_e32 v127.h, v2.l          ; encoding: [0xff,0x05,0x08,0x7d]
+
+v_cmpx_gt_f16 0.5, v127.l
+// GFX11: v_cmpx_gt_f16_e32 0.5, v127.l           ; encoding: [0xf0,0xfe,0x08,0x7d]
+
+v_cmpx_gt_f16 src_scc, v2.h
+// GFX11: v_cmpx_gt_f16_e32 src_scc, v2.h         ; encoding: [0xfd,0x04,0x09,0x7d]
+
+v_cmpx_gt_f16 0xfe0b, v127.h
+// GFX11: v_cmpx_gt_f16_e32 0xfe0b, v127.h        ; encoding: [0xff,0xfe,0x09,0x7d,0x0b,0xfe,0x00,0x00]
 
 v_cmpx_gt_f32 v1, v2
 // GFX11: v_cmpx_gt_f32_e32 v1, v2                ; encoding: [0x01,0x05,0x28,0x7d]
@@ -1655,50 +1715,65 @@ v_cmpx_gt_u64 src_scc, v[2:3]
 v_cmpx_gt_u64 0xaf123456, v[254:255]
 // GFX11: v_cmpx_gt_u64_e32 0xaf123456, v[254:255] ; encoding: [0xff,0xfc,0xb9,0x7d,0x56,0x34,0x12,0xaf]
 
-v_cmpx_le_f16 v1, v2
-// GFX11: v_cmpx_le_f16_e32 v1, v2                ; encoding: [0x01,0x05,0x06,0x7d]
+v_cmpx_le_f16 v1.l, v2.l
+// GFX11: v_cmpx_le_f16_e32 v1.l, v2.l            ; encoding: [0x01,0x05,0x06,0x7d]
+
+v_cmpx_le_f16 v127.l, v2.l
+// GFX11: v_cmpx_le_f16_e32 v127.l, v2.l          ; encoding: [0x7f,0x05,0x06,0x7d]
 
-v_cmpx_le_f16 v127, v2
-// GFX11: v_cmpx_le_f16_e32 v127, v2              ; encoding: [0x7f,0x05,0x06,0x7d]
+v_cmpx_le_f16 s1, v2.l
+// GFX11: v_cmpx_le_f16_e32 s1, v2.l              ; encoding: [0x01,0x04,0x06,0x7d]
 
-v_cmpx_le_f16 s1, v2
-// GFX11: v_cmpx_le_f16_e32 s1, v2                ; encoding: [0x01,0x04,0x06,0x7d]
+v_cmpx_le_f16 s105, v2.l
+// GFX11: v_cmpx_le_f16_e32 s105, v2.l            ; encoding: [0x69,0x04,0x06,0x7d]
 
-v_cmpx_le_f16 s105, v2
-// GFX11: v_cmpx_le_f16_e32 s105, v2              ; encoding: [0x69,0x04,0x06,0x7d]
+v_cmpx_le_f16 vcc_lo, v2.l
+// GFX11: v_cmpx_le_f16_e32 vcc_lo, v2.l          ; encoding: [0x6a,0x04,0x06,0x7d]
 
-v_cmpx_le_f16 vcc_lo, v2
-// GFX11: v_cmpx_le_f16_e32 vcc_lo, v2            ; encoding: [0x6a,0x04,0x06,0x7d]
+v_cmpx_le_f16 vcc_hi, v2.l
+// GFX11: v_cmpx_le_f16_e32 vcc_hi, v2.l          ; encoding: [0x6b,0x04,0x06,0x7d]
 
-v_cmpx_le_f16 vcc_hi, v2
-// GFX11: v_cmpx_le_f16_e32 vcc_hi, v2            ; encoding: [0x6b,0x04,0x06,0x7d]
+v_cmpx_le_f16 ttmp15, v2.l
+// GFX11: v_cmpx_le_f16_e32 ttmp15, v2.l          ; encoding: [0x7b,0x04,0x06,0x7d]
 
-v_cmpx_le_f16 ttmp15, v2
-// GFX11: v_cmpx_le_f16_e32 ttmp15, v2            ; encoding: [0x7b,0x04,0x06,0x7d]
+v_cmpx_le_f16 m0, v2.l
+// GFX11: v_cmpx_le_f16_e32 m0, v2.l              ; encoding: [0x7d,0x04,0x06,0x7d]
 
-v_cmpx_le_f16 m0, v2
-// GFX11: v_cmpx_le_f16_e32 m0, v2                ; encoding: [0x7d,0x04,0x06,0x7d]
+v_cmpx_le_f16 exec_lo, v2.l
+// GFX11: v_cmpx_le_f16_e32 exec_lo, v2.l         ; encoding: [0x7e,0x04,0x06,0x7d]
 
-v_cmpx_le_f16 exec_lo, v2
-// GFX11: v_cmpx_le_f16_e32 exec_lo, v2           ; encoding: [0x7e,0x04,0x06,0x7d]
+v_cmpx_le_f16 exec_hi, v2.l
+// GFX11: v_cmpx_le_f16_e32 exec_hi, v2.l         ; encoding: [0x7f,0x04,0x06,0x7d]
 
-v_cmpx_le_f16 exec_hi, v2
-// GFX11: v_cmpx_le_f16_e32 exec_hi, v2           ; encoding: [0x7f,0x04,0x06,0x7d]
+v_cmpx_le_f16 null, v2.l
+// GFX11: v_cmpx_le_f16_e32 null, v2.l            ; encoding: [0x7c,0x04,0x06,0x7d]
 
-v_cmpx_le_f16 null, v2
-// GFX11: v_cmpx_le_f16_e32 null, v2              ; encoding: [0x7c,0x04,0x06,0x7d]
+v_cmpx_le_f16 -1, v2.l
+// GFX11: v_cmpx_le_f16_e32 -1, v2.l              ; encoding: [0xc1,0x04,0x06,0x7d]
 
-v_cmpx_le_f16 -1, v2
-// GFX11: v_cmpx_le_f16_e32 -1, v2                ; encoding: [0xc1,0x04,0x06,0x7d]
+v_cmpx_le_f16 0.5, v2.l
+// GFX11: v_cmpx_le_f16_e32 0.5, v2.l             ; encoding: [0xf0,0x04,0x06,0x7d]
 
-v_cmpx_le_f16 0.5, v2
-// GFX11: v_cmpx_le_f16_e32 0.5, v2               ; encoding: [0xf0,0x04,0x06,0x7d]
+v_cmpx_le_f16 src_scc, v2.l
+// GFX11: v_cmpx_le_f16_e32 src_scc, v2.l         ; encoding: [0xfd,0x04,0x06,0x7d]
 
-v_cmpx_le_f16 src_scc, v2
-// GFX11: v_cmpx_le_f16_e32 src_scc, v2           ; encoding: [0xfd,0x04,0x06,0x7d]
+v_cmpx_le_f16 0xfe0b, v127.l
+// GFX11: v_cmpx_le_f16_e32 0xfe0b, v127.l        ; encoding: [0xff,0xfe,0x06,0x7d,0x0b,0xfe,0x00,0x00]
 
-v_cmpx_le_f16 0xfe0b, v127
-// GFX11: v_cmpx_le_f16_e32 0xfe0b, v127          ; encoding: [0xff,0xfe,0x06,0x7d,0x0b,0xfe,0x00,0x00]
+v_cmpx_le_f16 v1.h, v2.l
+// GFX11: v_cmpx_le_f16_e32 v1.h, v2.l            ; encoding: [0x81,0x05,0x06,0x7d]
+
+v_cmpx_le_f16 v127.h, v2.l
+// GFX11: v_cmpx_le_f16_e32 v127.h, v2.l          ; encoding: [0xff,0x05,0x06,0x7d]
+
+v_cmpx_le_f16 0.5, v127.l
+// GFX11: v_cmpx_le_f16_e32 0.5, v127.l           ; encoding: [0xf0,0xfe,0x06,0x7d]
+
+v_cmpx_le_f16 src_scc, v2.h
+// GFX11: v_cmpx_le_f16_e32 src_scc, v2.h         ; encoding: [0xfd,0x04,0x07,0x7d]
+
+v_cmpx_le_f16 0xfe0b, v127.h
+// GFX11: v_cmpx_le_f16_e32 0xfe0b, v127.h        ; encoding: [0xff,0xfe,0x07,0x7d,0x0b,0xfe,0x00,0x00]
 
 v_cmpx_le_f32 v1, v2
 // GFX11: v_cmpx_le_f32_e32 v1, v2                ; encoding: [0x01,0x05,0x26,0x7d]
@@ -2063,50 +2138,65 @@ v_cmpx_le_u64 src_scc, v[2:3]
 v_cmpx_le_u64 0xaf123456, v[254:255]
 // GFX11: v_cmpx_le_u64_e32 0xaf123456, v[254:255] ; encoding: [0xff,0xfc,0xb7,0x7d,0x56,0x34,0x12,0xaf]
 
-v_cmpx_lg_f16 v1, v2
-// GFX11: v_cmpx_lg_f16_e32 v1, v2                ; encoding: [0x01,0x05,0x0a,0x7d]
+v_cmpx_lg_f16 v1.l, v2.l
+// GFX11: v_cmpx_lg_f16_e32 v1.l, v2.l            ; encoding: [0x01,0x05,0x0a,0x7d]
+
+v_cmpx_lg_f16 v127.l, v2.l
+// GFX11: v_cmpx_lg_f16_e32 v127.l, v2.l          ; encoding: [0x7f,0x05,0x0a,0x7d]
+
+v_cmpx_lg_f16 s1, v2.l
+// GFX11: v_cmpx_lg_f16_e32 s1, v2.l              ; encoding: [0x01,0x04,0x0a,0x7d]
 
-v_cmpx_lg_f16 v127, v2
-// GFX11: v_cmpx_lg_f16_e32 v127, v2              ; encoding: [0x7f,0x05,0x0a,0x7d]
+v_cmpx_lg_f16 s105, v2.l
+// GFX11: v_cmpx_lg_f16_e32 s105, v2.l            ; encoding: [0x69,0x04,0x0a,0x7d]
 
-v_cmpx_lg_f16 s1, v2
-// GFX11: v_cmpx_lg_f16_e32 s1, v2                ; encoding: [0x01,0x04,0x0a,0x7d]
+v_cmpx_lg_f16 vcc_lo, v2.l
+// GFX11: v_cmpx_lg_f16_e32 vcc_lo, v2.l          ; encoding: [0x6a,0x04,0x0a,0x7d]
 
-v_cmpx_lg_f16 s105, v2
-// GFX11: v_cmpx_lg_f16_e32 s105, v2              ; encoding: [0x69,0x04,0x0a,0x7d]
+v_cmpx_lg_f16 vcc_hi, v2.l
+// GFX11: v_cmpx_lg_f16_e32 vcc_hi, v2.l          ; encoding: [0x6b,0x04,0x0a,0x7d]
 
-v_cmpx_lg_f16 vcc_lo, v2
-// GFX11: v_cmpx_lg_f16_e32 vcc_lo, v2            ; encoding: [0x6a,0x04,0x0a,0x7d]
+v_cmpx_lg_f16 ttmp15, v2.l
+// GFX11: v_cmpx_lg_f16_e32 ttmp15, v2.l          ; encoding: [0x7b,0x04,0x0a,0x7d]
 
-v_cmpx_lg_f16 vcc_hi, v2
-// GFX11: v_cmpx_lg_f16_e32 vcc_hi, v2            ; encoding: [0x6b,0x04,0x0a,0x7d]
+v_cmpx_lg_f16 m0, v2.l
+// GFX11: v_cmpx_lg_f16_e32 m0, v2.l              ; encoding: [0x7d,0x04,0x0a,0x7d]
 
-v_cmpx_lg_f16 ttmp15, v2
-// GFX11: v_cmpx_lg_f16_e32 ttmp15, v2            ; encoding: [0x7b,0x04,0x0a,0x7d]
+v_cmpx_lg_f16 exec_lo, v2.l
+// GFX11: v_cmpx_lg_f16_e32 exec_lo, v2.l         ; encoding: [0x7e,0x04,0x0a,0x7d]
 
-v_cmpx_lg_f16 m0, v2
-// GFX11: v_cmpx_lg_f16_e32 m0, v2                ; encoding: [0x7d,0x04,0x0a,0x7d]
+v_cmpx_lg_f16 exec_hi, v2.l
+// GFX11: v_cmpx_lg_f16_e32 exec_hi, v2.l         ; encoding: [0x7f,0x04,0x0a,0x7d]
 
-v_cmpx_lg_f16 exec_lo, v2
-// GFX11: v_cmpx_lg_f16_e32 exec_lo, v2           ; encoding: [0x7e,0x04,0x0a,0x7d]
+v_cmpx_lg_f16 null, v2.l
+// GFX11: v_cmpx_lg_f16_e32 null, v2.l            ; encoding: [0x7c,0x04,0x0a,0x7d]
 
-v_cmpx_lg_f16 exec_hi, v2
-// GFX11: v_cmpx_lg_f16_e32 exec_hi, v2           ; encoding: [0x7f,0x04,0x0a,0x7d]
+v_cmpx_lg_f16 -1, v2.l
+// GFX11: v_cmpx_lg_f16_e32 -1, v2.l              ; encoding: [0xc1,0x04,0x0a,0x7d]
 
-v_cmpx_lg_f16 null, v2
-// GFX11: v_cmpx_lg_f16_e32 null, v2              ; encoding: [0x7c,0x04,0x0a,0x7d]
+v_cmpx_lg_f16 0.5, v2.l
+// GFX11: v_cmpx_lg_f16_e32 0.5, v2.l             ; encoding: [0xf0,0x04,0x0a,0x7d]
 
-v_cmpx_lg_f16 -1, v2
-// GFX11: v_cmpx_lg_f16_e32 -1, v2                ; encoding: [0xc1,0x04,0x0a,0x7d]
+v_cmpx_lg_f16 src_scc, v2.l
+// GFX11: v_cmpx_lg_f16_e32 src_scc, v2.l         ; encoding: [0xfd,0x04,0x0a,0x7d]
 
-v_cmpx_lg_f16 0.5, v2
-// GFX11: v_cmpx_lg_f16_e32 0.5, v2               ; encoding: [0xf0,0x04,0x0a,0x7d]
+v_cmpx_lg_f16 0xfe0b, v127.l
+// GFX11: v_cmpx_lg_f16_e32 0xfe0b, v127.l        ; encoding: [0xff,0xfe,0x0a,0x7d,0x0b,0xfe,0x00,0x00]
 
-v_cmpx_lg_f16 src_scc, v2
-// GFX11: v_cmpx_lg_f16_e32 src_scc, v2           ; encoding: [0xfd,0x04,0x0a,0x7d]
+v_cmpx_lg_f16 v1.h, v2.l
+// GFX11: v_cmpx_lg_f16_e32 v1.h, v2.l            ; encoding: [0x81,0x05,0x0a,0x7d]
 
-v_cmpx_lg_f16 0xfe0b, v127
-// GFX11: v_cmpx_lg_f16_e32 0xfe0b, v127          ; encoding: [0xff,0xfe,0x0a,0x7d,0x0b,0xfe,0x00,0x00]
+v_cmpx_lg_f16 v127.h, v2.l
+// GFX11: v_cmpx_lg_f16_e32 v127.h, v2.l          ; encoding: [0xff,0x05,0x0a,0x7d]
+
+v_cmpx_lg_f16 0.5, v127.l
+// GFX11: v_cmpx_lg_f16_e32 0.5, v127.l           ; encoding: [0xf0,0xfe,0x0a,0x7d]
+
+v_cmpx_lg_f16 src_scc, v2.h
+// GFX11: v_cmpx_lg_f16_e32 src_scc, v2.h         ; encoding: [0xfd,0x04,0x0b,0x7d]
+
+v_cmpx_lg_f16 0xfe0b, v127.h
+// GFX11: v_cmpx_lg_f16_e32 0xfe0b, v127.h        ; encoding: [0xff,0xfe,0x0b,0x7d,0x0b,0xfe,0x00,0x00]
 
 v_cmpx_lg_f32 v1, v2
 // GFX11: v_cmpx_lg_f32_e32 v1, v2                ; encoding: [0x01,0x05,0x2a,0x7d]
@@ -2894,50 +2984,65 @@ v_cmpx_ne_u64 src_scc, v[2:3]
 v_cmpx_ne_u64 0xaf123456, v[254:255]
 // GFX11: v_cmpx_ne_u64_e32 0xaf123456, v[254:255] ; encoding: [0xff,0xfc,0xbb,0x7d,0x56,0x34,0x12,0xaf]
 
-v_cmpx_neq_f16 v1, v2
-// GFX11: v_cmpx_neq_f16_e32 v1, v2               ; encoding: [0x01,0x05,0x1a,0x7d]
+v_cmpx_neq_f16 v1.l, v2.l
+// GFX11: v_cmpx_neq_f16_e32 v1.l, v2.l           ; encoding: [0x01,0x05,0x1a,0x7d]
+
+v_cmpx_neq_f16 v127.l, v2.l
+// GFX11: v_cmpx_neq_f16_e32 v127.l, v2.l         ; encoding: [0x7f,0x05,0x1a,0x7d]
+
+v_cmpx_neq_f16 s1, v2.l
+// GFX11: v_cmpx_neq_f16_e32 s1, v2.l             ; encoding: [0x01,0x04,0x1a,0x7d]
 
-v_cmpx_neq_f16 v127, v2
-// GFX11: v_cmpx_neq_f16_e32 v127, v2             ; encoding: [0x7f,0x05,0x1a,0x7d]
+v_cmpx_neq_f16 s105, v2.l
+// GFX11: v_cmpx_neq_f16_e32 s105, v2.l           ; encoding: [0x69,0x04,0x1a,0x7d]
 
-v_cmpx_neq_f16 s1, v2
-// GFX11: v_cmpx_neq_f16_e32 s1, v2               ; encoding: [0x01,0x04,0x1a,0x7d]
+v_cmpx_neq_f16 vcc_lo, v2.l
+// GFX11: v_cmpx_neq_f16_e32 vcc_lo, v2.l         ; encoding: [0x6a,0x04,0x1a,0x7d]
 
-v_cmpx_neq_f16 s105, v2
-// GFX11: v_cmpx_neq_f16_e32 s105, v2             ; encoding: [0x69,0x04,0x1a,0x7d]
+v_cmpx_neq_f16 vcc_hi, v2.l
+// GFX11: v_cmpx_neq_f16_e32 vcc_hi, v2.l         ; encoding: [0x6b,0x04,0x1a,0x7d]
 
-v_cmpx_neq_f16 vcc_lo, v2
-// GFX11: v_cmpx_neq_f16_e32 vcc_lo, v2           ; encoding: [0x6a,0x04,0x1a,0x7d]
+v_cmpx_neq_f16 ttmp15, v2.l
+// GFX11: v_cmpx_neq_f16_e32 ttmp15, v2.l         ; encoding: [0x7b,0x04,0x1a,0x7d]
 
-v_cmpx_neq_f16 vcc_hi, v2
-// GFX11: v_cmpx_neq_f16_e32 vcc_hi, v2           ; encoding: [0x6b,0x04,0x1a,0x7d]
+v_cmpx_neq_f16 m0, v2.l
+// GFX11: v_cmpx_neq_f16_e32 m0, v2.l             ; encoding: [0x7d,0x04,0x1a,0x7d]
 
-v_cmpx_neq_f16 ttmp15, v2
-// GFX11: v_cmpx_neq_f16_e32 ttmp15, v2           ; encoding: [0x7b,0x04,0x1a,0x7d]
+v_cmpx_neq_f16 exec_lo, v2.l
+// GFX11: v_cmpx_neq_f16_e32 exec_lo, v2.l        ; encoding: [0x7e,0x04,0x1a,0x7d]
 
-v_cmpx_neq_f16 m0, v2
-// GFX11: v_cmpx_neq_f16_e32 m0, v2               ; encoding: [0x7d,0x04,0x1a,0x7d]
+v_cmpx_neq_f16 exec_hi, v2.l
+// GFX11: v_cmpx_neq_f16_e32 exec_hi, v2.l        ; encoding: [0x7f,0x04,0x1a,0x7d]
 
-v_cmpx_neq_f16 exec_lo, v2
-// GFX11: v_cmpx_neq_f16_e32 exec_lo, v2          ; encoding: [0x7e,0x04,0x1a,0x7d]
+v_cmpx_neq_f16 null, v2.l
+// GFX11: v_cmpx_neq_f16_e32 null, v2.l           ; encoding: [0x7c,0x04,0x1a,0x7d]
 
-v_cmpx_neq_f16 exec_hi, v2
-// GFX11: v_cmpx_neq_f16_e32 exec_hi, v2          ; encoding: [0x7f,0x04,0x1a,0x7d]
+v_cmpx_neq_f16 -1, v2.l
+// GFX11: v_cmpx_neq_f16_e32 -1, v2.l             ; encoding: [0xc1,0x04,0x1a,0x7d]
 
-v_cmpx_neq_f16 null, v2
-// GFX11: v_cmpx_neq_f16_e32 null, v2             ; encoding: [0x7c,0x04,0x1a,0x7d]
+v_cmpx_neq_f16 0.5, v2.l
+// GFX11: v_cmpx_neq_f16_e32 0.5, v2.l            ; encoding: [0xf0,0x04,0x1a,0x7d]
 
-v_cmpx_neq_f16 -1, v2
-// GFX11: v_cmpx_neq_f16_e32 -1, v2               ; encoding: [0xc1,0x04,0x1a,0x7d]
+v_cmpx_neq_f16 src_scc, v2.l
+// GFX11: v_cmpx_neq_f16_e32 src_scc, v2.l        ; encoding: [0xfd,0x04,0x1a,0x7d]
 
-v_cmpx_neq_f16 0.5, v2
-// GFX11: v_cmpx_neq_f16_e32 0.5, v2              ; encoding: [0xf0,0x04,0x1a,0x7d]
+v_cmpx_neq_f16 0xfe0b, v127.l
+// GFX11: v_cmpx_neq_f16_e32 0xfe0b, v127.l       ; encoding: [0xff,0xfe,0x1a,0x7d,0x0b,0xfe,0x00,0x00]
 
-v_cmpx_neq_f16 src_scc, v2
-// GFX11: v_cmpx_neq_f16_e32 src_scc, v2          ; encoding: [0xfd,0x04,0x1a,0x7d]
+v_cmpx_neq_f16 v1.h, v2.l
+// GFX11: v_cmpx_neq_f16_e32 v1.h, v2.l           ; encoding: [0x81,0x05,0x1a,0x7d]
 
-v_cmpx_neq_f16 0xfe0b, v127
-// GFX11: v_cmpx_neq_f16_e32 0xfe0b, v127         ; encoding: [0xff,0xfe,0x1a,0x7d,0x0b,0xfe,0x00,0x00]
+v_cmpx_neq_f16 v127.h, v2.l
+// GFX11: v_cmpx_neq_f16_e32 v127.h, v2.l         ; encoding: [0xff,0x05,0x1a,0x7d]
+
+v_cmpx_neq_f16 0.5, v127.l
+// GFX11: v_cmpx_neq_f16_e32 0.5, v127.l          ; encoding: [0xf0,0xfe,0x1a,0x7d]
+
+v_cmpx_neq_f16 src_scc, v2.h
+// GFX11: v_cmpx_neq_f16_e32 src_scc, v2.h        ; encoding: [0xfd,0x04,0x1b,0x7d]
+
+v_cmpx_neq_f16 0xfe0b, v127.h
+// GFX11: v_cmpx_neq_f16_e32 0xfe0b, v127.h       ; encoding: [0xff,0xfe,0x1b,0x7d,0x0b,0xfe,0x00,0x00]
 
 v_cmpx_neq_f32 v1, v2
 // GFX11: v_cmpx_neq_f32_e32 v1, v2               ; encoding: [0x01,0x05,0x3a,0x7d]
@@ -3020,50 +3125,65 @@ v_cmpx_neq_f64 src_scc, v[2:3]
 v_cmpx_neq_f64 0xaf123456, v[254:255]
 // GFX11: v_cmpx_neq_f64_e32 0xaf123456, v[254:255] ; encoding: [0xff,0xfc,0x5b,0x7d,0x56,0x34,0x12,0xaf]
 
-v_cmpx_nge_f16 v1, v2
-// GFX11: v_cmpx_nge_f16_e32 v1, v2               ; encoding: [0x01,0x05,0x12,0x7d]
+v_cmpx_nge_f16 v1.l, v2.l
+// GFX11: v_cmpx_nge_f16_e32 v1.l, v2.l           ; encoding: [0x01,0x05,0x12,0x7d]
+
+v_cmpx_nge_f16 v127.l, v2.l
+// GFX11: v_cmpx_nge_f16_e32 v127.l, v2.l         ; encoding: [0x7f,0x05,0x12,0x7d]
+
+v_cmpx_nge_f16 s1, v2.l
+// GFX11: v_cmpx_nge_f16_e32 s1, v2.l             ; encoding: [0x01,0x04,0x12,0x7d]
 
-v_cmpx_nge_f16 v127, v2
-// GFX11: v_cmpx_nge_f16_e32 v127, v2             ; encoding: [0x7f,0x05,0x12,0x7d]
+v_cmpx_nge_f16 s105, v2.l
+// GFX11: v_cmpx_nge_f16_e32 s105, v2.l           ; encoding: [0x69,0x04,0x12,0x7d]
 
-v_cmpx_nge_f16 s1, v2
-// GFX11: v_cmpx_nge_f16_e32 s1, v2               ; encoding: [0x01,0x04,0x12,0x7d]
+v_cmpx_nge_f16 vcc_lo, v2.l
+// GFX11: v_cmpx_nge_f16_e32 vcc_lo, v2.l         ; encoding: [0x6a,0x04,0x12,0x7d]
 
-v_cmpx_nge_f16 s105, v2
-// GFX11: v_cmpx_nge_f16_e32 s105, v2             ; encoding: [0x69,0x04,0x12,0x7d]
+v_cmpx_nge_f16 vcc_hi, v2.l
+// GFX11: v_cmpx_nge_f16_e32 vcc_hi, v2.l         ; encoding: [0x6b,0x04,0x12,0x7d]
 
-v_cmpx_nge_f16 vcc_lo, v2
-// GFX11: v_cmpx_nge_f16_e32 vcc_lo, v2           ; encoding: [0x6a,0x04,0x12,0x7d]
+v_cmpx_nge_f16 ttmp15, v2.l
+// GFX11: v_cmpx_nge_f16_e32 ttmp15, v2.l         ; encoding: [0x7b,0x04,0x12,0x7d]
 
-v_cmpx_nge_f16 vcc_hi, v2
-// GFX11: v_cmpx_nge_f16_e32 vcc_hi, v2           ; encoding: [0x6b,0x04,0x12,0x7d]
+v_cmpx_nge_f16 m0, v2.l
+// GFX11: v_cmpx_nge_f16_e32 m0, v2.l             ; encoding: [0x7d,0x04,0x12,0x7d]
 
-v_cmpx_nge_f16 ttmp15, v2
-// GFX11: v_cmpx_nge_f16_e32 ttmp15, v2           ; encoding: [0x7b,0x04,0x12,0x7d]
+v_cmpx_nge_f16 exec_lo, v2.l
+// GFX11: v_cmpx_nge_f16_e32 exec_lo, v2.l        ; encoding: [0x7e,0x04,0x12,0x7d]
 
-v_cmpx_nge_f16 m0, v2
-// GFX11: v_cmpx_nge_f16_e32 m0, v2               ; encoding: [0x7d,0x04,0x12,0x7d]
+v_cmpx_nge_f16 exec_hi, v2.l
+// GFX11: v_cmpx_nge_f16_e32 exec_hi, v2.l        ; encoding: [0x7f,0x04,0x12,0x7d]
 
-v_cmpx_nge_f16 exec_lo, v2
-// GFX11: v_cmpx_nge_f16_e32 exec_lo, v2          ; encoding: [0x7e,0x04,0x12,0x7d]
+v_cmpx_nge_f16 null, v2.l
+// GFX11: v_cmpx_nge_f16_e32 null, v2.l           ; encoding: [0x7c,0x04,0x12,0x7d]
 
-v_cmpx_nge_f16 exec_hi, v2
-// GFX11: v_cmpx_nge_f16_e32 exec_hi, v2          ; encoding: [0x7f,0x04,0x12,0x7d]
+v_cmpx_nge_f16 -1, v2.l
+// GFX11: v_cmpx_nge_f16_e32 -1, v2.l             ; encoding: [0xc1,0x04,0x12,0x7d]
 
-v_cmpx_nge_f16 null, v2
-// GFX11: v_cmpx_nge_f16_e32 null, v2             ; encoding: [0x7c,0x04,0x12,0x7d]
+v_cmpx_nge_f16 0.5, v2.l
+// GFX11: v_cmpx_nge_f16_e32 0.5, v2.l            ; encoding: [0xf0,0x04,0x12,0x7d]
 
-v_cmpx_nge_f16 -1, v2
-// GFX11: v_cmpx_nge_f16_e32 -1, v2               ; encoding: [0xc1,0x04,0x12,0x7d]
+v_cmpx_nge_f16 src_scc, v2.l
+// GFX11: v_cmpx_nge_f16_e32 src_scc, v2.l        ; encoding: [0xfd,0x04,0x12,0x7d]
 
-v_cmpx_nge_f16 0.5, v2
-// GFX11: v_cmpx_nge_f16_e32 0.5, v2              ; encoding: [0xf0,0x04,0x12,0x7d]
+v_cmpx_nge_f16 0xfe0b, v127.l
+// GFX11: v_cmpx_nge_f16_e32 0xfe0b, v127.l       ; encoding: [0xff,0xfe,0x12,0x7d,0x0b,0xfe,0x00,0x00]
 
-v_cmpx_nge_f16 src_scc, v2
-// GFX11: v_cmpx_nge_f16_e32 src_scc, v2          ; encoding: [0xfd,0x04,0x12,0x7d]
+v_cmpx_nge_f16 v1.h, v2.l
+// GFX11: v_cmpx_nge_f16_e32 v1.h, v2.l           ; encoding: [0x81,0x05,0x12,0x7d]
 
-v_cmpx_nge_f16 0xfe0b, v127
-// GFX11: v_cmpx_nge_f16_e32 0xfe0b, v127         ; encoding: [0xff,0xfe,0x12,0x7d,0x0b,0xfe,0x00,0x00]
+v_cmpx_nge_f16 v127.h, v2.l
+// GFX11: v_cmpx_nge_f16_e32 v127.h, v2.l         ; encoding: [0xff,0x05,0x12,0x7d]
+
+v_cmpx_nge_f16 0.5, v127.l
+// GFX11: v_cmpx_nge_f16_e32 0.5, v127.l          ; encoding: [0xf0,0xfe,0x12,0x7d]
+
+v_cmpx_nge_f16 src_scc, v2.h
+// GFX11: v_cmpx_nge_f16_e32 src_scc, v2.h        ; encoding: [0xfd,0x04,0x13,0x7d]
+
+v_cmpx_nge_f16 0xfe0b, v127.h
+// GFX11: v_cmpx_nge_f16_e32 0xfe0b, v127.h       ; encoding: [0xff,0xfe,0x13,0x7d,0x0b,0xfe,0x00,0x00]
 
 v_cmpx_nge_f32 v1, v2
 // GFX11: v_cmpx_nge_f32_e32 v1, v2               ; encoding: [0x01,0x05,0x32,0x7d]
@@ -3146,50 +3266,65 @@ v_cmpx_nge_f64 src_scc, v[2:3]
 v_cmpx_nge_f64 0xaf123456, v[254:255]
 // GFX11: v_cmpx_nge_f64_e32 0xaf123456, v[254:255] ; encoding: [0xff,0xfc,0x53,0x7d,0x56,0x34,0x12,0xaf]
 
-v_cmpx_ngt_f16 v1, v2
-// GFX11: v_cmpx_ngt_f16_e32 v1, v2               ; encoding: [0x01,0x05,0x16,0x7d]
+v_cmpx_ngt_f16 v1.l, v2.l
+// GFX11: v_cmpx_ngt_f16_e32 v1.l, v2.l           ; encoding: [0x01,0x05,0x16,0x7d]
+
+v_cmpx_ngt_f16 v127.l, v2.l
+// GFX11: v_cmpx_ngt_f16_e32 v127.l, v2.l         ; encoding: [0x7f,0x05,0x16,0x7d]
+
+v_cmpx_ngt_f16 s1, v2.l
+// GFX11: v_cmpx_ngt_f16_e32 s1, v2.l             ; encoding: [0x01,0x04,0x16,0x7d]
 
-v_cmpx_ngt_f16 v127, v2
-// GFX11: v_cmpx_ngt_f16_e32 v127, v2             ; encoding: [0x7f,0x05,0x16,0x7d]
+v_cmpx_ngt_f16 s105, v2.l
+// GFX11: v_cmpx_ngt_f16_e32 s105, v2.l           ; encoding: [0x69,0x04,0x16,0x7d]
 
-v_cmpx_ngt_f16 s1, v2
-// GFX11: v_cmpx_ngt_f16_e32 s1, v2               ; encoding: [0x01,0x04,0x16,0x7d]
+v_cmpx_ngt_f16 vcc_lo, v2.l
+// GFX11: v_cmpx_ngt_f16_e32 vcc_lo, v2.l         ; encoding: [0x6a,0x04,0x16,0x7d]
 
-v_cmpx_ngt_f16 s105, v2
-// GFX11: v_cmpx_ngt_f16_e32 s105, v2             ; encoding: [0x69,0x04,0x16,0x7d]
+v_cmpx_ngt_f16 vcc_hi, v2.l
+// GFX11: v_cmpx_ngt_f16_e32 vcc_hi, v2.l         ; encoding: [0x6b,0x04,0x16,0x7d]
 
-v_cmpx_ngt_f16 vcc_lo, v2
-// GFX11: v_cmpx_ngt_f16_e32 vcc_lo, v2           ; encoding: [0x6a,0x04,0x16,0x7d]
+v_cmpx_ngt_f16 ttmp15, v2.l
+// GFX11: v_cmpx_ngt_f16_e32 ttmp15, v2.l         ; encoding: [0x7b,0x04,0x16,0x7d]
 
-v_cmpx_ngt_f16 vcc_hi, v2
-// GFX11: v_cmpx_ngt_f16_e32 vcc_hi, v2           ; encoding: [0x6b,0x04,0x16,0x7d]
+v_cmpx_ngt_f16 m0, v2.l
+// GFX11: v_cmpx_ngt_f16_e32 m0, v2.l             ; encoding: [0x7d,0x04,0x16,0x7d]
 
-v_cmpx_ngt_f16 ttmp15, v2
-// GFX11: v_cmpx_ngt_f16_e32 ttmp15, v2           ; encoding: [0x7b,0x04,0x16,0x7d]
+v_cmpx_ngt_f16 exec_lo, v2.l
+// GFX11: v_cmpx_ngt_f16_e32 exec_lo, v2.l        ; encoding: [0x7e,0x04,0x16,0x7d]
 
-v_cmpx_ngt_f16 m0, v2
-// GFX11: v_cmpx_ngt_f16_e32 m0, v2               ; encoding: [0x7d,0x04,0x16,0x7d]
+v_cmpx_ngt_f16 exec_hi, v2.l
+// GFX11: v_cmpx_ngt_f16_e32 exec_hi, v2.l        ; encoding: [0x7f,0x04,0x16,0x7d]
 
-v_cmpx_ngt_f16 exec_lo, v2
-// GFX11: v_cmpx_ngt_f16_e32 exec_lo, v2          ; encoding: [0x7e,0x04,0x16,0x7d]
+v_cmpx_ngt_f16 null, v2.l
+// GFX11: v_cmpx_ngt_f16_e32 null, v2.l           ; encoding: [0x7c,0x04,0x16,0x7d]
 
-v_cmpx_ngt_f16 exec_hi, v2
-// GFX11: v_cmpx_ngt_f16_e32 exec_hi, v2          ; encoding: [0x7f,0x04,0x16,0x7d]
+v_cmpx_ngt_f16 -1, v2.l
+// GFX11: v_cmpx_ngt_f16_e32 -1, v2.l             ; encoding: [0xc1,0x04,0x16,0x7d]
 
-v_cmpx_ngt_f16 null, v2
-// GFX11: v_cmpx_ngt_f16_e32 null, v2             ; encoding: [0x7c,0x04,0x16,0x7d]
+v_cmpx_ngt_f16 0.5, v2.l
+// GFX11: v_cmpx_ngt_f16_e32 0.5, v2.l            ; encoding: [0xf0,0x04,0x16,0x7d]
 
-v_cmpx_ngt_f16 -1, v2
-// GFX11: v_cmpx_ngt_f16_e32 -1, v2               ; encoding: [0xc1,0x04,0x16,0x7d]
+v_cmpx_ngt_f16 src_scc, v2.l
+// GFX11: v_cmpx_ngt_f16_e32 src_scc, v2.l        ; encoding: [0xfd,0x04,0x16,0x7d]
 
-v_cmpx_ngt_f16 0.5, v2
-// GFX11: v_cmpx_ngt_f16_e32 0.5, v2              ; encoding: [0xf0,0x04,0x16,0x7d]
+v_cmpx_ngt_f16 0xfe0b, v127.l
+// GFX11: v_cmpx_ngt_f16_e32 0xfe0b, v127.l       ; encoding: [0xff,0xfe,0x16,0x7d,0x0b,0xfe,0x00,0x00]
 
-v_cmpx_ngt_f16 src_scc, v2
-// GFX11: v_cmpx_ngt_f16_e32 src_scc, v2          ; encoding: [0xfd,0x04,0x16,0x7d]
+v_cmpx_ngt_f16 v1.h, v2.l
+// GFX11: v_cmpx_ngt_f16_e32 v1.h, v2.l           ; encoding: [0x81,0x05,0x16,0x7d]
 
-v_cmpx_ngt_f16 0xfe0b, v127
-// GFX11: v_cmpx_ngt_f16_e32 0xfe0b, v127         ; encoding: [0xff,0xfe,0x16,0x7d,0x0b,0xfe,0x00,0x00]
+v_cmpx_ngt_f16 v127.h, v2.l
+// GFX11: v_cmpx_ngt_f16_e32 v127.h, v2.l         ; encoding: [0xff,0x05,0x16,0x7d]
+
+v_cmpx_ngt_f16 0.5, v127.l
+// GFX11: v_cmpx_ngt_f16_e32 0.5, v127.l          ; encoding: [0xf0,0xfe,0x16,0x7d]
+
+v_cmpx_ngt_f16 src_scc, v2.h
+// GFX11: v_cmpx_ngt_f16_e32 src_scc, v2.h        ; encoding: [0xfd,0x04,0x17,0x7d]
+
+v_cmpx_ngt_f16 0xfe0b, v127.h
+// GFX11: v_cmpx_ngt_f16_e32 0xfe0b, v127.h       ; encoding: [0xff,0xfe,0x17,0x7d,0x0b,0xfe,0x00,0x00]
 
 v_cmpx_ngt_f32 v1, v2
 // GFX11: v_cmpx_ngt_f32_e32 v1, v2               ; encoding: [0x01,0x05,0x36,0x7d]
@@ -3272,50 +3407,65 @@ v_cmpx_ngt_f64 src_scc, v[2:3]
 v_cmpx_ngt_f64 0xaf123456, v[254:255]
 // GFX11: v_cmpx_ngt_f64_e32 0xaf123456, v[254:255] ; encoding: [0xff,0xfc,0x57,0x7d,0x56,0x34,0x12,0xaf]
 
-v_cmpx_nle_f16 v1, v2
-// GFX11: v_cmpx_nle_f16_e32 v1, v2               ; encoding: [0x01,0x05,0x18,0x7d]
+v_cmpx_nle_f16 v1.l, v2.l
+// GFX11: v_cmpx_nle_f16_e32 v1.l, v2.l           ; encoding: [0x01,0x05,0x18,0x7d]
+
+v_cmpx_nle_f16 v127.l, v2.l
+// GFX11: v_cmpx_nle_f16_e32 v127.l, v2.l         ; encoding: [0x7f,0x05,0x18,0x7d]
+
+v_cmpx_nle_f16 s1, v2.l
+// GFX11: v_cmpx_nle_f16_e32 s1, v2.l             ; encoding: [0x01,0x04,0x18,0x7d]
+
+v_cmpx_nle_f16 s105, v2.l
+// GFX11: v_cmpx_nle_f16_e32 s105, v2.l           ; encoding: [0x69,0x04,0x18,0x7d]
 
-v_cmpx_nle_f16 v127, v2
-// GFX11: v_cmpx_nle_f16_e32 v127, v2             ; encoding: [0x7f,0x05,0x18,0x7d]
+v_cmpx_nle_f16 vcc_lo, v2.l
+// GFX11: v_cmpx_nle_f16_e32 vcc_lo, v2.l         ; encoding: [0x6a,0x04,0x18,0x7d]
 
-v_cmpx_nle_f16 s1, v2
-// GFX11: v_cmpx_nle_f16_e32 s1, v2               ; encoding: [0x01,0x04,0x18,0x7d]
+v_cmpx_nle_f16 vcc_hi, v2.l
+// GFX11: v_cmpx_nle_f16_e32 vcc_hi, v2.l         ; encoding: [0x6b,0x04,0x18,0x7d]
 
-v_cmpx_nle_f16 s105, v2
-// GFX11: v_cmpx_nle_f16_e32 s105, v2             ; encoding: [0x69,0x04,0x18,0x7d]
+v_cmpx_nle_f16 ttmp15, v2.l
+// GFX11: v_cmpx_nle_f16_e32 ttmp15, v2.l         ; encoding: [0x7b,0x04,0x18,0x7d]
 
-v_cmpx_nle_f16 vcc_lo, v2
-// GFX11: v_cmpx_nle_f16_e32 vcc_lo, v2           ; encoding: [0x6a,0x04,0x18,0x7d]
+v_cmpx_nle_f16 m0, v2.l
+// GFX11: v_cmpx_nle_f16_e32 m0, v2.l             ; encoding: [0x7d,0x04,0x18,0x7d]
 
-v_cmpx_nle_f16 vcc_hi, v2
-// GFX11: v_cmpx_nle_f16_e32 vcc_hi, v2           ; encoding: [0x6b,0x04,0x18,0x7d]
+v_cmpx_nle_f16 exec_lo, v2.l
+// GFX11: v_cmpx_nle_f16_e32 exec_lo, v2.l        ; encoding: [0x7e,0x04,0x18,0x7d]
 
-v_cmpx_nle_f16 ttmp15, v2
-// GFX11: v_cmpx_nle_f16_e32 ttmp15, v2           ; encoding: [0x7b,0x04,0x18,0x7d]
+v_cmpx_nle_f16 exec_hi, v2.l
+// GFX11: v_cmpx_nle_f16_e32 exec_hi, v2.l        ; encoding: [0x7f,0x04,0x18,0x7d]
 
-v_cmpx_nle_f16 m0, v2
-// GFX11: v_cmpx_nle_f16_e32 m0, v2               ; encoding: [0x7d,0x04,0x18,0x7d]
+v_cmpx_nle_f16 null, v2.l
+// GFX11: v_cmpx_nle_f16_e32 null, v2.l           ; encoding: [0x7c,0x04,0x18,0x7d]
 
-v_cmpx_nle_f16 exec_lo, v2
-// GFX11: v_cmpx_nle_f16_e32 exec_lo, v2          ; encoding: [0x7e,0x04,0x18,0x7d]
+v_cmpx_nle_f16 -1, v2.l
+// GFX11: v_cmpx_nle_f16_e32 -1, v2.l             ; encoding: [0xc1,0x04,0x18,0x7d]
 
-v_cmpx_nle_f16 exec_hi, v2
-// GFX11: v_cmpx_nle_f16_e32 exec_hi, v2          ; encoding: [0x7f,0x04,0x18,0x7d]
+v_cmpx_nle_f16 0.5, v2.l
+// GFX11: v_cmpx_nle_f16_e32 0.5, v2.l            ; encoding: [0xf0,0x04,0x18,0x7d]
 
-v_cmpx_nle_f16 null, v2
-// GFX11: v_cmpx_nle_f16_e32 null, v2             ; encoding: [0x7c,0x04,0x18,0x7d]
+v_cmpx_nle_f16 src_scc, v2.l
+// GFX11: v_cmpx_nle_f16_e32 src_scc, v2.l        ; encoding: [0xfd,0x04,0x18,0x7d]
 
-v_cmpx_nle_f16 -1, v2
-// GFX11: v_cmpx_nle_f16_e32 -1, v2               ; encoding: [0xc1,0x04,0x18,0x7d]
+v_cmpx_nle_f16 0xfe0b, v127.l
+// GFX11: v_cmpx_nle_f16_e32 0xfe0b, v127.l       ; encoding: [0xff,0xfe,0x18,0x7d,0x0b,0xfe,0x00,0x00]
 
-v_cmpx_nle_f16 0.5, v2
-// GFX11: v_cmpx_nle_f16_e32 0.5, v2              ; encoding: [0xf0,0x04,0x18,0x7d]
+v_cmpx_nle_f16 v1.h, v2.l
+// GFX11: v_cmpx_nle_f16_e32 v1.h, v2.l           ; encoding: [0x81,0x05,0x18,0x7d]
 
-v_cmpx_nle_f16 src_scc, v2
-// GFX11: v_cmpx_nle_f16_e32 src_scc, v2          ; encoding: [0xfd,0x04,0x18,0x7d]
+v_cmpx_nle_f16 v127.h, v2.l
+// GFX11: v_cmpx_nle_f16_e32 v127.h, v2.l         ; encoding: [0xff,0x05,0x18,0x7d]
 
-v_cmpx_nle_f16 0xfe0b, v127
-// GFX11: v_cmpx_nle_f16_e32 0xfe0b, v127         ; encoding: [0xff,0xfe,0x18,0x7d,0x0b,0xfe,0x00,0x00]
+v_cmpx_nle_f16 0.5, v127.l
+// GFX11: v_cmpx_nle_f16_e32 0.5, v127.l          ; encoding: [0xf0,0xfe,0x18,0x7d]
+
+v_cmpx_nle_f16 src_scc, v2.h
+// GFX11: v_cmpx_nle_f16_e32 src_scc, v2.h        ; encoding: [0xfd,0x04,0x19,0x7d]
+
+v_cmpx_nle_f16 0xfe0b, v127.h
+// GFX11: v_cmpx_nle_f16_e32 0xfe0b, v127.h       ; encoding: [0xff,0xfe,0x19,0x7d,0x0b,0xfe,0x00,0x00]
 
 v_cmpx_nle_f32 v1, v2
 // GFX11: v_cmpx_nle_f32_e32 v1, v2               ; encoding: [0x01,0x05,0x38,0x7d]
@@ -3398,50 +3548,65 @@ v_cmpx_nle_f64 src_scc, v[2:3]
 v_cmpx_nle_f64 0xaf123456, v[254:255]
 // GFX11: v_cmpx_nle_f64_e32 0xaf123456, v[254:255] ; encoding: [0xff,0xfc,0x59,0x7d,0x56,0x34,0x12,0xaf]
 
-v_cmpx_nlg_f16 v1, v2
-// GFX11: v_cmpx_nlg_f16_e32 v1, v2               ; encoding: [0x01,0x05,0x14,0x7d]
+v_cmpx_nlg_f16 v1.l, v2.l
+// GFX11: v_cmpx_nlg_f16_e32 v1.l, v2.l           ; encoding: [0x01,0x05,0x14,0x7d]
+
+v_cmpx_nlg_f16 v127.l, v2.l
+// GFX11: v_cmpx_nlg_f16_e32 v127.l, v2.l         ; encoding: [0x7f,0x05,0x14,0x7d]
+
+v_cmpx_nlg_f16 s1, v2.l
+// GFX11: v_cmpx_nlg_f16_e32 s1, v2.l             ; encoding: [0x01,0x04,0x14,0x7d]
+
+v_cmpx_nlg_f16 s105, v2.l
+// GFX11: v_cmpx_nlg_f16_e32 s105, v2.l           ; encoding: [0x69,0x04,0x14,0x7d]
 
-v_cmpx_nlg_f16 v127, v2
-// GFX11: v_cmpx_nlg_f16_e32 v127, v2             ; encoding: [0x7f,0x05,0x14,0x7d]
+v_cmpx_nlg_f16 vcc_lo, v2.l
+// GFX11: v_cmpx_nlg_f16_e32 vcc_lo, v2.l         ; encoding: [0x6a,0x04,0x14,0x7d]
 
-v_cmpx_nlg_f16 s1, v2
-// GFX11: v_cmpx_nlg_f16_e32 s1, v2               ; encoding: [0x01,0x04,0x14,0x7d]
+v_cmpx_nlg_f16 vcc_hi, v2.l
+// GFX11: v_cmpx_nlg_f16_e32 vcc_hi, v2.l         ; encoding: [0x6b,0x04,0x14,0x7d]
 
-v_cmpx_nlg_f16 s105, v2
-// GFX11: v_cmpx_nlg_f16_e32 s105, v2             ; encoding: [0x69,0x04,0x14,0x7d]
+v_cmpx_nlg_f16 ttmp15, v2.l
+// GFX11: v_cmpx_nlg_f16_e32 ttmp15, v2.l         ; encoding: [0x7b,0x04,0x14,0x7d]
 
-v_cmpx_nlg_f16 vcc_lo, v2
-// GFX11: v_cmpx_nlg_f16_e32 vcc_lo, v2           ; encoding: [0x6a,0x04,0x14,0x7d]
+v_cmpx_nlg_f16 m0, v2.l
+// GFX11: v_cmpx_nlg_f16_e32 m0, v2.l             ; encoding: [0x7d,0x04,0x14,0x7d]
 
-v_cmpx_nlg_f16 vcc_hi, v2
-// GFX11: v_cmpx_nlg_f16_e32 vcc_hi, v2           ; encoding: [0x6b,0x04,0x14,0x7d]
+v_cmpx_nlg_f16 exec_lo, v2.l
+// GFX11: v_cmpx_nlg_f16_e32 exec_lo, v2.l        ; encoding: [0x7e,0x04,0x14,0x7d]
 
-v_cmpx_nlg_f16 ttmp15, v2
-// GFX11: v_cmpx_nlg_f16_e32 ttmp15, v2           ; encoding: [0x7b,0x04,0x14,0x7d]
+v_cmpx_nlg_f16 exec_hi, v2.l
+// GFX11: v_cmpx_nlg_f16_e32 exec_hi, v2.l        ; encoding: [0x7f,0x04,0x14,0x7d]
 
-v_cmpx_nlg_f16 m0, v2
-// GFX11: v_cmpx_nlg_f16_e32 m0, v2               ; encoding: [0x7d,0x04,0x14,0x7d]
+v_cmpx_nlg_f16 null, v2.l
+// GFX11: v_cmpx_nlg_f16_e32 null, v2.l           ; encoding: [0x7c,0x04,0x14,0x7d]
 
-v_cmpx_nlg_f16 exec_lo, v2
-// GFX11: v_cmpx_nlg_f16_e32 exec_lo, v2          ; encoding: [0x7e,0x04,0x14,0x7d]
+v_cmpx_nlg_f16 -1, v2.l
+// GFX11: v_cmpx_nlg_f16_e32 -1, v2.l             ; encoding: [0xc1,0x04,0x14,0x7d]
 
-v_cmpx_nlg_f16 exec_hi, v2
-// GFX11: v_cmpx_nlg_f16_e32 exec_hi, v2          ; encoding: [0x7f,0x04,0x14,0x7d]
+v_cmpx_nlg_f16 0.5, v2.l
+// GFX11: v_cmpx_nlg_f16_e32 0.5, v2.l            ; encoding: [0xf0,0x04,0x14,0x7d]
 
-v_cmpx_nlg_f16 null, v2
-// GFX11: v_cmpx_nlg_f16_e32 null, v2             ; encoding: [0x7c,0x04,0x14,0x7d]
+v_cmpx_nlg_f16 src_scc, v2.l
+// GFX11: v_cmpx_nlg_f16_e32 src_scc, v2.l        ; encoding: [0xfd,0x04,0x14,0x7d]
 
-v_cmpx_nlg_f16 -1, v2
-// GFX11: v_cmpx_nlg_f16_e32 -1, v2               ; encoding: [0xc1,0x04,0x14,0x7d]
+v_cmpx_nlg_f16 0xfe0b, v127.l
+// GFX11: v_cmpx_nlg_f16_e32 0xfe0b, v127.l       ; encoding: [0xff,0xfe,0x14,0x7d,0x0b,0xfe,0x00,0x00]
 
-v_cmpx_nlg_f16 0.5, v2
-// GFX11: v_cmpx_nlg_f16_e32 0.5, v2              ; encoding: [0xf0,0x04,0x14,0x7d]
+v_cmpx_nlg_f16 v1.h, v2.l
+// GFX11: v_cmpx_nlg_f16_e32 v1.h, v2.l           ; encoding: [0x81,0x05,0x14,0x7d]
 
-v_cmpx_nlg_f16 src_scc, v2
-// GFX11: v_cmpx_nlg_f16_e32 src_scc, v2          ; encoding: [0xfd,0x04,0x14,0x7d]
+v_cmpx_nlg_f16 v127.h, v2.l
+// GFX11: v_cmpx_nlg_f16_e32 v127.h, v2.l         ; encoding: [0xff,0x05,0x14,0x7d]
 
-v_cmpx_nlg_f16 0xfe0b, v127
-// GFX11: v_cmpx_nlg_f16_e32 0xfe0b, v127         ; encoding: [0xff,0xfe,0x14,0x7d,0x0b,0xfe,0x00,0x00]
+v_cmpx_nlg_f16 0.5, v127.l
+// GFX11: v_cmpx_nlg_f16_e32 0.5, v127.l          ; encoding: [0xf0,0xfe,0x14,0x7d]
+
+v_cmpx_nlg_f16 src_scc, v2.h
+// GFX11: v_cmpx_nlg_f16_e32 src_scc, v2.h        ; encoding: [0xfd,0x04,0x15,0x7d]
+
+v_cmpx_nlg_f16 0xfe0b, v127.h
+// GFX11: v_cmpx_nlg_f16_e32 0xfe0b, v127.h       ; encoding: [0xff,0xfe,0x15,0x7d,0x0b,0xfe,0x00,0x00]
 
 v_cmpx_nlg_f32 v1, v2
 // GFX11: v_cmpx_nlg_f32_e32 v1, v2               ; encoding: [0x01,0x05,0x34,0x7d]
@@ -3524,50 +3689,65 @@ v_cmpx_nlg_f64 src_scc, v[2:3]
 v_cmpx_nlg_f64 0xaf123456, v[254:255]
 // GFX11: v_cmpx_nlg_f64_e32 0xaf123456, v[254:255] ; encoding: [0xff,0xfc,0x55,0x7d,0x56,0x34,0x12,0xaf]
 
-v_cmpx_nlt_f16 v1, v2
-// GFX11: v_cmpx_nlt_f16_e32 v1, v2               ; encoding: [0x01,0x05,0x1c,0x7d]
+v_cmpx_nlt_f16 v1.l, v2.l
+// GFX11: v_cmpx_nlt_f16_e32 v1.l, v2.l           ; encoding: [0x01,0x05,0x1c,0x7d]
+
+v_cmpx_nlt_f16 v127.l, v2.l
+// GFX11: v_cmpx_nlt_f16_e32 v127.l, v2.l         ; encoding: [0x7f,0x05,0x1c,0x7d]
+
+v_cmpx_nlt_f16 s1, v2.l
+// GFX11: v_cmpx_nlt_f16_e32 s1, v2.l             ; encoding: [0x01,0x04,0x1c,0x7d]
+
+v_cmpx_nlt_f16 s105, v2.l
+// GFX11: v_cmpx_nlt_f16_e32 s105, v2.l           ; encoding: [0x69,0x04,0x1c,0x7d]
 
-v_cmpx_nlt_f16 v127, v2
-// GFX11: v_cmpx_nlt_f16_e32 v127, v2             ; encoding: [0x7f,0x05,0x1c,0x7d]
+v_cmpx_nlt_f16 vcc_lo, v2.l
+// GFX11: v_cmpx_nlt_f16_e32 vcc_lo, v2.l         ; encoding: [0x6a,0x04,0x1c,0x7d]
 
-v_cmpx_nlt_f16 s1, v2
-// GFX11: v_cmpx_nlt_f16_e32 s1, v2               ; encoding: [0x01,0x04,0x1c,0x7d]
+v_cmpx_nlt_f16 vcc_hi, v2.l
+// GFX11: v_cmpx_nlt_f16_e32 vcc_hi, v2.l         ; encoding: [0x6b,0x04,0x1c,0x7d]
 
-v_cmpx_nlt_f16 s105, v2
-// GFX11: v_cmpx_nlt_f16_e32 s105, v2             ; encoding: [0x69,0x04,0x1c,0x7d]
+v_cmpx_nlt_f16 ttmp15, v2.l
+// GFX11: v_cmpx_nlt_f16_e32 ttmp15, v2.l         ; encoding: [0x7b,0x04,0x1c,0x7d]
 
-v_cmpx_nlt_f16 vcc_lo, v2
-// GFX11: v_cmpx_nlt_f16_e32 vcc_lo, v2           ; encoding: [0x6a,0x04,0x1c,0x7d]
+v_cmpx_nlt_f16 m0, v2.l
+// GFX11: v_cmpx_nlt_f16_e32 m0, v2.l             ; encoding: [0x7d,0x04,0x1c,0x7d]
 
-v_cmpx_nlt_f16 vcc_hi, v2
-// GFX11: v_cmpx_nlt_f16_e32 vcc_hi, v2           ; encoding: [0x6b,0x04,0x1c,0x7d]
+v_cmpx_nlt_f16 exec_lo, v2.l
+// GFX11: v_cmpx_nlt_f16_e32 exec_lo, v2.l        ; encoding: [0x7e,0x04,0x1c,0x7d]
 
-v_cmpx_nlt_f16 ttmp15, v2
-// GFX11: v_cmpx_nlt_f16_e32 ttmp15, v2           ; encoding: [0x7b,0x04,0x1c,0x7d]
+v_cmpx_nlt_f16 exec_hi, v2.l
+// GFX11: v_cmpx_nlt_f16_e32 exec_hi, v2.l        ; encoding: [0x7f,0x04,0x1c,0x7d]
 
-v_cmpx_nlt_f16 m0, v2
-// GFX11: v_cmpx_nlt_f16_e32 m0, v2               ; encoding: [0x7d,0x04,0x1c,0x7d]
+v_cmpx_nlt_f16 null, v2.l
+// GFX11: v_cmpx_nlt_f16_e32 null, v2.l           ; encoding: [0x7c,0x04,0x1c,0x7d]
 
-v_cmpx_nlt_f16 exec_lo, v2
-// GFX11: v_cmpx_nlt_f16_e32 exec_lo, v2          ; encoding: [0x7e,0x04,0x1c,0x7d]
+v_cmpx_nlt_f16 -1, v2.l
+// GFX11: v_cmpx_nlt_f16_e32 -1, v2.l             ; encoding: [0xc1,0x04,0x1c,0x7d]
 
-v_cmpx_nlt_f16 exec_hi, v2
-// GFX11: v_cmpx_nlt_f16_e32 exec_hi, v2          ; encoding: [0x7f,0x04,0x1c,0x7d]
+v_cmpx_nlt_f16 0.5, v2.l
+// GFX11: v_cmpx_nlt_f16_e32 0.5, v2.l            ; encoding: [0xf0,0x04,0x1c,0x7d]
 
-v_cmpx_nlt_f16 null, v2
-// GFX11: v_cmpx_nlt_f16_e32 null, v2             ; encoding: [0x7c,0x04,0x1c,0x7d]
+v_cmpx_nlt_f16 src_scc, v2.l
+// GFX11: v_cmpx_nlt_f16_e32 src_scc, v2.l        ; encoding: [0xfd,0x04,0x1c,0x7d]
 
-v_cmpx_nlt_f16 -1, v2
-// GFX11: v_cmpx_nlt_f16_e32 -1, v2               ; encoding: [0xc1,0x04,0x1c,0x7d]
+v_cmpx_nlt_f16 0xfe0b, v127.l
+// GFX11: v_cmpx_nlt_f16_e32 0xfe0b, v127.l       ; encoding: [0xff,0xfe,0x1c,0x7d,0x0b,0xfe,0x00,0x00]
 
-v_cmpx_nlt_f16 0.5, v2
-// GFX11: v_cmpx_nlt_f16_e32 0.5, v2              ; encoding: [0xf0,0x04,0x1c,0x7d]
+v_cmpx_nlt_f16 v1.h, v2.l
+// GFX11: v_cmpx_nlt_f16_e32 v1.h, v2.l           ; encoding: [0x81,0x05,0x1c,0x7d]
 
-v_cmpx_nlt_f16 src_scc, v2
-// GFX11: v_cmpx_nlt_f16_e32 src_scc, v2          ; encoding: [0xfd,0x04,0x1c,0x7d]
+v_cmpx_nlt_f16 v127.h, v2.l
+// GFX11: v_cmpx_nlt_f16_e32 v127.h, v2.l         ; encoding: [0xff,0x05,0x1c,0x7d]
 
-v_cmpx_nlt_f16 0xfe0b, v127
-// GFX11: v_cmpx_nlt_f16_e32 0xfe0b, v127         ; encoding: [0xff,0xfe,0x1c,0x7d,0x0b,0xfe,0x00,0x00]
+v_cmpx_nlt_f16 0.5, v127.l
+// GFX11: v_cmpx_nlt_f16_e32 0.5, v127.l          ; encoding: [0xf0,0xfe,0x1c,0x7d]
+
+v_cmpx_nlt_f16 src_scc, v2.h
+// GFX11: v_cmpx_nlt_f16_e32 src_scc, v2.h        ; encoding: [0xfd,0x04,0x1d,0x7d]
+
+v_cmpx_nlt_f16 0xfe0b, v127.h
+// GFX11: v_cmpx_nlt_f16_e32 0xfe0b, v127.h       ; encoding: [0xff,0xfe,0x1d,0x7d,0x0b,0xfe,0x00,0x00]
 
 v_cmpx_nlt_f32 v1, v2
 // GFX11: v_cmpx_nlt_f32_e32 v1, v2               ; encoding: [0x01,0x05,0x3c,0x7d]
@@ -3650,50 +3830,65 @@ v_cmpx_nlt_f64 src_scc, v[2:3]
 v_cmpx_nlt_f64 0xaf123456, v[254:255]
 // GFX11: v_cmpx_nlt_f64_e32 0xaf123456, v[254:255] ; encoding: [0xff,0xfc,0x5d,0x7d,0x56,0x34,0x12,0xaf]
 
-v_cmpx_o_f16 v1, v2
-// GFX11: v_cmpx_o_f16_e32 v1, v2                 ; encoding: [0x01,0x05,0x0e,0x7d]
+v_cmpx_o_f16 v1.l, v2.l
+// GFX11: v_cmpx_o_f16_e32 v1.l, v2.l             ; encoding: [0x01,0x05,0x0e,0x7d]
+
+v_cmpx_o_f16 v127.l, v2.l
+// GFX11: v_cmpx_o_f16_e32 v127.l, v2.l           ; encoding: [0x7f,0x05,0x0e,0x7d]
+
+v_cmpx_o_f16 s1, v2.l
+// GFX11: v_cmpx_o_f16_e32 s1, v2.l               ; encoding: [0x01,0x04,0x0e,0x7d]
+
+v_cmpx_o_f16 s105, v2.l
+// GFX11: v_cmpx_o_f16_e32 s105, v2.l             ; encoding: [0x69,0x04,0x0e,0x7d]
 
-v_cmpx_o_f16 v127, v2
-// GFX11: v_cmpx_o_f16_e32 v127, v2               ; encoding: [0x7f,0x05,0x0e,0x7d]
+v_cmpx_o_f16 vcc_lo, v2.l
+// GFX11: v_cmpx_o_f16_e32 vcc_lo, v2.l           ; encoding: [0x6a,0x04,0x0e,0x7d]
 
-v_cmpx_o_f16 s1, v2
-// GFX11: v_cmpx_o_f16_e32 s1, v2                 ; encoding: [0x01,0x04,0x0e,0x7d]
+v_cmpx_o_f16 vcc_hi, v2.l
+// GFX11: v_cmpx_o_f16_e32 vcc_hi, v2.l           ; encoding: [0x6b,0x04,0x0e,0x7d]
 
-v_cmpx_o_f16 s105, v2
-// GFX11: v_cmpx_o_f16_e32 s105, v2               ; encoding: [0x69,0x04,0x0e,0x7d]
+v_cmpx_o_f16 ttmp15, v2.l
+// GFX11: v_cmpx_o_f16_e32 ttmp15, v2.l           ; encoding: [0x7b,0x04,0x0e,0x7d]
 
-v_cmpx_o_f16 vcc_lo, v2
-// GFX11: v_cmpx_o_f16_e32 vcc_lo, v2             ; encoding: [0x6a,0x04,0x0e,0x7d]
+v_cmpx_o_f16 m0, v2.l
+// GFX11: v_cmpx_o_f16_e32 m0, v2.l               ; encoding: [0x7d,0x04,0x0e,0x7d]
 
-v_cmpx_o_f16 vcc_hi, v2
-// GFX11: v_cmpx_o_f16_e32 vcc_hi, v2             ; encoding: [0x6b,0x04,0x0e,0x7d]
+v_cmpx_o_f16 exec_lo, v2.l
+// GFX11: v_cmpx_o_f16_e32 exec_lo, v2.l          ; encoding: [0x7e,0x04,0x0e,0x7d]
 
-v_cmpx_o_f16 ttmp15, v2
-// GFX11: v_cmpx_o_f16_e32 ttmp15, v2             ; encoding: [0x7b,0x04,0x0e,0x7d]
+v_cmpx_o_f16 exec_hi, v2.l
+// GFX11: v_cmpx_o_f16_e32 exec_hi, v2.l          ; encoding: [0x7f,0x04,0x0e,0x7d]
 
-v_cmpx_o_f16 m0, v2
-// GFX11: v_cmpx_o_f16_e32 m0, v2                 ; encoding: [0x7d,0x04,0x0e,0x7d]
+v_cmpx_o_f16 null, v2.l
+// GFX11: v_cmpx_o_f16_e32 null, v2.l             ; encoding: [0x7c,0x04,0x0e,0x7d]
 
-v_cmpx_o_f16 exec_lo, v2
-// GFX11: v_cmpx_o_f16_e32 exec_lo, v2            ; encoding: [0x7e,0x04,0x0e,0x7d]
+v_cmpx_o_f16 -1, v2.l
+// GFX11: v_cmpx_o_f16_e32 -1, v2.l               ; encoding: [0xc1,0x04,0x0e,0x7d]
 
-v_cmpx_o_f16 exec_hi, v2
-// GFX11: v_cmpx_o_f16_e32 exec_hi, v2            ; encoding: [0x7f,0x04,0x0e,0x7d]
+v_cmpx_o_f16 0.5, v2.l
+// GFX11: v_cmpx_o_f16_e32 0.5, v2.l              ; encoding: [0xf0,0x04,0x0e,0x7d]
 
-v_cmpx_o_f16 null, v2
-// GFX11: v_cmpx_o_f16_e32 null, v2               ; encoding: [0x7c,0x04,0x0e,0x7d]
+v_cmpx_o_f16 src_scc, v2.l
+// GFX11: v_cmpx_o_f16_e32 src_scc, v2.l          ; encoding: [0xfd,0x04,0x0e,0x7d]
 
-v_cmpx_o_f16 -1, v2
-// GFX11: v_cmpx_o_f16_e32 -1, v2                 ; encoding: [0xc1,0x04,0x0e,0x7d]
+v_cmpx_o_f16 0xfe0b, v127.l
+// GFX11: v_cmpx_o_f16_e32 0xfe0b, v127.l         ; encoding: [0xff,0xfe,0x0e,0x7d,0x0b,0xfe,0x00,0x00]
 
-v_cmpx_o_f16 0.5, v2
-// GFX11: v_cmpx_o_f16_e32 0.5, v2                ; encoding: [0xf0,0x04,0x0e,0x7d]
+v_cmpx_o_f16 v1.h, v2.l
+// GFX11: v_cmpx_o_f16_e32 v1.h, v2.l             ; encoding: [0x81,0x05,0x0e,0x7d]
 
-v_cmpx_o_f16 src_scc, v2
-// GFX11: v_cmpx_o_f16_e32 src_scc, v2            ; encoding: [0xfd,0x04,0x0e,0x7d]
+v_cmpx_o_f16 v127.h, v2.l
+// GFX11: v_cmpx_o_f16_e32 v127.h, v2.l           ; encoding: [0xff,0x05,0x0e,0x7d]
 
-v_cmpx_o_f16 0xfe0b, v127
-// GFX11: v_cmpx_o_f16_e32 0xfe0b, v127           ; encoding: [0xff,0xfe,0x0e,0x7d,0x0b,0xfe,0x00,0x00]
+v_cmpx_o_f16 0.5, v127.l
+// GFX11: v_cmpx_o_f16_e32 0.5, v127.l            ; encoding: [0xf0,0xfe,0x0e,0x7d]
+
+v_cmpx_o_f16 src_scc, v2.h
+// GFX11: v_cmpx_o_f16_e32 src_scc, v2.h          ; encoding: [0xfd,0x04,0x0f,0x7d]
+
+v_cmpx_o_f16 0xfe0b, v127.h
+// GFX11: v_cmpx_o_f16_e32 0xfe0b, v127.h         ; encoding: [0xff,0xfe,0x0f,0x7d,0x0b,0xfe,0x00,0x00]
 
 v_cmpx_o_f32 v1, v2
 // GFX11: v_cmpx_o_f32_e32 v1, v2                 ; encoding: [0x01,0x05,0x2e,0x7d]
@@ -3776,50 +3971,80 @@ v_cmpx_o_f64 src_scc, v[2:3]
 v_cmpx_o_f64 0xaf123456, v[254:255]
 // GFX11: v_cmpx_o_f64_e32 0xaf123456, v[254:255] ; encoding: [0xff,0xfc,0x4f,0x7d,0x56,0x34,0x12,0xaf]
 
-v_cmpx_t_f16 v1, v2
-// GFX11: v_cmpx_t_f16_e32 v1, v2                 ; encoding: [0x01,0x05,0x1e,0x7d]
+v_cmpx_t_f16 v1.l, v2.l
+// GFX11: v_cmpx_t_f16_e32 v1.l, v2.l             ; encoding: [0x01,0x05,0x1e,0x7d]
 
-v_cmpx_t_f16 v127, v2
-// GFX11: v_cmpx_t_f16_e32 v127, v2               ; encoding: [0x7f,0x05,0x1e,0x7d]
+v_cmpx_t_f16 v127.l, v2.l
+// GFX11: v_cmpx_t_f16_e32 v127.l, v2.l           ; encoding: [0x7f,0x05,0x1e,0x7d]
 
-v_cmpx_t_f16 s1, v2
-// GFX11: v_cmpx_t_f16_e32 s1, v2                 ; encoding: [0x01,0x04,0x1e,0x7d]
+v_cmpx_t_f16 s1, v2.l
+// GFX11: v_cmpx_t_f16_e32 s1, v2.l               ; encoding: [0x01,0x04,0x1e,0x7d]
 
-v_cmpx_t_f16 s105, v2
-// GFX11: v_cmpx_t_f16_e32 s105, v2               ; encoding: [0x69,0x04,0x1e,0x7d]
+v_cmpx_t_f16 s105, v2.l
+// GFX11: v_cmpx_t_f16_e32 s105, v2.l             ; encoding: [0x69,0x04,0x1e,0x7d]
 
-v_cmpx_t_f16 vcc_lo, v2
-// GFX11: v_cmpx_t_f16_e32 vcc_lo, v2             ; encoding: [0x6a,0x04,0x1e,0x7d]
+v_cmpx_t_f16 vcc_lo, v2.l
+// GFX11: v_cmpx_t_f16_e32 vcc_lo, v2.l           ; encoding: [0x6a,0x04,0x1e,0x7d]
 
-v_cmpx_t_f16 vcc_hi, v2
-// GFX11: v_cmpx_t_f16_e32 vcc_hi, v2             ; encoding: [0x6b,0x04,0x1e,0x7d]
+v_cmpx_t_f16 vcc_hi, v2.l
+// GFX11: v_cmpx_t_f16_e32 vcc_hi, v2.l           ; encoding: [0x6b,0x04,0x1e,0x7d]
 
-v_cmpx_t_f16 ttmp15, v2
-// GFX11: v_cmpx_t_f16_e32 ttmp15, v2             ; encoding: [0x7b,0x04,0x1e,0x7d]
+v_cmpx_t_f16 ttmp15, v2.l
+// GFX11: v_cmpx_t_f16_e32 ttmp15, v2.l           ; encoding: [0x7b,0x04,0x1e,0x7d]
 
-v_cmpx_t_f16 m0, v2
-// GFX11: v_cmpx_t_f16_e32 m0, v2                 ; encoding: [0x7d,0x04,0x1e,0x7d]
+v_cmpx_t_f16 m0, v2.l
+// GFX11: v_cmpx_t_f16_e32 m0, v2.l               ; encoding: [0x7d,0x04,0x1e,0x7d]
 
-v_cmpx_t_f16 exec_lo, v2
-// GFX11: v_cmpx_t_f16_e32 exec_lo, v2            ; encoding: [0x7e,0x04,0x1e,0x7d]
+v_cmpx_t_f16 exec_lo, v2.l
+// GFX11: v_cmpx_t_f16_e32 exec_lo, v2.l          ; encoding: [0x7e,0x04,0x1e,0x7d]
 
-v_cmpx_t_f16 exec_hi, v2
-// GFX11: v_cmpx_t_f16_e32 exec_hi, v2            ; encoding: [0x7f,0x04,0x1e,0x7d]
+v_cmpx_t_f16 exec_hi, v2.l
+// GFX11: v_cmpx_t_f16_e32 exec_hi, v2.l          ; encoding: [0x7f,0x04,0x1e,0x7d]
 
-v_cmpx_t_f16 null, v2
-// GFX11: v_cmpx_t_f16_e32 null, v2               ; encoding: [0x7c,0x04,0x1e,0x7d]
+v_cmpx_t_f16 null, v2.l
+// GFX11: v_cmpx_t_f16_e32 null, v2.l             ; encoding: [0x7c,0x04,0x1e,0x7d]
 
-v_cmpx_t_f16 -1, v2
-// GFX11: v_cmpx_t_f16_e32 -1, v2                 ; encoding: [0xc1,0x04,0x1e,0x7d]
+v_cmpx_t_f16 -1, v2.l
+// GFX11: v_cmpx_t_f16_e32 -1, v2.l               ; encoding: [0xc1,0x04,0x1e,0x7d]
 
-v_cmpx_t_f16 0.5, v2
-// GFX11: v_cmpx_t_f16_e32 0.5, v2                ; encoding: [0xf0,0x04,0x1e,0x7d]
+v_cmpx_t_f16 0.5, v2.l
+// GFX11: v_cmpx_t_f16_e32 0.5, v2.l              ; encoding: [0xf0,0x04,0x1e,0x7d]
 
-v_cmpx_t_f16 src_scc, v2
-// GFX11: v_cmpx_t_f16_e32 src_scc, v2            ; encoding: [0xfd,0x04,0x1e,0x7d]
+v_cmpx_t_f16 src_scc, v2.l
+// GFX11: v_cmpx_t_f16_e32 src_scc, v2.l          ; encoding: [0xfd,0x04,0x1e,0x7d]
 
-v_cmpx_t_f16 0xfe0b, v127
-// GFX11: v_cmpx_t_f16_e32 0xfe0b, v127           ; encoding: [0xff,0xfe,0x1e,0x7d,0x0b,0xfe,0x00,0x00]
+v_cmpx_t_f16 0xfe0b, v127.l
+// GFX11: v_cmpx_t_f16_e32 0xfe0b, v127.l         ; encoding: [0xff,0xfe,0x1e,0x7d,0x0b,0xfe,0x00,0x00]
+
+v_cmpx_t_f16 v1.h, v2.l
+// GFX11: v_cmpx_t_f16_e32 v1.h, v2.l             ; encoding: [0x81,0x05,0x1e,0x7d]
+
+v_cmpx_tru_f16 v1.h, v2.l
+// GFX11: v_cmpx_t_f16_e32 v1.h, v2.l             ; encoding: [0x81,0x05,0x1e,0x7d]
+
+v_cmpx_t_f16 v127.h, v2.l
+// GFX11: v_cmpx_t_f16_e32 v127.h, v2.l           ; encoding: [0xff,0x05,0x1e,0x7d]
+
+v_cmpx_tru_f16 v127.h, v2.l
+// GFX11: v_cmpx_t_f16_e32 v127.h, v2.l           ; encoding: [0xff,0x05,0x1e,0x7d]
+
+v_cmpx_t_f16 0.5, v127.l
+// GFX11: v_cmpx_t_f16_e32 0.5, v127.l            ; encoding: [0xf0,0xfe,0x1e,0x7d]
+
+v_cmpx_tru_f16 0.5, v127.l
+// GFX11: v_cmpx_t_f16_e32 0.5, v127.l            ; encoding: [0xf0,0xfe,0x1e,0x7d]
+
+v_cmpx_t_f16 src_scc, v2.h
+// GFX11: v_cmpx_t_f16_e32 src_scc, v2.h          ; encoding: [0xfd,0x04,0x1f,0x7d]
+
+v_cmpx_tru_f16 src_scc, v2.h
+// GFX11: v_cmpx_t_f16_e32 src_scc, v2.h          ; encoding: [0xfd,0x04,0x1f,0x7d]
+
+v_cmpx_t_f16 0xfe0b, v127.h
+// GFX11: v_cmpx_t_f16_e32 0xfe0b, v127.h         ; encoding: [0xff,0xfe,0x1f,0x7d,0x0b,0xfe,0x00,0x00]
+
+v_cmpx_tru_f16 0xfe0b, v127.h
+// GFX11: v_cmpx_t_f16_e32 0xfe0b, v127.h         ; encoding: [0xff,0xfe,0x1f,0x7d,0x0b,0xfe,0x00,0x00]
 
 v_cmpx_t_f32 v1, v2
 // GFX11: v_cmpx_t_f32_e32 v1, v2                 ; encoding: [0x01,0x05,0x3e,0x7d]
@@ -4064,41 +4289,41 @@ v_cmpx_t_u64 src_scc, v[2:3]
 v_cmpx_t_u64 0xaf123456, v[254:255]
 // GFX11: v_cmpx_t_u64_e32 0xaf123456, v[254:255] ; encoding: [0xff,0xfc,0xbf,0x7d,0x56,0x34,0x12,0xaf]
 
-v_cmpx_tru_f16 v1, v2
-// GFX11: v_cmpx_t_f16_e32 v1, v2                 ; encoding: [0x01,0x05,0x1e,0x7d]
+v_cmpx_tru_f16 v1.l, v2.l
+// GFX11: v_cmpx_t_f16_e32 v1.l, v2.l             ; encoding: [0x01,0x05,0x1e,0x7d]
 
-v_cmpx_tru_f16 v127, v2
-// GFX11: v_cmpx_t_f16_e32 v127, v2               ; encoding: [0x7f,0x05,0x1e,0x7d]
+v_cmpx_tru_f16 v127.l, v2.l
+// GFX11: v_cmpx_t_f16_e32 v127.l, v2.l           ; encoding: [0x7f,0x05,0x1e,0x7d]
 
-v_cmpx_tru_f16 s1, v2
-// GFX11: v_cmpx_t_f16_e32 s1, v2                 ; encoding: [0x01,0x04,0x1e,0x7d]
+v_cmpx_tru_f16 s1, v2.l
+// GFX11: v_cmpx_t_f16_e32 s1, v2.l               ; encoding: [0x01,0x04,0x1e,0x7d]
 
-v_cmpx_tru_f16 s105, v2
-// GFX11: v_cmpx_t_f16_e32 s105, v2               ; encoding: [0x69,0x04,0x1e,0x7d]
+v_cmpx_tru_f16 s105, v2.l
+// GFX11: v_cmpx_t_f16_e32 s105, v2.l             ; encoding: [0x69,0x04,0x1e,0x7d]
 
-v_cmpx_tru_f16 vcc_lo, v2
-// GFX11: v_cmpx_t_f16_e32 vcc_lo, v2             ; encoding: [0x6a,0x04,0x1e,0x7d]
+v_cmpx_tru_f16 vcc_lo, v2.l
+// GFX11: v_cmpx_t_f16_e32 vcc_lo, v2.l           ; encoding: [0x6a,0x04,0x1e,0x7d]
 
-v_cmpx_tru_f16 vcc_hi, v2
-// GFX11: v_cmpx_t_f16_e32 vcc_hi, v2             ; encoding: [0x6b,0x04,0x1e,0x7d]
+v_cmpx_tru_f16 vcc_hi, v2.l
+// GFX11: v_cmpx_t_f16_e32 vcc_hi, v2.l           ; encoding: [0x6b,0x04,0x1e,0x7d]
 
-v_cmpx_tru_f16 ttmp15, v2
-// GFX11: v_cmpx_t_f16_e32 ttmp15, v2             ; encoding: [0x7b,0x04,0x1e,0x7d]
+v_cmpx_tru_f16 ttmp15, v2.l
+// GFX11: v_cmpx_t_f16_e32 ttmp15, v2.l           ; encoding: [0x7b,0x04,0x1e,0x7d]
 
-v_cmpx_tru_f16 m0, v2
-// GFX11: v_cmpx_t_f16_e32 m0, v2                 ; encoding: [0x7d,0x04,0x1e,0x7d]
+v_cmpx_tru_f16 m0, v2.l
+// GFX11: v_cmpx_t_f16_e32 m0, v2.l               ; encoding: [0x7d,0x04,0x1e,0x7d]
 
-v_cmpx_tru_f16 exec_lo, v2
-// GFX11: v_cmpx_t_f16_e32 exec_lo, v2            ; encoding: [0x7e,0x04,0x1e,0x7d]
+v_cmpx_tru_f16 exec_lo, v2.l
+// GFX11: v_cmpx_t_f16_e32 exec_lo, v2.l          ; encoding: [0x7e,0x04,0x1e,0x7d]
 
-v_cmpx_tru_f16 exec_hi, v2
-// GFX11: v_cmpx_t_f16_e32 exec_hi, v2            ; encoding: [0x7f,0x04,0x1e,0x7d]
+v_cmpx_tru_f16 exec_hi, v2.l
+// GFX11: v_cmpx_t_f16_e32 exec_hi, v2.l          ; encoding: [0x7f,0x04,0x1e,0x7d]
 
-v_cmpx_tru_f16 null, v2
-// GFX11: v_cmpx_t_f16_e32 null, v2               ; encoding: [0x7c,0x04,0x1e,0x7d]
+v_cmpx_tru_f16 null, v2.l
+// GFX11: v_cmpx_t_f16_e32 null, v2.l             ; encoding: [0x7c,0x04,0x1e,0x7d]
 
-v_cmpx_tru_f16 -1, v2
-// GFX11: v_cmpx_t_f16_e32 -1, v2                 ; encoding: [0xc1,0x04,0x1e,0x7d]
+v_cmpx_tru_f16 -1, v2.l
+// GFX11: v_cmpx_t_f16_e32 -1, v2.l               ; encoding: [0xc1,0x04,0x1e,0x7d]
 
 v_cmpx_tru_f16 0.5, v2
 // GFX11: v_cmpx_t_f16_e32 0.5, v2                ; encoding: [0xf0,0x04,0x1e,0x7d]
@@ -4190,50 +4415,65 @@ v_cmpx_tru_f64 src_scc, v[2:3]
 v_cmpx_tru_f64 0xaf123456, v[254:255]
 // GFX11: v_cmpx_t_f64_e32 0xaf123456, v[254:255] ; encoding: [0xff,0xfc,0x5f,0x7d,0x56,0x34,0x12,0xaf]
 
-v_cmpx_u_f16 v1, v2
-// GFX11: v_cmpx_u_f16_e32 v1, v2                 ; encoding: [0x01,0x05,0x10,0x7d]
+v_cmpx_u_f16 v1.l, v2.l
+// GFX11: v_cmpx_u_f16_e32 v1.l, v2.l             ; encoding: [0x01,0x05,0x10,0x7d]
+
+v_cmpx_u_f16 v127.l, v2.l
+// GFX11: v_cmpx_u_f16_e32 v127.l, v2.l           ; encoding: [0x7f,0x05,0x10,0x7d]
+
+v_cmpx_u_f16 s1, v2.l
+// GFX11: v_cmpx_u_f16_e32 s1, v2.l               ; encoding: [0x01,0x04,0x10,0x7d]
+
+v_cmpx_u_f16 s105, v2.l
+// GFX11: v_cmpx_u_f16_e32 s105, v2.l             ; encoding: [0x69,0x04,0x10,0x7d]
+
+v_cmpx_u_f16 vcc_lo, v2.l
+// GFX11: v_cmpx_u_f16_e32 vcc_lo, v2.l           ; encoding: [0x6a,0x04,0x10,0x7d]
+
+v_cmpx_u_f16 vcc_hi, v2.l
+// GFX11: v_cmpx_u_f16_e32 vcc_hi, v2.l           ; encoding: [0x6b,0x04,0x10,0x7d]
 
-v_cmpx_u_f16 v127, v2
-// GFX11: v_cmpx_u_f16_e32 v127, v2               ; encoding: [0x7f,0x05,0x10,0x7d]
+v_cmpx_u_f16 ttmp15, v2.l
+// GFX11: v_cmpx_u_f16_e32 ttmp15, v2.l           ; encoding: [0x7b,0x04,0x10,0x7d]
 
-v_cmpx_u_f16 s1, v2
-// GFX11: v_cmpx_u_f16_e32 s1, v2                 ; encoding: [0x01,0x04,0x10,0x7d]
+v_cmpx_u_f16 m0, v2.l
+// GFX11: v_cmpx_u_f16_e32 m0, v2.l               ; encoding: [0x7d,0x04,0x10,0x7d]
 
-v_cmpx_u_f16 s105, v2
-// GFX11: v_cmpx_u_f16_e32 s105, v2               ; encoding: [0x69,0x04,0x10,0x7d]
+v_cmpx_u_f16 exec_lo, v2.l
+// GFX11: v_cmpx_u_f16_e32 exec_lo, v2.l          ; encoding: [0x7e,0x04,0x10,0x7d]
 
-v_cmpx_u_f16 vcc_lo, v2
-// GFX11: v_cmpx_u_f16_e32 vcc_lo, v2             ; encoding: [0x6a,0x04,0x10,0x7d]
+v_cmpx_u_f16 exec_hi, v2.l
+// GFX11: v_cmpx_u_f16_e32 exec_hi, v2.l          ; encoding: [0x7f,0x04,0x10,0x7d]
 
-v_cmpx_u_f16 vcc_hi, v2
-// GFX11: v_cmpx_u_f16_e32 vcc_hi, v2             ; encoding: [0x6b,0x04,0x10,0x7d]
+v_cmpx_u_f16 null, v2.l
+// GFX11: v_cmpx_u_f16_e32 null, v2.l             ; encoding: [0x7c,0x04,0x10,0x7d]
 
-v_cmpx_u_f16 ttmp15, v2
-// GFX11: v_cmpx_u_f16_e32 ttmp15, v2             ; encoding: [0x7b,0x04,0x10,0x7d]
+v_cmpx_u_f16 -1, v2.l
+// GFX11: v_cmpx_u_f16_e32 -1, v2.l               ; encoding: [0xc1,0x04,0x10,0x7d]
 
-v_cmpx_u_f16 m0, v2
-// GFX11: v_cmpx_u_f16_e32 m0, v2                 ; encoding: [0x7d,0x04,0x10,0x7d]
+v_cmpx_u_f16 0.5, v2.l
+// GFX11: v_cmpx_u_f16_e32 0.5, v2.l              ; encoding: [0xf0,0x04,0x10,0x7d]
 
-v_cmpx_u_f16 exec_lo, v2
-// GFX11: v_cmpx_u_f16_e32 exec_lo, v2            ; encoding: [0x7e,0x04,0x10,0x7d]
+v_cmpx_u_f16 src_scc, v2.l
+// GFX11: v_cmpx_u_f16_e32 src_scc, v2.l          ; encoding: [0xfd,0x04,0x10,0x7d]
 
-v_cmpx_u_f16 exec_hi, v2
-// GFX11: v_cmpx_u_f16_e32 exec_hi, v2            ; encoding: [0x7f,0x04,0x10,0x7d]
+v_cmpx_u_f16 0xfe0b, v127.l
+// GFX11: v_cmpx_u_f16_e32 0xfe0b, v127.l         ; encoding: [0xff,0xfe,0x10,0x7d,0x0b,0xfe,0x00,0x00]
 
-v_cmpx_u_f16 null, v2
-// GFX11: v_cmpx_u_f16_e32 null, v2               ; encoding: [0x7c,0x04,0x10,0x7d]
+v_cmpx_u_f16 v1.h, v2.l
+// GFX11: v_cmpx_u_f16_e32 v1.h, v2.l             ; encoding: [0x81,0x05,0x10,0x7d]
 
-v_cmpx_u_f16 -1, v2
-// GFX11: v_cmpx_u_f16_e32 -1, v2                 ; encoding: [0xc1,0x04,0x10,0x7d]
+v_cmpx_u_f16 v127.h, v2.l
+// GFX11: v_cmpx_u_f16_e32 v127.h, v2.l           ; encoding: [0xff,0x05,0x10,0x7d]
 
-v_cmpx_u_f16 0.5, v2
-// GFX11: v_cmpx_u_f16_e32 0.5, v2                ; encoding: [0xf0,0x04,0x10,0x7d]
+v_cmpx_u_f16 0.5, v127.l
+// GFX11: v_cmpx_u_f16_e32 0.5, v127.l            ; encoding: [0xf0,0xfe,0x10,0x7d]
 
-v_cmpx_u_f16 src_scc, v2
-// GFX11: v_cmpx_u_f16_e32 src_scc, v2            ; encoding: [0xfd,0x04,0x10,0x7d]
+v_cmpx_u_f16 src_scc, v2.h
+// GFX11: v_cmpx_u_f16_e32 src_scc, v2.h          ; encoding: [0xfd,0x04,0x11,0x7d]
 
-v_cmpx_u_f16 0xfe0b, v127
-// GFX11: v_cmpx_u_f16_e32 0xfe0b, v127           ; encoding: [0xff,0xfe,0x10,0x7d,0x0b,0xfe,0x00,0x00]
+v_cmpx_u_f16 0xfe0b, v127.h
+// GFX11: v_cmpx_u_f16_e32 0xfe0b, v127.h         ; encoding: [0xff,0xfe,0x11,0x7d,0x0b,0xfe,0x00,0x00]
 
 v_cmpx_u_f32 v1, v2
 // GFX11: v_cmpx_u_f32_e32 v1, v2                 ; encoding: [0x01,0x05,0x30,0x7d]
diff --git a/llvm/test/MC/AMDGPU/gfx11_asm_vopcx_dpp16.s b/llvm/test/MC/AMDGPU/gfx11_asm_vopcx_dpp16.s
index ddaa30af953b8..1864a32c9f133 100644
--- a/llvm/test/MC/AMDGPU/gfx11_asm_vopcx_dpp16.s
+++ b/llvm/test/MC/AMDGPU/gfx11_asm_vopcx_dpp16.s
@@ -95,47 +95,56 @@ v_cmpx_class_f32 v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
 v_cmpx_class_f32 -|v255|, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
 // GFX11: v_cmpx_class_f32 -|v255|, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xfa,0xfe,0xfd,0x7d,0xff,0x6f,0x35,0x30]
 
-v_cmpx_eq_f16 v1, v2 quad_perm:[3,2,1,0]
-// GFX11: v_cmpx_eq_f16 v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x04,0x7d,0x01,0x1b,0x00,0xff]
+v_cmpx_eq_f16 v1.l, v2.l quad_perm:[3,2,1,0]
+// GFX11: v_cmpx_eq_f16 v1.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x04,0x7d,0x01,0x1b,0x00,0xff]
 
-v_cmpx_eq_f16 v1, v2 quad_perm:[0,1,2,3]
-// GFX11: v_cmpx_eq_f16 v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x04,0x7d,0x01,0xe4,0x00,0xff]
+v_cmpx_eq_f16 v1.l, v2.l quad_perm:[0,1,2,3]
+// GFX11: v_cmpx_eq_f16 v1.l, v2.l quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x04,0x7d,0x01,0xe4,0x00,0xff]
 
-v_cmpx_eq_f16 v1, v2 row_mirror
-// GFX11: v_cmpx_eq_f16 v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x04,0x7d,0x01,0x40,0x01,0xff]
+v_cmpx_eq_f16 v1.l, v2.l row_mirror
+// GFX11: v_cmpx_eq_f16 v1.l, v2.l row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x04,0x7d,0x01,0x40,0x01,0xff]
 
-v_cmpx_eq_f16 v1, v2 row_half_mirror
-// GFX11: v_cmpx_eq_f16 v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x04,0x7d,0x01,0x41,0x01,0xff]
+v_cmpx_eq_f16 v1.l, v2.l row_half_mirror
+// GFX11: v_cmpx_eq_f16 v1.l, v2.l row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x04,0x7d,0x01,0x41,0x01,0xff]
 
-v_cmpx_eq_f16 v1, v2 row_shl:1
-// GFX11: v_cmpx_eq_f16 v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x04,0x7d,0x01,0x01,0x01,0xff]
+v_cmpx_eq_f16 v1.l, v2.l row_shl:1
+// GFX11: v_cmpx_eq_f16 v1.l, v2.l row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x04,0x7d,0x01,0x01,0x01,0xff]
 
-v_cmpx_eq_f16 v1, v2 row_shl:15
-// GFX11: v_cmpx_eq_f16 v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x04,0x7d,0x01,0x0f,0x01,0xff]
+v_cmpx_eq_f16 v1.l, v2.l row_shl:15
+// GFX11: v_cmpx_eq_f16 v1.l, v2.l row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x04,0x7d,0x01,0x0f,0x01,0xff]
 
-v_cmpx_eq_f16 v1, v2 row_shr:1
-// GFX11: v_cmpx_eq_f16 v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x04,0x7d,0x01,0x11,0x01,0xff]
+v_cmpx_eq_f16 v1.l, v2.l row_shr:1
+// GFX11: v_cmpx_eq_f16 v1.l, v2.l row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x04,0x7d,0x01,0x11,0x01,0xff]
 
-v_cmpx_eq_f16 v1, v2 row_shr:15
-// GFX11: v_cmpx_eq_f16 v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x04,0x7d,0x01,0x1f,0x01,0xff]
+v_cmpx_eq_f16 v1.l, v2.l row_shr:15
+// GFX11: v_cmpx_eq_f16 v1.l, v2.l row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x04,0x7d,0x01,0x1f,0x01,0xff]
 
-v_cmpx_eq_f16 v1, v2 row_ror:1
-// GFX11: v_cmpx_eq_f16 v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x04,0x7d,0x01,0x21,0x01,0xff]
+v_cmpx_eq_f16 v1.l, v2.l row_ror:1
+// GFX11: v_cmpx_eq_f16 v1.l, v2.l row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x04,0x7d,0x01,0x21,0x01,0xff]
 
-v_cmpx_eq_f16 v1, v2 row_ror:15
-// GFX11: v_cmpx_eq_f16 v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x04,0x7d,0x01,0x2f,0x01,0xff]
+v_cmpx_eq_f16 v1.l, v2.l row_ror:15
+// GFX11: v_cmpx_eq_f16 v1.l, v2.l row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x04,0x7d,0x01,0x2f,0x01,0xff]
 
-v_cmpx_eq_f16 v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
-// GFX11: v_cmpx_eq_f16 v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x04,0x7d,0x01,0x50,0x01,0xff]
+v_cmpx_eq_f16 v1.l, v2.l row_share:0 row_mask:0xf bank_mask:0xf
+// GFX11: v_cmpx_eq_f16 v1.l, v2.l row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x04,0x7d,0x01,0x50,0x01,0xff]
 
-v_cmpx_eq_f16 v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
-// GFX11: v_cmpx_eq_f16 v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x04,0x7d,0x01,0x5f,0x01,0x01]
+v_cmpx_eq_f16 v1.l, v2.l row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: v_cmpx_eq_f16 v1.l, v2.l row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x04,0x7d,0x01,0x5f,0x01,0x01]
 
-v_cmpx_eq_f16 v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
-// GFX11: v_cmpx_eq_f16 v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0xfa,0x04,0x04,0x7d,0x01,0x60,0x09,0x13]
+v_cmpx_eq_f16 v1.l, v2.l row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1
+// GFX11: v_cmpx_eq_f16 v1.l, v2.l row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0xfa,0x04,0x04,0x7d,0x01,0x60,0x09,0x13]
 
-v_cmpx_eq_f16 -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
-// GFX11: v_cmpx_eq_f16 -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xfa,0xfe,0x04,0x7d,0x7f,0x6f,0xf5,0x30]
+v_cmpx_eq_f16 -|v127.l|, -|v127.l| row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1
+// GFX11: v_cmpx_eq_f16 -|v127.l|, -|v127.l| row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xfa,0xfe,0x04,0x7d,0x7f,0x6f,0xf5,0x30]
+
+v_cmpx_eq_f16 v127.l, v127.l row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: v_cmpx_eq_f16 v127.l, v127.l row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0xfe,0x04,0x7d,0x7f,0x5f,0x01,0x01]
+
+v_cmpx_eq_f16 v1.h, v2.h row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: v_cmpx_eq_f16 v1.h, v2.h row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0xfa,0x04,0x05,0x7d,0x81,0x60,0x09,0x13]
+
+v_cmpx_eq_f16 -|v127.h|, -|v127.h| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: v_cmpx_eq_f16 -|v127.h|, -|v127.h| row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xfa,0xfe,0x05,0x7d,0xff,0x6f,0xf5,0x30]
 
 v_cmpx_eq_f32 v1, v2 quad_perm:[3,2,1,0]
 // GFX11: v_cmpx_eq_f32 v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x24,0x7d,0x01,0x1b,0x00,0xff]
@@ -365,47 +374,56 @@ v_cmpx_eq_u32 v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
 v_cmpx_eq_u32 v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
 // GFX11: v_cmpx_eq_u32 v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xfa,0xfe,0x95,0x7d,0xff,0x6f,0x05,0x30]
 
-v_cmpx_f_f16 v1, v2 quad_perm:[3,2,1,0]
-// GFX11: v_cmpx_f_f16 v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x00,0x7d,0x01,0x1b,0x00,0xff]
+v_cmpx_f_f16 v1.l, v2.l quad_perm:[3,2,1,0]
+// GFX11: v_cmpx_f_f16 v1.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x00,0x7d,0x01,0x1b,0x00,0xff]
+
+v_cmpx_f_f16 v1.l, v2.l quad_perm:[0,1,2,3]
+// GFX11: v_cmpx_f_f16 v1.l, v2.l quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x00,0x7d,0x01,0xe4,0x00,0xff]
 
-v_cmpx_f_f16 v1, v2 quad_perm:[0,1,2,3]
-// GFX11: v_cmpx_f_f16 v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x00,0x7d,0x01,0xe4,0x00,0xff]
+v_cmpx_f_f16 v1.l, v2.l row_mirror
+// GFX11: v_cmpx_f_f16 v1.l, v2.l row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x00,0x7d,0x01,0x40,0x01,0xff]
 
-v_cmpx_f_f16 v1, v2 row_mirror
-// GFX11: v_cmpx_f_f16 v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x00,0x7d,0x01,0x40,0x01,0xff]
+v_cmpx_f_f16 v1.l, v2.l row_half_mirror
+// GFX11: v_cmpx_f_f16 v1.l, v2.l row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x00,0x7d,0x01,0x41,0x01,0xff]
 
-v_cmpx_f_f16 v1, v2 row_half_mirror
-// GFX11: v_cmpx_f_f16 v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x00,0x7d,0x01,0x41,0x01,0xff]
+v_cmpx_f_f16 v1.l, v2.l row_shl:1
+// GFX11: v_cmpx_f_f16 v1.l, v2.l row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x00,0x7d,0x01,0x01,0x01,0xff]
 
-v_cmpx_f_f16 v1, v2 row_shl:1
-// GFX11: v_cmpx_f_f16 v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x00,0x7d,0x01,0x01,0x01,0xff]
+v_cmpx_f_f16 v1.l, v2.l row_shl:15
+// GFX11: v_cmpx_f_f16 v1.l, v2.l row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x00,0x7d,0x01,0x0f,0x01,0xff]
 
-v_cmpx_f_f16 v1, v2 row_shl:15
-// GFX11: v_cmpx_f_f16 v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x00,0x7d,0x01,0x0f,0x01,0xff]
+v_cmpx_f_f16 v1.l, v2.l row_shr:1
+// GFX11: v_cmpx_f_f16 v1.l, v2.l row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x00,0x7d,0x01,0x11,0x01,0xff]
 
-v_cmpx_f_f16 v1, v2 row_shr:1
-// GFX11: v_cmpx_f_f16 v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x00,0x7d,0x01,0x11,0x01,0xff]
+v_cmpx_f_f16 v1.l, v2.l row_shr:15
+// GFX11: v_cmpx_f_f16 v1.l, v2.l row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x00,0x7d,0x01,0x1f,0x01,0xff]
 
-v_cmpx_f_f16 v1, v2 row_shr:15
-// GFX11: v_cmpx_f_f16 v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x00,0x7d,0x01,0x1f,0x01,0xff]
+v_cmpx_f_f16 v1.l, v2.l row_ror:1
+// GFX11: v_cmpx_f_f16 v1.l, v2.l row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x00,0x7d,0x01,0x21,0x01,0xff]
 
-v_cmpx_f_f16 v1, v2 row_ror:1
-// GFX11: v_cmpx_f_f16 v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x00,0x7d,0x01,0x21,0x01,0xff]
+v_cmpx_f_f16 v1.l, v2.l row_ror:15
+// GFX11: v_cmpx_f_f16 v1.l, v2.l row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x00,0x7d,0x01,0x2f,0x01,0xff]
 
-v_cmpx_f_f16 v1, v2 row_ror:15
-// GFX11: v_cmpx_f_f16 v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x00,0x7d,0x01,0x2f,0x01,0xff]
+v_cmpx_f_f16 v1.l, v2.l row_share:0 row_mask:0xf bank_mask:0xf
+// GFX11: v_cmpx_f_f16 v1.l, v2.l row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x00,0x7d,0x01,0x50,0x01,0xff]
 
-v_cmpx_f_f16 v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
-// GFX11: v_cmpx_f_f16 v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x00,0x7d,0x01,0x50,0x01,0xff]
+v_cmpx_f_f16 v1.l, v2.l row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: v_cmpx_f_f16 v1.l, v2.l row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x00,0x7d,0x01,0x5f,0x01,0x01]
 
-v_cmpx_f_f16 v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
-// GFX11: v_cmpx_f_f16 v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x00,0x7d,0x01,0x5f,0x01,0x01]
+v_cmpx_f_f16 v1.l, v2.l row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1
+// GFX11: v_cmpx_f_f16 v1.l, v2.l row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0xfa,0x04,0x00,0x7d,0x01,0x60,0x09,0x13]
 
-v_cmpx_f_f16 v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
-// GFX11: v_cmpx_f_f16 v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0xfa,0x04,0x00,0x7d,0x01,0x60,0x09,0x13]
+v_cmpx_f_f16 -|v127.l|, -|v127.l| row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1
+// GFX11: v_cmpx_f_f16 -|v127.l|, -|v127.l| row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xfa,0xfe,0x00,0x7d,0x7f,0x6f,0xf5,0x30]
 
-v_cmpx_f_f16 -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
-// GFX11: v_cmpx_f_f16 -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xfa,0xfe,0x00,0x7d,0x7f,0x6f,0xf5,0x30]
+v_cmpx_f_f16 v127.l, v127.l row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: v_cmpx_f_f16 v127.l, v127.l row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0xfe,0x00,0x7d,0x7f,0x5f,0x01,0x01]
+
+v_cmpx_f_f16 v1.h, v2.h row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: v_cmpx_f_f16 v1.h, v2.h row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0xfa,0x04,0x01,0x7d,0x81,0x60,0x09,0x13]
+
+v_cmpx_f_f16 -|v127.h|, -|v127.h| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: v_cmpx_f_f16 -|v127.h|, -|v127.h| row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xfa,0xfe,0x01,0x7d,0xff,0x6f,0xf5,0x30]
 
 v_cmpx_f_f32 v1, v2 quad_perm:[3,2,1,0]
 // GFX11: v_cmpx_f_f32 v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x20,0x7d,0x01,0x1b,0x00,0xff]
@@ -533,47 +551,56 @@ v_cmpx_f_u32 v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
 v_cmpx_f_u32 v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
 // GFX11: v_cmpx_f_u32 v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xfa,0xfe,0x91,0x7d,0xff,0x6f,0x05,0x30]
 
-v_cmpx_ge_f16 v1, v2 quad_perm:[3,2,1,0]
-// GFX11: v_cmpx_ge_f16 v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0c,0x7d,0x01,0x1b,0x00,0xff]
+v_cmpx_ge_f16 v1.l, v2.l quad_perm:[3,2,1,0]
+// GFX11: v_cmpx_ge_f16 v1.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0c,0x7d,0x01,0x1b,0x00,0xff]
+
+v_cmpx_ge_f16 v1.l, v2.l quad_perm:[0,1,2,3]
+// GFX11: v_cmpx_ge_f16 v1.l, v2.l quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0c,0x7d,0x01,0xe4,0x00,0xff]
 
-v_cmpx_ge_f16 v1, v2 quad_perm:[0,1,2,3]
-// GFX11: v_cmpx_ge_f16 v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0c,0x7d,0x01,0xe4,0x00,0xff]
+v_cmpx_ge_f16 v1.l, v2.l row_mirror
+// GFX11: v_cmpx_ge_f16 v1.l, v2.l row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0c,0x7d,0x01,0x40,0x01,0xff]
 
-v_cmpx_ge_f16 v1, v2 row_mirror
-// GFX11: v_cmpx_ge_f16 v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0c,0x7d,0x01,0x40,0x01,0xff]
+v_cmpx_ge_f16 v1.l, v2.l row_half_mirror
+// GFX11: v_cmpx_ge_f16 v1.l, v2.l row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0c,0x7d,0x01,0x41,0x01,0xff]
 
-v_cmpx_ge_f16 v1, v2 row_half_mirror
-// GFX11: v_cmpx_ge_f16 v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0c,0x7d,0x01,0x41,0x01,0xff]
+v_cmpx_ge_f16 v1.l, v2.l row_shl:1
+// GFX11: v_cmpx_ge_f16 v1.l, v2.l row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0c,0x7d,0x01,0x01,0x01,0xff]
 
-v_cmpx_ge_f16 v1, v2 row_shl:1
-// GFX11: v_cmpx_ge_f16 v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0c,0x7d,0x01,0x01,0x01,0xff]
+v_cmpx_ge_f16 v1.l, v2.l row_shl:15
+// GFX11: v_cmpx_ge_f16 v1.l, v2.l row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0c,0x7d,0x01,0x0f,0x01,0xff]
 
-v_cmpx_ge_f16 v1, v2 row_shl:15
-// GFX11: v_cmpx_ge_f16 v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0c,0x7d,0x01,0x0f,0x01,0xff]
+v_cmpx_ge_f16 v1.l, v2.l row_shr:1
+// GFX11: v_cmpx_ge_f16 v1.l, v2.l row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0c,0x7d,0x01,0x11,0x01,0xff]
 
-v_cmpx_ge_f16 v1, v2 row_shr:1
-// GFX11: v_cmpx_ge_f16 v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0c,0x7d,0x01,0x11,0x01,0xff]
+v_cmpx_ge_f16 v1.l, v2.l row_shr:15
+// GFX11: v_cmpx_ge_f16 v1.l, v2.l row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0c,0x7d,0x01,0x1f,0x01,0xff]
 
-v_cmpx_ge_f16 v1, v2 row_shr:15
-// GFX11: v_cmpx_ge_f16 v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0c,0x7d,0x01,0x1f,0x01,0xff]
+v_cmpx_ge_f16 v1.l, v2.l row_ror:1
+// GFX11: v_cmpx_ge_f16 v1.l, v2.l row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0c,0x7d,0x01,0x21,0x01,0xff]
 
-v_cmpx_ge_f16 v1, v2 row_ror:1
-// GFX11: v_cmpx_ge_f16 v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0c,0x7d,0x01,0x21,0x01,0xff]
+v_cmpx_ge_f16 v1.l, v2.l row_ror:15
+// GFX11: v_cmpx_ge_f16 v1.l, v2.l row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0c,0x7d,0x01,0x2f,0x01,0xff]
 
-v_cmpx_ge_f16 v1, v2 row_ror:15
-// GFX11: v_cmpx_ge_f16 v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0c,0x7d,0x01,0x2f,0x01,0xff]
+v_cmpx_ge_f16 v1.l, v2.l row_share:0 row_mask:0xf bank_mask:0xf
+// GFX11: v_cmpx_ge_f16 v1.l, v2.l row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0c,0x7d,0x01,0x50,0x01,0xff]
 
-v_cmpx_ge_f16 v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
-// GFX11: v_cmpx_ge_f16 v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0c,0x7d,0x01,0x50,0x01,0xff]
+v_cmpx_ge_f16 v1.l, v2.l row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: v_cmpx_ge_f16 v1.l, v2.l row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x0c,0x7d,0x01,0x5f,0x01,0x01]
 
-v_cmpx_ge_f16 v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
-// GFX11: v_cmpx_ge_f16 v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x0c,0x7d,0x01,0x5f,0x01,0x01]
+v_cmpx_ge_f16 v1.l, v2.l row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1
+// GFX11: v_cmpx_ge_f16 v1.l, v2.l row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0xfa,0x04,0x0c,0x7d,0x01,0x60,0x09,0x13]
 
-v_cmpx_ge_f16 v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
-// GFX11: v_cmpx_ge_f16 v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0xfa,0x04,0x0c,0x7d,0x01,0x60,0x09,0x13]
+v_cmpx_ge_f16 -|v127.l|, -|v127.l| row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1
+// GFX11: v_cmpx_ge_f16 -|v127.l|, -|v127.l| row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xfa,0xfe,0x0c,0x7d,0x7f,0x6f,0xf5,0x30]
 
-v_cmpx_ge_f16 -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
-// GFX11: v_cmpx_ge_f16 -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xfa,0xfe,0x0c,0x7d,0x7f,0x6f,0xf5,0x30]
+v_cmpx_ge_f16 v127.l, v127.l row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: v_cmpx_ge_f16 v127.l, v127.l row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0xfe,0x0c,0x7d,0x7f,0x5f,0x01,0x01]
+
+v_cmpx_ge_f16 v1.h, v2.h row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: v_cmpx_ge_f16 v1.h, v2.h row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0xfa,0x04,0x0d,0x7d,0x81,0x60,0x09,0x13]
+
+v_cmpx_ge_f16 -|v127.h|, -|v127.h| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: v_cmpx_ge_f16 -|v127.h|, -|v127.h| row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xfa,0xfe,0x0d,0x7d,0xff,0x6f,0xf5,0x30]
 
 v_cmpx_ge_f32 v1, v2 quad_perm:[3,2,1,0]
 // GFX11: v_cmpx_ge_f32 v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x2c,0x7d,0x01,0x1b,0x00,0xff]
@@ -803,47 +830,56 @@ v_cmpx_ge_u32 v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
 v_cmpx_ge_u32 v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
 // GFX11: v_cmpx_ge_u32 v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xfa,0xfe,0x9d,0x7d,0xff,0x6f,0x05,0x30]
 
-v_cmpx_gt_f16 v1, v2 quad_perm:[3,2,1,0]
-// GFX11: v_cmpx_gt_f16 v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x08,0x7d,0x01,0x1b,0x00,0xff]
+v_cmpx_gt_f16 v1.l, v2.l quad_perm:[3,2,1,0]
+// GFX11: v_cmpx_gt_f16 v1.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x08,0x7d,0x01,0x1b,0x00,0xff]
+
+v_cmpx_gt_f16 v1.l, v2.l quad_perm:[0,1,2,3]
+// GFX11: v_cmpx_gt_f16 v1.l, v2.l quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x08,0x7d,0x01,0xe4,0x00,0xff]
 
-v_cmpx_gt_f16 v1, v2 quad_perm:[0,1,2,3]
-// GFX11: v_cmpx_gt_f16 v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x08,0x7d,0x01,0xe4,0x00,0xff]
+v_cmpx_gt_f16 v1.l, v2.l row_mirror
+// GFX11: v_cmpx_gt_f16 v1.l, v2.l row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x08,0x7d,0x01,0x40,0x01,0xff]
 
-v_cmpx_gt_f16 v1, v2 row_mirror
-// GFX11: v_cmpx_gt_f16 v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x08,0x7d,0x01,0x40,0x01,0xff]
+v_cmpx_gt_f16 v1.l, v2.l row_half_mirror
+// GFX11: v_cmpx_gt_f16 v1.l, v2.l row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x08,0x7d,0x01,0x41,0x01,0xff]
 
-v_cmpx_gt_f16 v1, v2 row_half_mirror
-// GFX11: v_cmpx_gt_f16 v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x08,0x7d,0x01,0x41,0x01,0xff]
+v_cmpx_gt_f16 v1.l, v2.l row_shl:1
+// GFX11: v_cmpx_gt_f16 v1.l, v2.l row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x08,0x7d,0x01,0x01,0x01,0xff]
 
-v_cmpx_gt_f16 v1, v2 row_shl:1
-// GFX11: v_cmpx_gt_f16 v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x08,0x7d,0x01,0x01,0x01,0xff]
+v_cmpx_gt_f16 v1.l, v2.l row_shl:15
+// GFX11: v_cmpx_gt_f16 v1.l, v2.l row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x08,0x7d,0x01,0x0f,0x01,0xff]
 
-v_cmpx_gt_f16 v1, v2 row_shl:15
-// GFX11: v_cmpx_gt_f16 v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x08,0x7d,0x01,0x0f,0x01,0xff]
+v_cmpx_gt_f16 v1.l, v2.l row_shr:1
+// GFX11: v_cmpx_gt_f16 v1.l, v2.l row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x08,0x7d,0x01,0x11,0x01,0xff]
 
-v_cmpx_gt_f16 v1, v2 row_shr:1
-// GFX11: v_cmpx_gt_f16 v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x08,0x7d,0x01,0x11,0x01,0xff]
+v_cmpx_gt_f16 v1.l, v2.l row_shr:15
+// GFX11: v_cmpx_gt_f16 v1.l, v2.l row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x08,0x7d,0x01,0x1f,0x01,0xff]
 
-v_cmpx_gt_f16 v1, v2 row_shr:15
-// GFX11: v_cmpx_gt_f16 v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x08,0x7d,0x01,0x1f,0x01,0xff]
+v_cmpx_gt_f16 v1.l, v2.l row_ror:1
+// GFX11: v_cmpx_gt_f16 v1.l, v2.l row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x08,0x7d,0x01,0x21,0x01,0xff]
 
-v_cmpx_gt_f16 v1, v2 row_ror:1
-// GFX11: v_cmpx_gt_f16 v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x08,0x7d,0x01,0x21,0x01,0xff]
+v_cmpx_gt_f16 v1.l, v2.l row_ror:15
+// GFX11: v_cmpx_gt_f16 v1.l, v2.l row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x08,0x7d,0x01,0x2f,0x01,0xff]
 
-v_cmpx_gt_f16 v1, v2 row_ror:15
-// GFX11: v_cmpx_gt_f16 v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x08,0x7d,0x01,0x2f,0x01,0xff]
+v_cmpx_gt_f16 v1.l, v2.l row_share:0 row_mask:0xf bank_mask:0xf
+// GFX11: v_cmpx_gt_f16 v1.l, v2.l row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x08,0x7d,0x01,0x50,0x01,0xff]
 
-v_cmpx_gt_f16 v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
-// GFX11: v_cmpx_gt_f16 v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x08,0x7d,0x01,0x50,0x01,0xff]
+v_cmpx_gt_f16 v1.l, v2.l row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: v_cmpx_gt_f16 v1.l, v2.l row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x08,0x7d,0x01,0x5f,0x01,0x01]
 
-v_cmpx_gt_f16 v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
-// GFX11: v_cmpx_gt_f16 v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x08,0x7d,0x01,0x5f,0x01,0x01]
+v_cmpx_gt_f16 v1.l, v2.l row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1
+// GFX11: v_cmpx_gt_f16 v1.l, v2.l row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0xfa,0x04,0x08,0x7d,0x01,0x60,0x09,0x13]
 
-v_cmpx_gt_f16 v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
-// GFX11: v_cmpx_gt_f16 v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0xfa,0x04,0x08,0x7d,0x01,0x60,0x09,0x13]
+v_cmpx_gt_f16 -|v127.l|, -|v127.l| row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1
+// GFX11: v_cmpx_gt_f16 -|v127.l|, -|v127.l| row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xfa,0xfe,0x08,0x7d,0x7f,0x6f,0xf5,0x30]
 
-v_cmpx_gt_f16 -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
-// GFX11: v_cmpx_gt_f16 -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xfa,0xfe,0x08,0x7d,0x7f,0x6f,0xf5,0x30]
+v_cmpx_gt_f16 v127.l, v127.l row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: v_cmpx_gt_f16 v127.l, v127.l row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0xfe,0x08,0x7d,0x7f,0x5f,0x01,0x01]
+
+v_cmpx_gt_f16 v1.h, v2.h row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: v_cmpx_gt_f16 v1.h, v2.h row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0xfa,0x04,0x09,0x7d,0x81,0x60,0x09,0x13]
+
+v_cmpx_gt_f16 -|v127.h|, -|v127.h| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: v_cmpx_gt_f16 -|v127.h|, -|v127.h| row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xfa,0xfe,0x09,0x7d,0xff,0x6f,0xf5,0x30]
 
 v_cmpx_gt_f32 v1, v2 quad_perm:[3,2,1,0]
 // GFX11: v_cmpx_gt_f32 v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x28,0x7d,0x01,0x1b,0x00,0xff]
@@ -1073,47 +1109,56 @@ v_cmpx_gt_u32 v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
 v_cmpx_gt_u32 v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
 // GFX11: v_cmpx_gt_u32 v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xfa,0xfe,0x99,0x7d,0xff,0x6f,0x05,0x30]
 
-v_cmpx_le_f16 v1, v2 quad_perm:[3,2,1,0]
-// GFX11: v_cmpx_le_f16 v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x06,0x7d,0x01,0x1b,0x00,0xff]
+v_cmpx_le_f16 v1.l, v2.l quad_perm:[3,2,1,0]
+// GFX11: v_cmpx_le_f16 v1.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x06,0x7d,0x01,0x1b,0x00,0xff]
+
+v_cmpx_le_f16 v1.l, v2.l quad_perm:[0,1,2,3]
+// GFX11: v_cmpx_le_f16 v1.l, v2.l quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x06,0x7d,0x01,0xe4,0x00,0xff]
 
-v_cmpx_le_f16 v1, v2 quad_perm:[0,1,2,3]
-// GFX11: v_cmpx_le_f16 v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x06,0x7d,0x01,0xe4,0x00,0xff]
+v_cmpx_le_f16 v1.l, v2.l row_mirror
+// GFX11: v_cmpx_le_f16 v1.l, v2.l row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x06,0x7d,0x01,0x40,0x01,0xff]
 
-v_cmpx_le_f16 v1, v2 row_mirror
-// GFX11: v_cmpx_le_f16 v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x06,0x7d,0x01,0x40,0x01,0xff]
+v_cmpx_le_f16 v1.l, v2.l row_half_mirror
+// GFX11: v_cmpx_le_f16 v1.l, v2.l row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x06,0x7d,0x01,0x41,0x01,0xff]
 
-v_cmpx_le_f16 v1, v2 row_half_mirror
-// GFX11: v_cmpx_le_f16 v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x06,0x7d,0x01,0x41,0x01,0xff]
+v_cmpx_le_f16 v1.l, v2.l row_shl:1
+// GFX11: v_cmpx_le_f16 v1.l, v2.l row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x06,0x7d,0x01,0x01,0x01,0xff]
 
-v_cmpx_le_f16 v1, v2 row_shl:1
-// GFX11: v_cmpx_le_f16 v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x06,0x7d,0x01,0x01,0x01,0xff]
+v_cmpx_le_f16 v1.l, v2.l row_shl:15
+// GFX11: v_cmpx_le_f16 v1.l, v2.l row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x06,0x7d,0x01,0x0f,0x01,0xff]
 
-v_cmpx_le_f16 v1, v2 row_shl:15
-// GFX11: v_cmpx_le_f16 v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x06,0x7d,0x01,0x0f,0x01,0xff]
+v_cmpx_le_f16 v1.l, v2.l row_shr:1
+// GFX11: v_cmpx_le_f16 v1.l, v2.l row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x06,0x7d,0x01,0x11,0x01,0xff]
 
-v_cmpx_le_f16 v1, v2 row_shr:1
-// GFX11: v_cmpx_le_f16 v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x06,0x7d,0x01,0x11,0x01,0xff]
+v_cmpx_le_f16 v1.l, v2.l row_shr:15
+// GFX11: v_cmpx_le_f16 v1.l, v2.l row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x06,0x7d,0x01,0x1f,0x01,0xff]
 
-v_cmpx_le_f16 v1, v2 row_shr:15
-// GFX11: v_cmpx_le_f16 v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x06,0x7d,0x01,0x1f,0x01,0xff]
+v_cmpx_le_f16 v1.l, v2.l row_ror:1
+// GFX11: v_cmpx_le_f16 v1.l, v2.l row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x06,0x7d,0x01,0x21,0x01,0xff]
 
-v_cmpx_le_f16 v1, v2 row_ror:1
-// GFX11: v_cmpx_le_f16 v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x06,0x7d,0x01,0x21,0x01,0xff]
+v_cmpx_le_f16 v1.l, v2.l row_ror:15
+// GFX11: v_cmpx_le_f16 v1.l, v2.l row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x06,0x7d,0x01,0x2f,0x01,0xff]
 
-v_cmpx_le_f16 v1, v2 row_ror:15
-// GFX11: v_cmpx_le_f16 v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x06,0x7d,0x01,0x2f,0x01,0xff]
+v_cmpx_le_f16 v1.l, v2.l row_share:0 row_mask:0xf bank_mask:0xf
+// GFX11: v_cmpx_le_f16 v1.l, v2.l row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x06,0x7d,0x01,0x50,0x01,0xff]
 
-v_cmpx_le_f16 v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
-// GFX11: v_cmpx_le_f16 v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x06,0x7d,0x01,0x50,0x01,0xff]
+v_cmpx_le_f16 v1.l, v2.l row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: v_cmpx_le_f16 v1.l, v2.l row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x06,0x7d,0x01,0x5f,0x01,0x01]
 
-v_cmpx_le_f16 v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
-// GFX11: v_cmpx_le_f16 v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x06,0x7d,0x01,0x5f,0x01,0x01]
+v_cmpx_le_f16 v1.l, v2.l row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1
+// GFX11: v_cmpx_le_f16 v1.l, v2.l row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0xfa,0x04,0x06,0x7d,0x01,0x60,0x09,0x13]
 
-v_cmpx_le_f16 v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
-// GFX11: v_cmpx_le_f16 v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0xfa,0x04,0x06,0x7d,0x01,0x60,0x09,0x13]
+v_cmpx_le_f16 -|v127.l|, -|v127.l| row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1
+// GFX11: v_cmpx_le_f16 -|v127.l|, -|v127.l| row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xfa,0xfe,0x06,0x7d,0x7f,0x6f,0xf5,0x30]
 
-v_cmpx_le_f16 -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
-// GFX11: v_cmpx_le_f16 -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xfa,0xfe,0x06,0x7d,0x7f,0x6f,0xf5,0x30]
+v_cmpx_le_f16 v127.l, v127.l row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: v_cmpx_le_f16 v127.l, v127.l row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0xfe,0x06,0x7d,0x7f,0x5f,0x01,0x01]
+
+v_cmpx_le_f16 v1.h, v2.h row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: v_cmpx_le_f16 v1.h, v2.h row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0xfa,0x04,0x07,0x7d,0x81,0x60,0x09,0x13]
+
+v_cmpx_le_f16 -|v127.h|, -|v127.h| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: v_cmpx_le_f16 -|v127.h|, -|v127.h| row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xfa,0xfe,0x07,0x7d,0xff,0x6f,0xf5,0x30]
 
 v_cmpx_le_f32 v1, v2 quad_perm:[3,2,1,0]
 // GFX11: v_cmpx_le_f32 v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x26,0x7d,0x01,0x1b,0x00,0xff]
@@ -1343,47 +1388,56 @@ v_cmpx_le_u32 v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
 v_cmpx_le_u32 v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
 // GFX11: v_cmpx_le_u32 v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xfa,0xfe,0x97,0x7d,0xff,0x6f,0x05,0x30]
 
-v_cmpx_lg_f16 v1, v2 quad_perm:[3,2,1,0]
-// GFX11: v_cmpx_lg_f16 v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x7d,0x01,0x1b,0x00,0xff]
+v_cmpx_lg_f16 v1.l, v2.l quad_perm:[3,2,1,0]
+// GFX11: v_cmpx_lg_f16 v1.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x7d,0x01,0x1b,0x00,0xff]
+
+v_cmpx_lg_f16 v1.l, v2.l quad_perm:[0,1,2,3]
+// GFX11: v_cmpx_lg_f16 v1.l, v2.l quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x7d,0x01,0xe4,0x00,0xff]
+
+v_cmpx_lg_f16 v1.l, v2.l row_mirror
+// GFX11: v_cmpx_lg_f16 v1.l, v2.l row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x7d,0x01,0x40,0x01,0xff]
 
-v_cmpx_lg_f16 v1, v2 quad_perm:[0,1,2,3]
-// GFX11: v_cmpx_lg_f16 v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x7d,0x01,0xe4,0x00,0xff]
+v_cmpx_lg_f16 v1.l, v2.l row_half_mirror
+// GFX11: v_cmpx_lg_f16 v1.l, v2.l row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x7d,0x01,0x41,0x01,0xff]
 
-v_cmpx_lg_f16 v1, v2 row_mirror
-// GFX11: v_cmpx_lg_f16 v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x7d,0x01,0x40,0x01,0xff]
+v_cmpx_lg_f16 v1.l, v2.l row_shl:1
+// GFX11: v_cmpx_lg_f16 v1.l, v2.l row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x7d,0x01,0x01,0x01,0xff]
 
-v_cmpx_lg_f16 v1, v2 row_half_mirror
-// GFX11: v_cmpx_lg_f16 v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x7d,0x01,0x41,0x01,0xff]
+v_cmpx_lg_f16 v1.l, v2.l row_shl:15
+// GFX11: v_cmpx_lg_f16 v1.l, v2.l row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x7d,0x01,0x0f,0x01,0xff]
 
-v_cmpx_lg_f16 v1, v2 row_shl:1
-// GFX11: v_cmpx_lg_f16 v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x7d,0x01,0x01,0x01,0xff]
+v_cmpx_lg_f16 v1.l, v2.l row_shr:1
+// GFX11: v_cmpx_lg_f16 v1.l, v2.l row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x7d,0x01,0x11,0x01,0xff]
 
-v_cmpx_lg_f16 v1, v2 row_shl:15
-// GFX11: v_cmpx_lg_f16 v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x7d,0x01,0x0f,0x01,0xff]
+v_cmpx_lg_f16 v1.l, v2.l row_shr:15
+// GFX11: v_cmpx_lg_f16 v1.l, v2.l row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x7d,0x01,0x1f,0x01,0xff]
 
-v_cmpx_lg_f16 v1, v2 row_shr:1
-// GFX11: v_cmpx_lg_f16 v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x7d,0x01,0x11,0x01,0xff]
+v_cmpx_lg_f16 v1.l, v2.l row_ror:1
+// GFX11: v_cmpx_lg_f16 v1.l, v2.l row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x7d,0x01,0x21,0x01,0xff]
 
-v_cmpx_lg_f16 v1, v2 row_shr:15
-// GFX11: v_cmpx_lg_f16 v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x7d,0x01,0x1f,0x01,0xff]
+v_cmpx_lg_f16 v1.l, v2.l row_ror:15
+// GFX11: v_cmpx_lg_f16 v1.l, v2.l row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x7d,0x01,0x2f,0x01,0xff]
 
-v_cmpx_lg_f16 v1, v2 row_ror:1
-// GFX11: v_cmpx_lg_f16 v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x7d,0x01,0x21,0x01,0xff]
+v_cmpx_lg_f16 v1.l, v2.l row_share:0 row_mask:0xf bank_mask:0xf
+// GFX11: v_cmpx_lg_f16 v1.l, v2.l row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x7d,0x01,0x50,0x01,0xff]
 
-v_cmpx_lg_f16 v1, v2 row_ror:15
-// GFX11: v_cmpx_lg_f16 v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x7d,0x01,0x2f,0x01,0xff]
+v_cmpx_lg_f16 v1.l, v2.l row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: v_cmpx_lg_f16 v1.l, v2.l row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x0a,0x7d,0x01,0x5f,0x01,0x01]
 
-v_cmpx_lg_f16 v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
-// GFX11: v_cmpx_lg_f16 v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x7d,0x01,0x50,0x01,0xff]
+v_cmpx_lg_f16 v1.l, v2.l row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1
+// GFX11: v_cmpx_lg_f16 v1.l, v2.l row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0xfa,0x04,0x0a,0x7d,0x01,0x60,0x09,0x13]
 
-v_cmpx_lg_f16 v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
-// GFX11: v_cmpx_lg_f16 v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x0a,0x7d,0x01,0x5f,0x01,0x01]
+v_cmpx_lg_f16 -|v127.l|, -|v127.l| row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1
+// GFX11: v_cmpx_lg_f16 -|v127.l|, -|v127.l| row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xfa,0xfe,0x0a,0x7d,0x7f,0x6f,0xf5,0x30]
 
-v_cmpx_lg_f16 v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
-// GFX11: v_cmpx_lg_f16 v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0xfa,0x04,0x0a,0x7d,0x01,0x60,0x09,0x13]
+v_cmpx_lg_f16 v127.l, v127.l row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: v_cmpx_lg_f16 v127.l, v127.l row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0xfe,0x0a,0x7d,0x7f,0x5f,0x01,0x01]
 
-v_cmpx_lg_f16 -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
-// GFX11: v_cmpx_lg_f16 -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xfa,0xfe,0x0a,0x7d,0x7f,0x6f,0xf5,0x30]
+v_cmpx_lg_f16 v1.h, v2.h row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: v_cmpx_lg_f16 v1.h, v2.h row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0xfa,0x04,0x0b,0x7d,0x81,0x60,0x09,0x13]
+
+v_cmpx_lg_f16 -|v127.h|, -|v127.h| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: v_cmpx_lg_f16 -|v127.h|, -|v127.h| row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xfa,0xfe,0x0b,0x7d,0xff,0x6f,0xf5,0x30]
 
 v_cmpx_lg_f32 v1, v2 quad_perm:[3,2,1,0]
 // GFX11: v_cmpx_lg_f32 v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x2a,0x7d,0x01,0x1b,0x00,0xff]
@@ -1892,47 +1946,56 @@ v_cmpx_ne_u32 v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
 v_cmpx_ne_u32 v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
 // GFX11: v_cmpx_ne_u32 v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xfa,0xfe,0x9b,0x7d,0xff,0x6f,0x05,0x30]
 
-v_cmpx_neq_f16 v1, v2 quad_perm:[3,2,1,0]
-// GFX11: v_cmpx_neq_f16 v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1a,0x7d,0x01,0x1b,0x00,0xff]
+v_cmpx_neq_f16 v1.l, v2.l quad_perm:[3,2,1,0]
+// GFX11: v_cmpx_neq_f16 v1.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1a,0x7d,0x01,0x1b,0x00,0xff]
+
+v_cmpx_neq_f16 v1.l, v2.l quad_perm:[0,1,2,3]
+// GFX11: v_cmpx_neq_f16 v1.l, v2.l quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1a,0x7d,0x01,0xe4,0x00,0xff]
+
+v_cmpx_neq_f16 v1.l, v2.l row_mirror
+// GFX11: v_cmpx_neq_f16 v1.l, v2.l row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1a,0x7d,0x01,0x40,0x01,0xff]
 
-v_cmpx_neq_f16 v1, v2 quad_perm:[0,1,2,3]
-// GFX11: v_cmpx_neq_f16 v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1a,0x7d,0x01,0xe4,0x00,0xff]
+v_cmpx_neq_f16 v1.l, v2.l row_half_mirror
+// GFX11: v_cmpx_neq_f16 v1.l, v2.l row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1a,0x7d,0x01,0x41,0x01,0xff]
 
-v_cmpx_neq_f16 v1, v2 row_mirror
-// GFX11: v_cmpx_neq_f16 v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1a,0x7d,0x01,0x40,0x01,0xff]
+v_cmpx_neq_f16 v1.l, v2.l row_shl:1
+// GFX11: v_cmpx_neq_f16 v1.l, v2.l row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1a,0x7d,0x01,0x01,0x01,0xff]
 
-v_cmpx_neq_f16 v1, v2 row_half_mirror
-// GFX11: v_cmpx_neq_f16 v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1a,0x7d,0x01,0x41,0x01,0xff]
+v_cmpx_neq_f16 v1.l, v2.l row_shl:15
+// GFX11: v_cmpx_neq_f16 v1.l, v2.l row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1a,0x7d,0x01,0x0f,0x01,0xff]
 
-v_cmpx_neq_f16 v1, v2 row_shl:1
-// GFX11: v_cmpx_neq_f16 v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1a,0x7d,0x01,0x01,0x01,0xff]
+v_cmpx_neq_f16 v1.l, v2.l row_shr:1
+// GFX11: v_cmpx_neq_f16 v1.l, v2.l row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1a,0x7d,0x01,0x11,0x01,0xff]
 
-v_cmpx_neq_f16 v1, v2 row_shl:15
-// GFX11: v_cmpx_neq_f16 v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1a,0x7d,0x01,0x0f,0x01,0xff]
+v_cmpx_neq_f16 v1.l, v2.l row_shr:15
+// GFX11: v_cmpx_neq_f16 v1.l, v2.l row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1a,0x7d,0x01,0x1f,0x01,0xff]
 
-v_cmpx_neq_f16 v1, v2 row_shr:1
-// GFX11: v_cmpx_neq_f16 v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1a,0x7d,0x01,0x11,0x01,0xff]
+v_cmpx_neq_f16 v1.l, v2.l row_ror:1
+// GFX11: v_cmpx_neq_f16 v1.l, v2.l row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1a,0x7d,0x01,0x21,0x01,0xff]
 
-v_cmpx_neq_f16 v1, v2 row_shr:15
-// GFX11: v_cmpx_neq_f16 v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1a,0x7d,0x01,0x1f,0x01,0xff]
+v_cmpx_neq_f16 v1.l, v2.l row_ror:15
+// GFX11: v_cmpx_neq_f16 v1.l, v2.l row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1a,0x7d,0x01,0x2f,0x01,0xff]
 
-v_cmpx_neq_f16 v1, v2 row_ror:1
-// GFX11: v_cmpx_neq_f16 v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1a,0x7d,0x01,0x21,0x01,0xff]
+v_cmpx_neq_f16 v1.l, v2.l row_share:0 row_mask:0xf bank_mask:0xf
+// GFX11: v_cmpx_neq_f16 v1.l, v2.l row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1a,0x7d,0x01,0x50,0x01,0xff]
 
-v_cmpx_neq_f16 v1, v2 row_ror:15
-// GFX11: v_cmpx_neq_f16 v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1a,0x7d,0x01,0x2f,0x01,0xff]
+v_cmpx_neq_f16 v1.l, v2.l row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: v_cmpx_neq_f16 v1.l, v2.l row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x1a,0x7d,0x01,0x5f,0x01,0x01]
 
-v_cmpx_neq_f16 v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
-// GFX11: v_cmpx_neq_f16 v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1a,0x7d,0x01,0x50,0x01,0xff]
+v_cmpx_neq_f16 v1.l, v2.l row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1
+// GFX11: v_cmpx_neq_f16 v1.l, v2.l row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0xfa,0x04,0x1a,0x7d,0x01,0x60,0x09,0x13]
 
-v_cmpx_neq_f16 v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
-// GFX11: v_cmpx_neq_f16 v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x1a,0x7d,0x01,0x5f,0x01,0x01]
+v_cmpx_neq_f16 -|v127.l|, -|v127.l| row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1
+// GFX11: v_cmpx_neq_f16 -|v127.l|, -|v127.l| row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xfa,0xfe,0x1a,0x7d,0x7f,0x6f,0xf5,0x30]
 
-v_cmpx_neq_f16 v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
-// GFX11: v_cmpx_neq_f16 v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0xfa,0x04,0x1a,0x7d,0x01,0x60,0x09,0x13]
+v_cmpx_neq_f16 v127.l, v127.l row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: v_cmpx_neq_f16 v127.l, v127.l row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0xfe,0x1a,0x7d,0x7f,0x5f,0x01,0x01]
 
-v_cmpx_neq_f16 -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
-// GFX11: v_cmpx_neq_f16 -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xfa,0xfe,0x1a,0x7d,0x7f,0x6f,0xf5,0x30]
+v_cmpx_neq_f16 v1.h, v2.h row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: v_cmpx_neq_f16 v1.h, v2.h row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0xfa,0x04,0x1b,0x7d,0x81,0x60,0x09,0x13]
+
+v_cmpx_neq_f16 -|v127.h|, -|v127.h| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: v_cmpx_neq_f16 -|v127.h|, -|v127.h| row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xfa,0xfe,0x1b,0x7d,0xff,0x6f,0xf5,0x30]
 
 v_cmpx_neq_f32 v1, v2 quad_perm:[3,2,1,0]
 // GFX11: v_cmpx_neq_f32 v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x3a,0x7d,0x01,0x1b,0x00,0xff]
@@ -1976,47 +2039,56 @@ v_cmpx_neq_f32 v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
 v_cmpx_neq_f32 -|v255|, -|v255| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
 // GFX11: v_cmpx_neq_f32 -|v255|, -|v255| row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xfa,0xfe,0x3b,0x7d,0xff,0x6f,0xf5,0x30]
 
-v_cmpx_nge_f16 v1, v2 quad_perm:[3,2,1,0]
-// GFX11: v_cmpx_nge_f16 v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x12,0x7d,0x01,0x1b,0x00,0xff]
+v_cmpx_nge_f16 v1.l, v2.l quad_perm:[3,2,1,0]
+// GFX11: v_cmpx_nge_f16 v1.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x12,0x7d,0x01,0x1b,0x00,0xff]
+
+v_cmpx_nge_f16 v1.l, v2.l quad_perm:[0,1,2,3]
+// GFX11: v_cmpx_nge_f16 v1.l, v2.l quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x12,0x7d,0x01,0xe4,0x00,0xff]
+
+v_cmpx_nge_f16 v1.l, v2.l row_mirror
+// GFX11: v_cmpx_nge_f16 v1.l, v2.l row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x12,0x7d,0x01,0x40,0x01,0xff]
 
-v_cmpx_nge_f16 v1, v2 quad_perm:[0,1,2,3]
-// GFX11: v_cmpx_nge_f16 v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x12,0x7d,0x01,0xe4,0x00,0xff]
+v_cmpx_nge_f16 v1.l, v2.l row_half_mirror
+// GFX11: v_cmpx_nge_f16 v1.l, v2.l row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x12,0x7d,0x01,0x41,0x01,0xff]
 
-v_cmpx_nge_f16 v1, v2 row_mirror
-// GFX11: v_cmpx_nge_f16 v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x12,0x7d,0x01,0x40,0x01,0xff]
+v_cmpx_nge_f16 v1.l, v2.l row_shl:1
+// GFX11: v_cmpx_nge_f16 v1.l, v2.l row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x12,0x7d,0x01,0x01,0x01,0xff]
 
-v_cmpx_nge_f16 v1, v2 row_half_mirror
-// GFX11: v_cmpx_nge_f16 v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x12,0x7d,0x01,0x41,0x01,0xff]
+v_cmpx_nge_f16 v1.l, v2.l row_shl:15
+// GFX11: v_cmpx_nge_f16 v1.l, v2.l row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x12,0x7d,0x01,0x0f,0x01,0xff]
 
-v_cmpx_nge_f16 v1, v2 row_shl:1
-// GFX11: v_cmpx_nge_f16 v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x12,0x7d,0x01,0x01,0x01,0xff]
+v_cmpx_nge_f16 v1.l, v2.l row_shr:1
+// GFX11: v_cmpx_nge_f16 v1.l, v2.l row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x12,0x7d,0x01,0x11,0x01,0xff]
 
-v_cmpx_nge_f16 v1, v2 row_shl:15
-// GFX11: v_cmpx_nge_f16 v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x12,0x7d,0x01,0x0f,0x01,0xff]
+v_cmpx_nge_f16 v1.l, v2.l row_shr:15
+// GFX11: v_cmpx_nge_f16 v1.l, v2.l row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x12,0x7d,0x01,0x1f,0x01,0xff]
 
-v_cmpx_nge_f16 v1, v2 row_shr:1
-// GFX11: v_cmpx_nge_f16 v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x12,0x7d,0x01,0x11,0x01,0xff]
+v_cmpx_nge_f16 v1.l, v2.l row_ror:1
+// GFX11: v_cmpx_nge_f16 v1.l, v2.l row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x12,0x7d,0x01,0x21,0x01,0xff]
 
-v_cmpx_nge_f16 v1, v2 row_shr:15
-// GFX11: v_cmpx_nge_f16 v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x12,0x7d,0x01,0x1f,0x01,0xff]
+v_cmpx_nge_f16 v1.l, v2.l row_ror:15
+// GFX11: v_cmpx_nge_f16 v1.l, v2.l row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x12,0x7d,0x01,0x2f,0x01,0xff]
 
-v_cmpx_nge_f16 v1, v2 row_ror:1
-// GFX11: v_cmpx_nge_f16 v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x12,0x7d,0x01,0x21,0x01,0xff]
+v_cmpx_nge_f16 v1.l, v2.l row_share:0 row_mask:0xf bank_mask:0xf
+// GFX11: v_cmpx_nge_f16 v1.l, v2.l row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x12,0x7d,0x01,0x50,0x01,0xff]
 
-v_cmpx_nge_f16 v1, v2 row_ror:15
-// GFX11: v_cmpx_nge_f16 v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x12,0x7d,0x01,0x2f,0x01,0xff]
+v_cmpx_nge_f16 v1.l, v2.l row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: v_cmpx_nge_f16 v1.l, v2.l row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x12,0x7d,0x01,0x5f,0x01,0x01]
 
-v_cmpx_nge_f16 v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
-// GFX11: v_cmpx_nge_f16 v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x12,0x7d,0x01,0x50,0x01,0xff]
+v_cmpx_nge_f16 v1.l, v2.l row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1
+// GFX11: v_cmpx_nge_f16 v1.l, v2.l row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0xfa,0x04,0x12,0x7d,0x01,0x60,0x09,0x13]
 
-v_cmpx_nge_f16 v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
-// GFX11: v_cmpx_nge_f16 v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x12,0x7d,0x01,0x5f,0x01,0x01]
+v_cmpx_nge_f16 -|v127.l|, -|v127.l| row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1
+// GFX11: v_cmpx_nge_f16 -|v127.l|, -|v127.l| row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xfa,0xfe,0x12,0x7d,0x7f,0x6f,0xf5,0x30]
 
-v_cmpx_nge_f16 v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
-// GFX11: v_cmpx_nge_f16 v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0xfa,0x04,0x12,0x7d,0x01,0x60,0x09,0x13]
+v_cmpx_nge_f16 v127.l, v127.l row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: v_cmpx_nge_f16 v127.l, v127.l row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0xfe,0x12,0x7d,0x7f,0x5f,0x01,0x01]
 
-v_cmpx_nge_f16 -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
-// GFX11: v_cmpx_nge_f16 -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xfa,0xfe,0x12,0x7d,0x7f,0x6f,0xf5,0x30]
+v_cmpx_nge_f16 v1.h, v2.h row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: v_cmpx_nge_f16 v1.h, v2.h row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0xfa,0x04,0x13,0x7d,0x81,0x60,0x09,0x13]
+
+v_cmpx_nge_f16 -|v127.h|, -|v127.h| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: v_cmpx_nge_f16 -|v127.h|, -|v127.h| row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xfa,0xfe,0x13,0x7d,0xff,0x6f,0xf5,0x30]
 
 v_cmpx_nge_f32 v1, v2 quad_perm:[3,2,1,0]
 // GFX11: v_cmpx_nge_f32 v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x32,0x7d,0x01,0x1b,0x00,0xff]
@@ -2060,47 +2132,56 @@ v_cmpx_nge_f32 v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
 v_cmpx_nge_f32 -|v255|, -|v255| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
 // GFX11: v_cmpx_nge_f32 -|v255|, -|v255| row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xfa,0xfe,0x33,0x7d,0xff,0x6f,0xf5,0x30]
 
-v_cmpx_ngt_f16 v1, v2 quad_perm:[3,2,1,0]
-// GFX11: v_cmpx_ngt_f16 v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x16,0x7d,0x01,0x1b,0x00,0xff]
+v_cmpx_ngt_f16 v1.l, v2.l quad_perm:[3,2,1,0]
+// GFX11: v_cmpx_ngt_f16 v1.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x16,0x7d,0x01,0x1b,0x00,0xff]
+
+v_cmpx_ngt_f16 v1.l, v2.l quad_perm:[0,1,2,3]
+// GFX11: v_cmpx_ngt_f16 v1.l, v2.l quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x16,0x7d,0x01,0xe4,0x00,0xff]
+
+v_cmpx_ngt_f16 v1.l, v2.l row_mirror
+// GFX11: v_cmpx_ngt_f16 v1.l, v2.l row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x16,0x7d,0x01,0x40,0x01,0xff]
 
-v_cmpx_ngt_f16 v1, v2 quad_perm:[0,1,2,3]
-// GFX11: v_cmpx_ngt_f16 v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x16,0x7d,0x01,0xe4,0x00,0xff]
+v_cmpx_ngt_f16 v1.l, v2.l row_half_mirror
+// GFX11: v_cmpx_ngt_f16 v1.l, v2.l row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x16,0x7d,0x01,0x41,0x01,0xff]
 
-v_cmpx_ngt_f16 v1, v2 row_mirror
-// GFX11: v_cmpx_ngt_f16 v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x16,0x7d,0x01,0x40,0x01,0xff]
+v_cmpx_ngt_f16 v1.l, v2.l row_shl:1
+// GFX11: v_cmpx_ngt_f16 v1.l, v2.l row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x16,0x7d,0x01,0x01,0x01,0xff]
 
-v_cmpx_ngt_f16 v1, v2 row_half_mirror
-// GFX11: v_cmpx_ngt_f16 v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x16,0x7d,0x01,0x41,0x01,0xff]
+v_cmpx_ngt_f16 v1.l, v2.l row_shl:15
+// GFX11: v_cmpx_ngt_f16 v1.l, v2.l row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x16,0x7d,0x01,0x0f,0x01,0xff]
 
-v_cmpx_ngt_f16 v1, v2 row_shl:1
-// GFX11: v_cmpx_ngt_f16 v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x16,0x7d,0x01,0x01,0x01,0xff]
+v_cmpx_ngt_f16 v1.l, v2.l row_shr:1
+// GFX11: v_cmpx_ngt_f16 v1.l, v2.l row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x16,0x7d,0x01,0x11,0x01,0xff]
 
-v_cmpx_ngt_f16 v1, v2 row_shl:15
-// GFX11: v_cmpx_ngt_f16 v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x16,0x7d,0x01,0x0f,0x01,0xff]
+v_cmpx_ngt_f16 v1.l, v2.l row_shr:15
+// GFX11: v_cmpx_ngt_f16 v1.l, v2.l row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x16,0x7d,0x01,0x1f,0x01,0xff]
 
-v_cmpx_ngt_f16 v1, v2 row_shr:1
-// GFX11: v_cmpx_ngt_f16 v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x16,0x7d,0x01,0x11,0x01,0xff]
+v_cmpx_ngt_f16 v1.l, v2.l row_ror:1
+// GFX11: v_cmpx_ngt_f16 v1.l, v2.l row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x16,0x7d,0x01,0x21,0x01,0xff]
 
-v_cmpx_ngt_f16 v1, v2 row_shr:15
-// GFX11: v_cmpx_ngt_f16 v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x16,0x7d,0x01,0x1f,0x01,0xff]
+v_cmpx_ngt_f16 v1.l, v2.l row_ror:15
+// GFX11: v_cmpx_ngt_f16 v1.l, v2.l row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x16,0x7d,0x01,0x2f,0x01,0xff]
 
-v_cmpx_ngt_f16 v1, v2 row_ror:1
-// GFX11: v_cmpx_ngt_f16 v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x16,0x7d,0x01,0x21,0x01,0xff]
+v_cmpx_ngt_f16 v1.l, v2.l row_share:0 row_mask:0xf bank_mask:0xf
+// GFX11: v_cmpx_ngt_f16 v1.l, v2.l row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x16,0x7d,0x01,0x50,0x01,0xff]
 
-v_cmpx_ngt_f16 v1, v2 row_ror:15
-// GFX11: v_cmpx_ngt_f16 v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x16,0x7d,0x01,0x2f,0x01,0xff]
+v_cmpx_ngt_f16 v1.l, v2.l row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: v_cmpx_ngt_f16 v1.l, v2.l row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x16,0x7d,0x01,0x5f,0x01,0x01]
 
-v_cmpx_ngt_f16 v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
-// GFX11: v_cmpx_ngt_f16 v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x16,0x7d,0x01,0x50,0x01,0xff]
+v_cmpx_ngt_f16 v1.l, v2.l row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1
+// GFX11: v_cmpx_ngt_f16 v1.l, v2.l row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0xfa,0x04,0x16,0x7d,0x01,0x60,0x09,0x13]
 
-v_cmpx_ngt_f16 v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
-// GFX11: v_cmpx_ngt_f16 v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x16,0x7d,0x01,0x5f,0x01,0x01]
+v_cmpx_ngt_f16 -|v127.l|, -|v127.l| row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1
+// GFX11: v_cmpx_ngt_f16 -|v127.l|, -|v127.l| row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xfa,0xfe,0x16,0x7d,0x7f,0x6f,0xf5,0x30]
 
-v_cmpx_ngt_f16 v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
-// GFX11: v_cmpx_ngt_f16 v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0xfa,0x04,0x16,0x7d,0x01,0x60,0x09,0x13]
+v_cmpx_ngt_f16 v127.l, v127.l row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: v_cmpx_ngt_f16 v127.l, v127.l row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0xfe,0x16,0x7d,0x7f,0x5f,0x01,0x01]
 
-v_cmpx_ngt_f16 -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
-// GFX11: v_cmpx_ngt_f16 -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xfa,0xfe,0x16,0x7d,0x7f,0x6f,0xf5,0x30]
+v_cmpx_ngt_f16 v1.h, v2.h row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: v_cmpx_ngt_f16 v1.h, v2.h row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0xfa,0x04,0x17,0x7d,0x81,0x60,0x09,0x13]
+
+v_cmpx_ngt_f16 -|v127.h|, -|v127.h| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: v_cmpx_ngt_f16 -|v127.h|, -|v127.h| row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xfa,0xfe,0x17,0x7d,0xff,0x6f,0xf5,0x30]
 
 v_cmpx_ngt_f32 v1, v2 quad_perm:[3,2,1,0]
 // GFX11: v_cmpx_ngt_f32 v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x36,0x7d,0x01,0x1b,0x00,0xff]
@@ -2144,47 +2225,56 @@ v_cmpx_ngt_f32 v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
 v_cmpx_ngt_f32 -|v255|, -|v255| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
 // GFX11: v_cmpx_ngt_f32 -|v255|, -|v255| row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xfa,0xfe,0x37,0x7d,0xff,0x6f,0xf5,0x30]
 
-v_cmpx_nle_f16 v1, v2 quad_perm:[3,2,1,0]
-// GFX11: v_cmpx_nle_f16 v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x18,0x7d,0x01,0x1b,0x00,0xff]
+v_cmpx_nle_f16 v1.l, v2.l quad_perm:[3,2,1,0]
+// GFX11: v_cmpx_nle_f16 v1.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x18,0x7d,0x01,0x1b,0x00,0xff]
+
+v_cmpx_nle_f16 v1.l, v2.l quad_perm:[0,1,2,3]
+// GFX11: v_cmpx_nle_f16 v1.l, v2.l quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x18,0x7d,0x01,0xe4,0x00,0xff]
+
+v_cmpx_nle_f16 v1.l, v2.l row_mirror
+// GFX11: v_cmpx_nle_f16 v1.l, v2.l row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x18,0x7d,0x01,0x40,0x01,0xff]
+
+v_cmpx_nle_f16 v1.l, v2.l row_half_mirror
+// GFX11: v_cmpx_nle_f16 v1.l, v2.l row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x18,0x7d,0x01,0x41,0x01,0xff]
 
-v_cmpx_nle_f16 v1, v2 quad_perm:[0,1,2,3]
-// GFX11: v_cmpx_nle_f16 v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x18,0x7d,0x01,0xe4,0x00,0xff]
+v_cmpx_nle_f16 v1.l, v2.l row_shl:1
+// GFX11: v_cmpx_nle_f16 v1.l, v2.l row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x18,0x7d,0x01,0x01,0x01,0xff]
 
-v_cmpx_nle_f16 v1, v2 row_mirror
-// GFX11: v_cmpx_nle_f16 v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x18,0x7d,0x01,0x40,0x01,0xff]
+v_cmpx_nle_f16 v1.l, v2.l row_shl:15
+// GFX11: v_cmpx_nle_f16 v1.l, v2.l row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x18,0x7d,0x01,0x0f,0x01,0xff]
 
-v_cmpx_nle_f16 v1, v2 row_half_mirror
-// GFX11: v_cmpx_nle_f16 v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x18,0x7d,0x01,0x41,0x01,0xff]
+v_cmpx_nle_f16 v1.l, v2.l row_shr:1
+// GFX11: v_cmpx_nle_f16 v1.l, v2.l row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x18,0x7d,0x01,0x11,0x01,0xff]
 
-v_cmpx_nle_f16 v1, v2 row_shl:1
-// GFX11: v_cmpx_nle_f16 v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x18,0x7d,0x01,0x01,0x01,0xff]
+v_cmpx_nle_f16 v1.l, v2.l row_shr:15
+// GFX11: v_cmpx_nle_f16 v1.l, v2.l row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x18,0x7d,0x01,0x1f,0x01,0xff]
 
-v_cmpx_nle_f16 v1, v2 row_shl:15
-// GFX11: v_cmpx_nle_f16 v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x18,0x7d,0x01,0x0f,0x01,0xff]
+v_cmpx_nle_f16 v1.l, v2.l row_ror:1
+// GFX11: v_cmpx_nle_f16 v1.l, v2.l row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x18,0x7d,0x01,0x21,0x01,0xff]
 
-v_cmpx_nle_f16 v1, v2 row_shr:1
-// GFX11: v_cmpx_nle_f16 v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x18,0x7d,0x01,0x11,0x01,0xff]
+v_cmpx_nle_f16 v1.l, v2.l row_ror:15
+// GFX11: v_cmpx_nle_f16 v1.l, v2.l row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x18,0x7d,0x01,0x2f,0x01,0xff]
 
-v_cmpx_nle_f16 v1, v2 row_shr:15
-// GFX11: v_cmpx_nle_f16 v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x18,0x7d,0x01,0x1f,0x01,0xff]
+v_cmpx_nle_f16 v1.l, v2.l row_share:0 row_mask:0xf bank_mask:0xf
+// GFX11: v_cmpx_nle_f16 v1.l, v2.l row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x18,0x7d,0x01,0x50,0x01,0xff]
 
-v_cmpx_nle_f16 v1, v2 row_ror:1
-// GFX11: v_cmpx_nle_f16 v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x18,0x7d,0x01,0x21,0x01,0xff]
+v_cmpx_nle_f16 v1.l, v2.l row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: v_cmpx_nle_f16 v1.l, v2.l row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x18,0x7d,0x01,0x5f,0x01,0x01]
 
-v_cmpx_nle_f16 v1, v2 row_ror:15
-// GFX11: v_cmpx_nle_f16 v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x18,0x7d,0x01,0x2f,0x01,0xff]
+v_cmpx_nle_f16 v1.l, v2.l row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1
+// GFX11: v_cmpx_nle_f16 v1.l, v2.l row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0xfa,0x04,0x18,0x7d,0x01,0x60,0x09,0x13]
 
-v_cmpx_nle_f16 v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
-// GFX11: v_cmpx_nle_f16 v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x18,0x7d,0x01,0x50,0x01,0xff]
+v_cmpx_nle_f16 -|v127.l|, -|v127.l| row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1
+// GFX11: v_cmpx_nle_f16 -|v127.l|, -|v127.l| row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xfa,0xfe,0x18,0x7d,0x7f,0x6f,0xf5,0x30]
 
-v_cmpx_nle_f16 v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
-// GFX11: v_cmpx_nle_f16 v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x18,0x7d,0x01,0x5f,0x01,0x01]
+v_cmpx_nle_f16 v127.l, v127.l row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: v_cmpx_nle_f16 v127.l, v127.l row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0xfe,0x18,0x7d,0x7f,0x5f,0x01,0x01]
 
-v_cmpx_nle_f16 v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
-// GFX11: v_cmpx_nle_f16 v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0xfa,0x04,0x18,0x7d,0x01,0x60,0x09,0x13]
+v_cmpx_nle_f16 v1.h, v2.h row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: v_cmpx_nle_f16 v1.h, v2.h row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0xfa,0x04,0x19,0x7d,0x81,0x60,0x09,0x13]
 
-v_cmpx_nle_f16 -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
-// GFX11: v_cmpx_nle_f16 -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xfa,0xfe,0x18,0x7d,0x7f,0x6f,0xf5,0x30]
+v_cmpx_nle_f16 -|v127.h|, -|v127.h| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: v_cmpx_nle_f16 -|v127.h|, -|v127.h| row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xfa,0xfe,0x19,0x7d,0xff,0x6f,0xf5,0x30]
 
 v_cmpx_nle_f32 v1, v2 quad_perm:[3,2,1,0]
 // GFX11: v_cmpx_nle_f32 v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x38,0x7d,0x01,0x1b,0x00,0xff]
@@ -2228,47 +2318,56 @@ v_cmpx_nle_f32 v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
 v_cmpx_nle_f32 -|v255|, -|v255| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
 // GFX11: v_cmpx_nle_f32 -|v255|, -|v255| row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xfa,0xfe,0x39,0x7d,0xff,0x6f,0xf5,0x30]
 
-v_cmpx_nlg_f16 v1, v2 quad_perm:[3,2,1,0]
-// GFX11: v_cmpx_nlg_f16 v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x14,0x7d,0x01,0x1b,0x00,0xff]
+v_cmpx_nlg_f16 v1.l, v2.l quad_perm:[3,2,1,0]
+// GFX11: v_cmpx_nlg_f16 v1.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x14,0x7d,0x01,0x1b,0x00,0xff]
+
+v_cmpx_nlg_f16 v1.l, v2.l quad_perm:[0,1,2,3]
+// GFX11: v_cmpx_nlg_f16 v1.l, v2.l quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x14,0x7d,0x01,0xe4,0x00,0xff]
+
+v_cmpx_nlg_f16 v1.l, v2.l row_mirror
+// GFX11: v_cmpx_nlg_f16 v1.l, v2.l row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x14,0x7d,0x01,0x40,0x01,0xff]
+
+v_cmpx_nlg_f16 v1.l, v2.l row_half_mirror
+// GFX11: v_cmpx_nlg_f16 v1.l, v2.l row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x14,0x7d,0x01,0x41,0x01,0xff]
 
-v_cmpx_nlg_f16 v1, v2 quad_perm:[0,1,2,3]
-// GFX11: v_cmpx_nlg_f16 v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x14,0x7d,0x01,0xe4,0x00,0xff]
+v_cmpx_nlg_f16 v1.l, v2.l row_shl:1
+// GFX11: v_cmpx_nlg_f16 v1.l, v2.l row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x14,0x7d,0x01,0x01,0x01,0xff]
 
-v_cmpx_nlg_f16 v1, v2 row_mirror
-// GFX11: v_cmpx_nlg_f16 v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x14,0x7d,0x01,0x40,0x01,0xff]
+v_cmpx_nlg_f16 v1.l, v2.l row_shl:15
+// GFX11: v_cmpx_nlg_f16 v1.l, v2.l row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x14,0x7d,0x01,0x0f,0x01,0xff]
 
-v_cmpx_nlg_f16 v1, v2 row_half_mirror
-// GFX11: v_cmpx_nlg_f16 v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x14,0x7d,0x01,0x41,0x01,0xff]
+v_cmpx_nlg_f16 v1.l, v2.l row_shr:1
+// GFX11: v_cmpx_nlg_f16 v1.l, v2.l row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x14,0x7d,0x01,0x11,0x01,0xff]
 
-v_cmpx_nlg_f16 v1, v2 row_shl:1
-// GFX11: v_cmpx_nlg_f16 v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x14,0x7d,0x01,0x01,0x01,0xff]
+v_cmpx_nlg_f16 v1.l, v2.l row_shr:15
+// GFX11: v_cmpx_nlg_f16 v1.l, v2.l row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x14,0x7d,0x01,0x1f,0x01,0xff]
 
-v_cmpx_nlg_f16 v1, v2 row_shl:15
-// GFX11: v_cmpx_nlg_f16 v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x14,0x7d,0x01,0x0f,0x01,0xff]
+v_cmpx_nlg_f16 v1.l, v2.l row_ror:1
+// GFX11: v_cmpx_nlg_f16 v1.l, v2.l row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x14,0x7d,0x01,0x21,0x01,0xff]
 
-v_cmpx_nlg_f16 v1, v2 row_shr:1
-// GFX11: v_cmpx_nlg_f16 v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x14,0x7d,0x01,0x11,0x01,0xff]
+v_cmpx_nlg_f16 v1.l, v2.l row_ror:15
+// GFX11: v_cmpx_nlg_f16 v1.l, v2.l row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x14,0x7d,0x01,0x2f,0x01,0xff]
 
-v_cmpx_nlg_f16 v1, v2 row_shr:15
-// GFX11: v_cmpx_nlg_f16 v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x14,0x7d,0x01,0x1f,0x01,0xff]
+v_cmpx_nlg_f16 v1.l, v2.l row_share:0 row_mask:0xf bank_mask:0xf
+// GFX11: v_cmpx_nlg_f16 v1.l, v2.l row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x14,0x7d,0x01,0x50,0x01,0xff]
 
-v_cmpx_nlg_f16 v1, v2 row_ror:1
-// GFX11: v_cmpx_nlg_f16 v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x14,0x7d,0x01,0x21,0x01,0xff]
+v_cmpx_nlg_f16 v1.l, v2.l row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: v_cmpx_nlg_f16 v1.l, v2.l row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x14,0x7d,0x01,0x5f,0x01,0x01]
 
-v_cmpx_nlg_f16 v1, v2 row_ror:15
-// GFX11: v_cmpx_nlg_f16 v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x14,0x7d,0x01,0x2f,0x01,0xff]
+v_cmpx_nlg_f16 v1.l, v2.l row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1
+// GFX11: v_cmpx_nlg_f16 v1.l, v2.l row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0xfa,0x04,0x14,0x7d,0x01,0x60,0x09,0x13]
 
-v_cmpx_nlg_f16 v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
-// GFX11: v_cmpx_nlg_f16 v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x14,0x7d,0x01,0x50,0x01,0xff]
+v_cmpx_nlg_f16 -|v127.l|, -|v127.l| row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1
+// GFX11: v_cmpx_nlg_f16 -|v127.l|, -|v127.l| row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xfa,0xfe,0x14,0x7d,0x7f,0x6f,0xf5,0x30]
 
-v_cmpx_nlg_f16 v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
-// GFX11: v_cmpx_nlg_f16 v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x14,0x7d,0x01,0x5f,0x01,0x01]
+v_cmpx_nlg_f16 v127.l, v127.l row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: v_cmpx_nlg_f16 v127.l, v127.l row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0xfe,0x14,0x7d,0x7f,0x5f,0x01,0x01]
 
-v_cmpx_nlg_f16 v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
-// GFX11: v_cmpx_nlg_f16 v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0xfa,0x04,0x14,0x7d,0x01,0x60,0x09,0x13]
+v_cmpx_nlg_f16 v1.h, v2.h row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: v_cmpx_nlg_f16 v1.h, v2.h row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0xfa,0x04,0x15,0x7d,0x81,0x60,0x09,0x13]
 
-v_cmpx_nlg_f16 -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
-// GFX11: v_cmpx_nlg_f16 -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xfa,0xfe,0x14,0x7d,0x7f,0x6f,0xf5,0x30]
+v_cmpx_nlg_f16 -|v127.h|, -|v127.h| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: v_cmpx_nlg_f16 -|v127.h|, -|v127.h| row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xfa,0xfe,0x15,0x7d,0xff,0x6f,0xf5,0x30]
 
 v_cmpx_nlg_f32 v1, v2 quad_perm:[3,2,1,0]
 // GFX11: v_cmpx_nlg_f32 v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x34,0x7d,0x01,0x1b,0x00,0xff]
@@ -2312,47 +2411,56 @@ v_cmpx_nlg_f32 v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
 v_cmpx_nlg_f32 -|v255|, -|v255| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
 // GFX11: v_cmpx_nlg_f32 -|v255|, -|v255| row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xfa,0xfe,0x35,0x7d,0xff,0x6f,0xf5,0x30]
 
-v_cmpx_nlt_f16 v1, v2 quad_perm:[3,2,1,0]
-// GFX11: v_cmpx_nlt_f16 v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1c,0x7d,0x01,0x1b,0x00,0xff]
+v_cmpx_nlt_f16 v1.l, v2.l quad_perm:[3,2,1,0]
+// GFX11: v_cmpx_nlt_f16 v1.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1c,0x7d,0x01,0x1b,0x00,0xff]
 
-v_cmpx_nlt_f16 v1, v2 quad_perm:[0,1,2,3]
-// GFX11: v_cmpx_nlt_f16 v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1c,0x7d,0x01,0xe4,0x00,0xff]
+v_cmpx_nlt_f16 v1.l, v2.l quad_perm:[0,1,2,3]
+// GFX11: v_cmpx_nlt_f16 v1.l, v2.l quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1c,0x7d,0x01,0xe4,0x00,0xff]
 
-v_cmpx_nlt_f16 v1, v2 row_mirror
-// GFX11: v_cmpx_nlt_f16 v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1c,0x7d,0x01,0x40,0x01,0xff]
+v_cmpx_nlt_f16 v1.l, v2.l row_mirror
+// GFX11: v_cmpx_nlt_f16 v1.l, v2.l row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1c,0x7d,0x01,0x40,0x01,0xff]
 
-v_cmpx_nlt_f16 v1, v2 row_half_mirror
-// GFX11: v_cmpx_nlt_f16 v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1c,0x7d,0x01,0x41,0x01,0xff]
+v_cmpx_nlt_f16 v1.l, v2.l row_half_mirror
+// GFX11: v_cmpx_nlt_f16 v1.l, v2.l row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1c,0x7d,0x01,0x41,0x01,0xff]
 
-v_cmpx_nlt_f16 v1, v2 row_shl:1
-// GFX11: v_cmpx_nlt_f16 v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1c,0x7d,0x01,0x01,0x01,0xff]
+v_cmpx_nlt_f16 v1.l, v2.l row_shl:1
+// GFX11: v_cmpx_nlt_f16 v1.l, v2.l row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1c,0x7d,0x01,0x01,0x01,0xff]
 
-v_cmpx_nlt_f16 v1, v2 row_shl:15
-// GFX11: v_cmpx_nlt_f16 v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1c,0x7d,0x01,0x0f,0x01,0xff]
+v_cmpx_nlt_f16 v1.l, v2.l row_shl:15
+// GFX11: v_cmpx_nlt_f16 v1.l, v2.l row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1c,0x7d,0x01,0x0f,0x01,0xff]
 
-v_cmpx_nlt_f16 v1, v2 row_shr:1
-// GFX11: v_cmpx_nlt_f16 v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1c,0x7d,0x01,0x11,0x01,0xff]
+v_cmpx_nlt_f16 v1.l, v2.l row_shr:1
+// GFX11: v_cmpx_nlt_f16 v1.l, v2.l row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1c,0x7d,0x01,0x11,0x01,0xff]
 
-v_cmpx_nlt_f16 v1, v2 row_shr:15
-// GFX11: v_cmpx_nlt_f16 v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1c,0x7d,0x01,0x1f,0x01,0xff]
+v_cmpx_nlt_f16 v1.l, v2.l row_shr:15
+// GFX11: v_cmpx_nlt_f16 v1.l, v2.l row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1c,0x7d,0x01,0x1f,0x01,0xff]
 
-v_cmpx_nlt_f16 v1, v2 row_ror:1
-// GFX11: v_cmpx_nlt_f16 v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1c,0x7d,0x01,0x21,0x01,0xff]
+v_cmpx_nlt_f16 v1.l, v2.l row_ror:1
+// GFX11: v_cmpx_nlt_f16 v1.l, v2.l row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1c,0x7d,0x01,0x21,0x01,0xff]
 
-v_cmpx_nlt_f16 v1, v2 row_ror:15
-// GFX11: v_cmpx_nlt_f16 v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1c,0x7d,0x01,0x2f,0x01,0xff]
+v_cmpx_nlt_f16 v1.l, v2.l row_ror:15
+// GFX11: v_cmpx_nlt_f16 v1.l, v2.l row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1c,0x7d,0x01,0x2f,0x01,0xff]
 
-v_cmpx_nlt_f16 v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
-// GFX11: v_cmpx_nlt_f16 v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1c,0x7d,0x01,0x50,0x01,0xff]
+v_cmpx_nlt_f16 v1.l, v2.l row_share:0 row_mask:0xf bank_mask:0xf
+// GFX11: v_cmpx_nlt_f16 v1.l, v2.l row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1c,0x7d,0x01,0x50,0x01,0xff]
 
-v_cmpx_nlt_f16 v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
-// GFX11: v_cmpx_nlt_f16 v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x1c,0x7d,0x01,0x5f,0x01,0x01]
+v_cmpx_nlt_f16 v1.l, v2.l row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: v_cmpx_nlt_f16 v1.l, v2.l row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x1c,0x7d,0x01,0x5f,0x01,0x01]
 
-v_cmpx_nlt_f16 v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
-// GFX11: v_cmpx_nlt_f16 v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0xfa,0x04,0x1c,0x7d,0x01,0x60,0x09,0x13]
+v_cmpx_nlt_f16 v1.l, v2.l row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1
+// GFX11: v_cmpx_nlt_f16 v1.l, v2.l row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0xfa,0x04,0x1c,0x7d,0x01,0x60,0x09,0x13]
 
-v_cmpx_nlt_f16 -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
-// GFX11: v_cmpx_nlt_f16 -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xfa,0xfe,0x1c,0x7d,0x7f,0x6f,0xf5,0x30]
+v_cmpx_nlt_f16 -|v127.l|, -|v127.l| row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1
+// GFX11: v_cmpx_nlt_f16 -|v127.l|, -|v127.l| row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xfa,0xfe,0x1c,0x7d,0x7f,0x6f,0xf5,0x30]
+
+v_cmpx_nlt_f16 v127.l, v127.l row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: v_cmpx_nlt_f16 v127.l, v127.l row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0xfe,0x1c,0x7d,0x7f,0x5f,0x01,0x01]
+
+v_cmpx_nlt_f16 v1.h, v2.h row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: v_cmpx_nlt_f16 v1.h, v2.h row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0xfa,0x04,0x1d,0x7d,0x81,0x60,0x09,0x13]
+
+v_cmpx_nlt_f16 -|v127.h|, -|v127.h| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: v_cmpx_nlt_f16 -|v127.h|, -|v127.h| row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xfa,0xfe,0x1d,0x7d,0xff,0x6f,0xf5,0x30]
 
 v_cmpx_nlt_f32 v1, v2 quad_perm:[3,2,1,0]
 // GFX11: v_cmpx_nlt_f32 v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x3c,0x7d,0x01,0x1b,0x00,0xff]
@@ -2396,47 +2504,56 @@ v_cmpx_nlt_f32 v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
 v_cmpx_nlt_f32 -|v255|, -|v255| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
 // GFX11: v_cmpx_nlt_f32 -|v255|, -|v255| row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xfa,0xfe,0x3d,0x7d,0xff,0x6f,0xf5,0x30]
 
-v_cmpx_o_f16 v1, v2 quad_perm:[3,2,1,0]
-// GFX11: v_cmpx_o_f16 v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0e,0x7d,0x01,0x1b,0x00,0xff]
+v_cmpx_o_f16 v1.l, v2.l quad_perm:[3,2,1,0]
+// GFX11: v_cmpx_o_f16 v1.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0e,0x7d,0x01,0x1b,0x00,0xff]
+
+v_cmpx_o_f16 v1.l, v2.l quad_perm:[0,1,2,3]
+// GFX11: v_cmpx_o_f16 v1.l, v2.l quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0e,0x7d,0x01,0xe4,0x00,0xff]
+
+v_cmpx_o_f16 v1.l, v2.l row_mirror
+// GFX11: v_cmpx_o_f16 v1.l, v2.l row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0e,0x7d,0x01,0x40,0x01,0xff]
+
+v_cmpx_o_f16 v1.l, v2.l row_half_mirror
+// GFX11: v_cmpx_o_f16 v1.l, v2.l row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0e,0x7d,0x01,0x41,0x01,0xff]
 
-v_cmpx_o_f16 v1, v2 quad_perm:[0,1,2,3]
-// GFX11: v_cmpx_o_f16 v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0e,0x7d,0x01,0xe4,0x00,0xff]
+v_cmpx_o_f16 v1.l, v2.l row_shl:1
+// GFX11: v_cmpx_o_f16 v1.l, v2.l row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0e,0x7d,0x01,0x01,0x01,0xff]
 
-v_cmpx_o_f16 v1, v2 row_mirror
-// GFX11: v_cmpx_o_f16 v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0e,0x7d,0x01,0x40,0x01,0xff]
+v_cmpx_o_f16 v1.l, v2.l row_shl:15
+// GFX11: v_cmpx_o_f16 v1.l, v2.l row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0e,0x7d,0x01,0x0f,0x01,0xff]
 
-v_cmpx_o_f16 v1, v2 row_half_mirror
-// GFX11: v_cmpx_o_f16 v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0e,0x7d,0x01,0x41,0x01,0xff]
+v_cmpx_o_f16 v1.l, v2.l row_shr:1
+// GFX11: v_cmpx_o_f16 v1.l, v2.l row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0e,0x7d,0x01,0x11,0x01,0xff]
 
-v_cmpx_o_f16 v1, v2 row_shl:1
-// GFX11: v_cmpx_o_f16 v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0e,0x7d,0x01,0x01,0x01,0xff]
+v_cmpx_o_f16 v1.l, v2.l row_shr:15
+// GFX11: v_cmpx_o_f16 v1.l, v2.l row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0e,0x7d,0x01,0x1f,0x01,0xff]
 
-v_cmpx_o_f16 v1, v2 row_shl:15
-// GFX11: v_cmpx_o_f16 v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0e,0x7d,0x01,0x0f,0x01,0xff]
+v_cmpx_o_f16 v1.l, v2.l row_ror:1
+// GFX11: v_cmpx_o_f16 v1.l, v2.l row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0e,0x7d,0x01,0x21,0x01,0xff]
 
-v_cmpx_o_f16 v1, v2 row_shr:1
-// GFX11: v_cmpx_o_f16 v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0e,0x7d,0x01,0x11,0x01,0xff]
+v_cmpx_o_f16 v1.l, v2.l row_ror:15
+// GFX11: v_cmpx_o_f16 v1.l, v2.l row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0e,0x7d,0x01,0x2f,0x01,0xff]
 
-v_cmpx_o_f16 v1, v2 row_shr:15
-// GFX11: v_cmpx_o_f16 v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0e,0x7d,0x01,0x1f,0x01,0xff]
+v_cmpx_o_f16 v1.l, v2.l row_share:0 row_mask:0xf bank_mask:0xf
+// GFX11: v_cmpx_o_f16 v1.l, v2.l row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0e,0x7d,0x01,0x50,0x01,0xff]
 
-v_cmpx_o_f16 v1, v2 row_ror:1
-// GFX11: v_cmpx_o_f16 v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0e,0x7d,0x01,0x21,0x01,0xff]
+v_cmpx_o_f16 v1.l, v2.l row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: v_cmpx_o_f16 v1.l, v2.l row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x0e,0x7d,0x01,0x5f,0x01,0x01]
 
-v_cmpx_o_f16 v1, v2 row_ror:15
-// GFX11: v_cmpx_o_f16 v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0e,0x7d,0x01,0x2f,0x01,0xff]
+v_cmpx_o_f16 v1.l, v2.l row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1
+// GFX11: v_cmpx_o_f16 v1.l, v2.l row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0xfa,0x04,0x0e,0x7d,0x01,0x60,0x09,0x13]
 
-v_cmpx_o_f16 v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
-// GFX11: v_cmpx_o_f16 v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0e,0x7d,0x01,0x50,0x01,0xff]
+v_cmpx_o_f16 -|v127.l|, -|v127.l| row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1
+// GFX11: v_cmpx_o_f16 -|v127.l|, -|v127.l| row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xfa,0xfe,0x0e,0x7d,0x7f,0x6f,0xf5,0x30]
 
-v_cmpx_o_f16 v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
-// GFX11: v_cmpx_o_f16 v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x0e,0x7d,0x01,0x5f,0x01,0x01]
+v_cmpx_o_f16 v127.l, v127.l row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: v_cmpx_o_f16 v127.l, v127.l row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0xfe,0x0e,0x7d,0x7f,0x5f,0x01,0x01]
 
-v_cmpx_o_f16 v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
-// GFX11: v_cmpx_o_f16 v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0xfa,0x04,0x0e,0x7d,0x01,0x60,0x09,0x13]
+v_cmpx_o_f16 v1.h, v2.h row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: v_cmpx_o_f16 v1.h, v2.h row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0xfa,0x04,0x0f,0x7d,0x81,0x60,0x09,0x13]
 
-v_cmpx_o_f16 -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
-// GFX11: v_cmpx_o_f16 -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xfa,0xfe,0x0e,0x7d,0x7f,0x6f,0xf5,0x30]
+v_cmpx_o_f16 -|v127.h|, -|v127.h| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: v_cmpx_o_f16 -|v127.h|, -|v127.h| row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xfa,0xfe,0x0f,0x7d,0xff,0x6f,0xf5,0x30]
 
 v_cmpx_o_f32 v1, v2 quad_perm:[3,2,1,0]
 // GFX11: v_cmpx_o_f32 v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x2e,0x7d,0x01,0x1b,0x00,0xff]
@@ -2480,47 +2597,65 @@ v_cmpx_o_f32 v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
 v_cmpx_o_f32 -|v255|, -|v255| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
 // GFX11: v_cmpx_o_f32 -|v255|, -|v255| row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xfa,0xfe,0x2f,0x7d,0xff,0x6f,0xf5,0x30]
 
-v_cmpx_t_f16 v1, v2 quad_perm:[3,2,1,0]
-// GFX11: v_cmpx_t_f16 v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1e,0x7d,0x01,0x1b,0x00,0xff]
+v_cmpx_t_f16 v1.l, v2.l quad_perm:[3,2,1,0]
+// GFX11: v_cmpx_t_f16 v1.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1e,0x7d,0x01,0x1b,0x00,0xff]
 
-v_cmpx_t_f16 v1, v2 quad_perm:[0,1,2,3]
-// GFX11: v_cmpx_t_f16 v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1e,0x7d,0x01,0xe4,0x00,0xff]
+v_cmpx_t_f16 v1.l, v2.l quad_perm:[0,1,2,3]
+// GFX11: v_cmpx_t_f16 v1.l, v2.l quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1e,0x7d,0x01,0xe4,0x00,0xff]
 
-v_cmpx_t_f16 v1, v2 row_mirror
-// GFX11: v_cmpx_t_f16 v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1e,0x7d,0x01,0x40,0x01,0xff]
+v_cmpx_t_f16 v1.l, v2.l row_mirror
+// GFX11: v_cmpx_t_f16 v1.l, v2.l row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1e,0x7d,0x01,0x40,0x01,0xff]
 
-v_cmpx_t_f16 v1, v2 row_half_mirror
-// GFX11: v_cmpx_t_f16 v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1e,0x7d,0x01,0x41,0x01,0xff]
+v_cmpx_t_f16 v1.l, v2.l row_half_mirror
+// GFX11: v_cmpx_t_f16 v1.l, v2.l row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1e,0x7d,0x01,0x41,0x01,0xff]
 
-v_cmpx_t_f16 v1, v2 row_shl:1
-// GFX11: v_cmpx_t_f16 v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1e,0x7d,0x01,0x01,0x01,0xff]
+v_cmpx_t_f16 v1.l, v2.l row_shl:1
+// GFX11: v_cmpx_t_f16 v1.l, v2.l row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1e,0x7d,0x01,0x01,0x01,0xff]
 
-v_cmpx_t_f16 v1, v2 row_shl:15
-// GFX11: v_cmpx_t_f16 v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1e,0x7d,0x01,0x0f,0x01,0xff]
+v_cmpx_t_f16 v1.l, v2.l row_shl:15
+// GFX11: v_cmpx_t_f16 v1.l, v2.l row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1e,0x7d,0x01,0x0f,0x01,0xff]
 
-v_cmpx_t_f16 v1, v2 row_shr:1
-// GFX11: v_cmpx_t_f16 v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1e,0x7d,0x01,0x11,0x01,0xff]
+v_cmpx_t_f16 v1.l, v2.l row_shr:1
+// GFX11: v_cmpx_t_f16 v1.l, v2.l row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1e,0x7d,0x01,0x11,0x01,0xff]
 
-v_cmpx_t_f16 v1, v2 row_shr:15
-// GFX11: v_cmpx_t_f16 v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1e,0x7d,0x01,0x1f,0x01,0xff]
+v_cmpx_t_f16 v1.l, v2.l row_shr:15
+// GFX11: v_cmpx_t_f16 v1.l, v2.l row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1e,0x7d,0x01,0x1f,0x01,0xff]
 
-v_cmpx_t_f16 v1, v2 row_ror:1
-// GFX11: v_cmpx_t_f16 v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1e,0x7d,0x01,0x21,0x01,0xff]
+v_cmpx_t_f16 v1.l, v2.l row_ror:1
+// GFX11: v_cmpx_t_f16 v1.l, v2.l row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1e,0x7d,0x01,0x21,0x01,0xff]
 
-v_cmpx_t_f16 v1, v2 row_ror:15
-// GFX11: v_cmpx_t_f16 v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1e,0x7d,0x01,0x2f,0x01,0xff]
+v_cmpx_t_f16 v1.l, v2.l row_ror:15
+// GFX11: v_cmpx_t_f16 v1.l, v2.l row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1e,0x7d,0x01,0x2f,0x01,0xff]
 
-v_cmpx_t_f16 v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
-// GFX11: v_cmpx_t_f16 v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1e,0x7d,0x01,0x50,0x01,0xff]
+v_cmpx_t_f16 v1.l, v2.l row_share:0 row_mask:0xf bank_mask:0xf
+// GFX11: v_cmpx_t_f16 v1.l, v2.l row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1e,0x7d,0x01,0x50,0x01,0xff]
 
-v_cmpx_t_f16 v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
-// GFX11: v_cmpx_t_f16 v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x1e,0x7d,0x01,0x5f,0x01,0x01]
+v_cmpx_t_f16 v1.l, v2.l row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: v_cmpx_t_f16 v1.l, v2.l row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x1e,0x7d,0x01,0x5f,0x01,0x01]
 
-v_cmpx_t_f16 v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
-// GFX11: v_cmpx_t_f16 v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0xfa,0x04,0x1e,0x7d,0x01,0x60,0x09,0x13]
+v_cmpx_t_f16 v1.l, v2.l row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: v_cmpx_t_f16 v1.l, v2.l row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0xfa,0x04,0x1e,0x7d,0x01,0x60,0x09,0x13]
 
-v_cmpx_t_f16 -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
-// GFX11: v_cmpx_t_f16 -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xfa,0xfe,0x1e,0x7d,0x7f,0x6f,0xf5,0x30]
+v_cmpx_t_f16 -|v127.l|, -|v127.l| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: v_cmpx_t_f16 -|v127.l|, -|v127.l| row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xfa,0xfe,0x1e,0x7d,0x7f,0x6f,0xf5,0x30]
+
+v_cmpx_t_f16 v127.l, v127.l row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: v_cmpx_t_f16 v127.l, v127.l row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0xfe,0x1e,0x7d,0x7f,0x5f,0x01,0x01]
+
+v_cmpx_tru_f16 v127.l, v127.l row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: v_cmpx_t_f16 v127.l, v127.l row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0xfe,0x1e,0x7d,0x7f,0x5f,0x01,0x01]
+
+v_cmpx_t_f16 v1.h, v2.h row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: v_cmpx_t_f16 v1.h, v2.h row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0xfa,0x04,0x1f,0x7d,0x81,0x60,0x09,0x13]
+
+v_cmpx_tru_f16 v1.h, v2.h row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: v_cmpx_t_f16 v1.h, v2.h row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0xfa,0x04,0x1f,0x7d,0x81,0x60,0x09,0x13]
+
+v_cmpx_t_f16 -|v127.h|, -|v127.h| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: v_cmpx_t_f16 -|v127.h|, -|v127.h| row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xfa,0xfe,0x1f,0x7d,0xff,0x6f,0xf5,0x30]
+
+v_cmpx_tru_f16 -|v127.h|, -|v127.h| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: v_cmpx_t_f16 -|v127.h|, -|v127.h| row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xfa,0xfe,0x1f,0x7d,0xff,0x6f,0xf5,0x30]
 
 v_cmpx_t_f32 v1, v2 quad_perm:[3,2,1,0]
 // GFX11: v_cmpx_t_f32 v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x3e,0x7d,0x01,0x1b,0x00,0xff]
@@ -2648,38 +2783,38 @@ v_cmpx_t_u32 v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
 v_cmpx_t_u32 v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
 // GFX11: v_cmpx_t_u32 v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xfa,0xfe,0x9f,0x7d,0xff,0x6f,0x05,0x30]
 
-v_cmpx_tru_f16 v1, v2 quad_perm:[3,2,1,0]
-// GFX11: v_cmpx_t_f16 v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1e,0x7d,0x01,0x1b,0x00,0xff]
+v_cmpx_tru_f16 v1.l, v2.l quad_perm:[3,2,1,0]
+// GFX11: v_cmpx_t_f16 v1.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1e,0x7d,0x01,0x1b,0x00,0xff]
 
-v_cmpx_tru_f16 v1, v2 quad_perm:[0,1,2,3]
-// GFX11: v_cmpx_t_f16 v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1e,0x7d,0x01,0xe4,0x00,0xff]
+v_cmpx_tru_f16 v1.l, v2.l quad_perm:[0,1,2,3]
+// GFX11: v_cmpx_t_f16 v1.l, v2.l quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1e,0x7d,0x01,0xe4,0x00,0xff]
 
-v_cmpx_tru_f16 v1, v2 row_mirror
-// GFX11: v_cmpx_t_f16 v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1e,0x7d,0x01,0x40,0x01,0xff]
+v_cmpx_tru_f16 v1.l, v2.l row_mirror
+// GFX11: v_cmpx_t_f16 v1.l, v2.l row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1e,0x7d,0x01,0x40,0x01,0xff]
 
-v_cmpx_tru_f16 v1, v2 row_half_mirror
-// GFX11: v_cmpx_t_f16 v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1e,0x7d,0x01,0x41,0x01,0xff]
+v_cmpx_tru_f16 v1.l, v2.l row_half_mirror
+// GFX11: v_cmpx_t_f16 v1.l, v2.l row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1e,0x7d,0x01,0x41,0x01,0xff]
 
-v_cmpx_tru_f16 v1, v2 row_shl:1
-// GFX11: v_cmpx_t_f16 v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1e,0x7d,0x01,0x01,0x01,0xff]
+v_cmpx_tru_f16 v1.l, v2.l row_shl:1
+// GFX11: v_cmpx_t_f16 v1.l, v2.l row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1e,0x7d,0x01,0x01,0x01,0xff]
 
-v_cmpx_tru_f16 v1, v2 row_shl:15
-// GFX11: v_cmpx_t_f16 v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1e,0x7d,0x01,0x0f,0x01,0xff]
+v_cmpx_tru_f16 v1.l, v2.l row_shl:15
+// GFX11: v_cmpx_t_f16 v1.l, v2.l row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1e,0x7d,0x01,0x0f,0x01,0xff]
 
-v_cmpx_tru_f16 v1, v2 row_shr:1
-// GFX11: v_cmpx_t_f16 v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1e,0x7d,0x01,0x11,0x01,0xff]
+v_cmpx_tru_f16 v1.l, v2.l row_shr:1
+// GFX11: v_cmpx_t_f16 v1.l, v2.l row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1e,0x7d,0x01,0x11,0x01,0xff]
 
-v_cmpx_tru_f16 v1, v2 row_shr:15
-// GFX11: v_cmpx_t_f16 v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1e,0x7d,0x01,0x1f,0x01,0xff]
+v_cmpx_tru_f16 v1.l, v2.l row_shr:15
+// GFX11: v_cmpx_t_f16 v1.l, v2.l row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1e,0x7d,0x01,0x1f,0x01,0xff]
 
-v_cmpx_tru_f16 v1, v2 row_ror:1
-// GFX11: v_cmpx_t_f16 v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1e,0x7d,0x01,0x21,0x01,0xff]
+v_cmpx_tru_f16 v1.l, v2.l row_ror:1
+// GFX11: v_cmpx_t_f16 v1.l, v2.l row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1e,0x7d,0x01,0x21,0x01,0xff]
 
-v_cmpx_tru_f16 v1, v2 row_ror:15
-// GFX11: v_cmpx_t_f16 v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1e,0x7d,0x01,0x2f,0x01,0xff]
+v_cmpx_tru_f16 v1.l, v2.l row_ror:15
+// GFX11: v_cmpx_t_f16 v1.l, v2.l row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1e,0x7d,0x01,0x2f,0x01,0xff]
 
-v_cmpx_tru_f16 v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
-// GFX11: v_cmpx_t_f16 v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1e,0x7d,0x01,0x50,0x01,0xff]
+v_cmpx_tru_f16 v1.l, v2.l row_share:0 row_mask:0xf bank_mask:0xf
+// GFX11: v_cmpx_t_f16 v1.l, v2.l row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1e,0x7d,0x01,0x50,0x01,0xff]
 
 v_cmpx_tru_f16 v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
 // GFX11: v_cmpx_t_f16 v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x1e,0x7d,0x01,0x5f,0x01,0x01]
@@ -2732,47 +2867,56 @@ v_cmpx_tru_f32 v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
 v_cmpx_tru_f32 -|v255|, -|v255| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
 // GFX11: v_cmpx_t_f32 -|v255|, -|v255| row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xfa,0xfe,0x3f,0x7d,0xff,0x6f,0xf5,0x30]
 
-v_cmpx_u_f16 v1, v2 quad_perm:[3,2,1,0]
-// GFX11: v_cmpx_u_f16 v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x10,0x7d,0x01,0x1b,0x00,0xff]
+v_cmpx_u_f16 v1.l, v2.l quad_perm:[3,2,1,0]
+// GFX11: v_cmpx_u_f16 v1.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x10,0x7d,0x01,0x1b,0x00,0xff]
+
+v_cmpx_u_f16 v1.l, v2.l quad_perm:[0,1,2,3]
+// GFX11: v_cmpx_u_f16 v1.l, v2.l quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x10,0x7d,0x01,0xe4,0x00,0xff]
+
+v_cmpx_u_f16 v1.l, v2.l row_mirror
+// GFX11: v_cmpx_u_f16 v1.l, v2.l row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x10,0x7d,0x01,0x40,0x01,0xff]
+
+v_cmpx_u_f16 v1.l, v2.l row_half_mirror
+// GFX11: v_cmpx_u_f16 v1.l, v2.l row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x10,0x7d,0x01,0x41,0x01,0xff]
 
-v_cmpx_u_f16 v1, v2 quad_perm:[0,1,2,3]
-// GFX11: v_cmpx_u_f16 v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x10,0x7d,0x01,0xe4,0x00,0xff]
+v_cmpx_u_f16 v1.l, v2.l row_shl:1
+// GFX11: v_cmpx_u_f16 v1.l, v2.l row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x10,0x7d,0x01,0x01,0x01,0xff]
 
-v_cmpx_u_f16 v1, v2 row_mirror
-// GFX11: v_cmpx_u_f16 v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x10,0x7d,0x01,0x40,0x01,0xff]
+v_cmpx_u_f16 v1.l, v2.l row_shl:15
+// GFX11: v_cmpx_u_f16 v1.l, v2.l row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x10,0x7d,0x01,0x0f,0x01,0xff]
 
-v_cmpx_u_f16 v1, v2 row_half_mirror
-// GFX11: v_cmpx_u_f16 v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x10,0x7d,0x01,0x41,0x01,0xff]
+v_cmpx_u_f16 v1.l, v2.l row_shr:1
+// GFX11: v_cmpx_u_f16 v1.l, v2.l row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x10,0x7d,0x01,0x11,0x01,0xff]
 
-v_cmpx_u_f16 v1, v2 row_shl:1
-// GFX11: v_cmpx_u_f16 v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x10,0x7d,0x01,0x01,0x01,0xff]
+v_cmpx_u_f16 v1.l, v2.l row_shr:15
+// GFX11: v_cmpx_u_f16 v1.l, v2.l row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x10,0x7d,0x01,0x1f,0x01,0xff]
 
-v_cmpx_u_f16 v1, v2 row_shl:15
-// GFX11: v_cmpx_u_f16 v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x10,0x7d,0x01,0x0f,0x01,0xff]
+v_cmpx_u_f16 v1.l, v2.l row_ror:1
+// GFX11: v_cmpx_u_f16 v1.l, v2.l row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x10,0x7d,0x01,0x21,0x01,0xff]
 
-v_cmpx_u_f16 v1, v2 row_shr:1
-// GFX11: v_cmpx_u_f16 v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x10,0x7d,0x01,0x11,0x01,0xff]
+v_cmpx_u_f16 v1.l, v2.l row_ror:15
+// GFX11: v_cmpx_u_f16 v1.l, v2.l row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x10,0x7d,0x01,0x2f,0x01,0xff]
 
-v_cmpx_u_f16 v1, v2 row_shr:15
-// GFX11: v_cmpx_u_f16 v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x10,0x7d,0x01,0x1f,0x01,0xff]
+v_cmpx_u_f16 v1.l, v2.l row_share:0 row_mask:0xf bank_mask:0xf
+// GFX11: v_cmpx_u_f16 v1.l, v2.l row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x10,0x7d,0x01,0x50,0x01,0xff]
 
-v_cmpx_u_f16 v1, v2 row_ror:1
-// GFX11: v_cmpx_u_f16 v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x10,0x7d,0x01,0x21,0x01,0xff]
+v_cmpx_u_f16 v1.l, v2.l row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: v_cmpx_u_f16 v1.l, v2.l row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x10,0x7d,0x01,0x5f,0x01,0x01]
 
-v_cmpx_u_f16 v1, v2 row_ror:15
-// GFX11: v_cmpx_u_f16 v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x10,0x7d,0x01,0x2f,0x01,0xff]
+v_cmpx_u_f16 v1.l, v2.l row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1
+// GFX11: v_cmpx_u_f16 v1.l, v2.l row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0xfa,0x04,0x10,0x7d,0x01,0x60,0x09,0x13]
 
-v_cmpx_u_f16 v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
-// GFX11: v_cmpx_u_f16 v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x10,0x7d,0x01,0x50,0x01,0xff]
+v_cmpx_u_f16 -|v127.l|, -|v127.l| row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1
+// GFX11: v_cmpx_u_f16 -|v127.l|, -|v127.l| row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xfa,0xfe,0x10,0x7d,0x7f,0x6f,0xf5,0x30]
 
-v_cmpx_u_f16 v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
-// GFX11: v_cmpx_u_f16 v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x10,0x7d,0x01,0x5f,0x01,0x01]
+v_cmpx_u_f16 v127.l, v127.l row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: v_cmpx_u_f16 v127.l, v127.l row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0xfe,0x10,0x7d,0x7f,0x5f,0x01,0x01]
 
-v_cmpx_u_f16 v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
-// GFX11: v_cmpx_u_f16 v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0xfa,0x04,0x10,0x7d,0x01,0x60,0x09,0x13]
+v_cmpx_u_f16 v1.h, v2.h row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: v_cmpx_u_f16 v1.h, v2.h row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0xfa,0x04,0x11,0x7d,0x81,0x60,0x09,0x13]
 
-v_cmpx_u_f16 -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
-// GFX11: v_cmpx_u_f16 -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xfa,0xfe,0x10,0x7d,0x7f,0x6f,0xf5,0x30]
+v_cmpx_u_f16 -|v127.h|, -|v127.h| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: v_cmpx_u_f16 -|v127.h|, -|v127.h| row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xfa,0xfe,0x11,0x7d,0xff,0x6f,0xf5,0x30]
 
 v_cmpx_u_f32 v1, v2 quad_perm:[3,2,1,0]
 // GFX11: v_cmpx_u_f32 v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x30,0x7d,0x01,0x1b,0x00,0xff]
diff --git a/llvm/test/MC/AMDGPU/gfx11_asm_vopcx_dpp8.s b/llvm/test/MC/AMDGPU/gfx11_asm_vopcx_dpp8.s
index 1cead89c0a82e..1d664e4ecb902 100644
--- a/llvm/test/MC/AMDGPU/gfx11_asm_vopcx_dpp8.s
+++ b/llvm/test/MC/AMDGPU/gfx11_asm_vopcx_dpp8.s
@@ -29,14 +29,23 @@ v_cmpx_class_f32 v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
 v_cmpx_class_f32 v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
 // GFX11: v_cmpx_class_f32 v255, v255 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xe9,0xfe,0xfd,0x7d,0xff,0x00,0x00,0x00]
 
-v_cmpx_eq_f16 v1, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmpx_eq_f16 v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x04,0x7d,0x01,0x77,0x39,0x05]
+v_cmpx_eq_f16 v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_eq_f16 v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x04,0x7d,0x01,0x77,0x39,0x05]
 
-v_cmpx_eq_f16 v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
-// GFX11: v_cmpx_eq_f16 v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0xea,0x04,0x04,0x7d,0x01,0x77,0x39,0x05]
+v_cmpx_eq_f16 v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: v_cmpx_eq_f16 v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0xea,0x04,0x04,0x7d,0x01,0x77,0x39,0x05]
 
-v_cmpx_eq_f16 v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:0
-// GFX11: v_cmpx_eq_f16 v127, v127 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xe9,0xfe,0x04,0x7d,0x7f,0x00,0x00,0x00]
+v_cmpx_eq_f16 v127.l, v127.l dpp8:[0,0,0,0,0,0,0,0]
+// GFX11: v_cmpx_eq_f16 v127.l, v127.l dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xe9,0xfe,0x04,0x7d,0x7f,0x00,0x00,0x00]
+
+v_cmpx_eq_f16 v127.l, v127.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_eq_f16 v127.l, v127.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0xfe,0x04,0x7d,0x7f,0x77,0x39,0x05]
+
+v_cmpx_eq_f16 v1.h, v2.h dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: v_cmpx_eq_f16 v1.h, v2.h dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0xea,0x04,0x05,0x7d,0x81,0x77,0x39,0x05]
+
+v_cmpx_eq_f16 v127.h, v127.h dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: v_cmpx_eq_f16 v127.h, v127.h dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xe9,0xfe,0x05,0x7d,0xff,0x00,0x00,0x00]
 
 v_cmpx_eq_f32 v1, v2 dpp8:[7,6,5,4,3,2,1,0]
 // GFX11: v_cmpx_eq_f32 v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x24,0x7d,0x01,0x77,0x39,0x05]
@@ -101,14 +110,23 @@ v_cmpx_eq_u32 v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
 v_cmpx_eq_u32 v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
 // GFX11: v_cmpx_eq_u32 v255, v255 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xe9,0xfe,0x95,0x7d,0xff,0x00,0x00,0x00]
 
-v_cmpx_f_f16 v1, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmpx_f_f16 v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x00,0x7d,0x01,0x77,0x39,0x05]
+v_cmpx_f_f16 v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_f_f16 v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x00,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_f_f16 v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: v_cmpx_f_f16 v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0xea,0x04,0x00,0x7d,0x01,0x77,0x39,0x05]
 
-v_cmpx_f_f16 v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
-// GFX11: v_cmpx_f_f16 v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0xea,0x04,0x00,0x7d,0x01,0x77,0x39,0x05]
+v_cmpx_f_f16 v127.l, v127.l dpp8:[0,0,0,0,0,0,0,0]
+// GFX11: v_cmpx_f_f16 v127.l, v127.l dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xe9,0xfe,0x00,0x7d,0x7f,0x00,0x00,0x00]
 
-v_cmpx_f_f16 v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:0
-// GFX11: v_cmpx_f_f16 v127, v127 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xe9,0xfe,0x00,0x7d,0x7f,0x00,0x00,0x00]
+v_cmpx_f_f16 v127.l, v127.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_f_f16 v127.l, v127.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0xfe,0x00,0x7d,0x7f,0x77,0x39,0x05]
+
+v_cmpx_f_f16 v1.h, v2.h dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: v_cmpx_f_f16 v1.h, v2.h dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0xea,0x04,0x01,0x7d,0x81,0x77,0x39,0x05]
+
+v_cmpx_f_f16 v127.h, v127.h dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: v_cmpx_f_f16 v127.h, v127.h dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xe9,0xfe,0x01,0x7d,0xff,0x00,0x00,0x00]
 
 v_cmpx_f_f32 v1, v2 dpp8:[7,6,5,4,3,2,1,0]
 // GFX11: v_cmpx_f_f32 v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x20,0x7d,0x01,0x77,0x39,0x05]
@@ -137,14 +155,23 @@ v_cmpx_f_u32 v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
 v_cmpx_f_u32 v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
 // GFX11: v_cmpx_f_u32 v255, v255 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xe9,0xfe,0x91,0x7d,0xff,0x00,0x00,0x00]
 
-v_cmpx_ge_f16 v1, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmpx_ge_f16 v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x0c,0x7d,0x01,0x77,0x39,0x05]
+v_cmpx_ge_f16 v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_ge_f16 v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x0c,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_ge_f16 v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: v_cmpx_ge_f16 v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0xea,0x04,0x0c,0x7d,0x01,0x77,0x39,0x05]
 
-v_cmpx_ge_f16 v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
-// GFX11: v_cmpx_ge_f16 v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0xea,0x04,0x0c,0x7d,0x01,0x77,0x39,0x05]
+v_cmpx_ge_f16 v127.l, v127.l dpp8:[0,0,0,0,0,0,0,0]
+// GFX11: v_cmpx_ge_f16 v127.l, v127.l dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xe9,0xfe,0x0c,0x7d,0x7f,0x00,0x00,0x00]
 
-v_cmpx_ge_f16 v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:0
-// GFX11: v_cmpx_ge_f16 v127, v127 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xe9,0xfe,0x0c,0x7d,0x7f,0x00,0x00,0x00]
+v_cmpx_ge_f16 v127.l, v127.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_ge_f16 v127.l, v127.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0xfe,0x0c,0x7d,0x7f,0x77,0x39,0x05]
+
+v_cmpx_ge_f16 v1.h, v2.h dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: v_cmpx_ge_f16 v1.h, v2.h dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0xea,0x04,0x0d,0x7d,0x81,0x77,0x39,0x05]
+
+v_cmpx_ge_f16 v127.h, v127.h dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: v_cmpx_ge_f16 v127.h, v127.h dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xe9,0xfe,0x0d,0x7d,0xff,0x00,0x00,0x00]
 
 v_cmpx_ge_f32 v1, v2 dpp8:[7,6,5,4,3,2,1,0]
 // GFX11: v_cmpx_ge_f32 v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x2c,0x7d,0x01,0x77,0x39,0x05]
@@ -209,14 +236,23 @@ v_cmpx_ge_u32 v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
 v_cmpx_ge_u32 v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
 // GFX11: v_cmpx_ge_u32 v255, v255 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xe9,0xfe,0x9d,0x7d,0xff,0x00,0x00,0x00]
 
-v_cmpx_gt_f16 v1, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmpx_gt_f16 v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x08,0x7d,0x01,0x77,0x39,0x05]
+v_cmpx_gt_f16 v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_gt_f16 v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x08,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_gt_f16 v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: v_cmpx_gt_f16 v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0xea,0x04,0x08,0x7d,0x01,0x77,0x39,0x05]
 
-v_cmpx_gt_f16 v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
-// GFX11: v_cmpx_gt_f16 v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0xea,0x04,0x08,0x7d,0x01,0x77,0x39,0x05]
+v_cmpx_gt_f16 v127.l, v127.l dpp8:[0,0,0,0,0,0,0,0]
+// GFX11: v_cmpx_gt_f16 v127.l, v127.l dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xe9,0xfe,0x08,0x7d,0x7f,0x00,0x00,0x00]
 
-v_cmpx_gt_f16 v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:0
-// GFX11: v_cmpx_gt_f16 v127, v127 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xe9,0xfe,0x08,0x7d,0x7f,0x00,0x00,0x00]
+v_cmpx_gt_f16 v127.l, v127.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_gt_f16 v127.l, v127.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0xfe,0x08,0x7d,0x7f,0x77,0x39,0x05]
+
+v_cmpx_gt_f16 v1.h, v2.h dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: v_cmpx_gt_f16 v1.h, v2.h dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0xea,0x04,0x09,0x7d,0x81,0x77,0x39,0x05]
+
+v_cmpx_gt_f16 v127.h, v127.h dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: v_cmpx_gt_f16 v127.h, v127.h dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xe9,0xfe,0x09,0x7d,0xff,0x00,0x00,0x00]
 
 v_cmpx_gt_f32 v1, v2 dpp8:[7,6,5,4,3,2,1,0]
 // GFX11: v_cmpx_gt_f32 v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x28,0x7d,0x01,0x77,0x39,0x05]
@@ -281,14 +317,23 @@ v_cmpx_gt_u32 v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
 v_cmpx_gt_u32 v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
 // GFX11: v_cmpx_gt_u32 v255, v255 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xe9,0xfe,0x99,0x7d,0xff,0x00,0x00,0x00]
 
-v_cmpx_le_f16 v1, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmpx_le_f16 v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x06,0x7d,0x01,0x77,0x39,0x05]
+v_cmpx_le_f16 v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_le_f16 v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x06,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_le_f16 v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: v_cmpx_le_f16 v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0xea,0x04,0x06,0x7d,0x01,0x77,0x39,0x05]
 
-v_cmpx_le_f16 v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
-// GFX11: v_cmpx_le_f16 v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0xea,0x04,0x06,0x7d,0x01,0x77,0x39,0x05]
+v_cmpx_le_f16 v127.l, v127.l dpp8:[0,0,0,0,0,0,0,0]
+// GFX11: v_cmpx_le_f16 v127.l, v127.l dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xe9,0xfe,0x06,0x7d,0x7f,0x00,0x00,0x00]
 
-v_cmpx_le_f16 v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:0
-// GFX11: v_cmpx_le_f16 v127, v127 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xe9,0xfe,0x06,0x7d,0x7f,0x00,0x00,0x00]
+v_cmpx_le_f16 v127.l, v127.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_le_f16 v127.l, v127.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0xfe,0x06,0x7d,0x7f,0x77,0x39,0x05]
+
+v_cmpx_le_f16 v1.h, v2.h dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: v_cmpx_le_f16 v1.h, v2.h dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0xea,0x04,0x07,0x7d,0x81,0x77,0x39,0x05]
+
+v_cmpx_le_f16 v127.h, v127.h dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: v_cmpx_le_f16 v127.h, v127.h dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xe9,0xfe,0x07,0x7d,0xff,0x00,0x00,0x00]
 
 v_cmpx_le_f32 v1, v2 dpp8:[7,6,5,4,3,2,1,0]
 // GFX11: v_cmpx_le_f32 v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x26,0x7d,0x01,0x77,0x39,0x05]
@@ -353,14 +398,23 @@ v_cmpx_le_u32 v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
 v_cmpx_le_u32 v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
 // GFX11: v_cmpx_le_u32 v255, v255 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xe9,0xfe,0x97,0x7d,0xff,0x00,0x00,0x00]
 
-v_cmpx_lg_f16 v1, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmpx_lg_f16 v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x0a,0x7d,0x01,0x77,0x39,0x05]
+v_cmpx_lg_f16 v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_lg_f16 v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x0a,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_lg_f16 v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: v_cmpx_lg_f16 v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0xea,0x04,0x0a,0x7d,0x01,0x77,0x39,0x05]
 
-v_cmpx_lg_f16 v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
-// GFX11: v_cmpx_lg_f16 v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0xea,0x04,0x0a,0x7d,0x01,0x77,0x39,0x05]
+v_cmpx_lg_f16 v127.l, v127.l dpp8:[0,0,0,0,0,0,0,0]
+// GFX11: v_cmpx_lg_f16 v127.l, v127.l dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xe9,0xfe,0x0a,0x7d,0x7f,0x00,0x00,0x00]
 
-v_cmpx_lg_f16 v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:0
-// GFX11: v_cmpx_lg_f16 v127, v127 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xe9,0xfe,0x0a,0x7d,0x7f,0x00,0x00,0x00]
+v_cmpx_lg_f16 v127.l, v127.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_lg_f16 v127.l, v127.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0xfe,0x0a,0x7d,0x7f,0x77,0x39,0x05]
+
+v_cmpx_lg_f16 v1.h, v2.h dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: v_cmpx_lg_f16 v1.h, v2.h dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0xea,0x04,0x0b,0x7d,0x81,0x77,0x39,0x05]
+
+v_cmpx_lg_f16 v127.h, v127.h dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: v_cmpx_lg_f16 v127.h, v127.h dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xe9,0xfe,0x0b,0x7d,0xff,0x00,0x00,0x00]
 
 v_cmpx_lg_f32 v1, v2 dpp8:[7,6,5,4,3,2,1,0]
 // GFX11: v_cmpx_lg_f32 v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x2a,0x7d,0x01,0x77,0x39,0x05]
@@ -506,14 +560,23 @@ v_cmpx_ne_u32 v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
 v_cmpx_ne_u32 v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
 // GFX11: v_cmpx_ne_u32 v255, v255 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xe9,0xfe,0x9b,0x7d,0xff,0x00,0x00,0x00]
 
-v_cmpx_neq_f16 v1, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmpx_neq_f16 v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x1a,0x7d,0x01,0x77,0x39,0x05]
+v_cmpx_neq_f16 v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_neq_f16 v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x1a,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_neq_f16 v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: v_cmpx_neq_f16 v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0xea,0x04,0x1a,0x7d,0x01,0x77,0x39,0x05]
 
-v_cmpx_neq_f16 v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
-// GFX11: v_cmpx_neq_f16 v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0xea,0x04,0x1a,0x7d,0x01,0x77,0x39,0x05]
+v_cmpx_neq_f16 v127.l, v127.l dpp8:[0,0,0,0,0,0,0,0]
+// GFX11: v_cmpx_neq_f16 v127.l, v127.l dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xe9,0xfe,0x1a,0x7d,0x7f,0x00,0x00,0x00]
 
-v_cmpx_neq_f16 v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:0
-// GFX11: v_cmpx_neq_f16 v127, v127 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xe9,0xfe,0x1a,0x7d,0x7f,0x00,0x00,0x00]
+v_cmpx_neq_f16 v127.l, v127.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_neq_f16 v127.l, v127.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0xfe,0x1a,0x7d,0x7f,0x77,0x39,0x05]
+
+v_cmpx_neq_f16 v1.h, v2.h dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: v_cmpx_neq_f16 v1.h, v2.h dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0xea,0x04,0x1b,0x7d,0x81,0x77,0x39,0x05]
+
+v_cmpx_neq_f16 v127.h, v127.h dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: v_cmpx_neq_f16 v127.h, v127.h dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xe9,0xfe,0x1b,0x7d,0xff,0x00,0x00,0x00]
 
 v_cmpx_neq_f32 v1, v2 dpp8:[7,6,5,4,3,2,1,0]
 // GFX11: v_cmpx_neq_f32 v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x3a,0x7d,0x01,0x77,0x39,0x05]
@@ -524,14 +587,23 @@ v_cmpx_neq_f32 v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
 v_cmpx_neq_f32 v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
 // GFX11: v_cmpx_neq_f32 v255, v255 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xe9,0xfe,0x3b,0x7d,0xff,0x00,0x00,0x00]
 
-v_cmpx_nge_f16 v1, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmpx_nge_f16 v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x12,0x7d,0x01,0x77,0x39,0x05]
+v_cmpx_nge_f16 v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_nge_f16 v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x12,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_nge_f16 v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: v_cmpx_nge_f16 v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0xea,0x04,0x12,0x7d,0x01,0x77,0x39,0x05]
 
-v_cmpx_nge_f16 v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
-// GFX11: v_cmpx_nge_f16 v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0xea,0x04,0x12,0x7d,0x01,0x77,0x39,0x05]
+v_cmpx_nge_f16 v127.l, v127.l dpp8:[0,0,0,0,0,0,0,0]
+// GFX11: v_cmpx_nge_f16 v127.l, v127.l dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xe9,0xfe,0x12,0x7d,0x7f,0x00,0x00,0x00]
 
-v_cmpx_nge_f16 v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:0
-// GFX11: v_cmpx_nge_f16 v127, v127 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xe9,0xfe,0x12,0x7d,0x7f,0x00,0x00,0x00]
+v_cmpx_nge_f16 v127.l, v127.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_nge_f16 v127.l, v127.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0xfe,0x12,0x7d,0x7f,0x77,0x39,0x05]
+
+v_cmpx_nge_f16 v1.h, v2.h dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: v_cmpx_nge_f16 v1.h, v2.h dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0xea,0x04,0x13,0x7d,0x81,0x77,0x39,0x05]
+
+v_cmpx_nge_f16 v127.h, v127.h dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: v_cmpx_nge_f16 v127.h, v127.h dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xe9,0xfe,0x13,0x7d,0xff,0x00,0x00,0x00]
 
 v_cmpx_nge_f32 v1, v2 dpp8:[7,6,5,4,3,2,1,0]
 // GFX11: v_cmpx_nge_f32 v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x32,0x7d,0x01,0x77,0x39,0x05]
@@ -542,14 +614,23 @@ v_cmpx_nge_f32 v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
 v_cmpx_nge_f32 v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
 // GFX11: v_cmpx_nge_f32 v255, v255 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xe9,0xfe,0x33,0x7d,0xff,0x00,0x00,0x00]
 
-v_cmpx_ngt_f16 v1, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmpx_ngt_f16 v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x16,0x7d,0x01,0x77,0x39,0x05]
+v_cmpx_ngt_f16 v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_ngt_f16 v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x16,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_ngt_f16 v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: v_cmpx_ngt_f16 v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0xea,0x04,0x16,0x7d,0x01,0x77,0x39,0x05]
 
-v_cmpx_ngt_f16 v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
-// GFX11: v_cmpx_ngt_f16 v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0xea,0x04,0x16,0x7d,0x01,0x77,0x39,0x05]
+v_cmpx_ngt_f16 v127.l, v127.l dpp8:[0,0,0,0,0,0,0,0]
+// GFX11: v_cmpx_ngt_f16 v127.l, v127.l dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xe9,0xfe,0x16,0x7d,0x7f,0x00,0x00,0x00]
 
-v_cmpx_ngt_f16 v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:0
-// GFX11: v_cmpx_ngt_f16 v127, v127 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xe9,0xfe,0x16,0x7d,0x7f,0x00,0x00,0x00]
+v_cmpx_ngt_f16 v127.l, v127.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_ngt_f16 v127.l, v127.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0xfe,0x16,0x7d,0x7f,0x77,0x39,0x05]
+
+v_cmpx_ngt_f16 v1.h, v2.h dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: v_cmpx_ngt_f16 v1.h, v2.h dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0xea,0x04,0x17,0x7d,0x81,0x77,0x39,0x05]
+
+v_cmpx_ngt_f16 v127.h, v127.h dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: v_cmpx_ngt_f16 v127.h, v127.h dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xe9,0xfe,0x17,0x7d,0xff,0x00,0x00,0x00]
 
 v_cmpx_ngt_f32 v1, v2 dpp8:[7,6,5,4,3,2,1,0]
 // GFX11: v_cmpx_ngt_f32 v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x36,0x7d,0x01,0x77,0x39,0x05]
@@ -560,14 +641,23 @@ v_cmpx_ngt_f32 v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
 v_cmpx_ngt_f32 v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
 // GFX11: v_cmpx_ngt_f32 v255, v255 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xe9,0xfe,0x37,0x7d,0xff,0x00,0x00,0x00]
 
-v_cmpx_nle_f16 v1, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmpx_nle_f16 v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x18,0x7d,0x01,0x77,0x39,0x05]
+v_cmpx_nle_f16 v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_nle_f16 v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x18,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_nle_f16 v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: v_cmpx_nle_f16 v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0xea,0x04,0x18,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_nle_f16 v127.l, v127.l dpp8:[0,0,0,0,0,0,0,0]
+// GFX11: v_cmpx_nle_f16 v127.l, v127.l dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xe9,0xfe,0x18,0x7d,0x7f,0x00,0x00,0x00]
 
-v_cmpx_nle_f16 v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
-// GFX11: v_cmpx_nle_f16 v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0xea,0x04,0x18,0x7d,0x01,0x77,0x39,0x05]
+v_cmpx_nle_f16 v127.l, v127.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_nle_f16 v127.l, v127.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0xfe,0x18,0x7d,0x7f,0x77,0x39,0x05]
 
-v_cmpx_nle_f16 v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:0
-// GFX11: v_cmpx_nle_f16 v127, v127 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xe9,0xfe,0x18,0x7d,0x7f,0x00,0x00,0x00]
+v_cmpx_nle_f16 v1.h, v2.h dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: v_cmpx_nle_f16 v1.h, v2.h dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0xea,0x04,0x19,0x7d,0x81,0x77,0x39,0x05]
+
+v_cmpx_nle_f16 v127.h, v127.h dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: v_cmpx_nle_f16 v127.h, v127.h dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xe9,0xfe,0x19,0x7d,0xff,0x00,0x00,0x00]
 
 v_cmpx_nle_f32 v1, v2 dpp8:[7,6,5,4,3,2,1,0]
 // GFX11: v_cmpx_nle_f32 v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x38,0x7d,0x01,0x77,0x39,0x05]
@@ -578,14 +668,23 @@ v_cmpx_nle_f32 v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
 v_cmpx_nle_f32 v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
 // GFX11: v_cmpx_nle_f32 v255, v255 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xe9,0xfe,0x39,0x7d,0xff,0x00,0x00,0x00]
 
-v_cmpx_nlg_f16 v1, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmpx_nlg_f16 v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x14,0x7d,0x01,0x77,0x39,0x05]
+v_cmpx_nlg_f16 v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_nlg_f16 v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x14,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_nlg_f16 v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: v_cmpx_nlg_f16 v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0xea,0x04,0x14,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_nlg_f16 v127.l, v127.l dpp8:[0,0,0,0,0,0,0,0]
+// GFX11: v_cmpx_nlg_f16 v127.l, v127.l dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xe9,0xfe,0x14,0x7d,0x7f,0x00,0x00,0x00]
 
-v_cmpx_nlg_f16 v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
-// GFX11: v_cmpx_nlg_f16 v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0xea,0x04,0x14,0x7d,0x01,0x77,0x39,0x05]
+v_cmpx_nlg_f16 v127.l, v127.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_nlg_f16 v127.l, v127.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0xfe,0x14,0x7d,0x7f,0x77,0x39,0x05]
 
-v_cmpx_nlg_f16 v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:0
-// GFX11: v_cmpx_nlg_f16 v127, v127 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xe9,0xfe,0x14,0x7d,0x7f,0x00,0x00,0x00]
+v_cmpx_nlg_f16 v1.h, v2.h dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: v_cmpx_nlg_f16 v1.h, v2.h dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0xea,0x04,0x15,0x7d,0x81,0x77,0x39,0x05]
+
+v_cmpx_nlg_f16 v127.h, v127.h dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: v_cmpx_nlg_f16 v127.h, v127.h dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xe9,0xfe,0x15,0x7d,0xff,0x00,0x00,0x00]
 
 v_cmpx_nlg_f32 v1, v2 dpp8:[7,6,5,4,3,2,1,0]
 // GFX11: v_cmpx_nlg_f32 v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x34,0x7d,0x01,0x77,0x39,0x05]
@@ -596,14 +695,23 @@ v_cmpx_nlg_f32 v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
 v_cmpx_nlg_f32 v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
 // GFX11: v_cmpx_nlg_f32 v255, v255 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xe9,0xfe,0x35,0x7d,0xff,0x00,0x00,0x00]
 
-v_cmpx_nlt_f16 v1, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmpx_nlt_f16 v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x1c,0x7d,0x01,0x77,0x39,0x05]
+v_cmpx_nlt_f16 v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_nlt_f16 v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x1c,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_nlt_f16 v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: v_cmpx_nlt_f16 v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0xea,0x04,0x1c,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_nlt_f16 v127.l, v127.l dpp8:[0,0,0,0,0,0,0,0]
+// GFX11: v_cmpx_nlt_f16 v127.l, v127.l dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xe9,0xfe,0x1c,0x7d,0x7f,0x00,0x00,0x00]
 
-v_cmpx_nlt_f16 v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
-// GFX11: v_cmpx_nlt_f16 v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0xea,0x04,0x1c,0x7d,0x01,0x77,0x39,0x05]
+v_cmpx_nlt_f16 v127.l, v127.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_nlt_f16 v127.l, v127.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0xfe,0x1c,0x7d,0x7f,0x77,0x39,0x05]
 
-v_cmpx_nlt_f16 v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:0
-// GFX11: v_cmpx_nlt_f16 v127, v127 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xe9,0xfe,0x1c,0x7d,0x7f,0x00,0x00,0x00]
+v_cmpx_nlt_f16 v1.h, v2.h dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: v_cmpx_nlt_f16 v1.h, v2.h dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0xea,0x04,0x1d,0x7d,0x81,0x77,0x39,0x05]
+
+v_cmpx_nlt_f16 v127.h, v127.h dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: v_cmpx_nlt_f16 v127.h, v127.h dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xe9,0xfe,0x1d,0x7d,0xff,0x00,0x00,0x00]
 
 v_cmpx_nlt_f32 v1, v2 dpp8:[7,6,5,4,3,2,1,0]
 // GFX11: v_cmpx_nlt_f32 v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x3c,0x7d,0x01,0x77,0x39,0x05]
@@ -614,14 +722,23 @@ v_cmpx_nlt_f32 v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
 v_cmpx_nlt_f32 v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
 // GFX11: v_cmpx_nlt_f32 v255, v255 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xe9,0xfe,0x3d,0x7d,0xff,0x00,0x00,0x00]
 
-v_cmpx_o_f16 v1, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmpx_o_f16 v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x0e,0x7d,0x01,0x77,0x39,0x05]
+v_cmpx_o_f16 v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_o_f16 v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x0e,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_o_f16 v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: v_cmpx_o_f16 v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0xea,0x04,0x0e,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_o_f16 v127.l, v127.l dpp8:[0,0,0,0,0,0,0,0]
+// GFX11: v_cmpx_o_f16 v127.l, v127.l dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xe9,0xfe,0x0e,0x7d,0x7f,0x00,0x00,0x00]
 
-v_cmpx_o_f16 v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
-// GFX11: v_cmpx_o_f16 v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0xea,0x04,0x0e,0x7d,0x01,0x77,0x39,0x05]
+v_cmpx_o_f16 v127.l, v127.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_o_f16 v127.l, v127.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0xfe,0x0e,0x7d,0x7f,0x77,0x39,0x05]
 
-v_cmpx_o_f16 v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:0
-// GFX11: v_cmpx_o_f16 v127, v127 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xe9,0xfe,0x0e,0x7d,0x7f,0x00,0x00,0x00]
+v_cmpx_o_f16 v1.h, v2.h dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: v_cmpx_o_f16 v1.h, v2.h dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0xea,0x04,0x0f,0x7d,0x81,0x77,0x39,0x05]
+
+v_cmpx_o_f16 v127.h, v127.h dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: v_cmpx_o_f16 v127.h, v127.h dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xe9,0xfe,0x0f,0x7d,0xff,0x00,0x00,0x00]
 
 v_cmpx_o_f32 v1, v2 dpp8:[7,6,5,4,3,2,1,0]
 // GFX11: v_cmpx_o_f32 v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x2e,0x7d,0x01,0x77,0x39,0x05]
@@ -632,14 +749,32 @@ v_cmpx_o_f32 v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
 v_cmpx_o_f32 v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
 // GFX11: v_cmpx_o_f32 v255, v255 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xe9,0xfe,0x2f,0x7d,0xff,0x00,0x00,0x00]
 
-v_cmpx_t_f16 v1, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmpx_t_f16 v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x1e,0x7d,0x01,0x77,0x39,0x05]
+v_cmpx_t_f16 v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_t_f16 v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x1e,0x7d,0x01,0x77,0x39,0x05]
 
-v_cmpx_t_f16 v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
-// GFX11: v_cmpx_t_f16 v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0xea,0x04,0x1e,0x7d,0x01,0x77,0x39,0x05]
+v_cmpx_t_f16 v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: v_cmpx_t_f16 v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0xea,0x04,0x1e,0x7d,0x01,0x77,0x39,0x05]
 
-v_cmpx_t_f16 v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:0
-// GFX11: v_cmpx_t_f16 v127, v127 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xe9,0xfe,0x1e,0x7d,0x7f,0x00,0x00,0x00]
+v_cmpx_t_f16 v127.l, v127.l dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: v_cmpx_t_f16 v127.l, v127.l dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xe9,0xfe,0x1e,0x7d,0x7f,0x00,0x00,0x00]
+
+v_cmpx_t_f16 v127.l, v127.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_t_f16 v127.l, v127.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0xfe,0x1e,0x7d,0x7f,0x77,0x39,0x05]
+
+v_cmpx_tru_f16 v127.l, v127.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_t_f16 v127.l, v127.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0xfe,0x1e,0x7d,0x7f,0x77,0x39,0x05]
+
+v_cmpx_t_f16 v1.h, v2.h dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: v_cmpx_t_f16 v1.h, v2.h dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0xea,0x04,0x1f,0x7d,0x81,0x77,0x39,0x05]
+
+v_cmpx_tru_f16 v1.h, v2.h dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: v_cmpx_t_f16 v1.h, v2.h dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0xea,0x04,0x1f,0x7d,0x81,0x77,0x39,0x05]
+
+v_cmpx_t_f16 v127.h, v127.h dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: v_cmpx_t_f16 v127.h, v127.h dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xe9,0xfe,0x1f,0x7d,0xff,0x00,0x00,0x00]
+
+v_cmpx_tru_f16 v127.h, v127.h dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: v_cmpx_t_f16 v127.h, v127.h dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xe9,0xfe,0x1f,0x7d,0xff,0x00,0x00,0x00]
 
 v_cmpx_t_f32 v1, v2 dpp8:[7,6,5,4,3,2,1,0]
 // GFX11: v_cmpx_t_f32 v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x3e,0x7d,0x01,0x77,0x39,0x05]
@@ -668,8 +803,8 @@ v_cmpx_t_u32 v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
 v_cmpx_t_u32 v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
 // GFX11: v_cmpx_t_u32 v255, v255 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xe9,0xfe,0x9f,0x7d,0xff,0x00,0x00,0x00]
 
-v_cmpx_tru_f16 v1, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmpx_t_f16 v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x1e,0x7d,0x01,0x77,0x39,0x05]
+v_cmpx_tru_f16 v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_t_f16 v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x1e,0x7d,0x01,0x77,0x39,0x05]
 
 v_cmpx_tru_f16 v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
 // GFX11: v_cmpx_t_f16 v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0xea,0x04,0x1e,0x7d,0x01,0x77,0x39,0x05]
@@ -686,14 +821,23 @@ v_cmpx_tru_f32 v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
 v_cmpx_tru_f32 v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
 // GFX11: v_cmpx_t_f32 v255, v255 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xe9,0xfe,0x3f,0x7d,0xff,0x00,0x00,0x00]
 
-v_cmpx_u_f16 v1, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmpx_u_f16 v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x10,0x7d,0x01,0x77,0x39,0x05]
+v_cmpx_u_f16 v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_u_f16 v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x10,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_u_f16 v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: v_cmpx_u_f16 v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0xea,0x04,0x10,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_u_f16 v127.l, v127.l dpp8:[0,0,0,0,0,0,0,0]
+// GFX11: v_cmpx_u_f16 v127.l, v127.l dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xe9,0xfe,0x10,0x7d,0x7f,0x00,0x00,0x00]
+
+v_cmpx_u_f16 v127.l, v127.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_u_f16 v127.l, v127.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0xfe,0x10,0x7d,0x7f,0x77,0x39,0x05]
 
-v_cmpx_u_f16 v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
-// GFX11: v_cmpx_u_f16 v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0xea,0x04,0x10,0x7d,0x01,0x77,0x39,0x05]
+v_cmpx_u_f16 v1.h, v2.h dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: v_cmpx_u_f16 v1.h, v2.h dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0xea,0x04,0x11,0x7d,0x81,0x77,0x39,0x05]
 
-v_cmpx_u_f16 v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:0
-// GFX11: v_cmpx_u_f16 v127, v127 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xe9,0xfe,0x10,0x7d,0x7f,0x00,0x00,0x00]
+v_cmpx_u_f16 v127.h, v127.h dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: v_cmpx_u_f16 v127.h, v127.h dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xe9,0xfe,0x11,0x7d,0xff,0x00,0x00,0x00]
 
 v_cmpx_u_f32 v1, v2 dpp8:[7,6,5,4,3,2,1,0]
 // GFX11: v_cmpx_u_f32 v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x30,0x7d,0x01,0x77,0x39,0x05]
diff --git a/llvm/test/MC/AMDGPU/gfx11_asm_vopcx_t16_err.s b/llvm/test/MC/AMDGPU/gfx11_asm_vopcx_t16_err.s
index 5cab502e99647..a1ab032e85d75 100644
--- a/llvm/test/MC/AMDGPU/gfx11_asm_vopcx_t16_err.s
+++ b/llvm/test/MC/AMDGPU/gfx11_asm_vopcx_t16_err.s
@@ -37,23 +37,41 @@ v_cmpx_class_f16_e32 v255.l, v2.l dpp8:[7,6,5,4,3,2,1,0]
 v_cmpx_class_f16_e32 v255.l, v2.l quad_perm:[3,2,1,0]
 // GFX11: :[[@LINE-1]]:22: error: invalid operand for instruction
 
-v_cmpx_eq_f16_e32 v1, v255
-// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+v_cmpx_eq_f16_e32 v1.h, v255.h
+// GFX11: :[[@LINE-1]]:25: error: invalid operand for instruction
 
-v_cmpx_eq_f16_e32 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:28: error: invalid operand for instruction
+v_cmpx_eq_f16_e32 v1.h, v255.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:25: error: invalid operand for instruction
 
-v_cmpx_eq_f16_e32 v1, v255 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:28: error: invalid operand for instruction
+v_cmpx_eq_f16_e32 v1.h, v255.h quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:25: error: invalid operand for instruction
 
-v_cmpx_eq_f16_e32 v255, v2
-// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+v_cmpx_eq_f16_e32 v1.l, v255.l
+// GFX11: :[[@LINE-1]]:25: error: invalid operand for instruction
 
-v_cmpx_eq_f16_e32 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:28: error: invalid operand for instruction
+v_cmpx_eq_f16_e32 v1.l, v255.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:25: error: invalid operand for instruction
 
-v_cmpx_eq_f16_e32 v255, v2 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:28: error: invalid operand for instruction
+v_cmpx_eq_f16_e32 v1.l, v255.l quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:25: error: invalid operand for instruction
+
+v_cmpx_eq_f16_e32 v255.h, v2.h
+// GFX11: :[[@LINE-1]]:19: error: invalid operand for instruction
+
+v_cmpx_eq_f16_e32 v255.h, v2.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:19: error: invalid operand for instruction
+
+v_cmpx_eq_f16_e32 v255.h, v2.h quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:19: error: invalid operand for instruction
+
+v_cmpx_eq_f16_e32 v255.l, v2.l
+// GFX11: :[[@LINE-1]]:19: error: invalid operand for instruction
+
+v_cmpx_eq_f16_e32 v255.l, v2.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:19: error: invalid operand for instruction
+
+v_cmpx_eq_f16_e32 v255.l, v2.l quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:19: error: invalid operand for instruction
 
 v_cmpx_eq_i16_e32 v1.h, v255.h
 // GFX11: :[[@LINE-1]]:25: error: invalid operand for instruction
@@ -127,41 +145,77 @@ v_cmpx_eq_u16_e32 v255.l, v2.l dpp8:[7,6,5,4,3,2,1,0]
 v_cmpx_eq_u16_e32 v255.l, v2.l quad_perm:[3,2,1,0]
 // GFX11: :[[@LINE-1]]:19: error: invalid operand for instruction
 
-v_cmpx_f_f16_e32 v1, v255
-// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+v_cmpx_f_f16_e32 v1.h, v255.h
+// GFX11: :[[@LINE-1]]:24: error: invalid operand for instruction
 
-v_cmpx_f_f16_e32 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:27: error: invalid operand for instruction
+v_cmpx_f_f16_e32 v1.h, v255.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:24: error: invalid operand for instruction
 
-v_cmpx_f_f16_e32 v1, v255 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:27: error: invalid operand for instruction
+v_cmpx_f_f16_e32 v1.h, v255.h quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:24: error: invalid operand for instruction
 
-v_cmpx_f_f16_e32 v255, v2
-// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+v_cmpx_f_f16_e32 v1.l, v255.l
+// GFX11: :[[@LINE-1]]:24: error: invalid operand for instruction
 
-v_cmpx_f_f16_e32 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:27: error: invalid operand for instruction
+v_cmpx_f_f16_e32 v1.l, v255.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:24: error: invalid operand for instruction
 
-v_cmpx_f_f16_e32 v255, v2 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:27: error: invalid operand for instruction
+v_cmpx_f_f16_e32 v1.l, v255.l quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:24: error: invalid operand for instruction
 
-v_cmpx_ge_f16_e32 v1, v255
-// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+v_cmpx_f_f16_e32 v255.h, v2.h
+// GFX11: :[[@LINE-1]]:18: error: invalid operand for instruction
 
-v_cmpx_ge_f16_e32 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:28: error: invalid operand for instruction
+v_cmpx_f_f16_e32 v255.h, v2.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:18: error: invalid operand for instruction
 
-v_cmpx_ge_f16_e32 v1, v255 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:28: error: invalid operand for instruction
+v_cmpx_f_f16_e32 v255.h, v2.h quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:18: error: invalid operand for instruction
 
-v_cmpx_ge_f16_e32 v255, v2
-// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+v_cmpx_f_f16_e32 v255.l, v2.l
+// GFX11: :[[@LINE-1]]:18: error: invalid operand for instruction
 
-v_cmpx_ge_f16_e32 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:28: error: invalid operand for instruction
+v_cmpx_f_f16_e32 v255.l, v2.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:18: error: invalid operand for instruction
 
-v_cmpx_ge_f16_e32 v255, v2 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:28: error: invalid operand for instruction
+v_cmpx_f_f16_e32 v255.l, v2.l quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:18: error: invalid operand for instruction
+
+v_cmpx_ge_f16_e32 v1.h, v255.h
+// GFX11: :[[@LINE-1]]:25: error: invalid operand for instruction
+
+v_cmpx_ge_f16_e32 v1.h, v255.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:25: error: invalid operand for instruction
+
+v_cmpx_ge_f16_e32 v1.h, v255.h quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:25: error: invalid operand for instruction
+
+v_cmpx_ge_f16_e32 v1.l, v255.l
+// GFX11: :[[@LINE-1]]:25: error: invalid operand for instruction
+
+v_cmpx_ge_f16_e32 v1.l, v255.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:25: error: invalid operand for instruction
+
+v_cmpx_ge_f16_e32 v1.l, v255.l quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:25: error: invalid operand for instruction
+
+v_cmpx_ge_f16_e32 v255.h, v2.h
+// GFX11: :[[@LINE-1]]:19: error: invalid operand for instruction
+
+v_cmpx_ge_f16_e32 v255.h, v2.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:19: error: invalid operand for instruction
+
+v_cmpx_ge_f16_e32 v255.h, v2.h quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:19: error: invalid operand for instruction
+
+v_cmpx_ge_f16_e32 v255.l, v2.l
+// GFX11: :[[@LINE-1]]:19: error: invalid operand for instruction
+
+v_cmpx_ge_f16_e32 v255.l, v2.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:19: error: invalid operand for instruction
+
+v_cmpx_ge_f16_e32 v255.l, v2.l quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:19: error: invalid operand for instruction
 
 v_cmpx_ge_i16_e32 v1.h, v255.h
 // GFX11: :[[@LINE-1]]:25: error: invalid operand for instruction
@@ -235,23 +289,41 @@ v_cmpx_ge_u16_e32 v255.l, v2.l dpp8:[7,6,5,4,3,2,1,0]
 v_cmpx_ge_u16_e32 v255.l, v2.l quad_perm:[3,2,1,0]
 // GFX11: :[[@LINE-1]]:19: error: invalid operand for instruction
 
-v_cmpx_gt_f16_e32 v1, v255
-// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+v_cmpx_gt_f16_e32 v1.h, v255.h
+// GFX11: :[[@LINE-1]]:25: error: invalid operand for instruction
 
-v_cmpx_gt_f16_e32 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:28: error: invalid operand for instruction
+v_cmpx_gt_f16_e32 v1.h, v255.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:25: error: invalid operand for instruction
 
-v_cmpx_gt_f16_e32 v1, v255 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:28: error: invalid operand for instruction
+v_cmpx_gt_f16_e32 v1.h, v255.h quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:25: error: invalid operand for instruction
 
-v_cmpx_gt_f16_e32 v255, v2
-// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+v_cmpx_gt_f16_e32 v1.l, v255.l
+// GFX11: :[[@LINE-1]]:25: error: invalid operand for instruction
 
-v_cmpx_gt_f16_e32 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:28: error: invalid operand for instruction
+v_cmpx_gt_f16_e32 v1.l, v255.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:25: error: invalid operand for instruction
 
-v_cmpx_gt_f16_e32 v255, v2 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:28: error: invalid operand for instruction
+v_cmpx_gt_f16_e32 v1.l, v255.l quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:25: error: invalid operand for instruction
+
+v_cmpx_gt_f16_e32 v255.h, v2.h
+// GFX11: :[[@LINE-1]]:19: error: invalid operand for instruction
+
+v_cmpx_gt_f16_e32 v255.h, v2.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:19: error: invalid operand for instruction
+
+v_cmpx_gt_f16_e32 v255.h, v2.h quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:19: error: invalid operand for instruction
+
+v_cmpx_gt_f16_e32 v255.l, v2.l
+// GFX11: :[[@LINE-1]]:19: error: invalid operand for instruction
+
+v_cmpx_gt_f16_e32 v255.l, v2.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:19: error: invalid operand for instruction
+
+v_cmpx_gt_f16_e32 v255.l, v2.l quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:19: error: invalid operand for instruction
 
 v_cmpx_gt_i16_e32 v1.h, v255.h
 // GFX11: :[[@LINE-1]]:25: error: invalid operand for instruction
@@ -325,23 +397,41 @@ v_cmpx_gt_u16_e32 v255.l, v2.l dpp8:[7,6,5,4,3,2,1,0]
 v_cmpx_gt_u16_e32 v255.l, v2.l quad_perm:[3,2,1,0]
 // GFX11: :[[@LINE-1]]:19: error: invalid operand for instruction
 
-v_cmpx_le_f16_e32 v1, v255
-// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+v_cmpx_le_f16_e32 v1.h, v255.h
+// GFX11: :[[@LINE-1]]:25: error: invalid operand for instruction
 
-v_cmpx_le_f16_e32 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:28: error: invalid operand for instruction
+v_cmpx_le_f16_e32 v1.h, v255.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:25: error: invalid operand for instruction
 
-v_cmpx_le_f16_e32 v1, v255 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:28: error: invalid operand for instruction
+v_cmpx_le_f16_e32 v1.h, v255.h quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:25: error: invalid operand for instruction
 
-v_cmpx_le_f16_e32 v255, v2
-// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+v_cmpx_le_f16_e32 v1.l, v255.l
+// GFX11: :[[@LINE-1]]:25: error: invalid operand for instruction
 
-v_cmpx_le_f16_e32 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:28: error: invalid operand for instruction
+v_cmpx_le_f16_e32 v1.l, v255.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:25: error: invalid operand for instruction
 
-v_cmpx_le_f16_e32 v255, v2 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:28: error: invalid operand for instruction
+v_cmpx_le_f16_e32 v1.l, v255.l quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:25: error: invalid operand for instruction
+
+v_cmpx_le_f16_e32 v255.h, v2.h
+// GFX11: :[[@LINE-1]]:19: error: invalid operand for instruction
+
+v_cmpx_le_f16_e32 v255.h, v2.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:19: error: invalid operand for instruction
+
+v_cmpx_le_f16_e32 v255.h, v2.h quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:19: error: invalid operand for instruction
+
+v_cmpx_le_f16_e32 v255.l, v2.l
+// GFX11: :[[@LINE-1]]:19: error: invalid operand for instruction
+
+v_cmpx_le_f16_e32 v255.l, v2.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:19: error: invalid operand for instruction
+
+v_cmpx_le_f16_e32 v255.l, v2.l quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:19: error: invalid operand for instruction
 
 v_cmpx_le_i16_e32 v1.h, v255.h
 // GFX11: :[[@LINE-1]]:25: error: invalid operand for instruction
@@ -415,23 +505,41 @@ v_cmpx_le_u16_e32 v255.l, v2.l dpp8:[7,6,5,4,3,2,1,0]
 v_cmpx_le_u16_e32 v255.l, v2.l quad_perm:[3,2,1,0]
 // GFX11: :[[@LINE-1]]:19: error: invalid operand for instruction
 
-v_cmpx_lg_f16_e32 v1, v255
-// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+v_cmpx_lg_f16_e32 v1.h, v255.h
+// GFX11: :[[@LINE-1]]:25: error: invalid operand for instruction
 
-v_cmpx_lg_f16_e32 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:28: error: invalid operand for instruction
+v_cmpx_lg_f16_e32 v1.h, v255.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:25: error: invalid operand for instruction
 
-v_cmpx_lg_f16_e32 v1, v255 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:28: error: invalid operand for instruction
+v_cmpx_lg_f16_e32 v1.h, v255.h quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:25: error: invalid operand for instruction
 
-v_cmpx_lg_f16_e32 v255, v2
-// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+v_cmpx_lg_f16_e32 v1.l, v255.l
+// GFX11: :[[@LINE-1]]:25: error: invalid operand for instruction
 
-v_cmpx_lg_f16_e32 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:28: error: invalid operand for instruction
+v_cmpx_lg_f16_e32 v1.l, v255.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:25: error: invalid operand for instruction
 
-v_cmpx_lg_f16_e32 v255, v2 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:28: error: invalid operand for instruction
+v_cmpx_lg_f16_e32 v1.l, v255.l quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:25: error: invalid operand for instruction
+
+v_cmpx_lg_f16_e32 v255.h, v2.h
+// GFX11: :[[@LINE-1]]:19: error: invalid operand for instruction
+
+v_cmpx_lg_f16_e32 v255.h, v2.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:19: error: invalid operand for instruction
+
+v_cmpx_lg_f16_e32 v255.h, v2.h quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:19: error: invalid operand for instruction
+
+v_cmpx_lg_f16_e32 v255.l, v2.l
+// GFX11: :[[@LINE-1]]:19: error: invalid operand for instruction
+
+v_cmpx_lg_f16_e32 v255.l, v2.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:19: error: invalid operand for instruction
+
+v_cmpx_lg_f16_e32 v255.l, v2.l quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:19: error: invalid operand for instruction
 
 v_cmpx_lt_f16_e32 v1.h, v255.h
 // GFX11: :[[@LINE-1]]:25: error: invalid operand for instruction
@@ -613,148 +721,292 @@ v_cmpx_ne_u16_e32 v255.l, v2.l dpp8:[7,6,5,4,3,2,1,0]
 v_cmpx_ne_u16_e32 v255.l, v2.l quad_perm:[3,2,1,0]
 // GFX11: :[[@LINE-1]]:19: error: invalid operand for instruction
 
-v_cmpx_neq_f16_e32 v1, v255
-// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+v_cmpx_neq_f16_e32 v1.h, v255.h
+// GFX11: :[[@LINE-1]]:26: error: invalid operand for instruction
+
+v_cmpx_neq_f16_e32 v1.h, v255.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:26: error: invalid operand for instruction
+
+v_cmpx_neq_f16_e32 v1.h, v255.h quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:26: error: invalid operand for instruction
+
+v_cmpx_neq_f16_e32 v1.l, v255.l
+// GFX11: :[[@LINE-1]]:26: error: invalid operand for instruction
 
-v_cmpx_neq_f16_e32 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:29: error: invalid operand for instruction
+v_cmpx_neq_f16_e32 v1.l, v255.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:26: error: invalid operand for instruction
 
-v_cmpx_neq_f16_e32 v1, v255 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:29: error: invalid operand for instruction
+v_cmpx_neq_f16_e32 v1.l, v255.l quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:26: error: invalid operand for instruction
 
-v_cmpx_neq_f16_e32 v255, v2
-// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+v_cmpx_neq_f16_e32 v255.h, v2.h
+// GFX11: :[[@LINE-1]]:20: error: invalid operand for instruction
+
+v_cmpx_neq_f16_e32 v255.h, v2.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:20: error: invalid operand for instruction
 
-v_cmpx_neq_f16_e32 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:29: error: invalid operand for instruction
+v_cmpx_neq_f16_e32 v255.h, v2.h quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:20: error: invalid operand for instruction
 
-v_cmpx_neq_f16_e32 v255, v2 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:29: error: invalid operand for instruction
+v_cmpx_neq_f16_e32 v255.l, v2.l
+// GFX11: :[[@LINE-1]]:20: error: invalid operand for instruction
 
-v_cmpx_nge_f16_e32 v1, v255
-// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+v_cmpx_neq_f16_e32 v255.l, v2.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:20: error: invalid operand for instruction
 
-v_cmpx_nge_f16_e32 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:29: error: invalid operand for instruction
+v_cmpx_neq_f16_e32 v255.l, v2.l quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:20: error: invalid operand for instruction
 
-v_cmpx_nge_f16_e32 v1, v255 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:29: error: invalid operand for instruction
+v_cmpx_nge_f16_e32 v1.h, v255.h
+// GFX11: :[[@LINE-1]]:26: error: invalid operand for instruction
 
-v_cmpx_nge_f16_e32 v255, v2
-// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+v_cmpx_nge_f16_e32 v1.h, v255.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:26: error: invalid operand for instruction
 
-v_cmpx_nge_f16_e32 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:29: error: invalid operand for instruction
+v_cmpx_nge_f16_e32 v1.h, v255.h quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:26: error: invalid operand for instruction
 
-v_cmpx_nge_f16_e32 v255, v2 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:29: error: invalid operand for instruction
+v_cmpx_nge_f16_e32 v1.l, v255.l
+// GFX11: :[[@LINE-1]]:26: error: invalid operand for instruction
 
-v_cmpx_ngt_f16_e32 v1, v255
-// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+v_cmpx_nge_f16_e32 v1.l, v255.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:26: error: invalid operand for instruction
 
-v_cmpx_ngt_f16_e32 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:29: error: invalid operand for instruction
+v_cmpx_nge_f16_e32 v1.l, v255.l quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:26: error: invalid operand for instruction
 
-v_cmpx_ngt_f16_e32 v1, v255 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:29: error: invalid operand for instruction
+v_cmpx_nge_f16_e32 v255.h, v2.h
+// GFX11: :[[@LINE-1]]:20: error: invalid operand for instruction
 
-v_cmpx_ngt_f16_e32 v255, v2
-// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+v_cmpx_nge_f16_e32 v255.h, v2.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:20: error: invalid operand for instruction
 
-v_cmpx_ngt_f16_e32 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:29: error: invalid operand for instruction
+v_cmpx_nge_f16_e32 v255.h, v2.h quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:20: error: invalid operand for instruction
 
-v_cmpx_ngt_f16_e32 v255, v2 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:29: error: invalid operand for instruction
+v_cmpx_nge_f16_e32 v255.l, v2.l
+// GFX11: :[[@LINE-1]]:20: error: invalid operand for instruction
 
-v_cmpx_nle_f16_e32 v1, v255
-// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+v_cmpx_nge_f16_e32 v255.l, v2.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:20: error: invalid operand for instruction
 
-v_cmpx_nle_f16_e32 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:29: error: invalid operand for instruction
+v_cmpx_nge_f16_e32 v255.l, v2.l quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:20: error: invalid operand for instruction
 
-v_cmpx_nle_f16_e32 v1, v255 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:29: error: invalid operand for instruction
+v_cmpx_ngt_f16_e32 v1.h, v255.h
+// GFX11: :[[@LINE-1]]:26: error: invalid operand for instruction
 
-v_cmpx_nle_f16_e32 v255, v2
-// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+v_cmpx_ngt_f16_e32 v1.h, v255.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:26: error: invalid operand for instruction
 
-v_cmpx_nle_f16_e32 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:29: error: invalid operand for instruction
+v_cmpx_ngt_f16_e32 v1.h, v255.h quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:26: error: invalid operand for instruction
 
-v_cmpx_nle_f16_e32 v255, v2 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:29: error: invalid operand for instruction
+v_cmpx_ngt_f16_e32 v1.l, v255.l
+// GFX11: :[[@LINE-1]]:26: error: invalid operand for instruction
 
-v_cmpx_nlg_f16_e32 v1, v255
-// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+v_cmpx_ngt_f16_e32 v1.l, v255.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:26: error: invalid operand for instruction
 
-v_cmpx_nlg_f16_e32 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:29: error: invalid operand for instruction
+v_cmpx_ngt_f16_e32 v1.l, v255.l quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:26: error: invalid operand for instruction
 
-v_cmpx_nlg_f16_e32 v1, v255 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:29: error: invalid operand for instruction
+v_cmpx_ngt_f16_e32 v255.h, v2.h
+// GFX11: :[[@LINE-1]]:20: error: invalid operand for instruction
 
-v_cmpx_nlg_f16_e32 v255, v2
-// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+v_cmpx_ngt_f16_e32 v255.h, v2.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:20: error: invalid operand for instruction
 
-v_cmpx_nlg_f16_e32 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:29: error: invalid operand for instruction
+v_cmpx_ngt_f16_e32 v255.h, v2.h quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:20: error: invalid operand for instruction
 
-v_cmpx_nlg_f16_e32 v255, v2 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:29: error: invalid operand for instruction
+v_cmpx_ngt_f16_e32 v255.l, v2.l
+// GFX11: :[[@LINE-1]]:20: error: invalid operand for instruction
 
-v_cmpx_nlt_f16_e32 v1, v255
-// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+v_cmpx_ngt_f16_e32 v255.l, v2.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:20: error: invalid operand for instruction
 
-v_cmpx_nlt_f16_e32 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:29: error: invalid operand for instruction
+v_cmpx_ngt_f16_e32 v255.l, v2.l quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:20: error: invalid operand for instruction
 
-v_cmpx_nlt_f16_e32 v1, v255 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:29: error: invalid operand for instruction
+v_cmpx_nle_f16_e32 v1.h, v255.h
+// GFX11: :[[@LINE-1]]:26: error: invalid operand for instruction
 
-v_cmpx_nlt_f16_e32 v255, v2
-// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+v_cmpx_nle_f16_e32 v1.h, v255.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:26: error: invalid operand for instruction
 
-v_cmpx_nlt_f16_e32 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:29: error: invalid operand for instruction
+v_cmpx_nle_f16_e32 v1.h, v255.h quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:26: error: invalid operand for instruction
 
-v_cmpx_nlt_f16_e32 v255, v2 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:29: error: invalid operand for instruction
+v_cmpx_nle_f16_e32 v1.l, v255.l
+// GFX11: :[[@LINE-1]]:26: error: invalid operand for instruction
 
-v_cmpx_o_f16_e32 v1, v255
-// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+v_cmpx_nle_f16_e32 v1.l, v255.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:26: error: invalid operand for instruction
 
-v_cmpx_o_f16_e32 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:27: error: invalid operand for instruction
+v_cmpx_nle_f16_e32 v1.l, v255.l quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:26: error: invalid operand for instruction
 
-v_cmpx_o_f16_e32 v1, v255 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:27: error: invalid operand for instruction
+v_cmpx_nle_f16_e32 v255.h, v2.h
+// GFX11: :[[@LINE-1]]:20: error: invalid operand for instruction
 
-v_cmpx_o_f16_e32 v255, v2
-// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+v_cmpx_nle_f16_e32 v255.h, v2.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:20: error: invalid operand for instruction
 
-v_cmpx_o_f16_e32 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:27: error: invalid operand for instruction
+v_cmpx_nle_f16_e32 v255.h, v2.h quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:20: error: invalid operand for instruction
 
-v_cmpx_o_f16_e32 v255, v2 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:27: error: invalid operand for instruction
+v_cmpx_nle_f16_e32 v255.l, v2.l
+// GFX11: :[[@LINE-1]]:20: error: invalid operand for instruction
 
-v_cmpx_t_f16_e32 v1, v255
-// GFX11: :[[@LINE-1]]:22: error: invalid operand for instruction
+v_cmpx_nle_f16_e32 v255.l, v2.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:20: error: invalid operand for instruction
 
-v_cmpx_t_f16_e32 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:22: error: invalid operand for instruction
+v_cmpx_nle_f16_e32 v255.l, v2.l quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:20: error: invalid operand for instruction
 
-v_cmpx_t_f16_e32 v1, v255 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:22: error: invalid operand for instruction
+v_cmpx_nlg_f16_e32 v1.h, v255.h
+// GFX11: :[[@LINE-1]]:26: error: invalid operand for instruction
 
-v_cmpx_t_f16_e32 v255, v2
+v_cmpx_nlg_f16_e32 v1.h, v255.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:26: error: invalid operand for instruction
+
+v_cmpx_nlg_f16_e32 v1.h, v255.h quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:26: error: invalid operand for instruction
+
+v_cmpx_nlg_f16_e32 v1.l, v255.l
+// GFX11: :[[@LINE-1]]:26: error: invalid operand for instruction
+
+v_cmpx_nlg_f16_e32 v1.l, v255.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:26: error: invalid operand for instruction
+
+v_cmpx_nlg_f16_e32 v1.l, v255.l quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:26: error: invalid operand for instruction
+
+v_cmpx_nlg_f16_e32 v255.h, v2.h
+// GFX11: :[[@LINE-1]]:20: error: invalid operand for instruction
+
+v_cmpx_nlg_f16_e32 v255.h, v2.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:20: error: invalid operand for instruction
+
+v_cmpx_nlg_f16_e32 v255.h, v2.h quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:20: error: invalid operand for instruction
+
+v_cmpx_nlg_f16_e32 v255.l, v2.l
+// GFX11: :[[@LINE-1]]:20: error: invalid operand for instruction
+
+v_cmpx_nlg_f16_e32 v255.l, v2.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:20: error: invalid operand for instruction
+
+v_cmpx_nlg_f16_e32 v255.l, v2.l quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:20: error: invalid operand for instruction
+
+v_cmpx_nlt_f16_e32 v1.h, v255.h
+// GFX11: :[[@LINE-1]]:26: error: invalid operand for instruction
+
+v_cmpx_nlt_f16_e32 v1.h, v255.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:26: error: invalid operand for instruction
+
+v_cmpx_nlt_f16_e32 v1.h, v255.h quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:26: error: invalid operand for instruction
+
+v_cmpx_nlt_f16_e32 v1.l, v255.l
+// GFX11: :[[@LINE-1]]:26: error: invalid operand for instruction
+
+v_cmpx_nlt_f16_e32 v1.l, v255.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:26: error: invalid operand for instruction
+
+v_cmpx_nlt_f16_e32 v1.l, v255.l quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:26: error: invalid operand for instruction
+
+v_cmpx_nlt_f16_e32 v255.h, v2.h
+// GFX11: :[[@LINE-1]]:20: error: invalid operand for instruction
+
+v_cmpx_nlt_f16_e32 v255.h, v2.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:20: error: invalid operand for instruction
+
+v_cmpx_nlt_f16_e32 v255.h, v2.h quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:20: error: invalid operand for instruction
+
+v_cmpx_nlt_f16_e32 v255.l, v2.l
+// GFX11: :[[@LINE-1]]:20: error: invalid operand for instruction
+
+v_cmpx_nlt_f16_e32 v255.l, v2.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:20: error: invalid operand for instruction
+
+v_cmpx_nlt_f16_e32 v255.l, v2.l quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:20: error: invalid operand for instruction
+
+v_cmpx_o_f16_e32 v1.h, v255.h
+// GFX11: :[[@LINE-1]]:24: error: invalid operand for instruction
+
+v_cmpx_o_f16_e32 v1.h, v255.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:24: error: invalid operand for instruction
+
+v_cmpx_o_f16_e32 v1.h, v255.h quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:24: error: invalid operand for instruction
+
+v_cmpx_o_f16_e32 v1.l, v255.l
+// GFX11: :[[@LINE-1]]:24: error: invalid operand for instruction
+
+v_cmpx_o_f16_e32 v1.l, v255.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:24: error: invalid operand for instruction
+
+v_cmpx_o_f16_e32 v1.l, v255.l quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:24: error: invalid operand for instruction
+
+v_cmpx_o_f16_e32 v255.h, v2.h
+// GFX11: :[[@LINE-1]]:18: error: invalid operand for instruction
+
+v_cmpx_o_f16_e32 v255.h, v2.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:18: error: invalid operand for instruction
+
+v_cmpx_o_f16_e32 v255.h, v2.h quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:18: error: invalid operand for instruction
+
+v_cmpx_o_f16_e32 v255.l, v2.l
 // GFX11: :[[@LINE-1]]:18: error: invalid operand for instruction
 
-v_cmpx_t_f16_e32 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
+v_cmpx_o_f16_e32 v255.l, v2.l dpp8:[7,6,5,4,3,2,1,0]
 // GFX11: :[[@LINE-1]]:18: error: invalid operand for instruction
 
-v_cmpx_t_f16_e32 v255, v2 quad_perm:[3,2,1,0]
+v_cmpx_o_f16_e32 v255.l, v2.l quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:18: error: invalid operand for instruction
+
+v_cmpx_t_f16_e32 v1.h, v255.h
+// GFX11: :[[@LINE-1]]:24: error: invalid operand for instruction
+
+v_cmpx_t_f16_e32 v1.h, v255.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:24: error: invalid operand for instruction
+
+v_cmpx_t_f16_e32 v1.h, v255.h quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:24: error: invalid operand for instruction
+
+v_cmpx_t_f16_e32 v1.l, v255.l
+// GFX11: :[[@LINE-1]]:24: error: invalid operand for instruction
+
+v_cmpx_t_f16_e32 v1.l, v255.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:24: error: invalid operand for instruction
+
+v_cmpx_t_f16_e32 v1.l, v255.l quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:24: error: invalid operand for instruction
+
+v_cmpx_t_f16_e32 v255.h, v2.h
+// GFX11: :[[@LINE-1]]:18: error: invalid operand for instruction
+
+v_cmpx_t_f16_e32 v255.h, v2.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:18: error: invalid operand for instruction
+
+v_cmpx_t_f16_e32 v255.h, v2.h quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:18: error: invalid operand for instruction
+
+v_cmpx_t_f16_e32 v255.l, v2.l
+// GFX11: :[[@LINE-1]]:18: error: invalid operand for instruction
+
+v_cmpx_t_f16_e32 v255.l, v2.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:18: error: invalid operand for instruction
+
+v_cmpx_t_f16_e32 v255.l, v2.l quad_perm:[3,2,1,0]
 // GFX11: :[[@LINE-1]]:18: error: invalid operand for instruction
 
 v_cmpx_tru_f16_e32 v1, v255
@@ -775,20 +1027,38 @@ v_cmpx_tru_f16_e32 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
 v_cmpx_tru_f16_e32 v255, v2 quad_perm:[3,2,1,0]
 // GFX11: :[[@LINE-1]]:20: error: invalid operand for instruction
 
-v_cmpx_u_f16_e32 v1, v255
-// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+v_cmpx_u_f16_e32 v1.h, v255.h
+// GFX11: :[[@LINE-1]]:24: error: invalid operand for instruction
 
-v_cmpx_u_f16_e32 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:27: error: invalid operand for instruction
+v_cmpx_u_f16_e32 v1.h, v255.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:24: error: invalid operand for instruction
 
-v_cmpx_u_f16_e32 v1, v255 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:27: error: invalid operand for instruction
+v_cmpx_u_f16_e32 v1.h, v255.h quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:24: error: invalid operand for instruction
 
-v_cmpx_u_f16_e32 v255, v2
-// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+v_cmpx_u_f16_e32 v1.l, v255.l
+// GFX11: :[[@LINE-1]]:24: error: invalid operand for instruction
+
+v_cmpx_u_f16_e32 v1.l, v255.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:24: error: invalid operand for instruction
+
+v_cmpx_u_f16_e32 v1.l, v255.l quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:24: error: invalid operand for instruction
+
+v_cmpx_u_f16_e32 v255.h, v2.h
+// GFX11: :[[@LINE-1]]:18: error: invalid operand for instruction
 
-v_cmpx_u_f16_e32 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:27: error: invalid operand for instruction
+v_cmpx_u_f16_e32 v255.h, v2.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:18: error: invalid operand for instruction
 
-v_cmpx_u_f16_e32 v255, v2 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:27: error: invalid operand for instruction
+v_cmpx_u_f16_e32 v255.h, v2.h quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:18: error: invalid operand for instruction
+
+v_cmpx_u_f16_e32 v255.l, v2.l
+// GFX11: :[[@LINE-1]]:18: error: invalid operand for instruction
+
+v_cmpx_u_f16_e32 v255.l, v2.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:18: error: invalid operand for instruction
+
+v_cmpx_u_f16_e32 v255.l, v2.l quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:18: error: invalid operand for instruction
diff --git a/llvm/test/MC/AMDGPU/gfx11_asm_vopcx_t16_promote.s b/llvm/test/MC/AMDGPU/gfx11_asm_vopcx_t16_promote.s
index 5102a32075066..7e5e59b9668a3 100644
--- a/llvm/test/MC/AMDGPU/gfx11_asm_vopcx_t16_promote.s
+++ b/llvm/test/MC/AMDGPU/gfx11_asm_vopcx_t16_promote.s
@@ -2,7 +2,7 @@
 // RUN: llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize64,+real-true16 -show-encoding %s 2>&1 | FileCheck --check-prefix=GFX11 %s
 
 v_cmpx_class_f16 v1.h, v255.h
-// GFX11: v_cmpx_class_f16_e64 v1.h, v255.h       ; encoding: [0x7e,0x18,0xfd,0xd4,0x01,0xff,0x03,0x00]
+// GFX11: v_cmpx_class_f16_e64 v1.h, v255.h op_sel:[1,1,0] ; encoding: [0x7e,0x18,0xfd,0xd4,0x01,0xff,0x03,0x00]
 
 v_cmpx_class_f16 v1.h, v255.h dpp8:[7,6,5,4,3,2,1,0]
 // GFX11: v_cmpx_class_f16_e64_dpp v1.h, v255.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x18,0xfd,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
@@ -20,7 +20,7 @@ v_cmpx_class_f16 v1.l, v255.l quad_perm:[3,2,1,0]
 // GFX11: v_cmpx_class_f16_e64_dpp v1.l, v255.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xfd,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
 
 v_cmpx_class_f16 v255.h, v2.h
-// GFX11: v_cmpx_class_f16_e64 v255.h, v2.h       ; encoding: [0x7e,0x18,0xfd,0xd4,0xff,0x05,0x02,0x00]
+// GFX11: v_cmpx_class_f16_e64 v255.h, v2.h op_sel:[1,1,0] ; encoding: [0x7e,0x18,0xfd,0xd4,0xff,0x05,0x02,0x00]
 
 v_cmpx_class_f16 v255.h, v2.h dpp8:[7,6,5,4,3,2,1,0]
 // GFX11: v_cmpx_class_f16_e64_dpp v255.h, v2.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x18,0xfd,0xd4,0xe9,0x04,0x02,0x00,0xff,0x77,0x39,0x05]
@@ -37,26 +37,44 @@ v_cmpx_class_f16 v255.l, v2.l dpp8:[7,6,5,4,3,2,1,0]
 v_cmpx_class_f16 v255.l, v2.l quad_perm:[3,2,1,0]
 // GFX11: v_cmpx_class_f16_e64_dpp v255.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xfd,0xd4,0xfa,0x04,0x02,0x00,0xff,0x1b,0x00,0xff]
 
-v_cmpx_eq_f16 v1, v255
-// GFX11: v_cmpx_eq_f16_e64 v1, v255              ; encoding: [0x7e,0x00,0x82,0xd4,0x01,0xff,0x03,0x00]
+v_cmpx_eq_f16 v1.h, v255.h
+// GFX11: v_cmpx_eq_f16_e64 v1.h, v255.h op_sel:[1,1,0] ; encoding: [0x7e,0x18,0x82,0xd4,0x01,0xff,0x03,0x00]
 
-v_cmpx_eq_f16 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmpx_eq_f16_e64_dpp v1, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x82,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
+v_cmpx_eq_f16 v1.h, v255.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_eq_f16_e64_dpp v1.h, v255.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x18,0x82,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
 
-v_cmpx_eq_f16 v1, v255 quad_perm:[3,2,1,0]
-// GFX11: v_cmpx_eq_f16_e64_dpp v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x82,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
+v_cmpx_eq_f16 v1.h, v255.h quad_perm:[3,2,1,0]
+// GFX11: v_cmpx_eq_f16_e64_dpp v1.h, v255.h op_sel:[1,1] quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x18,0x82,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
 
-v_cmpx_eq_f16 v255, v2
-// GFX11: v_cmpx_eq_f16_e64 v255, v2              ; encoding: [0x7e,0x00,0x82,0xd4,0xff,0x05,0x02,0x00]
+v_cmpx_eq_f16 v1.l, v255.l
+// GFX11: v_cmpx_eq_f16_e64 v1.l, v255.l          ; encoding: [0x7e,0x00,0x82,0xd4,0x01,0xff,0x03,0x00]
 
-v_cmpx_eq_f16 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmpx_eq_f16_e64_dpp v255, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x82,0xd4,0xe9,0x04,0x02,0x00,0xff,0x77,0x39,0x05]
+v_cmpx_eq_f16 v1.l, v255.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_eq_f16_e64_dpp v1.l, v255.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x82,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
 
-v_cmpx_eq_f16 v255, v2 quad_perm:[3,2,1,0]
-// GFX11: v_cmpx_eq_f16_e64_dpp v255, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x82,0xd4,0xfa,0x04,0x02,0x00,0xff,0x1b,0x00,0xff]
+v_cmpx_eq_f16 v1.l, v255.l quad_perm:[3,2,1,0]
+// GFX11: v_cmpx_eq_f16_e64_dpp v1.l, v255.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x82,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
+
+v_cmpx_eq_f16 v255.h, v2.h
+// GFX11: v_cmpx_eq_f16_e64 v255.h, v2.h op_sel:[1,1,0] ; encoding: [0x7e,0x18,0x82,0xd4,0xff,0x05,0x02,0x00]
+
+v_cmpx_eq_f16 v255.h, v2.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_eq_f16_e64_dpp v255.h, v2.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x18,0x82,0xd4,0xe9,0x04,0x02,0x00,0xff,0x77,0x39,0x05]
+
+v_cmpx_eq_f16 v255.h, v2.h quad_perm:[3,2,1,0]
+// GFX11: v_cmpx_eq_f16_e64_dpp v255.h, v2.h op_sel:[1,1] quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x18,0x82,0xd4,0xfa,0x04,0x02,0x00,0xff,0x1b,0x00,0xff]
+
+v_cmpx_eq_f16 v255.l, v2.l
+// GFX11: v_cmpx_eq_f16_e64 v255.l, v2.l          ; encoding: [0x7e,0x00,0x82,0xd4,0xff,0x05,0x02,0x00]
+
+v_cmpx_eq_f16 v255.l, v2.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_eq_f16_e64_dpp v255.l, v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x82,0xd4,0xe9,0x04,0x02,0x00,0xff,0x77,0x39,0x05]
+
+v_cmpx_eq_f16 v255.l, v2.l quad_perm:[3,2,1,0]
+// GFX11: v_cmpx_eq_f16_e64_dpp v255.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x82,0xd4,0xfa,0x04,0x02,0x00,0xff,0x1b,0x00,0xff]
 
 v_cmpx_eq_i16 v1.h, v255.h
-// GFX11: v_cmpx_eq_i16_e64 v1.h, v255.h          ; encoding: [0x7e,0x18,0xb2,0xd4,0x01,0xff,0x03,0x00]
+// GFX11: v_cmpx_eq_i16_e64 v1.h, v255.h op_sel:[1,1,0] ; encoding: [0x7e,0x18,0xb2,0xd4,0x01,0xff,0x03,0x00]
 
 v_cmpx_eq_i16 v1.h, v255.h dpp8:[7,6,5,4,3,2,1,0]
 // GFX11: v_cmpx_eq_i16_e64_dpp v1.h, v255.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x18,0xb2,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
@@ -74,7 +92,7 @@ v_cmpx_eq_i16 v1.l, v255.l quad_perm:[3,2,1,0]
 // GFX11: v_cmpx_eq_i16_e64_dpp v1.l, v255.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xb2,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
 
 v_cmpx_eq_i16 v255.h, v2.h
-// GFX11: v_cmpx_eq_i16_e64 v255.h, v2.h          ; encoding: [0x7e,0x18,0xb2,0xd4,0xff,0x05,0x02,0x00]
+// GFX11: v_cmpx_eq_i16_e64 v255.h, v2.h op_sel:[1,1,0] ; encoding: [0x7e,0x18,0xb2,0xd4,0xff,0x05,0x02,0x00]
 
 v_cmpx_eq_i16 v255.h, v2.h dpp8:[7,6,5,4,3,2,1,0]
 // GFX11: v_cmpx_eq_i16_e64_dpp v255.h, v2.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x18,0xb2,0xd4,0xe9,0x04,0x02,0x00,0xff,0x77,0x39,0x05]
@@ -92,7 +110,7 @@ v_cmpx_eq_i16 v255.l, v2.l quad_perm:[3,2,1,0]
 // GFX11: v_cmpx_eq_i16_e64_dpp v255.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xb2,0xd4,0xfa,0x04,0x02,0x00,0xff,0x1b,0x00,0xff]
 
 v_cmpx_eq_u16 v1.h, v255.h
-// GFX11: v_cmpx_eq_u16_e64 v1.h, v255.h          ; encoding: [0x7e,0x18,0xba,0xd4,0x01,0xff,0x03,0x00]
+// GFX11: v_cmpx_eq_u16_e64 v1.h, v255.h op_sel:[1,1,0] ; encoding: [0x7e,0x18,0xba,0xd4,0x01,0xff,0x03,0x00]
 
 v_cmpx_eq_u16 v1.h, v255.h dpp8:[7,6,5,4,3,2,1,0]
 // GFX11: v_cmpx_eq_u16_e64_dpp v1.h, v255.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x18,0xba,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
@@ -110,7 +128,7 @@ v_cmpx_eq_u16 v1.l, v255.l quad_perm:[3,2,1,0]
 // GFX11: v_cmpx_eq_u16_e64_dpp v1.l, v255.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xba,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
 
 v_cmpx_eq_u16 v255.h, v2.h
-// GFX11: v_cmpx_eq_u16_e64 v255.h, v2.h          ; encoding: [0x7e,0x18,0xba,0xd4,0xff,0x05,0x02,0x00]
+// GFX11: v_cmpx_eq_u16_e64 v255.h, v2.h op_sel:[1,1,0] ; encoding: [0x7e,0x18,0xba,0xd4,0xff,0x05,0x02,0x00]
 
 v_cmpx_eq_u16 v255.h, v2.h dpp8:[7,6,5,4,3,2,1,0]
 // GFX11: v_cmpx_eq_u16_e64_dpp v255.h, v2.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x18,0xba,0xd4,0xe9,0x04,0x02,0x00,0xff,0x77,0x39,0x05]
@@ -127,44 +145,80 @@ v_cmpx_eq_u16 v255.l, v2.l dpp8:[7,6,5,4,3,2,1,0]
 v_cmpx_eq_u16 v255.l, v2.l quad_perm:[3,2,1,0]
 // GFX11: v_cmpx_eq_u16_e64_dpp v255.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xba,0xd4,0xfa,0x04,0x02,0x00,0xff,0x1b,0x00,0xff]
 
-v_cmpx_f_f16 v1, v255
-// GFX11: v_cmpx_f_f16_e64 v1, v255               ; encoding: [0x7e,0x00,0x80,0xd4,0x01,0xff,0x03,0x00]
+v_cmpx_f_f16 v1.h, v255.h
+// GFX11: v_cmpx_f_f16_e64 v1.h, v255.h op_sel:[1,1,0] ; encoding: [0x7e,0x18,0x80,0xd4,0x01,0xff,0x03,0x00]
+
+v_cmpx_f_f16 v1.h, v255.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_f_f16_e64_dpp v1.h, v255.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x18,0x80,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
+
+v_cmpx_f_f16 v1.h, v255.h quad_perm:[3,2,1,0]
+// GFX11: v_cmpx_f_f16_e64_dpp v1.h, v255.h op_sel:[1,1] quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x18,0x80,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
+
+v_cmpx_f_f16 v1.l, v255.l
+// GFX11: v_cmpx_f_f16_e64 v1.l, v255.l           ; encoding: [0x7e,0x00,0x80,0xd4,0x01,0xff,0x03,0x00]
+
+v_cmpx_f_f16 v1.l, v255.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_f_f16_e64_dpp v1.l, v255.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x80,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
+
+v_cmpx_f_f16 v1.l, v255.l quad_perm:[3,2,1,0]
+// GFX11: v_cmpx_f_f16_e64_dpp v1.l, v255.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x80,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
+
+v_cmpx_f_f16 v255.h, v2.h
+// GFX11: v_cmpx_f_f16_e64 v255.h, v2.h op_sel:[1,1,0] ; encoding: [0x7e,0x18,0x80,0xd4,0xff,0x05,0x02,0x00]
 
-v_cmpx_f_f16 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmpx_f_f16_e64_dpp v1, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x80,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
+v_cmpx_f_f16 v255.h, v2.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_f_f16_e64_dpp v255.h, v2.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x18,0x80,0xd4,0xe9,0x04,0x02,0x00,0xff,0x77,0x39,0x05]
 
-v_cmpx_f_f16 v1, v255 quad_perm:[3,2,1,0]
-// GFX11: v_cmpx_f_f16_e64_dpp v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x80,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
+v_cmpx_f_f16 v255.h, v2.h quad_perm:[3,2,1,0]
+// GFX11: v_cmpx_f_f16_e64_dpp v255.h, v2.h op_sel:[1,1] quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x18,0x80,0xd4,0xfa,0x04,0x02,0x00,0xff,0x1b,0x00,0xff]
 
-v_cmpx_f_f16 v255, v2
-// GFX11: v_cmpx_f_f16_e64 v255, v2               ; encoding: [0x7e,0x00,0x80,0xd4,0xff,0x05,0x02,0x00]
+v_cmpx_f_f16 v255.l, v2.l
+// GFX11: v_cmpx_f_f16_e64 v255.l, v2.l           ; encoding: [0x7e,0x00,0x80,0xd4,0xff,0x05,0x02,0x00]
 
-v_cmpx_f_f16 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmpx_f_f16_e64_dpp v255, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x80,0xd4,0xe9,0x04,0x02,0x00,0xff,0x77,0x39,0x05]
+v_cmpx_f_f16 v255.l, v2.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_f_f16_e64_dpp v255.l, v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x80,0xd4,0xe9,0x04,0x02,0x00,0xff,0x77,0x39,0x05]
 
-v_cmpx_f_f16 v255, v2 quad_perm:[3,2,1,0]
-// GFX11: v_cmpx_f_f16_e64_dpp v255, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x80,0xd4,0xfa,0x04,0x02,0x00,0xff,0x1b,0x00,0xff]
+v_cmpx_f_f16 v255.l, v2.l quad_perm:[3,2,1,0]
+// GFX11: v_cmpx_f_f16_e64_dpp v255.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x80,0xd4,0xfa,0x04,0x02,0x00,0xff,0x1b,0x00,0xff]
 
-v_cmpx_ge_f16 v1, v255
-// GFX11: v_cmpx_ge_f16_e64 v1, v255              ; encoding: [0x7e,0x00,0x86,0xd4,0x01,0xff,0x03,0x00]
+v_cmpx_ge_f16 v1.h, v255.h
+// GFX11: v_cmpx_ge_f16_e64 v1.h, v255.h op_sel:[1,1,0] ; encoding: [0x7e,0x18,0x86,0xd4,0x01,0xff,0x03,0x00]
 
-v_cmpx_ge_f16 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmpx_ge_f16_e64_dpp v1, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x86,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
+v_cmpx_ge_f16 v1.h, v255.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_ge_f16_e64_dpp v1.h, v255.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x18,0x86,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
 
-v_cmpx_ge_f16 v1, v255 quad_perm:[3,2,1,0]
-// GFX11: v_cmpx_ge_f16_e64_dpp v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x86,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
+v_cmpx_ge_f16 v1.h, v255.h quad_perm:[3,2,1,0]
+// GFX11: v_cmpx_ge_f16_e64_dpp v1.h, v255.h op_sel:[1,1] quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x18,0x86,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
 
-v_cmpx_ge_f16 v255, v2
-// GFX11: v_cmpx_ge_f16_e64 v255, v2              ; encoding: [0x7e,0x00,0x86,0xd4,0xff,0x05,0x02,0x00]
+v_cmpx_ge_f16 v1.l, v255.l
+// GFX11: v_cmpx_ge_f16_e64 v1.l, v255.l          ; encoding: [0x7e,0x00,0x86,0xd4,0x01,0xff,0x03,0x00]
 
-v_cmpx_ge_f16 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmpx_ge_f16_e64_dpp v255, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x86,0xd4,0xe9,0x04,0x02,0x00,0xff,0x77,0x39,0x05]
+v_cmpx_ge_f16 v1.l, v255.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_ge_f16_e64_dpp v1.l, v255.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x86,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
 
-v_cmpx_ge_f16 v255, v2 quad_perm:[3,2,1,0]
-// GFX11: v_cmpx_ge_f16_e64_dpp v255, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x86,0xd4,0xfa,0x04,0x02,0x00,0xff,0x1b,0x00,0xff]
+v_cmpx_ge_f16 v1.l, v255.l quad_perm:[3,2,1,0]
+// GFX11: v_cmpx_ge_f16_e64_dpp v1.l, v255.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x86,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
+
+v_cmpx_ge_f16 v255.h, v2.h
+// GFX11: v_cmpx_ge_f16_e64 v255.h, v2.h op_sel:[1,1,0] ; encoding: [0x7e,0x18,0x86,0xd4,0xff,0x05,0x02,0x00]
+
+v_cmpx_ge_f16 v255.h, v2.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_ge_f16_e64_dpp v255.h, v2.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x18,0x86,0xd4,0xe9,0x04,0x02,0x00,0xff,0x77,0x39,0x05]
+
+v_cmpx_ge_f16 v255.h, v2.h quad_perm:[3,2,1,0]
+// GFX11: v_cmpx_ge_f16_e64_dpp v255.h, v2.h op_sel:[1,1] quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x18,0x86,0xd4,0xfa,0x04,0x02,0x00,0xff,0x1b,0x00,0xff]
+
+v_cmpx_ge_f16 v255.l, v2.l
+// GFX11: v_cmpx_ge_f16_e64 v255.l, v2.l          ; encoding: [0x7e,0x00,0x86,0xd4,0xff,0x05,0x02,0x00]
+
+v_cmpx_ge_f16 v255.l, v2.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_ge_f16_e64_dpp v255.l, v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x86,0xd4,0xe9,0x04,0x02,0x00,0xff,0x77,0x39,0x05]
+
+v_cmpx_ge_f16 v255.l, v2.l quad_perm:[3,2,1,0]
+// GFX11: v_cmpx_ge_f16_e64_dpp v255.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x86,0xd4,0xfa,0x04,0x02,0x00,0xff,0x1b,0x00,0xff]
 
 v_cmpx_ge_i16 v1.h, v255.h
-// GFX11: v_cmpx_ge_i16_e64 v1.h, v255.h          ; encoding: [0x7e,0x18,0xb6,0xd4,0x01,0xff,0x03,0x00]
+// GFX11: v_cmpx_ge_i16_e64 v1.h, v255.h op_sel:[1,1,0] ; encoding: [0x7e,0x18,0xb6,0xd4,0x01,0xff,0x03,0x00]
 
 v_cmpx_ge_i16 v1.h, v255.h dpp8:[7,6,5,4,3,2,1,0]
 // GFX11: v_cmpx_ge_i16_e64_dpp v1.h, v255.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x18,0xb6,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
@@ -182,7 +236,7 @@ v_cmpx_ge_i16 v1.l, v255.l quad_perm:[3,2,1,0]
 // GFX11: v_cmpx_ge_i16_e64_dpp v1.l, v255.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xb6,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
 
 v_cmpx_ge_i16 v255.h, v2.h
-// GFX11: v_cmpx_ge_i16_e64 v255.h, v2.h          ; encoding: [0x7e,0x18,0xb6,0xd4,0xff,0x05,0x02,0x00]
+// GFX11: v_cmpx_ge_i16_e64 v255.h, v2.h op_sel:[1,1,0] ; encoding: [0x7e,0x18,0xb6,0xd4,0xff,0x05,0x02,0x00]
 
 v_cmpx_ge_i16 v255.h, v2.h dpp8:[7,6,5,4,3,2,1,0]
 // GFX11: v_cmpx_ge_i16_e64_dpp v255.h, v2.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x18,0xb6,0xd4,0xe9,0x04,0x02,0x00,0xff,0x77,0x39,0x05]
@@ -200,7 +254,7 @@ v_cmpx_ge_i16 v255.l, v2.l quad_perm:[3,2,1,0]
 // GFX11: v_cmpx_ge_i16_e64_dpp v255.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xb6,0xd4,0xfa,0x04,0x02,0x00,0xff,0x1b,0x00,0xff]
 
 v_cmpx_ge_u16 v1.h, v255.h
-// GFX11: v_cmpx_ge_u16_e64 v1.h, v255.h          ; encoding: [0x7e,0x18,0xbe,0xd4,0x01,0xff,0x03,0x00]
+// GFX11: v_cmpx_ge_u16_e64 v1.h, v255.h op_sel:[1,1,0] ; encoding: [0x7e,0x18,0xbe,0xd4,0x01,0xff,0x03,0x00]
 
 v_cmpx_ge_u16 v1.h, v255.h dpp8:[7,6,5,4,3,2,1,0]
 // GFX11: v_cmpx_ge_u16_e64_dpp v1.h, v255.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x18,0xbe,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
@@ -218,7 +272,7 @@ v_cmpx_ge_u16 v1.l, v255.l quad_perm:[3,2,1,0]
 // GFX11: v_cmpx_ge_u16_e64_dpp v1.l, v255.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xbe,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
 
 v_cmpx_ge_u16 v255.h, v2.h
-// GFX11: v_cmpx_ge_u16_e64 v255.h, v2.h          ; encoding: [0x7e,0x18,0xbe,0xd4,0xff,0x05,0x02,0x00]
+// GFX11: v_cmpx_ge_u16_e64 v255.h, v2.h op_sel:[1,1,0] ; encoding: [0x7e,0x18,0xbe,0xd4,0xff,0x05,0x02,0x00]
 
 v_cmpx_ge_u16 v255.h, v2.h dpp8:[7,6,5,4,3,2,1,0]
 // GFX11: v_cmpx_ge_u16_e64_dpp v255.h, v2.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x18,0xbe,0xd4,0xe9,0x04,0x02,0x00,0xff,0x77,0x39,0x05]
@@ -235,26 +289,44 @@ v_cmpx_ge_u16 v255.l, v2.l dpp8:[7,6,5,4,3,2,1,0]
 v_cmpx_ge_u16 v255.l, v2.l quad_perm:[3,2,1,0]
 // GFX11: v_cmpx_ge_u16_e64_dpp v255.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xbe,0xd4,0xfa,0x04,0x02,0x00,0xff,0x1b,0x00,0xff]
 
-v_cmpx_gt_f16 v1, v255
-// GFX11: v_cmpx_gt_f16_e64 v1, v255              ; encoding: [0x7e,0x00,0x84,0xd4,0x01,0xff,0x03,0x00]
+v_cmpx_gt_f16 v1.h, v255.h
+// GFX11: v_cmpx_gt_f16_e64 v1.h, v255.h op_sel:[1,1,0] ; encoding: [0x7e,0x18,0x84,0xd4,0x01,0xff,0x03,0x00]
+
+v_cmpx_gt_f16 v1.h, v255.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_gt_f16_e64_dpp v1.h, v255.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x18,0x84,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
+
+v_cmpx_gt_f16 v1.h, v255.h quad_perm:[3,2,1,0]
+// GFX11: v_cmpx_gt_f16_e64_dpp v1.h, v255.h op_sel:[1,1] quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x18,0x84,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
+
+v_cmpx_gt_f16 v1.l, v255.l
+// GFX11: v_cmpx_gt_f16_e64 v1.l, v255.l          ; encoding: [0x7e,0x00,0x84,0xd4,0x01,0xff,0x03,0x00]
+
+v_cmpx_gt_f16 v1.l, v255.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_gt_f16_e64_dpp v1.l, v255.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x84,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
+
+v_cmpx_gt_f16 v1.l, v255.l quad_perm:[3,2,1,0]
+// GFX11: v_cmpx_gt_f16_e64_dpp v1.l, v255.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x84,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
+
+v_cmpx_gt_f16 v255.h, v2.h
+// GFX11: v_cmpx_gt_f16_e64 v255.h, v2.h op_sel:[1,1,0] ; encoding: [0x7e,0x18,0x84,0xd4,0xff,0x05,0x02,0x00]
 
-v_cmpx_gt_f16 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmpx_gt_f16_e64_dpp v1, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x84,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
+v_cmpx_gt_f16 v255.h, v2.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_gt_f16_e64_dpp v255.h, v2.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x18,0x84,0xd4,0xe9,0x04,0x02,0x00,0xff,0x77,0x39,0x05]
 
-v_cmpx_gt_f16 v1, v255 quad_perm:[3,2,1,0]
-// GFX11: v_cmpx_gt_f16_e64_dpp v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x84,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
+v_cmpx_gt_f16 v255.h, v2.h quad_perm:[3,2,1,0]
+// GFX11: v_cmpx_gt_f16_e64_dpp v255.h, v2.h op_sel:[1,1] quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x18,0x84,0xd4,0xfa,0x04,0x02,0x00,0xff,0x1b,0x00,0xff]
 
-v_cmpx_gt_f16 v255, v2
-// GFX11: v_cmpx_gt_f16_e64 v255, v2              ; encoding: [0x7e,0x00,0x84,0xd4,0xff,0x05,0x02,0x00]
+v_cmpx_gt_f16 v255.l, v2.l
+// GFX11: v_cmpx_gt_f16_e64 v255.l, v2.l          ; encoding: [0x7e,0x00,0x84,0xd4,0xff,0x05,0x02,0x00]
 
-v_cmpx_gt_f16 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmpx_gt_f16_e64_dpp v255, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x84,0xd4,0xe9,0x04,0x02,0x00,0xff,0x77,0x39,0x05]
+v_cmpx_gt_f16 v255.l, v2.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_gt_f16_e64_dpp v255.l, v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x84,0xd4,0xe9,0x04,0x02,0x00,0xff,0x77,0x39,0x05]
 
-v_cmpx_gt_f16 v255, v2 quad_perm:[3,2,1,0]
-// GFX11: v_cmpx_gt_f16_e64_dpp v255, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x84,0xd4,0xfa,0x04,0x02,0x00,0xff,0x1b,0x00,0xff]
+v_cmpx_gt_f16 v255.l, v2.l quad_perm:[3,2,1,0]
+// GFX11: v_cmpx_gt_f16_e64_dpp v255.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x84,0xd4,0xfa,0x04,0x02,0x00,0xff,0x1b,0x00,0xff]
 
 v_cmpx_gt_i16 v1.h, v255.h
-// GFX11: v_cmpx_gt_i16_e64 v1.h, v255.h          ; encoding: [0x7e,0x18,0xb4,0xd4,0x01,0xff,0x03,0x00]
+// GFX11: v_cmpx_gt_i16_e64 v1.h, v255.h op_sel:[1,1,0] ; encoding: [0x7e,0x18,0xb4,0xd4,0x01,0xff,0x03,0x00]
 
 v_cmpx_gt_i16 v1.h, v255.h dpp8:[7,6,5,4,3,2,1,0]
 // GFX11: v_cmpx_gt_i16_e64_dpp v1.h, v255.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x18,0xb4,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
@@ -272,7 +344,7 @@ v_cmpx_gt_i16 v1.l, v255.l quad_perm:[3,2,1,0]
 // GFX11: v_cmpx_gt_i16_e64_dpp v1.l, v255.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xb4,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
 
 v_cmpx_gt_i16 v255.h, v2.h
-// GFX11: v_cmpx_gt_i16_e64 v255.h, v2.h          ; encoding: [0x7e,0x18,0xb4,0xd4,0xff,0x05,0x02,0x00]
+// GFX11: v_cmpx_gt_i16_e64 v255.h, v2.h op_sel:[1,1,0] ; encoding: [0x7e,0x18,0xb4,0xd4,0xff,0x05,0x02,0x00]
 
 v_cmpx_gt_i16 v255.h, v2.h dpp8:[7,6,5,4,3,2,1,0]
 // GFX11: v_cmpx_gt_i16_e64_dpp v255.h, v2.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x18,0xb4,0xd4,0xe9,0x04,0x02,0x00,0xff,0x77,0x39,0x05]
@@ -290,7 +362,7 @@ v_cmpx_gt_i16 v255.l, v2.l quad_perm:[3,2,1,0]
 // GFX11: v_cmpx_gt_i16_e64_dpp v255.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xb4,0xd4,0xfa,0x04,0x02,0x00,0xff,0x1b,0x00,0xff]
 
 v_cmpx_gt_u16 v1.h, v255.h
-// GFX11: v_cmpx_gt_u16_e64 v1.h, v255.h          ; encoding: [0x7e,0x18,0xbc,0xd4,0x01,0xff,0x03,0x00]
+// GFX11: v_cmpx_gt_u16_e64 v1.h, v255.h op_sel:[1,1,0] ; encoding: [0x7e,0x18,0xbc,0xd4,0x01,0xff,0x03,0x00]
 
 v_cmpx_gt_u16 v1.h, v255.h dpp8:[7,6,5,4,3,2,1,0]
 // GFX11: v_cmpx_gt_u16_e64_dpp v1.h, v255.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x18,0xbc,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
@@ -308,7 +380,7 @@ v_cmpx_gt_u16 v1.l, v255.l quad_perm:[3,2,1,0]
 // GFX11: v_cmpx_gt_u16_e64_dpp v1.l, v255.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xbc,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
 
 v_cmpx_gt_u16 v255.h, v2.h
-// GFX11: v_cmpx_gt_u16_e64 v255.h, v2.h          ; encoding: [0x7e,0x18,0xbc,0xd4,0xff,0x05,0x02,0x00]
+// GFX11: v_cmpx_gt_u16_e64 v255.h, v2.h op_sel:[1,1,0] ; encoding: [0x7e,0x18,0xbc,0xd4,0xff,0x05,0x02,0x00]
 
 v_cmpx_gt_u16 v255.h, v2.h dpp8:[7,6,5,4,3,2,1,0]
 // GFX11: v_cmpx_gt_u16_e64_dpp v255.h, v2.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x18,0xbc,0xd4,0xe9,0x04,0x02,0x00,0xff,0x77,0x39,0x05]
@@ -325,26 +397,44 @@ v_cmpx_gt_u16 v255.l, v2.l dpp8:[7,6,5,4,3,2,1,0]
 v_cmpx_gt_u16 v255.l, v2.l quad_perm:[3,2,1,0]
 // GFX11: v_cmpx_gt_u16_e64_dpp v255.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xbc,0xd4,0xfa,0x04,0x02,0x00,0xff,0x1b,0x00,0xff]
 
-v_cmpx_le_f16 v1, v255
-// GFX11: v_cmpx_le_f16_e64 v1, v255              ; encoding: [0x7e,0x00,0x83,0xd4,0x01,0xff,0x03,0x00]
+v_cmpx_le_f16 v1.h, v255.h
+// GFX11: v_cmpx_le_f16_e64 v1.h, v255.h op_sel:[1,1,0] ; encoding: [0x7e,0x18,0x83,0xd4,0x01,0xff,0x03,0x00]
 
-v_cmpx_le_f16 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmpx_le_f16_e64_dpp v1, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x83,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
+v_cmpx_le_f16 v1.h, v255.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_le_f16_e64_dpp v1.h, v255.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x18,0x83,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
 
-v_cmpx_le_f16 v1, v255 quad_perm:[3,2,1,0]
-// GFX11: v_cmpx_le_f16_e64_dpp v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x83,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
+v_cmpx_le_f16 v1.h, v255.h quad_perm:[3,2,1,0]
+// GFX11: v_cmpx_le_f16_e64_dpp v1.h, v255.h op_sel:[1,1] quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x18,0x83,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
 
-v_cmpx_le_f16 v255, v2
-// GFX11: v_cmpx_le_f16_e64 v255, v2              ; encoding: [0x7e,0x00,0x83,0xd4,0xff,0x05,0x02,0x00]
+v_cmpx_le_f16 v1.l, v255.l
+// GFX11: v_cmpx_le_f16_e64 v1.l, v255.l          ; encoding: [0x7e,0x00,0x83,0xd4,0x01,0xff,0x03,0x00]
 
-v_cmpx_le_f16 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmpx_le_f16_e64_dpp v255, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x83,0xd4,0xe9,0x04,0x02,0x00,0xff,0x77,0x39,0x05]
+v_cmpx_le_f16 v1.l, v255.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_le_f16_e64_dpp v1.l, v255.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x83,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
 
-v_cmpx_le_f16 v255, v2 quad_perm:[3,2,1,0]
-// GFX11: v_cmpx_le_f16_e64_dpp v255, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x83,0xd4,0xfa,0x04,0x02,0x00,0xff,0x1b,0x00,0xff]
+v_cmpx_le_f16 v1.l, v255.l quad_perm:[3,2,1,0]
+// GFX11: v_cmpx_le_f16_e64_dpp v1.l, v255.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x83,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
+
+v_cmpx_le_f16 v255.h, v2.h
+// GFX11: v_cmpx_le_f16_e64 v255.h, v2.h op_sel:[1,1,0] ; encoding: [0x7e,0x18,0x83,0xd4,0xff,0x05,0x02,0x00]
+
+v_cmpx_le_f16 v255.h, v2.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_le_f16_e64_dpp v255.h, v2.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x18,0x83,0xd4,0xe9,0x04,0x02,0x00,0xff,0x77,0x39,0x05]
+
+v_cmpx_le_f16 v255.h, v2.h quad_perm:[3,2,1,0]
+// GFX11: v_cmpx_le_f16_e64_dpp v255.h, v2.h op_sel:[1,1] quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x18,0x83,0xd4,0xfa,0x04,0x02,0x00,0xff,0x1b,0x00,0xff]
+
+v_cmpx_le_f16 v255.l, v2.l
+// GFX11: v_cmpx_le_f16_e64 v255.l, v2.l          ; encoding: [0x7e,0x00,0x83,0xd4,0xff,0x05,0x02,0x00]
+
+v_cmpx_le_f16 v255.l, v2.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_le_f16_e64_dpp v255.l, v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x83,0xd4,0xe9,0x04,0x02,0x00,0xff,0x77,0x39,0x05]
+
+v_cmpx_le_f16 v255.l, v2.l quad_perm:[3,2,1,0]
+// GFX11: v_cmpx_le_f16_e64_dpp v255.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x83,0xd4,0xfa,0x04,0x02,0x00,0xff,0x1b,0x00,0xff]
 
 v_cmpx_le_i16 v1.h, v255.h
-// GFX11: v_cmpx_le_i16_e64 v1.h, v255.h          ; encoding: [0x7e,0x18,0xb3,0xd4,0x01,0xff,0x03,0x00]
+// GFX11: v_cmpx_le_i16_e64 v1.h, v255.h op_sel:[1,1,0] ; encoding: [0x7e,0x18,0xb3,0xd4,0x01,0xff,0x03,0x00]
 
 v_cmpx_le_i16 v1.h, v255.h dpp8:[7,6,5,4,3,2,1,0]
 // GFX11: v_cmpx_le_i16_e64_dpp v1.h, v255.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x18,0xb3,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
@@ -362,7 +452,7 @@ v_cmpx_le_i16 v1.l, v255.l quad_perm:[3,2,1,0]
 // GFX11: v_cmpx_le_i16_e64_dpp v1.l, v255.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xb3,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
 
 v_cmpx_le_i16 v255.h, v2.h
-// GFX11: v_cmpx_le_i16_e64 v255.h, v2.h          ; encoding: [0x7e,0x18,0xb3,0xd4,0xff,0x05,0x02,0x00]
+// GFX11: v_cmpx_le_i16_e64 v255.h, v2.h op_sel:[1,1,0] ; encoding: [0x7e,0x18,0xb3,0xd4,0xff,0x05,0x02,0x00]
 
 v_cmpx_le_i16 v255.h, v2.h dpp8:[7,6,5,4,3,2,1,0]
 // GFX11: v_cmpx_le_i16_e64_dpp v255.h, v2.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x18,0xb3,0xd4,0xe9,0x04,0x02,0x00,0xff,0x77,0x39,0x05]
@@ -380,7 +470,7 @@ v_cmpx_le_i16 v255.l, v2.l quad_perm:[3,2,1,0]
 // GFX11: v_cmpx_le_i16_e64_dpp v255.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xb3,0xd4,0xfa,0x04,0x02,0x00,0xff,0x1b,0x00,0xff]
 
 v_cmpx_le_u16 v1.h, v255.h
-// GFX11: v_cmpx_le_u16_e64 v1.h, v255.h          ; encoding: [0x7e,0x18,0xbb,0xd4,0x01,0xff,0x03,0x00]
+// GFX11: v_cmpx_le_u16_e64 v1.h, v255.h op_sel:[1,1,0] ; encoding: [0x7e,0x18,0xbb,0xd4,0x01,0xff,0x03,0x00]
 
 v_cmpx_le_u16 v1.h, v255.h dpp8:[7,6,5,4,3,2,1,0]
 // GFX11: v_cmpx_le_u16_e64_dpp v1.h, v255.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x18,0xbb,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
@@ -398,7 +488,7 @@ v_cmpx_le_u16 v1.l, v255.l quad_perm:[3,2,1,0]
 // GFX11: v_cmpx_le_u16_e64_dpp v1.l, v255.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xbb,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
 
 v_cmpx_le_u16 v255.h, v2.h
-// GFX11: v_cmpx_le_u16_e64 v255.h, v2.h          ; encoding: [0x7e,0x18,0xbb,0xd4,0xff,0x05,0x02,0x00]
+// GFX11: v_cmpx_le_u16_e64 v255.h, v2.h op_sel:[1,1,0] ; encoding: [0x7e,0x18,0xbb,0xd4,0xff,0x05,0x02,0x00]
 
 v_cmpx_le_u16 v255.h, v2.h dpp8:[7,6,5,4,3,2,1,0]
 // GFX11: v_cmpx_le_u16_e64_dpp v255.h, v2.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x18,0xbb,0xd4,0xe9,0x04,0x02,0x00,0xff,0x77,0x39,0x05]
@@ -415,26 +505,44 @@ v_cmpx_le_u16 v255.l, v2.l dpp8:[7,6,5,4,3,2,1,0]
 v_cmpx_le_u16 v255.l, v2.l quad_perm:[3,2,1,0]
 // GFX11: v_cmpx_le_u16_e64_dpp v255.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xbb,0xd4,0xfa,0x04,0x02,0x00,0xff,0x1b,0x00,0xff]
 
-v_cmpx_lg_f16 v1, v255
-// GFX11: v_cmpx_lg_f16_e64 v1, v255              ; encoding: [0x7e,0x00,0x85,0xd4,0x01,0xff,0x03,0x00]
+v_cmpx_lg_f16 v1.h, v255.h
+// GFX11: v_cmpx_lg_f16_e64 v1.h, v255.h op_sel:[1,1,0] ; encoding: [0x7e,0x18,0x85,0xd4,0x01,0xff,0x03,0x00]
+
+v_cmpx_lg_f16 v1.h, v255.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_lg_f16_e64_dpp v1.h, v255.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x18,0x85,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
+
+v_cmpx_lg_f16 v1.h, v255.h quad_perm:[3,2,1,0]
+// GFX11: v_cmpx_lg_f16_e64_dpp v1.h, v255.h op_sel:[1,1] quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x18,0x85,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
+
+v_cmpx_lg_f16 v1.l, v255.l
+// GFX11: v_cmpx_lg_f16_e64 v1.l, v255.l          ; encoding: [0x7e,0x00,0x85,0xd4,0x01,0xff,0x03,0x00]
+
+v_cmpx_lg_f16 v1.l, v255.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_lg_f16_e64_dpp v1.l, v255.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x85,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
+
+v_cmpx_lg_f16 v1.l, v255.l quad_perm:[3,2,1,0]
+// GFX11: v_cmpx_lg_f16_e64_dpp v1.l, v255.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x85,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
+
+v_cmpx_lg_f16 v255.h, v2.h
+// GFX11: v_cmpx_lg_f16_e64 v255.h, v2.h op_sel:[1,1,0] ; encoding: [0x7e,0x18,0x85,0xd4,0xff,0x05,0x02,0x00]
 
-v_cmpx_lg_f16 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmpx_lg_f16_e64_dpp v1, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x85,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
+v_cmpx_lg_f16 v255.h, v2.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_lg_f16_e64_dpp v255.h, v2.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x18,0x85,0xd4,0xe9,0x04,0x02,0x00,0xff,0x77,0x39,0x05]
 
-v_cmpx_lg_f16 v1, v255 quad_perm:[3,2,1,0]
-// GFX11: v_cmpx_lg_f16_e64_dpp v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x85,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
+v_cmpx_lg_f16 v255.h, v2.h quad_perm:[3,2,1,0]
+// GFX11: v_cmpx_lg_f16_e64_dpp v255.h, v2.h op_sel:[1,1] quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x18,0x85,0xd4,0xfa,0x04,0x02,0x00,0xff,0x1b,0x00,0xff]
 
-v_cmpx_lg_f16 v255, v2
-// GFX11: v_cmpx_lg_f16_e64 v255, v2              ; encoding: [0x7e,0x00,0x85,0xd4,0xff,0x05,0x02,0x00]
+v_cmpx_lg_f16 v255.l, v2.l
+// GFX11: v_cmpx_lg_f16_e64 v255.l, v2.l          ; encoding: [0x7e,0x00,0x85,0xd4,0xff,0x05,0x02,0x00]
 
-v_cmpx_lg_f16 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmpx_lg_f16_e64_dpp v255, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x85,0xd4,0xe9,0x04,0x02,0x00,0xff,0x77,0x39,0x05]
+v_cmpx_lg_f16 v255.l, v2.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_lg_f16_e64_dpp v255.l, v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x85,0xd4,0xe9,0x04,0x02,0x00,0xff,0x77,0x39,0x05]
 
-v_cmpx_lg_f16 v255, v2 quad_perm:[3,2,1,0]
-// GFX11: v_cmpx_lg_f16_e64_dpp v255, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x85,0xd4,0xfa,0x04,0x02,0x00,0xff,0x1b,0x00,0xff]
+v_cmpx_lg_f16 v255.l, v2.l quad_perm:[3,2,1,0]
+// GFX11: v_cmpx_lg_f16_e64_dpp v255.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x85,0xd4,0xfa,0x04,0x02,0x00,0xff,0x1b,0x00,0xff]
 
 v_cmpx_lt_f16 v1.h, v255.h
-// GFX11: v_cmpx_lt_f16_e64 v1.h, v255.h          ; encoding: [0x7e,0x18,0x81,0xd4,0x01,0xff,0x03,0x00]
+// GFX11: v_cmpx_lt_f16_e64 v1.h, v255.h op_sel:[1,1,0] ; encoding: [0x7e,0x18,0x81,0xd4,0x01,0xff,0x03,0x00]
 
 v_cmpx_lt_f16 v1.h, v255.h dpp8:[7,6,5,4,3,2,1,0]
 // GFX11: v_cmpx_lt_f16_e64_dpp v1.h, v255.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x18,0x81,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
@@ -452,7 +560,7 @@ v_cmpx_lt_f16 v1.l, v255.l quad_perm:[3,2,1,0]
 // GFX11: v_cmpx_lt_f16_e64_dpp v1.l, v255.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x81,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
 
 v_cmpx_lt_f16 v255.h, v2.h
-// GFX11: v_cmpx_lt_f16_e64 v255.h, v2.h          ; encoding: [0x7e,0x18,0x81,0xd4,0xff,0x05,0x02,0x00]
+// GFX11: v_cmpx_lt_f16_e64 v255.h, v2.h op_sel:[1,1,0] ; encoding: [0x7e,0x18,0x81,0xd4,0xff,0x05,0x02,0x00]
 
 v_cmpx_lt_f16 v255.h, v2.h dpp8:[7,6,5,4,3,2,1,0]
 // GFX11: v_cmpx_lt_f16_e64_dpp v255.h, v2.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x18,0x81,0xd4,0xe9,0x04,0x02,0x00,0xff,0x77,0x39,0x05]
@@ -470,7 +578,7 @@ v_cmpx_lt_f16 v255.l, v2.l quad_perm:[3,2,1,0]
 // GFX11: v_cmpx_lt_f16_e64_dpp v255.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x81,0xd4,0xfa,0x04,0x02,0x00,0xff,0x1b,0x00,0xff]
 
 v_cmpx_lt_i16 v1.h, v255.h
-// GFX11: v_cmpx_lt_i16_e64 v1.h, v255.h          ; encoding: [0x7e,0x18,0xb1,0xd4,0x01,0xff,0x03,0x00]
+// GFX11: v_cmpx_lt_i16_e64 v1.h, v255.h op_sel:[1,1,0] ; encoding: [0x7e,0x18,0xb1,0xd4,0x01,0xff,0x03,0x00]
 
 v_cmpx_lt_i16 v1.h, v255.h dpp8:[7,6,5,4,3,2,1,0]
 // GFX11: v_cmpx_lt_i16_e64_dpp v1.h, v255.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x18,0xb1,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
@@ -488,7 +596,7 @@ v_cmpx_lt_i16 v1.l, v255.l quad_perm:[3,2,1,0]
 // GFX11: v_cmpx_lt_i16_e64_dpp v1.l, v255.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xb1,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
 
 v_cmpx_lt_i16 v255.h, v2.h
-// GFX11: v_cmpx_lt_i16_e64 v255.h, v2.h          ; encoding: [0x7e,0x18,0xb1,0xd4,0xff,0x05,0x02,0x00]
+// GFX11: v_cmpx_lt_i16_e64 v255.h, v2.h op_sel:[1,1,0] ; encoding: [0x7e,0x18,0xb1,0xd4,0xff,0x05,0x02,0x00]
 
 v_cmpx_lt_i16 v255.h, v2.h dpp8:[7,6,5,4,3,2,1,0]
 // GFX11: v_cmpx_lt_i16_e64_dpp v255.h, v2.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x18,0xb1,0xd4,0xe9,0x04,0x02,0x00,0xff,0x77,0x39,0x05]
@@ -506,7 +614,7 @@ v_cmpx_lt_i16 v255.l, v2.l quad_perm:[3,2,1,0]
 // GFX11: v_cmpx_lt_i16_e64_dpp v255.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xb1,0xd4,0xfa,0x04,0x02,0x00,0xff,0x1b,0x00,0xff]
 
 v_cmpx_lt_u16 v1.h, v255.h
-// GFX11: v_cmpx_lt_u16_e64 v1.h, v255.h          ; encoding: [0x7e,0x18,0xb9,0xd4,0x01,0xff,0x03,0x00]
+// GFX11: v_cmpx_lt_u16_e64 v1.h, v255.h op_sel:[1,1,0] ; encoding: [0x7e,0x18,0xb9,0xd4,0x01,0xff,0x03,0x00]
 
 v_cmpx_lt_u16 v1.h, v255.h dpp8:[7,6,5,4,3,2,1,0]
 // GFX11: v_cmpx_lt_u16_e64_dpp v1.h, v255.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x18,0xb9,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
@@ -524,7 +632,7 @@ v_cmpx_lt_u16 v1.l, v255.l quad_perm:[3,2,1,0]
 // GFX11: v_cmpx_lt_u16_e64_dpp v1.l, v255.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xb9,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
 
 v_cmpx_lt_u16 v255.h, v2.h
-// GFX11: v_cmpx_lt_u16_e64 v255.h, v2.h          ; encoding: [0x7e,0x18,0xb9,0xd4,0xff,0x05,0x02,0x00]
+// GFX11: v_cmpx_lt_u16_e64 v255.h, v2.h op_sel:[1,1,0] ; encoding: [0x7e,0x18,0xb9,0xd4,0xff,0x05,0x02,0x00]
 
 v_cmpx_lt_u16 v255.h, v2.h dpp8:[7,6,5,4,3,2,1,0]
 // GFX11: v_cmpx_lt_u16_e64_dpp v255.h, v2.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x18,0xb9,0xd4,0xe9,0x04,0x02,0x00,0xff,0x77,0x39,0x05]
@@ -542,7 +650,7 @@ v_cmpx_lt_u16 v255.l, v2.l quad_perm:[3,2,1,0]
 // GFX11: v_cmpx_lt_u16_e64_dpp v255.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xb9,0xd4,0xfa,0x04,0x02,0x00,0xff,0x1b,0x00,0xff]
 
 v_cmpx_ne_i16 v1.h, v255.h
-// GFX11: v_cmpx_ne_i16_e64 v1.h, v255.h          ; encoding: [0x7e,0x18,0xb5,0xd4,0x01,0xff,0x03,0x00]
+// GFX11: v_cmpx_ne_i16_e64 v1.h, v255.h op_sel:[1,1,0] ; encoding: [0x7e,0x18,0xb5,0xd4,0x01,0xff,0x03,0x00]
 
 v_cmpx_ne_i16 v1.h, v255.h dpp8:[7,6,5,4,3,2,1,0]
 // GFX11: v_cmpx_ne_i16_e64_dpp v1.h, v255.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x18,0xb5,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
@@ -560,7 +668,7 @@ v_cmpx_ne_i16 v1.l, v255.l quad_perm:[3,2,1,0]
 // GFX11: v_cmpx_ne_i16_e64_dpp v1.l, v255.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xb5,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
 
 v_cmpx_ne_i16 v255.h, v2.h
-// GFX11: v_cmpx_ne_i16_e64 v255.h, v2.h          ; encoding: [0x7e,0x18,0xb5,0xd4,0xff,0x05,0x02,0x00]
+// GFX11: v_cmpx_ne_i16_e64 v255.h, v2.h op_sel:[1,1,0] ; encoding: [0x7e,0x18,0xb5,0xd4,0xff,0x05,0x02,0x00]
 
 v_cmpx_ne_i16 v255.h, v2.h dpp8:[7,6,5,4,3,2,1,0]
 // GFX11: v_cmpx_ne_i16_e64_dpp v255.h, v2.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x18,0xb5,0xd4,0xe9,0x04,0x02,0x00,0xff,0x77,0x39,0x05]
@@ -578,7 +686,7 @@ v_cmpx_ne_i16 v255.l, v2.l quad_perm:[3,2,1,0]
 // GFX11: v_cmpx_ne_i16_e64_dpp v255.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xb5,0xd4,0xfa,0x04,0x02,0x00,0xff,0x1b,0x00,0xff]
 
 v_cmpx_ne_u16 v1.h, v255.h
-// GFX11: v_cmpx_ne_u16_e64 v1.h, v255.h          ; encoding: [0x7e,0x18,0xbd,0xd4,0x01,0xff,0x03,0x00]
+// GFX11: v_cmpx_ne_u16_e64 v1.h, v255.h op_sel:[1,1,0] ; encoding: [0x7e,0x18,0xbd,0xd4,0x01,0xff,0x03,0x00]
 
 v_cmpx_ne_u16 v1.h, v255.h dpp8:[7,6,5,4,3,2,1,0]
 // GFX11: v_cmpx_ne_u16_e64_dpp v1.h, v255.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x18,0xbd,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
@@ -596,7 +704,7 @@ v_cmpx_ne_u16 v1.l, v255.l quad_perm:[3,2,1,0]
 // GFX11: v_cmpx_ne_u16_e64_dpp v1.l, v255.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xbd,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
 
 v_cmpx_ne_u16 v255.h, v2.h
-// GFX11: v_cmpx_ne_u16_e64 v255.h, v2.h          ; encoding: [0x7e,0x18,0xbd,0xd4,0xff,0x05,0x02,0x00]
+// GFX11: v_cmpx_ne_u16_e64 v255.h, v2.h op_sel:[1,1,0] ; encoding: [0x7e,0x18,0xbd,0xd4,0xff,0x05,0x02,0x00]
 
 v_cmpx_ne_u16 v255.h, v2.h dpp8:[7,6,5,4,3,2,1,0]
 // GFX11: v_cmpx_ne_u16_e64_dpp v255.h, v2.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x18,0xbd,0xd4,0xe9,0x04,0x02,0x00,0xff,0x77,0x39,0x05]
@@ -613,182 +721,362 @@ v_cmpx_ne_u16 v255.l, v2.l dpp8:[7,6,5,4,3,2,1,0]
 v_cmpx_ne_u16 v255.l, v2.l quad_perm:[3,2,1,0]
 // GFX11: v_cmpx_ne_u16_e64_dpp v255.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xbd,0xd4,0xfa,0x04,0x02,0x00,0xff,0x1b,0x00,0xff]
 
-v_cmpx_neq_f16 v1, v255
-// GFX11: v_cmpx_neq_f16_e64 v1, v255             ; encoding: [0x7e,0x00,0x8d,0xd4,0x01,0xff,0x03,0x00]
+v_cmpx_neq_f16 v1.h, v255.h
+// GFX11: v_cmpx_neq_f16_e64 v1.h, v255.h op_sel:[1,1,0] ; encoding: [0x7e,0x18,0x8d,0xd4,0x01,0xff,0x03,0x00]
+
+v_cmpx_neq_f16 v1.h, v255.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_neq_f16_e64_dpp v1.h, v255.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x18,0x8d,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
+
+v_cmpx_neq_f16 v1.h, v255.h quad_perm:[3,2,1,0]
+// GFX11: v_cmpx_neq_f16_e64_dpp v1.h, v255.h op_sel:[1,1] quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x18,0x8d,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
+
+v_cmpx_neq_f16 v1.l, v255.l
+// GFX11: v_cmpx_neq_f16_e64 v1.l, v255.l         ; encoding: [0x7e,0x00,0x8d,0xd4,0x01,0xff,0x03,0x00]
+
+v_cmpx_neq_f16 v1.l, v255.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_neq_f16_e64_dpp v1.l, v255.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x8d,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
+
+v_cmpx_neq_f16 v1.l, v255.l quad_perm:[3,2,1,0]
+// GFX11: v_cmpx_neq_f16_e64_dpp v1.l, v255.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8d,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
+
+v_cmpx_neq_f16 v255.h, v2.h
+// GFX11: v_cmpx_neq_f16_e64 v255.h, v2.h op_sel:[1,1,0] ; encoding: [0x7e,0x18,0x8d,0xd4,0xff,0x05,0x02,0x00]
+
+v_cmpx_neq_f16 v255.h, v2.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_neq_f16_e64_dpp v255.h, v2.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x18,0x8d,0xd4,0xe9,0x04,0x02,0x00,0xff,0x77,0x39,0x05]
+
+v_cmpx_neq_f16 v255.h, v2.h quad_perm:[3,2,1,0]
+// GFX11: v_cmpx_neq_f16_e64_dpp v255.h, v2.h op_sel:[1,1] quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x18,0x8d,0xd4,0xfa,0x04,0x02,0x00,0xff,0x1b,0x00,0xff]
+
+v_cmpx_neq_f16 v255.l, v2.l
+// GFX11: v_cmpx_neq_f16_e64 v255.l, v2.l         ; encoding: [0x7e,0x00,0x8d,0xd4,0xff,0x05,0x02,0x00]
+
+v_cmpx_neq_f16 v255.l, v2.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_neq_f16_e64_dpp v255.l, v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x8d,0xd4,0xe9,0x04,0x02,0x00,0xff,0x77,0x39,0x05]
+
+v_cmpx_neq_f16 v255.l, v2.l quad_perm:[3,2,1,0]
+// GFX11: v_cmpx_neq_f16_e64_dpp v255.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8d,0xd4,0xfa,0x04,0x02,0x00,0xff,0x1b,0x00,0xff]
+
+v_cmpx_nge_f16 v1.h, v255.h
+// GFX11: v_cmpx_nge_f16_e64 v1.h, v255.h op_sel:[1,1,0] ; encoding: [0x7e,0x18,0x89,0xd4,0x01,0xff,0x03,0x00]
+
+v_cmpx_nge_f16 v1.h, v255.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_nge_f16_e64_dpp v1.h, v255.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x18,0x89,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
+
+v_cmpx_nge_f16 v1.h, v255.h quad_perm:[3,2,1,0]
+// GFX11: v_cmpx_nge_f16_e64_dpp v1.h, v255.h op_sel:[1,1] quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x18,0x89,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
+
+v_cmpx_nge_f16 v1.l, v255.l
+// GFX11: v_cmpx_nge_f16_e64 v1.l, v255.l         ; encoding: [0x7e,0x00,0x89,0xd4,0x01,0xff,0x03,0x00]
+
+v_cmpx_nge_f16 v1.l, v255.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_nge_f16_e64_dpp v1.l, v255.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x89,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
+
+v_cmpx_nge_f16 v1.l, v255.l quad_perm:[3,2,1,0]
+// GFX11: v_cmpx_nge_f16_e64_dpp v1.l, v255.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x89,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
+
+v_cmpx_nge_f16 v255.h, v2.h
+// GFX11: v_cmpx_nge_f16_e64 v255.h, v2.h op_sel:[1,1,0] ; encoding: [0x7e,0x18,0x89,0xd4,0xff,0x05,0x02,0x00]
+
+v_cmpx_nge_f16 v255.h, v2.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_nge_f16_e64_dpp v255.h, v2.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x18,0x89,0xd4,0xe9,0x04,0x02,0x00,0xff,0x77,0x39,0x05]
+
+v_cmpx_nge_f16 v255.h, v2.h quad_perm:[3,2,1,0]
+// GFX11: v_cmpx_nge_f16_e64_dpp v255.h, v2.h op_sel:[1,1] quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x18,0x89,0xd4,0xfa,0x04,0x02,0x00,0xff,0x1b,0x00,0xff]
+
+v_cmpx_nge_f16 v255.l, v2.l
+// GFX11: v_cmpx_nge_f16_e64 v255.l, v2.l         ; encoding: [0x7e,0x00,0x89,0xd4,0xff,0x05,0x02,0x00]
+
+v_cmpx_nge_f16 v255.l, v2.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_nge_f16_e64_dpp v255.l, v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x89,0xd4,0xe9,0x04,0x02,0x00,0xff,0x77,0x39,0x05]
+
+v_cmpx_nge_f16 v255.l, v2.l quad_perm:[3,2,1,0]
+// GFX11: v_cmpx_nge_f16_e64_dpp v255.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x89,0xd4,0xfa,0x04,0x02,0x00,0xff,0x1b,0x00,0xff]
+
+v_cmpx_ngt_f16 v1.h, v255.h
+// GFX11: v_cmpx_ngt_f16_e64 v1.h, v255.h op_sel:[1,1,0] ; encoding: [0x7e,0x18,0x8b,0xd4,0x01,0xff,0x03,0x00]
+
+v_cmpx_ngt_f16 v1.h, v255.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_ngt_f16_e64_dpp v1.h, v255.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x18,0x8b,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
+
+v_cmpx_ngt_f16 v1.h, v255.h quad_perm:[3,2,1,0]
+// GFX11: v_cmpx_ngt_f16_e64_dpp v1.h, v255.h op_sel:[1,1] quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x18,0x8b,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
+
+v_cmpx_ngt_f16 v1.l, v255.l
+// GFX11: v_cmpx_ngt_f16_e64 v1.l, v255.l         ; encoding: [0x7e,0x00,0x8b,0xd4,0x01,0xff,0x03,0x00]
+
+v_cmpx_ngt_f16 v1.l, v255.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_ngt_f16_e64_dpp v1.l, v255.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x8b,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
+
+v_cmpx_ngt_f16 v1.l, v255.l quad_perm:[3,2,1,0]
+// GFX11: v_cmpx_ngt_f16_e64_dpp v1.l, v255.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8b,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
+
+v_cmpx_ngt_f16 v255.h, v2.h
+// GFX11: v_cmpx_ngt_f16_e64 v255.h, v2.h op_sel:[1,1,0] ; encoding: [0x7e,0x18,0x8b,0xd4,0xff,0x05,0x02,0x00]
+
+v_cmpx_ngt_f16 v255.h, v2.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_ngt_f16_e64_dpp v255.h, v2.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x18,0x8b,0xd4,0xe9,0x04,0x02,0x00,0xff,0x77,0x39,0x05]
+
+v_cmpx_ngt_f16 v255.h, v2.h quad_perm:[3,2,1,0]
+// GFX11: v_cmpx_ngt_f16_e64_dpp v255.h, v2.h op_sel:[1,1] quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x18,0x8b,0xd4,0xfa,0x04,0x02,0x00,0xff,0x1b,0x00,0xff]
+
+v_cmpx_ngt_f16 v255.l, v2.l
+// GFX11: v_cmpx_ngt_f16_e64 v255.l, v2.l         ; encoding: [0x7e,0x00,0x8b,0xd4,0xff,0x05,0x02,0x00]
+
+v_cmpx_ngt_f16 v255.l, v2.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_ngt_f16_e64_dpp v255.l, v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x8b,0xd4,0xe9,0x04,0x02,0x00,0xff,0x77,0x39,0x05]
+
+v_cmpx_ngt_f16 v255.l, v2.l quad_perm:[3,2,1,0]
+// GFX11: v_cmpx_ngt_f16_e64_dpp v255.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8b,0xd4,0xfa,0x04,0x02,0x00,0xff,0x1b,0x00,0xff]
+
+v_cmpx_nle_f16 v1.h, v255.h
+// GFX11: v_cmpx_nle_f16_e64 v1.h, v255.h op_sel:[1,1,0] ; encoding: [0x7e,0x18,0x8c,0xd4,0x01,0xff,0x03,0x00]
+
+v_cmpx_nle_f16 v1.h, v255.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_nle_f16_e64_dpp v1.h, v255.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x18,0x8c,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
+
+v_cmpx_nle_f16 v1.h, v255.h quad_perm:[3,2,1,0]
+// GFX11: v_cmpx_nle_f16_e64_dpp v1.h, v255.h op_sel:[1,1] quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x18,0x8c,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
+
+v_cmpx_nle_f16 v1.l, v255.l
+// GFX11: v_cmpx_nle_f16_e64 v1.l, v255.l         ; encoding: [0x7e,0x00,0x8c,0xd4,0x01,0xff,0x03,0x00]
+
+v_cmpx_nle_f16 v1.l, v255.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_nle_f16_e64_dpp v1.l, v255.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x8c,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
+
+v_cmpx_nle_f16 v1.l, v255.l quad_perm:[3,2,1,0]
+// GFX11: v_cmpx_nle_f16_e64_dpp v1.l, v255.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8c,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
+
+v_cmpx_nle_f16 v255.h, v2.h
+// GFX11: v_cmpx_nle_f16_e64 v255.h, v2.h op_sel:[1,1,0] ; encoding: [0x7e,0x18,0x8c,0xd4,0xff,0x05,0x02,0x00]
+
+v_cmpx_nle_f16 v255.h, v2.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_nle_f16_e64_dpp v255.h, v2.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x18,0x8c,0xd4,0xe9,0x04,0x02,0x00,0xff,0x77,0x39,0x05]
+
+v_cmpx_nle_f16 v255.h, v2.h quad_perm:[3,2,1,0]
+// GFX11: v_cmpx_nle_f16_e64_dpp v255.h, v2.h op_sel:[1,1] quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x18,0x8c,0xd4,0xfa,0x04,0x02,0x00,0xff,0x1b,0x00,0xff]
+
+v_cmpx_nle_f16 v255.l, v2.l
+// GFX11: v_cmpx_nle_f16_e64 v255.l, v2.l         ; encoding: [0x7e,0x00,0x8c,0xd4,0xff,0x05,0x02,0x00]
+
+v_cmpx_nle_f16 v255.l, v2.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_nle_f16_e64_dpp v255.l, v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x8c,0xd4,0xe9,0x04,0x02,0x00,0xff,0x77,0x39,0x05]
+
+v_cmpx_nle_f16 v255.l, v2.l quad_perm:[3,2,1,0]
+// GFX11: v_cmpx_nle_f16_e64_dpp v255.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8c,0xd4,0xfa,0x04,0x02,0x00,0xff,0x1b,0x00,0xff]
+
+v_cmpx_nlg_f16 v1.h, v255.h
+// GFX11: v_cmpx_nlg_f16_e64 v1.h, v255.h op_sel:[1,1,0] ; encoding: [0x7e,0x18,0x8a,0xd4,0x01,0xff,0x03,0x00]
+
+v_cmpx_nlg_f16 v1.h, v255.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_nlg_f16_e64_dpp v1.h, v255.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x18,0x8a,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
+
+v_cmpx_nlg_f16 v1.h, v255.h quad_perm:[3,2,1,0]
+// GFX11: v_cmpx_nlg_f16_e64_dpp v1.h, v255.h op_sel:[1,1] quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x18,0x8a,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
+
+v_cmpx_nlg_f16 v1.l, v255.l
+// GFX11: v_cmpx_nlg_f16_e64 v1.l, v255.l         ; encoding: [0x7e,0x00,0x8a,0xd4,0x01,0xff,0x03,0x00]
+
+v_cmpx_nlg_f16 v1.l, v255.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_nlg_f16_e64_dpp v1.l, v255.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x8a,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
+
+v_cmpx_nlg_f16 v1.l, v255.l quad_perm:[3,2,1,0]
+// GFX11: v_cmpx_nlg_f16_e64_dpp v1.l, v255.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8a,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
+
+v_cmpx_nlg_f16 v255.h, v2.h
+// GFX11: v_cmpx_nlg_f16_e64 v255.h, v2.h op_sel:[1,1,0] ; encoding: [0x7e,0x18,0x8a,0xd4,0xff,0x05,0x02,0x00]
+
+v_cmpx_nlg_f16 v255.h, v2.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_nlg_f16_e64_dpp v255.h, v2.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x18,0x8a,0xd4,0xe9,0x04,0x02,0x00,0xff,0x77,0x39,0x05]
+
+v_cmpx_nlg_f16 v255.h, v2.h quad_perm:[3,2,1,0]
+// GFX11: v_cmpx_nlg_f16_e64_dpp v255.h, v2.h op_sel:[1,1] quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x18,0x8a,0xd4,0xfa,0x04,0x02,0x00,0xff,0x1b,0x00,0xff]
+
+v_cmpx_nlg_f16 v255.l, v2.l
+// GFX11: v_cmpx_nlg_f16_e64 v255.l, v2.l         ; encoding: [0x7e,0x00,0x8a,0xd4,0xff,0x05,0x02,0x00]
+
+v_cmpx_nlg_f16 v255.l, v2.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_nlg_f16_e64_dpp v255.l, v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x8a,0xd4,0xe9,0x04,0x02,0x00,0xff,0x77,0x39,0x05]
+
+v_cmpx_nlg_f16 v255.l, v2.l quad_perm:[3,2,1,0]
+// GFX11: v_cmpx_nlg_f16_e64_dpp v255.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8a,0xd4,0xfa,0x04,0x02,0x00,0xff,0x1b,0x00,0xff]
+
+v_cmpx_nlt_f16 v1.h, v255.h
+// GFX11: v_cmpx_nlt_f16_e64 v1.h, v255.h op_sel:[1,1,0] ; encoding: [0x7e,0x18,0x8e,0xd4,0x01,0xff,0x03,0x00]
 
-v_cmpx_neq_f16 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmpx_neq_f16_e64_dpp v1, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x8d,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
+v_cmpx_nlt_f16 v1.h, v255.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_nlt_f16_e64_dpp v1.h, v255.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x18,0x8e,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
 
-v_cmpx_neq_f16 v1, v255 quad_perm:[3,2,1,0]
-// GFX11: v_cmpx_neq_f16_e64_dpp v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8d,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
+v_cmpx_nlt_f16 v1.h, v255.h quad_perm:[3,2,1,0]
+// GFX11: v_cmpx_nlt_f16_e64_dpp v1.h, v255.h op_sel:[1,1] quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x18,0x8e,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
 
-v_cmpx_neq_f16 v255, v2
-// GFX11: v_cmpx_neq_f16_e64 v255, v2             ; encoding: [0x7e,0x00,0x8d,0xd4,0xff,0x05,0x02,0x00]
+v_cmpx_nlt_f16 v1.l, v255.l
+// GFX11: v_cmpx_nlt_f16_e64 v1.l, v255.l         ; encoding: [0x7e,0x00,0x8e,0xd4,0x01,0xff,0x03,0x00]
 
-v_cmpx_neq_f16 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmpx_neq_f16_e64_dpp v255, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x8d,0xd4,0xe9,0x04,0x02,0x00,0xff,0x77,0x39,0x05]
+v_cmpx_nlt_f16 v1.l, v255.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_nlt_f16_e64_dpp v1.l, v255.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x8e,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
 
-v_cmpx_neq_f16 v255, v2 quad_perm:[3,2,1,0]
-// GFX11: v_cmpx_neq_f16_e64_dpp v255, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8d,0xd4,0xfa,0x04,0x02,0x00,0xff,0x1b,0x00,0xff]
+v_cmpx_nlt_f16 v1.l, v255.l quad_perm:[3,2,1,0]
+// GFX11: v_cmpx_nlt_f16_e64_dpp v1.l, v255.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8e,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
 
-v_cmpx_nge_f16 v1, v255
-// GFX11: v_cmpx_nge_f16_e64 v1, v255             ; encoding: [0x7e,0x00,0x89,0xd4,0x01,0xff,0x03,0x00]
+v_cmpx_nlt_f16 v255.h, v2.h
+// GFX11: v_cmpx_nlt_f16_e64 v255.h, v2.h op_sel:[1,1,0] ; encoding: [0x7e,0x18,0x8e,0xd4,0xff,0x05,0x02,0x00]
 
-v_cmpx_nge_f16 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmpx_nge_f16_e64_dpp v1, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x89,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
+v_cmpx_nlt_f16 v255.h, v2.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_nlt_f16_e64_dpp v255.h, v2.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x18,0x8e,0xd4,0xe9,0x04,0x02,0x00,0xff,0x77,0x39,0x05]
 
-v_cmpx_nge_f16 v1, v255 quad_perm:[3,2,1,0]
-// GFX11: v_cmpx_nge_f16_e64_dpp v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x89,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
+v_cmpx_nlt_f16 v255.h, v2.h quad_perm:[3,2,1,0]
+// GFX11: v_cmpx_nlt_f16_e64_dpp v255.h, v2.h op_sel:[1,1] quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x18,0x8e,0xd4,0xfa,0x04,0x02,0x00,0xff,0x1b,0x00,0xff]
 
-v_cmpx_nge_f16 v255, v2
-// GFX11: v_cmpx_nge_f16_e64 v255, v2             ; encoding: [0x7e,0x00,0x89,0xd4,0xff,0x05,0x02,0x00]
+v_cmpx_nlt_f16 v255.l, v2.l
+// GFX11: v_cmpx_nlt_f16_e64 v255.l, v2.l         ; encoding: [0x7e,0x00,0x8e,0xd4,0xff,0x05,0x02,0x00]
 
-v_cmpx_nge_f16 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmpx_nge_f16_e64_dpp v255, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x89,0xd4,0xe9,0x04,0x02,0x00,0xff,0x77,0x39,0x05]
+v_cmpx_nlt_f16 v255.l, v2.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_nlt_f16_e64_dpp v255.l, v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x8e,0xd4,0xe9,0x04,0x02,0x00,0xff,0x77,0x39,0x05]
 
-v_cmpx_nge_f16 v255, v2 quad_perm:[3,2,1,0]
-// GFX11: v_cmpx_nge_f16_e64_dpp v255, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x89,0xd4,0xfa,0x04,0x02,0x00,0xff,0x1b,0x00,0xff]
+v_cmpx_nlt_f16 v255.l, v2.l quad_perm:[3,2,1,0]
+// GFX11: v_cmpx_nlt_f16_e64_dpp v255.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8e,0xd4,0xfa,0x04,0x02,0x00,0xff,0x1b,0x00,0xff]
 
-v_cmpx_ngt_f16 v1, v255
-// GFX11: v_cmpx_ngt_f16_e64 v1, v255             ; encoding: [0x7e,0x00,0x8b,0xd4,0x01,0xff,0x03,0x00]
+v_cmpx_o_f16 v1.h, v255.h
+// GFX11: v_cmpx_o_f16_e64 v1.h, v255.h op_sel:[1,1,0] ; encoding: [0x7e,0x18,0x87,0xd4,0x01,0xff,0x03,0x00]
 
-v_cmpx_ngt_f16 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmpx_ngt_f16_e64_dpp v1, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x8b,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
+v_cmpx_o_f16 v1.h, v255.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_o_f16_e64_dpp v1.h, v255.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x18,0x87,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
 
-v_cmpx_ngt_f16 v1, v255 quad_perm:[3,2,1,0]
-// GFX11: v_cmpx_ngt_f16_e64_dpp v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8b,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
+v_cmpx_o_f16 v1.h, v255.h quad_perm:[3,2,1,0]
+// GFX11: v_cmpx_o_f16_e64_dpp v1.h, v255.h op_sel:[1,1] quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x18,0x87,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
 
-v_cmpx_ngt_f16 v255, v2
-// GFX11: v_cmpx_ngt_f16_e64 v255, v2             ; encoding: [0x7e,0x00,0x8b,0xd4,0xff,0x05,0x02,0x00]
+v_cmpx_o_f16 v1.l, v255.l
+// GFX11: v_cmpx_o_f16_e64 v1.l, v255.l           ; encoding: [0x7e,0x00,0x87,0xd4,0x01,0xff,0x03,0x00]
 
-v_cmpx_ngt_f16 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmpx_ngt_f16_e64_dpp v255, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x8b,0xd4,0xe9,0x04,0x02,0x00,0xff,0x77,0x39,0x05]
+v_cmpx_o_f16 v1.l, v255.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_o_f16_e64_dpp v1.l, v255.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x87,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
 
-v_cmpx_ngt_f16 v255, v2 quad_perm:[3,2,1,0]
-// GFX11: v_cmpx_ngt_f16_e64_dpp v255, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8b,0xd4,0xfa,0x04,0x02,0x00,0xff,0x1b,0x00,0xff]
+v_cmpx_o_f16 v1.l, v255.l quad_perm:[3,2,1,0]
+// GFX11: v_cmpx_o_f16_e64_dpp v1.l, v255.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x87,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
 
-v_cmpx_nle_f16 v1, v255
-// GFX11: v_cmpx_nle_f16_e64 v1, v255             ; encoding: [0x7e,0x00,0x8c,0xd4,0x01,0xff,0x03,0x00]
+v_cmpx_o_f16 v255.h, v2.h
+// GFX11: v_cmpx_o_f16_e64 v255.h, v2.h op_sel:[1,1,0] ; encoding: [0x7e,0x18,0x87,0xd4,0xff,0x05,0x02,0x00]
 
-v_cmpx_nle_f16 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmpx_nle_f16_e64_dpp v1, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x8c,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
+v_cmpx_o_f16 v255.h, v2.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_o_f16_e64_dpp v255.h, v2.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x18,0x87,0xd4,0xe9,0x04,0x02,0x00,0xff,0x77,0x39,0x05]
 
-v_cmpx_nle_f16 v1, v255 quad_perm:[3,2,1,0]
-// GFX11: v_cmpx_nle_f16_e64_dpp v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8c,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
+v_cmpx_o_f16 v255.h, v2.h quad_perm:[3,2,1,0]
+// GFX11: v_cmpx_o_f16_e64_dpp v255.h, v2.h op_sel:[1,1] quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x18,0x87,0xd4,0xfa,0x04,0x02,0x00,0xff,0x1b,0x00,0xff]
 
-v_cmpx_nle_f16 v255, v2
-// GFX11: v_cmpx_nle_f16_e64 v255, v2             ; encoding: [0x7e,0x00,0x8c,0xd4,0xff,0x05,0x02,0x00]
+v_cmpx_o_f16 v255.l, v2.l
+// GFX11: v_cmpx_o_f16_e64 v255.l, v2.l           ; encoding: [0x7e,0x00,0x87,0xd4,0xff,0x05,0x02,0x00]
 
-v_cmpx_nle_f16 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmpx_nle_f16_e64_dpp v255, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x8c,0xd4,0xe9,0x04,0x02,0x00,0xff,0x77,0x39,0x05]
+v_cmpx_o_f16 v255.l, v2.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_o_f16_e64_dpp v255.l, v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x87,0xd4,0xe9,0x04,0x02,0x00,0xff,0x77,0x39,0x05]
 
-v_cmpx_nle_f16 v255, v2 quad_perm:[3,2,1,0]
-// GFX11: v_cmpx_nle_f16_e64_dpp v255, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8c,0xd4,0xfa,0x04,0x02,0x00,0xff,0x1b,0x00,0xff]
+v_cmpx_o_f16 v255.l, v2.l quad_perm:[3,2,1,0]
+// GFX11: v_cmpx_o_f16_e64_dpp v255.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x87,0xd4,0xfa,0x04,0x02,0x00,0xff,0x1b,0x00,0xff]
 
-v_cmpx_nlg_f16 v1, v255
-// GFX11: v_cmpx_nlg_f16_e64 v1, v255             ; encoding: [0x7e,0x00,0x8a,0xd4,0x01,0xff,0x03,0x00]
+v_cmpx_t_f16 v1.h, v255.h
+// GFX11: v_cmpx_t_f16_e64 v1.h, v255.h op_sel:[1,1,0] ; encoding: [0x7e,0x18,0x8f,0xd4,0x01,0xff,0x03,0x00]
 
-v_cmpx_nlg_f16 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmpx_nlg_f16_e64_dpp v1, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x8a,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
+v_cmpx_t_f16 v1.h, v255.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_t_f16_e64_dpp v1.h, v255.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x18,0x8f,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
 
-v_cmpx_nlg_f16 v1, v255 quad_perm:[3,2,1,0]
-// GFX11: v_cmpx_nlg_f16_e64_dpp v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8a,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
+v_cmpx_t_f16 v1.h, v255.h quad_perm:[3,2,1,0]
+// GFX11: v_cmpx_t_f16_e64_dpp v1.h, v255.h op_sel:[1,1] quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x18,0x8f,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
 
-v_cmpx_nlg_f16 v255, v2
-// GFX11: v_cmpx_nlg_f16_e64 v255, v2             ; encoding: [0x7e,0x00,0x8a,0xd4,0xff,0x05,0x02,0x00]
+v_cmpx_t_f16 v1.l, v255.l
+// GFX11: v_cmpx_t_f16_e64 v1.l, v255.l           ; encoding: [0x7e,0x00,0x8f,0xd4,0x01,0xff,0x03,0x00]
 
-v_cmpx_nlg_f16 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmpx_nlg_f16_e64_dpp v255, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x8a,0xd4,0xe9,0x04,0x02,0x00,0xff,0x77,0x39,0x05]
+v_cmpx_t_f16 v1.l, v255.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_t_f16_e64_dpp v1.l, v255.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x8f,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
 
-v_cmpx_nlg_f16 v255, v2 quad_perm:[3,2,1,0]
-// GFX11: v_cmpx_nlg_f16_e64_dpp v255, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8a,0xd4,0xfa,0x04,0x02,0x00,0xff,0x1b,0x00,0xff]
+v_cmpx_t_f16 v1.l, v255.l quad_perm:[3,2,1,0]
+// GFX11: v_cmpx_t_f16_e64_dpp v1.l, v255.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8f,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
 
-v_cmpx_nlt_f16 v1, v255
-// GFX11: v_cmpx_nlt_f16_e64 v1, v255             ; encoding: [0x7e,0x00,0x8e,0xd4,0x01,0xff,0x03,0x00]
+v_cmpx_t_f16 v255.h, v2.h
+// GFX11: v_cmpx_t_f16_e64 v255.h, v2.h op_sel:[1,1,0] ; encoding: [0x7e,0x18,0x8f,0xd4,0xff,0x05,0x02,0x00]
 
-v_cmpx_nlt_f16 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmpx_nlt_f16_e64_dpp v1, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x8e,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
+v_cmpx_t_f16 v255.h, v2.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_t_f16_e64_dpp v255.h, v2.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x18,0x8f,0xd4,0xe9,0x04,0x02,0x00,0xff,0x77,0x39,0x05]
 
-v_cmpx_nlt_f16 v1, v255 quad_perm:[3,2,1,0]
-// GFX11: v_cmpx_nlt_f16_e64_dpp v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8e,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
+v_cmpx_t_f16 v255.h, v2.h quad_perm:[3,2,1,0]
+// GFX11: v_cmpx_t_f16_e64_dpp v255.h, v2.h op_sel:[1,1] quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x18,0x8f,0xd4,0xfa,0x04,0x02,0x00,0xff,0x1b,0x00,0xff]
 
-v_cmpx_nlt_f16 v255, v2
-// GFX11: v_cmpx_nlt_f16_e64 v255, v2             ; encoding: [0x7e,0x00,0x8e,0xd4,0xff,0x05,0x02,0x00]
+v_cmpx_t_f16 v255.l, v2.l
+// GFX11: v_cmpx_t_f16_e64 v255.l, v2.l           ; encoding: [0x7e,0x00,0x8f,0xd4,0xff,0x05,0x02,0x00]
 
-v_cmpx_nlt_f16 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmpx_nlt_f16_e64_dpp v255, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x8e,0xd4,0xe9,0x04,0x02,0x00,0xff,0x77,0x39,0x05]
+v_cmpx_t_f16 v255.l, v2.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_t_f16_e64_dpp v255.l, v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x8f,0xd4,0xe9,0x04,0x02,0x00,0xff,0x77,0x39,0x05]
 
-v_cmpx_nlt_f16 v255, v2 quad_perm:[3,2,1,0]
-// GFX11: v_cmpx_nlt_f16_e64_dpp v255, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8e,0xd4,0xfa,0x04,0x02,0x00,0xff,0x1b,0x00,0xff]
+v_cmpx_t_f16 v255.l, v2.l quad_perm:[3,2,1,0]
+// GFX11: v_cmpx_t_f16_e64_dpp v255.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8f,0xd4,0xfa,0x04,0x02,0x00,0xff,0x1b,0x00,0xff]
 
-v_cmpx_o_f16 v1, v255
-// GFX11: v_cmpx_o_f16_e64 v1, v255               ; encoding: [0x7e,0x00,0x87,0xd4,0x01,0xff,0x03,0x00]
+v_cmpx_tru_f16 v1.h, v255.h
+// GFX11: v_cmpx_t_f16_e64 v1.h, v255.h op_sel:[1,1,0] ; encoding: [0x7e,0x18,0x8f,0xd4,0x01,0xff,0x03,0x00]
 
-v_cmpx_o_f16 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmpx_o_f16_e64_dpp v1, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x87,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
+v_cmpx_tru_f16 v1.h, v255.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_t_f16_e64_dpp v1.h, v255.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x18,0x8f,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
 
-v_cmpx_o_f16 v1, v255 quad_perm:[3,2,1,0]
-// GFX11: v_cmpx_o_f16_e64_dpp v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x87,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
+v_cmpx_tru_f16 v1.h, v255.h quad_perm:[3,2,1,0]
+// GFX11: v_cmpx_t_f16_e64_dpp v1.h, v255.h op_sel:[1,1] quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x18,0x8f,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
 
-v_cmpx_o_f16 v255, v2
-// GFX11: v_cmpx_o_f16_e64 v255, v2               ; encoding: [0x7e,0x00,0x87,0xd4,0xff,0x05,0x02,0x00]
+v_cmpx_tru_f16 v1.l, v255.l
+// GFX11: v_cmpx_t_f16_e64 v1.l, v255.l           ; encoding: [0x7e,0x00,0x8f,0xd4,0x01,0xff,0x03,0x00]
 
-v_cmpx_o_f16 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmpx_o_f16_e64_dpp v255, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x87,0xd4,0xe9,0x04,0x02,0x00,0xff,0x77,0x39,0x05]
+v_cmpx_tru_f16 v1.l, v255.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_t_f16_e64_dpp v1.l, v255.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x8f,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
 
-v_cmpx_o_f16 v255, v2 quad_perm:[3,2,1,0]
-// GFX11: v_cmpx_o_f16_e64_dpp v255, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x87,0xd4,0xfa,0x04,0x02,0x00,0xff,0x1b,0x00,0xff]
+v_cmpx_tru_f16 v1.l, v255.l quad_perm:[3,2,1,0]
+// GFX11: v_cmpx_t_f16_e64_dpp v1.l, v255.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8f,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
 
-v_cmpx_t_f16 v1, v255
-// GFX11: v_cmpx_t_f16_e64 v1, v255               ; encoding: [0x7e,0x00,0x8f,0xd4,0x01,0xff,0x03,0x00]
+v_cmpx_tru_f16 v255.h, v2.h
+// GFX11: v_cmpx_t_f16_e64 v255.h, v2.h op_sel:[1,1,0] ; encoding: [0x7e,0x18,0x8f,0xd4,0xff,0x05,0x02,0x00]
 
-v_cmpx_t_f16 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmpx_t_f16_e64_dpp v1, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x8f,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
+v_cmpx_tru_f16 v255.h, v2.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_t_f16_e64_dpp v255.h, v2.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x18,0x8f,0xd4,0xe9,0x04,0x02,0x00,0xff,0x77,0x39,0x05]
 
-v_cmpx_t_f16 v1, v255 quad_perm:[3,2,1,0]
-// GFX11: v_cmpx_t_f16_e64_dpp v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8f,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
+v_cmpx_tru_f16 v255.h, v2.h quad_perm:[3,2,1,0]
+// GFX11: v_cmpx_t_f16_e64_dpp v255.h, v2.h op_sel:[1,1] quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x18,0x8f,0xd4,0xfa,0x04,0x02,0x00,0xff,0x1b,0x00,0xff]
 
-v_cmpx_t_f16 v255, v2
-// GFX11: v_cmpx_t_f16_e64 v255, v2               ; encoding: [0x7e,0x00,0x8f,0xd4,0xff,0x05,0x02,0x00]
+v_cmpx_tru_f16 v255.l, v2.l
+// GFX11: v_cmpx_t_f16_e64 v255.l, v2.l           ; encoding: [0x7e,0x00,0x8f,0xd4,0xff,0x05,0x02,0x00]
 
-v_cmpx_t_f16 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmpx_t_f16_e64_dpp v255, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x8f,0xd4,0xe9,0x04,0x02,0x00,0xff,0x77,0x39,0x05]
+v_cmpx_tru_f16 v255.l, v2.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_t_f16_e64_dpp v255.l, v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x8f,0xd4,0xe9,0x04,0x02,0x00,0xff,0x77,0x39,0x05]
 
-v_cmpx_t_f16 v255, v2 quad_perm:[3,2,1,0]
-// GFX11: v_cmpx_t_f16_e64_dpp v255, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8f,0xd4,0xfa,0x04,0x02,0x00,0xff,0x1b,0x00,0xff]
+v_cmpx_tru_f16 v255.l, v2.l quad_perm:[3,2,1,0]
+// GFX11: v_cmpx_t_f16_e64_dpp v255.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8f,0xd4,0xfa,0x04,0x02,0x00,0xff,0x1b,0x00,0xff]
 
-v_cmpx_tru_f16 v1, v255
-// GFX11: v_cmpx_t_f16_e64 v1, v255               ; encoding: [0x7e,0x00,0x8f,0xd4,0x01,0xff,0x03,0x00]
+v_cmpx_u_f16 v1.h, v255.h
+// GFX11: v_cmpx_u_f16_e64 v1.h, v255.h op_sel:[1,1,0] ; encoding: [0x7e,0x18,0x88,0xd4,0x01,0xff,0x03,0x00]
 
-v_cmpx_tru_f16 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmpx_t_f16_e64_dpp v1, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x8f,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
+v_cmpx_u_f16 v1.h, v255.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_u_f16_e64_dpp v1.h, v255.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x18,0x88,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
 
-v_cmpx_tru_f16 v1, v255 quad_perm:[3,2,1,0]
-// GFX11: v_cmpx_t_f16_e64_dpp v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8f,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
+v_cmpx_u_f16 v1.h, v255.h quad_perm:[3,2,1,0]
+// GFX11: v_cmpx_u_f16_e64_dpp v1.h, v255.h op_sel:[1,1] quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x18,0x88,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
 
-v_cmpx_tru_f16 v255, v2
-// GFX11: v_cmpx_t_f16_e64 v255, v2               ; encoding: [0x7e,0x00,0x8f,0xd4,0xff,0x05,0x02,0x00]
+v_cmpx_u_f16 v1.l, v255.l
+// GFX11: v_cmpx_u_f16_e64 v1.l, v255.l           ; encoding: [0x7e,0x00,0x88,0xd4,0x01,0xff,0x03,0x00]
 
-v_cmpx_tru_f16 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmpx_t_f16_e64_dpp v255, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x8f,0xd4,0xe9,0x04,0x02,0x00,0xff,0x77,0x39,0x05]
+v_cmpx_u_f16 v1.l, v255.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_u_f16_e64_dpp v1.l, v255.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x88,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
 
-v_cmpx_tru_f16 v255, v2 quad_perm:[3,2,1,0]
-// GFX11: v_cmpx_t_f16_e64_dpp v255, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8f,0xd4,0xfa,0x04,0x02,0x00,0xff,0x1b,0x00,0xff]
+v_cmpx_u_f16 v1.l, v255.l quad_perm:[3,2,1,0]
+// GFX11: v_cmpx_u_f16_e64_dpp v1.l, v255.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x88,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
 
-v_cmpx_u_f16 v1, v255
-// GFX11: v_cmpx_u_f16_e64 v1, v255               ; encoding: [0x7e,0x00,0x88,0xd4,0x01,0xff,0x03,0x00]
+v_cmpx_u_f16 v255.h, v2.h
+// GFX11: v_cmpx_u_f16_e64 v255.h, v2.h op_sel:[1,1,0] ; encoding: [0x7e,0x18,0x88,0xd4,0xff,0x05,0x02,0x00]
 
-v_cmpx_u_f16 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmpx_u_f16_e64_dpp v1, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x88,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
+v_cmpx_u_f16 v255.h, v2.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_u_f16_e64_dpp v255.h, v2.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x18,0x88,0xd4,0xe9,0x04,0x02,0x00,0xff,0x77,0x39,0x05]
 
-v_cmpx_u_f16 v1, v255 quad_perm:[3,2,1,0]
-// GFX11: v_cmpx_u_f16_e64_dpp v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x88,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
+v_cmpx_u_f16 v255.h, v2.h quad_perm:[3,2,1,0]
+// GFX11: v_cmpx_u_f16_e64_dpp v255.h, v2.h op_sel:[1,1] quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x18,0x88,0xd4,0xfa,0x04,0x02,0x00,0xff,0x1b,0x00,0xff]
 
-v_cmpx_u_f16 v255, v2
-// GFX11: v_cmpx_u_f16_e64 v255, v2               ; encoding: [0x7e,0x00,0x88,0xd4,0xff,0x05,0x02,0x00]
+v_cmpx_u_f16 v255.l, v2.l
+// GFX11: v_cmpx_u_f16_e64 v255.l, v2.l           ; encoding: [0x7e,0x00,0x88,0xd4,0xff,0x05,0x02,0x00]
 
-v_cmpx_u_f16 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmpx_u_f16_e64_dpp v255, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x88,0xd4,0xe9,0x04,0x02,0x00,0xff,0x77,0x39,0x05]
+v_cmpx_u_f16 v255.l, v2.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_u_f16_e64_dpp v255.l, v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x88,0xd4,0xe9,0x04,0x02,0x00,0xff,0x77,0x39,0x05]
 
-v_cmpx_u_f16 v255, v2 quad_perm:[3,2,1,0]
-// GFX11: v_cmpx_u_f16_e64_dpp v255, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x88,0xd4,0xfa,0x04,0x02,0x00,0xff,0x1b,0x00,0xff]
+v_cmpx_u_f16 v255.l, v2.l quad_perm:[3,2,1,0]
+// GFX11: v_cmpx_u_f16_e64_dpp v255.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x88,0xd4,0xfa,0x04,0x02,0x00,0xff,0x1b,0x00,0xff]
diff --git a/llvm/test/MC/AMDGPU/gfx11_unsupported.s b/llvm/test/MC/AMDGPU/gfx11_unsupported.s
index 640ffb65a76bf..d5d10ce548c4d 100644
--- a/llvm/test/MC/AMDGPU/gfx11_unsupported.s
+++ b/llvm/test/MC/AMDGPU/gfx11_unsupported.s
@@ -2067,3 +2067,6 @@ ds_subrev_rtn_u64 v[5:6], v1, v[2:3]
 
 v_dot2c_f32_bf16 v5, v1, v2
 // CHECK: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+ds_bpermute_fi_b32 v5, v1, v2
+// CHECK: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
diff --git a/llvm/test/MC/AMDGPU/gfx12_asm_vop3.s b/llvm/test/MC/AMDGPU/gfx12_asm_vop3.s
index cd4ed2b9458e6..72d201e060df7 100644
--- a/llvm/test/MC/AMDGPU/gfx12_asm_vop3.s
+++ b/llvm/test/MC/AMDGPU/gfx12_asm_vop3.s
@@ -452,6 +452,9 @@ v_alignbyte_b32 v5, src_scc, vcc_lo, -1
 v_alignbyte_b32 v255, 0xaf123456, vcc_hi, null
 // GFX12: v_alignbyte_b32 v255, 0xaf123456, vcc_hi, null ; encoding: [0xff,0x00,0x17,0xd6,0xff,0xd6,0xf0,0x01,0x56,0x34,0x12,0xaf]
 
+v_alignbyte_b32 v5, vcc_hi, 0xaf123456, v255.h
+// GFX12: v_alignbyte_b32 v5, vcc_hi, 0xaf123456, v255.h op_sel:[0,0,1,0] ; encoding: [0x05,0x20,0x17,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf]
+
 v_and_b16 v5.l, v1.l, v2.l
 // GFX12: v_and_b16 v5.l, v1.l, v2.l              ; encoding: [0x05,0x00,0x62,0xd7,0x01,0x05,0x02,0x00]
 
@@ -2816,11 +2819,11 @@ v_mad_i16 v5.l, vcc_hi, 0xfe0b, v255.h
 v_mad_i16 v255.h, 0xfe0b, vcc_hi, null clamp
 // GFX12: v_mad_i16 v255.h, 0xfe0b, vcc_hi, null op_sel:[0,0,0,1] clamp ; encoding: [0xff,0xc0,0x53,0xd6,0xff,0xd6,0xf0,0x01,0x0b,0xfe,0x00,0x00]
 
-v_mad_i32_i16 v5, v1, v2, v3
-// GFX12: v_mad_i32_i16 v5, v1, v2, v3            ; encoding: [0x05,0x00,0x5a,0xd6,0x01,0x05,0x0e,0x04]
+v_mad_i32_i16 v5, v1.l, v2.l, v3
+// GFX12: v_mad_i32_i16 v5, v1.l, v2.l, v3        ; encoding: [0x05,0x00,0x5a,0xd6,0x01,0x05,0x0e,0x04]
 
-v_mad_i32_i16 v5, v255, v255, s3
-// GFX12: v_mad_i32_i16 v5, v255, v255, s3        ; encoding: [0x05,0x00,0x5a,0xd6,0xff,0xff,0x0f,0x00]
+v_mad_i32_i16 v5, v255.l, v255.l, s3
+// GFX12: v_mad_i32_i16 v5, v255.l, v255.l, s3    ; encoding: [0x05,0x00,0x5a,0xd6,0xff,0xff,0x0f,0x00]
 
 v_mad_i32_i16 v5, s1, s2, v255
 // GFX12: v_mad_i32_i16 v5, s1, s2, v255          ; encoding: [0x05,0x00,0x5a,0xd6,0x01,0x04,0xfc,0x07]
@@ -2861,6 +2864,18 @@ v_mad_i32_i16 v5, src_scc, vcc_lo, src_scc op_sel:[1,0,0,0]
 v_mad_i32_i16 v255, 0xfe0b, vcc_hi, 0.5 op_sel:[0,1,0,0] clamp
 // GFX12: v_mad_i32_i16 v255, 0xfe0b, vcc_hi, 0.5 op_sel:[0,1,0,0] clamp ; encoding: [0xff,0x90,0x5a,0xd6,0xff,0xd6,0xc0,0x03,0x0b,0xfe,0x00,0x00]
 
+v_mad_i32_i16 v5, v1.h, v2.l, v3
+// GFX12: v_mad_i32_i16 v5, v1.h, v2.l, v3 op_sel:[1,0,0,0] ; encoding: [0x05,0x08,0x5a,0xd6,0x01,0x05,0x0e,0x04]
+
+v_mad_i32_i16 v5, v255.l, v255.h, s3
+// GFX12: v_mad_i32_i16 v5, v255.l, v255.h, s3 op_sel:[0,1,0,0] ; encoding: [0x05,0x10,0x5a,0xd6,0xff,0xff,0x0f,0x00]
+
+v_mad_i32_i16 v5, src_scc, vcc_lo, src_scc
+// GFX12: v_mad_i32_i16 v5, src_scc, vcc_lo, src_scc ; encoding: [0x05,0x00,0x5a,0xd6,0xfd,0xd4,0xf4,0x03]
+
+v_mad_i32_i16 v255, 0xfe0b, vcc_hi, 0.5 clamp
+// GFX12: v_mad_i32_i16 v255, 0xfe0b, vcc_hi, 0.5 clamp ; encoding: [0xff,0x80,0x5a,0xd6,0xff,0xd6,0xc0,0x03,0x0b,0xfe,0x00,0x00]
+
 v_mad_i32_i24 v5, v1, v2, s3
 // GFX12: v_mad_i32_i24 v5, v1, v2, s3            ; encoding: [0x05,0x00,0x0a,0xd6,0x01,0x05,0x0e,0x00]
 
@@ -3038,11 +3053,11 @@ v_mad_u16 v5.l, vcc_hi, 0xfe0b, v255.h
 v_mad_u16 v255.h, 0xfe0b, vcc_hi, null clamp
 // GFX12: v_mad_u16 v255.h, 0xfe0b, vcc_hi, null op_sel:[0,0,0,1] clamp ; encoding: [0xff,0xc0,0x41,0xd6,0xff,0xd6,0xf0,0x01,0x0b,0xfe,0x00,0x00]
 
-v_mad_u32_u16 v5, v1, v2, v3
-// GFX12: v_mad_u32_u16 v5, v1, v2, v3            ; encoding: [0x05,0x00,0x59,0xd6,0x01,0x05,0x0e,0x04]
+v_mad_u32_u16 v5, v1.l, v2.l, v3
+// GFX12: v_mad_u32_u16 v5, v1.l, v2.l, v3        ; encoding: [0x05,0x00,0x59,0xd6,0x01,0x05,0x0e,0x04]
 
-v_mad_u32_u16 v5, v255, v255, s3
-// GFX12: v_mad_u32_u16 v5, v255, v255, s3        ; encoding: [0x05,0x00,0x59,0xd6,0xff,0xff,0x0f,0x00]
+v_mad_u32_u16 v5, v255.l, v255.l, s3
+// GFX12: v_mad_u32_u16 v5, v255.l, v255.l, s3    ; encoding: [0x05,0x00,0x59,0xd6,0xff,0xff,0x0f,0x00]
 
 v_mad_u32_u16 v5, s1, s2, v255
 // GFX12: v_mad_u32_u16 v5, s1, s2, v255          ; encoding: [0x05,0x00,0x59,0xd6,0x01,0x04,0xfc,0x07]
@@ -3083,6 +3098,18 @@ v_mad_u32_u16 v5, src_scc, vcc_lo, src_scc op_sel:[1,0,0,0]
 v_mad_u32_u16 v255, 0xfe0b, vcc_hi, 0.5 op_sel:[0,1,0,0] clamp
 // GFX12: v_mad_u32_u16 v255, 0xfe0b, vcc_hi, 0.5 op_sel:[0,1,0,0] clamp ; encoding: [0xff,0x90,0x59,0xd6,0xff,0xd6,0xc0,0x03,0x0b,0xfe,0x00,0x00]
 
+v_mad_u32_u16 v5, v1.h, v2.l, v3
+// GFX12: v_mad_u32_u16 v5, v1.h, v2.l, v3 op_sel:[1,0,0,0] ; encoding: [0x05,0x08,0x59,0xd6,0x01,0x05,0x0e,0x04]
+
+v_mad_u32_u16 v5, v255.l, v255.h, s3
+// GFX12: v_mad_u32_u16 v5, v255.l, v255.h, s3 op_sel:[0,1,0,0] ; encoding: [0x05,0x10,0x59,0xd6,0xff,0xff,0x0f,0x00]
+
+v_mad_u32_u16 v5, src_scc, vcc_lo, src_scc
+// GFX12: v_mad_u32_u16 v5, src_scc, vcc_lo, src_scc ; encoding: [0x05,0x00,0x59,0xd6,0xfd,0xd4,0xf4,0x03]
+
+v_mad_u32_u16 v255, 0xfe0b, vcc_hi, 0.5 clamp
+// GFX12: v_mad_u32_u16 v255, 0xfe0b, vcc_hi, 0.5 clamp ; encoding: [0xff,0x80,0x59,0xd6,0xff,0xd6,0xc0,0x03,0x0b,0xfe,0x00,0x00]
+
 v_mad_u32_u24 v5, v1, v2, s3
 // GFX12: v_mad_u32_u24 v5, v1, v2, s3            ; encoding: [0x05,0x00,0x0b,0xd6,0x01,0x05,0x0e,0x00]
 
@@ -6445,101 +6472,119 @@ v_maximum_f32 v5, -src_scc, |vcc_lo|
 v_maximum_f32 v255, -|0xaf123456|, -|vcc_hi|
 // GFX12: v_maximum_f32 v255, -|0xaf123456|, -|vcc_hi| ; encoding: [0xff,0x03,0x66,0xd7,0xff,0xd6,0x00,0x60,0x56,0x34,0x12,0xaf]
 
-v_minimum_f16 v5, v1, v2
-// GFX12: v_minimum_f16 v5, v1, v2                ; encoding: [0x05,0x00,0x67,0xd7,0x01,0x05,0x02,0x00]
+v_minimum_f16 v5.l, v1.l, v2.l
+// GFX12: v_minimum_f16 v5.l, v1.l, v2.l          ; encoding: [0x05,0x00,0x67,0xd7,0x01,0x05,0x02,0x00]
+
+v_minimum_f16 v5.l, v255.l, v255.l
+// GFX12: v_minimum_f16 v5.l, v255.l, v255.l      ; encoding: [0x05,0x00,0x67,0xd7,0xff,0xff,0x03,0x00]
+
+v_minimum_f16 v5.l, s1, s2
+// GFX12: v_minimum_f16 v5.l, s1, s2              ; encoding: [0x05,0x00,0x67,0xd7,0x01,0x04,0x00,0x00]
+
+v_minimum_f16 v5.l, s105, s105
+// GFX12: v_minimum_f16 v5.l, s105, s105          ; encoding: [0x05,0x00,0x67,0xd7,0x69,0xd2,0x00,0x00]
 
-v_minimum_f16 v5, v255, v255
-// GFX12: v_minimum_f16 v5, v255, v255            ; encoding: [0x05,0x00,0x67,0xd7,0xff,0xff,0x03,0x00]
+v_minimum_f16 v5.l, vcc_lo, ttmp15
+// GFX12: v_minimum_f16 v5.l, vcc_lo, ttmp15      ; encoding: [0x05,0x00,0x67,0xd7,0x6a,0xf6,0x00,0x00]
 
-v_minimum_f16 v5, s1, s2
-// GFX12: v_minimum_f16 v5, s1, s2                ; encoding: [0x05,0x00,0x67,0xd7,0x01,0x04,0x00,0x00]
+v_minimum_f16 v5.l, vcc_hi, 0xaf12
+// GFX12: v_minimum_f16 v5.l, vcc_hi, 0xaf12      ; encoding: [0x05,0x00,0x67,0xd7,0x6b,0xfe,0x01,0x00,0x12,0xaf,0x00,0x00]
 
-v_minimum_f16 v5, s105, s105
-// GFX12: v_minimum_f16 v5, s105, s105            ; encoding: [0x05,0x00,0x67,0xd7,0x69,0xd2,0x00,0x00]
+v_minimum_f16 v5.l, ttmp15, src_scc
+// GFX12: v_minimum_f16 v5.l, ttmp15, src_scc     ; encoding: [0x05,0x00,0x67,0xd7,0x7b,0xfa,0x01,0x00]
 
-v_minimum_f16 v5, vcc_lo, ttmp15
-// GFX12: v_minimum_f16 v5, vcc_lo, ttmp15        ; encoding: [0x05,0x00,0x67,0xd7,0x6a,0xf6,0x00,0x00]
+v_minimum_f16 v5.l, m0, 0.5
+// GFX12: v_minimum_f16 v5.l, m0, 0.5             ; encoding: [0x05,0x00,0x67,0xd7,0x7d,0xe0,0x01,0x00]
 
-v_minimum_f16 v5, vcc_hi, 0xaf12
-// GFX12: v_minimum_f16 v5, vcc_hi, 0xaf12        ; encoding: [0x05,0x00,0x67,0xd7,0x6b,0xfe,0x01,0x00,0x12,0xaf,0x00,0x00]
+v_minimum_f16 v5.l, exec_lo, -1
+// GFX12: v_minimum_f16 v5.l, exec_lo, -1         ; encoding: [0x05,0x00,0x67,0xd7,0x7e,0x82,0x01,0x00]
 
-v_minimum_f16 v5, ttmp15, src_scc
-// GFX12: v_minimum_f16 v5, ttmp15, src_scc       ; encoding: [0x05,0x00,0x67,0xd7,0x7b,0xfa,0x01,0x00]
+v_minimum_f16 v5.l, |exec_hi|, null
+// GFX12: v_minimum_f16 v5.l, |exec_hi|, null     ; encoding: [0x05,0x01,0x67,0xd7,0x7f,0xf8,0x00,0x00]
 
-v_minimum_f16 v5, m0, 0.5
-// GFX12: v_minimum_f16 v5, m0, 0.5               ; encoding: [0x05,0x00,0x67,0xd7,0x7d,0xe0,0x01,0x00]
+v_minimum_f16 v5.l, null, exec_lo
+// GFX12: v_minimum_f16 v5.l, null, exec_lo       ; encoding: [0x05,0x00,0x67,0xd7,0x7c,0xfc,0x00,0x00]
 
-v_minimum_f16 v5, exec_lo, -1
-// GFX12: v_minimum_f16 v5, exec_lo, -1           ; encoding: [0x05,0x00,0x67,0xd7,0x7e,0x82,0x01,0x00]
+v_minimum_f16 v5.l, -1, exec_hi
+// GFX12: v_minimum_f16 v5.l, -1, exec_hi         ; encoding: [0x05,0x00,0x67,0xd7,0xc1,0xfe,0x00,0x00]
 
-v_minimum_f16 v5, |exec_hi|, null
-// GFX12: v_minimum_f16 v5, |exec_hi|, null       ; encoding: [0x05,0x01,0x67,0xd7,0x7f,0xf8,0x00,0x00]
+v_minimum_f16 v5.l, 0.5, -m0
+// GFX12: v_minimum_f16 v5.l, 0.5, -m0            ; encoding: [0x05,0x00,0x67,0xd7,0xf0,0xfa,0x00,0x40]
 
-v_minimum_f16 v5, null, exec_lo
-// GFX12: v_minimum_f16 v5, null, exec_lo         ; encoding: [0x05,0x00,0x67,0xd7,0x7c,0xfc,0x00,0x00]
+v_minimum_f16 v5.l, -src_scc, |vcc_lo|
+// GFX12: v_minimum_f16 v5.l, -src_scc, |vcc_lo|  ; encoding: [0x05,0x02,0x67,0xd7,0xfd,0xd4,0x00,0x20]
 
-v_minimum_f16 v5, -1, exec_hi
-// GFX12: v_minimum_f16 v5, -1, exec_hi           ; encoding: [0x05,0x00,0x67,0xd7,0xc1,0xfe,0x00,0x00]
+v_minimum_f16 v255.l, -|0xaf12|, -|vcc_hi|
+// GFX12: v_minimum_f16 v255.l, -|0xaf12|, -|vcc_hi| ; encoding: [0xff,0x03,0x67,0xd7,0xff,0xd6,0x00,0x60,0x12,0xaf,0x00,0x00]
 
-v_minimum_f16 v5, 0.5, -m0
-// GFX12: v_minimum_f16 v5, 0.5, -m0              ; encoding: [0x05,0x00,0x67,0xd7,0xf0,0xfa,0x00,0x40]
+v_minimum_f16 v205.l, v201.l, v200.l
+// GFX12: v_minimum_f16 v205.l, v201.l, v200.l    ; encoding: [0xcd,0x00,0x67,0xd7,0xc9,0x91,0x03,0x00]
 
-v_minimum_f16 v5, -src_scc, |vcc_lo|
-// GFX12: v_minimum_f16 v5, -src_scc, |vcc_lo|    ; encoding: [0x05,0x02,0x67,0xd7,0xfd,0xd4,0x00,0x20]
+v_minimum_f16 v5.l, v1.h, v2.l
+// GFX12: v_minimum_f16 v5.l, v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x05,0x08,0x67,0xd7,0x01,0x05,0x02,0x00]
 
-v_minimum_f16 v255, -|0xaf12|, -|vcc_hi|
-// GFX12: v_minimum_f16 v255, -|0xaf12|, -|vcc_hi| ; encoding: [0xff,0x03,0x67,0xd7,0xff,0xd6,0x00,0x60,0x12,0xaf,0x00,0x00]
+v_minimum_f16 v5.l, v255.l, v255.h
+// GFX12: v_minimum_f16 v5.l, v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x05,0x10,0x67,0xd7,0xff,0xff,0x03,0x00]
 
-v_minimum_f16 v205, v201, v200
-// GFX12: v_minimum_f16 v205, v201, v200          ; encoding: [0xcd,0x00,0x67,0xd7,0xc9,0x91,0x03,0x00]
+v_minimum_f16 v255.h, 0xfe0b, vcc_hi
+// GFX12: v_minimum_f16 v255.h, 0xfe0b, vcc_hi op_sel:[0,0,1] ; encoding: [0xff,0x40,0x67,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
 
-v_maximum_f16 v5, v1, v2
-// GFX12: v_maximum_f16 v5, v1, v2                ; encoding: [0x05,0x00,0x68,0xd7,0x01,0x05,0x02,0x00]
+v_maximum_f16 v5.l, v1.l, v2.l
+// GFX12: v_maximum_f16 v5.l, v1.l, v2.l          ; encoding: [0x05,0x00,0x68,0xd7,0x01,0x05,0x02,0x00]
 
-v_maximum_f16 v5, v255, v255
-// GFX12: v_maximum_f16 v5, v255, v255            ; encoding: [0x05,0x00,0x68,0xd7,0xff,0xff,0x03,0x00]
+v_maximum_f16 v5.l, v255.l, v255.l
+// GFX12: v_maximum_f16 v5.l, v255.l, v255.l      ; encoding: [0x05,0x00,0x68,0xd7,0xff,0xff,0x03,0x00]
 
-v_maximum_f16 v5, s1, s2
-// GFX12: v_maximum_f16 v5, s1, s2                ; encoding: [0x05,0x00,0x68,0xd7,0x01,0x04,0x00,0x00]
+v_maximum_f16 v5.l, s1, s2
+// GFX12: v_maximum_f16 v5.l, s1, s2              ; encoding: [0x05,0x00,0x68,0xd7,0x01,0x04,0x00,0x00]
 
-v_maximum_f16 v5, s105, s105
-// GFX12: v_maximum_f16 v5, s105, s105            ; encoding: [0x05,0x00,0x68,0xd7,0x69,0xd2,0x00,0x00]
+v_maximum_f16 v5.l, s105, s105
+// GFX12: v_maximum_f16 v5.l, s105, s105          ; encoding: [0x05,0x00,0x68,0xd7,0x69,0xd2,0x00,0x00]
 
-v_maximum_f16 v5, vcc_lo, ttmp15
-// GFX12: v_maximum_f16 v5, vcc_lo, ttmp15        ; encoding: [0x05,0x00,0x68,0xd7,0x6a,0xf6,0x00,0x00]
+v_maximum_f16 v5.l, vcc_lo, ttmp15
+// GFX12: v_maximum_f16 v5.l, vcc_lo, ttmp15      ; encoding: [0x05,0x00,0x68,0xd7,0x6a,0xf6,0x00,0x00]
 
-v_maximum_f16 v5, vcc_hi, 0xaf12
-// GFX12: v_maximum_f16 v5, vcc_hi, 0xaf12        ; encoding: [0x05,0x00,0x68,0xd7,0x6b,0xfe,0x01,0x00,0x12,0xaf,0x00,0x00]
+v_maximum_f16 v5.l, vcc_hi, 0xaf12
+// GFX12: v_maximum_f16 v5.l, vcc_hi, 0xaf12      ; encoding: [0x05,0x00,0x68,0xd7,0x6b,0xfe,0x01,0x00,0x12,0xaf,0x00,0x00]
 
-v_maximum_f16 v5, ttmp15, src_scc
-// GFX12: v_maximum_f16 v5, ttmp15, src_scc       ; encoding: [0x05,0x00,0x68,0xd7,0x7b,0xfa,0x01,0x00]
+v_maximum_f16 v5.l, ttmp15, src_scc
+// GFX12: v_maximum_f16 v5.l, ttmp15, src_scc     ; encoding: [0x05,0x00,0x68,0xd7,0x7b,0xfa,0x01,0x00]
 
-v_maximum_f16 v5, m0, 0.5
-// GFX12: v_maximum_f16 v5, m0, 0.5               ; encoding: [0x05,0x00,0x68,0xd7,0x7d,0xe0,0x01,0x00]
+v_maximum_f16 v5.l, m0, 0.5
+// GFX12: v_maximum_f16 v5.l, m0, 0.5             ; encoding: [0x05,0x00,0x68,0xd7,0x7d,0xe0,0x01,0x00]
 
-v_maximum_f16 v5, exec_lo, -1
-// GFX12: v_maximum_f16 v5, exec_lo, -1           ; encoding: [0x05,0x00,0x68,0xd7,0x7e,0x82,0x01,0x00]
+v_maximum_f16 v5.l, exec_lo, -1
+// GFX12: v_maximum_f16 v5.l, exec_lo, -1         ; encoding: [0x05,0x00,0x68,0xd7,0x7e,0x82,0x01,0x00]
 
-v_maximum_f16 v5, |exec_hi|, null
-// GFX12: v_maximum_f16 v5, |exec_hi|, null       ; encoding: [0x05,0x01,0x68,0xd7,0x7f,0xf8,0x00,0x00]
+v_maximum_f16 v5.l, |exec_hi|, null
+// GFX12: v_maximum_f16 v5.l, |exec_hi|, null     ; encoding: [0x05,0x01,0x68,0xd7,0x7f,0xf8,0x00,0x00]
 
-v_maximum_f16 v5, null, exec_lo
-// GFX12: v_maximum_f16 v5, null, exec_lo         ; encoding: [0x05,0x00,0x68,0xd7,0x7c,0xfc,0x00,0x00]
+v_maximum_f16 v5.l, null, exec_lo
+// GFX12: v_maximum_f16 v5.l, null, exec_lo       ; encoding: [0x05,0x00,0x68,0xd7,0x7c,0xfc,0x00,0x00]
 
-v_maximum_f16 v5, -1, exec_hi
-// GFX12: v_maximum_f16 v5, -1, exec_hi           ; encoding: [0x05,0x00,0x68,0xd7,0xc1,0xfe,0x00,0x00]
+v_maximum_f16 v5.l, -1, exec_hi
+// GFX12: v_maximum_f16 v5.l, -1, exec_hi         ; encoding: [0x05,0x00,0x68,0xd7,0xc1,0xfe,0x00,0x00]
 
-v_maximum_f16 v5, 0.5, -m0
-// GFX12: v_maximum_f16 v5, 0.5, -m0              ; encoding: [0x05,0x00,0x68,0xd7,0xf0,0xfa,0x00,0x40]
+v_maximum_f16 v5.l, 0.5, -m0
+// GFX12: v_maximum_f16 v5.l, 0.5, -m0            ; encoding: [0x05,0x00,0x68,0xd7,0xf0,0xfa,0x00,0x40]
 
-v_maximum_f16 v5, -src_scc, |vcc_lo|
-// GFX12: v_maximum_f16 v5, -src_scc, |vcc_lo|    ; encoding: [0x05,0x02,0x68,0xd7,0xfd,0xd4,0x00,0x20]
+v_maximum_f16 v5.l, -src_scc, |vcc_lo|
+// GFX12: v_maximum_f16 v5.l, -src_scc, |vcc_lo|  ; encoding: [0x05,0x02,0x68,0xd7,0xfd,0xd4,0x00,0x20]
 
-v_maximum_f16 v255, -|0xaf12|, -|vcc_hi|
-// GFX12: v_maximum_f16 v255, -|0xaf12|, -|vcc_hi| ; encoding: [0xff,0x03,0x68,0xd7,0xff,0xd6,0x00,0x60,0x12,0xaf,0x00,0x00]
+v_maximum_f16 v255.l, -|0xaf12|, -|vcc_hi|
+// GFX12: v_maximum_f16 v255.l, -|0xaf12|, -|vcc_hi| ; encoding: [0xff,0x03,0x68,0xd7,0xff,0xd6,0x00,0x60,0x12,0xaf,0x00,0x00]
 
-v_maximum_f16 v205, v201, v200
-// GFX12: v_maximum_f16 v205, v201, v200          ; encoding: [0xcd,0x00,0x68,0xd7,0xc9,0x91,0x03,0x00]
+v_maximum_f16 v205.l, v201.l, v200.l
+// GFX12: v_maximum_f16 v205.l, v201.l, v200.l    ; encoding: [0xcd,0x00,0x68,0xd7,0xc9,0x91,0x03,0x00]
+
+v_maximum_f16 v5.l, v1.h, v2.l
+// GFX12: v_maximum_f16 v5.l, v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x05,0x08,0x68,0xd7,0x01,0x05,0x02,0x00]
+
+v_maximum_f16 v5.l, v255.l, v255.h
+// GFX12: v_maximum_f16 v5.l, v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x05,0x10,0x68,0xd7,0xff,0xff,0x03,0x00]
+
+v_maximum_f16 v255.h, 0xfe0b, vcc_hi
+// GFX12: v_maximum_f16 v255.h, 0xfe0b, vcc_hi op_sel:[0,0,1] ; encoding: [0xff,0x40,0x68,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
 
 v_minimum_f64 v[5:6], v[1:2], v[3:4]
 // GFX12: v_minimum_f64 v[5:6], v[1:2], v[3:4]    ; encoding: [0x05,0x00,0x41,0xd7,0x01,0x07,0x02,0x00]
@@ -6721,92 +6766,116 @@ v_maximum3_f32 v5, -src_scc, |vcc_lo|, -1 mul:4
 v_maximum3_f32 v255, -|0xaf123456|, -|vcc_hi|, null clamp div:2
 // GFX12: v_maximum3_f32 v255, -|0xaf123456|, -|vcc_hi|, null clamp div:2 ; encoding: [0xff,0x83,0x2e,0xd6,0xff,0xd6,0xf0,0x79,0x56,0x34,0x12,0xaf]
 
-v_minimum3_f16 v5, v1, v2, s3
-// GFX12: v_minimum3_f16 v5, v1, v2, s3           ; encoding: [0x05,0x00,0x2f,0xd6,0x01,0x05,0x0e,0x00]
+v_minimum3_f16 v5.l, v1.l, v2.l, s3
+// GFX12: v_minimum3_f16 v5.l, v1.l, v2.l, s3     ; encoding: [0x05,0x00,0x2f,0xd6,0x01,0x05,0x0e,0x00]
+
+v_minimum3_f16 v5.l, v255.l, s2, s105
+// GFX12: v_minimum3_f16 v5.l, v255.l, s2, s105   ; encoding: [0x05,0x00,0x2f,0xd6,0xff,0x05,0xa4,0x01]
 
-v_minimum3_f16 v5, v255, s2, s105
-// GFX12: v_minimum3_f16 v5, v255, s2, s105       ; encoding: [0x05,0x00,0x2f,0xd6,0xff,0x05,0xa4,0x01]
+v_minimum3_f16 v5.l, s1, v255.l, exec_hi
+// GFX12: v_minimum3_f16 v5.l, s1, v255.l, exec_hi ; encoding: [0x05,0x00,0x2f,0xd6,0x01,0xfe,0xff,0x01]
 
-v_minimum3_f16 v5, s1, v255, exec_hi
-// GFX12: v_minimum3_f16 v5, s1, v255, exec_hi    ; encoding: [0x05,0x00,0x2f,0xd6,0x01,0xfe,0xff,0x01]
+v_minimum3_f16 v5.l, s105, s105, exec_lo
+// GFX12: v_minimum3_f16 v5.l, s105, s105, exec_lo ; encoding: [0x05,0x00,0x2f,0xd6,0x69,0xd2,0xf8,0x01]
 
-v_minimum3_f16 v5, s105, s105, exec_lo
-// GFX12: v_minimum3_f16 v5, s105, s105, exec_lo  ; encoding: [0x05,0x00,0x2f,0xd6,0x69,0xd2,0xf8,0x01]
+v_minimum3_f16 v5.l, vcc_lo, ttmp15, v3.l
+// GFX12: v_minimum3_f16 v5.l, vcc_lo, ttmp15, v3.l ; encoding: [0x05,0x00,0x2f,0xd6,0x6a,0xf6,0x0c,0x04]
 
-v_minimum3_f16 v5, vcc_lo, ttmp15, v3
-// GFX12: v_minimum3_f16 v5, vcc_lo, ttmp15, v3   ; encoding: [0x05,0x00,0x2f,0xd6,0x6a,0xf6,0x0c,0x04]
+v_minimum3_f16 v5.l, vcc_hi, 0xaf12, v255.l
+// GFX12: v_minimum3_f16 v5.l, vcc_hi, 0xaf12, v255.l ; encoding: [0x05,0x00,0x2f,0xd6,0x6b,0xfe,0xfd,0x07,0x12,0xaf,0x00,0x00]
 
-v_minimum3_f16 v5, vcc_hi, 0xaf12, v255
-// GFX12: v_minimum3_f16 v5, vcc_hi, 0xaf12, v255 ; encoding: [0x05,0x00,0x2f,0xd6,0x6b,0xfe,0xfd,0x07,0x12,0xaf,0x00,0x00]
+v_minimum3_f16 v5.l, -|ttmp15|, -|src_scc|, -|ttmp15|
+// GFX12: v_minimum3_f16 v5.l, -|ttmp15|, -|src_scc|, -|ttmp15| ; encoding: [0x05,0x07,0x2f,0xd6,0x7b,0xfa,0xed,0xe1]
 
-v_minimum3_f16 v5, -|ttmp15|, -|src_scc|, -|ttmp15|
-// GFX12: v_minimum3_f16 v5, -|ttmp15|, -|src_scc|, -|ttmp15| ; encoding: [0x05,0x07,0x2f,0xd6,0x7b,0xfa,0xed,0xe1]
+v_minimum3_f16 v5.l, m0, 0.5, m0
+// GFX12: v_minimum3_f16 v5.l, m0, 0.5, m0        ; encoding: [0x05,0x00,0x2f,0xd6,0x7d,0xe0,0xf5,0x01]
 
-v_minimum3_f16 v5, m0, 0.5, m0
-// GFX12: v_minimum3_f16 v5, m0, 0.5, m0          ; encoding: [0x05,0x00,0x2f,0xd6,0x7d,0xe0,0xf5,0x01]
+v_minimum3_f16 v5.l, |exec_lo|, -1, vcc_hi
+// GFX12: v_minimum3_f16 v5.l, |exec_lo|, -1, vcc_hi ; encoding: [0x05,0x01,0x2f,0xd6,0x7e,0x82,0xad,0x01]
 
-v_minimum3_f16 v5, |exec_lo|, -1, vcc_hi
-// GFX12: v_minimum3_f16 v5, |exec_lo|, -1, vcc_hi ; encoding: [0x05,0x01,0x2f,0xd6,0x7e,0x82,0xad,0x01]
+v_minimum3_f16 v5.l, -|exec_hi|, null, -|vcc_lo|
+// GFX12: v_minimum3_f16 v5.l, -|exec_hi|, null, -|vcc_lo| ; encoding: [0x05,0x05,0x2f,0xd6,0x7f,0xf8,0xa8,0xa1]
 
-v_minimum3_f16 v5, -|exec_hi|, null, -|vcc_lo|
-// GFX12: v_minimum3_f16 v5, -|exec_hi|, null, -|vcc_lo| ; encoding: [0x05,0x05,0x2f,0xd6,0x7f,0xf8,0xa8,0xa1]
+v_minimum3_f16 v5.l, null, exec_lo, -|0xaf12|
+// GFX12: v_minimum3_f16 v5.l, null, exec_lo, -|0xaf12| ; encoding: [0x05,0x04,0x2f,0xd6,0x7c,0xfc,0xfc,0x83,0x12,0xaf,0x00,0x00]
 
-v_minimum3_f16 v5, null, exec_lo, -|0xaf12|
-// GFX12: v_minimum3_f16 v5, null, exec_lo, -|0xaf12| ; encoding: [0x05,0x04,0x2f,0xd6,0x7c,0xfc,0xfc,0x83,0x12,0xaf,0x00,0x00]
+v_minimum3_f16 v5.l, -1, -|exec_hi|, -|src_scc|
+// GFX12: v_minimum3_f16 v5.l, -1, -|exec_hi|, -|src_scc| ; encoding: [0x05,0x06,0x2f,0xd6,0xc1,0xfe,0xf4,0xc3]
 
-v_minimum3_f16 v5, -1, -|exec_hi|, -|src_scc|
-// GFX12: v_minimum3_f16 v5, -1, -|exec_hi|, -|src_scc| ; encoding: [0x05,0x06,0x2f,0xd6,0xc1,0xfe,0xf4,0xc3]
+v_minimum3_f16 v5.l, 0.5, -m0, 0.5
+// GFX12: v_minimum3_f16 v5.l, 0.5, -m0, 0.5      ; encoding: [0x05,0x00,0x2f,0xd6,0xf0,0xfa,0xc0,0x43]
 
-v_minimum3_f16 v5, 0.5, -m0, 0.5
-// GFX12: v_minimum3_f16 v5, 0.5, -m0, 0.5        ; encoding: [0x05,0x00,0x2f,0xd6,0xf0,0xfa,0xc0,0x43]
+v_minimum3_f16 v5.l, -src_scc, |vcc_lo|, -1
+// GFX12: v_minimum3_f16 v5.l, -src_scc, |vcc_lo|, -1 ; encoding: [0x05,0x02,0x2f,0xd6,0xfd,0xd4,0x04,0x23]
 
-v_minimum3_f16 v5, -src_scc, |vcc_lo|, -1
-// GFX12: v_minimum3_f16 v5, -src_scc, |vcc_lo|, -1 ; encoding: [0x05,0x02,0x2f,0xd6,0xfd,0xd4,0x04,0x23]
+v_minimum3_f16 v255.l, -|0xaf12|, -|vcc_hi|, null clamp
+// GFX12: v_minimum3_f16 v255.l, -|0xaf12|, -|vcc_hi|, null clamp ; encoding: [0xff,0x83,0x2f,0xd6,0xff,0xd6,0xf0,0x61,0x12,0xaf,0x00,0x00]
 
-v_minimum3_f16 v255, -|0xaf12|, -|vcc_hi|, null clamp
-// GFX12: v_minimum3_f16 v255, -|0xaf12|, -|vcc_hi|, null clamp ; encoding: [0xff,0x83,0x2f,0xd6,0xff,0xd6,0xf0,0x61,0x12,0xaf,0x00,0x00]
+v_minimum3_f16 v5.h, v1.h, v2.h, s3
+// GFX12: v_minimum3_f16 v5.h, v1.h, v2.h, s3 op_sel:[1,1,0,1] ; encoding: [0x05,0x58,0x2f,0xd6,0x01,0x05,0x0e,0x00]
 
-v_maximum3_f16 v5, v1, v2, s3
-// GFX12: v_maximum3_f16 v5, v1, v2, s3           ; encoding: [0x05,0x00,0x30,0xd6,0x01,0x05,0x0e,0x00]
+v_minimum3_f16 v5.l, v255.h, s2, s105
+// GFX12: v_minimum3_f16 v5.l, v255.h, s2, s105 op_sel:[1,0,0,0] ; encoding: [0x05,0x08,0x2f,0xd6,0xff,0x05,0xa4,0x01]
 
-v_maximum3_f16 v5, v255, s2, s105
-// GFX12: v_maximum3_f16 v5, v255, s2, s105       ; encoding: [0x05,0x00,0x30,0xd6,0xff,0x05,0xa4,0x01]
+v_minimum3_f16 v5.h, -src_scc, |vcc_hi|, -1
+// GFX12: v_minimum3_f16 v5.h, -src_scc, |vcc_hi|, -1 op_sel:[0,0,0,1] ; encoding: [0x05,0x42,0x2f,0xd6,0xfd,0xd6,0x04,0x23]
 
-v_maximum3_f16 v5, s1, v255, exec_hi
-// GFX12: v_maximum3_f16 v5, s1, v255, exec_hi    ; encoding: [0x05,0x00,0x30,0xd6,0x01,0xfe,0xff,0x01]
+v_minimum3_f16 v5.h, vcc_hi, 0xaf12, v255.h
+// GFX12: v_minimum3_f16 v5.h, vcc_hi, 0xaf12, v255.h op_sel:[0,0,1,1] ; encoding: [0x05,0x60,0x2f,0xd6,0x6b,0xfe,0xfd,0x07,0x12,0xaf,0x00,0x00]
 
-v_maximum3_f16 v5, s105, s105, exec_lo
-// GFX12: v_maximum3_f16 v5, s105, s105, exec_lo  ; encoding: [0x05,0x00,0x30,0xd6,0x69,0xd2,0xf8,0x01]
+v_maximum3_f16 v5.l, v1.l, v2.l, s3
+// GFX12: v_maximum3_f16 v5.l, v1.l, v2.l, s3     ; encoding: [0x05,0x00,0x30,0xd6,0x01,0x05,0x0e,0x00]
 
-v_maximum3_f16 v5, vcc_lo, ttmp15, v3
-// GFX12: v_maximum3_f16 v5, vcc_lo, ttmp15, v3   ; encoding: [0x05,0x00,0x30,0xd6,0x6a,0xf6,0x0c,0x04]
+v_maximum3_f16 v5.l, v255.l, s2, s105
+// GFX12: v_maximum3_f16 v5.l, v255.l, s2, s105   ; encoding: [0x05,0x00,0x30,0xd6,0xff,0x05,0xa4,0x01]
 
-v_maximum3_f16 v5, vcc_hi, 0xaf12, v255
-// GFX12: v_maximum3_f16 v5, vcc_hi, 0xaf12, v255 ; encoding: [0x05,0x00,0x30,0xd6,0x6b,0xfe,0xfd,0x07,0x12,0xaf,0x00,0x00]
+v_maximum3_f16 v5.l, s1, v255.l, exec_hi
+// GFX12: v_maximum3_f16 v5.l, s1, v255.l, exec_hi ; encoding: [0x05,0x00,0x30,0xd6,0x01,0xfe,0xff,0x01]
 
-v_maximum3_f16 v5, -|ttmp15|, -|src_scc|, -|ttmp15|
-// GFX12: v_maximum3_f16 v5, -|ttmp15|, -|src_scc|, -|ttmp15| ; encoding: [0x05,0x07,0x30,0xd6,0x7b,0xfa,0xed,0xe1]
+v_maximum3_f16 v5.l, s105, s105, exec_lo
+// GFX12: v_maximum3_f16 v5.l, s105, s105, exec_lo ; encoding: [0x05,0x00,0x30,0xd6,0x69,0xd2,0xf8,0x01]
 
-v_maximum3_f16 v5, m0, 0.5, m0
-// GFX12: v_maximum3_f16 v5, m0, 0.5, m0          ; encoding: [0x05,0x00,0x30,0xd6,0x7d,0xe0,0xf5,0x01]
+v_maximum3_f16 v5.l, vcc_lo, ttmp15, v3.l
+// GFX12: v_maximum3_f16 v5.l, vcc_lo, ttmp15, v3.l ; encoding: [0x05,0x00,0x30,0xd6,0x6a,0xf6,0x0c,0x04]
 
-v_maximum3_f16 v5, |exec_lo|, -1, vcc_hi
-// GFX12: v_maximum3_f16 v5, |exec_lo|, -1, vcc_hi ; encoding: [0x05,0x01,0x30,0xd6,0x7e,0x82,0xad,0x01]
+v_maximum3_f16 v5.l, vcc_hi, 0xaf12, v255.l
+// GFX12: v_maximum3_f16 v5.l, vcc_hi, 0xaf12, v255.l ; encoding: [0x05,0x00,0x30,0xd6,0x6b,0xfe,0xfd,0x07,0x12,0xaf,0x00,0x00]
 
-v_maximum3_f16 v5, -|exec_hi|, null, -|vcc_lo|
-// GFX12: v_maximum3_f16 v5, -|exec_hi|, null, -|vcc_lo| ; encoding: [0x05,0x05,0x30,0xd6,0x7f,0xf8,0xa8,0xa1]
+v_maximum3_f16 v5.l, -|ttmp15|, -|src_scc|, -|ttmp15|
+// GFX12: v_maximum3_f16 v5.l, -|ttmp15|, -|src_scc|, -|ttmp15| ; encoding: [0x05,0x07,0x30,0xd6,0x7b,0xfa,0xed,0xe1]
 
-v_maximum3_f16 v5, null, exec_lo, -|0xaf12|
-// GFX12: v_maximum3_f16 v5, null, exec_lo, -|0xaf12| ; encoding: [0x05,0x04,0x30,0xd6,0x7c,0xfc,0xfc,0x83,0x12,0xaf,0x00,0x00]
+v_maximum3_f16 v5.l, m0, 0.5, m0
+// GFX12: v_maximum3_f16 v5.l, m0, 0.5, m0        ; encoding: [0x05,0x00,0x30,0xd6,0x7d,0xe0,0xf5,0x01]
 
-v_maximum3_f16 v5, -1, -|exec_hi|, -|src_scc|
-// GFX12: v_maximum3_f16 v5, -1, -|exec_hi|, -|src_scc| ; encoding: [0x05,0x06,0x30,0xd6,0xc1,0xfe,0xf4,0xc3]
+v_maximum3_f16 v5.l, |exec_lo|, -1, vcc_hi
+// GFX12: v_maximum3_f16 v5.l, |exec_lo|, -1, vcc_hi ; encoding: [0x05,0x01,0x30,0xd6,0x7e,0x82,0xad,0x01]
 
-v_maximum3_f16 v5, 0.5, -m0, 0.5
-// GFX12: v_maximum3_f16 v5, 0.5, -m0, 0.5        ; encoding: [0x05,0x00,0x30,0xd6,0xf0,0xfa,0xc0,0x43]
+v_maximum3_f16 v5.l, -|exec_hi|, null, -|vcc_lo|
+// GFX12: v_maximum3_f16 v5.l, -|exec_hi|, null, -|vcc_lo| ; encoding: [0x05,0x05,0x30,0xd6,0x7f,0xf8,0xa8,0xa1]
 
-v_maximum3_f16 v5, -src_scc, |vcc_lo|, -1
-// GFX12: v_maximum3_f16 v5, -src_scc, |vcc_lo|, -1 ; encoding: [0x05,0x02,0x30,0xd6,0xfd,0xd4,0x04,0x23]
+v_maximum3_f16 v5.l, null, exec_lo, -|0xaf12|
+// GFX12: v_maximum3_f16 v5.l, null, exec_lo, -|0xaf12| ; encoding: [0x05,0x04,0x30,0xd6,0x7c,0xfc,0xfc,0x83,0x12,0xaf,0x00,0x00]
+
+v_maximum3_f16 v5.l, -1, -|exec_hi|, -|src_scc|
+// GFX12: v_maximum3_f16 v5.l, -1, -|exec_hi|, -|src_scc| ; encoding: [0x05,0x06,0x30,0xd6,0xc1,0xfe,0xf4,0xc3]
+
+v_maximum3_f16 v5.l, 0.5, -m0, 0.5
+// GFX12: v_maximum3_f16 v5.l, 0.5, -m0, 0.5      ; encoding: [0x05,0x00,0x30,0xd6,0xf0,0xfa,0xc0,0x43]
+
+v_maximum3_f16 v5.l, -src_scc, |vcc_lo|, -1
+// GFX12: v_maximum3_f16 v5.l, -src_scc, |vcc_lo|, -1 ; encoding: [0x05,0x02,0x30,0xd6,0xfd,0xd4,0x04,0x23]
+
+v_maximum3_f16 v5.h, v1.h, v2.h, s3
+// GFX12: v_maximum3_f16 v5.h, v1.h, v2.h, s3 op_sel:[1,1,0,1] ; encoding: [0x05,0x58,0x30,0xd6,0x01,0x05,0x0e,0x00]
+
+v_maximum3_f16 v5.l, v255.h, s2, s105
+// GFX12: v_maximum3_f16 v5.l, v255.h, s2, s105 op_sel:[1,0,0,0] ; encoding: [0x05,0x08,0x30,0xd6,0xff,0x05,0xa4,0x01]
+
+v_maximum3_f16 v5.h, -src_scc, |vcc_hi|, -1
+// GFX12: v_maximum3_f16 v5.h, -src_scc, |vcc_hi|, -1 op_sel:[0,0,0,1] ; encoding: [0x05,0x42,0x30,0xd6,0xfd,0xd6,0x04,0x23]
+
+v_maximum3_f16 v5.h, vcc_hi, 0xaf12, v255.h
+// GFX12: v_maximum3_f16 v5.h, vcc_hi, 0xaf12, v255.h op_sel:[0,0,1,1] ; encoding: [0x05,0x60,0x30,0xd6,0x6b,0xfe,0xfd,0x07,0x12,0xaf,0x00,0x00]
 
 v_maximumminimum_f32 v5, v1, v2, s3
 // GFX12: v_maximumminimum_f32 v5, v1, v2, s3     ; encoding: [0x05,0x00,0x6d,0xd6,0x01,0x05,0x0e,0x00]
@@ -6898,92 +6967,116 @@ v_minimummaximum_f32 v5, -src_scc, |vcc_lo|, -1 mul:4
 v_minimummaximum_f32 v255, -|0xaf123456|, -|vcc_hi|, null clamp div:2
 // GFX12: v_minimummaximum_f32 v255, -|0xaf123456|, -|vcc_hi|, null clamp div:2 ; encoding: [0xff,0x83,0x6c,0xd6,0xff,0xd6,0xf0,0x79,0x56,0x34,0x12,0xaf]
 
-v_maximumminimum_f16 v5, v1, v2, s3
-// GFX12: v_maximumminimum_f16 v5, v1, v2, s3     ; encoding: [0x05,0x00,0x6f,0xd6,0x01,0x05,0x0e,0x00]
+v_maximumminimum_f16 v5.l, v1.l, v2.l, s3
+// GFX12: v_maximumminimum_f16 v5.l, v1.l, v2.l, s3 ; encoding: [0x05,0x00,0x6f,0xd6,0x01,0x05,0x0e,0x00]
+
+v_maximumminimum_f16 v5.l, v255.l, s2, s105
+// GFX12: v_maximumminimum_f16 v5.l, v255.l, s2, s105 ; encoding: [0x05,0x00,0x6f,0xd6,0xff,0x05,0xa4,0x01]
+
+v_maximumminimum_f16 v5.l, s1, v255.l, exec_hi
+// GFX12: v_maximumminimum_f16 v5.l, s1, v255.l, exec_hi ; encoding: [0x05,0x00,0x6f,0xd6,0x01,0xfe,0xff,0x01]
+
+v_maximumminimum_f16 v5.l, s105, s105, exec_lo
+// GFX12: v_maximumminimum_f16 v5.l, s105, s105, exec_lo ; encoding: [0x05,0x00,0x6f,0xd6,0x69,0xd2,0xf8,0x01]
+
+v_maximumminimum_f16 v5.l, vcc_lo, ttmp15, v3.l
+// GFX12: v_maximumminimum_f16 v5.l, vcc_lo, ttmp15, v3.l ; encoding: [0x05,0x00,0x6f,0xd6,0x6a,0xf6,0x0c,0x04]
+
+v_maximumminimum_f16 v5.l, vcc_hi, 0xaf12, v255.l
+// GFX12: v_maximumminimum_f16 v5.l, vcc_hi, 0xaf12, v255.l ; encoding: [0x05,0x00,0x6f,0xd6,0x6b,0xfe,0xfd,0x07,0x12,0xaf,0x00,0x00]
+
+v_maximumminimum_f16 v5.l, -|ttmp15|, -|src_scc|, -|ttmp15|
+// GFX12: v_maximumminimum_f16 v5.l, -|ttmp15|, -|src_scc|, -|ttmp15| ; encoding: [0x05,0x07,0x6f,0xd6,0x7b,0xfa,0xed,0xe1]
+
+v_maximumminimum_f16 v5.l, m0, 0.5, m0
+// GFX12: v_maximumminimum_f16 v5.l, m0, 0.5, m0  ; encoding: [0x05,0x00,0x6f,0xd6,0x7d,0xe0,0xf5,0x01]
+
+v_maximumminimum_f16 v5.l, |exec_lo|, -1, vcc_hi
+// GFX12: v_maximumminimum_f16 v5.l, |exec_lo|, -1, vcc_hi ; encoding: [0x05,0x01,0x6f,0xd6,0x7e,0x82,0xad,0x01]
 
-v_maximumminimum_f16 v5, v255, s2, s105
-// GFX12: v_maximumminimum_f16 v5, v255, s2, s105 ; encoding: [0x05,0x00,0x6f,0xd6,0xff,0x05,0xa4,0x01]
+v_maximumminimum_f16 v5.l, -|exec_hi|, null, -|vcc_lo|
+// GFX12: v_maximumminimum_f16 v5.l, -|exec_hi|, null, -|vcc_lo| ; encoding: [0x05,0x05,0x6f,0xd6,0x7f,0xf8,0xa8,0xa1]
 
-v_maximumminimum_f16 v5, s1, v255, exec_hi
-// GFX12: v_maximumminimum_f16 v5, s1, v255, exec_hi ; encoding: [0x05,0x00,0x6f,0xd6,0x01,0xfe,0xff,0x01]
+v_maximumminimum_f16 v5.l, null, exec_lo, -|0xaf12|
+// GFX12: v_maximumminimum_f16 v5.l, null, exec_lo, -|0xaf12| ; encoding: [0x05,0x04,0x6f,0xd6,0x7c,0xfc,0xfc,0x83,0x12,0xaf,0x00,0x00]
 
-v_maximumminimum_f16 v5, s105, s105, exec_lo
-// GFX12: v_maximumminimum_f16 v5, s105, s105, exec_lo ; encoding: [0x05,0x00,0x6f,0xd6,0x69,0xd2,0xf8,0x01]
+v_maximumminimum_f16 v5.l, -1, -|exec_hi|, -|src_scc|
+// GFX12: v_maximumminimum_f16 v5.l, -1, -|exec_hi|, -|src_scc| ; encoding: [0x05,0x06,0x6f,0xd6,0xc1,0xfe,0xf4,0xc3]
 
-v_maximumminimum_f16 v5, vcc_lo, ttmp15, v3
-// GFX12: v_maximumminimum_f16 v5, vcc_lo, ttmp15, v3 ; encoding: [0x05,0x00,0x6f,0xd6,0x6a,0xf6,0x0c,0x04]
+v_maximumminimum_f16 v5.l, 0.5, -m0, 0.5
+// GFX12: v_maximumminimum_f16 v5.l, 0.5, -m0, 0.5 ; encoding: [0x05,0x00,0x6f,0xd6,0xf0,0xfa,0xc0,0x43]
 
-v_maximumminimum_f16 v5, vcc_hi, 0xaf12, v255
-// GFX12: v_maximumminimum_f16 v5, vcc_hi, 0xaf12, v255 ; encoding: [0x05,0x00,0x6f,0xd6,0x6b,0xfe,0xfd,0x07,0x12,0xaf,0x00,0x00]
+v_maximumminimum_f16 v5.l, -src_scc, |vcc_lo|, -1
+// GFX12: v_maximumminimum_f16 v5.l, -src_scc, |vcc_lo|, -1 ; encoding: [0x05,0x02,0x6f,0xd6,0xfd,0xd4,0x04,0x23]
 
-v_maximumminimum_f16 v5, -|ttmp15|, -|src_scc|, -|ttmp15|
-// GFX12: v_maximumminimum_f16 v5, -|ttmp15|, -|src_scc|, -|ttmp15| ; encoding: [0x05,0x07,0x6f,0xd6,0x7b,0xfa,0xed,0xe1]
+v_maximumminimum_f16 v255.l, -|0xaf12|, -|vcc_hi|, null clamp
+// GFX12: v_maximumminimum_f16 v255.l, -|0xaf12|, -|vcc_hi|, null clamp ; encoding: [0xff,0x83,0x6f,0xd6,0xff,0xd6,0xf0,0x61,0x12,0xaf,0x00,0x00]
 
-v_maximumminimum_f16 v5, m0, 0.5, m0
-// GFX12: v_maximumminimum_f16 v5, m0, 0.5, m0    ; encoding: [0x05,0x00,0x6f,0xd6,0x7d,0xe0,0xf5,0x01]
+v_maximumminimum_f16 v5.h, v1.h, v2.h, s3
+// GFX12: v_maximumminimum_f16 v5.h, v1.h, v2.h, s3 op_sel:[1,1,0,1] ; encoding: [0x05,0x58,0x6f,0xd6,0x01,0x05,0x0e,0x00]
 
-v_maximumminimum_f16 v5, |exec_lo|, -1, vcc_hi
-// GFX12: v_maximumminimum_f16 v5, |exec_lo|, -1, vcc_hi ; encoding: [0x05,0x01,0x6f,0xd6,0x7e,0x82,0xad,0x01]
+v_maximumminimum_f16 v5.h, s1, v255.h, exec_hi
+// GFX12: v_maximumminimum_f16 v5.h, s1, v255.h, exec_hi op_sel:[0,1,0,1] ; encoding: [0x05,0x50,0x6f,0xd6,0x01,0xfe,0xff,0x01]
 
-v_maximumminimum_f16 v5, -|exec_hi|, null, -|vcc_lo|
-// GFX12: v_maximumminimum_f16 v5, -|exec_hi|, null, -|vcc_lo| ; encoding: [0x05,0x05,0x6f,0xd6,0x7f,0xf8,0xa8,0xa1]
+v_maximumminimum_f16 v5.h, -src_scc, |vcc_hi|, -1
+// GFX12: v_maximumminimum_f16 v5.h, -src_scc, |vcc_hi|, -1 op_sel:[0,0,0,1] ; encoding: [0x05,0x42,0x6f,0xd6,0xfd,0xd6,0x04,0x23]
 
-v_maximumminimum_f16 v5, null, exec_lo, -|0xaf12|
-// GFX12: v_maximumminimum_f16 v5, null, exec_lo, -|0xaf12| ; encoding: [0x05,0x04,0x6f,0xd6,0x7c,0xfc,0xfc,0x83,0x12,0xaf,0x00,0x00]
+v_maximumminimum_f16 v255.h, -|0xaf12|, -|vcc_hi|, null clamp
+// GFX12: v_maximumminimum_f16 v255.h, -|0xaf12|, -|vcc_hi|, null op_sel:[0,0,0,1] clamp ; encoding: [0xff,0xc3,0x6f,0xd6,0xff,0xd6,0xf0,0x61,0x12,0xaf,0x00,0x00]
 
-v_maximumminimum_f16 v5, -1, -|exec_hi|, -|src_scc|
-// GFX12: v_maximumminimum_f16 v5, -1, -|exec_hi|, -|src_scc| ; encoding: [0x05,0x06,0x6f,0xd6,0xc1,0xfe,0xf4,0xc3]
+v_minimummaximum_f16 v5.l, v1.l, v2.l, s3
+// GFX12: v_minimummaximum_f16 v5.l, v1.l, v2.l, s3 ; encoding: [0x05,0x00,0x6e,0xd6,0x01,0x05,0x0e,0x00]
 
-v_maximumminimum_f16 v5, 0.5, -m0, 0.5
-// GFX12: v_maximumminimum_f16 v5, 0.5, -m0, 0.5  ; encoding: [0x05,0x00,0x6f,0xd6,0xf0,0xfa,0xc0,0x43]
+v_minimummaximum_f16 v5.l, v255.l, s2, s105
+// GFX12: v_minimummaximum_f16 v5.l, v255.l, s2, s105 ; encoding: [0x05,0x00,0x6e,0xd6,0xff,0x05,0xa4,0x01]
 
-v_maximumminimum_f16 v5, -src_scc, |vcc_lo|, -1
-// GFX12: v_maximumminimum_f16 v5, -src_scc, |vcc_lo|, -1 ; encoding: [0x05,0x02,0x6f,0xd6,0xfd,0xd4,0x04,0x23]
+v_minimummaximum_f16 v5.l, s1, v255.l, exec_hi
+// GFX12: v_minimummaximum_f16 v5.l, s1, v255.l, exec_hi ; encoding: [0x05,0x00,0x6e,0xd6,0x01,0xfe,0xff,0x01]
 
-v_maximumminimum_f16 v255, -|0xaf12|, -|vcc_hi|, null clamp
-// GFX12: v_maximumminimum_f16 v255, -|0xaf12|, -|vcc_hi|, null clamp ; encoding: [0xff,0x83,0x6f,0xd6,0xff,0xd6,0xf0,0x61,0x12,0xaf,0x00,0x00]
+v_minimummaximum_f16 v5.l, s105, s105, exec_lo
+// GFX12: v_minimummaximum_f16 v5.l, s105, s105, exec_lo ; encoding: [0x05,0x00,0x6e,0xd6,0x69,0xd2,0xf8,0x01]
 
-v_minimummaximum_f16 v5, v1, v2, s3
-// GFX12: v_minimummaximum_f16 v5, v1, v2, s3     ; encoding: [0x05,0x00,0x6e,0xd6,0x01,0x05,0x0e,0x00]
+v_minimummaximum_f16 v5.l, vcc_lo, ttmp15, v3.l
+// GFX12: v_minimummaximum_f16 v5.l, vcc_lo, ttmp15, v3.l ; encoding: [0x05,0x00,0x6e,0xd6,0x6a,0xf6,0x0c,0x04]
 
-v_minimummaximum_f16 v5, v255, s2, s105
-// GFX12: v_minimummaximum_f16 v5, v255, s2, s105 ; encoding: [0x05,0x00,0x6e,0xd6,0xff,0x05,0xa4,0x01]
+v_minimummaximum_f16 v5.l, vcc_hi, 0xaf12, v255.l
+// GFX12: v_minimummaximum_f16 v5.l, vcc_hi, 0xaf12, v255.l ; encoding: [0x05,0x00,0x6e,0xd6,0x6b,0xfe,0xfd,0x07,0x12,0xaf,0x00,0x00]
 
-v_minimummaximum_f16 v5, s1, v255, exec_hi
-// GFX12: v_minimummaximum_f16 v5, s1, v255, exec_hi ; encoding: [0x05,0x00,0x6e,0xd6,0x01,0xfe,0xff,0x01]
+v_minimummaximum_f16 v5.l, -|ttmp15|, -|src_scc|, -|ttmp15|
+// GFX12: v_minimummaximum_f16 v5.l, -|ttmp15|, -|src_scc|, -|ttmp15| ; encoding: [0x05,0x07,0x6e,0xd6,0x7b,0xfa,0xed,0xe1]
 
-v_minimummaximum_f16 v5, s105, s105, exec_lo
-// GFX12: v_minimummaximum_f16 v5, s105, s105, exec_lo ; encoding: [0x05,0x00,0x6e,0xd6,0x69,0xd2,0xf8,0x01]
+v_minimummaximum_f16 v5.l, m0, 0.5, m0
+// GFX12: v_minimummaximum_f16 v5.l, m0, 0.5, m0  ; encoding: [0x05,0x00,0x6e,0xd6,0x7d,0xe0,0xf5,0x01]
 
-v_minimummaximum_f16 v5, vcc_lo, ttmp15, v3
-// GFX12: v_minimummaximum_f16 v5, vcc_lo, ttmp15, v3 ; encoding: [0x05,0x00,0x6e,0xd6,0x6a,0xf6,0x0c,0x04]
+v_minimummaximum_f16 v5.l, |exec_lo|, -1, vcc_hi
+// GFX12: v_minimummaximum_f16 v5.l, |exec_lo|, -1, vcc_hi ; encoding: [0x05,0x01,0x6e,0xd6,0x7e,0x82,0xad,0x01]
 
-v_minimummaximum_f16 v5, vcc_hi, 0xaf12, v255
-// GFX12: v_minimummaximum_f16 v5, vcc_hi, 0xaf12, v255 ; encoding: [0x05,0x00,0x6e,0xd6,0x6b,0xfe,0xfd,0x07,0x12,0xaf,0x00,0x00]
+v_minimummaximum_f16 v5.l, -|exec_hi|, null, -|vcc_lo|
+// GFX12: v_minimummaximum_f16 v5.l, -|exec_hi|, null, -|vcc_lo| ; encoding: [0x05,0x05,0x6e,0xd6,0x7f,0xf8,0xa8,0xa1]
 
-v_minimummaximum_f16 v5, -|ttmp15|, -|src_scc|, -|ttmp15|
-// GFX12: v_minimummaximum_f16 v5, -|ttmp15|, -|src_scc|, -|ttmp15| ; encoding: [0x05,0x07,0x6e,0xd6,0x7b,0xfa,0xed,0xe1]
+v_minimummaximum_f16 v5.l, null, exec_lo, -|0xaf12|
+// GFX12: v_minimummaximum_f16 v5.l, null, exec_lo, -|0xaf12| ; encoding: [0x05,0x04,0x6e,0xd6,0x7c,0xfc,0xfc,0x83,0x12,0xaf,0x00,0x00]
 
-v_minimummaximum_f16 v5, m0, 0.5, m0
-// GFX12: v_minimummaximum_f16 v5, m0, 0.5, m0    ; encoding: [0x05,0x00,0x6e,0xd6,0x7d,0xe0,0xf5,0x01]
+v_minimummaximum_f16 v5.l, -1, -|exec_hi|, -|src_scc|
+// GFX12: v_minimummaximum_f16 v5.l, -1, -|exec_hi|, -|src_scc| ; encoding: [0x05,0x06,0x6e,0xd6,0xc1,0xfe,0xf4,0xc3]
 
-v_minimummaximum_f16 v5, |exec_lo|, -1, vcc_hi
-// GFX12: v_minimummaximum_f16 v5, |exec_lo|, -1, vcc_hi ; encoding: [0x05,0x01,0x6e,0xd6,0x7e,0x82,0xad,0x01]
+v_minimummaximum_f16 v5.l, 0.5, -m0, 0.5
+// GFX12: v_minimummaximum_f16 v5.l, 0.5, -m0, 0.5 ; encoding: [0x05,0x00,0x6e,0xd6,0xf0,0xfa,0xc0,0x43]
 
-v_minimummaximum_f16 v5, -|exec_hi|, null, -|vcc_lo|
-// GFX12: v_minimummaximum_f16 v5, -|exec_hi|, null, -|vcc_lo| ; encoding: [0x05,0x05,0x6e,0xd6,0x7f,0xf8,0xa8,0xa1]
+v_minimummaximum_f16 v5.l, -src_scc, |vcc_lo|, -1
+// GFX12: v_minimummaximum_f16 v5.l, -src_scc, |vcc_lo|, -1 ; encoding: [0x05,0x02,0x6e,0xd6,0xfd,0xd4,0x04,0x23]
 
-v_minimummaximum_f16 v5, null, exec_lo, -|0xaf12|
-// GFX12: v_minimummaximum_f16 v5, null, exec_lo, -|0xaf12| ; encoding: [0x05,0x04,0x6e,0xd6,0x7c,0xfc,0xfc,0x83,0x12,0xaf,0x00,0x00]
+v_minimummaximum_f16 v5.h, v1.h, v2.h, s3
+// GFX12: v_minimummaximum_f16 v5.h, v1.h, v2.h, s3 op_sel:[1,1,0,1] ; encoding: [0x05,0x58,0x6e,0xd6,0x01,0x05,0x0e,0x00]
 
-v_minimummaximum_f16 v5, -1, -|exec_hi|, -|src_scc|
-// GFX12: v_minimummaximum_f16 v5, -1, -|exec_hi|, -|src_scc| ; encoding: [0x05,0x06,0x6e,0xd6,0xc1,0xfe,0xf4,0xc3]
+v_minimummaximum_f16 v5.h, s1, v255.h, exec_hi
+// GFX12: v_minimummaximum_f16 v5.h, s1, v255.h, exec_hi op_sel:[0,1,0,1] ; encoding: [0x05,0x50,0x6e,0xd6,0x01,0xfe,0xff,0x01]
 
-v_minimummaximum_f16 v5, 0.5, -m0, 0.5
-// GFX12: v_minimummaximum_f16 v5, 0.5, -m0, 0.5  ; encoding: [0x05,0x00,0x6e,0xd6,0xf0,0xfa,0xc0,0x43]
+v_minimummaximum_f16 v5.h, -src_scc, |vcc_hi|, -1
+// GFX12: v_minimummaximum_f16 v5.h, -src_scc, |vcc_hi|, -1 op_sel:[0,0,0,1] ; encoding: [0x05,0x42,0x6e,0xd6,0xfd,0xd6,0x04,0x23]
 
-v_minimummaximum_f16 v5, -src_scc, |vcc_lo|, -1
-// GFX12: v_minimummaximum_f16 v5, -src_scc, |vcc_lo|, -1 ; encoding: [0x05,0x02,0x6e,0xd6,0xfd,0xd4,0x04,0x23]
+v_minimummaximum_f16 v255.h, -|0xaf12|, -|vcc_hi|, null clamp
+// GFX12: v_minimummaximum_f16 v255.h, -|0xaf12|, -|vcc_hi|, null op_sel:[0,0,0,1] clamp ; encoding: [0xff,0xc3,0x6e,0xd6,0xff,0xd6,0xf0,0x61,0x12,0xaf,0x00,0x00]
 
 v_s_exp_f32 s5, s1
 // GFX12: v_s_exp_f32 s5, s1                      ; encoding: [0x05,0x00,0x80,0xd6,0x01,0x00,0x00,0x00]
diff --git a/llvm/test/MC/AMDGPU/gfx12_asm_vop3_dpp16.s b/llvm/test/MC/AMDGPU/gfx12_asm_vop3_dpp16.s
index 78ce7451c1ba7..08be9225890c2 100644
--- a/llvm/test/MC/AMDGPU/gfx12_asm_vop3_dpp16.s
+++ b/llvm/test/MC/AMDGPU/gfx12_asm_vop3_dpp16.s
@@ -485,6 +485,9 @@ v_alignbyte_b32_e64_dpp v5, v1, v2, -1 row_xmask:0 row_mask:0x1 bank_mask:0x3 bo
 v_alignbyte_b32_e64_dpp v255, v255, v255, src_scc row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
 // GFX12: v_alignbyte_b32_e64_dpp v255, v255, v255, src_scc row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xff,0x00,0x17,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x05,0x30]
 
+v_alignbyte_b32_e64_dpp v5, v1, v2, v255.h row_mirror
+// GFX12: v_alignbyte_b32_e64_dpp v5, v1, v2, v255.h op_sel:[0,0,1,0] row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x20,0x17,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x40,0x01,0xff]
+
 v_and_b16_e64_dpp v5.l, v1.l, v2.l quad_perm:[3,2,1,0]
 // GFX12: v_and_b16_e64_dpp v5.l, v1.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x62,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
 
@@ -2259,53 +2262,59 @@ v_mad_i16_e64_dpp v5.l, v1.l, v2.h, -1 row_xmask:0 row_mask:0x1 bank_mask:0x3 bo
 v_mad_i16_e64_dpp v255.h, v255.l, v255.l, src_scc clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
 // GFX12: v_mad_i16_e64_dpp v255.h, v255.l, v255.l, src_scc op_sel:[0,0,0,1] clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xff,0xc0,0x53,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x05,0x30]
 
-v_mad_i32_i16_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0]
-// GFX12: v_mad_i32_i16_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5a,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
+v_mad_i32_i16_e64_dpp v5, v1.l, v2.l, v3 quad_perm:[3,2,1,0]
+// GFX12: v_mad_i32_i16_e64_dpp v5, v1.l, v2.l, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5a,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
+
+v_mad_i32_i16_e64_dpp v5, v1.l, s2, v3 quad_perm:[3,2,1,0]
+// GFX12: v_mad_i32_i16_e64_dpp v5, v1.l, s2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5a,0xd6,0xfa,0x04,0x0c,0x04,0x01,0x1b,0x00,0xff]
 
-v_mad_i32_i16_e64_dpp v5, v1, s2, v3 quad_perm:[3,2,1,0]
-// GFX12: v_mad_i32_i16_e64_dpp v5, v1, s2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5a,0xd6,0xfa,0x04,0x0c,0x04,0x01,0x1b,0x00,0xff]
+v_mad_i32_i16_e64_dpp v5, v1.l, 10, v3 quad_perm:[3,2,1,0]
+// GFX12: v_mad_i32_i16_e64_dpp v5, v1.l, 10, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5a,0xd6,0xfa,0x14,0x0d,0x04,0x01,0x1b,0x00,0xff]
 
-v_mad_i32_i16_e64_dpp v5, v1, 10, v3 quad_perm:[3,2,1,0]
-// GFX12: v_mad_i32_i16_e64_dpp v5, v1, 10, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5a,0xd6,0xfa,0x14,0x0d,0x04,0x01,0x1b,0x00,0xff]
+v_mad_i32_i16_e64_dpp v5, v1.l, v2.l, v3 quad_perm:[0,1,2,3]
+// GFX12: v_mad_i32_i16_e64_dpp v5, v1.l, v2.l, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5a,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
 
-v_mad_i32_i16_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3]
-// GFX12: v_mad_i32_i16_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5a,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
+v_mad_i32_i16_e64_dpp v5, v1.l, v2.l, v3 row_mirror
+// GFX12: v_mad_i32_i16_e64_dpp v5, v1.l, v2.l, v3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5a,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
 
-v_mad_i32_i16_e64_dpp v5, v1, v2, v3 row_mirror
-// GFX12: v_mad_i32_i16_e64_dpp v5, v1, v2, v3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5a,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
+v_mad_i32_i16_e64_dpp v5, v1.l, v2.l, v255 row_half_mirror
+// GFX12: v_mad_i32_i16_e64_dpp v5, v1.l, v2.l, v255 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5a,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff]
 
-v_mad_i32_i16_e64_dpp v5, v1, v2, v255 row_half_mirror
-// GFX12: v_mad_i32_i16_e64_dpp v5, v1, v2, v255 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5a,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff]
+v_mad_i32_i16_e64_dpp v5, v1.l, v2.l, s105 row_shl:1
+// GFX12: v_mad_i32_i16_e64_dpp v5, v1.l, v2.l, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5a,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
 
-v_mad_i32_i16_e64_dpp v5, v1, v2, s105 row_shl:1
-// GFX12: v_mad_i32_i16_e64_dpp v5, v1, v2, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5a,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
+v_mad_i32_i16_e64_dpp v5, v1.l, v2.l, vcc_hi row_shl:15
+// GFX12: v_mad_i32_i16_e64_dpp v5, v1.l, v2.l, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5a,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
 
-v_mad_i32_i16_e64_dpp v5, v1, v2, vcc_hi row_shl:15
-// GFX12: v_mad_i32_i16_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5a,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
+v_mad_i32_i16_e64_dpp v5, v1.l, v2.l, vcc_lo row_shr:1
+// GFX12: v_mad_i32_i16_e64_dpp v5, v1.l, v2.l, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5a,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
 
-v_mad_i32_i16_e64_dpp v5, v1, v2, vcc_lo row_shr:1
-// GFX12: v_mad_i32_i16_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5a,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
+v_mad_i32_i16_e64_dpp v5, v1.l, v2.l, ttmp15 row_shr:15
+// GFX12: v_mad_i32_i16_e64_dpp v5, v1.l, v2.l, ttmp15 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5a,0xd6,0xfa,0x04,0xee,0x01,0x01,0x1f,0x01,0xff]
 
-v_mad_i32_i16_e64_dpp v5, v1, v2, ttmp15 row_shr:15
-// GFX12: v_mad_i32_i16_e64_dpp v5, v1, v2, ttmp15 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5a,0xd6,0xfa,0x04,0xee,0x01,0x01,0x1f,0x01,0xff]
+v_mad_i32_i16_e64_dpp v5, v1.l, v2.l, exec_hi row_ror:1
+// GFX12: v_mad_i32_i16_e64_dpp v5, v1.l, v2.l, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5a,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x21,0x01,0xff]
 
-v_mad_i32_i16_e64_dpp v5, v1, v2, exec_hi row_ror:1
-// GFX12: v_mad_i32_i16_e64_dpp v5, v1, v2, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5a,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x21,0x01,0xff]
+v_mad_i32_i16_e64_dpp v5, v1.l, v2.l, exec_lo row_ror:15
+// GFX12: v_mad_i32_i16_e64_dpp v5, v1.l, v2.l, exec_lo row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5a,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x2f,0x01,0xff]
 
-v_mad_i32_i16_e64_dpp v5, v1, v2, exec_lo row_ror:15
-// GFX12: v_mad_i32_i16_e64_dpp v5, v1, v2, exec_lo row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5a,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x2f,0x01,0xff]
+v_mad_i32_i16_e64_dpp v5, v1.l, v2.l, null row_share:0 row_mask:0xf bank_mask:0xf
+// GFX12: v_mad_i32_i16_e64_dpp v5, v1.l, v2.l, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5a,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x50,0x01,0xff]
 
-v_mad_i32_i16_e64_dpp v5, v1, v2, null row_share:0 row_mask:0xf bank_mask:0xf
-// GFX12: v_mad_i32_i16_e64_dpp v5, v1, v2, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5a,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x50,0x01,0xff]
+v_mad_i32_i16_e64_dpp v5, v1.l, v2.l, -1 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX12: v_mad_i32_i16_e64_dpp v5, v1.l, v2.l, -1 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x5a,0xd6,0xfa,0x04,0x06,0x03,0x01,0x5f,0x01,0x01]
 
-v_mad_i32_i16_e64_dpp v5, v1, v2, -1 row_share:15 row_mask:0x0 bank_mask:0x1
-// GFX12: v_mad_i32_i16_e64_dpp v5, v1, v2, -1 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x5a,0xd6,0xfa,0x04,0x06,0x03,0x01,0x5f,0x01,0x01]
+v_mad_i32_i16_e64_dpp v5, v1.l, v2.l, 0.5 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX12: v_mad_i32_i16_e64_dpp v5, v1.l, v2.l, 0.5 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x05,0x00,0x5a,0xd6,0xfa,0x04,0xc2,0x03,0x01,0x60,0x09,0x13]
 
-v_mad_i32_i16_e64_dpp v5, v1, v2, 0.5 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
-// GFX12: v_mad_i32_i16_e64_dpp v5, v1, v2, 0.5 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x05,0x00,0x5a,0xd6,0xfa,0x04,0xc2,0x03,0x01,0x60,0x09,0x13]
+v_mad_i32_i16_e64_dpp v255, v255.l, v255.l, src_scc clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX12: v_mad_i32_i16_e64_dpp v255, v255.l, v255.l, src_scc clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xff,0x80,0x5a,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x05,0x30]
 
-v_mad_i32_i16_e64_dpp v255, v255, v255, src_scc clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
-// GFX12: v_mad_i32_i16_e64_dpp v255, v255, v255, src_scc clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xff,0x80,0x5a,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x05,0x30]
+v_mad_i32_i16_e64_dpp v5, v1.h, v2.l, 0.5 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX12: v_mad_i32_i16_e64_dpp v5, v1.h, v2.l, 0.5 op_sel:[1,0,0,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x05,0x08,0x5a,0xd6,0xfa,0x04,0xc2,0x03,0x01,0x60,0x09,0x13]
+
+v_mad_i32_i16_e64_dpp v255, v255.l, v255.h, src_scc clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX12: v_mad_i32_i16_e64_dpp v255, v255.l, v255.h, src_scc op_sel:[0,1,0,0] clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xff,0x90,0x5a,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x05,0x30]
 
 v_mad_i32_i24_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0]
 // GFX12: v_mad_i32_i24_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0a,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
@@ -2418,53 +2427,59 @@ v_mad_u16_e64_dpp v5.l, v1.l, v2.h, -1 row_xmask:0 row_mask:0x1 bank_mask:0x3 bo
 v_mad_u16_e64_dpp v255.h, v255.l, v255.l, src_scc clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
 // GFX12: v_mad_u16_e64_dpp v255.h, v255.l, v255.l, src_scc op_sel:[0,0,0,1] clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xff,0xc0,0x41,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x05,0x30]
 
-v_mad_u32_u16_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0]
-// GFX12: v_mad_u32_u16_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x59,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
+v_mad_u32_u16_e64_dpp v5, v1.l, v2.l, v3 quad_perm:[3,2,1,0]
+// GFX12: v_mad_u32_u16_e64_dpp v5, v1.l, v2.l, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x59,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
+
+v_mad_u32_u16_e64_dpp v5, v1.l, s2, v3 quad_perm:[3,2,1,0]
+// GFX12: v_mad_u32_u16_e64_dpp v5, v1.l, s2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x59,0xd6,0xfa,0x04,0x0c,0x04,0x01,0x1b,0x00,0xff]
 
-v_mad_u32_u16_e64_dpp v5, v1, s2, v3 quad_perm:[3,2,1,0]
-// GFX12: v_mad_u32_u16_e64_dpp v5, v1, s2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x59,0xd6,0xfa,0x04,0x0c,0x04,0x01,0x1b,0x00,0xff]
+v_mad_u32_u16_e64_dpp v5, v1.l, 10, v3 quad_perm:[3,2,1,0]
+// GFX12: v_mad_u32_u16_e64_dpp v5, v1.l, 10, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x59,0xd6,0xfa,0x14,0x0d,0x04,0x01,0x1b,0x00,0xff]
 
-v_mad_u32_u16_e64_dpp v5, v1, 10, v3 quad_perm:[3,2,1,0]
-// GFX12: v_mad_u32_u16_e64_dpp v5, v1, 10, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x59,0xd6,0xfa,0x14,0x0d,0x04,0x01,0x1b,0x00,0xff]
+v_mad_u32_u16_e64_dpp v5, v1.l, v2.l, v3 quad_perm:[0,1,2,3]
+// GFX12: v_mad_u32_u16_e64_dpp v5, v1.l, v2.l, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x59,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
 
-v_mad_u32_u16_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3]
-// GFX12: v_mad_u32_u16_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x59,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
+v_mad_u32_u16_e64_dpp v5, v1.l, v2.l, v3 row_mirror
+// GFX12: v_mad_u32_u16_e64_dpp v5, v1.l, v2.l, v3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x59,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
 
-v_mad_u32_u16_e64_dpp v5, v1, v2, v3 row_mirror
-// GFX12: v_mad_u32_u16_e64_dpp v5, v1, v2, v3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x59,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
+v_mad_u32_u16_e64_dpp v5, v1.l, v2.l, v255 row_half_mirror
+// GFX12: v_mad_u32_u16_e64_dpp v5, v1.l, v2.l, v255 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x59,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff]
 
-v_mad_u32_u16_e64_dpp v5, v1, v2, v255 row_half_mirror
-// GFX12: v_mad_u32_u16_e64_dpp v5, v1, v2, v255 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x59,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff]
+v_mad_u32_u16_e64_dpp v5, v1.l, v2.l, s105 row_shl:1
+// GFX12: v_mad_u32_u16_e64_dpp v5, v1.l, v2.l, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x59,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
 
-v_mad_u32_u16_e64_dpp v5, v1, v2, s105 row_shl:1
-// GFX12: v_mad_u32_u16_e64_dpp v5, v1, v2, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x59,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
+v_mad_u32_u16_e64_dpp v5, v1.l, v2.l, vcc_hi row_shl:15
+// GFX12: v_mad_u32_u16_e64_dpp v5, v1.l, v2.l, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x59,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
 
-v_mad_u32_u16_e64_dpp v5, v1, v2, vcc_hi row_shl:15
-// GFX12: v_mad_u32_u16_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x59,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
+v_mad_u32_u16_e64_dpp v5, v1.l, v2.l, vcc_lo row_shr:1
+// GFX12: v_mad_u32_u16_e64_dpp v5, v1.l, v2.l, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x59,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
 
-v_mad_u32_u16_e64_dpp v5, v1, v2, vcc_lo row_shr:1
-// GFX12: v_mad_u32_u16_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x59,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
+v_mad_u32_u16_e64_dpp v5, v1.l, v2.l, ttmp15 row_shr:15
+// GFX12: v_mad_u32_u16_e64_dpp v5, v1.l, v2.l, ttmp15 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x59,0xd6,0xfa,0x04,0xee,0x01,0x01,0x1f,0x01,0xff]
 
-v_mad_u32_u16_e64_dpp v5, v1, v2, ttmp15 row_shr:15
-// GFX12: v_mad_u32_u16_e64_dpp v5, v1, v2, ttmp15 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x59,0xd6,0xfa,0x04,0xee,0x01,0x01,0x1f,0x01,0xff]
+v_mad_u32_u16_e64_dpp v5, v1.l, v2.l, exec_hi row_ror:1
+// GFX12: v_mad_u32_u16_e64_dpp v5, v1.l, v2.l, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x59,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x21,0x01,0xff]
 
-v_mad_u32_u16_e64_dpp v5, v1, v2, exec_hi row_ror:1
-// GFX12: v_mad_u32_u16_e64_dpp v5, v1, v2, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x59,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x21,0x01,0xff]
+v_mad_u32_u16_e64_dpp v5, v1.l, v2.l, exec_lo row_ror:15
+// GFX12: v_mad_u32_u16_e64_dpp v5, v1.l, v2.l, exec_lo row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x59,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x2f,0x01,0xff]
 
-v_mad_u32_u16_e64_dpp v5, v1, v2, exec_lo row_ror:15
-// GFX12: v_mad_u32_u16_e64_dpp v5, v1, v2, exec_lo row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x59,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x2f,0x01,0xff]
+v_mad_u32_u16_e64_dpp v5, v1.l, v2.l, null row_share:0 row_mask:0xf bank_mask:0xf
+// GFX12: v_mad_u32_u16_e64_dpp v5, v1.l, v2.l, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x59,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x50,0x01,0xff]
 
-v_mad_u32_u16_e64_dpp v5, v1, v2, null row_share:0 row_mask:0xf bank_mask:0xf
-// GFX12: v_mad_u32_u16_e64_dpp v5, v1, v2, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x59,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x50,0x01,0xff]
+v_mad_u32_u16_e64_dpp v5, v1.l, v2.l, -1 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX12: v_mad_u32_u16_e64_dpp v5, v1.l, v2.l, -1 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x59,0xd6,0xfa,0x04,0x06,0x03,0x01,0x5f,0x01,0x01]
 
-v_mad_u32_u16_e64_dpp v5, v1, v2, -1 row_share:15 row_mask:0x0 bank_mask:0x1
-// GFX12: v_mad_u32_u16_e64_dpp v5, v1, v2, -1 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x59,0xd6,0xfa,0x04,0x06,0x03,0x01,0x5f,0x01,0x01]
+v_mad_u32_u16_e64_dpp v5, v1.l, v2.l, 0.5 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX12: v_mad_u32_u16_e64_dpp v5, v1.l, v2.l, 0.5 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x05,0x00,0x59,0xd6,0xfa,0x04,0xc2,0x03,0x01,0x60,0x09,0x13]
 
-v_mad_u32_u16_e64_dpp v5, v1, v2, 0.5 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
-// GFX12: v_mad_u32_u16_e64_dpp v5, v1, v2, 0.5 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x05,0x00,0x59,0xd6,0xfa,0x04,0xc2,0x03,0x01,0x60,0x09,0x13]
+v_mad_u32_u16_e64_dpp v255, v255.l, v255.l, src_scc clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX12: v_mad_u32_u16_e64_dpp v255, v255.l, v255.l, src_scc clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xff,0x80,0x59,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x05,0x30]
 
-v_mad_u32_u16_e64_dpp v255, v255, v255, src_scc clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
-// GFX12: v_mad_u32_u16_e64_dpp v255, v255, v255, src_scc clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xff,0x80,0x59,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x05,0x30]
+v_mad_u32_u16_e64_dpp v5, v1.h, v2.l, 0.5 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX12: v_mad_u32_u16_e64_dpp v5, v1.h, v2.l, 0.5 op_sel:[1,0,0,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x05,0x08,0x59,0xd6,0xfa,0x04,0xc2,0x03,0x01,0x60,0x09,0x13]
+
+v_mad_u32_u16_e64_dpp v255, v255.l, v255.h, src_scc clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX12: v_mad_u32_u16_e64_dpp v255, v255.l, v255.h, src_scc op_sel:[0,1,0,0] clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xff,0x90,0x59,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x05,0x30]
 
 v_mad_u32_u24_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0]
 // GFX12: v_mad_u32_u24_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0b,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
@@ -5399,11 +5414,11 @@ v_mad_i16_e64_dpp v5.l, v1.l, v2.l, -1 op_sel:[0,0,1,0] row_xmask:0 row_mask:0x1
 v_mad_i16_e64_dpp v255.h, v255.l, v255.l, src_scc op_sel:[0,0,0,1] clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1
 // GFX12: v_mad_i16_e64_dpp v255.h, v255.l, v255.l, src_scc op_sel:[0,0,0,1] clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0xc0,0x53,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30]
 
-v_mad_i32_i16_e64_dpp v5, v1, v2, 0.5 op_sel:[1,0,0,0] row_xmask:0 row_mask:0x1 bank_mask:0x3
-// GFX12: v_mad_i32_i16_e64_dpp v5, v1, v2, 0.5 op_sel:[1,0,0,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x08,0x5a,0xd6,0xfa,0x04,0xc2,0x03,0x01,0x60,0x01,0x13]
+v_mad_i32_i16_e64_dpp v5, v1.h, v2.l, 0.5 op_sel:[1,0,0,0] row_xmask:0 row_mask:0x1 bank_mask:0x3
+// GFX12: v_mad_i32_i16_e64_dpp v5, v1.h, v2.l, 0.5 op_sel:[1,0,0,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x08,0x5a,0xd6,0xfa,0x04,0xc2,0x03,0x01,0x60,0x01,0x13]
 
-v_mad_i32_i16_e64_dpp v255, v255, v255, src_scc op_sel:[0,1,0,0] clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1
-// GFX12: v_mad_i32_i16_e64_dpp v255, v255, v255, src_scc op_sel:[0,1,0,0] clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x90,0x5a,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30]
+v_mad_i32_i16_e64_dpp v255, v255.l, v255.h, src_scc op_sel:[0,1,0,0] clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1
+// GFX12: v_mad_i32_i16_e64_dpp v255, v255.l, v255.h, src_scc op_sel:[0,1,0,0] clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x90,0x5a,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30]
 
 v_mad_u16_e64_dpp v5.h, v1.h, v2.h, exec_hi op_sel:[1,1,1,1] row_ror:15 row_mask:0xf bank_mask:0xf
 // GFX12: v_mad_u16_e64_dpp v5.h, v1.h, v2.h, exec_hi op_sel:[1,1,1,1] row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x78,0x41,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x2f,0x01,0xff]
@@ -5420,11 +5435,11 @@ v_mad_u16_e64_dpp v5.l, v1.l, v2.l, -1 op_sel:[0,0,1,0] row_xmask:0 row_mask:0x1
 v_mad_u16_e64_dpp v255.h, v255.l, v255.l, src_scc op_sel:[0,0,0,1] clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1
 // GFX12: v_mad_u16_e64_dpp v255.h, v255.l, v255.l, src_scc op_sel:[0,0,0,1] clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0xc0,0x41,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30]
 
-v_mad_u32_u16_e64_dpp v5, v1, v2, 0.5 op_sel:[1,0,0,0] row_xmask:0 row_mask:0x1 bank_mask:0x3
-// GFX12: v_mad_u32_u16_e64_dpp v5, v1, v2, 0.5 op_sel:[1,0,0,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x08,0x59,0xd6,0xfa,0x04,0xc2,0x03,0x01,0x60,0x01,0x13]
+v_mad_u32_u16_e64_dpp v5, v1.h, v2.l, 0.5 op_sel:[1,0,0,0] row_xmask:0 row_mask:0x1 bank_mask:0x3
+// GFX12: v_mad_u32_u16_e64_dpp v5, v1.h, v2.l, 0.5 op_sel:[1,0,0,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x08,0x59,0xd6,0xfa,0x04,0xc2,0x03,0x01,0x60,0x01,0x13]
 
-v_mad_u32_u16_e64_dpp v255, v255, v255, src_scc op_sel:[0,1,0,0] clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1
-// GFX12: v_mad_u32_u16_e64_dpp v255, v255, v255, src_scc op_sel:[0,1,0,0] clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x90,0x59,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30]
+v_mad_u32_u16_e64_dpp v255, v255.l, v255.h, src_scc op_sel:[0,1,0,0] clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1
+// GFX12: v_mad_u32_u16_e64_dpp v255, v255.l, v255.h, src_scc op_sel:[0,1,0,0] clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x90,0x59,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30]
 
 v_max3_num_f16_e64_dpp v5.h, -v1.h, v2.h, |exec_lo| op_sel:[1,1,1,1] row_ror:15 row_mask:0xf bank_mask:0xf
 // GFX12: v_max3_num_f16_e64_dpp v5.h, -v1.h, v2.h, |exec_lo| op_sel:[1,1,1,1] row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x7c,0x2c,0xd6,0xfa,0x04,0xfa,0x21,0x01,0x2f,0x01,0xff]
@@ -5705,89 +5720,107 @@ v_maximum_f32 v5, -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1
 v_maximum_f32 v255, -|v255|, -|v255| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
 // GFX12: v_maximum_f32_e64_dpp v255, -|v255|, -|v255| row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xff,0x03,0x66,0xd7,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x05,0x30]
 
-v_minimum_f16 v5, v1, v2 quad_perm:[3,2,1,0]
-// GFX12: v_minimum_f16_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x67,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+v_minimum_f16 v5.l, v1.l, v2.l quad_perm:[3,2,1,0]
+// GFX12: v_minimum_f16_e64_dpp v5.l, v1.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x67,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+
+v_minimum_f16 v5.l, v1.l, v2.l quad_perm:[0,1,2,3]
+// GFX12: v_minimum_f16_e64_dpp v5.l, v1.l, v2.l quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x67,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+
+v_minimum_f16 v5.l, v1.l, v2.l row_mirror
+// GFX12: v_minimum_f16_e64_dpp v5.l, v1.l, v2.l row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x67,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+
+v_minimum_f16 v5.l, v1.l, v2.l row_half_mirror
+// GFX12: v_minimum_f16_e64_dpp v5.l, v1.l, v2.l row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x67,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
 
-v_minimum_f16 v5, v1, v2 quad_perm:[0,1,2,3]
-// GFX12: v_minimum_f16_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x67,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+v_minimum_f16 v5.l, v1.l, v2.l row_shl:1
+// GFX12: v_minimum_f16_e64_dpp v5.l, v1.l, v2.l row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x67,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
 
-v_minimum_f16 v5, v1, v2 row_mirror
-// GFX12: v_minimum_f16_e64_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x67,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+v_minimum_f16 v5.l, v1.l, v2.l row_shl:15
+// GFX12: v_minimum_f16_e64_dpp v5.l, v1.l, v2.l row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x67,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
 
-v_minimum_f16 v5, v1, v2 row_half_mirror
-// GFX12: v_minimum_f16_e64_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x67,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+v_minimum_f16 v5.l, v1.l, v2.l row_shr:1
+// GFX12: v_minimum_f16_e64_dpp v5.l, v1.l, v2.l row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x67,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
 
-v_minimum_f16 v5, v1, v2 row_shl:1
-// GFX12: v_minimum_f16_e64_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x67,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+v_minimum_f16 v5.l, v1.l, v2.l row_shr:15
+// GFX12: v_minimum_f16_e64_dpp v5.l, v1.l, v2.l row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x67,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
 
-v_minimum_f16 v5, v1, v2 row_shl:15
-// GFX12: v_minimum_f16_e64_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x67,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+v_minimum_f16 v5.l, v1.l, v2.l row_ror:1
+// GFX12: v_minimum_f16_e64_dpp v5.l, v1.l, v2.l row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x67,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
 
-v_minimum_f16 v5, v1, v2 row_shr:1
-// GFX12: v_minimum_f16_e64_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x67,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+v_minimum_f16 v5.l, v1.l, v2.l row_ror:15
+// GFX12: v_minimum_f16_e64_dpp v5.l, v1.l, v2.l row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x67,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
 
-v_minimum_f16 v5, v1, v2 row_shr:15
-// GFX12: v_minimum_f16_e64_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x67,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+v_minimum_f16 v5.l, v1.l, v2.l row_share:0 row_mask:0xf bank_mask:0xf
+// GFX12: v_minimum_f16_e64_dpp v5.l, v1.l, v2.l row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x67,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 
-v_minimum_f16 v5, v1, v2 row_ror:1
-// GFX12: v_minimum_f16_e64_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x67,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+v_minimum_f16 v5.l, |v1.l|, -v2.l row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX12: v_minimum_f16_e64_dpp v5.l, |v1.l|, -v2.l row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x01,0x67,0xd7,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
 
-v_minimum_f16 v5, v1, v2 row_ror:15
-// GFX12: v_minimum_f16_e64_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x67,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+v_minimum_f16 v5.l, -v1.l, |v2.l| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX12: v_minimum_f16_e64_dpp v5.l, -v1.l, |v2.l| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x05,0x02,0x67,0xd7,0xfa,0x04,0x02,0x20,0x01,0x60,0x09,0x13]
 
-v_minimum_f16 v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
-// GFX12: v_minimum_f16_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x67,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+v_minimum_f16 v255.l, -|v255.l|, -|v255.l| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX12: v_minimum_f16_e64_dpp v255.l, -|v255.l|, -|v255.l| row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xff,0x03,0x67,0xd7,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x05,0x30]
 
-v_minimum_f16 v5, |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1
-// GFX12: v_minimum_f16_e64_dpp v5, |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x01,0x67,0xd7,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+v_minimum_f16 v5.h, v1.h, v2.l quad_perm:[3,2,1,0]
+// GFX12: v_minimum_f16_e64_dpp v5.h, v1.h, v2.l op_sel:[1,0,1] quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x48,0x67,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
 
-v_minimum_f16 v5, -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
-// GFX12: v_minimum_f16_e64_dpp v5, -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x05,0x02,0x67,0xd7,0xfa,0x04,0x02,0x20,0x01,0x60,0x09,0x13]
+v_minimum_f16 v5.h, v1.h, v2.l row_ror:15
+// GFX12: v_minimum_f16_e64_dpp v5.h, v1.h, v2.l op_sel:[1,0,1] row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x48,0x67,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
 
-v_minimum_f16 v255, -|v255|, -|v255| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
-// GFX12: v_minimum_f16_e64_dpp v255, -|v255|, -|v255| row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xff,0x03,0x67,0xd7,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x05,0x30]
+v_minimum_f16 v255.h, -|v255.h|, -|v255.h| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX12: v_minimum_f16_e64_dpp v255.h, -|v255.h|, -|v255.h| op_sel:[1,1,1] row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xff,0x5b,0x67,0xd7,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x05,0x30]
 
-v_maximum_f16 v5, v1, v2 quad_perm:[3,2,1,0]
-// GFX12: v_maximum_f16_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x68,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+v_maximum_f16 v5.l, v1.l, v2.l quad_perm:[3,2,1,0]
+// GFX12: v_maximum_f16_e64_dpp v5.l, v1.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x68,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
 
-v_maximum_f16 v5, v1, v2 quad_perm:[0,1,2,3]
-// GFX12: v_maximum_f16_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x68,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+v_maximum_f16 v5.l, v1.l, v2.l quad_perm:[0,1,2,3]
+// GFX12: v_maximum_f16_e64_dpp v5.l, v1.l, v2.l quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x68,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
 
-v_maximum_f16 v5, v1, v2 row_mirror
-// GFX12: v_maximum_f16_e64_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x68,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+v_maximum_f16 v5.l, v1.l, v2.l row_mirror
+// GFX12: v_maximum_f16_e64_dpp v5.l, v1.l, v2.l row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x68,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
 
-v_maximum_f16 v5, v1, v2 row_half_mirror
-// GFX12: v_maximum_f16_e64_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x68,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+v_maximum_f16 v5.l, v1.l, v2.l row_half_mirror
+// GFX12: v_maximum_f16_e64_dpp v5.l, v1.l, v2.l row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x68,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
 
-v_maximum_f16 v5, v1, v2 row_shl:1
-// GFX12: v_maximum_f16_e64_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x68,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+v_maximum_f16 v5.l, v1.l, v2.l row_shl:1
+// GFX12: v_maximum_f16_e64_dpp v5.l, v1.l, v2.l row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x68,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
 
-v_maximum_f16 v5, v1, v2 row_shl:15
-// GFX12: v_maximum_f16_e64_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x68,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+v_maximum_f16 v5.l, v1.l, v2.l row_shl:15
+// GFX12: v_maximum_f16_e64_dpp v5.l, v1.l, v2.l row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x68,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
 
-v_maximum_f16 v5, v1, v2 row_shr:1
-// GFX12: v_maximum_f16_e64_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x68,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+v_maximum_f16 v5.l, v1.l, v2.l row_shr:1
+// GFX12: v_maximum_f16_e64_dpp v5.l, v1.l, v2.l row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x68,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
 
-v_maximum_f16 v5, v1, v2 row_shr:15
-// GFX12: v_maximum_f16_e64_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x68,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+v_maximum_f16 v5.l, v1.l, v2.l row_shr:15
+// GFX12: v_maximum_f16_e64_dpp v5.l, v1.l, v2.l row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x68,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
 
-v_maximum_f16 v5, v1, v2 row_ror:1
-// GFX12: v_maximum_f16_e64_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x68,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+v_maximum_f16 v5.l, v1.l, v2.l row_ror:1
+// GFX12: v_maximum_f16_e64_dpp v5.l, v1.l, v2.l row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x68,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
 
-v_maximum_f16 v5, v1, v2 row_ror:15
-// GFX12: v_maximum_f16_e64_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x68,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+v_maximum_f16 v5.l, v1.l, v2.l row_ror:15
+// GFX12: v_maximum_f16_e64_dpp v5.l, v1.l, v2.l row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x68,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
 
-v_maximum_f16 v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
-// GFX12: v_maximum_f16_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x68,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+v_maximum_f16 v5.l, v1.l, v2.l row_share:0 row_mask:0xf bank_mask:0xf
+// GFX12: v_maximum_f16_e64_dpp v5.l, v1.l, v2.l row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x68,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 
-v_maximum_f16 v5, |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1
-// GFX12: v_maximum_f16_e64_dpp v5, |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x01,0x68,0xd7,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+v_maximum_f16 v5.l, |v1.l|, -v2.l row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX12: v_maximum_f16_e64_dpp v5.l, |v1.l|, -v2.l row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x01,0x68,0xd7,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
 
-v_maximum_f16 v5, -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
-// GFX12: v_maximum_f16_e64_dpp v5, -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x05,0x02,0x68,0xd7,0xfa,0x04,0x02,0x20,0x01,0x60,0x09,0x13]
+v_maximum_f16 v5.l, -v1.l, |v2.l| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX12: v_maximum_f16_e64_dpp v5.l, -v1.l, |v2.l| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x05,0x02,0x68,0xd7,0xfa,0x04,0x02,0x20,0x01,0x60,0x09,0x13]
 
-v_maximum_f16 v255, -|v255|, -|v255| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
-// GFX12: v_maximum_f16_e64_dpp v255, -|v255|, -|v255| row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xff,0x03,0x68,0xd7,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x05,0x30]
+v_maximum_f16 v255.l, -|v255.l|, -|v255.l| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX12: v_maximum_f16_e64_dpp v255.l, -|v255.l|, -|v255.l| row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xff,0x03,0x68,0xd7,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x05,0x30]
+
+v_maximum_f16 v5.h, v1.h, v2.l quad_perm:[3,2,1,0]
+// GFX12: v_maximum_f16_e64_dpp v5.h, v1.h, v2.l op_sel:[1,0,1] quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x48,0x68,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+
+v_maximum_f16 v5.h, v1.h, v2.l row_ror:15
+// GFX12: v_maximum_f16_e64_dpp v5.h, v1.h, v2.l op_sel:[1,0,1] row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x48,0x68,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+
+v_maximum_f16 v255.h, -|v255.h|, -|v255.h| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX12: v_maximum_f16_e64_dpp v255.h, -|v255.h|, -|v255.h| op_sel:[1,1,1] row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xff,0x5b,0x68,0xd7,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x05,0x30]
 
 v_minimum3_f32 v5, v1, v2, v3 quad_perm:[3,2,1,0]
 // GFX12: v_minimum3_f32_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x2d,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
@@ -5885,101 +5918,131 @@ v_maximum3_f32 v5, v1, -|v2|, -|0.5| mul:4 row_xmask:0 row_mask:0x1 bank_mask:0x
 v_maximum3_f32 v255, -|v255|, -|v255|, -|src_scc| clamp div:2 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
 // GFX12: v_maximum3_f32_e64_dpp v255, -|v255|, -|v255|, -|src_scc| clamp div:2 row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xff,0x87,0x2e,0xd6,0xfa,0xfe,0xf7,0xfb,0xff,0x6f,0x05,0x30]
 
-v_minimum3_f16 v5, v1, v2, v3 quad_perm:[3,2,1,0]
-// GFX12: v_minimum3_f16_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x2f,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
+v_minimum3_f16 v5.l, v1.l, v2.l, v3.l quad_perm:[3,2,1,0]
+// GFX12: v_minimum3_f16_e64_dpp v5.l, v1.l, v2.l, v3.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x2f,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
+
+v_minimum3_f16 v5.l, v1.l, s2, v3.l quad_perm:[3,2,1,0]
+// GFX12: v_minimum3_f16_e64_dpp v5.l, v1.l, s2, v3.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x2f,0xd6,0xfa,0x04,0x0c,0x04,0x01,0x1b,0x00,0xff]
+
+v_minimum3_f16 v5.l, v1.l, 2.0, v3.l quad_perm:[3,2,1,0]
+// GFX12: v_minimum3_f16_e64_dpp v5.l, v1.l, 2.0, v3.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x2f,0xd6,0xfa,0xe8,0x0d,0x04,0x01,0x1b,0x00,0xff]
+
+v_minimum3_f16 v5.l, v1.l, v2.l, v3.l quad_perm:[0,1,2,3]
+// GFX12: v_minimum3_f16_e64_dpp v5.l, v1.l, v2.l, v3.l quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x2f,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
+
+v_minimum3_f16 v5.l, v1.l, v2.l, v3.l row_mirror
+// GFX12: v_minimum3_f16_e64_dpp v5.l, v1.l, v2.l, v3.l row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x2f,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
+
+v_minimum3_f16 v5.l, v1.l, v2.l, v255.l row_half_mirror
+// GFX12: v_minimum3_f16_e64_dpp v5.l, v1.l, v2.l, v255.l row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x2f,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff]
 
-v_minimum3_f16 v5, v1, s2, v3 quad_perm:[3,2,1,0]
-// GFX12: v_minimum3_f16_e64_dpp v5, v1, s2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x2f,0xd6,0xfa,0x04,0x0c,0x04,0x01,0x1b,0x00,0xff]
+v_minimum3_f16 v5.l, v1.l, v2.l, s105 row_shl:1
+// GFX12: v_minimum3_f16_e64_dpp v5.l, v1.l, v2.l, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x2f,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
 
-v_minimum3_f16 v5, v1, 2.0, v3 quad_perm:[3,2,1,0]
-// GFX12: v_minimum3_f16_e64_dpp v5, v1, 2.0, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x2f,0xd6,0xfa,0xe8,0x0d,0x04,0x01,0x1b,0x00,0xff]
+v_minimum3_f16 v5.l, v1.l, v2.l, vcc_hi row_shl:15
+// GFX12: v_minimum3_f16_e64_dpp v5.l, v1.l, v2.l, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x2f,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
 
-v_minimum3_f16 v5, v1, v2, v3 quad_perm:[0,1,2,3]
-// GFX12: v_minimum3_f16_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x2f,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
+v_minimum3_f16 v5.l, v1.l, v2.l, vcc_lo row_shr:1
+// GFX12: v_minimum3_f16_e64_dpp v5.l, v1.l, v2.l, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x2f,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
 
-v_minimum3_f16 v5, v1, v2, v3 row_mirror
-// GFX12: v_minimum3_f16_e64_dpp v5, v1, v2, v3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x2f,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
+v_minimum3_f16 v5.l, |v1.l|, v2.l, -ttmp15 row_shr:15
+// GFX12: v_minimum3_f16_e64_dpp v5.l, |v1.l|, v2.l, -ttmp15 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x01,0x2f,0xd6,0xfa,0x04,0xee,0x81,0x01,0x1f,0x01,0xff]
 
-v_minimum3_f16 v5, v1, v2, v255 row_half_mirror
-// GFX12: v_minimum3_f16_e64_dpp v5, v1, v2, v255 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x2f,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff]
+v_minimum3_f16 v5.l, v1.l, -|v2.l|, exec_hi row_ror:1
+// GFX12: v_minimum3_f16_e64_dpp v5.l, v1.l, -|v2.l|, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x02,0x2f,0xd6,0xfa,0x04,0xfe,0x41,0x01,0x21,0x01,0xff]
 
-v_minimum3_f16 v5, v1, v2, s105 row_shl:1
-// GFX12: v_minimum3_f16_e64_dpp v5, v1, v2, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x2f,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
+v_minimum3_f16 v5.l, -v1.l, v2.l, |exec_lo| row_ror:15
+// GFX12: v_minimum3_f16_e64_dpp v5.l, -v1.l, v2.l, |exec_lo| row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x04,0x2f,0xd6,0xfa,0x04,0xfa,0x21,0x01,0x2f,0x01,0xff]
 
-v_minimum3_f16 v5, v1, v2, vcc_hi row_shl:15
-// GFX12: v_minimum3_f16_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x2f,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
+v_minimum3_f16 v5.l, -|v1.l|, -|v2.l|, null row_share:0 row_mask:0xf bank_mask:0xf
+// GFX12: v_minimum3_f16_e64_dpp v5.l, -|v1.l|, -|v2.l|, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x03,0x2f,0xd6,0xfa,0x04,0xf2,0x61,0x01,0x50,0x01,0xff]
 
-v_minimum3_f16 v5, v1, v2, vcc_lo row_shr:1
-// GFX12: v_minimum3_f16_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x2f,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
+v_minimum3_f16 v5.l, -|v1.l|, v2.l, -|-1| row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX12: v_minimum3_f16_e64_dpp v5.l, -|v1.l|, v2.l, -|-1| row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x05,0x2f,0xd6,0xfa,0x04,0x06,0xa3,0x01,0x5f,0x01,0x01]
 
-v_minimum3_f16 v5, |v1|, v2, -ttmp15 row_shr:15
-// GFX12: v_minimum3_f16_e64_dpp v5, |v1|, v2, -ttmp15 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x01,0x2f,0xd6,0xfa,0x04,0xee,0x81,0x01,0x1f,0x01,0xff]
+v_minimum3_f16 v5.l, v1.l, -|v2.l|, -|0.5| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX12: v_minimum3_f16_e64_dpp v5.l, v1.l, -|v2.l|, -|0.5| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x05,0x06,0x2f,0xd6,0xfa,0x04,0xc2,0xc3,0x01,0x60,0x09,0x13]
 
-v_minimum3_f16 v5, v1, -|v2|, exec_hi row_ror:1
-// GFX12: v_minimum3_f16_e64_dpp v5, v1, -|v2|, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x02,0x2f,0xd6,0xfa,0x04,0xfe,0x41,0x01,0x21,0x01,0xff]
+v_minimum3_f16 v255.l, -|v255.l|, -|v255.l|, -|src_scc| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX12: v_minimum3_f16_e64_dpp v255.l, -|v255.l|, -|v255.l|, -|src_scc| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xff,0x87,0x2f,0xd6,0xfa,0xfe,0xf7,0xe3,0xff,0x6f,0x05,0x30]
 
-v_minimum3_f16 v5, -v1, v2, |exec_lo| row_ror:15
-// GFX12: v_minimum3_f16_e64_dpp v5, -v1, v2, |exec_lo| row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x04,0x2f,0xd6,0xfa,0x04,0xfa,0x21,0x01,0x2f,0x01,0xff]
+v_minimum3_f16 v5.h, v1.h, v2.h, v3.h quad_perm:[3,2,1,0]
+// GFX12: v_minimum3_f16_e64_dpp v5.h, v1.h, v2.h, v3.h op_sel:[1,1,1,1] quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x78,0x2f,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 
-v_minimum3_f16 v5, -|v1|, -|v2|, null row_share:0 row_mask:0xf bank_mask:0xf
-// GFX12: v_minimum3_f16_e64_dpp v5, -|v1|, -|v2|, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x03,0x2f,0xd6,0xfa,0x04,0xf2,0x61,0x01,0x50,0x01,0xff]
+v_minimum3_f16 v5.l, v1.l, v2.l, v255.h quad_perm:[0,1,2,3]
+// GFX12: v_minimum3_f16_e64_dpp v5.l, v1.l, v2.l, v255.h op_sel:[0,0,1,0] quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x20,0x2f,0xd6,0xfa,0x04,0xfe,0x07,0x01,0xe4,0x00,0xff]
 
-v_minimum3_f16 v5, -|v1|, v2, -|-1| row_share:15 row_mask:0x0 bank_mask:0x1
-// GFX12: v_minimum3_f16_e64_dpp v5, -|v1|, v2, -|-1| row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x05,0x2f,0xd6,0xfa,0x04,0x06,0xa3,0x01,0x5f,0x01,0x01]
+v_minimum3_f16 v5.l, -v1.h, |v2.l|, -1 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX12: v_minimum3_f16_e64_dpp v5.l, -v1.h, |v2.l|, -1 op_sel:[1,0,0,0] row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x0a,0x2f,0xd6,0xfa,0x04,0x06,0x23,0x01,0x5f,0x01,0x01]
 
-v_minimum3_f16 v5, v1, -|v2|, -|0.5| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
-// GFX12: v_minimum3_f16_e64_dpp v5, v1, -|v2|, -|0.5| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x05,0x06,0x2f,0xd6,0xfa,0x04,0xc2,0xc3,0x01,0x60,0x09,0x13]
+v_minimum3_f16 v5.l, -|v1.l|, -|v2.h|, 0.5 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX12: v_minimum3_f16_e64_dpp v5.l, -|v1.l|, -|v2.h|, 0.5 op_sel:[0,1,0,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x05,0x13,0x2f,0xd6,0xfa,0x04,0xc2,0x63,0x01,0x60,0x09,0x13]
 
-v_minimum3_f16 v255, -|v255|, -|v255|, -|src_scc| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
-// GFX12: v_minimum3_f16_e64_dpp v255, -|v255|, -|v255|, -|src_scc| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xff,0x87,0x2f,0xd6,0xfa,0xfe,0xf7,0xe3,0xff,0x6f,0x05,0x30]
+v_minimum3_f16 v255.h, -|v255.l|, -|v255.l|, -|src_scc| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX12: v_minimum3_f16_e64_dpp v255.h, -|v255.l|, -|v255.l|, -|src_scc| op_sel:[0,0,0,1] clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xff,0xc7,0x2f,0xd6,0xfa,0xfe,0xf7,0xe3,0xff,0x6f,0x05,0x30]
 
-v_maximum3_f16 v5, v1, v2, v3 quad_perm:[3,2,1,0]
-// GFX12: v_maximum3_f16_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x30,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
+v_maximum3_f16 v5.l, v1.l, v2.l, v3.l quad_perm:[3,2,1,0]
+// GFX12: v_maximum3_f16_e64_dpp v5.l, v1.l, v2.l, v3.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x30,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 
-v_maximum3_f16 v5, v1, s2, v3 quad_perm:[3,2,1,0]
-// GFX12: v_maximum3_f16_e64_dpp v5, v1, s2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x30,0xd6,0xfa,0x04,0x0c,0x04,0x01,0x1b,0x00,0xff]
+v_maximum3_f16 v5.l, v1.l, s2, v3.l quad_perm:[3,2,1,0]
+// GFX12: v_maximum3_f16_e64_dpp v5.l, v1.l, s2, v3.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x30,0xd6,0xfa,0x04,0x0c,0x04,0x01,0x1b,0x00,0xff]
 
-v_maximum3_f16 v5, v1, 2.0, v3 quad_perm:[3,2,1,0]
-// GFX12: v_maximum3_f16_e64_dpp v5, v1, 2.0, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x30,0xd6,0xfa,0xe8,0x0d,0x04,0x01,0x1b,0x00,0xff]
+v_maximum3_f16 v5.l, v1.l, 2.0, v3.l quad_perm:[3,2,1,0]
+// GFX12: v_maximum3_f16_e64_dpp v5.l, v1.l, 2.0, v3.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x30,0xd6,0xfa,0xe8,0x0d,0x04,0x01,0x1b,0x00,0xff]
 
-v_maximum3_f16 v5, v1, v2, v3 quad_perm:[0,1,2,3]
-// GFX12: v_maximum3_f16_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x30,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
+v_maximum3_f16 v5.l, v1.l, v2.l, v3.l quad_perm:[0,1,2,3]
+// GFX12: v_maximum3_f16_e64_dpp v5.l, v1.l, v2.l, v3.l quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x30,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
 
-v_maximum3_f16 v5, v1, v2, v3 row_mirror
-// GFX12: v_maximum3_f16_e64_dpp v5, v1, v2, v3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x30,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
+v_maximum3_f16 v5.l, v1.l, v2.l, v3.l row_mirror
+// GFX12: v_maximum3_f16_e64_dpp v5.l, v1.l, v2.l, v3.l row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x30,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
 
-v_maximum3_f16 v5, v1, v2, v255 row_half_mirror
-// GFX12: v_maximum3_f16_e64_dpp v5, v1, v2, v255 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x30,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff]
+v_maximum3_f16 v5.l, v1.l, v2.l, v255.l row_half_mirror
+// GFX12: v_maximum3_f16_e64_dpp v5.l, v1.l, v2.l, v255.l row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x30,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff]
 
-v_maximum3_f16 v5, v1, v2, s105 row_shl:1
-// GFX12: v_maximum3_f16_e64_dpp v5, v1, v2, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x30,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
+v_maximum3_f16 v5.l, v1.l, v2.l, s105 row_shl:1
+// GFX12: v_maximum3_f16_e64_dpp v5.l, v1.l, v2.l, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x30,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
 
-v_maximum3_f16 v5, v1, v2, vcc_hi row_shl:15
-// GFX12: v_maximum3_f16_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x30,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
+v_maximum3_f16 v5.l, v1.l, v2.l, vcc_hi row_shl:15
+// GFX12: v_maximum3_f16_e64_dpp v5.l, v1.l, v2.l, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x30,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
 
-v_maximum3_f16 v5, v1, v2, vcc_lo row_shr:1
-// GFX12: v_maximum3_f16_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x30,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
+v_maximum3_f16 v5.l, v1.l, v2.l, vcc_lo row_shr:1
+// GFX12: v_maximum3_f16_e64_dpp v5.l, v1.l, v2.l, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x30,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
 
-v_maximum3_f16 v5, |v1|, v2, -ttmp15 row_shr:15
-// GFX12: v_maximum3_f16_e64_dpp v5, |v1|, v2, -ttmp15 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x01,0x30,0xd6,0xfa,0x04,0xee,0x81,0x01,0x1f,0x01,0xff]
+v_maximum3_f16 v5.l, |v1.l|, v2.l, -ttmp15 row_shr:15
+// GFX12: v_maximum3_f16_e64_dpp v5.l, |v1.l|, v2.l, -ttmp15 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x01,0x30,0xd6,0xfa,0x04,0xee,0x81,0x01,0x1f,0x01,0xff]
 
-v_maximum3_f16 v5, v1, -|v2|, exec_hi row_ror:1
-// GFX12: v_maximum3_f16_e64_dpp v5, v1, -|v2|, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x02,0x30,0xd6,0xfa,0x04,0xfe,0x41,0x01,0x21,0x01,0xff]
+v_maximum3_f16 v5.l, v1.l, -|v2.l|, exec_hi row_ror:1
+// GFX12: v_maximum3_f16_e64_dpp v5.l, v1.l, -|v2.l|, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x02,0x30,0xd6,0xfa,0x04,0xfe,0x41,0x01,0x21,0x01,0xff]
 
-v_maximum3_f16 v5, -v1, v2, |exec_lo| row_ror:15
-// GFX12: v_maximum3_f16_e64_dpp v5, -v1, v2, |exec_lo| row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x04,0x30,0xd6,0xfa,0x04,0xfa,0x21,0x01,0x2f,0x01,0xff]
+v_maximum3_f16 v5.l, -v1.l, v2.l, |exec_lo| row_ror:15
+// GFX12: v_maximum3_f16_e64_dpp v5.l, -v1.l, v2.l, |exec_lo| row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x04,0x30,0xd6,0xfa,0x04,0xfa,0x21,0x01,0x2f,0x01,0xff]
 
-v_maximum3_f16 v5, -|v1|, -|v2|, null row_share:0 row_mask:0xf bank_mask:0xf
-// GFX12: v_maximum3_f16_e64_dpp v5, -|v1|, -|v2|, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x03,0x30,0xd6,0xfa,0x04,0xf2,0x61,0x01,0x50,0x01,0xff]
+v_maximum3_f16 v5.l, -|v1.l|, -|v2.l|, null row_share:0 row_mask:0xf bank_mask:0xf
+// GFX12: v_maximum3_f16_e64_dpp v5.l, -|v1.l|, -|v2.l|, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x03,0x30,0xd6,0xfa,0x04,0xf2,0x61,0x01,0x50,0x01,0xff]
 
-v_maximum3_f16 v5, -|v1|, v2, -|-1| row_share:15 row_mask:0x0 bank_mask:0x1
-// GFX12: v_maximum3_f16_e64_dpp v5, -|v1|, v2, -|-1| row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x05,0x30,0xd6,0xfa,0x04,0x06,0xa3,0x01,0x5f,0x01,0x01]
+v_maximum3_f16 v5.l, -|v1.l|, v2.l, -|-1| row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX12: v_maximum3_f16_e64_dpp v5.l, -|v1.l|, v2.l, -|-1| row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x05,0x30,0xd6,0xfa,0x04,0x06,0xa3,0x01,0x5f,0x01,0x01]
 
-v_maximum3_f16 v5, v1, -|v2|, -|0.5| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
-// GFX12: v_maximum3_f16_e64_dpp v5, v1, -|v2|, -|0.5| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x05,0x06,0x30,0xd6,0xfa,0x04,0xc2,0xc3,0x01,0x60,0x09,0x13]
+v_maximum3_f16 v5.l, v1.l, -|v2.l|, -|0.5| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX12: v_maximum3_f16_e64_dpp v5.l, v1.l, -|v2.l|, -|0.5| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x05,0x06,0x30,0xd6,0xfa,0x04,0xc2,0xc3,0x01,0x60,0x09,0x13]
 
-v_maximum3_f16 v255, -|v255|, -|v255|, -|src_scc| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
-// GFX12: v_maximum3_f16_e64_dpp v255, -|v255|, -|v255|, -|src_scc| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xff,0x87,0x30,0xd6,0xfa,0xfe,0xf7,0xe3,0xff,0x6f,0x05,0x30]
+v_maximum3_f16 v255.l, -|v255.l|, -|v255.l|, -|src_scc| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX12: v_maximum3_f16_e64_dpp v255.l, -|v255.l|, -|v255.l|, -|src_scc| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xff,0x87,0x30,0xd6,0xfa,0xfe,0xf7,0xe3,0xff,0x6f,0x05,0x30]
+
+v_maximum3_f16 v5.h, v1.h, v2.h, v3.h quad_perm:[3,2,1,0]
+// GFX12: v_maximum3_f16_e64_dpp v5.h, v1.h, v2.h, v3.h op_sel:[1,1,1,1] quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x78,0x30,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
+
+v_maximum3_f16 v5.l, v1.l, v2.l, v255.h quad_perm:[0,1,2,3]
+// GFX12: v_maximum3_f16_e64_dpp v5.l, v1.l, v2.l, v255.h op_sel:[0,0,1,0] quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x20,0x30,0xd6,0xfa,0x04,0xfe,0x07,0x01,0xe4,0x00,0xff]
+
+v_maximum3_f16 v5.l, -v1.h, |v2.l|, -1 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX12: v_maximum3_f16_e64_dpp v5.l, -v1.h, |v2.l|, -1 op_sel:[1,0,0,0] row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x0a,0x30,0xd6,0xfa,0x04,0x06,0x23,0x01,0x5f,0x01,0x01]
+
+v_maximum3_f16 v5.l, -|v1.l|, -|v2.h|, 0.5 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX12: v_maximum3_f16_e64_dpp v5.l, -|v1.l|, -|v2.h|, 0.5 op_sel:[0,1,0,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x05,0x13,0x30,0xd6,0xfa,0x04,0xc2,0x63,0x01,0x60,0x09,0x13]
+
+v_maximum3_f16 v255.h, -|v255.l|, -|v255.l|, -|src_scc| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX12: v_maximum3_f16_e64_dpp v255.h, -|v255.l|, -|v255.l|, -|src_scc| op_sel:[0,0,0,1] clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xff,0xc7,0x30,0xd6,0xfa,0xfe,0xf7,0xe3,0xff,0x6f,0x05,0x30]
 
 v_maximumminimum_f32 v5, v1, v2, v3 quad_perm:[0,1,2,3]
 // GFX12: v_maximumminimum_f32_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x6d,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
@@ -6068,98 +6131,128 @@ v_minimummaximum_f32 v5, v1, -|v2|, -|0.5| mul:4 row_xmask:0 row_mask:0x1 bank_m
 v_minimummaximum_f32 v255, -|v255|, -|v255|, -|src_scc| clamp div:2 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
 // GFX12: v_minimummaximum_f32_e64_dpp v255, -|v255|, -|v255|, -|src_scc| clamp div:2 row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xff,0x87,0x6c,0xd6,0xfa,0xfe,0xf7,0xfb,0xff,0x6f,0x05,0x30]
 
-v_maximumminimum_f16 v5, v1, v2, v3 quad_perm:[3,2,1,0]
-// GFX12: v_maximumminimum_f16_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x6f,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
+v_maximumminimum_f16 v5.l, v1.l, v2.l, v3.l quad_perm:[3,2,1,0]
+// GFX12: v_maximumminimum_f16_e64_dpp v5.l, v1.l, v2.l, v3.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x6f,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
+
+v_maximumminimum_f16 v5.l, v1.l, s2, v3.l quad_perm:[3,2,1,0]
+// GFX12: v_maximumminimum_f16_e64_dpp v5.l, v1.l, s2, v3.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x6f,0xd6,0xfa,0x04,0x0c,0x04,0x01,0x1b,0x00,0xff]
+
+v_maximumminimum_f16 v5.l, v1.l, 2.0, v3.l quad_perm:[3,2,1,0]
+// GFX12: v_maximumminimum_f16_e64_dpp v5.l, v1.l, 2.0, v3.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x6f,0xd6,0xfa,0xe8,0x0d,0x04,0x01,0x1b,0x00,0xff]
+
+v_maximumminimum_f16 v5.l, v1.l, v2.l, v3.l quad_perm:[0,1,2,3]
+// GFX12: v_maximumminimum_f16_e64_dpp v5.l, v1.l, v2.l, v3.l quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x6f,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
+
+v_maximumminimum_f16 v5.l, v1.l, v2.l, v3.l row_mirror
+// GFX12: v_maximumminimum_f16_e64_dpp v5.l, v1.l, v2.l, v3.l row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x6f,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
+
+v_maximumminimum_f16 v5.l, v1.l, v2.l, v255.l row_half_mirror
+// GFX12: v_maximumminimum_f16_e64_dpp v5.l, v1.l, v2.l, v255.l row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x6f,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff]
+
+v_maximumminimum_f16 v5.l, v1.l, v2.l, s105 row_shl:1
+// GFX12: v_maximumminimum_f16_e64_dpp v5.l, v1.l, v2.l, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x6f,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
+
+v_maximumminimum_f16 v5.l, v1.l, v2.l, vcc_hi row_shl:15
+// GFX12: v_maximumminimum_f16_e64_dpp v5.l, v1.l, v2.l, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x6f,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
+
+v_maximumminimum_f16 v5.l, v1.l, v2.l, vcc_lo row_shr:1
+// GFX12: v_maximumminimum_f16_e64_dpp v5.l, v1.l, v2.l, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x6f,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
+
+v_maximumminimum_f16 v5.l, |v1.l|, v2.l, -ttmp15 row_shr:15
+// GFX12: v_maximumminimum_f16_e64_dpp v5.l, |v1.l|, v2.l, -ttmp15 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x01,0x6f,0xd6,0xfa,0x04,0xee,0x81,0x01,0x1f,0x01,0xff]
+
+v_maximumminimum_f16 v5.l, v1.l, -|v2.l|, exec_hi row_ror:1
+// GFX12: v_maximumminimum_f16_e64_dpp v5.l, v1.l, -|v2.l|, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x02,0x6f,0xd6,0xfa,0x04,0xfe,0x41,0x01,0x21,0x01,0xff]
 
-v_maximumminimum_f16 v5, v1, s2, v3 quad_perm:[3,2,1,0]
-// GFX12: v_maximumminimum_f16_e64_dpp v5, v1, s2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x6f,0xd6,0xfa,0x04,0x0c,0x04,0x01,0x1b,0x00,0xff]
+v_maximumminimum_f16 v5.l, -v1.l, v2.l, |exec_lo| row_ror:15
+// GFX12: v_maximumminimum_f16_e64_dpp v5.l, -v1.l, v2.l, |exec_lo| row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x04,0x6f,0xd6,0xfa,0x04,0xfa,0x21,0x01,0x2f,0x01,0xff]
 
-v_maximumminimum_f16 v5, v1, 2.0, v3 quad_perm:[3,2,1,0]
-// GFX12: v_maximumminimum_f16_e64_dpp v5, v1, 2.0, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x6f,0xd6,0xfa,0xe8,0x0d,0x04,0x01,0x1b,0x00,0xff]
+v_maximumminimum_f16 v5.l, -|v1.l|, -|v2.l|, null row_share:0 row_mask:0xf bank_mask:0xf
+// GFX12: v_maximumminimum_f16_e64_dpp v5.l, -|v1.l|, -|v2.l|, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x03,0x6f,0xd6,0xfa,0x04,0xf2,0x61,0x01,0x50,0x01,0xff]
 
-v_maximumminimum_f16 v5, v1, v2, v3 quad_perm:[0,1,2,3]
-// GFX12: v_maximumminimum_f16_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x6f,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
+v_maximumminimum_f16 v5.l, -|v1.l|, v2.l, -|-1| row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX12: v_maximumminimum_f16_e64_dpp v5.l, -|v1.l|, v2.l, -|-1| row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x05,0x6f,0xd6,0xfa,0x04,0x06,0xa3,0x01,0x5f,0x01,0x01]
 
-v_maximumminimum_f16 v5, v1, v2, v3 row_mirror
-// GFX12: v_maximumminimum_f16_e64_dpp v5, v1, v2, v3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x6f,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
+v_maximumminimum_f16 v5.l, v1.l, -|v2.l|, -|0.5| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX12: v_maximumminimum_f16_e64_dpp v5.l, v1.l, -|v2.l|, -|0.5| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x05,0x06,0x6f,0xd6,0xfa,0x04,0xc2,0xc3,0x01,0x60,0x09,0x13]
 
-v_maximumminimum_f16 v5, v1, v2, v255 row_half_mirror
-// GFX12: v_maximumminimum_f16_e64_dpp v5, v1, v2, v255 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x6f,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff]
+v_maximumminimum_f16 v255.l, -|v255.l|, -|v255.l|, -|src_scc| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX12: v_maximumminimum_f16_e64_dpp v255.l, -|v255.l|, -|v255.l|, -|src_scc| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xff,0x87,0x6f,0xd6,0xfa,0xfe,0xf7,0xe3,0xff,0x6f,0x05,0x30]
 
-v_maximumminimum_f16 v5, v1, v2, s105 row_shl:1
-// GFX12: v_maximumminimum_f16_e64_dpp v5, v1, v2, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x6f,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
+v_maximumminimum_f16 v5.h, v1.h, v2.h, v3.h quad_perm:[3,2,1,0]
+// GFX12: v_maximumminimum_f16_e64_dpp v5.h, v1.h, v2.h, v3.h op_sel:[1,1,1,1] quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x78,0x6f,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 
-v_maximumminimum_f16 v5, v1, v2, vcc_hi row_shl:15
-// GFX12: v_maximumminimum_f16_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x6f,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
+v_maximumminimum_f16 v5.l, v1.l, v2.l, v255.h quad_perm:[0,1,2,3]
+// GFX12: v_maximumminimum_f16_e64_dpp v5.l, v1.l, v2.l, v255.h op_sel:[0,0,1,0] quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x20,0x6f,0xd6,0xfa,0x04,0xfe,0x07,0x01,0xe4,0x00,0xff]
 
-v_maximumminimum_f16 v5, v1, v2, vcc_lo row_shr:1
-// GFX12: v_maximumminimum_f16_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x6f,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
+v_maximumminimum_f16 v5.l, -v1.h, |v2.l|, -1 mul:2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX12: v_maximumminimum_f16_e64_dpp v5.l, -v1.h, |v2.l|, -1 op_sel:[1,0,0,0] mul:2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x0a,0x6f,0xd6,0xfa,0x04,0x06,0x2b,0x01,0x5f,0x01,0x01]
 
-v_maximumminimum_f16 v5, |v1|, v2, -ttmp15 row_shr:15
-// GFX12: v_maximumminimum_f16_e64_dpp v5, |v1|, v2, -ttmp15 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x01,0x6f,0xd6,0xfa,0x04,0xee,0x81,0x01,0x1f,0x01,0xff]
+v_maximumminimum_f16 v5.l, -|v1.l|, -|v2.h|, 0.5 mul:4 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX12: v_maximumminimum_f16_e64_dpp v5.l, -|v1.l|, -|v2.h|, 0.5 op_sel:[0,1,0,0] mul:4 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x05,0x13,0x6f,0xd6,0xfa,0x04,0xc2,0x73,0x01,0x60,0x09,0x13]
 
-v_maximumminimum_f16 v5, v1, -|v2|, exec_hi row_ror:1
-// GFX12: v_maximumminimum_f16_e64_dpp v5, v1, -|v2|, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x02,0x6f,0xd6,0xfa,0x04,0xfe,0x41,0x01,0x21,0x01,0xff]
+v_maximumminimum_f16 v255.h, -|v255.l|, -|v255.l|, -|src_scc| clamp div:2 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX12: v_maximumminimum_f16_e64_dpp v255.h, -|v255.l|, -|v255.l|, -|src_scc| op_sel:[0,0,0,1] clamp div:2 row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xff,0xc7,0x6f,0xd6,0xfa,0xfe,0xf7,0xfb,0xff,0x6f,0x05,0x30]
 
-v_maximumminimum_f16 v5, -v1, v2, |exec_lo| row_ror:15
-// GFX12: v_maximumminimum_f16_e64_dpp v5, -v1, v2, |exec_lo| row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x04,0x6f,0xd6,0xfa,0x04,0xfa,0x21,0x01,0x2f,0x01,0xff]
+v_minimummaximum_f16 v5.l, v1.l, v2.l, v3.l quad_perm:[3,2,1,0]
+// GFX12: v_minimummaximum_f16_e64_dpp v5.l, v1.l, v2.l, v3.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x6e,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 
-v_maximumminimum_f16 v5, -|v1|, -|v2|, null row_share:0 row_mask:0xf bank_mask:0xf
-// GFX12: v_maximumminimum_f16_e64_dpp v5, -|v1|, -|v2|, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x03,0x6f,0xd6,0xfa,0x04,0xf2,0x61,0x01,0x50,0x01,0xff]
+v_minimummaximum_f16 v5.l, v1.l, s2, v3.l quad_perm:[3,2,1,0]
+// GFX12: v_minimummaximum_f16_e64_dpp v5.l, v1.l, s2, v3.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x6e,0xd6,0xfa,0x04,0x0c,0x04,0x01,0x1b,0x00,0xff]
 
-v_maximumminimum_f16 v5, -|v1|, v2, -|-1| row_share:15 row_mask:0x0 bank_mask:0x1
-// GFX12: v_maximumminimum_f16_e64_dpp v5, -|v1|, v2, -|-1| row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x05,0x6f,0xd6,0xfa,0x04,0x06,0xa3,0x01,0x5f,0x01,0x01]
+v_minimummaximum_f16 v5.l, v1.l, 2.0, v3.l quad_perm:[3,2,1,0]
+// GFX12: v_minimummaximum_f16_e64_dpp v5.l, v1.l, 2.0, v3.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x6e,0xd6,0xfa,0xe8,0x0d,0x04,0x01,0x1b,0x00,0xff]
 
-v_maximumminimum_f16 v5, v1, -|v2|, -|0.5| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
-// GFX12: v_maximumminimum_f16_e64_dpp v5, v1, -|v2|, -|0.5| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x05,0x06,0x6f,0xd6,0xfa,0x04,0xc2,0xc3,0x01,0x60,0x09,0x13]
+v_minimummaximum_f16 v5.l, v1.l, v2.l, v3.l quad_perm:[0,1,2,3]
+// GFX12: v_minimummaximum_f16_e64_dpp v5.l, v1.l, v2.l, v3.l quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x6e,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
 
-v_maximumminimum_f16 v255, -|v255|, -|v255|, -|src_scc| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
-// GFX12: v_maximumminimum_f16_e64_dpp v255, -|v255|, -|v255|, -|src_scc| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xff,0x87,0x6f,0xd6,0xfa,0xfe,0xf7,0xe3,0xff,0x6f,0x05,0x30]
+v_minimummaximum_f16 v5.l, v1.l, v2.l, v3.l row_mirror
+// GFX12: v_minimummaximum_f16_e64_dpp v5.l, v1.l, v2.l, v3.l row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x6e,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
 
-v_minimummaximum_f16 v5, v1, v2, v3 quad_perm:[3,2,1,0]
-// GFX12: v_minimummaximum_f16_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x6e,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
+v_minimummaximum_f16 v5.l, v1.l, v2.l, v255.l row_half_mirror
+// GFX12: v_minimummaximum_f16_e64_dpp v5.l, v1.l, v2.l, v255.l row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x6e,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff]
 
-v_minimummaximum_f16 v5, v1, s2, v3 quad_perm:[3,2,1,0]
-// GFX12: v_minimummaximum_f16_e64_dpp v5, v1, s2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x6e,0xd6,0xfa,0x04,0x0c,0x04,0x01,0x1b,0x00,0xff]
+v_minimummaximum_f16 v5.l, v1.l, v2.l, s105 row_shl:1
+// GFX12: v_minimummaximum_f16_e64_dpp v5.l, v1.l, v2.l, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x6e,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
 
-v_minimummaximum_f16 v5, v1, 2.0, v3 quad_perm:[3,2,1,0]
-// GFX12: v_minimummaximum_f16_e64_dpp v5, v1, 2.0, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x6e,0xd6,0xfa,0xe8,0x0d,0x04,0x01,0x1b,0x00,0xff]
+v_minimummaximum_f16 v5.l, v1.l, v2.l, vcc_hi row_shl:15
+// GFX12: v_minimummaximum_f16_e64_dpp v5.l, v1.l, v2.l, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x6e,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
 
-v_minimummaximum_f16 v5, v1, v2, v3 quad_perm:[0,1,2,3]
-// GFX12: v_minimummaximum_f16_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x6e,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
+v_minimummaximum_f16 v5.l, v1.l, v2.l, vcc_lo row_shr:1
+// GFX12: v_minimummaximum_f16_e64_dpp v5.l, v1.l, v2.l, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x6e,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
 
-v_minimummaximum_f16 v5, v1, v2, v3 row_mirror
-// GFX12: v_minimummaximum_f16_e64_dpp v5, v1, v2, v3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x6e,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
+v_minimummaximum_f16 v5.l, |v1.l|, v2.l, -ttmp15 row_shr:15
+// GFX12: v_minimummaximum_f16_e64_dpp v5.l, |v1.l|, v2.l, -ttmp15 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x01,0x6e,0xd6,0xfa,0x04,0xee,0x81,0x01,0x1f,0x01,0xff]
 
-v_minimummaximum_f16 v5, v1, v2, v255 row_half_mirror
-// GFX12: v_minimummaximum_f16_e64_dpp v5, v1, v2, v255 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x6e,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff]
+v_minimummaximum_f16 v5.l, v1.l, -|v2.l|, exec_hi row_ror:1
+// GFX12: v_minimummaximum_f16_e64_dpp v5.l, v1.l, -|v2.l|, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x02,0x6e,0xd6,0xfa,0x04,0xfe,0x41,0x01,0x21,0x01,0xff]
 
-v_minimummaximum_f16 v5, v1, v2, s105 row_shl:1
-// GFX12: v_minimummaximum_f16_e64_dpp v5, v1, v2, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x6e,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
+v_minimummaximum_f16 v5.l, -v1.l, v2.l, |exec_lo| row_ror:15
+// GFX12: v_minimummaximum_f16_e64_dpp v5.l, -v1.l, v2.l, |exec_lo| row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x04,0x6e,0xd6,0xfa,0x04,0xfa,0x21,0x01,0x2f,0x01,0xff]
 
-v_minimummaximum_f16 v5, v1, v2, vcc_hi row_shl:15
-// GFX12: v_minimummaximum_f16_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x6e,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
+v_minimummaximum_f16 v5.l, -|v1.l|, -|v2.l|, null row_share:0 row_mask:0xf bank_mask:0xf
+// GFX12: v_minimummaximum_f16_e64_dpp v5.l, -|v1.l|, -|v2.l|, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x03,0x6e,0xd6,0xfa,0x04,0xf2,0x61,0x01,0x50,0x01,0xff]
 
-v_minimummaximum_f16 v5, v1, v2, vcc_lo row_shr:1
-// GFX12: v_minimummaximum_f16_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x6e,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
+v_minimummaximum_f16 v5.l, -|v1.l|, v2.l, -|-1| row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX12: v_minimummaximum_f16_e64_dpp v5.l, -|v1.l|, v2.l, -|-1| row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x05,0x6e,0xd6,0xfa,0x04,0x06,0xa3,0x01,0x5f,0x01,0x01]
 
-v_minimummaximum_f16 v5, |v1|, v2, -ttmp15 row_shr:15
-// GFX12: v_minimummaximum_f16_e64_dpp v5, |v1|, v2, -ttmp15 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x01,0x6e,0xd6,0xfa,0x04,0xee,0x81,0x01,0x1f,0x01,0xff]
+v_minimummaximum_f16 v5.l, v1.l, -|v2.l|, -|0.5| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX12: v_minimummaximum_f16_e64_dpp v5.l, v1.l, -|v2.l|, -|0.5| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x05,0x06,0x6e,0xd6,0xfa,0x04,0xc2,0xc3,0x01,0x60,0x09,0x13]
 
-v_minimummaximum_f16 v5, v1, -|v2|, exec_hi row_ror:1
-// GFX12: v_minimummaximum_f16_e64_dpp v5, v1, -|v2|, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x02,0x6e,0xd6,0xfa,0x04,0xfe,0x41,0x01,0x21,0x01,0xff]
+v_minimummaximum_f16 v255.l, -|v255.l|, -|v255.l|, -|src_scc| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX12: v_minimummaximum_f16_e64_dpp v255.l, -|v255.l|, -|v255.l|, -|src_scc| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xff,0x87,0x6e,0xd6,0xfa,0xfe,0xf7,0xe3,0xff,0x6f,0x05,0x30]
 
-v_minimummaximum_f16 v5, -v1, v2, |exec_lo| row_ror:15
-// GFX12: v_minimummaximum_f16_e64_dpp v5, -v1, v2, |exec_lo| row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x04,0x6e,0xd6,0xfa,0x04,0xfa,0x21,0x01,0x2f,0x01,0xff]
+v_minimummaximum_f16 v5.h, v1.h, v2.h, v3.h quad_perm:[3,2,1,0]
+// GFX12: v_minimummaximum_f16_e64_dpp v5.h, v1.h, v2.h, v3.h op_sel:[1,1,1,1] quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x78,0x6e,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 
-v_minimummaximum_f16 v5, -|v1|, -|v2|, null row_share:0 row_mask:0xf bank_mask:0xf
-// GFX12: v_minimummaximum_f16_e64_dpp v5, -|v1|, -|v2|, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x03,0x6e,0xd6,0xfa,0x04,0xf2,0x61,0x01,0x50,0x01,0xff]
+v_minimummaximum_f16 v5.l, v1.l, v2.l, v255.h quad_perm:[0,1,2,3]
+// GFX12: v_minimummaximum_f16_e64_dpp v5.l, v1.l, v2.l, v255.h op_sel:[0,0,1,0] quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x20,0x6e,0xd6,0xfa,0x04,0xfe,0x07,0x01,0xe4,0x00,0xff]
 
-v_minimummaximum_f16 v5, -|v1|, v2, -|-1| row_share:15 row_mask:0x0 bank_mask:0x1
-// GFX12: v_minimummaximum_f16_e64_dpp v5, -|v1|, v2, -|-1| row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x05,0x6e,0xd6,0xfa,0x04,0x06,0xa3,0x01,0x5f,0x01,0x01]
+v_minimummaximum_f16 v5.l, -v1.h, |v2.l|, -1 mul:2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX12: v_minimummaximum_f16_e64_dpp v5.l, -v1.h, |v2.l|, -1 op_sel:[1,0,0,0] mul:2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x0a,0x6e,0xd6,0xfa,0x04,0x06,0x2b,0x01,0x5f,0x01,0x01]
 
-v_minimummaximum_f16 v5, v1, -|v2|, -|0.5| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
-// GFX12: v_minimummaximum_f16_e64_dpp v5, v1, -|v2|, -|0.5| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x05,0x06,0x6e,0xd6,0xfa,0x04,0xc2,0xc3,0x01,0x60,0x09,0x13]
+v_minimummaximum_f16 v5.l, -|v1.l|, -|v2.h|, 0.5 mul:4 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX12: v_minimummaximum_f16_e64_dpp v5.l, -|v1.l|, -|v2.h|, 0.5 op_sel:[0,1,0,0] mul:4 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x05,0x13,0x6e,0xd6,0xfa,0x04,0xc2,0x73,0x01,0x60,0x09,0x13]
 
-v_minimummaximum_f16 v255, -|v255|, -|v255|, -|src_scc| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
-// GFX12: v_minimummaximum_f16_e64_dpp v255, -|v255|, -|v255|, -|src_scc| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xff,0x87,0x6e,0xd6,0xfa,0xfe,0xf7,0xe3,0xff,0x6f,0x05,0x30]
+v_minimummaximum_f16 v255.h, -|v255.l|, -|v255.l|, -|src_scc| clamp div:2 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX12: v_minimummaximum_f16_e64_dpp v255.h, -|v255.l|, -|v255.l|, -|src_scc| op_sel:[0,0,0,1] clamp div:2 row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xff,0xc7,0x6e,0xd6,0xfa,0xfe,0xf7,0xfb,0xff,0x6f,0x05,0x30]
diff --git a/llvm/test/MC/AMDGPU/gfx12_asm_vop3_dpp8.s b/llvm/test/MC/AMDGPU/gfx12_asm_vop3_dpp8.s
index b41f92b889368..0523b0fe8c9a4 100644
--- a/llvm/test/MC/AMDGPU/gfx12_asm_vop3_dpp8.s
+++ b/llvm/test/MC/AMDGPU/gfx12_asm_vop3_dpp8.s
@@ -288,6 +288,9 @@ v_alignbyte_b32_e64_dpp v5, v1, v2, -1 dpp8:[7,6,5,4,3,2,1,0] fi:1
 v_alignbyte_b32_e64_dpp v255, v255, v255, src_scc dpp8:[0,0,0,0,0,0,0,0] fi:0
 // GFX12: v_alignbyte_b32_e64_dpp v255, v255, v255, src_scc dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x00,0x17,0xd6,0xe9,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00]
 
+v_alignbyte_b32_e64_dpp v5, v1, v2, v255.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_alignbyte_b32_e64_dpp v5, v1, v2, v255.h op_sel:[0,0,1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x20,0x17,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
+
 v_and_b16_e64_dpp v5.l, v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0]
 // GFX12: v_and_b16_e64_dpp v5.l, v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x62,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 
@@ -1357,47 +1360,53 @@ v_mad_i16_e64_dpp v5.l, v1.l, v2.h, -1 dpp8:[7,6,5,4,3,2,1,0] fi:1
 v_mad_i16_e64_dpp v255.h, v255.l, v255.l, src_scc clamp dpp8:[0,0,0,0,0,0,0,0] fi:0
 // GFX12: v_mad_i16_e64_dpp v255.h, v255.l, v255.l, src_scc op_sel:[0,0,0,1] clamp dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0xc0,0x53,0xd6,0xe9,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00]
 
-v_mad_i32_i16_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_mad_i32_i16_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x5a,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
+v_mad_i32_i16_e64_dpp v5, v1.l, v2.l, v3 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_mad_i32_i16_e64_dpp v5, v1.l, v2.l, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x5a,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
+
+v_mad_i32_i16_e64_dpp v5, v1.l, s2, v3 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_mad_i32_i16_e64_dpp v5, v1.l, s2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x5a,0xd6,0xe9,0x04,0x0c,0x04,0x01,0x77,0x39,0x05]
 
-v_mad_i32_i16_e64_dpp v5, v1, s2, v3 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_mad_i32_i16_e64_dpp v5, v1, s2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x5a,0xd6,0xe9,0x04,0x0c,0x04,0x01,0x77,0x39,0x05]
+v_mad_i32_i16_e64_dpp v5, v1.l, 10, v3 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_mad_i32_i16_e64_dpp v5, v1.l, 10, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x5a,0xd6,0xe9,0x14,0x0d,0x04,0x01,0x77,0x39,0x05]
 
-v_mad_i32_i16_e64_dpp v5, v1, 10, v3 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_mad_i32_i16_e64_dpp v5, v1, 10, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x5a,0xd6,0xe9,0x14,0x0d,0x04,0x01,0x77,0x39,0x05]
+v_mad_i32_i16_e64_dpp v5, v1.l, v2.l, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_mad_i32_i16_e64_dpp v5, v1.l, v2.l, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x5a,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 
-v_mad_i32_i16_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_mad_i32_i16_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x5a,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
+v_mad_i32_i16_e64_dpp v5, v1.l, v2.l, s105 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_mad_i32_i16_e64_dpp v5, v1.l, v2.l, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x5a,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 
-v_mad_i32_i16_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_mad_i32_i16_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x5a,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
+v_mad_i32_i16_e64_dpp v5, v1.l, v2.l, vcc_hi dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_mad_i32_i16_e64_dpp v5, v1.l, v2.l, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x5a,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 
-v_mad_i32_i16_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_mad_i32_i16_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x5a,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
+v_mad_i32_i16_e64_dpp v5, v1.l, v2.l, vcc_lo dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_mad_i32_i16_e64_dpp v5, v1.l, v2.l, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x5a,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 
-v_mad_i32_i16_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_mad_i32_i16_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x5a,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
+v_mad_i32_i16_e64_dpp v5, v1.l, v2.l, ttmp15 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_mad_i32_i16_e64_dpp v5, v1.l, v2.l, ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x5a,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05]
 
-v_mad_i32_i16_e64_dpp v5, v1, v2, ttmp15 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_mad_i32_i16_e64_dpp v5, v1, v2, ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x5a,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05]
+v_mad_i32_i16_e64_dpp v5, v1.l, v2.l, exec_hi dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_mad_i32_i16_e64_dpp v5, v1.l, v2.l, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x5a,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05]
 
-v_mad_i32_i16_e64_dpp v5, v1, v2, exec_hi dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_mad_i32_i16_e64_dpp v5, v1, v2, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x5a,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05]
+v_mad_i32_i16_e64_dpp v5, v1.l, v2.l, exec_lo dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_mad_i32_i16_e64_dpp v5, v1.l, v2.l, exec_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x5a,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05]
 
-v_mad_i32_i16_e64_dpp v5, v1, v2, exec_lo dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_mad_i32_i16_e64_dpp v5, v1, v2, exec_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x5a,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05]
+v_mad_i32_i16_e64_dpp v5, v1.l, v2.l, null dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_mad_i32_i16_e64_dpp v5, v1.l, v2.l, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x5a,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05]
 
-v_mad_i32_i16_e64_dpp v5, v1, v2, null dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_mad_i32_i16_e64_dpp v5, v1, v2, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x5a,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05]
+v_mad_i32_i16_e64_dpp v5, v1.l, v2.l, -1 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_mad_i32_i16_e64_dpp v5, v1.l, v2.l, -1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x5a,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05]
 
-v_mad_i32_i16_e64_dpp v5, v1, v2, -1 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_mad_i32_i16_e64_dpp v5, v1, v2, -1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x5a,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05]
+v_mad_i32_i16_e64_dpp v5, v1.l, v2.l, 0.5 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX12: v_mad_i32_i16_e64_dpp v5, v1.l, v2.l, 0.5 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x05,0x00,0x5a,0xd6,0xea,0x04,0xc2,0x03,0x01,0x77,0x39,0x05]
 
-v_mad_i32_i16_e64_dpp v5, v1, v2, 0.5 dpp8:[7,6,5,4,3,2,1,0] fi:1
-// GFX12: v_mad_i32_i16_e64_dpp v5, v1, v2, 0.5 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x05,0x00,0x5a,0xd6,0xea,0x04,0xc2,0x03,0x01,0x77,0x39,0x05]
+v_mad_i32_i16_e64_dpp v255, v255.l, v255.l, src_scc clamp dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX12: v_mad_i32_i16_e64_dpp v255, v255.l, v255.l, src_scc clamp dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x80,0x5a,0xd6,0xe9,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00]
 
-v_mad_i32_i16_e64_dpp v255, v255, v255, src_scc clamp dpp8:[0,0,0,0,0,0,0,0] fi:0
-// GFX12: v_mad_i32_i16_e64_dpp v255, v255, v255, src_scc clamp dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x80,0x5a,0xd6,0xe9,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00]
+v_mad_i32_i16_e64_dpp v5, v1.h, v2.l, 0.5 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX12: v_mad_i32_i16_e64_dpp v5, v1.h, v2.l, 0.5 op_sel:[1,0,0,0] dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x05,0x08,0x5a,0xd6,0xea,0x04,0xc2,0x03,0x01,0x77,0x39,0x05]
+
+v_mad_i32_i16_e64_dpp v255, v255.l, v255.h, src_scc clamp dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX12: v_mad_i32_i16_e64_dpp v255, v255.l, v255.h, src_scc op_sel:[0,1,0,0] clamp dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x90,0x5a,0xd6,0xe9,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00]
 
 v_mad_i32_i24_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0]
 // GFX12: v_mad_i32_i24_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0a,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
@@ -1495,47 +1504,53 @@ v_mad_u16_e64_dpp v5.l, v1.l, v2.h, -1 dpp8:[7,6,5,4,3,2,1,0] fi:1
 v_mad_u16_e64_dpp v255.h, v255.l, v255.l, src_scc clamp dpp8:[0,0,0,0,0,0,0,0] fi:0
 // GFX12: v_mad_u16_e64_dpp v255.h, v255.l, v255.l, src_scc op_sel:[0,0,0,1] clamp dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0xc0,0x41,0xd6,0xe9,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00]
 
-v_mad_u32_u16_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_mad_u32_u16_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x59,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
+v_mad_u32_u16_e64_dpp v5, v1.l, v2.l, v3 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_mad_u32_u16_e64_dpp v5, v1.l, v2.l, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x59,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
+
+v_mad_u32_u16_e64_dpp v5, v1.l, s2, v3 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_mad_u32_u16_e64_dpp v5, v1.l, s2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x59,0xd6,0xe9,0x04,0x0c,0x04,0x01,0x77,0x39,0x05]
 
-v_mad_u32_u16_e64_dpp v5, v1, s2, v3 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_mad_u32_u16_e64_dpp v5, v1, s2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x59,0xd6,0xe9,0x04,0x0c,0x04,0x01,0x77,0x39,0x05]
+v_mad_u32_u16_e64_dpp v5, v1.l, 10, v3 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_mad_u32_u16_e64_dpp v5, v1.l, 10, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x59,0xd6,0xe9,0x14,0x0d,0x04,0x01,0x77,0x39,0x05]
 
-v_mad_u32_u16_e64_dpp v5, v1, 10, v3 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_mad_u32_u16_e64_dpp v5, v1, 10, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x59,0xd6,0xe9,0x14,0x0d,0x04,0x01,0x77,0x39,0x05]
+v_mad_u32_u16_e64_dpp v5, v1.l, v2.l, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_mad_u32_u16_e64_dpp v5, v1.l, v2.l, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x59,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 
-v_mad_u32_u16_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_mad_u32_u16_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x59,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
+v_mad_u32_u16_e64_dpp v5, v1.l, v2.l, s105 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_mad_u32_u16_e64_dpp v5, v1.l, v2.l, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x59,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 
-v_mad_u32_u16_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_mad_u32_u16_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x59,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
+v_mad_u32_u16_e64_dpp v5, v1.l, v2.l, vcc_hi dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_mad_u32_u16_e64_dpp v5, v1.l, v2.l, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x59,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 
-v_mad_u32_u16_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_mad_u32_u16_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x59,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
+v_mad_u32_u16_e64_dpp v5, v1.l, v2.l, vcc_lo dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_mad_u32_u16_e64_dpp v5, v1.l, v2.l, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x59,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 
-v_mad_u32_u16_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_mad_u32_u16_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x59,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
+v_mad_u32_u16_e64_dpp v5, v1.l, v2.l, ttmp15 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_mad_u32_u16_e64_dpp v5, v1.l, v2.l, ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x59,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05]
 
-v_mad_u32_u16_e64_dpp v5, v1, v2, ttmp15 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_mad_u32_u16_e64_dpp v5, v1, v2, ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x59,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05]
+v_mad_u32_u16_e64_dpp v5, v1.l, v2.l, exec_hi dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_mad_u32_u16_e64_dpp v5, v1.l, v2.l, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x59,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05]
 
-v_mad_u32_u16_e64_dpp v5, v1, v2, exec_hi dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_mad_u32_u16_e64_dpp v5, v1, v2, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x59,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05]
+v_mad_u32_u16_e64_dpp v5, v1.l, v2.l, exec_lo dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_mad_u32_u16_e64_dpp v5, v1.l, v2.l, exec_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x59,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05]
 
-v_mad_u32_u16_e64_dpp v5, v1, v2, exec_lo dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_mad_u32_u16_e64_dpp v5, v1, v2, exec_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x59,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05]
+v_mad_u32_u16_e64_dpp v5, v1.l, v2.l, null dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_mad_u32_u16_e64_dpp v5, v1.l, v2.l, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x59,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05]
 
-v_mad_u32_u16_e64_dpp v5, v1, v2, null dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_mad_u32_u16_e64_dpp v5, v1, v2, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x59,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05]
+v_mad_u32_u16_e64_dpp v5, v1.l, v2.l, -1 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_mad_u32_u16_e64_dpp v5, v1.l, v2.l, -1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x59,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05]
 
-v_mad_u32_u16_e64_dpp v5, v1, v2, -1 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_mad_u32_u16_e64_dpp v5, v1, v2, -1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x59,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05]
+v_mad_u32_u16_e64_dpp v5, v1.l, v2.l, 0.5 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX12: v_mad_u32_u16_e64_dpp v5, v1.l, v2.l, 0.5 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x05,0x00,0x59,0xd6,0xea,0x04,0xc2,0x03,0x01,0x77,0x39,0x05]
 
-v_mad_u32_u16_e64_dpp v5, v1, v2, 0.5 dpp8:[7,6,5,4,3,2,1,0] fi:1
-// GFX12: v_mad_u32_u16_e64_dpp v5, v1, v2, 0.5 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x05,0x00,0x59,0xd6,0xea,0x04,0xc2,0x03,0x01,0x77,0x39,0x05]
+v_mad_u32_u16_e64_dpp v255, v255.l, v255.l, src_scc clamp dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX12: v_mad_u32_u16_e64_dpp v255, v255.l, v255.l, src_scc clamp dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x80,0x59,0xd6,0xe9,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00]
 
-v_mad_u32_u16_e64_dpp v255, v255, v255, src_scc clamp dpp8:[0,0,0,0,0,0,0,0] fi:0
-// GFX12: v_mad_u32_u16_e64_dpp v255, v255, v255, src_scc clamp dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x80,0x59,0xd6,0xe9,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00]
+v_mad_u32_u16_e64_dpp v5, v1.h, v2.l, 0.5 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX12: v_mad_u32_u16_e64_dpp v5, v1.h, v2.l, 0.5 op_sel:[1,0,0,0] dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x05,0x08,0x59,0xd6,0xea,0x04,0xc2,0x03,0x01,0x77,0x39,0x05]
+
+v_mad_u32_u16_e64_dpp v255, v255.l, v255.h, src_scc clamp dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX12: v_mad_u32_u16_e64_dpp v255, v255.l, v255.h, src_scc op_sel:[0,1,0,0] clamp dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x90,0x59,0xd6,0xe9,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00]
 
 v_mad_u32_u24_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0]
 // GFX12: v_mad_u32_u24_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0b,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
@@ -3656,11 +3671,11 @@ v_mad_i16_e64_dpp v5.l, v1.l, v2.l, -1 op_sel:[0,0,1,0] dpp8:[7,6,5,4,3,2,1,0]
 v_mad_i16_e64_dpp v255.h, v255.l, v255.l, src_scc op_sel:[0,0,0,1] clamp dpp8:[0,0,0,0,0,0,0,0] fi:1
 // GFX12: v_mad_i16_e64_dpp v255.h, v255.l, v255.l, src_scc op_sel:[0,0,0,1] clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0xc0,0x53,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00]
 
-v_mad_i32_i16_e64_dpp v5, v1, v2, 0.5 op_sel:[1,0,0,0] dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_mad_i32_i16_e64_dpp v5, v1, v2, 0.5 op_sel:[1,0,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x08,0x5a,0xd6,0xe9,0x04,0xc2,0x03,0x01,0x77,0x39,0x05]
+v_mad_i32_i16_e64_dpp v5, v1.h, v2.l, 0.5 op_sel:[1,0,0,0] dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_mad_i32_i16_e64_dpp v5, v1.h, v2.l, 0.5 op_sel:[1,0,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x08,0x5a,0xd6,0xe9,0x04,0xc2,0x03,0x01,0x77,0x39,0x05]
 
-v_mad_i32_i16_e64_dpp v255, v255, v255, src_scc op_sel:[0,1,0,0] clamp dpp8:[0,0,0,0,0,0,0,0] fi:1
-// GFX12: v_mad_i32_i16_e64_dpp v255, v255, v255, src_scc op_sel:[0,1,0,0] clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x90,0x5a,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00]
+v_mad_i32_i16_e64_dpp v255, v255.l, v255.h, src_scc op_sel:[0,1,0,0] clamp dpp8:[0,0,0,0,0,0,0,0] fi:1
+// GFX12: v_mad_i32_i16_e64_dpp v255, v255.l, v255.h, src_scc op_sel:[0,1,0,0] clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x90,0x5a,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00]
 
 v_mad_u16_e64_dpp v5.h, v1.h, v2.h, exec_hi op_sel:[1,1,1,1] dpp8:[7,6,5,4,3,2,1,0]
 // GFX12: v_mad_u16_e64_dpp v5.h, v1.h, v2.h, exec_hi op_sel:[1,1,1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x78,0x41,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05]
@@ -3677,11 +3692,11 @@ v_mad_u16_e64_dpp v5.l, v1.l, v2.l, -1 op_sel:[0,0,1,0] dpp8:[7,6,5,4,3,2,1,0]
 v_mad_u16_e64_dpp v255.h, v255.l, v255.l, src_scc op_sel:[0,0,0,1] clamp dpp8:[0,0,0,0,0,0,0,0] fi:1
 // GFX12: v_mad_u16_e64_dpp v255.h, v255.l, v255.l, src_scc op_sel:[0,0,0,1] clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0xc0,0x41,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00]
 
-v_mad_u32_u16_e64_dpp v5, v1, v2, 0.5 op_sel:[1,0,0,0] dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_mad_u32_u16_e64_dpp v5, v1, v2, 0.5 op_sel:[1,0,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x08,0x59,0xd6,0xe9,0x04,0xc2,0x03,0x01,0x77,0x39,0x05]
+v_mad_u32_u16_e64_dpp v5, v1.h, v2.l, 0.5 op_sel:[1,0,0,0] dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_mad_u32_u16_e64_dpp v5, v1.h, v2.l, 0.5 op_sel:[1,0,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x08,0x59,0xd6,0xe9,0x04,0xc2,0x03,0x01,0x77,0x39,0x05]
 
-v_mad_u32_u16_e64_dpp v255, v255, v255, src_scc op_sel:[0,1,0,0] clamp dpp8:[0,0,0,0,0,0,0,0] fi:1
-// GFX12: v_mad_u32_u16_e64_dpp v255, v255, v255, src_scc op_sel:[0,1,0,0] clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x90,0x59,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00]
+v_mad_u32_u16_e64_dpp v255, v255.l, v255.h, src_scc op_sel:[0,1,0,0] clamp dpp8:[0,0,0,0,0,0,0,0] fi:1
+// GFX12: v_mad_u32_u16_e64_dpp v255, v255.l, v255.h, src_scc op_sel:[0,1,0,0] clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x90,0x59,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00]
 
 v_max3_num_f16_e64_dpp v5.h, -v1.h, v2.h, |exec_lo| op_sel:[1,1,1,1] dpp8:[7,6,5,4,3,2,1,0]
 // GFX12: v_max3_num_f16_e64_dpp v5.h, -v1.h, v2.h, |exec_lo| op_sel:[1,1,1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x7c,0x2c,0xd6,0xe9,0x04,0xfa,0x21,0x01,0x77,0x39,0x05]
@@ -3902,29 +3917,35 @@ v_maximum_f32 v5, -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1
 v_maximum_f32 v255, -|v255|, -|v255| dpp8:[0,0,0,0,0,0,0,0] fi:0
 // GFX12: v_maximum_f32_e64_dpp v255, -|v255|, -|v255| dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x03,0x66,0xd7,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
 
-v_minimum_f16 v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_minimum_f16_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x67,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+v_minimum_f16 v5.l, v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_minimum_f16_e64_dpp v5.l, v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x67,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+
+v_minimum_f16 v5.l, |v1.l|, -v2.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_minimum_f16_e64_dpp v5.l, |v1.l|, -v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x67,0xd7,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+
+v_minimum_f16 v5.l, -v1.l, |v2.l| dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX12: v_minimum_f16_e64_dpp v5.l, -v1.l, |v2.l| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x05,0x02,0x67,0xd7,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
 
-v_minimum_f16 v5, |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_minimum_f16_e64_dpp v5, |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x67,0xd7,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+v_minimum_f16 v255.l, -|v255.l|, -|v255.l| dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX12: v_minimum_f16_e64_dpp v255.l, -|v255.l|, -|v255.l| dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x03,0x67,0xd7,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
 
-v_minimum_f16 v5, -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1
-// GFX12: v_minimum_f16_e64_dpp v5, -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x05,0x02,0x67,0xd7,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+v_minimum_f16 v255.h, -|v255.h|, -|v255.h| dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX12: v_minimum_f16_e64_dpp v255.h, -|v255.h|, -|v255.h| op_sel:[1,1,1] dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x5b,0x67,0xd7,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
 
-v_minimum_f16 v255, -|v255|, -|v255| dpp8:[0,0,0,0,0,0,0,0] fi:0
-// GFX12: v_minimum_f16_e64_dpp v255, -|v255|, -|v255| dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x03,0x67,0xd7,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+v_maximum_f16 v5.l, v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_maximum_f16_e64_dpp v5.l, v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x68,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 
-v_maximum_f16 v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_maximum_f16_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x68,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+v_maximum_f16 v5.l, |v1.l|, -v2.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_maximum_f16_e64_dpp v5.l, |v1.l|, -v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x68,0xd7,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
 
-v_maximum_f16 v5, |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_maximum_f16_e64_dpp v5, |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x68,0xd7,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+v_maximum_f16 v5.l, -v1.l, |v2.l| dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX12: v_maximum_f16_e64_dpp v5.l, -v1.l, |v2.l| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x05,0x02,0x68,0xd7,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
 
-v_maximum_f16 v5, -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1
-// GFX12: v_maximum_f16_e64_dpp v5, -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x05,0x02,0x68,0xd7,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+v_maximum_f16 v255.l, -|v255.l|, -|v255.l| dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX12: v_maximum_f16_e64_dpp v255.l, -|v255.l|, -|v255.l| dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x03,0x68,0xd7,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
 
-v_maximum_f16 v255, -|v255|, -|v255| dpp8:[0,0,0,0,0,0,0,0] fi:0
-// GFX12: v_maximum_f16_e64_dpp v255, -|v255|, -|v255| dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x03,0x68,0xd7,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+v_maximum_f16 v255.h, -|v255.h|, -|v255.h| dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX12: v_maximum_f16_e64_dpp v255.h, -|v255.h|, -|v255.h| op_sel:[1,1,1] dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x5b,0x68,0xd7,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
 
 v_minimum3_f32 v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0]
 // GFX12: v_minimum3_f32_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x2d,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
@@ -4010,89 +4031,119 @@ v_maximum3_f32 v5, v1, -|v2|, -|0.5| mul:4 dpp8:[7,6,5,4,3,2,1,0] fi:1
 v_maximum3_f32 v255, -|v255|, -|v255|, -|src_scc| clamp div:2 dpp8:[0,0,0,0,0,0,0,0] fi:0
 // GFX12: v_maximum3_f32_e64_dpp v255, -|v255|, -|v255|, -|src_scc| clamp div:2 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x87,0x2e,0xd6,0xe9,0xfe,0xf7,0xfb,0xff,0x00,0x00,0x00]
 
-v_minimum3_f16 v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_minimum3_f16_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x2f,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
+v_minimum3_f16 v5.l, v1.l, v2.l, v3.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_minimum3_f16_e64_dpp v5.l, v1.l, v2.l, v3.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x2f,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
+
+v_minimum3_f16 v5.l, v1.l, s2, v3.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_minimum3_f16_e64_dpp v5.l, v1.l, s2, v3.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x2f,0xd6,0xe9,0x04,0x0c,0x04,0x01,0x77,0x39,0x05]
+
+v_minimum3_f16 v5.l, v1.l, 2.0, v3.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_minimum3_f16_e64_dpp v5.l, v1.l, 2.0, v3.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x2f,0xd6,0xe9,0xe8,0x0d,0x04,0x01,0x77,0x39,0x05]
+
+v_minimum3_f16 v5.l, v1.l, v2.l, v255.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_minimum3_f16_e64_dpp v5.l, v1.l, v2.l, v255.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x2f,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
+
+v_minimum3_f16 v5.l, v1.l, v2.l, s105 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_minimum3_f16_e64_dpp v5.l, v1.l, v2.l, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x2f,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
+
+v_minimum3_f16 v5.l, v1.l, v2.l, vcc_hi dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_minimum3_f16_e64_dpp v5.l, v1.l, v2.l, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x2f,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
+
+v_minimum3_f16 v5.l, v1.l, v2.l, vcc_lo dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_minimum3_f16_e64_dpp v5.l, v1.l, v2.l, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x2f,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
+
+v_minimum3_f16 v5.l, |v1.l|, v2.l, -ttmp15 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_minimum3_f16_e64_dpp v5.l, |v1.l|, v2.l, -ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x2f,0xd6,0xe9,0x04,0xee,0x81,0x01,0x77,0x39,0x05]
 
-v_minimum3_f16 v5, v1, s2, v3 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_minimum3_f16_e64_dpp v5, v1, s2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x2f,0xd6,0xe9,0x04,0x0c,0x04,0x01,0x77,0x39,0x05]
+v_minimum3_f16 v5.l, v1.l, -|v2.l|, exec_hi dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_minimum3_f16_e64_dpp v5.l, v1.l, -|v2.l|, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x02,0x2f,0xd6,0xe9,0x04,0xfe,0x41,0x01,0x77,0x39,0x05]
 
-v_minimum3_f16 v5, v1, 2.0, v3 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_minimum3_f16_e64_dpp v5, v1, 2.0, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x2f,0xd6,0xe9,0xe8,0x0d,0x04,0x01,0x77,0x39,0x05]
+v_minimum3_f16 v5.l, -v1.l, v2.l, |exec_lo| dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_minimum3_f16_e64_dpp v5.l, -v1.l, v2.l, |exec_lo| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x04,0x2f,0xd6,0xe9,0x04,0xfa,0x21,0x01,0x77,0x39,0x05]
 
-v_minimum3_f16 v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_minimum3_f16_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x2f,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
+v_minimum3_f16 v5.l, -|v1.l|, -|v2.l|, null dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_minimum3_f16_e64_dpp v5.l, -|v1.l|, -|v2.l|, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x03,0x2f,0xd6,0xe9,0x04,0xf2,0x61,0x01,0x77,0x39,0x05]
 
-v_minimum3_f16 v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_minimum3_f16_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x2f,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
+v_minimum3_f16 v5.l, -|v1.l|, v2.l, -|-1| dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_minimum3_f16_e64_dpp v5.l, -|v1.l|, v2.l, -|-1| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x05,0x2f,0xd6,0xe9,0x04,0x06,0xa3,0x01,0x77,0x39,0x05]
 
-v_minimum3_f16 v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_minimum3_f16_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x2f,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
+v_minimum3_f16 v5.l, v1.l, -|v2.l|, -|0.5| dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX12: v_minimum3_f16_e64_dpp v5.l, v1.l, -|v2.l|, -|0.5| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x05,0x06,0x2f,0xd6,0xea,0x04,0xc2,0xc3,0x01,0x77,0x39,0x05]
 
-v_minimum3_f16 v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_minimum3_f16_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x2f,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
+v_minimum3_f16 v255.l, -|v255.l|, -|v255.l|, -|src_scc| clamp dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX12: v_minimum3_f16_e64_dpp v255.l, -|v255.l|, -|v255.l|, -|src_scc| clamp dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x87,0x2f,0xd6,0xe9,0xfe,0xf7,0xe3,0xff,0x00,0x00,0x00]
 
-v_minimum3_f16 v5, |v1|, v2, -ttmp15 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_minimum3_f16_e64_dpp v5, |v1|, v2, -ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x2f,0xd6,0xe9,0x04,0xee,0x81,0x01,0x77,0x39,0x05]
+v_minimum3_f16 v5.h, v1.h, v2.h, v3.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_minimum3_f16_e64_dpp v5.h, v1.h, v2.h, v3.h op_sel:[1,1,1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x78,0x2f,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 
-v_minimum3_f16 v5, v1, -|v2|, exec_hi dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_minimum3_f16_e64_dpp v5, v1, -|v2|, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x02,0x2f,0xd6,0xe9,0x04,0xfe,0x41,0x01,0x77,0x39,0x05]
+v_minimum3_f16 v5.l, v1.l, v2.l, v255.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_minimum3_f16_e64_dpp v5.l, v1.l, v2.l, v255.h op_sel:[0,0,1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x20,0x2f,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 
-v_minimum3_f16 v5, -v1, v2, |exec_lo| dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_minimum3_f16_e64_dpp v5, -v1, v2, |exec_lo| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x04,0x2f,0xd6,0xe9,0x04,0xfa,0x21,0x01,0x77,0x39,0x05]
+v_minimum3_f16 v5.l, -v1.h, |v2.l|, -1 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_minimum3_f16_e64_dpp v5.l, -v1.h, |v2.l|, -1 op_sel:[1,0,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x0a,0x2f,0xd6,0xe9,0x04,0x06,0x23,0x01,0x77,0x39,0x05]
 
-v_minimum3_f16 v5, -|v1|, -|v2|, null dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_minimum3_f16_e64_dpp v5, -|v1|, -|v2|, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x03,0x2f,0xd6,0xe9,0x04,0xf2,0x61,0x01,0x77,0x39,0x05]
+v_minimum3_f16 v5.l, -|v1.l|, -|v2.h|, 0.5 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX12: v_minimum3_f16_e64_dpp v5.l, -|v1.l|, -|v2.h|, 0.5 op_sel:[0,1,0,0] dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x05,0x13,0x2f,0xd6,0xea,0x04,0xc2,0x63,0x01,0x77,0x39,0x05]
 
-v_minimum3_f16 v5, -|v1|, v2, -|-1| dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_minimum3_f16_e64_dpp v5, -|v1|, v2, -|-1| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x05,0x2f,0xd6,0xe9,0x04,0x06,0xa3,0x01,0x77,0x39,0x05]
+v_minimum3_f16 v255.h, -|v255.l|, -|v255.l|, -|src_scc| clamp dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX12: v_minimum3_f16_e64_dpp v255.h, -|v255.l|, -|v255.l|, -|src_scc| op_sel:[0,0,0,1] clamp dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0xc7,0x2f,0xd6,0xe9,0xfe,0xf7,0xe3,0xff,0x00,0x00,0x00]
 
-v_minimum3_f16 v5, v1, -|v2|, -|0.5| dpp8:[7,6,5,4,3,2,1,0] fi:1
-// GFX12: v_minimum3_f16_e64_dpp v5, v1, -|v2|, -|0.5| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x05,0x06,0x2f,0xd6,0xea,0x04,0xc2,0xc3,0x01,0x77,0x39,0x05]
+v_maximum3_f16 v5.l, v1.l, v2.l, v3.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_maximum3_f16_e64_dpp v5.l, v1.l, v2.l, v3.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x30,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 
-v_minimum3_f16 v255, -|v255|, -|v255|, -|src_scc| clamp dpp8:[0,0,0,0,0,0,0,0] fi:0
-// GFX12: v_minimum3_f16_e64_dpp v255, -|v255|, -|v255|, -|src_scc| clamp dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x87,0x2f,0xd6,0xe9,0xfe,0xf7,0xe3,0xff,0x00,0x00,0x00]
+v_maximum3_f16 v5.l, v1.l, s2, v3.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_maximum3_f16_e64_dpp v5.l, v1.l, s2, v3.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x30,0xd6,0xe9,0x04,0x0c,0x04,0x01,0x77,0x39,0x05]
 
-v_maximum3_f16 v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_maximum3_f16_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x30,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
+v_maximum3_f16 v5.l, v1.l, 2.0, v3.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_maximum3_f16_e64_dpp v5.l, v1.l, 2.0, v3.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x30,0xd6,0xe9,0xe8,0x0d,0x04,0x01,0x77,0x39,0x05]
 
-v_maximum3_f16 v5, v1, s2, v3 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_maximum3_f16_e64_dpp v5, v1, s2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x30,0xd6,0xe9,0x04,0x0c,0x04,0x01,0x77,0x39,0x05]
+v_maximum3_f16 v5.l, v1.l, v2.l, v255.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_maximum3_f16_e64_dpp v5.l, v1.l, v2.l, v255.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x30,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 
-v_maximum3_f16 v5, v1, 2.0, v3 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_maximum3_f16_e64_dpp v5, v1, 2.0, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x30,0xd6,0xe9,0xe8,0x0d,0x04,0x01,0x77,0x39,0x05]
+v_maximum3_f16 v5.l, v1.l, v2.l, s105 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_maximum3_f16_e64_dpp v5.l, v1.l, v2.l, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x30,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 
-v_maximum3_f16 v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_maximum3_f16_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x30,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
+v_maximum3_f16 v5.l, v1.l, v2.l, vcc_hi dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_maximum3_f16_e64_dpp v5.l, v1.l, v2.l, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x30,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 
-v_maximum3_f16 v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_maximum3_f16_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x30,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
+v_maximum3_f16 v5.l, v1.l, v2.l, vcc_lo dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_maximum3_f16_e64_dpp v5.l, v1.l, v2.l, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x30,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 
-v_maximum3_f16 v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_maximum3_f16_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x30,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
+v_maximum3_f16 v5.l, |v1.l|, v2.l, -ttmp15 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_maximum3_f16_e64_dpp v5.l, |v1.l|, v2.l, -ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x30,0xd6,0xe9,0x04,0xee,0x81,0x01,0x77,0x39,0x05]
 
-v_maximum3_f16 v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_maximum3_f16_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x30,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
+v_maximum3_f16 v5.l, v1.l, -|v2.l|, exec_hi dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_maximum3_f16_e64_dpp v5.l, v1.l, -|v2.l|, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x02,0x30,0xd6,0xe9,0x04,0xfe,0x41,0x01,0x77,0x39,0x05]
 
-v_maximum3_f16 v5, |v1|, v2, -ttmp15 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_maximum3_f16_e64_dpp v5, |v1|, v2, -ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x30,0xd6,0xe9,0x04,0xee,0x81,0x01,0x77,0x39,0x05]
+v_maximum3_f16 v5.l, -v1.l, v2.l, |exec_lo| dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_maximum3_f16_e64_dpp v5.l, -v1.l, v2.l, |exec_lo| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x04,0x30,0xd6,0xe9,0x04,0xfa,0x21,0x01,0x77,0x39,0x05]
 
-v_maximum3_f16 v5, v1, -|v2|, exec_hi dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_maximum3_f16_e64_dpp v5, v1, -|v2|, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x02,0x30,0xd6,0xe9,0x04,0xfe,0x41,0x01,0x77,0x39,0x05]
+v_maximum3_f16 v5.l, -|v1.l|, -|v2.l|, null dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_maximum3_f16_e64_dpp v5.l, -|v1.l|, -|v2.l|, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x03,0x30,0xd6,0xe9,0x04,0xf2,0x61,0x01,0x77,0x39,0x05]
 
-v_maximum3_f16 v5, -v1, v2, |exec_lo| dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_maximum3_f16_e64_dpp v5, -v1, v2, |exec_lo| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x04,0x30,0xd6,0xe9,0x04,0xfa,0x21,0x01,0x77,0x39,0x05]
+v_maximum3_f16 v5.l, -|v1.l|, v2.l, -|-1| dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_maximum3_f16_e64_dpp v5.l, -|v1.l|, v2.l, -|-1| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x05,0x30,0xd6,0xe9,0x04,0x06,0xa3,0x01,0x77,0x39,0x05]
 
-v_maximum3_f16 v5, -|v1|, -|v2|, null dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_maximum3_f16_e64_dpp v5, -|v1|, -|v2|, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x03,0x30,0xd6,0xe9,0x04,0xf2,0x61,0x01,0x77,0x39,0x05]
+v_maximum3_f16 v5.l, v1.l, -|v2.l| -|0.5| dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX12: v_maximum3_f16_e64_dpp v5.l, v1.l, -|v2.l|, -|0.5| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x05,0x06,0x30,0xd6,0xea,0x04,0xc2,0xc3,0x01,0x77,0x39,0x05]
 
-v_maximum3_f16 v5, -|v1|, v2, -|-1| dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_maximum3_f16_e64_dpp v5, -|v1|, v2, -|-1| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x05,0x30,0xd6,0xe9,0x04,0x06,0xa3,0x01,0x77,0x39,0x05]
+v_maximum3_f16 v255.l, -|v255.l|, -|v255.l|, -|src_scc| clamp dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX12: v_maximum3_f16_e64_dpp v255.l, -|v255.l|, -|v255.l|, -|src_scc| clamp dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x87,0x30,0xd6,0xe9,0xfe,0xf7,0xe3,0xff,0x00,0x00,0x00]
 
-v_maximum3_f16 v5, v1, -|v2|, -|0.5| dpp8:[7,6,5,4,3,2,1,0] fi:1
-// GFX12: v_maximum3_f16_e64_dpp v5, v1, -|v2|, -|0.5| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x05,0x06,0x30,0xd6,0xea,0x04,0xc2,0xc3,0x01,0x77,0x39,0x05]
+v_maximum3_f16 v5.h, v1.h, v2.h, v3.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_maximum3_f16_e64_dpp v5.h, v1.h, v2.h, v3.h op_sel:[1,1,1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x78,0x30,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 
-v_maximum3_f16 v255, -|v255|, -|v255|, -|src_scc| clamp dpp8:[0,0,0,0,0,0,0,0] fi:0
-// GFX12: v_maximum3_f16_e64_dpp v255, -|v255|, -|v255|, -|src_scc| clamp dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x87,0x30,0xd6,0xe9,0xfe,0xf7,0xe3,0xff,0x00,0x00,0x00]
+v_maximum3_f16 v5.l, v1.l, v2.l, v255.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_maximum3_f16_e64_dpp v5.l, v1.l, v2.l, v255.h op_sel:[0,0,1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x20,0x30,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
+
+v_maximum3_f16 v5.l, -v1.h, |v2.l|, -1 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_maximum3_f16_e64_dpp v5.l, -v1.h, |v2.l|, -1 op_sel:[1,0,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x0a,0x30,0xd6,0xe9,0x04,0x06,0x23,0x01,0x77,0x39,0x05]
+
+v_maximum3_f16 v5.l, -|v1.l|, -|v2.h|, 0.5 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX12: v_maximum3_f16_e64_dpp v5.l, -|v1.l|, -|v2.h|, 0.5 op_sel:[0,1,0,0] dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x05,0x13,0x30,0xd6,0xea,0x04,0xc2,0x63,0x01,0x77,0x39,0x05]
+
+v_maximum3_f16 v255.h, -|v255.l|, -|v255.l|, -|src_scc| clamp dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX12: v_maximum3_f16_e64_dpp v255.h, -|v255.l|, -|v255.l|, -|src_scc| op_sel:[0,0,0,1] clamp dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0xc7,0x30,0xd6,0xe9,0xfe,0xf7,0xe3,0xff,0x00,0x00,0x00]
 
 v_maximumminimum_f32 v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0]
 // GFX12: v_maximumminimum_f32_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x6d,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
@@ -4178,86 +4229,116 @@ v_minimummaximum_f32 v5, v1, -|v2|, -|0.5| mul:4 dpp8:[7,6,5,4,3,2,1,0] fi:1
 v_minimummaximum_f32 v255, -|v255|, -|v255|, -|src_scc| clamp div:2 dpp8:[0,0,0,0,0,0,0,0] fi:0
 // GFX12: v_minimummaximum_f32_e64_dpp v255, -|v255|, -|v255|, -|src_scc| clamp div:2 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x87,0x6c,0xd6,0xe9,0xfe,0xf7,0xfb,0xff,0x00,0x00,0x00]
 
-v_maximumminimum_f16 v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_maximumminimum_f16_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x6f,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
+v_maximumminimum_f16 v5.l, v1.l, v2.l, v3.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_maximumminimum_f16_e64_dpp v5.l, v1.l, v2.l, v3.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x6f,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
+
+v_maximumminimum_f16 v5.l, v1.l, s2, v3.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_maximumminimum_f16_e64_dpp v5.l, v1.l, s2, v3.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x6f,0xd6,0xe9,0x04,0x0c,0x04,0x01,0x77,0x39,0x05]
+
+v_maximumminimum_f16 v5.l, v1.l, 2.0, v3.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_maximumminimum_f16_e64_dpp v5.l, v1.l, 2.0, v3.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x6f,0xd6,0xe9,0xe8,0x0d,0x04,0x01,0x77,0x39,0x05]
+
+v_maximumminimum_f16 v5.l, v1.l, v2.l, v255.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_maximumminimum_f16_e64_dpp v5.l, v1.l, v2.l, v255.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x6f,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
+
+v_maximumminimum_f16 v5.l, v1.l, v2.l, s105 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_maximumminimum_f16_e64_dpp v5.l, v1.l, v2.l, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x6f,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
+
+v_maximumminimum_f16 v5.l, v1.l, v2.l, vcc_hi dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_maximumminimum_f16_e64_dpp v5.l, v1.l, v2.l, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x6f,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
+
+v_maximumminimum_f16 v5.l, v1.l, v2.l, vcc_lo dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_maximumminimum_f16_e64_dpp v5.l, v1.l, v2.l, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x6f,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
+
+v_maximumminimum_f16 v5.l, |v1.l|, v2.l, -ttmp15 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_maximumminimum_f16_e64_dpp v5.l, |v1.l|, v2.l, -ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x6f,0xd6,0xe9,0x04,0xee,0x81,0x01,0x77,0x39,0x05]
+
+v_maximumminimum_f16 v5.l, v1.l, -|v2.l|, exec_hi dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_maximumminimum_f16_e64_dpp v5.l, v1.l, -|v2.l|, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x02,0x6f,0xd6,0xe9,0x04,0xfe,0x41,0x01,0x77,0x39,0x05]
+
+v_maximumminimum_f16 v5.l, -v1.l, v2.l, |exec_lo| dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_maximumminimum_f16_e64_dpp v5.l, -v1.l, v2.l, |exec_lo| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x04,0x6f,0xd6,0xe9,0x04,0xfa,0x21,0x01,0x77,0x39,0x05]
+
+v_maximumminimum_f16 v5.l, -|v1.l|, -|v2.l|, null dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_maximumminimum_f16_e64_dpp v5.l, -|v1.l|, -|v2.l|, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x03,0x6f,0xd6,0xe9,0x04,0xf2,0x61,0x01,0x77,0x39,0x05]
 
-v_maximumminimum_f16 v5, v1, s2, v3 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_maximumminimum_f16_e64_dpp v5, v1, s2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x6f,0xd6,0xe9,0x04,0x0c,0x04,0x01,0x77,0x39,0x05]
+v_maximumminimum_f16 v5.l, -|v1.l|, v2.l, -|-1| dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_maximumminimum_f16_e64_dpp v5.l, -|v1.l|, v2.l, -|-1| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x05,0x6f,0xd6,0xe9,0x04,0x06,0xa3,0x01,0x77,0x39,0x05]
 
-v_maximumminimum_f16 v5, v1, 2.0, v3 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_maximumminimum_f16_e64_dpp v5, v1, 2.0, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x6f,0xd6,0xe9,0xe8,0x0d,0x04,0x01,0x77,0x39,0x05]
+v_maximumminimum_f16 v5.l, v1.l, -|v2.l|, -|0.5| dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX12: v_maximumminimum_f16_e64_dpp v5.l, v1.l, -|v2.l|, -|0.5| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x05,0x06,0x6f,0xd6,0xea,0x04,0xc2,0xc3,0x01,0x77,0x39,0x05]
 
-v_maximumminimum_f16 v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_maximumminimum_f16_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x6f,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
+v_maximumminimum_f16 v255.l, -|v255.l|, -|v255.l|, -|src_scc| clamp dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX12: v_maximumminimum_f16_e64_dpp v255.l, -|v255.l|, -|v255.l|, -|src_scc| clamp dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x87,0x6f,0xd6,0xe9,0xfe,0xf7,0xe3,0xff,0x00,0x00,0x00]
 
-v_maximumminimum_f16 v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_maximumminimum_f16_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x6f,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
+v_maximumminimum_f16 v5.h, v1.h, v2.h, v3.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_maximumminimum_f16_e64_dpp v5.h, v1.h, v2.h, v3.h op_sel:[1,1,1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x78,0x6f,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 
-v_maximumminimum_f16 v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_maximumminimum_f16_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x6f,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
+v_maximumminimum_f16 v5.l, v1.l, v2.l, v255.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_maximumminimum_f16_e64_dpp v5.l, v1.l, v2.l, v255.h op_sel:[0,0,1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x20,0x6f,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 
-v_maximumminimum_f16 v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_maximumminimum_f16_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x6f,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
+v_maximumminimum_f16 v5.l, -v1.h, |v2.l|, -1 mul:2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_maximumminimum_f16_e64_dpp v5.l, -v1.h, |v2.l|, -1 op_sel:[1,0,0,0] mul:2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x0a,0x6f,0xd6,0xe9,0x04,0x06,0x2b,0x01,0x77,0x39,0x05]
 
-v_maximumminimum_f16 v5, |v1|, v2, -ttmp15 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_maximumminimum_f16_e64_dpp v5, |v1|, v2, -ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x6f,0xd6,0xe9,0x04,0xee,0x81,0x01,0x77,0x39,0x05]
+v_maximumminimum_f16 v5.l, -|v1.l|, -|v2.h|, 0.5 mul:4 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX12: v_maximumminimum_f16_e64_dpp v5.l, -|v1.l|, -|v2.h|, 0.5 op_sel:[0,1,0,0] mul:4 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x05,0x13,0x6f,0xd6,0xea,0x04,0xc2,0x73,0x01,0x77,0x39,0x05]
 
-v_maximumminimum_f16 v5, v1, -|v2|, exec_hi dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_maximumminimum_f16_e64_dpp v5, v1, -|v2|, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x02,0x6f,0xd6,0xe9,0x04,0xfe,0x41,0x01,0x77,0x39,0x05]
+v_maximumminimum_f16 v255.h, -|v255.l|, -|v255.l|, -|src_scc| clamp div:2 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX12: v_maximumminimum_f16_e64_dpp v255.h, -|v255.l|, -|v255.l|, -|src_scc| op_sel:[0,0,0,1] clamp div:2 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0xc7,0x6f,0xd6,0xe9,0xfe,0xf7,0xfb,0xff,0x00,0x00,0x00]
 
-v_maximumminimum_f16 v5, -v1, v2, |exec_lo| dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_maximumminimum_f16_e64_dpp v5, -v1, v2, |exec_lo| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x04,0x6f,0xd6,0xe9,0x04,0xfa,0x21,0x01,0x77,0x39,0x05]
+v_minimummaximum_f16 v5.l, v1.l, v2.l, v3.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_minimummaximum_f16_e64_dpp v5.l, v1.l, v2.l, v3.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x6e,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 
-v_maximumminimum_f16 v5, -|v1|, -|v2|, null dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_maximumminimum_f16_e64_dpp v5, -|v1|, -|v2|, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x03,0x6f,0xd6,0xe9,0x04,0xf2,0x61,0x01,0x77,0x39,0x05]
+v_minimummaximum_f16 v5.l, v1.l, s2, v3.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_minimummaximum_f16_e64_dpp v5.l, v1.l, s2, v3.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x6e,0xd6,0xe9,0x04,0x0c,0x04,0x01,0x77,0x39,0x05]
 
-v_maximumminimum_f16 v5, -|v1|, v2, -|-1| dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_maximumminimum_f16_e64_dpp v5, -|v1|, v2, -|-1| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x05,0x6f,0xd6,0xe9,0x04,0x06,0xa3,0x01,0x77,0x39,0x05]
+v_minimummaximum_f16 v5.l, v1.l, 2.0, v3.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_minimummaximum_f16_e64_dpp v5.l, v1.l, 2.0, v3.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x6e,0xd6,0xe9,0xe8,0x0d,0x04,0x01,0x77,0x39,0x05]
 
-v_maximumminimum_f16 v5, v1, -|v2|, -|0.5| dpp8:[7,6,5,4,3,2,1,0] fi:1
-// GFX12: v_maximumminimum_f16_e64_dpp v5, v1, -|v2|, -|0.5| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x05,0x06,0x6f,0xd6,0xea,0x04,0xc2,0xc3,0x01,0x77,0x39,0x05]
+v_minimummaximum_f16 v5.l, v1.l, v2.l, v255.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_minimummaximum_f16_e64_dpp v5.l, v1.l, v2.l, v255.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x6e,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 
-v_maximumminimum_f16 v255, -|v255|, -|v255|, -|src_scc| clamp dpp8:[0,0,0,0,0,0,0,0] fi:0
-// GFX12: v_maximumminimum_f16_e64_dpp v255, -|v255|, -|v255|, -|src_scc| clamp dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x87,0x6f,0xd6,0xe9,0xfe,0xf7,0xe3,0xff,0x00,0x00,0x00]
+v_minimummaximum_f16 v5.l, v1.l, v2.l, s105 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_minimummaximum_f16_e64_dpp v5.l, v1.l, v2.l, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x6e,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 
-v_minimummaximum_f16 v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_minimummaximum_f16_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x6e,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
+v_minimummaximum_f16 v5.l, v1.l, v2.l, vcc_hi dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_minimummaximum_f16_e64_dpp v5.l, v1.l, v2.l, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x6e,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 
-v_minimummaximum_f16 v5, v1, s2, v3 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_minimummaximum_f16_e64_dpp v5, v1, s2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x6e,0xd6,0xe9,0x04,0x0c,0x04,0x01,0x77,0x39,0x05]
+v_minimummaximum_f16 v5.l, v1.l, v2.l, vcc_lo dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_minimummaximum_f16_e64_dpp v5.l, v1.l, v2.l, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x6e,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 
-v_minimummaximum_f16 v5, v1, 2.0, v3 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_minimummaximum_f16_e64_dpp v5, v1, 2.0, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x6e,0xd6,0xe9,0xe8,0x0d,0x04,0x01,0x77,0x39,0x05]
+v_minimummaximum_f16 v5.l, |v1.l|, v2.l, -ttmp15 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_minimummaximum_f16_e64_dpp v5.l, |v1.l|, v2.l, -ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x6e,0xd6,0xe9,0x04,0xee,0x81,0x01,0x77,0x39,0x05]
 
-v_minimummaximum_f16 v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_minimummaximum_f16_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x6e,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
+v_minimummaximum_f16 v5.l, v1.l, -|v2.l|, exec_hi dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_minimummaximum_f16_e64_dpp v5.l, v1.l, -|v2.l|, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x02,0x6e,0xd6,0xe9,0x04,0xfe,0x41,0x01,0x77,0x39,0x05]
 
-v_minimummaximum_f16 v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_minimummaximum_f16_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x6e,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
+v_minimummaximum_f16 v5.l, -v1.l, v2.l, |exec_lo| dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_minimummaximum_f16_e64_dpp v5.l, -v1.l, v2.l, |exec_lo| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x04,0x6e,0xd6,0xe9,0x04,0xfa,0x21,0x01,0x77,0x39,0x05]
 
-v_minimummaximum_f16 v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_minimummaximum_f16_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x6e,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
+v_minimummaximum_f16 v5.l, -|v1.l|, -|v2.l|, null dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_minimummaximum_f16_e64_dpp v5.l, -|v1.l|, -|v2.l|, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x03,0x6e,0xd6,0xe9,0x04,0xf2,0x61,0x01,0x77,0x39,0x05]
 
-v_minimummaximum_f16 v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_minimummaximum_f16_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x6e,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
+v_minimummaximum_f16 v5.l, -|v1.l|, v2.l, -|-1| dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_minimummaximum_f16_e64_dpp v5.l, -|v1.l|, v2.l, -|-1| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x05,0x6e,0xd6,0xe9,0x04,0x06,0xa3,0x01,0x77,0x39,0x05]
 
-v_minimummaximum_f16 v5, |v1|, v2, -ttmp15 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_minimummaximum_f16_e64_dpp v5, |v1|, v2, -ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x6e,0xd6,0xe9,0x04,0xee,0x81,0x01,0x77,0x39,0x05]
+v_minimummaximum_f16 v5.l, v1.l, -|v2.l|, -|0.5| dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX12: v_minimummaximum_f16_e64_dpp v5.l, v1.l, -|v2.l|, -|0.5| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x05,0x06,0x6e,0xd6,0xea,0x04,0xc2,0xc3,0x01,0x77,0x39,0x05]
 
-v_minimummaximum_f16 v5, v1, -|v2|, exec_hi dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_minimummaximum_f16_e64_dpp v5, v1, -|v2|, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x02,0x6e,0xd6,0xe9,0x04,0xfe,0x41,0x01,0x77,0x39,0x05]
+v_minimummaximum_f16 v255.l, -|v255.l|, -|v255.l|, -|src_scc| clamp dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX12: v_minimummaximum_f16_e64_dpp v255.l, -|v255.l|, -|v255.l|, -|src_scc| clamp dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x87,0x6e,0xd6,0xe9,0xfe,0xf7,0xe3,0xff,0x00,0x00,0x00]
 
-v_minimummaximum_f16 v5, -v1, v2, |exec_lo| dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_minimummaximum_f16_e64_dpp v5, -v1, v2, |exec_lo| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x04,0x6e,0xd6,0xe9,0x04,0xfa,0x21,0x01,0x77,0x39,0x05]
+v_minimummaximum_f16 v5.h, v1.h, v2.h, v3.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_minimummaximum_f16_e64_dpp v5.h, v1.h, v2.h, v3.h op_sel:[1,1,1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x78,0x6e,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 
-v_minimummaximum_f16 v5, -|v1|, -|v2|, null dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_minimummaximum_f16_e64_dpp v5, -|v1|, -|v2|, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x03,0x6e,0xd6,0xe9,0x04,0xf2,0x61,0x01,0x77,0x39,0x05]
+v_minimummaximum_f16 v5.l, v1.l, v2.l, v255.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_minimummaximum_f16_e64_dpp v5.l, v1.l, v2.l, v255.h op_sel:[0,0,1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x20,0x6e,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 
-v_minimummaximum_f16 v5, -|v1|, v2, -|-1| dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_minimummaximum_f16_e64_dpp v5, -|v1|, v2, -|-1| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x05,0x6e,0xd6,0xe9,0x04,0x06,0xa3,0x01,0x77,0x39,0x05]
+v_minimummaximum_f16 v5.l, -v1.h, |v2.l|, -1 mul:2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_minimummaximum_f16_e64_dpp v5.l, -v1.h, |v2.l|, -1 op_sel:[1,0,0,0] mul:2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x0a,0x6e,0xd6,0xe9,0x04,0x06,0x2b,0x01,0x77,0x39,0x05]
 
-v_minimummaximum_f16 v5, v1, -|v2|, -|0.5| dpp8:[7,6,5,4,3,2,1,0] fi:1
-// GFX12: v_minimummaximum_f16_e64_dpp v5, v1, -|v2|, -|0.5| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x05,0x06,0x6e,0xd6,0xea,0x04,0xc2,0xc3,0x01,0x77,0x39,0x05]
+v_minimummaximum_f16 v5.l, -|v1.l|, -|v2.h|, 0.5 mul:4 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX12: v_minimummaximum_f16_e64_dpp v5.l, -|v1.l|, -|v2.h|, 0.5 op_sel:[0,1,0,0] mul:4 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x05,0x13,0x6e,0xd6,0xea,0x04,0xc2,0x73,0x01,0x77,0x39,0x05]
 
-v_minimummaximum_f16 v255, -|v255|, -|v255|, -|src_scc| clamp dpp8:[0,0,0,0,0,0,0,0] fi:0
-// GFX12: v_minimummaximum_f16_e64_dpp v255, -|v255|, -|v255|, -|src_scc| clamp dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x87,0x6e,0xd6,0xe9,0xfe,0xf7,0xe3,0xff,0x00,0x00,0x00]
+v_minimummaximum_f16 v255.h, -|v255.l|, -|v255.l|, -|src_scc| clamp div:2 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX12: v_minimummaximum_f16_e64_dpp v255.h, -|v255.l|, -|v255.l|, -|src_scc| op_sel:[0,0,0,1] clamp div:2 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0xc7,0x6e,0xd6,0xe9,0xfe,0xf7,0xfb,0xff,0x00,0x00,0x00]
diff --git a/llvm/test/MC/AMDGPU/gfx12_asm_vop3c.s b/llvm/test/MC/AMDGPU/gfx12_asm_vop3c.s
index d73ffb6fd2f50..bc8b2bca79be4 100644
--- a/llvm/test/MC/AMDGPU/gfx12_asm_vop3c.s
+++ b/llvm/test/MC/AMDGPU/gfx12_asm_vop3c.s
@@ -120,19 +120,19 @@ v_cmp_class_f16_e64 null, -|0xfe0b|, vcc_hi
 // GFX12: v_cmp_class_f16_e64 null, -|0xfe0b|, vcc_hi ; encoding: [0x7c,0x01,0x7d,0xd4,0xff,0xd6,0x00,0x20,0x0b,0xfe,0x00,0x00]
 
 v_cmp_class_f16_e64 s5, v255.h, v2.l
-// W32: v_cmp_class_f16_e64 s5, v255.h, v2.l    ; encoding: [0x05,0x08,0x7d,0xd4,0xff,0x05,0x02,0x00]
+// W32: v_cmp_class_f16_e64 s5, v255.h, v2.l op_sel:[1,0,0] ; encoding: [0x05,0x08,0x7d,0xd4,0xff,0x05,0x02,0x00]
 // W64-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
 
 v_cmp_class_f16_e64 s5, s105, v255.h
-// W32: v_cmp_class_f16_e64 s5, s105, v255.h    ; encoding: [0x05,0x10,0x7d,0xd4,0x69,0xfe,0x03,0x00]
+// W32: v_cmp_class_f16_e64 s5, s105, v255.h op_sel:[0,1,0] ; encoding: [0x05,0x10,0x7d,0xd4,0x69,0xfe,0x03,0x00]
 // W64-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
 
 v_cmp_class_f16_e64 s[10:11], v255.h, v2.l
-// W64: v_cmp_class_f16_e64 s[10:11], v255.h, v2.l ; encoding: [0x0a,0x08,0x7d,0xd4,0xff,0x05,0x02,0x00]
+// W64: v_cmp_class_f16_e64 s[10:11], v255.h, v2.l op_sel:[1,0,0] ; encoding: [0x0a,0x08,0x7d,0xd4,0xff,0x05,0x02,0x00]
 // W32-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
 
 v_cmp_class_f16_e64 s[10:11], s105, v255.h
-// W64: v_cmp_class_f16_e64 s[10:11], s105, v255.h ; encoding: [0x0a,0x10,0x7d,0xd4,0x69,0xfe,0x03,0x00]
+// W64: v_cmp_class_f16_e64 s[10:11], s105, v255.h op_sel:[0,1,0] ; encoding: [0x0a,0x10,0x7d,0xd4,0x69,0xfe,0x03,0x00]
 // W32-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
 
 v_cmp_class_f16_e64 vcc_lo, 0.5, m0
diff --git a/llvm/test/MC/AMDGPU/gfx12_asm_vop3cx.s b/llvm/test/MC/AMDGPU/gfx12_asm_vop3cx.s
index d7bec00b83080..e40b6a77c9b27 100644
--- a/llvm/test/MC/AMDGPU/gfx12_asm_vop3cx.s
+++ b/llvm/test/MC/AMDGPU/gfx12_asm_vop3cx.s
@@ -48,13 +48,13 @@ v_cmpx_class_f16_e64 -|0xfe0b|, vcc_hi
 // GFX12: v_cmpx_class_f16_e64 -|0xfe0b|, vcc_hi  ; encoding: [0x7e,0x01,0xfd,0xd4,0xff,0xd6,0x00,0x20,0x0b,0xfe,0x00,0x00]
 
 v_cmpx_class_f16_e64 v1.h, v2.h
-// GFX12: v_cmpx_class_f16_e64 v1.h, v2.h         ; encoding: [0x7e,0x18,0xfd,0xd4,0x01,0x05,0x02,0x00]
+// GFX12: v_cmpx_class_f16_e64 v1.h, v2.h op_sel:[1,1,0] ; encoding: [0x7e,0x18,0xfd,0xd4,0x01,0x05,0x02,0x00]
 
 v_cmpx_class_f16_e64 v255.h, v2.l
-// GFX12: v_cmpx_class_f16_e64 v255.h, v2.l       ; encoding: [0x7e,0x08,0xfd,0xd4,0xff,0x05,0x02,0x00]
+// GFX12: v_cmpx_class_f16_e64 v255.h, v2.l op_sel:[1,0,0] ; encoding: [0x7e,0x08,0xfd,0xd4,0xff,0x05,0x02,0x00]
 
 v_cmpx_class_f16_e64 s105, v255.h
-// GFX12: v_cmpx_class_f16_e64 s105, v255.h       ; encoding: [0x7e,0x10,0xfd,0xd4,0x69,0xfe,0x03,0x00]
+// GFX12: v_cmpx_class_f16_e64 s105, v255.h op_sel:[0,1,0] ; encoding: [0x7e,0x10,0xfd,0xd4,0x69,0xfe,0x03,0x00]
 
 v_cmpx_class_f32_e64 v1, v2
 // GFX12: v_cmpx_class_f32_e64 v1, v2             ; encoding: [0x7e,0x00,0xfe,0xd4,0x01,0x05,0x02,0x00]
@@ -146,11 +146,11 @@ v_cmpx_class_f64_e64 -|src_scc|, src_scc
 v_cmpx_class_f64_e64 0xaf123456, 0xaf123456
 // GFX12: v_cmpx_class_f64_e64 0xaf123456, 0xaf123456 ; encoding: [0x7e,0x00,0xff,0xd4,0xff,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
 
-v_cmpx_eq_f16_e64 v1, v2
-// GFX12: v_cmpx_eq_f16_e64 v1, v2                ; encoding: [0x7e,0x00,0x82,0xd4,0x01,0x05,0x02,0x00]
+v_cmpx_eq_f16_e64 v1.l, v2.l
+// GFX12: v_cmpx_eq_f16_e64 v1.l, v2.l            ; encoding: [0x7e,0x00,0x82,0xd4,0x01,0x05,0x02,0x00]
 
-v_cmpx_eq_f16_e64 v255, v255
-// GFX12: v_cmpx_eq_f16_e64 v255, v255            ; encoding: [0x7e,0x00,0x82,0xd4,0xff,0xff,0x03,0x00]
+v_cmpx_eq_f16_e64 v255.l, v255.l
+// GFX12: v_cmpx_eq_f16_e64 v255.l, v255.l        ; encoding: [0x7e,0x00,0x82,0xd4,0xff,0xff,0x03,0x00]
 
 v_cmpx_eq_f16_e64 s1, s2
 // GFX12: v_cmpx_eq_f16_e64 s1, s2                ; encoding: [0x7e,0x00,0x82,0xd4,0x01,0x04,0x00,0x00]
@@ -191,6 +191,12 @@ v_cmpx_eq_f16_e64 -src_scc, |vcc_lo|
 v_cmpx_eq_f16_e64 -|0xfe0b|, -|vcc_hi| clamp
 // GFX12: v_cmpx_eq_f16_e64 -|0xfe0b|, -|vcc_hi| clamp ; encoding: [0x7e,0x83,0x82,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00]
 
+v_cmpx_eq_f16_e64 v1.h, v2.l
+// GFX12: v_cmpx_eq_f16_e64 v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x7e,0x08,0x82,0xd4,0x01,0x05,0x02,0x00]
+
+v_cmpx_eq_f16_e64 v255.l, v255.h
+// GFX12: v_cmpx_eq_f16_e64 v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x7e,0x10,0x82,0xd4,0xff,0xff,0x03,0x00]
+
 v_cmpx_eq_f32_e64 v1, v2
 // GFX12: v_cmpx_eq_f32_e64 v1, v2                ; encoding: [0x7e,0x00,0x92,0xd4,0x01,0x05,0x02,0x00]
 
@@ -318,10 +324,10 @@ v_cmpx_eq_i16_e64 0xfe0b, vcc_hi
 // GFX12: v_cmpx_eq_i16_e64 0xfe0b, vcc_hi        ; encoding: [0x7e,0x00,0xb2,0xd4,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
 
 v_cmpx_eq_i16_e64 v1.h, v2.l
-// GFX12: v_cmpx_eq_i16_e64 v1.h, v2.l            ; encoding: [0x7e,0x08,0xb2,0xd4,0x01,0x05,0x02,0x00]
+// GFX12: v_cmpx_eq_i16_e64 v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x7e,0x08,0xb2,0xd4,0x01,0x05,0x02,0x00]
 
 v_cmpx_eq_i16_e64 v255.l, v255.h
-// GFX12: v_cmpx_eq_i16_e64 v255.l, v255.h        ; encoding: [0x7e,0x10,0xb2,0xd4,0xff,0xff,0x03,0x00]
+// GFX12: v_cmpx_eq_i16_e64 v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x7e,0x10,0xb2,0xd4,0xff,0xff,0x03,0x00]
 
 v_cmpx_eq_i32_e64 v1, v2
 // GFX12: v_cmpx_eq_i32_e64 v1, v2                ; encoding: [0x7e,0x00,0xc2,0xd4,0x01,0x05,0x02,0x00]
@@ -450,10 +456,10 @@ v_cmpx_eq_u16_e64 0xfe0b, vcc_hi
 // GFX12: v_cmpx_eq_u16_e64 0xfe0b, vcc_hi        ; encoding: [0x7e,0x00,0xba,0xd4,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
 
 v_cmpx_eq_u16_e64 v1.h, v2.l
-// GFX12: v_cmpx_eq_u16_e64 v1.h, v2.l            ; encoding: [0x7e,0x08,0xba,0xd4,0x01,0x05,0x02,0x00]
+// GFX12: v_cmpx_eq_u16_e64 v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x7e,0x08,0xba,0xd4,0x01,0x05,0x02,0x00]
 
 v_cmpx_eq_u16_e64 v255.l, v255.h
-// GFX12: v_cmpx_eq_u16_e64 v255.l, v255.h        ; encoding: [0x7e,0x10,0xba,0xd4,0xff,0xff,0x03,0x00]
+// GFX12: v_cmpx_eq_u16_e64 v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x7e,0x10,0xba,0xd4,0xff,0xff,0x03,0x00]
 
 v_cmpx_eq_u32_e64 v1, v2
 // GFX12: v_cmpx_eq_u32_e64 v1, v2                ; encoding: [0x7e,0x00,0xca,0xd4,0x01,0x05,0x02,0x00]
@@ -536,11 +542,11 @@ v_cmpx_eq_u64_e64 src_scc, exec
 v_cmpx_eq_u64_e64 0xaf123456, vcc
 // GFX12: v_cmpx_eq_u64_e64 0xaf123456, vcc       ; encoding: [0x7e,0x00,0xda,0xd4,0xff,0xd4,0x00,0x00,0x56,0x34,0x12,0xaf]
 
-v_cmpx_ge_f16_e64 v1, v2
-// GFX12: v_cmpx_ge_f16_e64 v1, v2                ; encoding: [0x7e,0x00,0x86,0xd4,0x01,0x05,0x02,0x00]
+v_cmpx_ge_f16_e64 v1.l, v2.l
+// GFX12: v_cmpx_ge_f16_e64 v1.l, v2.l            ; encoding: [0x7e,0x00,0x86,0xd4,0x01,0x05,0x02,0x00]
 
-v_cmpx_ge_f16_e64 v255, v255
-// GFX12: v_cmpx_ge_f16_e64 v255, v255            ; encoding: [0x7e,0x00,0x86,0xd4,0xff,0xff,0x03,0x00]
+v_cmpx_ge_f16_e64 v255.l, v255.l
+// GFX12: v_cmpx_ge_f16_e64 v255.l, v255.l        ; encoding: [0x7e,0x00,0x86,0xd4,0xff,0xff,0x03,0x00]
 
 v_cmpx_ge_f16_e64 s1, s2
 // GFX12: v_cmpx_ge_f16_e64 s1, s2                ; encoding: [0x7e,0x00,0x86,0xd4,0x01,0x04,0x00,0x00]
@@ -581,6 +587,12 @@ v_cmpx_ge_f16_e64 -src_scc, |vcc_lo|
 v_cmpx_ge_f16_e64 -|0xfe0b|, -|vcc_hi| clamp
 // GFX12: v_cmpx_ge_f16_e64 -|0xfe0b|, -|vcc_hi| clamp ; encoding: [0x7e,0x83,0x86,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00]
 
+v_cmpx_ge_f16_e64 v1.h, v2.l
+// GFX12: v_cmpx_ge_f16_e64 v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x7e,0x08,0x86,0xd4,0x01,0x05,0x02,0x00]
+
+v_cmpx_ge_f16_e64 v255.l, v255.h
+// GFX12: v_cmpx_ge_f16_e64 v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x7e,0x10,0x86,0xd4,0xff,0xff,0x03,0x00]
+
 v_cmpx_ge_f32_e64 v1, v2
 // GFX12: v_cmpx_ge_f32_e64 v1, v2                ; encoding: [0x7e,0x00,0x96,0xd4,0x01,0x05,0x02,0x00]
 
@@ -708,10 +720,10 @@ v_cmpx_ge_i16_e64 0xfe0b, vcc_hi
 // GFX12: v_cmpx_ge_i16_e64 0xfe0b, vcc_hi        ; encoding: [0x7e,0x00,0xb6,0xd4,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
 
 v_cmpx_ge_i16_e64 v1.h, v2.l
-// GFX12: v_cmpx_ge_i16_e64 v1.h, v2.l            ; encoding: [0x7e,0x08,0xb6,0xd4,0x01,0x05,0x02,0x00]
+// GFX12: v_cmpx_ge_i16_e64 v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x7e,0x08,0xb6,0xd4,0x01,0x05,0x02,0x00]
 
 v_cmpx_ge_i16_e64 v255.l, v255.h
-// GFX12: v_cmpx_ge_i16_e64 v255.l, v255.h        ; encoding: [0x7e,0x10,0xb6,0xd4,0xff,0xff,0x03,0x00]
+// GFX12: v_cmpx_ge_i16_e64 v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x7e,0x10,0xb6,0xd4,0xff,0xff,0x03,0x00]
 
 v_cmpx_ge_i32_e64 v1, v2
 // GFX12: v_cmpx_ge_i32_e64 v1, v2                ; encoding: [0x7e,0x00,0xc6,0xd4,0x01,0x05,0x02,0x00]
@@ -840,10 +852,10 @@ v_cmpx_ge_u16_e64 0xfe0b, vcc_hi
 // GFX12: v_cmpx_ge_u16_e64 0xfe0b, vcc_hi        ; encoding: [0x7e,0x00,0xbe,0xd4,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
 
 v_cmpx_ge_u16_e64 v1.h, v2.l
-// GFX12: v_cmpx_ge_u16_e64 v1.h, v2.l            ; encoding: [0x7e,0x08,0xbe,0xd4,0x01,0x05,0x02,0x00]
+// GFX12: v_cmpx_ge_u16_e64 v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x7e,0x08,0xbe,0xd4,0x01,0x05,0x02,0x00]
 
 v_cmpx_ge_u16_e64 v255.l, v255.h
-// GFX12: v_cmpx_ge_u16_e64 v255.l, v255.h        ; encoding: [0x7e,0x10,0xbe,0xd4,0xff,0xff,0x03,0x00]
+// GFX12: v_cmpx_ge_u16_e64 v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x7e,0x10,0xbe,0xd4,0xff,0xff,0x03,0x00]
 
 v_cmpx_ge_u32_e64 v1, v2
 // GFX12: v_cmpx_ge_u32_e64 v1, v2                ; encoding: [0x7e,0x00,0xce,0xd4,0x01,0x05,0x02,0x00]
@@ -926,11 +938,11 @@ v_cmpx_ge_u64_e64 src_scc, exec
 v_cmpx_ge_u64_e64 0xaf123456, vcc
 // GFX12: v_cmpx_ge_u64_e64 0xaf123456, vcc       ; encoding: [0x7e,0x00,0xde,0xd4,0xff,0xd4,0x00,0x00,0x56,0x34,0x12,0xaf]
 
-v_cmpx_gt_f16_e64 v1, v2
-// GFX12: v_cmpx_gt_f16_e64 v1, v2                ; encoding: [0x7e,0x00,0x84,0xd4,0x01,0x05,0x02,0x00]
+v_cmpx_gt_f16_e64 v1.l, v2.l
+// GFX12: v_cmpx_gt_f16_e64 v1.l, v2.l            ; encoding: [0x7e,0x00,0x84,0xd4,0x01,0x05,0x02,0x00]
 
-v_cmpx_gt_f16_e64 v255, v255
-// GFX12: v_cmpx_gt_f16_e64 v255, v255            ; encoding: [0x7e,0x00,0x84,0xd4,0xff,0xff,0x03,0x00]
+v_cmpx_gt_f16_e64 v255.l, v255.l
+// GFX12: v_cmpx_gt_f16_e64 v255.l, v255.l        ; encoding: [0x7e,0x00,0x84,0xd4,0xff,0xff,0x03,0x00]
 
 v_cmpx_gt_f16_e64 s1, s2
 // GFX12: v_cmpx_gt_f16_e64 s1, s2                ; encoding: [0x7e,0x00,0x84,0xd4,0x01,0x04,0x00,0x00]
@@ -971,6 +983,12 @@ v_cmpx_gt_f16_e64 -src_scc, |vcc_lo|
 v_cmpx_gt_f16_e64 -|0xfe0b|, -|vcc_hi| clamp
 // GFX12: v_cmpx_gt_f16_e64 -|0xfe0b|, -|vcc_hi| clamp ; encoding: [0x7e,0x83,0x84,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00]
 
+v_cmpx_gt_f16_e64 v1.h, v2.l
+// GFX12: v_cmpx_gt_f16_e64 v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x7e,0x08,0x84,0xd4,0x01,0x05,0x02,0x00]
+
+v_cmpx_gt_f16_e64 v255.l, v255.h
+// GFX12: v_cmpx_gt_f16_e64 v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x7e,0x10,0x84,0xd4,0xff,0xff,0x03,0x00]
+
 v_cmpx_gt_f32_e64 v1, v2
 // GFX12: v_cmpx_gt_f32_e64 v1, v2                ; encoding: [0x7e,0x00,0x94,0xd4,0x01,0x05,0x02,0x00]
 
@@ -1098,10 +1116,10 @@ v_cmpx_gt_i16_e64 0xfe0b, vcc_hi
 // GFX12: v_cmpx_gt_i16_e64 0xfe0b, vcc_hi        ; encoding: [0x7e,0x00,0xb4,0xd4,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
 
 v_cmpx_gt_i16_e64 v1.h, v2.l
-// GFX12: v_cmpx_gt_i16_e64 v1.h, v2.l            ; encoding: [0x7e,0x08,0xb4,0xd4,0x01,0x05,0x02,0x00]
+// GFX12: v_cmpx_gt_i16_e64 v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x7e,0x08,0xb4,0xd4,0x01,0x05,0x02,0x00]
 
 v_cmpx_gt_i16_e64 v255.l, v255.h
-// GFX12: v_cmpx_gt_i16_e64 v255.l, v255.h        ; encoding: [0x7e,0x10,0xb4,0xd4,0xff,0xff,0x03,0x00]
+// GFX12: v_cmpx_gt_i16_e64 v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x7e,0x10,0xb4,0xd4,0xff,0xff,0x03,0x00]
 
 v_cmpx_gt_i32_e64 v1, v2
 // GFX12: v_cmpx_gt_i32_e64 v1, v2                ; encoding: [0x7e,0x00,0xc4,0xd4,0x01,0x05,0x02,0x00]
@@ -1230,10 +1248,10 @@ v_cmpx_gt_u16_e64 0xfe0b, vcc_hi
 // GFX12: v_cmpx_gt_u16_e64 0xfe0b, vcc_hi        ; encoding: [0x7e,0x00,0xbc,0xd4,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
 
 v_cmpx_gt_u16_e64 v1.h, v2.l
-// GFX12: v_cmpx_gt_u16_e64 v1.h, v2.l            ; encoding: [0x7e,0x08,0xbc,0xd4,0x01,0x05,0x02,0x00]
+// GFX12: v_cmpx_gt_u16_e64 v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x7e,0x08,0xbc,0xd4,0x01,0x05,0x02,0x00]
 
 v_cmpx_gt_u16_e64 v255.l, v255.h
-// GFX12: v_cmpx_gt_u16_e64 v255.l, v255.h        ; encoding: [0x7e,0x10,0xbc,0xd4,0xff,0xff,0x03,0x00]
+// GFX12: v_cmpx_gt_u16_e64 v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x7e,0x10,0xbc,0xd4,0xff,0xff,0x03,0x00]
 
 v_cmpx_gt_u32_e64 v1, v2
 // GFX12: v_cmpx_gt_u32_e64 v1, v2                ; encoding: [0x7e,0x00,0xcc,0xd4,0x01,0x05,0x02,0x00]
@@ -1316,11 +1334,11 @@ v_cmpx_gt_u64_e64 src_scc, exec
 v_cmpx_gt_u64_e64 0xaf123456, vcc
 // GFX12: v_cmpx_gt_u64_e64 0xaf123456, vcc       ; encoding: [0x7e,0x00,0xdc,0xd4,0xff,0xd4,0x00,0x00,0x56,0x34,0x12,0xaf]
 
-v_cmpx_le_f16_e64 v1, v2
-// GFX12: v_cmpx_le_f16_e64 v1, v2                ; encoding: [0x7e,0x00,0x83,0xd4,0x01,0x05,0x02,0x00]
+v_cmpx_le_f16_e64 v1.l, v2.l
+// GFX12: v_cmpx_le_f16_e64 v1.l, v2.l            ; encoding: [0x7e,0x00,0x83,0xd4,0x01,0x05,0x02,0x00]
 
-v_cmpx_le_f16_e64 v255, v255
-// GFX12: v_cmpx_le_f16_e64 v255, v255            ; encoding: [0x7e,0x00,0x83,0xd4,0xff,0xff,0x03,0x00]
+v_cmpx_le_f16_e64 v255.l, v255.l
+// GFX12: v_cmpx_le_f16_e64 v255.l, v255.l        ; encoding: [0x7e,0x00,0x83,0xd4,0xff,0xff,0x03,0x00]
 
 v_cmpx_le_f16_e64 s1, s2
 // GFX12: v_cmpx_le_f16_e64 s1, s2                ; encoding: [0x7e,0x00,0x83,0xd4,0x01,0x04,0x00,0x00]
@@ -1361,6 +1379,12 @@ v_cmpx_le_f16_e64 -src_scc, |vcc_lo|
 v_cmpx_le_f16_e64 -|0xfe0b|, -|vcc_hi| clamp
 // GFX12: v_cmpx_le_f16_e64 -|0xfe0b|, -|vcc_hi| clamp ; encoding: [0x7e,0x83,0x83,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00]
 
+v_cmpx_le_f16_e64 v1.h, v2.l
+// GFX12: v_cmpx_le_f16_e64 v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x7e,0x08,0x83,0xd4,0x01,0x05,0x02,0x00]
+
+v_cmpx_le_f16_e64 v255.l, v255.h
+// GFX12: v_cmpx_le_f16_e64 v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x7e,0x10,0x83,0xd4,0xff,0xff,0x03,0x00]
+
 v_cmpx_le_f32_e64 v1, v2
 // GFX12: v_cmpx_le_f32_e64 v1, v2                ; encoding: [0x7e,0x00,0x93,0xd4,0x01,0x05,0x02,0x00]
 
@@ -1488,10 +1512,10 @@ v_cmpx_le_i16_e64 0xfe0b, vcc_hi
 // GFX12: v_cmpx_le_i16_e64 0xfe0b, vcc_hi        ; encoding: [0x7e,0x00,0xb3,0xd4,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
 
 v_cmpx_le_i16_e64 v1.h, v2.l
-// GFX12: v_cmpx_le_i16_e64 v1.h, v2.l            ; encoding: [0x7e,0x08,0xb3,0xd4,0x01,0x05,0x02,0x00]
+// GFX12: v_cmpx_le_i16_e64 v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x7e,0x08,0xb3,0xd4,0x01,0x05,0x02,0x00]
 
 v_cmpx_le_i16_e64 v255.l, v255.h
-// GFX12: v_cmpx_le_i16_e64 v255.l, v255.h        ; encoding: [0x7e,0x10,0xb3,0xd4,0xff,0xff,0x03,0x00]
+// GFX12: v_cmpx_le_i16_e64 v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x7e,0x10,0xb3,0xd4,0xff,0xff,0x03,0x00]
 
 v_cmpx_le_i32_e64 v1, v2
 // GFX12: v_cmpx_le_i32_e64 v1, v2                ; encoding: [0x7e,0x00,0xc3,0xd4,0x01,0x05,0x02,0x00]
@@ -1620,10 +1644,10 @@ v_cmpx_le_u16_e64 0xfe0b, vcc_hi
 // GFX12: v_cmpx_le_u16_e64 0xfe0b, vcc_hi        ; encoding: [0x7e,0x00,0xbb,0xd4,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
 
 v_cmpx_le_u16_e64 v1.h, v2.l
-// GFX12: v_cmpx_le_u16_e64 v1.h, v2.l            ; encoding: [0x7e,0x08,0xbb,0xd4,0x01,0x05,0x02,0x00]
+// GFX12: v_cmpx_le_u16_e64 v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x7e,0x08,0xbb,0xd4,0x01,0x05,0x02,0x00]
 
 v_cmpx_le_u16_e64 v255.l, v255.h
-// GFX12: v_cmpx_le_u16_e64 v255.l, v255.h        ; encoding: [0x7e,0x10,0xbb,0xd4,0xff,0xff,0x03,0x00]
+// GFX12: v_cmpx_le_u16_e64 v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x7e,0x10,0xbb,0xd4,0xff,0xff,0x03,0x00]
 
 v_cmpx_le_u32_e64 v1, v2
 // GFX12: v_cmpx_le_u32_e64 v1, v2                ; encoding: [0x7e,0x00,0xcb,0xd4,0x01,0x05,0x02,0x00]
@@ -1706,11 +1730,11 @@ v_cmpx_le_u64_e64 src_scc, exec
 v_cmpx_le_u64_e64 0xaf123456, vcc
 // GFX12: v_cmpx_le_u64_e64 0xaf123456, vcc       ; encoding: [0x7e,0x00,0xdb,0xd4,0xff,0xd4,0x00,0x00,0x56,0x34,0x12,0xaf]
 
-v_cmpx_lg_f16_e64 v1, v2
-// GFX12: v_cmpx_lg_f16_e64 v1, v2                ; encoding: [0x7e,0x00,0x85,0xd4,0x01,0x05,0x02,0x00]
+v_cmpx_lg_f16_e64 v1.l, v2.l
+// GFX12: v_cmpx_lg_f16_e64 v1.l, v2.l            ; encoding: [0x7e,0x00,0x85,0xd4,0x01,0x05,0x02,0x00]
 
-v_cmpx_lg_f16_e64 v255, v255
-// GFX12: v_cmpx_lg_f16_e64 v255, v255            ; encoding: [0x7e,0x00,0x85,0xd4,0xff,0xff,0x03,0x00]
+v_cmpx_lg_f16_e64 v255.l, v255.l
+// GFX12: v_cmpx_lg_f16_e64 v255.l, v255.l        ; encoding: [0x7e,0x00,0x85,0xd4,0xff,0xff,0x03,0x00]
 
 v_cmpx_lg_f16_e64 s1, s2
 // GFX12: v_cmpx_lg_f16_e64 s1, s2                ; encoding: [0x7e,0x00,0x85,0xd4,0x01,0x04,0x00,0x00]
@@ -1751,6 +1775,12 @@ v_cmpx_lg_f16_e64 -src_scc, |vcc_lo|
 v_cmpx_lg_f16_e64 -|0xfe0b|, -|vcc_hi| clamp
 // GFX12: v_cmpx_lg_f16_e64 -|0xfe0b|, -|vcc_hi| clamp ; encoding: [0x7e,0x83,0x85,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00]
 
+v_cmpx_lg_f16_e64 v1.h, v2.l
+// GFX12: v_cmpx_lg_f16_e64 v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x7e,0x08,0x85,0xd4,0x01,0x05,0x02,0x00]
+
+v_cmpx_lg_f16_e64 v255.l, v255.h
+// GFX12: v_cmpx_lg_f16_e64 v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x7e,0x10,0x85,0xd4,0xff,0xff,0x03,0x00]
+
 v_cmpx_lg_f32_e64 v1, v2
 // GFX12: v_cmpx_lg_f32_e64 v1, v2                ; encoding: [0x7e,0x00,0x95,0xd4,0x01,0x05,0x02,0x00]
 
@@ -1878,10 +1908,10 @@ v_cmpx_lt_f16_e64 -|0xfe0b|, -|vcc_hi| clamp
 // GFX12: v_cmpx_lt_f16_e64 -|0xfe0b|, -|vcc_hi| clamp ; encoding: [0x7e,0x83,0x81,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00]
 
 v_cmpx_lt_f16_e64 v1.h, v2.l
-// GFX12: v_cmpx_lt_f16_e64 v1.h, v2.l            ; encoding: [0x7e,0x08,0x81,0xd4,0x01,0x05,0x02,0x00]
+// GFX12: v_cmpx_lt_f16_e64 v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x7e,0x08,0x81,0xd4,0x01,0x05,0x02,0x00]
 
 v_cmpx_lt_f16_e64 v255.l, v255.h
-// GFX12: v_cmpx_lt_f16_e64 v255.l, v255.h        ; encoding: [0x7e,0x10,0x81,0xd4,0xff,0xff,0x03,0x00]
+// GFX12: v_cmpx_lt_f16_e64 v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x7e,0x10,0x81,0xd4,0xff,0xff,0x03,0x00]
 
 v_cmpx_lt_f32_e64 v1, v2
 // GFX12: v_cmpx_lt_f32_e64 v1, v2                ; encoding: [0x7e,0x00,0x91,0xd4,0x01,0x05,0x02,0x00]
@@ -2010,10 +2040,10 @@ v_cmpx_lt_i16_e64 0xfe0b, vcc_hi
 // GFX12: v_cmpx_lt_i16_e64 0xfe0b, vcc_hi        ; encoding: [0x7e,0x00,0xb1,0xd4,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
 
 v_cmpx_lt_i16_e64 v1.h, v2.l
-// GFX12: v_cmpx_lt_i16_e64 v1.h, v2.l            ; encoding: [0x7e,0x08,0xb1,0xd4,0x01,0x05,0x02,0x00]
+// GFX12: v_cmpx_lt_i16_e64 v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x7e,0x08,0xb1,0xd4,0x01,0x05,0x02,0x00]
 
 v_cmpx_lt_i16_e64 v255.l, v255.h
-// GFX12: v_cmpx_lt_i16_e64 v255.l, v255.h        ; encoding: [0x7e,0x10,0xb1,0xd4,0xff,0xff,0x03,0x00]
+// GFX12: v_cmpx_lt_i16_e64 v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x7e,0x10,0xb1,0xd4,0xff,0xff,0x03,0x00]
 
 v_cmpx_lt_i32_e64 v1, v2
 // GFX12: v_cmpx_lt_i32_e64 v1, v2                ; encoding: [0x7e,0x00,0xc1,0xd4,0x01,0x05,0x02,0x00]
@@ -2142,10 +2172,10 @@ v_cmpx_lt_u16_e64 0xfe0b, vcc_hi
 // GFX12: v_cmpx_lt_u16_e64 0xfe0b, vcc_hi        ; encoding: [0x7e,0x00,0xb9,0xd4,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
 
 v_cmpx_lt_u16_e64 v1.h, v2.l
-// GFX12: v_cmpx_lt_u16_e64 v1.h, v2.l            ; encoding: [0x7e,0x08,0xb9,0xd4,0x01,0x05,0x02,0x00]
+// GFX12: v_cmpx_lt_u16_e64 v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x7e,0x08,0xb9,0xd4,0x01,0x05,0x02,0x00]
 
 v_cmpx_lt_u16_e64 v255.l, v255.h
-// GFX12: v_cmpx_lt_u16_e64 v255.l, v255.h        ; encoding: [0x7e,0x10,0xb9,0xd4,0xff,0xff,0x03,0x00]
+// GFX12: v_cmpx_lt_u16_e64 v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x7e,0x10,0xb9,0xd4,0xff,0xff,0x03,0x00]
 
 v_cmpx_lt_u32_e64 v1, v2
 // GFX12: v_cmpx_lt_u32_e64 v1, v2                ; encoding: [0x7e,0x00,0xc9,0xd4,0x01,0x05,0x02,0x00]
@@ -2274,10 +2304,10 @@ v_cmpx_ne_i16_e64 0xfe0b, vcc_hi
 // GFX12: v_cmpx_ne_i16_e64 0xfe0b, vcc_hi        ; encoding: [0x7e,0x00,0xb5,0xd4,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
 
 v_cmpx_ne_i16_e64 v1.h, v2.l
-// GFX12: v_cmpx_ne_i16_e64 v1.h, v2.l            ; encoding: [0x7e,0x08,0xb5,0xd4,0x01,0x05,0x02,0x00]
+// GFX12: v_cmpx_ne_i16_e64 v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x7e,0x08,0xb5,0xd4,0x01,0x05,0x02,0x00]
 
 v_cmpx_ne_i16_e64 v255.l, v255.h
-// GFX12: v_cmpx_ne_i16_e64 v255.l, v255.h        ; encoding: [0x7e,0x10,0xb5,0xd4,0xff,0xff,0x03,0x00]
+// GFX12: v_cmpx_ne_i16_e64 v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x7e,0x10,0xb5,0xd4,0xff,0xff,0x03,0x00]
 
 v_cmpx_ne_i32_e64 v1, v2
 // GFX12: v_cmpx_ne_i32_e64 v1, v2                ; encoding: [0x7e,0x00,0xc5,0xd4,0x01,0x05,0x02,0x00]
@@ -2406,10 +2436,10 @@ v_cmpx_ne_u16_e64 0xfe0b, vcc_hi
 // GFX12: v_cmpx_ne_u16_e64 0xfe0b, vcc_hi        ; encoding: [0x7e,0x00,0xbd,0xd4,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
 
 v_cmpx_ne_u16_e64 v1.h, v2.l
-// GFX12: v_cmpx_ne_u16_e64 v1.h, v2.l            ; encoding: [0x7e,0x08,0xbd,0xd4,0x01,0x05,0x02,0x00]
+// GFX12: v_cmpx_ne_u16_e64 v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x7e,0x08,0xbd,0xd4,0x01,0x05,0x02,0x00]
 
 v_cmpx_ne_u16_e64 v255.l, v255.h
-// GFX12: v_cmpx_ne_u16_e64 v255.l, v255.h        ; encoding: [0x7e,0x10,0xbd,0xd4,0xff,0xff,0x03,0x00]
+// GFX12: v_cmpx_ne_u16_e64 v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x7e,0x10,0xbd,0xd4,0xff,0xff,0x03,0x00]
 
 v_cmpx_ne_u32_e64 v1, v2
 // GFX12: v_cmpx_ne_u32_e64 v1, v2                ; encoding: [0x7e,0x00,0xcd,0xd4,0x01,0x05,0x02,0x00]
@@ -2492,11 +2522,11 @@ v_cmpx_ne_u64_e64 src_scc, exec
 v_cmpx_ne_u64_e64 0xaf123456, vcc
 // GFX12: v_cmpx_ne_u64_e64 0xaf123456, vcc       ; encoding: [0x7e,0x00,0xdd,0xd4,0xff,0xd4,0x00,0x00,0x56,0x34,0x12,0xaf]
 
-v_cmpx_neq_f16_e64 v1, v2
-// GFX12: v_cmpx_neq_f16_e64 v1, v2               ; encoding: [0x7e,0x00,0x8d,0xd4,0x01,0x05,0x02,0x00]
+v_cmpx_neq_f16_e64 v1.l, v2.l
+// GFX12: v_cmpx_neq_f16_e64 v1.l, v2.l           ; encoding: [0x7e,0x00,0x8d,0xd4,0x01,0x05,0x02,0x00]
 
-v_cmpx_neq_f16_e64 v255, v255
-// GFX12: v_cmpx_neq_f16_e64 v255, v255           ; encoding: [0x7e,0x00,0x8d,0xd4,0xff,0xff,0x03,0x00]
+v_cmpx_neq_f16_e64 v255.l, v255.l
+// GFX12: v_cmpx_neq_f16_e64 v255.l, v255.l       ; encoding: [0x7e,0x00,0x8d,0xd4,0xff,0xff,0x03,0x00]
 
 v_cmpx_neq_f16_e64 s1, s2
 // GFX12: v_cmpx_neq_f16_e64 s1, s2               ; encoding: [0x7e,0x00,0x8d,0xd4,0x01,0x04,0x00,0x00]
@@ -2537,6 +2567,12 @@ v_cmpx_neq_f16_e64 -src_scc, |vcc_lo|
 v_cmpx_neq_f16_e64 -|0xfe0b|, -|vcc_hi| clamp
 // GFX12: v_cmpx_neq_f16_e64 -|0xfe0b|, -|vcc_hi| clamp ; encoding: [0x7e,0x83,0x8d,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00]
 
+v_cmpx_neq_f16_e64 v1.h, v2.l
+// GFX12: v_cmpx_neq_f16_e64 v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x7e,0x08,0x8d,0xd4,0x01,0x05,0x02,0x00]
+
+v_cmpx_neq_f16_e64 v255.l, v255.h
+// GFX12: v_cmpx_neq_f16_e64 v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x7e,0x10,0x8d,0xd4,0xff,0xff,0x03,0x00]
+
 v_cmpx_neq_f32_e64 v1, v2
 // GFX12: v_cmpx_neq_f32_e64 v1, v2               ; encoding: [0x7e,0x00,0x9d,0xd4,0x01,0x05,0x02,0x00]
 
@@ -2618,11 +2654,11 @@ v_cmpx_neq_f64_e64 -|src_scc|, -|exec|
 v_cmpx_neq_f64_e64 0xaf123456, -|vcc| clamp
 // GFX12: v_cmpx_neq_f64_e64 0xaf123456, -|vcc| clamp ; encoding: [0x7e,0x82,0xad,0xd4,0xff,0xd4,0x00,0x40,0x56,0x34,0x12,0xaf]
 
-v_cmpx_nge_f16_e64 v1, v2
-// GFX12: v_cmpx_nge_f16_e64 v1, v2               ; encoding: [0x7e,0x00,0x89,0xd4,0x01,0x05,0x02,0x00]
+v_cmpx_nge_f16_e64 v1.l, v2.l
+// GFX12: v_cmpx_nge_f16_e64 v1.l, v2.l           ; encoding: [0x7e,0x00,0x89,0xd4,0x01,0x05,0x02,0x00]
 
-v_cmpx_nge_f16_e64 v255, v255
-// GFX12: v_cmpx_nge_f16_e64 v255, v255           ; encoding: [0x7e,0x00,0x89,0xd4,0xff,0xff,0x03,0x00]
+v_cmpx_nge_f16_e64 v255.l, v255.l
+// GFX12: v_cmpx_nge_f16_e64 v255.l, v255.l       ; encoding: [0x7e,0x00,0x89,0xd4,0xff,0xff,0x03,0x00]
 
 v_cmpx_nge_f16_e64 s1, s2
 // GFX12: v_cmpx_nge_f16_e64 s1, s2               ; encoding: [0x7e,0x00,0x89,0xd4,0x01,0x04,0x00,0x00]
@@ -2663,6 +2699,12 @@ v_cmpx_nge_f16_e64 -src_scc, |vcc_lo|
 v_cmpx_nge_f16_e64 -|0xfe0b|, -|vcc_hi| clamp
 // GFX12: v_cmpx_nge_f16_e64 -|0xfe0b|, -|vcc_hi| clamp ; encoding: [0x7e,0x83,0x89,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00]
 
+v_cmpx_nge_f16_e64 v1.h, v2.l
+// GFX12: v_cmpx_nge_f16_e64 v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x7e,0x08,0x89,0xd4,0x01,0x05,0x02,0x00]
+
+v_cmpx_nge_f16_e64 v255.l, v255.h
+// GFX12: v_cmpx_nge_f16_e64 v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x7e,0x10,0x89,0xd4,0xff,0xff,0x03,0x00]
+
 v_cmpx_nge_f32_e64 v1, v2
 // GFX12: v_cmpx_nge_f32_e64 v1, v2               ; encoding: [0x7e,0x00,0x99,0xd4,0x01,0x05,0x02,0x00]
 
@@ -2744,11 +2786,11 @@ v_cmpx_nge_f64_e64 -|src_scc|, -|exec|
 v_cmpx_nge_f64_e64 0xaf123456, -|vcc| clamp
 // GFX12: v_cmpx_nge_f64_e64 0xaf123456, -|vcc| clamp ; encoding: [0x7e,0x82,0xa9,0xd4,0xff,0xd4,0x00,0x40,0x56,0x34,0x12,0xaf]
 
-v_cmpx_ngt_f16_e64 v1, v2
-// GFX12: v_cmpx_ngt_f16_e64 v1, v2               ; encoding: [0x7e,0x00,0x8b,0xd4,0x01,0x05,0x02,0x00]
+v_cmpx_ngt_f16_e64 v1.l, v2.l
+// GFX12: v_cmpx_ngt_f16_e64 v1.l, v2.l           ; encoding: [0x7e,0x00,0x8b,0xd4,0x01,0x05,0x02,0x00]
 
-v_cmpx_ngt_f16_e64 v255, v255
-// GFX12: v_cmpx_ngt_f16_e64 v255, v255           ; encoding: [0x7e,0x00,0x8b,0xd4,0xff,0xff,0x03,0x00]
+v_cmpx_ngt_f16_e64 v255.l, v255.l
+// GFX12: v_cmpx_ngt_f16_e64 v255.l, v255.l       ; encoding: [0x7e,0x00,0x8b,0xd4,0xff,0xff,0x03,0x00]
 
 v_cmpx_ngt_f16_e64 s1, s2
 // GFX12: v_cmpx_ngt_f16_e64 s1, s2               ; encoding: [0x7e,0x00,0x8b,0xd4,0x01,0x04,0x00,0x00]
@@ -2789,6 +2831,12 @@ v_cmpx_ngt_f16_e64 -src_scc, |vcc_lo|
 v_cmpx_ngt_f16_e64 -|0xfe0b|, -|vcc_hi| clamp
 // GFX12: v_cmpx_ngt_f16_e64 -|0xfe0b|, -|vcc_hi| clamp ; encoding: [0x7e,0x83,0x8b,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00]
 
+v_cmpx_ngt_f16_e64 v1.h, v2.l
+// GFX12: v_cmpx_ngt_f16_e64 v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x7e,0x08,0x8b,0xd4,0x01,0x05,0x02,0x00]
+
+v_cmpx_ngt_f16_e64 v255.l, v255.h
+// GFX12: v_cmpx_ngt_f16_e64 v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x7e,0x10,0x8b,0xd4,0xff,0xff,0x03,0x00]
+
 v_cmpx_ngt_f32_e64 v1, v2
 // GFX12: v_cmpx_ngt_f32_e64 v1, v2               ; encoding: [0x7e,0x00,0x9b,0xd4,0x01,0x05,0x02,0x00]
 
@@ -2870,11 +2918,11 @@ v_cmpx_ngt_f64_e64 -|src_scc|, -|exec|
 v_cmpx_ngt_f64_e64 0xaf123456, -|vcc| clamp
 // GFX12: v_cmpx_ngt_f64_e64 0xaf123456, -|vcc| clamp ; encoding: [0x7e,0x82,0xab,0xd4,0xff,0xd4,0x00,0x40,0x56,0x34,0x12,0xaf]
 
-v_cmpx_nle_f16_e64 v1, v2
-// GFX12: v_cmpx_nle_f16_e64 v1, v2               ; encoding: [0x7e,0x00,0x8c,0xd4,0x01,0x05,0x02,0x00]
+v_cmpx_nle_f16_e64 v1.l, v2.l
+// GFX12: v_cmpx_nle_f16_e64 v1.l, v2.l           ; encoding: [0x7e,0x00,0x8c,0xd4,0x01,0x05,0x02,0x00]
 
-v_cmpx_nle_f16_e64 v255, v255
-// GFX12: v_cmpx_nle_f16_e64 v255, v255           ; encoding: [0x7e,0x00,0x8c,0xd4,0xff,0xff,0x03,0x00]
+v_cmpx_nle_f16_e64 v255.l, v255.l
+// GFX12: v_cmpx_nle_f16_e64 v255.l, v255.l       ; encoding: [0x7e,0x00,0x8c,0xd4,0xff,0xff,0x03,0x00]
 
 v_cmpx_nle_f16_e64 s1, s2
 // GFX12: v_cmpx_nle_f16_e64 s1, s2               ; encoding: [0x7e,0x00,0x8c,0xd4,0x01,0x04,0x00,0x00]
@@ -2915,6 +2963,12 @@ v_cmpx_nle_f16_e64 -src_scc, |vcc_lo|
 v_cmpx_nle_f16_e64 -|0xfe0b|, -|vcc_hi| clamp
 // GFX12: v_cmpx_nle_f16_e64 -|0xfe0b|, -|vcc_hi| clamp ; encoding: [0x7e,0x83,0x8c,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00]
 
+v_cmpx_nle_f16_e64 v1.h, v2.l
+// GFX12: v_cmpx_nle_f16_e64 v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x7e,0x08,0x8c,0xd4,0x01,0x05,0x02,0x00]
+
+v_cmpx_nle_f16_e64 v255.l, v255.h
+// GFX12: v_cmpx_nle_f16_e64 v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x7e,0x10,0x8c,0xd4,0xff,0xff,0x03,0x00]
+
 v_cmpx_nle_f32_e64 v1, v2
 // GFX12: v_cmpx_nle_f32_e64 v1, v2               ; encoding: [0x7e,0x00,0x9c,0xd4,0x01,0x05,0x02,0x00]
 
@@ -2996,11 +3050,11 @@ v_cmpx_nle_f64_e64 -|src_scc|, -|exec|
 v_cmpx_nle_f64_e64 0xaf123456, -|vcc| clamp
 // GFX12: v_cmpx_nle_f64_e64 0xaf123456, -|vcc| clamp ; encoding: [0x7e,0x82,0xac,0xd4,0xff,0xd4,0x00,0x40,0x56,0x34,0x12,0xaf]
 
-v_cmpx_nlg_f16_e64 v1, v2
-// GFX12: v_cmpx_nlg_f16_e64 v1, v2               ; encoding: [0x7e,0x00,0x8a,0xd4,0x01,0x05,0x02,0x00]
+v_cmpx_nlg_f16_e64 v1.l, v2.l
+// GFX12: v_cmpx_nlg_f16_e64 v1.l, v2.l           ; encoding: [0x7e,0x00,0x8a,0xd4,0x01,0x05,0x02,0x00]
 
-v_cmpx_nlg_f16_e64 v255, v255
-// GFX12: v_cmpx_nlg_f16_e64 v255, v255           ; encoding: [0x7e,0x00,0x8a,0xd4,0xff,0xff,0x03,0x00]
+v_cmpx_nlg_f16_e64 v255.l, v255.l
+// GFX12: v_cmpx_nlg_f16_e64 v255.l, v255.l       ; encoding: [0x7e,0x00,0x8a,0xd4,0xff,0xff,0x03,0x00]
 
 v_cmpx_nlg_f16_e64 s1, s2
 // GFX12: v_cmpx_nlg_f16_e64 s1, s2               ; encoding: [0x7e,0x00,0x8a,0xd4,0x01,0x04,0x00,0x00]
@@ -3041,6 +3095,12 @@ v_cmpx_nlg_f16_e64 -src_scc, |vcc_lo|
 v_cmpx_nlg_f16_e64 -|0xfe0b|, -|vcc_hi| clamp
 // GFX12: v_cmpx_nlg_f16_e64 -|0xfe0b|, -|vcc_hi| clamp ; encoding: [0x7e,0x83,0x8a,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00]
 
+v_cmpx_nlg_f16_e64 v1.h, v2.l
+// GFX12: v_cmpx_nlg_f16_e64 v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x7e,0x08,0x8a,0xd4,0x01,0x05,0x02,0x00]
+
+v_cmpx_nlg_f16_e64 v255.l, v255.h
+// GFX12: v_cmpx_nlg_f16_e64 v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x7e,0x10,0x8a,0xd4,0xff,0xff,0x03,0x00]
+
 v_cmpx_nlg_f32_e64 v1, v2
 // GFX12: v_cmpx_nlg_f32_e64 v1, v2               ; encoding: [0x7e,0x00,0x9a,0xd4,0x01,0x05,0x02,0x00]
 
@@ -3122,11 +3182,11 @@ v_cmpx_nlg_f64_e64 -|src_scc|, -|exec|
 v_cmpx_nlg_f64_e64 0xaf123456, -|vcc| clamp
 // GFX12: v_cmpx_nlg_f64_e64 0xaf123456, -|vcc| clamp ; encoding: [0x7e,0x82,0xaa,0xd4,0xff,0xd4,0x00,0x40,0x56,0x34,0x12,0xaf]
 
-v_cmpx_nlt_f16_e64 v1, v2
-// GFX12: v_cmpx_nlt_f16_e64 v1, v2               ; encoding: [0x7e,0x00,0x8e,0xd4,0x01,0x05,0x02,0x00]
+v_cmpx_nlt_f16_e64 v1.l, v2.l
+// GFX12: v_cmpx_nlt_f16_e64 v1.l, v2.l           ; encoding: [0x7e,0x00,0x8e,0xd4,0x01,0x05,0x02,0x00]
 
-v_cmpx_nlt_f16_e64 v255, v255
-// GFX12: v_cmpx_nlt_f16_e64 v255, v255           ; encoding: [0x7e,0x00,0x8e,0xd4,0xff,0xff,0x03,0x00]
+v_cmpx_nlt_f16_e64 v255.l, v255.l
+// GFX12: v_cmpx_nlt_f16_e64 v255.l, v255.l       ; encoding: [0x7e,0x00,0x8e,0xd4,0xff,0xff,0x03,0x00]
 
 v_cmpx_nlt_f16_e64 s1, s2
 // GFX12: v_cmpx_nlt_f16_e64 s1, s2               ; encoding: [0x7e,0x00,0x8e,0xd4,0x01,0x04,0x00,0x00]
@@ -3167,6 +3227,12 @@ v_cmpx_nlt_f16_e64 -src_scc, |vcc_lo|
 v_cmpx_nlt_f16_e64 -|0xfe0b|, -|vcc_hi| clamp
 // GFX12: v_cmpx_nlt_f16_e64 -|0xfe0b|, -|vcc_hi| clamp ; encoding: [0x7e,0x83,0x8e,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00]
 
+v_cmpx_nlt_f16_e64 v1.h, v2.l
+// GFX12: v_cmpx_nlt_f16_e64 v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x7e,0x08,0x8e,0xd4,0x01,0x05,0x02,0x00]
+
+v_cmpx_nlt_f16_e64 v255.l, v255.h
+// GFX12: v_cmpx_nlt_f16_e64 v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x7e,0x10,0x8e,0xd4,0xff,0xff,0x03,0x00]
+
 v_cmpx_nlt_f32_e64 v1, v2
 // GFX12: v_cmpx_nlt_f32_e64 v1, v2               ; encoding: [0x7e,0x00,0x9e,0xd4,0x01,0x05,0x02,0x00]
 
@@ -3248,11 +3314,11 @@ v_cmpx_nlt_f64_e64 -|src_scc|, -|exec|
 v_cmpx_nlt_f64_e64 0xaf123456, -|vcc| clamp
 // GFX12: v_cmpx_nlt_f64_e64 0xaf123456, -|vcc| clamp ; encoding: [0x7e,0x82,0xae,0xd4,0xff,0xd4,0x00,0x40,0x56,0x34,0x12,0xaf]
 
-v_cmpx_o_f16_e64 v1, v2
-// GFX12: v_cmpx_o_f16_e64 v1, v2                 ; encoding: [0x7e,0x00,0x87,0xd4,0x01,0x05,0x02,0x00]
+v_cmpx_o_f16_e64 v1.l, v2.l
+// GFX12: v_cmpx_o_f16_e64 v1.l, v2.l             ; encoding: [0x7e,0x00,0x87,0xd4,0x01,0x05,0x02,0x00]
 
-v_cmpx_o_f16_e64 v255, v255
-// GFX12: v_cmpx_o_f16_e64 v255, v255             ; encoding: [0x7e,0x00,0x87,0xd4,0xff,0xff,0x03,0x00]
+v_cmpx_o_f16_e64 v255.l, v255.l
+// GFX12: v_cmpx_o_f16_e64 v255.l, v255.l         ; encoding: [0x7e,0x00,0x87,0xd4,0xff,0xff,0x03,0x00]
 
 v_cmpx_o_f16_e64 s1, s2
 // GFX12: v_cmpx_o_f16_e64 s1, s2                 ; encoding: [0x7e,0x00,0x87,0xd4,0x01,0x04,0x00,0x00]
@@ -3293,6 +3359,12 @@ v_cmpx_o_f16_e64 -src_scc, |vcc_lo|
 v_cmpx_o_f16_e64 -|0xfe0b|, -|vcc_hi| clamp
 // GFX12: v_cmpx_o_f16_e64 -|0xfe0b|, -|vcc_hi| clamp ; encoding: [0x7e,0x83,0x87,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00]
 
+v_cmpx_o_f16_e64 v1.h, v2.l
+// GFX12: v_cmpx_o_f16_e64 v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x7e,0x08,0x87,0xd4,0x01,0x05,0x02,0x00]
+
+v_cmpx_o_f16_e64 v255.l, v255.h
+// GFX12: v_cmpx_o_f16_e64 v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x7e,0x10,0x87,0xd4,0xff,0xff,0x03,0x00]
+
 v_cmpx_o_f32_e64 v1, v2
 // GFX12: v_cmpx_o_f32_e64 v1, v2                 ; encoding: [0x7e,0x00,0x97,0xd4,0x01,0x05,0x02,0x00]
 
@@ -3374,11 +3446,11 @@ v_cmpx_o_f64_e64 -|src_scc|, -|exec|
 v_cmpx_o_f64_e64 0xaf123456, -|vcc| clamp
 // GFX12: v_cmpx_o_f64_e64 0xaf123456, -|vcc| clamp ; encoding: [0x7e,0x82,0xa7,0xd4,0xff,0xd4,0x00,0x40,0x56,0x34,0x12,0xaf]
 
-v_cmpx_u_f16_e64 v1, v2
-// GFX12: v_cmpx_u_f16_e64 v1, v2                 ; encoding: [0x7e,0x00,0x88,0xd4,0x01,0x05,0x02,0x00]
+v_cmpx_u_f16_e64 v1.l, v2.l
+// GFX12: v_cmpx_u_f16_e64 v1.l, v2.l             ; encoding: [0x7e,0x00,0x88,0xd4,0x01,0x05,0x02,0x00]
 
-v_cmpx_u_f16_e64 v255, v255
-// GFX12: v_cmpx_u_f16_e64 v255, v255             ; encoding: [0x7e,0x00,0x88,0xd4,0xff,0xff,0x03,0x00]
+v_cmpx_u_f16_e64 v255.l, v255.l
+// GFX12: v_cmpx_u_f16_e64 v255.l, v255.l         ; encoding: [0x7e,0x00,0x88,0xd4,0xff,0xff,0x03,0x00]
 
 v_cmpx_u_f16_e64 s1, s2
 // GFX12: v_cmpx_u_f16_e64 s1, s2                 ; encoding: [0x7e,0x00,0x88,0xd4,0x01,0x04,0x00,0x00]
@@ -3419,6 +3491,12 @@ v_cmpx_u_f16_e64 -src_scc, |vcc_lo|
 v_cmpx_u_f16_e64 -|0xfe0b|, -|vcc_hi| clamp
 // GFX12: v_cmpx_u_f16_e64 -|0xfe0b|, -|vcc_hi| clamp ; encoding: [0x7e,0x83,0x88,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00]
 
+v_cmpx_u_f16_e64 v1.h, v2.l
+// GFX12: v_cmpx_u_f16_e64 v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x7e,0x08,0x88,0xd4,0x01,0x05,0x02,0x00]
+
+v_cmpx_u_f16_e64 v255.l, v255.h
+// GFX12: v_cmpx_u_f16_e64 v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x7e,0x10,0x88,0xd4,0xff,0xff,0x03,0x00]
+
 v_cmpx_u_f32_e64 v1, v2
 // GFX12: v_cmpx_u_f32_e64 v1, v2                 ; encoding: [0x7e,0x00,0x98,0xd4,0x01,0x05,0x02,0x00]
 
diff --git a/llvm/test/MC/AMDGPU/gfx12_asm_vop3cx_dpp16.s b/llvm/test/MC/AMDGPU/gfx12_asm_vop3cx_dpp16.s
index faad68f902d5f..aca2b4792f74a 100644
--- a/llvm/test/MC/AMDGPU/gfx12_asm_vop3cx_dpp16.s
+++ b/llvm/test/MC/AMDGPU/gfx12_asm_vop3cx_dpp16.s
@@ -107,53 +107,62 @@ v_cmpx_class_f32_e64_dpp v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctr
 v_cmpx_class_f32_e64_dpp -|v255|, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
 // GFX12: v_cmpx_class_f32_e64_dpp -|v255|, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0x7e,0x01,0xfe,0xd4,0xfa,0xfe,0x03,0x20,0xff,0x6f,0x05,0x30]
 
-v_cmpx_eq_f16_e64_dpp v1, v2 quad_perm:[3,2,1,0]
-// GFX12: v_cmpx_eq_f16_e64_dpp v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x82,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+v_cmpx_eq_f16_e64_dpp v1.l, v2.l quad_perm:[3,2,1,0]
+// GFX12: v_cmpx_eq_f16_e64_dpp v1.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x82,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
 
-v_cmpx_eq_f16_e64_dpp v1, s2 quad_perm:[3,2,1,0]
-// GFX12: v_cmpx_eq_f16_e64_dpp v1, s2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x82,0xd4,0xfa,0x04,0x00,0x00,0x01,0x1b,0x00,0xff]
+v_cmpx_eq_f16_e64_dpp v1.l, s2 quad_perm:[3,2,1,0]
+// GFX12: v_cmpx_eq_f16_e64_dpp v1.l, s2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x82,0xd4,0xfa,0x04,0x00,0x00,0x01,0x1b,0x00,0xff]
 
-v_cmpx_eq_f16_e64_dpp v1, 2.0 quad_perm:[3,2,1,0]
-// GFX12: v_cmpx_eq_f16_e64_dpp v1, 2.0 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x82,0xd4,0xfa,0xe8,0x01,0x00,0x01,0x1b,0x00,0xff]
+v_cmpx_eq_f16_e64_dpp v1.l, 2.0 quad_perm:[3,2,1,0]
+// GFX12: v_cmpx_eq_f16_e64_dpp v1.l, 2.0 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x82,0xd4,0xfa,0xe8,0x01,0x00,0x01,0x1b,0x00,0xff]
 
-v_cmpx_eq_f16_e64_dpp v1, v2 quad_perm:[0,1,2,3]
-// GFX12: v_cmpx_eq_f16_e64_dpp v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x82,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+v_cmpx_eq_f16_e64_dpp v1.l, v2.l quad_perm:[0,1,2,3]
+// GFX12: v_cmpx_eq_f16_e64_dpp v1.l, v2.l quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x82,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
 
-v_cmpx_eq_f16_e64_dpp v1, v2 row_mirror
-// GFX12: v_cmpx_eq_f16_e64_dpp v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x82,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+v_cmpx_eq_f16_e64_dpp v1.l, v2.l row_mirror
+// GFX12: v_cmpx_eq_f16_e64_dpp v1.l, v2.l row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x82,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
 
-v_cmpx_eq_f16_e64_dpp v1, v2 row_half_mirror
-// GFX12: v_cmpx_eq_f16_e64_dpp v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x82,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+v_cmpx_eq_f16_e64_dpp v1.l, v2.l row_half_mirror
+// GFX12: v_cmpx_eq_f16_e64_dpp v1.l, v2.l row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x82,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
 
-v_cmpx_eq_f16_e64_dpp v1, v2 row_shl:1
-// GFX12: v_cmpx_eq_f16_e64_dpp v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x82,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+v_cmpx_eq_f16_e64_dpp v1.l, v2.l row_shl:1
+// GFX12: v_cmpx_eq_f16_e64_dpp v1.l, v2.l row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x82,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
 
-v_cmpx_eq_f16_e64_dpp v1, v2 row_shl:15
-// GFX12: v_cmpx_eq_f16_e64_dpp v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x82,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+v_cmpx_eq_f16_e64_dpp v1.l, v2.l row_shl:15
+// GFX12: v_cmpx_eq_f16_e64_dpp v1.l, v2.l row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x82,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
 
-v_cmpx_eq_f16_e64_dpp v1, v2 row_shr:1
-// GFX12: v_cmpx_eq_f16_e64_dpp v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x82,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+v_cmpx_eq_f16_e64_dpp v1.l, v2.l row_shr:1
+// GFX12: v_cmpx_eq_f16_e64_dpp v1.l, v2.l row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x82,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
 
-v_cmpx_eq_f16_e64_dpp v1, v2 row_shr:15
-// GFX12: v_cmpx_eq_f16_e64_dpp v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x82,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+v_cmpx_eq_f16_e64_dpp v1.l, v2.l row_shr:15
+// GFX12: v_cmpx_eq_f16_e64_dpp v1.l, v2.l row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x82,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
 
-v_cmpx_eq_f16_e64_dpp v1, v2 row_ror:1
-// GFX12: v_cmpx_eq_f16_e64_dpp v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x82,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+v_cmpx_eq_f16_e64_dpp v1.l, v2.l row_ror:1
+// GFX12: v_cmpx_eq_f16_e64_dpp v1.l, v2.l row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x82,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
 
-v_cmpx_eq_f16_e64_dpp v1, v2 row_ror:15
-// GFX12: v_cmpx_eq_f16_e64_dpp v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x82,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+v_cmpx_eq_f16_e64_dpp v1.l, v2.l row_ror:15
+// GFX12: v_cmpx_eq_f16_e64_dpp v1.l, v2.l row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x82,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
 
-v_cmpx_eq_f16_e64_dpp v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
-// GFX12: v_cmpx_eq_f16_e64_dpp v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x82,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+v_cmpx_eq_f16_e64_dpp v1.l, v2.l row_share:0 row_mask:0xf bank_mask:0xf
+// GFX12: v_cmpx_eq_f16_e64_dpp v1.l, v2.l row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x82,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 
-v_cmpx_eq_f16_e64_dpp |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1
-// GFX12: v_cmpx_eq_f16_e64_dpp |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x01,0x82,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+v_cmpx_eq_f16_e64_dpp |v1.l|, -v2.l row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX12: v_cmpx_eq_f16_e64_dpp |v1.l|, -v2.l row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x01,0x82,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
 
-v_cmpx_eq_f16_e64_dpp -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
-// GFX12: v_cmpx_eq_f16_e64_dpp -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7e,0x02,0x82,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x09,0x13]
+v_cmpx_eq_f16_e64_dpp -v1.l, |v2.l| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX12: v_cmpx_eq_f16_e64_dpp -v1.l, |v2.l| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7e,0x02,0x82,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x09,0x13]
 
-v_cmpx_eq_f16_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
-// GFX12: v_cmpx_eq_f16_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0x7e,0x83,0x82,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x05,0x30]
+v_cmpx_eq_f16_e64_dpp -|v255.l|, -|v255.l| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX12: v_cmpx_eq_f16_e64_dpp -|v255.l|, -|v255.l| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0x7e,0x83,0x82,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x05,0x30]
+
+v_cmpx_eq_f16_e64_dpp |v1.h|, -v2.h row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX12: v_cmpx_eq_f16_e64_dpp |v1.h|, -v2.h op_sel:[1,1] row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x19,0x82,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+
+v_cmpx_eq_f16_e64_dpp -v1.h, |v2.l| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX12: v_cmpx_eq_f16_e64_dpp -v1.h, |v2.l| op_sel:[1,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7e,0x0a,0x82,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x09,0x13]
+
+v_cmpx_eq_f16_e64_dpp -|v255.l|, -|v255.h| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX12: v_cmpx_eq_f16_e64_dpp -|v255.l|, -|v255.h| op_sel:[0,1] clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0x7e,0x93,0x82,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x05,0x30]
 
 v_cmpx_eq_f32_e64_dpp v1, v2 quad_perm:[3,2,1,0]
 // GFX12: v_cmpx_eq_f32_e64_dpp v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x92,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
@@ -413,53 +422,62 @@ v_cmpx_eq_u32_e64_dpp v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1
 v_cmpx_eq_u32_e64_dpp v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
 // GFX12: v_cmpx_eq_u32_e64_dpp v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0x7e,0x00,0xca,0xd4,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x05,0x30]
 
-v_cmpx_ge_f16_e64_dpp v1, v2 quad_perm:[3,2,1,0]
-// GFX12: v_cmpx_ge_f16_e64_dpp v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x86,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+v_cmpx_ge_f16_e64_dpp v1.l, v2.l quad_perm:[3,2,1,0]
+// GFX12: v_cmpx_ge_f16_e64_dpp v1.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x86,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+
+v_cmpx_ge_f16_e64_dpp v1.l, s2 quad_perm:[3,2,1,0]
+// GFX12: v_cmpx_ge_f16_e64_dpp v1.l, s2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x86,0xd4,0xfa,0x04,0x00,0x00,0x01,0x1b,0x00,0xff]
+
+v_cmpx_ge_f16_e64_dpp v1.l, 2.0 quad_perm:[3,2,1,0]
+// GFX12: v_cmpx_ge_f16_e64_dpp v1.l, 2.0 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x86,0xd4,0xfa,0xe8,0x01,0x00,0x01,0x1b,0x00,0xff]
 
-v_cmpx_ge_f16_e64_dpp v1, s2 quad_perm:[3,2,1,0]
-// GFX12: v_cmpx_ge_f16_e64_dpp v1, s2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x86,0xd4,0xfa,0x04,0x00,0x00,0x01,0x1b,0x00,0xff]
+v_cmpx_ge_f16_e64_dpp v1.l, v2.l quad_perm:[0,1,2,3]
+// GFX12: v_cmpx_ge_f16_e64_dpp v1.l, v2.l quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x86,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
 
-v_cmpx_ge_f16_e64_dpp v1, 2.0 quad_perm:[3,2,1,0]
-// GFX12: v_cmpx_ge_f16_e64_dpp v1, 2.0 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x86,0xd4,0xfa,0xe8,0x01,0x00,0x01,0x1b,0x00,0xff]
+v_cmpx_ge_f16_e64_dpp v1.l, v2.l row_mirror
+// GFX12: v_cmpx_ge_f16_e64_dpp v1.l, v2.l row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x86,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
 
-v_cmpx_ge_f16_e64_dpp v1, v2 quad_perm:[0,1,2,3]
-// GFX12: v_cmpx_ge_f16_e64_dpp v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x86,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+v_cmpx_ge_f16_e64_dpp v1.l, v2.l row_half_mirror
+// GFX12: v_cmpx_ge_f16_e64_dpp v1.l, v2.l row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x86,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
 
-v_cmpx_ge_f16_e64_dpp v1, v2 row_mirror
-// GFX12: v_cmpx_ge_f16_e64_dpp v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x86,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+v_cmpx_ge_f16_e64_dpp v1.l, v2.l row_shl:1
+// GFX12: v_cmpx_ge_f16_e64_dpp v1.l, v2.l row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x86,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
 
-v_cmpx_ge_f16_e64_dpp v1, v2 row_half_mirror
-// GFX12: v_cmpx_ge_f16_e64_dpp v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x86,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+v_cmpx_ge_f16_e64_dpp v1.l, v2.l row_shl:15
+// GFX12: v_cmpx_ge_f16_e64_dpp v1.l, v2.l row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x86,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
 
-v_cmpx_ge_f16_e64_dpp v1, v2 row_shl:1
-// GFX12: v_cmpx_ge_f16_e64_dpp v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x86,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+v_cmpx_ge_f16_e64_dpp v1.l, v2.l row_shr:1
+// GFX12: v_cmpx_ge_f16_e64_dpp v1.l, v2.l row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x86,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
 
-v_cmpx_ge_f16_e64_dpp v1, v2 row_shl:15
-// GFX12: v_cmpx_ge_f16_e64_dpp v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x86,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+v_cmpx_ge_f16_e64_dpp v1.l, v2.l row_shr:15
+// GFX12: v_cmpx_ge_f16_e64_dpp v1.l, v2.l row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x86,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
 
-v_cmpx_ge_f16_e64_dpp v1, v2 row_shr:1
-// GFX12: v_cmpx_ge_f16_e64_dpp v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x86,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+v_cmpx_ge_f16_e64_dpp v1.l, v2.l row_ror:1
+// GFX12: v_cmpx_ge_f16_e64_dpp v1.l, v2.l row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x86,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
 
-v_cmpx_ge_f16_e64_dpp v1, v2 row_shr:15
-// GFX12: v_cmpx_ge_f16_e64_dpp v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x86,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+v_cmpx_ge_f16_e64_dpp v1.l, v2.l row_ror:15
+// GFX12: v_cmpx_ge_f16_e64_dpp v1.l, v2.l row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x86,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
 
-v_cmpx_ge_f16_e64_dpp v1, v2 row_ror:1
-// GFX12: v_cmpx_ge_f16_e64_dpp v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x86,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+v_cmpx_ge_f16_e64_dpp v1.l, v2.l row_share:0 row_mask:0xf bank_mask:0xf
+// GFX12: v_cmpx_ge_f16_e64_dpp v1.l, v2.l row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x86,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 
-v_cmpx_ge_f16_e64_dpp v1, v2 row_ror:15
-// GFX12: v_cmpx_ge_f16_e64_dpp v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x86,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+v_cmpx_ge_f16_e64_dpp |v1.l|, -v2.l row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX12: v_cmpx_ge_f16_e64_dpp |v1.l|, -v2.l row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x01,0x86,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
 
-v_cmpx_ge_f16_e64_dpp v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
-// GFX12: v_cmpx_ge_f16_e64_dpp v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x86,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+v_cmpx_ge_f16_e64_dpp -v1.l, |v2.l| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX12: v_cmpx_ge_f16_e64_dpp -v1.l, |v2.l| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7e,0x02,0x86,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x09,0x13]
 
-v_cmpx_ge_f16_e64_dpp |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1
-// GFX12: v_cmpx_ge_f16_e64_dpp |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x01,0x86,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+v_cmpx_ge_f16_e64_dpp -|v255.l|, -|v255.l| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX12: v_cmpx_ge_f16_e64_dpp -|v255.l|, -|v255.l| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0x7e,0x83,0x86,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x05,0x30]
 
-v_cmpx_ge_f16_e64_dpp -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
-// GFX12: v_cmpx_ge_f16_e64_dpp -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7e,0x02,0x86,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x09,0x13]
+v_cmpx_ge_f16_e64_dpp |v1.h|, -v2.h row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX12: v_cmpx_ge_f16_e64_dpp |v1.h|, -v2.h op_sel:[1,1] row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x19,0x86,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
 
-v_cmpx_ge_f16_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
-// GFX12: v_cmpx_ge_f16_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0x7e,0x83,0x86,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x05,0x30]
+v_cmpx_ge_f16_e64_dpp -v1.h, |v2.l| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX12: v_cmpx_ge_f16_e64_dpp -v1.h, |v2.l| op_sel:[1,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7e,0x0a,0x86,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x09,0x13]
+
+v_cmpx_ge_f16_e64_dpp -|v255.l|, -|v255.h| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX12: v_cmpx_ge_f16_e64_dpp -|v255.l|, -|v255.h| op_sel:[0,1] clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0x7e,0x93,0x86,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x05,0x30]
 
 v_cmpx_ge_f32_e64_dpp v1, v2 quad_perm:[3,2,1,0]
 // GFX12: v_cmpx_ge_f32_e64_dpp v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x96,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
@@ -719,53 +737,62 @@ v_cmpx_ge_u32_e64_dpp v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1
 v_cmpx_ge_u32_e64_dpp v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
 // GFX12: v_cmpx_ge_u32_e64_dpp v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0x7e,0x00,0xce,0xd4,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x05,0x30]
 
-v_cmpx_gt_f16_e64_dpp v1, v2 quad_perm:[3,2,1,0]
-// GFX12: v_cmpx_gt_f16_e64_dpp v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x84,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+v_cmpx_gt_f16_e64_dpp v1.l, v2.l quad_perm:[3,2,1,0]
+// GFX12: v_cmpx_gt_f16_e64_dpp v1.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x84,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+
+v_cmpx_gt_f16_e64_dpp v1.l, s2 quad_perm:[3,2,1,0]
+// GFX12: v_cmpx_gt_f16_e64_dpp v1.l, s2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x84,0xd4,0xfa,0x04,0x00,0x00,0x01,0x1b,0x00,0xff]
+
+v_cmpx_gt_f16_e64_dpp v1.l, 2.0 quad_perm:[3,2,1,0]
+// GFX12: v_cmpx_gt_f16_e64_dpp v1.l, 2.0 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x84,0xd4,0xfa,0xe8,0x01,0x00,0x01,0x1b,0x00,0xff]
 
-v_cmpx_gt_f16_e64_dpp v1, s2 quad_perm:[3,2,1,0]
-// GFX12: v_cmpx_gt_f16_e64_dpp v1, s2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x84,0xd4,0xfa,0x04,0x00,0x00,0x01,0x1b,0x00,0xff]
+v_cmpx_gt_f16_e64_dpp v1.l, v2.l quad_perm:[0,1,2,3]
+// GFX12: v_cmpx_gt_f16_e64_dpp v1.l, v2.l quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x84,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
 
-v_cmpx_gt_f16_e64_dpp v1, 2.0 quad_perm:[3,2,1,0]
-// GFX12: v_cmpx_gt_f16_e64_dpp v1, 2.0 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x84,0xd4,0xfa,0xe8,0x01,0x00,0x01,0x1b,0x00,0xff]
+v_cmpx_gt_f16_e64_dpp v1.l, v2.l row_mirror
+// GFX12: v_cmpx_gt_f16_e64_dpp v1.l, v2.l row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x84,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
 
-v_cmpx_gt_f16_e64_dpp v1, v2 quad_perm:[0,1,2,3]
-// GFX12: v_cmpx_gt_f16_e64_dpp v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x84,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+v_cmpx_gt_f16_e64_dpp v1.l, v2.l row_half_mirror
+// GFX12: v_cmpx_gt_f16_e64_dpp v1.l, v2.l row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x84,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
 
-v_cmpx_gt_f16_e64_dpp v1, v2 row_mirror
-// GFX12: v_cmpx_gt_f16_e64_dpp v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x84,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+v_cmpx_gt_f16_e64_dpp v1.l, v2.l row_shl:1
+// GFX12: v_cmpx_gt_f16_e64_dpp v1.l, v2.l row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x84,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
 
-v_cmpx_gt_f16_e64_dpp v1, v2 row_half_mirror
-// GFX12: v_cmpx_gt_f16_e64_dpp v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x84,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+v_cmpx_gt_f16_e64_dpp v1.l, v2.l row_shl:15
+// GFX12: v_cmpx_gt_f16_e64_dpp v1.l, v2.l row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x84,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
 
-v_cmpx_gt_f16_e64_dpp v1, v2 row_shl:1
-// GFX12: v_cmpx_gt_f16_e64_dpp v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x84,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+v_cmpx_gt_f16_e64_dpp v1.l, v2.l row_shr:1
+// GFX12: v_cmpx_gt_f16_e64_dpp v1.l, v2.l row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x84,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
 
-v_cmpx_gt_f16_e64_dpp v1, v2 row_shl:15
-// GFX12: v_cmpx_gt_f16_e64_dpp v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x84,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+v_cmpx_gt_f16_e64_dpp v1.l, v2.l row_shr:15
+// GFX12: v_cmpx_gt_f16_e64_dpp v1.l, v2.l row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x84,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
 
-v_cmpx_gt_f16_e64_dpp v1, v2 row_shr:1
-// GFX12: v_cmpx_gt_f16_e64_dpp v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x84,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+v_cmpx_gt_f16_e64_dpp v1.l, v2.l row_ror:1
+// GFX12: v_cmpx_gt_f16_e64_dpp v1.l, v2.l row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x84,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
 
-v_cmpx_gt_f16_e64_dpp v1, v2 row_shr:15
-// GFX12: v_cmpx_gt_f16_e64_dpp v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x84,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+v_cmpx_gt_f16_e64_dpp v1.l, v2.l row_ror:15
+// GFX12: v_cmpx_gt_f16_e64_dpp v1.l, v2.l row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x84,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
 
-v_cmpx_gt_f16_e64_dpp v1, v2 row_ror:1
-// GFX12: v_cmpx_gt_f16_e64_dpp v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x84,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+v_cmpx_gt_f16_e64_dpp v1.l, v2.l row_share:0 row_mask:0xf bank_mask:0xf
+// GFX12: v_cmpx_gt_f16_e64_dpp v1.l, v2.l row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x84,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 
-v_cmpx_gt_f16_e64_dpp v1, v2 row_ror:15
-// GFX12: v_cmpx_gt_f16_e64_dpp v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x84,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+v_cmpx_gt_f16_e64_dpp |v1.l|, -v2.l row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX12: v_cmpx_gt_f16_e64_dpp |v1.l|, -v2.l row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x01,0x84,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
 
-v_cmpx_gt_f16_e64_dpp v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
-// GFX12: v_cmpx_gt_f16_e64_dpp v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x84,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+v_cmpx_gt_f16_e64_dpp -v1.l, |v2.l| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX12: v_cmpx_gt_f16_e64_dpp -v1.l, |v2.l| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7e,0x02,0x84,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x09,0x13]
 
-v_cmpx_gt_f16_e64_dpp |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1
-// GFX12: v_cmpx_gt_f16_e64_dpp |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x01,0x84,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+v_cmpx_gt_f16_e64_dpp -|v255.l|, -|v255.l| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX12: v_cmpx_gt_f16_e64_dpp -|v255.l|, -|v255.l| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0x7e,0x83,0x84,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x05,0x30]
 
-v_cmpx_gt_f16_e64_dpp -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
-// GFX12: v_cmpx_gt_f16_e64_dpp -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7e,0x02,0x84,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x09,0x13]
+v_cmpx_gt_f16_e64_dpp |v1.h|, -v2.h row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX12: v_cmpx_gt_f16_e64_dpp |v1.h|, -v2.h op_sel:[1,1] row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x19,0x84,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
 
-v_cmpx_gt_f16_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
-// GFX12: v_cmpx_gt_f16_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0x7e,0x83,0x84,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x05,0x30]
+v_cmpx_gt_f16_e64_dpp -v1.h, |v2.l| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX12: v_cmpx_gt_f16_e64_dpp -v1.h, |v2.l| op_sel:[1,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7e,0x0a,0x84,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x09,0x13]
+
+v_cmpx_gt_f16_e64_dpp -|v255.l|, -|v255.h| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX12: v_cmpx_gt_f16_e64_dpp -|v255.l|, -|v255.h| op_sel:[0,1] clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0x7e,0x93,0x84,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x05,0x30]
 
 v_cmpx_gt_f32_e64_dpp v1, v2 quad_perm:[3,2,1,0]
 // GFX12: v_cmpx_gt_f32_e64_dpp v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x94,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
@@ -1025,53 +1052,62 @@ v_cmpx_gt_u32_e64_dpp v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1
 v_cmpx_gt_u32_e64_dpp v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
 // GFX12: v_cmpx_gt_u32_e64_dpp v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0x7e,0x00,0xcc,0xd4,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x05,0x30]
 
-v_cmpx_le_f16_e64_dpp v1, v2 quad_perm:[3,2,1,0]
-// GFX12: v_cmpx_le_f16_e64_dpp v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x83,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+v_cmpx_le_f16_e64_dpp v1.l, v2.l quad_perm:[3,2,1,0]
+// GFX12: v_cmpx_le_f16_e64_dpp v1.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x83,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+
+v_cmpx_le_f16_e64_dpp v1.l, s2 quad_perm:[3,2,1,0]
+// GFX12: v_cmpx_le_f16_e64_dpp v1.l, s2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x83,0xd4,0xfa,0x04,0x00,0x00,0x01,0x1b,0x00,0xff]
 
-v_cmpx_le_f16_e64_dpp v1, s2 quad_perm:[3,2,1,0]
-// GFX12: v_cmpx_le_f16_e64_dpp v1, s2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x83,0xd4,0xfa,0x04,0x00,0x00,0x01,0x1b,0x00,0xff]
+v_cmpx_le_f16_e64_dpp v1.l, 2.0 quad_perm:[3,2,1,0]
+// GFX12: v_cmpx_le_f16_e64_dpp v1.l, 2.0 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x83,0xd4,0xfa,0xe8,0x01,0x00,0x01,0x1b,0x00,0xff]
 
-v_cmpx_le_f16_e64_dpp v1, 2.0 quad_perm:[3,2,1,0]
-// GFX12: v_cmpx_le_f16_e64_dpp v1, 2.0 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x83,0xd4,0xfa,0xe8,0x01,0x00,0x01,0x1b,0x00,0xff]
+v_cmpx_le_f16_e64_dpp v1.l, v2.l quad_perm:[0,1,2,3]
+// GFX12: v_cmpx_le_f16_e64_dpp v1.l, v2.l quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x83,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
 
-v_cmpx_le_f16_e64_dpp v1, v2 quad_perm:[0,1,2,3]
-// GFX12: v_cmpx_le_f16_e64_dpp v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x83,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+v_cmpx_le_f16_e64_dpp v1.l, v2.l row_mirror
+// GFX12: v_cmpx_le_f16_e64_dpp v1.l, v2.l row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x83,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
 
-v_cmpx_le_f16_e64_dpp v1, v2 row_mirror
-// GFX12: v_cmpx_le_f16_e64_dpp v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x83,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+v_cmpx_le_f16_e64_dpp v1.l, v2.l row_half_mirror
+// GFX12: v_cmpx_le_f16_e64_dpp v1.l, v2.l row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x83,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
 
-v_cmpx_le_f16_e64_dpp v1, v2 row_half_mirror
-// GFX12: v_cmpx_le_f16_e64_dpp v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x83,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+v_cmpx_le_f16_e64_dpp v1.l, v2.l row_shl:1
+// GFX12: v_cmpx_le_f16_e64_dpp v1.l, v2.l row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x83,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
 
-v_cmpx_le_f16_e64_dpp v1, v2 row_shl:1
-// GFX12: v_cmpx_le_f16_e64_dpp v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x83,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+v_cmpx_le_f16_e64_dpp v1.l, v2.l row_shl:15
+// GFX12: v_cmpx_le_f16_e64_dpp v1.l, v2.l row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x83,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
 
-v_cmpx_le_f16_e64_dpp v1, v2 row_shl:15
-// GFX12: v_cmpx_le_f16_e64_dpp v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x83,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+v_cmpx_le_f16_e64_dpp v1.l, v2.l row_shr:1
+// GFX12: v_cmpx_le_f16_e64_dpp v1.l, v2.l row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x83,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
 
-v_cmpx_le_f16_e64_dpp v1, v2 row_shr:1
-// GFX12: v_cmpx_le_f16_e64_dpp v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x83,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+v_cmpx_le_f16_e64_dpp v1.l, v2.l row_shr:15
+// GFX12: v_cmpx_le_f16_e64_dpp v1.l, v2.l row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x83,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
 
-v_cmpx_le_f16_e64_dpp v1, v2 row_shr:15
-// GFX12: v_cmpx_le_f16_e64_dpp v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x83,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+v_cmpx_le_f16_e64_dpp v1.l, v2.l row_ror:1
+// GFX12: v_cmpx_le_f16_e64_dpp v1.l, v2.l row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x83,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
 
-v_cmpx_le_f16_e64_dpp v1, v2 row_ror:1
-// GFX12: v_cmpx_le_f16_e64_dpp v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x83,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+v_cmpx_le_f16_e64_dpp v1.l, v2.l row_ror:15
+// GFX12: v_cmpx_le_f16_e64_dpp v1.l, v2.l row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x83,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
 
-v_cmpx_le_f16_e64_dpp v1, v2 row_ror:15
-// GFX12: v_cmpx_le_f16_e64_dpp v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x83,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+v_cmpx_le_f16_e64_dpp v1.l, v2.l row_share:0 row_mask:0xf bank_mask:0xf
+// GFX12: v_cmpx_le_f16_e64_dpp v1.l, v2.l row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x83,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 
-v_cmpx_le_f16_e64_dpp v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
-// GFX12: v_cmpx_le_f16_e64_dpp v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x83,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+v_cmpx_le_f16_e64_dpp |v1.l|, -v2.l row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX12: v_cmpx_le_f16_e64_dpp |v1.l|, -v2.l row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x01,0x83,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
 
-v_cmpx_le_f16_e64_dpp |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1
-// GFX12: v_cmpx_le_f16_e64_dpp |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x01,0x83,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+v_cmpx_le_f16_e64_dpp -v1.l, |v2.l| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX12: v_cmpx_le_f16_e64_dpp -v1.l, |v2.l| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7e,0x02,0x83,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x09,0x13]
 
-v_cmpx_le_f16_e64_dpp -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
-// GFX12: v_cmpx_le_f16_e64_dpp -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7e,0x02,0x83,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x09,0x13]
+v_cmpx_le_f16_e64_dpp -|v255.l|, -|v255.l| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX12: v_cmpx_le_f16_e64_dpp -|v255.l|, -|v255.l| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0x7e,0x83,0x83,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x05,0x30]
 
-v_cmpx_le_f16_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
-// GFX12: v_cmpx_le_f16_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0x7e,0x83,0x83,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x05,0x30]
+v_cmpx_le_f16_e64_dpp |v1.h|, -v2.h row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX12: v_cmpx_le_f16_e64_dpp |v1.h|, -v2.h op_sel:[1,1] row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x19,0x83,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+
+v_cmpx_le_f16_e64_dpp -v1.h, |v2.l| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX12: v_cmpx_le_f16_e64_dpp -v1.h, |v2.l| op_sel:[1,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7e,0x0a,0x83,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x09,0x13]
+
+v_cmpx_le_f16_e64_dpp -|v255.l|, -|v255.h| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX12: v_cmpx_le_f16_e64_dpp -|v255.l|, -|v255.h| op_sel:[0,1] clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0x7e,0x93,0x83,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x05,0x30]
 
 v_cmpx_le_f32_e64_dpp v1, v2 quad_perm:[3,2,1,0]
 // GFX12: v_cmpx_le_f32_e64_dpp v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x93,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
@@ -1331,53 +1367,62 @@ v_cmpx_le_u32_e64_dpp v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1
 v_cmpx_le_u32_e64_dpp v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
 // GFX12: v_cmpx_le_u32_e64_dpp v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0x7e,0x00,0xcb,0xd4,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x05,0x30]
 
-v_cmpx_lg_f16_e64_dpp v1, v2 quad_perm:[3,2,1,0]
-// GFX12: v_cmpx_lg_f16_e64_dpp v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x85,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+v_cmpx_lg_f16_e64_dpp v1.l, v2.l quad_perm:[3,2,1,0]
+// GFX12: v_cmpx_lg_f16_e64_dpp v1.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x85,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+
+v_cmpx_lg_f16_e64_dpp v1.l, s2 quad_perm:[3,2,1,0]
+// GFX12: v_cmpx_lg_f16_e64_dpp v1.l, s2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x85,0xd4,0xfa,0x04,0x00,0x00,0x01,0x1b,0x00,0xff]
+
+v_cmpx_lg_f16_e64_dpp v1.l, 2.0 quad_perm:[3,2,1,0]
+// GFX12: v_cmpx_lg_f16_e64_dpp v1.l, 2.0 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x85,0xd4,0xfa,0xe8,0x01,0x00,0x01,0x1b,0x00,0xff]
+
+v_cmpx_lg_f16_e64_dpp v1.l, v2.l quad_perm:[0,1,2,3]
+// GFX12: v_cmpx_lg_f16_e64_dpp v1.l, v2.l quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x85,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
 
-v_cmpx_lg_f16_e64_dpp v1, s2 quad_perm:[3,2,1,0]
-// GFX12: v_cmpx_lg_f16_e64_dpp v1, s2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x85,0xd4,0xfa,0x04,0x00,0x00,0x01,0x1b,0x00,0xff]
+v_cmpx_lg_f16_e64_dpp v1.l, v2.l row_mirror
+// GFX12: v_cmpx_lg_f16_e64_dpp v1.l, v2.l row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x85,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
 
-v_cmpx_lg_f16_e64_dpp v1, 2.0 quad_perm:[3,2,1,0]
-// GFX12: v_cmpx_lg_f16_e64_dpp v1, 2.0 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x85,0xd4,0xfa,0xe8,0x01,0x00,0x01,0x1b,0x00,0xff]
+v_cmpx_lg_f16_e64_dpp v1.l, v2.l row_half_mirror
+// GFX12: v_cmpx_lg_f16_e64_dpp v1.l, v2.l row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x85,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
 
-v_cmpx_lg_f16_e64_dpp v1, v2 quad_perm:[0,1,2,3]
-// GFX12: v_cmpx_lg_f16_e64_dpp v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x85,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+v_cmpx_lg_f16_e64_dpp v1.l, v2.l row_shl:1
+// GFX12: v_cmpx_lg_f16_e64_dpp v1.l, v2.l row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x85,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
 
-v_cmpx_lg_f16_e64_dpp v1, v2 row_mirror
-// GFX12: v_cmpx_lg_f16_e64_dpp v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x85,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+v_cmpx_lg_f16_e64_dpp v1.l, v2.l row_shl:15
+// GFX12: v_cmpx_lg_f16_e64_dpp v1.l, v2.l row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x85,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
 
-v_cmpx_lg_f16_e64_dpp v1, v2 row_half_mirror
-// GFX12: v_cmpx_lg_f16_e64_dpp v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x85,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+v_cmpx_lg_f16_e64_dpp v1.l, v2.l row_shr:1
+// GFX12: v_cmpx_lg_f16_e64_dpp v1.l, v2.l row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x85,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
 
-v_cmpx_lg_f16_e64_dpp v1, v2 row_shl:1
-// GFX12: v_cmpx_lg_f16_e64_dpp v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x85,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+v_cmpx_lg_f16_e64_dpp v1.l, v2.l row_shr:15
+// GFX12: v_cmpx_lg_f16_e64_dpp v1.l, v2.l row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x85,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
 
-v_cmpx_lg_f16_e64_dpp v1, v2 row_shl:15
-// GFX12: v_cmpx_lg_f16_e64_dpp v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x85,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+v_cmpx_lg_f16_e64_dpp v1.l, v2.l row_ror:1
+// GFX12: v_cmpx_lg_f16_e64_dpp v1.l, v2.l row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x85,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
 
-v_cmpx_lg_f16_e64_dpp v1, v2 row_shr:1
-// GFX12: v_cmpx_lg_f16_e64_dpp v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x85,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+v_cmpx_lg_f16_e64_dpp v1.l, v2.l row_ror:15
+// GFX12: v_cmpx_lg_f16_e64_dpp v1.l, v2.l row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x85,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
 
-v_cmpx_lg_f16_e64_dpp v1, v2 row_shr:15
-// GFX12: v_cmpx_lg_f16_e64_dpp v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x85,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+v_cmpx_lg_f16_e64_dpp v1.l, v2.l row_share:0 row_mask:0xf bank_mask:0xf
+// GFX12: v_cmpx_lg_f16_e64_dpp v1.l, v2.l row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x85,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 
-v_cmpx_lg_f16_e64_dpp v1, v2 row_ror:1
-// GFX12: v_cmpx_lg_f16_e64_dpp v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x85,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+v_cmpx_lg_f16_e64_dpp |v1.l|, -v2.l row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX12: v_cmpx_lg_f16_e64_dpp |v1.l|, -v2.l row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x01,0x85,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
 
-v_cmpx_lg_f16_e64_dpp v1, v2 row_ror:15
-// GFX12: v_cmpx_lg_f16_e64_dpp v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x85,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+v_cmpx_lg_f16_e64_dpp -v1.l, |v2.l| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX12: v_cmpx_lg_f16_e64_dpp -v1.l, |v2.l| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7e,0x02,0x85,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x09,0x13]
 
-v_cmpx_lg_f16_e64_dpp v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
-// GFX12: v_cmpx_lg_f16_e64_dpp v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x85,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+v_cmpx_lg_f16_e64_dpp -|v255.l|, -|v255.l| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX12: v_cmpx_lg_f16_e64_dpp -|v255.l|, -|v255.l| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0x7e,0x83,0x85,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x05,0x30]
 
-v_cmpx_lg_f16_e64_dpp |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1
-// GFX12: v_cmpx_lg_f16_e64_dpp |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x01,0x85,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+v_cmpx_lg_f16_e64_dpp |v1.h|, -v2.h row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX12: v_cmpx_lg_f16_e64_dpp |v1.h|, -v2.h op_sel:[1,1] row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x19,0x85,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
 
-v_cmpx_lg_f16_e64_dpp -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
-// GFX12: v_cmpx_lg_f16_e64_dpp -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7e,0x02,0x85,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x09,0x13]
+v_cmpx_lg_f16_e64_dpp -v1.h, |v2.l| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX12: v_cmpx_lg_f16_e64_dpp -v1.h, |v2.l| op_sel:[1,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7e,0x0a,0x85,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x09,0x13]
 
-v_cmpx_lg_f16_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
-// GFX12: v_cmpx_lg_f16_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0x7e,0x83,0x85,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x05,0x30]
+v_cmpx_lg_f16_e64_dpp -|v255.l|, -|v255.h| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX12: v_cmpx_lg_f16_e64_dpp -|v255.l|, -|v255.h| op_sel:[0,1] clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0x7e,0x93,0x85,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x05,0x30]
 
 v_cmpx_lg_f32_e64_dpp v1, v2 quad_perm:[3,2,1,0]
 // GFX12: v_cmpx_lg_f32_e64_dpp v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x95,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
@@ -1952,53 +1997,62 @@ v_cmpx_ne_u32_e64_dpp v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1
 v_cmpx_ne_u32_e64_dpp v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
 // GFX12: v_cmpx_ne_u32_e64_dpp v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0x7e,0x00,0xcd,0xd4,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x05,0x30]
 
-v_cmpx_neq_f16_e64_dpp v1, v2 quad_perm:[3,2,1,0]
-// GFX12: v_cmpx_neq_f16_e64_dpp v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+v_cmpx_neq_f16_e64_dpp v1.l, v2.l quad_perm:[3,2,1,0]
+// GFX12: v_cmpx_neq_f16_e64_dpp v1.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
 
-v_cmpx_neq_f16_e64_dpp v1, s2 quad_perm:[3,2,1,0]
-// GFX12: v_cmpx_neq_f16_e64_dpp v1, s2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8d,0xd4,0xfa,0x04,0x00,0x00,0x01,0x1b,0x00,0xff]
+v_cmpx_neq_f16_e64_dpp v1.l, s2 quad_perm:[3,2,1,0]
+// GFX12: v_cmpx_neq_f16_e64_dpp v1.l, s2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8d,0xd4,0xfa,0x04,0x00,0x00,0x01,0x1b,0x00,0xff]
 
-v_cmpx_neq_f16_e64_dpp v1, 2.0 quad_perm:[3,2,1,0]
-// GFX12: v_cmpx_neq_f16_e64_dpp v1, 2.0 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8d,0xd4,0xfa,0xe8,0x01,0x00,0x01,0x1b,0x00,0xff]
+v_cmpx_neq_f16_e64_dpp v1.l, 2.0 quad_perm:[3,2,1,0]
+// GFX12: v_cmpx_neq_f16_e64_dpp v1.l, 2.0 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8d,0xd4,0xfa,0xe8,0x01,0x00,0x01,0x1b,0x00,0xff]
 
-v_cmpx_neq_f16_e64_dpp v1, v2 quad_perm:[0,1,2,3]
-// GFX12: v_cmpx_neq_f16_e64_dpp v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8d,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+v_cmpx_neq_f16_e64_dpp v1.l, v2.l quad_perm:[0,1,2,3]
+// GFX12: v_cmpx_neq_f16_e64_dpp v1.l, v2.l quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8d,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
 
-v_cmpx_neq_f16_e64_dpp v1, v2 row_mirror
-// GFX12: v_cmpx_neq_f16_e64_dpp v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+v_cmpx_neq_f16_e64_dpp v1.l, v2.l row_mirror
+// GFX12: v_cmpx_neq_f16_e64_dpp v1.l, v2.l row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
 
-v_cmpx_neq_f16_e64_dpp v1, v2 row_half_mirror
-// GFX12: v_cmpx_neq_f16_e64_dpp v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+v_cmpx_neq_f16_e64_dpp v1.l, v2.l row_half_mirror
+// GFX12: v_cmpx_neq_f16_e64_dpp v1.l, v2.l row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
 
-v_cmpx_neq_f16_e64_dpp v1, v2 row_shl:1
-// GFX12: v_cmpx_neq_f16_e64_dpp v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+v_cmpx_neq_f16_e64_dpp v1.l, v2.l row_shl:1
+// GFX12: v_cmpx_neq_f16_e64_dpp v1.l, v2.l row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
 
-v_cmpx_neq_f16_e64_dpp v1, v2 row_shl:15
-// GFX12: v_cmpx_neq_f16_e64_dpp v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+v_cmpx_neq_f16_e64_dpp v1.l, v2.l row_shl:15
+// GFX12: v_cmpx_neq_f16_e64_dpp v1.l, v2.l row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
 
-v_cmpx_neq_f16_e64_dpp v1, v2 row_shr:1
-// GFX12: v_cmpx_neq_f16_e64_dpp v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+v_cmpx_neq_f16_e64_dpp v1.l, v2.l row_shr:1
+// GFX12: v_cmpx_neq_f16_e64_dpp v1.l, v2.l row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
 
-v_cmpx_neq_f16_e64_dpp v1, v2 row_shr:15
-// GFX12: v_cmpx_neq_f16_e64_dpp v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+v_cmpx_neq_f16_e64_dpp v1.l, v2.l row_shr:15
+// GFX12: v_cmpx_neq_f16_e64_dpp v1.l, v2.l row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
 
-v_cmpx_neq_f16_e64_dpp v1, v2 row_ror:1
-// GFX12: v_cmpx_neq_f16_e64_dpp v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+v_cmpx_neq_f16_e64_dpp v1.l, v2.l row_ror:1
+// GFX12: v_cmpx_neq_f16_e64_dpp v1.l, v2.l row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
 
-v_cmpx_neq_f16_e64_dpp v1, v2 row_ror:15
-// GFX12: v_cmpx_neq_f16_e64_dpp v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+v_cmpx_neq_f16_e64_dpp v1.l, v2.l row_ror:15
+// GFX12: v_cmpx_neq_f16_e64_dpp v1.l, v2.l row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
 
-v_cmpx_neq_f16_e64_dpp v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
-// GFX12: v_cmpx_neq_f16_e64_dpp v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+v_cmpx_neq_f16_e64_dpp v1.l, v2.l row_share:0 row_mask:0xf bank_mask:0xf
+// GFX12: v_cmpx_neq_f16_e64_dpp v1.l, v2.l row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 
-v_cmpx_neq_f16_e64_dpp |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1
-// GFX12: v_cmpx_neq_f16_e64_dpp |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x01,0x8d,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+v_cmpx_neq_f16_e64_dpp |v1.l|, -v2.l row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX12: v_cmpx_neq_f16_e64_dpp |v1.l|, -v2.l row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x01,0x8d,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
 
-v_cmpx_neq_f16_e64_dpp -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
-// GFX12: v_cmpx_neq_f16_e64_dpp -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7e,0x02,0x8d,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x09,0x13]
+v_cmpx_neq_f16_e64_dpp -v1.l, |v2.l| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX12: v_cmpx_neq_f16_e64_dpp -v1.l, |v2.l| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7e,0x02,0x8d,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x09,0x13]
 
-v_cmpx_neq_f16_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
-// GFX12: v_cmpx_neq_f16_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0x7e,0x83,0x8d,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x05,0x30]
+v_cmpx_neq_f16_e64_dpp -|v255.l|, -|v255.l| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX12: v_cmpx_neq_f16_e64_dpp -|v255.l|, -|v255.l| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0x7e,0x83,0x8d,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x05,0x30]
+
+v_cmpx_neq_f16_e64_dpp |v1.h|, -v2.h row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX12: v_cmpx_neq_f16_e64_dpp |v1.h|, -v2.h op_sel:[1,1] row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x19,0x8d,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+
+v_cmpx_neq_f16_e64_dpp -v1.h, |v2.l| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX12: v_cmpx_neq_f16_e64_dpp -v1.h, |v2.l| op_sel:[1,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7e,0x0a,0x8d,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x09,0x13]
+
+v_cmpx_neq_f16_e64_dpp -|v255.l|, -|v255.h| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX12: v_cmpx_neq_f16_e64_dpp -|v255.l|, -|v255.h| op_sel:[0,1] clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0x7e,0x93,0x8d,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x05,0x30]
 
 v_cmpx_neq_f32_e64_dpp v1, v2 quad_perm:[3,2,1,0]
 // GFX12: v_cmpx_neq_f32_e64_dpp v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x9d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
@@ -2048,53 +2102,62 @@ v_cmpx_neq_f32_e64_dpp -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ct
 v_cmpx_neq_f32_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
 // GFX12: v_cmpx_neq_f32_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0x7e,0x83,0x9d,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x05,0x30]
 
-v_cmpx_nge_f16_e64_dpp v1, v2 quad_perm:[3,2,1,0]
-// GFX12: v_cmpx_nge_f16_e64_dpp v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x89,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+v_cmpx_nge_f16_e64_dpp v1.l, v2.l quad_perm:[3,2,1,0]
+// GFX12: v_cmpx_nge_f16_e64_dpp v1.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x89,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+
+v_cmpx_nge_f16_e64_dpp v1.l, s2 quad_perm:[3,2,1,0]
+// GFX12: v_cmpx_nge_f16_e64_dpp v1.l, s2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x89,0xd4,0xfa,0x04,0x00,0x00,0x01,0x1b,0x00,0xff]
+
+v_cmpx_nge_f16_e64_dpp v1.l, 2.0 quad_perm:[3,2,1,0]
+// GFX12: v_cmpx_nge_f16_e64_dpp v1.l, 2.0 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x89,0xd4,0xfa,0xe8,0x01,0x00,0x01,0x1b,0x00,0xff]
 
-v_cmpx_nge_f16_e64_dpp v1, s2 quad_perm:[3,2,1,0]
-// GFX12: v_cmpx_nge_f16_e64_dpp v1, s2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x89,0xd4,0xfa,0x04,0x00,0x00,0x01,0x1b,0x00,0xff]
+v_cmpx_nge_f16_e64_dpp v1.l, v2.l quad_perm:[0,1,2,3]
+// GFX12: v_cmpx_nge_f16_e64_dpp v1.l, v2.l quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x89,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
 
-v_cmpx_nge_f16_e64_dpp v1, 2.0 quad_perm:[3,2,1,0]
-// GFX12: v_cmpx_nge_f16_e64_dpp v1, 2.0 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x89,0xd4,0xfa,0xe8,0x01,0x00,0x01,0x1b,0x00,0xff]
+v_cmpx_nge_f16_e64_dpp v1.l, v2.l row_mirror
+// GFX12: v_cmpx_nge_f16_e64_dpp v1.l, v2.l row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x89,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
 
-v_cmpx_nge_f16_e64_dpp v1, v2 quad_perm:[0,1,2,3]
-// GFX12: v_cmpx_nge_f16_e64_dpp v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x89,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+v_cmpx_nge_f16_e64_dpp v1.l, v2.l row_half_mirror
+// GFX12: v_cmpx_nge_f16_e64_dpp v1.l, v2.l row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x89,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
 
-v_cmpx_nge_f16_e64_dpp v1, v2 row_mirror
-// GFX12: v_cmpx_nge_f16_e64_dpp v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x89,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+v_cmpx_nge_f16_e64_dpp v1.l, v2.l row_shl:1
+// GFX12: v_cmpx_nge_f16_e64_dpp v1.l, v2.l row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x89,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
 
-v_cmpx_nge_f16_e64_dpp v1, v2 row_half_mirror
-// GFX12: v_cmpx_nge_f16_e64_dpp v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x89,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+v_cmpx_nge_f16_e64_dpp v1.l, v2.l row_shl:15
+// GFX12: v_cmpx_nge_f16_e64_dpp v1.l, v2.l row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x89,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
 
-v_cmpx_nge_f16_e64_dpp v1, v2 row_shl:1
-// GFX12: v_cmpx_nge_f16_e64_dpp v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x89,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+v_cmpx_nge_f16_e64_dpp v1.l, v2.l row_shr:1
+// GFX12: v_cmpx_nge_f16_e64_dpp v1.l, v2.l row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x89,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
 
-v_cmpx_nge_f16_e64_dpp v1, v2 row_shl:15
-// GFX12: v_cmpx_nge_f16_e64_dpp v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x89,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+v_cmpx_nge_f16_e64_dpp v1.l, v2.l row_shr:15
+// GFX12: v_cmpx_nge_f16_e64_dpp v1.l, v2.l row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x89,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
 
-v_cmpx_nge_f16_e64_dpp v1, v2 row_shr:1
-// GFX12: v_cmpx_nge_f16_e64_dpp v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x89,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+v_cmpx_nge_f16_e64_dpp v1.l, v2.l row_ror:1
+// GFX12: v_cmpx_nge_f16_e64_dpp v1.l, v2.l row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x89,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
 
-v_cmpx_nge_f16_e64_dpp v1, v2 row_shr:15
-// GFX12: v_cmpx_nge_f16_e64_dpp v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x89,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+v_cmpx_nge_f16_e64_dpp v1.l, v2.l row_ror:15
+// GFX12: v_cmpx_nge_f16_e64_dpp v1.l, v2.l row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x89,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
 
-v_cmpx_nge_f16_e64_dpp v1, v2 row_ror:1
-// GFX12: v_cmpx_nge_f16_e64_dpp v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x89,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+v_cmpx_nge_f16_e64_dpp v1.l, v2.l row_share:0 row_mask:0xf bank_mask:0xf
+// GFX12: v_cmpx_nge_f16_e64_dpp v1.l, v2.l row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x89,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 
-v_cmpx_nge_f16_e64_dpp v1, v2 row_ror:15
-// GFX12: v_cmpx_nge_f16_e64_dpp v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x89,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+v_cmpx_nge_f16_e64_dpp |v1.l|, -v2.l row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX12: v_cmpx_nge_f16_e64_dpp |v1.l|, -v2.l row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x01,0x89,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
 
-v_cmpx_nge_f16_e64_dpp v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
-// GFX12: v_cmpx_nge_f16_e64_dpp v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x89,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+v_cmpx_nge_f16_e64_dpp -v1.l, |v2.l| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX12: v_cmpx_nge_f16_e64_dpp -v1.l, |v2.l| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7e,0x02,0x89,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x09,0x13]
 
-v_cmpx_nge_f16_e64_dpp |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1
-// GFX12: v_cmpx_nge_f16_e64_dpp |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x01,0x89,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+v_cmpx_nge_f16_e64_dpp -|v255.l|, -|v255.l| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX12: v_cmpx_nge_f16_e64_dpp -|v255.l|, -|v255.l| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0x7e,0x83,0x89,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x05,0x30]
 
-v_cmpx_nge_f16_e64_dpp -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
-// GFX12: v_cmpx_nge_f16_e64_dpp -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7e,0x02,0x89,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x09,0x13]
+v_cmpx_nge_f16_e64_dpp |v1.h|, -v2.h row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX12: v_cmpx_nge_f16_e64_dpp |v1.h|, -v2.h op_sel:[1,1] row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x19,0x89,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
 
-v_cmpx_nge_f16_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
-// GFX12: v_cmpx_nge_f16_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0x7e,0x83,0x89,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x05,0x30]
+v_cmpx_nge_f16_e64_dpp -v1.h, |v2.l| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX12: v_cmpx_nge_f16_e64_dpp -v1.h, |v2.l| op_sel:[1,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7e,0x0a,0x89,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x09,0x13]
+
+v_cmpx_nge_f16_e64_dpp -|v255.l|, -|v255.h| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX12: v_cmpx_nge_f16_e64_dpp -|v255.l|, -|v255.h| op_sel:[0,1] clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0x7e,0x93,0x89,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x05,0x30]
 
 v_cmpx_nge_f32_e64_dpp v1, v2 quad_perm:[3,2,1,0]
 // GFX12: v_cmpx_nge_f32_e64_dpp v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x99,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
@@ -2144,53 +2207,62 @@ v_cmpx_nge_f32_e64_dpp -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ct
 v_cmpx_nge_f32_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
 // GFX12: v_cmpx_nge_f32_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0x7e,0x83,0x99,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x05,0x30]
 
-v_cmpx_ngt_f16_e64_dpp v1, v2 quad_perm:[3,2,1,0]
-// GFX12: v_cmpx_ngt_f16_e64_dpp v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8b,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+v_cmpx_ngt_f16_e64_dpp v1.l, v2.l quad_perm:[3,2,1,0]
+// GFX12: v_cmpx_ngt_f16_e64_dpp v1.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8b,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+
+v_cmpx_ngt_f16_e64_dpp v1.l, s2 quad_perm:[3,2,1,0]
+// GFX12: v_cmpx_ngt_f16_e64_dpp v1.l, s2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8b,0xd4,0xfa,0x04,0x00,0x00,0x01,0x1b,0x00,0xff]
+
+v_cmpx_ngt_f16_e64_dpp v1.l, 2.0 quad_perm:[3,2,1,0]
+// GFX12: v_cmpx_ngt_f16_e64_dpp v1.l, 2.0 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8b,0xd4,0xfa,0xe8,0x01,0x00,0x01,0x1b,0x00,0xff]
 
-v_cmpx_ngt_f16_e64_dpp v1, s2 quad_perm:[3,2,1,0]
-// GFX12: v_cmpx_ngt_f16_e64_dpp v1, s2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8b,0xd4,0xfa,0x04,0x00,0x00,0x01,0x1b,0x00,0xff]
+v_cmpx_ngt_f16_e64_dpp v1.l, v2.l quad_perm:[0,1,2,3]
+// GFX12: v_cmpx_ngt_f16_e64_dpp v1.l, v2.l quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8b,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
 
-v_cmpx_ngt_f16_e64_dpp v1, 2.0 quad_perm:[3,2,1,0]
-// GFX12: v_cmpx_ngt_f16_e64_dpp v1, 2.0 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8b,0xd4,0xfa,0xe8,0x01,0x00,0x01,0x1b,0x00,0xff]
+v_cmpx_ngt_f16_e64_dpp v1.l, v2.l row_mirror
+// GFX12: v_cmpx_ngt_f16_e64_dpp v1.l, v2.l row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8b,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
 
-v_cmpx_ngt_f16_e64_dpp v1, v2 quad_perm:[0,1,2,3]
-// GFX12: v_cmpx_ngt_f16_e64_dpp v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8b,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+v_cmpx_ngt_f16_e64_dpp v1.l, v2.l row_half_mirror
+// GFX12: v_cmpx_ngt_f16_e64_dpp v1.l, v2.l row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8b,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
 
-v_cmpx_ngt_f16_e64_dpp v1, v2 row_mirror
-// GFX12: v_cmpx_ngt_f16_e64_dpp v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8b,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+v_cmpx_ngt_f16_e64_dpp v1.l, v2.l row_shl:1
+// GFX12: v_cmpx_ngt_f16_e64_dpp v1.l, v2.l row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8b,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
 
-v_cmpx_ngt_f16_e64_dpp v1, v2 row_half_mirror
-// GFX12: v_cmpx_ngt_f16_e64_dpp v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8b,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+v_cmpx_ngt_f16_e64_dpp v1.l, v2.l row_shl:15
+// GFX12: v_cmpx_ngt_f16_e64_dpp v1.l, v2.l row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8b,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
 
-v_cmpx_ngt_f16_e64_dpp v1, v2 row_shl:1
-// GFX12: v_cmpx_ngt_f16_e64_dpp v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8b,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+v_cmpx_ngt_f16_e64_dpp v1.l, v2.l row_shr:1
+// GFX12: v_cmpx_ngt_f16_e64_dpp v1.l, v2.l row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8b,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
 
-v_cmpx_ngt_f16_e64_dpp v1, v2 row_shl:15
-// GFX12: v_cmpx_ngt_f16_e64_dpp v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8b,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+v_cmpx_ngt_f16_e64_dpp v1.l, v2.l row_shr:15
+// GFX12: v_cmpx_ngt_f16_e64_dpp v1.l, v2.l row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8b,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
 
-v_cmpx_ngt_f16_e64_dpp v1, v2 row_shr:1
-// GFX12: v_cmpx_ngt_f16_e64_dpp v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8b,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+v_cmpx_ngt_f16_e64_dpp v1.l, v2.l row_ror:1
+// GFX12: v_cmpx_ngt_f16_e64_dpp v1.l, v2.l row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8b,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
 
-v_cmpx_ngt_f16_e64_dpp v1, v2 row_shr:15
-// GFX12: v_cmpx_ngt_f16_e64_dpp v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8b,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+v_cmpx_ngt_f16_e64_dpp v1.l, v2.l row_ror:15
+// GFX12: v_cmpx_ngt_f16_e64_dpp v1.l, v2.l row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8b,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
 
-v_cmpx_ngt_f16_e64_dpp v1, v2 row_ror:1
-// GFX12: v_cmpx_ngt_f16_e64_dpp v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8b,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+v_cmpx_ngt_f16_e64_dpp v1.l, v2.l row_share:0 row_mask:0xf bank_mask:0xf
+// GFX12: v_cmpx_ngt_f16_e64_dpp v1.l, v2.l row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8b,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 
-v_cmpx_ngt_f16_e64_dpp v1, v2 row_ror:15
-// GFX12: v_cmpx_ngt_f16_e64_dpp v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8b,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+v_cmpx_ngt_f16_e64_dpp |v1.l|, -v2.l row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX12: v_cmpx_ngt_f16_e64_dpp |v1.l|, -v2.l row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x01,0x8b,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
 
-v_cmpx_ngt_f16_e64_dpp v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
-// GFX12: v_cmpx_ngt_f16_e64_dpp v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8b,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+v_cmpx_ngt_f16_e64_dpp -v1.l, |v2.l| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX12: v_cmpx_ngt_f16_e64_dpp -v1.l, |v2.l| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7e,0x02,0x8b,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x09,0x13]
 
-v_cmpx_ngt_f16_e64_dpp |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1
-// GFX12: v_cmpx_ngt_f16_e64_dpp |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x01,0x8b,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+v_cmpx_ngt_f16_e64_dpp -|v255.l|, -|v255.l| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX12: v_cmpx_ngt_f16_e64_dpp -|v255.l|, -|v255.l| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0x7e,0x83,0x8b,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x05,0x30]
 
-v_cmpx_ngt_f16_e64_dpp -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
-// GFX12: v_cmpx_ngt_f16_e64_dpp -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7e,0x02,0x8b,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x09,0x13]
+v_cmpx_ngt_f16_e64_dpp |v1.h|, -v2.h row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX12: v_cmpx_ngt_f16_e64_dpp |v1.h|, -v2.h op_sel:[1,1] row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x19,0x8b,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
 
-v_cmpx_ngt_f16_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
-// GFX12: v_cmpx_ngt_f16_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0x7e,0x83,0x8b,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x05,0x30]
+v_cmpx_ngt_f16_e64_dpp -v1.h, |v2.l| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX12: v_cmpx_ngt_f16_e64_dpp -v1.h, |v2.l| op_sel:[1,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7e,0x0a,0x8b,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x09,0x13]
+
+v_cmpx_ngt_f16_e64_dpp -|v255.l|, -|v255.h| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX12: v_cmpx_ngt_f16_e64_dpp -|v255.l|, -|v255.h| op_sel:[0,1] clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0x7e,0x93,0x8b,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x05,0x30]
 
 v_cmpx_ngt_f32_e64_dpp v1, v2 quad_perm:[3,2,1,0]
 // GFX12: v_cmpx_ngt_f32_e64_dpp v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x9b,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
@@ -2240,53 +2312,62 @@ v_cmpx_ngt_f32_e64_dpp -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ct
 v_cmpx_ngt_f32_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
 // GFX12: v_cmpx_ngt_f32_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0x7e,0x83,0x9b,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x05,0x30]
 
-v_cmpx_nle_f16_e64_dpp v1, v2 quad_perm:[3,2,1,0]
-// GFX12: v_cmpx_nle_f16_e64_dpp v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8c,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+v_cmpx_nle_f16_e64_dpp v1.l, v2.l quad_perm:[3,2,1,0]
+// GFX12: v_cmpx_nle_f16_e64_dpp v1.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8c,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+
+v_cmpx_nle_f16_e64_dpp v1.l, s2 quad_perm:[3,2,1,0]
+// GFX12: v_cmpx_nle_f16_e64_dpp v1.l, s2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8c,0xd4,0xfa,0x04,0x00,0x00,0x01,0x1b,0x00,0xff]
 
-v_cmpx_nle_f16_e64_dpp v1, s2 quad_perm:[3,2,1,0]
-// GFX12: v_cmpx_nle_f16_e64_dpp v1, s2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8c,0xd4,0xfa,0x04,0x00,0x00,0x01,0x1b,0x00,0xff]
+v_cmpx_nle_f16_e64_dpp v1.l, 2.0 quad_perm:[3,2,1,0]
+// GFX12: v_cmpx_nle_f16_e64_dpp v1.l, 2.0 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8c,0xd4,0xfa,0xe8,0x01,0x00,0x01,0x1b,0x00,0xff]
 
-v_cmpx_nle_f16_e64_dpp v1, 2.0 quad_perm:[3,2,1,0]
-// GFX12: v_cmpx_nle_f16_e64_dpp v1, 2.0 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8c,0xd4,0xfa,0xe8,0x01,0x00,0x01,0x1b,0x00,0xff]
+v_cmpx_nle_f16_e64_dpp v1.l, v2.l quad_perm:[0,1,2,3]
+// GFX12: v_cmpx_nle_f16_e64_dpp v1.l, v2.l quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8c,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
 
-v_cmpx_nle_f16_e64_dpp v1, v2 quad_perm:[0,1,2,3]
-// GFX12: v_cmpx_nle_f16_e64_dpp v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8c,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+v_cmpx_nle_f16_e64_dpp v1.l, v2.l row_mirror
+// GFX12: v_cmpx_nle_f16_e64_dpp v1.l, v2.l row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8c,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
 
-v_cmpx_nle_f16_e64_dpp v1, v2 row_mirror
-// GFX12: v_cmpx_nle_f16_e64_dpp v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8c,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+v_cmpx_nle_f16_e64_dpp v1.l, v2.l row_half_mirror
+// GFX12: v_cmpx_nle_f16_e64_dpp v1.l, v2.l row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8c,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
 
-v_cmpx_nle_f16_e64_dpp v1, v2 row_half_mirror
-// GFX12: v_cmpx_nle_f16_e64_dpp v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8c,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+v_cmpx_nle_f16_e64_dpp v1.l, v2.l row_shl:1
+// GFX12: v_cmpx_nle_f16_e64_dpp v1.l, v2.l row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8c,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
 
-v_cmpx_nle_f16_e64_dpp v1, v2 row_shl:1
-// GFX12: v_cmpx_nle_f16_e64_dpp v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8c,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+v_cmpx_nle_f16_e64_dpp v1.l, v2.l row_shl:15
+// GFX12: v_cmpx_nle_f16_e64_dpp v1.l, v2.l row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8c,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
 
-v_cmpx_nle_f16_e64_dpp v1, v2 row_shl:15
-// GFX12: v_cmpx_nle_f16_e64_dpp v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8c,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+v_cmpx_nle_f16_e64_dpp v1.l, v2.l row_shr:1
+// GFX12: v_cmpx_nle_f16_e64_dpp v1.l, v2.l row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8c,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
 
-v_cmpx_nle_f16_e64_dpp v1, v2 row_shr:1
-// GFX12: v_cmpx_nle_f16_e64_dpp v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8c,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+v_cmpx_nle_f16_e64_dpp v1.l, v2.l row_shr:15
+// GFX12: v_cmpx_nle_f16_e64_dpp v1.l, v2.l row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8c,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
 
-v_cmpx_nle_f16_e64_dpp v1, v2 row_shr:15
-// GFX12: v_cmpx_nle_f16_e64_dpp v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8c,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+v_cmpx_nle_f16_e64_dpp v1.l, v2.l row_ror:1
+// GFX12: v_cmpx_nle_f16_e64_dpp v1.l, v2.l row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8c,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
 
-v_cmpx_nle_f16_e64_dpp v1, v2 row_ror:1
-// GFX12: v_cmpx_nle_f16_e64_dpp v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8c,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+v_cmpx_nle_f16_e64_dpp v1.l, v2.l row_ror:15
+// GFX12: v_cmpx_nle_f16_e64_dpp v1.l, v2.l row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8c,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
 
-v_cmpx_nle_f16_e64_dpp v1, v2 row_ror:15
-// GFX12: v_cmpx_nle_f16_e64_dpp v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8c,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+v_cmpx_nle_f16_e64_dpp v1.l, v2.l row_share:0 row_mask:0xf bank_mask:0xf
+// GFX12: v_cmpx_nle_f16_e64_dpp v1.l, v2.l row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8c,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 
-v_cmpx_nle_f16_e64_dpp v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
-// GFX12: v_cmpx_nle_f16_e64_dpp v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8c,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+v_cmpx_nle_f16_e64_dpp |v1.l|, -v2.l row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX12: v_cmpx_nle_f16_e64_dpp |v1.l|, -v2.l row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x01,0x8c,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
 
-v_cmpx_nle_f16_e64_dpp |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1
-// GFX12: v_cmpx_nle_f16_e64_dpp |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x01,0x8c,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+v_cmpx_nle_f16_e64_dpp -v1.l, |v2.l| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX12: v_cmpx_nle_f16_e64_dpp -v1.l, |v2.l| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7e,0x02,0x8c,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x09,0x13]
 
-v_cmpx_nle_f16_e64_dpp -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
-// GFX12: v_cmpx_nle_f16_e64_dpp -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7e,0x02,0x8c,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x09,0x13]
+v_cmpx_nle_f16_e64_dpp -|v255.l|, -|v255.l| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX12: v_cmpx_nle_f16_e64_dpp -|v255.l|, -|v255.l| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0x7e,0x83,0x8c,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x05,0x30]
 
-v_cmpx_nle_f16_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
-// GFX12: v_cmpx_nle_f16_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0x7e,0x83,0x8c,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x05,0x30]
+v_cmpx_nle_f16_e64_dpp |v1.h|, -v2.h row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX12: v_cmpx_nle_f16_e64_dpp |v1.h|, -v2.h op_sel:[1,1] row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x19,0x8c,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+
+v_cmpx_nle_f16_e64_dpp -v1.h, |v2.l| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX12: v_cmpx_nle_f16_e64_dpp -v1.h, |v2.l| op_sel:[1,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7e,0x0a,0x8c,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x09,0x13]
+
+v_cmpx_nle_f16_e64_dpp -|v255.l|, -|v255.h| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX12: v_cmpx_nle_f16_e64_dpp -|v255.l|, -|v255.h| op_sel:[0,1] clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0x7e,0x93,0x8c,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x05,0x30]
 
 v_cmpx_nle_f32_e64_dpp v1, v2 quad_perm:[3,2,1,0]
 // GFX12: v_cmpx_nle_f32_e64_dpp v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x9c,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
@@ -2336,53 +2417,62 @@ v_cmpx_nle_f32_e64_dpp -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ct
 v_cmpx_nle_f32_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
 // GFX12: v_cmpx_nle_f32_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0x7e,0x83,0x9c,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x05,0x30]
 
-v_cmpx_nlg_f16_e64_dpp v1, v2 quad_perm:[3,2,1,0]
-// GFX12: v_cmpx_nlg_f16_e64_dpp v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8a,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+v_cmpx_nlg_f16_e64_dpp v1.l, v2.l quad_perm:[3,2,1,0]
+// GFX12: v_cmpx_nlg_f16_e64_dpp v1.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8a,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+
+v_cmpx_nlg_f16_e64_dpp v1.l, s2 quad_perm:[3,2,1,0]
+// GFX12: v_cmpx_nlg_f16_e64_dpp v1.l, s2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8a,0xd4,0xfa,0x04,0x00,0x00,0x01,0x1b,0x00,0xff]
+
+v_cmpx_nlg_f16_e64_dpp v1.l, 2.0 quad_perm:[3,2,1,0]
+// GFX12: v_cmpx_nlg_f16_e64_dpp v1.l, 2.0 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8a,0xd4,0xfa,0xe8,0x01,0x00,0x01,0x1b,0x00,0xff]
+
+v_cmpx_nlg_f16_e64_dpp v1.l, v2.l quad_perm:[0,1,2,3]
+// GFX12: v_cmpx_nlg_f16_e64_dpp v1.l, v2.l quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8a,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
 
-v_cmpx_nlg_f16_e64_dpp v1, s2 quad_perm:[3,2,1,0]
-// GFX12: v_cmpx_nlg_f16_e64_dpp v1, s2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8a,0xd4,0xfa,0x04,0x00,0x00,0x01,0x1b,0x00,0xff]
+v_cmpx_nlg_f16_e64_dpp v1.l, v2.l row_mirror
+// GFX12: v_cmpx_nlg_f16_e64_dpp v1.l, v2.l row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8a,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
 
-v_cmpx_nlg_f16_e64_dpp v1, 2.0 quad_perm:[3,2,1,0]
-// GFX12: v_cmpx_nlg_f16_e64_dpp v1, 2.0 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8a,0xd4,0xfa,0xe8,0x01,0x00,0x01,0x1b,0x00,0xff]
+v_cmpx_nlg_f16_e64_dpp v1.l, v2.l row_half_mirror
+// GFX12: v_cmpx_nlg_f16_e64_dpp v1.l, v2.l row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8a,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
 
-v_cmpx_nlg_f16_e64_dpp v1, v2 quad_perm:[0,1,2,3]
-// GFX12: v_cmpx_nlg_f16_e64_dpp v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8a,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+v_cmpx_nlg_f16_e64_dpp v1.l, v2.l row_shl:1
+// GFX12: v_cmpx_nlg_f16_e64_dpp v1.l, v2.l row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8a,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
 
-v_cmpx_nlg_f16_e64_dpp v1, v2 row_mirror
-// GFX12: v_cmpx_nlg_f16_e64_dpp v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8a,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+v_cmpx_nlg_f16_e64_dpp v1.l, v2.l row_shl:15
+// GFX12: v_cmpx_nlg_f16_e64_dpp v1.l, v2.l row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8a,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
 
-v_cmpx_nlg_f16_e64_dpp v1, v2 row_half_mirror
-// GFX12: v_cmpx_nlg_f16_e64_dpp v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8a,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+v_cmpx_nlg_f16_e64_dpp v1.l, v2.l row_shr:1
+// GFX12: v_cmpx_nlg_f16_e64_dpp v1.l, v2.l row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8a,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
 
-v_cmpx_nlg_f16_e64_dpp v1, v2 row_shl:1
-// GFX12: v_cmpx_nlg_f16_e64_dpp v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8a,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+v_cmpx_nlg_f16_e64_dpp v1.l, v2.l row_shr:15
+// GFX12: v_cmpx_nlg_f16_e64_dpp v1.l, v2.l row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8a,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
 
-v_cmpx_nlg_f16_e64_dpp v1, v2 row_shl:15
-// GFX12: v_cmpx_nlg_f16_e64_dpp v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8a,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+v_cmpx_nlg_f16_e64_dpp v1.l, v2.l row_ror:1
+// GFX12: v_cmpx_nlg_f16_e64_dpp v1.l, v2.l row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8a,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
 
-v_cmpx_nlg_f16_e64_dpp v1, v2 row_shr:1
-// GFX12: v_cmpx_nlg_f16_e64_dpp v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8a,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+v_cmpx_nlg_f16_e64_dpp v1.l, v2.l row_ror:15
+// GFX12: v_cmpx_nlg_f16_e64_dpp v1.l, v2.l row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8a,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
 
-v_cmpx_nlg_f16_e64_dpp v1, v2 row_shr:15
-// GFX12: v_cmpx_nlg_f16_e64_dpp v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8a,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+v_cmpx_nlg_f16_e64_dpp v1.l, v2.l row_share:0 row_mask:0xf bank_mask:0xf
+// GFX12: v_cmpx_nlg_f16_e64_dpp v1.l, v2.l row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8a,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 
-v_cmpx_nlg_f16_e64_dpp v1, v2 row_ror:1
-// GFX12: v_cmpx_nlg_f16_e64_dpp v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8a,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+v_cmpx_nlg_f16_e64_dpp |v1.l|, -v2.l row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX12: v_cmpx_nlg_f16_e64_dpp |v1.l|, -v2.l row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x01,0x8a,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
 
-v_cmpx_nlg_f16_e64_dpp v1, v2 row_ror:15
-// GFX12: v_cmpx_nlg_f16_e64_dpp v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8a,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+v_cmpx_nlg_f16_e64_dpp -v1.l, |v2.l| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX12: v_cmpx_nlg_f16_e64_dpp -v1.l, |v2.l| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7e,0x02,0x8a,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x09,0x13]
 
-v_cmpx_nlg_f16_e64_dpp v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
-// GFX12: v_cmpx_nlg_f16_e64_dpp v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8a,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+v_cmpx_nlg_f16_e64_dpp -|v255.l|, -|v255.l| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX12: v_cmpx_nlg_f16_e64_dpp -|v255.l|, -|v255.l| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0x7e,0x83,0x8a,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x05,0x30]
 
-v_cmpx_nlg_f16_e64_dpp |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1
-// GFX12: v_cmpx_nlg_f16_e64_dpp |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x01,0x8a,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+v_cmpx_nlg_f16_e64_dpp |v1.h|, -v2.h row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX12: v_cmpx_nlg_f16_e64_dpp |v1.h|, -v2.h op_sel:[1,1] row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x19,0x8a,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
 
-v_cmpx_nlg_f16_e64_dpp -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
-// GFX12: v_cmpx_nlg_f16_e64_dpp -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7e,0x02,0x8a,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x09,0x13]
+v_cmpx_nlg_f16_e64_dpp -v1.h, |v2.l| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX12: v_cmpx_nlg_f16_e64_dpp -v1.h, |v2.l| op_sel:[1,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7e,0x0a,0x8a,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x09,0x13]
 
-v_cmpx_nlg_f16_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
-// GFX12: v_cmpx_nlg_f16_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0x7e,0x83,0x8a,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x05,0x30]
+v_cmpx_nlg_f16_e64_dpp -|v255.l|, -|v255.h| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX12: v_cmpx_nlg_f16_e64_dpp -|v255.l|, -|v255.h| op_sel:[0,1] clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0x7e,0x93,0x8a,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x05,0x30]
 
 v_cmpx_nlg_f32_e64_dpp v1, v2 quad_perm:[3,2,1,0]
 // GFX12: v_cmpx_nlg_f32_e64_dpp v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x9a,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
@@ -2432,53 +2522,62 @@ v_cmpx_nlg_f32_e64_dpp -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ct
 v_cmpx_nlg_f32_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
 // GFX12: v_cmpx_nlg_f32_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0x7e,0x83,0x9a,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x05,0x30]
 
-v_cmpx_nlt_f16_e64_dpp v1, v2 quad_perm:[3,2,1,0]
-// GFX12: v_cmpx_nlt_f16_e64_dpp v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8e,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+v_cmpx_nlt_f16_e64_dpp v1.l, v2.l quad_perm:[3,2,1,0]
+// GFX12: v_cmpx_nlt_f16_e64_dpp v1.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8e,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
 
-v_cmpx_nlt_f16_e64_dpp v1, s2 quad_perm:[3,2,1,0]
-// GFX12: v_cmpx_nlt_f16_e64_dpp v1, s2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8e,0xd4,0xfa,0x04,0x00,0x00,0x01,0x1b,0x00,0xff]
+v_cmpx_nlt_f16_e64_dpp v1.l, s2 quad_perm:[3,2,1,0]
+// GFX12: v_cmpx_nlt_f16_e64_dpp v1.l, s2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8e,0xd4,0xfa,0x04,0x00,0x00,0x01,0x1b,0x00,0xff]
 
-v_cmpx_nlt_f16_e64_dpp v1, 2.0 quad_perm:[3,2,1,0]
-// GFX12: v_cmpx_nlt_f16_e64_dpp v1, 2.0 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8e,0xd4,0xfa,0xe8,0x01,0x00,0x01,0x1b,0x00,0xff]
+v_cmpx_nlt_f16_e64_dpp v1.l, 2.0 quad_perm:[3,2,1,0]
+// GFX12: v_cmpx_nlt_f16_e64_dpp v1.l, 2.0 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8e,0xd4,0xfa,0xe8,0x01,0x00,0x01,0x1b,0x00,0xff]
 
-v_cmpx_nlt_f16_e64_dpp v1, v2 quad_perm:[0,1,2,3]
-// GFX12: v_cmpx_nlt_f16_e64_dpp v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8e,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+v_cmpx_nlt_f16_e64_dpp v1.l, v2.l quad_perm:[0,1,2,3]
+// GFX12: v_cmpx_nlt_f16_e64_dpp v1.l, v2.l quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8e,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
 
-v_cmpx_nlt_f16_e64_dpp v1, v2 row_mirror
-// GFX12: v_cmpx_nlt_f16_e64_dpp v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8e,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+v_cmpx_nlt_f16_e64_dpp v1.l, v2.l row_mirror
+// GFX12: v_cmpx_nlt_f16_e64_dpp v1.l, v2.l row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8e,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
 
-v_cmpx_nlt_f16_e64_dpp v1, v2 row_half_mirror
-// GFX12: v_cmpx_nlt_f16_e64_dpp v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8e,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+v_cmpx_nlt_f16_e64_dpp v1.l, v2.l row_half_mirror
+// GFX12: v_cmpx_nlt_f16_e64_dpp v1.l, v2.l row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8e,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
 
-v_cmpx_nlt_f16_e64_dpp v1, v2 row_shl:1
-// GFX12: v_cmpx_nlt_f16_e64_dpp v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8e,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+v_cmpx_nlt_f16_e64_dpp v1.l, v2.l row_shl:1
+// GFX12: v_cmpx_nlt_f16_e64_dpp v1.l, v2.l row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8e,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
 
-v_cmpx_nlt_f16_e64_dpp v1, v2 row_shl:15
-// GFX12: v_cmpx_nlt_f16_e64_dpp v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8e,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+v_cmpx_nlt_f16_e64_dpp v1.l, v2.l row_shl:15
+// GFX12: v_cmpx_nlt_f16_e64_dpp v1.l, v2.l row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8e,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
 
-v_cmpx_nlt_f16_e64_dpp v1, v2 row_shr:1
-// GFX12: v_cmpx_nlt_f16_e64_dpp v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8e,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+v_cmpx_nlt_f16_e64_dpp v1.l, v2.l row_shr:1
+// GFX12: v_cmpx_nlt_f16_e64_dpp v1.l, v2.l row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8e,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
 
-v_cmpx_nlt_f16_e64_dpp v1, v2 row_shr:15
-// GFX12: v_cmpx_nlt_f16_e64_dpp v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8e,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+v_cmpx_nlt_f16_e64_dpp v1.l, v2.l row_shr:15
+// GFX12: v_cmpx_nlt_f16_e64_dpp v1.l, v2.l row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8e,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
 
-v_cmpx_nlt_f16_e64_dpp v1, v2 row_ror:1
-// GFX12: v_cmpx_nlt_f16_e64_dpp v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8e,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+v_cmpx_nlt_f16_e64_dpp v1.l, v2.l row_ror:1
+// GFX12: v_cmpx_nlt_f16_e64_dpp v1.l, v2.l row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8e,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
 
-v_cmpx_nlt_f16_e64_dpp v1, v2 row_ror:15
-// GFX12: v_cmpx_nlt_f16_e64_dpp v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8e,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+v_cmpx_nlt_f16_e64_dpp v1.l, v2.l row_ror:15
+// GFX12: v_cmpx_nlt_f16_e64_dpp v1.l, v2.l row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8e,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
 
-v_cmpx_nlt_f16_e64_dpp v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
-// GFX12: v_cmpx_nlt_f16_e64_dpp v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8e,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+v_cmpx_nlt_f16_e64_dpp v1.l, v2.l row_share:0 row_mask:0xf bank_mask:0xf
+// GFX12: v_cmpx_nlt_f16_e64_dpp v1.l, v2.l row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8e,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 
-v_cmpx_nlt_f16_e64_dpp |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1
-// GFX12: v_cmpx_nlt_f16_e64_dpp |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x01,0x8e,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+v_cmpx_nlt_f16_e64_dpp |v1.l|, -v2.l row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX12: v_cmpx_nlt_f16_e64_dpp |v1.l|, -v2.l row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x01,0x8e,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
 
-v_cmpx_nlt_f16_e64_dpp -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
-// GFX12: v_cmpx_nlt_f16_e64_dpp -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7e,0x02,0x8e,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x09,0x13]
+v_cmpx_nlt_f16_e64_dpp -v1.l, |v2.l| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX12: v_cmpx_nlt_f16_e64_dpp -v1.l, |v2.l| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7e,0x02,0x8e,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x09,0x13]
 
-v_cmpx_nlt_f16_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
-// GFX12: v_cmpx_nlt_f16_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0x7e,0x83,0x8e,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x05,0x30]
+v_cmpx_nlt_f16_e64_dpp -|v255.l|, -|v255.l| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX12: v_cmpx_nlt_f16_e64_dpp -|v255.l|, -|v255.l| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0x7e,0x83,0x8e,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x05,0x30]
+
+v_cmpx_nlt_f16_e64_dpp |v1.h|, -v2.h row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX12: v_cmpx_nlt_f16_e64_dpp |v1.h|, -v2.h op_sel:[1,1] row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x19,0x8e,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+
+v_cmpx_nlt_f16_e64_dpp -v1.h, |v2.l| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX12: v_cmpx_nlt_f16_e64_dpp -v1.h, |v2.l| op_sel:[1,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7e,0x0a,0x8e,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x09,0x13]
+
+v_cmpx_nlt_f16_e64_dpp -|v255.l|, -|v255.h| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX12: v_cmpx_nlt_f16_e64_dpp -|v255.l|, -|v255.h| op_sel:[0,1] clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0x7e,0x93,0x8e,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x05,0x30]
 
 v_cmpx_nlt_f32_e64_dpp v1, v2 quad_perm:[3,2,1,0]
 // GFX12: v_cmpx_nlt_f32_e64_dpp v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x9e,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
@@ -2528,53 +2627,62 @@ v_cmpx_nlt_f32_e64_dpp -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ct
 v_cmpx_nlt_f32_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
 // GFX12: v_cmpx_nlt_f32_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0x7e,0x83,0x9e,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x05,0x30]
 
-v_cmpx_o_f16_e64_dpp v1, v2 quad_perm:[3,2,1,0]
-// GFX12: v_cmpx_o_f16_e64_dpp v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x87,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+v_cmpx_o_f16_e64_dpp v1.l, v2.l quad_perm:[3,2,1,0]
+// GFX12: v_cmpx_o_f16_e64_dpp v1.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x87,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+
+v_cmpx_o_f16_e64_dpp v1.l, s2 quad_perm:[3,2,1,0]
+// GFX12: v_cmpx_o_f16_e64_dpp v1.l, s2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x87,0xd4,0xfa,0x04,0x00,0x00,0x01,0x1b,0x00,0xff]
+
+v_cmpx_o_f16_e64_dpp v1.l, 2.0 quad_perm:[3,2,1,0]
+// GFX12: v_cmpx_o_f16_e64_dpp v1.l, 2.0 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x87,0xd4,0xfa,0xe8,0x01,0x00,0x01,0x1b,0x00,0xff]
 
-v_cmpx_o_f16_e64_dpp v1, s2 quad_perm:[3,2,1,0]
-// GFX12: v_cmpx_o_f16_e64_dpp v1, s2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x87,0xd4,0xfa,0x04,0x00,0x00,0x01,0x1b,0x00,0xff]
+v_cmpx_o_f16_e64_dpp v1.l, v2.l quad_perm:[0,1,2,3]
+// GFX12: v_cmpx_o_f16_e64_dpp v1.l, v2.l quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x87,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
 
-v_cmpx_o_f16_e64_dpp v1, 2.0 quad_perm:[3,2,1,0]
-// GFX12: v_cmpx_o_f16_e64_dpp v1, 2.0 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x87,0xd4,0xfa,0xe8,0x01,0x00,0x01,0x1b,0x00,0xff]
+v_cmpx_o_f16_e64_dpp v1.l, v2.l row_mirror
+// GFX12: v_cmpx_o_f16_e64_dpp v1.l, v2.l row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x87,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
 
-v_cmpx_o_f16_e64_dpp v1, v2 quad_perm:[0,1,2,3]
-// GFX12: v_cmpx_o_f16_e64_dpp v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x87,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+v_cmpx_o_f16_e64_dpp v1.l, v2.l row_half_mirror
+// GFX12: v_cmpx_o_f16_e64_dpp v1.l, v2.l row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x87,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
 
-v_cmpx_o_f16_e64_dpp v1, v2 row_mirror
-// GFX12: v_cmpx_o_f16_e64_dpp v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x87,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+v_cmpx_o_f16_e64_dpp v1.l, v2.l row_shl:1
+// GFX12: v_cmpx_o_f16_e64_dpp v1.l, v2.l row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x87,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
 
-v_cmpx_o_f16_e64_dpp v1, v2 row_half_mirror
-// GFX12: v_cmpx_o_f16_e64_dpp v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x87,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+v_cmpx_o_f16_e64_dpp v1.l, v2.l row_shl:15
+// GFX12: v_cmpx_o_f16_e64_dpp v1.l, v2.l row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x87,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
 
-v_cmpx_o_f16_e64_dpp v1, v2 row_shl:1
-// GFX12: v_cmpx_o_f16_e64_dpp v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x87,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+v_cmpx_o_f16_e64_dpp v1.l, v2.l row_shr:1
+// GFX12: v_cmpx_o_f16_e64_dpp v1.l, v2.l row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x87,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
 
-v_cmpx_o_f16_e64_dpp v1, v2 row_shl:15
-// GFX12: v_cmpx_o_f16_e64_dpp v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x87,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+v_cmpx_o_f16_e64_dpp v1.l, v2.l row_shr:15
+// GFX12: v_cmpx_o_f16_e64_dpp v1.l, v2.l row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x87,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
 
-v_cmpx_o_f16_e64_dpp v1, v2 row_shr:1
-// GFX12: v_cmpx_o_f16_e64_dpp v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x87,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+v_cmpx_o_f16_e64_dpp v1.l, v2.l row_ror:1
+// GFX12: v_cmpx_o_f16_e64_dpp v1.l, v2.l row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x87,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
 
-v_cmpx_o_f16_e64_dpp v1, v2 row_shr:15
-// GFX12: v_cmpx_o_f16_e64_dpp v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x87,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+v_cmpx_o_f16_e64_dpp v1.l, v2.l row_ror:15
+// GFX12: v_cmpx_o_f16_e64_dpp v1.l, v2.l row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x87,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
 
-v_cmpx_o_f16_e64_dpp v1, v2 row_ror:1
-// GFX12: v_cmpx_o_f16_e64_dpp v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x87,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+v_cmpx_o_f16_e64_dpp v1.l, v2.l row_share:0 row_mask:0xf bank_mask:0xf
+// GFX12: v_cmpx_o_f16_e64_dpp v1.l, v2.l row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x87,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 
-v_cmpx_o_f16_e64_dpp v1, v2 row_ror:15
-// GFX12: v_cmpx_o_f16_e64_dpp v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x87,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+v_cmpx_o_f16_e64_dpp |v1.l|, -v2.l row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX12: v_cmpx_o_f16_e64_dpp |v1.l|, -v2.l row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x01,0x87,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
 
-v_cmpx_o_f16_e64_dpp v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
-// GFX12: v_cmpx_o_f16_e64_dpp v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x87,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+v_cmpx_o_f16_e64_dpp -v1.l, |v2.l| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX12: v_cmpx_o_f16_e64_dpp -v1.l, |v2.l| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7e,0x02,0x87,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x09,0x13]
 
-v_cmpx_o_f16_e64_dpp |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1
-// GFX12: v_cmpx_o_f16_e64_dpp |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x01,0x87,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+v_cmpx_o_f16_e64_dpp -|v255.l|, -|v255.l| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX12: v_cmpx_o_f16_e64_dpp -|v255.l|, -|v255.l| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0x7e,0x83,0x87,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x05,0x30]
 
-v_cmpx_o_f16_e64_dpp -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
-// GFX12: v_cmpx_o_f16_e64_dpp -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7e,0x02,0x87,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x09,0x13]
+v_cmpx_o_f16_e64_dpp |v1.h|, -v2.h row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX12: v_cmpx_o_f16_e64_dpp |v1.h|, -v2.h op_sel:[1,1] row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x19,0x87,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
 
-v_cmpx_o_f16_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
-// GFX12: v_cmpx_o_f16_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0x7e,0x83,0x87,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x05,0x30]
+v_cmpx_o_f16_e64_dpp -v1.h, |v2.l| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX12: v_cmpx_o_f16_e64_dpp -v1.h, |v2.l| op_sel:[1,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7e,0x0a,0x87,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x09,0x13]
+
+v_cmpx_o_f16_e64_dpp -|v255.l|, -|v255.h| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX12: v_cmpx_o_f16_e64_dpp -|v255.l|, -|v255.h| op_sel:[0,1] clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0x7e,0x93,0x87,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x05,0x30]
 
 v_cmpx_o_f32_e64_dpp v1, v2 quad_perm:[3,2,1,0]
 // GFX12: v_cmpx_o_f32_e64_dpp v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x97,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
@@ -2624,53 +2732,62 @@ v_cmpx_o_f32_e64_dpp -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl
 v_cmpx_o_f32_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
 // GFX12: v_cmpx_o_f32_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0x7e,0x83,0x97,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x05,0x30]
 
-v_cmpx_u_f16_e64_dpp v1, v2 quad_perm:[3,2,1,0]
-// GFX12: v_cmpx_u_f16_e64_dpp v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x88,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+v_cmpx_u_f16_e64_dpp v1.l, v2.l quad_perm:[3,2,1,0]
+// GFX12: v_cmpx_u_f16_e64_dpp v1.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x88,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+
+v_cmpx_u_f16_e64_dpp v1.l, s2 quad_perm:[3,2,1,0]
+// GFX12: v_cmpx_u_f16_e64_dpp v1.l, s2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x88,0xd4,0xfa,0x04,0x00,0x00,0x01,0x1b,0x00,0xff]
+
+v_cmpx_u_f16_e64_dpp v1.l, 2.0 quad_perm:[3,2,1,0]
+// GFX12: v_cmpx_u_f16_e64_dpp v1.l, 2.0 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x88,0xd4,0xfa,0xe8,0x01,0x00,0x01,0x1b,0x00,0xff]
+
+v_cmpx_u_f16_e64_dpp v1.l, v2.l quad_perm:[0,1,2,3]
+// GFX12: v_cmpx_u_f16_e64_dpp v1.l, v2.l quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x88,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
 
-v_cmpx_u_f16_e64_dpp v1, s2 quad_perm:[3,2,1,0]
-// GFX12: v_cmpx_u_f16_e64_dpp v1, s2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x88,0xd4,0xfa,0x04,0x00,0x00,0x01,0x1b,0x00,0xff]
+v_cmpx_u_f16_e64_dpp v1.l, v2.l row_mirror
+// GFX12: v_cmpx_u_f16_e64_dpp v1.l, v2.l row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x88,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
 
-v_cmpx_u_f16_e64_dpp v1, 2.0 quad_perm:[3,2,1,0]
-// GFX12: v_cmpx_u_f16_e64_dpp v1, 2.0 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x88,0xd4,0xfa,0xe8,0x01,0x00,0x01,0x1b,0x00,0xff]
+v_cmpx_u_f16_e64_dpp v1.l, v2.l row_half_mirror
+// GFX12: v_cmpx_u_f16_e64_dpp v1.l, v2.l row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x88,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
 
-v_cmpx_u_f16_e64_dpp v1, v2 quad_perm:[0,1,2,3]
-// GFX12: v_cmpx_u_f16_e64_dpp v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x88,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+v_cmpx_u_f16_e64_dpp v1.l, v2.l row_shl:1
+// GFX12: v_cmpx_u_f16_e64_dpp v1.l, v2.l row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x88,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
 
-v_cmpx_u_f16_e64_dpp v1, v2 row_mirror
-// GFX12: v_cmpx_u_f16_e64_dpp v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x88,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+v_cmpx_u_f16_e64_dpp v1.l, v2.l row_shl:15
+// GFX12: v_cmpx_u_f16_e64_dpp v1.l, v2.l row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x88,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
 
-v_cmpx_u_f16_e64_dpp v1, v2 row_half_mirror
-// GFX12: v_cmpx_u_f16_e64_dpp v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x88,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+v_cmpx_u_f16_e64_dpp v1.l, v2.l row_shr:1
+// GFX12: v_cmpx_u_f16_e64_dpp v1.l, v2.l row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x88,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
 
-v_cmpx_u_f16_e64_dpp v1, v2 row_shl:1
-// GFX12: v_cmpx_u_f16_e64_dpp v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x88,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+v_cmpx_u_f16_e64_dpp v1.l, v2.l row_shr:15
+// GFX12: v_cmpx_u_f16_e64_dpp v1.l, v2.l row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x88,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
 
-v_cmpx_u_f16_e64_dpp v1, v2 row_shl:15
-// GFX12: v_cmpx_u_f16_e64_dpp v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x88,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+v_cmpx_u_f16_e64_dpp v1.l, v2.l row_ror:1
+// GFX12: v_cmpx_u_f16_e64_dpp v1.l, v2.l row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x88,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
 
-v_cmpx_u_f16_e64_dpp v1, v2 row_shr:1
-// GFX12: v_cmpx_u_f16_e64_dpp v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x88,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+v_cmpx_u_f16_e64_dpp v1.l, v2.l row_ror:15
+// GFX12: v_cmpx_u_f16_e64_dpp v1.l, v2.l row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x88,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
 
-v_cmpx_u_f16_e64_dpp v1, v2 row_shr:15
-// GFX12: v_cmpx_u_f16_e64_dpp v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x88,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+v_cmpx_u_f16_e64_dpp v1.l, v2.l row_share:0 row_mask:0xf bank_mask:0xf
+// GFX12: v_cmpx_u_f16_e64_dpp v1.l, v2.l row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x88,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 
-v_cmpx_u_f16_e64_dpp v1, v2 row_ror:1
-// GFX12: v_cmpx_u_f16_e64_dpp v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x88,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+v_cmpx_u_f16_e64_dpp |v1.l|, -v2.l row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX12: v_cmpx_u_f16_e64_dpp |v1.l|, -v2.l row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x01,0x88,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
 
-v_cmpx_u_f16_e64_dpp v1, v2 row_ror:15
-// GFX12: v_cmpx_u_f16_e64_dpp v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x88,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+v_cmpx_u_f16_e64_dpp -v1.l, |v2.l| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX12: v_cmpx_u_f16_e64_dpp -v1.l, |v2.l| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7e,0x02,0x88,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x09,0x13]
 
-v_cmpx_u_f16_e64_dpp v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
-// GFX12: v_cmpx_u_f16_e64_dpp v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x88,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+v_cmpx_u_f16_e64_dpp -|v255.l|, -|v255.l| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX12: v_cmpx_u_f16_e64_dpp -|v255.l|, -|v255.l| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0x7e,0x83,0x88,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x05,0x30]
 
-v_cmpx_u_f16_e64_dpp |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1
-// GFX12: v_cmpx_u_f16_e64_dpp |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x01,0x88,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+v_cmpx_u_f16_e64_dpp |v1.h|, -v2.h row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX12: v_cmpx_u_f16_e64_dpp |v1.h|, -v2.h op_sel:[1,1] row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x19,0x88,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
 
-v_cmpx_u_f16_e64_dpp -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
-// GFX12: v_cmpx_u_f16_e64_dpp -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7e,0x02,0x88,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x09,0x13]
+v_cmpx_u_f16_e64_dpp -v1.h, |v2.l| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX12: v_cmpx_u_f16_e64_dpp -v1.h, |v2.l| op_sel:[1,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7e,0x0a,0x88,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x09,0x13]
 
-v_cmpx_u_f16_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
-// GFX12: v_cmpx_u_f16_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0x7e,0x83,0x88,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x05,0x30]
+v_cmpx_u_f16_e64_dpp -|v255.l|, -|v255.h| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX12: v_cmpx_u_f16_e64_dpp -|v255.l|, -|v255.h| op_sel:[0,1] clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0x7e,0x93,0x88,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x05,0x30]
 
 v_cmpx_u_f32_e64_dpp v1, v2 quad_perm:[3,2,1,0]
 // GFX12: v_cmpx_u_f32_e64_dpp v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x98,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
diff --git a/llvm/test/MC/AMDGPU/gfx12_asm_vop3cx_dpp8.s b/llvm/test/MC/AMDGPU/gfx12_asm_vop3cx_dpp8.s
index 588ad2b75a410..3503f3d62d737 100644
--- a/llvm/test/MC/AMDGPU/gfx12_asm_vop3cx_dpp8.s
+++ b/llvm/test/MC/AMDGPU/gfx12_asm_vop3cx_dpp8.s
@@ -41,23 +41,32 @@ v_cmpx_class_f32_e64_dpp v1, 2.0 dpp8:[7,6,5,4,3,2,1,0] fi:1
 v_cmpx_class_f32_e64_dpp -|v255|, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
 // GFX12: v_cmpx_class_f32_e64_dpp -|v255|, v255 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0x7e,0x01,0xfe,0xd4,0xe9,0xfe,0x03,0x20,0xff,0x00,0x00,0x00]
 
-v_cmpx_eq_f16_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_cmpx_eq_f16_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x82,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+v_cmpx_eq_f16_e64_dpp v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_cmpx_eq_f16_e64_dpp v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x82,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 
-v_cmpx_eq_f16_e64_dpp |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_cmpx_eq_f16_e64_dpp |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x01,0x82,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+v_cmpx_eq_f16_e64_dpp |v1.l|, -v2.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_cmpx_eq_f16_e64_dpp |v1.l|, -v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x01,0x82,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
 
-v_cmpx_eq_f16_e64_dpp -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1
-// GFX12: v_cmpx_eq_f16_e64_dpp -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7e,0x02,0x82,0xd4,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+v_cmpx_eq_f16_e64_dpp -v1.l, |v2.l| dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX12: v_cmpx_eq_f16_e64_dpp -v1.l, |v2.l| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7e,0x02,0x82,0xd4,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
 
-v_cmpx_eq_f16_e64_dpp -v1, |s2| dpp8:[7,6,5,4,3,2,1,0] fi:1
-// GFX12: v_cmpx_eq_f16_e64_dpp -v1, |s2| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7e,0x02,0x82,0xd4,0xea,0x04,0x00,0x20,0x01,0x77,0x39,0x05]
+v_cmpx_eq_f16_e64_dpp -v1.l, |s2| dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX12: v_cmpx_eq_f16_e64_dpp -v1.l, |s2| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7e,0x02,0x82,0xd4,0xea,0x04,0x00,0x20,0x01,0x77,0x39,0x05]
 
-v_cmpx_eq_f16_e64_dpp -v1, |2.0| dpp8:[7,6,5,4,3,2,1,0] fi:1
-// GFX12: v_cmpx_eq_f16_e64_dpp -v1, |2.0| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7e,0x02,0x82,0xd4,0xea,0xe8,0x01,0x20,0x01,0x77,0x39,0x05]
+v_cmpx_eq_f16_e64_dpp -v1.l, |2.0| dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX12: v_cmpx_eq_f16_e64_dpp -v1.l, |2.0| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7e,0x02,0x82,0xd4,0xea,0xe8,0x01,0x20,0x01,0x77,0x39,0x05]
 
-v_cmpx_eq_f16_e64_dpp -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] fi:0
-// GFX12: v_cmpx_eq_f16_e64_dpp -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0x7e,0x83,0x82,0xd4,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+v_cmpx_eq_f16_e64_dpp -|v255.l|, -|v255.l| clamp dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX12: v_cmpx_eq_f16_e64_dpp -|v255.l|, -|v255.l| clamp dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0x7e,0x83,0x82,0xd4,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+
+v_cmpx_eq_f16_e64_dpp |v1.h|, -v2.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_cmpx_eq_f16_e64_dpp |v1.h|, -v2.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x19,0x82,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+
+v_cmpx_eq_f16_e64_dpp -v1.h, |v2.l| dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX12: v_cmpx_eq_f16_e64_dpp -v1.h, |v2.l| op_sel:[1,0] dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7e,0x0a,0x82,0xd4,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+
+v_cmpx_eq_f16_e64_dpp -|v255.l|, -|v255.h| clamp dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX12: v_cmpx_eq_f16_e64_dpp -|v255.l|, -|v255.h| op_sel:[0,1] clamp dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0x7e,0x93,0x82,0xd4,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
 
 v_cmpx_eq_f32_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0]
 // GFX12: v_cmpx_eq_f32_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x92,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
@@ -155,23 +164,32 @@ v_cmpx_eq_u32_e64_dpp v1, 10 dpp8:[7,6,5,4,3,2,1,0] fi:1
 v_cmpx_eq_u32_e64_dpp v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
 // GFX12: v_cmpx_eq_u32_e64_dpp v255, v255 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0x7e,0x00,0xca,0xd4,0xe9,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 
-v_cmpx_ge_f16_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_cmpx_ge_f16_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x86,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+v_cmpx_ge_f16_e64_dpp v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_cmpx_ge_f16_e64_dpp v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x86,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+
+v_cmpx_ge_f16_e64_dpp |v1.l|, -v2.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_cmpx_ge_f16_e64_dpp |v1.l|, -v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x01,0x86,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+
+v_cmpx_ge_f16_e64_dpp -v1.l, |v2.l| dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX12: v_cmpx_ge_f16_e64_dpp -v1.l, |v2.l| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7e,0x02,0x86,0xd4,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
 
-v_cmpx_ge_f16_e64_dpp |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_cmpx_ge_f16_e64_dpp |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x01,0x86,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+v_cmpx_ge_f16_e64_dpp -v1.l, |s2| dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX12: v_cmpx_ge_f16_e64_dpp -v1.l, |s2| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7e,0x02,0x86,0xd4,0xea,0x04,0x00,0x20,0x01,0x77,0x39,0x05]
 
-v_cmpx_ge_f16_e64_dpp -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1
-// GFX12: v_cmpx_ge_f16_e64_dpp -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7e,0x02,0x86,0xd4,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+v_cmpx_ge_f16_e64_dpp -v1.l, |2.0| dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX12: v_cmpx_ge_f16_e64_dpp -v1.l, |2.0| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7e,0x02,0x86,0xd4,0xea,0xe8,0x01,0x20,0x01,0x77,0x39,0x05]
 
-v_cmpx_ge_f16_e64_dpp -v1, |s2| dpp8:[7,6,5,4,3,2,1,0] fi:1
-// GFX12: v_cmpx_ge_f16_e64_dpp -v1, |s2| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7e,0x02,0x86,0xd4,0xea,0x04,0x00,0x20,0x01,0x77,0x39,0x05]
+v_cmpx_ge_f16_e64_dpp -|v255.l|, -|v255.l| clamp dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX12: v_cmpx_ge_f16_e64_dpp -|v255.l|, -|v255.l| clamp dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0x7e,0x83,0x86,0xd4,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
 
-v_cmpx_ge_f16_e64_dpp -v1, |2.0| dpp8:[7,6,5,4,3,2,1,0] fi:1
-// GFX12: v_cmpx_ge_f16_e64_dpp -v1, |2.0| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7e,0x02,0x86,0xd4,0xea,0xe8,0x01,0x20,0x01,0x77,0x39,0x05]
+v_cmpx_ge_f16_e64_dpp |v1.h|, -v2.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_cmpx_ge_f16_e64_dpp |v1.h|, -v2.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x19,0x86,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
 
-v_cmpx_ge_f16_e64_dpp -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] fi:0
-// GFX12: v_cmpx_ge_f16_e64_dpp -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0x7e,0x83,0x86,0xd4,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+v_cmpx_ge_f16_e64_dpp -v1.h, |v2.l| dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX12: v_cmpx_ge_f16_e64_dpp -v1.h, |v2.l| op_sel:[1,0] dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7e,0x0a,0x86,0xd4,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+
+v_cmpx_ge_f16_e64_dpp -|v255.l|, -|v255.h| clamp dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX12: v_cmpx_ge_f16_e64_dpp -|v255.l|, -|v255.h| op_sel:[0,1] clamp dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0x7e,0x93,0x86,0xd4,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
 
 v_cmpx_ge_f32_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0]
 // GFX12: v_cmpx_ge_f32_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x96,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
@@ -269,23 +287,32 @@ v_cmpx_ge_u32_e64_dpp v1, 10 dpp8:[7,6,5,4,3,2,1,0] fi:1
 v_cmpx_ge_u32_e64_dpp v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
 // GFX12: v_cmpx_ge_u32_e64_dpp v255, v255 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0x7e,0x00,0xce,0xd4,0xe9,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 
-v_cmpx_gt_f16_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_cmpx_gt_f16_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x84,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+v_cmpx_gt_f16_e64_dpp v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_cmpx_gt_f16_e64_dpp v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x84,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+
+v_cmpx_gt_f16_e64_dpp |v1.l|, -v2.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_cmpx_gt_f16_e64_dpp |v1.l|, -v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x01,0x84,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+
+v_cmpx_gt_f16_e64_dpp -v1.l, |v2.l| dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX12: v_cmpx_gt_f16_e64_dpp -v1.l, |v2.l| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7e,0x02,0x84,0xd4,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
 
-v_cmpx_gt_f16_e64_dpp |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_cmpx_gt_f16_e64_dpp |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x01,0x84,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+v_cmpx_gt_f16_e64_dpp -v1.l, |s2| dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX12: v_cmpx_gt_f16_e64_dpp -v1.l, |s2| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7e,0x02,0x84,0xd4,0xea,0x04,0x00,0x20,0x01,0x77,0x39,0x05]
 
-v_cmpx_gt_f16_e64_dpp -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1
-// GFX12: v_cmpx_gt_f16_e64_dpp -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7e,0x02,0x84,0xd4,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+v_cmpx_gt_f16_e64_dpp -v1.l, |2.0| dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX12: v_cmpx_gt_f16_e64_dpp -v1.l, |2.0| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7e,0x02,0x84,0xd4,0xea,0xe8,0x01,0x20,0x01,0x77,0x39,0x05]
 
-v_cmpx_gt_f16_e64_dpp -v1, |s2| dpp8:[7,6,5,4,3,2,1,0] fi:1
-// GFX12: v_cmpx_gt_f16_e64_dpp -v1, |s2| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7e,0x02,0x84,0xd4,0xea,0x04,0x00,0x20,0x01,0x77,0x39,0x05]
+v_cmpx_gt_f16_e64_dpp -|v255.l|, -|v255.l| clamp dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX12: v_cmpx_gt_f16_e64_dpp -|v255.l|, -|v255.l| clamp dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0x7e,0x83,0x84,0xd4,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
 
-v_cmpx_gt_f16_e64_dpp -v1, |2.0| dpp8:[7,6,5,4,3,2,1,0] fi:1
-// GFX12: v_cmpx_gt_f16_e64_dpp -v1, |2.0| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7e,0x02,0x84,0xd4,0xea,0xe8,0x01,0x20,0x01,0x77,0x39,0x05]
+v_cmpx_gt_f16_e64_dpp |v1.h|, -v2.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_cmpx_gt_f16_e64_dpp |v1.h|, -v2.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x19,0x84,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
 
-v_cmpx_gt_f16_e64_dpp -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] fi:0
-// GFX12: v_cmpx_gt_f16_e64_dpp -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0x7e,0x83,0x84,0xd4,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+v_cmpx_gt_f16_e64_dpp -v1.h, |v2.l| dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX12: v_cmpx_gt_f16_e64_dpp -v1.h, |v2.l| op_sel:[1,0] dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7e,0x0a,0x84,0xd4,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+
+v_cmpx_gt_f16_e64_dpp -|v255.l|, -|v255.h| clamp dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX12: v_cmpx_gt_f16_e64_dpp -|v255.l|, -|v255.h| op_sel:[0,1] clamp dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0x7e,0x93,0x84,0xd4,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
 
 v_cmpx_gt_f32_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0]
 // GFX12: v_cmpx_gt_f32_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x94,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
@@ -383,23 +410,32 @@ v_cmpx_gt_u32_e64_dpp v1, 10 dpp8:[7,6,5,4,3,2,1,0] fi:1
 v_cmpx_gt_u32_e64_dpp v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
 // GFX12: v_cmpx_gt_u32_e64_dpp v255, v255 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0x7e,0x00,0xcc,0xd4,0xe9,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 
-v_cmpx_le_f16_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_cmpx_le_f16_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x83,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+v_cmpx_le_f16_e64_dpp v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_cmpx_le_f16_e64_dpp v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x83,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+
+v_cmpx_le_f16_e64_dpp |v1.l|, -v2.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_cmpx_le_f16_e64_dpp |v1.l|, -v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x01,0x83,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
 
-v_cmpx_le_f16_e64_dpp |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_cmpx_le_f16_e64_dpp |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x01,0x83,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+v_cmpx_le_f16_e64_dpp -v1.l, |v2.l| dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX12: v_cmpx_le_f16_e64_dpp -v1.l, |v2.l| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7e,0x02,0x83,0xd4,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
 
-v_cmpx_le_f16_e64_dpp -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1
-// GFX12: v_cmpx_le_f16_e64_dpp -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7e,0x02,0x83,0xd4,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+v_cmpx_le_f16_e64_dpp -v1.l, |s2| dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX12: v_cmpx_le_f16_e64_dpp -v1.l, |s2| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7e,0x02,0x83,0xd4,0xea,0x04,0x00,0x20,0x01,0x77,0x39,0x05]
 
-v_cmpx_le_f16_e64_dpp -v1, |s2| dpp8:[7,6,5,4,3,2,1,0] fi:1
-// GFX12: v_cmpx_le_f16_e64_dpp -v1, |s2| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7e,0x02,0x83,0xd4,0xea,0x04,0x00,0x20,0x01,0x77,0x39,0x05]
+v_cmpx_le_f16_e64_dpp -v1.l, |2.0| dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX12: v_cmpx_le_f16_e64_dpp -v1.l, |2.0| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7e,0x02,0x83,0xd4,0xea,0xe8,0x01,0x20,0x01,0x77,0x39,0x05]
 
-v_cmpx_le_f16_e64_dpp -v1, |2.0| dpp8:[7,6,5,4,3,2,1,0] fi:1
-// GFX12: v_cmpx_le_f16_e64_dpp -v1, |2.0| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7e,0x02,0x83,0xd4,0xea,0xe8,0x01,0x20,0x01,0x77,0x39,0x05]
+v_cmpx_le_f16_e64_dpp -|v255.l|, -|v255.l| clamp dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX12: v_cmpx_le_f16_e64_dpp -|v255.l|, -|v255.l| clamp dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0x7e,0x83,0x83,0xd4,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
 
-v_cmpx_le_f16_e64_dpp -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] fi:0
-// GFX12: v_cmpx_le_f16_e64_dpp -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0x7e,0x83,0x83,0xd4,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+v_cmpx_le_f16_e64_dpp |v1.h|, -v2.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_cmpx_le_f16_e64_dpp |v1.h|, -v2.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x19,0x83,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+
+v_cmpx_le_f16_e64_dpp -v1.h, |v2.l| dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX12: v_cmpx_le_f16_e64_dpp -v1.h, |v2.l| op_sel:[1,0] dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7e,0x0a,0x83,0xd4,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+
+v_cmpx_le_f16_e64_dpp -|v255.l|, -|v255.h| clamp dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX12: v_cmpx_le_f16_e64_dpp -|v255.l|, -|v255.h| op_sel:[0,1] clamp dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0x7e,0x93,0x83,0xd4,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
 
 v_cmpx_le_f32_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0]
 // GFX12: v_cmpx_le_f32_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x93,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
@@ -497,23 +533,32 @@ v_cmpx_le_u32_e64_dpp v1, 10 dpp8:[7,6,5,4,3,2,1,0] fi:1
 v_cmpx_le_u32_e64_dpp v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
 // GFX12: v_cmpx_le_u32_e64_dpp v255, v255 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0x7e,0x00,0xcb,0xd4,0xe9,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 
-v_cmpx_lg_f16_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_cmpx_lg_f16_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x85,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+v_cmpx_lg_f16_e64_dpp v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_cmpx_lg_f16_e64_dpp v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x85,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+
+v_cmpx_lg_f16_e64_dpp |v1.l|, -v2.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_cmpx_lg_f16_e64_dpp |v1.l|, -v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x01,0x85,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+
+v_cmpx_lg_f16_e64_dpp -v1.l, |v2.l| dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX12: v_cmpx_lg_f16_e64_dpp -v1.l, |v2.l| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7e,0x02,0x85,0xd4,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+
+v_cmpx_lg_f16_e64_dpp -v1.l, |s2| dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX12: v_cmpx_lg_f16_e64_dpp -v1.l, |s2| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7e,0x02,0x85,0xd4,0xea,0x04,0x00,0x20,0x01,0x77,0x39,0x05]
 
-v_cmpx_lg_f16_e64_dpp |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_cmpx_lg_f16_e64_dpp |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x01,0x85,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+v_cmpx_lg_f16_e64_dpp -v1.l, |2.0| dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX12: v_cmpx_lg_f16_e64_dpp -v1.l, |2.0| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7e,0x02,0x85,0xd4,0xea,0xe8,0x01,0x20,0x01,0x77,0x39,0x05]
 
-v_cmpx_lg_f16_e64_dpp -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1
-// GFX12: v_cmpx_lg_f16_e64_dpp -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7e,0x02,0x85,0xd4,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+v_cmpx_lg_f16_e64_dpp -|v255.l|, -|v255.l| clamp dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX12: v_cmpx_lg_f16_e64_dpp -|v255.l|, -|v255.l| clamp dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0x7e,0x83,0x85,0xd4,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
 
-v_cmpx_lg_f16_e64_dpp -v1, |s2| dpp8:[7,6,5,4,3,2,1,0] fi:1
-// GFX12: v_cmpx_lg_f16_e64_dpp -v1, |s2| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7e,0x02,0x85,0xd4,0xea,0x04,0x00,0x20,0x01,0x77,0x39,0x05]
+v_cmpx_lg_f16_e64_dpp |v1.h|, -v2.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_cmpx_lg_f16_e64_dpp |v1.h|, -v2.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x19,0x85,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
 
-v_cmpx_lg_f16_e64_dpp -v1, |2.0| dpp8:[7,6,5,4,3,2,1,0] fi:1
-// GFX12: v_cmpx_lg_f16_e64_dpp -v1, |2.0| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7e,0x02,0x85,0xd4,0xea,0xe8,0x01,0x20,0x01,0x77,0x39,0x05]
+v_cmpx_lg_f16_e64_dpp -v1.h, |v2.l| dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX12: v_cmpx_lg_f16_e64_dpp -v1.h, |v2.l| op_sel:[1,0] dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7e,0x0a,0x85,0xd4,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
 
-v_cmpx_lg_f16_e64_dpp -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] fi:0
-// GFX12: v_cmpx_lg_f16_e64_dpp -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0x7e,0x83,0x85,0xd4,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+v_cmpx_lg_f16_e64_dpp -|v255.l|, -|v255.h| clamp dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX12: v_cmpx_lg_f16_e64_dpp -|v255.l|, -|v255.h| op_sel:[0,1] clamp dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0x7e,0x93,0x85,0xd4,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
 
 v_cmpx_lg_f32_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0]
 // GFX12: v_cmpx_lg_f32_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x95,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
@@ -734,23 +779,32 @@ v_cmpx_ne_u32_e64_dpp v1, 10 dpp8:[7,6,5,4,3,2,1,0] fi:1
 v_cmpx_ne_u32_e64_dpp v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
 // GFX12: v_cmpx_ne_u32_e64_dpp v255, v255 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0x7e,0x00,0xcd,0xd4,0xe9,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 
-v_cmpx_neq_f16_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_cmpx_neq_f16_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x8d,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+v_cmpx_neq_f16_e64_dpp v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_cmpx_neq_f16_e64_dpp v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x8d,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 
-v_cmpx_neq_f16_e64_dpp |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_cmpx_neq_f16_e64_dpp |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x01,0x8d,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+v_cmpx_neq_f16_e64_dpp |v1.l|, -v2.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_cmpx_neq_f16_e64_dpp |v1.l|, -v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x01,0x8d,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
 
-v_cmpx_neq_f16_e64_dpp -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1
-// GFX12: v_cmpx_neq_f16_e64_dpp -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7e,0x02,0x8d,0xd4,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+v_cmpx_neq_f16_e64_dpp -v1.l, |v2.l| dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX12: v_cmpx_neq_f16_e64_dpp -v1.l, |v2.l| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7e,0x02,0x8d,0xd4,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
 
-v_cmpx_neq_f16_e64_dpp -v1, |s2| dpp8:[7,6,5,4,3,2,1,0] fi:1
-// GFX12: v_cmpx_neq_f16_e64_dpp -v1, |s2| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7e,0x02,0x8d,0xd4,0xea,0x04,0x00,0x20,0x01,0x77,0x39,0x05]
+v_cmpx_neq_f16_e64_dpp -v1.l, |s2| dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX12: v_cmpx_neq_f16_e64_dpp -v1.l, |s2| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7e,0x02,0x8d,0xd4,0xea,0x04,0x00,0x20,0x01,0x77,0x39,0x05]
 
-v_cmpx_neq_f16_e64_dpp -v1, |2.0| dpp8:[7,6,5,4,3,2,1,0] fi:1
-// GFX12: v_cmpx_neq_f16_e64_dpp -v1, |2.0| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7e,0x02,0x8d,0xd4,0xea,0xe8,0x01,0x20,0x01,0x77,0x39,0x05]
+v_cmpx_neq_f16_e64_dpp -v1.l, |2.0| dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX12: v_cmpx_neq_f16_e64_dpp -v1.l, |2.0| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7e,0x02,0x8d,0xd4,0xea,0xe8,0x01,0x20,0x01,0x77,0x39,0x05]
 
-v_cmpx_neq_f16_e64_dpp -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] fi:0
-// GFX12: v_cmpx_neq_f16_e64_dpp -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0x7e,0x83,0x8d,0xd4,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+v_cmpx_neq_f16_e64_dpp -|v255.l|, -|v255.l| clamp dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX12: v_cmpx_neq_f16_e64_dpp -|v255.l|, -|v255.l| clamp dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0x7e,0x83,0x8d,0xd4,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+
+v_cmpx_neq_f16_e64_dpp |v1.h|, -v2.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_cmpx_neq_f16_e64_dpp |v1.h|, -v2.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x19,0x8d,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+
+v_cmpx_neq_f16_e64_dpp -v1.h, |v2.l| dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX12: v_cmpx_neq_f16_e64_dpp -v1.h, |v2.l| op_sel:[1,0] dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7e,0x0a,0x8d,0xd4,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+
+v_cmpx_neq_f16_e64_dpp -|v255.l|, -|v255.h| clamp dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX12: v_cmpx_neq_f16_e64_dpp -|v255.l|, -|v255.h| op_sel:[0,1] clamp dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0x7e,0x93,0x8d,0xd4,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
 
 v_cmpx_neq_f32_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0]
 // GFX12: v_cmpx_neq_f32_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x9d,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
@@ -770,23 +824,32 @@ v_cmpx_neq_f32_e64_dpp -v1, |2.0| dpp8:[7,6,5,4,3,2,1,0] fi:1
 v_cmpx_neq_f32_e64_dpp -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] fi:0
 // GFX12: v_cmpx_neq_f32_e64_dpp -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0x7e,0x83,0x9d,0xd4,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
 
-v_cmpx_nge_f16_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_cmpx_nge_f16_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x89,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+v_cmpx_nge_f16_e64_dpp v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_cmpx_nge_f16_e64_dpp v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x89,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+
+v_cmpx_nge_f16_e64_dpp |v1.l|, -v2.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_cmpx_nge_f16_e64_dpp |v1.l|, -v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x01,0x89,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+
+v_cmpx_nge_f16_e64_dpp -v1.l, |v2.l| dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX12: v_cmpx_nge_f16_e64_dpp -v1.l, |v2.l| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7e,0x02,0x89,0xd4,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
 
-v_cmpx_nge_f16_e64_dpp |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_cmpx_nge_f16_e64_dpp |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x01,0x89,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+v_cmpx_nge_f16_e64_dpp -v1.l, |s2| dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX12: v_cmpx_nge_f16_e64_dpp -v1.l, |s2| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7e,0x02,0x89,0xd4,0xea,0x04,0x00,0x20,0x01,0x77,0x39,0x05]
 
-v_cmpx_nge_f16_e64_dpp -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1
-// GFX12: v_cmpx_nge_f16_e64_dpp -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7e,0x02,0x89,0xd4,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+v_cmpx_nge_f16_e64_dpp -v1.l, |2.0| dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX12: v_cmpx_nge_f16_e64_dpp -v1.l, |2.0| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7e,0x02,0x89,0xd4,0xea,0xe8,0x01,0x20,0x01,0x77,0x39,0x05]
 
-v_cmpx_nge_f16_e64_dpp -v1, |s2| dpp8:[7,6,5,4,3,2,1,0] fi:1
-// GFX12: v_cmpx_nge_f16_e64_dpp -v1, |s2| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7e,0x02,0x89,0xd4,0xea,0x04,0x00,0x20,0x01,0x77,0x39,0x05]
+v_cmpx_nge_f16_e64_dpp -|v255.l|, -|v255.l| clamp dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX12: v_cmpx_nge_f16_e64_dpp -|v255.l|, -|v255.l| clamp dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0x7e,0x83,0x89,0xd4,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
 
-v_cmpx_nge_f16_e64_dpp -v1, |2.0| dpp8:[7,6,5,4,3,2,1,0] fi:1
-// GFX12: v_cmpx_nge_f16_e64_dpp -v1, |2.0| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7e,0x02,0x89,0xd4,0xea,0xe8,0x01,0x20,0x01,0x77,0x39,0x05]
+v_cmpx_nge_f16_e64_dpp |v1.h|, -v2.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_cmpx_nge_f16_e64_dpp |v1.h|, -v2.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x19,0x89,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
 
-v_cmpx_nge_f16_e64_dpp -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] fi:0
-// GFX12: v_cmpx_nge_f16_e64_dpp -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0x7e,0x83,0x89,0xd4,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+v_cmpx_nge_f16_e64_dpp -v1.h, |v2.l| dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX12: v_cmpx_nge_f16_e64_dpp -v1.h, |v2.l| op_sel:[1,0] dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7e,0x0a,0x89,0xd4,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+
+v_cmpx_nge_f16_e64_dpp -|v255.l|, -|v255.h| clamp dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX12: v_cmpx_nge_f16_e64_dpp -|v255.l|, -|v255.h| op_sel:[0,1] clamp dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0x7e,0x93,0x89,0xd4,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
 
 v_cmpx_nge_f32_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0]
 // GFX12: v_cmpx_nge_f32_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x99,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
@@ -806,23 +869,32 @@ v_cmpx_nge_f32_e64_dpp -v1, |2.0| dpp8:[7,6,5,4,3,2,1,0] fi:1
 v_cmpx_nge_f32_e64_dpp -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] fi:0
 // GFX12: v_cmpx_nge_f32_e64_dpp -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0x7e,0x83,0x99,0xd4,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
 
-v_cmpx_ngt_f16_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_cmpx_ngt_f16_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x8b,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+v_cmpx_ngt_f16_e64_dpp v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_cmpx_ngt_f16_e64_dpp v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x8b,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+
+v_cmpx_ngt_f16_e64_dpp |v1.l|, -v2.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_cmpx_ngt_f16_e64_dpp |v1.l|, -v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x01,0x8b,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+
+v_cmpx_ngt_f16_e64_dpp -v1.l, |v2.l| dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX12: v_cmpx_ngt_f16_e64_dpp -v1.l, |v2.l| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7e,0x02,0x8b,0xd4,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
 
-v_cmpx_ngt_f16_e64_dpp |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_cmpx_ngt_f16_e64_dpp |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x01,0x8b,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+v_cmpx_ngt_f16_e64_dpp -v1.l, |s2| dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX12: v_cmpx_ngt_f16_e64_dpp -v1.l, |s2| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7e,0x02,0x8b,0xd4,0xea,0x04,0x00,0x20,0x01,0x77,0x39,0x05]
 
-v_cmpx_ngt_f16_e64_dpp -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1
-// GFX12: v_cmpx_ngt_f16_e64_dpp -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7e,0x02,0x8b,0xd4,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+v_cmpx_ngt_f16_e64_dpp -v1.l, |2.0| dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX12: v_cmpx_ngt_f16_e64_dpp -v1.l, |2.0| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7e,0x02,0x8b,0xd4,0xea,0xe8,0x01,0x20,0x01,0x77,0x39,0x05]
 
-v_cmpx_ngt_f16_e64_dpp -v1, |s2| dpp8:[7,6,5,4,3,2,1,0] fi:1
-// GFX12: v_cmpx_ngt_f16_e64_dpp -v1, |s2| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7e,0x02,0x8b,0xd4,0xea,0x04,0x00,0x20,0x01,0x77,0x39,0x05]
+v_cmpx_ngt_f16_e64_dpp -|v255.l|, -|v255.l| clamp dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX12: v_cmpx_ngt_f16_e64_dpp -|v255.l|, -|v255.l| clamp dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0x7e,0x83,0x8b,0xd4,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
 
-v_cmpx_ngt_f16_e64_dpp -v1, |2.0| dpp8:[7,6,5,4,3,2,1,0] fi:1
-// GFX12: v_cmpx_ngt_f16_e64_dpp -v1, |2.0| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7e,0x02,0x8b,0xd4,0xea,0xe8,0x01,0x20,0x01,0x77,0x39,0x05]
+v_cmpx_ngt_f16_e64_dpp |v1.h|, -v2.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_cmpx_ngt_f16_e64_dpp |v1.h|, -v2.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x19,0x8b,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
 
-v_cmpx_ngt_f16_e64_dpp -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] fi:0
-// GFX12: v_cmpx_ngt_f16_e64_dpp -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0x7e,0x83,0x8b,0xd4,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+v_cmpx_ngt_f16_e64_dpp -v1.h, |v2.l| dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX12: v_cmpx_ngt_f16_e64_dpp -v1.h, |v2.l| op_sel:[1,0] dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7e,0x0a,0x8b,0xd4,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+
+v_cmpx_ngt_f16_e64_dpp -|v255.l|, -|v255.h| clamp dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX12: v_cmpx_ngt_f16_e64_dpp -|v255.l|, -|v255.h| op_sel:[0,1] clamp dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0x7e,0x93,0x8b,0xd4,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
 
 v_cmpx_ngt_f32_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0]
 // GFX12: v_cmpx_ngt_f32_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x9b,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
@@ -842,23 +914,32 @@ v_cmpx_ngt_f32_e64_dpp -v1, |2.0| dpp8:[7,6,5,4,3,2,1,0] fi:1
 v_cmpx_ngt_f32_e64_dpp -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] fi:0
 // GFX12: v_cmpx_ngt_f32_e64_dpp -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0x7e,0x83,0x9b,0xd4,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
 
-v_cmpx_nle_f16_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_cmpx_nle_f16_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x8c,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+v_cmpx_nle_f16_e64_dpp v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_cmpx_nle_f16_e64_dpp v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x8c,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+
+v_cmpx_nle_f16_e64_dpp |v1.l|, -v2.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_cmpx_nle_f16_e64_dpp |v1.l|, -v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x01,0x8c,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
 
-v_cmpx_nle_f16_e64_dpp |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_cmpx_nle_f16_e64_dpp |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x01,0x8c,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+v_cmpx_nle_f16_e64_dpp -v1.l, |v2.l| dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX12: v_cmpx_nle_f16_e64_dpp -v1.l, |v2.l| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7e,0x02,0x8c,0xd4,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
 
-v_cmpx_nle_f16_e64_dpp -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1
-// GFX12: v_cmpx_nle_f16_e64_dpp -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7e,0x02,0x8c,0xd4,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+v_cmpx_nle_f16_e64_dpp -v1.l, |s2| dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX12: v_cmpx_nle_f16_e64_dpp -v1.l, |s2| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7e,0x02,0x8c,0xd4,0xea,0x04,0x00,0x20,0x01,0x77,0x39,0x05]
 
-v_cmpx_nle_f16_e64_dpp -v1, |s2| dpp8:[7,6,5,4,3,2,1,0] fi:1
-// GFX12: v_cmpx_nle_f16_e64_dpp -v1, |s2| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7e,0x02,0x8c,0xd4,0xea,0x04,0x00,0x20,0x01,0x77,0x39,0x05]
+v_cmpx_nle_f16_e64_dpp -v1.l, |2.0| dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX12: v_cmpx_nle_f16_e64_dpp -v1.l, |2.0| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7e,0x02,0x8c,0xd4,0xea,0xe8,0x01,0x20,0x01,0x77,0x39,0x05]
 
-v_cmpx_nle_f16_e64_dpp -v1, |2.0| dpp8:[7,6,5,4,3,2,1,0] fi:1
-// GFX12: v_cmpx_nle_f16_e64_dpp -v1, |2.0| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7e,0x02,0x8c,0xd4,0xea,0xe8,0x01,0x20,0x01,0x77,0x39,0x05]
+v_cmpx_nle_f16_e64_dpp -|v255.l|, -|v255.l| clamp dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX12: v_cmpx_nle_f16_e64_dpp -|v255.l|, -|v255.l| clamp dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0x7e,0x83,0x8c,0xd4,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
 
-v_cmpx_nle_f16_e64_dpp -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] fi:0
-// GFX12: v_cmpx_nle_f16_e64_dpp -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0x7e,0x83,0x8c,0xd4,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+v_cmpx_nle_f16_e64_dpp |v1.h|, -v2.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_cmpx_nle_f16_e64_dpp |v1.h|, -v2.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x19,0x8c,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+
+v_cmpx_nle_f16_e64_dpp -v1.h, |v2.l| dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX12: v_cmpx_nle_f16_e64_dpp -v1.h, |v2.l| op_sel:[1,0] dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7e,0x0a,0x8c,0xd4,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+
+v_cmpx_nle_f16_e64_dpp -|v255.l|, -|v255.h| clamp dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX12: v_cmpx_nle_f16_e64_dpp -|v255.l|, -|v255.h| op_sel:[0,1] clamp dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0x7e,0x93,0x8c,0xd4,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
 
 v_cmpx_nle_f32_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0]
 // GFX12: v_cmpx_nle_f32_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x9c,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
@@ -878,23 +959,32 @@ v_cmpx_nle_f32_e64_dpp -v1, |2.0| dpp8:[7,6,5,4,3,2,1,0] fi:1
 v_cmpx_nle_f32_e64_dpp -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] fi:0
 // GFX12: v_cmpx_nle_f32_e64_dpp -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0x7e,0x83,0x9c,0xd4,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
 
-v_cmpx_nlg_f16_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_cmpx_nlg_f16_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x8a,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+v_cmpx_nlg_f16_e64_dpp v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_cmpx_nlg_f16_e64_dpp v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x8a,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+
+v_cmpx_nlg_f16_e64_dpp |v1.l|, -v2.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_cmpx_nlg_f16_e64_dpp |v1.l|, -v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x01,0x8a,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+
+v_cmpx_nlg_f16_e64_dpp -v1.l, |v2.l| dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX12: v_cmpx_nlg_f16_e64_dpp -v1.l, |v2.l| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7e,0x02,0x8a,0xd4,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+
+v_cmpx_nlg_f16_e64_dpp -v1.l, |s2| dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX12: v_cmpx_nlg_f16_e64_dpp -v1.l, |s2| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7e,0x02,0x8a,0xd4,0xea,0x04,0x00,0x20,0x01,0x77,0x39,0x05]
 
-v_cmpx_nlg_f16_e64_dpp |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_cmpx_nlg_f16_e64_dpp |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x01,0x8a,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+v_cmpx_nlg_f16_e64_dpp -v1.l, |2.0| dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX12: v_cmpx_nlg_f16_e64_dpp -v1.l, |2.0| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7e,0x02,0x8a,0xd4,0xea,0xe8,0x01,0x20,0x01,0x77,0x39,0x05]
 
-v_cmpx_nlg_f16_e64_dpp -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1
-// GFX12: v_cmpx_nlg_f16_e64_dpp -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7e,0x02,0x8a,0xd4,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+v_cmpx_nlg_f16_e64_dpp -|v255.l|, -|v255.l| clamp dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX12: v_cmpx_nlg_f16_e64_dpp -|v255.l|, -|v255.l| clamp dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0x7e,0x83,0x8a,0xd4,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
 
-v_cmpx_nlg_f16_e64_dpp -v1, |s2| dpp8:[7,6,5,4,3,2,1,0] fi:1
-// GFX12: v_cmpx_nlg_f16_e64_dpp -v1, |s2| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7e,0x02,0x8a,0xd4,0xea,0x04,0x00,0x20,0x01,0x77,0x39,0x05]
+v_cmpx_nlg_f16_e64_dpp |v1.h|, -v2.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_cmpx_nlg_f16_e64_dpp |v1.h|, -v2.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x19,0x8a,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
 
-v_cmpx_nlg_f16_e64_dpp -v1, |2.0| dpp8:[7,6,5,4,3,2,1,0] fi:1
-// GFX12: v_cmpx_nlg_f16_e64_dpp -v1, |2.0| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7e,0x02,0x8a,0xd4,0xea,0xe8,0x01,0x20,0x01,0x77,0x39,0x05]
+v_cmpx_nlg_f16_e64_dpp -v1.h, |v2.l| dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX12: v_cmpx_nlg_f16_e64_dpp -v1.h, |v2.l| op_sel:[1,0] dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7e,0x0a,0x8a,0xd4,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
 
-v_cmpx_nlg_f16_e64_dpp -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] fi:0
-// GFX12: v_cmpx_nlg_f16_e64_dpp -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0x7e,0x83,0x8a,0xd4,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+v_cmpx_nlg_f16_e64_dpp -|v255.l|, -|v255.h| clamp dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX12: v_cmpx_nlg_f16_e64_dpp -|v255.l|, -|v255.h| op_sel:[0,1] clamp dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0x7e,0x93,0x8a,0xd4,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
 
 v_cmpx_nlg_f32_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0]
 // GFX12: v_cmpx_nlg_f32_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x9a,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
@@ -914,23 +1004,32 @@ v_cmpx_nlg_f32_e64_dpp -v1, |2.0| dpp8:[7,6,5,4,3,2,1,0] fi:1
 v_cmpx_nlg_f32_e64_dpp -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] fi:0
 // GFX12: v_cmpx_nlg_f32_e64_dpp -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0x7e,0x83,0x9a,0xd4,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
 
-v_cmpx_nlt_f16_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_cmpx_nlt_f16_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x8e,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+v_cmpx_nlt_f16_e64_dpp v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_cmpx_nlt_f16_e64_dpp v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x8e,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 
-v_cmpx_nlt_f16_e64_dpp |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_cmpx_nlt_f16_e64_dpp |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x01,0x8e,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+v_cmpx_nlt_f16_e64_dpp |v1.l|, -v2.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_cmpx_nlt_f16_e64_dpp |v1.l|, -v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x01,0x8e,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
 
-v_cmpx_nlt_f16_e64_dpp -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1
-// GFX12: v_cmpx_nlt_f16_e64_dpp -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7e,0x02,0x8e,0xd4,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+v_cmpx_nlt_f16_e64_dpp -v1.l, |v2.l| dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX12: v_cmpx_nlt_f16_e64_dpp -v1.l, |v2.l| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7e,0x02,0x8e,0xd4,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
 
-v_cmpx_nlt_f16_e64_dpp -v1, |s2| dpp8:[7,6,5,4,3,2,1,0] fi:1
-// GFX12: v_cmpx_nlt_f16_e64_dpp -v1, |s2| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7e,0x02,0x8e,0xd4,0xea,0x04,0x00,0x20,0x01,0x77,0x39,0x05]
+v_cmpx_nlt_f16_e64_dpp -v1.l, |s2| dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX12: v_cmpx_nlt_f16_e64_dpp -v1.l, |s2| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7e,0x02,0x8e,0xd4,0xea,0x04,0x00,0x20,0x01,0x77,0x39,0x05]
 
-v_cmpx_nlt_f16_e64_dpp -v1, |2.0| dpp8:[7,6,5,4,3,2,1,0] fi:1
-// GFX12: v_cmpx_nlt_f16_e64_dpp -v1, |2.0| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7e,0x02,0x8e,0xd4,0xea,0xe8,0x01,0x20,0x01,0x77,0x39,0x05]
+v_cmpx_nlt_f16_e64_dpp -v1.l, |2.0| dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX12: v_cmpx_nlt_f16_e64_dpp -v1.l, |2.0| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7e,0x02,0x8e,0xd4,0xea,0xe8,0x01,0x20,0x01,0x77,0x39,0x05]
 
-v_cmpx_nlt_f16_e64_dpp -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] fi:0
-// GFX12: v_cmpx_nlt_f16_e64_dpp -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0x7e,0x83,0x8e,0xd4,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+v_cmpx_nlt_f16_e64_dpp -|v255.l|, -|v255.l| clamp dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX12: v_cmpx_nlt_f16_e64_dpp -|v255.l|, -|v255.l| clamp dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0x7e,0x83,0x8e,0xd4,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+
+v_cmpx_nlt_f16_e64_dpp |v1.h|, -v2.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_cmpx_nlt_f16_e64_dpp |v1.h|, -v2.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x19,0x8e,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+
+v_cmpx_nlt_f16_e64_dpp -v1.h, |v2.l| dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX12: v_cmpx_nlt_f16_e64_dpp -v1.h, |v2.l| op_sel:[1,0] dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7e,0x0a,0x8e,0xd4,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+
+v_cmpx_nlt_f16_e64_dpp -|v255.l|, -|v255.h| clamp dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX12: v_cmpx_nlt_f16_e64_dpp -|v255.l|, -|v255.h| op_sel:[0,1] clamp dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0x7e,0x93,0x8e,0xd4,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
 
 v_cmpx_nlt_f32_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0]
 // GFX12: v_cmpx_nlt_f32_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x9e,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
@@ -950,23 +1049,32 @@ v_cmpx_nlt_f32_e64_dpp -v1, |2.0| dpp8:[7,6,5,4,3,2,1,0] fi:1
 v_cmpx_nlt_f32_e64_dpp -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] fi:0
 // GFX12: v_cmpx_nlt_f32_e64_dpp -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0x7e,0x83,0x9e,0xd4,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
 
-v_cmpx_o_f16_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_cmpx_o_f16_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x87,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+v_cmpx_o_f16_e64_dpp v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_cmpx_o_f16_e64_dpp v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x87,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+
+v_cmpx_o_f16_e64_dpp |v1.l|, -v2.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_cmpx_o_f16_e64_dpp |v1.l|, -v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x01,0x87,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+
+v_cmpx_o_f16_e64_dpp -v1.l, |v2.l| dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX12: v_cmpx_o_f16_e64_dpp -v1.l, |v2.l| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7e,0x02,0x87,0xd4,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
 
-v_cmpx_o_f16_e64_dpp |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_cmpx_o_f16_e64_dpp |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x01,0x87,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+v_cmpx_o_f16_e64_dpp -v1.l, |s2| dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX12: v_cmpx_o_f16_e64_dpp -v1.l, |s2| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7e,0x02,0x87,0xd4,0xea,0x04,0x00,0x20,0x01,0x77,0x39,0x05]
 
-v_cmpx_o_f16_e64_dpp -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1
-// GFX12: v_cmpx_o_f16_e64_dpp -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7e,0x02,0x87,0xd4,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+v_cmpx_o_f16_e64_dpp -v1.l, |2.0| dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX12: v_cmpx_o_f16_e64_dpp -v1.l, |2.0| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7e,0x02,0x87,0xd4,0xea,0xe8,0x01,0x20,0x01,0x77,0x39,0x05]
 
-v_cmpx_o_f16_e64_dpp -v1, |s2| dpp8:[7,6,5,4,3,2,1,0] fi:1
-// GFX12: v_cmpx_o_f16_e64_dpp -v1, |s2| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7e,0x02,0x87,0xd4,0xea,0x04,0x00,0x20,0x01,0x77,0x39,0x05]
+v_cmpx_o_f16_e64_dpp -|v255.l|, -|v255.l| clamp dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX12: v_cmpx_o_f16_e64_dpp -|v255.l|, -|v255.l| clamp dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0x7e,0x83,0x87,0xd4,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
 
-v_cmpx_o_f16_e64_dpp -v1, |2.0| dpp8:[7,6,5,4,3,2,1,0] fi:1
-// GFX12: v_cmpx_o_f16_e64_dpp -v1, |2.0| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7e,0x02,0x87,0xd4,0xea,0xe8,0x01,0x20,0x01,0x77,0x39,0x05]
+v_cmpx_o_f16_e64_dpp |v1.h|, -v2.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_cmpx_o_f16_e64_dpp |v1.h|, -v2.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x19,0x87,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
 
-v_cmpx_o_f16_e64_dpp -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] fi:0
-// GFX12: v_cmpx_o_f16_e64_dpp -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0x7e,0x83,0x87,0xd4,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+v_cmpx_o_f16_e64_dpp -v1.h, |v2.l| dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX12: v_cmpx_o_f16_e64_dpp -v1.h, |v2.l| op_sel:[1,0] dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7e,0x0a,0x87,0xd4,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+
+v_cmpx_o_f16_e64_dpp -|v255.l|, -|v255.h| clamp dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX12: v_cmpx_o_f16_e64_dpp -|v255.l|, -|v255.h| op_sel:[0,1] clamp dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0x7e,0x93,0x87,0xd4,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
 
 v_cmpx_o_f32_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0]
 // GFX12: v_cmpx_o_f32_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x97,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
@@ -986,23 +1094,32 @@ v_cmpx_o_f32_e64_dpp -v1, |2.0| dpp8:[7,6,5,4,3,2,1,0] fi:1
 v_cmpx_o_f32_e64_dpp -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] fi:0
 // GFX12: v_cmpx_o_f32_e64_dpp -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0x7e,0x83,0x97,0xd4,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
 
-v_cmpx_u_f16_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_cmpx_u_f16_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x88,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+v_cmpx_u_f16_e64_dpp v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_cmpx_u_f16_e64_dpp v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x88,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+
+v_cmpx_u_f16_e64_dpp |v1.l|, -v2.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_cmpx_u_f16_e64_dpp |v1.l|, -v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x01,0x88,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+
+v_cmpx_u_f16_e64_dpp -v1.l, |v2.l| dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX12: v_cmpx_u_f16_e64_dpp -v1.l, |v2.l| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7e,0x02,0x88,0xd4,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+
+v_cmpx_u_f16_e64_dpp -v1.l, |s2| dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX12: v_cmpx_u_f16_e64_dpp -v1.l, |s2| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7e,0x02,0x88,0xd4,0xea,0x04,0x00,0x20,0x01,0x77,0x39,0x05]
 
-v_cmpx_u_f16_e64_dpp |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_cmpx_u_f16_e64_dpp |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x01,0x88,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+v_cmpx_u_f16_e64_dpp -v1.l, |2.0| dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX12: v_cmpx_u_f16_e64_dpp -v1.l, |2.0| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7e,0x02,0x88,0xd4,0xea,0xe8,0x01,0x20,0x01,0x77,0x39,0x05]
 
-v_cmpx_u_f16_e64_dpp -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1
-// GFX12: v_cmpx_u_f16_e64_dpp -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7e,0x02,0x88,0xd4,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+v_cmpx_u_f16_e64_dpp -|v255.l|, -|v255.l| clamp dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX12: v_cmpx_u_f16_e64_dpp -|v255.l|, -|v255.l| clamp dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0x7e,0x83,0x88,0xd4,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
 
-v_cmpx_u_f16_e64_dpp -v1, |s2| dpp8:[7,6,5,4,3,2,1,0] fi:1
-// GFX12: v_cmpx_u_f16_e64_dpp -v1, |s2| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7e,0x02,0x88,0xd4,0xea,0x04,0x00,0x20,0x01,0x77,0x39,0x05]
+v_cmpx_u_f16_e64_dpp |v1.h|, -v2.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_cmpx_u_f16_e64_dpp |v1.h|, -v2.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x19,0x88,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
 
-v_cmpx_u_f16_e64_dpp -v1, |2.0| dpp8:[7,6,5,4,3,2,1,0] fi:1
-// GFX12: v_cmpx_u_f16_e64_dpp -v1, |2.0| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7e,0x02,0x88,0xd4,0xea,0xe8,0x01,0x20,0x01,0x77,0x39,0x05]
+v_cmpx_u_f16_e64_dpp -v1.h, |v2.l| dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX12: v_cmpx_u_f16_e64_dpp -v1.h, |v2.l| op_sel:[1,0] dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7e,0x0a,0x88,0xd4,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
 
-v_cmpx_u_f16_e64_dpp -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] fi:0
-// GFX12: v_cmpx_u_f16_e64_dpp -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0x7e,0x83,0x88,0xd4,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+v_cmpx_u_f16_e64_dpp -|v255.l|, -|v255.h| clamp dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX12: v_cmpx_u_f16_e64_dpp -|v255.l|, -|v255.h| op_sel:[0,1] clamp dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0x7e,0x93,0x88,0xd4,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
 
 v_cmpx_u_f32_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0]
 // GFX12: v_cmpx_u_f32_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x98,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
diff --git a/llvm/test/MC/AMDGPU/gfx12_asm_vopc_t16_promote.s b/llvm/test/MC/AMDGPU/gfx12_asm_vopc_t16_promote.s
index 2005f4823b65b..1163aaf3f8270 100644
--- a/llvm/test/MC/AMDGPU/gfx12_asm_vopc_t16_promote.s
+++ b/llvm/test/MC/AMDGPU/gfx12_asm_vopc_t16_promote.s
@@ -5,7 +5,7 @@
 // RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize64,+real-true16 %s 2>&1 > /dev/null | FileCheck --check-prefix=W64-ERR --implicit-check-not=error: %s
 
 v_cmp_class_f16 vcc, v1.h, v255.h
-// W64: v_cmp_class_f16_e64 vcc, v1.h, v255.h   ; encoding: [0x6a,0x18,0x7d,0xd4,0x01,0xff,0x03,0x00]
+// W64: v_cmp_class_f16_e64 vcc, v1.h, v255.h op_sel:[1,1,0] ; encoding: [0x6a,0x18,0x7d,0xd4,0x01,0xff,0x03,0x00]
 // W32-ERR: :[[@LINE-2]]:17: error: invalid operand for instruction
 
 v_cmp_class_f16 vcc, v1.h, v255.h dpp8:[7,6,5,4,3,2,1,0]
@@ -29,7 +29,7 @@ v_cmp_class_f16 vcc, v1.l, v255.l quad_perm:[3,2,1,0]
 // W32-ERR: :[[@LINE-2]]:17: error: invalid operand for instruction
 
 v_cmp_class_f16 vcc, v127.h, v255.h
-// W64: v_cmp_class_f16_e64 vcc, v127.h, v255.h ; encoding: [0x6a,0x18,0x7d,0xd4,0x7f,0xff,0x03,0x00]
+// W64: v_cmp_class_f16_e64 vcc, v127.h, v255.h op_sel:[1,1,0] ; encoding: [0x6a,0x18,0x7d,0xd4,0x7f,0xff,0x03,0x00]
 // W32-ERR: :[[@LINE-2]]:17: error: invalid operand for instruction
 
 v_cmp_class_f16 vcc, v127.h, v255.h dpp8:[7,6,5,4,3,2,1,0]
@@ -53,7 +53,7 @@ v_cmp_class_f16 vcc, v127.l, v255.l quad_perm:[3,2,1,0]
 // W32-ERR: :[[@LINE-2]]:17: error: invalid operand for instruction
 
 v_cmp_class_f16 vcc, v128.h, v2.h
-// W64: v_cmp_class_f16_e64 vcc, v128.h, v2.h   ; encoding: [0x6a,0x18,0x7d,0xd4,0x80,0x05,0x02,0x00]
+// W64: v_cmp_class_f16_e64 vcc, v128.h, v2.h op_sel:[1,1,0] ; encoding: [0x6a,0x18,0x7d,0xd4,0x80,0x05,0x02,0x00]
 // W32-ERR: :[[@LINE-2]]:17: error: invalid operand for instruction
 
 v_cmp_class_f16 vcc, v128.h, v2.h dpp8:[7,6,5,4,3,2,1,0]
@@ -77,7 +77,7 @@ v_cmp_class_f16 vcc, v128.l, v2.l quad_perm:[3,2,1,0]
 // W32-ERR: :[[@LINE-2]]:17: error: invalid operand for instruction
 
 v_cmp_class_f16 vcc, vcc_hi, v255.h
-// W64: v_cmp_class_f16_e64 vcc, vcc_hi, v255.h ; encoding: [0x6a,0x10,0x7d,0xd4,0x6b,0xfe,0x03,0x00]
+// W64: v_cmp_class_f16_e64 vcc, vcc_hi, v255.h op_sel:[0,1,0] ; encoding: [0x6a,0x10,0x7d,0xd4,0x6b,0xfe,0x03,0x00]
 // W32-ERR: :[[@LINE-2]]:17: error: invalid operand for instruction
 
 v_cmp_class_f16 vcc, vcc_hi, v255.l
@@ -85,7 +85,7 @@ v_cmp_class_f16 vcc, vcc_hi, v255.l
 // W32-ERR: :[[@LINE-2]]:17: error: invalid operand for instruction
 
 v_cmp_class_f16 vcc, vcc_lo, v255.h
-// W64: v_cmp_class_f16_e64 vcc, vcc_lo, v255.h ; encoding: [0x6a,0x10,0x7d,0xd4,0x6a,0xfe,0x03,0x00]
+// W64: v_cmp_class_f16_e64 vcc, vcc_lo, v255.h op_sel:[0,1,0] ; encoding: [0x6a,0x10,0x7d,0xd4,0x6a,0xfe,0x03,0x00]
 // W32-ERR: :[[@LINE-2]]:17: error: invalid operand for instruction
 
 v_cmp_class_f16 vcc, vcc_lo, v255.l
@@ -93,7 +93,7 @@ v_cmp_class_f16 vcc, vcc_lo, v255.l
 // W32-ERR: :[[@LINE-2]]:17: error: invalid operand for instruction
 
 v_cmp_class_f16 vcc_lo, v127.h, v255.h
-// W32: v_cmp_class_f16_e64 vcc_lo, v127.h, v255.h ; encoding: [0x6a,0x18,0x7d,0xd4,0x7f,0xff,0x03,0x00]
+// W32: v_cmp_class_f16_e64 vcc_lo, v127.h, v255.h op_sel:[1,1,0] ; encoding: [0x6a,0x18,0x7d,0xd4,0x7f,0xff,0x03,0x00]
 // W64-ERR: :[[@LINE-2]]:17: error: invalid operand for instruction
 
 v_cmp_class_f16 vcc_lo, v127.h, v255.h dpp8:[7,6,5,4,3,2,1,0]
@@ -117,7 +117,7 @@ v_cmp_class_f16 vcc_lo, v127.l, v255.l quad_perm:[3,2,1,0]
 // W64-ERR: :[[@LINE-2]]:17: error: invalid operand for instruction
 
 v_cmp_class_f16 vcc_lo, v128.h, v2.h
-// W32: v_cmp_class_f16_e64 vcc_lo, v128.h, v2.h ; encoding: [0x6a,0x18,0x7d,0xd4,0x80,0x05,0x02,0x00]
+// W32: v_cmp_class_f16_e64 vcc_lo, v128.h, v2.h op_sel:[1,1,0] ; encoding: [0x6a,0x18,0x7d,0xd4,0x80,0x05,0x02,0x00]
 // W64-ERR: :[[@LINE-2]]:17: error: invalid operand for instruction
 
 v_cmp_class_f16 vcc_lo, v128.h, v2.h dpp8:[7,6,5,4,3,2,1,0]
@@ -141,7 +141,7 @@ v_cmp_class_f16 vcc_lo, v128.l, v2.l quad_perm:[3,2,1,0]
 // W64-ERR: :[[@LINE-2]]:17: error: invalid operand for instruction
 
 v_cmp_class_f16 vcc_lo, vcc_hi, v255.h
-// W32: v_cmp_class_f16_e64 vcc_lo, vcc_hi, v255.h ; encoding: [0x6a,0x10,0x7d,0xd4,0x6b,0xfe,0x03,0x00]
+// W32: v_cmp_class_f16_e64 vcc_lo, vcc_hi, v255.h op_sel:[0,1,0] ; encoding: [0x6a,0x10,0x7d,0xd4,0x6b,0xfe,0x03,0x00]
 // W64-ERR: :[[@LINE-2]]:17: error: invalid operand for instruction
 
 v_cmp_class_f16 vcc_lo, vcc_hi, v255.l
@@ -149,7 +149,7 @@ v_cmp_class_f16 vcc_lo, vcc_hi, v255.l
 // W64-ERR: :[[@LINE-2]]:17: error: invalid operand for instruction
 
 v_cmp_class_f16 vcc_lo, vcc_lo, v255.h
-// W32: v_cmp_class_f16_e64 vcc_lo, vcc_lo, v255.h ; encoding: [0x6a,0x10,0x7d,0xd4,0x6a,0xfe,0x03,0x00]
+// W32: v_cmp_class_f16_e64 vcc_lo, vcc_lo, v255.h op_sel:[0,1,0] ; encoding: [0x6a,0x10,0x7d,0xd4,0x6a,0xfe,0x03,0x00]
 // W64-ERR: :[[@LINE-2]]:17: error: invalid operand for instruction
 
 v_cmp_class_f16 vcc_lo, vcc_lo, v255.l
diff --git a/llvm/test/MC/AMDGPU/gfx12_asm_vopcx.s b/llvm/test/MC/AMDGPU/gfx12_asm_vopcx.s
index 4d43b98978eb5..59634ba3cd64a 100644
--- a/llvm/test/MC/AMDGPU/gfx12_asm_vopcx.s
+++ b/llvm/test/MC/AMDGPU/gfx12_asm_vopcx.s
@@ -140,50 +140,62 @@ v_cmpx_class_f64 src_scc, v2
 v_cmpx_class_f64 0xaf123456, v255
 // GFX12: v_cmpx_class_f64_e32 0xaf123456, v255   ; encoding: [0xff,0xfe,0xff,0x7d,0x56,0x34,0x12,0xaf]
 
-v_cmpx_eq_f16 v1, v2
-// GFX12: v_cmpx_eq_f16_e32 v1, v2                ; encoding: [0x01,0x05,0x04,0x7d]
+v_cmpx_eq_f16 v1.l, v2.l
+// GFX12: v_cmpx_eq_f16_e32 v1.l, v2.l            ; encoding: [0x01,0x05,0x04,0x7d]
 
-v_cmpx_eq_f16 v127, v2
-// GFX12: v_cmpx_eq_f16_e32 v127, v2              ; encoding: [0x7f,0x05,0x04,0x7d]
+v_cmpx_eq_f16 v127.l, v2.l
+// GFX12: v_cmpx_eq_f16_e32 v127.l, v2.l          ; encoding: [0x7f,0x05,0x04,0x7d]
 
-v_cmpx_eq_f16 s1, v2
-// GFX12: v_cmpx_eq_f16_e32 s1, v2                ; encoding: [0x01,0x04,0x04,0x7d]
+v_cmpx_eq_f16 s1, v2.l
+// GFX12: v_cmpx_eq_f16_e32 s1, v2.l              ; encoding: [0x01,0x04,0x04,0x7d]
 
-v_cmpx_eq_f16 s105, v2
-// GFX12: v_cmpx_eq_f16_e32 s105, v2              ; encoding: [0x69,0x04,0x04,0x7d]
+v_cmpx_eq_f16 s105, v2.l
+// GFX12: v_cmpx_eq_f16_e32 s105, v2.l            ; encoding: [0x69,0x04,0x04,0x7d]
 
-v_cmpx_eq_f16 vcc_lo, v2
-// GFX12: v_cmpx_eq_f16_e32 vcc_lo, v2            ; encoding: [0x6a,0x04,0x04,0x7d]
+v_cmpx_eq_f16 vcc_lo, v2.l
+// GFX12: v_cmpx_eq_f16_e32 vcc_lo, v2.l          ; encoding: [0x6a,0x04,0x04,0x7d]
 
-v_cmpx_eq_f16 vcc_hi, v2
-// GFX12: v_cmpx_eq_f16_e32 vcc_hi, v2            ; encoding: [0x6b,0x04,0x04,0x7d]
+v_cmpx_eq_f16 vcc_hi, v2.l
+// GFX12: v_cmpx_eq_f16_e32 vcc_hi, v2.l          ; encoding: [0x6b,0x04,0x04,0x7d]
 
-v_cmpx_eq_f16 ttmp15, v2
-// GFX12: v_cmpx_eq_f16_e32 ttmp15, v2            ; encoding: [0x7b,0x04,0x04,0x7d]
+v_cmpx_eq_f16 ttmp15, v2.l
+// GFX12: v_cmpx_eq_f16_e32 ttmp15, v2.l          ; encoding: [0x7b,0x04,0x04,0x7d]
 
-v_cmpx_eq_f16 m0, v2
-// GFX12: v_cmpx_eq_f16_e32 m0, v2                ; encoding: [0x7d,0x04,0x04,0x7d]
+v_cmpx_eq_f16 m0, v2.l
+// GFX12: v_cmpx_eq_f16_e32 m0, v2.l              ; encoding: [0x7d,0x04,0x04,0x7d]
 
-v_cmpx_eq_f16 exec_lo, v2
-// GFX12: v_cmpx_eq_f16_e32 exec_lo, v2           ; encoding: [0x7e,0x04,0x04,0x7d]
+v_cmpx_eq_f16 exec_lo, v2.l
+// GFX12: v_cmpx_eq_f16_e32 exec_lo, v2.l         ; encoding: [0x7e,0x04,0x04,0x7d]
 
-v_cmpx_eq_f16 exec_hi, v2
-// GFX12: v_cmpx_eq_f16_e32 exec_hi, v2           ; encoding: [0x7f,0x04,0x04,0x7d]
+v_cmpx_eq_f16 exec_hi, v2.l
+// GFX12: v_cmpx_eq_f16_e32 exec_hi, v2.l         ; encoding: [0x7f,0x04,0x04,0x7d]
 
-v_cmpx_eq_f16 null, v2
-// GFX12: v_cmpx_eq_f16_e32 null, v2              ; encoding: [0x7c,0x04,0x04,0x7d]
+v_cmpx_eq_f16 null, v2.l
+// GFX12: v_cmpx_eq_f16_e32 null, v2.l            ; encoding: [0x7c,0x04,0x04,0x7d]
 
-v_cmpx_eq_f16 -1, v2
-// GFX12: v_cmpx_eq_f16_e32 -1, v2                ; encoding: [0xc1,0x04,0x04,0x7d]
+v_cmpx_eq_f16 -1, v2.l
+// GFX12: v_cmpx_eq_f16_e32 -1, v2.l              ; encoding: [0xc1,0x04,0x04,0x7d]
 
-v_cmpx_eq_f16 0.5, v2
-// GFX12: v_cmpx_eq_f16_e32 0.5, v2               ; encoding: [0xf0,0x04,0x04,0x7d]
+v_cmpx_eq_f16 0.5, v2.l
+// GFX12: v_cmpx_eq_f16_e32 0.5, v2.l             ; encoding: [0xf0,0x04,0x04,0x7d]
 
-v_cmpx_eq_f16 src_scc, v2
-// GFX12: v_cmpx_eq_f16_e32 src_scc, v2           ; encoding: [0xfd,0x04,0x04,0x7d]
+v_cmpx_eq_f16 src_scc, v2.l
+// GFX12: v_cmpx_eq_f16_e32 src_scc, v2.l         ; encoding: [0xfd,0x04,0x04,0x7d]
 
-v_cmpx_eq_f16 0xfe0b, v127
-// GFX12: v_cmpx_eq_f16_e32 0xfe0b, v127          ; encoding: [0xff,0xfe,0x04,0x7d,0x0b,0xfe,0x00,0x00]
+v_cmpx_eq_f16 0xfe0b, v127.l
+// GFX12: v_cmpx_eq_f16_e32 0xfe0b, v127.l        ; encoding: [0xff,0xfe,0x04,0x7d,0x0b,0xfe,0x00,0x00]
+
+v_cmpx_eq_f16 v1.h, v2.l
+// GFX12: v_cmpx_eq_f16_e32 v1.h, v2.l            ; encoding: [0x81,0x05,0x04,0x7d]
+
+v_cmpx_eq_f16 v127.h, v2.l
+// GFX12: v_cmpx_eq_f16_e32 v127.h, v2.l          ; encoding: [0xff,0x05,0x04,0x7d]
+
+v_cmpx_eq_f16 src_scc, v2.h
+// GFX12: v_cmpx_eq_f16_e32 src_scc, v2.h         ; encoding: [0xfd,0x04,0x05,0x7d]
+
+v_cmpx_eq_f16 0xfe0b, v127.h
+// GFX12: v_cmpx_eq_f16_e32 0xfe0b, v127.h        ; encoding: [0xff,0xfe,0x05,0x7d,0x0b,0xfe,0x00,0x00]
 
 v_cmpx_eq_f32 v1, v2
 // GFX12: v_cmpx_eq_f32_e32 v1, v2                ; encoding: [0x01,0x05,0x24,0x7d]
@@ -542,50 +554,62 @@ v_cmpx_eq_u64 src_scc, v[2:3]
 v_cmpx_eq_u64 0xaf123456, v[254:255]
 // GFX12: v_cmpx_eq_u64_e32 0xaf123456, v[254:255] ; encoding: [0xff,0xfc,0xb5,0x7d,0x56,0x34,0x12,0xaf]
 
-v_cmpx_ge_f16 v1, v2
-// GFX12: v_cmpx_ge_f16_e32 v1, v2                ; encoding: [0x01,0x05,0x0c,0x7d]
+v_cmpx_ge_f16 v1.l, v2.l
+// GFX12: v_cmpx_ge_f16_e32 v1.l, v2.l            ; encoding: [0x01,0x05,0x0c,0x7d]
+
+v_cmpx_ge_f16 v127.l, v2.l
+// GFX12: v_cmpx_ge_f16_e32 v127.l, v2.l          ; encoding: [0x7f,0x05,0x0c,0x7d]
+
+v_cmpx_ge_f16 s1, v2.l
+// GFX12: v_cmpx_ge_f16_e32 s1, v2.l              ; encoding: [0x01,0x04,0x0c,0x7d]
+
+v_cmpx_ge_f16 s105, v2.l
+// GFX12: v_cmpx_ge_f16_e32 s105, v2.l            ; encoding: [0x69,0x04,0x0c,0x7d]
 
-v_cmpx_ge_f16 v127, v2
-// GFX12: v_cmpx_ge_f16_e32 v127, v2              ; encoding: [0x7f,0x05,0x0c,0x7d]
+v_cmpx_ge_f16 vcc_lo, v2.l
+// GFX12: v_cmpx_ge_f16_e32 vcc_lo, v2.l          ; encoding: [0x6a,0x04,0x0c,0x7d]
 
-v_cmpx_ge_f16 s1, v2
-// GFX12: v_cmpx_ge_f16_e32 s1, v2                ; encoding: [0x01,0x04,0x0c,0x7d]
+v_cmpx_ge_f16 vcc_hi, v2.l
+// GFX12: v_cmpx_ge_f16_e32 vcc_hi, v2.l          ; encoding: [0x6b,0x04,0x0c,0x7d]
 
-v_cmpx_ge_f16 s105, v2
-// GFX12: v_cmpx_ge_f16_e32 s105, v2              ; encoding: [0x69,0x04,0x0c,0x7d]
+v_cmpx_ge_f16 ttmp15, v2.l
+// GFX12: v_cmpx_ge_f16_e32 ttmp15, v2.l          ; encoding: [0x7b,0x04,0x0c,0x7d]
 
-v_cmpx_ge_f16 vcc_lo, v2
-// GFX12: v_cmpx_ge_f16_e32 vcc_lo, v2            ; encoding: [0x6a,0x04,0x0c,0x7d]
+v_cmpx_ge_f16 m0, v2.l
+// GFX12: v_cmpx_ge_f16_e32 m0, v2.l              ; encoding: [0x7d,0x04,0x0c,0x7d]
 
-v_cmpx_ge_f16 vcc_hi, v2
-// GFX12: v_cmpx_ge_f16_e32 vcc_hi, v2            ; encoding: [0x6b,0x04,0x0c,0x7d]
+v_cmpx_ge_f16 exec_lo, v2.l
+// GFX12: v_cmpx_ge_f16_e32 exec_lo, v2.l         ; encoding: [0x7e,0x04,0x0c,0x7d]
 
-v_cmpx_ge_f16 ttmp15, v2
-// GFX12: v_cmpx_ge_f16_e32 ttmp15, v2            ; encoding: [0x7b,0x04,0x0c,0x7d]
+v_cmpx_ge_f16 exec_hi, v2.l
+// GFX12: v_cmpx_ge_f16_e32 exec_hi, v2.l         ; encoding: [0x7f,0x04,0x0c,0x7d]
 
-v_cmpx_ge_f16 m0, v2
-// GFX12: v_cmpx_ge_f16_e32 m0, v2                ; encoding: [0x7d,0x04,0x0c,0x7d]
+v_cmpx_ge_f16 null, v2.l
+// GFX12: v_cmpx_ge_f16_e32 null, v2.l            ; encoding: [0x7c,0x04,0x0c,0x7d]
 
-v_cmpx_ge_f16 exec_lo, v2
-// GFX12: v_cmpx_ge_f16_e32 exec_lo, v2           ; encoding: [0x7e,0x04,0x0c,0x7d]
+v_cmpx_ge_f16 -1, v2.l
+// GFX12: v_cmpx_ge_f16_e32 -1, v2.l              ; encoding: [0xc1,0x04,0x0c,0x7d]
 
-v_cmpx_ge_f16 exec_hi, v2
-// GFX12: v_cmpx_ge_f16_e32 exec_hi, v2           ; encoding: [0x7f,0x04,0x0c,0x7d]
+v_cmpx_ge_f16 0.5, v2.l
+// GFX12: v_cmpx_ge_f16_e32 0.5, v2.l             ; encoding: [0xf0,0x04,0x0c,0x7d]
 
-v_cmpx_ge_f16 null, v2
-// GFX12: v_cmpx_ge_f16_e32 null, v2              ; encoding: [0x7c,0x04,0x0c,0x7d]
+v_cmpx_ge_f16 src_scc, v2.l
+// GFX12: v_cmpx_ge_f16_e32 src_scc, v2.l         ; encoding: [0xfd,0x04,0x0c,0x7d]
 
-v_cmpx_ge_f16 -1, v2
-// GFX12: v_cmpx_ge_f16_e32 -1, v2                ; encoding: [0xc1,0x04,0x0c,0x7d]
+v_cmpx_ge_f16 0xfe0b, v127.l
+// GFX12: v_cmpx_ge_f16_e32 0xfe0b, v127.l        ; encoding: [0xff,0xfe,0x0c,0x7d,0x0b,0xfe,0x00,0x00]
 
-v_cmpx_ge_f16 0.5, v2
-// GFX12: v_cmpx_ge_f16_e32 0.5, v2               ; encoding: [0xf0,0x04,0x0c,0x7d]
+v_cmpx_ge_f16 v1.h, v2.l
+// GFX12: v_cmpx_ge_f16_e32 v1.h, v2.l            ; encoding: [0x81,0x05,0x0c,0x7d]
 
-v_cmpx_ge_f16 src_scc, v2
-// GFX12: v_cmpx_ge_f16_e32 src_scc, v2           ; encoding: [0xfd,0x04,0x0c,0x7d]
+v_cmpx_ge_f16 v127.h, v2.l
+// GFX12: v_cmpx_ge_f16_e32 v127.h, v2.l          ; encoding: [0xff,0x05,0x0c,0x7d]
 
-v_cmpx_ge_f16 0xfe0b, v127
-// GFX12: v_cmpx_ge_f16_e32 0xfe0b, v127          ; encoding: [0xff,0xfe,0x0c,0x7d,0x0b,0xfe,0x00,0x00]
+v_cmpx_ge_f16 src_scc, v2.h
+// GFX12: v_cmpx_ge_f16_e32 src_scc, v2.h         ; encoding: [0xfd,0x04,0x0d,0x7d]
+
+v_cmpx_ge_f16 0xfe0b, v127.h
+// GFX12: v_cmpx_ge_f16_e32 0xfe0b, v127.h        ; encoding: [0xff,0xfe,0x0d,0x7d,0x0b,0xfe,0x00,0x00]
 
 v_cmpx_ge_f32 v1, v2
 // GFX12: v_cmpx_ge_f32_e32 v1, v2                ; encoding: [0x01,0x05,0x2c,0x7d]
@@ -944,50 +968,62 @@ v_cmpx_ge_u64 src_scc, v[2:3]
 v_cmpx_ge_u64 0xaf123456, v[254:255]
 // GFX12: v_cmpx_ge_u64_e32 0xaf123456, v[254:255] ; encoding: [0xff,0xfc,0xbd,0x7d,0x56,0x34,0x12,0xaf]
 
-v_cmpx_gt_f16 v1, v2
-// GFX12: v_cmpx_gt_f16_e32 v1, v2                ; encoding: [0x01,0x05,0x08,0x7d]
+v_cmpx_gt_f16 v1.l, v2.l
+// GFX12: v_cmpx_gt_f16_e32 v1.l, v2.l            ; encoding: [0x01,0x05,0x08,0x7d]
+
+v_cmpx_gt_f16 v127.l, v2.l
+// GFX12: v_cmpx_gt_f16_e32 v127.l, v2.l          ; encoding: [0x7f,0x05,0x08,0x7d]
+
+v_cmpx_gt_f16 s1, v2.l
+// GFX12: v_cmpx_gt_f16_e32 s1, v2.l              ; encoding: [0x01,0x04,0x08,0x7d]
 
-v_cmpx_gt_f16 v127, v2
-// GFX12: v_cmpx_gt_f16_e32 v127, v2              ; encoding: [0x7f,0x05,0x08,0x7d]
+v_cmpx_gt_f16 s105, v2.l
+// GFX12: v_cmpx_gt_f16_e32 s105, v2.l            ; encoding: [0x69,0x04,0x08,0x7d]
 
-v_cmpx_gt_f16 s1, v2
-// GFX12: v_cmpx_gt_f16_e32 s1, v2                ; encoding: [0x01,0x04,0x08,0x7d]
+v_cmpx_gt_f16 vcc_lo, v2.l
+// GFX12: v_cmpx_gt_f16_e32 vcc_lo, v2.l          ; encoding: [0x6a,0x04,0x08,0x7d]
 
-v_cmpx_gt_f16 s105, v2
-// GFX12: v_cmpx_gt_f16_e32 s105, v2              ; encoding: [0x69,0x04,0x08,0x7d]
+v_cmpx_gt_f16 vcc_hi, v2.l
+// GFX12: v_cmpx_gt_f16_e32 vcc_hi, v2.l          ; encoding: [0x6b,0x04,0x08,0x7d]
 
-v_cmpx_gt_f16 vcc_lo, v2
-// GFX12: v_cmpx_gt_f16_e32 vcc_lo, v2            ; encoding: [0x6a,0x04,0x08,0x7d]
+v_cmpx_gt_f16 ttmp15, v2.l
+// GFX12: v_cmpx_gt_f16_e32 ttmp15, v2.l          ; encoding: [0x7b,0x04,0x08,0x7d]
 
-v_cmpx_gt_f16 vcc_hi, v2
-// GFX12: v_cmpx_gt_f16_e32 vcc_hi, v2            ; encoding: [0x6b,0x04,0x08,0x7d]
+v_cmpx_gt_f16 m0, v2.l
+// GFX12: v_cmpx_gt_f16_e32 m0, v2.l              ; encoding: [0x7d,0x04,0x08,0x7d]
 
-v_cmpx_gt_f16 ttmp15, v2
-// GFX12: v_cmpx_gt_f16_e32 ttmp15, v2            ; encoding: [0x7b,0x04,0x08,0x7d]
+v_cmpx_gt_f16 exec_lo, v2.l
+// GFX12: v_cmpx_gt_f16_e32 exec_lo, v2.l         ; encoding: [0x7e,0x04,0x08,0x7d]
 
-v_cmpx_gt_f16 m0, v2
-// GFX12: v_cmpx_gt_f16_e32 m0, v2                ; encoding: [0x7d,0x04,0x08,0x7d]
+v_cmpx_gt_f16 exec_hi, v2.l
+// GFX12: v_cmpx_gt_f16_e32 exec_hi, v2.l         ; encoding: [0x7f,0x04,0x08,0x7d]
 
-v_cmpx_gt_f16 exec_lo, v2
-// GFX12: v_cmpx_gt_f16_e32 exec_lo, v2           ; encoding: [0x7e,0x04,0x08,0x7d]
+v_cmpx_gt_f16 null, v2.l
+// GFX12: v_cmpx_gt_f16_e32 null, v2.l            ; encoding: [0x7c,0x04,0x08,0x7d]
 
-v_cmpx_gt_f16 exec_hi, v2
-// GFX12: v_cmpx_gt_f16_e32 exec_hi, v2           ; encoding: [0x7f,0x04,0x08,0x7d]
+v_cmpx_gt_f16 -1, v2.l
+// GFX12: v_cmpx_gt_f16_e32 -1, v2.l              ; encoding: [0xc1,0x04,0x08,0x7d]
 
-v_cmpx_gt_f16 null, v2
-// GFX12: v_cmpx_gt_f16_e32 null, v2              ; encoding: [0x7c,0x04,0x08,0x7d]
+v_cmpx_gt_f16 0.5, v2.l
+// GFX12: v_cmpx_gt_f16_e32 0.5, v2.l             ; encoding: [0xf0,0x04,0x08,0x7d]
 
-v_cmpx_gt_f16 -1, v2
-// GFX12: v_cmpx_gt_f16_e32 -1, v2                ; encoding: [0xc1,0x04,0x08,0x7d]
+v_cmpx_gt_f16 src_scc, v2.l
+// GFX12: v_cmpx_gt_f16_e32 src_scc, v2.l         ; encoding: [0xfd,0x04,0x08,0x7d]
 
-v_cmpx_gt_f16 0.5, v2
-// GFX12: v_cmpx_gt_f16_e32 0.5, v2               ; encoding: [0xf0,0x04,0x08,0x7d]
+v_cmpx_gt_f16 0xfe0b, v127.l
+// GFX12: v_cmpx_gt_f16_e32 0xfe0b, v127.l        ; encoding: [0xff,0xfe,0x08,0x7d,0x0b,0xfe,0x00,0x00]
 
-v_cmpx_gt_f16 src_scc, v2
-// GFX12: v_cmpx_gt_f16_e32 src_scc, v2           ; encoding: [0xfd,0x04,0x08,0x7d]
+v_cmpx_gt_f16 v1.h, v2.l
+// GFX12: v_cmpx_gt_f16_e32 v1.h, v2.l            ; encoding: [0x81,0x05,0x08,0x7d]
 
-v_cmpx_gt_f16 0xfe0b, v127
-// GFX12: v_cmpx_gt_f16_e32 0xfe0b, v127          ; encoding: [0xff,0xfe,0x08,0x7d,0x0b,0xfe,0x00,0x00]
+v_cmpx_gt_f16 v127.h, v2.l
+// GFX12: v_cmpx_gt_f16_e32 v127.h, v2.l          ; encoding: [0xff,0x05,0x08,0x7d]
+
+v_cmpx_gt_f16 src_scc, v2.h
+// GFX12: v_cmpx_gt_f16_e32 src_scc, v2.h         ; encoding: [0xfd,0x04,0x09,0x7d]
+
+v_cmpx_gt_f16 0xfe0b, v127.h
+// GFX12: v_cmpx_gt_f16_e32 0xfe0b, v127.h        ; encoding: [0xff,0xfe,0x09,0x7d,0x0b,0xfe,0x00,0x00]
 
 v_cmpx_gt_f32 v1, v2
 // GFX12: v_cmpx_gt_f32_e32 v1, v2                ; encoding: [0x01,0x05,0x28,0x7d]
@@ -1346,50 +1382,62 @@ v_cmpx_gt_u64 src_scc, v[2:3]
 v_cmpx_gt_u64 0xaf123456, v[254:255]
 // GFX12: v_cmpx_gt_u64_e32 0xaf123456, v[254:255] ; encoding: [0xff,0xfc,0xb9,0x7d,0x56,0x34,0x12,0xaf]
 
-v_cmpx_le_f16 v1, v2
-// GFX12: v_cmpx_le_f16_e32 v1, v2                ; encoding: [0x01,0x05,0x06,0x7d]
+v_cmpx_le_f16 v1.l, v2.l
+// GFX12: v_cmpx_le_f16_e32 v1.l, v2.l            ; encoding: [0x01,0x05,0x06,0x7d]
+
+v_cmpx_le_f16 v127.l, v2.l
+// GFX12: v_cmpx_le_f16_e32 v127.l, v2.l          ; encoding: [0x7f,0x05,0x06,0x7d]
 
-v_cmpx_le_f16 v127, v2
-// GFX12: v_cmpx_le_f16_e32 v127, v2              ; encoding: [0x7f,0x05,0x06,0x7d]
+v_cmpx_le_f16 s1, v2.l
+// GFX12: v_cmpx_le_f16_e32 s1, v2.l              ; encoding: [0x01,0x04,0x06,0x7d]
 
-v_cmpx_le_f16 s1, v2
-// GFX12: v_cmpx_le_f16_e32 s1, v2                ; encoding: [0x01,0x04,0x06,0x7d]
+v_cmpx_le_f16 s105, v2.l
+// GFX12: v_cmpx_le_f16_e32 s105, v2.l            ; encoding: [0x69,0x04,0x06,0x7d]
 
-v_cmpx_le_f16 s105, v2
-// GFX12: v_cmpx_le_f16_e32 s105, v2              ; encoding: [0x69,0x04,0x06,0x7d]
+v_cmpx_le_f16 vcc_lo, v2.l
+// GFX12: v_cmpx_le_f16_e32 vcc_lo, v2.l          ; encoding: [0x6a,0x04,0x06,0x7d]
 
-v_cmpx_le_f16 vcc_lo, v2
-// GFX12: v_cmpx_le_f16_e32 vcc_lo, v2            ; encoding: [0x6a,0x04,0x06,0x7d]
+v_cmpx_le_f16 vcc_hi, v2.l
+// GFX12: v_cmpx_le_f16_e32 vcc_hi, v2.l          ; encoding: [0x6b,0x04,0x06,0x7d]
 
-v_cmpx_le_f16 vcc_hi, v2
-// GFX12: v_cmpx_le_f16_e32 vcc_hi, v2            ; encoding: [0x6b,0x04,0x06,0x7d]
+v_cmpx_le_f16 ttmp15, v2.l
+// GFX12: v_cmpx_le_f16_e32 ttmp15, v2.l          ; encoding: [0x7b,0x04,0x06,0x7d]
 
-v_cmpx_le_f16 ttmp15, v2
-// GFX12: v_cmpx_le_f16_e32 ttmp15, v2            ; encoding: [0x7b,0x04,0x06,0x7d]
+v_cmpx_le_f16 m0, v2.l
+// GFX12: v_cmpx_le_f16_e32 m0, v2.l              ; encoding: [0x7d,0x04,0x06,0x7d]
 
-v_cmpx_le_f16 m0, v2
-// GFX12: v_cmpx_le_f16_e32 m0, v2                ; encoding: [0x7d,0x04,0x06,0x7d]
+v_cmpx_le_f16 exec_lo, v2.l
+// GFX12: v_cmpx_le_f16_e32 exec_lo, v2.l         ; encoding: [0x7e,0x04,0x06,0x7d]
 
-v_cmpx_le_f16 exec_lo, v2
-// GFX12: v_cmpx_le_f16_e32 exec_lo, v2           ; encoding: [0x7e,0x04,0x06,0x7d]
+v_cmpx_le_f16 exec_hi, v2.l
+// GFX12: v_cmpx_le_f16_e32 exec_hi, v2.l         ; encoding: [0x7f,0x04,0x06,0x7d]
 
-v_cmpx_le_f16 exec_hi, v2
-// GFX12: v_cmpx_le_f16_e32 exec_hi, v2           ; encoding: [0x7f,0x04,0x06,0x7d]
+v_cmpx_le_f16 null, v2.l
+// GFX12: v_cmpx_le_f16_e32 null, v2.l            ; encoding: [0x7c,0x04,0x06,0x7d]
 
-v_cmpx_le_f16 null, v2
-// GFX12: v_cmpx_le_f16_e32 null, v2              ; encoding: [0x7c,0x04,0x06,0x7d]
+v_cmpx_le_f16 -1, v2.l
+// GFX12: v_cmpx_le_f16_e32 -1, v2.l              ; encoding: [0xc1,0x04,0x06,0x7d]
 
-v_cmpx_le_f16 -1, v2
-// GFX12: v_cmpx_le_f16_e32 -1, v2                ; encoding: [0xc1,0x04,0x06,0x7d]
+v_cmpx_le_f16 0.5, v2.l
+// GFX12: v_cmpx_le_f16_e32 0.5, v2.l             ; encoding: [0xf0,0x04,0x06,0x7d]
 
-v_cmpx_le_f16 0.5, v2
-// GFX12: v_cmpx_le_f16_e32 0.5, v2               ; encoding: [0xf0,0x04,0x06,0x7d]
+v_cmpx_le_f16 src_scc, v2.l
+// GFX12: v_cmpx_le_f16_e32 src_scc, v2.l         ; encoding: [0xfd,0x04,0x06,0x7d]
 
-v_cmpx_le_f16 src_scc, v2
-// GFX12: v_cmpx_le_f16_e32 src_scc, v2           ; encoding: [0xfd,0x04,0x06,0x7d]
+v_cmpx_le_f16 0xfe0b, v127.l
+// GFX12: v_cmpx_le_f16_e32 0xfe0b, v127.l        ; encoding: [0xff,0xfe,0x06,0x7d,0x0b,0xfe,0x00,0x00]
 
-v_cmpx_le_f16 0xfe0b, v127
-// GFX12: v_cmpx_le_f16_e32 0xfe0b, v127          ; encoding: [0xff,0xfe,0x06,0x7d,0x0b,0xfe,0x00,0x00]
+v_cmpx_le_f16 v1.h, v2.l
+// GFX12: v_cmpx_le_f16_e32 v1.h, v2.l            ; encoding: [0x81,0x05,0x06,0x7d]
+
+v_cmpx_le_f16 v127.h, v2.l
+// GFX12: v_cmpx_le_f16_e32 v127.h, v2.l          ; encoding: [0xff,0x05,0x06,0x7d]
+
+v_cmpx_le_f16 src_scc, v2.h
+// GFX12: v_cmpx_le_f16_e32 src_scc, v2.h         ; encoding: [0xfd,0x04,0x07,0x7d]
+
+v_cmpx_le_f16 0xfe0b, v127.h
+// GFX12: v_cmpx_le_f16_e32 0xfe0b, v127.h        ; encoding: [0xff,0xfe,0x07,0x7d,0x0b,0xfe,0x00,0x00]
 
 v_cmpx_le_f32 v1, v2
 // GFX12: v_cmpx_le_f32_e32 v1, v2                ; encoding: [0x01,0x05,0x26,0x7d]
@@ -1748,50 +1796,62 @@ v_cmpx_le_u64 src_scc, v[2:3]
 v_cmpx_le_u64 0xaf123456, v[254:255]
 // GFX12: v_cmpx_le_u64_e32 0xaf123456, v[254:255] ; encoding: [0xff,0xfc,0xb7,0x7d,0x56,0x34,0x12,0xaf]
 
-v_cmpx_lg_f16 v1, v2
-// GFX12: v_cmpx_lg_f16_e32 v1, v2                ; encoding: [0x01,0x05,0x0a,0x7d]
+v_cmpx_lg_f16 v1.l, v2.l
+// GFX12: v_cmpx_lg_f16_e32 v1.l, v2.l            ; encoding: [0x01,0x05,0x0a,0x7d]
+
+v_cmpx_lg_f16 v127.l, v2.l
+// GFX12: v_cmpx_lg_f16_e32 v127.l, v2.l          ; encoding: [0x7f,0x05,0x0a,0x7d]
+
+v_cmpx_lg_f16 s1, v2.l
+// GFX12: v_cmpx_lg_f16_e32 s1, v2.l              ; encoding: [0x01,0x04,0x0a,0x7d]
+
+v_cmpx_lg_f16 s105, v2.l
+// GFX12: v_cmpx_lg_f16_e32 s105, v2.l            ; encoding: [0x69,0x04,0x0a,0x7d]
+
+v_cmpx_lg_f16 vcc_lo, v2.l
+// GFX12: v_cmpx_lg_f16_e32 vcc_lo, v2.l          ; encoding: [0x6a,0x04,0x0a,0x7d]
 
-v_cmpx_lg_f16 v127, v2
-// GFX12: v_cmpx_lg_f16_e32 v127, v2              ; encoding: [0x7f,0x05,0x0a,0x7d]
+v_cmpx_lg_f16 vcc_hi, v2.l
+// GFX12: v_cmpx_lg_f16_e32 vcc_hi, v2.l          ; encoding: [0x6b,0x04,0x0a,0x7d]
 
-v_cmpx_lg_f16 s1, v2
-// GFX12: v_cmpx_lg_f16_e32 s1, v2                ; encoding: [0x01,0x04,0x0a,0x7d]
+v_cmpx_lg_f16 ttmp15, v2.l
+// GFX12: v_cmpx_lg_f16_e32 ttmp15, v2.l          ; encoding: [0x7b,0x04,0x0a,0x7d]
 
-v_cmpx_lg_f16 s105, v2
-// GFX12: v_cmpx_lg_f16_e32 s105, v2              ; encoding: [0x69,0x04,0x0a,0x7d]
+v_cmpx_lg_f16 m0, v2.l
+// GFX12: v_cmpx_lg_f16_e32 m0, v2.l              ; encoding: [0x7d,0x04,0x0a,0x7d]
 
-v_cmpx_lg_f16 vcc_lo, v2
-// GFX12: v_cmpx_lg_f16_e32 vcc_lo, v2            ; encoding: [0x6a,0x04,0x0a,0x7d]
+v_cmpx_lg_f16 exec_lo, v2.l
+// GFX12: v_cmpx_lg_f16_e32 exec_lo, v2.l         ; encoding: [0x7e,0x04,0x0a,0x7d]
 
-v_cmpx_lg_f16 vcc_hi, v2
-// GFX12: v_cmpx_lg_f16_e32 vcc_hi, v2            ; encoding: [0x6b,0x04,0x0a,0x7d]
+v_cmpx_lg_f16 exec_hi, v2.l
+// GFX12: v_cmpx_lg_f16_e32 exec_hi, v2.l         ; encoding: [0x7f,0x04,0x0a,0x7d]
 
-v_cmpx_lg_f16 ttmp15, v2
-// GFX12: v_cmpx_lg_f16_e32 ttmp15, v2            ; encoding: [0x7b,0x04,0x0a,0x7d]
+v_cmpx_lg_f16 null, v2.l
+// GFX12: v_cmpx_lg_f16_e32 null, v2.l            ; encoding: [0x7c,0x04,0x0a,0x7d]
 
-v_cmpx_lg_f16 m0, v2
-// GFX12: v_cmpx_lg_f16_e32 m0, v2                ; encoding: [0x7d,0x04,0x0a,0x7d]
+v_cmpx_lg_f16 -1, v2.l
+// GFX12: v_cmpx_lg_f16_e32 -1, v2.l              ; encoding: [0xc1,0x04,0x0a,0x7d]
 
-v_cmpx_lg_f16 exec_lo, v2
-// GFX12: v_cmpx_lg_f16_e32 exec_lo, v2           ; encoding: [0x7e,0x04,0x0a,0x7d]
+v_cmpx_lg_f16 0.5, v2.l
+// GFX12: v_cmpx_lg_f16_e32 0.5, v2.l             ; encoding: [0xf0,0x04,0x0a,0x7d]
 
-v_cmpx_lg_f16 exec_hi, v2
-// GFX12: v_cmpx_lg_f16_e32 exec_hi, v2           ; encoding: [0x7f,0x04,0x0a,0x7d]
+v_cmpx_lg_f16 src_scc, v2.l
+// GFX12: v_cmpx_lg_f16_e32 src_scc, v2.l         ; encoding: [0xfd,0x04,0x0a,0x7d]
 
-v_cmpx_lg_f16 null, v2
-// GFX12: v_cmpx_lg_f16_e32 null, v2              ; encoding: [0x7c,0x04,0x0a,0x7d]
+v_cmpx_lg_f16 0xfe0b, v127.l
+// GFX12: v_cmpx_lg_f16_e32 0xfe0b, v127.l        ; encoding: [0xff,0xfe,0x0a,0x7d,0x0b,0xfe,0x00,0x00]
 
-v_cmpx_lg_f16 -1, v2
-// GFX12: v_cmpx_lg_f16_e32 -1, v2                ; encoding: [0xc1,0x04,0x0a,0x7d]
+v_cmpx_lg_f16 v1.h, v2.l
+// GFX12: v_cmpx_lg_f16_e32 v1.h, v2.l            ; encoding: [0x81,0x05,0x0a,0x7d]
 
-v_cmpx_lg_f16 0.5, v2
-// GFX12: v_cmpx_lg_f16_e32 0.5, v2               ; encoding: [0xf0,0x04,0x0a,0x7d]
+v_cmpx_lg_f16 v127.h, v2.l
+// GFX12: v_cmpx_lg_f16_e32 v127.h, v2.l          ; encoding: [0xff,0x05,0x0a,0x7d]
 
-v_cmpx_lg_f16 src_scc, v2
-// GFX12: v_cmpx_lg_f16_e32 src_scc, v2           ; encoding: [0xfd,0x04,0x0a,0x7d]
+v_cmpx_lg_f16 src_scc, v2.h
+// GFX12: v_cmpx_lg_f16_e32 src_scc, v2.h         ; encoding: [0xfd,0x04,0x0b,0x7d]
 
-v_cmpx_lg_f16 0xfe0b, v127
-// GFX12: v_cmpx_lg_f16_e32 0xfe0b, v127          ; encoding: [0xff,0xfe,0x0a,0x7d,0x0b,0xfe,0x00,0x00]
+v_cmpx_lg_f16 0xfe0b, v127.h
+// GFX12: v_cmpx_lg_f16_e32 0xfe0b, v127.h        ; encoding: [0xff,0xfe,0x0b,0x7d,0x0b,0xfe,0x00,0x00]
 
 v_cmpx_lg_f32 v1, v2
 // GFX12: v_cmpx_lg_f32_e32 v1, v2                ; encoding: [0x01,0x05,0x2a,0x7d]
@@ -2564,50 +2624,62 @@ v_cmpx_ne_u64 src_scc, v[2:3]
 v_cmpx_ne_u64 0xaf123456, v[254:255]
 // GFX12: v_cmpx_ne_u64_e32 0xaf123456, v[254:255] ; encoding: [0xff,0xfc,0xbb,0x7d,0x56,0x34,0x12,0xaf]
 
-v_cmpx_neq_f16 v1, v2
-// GFX12: v_cmpx_neq_f16_e32 v1, v2               ; encoding: [0x01,0x05,0x1a,0x7d]
+v_cmpx_neq_f16 v1.l, v2.l
+// GFX12: v_cmpx_neq_f16_e32 v1.l, v2.l           ; encoding: [0x01,0x05,0x1a,0x7d]
 
-v_cmpx_neq_f16 v127, v2
-// GFX12: v_cmpx_neq_f16_e32 v127, v2             ; encoding: [0x7f,0x05,0x1a,0x7d]
+v_cmpx_neq_f16 v127.l, v2.l
+// GFX12: v_cmpx_neq_f16_e32 v127.l, v2.l         ; encoding: [0x7f,0x05,0x1a,0x7d]
 
-v_cmpx_neq_f16 s1, v2
-// GFX12: v_cmpx_neq_f16_e32 s1, v2               ; encoding: [0x01,0x04,0x1a,0x7d]
+v_cmpx_neq_f16 s1, v2.l
+// GFX12: v_cmpx_neq_f16_e32 s1, v2.l             ; encoding: [0x01,0x04,0x1a,0x7d]
 
-v_cmpx_neq_f16 s105, v2
-// GFX12: v_cmpx_neq_f16_e32 s105, v2             ; encoding: [0x69,0x04,0x1a,0x7d]
+v_cmpx_neq_f16 s105, v2.l
+// GFX12: v_cmpx_neq_f16_e32 s105, v2.l           ; encoding: [0x69,0x04,0x1a,0x7d]
 
-v_cmpx_neq_f16 vcc_lo, v2
-// GFX12: v_cmpx_neq_f16_e32 vcc_lo, v2           ; encoding: [0x6a,0x04,0x1a,0x7d]
+v_cmpx_neq_f16 vcc_lo, v2.l
+// GFX12: v_cmpx_neq_f16_e32 vcc_lo, v2.l         ; encoding: [0x6a,0x04,0x1a,0x7d]
 
-v_cmpx_neq_f16 vcc_hi, v2
-// GFX12: v_cmpx_neq_f16_e32 vcc_hi, v2           ; encoding: [0x6b,0x04,0x1a,0x7d]
+v_cmpx_neq_f16 vcc_hi, v2.l
+// GFX12: v_cmpx_neq_f16_e32 vcc_hi, v2.l         ; encoding: [0x6b,0x04,0x1a,0x7d]
 
-v_cmpx_neq_f16 ttmp15, v2
-// GFX12: v_cmpx_neq_f16_e32 ttmp15, v2           ; encoding: [0x7b,0x04,0x1a,0x7d]
+v_cmpx_neq_f16 ttmp15, v2.l
+// GFX12: v_cmpx_neq_f16_e32 ttmp15, v2.l         ; encoding: [0x7b,0x04,0x1a,0x7d]
 
-v_cmpx_neq_f16 m0, v2
-// GFX12: v_cmpx_neq_f16_e32 m0, v2               ; encoding: [0x7d,0x04,0x1a,0x7d]
+v_cmpx_neq_f16 m0, v2.l
+// GFX12: v_cmpx_neq_f16_e32 m0, v2.l             ; encoding: [0x7d,0x04,0x1a,0x7d]
 
-v_cmpx_neq_f16 exec_lo, v2
-// GFX12: v_cmpx_neq_f16_e32 exec_lo, v2          ; encoding: [0x7e,0x04,0x1a,0x7d]
+v_cmpx_neq_f16 exec_lo, v2.l
+// GFX12: v_cmpx_neq_f16_e32 exec_lo, v2.l        ; encoding: [0x7e,0x04,0x1a,0x7d]
 
-v_cmpx_neq_f16 exec_hi, v2
-// GFX12: v_cmpx_neq_f16_e32 exec_hi, v2          ; encoding: [0x7f,0x04,0x1a,0x7d]
+v_cmpx_neq_f16 exec_hi, v2.l
+// GFX12: v_cmpx_neq_f16_e32 exec_hi, v2.l        ; encoding: [0x7f,0x04,0x1a,0x7d]
 
-v_cmpx_neq_f16 null, v2
-// GFX12: v_cmpx_neq_f16_e32 null, v2             ; encoding: [0x7c,0x04,0x1a,0x7d]
+v_cmpx_neq_f16 null, v2.l
+// GFX12: v_cmpx_neq_f16_e32 null, v2.l           ; encoding: [0x7c,0x04,0x1a,0x7d]
 
-v_cmpx_neq_f16 -1, v2
-// GFX12: v_cmpx_neq_f16_e32 -1, v2               ; encoding: [0xc1,0x04,0x1a,0x7d]
+v_cmpx_neq_f16 -1, v2.l
+// GFX12: v_cmpx_neq_f16_e32 -1, v2.l             ; encoding: [0xc1,0x04,0x1a,0x7d]
 
-v_cmpx_neq_f16 0.5, v2
-// GFX12: v_cmpx_neq_f16_e32 0.5, v2              ; encoding: [0xf0,0x04,0x1a,0x7d]
+v_cmpx_neq_f16 0.5, v2.l
+// GFX12: v_cmpx_neq_f16_e32 0.5, v2.l            ; encoding: [0xf0,0x04,0x1a,0x7d]
 
-v_cmpx_neq_f16 src_scc, v2
-// GFX12: v_cmpx_neq_f16_e32 src_scc, v2          ; encoding: [0xfd,0x04,0x1a,0x7d]
+v_cmpx_neq_f16 src_scc, v2.l
+// GFX12: v_cmpx_neq_f16_e32 src_scc, v2.l        ; encoding: [0xfd,0x04,0x1a,0x7d]
 
-v_cmpx_neq_f16 0xfe0b, v127
-// GFX12: v_cmpx_neq_f16_e32 0xfe0b, v127         ; encoding: [0xff,0xfe,0x1a,0x7d,0x0b,0xfe,0x00,0x00]
+v_cmpx_neq_f16 0xfe0b, v127.l
+// GFX12: v_cmpx_neq_f16_e32 0xfe0b, v127.l       ; encoding: [0xff,0xfe,0x1a,0x7d,0x0b,0xfe,0x00,0x00]
+
+v_cmpx_neq_f16 v1.h, v2.l
+// GFX12: v_cmpx_neq_f16_e32 v1.h, v2.l           ; encoding: [0x81,0x05,0x1a,0x7d]
+
+v_cmpx_neq_f16 v127.h, v2.l
+// GFX12: v_cmpx_neq_f16_e32 v127.h, v2.l         ; encoding: [0xff,0x05,0x1a,0x7d]
+
+v_cmpx_neq_f16 src_scc, v2.h
+// GFX12: v_cmpx_neq_f16_e32 src_scc, v2.h        ; encoding: [0xfd,0x04,0x1b,0x7d]
+
+v_cmpx_neq_f16 0xfe0b, v127.h
+// GFX12: v_cmpx_neq_f16_e32 0xfe0b, v127.h       ; encoding: [0xff,0xfe,0x1b,0x7d,0x0b,0xfe,0x00,0x00]
 
 v_cmpx_neq_f32 v1, v2
 // GFX12: v_cmpx_neq_f32_e32 v1, v2               ; encoding: [0x01,0x05,0x3a,0x7d]
@@ -2690,50 +2762,62 @@ v_cmpx_neq_f64 src_scc, v[2:3]
 v_cmpx_neq_f64 0xaf123456, v[254:255]
 // GFX12: v_cmpx_neq_f64_e32 0xaf123456, v[254:255] ; encoding: [0xff,0xfc,0x5b,0x7d,0x56,0x34,0x12,0xaf]
 
-v_cmpx_nge_f16 v1, v2
-// GFX12: v_cmpx_nge_f16_e32 v1, v2               ; encoding: [0x01,0x05,0x12,0x7d]
+v_cmpx_nge_f16 v1.l, v2.l
+// GFX12: v_cmpx_nge_f16_e32 v1.l, v2.l           ; encoding: [0x01,0x05,0x12,0x7d]
+
+v_cmpx_nge_f16 v127.l, v2.l
+// GFX12: v_cmpx_nge_f16_e32 v127.l, v2.l         ; encoding: [0x7f,0x05,0x12,0x7d]
+
+v_cmpx_nge_f16 s1, v2.l
+// GFX12: v_cmpx_nge_f16_e32 s1, v2.l             ; encoding: [0x01,0x04,0x12,0x7d]
 
-v_cmpx_nge_f16 v127, v2
-// GFX12: v_cmpx_nge_f16_e32 v127, v2             ; encoding: [0x7f,0x05,0x12,0x7d]
+v_cmpx_nge_f16 s105, v2.l
+// GFX12: v_cmpx_nge_f16_e32 s105, v2.l           ; encoding: [0x69,0x04,0x12,0x7d]
 
-v_cmpx_nge_f16 s1, v2
-// GFX12: v_cmpx_nge_f16_e32 s1, v2               ; encoding: [0x01,0x04,0x12,0x7d]
+v_cmpx_nge_f16 vcc_lo, v2.l
+// GFX12: v_cmpx_nge_f16_e32 vcc_lo, v2.l         ; encoding: [0x6a,0x04,0x12,0x7d]
 
-v_cmpx_nge_f16 s105, v2
-// GFX12: v_cmpx_nge_f16_e32 s105, v2             ; encoding: [0x69,0x04,0x12,0x7d]
+v_cmpx_nge_f16 vcc_hi, v2.l
+// GFX12: v_cmpx_nge_f16_e32 vcc_hi, v2.l         ; encoding: [0x6b,0x04,0x12,0x7d]
 
-v_cmpx_nge_f16 vcc_lo, v2
-// GFX12: v_cmpx_nge_f16_e32 vcc_lo, v2           ; encoding: [0x6a,0x04,0x12,0x7d]
+v_cmpx_nge_f16 ttmp15, v2.l
+// GFX12: v_cmpx_nge_f16_e32 ttmp15, v2.l         ; encoding: [0x7b,0x04,0x12,0x7d]
 
-v_cmpx_nge_f16 vcc_hi, v2
-// GFX12: v_cmpx_nge_f16_e32 vcc_hi, v2           ; encoding: [0x6b,0x04,0x12,0x7d]
+v_cmpx_nge_f16 m0, v2.l
+// GFX12: v_cmpx_nge_f16_e32 m0, v2.l             ; encoding: [0x7d,0x04,0x12,0x7d]
 
-v_cmpx_nge_f16 ttmp15, v2
-// GFX12: v_cmpx_nge_f16_e32 ttmp15, v2           ; encoding: [0x7b,0x04,0x12,0x7d]
+v_cmpx_nge_f16 exec_lo, v2.l
+// GFX12: v_cmpx_nge_f16_e32 exec_lo, v2.l        ; encoding: [0x7e,0x04,0x12,0x7d]
 
-v_cmpx_nge_f16 m0, v2
-// GFX12: v_cmpx_nge_f16_e32 m0, v2               ; encoding: [0x7d,0x04,0x12,0x7d]
+v_cmpx_nge_f16 exec_hi, v2.l
+// GFX12: v_cmpx_nge_f16_e32 exec_hi, v2.l        ; encoding: [0x7f,0x04,0x12,0x7d]
 
-v_cmpx_nge_f16 exec_lo, v2
-// GFX12: v_cmpx_nge_f16_e32 exec_lo, v2          ; encoding: [0x7e,0x04,0x12,0x7d]
+v_cmpx_nge_f16 null, v2.l
+// GFX12: v_cmpx_nge_f16_e32 null, v2.l           ; encoding: [0x7c,0x04,0x12,0x7d]
 
-v_cmpx_nge_f16 exec_hi, v2
-// GFX12: v_cmpx_nge_f16_e32 exec_hi, v2          ; encoding: [0x7f,0x04,0x12,0x7d]
+v_cmpx_nge_f16 -1, v2.l
+// GFX12: v_cmpx_nge_f16_e32 -1, v2.l             ; encoding: [0xc1,0x04,0x12,0x7d]
 
-v_cmpx_nge_f16 null, v2
-// GFX12: v_cmpx_nge_f16_e32 null, v2             ; encoding: [0x7c,0x04,0x12,0x7d]
+v_cmpx_nge_f16 0.5, v2.l
+// GFX12: v_cmpx_nge_f16_e32 0.5, v2.l            ; encoding: [0xf0,0x04,0x12,0x7d]
 
-v_cmpx_nge_f16 -1, v2
-// GFX12: v_cmpx_nge_f16_e32 -1, v2               ; encoding: [0xc1,0x04,0x12,0x7d]
+v_cmpx_nge_f16 src_scc, v2.l
+// GFX12: v_cmpx_nge_f16_e32 src_scc, v2.l        ; encoding: [0xfd,0x04,0x12,0x7d]
 
-v_cmpx_nge_f16 0.5, v2
-// GFX12: v_cmpx_nge_f16_e32 0.5, v2              ; encoding: [0xf0,0x04,0x12,0x7d]
+v_cmpx_nge_f16 0xfe0b, v127.l
+// GFX12: v_cmpx_nge_f16_e32 0xfe0b, v127.l       ; encoding: [0xff,0xfe,0x12,0x7d,0x0b,0xfe,0x00,0x00]
 
-v_cmpx_nge_f16 src_scc, v2
-// GFX12: v_cmpx_nge_f16_e32 src_scc, v2          ; encoding: [0xfd,0x04,0x12,0x7d]
+v_cmpx_nge_f16 v1.h, v2.l
+// GFX12: v_cmpx_nge_f16_e32 v1.h, v2.l           ; encoding: [0x81,0x05,0x12,0x7d]
 
-v_cmpx_nge_f16 0xfe0b, v127
-// GFX12: v_cmpx_nge_f16_e32 0xfe0b, v127         ; encoding: [0xff,0xfe,0x12,0x7d,0x0b,0xfe,0x00,0x00]
+v_cmpx_nge_f16 v127.h, v2.l
+// GFX12: v_cmpx_nge_f16_e32 v127.h, v2.l         ; encoding: [0xff,0x05,0x12,0x7d]
+
+v_cmpx_nge_f16 src_scc, v2.h
+// GFX12: v_cmpx_nge_f16_e32 src_scc, v2.h        ; encoding: [0xfd,0x04,0x13,0x7d]
+
+v_cmpx_nge_f16 0xfe0b, v127.h
+// GFX12: v_cmpx_nge_f16_e32 0xfe0b, v127.h       ; encoding: [0xff,0xfe,0x13,0x7d,0x0b,0xfe,0x00,0x00]
 
 v_cmpx_nge_f32 v1, v2
 // GFX12: v_cmpx_nge_f32_e32 v1, v2               ; encoding: [0x01,0x05,0x32,0x7d]
@@ -2816,50 +2900,62 @@ v_cmpx_nge_f64 src_scc, v[2:3]
 v_cmpx_nge_f64 0xaf123456, v[254:255]
 // GFX12: v_cmpx_nge_f64_e32 0xaf123456, v[254:255] ; encoding: [0xff,0xfc,0x53,0x7d,0x56,0x34,0x12,0xaf]
 
-v_cmpx_ngt_f16 v1, v2
-// GFX12: v_cmpx_ngt_f16_e32 v1, v2               ; encoding: [0x01,0x05,0x16,0x7d]
+v_cmpx_ngt_f16 v1.l, v2.l
+// GFX12: v_cmpx_ngt_f16_e32 v1.l, v2.l           ; encoding: [0x01,0x05,0x16,0x7d]
+
+v_cmpx_ngt_f16 v127.l, v2.l
+// GFX12: v_cmpx_ngt_f16_e32 v127.l, v2.l         ; encoding: [0x7f,0x05,0x16,0x7d]
+
+v_cmpx_ngt_f16 s1, v2.l
+// GFX12: v_cmpx_ngt_f16_e32 s1, v2.l             ; encoding: [0x01,0x04,0x16,0x7d]
 
-v_cmpx_ngt_f16 v127, v2
-// GFX12: v_cmpx_ngt_f16_e32 v127, v2             ; encoding: [0x7f,0x05,0x16,0x7d]
+v_cmpx_ngt_f16 s105, v2.l
+// GFX12: v_cmpx_ngt_f16_e32 s105, v2.l           ; encoding: [0x69,0x04,0x16,0x7d]
 
-v_cmpx_ngt_f16 s1, v2
-// GFX12: v_cmpx_ngt_f16_e32 s1, v2               ; encoding: [0x01,0x04,0x16,0x7d]
+v_cmpx_ngt_f16 vcc_lo, v2.l
+// GFX12: v_cmpx_ngt_f16_e32 vcc_lo, v2.l         ; encoding: [0x6a,0x04,0x16,0x7d]
 
-v_cmpx_ngt_f16 s105, v2
-// GFX12: v_cmpx_ngt_f16_e32 s105, v2             ; encoding: [0x69,0x04,0x16,0x7d]
+v_cmpx_ngt_f16 vcc_hi, v2.l
+// GFX12: v_cmpx_ngt_f16_e32 vcc_hi, v2.l         ; encoding: [0x6b,0x04,0x16,0x7d]
 
-v_cmpx_ngt_f16 vcc_lo, v2
-// GFX12: v_cmpx_ngt_f16_e32 vcc_lo, v2           ; encoding: [0x6a,0x04,0x16,0x7d]
+v_cmpx_ngt_f16 ttmp15, v2.l
+// GFX12: v_cmpx_ngt_f16_e32 ttmp15, v2.l         ; encoding: [0x7b,0x04,0x16,0x7d]
 
-v_cmpx_ngt_f16 vcc_hi, v2
-// GFX12: v_cmpx_ngt_f16_e32 vcc_hi, v2           ; encoding: [0x6b,0x04,0x16,0x7d]
+v_cmpx_ngt_f16 m0, v2.l
+// GFX12: v_cmpx_ngt_f16_e32 m0, v2.l             ; encoding: [0x7d,0x04,0x16,0x7d]
 
-v_cmpx_ngt_f16 ttmp15, v2
-// GFX12: v_cmpx_ngt_f16_e32 ttmp15, v2           ; encoding: [0x7b,0x04,0x16,0x7d]
+v_cmpx_ngt_f16 exec_lo, v2.l
+// GFX12: v_cmpx_ngt_f16_e32 exec_lo, v2.l        ; encoding: [0x7e,0x04,0x16,0x7d]
 
-v_cmpx_ngt_f16 m0, v2
-// GFX12: v_cmpx_ngt_f16_e32 m0, v2               ; encoding: [0x7d,0x04,0x16,0x7d]
+v_cmpx_ngt_f16 exec_hi, v2.l
+// GFX12: v_cmpx_ngt_f16_e32 exec_hi, v2.l        ; encoding: [0x7f,0x04,0x16,0x7d]
 
-v_cmpx_ngt_f16 exec_lo, v2
-// GFX12: v_cmpx_ngt_f16_e32 exec_lo, v2          ; encoding: [0x7e,0x04,0x16,0x7d]
+v_cmpx_ngt_f16 null, v2.l
+// GFX12: v_cmpx_ngt_f16_e32 null, v2.l           ; encoding: [0x7c,0x04,0x16,0x7d]
 
-v_cmpx_ngt_f16 exec_hi, v2
-// GFX12: v_cmpx_ngt_f16_e32 exec_hi, v2          ; encoding: [0x7f,0x04,0x16,0x7d]
+v_cmpx_ngt_f16 -1, v2.l
+// GFX12: v_cmpx_ngt_f16_e32 -1, v2.l             ; encoding: [0xc1,0x04,0x16,0x7d]
 
-v_cmpx_ngt_f16 null, v2
-// GFX12: v_cmpx_ngt_f16_e32 null, v2             ; encoding: [0x7c,0x04,0x16,0x7d]
+v_cmpx_ngt_f16 0.5, v2.l
+// GFX12: v_cmpx_ngt_f16_e32 0.5, v2.l            ; encoding: [0xf0,0x04,0x16,0x7d]
 
-v_cmpx_ngt_f16 -1, v2
-// GFX12: v_cmpx_ngt_f16_e32 -1, v2               ; encoding: [0xc1,0x04,0x16,0x7d]
+v_cmpx_ngt_f16 src_scc, v2.l
+// GFX12: v_cmpx_ngt_f16_e32 src_scc, v2.l        ; encoding: [0xfd,0x04,0x16,0x7d]
 
-v_cmpx_ngt_f16 0.5, v2
-// GFX12: v_cmpx_ngt_f16_e32 0.5, v2              ; encoding: [0xf0,0x04,0x16,0x7d]
+v_cmpx_ngt_f16 0xfe0b, v127.l
+// GFX12: v_cmpx_ngt_f16_e32 0xfe0b, v127.l       ; encoding: [0xff,0xfe,0x16,0x7d,0x0b,0xfe,0x00,0x00]
 
-v_cmpx_ngt_f16 src_scc, v2
-// GFX12: v_cmpx_ngt_f16_e32 src_scc, v2          ; encoding: [0xfd,0x04,0x16,0x7d]
+v_cmpx_ngt_f16 v1.h, v2.l
+// GFX12: v_cmpx_ngt_f16_e32 v1.h, v2.l           ; encoding: [0x81,0x05,0x16,0x7d]
 
-v_cmpx_ngt_f16 0xfe0b, v127
-// GFX12: v_cmpx_ngt_f16_e32 0xfe0b, v127         ; encoding: [0xff,0xfe,0x16,0x7d,0x0b,0xfe,0x00,0x00]
+v_cmpx_ngt_f16 v127.h, v2.l
+// GFX12: v_cmpx_ngt_f16_e32 v127.h, v2.l         ; encoding: [0xff,0x05,0x16,0x7d]
+
+v_cmpx_ngt_f16 src_scc, v2.h
+// GFX12: v_cmpx_ngt_f16_e32 src_scc, v2.h        ; encoding: [0xfd,0x04,0x17,0x7d]
+
+v_cmpx_ngt_f16 0xfe0b, v127.h
+// GFX12: v_cmpx_ngt_f16_e32 0xfe0b, v127.h       ; encoding: [0xff,0xfe,0x17,0x7d,0x0b,0xfe,0x00,0x00]
 
 v_cmpx_ngt_f32 v1, v2
 // GFX12: v_cmpx_ngt_f32_e32 v1, v2               ; encoding: [0x01,0x05,0x36,0x7d]
@@ -2942,50 +3038,62 @@ v_cmpx_ngt_f64 src_scc, v[2:3]
 v_cmpx_ngt_f64 0xaf123456, v[254:255]
 // GFX12: v_cmpx_ngt_f64_e32 0xaf123456, v[254:255] ; encoding: [0xff,0xfc,0x57,0x7d,0x56,0x34,0x12,0xaf]
 
-v_cmpx_nle_f16 v1, v2
-// GFX12: v_cmpx_nle_f16_e32 v1, v2               ; encoding: [0x01,0x05,0x18,0x7d]
+v_cmpx_nle_f16 v1.l, v2.l
+// GFX12: v_cmpx_nle_f16_e32 v1.l, v2.l           ; encoding: [0x01,0x05,0x18,0x7d]
+
+v_cmpx_nle_f16 v127.l, v2.l
+// GFX12: v_cmpx_nle_f16_e32 v127.l, v2.l         ; encoding: [0x7f,0x05,0x18,0x7d]
 
-v_cmpx_nle_f16 v127, v2
-// GFX12: v_cmpx_nle_f16_e32 v127, v2             ; encoding: [0x7f,0x05,0x18,0x7d]
+v_cmpx_nle_f16 s1, v2.l
+// GFX12: v_cmpx_nle_f16_e32 s1, v2.l             ; encoding: [0x01,0x04,0x18,0x7d]
 
-v_cmpx_nle_f16 s1, v2
-// GFX12: v_cmpx_nle_f16_e32 s1, v2               ; encoding: [0x01,0x04,0x18,0x7d]
+v_cmpx_nle_f16 s105, v2.l
+// GFX12: v_cmpx_nle_f16_e32 s105, v2.l           ; encoding: [0x69,0x04,0x18,0x7d]
 
-v_cmpx_nle_f16 s105, v2
-// GFX12: v_cmpx_nle_f16_e32 s105, v2             ; encoding: [0x69,0x04,0x18,0x7d]
+v_cmpx_nle_f16 vcc_lo, v2.l
+// GFX12: v_cmpx_nle_f16_e32 vcc_lo, v2.l         ; encoding: [0x6a,0x04,0x18,0x7d]
 
-v_cmpx_nle_f16 vcc_lo, v2
-// GFX12: v_cmpx_nle_f16_e32 vcc_lo, v2           ; encoding: [0x6a,0x04,0x18,0x7d]
+v_cmpx_nle_f16 vcc_hi, v2.l
+// GFX12: v_cmpx_nle_f16_e32 vcc_hi, v2.l         ; encoding: [0x6b,0x04,0x18,0x7d]
 
-v_cmpx_nle_f16 vcc_hi, v2
-// GFX12: v_cmpx_nle_f16_e32 vcc_hi, v2           ; encoding: [0x6b,0x04,0x18,0x7d]
+v_cmpx_nle_f16 ttmp15, v2.l
+// GFX12: v_cmpx_nle_f16_e32 ttmp15, v2.l         ; encoding: [0x7b,0x04,0x18,0x7d]
 
-v_cmpx_nle_f16 ttmp15, v2
-// GFX12: v_cmpx_nle_f16_e32 ttmp15, v2           ; encoding: [0x7b,0x04,0x18,0x7d]
+v_cmpx_nle_f16 m0, v2.l
+// GFX12: v_cmpx_nle_f16_e32 m0, v2.l             ; encoding: [0x7d,0x04,0x18,0x7d]
 
-v_cmpx_nle_f16 m0, v2
-// GFX12: v_cmpx_nle_f16_e32 m0, v2               ; encoding: [0x7d,0x04,0x18,0x7d]
+v_cmpx_nle_f16 exec_lo, v2.l
+// GFX12: v_cmpx_nle_f16_e32 exec_lo, v2.l        ; encoding: [0x7e,0x04,0x18,0x7d]
 
-v_cmpx_nle_f16 exec_lo, v2
-// GFX12: v_cmpx_nle_f16_e32 exec_lo, v2          ; encoding: [0x7e,0x04,0x18,0x7d]
+v_cmpx_nle_f16 exec_hi, v2.l
+// GFX12: v_cmpx_nle_f16_e32 exec_hi, v2.l        ; encoding: [0x7f,0x04,0x18,0x7d]
 
-v_cmpx_nle_f16 exec_hi, v2
-// GFX12: v_cmpx_nle_f16_e32 exec_hi, v2          ; encoding: [0x7f,0x04,0x18,0x7d]
+v_cmpx_nle_f16 null, v2.l
+// GFX12: v_cmpx_nle_f16_e32 null, v2.l           ; encoding: [0x7c,0x04,0x18,0x7d]
 
-v_cmpx_nle_f16 null, v2
-// GFX12: v_cmpx_nle_f16_e32 null, v2             ; encoding: [0x7c,0x04,0x18,0x7d]
+v_cmpx_nle_f16 -1, v2.l
+// GFX12: v_cmpx_nle_f16_e32 -1, v2.l             ; encoding: [0xc1,0x04,0x18,0x7d]
 
-v_cmpx_nle_f16 -1, v2
-// GFX12: v_cmpx_nle_f16_e32 -1, v2               ; encoding: [0xc1,0x04,0x18,0x7d]
+v_cmpx_nle_f16 0.5, v2.l
+// GFX12: v_cmpx_nle_f16_e32 0.5, v2.l            ; encoding: [0xf0,0x04,0x18,0x7d]
 
-v_cmpx_nle_f16 0.5, v2
-// GFX12: v_cmpx_nle_f16_e32 0.5, v2              ; encoding: [0xf0,0x04,0x18,0x7d]
+v_cmpx_nle_f16 src_scc, v2.l
+// GFX12: v_cmpx_nle_f16_e32 src_scc, v2.l        ; encoding: [0xfd,0x04,0x18,0x7d]
 
-v_cmpx_nle_f16 src_scc, v2
-// GFX12: v_cmpx_nle_f16_e32 src_scc, v2          ; encoding: [0xfd,0x04,0x18,0x7d]
+v_cmpx_nle_f16 0xfe0b, v127.l
+// GFX12: v_cmpx_nle_f16_e32 0xfe0b, v127.l       ; encoding: [0xff,0xfe,0x18,0x7d,0x0b,0xfe,0x00,0x00]
 
-v_cmpx_nle_f16 0xfe0b, v127
-// GFX12: v_cmpx_nle_f16_e32 0xfe0b, v127         ; encoding: [0xff,0xfe,0x18,0x7d,0x0b,0xfe,0x00,0x00]
+v_cmpx_nle_f16 v1.h, v2.l
+// GFX12: v_cmpx_nle_f16_e32 v1.h, v2.l           ; encoding: [0x81,0x05,0x18,0x7d]
+
+v_cmpx_nle_f16 v127.h, v2.l
+// GFX12: v_cmpx_nle_f16_e32 v127.h, v2.l         ; encoding: [0xff,0x05,0x18,0x7d]
+
+v_cmpx_nle_f16 src_scc, v2.h
+// GFX12: v_cmpx_nle_f16_e32 src_scc, v2.h        ; encoding: [0xfd,0x04,0x19,0x7d]
+
+v_cmpx_nle_f16 0xfe0b, v127.h
+// GFX12: v_cmpx_nle_f16_e32 0xfe0b, v127.h       ; encoding: [0xff,0xfe,0x19,0x7d,0x0b,0xfe,0x00,0x00]
 
 v_cmpx_nle_f32 v1, v2
 // GFX12: v_cmpx_nle_f32_e32 v1, v2               ; encoding: [0x01,0x05,0x38,0x7d]
@@ -3068,50 +3176,62 @@ v_cmpx_nle_f64 src_scc, v[2:3]
 v_cmpx_nle_f64 0xaf123456, v[254:255]
 // GFX12: v_cmpx_nle_f64_e32 0xaf123456, v[254:255] ; encoding: [0xff,0xfc,0x59,0x7d,0x56,0x34,0x12,0xaf]
 
-v_cmpx_nlg_f16 v1, v2
-// GFX12: v_cmpx_nlg_f16_e32 v1, v2               ; encoding: [0x01,0x05,0x14,0x7d]
+v_cmpx_nlg_f16 v1.l, v2.l
+// GFX12: v_cmpx_nlg_f16_e32 v1.l, v2.l           ; encoding: [0x01,0x05,0x14,0x7d]
+
+v_cmpx_nlg_f16 v127.l, v2.l
+// GFX12: v_cmpx_nlg_f16_e32 v127.l, v2.l         ; encoding: [0x7f,0x05,0x14,0x7d]
+
+v_cmpx_nlg_f16 s1, v2.l
+// GFX12: v_cmpx_nlg_f16_e32 s1, v2.l             ; encoding: [0x01,0x04,0x14,0x7d]
+
+v_cmpx_nlg_f16 s105, v2.l
+// GFX12: v_cmpx_nlg_f16_e32 s105, v2.l           ; encoding: [0x69,0x04,0x14,0x7d]
 
-v_cmpx_nlg_f16 v127, v2
-// GFX12: v_cmpx_nlg_f16_e32 v127, v2             ; encoding: [0x7f,0x05,0x14,0x7d]
+v_cmpx_nlg_f16 vcc_lo, v2.l
+// GFX12: v_cmpx_nlg_f16_e32 vcc_lo, v2.l         ; encoding: [0x6a,0x04,0x14,0x7d]
 
-v_cmpx_nlg_f16 s1, v2
-// GFX12: v_cmpx_nlg_f16_e32 s1, v2               ; encoding: [0x01,0x04,0x14,0x7d]
+v_cmpx_nlg_f16 vcc_hi, v2.l
+// GFX12: v_cmpx_nlg_f16_e32 vcc_hi, v2.l         ; encoding: [0x6b,0x04,0x14,0x7d]
 
-v_cmpx_nlg_f16 s105, v2
-// GFX12: v_cmpx_nlg_f16_e32 s105, v2             ; encoding: [0x69,0x04,0x14,0x7d]
+v_cmpx_nlg_f16 ttmp15, v2.l
+// GFX12: v_cmpx_nlg_f16_e32 ttmp15, v2.l         ; encoding: [0x7b,0x04,0x14,0x7d]
 
-v_cmpx_nlg_f16 vcc_lo, v2
-// GFX12: v_cmpx_nlg_f16_e32 vcc_lo, v2           ; encoding: [0x6a,0x04,0x14,0x7d]
+v_cmpx_nlg_f16 m0, v2.l
+// GFX12: v_cmpx_nlg_f16_e32 m0, v2.l             ; encoding: [0x7d,0x04,0x14,0x7d]
 
-v_cmpx_nlg_f16 vcc_hi, v2
-// GFX12: v_cmpx_nlg_f16_e32 vcc_hi, v2           ; encoding: [0x6b,0x04,0x14,0x7d]
+v_cmpx_nlg_f16 exec_lo, v2.l
+// GFX12: v_cmpx_nlg_f16_e32 exec_lo, v2.l        ; encoding: [0x7e,0x04,0x14,0x7d]
 
-v_cmpx_nlg_f16 ttmp15, v2
-// GFX12: v_cmpx_nlg_f16_e32 ttmp15, v2           ; encoding: [0x7b,0x04,0x14,0x7d]
+v_cmpx_nlg_f16 exec_hi, v2.l
+// GFX12: v_cmpx_nlg_f16_e32 exec_hi, v2.l        ; encoding: [0x7f,0x04,0x14,0x7d]
 
-v_cmpx_nlg_f16 m0, v2
-// GFX12: v_cmpx_nlg_f16_e32 m0, v2               ; encoding: [0x7d,0x04,0x14,0x7d]
+v_cmpx_nlg_f16 null, v2.l
+// GFX12: v_cmpx_nlg_f16_e32 null, v2.l           ; encoding: [0x7c,0x04,0x14,0x7d]
 
-v_cmpx_nlg_f16 exec_lo, v2
-// GFX12: v_cmpx_nlg_f16_e32 exec_lo, v2          ; encoding: [0x7e,0x04,0x14,0x7d]
+v_cmpx_nlg_f16 -1, v2.l
+// GFX12: v_cmpx_nlg_f16_e32 -1, v2.l             ; encoding: [0xc1,0x04,0x14,0x7d]
 
-v_cmpx_nlg_f16 exec_hi, v2
-// GFX12: v_cmpx_nlg_f16_e32 exec_hi, v2          ; encoding: [0x7f,0x04,0x14,0x7d]
+v_cmpx_nlg_f16 0.5, v2.l
+// GFX12: v_cmpx_nlg_f16_e32 0.5, v2.l            ; encoding: [0xf0,0x04,0x14,0x7d]
 
-v_cmpx_nlg_f16 null, v2
-// GFX12: v_cmpx_nlg_f16_e32 null, v2             ; encoding: [0x7c,0x04,0x14,0x7d]
+v_cmpx_nlg_f16 src_scc, v2.l
+// GFX12: v_cmpx_nlg_f16_e32 src_scc, v2.l        ; encoding: [0xfd,0x04,0x14,0x7d]
 
-v_cmpx_nlg_f16 -1, v2
-// GFX12: v_cmpx_nlg_f16_e32 -1, v2               ; encoding: [0xc1,0x04,0x14,0x7d]
+v_cmpx_nlg_f16 0xfe0b, v127.l
+// GFX12: v_cmpx_nlg_f16_e32 0xfe0b, v127.l       ; encoding: [0xff,0xfe,0x14,0x7d,0x0b,0xfe,0x00,0x00]
 
-v_cmpx_nlg_f16 0.5, v2
-// GFX12: v_cmpx_nlg_f16_e32 0.5, v2              ; encoding: [0xf0,0x04,0x14,0x7d]
+v_cmpx_nlg_f16 v1.h, v2.l
+// GFX12: v_cmpx_nlg_f16_e32 v1.h, v2.l           ; encoding: [0x81,0x05,0x14,0x7d]
 
-v_cmpx_nlg_f16 src_scc, v2
-// GFX12: v_cmpx_nlg_f16_e32 src_scc, v2          ; encoding: [0xfd,0x04,0x14,0x7d]
+v_cmpx_nlg_f16 v127.h, v2.l
+// GFX12: v_cmpx_nlg_f16_e32 v127.h, v2.l         ; encoding: [0xff,0x05,0x14,0x7d]
 
-v_cmpx_nlg_f16 0xfe0b, v127
-// GFX12: v_cmpx_nlg_f16_e32 0xfe0b, v127         ; encoding: [0xff,0xfe,0x14,0x7d,0x0b,0xfe,0x00,0x00]
+v_cmpx_nlg_f16 src_scc, v2.h
+// GFX12: v_cmpx_nlg_f16_e32 src_scc, v2.h        ; encoding: [0xfd,0x04,0x15,0x7d]
+
+v_cmpx_nlg_f16 0xfe0b, v127.h
+// GFX12: v_cmpx_nlg_f16_e32 0xfe0b, v127.h       ; encoding: [0xff,0xfe,0x15,0x7d,0x0b,0xfe,0x00,0x00]
 
 v_cmpx_nlg_f32 v1, v2
 // GFX12: v_cmpx_nlg_f32_e32 v1, v2               ; encoding: [0x01,0x05,0x34,0x7d]
@@ -3194,50 +3314,62 @@ v_cmpx_nlg_f64 src_scc, v[2:3]
 v_cmpx_nlg_f64 0xaf123456, v[254:255]
 // GFX12: v_cmpx_nlg_f64_e32 0xaf123456, v[254:255] ; encoding: [0xff,0xfc,0x55,0x7d,0x56,0x34,0x12,0xaf]
 
-v_cmpx_nlt_f16 v1, v2
-// GFX12: v_cmpx_nlt_f16_e32 v1, v2               ; encoding: [0x01,0x05,0x1c,0x7d]
+v_cmpx_nlt_f16 v1.l, v2.l
+// GFX12: v_cmpx_nlt_f16_e32 v1.l, v2.l           ; encoding: [0x01,0x05,0x1c,0x7d]
+
+v_cmpx_nlt_f16 v127.l, v2.l
+// GFX12: v_cmpx_nlt_f16_e32 v127.l, v2.l         ; encoding: [0x7f,0x05,0x1c,0x7d]
+
+v_cmpx_nlt_f16 s1, v2.l
+// GFX12: v_cmpx_nlt_f16_e32 s1, v2.l             ; encoding: [0x01,0x04,0x1c,0x7d]
+
+v_cmpx_nlt_f16 s105, v2.l
+// GFX12: v_cmpx_nlt_f16_e32 s105, v2.l           ; encoding: [0x69,0x04,0x1c,0x7d]
 
-v_cmpx_nlt_f16 v127, v2
-// GFX12: v_cmpx_nlt_f16_e32 v127, v2             ; encoding: [0x7f,0x05,0x1c,0x7d]
+v_cmpx_nlt_f16 vcc_lo, v2.l
+// GFX12: v_cmpx_nlt_f16_e32 vcc_lo, v2.l         ; encoding: [0x6a,0x04,0x1c,0x7d]
 
-v_cmpx_nlt_f16 s1, v2
-// GFX12: v_cmpx_nlt_f16_e32 s1, v2               ; encoding: [0x01,0x04,0x1c,0x7d]
+v_cmpx_nlt_f16 vcc_hi, v2.l
+// GFX12: v_cmpx_nlt_f16_e32 vcc_hi, v2.l         ; encoding: [0x6b,0x04,0x1c,0x7d]
 
-v_cmpx_nlt_f16 s105, v2
-// GFX12: v_cmpx_nlt_f16_e32 s105, v2             ; encoding: [0x69,0x04,0x1c,0x7d]
+v_cmpx_nlt_f16 ttmp15, v2.l
+// GFX12: v_cmpx_nlt_f16_e32 ttmp15, v2.l         ; encoding: [0x7b,0x04,0x1c,0x7d]
 
-v_cmpx_nlt_f16 vcc_lo, v2
-// GFX12: v_cmpx_nlt_f16_e32 vcc_lo, v2           ; encoding: [0x6a,0x04,0x1c,0x7d]
+v_cmpx_nlt_f16 m0, v2.l
+// GFX12: v_cmpx_nlt_f16_e32 m0, v2.l             ; encoding: [0x7d,0x04,0x1c,0x7d]
 
-v_cmpx_nlt_f16 vcc_hi, v2
-// GFX12: v_cmpx_nlt_f16_e32 vcc_hi, v2           ; encoding: [0x6b,0x04,0x1c,0x7d]
+v_cmpx_nlt_f16 exec_lo, v2.l
+// GFX12: v_cmpx_nlt_f16_e32 exec_lo, v2.l        ; encoding: [0x7e,0x04,0x1c,0x7d]
 
-v_cmpx_nlt_f16 ttmp15, v2
-// GFX12: v_cmpx_nlt_f16_e32 ttmp15, v2           ; encoding: [0x7b,0x04,0x1c,0x7d]
+v_cmpx_nlt_f16 exec_hi, v2.l
+// GFX12: v_cmpx_nlt_f16_e32 exec_hi, v2.l        ; encoding: [0x7f,0x04,0x1c,0x7d]
 
-v_cmpx_nlt_f16 m0, v2
-// GFX12: v_cmpx_nlt_f16_e32 m0, v2               ; encoding: [0x7d,0x04,0x1c,0x7d]
+v_cmpx_nlt_f16 null, v2.l
+// GFX12: v_cmpx_nlt_f16_e32 null, v2.l           ; encoding: [0x7c,0x04,0x1c,0x7d]
 
-v_cmpx_nlt_f16 exec_lo, v2
-// GFX12: v_cmpx_nlt_f16_e32 exec_lo, v2          ; encoding: [0x7e,0x04,0x1c,0x7d]
+v_cmpx_nlt_f16 -1, v2.l
+// GFX12: v_cmpx_nlt_f16_e32 -1, v2.l             ; encoding: [0xc1,0x04,0x1c,0x7d]
 
-v_cmpx_nlt_f16 exec_hi, v2
-// GFX12: v_cmpx_nlt_f16_e32 exec_hi, v2          ; encoding: [0x7f,0x04,0x1c,0x7d]
+v_cmpx_nlt_f16 0.5, v2.l
+// GFX12: v_cmpx_nlt_f16_e32 0.5, v2.l            ; encoding: [0xf0,0x04,0x1c,0x7d]
 
-v_cmpx_nlt_f16 null, v2
-// GFX12: v_cmpx_nlt_f16_e32 null, v2             ; encoding: [0x7c,0x04,0x1c,0x7d]
+v_cmpx_nlt_f16 src_scc, v2.l
+// GFX12: v_cmpx_nlt_f16_e32 src_scc, v2.l        ; encoding: [0xfd,0x04,0x1c,0x7d]
 
-v_cmpx_nlt_f16 -1, v2
-// GFX12: v_cmpx_nlt_f16_e32 -1, v2               ; encoding: [0xc1,0x04,0x1c,0x7d]
+v_cmpx_nlt_f16 0xfe0b, v127.l
+// GFX12: v_cmpx_nlt_f16_e32 0xfe0b, v127.l       ; encoding: [0xff,0xfe,0x1c,0x7d,0x0b,0xfe,0x00,0x00]
 
-v_cmpx_nlt_f16 0.5, v2
-// GFX12: v_cmpx_nlt_f16_e32 0.5, v2              ; encoding: [0xf0,0x04,0x1c,0x7d]
+v_cmpx_nlt_f16 v1.h, v2.l
+// GFX12: v_cmpx_nlt_f16_e32 v1.h, v2.l           ; encoding: [0x81,0x05,0x1c,0x7d]
 
-v_cmpx_nlt_f16 src_scc, v2
-// GFX12: v_cmpx_nlt_f16_e32 src_scc, v2          ; encoding: [0xfd,0x04,0x1c,0x7d]
+v_cmpx_nlt_f16 v127.h, v2.l
+// GFX12: v_cmpx_nlt_f16_e32 v127.h, v2.l         ; encoding: [0xff,0x05,0x1c,0x7d]
 
-v_cmpx_nlt_f16 0xfe0b, v127
-// GFX12: v_cmpx_nlt_f16_e32 0xfe0b, v127         ; encoding: [0xff,0xfe,0x1c,0x7d,0x0b,0xfe,0x00,0x00]
+v_cmpx_nlt_f16 src_scc, v2.h
+// GFX12: v_cmpx_nlt_f16_e32 src_scc, v2.h        ; encoding: [0xfd,0x04,0x1d,0x7d]
+
+v_cmpx_nlt_f16 0xfe0b, v127.h
+// GFX12: v_cmpx_nlt_f16_e32 0xfe0b, v127.h       ; encoding: [0xff,0xfe,0x1d,0x7d,0x0b,0xfe,0x00,0x00]
 
 v_cmpx_nlt_f32 v1, v2
 // GFX12: v_cmpx_nlt_f32_e32 v1, v2               ; encoding: [0x01,0x05,0x3c,0x7d]
@@ -3320,50 +3452,62 @@ v_cmpx_nlt_f64 src_scc, v[2:3]
 v_cmpx_nlt_f64 0xaf123456, v[254:255]
 // GFX12: v_cmpx_nlt_f64_e32 0xaf123456, v[254:255] ; encoding: [0xff,0xfc,0x5d,0x7d,0x56,0x34,0x12,0xaf]
 
-v_cmpx_o_f16 v1, v2
-// GFX12: v_cmpx_o_f16_e32 v1, v2                 ; encoding: [0x01,0x05,0x0e,0x7d]
+v_cmpx_o_f16 v1.l, v2.l
+// GFX12: v_cmpx_o_f16_e32 v1.l, v2.l             ; encoding: [0x01,0x05,0x0e,0x7d]
+
+v_cmpx_o_f16 v127.l, v2.l
+// GFX12: v_cmpx_o_f16_e32 v127.l, v2.l           ; encoding: [0x7f,0x05,0x0e,0x7d]
+
+v_cmpx_o_f16 s1, v2.l
+// GFX12: v_cmpx_o_f16_e32 s1, v2.l               ; encoding: [0x01,0x04,0x0e,0x7d]
 
-v_cmpx_o_f16 v127, v2
-// GFX12: v_cmpx_o_f16_e32 v127, v2               ; encoding: [0x7f,0x05,0x0e,0x7d]
+v_cmpx_o_f16 s105, v2.l
+// GFX12: v_cmpx_o_f16_e32 s105, v2.l             ; encoding: [0x69,0x04,0x0e,0x7d]
 
-v_cmpx_o_f16 s1, v2
-// GFX12: v_cmpx_o_f16_e32 s1, v2                 ; encoding: [0x01,0x04,0x0e,0x7d]
+v_cmpx_o_f16 vcc_lo, v2.l
+// GFX12: v_cmpx_o_f16_e32 vcc_lo, v2.l           ; encoding: [0x6a,0x04,0x0e,0x7d]
 
-v_cmpx_o_f16 s105, v2
-// GFX12: v_cmpx_o_f16_e32 s105, v2               ; encoding: [0x69,0x04,0x0e,0x7d]
+v_cmpx_o_f16 vcc_hi, v2.l
+// GFX12: v_cmpx_o_f16_e32 vcc_hi, v2.l           ; encoding: [0x6b,0x04,0x0e,0x7d]
 
-v_cmpx_o_f16 vcc_lo, v2
-// GFX12: v_cmpx_o_f16_e32 vcc_lo, v2             ; encoding: [0x6a,0x04,0x0e,0x7d]
+v_cmpx_o_f16 ttmp15, v2.l
+// GFX12: v_cmpx_o_f16_e32 ttmp15, v2.l           ; encoding: [0x7b,0x04,0x0e,0x7d]
 
-v_cmpx_o_f16 vcc_hi, v2
-// GFX12: v_cmpx_o_f16_e32 vcc_hi, v2             ; encoding: [0x6b,0x04,0x0e,0x7d]
+v_cmpx_o_f16 m0, v2.l
+// GFX12: v_cmpx_o_f16_e32 m0, v2.l               ; encoding: [0x7d,0x04,0x0e,0x7d]
 
-v_cmpx_o_f16 ttmp15, v2
-// GFX12: v_cmpx_o_f16_e32 ttmp15, v2             ; encoding: [0x7b,0x04,0x0e,0x7d]
+v_cmpx_o_f16 exec_lo, v2.l
+// GFX12: v_cmpx_o_f16_e32 exec_lo, v2.l          ; encoding: [0x7e,0x04,0x0e,0x7d]
 
-v_cmpx_o_f16 m0, v2
-// GFX12: v_cmpx_o_f16_e32 m0, v2                 ; encoding: [0x7d,0x04,0x0e,0x7d]
+v_cmpx_o_f16 exec_hi, v2.l
+// GFX12: v_cmpx_o_f16_e32 exec_hi, v2.l          ; encoding: [0x7f,0x04,0x0e,0x7d]
 
-v_cmpx_o_f16 exec_lo, v2
-// GFX12: v_cmpx_o_f16_e32 exec_lo, v2            ; encoding: [0x7e,0x04,0x0e,0x7d]
+v_cmpx_o_f16 null, v2.l
+// GFX12: v_cmpx_o_f16_e32 null, v2.l             ; encoding: [0x7c,0x04,0x0e,0x7d]
 
-v_cmpx_o_f16 exec_hi, v2
-// GFX12: v_cmpx_o_f16_e32 exec_hi, v2            ; encoding: [0x7f,0x04,0x0e,0x7d]
+v_cmpx_o_f16 -1, v2.l
+// GFX12: v_cmpx_o_f16_e32 -1, v2.l               ; encoding: [0xc1,0x04,0x0e,0x7d]
 
-v_cmpx_o_f16 null, v2
-// GFX12: v_cmpx_o_f16_e32 null, v2               ; encoding: [0x7c,0x04,0x0e,0x7d]
+v_cmpx_o_f16 0.5, v2.l
+// GFX12: v_cmpx_o_f16_e32 0.5, v2.l              ; encoding: [0xf0,0x04,0x0e,0x7d]
 
-v_cmpx_o_f16 -1, v2
-// GFX12: v_cmpx_o_f16_e32 -1, v2                 ; encoding: [0xc1,0x04,0x0e,0x7d]
+v_cmpx_o_f16 src_scc, v2.l
+// GFX12: v_cmpx_o_f16_e32 src_scc, v2.l          ; encoding: [0xfd,0x04,0x0e,0x7d]
 
-v_cmpx_o_f16 0.5, v2
-// GFX12: v_cmpx_o_f16_e32 0.5, v2                ; encoding: [0xf0,0x04,0x0e,0x7d]
+v_cmpx_o_f16 0xfe0b, v127.l
+// GFX12: v_cmpx_o_f16_e32 0xfe0b, v127.l         ; encoding: [0xff,0xfe,0x0e,0x7d,0x0b,0xfe,0x00,0x00]
 
-v_cmpx_o_f16 src_scc, v2
-// GFX12: v_cmpx_o_f16_e32 src_scc, v2            ; encoding: [0xfd,0x04,0x0e,0x7d]
+v_cmpx_o_f16 v1.h, v2.l
+// GFX12: v_cmpx_o_f16_e32 v1.h, v2.l             ; encoding: [0x81,0x05,0x0e,0x7d]
 
-v_cmpx_o_f16 0xfe0b, v127
-// GFX12: v_cmpx_o_f16_e32 0xfe0b, v127           ; encoding: [0xff,0xfe,0x0e,0x7d,0x0b,0xfe,0x00,0x00]
+v_cmpx_o_f16 v127.h, v2.l
+// GFX12: v_cmpx_o_f16_e32 v127.h, v2.l           ; encoding: [0xff,0x05,0x0e,0x7d]
+
+v_cmpx_o_f16 src_scc, v2.h
+// GFX12: v_cmpx_o_f16_e32 src_scc, v2.h          ; encoding: [0xfd,0x04,0x0f,0x7d]
+
+v_cmpx_o_f16 0xfe0b, v127.h
+// GFX12: v_cmpx_o_f16_e32 0xfe0b, v127.h         ; encoding: [0xff,0xfe,0x0f,0x7d,0x0b,0xfe,0x00,0x00]
 
 v_cmpx_o_f32 v1, v2
 // GFX12: v_cmpx_o_f32_e32 v1, v2                 ; encoding: [0x01,0x05,0x2e,0x7d]
@@ -3446,50 +3590,62 @@ v_cmpx_o_f64 src_scc, v[2:3]
 v_cmpx_o_f64 0xaf123456, v[254:255]
 // GFX12: v_cmpx_o_f64_e32 0xaf123456, v[254:255] ; encoding: [0xff,0xfc,0x4f,0x7d,0x56,0x34,0x12,0xaf]
 
-v_cmpx_u_f16 v1, v2
-// GFX12: v_cmpx_u_f16_e32 v1, v2                 ; encoding: [0x01,0x05,0x10,0x7d]
+v_cmpx_u_f16 v1.l, v2.l
+// GFX12: v_cmpx_u_f16_e32 v1.l, v2.l             ; encoding: [0x01,0x05,0x10,0x7d]
+
+v_cmpx_u_f16 v127.l, v2.l
+// GFX12: v_cmpx_u_f16_e32 v127.l, v2.l           ; encoding: [0x7f,0x05,0x10,0x7d]
+
+v_cmpx_u_f16 s1, v2.l
+// GFX12: v_cmpx_u_f16_e32 s1, v2.l               ; encoding: [0x01,0x04,0x10,0x7d]
+
+v_cmpx_u_f16 s105, v2.l
+// GFX12: v_cmpx_u_f16_e32 s105, v2.l             ; encoding: [0x69,0x04,0x10,0x7d]
+
+v_cmpx_u_f16 vcc_lo, v2.l
+// GFX12: v_cmpx_u_f16_e32 vcc_lo, v2.l           ; encoding: [0x6a,0x04,0x10,0x7d]
 
-v_cmpx_u_f16 v127, v2
-// GFX12: v_cmpx_u_f16_e32 v127, v2               ; encoding: [0x7f,0x05,0x10,0x7d]
+v_cmpx_u_f16 vcc_hi, v2.l
+// GFX12: v_cmpx_u_f16_e32 vcc_hi, v2.l           ; encoding: [0x6b,0x04,0x10,0x7d]
 
-v_cmpx_u_f16 s1, v2
-// GFX12: v_cmpx_u_f16_e32 s1, v2                 ; encoding: [0x01,0x04,0x10,0x7d]
+v_cmpx_u_f16 ttmp15, v2.l
+// GFX12: v_cmpx_u_f16_e32 ttmp15, v2.l           ; encoding: [0x7b,0x04,0x10,0x7d]
 
-v_cmpx_u_f16 s105, v2
-// GFX12: v_cmpx_u_f16_e32 s105, v2               ; encoding: [0x69,0x04,0x10,0x7d]
+v_cmpx_u_f16 m0, v2.l
+// GFX12: v_cmpx_u_f16_e32 m0, v2.l               ; encoding: [0x7d,0x04,0x10,0x7d]
 
-v_cmpx_u_f16 vcc_lo, v2
-// GFX12: v_cmpx_u_f16_e32 vcc_lo, v2             ; encoding: [0x6a,0x04,0x10,0x7d]
+v_cmpx_u_f16 exec_lo, v2.l
+// GFX12: v_cmpx_u_f16_e32 exec_lo, v2.l          ; encoding: [0x7e,0x04,0x10,0x7d]
 
-v_cmpx_u_f16 vcc_hi, v2
-// GFX12: v_cmpx_u_f16_e32 vcc_hi, v2             ; encoding: [0x6b,0x04,0x10,0x7d]
+v_cmpx_u_f16 exec_hi, v2.l
+// GFX12: v_cmpx_u_f16_e32 exec_hi, v2.l          ; encoding: [0x7f,0x04,0x10,0x7d]
 
-v_cmpx_u_f16 ttmp15, v2
-// GFX12: v_cmpx_u_f16_e32 ttmp15, v2             ; encoding: [0x7b,0x04,0x10,0x7d]
+v_cmpx_u_f16 null, v2.l
+// GFX12: v_cmpx_u_f16_e32 null, v2.l             ; encoding: [0x7c,0x04,0x10,0x7d]
 
-v_cmpx_u_f16 m0, v2
-// GFX12: v_cmpx_u_f16_e32 m0, v2                 ; encoding: [0x7d,0x04,0x10,0x7d]
+v_cmpx_u_f16 -1, v2.l
+// GFX12: v_cmpx_u_f16_e32 -1, v2.l               ; encoding: [0xc1,0x04,0x10,0x7d]
 
-v_cmpx_u_f16 exec_lo, v2
-// GFX12: v_cmpx_u_f16_e32 exec_lo, v2            ; encoding: [0x7e,0x04,0x10,0x7d]
+v_cmpx_u_f16 0.5, v2.l
+// GFX12: v_cmpx_u_f16_e32 0.5, v2.l              ; encoding: [0xf0,0x04,0x10,0x7d]
 
-v_cmpx_u_f16 exec_hi, v2
-// GFX12: v_cmpx_u_f16_e32 exec_hi, v2            ; encoding: [0x7f,0x04,0x10,0x7d]
+v_cmpx_u_f16 src_scc, v2.l
+// GFX12: v_cmpx_u_f16_e32 src_scc, v2.l          ; encoding: [0xfd,0x04,0x10,0x7d]
 
-v_cmpx_u_f16 null, v2
-// GFX12: v_cmpx_u_f16_e32 null, v2               ; encoding: [0x7c,0x04,0x10,0x7d]
+v_cmpx_u_f16 0xfe0b, v127.l
+// GFX12: v_cmpx_u_f16_e32 0xfe0b, v127.l         ; encoding: [0xff,0xfe,0x10,0x7d,0x0b,0xfe,0x00,0x00]
 
-v_cmpx_u_f16 -1, v2
-// GFX12: v_cmpx_u_f16_e32 -1, v2                 ; encoding: [0xc1,0x04,0x10,0x7d]
+v_cmpx_u_f16 v1.h, v2.l
+// GFX12: v_cmpx_u_f16_e32 v1.h, v2.l             ; encoding: [0x81,0x05,0x10,0x7d]
 
-v_cmpx_u_f16 0.5, v2
-// GFX12: v_cmpx_u_f16_e32 0.5, v2                ; encoding: [0xf0,0x04,0x10,0x7d]
+v_cmpx_u_f16 v127.h, v2.l
+// GFX12: v_cmpx_u_f16_e32 v127.h, v2.l           ; encoding: [0xff,0x05,0x10,0x7d]
 
-v_cmpx_u_f16 src_scc, v2
-// GFX12: v_cmpx_u_f16_e32 src_scc, v2            ; encoding: [0xfd,0x04,0x10,0x7d]
+v_cmpx_u_f16 src_scc, v2.h
+// GFX12: v_cmpx_u_f16_e32 src_scc, v2.h          ; encoding: [0xfd,0x04,0x11,0x7d]
 
-v_cmpx_u_f16 0xfe0b, v127
-// GFX12: v_cmpx_u_f16_e32 0xfe0b, v127           ; encoding: [0xff,0xfe,0x10,0x7d,0x0b,0xfe,0x00,0x00]
+v_cmpx_u_f16 0xfe0b, v127.h
+// GFX12: v_cmpx_u_f16_e32 0xfe0b, v127.h         ; encoding: [0xff,0xfe,0x11,0x7d,0x0b,0xfe,0x00,0x00]
 
 v_cmpx_u_f32 v1, v2
 // GFX12: v_cmpx_u_f32_e32 v1, v2                 ; encoding: [0x01,0x05,0x30,0x7d]
diff --git a/llvm/test/MC/AMDGPU/gfx12_asm_vopcx_dpp16.s b/llvm/test/MC/AMDGPU/gfx12_asm_vopcx_dpp16.s
index 5c54d1ad5788c..3a3a89b8a9932 100644
--- a/llvm/test/MC/AMDGPU/gfx12_asm_vopcx_dpp16.s
+++ b/llvm/test/MC/AMDGPU/gfx12_asm_vopcx_dpp16.s
@@ -92,47 +92,53 @@ v_cmpx_class_f32 v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
 v_cmpx_class_f32 -|v255|, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
 // GFX12: v_cmpx_class_f32 -|v255|, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xfa,0xfe,0xfd,0x7d,0xff,0x6f,0x35,0x30]
 
-v_cmpx_eq_f16 v1, v2 quad_perm:[3,2,1,0]
-// GFX12: v_cmpx_eq_f16 v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x04,0x7d,0x01,0x1b,0x00,0xff]
+v_cmpx_eq_f16 v1.l, v2.l quad_perm:[3,2,1,0]
+// GFX12: v_cmpx_eq_f16 v1.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x04,0x7d,0x01,0x1b,0x00,0xff]
 
-v_cmpx_eq_f16 v1, v2 quad_perm:[0,1,2,3]
-// GFX12: v_cmpx_eq_f16 v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x04,0x7d,0x01,0xe4,0x00,0xff]
+v_cmpx_eq_f16 v1.l, v2.l quad_perm:[0,1,2,3]
+// GFX12: v_cmpx_eq_f16 v1.l, v2.l quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x04,0x7d,0x01,0xe4,0x00,0xff]
 
-v_cmpx_eq_f16 v1, v2 row_mirror
-// GFX12: v_cmpx_eq_f16 v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x04,0x7d,0x01,0x40,0x01,0xff]
+v_cmpx_eq_f16 v1.l, v2.l row_mirror
+// GFX12: v_cmpx_eq_f16 v1.l, v2.l row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x04,0x7d,0x01,0x40,0x01,0xff]
 
-v_cmpx_eq_f16 v1, v2 row_half_mirror
-// GFX12: v_cmpx_eq_f16 v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x04,0x7d,0x01,0x41,0x01,0xff]
+v_cmpx_eq_f16 v1.l, v2.l row_half_mirror
+// GFX12: v_cmpx_eq_f16 v1.l, v2.l row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x04,0x7d,0x01,0x41,0x01,0xff]
 
-v_cmpx_eq_f16 v1, v2 row_shl:1
-// GFX12: v_cmpx_eq_f16 v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x04,0x7d,0x01,0x01,0x01,0xff]
+v_cmpx_eq_f16 v1.l, v2.l row_shl:1
+// GFX12: v_cmpx_eq_f16 v1.l, v2.l row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x04,0x7d,0x01,0x01,0x01,0xff]
 
-v_cmpx_eq_f16 v1, v2 row_shl:15
-// GFX12: v_cmpx_eq_f16 v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x04,0x7d,0x01,0x0f,0x01,0xff]
+v_cmpx_eq_f16 v1.l, v2.l row_shl:15
+// GFX12: v_cmpx_eq_f16 v1.l, v2.l row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x04,0x7d,0x01,0x0f,0x01,0xff]
 
-v_cmpx_eq_f16 v1, v2 row_shr:1
-// GFX12: v_cmpx_eq_f16 v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x04,0x7d,0x01,0x11,0x01,0xff]
+v_cmpx_eq_f16 v1.l, v2.l row_shr:1
+// GFX12: v_cmpx_eq_f16 v1.l, v2.l row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x04,0x7d,0x01,0x11,0x01,0xff]
 
-v_cmpx_eq_f16 v1, v2 row_shr:15
-// GFX12: v_cmpx_eq_f16 v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x04,0x7d,0x01,0x1f,0x01,0xff]
+v_cmpx_eq_f16 v1.l, v2.l row_shr:15
+// GFX12: v_cmpx_eq_f16 v1.l, v2.l row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x04,0x7d,0x01,0x1f,0x01,0xff]
 
-v_cmpx_eq_f16 v1, v2 row_ror:1
-// GFX12: v_cmpx_eq_f16 v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x04,0x7d,0x01,0x21,0x01,0xff]
+v_cmpx_eq_f16 v1.l, v2.l row_ror:1
+// GFX12: v_cmpx_eq_f16 v1.l, v2.l row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x04,0x7d,0x01,0x21,0x01,0xff]
 
-v_cmpx_eq_f16 v1, v2 row_ror:15
-// GFX12: v_cmpx_eq_f16 v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x04,0x7d,0x01,0x2f,0x01,0xff]
+v_cmpx_eq_f16 v1.l, v2.l row_ror:15
+// GFX12: v_cmpx_eq_f16 v1.l, v2.l row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x04,0x7d,0x01,0x2f,0x01,0xff]
 
-v_cmpx_eq_f16 v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
-// GFX12: v_cmpx_eq_f16 v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x04,0x7d,0x01,0x50,0x01,0xff]
+v_cmpx_eq_f16 v1.l, v2.l row_share:0 row_mask:0xf bank_mask:0xf
+// GFX12: v_cmpx_eq_f16 v1.l, v2.l row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x04,0x7d,0x01,0x50,0x01,0xff]
 
-v_cmpx_eq_f16 v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
-// GFX12: v_cmpx_eq_f16 v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x04,0x7d,0x01,0x5f,0x01,0x01]
+v_cmpx_eq_f16 v1.l, v2.l row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX12: v_cmpx_eq_f16 v1.l, v2.l row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x04,0x7d,0x01,0x5f,0x01,0x01]
 
-v_cmpx_eq_f16 v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
-// GFX12: v_cmpx_eq_f16 v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0xfa,0x04,0x04,0x7d,0x01,0x60,0x09,0x13]
+v_cmpx_eq_f16 v1.l, v2.l row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1
+// GFX12: v_cmpx_eq_f16 v1.l, v2.l row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0xfa,0x04,0x04,0x7d,0x01,0x60,0x09,0x13]
 
-v_cmpx_eq_f16 -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
-// GFX12: v_cmpx_eq_f16 -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xfa,0xfe,0x04,0x7d,0x7f,0x6f,0xf5,0x30]
+v_cmpx_eq_f16 -|v127.l|, -|v127.l| row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1
+// GFX12: v_cmpx_eq_f16 -|v127.l|, -|v127.l| row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xfa,0xfe,0x04,0x7d,0x7f,0x6f,0xf5,0x30]
+
+v_cmpx_eq_f16 v1.h, v2.h row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX12: v_cmpx_eq_f16 v1.h, v2.h row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0xfa,0x04,0x05,0x7d,0x81,0x60,0x09,0x13]
+
+v_cmpx_eq_f16 -|v127.h|, -|v127.h| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX12: v_cmpx_eq_f16 -|v127.h|, -|v127.h| row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xfa,0xfe,0x05,0x7d,0xff,0x6f,0xf5,0x30]
 
 v_cmpx_eq_f32 v1, v2 quad_perm:[3,2,1,0]
 // GFX12: v_cmpx_eq_f32 v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x24,0x7d,0x01,0x1b,0x00,0xff]
@@ -356,47 +362,53 @@ v_cmpx_eq_u32 v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
 v_cmpx_eq_u32 v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
 // GFX12: v_cmpx_eq_u32 v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xfa,0xfe,0x95,0x7d,0xff,0x6f,0x05,0x30]
 
-v_cmpx_ge_f16 v1, v2 quad_perm:[3,2,1,0]
-// GFX12: v_cmpx_ge_f16 v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0c,0x7d,0x01,0x1b,0x00,0xff]
+v_cmpx_ge_f16 v1.l, v2.l quad_perm:[3,2,1,0]
+// GFX12: v_cmpx_ge_f16 v1.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0c,0x7d,0x01,0x1b,0x00,0xff]
+
+v_cmpx_ge_f16 v1.l, v2.l quad_perm:[0,1,2,3]
+// GFX12: v_cmpx_ge_f16 v1.l, v2.l quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0c,0x7d,0x01,0xe4,0x00,0xff]
+
+v_cmpx_ge_f16 v1.l, v2.l row_mirror
+// GFX12: v_cmpx_ge_f16 v1.l, v2.l row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0c,0x7d,0x01,0x40,0x01,0xff]
 
-v_cmpx_ge_f16 v1, v2 quad_perm:[0,1,2,3]
-// GFX12: v_cmpx_ge_f16 v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0c,0x7d,0x01,0xe4,0x00,0xff]
+v_cmpx_ge_f16 v1.l, v2.l row_half_mirror
+// GFX12: v_cmpx_ge_f16 v1.l, v2.l row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0c,0x7d,0x01,0x41,0x01,0xff]
 
-v_cmpx_ge_f16 v1, v2 row_mirror
-// GFX12: v_cmpx_ge_f16 v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0c,0x7d,0x01,0x40,0x01,0xff]
+v_cmpx_ge_f16 v1.l, v2.l row_shl:1
+// GFX12: v_cmpx_ge_f16 v1.l, v2.l row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0c,0x7d,0x01,0x01,0x01,0xff]
 
-v_cmpx_ge_f16 v1, v2 row_half_mirror
-// GFX12: v_cmpx_ge_f16 v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0c,0x7d,0x01,0x41,0x01,0xff]
+v_cmpx_ge_f16 v1.l, v2.l row_shl:15
+// GFX12: v_cmpx_ge_f16 v1.l, v2.l row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0c,0x7d,0x01,0x0f,0x01,0xff]
 
-v_cmpx_ge_f16 v1, v2 row_shl:1
-// GFX12: v_cmpx_ge_f16 v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0c,0x7d,0x01,0x01,0x01,0xff]
+v_cmpx_ge_f16 v1.l, v2.l row_shr:1
+// GFX12: v_cmpx_ge_f16 v1.l, v2.l row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0c,0x7d,0x01,0x11,0x01,0xff]
 
-v_cmpx_ge_f16 v1, v2 row_shl:15
-// GFX12: v_cmpx_ge_f16 v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0c,0x7d,0x01,0x0f,0x01,0xff]
+v_cmpx_ge_f16 v1.l, v2.l row_shr:15
+// GFX12: v_cmpx_ge_f16 v1.l, v2.l row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0c,0x7d,0x01,0x1f,0x01,0xff]
 
-v_cmpx_ge_f16 v1, v2 row_shr:1
-// GFX12: v_cmpx_ge_f16 v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0c,0x7d,0x01,0x11,0x01,0xff]
+v_cmpx_ge_f16 v1.l, v2.l row_ror:1
+// GFX12: v_cmpx_ge_f16 v1.l, v2.l row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0c,0x7d,0x01,0x21,0x01,0xff]
 
-v_cmpx_ge_f16 v1, v2 row_shr:15
-// GFX12: v_cmpx_ge_f16 v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0c,0x7d,0x01,0x1f,0x01,0xff]
+v_cmpx_ge_f16 v1.l, v2.l row_ror:15
+// GFX12: v_cmpx_ge_f16 v1.l, v2.l row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0c,0x7d,0x01,0x2f,0x01,0xff]
 
-v_cmpx_ge_f16 v1, v2 row_ror:1
-// GFX12: v_cmpx_ge_f16 v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0c,0x7d,0x01,0x21,0x01,0xff]
+v_cmpx_ge_f16 v1.l, v2.l row_share:0 row_mask:0xf bank_mask:0xf
+// GFX12: v_cmpx_ge_f16 v1.l, v2.l row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0c,0x7d,0x01,0x50,0x01,0xff]
 
-v_cmpx_ge_f16 v1, v2 row_ror:15
-// GFX12: v_cmpx_ge_f16 v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0c,0x7d,0x01,0x2f,0x01,0xff]
+v_cmpx_ge_f16 v1.l, v2.l row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX12: v_cmpx_ge_f16 v1.l, v2.l row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x0c,0x7d,0x01,0x5f,0x01,0x01]
 
-v_cmpx_ge_f16 v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
-// GFX12: v_cmpx_ge_f16 v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0c,0x7d,0x01,0x50,0x01,0xff]
+v_cmpx_ge_f16 v1.l, v2.l row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1
+// GFX12: v_cmpx_ge_f16 v1.l, v2.l row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0xfa,0x04,0x0c,0x7d,0x01,0x60,0x09,0x13]
 
-v_cmpx_ge_f16 v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
-// GFX12: v_cmpx_ge_f16 v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x0c,0x7d,0x01,0x5f,0x01,0x01]
+v_cmpx_ge_f16 -|v127.l|, -|v127.l| row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1
+// GFX12: v_cmpx_ge_f16 -|v127.l|, -|v127.l| row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xfa,0xfe,0x0c,0x7d,0x7f,0x6f,0xf5,0x30]
 
-v_cmpx_ge_f16 v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
-// GFX12: v_cmpx_ge_f16 v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0xfa,0x04,0x0c,0x7d,0x01,0x60,0x09,0x13]
+v_cmpx_ge_f16 v1.h, v2.h row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX12: v_cmpx_ge_f16 v1.h, v2.h row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0xfa,0x04,0x0d,0x7d,0x81,0x60,0x09,0x13]
 
-v_cmpx_ge_f16 -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
-// GFX12: v_cmpx_ge_f16 -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xfa,0xfe,0x0c,0x7d,0x7f,0x6f,0xf5,0x30]
+v_cmpx_ge_f16 -|v127.h|, -|v127.h| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX12: v_cmpx_ge_f16 -|v127.h|, -|v127.h| row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xfa,0xfe,0x0d,0x7d,0xff,0x6f,0xf5,0x30]
 
 v_cmpx_ge_f32 v1, v2 quad_perm:[3,2,1,0]
 // GFX12: v_cmpx_ge_f32 v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x2c,0x7d,0x01,0x1b,0x00,0xff]
@@ -620,47 +632,53 @@ v_cmpx_ge_u32 v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
 v_cmpx_ge_u32 v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
 // GFX12: v_cmpx_ge_u32 v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xfa,0xfe,0x9d,0x7d,0xff,0x6f,0x05,0x30]
 
-v_cmpx_gt_f16 v1, v2 quad_perm:[3,2,1,0]
-// GFX12: v_cmpx_gt_f16 v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x08,0x7d,0x01,0x1b,0x00,0xff]
+v_cmpx_gt_f16 v1.l, v2.l quad_perm:[3,2,1,0]
+// GFX12: v_cmpx_gt_f16 v1.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x08,0x7d,0x01,0x1b,0x00,0xff]
 
-v_cmpx_gt_f16 v1, v2 quad_perm:[0,1,2,3]
-// GFX12: v_cmpx_gt_f16 v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x08,0x7d,0x01,0xe4,0x00,0xff]
+v_cmpx_gt_f16 v1.l, v2.l quad_perm:[0,1,2,3]
+// GFX12: v_cmpx_gt_f16 v1.l, v2.l quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x08,0x7d,0x01,0xe4,0x00,0xff]
 
-v_cmpx_gt_f16 v1, v2 row_mirror
-// GFX12: v_cmpx_gt_f16 v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x08,0x7d,0x01,0x40,0x01,0xff]
+v_cmpx_gt_f16 v1.l, v2.l row_mirror
+// GFX12: v_cmpx_gt_f16 v1.l, v2.l row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x08,0x7d,0x01,0x40,0x01,0xff]
 
-v_cmpx_gt_f16 v1, v2 row_half_mirror
-// GFX12: v_cmpx_gt_f16 v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x08,0x7d,0x01,0x41,0x01,0xff]
+v_cmpx_gt_f16 v1.l, v2.l row_half_mirror
+// GFX12: v_cmpx_gt_f16 v1.l, v2.l row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x08,0x7d,0x01,0x41,0x01,0xff]
 
-v_cmpx_gt_f16 v1, v2 row_shl:1
-// GFX12: v_cmpx_gt_f16 v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x08,0x7d,0x01,0x01,0x01,0xff]
+v_cmpx_gt_f16 v1.l, v2.l row_shl:1
+// GFX12: v_cmpx_gt_f16 v1.l, v2.l row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x08,0x7d,0x01,0x01,0x01,0xff]
 
-v_cmpx_gt_f16 v1, v2 row_shl:15
-// GFX12: v_cmpx_gt_f16 v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x08,0x7d,0x01,0x0f,0x01,0xff]
+v_cmpx_gt_f16 v1.l, v2.l row_shl:15
+// GFX12: v_cmpx_gt_f16 v1.l, v2.l row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x08,0x7d,0x01,0x0f,0x01,0xff]
 
-v_cmpx_gt_f16 v1, v2 row_shr:1
-// GFX12: v_cmpx_gt_f16 v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x08,0x7d,0x01,0x11,0x01,0xff]
+v_cmpx_gt_f16 v1.l, v2.l row_shr:1
+// GFX12: v_cmpx_gt_f16 v1.l, v2.l row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x08,0x7d,0x01,0x11,0x01,0xff]
 
-v_cmpx_gt_f16 v1, v2 row_shr:15
-// GFX12: v_cmpx_gt_f16 v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x08,0x7d,0x01,0x1f,0x01,0xff]
+v_cmpx_gt_f16 v1.l, v2.l row_shr:15
+// GFX12: v_cmpx_gt_f16 v1.l, v2.l row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x08,0x7d,0x01,0x1f,0x01,0xff]
 
-v_cmpx_gt_f16 v1, v2 row_ror:1
-// GFX12: v_cmpx_gt_f16 v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x08,0x7d,0x01,0x21,0x01,0xff]
+v_cmpx_gt_f16 v1.l, v2.l row_ror:1
+// GFX12: v_cmpx_gt_f16 v1.l, v2.l row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x08,0x7d,0x01,0x21,0x01,0xff]
 
-v_cmpx_gt_f16 v1, v2 row_ror:15
-// GFX12: v_cmpx_gt_f16 v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x08,0x7d,0x01,0x2f,0x01,0xff]
+v_cmpx_gt_f16 v1.l, v2.l row_ror:15
+// GFX12: v_cmpx_gt_f16 v1.l, v2.l row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x08,0x7d,0x01,0x2f,0x01,0xff]
 
-v_cmpx_gt_f16 v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
-// GFX12: v_cmpx_gt_f16 v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x08,0x7d,0x01,0x50,0x01,0xff]
+v_cmpx_gt_f16 v1.l, v2.l row_share:0 row_mask:0xf bank_mask:0xf
+// GFX12: v_cmpx_gt_f16 v1.l, v2.l row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x08,0x7d,0x01,0x50,0x01,0xff]
 
-v_cmpx_gt_f16 v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
-// GFX12: v_cmpx_gt_f16 v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x08,0x7d,0x01,0x5f,0x01,0x01]
+v_cmpx_gt_f16 v1.l, v2.l row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX12: v_cmpx_gt_f16 v1.l, v2.l row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x08,0x7d,0x01,0x5f,0x01,0x01]
 
-v_cmpx_gt_f16 v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
-// GFX12: v_cmpx_gt_f16 v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0xfa,0x04,0x08,0x7d,0x01,0x60,0x09,0x13]
+v_cmpx_gt_f16 v1.l, v2.l row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1
+// GFX12: v_cmpx_gt_f16 v1.l, v2.l row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0xfa,0x04,0x08,0x7d,0x01,0x60,0x09,0x13]
 
-v_cmpx_gt_f16 -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
-// GFX12: v_cmpx_gt_f16 -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xfa,0xfe,0x08,0x7d,0x7f,0x6f,0xf5,0x30]
+v_cmpx_gt_f16 -|v127.l|, -|v127.l| row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1
+// GFX12: v_cmpx_gt_f16 -|v127.l|, -|v127.l| row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xfa,0xfe,0x08,0x7d,0x7f,0x6f,0xf5,0x30]
+
+v_cmpx_gt_f16 v1.h, v2.h row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX12: v_cmpx_gt_f16 v1.h, v2.h row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0xfa,0x04,0x09,0x7d,0x81,0x60,0x09,0x13]
+
+v_cmpx_gt_f16 -|v127.h|, -|v127.h| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX12: v_cmpx_gt_f16 -|v127.h|, -|v127.h| row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xfa,0xfe,0x09,0x7d,0xff,0x6f,0xf5,0x30]
 
 v_cmpx_gt_f32 v1, v2 quad_perm:[3,2,1,0]
 // GFX12: v_cmpx_gt_f32 v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x28,0x7d,0x01,0x1b,0x00,0xff]
@@ -884,47 +902,53 @@ v_cmpx_gt_u32 v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
 v_cmpx_gt_u32 v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
 // GFX12: v_cmpx_gt_u32 v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xfa,0xfe,0x99,0x7d,0xff,0x6f,0x05,0x30]
 
-v_cmpx_le_f16 v1, v2 quad_perm:[3,2,1,0]
-// GFX12: v_cmpx_le_f16 v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x06,0x7d,0x01,0x1b,0x00,0xff]
+v_cmpx_le_f16 v1.l, v2.l quad_perm:[3,2,1,0]
+// GFX12: v_cmpx_le_f16 v1.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x06,0x7d,0x01,0x1b,0x00,0xff]
+
+v_cmpx_le_f16 v1.l, v2.l quad_perm:[0,1,2,3]
+// GFX12: v_cmpx_le_f16 v1.l, v2.l quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x06,0x7d,0x01,0xe4,0x00,0xff]
 
-v_cmpx_le_f16 v1, v2 quad_perm:[0,1,2,3]
-// GFX12: v_cmpx_le_f16 v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x06,0x7d,0x01,0xe4,0x00,0xff]
+v_cmpx_le_f16 v1.l, v2.l row_mirror
+// GFX12: v_cmpx_le_f16 v1.l, v2.l row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x06,0x7d,0x01,0x40,0x01,0xff]
 
-v_cmpx_le_f16 v1, v2 row_mirror
-// GFX12: v_cmpx_le_f16 v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x06,0x7d,0x01,0x40,0x01,0xff]
+v_cmpx_le_f16 v1.l, v2.l row_half_mirror
+// GFX12: v_cmpx_le_f16 v1.l, v2.l row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x06,0x7d,0x01,0x41,0x01,0xff]
 
-v_cmpx_le_f16 v1, v2 row_half_mirror
-// GFX12: v_cmpx_le_f16 v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x06,0x7d,0x01,0x41,0x01,0xff]
+v_cmpx_le_f16 v1.l, v2.l row_shl:1
+// GFX12: v_cmpx_le_f16 v1.l, v2.l row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x06,0x7d,0x01,0x01,0x01,0xff]
 
-v_cmpx_le_f16 v1, v2 row_shl:1
-// GFX12: v_cmpx_le_f16 v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x06,0x7d,0x01,0x01,0x01,0xff]
+v_cmpx_le_f16 v1.l, v2.l row_shl:15
+// GFX12: v_cmpx_le_f16 v1.l, v2.l row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x06,0x7d,0x01,0x0f,0x01,0xff]
 
-v_cmpx_le_f16 v1, v2 row_shl:15
-// GFX12: v_cmpx_le_f16 v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x06,0x7d,0x01,0x0f,0x01,0xff]
+v_cmpx_le_f16 v1.l, v2.l row_shr:1
+// GFX12: v_cmpx_le_f16 v1.l, v2.l row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x06,0x7d,0x01,0x11,0x01,0xff]
 
-v_cmpx_le_f16 v1, v2 row_shr:1
-// GFX12: v_cmpx_le_f16 v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x06,0x7d,0x01,0x11,0x01,0xff]
+v_cmpx_le_f16 v1.l, v2.l row_shr:15
+// GFX12: v_cmpx_le_f16 v1.l, v2.l row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x06,0x7d,0x01,0x1f,0x01,0xff]
 
-v_cmpx_le_f16 v1, v2 row_shr:15
-// GFX12: v_cmpx_le_f16 v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x06,0x7d,0x01,0x1f,0x01,0xff]
+v_cmpx_le_f16 v1.l, v2.l row_ror:1
+// GFX12: v_cmpx_le_f16 v1.l, v2.l row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x06,0x7d,0x01,0x21,0x01,0xff]
 
-v_cmpx_le_f16 v1, v2 row_ror:1
-// GFX12: v_cmpx_le_f16 v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x06,0x7d,0x01,0x21,0x01,0xff]
+v_cmpx_le_f16 v1.l, v2.l row_ror:15
+// GFX12: v_cmpx_le_f16 v1.l, v2.l row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x06,0x7d,0x01,0x2f,0x01,0xff]
 
-v_cmpx_le_f16 v1, v2 row_ror:15
-// GFX12: v_cmpx_le_f16 v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x06,0x7d,0x01,0x2f,0x01,0xff]
+v_cmpx_le_f16 v1.l, v2.l row_share:0 row_mask:0xf bank_mask:0xf
+// GFX12: v_cmpx_le_f16 v1.l, v2.l row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x06,0x7d,0x01,0x50,0x01,0xff]
 
-v_cmpx_le_f16 v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
-// GFX12: v_cmpx_le_f16 v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x06,0x7d,0x01,0x50,0x01,0xff]
+v_cmpx_le_f16 v1.l, v2.l row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX12: v_cmpx_le_f16 v1.l, v2.l row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x06,0x7d,0x01,0x5f,0x01,0x01]
 
-v_cmpx_le_f16 v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
-// GFX12: v_cmpx_le_f16 v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x06,0x7d,0x01,0x5f,0x01,0x01]
+v_cmpx_le_f16 v1.l, v2.l row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1
+// GFX12: v_cmpx_le_f16 v1.l, v2.l row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0xfa,0x04,0x06,0x7d,0x01,0x60,0x09,0x13]
 
-v_cmpx_le_f16 v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
-// GFX12: v_cmpx_le_f16 v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0xfa,0x04,0x06,0x7d,0x01,0x60,0x09,0x13]
+v_cmpx_le_f16 -|v127.l|, -|v127.l| row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1
+// GFX12: v_cmpx_le_f16 -|v127.l|, -|v127.l| row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xfa,0xfe,0x06,0x7d,0x7f,0x6f,0xf5,0x30]
 
-v_cmpx_le_f16 -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
-// GFX12: v_cmpx_le_f16 -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xfa,0xfe,0x06,0x7d,0x7f,0x6f,0xf5,0x30]
+v_cmpx_le_f16 v1.h, v2.h row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX12: v_cmpx_le_f16 v1.h, v2.h row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0xfa,0x04,0x07,0x7d,0x81,0x60,0x09,0x13]
+
+v_cmpx_le_f16 -|v127.h|, -|v127.h| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX12: v_cmpx_le_f16 -|v127.h|, -|v127.h| row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xfa,0xfe,0x07,0x7d,0xff,0x6f,0xf5,0x30]
 
 v_cmpx_le_f32 v1, v2 quad_perm:[3,2,1,0]
 // GFX12: v_cmpx_le_f32 v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x26,0x7d,0x01,0x1b,0x00,0xff]
@@ -1148,47 +1172,53 @@ v_cmpx_le_u32 v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
 v_cmpx_le_u32 v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
 // GFX12: v_cmpx_le_u32 v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xfa,0xfe,0x97,0x7d,0xff,0x6f,0x05,0x30]
 
-v_cmpx_lg_f16 v1, v2 quad_perm:[3,2,1,0]
-// GFX12: v_cmpx_lg_f16 v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x7d,0x01,0x1b,0x00,0xff]
+v_cmpx_lg_f16 v1.l, v2.l quad_perm:[3,2,1,0]
+// GFX12: v_cmpx_lg_f16 v1.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x7d,0x01,0x1b,0x00,0xff]
+
+v_cmpx_lg_f16 v1.l, v2.l quad_perm:[0,1,2,3]
+// GFX12: v_cmpx_lg_f16 v1.l, v2.l quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x7d,0x01,0xe4,0x00,0xff]
+
+v_cmpx_lg_f16 v1.l, v2.l row_mirror
+// GFX12: v_cmpx_lg_f16 v1.l, v2.l row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x7d,0x01,0x40,0x01,0xff]
 
-v_cmpx_lg_f16 v1, v2 quad_perm:[0,1,2,3]
-// GFX12: v_cmpx_lg_f16 v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x7d,0x01,0xe4,0x00,0xff]
+v_cmpx_lg_f16 v1.l, v2.l row_half_mirror
+// GFX12: v_cmpx_lg_f16 v1.l, v2.l row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x7d,0x01,0x41,0x01,0xff]
 
-v_cmpx_lg_f16 v1, v2 row_mirror
-// GFX12: v_cmpx_lg_f16 v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x7d,0x01,0x40,0x01,0xff]
+v_cmpx_lg_f16 v1.l, v2.l row_shl:1
+// GFX12: v_cmpx_lg_f16 v1.l, v2.l row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x7d,0x01,0x01,0x01,0xff]
 
-v_cmpx_lg_f16 v1, v2 row_half_mirror
-// GFX12: v_cmpx_lg_f16 v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x7d,0x01,0x41,0x01,0xff]
+v_cmpx_lg_f16 v1.l, v2.l row_shl:15
+// GFX12: v_cmpx_lg_f16 v1.l, v2.l row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x7d,0x01,0x0f,0x01,0xff]
 
-v_cmpx_lg_f16 v1, v2 row_shl:1
-// GFX12: v_cmpx_lg_f16 v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x7d,0x01,0x01,0x01,0xff]
+v_cmpx_lg_f16 v1.l, v2.l row_shr:1
+// GFX12: v_cmpx_lg_f16 v1.l, v2.l row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x7d,0x01,0x11,0x01,0xff]
 
-v_cmpx_lg_f16 v1, v2 row_shl:15
-// GFX12: v_cmpx_lg_f16 v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x7d,0x01,0x0f,0x01,0xff]
+v_cmpx_lg_f16 v1.l, v2.l row_shr:15
+// GFX12: v_cmpx_lg_f16 v1.l, v2.l row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x7d,0x01,0x1f,0x01,0xff]
 
-v_cmpx_lg_f16 v1, v2 row_shr:1
-// GFX12: v_cmpx_lg_f16 v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x7d,0x01,0x11,0x01,0xff]
+v_cmpx_lg_f16 v1.l, v2.l row_ror:1
+// GFX12: v_cmpx_lg_f16 v1.l, v2.l row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x7d,0x01,0x21,0x01,0xff]
 
-v_cmpx_lg_f16 v1, v2 row_shr:15
-// GFX12: v_cmpx_lg_f16 v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x7d,0x01,0x1f,0x01,0xff]
+v_cmpx_lg_f16 v1.l, v2.l row_ror:15
+// GFX12: v_cmpx_lg_f16 v1.l, v2.l row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x7d,0x01,0x2f,0x01,0xff]
 
-v_cmpx_lg_f16 v1, v2 row_ror:1
-// GFX12: v_cmpx_lg_f16 v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x7d,0x01,0x21,0x01,0xff]
+v_cmpx_lg_f16 v1.l, v2.l row_share:0 row_mask:0xf bank_mask:0xf
+// GFX12: v_cmpx_lg_f16 v1.l, v2.l row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x7d,0x01,0x50,0x01,0xff]
 
-v_cmpx_lg_f16 v1, v2 row_ror:15
-// GFX12: v_cmpx_lg_f16 v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x7d,0x01,0x2f,0x01,0xff]
+v_cmpx_lg_f16 v1.l, v2.l row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX12: v_cmpx_lg_f16 v1.l, v2.l row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x0a,0x7d,0x01,0x5f,0x01,0x01]
 
-v_cmpx_lg_f16 v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
-// GFX12: v_cmpx_lg_f16 v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x7d,0x01,0x50,0x01,0xff]
+v_cmpx_lg_f16 v1.l, v2.l row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1
+// GFX12: v_cmpx_lg_f16 v1.l, v2.l row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0xfa,0x04,0x0a,0x7d,0x01,0x60,0x09,0x13]
 
-v_cmpx_lg_f16 v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
-// GFX12: v_cmpx_lg_f16 v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x0a,0x7d,0x01,0x5f,0x01,0x01]
+v_cmpx_lg_f16 -|v127.l|, -|v127.l| row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1
+// GFX12: v_cmpx_lg_f16 -|v127.l|, -|v127.l| row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xfa,0xfe,0x0a,0x7d,0x7f,0x6f,0xf5,0x30]
 
-v_cmpx_lg_f16 v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
-// GFX12: v_cmpx_lg_f16 v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0xfa,0x04,0x0a,0x7d,0x01,0x60,0x09,0x13]
+v_cmpx_lg_f16 v1.h, v2.h row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX12: v_cmpx_lg_f16 v1.h, v2.h row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0xfa,0x04,0x0b,0x7d,0x81,0x60,0x09,0x13]
 
-v_cmpx_lg_f16 -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
-// GFX12: v_cmpx_lg_f16 -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xfa,0xfe,0x0a,0x7d,0x7f,0x6f,0xf5,0x30]
+v_cmpx_lg_f16 -|v127.h|, -|v127.h| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX12: v_cmpx_lg_f16 -|v127.h|, -|v127.h| row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xfa,0xfe,0x0b,0x7d,0xff,0x6f,0xf5,0x30]
 
 v_cmpx_lg_f32 v1, v2 quad_perm:[3,2,1,0]
 // GFX12: v_cmpx_lg_f32 v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x2a,0x7d,0x01,0x1b,0x00,0xff]
@@ -1682,47 +1712,53 @@ v_cmpx_ne_u32 v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
 v_cmpx_ne_u32 v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
 // GFX12: v_cmpx_ne_u32 v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xfa,0xfe,0x9b,0x7d,0xff,0x6f,0x05,0x30]
 
-v_cmpx_neq_f16 v1, v2 quad_perm:[3,2,1,0]
-// GFX12: v_cmpx_neq_f16 v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1a,0x7d,0x01,0x1b,0x00,0xff]
+v_cmpx_neq_f16 v1.l, v2.l quad_perm:[3,2,1,0]
+// GFX12: v_cmpx_neq_f16 v1.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1a,0x7d,0x01,0x1b,0x00,0xff]
 
-v_cmpx_neq_f16 v1, v2 quad_perm:[0,1,2,3]
-// GFX12: v_cmpx_neq_f16 v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1a,0x7d,0x01,0xe4,0x00,0xff]
+v_cmpx_neq_f16 v1.l, v2.l quad_perm:[0,1,2,3]
+// GFX12: v_cmpx_neq_f16 v1.l, v2.l quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1a,0x7d,0x01,0xe4,0x00,0xff]
 
-v_cmpx_neq_f16 v1, v2 row_mirror
-// GFX12: v_cmpx_neq_f16 v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1a,0x7d,0x01,0x40,0x01,0xff]
+v_cmpx_neq_f16 v1.l, v2.l row_mirror
+// GFX12: v_cmpx_neq_f16 v1.l, v2.l row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1a,0x7d,0x01,0x40,0x01,0xff]
 
-v_cmpx_neq_f16 v1, v2 row_half_mirror
-// GFX12: v_cmpx_neq_f16 v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1a,0x7d,0x01,0x41,0x01,0xff]
+v_cmpx_neq_f16 v1.l, v2.l row_half_mirror
+// GFX12: v_cmpx_neq_f16 v1.l, v2.l row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1a,0x7d,0x01,0x41,0x01,0xff]
 
-v_cmpx_neq_f16 v1, v2 row_shl:1
-// GFX12: v_cmpx_neq_f16 v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1a,0x7d,0x01,0x01,0x01,0xff]
+v_cmpx_neq_f16 v1.l, v2.l row_shl:1
+// GFX12: v_cmpx_neq_f16 v1.l, v2.l row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1a,0x7d,0x01,0x01,0x01,0xff]
 
-v_cmpx_neq_f16 v1, v2 row_shl:15
-// GFX12: v_cmpx_neq_f16 v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1a,0x7d,0x01,0x0f,0x01,0xff]
+v_cmpx_neq_f16 v1.l, v2.l row_shl:15
+// GFX12: v_cmpx_neq_f16 v1.l, v2.l row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1a,0x7d,0x01,0x0f,0x01,0xff]
 
-v_cmpx_neq_f16 v1, v2 row_shr:1
-// GFX12: v_cmpx_neq_f16 v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1a,0x7d,0x01,0x11,0x01,0xff]
+v_cmpx_neq_f16 v1.l, v2.l row_shr:1
+// GFX12: v_cmpx_neq_f16 v1.l, v2.l row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1a,0x7d,0x01,0x11,0x01,0xff]
 
-v_cmpx_neq_f16 v1, v2 row_shr:15
-// GFX12: v_cmpx_neq_f16 v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1a,0x7d,0x01,0x1f,0x01,0xff]
+v_cmpx_neq_f16 v1.l, v2.l row_shr:15
+// GFX12: v_cmpx_neq_f16 v1.l, v2.l row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1a,0x7d,0x01,0x1f,0x01,0xff]
 
-v_cmpx_neq_f16 v1, v2 row_ror:1
-// GFX12: v_cmpx_neq_f16 v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1a,0x7d,0x01,0x21,0x01,0xff]
+v_cmpx_neq_f16 v1.l, v2.l row_ror:1
+// GFX12: v_cmpx_neq_f16 v1.l, v2.l row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1a,0x7d,0x01,0x21,0x01,0xff]
 
-v_cmpx_neq_f16 v1, v2 row_ror:15
-// GFX12: v_cmpx_neq_f16 v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1a,0x7d,0x01,0x2f,0x01,0xff]
+v_cmpx_neq_f16 v1.l, v2.l row_ror:15
+// GFX12: v_cmpx_neq_f16 v1.l, v2.l row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1a,0x7d,0x01,0x2f,0x01,0xff]
 
-v_cmpx_neq_f16 v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
-// GFX12: v_cmpx_neq_f16 v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1a,0x7d,0x01,0x50,0x01,0xff]
+v_cmpx_neq_f16 v1.l, v2.l row_share:0 row_mask:0xf bank_mask:0xf
+// GFX12: v_cmpx_neq_f16 v1.l, v2.l row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1a,0x7d,0x01,0x50,0x01,0xff]
 
-v_cmpx_neq_f16 v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
-// GFX12: v_cmpx_neq_f16 v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x1a,0x7d,0x01,0x5f,0x01,0x01]
+v_cmpx_neq_f16 v1.l, v2.l row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX12: v_cmpx_neq_f16 v1.l, v2.l row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x1a,0x7d,0x01,0x5f,0x01,0x01]
 
-v_cmpx_neq_f16 v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
-// GFX12: v_cmpx_neq_f16 v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0xfa,0x04,0x1a,0x7d,0x01,0x60,0x09,0x13]
+v_cmpx_neq_f16 v1.l, v2.l row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1
+// GFX12: v_cmpx_neq_f16 v1.l, v2.l row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0xfa,0x04,0x1a,0x7d,0x01,0x60,0x09,0x13]
 
-v_cmpx_neq_f16 -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
-// GFX12: v_cmpx_neq_f16 -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xfa,0xfe,0x1a,0x7d,0x7f,0x6f,0xf5,0x30]
+v_cmpx_neq_f16 -|v127.l|, -|v127.l| row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1
+// GFX12: v_cmpx_neq_f16 -|v127.l|, -|v127.l| row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xfa,0xfe,0x1a,0x7d,0x7f,0x6f,0xf5,0x30]
+
+v_cmpx_neq_f16 v1.h, v2.h row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX12: v_cmpx_neq_f16 v1.h, v2.h row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0xfa,0x04,0x1b,0x7d,0x81,0x60,0x09,0x13]
+
+v_cmpx_neq_f16 -|v127.h|, -|v127.h| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX12: v_cmpx_neq_f16 -|v127.h|, -|v127.h| row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xfa,0xfe,0x1b,0x7d,0xff,0x6f,0xf5,0x30]
 
 v_cmpx_neq_f32 v1, v2 quad_perm:[3,2,1,0]
 // GFX12: v_cmpx_neq_f32 v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x3a,0x7d,0x01,0x1b,0x00,0xff]
@@ -1766,47 +1802,53 @@ v_cmpx_neq_f32 v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
 v_cmpx_neq_f32 -|v255|, -|v255| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
 // GFX12: v_cmpx_neq_f32 -|v255|, -|v255| row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xfa,0xfe,0x3b,0x7d,0xff,0x6f,0xf5,0x30]
 
-v_cmpx_nge_f16 v1, v2 quad_perm:[3,2,1,0]
-// GFX12: v_cmpx_nge_f16 v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x12,0x7d,0x01,0x1b,0x00,0xff]
+v_cmpx_nge_f16 v1.l, v2.l quad_perm:[3,2,1,0]
+// GFX12: v_cmpx_nge_f16 v1.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x12,0x7d,0x01,0x1b,0x00,0xff]
+
+v_cmpx_nge_f16 v1.l, v2.l quad_perm:[0,1,2,3]
+// GFX12: v_cmpx_nge_f16 v1.l, v2.l quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x12,0x7d,0x01,0xe4,0x00,0xff]
 
-v_cmpx_nge_f16 v1, v2 quad_perm:[0,1,2,3]
-// GFX12: v_cmpx_nge_f16 v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x12,0x7d,0x01,0xe4,0x00,0xff]
+v_cmpx_nge_f16 v1.l, v2.l row_mirror
+// GFX12: v_cmpx_nge_f16 v1.l, v2.l row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x12,0x7d,0x01,0x40,0x01,0xff]
 
-v_cmpx_nge_f16 v1, v2 row_mirror
-// GFX12: v_cmpx_nge_f16 v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x12,0x7d,0x01,0x40,0x01,0xff]
+v_cmpx_nge_f16 v1.l, v2.l row_half_mirror
+// GFX12: v_cmpx_nge_f16 v1.l, v2.l row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x12,0x7d,0x01,0x41,0x01,0xff]
 
-v_cmpx_nge_f16 v1, v2 row_half_mirror
-// GFX12: v_cmpx_nge_f16 v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x12,0x7d,0x01,0x41,0x01,0xff]
+v_cmpx_nge_f16 v1.l, v2.l row_shl:1
+// GFX12: v_cmpx_nge_f16 v1.l, v2.l row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x12,0x7d,0x01,0x01,0x01,0xff]
 
-v_cmpx_nge_f16 v1, v2 row_shl:1
-// GFX12: v_cmpx_nge_f16 v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x12,0x7d,0x01,0x01,0x01,0xff]
+v_cmpx_nge_f16 v1.l, v2.l row_shl:15
+// GFX12: v_cmpx_nge_f16 v1.l, v2.l row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x12,0x7d,0x01,0x0f,0x01,0xff]
 
-v_cmpx_nge_f16 v1, v2 row_shl:15
-// GFX12: v_cmpx_nge_f16 v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x12,0x7d,0x01,0x0f,0x01,0xff]
+v_cmpx_nge_f16 v1.l, v2.l row_shr:1
+// GFX12: v_cmpx_nge_f16 v1.l, v2.l row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x12,0x7d,0x01,0x11,0x01,0xff]
 
-v_cmpx_nge_f16 v1, v2 row_shr:1
-// GFX12: v_cmpx_nge_f16 v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x12,0x7d,0x01,0x11,0x01,0xff]
+v_cmpx_nge_f16 v1.l, v2.l row_shr:15
+// GFX12: v_cmpx_nge_f16 v1.l, v2.l row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x12,0x7d,0x01,0x1f,0x01,0xff]
 
-v_cmpx_nge_f16 v1, v2 row_shr:15
-// GFX12: v_cmpx_nge_f16 v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x12,0x7d,0x01,0x1f,0x01,0xff]
+v_cmpx_nge_f16 v1.l, v2.l row_ror:1
+// GFX12: v_cmpx_nge_f16 v1.l, v2.l row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x12,0x7d,0x01,0x21,0x01,0xff]
 
-v_cmpx_nge_f16 v1, v2 row_ror:1
-// GFX12: v_cmpx_nge_f16 v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x12,0x7d,0x01,0x21,0x01,0xff]
+v_cmpx_nge_f16 v1.l, v2.l row_ror:15
+// GFX12: v_cmpx_nge_f16 v1.l, v2.l row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x12,0x7d,0x01,0x2f,0x01,0xff]
 
-v_cmpx_nge_f16 v1, v2 row_ror:15
-// GFX12: v_cmpx_nge_f16 v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x12,0x7d,0x01,0x2f,0x01,0xff]
+v_cmpx_nge_f16 v1.l, v2.l row_share:0 row_mask:0xf bank_mask:0xf
+// GFX12: v_cmpx_nge_f16 v1.l, v2.l row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x12,0x7d,0x01,0x50,0x01,0xff]
 
-v_cmpx_nge_f16 v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
-// GFX12: v_cmpx_nge_f16 v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x12,0x7d,0x01,0x50,0x01,0xff]
+v_cmpx_nge_f16 v1.l, v2.l row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX12: v_cmpx_nge_f16 v1.l, v2.l row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x12,0x7d,0x01,0x5f,0x01,0x01]
 
-v_cmpx_nge_f16 v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
-// GFX12: v_cmpx_nge_f16 v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x12,0x7d,0x01,0x5f,0x01,0x01]
+v_cmpx_nge_f16 v1.l, v2.l row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1
+// GFX12: v_cmpx_nge_f16 v1.l, v2.l row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0xfa,0x04,0x12,0x7d,0x01,0x60,0x09,0x13]
 
-v_cmpx_nge_f16 v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
-// GFX12: v_cmpx_nge_f16 v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0xfa,0x04,0x12,0x7d,0x01,0x60,0x09,0x13]
+v_cmpx_nge_f16 -|v127.l|, -|v127.l| row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1
+// GFX12: v_cmpx_nge_f16 -|v127.l|, -|v127.l| row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xfa,0xfe,0x12,0x7d,0x7f,0x6f,0xf5,0x30]
 
-v_cmpx_nge_f16 -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
-// GFX12: v_cmpx_nge_f16 -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xfa,0xfe,0x12,0x7d,0x7f,0x6f,0xf5,0x30]
+v_cmpx_nge_f16 v1.h, v2.h row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX12: v_cmpx_nge_f16 v1.h, v2.h row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0xfa,0x04,0x13,0x7d,0x81,0x60,0x09,0x13]
+
+v_cmpx_nge_f16 -|v127.h|, -|v127.h| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX12: v_cmpx_nge_f16 -|v127.h|, -|v127.h| row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xfa,0xfe,0x13,0x7d,0xff,0x6f,0xf5,0x30]
 
 v_cmpx_nge_f32 v1, v2 quad_perm:[3,2,1,0]
 // GFX12: v_cmpx_nge_f32 v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x32,0x7d,0x01,0x1b,0x00,0xff]
@@ -1850,47 +1892,53 @@ v_cmpx_nge_f32 v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
 v_cmpx_nge_f32 -|v255|, -|v255| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
 // GFX12: v_cmpx_nge_f32 -|v255|, -|v255| row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xfa,0xfe,0x33,0x7d,0xff,0x6f,0xf5,0x30]
 
-v_cmpx_ngt_f16 v1, v2 quad_perm:[3,2,1,0]
-// GFX12: v_cmpx_ngt_f16 v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x16,0x7d,0x01,0x1b,0x00,0xff]
+v_cmpx_ngt_f16 v1.l, v2.l quad_perm:[3,2,1,0]
+// GFX12: v_cmpx_ngt_f16 v1.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x16,0x7d,0x01,0x1b,0x00,0xff]
+
+v_cmpx_ngt_f16 v1.l, v2.l quad_perm:[0,1,2,3]
+// GFX12: v_cmpx_ngt_f16 v1.l, v2.l quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x16,0x7d,0x01,0xe4,0x00,0xff]
 
-v_cmpx_ngt_f16 v1, v2 quad_perm:[0,1,2,3]
-// GFX12: v_cmpx_ngt_f16 v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x16,0x7d,0x01,0xe4,0x00,0xff]
+v_cmpx_ngt_f16 v1.l, v2.l row_mirror
+// GFX12: v_cmpx_ngt_f16 v1.l, v2.l row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x16,0x7d,0x01,0x40,0x01,0xff]
 
-v_cmpx_ngt_f16 v1, v2 row_mirror
-// GFX12: v_cmpx_ngt_f16 v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x16,0x7d,0x01,0x40,0x01,0xff]
+v_cmpx_ngt_f16 v1.l, v2.l row_half_mirror
+// GFX12: v_cmpx_ngt_f16 v1.l, v2.l row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x16,0x7d,0x01,0x41,0x01,0xff]
 
-v_cmpx_ngt_f16 v1, v2 row_half_mirror
-// GFX12: v_cmpx_ngt_f16 v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x16,0x7d,0x01,0x41,0x01,0xff]
+v_cmpx_ngt_f16 v1.l, v2.l row_shl:1
+// GFX12: v_cmpx_ngt_f16 v1.l, v2.l row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x16,0x7d,0x01,0x01,0x01,0xff]
 
-v_cmpx_ngt_f16 v1, v2 row_shl:1
-// GFX12: v_cmpx_ngt_f16 v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x16,0x7d,0x01,0x01,0x01,0xff]
+v_cmpx_ngt_f16 v1.l, v2.l row_shl:15
+// GFX12: v_cmpx_ngt_f16 v1.l, v2.l row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x16,0x7d,0x01,0x0f,0x01,0xff]
 
-v_cmpx_ngt_f16 v1, v2 row_shl:15
-// GFX12: v_cmpx_ngt_f16 v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x16,0x7d,0x01,0x0f,0x01,0xff]
+v_cmpx_ngt_f16 v1.l, v2.l row_shr:1
+// GFX12: v_cmpx_ngt_f16 v1.l, v2.l row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x16,0x7d,0x01,0x11,0x01,0xff]
 
-v_cmpx_ngt_f16 v1, v2 row_shr:1
-// GFX12: v_cmpx_ngt_f16 v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x16,0x7d,0x01,0x11,0x01,0xff]
+v_cmpx_ngt_f16 v1.l, v2.l row_shr:15
+// GFX12: v_cmpx_ngt_f16 v1.l, v2.l row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x16,0x7d,0x01,0x1f,0x01,0xff]
 
-v_cmpx_ngt_f16 v1, v2 row_shr:15
-// GFX12: v_cmpx_ngt_f16 v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x16,0x7d,0x01,0x1f,0x01,0xff]
+v_cmpx_ngt_f16 v1.l, v2.l row_ror:1
+// GFX12: v_cmpx_ngt_f16 v1.l, v2.l row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x16,0x7d,0x01,0x21,0x01,0xff]
 
-v_cmpx_ngt_f16 v1, v2 row_ror:1
-// GFX12: v_cmpx_ngt_f16 v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x16,0x7d,0x01,0x21,0x01,0xff]
+v_cmpx_ngt_f16 v1.l, v2.l row_ror:15
+// GFX12: v_cmpx_ngt_f16 v1.l, v2.l row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x16,0x7d,0x01,0x2f,0x01,0xff]
 
-v_cmpx_ngt_f16 v1, v2 row_ror:15
-// GFX12: v_cmpx_ngt_f16 v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x16,0x7d,0x01,0x2f,0x01,0xff]
+v_cmpx_ngt_f16 v1.l, v2.l row_share:0 row_mask:0xf bank_mask:0xf
+// GFX12: v_cmpx_ngt_f16 v1.l, v2.l row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x16,0x7d,0x01,0x50,0x01,0xff]
 
-v_cmpx_ngt_f16 v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
-// GFX12: v_cmpx_ngt_f16 v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x16,0x7d,0x01,0x50,0x01,0xff]
+v_cmpx_ngt_f16 v1.l, v2.l row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX12: v_cmpx_ngt_f16 v1.l, v2.l row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x16,0x7d,0x01,0x5f,0x01,0x01]
 
-v_cmpx_ngt_f16 v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
-// GFX12: v_cmpx_ngt_f16 v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x16,0x7d,0x01,0x5f,0x01,0x01]
+v_cmpx_ngt_f16 v1.l, v2.l row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1
+// GFX12: v_cmpx_ngt_f16 v1.l, v2.l row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0xfa,0x04,0x16,0x7d,0x01,0x60,0x09,0x13]
 
-v_cmpx_ngt_f16 v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
-// GFX12: v_cmpx_ngt_f16 v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0xfa,0x04,0x16,0x7d,0x01,0x60,0x09,0x13]
+v_cmpx_ngt_f16 -|v127.l|, -|v127.l| row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1
+// GFX12: v_cmpx_ngt_f16 -|v127.l|, -|v127.l| row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xfa,0xfe,0x16,0x7d,0x7f,0x6f,0xf5,0x30]
 
-v_cmpx_ngt_f16 -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
-// GFX12: v_cmpx_ngt_f16 -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xfa,0xfe,0x16,0x7d,0x7f,0x6f,0xf5,0x30]
+v_cmpx_ngt_f16 v1.h, v2.h row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX12: v_cmpx_ngt_f16 v1.h, v2.h row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0xfa,0x04,0x17,0x7d,0x81,0x60,0x09,0x13]
+
+v_cmpx_ngt_f16 -|v127.h|, -|v127.h| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX12: v_cmpx_ngt_f16 -|v127.h|, -|v127.h| row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xfa,0xfe,0x17,0x7d,0xff,0x6f,0xf5,0x30]
 
 v_cmpx_ngt_f32 v1, v2 quad_perm:[3,2,1,0]
 // GFX12: v_cmpx_ngt_f32 v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x36,0x7d,0x01,0x1b,0x00,0xff]
@@ -1934,47 +1982,53 @@ v_cmpx_ngt_f32 v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
 v_cmpx_ngt_f32 -|v255|, -|v255| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
 // GFX12: v_cmpx_ngt_f32 -|v255|, -|v255| row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xfa,0xfe,0x37,0x7d,0xff,0x6f,0xf5,0x30]
 
-v_cmpx_nle_f16 v1, v2 quad_perm:[3,2,1,0]
-// GFX12: v_cmpx_nle_f16 v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x18,0x7d,0x01,0x1b,0x00,0xff]
+v_cmpx_nle_f16 v1.l, v2.l quad_perm:[3,2,1,0]
+// GFX12: v_cmpx_nle_f16 v1.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x18,0x7d,0x01,0x1b,0x00,0xff]
+
+v_cmpx_nle_f16 v1.l, v2.l quad_perm:[0,1,2,3]
+// GFX12: v_cmpx_nle_f16 v1.l, v2.l quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x18,0x7d,0x01,0xe4,0x00,0xff]
 
-v_cmpx_nle_f16 v1, v2 quad_perm:[0,1,2,3]
-// GFX12: v_cmpx_nle_f16 v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x18,0x7d,0x01,0xe4,0x00,0xff]
+v_cmpx_nle_f16 v1.l, v2.l row_mirror
+// GFX12: v_cmpx_nle_f16 v1.l, v2.l row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x18,0x7d,0x01,0x40,0x01,0xff]
 
-v_cmpx_nle_f16 v1, v2 row_mirror
-// GFX12: v_cmpx_nle_f16 v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x18,0x7d,0x01,0x40,0x01,0xff]
+v_cmpx_nle_f16 v1.l, v2.l row_half_mirror
+// GFX12: v_cmpx_nle_f16 v1.l, v2.l row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x18,0x7d,0x01,0x41,0x01,0xff]
 
-v_cmpx_nle_f16 v1, v2 row_half_mirror
-// GFX12: v_cmpx_nle_f16 v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x18,0x7d,0x01,0x41,0x01,0xff]
+v_cmpx_nle_f16 v1.l, v2.l row_shl:1
+// GFX12: v_cmpx_nle_f16 v1.l, v2.l row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x18,0x7d,0x01,0x01,0x01,0xff]
 
-v_cmpx_nle_f16 v1, v2 row_shl:1
-// GFX12: v_cmpx_nle_f16 v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x18,0x7d,0x01,0x01,0x01,0xff]
+v_cmpx_nle_f16 v1.l, v2.l row_shl:15
+// GFX12: v_cmpx_nle_f16 v1.l, v2.l row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x18,0x7d,0x01,0x0f,0x01,0xff]
 
-v_cmpx_nle_f16 v1, v2 row_shl:15
-// GFX12: v_cmpx_nle_f16 v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x18,0x7d,0x01,0x0f,0x01,0xff]
+v_cmpx_nle_f16 v1.l, v2.l row_shr:1
+// GFX12: v_cmpx_nle_f16 v1.l, v2.l row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x18,0x7d,0x01,0x11,0x01,0xff]
 
-v_cmpx_nle_f16 v1, v2 row_shr:1
-// GFX12: v_cmpx_nle_f16 v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x18,0x7d,0x01,0x11,0x01,0xff]
+v_cmpx_nle_f16 v1.l, v2.l row_shr:15
+// GFX12: v_cmpx_nle_f16 v1.l, v2.l row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x18,0x7d,0x01,0x1f,0x01,0xff]
 
-v_cmpx_nle_f16 v1, v2 row_shr:15
-// GFX12: v_cmpx_nle_f16 v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x18,0x7d,0x01,0x1f,0x01,0xff]
+v_cmpx_nle_f16 v1.l, v2.l row_ror:1
+// GFX12: v_cmpx_nle_f16 v1.l, v2.l row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x18,0x7d,0x01,0x21,0x01,0xff]
 
-v_cmpx_nle_f16 v1, v2 row_ror:1
-// GFX12: v_cmpx_nle_f16 v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x18,0x7d,0x01,0x21,0x01,0xff]
+v_cmpx_nle_f16 v1.l, v2.l row_ror:15
+// GFX12: v_cmpx_nle_f16 v1.l, v2.l row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x18,0x7d,0x01,0x2f,0x01,0xff]
 
-v_cmpx_nle_f16 v1, v2 row_ror:15
-// GFX12: v_cmpx_nle_f16 v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x18,0x7d,0x01,0x2f,0x01,0xff]
+v_cmpx_nle_f16 v1.l, v2.l row_share:0 row_mask:0xf bank_mask:0xf
+// GFX12: v_cmpx_nle_f16 v1.l, v2.l row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x18,0x7d,0x01,0x50,0x01,0xff]
 
-v_cmpx_nle_f16 v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
-// GFX12: v_cmpx_nle_f16 v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x18,0x7d,0x01,0x50,0x01,0xff]
+v_cmpx_nle_f16 v1.l, v2.l row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX12: v_cmpx_nle_f16 v1.l, v2.l row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x18,0x7d,0x01,0x5f,0x01,0x01]
 
-v_cmpx_nle_f16 v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
-// GFX12: v_cmpx_nle_f16 v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x18,0x7d,0x01,0x5f,0x01,0x01]
+v_cmpx_nle_f16 v1.l, v2.l row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1
+// GFX12: v_cmpx_nle_f16 v1.l, v2.l row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0xfa,0x04,0x18,0x7d,0x01,0x60,0x09,0x13]
 
-v_cmpx_nle_f16 v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
-// GFX12: v_cmpx_nle_f16 v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0xfa,0x04,0x18,0x7d,0x01,0x60,0x09,0x13]
+v_cmpx_nle_f16 -|v127.l|, -|v127.l| row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1
+// GFX12: v_cmpx_nle_f16 -|v127.l|, -|v127.l| row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xfa,0xfe,0x18,0x7d,0x7f,0x6f,0xf5,0x30]
 
-v_cmpx_nle_f16 -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
-// GFX12: v_cmpx_nle_f16 -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xfa,0xfe,0x18,0x7d,0x7f,0x6f,0xf5,0x30]
+v_cmpx_nle_f16 v1.h, v2.h row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX12: v_cmpx_nle_f16 v1.h, v2.h row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0xfa,0x04,0x19,0x7d,0x81,0x60,0x09,0x13]
+
+v_cmpx_nle_f16 -|v127.h|, -|v127.h| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX12: v_cmpx_nle_f16 -|v127.h|, -|v127.h| row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xfa,0xfe,0x19,0x7d,0xff,0x6f,0xf5,0x30]
 
 v_cmpx_nle_f32 v1, v2 quad_perm:[3,2,1,0]
 // GFX12: v_cmpx_nle_f32 v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x38,0x7d,0x01,0x1b,0x00,0xff]
@@ -2018,47 +2072,53 @@ v_cmpx_nle_f32 v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
 v_cmpx_nle_f32 -|v255|, -|v255| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
 // GFX12: v_cmpx_nle_f32 -|v255|, -|v255| row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xfa,0xfe,0x39,0x7d,0xff,0x6f,0xf5,0x30]
 
-v_cmpx_nlg_f16 v1, v2 quad_perm:[3,2,1,0]
-// GFX12: v_cmpx_nlg_f16 v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x14,0x7d,0x01,0x1b,0x00,0xff]
+v_cmpx_nlg_f16 v1.l, v2.l quad_perm:[3,2,1,0]
+// GFX12: v_cmpx_nlg_f16 v1.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x14,0x7d,0x01,0x1b,0x00,0xff]
+
+v_cmpx_nlg_f16 v1.l, v2.l quad_perm:[0,1,2,3]
+// GFX12: v_cmpx_nlg_f16 v1.l, v2.l quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x14,0x7d,0x01,0xe4,0x00,0xff]
+
+v_cmpx_nlg_f16 v1.l, v2.l row_mirror
+// GFX12: v_cmpx_nlg_f16 v1.l, v2.l row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x14,0x7d,0x01,0x40,0x01,0xff]
 
-v_cmpx_nlg_f16 v1, v2 quad_perm:[0,1,2,3]
-// GFX12: v_cmpx_nlg_f16 v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x14,0x7d,0x01,0xe4,0x00,0xff]
+v_cmpx_nlg_f16 v1.l, v2.l row_half_mirror
+// GFX12: v_cmpx_nlg_f16 v1.l, v2.l row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x14,0x7d,0x01,0x41,0x01,0xff]
 
-v_cmpx_nlg_f16 v1, v2 row_mirror
-// GFX12: v_cmpx_nlg_f16 v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x14,0x7d,0x01,0x40,0x01,0xff]
+v_cmpx_nlg_f16 v1.l, v2.l row_shl:1
+// GFX12: v_cmpx_nlg_f16 v1.l, v2.l row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x14,0x7d,0x01,0x01,0x01,0xff]
 
-v_cmpx_nlg_f16 v1, v2 row_half_mirror
-// GFX12: v_cmpx_nlg_f16 v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x14,0x7d,0x01,0x41,0x01,0xff]
+v_cmpx_nlg_f16 v1.l, v2.l row_shl:15
+// GFX12: v_cmpx_nlg_f16 v1.l, v2.l row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x14,0x7d,0x01,0x0f,0x01,0xff]
 
-v_cmpx_nlg_f16 v1, v2 row_shl:1
-// GFX12: v_cmpx_nlg_f16 v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x14,0x7d,0x01,0x01,0x01,0xff]
+v_cmpx_nlg_f16 v1.l, v2.l row_shr:1
+// GFX12: v_cmpx_nlg_f16 v1.l, v2.l row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x14,0x7d,0x01,0x11,0x01,0xff]
 
-v_cmpx_nlg_f16 v1, v2 row_shl:15
-// GFX12: v_cmpx_nlg_f16 v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x14,0x7d,0x01,0x0f,0x01,0xff]
+v_cmpx_nlg_f16 v1.l, v2.l row_shr:15
+// GFX12: v_cmpx_nlg_f16 v1.l, v2.l row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x14,0x7d,0x01,0x1f,0x01,0xff]
 
-v_cmpx_nlg_f16 v1, v2 row_shr:1
-// GFX12: v_cmpx_nlg_f16 v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x14,0x7d,0x01,0x11,0x01,0xff]
+v_cmpx_nlg_f16 v1.l, v2.l row_ror:1
+// GFX12: v_cmpx_nlg_f16 v1.l, v2.l row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x14,0x7d,0x01,0x21,0x01,0xff]
 
-v_cmpx_nlg_f16 v1, v2 row_shr:15
-// GFX12: v_cmpx_nlg_f16 v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x14,0x7d,0x01,0x1f,0x01,0xff]
+v_cmpx_nlg_f16 v1.l, v2.l row_ror:15
+// GFX12: v_cmpx_nlg_f16 v1.l, v2.l row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x14,0x7d,0x01,0x2f,0x01,0xff]
 
-v_cmpx_nlg_f16 v1, v2 row_ror:1
-// GFX12: v_cmpx_nlg_f16 v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x14,0x7d,0x01,0x21,0x01,0xff]
+v_cmpx_nlg_f16 v1.l, v2.l row_share:0 row_mask:0xf bank_mask:0xf
+// GFX12: v_cmpx_nlg_f16 v1.l, v2.l row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x14,0x7d,0x01,0x50,0x01,0xff]
 
-v_cmpx_nlg_f16 v1, v2 row_ror:15
-// GFX12: v_cmpx_nlg_f16 v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x14,0x7d,0x01,0x2f,0x01,0xff]
+v_cmpx_nlg_f16 v1.l, v2.l row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX12: v_cmpx_nlg_f16 v1.l, v2.l row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x14,0x7d,0x01,0x5f,0x01,0x01]
 
-v_cmpx_nlg_f16 v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
-// GFX12: v_cmpx_nlg_f16 v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x14,0x7d,0x01,0x50,0x01,0xff]
+v_cmpx_nlg_f16 v1.l, v2.l row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1
+// GFX12: v_cmpx_nlg_f16 v1.l, v2.l row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0xfa,0x04,0x14,0x7d,0x01,0x60,0x09,0x13]
 
-v_cmpx_nlg_f16 v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
-// GFX12: v_cmpx_nlg_f16 v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x14,0x7d,0x01,0x5f,0x01,0x01]
+v_cmpx_nlg_f16 -|v127.l|, -|v127.l| row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1
+// GFX12: v_cmpx_nlg_f16 -|v127.l|, -|v127.l| row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xfa,0xfe,0x14,0x7d,0x7f,0x6f,0xf5,0x30]
 
-v_cmpx_nlg_f16 v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
-// GFX12: v_cmpx_nlg_f16 v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0xfa,0x04,0x14,0x7d,0x01,0x60,0x09,0x13]
+v_cmpx_nlg_f16 v1.h, v2.h row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX12: v_cmpx_nlg_f16 v1.h, v2.h row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0xfa,0x04,0x15,0x7d,0x81,0x60,0x09,0x13]
 
-v_cmpx_nlg_f16 -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
-// GFX12: v_cmpx_nlg_f16 -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xfa,0xfe,0x14,0x7d,0x7f,0x6f,0xf5,0x30]
+v_cmpx_nlg_f16 -|v127.h|, -|v127.h| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX12: v_cmpx_nlg_f16 -|v127.h|, -|v127.h| row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xfa,0xfe,0x15,0x7d,0xff,0x6f,0xf5,0x30]
 
 v_cmpx_nlg_f32 v1, v2 quad_perm:[3,2,1,0]
 // GFX12: v_cmpx_nlg_f32 v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x34,0x7d,0x01,0x1b,0x00,0xff]
@@ -2102,47 +2162,53 @@ v_cmpx_nlg_f32 v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
 v_cmpx_nlg_f32 -|v255|, -|v255| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
 // GFX12: v_cmpx_nlg_f32 -|v255|, -|v255| row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xfa,0xfe,0x35,0x7d,0xff,0x6f,0xf5,0x30]
 
-v_cmpx_nlt_f16 v1, v2 quad_perm:[3,2,1,0]
-// GFX12: v_cmpx_nlt_f16 v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1c,0x7d,0x01,0x1b,0x00,0xff]
+v_cmpx_nlt_f16 v1.l, v2.l quad_perm:[3,2,1,0]
+// GFX12: v_cmpx_nlt_f16 v1.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1c,0x7d,0x01,0x1b,0x00,0xff]
 
-v_cmpx_nlt_f16 v1, v2 quad_perm:[0,1,2,3]
-// GFX12: v_cmpx_nlt_f16 v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1c,0x7d,0x01,0xe4,0x00,0xff]
+v_cmpx_nlt_f16 v1.l, v2.l quad_perm:[0,1,2,3]
+// GFX12: v_cmpx_nlt_f16 v1.l, v2.l quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1c,0x7d,0x01,0xe4,0x00,0xff]
 
-v_cmpx_nlt_f16 v1, v2 row_mirror
-// GFX12: v_cmpx_nlt_f16 v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1c,0x7d,0x01,0x40,0x01,0xff]
+v_cmpx_nlt_f16 v1.l, v2.l row_mirror
+// GFX12: v_cmpx_nlt_f16 v1.l, v2.l row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1c,0x7d,0x01,0x40,0x01,0xff]
 
-v_cmpx_nlt_f16 v1, v2 row_half_mirror
-// GFX12: v_cmpx_nlt_f16 v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1c,0x7d,0x01,0x41,0x01,0xff]
+v_cmpx_nlt_f16 v1.l, v2.l row_half_mirror
+// GFX12: v_cmpx_nlt_f16 v1.l, v2.l row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1c,0x7d,0x01,0x41,0x01,0xff]
 
-v_cmpx_nlt_f16 v1, v2 row_shl:1
-// GFX12: v_cmpx_nlt_f16 v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1c,0x7d,0x01,0x01,0x01,0xff]
+v_cmpx_nlt_f16 v1.l, v2.l row_shl:1
+// GFX12: v_cmpx_nlt_f16 v1.l, v2.l row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1c,0x7d,0x01,0x01,0x01,0xff]
 
-v_cmpx_nlt_f16 v1, v2 row_shl:15
-// GFX12: v_cmpx_nlt_f16 v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1c,0x7d,0x01,0x0f,0x01,0xff]
+v_cmpx_nlt_f16 v1.l, v2.l row_shl:15
+// GFX12: v_cmpx_nlt_f16 v1.l, v2.l row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1c,0x7d,0x01,0x0f,0x01,0xff]
 
-v_cmpx_nlt_f16 v1, v2 row_shr:1
-// GFX12: v_cmpx_nlt_f16 v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1c,0x7d,0x01,0x11,0x01,0xff]
+v_cmpx_nlt_f16 v1.l, v2.l row_shr:1
+// GFX12: v_cmpx_nlt_f16 v1.l, v2.l row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1c,0x7d,0x01,0x11,0x01,0xff]
 
-v_cmpx_nlt_f16 v1, v2 row_shr:15
-// GFX12: v_cmpx_nlt_f16 v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1c,0x7d,0x01,0x1f,0x01,0xff]
+v_cmpx_nlt_f16 v1.l, v2.l row_shr:15
+// GFX12: v_cmpx_nlt_f16 v1.l, v2.l row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1c,0x7d,0x01,0x1f,0x01,0xff]
 
-v_cmpx_nlt_f16 v1, v2 row_ror:1
-// GFX12: v_cmpx_nlt_f16 v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1c,0x7d,0x01,0x21,0x01,0xff]
+v_cmpx_nlt_f16 v1.l, v2.l row_ror:1
+// GFX12: v_cmpx_nlt_f16 v1.l, v2.l row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1c,0x7d,0x01,0x21,0x01,0xff]
 
-v_cmpx_nlt_f16 v1, v2 row_ror:15
-// GFX12: v_cmpx_nlt_f16 v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1c,0x7d,0x01,0x2f,0x01,0xff]
+v_cmpx_nlt_f16 v1.l, v2.l row_ror:15
+// GFX12: v_cmpx_nlt_f16 v1.l, v2.l row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1c,0x7d,0x01,0x2f,0x01,0xff]
 
-v_cmpx_nlt_f16 v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
-// GFX12: v_cmpx_nlt_f16 v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1c,0x7d,0x01,0x50,0x01,0xff]
+v_cmpx_nlt_f16 v1.l, v2.l row_share:0 row_mask:0xf bank_mask:0xf
+// GFX12: v_cmpx_nlt_f16 v1.l, v2.l row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1c,0x7d,0x01,0x50,0x01,0xff]
 
-v_cmpx_nlt_f16 v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
-// GFX12: v_cmpx_nlt_f16 v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x1c,0x7d,0x01,0x5f,0x01,0x01]
+v_cmpx_nlt_f16 v1.l, v2.l row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX12: v_cmpx_nlt_f16 v1.l, v2.l row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x1c,0x7d,0x01,0x5f,0x01,0x01]
 
-v_cmpx_nlt_f16 v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
-// GFX12: v_cmpx_nlt_f16 v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0xfa,0x04,0x1c,0x7d,0x01,0x60,0x09,0x13]
+v_cmpx_nlt_f16 v1.l, v2.l row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1
+// GFX12: v_cmpx_nlt_f16 v1.l, v2.l row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0xfa,0x04,0x1c,0x7d,0x01,0x60,0x09,0x13]
 
-v_cmpx_nlt_f16 -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
-// GFX12: v_cmpx_nlt_f16 -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xfa,0xfe,0x1c,0x7d,0x7f,0x6f,0xf5,0x30]
+v_cmpx_nlt_f16 -|v127.l|, -|v127.l| row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1
+// GFX12: v_cmpx_nlt_f16 -|v127.l|, -|v127.l| row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xfa,0xfe,0x1c,0x7d,0x7f,0x6f,0xf5,0x30]
+
+v_cmpx_nlt_f16 v1.h, v2.h row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX12: v_cmpx_nlt_f16 v1.h, v2.h row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0xfa,0x04,0x1d,0x7d,0x81,0x60,0x09,0x13]
+
+v_cmpx_nlt_f16 -|v127.h|, -|v127.h| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX12: v_cmpx_nlt_f16 -|v127.h|, -|v127.h| row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xfa,0xfe,0x1d,0x7d,0xff,0x6f,0xf5,0x30]
 
 v_cmpx_nlt_f32 v1, v2 quad_perm:[3,2,1,0]
 // GFX12: v_cmpx_nlt_f32 v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x3c,0x7d,0x01,0x1b,0x00,0xff]
@@ -2186,47 +2252,53 @@ v_cmpx_nlt_f32 v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
 v_cmpx_nlt_f32 -|v255|, -|v255| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
 // GFX12: v_cmpx_nlt_f32 -|v255|, -|v255| row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xfa,0xfe,0x3d,0x7d,0xff,0x6f,0xf5,0x30]
 
-v_cmpx_o_f16 v1, v2 quad_perm:[3,2,1,0]
-// GFX12: v_cmpx_o_f16 v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0e,0x7d,0x01,0x1b,0x00,0xff]
+v_cmpx_o_f16 v1.l, v2.l quad_perm:[3,2,1,0]
+// GFX12: v_cmpx_o_f16 v1.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0e,0x7d,0x01,0x1b,0x00,0xff]
+
+v_cmpx_o_f16 v1.l, v2.l quad_perm:[0,1,2,3]
+// GFX12: v_cmpx_o_f16 v1.l, v2.l quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0e,0x7d,0x01,0xe4,0x00,0xff]
 
-v_cmpx_o_f16 v1, v2 quad_perm:[0,1,2,3]
-// GFX12: v_cmpx_o_f16 v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0e,0x7d,0x01,0xe4,0x00,0xff]
+v_cmpx_o_f16 v1.l, v2.l row_mirror
+// GFX12: v_cmpx_o_f16 v1.l, v2.l row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0e,0x7d,0x01,0x40,0x01,0xff]
 
-v_cmpx_o_f16 v1, v2 row_mirror
-// GFX12: v_cmpx_o_f16 v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0e,0x7d,0x01,0x40,0x01,0xff]
+v_cmpx_o_f16 v1.l, v2.l row_half_mirror
+// GFX12: v_cmpx_o_f16 v1.l, v2.l row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0e,0x7d,0x01,0x41,0x01,0xff]
 
-v_cmpx_o_f16 v1, v2 row_half_mirror
-// GFX12: v_cmpx_o_f16 v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0e,0x7d,0x01,0x41,0x01,0xff]
+v_cmpx_o_f16 v1.l, v2.l row_shl:1
+// GFX12: v_cmpx_o_f16 v1.l, v2.l row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0e,0x7d,0x01,0x01,0x01,0xff]
 
-v_cmpx_o_f16 v1, v2 row_shl:1
-// GFX12: v_cmpx_o_f16 v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0e,0x7d,0x01,0x01,0x01,0xff]
+v_cmpx_o_f16 v1.l, v2.l row_shl:15
+// GFX12: v_cmpx_o_f16 v1.l, v2.l row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0e,0x7d,0x01,0x0f,0x01,0xff]
 
-v_cmpx_o_f16 v1, v2 row_shl:15
-// GFX12: v_cmpx_o_f16 v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0e,0x7d,0x01,0x0f,0x01,0xff]
+v_cmpx_o_f16 v1.l, v2.l row_shr:1
+// GFX12: v_cmpx_o_f16 v1.l, v2.l row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0e,0x7d,0x01,0x11,0x01,0xff]
 
-v_cmpx_o_f16 v1, v2 row_shr:1
-// GFX12: v_cmpx_o_f16 v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0e,0x7d,0x01,0x11,0x01,0xff]
+v_cmpx_o_f16 v1.l, v2.l row_shr:15
+// GFX12: v_cmpx_o_f16 v1.l, v2.l row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0e,0x7d,0x01,0x1f,0x01,0xff]
 
-v_cmpx_o_f16 v1, v2 row_shr:15
-// GFX12: v_cmpx_o_f16 v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0e,0x7d,0x01,0x1f,0x01,0xff]
+v_cmpx_o_f16 v1.l, v2.l row_ror:1
+// GFX12: v_cmpx_o_f16 v1.l, v2.l row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0e,0x7d,0x01,0x21,0x01,0xff]
 
-v_cmpx_o_f16 v1, v2 row_ror:1
-// GFX12: v_cmpx_o_f16 v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0e,0x7d,0x01,0x21,0x01,0xff]
+v_cmpx_o_f16 v1.l, v2.l row_ror:15
+// GFX12: v_cmpx_o_f16 v1.l, v2.l row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0e,0x7d,0x01,0x2f,0x01,0xff]
 
-v_cmpx_o_f16 v1, v2 row_ror:15
-// GFX12: v_cmpx_o_f16 v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0e,0x7d,0x01,0x2f,0x01,0xff]
+v_cmpx_o_f16 v1.l, v2.l row_share:0 row_mask:0xf bank_mask:0xf
+// GFX12: v_cmpx_o_f16 v1.l, v2.l row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0e,0x7d,0x01,0x50,0x01,0xff]
 
-v_cmpx_o_f16 v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
-// GFX12: v_cmpx_o_f16 v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0e,0x7d,0x01,0x50,0x01,0xff]
+v_cmpx_o_f16 v1.l, v2.l row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX12: v_cmpx_o_f16 v1.l, v2.l row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x0e,0x7d,0x01,0x5f,0x01,0x01]
 
-v_cmpx_o_f16 v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
-// GFX12: v_cmpx_o_f16 v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x0e,0x7d,0x01,0x5f,0x01,0x01]
+v_cmpx_o_f16 v1.l, v2.l row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1
+// GFX12: v_cmpx_o_f16 v1.l, v2.l row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0xfa,0x04,0x0e,0x7d,0x01,0x60,0x09,0x13]
 
-v_cmpx_o_f16 v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
-// GFX12: v_cmpx_o_f16 v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0xfa,0x04,0x0e,0x7d,0x01,0x60,0x09,0x13]
+v_cmpx_o_f16 -|v127.l|, -|v127.l| row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1
+// GFX12: v_cmpx_o_f16 -|v127.l|, -|v127.l| row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xfa,0xfe,0x0e,0x7d,0x7f,0x6f,0xf5,0x30]
 
-v_cmpx_o_f16 -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
-// GFX12: v_cmpx_o_f16 -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xfa,0xfe,0x0e,0x7d,0x7f,0x6f,0xf5,0x30]
+v_cmpx_o_f16 v1.h, v2.h row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX12: v_cmpx_o_f16 v1.h, v2.h row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0xfa,0x04,0x0f,0x7d,0x81,0x60,0x09,0x13]
+
+v_cmpx_o_f16 -|v127.h|, -|v127.h| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX12: v_cmpx_o_f16 -|v127.h|, -|v127.h| row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xfa,0xfe,0x0f,0x7d,0xff,0x6f,0xf5,0x30]
 
 v_cmpx_o_f32 v1, v2 quad_perm:[3,2,1,0]
 // GFX12: v_cmpx_o_f32 v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x2e,0x7d,0x01,0x1b,0x00,0xff]
@@ -2270,47 +2342,53 @@ v_cmpx_o_f32 v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
 v_cmpx_o_f32 -|v255|, -|v255| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
 // GFX12: v_cmpx_o_f32 -|v255|, -|v255| row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xfa,0xfe,0x2f,0x7d,0xff,0x6f,0xf5,0x30]
 
-v_cmpx_u_f16 v1, v2 quad_perm:[3,2,1,0]
-// GFX12: v_cmpx_u_f16 v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x10,0x7d,0x01,0x1b,0x00,0xff]
+v_cmpx_u_f16 v1.l, v2.l quad_perm:[3,2,1,0]
+// GFX12: v_cmpx_u_f16 v1.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x10,0x7d,0x01,0x1b,0x00,0xff]
+
+v_cmpx_u_f16 v1.l, v2.l quad_perm:[0,1,2,3]
+// GFX12: v_cmpx_u_f16 v1.l, v2.l quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x10,0x7d,0x01,0xe4,0x00,0xff]
+
+v_cmpx_u_f16 v1.l, v2.l row_mirror
+// GFX12: v_cmpx_u_f16 v1.l, v2.l row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x10,0x7d,0x01,0x40,0x01,0xff]
 
-v_cmpx_u_f16 v1, v2 quad_perm:[0,1,2,3]
-// GFX12: v_cmpx_u_f16 v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x10,0x7d,0x01,0xe4,0x00,0xff]
+v_cmpx_u_f16 v1.l, v2.l row_half_mirror
+// GFX12: v_cmpx_u_f16 v1.l, v2.l row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x10,0x7d,0x01,0x41,0x01,0xff]
 
-v_cmpx_u_f16 v1, v2 row_mirror
-// GFX12: v_cmpx_u_f16 v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x10,0x7d,0x01,0x40,0x01,0xff]
+v_cmpx_u_f16 v1.l, v2.l row_shl:1
+// GFX12: v_cmpx_u_f16 v1.l, v2.l row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x10,0x7d,0x01,0x01,0x01,0xff]
 
-v_cmpx_u_f16 v1, v2 row_half_mirror
-// GFX12: v_cmpx_u_f16 v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x10,0x7d,0x01,0x41,0x01,0xff]
+v_cmpx_u_f16 v1.l, v2.l row_shl:15
+// GFX12: v_cmpx_u_f16 v1.l, v2.l row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x10,0x7d,0x01,0x0f,0x01,0xff]
 
-v_cmpx_u_f16 v1, v2 row_shl:1
-// GFX12: v_cmpx_u_f16 v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x10,0x7d,0x01,0x01,0x01,0xff]
+v_cmpx_u_f16 v1.l, v2.l row_shr:1
+// GFX12: v_cmpx_u_f16 v1.l, v2.l row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x10,0x7d,0x01,0x11,0x01,0xff]
 
-v_cmpx_u_f16 v1, v2 row_shl:15
-// GFX12: v_cmpx_u_f16 v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x10,0x7d,0x01,0x0f,0x01,0xff]
+v_cmpx_u_f16 v1.l, v2.l row_shr:15
+// GFX12: v_cmpx_u_f16 v1.l, v2.l row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x10,0x7d,0x01,0x1f,0x01,0xff]
 
-v_cmpx_u_f16 v1, v2 row_shr:1
-// GFX12: v_cmpx_u_f16 v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x10,0x7d,0x01,0x11,0x01,0xff]
+v_cmpx_u_f16 v1.l, v2.l row_ror:1
+// GFX12: v_cmpx_u_f16 v1.l, v2.l row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x10,0x7d,0x01,0x21,0x01,0xff]
 
-v_cmpx_u_f16 v1, v2 row_shr:15
-// GFX12: v_cmpx_u_f16 v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x10,0x7d,0x01,0x1f,0x01,0xff]
+v_cmpx_u_f16 v1.l, v2.l row_ror:15
+// GFX12: v_cmpx_u_f16 v1.l, v2.l row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x10,0x7d,0x01,0x2f,0x01,0xff]
 
-v_cmpx_u_f16 v1, v2 row_ror:1
-// GFX12: v_cmpx_u_f16 v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x10,0x7d,0x01,0x21,0x01,0xff]
+v_cmpx_u_f16 v1.l, v2.l row_share:0 row_mask:0xf bank_mask:0xf
+// GFX12: v_cmpx_u_f16 v1.l, v2.l row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x10,0x7d,0x01,0x50,0x01,0xff]
 
-v_cmpx_u_f16 v1, v2 row_ror:15
-// GFX12: v_cmpx_u_f16 v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x10,0x7d,0x01,0x2f,0x01,0xff]
+v_cmpx_u_f16 v1.l, v2.l row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX12: v_cmpx_u_f16 v1.l, v2.l row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x10,0x7d,0x01,0x5f,0x01,0x01]
 
-v_cmpx_u_f16 v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
-// GFX12: v_cmpx_u_f16 v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x10,0x7d,0x01,0x50,0x01,0xff]
+v_cmpx_u_f16 v1.l, v2.l row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1
+// GFX12: v_cmpx_u_f16 v1.l, v2.l row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0xfa,0x04,0x10,0x7d,0x01,0x60,0x09,0x13]
 
-v_cmpx_u_f16 v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
-// GFX12: v_cmpx_u_f16 v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x10,0x7d,0x01,0x5f,0x01,0x01]
+v_cmpx_u_f16 -|v127.l|, -|v127.l| row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1
+// GFX12: v_cmpx_u_f16 -|v127.l|, -|v127.l| row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xfa,0xfe,0x10,0x7d,0x7f,0x6f,0xf5,0x30]
 
-v_cmpx_u_f16 v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
-// GFX12: v_cmpx_u_f16 v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0xfa,0x04,0x10,0x7d,0x01,0x60,0x09,0x13]
+v_cmpx_u_f16 v1.h, v2.h row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX12: v_cmpx_u_f16 v1.h, v2.h row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0xfa,0x04,0x11,0x7d,0x81,0x60,0x09,0x13]
 
-v_cmpx_u_f16 -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
-// GFX12: v_cmpx_u_f16 -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xfa,0xfe,0x10,0x7d,0x7f,0x6f,0xf5,0x30]
+v_cmpx_u_f16 -|v127.h|, -|v127.h| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX12: v_cmpx_u_f16 -|v127.h|, -|v127.h| row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xfa,0xfe,0x11,0x7d,0xff,0x6f,0xf5,0x30]
 
 v_cmpx_u_f32 v1, v2 quad_perm:[3,2,1,0]
 // GFX12: v_cmpx_u_f32 v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x30,0x7d,0x01,0x1b,0x00,0xff]
diff --git a/llvm/test/MC/AMDGPU/gfx12_asm_vopcx_dpp8.s b/llvm/test/MC/AMDGPU/gfx12_asm_vopcx_dpp8.s
index c6e7fd1aa96da..0f30751003373 100644
--- a/llvm/test/MC/AMDGPU/gfx12_asm_vopcx_dpp8.s
+++ b/llvm/test/MC/AMDGPU/gfx12_asm_vopcx_dpp8.s
@@ -26,14 +26,20 @@ v_cmpx_class_f32 v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
 v_cmpx_class_f32 v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
 // GFX12: v_cmpx_class_f32 v255, v255 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xe9,0xfe,0xfd,0x7d,0xff,0x00,0x00,0x00]
 
-v_cmpx_eq_f16 v1, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_cmpx_eq_f16 v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x04,0x7d,0x01,0x77,0x39,0x05]
+v_cmpx_eq_f16 v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_cmpx_eq_f16 v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x04,0x7d,0x01,0x77,0x39,0x05]
 
-v_cmpx_eq_f16 v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
-// GFX12: v_cmpx_eq_f16 v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0xea,0x04,0x04,0x7d,0x01,0x77,0x39,0x05]
+v_cmpx_eq_f16 v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX12: v_cmpx_eq_f16 v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0xea,0x04,0x04,0x7d,0x01,0x77,0x39,0x05]
 
-v_cmpx_eq_f16 v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:0
-// GFX12: v_cmpx_eq_f16 v127, v127 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xe9,0xfe,0x04,0x7d,0x7f,0x00,0x00,0x00]
+v_cmpx_eq_f16 v127.l, v127.l dpp8:[0,0,0,0,0,0,0,0]
+// GFX12: v_cmpx_eq_f16 v127.l, v127.l dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xe9,0xfe,0x04,0x7d,0x7f,0x00,0x00,0x00]
+
+v_cmpx_eq_f16 v1.h, v2.h dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX12: v_cmpx_eq_f16 v1.h, v2.h dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0xea,0x04,0x05,0x7d,0x81,0x77,0x39,0x05]
+
+v_cmpx_eq_f16 v127.h, v127.h dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX12: v_cmpx_eq_f16 v127.h, v127.h dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xe9,0xfe,0x05,0x7d,0xff,0x00,0x00,0x00]
 
 v_cmpx_eq_f32 v1, v2 dpp8:[7,6,5,4,3,2,1,0]
 // GFX12: v_cmpx_eq_f32 v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x24,0x7d,0x01,0x77,0x39,0x05]
@@ -92,14 +98,20 @@ v_cmpx_eq_u32 v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
 v_cmpx_eq_u32 v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
 // GFX12: v_cmpx_eq_u32 v255, v255 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xe9,0xfe,0x95,0x7d,0xff,0x00,0x00,0x00]
 
-v_cmpx_ge_f16 v1, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_cmpx_ge_f16 v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x0c,0x7d,0x01,0x77,0x39,0x05]
+v_cmpx_ge_f16 v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_cmpx_ge_f16 v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x0c,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_ge_f16 v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX12: v_cmpx_ge_f16 v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0xea,0x04,0x0c,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_ge_f16 v127.l, v127.l dpp8:[0,0,0,0,0,0,0,0]
+// GFX12: v_cmpx_ge_f16 v127.l, v127.l dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xe9,0xfe,0x0c,0x7d,0x7f,0x00,0x00,0x00]
 
-v_cmpx_ge_f16 v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
-// GFX12: v_cmpx_ge_f16 v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0xea,0x04,0x0c,0x7d,0x01,0x77,0x39,0x05]
+v_cmpx_ge_f16 v1.h, v2.h dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX12: v_cmpx_ge_f16 v1.h, v2.h dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0xea,0x04,0x0d,0x7d,0x81,0x77,0x39,0x05]
 
-v_cmpx_ge_f16 v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:0
-// GFX12: v_cmpx_ge_f16 v127, v127 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xe9,0xfe,0x0c,0x7d,0x7f,0x00,0x00,0x00]
+v_cmpx_ge_f16 v127.h, v127.h dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX12: v_cmpx_ge_f16 v127.h, v127.h dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xe9,0xfe,0x0d,0x7d,0xff,0x00,0x00,0x00]
 
 v_cmpx_ge_f32 v1, v2 dpp8:[7,6,5,4,3,2,1,0]
 // GFX12: v_cmpx_ge_f32 v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x2c,0x7d,0x01,0x77,0x39,0x05]
@@ -158,14 +170,20 @@ v_cmpx_ge_u32 v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
 v_cmpx_ge_u32 v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
 // GFX12: v_cmpx_ge_u32 v255, v255 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xe9,0xfe,0x9d,0x7d,0xff,0x00,0x00,0x00]
 
-v_cmpx_gt_f16 v1, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_cmpx_gt_f16 v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x08,0x7d,0x01,0x77,0x39,0x05]
+v_cmpx_gt_f16 v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_cmpx_gt_f16 v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x08,0x7d,0x01,0x77,0x39,0x05]
 
-v_cmpx_gt_f16 v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
-// GFX12: v_cmpx_gt_f16 v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0xea,0x04,0x08,0x7d,0x01,0x77,0x39,0x05]
+v_cmpx_gt_f16 v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX12: v_cmpx_gt_f16 v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0xea,0x04,0x08,0x7d,0x01,0x77,0x39,0x05]
 
-v_cmpx_gt_f16 v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:0
-// GFX12: v_cmpx_gt_f16 v127, v127 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xe9,0xfe,0x08,0x7d,0x7f,0x00,0x00,0x00]
+v_cmpx_gt_f16 v127.l, v127.l dpp8:[0,0,0,0,0,0,0,0]
+// GFX12: v_cmpx_gt_f16 v127.l, v127.l dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xe9,0xfe,0x08,0x7d,0x7f,0x00,0x00,0x00]
+
+v_cmpx_gt_f16 v1.h, v2.h dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX12: v_cmpx_gt_f16 v1.h, v2.h dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0xea,0x04,0x09,0x7d,0x81,0x77,0x39,0x05]
+
+v_cmpx_gt_f16 v127.h, v127.h dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX12: v_cmpx_gt_f16 v127.h, v127.h dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xe9,0xfe,0x09,0x7d,0xff,0x00,0x00,0x00]
 
 v_cmpx_gt_f32 v1, v2 dpp8:[7,6,5,4,3,2,1,0]
 // GFX12: v_cmpx_gt_f32 v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x28,0x7d,0x01,0x77,0x39,0x05]
@@ -224,14 +242,20 @@ v_cmpx_gt_u32 v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
 v_cmpx_gt_u32 v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
 // GFX12: v_cmpx_gt_u32 v255, v255 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xe9,0xfe,0x99,0x7d,0xff,0x00,0x00,0x00]
 
-v_cmpx_le_f16 v1, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_cmpx_le_f16 v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x06,0x7d,0x01,0x77,0x39,0x05]
+v_cmpx_le_f16 v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_cmpx_le_f16 v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x06,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_le_f16 v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX12: v_cmpx_le_f16 v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0xea,0x04,0x06,0x7d,0x01,0x77,0x39,0x05]
 
-v_cmpx_le_f16 v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
-// GFX12: v_cmpx_le_f16 v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0xea,0x04,0x06,0x7d,0x01,0x77,0x39,0x05]
+v_cmpx_le_f16 v127.l, v127.l dpp8:[0,0,0,0,0,0,0,0]
+// GFX12: v_cmpx_le_f16 v127.l, v127.l dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xe9,0xfe,0x06,0x7d,0x7f,0x00,0x00,0x00]
 
-v_cmpx_le_f16 v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:0
-// GFX12: v_cmpx_le_f16 v127, v127 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xe9,0xfe,0x06,0x7d,0x7f,0x00,0x00,0x00]
+v_cmpx_le_f16 v1.h, v2.h dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX12: v_cmpx_le_f16 v1.h, v2.h dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0xea,0x04,0x07,0x7d,0x81,0x77,0x39,0x05]
+
+v_cmpx_le_f16 v127.h, v127.h dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX12: v_cmpx_le_f16 v127.h, v127.h dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xe9,0xfe,0x07,0x7d,0xff,0x00,0x00,0x00]
 
 v_cmpx_le_f32 v1, v2 dpp8:[7,6,5,4,3,2,1,0]
 // GFX12: v_cmpx_le_f32 v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x26,0x7d,0x01,0x77,0x39,0x05]
@@ -290,14 +314,20 @@ v_cmpx_le_u32 v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
 v_cmpx_le_u32 v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
 // GFX12: v_cmpx_le_u32 v255, v255 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xe9,0xfe,0x97,0x7d,0xff,0x00,0x00,0x00]
 
-v_cmpx_lg_f16 v1, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_cmpx_lg_f16 v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x0a,0x7d,0x01,0x77,0x39,0x05]
+v_cmpx_lg_f16 v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_cmpx_lg_f16 v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x0a,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_lg_f16 v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX12: v_cmpx_lg_f16 v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0xea,0x04,0x0a,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_lg_f16 v127.l, v127.l dpp8:[0,0,0,0,0,0,0,0]
+// GFX12: v_cmpx_lg_f16 v127.l, v127.l dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xe9,0xfe,0x0a,0x7d,0x7f,0x00,0x00,0x00]
 
-v_cmpx_lg_f16 v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
-// GFX12: v_cmpx_lg_f16 v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0xea,0x04,0x0a,0x7d,0x01,0x77,0x39,0x05]
+v_cmpx_lg_f16 v1.h, v2.h dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX12: v_cmpx_lg_f16 v1.h, v2.h dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0xea,0x04,0x0b,0x7d,0x81,0x77,0x39,0x05]
 
-v_cmpx_lg_f16 v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:0
-// GFX12: v_cmpx_lg_f16 v127, v127 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xe9,0xfe,0x0a,0x7d,0x7f,0x00,0x00,0x00]
+v_cmpx_lg_f16 v127.h, v127.h dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX12: v_cmpx_lg_f16 v127.h, v127.h dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xe9,0xfe,0x0b,0x7d,0xff,0x00,0x00,0x00]
 
 v_cmpx_lg_f32 v1, v2 dpp8:[7,6,5,4,3,2,1,0]
 // GFX12: v_cmpx_lg_f32 v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x2a,0x7d,0x01,0x77,0x39,0x05]
@@ -428,14 +458,20 @@ v_cmpx_ne_u32 v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
 v_cmpx_ne_u32 v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
 // GFX12: v_cmpx_ne_u32 v255, v255 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xe9,0xfe,0x9b,0x7d,0xff,0x00,0x00,0x00]
 
-v_cmpx_neq_f16 v1, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_cmpx_neq_f16 v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x1a,0x7d,0x01,0x77,0x39,0x05]
+v_cmpx_neq_f16 v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_cmpx_neq_f16 v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x1a,0x7d,0x01,0x77,0x39,0x05]
 
-v_cmpx_neq_f16 v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
-// GFX12: v_cmpx_neq_f16 v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0xea,0x04,0x1a,0x7d,0x01,0x77,0x39,0x05]
+v_cmpx_neq_f16 v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX12: v_cmpx_neq_f16 v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0xea,0x04,0x1a,0x7d,0x01,0x77,0x39,0x05]
 
-v_cmpx_neq_f16 v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:0
-// GFX12: v_cmpx_neq_f16 v127, v127 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xe9,0xfe,0x1a,0x7d,0x7f,0x00,0x00,0x00]
+v_cmpx_neq_f16 v127.l, v127.l dpp8:[0,0,0,0,0,0,0,0]
+// GFX12: v_cmpx_neq_f16 v127.l, v127.l dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xe9,0xfe,0x1a,0x7d,0x7f,0x00,0x00,0x00]
+
+v_cmpx_neq_f16 v1.h, v2.h dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX12: v_cmpx_neq_f16 v1.h, v2.h dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0xea,0x04,0x1b,0x7d,0x81,0x77,0x39,0x05]
+
+v_cmpx_neq_f16 v127.h, v127.h dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX12: v_cmpx_neq_f16 v127.h, v127.h dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xe9,0xfe,0x1b,0x7d,0xff,0x00,0x00,0x00]
 
 v_cmpx_neq_f32 v1, v2 dpp8:[7,6,5,4,3,2,1,0]
 // GFX12: v_cmpx_neq_f32 v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x3a,0x7d,0x01,0x77,0x39,0x05]
@@ -446,14 +482,20 @@ v_cmpx_neq_f32 v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
 v_cmpx_neq_f32 v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
 // GFX12: v_cmpx_neq_f32 v255, v255 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xe9,0xfe,0x3b,0x7d,0xff,0x00,0x00,0x00]
 
-v_cmpx_nge_f16 v1, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_cmpx_nge_f16 v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x12,0x7d,0x01,0x77,0x39,0x05]
+v_cmpx_nge_f16 v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_cmpx_nge_f16 v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x12,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_nge_f16 v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX12: v_cmpx_nge_f16 v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0xea,0x04,0x12,0x7d,0x01,0x77,0x39,0x05]
 
-v_cmpx_nge_f16 v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
-// GFX12: v_cmpx_nge_f16 v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0xea,0x04,0x12,0x7d,0x01,0x77,0x39,0x05]
+v_cmpx_nge_f16 v127.l, v127.l dpp8:[0,0,0,0,0,0,0,0]
+// GFX12: v_cmpx_nge_f16 v127.l, v127.l dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xe9,0xfe,0x12,0x7d,0x7f,0x00,0x00,0x00]
 
-v_cmpx_nge_f16 v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:0
-// GFX12: v_cmpx_nge_f16 v127, v127 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xe9,0xfe,0x12,0x7d,0x7f,0x00,0x00,0x00]
+v_cmpx_nge_f16 v1.h, v2.h dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX12: v_cmpx_nge_f16 v1.h, v2.h dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0xea,0x04,0x13,0x7d,0x81,0x77,0x39,0x05]
+
+v_cmpx_nge_f16 v127.h, v127.h dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX12: v_cmpx_nge_f16 v127.h, v127.h dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xe9,0xfe,0x13,0x7d,0xff,0x00,0x00,0x00]
 
 v_cmpx_nge_f32 v1, v2 dpp8:[7,6,5,4,3,2,1,0]
 // GFX12: v_cmpx_nge_f32 v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x32,0x7d,0x01,0x77,0x39,0x05]
@@ -464,14 +506,20 @@ v_cmpx_nge_f32 v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
 v_cmpx_nge_f32 v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
 // GFX12: v_cmpx_nge_f32 v255, v255 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xe9,0xfe,0x33,0x7d,0xff,0x00,0x00,0x00]
 
-v_cmpx_ngt_f16 v1, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_cmpx_ngt_f16 v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x16,0x7d,0x01,0x77,0x39,0x05]
+v_cmpx_ngt_f16 v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_cmpx_ngt_f16 v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x16,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_ngt_f16 v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX12: v_cmpx_ngt_f16 v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0xea,0x04,0x16,0x7d,0x01,0x77,0x39,0x05]
 
-v_cmpx_ngt_f16 v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
-// GFX12: v_cmpx_ngt_f16 v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0xea,0x04,0x16,0x7d,0x01,0x77,0x39,0x05]
+v_cmpx_ngt_f16 v127.l, v127.l dpp8:[0,0,0,0,0,0,0,0]
+// GFX12: v_cmpx_ngt_f16 v127.l, v127.l dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xe9,0xfe,0x16,0x7d,0x7f,0x00,0x00,0x00]
 
-v_cmpx_ngt_f16 v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:0
-// GFX12: v_cmpx_ngt_f16 v127, v127 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xe9,0xfe,0x16,0x7d,0x7f,0x00,0x00,0x00]
+v_cmpx_ngt_f16 v1.h, v2.h dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX12: v_cmpx_ngt_f16 v1.h, v2.h dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0xea,0x04,0x17,0x7d,0x81,0x77,0x39,0x05]
+
+v_cmpx_ngt_f16 v127.h, v127.h dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX12: v_cmpx_ngt_f16 v127.h, v127.h dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xe9,0xfe,0x17,0x7d,0xff,0x00,0x00,0x00]
 
 v_cmpx_ngt_f32 v1, v2 dpp8:[7,6,5,4,3,2,1,0]
 // GFX12: v_cmpx_ngt_f32 v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x36,0x7d,0x01,0x77,0x39,0x05]
@@ -482,14 +530,20 @@ v_cmpx_ngt_f32 v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
 v_cmpx_ngt_f32 v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
 // GFX12: v_cmpx_ngt_f32 v255, v255 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xe9,0xfe,0x37,0x7d,0xff,0x00,0x00,0x00]
 
-v_cmpx_nle_f16 v1, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_cmpx_nle_f16 v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x18,0x7d,0x01,0x77,0x39,0x05]
+v_cmpx_nle_f16 v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_cmpx_nle_f16 v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x18,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_nle_f16 v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX12: v_cmpx_nle_f16 v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0xea,0x04,0x18,0x7d,0x01,0x77,0x39,0x05]
 
-v_cmpx_nle_f16 v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
-// GFX12: v_cmpx_nle_f16 v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0xea,0x04,0x18,0x7d,0x01,0x77,0x39,0x05]
+v_cmpx_nle_f16 v127.l, v127.l dpp8:[0,0,0,0,0,0,0,0]
+// GFX12: v_cmpx_nle_f16 v127.l, v127.l dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xe9,0xfe,0x18,0x7d,0x7f,0x00,0x00,0x00]
 
-v_cmpx_nle_f16 v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:0
-// GFX12: v_cmpx_nle_f16 v127, v127 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xe9,0xfe,0x18,0x7d,0x7f,0x00,0x00,0x00]
+v_cmpx_nle_f16 v1.h, v2.h dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX12: v_cmpx_nle_f16 v1.h, v2.h dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0xea,0x04,0x19,0x7d,0x81,0x77,0x39,0x05]
+
+v_cmpx_nle_f16 v127.h, v127.h dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX12: v_cmpx_nle_f16 v127.h, v127.h dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xe9,0xfe,0x19,0x7d,0xff,0x00,0x00,0x00]
 
 v_cmpx_nle_f32 v1, v2 dpp8:[7,6,5,4,3,2,1,0]
 // GFX12: v_cmpx_nle_f32 v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x38,0x7d,0x01,0x77,0x39,0x05]
@@ -500,14 +554,20 @@ v_cmpx_nle_f32 v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
 v_cmpx_nle_f32 v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
 // GFX12: v_cmpx_nle_f32 v255, v255 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xe9,0xfe,0x39,0x7d,0xff,0x00,0x00,0x00]
 
-v_cmpx_nlg_f16 v1, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_cmpx_nlg_f16 v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x14,0x7d,0x01,0x77,0x39,0x05]
+v_cmpx_nlg_f16 v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_cmpx_nlg_f16 v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x14,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_nlg_f16 v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX12: v_cmpx_nlg_f16 v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0xea,0x04,0x14,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_nlg_f16 v127.l, v127.l dpp8:[0,0,0,0,0,0,0,0]
+// GFX12: v_cmpx_nlg_f16 v127.l, v127.l dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xe9,0xfe,0x14,0x7d,0x7f,0x00,0x00,0x00]
 
-v_cmpx_nlg_f16 v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
-// GFX12: v_cmpx_nlg_f16 v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0xea,0x04,0x14,0x7d,0x01,0x77,0x39,0x05]
+v_cmpx_nlg_f16 v1.h, v2.h dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX12: v_cmpx_nlg_f16 v1.h, v2.h dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0xea,0x04,0x15,0x7d,0x81,0x77,0x39,0x05]
 
-v_cmpx_nlg_f16 v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:0
-// GFX12: v_cmpx_nlg_f16 v127, v127 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xe9,0xfe,0x14,0x7d,0x7f,0x00,0x00,0x00]
+v_cmpx_nlg_f16 v127.h, v127.h dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX12: v_cmpx_nlg_f16 v127.h, v127.h dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xe9,0xfe,0x15,0x7d,0xff,0x00,0x00,0x00]
 
 v_cmpx_nlg_f32 v1, v2 dpp8:[7,6,5,4,3,2,1,0]
 // GFX12: v_cmpx_nlg_f32 v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x34,0x7d,0x01,0x77,0x39,0x05]
@@ -518,14 +578,20 @@ v_cmpx_nlg_f32 v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
 v_cmpx_nlg_f32 v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
 // GFX12: v_cmpx_nlg_f32 v255, v255 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xe9,0xfe,0x35,0x7d,0xff,0x00,0x00,0x00]
 
-v_cmpx_nlt_f16 v1, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_cmpx_nlt_f16 v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x1c,0x7d,0x01,0x77,0x39,0x05]
+v_cmpx_nlt_f16 v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_cmpx_nlt_f16 v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x1c,0x7d,0x01,0x77,0x39,0x05]
 
-v_cmpx_nlt_f16 v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
-// GFX12: v_cmpx_nlt_f16 v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0xea,0x04,0x1c,0x7d,0x01,0x77,0x39,0x05]
+v_cmpx_nlt_f16 v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX12: v_cmpx_nlt_f16 v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0xea,0x04,0x1c,0x7d,0x01,0x77,0x39,0x05]
 
-v_cmpx_nlt_f16 v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:0
-// GFX12: v_cmpx_nlt_f16 v127, v127 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xe9,0xfe,0x1c,0x7d,0x7f,0x00,0x00,0x00]
+v_cmpx_nlt_f16 v127.l, v127.l dpp8:[0,0,0,0,0,0,0,0]
+// GFX12: v_cmpx_nlt_f16 v127.l, v127.l dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xe9,0xfe,0x1c,0x7d,0x7f,0x00,0x00,0x00]
+
+v_cmpx_nlt_f16 v1.h, v2.h dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX12: v_cmpx_nlt_f16 v1.h, v2.h dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0xea,0x04,0x1d,0x7d,0x81,0x77,0x39,0x05]
+
+v_cmpx_nlt_f16 v127.h, v127.h dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX12: v_cmpx_nlt_f16 v127.h, v127.h dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xe9,0xfe,0x1d,0x7d,0xff,0x00,0x00,0x00]
 
 v_cmpx_nlt_f32 v1, v2 dpp8:[7,6,5,4,3,2,1,0]
 // GFX12: v_cmpx_nlt_f32 v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x3c,0x7d,0x01,0x77,0x39,0x05]
@@ -536,14 +602,20 @@ v_cmpx_nlt_f32 v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
 v_cmpx_nlt_f32 v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
 // GFX12: v_cmpx_nlt_f32 v255, v255 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xe9,0xfe,0x3d,0x7d,0xff,0x00,0x00,0x00]
 
-v_cmpx_o_f16 v1, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_cmpx_o_f16 v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x0e,0x7d,0x01,0x77,0x39,0x05]
+v_cmpx_o_f16 v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_cmpx_o_f16 v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x0e,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_o_f16 v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX12: v_cmpx_o_f16 v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0xea,0x04,0x0e,0x7d,0x01,0x77,0x39,0x05]
 
-v_cmpx_o_f16 v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
-// GFX12: v_cmpx_o_f16 v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0xea,0x04,0x0e,0x7d,0x01,0x77,0x39,0x05]
+v_cmpx_o_f16 v127.l, v127.l dpp8:[0,0,0,0,0,0,0,0]
+// GFX12: v_cmpx_o_f16 v127.l, v127.l dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xe9,0xfe,0x0e,0x7d,0x7f,0x00,0x00,0x00]
 
-v_cmpx_o_f16 v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:0
-// GFX12: v_cmpx_o_f16 v127, v127 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xe9,0xfe,0x0e,0x7d,0x7f,0x00,0x00,0x00]
+v_cmpx_o_f16 v1.h, v2.h dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX12: v_cmpx_o_f16 v1.h, v2.h dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0xea,0x04,0x0f,0x7d,0x81,0x77,0x39,0x05]
+
+v_cmpx_o_f16 v127.h, v127.h dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX12: v_cmpx_o_f16 v127.h, v127.h dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xe9,0xfe,0x0f,0x7d,0xff,0x00,0x00,0x00]
 
 v_cmpx_o_f32 v1, v2 dpp8:[7,6,5,4,3,2,1,0]
 // GFX12: v_cmpx_o_f32 v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x2e,0x7d,0x01,0x77,0x39,0x05]
@@ -554,14 +626,20 @@ v_cmpx_o_f32 v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
 v_cmpx_o_f32 v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
 // GFX12: v_cmpx_o_f32 v255, v255 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xe9,0xfe,0x2f,0x7d,0xff,0x00,0x00,0x00]
 
-v_cmpx_u_f16 v1, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_cmpx_u_f16 v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x10,0x7d,0x01,0x77,0x39,0x05]
+v_cmpx_u_f16 v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_cmpx_u_f16 v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x10,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_u_f16 v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX12: v_cmpx_u_f16 v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0xea,0x04,0x10,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_u_f16 v127.l, v127.l dpp8:[0,0,0,0,0,0,0,0]
+// GFX12: v_cmpx_u_f16 v127.l, v127.l dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xe9,0xfe,0x10,0x7d,0x7f,0x00,0x00,0x00]
 
-v_cmpx_u_f16 v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
-// GFX12: v_cmpx_u_f16 v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0xea,0x04,0x10,0x7d,0x01,0x77,0x39,0x05]
+v_cmpx_u_f16 v1.h, v2.h dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX12: v_cmpx_u_f16 v1.h, v2.h dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0xea,0x04,0x11,0x7d,0x81,0x77,0x39,0x05]
 
-v_cmpx_u_f16 v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:0
-// GFX12: v_cmpx_u_f16 v127, v127 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xe9,0xfe,0x10,0x7d,0x7f,0x00,0x00,0x00]
+v_cmpx_u_f16 v127.h, v127.h dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX12: v_cmpx_u_f16 v127.h, v127.h dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xe9,0xfe,0x11,0x7d,0xff,0x00,0x00,0x00]
 
 v_cmpx_u_f32 v1, v2 dpp8:[7,6,5,4,3,2,1,0]
 // GFX12: v_cmpx_u_f32 v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x30,0x7d,0x01,0x77,0x39,0x05]
diff --git a/llvm/test/MC/AMDGPU/gfx12_asm_vopcx_t16_err.s b/llvm/test/MC/AMDGPU/gfx12_asm_vopcx_t16_err.s
index 7ba3aff6c80ca..58c355ed56ab1 100644
--- a/llvm/test/MC/AMDGPU/gfx12_asm_vopcx_t16_err.s
+++ b/llvm/test/MC/AMDGPU/gfx12_asm_vopcx_t16_err.s
@@ -37,23 +37,41 @@ v_cmpx_class_f16_e32 v255.l, v2.l dpp8:[7,6,5,4,3,2,1,0]
 v_cmpx_class_f16_e32 v255.l, v2.l quad_perm:[3,2,1,0]
 // GFX12: :[[@LINE-1]]:22: error: invalid operand for instruction
 
-v_cmpx_eq_f16_e32 v1, v255
-// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+v_cmpx_eq_f16_e32 v1.h, v255.h
+// GFX12: :[[@LINE-1]]:25: error: invalid operand for instruction
 
-v_cmpx_eq_f16_e32 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:28: error: invalid operand for instruction
+v_cmpx_eq_f16_e32 v1.h, v255.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:25: error: invalid operand for instruction
 
-v_cmpx_eq_f16_e32 v1, v255 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:28: error: invalid operand for instruction
+v_cmpx_eq_f16_e32 v1.h, v255.h quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:25: error: invalid operand for instruction
 
-v_cmpx_eq_f16_e32 v255, v2
-// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+v_cmpx_eq_f16_e32 v1.l, v255.l
+// GFX12: :[[@LINE-1]]:25: error: invalid operand for instruction
 
-v_cmpx_eq_f16_e32 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:28: error: invalid operand for instruction
+v_cmpx_eq_f16_e32 v1.l, v255.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:25: error: invalid operand for instruction
 
-v_cmpx_eq_f16_e32 v255, v2 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:28: error: invalid operand for instruction
+v_cmpx_eq_f16_e32 v1.l, v255.l quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:25: error: invalid operand for instruction
+
+v_cmpx_eq_f16_e32 v255.h, v2.h
+// GFX12: :[[@LINE-1]]:19: error: invalid operand for instruction
+
+v_cmpx_eq_f16_e32 v255.h, v2.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:19: error: invalid operand for instruction
+
+v_cmpx_eq_f16_e32 v255.h, v2.h quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:19: error: invalid operand for instruction
+
+v_cmpx_eq_f16_e32 v255.l, v2.l
+// GFX12: :[[@LINE-1]]:19: error: invalid operand for instruction
+
+v_cmpx_eq_f16_e32 v255.l, v2.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:19: error: invalid operand for instruction
+
+v_cmpx_eq_f16_e32 v255.l, v2.l quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:19: error: invalid operand for instruction
 
 v_cmpx_eq_i16_e32 v1.h, v255.h
 // GFX12: :[[@LINE-1]]:25: error: invalid operand for instruction
@@ -127,23 +145,41 @@ v_cmpx_eq_u16_e32 v255.l, v2.l dpp8:[7,6,5,4,3,2,1,0]
 v_cmpx_eq_u16_e32 v255.l, v2.l quad_perm:[3,2,1,0]
 // GFX12: :[[@LINE-1]]:19: error: invalid operand for instruction
 
-v_cmpx_ge_f16_e32 v1, v255
-// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+v_cmpx_ge_f16_e32 v1.h, v255.h
+// GFX12: :[[@LINE-1]]:25: error: invalid operand for instruction
 
-v_cmpx_ge_f16_e32 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:28: error: invalid operand for instruction
+v_cmpx_ge_f16_e32 v1.h, v255.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:25: error: invalid operand for instruction
 
-v_cmpx_ge_f16_e32 v1, v255 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:28: error: invalid operand for instruction
+v_cmpx_ge_f16_e32 v1.h, v255.h quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:25: error: invalid operand for instruction
 
-v_cmpx_ge_f16_e32 v255, v2
-// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+v_cmpx_ge_f16_e32 v1.l, v255.l
+// GFX12: :[[@LINE-1]]:25: error: invalid operand for instruction
 
-v_cmpx_ge_f16_e32 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:28: error: invalid operand for instruction
+v_cmpx_ge_f16_e32 v1.l, v255.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:25: error: invalid operand for instruction
 
-v_cmpx_ge_f16_e32 v255, v2 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:28: error: invalid operand for instruction
+v_cmpx_ge_f16_e32 v1.l, v255.l quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:25: error: invalid operand for instruction
+
+v_cmpx_ge_f16_e32 v255.h, v2.h
+// GFX12: :[[@LINE-1]]:19: error: invalid operand for instruction
+
+v_cmpx_ge_f16_e32 v255.h, v2.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:19: error: invalid operand for instruction
+
+v_cmpx_ge_f16_e32 v255.h, v2.h quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:19: error: invalid operand for instruction
+
+v_cmpx_ge_f16_e32 v255.l, v2.l
+// GFX12: :[[@LINE-1]]:19: error: invalid operand for instruction
+
+v_cmpx_ge_f16_e32 v255.l, v2.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:19: error: invalid operand for instruction
+
+v_cmpx_ge_f16_e32 v255.l, v2.l quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:19: error: invalid operand for instruction
 
 v_cmpx_ge_i16_e32 v1.h, v255.h
 // GFX12: :[[@LINE-1]]:25: error: invalid operand for instruction
@@ -217,23 +253,41 @@ v_cmpx_ge_u16_e32 v255.l, v2.l dpp8:[7,6,5,4,3,2,1,0]
 v_cmpx_ge_u16_e32 v255.l, v2.l quad_perm:[3,2,1,0]
 // GFX12: :[[@LINE-1]]:19: error: invalid operand for instruction
 
-v_cmpx_gt_f16_e32 v1, v255
-// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+v_cmpx_gt_f16_e32 v1.h, v255.h
+// GFX12: :[[@LINE-1]]:25: error: invalid operand for instruction
+
+v_cmpx_gt_f16_e32 v1.h, v255.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:25: error: invalid operand for instruction
 
-v_cmpx_gt_f16_e32 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:28: error: invalid operand for instruction
+v_cmpx_gt_f16_e32 v1.h, v255.h quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:25: error: invalid operand for instruction
 
-v_cmpx_gt_f16_e32 v1, v255 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:28: error: invalid operand for instruction
+v_cmpx_gt_f16_e32 v1.l, v255.l
+// GFX12: :[[@LINE-1]]:25: error: invalid operand for instruction
 
-v_cmpx_gt_f16_e32 v255, v2
-// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+v_cmpx_gt_f16_e32 v1.l, v255.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:25: error: invalid operand for instruction
 
-v_cmpx_gt_f16_e32 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:28: error: invalid operand for instruction
+v_cmpx_gt_f16_e32 v1.l, v255.l quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:25: error: invalid operand for instruction
 
-v_cmpx_gt_f16_e32 v255, v2 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:28: error: invalid operand for instruction
+v_cmpx_gt_f16_e32 v255.h, v2.h
+// GFX12: :[[@LINE-1]]:19: error: invalid operand for instruction
+
+v_cmpx_gt_f16_e32 v255.h, v2.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:19: error: invalid operand for instruction
+
+v_cmpx_gt_f16_e32 v255.h, v2.h quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:19: error: invalid operand for instruction
+
+v_cmpx_gt_f16_e32 v255.l, v2.l
+// GFX12: :[[@LINE-1]]:19: error: invalid operand for instruction
+
+v_cmpx_gt_f16_e32 v255.l, v2.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:19: error: invalid operand for instruction
+
+v_cmpx_gt_f16_e32 v255.l, v2.l quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:19: error: invalid operand for instruction
 
 v_cmpx_gt_i16_e32 v1.h, v255.h
 // GFX12: :[[@LINE-1]]:25: error: invalid operand for instruction
@@ -307,23 +361,41 @@ v_cmpx_gt_u16_e32 v255.l, v2.l dpp8:[7,6,5,4,3,2,1,0]
 v_cmpx_gt_u16_e32 v255.l, v2.l quad_perm:[3,2,1,0]
 // GFX12: :[[@LINE-1]]:19: error: invalid operand for instruction
 
-v_cmpx_le_f16_e32 v1, v255
-// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+v_cmpx_le_f16_e32 v1.h, v255.h
+// GFX12: :[[@LINE-1]]:25: error: invalid operand for instruction
 
-v_cmpx_le_f16_e32 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:28: error: invalid operand for instruction
+v_cmpx_le_f16_e32 v1.h, v255.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:25: error: invalid operand for instruction
 
-v_cmpx_le_f16_e32 v1, v255 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:28: error: invalid operand for instruction
+v_cmpx_le_f16_e32 v1.h, v255.h quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:25: error: invalid operand for instruction
 
-v_cmpx_le_f16_e32 v255, v2
-// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+v_cmpx_le_f16_e32 v1.l, v255.l
+// GFX12: :[[@LINE-1]]:25: error: invalid operand for instruction
 
-v_cmpx_le_f16_e32 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:28: error: invalid operand for instruction
+v_cmpx_le_f16_e32 v1.l, v255.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:25: error: invalid operand for instruction
 
-v_cmpx_le_f16_e32 v255, v2 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:28: error: invalid operand for instruction
+v_cmpx_le_f16_e32 v1.l, v255.l quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:25: error: invalid operand for instruction
+
+v_cmpx_le_f16_e32 v255.h, v2.h
+// GFX12: :[[@LINE-1]]:19: error: invalid operand for instruction
+
+v_cmpx_le_f16_e32 v255.h, v2.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:19: error: invalid operand for instruction
+
+v_cmpx_le_f16_e32 v255.h, v2.h quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:19: error: invalid operand for instruction
+
+v_cmpx_le_f16_e32 v255.l, v2.l
+// GFX12: :[[@LINE-1]]:19: error: invalid operand for instruction
+
+v_cmpx_le_f16_e32 v255.l, v2.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:19: error: invalid operand for instruction
+
+v_cmpx_le_f16_e32 v255.l, v2.l quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:19: error: invalid operand for instruction
 
 v_cmpx_le_i16_e32 v1.h, v255.h
 // GFX12: :[[@LINE-1]]:25: error: invalid operand for instruction
@@ -397,23 +469,41 @@ v_cmpx_le_u16_e32 v255.l, v2.l dpp8:[7,6,5,4,3,2,1,0]
 v_cmpx_le_u16_e32 v255.l, v2.l quad_perm:[3,2,1,0]
 // GFX12: :[[@LINE-1]]:19: error: invalid operand for instruction
 
-v_cmpx_lg_f16_e32 v1, v255
-// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+v_cmpx_lg_f16_e32 v1.h, v255.h
+// GFX12: :[[@LINE-1]]:25: error: invalid operand for instruction
+
+v_cmpx_lg_f16_e32 v1.h, v255.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:25: error: invalid operand for instruction
 
-v_cmpx_lg_f16_e32 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:28: error: invalid operand for instruction
+v_cmpx_lg_f16_e32 v1.h, v255.h quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:25: error: invalid operand for instruction
 
-v_cmpx_lg_f16_e32 v1, v255 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:28: error: invalid operand for instruction
+v_cmpx_lg_f16_e32 v1.l, v255.l
+// GFX12: :[[@LINE-1]]:25: error: invalid operand for instruction
 
-v_cmpx_lg_f16_e32 v255, v2
-// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+v_cmpx_lg_f16_e32 v1.l, v255.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:25: error: invalid operand for instruction
 
-v_cmpx_lg_f16_e32 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:28: error: invalid operand for instruction
+v_cmpx_lg_f16_e32 v1.l, v255.l quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:25: error: invalid operand for instruction
 
-v_cmpx_lg_f16_e32 v255, v2 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:28: error: invalid operand for instruction
+v_cmpx_lg_f16_e32 v255.h, v2.h
+// GFX12: :[[@LINE-1]]:19: error: invalid operand for instruction
+
+v_cmpx_lg_f16_e32 v255.h, v2.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:19: error: invalid operand for instruction
+
+v_cmpx_lg_f16_e32 v255.h, v2.h quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:19: error: invalid operand for instruction
+
+v_cmpx_lg_f16_e32 v255.l, v2.l
+// GFX12: :[[@LINE-1]]:19: error: invalid operand for instruction
+
+v_cmpx_lg_f16_e32 v255.l, v2.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:19: error: invalid operand for instruction
+
+v_cmpx_lg_f16_e32 v255.l, v2.l quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:19: error: invalid operand for instruction
 
 v_cmpx_lt_f16_e32 v1.h, v255.h
 // GFX12: :[[@LINE-1]]:25: error: invalid operand for instruction
@@ -595,146 +685,290 @@ v_cmpx_ne_u16_e32 v255.l, v2.l dpp8:[7,6,5,4,3,2,1,0]
 v_cmpx_ne_u16_e32 v255.l, v2.l quad_perm:[3,2,1,0]
 // GFX12: :[[@LINE-1]]:19: error: invalid operand for instruction
 
-v_cmpx_neq_f16_e32 v1, v255
-// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+v_cmpx_neq_f16_e32 v1.h, v255.h
+// GFX12: :[[@LINE-1]]:26: error: invalid operand for instruction
+
+v_cmpx_neq_f16_e32 v1.h, v255.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:26: error: invalid operand for instruction
+
+v_cmpx_neq_f16_e32 v1.h, v255.h quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:26: error: invalid operand for instruction
+
+v_cmpx_neq_f16_e32 v1.l, v255.l
+// GFX12: :[[@LINE-1]]:26: error: invalid operand for instruction
+
+v_cmpx_neq_f16_e32 v1.l, v255.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:26: error: invalid operand for instruction
+
+v_cmpx_neq_f16_e32 v1.l, v255.l quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:26: error: invalid operand for instruction
+
+v_cmpx_neq_f16_e32 v255.h, v2.h
+// GFX12: :[[@LINE-1]]:20: error: invalid operand for instruction
+
+v_cmpx_neq_f16_e32 v255.h, v2.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:20: error: invalid operand for instruction
+
+v_cmpx_neq_f16_e32 v255.h, v2.h quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:20: error: invalid operand for instruction
+
+v_cmpx_neq_f16_e32 v255.l, v2.l
+// GFX12: :[[@LINE-1]]:20: error: invalid operand for instruction
+
+v_cmpx_neq_f16_e32 v255.l, v2.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:20: error: invalid operand for instruction
+
+v_cmpx_neq_f16_e32 v255.l, v2.l quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:20: error: invalid operand for instruction
+
+v_cmpx_nge_f16_e32 v1.h, v255.h
+// GFX12: :[[@LINE-1]]:26: error: invalid operand for instruction
+
+v_cmpx_nge_f16_e32 v1.h, v255.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:26: error: invalid operand for instruction
+
+v_cmpx_nge_f16_e32 v1.h, v255.h quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:26: error: invalid operand for instruction
+
+v_cmpx_nge_f16_e32 v1.l, v255.l
+// GFX12: :[[@LINE-1]]:26: error: invalid operand for instruction
+
+v_cmpx_nge_f16_e32 v1.l, v255.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:26: error: invalid operand for instruction
+
+v_cmpx_nge_f16_e32 v1.l, v255.l quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:26: error: invalid operand for instruction
+
+v_cmpx_nge_f16_e32 v255.h, v2.h
+// GFX12: :[[@LINE-1]]:20: error: invalid operand for instruction
+
+v_cmpx_nge_f16_e32 v255.h, v2.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:20: error: invalid operand for instruction
+
+v_cmpx_nge_f16_e32 v255.h, v2.h quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:20: error: invalid operand for instruction
+
+v_cmpx_nge_f16_e32 v255.l, v2.l
+// GFX12: :[[@LINE-1]]:20: error: invalid operand for instruction
+
+v_cmpx_nge_f16_e32 v255.l, v2.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:20: error: invalid operand for instruction
+
+v_cmpx_nge_f16_e32 v255.l, v2.l quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:20: error: invalid operand for instruction
+
+v_cmpx_ngt_f16_e32 v1.h, v255.h
+// GFX12: :[[@LINE-1]]:26: error: invalid operand for instruction
+
+v_cmpx_ngt_f16_e32 v1.h, v255.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:26: error: invalid operand for instruction
+
+v_cmpx_ngt_f16_e32 v1.h, v255.h quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:26: error: invalid operand for instruction
+
+v_cmpx_ngt_f16_e32 v1.l, v255.l
+// GFX12: :[[@LINE-1]]:26: error: invalid operand for instruction
+
+v_cmpx_ngt_f16_e32 v1.l, v255.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:26: error: invalid operand for instruction
+
+v_cmpx_ngt_f16_e32 v1.l, v255.l quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:26: error: invalid operand for instruction
+
+v_cmpx_ngt_f16_e32 v255.h, v2.h
+// GFX12: :[[@LINE-1]]:20: error: invalid operand for instruction
+
+v_cmpx_ngt_f16_e32 v255.h, v2.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:20: error: invalid operand for instruction
+
+v_cmpx_ngt_f16_e32 v255.h, v2.h quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:20: error: invalid operand for instruction
+
+v_cmpx_ngt_f16_e32 v255.l, v2.l
+// GFX12: :[[@LINE-1]]:20: error: invalid operand for instruction
+
+v_cmpx_ngt_f16_e32 v255.l, v2.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:20: error: invalid operand for instruction
+
+v_cmpx_ngt_f16_e32 v255.l, v2.l quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:20: error: invalid operand for instruction
+
+v_cmpx_nle_f16_e32 v1.h, v255.h
+// GFX12: :[[@LINE-1]]:26: error: invalid operand for instruction
+
+v_cmpx_nle_f16_e32 v1.h, v255.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:26: error: invalid operand for instruction
+
+v_cmpx_nle_f16_e32 v1.h, v255.h quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:26: error: invalid operand for instruction
+
+v_cmpx_nle_f16_e32 v1.l, v255.l
+// GFX12: :[[@LINE-1]]:26: error: invalid operand for instruction
+
+v_cmpx_nle_f16_e32 v1.l, v255.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:26: error: invalid operand for instruction
+
+v_cmpx_nle_f16_e32 v1.l, v255.l quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:26: error: invalid operand for instruction
+
+v_cmpx_nle_f16_e32 v255.h, v2.h
+// GFX12: :[[@LINE-1]]:20: error: invalid operand for instruction
+
+v_cmpx_nle_f16_e32 v255.h, v2.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:20: error: invalid operand for instruction
+
+v_cmpx_nle_f16_e32 v255.h, v2.h quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:20: error: invalid operand for instruction
+
+v_cmpx_nle_f16_e32 v255.l, v2.l
+// GFX12: :[[@LINE-1]]:20: error: invalid operand for instruction
+
+v_cmpx_nle_f16_e32 v255.l, v2.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:20: error: invalid operand for instruction
+
+v_cmpx_nle_f16_e32 v255.l, v2.l quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:20: error: invalid operand for instruction
+
+v_cmpx_nlg_f16_e32 v1.h, v255.h
+// GFX12: :[[@LINE-1]]:26: error: invalid operand for instruction
 
-v_cmpx_neq_f16_e32 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:29: error: invalid operand for instruction
+v_cmpx_nlg_f16_e32 v1.h, v255.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:26: error: invalid operand for instruction
 
-v_cmpx_neq_f16_e32 v1, v255 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:29: error: invalid operand for instruction
+v_cmpx_nlg_f16_e32 v1.h, v255.h quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:26: error: invalid operand for instruction
 
-v_cmpx_neq_f16_e32 v255, v2
-// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+v_cmpx_nlg_f16_e32 v1.l, v255.l
+// GFX12: :[[@LINE-1]]:26: error: invalid operand for instruction
 
-v_cmpx_neq_f16_e32 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:29: error: invalid operand for instruction
+v_cmpx_nlg_f16_e32 v1.l, v255.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:26: error: invalid operand for instruction
 
-v_cmpx_neq_f16_e32 v255, v2 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:29: error: invalid operand for instruction
+v_cmpx_nlg_f16_e32 v1.l, v255.l quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:26: error: invalid operand for instruction
 
-v_cmpx_nge_f16_e32 v1, v255
-// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+v_cmpx_nlg_f16_e32 v255.h, v2.h
+// GFX12: :[[@LINE-1]]:20: error: invalid operand for instruction
 
-v_cmpx_nge_f16_e32 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:29: error: invalid operand for instruction
+v_cmpx_nlg_f16_e32 v255.h, v2.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:20: error: invalid operand for instruction
 
-v_cmpx_nge_f16_e32 v1, v255 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:29: error: invalid operand for instruction
+v_cmpx_nlg_f16_e32 v255.h, v2.h quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:20: error: invalid operand for instruction
 
-v_cmpx_nge_f16_e32 v255, v2
-// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+v_cmpx_nlg_f16_e32 v255.l, v2.l
+// GFX12: :[[@LINE-1]]:20: error: invalid operand for instruction
 
-v_cmpx_nge_f16_e32 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:29: error: invalid operand for instruction
+v_cmpx_nlg_f16_e32 v255.l, v2.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:20: error: invalid operand for instruction
 
-v_cmpx_nge_f16_e32 v255, v2 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:29: error: invalid operand for instruction
+v_cmpx_nlg_f16_e32 v255.l, v2.l quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:20: error: invalid operand for instruction
 
-v_cmpx_ngt_f16_e32 v1, v255
-// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+v_cmpx_nlt_f16_e32 v1.h, v255.h
+// GFX12: :[[@LINE-1]]:26: error: invalid operand for instruction
 
-v_cmpx_ngt_f16_e32 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:29: error: invalid operand for instruction
+v_cmpx_nlt_f16_e32 v1.h, v255.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:26: error: invalid operand for instruction
 
-v_cmpx_ngt_f16_e32 v1, v255 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:29: error: invalid operand for instruction
+v_cmpx_nlt_f16_e32 v1.h, v255.h quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:26: error: invalid operand for instruction
 
-v_cmpx_ngt_f16_e32 v255, v2
-// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+v_cmpx_nlt_f16_e32 v1.l, v255.l
+// GFX12: :[[@LINE-1]]:26: error: invalid operand for instruction
 
-v_cmpx_ngt_f16_e32 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:29: error: invalid operand for instruction
+v_cmpx_nlt_f16_e32 v1.l, v255.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:26: error: invalid operand for instruction
 
-v_cmpx_ngt_f16_e32 v255, v2 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:29: error: invalid operand for instruction
+v_cmpx_nlt_f16_e32 v1.l, v255.l quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:26: error: invalid operand for instruction
 
-v_cmpx_nle_f16_e32 v1, v255
-// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+v_cmpx_nlt_f16_e32 v255.h, v2.h
+// GFX12: :[[@LINE-1]]:20: error: invalid operand for instruction
 
-v_cmpx_nle_f16_e32 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:29: error: invalid operand for instruction
+v_cmpx_nlt_f16_e32 v255.h, v2.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:20: error: invalid operand for instruction
 
-v_cmpx_nle_f16_e32 v1, v255 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:29: error: invalid operand for instruction
+v_cmpx_nlt_f16_e32 v255.h, v2.h quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:20: error: invalid operand for instruction
 
-v_cmpx_nle_f16_e32 v255, v2
-// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+v_cmpx_nlt_f16_e32 v255.l, v2.l
+// GFX12: :[[@LINE-1]]:20: error: invalid operand for instruction
 
-v_cmpx_nle_f16_e32 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:29: error: invalid operand for instruction
+v_cmpx_nlt_f16_e32 v255.l, v2.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:20: error: invalid operand for instruction
 
-v_cmpx_nle_f16_e32 v255, v2 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:29: error: invalid operand for instruction
+v_cmpx_nlt_f16_e32 v255.l, v2.l quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:20: error: invalid operand for instruction
 
-v_cmpx_nlg_f16_e32 v1, v255
-// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+v_cmpx_o_f16_e32 v1.h, v255.h
+// GFX12: :[[@LINE-1]]:24: error: invalid operand for instruction
 
-v_cmpx_nlg_f16_e32 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:29: error: invalid operand for instruction
+v_cmpx_o_f16_e32 v1.h, v255.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:24: error: invalid operand for instruction
 
-v_cmpx_nlg_f16_e32 v1, v255 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:29: error: invalid operand for instruction
+v_cmpx_o_f16_e32 v1.h, v255.h quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:24: error: invalid operand for instruction
 
-v_cmpx_nlg_f16_e32 v255, v2
-// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+v_cmpx_o_f16_e32 v1.l, v255.l
+// GFX12: :[[@LINE-1]]:24: error: invalid operand for instruction
 
-v_cmpx_nlg_f16_e32 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:29: error: invalid operand for instruction
+v_cmpx_o_f16_e32 v1.l, v255.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:24: error: invalid operand for instruction
 
-v_cmpx_nlg_f16_e32 v255, v2 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:29: error: invalid operand for instruction
+v_cmpx_o_f16_e32 v1.l, v255.l quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:24: error: invalid operand for instruction
 
-v_cmpx_nlt_f16_e32 v1, v255
-// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+v_cmpx_o_f16_e32 v255.h, v2.h
+// GFX12: :[[@LINE-1]]:18: error: invalid operand for instruction
 
-v_cmpx_nlt_f16_e32 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:29: error: invalid operand for instruction
+v_cmpx_o_f16_e32 v255.h, v2.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:18: error: invalid operand for instruction
 
-v_cmpx_nlt_f16_e32 v1, v255 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:29: error: invalid operand for instruction
+v_cmpx_o_f16_e32 v255.h, v2.h quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:18: error: invalid operand for instruction
 
-v_cmpx_nlt_f16_e32 v255, v2
-// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+v_cmpx_o_f16_e32 v255.l, v2.l
+// GFX12: :[[@LINE-1]]:18: error: invalid operand for instruction
 
-v_cmpx_nlt_f16_e32 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:29: error: invalid operand for instruction
+v_cmpx_o_f16_e32 v255.l, v2.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:18: error: invalid operand for instruction
 
-v_cmpx_nlt_f16_e32 v255, v2 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:29: error: invalid operand for instruction
+v_cmpx_o_f16_e32 v255.l, v2.l quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:18: error: invalid operand for instruction
 
-v_cmpx_o_f16_e32 v1, v255
-// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+v_cmpx_u_f16_e32 v1.h, v255.h
+// GFX12: :[[@LINE-1]]:24: error: invalid operand for instruction
 
-v_cmpx_o_f16_e32 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:27: error: invalid operand for instruction
+v_cmpx_u_f16_e32 v1.h, v255.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:24: error: invalid operand for instruction
 
-v_cmpx_o_f16_e32 v1, v255 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:27: error: invalid operand for instruction
+v_cmpx_u_f16_e32 v1.h, v255.h quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:24: error: invalid operand for instruction
 
-v_cmpx_o_f16_e32 v255, v2
-// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+v_cmpx_u_f16_e32 v1.l, v255.l
+// GFX12: :[[@LINE-1]]:24: error: invalid operand for instruction
 
-v_cmpx_o_f16_e32 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:27: error: invalid operand for instruction
+v_cmpx_u_f16_e32 v1.l, v255.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:24: error: invalid operand for instruction
 
-v_cmpx_o_f16_e32 v255, v2 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:27: error: invalid operand for instruction
+v_cmpx_u_f16_e32 v1.l, v255.l quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:24: error: invalid operand for instruction
 
-v_cmpx_u_f16_e32 v1, v255
-// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+v_cmpx_u_f16_e32 v255.h, v2.h
+// GFX12: :[[@LINE-1]]:18: error: invalid operand for instruction
 
-v_cmpx_u_f16_e32 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:27: error: invalid operand for instruction
+v_cmpx_u_f16_e32 v255.h, v2.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:18: error: invalid operand for instruction
 
-v_cmpx_u_f16_e32 v1, v255 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:27: error: invalid operand for instruction
+v_cmpx_u_f16_e32 v255.h, v2.h quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:18: error: invalid operand for instruction
 
-v_cmpx_u_f16_e32 v255, v2
-// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+v_cmpx_u_f16_e32 v255.l, v2.l
+// GFX12: :[[@LINE-1]]:18: error: invalid operand for instruction
 
-v_cmpx_u_f16_e32 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:27: error: invalid operand for instruction
+v_cmpx_u_f16_e32 v255.l, v2.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:18: error: invalid operand for instruction
 
-v_cmpx_u_f16_e32 v255, v2 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:27: error: invalid operand for instruction
+v_cmpx_u_f16_e32 v255.l, v2.l quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:18: error: invalid operand for instruction
diff --git a/llvm/test/MC/AMDGPU/gfx12_asm_vopcx_t16_promote.s b/llvm/test/MC/AMDGPU/gfx12_asm_vopcx_t16_promote.s
index b7423dcde03d4..b6f907c2bbff4 100644
--- a/llvm/test/MC/AMDGPU/gfx12_asm_vopcx_t16_promote.s
+++ b/llvm/test/MC/AMDGPU/gfx12_asm_vopcx_t16_promote.s
@@ -2,7 +2,7 @@
 // RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize64,+real-true16 -show-encoding %s 2>&1 | FileCheck --check-prefix=GFX12 %s
 
 v_cmpx_class_f16 v1.h, v255.h
-// GFX12: v_cmpx_class_f16_e64 v1.h, v255.h       ; encoding: [0x7e,0x18,0xfd,0xd4,0x01,0xff,0x03,0x00]
+// GFX12: v_cmpx_class_f16_e64 v1.h, v255.h op_sel:[1,1,0] ; encoding: [0x7e,0x18,0xfd,0xd4,0x01,0xff,0x03,0x00]
 
 v_cmpx_class_f16 v1.h, v255.h dpp8:[7,6,5,4,3,2,1,0]
 // GFX12: v_cmpx_class_f16_e64_dpp v1.h, v255.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x18,0xfd,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
@@ -20,7 +20,7 @@ v_cmpx_class_f16 v1.l, v255.l quad_perm:[3,2,1,0]
 // GFX12: v_cmpx_class_f16_e64_dpp v1.l, v255.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xfd,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
 
 v_cmpx_class_f16 v255.h, v2.h
-// GFX12: v_cmpx_class_f16_e64 v255.h, v2.h       ; encoding: [0x7e,0x18,0xfd,0xd4,0xff,0x05,0x02,0x00]
+// GFX12: v_cmpx_class_f16_e64 v255.h, v2.h op_sel:[1,1,0] ; encoding: [0x7e,0x18,0xfd,0xd4,0xff,0x05,0x02,0x00]
 
 v_cmpx_class_f16 v255.h, v2.h dpp8:[7,6,5,4,3,2,1,0]
 // GFX12: v_cmpx_class_f16_e64_dpp v255.h, v2.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x18,0xfd,0xd4,0xe9,0x04,0x02,0x00,0xff,0x77,0x39,0x05]
@@ -37,26 +37,44 @@ v_cmpx_class_f16 v255.l, v2.l dpp8:[7,6,5,4,3,2,1,0]
 v_cmpx_class_f16 v255.l, v2.l quad_perm:[3,2,1,0]
 // GFX12: v_cmpx_class_f16_e64_dpp v255.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xfd,0xd4,0xfa,0x04,0x02,0x00,0xff,0x1b,0x00,0xff]
 
-v_cmpx_eq_f16 v1, v255
-// GFX12: v_cmpx_eq_f16_e64 v1, v255              ; encoding: [0x7e,0x00,0x82,0xd4,0x01,0xff,0x03,0x00]
+v_cmpx_eq_f16 v1.h, v255.h
+// GFX12: v_cmpx_eq_f16_e64 v1.h, v255.h op_sel:[1,1,0] ; encoding: [0x7e,0x18,0x82,0xd4,0x01,0xff,0x03,0x00]
 
-v_cmpx_eq_f16 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_cmpx_eq_f16_e64_dpp v1, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x82,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
+v_cmpx_eq_f16 v1.h, v255.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_cmpx_eq_f16_e64_dpp v1.h, v255.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x18,0x82,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
 
-v_cmpx_eq_f16 v1, v255 quad_perm:[3,2,1,0]
-// GFX12: v_cmpx_eq_f16_e64_dpp v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x82,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
+v_cmpx_eq_f16 v1.h, v255.h quad_perm:[3,2,1,0]
+// GFX12: v_cmpx_eq_f16_e64_dpp v1.h, v255.h op_sel:[1,1] quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x18,0x82,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
 
-v_cmpx_eq_f16 v255, v2
-// GFX12: v_cmpx_eq_f16_e64 v255, v2              ; encoding: [0x7e,0x00,0x82,0xd4,0xff,0x05,0x02,0x00]
+v_cmpx_eq_f16 v1.l, v255.l
+// GFX12: v_cmpx_eq_f16_e64 v1.l, v255.l          ; encoding: [0x7e,0x00,0x82,0xd4,0x01,0xff,0x03,0x00]
 
-v_cmpx_eq_f16 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_cmpx_eq_f16_e64_dpp v255, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x82,0xd4,0xe9,0x04,0x02,0x00,0xff,0x77,0x39,0x05]
+v_cmpx_eq_f16 v1.l, v255.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_cmpx_eq_f16_e64_dpp v1.l, v255.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x82,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
 
-v_cmpx_eq_f16 v255, v2 quad_perm:[3,2,1,0]
-// GFX12: v_cmpx_eq_f16_e64_dpp v255, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x82,0xd4,0xfa,0x04,0x02,0x00,0xff,0x1b,0x00,0xff]
+v_cmpx_eq_f16 v1.l, v255.l quad_perm:[3,2,1,0]
+// GFX12: v_cmpx_eq_f16_e64_dpp v1.l, v255.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x82,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
+
+v_cmpx_eq_f16 v255.h, v2.h
+// GFX12: v_cmpx_eq_f16_e64 v255.h, v2.h op_sel:[1,1,0] ; encoding: [0x7e,0x18,0x82,0xd4,0xff,0x05,0x02,0x00]
+
+v_cmpx_eq_f16 v255.h, v2.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_cmpx_eq_f16_e64_dpp v255.h, v2.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x18,0x82,0xd4,0xe9,0x04,0x02,0x00,0xff,0x77,0x39,0x05]
+
+v_cmpx_eq_f16 v255.h, v2.h quad_perm:[3,2,1,0]
+// GFX12: v_cmpx_eq_f16_e64_dpp v255.h, v2.h op_sel:[1,1] quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x18,0x82,0xd4,0xfa,0x04,0x02,0x00,0xff,0x1b,0x00,0xff]
+
+v_cmpx_eq_f16 v255.l, v2.l
+// GFX12: v_cmpx_eq_f16_e64 v255.l, v2.l          ; encoding: [0x7e,0x00,0x82,0xd4,0xff,0x05,0x02,0x00]
+
+v_cmpx_eq_f16 v255.l, v2.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_cmpx_eq_f16_e64_dpp v255.l, v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x82,0xd4,0xe9,0x04,0x02,0x00,0xff,0x77,0x39,0x05]
+
+v_cmpx_eq_f16 v255.l, v2.l quad_perm:[3,2,1,0]
+// GFX12: v_cmpx_eq_f16_e64_dpp v255.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x82,0xd4,0xfa,0x04,0x02,0x00,0xff,0x1b,0x00,0xff]
 
 v_cmpx_eq_i16 v1.h, v255.h
-// GFX12: v_cmpx_eq_i16_e64 v1.h, v255.h          ; encoding: [0x7e,0x18,0xb2,0xd4,0x01,0xff,0x03,0x00]
+// GFX12: v_cmpx_eq_i16_e64 v1.h, v255.h op_sel:[1,1,0] ; encoding: [0x7e,0x18,0xb2,0xd4,0x01,0xff,0x03,0x00]
 
 v_cmpx_eq_i16 v1.h, v255.h dpp8:[7,6,5,4,3,2,1,0]
 // GFX12: v_cmpx_eq_i16_e64_dpp v1.h, v255.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x18,0xb2,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
@@ -74,7 +92,7 @@ v_cmpx_eq_i16 v1.l, v255.l quad_perm:[3,2,1,0]
 // GFX12: v_cmpx_eq_i16_e64_dpp v1.l, v255.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xb2,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
 
 v_cmpx_eq_i16 v255.h, v2.h
-// GFX12: v_cmpx_eq_i16_e64 v255.h, v2.h          ; encoding: [0x7e,0x18,0xb2,0xd4,0xff,0x05,0x02,0x00]
+// GFX12: v_cmpx_eq_i16_e64 v255.h, v2.h op_sel:[1,1,0] ; encoding: [0x7e,0x18,0xb2,0xd4,0xff,0x05,0x02,0x00]
 
 v_cmpx_eq_i16 v255.h, v2.h dpp8:[7,6,5,4,3,2,1,0]
 // GFX12: v_cmpx_eq_i16_e64_dpp v255.h, v2.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x18,0xb2,0xd4,0xe9,0x04,0x02,0x00,0xff,0x77,0x39,0x05]
@@ -92,7 +110,7 @@ v_cmpx_eq_i16 v255.l, v2.l quad_perm:[3,2,1,0]
 // GFX12: v_cmpx_eq_i16_e64_dpp v255.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xb2,0xd4,0xfa,0x04,0x02,0x00,0xff,0x1b,0x00,0xff]
 
 v_cmpx_eq_u16 v1.h, v255.h
-// GFX12: v_cmpx_eq_u16_e64 v1.h, v255.h          ; encoding: [0x7e,0x18,0xba,0xd4,0x01,0xff,0x03,0x00]
+// GFX12: v_cmpx_eq_u16_e64 v1.h, v255.h op_sel:[1,1,0] ; encoding: [0x7e,0x18,0xba,0xd4,0x01,0xff,0x03,0x00]
 
 v_cmpx_eq_u16 v1.h, v255.h dpp8:[7,6,5,4,3,2,1,0]
 // GFX12: v_cmpx_eq_u16_e64_dpp v1.h, v255.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x18,0xba,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
@@ -110,7 +128,7 @@ v_cmpx_eq_u16 v1.l, v255.l quad_perm:[3,2,1,0]
 // GFX12: v_cmpx_eq_u16_e64_dpp v1.l, v255.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xba,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
 
 v_cmpx_eq_u16 v255.h, v2.h
-// GFX12: v_cmpx_eq_u16_e64 v255.h, v2.h          ; encoding: [0x7e,0x18,0xba,0xd4,0xff,0x05,0x02,0x00]
+// GFX12: v_cmpx_eq_u16_e64 v255.h, v2.h op_sel:[1,1,0] ; encoding: [0x7e,0x18,0xba,0xd4,0xff,0x05,0x02,0x00]
 
 v_cmpx_eq_u16 v255.h, v2.h dpp8:[7,6,5,4,3,2,1,0]
 // GFX12: v_cmpx_eq_u16_e64_dpp v255.h, v2.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x18,0xba,0xd4,0xe9,0x04,0x02,0x00,0xff,0x77,0x39,0x05]
@@ -127,26 +145,44 @@ v_cmpx_eq_u16 v255.l, v2.l dpp8:[7,6,5,4,3,2,1,0]
 v_cmpx_eq_u16 v255.l, v2.l quad_perm:[3,2,1,0]
 // GFX12: v_cmpx_eq_u16_e64_dpp v255.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xba,0xd4,0xfa,0x04,0x02,0x00,0xff,0x1b,0x00,0xff]
 
-v_cmpx_ge_f16 v1, v255
-// GFX12: v_cmpx_ge_f16_e64 v1, v255              ; encoding: [0x7e,0x00,0x86,0xd4,0x01,0xff,0x03,0x00]
+v_cmpx_ge_f16 v1.h, v255.h
+// GFX12: v_cmpx_ge_f16_e64 v1.h, v255.h op_sel:[1,1,0] ; encoding: [0x7e,0x18,0x86,0xd4,0x01,0xff,0x03,0x00]
+
+v_cmpx_ge_f16 v1.h, v255.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_cmpx_ge_f16_e64_dpp v1.h, v255.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x18,0x86,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
+
+v_cmpx_ge_f16 v1.h, v255.h quad_perm:[3,2,1,0]
+// GFX12: v_cmpx_ge_f16_e64_dpp v1.h, v255.h op_sel:[1,1] quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x18,0x86,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
+
+v_cmpx_ge_f16 v1.l, v255.l
+// GFX12: v_cmpx_ge_f16_e64 v1.l, v255.l          ; encoding: [0x7e,0x00,0x86,0xd4,0x01,0xff,0x03,0x00]
+
+v_cmpx_ge_f16 v1.l, v255.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_cmpx_ge_f16_e64_dpp v1.l, v255.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x86,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
 
-v_cmpx_ge_f16 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_cmpx_ge_f16_e64_dpp v1, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x86,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
+v_cmpx_ge_f16 v1.l, v255.l quad_perm:[3,2,1,0]
+// GFX12: v_cmpx_ge_f16_e64_dpp v1.l, v255.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x86,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
 
-v_cmpx_ge_f16 v1, v255 quad_perm:[3,2,1,0]
-// GFX12: v_cmpx_ge_f16_e64_dpp v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x86,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
+v_cmpx_ge_f16 v255.h, v2.h
+// GFX12: v_cmpx_ge_f16_e64 v255.h, v2.h op_sel:[1,1,0] ; encoding: [0x7e,0x18,0x86,0xd4,0xff,0x05,0x02,0x00]
 
-v_cmpx_ge_f16 v255, v2
-// GFX12: v_cmpx_ge_f16_e64 v255, v2              ; encoding: [0x7e,0x00,0x86,0xd4,0xff,0x05,0x02,0x00]
+v_cmpx_ge_f16 v255.h, v2.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_cmpx_ge_f16_e64_dpp v255.h, v2.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x18,0x86,0xd4,0xe9,0x04,0x02,0x00,0xff,0x77,0x39,0x05]
 
-v_cmpx_ge_f16 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_cmpx_ge_f16_e64_dpp v255, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x86,0xd4,0xe9,0x04,0x02,0x00,0xff,0x77,0x39,0x05]
+v_cmpx_ge_f16 v255.h, v2.h quad_perm:[3,2,1,0]
+// GFX12: v_cmpx_ge_f16_e64_dpp v255.h, v2.h op_sel:[1,1] quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x18,0x86,0xd4,0xfa,0x04,0x02,0x00,0xff,0x1b,0x00,0xff]
 
-v_cmpx_ge_f16 v255, v2 quad_perm:[3,2,1,0]
-// GFX12: v_cmpx_ge_f16_e64_dpp v255, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x86,0xd4,0xfa,0x04,0x02,0x00,0xff,0x1b,0x00,0xff]
+v_cmpx_ge_f16 v255.l, v2.l
+// GFX12: v_cmpx_ge_f16_e64 v255.l, v2.l          ; encoding: [0x7e,0x00,0x86,0xd4,0xff,0x05,0x02,0x00]
+
+v_cmpx_ge_f16 v255.l, v2.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_cmpx_ge_f16_e64_dpp v255.l, v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x86,0xd4,0xe9,0x04,0x02,0x00,0xff,0x77,0x39,0x05]
+
+v_cmpx_ge_f16 v255.l, v2.l quad_perm:[3,2,1,0]
+// GFX12: v_cmpx_ge_f16_e64_dpp v255.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x86,0xd4,0xfa,0x04,0x02,0x00,0xff,0x1b,0x00,0xff]
 
 v_cmpx_ge_i16 v1.h, v255.h
-// GFX12: v_cmpx_ge_i16_e64 v1.h, v255.h          ; encoding: [0x7e,0x18,0xb6,0xd4,0x01,0xff,0x03,0x00]
+// GFX12: v_cmpx_ge_i16_e64 v1.h, v255.h op_sel:[1,1,0] ; encoding: [0x7e,0x18,0xb6,0xd4,0x01,0xff,0x03,0x00]
 
 v_cmpx_ge_i16 v1.h, v255.h dpp8:[7,6,5,4,3,2,1,0]
 // GFX12: v_cmpx_ge_i16_e64_dpp v1.h, v255.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x18,0xb6,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
@@ -164,7 +200,7 @@ v_cmpx_ge_i16 v1.l, v255.l quad_perm:[3,2,1,0]
 // GFX12: v_cmpx_ge_i16_e64_dpp v1.l, v255.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xb6,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
 
 v_cmpx_ge_i16 v255.h, v2.h
-// GFX12: v_cmpx_ge_i16_e64 v255.h, v2.h          ; encoding: [0x7e,0x18,0xb6,0xd4,0xff,0x05,0x02,0x00]
+// GFX12: v_cmpx_ge_i16_e64 v255.h, v2.h op_sel:[1,1,0] ; encoding: [0x7e,0x18,0xb6,0xd4,0xff,0x05,0x02,0x00]
 
 v_cmpx_ge_i16 v255.h, v2.h dpp8:[7,6,5,4,3,2,1,0]
 // GFX12: v_cmpx_ge_i16_e64_dpp v255.h, v2.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x18,0xb6,0xd4,0xe9,0x04,0x02,0x00,0xff,0x77,0x39,0x05]
@@ -182,7 +218,7 @@ v_cmpx_ge_i16 v255.l, v2.l quad_perm:[3,2,1,0]
 // GFX12: v_cmpx_ge_i16_e64_dpp v255.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xb6,0xd4,0xfa,0x04,0x02,0x00,0xff,0x1b,0x00,0xff]
 
 v_cmpx_ge_u16 v1.h, v255.h
-// GFX12: v_cmpx_ge_u16_e64 v1.h, v255.h          ; encoding: [0x7e,0x18,0xbe,0xd4,0x01,0xff,0x03,0x00]
+// GFX12: v_cmpx_ge_u16_e64 v1.h, v255.h op_sel:[1,1,0] ; encoding: [0x7e,0x18,0xbe,0xd4,0x01,0xff,0x03,0x00]
 
 v_cmpx_ge_u16 v1.h, v255.h dpp8:[7,6,5,4,3,2,1,0]
 // GFX12: v_cmpx_ge_u16_e64_dpp v1.h, v255.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x18,0xbe,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
@@ -200,7 +236,7 @@ v_cmpx_ge_u16 v1.l, v255.l quad_perm:[3,2,1,0]
 // GFX12: v_cmpx_ge_u16_e64_dpp v1.l, v255.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xbe,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
 
 v_cmpx_ge_u16 v255.h, v2.h
-// GFX12: v_cmpx_ge_u16_e64 v255.h, v2.h          ; encoding: [0x7e,0x18,0xbe,0xd4,0xff,0x05,0x02,0x00]
+// GFX12: v_cmpx_ge_u16_e64 v255.h, v2.h op_sel:[1,1,0] ; encoding: [0x7e,0x18,0xbe,0xd4,0xff,0x05,0x02,0x00]
 
 v_cmpx_ge_u16 v255.h, v2.h dpp8:[7,6,5,4,3,2,1,0]
 // GFX12: v_cmpx_ge_u16_e64_dpp v255.h, v2.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x18,0xbe,0xd4,0xe9,0x04,0x02,0x00,0xff,0x77,0x39,0x05]
@@ -217,26 +253,44 @@ v_cmpx_ge_u16 v255.l, v2.l dpp8:[7,6,5,4,3,2,1,0]
 v_cmpx_ge_u16 v255.l, v2.l quad_perm:[3,2,1,0]
 // GFX12: v_cmpx_ge_u16_e64_dpp v255.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xbe,0xd4,0xfa,0x04,0x02,0x00,0xff,0x1b,0x00,0xff]
 
-v_cmpx_gt_f16 v1, v255
-// GFX12: v_cmpx_gt_f16_e64 v1, v255              ; encoding: [0x7e,0x00,0x84,0xd4,0x01,0xff,0x03,0x00]
+v_cmpx_gt_f16 v1.h, v255.h
+// GFX12: v_cmpx_gt_f16_e64 v1.h, v255.h op_sel:[1,1,0] ; encoding: [0x7e,0x18,0x84,0xd4,0x01,0xff,0x03,0x00]
+
+v_cmpx_gt_f16 v1.h, v255.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_cmpx_gt_f16_e64_dpp v1.h, v255.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x18,0x84,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
+
+v_cmpx_gt_f16 v1.h, v255.h quad_perm:[3,2,1,0]
+// GFX12: v_cmpx_gt_f16_e64_dpp v1.h, v255.h op_sel:[1,1] quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x18,0x84,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
+
+v_cmpx_gt_f16 v1.l, v255.l
+// GFX12: v_cmpx_gt_f16_e64 v1.l, v255.l          ; encoding: [0x7e,0x00,0x84,0xd4,0x01,0xff,0x03,0x00]
 
-v_cmpx_gt_f16 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_cmpx_gt_f16_e64_dpp v1, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x84,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
+v_cmpx_gt_f16 v1.l, v255.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_cmpx_gt_f16_e64_dpp v1.l, v255.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x84,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
 
-v_cmpx_gt_f16 v1, v255 quad_perm:[3,2,1,0]
-// GFX12: v_cmpx_gt_f16_e64_dpp v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x84,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
+v_cmpx_gt_f16 v1.l, v255.l quad_perm:[3,2,1,0]
+// GFX12: v_cmpx_gt_f16_e64_dpp v1.l, v255.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x84,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
 
-v_cmpx_gt_f16 v255, v2
-// GFX12: v_cmpx_gt_f16_e64 v255, v2              ; encoding: [0x7e,0x00,0x84,0xd4,0xff,0x05,0x02,0x00]
+v_cmpx_gt_f16 v255.h, v2.h
+// GFX12: v_cmpx_gt_f16_e64 v255.h, v2.h op_sel:[1,1,0] ; encoding: [0x7e,0x18,0x84,0xd4,0xff,0x05,0x02,0x00]
 
-v_cmpx_gt_f16 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_cmpx_gt_f16_e64_dpp v255, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x84,0xd4,0xe9,0x04,0x02,0x00,0xff,0x77,0x39,0x05]
+v_cmpx_gt_f16 v255.h, v2.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_cmpx_gt_f16_e64_dpp v255.h, v2.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x18,0x84,0xd4,0xe9,0x04,0x02,0x00,0xff,0x77,0x39,0x05]
 
-v_cmpx_gt_f16 v255, v2 quad_perm:[3,2,1,0]
-// GFX12: v_cmpx_gt_f16_e64_dpp v255, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x84,0xd4,0xfa,0x04,0x02,0x00,0xff,0x1b,0x00,0xff]
+v_cmpx_gt_f16 v255.h, v2.h quad_perm:[3,2,1,0]
+// GFX12: v_cmpx_gt_f16_e64_dpp v255.h, v2.h op_sel:[1,1] quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x18,0x84,0xd4,0xfa,0x04,0x02,0x00,0xff,0x1b,0x00,0xff]
+
+v_cmpx_gt_f16 v255.l, v2.l
+// GFX12: v_cmpx_gt_f16_e64 v255.l, v2.l          ; encoding: [0x7e,0x00,0x84,0xd4,0xff,0x05,0x02,0x00]
+
+v_cmpx_gt_f16 v255.l, v2.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_cmpx_gt_f16_e64_dpp v255.l, v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x84,0xd4,0xe9,0x04,0x02,0x00,0xff,0x77,0x39,0x05]
+
+v_cmpx_gt_f16 v255.l, v2.l quad_perm:[3,2,1,0]
+// GFX12: v_cmpx_gt_f16_e64_dpp v255.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x84,0xd4,0xfa,0x04,0x02,0x00,0xff,0x1b,0x00,0xff]
 
 v_cmpx_gt_i16 v1.h, v255.h
-// GFX12: v_cmpx_gt_i16_e64 v1.h, v255.h          ; encoding: [0x7e,0x18,0xb4,0xd4,0x01,0xff,0x03,0x00]
+// GFX12: v_cmpx_gt_i16_e64 v1.h, v255.h op_sel:[1,1,0] ; encoding: [0x7e,0x18,0xb4,0xd4,0x01,0xff,0x03,0x00]
 
 v_cmpx_gt_i16 v1.h, v255.h dpp8:[7,6,5,4,3,2,1,0]
 // GFX12: v_cmpx_gt_i16_e64_dpp v1.h, v255.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x18,0xb4,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
@@ -254,7 +308,7 @@ v_cmpx_gt_i16 v1.l, v255.l quad_perm:[3,2,1,0]
 // GFX12: v_cmpx_gt_i16_e64_dpp v1.l, v255.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xb4,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
 
 v_cmpx_gt_i16 v255.h, v2.h
-// GFX12: v_cmpx_gt_i16_e64 v255.h, v2.h          ; encoding: [0x7e,0x18,0xb4,0xd4,0xff,0x05,0x02,0x00]
+// GFX12: v_cmpx_gt_i16_e64 v255.h, v2.h op_sel:[1,1,0] ; encoding: [0x7e,0x18,0xb4,0xd4,0xff,0x05,0x02,0x00]
 
 v_cmpx_gt_i16 v255.h, v2.h dpp8:[7,6,5,4,3,2,1,0]
 // GFX12: v_cmpx_gt_i16_e64_dpp v255.h, v2.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x18,0xb4,0xd4,0xe9,0x04,0x02,0x00,0xff,0x77,0x39,0x05]
@@ -272,7 +326,7 @@ v_cmpx_gt_i16 v255.l, v2.l quad_perm:[3,2,1,0]
 // GFX12: v_cmpx_gt_i16_e64_dpp v255.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xb4,0xd4,0xfa,0x04,0x02,0x00,0xff,0x1b,0x00,0xff]
 
 v_cmpx_gt_u16 v1.h, v255.h
-// GFX12: v_cmpx_gt_u16_e64 v1.h, v255.h          ; encoding: [0x7e,0x18,0xbc,0xd4,0x01,0xff,0x03,0x00]
+// GFX12: v_cmpx_gt_u16_e64 v1.h, v255.h op_sel:[1,1,0] ; encoding: [0x7e,0x18,0xbc,0xd4,0x01,0xff,0x03,0x00]
 
 v_cmpx_gt_u16 v1.h, v255.h dpp8:[7,6,5,4,3,2,1,0]
 // GFX12: v_cmpx_gt_u16_e64_dpp v1.h, v255.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x18,0xbc,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
@@ -290,7 +344,7 @@ v_cmpx_gt_u16 v1.l, v255.l quad_perm:[3,2,1,0]
 // GFX12: v_cmpx_gt_u16_e64_dpp v1.l, v255.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xbc,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
 
 v_cmpx_gt_u16 v255.h, v2.h
-// GFX12: v_cmpx_gt_u16_e64 v255.h, v2.h          ; encoding: [0x7e,0x18,0xbc,0xd4,0xff,0x05,0x02,0x00]
+// GFX12: v_cmpx_gt_u16_e64 v255.h, v2.h op_sel:[1,1,0] ; encoding: [0x7e,0x18,0xbc,0xd4,0xff,0x05,0x02,0x00]
 
 v_cmpx_gt_u16 v255.h, v2.h dpp8:[7,6,5,4,3,2,1,0]
 // GFX12: v_cmpx_gt_u16_e64_dpp v255.h, v2.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x18,0xbc,0xd4,0xe9,0x04,0x02,0x00,0xff,0x77,0x39,0x05]
@@ -307,26 +361,44 @@ v_cmpx_gt_u16 v255.l, v2.l dpp8:[7,6,5,4,3,2,1,0]
 v_cmpx_gt_u16 v255.l, v2.l quad_perm:[3,2,1,0]
 // GFX12: v_cmpx_gt_u16_e64_dpp v255.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xbc,0xd4,0xfa,0x04,0x02,0x00,0xff,0x1b,0x00,0xff]
 
-v_cmpx_le_f16 v1, v255
-// GFX12: v_cmpx_le_f16_e64 v1, v255              ; encoding: [0x7e,0x00,0x83,0xd4,0x01,0xff,0x03,0x00]
+v_cmpx_le_f16 v1.h, v255.h
+// GFX12: v_cmpx_le_f16_e64 v1.h, v255.h op_sel:[1,1,0] ; encoding: [0x7e,0x18,0x83,0xd4,0x01,0xff,0x03,0x00]
+
+v_cmpx_le_f16 v1.h, v255.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_cmpx_le_f16_e64_dpp v1.h, v255.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x18,0x83,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
+
+v_cmpx_le_f16 v1.h, v255.h quad_perm:[3,2,1,0]
+// GFX12: v_cmpx_le_f16_e64_dpp v1.h, v255.h op_sel:[1,1] quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x18,0x83,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
 
-v_cmpx_le_f16 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_cmpx_le_f16_e64_dpp v1, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x83,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
+v_cmpx_le_f16 v1.l, v255.l
+// GFX12: v_cmpx_le_f16_e64 v1.l, v255.l          ; encoding: [0x7e,0x00,0x83,0xd4,0x01,0xff,0x03,0x00]
 
-v_cmpx_le_f16 v1, v255 quad_perm:[3,2,1,0]
-// GFX12: v_cmpx_le_f16_e64_dpp v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x83,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
+v_cmpx_le_f16 v1.l, v255.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_cmpx_le_f16_e64_dpp v1.l, v255.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x83,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
 
-v_cmpx_le_f16 v255, v2
-// GFX12: v_cmpx_le_f16_e64 v255, v2              ; encoding: [0x7e,0x00,0x83,0xd4,0xff,0x05,0x02,0x00]
+v_cmpx_le_f16 v1.l, v255.l quad_perm:[3,2,1,0]
+// GFX12: v_cmpx_le_f16_e64_dpp v1.l, v255.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x83,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
 
-v_cmpx_le_f16 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_cmpx_le_f16_e64_dpp v255, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x83,0xd4,0xe9,0x04,0x02,0x00,0xff,0x77,0x39,0x05]
+v_cmpx_le_f16 v255.h, v2.h
+// GFX12: v_cmpx_le_f16_e64 v255.h, v2.h op_sel:[1,1,0] ; encoding: [0x7e,0x18,0x83,0xd4,0xff,0x05,0x02,0x00]
 
-v_cmpx_le_f16 v255, v2 quad_perm:[3,2,1,0]
-// GFX12: v_cmpx_le_f16_e64_dpp v255, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x83,0xd4,0xfa,0x04,0x02,0x00,0xff,0x1b,0x00,0xff]
+v_cmpx_le_f16 v255.h, v2.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_cmpx_le_f16_e64_dpp v255.h, v2.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x18,0x83,0xd4,0xe9,0x04,0x02,0x00,0xff,0x77,0x39,0x05]
+
+v_cmpx_le_f16 v255.h, v2.h quad_perm:[3,2,1,0]
+// GFX12: v_cmpx_le_f16_e64_dpp v255.h, v2.h op_sel:[1,1] quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x18,0x83,0xd4,0xfa,0x04,0x02,0x00,0xff,0x1b,0x00,0xff]
+
+v_cmpx_le_f16 v255.l, v2.l
+// GFX12: v_cmpx_le_f16_e64 v255.l, v2.l          ; encoding: [0x7e,0x00,0x83,0xd4,0xff,0x05,0x02,0x00]
+
+v_cmpx_le_f16 v255.l, v2.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_cmpx_le_f16_e64_dpp v255.l, v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x83,0xd4,0xe9,0x04,0x02,0x00,0xff,0x77,0x39,0x05]
+
+v_cmpx_le_f16 v255.l, v2.l quad_perm:[3,2,1,0]
+// GFX12: v_cmpx_le_f16_e64_dpp v255.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x83,0xd4,0xfa,0x04,0x02,0x00,0xff,0x1b,0x00,0xff]
 
 v_cmpx_le_i16 v1.h, v255.h
-// GFX12: v_cmpx_le_i16_e64 v1.h, v255.h          ; encoding: [0x7e,0x18,0xb3,0xd4,0x01,0xff,0x03,0x00]
+// GFX12: v_cmpx_le_i16_e64 v1.h, v255.h op_sel:[1,1,0] ; encoding: [0x7e,0x18,0xb3,0xd4,0x01,0xff,0x03,0x00]
 
 v_cmpx_le_i16 v1.h, v255.h dpp8:[7,6,5,4,3,2,1,0]
 // GFX12: v_cmpx_le_i16_e64_dpp v1.h, v255.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x18,0xb3,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
@@ -344,7 +416,7 @@ v_cmpx_le_i16 v1.l, v255.l quad_perm:[3,2,1,0]
 // GFX12: v_cmpx_le_i16_e64_dpp v1.l, v255.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xb3,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
 
 v_cmpx_le_i16 v255.h, v2.h
-// GFX12: v_cmpx_le_i16_e64 v255.h, v2.h          ; encoding: [0x7e,0x18,0xb3,0xd4,0xff,0x05,0x02,0x00]
+// GFX12: v_cmpx_le_i16_e64 v255.h, v2.h op_sel:[1,1,0] ; encoding: [0x7e,0x18,0xb3,0xd4,0xff,0x05,0x02,0x00]
 
 v_cmpx_le_i16 v255.h, v2.h dpp8:[7,6,5,4,3,2,1,0]
 // GFX12: v_cmpx_le_i16_e64_dpp v255.h, v2.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x18,0xb3,0xd4,0xe9,0x04,0x02,0x00,0xff,0x77,0x39,0x05]
@@ -362,7 +434,7 @@ v_cmpx_le_i16 v255.l, v2.l quad_perm:[3,2,1,0]
 // GFX12: v_cmpx_le_i16_e64_dpp v255.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xb3,0xd4,0xfa,0x04,0x02,0x00,0xff,0x1b,0x00,0xff]
 
 v_cmpx_le_u16 v1.h, v255.h
-// GFX12: v_cmpx_le_u16_e64 v1.h, v255.h          ; encoding: [0x7e,0x18,0xbb,0xd4,0x01,0xff,0x03,0x00]
+// GFX12: v_cmpx_le_u16_e64 v1.h, v255.h op_sel:[1,1,0] ; encoding: [0x7e,0x18,0xbb,0xd4,0x01,0xff,0x03,0x00]
 
 v_cmpx_le_u16 v1.h, v255.h dpp8:[7,6,5,4,3,2,1,0]
 // GFX12: v_cmpx_le_u16_e64_dpp v1.h, v255.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x18,0xbb,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
@@ -380,7 +452,7 @@ v_cmpx_le_u16 v1.l, v255.l quad_perm:[3,2,1,0]
 // GFX12: v_cmpx_le_u16_e64_dpp v1.l, v255.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xbb,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
 
 v_cmpx_le_u16 v255.h, v2.h
-// GFX12: v_cmpx_le_u16_e64 v255.h, v2.h          ; encoding: [0x7e,0x18,0xbb,0xd4,0xff,0x05,0x02,0x00]
+// GFX12: v_cmpx_le_u16_e64 v255.h, v2.h op_sel:[1,1,0] ; encoding: [0x7e,0x18,0xbb,0xd4,0xff,0x05,0x02,0x00]
 
 v_cmpx_le_u16 v255.h, v2.h dpp8:[7,6,5,4,3,2,1,0]
 // GFX12: v_cmpx_le_u16_e64_dpp v255.h, v2.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x18,0xbb,0xd4,0xe9,0x04,0x02,0x00,0xff,0x77,0x39,0x05]
@@ -397,26 +469,44 @@ v_cmpx_le_u16 v255.l, v2.l dpp8:[7,6,5,4,3,2,1,0]
 v_cmpx_le_u16 v255.l, v2.l quad_perm:[3,2,1,0]
 // GFX12: v_cmpx_le_u16_e64_dpp v255.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xbb,0xd4,0xfa,0x04,0x02,0x00,0xff,0x1b,0x00,0xff]
 
-v_cmpx_lg_f16 v1, v255
-// GFX12: v_cmpx_lg_f16_e64 v1, v255              ; encoding: [0x7e,0x00,0x85,0xd4,0x01,0xff,0x03,0x00]
+v_cmpx_lg_f16 v1.h, v255.h
+// GFX12: v_cmpx_lg_f16_e64 v1.h, v255.h op_sel:[1,1,0] ; encoding: [0x7e,0x18,0x85,0xd4,0x01,0xff,0x03,0x00]
+
+v_cmpx_lg_f16 v1.h, v255.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_cmpx_lg_f16_e64_dpp v1.h, v255.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x18,0x85,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
+
+v_cmpx_lg_f16 v1.h, v255.h quad_perm:[3,2,1,0]
+// GFX12: v_cmpx_lg_f16_e64_dpp v1.h, v255.h op_sel:[1,1] quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x18,0x85,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
+
+v_cmpx_lg_f16 v1.l, v255.l
+// GFX12: v_cmpx_lg_f16_e64 v1.l, v255.l          ; encoding: [0x7e,0x00,0x85,0xd4,0x01,0xff,0x03,0x00]
+
+v_cmpx_lg_f16 v1.l, v255.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_cmpx_lg_f16_e64_dpp v1.l, v255.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x85,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
+
+v_cmpx_lg_f16 v1.l, v255.l quad_perm:[3,2,1,0]
+// GFX12: v_cmpx_lg_f16_e64_dpp v1.l, v255.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x85,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
+
+v_cmpx_lg_f16 v255.h, v2.h
+// GFX12: v_cmpx_lg_f16_e64 v255.h, v2.h op_sel:[1,1,0] ; encoding: [0x7e,0x18,0x85,0xd4,0xff,0x05,0x02,0x00]
 
-v_cmpx_lg_f16 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_cmpx_lg_f16_e64_dpp v1, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x85,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
+v_cmpx_lg_f16 v255.h, v2.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_cmpx_lg_f16_e64_dpp v255.h, v2.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x18,0x85,0xd4,0xe9,0x04,0x02,0x00,0xff,0x77,0x39,0x05]
 
-v_cmpx_lg_f16 v1, v255 quad_perm:[3,2,1,0]
-// GFX12: v_cmpx_lg_f16_e64_dpp v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x85,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
+v_cmpx_lg_f16 v255.h, v2.h quad_perm:[3,2,1,0]
+// GFX12: v_cmpx_lg_f16_e64_dpp v255.h, v2.h op_sel:[1,1] quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x18,0x85,0xd4,0xfa,0x04,0x02,0x00,0xff,0x1b,0x00,0xff]
 
-v_cmpx_lg_f16 v255, v2
-// GFX12: v_cmpx_lg_f16_e64 v255, v2              ; encoding: [0x7e,0x00,0x85,0xd4,0xff,0x05,0x02,0x00]
+v_cmpx_lg_f16 v255.l, v2.l
+// GFX12: v_cmpx_lg_f16_e64 v255.l, v2.l          ; encoding: [0x7e,0x00,0x85,0xd4,0xff,0x05,0x02,0x00]
 
-v_cmpx_lg_f16 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_cmpx_lg_f16_e64_dpp v255, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x85,0xd4,0xe9,0x04,0x02,0x00,0xff,0x77,0x39,0x05]
+v_cmpx_lg_f16 v255.l, v2.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_cmpx_lg_f16_e64_dpp v255.l, v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x85,0xd4,0xe9,0x04,0x02,0x00,0xff,0x77,0x39,0x05]
 
-v_cmpx_lg_f16 v255, v2 quad_perm:[3,2,1,0]
-// GFX12: v_cmpx_lg_f16_e64_dpp v255, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x85,0xd4,0xfa,0x04,0x02,0x00,0xff,0x1b,0x00,0xff]
+v_cmpx_lg_f16 v255.l, v2.l quad_perm:[3,2,1,0]
+// GFX12: v_cmpx_lg_f16_e64_dpp v255.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x85,0xd4,0xfa,0x04,0x02,0x00,0xff,0x1b,0x00,0xff]
 
 v_cmpx_lt_f16 v1.h, v255.h
-// GFX12: v_cmpx_lt_f16_e64 v1.h, v255.h          ; encoding: [0x7e,0x18,0x81,0xd4,0x01,0xff,0x03,0x00]
+// GFX12: v_cmpx_lt_f16_e64 v1.h, v255.h op_sel:[1,1,0] ; encoding: [0x7e,0x18,0x81,0xd4,0x01,0xff,0x03,0x00]
 
 v_cmpx_lt_f16 v1.h, v255.h dpp8:[7,6,5,4,3,2,1,0]
 // GFX12: v_cmpx_lt_f16_e64_dpp v1.h, v255.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x18,0x81,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
@@ -434,7 +524,7 @@ v_cmpx_lt_f16 v1.l, v255.l quad_perm:[3,2,1,0]
 // GFX12: v_cmpx_lt_f16_e64_dpp v1.l, v255.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x81,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
 
 v_cmpx_lt_f16 v255.h, v2.h
-// GFX12: v_cmpx_lt_f16_e64 v255.h, v2.h          ; encoding: [0x7e,0x18,0x81,0xd4,0xff,0x05,0x02,0x00]
+// GFX12: v_cmpx_lt_f16_e64 v255.h, v2.h op_sel:[1,1,0] ; encoding: [0x7e,0x18,0x81,0xd4,0xff,0x05,0x02,0x00]
 
 v_cmpx_lt_f16 v255.h, v2.h dpp8:[7,6,5,4,3,2,1,0]
 // GFX12: v_cmpx_lt_f16_e64_dpp v255.h, v2.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x18,0x81,0xd4,0xe9,0x04,0x02,0x00,0xff,0x77,0x39,0x05]
@@ -452,7 +542,7 @@ v_cmpx_lt_f16 v255.l, v2.l quad_perm:[3,2,1,0]
 // GFX12: v_cmpx_lt_f16_e64_dpp v255.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x81,0xd4,0xfa,0x04,0x02,0x00,0xff,0x1b,0x00,0xff]
 
 v_cmpx_lt_i16 v1.h, v255.h
-// GFX12: v_cmpx_lt_i16_e64 v1.h, v255.h          ; encoding: [0x7e,0x18,0xb1,0xd4,0x01,0xff,0x03,0x00]
+// GFX12: v_cmpx_lt_i16_e64 v1.h, v255.h op_sel:[1,1,0] ; encoding: [0x7e,0x18,0xb1,0xd4,0x01,0xff,0x03,0x00]
 
 v_cmpx_lt_i16 v1.h, v255.h dpp8:[7,6,5,4,3,2,1,0]
 // GFX12: v_cmpx_lt_i16_e64_dpp v1.h, v255.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x18,0xb1,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
@@ -470,7 +560,7 @@ v_cmpx_lt_i16 v1.l, v255.l quad_perm:[3,2,1,0]
 // GFX12: v_cmpx_lt_i16_e64_dpp v1.l, v255.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xb1,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
 
 v_cmpx_lt_i16 v255.h, v2.h
-// GFX12: v_cmpx_lt_i16_e64 v255.h, v2.h          ; encoding: [0x7e,0x18,0xb1,0xd4,0xff,0x05,0x02,0x00]
+// GFX12: v_cmpx_lt_i16_e64 v255.h, v2.h op_sel:[1,1,0] ; encoding: [0x7e,0x18,0xb1,0xd4,0xff,0x05,0x02,0x00]
 
 v_cmpx_lt_i16 v255.h, v2.h dpp8:[7,6,5,4,3,2,1,0]
 // GFX12: v_cmpx_lt_i16_e64_dpp v255.h, v2.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x18,0xb1,0xd4,0xe9,0x04,0x02,0x00,0xff,0x77,0x39,0x05]
@@ -488,7 +578,7 @@ v_cmpx_lt_i16 v255.l, v2.l quad_perm:[3,2,1,0]
 // GFX12: v_cmpx_lt_i16_e64_dpp v255.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xb1,0xd4,0xfa,0x04,0x02,0x00,0xff,0x1b,0x00,0xff]
 
 v_cmpx_lt_u16 v1.h, v255.h
-// GFX12: v_cmpx_lt_u16_e64 v1.h, v255.h          ; encoding: [0x7e,0x18,0xb9,0xd4,0x01,0xff,0x03,0x00]
+// GFX12: v_cmpx_lt_u16_e64 v1.h, v255.h op_sel:[1,1,0] ; encoding: [0x7e,0x18,0xb9,0xd4,0x01,0xff,0x03,0x00]
 
 v_cmpx_lt_u16 v1.h, v255.h dpp8:[7,6,5,4,3,2,1,0]
 // GFX12: v_cmpx_lt_u16_e64_dpp v1.h, v255.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x18,0xb9,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
@@ -506,7 +596,7 @@ v_cmpx_lt_u16 v1.l, v255.l quad_perm:[3,2,1,0]
 // GFX12: v_cmpx_lt_u16_e64_dpp v1.l, v255.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xb9,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
 
 v_cmpx_lt_u16 v255.h, v2.h
-// GFX12: v_cmpx_lt_u16_e64 v255.h, v2.h          ; encoding: [0x7e,0x18,0xb9,0xd4,0xff,0x05,0x02,0x00]
+// GFX12: v_cmpx_lt_u16_e64 v255.h, v2.h op_sel:[1,1,0] ; encoding: [0x7e,0x18,0xb9,0xd4,0xff,0x05,0x02,0x00]
 
 v_cmpx_lt_u16 v255.h, v2.h dpp8:[7,6,5,4,3,2,1,0]
 // GFX12: v_cmpx_lt_u16_e64_dpp v255.h, v2.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x18,0xb9,0xd4,0xe9,0x04,0x02,0x00,0xff,0x77,0x39,0x05]
@@ -524,7 +614,7 @@ v_cmpx_lt_u16 v255.l, v2.l quad_perm:[3,2,1,0]
 // GFX12: v_cmpx_lt_u16_e64_dpp v255.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xb9,0xd4,0xfa,0x04,0x02,0x00,0xff,0x1b,0x00,0xff]
 
 v_cmpx_ne_i16 v1.h, v255.h
-// GFX12: v_cmpx_ne_i16_e64 v1.h, v255.h          ; encoding: [0x7e,0x18,0xb5,0xd4,0x01,0xff,0x03,0x00]
+// GFX12: v_cmpx_ne_i16_e64 v1.h, v255.h op_sel:[1,1,0] ; encoding: [0x7e,0x18,0xb5,0xd4,0x01,0xff,0x03,0x00]
 
 v_cmpx_ne_i16 v1.h, v255.h dpp8:[7,6,5,4,3,2,1,0]
 // GFX12: v_cmpx_ne_i16_e64_dpp v1.h, v255.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x18,0xb5,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
@@ -542,7 +632,7 @@ v_cmpx_ne_i16 v1.l, v255.l quad_perm:[3,2,1,0]
 // GFX12: v_cmpx_ne_i16_e64_dpp v1.l, v255.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xb5,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
 
 v_cmpx_ne_i16 v255.h, v2.h
-// GFX12: v_cmpx_ne_i16_e64 v255.h, v2.h          ; encoding: [0x7e,0x18,0xb5,0xd4,0xff,0x05,0x02,0x00]
+// GFX12: v_cmpx_ne_i16_e64 v255.h, v2.h op_sel:[1,1,0] ; encoding: [0x7e,0x18,0xb5,0xd4,0xff,0x05,0x02,0x00]
 
 v_cmpx_ne_i16 v255.h, v2.h dpp8:[7,6,5,4,3,2,1,0]
 // GFX12: v_cmpx_ne_i16_e64_dpp v255.h, v2.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x18,0xb5,0xd4,0xe9,0x04,0x02,0x00,0xff,0x77,0x39,0x05]
@@ -560,7 +650,7 @@ v_cmpx_ne_i16 v255.l, v2.l quad_perm:[3,2,1,0]
 // GFX12: v_cmpx_ne_i16_e64_dpp v255.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xb5,0xd4,0xfa,0x04,0x02,0x00,0xff,0x1b,0x00,0xff]
 
 v_cmpx_ne_u16 v1.h, v255.h
-// GFX12: v_cmpx_ne_u16_e64 v1.h, v255.h          ; encoding: [0x7e,0x18,0xbd,0xd4,0x01,0xff,0x03,0x00]
+// GFX12: v_cmpx_ne_u16_e64 v1.h, v255.h op_sel:[1,1,0] ; encoding: [0x7e,0x18,0xbd,0xd4,0x01,0xff,0x03,0x00]
 
 v_cmpx_ne_u16 v1.h, v255.h dpp8:[7,6,5,4,3,2,1,0]
 // GFX12: v_cmpx_ne_u16_e64_dpp v1.h, v255.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x18,0xbd,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
@@ -578,7 +668,7 @@ v_cmpx_ne_u16 v1.l, v255.l quad_perm:[3,2,1,0]
 // GFX12: v_cmpx_ne_u16_e64_dpp v1.l, v255.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xbd,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
 
 v_cmpx_ne_u16 v255.h, v2.h
-// GFX12: v_cmpx_ne_u16_e64 v255.h, v2.h          ; encoding: [0x7e,0x18,0xbd,0xd4,0xff,0x05,0x02,0x00]
+// GFX12: v_cmpx_ne_u16_e64 v255.h, v2.h op_sel:[1,1,0] ; encoding: [0x7e,0x18,0xbd,0xd4,0xff,0x05,0x02,0x00]
 
 v_cmpx_ne_u16 v255.h, v2.h dpp8:[7,6,5,4,3,2,1,0]
 // GFX12: v_cmpx_ne_u16_e64_dpp v255.h, v2.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x18,0xbd,0xd4,0xe9,0x04,0x02,0x00,0xff,0x77,0x39,0x05]
@@ -595,146 +685,290 @@ v_cmpx_ne_u16 v255.l, v2.l dpp8:[7,6,5,4,3,2,1,0]
 v_cmpx_ne_u16 v255.l, v2.l quad_perm:[3,2,1,0]
 // GFX12: v_cmpx_ne_u16_e64_dpp v255.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xbd,0xd4,0xfa,0x04,0x02,0x00,0xff,0x1b,0x00,0xff]
 
-v_cmpx_neq_f16 v1, v255
-// GFX12: v_cmpx_neq_f16_e64 v1, v255             ; encoding: [0x7e,0x00,0x8d,0xd4,0x01,0xff,0x03,0x00]
+v_cmpx_neq_f16 v1.h, v255.h
+// GFX12: v_cmpx_neq_f16_e64 v1.h, v255.h op_sel:[1,1,0] ; encoding: [0x7e,0x18,0x8d,0xd4,0x01,0xff,0x03,0x00]
+
+v_cmpx_neq_f16 v1.h, v255.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_cmpx_neq_f16_e64_dpp v1.h, v255.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x18,0x8d,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
+
+v_cmpx_neq_f16 v1.h, v255.h quad_perm:[3,2,1,0]
+// GFX12: v_cmpx_neq_f16_e64_dpp v1.h, v255.h op_sel:[1,1] quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x18,0x8d,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
+
+v_cmpx_neq_f16 v1.l, v255.l
+// GFX12: v_cmpx_neq_f16_e64 v1.l, v255.l         ; encoding: [0x7e,0x00,0x8d,0xd4,0x01,0xff,0x03,0x00]
+
+v_cmpx_neq_f16 v1.l, v255.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_cmpx_neq_f16_e64_dpp v1.l, v255.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x8d,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
+
+v_cmpx_neq_f16 v1.l, v255.l quad_perm:[3,2,1,0]
+// GFX12: v_cmpx_neq_f16_e64_dpp v1.l, v255.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8d,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
+
+v_cmpx_neq_f16 v255.h, v2.h
+// GFX12: v_cmpx_neq_f16_e64 v255.h, v2.h op_sel:[1,1,0] ; encoding: [0x7e,0x18,0x8d,0xd4,0xff,0x05,0x02,0x00]
+
+v_cmpx_neq_f16 v255.h, v2.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_cmpx_neq_f16_e64_dpp v255.h, v2.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x18,0x8d,0xd4,0xe9,0x04,0x02,0x00,0xff,0x77,0x39,0x05]
+
+v_cmpx_neq_f16 v255.h, v2.h quad_perm:[3,2,1,0]
+// GFX12: v_cmpx_neq_f16_e64_dpp v255.h, v2.h op_sel:[1,1] quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x18,0x8d,0xd4,0xfa,0x04,0x02,0x00,0xff,0x1b,0x00,0xff]
+
+v_cmpx_neq_f16 v255.l, v2.l
+// GFX12: v_cmpx_neq_f16_e64 v255.l, v2.l         ; encoding: [0x7e,0x00,0x8d,0xd4,0xff,0x05,0x02,0x00]
+
+v_cmpx_neq_f16 v255.l, v2.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_cmpx_neq_f16_e64_dpp v255.l, v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x8d,0xd4,0xe9,0x04,0x02,0x00,0xff,0x77,0x39,0x05]
+
+v_cmpx_neq_f16 v255.l, v2.l quad_perm:[3,2,1,0]
+// GFX12: v_cmpx_neq_f16_e64_dpp v255.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8d,0xd4,0xfa,0x04,0x02,0x00,0xff,0x1b,0x00,0xff]
+
+v_cmpx_nge_f16 v1.h, v255.h
+// GFX12: v_cmpx_nge_f16_e64 v1.h, v255.h op_sel:[1,1,0] ; encoding: [0x7e,0x18,0x89,0xd4,0x01,0xff,0x03,0x00]
+
+v_cmpx_nge_f16 v1.h, v255.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_cmpx_nge_f16_e64_dpp v1.h, v255.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x18,0x89,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
+
+v_cmpx_nge_f16 v1.h, v255.h quad_perm:[3,2,1,0]
+// GFX12: v_cmpx_nge_f16_e64_dpp v1.h, v255.h op_sel:[1,1] quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x18,0x89,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
+
+v_cmpx_nge_f16 v1.l, v255.l
+// GFX12: v_cmpx_nge_f16_e64 v1.l, v255.l         ; encoding: [0x7e,0x00,0x89,0xd4,0x01,0xff,0x03,0x00]
+
+v_cmpx_nge_f16 v1.l, v255.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_cmpx_nge_f16_e64_dpp v1.l, v255.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x89,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
+
+v_cmpx_nge_f16 v1.l, v255.l quad_perm:[3,2,1,0]
+// GFX12: v_cmpx_nge_f16_e64_dpp v1.l, v255.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x89,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
+
+v_cmpx_nge_f16 v255.h, v2.h
+// GFX12: v_cmpx_nge_f16_e64 v255.h, v2.h op_sel:[1,1,0] ; encoding: [0x7e,0x18,0x89,0xd4,0xff,0x05,0x02,0x00]
+
+v_cmpx_nge_f16 v255.h, v2.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_cmpx_nge_f16_e64_dpp v255.h, v2.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x18,0x89,0xd4,0xe9,0x04,0x02,0x00,0xff,0x77,0x39,0x05]
+
+v_cmpx_nge_f16 v255.h, v2.h quad_perm:[3,2,1,0]
+// GFX12: v_cmpx_nge_f16_e64_dpp v255.h, v2.h op_sel:[1,1] quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x18,0x89,0xd4,0xfa,0x04,0x02,0x00,0xff,0x1b,0x00,0xff]
+
+v_cmpx_nge_f16 v255.l, v2.l
+// GFX12: v_cmpx_nge_f16_e64 v255.l, v2.l         ; encoding: [0x7e,0x00,0x89,0xd4,0xff,0x05,0x02,0x00]
+
+v_cmpx_nge_f16 v255.l, v2.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_cmpx_nge_f16_e64_dpp v255.l, v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x89,0xd4,0xe9,0x04,0x02,0x00,0xff,0x77,0x39,0x05]
+
+v_cmpx_nge_f16 v255.l, v2.l quad_perm:[3,2,1,0]
+// GFX12: v_cmpx_nge_f16_e64_dpp v255.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x89,0xd4,0xfa,0x04,0x02,0x00,0xff,0x1b,0x00,0xff]
+
+v_cmpx_ngt_f16 v1.h, v255.h
+// GFX12: v_cmpx_ngt_f16_e64 v1.h, v255.h op_sel:[1,1,0] ; encoding: [0x7e,0x18,0x8b,0xd4,0x01,0xff,0x03,0x00]
+
+v_cmpx_ngt_f16 v1.h, v255.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_cmpx_ngt_f16_e64_dpp v1.h, v255.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x18,0x8b,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
+
+v_cmpx_ngt_f16 v1.h, v255.h quad_perm:[3,2,1,0]
+// GFX12: v_cmpx_ngt_f16_e64_dpp v1.h, v255.h op_sel:[1,1] quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x18,0x8b,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
+
+v_cmpx_ngt_f16 v1.l, v255.l
+// GFX12: v_cmpx_ngt_f16_e64 v1.l, v255.l         ; encoding: [0x7e,0x00,0x8b,0xd4,0x01,0xff,0x03,0x00]
+
+v_cmpx_ngt_f16 v1.l, v255.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_cmpx_ngt_f16_e64_dpp v1.l, v255.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x8b,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
+
+v_cmpx_ngt_f16 v1.l, v255.l quad_perm:[3,2,1,0]
+// GFX12: v_cmpx_ngt_f16_e64_dpp v1.l, v255.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8b,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
+
+v_cmpx_ngt_f16 v255.h, v2.h
+// GFX12: v_cmpx_ngt_f16_e64 v255.h, v2.h op_sel:[1,1,0] ; encoding: [0x7e,0x18,0x8b,0xd4,0xff,0x05,0x02,0x00]
+
+v_cmpx_ngt_f16 v255.h, v2.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_cmpx_ngt_f16_e64_dpp v255.h, v2.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x18,0x8b,0xd4,0xe9,0x04,0x02,0x00,0xff,0x77,0x39,0x05]
+
+v_cmpx_ngt_f16 v255.h, v2.h quad_perm:[3,2,1,0]
+// GFX12: v_cmpx_ngt_f16_e64_dpp v255.h, v2.h op_sel:[1,1] quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x18,0x8b,0xd4,0xfa,0x04,0x02,0x00,0xff,0x1b,0x00,0xff]
+
+v_cmpx_ngt_f16 v255.l, v2.l
+// GFX12: v_cmpx_ngt_f16_e64 v255.l, v2.l         ; encoding: [0x7e,0x00,0x8b,0xd4,0xff,0x05,0x02,0x00]
+
+v_cmpx_ngt_f16 v255.l, v2.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_cmpx_ngt_f16_e64_dpp v255.l, v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x8b,0xd4,0xe9,0x04,0x02,0x00,0xff,0x77,0x39,0x05]
+
+v_cmpx_ngt_f16 v255.l, v2.l quad_perm:[3,2,1,0]
+// GFX12: v_cmpx_ngt_f16_e64_dpp v255.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8b,0xd4,0xfa,0x04,0x02,0x00,0xff,0x1b,0x00,0xff]
+
+v_cmpx_nle_f16 v1.h, v255.h
+// GFX12: v_cmpx_nle_f16_e64 v1.h, v255.h op_sel:[1,1,0] ; encoding: [0x7e,0x18,0x8c,0xd4,0x01,0xff,0x03,0x00]
+
+v_cmpx_nle_f16 v1.h, v255.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_cmpx_nle_f16_e64_dpp v1.h, v255.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x18,0x8c,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
+
+v_cmpx_nle_f16 v1.h, v255.h quad_perm:[3,2,1,0]
+// GFX12: v_cmpx_nle_f16_e64_dpp v1.h, v255.h op_sel:[1,1] quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x18,0x8c,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
+
+v_cmpx_nle_f16 v1.l, v255.l
+// GFX12: v_cmpx_nle_f16_e64 v1.l, v255.l         ; encoding: [0x7e,0x00,0x8c,0xd4,0x01,0xff,0x03,0x00]
+
+v_cmpx_nle_f16 v1.l, v255.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_cmpx_nle_f16_e64_dpp v1.l, v255.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x8c,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
+
+v_cmpx_nle_f16 v1.l, v255.l quad_perm:[3,2,1,0]
+// GFX12: v_cmpx_nle_f16_e64_dpp v1.l, v255.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8c,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
+
+v_cmpx_nle_f16 v255.h, v2.h
+// GFX12: v_cmpx_nle_f16_e64 v255.h, v2.h op_sel:[1,1,0] ; encoding: [0x7e,0x18,0x8c,0xd4,0xff,0x05,0x02,0x00]
+
+v_cmpx_nle_f16 v255.h, v2.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_cmpx_nle_f16_e64_dpp v255.h, v2.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x18,0x8c,0xd4,0xe9,0x04,0x02,0x00,0xff,0x77,0x39,0x05]
+
+v_cmpx_nle_f16 v255.h, v2.h quad_perm:[3,2,1,0]
+// GFX12: v_cmpx_nle_f16_e64_dpp v255.h, v2.h op_sel:[1,1] quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x18,0x8c,0xd4,0xfa,0x04,0x02,0x00,0xff,0x1b,0x00,0xff]
+
+v_cmpx_nle_f16 v255.l, v2.l
+// GFX12: v_cmpx_nle_f16_e64 v255.l, v2.l         ; encoding: [0x7e,0x00,0x8c,0xd4,0xff,0x05,0x02,0x00]
+
+v_cmpx_nle_f16 v255.l, v2.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_cmpx_nle_f16_e64_dpp v255.l, v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x8c,0xd4,0xe9,0x04,0x02,0x00,0xff,0x77,0x39,0x05]
+
+v_cmpx_nle_f16 v255.l, v2.l quad_perm:[3,2,1,0]
+// GFX12: v_cmpx_nle_f16_e64_dpp v255.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8c,0xd4,0xfa,0x04,0x02,0x00,0xff,0x1b,0x00,0xff]
+
+v_cmpx_nlg_f16 v1.h, v255.h
+// GFX12: v_cmpx_nlg_f16_e64 v1.h, v255.h op_sel:[1,1,0] ; encoding: [0x7e,0x18,0x8a,0xd4,0x01,0xff,0x03,0x00]
 
-v_cmpx_neq_f16 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_cmpx_neq_f16_e64_dpp v1, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x8d,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
+v_cmpx_nlg_f16 v1.h, v255.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_cmpx_nlg_f16_e64_dpp v1.h, v255.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x18,0x8a,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
 
-v_cmpx_neq_f16 v1, v255 quad_perm:[3,2,1,0]
-// GFX12: v_cmpx_neq_f16_e64_dpp v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8d,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
+v_cmpx_nlg_f16 v1.h, v255.h quad_perm:[3,2,1,0]
+// GFX12: v_cmpx_nlg_f16_e64_dpp v1.h, v255.h op_sel:[1,1] quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x18,0x8a,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
 
-v_cmpx_neq_f16 v255, v2
-// GFX12: v_cmpx_neq_f16_e64 v255, v2             ; encoding: [0x7e,0x00,0x8d,0xd4,0xff,0x05,0x02,0x00]
+v_cmpx_nlg_f16 v1.l, v255.l
+// GFX12: v_cmpx_nlg_f16_e64 v1.l, v255.l         ; encoding: [0x7e,0x00,0x8a,0xd4,0x01,0xff,0x03,0x00]
 
-v_cmpx_neq_f16 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_cmpx_neq_f16_e64_dpp v255, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x8d,0xd4,0xe9,0x04,0x02,0x00,0xff,0x77,0x39,0x05]
+v_cmpx_nlg_f16 v1.l, v255.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_cmpx_nlg_f16_e64_dpp v1.l, v255.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x8a,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
 
-v_cmpx_neq_f16 v255, v2 quad_perm:[3,2,1,0]
-// GFX12: v_cmpx_neq_f16_e64_dpp v255, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8d,0xd4,0xfa,0x04,0x02,0x00,0xff,0x1b,0x00,0xff]
+v_cmpx_nlg_f16 v1.l, v255.l quad_perm:[3,2,1,0]
+// GFX12: v_cmpx_nlg_f16_e64_dpp v1.l, v255.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8a,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
 
-v_cmpx_nge_f16 v1, v255
-// GFX12: v_cmpx_nge_f16_e64 v1, v255             ; encoding: [0x7e,0x00,0x89,0xd4,0x01,0xff,0x03,0x00]
+v_cmpx_nlg_f16 v255.h, v2.h
+// GFX12: v_cmpx_nlg_f16_e64 v255.h, v2.h op_sel:[1,1,0] ; encoding: [0x7e,0x18,0x8a,0xd4,0xff,0x05,0x02,0x00]
 
-v_cmpx_nge_f16 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_cmpx_nge_f16_e64_dpp v1, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x89,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
+v_cmpx_nlg_f16 v255.h, v2.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_cmpx_nlg_f16_e64_dpp v255.h, v2.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x18,0x8a,0xd4,0xe9,0x04,0x02,0x00,0xff,0x77,0x39,0x05]
 
-v_cmpx_nge_f16 v1, v255 quad_perm:[3,2,1,0]
-// GFX12: v_cmpx_nge_f16_e64_dpp v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x89,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
+v_cmpx_nlg_f16 v255.h, v2.h quad_perm:[3,2,1,0]
+// GFX12: v_cmpx_nlg_f16_e64_dpp v255.h, v2.h op_sel:[1,1] quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x18,0x8a,0xd4,0xfa,0x04,0x02,0x00,0xff,0x1b,0x00,0xff]
 
-v_cmpx_nge_f16 v255, v2
-// GFX12: v_cmpx_nge_f16_e64 v255, v2             ; encoding: [0x7e,0x00,0x89,0xd4,0xff,0x05,0x02,0x00]
+v_cmpx_nlg_f16 v255.l, v2.l
+// GFX12: v_cmpx_nlg_f16_e64 v255.l, v2.l         ; encoding: [0x7e,0x00,0x8a,0xd4,0xff,0x05,0x02,0x00]
 
-v_cmpx_nge_f16 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_cmpx_nge_f16_e64_dpp v255, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x89,0xd4,0xe9,0x04,0x02,0x00,0xff,0x77,0x39,0x05]
+v_cmpx_nlg_f16 v255.l, v2.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_cmpx_nlg_f16_e64_dpp v255.l, v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x8a,0xd4,0xe9,0x04,0x02,0x00,0xff,0x77,0x39,0x05]
 
-v_cmpx_nge_f16 v255, v2 quad_perm:[3,2,1,0]
-// GFX12: v_cmpx_nge_f16_e64_dpp v255, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x89,0xd4,0xfa,0x04,0x02,0x00,0xff,0x1b,0x00,0xff]
+v_cmpx_nlg_f16 v255.l, v2.l quad_perm:[3,2,1,0]
+// GFX12: v_cmpx_nlg_f16_e64_dpp v255.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8a,0xd4,0xfa,0x04,0x02,0x00,0xff,0x1b,0x00,0xff]
 
-v_cmpx_ngt_f16 v1, v255
-// GFX12: v_cmpx_ngt_f16_e64 v1, v255             ; encoding: [0x7e,0x00,0x8b,0xd4,0x01,0xff,0x03,0x00]
+v_cmpx_nlt_f16 v1.h, v255.h
+// GFX12: v_cmpx_nlt_f16_e64 v1.h, v255.h op_sel:[1,1,0] ; encoding: [0x7e,0x18,0x8e,0xd4,0x01,0xff,0x03,0x00]
 
-v_cmpx_ngt_f16 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_cmpx_ngt_f16_e64_dpp v1, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x8b,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
+v_cmpx_nlt_f16 v1.h, v255.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_cmpx_nlt_f16_e64_dpp v1.h, v255.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x18,0x8e,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
 
-v_cmpx_ngt_f16 v1, v255 quad_perm:[3,2,1,0]
-// GFX12: v_cmpx_ngt_f16_e64_dpp v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8b,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
+v_cmpx_nlt_f16 v1.h, v255.h quad_perm:[3,2,1,0]
+// GFX12: v_cmpx_nlt_f16_e64_dpp v1.h, v255.h op_sel:[1,1] quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x18,0x8e,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
 
-v_cmpx_ngt_f16 v255, v2
-// GFX12: v_cmpx_ngt_f16_e64 v255, v2             ; encoding: [0x7e,0x00,0x8b,0xd4,0xff,0x05,0x02,0x00]
+v_cmpx_nlt_f16 v1.l, v255.l
+// GFX12: v_cmpx_nlt_f16_e64 v1.l, v255.l         ; encoding: [0x7e,0x00,0x8e,0xd4,0x01,0xff,0x03,0x00]
 
-v_cmpx_ngt_f16 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_cmpx_ngt_f16_e64_dpp v255, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x8b,0xd4,0xe9,0x04,0x02,0x00,0xff,0x77,0x39,0x05]
+v_cmpx_nlt_f16 v1.l, v255.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_cmpx_nlt_f16_e64_dpp v1.l, v255.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x8e,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
 
-v_cmpx_ngt_f16 v255, v2 quad_perm:[3,2,1,0]
-// GFX12: v_cmpx_ngt_f16_e64_dpp v255, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8b,0xd4,0xfa,0x04,0x02,0x00,0xff,0x1b,0x00,0xff]
+v_cmpx_nlt_f16 v1.l, v255.l quad_perm:[3,2,1,0]
+// GFX12: v_cmpx_nlt_f16_e64_dpp v1.l, v255.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8e,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
 
-v_cmpx_nle_f16 v1, v255
-// GFX12: v_cmpx_nle_f16_e64 v1, v255             ; encoding: [0x7e,0x00,0x8c,0xd4,0x01,0xff,0x03,0x00]
+v_cmpx_nlt_f16 v255.h, v2.h
+// GFX12: v_cmpx_nlt_f16_e64 v255.h, v2.h op_sel:[1,1,0] ; encoding: [0x7e,0x18,0x8e,0xd4,0xff,0x05,0x02,0x00]
 
-v_cmpx_nle_f16 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_cmpx_nle_f16_e64_dpp v1, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x8c,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
+v_cmpx_nlt_f16 v255.h, v2.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_cmpx_nlt_f16_e64_dpp v255.h, v2.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x18,0x8e,0xd4,0xe9,0x04,0x02,0x00,0xff,0x77,0x39,0x05]
 
-v_cmpx_nle_f16 v1, v255 quad_perm:[3,2,1,0]
-// GFX12: v_cmpx_nle_f16_e64_dpp v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8c,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
+v_cmpx_nlt_f16 v255.h, v2.h quad_perm:[3,2,1,0]
+// GFX12: v_cmpx_nlt_f16_e64_dpp v255.h, v2.h op_sel:[1,1] quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x18,0x8e,0xd4,0xfa,0x04,0x02,0x00,0xff,0x1b,0x00,0xff]
 
-v_cmpx_nle_f16 v255, v2
-// GFX12: v_cmpx_nle_f16_e64 v255, v2             ; encoding: [0x7e,0x00,0x8c,0xd4,0xff,0x05,0x02,0x00]
+v_cmpx_nlt_f16 v255.l, v2.l
+// GFX12: v_cmpx_nlt_f16_e64 v255.l, v2.l         ; encoding: [0x7e,0x00,0x8e,0xd4,0xff,0x05,0x02,0x00]
 
-v_cmpx_nle_f16 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_cmpx_nle_f16_e64_dpp v255, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x8c,0xd4,0xe9,0x04,0x02,0x00,0xff,0x77,0x39,0x05]
+v_cmpx_nlt_f16 v255.l, v2.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_cmpx_nlt_f16_e64_dpp v255.l, v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x8e,0xd4,0xe9,0x04,0x02,0x00,0xff,0x77,0x39,0x05]
 
-v_cmpx_nle_f16 v255, v2 quad_perm:[3,2,1,0]
-// GFX12: v_cmpx_nle_f16_e64_dpp v255, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8c,0xd4,0xfa,0x04,0x02,0x00,0xff,0x1b,0x00,0xff]
+v_cmpx_nlt_f16 v255.l, v2.l quad_perm:[3,2,1,0]
+// GFX12: v_cmpx_nlt_f16_e64_dpp v255.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8e,0xd4,0xfa,0x04,0x02,0x00,0xff,0x1b,0x00,0xff]
 
-v_cmpx_nlg_f16 v1, v255
-// GFX12: v_cmpx_nlg_f16_e64 v1, v255             ; encoding: [0x7e,0x00,0x8a,0xd4,0x01,0xff,0x03,0x00]
+v_cmpx_o_f16 v1.h, v255.h
+// GFX12: v_cmpx_o_f16_e64 v1.h, v255.h op_sel:[1,1,0] ; encoding: [0x7e,0x18,0x87,0xd4,0x01,0xff,0x03,0x00]
 
-v_cmpx_nlg_f16 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_cmpx_nlg_f16_e64_dpp v1, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x8a,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
+v_cmpx_o_f16 v1.h, v255.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_cmpx_o_f16_e64_dpp v1.h, v255.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x18,0x87,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
 
-v_cmpx_nlg_f16 v1, v255 quad_perm:[3,2,1,0]
-// GFX12: v_cmpx_nlg_f16_e64_dpp v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8a,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
+v_cmpx_o_f16 v1.h, v255.h quad_perm:[3,2,1,0]
+// GFX12: v_cmpx_o_f16_e64_dpp v1.h, v255.h op_sel:[1,1] quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x18,0x87,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
 
-v_cmpx_nlg_f16 v255, v2
-// GFX12: v_cmpx_nlg_f16_e64 v255, v2             ; encoding: [0x7e,0x00,0x8a,0xd4,0xff,0x05,0x02,0x00]
+v_cmpx_o_f16 v1.l, v255.l
+// GFX12: v_cmpx_o_f16_e64 v1.l, v255.l           ; encoding: [0x7e,0x00,0x87,0xd4,0x01,0xff,0x03,0x00]
 
-v_cmpx_nlg_f16 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_cmpx_nlg_f16_e64_dpp v255, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x8a,0xd4,0xe9,0x04,0x02,0x00,0xff,0x77,0x39,0x05]
+v_cmpx_o_f16 v1.l, v255.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_cmpx_o_f16_e64_dpp v1.l, v255.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x87,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
 
-v_cmpx_nlg_f16 v255, v2 quad_perm:[3,2,1,0]
-// GFX12: v_cmpx_nlg_f16_e64_dpp v255, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8a,0xd4,0xfa,0x04,0x02,0x00,0xff,0x1b,0x00,0xff]
+v_cmpx_o_f16 v1.l, v255.l quad_perm:[3,2,1,0]
+// GFX12: v_cmpx_o_f16_e64_dpp v1.l, v255.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x87,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
 
-v_cmpx_nlt_f16 v1, v255
-// GFX12: v_cmpx_nlt_f16_e64 v1, v255             ; encoding: [0x7e,0x00,0x8e,0xd4,0x01,0xff,0x03,0x00]
+v_cmpx_o_f16 v255.h, v2.h
+// GFX12: v_cmpx_o_f16_e64 v255.h, v2.h op_sel:[1,1,0] ; encoding: [0x7e,0x18,0x87,0xd4,0xff,0x05,0x02,0x00]
 
-v_cmpx_nlt_f16 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_cmpx_nlt_f16_e64_dpp v1, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x8e,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
+v_cmpx_o_f16 v255.h, v2.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_cmpx_o_f16_e64_dpp v255.h, v2.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x18,0x87,0xd4,0xe9,0x04,0x02,0x00,0xff,0x77,0x39,0x05]
 
-v_cmpx_nlt_f16 v1, v255 quad_perm:[3,2,1,0]
-// GFX12: v_cmpx_nlt_f16_e64_dpp v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8e,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
+v_cmpx_o_f16 v255.h, v2.h quad_perm:[3,2,1,0]
+// GFX12: v_cmpx_o_f16_e64_dpp v255.h, v2.h op_sel:[1,1] quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x18,0x87,0xd4,0xfa,0x04,0x02,0x00,0xff,0x1b,0x00,0xff]
 
-v_cmpx_nlt_f16 v255, v2
-// GFX12: v_cmpx_nlt_f16_e64 v255, v2             ; encoding: [0x7e,0x00,0x8e,0xd4,0xff,0x05,0x02,0x00]
+v_cmpx_o_f16 v255.l, v2.l
+// GFX12: v_cmpx_o_f16_e64 v255.l, v2.l           ; encoding: [0x7e,0x00,0x87,0xd4,0xff,0x05,0x02,0x00]
 
-v_cmpx_nlt_f16 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_cmpx_nlt_f16_e64_dpp v255, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x8e,0xd4,0xe9,0x04,0x02,0x00,0xff,0x77,0x39,0x05]
+v_cmpx_o_f16 v255.l, v2.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_cmpx_o_f16_e64_dpp v255.l, v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x87,0xd4,0xe9,0x04,0x02,0x00,0xff,0x77,0x39,0x05]
 
-v_cmpx_nlt_f16 v255, v2 quad_perm:[3,2,1,0]
-// GFX12: v_cmpx_nlt_f16_e64_dpp v255, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8e,0xd4,0xfa,0x04,0x02,0x00,0xff,0x1b,0x00,0xff]
+v_cmpx_o_f16 v255.l, v2.l quad_perm:[3,2,1,0]
+// GFX12: v_cmpx_o_f16_e64_dpp v255.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x87,0xd4,0xfa,0x04,0x02,0x00,0xff,0x1b,0x00,0xff]
 
-v_cmpx_o_f16 v1, v255
-// GFX12: v_cmpx_o_f16_e64 v1, v255               ; encoding: [0x7e,0x00,0x87,0xd4,0x01,0xff,0x03,0x00]
+v_cmpx_u_f16 v1.h, v255.h
+// GFX12: v_cmpx_u_f16_e64 v1.h, v255.h op_sel:[1,1,0] ; encoding: [0x7e,0x18,0x88,0xd4,0x01,0xff,0x03,0x00]
 
-v_cmpx_o_f16 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_cmpx_o_f16_e64_dpp v1, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x87,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
+v_cmpx_u_f16 v1.h, v255.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_cmpx_u_f16_e64_dpp v1.h, v255.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x18,0x88,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
 
-v_cmpx_o_f16 v1, v255 quad_perm:[3,2,1,0]
-// GFX12: v_cmpx_o_f16_e64_dpp v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x87,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
+v_cmpx_u_f16 v1.h, v255.h quad_perm:[3,2,1,0]
+// GFX12: v_cmpx_u_f16_e64_dpp v1.h, v255.h op_sel:[1,1] quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x18,0x88,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
 
-v_cmpx_o_f16 v255, v2
-// GFX12: v_cmpx_o_f16_e64 v255, v2               ; encoding: [0x7e,0x00,0x87,0xd4,0xff,0x05,0x02,0x00]
+v_cmpx_u_f16 v1.l, v255.l
+// GFX12: v_cmpx_u_f16_e64 v1.l, v255.l           ; encoding: [0x7e,0x00,0x88,0xd4,0x01,0xff,0x03,0x00]
 
-v_cmpx_o_f16 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_cmpx_o_f16_e64_dpp v255, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x87,0xd4,0xe9,0x04,0x02,0x00,0xff,0x77,0x39,0x05]
+v_cmpx_u_f16 v1.l, v255.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_cmpx_u_f16_e64_dpp v1.l, v255.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x88,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
 
-v_cmpx_o_f16 v255, v2 quad_perm:[3,2,1,0]
-// GFX12: v_cmpx_o_f16_e64_dpp v255, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x87,0xd4,0xfa,0x04,0x02,0x00,0xff,0x1b,0x00,0xff]
+v_cmpx_u_f16 v1.l, v255.l quad_perm:[3,2,1,0]
+// GFX12: v_cmpx_u_f16_e64_dpp v1.l, v255.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x88,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
 
-v_cmpx_u_f16 v1, v255
-// GFX12: v_cmpx_u_f16_e64 v1, v255               ; encoding: [0x7e,0x00,0x88,0xd4,0x01,0xff,0x03,0x00]
+v_cmpx_u_f16 v255.h, v2.h
+// GFX12: v_cmpx_u_f16_e64 v255.h, v2.h op_sel:[1,1,0] ; encoding: [0x7e,0x18,0x88,0xd4,0xff,0x05,0x02,0x00]
 
-v_cmpx_u_f16 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_cmpx_u_f16_e64_dpp v1, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x88,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
+v_cmpx_u_f16 v255.h, v2.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_cmpx_u_f16_e64_dpp v255.h, v2.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x18,0x88,0xd4,0xe9,0x04,0x02,0x00,0xff,0x77,0x39,0x05]
 
-v_cmpx_u_f16 v1, v255 quad_perm:[3,2,1,0]
-// GFX12: v_cmpx_u_f16_e64_dpp v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x88,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
+v_cmpx_u_f16 v255.h, v2.h quad_perm:[3,2,1,0]
+// GFX12: v_cmpx_u_f16_e64_dpp v255.h, v2.h op_sel:[1,1] quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x18,0x88,0xd4,0xfa,0x04,0x02,0x00,0xff,0x1b,0x00,0xff]
 
-v_cmpx_u_f16 v255, v2
-// GFX12: v_cmpx_u_f16_e64 v255, v2               ; encoding: [0x7e,0x00,0x88,0xd4,0xff,0x05,0x02,0x00]
+v_cmpx_u_f16 v255.l, v2.l
+// GFX12: v_cmpx_u_f16_e64 v255.l, v2.l           ; encoding: [0x7e,0x00,0x88,0xd4,0xff,0x05,0x02,0x00]
 
-v_cmpx_u_f16 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_cmpx_u_f16_e64_dpp v255, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x88,0xd4,0xe9,0x04,0x02,0x00,0xff,0x77,0x39,0x05]
+v_cmpx_u_f16 v255.l, v2.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_cmpx_u_f16_e64_dpp v255.l, v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x88,0xd4,0xe9,0x04,0x02,0x00,0xff,0x77,0x39,0x05]
 
-v_cmpx_u_f16 v255, v2 quad_perm:[3,2,1,0]
-// GFX12: v_cmpx_u_f16_e64_dpp v255, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x88,0xd4,0xfa,0x04,0x02,0x00,0xff,0x1b,0x00,0xff]
+v_cmpx_u_f16 v255.l, v2.l quad_perm:[3,2,1,0]
+// GFX12: v_cmpx_u_f16_e64_dpp v255.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x88,0xd4,0xfa,0x04,0x02,0x00,0xff,0x1b,0x00,0xff]
diff --git a/llvm/test/MC/AMDGPU/gfx950-unsupported.s b/llvm/test/MC/AMDGPU/gfx950-unsupported.s
index 225784177ae18..8bdab2da2394c 100644
--- a/llvm/test/MC/AMDGPU/gfx950-unsupported.s
+++ b/llvm/test/MC/AMDGPU/gfx950-unsupported.s
@@ -239,10 +239,6 @@ ds_read_b64_tr_b16 v[2:3], v2 offset:-64
 //===----------------------------------------------------------------------===//
 // ds_read_b96_tr_b6
 //===----------------------------------------------------------------------===//
-ds_read_b96_tr_b6 v[1:3], v0
-// ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid register class: vgpr tuples must be 64 bit aligned
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: instruction not supported on this GPU
-
 ds_read_b96_tr_b6 v1, v0
 // ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: instruction not supported on this GPU
diff --git a/llvm/test/MC/AMDGPU/gfx950_asm_read_tr.s b/llvm/test/MC/AMDGPU/gfx950_asm_read_tr.s
index a6907caafcbb6..e6606ac8b72d0 100644
--- a/llvm/test/MC/AMDGPU/gfx950_asm_read_tr.s
+++ b/llvm/test/MC/AMDGPU/gfx950_asm_read_tr.s
@@ -32,3 +32,11 @@ ds_read_b96_tr_b6 v[0:2], v0
 ds_read_b96_tr_b6 v[2:4], v2 offset:64
 // GFX940-ERR: [[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
 // GFX950: encoding: [0x40,0x00,0xc2,0xd9,0x02,0x00,0x00,0x02]
+
+ds_read_b96_tr_b6 v[1:3], v0
+// GFX940-ERR: [[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+// GFX950: encoding: [0x00,0x00,0xc2,0xd9,0x00,0x00,0x00,0x01]
+
+ds_read_b96_tr_b6 v[1:3], v2 offset:64
+// GFX940-ERR: [[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+// GFX950: encoding: [0x40,0x00,0xc2,0xd9,0x02,0x00,0x00,0x01]
diff --git a/llvm/test/MC/AMDGPU/hsa-diag-v4.s b/llvm/test/MC/AMDGPU/hsa-diag-v4.s
index 3733b162edcfb..9ab177cf2b125 100644
--- a/llvm/test/MC/AMDGPU/hsa-diag-v4.s
+++ b/llvm/test/MC/AMDGPU/hsa-diag-v4.s
@@ -280,6 +280,16 @@
   .amdhsa_shared_vgpr_count 15
 .end_amdhsa_kernel
 
+// GCN-LABEL: warning: test_amdhsa_inst_pref_size_invalid
+// PREGFX10: error: directive requires gfx11+
+// NONAMDHSA: error: unknown directive
+.warning "test_amdhsa_inst_pref_size_invalid"
+.amdhsa_kernel test_amdhsa_inst_pref_size_invalid
+  .amdhsa_next_free_vgpr 273
+  .amdhsa_next_free_sgpr 0
+  .amdhsa_inst_pref_size 15
+.end_amdhsa_kernel
+
 // GCN-LABEL: warning: test_next_free_vgpr_invalid
 // AMDHSA: error: .amdgcn.next_free_{v,s}gpr symbols must be absolute expressions
 // NONAMDHSA-NOT: error:
diff --git a/llvm/test/MC/AMDGPU/hsa-gfx12-v4.s b/llvm/test/MC/AMDGPU/hsa-gfx12-v4.s
index ea649bc76116a..e90a976008229 100644
--- a/llvm/test/MC/AMDGPU/hsa-gfx12-v4.s
+++ b/llvm/test/MC/AMDGPU/hsa-gfx12-v4.s
@@ -33,7 +33,7 @@
 // complete
 // OBJDUMP-NEXT: 0040 01000000 01000000 08000000 00000000
 // OBJDUMP-NEXT: 0050 00000000 00000000 00000000 00000000
-// OBJDUMP-NEXT: 0060 00000000 00000000 00000000 00000000
+// OBJDUMP-NEXT: 0060 00000000 00000000 00000000 f00f0000
 // OBJDUMP-NEXT: 0070 015021e4 1f0f007f 5e040000 00000000
 // special_sgpr
 // OBJDUMP-NEXT: 0080 00000000 00000000 00000000 00000000
@@ -120,6 +120,7 @@ disabled_user_sgpr:
   .amdhsa_workgroup_processor_mode 1
   .amdhsa_memory_ordered 1
   .amdhsa_forward_progress 1
+  .amdhsa_inst_pref_size 255
   .amdhsa_round_robin_scheduling 1
   .amdhsa_exception_fp_ieee_invalid_op 1
   .amdhsa_exception_fp_denorm_src 1
@@ -158,6 +159,7 @@ disabled_user_sgpr:
 // ASM-NEXT: .amdhsa_workgroup_processor_mode 1
 // ASM-NEXT: .amdhsa_memory_ordered 1
 // ASM-NEXT: .amdhsa_forward_progress 1
+// ASM-NEXT: .amdhsa_inst_pref_size 255
 // ASM-NEXT: .amdhsa_round_robin_scheduling 1
 // ASM-NEXT: .amdhsa_exception_fp_ieee_invalid_op 1
 // ASM-NEXT: .amdhsa_exception_fp_denorm_src 1
diff --git a/llvm/test/MC/AMDGPU/hsa-sym-exprs-gfx11.s b/llvm/test/MC/AMDGPU/hsa-sym-exprs-gfx11.s
index 85a7ad05b00f4..68cf28f2ac49d 100644
--- a/llvm/test/MC/AMDGPU/hsa-sym-exprs-gfx11.s
+++ b/llvm/test/MC/AMDGPU/hsa-sym-exprs-gfx11.s
@@ -133,6 +133,7 @@ expr_defined:
 // ASM-NEXT: .amdhsa_memory_ordered (((((((((((((((((((1621884928|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~63))|(((alignto(max(defined_value+4, 1), 8))/8)-1))&(~960))&1073741824)>>30
 // ASM-NEXT: .amdhsa_forward_progress (((((((((((((((((((1621884928|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~63))|(((alignto(max(defined_value+4, 1), 8))/8)-1))&(~960))&2147483648)>>31
 // ASM-NEXT: .amdhsa_shared_vgpr_count 0
+// ASM-NEXT: .amdhsa_inst_pref_size 0
 // ASM-NEXT: .amdhsa_exception_fp_ieee_invalid_op (((((((((((((((((((((((((((128|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~1))|defined_boolean)&(~62))&16777216)>>24
 // ASM-NEXT: .amdhsa_exception_fp_denorm_src (((((((((((((((((((((((((((128|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~1))|defined_boolean)&(~62))&33554432)>>25
 // ASM-NEXT: .amdhsa_exception_fp_ieee_div_zero (((((((((((((((((((((((((((128|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~1))|defined_boolean)&(~62))&67108864)>>26
@@ -180,6 +181,7 @@ expr_defined:
 // ASM-NEXT: .amdhsa_memory_ordered 1
 // ASM-NEXT: .amdhsa_forward_progress 1
 // ASM-NEXT: .amdhsa_shared_vgpr_count 0
+// ASM-NEXT: .amdhsa_inst_pref_size 0
 // ASM-NEXT: .amdhsa_exception_fp_ieee_invalid_op 1
 // ASM-NEXT: .amdhsa_exception_fp_denorm_src 1
 // ASM-NEXT: .amdhsa_exception_fp_ieee_div_zero 1
diff --git a/llvm/test/MC/AMDGPU/hsa-sym-exprs-gfx12.s b/llvm/test/MC/AMDGPU/hsa-sym-exprs-gfx12.s
index 51d0fb30b320c..6f7a9a2605681 100644
--- a/llvm/test/MC/AMDGPU/hsa-sym-exprs-gfx12.s
+++ b/llvm/test/MC/AMDGPU/hsa-sym-exprs-gfx12.s
@@ -9,12 +9,12 @@
 // expr_defined_later
 // OBJDUMP-NEXT: 0000 2b000000 2c000000 00000000 00000000
 // OBJDUMP-NEXT: 0010 00000000 00000000 00000000 00000000
-// OBJDUMP-NEXT: 0020 00000000 00000000 00000000 00000000
+// OBJDUMP-NEXT: 0020 00000000 00000000 00000000 f0020000
 // OBJDUMP-NEXT: 0030 05f02fe4 811f007f 000c0000 00000000
 // expr_defined
 // OBJDUMP-NEXT: 0040 2a000000 2b000000 00000000 00000000
 // OBJDUMP-NEXT: 0050 00000000 00000000 00000000 00000000
-// OBJDUMP-NEXT: 0060 00000000 00000000 00000000 00000000
+// OBJDUMP-NEXT: 0060 00000000 00000000 00000000 f0020000
 // OBJDUMP-NEXT: 0070 05f02fe4 811f007f 000c0000 00000000
 
 .text
@@ -53,6 +53,7 @@ expr_defined:
   .amdhsa_workgroup_processor_mode defined_boolean
   .amdhsa_memory_ordered defined_boolean
   .amdhsa_forward_progress defined_boolean
+  .amdhsa_inst_pref_size defined_value+6
   .amdhsa_exception_fp_ieee_invalid_op defined_boolean
   .amdhsa_exception_fp_denorm_src defined_boolean
   .amdhsa_exception_fp_ieee_div_zero defined_boolean
@@ -89,6 +90,7 @@ expr_defined:
   .amdhsa_workgroup_processor_mode defined_boolean
   .amdhsa_memory_ordered defined_boolean
   .amdhsa_forward_progress defined_boolean
+  .amdhsa_inst_pref_size defined_value+6
   .amdhsa_exception_fp_ieee_invalid_op defined_boolean
   .amdhsa_exception_fp_denorm_src defined_boolean
   .amdhsa_exception_fp_ieee_div_zero defined_boolean
@@ -132,6 +134,7 @@ expr_defined:
 // ASM-NEXT: .amdhsa_workgroup_processor_mode (((((((((((((((((((((1611399168|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~2097152))|(defined_boolean<<21))&(~63))|(((alignto(max(defined_value+4, 1), 8))/8)-1))&(~960))&536870912)>>29
 // ASM-NEXT: .amdhsa_memory_ordered (((((((((((((((((((((1611399168|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~2097152))|(defined_boolean<<21))&(~63))|(((alignto(max(defined_value+4, 1), 8))/8)-1))&(~960))&1073741824)>>30
 // ASM-NEXT: .amdhsa_forward_progress (((((((((((((((((((((1611399168|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~2097152))|(defined_boolean<<21))&(~63))|(((alignto(max(defined_value+4, 1), 8))/8)-1))&(~960))&2147483648)>>31
+// ASM-NEXT: .amdhsa_inst_pref_size (((defined_value+6)<<4)&4080)>>4
 // ASM-NEXT: .amdhsa_round_robin_scheduling (((((((((((((((((((((1611399168|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~2097152))|(defined_boolean<<21))&(~63))|(((alignto(max(defined_value+4, 1), 8))/8)-1))&(~960))&2097152)>>21
 // ASM-NEXT: .amdhsa_exception_fp_ieee_invalid_op (((((((((((((((((((((((((((128|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~1))|defined_boolean)&(~62))&16777216)>>24
 // ASM-NEXT: .amdhsa_exception_fp_denorm_src (((((((((((((((((((((((((((128|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~1))|defined_boolean)&(~62))&33554432)>>25
@@ -177,6 +180,7 @@ expr_defined:
 // ASM-NEXT: .amdhsa_workgroup_processor_mode 1
 // ASM-NEXT: .amdhsa_memory_ordered 1
 // ASM-NEXT: .amdhsa_forward_progress 1
+// ASM-NEXT: .amdhsa_inst_pref_size 47
 // ASM-NEXT: .amdhsa_round_robin_scheduling 1
 // ASM-NEXT: .amdhsa_exception_fp_ieee_invalid_op 1
 // ASM-NEXT: .amdhsa_exception_fp_denorm_src 1
diff --git a/llvm/test/MC/ARM/cortex-r52-nofp.s b/llvm/test/MC/ARM/cortex-r52-nofp.s
new file mode 100644
index 0000000000000..cc72cecd13115
--- /dev/null
+++ b/llvm/test/MC/ARM/cortex-r52-nofp.s
@@ -0,0 +1,9 @@
+@ RUN: llvm-mc -triple armv8-none-eabi -mcpu=cortex-r52 -mattr=+nosimd+nofp.dp %s -o - | FileCheck %s -check-prefix=CHECK-NO-FP
+@ RUN: llvm-mc -triple armv8-none-eabi -mcpu=cortex-r52 %s -o - | FileCheck %s -check-prefix=CHECK-FP
+
+.text
+vadd.f32 s0, s1, s2
+@ CHECK-NO-FP: vadd.f32 s0, s1, s2
+@ CHECK-FP: vadd.f32 s0, s1, s2
+@ CHECK-NOT-NO-FP: error: instruction requires: VPF2
+@ CHECK-NOT-FP: error: instruction requires: VPF2
diff --git a/llvm/test/MC/ARM/vcvt-fixed-point-errors.s b/llvm/test/MC/ARM/vcvt-fixed-point-errors.s
new file mode 100644
index 0000000000000..90e9da054a908
--- /dev/null
+++ b/llvm/test/MC/ARM/vcvt-fixed-point-errors.s
@@ -0,0 +1,51 @@
+// RUN: not llvm-mc -triple=armv8a-none-eabi -mattr=+fullfp16 < %s 2>&1 | FileCheck %s
+
+  vcvt.u16.f16 s0, s1, #1
+// CHECK: [[@LINE-1]]{{.*}}error: source and destination registers must be the same
+  vcvt.s16.f16 s0, s1, #1
+// CHECK: [[@LINE-1]]{{.*}}error: source and destination registers must be the same
+  vcvt.u32.f16 s0, s1, #1
+// CHECK: [[@LINE-1]]{{.*}}error: source and destination registers must be the same
+  vcvt.s32.f16 s0, s1, #1
+// CHECK: [[@LINE-1]]{{.*}}error: source and destination registers must be the same
+  vcvt.u16.f32 s0, s1, #1
+// CHECK: [[@LINE-1]]{{.*}}error: source and destination registers must be the same
+  vcvt.s16.f32 s0, s1, #1
+// CHECK: [[@LINE-1]]{{.*}}error: source and destination registers must be the same
+  vcvt.u32.f32 s0, s1, #1
+// CHECK: [[@LINE-1]]{{.*}}error: source and destination registers must be the same
+  vcvt.s32.f32 s0, s1, #1
+// CHECK: [[@LINE-1]]{{.*}}error: source and destination registers must be the same
+  vcvt.u16.f64 d0, d1, #1
+// CHECK: [[@LINE-1]]{{.*}}error: source and destination registers must be the same
+  vcvt.s16.f64 d0, d1, #1
+// CHECK: [[@LINE-1]]{{.*}}error: source and destination registers must be the same
+  vcvt.u32.f64 d0, d1, #1
+// CHECK: [[@LINE-1]]{{.*}}error: source and destination registers must be the same
+  vcvt.s32.f64 d0, d1, #1
+// CHECK: [[@LINE-1]]{{.*}}error: source and destination registers must be the same
+  vcvt.f16.u16 s0, s1, #1
+// CHECK: [[@LINE-1]]{{.*}}error: source and destination registers must be the same
+  vcvt.f16.s16 s0, s1, #1
+// CHECK: [[@LINE-1]]{{.*}}error: source and destination registers must be the same
+  vcvt.f16.u32 s0, s1, #1
+// CHECK: [[@LINE-1]]{{.*}}error: source and destination registers must be the same
+  vcvt.f16.s32 s0, s1, #1
+// CHECK: [[@LINE-1]]{{.*}}error: source and destination registers must be the same
+  vcvt.f32.u16 s0, s1, #1
+// CHECK: [[@LINE-1]]{{.*}}error: source and destination registers must be the same
+  vcvt.f32.s16 s0, s1, #1
+// CHECK: [[@LINE-1]]{{.*}}error: source and destination registers must be the same
+  vcvt.f32.u32 s0, s1, #1
+// CHECK: [[@LINE-1]]{{.*}}error: source and destination registers must be the same
+  vcvt.f32.s32 s0, s1, #1
+// CHECK: [[@LINE-1]]{{.*}}error: source and destination registers must be the same
+  vcvt.f64.u16 d0, d1, #1
+// CHECK: [[@LINE-1]]{{.*}}error: source and destination registers must be the same
+  vcvt.f64.s16 d0, d1, #1
+// CHECK: [[@LINE-1]]{{.*}}error: source and destination registers must be the same
+  vcvt.f64.u32 d0, d1, #1
+// CHECK: [[@LINE-1]]{{.*}}error: source and destination registers must be the same
+  vcvt.f64.s32 d0, d1, #1
+// CHECK: [[@LINE-1]]{{.*}}error: source and destination registers must be the same
+
diff --git a/llvm/test/MC/AsmParser/directive_seh.s b/llvm/test/MC/AsmParser/directive_seh.s
index f4cd29fb5e954..8eb50ed996dd3 100644
--- a/llvm/test/MC/AsmParser/directive_seh.s
+++ b/llvm/test/MC/AsmParser/directive_seh.s
@@ -49,9 +49,13 @@ func:
 # CHECK: .seh_startchained
 # CHECK: .seh_endprologue
 # CHECK: .seh_endchained
+    .seh_startepilogue
+# CHECK: .seh_startepilogue
     lea (%rbx), %rsp
     pop %rbx
     addq $24, %rsp
+    .seh_endepilogue
+# CHECK: .seh_endepilogue
     ret
     .seh_endproc
 # CHECK: .seh_endproc
diff --git a/llvm/test/MC/AsmParser/seh-directive-errors.s b/llvm/test/MC/AsmParser/seh-directive-errors.s
index 6d1486916caf3..c8a9d5d12c31d 100644
--- a/llvm/test/MC/AsmParser/seh-directive-errors.s
+++ b/llvm/test/MC/AsmParser/seh-directive-errors.s
@@ -11,6 +11,12 @@
 	.seh_stackalloc 32
 	# CHECK: :[[@LINE-1]]:{{[0-9]+}}: error: .seh_ directive must appear within an active frame
 
+	.seh_startepilogue
+	# CHECK: :[[@LINE-1]]:{{[0-9]+}}: error: .seh_ directive must appear within an active frame
+
+	.seh_endepilogue
+	# CHECK: :[[@LINE-1]]:{{[0-9]+}}: error: .seh_ directive must appear within an active frame
+
 	.def	 f;
 	.scl	2;
 	.type	32;
@@ -31,12 +37,18 @@ f:                                      # @f
 	.seh_stackalloc 7
 	# CHECK: :[[@LINE-1]]:{{[0-9]+}}: error: stack allocation size is not a multiple of 8
 	.seh_stackalloc 32
+	.seh_startepilogue
+	# CHECK: :[[@LINE-1]]:{{[0-9]+}}: error: starting epilogue (.seh_startepilogue) before prologue has ended (.seh_endprologue) in f
 	.seh_endprologue
 	nop
+	.seh_endepilogue
+	# CHECK: :[[@LINE-1]]:{{[0-9]+}}: error: Stray .seh_endepilogue in f
+	.seh_startepilogue
 	addq	$32, %rsp
 	popq	%rbx
 	popq	%rdi
 	popq	%rsi
+	.seh_endepilogue
 	retq
 	.seh_handlerdata
 	.text
@@ -63,8 +75,10 @@ g:
 	.seh_setframe 3, 128
 	# CHECK: :[[@LINE-1]]:{{[0-9]+}}: error: frame register and offset can be set at most once
 	nop
+	.seh_startepilogue
 	popq %rsi
 	popq %rbp
+	.seh_endepilogue
 	retq
 	.seh_endproc
 
@@ -90,9 +104,11 @@ h:                                      # @h
         addsd   %xmm6, %xmm7
         callq   getdbl
         addsd   %xmm7, %xmm0
+		.seh_startepilogue
         movaps  32(%rsp), %xmm6         # 16-byte Reload
         movaps  48(%rsp), %xmm7         # 16-byte Reload
         addq    $72, %rsp
+		.seh_endepilogue
         retq
         .seh_handlerdata
         .text
diff --git a/llvm/test/MC/COFF/cv-def-range-align.s b/llvm/test/MC/COFF/cv-def-range-align.s
index 2b6c18ad346b0..358383ac8f327 100644
--- a/llvm/test/MC/COFF/cv-def-range-align.s
+++ b/llvm/test/MC/COFF/cv-def-range-align.s
@@ -48,7 +48,9 @@ max:                                    # @max
 .LBB0_2:                                # %cond.false
 	movl	4(%rsp), %eax
 .LBB0_3:                                # %cond.end
+	.seh_startepilogue
 	popq	%rcx
+	.seh_endepilogue
 	retq
 .Ltmp1:
 .Lfunc_end0:
diff --git a/llvm/test/MC/COFF/cv-inline-linetable-unlikely.s b/llvm/test/MC/COFF/cv-inline-linetable-unlikely.s
index 862615f48fdc6..7d2f6d92f1175 100644
--- a/llvm/test/MC/COFF/cv-inline-linetable-unlikely.s
+++ b/llvm/test/MC/COFF/cv-inline-linetable-unlikely.s
@@ -62,8 +62,10 @@ g:                                      # @g
 	jne	.LBB0_1
 	.cv_loc	0 1 10 17               # t.cpp:10:17
 	movl	$0, unlikely_cond(%rip)
+	.seh_startepilogue
 	.cv_loc	0 1 11 1                # t.cpp:11:1
 	addq	$40, %rsp
+	.seh_endepilogue
 	retq
 
 .LBB0_1:                                # %if.then.i
diff --git a/llvm/test/MC/COFF/seh-align2.s b/llvm/test/MC/COFF/seh-align2.s
index 7981c18c2c251..cf46cf6b00841 100644
--- a/llvm/test/MC/COFF/seh-align2.s
+++ b/llvm/test/MC/COFF/seh-align2.s
@@ -73,6 +73,8 @@
     .text
     .seh_endprologue
 func:
+    .seh_startepilogue
     addq $24, %rsp
+    .seh_endepilogue
     ret
     .seh_endproc
diff --git a/llvm/test/MC/COFF/seh-align3.s b/llvm/test/MC/COFF/seh-align3.s
index 9b3e39d08a196..9967e254b7eaf 100644
--- a/llvm/test/MC/COFF/seh-align3.s
+++ b/llvm/test/MC/COFF/seh-align3.s
@@ -77,7 +77,9 @@
     .text
     .seh_endprologue
 func:
+    .seh_startepilogue
     pop %r13
     pop %r12
+    .seh_endepilogue
     ret
     .seh_endproc
diff --git a/llvm/test/MC/COFF/seh-linkonce.s b/llvm/test/MC/COFF/seh-linkonce.s
index 52feca020386e..0c5d8ecadb72a 100644
--- a/llvm/test/MC/COFF/seh-linkonce.s
+++ b/llvm/test/MC/COFF/seh-linkonce.s
@@ -21,7 +21,9 @@ weak_func:                              # @weak_func
 .Ltmp3:
         .seh_endprologue
         xorl    %eax, %eax
+        .seh_startepilogue
         popq    %rbp
+        .seh_endepilogue
         retq
 .Leh_func_end0:
 .Ltmp4:
diff --git a/llvm/test/MC/COFF/seh-section-2.s b/llvm/test/MC/COFF/seh-section-2.s
index 390a20b3e4f9c..e34cc3ba67439 100644
--- a/llvm/test/MC/COFF/seh-section-2.s
+++ b/llvm/test/MC/COFF/seh-section-2.s
@@ -21,7 +21,9 @@ f:                                      # @f
         .seh_endprologue
         callq   g
         nop
+        .seh_startepilogue
         addq    $40, %rsp
+        .seh_endepilogue
         retq
         .seh_handlerdata
         .section        .text,"xr",discard,f
diff --git a/llvm/test/MC/COFF/seh-section.s b/llvm/test/MC/COFF/seh-section.s
index 7bdc11e7723a2..17c609a29ea79 100644
--- a/llvm/test/MC/COFF/seh-section.s
+++ b/llvm/test/MC/COFF/seh-section.s
@@ -68,8 +68,10 @@ foo:
     pushq %rbp
     .seh_pushreg %rbp
     .seh_endprologue
+    .seh_startepilogue
     popq %rbp
     addq $8, %rsp
+    .seh_endepilogue
     ret
     .seh_endproc
 
@@ -83,8 +85,10 @@ bar:
     pushq %rbp
     .seh_pushreg %rbp
     .seh_endprologue
+    .seh_startepilogue
     popq %rbp
     addq $8, %rsp
+    .seh_endepilogue
     ret
     .seh_endproc
 
@@ -98,8 +102,10 @@ baz:
     pushq %rbp
     .seh_pushreg %rbp
     .seh_endprologue
+    .seh_startepilogue
     popq %rbp
     addq $8, %rsp
+    .seh_endepilogue
     ret
     .seh_endproc
 
diff --git a/llvm/test/MC/COFF/seh.s b/llvm/test/MC/COFF/seh.s
index 201cdb1d07d1c..31e845397f2d0 100644
--- a/llvm/test/MC/COFF/seh.s
+++ b/llvm/test/MC/COFF/seh.s
@@ -144,9 +144,11 @@ func:
     .seh_startchained
     .seh_endprologue
     .seh_endchained
+    .seh_startepilogue
     lea (%rbx), %rsp
     pop %rbx
     addq $24, %rsp
+    .seh_endepilogue
     ret
     .seh_endproc
 
diff --git a/llvm/test/MC/Disassembler/AMDGPU/gfx11_dasm_vop3.txt b/llvm/test/MC/Disassembler/AMDGPU/gfx11_dasm_vop3.txt
index 05174e3128919..b74128b21f563 100644
--- a/llvm/test/MC/Disassembler/AMDGPU/gfx11_dasm_vop3.txt
+++ b/llvm/test/MC/Disassembler/AMDGPU/gfx11_dasm_vop3.txt
@@ -508,10 +508,16 @@
 # GFX11: v_alignbyte_b32 v5, s105, s105, s105    ; encoding: [0x05,0x00,0x17,0xd6,0x69,0xd2,0xa4,0x01]
 
 0x05,0x00,0x17,0xd6,0x6a,0xf6,0x0c,0x04
-# GFX11: v_alignbyte_b32 v5, vcc_lo, ttmp15, v3  ; encoding: [0x05,0x00,0x17,0xd6,0x6a,0xf6,0x0c,0x04]
+# W32-REAL16: v_alignbyte_b32 v5, vcc_lo, ttmp15, v3.l ; encoding: [0x05,0x00,0x17,0xd6,0x6a,0xf6,0x0c,0x04]
+# W32-FAKE16: v_alignbyte_b32 v5, vcc_lo, ttmp15, v3  ; encoding: [0x05,0x00,0x17,0xd6,0x6a,0xf6,0x0c,0x04]
+# W64-REAL16: v_alignbyte_b32 v5, vcc_lo, ttmp15, v3.l ; encoding: [0x05,0x00,0x17,0xd6,0x6a,0xf6,0x0c,0x04]
+# W64-FAKE16: v_alignbyte_b32 v5, vcc_lo, ttmp15, v3  ; encoding: [0x05,0x00,0x17,0xd6,0x6a,0xf6,0x0c,0x04]
 
 0x05,0x00,0x17,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf
-# GFX11: v_alignbyte_b32 v5, vcc_hi, 0xaf123456, v255 ; encoding: [0x05,0x00,0x17,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf]
+# W32-REAL16: v_alignbyte_b32 v5, vcc_hi, 0xaf123456, v255.l ; encoding: [0x05,0x00,0x17,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf]
+# W32-FAKE16: v_alignbyte_b32 v5, vcc_hi, 0xaf123456, v255 ; encoding: [0x05,0x00,0x17,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf]
+# W64-REAL16: v_alignbyte_b32 v5, vcc_hi, 0xaf123456, v255.l ; encoding: [0x05,0x00,0x17,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf]
+# W64-FAKE16: v_alignbyte_b32 v5, vcc_hi, 0xaf123456, v255 ; encoding: [0x05,0x00,0x17,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf]
 
 0x05,0x00,0x17,0xd6,0x7b,0xfa,0xed,0x01
 # GFX11: v_alignbyte_b32 v5, ttmp15, src_scc, ttmp15 ; encoding: [0x05,0x00,0x17,0xd6,0x7b,0xfa,0xed,0x01]
@@ -540,6 +546,12 @@
 0xff,0x00,0x17,0xd6,0xff,0xd6,0xf0,0x01,0x56,0x34,0x12,0xaf
 # GFX11: v_alignbyte_b32 v255, 0xaf123456, vcc_hi, null ; encoding: [0xff,0x00,0x17,0xd6,0xff,0xd6,0xf0,0x01,0x56,0x34,0x12,0xaf]
 
+0x05,0x20,0x17,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf
+# W32-REAL16: v_alignbyte_b32 v5, vcc_hi, 0xaf123456, v255.h op_sel:[0,0,1,0] ; encoding: [0x05,0x20,0x17,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf]
+# W32-FAKE16: v_alignbyte_b32 v5, vcc_hi, 0xaf123456, v255 op_sel:[0,0,1,0] ; encoding: [0x05,0x20,0x17,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf]
+# W64-REAL16: v_alignbyte_b32 v5, vcc_hi, 0xaf123456, v255.h op_sel:[0,0,1,0] ; encoding: [0x05,0x20,0x17,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf]
+# W64-FAKE16: v_alignbyte_b32 v5, vcc_hi, 0xaf123456, v255 op_sel:[0,0,1,0] ; encoding: [0x05,0x20,0x17,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf]
+
 0x05,0x00,0x62,0xd7,0x01,0x05,0x02,0x00
 # W32-REAL16: v_and_b16 v5.l, v1.l, v2.l              ; encoding: [0x05,0x00,0x62,0xd7,0x01,0x05,0x02,0x00]
 # W32-FAKE16: v_and_b16 v5, v1, v2                    ; encoding: [0x05,0x00,0x62,0xd7,0x01,0x05,0x02,0x00]
@@ -3258,10 +3270,16 @@
 # W64-FAKE16: v_mad_i16 v255, 0xfe0b, vcc_hi, null op_sel:[0,0,0,1] clamp ; encoding: [0xff,0xc0,0x53,0xd6,0xff,0xd6,0xf0,0x01,0x0b,0xfe,0x00,0x00]
 
 0x05,0x00,0x5a,0xd6,0x01,0x05,0x0e,0x04
-# GFX11: v_mad_i32_i16 v5, v1, v2, v3            ; encoding: [0x05,0x00,0x5a,0xd6,0x01,0x05,0x0e,0x04]
+# W32-REAL16: v_mad_i32_i16 v5, v1.l, v2.l, v3        ; encoding: [0x05,0x00,0x5a,0xd6,0x01,0x05,0x0e,0x04]
+# W32-FAKE16: v_mad_i32_i16 v5, v1, v2, v3            ; encoding: [0x05,0x00,0x5a,0xd6,0x01,0x05,0x0e,0x04]
+# W64-REAL16: v_mad_i32_i16 v5, v1.l, v2.l, v3        ; encoding: [0x05,0x00,0x5a,0xd6,0x01,0x05,0x0e,0x04]
+# W64-FAKE16: v_mad_i32_i16 v5, v1, v2, v3            ; encoding: [0x05,0x00,0x5a,0xd6,0x01,0x05,0x0e,0x04]
 
 0x05,0x00,0x5a,0xd6,0xff,0xff,0x0f,0x00
-# GFX11: v_mad_i32_i16 v5, v255, v255, s3        ; encoding: [0x05,0x00,0x5a,0xd6,0xff,0xff,0x0f,0x00]
+# W32-REAL16: v_mad_i32_i16 v5, v255.l, v255.l, s3    ; encoding: [0x05,0x00,0x5a,0xd6,0xff,0xff,0x0f,0x00]
+# W32-FAKE16: v_mad_i32_i16 v5, v255, v255, s3        ; encoding: [0x05,0x00,0x5a,0xd6,0xff,0xff,0x0f,0x00]
+# W64-REAL16: v_mad_i32_i16 v5, v255.l, v255.l, s3    ; encoding: [0x05,0x00,0x5a,0xd6,0xff,0xff,0x0f,0x00]
+# W64-FAKE16: v_mad_i32_i16 v5, v255, v255, s3        ; encoding: [0x05,0x00,0x5a,0xd6,0xff,0xff,0x0f,0x00]
 
 0x05,0x00,0x5a,0xd6,0x01,0x04,0xfc,0x07
 # GFX11: v_mad_i32_i16 v5, s1, s2, v255          ; encoding: [0x05,0x00,0x5a,0xd6,0x01,0x04,0xfc,0x07]
@@ -3302,6 +3320,24 @@
 0xff,0x90,0x5a,0xd6,0xff,0xd6,0xc0,0x03,0x0b,0xfe,0x00,0x00
 # GFX11: v_mad_i32_i16 v255, 0xfe0b, vcc_hi, 0.5 op_sel:[0,1,0,0] clamp ; encoding: [0xff,0x90,0x5a,0xd6,0xff,0xd6,0xc0,0x03,0x0b,0xfe,0x00,0x00]
 
+0x05,0x08,0x5a,0xd6,0x01,0x05,0x0e,0x04
+# W32-REAL16: v_mad_i32_i16 v5, v1.h, v2.l, v3 op_sel:[1,0,0,0] ; encoding: [0x05,0x08,0x5a,0xd6,0x01,0x05,0x0e,0x04]
+# W32-FAKE16: v_mad_i32_i16 v5, v1, v2, v3 op_sel:[1,0,0,0] ; encoding: [0x05,0x08,0x5a,0xd6,0x01,0x05,0x0e,0x04]
+# W64-REAL16: v_mad_i32_i16 v5, v1.h, v2.l, v3 op_sel:[1,0,0,0] ; encoding: [0x05,0x08,0x5a,0xd6,0x01,0x05,0x0e,0x04]
+# W64-FAKE16: v_mad_i32_i16 v5, v1, v2, v3 op_sel:[1,0,0,0] ; encoding: [0x05,0x08,0x5a,0xd6,0x01,0x05,0x0e,0x04]
+
+0x05,0x10,0x5a,0xd6,0xff,0xff,0x0f,0x00
+# W32-REAL16: v_mad_i32_i16 v5, v255.l, v255.h, s3 op_sel:[0,1,0,0] ; encoding: [0x05,0x10,0x5a,0xd6,0xff,0xff,0x0f,0x00]
+# W32-FAKE16: v_mad_i32_i16 v5, v255, v255, s3 op_sel:[0,1,0,0] ; encoding: [0x05,0x10,0x5a,0xd6,0xff,0xff,0x0f,0x00]
+# W64-REAL16: v_mad_i32_i16 v5, v255.l, v255.h, s3 op_sel:[0,1,0,0] ; encoding: [0x05,0x10,0x5a,0xd6,0xff,0xff,0x0f,0x00]
+# W64-FAKE16: v_mad_i32_i16 v5, v255, v255, s3 op_sel:[0,1,0,0] ; encoding: [0x05,0x10,0x5a,0xd6,0xff,0xff,0x0f,0x00]
+
+0x05,0x00,0x5a,0xd6,0xfd,0xd4,0xf4,0x03
+# GFX11: v_mad_i32_i16 v5, src_scc, vcc_lo, src_scc ; encoding: [0x05,0x00,0x5a,0xd6,0xfd,0xd4,0xf4,0x03]
+
+0xff,0x80,0x5a,0xd6,0xff,0xd6,0xc0,0x03,0x0b,0xfe,0x00,0x00
+# GFX11: v_mad_i32_i16 v255, 0xfe0b, vcc_hi, 0.5 clamp ; encoding: [0xff,0x80,0x5a,0xd6,0xff,0xd6,0xc0,0x03,0x0b,0xfe,0x00,0x00]
+
 0x05,0x00,0x0a,0xd6,0x01,0x05,0x0e,0x00
 # GFX11: v_mad_i32_i24 v5, v1, v2, s3            ; encoding: [0x05,0x00,0x0a,0xd6,0x01,0x05,0x0e,0x00]
 
@@ -3521,10 +3557,16 @@
 # W64-FAKE16: v_mad_u16 v255, 0xfe0b, vcc_hi, null op_sel:[0,0,0,1] clamp ; encoding: [0xff,0xc0,0x41,0xd6,0xff,0xd6,0xf0,0x01,0x0b,0xfe,0x00,0x00]
 
 0x05,0x00,0x59,0xd6,0x01,0x05,0x0e,0x04
-# GFX11: v_mad_u32_u16 v5, v1, v2, v3            ; encoding: [0x05,0x00,0x59,0xd6,0x01,0x05,0x0e,0x04]
+# W32-REAL16: v_mad_u32_u16 v5, v1.l, v2.l, v3        ; encoding: [0x05,0x00,0x59,0xd6,0x01,0x05,0x0e,0x04]
+# W32-FAKE16: v_mad_u32_u16 v5, v1, v2, v3            ; encoding: [0x05,0x00,0x59,0xd6,0x01,0x05,0x0e,0x04]
+# W64-REAL16: v_mad_u32_u16 v5, v1.l, v2.l, v3        ; encoding: [0x05,0x00,0x59,0xd6,0x01,0x05,0x0e,0x04]
+# W64-FAKE16: v_mad_u32_u16 v5, v1, v2, v3            ; encoding: [0x05,0x00,0x59,0xd6,0x01,0x05,0x0e,0x04]
 
 0x05,0x00,0x59,0xd6,0xff,0xff,0x0f,0x00
-# GFX11: v_mad_u32_u16 v5, v255, v255, s3        ; encoding: [0x05,0x00,0x59,0xd6,0xff,0xff,0x0f,0x00]
+# W32-REAL16: v_mad_u32_u16 v5, v255.l, v255.l, s3    ; encoding: [0x05,0x00,0x59,0xd6,0xff,0xff,0x0f,0x00]
+# W32-FAKE16: v_mad_u32_u16 v5, v255, v255, s3        ; encoding: [0x05,0x00,0x59,0xd6,0xff,0xff,0x0f,0x00]
+# W64-REAL16: v_mad_u32_u16 v5, v255.l, v255.l, s3    ; encoding: [0x05,0x00,0x59,0xd6,0xff,0xff,0x0f,0x00]
+# W64-FAKE16: v_mad_u32_u16 v5, v255, v255, s3        ; encoding: [0x05,0x00,0x59,0xd6,0xff,0xff,0x0f,0x00]
 
 0x05,0x00,0x59,0xd6,0x01,0x04,0xfc,0x07
 # GFX11: v_mad_u32_u16 v5, s1, s2, v255          ; encoding: [0x05,0x00,0x59,0xd6,0x01,0x04,0xfc,0x07]
@@ -3565,6 +3607,24 @@
 0xff,0x90,0x59,0xd6,0xff,0xd6,0xc0,0x03,0x0b,0xfe,0x00,0x00
 # GFX11: v_mad_u32_u16 v255, 0xfe0b, vcc_hi, 0.5 op_sel:[0,1,0,0] clamp ; encoding: [0xff,0x90,0x59,0xd6,0xff,0xd6,0xc0,0x03,0x0b,0xfe,0x00,0x00]
 
+0x05,0x08,0x59,0xd6,0x01,0x05,0x0e,0x04
+# W32-REAL16: v_mad_u32_u16 v5, v1.h, v2.l, v3 op_sel:[1,0,0,0] ; encoding: [0x05,0x08,0x59,0xd6,0x01,0x05,0x0e,0x04]
+# W32-FAKE16: v_mad_u32_u16 v5, v1, v2, v3 op_sel:[1,0,0,0] ; encoding: [0x05,0x08,0x59,0xd6,0x01,0x05,0x0e,0x04]
+# W64-REAL16: v_mad_u32_u16 v5, v1.h, v2.l, v3 op_sel:[1,0,0,0] ; encoding: [0x05,0x08,0x59,0xd6,0x01,0x05,0x0e,0x04]
+# W64-FAKE16: v_mad_u32_u16 v5, v1, v2, v3 op_sel:[1,0,0,0] ; encoding: [0x05,0x08,0x59,0xd6,0x01,0x05,0x0e,0x04]
+
+0x05,0x10,0x59,0xd6,0xff,0xff,0x0f,0x00
+# W32-REAL16: v_mad_u32_u16 v5, v255.l, v255.h, s3 op_sel:[0,1,0,0] ; encoding: [0x05,0x10,0x59,0xd6,0xff,0xff,0x0f,0x00]
+# W32-FAKE16: v_mad_u32_u16 v5, v255, v255, s3 op_sel:[0,1,0,0] ; encoding: [0x05,0x10,0x59,0xd6,0xff,0xff,0x0f,0x00]
+# W64-REAL16: v_mad_u32_u16 v5, v255.l, v255.h, s3 op_sel:[0,1,0,0] ; encoding: [0x05,0x10,0x59,0xd6,0xff,0xff,0x0f,0x00]
+# W64-FAKE16: v_mad_u32_u16 v5, v255, v255, s3 op_sel:[0,1,0,0] ; encoding: [0x05,0x10,0x59,0xd6,0xff,0xff,0x0f,0x00]
+
+0x05,0x00,0x59,0xd6,0xfd,0xd4,0xf4,0x03
+# GFX11: v_mad_u32_u16 v5, src_scc, vcc_lo, src_scc ; encoding: [0x05,0x00,0x59,0xd6,0xfd,0xd4,0xf4,0x03]
+
+0xff,0x80,0x59,0xd6,0xff,0xd6,0xc0,0x03,0x0b,0xfe,0x00,0x00
+# GFX11: v_mad_u32_u16 v255, 0xfe0b, vcc_hi, 0.5 clamp ; encoding: [0xff,0x80,0x59,0xd6,0xff,0xd6,0xc0,0x03,0x0b,0xfe,0x00,0x00]
+
 0x05,0x00,0x0b,0xd6,0x01,0x05,0x0e,0x00
 # GFX11: v_mad_u32_u24 v5, v1, v2, s3            ; encoding: [0x05,0x00,0x0b,0xd6,0x01,0x05,0x0e,0x00]
 
diff --git a/llvm/test/MC/Disassembler/AMDGPU/gfx11_dasm_vop3_dpp16.txt b/llvm/test/MC/Disassembler/AMDGPU/gfx11_dasm_vop3_dpp16.txt
index c9ef3c714213d..8e7122b902326 100644
--- a/llvm/test/MC/Disassembler/AMDGPU/gfx11_dasm_vop3_dpp16.txt
+++ b/llvm/test/MC/Disassembler/AMDGPU/gfx11_dasm_vop3_dpp16.txt
@@ -228,19 +228,34 @@
 # GFX11: v_alignbit_b32_e64_dpp v255, v255, v255, src_scc row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x00,0x16,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30]
 
 0x05,0x00,0x17,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff
-# GFX11: v_alignbyte_b32_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x17,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
+# W32-REAL16: v_alignbyte_b32_e64_dpp v5, v1, v2, v3.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x17,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
+# W32-FAKE16: v_alignbyte_b32_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x17,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
+# W64-REAL16: v_alignbyte_b32_e64_dpp v5, v1, v2, v3.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x17,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
+# W64-FAKE16: v_alignbyte_b32_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x17,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 
 0x05,0x00,0x17,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff
-# GFX11: v_alignbyte_b32_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x17,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
+# W32-REAL16: v_alignbyte_b32_e64_dpp v5, v1, v2, v3.l quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x17,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
+# W32-FAKE16: v_alignbyte_b32_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x17,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
+# W64-REAL16: v_alignbyte_b32_e64_dpp v5, v1, v2, v3.l quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x17,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
+# W64-FAKE16: v_alignbyte_b32_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x17,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
 
 0x05,0x00,0x17,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff
-# GFX11: v_alignbyte_b32_e64_dpp v5, v1, v2, v3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x17,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
+# W32-REAL16: v_alignbyte_b32_e64_dpp v5, v1, v2, v3.l row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x17,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
+# W32-FAKE16: v_alignbyte_b32_e64_dpp v5, v1, v2, v3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x17,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
+# W64-REAL16: v_alignbyte_b32_e64_dpp v5, v1, v2, v3.l row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x17,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
+# W64-FAKE16: v_alignbyte_b32_e64_dpp v5, v1, v2, v3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x17,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
 
 0x05,0x00,0x17,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x41,0x01,0xff
-# GFX11: v_alignbyte_b32_e64_dpp v5, v1, v2, v3 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x17,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x41,0x01,0xff]
+# W32-REAL16: v_alignbyte_b32_e64_dpp v5, v1, v2, v3.l row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x17,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x41,0x01,0xff]
+# W32-FAKE16: v_alignbyte_b32_e64_dpp v5, v1, v2, v3 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x17,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x41,0x01,0xff]
+# W64-REAL16: v_alignbyte_b32_e64_dpp v5, v1, v2, v3.l row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x17,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x41,0x01,0xff]
+# W64-FAKE16: v_alignbyte_b32_e64_dpp v5, v1, v2, v3 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x17,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x41,0x01,0xff]
 
 0x05,0x00,0x17,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x01,0x01,0xff
-# GFX11: v_alignbyte_b32_e64_dpp v5, v1, v2, v255 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x17,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x01,0x01,0xff]
+# W32-REAL16: v_alignbyte_b32_e64_dpp v5, v1, v2, v255.l row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x17,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x01,0x01,0xff]
+# W32-FAKE16: v_alignbyte_b32_e64_dpp v5, v1, v2, v255 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x17,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x01,0x01,0xff]
+# W64-REAL16: v_alignbyte_b32_e64_dpp v5, v1, v2, v255.l row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x17,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x01,0x01,0xff]
+# W64-FAKE16: v_alignbyte_b32_e64_dpp v5, v1, v2, v255 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x17,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x01,0x01,0xff]
 
 0x05,0x00,0x17,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x0f,0x01,0xff
 # GFX11: v_alignbyte_b32_e64_dpp v5, v1, v2, s105 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x17,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x0f,0x01,0xff]
@@ -269,6 +284,12 @@
 0xff,0x00,0x17,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30
 # GFX11: v_alignbyte_b32_e64_dpp v255, v255, v255, src_scc row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x00,0x17,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30]
 
+0x05,0x20,0x17,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x40,0x01,0xff
+# W32-REAL16: v_alignbyte_b32_e64_dpp v5, v1, v2, v255.h op_sel:[0,0,1,0] row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x20,0x17,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x40,0x01,0xff]
+# W32-FAKE16: v_alignbyte_b32_e64_dpp v5, v1, v2, v255 op_sel:[0,0,1,0] row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x20,0x17,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x40,0x01,0xff]
+# W64-REAL16: v_alignbyte_b32_e64_dpp v5, v1, v2, v255.h op_sel:[0,0,1,0] row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x20,0x17,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x40,0x01,0xff]
+# W64-FAKE16: v_alignbyte_b32_e64_dpp v5, v1, v2, v255 op_sel:[0,0,1,0] row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x20,0x17,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x40,0x01,0xff]
+
 0x05,0x00,0x62,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff
 # W32-REAL16: v_and_b16_e64_dpp v5.l, v1.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x62,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
 # W32-FAKE16: v_and_b16_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x62,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
@@ -4868,46 +4889,88 @@
 # W64-FAKE16: v_mad_i16_e64_dpp v255, v255, v255, src_scc op_sel:[0,0,0,1] clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0xc0,0x53,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30]
 
 0x05,0x00,0x5a,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff
-# GFX11: v_mad_i32_i16_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5a,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
+# W32-REAL16: v_mad_i32_i16_e64_dpp v5, v1.l, v2.l, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5a,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
+# W32-FAKE16: v_mad_i32_i16_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5a,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
+# W64-REAL16: v_mad_i32_i16_e64_dpp v5, v1.l, v2.l, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5a,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
+# W64-FAKE16: v_mad_i32_i16_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5a,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 
 0x05,0x00,0x5a,0xd6,0xfa,0x04,0xfe,0x07,0x01,0xe4,0x00,0xff
-# GFX11: v_mad_i32_i16_e64_dpp v5, v1, v2, v255 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5a,0xd6,0xfa,0x04,0xfe,0x07,0x01,0xe4,0x00,0xff]
+# W32-REAL16: v_mad_i32_i16_e64_dpp v5, v1.l, v2.l, v255 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5a,0xd6,0xfa,0x04,0xfe,0x07,0x01,0xe4,0x00,0xff]
+# W32-FAKE16: v_mad_i32_i16_e64_dpp v5, v1, v2, v255 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5a,0xd6,0xfa,0x04,0xfe,0x07,0x01,0xe4,0x00,0xff]
+# W64-REAL16: v_mad_i32_i16_e64_dpp v5, v1.l, v2.l, v255 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5a,0xd6,0xfa,0x04,0xfe,0x07,0x01,0xe4,0x00,0xff]
+# W64-FAKE16: v_mad_i32_i16_e64_dpp v5, v1, v2, v255 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5a,0xd6,0xfa,0x04,0xfe,0x07,0x01,0xe4,0x00,0xff]
 
 0x05,0x00,0x5a,0xd6,0xfa,0x04,0x0e,0x00,0x01,0x40,0x01,0xff
-# GFX11: v_mad_i32_i16_e64_dpp v5, v1, v2, s3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5a,0xd6,0xfa,0x04,0x0e,0x00,0x01,0x40,0x01,0xff]
+# W32-REAL16: v_mad_i32_i16_e64_dpp v5, v1.l, v2.l, s3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5a,0xd6,0xfa,0x04,0x0e,0x00,0x01,0x40,0x01,0xff]
+# W32-FAKE16: v_mad_i32_i16_e64_dpp v5, v1, v2, s3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5a,0xd6,0xfa,0x04,0x0e,0x00,0x01,0x40,0x01,0xff]
+# W64-REAL16: v_mad_i32_i16_e64_dpp v5, v1.l, v2.l, s3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5a,0xd6,0xfa,0x04,0x0e,0x00,0x01,0x40,0x01,0xff]
+# W64-FAKE16: v_mad_i32_i16_e64_dpp v5, v1, v2, s3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5a,0xd6,0xfa,0x04,0x0e,0x00,0x01,0x40,0x01,0xff]
 
 0x05,0x00,0x5a,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x41,0x01,0xff
-# GFX11: v_mad_i32_i16_e64_dpp v5, v1, v2, s105 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5a,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x41,0x01,0xff]
+# W32-REAL16: v_mad_i32_i16_e64_dpp v5, v1.l, v2.l, s105 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5a,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x41,0x01,0xff]
+# W32-FAKE16: v_mad_i32_i16_e64_dpp v5, v1, v2, s105 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5a,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x41,0x01,0xff]
+# W64-REAL16: v_mad_i32_i16_e64_dpp v5, v1.l, v2.l, s105 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5a,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x41,0x01,0xff]
+# W64-FAKE16: v_mad_i32_i16_e64_dpp v5, v1, v2, s105 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5a,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x41,0x01,0xff]
 
 0x05,0x00,0x5a,0xd6,0xfa,0x04,0xee,0x01,0x01,0x01,0x01,0xff
-# GFX11: v_mad_i32_i16_e64_dpp v5, v1, v2, ttmp15 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5a,0xd6,0xfa,0x04,0xee,0x01,0x01,0x01,0x01,0xff]
+# W32-REAL16: v_mad_i32_i16_e64_dpp v5, v1.l, v2.l, ttmp15 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5a,0xd6,0xfa,0x04,0xee,0x01,0x01,0x01,0x01,0xff]
+# W32-FAKE16: v_mad_i32_i16_e64_dpp v5, v1, v2, ttmp15 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5a,0xd6,0xfa,0x04,0xee,0x01,0x01,0x01,0x01,0xff]
+# W64-REAL16: v_mad_i32_i16_e64_dpp v5, v1.l, v2.l, ttmp15 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5a,0xd6,0xfa,0x04,0xee,0x01,0x01,0x01,0x01,0xff]
+# W64-FAKE16: v_mad_i32_i16_e64_dpp v5, v1, v2, ttmp15 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5a,0xd6,0xfa,0x04,0xee,0x01,0x01,0x01,0x01,0xff]
 
 0x05,0x00,0x5a,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff
-# GFX11: v_mad_i32_i16_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5a,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
+# W32-REAL16: v_mad_i32_i16_e64_dpp v5, v1.l, v2.l, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5a,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
+# W32-FAKE16: v_mad_i32_i16_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5a,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
+# W64-REAL16: v_mad_i32_i16_e64_dpp v5, v1.l, v2.l, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5a,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
+# W64-FAKE16: v_mad_i32_i16_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5a,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
 
 0x05,0x00,0x5a,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff
-# GFX11: v_mad_i32_i16_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5a,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
+# W32-REAL16: v_mad_i32_i16_e64_dpp v5, v1.l, v2.l, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5a,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
+# W32-FAKE16: v_mad_i32_i16_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5a,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
+# W64-REAL16: v_mad_i32_i16_e64_dpp v5, v1.l, v2.l, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5a,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
+# W64-FAKE16: v_mad_i32_i16_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5a,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
 
 0x05,0x00,0x5a,0xd6,0xfa,0x04,0xf6,0x01,0x01,0x1f,0x01,0xff
-# GFX11: v_mad_i32_i16_e64_dpp v5, v1, v2, m0 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5a,0xd6,0xfa,0x04,0xf6,0x01,0x01,0x1f,0x01,0xff]
+# W32-REAL16: v_mad_i32_i16_e64_dpp v5, v1.l, v2.l, m0 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5a,0xd6,0xfa,0x04,0xf6,0x01,0x01,0x1f,0x01,0xff]
+# W32-FAKE16: v_mad_i32_i16_e64_dpp v5, v1, v2, m0 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5a,0xd6,0xfa,0x04,0xf6,0x01,0x01,0x1f,0x01,0xff]
+# W64-REAL16: v_mad_i32_i16_e64_dpp v5, v1.l, v2.l, m0 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5a,0xd6,0xfa,0x04,0xf6,0x01,0x01,0x1f,0x01,0xff]
+# W64-FAKE16: v_mad_i32_i16_e64_dpp v5, v1, v2, m0 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5a,0xd6,0xfa,0x04,0xf6,0x01,0x01,0x1f,0x01,0xff]
 
 0x05,0x00,0x5a,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x21,0x01,0xff
-# GFX11: v_mad_i32_i16_e64_dpp v5, v1, v2, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5a,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x21,0x01,0xff]
+# W32-REAL16: v_mad_i32_i16_e64_dpp v5, v1.l, v2.l, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5a,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x21,0x01,0xff]
+# W32-FAKE16: v_mad_i32_i16_e64_dpp v5, v1, v2, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5a,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x21,0x01,0xff]
+# W64-REAL16: v_mad_i32_i16_e64_dpp v5, v1.l, v2.l, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5a,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x21,0x01,0xff]
+# W64-FAKE16: v_mad_i32_i16_e64_dpp v5, v1, v2, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5a,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x21,0x01,0xff]
 
 0x05,0x00,0x5a,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x2f,0x01,0xff
-# GFX11: v_mad_i32_i16_e64_dpp v5, v1, v2, exec_lo row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5a,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x2f,0x01,0xff]
+# W32-REAL16: v_mad_i32_i16_e64_dpp v5, v1.l, v2.l, exec_lo row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5a,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x2f,0x01,0xff]
+# W32-FAKE16: v_mad_i32_i16_e64_dpp v5, v1, v2, exec_lo row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5a,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x2f,0x01,0xff]
+# W64-REAL16: v_mad_i32_i16_e64_dpp v5, v1.l, v2.l, exec_lo row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5a,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x2f,0x01,0xff]
+# W64-FAKE16: v_mad_i32_i16_e64_dpp v5, v1, v2, exec_lo row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5a,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x2f,0x01,0xff]
 
 0x05,0x00,0x5a,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x50,0x01,0xff
-# GFX11: v_mad_i32_i16_e64_dpp v5, v1, v2, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5a,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x50,0x01,0xff]
+# W32-REAL16: v_mad_i32_i16_e64_dpp v5, v1.l, v2.l, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5a,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x50,0x01,0xff]
+# W32-FAKE16: v_mad_i32_i16_e64_dpp v5, v1, v2, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5a,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x50,0x01,0xff]
+# W64-REAL16: v_mad_i32_i16_e64_dpp v5, v1.l, v2.l, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5a,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x50,0x01,0xff]
+# W64-FAKE16: v_mad_i32_i16_e64_dpp v5, v1, v2, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5a,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x50,0x01,0xff]
 
 0x05,0x00,0x5a,0xd6,0xfa,0x04,0x06,0x03,0x01,0x5f,0x01,0x01
-# GFX11: v_mad_i32_i16_e64_dpp v5, v1, v2, -1 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x5a,0xd6,0xfa,0x04,0x06,0x03,0x01,0x5f,0x01,0x01]
+# W32-REAL16: v_mad_i32_i16_e64_dpp v5, v1.l, v2.l, -1 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x5a,0xd6,0xfa,0x04,0x06,0x03,0x01,0x5f,0x01,0x01]
+# W32-FAKE16: v_mad_i32_i16_e64_dpp v5, v1, v2, -1 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x5a,0xd6,0xfa,0x04,0x06,0x03,0x01,0x5f,0x01,0x01]
+# W64-REAL16: v_mad_i32_i16_e64_dpp v5, v1.l, v2.l, -1 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x5a,0xd6,0xfa,0x04,0x06,0x03,0x01,0x5f,0x01,0x01]
+# W64-FAKE16: v_mad_i32_i16_e64_dpp v5, v1, v2, -1 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x5a,0xd6,0xfa,0x04,0x06,0x03,0x01,0x5f,0x01,0x01]
 
 0x05,0x08,0x5a,0xd6,0xfa,0x04,0xc2,0x03,0x01,0x60,0x01,0x13
-# GFX11: v_mad_i32_i16_e64_dpp v5, v1, v2, 0.5 op_sel:[1,0,0,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x08,0x5a,0xd6,0xfa,0x04,0xc2,0x03,0x01,0x60,0x01,0x13]
+# W32-REAL16: v_mad_i32_i16_e64_dpp v5, v1.h, v2.l, 0.5 op_sel:[1,0,0,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x08,0x5a,0xd6,0xfa,0x04,0xc2,0x03,0x01,0x60,0x01,0x13]
+# W32-FAKE16: v_mad_i32_i16_e64_dpp v5, v1, v2, 0.5 op_sel:[1,0,0,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x08,0x5a,0xd6,0xfa,0x04,0xc2,0x03,0x01,0x60,0x01,0x13]
+# W64-REAL16: v_mad_i32_i16_e64_dpp v5, v1.h, v2.l, 0.5 op_sel:[1,0,0,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x08,0x5a,0xd6,0xfa,0x04,0xc2,0x03,0x01,0x60,0x01,0x13]
+# W64-FAKE16: v_mad_i32_i16_e64_dpp v5, v1, v2, 0.5 op_sel:[1,0,0,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x08,0x5a,0xd6,0xfa,0x04,0xc2,0x03,0x01,0x60,0x01,0x13]
 
 0xff,0x90,0x5a,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30
-# GFX11: v_mad_i32_i16_e64_dpp v255, v255, v255, src_scc op_sel:[0,1,0,0] clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x90,0x5a,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30]
+# W32-REAL16: v_mad_i32_i16_e64_dpp v255, v255.l, v255.h, src_scc op_sel:[0,1,0,0] clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x90,0x5a,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30]
+# W32-FAKE16: v_mad_i32_i16_e64_dpp v255, v255, v255, src_scc op_sel:[0,1,0,0] clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x90,0x5a,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30]
+# W64-REAL16: v_mad_i32_i16_e64_dpp v255, v255.l, v255.h, src_scc op_sel:[0,1,0,0] clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x90,0x5a,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30]
+# W64-FAKE16: v_mad_i32_i16_e64_dpp v255, v255, v255, src_scc op_sel:[0,1,0,0] clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x90,0x5a,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30]
 
 0x05,0x00,0x41,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff
 # W32-REAL16: v_mad_u16_e64_dpp v5.l, v1.l, v2.l, v3.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x41,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
@@ -5024,46 +5087,88 @@
 # W64-FAKE16: v_mad_u16_e64_dpp v255, v255, v255, src_scc op_sel:[0,0,0,1] clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0xc0,0x41,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30]
 
 0x05,0x00,0x59,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff
-# GFX11: v_mad_u32_u16_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x59,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
+# W32-REAL16: v_mad_u32_u16_e64_dpp v5, v1.l, v2.l, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x59,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
+# W32-FAKE16: v_mad_u32_u16_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x59,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
+# W64-REAL16: v_mad_u32_u16_e64_dpp v5, v1.l, v2.l, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x59,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
+# W64-FAKE16: v_mad_u32_u16_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x59,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 
 0x05,0x00,0x59,0xd6,0xfa,0x04,0xfe,0x07,0x01,0xe4,0x00,0xff
-# GFX11: v_mad_u32_u16_e64_dpp v5, v1, v2, v255 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x59,0xd6,0xfa,0x04,0xfe,0x07,0x01,0xe4,0x00,0xff]
+# W32-REAL16: v_mad_u32_u16_e64_dpp v5, v1.l, v2.l, v255 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x59,0xd6,0xfa,0x04,0xfe,0x07,0x01,0xe4,0x00,0xff]
+# W32-FAKE16: v_mad_u32_u16_e64_dpp v5, v1, v2, v255 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x59,0xd6,0xfa,0x04,0xfe,0x07,0x01,0xe4,0x00,0xff]
+# W64-REAL16: v_mad_u32_u16_e64_dpp v5, v1.l, v2.l, v255 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x59,0xd6,0xfa,0x04,0xfe,0x07,0x01,0xe4,0x00,0xff]
+# W64-FAKE16: v_mad_u32_u16_e64_dpp v5, v1, v2, v255 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x59,0xd6,0xfa,0x04,0xfe,0x07,0x01,0xe4,0x00,0xff]
 
 0x05,0x00,0x59,0xd6,0xfa,0x04,0x0e,0x00,0x01,0x40,0x01,0xff
-# GFX11: v_mad_u32_u16_e64_dpp v5, v1, v2, s3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x59,0xd6,0xfa,0x04,0x0e,0x00,0x01,0x40,0x01,0xff]
+# W32-REAL16: v_mad_u32_u16_e64_dpp v5, v1.l, v2.l, s3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x59,0xd6,0xfa,0x04,0x0e,0x00,0x01,0x40,0x01,0xff]
+# W32-FAKE16: v_mad_u32_u16_e64_dpp v5, v1, v2, s3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x59,0xd6,0xfa,0x04,0x0e,0x00,0x01,0x40,0x01,0xff]
+# W64-REAL16: v_mad_u32_u16_e64_dpp v5, v1.l, v2.l, s3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x59,0xd6,0xfa,0x04,0x0e,0x00,0x01,0x40,0x01,0xff]
+# W64-FAKE16: v_mad_u32_u16_e64_dpp v5, v1, v2, s3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x59,0xd6,0xfa,0x04,0x0e,0x00,0x01,0x40,0x01,0xff]
 
 0x05,0x00,0x59,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x41,0x01,0xff
-# GFX11: v_mad_u32_u16_e64_dpp v5, v1, v2, s105 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x59,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x41,0x01,0xff]
+# W32-REAL16: v_mad_u32_u16_e64_dpp v5, v1.l, v2.l, s105 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x59,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x41,0x01,0xff]
+# W32-FAKE16: v_mad_u32_u16_e64_dpp v5, v1, v2, s105 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x59,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x41,0x01,0xff]
+# W64-REAL16: v_mad_u32_u16_e64_dpp v5, v1.l, v2.l, s105 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x59,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x41,0x01,0xff]
+# W64-FAKE16: v_mad_u32_u16_e64_dpp v5, v1, v2, s105 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x59,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x41,0x01,0xff]
 
 0x05,0x00,0x59,0xd6,0xfa,0x04,0xee,0x01,0x01,0x01,0x01,0xff
-# GFX11: v_mad_u32_u16_e64_dpp v5, v1, v2, ttmp15 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x59,0xd6,0xfa,0x04,0xee,0x01,0x01,0x01,0x01,0xff]
+# W32-REAL16: v_mad_u32_u16_e64_dpp v5, v1.l, v2.l, ttmp15 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x59,0xd6,0xfa,0x04,0xee,0x01,0x01,0x01,0x01,0xff]
+# W32-FAKE16: v_mad_u32_u16_e64_dpp v5, v1, v2, ttmp15 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x59,0xd6,0xfa,0x04,0xee,0x01,0x01,0x01,0x01,0xff]
+# W64-REAL16: v_mad_u32_u16_e64_dpp v5, v1.l, v2.l, ttmp15 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x59,0xd6,0xfa,0x04,0xee,0x01,0x01,0x01,0x01,0xff]
+# W64-FAKE16: v_mad_u32_u16_e64_dpp v5, v1, v2, ttmp15 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x59,0xd6,0xfa,0x04,0xee,0x01,0x01,0x01,0x01,0xff]
 
 0x05,0x00,0x59,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff
-# GFX11: v_mad_u32_u16_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x59,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
+# W32-REAL16: v_mad_u32_u16_e64_dpp v5, v1.l, v2.l, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x59,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
+# W32-FAKE16: v_mad_u32_u16_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x59,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
+# W64-REAL16: v_mad_u32_u16_e64_dpp v5, v1.l, v2.l, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x59,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
+# W64-FAKE16: v_mad_u32_u16_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x59,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
 
 0x05,0x00,0x59,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff
-# GFX11: v_mad_u32_u16_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x59,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
+# W32-REAL16: v_mad_u32_u16_e64_dpp v5, v1.l, v2.l, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x59,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
+# W32-FAKE16: v_mad_u32_u16_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x59,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
+# W64-REAL16: v_mad_u32_u16_e64_dpp v5, v1.l, v2.l, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x59,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
+# W64-FAKE16: v_mad_u32_u16_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x59,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
 
 0x05,0x00,0x59,0xd6,0xfa,0x04,0xf6,0x01,0x01,0x1f,0x01,0xff
-# GFX11: v_mad_u32_u16_e64_dpp v5, v1, v2, m0 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x59,0xd6,0xfa,0x04,0xf6,0x01,0x01,0x1f,0x01,0xff]
+# W32-REAL16: v_mad_u32_u16_e64_dpp v5, v1.l, v2.l, m0 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x59,0xd6,0xfa,0x04,0xf6,0x01,0x01,0x1f,0x01,0xff]
+# W32-FAKE16: v_mad_u32_u16_e64_dpp v5, v1, v2, m0 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x59,0xd6,0xfa,0x04,0xf6,0x01,0x01,0x1f,0x01,0xff]
+# W64-REAL16: v_mad_u32_u16_e64_dpp v5, v1.l, v2.l, m0 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x59,0xd6,0xfa,0x04,0xf6,0x01,0x01,0x1f,0x01,0xff]
+# W64-FAKE16: v_mad_u32_u16_e64_dpp v5, v1, v2, m0 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x59,0xd6,0xfa,0x04,0xf6,0x01,0x01,0x1f,0x01,0xff]
 
 0x05,0x00,0x59,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x21,0x01,0xff
-# GFX11: v_mad_u32_u16_e64_dpp v5, v1, v2, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x59,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x21,0x01,0xff]
+# W32-REAL16: v_mad_u32_u16_e64_dpp v5, v1.l, v2.l, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x59,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x21,0x01,0xff]
+# W32-FAKE16: v_mad_u32_u16_e64_dpp v5, v1, v2, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x59,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x21,0x01,0xff]
+# W64-REAL16: v_mad_u32_u16_e64_dpp v5, v1.l, v2.l, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x59,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x21,0x01,0xff]
+# W64-FAKE16: v_mad_u32_u16_e64_dpp v5, v1, v2, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x59,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x21,0x01,0xff]
 
 0x05,0x00,0x59,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x2f,0x01,0xff
-# GFX11: v_mad_u32_u16_e64_dpp v5, v1, v2, exec_lo row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x59,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x2f,0x01,0xff]
+# W32-REAL16: v_mad_u32_u16_e64_dpp v5, v1.l, v2.l, exec_lo row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x59,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x2f,0x01,0xff]
+# W32-FAKE16: v_mad_u32_u16_e64_dpp v5, v1, v2, exec_lo row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x59,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x2f,0x01,0xff]
+# W64-REAL16: v_mad_u32_u16_e64_dpp v5, v1.l, v2.l, exec_lo row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x59,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x2f,0x01,0xff]
+# W64-FAKE16: v_mad_u32_u16_e64_dpp v5, v1, v2, exec_lo row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x59,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x2f,0x01,0xff]
 
 0x05,0x00,0x59,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x50,0x01,0xff
-# GFX11: v_mad_u32_u16_e64_dpp v5, v1, v2, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x59,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x50,0x01,0xff]
+# W32-REAL16: v_mad_u32_u16_e64_dpp v5, v1.l, v2.l, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x59,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x50,0x01,0xff]
+# W32-FAKE16: v_mad_u32_u16_e64_dpp v5, v1, v2, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x59,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x50,0x01,0xff]
+# W64-REAL16: v_mad_u32_u16_e64_dpp v5, v1.l, v2.l, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x59,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x50,0x01,0xff]
+# W64-FAKE16: v_mad_u32_u16_e64_dpp v5, v1, v2, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x59,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x50,0x01,0xff]
 
 0x05,0x00,0x59,0xd6,0xfa,0x04,0x06,0x03,0x01,0x5f,0x01,0x01
-# GFX11: v_mad_u32_u16_e64_dpp v5, v1, v2, -1 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x59,0xd6,0xfa,0x04,0x06,0x03,0x01,0x5f,0x01,0x01]
+# W32-REAL16: v_mad_u32_u16_e64_dpp v5, v1.l, v2.l, -1 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x59,0xd6,0xfa,0x04,0x06,0x03,0x01,0x5f,0x01,0x01]
+# W32-FAKE16: v_mad_u32_u16_e64_dpp v5, v1, v2, -1 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x59,0xd6,0xfa,0x04,0x06,0x03,0x01,0x5f,0x01,0x01]
+# W64-REAL16: v_mad_u32_u16_e64_dpp v5, v1.l, v2.l, -1 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x59,0xd6,0xfa,0x04,0x06,0x03,0x01,0x5f,0x01,0x01]
+# W64-FAKE16: v_mad_u32_u16_e64_dpp v5, v1, v2, -1 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x59,0xd6,0xfa,0x04,0x06,0x03,0x01,0x5f,0x01,0x01]
 
 0x05,0x08,0x59,0xd6,0xfa,0x04,0xc2,0x03,0x01,0x60,0x01,0x13
-# GFX11: v_mad_u32_u16_e64_dpp v5, v1, v2, 0.5 op_sel:[1,0,0,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x08,0x59,0xd6,0xfa,0x04,0xc2,0x03,0x01,0x60,0x01,0x13]
+# W32-REAL16: v_mad_u32_u16_e64_dpp v5, v1.h, v2.l, 0.5 op_sel:[1,0,0,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x08,0x59,0xd6,0xfa,0x04,0xc2,0x03,0x01,0x60,0x01,0x13]
+# W32-FAKE16: v_mad_u32_u16_e64_dpp v5, v1, v2, 0.5 op_sel:[1,0,0,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x08,0x59,0xd6,0xfa,0x04,0xc2,0x03,0x01,0x60,0x01,0x13]
+# W64-REAL16: v_mad_u32_u16_e64_dpp v5, v1.h, v2.l, 0.5 op_sel:[1,0,0,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x08,0x59,0xd6,0xfa,0x04,0xc2,0x03,0x01,0x60,0x01,0x13]
+# W64-FAKE16: v_mad_u32_u16_e64_dpp v5, v1, v2, 0.5 op_sel:[1,0,0,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x08,0x59,0xd6,0xfa,0x04,0xc2,0x03,0x01,0x60,0x01,0x13]
 
 0xff,0x90,0x59,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30
-# GFX11: v_mad_u32_u16_e64_dpp v255, v255, v255, src_scc op_sel:[0,1,0,0] clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x90,0x59,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30]
+# W32-REAL16: v_mad_u32_u16_e64_dpp v255, v255.l, v255.h, src_scc op_sel:[0,1,0,0] clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x90,0x59,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30]
+# W32-FAKE16: v_mad_u32_u16_e64_dpp v255, v255, v255, src_scc op_sel:[0,1,0,0] clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x90,0x59,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30]
+# W64-REAL16: v_mad_u32_u16_e64_dpp v255, v255.l, v255.h, src_scc op_sel:[0,1,0,0] clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x90,0x59,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30]
+# W64-FAKE16: v_mad_u32_u16_e64_dpp v255, v255, v255, src_scc op_sel:[0,1,0,0] clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x90,0x59,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30]
 
 0x05,0x78,0x4c,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff
 # W32-REAL16: v_max3_f16_e64_dpp v5.h, v1.h, v2.h, v3.h op_sel:[1,1,1,1] quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x78,0x4c,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
diff --git a/llvm/test/MC/Disassembler/AMDGPU/gfx11_dasm_vop3_dpp16_from_vopcx.txt b/llvm/test/MC/Disassembler/AMDGPU/gfx11_dasm_vop3_dpp16_from_vopcx.txt
index 20250c1df729e..4a46eaead2390 100644
--- a/llvm/test/MC/Disassembler/AMDGPU/gfx11_dasm_vop3_dpp16_from_vopcx.txt
+++ b/llvm/test/MC/Disassembler/AMDGPU/gfx11_dasm_vop3_dpp16_from_vopcx.txt
@@ -115,46 +115,72 @@
 # GFX11: v_cmpx_class_f32_e64_dpp -|v255|, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0x7e,0x01,0xfe,0xd4,0xfa,0xfe,0x03,0x20,0xff,0x6f,0x0d,0x30]
 
 0x7e,0x00,0x82,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff
-# GFX11: v_cmpx_eq_f16_e64_dpp v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x82,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+# GFX11-REAL16: v_cmpx_eq_f16_e64_dpp v1.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x82,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+# GFX11-FAKE16: v_cmpx_eq_f16_e64_dpp v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x82,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
 
 0x7e,0x00,0x82,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff
-# GFX11: v_cmpx_eq_f16_e64_dpp v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x82,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+# GFX11-REAL16: v_cmpx_eq_f16_e64_dpp v1.l, v2.l quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x82,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+# GFX11-FAKE16: v_cmpx_eq_f16_e64_dpp v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x82,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
 
 0x7e,0x00,0x82,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff
-# GFX11: v_cmpx_eq_f16_e64_dpp v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x82,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+# GFX11-REAL16: v_cmpx_eq_f16_e64_dpp v1.l, v2.l row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x82,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_eq_f16_e64_dpp v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x82,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
 
 0x7e,0x00,0x82,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff
-# GFX11: v_cmpx_eq_f16_e64_dpp v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x82,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+# GFX11-REAL16: v_cmpx_eq_f16_e64_dpp v1.l, v2.l row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x82,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_eq_f16_e64_dpp v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x82,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
 
 0x7e,0x00,0x82,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff
-# GFX11: v_cmpx_eq_f16_e64_dpp v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x82,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+# GFX11-REAL16: v_cmpx_eq_f16_e64_dpp v1.l, v2.l row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x82,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_eq_f16_e64_dpp v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x82,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
 
 0x7e,0x00,0x82,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff
-# GFX11: v_cmpx_eq_f16_e64_dpp v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x82,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+# GFX11-REAL16: v_cmpx_eq_f16_e64_dpp v1.l, v2.l row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x82,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_eq_f16_e64_dpp v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x82,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
 
 0x7e,0x00,0x82,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff
-# GFX11: v_cmpx_eq_f16_e64_dpp v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x82,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+# GFX11-REAL16: v_cmpx_eq_f16_e64_dpp v1.l, v2.l row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x82,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_eq_f16_e64_dpp v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x82,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
 
 0x7e,0x00,0x82,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff
-# GFX11: v_cmpx_eq_f16_e64_dpp v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x82,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+# GFX11-REAL16: v_cmpx_eq_f16_e64_dpp v1.l, v2.l row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x82,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_eq_f16_e64_dpp v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x82,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
 
 0x7e,0x00,0x82,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff
-# GFX11: v_cmpx_eq_f16_e64_dpp v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x82,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+# GFX11-REAL16: v_cmpx_eq_f16_e64_dpp v1.l, v2.l row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x82,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_eq_f16_e64_dpp v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x82,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
 
 0x7e,0x00,0x82,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff
-# GFX11: v_cmpx_eq_f16_e64_dpp v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x82,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+# GFX11-REAL16: v_cmpx_eq_f16_e64_dpp v1.l, v2.l row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x82,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_eq_f16_e64_dpp v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x82,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
 
 0x7e,0x00,0x82,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff
-# GFX11: v_cmpx_eq_f16_e64_dpp v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x82,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+# GFX11-REAL16: v_cmpx_eq_f16_e64_dpp v1.l, v2.l row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x82,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_eq_f16_e64_dpp v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x82,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 
 0x7e,0x01,0x82,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01
-# GFX11: v_cmpx_eq_f16_e64_dpp |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x01,0x82,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+# GFX11-REAL16: v_cmpx_eq_f16_e64_dpp |v1.l|, -v2.l row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x01,0x82,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+# GFX11-FAKE16: v_cmpx_eq_f16_e64_dpp |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x01,0x82,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
 
 0x7e,0x02,0x82,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x01,0x13
-# GFX11: v_cmpx_eq_f16_e64_dpp -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x7e,0x02,0x82,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x01,0x13]
+# GFX11-REAL16: v_cmpx_eq_f16_e64_dpp -v1.l, |v2.l| row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x7e,0x02,0x82,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x01,0x13]
+# GFX11-FAKE16: v_cmpx_eq_f16_e64_dpp -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x7e,0x02,0x82,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x01,0x13]
 
 0x7e,0x83,0x82,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30
-# GFX11: v_cmpx_eq_f16_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0x7e,0x83,0x82,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30]
+# GFX11-REAL16: v_cmpx_eq_f16_e64_dpp -|v255.l|, -|v255.l| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0x7e,0x83,0x82,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30]
+# GFX11-FAKE16: v_cmpx_eq_f16_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0x7e,0x83,0x82,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30]
+
+0x7e,0x19,0x82,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01
+# GFX11-REAL16: v_cmpx_eq_f16_e64_dpp |v1.h|, -v2.h op_sel:[1,1] row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x19,0x82,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+# GFX11-FAKE16: v_cmpx_eq_f16_e64_dpp |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x01,0x82,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+
+0x7e,0x0a,0x82,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x01,0x13
+# GFX11-REAL16: v_cmpx_eq_f16_e64_dpp -v1.h, |v2.l| op_sel:[1,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x7e,0x0a,0x82,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x01,0x13]
+# GFX11-FAKE16: v_cmpx_eq_f16_e64_dpp -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x7e,0x02,0x82,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x01,0x13]
+
+0x7e,0x93,0x82,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30
+# GFX11-REAL16: v_cmpx_eq_f16_e64_dpp -|v255.l|, -|v255.h| op_sel:[0,1] clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0x7e,0x93,0x82,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30]
+# GFX11-FAKE16: v_cmpx_eq_f16_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0x7e,0x83,0x82,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30]
 
 0x7e,0x00,0x92,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff
 # GFX11: v_cmpx_eq_f32_e64_dpp v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x92,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
@@ -419,46 +445,72 @@
 # GFX11: v_cmpx_eq_u32_e64_dpp v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0x7e,0x00,0xca,0xd4,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 
 0x7e,0x00,0x80,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff
-# GFX11: v_cmpx_f_f16_e64_dpp v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x80,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+# GFX11-REAL16: v_cmpx_f_f16_e64_dpp v1.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x80,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+# GFX11-FAKE16: v_cmpx_f_f16_e64_dpp v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x80,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
 
 0x7e,0x00,0x80,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff
-# GFX11: v_cmpx_f_f16_e64_dpp v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x80,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+# GFX11-REAL16: v_cmpx_f_f16_e64_dpp v1.l, v2.l quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x80,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+# GFX11-FAKE16: v_cmpx_f_f16_e64_dpp v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x80,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
 
 0x7e,0x00,0x80,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff
-# GFX11: v_cmpx_f_f16_e64_dpp v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x80,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+# GFX11-REAL16: v_cmpx_f_f16_e64_dpp v1.l, v2.l row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x80,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_f_f16_e64_dpp v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x80,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
 
 0x7e,0x00,0x80,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff
-# GFX11: v_cmpx_f_f16_e64_dpp v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x80,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+# GFX11-REAL16: v_cmpx_f_f16_e64_dpp v1.l, v2.l row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x80,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_f_f16_e64_dpp v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x80,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
 
 0x7e,0x00,0x80,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff
-# GFX11: v_cmpx_f_f16_e64_dpp v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x80,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+# GFX11-REAL16: v_cmpx_f_f16_e64_dpp v1.l, v2.l row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x80,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_f_f16_e64_dpp v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x80,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
 
 0x7e,0x00,0x80,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff
-# GFX11: v_cmpx_f_f16_e64_dpp v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x80,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+# GFX11-REAL16: v_cmpx_f_f16_e64_dpp v1.l, v2.l row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x80,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_f_f16_e64_dpp v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x80,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
 
 0x7e,0x00,0x80,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff
-# GFX11: v_cmpx_f_f16_e64_dpp v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x80,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+# GFX11-REAL16: v_cmpx_f_f16_e64_dpp v1.l, v2.l row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x80,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_f_f16_e64_dpp v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x80,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
 
 0x7e,0x00,0x80,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff
-# GFX11: v_cmpx_f_f16_e64_dpp v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x80,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+# GFX11-REAL16: v_cmpx_f_f16_e64_dpp v1.l, v2.l row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x80,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_f_f16_e64_dpp v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x80,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
 
 0x7e,0x00,0x80,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff
-# GFX11: v_cmpx_f_f16_e64_dpp v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x80,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+# GFX11-REAL16: v_cmpx_f_f16_e64_dpp v1.l, v2.l row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x80,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_f_f16_e64_dpp v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x80,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
 
 0x7e,0x00,0x80,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff
-# GFX11: v_cmpx_f_f16_e64_dpp v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x80,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+# GFX11-REAL16: v_cmpx_f_f16_e64_dpp v1.l, v2.l row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x80,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_f_f16_e64_dpp v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x80,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
 
 0x7e,0x00,0x80,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff
-# GFX11: v_cmpx_f_f16_e64_dpp v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x80,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+# GFX11-REAL16: v_cmpx_f_f16_e64_dpp v1.l, v2.l row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x80,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_f_f16_e64_dpp v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x80,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 
 0x7e,0x01,0x80,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01
-# GFX11: v_cmpx_f_f16_e64_dpp |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x01,0x80,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+# GFX11-REAL16: v_cmpx_f_f16_e64_dpp |v1.l|, -v2.l row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x01,0x80,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+# GFX11-FAKE16: v_cmpx_f_f16_e64_dpp |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x01,0x80,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
 
 0x7e,0x02,0x80,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x01,0x13
-# GFX11: v_cmpx_f_f16_e64_dpp -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x7e,0x02,0x80,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x01,0x13]
+# GFX11-REAL16: v_cmpx_f_f16_e64_dpp -v1.l, |v2.l| row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x7e,0x02,0x80,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x01,0x13]
+# GFX11-FAKE16: v_cmpx_f_f16_e64_dpp -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x7e,0x02,0x80,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x01,0x13]
 
 0x7e,0x83,0x80,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30
-# GFX11: v_cmpx_f_f16_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0x7e,0x83,0x80,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30]
+# GFX11-REAL16: v_cmpx_f_f16_e64_dpp -|v255.l|, -|v255.l| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0x7e,0x83,0x80,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30]
+# GFX11-FAKE16: v_cmpx_f_f16_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0x7e,0x83,0x80,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30]
+
+0x7e,0x19,0x80,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01
+# GFX11-REAL16: v_cmpx_f_f16_e64_dpp |v1.h|, -v2.h op_sel:[1,1] row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x19,0x80,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+# GFX11-FAKE16: v_cmpx_f_f16_e64_dpp |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x01,0x80,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+
+0x7e,0x0a,0x80,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x01,0x13
+# GFX11-REAL16: v_cmpx_f_f16_e64_dpp -v1.h, |v2.l| op_sel:[1,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x7e,0x0a,0x80,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x01,0x13]
+# GFX11-FAKE16: v_cmpx_f_f16_e64_dpp -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x7e,0x02,0x80,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x01,0x13]
+
+0x7e,0x93,0x80,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30
+# GFX11-REAL16: v_cmpx_f_f16_e64_dpp -|v255.l|, -|v255.h| op_sel:[0,1] clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0x7e,0x93,0x80,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30]
+# GFX11-FAKE16: v_cmpx_f_f16_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0x7e,0x83,0x80,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30]
 
 0x7e,0x00,0x90,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff
 # GFX11: v_cmpx_f_f32_e64_dpp v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x90,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
@@ -587,46 +639,72 @@
 # GFX11: v_cmpx_f_u32_e64_dpp v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0x7e,0x00,0xc8,0xd4,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 
 0x7e,0x00,0x86,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff
-# GFX11: v_cmpx_ge_f16_e64_dpp v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x86,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+# GFX11-REAL16: v_cmpx_ge_f16_e64_dpp v1.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x86,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+# GFX11-FAKE16: v_cmpx_ge_f16_e64_dpp v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x86,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
 
 0x7e,0x00,0x86,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff
-# GFX11: v_cmpx_ge_f16_e64_dpp v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x86,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+# GFX11-REAL16: v_cmpx_ge_f16_e64_dpp v1.l, v2.l quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x86,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+# GFX11-FAKE16: v_cmpx_ge_f16_e64_dpp v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x86,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
 
 0x7e,0x00,0x86,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff
-# GFX11: v_cmpx_ge_f16_e64_dpp v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x86,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+# GFX11-REAL16: v_cmpx_ge_f16_e64_dpp v1.l, v2.l row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x86,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_ge_f16_e64_dpp v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x86,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
 
 0x7e,0x00,0x86,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff
-# GFX11: v_cmpx_ge_f16_e64_dpp v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x86,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+# GFX11-REAL16: v_cmpx_ge_f16_e64_dpp v1.l, v2.l row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x86,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_ge_f16_e64_dpp v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x86,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
 
 0x7e,0x00,0x86,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff
-# GFX11: v_cmpx_ge_f16_e64_dpp v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x86,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+# GFX11-REAL16: v_cmpx_ge_f16_e64_dpp v1.l, v2.l row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x86,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_ge_f16_e64_dpp v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x86,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
 
 0x7e,0x00,0x86,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff
-# GFX11: v_cmpx_ge_f16_e64_dpp v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x86,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+# GFX11-REAL16: v_cmpx_ge_f16_e64_dpp v1.l, v2.l row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x86,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_ge_f16_e64_dpp v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x86,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
 
 0x7e,0x00,0x86,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff
-# GFX11: v_cmpx_ge_f16_e64_dpp v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x86,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+# GFX11-REAL16: v_cmpx_ge_f16_e64_dpp v1.l, v2.l row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x86,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_ge_f16_e64_dpp v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x86,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
 
 0x7e,0x00,0x86,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff
-# GFX11: v_cmpx_ge_f16_e64_dpp v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x86,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+# GFX11-REAL16: v_cmpx_ge_f16_e64_dpp v1.l, v2.l row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x86,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_ge_f16_e64_dpp v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x86,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
 
 0x7e,0x00,0x86,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff
-# GFX11: v_cmpx_ge_f16_e64_dpp v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x86,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+# GFX11-REAL16: v_cmpx_ge_f16_e64_dpp v1.l, v2.l row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x86,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_ge_f16_e64_dpp v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x86,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
 
 0x7e,0x00,0x86,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff
-# GFX11: v_cmpx_ge_f16_e64_dpp v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x86,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+# GFX11-REAL16: v_cmpx_ge_f16_e64_dpp v1.l, v2.l row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x86,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_ge_f16_e64_dpp v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x86,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
 
 0x7e,0x00,0x86,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff
-# GFX11: v_cmpx_ge_f16_e64_dpp v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x86,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+# GFX11-REAL16: v_cmpx_ge_f16_e64_dpp v1.l, v2.l row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x86,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_ge_f16_e64_dpp v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x86,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 
 0x7e,0x01,0x86,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01
-# GFX11: v_cmpx_ge_f16_e64_dpp |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x01,0x86,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+# GFX11-REAL16: v_cmpx_ge_f16_e64_dpp |v1.l|, -v2.l row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x01,0x86,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+# GFX11-FAKE16: v_cmpx_ge_f16_e64_dpp |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x01,0x86,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
 
 0x7e,0x02,0x86,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x01,0x13
-# GFX11: v_cmpx_ge_f16_e64_dpp -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x7e,0x02,0x86,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x01,0x13]
+# GFX11-REAL16: v_cmpx_ge_f16_e64_dpp -v1.l, |v2.l| row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x7e,0x02,0x86,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x01,0x13]
+# GFX11-FAKE16: v_cmpx_ge_f16_e64_dpp -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x7e,0x02,0x86,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x01,0x13]
 
 0x7e,0x83,0x86,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30
-# GFX11: v_cmpx_ge_f16_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0x7e,0x83,0x86,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30]
+# GFX11-REAL16: v_cmpx_ge_f16_e64_dpp -|v255.l|, -|v255.l| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0x7e,0x83,0x86,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30]
+# GFX11-FAKE16: v_cmpx_ge_f16_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0x7e,0x83,0x86,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30]
+
+0x7e,0x19,0x86,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01
+# GFX11-REAL16: v_cmpx_ge_f16_e64_dpp |v1.h|, -v2.h op_sel:[1,1] row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x19,0x86,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+# GFX11-FAKE16: v_cmpx_ge_f16_e64_dpp |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x01,0x86,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+
+0x7e,0x0a,0x86,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x01,0x13
+# GFX11-REAL16: v_cmpx_ge_f16_e64_dpp -v1.h, |v2.l| op_sel:[1,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x7e,0x0a,0x86,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x01,0x13]
+# GFX11-FAKE16: v_cmpx_ge_f16_e64_dpp -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x7e,0x02,0x86,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x01,0x13]
+
+0x7e,0x93,0x86,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30
+# GFX11-REAL16: v_cmpx_ge_f16_e64_dpp -|v255.l|, -|v255.h| op_sel:[0,1] clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0x7e,0x93,0x86,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30]
+# GFX11-FAKE16: v_cmpx_ge_f16_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0x7e,0x83,0x86,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30]
 
 0x7e,0x00,0x96,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff
 # GFX11: v_cmpx_ge_f32_e64_dpp v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x96,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
@@ -891,46 +969,72 @@
 # GFX11: v_cmpx_ge_u32_e64_dpp v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0x7e,0x00,0xce,0xd4,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 
 0x7e,0x00,0x84,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff
-# GFX11: v_cmpx_gt_f16_e64_dpp v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x84,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+# GFX11-REAL16: v_cmpx_gt_f16_e64_dpp v1.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x84,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+# GFX11-FAKE16: v_cmpx_gt_f16_e64_dpp v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x84,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
 
 0x7e,0x00,0x84,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff
-# GFX11: v_cmpx_gt_f16_e64_dpp v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x84,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+# GFX11-REAL16: v_cmpx_gt_f16_e64_dpp v1.l, v2.l quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x84,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+# GFX11-FAKE16: v_cmpx_gt_f16_e64_dpp v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x84,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
 
 0x7e,0x00,0x84,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff
-# GFX11: v_cmpx_gt_f16_e64_dpp v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x84,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+# GFX11-REAL16: v_cmpx_gt_f16_e64_dpp v1.l, v2.l row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x84,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_gt_f16_e64_dpp v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x84,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
 
 0x7e,0x00,0x84,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff
-# GFX11: v_cmpx_gt_f16_e64_dpp v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x84,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+# GFX11-REAL16: v_cmpx_gt_f16_e64_dpp v1.l, v2.l row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x84,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_gt_f16_e64_dpp v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x84,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
 
 0x7e,0x00,0x84,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff
-# GFX11: v_cmpx_gt_f16_e64_dpp v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x84,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+# GFX11-REAL16: v_cmpx_gt_f16_e64_dpp v1.l, v2.l row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x84,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_gt_f16_e64_dpp v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x84,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
 
 0x7e,0x00,0x84,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff
-# GFX11: v_cmpx_gt_f16_e64_dpp v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x84,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+# GFX11-REAL16: v_cmpx_gt_f16_e64_dpp v1.l, v2.l row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x84,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_gt_f16_e64_dpp v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x84,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
 
 0x7e,0x00,0x84,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff
-# GFX11: v_cmpx_gt_f16_e64_dpp v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x84,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+# GFX11-REAL16: v_cmpx_gt_f16_e64_dpp v1.l, v2.l row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x84,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_gt_f16_e64_dpp v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x84,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
 
 0x7e,0x00,0x84,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff
-# GFX11: v_cmpx_gt_f16_e64_dpp v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x84,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+# GFX11-REAL16: v_cmpx_gt_f16_e64_dpp v1.l, v2.l row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x84,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_gt_f16_e64_dpp v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x84,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
 
 0x7e,0x00,0x84,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff
-# GFX11: v_cmpx_gt_f16_e64_dpp v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x84,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+# GFX11-REAL16: v_cmpx_gt_f16_e64_dpp v1.l, v2.l row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x84,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_gt_f16_e64_dpp v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x84,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
 
 0x7e,0x00,0x84,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff
-# GFX11: v_cmpx_gt_f16_e64_dpp v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x84,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+# GFX11-REAL16: v_cmpx_gt_f16_e64_dpp v1.l, v2.l row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x84,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_gt_f16_e64_dpp v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x84,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
 
 0x7e,0x00,0x84,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff
-# GFX11: v_cmpx_gt_f16_e64_dpp v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x84,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+# GFX11-REAL16: v_cmpx_gt_f16_e64_dpp v1.l, v2.l row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x84,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_gt_f16_e64_dpp v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x84,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 
 0x7e,0x01,0x84,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01
-# GFX11: v_cmpx_gt_f16_e64_dpp |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x01,0x84,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+# GFX11-REAL16: v_cmpx_gt_f16_e64_dpp |v1.l|, -v2.l row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x01,0x84,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+# GFX11-FAKE16: v_cmpx_gt_f16_e64_dpp |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x01,0x84,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
 
 0x7e,0x02,0x84,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x01,0x13
-# GFX11: v_cmpx_gt_f16_e64_dpp -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x7e,0x02,0x84,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x01,0x13]
+# GFX11-REAL16: v_cmpx_gt_f16_e64_dpp -v1.l, |v2.l| row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x7e,0x02,0x84,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x01,0x13]
+# GFX11-FAKE16: v_cmpx_gt_f16_e64_dpp -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x7e,0x02,0x84,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x01,0x13]
 
 0x7e,0x83,0x84,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30
-# GFX11: v_cmpx_gt_f16_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0x7e,0x83,0x84,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30]
+# GFX11-REAL16: v_cmpx_gt_f16_e64_dpp -|v255.l|, -|v255.l| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0x7e,0x83,0x84,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30]
+# GFX11-FAKE16: v_cmpx_gt_f16_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0x7e,0x83,0x84,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30]
+
+0x7e,0x19,0x84,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01
+# GFX11-REAL16: v_cmpx_gt_f16_e64_dpp |v1.h|, -v2.h op_sel:[1,1] row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x19,0x84,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+# GFX11-FAKE16: v_cmpx_gt_f16_e64_dpp |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x01,0x84,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+
+0x7e,0x0a,0x84,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x01,0x13
+# GFX11-REAL16: v_cmpx_gt_f16_e64_dpp -v1.h, |v2.l| op_sel:[1,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x7e,0x0a,0x84,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x01,0x13]
+# GFX11-FAKE16: v_cmpx_gt_f16_e64_dpp -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x7e,0x02,0x84,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x01,0x13]
+
+0x7e,0x93,0x84,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30
+# GFX11-REAL16: v_cmpx_gt_f16_e64_dpp -|v255.l|, -|v255.h| op_sel:[0,1] clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0x7e,0x93,0x84,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30]
+# GFX11-FAKE16: v_cmpx_gt_f16_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0x7e,0x83,0x84,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30]
 
 0x7e,0x00,0x94,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff
 # GFX11: v_cmpx_gt_f32_e64_dpp v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x94,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
@@ -1195,46 +1299,72 @@
 # GFX11: v_cmpx_gt_u32_e64_dpp v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0x7e,0x00,0xcc,0xd4,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 
 0x7e,0x00,0x83,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff
-# GFX11: v_cmpx_le_f16_e64_dpp v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x83,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+# GFX11-REAL16: v_cmpx_le_f16_e64_dpp v1.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x83,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+# GFX11-FAKE16: v_cmpx_le_f16_e64_dpp v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x83,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
 
 0x7e,0x00,0x83,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff
-# GFX11: v_cmpx_le_f16_e64_dpp v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x83,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+# GFX11-REAL16: v_cmpx_le_f16_e64_dpp v1.l, v2.l quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x83,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+# GFX11-FAKE16: v_cmpx_le_f16_e64_dpp v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x83,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
 
 0x7e,0x00,0x83,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff
-# GFX11: v_cmpx_le_f16_e64_dpp v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x83,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+# GFX11-REAL16: v_cmpx_le_f16_e64_dpp v1.l, v2.l row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x83,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_le_f16_e64_dpp v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x83,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
 
 0x7e,0x00,0x83,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff
-# GFX11: v_cmpx_le_f16_e64_dpp v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x83,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+# GFX11-REAL16: v_cmpx_le_f16_e64_dpp v1.l, v2.l row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x83,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_le_f16_e64_dpp v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x83,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
 
 0x7e,0x00,0x83,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff
-# GFX11: v_cmpx_le_f16_e64_dpp v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x83,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+# GFX11-REAL16: v_cmpx_le_f16_e64_dpp v1.l, v2.l row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x83,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_le_f16_e64_dpp v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x83,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
 
 0x7e,0x00,0x83,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff
-# GFX11: v_cmpx_le_f16_e64_dpp v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x83,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+# GFX11-REAL16: v_cmpx_le_f16_e64_dpp v1.l, v2.l row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x83,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_le_f16_e64_dpp v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x83,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
 
 0x7e,0x00,0x83,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff
-# GFX11: v_cmpx_le_f16_e64_dpp v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x83,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+# GFX11-REAL16: v_cmpx_le_f16_e64_dpp v1.l, v2.l row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x83,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_le_f16_e64_dpp v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x83,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
 
 0x7e,0x00,0x83,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff
-# GFX11: v_cmpx_le_f16_e64_dpp v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x83,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+# GFX11-REAL16: v_cmpx_le_f16_e64_dpp v1.l, v2.l row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x83,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_le_f16_e64_dpp v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x83,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
 
 0x7e,0x00,0x83,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff
-# GFX11: v_cmpx_le_f16_e64_dpp v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x83,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+# GFX11-REAL16: v_cmpx_le_f16_e64_dpp v1.l, v2.l row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x83,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_le_f16_e64_dpp v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x83,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
 
 0x7e,0x00,0x83,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff
-# GFX11: v_cmpx_le_f16_e64_dpp v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x83,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+# GFX11-REAL16: v_cmpx_le_f16_e64_dpp v1.l, v2.l row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x83,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_le_f16_e64_dpp v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x83,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
 
 0x7e,0x00,0x83,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff
-# GFX11: v_cmpx_le_f16_e64_dpp v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x83,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+# GFX11-REAL16: v_cmpx_le_f16_e64_dpp v1.l, v2.l row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x83,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_le_f16_e64_dpp v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x83,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 
 0x7e,0x01,0x83,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01
-# GFX11: v_cmpx_le_f16_e64_dpp |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x01,0x83,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+# GFX11-REAL16: v_cmpx_le_f16_e64_dpp |v1.l|, -v2.l row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x01,0x83,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+# GFX11-FAKE16: v_cmpx_le_f16_e64_dpp |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x01,0x83,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
 
 0x7e,0x02,0x83,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x01,0x13
-# GFX11: v_cmpx_le_f16_e64_dpp -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x7e,0x02,0x83,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x01,0x13]
+# GFX11-REAL16: v_cmpx_le_f16_e64_dpp -v1.l, |v2.l| row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x7e,0x02,0x83,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x01,0x13]
+# GFX11-FAKE16: v_cmpx_le_f16_e64_dpp -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x7e,0x02,0x83,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x01,0x13]
 
 0x7e,0x83,0x83,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30
-# GFX11: v_cmpx_le_f16_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0x7e,0x83,0x83,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30]
+# GFX11-REAL16: v_cmpx_le_f16_e64_dpp -|v255.l|, -|v255.l| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0x7e,0x83,0x83,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30]
+# GFX11-FAKE16: v_cmpx_le_f16_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0x7e,0x83,0x83,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30]
+
+0x7e,0x19,0x83,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01
+# GFX11-REAL16: v_cmpx_le_f16_e64_dpp |v1.h|, -v2.h op_sel:[1,1] row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x19,0x83,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+# GFX11-FAKE16: v_cmpx_le_f16_e64_dpp |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x01,0x83,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+
+0x7e,0x0a,0x83,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x01,0x13
+# GFX11-REAL16: v_cmpx_le_f16_e64_dpp -v1.h, |v2.l| op_sel:[1,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x7e,0x0a,0x83,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x01,0x13]
+# GFX11-FAKE16: v_cmpx_le_f16_e64_dpp -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x7e,0x02,0x83,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x01,0x13]
+
+0x7e,0x93,0x83,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30
+# GFX11-REAL16: v_cmpx_le_f16_e64_dpp -|v255.l|, -|v255.h| op_sel:[0,1] clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0x7e,0x93,0x83,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30]
+# GFX11-FAKE16: v_cmpx_le_f16_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0x7e,0x83,0x83,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30]
 
 0x7e,0x00,0x93,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff
 # GFX11: v_cmpx_le_f32_e64_dpp v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x93,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
@@ -1499,46 +1629,72 @@
 # GFX11: v_cmpx_le_u32_e64_dpp v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0x7e,0x00,0xcb,0xd4,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 
 0x7e,0x00,0x85,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff
-# GFX11: v_cmpx_lg_f16_e64_dpp v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x85,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+# GFX11-REAL16: v_cmpx_lg_f16_e64_dpp v1.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x85,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+# GFX11-FAKE16: v_cmpx_lg_f16_e64_dpp v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x85,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
 
 0x7e,0x00,0x85,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff
-# GFX11: v_cmpx_lg_f16_e64_dpp v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x85,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+# GFX11-REAL16: v_cmpx_lg_f16_e64_dpp v1.l, v2.l quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x85,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+# GFX11-FAKE16: v_cmpx_lg_f16_e64_dpp v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x85,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
 
 0x7e,0x00,0x85,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff
-# GFX11: v_cmpx_lg_f16_e64_dpp v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x85,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+# GFX11-REAL16: v_cmpx_lg_f16_e64_dpp v1.l, v2.l row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x85,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_lg_f16_e64_dpp v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x85,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
 
 0x7e,0x00,0x85,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff
-# GFX11: v_cmpx_lg_f16_e64_dpp v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x85,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+# GFX11-REAL16: v_cmpx_lg_f16_e64_dpp v1.l, v2.l row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x85,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_lg_f16_e64_dpp v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x85,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
 
 0x7e,0x00,0x85,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff
-# GFX11: v_cmpx_lg_f16_e64_dpp v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x85,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+# GFX11-REAL16: v_cmpx_lg_f16_e64_dpp v1.l, v2.l row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x85,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_lg_f16_e64_dpp v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x85,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
 
 0x7e,0x00,0x85,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff
-# GFX11: v_cmpx_lg_f16_e64_dpp v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x85,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+# GFX11-REAL16: v_cmpx_lg_f16_e64_dpp v1.l, v2.l row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x85,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_lg_f16_e64_dpp v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x85,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
 
 0x7e,0x00,0x85,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff
-# GFX11: v_cmpx_lg_f16_e64_dpp v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x85,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+# GFX11-REAL16: v_cmpx_lg_f16_e64_dpp v1.l, v2.l row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x85,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_lg_f16_e64_dpp v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x85,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
 
 0x7e,0x00,0x85,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff
-# GFX11: v_cmpx_lg_f16_e64_dpp v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x85,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+# GFX11-REAL16: v_cmpx_lg_f16_e64_dpp v1.l, v2.l row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x85,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_lg_f16_e64_dpp v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x85,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
 
 0x7e,0x00,0x85,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff
-# GFX11: v_cmpx_lg_f16_e64_dpp v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x85,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+# GFX11-REAL16: v_cmpx_lg_f16_e64_dpp v1.l, v2.l row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x85,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_lg_f16_e64_dpp v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x85,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
 
 0x7e,0x00,0x85,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff
-# GFX11: v_cmpx_lg_f16_e64_dpp v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x85,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+# GFX11-REAL16: v_cmpx_lg_f16_e64_dpp v1.l, v2.l row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x85,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_lg_f16_e64_dpp v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x85,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
 
 0x7e,0x00,0x85,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff
-# GFX11: v_cmpx_lg_f16_e64_dpp v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x85,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+# GFX11-REAL16: v_cmpx_lg_f16_e64_dpp v1.l, v2.l row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x85,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_lg_f16_e64_dpp v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x85,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 
 0x7e,0x01,0x85,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01
-# GFX11: v_cmpx_lg_f16_e64_dpp |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x01,0x85,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+# GFX11-REAL16: v_cmpx_lg_f16_e64_dpp |v1.l|, -v2.l row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x01,0x85,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+# GFX11-FAKE16: v_cmpx_lg_f16_e64_dpp |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x01,0x85,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
 
 0x7e,0x02,0x85,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x01,0x13
-# GFX11: v_cmpx_lg_f16_e64_dpp -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x7e,0x02,0x85,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x01,0x13]
+# GFX11-REAL16: v_cmpx_lg_f16_e64_dpp -v1.l, |v2.l| row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x7e,0x02,0x85,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x01,0x13]
+# GFX11-FAKE16: v_cmpx_lg_f16_e64_dpp -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x7e,0x02,0x85,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x01,0x13]
 
 0x7e,0x83,0x85,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30
-# GFX11: v_cmpx_lg_f16_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0x7e,0x83,0x85,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30]
+# GFX11-REAL16: v_cmpx_lg_f16_e64_dpp -|v255.l|, -|v255.l| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0x7e,0x83,0x85,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30]
+# GFX11-FAKE16: v_cmpx_lg_f16_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0x7e,0x83,0x85,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30]
+
+0x7e,0x19,0x85,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01
+# GFX11-REAL16: v_cmpx_lg_f16_e64_dpp |v1.h|, -v2.h op_sel:[1,1] row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x19,0x85,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+# GFX11-FAKE16: v_cmpx_lg_f16_e64_dpp |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x01,0x85,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+
+0x7e,0x0a,0x85,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x01,0x13
+# GFX11-REAL16: v_cmpx_lg_f16_e64_dpp -v1.h, |v2.l| op_sel:[1,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x7e,0x0a,0x85,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x01,0x13]
+# GFX11-FAKE16: v_cmpx_lg_f16_e64_dpp -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x7e,0x02,0x85,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x01,0x13]
+
+0x7e,0x93,0x85,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30
+# GFX11-REAL16: v_cmpx_lg_f16_e64_dpp -|v255.l|, -|v255.h| op_sel:[0,1] clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0x7e,0x93,0x85,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30]
+# GFX11-FAKE16: v_cmpx_lg_f16_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0x7e,0x83,0x85,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30]
 
 0x7e,0x00,0x95,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff
 # GFX11: v_cmpx_lg_f32_e64_dpp v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x95,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
@@ -2133,46 +2289,72 @@
 # GFX11: v_cmpx_ne_u32_e64_dpp v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0x7e,0x00,0xcd,0xd4,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 
 0x7e,0x00,0x8d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff
-# GFX11: v_cmpx_neq_f16_e64_dpp v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+# GFX11-REAL16: v_cmpx_neq_f16_e64_dpp v1.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+# GFX11-FAKE16: v_cmpx_neq_f16_e64_dpp v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
 
 0x7e,0x00,0x8d,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff
-# GFX11: v_cmpx_neq_f16_e64_dpp v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8d,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+# GFX11-REAL16: v_cmpx_neq_f16_e64_dpp v1.l, v2.l quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8d,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+# GFX11-FAKE16: v_cmpx_neq_f16_e64_dpp v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8d,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
 
 0x7e,0x00,0x8d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff
-# GFX11: v_cmpx_neq_f16_e64_dpp v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+# GFX11-REAL16: v_cmpx_neq_f16_e64_dpp v1.l, v2.l row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_neq_f16_e64_dpp v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
 
 0x7e,0x00,0x8d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff
-# GFX11: v_cmpx_neq_f16_e64_dpp v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+# GFX11-REAL16: v_cmpx_neq_f16_e64_dpp v1.l, v2.l row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_neq_f16_e64_dpp v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
 
 0x7e,0x00,0x8d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff
-# GFX11: v_cmpx_neq_f16_e64_dpp v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+# GFX11-REAL16: v_cmpx_neq_f16_e64_dpp v1.l, v2.l row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_neq_f16_e64_dpp v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
 
 0x7e,0x00,0x8d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff
-# GFX11: v_cmpx_neq_f16_e64_dpp v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+# GFX11-REAL16: v_cmpx_neq_f16_e64_dpp v1.l, v2.l row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_neq_f16_e64_dpp v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
 
 0x7e,0x00,0x8d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff
-# GFX11: v_cmpx_neq_f16_e64_dpp v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+# GFX11-REAL16: v_cmpx_neq_f16_e64_dpp v1.l, v2.l row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_neq_f16_e64_dpp v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
 
 0x7e,0x00,0x8d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff
-# GFX11: v_cmpx_neq_f16_e64_dpp v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+# GFX11-REAL16: v_cmpx_neq_f16_e64_dpp v1.l, v2.l row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_neq_f16_e64_dpp v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
 
 0x7e,0x00,0x8d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff
-# GFX11: v_cmpx_neq_f16_e64_dpp v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+# GFX11-REAL16: v_cmpx_neq_f16_e64_dpp v1.l, v2.l row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_neq_f16_e64_dpp v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
 
 0x7e,0x00,0x8d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff
-# GFX11: v_cmpx_neq_f16_e64_dpp v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+# GFX11-REAL16: v_cmpx_neq_f16_e64_dpp v1.l, v2.l row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_neq_f16_e64_dpp v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
 
 0x7e,0x00,0x8d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff
-# GFX11: v_cmpx_neq_f16_e64_dpp v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+# GFX11-REAL16: v_cmpx_neq_f16_e64_dpp v1.l, v2.l row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_neq_f16_e64_dpp v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 
 0x7e,0x01,0x8d,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01
-# GFX11: v_cmpx_neq_f16_e64_dpp |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x01,0x8d,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+# GFX11-REAL16: v_cmpx_neq_f16_e64_dpp |v1.l|, -v2.l row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x01,0x8d,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+# GFX11-FAKE16: v_cmpx_neq_f16_e64_dpp |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x01,0x8d,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
 
 0x7e,0x02,0x8d,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x01,0x13
-# GFX11: v_cmpx_neq_f16_e64_dpp -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x7e,0x02,0x8d,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x01,0x13]
+# GFX11-REAL16: v_cmpx_neq_f16_e64_dpp -v1.l, |v2.l| row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x7e,0x02,0x8d,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x01,0x13]
+# GFX11-FAKE16: v_cmpx_neq_f16_e64_dpp -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x7e,0x02,0x8d,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x01,0x13]
 
 0x7e,0x83,0x8d,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30
-# GFX11: v_cmpx_neq_f16_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0x7e,0x83,0x8d,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30]
+# GFX11-REAL16: v_cmpx_neq_f16_e64_dpp -|v255.l|, -|v255.l| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0x7e,0x83,0x8d,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30]
+# GFX11-FAKE16: v_cmpx_neq_f16_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0x7e,0x83,0x8d,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30]
+
+0x7e,0x19,0x8d,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01
+# GFX11-REAL16: v_cmpx_neq_f16_e64_dpp |v1.h|, -v2.h op_sel:[1,1] row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x19,0x8d,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+# GFX11-FAKE16: v_cmpx_neq_f16_e64_dpp |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x01,0x8d,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+
+0x7e,0x0a,0x8d,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x01,0x13
+# GFX11-REAL16: v_cmpx_neq_f16_e64_dpp -v1.h, |v2.l| op_sel:[1,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x7e,0x0a,0x8d,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x01,0x13]
+# GFX11-FAKE16: v_cmpx_neq_f16_e64_dpp -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x7e,0x02,0x8d,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x01,0x13]
+
+0x7e,0x93,0x8d,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30
+# GFX11-REAL16: v_cmpx_neq_f16_e64_dpp -|v255.l|, -|v255.h| op_sel:[0,1] clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0x7e,0x93,0x8d,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30]
+# GFX11-FAKE16: v_cmpx_neq_f16_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0x7e,0x83,0x8d,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30]
 
 0x7e,0x00,0x9d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff
 # GFX11: v_cmpx_neq_f32_e64_dpp v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x9d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
@@ -2217,46 +2399,72 @@
 # GFX11: v_cmpx_neq_f32_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0x7e,0x83,0x9d,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30]
 
 0x7e,0x00,0x89,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff
-# GFX11: v_cmpx_nge_f16_e64_dpp v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x89,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+# GFX11-REAL16: v_cmpx_nge_f16_e64_dpp v1.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x89,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+# GFX11-FAKE16: v_cmpx_nge_f16_e64_dpp v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x89,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
 
 0x7e,0x00,0x89,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff
-# GFX11: v_cmpx_nge_f16_e64_dpp v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x89,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+# GFX11-REAL16: v_cmpx_nge_f16_e64_dpp v1.l, v2.l quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x89,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+# GFX11-FAKE16: v_cmpx_nge_f16_e64_dpp v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x89,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
 
 0x7e,0x00,0x89,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff
-# GFX11: v_cmpx_nge_f16_e64_dpp v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x89,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+# GFX11-REAL16: v_cmpx_nge_f16_e64_dpp v1.l, v2.l row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x89,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_nge_f16_e64_dpp v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x89,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
 
 0x7e,0x00,0x89,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff
-# GFX11: v_cmpx_nge_f16_e64_dpp v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x89,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+# GFX11-REAL16: v_cmpx_nge_f16_e64_dpp v1.l, v2.l row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x89,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_nge_f16_e64_dpp v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x89,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
 
 0x7e,0x00,0x89,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff
-# GFX11: v_cmpx_nge_f16_e64_dpp v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x89,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+# GFX11-REAL16: v_cmpx_nge_f16_e64_dpp v1.l, v2.l row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x89,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_nge_f16_e64_dpp v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x89,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
 
 0x7e,0x00,0x89,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff
-# GFX11: v_cmpx_nge_f16_e64_dpp v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x89,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+# GFX11-REAL16: v_cmpx_nge_f16_e64_dpp v1.l, v2.l row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x89,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_nge_f16_e64_dpp v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x89,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
 
 0x7e,0x00,0x89,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff
-# GFX11: v_cmpx_nge_f16_e64_dpp v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x89,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+# GFX11-REAL16: v_cmpx_nge_f16_e64_dpp v1.l, v2.l row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x89,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_nge_f16_e64_dpp v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x89,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
 
 0x7e,0x00,0x89,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff
-# GFX11: v_cmpx_nge_f16_e64_dpp v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x89,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+# GFX11-REAL16: v_cmpx_nge_f16_e64_dpp v1.l, v2.l row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x89,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_nge_f16_e64_dpp v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x89,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
 
 0x7e,0x00,0x89,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff
-# GFX11: v_cmpx_nge_f16_e64_dpp v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x89,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+# GFX11-REAL16: v_cmpx_nge_f16_e64_dpp v1.l, v2.l row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x89,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_nge_f16_e64_dpp v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x89,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
 
 0x7e,0x00,0x89,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff
-# GFX11: v_cmpx_nge_f16_e64_dpp v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x89,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+# GFX11-REAL16: v_cmpx_nge_f16_e64_dpp v1.l, v2.l row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x89,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_nge_f16_e64_dpp v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x89,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
 
 0x7e,0x00,0x89,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff
-# GFX11: v_cmpx_nge_f16_e64_dpp v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x89,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+# GFX11-REAL16: v_cmpx_nge_f16_e64_dpp v1.l, v2.l row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x89,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_nge_f16_e64_dpp v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x89,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 
 0x7e,0x01,0x89,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01
-# GFX11: v_cmpx_nge_f16_e64_dpp |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x01,0x89,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+# GFX11-REAL16: v_cmpx_nge_f16_e64_dpp |v1.l|, -v2.l row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x01,0x89,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+# GFX11-FAKE16: v_cmpx_nge_f16_e64_dpp |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x01,0x89,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
 
 0x7e,0x02,0x89,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x01,0x13
-# GFX11: v_cmpx_nge_f16_e64_dpp -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x7e,0x02,0x89,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x01,0x13]
+# GFX11-REAL16: v_cmpx_nge_f16_e64_dpp -v1.l, |v2.l| row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x7e,0x02,0x89,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x01,0x13]
+# GFX11-FAKE16: v_cmpx_nge_f16_e64_dpp -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x7e,0x02,0x89,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x01,0x13]
 
 0x7e,0x83,0x89,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30
-# GFX11: v_cmpx_nge_f16_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0x7e,0x83,0x89,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30]
+# GFX11-REAL16: v_cmpx_nge_f16_e64_dpp -|v255.l|, -|v255.l| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0x7e,0x83,0x89,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30]
+# GFX11-FAKE16: v_cmpx_nge_f16_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0x7e,0x83,0x89,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30]
+
+0x7e,0x19,0x89,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01
+# GFX11-REAL16: v_cmpx_nge_f16_e64_dpp |v1.h|, -v2.h op_sel:[1,1] row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x19,0x89,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+# GFX11-FAKE16: v_cmpx_nge_f16_e64_dpp |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x01,0x89,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+
+0x7e,0x0a,0x89,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x01,0x13
+# GFX11-REAL16: v_cmpx_nge_f16_e64_dpp -v1.h, |v2.l| op_sel:[1,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x7e,0x0a,0x89,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x01,0x13]
+# GFX11-FAKE16: v_cmpx_nge_f16_e64_dpp -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x7e,0x02,0x89,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x01,0x13]
+
+0x7e,0x93,0x89,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30
+# GFX11-REAL16: v_cmpx_nge_f16_e64_dpp -|v255.l|, -|v255.h| op_sel:[0,1] clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0x7e,0x93,0x89,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30]
+# GFX11-FAKE16: v_cmpx_nge_f16_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0x7e,0x83,0x89,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30]
 
 0x7e,0x00,0x99,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff
 # GFX11: v_cmpx_nge_f32_e64_dpp v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x99,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
@@ -2301,46 +2509,72 @@
 # GFX11: v_cmpx_nge_f32_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0x7e,0x83,0x99,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30]
 
 0x7e,0x00,0x8b,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff
-# GFX11: v_cmpx_ngt_f16_e64_dpp v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8b,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+# GFX11-REAL16: v_cmpx_ngt_f16_e64_dpp v1.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8b,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+# GFX11-FAKE16: v_cmpx_ngt_f16_e64_dpp v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8b,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
 
 0x7e,0x00,0x8b,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff
-# GFX11: v_cmpx_ngt_f16_e64_dpp v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8b,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+# GFX11-REAL16: v_cmpx_ngt_f16_e64_dpp v1.l, v2.l quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8b,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+# GFX11-FAKE16: v_cmpx_ngt_f16_e64_dpp v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8b,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
 
 0x7e,0x00,0x8b,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff
-# GFX11: v_cmpx_ngt_f16_e64_dpp v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8b,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+# GFX11-REAL16: v_cmpx_ngt_f16_e64_dpp v1.l, v2.l row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8b,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_ngt_f16_e64_dpp v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8b,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
 
 0x7e,0x00,0x8b,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff
-# GFX11: v_cmpx_ngt_f16_e64_dpp v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8b,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+# GFX11-REAL16: v_cmpx_ngt_f16_e64_dpp v1.l, v2.l row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8b,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_ngt_f16_e64_dpp v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8b,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
 
 0x7e,0x00,0x8b,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff
-# GFX11: v_cmpx_ngt_f16_e64_dpp v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8b,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+# GFX11-REAL16: v_cmpx_ngt_f16_e64_dpp v1.l, v2.l row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8b,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_ngt_f16_e64_dpp v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8b,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
 
 0x7e,0x00,0x8b,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff
-# GFX11: v_cmpx_ngt_f16_e64_dpp v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8b,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+# GFX11-REAL16: v_cmpx_ngt_f16_e64_dpp v1.l, v2.l row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8b,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_ngt_f16_e64_dpp v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8b,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
 
 0x7e,0x00,0x8b,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff
-# GFX11: v_cmpx_ngt_f16_e64_dpp v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8b,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+# GFX11-REAL16: v_cmpx_ngt_f16_e64_dpp v1.l, v2.l row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8b,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_ngt_f16_e64_dpp v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8b,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
 
 0x7e,0x00,0x8b,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff
-# GFX11: v_cmpx_ngt_f16_e64_dpp v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8b,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+# GFX11-REAL16: v_cmpx_ngt_f16_e64_dpp v1.l, v2.l row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8b,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_ngt_f16_e64_dpp v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8b,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
 
 0x7e,0x00,0x8b,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff
-# GFX11: v_cmpx_ngt_f16_e64_dpp v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8b,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+# GFX11-REAL16: v_cmpx_ngt_f16_e64_dpp v1.l, v2.l row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8b,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_ngt_f16_e64_dpp v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8b,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
 
 0x7e,0x00,0x8b,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff
-# GFX11: v_cmpx_ngt_f16_e64_dpp v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8b,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+# GFX11-REAL16: v_cmpx_ngt_f16_e64_dpp v1.l, v2.l row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8b,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_ngt_f16_e64_dpp v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8b,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
 
 0x7e,0x00,0x8b,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff
-# GFX11: v_cmpx_ngt_f16_e64_dpp v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8b,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+# GFX11-REAL16: v_cmpx_ngt_f16_e64_dpp v1.l, v2.l row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8b,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_ngt_f16_e64_dpp v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8b,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 
 0x7e,0x01,0x8b,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01
-# GFX11: v_cmpx_ngt_f16_e64_dpp |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x01,0x8b,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+# GFX11-REAL16: v_cmpx_ngt_f16_e64_dpp |v1.l|, -v2.l row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x01,0x8b,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+# GFX11-FAKE16: v_cmpx_ngt_f16_e64_dpp |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x01,0x8b,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
 
 0x7e,0x02,0x8b,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x01,0x13
-# GFX11: v_cmpx_ngt_f16_e64_dpp -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x7e,0x02,0x8b,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x01,0x13]
+# GFX11-REAL16: v_cmpx_ngt_f16_e64_dpp -v1.l, |v2.l| row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x7e,0x02,0x8b,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x01,0x13]
+# GFX11-FAKE16: v_cmpx_ngt_f16_e64_dpp -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x7e,0x02,0x8b,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x01,0x13]
 
 0x7e,0x83,0x8b,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30
-# GFX11: v_cmpx_ngt_f16_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0x7e,0x83,0x8b,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30]
+# GFX11-REAL16: v_cmpx_ngt_f16_e64_dpp -|v255.l|, -|v255.l| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0x7e,0x83,0x8b,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30]
+# GFX11-FAKE16: v_cmpx_ngt_f16_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0x7e,0x83,0x8b,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30]
+
+0x7e,0x19,0x8b,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01
+# GFX11-REAL16: v_cmpx_ngt_f16_e64_dpp |v1.h|, -v2.h op_sel:[1,1] row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x19,0x8b,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+# GFX11-FAKE16: v_cmpx_ngt_f16_e64_dpp |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x01,0x8b,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+
+0x7e,0x0a,0x8b,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x01,0x13
+# GFX11-REAL16: v_cmpx_ngt_f16_e64_dpp -v1.h, |v2.l| op_sel:[1,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x7e,0x0a,0x8b,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x01,0x13]
+# GFX11-FAKE16: v_cmpx_ngt_f16_e64_dpp -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x7e,0x02,0x8b,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x01,0x13]
+
+0x7e,0x93,0x8b,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30
+# GFX11-REAL16: v_cmpx_ngt_f16_e64_dpp -|v255.l|, -|v255.h| op_sel:[0,1] clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0x7e,0x93,0x8b,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30]
+# GFX11-FAKE16: v_cmpx_ngt_f16_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0x7e,0x83,0x8b,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30]
 
 0x7e,0x00,0x9b,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff
 # GFX11: v_cmpx_ngt_f32_e64_dpp v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x9b,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
@@ -2385,46 +2619,72 @@
 # GFX11: v_cmpx_ngt_f32_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0x7e,0x83,0x9b,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30]
 
 0x7e,0x00,0x8c,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff
-# GFX11: v_cmpx_nle_f16_e64_dpp v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8c,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+# GFX11-REAL16: v_cmpx_nle_f16_e64_dpp v1.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8c,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+# GFX11-FAKE16: v_cmpx_nle_f16_e64_dpp v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8c,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
 
 0x7e,0x00,0x8c,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff
-# GFX11: v_cmpx_nle_f16_e64_dpp v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8c,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+# GFX11-REAL16: v_cmpx_nle_f16_e64_dpp v1.l, v2.l quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8c,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+# GFX11-FAKE16: v_cmpx_nle_f16_e64_dpp v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8c,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
 
 0x7e,0x00,0x8c,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff
-# GFX11: v_cmpx_nle_f16_e64_dpp v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8c,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+# GFX11-REAL16: v_cmpx_nle_f16_e64_dpp v1.l, v2.l row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8c,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_nle_f16_e64_dpp v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8c,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
 
 0x7e,0x00,0x8c,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff
-# GFX11: v_cmpx_nle_f16_e64_dpp v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8c,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+# GFX11-REAL16: v_cmpx_nle_f16_e64_dpp v1.l, v2.l row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8c,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_nle_f16_e64_dpp v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8c,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
 
 0x7e,0x00,0x8c,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff
-# GFX11: v_cmpx_nle_f16_e64_dpp v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8c,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+# GFX11-REAL16: v_cmpx_nle_f16_e64_dpp v1.l, v2.l row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8c,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_nle_f16_e64_dpp v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8c,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
 
 0x7e,0x00,0x8c,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff
-# GFX11: v_cmpx_nle_f16_e64_dpp v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8c,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+# GFX11-REAL16: v_cmpx_nle_f16_e64_dpp v1.l, v2.l row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8c,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_nle_f16_e64_dpp v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8c,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
 
 0x7e,0x00,0x8c,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff
-# GFX11: v_cmpx_nle_f16_e64_dpp v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8c,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+# GFX11-REAL16: v_cmpx_nle_f16_e64_dpp v1.l, v2.l row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8c,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_nle_f16_e64_dpp v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8c,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
 
 0x7e,0x00,0x8c,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff
-# GFX11: v_cmpx_nle_f16_e64_dpp v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8c,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+# GFX11-REAL16: v_cmpx_nle_f16_e64_dpp v1.l, v2.l row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8c,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_nle_f16_e64_dpp v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8c,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
 
 0x7e,0x00,0x8c,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff
-# GFX11: v_cmpx_nle_f16_e64_dpp v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8c,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+# GFX11-REAL16: v_cmpx_nle_f16_e64_dpp v1.l, v2.l row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8c,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_nle_f16_e64_dpp v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8c,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
 
 0x7e,0x00,0x8c,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff
-# GFX11: v_cmpx_nle_f16_e64_dpp v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8c,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+# GFX11-REAL16: v_cmpx_nle_f16_e64_dpp v1.l, v2.l row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8c,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_nle_f16_e64_dpp v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8c,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
 
 0x7e,0x00,0x8c,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff
-# GFX11: v_cmpx_nle_f16_e64_dpp v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8c,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+# GFX11-REAL16: v_cmpx_nle_f16_e64_dpp v1.l, v2.l row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8c,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_nle_f16_e64_dpp v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8c,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 
 0x7e,0x01,0x8c,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01
-# GFX11: v_cmpx_nle_f16_e64_dpp |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x01,0x8c,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+# GFX11-REAL16: v_cmpx_nle_f16_e64_dpp |v1.l|, -v2.l row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x01,0x8c,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+# GFX11-FAKE16: v_cmpx_nle_f16_e64_dpp |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x01,0x8c,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
 
 0x7e,0x02,0x8c,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x01,0x13
-# GFX11: v_cmpx_nle_f16_e64_dpp -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x7e,0x02,0x8c,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x01,0x13]
+# GFX11-REAL16: v_cmpx_nle_f16_e64_dpp -v1.l, |v2.l| row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x7e,0x02,0x8c,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x01,0x13]
+# GFX11-FAKE16: v_cmpx_nle_f16_e64_dpp -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x7e,0x02,0x8c,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x01,0x13]
 
 0x7e,0x83,0x8c,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30
-# GFX11: v_cmpx_nle_f16_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0x7e,0x83,0x8c,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30]
+# GFX11-REAL16: v_cmpx_nle_f16_e64_dpp -|v255.l|, -|v255.l| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0x7e,0x83,0x8c,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30]
+# GFX11-FAKE16: v_cmpx_nle_f16_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0x7e,0x83,0x8c,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30]
+
+0x7e,0x19,0x8c,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01
+# GFX11-REAL16: v_cmpx_nle_f16_e64_dpp |v1.h|, -v2.h op_sel:[1,1] row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x19,0x8c,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+# GFX11-FAKE16: v_cmpx_nle_f16_e64_dpp |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x01,0x8c,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+
+0x7e,0x0a,0x8c,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x01,0x13
+# GFX11-REAL16: v_cmpx_nle_f16_e64_dpp -v1.h, |v2.l| op_sel:[1,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x7e,0x0a,0x8c,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x01,0x13]
+# GFX11-FAKE16: v_cmpx_nle_f16_e64_dpp -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x7e,0x02,0x8c,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x01,0x13]
+
+0x7e,0x93,0x8c,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30
+# GFX11-REAL16: v_cmpx_nle_f16_e64_dpp -|v255.l|, -|v255.h| op_sel:[0,1] clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0x7e,0x93,0x8c,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30]
+# GFX11-FAKE16: v_cmpx_nle_f16_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0x7e,0x83,0x8c,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30]
 
 0x7e,0x00,0x9c,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff
 # GFX11: v_cmpx_nle_f32_e64_dpp v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x9c,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
@@ -2469,46 +2729,72 @@
 # GFX11: v_cmpx_nle_f32_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0x7e,0x83,0x9c,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30]
 
 0x7e,0x00,0x8a,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff
-# GFX11: v_cmpx_nlg_f16_e64_dpp v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8a,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+# GFX11-REAL16: v_cmpx_nlg_f16_e64_dpp v1.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8a,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+# GFX11-FAKE16: v_cmpx_nlg_f16_e64_dpp v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8a,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
 
 0x7e,0x00,0x8a,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff
-# GFX11: v_cmpx_nlg_f16_e64_dpp v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8a,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+# GFX11-REAL16: v_cmpx_nlg_f16_e64_dpp v1.l, v2.l quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8a,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+# GFX11-FAKE16: v_cmpx_nlg_f16_e64_dpp v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8a,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
 
 0x7e,0x00,0x8a,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff
-# GFX11: v_cmpx_nlg_f16_e64_dpp v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8a,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+# GFX11-REAL16: v_cmpx_nlg_f16_e64_dpp v1.l, v2.l row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8a,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_nlg_f16_e64_dpp v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8a,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
 
 0x7e,0x00,0x8a,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff
-# GFX11: v_cmpx_nlg_f16_e64_dpp v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8a,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+# GFX11-REAL16: v_cmpx_nlg_f16_e64_dpp v1.l, v2.l row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8a,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_nlg_f16_e64_dpp v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8a,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
 
 0x7e,0x00,0x8a,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff
-# GFX11: v_cmpx_nlg_f16_e64_dpp v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8a,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+# GFX11-REAL16: v_cmpx_nlg_f16_e64_dpp v1.l, v2.l row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8a,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_nlg_f16_e64_dpp v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8a,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
 
 0x7e,0x00,0x8a,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff
-# GFX11: v_cmpx_nlg_f16_e64_dpp v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8a,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+# GFX11-REAL16: v_cmpx_nlg_f16_e64_dpp v1.l, v2.l row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8a,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_nlg_f16_e64_dpp v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8a,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
 
 0x7e,0x00,0x8a,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff
-# GFX11: v_cmpx_nlg_f16_e64_dpp v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8a,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+# GFX11-REAL16: v_cmpx_nlg_f16_e64_dpp v1.l, v2.l row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8a,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_nlg_f16_e64_dpp v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8a,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
 
 0x7e,0x00,0x8a,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff
-# GFX11: v_cmpx_nlg_f16_e64_dpp v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8a,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+# GFX11-REAL16: v_cmpx_nlg_f16_e64_dpp v1.l, v2.l row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8a,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_nlg_f16_e64_dpp v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8a,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
 
 0x7e,0x00,0x8a,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff
-# GFX11: v_cmpx_nlg_f16_e64_dpp v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8a,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+# GFX11-REAL16: v_cmpx_nlg_f16_e64_dpp v1.l, v2.l row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8a,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_nlg_f16_e64_dpp v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8a,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
 
 0x7e,0x00,0x8a,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff
-# GFX11: v_cmpx_nlg_f16_e64_dpp v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8a,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+# GFX11-REAL16: v_cmpx_nlg_f16_e64_dpp v1.l, v2.l row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8a,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_nlg_f16_e64_dpp v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8a,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
 
 0x7e,0x00,0x8a,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff
-# GFX11: v_cmpx_nlg_f16_e64_dpp v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8a,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+# GFX11-REAL16: v_cmpx_nlg_f16_e64_dpp v1.l, v2.l row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8a,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_nlg_f16_e64_dpp v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8a,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 
 0x7e,0x01,0x8a,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01
-# GFX11: v_cmpx_nlg_f16_e64_dpp |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x01,0x8a,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+# GFX11-REAL16: v_cmpx_nlg_f16_e64_dpp |v1.l|, -v2.l row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x01,0x8a,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+# GFX11-FAKE16: v_cmpx_nlg_f16_e64_dpp |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x01,0x8a,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
 
 0x7e,0x02,0x8a,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x01,0x13
-# GFX11: v_cmpx_nlg_f16_e64_dpp -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x7e,0x02,0x8a,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x01,0x13]
+# GFX11-REAL16: v_cmpx_nlg_f16_e64_dpp -v1.l, |v2.l| row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x7e,0x02,0x8a,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x01,0x13]
+# GFX11-FAKE16: v_cmpx_nlg_f16_e64_dpp -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x7e,0x02,0x8a,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x01,0x13]
 
 0x7e,0x83,0x8a,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30
-# GFX11: v_cmpx_nlg_f16_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0x7e,0x83,0x8a,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30]
+# GFX11-REAL16: v_cmpx_nlg_f16_e64_dpp -|v255.l|, -|v255.l| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0x7e,0x83,0x8a,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30]
+# GFX11-FAKE16: v_cmpx_nlg_f16_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0x7e,0x83,0x8a,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30]
+
+0x7e,0x19,0x8a,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01
+# GFX11-REAL16: v_cmpx_nlg_f16_e64_dpp |v1.h|, -v2.h op_sel:[1,1] row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x19,0x8a,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+# GFX11-FAKE16: v_cmpx_nlg_f16_e64_dpp |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x01,0x8a,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+
+0x7e,0x0a,0x8a,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x01,0x13
+# GFX11-REAL16: v_cmpx_nlg_f16_e64_dpp -v1.h, |v2.l| op_sel:[1,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x7e,0x0a,0x8a,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x01,0x13]
+# GFX11-FAKE16: v_cmpx_nlg_f16_e64_dpp -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x7e,0x02,0x8a,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x01,0x13]
+
+0x7e,0x93,0x8a,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30
+# GFX11-REAL16: v_cmpx_nlg_f16_e64_dpp -|v255.l|, -|v255.h| op_sel:[0,1] clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0x7e,0x93,0x8a,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30]
+# GFX11-FAKE16: v_cmpx_nlg_f16_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0x7e,0x83,0x8a,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30]
 
 0x7e,0x00,0x9a,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff
 # GFX11: v_cmpx_nlg_f32_e64_dpp v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x9a,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
@@ -2553,46 +2839,72 @@
 # GFX11: v_cmpx_nlg_f32_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0x7e,0x83,0x9a,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30]
 
 0x7e,0x00,0x8e,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff
-# GFX11: v_cmpx_nlt_f16_e64_dpp v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8e,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+# GFX11-REAL16: v_cmpx_nlt_f16_e64_dpp v1.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8e,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+# GFX11-FAKE16: v_cmpx_nlt_f16_e64_dpp v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8e,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
 
 0x7e,0x00,0x8e,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff
-# GFX11: v_cmpx_nlt_f16_e64_dpp v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8e,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+# GFX11-REAL16: v_cmpx_nlt_f16_e64_dpp v1.l, v2.l quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8e,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+# GFX11-FAKE16: v_cmpx_nlt_f16_e64_dpp v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8e,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
 
 0x7e,0x00,0x8e,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff
-# GFX11: v_cmpx_nlt_f16_e64_dpp v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8e,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+# GFX11-REAL16: v_cmpx_nlt_f16_e64_dpp v1.l, v2.l row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8e,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_nlt_f16_e64_dpp v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8e,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
 
 0x7e,0x00,0x8e,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff
-# GFX11: v_cmpx_nlt_f16_e64_dpp v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8e,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+# GFX11-REAL16: v_cmpx_nlt_f16_e64_dpp v1.l, v2.l row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8e,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_nlt_f16_e64_dpp v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8e,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
 
 0x7e,0x00,0x8e,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff
-# GFX11: v_cmpx_nlt_f16_e64_dpp v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8e,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+# GFX11-REAL16: v_cmpx_nlt_f16_e64_dpp v1.l, v2.l row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8e,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_nlt_f16_e64_dpp v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8e,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
 
 0x7e,0x00,0x8e,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff
-# GFX11: v_cmpx_nlt_f16_e64_dpp v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8e,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+# GFX11-REAL16: v_cmpx_nlt_f16_e64_dpp v1.l, v2.l row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8e,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_nlt_f16_e64_dpp v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8e,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
 
 0x7e,0x00,0x8e,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff
-# GFX11: v_cmpx_nlt_f16_e64_dpp v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8e,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+# GFX11-REAL16: v_cmpx_nlt_f16_e64_dpp v1.l, v2.l row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8e,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_nlt_f16_e64_dpp v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8e,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
 
 0x7e,0x00,0x8e,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff
-# GFX11: v_cmpx_nlt_f16_e64_dpp v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8e,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+# GFX11-REAL16: v_cmpx_nlt_f16_e64_dpp v1.l, v2.l row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8e,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_nlt_f16_e64_dpp v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8e,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
 
 0x7e,0x00,0x8e,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff
-# GFX11: v_cmpx_nlt_f16_e64_dpp v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8e,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+# GFX11-REAL16: v_cmpx_nlt_f16_e64_dpp v1.l, v2.l row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8e,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_nlt_f16_e64_dpp v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8e,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
 
 0x7e,0x00,0x8e,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff
-# GFX11: v_cmpx_nlt_f16_e64_dpp v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8e,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+# GFX11-REAL16: v_cmpx_nlt_f16_e64_dpp v1.l, v2.l row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8e,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_nlt_f16_e64_dpp v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8e,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
 
 0x7e,0x00,0x8e,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff
-# GFX11: v_cmpx_nlt_f16_e64_dpp v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8e,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+# GFX11-REAL16: v_cmpx_nlt_f16_e64_dpp v1.l, v2.l row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8e,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_nlt_f16_e64_dpp v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8e,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 
 0x7e,0x01,0x8e,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01
-# GFX11: v_cmpx_nlt_f16_e64_dpp |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x01,0x8e,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+# GFX11-REAL16: v_cmpx_nlt_f16_e64_dpp |v1.l|, -v2.l row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x01,0x8e,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+# GFX11-FAKE16: v_cmpx_nlt_f16_e64_dpp |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x01,0x8e,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
 
 0x7e,0x02,0x8e,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x01,0x13
-# GFX11: v_cmpx_nlt_f16_e64_dpp -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x7e,0x02,0x8e,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x01,0x13]
+# GFX11-REAL16: v_cmpx_nlt_f16_e64_dpp -v1.l, |v2.l| row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x7e,0x02,0x8e,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x01,0x13]
+# GFX11-FAKE16: v_cmpx_nlt_f16_e64_dpp -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x7e,0x02,0x8e,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x01,0x13]
 
 0x7e,0x83,0x8e,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30
-# GFX11: v_cmpx_nlt_f16_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0x7e,0x83,0x8e,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30]
+# GFX11-REAL16: v_cmpx_nlt_f16_e64_dpp -|v255.l|, -|v255.l| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0x7e,0x83,0x8e,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30]
+# GFX11-FAKE16: v_cmpx_nlt_f16_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0x7e,0x83,0x8e,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30]
+
+0x7e,0x19,0x8e,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01
+# GFX11-REAL16: v_cmpx_nlt_f16_e64_dpp |v1.h|, -v2.h op_sel:[1,1] row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x19,0x8e,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+# GFX11-FAKE16: v_cmpx_nlt_f16_e64_dpp |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x01,0x8e,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+
+0x7e,0x0a,0x8e,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x01,0x13
+# GFX11-REAL16: v_cmpx_nlt_f16_e64_dpp -v1.h, |v2.l| op_sel:[1,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x7e,0x0a,0x8e,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x01,0x13]
+# GFX11-FAKE16: v_cmpx_nlt_f16_e64_dpp -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x7e,0x02,0x8e,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x01,0x13]
+
+0x7e,0x93,0x8e,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30
+# GFX11-REAL16: v_cmpx_nlt_f16_e64_dpp -|v255.l|, -|v255.h| op_sel:[0,1] clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0x7e,0x93,0x8e,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30]
+# GFX11-FAKE16: v_cmpx_nlt_f16_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0x7e,0x83,0x8e,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30]
 
 0x7e,0x00,0x9e,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff
 # GFX11: v_cmpx_nlt_f32_e64_dpp v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x9e,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
@@ -2637,46 +2949,72 @@
 # GFX11: v_cmpx_nlt_f32_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0x7e,0x83,0x9e,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30]
 
 0x7e,0x00,0x87,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff
-# GFX11: v_cmpx_o_f16_e64_dpp v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x87,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+# GFX11-REAL16: v_cmpx_o_f16_e64_dpp v1.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x87,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+# GFX11-FAKE16: v_cmpx_o_f16_e64_dpp v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x87,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
 
 0x7e,0x00,0x87,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff
-# GFX11: v_cmpx_o_f16_e64_dpp v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x87,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+# GFX11-REAL16: v_cmpx_o_f16_e64_dpp v1.l, v2.l quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x87,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+# GFX11-FAKE16: v_cmpx_o_f16_e64_dpp v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x87,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
 
 0x7e,0x00,0x87,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff
-# GFX11: v_cmpx_o_f16_e64_dpp v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x87,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+# GFX11-REAL16: v_cmpx_o_f16_e64_dpp v1.l, v2.l row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x87,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_o_f16_e64_dpp v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x87,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
 
 0x7e,0x00,0x87,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff
-# GFX11: v_cmpx_o_f16_e64_dpp v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x87,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+# GFX11-REAL16: v_cmpx_o_f16_e64_dpp v1.l, v2.l row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x87,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_o_f16_e64_dpp v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x87,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
 
 0x7e,0x00,0x87,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff
-# GFX11: v_cmpx_o_f16_e64_dpp v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x87,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+# GFX11-REAL16: v_cmpx_o_f16_e64_dpp v1.l, v2.l row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x87,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_o_f16_e64_dpp v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x87,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
 
 0x7e,0x00,0x87,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff
-# GFX11: v_cmpx_o_f16_e64_dpp v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x87,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+# GFX11-REAL16: v_cmpx_o_f16_e64_dpp v1.l, v2.l row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x87,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_o_f16_e64_dpp v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x87,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
 
 0x7e,0x00,0x87,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff
-# GFX11: v_cmpx_o_f16_e64_dpp v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x87,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+# GFX11-REAL16: v_cmpx_o_f16_e64_dpp v1.l, v2.l row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x87,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_o_f16_e64_dpp v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x87,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
 
 0x7e,0x00,0x87,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff
-# GFX11: v_cmpx_o_f16_e64_dpp v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x87,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+# GFX11-REAL16: v_cmpx_o_f16_e64_dpp v1.l, v2.l row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x87,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_o_f16_e64_dpp v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x87,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
 
 0x7e,0x00,0x87,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff
-# GFX11: v_cmpx_o_f16_e64_dpp v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x87,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+# GFX11-REAL16: v_cmpx_o_f16_e64_dpp v1.l, v2.l row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x87,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_o_f16_e64_dpp v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x87,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
 
 0x7e,0x00,0x87,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff
-# GFX11: v_cmpx_o_f16_e64_dpp v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x87,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+# GFX11-REAL16: v_cmpx_o_f16_e64_dpp v1.l, v2.l row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x87,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_o_f16_e64_dpp v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x87,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
 
 0x7e,0x00,0x87,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff
-# GFX11: v_cmpx_o_f16_e64_dpp v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x87,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+# GFX11-REAL16: v_cmpx_o_f16_e64_dpp v1.l, v2.l row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x87,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_o_f16_e64_dpp v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x87,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 
 0x7e,0x01,0x87,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01
-# GFX11: v_cmpx_o_f16_e64_dpp |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x01,0x87,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+# GFX11-REAL16: v_cmpx_o_f16_e64_dpp |v1.l|, -v2.l row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x01,0x87,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+# GFX11-FAKE16: v_cmpx_o_f16_e64_dpp |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x01,0x87,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
 
 0x7e,0x02,0x87,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x01,0x13
-# GFX11: v_cmpx_o_f16_e64_dpp -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x7e,0x02,0x87,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x01,0x13]
+# GFX11-REAL16: v_cmpx_o_f16_e64_dpp -v1.l, |v2.l| row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x7e,0x02,0x87,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x01,0x13]
+# GFX11-FAKE16: v_cmpx_o_f16_e64_dpp -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x7e,0x02,0x87,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x01,0x13]
 
 0x7e,0x83,0x87,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30
-# GFX11: v_cmpx_o_f16_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0x7e,0x83,0x87,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30]
+# GFX11-REAL16: v_cmpx_o_f16_e64_dpp -|v255.l|, -|v255.l| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0x7e,0x83,0x87,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30]
+# GFX11-FAKE16: v_cmpx_o_f16_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0x7e,0x83,0x87,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30]
+
+0x7e,0x19,0x87,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01
+# GFX11-REAL16: v_cmpx_o_f16_e64_dpp |v1.h|, -v2.h op_sel:[1,1] row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x19,0x87,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+# GFX11-FAKE16: v_cmpx_o_f16_e64_dpp |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x01,0x87,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+
+0x7e,0x0a,0x87,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x01,0x13
+# GFX11-REAL16: v_cmpx_o_f16_e64_dpp -v1.h, |v2.l| op_sel:[1,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x7e,0x0a,0x87,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x01,0x13]
+# GFX11-FAKE16: v_cmpx_o_f16_e64_dpp -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x7e,0x02,0x87,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x01,0x13]
+
+0x7e,0x93,0x87,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30
+# GFX11-REAL16: v_cmpx_o_f16_e64_dpp -|v255.l|, -|v255.h| op_sel:[0,1] clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0x7e,0x93,0x87,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30]
+# GFX11-FAKE16: v_cmpx_o_f16_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0x7e,0x83,0x87,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30]
 
 0x7e,0x00,0x97,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff
 # GFX11: v_cmpx_o_f32_e64_dpp v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x97,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
@@ -2721,46 +3059,72 @@
 # GFX11: v_cmpx_o_f32_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0x7e,0x83,0x97,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30]
 
 0x7e,0x00,0x8f,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff
-# GFX11: v_cmpx_t_f16_e64_dpp v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8f,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+# GFX11-REAL16: v_cmpx_t_f16_e64_dpp v1.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8f,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+# GFX11-FAKE16: v_cmpx_t_f16_e64_dpp v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8f,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
 
 0x7e,0x00,0x8f,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff
-# GFX11: v_cmpx_t_f16_e64_dpp v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8f,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+# GFX11-REAL16: v_cmpx_t_f16_e64_dpp v1.l, v2.l quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8f,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+# GFX11-FAKE16: v_cmpx_t_f16_e64_dpp v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8f,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
 
 0x7e,0x00,0x8f,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff
-# GFX11: v_cmpx_t_f16_e64_dpp v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8f,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+# GFX11-REAL16: v_cmpx_t_f16_e64_dpp v1.l, v2.l row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8f,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_t_f16_e64_dpp v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8f,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
 
 0x7e,0x00,0x8f,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff
-# GFX11: v_cmpx_t_f16_e64_dpp v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8f,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+# GFX11-REAL16: v_cmpx_t_f16_e64_dpp v1.l, v2.l row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8f,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_t_f16_e64_dpp v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8f,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
 
 0x7e,0x00,0x8f,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff
-# GFX11: v_cmpx_t_f16_e64_dpp v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8f,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+# GFX11-REAL16: v_cmpx_t_f16_e64_dpp v1.l, v2.l row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8f,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_t_f16_e64_dpp v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8f,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
 
 0x7e,0x00,0x8f,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff
-# GFX11: v_cmpx_t_f16_e64_dpp v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8f,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+# GFX11-REAL16: v_cmpx_t_f16_e64_dpp v1.l, v2.l row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8f,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_t_f16_e64_dpp v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8f,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
 
 0x7e,0x00,0x8f,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff
-# GFX11: v_cmpx_t_f16_e64_dpp v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8f,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+# GFX11-REAL16: v_cmpx_t_f16_e64_dpp v1.l, v2.l row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8f,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_t_f16_e64_dpp v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8f,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
 
 0x7e,0x00,0x8f,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff
-# GFX11: v_cmpx_t_f16_e64_dpp v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8f,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+# GFX11-REAL16: v_cmpx_t_f16_e64_dpp v1.l, v2.l row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8f,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_t_f16_e64_dpp v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8f,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
 
 0x7e,0x00,0x8f,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff
-# GFX11: v_cmpx_t_f16_e64_dpp v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8f,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+# GFX11-REAL16: v_cmpx_t_f16_e64_dpp v1.l, v2.l row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8f,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_t_f16_e64_dpp v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8f,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
 
 0x7e,0x00,0x8f,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff
-# GFX11: v_cmpx_t_f16_e64_dpp v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8f,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+# GFX11-REAL16: v_cmpx_t_f16_e64_dpp v1.l, v2.l row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8f,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_t_f16_e64_dpp v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8f,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
 
 0x7e,0x00,0x8f,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff
-# GFX11: v_cmpx_t_f16_e64_dpp v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8f,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+# GFX11-REAL16: v_cmpx_t_f16_e64_dpp v1.l, v2.l row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8f,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_t_f16_e64_dpp v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8f,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 
 0x7e,0x01,0x8f,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01
-# GFX11: v_cmpx_t_f16_e64_dpp |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x01,0x8f,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+# GFX11-REAL16: v_cmpx_t_f16_e64_dpp |v1.l|, -v2.l row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x01,0x8f,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+# GFX11-FAKE16: v_cmpx_t_f16_e64_dpp |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x01,0x8f,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
 
 0x7e,0x02,0x8f,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x01,0x13
-# GFX11: v_cmpx_t_f16_e64_dpp -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x7e,0x02,0x8f,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x01,0x13]
+# GFX11-REAL16: v_cmpx_t_f16_e64_dpp -v1.l, |v2.l| row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x7e,0x02,0x8f,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x01,0x13]
+# GFX11-FAKE16: v_cmpx_t_f16_e64_dpp -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x7e,0x02,0x8f,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x01,0x13]
 
 0x7e,0x83,0x8f,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30
-# GFX11: v_cmpx_t_f16_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0x7e,0x83,0x8f,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30]
+# GFX11-REAL16: v_cmpx_t_f16_e64_dpp -|v255.l|, -|v255.l| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0x7e,0x83,0x8f,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30]
+# GFX11-FAKE16: v_cmpx_t_f16_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0x7e,0x83,0x8f,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30]
+
+0x7e,0x19,0x8f,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01
+# GFX11-REAL16: v_cmpx_t_f16_e64_dpp |v1.h|, -v2.h op_sel:[1,1] row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x19,0x8f,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+# GFX11-FAKE16: v_cmpx_t_f16_e64_dpp |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x01,0x8f,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+
+0x7e,0x0a,0x8f,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x01,0x13
+# GFX11-REAL16: v_cmpx_t_f16_e64_dpp -v1.h, |v2.l| op_sel:[1,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x7e,0x0a,0x8f,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x01,0x13]
+# GFX11-FAKE16: v_cmpx_t_f16_e64_dpp -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x7e,0x02,0x8f,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x01,0x13]
+
+0x7e,0x93,0x8f,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30
+# GFX11-REAL16: v_cmpx_t_f16_e64_dpp -|v255.l|, -|v255.h| op_sel:[0,1] clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0x7e,0x93,0x8f,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30]
+# GFX11-FAKE16: v_cmpx_t_f16_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0x7e,0x83,0x8f,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30]
 
 0x7e,0x00,0x9f,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff
 # GFX11: v_cmpx_t_f32_e64_dpp v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x9f,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
@@ -2889,46 +3253,72 @@
 # GFX11: v_cmpx_t_u32_e64_dpp v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0x7e,0x00,0xcf,0xd4,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 
 0x7e,0x00,0x88,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff
-# GFX11: v_cmpx_u_f16_e64_dpp v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x88,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+# GFX11-REAL16: v_cmpx_u_f16_e64_dpp v1.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x88,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+# GFX11-FAKE16: v_cmpx_u_f16_e64_dpp v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x88,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
 
 0x7e,0x00,0x88,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff
-# GFX11: v_cmpx_u_f16_e64_dpp v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x88,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+# GFX11-REAL16: v_cmpx_u_f16_e64_dpp v1.l, v2.l quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x88,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+# GFX11-FAKE16: v_cmpx_u_f16_e64_dpp v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x88,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
 
 0x7e,0x00,0x88,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff
-# GFX11: v_cmpx_u_f16_e64_dpp v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x88,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+# GFX11-REAL16: v_cmpx_u_f16_e64_dpp v1.l, v2.l row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x88,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_u_f16_e64_dpp v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x88,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
 
 0x7e,0x00,0x88,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff
-# GFX11: v_cmpx_u_f16_e64_dpp v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x88,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+# GFX11-REAL16: v_cmpx_u_f16_e64_dpp v1.l, v2.l row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x88,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_u_f16_e64_dpp v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x88,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
 
 0x7e,0x00,0x88,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff
-# GFX11: v_cmpx_u_f16_e64_dpp v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x88,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+# GFX11-REAL16: v_cmpx_u_f16_e64_dpp v1.l, v2.l row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x88,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_u_f16_e64_dpp v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x88,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
 
 0x7e,0x00,0x88,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff
-# GFX11: v_cmpx_u_f16_e64_dpp v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x88,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+# GFX11-REAL16: v_cmpx_u_f16_e64_dpp v1.l, v2.l row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x88,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_u_f16_e64_dpp v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x88,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
 
 0x7e,0x00,0x88,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff
-# GFX11: v_cmpx_u_f16_e64_dpp v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x88,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+# GFX11-REAL16: v_cmpx_u_f16_e64_dpp v1.l, v2.l row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x88,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_u_f16_e64_dpp v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x88,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
 
 0x7e,0x00,0x88,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff
-# GFX11: v_cmpx_u_f16_e64_dpp v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x88,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+# GFX11-REAL16: v_cmpx_u_f16_e64_dpp v1.l, v2.l row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x88,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_u_f16_e64_dpp v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x88,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
 
 0x7e,0x00,0x88,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff
-# GFX11: v_cmpx_u_f16_e64_dpp v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x88,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+# GFX11-REAL16: v_cmpx_u_f16_e64_dpp v1.l, v2.l row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x88,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_u_f16_e64_dpp v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x88,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
 
 0x7e,0x00,0x88,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff
-# GFX11: v_cmpx_u_f16_e64_dpp v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x88,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+# GFX11-REAL16: v_cmpx_u_f16_e64_dpp v1.l, v2.l row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x88,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_u_f16_e64_dpp v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x88,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
 
 0x7e,0x00,0x88,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff
-# GFX11: v_cmpx_u_f16_e64_dpp v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x88,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+# GFX11-REAL16: v_cmpx_u_f16_e64_dpp v1.l, v2.l row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x88,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_u_f16_e64_dpp v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x88,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 
 0x7e,0x01,0x88,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01
-# GFX11: v_cmpx_u_f16_e64_dpp |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x01,0x88,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+# GFX11-REAL16: v_cmpx_u_f16_e64_dpp |v1.l|, -v2.l row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x01,0x88,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+# GFX11-FAKE16: v_cmpx_u_f16_e64_dpp |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x01,0x88,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
 
 0x7e,0x02,0x88,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x01,0x13
-# GFX11: v_cmpx_u_f16_e64_dpp -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x7e,0x02,0x88,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x01,0x13]
+# GFX11-REAL16: v_cmpx_u_f16_e64_dpp -v1.l, |v2.l| row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x7e,0x02,0x88,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x01,0x13]
+# GFX11-FAKE16: v_cmpx_u_f16_e64_dpp -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x7e,0x02,0x88,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x01,0x13]
 
 0x7e,0x83,0x88,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30
-# GFX11: v_cmpx_u_f16_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0x7e,0x83,0x88,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30]
+# GFX11-REAL16: v_cmpx_u_f16_e64_dpp -|v255.l|, -|v255.l| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0x7e,0x83,0x88,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30]
+# GFX11-FAKE16: v_cmpx_u_f16_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0x7e,0x83,0x88,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30]
+
+0x7e,0x19,0x88,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01
+# GFX11-REAL16: v_cmpx_u_f16_e64_dpp |v1.h|, -v2.h op_sel:[1,1] row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x19,0x88,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+# GFX11-FAKE16: v_cmpx_u_f16_e64_dpp |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x01,0x88,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+
+0x7e,0x0a,0x88,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x01,0x13
+# GFX11-REAL16: v_cmpx_u_f16_e64_dpp -v1.h, |v2.l| op_sel:[1,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x7e,0x0a,0x88,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x01,0x13]
+# GFX11-FAKE16: v_cmpx_u_f16_e64_dpp -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x7e,0x02,0x88,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x01,0x13]
+
+0x7e,0x93,0x88,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30
+# GFX11-REAL16: v_cmpx_u_f16_e64_dpp -|v255.l|, -|v255.h| op_sel:[0,1] clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0x7e,0x93,0x88,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30]
+# GFX11-FAKE16: v_cmpx_u_f16_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0x7e,0x83,0x88,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30]
 
 0x7e,0x00,0x98,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff
 # GFX11: v_cmpx_u_f32_e64_dpp v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x98,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
diff --git a/llvm/test/MC/Disassembler/AMDGPU/gfx11_dasm_vop3_dpp8.txt b/llvm/test/MC/Disassembler/AMDGPU/gfx11_dasm_vop3_dpp8.txt
index 1e74b5aec0cf3..f67eb32385407 100644
--- a/llvm/test/MC/Disassembler/AMDGPU/gfx11_dasm_vop3_dpp8.txt
+++ b/llvm/test/MC/Disassembler/AMDGPU/gfx11_dasm_vop3_dpp8.txt
@@ -135,10 +135,16 @@
 # GFX11: v_alignbit_b32_e64_dpp v255, v255, v255, src_scc dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x00,0x16,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00]
 
 0x05,0x00,0x17,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05
-# GFX11: v_alignbyte_b32_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x17,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
+# W32-REAL16: v_alignbyte_b32_e64_dpp v5, v1, v2, v3.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x17,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
+# W32-FAKE16: v_alignbyte_b32_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x17,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
+# W64-REAL16: v_alignbyte_b32_e64_dpp v5, v1, v2, v3.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x17,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
+# W64-FAKE16: v_alignbyte_b32_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x17,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 
 0x05,0x00,0x17,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05
-# GFX11: v_alignbyte_b32_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x17,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
+# W32-REAL16: v_alignbyte_b32_e64_dpp v5, v1, v2, v255.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x17,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
+# W32-FAKE16: v_alignbyte_b32_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x17,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
+# W64-REAL16: v_alignbyte_b32_e64_dpp v5, v1, v2, v255.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x17,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
+# W64-FAKE16: v_alignbyte_b32_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x17,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 
 0x05,0x00,0x17,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05
 # GFX11: v_alignbyte_b32_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x17,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
@@ -167,6 +173,12 @@
 0xff,0x00,0x17,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00
 # GFX11: v_alignbyte_b32_e64_dpp v255, v255, v255, src_scc dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x00,0x17,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00]
 
+0x05,0x20,0x17,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05
+# W32-REAL16: v_alignbyte_b32_e64_dpp v5, v1, v2, v255.h op_sel:[0,0,1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x20,0x17,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
+# W32-FAKE16: v_alignbyte_b32_e64_dpp v5, v1, v2, v255 op_sel:[0,0,1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x20,0x17,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
+# W64-REAL16: v_alignbyte_b32_e64_dpp v5, v1, v2, v255.h op_sel:[0,0,1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x20,0x17,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
+# W64-FAKE16: v_alignbyte_b32_e64_dpp v5, v1, v2, v255 op_sel:[0,0,1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x20,0x17,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
+
 0x05,0x00,0x62,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 # W32-REAL16: v_and_b16_e64_dpp v5.l, v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x62,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W32-FAKE16: v_and_b16_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x62,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
@@ -2948,46 +2960,88 @@
 # W64-FAKE16: v_mad_i16_e64_dpp v255, v255, v255, src_scc op_sel:[0,0,0,1] clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0xc0,0x53,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00]
 
 0x05,0x00,0x5a,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05
-# GFX11: v_mad_i32_i16_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x5a,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
+# W32-REAL16: v_mad_i32_i16_e64_dpp v5, v1.l, v2.l, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x5a,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
+# W32-FAKE16: v_mad_i32_i16_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x5a,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
+# W64-REAL16: v_mad_i32_i16_e64_dpp v5, v1.l, v2.l, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x5a,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
+# W64-FAKE16: v_mad_i32_i16_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x5a,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 
 0x05,0x00,0x5a,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05
-# GFX11: v_mad_i32_i16_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x5a,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
+# W32-REAL16: v_mad_i32_i16_e64_dpp v5, v1.l, v2.l, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x5a,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
+# W32-FAKE16: v_mad_i32_i16_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x5a,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
+# W64-REAL16: v_mad_i32_i16_e64_dpp v5, v1.l, v2.l, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x5a,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
+# W64-FAKE16: v_mad_i32_i16_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x5a,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 
 0x05,0x00,0x5a,0xd6,0xe9,0x04,0x0e,0x00,0x01,0x77,0x39,0x05
-# GFX11: v_mad_i32_i16_e64_dpp v5, v1, v2, s3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x5a,0xd6,0xe9,0x04,0x0e,0x00,0x01,0x77,0x39,0x05]
+# W32-REAL16: v_mad_i32_i16_e64_dpp v5, v1.l, v2.l, s3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x5a,0xd6,0xe9,0x04,0x0e,0x00,0x01,0x77,0x39,0x05]
+# W32-FAKE16: v_mad_i32_i16_e64_dpp v5, v1, v2, s3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x5a,0xd6,0xe9,0x04,0x0e,0x00,0x01,0x77,0x39,0x05]
+# W64-REAL16: v_mad_i32_i16_e64_dpp v5, v1.l, v2.l, s3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x5a,0xd6,0xe9,0x04,0x0e,0x00,0x01,0x77,0x39,0x05]
+# W64-FAKE16: v_mad_i32_i16_e64_dpp v5, v1, v2, s3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x5a,0xd6,0xe9,0x04,0x0e,0x00,0x01,0x77,0x39,0x05]
 
 0x05,0x00,0x5a,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05
-# GFX11: v_mad_i32_i16_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x5a,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
+# W32-REAL16: v_mad_i32_i16_e64_dpp v5, v1.l, v2.l, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x5a,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
+# W32-FAKE16: v_mad_i32_i16_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x5a,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
+# W64-REAL16: v_mad_i32_i16_e64_dpp v5, v1.l, v2.l, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x5a,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
+# W64-FAKE16: v_mad_i32_i16_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x5a,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 
 0x05,0x00,0x5a,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05
-# GFX11: v_mad_i32_i16_e64_dpp v5, v1, v2, ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x5a,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05]
+# W32-REAL16: v_mad_i32_i16_e64_dpp v5, v1.l, v2.l, ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x5a,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05]
+# W32-FAKE16: v_mad_i32_i16_e64_dpp v5, v1, v2, ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x5a,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05]
+# W64-REAL16: v_mad_i32_i16_e64_dpp v5, v1.l, v2.l, ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x5a,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05]
+# W64-FAKE16: v_mad_i32_i16_e64_dpp v5, v1, v2, ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x5a,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05]
 
 0x05,0x00,0x5a,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05
-# GFX11: v_mad_i32_i16_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x5a,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
+# W32-REAL16: v_mad_i32_i16_e64_dpp v5, v1.l, v2.l, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x5a,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
+# W32-FAKE16: v_mad_i32_i16_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x5a,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
+# W64-REAL16: v_mad_i32_i16_e64_dpp v5, v1.l, v2.l, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x5a,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
+# W64-FAKE16: v_mad_i32_i16_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x5a,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 
 0x05,0x00,0x5a,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05
-# GFX11: v_mad_i32_i16_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x5a,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
+# W32-REAL16: v_mad_i32_i16_e64_dpp v5, v1.l, v2.l, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x5a,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
+# W32-FAKE16: v_mad_i32_i16_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x5a,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
+# W64-REAL16: v_mad_i32_i16_e64_dpp v5, v1.l, v2.l, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x5a,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
+# W64-FAKE16: v_mad_i32_i16_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x5a,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 
 0x05,0x00,0x5a,0xd6,0xe9,0x04,0xf6,0x01,0x01,0x77,0x39,0x05
-# GFX11: v_mad_i32_i16_e64_dpp v5, v1, v2, m0 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x5a,0xd6,0xe9,0x04,0xf6,0x01,0x01,0x77,0x39,0x05]
+# W32-REAL16: v_mad_i32_i16_e64_dpp v5, v1.l, v2.l, m0 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x5a,0xd6,0xe9,0x04,0xf6,0x01,0x01,0x77,0x39,0x05]
+# W32-FAKE16: v_mad_i32_i16_e64_dpp v5, v1, v2, m0 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x5a,0xd6,0xe9,0x04,0xf6,0x01,0x01,0x77,0x39,0x05]
+# W64-REAL16: v_mad_i32_i16_e64_dpp v5, v1.l, v2.l, m0 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x5a,0xd6,0xe9,0x04,0xf6,0x01,0x01,0x77,0x39,0x05]
+# W64-FAKE16: v_mad_i32_i16_e64_dpp v5, v1, v2, m0 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x5a,0xd6,0xe9,0x04,0xf6,0x01,0x01,0x77,0x39,0x05]
 
 0x05,0x00,0x5a,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05
-# GFX11: v_mad_i32_i16_e64_dpp v5, v1, v2, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x5a,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05]
+# W32-REAL16: v_mad_i32_i16_e64_dpp v5, v1.l, v2.l, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x5a,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05]
+# W32-FAKE16: v_mad_i32_i16_e64_dpp v5, v1, v2, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x5a,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05]
+# W64-REAL16: v_mad_i32_i16_e64_dpp v5, v1.l, v2.l, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x5a,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05]
+# W64-FAKE16: v_mad_i32_i16_e64_dpp v5, v1, v2, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x5a,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05]
 
 0x05,0x00,0x5a,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05
-# GFX11: v_mad_i32_i16_e64_dpp v5, v1, v2, exec_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x5a,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05]
+# W32-REAL16: v_mad_i32_i16_e64_dpp v5, v1.l, v2.l, exec_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x5a,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05]
+# W32-FAKE16: v_mad_i32_i16_e64_dpp v5, v1, v2, exec_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x5a,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05]
+# W64-REAL16: v_mad_i32_i16_e64_dpp v5, v1.l, v2.l, exec_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x5a,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05]
+# W64-FAKE16: v_mad_i32_i16_e64_dpp v5, v1, v2, exec_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x5a,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05]
 
 0x05,0x00,0x5a,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05
-# GFX11: v_mad_i32_i16_e64_dpp v5, v1, v2, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x5a,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05]
+# W32-REAL16: v_mad_i32_i16_e64_dpp v5, v1.l, v2.l, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x5a,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05]
+# W32-FAKE16: v_mad_i32_i16_e64_dpp v5, v1, v2, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x5a,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05]
+# W64-REAL16: v_mad_i32_i16_e64_dpp v5, v1.l, v2.l, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x5a,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05]
+# W64-FAKE16: v_mad_i32_i16_e64_dpp v5, v1, v2, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x5a,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05]
 
 0x05,0x00,0x5a,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05
-# GFX11: v_mad_i32_i16_e64_dpp v5, v1, v2, -1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x5a,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05]
+# W32-REAL16: v_mad_i32_i16_e64_dpp v5, v1.l, v2.l, -1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x5a,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05]
+# W32-FAKE16: v_mad_i32_i16_e64_dpp v5, v1, v2, -1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x5a,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05]
+# W64-REAL16: v_mad_i32_i16_e64_dpp v5, v1.l, v2.l, -1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x5a,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05]
+# W64-FAKE16: v_mad_i32_i16_e64_dpp v5, v1, v2, -1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x5a,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05]
 
 0x05,0x08,0x5a,0xd6,0xe9,0x04,0xc2,0x03,0x01,0x77,0x39,0x05
-# GFX11: v_mad_i32_i16_e64_dpp v5, v1, v2, 0.5 op_sel:[1,0,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x08,0x5a,0xd6,0xe9,0x04,0xc2,0x03,0x01,0x77,0x39,0x05]
+# W32-REAL16: v_mad_i32_i16_e64_dpp v5, v1.h, v2.l, 0.5 op_sel:[1,0,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x08,0x5a,0xd6,0xe9,0x04,0xc2,0x03,0x01,0x77,0x39,0x05]
+# W32-FAKE16: v_mad_i32_i16_e64_dpp v5, v1, v2, 0.5 op_sel:[1,0,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x08,0x5a,0xd6,0xe9,0x04,0xc2,0x03,0x01,0x77,0x39,0x05]
+# W64-REAL16: v_mad_i32_i16_e64_dpp v5, v1.h, v2.l, 0.5 op_sel:[1,0,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x08,0x5a,0xd6,0xe9,0x04,0xc2,0x03,0x01,0x77,0x39,0x05]
+# W64-FAKE16: v_mad_i32_i16_e64_dpp v5, v1, v2, 0.5 op_sel:[1,0,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x08,0x5a,0xd6,0xe9,0x04,0xc2,0x03,0x01,0x77,0x39,0x05]
 
 0xff,0x90,0x5a,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00
-# GFX11: v_mad_i32_i16_e64_dpp v255, v255, v255, src_scc op_sel:[0,1,0,0] clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x90,0x5a,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00]
+# W32-REAL16: v_mad_i32_i16_e64_dpp v255, v255.l, v255.h, src_scc op_sel:[0,1,0,0] clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x90,0x5a,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00]
+# W32-FAKE16: v_mad_i32_i16_e64_dpp v255, v255, v255, src_scc op_sel:[0,1,0,0] clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x90,0x5a,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00]
+# W64-REAL16: v_mad_i32_i16_e64_dpp v255, v255.l, v255.h, src_scc op_sel:[0,1,0,0] clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x90,0x5a,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00]
+# W64-FAKE16: v_mad_i32_i16_e64_dpp v255, v255, v255, src_scc op_sel:[0,1,0,0] clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x90,0x5a,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00]
 
 0x05,0x78,0x41,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05
 # W32-REAL16: v_mad_u16_e64_dpp v5.h, v1.h, v2.h, v3.h op_sel:[1,1,1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x78,0x41,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
@@ -3098,46 +3152,88 @@
 # W64-FAKE16: v_mad_u16_e64_dpp v255, v255, v255, src_scc op_sel:[0,0,0,1] clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0xc0,0x41,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00]
 
 0x05,0x00,0x59,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05
-# GFX11: v_mad_u32_u16_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x59,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
+# W32-REAL16: v_mad_u32_u16_e64_dpp v5, v1.l, v2.l, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x59,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
+# W32-FAKE16: v_mad_u32_u16_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x59,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
+# W64-REAL16: v_mad_u32_u16_e64_dpp v5, v1.l, v2.l, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x59,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
+# W64-FAKE16: v_mad_u32_u16_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x59,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 
 0x05,0x00,0x59,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05
-# GFX11: v_mad_u32_u16_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x59,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
+# W32-REAL16: v_mad_u32_u16_e64_dpp v5, v1.l, v2.l, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x59,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
+# W32-FAKE16: v_mad_u32_u16_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x59,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
+# W64-REAL16: v_mad_u32_u16_e64_dpp v5, v1.l, v2.l, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x59,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
+# W64-FAKE16: v_mad_u32_u16_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x59,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 
 0x05,0x00,0x59,0xd6,0xe9,0x04,0x0e,0x00,0x01,0x77,0x39,0x05
-# GFX11: v_mad_u32_u16_e64_dpp v5, v1, v2, s3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x59,0xd6,0xe9,0x04,0x0e,0x00,0x01,0x77,0x39,0x05]
+# W32-REAL16: v_mad_u32_u16_e64_dpp v5, v1.l, v2.l, s3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x59,0xd6,0xe9,0x04,0x0e,0x00,0x01,0x77,0x39,0x05]
+# W32-FAKE16: v_mad_u32_u16_e64_dpp v5, v1, v2, s3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x59,0xd6,0xe9,0x04,0x0e,0x00,0x01,0x77,0x39,0x05]
+# W64-REAL16: v_mad_u32_u16_e64_dpp v5, v1.l, v2.l, s3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x59,0xd6,0xe9,0x04,0x0e,0x00,0x01,0x77,0x39,0x05]
+# W64-FAKE16: v_mad_u32_u16_e64_dpp v5, v1, v2, s3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x59,0xd6,0xe9,0x04,0x0e,0x00,0x01,0x77,0x39,0x05]
 
 0x05,0x00,0x59,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05
-# GFX11: v_mad_u32_u16_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x59,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
+# W32-REAL16: v_mad_u32_u16_e64_dpp v5, v1.l, v2.l, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x59,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
+# W32-FAKE16: v_mad_u32_u16_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x59,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
+# W64-REAL16: v_mad_u32_u16_e64_dpp v5, v1.l, v2.l, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x59,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
+# W64-FAKE16: v_mad_u32_u16_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x59,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 
 0x05,0x00,0x59,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05
-# GFX11: v_mad_u32_u16_e64_dpp v5, v1, v2, ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x59,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05]
+# W32-REAL16: v_mad_u32_u16_e64_dpp v5, v1.l, v2.l, ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x59,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05]
+# W32-FAKE16: v_mad_u32_u16_e64_dpp v5, v1, v2, ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x59,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05]
+# W64-REAL16: v_mad_u32_u16_e64_dpp v5, v1.l, v2.l, ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x59,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05]
+# W64-FAKE16: v_mad_u32_u16_e64_dpp v5, v1, v2, ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x59,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05]
 
 0x05,0x00,0x59,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05
-# GFX11: v_mad_u32_u16_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x59,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
+# W32-REAL16: v_mad_u32_u16_e64_dpp v5, v1.l, v2.l, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x59,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
+# W32-FAKE16: v_mad_u32_u16_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x59,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
+# W64-REAL16: v_mad_u32_u16_e64_dpp v5, v1.l, v2.l, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x59,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
+# W64-FAKE16: v_mad_u32_u16_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x59,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 
 0x05,0x00,0x59,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05
-# GFX11: v_mad_u32_u16_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x59,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
+# W32-REAL16: v_mad_u32_u16_e64_dpp v5, v1.l, v2.l, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x59,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
+# W32-FAKE16: v_mad_u32_u16_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x59,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
+# W64-REAL16: v_mad_u32_u16_e64_dpp v5, v1.l, v2.l, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x59,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
+# W64-FAKE16: v_mad_u32_u16_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x59,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 
 0x05,0x00,0x59,0xd6,0xe9,0x04,0xf6,0x01,0x01,0x77,0x39,0x05
-# GFX11: v_mad_u32_u16_e64_dpp v5, v1, v2, m0 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x59,0xd6,0xe9,0x04,0xf6,0x01,0x01,0x77,0x39,0x05]
+# W32-REAL16: v_mad_u32_u16_e64_dpp v5, v1.l, v2.l, m0 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x59,0xd6,0xe9,0x04,0xf6,0x01,0x01,0x77,0x39,0x05]
+# W32-FAKE16: v_mad_u32_u16_e64_dpp v5, v1, v2, m0 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x59,0xd6,0xe9,0x04,0xf6,0x01,0x01,0x77,0x39,0x05]
+# W64-REAL16: v_mad_u32_u16_e64_dpp v5, v1.l, v2.l, m0 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x59,0xd6,0xe9,0x04,0xf6,0x01,0x01,0x77,0x39,0x05]
+# W64-FAKE16: v_mad_u32_u16_e64_dpp v5, v1, v2, m0 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x59,0xd6,0xe9,0x04,0xf6,0x01,0x01,0x77,0x39,0x05]
 
 0x05,0x00,0x59,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05
-# GFX11: v_mad_u32_u16_e64_dpp v5, v1, v2, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x59,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05]
+# W32-REAL16: v_mad_u32_u16_e64_dpp v5, v1.l, v2.l, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x59,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05]
+# W32-FAKE16: v_mad_u32_u16_e64_dpp v5, v1, v2, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x59,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05]
+# W64-REAL16: v_mad_u32_u16_e64_dpp v5, v1.l, v2.l, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x59,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05]
+# W64-FAKE16: v_mad_u32_u16_e64_dpp v5, v1, v2, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x59,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05]
 
 0x05,0x00,0x59,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05
-# GFX11: v_mad_u32_u16_e64_dpp v5, v1, v2, exec_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x59,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05]
+# W32-REAL16: v_mad_u32_u16_e64_dpp v5, v1.l, v2.l, exec_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x59,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05]
+# W32-FAKE16: v_mad_u32_u16_e64_dpp v5, v1, v2, exec_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x59,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05]
+# W64-REAL16: v_mad_u32_u16_e64_dpp v5, v1.l, v2.l, exec_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x59,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05]
+# W64-FAKE16: v_mad_u32_u16_e64_dpp v5, v1, v2, exec_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x59,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05]
 
 0x05,0x00,0x59,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05
-# GFX11: v_mad_u32_u16_e64_dpp v5, v1, v2, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x59,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05]
+# W32-REAL16: v_mad_u32_u16_e64_dpp v5, v1.l, v2.l, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x59,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05]
+# W32-FAKE16: v_mad_u32_u16_e64_dpp v5, v1, v2, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x59,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05]
+# W64-REAL16: v_mad_u32_u16_e64_dpp v5, v1.l, v2.l, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x59,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05]
+# W64-FAKE16: v_mad_u32_u16_e64_dpp v5, v1, v2, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x59,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05]
 
 0x05,0x00,0x59,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05
-# GFX11: v_mad_u32_u16_e64_dpp v5, v1, v2, -1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x59,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05]
+# W32-REAL16: v_mad_u32_u16_e64_dpp v5, v1.l, v2.l, -1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x59,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05]
+# W32-FAKE16: v_mad_u32_u16_e64_dpp v5, v1, v2, -1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x59,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05]
+# W64-REAL16: v_mad_u32_u16_e64_dpp v5, v1.l, v2.l, -1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x59,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05]
+# W64-FAKE16: v_mad_u32_u16_e64_dpp v5, v1, v2, -1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x59,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05]
 
 0x05,0x08,0x59,0xd6,0xe9,0x04,0xc2,0x03,0x01,0x77,0x39,0x05
-# GFX11: v_mad_u32_u16_e64_dpp v5, v1, v2, 0.5 op_sel:[1,0,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x08,0x59,0xd6,0xe9,0x04,0xc2,0x03,0x01,0x77,0x39,0x05]
+# W32-REAL16: v_mad_u32_u16_e64_dpp v5, v1.h, v2.l, 0.5 op_sel:[1,0,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x08,0x59,0xd6,0xe9,0x04,0xc2,0x03,0x01,0x77,0x39,0x05]
+# W32-FAKE16: v_mad_u32_u16_e64_dpp v5, v1, v2, 0.5 op_sel:[1,0,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x08,0x59,0xd6,0xe9,0x04,0xc2,0x03,0x01,0x77,0x39,0x05]
+# W64-REAL16: v_mad_u32_u16_e64_dpp v5, v1.h, v2.l, 0.5 op_sel:[1,0,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x08,0x59,0xd6,0xe9,0x04,0xc2,0x03,0x01,0x77,0x39,0x05]
+# W64-FAKE16: v_mad_u32_u16_e64_dpp v5, v1, v2, 0.5 op_sel:[1,0,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x08,0x59,0xd6,0xe9,0x04,0xc2,0x03,0x01,0x77,0x39,0x05]
 
 0xff,0x90,0x59,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00
-# GFX11: v_mad_u32_u16_e64_dpp v255, v255, v255, src_scc op_sel:[0,1,0,0] clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x90,0x59,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00]
+# W32-REAL16: v_mad_u32_u16_e64_dpp v255, v255.l, v255.h, src_scc op_sel:[0,1,0,0] clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x90,0x59,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00]
+# W32-FAKE16: v_mad_u32_u16_e64_dpp v255, v255, v255, src_scc op_sel:[0,1,0,0] clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x90,0x59,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00]
+# W64-REAL16: v_mad_u32_u16_e64_dpp v255, v255.l, v255.h, src_scc op_sel:[0,1,0,0] clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x90,0x59,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00]
+# W64-FAKE16: v_mad_u32_u16_e64_dpp v255, v255, v255, src_scc op_sel:[0,1,0,0] clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x90,0x59,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00]
 
 0x05,0x78,0x4c,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05
 # W32-REAL16: v_max3_f16_e64_dpp v5.h, v1.h, v2.h, v3.h op_sel:[1,1,1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x78,0x4c,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
diff --git a/llvm/test/MC/Disassembler/AMDGPU/gfx11_dasm_vop3_dpp8_from_vopcx.txt b/llvm/test/MC/Disassembler/AMDGPU/gfx11_dasm_vop3_dpp8_from_vopcx.txt
index a1ef8f36e77be..4c197faf52c16 100644
--- a/llvm/test/MC/Disassembler/AMDGPU/gfx11_dasm_vop3_dpp8_from_vopcx.txt
+++ b/llvm/test/MC/Disassembler/AMDGPU/gfx11_dasm_vop3_dpp8_from_vopcx.txt
@@ -31,16 +31,32 @@
 # GFX11: v_cmpx_class_f32_e64_dpp -|v255|, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0x7e,0x01,0xfe,0xd4,0xea,0xfe,0x03,0x20,0xff,0x00,0x00,0x00]
 
 0x7e,0x00,0x82,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
-# GFX11: v_cmpx_eq_f16_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x82,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+# GFX11-REAL16: v_cmpx_eq_f16_e64_dpp v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x82,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+# GFX11-FAKE16: v_cmpx_eq_f16_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x82,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 
 0x7e,0x01,0x82,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05
-# GFX11: v_cmpx_eq_f16_e64_dpp |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x01,0x82,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+# GFX11-REAL16: v_cmpx_eq_f16_e64_dpp |v1.l|, -v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x01,0x82,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+# GFX11-FAKE16: v_cmpx_eq_f16_e64_dpp |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x01,0x82,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
 
 0x7e,0x02,0x82,0xd4,0xe9,0x04,0x02,0x20,0x01,0x77,0x39,0x05
-# GFX11: v_cmpx_eq_f16_e64_dpp -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x02,0x82,0xd4,0xe9,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+# GFX11-REAL16: v_cmpx_eq_f16_e64_dpp -v1.l, |v2.l| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x02,0x82,0xd4,0xe9,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+# GFX11-FAKE16: v_cmpx_eq_f16_e64_dpp -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x02,0x82,0xd4,0xe9,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
 
 0x7e,0x83,0x82,0xd4,0xea,0xfe,0x03,0x60,0xff,0x00,0x00,0x00
-# GFX11: v_cmpx_eq_f16_e64_dpp -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0x7e,0x83,0x82,0xd4,0xea,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+# GFX11-REAL16: v_cmpx_eq_f16_e64_dpp -|v255.l|, -|v255.l| clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0x7e,0x83,0x82,0xd4,0xea,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+# GFX11-FAKE16: v_cmpx_eq_f16_e64_dpp -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0x7e,0x83,0x82,0xd4,0xea,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+
+0x7e,0x19,0x82,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05
+# GFX11-REAL16: v_cmpx_eq_f16_e64_dpp |v1.h|, -v2.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x19,0x82,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+# GFX11-FAKE16: v_cmpx_eq_f16_e64_dpp |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x01,0x82,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+
+0x7e,0x0a,0x82,0xd4,0xe9,0x04,0x02,0x20,0x01,0x77,0x39,0x05
+# GFX11-REAL16: v_cmpx_eq_f16_e64_dpp -v1.h, |v2.l| op_sel:[1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x0a,0x82,0xd4,0xe9,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+# GFX11-FAKE16: v_cmpx_eq_f16_e64_dpp -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x02,0x82,0xd4,0xe9,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+
+0x7e,0x93,0x82,0xd4,0xea,0xfe,0x03,0x60,0xff,0x00,0x00,0x00
+# GFX11-REAL16: v_cmpx_eq_f16_e64_dpp -|v255.l|, -|v255.h| op_sel:[0,1] clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0x7e,0x93,0x82,0xd4,0xea,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+# GFX11-FAKE16: v_cmpx_eq_f16_e64_dpp -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0x7e,0x83,0x82,0xd4,0xea,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
 
 0x7e,0x00,0x92,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 # GFX11: v_cmpx_eq_f32_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x92,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
@@ -107,16 +123,32 @@
 # GFX11: v_cmpx_eq_u32_e64_dpp v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0x7e,0x00,0xca,0xd4,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 
 0x7e,0x00,0x80,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
-# GFX11: v_cmpx_f_f16_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x80,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+# GFX11-REAL16: v_cmpx_f_f16_e64_dpp v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x80,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+# GFX11-FAKE16: v_cmpx_f_f16_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x80,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 
 0x7e,0x01,0x80,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05
-# GFX11: v_cmpx_f_f16_e64_dpp |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x01,0x80,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+# GFX11-REAL16: v_cmpx_f_f16_e64_dpp |v1.l|, -v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x01,0x80,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+# GFX11-FAKE16: v_cmpx_f_f16_e64_dpp |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x01,0x80,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
 
 0x7e,0x02,0x80,0xd4,0xe9,0x04,0x02,0x20,0x01,0x77,0x39,0x05
-# GFX11: v_cmpx_f_f16_e64_dpp -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x02,0x80,0xd4,0xe9,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+# GFX11-REAL16: v_cmpx_f_f16_e64_dpp -v1.l, |v2.l| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x02,0x80,0xd4,0xe9,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+# GFX11-FAKE16: v_cmpx_f_f16_e64_dpp -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x02,0x80,0xd4,0xe9,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
 
 0x7e,0x83,0x80,0xd4,0xea,0xfe,0x03,0x60,0xff,0x00,0x00,0x00
-# GFX11: v_cmpx_f_f16_e64_dpp -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0x7e,0x83,0x80,0xd4,0xea,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+# GFX11-REAL16: v_cmpx_f_f16_e64_dpp -|v255.l|, -|v255.l| clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0x7e,0x83,0x80,0xd4,0xea,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+# GFX11-FAKE16: v_cmpx_f_f16_e64_dpp -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0x7e,0x83,0x80,0xd4,0xea,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+
+0x7e,0x19,0x80,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05
+# GFX11-REAL16: v_cmpx_f_f16_e64_dpp |v1.h|, -v2.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x19,0x80,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+# GFX11-FAKE16: v_cmpx_f_f16_e64_dpp |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x01,0x80,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+
+0x7e,0x0a,0x80,0xd4,0xe9,0x04,0x02,0x20,0x01,0x77,0x39,0x05
+# GFX11-REAL16: v_cmpx_f_f16_e64_dpp -v1.h, |v2.l| op_sel:[1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x0a,0x80,0xd4,0xe9,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+# GFX11-FAKE16: v_cmpx_f_f16_e64_dpp -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x02,0x80,0xd4,0xe9,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+
+0x7e,0x93,0x80,0xd4,0xea,0xfe,0x03,0x60,0xff,0x00,0x00,0x00
+# GFX11-REAL16: v_cmpx_f_f16_e64_dpp -|v255.l|, -|v255.h| op_sel:[0,1] clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0x7e,0x93,0x80,0xd4,0xea,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+# GFX11-FAKE16: v_cmpx_f_f16_e64_dpp -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0x7e,0x83,0x80,0xd4,0xea,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
 
 0x7e,0x00,0x90,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 # GFX11: v_cmpx_f_f32_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x90,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
@@ -143,16 +175,32 @@
 # GFX11: v_cmpx_f_u32_e64_dpp v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0x7e,0x00,0xc8,0xd4,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 
 0x7e,0x00,0x86,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
-# GFX11: v_cmpx_ge_f16_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x86,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+# GFX11-REAL16: v_cmpx_ge_f16_e64_dpp v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x86,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+# GFX11-FAKE16: v_cmpx_ge_f16_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x86,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 
 0x7e,0x01,0x86,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05
-# GFX11: v_cmpx_ge_f16_e64_dpp |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x01,0x86,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+# GFX11-REAL16: v_cmpx_ge_f16_e64_dpp |v1.l|, -v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x01,0x86,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+# GFX11-FAKE16: v_cmpx_ge_f16_e64_dpp |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x01,0x86,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
 
 0x7e,0x02,0x86,0xd4,0xe9,0x04,0x02,0x20,0x01,0x77,0x39,0x05
-# GFX11: v_cmpx_ge_f16_e64_dpp -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x02,0x86,0xd4,0xe9,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+# GFX11-REAL16: v_cmpx_ge_f16_e64_dpp -v1.l, |v2.l| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x02,0x86,0xd4,0xe9,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+# GFX11-FAKE16: v_cmpx_ge_f16_e64_dpp -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x02,0x86,0xd4,0xe9,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
 
 0x7e,0x83,0x86,0xd4,0xea,0xfe,0x03,0x60,0xff,0x00,0x00,0x00
-# GFX11: v_cmpx_ge_f16_e64_dpp -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0x7e,0x83,0x86,0xd4,0xea,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+# GFX11-REAL16: v_cmpx_ge_f16_e64_dpp -|v255.l|, -|v255.l| clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0x7e,0x83,0x86,0xd4,0xea,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+# GFX11-FAKE16: v_cmpx_ge_f16_e64_dpp -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0x7e,0x83,0x86,0xd4,0xea,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+
+0x7e,0x19,0x86,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05
+# GFX11-REAL16: v_cmpx_ge_f16_e64_dpp |v1.h|, -v2.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x19,0x86,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+# GFX11-FAKE16: v_cmpx_ge_f16_e64_dpp |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x01,0x86,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+
+0x7e,0x0a,0x86,0xd4,0xe9,0x04,0x02,0x20,0x01,0x77,0x39,0x05
+# GFX11-REAL16: v_cmpx_ge_f16_e64_dpp -v1.h, |v2.l| op_sel:[1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x0a,0x86,0xd4,0xe9,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+# GFX11-FAKE16: v_cmpx_ge_f16_e64_dpp -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x02,0x86,0xd4,0xe9,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+
+0x7e,0x93,0x86,0xd4,0xea,0xfe,0x03,0x60,0xff,0x00,0x00,0x00
+# GFX11-REAL16: v_cmpx_ge_f16_e64_dpp -|v255.l|, -|v255.h| op_sel:[0,1] clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0x7e,0x93,0x86,0xd4,0xea,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+# GFX11-FAKE16: v_cmpx_ge_f16_e64_dpp -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0x7e,0x83,0x86,0xd4,0xea,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
 
 0x7e,0x00,0x96,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 # GFX11: v_cmpx_ge_f32_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x96,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
@@ -219,16 +267,32 @@
 # GFX11: v_cmpx_ge_u32_e64_dpp v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0x7e,0x00,0xce,0xd4,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 
 0x7e,0x00,0x84,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
-# GFX11: v_cmpx_gt_f16_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x84,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+# GFX11-REAL16: v_cmpx_gt_f16_e64_dpp v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x84,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+# GFX11-FAKE16: v_cmpx_gt_f16_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x84,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 
 0x7e,0x01,0x84,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05
-# GFX11: v_cmpx_gt_f16_e64_dpp |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x01,0x84,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+# GFX11-REAL16: v_cmpx_gt_f16_e64_dpp |v1.l|, -v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x01,0x84,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+# GFX11-FAKE16: v_cmpx_gt_f16_e64_dpp |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x01,0x84,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
 
 0x7e,0x02,0x84,0xd4,0xe9,0x04,0x02,0x20,0x01,0x77,0x39,0x05
-# GFX11: v_cmpx_gt_f16_e64_dpp -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x02,0x84,0xd4,0xe9,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+# GFX11-REAL16: v_cmpx_gt_f16_e64_dpp -v1.l, |v2.l| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x02,0x84,0xd4,0xe9,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+# GFX11-FAKE16: v_cmpx_gt_f16_e64_dpp -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x02,0x84,0xd4,0xe9,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
 
 0x7e,0x83,0x84,0xd4,0xea,0xfe,0x03,0x60,0xff,0x00,0x00,0x00
-# GFX11: v_cmpx_gt_f16_e64_dpp -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0x7e,0x83,0x84,0xd4,0xea,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+# GFX11-REAL16: v_cmpx_gt_f16_e64_dpp -|v255.l|, -|v255.l| clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0x7e,0x83,0x84,0xd4,0xea,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+# GFX11-FAKE16: v_cmpx_gt_f16_e64_dpp -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0x7e,0x83,0x84,0xd4,0xea,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+
+0x7e,0x19,0x84,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05
+# GFX11-REAL16: v_cmpx_gt_f16_e64_dpp |v1.h|, -v2.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x19,0x84,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+# GFX11-FAKE16: v_cmpx_gt_f16_e64_dpp |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x01,0x84,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+
+0x7e,0x0a,0x84,0xd4,0xe9,0x04,0x02,0x20,0x01,0x77,0x39,0x05
+# GFX11-REAL16: v_cmpx_gt_f16_e64_dpp -v1.h, |v2.l| op_sel:[1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x0a,0x84,0xd4,0xe9,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+# GFX11-FAKE16: v_cmpx_gt_f16_e64_dpp -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x02,0x84,0xd4,0xe9,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+
+0x7e,0x93,0x84,0xd4,0xea,0xfe,0x03,0x60,0xff,0x00,0x00,0x00
+# GFX11-REAL16: v_cmpx_gt_f16_e64_dpp -|v255.l|, -|v255.h| op_sel:[0,1] clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0x7e,0x93,0x84,0xd4,0xea,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+# GFX11-FAKE16: v_cmpx_gt_f16_e64_dpp -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0x7e,0x83,0x84,0xd4,0xea,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
 
 0x7e,0x00,0x94,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 # GFX11: v_cmpx_gt_f32_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x94,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
@@ -295,16 +359,32 @@
 # GFX11: v_cmpx_gt_u32_e64_dpp v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0x7e,0x00,0xcc,0xd4,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 
 0x7e,0x00,0x83,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
-# GFX11: v_cmpx_le_f16_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x83,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+# GFX11-REAL16: v_cmpx_le_f16_e64_dpp v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x83,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+# GFX11-FAKE16: v_cmpx_le_f16_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x83,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 
 0x7e,0x01,0x83,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05
-# GFX11: v_cmpx_le_f16_e64_dpp |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x01,0x83,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+# GFX11-REAL16: v_cmpx_le_f16_e64_dpp |v1.l|, -v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x01,0x83,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+# GFX11-FAKE16: v_cmpx_le_f16_e64_dpp |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x01,0x83,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
 
 0x7e,0x02,0x83,0xd4,0xe9,0x04,0x02,0x20,0x01,0x77,0x39,0x05
-# GFX11: v_cmpx_le_f16_e64_dpp -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x02,0x83,0xd4,0xe9,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+# GFX11-REAL16: v_cmpx_le_f16_e64_dpp -v1.l, |v2.l| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x02,0x83,0xd4,0xe9,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+# GFX11-FAKE16: v_cmpx_le_f16_e64_dpp -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x02,0x83,0xd4,0xe9,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
 
 0x7e,0x83,0x83,0xd4,0xea,0xfe,0x03,0x60,0xff,0x00,0x00,0x00
-# GFX11: v_cmpx_le_f16_e64_dpp -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0x7e,0x83,0x83,0xd4,0xea,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+# GFX11-REAL16: v_cmpx_le_f16_e64_dpp -|v255.l|, -|v255.l| clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0x7e,0x83,0x83,0xd4,0xea,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+# GFX11-FAKE16: v_cmpx_le_f16_e64_dpp -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0x7e,0x83,0x83,0xd4,0xea,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+
+0x7e,0x19,0x83,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05
+# GFX11-REAL16: v_cmpx_le_f16_e64_dpp |v1.h|, -v2.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x19,0x83,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+# GFX11-FAKE16: v_cmpx_le_f16_e64_dpp |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x01,0x83,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+
+0x7e,0x0a,0x83,0xd4,0xe9,0x04,0x02,0x20,0x01,0x77,0x39,0x05
+# GFX11-REAL16: v_cmpx_le_f16_e64_dpp -v1.h, |v2.l| op_sel:[1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x0a,0x83,0xd4,0xe9,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+# GFX11-FAKE16: v_cmpx_le_f16_e64_dpp -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x02,0x83,0xd4,0xe9,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+
+0x7e,0x93,0x83,0xd4,0xea,0xfe,0x03,0x60,0xff,0x00,0x00,0x00
+# GFX11-REAL16: v_cmpx_le_f16_e64_dpp -|v255.l|, -|v255.h| op_sel:[0,1] clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0x7e,0x93,0x83,0xd4,0xea,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+# GFX11-FAKE16: v_cmpx_le_f16_e64_dpp -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0x7e,0x83,0x83,0xd4,0xea,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
 
 0x7e,0x00,0x93,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 # GFX11: v_cmpx_le_f32_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x93,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
@@ -371,16 +451,32 @@
 # GFX11: v_cmpx_le_u32_e64_dpp v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0x7e,0x00,0xcb,0xd4,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 
 0x7e,0x00,0x85,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
-# GFX11: v_cmpx_lg_f16_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x85,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+# GFX11-REAL16: v_cmpx_lg_f16_e64_dpp v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x85,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+# GFX11-FAKE16: v_cmpx_lg_f16_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x85,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 
 0x7e,0x01,0x85,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05
-# GFX11: v_cmpx_lg_f16_e64_dpp |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x01,0x85,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+# GFX11-REAL16: v_cmpx_lg_f16_e64_dpp |v1.l|, -v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x01,0x85,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+# GFX11-FAKE16: v_cmpx_lg_f16_e64_dpp |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x01,0x85,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
 
 0x7e,0x02,0x85,0xd4,0xe9,0x04,0x02,0x20,0x01,0x77,0x39,0x05
-# GFX11: v_cmpx_lg_f16_e64_dpp -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x02,0x85,0xd4,0xe9,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+# GFX11-REAL16: v_cmpx_lg_f16_e64_dpp -v1.l, |v2.l| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x02,0x85,0xd4,0xe9,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+# GFX11-FAKE16: v_cmpx_lg_f16_e64_dpp -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x02,0x85,0xd4,0xe9,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
 
 0x7e,0x83,0x85,0xd4,0xea,0xfe,0x03,0x60,0xff,0x00,0x00,0x00
-# GFX11: v_cmpx_lg_f16_e64_dpp -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0x7e,0x83,0x85,0xd4,0xea,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+# GFX11-REAL16: v_cmpx_lg_f16_e64_dpp -|v255.l|, -|v255.l| clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0x7e,0x83,0x85,0xd4,0xea,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+# GFX11-FAKE16: v_cmpx_lg_f16_e64_dpp -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0x7e,0x83,0x85,0xd4,0xea,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+
+0x7e,0x19,0x85,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05
+# GFX11-REAL16: v_cmpx_lg_f16_e64_dpp |v1.h|, -v2.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x19,0x85,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+# GFX11-FAKE16: v_cmpx_lg_f16_e64_dpp |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x01,0x85,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+
+0x7e,0x0a,0x85,0xd4,0xe9,0x04,0x02,0x20,0x01,0x77,0x39,0x05
+# GFX11-REAL16: v_cmpx_lg_f16_e64_dpp -v1.h, |v2.l| op_sel:[1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x0a,0x85,0xd4,0xe9,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+# GFX11-FAKE16: v_cmpx_lg_f16_e64_dpp -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x02,0x85,0xd4,0xe9,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+
+0x7e,0x93,0x85,0xd4,0xea,0xfe,0x03,0x60,0xff,0x00,0x00,0x00
+# GFX11-REAL16: v_cmpx_lg_f16_e64_dpp -|v255.l|, -|v255.h| op_sel:[0,1] clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0x7e,0x93,0x85,0xd4,0xea,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+# GFX11-FAKE16: v_cmpx_lg_f16_e64_dpp -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0x7e,0x83,0x85,0xd4,0xea,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
 
 0x7e,0x00,0x95,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 # GFX11: v_cmpx_lg_f32_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x95,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
@@ -539,16 +635,32 @@
 # GFX11: v_cmpx_ne_u32_e64_dpp v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0x7e,0x00,0xcd,0xd4,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 
 0x7e,0x00,0x8d,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
-# GFX11: v_cmpx_neq_f16_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x8d,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+# GFX11-REAL16: v_cmpx_neq_f16_e64_dpp v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x8d,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+# GFX11-FAKE16: v_cmpx_neq_f16_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x8d,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 
 0x7e,0x01,0x8d,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05
-# GFX11: v_cmpx_neq_f16_e64_dpp |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x01,0x8d,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+# GFX11-REAL16: v_cmpx_neq_f16_e64_dpp |v1.l|, -v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x01,0x8d,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+# GFX11-FAKE16: v_cmpx_neq_f16_e64_dpp |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x01,0x8d,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
 
 0x7e,0x02,0x8d,0xd4,0xe9,0x04,0x02,0x20,0x01,0x77,0x39,0x05
-# GFX11: v_cmpx_neq_f16_e64_dpp -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x02,0x8d,0xd4,0xe9,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+# GFX11-REAL16: v_cmpx_neq_f16_e64_dpp -v1.l, |v2.l| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x02,0x8d,0xd4,0xe9,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+# GFX11-FAKE16: v_cmpx_neq_f16_e64_dpp -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x02,0x8d,0xd4,0xe9,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
 
 0x7e,0x83,0x8d,0xd4,0xea,0xfe,0x03,0x60,0xff,0x00,0x00,0x00
-# GFX11: v_cmpx_neq_f16_e64_dpp -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0x7e,0x83,0x8d,0xd4,0xea,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+# GFX11-REAL16: v_cmpx_neq_f16_e64_dpp -|v255.l|, -|v255.l| clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0x7e,0x83,0x8d,0xd4,0xea,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+# GFX11-FAKE16: v_cmpx_neq_f16_e64_dpp -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0x7e,0x83,0x8d,0xd4,0xea,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+
+0x7e,0x19,0x8d,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05
+# GFX11-REAL16: v_cmpx_neq_f16_e64_dpp |v1.h|, -v2.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x19,0x8d,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+# GFX11-FAKE16: v_cmpx_neq_f16_e64_dpp |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x01,0x8d,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+
+0x7e,0x0a,0x8d,0xd4,0xe9,0x04,0x02,0x20,0x01,0x77,0x39,0x05
+# GFX11-REAL16: v_cmpx_neq_f16_e64_dpp -v1.h, |v2.l| op_sel:[1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x0a,0x8d,0xd4,0xe9,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+# GFX11-FAKE16: v_cmpx_neq_f16_e64_dpp -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x02,0x8d,0xd4,0xe9,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+
+0x7e,0x93,0x8d,0xd4,0xea,0xfe,0x03,0x60,0xff,0x00,0x00,0x00
+# GFX11-REAL16: v_cmpx_neq_f16_e64_dpp -|v255.l|, -|v255.h| op_sel:[0,1] clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0x7e,0x93,0x8d,0xd4,0xea,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+# GFX11-FAKE16: v_cmpx_neq_f16_e64_dpp -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0x7e,0x83,0x8d,0xd4,0xea,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
 
 0x7e,0x00,0x9d,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 # GFX11: v_cmpx_neq_f32_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x9d,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
@@ -563,16 +675,32 @@
 # GFX11: v_cmpx_neq_f32_e64_dpp -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0x7e,0x83,0x9d,0xd4,0xea,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
 
 0x7e,0x00,0x89,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
-# GFX11: v_cmpx_nge_f16_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x89,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+# GFX11-REAL16: v_cmpx_nge_f16_e64_dpp v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x89,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+# GFX11-FAKE16: v_cmpx_nge_f16_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x89,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 
 0x7e,0x01,0x89,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05
-# GFX11: v_cmpx_nge_f16_e64_dpp |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x01,0x89,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+# GFX11-REAL16: v_cmpx_nge_f16_e64_dpp |v1.l|, -v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x01,0x89,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+# GFX11-FAKE16: v_cmpx_nge_f16_e64_dpp |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x01,0x89,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
 
 0x7e,0x02,0x89,0xd4,0xe9,0x04,0x02,0x20,0x01,0x77,0x39,0x05
-# GFX11: v_cmpx_nge_f16_e64_dpp -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x02,0x89,0xd4,0xe9,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+# GFX11-REAL16: v_cmpx_nge_f16_e64_dpp -v1.l, |v2.l| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x02,0x89,0xd4,0xe9,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+# GFX11-FAKE16: v_cmpx_nge_f16_e64_dpp -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x02,0x89,0xd4,0xe9,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
 
 0x7e,0x83,0x89,0xd4,0xea,0xfe,0x03,0x60,0xff,0x00,0x00,0x00
-# GFX11: v_cmpx_nge_f16_e64_dpp -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0x7e,0x83,0x89,0xd4,0xea,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+# GFX11-REAL16: v_cmpx_nge_f16_e64_dpp -|v255.l|, -|v255.l| clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0x7e,0x83,0x89,0xd4,0xea,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+# GFX11-FAKE16: v_cmpx_nge_f16_e64_dpp -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0x7e,0x83,0x89,0xd4,0xea,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+
+0x7e,0x19,0x89,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05
+# GFX11-REAL16: v_cmpx_nge_f16_e64_dpp |v1.h|, -v2.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x19,0x89,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+# GFX11-FAKE16: v_cmpx_nge_f16_e64_dpp |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x01,0x89,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+
+0x7e,0x0a,0x89,0xd4,0xe9,0x04,0x02,0x20,0x01,0x77,0x39,0x05
+# GFX11-REAL16: v_cmpx_nge_f16_e64_dpp -v1.h, |v2.l| op_sel:[1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x0a,0x89,0xd4,0xe9,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+# GFX11-FAKE16: v_cmpx_nge_f16_e64_dpp -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x02,0x89,0xd4,0xe9,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+
+0x7e,0x93,0x89,0xd4,0xea,0xfe,0x03,0x60,0xff,0x00,0x00,0x00
+# GFX11-REAL16: v_cmpx_nge_f16_e64_dpp -|v255.l|, -|v255.h| op_sel:[0,1] clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0x7e,0x93,0x89,0xd4,0xea,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+# GFX11-FAKE16: v_cmpx_nge_f16_e64_dpp -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0x7e,0x83,0x89,0xd4,0xea,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
 
 0x7e,0x00,0x99,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 # GFX11: v_cmpx_nge_f32_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x99,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
@@ -587,16 +715,32 @@
 # GFX11: v_cmpx_nge_f32_e64_dpp -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0x7e,0x83,0x99,0xd4,0xea,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
 
 0x7e,0x00,0x8b,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
-# GFX11: v_cmpx_ngt_f16_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x8b,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+# GFX11-REAL16: v_cmpx_ngt_f16_e64_dpp v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x8b,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+# GFX11-FAKE16: v_cmpx_ngt_f16_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x8b,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 
 0x7e,0x01,0x8b,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05
-# GFX11: v_cmpx_ngt_f16_e64_dpp |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x01,0x8b,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+# GFX11-REAL16: v_cmpx_ngt_f16_e64_dpp |v1.l|, -v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x01,0x8b,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+# GFX11-FAKE16: v_cmpx_ngt_f16_e64_dpp |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x01,0x8b,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
 
 0x7e,0x02,0x8b,0xd4,0xe9,0x04,0x02,0x20,0x01,0x77,0x39,0x05
-# GFX11: v_cmpx_ngt_f16_e64_dpp -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x02,0x8b,0xd4,0xe9,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+# GFX11-REAL16: v_cmpx_ngt_f16_e64_dpp -v1.l, |v2.l| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x02,0x8b,0xd4,0xe9,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+# GFX11-FAKE16: v_cmpx_ngt_f16_e64_dpp -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x02,0x8b,0xd4,0xe9,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
 
 0x7e,0x83,0x8b,0xd4,0xea,0xfe,0x03,0x60,0xff,0x00,0x00,0x00
-# GFX11: v_cmpx_ngt_f16_e64_dpp -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0x7e,0x83,0x8b,0xd4,0xea,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+# GFX11-REAL16: v_cmpx_ngt_f16_e64_dpp -|v255.l|, -|v255.l| clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0x7e,0x83,0x8b,0xd4,0xea,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+# GFX11-FAKE16: v_cmpx_ngt_f16_e64_dpp -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0x7e,0x83,0x8b,0xd4,0xea,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+
+0x7e,0x19,0x8b,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05
+# GFX11-REAL16: v_cmpx_ngt_f16_e64_dpp |v1.h|, -v2.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x19,0x8b,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+# GFX11-FAKE16: v_cmpx_ngt_f16_e64_dpp |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x01,0x8b,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+
+0x7e,0x0a,0x8b,0xd4,0xe9,0x04,0x02,0x20,0x01,0x77,0x39,0x05
+# GFX11-REAL16: v_cmpx_ngt_f16_e64_dpp -v1.h, |v2.l| op_sel:[1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x0a,0x8b,0xd4,0xe9,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+# GFX11-FAKE16: v_cmpx_ngt_f16_e64_dpp -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x02,0x8b,0xd4,0xe9,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+
+0x7e,0x93,0x8b,0xd4,0xea,0xfe,0x03,0x60,0xff,0x00,0x00,0x00
+# GFX11-REAL16: v_cmpx_ngt_f16_e64_dpp -|v255.l|, -|v255.h| op_sel:[0,1] clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0x7e,0x93,0x8b,0xd4,0xea,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+# GFX11-FAKE16: v_cmpx_ngt_f16_e64_dpp -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0x7e,0x83,0x8b,0xd4,0xea,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
 
 0x7e,0x00,0x9b,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 # GFX11: v_cmpx_ngt_f32_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x9b,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
@@ -611,16 +755,32 @@
 # GFX11: v_cmpx_ngt_f32_e64_dpp -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0x7e,0x83,0x9b,0xd4,0xea,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
 
 0x7e,0x00,0x8c,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
-# GFX11: v_cmpx_nle_f16_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x8c,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+# GFX11-REAL16: v_cmpx_nle_f16_e64_dpp v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x8c,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+# GFX11-FAKE16: v_cmpx_nle_f16_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x8c,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 
 0x7e,0x01,0x8c,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05
-# GFX11: v_cmpx_nle_f16_e64_dpp |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x01,0x8c,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+# GFX11-REAL16: v_cmpx_nle_f16_e64_dpp |v1.l|, -v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x01,0x8c,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+# GFX11-FAKE16: v_cmpx_nle_f16_e64_dpp |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x01,0x8c,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
 
 0x7e,0x02,0x8c,0xd4,0xe9,0x04,0x02,0x20,0x01,0x77,0x39,0x05
-# GFX11: v_cmpx_nle_f16_e64_dpp -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x02,0x8c,0xd4,0xe9,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+# GFX11-REAL16: v_cmpx_nle_f16_e64_dpp -v1.l, |v2.l| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x02,0x8c,0xd4,0xe9,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+# GFX11-FAKE16: v_cmpx_nle_f16_e64_dpp -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x02,0x8c,0xd4,0xe9,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
 
 0x7e,0x83,0x8c,0xd4,0xea,0xfe,0x03,0x60,0xff,0x00,0x00,0x00
-# GFX11: v_cmpx_nle_f16_e64_dpp -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0x7e,0x83,0x8c,0xd4,0xea,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+# GFX11-REAL16: v_cmpx_nle_f16_e64_dpp -|v255.l|, -|v255.l| clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0x7e,0x83,0x8c,0xd4,0xea,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+# GFX11-FAKE16: v_cmpx_nle_f16_e64_dpp -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0x7e,0x83,0x8c,0xd4,0xea,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+
+0x7e,0x19,0x8c,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05
+# GFX11-REAL16: v_cmpx_nle_f16_e64_dpp |v1.h|, -v2.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x19,0x8c,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+# GFX11-FAKE16: v_cmpx_nle_f16_e64_dpp |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x01,0x8c,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+
+0x7e,0x0a,0x8c,0xd4,0xe9,0x04,0x02,0x20,0x01,0x77,0x39,0x05
+# GFX11-REAL16: v_cmpx_nle_f16_e64_dpp -v1.h, |v2.l| op_sel:[1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x0a,0x8c,0xd4,0xe9,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+# GFX11-FAKE16: v_cmpx_nle_f16_e64_dpp -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x02,0x8c,0xd4,0xe9,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+
+0x7e,0x93,0x8c,0xd4,0xea,0xfe,0x03,0x60,0xff,0x00,0x00,0x00
+# GFX11-REAL16: v_cmpx_nle_f16_e64_dpp -|v255.l|, -|v255.h| op_sel:[0,1] clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0x7e,0x93,0x8c,0xd4,0xea,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+# GFX11-FAKE16: v_cmpx_nle_f16_e64_dpp -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0x7e,0x83,0x8c,0xd4,0xea,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
 
 0x7e,0x00,0x9c,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 # GFX11: v_cmpx_nle_f32_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x9c,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
@@ -635,16 +795,32 @@
 # GFX11: v_cmpx_nle_f32_e64_dpp -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0x7e,0x83,0x9c,0xd4,0xea,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
 
 0x7e,0x00,0x8a,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
-# GFX11: v_cmpx_nlg_f16_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x8a,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+# GFX11-REAL16: v_cmpx_nlg_f16_e64_dpp v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x8a,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+# GFX11-FAKE16: v_cmpx_nlg_f16_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x8a,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 
 0x7e,0x01,0x8a,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05
-# GFX11: v_cmpx_nlg_f16_e64_dpp |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x01,0x8a,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+# GFX11-REAL16: v_cmpx_nlg_f16_e64_dpp |v1.l|, -v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x01,0x8a,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+# GFX11-FAKE16: v_cmpx_nlg_f16_e64_dpp |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x01,0x8a,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
 
 0x7e,0x02,0x8a,0xd4,0xe9,0x04,0x02,0x20,0x01,0x77,0x39,0x05
-# GFX11: v_cmpx_nlg_f16_e64_dpp -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x02,0x8a,0xd4,0xe9,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+# GFX11-REAL16: v_cmpx_nlg_f16_e64_dpp -v1.l, |v2.l| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x02,0x8a,0xd4,0xe9,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+# GFX11-FAKE16: v_cmpx_nlg_f16_e64_dpp -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x02,0x8a,0xd4,0xe9,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
 
 0x7e,0x83,0x8a,0xd4,0xea,0xfe,0x03,0x60,0xff,0x00,0x00,0x00
-# GFX11: v_cmpx_nlg_f16_e64_dpp -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0x7e,0x83,0x8a,0xd4,0xea,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+# GFX11-REAL16: v_cmpx_nlg_f16_e64_dpp -|v255.l|, -|v255.l| clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0x7e,0x83,0x8a,0xd4,0xea,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+# GFX11-FAKE16: v_cmpx_nlg_f16_e64_dpp -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0x7e,0x83,0x8a,0xd4,0xea,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+
+0x7e,0x19,0x8a,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05
+# GFX11-REAL16: v_cmpx_nlg_f16_e64_dpp |v1.h|, -v2.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x19,0x8a,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+# GFX11-FAKE16: v_cmpx_nlg_f16_e64_dpp |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x01,0x8a,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+
+0x7e,0x0a,0x8a,0xd4,0xe9,0x04,0x02,0x20,0x01,0x77,0x39,0x05
+# GFX11-REAL16: v_cmpx_nlg_f16_e64_dpp -v1.h, |v2.l| op_sel:[1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x0a,0x8a,0xd4,0xe9,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+# GFX11-FAKE16: v_cmpx_nlg_f16_e64_dpp -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x02,0x8a,0xd4,0xe9,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+
+0x7e,0x93,0x8a,0xd4,0xea,0xfe,0x03,0x60,0xff,0x00,0x00,0x00
+# GFX11-REAL16: v_cmpx_nlg_f16_e64_dpp -|v255.l|, -|v255.h| op_sel:[0,1] clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0x7e,0x93,0x8a,0xd4,0xea,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+# GFX11-FAKE16: v_cmpx_nlg_f16_e64_dpp -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0x7e,0x83,0x8a,0xd4,0xea,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
 
 0x7e,0x00,0x9a,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 # GFX11: v_cmpx_nlg_f32_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x9a,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
@@ -659,16 +835,32 @@
 # GFX11: v_cmpx_nlg_f32_e64_dpp -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0x7e,0x83,0x9a,0xd4,0xea,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
 
 0x7e,0x00,0x8e,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
-# GFX11: v_cmpx_nlt_f16_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x8e,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+# GFX11-REAL16: v_cmpx_nlt_f16_e64_dpp v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x8e,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+# GFX11-FAKE16: v_cmpx_nlt_f16_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x8e,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 
 0x7e,0x01,0x8e,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05
-# GFX11: v_cmpx_nlt_f16_e64_dpp |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x01,0x8e,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+# GFX11-REAL16: v_cmpx_nlt_f16_e64_dpp |v1.l|, -v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x01,0x8e,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+# GFX11-FAKE16: v_cmpx_nlt_f16_e64_dpp |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x01,0x8e,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
 
 0x7e,0x02,0x8e,0xd4,0xe9,0x04,0x02,0x20,0x01,0x77,0x39,0x05
-# GFX11: v_cmpx_nlt_f16_e64_dpp -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x02,0x8e,0xd4,0xe9,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+# GFX11-REAL16: v_cmpx_nlt_f16_e64_dpp -v1.l, |v2.l| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x02,0x8e,0xd4,0xe9,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+# GFX11-FAKE16: v_cmpx_nlt_f16_e64_dpp -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x02,0x8e,0xd4,0xe9,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
 
 0x7e,0x83,0x8e,0xd4,0xea,0xfe,0x03,0x60,0xff,0x00,0x00,0x00
-# GFX11: v_cmpx_nlt_f16_e64_dpp -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0x7e,0x83,0x8e,0xd4,0xea,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+# GFX11-REAL16: v_cmpx_nlt_f16_e64_dpp -|v255.l|, -|v255.l| clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0x7e,0x83,0x8e,0xd4,0xea,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+# GFX11-FAKE16: v_cmpx_nlt_f16_e64_dpp -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0x7e,0x83,0x8e,0xd4,0xea,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+
+0x7e,0x19,0x8e,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05
+# GFX11-REAL16: v_cmpx_nlt_f16_e64_dpp |v1.h|, -v2.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x19,0x8e,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+# GFX11-FAKE16: v_cmpx_nlt_f16_e64_dpp |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x01,0x8e,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+
+0x7e,0x0a,0x8e,0xd4,0xe9,0x04,0x02,0x20,0x01,0x77,0x39,0x05
+# GFX11-REAL16: v_cmpx_nlt_f16_e64_dpp -v1.h, |v2.l| op_sel:[1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x0a,0x8e,0xd4,0xe9,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+# GFX11-FAKE16: v_cmpx_nlt_f16_e64_dpp -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x02,0x8e,0xd4,0xe9,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+
+0x7e,0x93,0x8e,0xd4,0xea,0xfe,0x03,0x60,0xff,0x00,0x00,0x00
+# GFX11-REAL16: v_cmpx_nlt_f16_e64_dpp -|v255.l|, -|v255.h| op_sel:[0,1] clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0x7e,0x93,0x8e,0xd4,0xea,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+# GFX11-FAKE16: v_cmpx_nlt_f16_e64_dpp -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0x7e,0x83,0x8e,0xd4,0xea,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
 
 0x7e,0x00,0x9e,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 # GFX11: v_cmpx_nlt_f32_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x9e,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
@@ -683,16 +875,32 @@
 # GFX11: v_cmpx_nlt_f32_e64_dpp -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0x7e,0x83,0x9e,0xd4,0xea,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
 
 0x7e,0x00,0x87,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
-# GFX11: v_cmpx_o_f16_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x87,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+# GFX11-REAL16: v_cmpx_o_f16_e64_dpp v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x87,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+# GFX11-FAKE16: v_cmpx_o_f16_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x87,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 
 0x7e,0x01,0x87,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05
-# GFX11: v_cmpx_o_f16_e64_dpp |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x01,0x87,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+# GFX11-REAL16: v_cmpx_o_f16_e64_dpp |v1.l|, -v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x01,0x87,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+# GFX11-FAKE16: v_cmpx_o_f16_e64_dpp |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x01,0x87,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
 
 0x7e,0x02,0x87,0xd4,0xe9,0x04,0x02,0x20,0x01,0x77,0x39,0x05
-# GFX11: v_cmpx_o_f16_e64_dpp -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x02,0x87,0xd4,0xe9,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+# GFX11-REAL16: v_cmpx_o_f16_e64_dpp -v1.l, |v2.l| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x02,0x87,0xd4,0xe9,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+# GFX11-FAKE16: v_cmpx_o_f16_e64_dpp -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x02,0x87,0xd4,0xe9,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
 
 0x7e,0x83,0x87,0xd4,0xea,0xfe,0x03,0x60,0xff,0x00,0x00,0x00
-# GFX11: v_cmpx_o_f16_e64_dpp -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0x7e,0x83,0x87,0xd4,0xea,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+# GFX11-REAL16: v_cmpx_o_f16_e64_dpp -|v255.l|, -|v255.l| clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0x7e,0x83,0x87,0xd4,0xea,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+# GFX11-FAKE16: v_cmpx_o_f16_e64_dpp -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0x7e,0x83,0x87,0xd4,0xea,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+
+0x7e,0x19,0x87,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05
+# GFX11-REAL16: v_cmpx_o_f16_e64_dpp |v1.h|, -v2.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x19,0x87,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+# GFX11-FAKE16: v_cmpx_o_f16_e64_dpp |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x01,0x87,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+
+0x7e,0x0a,0x87,0xd4,0xe9,0x04,0x02,0x20,0x01,0x77,0x39,0x05
+# GFX11-REAL16: v_cmpx_o_f16_e64_dpp -v1.h, |v2.l| op_sel:[1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x0a,0x87,0xd4,0xe9,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+# GFX11-FAKE16: v_cmpx_o_f16_e64_dpp -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x02,0x87,0xd4,0xe9,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+
+0x7e,0x93,0x87,0xd4,0xea,0xfe,0x03,0x60,0xff,0x00,0x00,0x00
+# GFX11-REAL16: v_cmpx_o_f16_e64_dpp -|v255.l|, -|v255.h| op_sel:[0,1] clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0x7e,0x93,0x87,0xd4,0xea,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+# GFX11-FAKE16: v_cmpx_o_f16_e64_dpp -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0x7e,0x83,0x87,0xd4,0xea,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
 
 0x7e,0x00,0x97,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 # GFX11: v_cmpx_o_f32_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x97,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
@@ -707,16 +915,32 @@
 # GFX11: v_cmpx_o_f32_e64_dpp -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0x7e,0x83,0x97,0xd4,0xea,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
 
 0x7e,0x00,0x8f,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
-# GFX11: v_cmpx_t_f16_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x8f,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+# GFX11-REAL16: v_cmpx_t_f16_e64_dpp v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x8f,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+# GFX11-FAKE16: v_cmpx_t_f16_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x8f,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 
 0x7e,0x01,0x8f,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05
-# GFX11: v_cmpx_t_f16_e64_dpp |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x01,0x8f,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+# GFX11-REAL16: v_cmpx_t_f16_e64_dpp |v1.l|, -v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x01,0x8f,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+# GFX11-FAKE16: v_cmpx_t_f16_e64_dpp |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x01,0x8f,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
 
 0x7e,0x02,0x8f,0xd4,0xe9,0x04,0x02,0x20,0x01,0x77,0x39,0x05
-# GFX11: v_cmpx_t_f16_e64_dpp -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x02,0x8f,0xd4,0xe9,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+# GFX11-REAL16: v_cmpx_t_f16_e64_dpp -v1.l, |v2.l| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x02,0x8f,0xd4,0xe9,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+# GFX11-FAKE16: v_cmpx_t_f16_e64_dpp -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x02,0x8f,0xd4,0xe9,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
 
 0x7e,0x83,0x8f,0xd4,0xea,0xfe,0x03,0x60,0xff,0x00,0x00,0x00
-# GFX11: v_cmpx_t_f16_e64_dpp -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0x7e,0x83,0x8f,0xd4,0xea,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+# GFX11-REAL16: v_cmpx_t_f16_e64_dpp -|v255.l|, -|v255.l| clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0x7e,0x83,0x8f,0xd4,0xea,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+# GFX11-FAKE16: v_cmpx_t_f16_e64_dpp -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0x7e,0x83,0x8f,0xd4,0xea,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+
+0x7e,0x19,0x8f,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05
+# GFX11-REAL16: v_cmpx_t_f16_e64_dpp |v1.h|, -v2.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x19,0x8f,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+# GFX11-FAKE16: v_cmpx_t_f16_e64_dpp |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x01,0x8f,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+
+0x7e,0x0a,0x8f,0xd4,0xe9,0x04,0x02,0x20,0x01,0x77,0x39,0x05
+# GFX11-REAL16: v_cmpx_t_f16_e64_dpp -v1.h, |v2.l| op_sel:[1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x0a,0x8f,0xd4,0xe9,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+# GFX11-FAKE16: v_cmpx_t_f16_e64_dpp -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x02,0x8f,0xd4,0xe9,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+
+0x7e,0x93,0x8f,0xd4,0xea,0xfe,0x03,0x60,0xff,0x00,0x00,0x00
+# GFX11-REAL16: v_cmpx_t_f16_e64_dpp -|v255.l|, -|v255.h| op_sel:[0,1] clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0x7e,0x93,0x8f,0xd4,0xea,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+# GFX11-FAKE16: v_cmpx_t_f16_e64_dpp -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0x7e,0x83,0x8f,0xd4,0xea,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
 
 0x7e,0x00,0x9f,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 # GFX11: v_cmpx_t_f32_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x9f,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
@@ -743,16 +967,32 @@
 # GFX11: v_cmpx_t_u32_e64_dpp v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0x7e,0x00,0xcf,0xd4,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 
 0x7e,0x00,0x88,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
-# GFX11: v_cmpx_u_f16_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x88,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+# GFX11-REAL16: v_cmpx_u_f16_e64_dpp v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x88,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+# GFX11-FAKE16: v_cmpx_u_f16_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x88,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 
 0x7e,0x01,0x88,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05
-# GFX11: v_cmpx_u_f16_e64_dpp |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x01,0x88,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+# GFX11-REAL16: v_cmpx_u_f16_e64_dpp |v1.l|, -v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x01,0x88,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+# GFX11-FAKE16: v_cmpx_u_f16_e64_dpp |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x01,0x88,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
 
 0x7e,0x02,0x88,0xd4,0xe9,0x04,0x02,0x20,0x01,0x77,0x39,0x05
-# GFX11: v_cmpx_u_f16_e64_dpp -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x02,0x88,0xd4,0xe9,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+# GFX11-REAL16: v_cmpx_u_f16_e64_dpp -v1.l, |v2.l| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x02,0x88,0xd4,0xe9,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+# GFX11-FAKE16: v_cmpx_u_f16_e64_dpp -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x02,0x88,0xd4,0xe9,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
 
 0x7e,0x83,0x88,0xd4,0xea,0xfe,0x03,0x60,0xff,0x00,0x00,0x00
-# GFX11: v_cmpx_u_f16_e64_dpp -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0x7e,0x83,0x88,0xd4,0xea,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+# GFX11-REAL16: v_cmpx_u_f16_e64_dpp -|v255.l|, -|v255.l| clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0x7e,0x83,0x88,0xd4,0xea,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+# GFX11-FAKE16: v_cmpx_u_f16_e64_dpp -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0x7e,0x83,0x88,0xd4,0xea,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+
+0x7e,0x19,0x88,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05
+# GFX11-REAL16: v_cmpx_u_f16_e64_dpp |v1.h|, -v2.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x19,0x88,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+# GFX11-FAKE16: v_cmpx_u_f16_e64_dpp |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x01,0x88,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+
+0x7e,0x0a,0x88,0xd4,0xe9,0x04,0x02,0x20,0x01,0x77,0x39,0x05
+# GFX11-REAL16: v_cmpx_u_f16_e64_dpp -v1.h, |v2.l| op_sel:[1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x0a,0x88,0xd4,0xe9,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+# GFX11-FAKE16: v_cmpx_u_f16_e64_dpp -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x02,0x88,0xd4,0xe9,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+
+0x7e,0x93,0x88,0xd4,0xea,0xfe,0x03,0x60,0xff,0x00,0x00,0x00
+# GFX11-REAL16: v_cmpx_u_f16_e64_dpp -|v255.l|, -|v255.h| op_sel:[0,1] clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0x7e,0x93,0x88,0xd4,0xea,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+# GFX11-FAKE16: v_cmpx_u_f16_e64_dpp -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0x7e,0x83,0x88,0xd4,0xea,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
 
 0x7e,0x00,0x98,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 # GFX11: v_cmpx_u_f32_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x98,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
diff --git a/llvm/test/MC/Disassembler/AMDGPU/gfx11_dasm_vop3_from_vopcx.txt b/llvm/test/MC/Disassembler/AMDGPU/gfx11_dasm_vop3_from_vopcx.txt
index a3e9f92454e3a..40c34708d863e 100644
--- a/llvm/test/MC/Disassembler/AMDGPU/gfx11_dasm_vop3_from_vopcx.txt
+++ b/llvm/test/MC/Disassembler/AMDGPU/gfx11_dasm_vop3_from_vopcx.txt
@@ -160,10 +160,12 @@
 # GFX11: v_cmpx_class_f64_e64 0xaf123456, 0xaf123456 ; encoding: [0x7e,0x00,0xff,0xd4,0xff,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
 
 0x7e,0x00,0x82,0xd4,0x01,0x05,0x02,0x00
-# GFX11: v_cmpx_eq_f16_e64 v1, v2                ; encoding: [0x7e,0x00,0x82,0xd4,0x01,0x05,0x02,0x00]
+# GFX11-REAL16: v_cmpx_eq_f16_e64 v1.l, v2.l            ; encoding: [0x7e,0x00,0x82,0xd4,0x01,0x05,0x02,0x00]
+# GFX11-FAKE16: v_cmpx_eq_f16_e64 v1, v2                ; encoding: [0x7e,0x00,0x82,0xd4,0x01,0x05,0x02,0x00]
 
 0x7e,0x00,0x82,0xd4,0xff,0xff,0x03,0x00
-# GFX11: v_cmpx_eq_f16_e64 v255, v255            ; encoding: [0x7e,0x00,0x82,0xd4,0xff,0xff,0x03,0x00]
+# GFX11-REAL16: v_cmpx_eq_f16_e64 v255.l, v255.l        ; encoding: [0x7e,0x00,0x82,0xd4,0xff,0xff,0x03,0x00]
+# GFX11-FAKE16: v_cmpx_eq_f16_e64 v255, v255            ; encoding: [0x7e,0x00,0x82,0xd4,0xff,0xff,0x03,0x00]
 
 0x7e,0x00,0x82,0xd4,0x01,0x04,0x00,0x00
 # GFX11: v_cmpx_eq_f16_e64 s1, s2                ; encoding: [0x7e,0x00,0x82,0xd4,0x01,0x04,0x00,0x00]
@@ -204,6 +206,14 @@
 0x7e,0x83,0x82,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00
 # GFX11: v_cmpx_eq_f16_e64 -|0xfe0b|, -|vcc_hi| clamp ; encoding: [0x7e,0x83,0x82,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00]
 
+0x7e,0x08,0x82,0xd4,0x01,0x05,0x02,0x00
+# GFX11-REAL16: v_cmpx_eq_f16_e64 v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x7e,0x08,0x82,0xd4,0x01,0x05,0x02,0x00]
+# GFX11-FAKE16: v_cmpx_eq_f16_e64 v1, v2                ; encoding: [0x7e,0x00,0x82,0xd4,0x01,0x05,0x02,0x00]
+
+0x7e,0x10,0x82,0xd4,0xff,0xff,0x03,0x00
+# GFX11-REAL16: v_cmpx_eq_f16_e64 v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x7e,0x10,0x82,0xd4,0xff,0xff,0x03,0x00]
+# GFX11-FAKE16: v_cmpx_eq_f16_e64 v255, v255            ; encoding: [0x7e,0x00,0x82,0xd4,0xff,0xff,0x03,0x00]
+
 0x7e,0x00,0x92,0xd4,0x01,0x05,0x02,0x00
 # GFX11: v_cmpx_eq_f32_e64 v1, v2                ; encoding: [0x7e,0x00,0x92,0xd4,0x01,0x05,0x02,0x00]
 
@@ -558,10 +568,12 @@
 # GFX11: v_cmpx_eq_u64_e64 0xaf123456, vcc       ; encoding: [0x7e,0x00,0xda,0xd4,0xff,0xd4,0x00,0x00,0x56,0x34,0x12,0xaf]
 
 0x7e,0x00,0x80,0xd4,0x01,0x05,0x02,0x00
-# GFX11: v_cmpx_f_f16_e64 v1, v2                 ; encoding: [0x7e,0x00,0x80,0xd4,0x01,0x05,0x02,0x00]
+# GFX11-REAL16: v_cmpx_f_f16_e64 v1.l, v2.l             ; encoding: [0x7e,0x00,0x80,0xd4,0x01,0x05,0x02,0x00]
+# GFX11-FAKE16: v_cmpx_f_f16_e64 v1, v2                 ; encoding: [0x7e,0x00,0x80,0xd4,0x01,0x05,0x02,0x00]
 
 0x7e,0x00,0x80,0xd4,0xff,0xff,0x03,0x00
-# GFX11: v_cmpx_f_f16_e64 v255, v255             ; encoding: [0x7e,0x00,0x80,0xd4,0xff,0xff,0x03,0x00]
+# GFX11-REAL16: v_cmpx_f_f16_e64 v255.l, v255.l         ; encoding: [0x7e,0x00,0x80,0xd4,0xff,0xff,0x03,0x00]
+# GFX11-FAKE16: v_cmpx_f_f16_e64 v255, v255             ; encoding: [0x7e,0x00,0x80,0xd4,0xff,0xff,0x03,0x00]
 
 0x7e,0x00,0x80,0xd4,0x01,0x04,0x00,0x00
 # GFX11: v_cmpx_f_f16_e64 s1, s2                 ; encoding: [0x7e,0x00,0x80,0xd4,0x01,0x04,0x00,0x00]
@@ -602,6 +614,14 @@
 0x7e,0x83,0x80,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00
 # GFX11: v_cmpx_f_f16_e64 -|0xfe0b|, -|vcc_hi| clamp ; encoding: [0x7e,0x83,0x80,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00]
 
+0x7e,0x08,0x80,0xd4,0x01,0x05,0x02,0x00
+# GFX11-REAL16: v_cmpx_f_f16_e64 v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x7e,0x08,0x80,0xd4,0x01,0x05,0x02,0x00]
+# GFX11-FAKE16: v_cmpx_f_f16_e64 v1, v2                 ; encoding: [0x7e,0x00,0x80,0xd4,0x01,0x05,0x02,0x00]
+
+0x7e,0x10,0x80,0xd4,0xff,0xff,0x03,0x00
+# GFX11-REAL16: v_cmpx_f_f16_e64 v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x7e,0x10,0x80,0xd4,0xff,0xff,0x03,0x00]
+# GFX11-FAKE16: v_cmpx_f_f16_e64 v255, v255             ; encoding: [0x7e,0x00,0x80,0xd4,0xff,0xff,0x03,0x00]
+
 0x7e,0x00,0x90,0xd4,0x01,0x05,0x02,0x00
 # GFX11: v_cmpx_f_f32_e64 v1, v2                 ; encoding: [0x7e,0x00,0x90,0xd4,0x01,0x05,0x02,0x00]
 
@@ -846,10 +866,12 @@
 # GFX11: v_cmpx_f_u64_e64 0xaf123456, vcc        ; encoding: [0x7e,0x00,0xd8,0xd4,0xff,0xd4,0x00,0x00,0x56,0x34,0x12,0xaf]
 
 0x7e,0x00,0x86,0xd4,0x01,0x05,0x02,0x00
-# GFX11: v_cmpx_ge_f16_e64 v1, v2                ; encoding: [0x7e,0x00,0x86,0xd4,0x01,0x05,0x02,0x00]
+# GFX11-REAL16: v_cmpx_ge_f16_e64 v1.l, v2.l            ; encoding: [0x7e,0x00,0x86,0xd4,0x01,0x05,0x02,0x00]
+# GFX11-FAKE16: v_cmpx_ge_f16_e64 v1, v2                ; encoding: [0x7e,0x00,0x86,0xd4,0x01,0x05,0x02,0x00]
 
 0x7e,0x00,0x86,0xd4,0xff,0xff,0x03,0x00
-# GFX11: v_cmpx_ge_f16_e64 v255, v255            ; encoding: [0x7e,0x00,0x86,0xd4,0xff,0xff,0x03,0x00]
+# GFX11-REAL16: v_cmpx_ge_f16_e64 v255.l, v255.l        ; encoding: [0x7e,0x00,0x86,0xd4,0xff,0xff,0x03,0x00]
+# GFX11-FAKE16: v_cmpx_ge_f16_e64 v255, v255            ; encoding: [0x7e,0x00,0x86,0xd4,0xff,0xff,0x03,0x00]
 
 0x7e,0x00,0x86,0xd4,0x01,0x04,0x00,0x00
 # GFX11: v_cmpx_ge_f16_e64 s1, s2                ; encoding: [0x7e,0x00,0x86,0xd4,0x01,0x04,0x00,0x00]
@@ -890,6 +912,14 @@
 0x7e,0x83,0x86,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00
 # GFX11: v_cmpx_ge_f16_e64 -|0xfe0b|, -|vcc_hi| clamp ; encoding: [0x7e,0x83,0x86,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00]
 
+0x7e,0x08,0x86,0xd4,0x01,0x05,0x02,0x00
+# GFX11-REAL16: v_cmpx_ge_f16_e64 v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x7e,0x08,0x86,0xd4,0x01,0x05,0x02,0x00]
+# GFX11-FAKE16: v_cmpx_ge_f16_e64 v1, v2                ; encoding: [0x7e,0x00,0x86,0xd4,0x01,0x05,0x02,0x00]
+
+0x7e,0x10,0x86,0xd4,0xff,0xff,0x03,0x00
+# GFX11-REAL16: v_cmpx_ge_f16_e64 v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x7e,0x10,0x86,0xd4,0xff,0xff,0x03,0x00]
+# GFX11-FAKE16: v_cmpx_ge_f16_e64 v255, v255            ; encoding: [0x7e,0x00,0x86,0xd4,0xff,0xff,0x03,0x00]
+
 0x7e,0x00,0x96,0xd4,0x01,0x05,0x02,0x00
 # GFX11: v_cmpx_ge_f32_e64 v1, v2                ; encoding: [0x7e,0x00,0x96,0xd4,0x01,0x05,0x02,0x00]
 
@@ -1244,10 +1274,12 @@
 # GFX11: v_cmpx_ge_u64_e64 0xaf123456, vcc       ; encoding: [0x7e,0x00,0xde,0xd4,0xff,0xd4,0x00,0x00,0x56,0x34,0x12,0xaf]
 
 0x7e,0x00,0x84,0xd4,0x01,0x05,0x02,0x00
-# GFX11: v_cmpx_gt_f16_e64 v1, v2                ; encoding: [0x7e,0x00,0x84,0xd4,0x01,0x05,0x02,0x00]
+# GFX11-REAL16: v_cmpx_gt_f16_e64 v1.l, v2.l            ; encoding: [0x7e,0x00,0x84,0xd4,0x01,0x05,0x02,0x00]
+# GFX11-FAKE16: v_cmpx_gt_f16_e64 v1, v2                ; encoding: [0x7e,0x00,0x84,0xd4,0x01,0x05,0x02,0x00]
 
 0x7e,0x00,0x84,0xd4,0xff,0xff,0x03,0x00
-# GFX11: v_cmpx_gt_f16_e64 v255, v255            ; encoding: [0x7e,0x00,0x84,0xd4,0xff,0xff,0x03,0x00]
+# GFX11-REAL16: v_cmpx_gt_f16_e64 v255.l, v255.l        ; encoding: [0x7e,0x00,0x84,0xd4,0xff,0xff,0x03,0x00]
+# GFX11-FAKE16: v_cmpx_gt_f16_e64 v255, v255            ; encoding: [0x7e,0x00,0x84,0xd4,0xff,0xff,0x03,0x00]
 
 0x7e,0x00,0x84,0xd4,0x01,0x04,0x00,0x00
 # GFX11: v_cmpx_gt_f16_e64 s1, s2                ; encoding: [0x7e,0x00,0x84,0xd4,0x01,0x04,0x00,0x00]
@@ -1288,6 +1320,14 @@
 0x7e,0x83,0x84,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00
 # GFX11: v_cmpx_gt_f16_e64 -|0xfe0b|, -|vcc_hi| clamp ; encoding: [0x7e,0x83,0x84,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00]
 
+0x7e,0x08,0x84,0xd4,0x01,0x05,0x02,0x00
+# GFX11-REAL16: v_cmpx_gt_f16_e64 v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x7e,0x08,0x84,0xd4,0x01,0x05,0x02,0x00]
+# GFX11-FAKE16: v_cmpx_gt_f16_e64 v1, v2                ; encoding: [0x7e,0x00,0x84,0xd4,0x01,0x05,0x02,0x00]
+
+0x7e,0x10,0x84,0xd4,0xff,0xff,0x03,0x00
+# GFX11-REAL16: v_cmpx_gt_f16_e64 v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x7e,0x10,0x84,0xd4,0xff,0xff,0x03,0x00]
+# GFX11-FAKE16: v_cmpx_gt_f16_e64 v255, v255            ; encoding: [0x7e,0x00,0x84,0xd4,0xff,0xff,0x03,0x00]
+
 0x7e,0x00,0x94,0xd4,0x01,0x05,0x02,0x00
 # GFX11: v_cmpx_gt_f32_e64 v1, v2                ; encoding: [0x7e,0x00,0x94,0xd4,0x01,0x05,0x02,0x00]
 
@@ -1642,10 +1682,12 @@
 # GFX11: v_cmpx_gt_u64_e64 0xaf123456, vcc       ; encoding: [0x7e,0x00,0xdc,0xd4,0xff,0xd4,0x00,0x00,0x56,0x34,0x12,0xaf]
 
 0x7e,0x00,0x83,0xd4,0x01,0x05,0x02,0x00
-# GFX11: v_cmpx_le_f16_e64 v1, v2                ; encoding: [0x7e,0x00,0x83,0xd4,0x01,0x05,0x02,0x00]
+# GFX11-REAL16: v_cmpx_le_f16_e64 v1.l, v2.l            ; encoding: [0x7e,0x00,0x83,0xd4,0x01,0x05,0x02,0x00]
+# GFX11-FAKE16: v_cmpx_le_f16_e64 v1, v2                ; encoding: [0x7e,0x00,0x83,0xd4,0x01,0x05,0x02,0x00]
 
 0x7e,0x00,0x83,0xd4,0xff,0xff,0x03,0x00
-# GFX11: v_cmpx_le_f16_e64 v255, v255            ; encoding: [0x7e,0x00,0x83,0xd4,0xff,0xff,0x03,0x00]
+# GFX11-REAL16: v_cmpx_le_f16_e64 v255.l, v255.l        ; encoding: [0x7e,0x00,0x83,0xd4,0xff,0xff,0x03,0x00]
+# GFX11-FAKE16: v_cmpx_le_f16_e64 v255, v255            ; encoding: [0x7e,0x00,0x83,0xd4,0xff,0xff,0x03,0x00]
 
 0x7e,0x00,0x83,0xd4,0x01,0x04,0x00,0x00
 # GFX11: v_cmpx_le_f16_e64 s1, s2                ; encoding: [0x7e,0x00,0x83,0xd4,0x01,0x04,0x00,0x00]
@@ -1686,6 +1728,14 @@
 0x7e,0x83,0x83,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00
 # GFX11: v_cmpx_le_f16_e64 -|0xfe0b|, -|vcc_hi| clamp ; encoding: [0x7e,0x83,0x83,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00]
 
+0x7e,0x08,0x83,0xd4,0x01,0x05,0x02,0x00
+# GFX11-REAL16: v_cmpx_le_f16_e64 v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x7e,0x08,0x83,0xd4,0x01,0x05,0x02,0x00]
+# GFX11-FAKE16: v_cmpx_le_f16_e64 v1, v2                ; encoding: [0x7e,0x00,0x83,0xd4,0x01,0x05,0x02,0x00]
+
+0x7e,0x10,0x83,0xd4,0xff,0xff,0x03,0x00
+# GFX11-REAL16: v_cmpx_le_f16_e64 v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x7e,0x10,0x83,0xd4,0xff,0xff,0x03,0x00]
+# GFX11-FAKE16: v_cmpx_le_f16_e64 v255, v255            ; encoding: [0x7e,0x00,0x83,0xd4,0xff,0xff,0x03,0x00]
+
 0x7e,0x00,0x93,0xd4,0x01,0x05,0x02,0x00
 # GFX11: v_cmpx_le_f32_e64 v1, v2                ; encoding: [0x7e,0x00,0x93,0xd4,0x01,0x05,0x02,0x00]
 
@@ -2040,10 +2090,12 @@
 # GFX11: v_cmpx_le_u64_e64 0xaf123456, vcc       ; encoding: [0x7e,0x00,0xdb,0xd4,0xff,0xd4,0x00,0x00,0x56,0x34,0x12,0xaf]
 
 0x7e,0x00,0x85,0xd4,0x01,0x05,0x02,0x00
-# GFX11: v_cmpx_lg_f16_e64 v1, v2                ; encoding: [0x7e,0x00,0x85,0xd4,0x01,0x05,0x02,0x00]
+# GFX11-REAL16: v_cmpx_lg_f16_e64 v1.l, v2.l            ; encoding: [0x7e,0x00,0x85,0xd4,0x01,0x05,0x02,0x00]
+# GFX11-FAKE16: v_cmpx_lg_f16_e64 v1, v2                ; encoding: [0x7e,0x00,0x85,0xd4,0x01,0x05,0x02,0x00]
 
 0x7e,0x00,0x85,0xd4,0xff,0xff,0x03,0x00
-# GFX11: v_cmpx_lg_f16_e64 v255, v255            ; encoding: [0x7e,0x00,0x85,0xd4,0xff,0xff,0x03,0x00]
+# GFX11-REAL16: v_cmpx_lg_f16_e64 v255.l, v255.l        ; encoding: [0x7e,0x00,0x85,0xd4,0xff,0xff,0x03,0x00]
+# GFX11-FAKE16: v_cmpx_lg_f16_e64 v255, v255            ; encoding: [0x7e,0x00,0x85,0xd4,0xff,0xff,0x03,0x00]
 
 0x7e,0x00,0x85,0xd4,0x01,0x04,0x00,0x00
 # GFX11: v_cmpx_lg_f16_e64 s1, s2                ; encoding: [0x7e,0x00,0x85,0xd4,0x01,0x04,0x00,0x00]
@@ -2084,6 +2136,14 @@
 0x7e,0x83,0x85,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00
 # GFX11: v_cmpx_lg_f16_e64 -|0xfe0b|, -|vcc_hi| clamp ; encoding: [0x7e,0x83,0x85,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00]
 
+0x7e,0x08,0x85,0xd4,0x01,0x05,0x02,0x00
+# GFX11-REAL16: v_cmpx_lg_f16_e64 v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x7e,0x08,0x85,0xd4,0x01,0x05,0x02,0x00]
+# GFX11-FAKE16: v_cmpx_lg_f16_e64 v1, v2                ; encoding: [0x7e,0x00,0x85,0xd4,0x01,0x05,0x02,0x00]
+
+0x7e,0x10,0x85,0xd4,0xff,0xff,0x03,0x00
+# GFX11-REAL16: v_cmpx_lg_f16_e64 v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x7e,0x10,0x85,0xd4,0xff,0xff,0x03,0x00]
+# GFX11-FAKE16: v_cmpx_lg_f16_e64 v255, v255            ; encoding: [0x7e,0x00,0x85,0xd4,0xff,0xff,0x03,0x00]
+
 0x7e,0x00,0x95,0xd4,0x01,0x05,0x02,0x00
 # GFX11: v_cmpx_lg_f32_e64 v1, v2                ; encoding: [0x7e,0x00,0x95,0xd4,0x01,0x05,0x02,0x00]
 
@@ -2846,10 +2906,12 @@
 # GFX11: v_cmpx_ne_u64_e64 0xaf123456, vcc       ; encoding: [0x7e,0x00,0xdd,0xd4,0xff,0xd4,0x00,0x00,0x56,0x34,0x12,0xaf]
 
 0x7e,0x00,0x8d,0xd4,0x01,0x05,0x02,0x00
-# GFX11: v_cmpx_neq_f16_e64 v1, v2               ; encoding: [0x7e,0x00,0x8d,0xd4,0x01,0x05,0x02,0x00]
+# GFX11-REAL16: v_cmpx_neq_f16_e64 v1.l, v2.l           ; encoding: [0x7e,0x00,0x8d,0xd4,0x01,0x05,0x02,0x00]
+# GFX11-FAKE16: v_cmpx_neq_f16_e64 v1, v2               ; encoding: [0x7e,0x00,0x8d,0xd4,0x01,0x05,0x02,0x00]
 
 0x7e,0x00,0x8d,0xd4,0xff,0xff,0x03,0x00
-# GFX11: v_cmpx_neq_f16_e64 v255, v255           ; encoding: [0x7e,0x00,0x8d,0xd4,0xff,0xff,0x03,0x00]
+# GFX11-REAL16: v_cmpx_neq_f16_e64 v255.l, v255.l       ; encoding: [0x7e,0x00,0x8d,0xd4,0xff,0xff,0x03,0x00]
+# GFX11-FAKE16: v_cmpx_neq_f16_e64 v255, v255           ; encoding: [0x7e,0x00,0x8d,0xd4,0xff,0xff,0x03,0x00]
 
 0x7e,0x00,0x8d,0xd4,0x01,0x04,0x00,0x00
 # GFX11: v_cmpx_neq_f16_e64 s1, s2               ; encoding: [0x7e,0x00,0x8d,0xd4,0x01,0x04,0x00,0x00]
@@ -2890,6 +2952,14 @@
 0x7e,0x83,0x8d,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00
 # GFX11: v_cmpx_neq_f16_e64 -|0xfe0b|, -|vcc_hi| clamp ; encoding: [0x7e,0x83,0x8d,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00]
 
+0x7e,0x08,0x8d,0xd4,0x01,0x05,0x02,0x00
+# GFX11-REAL16: v_cmpx_neq_f16_e64 v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x7e,0x08,0x8d,0xd4,0x01,0x05,0x02,0x00]
+# GFX11-FAKE16: v_cmpx_neq_f16_e64 v1, v2               ; encoding: [0x7e,0x00,0x8d,0xd4,0x01,0x05,0x02,0x00]
+
+0x7e,0x10,0x8d,0xd4,0xff,0xff,0x03,0x00
+# GFX11-REAL16: v_cmpx_neq_f16_e64 v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x7e,0x10,0x8d,0xd4,0xff,0xff,0x03,0x00]
+# GFX11-FAKE16: v_cmpx_neq_f16_e64 v255, v255           ; encoding: [0x7e,0x00,0x8d,0xd4,0xff,0xff,0x03,0x00]
+
 0x7e,0x00,0x9d,0xd4,0x01,0x05,0x02,0x00
 # GFX11: v_cmpx_neq_f32_e64 v1, v2               ; encoding: [0x7e,0x00,0x9d,0xd4,0x01,0x05,0x02,0x00]
 
@@ -2972,10 +3042,12 @@
 # GFX11: v_cmpx_neq_f64_e64 0xaf123456, -|vcc| clamp ; encoding: [0x7e,0x82,0xad,0xd4,0xff,0xd4,0x00,0x40,0x56,0x34,0x12,0xaf]
 
 0x7e,0x00,0x89,0xd4,0x01,0x05,0x02,0x00
-# GFX11: v_cmpx_nge_f16_e64 v1, v2               ; encoding: [0x7e,0x00,0x89,0xd4,0x01,0x05,0x02,0x00]
+# GFX11-REAL16: v_cmpx_nge_f16_e64 v1.l, v2.l           ; encoding: [0x7e,0x00,0x89,0xd4,0x01,0x05,0x02,0x00]
+# GFX11-FAKE16: v_cmpx_nge_f16_e64 v1, v2               ; encoding: [0x7e,0x00,0x89,0xd4,0x01,0x05,0x02,0x00]
 
 0x7e,0x00,0x89,0xd4,0xff,0xff,0x03,0x00
-# GFX11: v_cmpx_nge_f16_e64 v255, v255           ; encoding: [0x7e,0x00,0x89,0xd4,0xff,0xff,0x03,0x00]
+# GFX11-REAL16: v_cmpx_nge_f16_e64 v255.l, v255.l       ; encoding: [0x7e,0x00,0x89,0xd4,0xff,0xff,0x03,0x00]
+# GFX11-FAKE16: v_cmpx_nge_f16_e64 v255, v255           ; encoding: [0x7e,0x00,0x89,0xd4,0xff,0xff,0x03,0x00]
 
 0x7e,0x00,0x89,0xd4,0x01,0x04,0x00,0x00
 # GFX11: v_cmpx_nge_f16_e64 s1, s2               ; encoding: [0x7e,0x00,0x89,0xd4,0x01,0x04,0x00,0x00]
@@ -3016,6 +3088,14 @@
 0x7e,0x83,0x89,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00
 # GFX11: v_cmpx_nge_f16_e64 -|0xfe0b|, -|vcc_hi| clamp ; encoding: [0x7e,0x83,0x89,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00]
 
+0x7e,0x08,0x89,0xd4,0x01,0x05,0x02,0x00
+# GFX11-REAL16: v_cmpx_nge_f16_e64 v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x7e,0x08,0x89,0xd4,0x01,0x05,0x02,0x00]
+# GFX11-FAKE16: v_cmpx_nge_f16_e64 v1, v2               ; encoding: [0x7e,0x00,0x89,0xd4,0x01,0x05,0x02,0x00]
+
+0x7e,0x10,0x89,0xd4,0xff,0xff,0x03,0x00
+# GFX11-REAL16: v_cmpx_nge_f16_e64 v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x7e,0x10,0x89,0xd4,0xff,0xff,0x03,0x00]
+# GFX11-FAKE16: v_cmpx_nge_f16_e64 v255, v255           ; encoding: [0x7e,0x00,0x89,0xd4,0xff,0xff,0x03,0x00]
+
 0x7e,0x00,0x99,0xd4,0x01,0x05,0x02,0x00
 # GFX11: v_cmpx_nge_f32_e64 v1, v2               ; encoding: [0x7e,0x00,0x99,0xd4,0x01,0x05,0x02,0x00]
 
@@ -3098,10 +3178,12 @@
 # GFX11: v_cmpx_nge_f64_e64 0xaf123456, -|vcc| clamp ; encoding: [0x7e,0x82,0xa9,0xd4,0xff,0xd4,0x00,0x40,0x56,0x34,0x12,0xaf]
 
 0x7e,0x00,0x8b,0xd4,0x01,0x05,0x02,0x00
-# GFX11: v_cmpx_ngt_f16_e64 v1, v2               ; encoding: [0x7e,0x00,0x8b,0xd4,0x01,0x05,0x02,0x00]
+# GFX11-REAL16: v_cmpx_ngt_f16_e64 v1.l, v2.l           ; encoding: [0x7e,0x00,0x8b,0xd4,0x01,0x05,0x02,0x00]
+# GFX11-FAKE16: v_cmpx_ngt_f16_e64 v1, v2               ; encoding: [0x7e,0x00,0x8b,0xd4,0x01,0x05,0x02,0x00]
 
 0x7e,0x00,0x8b,0xd4,0xff,0xff,0x03,0x00
-# GFX11: v_cmpx_ngt_f16_e64 v255, v255           ; encoding: [0x7e,0x00,0x8b,0xd4,0xff,0xff,0x03,0x00]
+# GFX11-REAL16: v_cmpx_ngt_f16_e64 v255.l, v255.l       ; encoding: [0x7e,0x00,0x8b,0xd4,0xff,0xff,0x03,0x00]
+# GFX11-FAKE16: v_cmpx_ngt_f16_e64 v255, v255           ; encoding: [0x7e,0x00,0x8b,0xd4,0xff,0xff,0x03,0x00]
 
 0x7e,0x00,0x8b,0xd4,0x01,0x04,0x00,0x00
 # GFX11: v_cmpx_ngt_f16_e64 s1, s2               ; encoding: [0x7e,0x00,0x8b,0xd4,0x01,0x04,0x00,0x00]
@@ -3142,6 +3224,14 @@
 0x7e,0x83,0x8b,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00
 # GFX11: v_cmpx_ngt_f16_e64 -|0xfe0b|, -|vcc_hi| clamp ; encoding: [0x7e,0x83,0x8b,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00]
 
+0x7e,0x08,0x8b,0xd4,0x01,0x05,0x02,0x00
+# GFX11-REAL16: v_cmpx_ngt_f16_e64 v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x7e,0x08,0x8b,0xd4,0x01,0x05,0x02,0x00]
+# GFX11-FAKE16: v_cmpx_ngt_f16_e64 v1, v2               ; encoding: [0x7e,0x00,0x8b,0xd4,0x01,0x05,0x02,0x00]
+
+0x7e,0x10,0x8b,0xd4,0xff,0xff,0x03,0x00
+# GFX11-REAL16: v_cmpx_ngt_f16_e64 v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x7e,0x10,0x8b,0xd4,0xff,0xff,0x03,0x00]
+# GFX11-FAKE16: v_cmpx_ngt_f16_e64 v255, v255           ; encoding: [0x7e,0x00,0x8b,0xd4,0xff,0xff,0x03,0x00]
+
 0x7e,0x00,0x9b,0xd4,0x01,0x05,0x02,0x00
 # GFX11: v_cmpx_ngt_f32_e64 v1, v2               ; encoding: [0x7e,0x00,0x9b,0xd4,0x01,0x05,0x02,0x00]
 
@@ -3224,10 +3314,12 @@
 # GFX11: v_cmpx_ngt_f64_e64 0xaf123456, -|vcc| clamp ; encoding: [0x7e,0x82,0xab,0xd4,0xff,0xd4,0x00,0x40,0x56,0x34,0x12,0xaf]
 
 0x7e,0x00,0x8c,0xd4,0x01,0x05,0x02,0x00
-# GFX11: v_cmpx_nle_f16_e64 v1, v2               ; encoding: [0x7e,0x00,0x8c,0xd4,0x01,0x05,0x02,0x00]
+# GFX11-REAL16: v_cmpx_nle_f16_e64 v1.l, v2.l           ; encoding: [0x7e,0x00,0x8c,0xd4,0x01,0x05,0x02,0x00]
+# GFX11-FAKE16: v_cmpx_nle_f16_e64 v1, v2               ; encoding: [0x7e,0x00,0x8c,0xd4,0x01,0x05,0x02,0x00]
 
 0x7e,0x00,0x8c,0xd4,0xff,0xff,0x03,0x00
-# GFX11: v_cmpx_nle_f16_e64 v255, v255           ; encoding: [0x7e,0x00,0x8c,0xd4,0xff,0xff,0x03,0x00]
+# GFX11-REAL16: v_cmpx_nle_f16_e64 v255.l, v255.l       ; encoding: [0x7e,0x00,0x8c,0xd4,0xff,0xff,0x03,0x00]
+# GFX11-FAKE16: v_cmpx_nle_f16_e64 v255, v255           ; encoding: [0x7e,0x00,0x8c,0xd4,0xff,0xff,0x03,0x00]
 
 0x7e,0x00,0x8c,0xd4,0x01,0x04,0x00,0x00
 # GFX11: v_cmpx_nle_f16_e64 s1, s2               ; encoding: [0x7e,0x00,0x8c,0xd4,0x01,0x04,0x00,0x00]
@@ -3268,6 +3360,14 @@
 0x7e,0x83,0x8c,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00
 # GFX11: v_cmpx_nle_f16_e64 -|0xfe0b|, -|vcc_hi| clamp ; encoding: [0x7e,0x83,0x8c,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00]
 
+0x7e,0x08,0x8c,0xd4,0x01,0x05,0x02,0x00
+# GFX11-REAL16: v_cmpx_nle_f16_e64 v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x7e,0x08,0x8c,0xd4,0x01,0x05,0x02,0x00]
+# GFX11-FAKE16: v_cmpx_nle_f16_e64 v1, v2               ; encoding: [0x7e,0x00,0x8c,0xd4,0x01,0x05,0x02,0x00]
+
+0x7e,0x10,0x8c,0xd4,0xff,0xff,0x03,0x00
+# GFX11-REAL16: v_cmpx_nle_f16_e64 v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x7e,0x10,0x8c,0xd4,0xff,0xff,0x03,0x00]
+# GFX11-FAKE16: v_cmpx_nle_f16_e64 v255, v255           ; encoding: [0x7e,0x00,0x8c,0xd4,0xff,0xff,0x03,0x00]
+
 0x7e,0x00,0x9c,0xd4,0x01,0x05,0x02,0x00
 # GFX11: v_cmpx_nle_f32_e64 v1, v2               ; encoding: [0x7e,0x00,0x9c,0xd4,0x01,0x05,0x02,0x00]
 
@@ -3350,10 +3450,12 @@
 # GFX11: v_cmpx_nle_f64_e64 0xaf123456, -|vcc| clamp ; encoding: [0x7e,0x82,0xac,0xd4,0xff,0xd4,0x00,0x40,0x56,0x34,0x12,0xaf]
 
 0x7e,0x00,0x8a,0xd4,0x01,0x05,0x02,0x00
-# GFX11: v_cmpx_nlg_f16_e64 v1, v2               ; encoding: [0x7e,0x00,0x8a,0xd4,0x01,0x05,0x02,0x00]
+# GFX11-REAL16: v_cmpx_nlg_f16_e64 v1.l, v2.l           ; encoding: [0x7e,0x00,0x8a,0xd4,0x01,0x05,0x02,0x00]
+# GFX11-FAKE16: v_cmpx_nlg_f16_e64 v1, v2               ; encoding: [0x7e,0x00,0x8a,0xd4,0x01,0x05,0x02,0x00]
 
 0x7e,0x00,0x8a,0xd4,0xff,0xff,0x03,0x00
-# GFX11: v_cmpx_nlg_f16_e64 v255, v255           ; encoding: [0x7e,0x00,0x8a,0xd4,0xff,0xff,0x03,0x00]
+# GFX11-REAL16: v_cmpx_nlg_f16_e64 v255.l, v255.l       ; encoding: [0x7e,0x00,0x8a,0xd4,0xff,0xff,0x03,0x00]
+# GFX11-FAKE16: v_cmpx_nlg_f16_e64 v255, v255           ; encoding: [0x7e,0x00,0x8a,0xd4,0xff,0xff,0x03,0x00]
 
 0x7e,0x00,0x8a,0xd4,0x01,0x04,0x00,0x00
 # GFX11: v_cmpx_nlg_f16_e64 s1, s2               ; encoding: [0x7e,0x00,0x8a,0xd4,0x01,0x04,0x00,0x00]
@@ -3394,6 +3496,14 @@
 0x7e,0x83,0x8a,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00
 # GFX11: v_cmpx_nlg_f16_e64 -|0xfe0b|, -|vcc_hi| clamp ; encoding: [0x7e,0x83,0x8a,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00]
 
+0x7e,0x08,0x8a,0xd4,0x01,0x05,0x02,0x00
+# GFX11-REAL16: v_cmpx_nlg_f16_e64 v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x7e,0x08,0x8a,0xd4,0x01,0x05,0x02,0x00]
+# GFX11-FAKE16: v_cmpx_nlg_f16_e64 v1, v2               ; encoding: [0x7e,0x00,0x8a,0xd4,0x01,0x05,0x02,0x00]
+
+0x7e,0x10,0x8a,0xd4,0xff,0xff,0x03,0x00
+# GFX11-REAL16: v_cmpx_nlg_f16_e64 v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x7e,0x10,0x8a,0xd4,0xff,0xff,0x03,0x00]
+# GFX11-FAKE16: v_cmpx_nlg_f16_e64 v255, v255           ; encoding: [0x7e,0x00,0x8a,0xd4,0xff,0xff,0x03,0x00]
+
 0x7e,0x00,0x9a,0xd4,0x01,0x05,0x02,0x00
 # GFX11: v_cmpx_nlg_f32_e64 v1, v2               ; encoding: [0x7e,0x00,0x9a,0xd4,0x01,0x05,0x02,0x00]
 
@@ -3476,10 +3586,12 @@
 # GFX11: v_cmpx_nlg_f64_e64 0xaf123456, -|vcc| clamp ; encoding: [0x7e,0x82,0xaa,0xd4,0xff,0xd4,0x00,0x40,0x56,0x34,0x12,0xaf]
 
 0x7e,0x00,0x8e,0xd4,0x01,0x05,0x02,0x00
-# GFX11: v_cmpx_nlt_f16_e64 v1, v2               ; encoding: [0x7e,0x00,0x8e,0xd4,0x01,0x05,0x02,0x00]
+# GFX11-REAL16: v_cmpx_nlt_f16_e64 v1.l, v2.l           ; encoding: [0x7e,0x00,0x8e,0xd4,0x01,0x05,0x02,0x00]
+# GFX11-FAKE16: v_cmpx_nlt_f16_e64 v1, v2               ; encoding: [0x7e,0x00,0x8e,0xd4,0x01,0x05,0x02,0x00]
 
 0x7e,0x00,0x8e,0xd4,0xff,0xff,0x03,0x00
-# GFX11: v_cmpx_nlt_f16_e64 v255, v255           ; encoding: [0x7e,0x00,0x8e,0xd4,0xff,0xff,0x03,0x00]
+# GFX11-REAL16: v_cmpx_nlt_f16_e64 v255.l, v255.l       ; encoding: [0x7e,0x00,0x8e,0xd4,0xff,0xff,0x03,0x00]
+# GFX11-FAKE16: v_cmpx_nlt_f16_e64 v255, v255           ; encoding: [0x7e,0x00,0x8e,0xd4,0xff,0xff,0x03,0x00]
 
 0x7e,0x00,0x8e,0xd4,0x01,0x04,0x00,0x00
 # GFX11: v_cmpx_nlt_f16_e64 s1, s2               ; encoding: [0x7e,0x00,0x8e,0xd4,0x01,0x04,0x00,0x00]
@@ -3520,6 +3632,14 @@
 0x7e,0x83,0x8e,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00
 # GFX11: v_cmpx_nlt_f16_e64 -|0xfe0b|, -|vcc_hi| clamp ; encoding: [0x7e,0x83,0x8e,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00]
 
+0x7e,0x08,0x8e,0xd4,0x01,0x05,0x02,0x00
+# GFX11-REAL16: v_cmpx_nlt_f16_e64 v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x7e,0x08,0x8e,0xd4,0x01,0x05,0x02,0x00]
+# GFX11-FAKE16: v_cmpx_nlt_f16_e64 v1, v2               ; encoding: [0x7e,0x00,0x8e,0xd4,0x01,0x05,0x02,0x00]
+
+0x7e,0x10,0x8e,0xd4,0xff,0xff,0x03,0x00
+# GFX11-REAL16: v_cmpx_nlt_f16_e64 v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x7e,0x10,0x8e,0xd4,0xff,0xff,0x03,0x00]
+# GFX11-FAKE16: v_cmpx_nlt_f16_e64 v255, v255           ; encoding: [0x7e,0x00,0x8e,0xd4,0xff,0xff,0x03,0x00]
+
 0x7e,0x00,0x9e,0xd4,0x01,0x05,0x02,0x00
 # GFX11: v_cmpx_nlt_f32_e64 v1, v2               ; encoding: [0x7e,0x00,0x9e,0xd4,0x01,0x05,0x02,0x00]
 
@@ -3602,10 +3722,12 @@
 # GFX11: v_cmpx_nlt_f64_e64 0xaf123456, -|vcc| clamp ; encoding: [0x7e,0x82,0xae,0xd4,0xff,0xd4,0x00,0x40,0x56,0x34,0x12,0xaf]
 
 0x7e,0x00,0x87,0xd4,0x01,0x05,0x02,0x00
-# GFX11: v_cmpx_o_f16_e64 v1, v2                 ; encoding: [0x7e,0x00,0x87,0xd4,0x01,0x05,0x02,0x00]
+# GFX11-REAL16: v_cmpx_o_f16_e64 v1.l, v2.l             ; encoding: [0x7e,0x00,0x87,0xd4,0x01,0x05,0x02,0x00]
+# GFX11-FAKE16: v_cmpx_o_f16_e64 v1, v2                 ; encoding: [0x7e,0x00,0x87,0xd4,0x01,0x05,0x02,0x00]
 
 0x7e,0x00,0x87,0xd4,0xff,0xff,0x03,0x00
-# GFX11: v_cmpx_o_f16_e64 v255, v255             ; encoding: [0x7e,0x00,0x87,0xd4,0xff,0xff,0x03,0x00]
+# GFX11-REAL16: v_cmpx_o_f16_e64 v255.l, v255.l         ; encoding: [0x7e,0x00,0x87,0xd4,0xff,0xff,0x03,0x00]
+# GFX11-FAKE16: v_cmpx_o_f16_e64 v255, v255             ; encoding: [0x7e,0x00,0x87,0xd4,0xff,0xff,0x03,0x00]
 
 0x7e,0x00,0x87,0xd4,0x01,0x04,0x00,0x00
 # GFX11: v_cmpx_o_f16_e64 s1, s2                 ; encoding: [0x7e,0x00,0x87,0xd4,0x01,0x04,0x00,0x00]
@@ -3646,6 +3768,14 @@
 0x7e,0x83,0x87,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00
 # GFX11: v_cmpx_o_f16_e64 -|0xfe0b|, -|vcc_hi| clamp ; encoding: [0x7e,0x83,0x87,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00]
 
+0x7e,0x08,0x87,0xd4,0x01,0x05,0x02,0x00
+# GFX11-REAL16: v_cmpx_o_f16_e64 v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x7e,0x08,0x87,0xd4,0x01,0x05,0x02,0x00]
+# GFX11-FAKE16: v_cmpx_o_f16_e64 v1, v2                 ; encoding: [0x7e,0x00,0x87,0xd4,0x01,0x05,0x02,0x00]
+
+0x7e,0x10,0x87,0xd4,0xff,0xff,0x03,0x00
+# GFX11-REAL16: v_cmpx_o_f16_e64 v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x7e,0x10,0x87,0xd4,0xff,0xff,0x03,0x00]
+# GFX11-FAKE16: v_cmpx_o_f16_e64 v255, v255             ; encoding: [0x7e,0x00,0x87,0xd4,0xff,0xff,0x03,0x00]
+
 0x7e,0x00,0x97,0xd4,0x01,0x05,0x02,0x00
 # GFX11: v_cmpx_o_f32_e64 v1, v2                 ; encoding: [0x7e,0x00,0x97,0xd4,0x01,0x05,0x02,0x00]
 
@@ -3728,10 +3858,12 @@
 # GFX11: v_cmpx_o_f64_e64 0xaf123456, -|vcc| clamp ; encoding: [0x7e,0x82,0xa7,0xd4,0xff,0xd4,0x00,0x40,0x56,0x34,0x12,0xaf]
 
 0x7e,0x00,0x8f,0xd4,0x01,0x05,0x02,0x00
-# GFX11: v_cmpx_t_f16_e64 v1, v2                 ; encoding: [0x7e,0x00,0x8f,0xd4,0x01,0x05,0x02,0x00]
+# GFX11-REAL16: v_cmpx_t_f16_e64 v1.l, v2.l             ; encoding: [0x7e,0x00,0x8f,0xd4,0x01,0x05,0x02,0x00]
+# GFX11-FAKE16: v_cmpx_t_f16_e64 v1, v2                 ; encoding: [0x7e,0x00,0x8f,0xd4,0x01,0x05,0x02,0x00]
 
 0x7e,0x00,0x8f,0xd4,0xff,0xff,0x03,0x00
-# GFX11: v_cmpx_t_f16_e64 v255, v255             ; encoding: [0x7e,0x00,0x8f,0xd4,0xff,0xff,0x03,0x00]
+# GFX11-REAL16: v_cmpx_t_f16_e64 v255.l, v255.l         ; encoding: [0x7e,0x00,0x8f,0xd4,0xff,0xff,0x03,0x00]
+# GFX11-FAKE16: v_cmpx_t_f16_e64 v255, v255             ; encoding: [0x7e,0x00,0x8f,0xd4,0xff,0xff,0x03,0x00]
 
 0x7e,0x00,0x8f,0xd4,0x01,0x04,0x00,0x00
 # GFX11: v_cmpx_t_f16_e64 s1, s2                 ; encoding: [0x7e,0x00,0x8f,0xd4,0x01,0x04,0x00,0x00]
@@ -3772,6 +3904,14 @@
 0x7e,0x83,0x8f,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00
 # GFX11: v_cmpx_t_f16_e64 -|0xfe0b|, -|vcc_hi| clamp ; encoding: [0x7e,0x83,0x8f,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00]
 
+0x7e,0x08,0x8f,0xd4,0x01,0x05,0x02,0x00
+# GFX11-REAL16: v_cmpx_t_f16_e64 v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x7e,0x08,0x8f,0xd4,0x01,0x05,0x02,0x00]
+# GFX11-FAKE16: v_cmpx_t_f16_e64 v1, v2                 ; encoding: [0x7e,0x00,0x8f,0xd4,0x01,0x05,0x02,0x00]
+
+0x7e,0x10,0x8f,0xd4,0xff,0xff,0x03,0x00
+# GFX11-REAL16: v_cmpx_t_f16_e64 v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x7e,0x10,0x8f,0xd4,0xff,0xff,0x03,0x00]
+# GFX11-FAKE16: v_cmpx_t_f16_e64 v255, v255             ; encoding: [0x7e,0x00,0x8f,0xd4,0xff,0xff,0x03,0x00]
+
 0x7e,0x00,0x9f,0xd4,0x01,0x05,0x02,0x00
 # GFX11: v_cmpx_t_f32_e64 v1, v2                 ; encoding: [0x7e,0x00,0x9f,0xd4,0x01,0x05,0x02,0x00]
 
@@ -4016,10 +4156,12 @@
 # GFX11: v_cmpx_t_u64_e64 0xaf123456, vcc        ; encoding: [0x7e,0x00,0xdf,0xd4,0xff,0xd4,0x00,0x00,0x56,0x34,0x12,0xaf]
 
 0x7e,0x00,0x88,0xd4,0x01,0x05,0x02,0x00
-# GFX11: v_cmpx_u_f16_e64 v1, v2                 ; encoding: [0x7e,0x00,0x88,0xd4,0x01,0x05,0x02,0x00]
+# GFX11-REAL16: v_cmpx_u_f16_e64 v1.l, v2.l             ; encoding: [0x7e,0x00,0x88,0xd4,0x01,0x05,0x02,0x00]
+# GFX11-FAKE16: v_cmpx_u_f16_e64 v1, v2                 ; encoding: [0x7e,0x00,0x88,0xd4,0x01,0x05,0x02,0x00]
 
 0x7e,0x00,0x88,0xd4,0xff,0xff,0x03,0x00
-# GFX11: v_cmpx_u_f16_e64 v255, v255             ; encoding: [0x7e,0x00,0x88,0xd4,0xff,0xff,0x03,0x00]
+# GFX11-REAL16: v_cmpx_u_f16_e64 v255.l, v255.l         ; encoding: [0x7e,0x00,0x88,0xd4,0xff,0xff,0x03,0x00]
+# GFX11-FAKE16: v_cmpx_u_f16_e64 v255, v255             ; encoding: [0x7e,0x00,0x88,0xd4,0xff,0xff,0x03,0x00]
 
 0x7e,0x00,0x88,0xd4,0x01,0x04,0x00,0x00
 # GFX11: v_cmpx_u_f16_e64 s1, s2                 ; encoding: [0x7e,0x00,0x88,0xd4,0x01,0x04,0x00,0x00]
@@ -4060,6 +4202,14 @@
 0x7e,0x83,0x88,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00
 # GFX11: v_cmpx_u_f16_e64 -|0xfe0b|, -|vcc_hi| clamp ; encoding: [0x7e,0x83,0x88,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00]
 
+0x7e,0x08,0x88,0xd4,0x01,0x05,0x02,0x00
+# GFX11-REAL16: v_cmpx_u_f16_e64 v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x7e,0x08,0x88,0xd4,0x01,0x05,0x02,0x00]
+# GFX11-FAKE16: v_cmpx_u_f16_e64 v1, v2                 ; encoding: [0x7e,0x00,0x88,0xd4,0x01,0x05,0x02,0x00]
+
+0x7e,0x10,0x88,0xd4,0xff,0xff,0x03,0x00
+# GFX11-REAL16: v_cmpx_u_f16_e64 v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x7e,0x10,0x88,0xd4,0xff,0xff,0x03,0x00]
+# GFX11-FAKE16: v_cmpx_u_f16_e64 v255, v255             ; encoding: [0x7e,0x00,0x88,0xd4,0xff,0xff,0x03,0x00]
+
 0x7e,0x00,0x98,0xd4,0x01,0x05,0x02,0x00
 # GFX11: v_cmpx_u_f32_e64 v1, v2                 ; encoding: [0x7e,0x00,0x98,0xd4,0x01,0x05,0x02,0x00]
 
diff --git a/llvm/test/MC/Disassembler/AMDGPU/gfx11_dasm_vopcx.txt b/llvm/test/MC/Disassembler/AMDGPU/gfx11_dasm_vopcx.txt
index f058a9b981625..3d16d19954ab9 100644
--- a/llvm/test/MC/Disassembler/AMDGPU/gfx11_dasm_vopcx.txt
+++ b/llvm/test/MC/Disassembler/AMDGPU/gfx11_dasm_vopcx.txt
@@ -166,49 +166,84 @@
 # GFX11: v_cmpx_class_f64_e32 0xaf123456, v255   ; encoding: [0xff,0xfe,0xff,0x7d,0x56,0x34,0x12,0xaf]
 
 0x01,0x05,0x04,0x7d
-# GFX11: v_cmpx_eq_f16_e32 v1, v2                ; encoding: [0x01,0x05,0x04,0x7d]
+# GFX11-REAL16: v_cmpx_eq_f16_e32 v1.l, v2.l            ; encoding: [0x01,0x05,0x04,0x7d]
+# GFX11-FAKE16: v_cmpx_eq_f16_e32 v1, v2                ; encoding: [0x01,0x05,0x04,0x7d]
 
 0x7f,0x05,0x04,0x7d
-# GFX11: v_cmpx_eq_f16_e32 v127, v2              ; encoding: [0x7f,0x05,0x04,0x7d]
+# GFX11-REAL16: v_cmpx_eq_f16_e32 v127.l, v2.l          ; encoding: [0x7f,0x05,0x04,0x7d]
+# GFX11-FAKE16: v_cmpx_eq_f16_e32 v127, v2              ; encoding: [0x7f,0x05,0x04,0x7d]
 
 0x01,0x04,0x04,0x7d
-# GFX11: v_cmpx_eq_f16_e32 s1, v2                ; encoding: [0x01,0x04,0x04,0x7d]
+# GFX11-REAL16: v_cmpx_eq_f16_e32 s1, v2.l              ; encoding: [0x01,0x04,0x04,0x7d]
+# GFX11-FAKE16: v_cmpx_eq_f16_e32 s1, v2                ; encoding: [0x01,0x04,0x04,0x7d]
 
 0x69,0x04,0x04,0x7d
-# GFX11: v_cmpx_eq_f16_e32 s105, v2              ; encoding: [0x69,0x04,0x04,0x7d]
+# GFX11-REAL16: v_cmpx_eq_f16_e32 s105, v2.l            ; encoding: [0x69,0x04,0x04,0x7d]
+# GFX11-FAKE16: v_cmpx_eq_f16_e32 s105, v2              ; encoding: [0x69,0x04,0x04,0x7d]
 
 0x6a,0x04,0x04,0x7d
-# GFX11: v_cmpx_eq_f16_e32 vcc_lo, v2            ; encoding: [0x6a,0x04,0x04,0x7d]
+# GFX11-REAL16: v_cmpx_eq_f16_e32 vcc_lo, v2.l          ; encoding: [0x6a,0x04,0x04,0x7d]
+# GFX11-FAKE16: v_cmpx_eq_f16_e32 vcc_lo, v2            ; encoding: [0x6a,0x04,0x04,0x7d]
 
 0x6b,0x04,0x04,0x7d
-# GFX11: v_cmpx_eq_f16_e32 vcc_hi, v2            ; encoding: [0x6b,0x04,0x04,0x7d]
+# GFX11-REAL16: v_cmpx_eq_f16_e32 vcc_hi, v2.l          ; encoding: [0x6b,0x04,0x04,0x7d]
+# GFX11-FAKE16: v_cmpx_eq_f16_e32 vcc_hi, v2            ; encoding: [0x6b,0x04,0x04,0x7d]
 
 0x7b,0x04,0x04,0x7d
-# GFX11: v_cmpx_eq_f16_e32 ttmp15, v2            ; encoding: [0x7b,0x04,0x04,0x7d]
+# GFX11-REAL16: v_cmpx_eq_f16_e32 ttmp15, v2.l          ; encoding: [0x7b,0x04,0x04,0x7d]
+# GFX11-FAKE16: v_cmpx_eq_f16_e32 ttmp15, v2            ; encoding: [0x7b,0x04,0x04,0x7d]
 
 0x7d,0x04,0x04,0x7d
-# GFX11: v_cmpx_eq_f16_e32 m0, v2                ; encoding: [0x7d,0x04,0x04,0x7d]
+# GFX11-REAL16: v_cmpx_eq_f16_e32 m0, v2.l              ; encoding: [0x7d,0x04,0x04,0x7d]
+# GFX11-FAKE16: v_cmpx_eq_f16_e32 m0, v2                ; encoding: [0x7d,0x04,0x04,0x7d]
 
 0x7e,0x04,0x04,0x7d
-# GFX11: v_cmpx_eq_f16_e32 exec_lo, v2           ; encoding: [0x7e,0x04,0x04,0x7d]
+# GFX11-REAL16: v_cmpx_eq_f16_e32 exec_lo, v2.l         ; encoding: [0x7e,0x04,0x04,0x7d]
+# GFX11-FAKE16: v_cmpx_eq_f16_e32 exec_lo, v2           ; encoding: [0x7e,0x04,0x04,0x7d]
 
 0x7f,0x04,0x04,0x7d
-# GFX11: v_cmpx_eq_f16_e32 exec_hi, v2           ; encoding: [0x7f,0x04,0x04,0x7d]
+# GFX11-REAL16: v_cmpx_eq_f16_e32 exec_hi, v2.l         ; encoding: [0x7f,0x04,0x04,0x7d]
+# GFX11-FAKE16: v_cmpx_eq_f16_e32 exec_hi, v2           ; encoding: [0x7f,0x04,0x04,0x7d]
 
 0x7c,0x04,0x04,0x7d
-# GFX11: v_cmpx_eq_f16_e32 null, v2              ; encoding: [0x7c,0x04,0x04,0x7d]
+# GFX11-REAL16: v_cmpx_eq_f16_e32 null, v2.l            ; encoding: [0x7c,0x04,0x04,0x7d]
+# GFX11-FAKE16: v_cmpx_eq_f16_e32 null, v2              ; encoding: [0x7c,0x04,0x04,0x7d]
 
 0xc1,0x04,0x04,0x7d
-# GFX11: v_cmpx_eq_f16_e32 -1, v2                ; encoding: [0xc1,0x04,0x04,0x7d]
+# GFX11-REAL16: v_cmpx_eq_f16_e32 -1, v2.l              ; encoding: [0xc1,0x04,0x04,0x7d]
+# GFX11-FAKE16: v_cmpx_eq_f16_e32 -1, v2                ; encoding: [0xc1,0x04,0x04,0x7d]
 
 0xf0,0x04,0x04,0x7d
-# GFX11: v_cmpx_eq_f16_e32 0.5, v2               ; encoding: [0xf0,0x04,0x04,0x7d]
+# GFX11-REAL16: v_cmpx_eq_f16_e32 0.5, v2.l             ; encoding: [0xf0,0x04,0x04,0x7d]
+# GFX11-FAKE16: v_cmpx_eq_f16_e32 0.5, v2               ; encoding: [0xf0,0x04,0x04,0x7d]
 
 0xfd,0x04,0x04,0x7d
-# GFX11: v_cmpx_eq_f16_e32 src_scc, v2           ; encoding: [0xfd,0x04,0x04,0x7d]
+# GFX11-REAL16: v_cmpx_eq_f16_e32 src_scc, v2.l         ; encoding: [0xfd,0x04,0x04,0x7d]
+# GFX11-FAKE16: v_cmpx_eq_f16_e32 src_scc, v2           ; encoding: [0xfd,0x04,0x04,0x7d]
 
 0xff,0xfe,0x04,0x7d,0x0b,0xfe,0x00,0x00
-# GFX11: v_cmpx_eq_f16_e32 0xfe0b, v127          ; encoding: [0xff,0xfe,0x04,0x7d,0x0b,0xfe,0x00,0x00]
+# GFX11-REAL16: v_cmpx_eq_f16_e32 0xfe0b, v127.l        ; encoding: [0xff,0xfe,0x04,0x7d,0x0b,0xfe,0x00,0x00]
+# GFX11-FAKE16: v_cmpx_eq_f16_e32 0xfe0b, v127          ; encoding: [0xff,0xfe,0x04,0x7d,0x0b,0xfe,0x00,0x00]
+
+0x81,0x05,0x04,0x7d
+# GFX11-REAL16: v_cmpx_eq_f16_e32 v1.h, v2.l            ; encoding: [0x81,0x05,0x04,0x7d]
+# GFX11-FAKE16: v_cmpx_eq_f16_e32 v129/*Invalid register, operand has 'VS_32_Lo128' register class*/, v2 ; encoding: [0x81,0x05,0x04,0x7d]
+
+0xff,0x05,0x04,0x7d
+# GFX11-REAL16: v_cmpx_eq_f16_e32 v127.h, v2.l          ; encoding: [0xff,0x05,0x04,0x7d]
+# GFX11-FAKE16: v_cmpx_eq_f16_e32 v255/*Invalid register, operand has 'VS_32_Lo128' register class*/, v2 ; encoding: [0xff,0x05,0x04,0x7d]
+
+0xf0,0xfe,0x04,0x7d
+# GFX11-REAL16: v_cmpx_eq_f16_e32 0.5, v127.l           ; encoding: [0xf0,0xfe,0x04,0x7d]
+# GFX11-FAKE16: v_cmpx_eq_f16_e32 0.5, v127             ; encoding: [0xf0,0xfe,0x04,0x7d]
+
+0xfd,0x04,0x05,0x7d
+# GFX11-REAL16: v_cmpx_eq_f16_e32 src_scc, v2.h         ; encoding: [0xfd,0x04,0x05,0x7d]
+# GFX11-FAKE16: v_cmpx_eq_f16_e32 src_scc, v130/*Invalid register, operand has 'VGPR_32_Lo128' register class*/ ; encoding: [0xfd,0x04,0x05,0x7d]
+
+0xff,0xfe,0x05,0x7d,0x0b,0xfe,0x00,0x00
+# GFX11-REAL16: v_cmpx_eq_f16_e32 0xfe0b, v127.h        ; encoding: [0xff,0xfe,0x05,0x7d,0x0b,0xfe,0x00,0x00]
+# GFX11-FAKE16: v_cmpx_eq_f16_e32 0xfe0b, v255/*Invalid register, operand has 'VGPR_32_Lo128' register class*/ ; encoding: [0xff,0xfe,0x05,0x7d,0x0b,0xfe,0x00,0x00]
 
 0x01,0x05,0x24,0x7d
 # GFX11: v_cmpx_eq_f32_e32 v1, v2                ; encoding: [0x01,0x05,0x24,0x7d]
@@ -614,49 +649,84 @@
 # GFX11: v_cmpx_eq_u64_e32 0xaf123456, v[254:255] ; encoding: [0xff,0xfc,0xb5,0x7d,0x56,0x34,0x12,0xaf]
 
 0x01,0x05,0x00,0x7d
-# GFX11: v_cmpx_f_f16_e32 v1, v2                 ; encoding: [0x01,0x05,0x00,0x7d]
+# GFX11-REAL16: v_cmpx_f_f16_e32 v1.l, v2.l             ; encoding: [0x01,0x05,0x00,0x7d]
+# GFX11-FAKE16: v_cmpx_f_f16_e32 v1, v2                 ; encoding: [0x01,0x05,0x00,0x7d]
 
 0x7f,0x05,0x00,0x7d
-# GFX11: v_cmpx_f_f16_e32 v127, v2               ; encoding: [0x7f,0x05,0x00,0x7d]
+# GFX11-REAL16: v_cmpx_f_f16_e32 v127.l, v2.l           ; encoding: [0x7f,0x05,0x00,0x7d]
+# GFX11-FAKE16: v_cmpx_f_f16_e32 v127, v2               ; encoding: [0x7f,0x05,0x00,0x7d]
 
 0x01,0x04,0x00,0x7d
-# GFX11: v_cmpx_f_f16_e32 s1, v2                 ; encoding: [0x01,0x04,0x00,0x7d]
+# GFX11-REAL16: v_cmpx_f_f16_e32 s1, v2.l               ; encoding: [0x01,0x04,0x00,0x7d]
+# GFX11-FAKE16: v_cmpx_f_f16_e32 s1, v2                 ; encoding: [0x01,0x04,0x00,0x7d]
 
 0x69,0x04,0x00,0x7d
-# GFX11: v_cmpx_f_f16_e32 s105, v2               ; encoding: [0x69,0x04,0x00,0x7d]
+# GFX11-REAL16: v_cmpx_f_f16_e32 s105, v2.l             ; encoding: [0x69,0x04,0x00,0x7d]
+# GFX11-FAKE16: v_cmpx_f_f16_e32 s105, v2               ; encoding: [0x69,0x04,0x00,0x7d]
 
 0x6a,0x04,0x00,0x7d
-# GFX11: v_cmpx_f_f16_e32 vcc_lo, v2             ; encoding: [0x6a,0x04,0x00,0x7d]
+# GFX11-REAL16: v_cmpx_f_f16_e32 vcc_lo, v2.l           ; encoding: [0x6a,0x04,0x00,0x7d]
+# GFX11-FAKE16: v_cmpx_f_f16_e32 vcc_lo, v2             ; encoding: [0x6a,0x04,0x00,0x7d]
 
 0x6b,0x04,0x00,0x7d
-# GFX11: v_cmpx_f_f16_e32 vcc_hi, v2             ; encoding: [0x6b,0x04,0x00,0x7d]
+# GFX11-REAL16: v_cmpx_f_f16_e32 vcc_hi, v2.l           ; encoding: [0x6b,0x04,0x00,0x7d]
+# GFX11-FAKE16: v_cmpx_f_f16_e32 vcc_hi, v2             ; encoding: [0x6b,0x04,0x00,0x7d]
 
 0x7b,0x04,0x00,0x7d
-# GFX11: v_cmpx_f_f16_e32 ttmp15, v2             ; encoding: [0x7b,0x04,0x00,0x7d]
+# GFX11-REAL16: v_cmpx_f_f16_e32 ttmp15, v2.l           ; encoding: [0x7b,0x04,0x00,0x7d]
+# GFX11-FAKE16: v_cmpx_f_f16_e32 ttmp15, v2             ; encoding: [0x7b,0x04,0x00,0x7d]
 
 0x7d,0x04,0x00,0x7d
-# GFX11: v_cmpx_f_f16_e32 m0, v2                 ; encoding: [0x7d,0x04,0x00,0x7d]
+# GFX11-REAL16: v_cmpx_f_f16_e32 m0, v2.l               ; encoding: [0x7d,0x04,0x00,0x7d]
+# GFX11-FAKE16: v_cmpx_f_f16_e32 m0, v2                 ; encoding: [0x7d,0x04,0x00,0x7d]
 
 0x7e,0x04,0x00,0x7d
-# GFX11: v_cmpx_f_f16_e32 exec_lo, v2            ; encoding: [0x7e,0x04,0x00,0x7d]
+# GFX11-REAL16: v_cmpx_f_f16_e32 exec_lo, v2.l          ; encoding: [0x7e,0x04,0x00,0x7d]
+# GFX11-FAKE16: v_cmpx_f_f16_e32 exec_lo, v2            ; encoding: [0x7e,0x04,0x00,0x7d]
 
 0x7f,0x04,0x00,0x7d
-# GFX11: v_cmpx_f_f16_e32 exec_hi, v2            ; encoding: [0x7f,0x04,0x00,0x7d]
+# GFX11-REAL16: v_cmpx_f_f16_e32 exec_hi, v2.l          ; encoding: [0x7f,0x04,0x00,0x7d]
+# GFX11-FAKE16: v_cmpx_f_f16_e32 exec_hi, v2            ; encoding: [0x7f,0x04,0x00,0x7d]
 
 0x7c,0x04,0x00,0x7d
-# GFX11: v_cmpx_f_f16_e32 null, v2               ; encoding: [0x7c,0x04,0x00,0x7d]
+# GFX11-REAL16: v_cmpx_f_f16_e32 null, v2.l             ; encoding: [0x7c,0x04,0x00,0x7d]
+# GFX11-FAKE16: v_cmpx_f_f16_e32 null, v2               ; encoding: [0x7c,0x04,0x00,0x7d]
 
 0xc1,0x04,0x00,0x7d
-# GFX11: v_cmpx_f_f16_e32 -1, v2                 ; encoding: [0xc1,0x04,0x00,0x7d]
+# GFX11-REAL16: v_cmpx_f_f16_e32 -1, v2.l               ; encoding: [0xc1,0x04,0x00,0x7d]
+# GFX11-FAKE16: v_cmpx_f_f16_e32 -1, v2                 ; encoding: [0xc1,0x04,0x00,0x7d]
 
 0xf0,0x04,0x00,0x7d
-# GFX11: v_cmpx_f_f16_e32 0.5, v2                ; encoding: [0xf0,0x04,0x00,0x7d]
+# GFX11-REAL16: v_cmpx_f_f16_e32 0.5, v2.l              ; encoding: [0xf0,0x04,0x00,0x7d]
+# GFX11-FAKE16: v_cmpx_f_f16_e32 0.5, v2                ; encoding: [0xf0,0x04,0x00,0x7d]
 
 0xfd,0x04,0x00,0x7d
-# GFX11: v_cmpx_f_f16_e32 src_scc, v2            ; encoding: [0xfd,0x04,0x00,0x7d]
+# GFX11-REAL16: v_cmpx_f_f16_e32 src_scc, v2.l          ; encoding: [0xfd,0x04,0x00,0x7d]
+# GFX11-FAKE16: v_cmpx_f_f16_e32 src_scc, v2            ; encoding: [0xfd,0x04,0x00,0x7d]
 
 0xff,0xfe,0x00,0x7d,0x0b,0xfe,0x00,0x00
-# GFX11: v_cmpx_f_f16_e32 0xfe0b, v127           ; encoding: [0xff,0xfe,0x00,0x7d,0x0b,0xfe,0x00,0x00]
+# GFX11-REAL16: v_cmpx_f_f16_e32 0xfe0b, v127.l         ; encoding: [0xff,0xfe,0x00,0x7d,0x0b,0xfe,0x00,0x00]
+# GFX11-FAKE16: v_cmpx_f_f16_e32 0xfe0b, v127           ; encoding: [0xff,0xfe,0x00,0x7d,0x0b,0xfe,0x00,0x00]
+
+0x81,0x05,0x00,0x7d
+# GFX11-REAL16: v_cmpx_f_f16_e32 v1.h, v2.l             ; encoding: [0x81,0x05,0x00,0x7d]
+# GFX11-FAKE16: v_cmpx_f_f16_e32 v129/*Invalid register, operand has 'VS_32_Lo128' register class*/, v2 ; encoding: [0x81,0x05,0x00,0x7d]
+
+0xff,0x05,0x00,0x7d
+# GFX11-REAL16: v_cmpx_f_f16_e32 v127.h, v2.l           ; encoding: [0xff,0x05,0x00,0x7d]
+# GFX11-FAKE16: v_cmpx_f_f16_e32 v255/*Invalid register, operand has 'VS_32_Lo128' register class*/, v2 ; encoding: [0xff,0x05,0x00,0x7d]
+
+0xf0,0xfe,0x00,0x7d
+# GFX11-REAL16: v_cmpx_f_f16_e32 0.5, v127.l            ; encoding: [0xf0,0xfe,0x00,0x7d]
+# GFX11-FAKE16: v_cmpx_f_f16_e32 0.5, v127              ; encoding: [0xf0,0xfe,0x00,0x7d]
+
+0xfd,0x04,0x01,0x7d
+# GFX11-REAL16: v_cmpx_f_f16_e32 src_scc, v2.h          ; encoding: [0xfd,0x04,0x01,0x7d]
+# GFX11-FAKE16: v_cmpx_f_f16_e32 src_scc, v130/*Invalid register, operand has 'VGPR_32_Lo128' register class*/ ; encoding: [0xfd,0x04,0x01,0x7d]
+
+0xff,0xfe,0x01,0x7d,0x0b,0xfe,0x00,0x00
+# GFX11-REAL16: v_cmpx_f_f16_e32 0xfe0b, v127.h         ; encoding: [0xff,0xfe,0x01,0x7d,0x0b,0xfe,0x00,0x00]
+# GFX11-FAKE16: v_cmpx_f_f16_e32 0xfe0b, v255/*Invalid register, operand has 'VGPR_32_Lo128' register class*/ ; encoding: [0xff,0xfe,0x01,0x7d,0x0b,0xfe,0x00,0x00]
 
 0x01,0x05,0x20,0x7d
 # GFX11: v_cmpx_f_f32_e32 v1, v2                 ; encoding: [0x01,0x05,0x20,0x7d]
@@ -902,49 +972,84 @@
 # GFX11: v_cmpx_f_u64_e32 0xaf123456, v[254:255] ; encoding: [0xff,0xfc,0xb1,0x7d,0x56,0x34,0x12,0xaf]
 
 0x01,0x05,0x0c,0x7d
-# GFX11: v_cmpx_ge_f16_e32 v1, v2                ; encoding: [0x01,0x05,0x0c,0x7d]
+# GFX11-REAL16: v_cmpx_ge_f16_e32 v1.l, v2.l            ; encoding: [0x01,0x05,0x0c,0x7d]
+# GFX11-FAKE16: v_cmpx_ge_f16_e32 v1, v2                ; encoding: [0x01,0x05,0x0c,0x7d]
 
 0x7f,0x05,0x0c,0x7d
-# GFX11: v_cmpx_ge_f16_e32 v127, v2              ; encoding: [0x7f,0x05,0x0c,0x7d]
+# GFX11-REAL16: v_cmpx_ge_f16_e32 v127.l, v2.l          ; encoding: [0x7f,0x05,0x0c,0x7d]
+# GFX11-FAKE16: v_cmpx_ge_f16_e32 v127, v2              ; encoding: [0x7f,0x05,0x0c,0x7d]
 
 0x01,0x04,0x0c,0x7d
-# GFX11: v_cmpx_ge_f16_e32 s1, v2                ; encoding: [0x01,0x04,0x0c,0x7d]
+# GFX11-REAL16: v_cmpx_ge_f16_e32 s1, v2.l              ; encoding: [0x01,0x04,0x0c,0x7d]
+# GFX11-FAKE16: v_cmpx_ge_f16_e32 s1, v2                ; encoding: [0x01,0x04,0x0c,0x7d]
 
 0x69,0x04,0x0c,0x7d
-# GFX11: v_cmpx_ge_f16_e32 s105, v2              ; encoding: [0x69,0x04,0x0c,0x7d]
+# GFX11-REAL16: v_cmpx_ge_f16_e32 s105, v2.l            ; encoding: [0x69,0x04,0x0c,0x7d]
+# GFX11-FAKE16: v_cmpx_ge_f16_e32 s105, v2              ; encoding: [0x69,0x04,0x0c,0x7d]
 
 0x6a,0x04,0x0c,0x7d
-# GFX11: v_cmpx_ge_f16_e32 vcc_lo, v2            ; encoding: [0x6a,0x04,0x0c,0x7d]
+# GFX11-REAL16: v_cmpx_ge_f16_e32 vcc_lo, v2.l          ; encoding: [0x6a,0x04,0x0c,0x7d]
+# GFX11-FAKE16: v_cmpx_ge_f16_e32 vcc_lo, v2            ; encoding: [0x6a,0x04,0x0c,0x7d]
 
 0x6b,0x04,0x0c,0x7d
-# GFX11: v_cmpx_ge_f16_e32 vcc_hi, v2            ; encoding: [0x6b,0x04,0x0c,0x7d]
+# GFX11-REAL16: v_cmpx_ge_f16_e32 vcc_hi, v2.l          ; encoding: [0x6b,0x04,0x0c,0x7d]
+# GFX11-FAKE16: v_cmpx_ge_f16_e32 vcc_hi, v2            ; encoding: [0x6b,0x04,0x0c,0x7d]
 
 0x7b,0x04,0x0c,0x7d
-# GFX11: v_cmpx_ge_f16_e32 ttmp15, v2            ; encoding: [0x7b,0x04,0x0c,0x7d]
+# GFX11-REAL16: v_cmpx_ge_f16_e32 ttmp15, v2.l          ; encoding: [0x7b,0x04,0x0c,0x7d]
+# GFX11-FAKE16: v_cmpx_ge_f16_e32 ttmp15, v2            ; encoding: [0x7b,0x04,0x0c,0x7d]
 
 0x7d,0x04,0x0c,0x7d
-# GFX11: v_cmpx_ge_f16_e32 m0, v2                ; encoding: [0x7d,0x04,0x0c,0x7d]
+# GFX11-REAL16: v_cmpx_ge_f16_e32 m0, v2.l              ; encoding: [0x7d,0x04,0x0c,0x7d]
+# GFX11-FAKE16: v_cmpx_ge_f16_e32 m0, v2                ; encoding: [0x7d,0x04,0x0c,0x7d]
 
 0x7e,0x04,0x0c,0x7d
-# GFX11: v_cmpx_ge_f16_e32 exec_lo, v2           ; encoding: [0x7e,0x04,0x0c,0x7d]
+# GFX11-REAL16: v_cmpx_ge_f16_e32 exec_lo, v2.l         ; encoding: [0x7e,0x04,0x0c,0x7d]
+# GFX11-FAKE16: v_cmpx_ge_f16_e32 exec_lo, v2           ; encoding: [0x7e,0x04,0x0c,0x7d]
 
 0x7f,0x04,0x0c,0x7d
-# GFX11: v_cmpx_ge_f16_e32 exec_hi, v2           ; encoding: [0x7f,0x04,0x0c,0x7d]
+# GFX11-REAL16: v_cmpx_ge_f16_e32 exec_hi, v2.l         ; encoding: [0x7f,0x04,0x0c,0x7d]
+# GFX11-FAKE16: v_cmpx_ge_f16_e32 exec_hi, v2           ; encoding: [0x7f,0x04,0x0c,0x7d]
 
 0x7c,0x04,0x0c,0x7d
-# GFX11: v_cmpx_ge_f16_e32 null, v2              ; encoding: [0x7c,0x04,0x0c,0x7d]
+# GFX11-REAL16: v_cmpx_ge_f16_e32 null, v2.l            ; encoding: [0x7c,0x04,0x0c,0x7d]
+# GFX11-FAKE16: v_cmpx_ge_f16_e32 null, v2              ; encoding: [0x7c,0x04,0x0c,0x7d]
 
 0xc1,0x04,0x0c,0x7d
-# GFX11: v_cmpx_ge_f16_e32 -1, v2                ; encoding: [0xc1,0x04,0x0c,0x7d]
+# GFX11-REAL16: v_cmpx_ge_f16_e32 -1, v2.l              ; encoding: [0xc1,0x04,0x0c,0x7d]
+# GFX11-FAKE16: v_cmpx_ge_f16_e32 -1, v2                ; encoding: [0xc1,0x04,0x0c,0x7d]
 
 0xf0,0x04,0x0c,0x7d
-# GFX11: v_cmpx_ge_f16_e32 0.5, v2               ; encoding: [0xf0,0x04,0x0c,0x7d]
+# GFX11-REAL16: v_cmpx_ge_f16_e32 0.5, v2.l             ; encoding: [0xf0,0x04,0x0c,0x7d]
+# GFX11-FAKE16: v_cmpx_ge_f16_e32 0.5, v2               ; encoding: [0xf0,0x04,0x0c,0x7d]
 
 0xfd,0x04,0x0c,0x7d
-# GFX11: v_cmpx_ge_f16_e32 src_scc, v2           ; encoding: [0xfd,0x04,0x0c,0x7d]
+# GFX11-REAL16: v_cmpx_ge_f16_e32 src_scc, v2.l         ; encoding: [0xfd,0x04,0x0c,0x7d]
+# GFX11-FAKE16: v_cmpx_ge_f16_e32 src_scc, v2           ; encoding: [0xfd,0x04,0x0c,0x7d]
 
 0xff,0xfe,0x0c,0x7d,0x0b,0xfe,0x00,0x00
-# GFX11: v_cmpx_ge_f16_e32 0xfe0b, v127          ; encoding: [0xff,0xfe,0x0c,0x7d,0x0b,0xfe,0x00,0x00]
+# GFX11-REAL16: v_cmpx_ge_f16_e32 0xfe0b, v127.l        ; encoding: [0xff,0xfe,0x0c,0x7d,0x0b,0xfe,0x00,0x00]
+# GFX11-FAKE16: v_cmpx_ge_f16_e32 0xfe0b, v127          ; encoding: [0xff,0xfe,0x0c,0x7d,0x0b,0xfe,0x00,0x00]
+
+0x81,0x05,0x0c,0x7d
+# GFX11-REAL16: v_cmpx_ge_f16_e32 v1.h, v2.l            ; encoding: [0x81,0x05,0x0c,0x7d]
+# GFX11-FAKE16: v_cmpx_ge_f16_e32 v129/*Invalid register, operand has 'VS_32_Lo128' register class*/, v2 ; encoding: [0x81,0x05,0x0c,0x7d]
+
+0xff,0x05,0x0c,0x7d
+# GFX11-REAL16: v_cmpx_ge_f16_e32 v127.h, v2.l          ; encoding: [0xff,0x05,0x0c,0x7d]
+# GFX11-FAKE16: v_cmpx_ge_f16_e32 v255/*Invalid register, operand has 'VS_32_Lo128' register class*/, v2 ; encoding: [0xff,0x05,0x0c,0x7d]
+
+0xf0,0xfe,0x0c,0x7d
+# GFX11-REAL16: v_cmpx_ge_f16_e32 0.5, v127.l           ; encoding: [0xf0,0xfe,0x0c,0x7d]
+# GFX11-FAKE16: v_cmpx_ge_f16_e32 0.5, v127             ; encoding: [0xf0,0xfe,0x0c,0x7d]
+
+0xfd,0x04,0x0d,0x7d
+# GFX11-REAL16: v_cmpx_ge_f16_e32 src_scc, v2.h         ; encoding: [0xfd,0x04,0x0d,0x7d]
+# GFX11-FAKE16: v_cmpx_ge_f16_e32 src_scc, v130/*Invalid register, operand has 'VGPR_32_Lo128' register class*/ ; encoding: [0xfd,0x04,0x0d,0x7d]
+
+0xff,0xfe,0x0d,0x7d,0x0b,0xfe,0x00,0x00
+# GFX11-REAL16: v_cmpx_ge_f16_e32 0xfe0b, v127.h        ; encoding: [0xff,0xfe,0x0d,0x7d,0x0b,0xfe,0x00,0x00]
+# GFX11-FAKE16: v_cmpx_ge_f16_e32 0xfe0b, v255/*Invalid register, operand has 'VGPR_32_Lo128' register class*/ ; encoding: [0xff,0xfe,0x0d,0x7d,0x0b,0xfe,0x00,0x00]
 
 0x01,0x05,0x2c,0x7d
 # GFX11: v_cmpx_ge_f32_e32 v1, v2                ; encoding: [0x01,0x05,0x2c,0x7d]
@@ -1350,49 +1455,84 @@
 # GFX11: v_cmpx_ge_u64_e32 0xaf123456, v[254:255] ; encoding: [0xff,0xfc,0xbd,0x7d,0x56,0x34,0x12,0xaf]
 
 0x01,0x05,0x08,0x7d
-# GFX11: v_cmpx_gt_f16_e32 v1, v2                ; encoding: [0x01,0x05,0x08,0x7d]
+# GFX11-REAL16: v_cmpx_gt_f16_e32 v1.l, v2.l            ; encoding: [0x01,0x05,0x08,0x7d]
+# GFX11-FAKE16: v_cmpx_gt_f16_e32 v1, v2                ; encoding: [0x01,0x05,0x08,0x7d]
 
 0x7f,0x05,0x08,0x7d
-# GFX11: v_cmpx_gt_f16_e32 v127, v2              ; encoding: [0x7f,0x05,0x08,0x7d]
+# GFX11-REAL16: v_cmpx_gt_f16_e32 v127.l, v2.l          ; encoding: [0x7f,0x05,0x08,0x7d]
+# GFX11-FAKE16: v_cmpx_gt_f16_e32 v127, v2              ; encoding: [0x7f,0x05,0x08,0x7d]
 
 0x01,0x04,0x08,0x7d
-# GFX11: v_cmpx_gt_f16_e32 s1, v2                ; encoding: [0x01,0x04,0x08,0x7d]
+# GFX11-REAL16: v_cmpx_gt_f16_e32 s1, v2.l              ; encoding: [0x01,0x04,0x08,0x7d]
+# GFX11-FAKE16: v_cmpx_gt_f16_e32 s1, v2                ; encoding: [0x01,0x04,0x08,0x7d]
 
 0x69,0x04,0x08,0x7d
-# GFX11: v_cmpx_gt_f16_e32 s105, v2              ; encoding: [0x69,0x04,0x08,0x7d]
+# GFX11-REAL16: v_cmpx_gt_f16_e32 s105, v2.l            ; encoding: [0x69,0x04,0x08,0x7d]
+# GFX11-FAKE16: v_cmpx_gt_f16_e32 s105, v2              ; encoding: [0x69,0x04,0x08,0x7d]
 
 0x6a,0x04,0x08,0x7d
-# GFX11: v_cmpx_gt_f16_e32 vcc_lo, v2            ; encoding: [0x6a,0x04,0x08,0x7d]
+# GFX11-REAL16: v_cmpx_gt_f16_e32 vcc_lo, v2.l          ; encoding: [0x6a,0x04,0x08,0x7d]
+# GFX11-FAKE16: v_cmpx_gt_f16_e32 vcc_lo, v2            ; encoding: [0x6a,0x04,0x08,0x7d]
 
 0x6b,0x04,0x08,0x7d
-# GFX11: v_cmpx_gt_f16_e32 vcc_hi, v2            ; encoding: [0x6b,0x04,0x08,0x7d]
+# GFX11-REAL16: v_cmpx_gt_f16_e32 vcc_hi, v2.l          ; encoding: [0x6b,0x04,0x08,0x7d]
+# GFX11-FAKE16: v_cmpx_gt_f16_e32 vcc_hi, v2            ; encoding: [0x6b,0x04,0x08,0x7d]
 
 0x7b,0x04,0x08,0x7d
-# GFX11: v_cmpx_gt_f16_e32 ttmp15, v2            ; encoding: [0x7b,0x04,0x08,0x7d]
+# GFX11-REAL16: v_cmpx_gt_f16_e32 ttmp15, v2.l          ; encoding: [0x7b,0x04,0x08,0x7d]
+# GFX11-FAKE16: v_cmpx_gt_f16_e32 ttmp15, v2            ; encoding: [0x7b,0x04,0x08,0x7d]
 
 0x7d,0x04,0x08,0x7d
-# GFX11: v_cmpx_gt_f16_e32 m0, v2                ; encoding: [0x7d,0x04,0x08,0x7d]
+# GFX11-REAL16: v_cmpx_gt_f16_e32 m0, v2.l              ; encoding: [0x7d,0x04,0x08,0x7d]
+# GFX11-FAKE16: v_cmpx_gt_f16_e32 m0, v2                ; encoding: [0x7d,0x04,0x08,0x7d]
 
 0x7e,0x04,0x08,0x7d
-# GFX11: v_cmpx_gt_f16_e32 exec_lo, v2           ; encoding: [0x7e,0x04,0x08,0x7d]
+# GFX11-REAL16: v_cmpx_gt_f16_e32 exec_lo, v2.l         ; encoding: [0x7e,0x04,0x08,0x7d]
+# GFX11-FAKE16: v_cmpx_gt_f16_e32 exec_lo, v2           ; encoding: [0x7e,0x04,0x08,0x7d]
 
 0x7f,0x04,0x08,0x7d
-# GFX11: v_cmpx_gt_f16_e32 exec_hi, v2           ; encoding: [0x7f,0x04,0x08,0x7d]
+# GFX11-REAL16: v_cmpx_gt_f16_e32 exec_hi, v2.l         ; encoding: [0x7f,0x04,0x08,0x7d]
+# GFX11-FAKE16: v_cmpx_gt_f16_e32 exec_hi, v2           ; encoding: [0x7f,0x04,0x08,0x7d]
 
 0x7c,0x04,0x08,0x7d
-# GFX11: v_cmpx_gt_f16_e32 null, v2              ; encoding: [0x7c,0x04,0x08,0x7d]
+# GFX11-REAL16: v_cmpx_gt_f16_e32 null, v2.l            ; encoding: [0x7c,0x04,0x08,0x7d]
+# GFX11-FAKE16: v_cmpx_gt_f16_e32 null, v2              ; encoding: [0x7c,0x04,0x08,0x7d]
 
 0xc1,0x04,0x08,0x7d
-# GFX11: v_cmpx_gt_f16_e32 -1, v2                ; encoding: [0xc1,0x04,0x08,0x7d]
+# GFX11-REAL16: v_cmpx_gt_f16_e32 -1, v2.l              ; encoding: [0xc1,0x04,0x08,0x7d]
+# GFX11-FAKE16: v_cmpx_gt_f16_e32 -1, v2                ; encoding: [0xc1,0x04,0x08,0x7d]
 
 0xf0,0x04,0x08,0x7d
-# GFX11: v_cmpx_gt_f16_e32 0.5, v2               ; encoding: [0xf0,0x04,0x08,0x7d]
+# GFX11-REAL16: v_cmpx_gt_f16_e32 0.5, v2.l             ; encoding: [0xf0,0x04,0x08,0x7d]
+# GFX11-FAKE16: v_cmpx_gt_f16_e32 0.5, v2               ; encoding: [0xf0,0x04,0x08,0x7d]
 
 0xfd,0x04,0x08,0x7d
-# GFX11: v_cmpx_gt_f16_e32 src_scc, v2           ; encoding: [0xfd,0x04,0x08,0x7d]
+# GFX11-REAL16: v_cmpx_gt_f16_e32 src_scc, v2.l         ; encoding: [0xfd,0x04,0x08,0x7d]
+# GFX11-FAKE16: v_cmpx_gt_f16_e32 src_scc, v2           ; encoding: [0xfd,0x04,0x08,0x7d]
 
 0xff,0xfe,0x08,0x7d,0x0b,0xfe,0x00,0x00
-# GFX11: v_cmpx_gt_f16_e32 0xfe0b, v127          ; encoding: [0xff,0xfe,0x08,0x7d,0x0b,0xfe,0x00,0x00]
+# GFX11-REAL16: v_cmpx_gt_f16_e32 0xfe0b, v127.l        ; encoding: [0xff,0xfe,0x08,0x7d,0x0b,0xfe,0x00,0x00]
+# GFX11-FAKE16: v_cmpx_gt_f16_e32 0xfe0b, v127          ; encoding: [0xff,0xfe,0x08,0x7d,0x0b,0xfe,0x00,0x00]
+
+0x81,0x05,0x08,0x7d
+# GFX11-REAL16: v_cmpx_gt_f16_e32 v1.h, v2.l            ; encoding: [0x81,0x05,0x08,0x7d]
+# GFX11-FAKE16: v_cmpx_gt_f16_e32 v129/*Invalid register, operand has 'VS_32_Lo128' register class*/, v2 ; encoding: [0x81,0x05,0x08,0x7d]
+
+0xff,0x05,0x08,0x7d
+# GFX11-REAL16: v_cmpx_gt_f16_e32 v127.h, v2.l          ; encoding: [0xff,0x05,0x08,0x7d]
+# GFX11-FAKE16: v_cmpx_gt_f16_e32 v255/*Invalid register, operand has 'VS_32_Lo128' register class*/, v2 ; encoding: [0xff,0x05,0x08,0x7d]
+
+0xf0,0xfe,0x08,0x7d
+# GFX11-REAL16: v_cmpx_gt_f16_e32 0.5, v127.l           ; encoding: [0xf0,0xfe,0x08,0x7d]
+# GFX11-FAKE16: v_cmpx_gt_f16_e32 0.5, v127             ; encoding: [0xf0,0xfe,0x08,0x7d]
+
+0xfd,0x04,0x09,0x7d
+# GFX11-REAL16: v_cmpx_gt_f16_e32 src_scc, v2.h         ; encoding: [0xfd,0x04,0x09,0x7d]
+# GFX11-FAKE16: v_cmpx_gt_f16_e32 src_scc, v130/*Invalid register, operand has 'VGPR_32_Lo128' register class*/ ; encoding: [0xfd,0x04,0x09,0x7d]
+
+0xff,0xfe,0x09,0x7d,0x0b,0xfe,0x00,0x00
+# GFX11-REAL16: v_cmpx_gt_f16_e32 0xfe0b, v127.h        ; encoding: [0xff,0xfe,0x09,0x7d,0x0b,0xfe,0x00,0x00]
+# GFX11-FAKE16: v_cmpx_gt_f16_e32 0xfe0b, v255/*Invalid register, operand has 'VGPR_32_Lo128' register class*/ ; encoding: [0xff,0xfe,0x09,0x7d,0x0b,0xfe,0x00,0x00]
 
 0x01,0x05,0x28,0x7d
 # GFX11: v_cmpx_gt_f32_e32 v1, v2                ; encoding: [0x01,0x05,0x28,0x7d]
@@ -1798,49 +1938,84 @@
 # GFX11: v_cmpx_gt_u64_e32 0xaf123456, v[254:255] ; encoding: [0xff,0xfc,0xb9,0x7d,0x56,0x34,0x12,0xaf]
 
 0x01,0x05,0x06,0x7d
-# GFX11: v_cmpx_le_f16_e32 v1, v2                ; encoding: [0x01,0x05,0x06,0x7d]
+# GFX11-REAL16: v_cmpx_le_f16_e32 v1.l, v2.l            ; encoding: [0x01,0x05,0x06,0x7d]
+# GFX11-FAKE16: v_cmpx_le_f16_e32 v1, v2                ; encoding: [0x01,0x05,0x06,0x7d]
 
 0x7f,0x05,0x06,0x7d
-# GFX11: v_cmpx_le_f16_e32 v127, v2              ; encoding: [0x7f,0x05,0x06,0x7d]
+# GFX11-REAL16: v_cmpx_le_f16_e32 v127.l, v2.l          ; encoding: [0x7f,0x05,0x06,0x7d]
+# GFX11-FAKE16: v_cmpx_le_f16_e32 v127, v2              ; encoding: [0x7f,0x05,0x06,0x7d]
 
 0x01,0x04,0x06,0x7d
-# GFX11: v_cmpx_le_f16_e32 s1, v2                ; encoding: [0x01,0x04,0x06,0x7d]
+# GFX11-REAL16: v_cmpx_le_f16_e32 s1, v2.l              ; encoding: [0x01,0x04,0x06,0x7d]
+# GFX11-FAKE16: v_cmpx_le_f16_e32 s1, v2                ; encoding: [0x01,0x04,0x06,0x7d]
 
 0x69,0x04,0x06,0x7d
-# GFX11: v_cmpx_le_f16_e32 s105, v2              ; encoding: [0x69,0x04,0x06,0x7d]
+# GFX11-REAL16: v_cmpx_le_f16_e32 s105, v2.l            ; encoding: [0x69,0x04,0x06,0x7d]
+# GFX11-FAKE16: v_cmpx_le_f16_e32 s105, v2              ; encoding: [0x69,0x04,0x06,0x7d]
 
 0x6a,0x04,0x06,0x7d
-# GFX11: v_cmpx_le_f16_e32 vcc_lo, v2            ; encoding: [0x6a,0x04,0x06,0x7d]
+# GFX11-REAL16: v_cmpx_le_f16_e32 vcc_lo, v2.l          ; encoding: [0x6a,0x04,0x06,0x7d]
+# GFX11-FAKE16: v_cmpx_le_f16_e32 vcc_lo, v2            ; encoding: [0x6a,0x04,0x06,0x7d]
 
 0x6b,0x04,0x06,0x7d
-# GFX11: v_cmpx_le_f16_e32 vcc_hi, v2            ; encoding: [0x6b,0x04,0x06,0x7d]
+# GFX11-REAL16: v_cmpx_le_f16_e32 vcc_hi, v2.l          ; encoding: [0x6b,0x04,0x06,0x7d]
+# GFX11-FAKE16: v_cmpx_le_f16_e32 vcc_hi, v2            ; encoding: [0x6b,0x04,0x06,0x7d]
 
 0x7b,0x04,0x06,0x7d
-# GFX11: v_cmpx_le_f16_e32 ttmp15, v2            ; encoding: [0x7b,0x04,0x06,0x7d]
+# GFX11-REAL16: v_cmpx_le_f16_e32 ttmp15, v2.l          ; encoding: [0x7b,0x04,0x06,0x7d]
+# GFX11-FAKE16: v_cmpx_le_f16_e32 ttmp15, v2            ; encoding: [0x7b,0x04,0x06,0x7d]
 
 0x7d,0x04,0x06,0x7d
-# GFX11: v_cmpx_le_f16_e32 m0, v2                ; encoding: [0x7d,0x04,0x06,0x7d]
+# GFX11-REAL16: v_cmpx_le_f16_e32 m0, v2.l              ; encoding: [0x7d,0x04,0x06,0x7d]
+# GFX11-FAKE16: v_cmpx_le_f16_e32 m0, v2                ; encoding: [0x7d,0x04,0x06,0x7d]
 
 0x7e,0x04,0x06,0x7d
-# GFX11: v_cmpx_le_f16_e32 exec_lo, v2           ; encoding: [0x7e,0x04,0x06,0x7d]
+# GFX11-REAL16: v_cmpx_le_f16_e32 exec_lo, v2.l         ; encoding: [0x7e,0x04,0x06,0x7d]
+# GFX11-FAKE16: v_cmpx_le_f16_e32 exec_lo, v2           ; encoding: [0x7e,0x04,0x06,0x7d]
 
 0x7f,0x04,0x06,0x7d
-# GFX11: v_cmpx_le_f16_e32 exec_hi, v2           ; encoding: [0x7f,0x04,0x06,0x7d]
+# GFX11-REAL16: v_cmpx_le_f16_e32 exec_hi, v2.l         ; encoding: [0x7f,0x04,0x06,0x7d]
+# GFX11-FAKE16: v_cmpx_le_f16_e32 exec_hi, v2           ; encoding: [0x7f,0x04,0x06,0x7d]
 
 0x7c,0x04,0x06,0x7d
-# GFX11: v_cmpx_le_f16_e32 null, v2              ; encoding: [0x7c,0x04,0x06,0x7d]
+# GFX11-REAL16: v_cmpx_le_f16_e32 null, v2.l            ; encoding: [0x7c,0x04,0x06,0x7d]
+# GFX11-FAKE16: v_cmpx_le_f16_e32 null, v2              ; encoding: [0x7c,0x04,0x06,0x7d]
 
 0xc1,0x04,0x06,0x7d
-# GFX11: v_cmpx_le_f16_e32 -1, v2                ; encoding: [0xc1,0x04,0x06,0x7d]
+# GFX11-REAL16: v_cmpx_le_f16_e32 -1, v2.l              ; encoding: [0xc1,0x04,0x06,0x7d]
+# GFX11-FAKE16: v_cmpx_le_f16_e32 -1, v2                ; encoding: [0xc1,0x04,0x06,0x7d]
 
 0xf0,0x04,0x06,0x7d
-# GFX11: v_cmpx_le_f16_e32 0.5, v2               ; encoding: [0xf0,0x04,0x06,0x7d]
+# GFX11-REAL16: v_cmpx_le_f16_e32 0.5, v2.l             ; encoding: [0xf0,0x04,0x06,0x7d]
+# GFX11-FAKE16: v_cmpx_le_f16_e32 0.5, v2               ; encoding: [0xf0,0x04,0x06,0x7d]
 
 0xfd,0x04,0x06,0x7d
-# GFX11: v_cmpx_le_f16_e32 src_scc, v2           ; encoding: [0xfd,0x04,0x06,0x7d]
+# GFX11-REAL16: v_cmpx_le_f16_e32 src_scc, v2.l         ; encoding: [0xfd,0x04,0x06,0x7d]
+# GFX11-FAKE16: v_cmpx_le_f16_e32 src_scc, v2           ; encoding: [0xfd,0x04,0x06,0x7d]
 
 0xff,0xfe,0x06,0x7d,0x0b,0xfe,0x00,0x00
-# GFX11: v_cmpx_le_f16_e32 0xfe0b, v127          ; encoding: [0xff,0xfe,0x06,0x7d,0x0b,0xfe,0x00,0x00]
+# GFX11-REAL16: v_cmpx_le_f16_e32 0xfe0b, v127.l        ; encoding: [0xff,0xfe,0x06,0x7d,0x0b,0xfe,0x00,0x00]
+# GFX11-FAKE16: v_cmpx_le_f16_e32 0xfe0b, v127          ; encoding: [0xff,0xfe,0x06,0x7d,0x0b,0xfe,0x00,0x00]
+
+0x81,0x05,0x06,0x7d
+# GFX11-REAL16: v_cmpx_le_f16_e32 v1.h, v2.l            ; encoding: [0x81,0x05,0x06,0x7d]
+# GFX11-FAKE16: v_cmpx_le_f16_e32 v129/*Invalid register, operand has 'VS_32_Lo128' register class*/, v2 ; encoding: [0x81,0x05,0x06,0x7d]
+
+0xff,0x05,0x06,0x7d
+# GFX11-REAL16: v_cmpx_le_f16_e32 v127.h, v2.l          ; encoding: [0xff,0x05,0x06,0x7d]
+# GFX11-FAKE16: v_cmpx_le_f16_e32 v255/*Invalid register, operand has 'VS_32_Lo128' register class*/, v2 ; encoding: [0xff,0x05,0x06,0x7d]
+
+0xf0,0xfe,0x06,0x7d
+# GFX11-REAL16: v_cmpx_le_f16_e32 0.5, v127.l           ; encoding: [0xf0,0xfe,0x06,0x7d]
+# GFX11-FAKE16: v_cmpx_le_f16_e32 0.5, v127             ; encoding: [0xf0,0xfe,0x06,0x7d]
+
+0xfd,0x04,0x07,0x7d
+# GFX11-REAL16: v_cmpx_le_f16_e32 src_scc, v2.h         ; encoding: [0xfd,0x04,0x07,0x7d]
+# GFX11-FAKE16: v_cmpx_le_f16_e32 src_scc, v130/*Invalid register, operand has 'VGPR_32_Lo128' register class*/ ; encoding: [0xfd,0x04,0x07,0x7d]
+
+0xff,0xfe,0x07,0x7d,0x0b,0xfe,0x00,0x00
+# GFX11-REAL16: v_cmpx_le_f16_e32 0xfe0b, v127.h        ; encoding: [0xff,0xfe,0x07,0x7d,0x0b,0xfe,0x00,0x00]
+# GFX11-FAKE16: v_cmpx_le_f16_e32 0xfe0b, v255/*Invalid register, operand has 'VGPR_32_Lo128' register class*/ ; encoding: [0xff,0xfe,0x07,0x7d,0x0b,0xfe,0x00,0x00]
 
 0x01,0x05,0x26,0x7d
 # GFX11: v_cmpx_le_f32_e32 v1, v2                ; encoding: [0x01,0x05,0x26,0x7d]
@@ -2246,49 +2421,84 @@
 # GFX11: v_cmpx_le_u64_e32 0xaf123456, v[254:255] ; encoding: [0xff,0xfc,0xb7,0x7d,0x56,0x34,0x12,0xaf]
 
 0x01,0x05,0x0a,0x7d
-# GFX11: v_cmpx_lg_f16_e32 v1, v2                ; encoding: [0x01,0x05,0x0a,0x7d]
+# GFX11-REAL16: v_cmpx_lg_f16_e32 v1.l, v2.l            ; encoding: [0x01,0x05,0x0a,0x7d]
+# GFX11-FAKE16: v_cmpx_lg_f16_e32 v1, v2                ; encoding: [0x01,0x05,0x0a,0x7d]
 
 0x7f,0x05,0x0a,0x7d
-# GFX11: v_cmpx_lg_f16_e32 v127, v2              ; encoding: [0x7f,0x05,0x0a,0x7d]
+# GFX11-REAL16: v_cmpx_lg_f16_e32 v127.l, v2.l          ; encoding: [0x7f,0x05,0x0a,0x7d]
+# GFX11-FAKE16: v_cmpx_lg_f16_e32 v127, v2              ; encoding: [0x7f,0x05,0x0a,0x7d]
 
 0x01,0x04,0x0a,0x7d
-# GFX11: v_cmpx_lg_f16_e32 s1, v2                ; encoding: [0x01,0x04,0x0a,0x7d]
+# GFX11-REAL16: v_cmpx_lg_f16_e32 s1, v2.l              ; encoding: [0x01,0x04,0x0a,0x7d]
+# GFX11-FAKE16: v_cmpx_lg_f16_e32 s1, v2                ; encoding: [0x01,0x04,0x0a,0x7d]
 
 0x69,0x04,0x0a,0x7d
-# GFX11: v_cmpx_lg_f16_e32 s105, v2              ; encoding: [0x69,0x04,0x0a,0x7d]
+# GFX11-REAL16: v_cmpx_lg_f16_e32 s105, v2.l            ; encoding: [0x69,0x04,0x0a,0x7d]
+# GFX11-FAKE16: v_cmpx_lg_f16_e32 s105, v2              ; encoding: [0x69,0x04,0x0a,0x7d]
 
 0x6a,0x04,0x0a,0x7d
-# GFX11: v_cmpx_lg_f16_e32 vcc_lo, v2            ; encoding: [0x6a,0x04,0x0a,0x7d]
+# GFX11-REAL16: v_cmpx_lg_f16_e32 vcc_lo, v2.l          ; encoding: [0x6a,0x04,0x0a,0x7d]
+# GFX11-FAKE16: v_cmpx_lg_f16_e32 vcc_lo, v2            ; encoding: [0x6a,0x04,0x0a,0x7d]
 
 0x6b,0x04,0x0a,0x7d
-# GFX11: v_cmpx_lg_f16_e32 vcc_hi, v2            ; encoding: [0x6b,0x04,0x0a,0x7d]
+# GFX11-REAL16: v_cmpx_lg_f16_e32 vcc_hi, v2.l          ; encoding: [0x6b,0x04,0x0a,0x7d]
+# GFX11-FAKE16: v_cmpx_lg_f16_e32 vcc_hi, v2            ; encoding: [0x6b,0x04,0x0a,0x7d]
 
 0x7b,0x04,0x0a,0x7d
-# GFX11: v_cmpx_lg_f16_e32 ttmp15, v2            ; encoding: [0x7b,0x04,0x0a,0x7d]
+# GFX11-REAL16: v_cmpx_lg_f16_e32 ttmp15, v2.l          ; encoding: [0x7b,0x04,0x0a,0x7d]
+# GFX11-FAKE16: v_cmpx_lg_f16_e32 ttmp15, v2            ; encoding: [0x7b,0x04,0x0a,0x7d]
 
 0x7d,0x04,0x0a,0x7d
-# GFX11: v_cmpx_lg_f16_e32 m0, v2                ; encoding: [0x7d,0x04,0x0a,0x7d]
+# GFX11-REAL16: v_cmpx_lg_f16_e32 m0, v2.l              ; encoding: [0x7d,0x04,0x0a,0x7d]
+# GFX11-FAKE16: v_cmpx_lg_f16_e32 m0, v2                ; encoding: [0x7d,0x04,0x0a,0x7d]
 
 0x7e,0x04,0x0a,0x7d
-# GFX11: v_cmpx_lg_f16_e32 exec_lo, v2           ; encoding: [0x7e,0x04,0x0a,0x7d]
+# GFX11-REAL16: v_cmpx_lg_f16_e32 exec_lo, v2.l         ; encoding: [0x7e,0x04,0x0a,0x7d]
+# GFX11-FAKE16: v_cmpx_lg_f16_e32 exec_lo, v2           ; encoding: [0x7e,0x04,0x0a,0x7d]
 
 0x7f,0x04,0x0a,0x7d
-# GFX11: v_cmpx_lg_f16_e32 exec_hi, v2           ; encoding: [0x7f,0x04,0x0a,0x7d]
+# GFX11-REAL16: v_cmpx_lg_f16_e32 exec_hi, v2.l         ; encoding: [0x7f,0x04,0x0a,0x7d]
+# GFX11-FAKE16: v_cmpx_lg_f16_e32 exec_hi, v2           ; encoding: [0x7f,0x04,0x0a,0x7d]
 
 0x7c,0x04,0x0a,0x7d
-# GFX11: v_cmpx_lg_f16_e32 null, v2              ; encoding: [0x7c,0x04,0x0a,0x7d]
+# GFX11-REAL16: v_cmpx_lg_f16_e32 null, v2.l            ; encoding: [0x7c,0x04,0x0a,0x7d]
+# GFX11-FAKE16: v_cmpx_lg_f16_e32 null, v2              ; encoding: [0x7c,0x04,0x0a,0x7d]
 
 0xc1,0x04,0x0a,0x7d
-# GFX11: v_cmpx_lg_f16_e32 -1, v2                ; encoding: [0xc1,0x04,0x0a,0x7d]
+# GFX11-REAL16: v_cmpx_lg_f16_e32 -1, v2.l              ; encoding: [0xc1,0x04,0x0a,0x7d]
+# GFX11-FAKE16: v_cmpx_lg_f16_e32 -1, v2                ; encoding: [0xc1,0x04,0x0a,0x7d]
 
 0xf0,0x04,0x0a,0x7d
-# GFX11: v_cmpx_lg_f16_e32 0.5, v2               ; encoding: [0xf0,0x04,0x0a,0x7d]
+# GFX11-REAL16: v_cmpx_lg_f16_e32 0.5, v2.l             ; encoding: [0xf0,0x04,0x0a,0x7d]
+# GFX11-FAKE16: v_cmpx_lg_f16_e32 0.5, v2               ; encoding: [0xf0,0x04,0x0a,0x7d]
 
 0xfd,0x04,0x0a,0x7d
-# GFX11: v_cmpx_lg_f16_e32 src_scc, v2           ; encoding: [0xfd,0x04,0x0a,0x7d]
+# GFX11-REAL16: v_cmpx_lg_f16_e32 src_scc, v2.l         ; encoding: [0xfd,0x04,0x0a,0x7d]
+# GFX11-FAKE16: v_cmpx_lg_f16_e32 src_scc, v2           ; encoding: [0xfd,0x04,0x0a,0x7d]
 
 0xff,0xfe,0x0a,0x7d,0x0b,0xfe,0x00,0x00
-# GFX11: v_cmpx_lg_f16_e32 0xfe0b, v127          ; encoding: [0xff,0xfe,0x0a,0x7d,0x0b,0xfe,0x00,0x00]
+# GFX11-REAL16: v_cmpx_lg_f16_e32 0xfe0b, v127.l        ; encoding: [0xff,0xfe,0x0a,0x7d,0x0b,0xfe,0x00,0x00]
+# GFX11-FAKE16: v_cmpx_lg_f16_e32 0xfe0b, v127          ; encoding: [0xff,0xfe,0x0a,0x7d,0x0b,0xfe,0x00,0x00]
+
+0x81,0x05,0x0a,0x7d
+# GFX11-REAL16: v_cmpx_lg_f16_e32 v1.h, v2.l            ; encoding: [0x81,0x05,0x0a,0x7d]
+# GFX11-FAKE16: v_cmpx_lg_f16_e32 v129/*Invalid register, operand has 'VS_32_Lo128' register class*/, v2 ; encoding: [0x81,0x05,0x0a,0x7d]
+
+0xff,0x05,0x0a,0x7d
+# GFX11-REAL16: v_cmpx_lg_f16_e32 v127.h, v2.l          ; encoding: [0xff,0x05,0x0a,0x7d]
+# GFX11-FAKE16: v_cmpx_lg_f16_e32 v255/*Invalid register, operand has 'VS_32_Lo128' register class*/, v2 ; encoding: [0xff,0x05,0x0a,0x7d]
+
+0xf0,0xfe,0x0a,0x7d
+# GFX11-REAL16: v_cmpx_lg_f16_e32 0.5, v127.l           ; encoding: [0xf0,0xfe,0x0a,0x7d]
+# GFX11-FAKE16: v_cmpx_lg_f16_e32 0.5, v127             ; encoding: [0xf0,0xfe,0x0a,0x7d]
+
+0xfd,0x04,0x0b,0x7d
+# GFX11-REAL16: v_cmpx_lg_f16_e32 src_scc, v2.h         ; encoding: [0xfd,0x04,0x0b,0x7d]
+# GFX11-FAKE16: v_cmpx_lg_f16_e32 src_scc, v130/*Invalid register, operand has 'VGPR_32_Lo128' register class*/ ; encoding: [0xfd,0x04,0x0b,0x7d]
+
+0xff,0xfe,0x0b,0x7d,0x0b,0xfe,0x00,0x00
+# GFX11-REAL16: v_cmpx_lg_f16_e32 0xfe0b, v127.h        ; encoding: [0xff,0xfe,0x0b,0x7d,0x0b,0xfe,0x00,0x00]
+# GFX11-FAKE16: v_cmpx_lg_f16_e32 0xfe0b, v255/*Invalid register, operand has 'VGPR_32_Lo128' register class*/ ; encoding: [0xff,0xfe,0x0b,0x7d,0x0b,0xfe,0x00,0x00]
 
 0x01,0x05,0x2a,0x7d
 # GFX11: v_cmpx_lg_f32_e32 v1, v2                ; encoding: [0x01,0x05,0x2a,0x7d]
@@ -3177,49 +3387,84 @@
 # GFX11: v_cmpx_ne_u64_e32 0xaf123456, v[254:255] ; encoding: [0xff,0xfc,0xbb,0x7d,0x56,0x34,0x12,0xaf]
 
 0x01,0x05,0x1a,0x7d
-# GFX11: v_cmpx_neq_f16_e32 v1, v2               ; encoding: [0x01,0x05,0x1a,0x7d]
+# GFX11-REAL16: v_cmpx_neq_f16_e32 v1.l, v2.l           ; encoding: [0x01,0x05,0x1a,0x7d]
+# GFX11-FAKE16: v_cmpx_neq_f16_e32 v1, v2               ; encoding: [0x01,0x05,0x1a,0x7d]
 
 0x7f,0x05,0x1a,0x7d
-# GFX11: v_cmpx_neq_f16_e32 v127, v2             ; encoding: [0x7f,0x05,0x1a,0x7d]
+# GFX11-REAL16: v_cmpx_neq_f16_e32 v127.l, v2.l         ; encoding: [0x7f,0x05,0x1a,0x7d]
+# GFX11-FAKE16: v_cmpx_neq_f16_e32 v127, v2             ; encoding: [0x7f,0x05,0x1a,0x7d]
 
 0x01,0x04,0x1a,0x7d
-# GFX11: v_cmpx_neq_f16_e32 s1, v2               ; encoding: [0x01,0x04,0x1a,0x7d]
+# GFX11-REAL16: v_cmpx_neq_f16_e32 s1, v2.l             ; encoding: [0x01,0x04,0x1a,0x7d]
+# GFX11-FAKE16: v_cmpx_neq_f16_e32 s1, v2               ; encoding: [0x01,0x04,0x1a,0x7d]
 
 0x69,0x04,0x1a,0x7d
-# GFX11: v_cmpx_neq_f16_e32 s105, v2             ; encoding: [0x69,0x04,0x1a,0x7d]
+# GFX11-REAL16: v_cmpx_neq_f16_e32 s105, v2.l           ; encoding: [0x69,0x04,0x1a,0x7d]
+# GFX11-FAKE16: v_cmpx_neq_f16_e32 s105, v2             ; encoding: [0x69,0x04,0x1a,0x7d]
 
 0x6a,0x04,0x1a,0x7d
-# GFX11: v_cmpx_neq_f16_e32 vcc_lo, v2           ; encoding: [0x6a,0x04,0x1a,0x7d]
+# GFX11-REAL16: v_cmpx_neq_f16_e32 vcc_lo, v2.l         ; encoding: [0x6a,0x04,0x1a,0x7d]
+# GFX11-FAKE16: v_cmpx_neq_f16_e32 vcc_lo, v2           ; encoding: [0x6a,0x04,0x1a,0x7d]
 
 0x6b,0x04,0x1a,0x7d
-# GFX11: v_cmpx_neq_f16_e32 vcc_hi, v2           ; encoding: [0x6b,0x04,0x1a,0x7d]
+# GFX11-REAL16: v_cmpx_neq_f16_e32 vcc_hi, v2.l         ; encoding: [0x6b,0x04,0x1a,0x7d]
+# GFX11-FAKE16: v_cmpx_neq_f16_e32 vcc_hi, v2           ; encoding: [0x6b,0x04,0x1a,0x7d]
 
 0x7b,0x04,0x1a,0x7d
-# GFX11: v_cmpx_neq_f16_e32 ttmp15, v2           ; encoding: [0x7b,0x04,0x1a,0x7d]
+# GFX11-REAL16: v_cmpx_neq_f16_e32 ttmp15, v2.l         ; encoding: [0x7b,0x04,0x1a,0x7d]
+# GFX11-FAKE16: v_cmpx_neq_f16_e32 ttmp15, v2           ; encoding: [0x7b,0x04,0x1a,0x7d]
 
 0x7d,0x04,0x1a,0x7d
-# GFX11: v_cmpx_neq_f16_e32 m0, v2               ; encoding: [0x7d,0x04,0x1a,0x7d]
+# GFX11-REAL16: v_cmpx_neq_f16_e32 m0, v2.l             ; encoding: [0x7d,0x04,0x1a,0x7d]
+# GFX11-FAKE16: v_cmpx_neq_f16_e32 m0, v2               ; encoding: [0x7d,0x04,0x1a,0x7d]
 
 0x7e,0x04,0x1a,0x7d
-# GFX11: v_cmpx_neq_f16_e32 exec_lo, v2          ; encoding: [0x7e,0x04,0x1a,0x7d]
+# GFX11-REAL16: v_cmpx_neq_f16_e32 exec_lo, v2.l        ; encoding: [0x7e,0x04,0x1a,0x7d]
+# GFX11-FAKE16: v_cmpx_neq_f16_e32 exec_lo, v2          ; encoding: [0x7e,0x04,0x1a,0x7d]
 
 0x7f,0x04,0x1a,0x7d
-# GFX11: v_cmpx_neq_f16_e32 exec_hi, v2          ; encoding: [0x7f,0x04,0x1a,0x7d]
+# GFX11-REAL16: v_cmpx_neq_f16_e32 exec_hi, v2.l        ; encoding: [0x7f,0x04,0x1a,0x7d]
+# GFX11-FAKE16: v_cmpx_neq_f16_e32 exec_hi, v2          ; encoding: [0x7f,0x04,0x1a,0x7d]
 
 0x7c,0x04,0x1a,0x7d
-# GFX11: v_cmpx_neq_f16_e32 null, v2             ; encoding: [0x7c,0x04,0x1a,0x7d]
+# GFX11-REAL16: v_cmpx_neq_f16_e32 null, v2.l           ; encoding: [0x7c,0x04,0x1a,0x7d]
+# GFX11-FAKE16: v_cmpx_neq_f16_e32 null, v2             ; encoding: [0x7c,0x04,0x1a,0x7d]
 
 0xc1,0x04,0x1a,0x7d
-# GFX11: v_cmpx_neq_f16_e32 -1, v2               ; encoding: [0xc1,0x04,0x1a,0x7d]
+# GFX11-REAL16: v_cmpx_neq_f16_e32 -1, v2.l             ; encoding: [0xc1,0x04,0x1a,0x7d]
+# GFX11-FAKE16: v_cmpx_neq_f16_e32 -1, v2               ; encoding: [0xc1,0x04,0x1a,0x7d]
 
 0xf0,0x04,0x1a,0x7d
-# GFX11: v_cmpx_neq_f16_e32 0.5, v2              ; encoding: [0xf0,0x04,0x1a,0x7d]
+# GFX11-REAL16: v_cmpx_neq_f16_e32 0.5, v2.l            ; encoding: [0xf0,0x04,0x1a,0x7d]
+# GFX11-FAKE16: v_cmpx_neq_f16_e32 0.5, v2              ; encoding: [0xf0,0x04,0x1a,0x7d]
 
 0xfd,0x04,0x1a,0x7d
-# GFX11: v_cmpx_neq_f16_e32 src_scc, v2          ; encoding: [0xfd,0x04,0x1a,0x7d]
+# GFX11-REAL16: v_cmpx_neq_f16_e32 src_scc, v2.l        ; encoding: [0xfd,0x04,0x1a,0x7d]
+# GFX11-FAKE16: v_cmpx_neq_f16_e32 src_scc, v2          ; encoding: [0xfd,0x04,0x1a,0x7d]
 
 0xff,0xfe,0x1a,0x7d,0x0b,0xfe,0x00,0x00
-# GFX11: v_cmpx_neq_f16_e32 0xfe0b, v127         ; encoding: [0xff,0xfe,0x1a,0x7d,0x0b,0xfe,0x00,0x00]
+# GFX11-REAL16: v_cmpx_neq_f16_e32 0xfe0b, v127.l       ; encoding: [0xff,0xfe,0x1a,0x7d,0x0b,0xfe,0x00,0x00]
+# GFX11-FAKE16: v_cmpx_neq_f16_e32 0xfe0b, v127         ; encoding: [0xff,0xfe,0x1a,0x7d,0x0b,0xfe,0x00,0x00]
+
+0x81,0x05,0x1a,0x7d
+# GFX11-REAL16: v_cmpx_neq_f16_e32 v1.h, v2.l           ; encoding: [0x81,0x05,0x1a,0x7d]
+# GFX11-FAKE16: v_cmpx_neq_f16_e32 v129/*Invalid register, operand has 'VS_32_Lo128' register class*/, v2 ; encoding: [0x81,0x05,0x1a,0x7d]
+
+0xff,0x05,0x1a,0x7d
+# GFX11-REAL16: v_cmpx_neq_f16_e32 v127.h, v2.l         ; encoding: [0xff,0x05,0x1a,0x7d]
+# GFX11-FAKE16: v_cmpx_neq_f16_e32 v255/*Invalid register, operand has 'VS_32_Lo128' register class*/, v2 ; encoding: [0xff,0x05,0x1a,0x7d]
+
+0xf0,0xfe,0x1a,0x7d
+# GFX11-REAL16: v_cmpx_neq_f16_e32 0.5, v127.l          ; encoding: [0xf0,0xfe,0x1a,0x7d]
+# GFX11-FAKE16: v_cmpx_neq_f16_e32 0.5, v127            ; encoding: [0xf0,0xfe,0x1a,0x7d]
+
+0xfd,0x04,0x1b,0x7d
+# GFX11-REAL16: v_cmpx_neq_f16_e32 src_scc, v2.h        ; encoding: [0xfd,0x04,0x1b,0x7d]
+# GFX11-FAKE16: v_cmpx_neq_f16_e32 src_scc, v130/*Invalid register, operand has 'VGPR_32_Lo128' register class*/ ; encoding: [0xfd,0x04,0x1b,0x7d]
+
+0xff,0xfe,0x1b,0x7d,0x0b,0xfe,0x00,0x00
+# GFX11-REAL16: v_cmpx_neq_f16_e32 0xfe0b, v127.h       ; encoding: [0xff,0xfe,0x1b,0x7d,0x0b,0xfe,0x00,0x00]
+# GFX11-FAKE16: v_cmpx_neq_f16_e32 0xfe0b, v255/*Invalid register, operand has 'VGPR_32_Lo128' register class*/ ; encoding: [0xff,0xfe,0x1b,0x7d,0x0b,0xfe,0x00,0x00]
 
 0x01,0x05,0x3a,0x7d
 # GFX11: v_cmpx_neq_f32_e32 v1, v2               ; encoding: [0x01,0x05,0x3a,0x7d]
@@ -3303,49 +3548,84 @@
 # GFX11: v_cmpx_neq_f64_e32 0xaf123456, v[254:255] ; encoding: [0xff,0xfc,0x5b,0x7d,0x56,0x34,0x12,0xaf]
 
 0x01,0x05,0x12,0x7d
-# GFX11: v_cmpx_nge_f16_e32 v1, v2               ; encoding: [0x01,0x05,0x12,0x7d]
+# GFX11-REAL16: v_cmpx_nge_f16_e32 v1.l, v2.l           ; encoding: [0x01,0x05,0x12,0x7d]
+# GFX11-FAKE16: v_cmpx_nge_f16_e32 v1, v2               ; encoding: [0x01,0x05,0x12,0x7d]
 
 0x7f,0x05,0x12,0x7d
-# GFX11: v_cmpx_nge_f16_e32 v127, v2             ; encoding: [0x7f,0x05,0x12,0x7d]
+# GFX11-REAL16: v_cmpx_nge_f16_e32 v127.l, v2.l         ; encoding: [0x7f,0x05,0x12,0x7d]
+# GFX11-FAKE16: v_cmpx_nge_f16_e32 v127, v2             ; encoding: [0x7f,0x05,0x12,0x7d]
 
 0x01,0x04,0x12,0x7d
-# GFX11: v_cmpx_nge_f16_e32 s1, v2               ; encoding: [0x01,0x04,0x12,0x7d]
+# GFX11-REAL16: v_cmpx_nge_f16_e32 s1, v2.l             ; encoding: [0x01,0x04,0x12,0x7d]
+# GFX11-FAKE16: v_cmpx_nge_f16_e32 s1, v2               ; encoding: [0x01,0x04,0x12,0x7d]
 
 0x69,0x04,0x12,0x7d
-# GFX11: v_cmpx_nge_f16_e32 s105, v2             ; encoding: [0x69,0x04,0x12,0x7d]
+# GFX11-REAL16: v_cmpx_nge_f16_e32 s105, v2.l           ; encoding: [0x69,0x04,0x12,0x7d]
+# GFX11-FAKE16: v_cmpx_nge_f16_e32 s105, v2             ; encoding: [0x69,0x04,0x12,0x7d]
 
 0x6a,0x04,0x12,0x7d
-# GFX11: v_cmpx_nge_f16_e32 vcc_lo, v2           ; encoding: [0x6a,0x04,0x12,0x7d]
+# GFX11-REAL16: v_cmpx_nge_f16_e32 vcc_lo, v2.l         ; encoding: [0x6a,0x04,0x12,0x7d]
+# GFX11-FAKE16: v_cmpx_nge_f16_e32 vcc_lo, v2           ; encoding: [0x6a,0x04,0x12,0x7d]
 
 0x6b,0x04,0x12,0x7d
-# GFX11: v_cmpx_nge_f16_e32 vcc_hi, v2           ; encoding: [0x6b,0x04,0x12,0x7d]
+# GFX11-REAL16: v_cmpx_nge_f16_e32 vcc_hi, v2.l         ; encoding: [0x6b,0x04,0x12,0x7d]
+# GFX11-FAKE16: v_cmpx_nge_f16_e32 vcc_hi, v2           ; encoding: [0x6b,0x04,0x12,0x7d]
 
 0x7b,0x04,0x12,0x7d
-# GFX11: v_cmpx_nge_f16_e32 ttmp15, v2           ; encoding: [0x7b,0x04,0x12,0x7d]
+# GFX11-REAL16: v_cmpx_nge_f16_e32 ttmp15, v2.l         ; encoding: [0x7b,0x04,0x12,0x7d]
+# GFX11-FAKE16: v_cmpx_nge_f16_e32 ttmp15, v2           ; encoding: [0x7b,0x04,0x12,0x7d]
 
 0x7d,0x04,0x12,0x7d
-# GFX11: v_cmpx_nge_f16_e32 m0, v2               ; encoding: [0x7d,0x04,0x12,0x7d]
+# GFX11-REAL16: v_cmpx_nge_f16_e32 m0, v2.l             ; encoding: [0x7d,0x04,0x12,0x7d]
+# GFX11-FAKE16: v_cmpx_nge_f16_e32 m0, v2               ; encoding: [0x7d,0x04,0x12,0x7d]
 
 0x7e,0x04,0x12,0x7d
-# GFX11: v_cmpx_nge_f16_e32 exec_lo, v2          ; encoding: [0x7e,0x04,0x12,0x7d]
+# GFX11-REAL16: v_cmpx_nge_f16_e32 exec_lo, v2.l        ; encoding: [0x7e,0x04,0x12,0x7d]
+# GFX11-FAKE16: v_cmpx_nge_f16_e32 exec_lo, v2          ; encoding: [0x7e,0x04,0x12,0x7d]
 
 0x7f,0x04,0x12,0x7d
-# GFX11: v_cmpx_nge_f16_e32 exec_hi, v2          ; encoding: [0x7f,0x04,0x12,0x7d]
+# GFX11-REAL16: v_cmpx_nge_f16_e32 exec_hi, v2.l        ; encoding: [0x7f,0x04,0x12,0x7d]
+# GFX11-FAKE16: v_cmpx_nge_f16_e32 exec_hi, v2          ; encoding: [0x7f,0x04,0x12,0x7d]
 
 0x7c,0x04,0x12,0x7d
-# GFX11: v_cmpx_nge_f16_e32 null, v2             ; encoding: [0x7c,0x04,0x12,0x7d]
+# GFX11-REAL16: v_cmpx_nge_f16_e32 null, v2.l           ; encoding: [0x7c,0x04,0x12,0x7d]
+# GFX11-FAKE16: v_cmpx_nge_f16_e32 null, v2             ; encoding: [0x7c,0x04,0x12,0x7d]
 
 0xc1,0x04,0x12,0x7d
-# GFX11: v_cmpx_nge_f16_e32 -1, v2               ; encoding: [0xc1,0x04,0x12,0x7d]
+# GFX11-REAL16: v_cmpx_nge_f16_e32 -1, v2.l             ; encoding: [0xc1,0x04,0x12,0x7d]
+# GFX11-FAKE16: v_cmpx_nge_f16_e32 -1, v2               ; encoding: [0xc1,0x04,0x12,0x7d]
 
 0xf0,0x04,0x12,0x7d
-# GFX11: v_cmpx_nge_f16_e32 0.5, v2              ; encoding: [0xf0,0x04,0x12,0x7d]
+# GFX11-REAL16: v_cmpx_nge_f16_e32 0.5, v2.l            ; encoding: [0xf0,0x04,0x12,0x7d]
+# GFX11-FAKE16: v_cmpx_nge_f16_e32 0.5, v2              ; encoding: [0xf0,0x04,0x12,0x7d]
 
 0xfd,0x04,0x12,0x7d
-# GFX11: v_cmpx_nge_f16_e32 src_scc, v2          ; encoding: [0xfd,0x04,0x12,0x7d]
+# GFX11-REAL16: v_cmpx_nge_f16_e32 src_scc, v2.l        ; encoding: [0xfd,0x04,0x12,0x7d]
+# GFX11-FAKE16: v_cmpx_nge_f16_e32 src_scc, v2          ; encoding: [0xfd,0x04,0x12,0x7d]
 
 0xff,0xfe,0x12,0x7d,0x0b,0xfe,0x00,0x00
-# GFX11: v_cmpx_nge_f16_e32 0xfe0b, v127         ; encoding: [0xff,0xfe,0x12,0x7d,0x0b,0xfe,0x00,0x00]
+# GFX11-REAL16: v_cmpx_nge_f16_e32 0xfe0b, v127.l       ; encoding: [0xff,0xfe,0x12,0x7d,0x0b,0xfe,0x00,0x00]
+# GFX11-FAKE16: v_cmpx_nge_f16_e32 0xfe0b, v127         ; encoding: [0xff,0xfe,0x12,0x7d,0x0b,0xfe,0x00,0x00]
+
+0x81,0x05,0x12,0x7d
+# GFX11-REAL16: v_cmpx_nge_f16_e32 v1.h, v2.l           ; encoding: [0x81,0x05,0x12,0x7d]
+# GFX11-FAKE16: v_cmpx_nge_f16_e32 v129/*Invalid register, operand has 'VS_32_Lo128' register class*/, v2 ; encoding: [0x81,0x05,0x12,0x7d]
+
+0xff,0x05,0x12,0x7d
+# GFX11-REAL16: v_cmpx_nge_f16_e32 v127.h, v2.l         ; encoding: [0xff,0x05,0x12,0x7d]
+# GFX11-FAKE16: v_cmpx_nge_f16_e32 v255/*Invalid register, operand has 'VS_32_Lo128' register class*/, v2 ; encoding: [0xff,0x05,0x12,0x7d]
+
+0xf0,0xfe,0x12,0x7d
+# GFX11-REAL16: v_cmpx_nge_f16_e32 0.5, v127.l          ; encoding: [0xf0,0xfe,0x12,0x7d]
+# GFX11-FAKE16: v_cmpx_nge_f16_e32 0.5, v127            ; encoding: [0xf0,0xfe,0x12,0x7d]
+
+0xfd,0x04,0x13,0x7d
+# GFX11-REAL16: v_cmpx_nge_f16_e32 src_scc, v2.h        ; encoding: [0xfd,0x04,0x13,0x7d]
+# GFX11-FAKE16: v_cmpx_nge_f16_e32 src_scc, v130/*Invalid register, operand has 'VGPR_32_Lo128' register class*/ ; encoding: [0xfd,0x04,0x13,0x7d]
+
+0xff,0xfe,0x13,0x7d,0x0b,0xfe,0x00,0x00
+# GFX11-REAL16: v_cmpx_nge_f16_e32 0xfe0b, v127.h       ; encoding: [0xff,0xfe,0x13,0x7d,0x0b,0xfe,0x00,0x00]
+# GFX11-FAKE16: v_cmpx_nge_f16_e32 0xfe0b, v255/*Invalid register, operand has 'VGPR_32_Lo128' register class*/ ; encoding: [0xff,0xfe,0x13,0x7d,0x0b,0xfe,0x00,0x00]
 
 0x01,0x05,0x32,0x7d
 # GFX11: v_cmpx_nge_f32_e32 v1, v2               ; encoding: [0x01,0x05,0x32,0x7d]
@@ -3429,49 +3709,84 @@
 # GFX11: v_cmpx_nge_f64_e32 0xaf123456, v[254:255] ; encoding: [0xff,0xfc,0x53,0x7d,0x56,0x34,0x12,0xaf]
 
 0x01,0x05,0x16,0x7d
-# GFX11: v_cmpx_ngt_f16_e32 v1, v2               ; encoding: [0x01,0x05,0x16,0x7d]
+# GFX11-REAL16: v_cmpx_ngt_f16_e32 v1.l, v2.l           ; encoding: [0x01,0x05,0x16,0x7d]
+# GFX11-FAKE16: v_cmpx_ngt_f16_e32 v1, v2               ; encoding: [0x01,0x05,0x16,0x7d]
 
 0x7f,0x05,0x16,0x7d
-# GFX11: v_cmpx_ngt_f16_e32 v127, v2             ; encoding: [0x7f,0x05,0x16,0x7d]
+# GFX11-REAL16: v_cmpx_ngt_f16_e32 v127.l, v2.l         ; encoding: [0x7f,0x05,0x16,0x7d]
+# GFX11-FAKE16: v_cmpx_ngt_f16_e32 v127, v2             ; encoding: [0x7f,0x05,0x16,0x7d]
 
 0x01,0x04,0x16,0x7d
-# GFX11: v_cmpx_ngt_f16_e32 s1, v2               ; encoding: [0x01,0x04,0x16,0x7d]
+# GFX11-REAL16: v_cmpx_ngt_f16_e32 s1, v2.l             ; encoding: [0x01,0x04,0x16,0x7d]
+# GFX11-FAKE16: v_cmpx_ngt_f16_e32 s1, v2               ; encoding: [0x01,0x04,0x16,0x7d]
 
 0x69,0x04,0x16,0x7d
-# GFX11: v_cmpx_ngt_f16_e32 s105, v2             ; encoding: [0x69,0x04,0x16,0x7d]
+# GFX11-REAL16: v_cmpx_ngt_f16_e32 s105, v2.l           ; encoding: [0x69,0x04,0x16,0x7d]
+# GFX11-FAKE16: v_cmpx_ngt_f16_e32 s105, v2             ; encoding: [0x69,0x04,0x16,0x7d]
 
 0x6a,0x04,0x16,0x7d
-# GFX11: v_cmpx_ngt_f16_e32 vcc_lo, v2           ; encoding: [0x6a,0x04,0x16,0x7d]
+# GFX11-REAL16: v_cmpx_ngt_f16_e32 vcc_lo, v2.l         ; encoding: [0x6a,0x04,0x16,0x7d]
+# GFX11-FAKE16: v_cmpx_ngt_f16_e32 vcc_lo, v2           ; encoding: [0x6a,0x04,0x16,0x7d]
 
 0x6b,0x04,0x16,0x7d
-# GFX11: v_cmpx_ngt_f16_e32 vcc_hi, v2           ; encoding: [0x6b,0x04,0x16,0x7d]
+# GFX11-REAL16: v_cmpx_ngt_f16_e32 vcc_hi, v2.l         ; encoding: [0x6b,0x04,0x16,0x7d]
+# GFX11-FAKE16: v_cmpx_ngt_f16_e32 vcc_hi, v2           ; encoding: [0x6b,0x04,0x16,0x7d]
 
 0x7b,0x04,0x16,0x7d
-# GFX11: v_cmpx_ngt_f16_e32 ttmp15, v2           ; encoding: [0x7b,0x04,0x16,0x7d]
+# GFX11-REAL16: v_cmpx_ngt_f16_e32 ttmp15, v2.l         ; encoding: [0x7b,0x04,0x16,0x7d]
+# GFX11-FAKE16: v_cmpx_ngt_f16_e32 ttmp15, v2           ; encoding: [0x7b,0x04,0x16,0x7d]
 
 0x7d,0x04,0x16,0x7d
-# GFX11: v_cmpx_ngt_f16_e32 m0, v2               ; encoding: [0x7d,0x04,0x16,0x7d]
+# GFX11-REAL16: v_cmpx_ngt_f16_e32 m0, v2.l             ; encoding: [0x7d,0x04,0x16,0x7d]
+# GFX11-FAKE16: v_cmpx_ngt_f16_e32 m0, v2               ; encoding: [0x7d,0x04,0x16,0x7d]
 
 0x7e,0x04,0x16,0x7d
-# GFX11: v_cmpx_ngt_f16_e32 exec_lo, v2          ; encoding: [0x7e,0x04,0x16,0x7d]
+# GFX11-REAL16: v_cmpx_ngt_f16_e32 exec_lo, v2.l        ; encoding: [0x7e,0x04,0x16,0x7d]
+# GFX11-FAKE16: v_cmpx_ngt_f16_e32 exec_lo, v2          ; encoding: [0x7e,0x04,0x16,0x7d]
 
 0x7f,0x04,0x16,0x7d
-# GFX11: v_cmpx_ngt_f16_e32 exec_hi, v2          ; encoding: [0x7f,0x04,0x16,0x7d]
+# GFX11-REAL16: v_cmpx_ngt_f16_e32 exec_hi, v2.l        ; encoding: [0x7f,0x04,0x16,0x7d]
+# GFX11-FAKE16: v_cmpx_ngt_f16_e32 exec_hi, v2          ; encoding: [0x7f,0x04,0x16,0x7d]
 
 0x7c,0x04,0x16,0x7d
-# GFX11: v_cmpx_ngt_f16_e32 null, v2             ; encoding: [0x7c,0x04,0x16,0x7d]
+# GFX11-REAL16: v_cmpx_ngt_f16_e32 null, v2.l           ; encoding: [0x7c,0x04,0x16,0x7d]
+# GFX11-FAKE16: v_cmpx_ngt_f16_e32 null, v2             ; encoding: [0x7c,0x04,0x16,0x7d]
 
 0xc1,0x04,0x16,0x7d
-# GFX11: v_cmpx_ngt_f16_e32 -1, v2               ; encoding: [0xc1,0x04,0x16,0x7d]
+# GFX11-REAL16: v_cmpx_ngt_f16_e32 -1, v2.l             ; encoding: [0xc1,0x04,0x16,0x7d]
+# GFX11-FAKE16: v_cmpx_ngt_f16_e32 -1, v2               ; encoding: [0xc1,0x04,0x16,0x7d]
 
 0xf0,0x04,0x16,0x7d
-# GFX11: v_cmpx_ngt_f16_e32 0.5, v2              ; encoding: [0xf0,0x04,0x16,0x7d]
+# GFX11-REAL16: v_cmpx_ngt_f16_e32 0.5, v2.l            ; encoding: [0xf0,0x04,0x16,0x7d]
+# GFX11-FAKE16: v_cmpx_ngt_f16_e32 0.5, v2              ; encoding: [0xf0,0x04,0x16,0x7d]
 
 0xfd,0x04,0x16,0x7d
-# GFX11: v_cmpx_ngt_f16_e32 src_scc, v2          ; encoding: [0xfd,0x04,0x16,0x7d]
+# GFX11-REAL16: v_cmpx_ngt_f16_e32 src_scc, v2.l        ; encoding: [0xfd,0x04,0x16,0x7d]
+# GFX11-FAKE16: v_cmpx_ngt_f16_e32 src_scc, v2          ; encoding: [0xfd,0x04,0x16,0x7d]
 
 0xff,0xfe,0x16,0x7d,0x0b,0xfe,0x00,0x00
-# GFX11: v_cmpx_ngt_f16_e32 0xfe0b, v127         ; encoding: [0xff,0xfe,0x16,0x7d,0x0b,0xfe,0x00,0x00]
+# GFX11-REAL16: v_cmpx_ngt_f16_e32 0xfe0b, v127.l       ; encoding: [0xff,0xfe,0x16,0x7d,0x0b,0xfe,0x00,0x00]
+# GFX11-FAKE16: v_cmpx_ngt_f16_e32 0xfe0b, v127         ; encoding: [0xff,0xfe,0x16,0x7d,0x0b,0xfe,0x00,0x00]
+
+0x81,0x05,0x16,0x7d
+# GFX11-REAL16: v_cmpx_ngt_f16_e32 v1.h, v2.l           ; encoding: [0x81,0x05,0x16,0x7d]
+# GFX11-FAKE16: v_cmpx_ngt_f16_e32 v129/*Invalid register, operand has 'VS_32_Lo128' register class*/, v2 ; encoding: [0x81,0x05,0x16,0x7d]
+
+0xff,0x05,0x16,0x7d
+# GFX11-REAL16: v_cmpx_ngt_f16_e32 v127.h, v2.l         ; encoding: [0xff,0x05,0x16,0x7d]
+# GFX11-FAKE16: v_cmpx_ngt_f16_e32 v255/*Invalid register, operand has 'VS_32_Lo128' register class*/, v2 ; encoding: [0xff,0x05,0x16,0x7d]
+
+0xf0,0xfe,0x16,0x7d
+# GFX11-REAL16: v_cmpx_ngt_f16_e32 0.5, v127.l          ; encoding: [0xf0,0xfe,0x16,0x7d]
+# GFX11-FAKE16: v_cmpx_ngt_f16_e32 0.5, v127            ; encoding: [0xf0,0xfe,0x16,0x7d]
+
+0xfd,0x04,0x17,0x7d
+# GFX11-REAL16: v_cmpx_ngt_f16_e32 src_scc, v2.h        ; encoding: [0xfd,0x04,0x17,0x7d]
+# GFX11-FAKE16: v_cmpx_ngt_f16_e32 src_scc, v130/*Invalid register, operand has 'VGPR_32_Lo128' register class*/ ; encoding: [0xfd,0x04,0x17,0x7d]
+
+0xff,0xfe,0x17,0x7d,0x0b,0xfe,0x00,0x00
+# GFX11-REAL16: v_cmpx_ngt_f16_e32 0xfe0b, v127.h       ; encoding: [0xff,0xfe,0x17,0x7d,0x0b,0xfe,0x00,0x00]
+# GFX11-FAKE16: v_cmpx_ngt_f16_e32 0xfe0b, v255/*Invalid register, operand has 'VGPR_32_Lo128' register class*/ ; encoding: [0xff,0xfe,0x17,0x7d,0x0b,0xfe,0x00,0x00]
 
 0x01,0x05,0x36,0x7d
 # GFX11: v_cmpx_ngt_f32_e32 v1, v2               ; encoding: [0x01,0x05,0x36,0x7d]
@@ -3555,49 +3870,84 @@
 # GFX11: v_cmpx_ngt_f64_e32 0xaf123456, v[254:255] ; encoding: [0xff,0xfc,0x57,0x7d,0x56,0x34,0x12,0xaf]
 
 0x01,0x05,0x18,0x7d
-# GFX11: v_cmpx_nle_f16_e32 v1, v2               ; encoding: [0x01,0x05,0x18,0x7d]
+# GFX11-REAL16: v_cmpx_nle_f16_e32 v1.l, v2.l           ; encoding: [0x01,0x05,0x18,0x7d]
+# GFX11-FAKE16: v_cmpx_nle_f16_e32 v1, v2               ; encoding: [0x01,0x05,0x18,0x7d]
 
 0x7f,0x05,0x18,0x7d
-# GFX11: v_cmpx_nle_f16_e32 v127, v2             ; encoding: [0x7f,0x05,0x18,0x7d]
+# GFX11-REAL16: v_cmpx_nle_f16_e32 v127.l, v2.l         ; encoding: [0x7f,0x05,0x18,0x7d]
+# GFX11-FAKE16: v_cmpx_nle_f16_e32 v127, v2             ; encoding: [0x7f,0x05,0x18,0x7d]
 
 0x01,0x04,0x18,0x7d
-# GFX11: v_cmpx_nle_f16_e32 s1, v2               ; encoding: [0x01,0x04,0x18,0x7d]
+# GFX11-REAL16: v_cmpx_nle_f16_e32 s1, v2.l             ; encoding: [0x01,0x04,0x18,0x7d]
+# GFX11-FAKE16: v_cmpx_nle_f16_e32 s1, v2               ; encoding: [0x01,0x04,0x18,0x7d]
 
 0x69,0x04,0x18,0x7d
-# GFX11: v_cmpx_nle_f16_e32 s105, v2             ; encoding: [0x69,0x04,0x18,0x7d]
+# GFX11-REAL16: v_cmpx_nle_f16_e32 s105, v2.l           ; encoding: [0x69,0x04,0x18,0x7d]
+# GFX11-FAKE16: v_cmpx_nle_f16_e32 s105, v2             ; encoding: [0x69,0x04,0x18,0x7d]
 
 0x6a,0x04,0x18,0x7d
-# GFX11: v_cmpx_nle_f16_e32 vcc_lo, v2           ; encoding: [0x6a,0x04,0x18,0x7d]
+# GFX11-REAL16: v_cmpx_nle_f16_e32 vcc_lo, v2.l         ; encoding: [0x6a,0x04,0x18,0x7d]
+# GFX11-FAKE16: v_cmpx_nle_f16_e32 vcc_lo, v2           ; encoding: [0x6a,0x04,0x18,0x7d]
 
 0x6b,0x04,0x18,0x7d
-# GFX11: v_cmpx_nle_f16_e32 vcc_hi, v2           ; encoding: [0x6b,0x04,0x18,0x7d]
+# GFX11-REAL16: v_cmpx_nle_f16_e32 vcc_hi, v2.l         ; encoding: [0x6b,0x04,0x18,0x7d]
+# GFX11-FAKE16: v_cmpx_nle_f16_e32 vcc_hi, v2           ; encoding: [0x6b,0x04,0x18,0x7d]
 
 0x7b,0x04,0x18,0x7d
-# GFX11: v_cmpx_nle_f16_e32 ttmp15, v2           ; encoding: [0x7b,0x04,0x18,0x7d]
+# GFX11-REAL16: v_cmpx_nle_f16_e32 ttmp15, v2.l         ; encoding: [0x7b,0x04,0x18,0x7d]
+# GFX11-FAKE16: v_cmpx_nle_f16_e32 ttmp15, v2           ; encoding: [0x7b,0x04,0x18,0x7d]
 
 0x7d,0x04,0x18,0x7d
-# GFX11: v_cmpx_nle_f16_e32 m0, v2               ; encoding: [0x7d,0x04,0x18,0x7d]
+# GFX11-REAL16: v_cmpx_nle_f16_e32 m0, v2.l             ; encoding: [0x7d,0x04,0x18,0x7d]
+# GFX11-FAKE16: v_cmpx_nle_f16_e32 m0, v2               ; encoding: [0x7d,0x04,0x18,0x7d]
 
 0x7e,0x04,0x18,0x7d
-# GFX11: v_cmpx_nle_f16_e32 exec_lo, v2          ; encoding: [0x7e,0x04,0x18,0x7d]
+# GFX11-REAL16: v_cmpx_nle_f16_e32 exec_lo, v2.l        ; encoding: [0x7e,0x04,0x18,0x7d]
+# GFX11-FAKE16: v_cmpx_nle_f16_e32 exec_lo, v2          ; encoding: [0x7e,0x04,0x18,0x7d]
 
 0x7f,0x04,0x18,0x7d
-# GFX11: v_cmpx_nle_f16_e32 exec_hi, v2          ; encoding: [0x7f,0x04,0x18,0x7d]
+# GFX11-REAL16: v_cmpx_nle_f16_e32 exec_hi, v2.l        ; encoding: [0x7f,0x04,0x18,0x7d]
+# GFX11-FAKE16: v_cmpx_nle_f16_e32 exec_hi, v2          ; encoding: [0x7f,0x04,0x18,0x7d]
 
 0x7c,0x04,0x18,0x7d
-# GFX11: v_cmpx_nle_f16_e32 null, v2             ; encoding: [0x7c,0x04,0x18,0x7d]
+# GFX11-REAL16: v_cmpx_nle_f16_e32 null, v2.l           ; encoding: [0x7c,0x04,0x18,0x7d]
+# GFX11-FAKE16: v_cmpx_nle_f16_e32 null, v2             ; encoding: [0x7c,0x04,0x18,0x7d]
 
 0xc1,0x04,0x18,0x7d
-# GFX11: v_cmpx_nle_f16_e32 -1, v2               ; encoding: [0xc1,0x04,0x18,0x7d]
+# GFX11-REAL16: v_cmpx_nle_f16_e32 -1, v2.l             ; encoding: [0xc1,0x04,0x18,0x7d]
+# GFX11-FAKE16: v_cmpx_nle_f16_e32 -1, v2               ; encoding: [0xc1,0x04,0x18,0x7d]
 
 0xf0,0x04,0x18,0x7d
-# GFX11: v_cmpx_nle_f16_e32 0.5, v2              ; encoding: [0xf0,0x04,0x18,0x7d]
+# GFX11-REAL16: v_cmpx_nle_f16_e32 0.5, v2.l            ; encoding: [0xf0,0x04,0x18,0x7d]
+# GFX11-FAKE16: v_cmpx_nle_f16_e32 0.5, v2              ; encoding: [0xf0,0x04,0x18,0x7d]
 
 0xfd,0x04,0x18,0x7d
-# GFX11: v_cmpx_nle_f16_e32 src_scc, v2          ; encoding: [0xfd,0x04,0x18,0x7d]
+# GFX11-REAL16: v_cmpx_nle_f16_e32 src_scc, v2.l        ; encoding: [0xfd,0x04,0x18,0x7d]
+# GFX11-FAKE16: v_cmpx_nle_f16_e32 src_scc, v2          ; encoding: [0xfd,0x04,0x18,0x7d]
 
 0xff,0xfe,0x18,0x7d,0x0b,0xfe,0x00,0x00
-# GFX11: v_cmpx_nle_f16_e32 0xfe0b, v127         ; encoding: [0xff,0xfe,0x18,0x7d,0x0b,0xfe,0x00,0x00]
+# GFX11-REAL16: v_cmpx_nle_f16_e32 0xfe0b, v127.l       ; encoding: [0xff,0xfe,0x18,0x7d,0x0b,0xfe,0x00,0x00]
+# GFX11-FAKE16: v_cmpx_nle_f16_e32 0xfe0b, v127         ; encoding: [0xff,0xfe,0x18,0x7d,0x0b,0xfe,0x00,0x00]
+
+0x81,0x05,0x18,0x7d
+# GFX11-REAL16: v_cmpx_nle_f16_e32 v1.h, v2.l           ; encoding: [0x81,0x05,0x18,0x7d]
+# GFX11-FAKE16: v_cmpx_nle_f16_e32 v129/*Invalid register, operand has 'VS_32_Lo128' register class*/, v2 ; encoding: [0x81,0x05,0x18,0x7d]
+
+0xff,0x05,0x18,0x7d
+# GFX11-REAL16: v_cmpx_nle_f16_e32 v127.h, v2.l         ; encoding: [0xff,0x05,0x18,0x7d]
+# GFX11-FAKE16: v_cmpx_nle_f16_e32 v255/*Invalid register, operand has 'VS_32_Lo128' register class*/, v2 ; encoding: [0xff,0x05,0x18,0x7d]
+
+0xf0,0xfe,0x18,0x7d
+# GFX11-REAL16: v_cmpx_nle_f16_e32 0.5, v127.l          ; encoding: [0xf0,0xfe,0x18,0x7d]
+# GFX11-FAKE16: v_cmpx_nle_f16_e32 0.5, v127            ; encoding: [0xf0,0xfe,0x18,0x7d]
+
+0xfd,0x04,0x19,0x7d
+# GFX11-REAL16: v_cmpx_nle_f16_e32 src_scc, v2.h        ; encoding: [0xfd,0x04,0x19,0x7d]
+# GFX11-FAKE16: v_cmpx_nle_f16_e32 src_scc, v130/*Invalid register, operand has 'VGPR_32_Lo128' register class*/ ; encoding: [0xfd,0x04,0x19,0x7d]
+
+0xff,0xfe,0x19,0x7d,0x0b,0xfe,0x00,0x00
+# GFX11-REAL16: v_cmpx_nle_f16_e32 0xfe0b, v127.h       ; encoding: [0xff,0xfe,0x19,0x7d,0x0b,0xfe,0x00,0x00]
+# GFX11-FAKE16: v_cmpx_nle_f16_e32 0xfe0b, v255/*Invalid register, operand has 'VGPR_32_Lo128' register class*/ ; encoding: [0xff,0xfe,0x19,0x7d,0x0b,0xfe,0x00,0x00]
 
 0x01,0x05,0x38,0x7d
 # GFX11: v_cmpx_nle_f32_e32 v1, v2               ; encoding: [0x01,0x05,0x38,0x7d]
@@ -3681,49 +4031,84 @@
 # GFX11: v_cmpx_nle_f64_e32 0xaf123456, v[254:255] ; encoding: [0xff,0xfc,0x59,0x7d,0x56,0x34,0x12,0xaf]
 
 0x01,0x05,0x14,0x7d
-# GFX11: v_cmpx_nlg_f16_e32 v1, v2               ; encoding: [0x01,0x05,0x14,0x7d]
+# GFX11-REAL16: v_cmpx_nlg_f16_e32 v1.l, v2.l           ; encoding: [0x01,0x05,0x14,0x7d]
+# GFX11-FAKE16: v_cmpx_nlg_f16_e32 v1, v2               ; encoding: [0x01,0x05,0x14,0x7d]
 
 0x7f,0x05,0x14,0x7d
-# GFX11: v_cmpx_nlg_f16_e32 v127, v2             ; encoding: [0x7f,0x05,0x14,0x7d]
+# GFX11-REAL16: v_cmpx_nlg_f16_e32 v127.l, v2.l         ; encoding: [0x7f,0x05,0x14,0x7d]
+# GFX11-FAKE16: v_cmpx_nlg_f16_e32 v127, v2             ; encoding: [0x7f,0x05,0x14,0x7d]
 
 0x01,0x04,0x14,0x7d
-# GFX11: v_cmpx_nlg_f16_e32 s1, v2               ; encoding: [0x01,0x04,0x14,0x7d]
+# GFX11-REAL16: v_cmpx_nlg_f16_e32 s1, v2.l             ; encoding: [0x01,0x04,0x14,0x7d]
+# GFX11-FAKE16: v_cmpx_nlg_f16_e32 s1, v2               ; encoding: [0x01,0x04,0x14,0x7d]
 
 0x69,0x04,0x14,0x7d
-# GFX11: v_cmpx_nlg_f16_e32 s105, v2             ; encoding: [0x69,0x04,0x14,0x7d]
+# GFX11-REAL16: v_cmpx_nlg_f16_e32 s105, v2.l           ; encoding: [0x69,0x04,0x14,0x7d]
+# GFX11-FAKE16: v_cmpx_nlg_f16_e32 s105, v2             ; encoding: [0x69,0x04,0x14,0x7d]
 
 0x6a,0x04,0x14,0x7d
-# GFX11: v_cmpx_nlg_f16_e32 vcc_lo, v2           ; encoding: [0x6a,0x04,0x14,0x7d]
+# GFX11-REAL16: v_cmpx_nlg_f16_e32 vcc_lo, v2.l         ; encoding: [0x6a,0x04,0x14,0x7d]
+# GFX11-FAKE16: v_cmpx_nlg_f16_e32 vcc_lo, v2           ; encoding: [0x6a,0x04,0x14,0x7d]
 
 0x6b,0x04,0x14,0x7d
-# GFX11: v_cmpx_nlg_f16_e32 vcc_hi, v2           ; encoding: [0x6b,0x04,0x14,0x7d]
+# GFX11-REAL16: v_cmpx_nlg_f16_e32 vcc_hi, v2.l         ; encoding: [0x6b,0x04,0x14,0x7d]
+# GFX11-FAKE16: v_cmpx_nlg_f16_e32 vcc_hi, v2           ; encoding: [0x6b,0x04,0x14,0x7d]
 
 0x7b,0x04,0x14,0x7d
-# GFX11: v_cmpx_nlg_f16_e32 ttmp15, v2           ; encoding: [0x7b,0x04,0x14,0x7d]
+# GFX11-REAL16: v_cmpx_nlg_f16_e32 ttmp15, v2.l         ; encoding: [0x7b,0x04,0x14,0x7d]
+# GFX11-FAKE16: v_cmpx_nlg_f16_e32 ttmp15, v2           ; encoding: [0x7b,0x04,0x14,0x7d]
 
 0x7d,0x04,0x14,0x7d
-# GFX11: v_cmpx_nlg_f16_e32 m0, v2               ; encoding: [0x7d,0x04,0x14,0x7d]
+# GFX11-REAL16: v_cmpx_nlg_f16_e32 m0, v2.l             ; encoding: [0x7d,0x04,0x14,0x7d]
+# GFX11-FAKE16: v_cmpx_nlg_f16_e32 m0, v2               ; encoding: [0x7d,0x04,0x14,0x7d]
 
 0x7e,0x04,0x14,0x7d
-# GFX11: v_cmpx_nlg_f16_e32 exec_lo, v2          ; encoding: [0x7e,0x04,0x14,0x7d]
+# GFX11-REAL16: v_cmpx_nlg_f16_e32 exec_lo, v2.l        ; encoding: [0x7e,0x04,0x14,0x7d]
+# GFX11-FAKE16: v_cmpx_nlg_f16_e32 exec_lo, v2          ; encoding: [0x7e,0x04,0x14,0x7d]
 
 0x7f,0x04,0x14,0x7d
-# GFX11: v_cmpx_nlg_f16_e32 exec_hi, v2          ; encoding: [0x7f,0x04,0x14,0x7d]
+# GFX11-REAL16: v_cmpx_nlg_f16_e32 exec_hi, v2.l        ; encoding: [0x7f,0x04,0x14,0x7d]
+# GFX11-FAKE16: v_cmpx_nlg_f16_e32 exec_hi, v2          ; encoding: [0x7f,0x04,0x14,0x7d]
 
 0x7c,0x04,0x14,0x7d
-# GFX11: v_cmpx_nlg_f16_e32 null, v2             ; encoding: [0x7c,0x04,0x14,0x7d]
+# GFX11-REAL16: v_cmpx_nlg_f16_e32 null, v2.l           ; encoding: [0x7c,0x04,0x14,0x7d]
+# GFX11-FAKE16: v_cmpx_nlg_f16_e32 null, v2             ; encoding: [0x7c,0x04,0x14,0x7d]
 
 0xc1,0x04,0x14,0x7d
-# GFX11: v_cmpx_nlg_f16_e32 -1, v2               ; encoding: [0xc1,0x04,0x14,0x7d]
+# GFX11-REAL16: v_cmpx_nlg_f16_e32 -1, v2.l             ; encoding: [0xc1,0x04,0x14,0x7d]
+# GFX11-FAKE16: v_cmpx_nlg_f16_e32 -1, v2               ; encoding: [0xc1,0x04,0x14,0x7d]
 
 0xf0,0x04,0x14,0x7d
-# GFX11: v_cmpx_nlg_f16_e32 0.5, v2              ; encoding: [0xf0,0x04,0x14,0x7d]
+# GFX11-REAL16: v_cmpx_nlg_f16_e32 0.5, v2.l            ; encoding: [0xf0,0x04,0x14,0x7d]
+# GFX11-FAKE16: v_cmpx_nlg_f16_e32 0.5, v2              ; encoding: [0xf0,0x04,0x14,0x7d]
 
 0xfd,0x04,0x14,0x7d
-# GFX11: v_cmpx_nlg_f16_e32 src_scc, v2          ; encoding: [0xfd,0x04,0x14,0x7d]
+# GFX11-REAL16: v_cmpx_nlg_f16_e32 src_scc, v2.l        ; encoding: [0xfd,0x04,0x14,0x7d]
+# GFX11-FAKE16: v_cmpx_nlg_f16_e32 src_scc, v2          ; encoding: [0xfd,0x04,0x14,0x7d]
 
 0xff,0xfe,0x14,0x7d,0x0b,0xfe,0x00,0x00
-# GFX11: v_cmpx_nlg_f16_e32 0xfe0b, v127         ; encoding: [0xff,0xfe,0x14,0x7d,0x0b,0xfe,0x00,0x00]
+# GFX11-REAL16: v_cmpx_nlg_f16_e32 0xfe0b, v127.l       ; encoding: [0xff,0xfe,0x14,0x7d,0x0b,0xfe,0x00,0x00]
+# GFX11-FAKE16: v_cmpx_nlg_f16_e32 0xfe0b, v127         ; encoding: [0xff,0xfe,0x14,0x7d,0x0b,0xfe,0x00,0x00]
+
+0x81,0x05,0x14,0x7d
+# GFX11-REAL16: v_cmpx_nlg_f16_e32 v1.h, v2.l           ; encoding: [0x81,0x05,0x14,0x7d]
+# GFX11-FAKE16: v_cmpx_nlg_f16_e32 v129/*Invalid register, operand has 'VS_32_Lo128' register class*/, v2 ; encoding: [0x81,0x05,0x14,0x7d]
+
+0xff,0x05,0x14,0x7d
+# GFX11-REAL16: v_cmpx_nlg_f16_e32 v127.h, v2.l         ; encoding: [0xff,0x05,0x14,0x7d]
+# GFX11-FAKE16: v_cmpx_nlg_f16_e32 v255/*Invalid register, operand has 'VS_32_Lo128' register class*/, v2 ; encoding: [0xff,0x05,0x14,0x7d]
+
+0xf0,0xfe,0x14,0x7d
+# GFX11-REAL16: v_cmpx_nlg_f16_e32 0.5, v127.l          ; encoding: [0xf0,0xfe,0x14,0x7d]
+# GFX11-FAKE16: v_cmpx_nlg_f16_e32 0.5, v127            ; encoding: [0xf0,0xfe,0x14,0x7d]
+
+0xfd,0x04,0x15,0x7d
+# GFX11-REAL16: v_cmpx_nlg_f16_e32 src_scc, v2.h        ; encoding: [0xfd,0x04,0x15,0x7d]
+# GFX11-FAKE16: v_cmpx_nlg_f16_e32 src_scc, v130/*Invalid register, operand has 'VGPR_32_Lo128' register class*/ ; encoding: [0xfd,0x04,0x15,0x7d]
+
+0xff,0xfe,0x15,0x7d,0x0b,0xfe,0x00,0x00
+# GFX11-REAL16: v_cmpx_nlg_f16_e32 0xfe0b, v127.h       ; encoding: [0xff,0xfe,0x15,0x7d,0x0b,0xfe,0x00,0x00]
+# GFX11-FAKE16: v_cmpx_nlg_f16_e32 0xfe0b, v255/*Invalid register, operand has 'VGPR_32_Lo128' register class*/ ; encoding: [0xff,0xfe,0x15,0x7d,0x0b,0xfe,0x00,0x00]
 
 0x01,0x05,0x34,0x7d
 # GFX11: v_cmpx_nlg_f32_e32 v1, v2               ; encoding: [0x01,0x05,0x34,0x7d]
@@ -3807,49 +4192,84 @@
 # GFX11: v_cmpx_nlg_f64_e32 0xaf123456, v[254:255] ; encoding: [0xff,0xfc,0x55,0x7d,0x56,0x34,0x12,0xaf]
 
 0x01,0x05,0x1c,0x7d
-# GFX11: v_cmpx_nlt_f16_e32 v1, v2               ; encoding: [0x01,0x05,0x1c,0x7d]
+# GFX11-REAL16: v_cmpx_nlt_f16_e32 v1.l, v2.l           ; encoding: [0x01,0x05,0x1c,0x7d]
+# GFX11-FAKE16: v_cmpx_nlt_f16_e32 v1, v2               ; encoding: [0x01,0x05,0x1c,0x7d]
 
 0x7f,0x05,0x1c,0x7d
-# GFX11: v_cmpx_nlt_f16_e32 v127, v2             ; encoding: [0x7f,0x05,0x1c,0x7d]
+# GFX11-REAL16: v_cmpx_nlt_f16_e32 v127.l, v2.l         ; encoding: [0x7f,0x05,0x1c,0x7d]
+# GFX11-FAKE16: v_cmpx_nlt_f16_e32 v127, v2             ; encoding: [0x7f,0x05,0x1c,0x7d]
 
 0x01,0x04,0x1c,0x7d
-# GFX11: v_cmpx_nlt_f16_e32 s1, v2               ; encoding: [0x01,0x04,0x1c,0x7d]
+# GFX11-REAL16: v_cmpx_nlt_f16_e32 s1, v2.l             ; encoding: [0x01,0x04,0x1c,0x7d]
+# GFX11-FAKE16: v_cmpx_nlt_f16_e32 s1, v2               ; encoding: [0x01,0x04,0x1c,0x7d]
 
 0x69,0x04,0x1c,0x7d
-# GFX11: v_cmpx_nlt_f16_e32 s105, v2             ; encoding: [0x69,0x04,0x1c,0x7d]
+# GFX11-REAL16: v_cmpx_nlt_f16_e32 s105, v2.l           ; encoding: [0x69,0x04,0x1c,0x7d]
+# GFX11-FAKE16: v_cmpx_nlt_f16_e32 s105, v2             ; encoding: [0x69,0x04,0x1c,0x7d]
 
 0x6a,0x04,0x1c,0x7d
-# GFX11: v_cmpx_nlt_f16_e32 vcc_lo, v2           ; encoding: [0x6a,0x04,0x1c,0x7d]
+# GFX11-REAL16: v_cmpx_nlt_f16_e32 vcc_lo, v2.l         ; encoding: [0x6a,0x04,0x1c,0x7d]
+# GFX11-FAKE16: v_cmpx_nlt_f16_e32 vcc_lo, v2           ; encoding: [0x6a,0x04,0x1c,0x7d]
 
 0x6b,0x04,0x1c,0x7d
-# GFX11: v_cmpx_nlt_f16_e32 vcc_hi, v2           ; encoding: [0x6b,0x04,0x1c,0x7d]
+# GFX11-REAL16: v_cmpx_nlt_f16_e32 vcc_hi, v2.l         ; encoding: [0x6b,0x04,0x1c,0x7d]
+# GFX11-FAKE16: v_cmpx_nlt_f16_e32 vcc_hi, v2           ; encoding: [0x6b,0x04,0x1c,0x7d]
 
 0x7b,0x04,0x1c,0x7d
-# GFX11: v_cmpx_nlt_f16_e32 ttmp15, v2           ; encoding: [0x7b,0x04,0x1c,0x7d]
+# GFX11-REAL16: v_cmpx_nlt_f16_e32 ttmp15, v2.l         ; encoding: [0x7b,0x04,0x1c,0x7d]
+# GFX11-FAKE16: v_cmpx_nlt_f16_e32 ttmp15, v2           ; encoding: [0x7b,0x04,0x1c,0x7d]
 
 0x7d,0x04,0x1c,0x7d
-# GFX11: v_cmpx_nlt_f16_e32 m0, v2               ; encoding: [0x7d,0x04,0x1c,0x7d]
+# GFX11-REAL16: v_cmpx_nlt_f16_e32 m0, v2.l             ; encoding: [0x7d,0x04,0x1c,0x7d]
+# GFX11-FAKE16: v_cmpx_nlt_f16_e32 m0, v2               ; encoding: [0x7d,0x04,0x1c,0x7d]
 
 0x7e,0x04,0x1c,0x7d
-# GFX11: v_cmpx_nlt_f16_e32 exec_lo, v2          ; encoding: [0x7e,0x04,0x1c,0x7d]
+# GFX11-REAL16: v_cmpx_nlt_f16_e32 exec_lo, v2.l        ; encoding: [0x7e,0x04,0x1c,0x7d]
+# GFX11-FAKE16: v_cmpx_nlt_f16_e32 exec_lo, v2          ; encoding: [0x7e,0x04,0x1c,0x7d]
 
 0x7f,0x04,0x1c,0x7d
-# GFX11: v_cmpx_nlt_f16_e32 exec_hi, v2          ; encoding: [0x7f,0x04,0x1c,0x7d]
+# GFX11-REAL16: v_cmpx_nlt_f16_e32 exec_hi, v2.l        ; encoding: [0x7f,0x04,0x1c,0x7d]
+# GFX11-FAKE16: v_cmpx_nlt_f16_e32 exec_hi, v2          ; encoding: [0x7f,0x04,0x1c,0x7d]
 
 0x7c,0x04,0x1c,0x7d
-# GFX11: v_cmpx_nlt_f16_e32 null, v2             ; encoding: [0x7c,0x04,0x1c,0x7d]
+# GFX11-REAL16: v_cmpx_nlt_f16_e32 null, v2.l           ; encoding: [0x7c,0x04,0x1c,0x7d]
+# GFX11-FAKE16: v_cmpx_nlt_f16_e32 null, v2             ; encoding: [0x7c,0x04,0x1c,0x7d]
 
 0xc1,0x04,0x1c,0x7d
-# GFX11: v_cmpx_nlt_f16_e32 -1, v2               ; encoding: [0xc1,0x04,0x1c,0x7d]
+# GFX11-REAL16: v_cmpx_nlt_f16_e32 -1, v2.l             ; encoding: [0xc1,0x04,0x1c,0x7d]
+# GFX11-FAKE16: v_cmpx_nlt_f16_e32 -1, v2               ; encoding: [0xc1,0x04,0x1c,0x7d]
 
 0xf0,0x04,0x1c,0x7d
-# GFX11: v_cmpx_nlt_f16_e32 0.5, v2              ; encoding: [0xf0,0x04,0x1c,0x7d]
+# GFX11-REAL16: v_cmpx_nlt_f16_e32 0.5, v2.l            ; encoding: [0xf0,0x04,0x1c,0x7d]
+# GFX11-FAKE16: v_cmpx_nlt_f16_e32 0.5, v2              ; encoding: [0xf0,0x04,0x1c,0x7d]
 
 0xfd,0x04,0x1c,0x7d
-# GFX11: v_cmpx_nlt_f16_e32 src_scc, v2          ; encoding: [0xfd,0x04,0x1c,0x7d]
+# GFX11-REAL16: v_cmpx_nlt_f16_e32 src_scc, v2.l        ; encoding: [0xfd,0x04,0x1c,0x7d]
+# GFX11-FAKE16: v_cmpx_nlt_f16_e32 src_scc, v2          ; encoding: [0xfd,0x04,0x1c,0x7d]
 
 0xff,0xfe,0x1c,0x7d,0x0b,0xfe,0x00,0x00
-# GFX11: v_cmpx_nlt_f16_e32 0xfe0b, v127         ; encoding: [0xff,0xfe,0x1c,0x7d,0x0b,0xfe,0x00,0x00]
+# GFX11-REAL16: v_cmpx_nlt_f16_e32 0xfe0b, v127.l       ; encoding: [0xff,0xfe,0x1c,0x7d,0x0b,0xfe,0x00,0x00]
+# GFX11-FAKE16: v_cmpx_nlt_f16_e32 0xfe0b, v127         ; encoding: [0xff,0xfe,0x1c,0x7d,0x0b,0xfe,0x00,0x00]
+
+0x81,0x05,0x1c,0x7d
+# GFX11-REAL16: v_cmpx_nlt_f16_e32 v1.h, v2.l           ; encoding: [0x81,0x05,0x1c,0x7d]
+# GFX11-FAKE16: v_cmpx_nlt_f16_e32 v129/*Invalid register, operand has 'VS_32_Lo128' register class*/, v2 ; encoding: [0x81,0x05,0x1c,0x7d]
+
+0xff,0x05,0x1c,0x7d
+# GFX11-REAL16: v_cmpx_nlt_f16_e32 v127.h, v2.l         ; encoding: [0xff,0x05,0x1c,0x7d]
+# GFX11-FAKE16: v_cmpx_nlt_f16_e32 v255/*Invalid register, operand has 'VS_32_Lo128' register class*/, v2 ; encoding: [0xff,0x05,0x1c,0x7d]
+
+0xf0,0xfe,0x1c,0x7d
+# GFX11-REAL16: v_cmpx_nlt_f16_e32 0.5, v127.l          ; encoding: [0xf0,0xfe,0x1c,0x7d]
+# GFX11-FAKE16: v_cmpx_nlt_f16_e32 0.5, v127            ; encoding: [0xf0,0xfe,0x1c,0x7d]
+
+0xfd,0x04,0x1d,0x7d
+# GFX11-REAL16: v_cmpx_nlt_f16_e32 src_scc, v2.h        ; encoding: [0xfd,0x04,0x1d,0x7d]
+# GFX11-FAKE16: v_cmpx_nlt_f16_e32 src_scc, v130/*Invalid register, operand has 'VGPR_32_Lo128' register class*/ ; encoding: [0xfd,0x04,0x1d,0x7d]
+
+0xff,0xfe,0x1d,0x7d,0x0b,0xfe,0x00,0x00
+# GFX11-REAL16: v_cmpx_nlt_f16_e32 0xfe0b, v127.h       ; encoding: [0xff,0xfe,0x1d,0x7d,0x0b,0xfe,0x00,0x00]
+# GFX11-FAKE16: v_cmpx_nlt_f16_e32 0xfe0b, v255/*Invalid register, operand has 'VGPR_32_Lo128' register class*/ ; encoding: [0xff,0xfe,0x1d,0x7d,0x0b,0xfe,0x00,0x00]
 
 0x01,0x05,0x3c,0x7d
 # GFX11: v_cmpx_nlt_f32_e32 v1, v2               ; encoding: [0x01,0x05,0x3c,0x7d]
@@ -3933,49 +4353,84 @@
 # GFX11: v_cmpx_nlt_f64_e32 0xaf123456, v[254:255] ; encoding: [0xff,0xfc,0x5d,0x7d,0x56,0x34,0x12,0xaf]
 
 0x01,0x05,0x0e,0x7d
-# GFX11: v_cmpx_o_f16_e32 v1, v2                 ; encoding: [0x01,0x05,0x0e,0x7d]
+# GFX11-REAL16: v_cmpx_o_f16_e32 v1.l, v2.l             ; encoding: [0x01,0x05,0x0e,0x7d]
+# GFX11-FAKE16: v_cmpx_o_f16_e32 v1, v2                 ; encoding: [0x01,0x05,0x0e,0x7d]
 
 0x7f,0x05,0x0e,0x7d
-# GFX11: v_cmpx_o_f16_e32 v127, v2               ; encoding: [0x7f,0x05,0x0e,0x7d]
+# GFX11-REAL16: v_cmpx_o_f16_e32 v127.l, v2.l           ; encoding: [0x7f,0x05,0x0e,0x7d]
+# GFX11-FAKE16: v_cmpx_o_f16_e32 v127, v2               ; encoding: [0x7f,0x05,0x0e,0x7d]
 
 0x01,0x04,0x0e,0x7d
-# GFX11: v_cmpx_o_f16_e32 s1, v2                 ; encoding: [0x01,0x04,0x0e,0x7d]
+# GFX11-REAL16: v_cmpx_o_f16_e32 s1, v2.l               ; encoding: [0x01,0x04,0x0e,0x7d]
+# GFX11-FAKE16: v_cmpx_o_f16_e32 s1, v2                 ; encoding: [0x01,0x04,0x0e,0x7d]
 
 0x69,0x04,0x0e,0x7d
-# GFX11: v_cmpx_o_f16_e32 s105, v2               ; encoding: [0x69,0x04,0x0e,0x7d]
+# GFX11-REAL16: v_cmpx_o_f16_e32 s105, v2.l             ; encoding: [0x69,0x04,0x0e,0x7d]
+# GFX11-FAKE16: v_cmpx_o_f16_e32 s105, v2               ; encoding: [0x69,0x04,0x0e,0x7d]
 
 0x6a,0x04,0x0e,0x7d
-# GFX11: v_cmpx_o_f16_e32 vcc_lo, v2             ; encoding: [0x6a,0x04,0x0e,0x7d]
+# GFX11-REAL16: v_cmpx_o_f16_e32 vcc_lo, v2.l           ; encoding: [0x6a,0x04,0x0e,0x7d]
+# GFX11-FAKE16: v_cmpx_o_f16_e32 vcc_lo, v2             ; encoding: [0x6a,0x04,0x0e,0x7d]
 
 0x6b,0x04,0x0e,0x7d
-# GFX11: v_cmpx_o_f16_e32 vcc_hi, v2             ; encoding: [0x6b,0x04,0x0e,0x7d]
+# GFX11-REAL16: v_cmpx_o_f16_e32 vcc_hi, v2.l           ; encoding: [0x6b,0x04,0x0e,0x7d]
+# GFX11-FAKE16: v_cmpx_o_f16_e32 vcc_hi, v2             ; encoding: [0x6b,0x04,0x0e,0x7d]
 
 0x7b,0x04,0x0e,0x7d
-# GFX11: v_cmpx_o_f16_e32 ttmp15, v2             ; encoding: [0x7b,0x04,0x0e,0x7d]
+# GFX11-REAL16: v_cmpx_o_f16_e32 ttmp15, v2.l           ; encoding: [0x7b,0x04,0x0e,0x7d]
+# GFX11-FAKE16: v_cmpx_o_f16_e32 ttmp15, v2             ; encoding: [0x7b,0x04,0x0e,0x7d]
 
 0x7d,0x04,0x0e,0x7d
-# GFX11: v_cmpx_o_f16_e32 m0, v2                 ; encoding: [0x7d,0x04,0x0e,0x7d]
+# GFX11-REAL16: v_cmpx_o_f16_e32 m0, v2.l               ; encoding: [0x7d,0x04,0x0e,0x7d]
+# GFX11-FAKE16: v_cmpx_o_f16_e32 m0, v2                 ; encoding: [0x7d,0x04,0x0e,0x7d]
 
 0x7e,0x04,0x0e,0x7d
-# GFX11: v_cmpx_o_f16_e32 exec_lo, v2            ; encoding: [0x7e,0x04,0x0e,0x7d]
+# GFX11-REAL16: v_cmpx_o_f16_e32 exec_lo, v2.l          ; encoding: [0x7e,0x04,0x0e,0x7d]
+# GFX11-FAKE16: v_cmpx_o_f16_e32 exec_lo, v2            ; encoding: [0x7e,0x04,0x0e,0x7d]
 
 0x7f,0x04,0x0e,0x7d
-# GFX11: v_cmpx_o_f16_e32 exec_hi, v2            ; encoding: [0x7f,0x04,0x0e,0x7d]
+# GFX11-REAL16: v_cmpx_o_f16_e32 exec_hi, v2.l          ; encoding: [0x7f,0x04,0x0e,0x7d]
+# GFX11-FAKE16: v_cmpx_o_f16_e32 exec_hi, v2            ; encoding: [0x7f,0x04,0x0e,0x7d]
 
 0x7c,0x04,0x0e,0x7d
-# GFX11: v_cmpx_o_f16_e32 null, v2               ; encoding: [0x7c,0x04,0x0e,0x7d]
+# GFX11-REAL16: v_cmpx_o_f16_e32 null, v2.l             ; encoding: [0x7c,0x04,0x0e,0x7d]
+# GFX11-FAKE16: v_cmpx_o_f16_e32 null, v2               ; encoding: [0x7c,0x04,0x0e,0x7d]
 
 0xc1,0x04,0x0e,0x7d
-# GFX11: v_cmpx_o_f16_e32 -1, v2                 ; encoding: [0xc1,0x04,0x0e,0x7d]
+# GFX11-REAL16: v_cmpx_o_f16_e32 -1, v2.l               ; encoding: [0xc1,0x04,0x0e,0x7d]
+# GFX11-FAKE16: v_cmpx_o_f16_e32 -1, v2                 ; encoding: [0xc1,0x04,0x0e,0x7d]
 
 0xf0,0x04,0x0e,0x7d
-# GFX11: v_cmpx_o_f16_e32 0.5, v2                ; encoding: [0xf0,0x04,0x0e,0x7d]
+# GFX11-REAL16: v_cmpx_o_f16_e32 0.5, v2.l              ; encoding: [0xf0,0x04,0x0e,0x7d]
+# GFX11-FAKE16: v_cmpx_o_f16_e32 0.5, v2                ; encoding: [0xf0,0x04,0x0e,0x7d]
 
 0xfd,0x04,0x0e,0x7d
-# GFX11: v_cmpx_o_f16_e32 src_scc, v2            ; encoding: [0xfd,0x04,0x0e,0x7d]
+# GFX11-REAL16: v_cmpx_o_f16_e32 src_scc, v2.l          ; encoding: [0xfd,0x04,0x0e,0x7d]
+# GFX11-FAKE16: v_cmpx_o_f16_e32 src_scc, v2            ; encoding: [0xfd,0x04,0x0e,0x7d]
 
 0xff,0xfe,0x0e,0x7d,0x0b,0xfe,0x00,0x00
-# GFX11: v_cmpx_o_f16_e32 0xfe0b, v127           ; encoding: [0xff,0xfe,0x0e,0x7d,0x0b,0xfe,0x00,0x00]
+# GFX11-REAL16: v_cmpx_o_f16_e32 0xfe0b, v127.l         ; encoding: [0xff,0xfe,0x0e,0x7d,0x0b,0xfe,0x00,0x00]
+# GFX11-FAKE16: v_cmpx_o_f16_e32 0xfe0b, v127           ; encoding: [0xff,0xfe,0x0e,0x7d,0x0b,0xfe,0x00,0x00]
+
+0x81,0x05,0x0e,0x7d
+# GFX11-REAL16: v_cmpx_o_f16_e32 v1.h, v2.l             ; encoding: [0x81,0x05,0x0e,0x7d]
+# GFX11-FAKE16: v_cmpx_o_f16_e32 v129/*Invalid register, operand has 'VS_32_Lo128' register class*/, v2 ; encoding: [0x81,0x05,0x0e,0x7d]
+
+0xff,0x05,0x0e,0x7d
+# GFX11-REAL16: v_cmpx_o_f16_e32 v127.h, v2.l           ; encoding: [0xff,0x05,0x0e,0x7d]
+# GFX11-FAKE16: v_cmpx_o_f16_e32 v255/*Invalid register, operand has 'VS_32_Lo128' register class*/, v2 ; encoding: [0xff,0x05,0x0e,0x7d]
+
+0xf0,0xfe,0x0e,0x7d
+# GFX11-REAL16: v_cmpx_o_f16_e32 0.5, v127.l            ; encoding: [0xf0,0xfe,0x0e,0x7d]
+# GFX11-FAKE16: v_cmpx_o_f16_e32 0.5, v127              ; encoding: [0xf0,0xfe,0x0e,0x7d]
+
+0xfd,0x04,0x0f,0x7d
+# GFX11-REAL16: v_cmpx_o_f16_e32 src_scc, v2.h          ; encoding: [0xfd,0x04,0x0f,0x7d]
+# GFX11-FAKE16: v_cmpx_o_f16_e32 src_scc, v130/*Invalid register, operand has 'VGPR_32_Lo128' register class*/ ; encoding: [0xfd,0x04,0x0f,0x7d]
+
+0xff,0xfe,0x0f,0x7d,0x0b,0xfe,0x00,0x00
+# GFX11-REAL16: v_cmpx_o_f16_e32 0xfe0b, v127.h         ; encoding: [0xff,0xfe,0x0f,0x7d,0x0b,0xfe,0x00,0x00]
+# GFX11-FAKE16: v_cmpx_o_f16_e32 0xfe0b, v255/*Invalid register, operand has 'VGPR_32_Lo128' register class*/ ; encoding: [0xff,0xfe,0x0f,0x7d,0x0b,0xfe,0x00,0x00]
 
 0x01,0x05,0x2e,0x7d
 # GFX11: v_cmpx_o_f32_e32 v1, v2                 ; encoding: [0x01,0x05,0x2e,0x7d]
@@ -4059,49 +4514,84 @@
 # GFX11: v_cmpx_o_f64_e32 0xaf123456, v[254:255] ; encoding: [0xff,0xfc,0x4f,0x7d,0x56,0x34,0x12,0xaf]
 
 0x01,0x05,0x1e,0x7d
-# GFX11: v_cmpx_t_f16_e32 v1, v2                 ; encoding: [0x01,0x05,0x1e,0x7d]
+# GFX11-REAL16: v_cmpx_t_f16_e32 v1.l, v2.l             ; encoding: [0x01,0x05,0x1e,0x7d]
+# GFX11-FAKE16: v_cmpx_t_f16_e32 v1, v2                 ; encoding: [0x01,0x05,0x1e,0x7d]
 
 0x7f,0x05,0x1e,0x7d
-# GFX11: v_cmpx_t_f16_e32 v127, v2               ; encoding: [0x7f,0x05,0x1e,0x7d]
+# GFX11-REAL16: v_cmpx_t_f16_e32 v127.l, v2.l           ; encoding: [0x7f,0x05,0x1e,0x7d]
+# GFX11-FAKE16: v_cmpx_t_f16_e32 v127, v2               ; encoding: [0x7f,0x05,0x1e,0x7d]
 
 0x01,0x04,0x1e,0x7d
-# GFX11: v_cmpx_t_f16_e32 s1, v2                 ; encoding: [0x01,0x04,0x1e,0x7d]
+# GFX11-REAL16: v_cmpx_t_f16_e32 s1, v2.l               ; encoding: [0x01,0x04,0x1e,0x7d]
+# GFX11-FAKE16: v_cmpx_t_f16_e32 s1, v2                 ; encoding: [0x01,0x04,0x1e,0x7d]
 
 0x69,0x04,0x1e,0x7d
-# GFX11: v_cmpx_t_f16_e32 s105, v2               ; encoding: [0x69,0x04,0x1e,0x7d]
+# GFX11-REAL16: v_cmpx_t_f16_e32 s105, v2.l             ; encoding: [0x69,0x04,0x1e,0x7d]
+# GFX11-FAKE16: v_cmpx_t_f16_e32 s105, v2               ; encoding: [0x69,0x04,0x1e,0x7d]
 
 0x6a,0x04,0x1e,0x7d
-# GFX11: v_cmpx_t_f16_e32 vcc_lo, v2             ; encoding: [0x6a,0x04,0x1e,0x7d]
+# GFX11-REAL16: v_cmpx_t_f16_e32 vcc_lo, v2.l           ; encoding: [0x6a,0x04,0x1e,0x7d]
+# GFX11-FAKE16: v_cmpx_t_f16_e32 vcc_lo, v2             ; encoding: [0x6a,0x04,0x1e,0x7d]
 
 0x6b,0x04,0x1e,0x7d
-# GFX11: v_cmpx_t_f16_e32 vcc_hi, v2             ; encoding: [0x6b,0x04,0x1e,0x7d]
+# GFX11-REAL16: v_cmpx_t_f16_e32 vcc_hi, v2.l           ; encoding: [0x6b,0x04,0x1e,0x7d]
+# GFX11-FAKE16: v_cmpx_t_f16_e32 vcc_hi, v2             ; encoding: [0x6b,0x04,0x1e,0x7d]
 
 0x7b,0x04,0x1e,0x7d
-# GFX11: v_cmpx_t_f16_e32 ttmp15, v2             ; encoding: [0x7b,0x04,0x1e,0x7d]
+# GFX11-REAL16: v_cmpx_t_f16_e32 ttmp15, v2.l           ; encoding: [0x7b,0x04,0x1e,0x7d]
+# GFX11-FAKE16: v_cmpx_t_f16_e32 ttmp15, v2             ; encoding: [0x7b,0x04,0x1e,0x7d]
 
 0x7d,0x04,0x1e,0x7d
-# GFX11: v_cmpx_t_f16_e32 m0, v2                 ; encoding: [0x7d,0x04,0x1e,0x7d]
+# GFX11-REAL16: v_cmpx_t_f16_e32 m0, v2.l               ; encoding: [0x7d,0x04,0x1e,0x7d]
+# GFX11-FAKE16: v_cmpx_t_f16_e32 m0, v2                 ; encoding: [0x7d,0x04,0x1e,0x7d]
 
 0x7e,0x04,0x1e,0x7d
-# GFX11: v_cmpx_t_f16_e32 exec_lo, v2            ; encoding: [0x7e,0x04,0x1e,0x7d]
+# GFX11-REAL16: v_cmpx_t_f16_e32 exec_lo, v2.l          ; encoding: [0x7e,0x04,0x1e,0x7d]
+# GFX11-FAKE16: v_cmpx_t_f16_e32 exec_lo, v2            ; encoding: [0x7e,0x04,0x1e,0x7d]
 
 0x7f,0x04,0x1e,0x7d
-# GFX11: v_cmpx_t_f16_e32 exec_hi, v2            ; encoding: [0x7f,0x04,0x1e,0x7d]
+# GFX11-REAL16: v_cmpx_t_f16_e32 exec_hi, v2.l          ; encoding: [0x7f,0x04,0x1e,0x7d]
+# GFX11-FAKE16: v_cmpx_t_f16_e32 exec_hi, v2            ; encoding: [0x7f,0x04,0x1e,0x7d]
 
 0x7c,0x04,0x1e,0x7d
-# GFX11: v_cmpx_t_f16_e32 null, v2               ; encoding: [0x7c,0x04,0x1e,0x7d]
+# GFX11-REAL16: v_cmpx_t_f16_e32 null, v2.l             ; encoding: [0x7c,0x04,0x1e,0x7d]
+# GFX11-FAKE16: v_cmpx_t_f16_e32 null, v2               ; encoding: [0x7c,0x04,0x1e,0x7d]
 
 0xc1,0x04,0x1e,0x7d
-# GFX11: v_cmpx_t_f16_e32 -1, v2                 ; encoding: [0xc1,0x04,0x1e,0x7d]
+# GFX11-REAL16: v_cmpx_t_f16_e32 -1, v2.l               ; encoding: [0xc1,0x04,0x1e,0x7d]
+# GFX11-FAKE16: v_cmpx_t_f16_e32 -1, v2                 ; encoding: [0xc1,0x04,0x1e,0x7d]
 
 0xf0,0x04,0x1e,0x7d
-# GFX11: v_cmpx_t_f16_e32 0.5, v2                ; encoding: [0xf0,0x04,0x1e,0x7d]
+# GFX11-REAL16: v_cmpx_t_f16_e32 0.5, v2.l              ; encoding: [0xf0,0x04,0x1e,0x7d]
+# GFX11-FAKE16: v_cmpx_t_f16_e32 0.5, v2                ; encoding: [0xf0,0x04,0x1e,0x7d]
 
 0xfd,0x04,0x1e,0x7d
-# GFX11: v_cmpx_t_f16_e32 src_scc, v2            ; encoding: [0xfd,0x04,0x1e,0x7d]
+# GFX11-REAL16: v_cmpx_t_f16_e32 src_scc, v2.l          ; encoding: [0xfd,0x04,0x1e,0x7d]
+# GFX11-FAKE16: v_cmpx_t_f16_e32 src_scc, v2            ; encoding: [0xfd,0x04,0x1e,0x7d]
 
 0xff,0xfe,0x1e,0x7d,0x0b,0xfe,0x00,0x00
-# GFX11: v_cmpx_t_f16_e32 0xfe0b, v127           ; encoding: [0xff,0xfe,0x1e,0x7d,0x0b,0xfe,0x00,0x00]
+# GFX11-REAL16: v_cmpx_t_f16_e32 0xfe0b, v127.l         ; encoding: [0xff,0xfe,0x1e,0x7d,0x0b,0xfe,0x00,0x00]
+# GFX11-FAKE16: v_cmpx_t_f16_e32 0xfe0b, v127           ; encoding: [0xff,0xfe,0x1e,0x7d,0x0b,0xfe,0x00,0x00]
+
+0x81,0x05,0x1e,0x7d
+# GFX11-REAL16: v_cmpx_t_f16_e32 v1.h, v2.l             ; encoding: [0x81,0x05,0x1e,0x7d]
+# GFX11-FAKE16: v_cmpx_t_f16_e32 v129/*Invalid register, operand has 'VS_32_Lo128' register class*/, v2 ; encoding: [0x81,0x05,0x1e,0x7d]
+
+0xff,0x05,0x1e,0x7d
+# GFX11-REAL16: v_cmpx_t_f16_e32 v127.h, v2.l           ; encoding: [0xff,0x05,0x1e,0x7d]
+# GFX11-FAKE16: v_cmpx_t_f16_e32 v255/*Invalid register, operand has 'VS_32_Lo128' register class*/, v2 ; encoding: [0xff,0x05,0x1e,0x7d]
+
+0xf0,0xfe,0x1e,0x7d
+# GFX11-REAL16: v_cmpx_t_f16_e32 0.5, v127.l            ; encoding: [0xf0,0xfe,0x1e,0x7d]
+# GFX11-FAKE16: v_cmpx_t_f16_e32 0.5, v127              ; encoding: [0xf0,0xfe,0x1e,0x7d]
+
+0xfd,0x04,0x1f,0x7d
+# GFX11-REAL16: v_cmpx_t_f16_e32 src_scc, v2.h          ; encoding: [0xfd,0x04,0x1f,0x7d]
+# GFX11-FAKE16: v_cmpx_t_f16_e32 src_scc, v130/*Invalid register, operand has 'VGPR_32_Lo128' register class*/ ; encoding: [0xfd,0x04,0x1f,0x7d]
+
+0xff,0xfe,0x1f,0x7d,0x0b,0xfe,0x00,0x00
+# GFX11-REAL16: v_cmpx_t_f16_e32 0xfe0b, v127.h         ; encoding: [0xff,0xfe,0x1f,0x7d,0x0b,0xfe,0x00,0x00]
+# GFX11-FAKE16: v_cmpx_t_f16_e32 0xfe0b, v255/*Invalid register, operand has 'VGPR_32_Lo128' register class*/ ; encoding: [0xff,0xfe,0x1f,0x7d,0x0b,0xfe,0x00,0x00]
 
 0x01,0x05,0x3e,0x7d
 # GFX11: v_cmpx_t_f32_e32 v1, v2                 ; encoding: [0x01,0x05,0x3e,0x7d]
@@ -4347,49 +4837,84 @@
 # GFX11: v_cmpx_t_u64_e32 0xaf123456, v[254:255] ; encoding: [0xff,0xfc,0xbf,0x7d,0x56,0x34,0x12,0xaf]
 
 0x01,0x05,0x10,0x7d
-# GFX11: v_cmpx_u_f16_e32 v1, v2                 ; encoding: [0x01,0x05,0x10,0x7d]
+# GFX11-REAL16: v_cmpx_u_f16_e32 v1.l, v2.l             ; encoding: [0x01,0x05,0x10,0x7d]
+# GFX11-FAKE16: v_cmpx_u_f16_e32 v1, v2                 ; encoding: [0x01,0x05,0x10,0x7d]
 
 0x7f,0x05,0x10,0x7d
-# GFX11: v_cmpx_u_f16_e32 v127, v2               ; encoding: [0x7f,0x05,0x10,0x7d]
+# GFX11-REAL16: v_cmpx_u_f16_e32 v127.l, v2.l           ; encoding: [0x7f,0x05,0x10,0x7d]
+# GFX11-FAKE16: v_cmpx_u_f16_e32 v127, v2               ; encoding: [0x7f,0x05,0x10,0x7d]
 
 0x01,0x04,0x10,0x7d
-# GFX11: v_cmpx_u_f16_e32 s1, v2                 ; encoding: [0x01,0x04,0x10,0x7d]
+# GFX11-REAL16: v_cmpx_u_f16_e32 s1, v2.l               ; encoding: [0x01,0x04,0x10,0x7d]
+# GFX11-FAKE16: v_cmpx_u_f16_e32 s1, v2                 ; encoding: [0x01,0x04,0x10,0x7d]
 
 0x69,0x04,0x10,0x7d
-# GFX11: v_cmpx_u_f16_e32 s105, v2               ; encoding: [0x69,0x04,0x10,0x7d]
+# GFX11-REAL16: v_cmpx_u_f16_e32 s105, v2.l             ; encoding: [0x69,0x04,0x10,0x7d]
+# GFX11-FAKE16: v_cmpx_u_f16_e32 s105, v2               ; encoding: [0x69,0x04,0x10,0x7d]
 
 0x6a,0x04,0x10,0x7d
-# GFX11: v_cmpx_u_f16_e32 vcc_lo, v2             ; encoding: [0x6a,0x04,0x10,0x7d]
+# GFX11-REAL16: v_cmpx_u_f16_e32 vcc_lo, v2.l           ; encoding: [0x6a,0x04,0x10,0x7d]
+# GFX11-FAKE16: v_cmpx_u_f16_e32 vcc_lo, v2             ; encoding: [0x6a,0x04,0x10,0x7d]
 
 0x6b,0x04,0x10,0x7d
-# GFX11: v_cmpx_u_f16_e32 vcc_hi, v2             ; encoding: [0x6b,0x04,0x10,0x7d]
+# GFX11-REAL16: v_cmpx_u_f16_e32 vcc_hi, v2.l           ; encoding: [0x6b,0x04,0x10,0x7d]
+# GFX11-FAKE16: v_cmpx_u_f16_e32 vcc_hi, v2             ; encoding: [0x6b,0x04,0x10,0x7d]
 
 0x7b,0x04,0x10,0x7d
-# GFX11: v_cmpx_u_f16_e32 ttmp15, v2             ; encoding: [0x7b,0x04,0x10,0x7d]
+# GFX11-REAL16: v_cmpx_u_f16_e32 ttmp15, v2.l           ; encoding: [0x7b,0x04,0x10,0x7d]
+# GFX11-FAKE16: v_cmpx_u_f16_e32 ttmp15, v2             ; encoding: [0x7b,0x04,0x10,0x7d]
 
 0x7d,0x04,0x10,0x7d
-# GFX11: v_cmpx_u_f16_e32 m0, v2                 ; encoding: [0x7d,0x04,0x10,0x7d]
+# GFX11-REAL16: v_cmpx_u_f16_e32 m0, v2.l               ; encoding: [0x7d,0x04,0x10,0x7d]
+# GFX11-FAKE16: v_cmpx_u_f16_e32 m0, v2                 ; encoding: [0x7d,0x04,0x10,0x7d]
 
 0x7e,0x04,0x10,0x7d
-# GFX11: v_cmpx_u_f16_e32 exec_lo, v2            ; encoding: [0x7e,0x04,0x10,0x7d]
+# GFX11-REAL16: v_cmpx_u_f16_e32 exec_lo, v2.l          ; encoding: [0x7e,0x04,0x10,0x7d]
+# GFX11-FAKE16: v_cmpx_u_f16_e32 exec_lo, v2            ; encoding: [0x7e,0x04,0x10,0x7d]
 
 0x7f,0x04,0x10,0x7d
-# GFX11: v_cmpx_u_f16_e32 exec_hi, v2            ; encoding: [0x7f,0x04,0x10,0x7d]
+# GFX11-REAL16: v_cmpx_u_f16_e32 exec_hi, v2.l          ; encoding: [0x7f,0x04,0x10,0x7d]
+# GFX11-FAKE16: v_cmpx_u_f16_e32 exec_hi, v2            ; encoding: [0x7f,0x04,0x10,0x7d]
 
 0x7c,0x04,0x10,0x7d
-# GFX11: v_cmpx_u_f16_e32 null, v2               ; encoding: [0x7c,0x04,0x10,0x7d]
+# GFX11-REAL16: v_cmpx_u_f16_e32 null, v2.l             ; encoding: [0x7c,0x04,0x10,0x7d]
+# GFX11-FAKE16: v_cmpx_u_f16_e32 null, v2               ; encoding: [0x7c,0x04,0x10,0x7d]
 
 0xc1,0x04,0x10,0x7d
-# GFX11: v_cmpx_u_f16_e32 -1, v2                 ; encoding: [0xc1,0x04,0x10,0x7d]
+# GFX11-REAL16: v_cmpx_u_f16_e32 -1, v2.l               ; encoding: [0xc1,0x04,0x10,0x7d]
+# GFX11-FAKE16: v_cmpx_u_f16_e32 -1, v2                 ; encoding: [0xc1,0x04,0x10,0x7d]
 
 0xf0,0x04,0x10,0x7d
-# GFX11: v_cmpx_u_f16_e32 0.5, v2                ; encoding: [0xf0,0x04,0x10,0x7d]
+# GFX11-REAL16: v_cmpx_u_f16_e32 0.5, v2.l              ; encoding: [0xf0,0x04,0x10,0x7d]
+# GFX11-FAKE16: v_cmpx_u_f16_e32 0.5, v2                ; encoding: [0xf0,0x04,0x10,0x7d]
 
 0xfd,0x04,0x10,0x7d
-# GFX11: v_cmpx_u_f16_e32 src_scc, v2            ; encoding: [0xfd,0x04,0x10,0x7d]
+# GFX11-REAL16: v_cmpx_u_f16_e32 src_scc, v2.l          ; encoding: [0xfd,0x04,0x10,0x7d]
+# GFX11-FAKE16: v_cmpx_u_f16_e32 src_scc, v2            ; encoding: [0xfd,0x04,0x10,0x7d]
 
 0xff,0xfe,0x10,0x7d,0x0b,0xfe,0x00,0x00
-# GFX11: v_cmpx_u_f16_e32 0xfe0b, v127           ; encoding: [0xff,0xfe,0x10,0x7d,0x0b,0xfe,0x00,0x00]
+# GFX11-REAL16: v_cmpx_u_f16_e32 0xfe0b, v127.l         ; encoding: [0xff,0xfe,0x10,0x7d,0x0b,0xfe,0x00,0x00]
+# GFX11-FAKE16: v_cmpx_u_f16_e32 0xfe0b, v127           ; encoding: [0xff,0xfe,0x10,0x7d,0x0b,0xfe,0x00,0x00]
+
+0x81,0x05,0x10,0x7d
+# GFX11-REAL16: v_cmpx_u_f16_e32 v1.h, v2.l             ; encoding: [0x81,0x05,0x10,0x7d]
+# GFX11-FAKE16: v_cmpx_u_f16_e32 v129/*Invalid register, operand has 'VS_32_Lo128' register class*/, v2 ; encoding: [0x81,0x05,0x10,0x7d]
+
+0xff,0x05,0x10,0x7d
+# GFX11-REAL16: v_cmpx_u_f16_e32 v127.h, v2.l           ; encoding: [0xff,0x05,0x10,0x7d]
+# GFX11-FAKE16: v_cmpx_u_f16_e32 v255/*Invalid register, operand has 'VS_32_Lo128' register class*/, v2 ; encoding: [0xff,0x05,0x10,0x7d]
+
+0xf0,0xfe,0x10,0x7d
+# GFX11-REAL16: v_cmpx_u_f16_e32 0.5, v127.l            ; encoding: [0xf0,0xfe,0x10,0x7d]
+# GFX11-FAKE16: v_cmpx_u_f16_e32 0.5, v127              ; encoding: [0xf0,0xfe,0x10,0x7d]
+
+0xfd,0x04,0x11,0x7d
+# GFX11-REAL16: v_cmpx_u_f16_e32 src_scc, v2.h          ; encoding: [0xfd,0x04,0x11,0x7d]
+# GFX11-FAKE16: v_cmpx_u_f16_e32 src_scc, v130/*Invalid register, operand has 'VGPR_32_Lo128' register class*/ ; encoding: [0xfd,0x04,0x11,0x7d]
+
+0xff,0xfe,0x11,0x7d,0x0b,0xfe,0x00,0x00
+# GFX11-REAL16: v_cmpx_u_f16_e32 0xfe0b, v127.h         ; encoding: [0xff,0xfe,0x11,0x7d,0x0b,0xfe,0x00,0x00]
+# GFX11-FAKE16: v_cmpx_u_f16_e32 0xfe0b, v255/*Invalid register, operand has 'VGPR_32_Lo128' register class*/ ; encoding: [0xff,0xfe,0x11,0x7d,0x0b,0xfe,0x00,0x00]
 
 0x01,0x05,0x30,0x7d
 # GFX11: v_cmpx_u_f32_e32 v1, v2                 ; encoding: [0x01,0x05,0x30,0x7d]
diff --git a/llvm/test/MC/Disassembler/AMDGPU/gfx11_dasm_vopcx_dpp16.txt b/llvm/test/MC/Disassembler/AMDGPU/gfx11_dasm_vopcx_dpp16.txt
index 5a57f93c65939..f55e646dda79b 100644
--- a/llvm/test/MC/Disassembler/AMDGPU/gfx11_dasm_vopcx_dpp16.txt
+++ b/llvm/test/MC/Disassembler/AMDGPU/gfx11_dasm_vopcx_dpp16.txt
@@ -115,46 +115,72 @@
 # GFX11: v_cmpx_class_f32 -|v255|, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0xfd,0x7d,0xff,0x6f,0x3d,0x30]
 
 0xfa,0x04,0x04,0x7d,0x01,0x1b,0x00,0xff
-# GFX11: v_cmpx_eq_f16 v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x04,0x7d,0x01,0x1b,0x00,0xff]
+# GFX11-REAL16: v_cmpx_eq_f16 v1.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x04,0x7d,0x01,0x1b,0x00,0xff]
+# GFX11-FAKE16: v_cmpx_eq_f16 v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x04,0x7d,0x01,0x1b,0x00,0xff]
 
 0xfa,0x04,0x04,0x7d,0x01,0xe4,0x00,0xff
-# GFX11: v_cmpx_eq_f16 v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x04,0x7d,0x01,0xe4,0x00,0xff]
+# GFX11-REAL16: v_cmpx_eq_f16 v1.l, v2.l quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x04,0x7d,0x01,0xe4,0x00,0xff]
+# GFX11-FAKE16: v_cmpx_eq_f16 v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x04,0x7d,0x01,0xe4,0x00,0xff]
 
 0xfa,0x04,0x04,0x7d,0x01,0x40,0x01,0xff
-# GFX11: v_cmpx_eq_f16 v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x04,0x7d,0x01,0x40,0x01,0xff]
+# GFX11-REAL16: v_cmpx_eq_f16 v1.l, v2.l row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x04,0x7d,0x01,0x40,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_eq_f16 v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x04,0x7d,0x01,0x40,0x01,0xff]
 
 0xfa,0x04,0x04,0x7d,0x01,0x41,0x01,0xff
-# GFX11: v_cmpx_eq_f16 v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x04,0x7d,0x01,0x41,0x01,0xff]
+# GFX11-REAL16: v_cmpx_eq_f16 v1.l, v2.l row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x04,0x7d,0x01,0x41,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_eq_f16 v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x04,0x7d,0x01,0x41,0x01,0xff]
 
 0xfa,0x04,0x04,0x7d,0x01,0x01,0x01,0xff
-# GFX11: v_cmpx_eq_f16 v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x04,0x7d,0x01,0x01,0x01,0xff]
+# GFX11-REAL16: v_cmpx_eq_f16 v1.l, v2.l row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x04,0x7d,0x01,0x01,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_eq_f16 v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x04,0x7d,0x01,0x01,0x01,0xff]
 
 0xfa,0x04,0x04,0x7d,0x01,0x0f,0x01,0xff
-# GFX11: v_cmpx_eq_f16 v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x04,0x7d,0x01,0x0f,0x01,0xff]
+# GFX11-REAL16: v_cmpx_eq_f16 v1.l, v2.l row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x04,0x7d,0x01,0x0f,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_eq_f16 v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x04,0x7d,0x01,0x0f,0x01,0xff]
 
 0xfa,0x04,0x04,0x7d,0x01,0x11,0x01,0xff
-# GFX11: v_cmpx_eq_f16 v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x04,0x7d,0x01,0x11,0x01,0xff]
+# GFX11-REAL16: v_cmpx_eq_f16 v1.l, v2.l row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x04,0x7d,0x01,0x11,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_eq_f16 v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x04,0x7d,0x01,0x11,0x01,0xff]
 
 0xfa,0x04,0x04,0x7d,0x01,0x1f,0x01,0xff
-# GFX11: v_cmpx_eq_f16 v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x04,0x7d,0x01,0x1f,0x01,0xff]
+# GFX11-REAL16: v_cmpx_eq_f16 v1.l, v2.l row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x04,0x7d,0x01,0x1f,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_eq_f16 v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x04,0x7d,0x01,0x1f,0x01,0xff]
 
 0xfa,0x04,0x04,0x7d,0x01,0x21,0x01,0xff
-# GFX11: v_cmpx_eq_f16 v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x04,0x7d,0x01,0x21,0x01,0xff]
+# GFX11-REAL16: v_cmpx_eq_f16 v1.l, v2.l row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x04,0x7d,0x01,0x21,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_eq_f16 v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x04,0x7d,0x01,0x21,0x01,0xff]
 
 0xfa,0x04,0x04,0x7d,0x01,0x2f,0x01,0xff
-# GFX11: v_cmpx_eq_f16 v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x04,0x7d,0x01,0x2f,0x01,0xff]
+# GFX11-REAL16: v_cmpx_eq_f16 v1.l, v2.l row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x04,0x7d,0x01,0x2f,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_eq_f16 v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x04,0x7d,0x01,0x2f,0x01,0xff]
 
 0xfa,0x04,0x04,0x7d,0x01,0x50,0x01,0xff
-# GFX11: v_cmpx_eq_f16 v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x04,0x7d,0x01,0x50,0x01,0xff]
+# GFX11-REAL16: v_cmpx_eq_f16 v1.l, v2.l row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x04,0x7d,0x01,0x50,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_eq_f16 v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x04,0x7d,0x01,0x50,0x01,0xff]
 
 0xfa,0x04,0x04,0x7d,0x01,0x5f,0x01,0x01
-# GFX11: v_cmpx_eq_f16 v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x04,0x7d,0x01,0x5f,0x01,0x01]
+# GFX11-REAL16: v_cmpx_eq_f16 v1.l, v2.l row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x04,0x7d,0x01,0x5f,0x01,0x01]
+# GFX11-FAKE16: v_cmpx_eq_f16 v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x04,0x7d,0x01,0x5f,0x01,0x01]
 
 0xfa,0x04,0x04,0x7d,0x01,0x60,0x01,0x13
-# GFX11: v_cmpx_eq_f16 v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x04,0x7d,0x01,0x60,0x01,0x13]
+# GFX11-REAL16: v_cmpx_eq_f16 v1.l, v2.l row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x04,0x7d,0x01,0x60,0x01,0x13]
+# GFX11-FAKE16: v_cmpx_eq_f16 v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x04,0x7d,0x01,0x60,0x01,0x13]
 
 0xfa,0xfe,0x04,0x7d,0x7f,0x6f,0xfd,0x30
-# GFX11: v_cmpx_eq_f16 -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0x04,0x7d,0x7f,0x6f,0xfd,0x30]
+# GFX11-REAL16: v_cmpx_eq_f16 -|v127.l|, -|v127.l| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0x04,0x7d,0x7f,0x6f,0xfd,0x30]
+# GFX11-FAKE16: v_cmpx_eq_f16 -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0x04,0x7d,0x7f,0x6f,0xfd,0x30]
+
+0xfa,0xfe,0x04,0x7d,0x7f,0x5f,0x01,0x01
+# GFX11-REAL16: v_cmpx_eq_f16 v127.l, v127.l row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0xfe,0x04,0x7d,0x7f,0x5f,0x01,0x01]
+# GFX11-FAKE16: v_cmpx_eq_f16 v127, v127 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0xfe,0x04,0x7d,0x7f,0x5f,0x01,0x01]
+
+0xfa,0x04,0x05,0x7d,0x81,0x60,0x01,0x13
+# GFX11-REAL16: v_cmpx_eq_f16 v1.h, v2.h row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x05,0x7d,0x81,0x60,0x01,0x13]
+# GFX11-FAKE16: v_cmpx_eq_f16 v129/*Invalid register, operand has 'VGPR_32_Lo128' register class*/, v130/*Invalid register, operand has 'VGPR_32_Lo128' register class*/ row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x05,0x7d,0x81,0x60,0x01,0x13]
+
+0xfa,0xfe,0x05,0x7d,0xff,0x6f,0xfd,0x30
+# GFX11-REAL16: v_cmpx_eq_f16 -|v127.h|, -|v127.h| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0x05,0x7d,0xff,0x6f,0xfd,0x30]
+# GFX11-FAKE16: v_cmpx_eq_f16 -|v255/*Invalid register, operand has 'VGPR_32_Lo128' register class*/|, -|v255/*Invalid register, operand has 'VGPR_32_Lo128' register class*/| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0x05,0x7d,0xff,0x6f,0xfd,0x30]
 
 0xfa,0x04,0x24,0x7d,0x01,0x1b,0x00,0xff
 # GFX11: v_cmpx_eq_f32 v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x24,0x7d,0x01,0x1b,0x00,0xff]
@@ -419,46 +445,72 @@
 # GFX11: v_cmpx_eq_u32 v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0x95,0x7d,0xff,0x6f,0x0d,0x30]
 
 0xfa,0x04,0x00,0x7d,0x01,0x1b,0x00,0xff
-# GFX11: v_cmpx_f_f16 v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x00,0x7d,0x01,0x1b,0x00,0xff]
+# GFX11-REAL16: v_cmpx_f_f16 v1.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x00,0x7d,0x01,0x1b,0x00,0xff]
+# GFX11-FAKE16: v_cmpx_f_f16 v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x00,0x7d,0x01,0x1b,0x00,0xff]
 
 0xfa,0x04,0x00,0x7d,0x01,0xe4,0x00,0xff
-# GFX11: v_cmpx_f_f16 v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x00,0x7d,0x01,0xe4,0x00,0xff]
+# GFX11-REAL16: v_cmpx_f_f16 v1.l, v2.l quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x00,0x7d,0x01,0xe4,0x00,0xff]
+# GFX11-FAKE16: v_cmpx_f_f16 v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x00,0x7d,0x01,0xe4,0x00,0xff]
 
 0xfa,0x04,0x00,0x7d,0x01,0x40,0x01,0xff
-# GFX11: v_cmpx_f_f16 v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x00,0x7d,0x01,0x40,0x01,0xff]
+# GFX11-REAL16: v_cmpx_f_f16 v1.l, v2.l row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x00,0x7d,0x01,0x40,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_f_f16 v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x00,0x7d,0x01,0x40,0x01,0xff]
 
 0xfa,0x04,0x00,0x7d,0x01,0x41,0x01,0xff
-# GFX11: v_cmpx_f_f16 v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x00,0x7d,0x01,0x41,0x01,0xff]
+# GFX11-REAL16: v_cmpx_f_f16 v1.l, v2.l row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x00,0x7d,0x01,0x41,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_f_f16 v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x00,0x7d,0x01,0x41,0x01,0xff]
 
 0xfa,0x04,0x00,0x7d,0x01,0x01,0x01,0xff
-# GFX11: v_cmpx_f_f16 v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x00,0x7d,0x01,0x01,0x01,0xff]
+# GFX11-REAL16: v_cmpx_f_f16 v1.l, v2.l row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x00,0x7d,0x01,0x01,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_f_f16 v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x00,0x7d,0x01,0x01,0x01,0xff]
 
 0xfa,0x04,0x00,0x7d,0x01,0x0f,0x01,0xff
-# GFX11: v_cmpx_f_f16 v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x00,0x7d,0x01,0x0f,0x01,0xff]
+# GFX11-REAL16: v_cmpx_f_f16 v1.l, v2.l row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x00,0x7d,0x01,0x0f,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_f_f16 v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x00,0x7d,0x01,0x0f,0x01,0xff]
 
 0xfa,0x04,0x00,0x7d,0x01,0x11,0x01,0xff
-# GFX11: v_cmpx_f_f16 v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x00,0x7d,0x01,0x11,0x01,0xff]
+# GFX11-REAL16: v_cmpx_f_f16 v1.l, v2.l row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x00,0x7d,0x01,0x11,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_f_f16 v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x00,0x7d,0x01,0x11,0x01,0xff]
 
 0xfa,0x04,0x00,0x7d,0x01,0x1f,0x01,0xff
-# GFX11: v_cmpx_f_f16 v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x00,0x7d,0x01,0x1f,0x01,0xff]
+# GFX11-REAL16: v_cmpx_f_f16 v1.l, v2.l row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x00,0x7d,0x01,0x1f,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_f_f16 v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x00,0x7d,0x01,0x1f,0x01,0xff]
 
 0xfa,0x04,0x00,0x7d,0x01,0x21,0x01,0xff
-# GFX11: v_cmpx_f_f16 v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x00,0x7d,0x01,0x21,0x01,0xff]
+# GFX11-REAL16: v_cmpx_f_f16 v1.l, v2.l row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x00,0x7d,0x01,0x21,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_f_f16 v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x00,0x7d,0x01,0x21,0x01,0xff]
 
 0xfa,0x04,0x00,0x7d,0x01,0x2f,0x01,0xff
-# GFX11: v_cmpx_f_f16 v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x00,0x7d,0x01,0x2f,0x01,0xff]
+# GFX11-REAL16: v_cmpx_f_f16 v1.l, v2.l row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x00,0x7d,0x01,0x2f,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_f_f16 v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x00,0x7d,0x01,0x2f,0x01,0xff]
 
 0xfa,0x04,0x00,0x7d,0x01,0x50,0x01,0xff
-# GFX11: v_cmpx_f_f16 v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x00,0x7d,0x01,0x50,0x01,0xff]
+# GFX11-REAL16: v_cmpx_f_f16 v1.l, v2.l row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x00,0x7d,0x01,0x50,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_f_f16 v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x00,0x7d,0x01,0x50,0x01,0xff]
 
 0xfa,0x04,0x00,0x7d,0x01,0x5f,0x01,0x01
-# GFX11: v_cmpx_f_f16 v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x00,0x7d,0x01,0x5f,0x01,0x01]
+# GFX11-REAL16: v_cmpx_f_f16 v1.l, v2.l row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x00,0x7d,0x01,0x5f,0x01,0x01]
+# GFX11-FAKE16: v_cmpx_f_f16 v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x00,0x7d,0x01,0x5f,0x01,0x01]
 
 0xfa,0x04,0x00,0x7d,0x01,0x60,0x01,0x13
-# GFX11: v_cmpx_f_f16 v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x00,0x7d,0x01,0x60,0x01,0x13]
+# GFX11-REAL16: v_cmpx_f_f16 v1.l, v2.l row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x00,0x7d,0x01,0x60,0x01,0x13]
+# GFX11-FAKE16: v_cmpx_f_f16 v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x00,0x7d,0x01,0x60,0x01,0x13]
 
 0xfa,0xfe,0x00,0x7d,0x7f,0x6f,0xfd,0x30
-# GFX11: v_cmpx_f_f16 -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0x00,0x7d,0x7f,0x6f,0xfd,0x30]
+# GFX11-REAL16: v_cmpx_f_f16 -|v127.l|, -|v127.l| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0x00,0x7d,0x7f,0x6f,0xfd,0x30]
+# GFX11-FAKE16: v_cmpx_f_f16 -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0x00,0x7d,0x7f,0x6f,0xfd,0x30]
+
+0xfa,0xfe,0x00,0x7d,0x7f,0x5f,0x01,0x01
+# GFX11-REAL16: v_cmpx_f_f16 v127.l, v127.l row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0xfe,0x00,0x7d,0x7f,0x5f,0x01,0x01]
+# GFX11-FAKE16: v_cmpx_f_f16 v127, v127 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0xfe,0x00,0x7d,0x7f,0x5f,0x01,0x01]
+
+0xfa,0x04,0x01,0x7d,0x81,0x60,0x01,0x13
+# GFX11-REAL16: v_cmpx_f_f16 v1.h, v2.h row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x01,0x7d,0x81,0x60,0x01,0x13]
+# GFX11-FAKE16: v_cmpx_f_f16 v129/*Invalid register, operand has 'VGPR_32_Lo128' register class*/, v130/*Invalid register, operand has 'VGPR_32_Lo128' register class*/ row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x01,0x7d,0x81,0x60,0x01,0x13]
+
+0xfa,0xfe,0x01,0x7d,0xff,0x6f,0xfd,0x30
+# GFX11-REAL16: v_cmpx_f_f16 -|v127.h|, -|v127.h| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0x01,0x7d,0xff,0x6f,0xfd,0x30]
+# GFX11-FAKE16: v_cmpx_f_f16 -|v255/*Invalid register, operand has 'VGPR_32_Lo128' register class*/|, -|v255/*Invalid register, operand has 'VGPR_32_Lo128' register class*/| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0x01,0x7d,0xff,0x6f,0xfd,0x30]
 
 0xfa,0x04,0x20,0x7d,0x01,0x1b,0x00,0xff
 # GFX11: v_cmpx_f_f32 v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x20,0x7d,0x01,0x1b,0x00,0xff]
@@ -587,46 +639,72 @@
 # GFX11: v_cmpx_f_u32 v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0x91,0x7d,0xff,0x6f,0x0d,0x30]
 
 0xfa,0x04,0x0c,0x7d,0x01,0x1b,0x00,0xff
-# GFX11: v_cmpx_ge_f16 v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0c,0x7d,0x01,0x1b,0x00,0xff]
+# GFX11-REAL16: v_cmpx_ge_f16 v1.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0c,0x7d,0x01,0x1b,0x00,0xff]
+# GFX11-FAKE16: v_cmpx_ge_f16 v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0c,0x7d,0x01,0x1b,0x00,0xff]
 
 0xfa,0x04,0x0c,0x7d,0x01,0xe4,0x00,0xff
-# GFX11: v_cmpx_ge_f16 v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0c,0x7d,0x01,0xe4,0x00,0xff]
+# GFX11-REAL16: v_cmpx_ge_f16 v1.l, v2.l quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0c,0x7d,0x01,0xe4,0x00,0xff]
+# GFX11-FAKE16: v_cmpx_ge_f16 v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0c,0x7d,0x01,0xe4,0x00,0xff]
 
 0xfa,0x04,0x0c,0x7d,0x01,0x40,0x01,0xff
-# GFX11: v_cmpx_ge_f16 v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0c,0x7d,0x01,0x40,0x01,0xff]
+# GFX11-REAL16: v_cmpx_ge_f16 v1.l, v2.l row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0c,0x7d,0x01,0x40,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_ge_f16 v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0c,0x7d,0x01,0x40,0x01,0xff]
 
 0xfa,0x04,0x0c,0x7d,0x01,0x41,0x01,0xff
-# GFX11: v_cmpx_ge_f16 v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0c,0x7d,0x01,0x41,0x01,0xff]
+# GFX11-REAL16: v_cmpx_ge_f16 v1.l, v2.l row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0c,0x7d,0x01,0x41,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_ge_f16 v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0c,0x7d,0x01,0x41,0x01,0xff]
 
 0xfa,0x04,0x0c,0x7d,0x01,0x01,0x01,0xff
-# GFX11: v_cmpx_ge_f16 v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0c,0x7d,0x01,0x01,0x01,0xff]
+# GFX11-REAL16: v_cmpx_ge_f16 v1.l, v2.l row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0c,0x7d,0x01,0x01,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_ge_f16 v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0c,0x7d,0x01,0x01,0x01,0xff]
 
 0xfa,0x04,0x0c,0x7d,0x01,0x0f,0x01,0xff
-# GFX11: v_cmpx_ge_f16 v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0c,0x7d,0x01,0x0f,0x01,0xff]
+# GFX11-REAL16: v_cmpx_ge_f16 v1.l, v2.l row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0c,0x7d,0x01,0x0f,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_ge_f16 v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0c,0x7d,0x01,0x0f,0x01,0xff]
 
 0xfa,0x04,0x0c,0x7d,0x01,0x11,0x01,0xff
-# GFX11: v_cmpx_ge_f16 v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0c,0x7d,0x01,0x11,0x01,0xff]
+# GFX11-REAL16: v_cmpx_ge_f16 v1.l, v2.l row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0c,0x7d,0x01,0x11,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_ge_f16 v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0c,0x7d,0x01,0x11,0x01,0xff]
 
 0xfa,0x04,0x0c,0x7d,0x01,0x1f,0x01,0xff
-# GFX11: v_cmpx_ge_f16 v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0c,0x7d,0x01,0x1f,0x01,0xff]
+# GFX11-REAL16: v_cmpx_ge_f16 v1.l, v2.l row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0c,0x7d,0x01,0x1f,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_ge_f16 v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0c,0x7d,0x01,0x1f,0x01,0xff]
 
 0xfa,0x04,0x0c,0x7d,0x01,0x21,0x01,0xff
-# GFX11: v_cmpx_ge_f16 v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0c,0x7d,0x01,0x21,0x01,0xff]
+# GFX11-REAL16: v_cmpx_ge_f16 v1.l, v2.l row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0c,0x7d,0x01,0x21,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_ge_f16 v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0c,0x7d,0x01,0x21,0x01,0xff]
 
 0xfa,0x04,0x0c,0x7d,0x01,0x2f,0x01,0xff
-# GFX11: v_cmpx_ge_f16 v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0c,0x7d,0x01,0x2f,0x01,0xff]
+# GFX11-REAL16: v_cmpx_ge_f16 v1.l, v2.l row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0c,0x7d,0x01,0x2f,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_ge_f16 v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0c,0x7d,0x01,0x2f,0x01,0xff]
 
 0xfa,0x04,0x0c,0x7d,0x01,0x50,0x01,0xff
-# GFX11: v_cmpx_ge_f16 v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0c,0x7d,0x01,0x50,0x01,0xff]
+# GFX11-REAL16: v_cmpx_ge_f16 v1.l, v2.l row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0c,0x7d,0x01,0x50,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_ge_f16 v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0c,0x7d,0x01,0x50,0x01,0xff]
 
 0xfa,0x04,0x0c,0x7d,0x01,0x5f,0x01,0x01
-# GFX11: v_cmpx_ge_f16 v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x0c,0x7d,0x01,0x5f,0x01,0x01]
+# GFX11-REAL16: v_cmpx_ge_f16 v1.l, v2.l row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x0c,0x7d,0x01,0x5f,0x01,0x01]
+# GFX11-FAKE16: v_cmpx_ge_f16 v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x0c,0x7d,0x01,0x5f,0x01,0x01]
 
 0xfa,0x04,0x0c,0x7d,0x01,0x60,0x01,0x13
-# GFX11: v_cmpx_ge_f16 v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x0c,0x7d,0x01,0x60,0x01,0x13]
+# GFX11-REAL16: v_cmpx_ge_f16 v1.l, v2.l row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x0c,0x7d,0x01,0x60,0x01,0x13]
+# GFX11-FAKE16: v_cmpx_ge_f16 v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x0c,0x7d,0x01,0x60,0x01,0x13]
 
 0xfa,0xfe,0x0c,0x7d,0x7f,0x6f,0xfd,0x30
-# GFX11: v_cmpx_ge_f16 -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0x0c,0x7d,0x7f,0x6f,0xfd,0x30]
+# GFX11-REAL16: v_cmpx_ge_f16 -|v127.l|, -|v127.l| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0x0c,0x7d,0x7f,0x6f,0xfd,0x30]
+# GFX11-FAKE16: v_cmpx_ge_f16 -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0x0c,0x7d,0x7f,0x6f,0xfd,0x30]
+
+0xfa,0xfe,0x0c,0x7d,0x7f,0x5f,0x01,0x01
+# GFX11-REAL16: v_cmpx_ge_f16 v127.l, v127.l row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0xfe,0x0c,0x7d,0x7f,0x5f,0x01,0x01]
+# GFX11-FAKE16: v_cmpx_ge_f16 v127, v127 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0xfe,0x0c,0x7d,0x7f,0x5f,0x01,0x01]
+
+0xfa,0x04,0x0d,0x7d,0x81,0x60,0x01,0x13
+# GFX11-REAL16: v_cmpx_ge_f16 v1.h, v2.h row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x0d,0x7d,0x81,0x60,0x01,0x13]
+# GFX11-FAKE16: v_cmpx_ge_f16 v129/*Invalid register, operand has 'VGPR_32_Lo128' register class*/, v130/*Invalid register, operand has 'VGPR_32_Lo128' register class*/ row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x0d,0x7d,0x81,0x60,0x01,0x13]
+
+0xfa,0xfe,0x0d,0x7d,0xff,0x6f,0xfd,0x30
+# GFX11-REAL16: v_cmpx_ge_f16 -|v127.h|, -|v127.h| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0x0d,0x7d,0xff,0x6f,0xfd,0x30]
+# GFX11-FAKE16: v_cmpx_ge_f16 -|v255/*Invalid register, operand has 'VGPR_32_Lo128' register class*/|, -|v255/*Invalid register, operand has 'VGPR_32_Lo128' register class*/| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0x0d,0x7d,0xff,0x6f,0xfd,0x30]
 
 0xfa,0x04,0x2c,0x7d,0x01,0x1b,0x00,0xff
 # GFX11: v_cmpx_ge_f32 v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x2c,0x7d,0x01,0x1b,0x00,0xff]
@@ -891,46 +969,72 @@
 # GFX11: v_cmpx_ge_u32 v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0x9d,0x7d,0xff,0x6f,0x0d,0x30]
 
 0xfa,0x04,0x08,0x7d,0x01,0x1b,0x00,0xff
-# GFX11: v_cmpx_gt_f16 v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x08,0x7d,0x01,0x1b,0x00,0xff]
+# GFX11-REAL16: v_cmpx_gt_f16 v1.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x08,0x7d,0x01,0x1b,0x00,0xff]
+# GFX11-FAKE16: v_cmpx_gt_f16 v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x08,0x7d,0x01,0x1b,0x00,0xff]
 
 0xfa,0x04,0x08,0x7d,0x01,0xe4,0x00,0xff
-# GFX11: v_cmpx_gt_f16 v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x08,0x7d,0x01,0xe4,0x00,0xff]
+# GFX11-REAL16: v_cmpx_gt_f16 v1.l, v2.l quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x08,0x7d,0x01,0xe4,0x00,0xff]
+# GFX11-FAKE16: v_cmpx_gt_f16 v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x08,0x7d,0x01,0xe4,0x00,0xff]
 
 0xfa,0x04,0x08,0x7d,0x01,0x40,0x01,0xff
-# GFX11: v_cmpx_gt_f16 v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x08,0x7d,0x01,0x40,0x01,0xff]
+# GFX11-REAL16: v_cmpx_gt_f16 v1.l, v2.l row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x08,0x7d,0x01,0x40,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_gt_f16 v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x08,0x7d,0x01,0x40,0x01,0xff]
 
 0xfa,0x04,0x08,0x7d,0x01,0x41,0x01,0xff
-# GFX11: v_cmpx_gt_f16 v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x08,0x7d,0x01,0x41,0x01,0xff]
+# GFX11-REAL16: v_cmpx_gt_f16 v1.l, v2.l row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x08,0x7d,0x01,0x41,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_gt_f16 v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x08,0x7d,0x01,0x41,0x01,0xff]
 
 0xfa,0x04,0x08,0x7d,0x01,0x01,0x01,0xff
-# GFX11: v_cmpx_gt_f16 v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x08,0x7d,0x01,0x01,0x01,0xff]
+# GFX11-REAL16: v_cmpx_gt_f16 v1.l, v2.l row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x08,0x7d,0x01,0x01,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_gt_f16 v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x08,0x7d,0x01,0x01,0x01,0xff]
 
 0xfa,0x04,0x08,0x7d,0x01,0x0f,0x01,0xff
-# GFX11: v_cmpx_gt_f16 v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x08,0x7d,0x01,0x0f,0x01,0xff]
+# GFX11-REAL16: v_cmpx_gt_f16 v1.l, v2.l row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x08,0x7d,0x01,0x0f,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_gt_f16 v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x08,0x7d,0x01,0x0f,0x01,0xff]
 
 0xfa,0x04,0x08,0x7d,0x01,0x11,0x01,0xff
-# GFX11: v_cmpx_gt_f16 v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x08,0x7d,0x01,0x11,0x01,0xff]
+# GFX11-REAL16: v_cmpx_gt_f16 v1.l, v2.l row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x08,0x7d,0x01,0x11,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_gt_f16 v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x08,0x7d,0x01,0x11,0x01,0xff]
 
 0xfa,0x04,0x08,0x7d,0x01,0x1f,0x01,0xff
-# GFX11: v_cmpx_gt_f16 v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x08,0x7d,0x01,0x1f,0x01,0xff]
+# GFX11-REAL16: v_cmpx_gt_f16 v1.l, v2.l row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x08,0x7d,0x01,0x1f,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_gt_f16 v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x08,0x7d,0x01,0x1f,0x01,0xff]
 
 0xfa,0x04,0x08,0x7d,0x01,0x21,0x01,0xff
-# GFX11: v_cmpx_gt_f16 v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x08,0x7d,0x01,0x21,0x01,0xff]
+# GFX11-REAL16: v_cmpx_gt_f16 v1.l, v2.l row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x08,0x7d,0x01,0x21,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_gt_f16 v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x08,0x7d,0x01,0x21,0x01,0xff]
 
 0xfa,0x04,0x08,0x7d,0x01,0x2f,0x01,0xff
-# GFX11: v_cmpx_gt_f16 v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x08,0x7d,0x01,0x2f,0x01,0xff]
+# GFX11-REAL16: v_cmpx_gt_f16 v1.l, v2.l row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x08,0x7d,0x01,0x2f,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_gt_f16 v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x08,0x7d,0x01,0x2f,0x01,0xff]
 
 0xfa,0x04,0x08,0x7d,0x01,0x50,0x01,0xff
-# GFX11: v_cmpx_gt_f16 v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x08,0x7d,0x01,0x50,0x01,0xff]
+# GFX11-REAL16: v_cmpx_gt_f16 v1.l, v2.l row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x08,0x7d,0x01,0x50,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_gt_f16 v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x08,0x7d,0x01,0x50,0x01,0xff]
 
 0xfa,0x04,0x08,0x7d,0x01,0x5f,0x01,0x01
-# GFX11: v_cmpx_gt_f16 v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x08,0x7d,0x01,0x5f,0x01,0x01]
+# GFX11-REAL16: v_cmpx_gt_f16 v1.l, v2.l row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x08,0x7d,0x01,0x5f,0x01,0x01]
+# GFX11-FAKE16: v_cmpx_gt_f16 v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x08,0x7d,0x01,0x5f,0x01,0x01]
 
 0xfa,0x04,0x08,0x7d,0x01,0x60,0x01,0x13
-# GFX11: v_cmpx_gt_f16 v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x08,0x7d,0x01,0x60,0x01,0x13]
+# GFX11-REAL16: v_cmpx_gt_f16 v1.l, v2.l row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x08,0x7d,0x01,0x60,0x01,0x13]
+# GFX11-FAKE16: v_cmpx_gt_f16 v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x08,0x7d,0x01,0x60,0x01,0x13]
 
 0xfa,0xfe,0x08,0x7d,0x7f,0x6f,0xfd,0x30
-# GFX11: v_cmpx_gt_f16 -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0x08,0x7d,0x7f,0x6f,0xfd,0x30]
+# GFX11-REAL16: v_cmpx_gt_f16 -|v127.l|, -|v127.l| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0x08,0x7d,0x7f,0x6f,0xfd,0x30]
+# GFX11-FAKE16: v_cmpx_gt_f16 -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0x08,0x7d,0x7f,0x6f,0xfd,0x30]
+
+0xfa,0xfe,0x08,0x7d,0x7f,0x5f,0x01,0x01
+# GFX11-REAL16: v_cmpx_gt_f16 v127.l, v127.l row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0xfe,0x08,0x7d,0x7f,0x5f,0x01,0x01]
+# GFX11-FAKE16: v_cmpx_gt_f16 v127, v127 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0xfe,0x08,0x7d,0x7f,0x5f,0x01,0x01]
+
+0xfa,0x04,0x09,0x7d,0x81,0x60,0x01,0x13
+# GFX11-REAL16: v_cmpx_gt_f16 v1.h, v2.h row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x09,0x7d,0x81,0x60,0x01,0x13]
+# GFX11-FAKE16: v_cmpx_gt_f16 v129/*Invalid register, operand has 'VGPR_32_Lo128' register class*/, v130/*Invalid register, operand has 'VGPR_32_Lo128' register class*/ row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x09,0x7d,0x81,0x60,0x01,0x13]
+
+0xfa,0xfe,0x09,0x7d,0xff,0x6f,0xfd,0x30
+# GFX11-REAL16: v_cmpx_gt_f16 -|v127.h|, -|v127.h| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0x09,0x7d,0xff,0x6f,0xfd,0x30]
+# GFX11-FAKE16: v_cmpx_gt_f16 -|v255/*Invalid register, operand has 'VGPR_32_Lo128' register class*/|, -|v255/*Invalid register, operand has 'VGPR_32_Lo128' register class*/| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0x09,0x7d,0xff,0x6f,0xfd,0x30]
 
 0xfa,0x04,0x28,0x7d,0x01,0x1b,0x00,0xff
 # GFX11: v_cmpx_gt_f32 v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x28,0x7d,0x01,0x1b,0x00,0xff]
@@ -1195,46 +1299,72 @@
 # GFX11: v_cmpx_gt_u32 v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0x99,0x7d,0xff,0x6f,0x0d,0x30]
 
 0xfa,0x04,0x06,0x7d,0x01,0x1b,0x00,0xff
-# GFX11: v_cmpx_le_f16 v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x06,0x7d,0x01,0x1b,0x00,0xff]
+# GFX11-REAL16: v_cmpx_le_f16 v1.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x06,0x7d,0x01,0x1b,0x00,0xff]
+# GFX11-FAKE16: v_cmpx_le_f16 v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x06,0x7d,0x01,0x1b,0x00,0xff]
 
 0xfa,0x04,0x06,0x7d,0x01,0xe4,0x00,0xff
-# GFX11: v_cmpx_le_f16 v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x06,0x7d,0x01,0xe4,0x00,0xff]
+# GFX11-REAL16: v_cmpx_le_f16 v1.l, v2.l quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x06,0x7d,0x01,0xe4,0x00,0xff]
+# GFX11-FAKE16: v_cmpx_le_f16 v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x06,0x7d,0x01,0xe4,0x00,0xff]
 
 0xfa,0x04,0x06,0x7d,0x01,0x40,0x01,0xff
-# GFX11: v_cmpx_le_f16 v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x06,0x7d,0x01,0x40,0x01,0xff]
+# GFX11-REAL16: v_cmpx_le_f16 v1.l, v2.l row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x06,0x7d,0x01,0x40,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_le_f16 v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x06,0x7d,0x01,0x40,0x01,0xff]
 
 0xfa,0x04,0x06,0x7d,0x01,0x41,0x01,0xff
-# GFX11: v_cmpx_le_f16 v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x06,0x7d,0x01,0x41,0x01,0xff]
+# GFX11-REAL16: v_cmpx_le_f16 v1.l, v2.l row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x06,0x7d,0x01,0x41,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_le_f16 v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x06,0x7d,0x01,0x41,0x01,0xff]
 
 0xfa,0x04,0x06,0x7d,0x01,0x01,0x01,0xff
-# GFX11: v_cmpx_le_f16 v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x06,0x7d,0x01,0x01,0x01,0xff]
+# GFX11-REAL16: v_cmpx_le_f16 v1.l, v2.l row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x06,0x7d,0x01,0x01,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_le_f16 v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x06,0x7d,0x01,0x01,0x01,0xff]
 
 0xfa,0x04,0x06,0x7d,0x01,0x0f,0x01,0xff
-# GFX11: v_cmpx_le_f16 v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x06,0x7d,0x01,0x0f,0x01,0xff]
+# GFX11-REAL16: v_cmpx_le_f16 v1.l, v2.l row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x06,0x7d,0x01,0x0f,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_le_f16 v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x06,0x7d,0x01,0x0f,0x01,0xff]
 
 0xfa,0x04,0x06,0x7d,0x01,0x11,0x01,0xff
-# GFX11: v_cmpx_le_f16 v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x06,0x7d,0x01,0x11,0x01,0xff]
+# GFX11-REAL16: v_cmpx_le_f16 v1.l, v2.l row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x06,0x7d,0x01,0x11,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_le_f16 v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x06,0x7d,0x01,0x11,0x01,0xff]
 
 0xfa,0x04,0x06,0x7d,0x01,0x1f,0x01,0xff
-# GFX11: v_cmpx_le_f16 v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x06,0x7d,0x01,0x1f,0x01,0xff]
+# GFX11-REAL16: v_cmpx_le_f16 v1.l, v2.l row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x06,0x7d,0x01,0x1f,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_le_f16 v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x06,0x7d,0x01,0x1f,0x01,0xff]
 
 0xfa,0x04,0x06,0x7d,0x01,0x21,0x01,0xff
-# GFX11: v_cmpx_le_f16 v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x06,0x7d,0x01,0x21,0x01,0xff]
+# GFX11-REAL16: v_cmpx_le_f16 v1.l, v2.l row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x06,0x7d,0x01,0x21,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_le_f16 v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x06,0x7d,0x01,0x21,0x01,0xff]
 
 0xfa,0x04,0x06,0x7d,0x01,0x2f,0x01,0xff
-# GFX11: v_cmpx_le_f16 v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x06,0x7d,0x01,0x2f,0x01,0xff]
+# GFX11-REAL16: v_cmpx_le_f16 v1.l, v2.l row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x06,0x7d,0x01,0x2f,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_le_f16 v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x06,0x7d,0x01,0x2f,0x01,0xff]
 
 0xfa,0x04,0x06,0x7d,0x01,0x50,0x01,0xff
-# GFX11: v_cmpx_le_f16 v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x06,0x7d,0x01,0x50,0x01,0xff]
+# GFX11-REAL16: v_cmpx_le_f16 v1.l, v2.l row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x06,0x7d,0x01,0x50,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_le_f16 v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x06,0x7d,0x01,0x50,0x01,0xff]
 
 0xfa,0x04,0x06,0x7d,0x01,0x5f,0x01,0x01
-# GFX11: v_cmpx_le_f16 v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x06,0x7d,0x01,0x5f,0x01,0x01]
+# GFX11-REAL16: v_cmpx_le_f16 v1.l, v2.l row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x06,0x7d,0x01,0x5f,0x01,0x01]
+# GFX11-FAKE16: v_cmpx_le_f16 v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x06,0x7d,0x01,0x5f,0x01,0x01]
 
 0xfa,0x04,0x06,0x7d,0x01,0x60,0x01,0x13
-# GFX11: v_cmpx_le_f16 v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x06,0x7d,0x01,0x60,0x01,0x13]
+# GFX11-REAL16: v_cmpx_le_f16 v1.l, v2.l row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x06,0x7d,0x01,0x60,0x01,0x13]
+# GFX11-FAKE16: v_cmpx_le_f16 v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x06,0x7d,0x01,0x60,0x01,0x13]
 
 0xfa,0xfe,0x06,0x7d,0x7f,0x6f,0xfd,0x30
-# GFX11: v_cmpx_le_f16 -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0x06,0x7d,0x7f,0x6f,0xfd,0x30]
+# GFX11-REAL16: v_cmpx_le_f16 -|v127.l|, -|v127.l| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0x06,0x7d,0x7f,0x6f,0xfd,0x30]
+# GFX11-FAKE16: v_cmpx_le_f16 -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0x06,0x7d,0x7f,0x6f,0xfd,0x30]
+
+0xfa,0xfe,0x06,0x7d,0x7f,0x5f,0x01,0x01
+# GFX11-REAL16: v_cmpx_le_f16 v127.l, v127.l row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0xfe,0x06,0x7d,0x7f,0x5f,0x01,0x01]
+# GFX11-FAKE16: v_cmpx_le_f16 v127, v127 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0xfe,0x06,0x7d,0x7f,0x5f,0x01,0x01]
+
+0xfa,0x04,0x07,0x7d,0x81,0x60,0x01,0x13
+# GFX11-REAL16: v_cmpx_le_f16 v1.h, v2.h row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x07,0x7d,0x81,0x60,0x01,0x13]
+# GFX11-FAKE16: v_cmpx_le_f16 v129/*Invalid register, operand has 'VGPR_32_Lo128' register class*/, v130/*Invalid register, operand has 'VGPR_32_Lo128' register class*/ row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x07,0x7d,0x81,0x60,0x01,0x13]
+
+0xfa,0xfe,0x07,0x7d,0xff,0x6f,0xfd,0x30
+# GFX11-REAL16: v_cmpx_le_f16 -|v127.h|, -|v127.h| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0x07,0x7d,0xff,0x6f,0xfd,0x30]
+# GFX11-FAKE16: v_cmpx_le_f16 -|v255/*Invalid register, operand has 'VGPR_32_Lo128' register class*/|, -|v255/*Invalid register, operand has 'VGPR_32_Lo128' register class*/| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0x07,0x7d,0xff,0x6f,0xfd,0x30]
 
 0xfa,0x04,0x26,0x7d,0x01,0x1b,0x00,0xff
 # GFX11: v_cmpx_le_f32 v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x26,0x7d,0x01,0x1b,0x00,0xff]
@@ -1499,46 +1629,72 @@
 # GFX11: v_cmpx_le_u32 v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0x97,0x7d,0xff,0x6f,0x0d,0x30]
 
 0xfa,0x04,0x0a,0x7d,0x01,0x1b,0x00,0xff
-# GFX11: v_cmpx_lg_f16 v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x7d,0x01,0x1b,0x00,0xff]
+# GFX11-REAL16: v_cmpx_lg_f16 v1.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x7d,0x01,0x1b,0x00,0xff]
+# GFX11-FAKE16: v_cmpx_lg_f16 v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x7d,0x01,0x1b,0x00,0xff]
 
 0xfa,0x04,0x0a,0x7d,0x01,0xe4,0x00,0xff
-# GFX11: v_cmpx_lg_f16 v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x7d,0x01,0xe4,0x00,0xff]
+# GFX11-REAL16: v_cmpx_lg_f16 v1.l, v2.l quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x7d,0x01,0xe4,0x00,0xff]
+# GFX11-FAKE16: v_cmpx_lg_f16 v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x7d,0x01,0xe4,0x00,0xff]
 
 0xfa,0x04,0x0a,0x7d,0x01,0x40,0x01,0xff
-# GFX11: v_cmpx_lg_f16 v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x7d,0x01,0x40,0x01,0xff]
+# GFX11-REAL16: v_cmpx_lg_f16 v1.l, v2.l row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x7d,0x01,0x40,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_lg_f16 v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x7d,0x01,0x40,0x01,0xff]
 
 0xfa,0x04,0x0a,0x7d,0x01,0x41,0x01,0xff
-# GFX11: v_cmpx_lg_f16 v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x7d,0x01,0x41,0x01,0xff]
+# GFX11-REAL16: v_cmpx_lg_f16 v1.l, v2.l row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x7d,0x01,0x41,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_lg_f16 v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x7d,0x01,0x41,0x01,0xff]
 
 0xfa,0x04,0x0a,0x7d,0x01,0x01,0x01,0xff
-# GFX11: v_cmpx_lg_f16 v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x7d,0x01,0x01,0x01,0xff]
+# GFX11-REAL16: v_cmpx_lg_f16 v1.l, v2.l row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x7d,0x01,0x01,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_lg_f16 v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x7d,0x01,0x01,0x01,0xff]
 
 0xfa,0x04,0x0a,0x7d,0x01,0x0f,0x01,0xff
-# GFX11: v_cmpx_lg_f16 v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x7d,0x01,0x0f,0x01,0xff]
+# GFX11-REAL16: v_cmpx_lg_f16 v1.l, v2.l row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x7d,0x01,0x0f,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_lg_f16 v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x7d,0x01,0x0f,0x01,0xff]
 
 0xfa,0x04,0x0a,0x7d,0x01,0x11,0x01,0xff
-# GFX11: v_cmpx_lg_f16 v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x7d,0x01,0x11,0x01,0xff]
+# GFX11-REAL16: v_cmpx_lg_f16 v1.l, v2.l row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x7d,0x01,0x11,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_lg_f16 v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x7d,0x01,0x11,0x01,0xff]
 
 0xfa,0x04,0x0a,0x7d,0x01,0x1f,0x01,0xff
-# GFX11: v_cmpx_lg_f16 v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x7d,0x01,0x1f,0x01,0xff]
+# GFX11-REAL16: v_cmpx_lg_f16 v1.l, v2.l row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x7d,0x01,0x1f,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_lg_f16 v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x7d,0x01,0x1f,0x01,0xff]
 
 0xfa,0x04,0x0a,0x7d,0x01,0x21,0x01,0xff
-# GFX11: v_cmpx_lg_f16 v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x7d,0x01,0x21,0x01,0xff]
+# GFX11-REAL16: v_cmpx_lg_f16 v1.l, v2.l row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x7d,0x01,0x21,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_lg_f16 v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x7d,0x01,0x21,0x01,0xff]
 
 0xfa,0x04,0x0a,0x7d,0x01,0x2f,0x01,0xff
-# GFX11: v_cmpx_lg_f16 v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x7d,0x01,0x2f,0x01,0xff]
+# GFX11-REAL16: v_cmpx_lg_f16 v1.l, v2.l row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x7d,0x01,0x2f,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_lg_f16 v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x7d,0x01,0x2f,0x01,0xff]
 
 0xfa,0x04,0x0a,0x7d,0x01,0x50,0x01,0xff
-# GFX11: v_cmpx_lg_f16 v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x7d,0x01,0x50,0x01,0xff]
+# GFX11-REAL16: v_cmpx_lg_f16 v1.l, v2.l row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x7d,0x01,0x50,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_lg_f16 v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x7d,0x01,0x50,0x01,0xff]
 
 0xfa,0x04,0x0a,0x7d,0x01,0x5f,0x01,0x01
-# GFX11: v_cmpx_lg_f16 v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x0a,0x7d,0x01,0x5f,0x01,0x01]
+# GFX11-REAL16: v_cmpx_lg_f16 v1.l, v2.l row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x0a,0x7d,0x01,0x5f,0x01,0x01]
+# GFX11-FAKE16: v_cmpx_lg_f16 v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x0a,0x7d,0x01,0x5f,0x01,0x01]
 
 0xfa,0x04,0x0a,0x7d,0x01,0x60,0x01,0x13
-# GFX11: v_cmpx_lg_f16 v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x0a,0x7d,0x01,0x60,0x01,0x13]
+# GFX11-REAL16: v_cmpx_lg_f16 v1.l, v2.l row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x0a,0x7d,0x01,0x60,0x01,0x13]
+# GFX11-FAKE16: v_cmpx_lg_f16 v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x0a,0x7d,0x01,0x60,0x01,0x13]
 
 0xfa,0xfe,0x0a,0x7d,0x7f,0x6f,0xfd,0x30
-# GFX11: v_cmpx_lg_f16 -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0x0a,0x7d,0x7f,0x6f,0xfd,0x30]
+# GFX11-REAL16: v_cmpx_lg_f16 -|v127.l|, -|v127.l| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0x0a,0x7d,0x7f,0x6f,0xfd,0x30]
+# GFX11-FAKE16: v_cmpx_lg_f16 -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0x0a,0x7d,0x7f,0x6f,0xfd,0x30]
+
+0xfa,0xfe,0x0a,0x7d,0x7f,0x5f,0x01,0x01
+# GFX11-REAL16: v_cmpx_lg_f16 v127.l, v127.l row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0xfe,0x0a,0x7d,0x7f,0x5f,0x01,0x01]
+# GFX11-FAKE16: v_cmpx_lg_f16 v127, v127 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0xfe,0x0a,0x7d,0x7f,0x5f,0x01,0x01]
+
+0xfa,0x04,0x0b,0x7d,0x81,0x60,0x01,0x13
+# GFX11-REAL16: v_cmpx_lg_f16 v1.h, v2.h row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x0b,0x7d,0x81,0x60,0x01,0x13]
+# GFX11-FAKE16: v_cmpx_lg_f16 v129/*Invalid register, operand has 'VGPR_32_Lo128' register class*/, v130/*Invalid register, operand has 'VGPR_32_Lo128' register class*/ row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x0b,0x7d,0x81,0x60,0x01,0x13]
+
+0xfa,0xfe,0x0b,0x7d,0xff,0x6f,0xfd,0x30
+# GFX11-REAL16: v_cmpx_lg_f16 -|v127.h|, -|v127.h| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0x0b,0x7d,0xff,0x6f,0xfd,0x30]
+# GFX11-FAKE16: v_cmpx_lg_f16 -|v255/*Invalid register, operand has 'VGPR_32_Lo128' register class*/|, -|v255/*Invalid register, operand has 'VGPR_32_Lo128' register class*/| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0x0b,0x7d,0xff,0x6f,0xfd,0x30]
 
 0xfa,0x04,0x2a,0x7d,0x01,0x1b,0x00,0xff
 # GFX11: v_cmpx_lg_f32 v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x2a,0x7d,0x01,0x1b,0x00,0xff]
@@ -2133,46 +2289,72 @@
 # GFX11: v_cmpx_ne_u32 v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0x9b,0x7d,0xff,0x6f,0x0d,0x30]
 
 0xfa,0x04,0x1a,0x7d,0x01,0x1b,0x00,0xff
-# GFX11: v_cmpx_neq_f16 v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1a,0x7d,0x01,0x1b,0x00,0xff]
+# GFX11-REAL16: v_cmpx_neq_f16 v1.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1a,0x7d,0x01,0x1b,0x00,0xff]
+# GFX11-FAKE16: v_cmpx_neq_f16 v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1a,0x7d,0x01,0x1b,0x00,0xff]
 
 0xfa,0x04,0x1a,0x7d,0x01,0xe4,0x00,0xff
-# GFX11: v_cmpx_neq_f16 v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1a,0x7d,0x01,0xe4,0x00,0xff]
+# GFX11-REAL16: v_cmpx_neq_f16 v1.l, v2.l quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1a,0x7d,0x01,0xe4,0x00,0xff]
+# GFX11-FAKE16: v_cmpx_neq_f16 v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1a,0x7d,0x01,0xe4,0x00,0xff]
 
 0xfa,0x04,0x1a,0x7d,0x01,0x40,0x01,0xff
-# GFX11: v_cmpx_neq_f16 v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1a,0x7d,0x01,0x40,0x01,0xff]
+# GFX11-REAL16: v_cmpx_neq_f16 v1.l, v2.l row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1a,0x7d,0x01,0x40,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_neq_f16 v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1a,0x7d,0x01,0x40,0x01,0xff]
 
 0xfa,0x04,0x1a,0x7d,0x01,0x41,0x01,0xff
-# GFX11: v_cmpx_neq_f16 v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1a,0x7d,0x01,0x41,0x01,0xff]
+# GFX11-REAL16: v_cmpx_neq_f16 v1.l, v2.l row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1a,0x7d,0x01,0x41,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_neq_f16 v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1a,0x7d,0x01,0x41,0x01,0xff]
 
 0xfa,0x04,0x1a,0x7d,0x01,0x01,0x01,0xff
-# GFX11: v_cmpx_neq_f16 v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1a,0x7d,0x01,0x01,0x01,0xff]
+# GFX11-REAL16: v_cmpx_neq_f16 v1.l, v2.l row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1a,0x7d,0x01,0x01,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_neq_f16 v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1a,0x7d,0x01,0x01,0x01,0xff]
 
 0xfa,0x04,0x1a,0x7d,0x01,0x0f,0x01,0xff
-# GFX11: v_cmpx_neq_f16 v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1a,0x7d,0x01,0x0f,0x01,0xff]
+# GFX11-REAL16: v_cmpx_neq_f16 v1.l, v2.l row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1a,0x7d,0x01,0x0f,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_neq_f16 v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1a,0x7d,0x01,0x0f,0x01,0xff]
 
 0xfa,0x04,0x1a,0x7d,0x01,0x11,0x01,0xff
-# GFX11: v_cmpx_neq_f16 v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1a,0x7d,0x01,0x11,0x01,0xff]
+# GFX11-REAL16: v_cmpx_neq_f16 v1.l, v2.l row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1a,0x7d,0x01,0x11,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_neq_f16 v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1a,0x7d,0x01,0x11,0x01,0xff]
 
 0xfa,0x04,0x1a,0x7d,0x01,0x1f,0x01,0xff
-# GFX11: v_cmpx_neq_f16 v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1a,0x7d,0x01,0x1f,0x01,0xff]
+# GFX11-REAL16: v_cmpx_neq_f16 v1.l, v2.l row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1a,0x7d,0x01,0x1f,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_neq_f16 v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1a,0x7d,0x01,0x1f,0x01,0xff]
 
 0xfa,0x04,0x1a,0x7d,0x01,0x21,0x01,0xff
-# GFX11: v_cmpx_neq_f16 v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1a,0x7d,0x01,0x21,0x01,0xff]
+# GFX11-REAL16: v_cmpx_neq_f16 v1.l, v2.l row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1a,0x7d,0x01,0x21,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_neq_f16 v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1a,0x7d,0x01,0x21,0x01,0xff]
 
 0xfa,0x04,0x1a,0x7d,0x01,0x2f,0x01,0xff
-# GFX11: v_cmpx_neq_f16 v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1a,0x7d,0x01,0x2f,0x01,0xff]
+# GFX11-REAL16: v_cmpx_neq_f16 v1.l, v2.l row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1a,0x7d,0x01,0x2f,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_neq_f16 v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1a,0x7d,0x01,0x2f,0x01,0xff]
 
 0xfa,0x04,0x1a,0x7d,0x01,0x50,0x01,0xff
-# GFX11: v_cmpx_neq_f16 v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1a,0x7d,0x01,0x50,0x01,0xff]
+# GFX11-REAL16: v_cmpx_neq_f16 v1.l, v2.l row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1a,0x7d,0x01,0x50,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_neq_f16 v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1a,0x7d,0x01,0x50,0x01,0xff]
 
 0xfa,0x04,0x1a,0x7d,0x01,0x5f,0x01,0x01
-# GFX11: v_cmpx_neq_f16 v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x1a,0x7d,0x01,0x5f,0x01,0x01]
+# GFX11-REAL16: v_cmpx_neq_f16 v1.l, v2.l row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x1a,0x7d,0x01,0x5f,0x01,0x01]
+# GFX11-FAKE16: v_cmpx_neq_f16 v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x1a,0x7d,0x01,0x5f,0x01,0x01]
 
 0xfa,0x04,0x1a,0x7d,0x01,0x60,0x01,0x13
-# GFX11: v_cmpx_neq_f16 v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x1a,0x7d,0x01,0x60,0x01,0x13]
+# GFX11-REAL16: v_cmpx_neq_f16 v1.l, v2.l row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x1a,0x7d,0x01,0x60,0x01,0x13]
+# GFX11-FAKE16: v_cmpx_neq_f16 v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x1a,0x7d,0x01,0x60,0x01,0x13]
 
 0xfa,0xfe,0x1a,0x7d,0x7f,0x6f,0xfd,0x30
-# GFX11: v_cmpx_neq_f16 -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0x1a,0x7d,0x7f,0x6f,0xfd,0x30]
+# GFX11-REAL16: v_cmpx_neq_f16 -|v127.l|, -|v127.l| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0x1a,0x7d,0x7f,0x6f,0xfd,0x30]
+# GFX11-FAKE16: v_cmpx_neq_f16 -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0x1a,0x7d,0x7f,0x6f,0xfd,0x30]
+
+0xfa,0xfe,0x1a,0x7d,0x7f,0x5f,0x01,0x01
+# GFX11-REAL16: v_cmpx_neq_f16 v127.l, v127.l row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0xfe,0x1a,0x7d,0x7f,0x5f,0x01,0x01]
+# GFX11-FAKE16: v_cmpx_neq_f16 v127, v127 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0xfe,0x1a,0x7d,0x7f,0x5f,0x01,0x01]
+
+0xfa,0x04,0x1b,0x7d,0x81,0x60,0x01,0x13
+# GFX11-REAL16: v_cmpx_neq_f16 v1.h, v2.h row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x1b,0x7d,0x81,0x60,0x01,0x13]
+# GFX11-FAKE16: v_cmpx_neq_f16 v129/*Invalid register, operand has 'VGPR_32_Lo128' register class*/, v130/*Invalid register, operand has 'VGPR_32_Lo128' register class*/ row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x1b,0x7d,0x81,0x60,0x01,0x13]
+
+0xfa,0xfe,0x1b,0x7d,0xff,0x6f,0xfd,0x30
+# GFX11-REAL16: v_cmpx_neq_f16 -|v127.h|, -|v127.h| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0x1b,0x7d,0xff,0x6f,0xfd,0x30]
+# GFX11-FAKE16: v_cmpx_neq_f16 -|v255/*Invalid register, operand has 'VGPR_32_Lo128' register class*/|, -|v255/*Invalid register, operand has 'VGPR_32_Lo128' register class*/| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0x1b,0x7d,0xff,0x6f,0xfd,0x30]
 
 0xfa,0x04,0x3a,0x7d,0x01,0x1b,0x00,0xff
 # GFX11: v_cmpx_neq_f32 v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x3a,0x7d,0x01,0x1b,0x00,0xff]
@@ -2217,46 +2399,72 @@
 # GFX11: v_cmpx_neq_f32 -|v255|, -|v255| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0x3b,0x7d,0xff,0x6f,0xfd,0x30]
 
 0xfa,0x04,0x12,0x7d,0x01,0x1b,0x00,0xff
-# GFX11: v_cmpx_nge_f16 v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x12,0x7d,0x01,0x1b,0x00,0xff]
+# GFX11-REAL16: v_cmpx_nge_f16 v1.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x12,0x7d,0x01,0x1b,0x00,0xff]
+# GFX11-FAKE16: v_cmpx_nge_f16 v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x12,0x7d,0x01,0x1b,0x00,0xff]
 
 0xfa,0x04,0x12,0x7d,0x01,0xe4,0x00,0xff
-# GFX11: v_cmpx_nge_f16 v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x12,0x7d,0x01,0xe4,0x00,0xff]
+# GFX11-REAL16: v_cmpx_nge_f16 v1.l, v2.l quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x12,0x7d,0x01,0xe4,0x00,0xff]
+# GFX11-FAKE16: v_cmpx_nge_f16 v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x12,0x7d,0x01,0xe4,0x00,0xff]
 
 0xfa,0x04,0x12,0x7d,0x01,0x40,0x01,0xff
-# GFX11: v_cmpx_nge_f16 v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x12,0x7d,0x01,0x40,0x01,0xff]
+# GFX11-REAL16: v_cmpx_nge_f16 v1.l, v2.l row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x12,0x7d,0x01,0x40,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_nge_f16 v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x12,0x7d,0x01,0x40,0x01,0xff]
 
 0xfa,0x04,0x12,0x7d,0x01,0x41,0x01,0xff
-# GFX11: v_cmpx_nge_f16 v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x12,0x7d,0x01,0x41,0x01,0xff]
+# GFX11-REAL16: v_cmpx_nge_f16 v1.l, v2.l row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x12,0x7d,0x01,0x41,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_nge_f16 v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x12,0x7d,0x01,0x41,0x01,0xff]
 
 0xfa,0x04,0x12,0x7d,0x01,0x01,0x01,0xff
-# GFX11: v_cmpx_nge_f16 v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x12,0x7d,0x01,0x01,0x01,0xff]
+# GFX11-REAL16: v_cmpx_nge_f16 v1.l, v2.l row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x12,0x7d,0x01,0x01,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_nge_f16 v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x12,0x7d,0x01,0x01,0x01,0xff]
 
 0xfa,0x04,0x12,0x7d,0x01,0x0f,0x01,0xff
-# GFX11: v_cmpx_nge_f16 v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x12,0x7d,0x01,0x0f,0x01,0xff]
+# GFX11-REAL16: v_cmpx_nge_f16 v1.l, v2.l row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x12,0x7d,0x01,0x0f,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_nge_f16 v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x12,0x7d,0x01,0x0f,0x01,0xff]
 
 0xfa,0x04,0x12,0x7d,0x01,0x11,0x01,0xff
-# GFX11: v_cmpx_nge_f16 v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x12,0x7d,0x01,0x11,0x01,0xff]
+# GFX11-REAL16: v_cmpx_nge_f16 v1.l, v2.l row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x12,0x7d,0x01,0x11,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_nge_f16 v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x12,0x7d,0x01,0x11,0x01,0xff]
 
 0xfa,0x04,0x12,0x7d,0x01,0x1f,0x01,0xff
-# GFX11: v_cmpx_nge_f16 v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x12,0x7d,0x01,0x1f,0x01,0xff]
+# GFX11-REAL16: v_cmpx_nge_f16 v1.l, v2.l row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x12,0x7d,0x01,0x1f,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_nge_f16 v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x12,0x7d,0x01,0x1f,0x01,0xff]
 
 0xfa,0x04,0x12,0x7d,0x01,0x21,0x01,0xff
-# GFX11: v_cmpx_nge_f16 v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x12,0x7d,0x01,0x21,0x01,0xff]
+# GFX11-REAL16: v_cmpx_nge_f16 v1.l, v2.l row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x12,0x7d,0x01,0x21,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_nge_f16 v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x12,0x7d,0x01,0x21,0x01,0xff]
 
 0xfa,0x04,0x12,0x7d,0x01,0x2f,0x01,0xff
-# GFX11: v_cmpx_nge_f16 v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x12,0x7d,0x01,0x2f,0x01,0xff]
+# GFX11-REAL16: v_cmpx_nge_f16 v1.l, v2.l row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x12,0x7d,0x01,0x2f,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_nge_f16 v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x12,0x7d,0x01,0x2f,0x01,0xff]
 
 0xfa,0x04,0x12,0x7d,0x01,0x50,0x01,0xff
-# GFX11: v_cmpx_nge_f16 v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x12,0x7d,0x01,0x50,0x01,0xff]
+# GFX11-REAL16: v_cmpx_nge_f16 v1.l, v2.l row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x12,0x7d,0x01,0x50,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_nge_f16 v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x12,0x7d,0x01,0x50,0x01,0xff]
 
 0xfa,0x04,0x12,0x7d,0x01,0x5f,0x01,0x01
-# GFX11: v_cmpx_nge_f16 v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x12,0x7d,0x01,0x5f,0x01,0x01]
+# GFX11-REAL16: v_cmpx_nge_f16 v1.l, v2.l row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x12,0x7d,0x01,0x5f,0x01,0x01]
+# GFX11-FAKE16: v_cmpx_nge_f16 v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x12,0x7d,0x01,0x5f,0x01,0x01]
 
 0xfa,0x04,0x12,0x7d,0x01,0x60,0x01,0x13
-# GFX11: v_cmpx_nge_f16 v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x12,0x7d,0x01,0x60,0x01,0x13]
+# GFX11-REAL16: v_cmpx_nge_f16 v1.l, v2.l row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x12,0x7d,0x01,0x60,0x01,0x13]
+# GFX11-FAKE16: v_cmpx_nge_f16 v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x12,0x7d,0x01,0x60,0x01,0x13]
 
 0xfa,0xfe,0x12,0x7d,0x7f,0x6f,0xfd,0x30
-# GFX11: v_cmpx_nge_f16 -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0x12,0x7d,0x7f,0x6f,0xfd,0x30]
+# GFX11-REAL16: v_cmpx_nge_f16 -|v127.l|, -|v127.l| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0x12,0x7d,0x7f,0x6f,0xfd,0x30]
+# GFX11-FAKE16: v_cmpx_nge_f16 -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0x12,0x7d,0x7f,0x6f,0xfd,0x30]
+
+0xfa,0xfe,0x12,0x7d,0x7f,0x5f,0x01,0x01
+# GFX11-REAL16: v_cmpx_nge_f16 v127.l, v127.l row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0xfe,0x12,0x7d,0x7f,0x5f,0x01,0x01]
+# GFX11-FAKE16: v_cmpx_nge_f16 v127, v127 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0xfe,0x12,0x7d,0x7f,0x5f,0x01,0x01]
+
+0xfa,0x04,0x13,0x7d,0x81,0x60,0x01,0x13
+# GFX11-REAL16: v_cmpx_nge_f16 v1.h, v2.h row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x13,0x7d,0x81,0x60,0x01,0x13]
+# GFX11-FAKE16: v_cmpx_nge_f16 v129/*Invalid register, operand has 'VGPR_32_Lo128' register class*/, v130/*Invalid register, operand has 'VGPR_32_Lo128' register class*/ row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x13,0x7d,0x81,0x60,0x01,0x13]
+
+0xfa,0xfe,0x13,0x7d,0xff,0x6f,0xfd,0x30
+# GFX11-REAL16: v_cmpx_nge_f16 -|v127.h|, -|v127.h| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0x13,0x7d,0xff,0x6f,0xfd,0x30]
+# GFX11-FAKE16: v_cmpx_nge_f16 -|v255/*Invalid register, operand has 'VGPR_32_Lo128' register class*/|, -|v255/*Invalid register, operand has 'VGPR_32_Lo128' register class*/| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0x13,0x7d,0xff,0x6f,0xfd,0x30]
 
 0xfa,0x04,0x32,0x7d,0x01,0x1b,0x00,0xff
 # GFX11: v_cmpx_nge_f32 v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x32,0x7d,0x01,0x1b,0x00,0xff]
@@ -2301,46 +2509,72 @@
 # GFX11: v_cmpx_nge_f32 -|v255|, -|v255| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0x33,0x7d,0xff,0x6f,0xfd,0x30]
 
 0xfa,0x04,0x16,0x7d,0x01,0x1b,0x00,0xff
-# GFX11: v_cmpx_ngt_f16 v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x16,0x7d,0x01,0x1b,0x00,0xff]
+# GFX11-REAL16: v_cmpx_ngt_f16 v1.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x16,0x7d,0x01,0x1b,0x00,0xff]
+# GFX11-FAKE16: v_cmpx_ngt_f16 v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x16,0x7d,0x01,0x1b,0x00,0xff]
 
 0xfa,0x04,0x16,0x7d,0x01,0xe4,0x00,0xff
-# GFX11: v_cmpx_ngt_f16 v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x16,0x7d,0x01,0xe4,0x00,0xff]
+# GFX11-REAL16: v_cmpx_ngt_f16 v1.l, v2.l quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x16,0x7d,0x01,0xe4,0x00,0xff]
+# GFX11-FAKE16: v_cmpx_ngt_f16 v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x16,0x7d,0x01,0xe4,0x00,0xff]
 
 0xfa,0x04,0x16,0x7d,0x01,0x40,0x01,0xff
-# GFX11: v_cmpx_ngt_f16 v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x16,0x7d,0x01,0x40,0x01,0xff]
+# GFX11-REAL16: v_cmpx_ngt_f16 v1.l, v2.l row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x16,0x7d,0x01,0x40,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_ngt_f16 v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x16,0x7d,0x01,0x40,0x01,0xff]
 
 0xfa,0x04,0x16,0x7d,0x01,0x41,0x01,0xff
-# GFX11: v_cmpx_ngt_f16 v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x16,0x7d,0x01,0x41,0x01,0xff]
+# GFX11-REAL16: v_cmpx_ngt_f16 v1.l, v2.l row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x16,0x7d,0x01,0x41,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_ngt_f16 v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x16,0x7d,0x01,0x41,0x01,0xff]
 
 0xfa,0x04,0x16,0x7d,0x01,0x01,0x01,0xff
-# GFX11: v_cmpx_ngt_f16 v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x16,0x7d,0x01,0x01,0x01,0xff]
+# GFX11-REAL16: v_cmpx_ngt_f16 v1.l, v2.l row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x16,0x7d,0x01,0x01,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_ngt_f16 v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x16,0x7d,0x01,0x01,0x01,0xff]
 
 0xfa,0x04,0x16,0x7d,0x01,0x0f,0x01,0xff
-# GFX11: v_cmpx_ngt_f16 v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x16,0x7d,0x01,0x0f,0x01,0xff]
+# GFX11-REAL16: v_cmpx_ngt_f16 v1.l, v2.l row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x16,0x7d,0x01,0x0f,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_ngt_f16 v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x16,0x7d,0x01,0x0f,0x01,0xff]
 
 0xfa,0x04,0x16,0x7d,0x01,0x11,0x01,0xff
-# GFX11: v_cmpx_ngt_f16 v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x16,0x7d,0x01,0x11,0x01,0xff]
+# GFX11-REAL16: v_cmpx_ngt_f16 v1.l, v2.l row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x16,0x7d,0x01,0x11,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_ngt_f16 v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x16,0x7d,0x01,0x11,0x01,0xff]
 
 0xfa,0x04,0x16,0x7d,0x01,0x1f,0x01,0xff
-# GFX11: v_cmpx_ngt_f16 v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x16,0x7d,0x01,0x1f,0x01,0xff]
+# GFX11-REAL16: v_cmpx_ngt_f16 v1.l, v2.l row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x16,0x7d,0x01,0x1f,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_ngt_f16 v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x16,0x7d,0x01,0x1f,0x01,0xff]
 
 0xfa,0x04,0x16,0x7d,0x01,0x21,0x01,0xff
-# GFX11: v_cmpx_ngt_f16 v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x16,0x7d,0x01,0x21,0x01,0xff]
+# GFX11-REAL16: v_cmpx_ngt_f16 v1.l, v2.l row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x16,0x7d,0x01,0x21,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_ngt_f16 v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x16,0x7d,0x01,0x21,0x01,0xff]
 
 0xfa,0x04,0x16,0x7d,0x01,0x2f,0x01,0xff
-# GFX11: v_cmpx_ngt_f16 v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x16,0x7d,0x01,0x2f,0x01,0xff]
+# GFX11-REAL16: v_cmpx_ngt_f16 v1.l, v2.l row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x16,0x7d,0x01,0x2f,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_ngt_f16 v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x16,0x7d,0x01,0x2f,0x01,0xff]
 
 0xfa,0x04,0x16,0x7d,0x01,0x50,0x01,0xff
-# GFX11: v_cmpx_ngt_f16 v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x16,0x7d,0x01,0x50,0x01,0xff]
+# GFX11-REAL16: v_cmpx_ngt_f16 v1.l, v2.l row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x16,0x7d,0x01,0x50,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_ngt_f16 v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x16,0x7d,0x01,0x50,0x01,0xff]
 
 0xfa,0x04,0x16,0x7d,0x01,0x5f,0x01,0x01
-# GFX11: v_cmpx_ngt_f16 v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x16,0x7d,0x01,0x5f,0x01,0x01]
+# GFX11-REAL16: v_cmpx_ngt_f16 v1.l, v2.l row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x16,0x7d,0x01,0x5f,0x01,0x01]
+# GFX11-FAKE16: v_cmpx_ngt_f16 v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x16,0x7d,0x01,0x5f,0x01,0x01]
 
 0xfa,0x04,0x16,0x7d,0x01,0x60,0x01,0x13
-# GFX11: v_cmpx_ngt_f16 v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x16,0x7d,0x01,0x60,0x01,0x13]
+# GFX11-REAL16: v_cmpx_ngt_f16 v1.l, v2.l row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x16,0x7d,0x01,0x60,0x01,0x13]
+# GFX11-FAKE16: v_cmpx_ngt_f16 v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x16,0x7d,0x01,0x60,0x01,0x13]
 
 0xfa,0xfe,0x16,0x7d,0x7f,0x6f,0xfd,0x30
-# GFX11: v_cmpx_ngt_f16 -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0x16,0x7d,0x7f,0x6f,0xfd,0x30]
+# GFX11-REAL16: v_cmpx_ngt_f16 -|v127.l|, -|v127.l| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0x16,0x7d,0x7f,0x6f,0xfd,0x30]
+# GFX11-FAKE16: v_cmpx_ngt_f16 -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0x16,0x7d,0x7f,0x6f,0xfd,0x30]
+
+0xfa,0xfe,0x16,0x7d,0x7f,0x5f,0x01,0x01
+# GFX11-REAL16: v_cmpx_ngt_f16 v127.l, v127.l row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0xfe,0x16,0x7d,0x7f,0x5f,0x01,0x01]
+# GFX11-FAKE16: v_cmpx_ngt_f16 v127, v127 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0xfe,0x16,0x7d,0x7f,0x5f,0x01,0x01]
+
+0xfa,0x04,0x17,0x7d,0x81,0x60,0x01,0x13
+# GFX11-REAL16: v_cmpx_ngt_f16 v1.h, v2.h row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x17,0x7d,0x81,0x60,0x01,0x13]
+# GFX11-FAKE16: v_cmpx_ngt_f16 v129/*Invalid register, operand has 'VGPR_32_Lo128' register class*/, v130/*Invalid register, operand has 'VGPR_32_Lo128' register class*/ row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x17,0x7d,0x81,0x60,0x01,0x13]
+
+0xfa,0xfe,0x17,0x7d,0xff,0x6f,0xfd,0x30
+# GFX11-REAL16: v_cmpx_ngt_f16 -|v127.h|, -|v127.h| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0x17,0x7d,0xff,0x6f,0xfd,0x30]
+# GFX11-FAKE16: v_cmpx_ngt_f16 -|v255/*Invalid register, operand has 'VGPR_32_Lo128' register class*/|, -|v255/*Invalid register, operand has 'VGPR_32_Lo128' register class*/| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0x17,0x7d,0xff,0x6f,0xfd,0x30]
 
 0xfa,0x04,0x36,0x7d,0x01,0x1b,0x00,0xff
 # GFX11: v_cmpx_ngt_f32 v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x36,0x7d,0x01,0x1b,0x00,0xff]
@@ -2385,46 +2619,72 @@
 # GFX11: v_cmpx_ngt_f32 -|v255|, -|v255| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0x37,0x7d,0xff,0x6f,0xfd,0x30]
 
 0xfa,0x04,0x18,0x7d,0x01,0x1b,0x00,0xff
-# GFX11: v_cmpx_nle_f16 v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x18,0x7d,0x01,0x1b,0x00,0xff]
+# GFX11-REAL16: v_cmpx_nle_f16 v1.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x18,0x7d,0x01,0x1b,0x00,0xff]
+# GFX11-FAKE16: v_cmpx_nle_f16 v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x18,0x7d,0x01,0x1b,0x00,0xff]
 
 0xfa,0x04,0x18,0x7d,0x01,0xe4,0x00,0xff
-# GFX11: v_cmpx_nle_f16 v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x18,0x7d,0x01,0xe4,0x00,0xff]
+# GFX11-REAL16: v_cmpx_nle_f16 v1.l, v2.l quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x18,0x7d,0x01,0xe4,0x00,0xff]
+# GFX11-FAKE16: v_cmpx_nle_f16 v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x18,0x7d,0x01,0xe4,0x00,0xff]
 
 0xfa,0x04,0x18,0x7d,0x01,0x40,0x01,0xff
-# GFX11: v_cmpx_nle_f16 v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x18,0x7d,0x01,0x40,0x01,0xff]
+# GFX11-REAL16: v_cmpx_nle_f16 v1.l, v2.l row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x18,0x7d,0x01,0x40,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_nle_f16 v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x18,0x7d,0x01,0x40,0x01,0xff]
 
 0xfa,0x04,0x18,0x7d,0x01,0x41,0x01,0xff
-# GFX11: v_cmpx_nle_f16 v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x18,0x7d,0x01,0x41,0x01,0xff]
+# GFX11-REAL16: v_cmpx_nle_f16 v1.l, v2.l row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x18,0x7d,0x01,0x41,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_nle_f16 v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x18,0x7d,0x01,0x41,0x01,0xff]
 
 0xfa,0x04,0x18,0x7d,0x01,0x01,0x01,0xff
-# GFX11: v_cmpx_nle_f16 v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x18,0x7d,0x01,0x01,0x01,0xff]
+# GFX11-REAL16: v_cmpx_nle_f16 v1.l, v2.l row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x18,0x7d,0x01,0x01,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_nle_f16 v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x18,0x7d,0x01,0x01,0x01,0xff]
 
 0xfa,0x04,0x18,0x7d,0x01,0x0f,0x01,0xff
-# GFX11: v_cmpx_nle_f16 v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x18,0x7d,0x01,0x0f,0x01,0xff]
+# GFX11-REAL16: v_cmpx_nle_f16 v1.l, v2.l row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x18,0x7d,0x01,0x0f,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_nle_f16 v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x18,0x7d,0x01,0x0f,0x01,0xff]
 
 0xfa,0x04,0x18,0x7d,0x01,0x11,0x01,0xff
-# GFX11: v_cmpx_nle_f16 v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x18,0x7d,0x01,0x11,0x01,0xff]
+# GFX11-REAL16: v_cmpx_nle_f16 v1.l, v2.l row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x18,0x7d,0x01,0x11,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_nle_f16 v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x18,0x7d,0x01,0x11,0x01,0xff]
 
 0xfa,0x04,0x18,0x7d,0x01,0x1f,0x01,0xff
-# GFX11: v_cmpx_nle_f16 v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x18,0x7d,0x01,0x1f,0x01,0xff]
+# GFX11-REAL16: v_cmpx_nle_f16 v1.l, v2.l row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x18,0x7d,0x01,0x1f,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_nle_f16 v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x18,0x7d,0x01,0x1f,0x01,0xff]
 
 0xfa,0x04,0x18,0x7d,0x01,0x21,0x01,0xff
-# GFX11: v_cmpx_nle_f16 v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x18,0x7d,0x01,0x21,0x01,0xff]
+# GFX11-REAL16: v_cmpx_nle_f16 v1.l, v2.l row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x18,0x7d,0x01,0x21,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_nle_f16 v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x18,0x7d,0x01,0x21,0x01,0xff]
 
 0xfa,0x04,0x18,0x7d,0x01,0x2f,0x01,0xff
-# GFX11: v_cmpx_nle_f16 v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x18,0x7d,0x01,0x2f,0x01,0xff]
+# GFX11-REAL16: v_cmpx_nle_f16 v1.l, v2.l row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x18,0x7d,0x01,0x2f,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_nle_f16 v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x18,0x7d,0x01,0x2f,0x01,0xff]
 
 0xfa,0x04,0x18,0x7d,0x01,0x50,0x01,0xff
-# GFX11: v_cmpx_nle_f16 v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x18,0x7d,0x01,0x50,0x01,0xff]
+# GFX11-REAL16: v_cmpx_nle_f16 v1.l, v2.l row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x18,0x7d,0x01,0x50,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_nle_f16 v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x18,0x7d,0x01,0x50,0x01,0xff]
 
 0xfa,0x04,0x18,0x7d,0x01,0x5f,0x01,0x01
-# GFX11: v_cmpx_nle_f16 v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x18,0x7d,0x01,0x5f,0x01,0x01]
+# GFX11-REAL16: v_cmpx_nle_f16 v1.l, v2.l row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x18,0x7d,0x01,0x5f,0x01,0x01]
+# GFX11-FAKE16: v_cmpx_nle_f16 v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x18,0x7d,0x01,0x5f,0x01,0x01]
 
 0xfa,0x04,0x18,0x7d,0x01,0x60,0x01,0x13
-# GFX11: v_cmpx_nle_f16 v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x18,0x7d,0x01,0x60,0x01,0x13]
+# GFX11-REAL16: v_cmpx_nle_f16 v1.l, v2.l row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x18,0x7d,0x01,0x60,0x01,0x13]
+# GFX11-FAKE16: v_cmpx_nle_f16 v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x18,0x7d,0x01,0x60,0x01,0x13]
 
 0xfa,0xfe,0x18,0x7d,0x7f,0x6f,0xfd,0x30
-# GFX11: v_cmpx_nle_f16 -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0x18,0x7d,0x7f,0x6f,0xfd,0x30]
+# GFX11-REAL16: v_cmpx_nle_f16 -|v127.l|, -|v127.l| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0x18,0x7d,0x7f,0x6f,0xfd,0x30]
+# GFX11-FAKE16: v_cmpx_nle_f16 -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0x18,0x7d,0x7f,0x6f,0xfd,0x30]
+
+0xfa,0xfe,0x18,0x7d,0x7f,0x5f,0x01,0x01
+# GFX11-REAL16: v_cmpx_nle_f16 v127.l, v127.l row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0xfe,0x18,0x7d,0x7f,0x5f,0x01,0x01]
+# GFX11-FAKE16: v_cmpx_nle_f16 v127, v127 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0xfe,0x18,0x7d,0x7f,0x5f,0x01,0x01]
+
+0xfa,0x04,0x19,0x7d,0x81,0x60,0x01,0x13
+# GFX11-REAL16: v_cmpx_nle_f16 v1.h, v2.h row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x19,0x7d,0x81,0x60,0x01,0x13]
+# GFX11-FAKE16: v_cmpx_nle_f16 v129/*Invalid register, operand has 'VGPR_32_Lo128' register class*/, v130/*Invalid register, operand has 'VGPR_32_Lo128' register class*/ row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x19,0x7d,0x81,0x60,0x01,0x13]
+
+0xfa,0xfe,0x19,0x7d,0xff,0x6f,0xfd,0x30
+# GFX11-REAL16: v_cmpx_nle_f16 -|v127.h|, -|v127.h| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0x19,0x7d,0xff,0x6f,0xfd,0x30]
+# GFX11-FAKE16: v_cmpx_nle_f16 -|v255/*Invalid register, operand has 'VGPR_32_Lo128' register class*/|, -|v255/*Invalid register, operand has 'VGPR_32_Lo128' register class*/| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0x19,0x7d,0xff,0x6f,0xfd,0x30]
 
 0xfa,0x04,0x38,0x7d,0x01,0x1b,0x00,0xff
 # GFX11: v_cmpx_nle_f32 v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x38,0x7d,0x01,0x1b,0x00,0xff]
@@ -2469,46 +2729,72 @@
 # GFX11: v_cmpx_nle_f32 -|v255|, -|v255| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0x39,0x7d,0xff,0x6f,0xfd,0x30]
 
 0xfa,0x04,0x14,0x7d,0x01,0x1b,0x00,0xff
-# GFX11: v_cmpx_nlg_f16 v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x14,0x7d,0x01,0x1b,0x00,0xff]
+# GFX11-REAL16: v_cmpx_nlg_f16 v1.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x14,0x7d,0x01,0x1b,0x00,0xff]
+# GFX11-FAKE16: v_cmpx_nlg_f16 v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x14,0x7d,0x01,0x1b,0x00,0xff]
 
 0xfa,0x04,0x14,0x7d,0x01,0xe4,0x00,0xff
-# GFX11: v_cmpx_nlg_f16 v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x14,0x7d,0x01,0xe4,0x00,0xff]
+# GFX11-REAL16: v_cmpx_nlg_f16 v1.l, v2.l quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x14,0x7d,0x01,0xe4,0x00,0xff]
+# GFX11-FAKE16: v_cmpx_nlg_f16 v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x14,0x7d,0x01,0xe4,0x00,0xff]
 
 0xfa,0x04,0x14,0x7d,0x01,0x40,0x01,0xff
-# GFX11: v_cmpx_nlg_f16 v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x14,0x7d,0x01,0x40,0x01,0xff]
+# GFX11-REAL16: v_cmpx_nlg_f16 v1.l, v2.l row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x14,0x7d,0x01,0x40,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_nlg_f16 v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x14,0x7d,0x01,0x40,0x01,0xff]
 
 0xfa,0x04,0x14,0x7d,0x01,0x41,0x01,0xff
-# GFX11: v_cmpx_nlg_f16 v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x14,0x7d,0x01,0x41,0x01,0xff]
+# GFX11-REAL16: v_cmpx_nlg_f16 v1.l, v2.l row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x14,0x7d,0x01,0x41,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_nlg_f16 v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x14,0x7d,0x01,0x41,0x01,0xff]
 
 0xfa,0x04,0x14,0x7d,0x01,0x01,0x01,0xff
-# GFX11: v_cmpx_nlg_f16 v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x14,0x7d,0x01,0x01,0x01,0xff]
+# GFX11-REAL16: v_cmpx_nlg_f16 v1.l, v2.l row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x14,0x7d,0x01,0x01,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_nlg_f16 v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x14,0x7d,0x01,0x01,0x01,0xff]
 
 0xfa,0x04,0x14,0x7d,0x01,0x0f,0x01,0xff
-# GFX11: v_cmpx_nlg_f16 v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x14,0x7d,0x01,0x0f,0x01,0xff]
+# GFX11-REAL16: v_cmpx_nlg_f16 v1.l, v2.l row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x14,0x7d,0x01,0x0f,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_nlg_f16 v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x14,0x7d,0x01,0x0f,0x01,0xff]
 
 0xfa,0x04,0x14,0x7d,0x01,0x11,0x01,0xff
-# GFX11: v_cmpx_nlg_f16 v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x14,0x7d,0x01,0x11,0x01,0xff]
+# GFX11-REAL16: v_cmpx_nlg_f16 v1.l, v2.l row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x14,0x7d,0x01,0x11,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_nlg_f16 v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x14,0x7d,0x01,0x11,0x01,0xff]
 
 0xfa,0x04,0x14,0x7d,0x01,0x1f,0x01,0xff
-# GFX11: v_cmpx_nlg_f16 v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x14,0x7d,0x01,0x1f,0x01,0xff]
+# GFX11-REAL16: v_cmpx_nlg_f16 v1.l, v2.l row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x14,0x7d,0x01,0x1f,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_nlg_f16 v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x14,0x7d,0x01,0x1f,0x01,0xff]
 
 0xfa,0x04,0x14,0x7d,0x01,0x21,0x01,0xff
-# GFX11: v_cmpx_nlg_f16 v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x14,0x7d,0x01,0x21,0x01,0xff]
+# GFX11-REAL16: v_cmpx_nlg_f16 v1.l, v2.l row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x14,0x7d,0x01,0x21,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_nlg_f16 v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x14,0x7d,0x01,0x21,0x01,0xff]
 
 0xfa,0x04,0x14,0x7d,0x01,0x2f,0x01,0xff
-# GFX11: v_cmpx_nlg_f16 v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x14,0x7d,0x01,0x2f,0x01,0xff]
+# GFX11-REAL16: v_cmpx_nlg_f16 v1.l, v2.l row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x14,0x7d,0x01,0x2f,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_nlg_f16 v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x14,0x7d,0x01,0x2f,0x01,0xff]
 
 0xfa,0x04,0x14,0x7d,0x01,0x50,0x01,0xff
-# GFX11: v_cmpx_nlg_f16 v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x14,0x7d,0x01,0x50,0x01,0xff]
+# GFX11-REAL16: v_cmpx_nlg_f16 v1.l, v2.l row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x14,0x7d,0x01,0x50,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_nlg_f16 v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x14,0x7d,0x01,0x50,0x01,0xff]
 
 0xfa,0x04,0x14,0x7d,0x01,0x5f,0x01,0x01
-# GFX11: v_cmpx_nlg_f16 v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x14,0x7d,0x01,0x5f,0x01,0x01]
+# GFX11-REAL16: v_cmpx_nlg_f16 v1.l, v2.l row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x14,0x7d,0x01,0x5f,0x01,0x01]
+# GFX11-FAKE16: v_cmpx_nlg_f16 v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x14,0x7d,0x01,0x5f,0x01,0x01]
 
 0xfa,0x04,0x14,0x7d,0x01,0x60,0x01,0x13
-# GFX11: v_cmpx_nlg_f16 v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x14,0x7d,0x01,0x60,0x01,0x13]
+# GFX11-REAL16: v_cmpx_nlg_f16 v1.l, v2.l row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x14,0x7d,0x01,0x60,0x01,0x13]
+# GFX11-FAKE16: v_cmpx_nlg_f16 v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x14,0x7d,0x01,0x60,0x01,0x13]
 
 0xfa,0xfe,0x14,0x7d,0x7f,0x6f,0xfd,0x30
-# GFX11: v_cmpx_nlg_f16 -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0x14,0x7d,0x7f,0x6f,0xfd,0x30]
+# GFX11-REAL16: v_cmpx_nlg_f16 -|v127.l|, -|v127.l| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0x14,0x7d,0x7f,0x6f,0xfd,0x30]
+# GFX11-FAKE16: v_cmpx_nlg_f16 -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0x14,0x7d,0x7f,0x6f,0xfd,0x30]
+
+0xfa,0xfe,0x14,0x7d,0x7f,0x5f,0x01,0x01
+# GFX11-REAL16: v_cmpx_nlg_f16 v127.l, v127.l row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0xfe,0x14,0x7d,0x7f,0x5f,0x01,0x01]
+# GFX11-FAKE16: v_cmpx_nlg_f16 v127, v127 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0xfe,0x14,0x7d,0x7f,0x5f,0x01,0x01]
+
+0xfa,0x04,0x15,0x7d,0x81,0x60,0x01,0x13
+# GFX11-REAL16: v_cmpx_nlg_f16 v1.h, v2.h row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x15,0x7d,0x81,0x60,0x01,0x13]
+# GFX11-FAKE16: v_cmpx_nlg_f16 v129/*Invalid register, operand has 'VGPR_32_Lo128' register class*/, v130/*Invalid register, operand has 'VGPR_32_Lo128' register class*/ row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x15,0x7d,0x81,0x60,0x01,0x13]
+
+0xfa,0xfe,0x15,0x7d,0xff,0x6f,0xfd,0x30
+# GFX11-REAL16: v_cmpx_nlg_f16 -|v127.h|, -|v127.h| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0x15,0x7d,0xff,0x6f,0xfd,0x30]
+# GFX11-FAKE16: v_cmpx_nlg_f16 -|v255/*Invalid register, operand has 'VGPR_32_Lo128' register class*/|, -|v255/*Invalid register, operand has 'VGPR_32_Lo128' register class*/| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0x15,0x7d,0xff,0x6f,0xfd,0x30]
 
 0xfa,0x04,0x34,0x7d,0x01,0x1b,0x00,0xff
 # GFX11: v_cmpx_nlg_f32 v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x34,0x7d,0x01,0x1b,0x00,0xff]
@@ -2553,46 +2839,72 @@
 # GFX11: v_cmpx_nlg_f32 -|v255|, -|v255| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0x35,0x7d,0xff,0x6f,0xfd,0x30]
 
 0xfa,0x04,0x1c,0x7d,0x01,0x1b,0x00,0xff
-# GFX11: v_cmpx_nlt_f16 v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1c,0x7d,0x01,0x1b,0x00,0xff]
+# GFX11-REAL16: v_cmpx_nlt_f16 v1.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1c,0x7d,0x01,0x1b,0x00,0xff]
+# GFX11-FAKE16: v_cmpx_nlt_f16 v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1c,0x7d,0x01,0x1b,0x00,0xff]
 
 0xfa,0x04,0x1c,0x7d,0x01,0xe4,0x00,0xff
-# GFX11: v_cmpx_nlt_f16 v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1c,0x7d,0x01,0xe4,0x00,0xff]
+# GFX11-REAL16: v_cmpx_nlt_f16 v1.l, v2.l quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1c,0x7d,0x01,0xe4,0x00,0xff]
+# GFX11-FAKE16: v_cmpx_nlt_f16 v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1c,0x7d,0x01,0xe4,0x00,0xff]
 
 0xfa,0x04,0x1c,0x7d,0x01,0x40,0x01,0xff
-# GFX11: v_cmpx_nlt_f16 v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1c,0x7d,0x01,0x40,0x01,0xff]
+# GFX11-REAL16: v_cmpx_nlt_f16 v1.l, v2.l row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1c,0x7d,0x01,0x40,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_nlt_f16 v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1c,0x7d,0x01,0x40,0x01,0xff]
 
 0xfa,0x04,0x1c,0x7d,0x01,0x41,0x01,0xff
-# GFX11: v_cmpx_nlt_f16 v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1c,0x7d,0x01,0x41,0x01,0xff]
+# GFX11-REAL16: v_cmpx_nlt_f16 v1.l, v2.l row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1c,0x7d,0x01,0x41,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_nlt_f16 v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1c,0x7d,0x01,0x41,0x01,0xff]
 
 0xfa,0x04,0x1c,0x7d,0x01,0x01,0x01,0xff
-# GFX11: v_cmpx_nlt_f16 v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1c,0x7d,0x01,0x01,0x01,0xff]
+# GFX11-REAL16: v_cmpx_nlt_f16 v1.l, v2.l row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1c,0x7d,0x01,0x01,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_nlt_f16 v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1c,0x7d,0x01,0x01,0x01,0xff]
 
 0xfa,0x04,0x1c,0x7d,0x01,0x0f,0x01,0xff
-# GFX11: v_cmpx_nlt_f16 v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1c,0x7d,0x01,0x0f,0x01,0xff]
+# GFX11-REAL16: v_cmpx_nlt_f16 v1.l, v2.l row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1c,0x7d,0x01,0x0f,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_nlt_f16 v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1c,0x7d,0x01,0x0f,0x01,0xff]
 
 0xfa,0x04,0x1c,0x7d,0x01,0x11,0x01,0xff
-# GFX11: v_cmpx_nlt_f16 v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1c,0x7d,0x01,0x11,0x01,0xff]
+# GFX11-REAL16: v_cmpx_nlt_f16 v1.l, v2.l row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1c,0x7d,0x01,0x11,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_nlt_f16 v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1c,0x7d,0x01,0x11,0x01,0xff]
 
 0xfa,0x04,0x1c,0x7d,0x01,0x1f,0x01,0xff
-# GFX11: v_cmpx_nlt_f16 v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1c,0x7d,0x01,0x1f,0x01,0xff]
+# GFX11-REAL16: v_cmpx_nlt_f16 v1.l, v2.l row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1c,0x7d,0x01,0x1f,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_nlt_f16 v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1c,0x7d,0x01,0x1f,0x01,0xff]
 
 0xfa,0x04,0x1c,0x7d,0x01,0x21,0x01,0xff
-# GFX11: v_cmpx_nlt_f16 v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1c,0x7d,0x01,0x21,0x01,0xff]
+# GFX11-REAL16: v_cmpx_nlt_f16 v1.l, v2.l row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1c,0x7d,0x01,0x21,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_nlt_f16 v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1c,0x7d,0x01,0x21,0x01,0xff]
 
 0xfa,0x04,0x1c,0x7d,0x01,0x2f,0x01,0xff
-# GFX11: v_cmpx_nlt_f16 v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1c,0x7d,0x01,0x2f,0x01,0xff]
+# GFX11-REAL16: v_cmpx_nlt_f16 v1.l, v2.l row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1c,0x7d,0x01,0x2f,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_nlt_f16 v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1c,0x7d,0x01,0x2f,0x01,0xff]
 
 0xfa,0x04,0x1c,0x7d,0x01,0x50,0x01,0xff
-# GFX11: v_cmpx_nlt_f16 v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1c,0x7d,0x01,0x50,0x01,0xff]
+# GFX11-REAL16: v_cmpx_nlt_f16 v1.l, v2.l row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1c,0x7d,0x01,0x50,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_nlt_f16 v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1c,0x7d,0x01,0x50,0x01,0xff]
 
 0xfa,0x04,0x1c,0x7d,0x01,0x5f,0x01,0x01
-# GFX11: v_cmpx_nlt_f16 v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x1c,0x7d,0x01,0x5f,0x01,0x01]
+# GFX11-REAL16: v_cmpx_nlt_f16 v1.l, v2.l row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x1c,0x7d,0x01,0x5f,0x01,0x01]
+# GFX11-FAKE16: v_cmpx_nlt_f16 v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x1c,0x7d,0x01,0x5f,0x01,0x01]
 
 0xfa,0x04,0x1c,0x7d,0x01,0x60,0x01,0x13
-# GFX11: v_cmpx_nlt_f16 v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x1c,0x7d,0x01,0x60,0x01,0x13]
+# GFX11-REAL16: v_cmpx_nlt_f16 v1.l, v2.l row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x1c,0x7d,0x01,0x60,0x01,0x13]
+# GFX11-FAKE16: v_cmpx_nlt_f16 v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x1c,0x7d,0x01,0x60,0x01,0x13]
 
 0xfa,0xfe,0x1c,0x7d,0x7f,0x6f,0xfd,0x30
-# GFX11: v_cmpx_nlt_f16 -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0x1c,0x7d,0x7f,0x6f,0xfd,0x30]
+# GFX11-REAL16: v_cmpx_nlt_f16 -|v127.l|, -|v127.l| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0x1c,0x7d,0x7f,0x6f,0xfd,0x30]
+# GFX11-FAKE16: v_cmpx_nlt_f16 -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0x1c,0x7d,0x7f,0x6f,0xfd,0x30]
+
+0xfa,0xfe,0x1c,0x7d,0x7f,0x5f,0x01,0x01
+# GFX11-REAL16: v_cmpx_nlt_f16 v127.l, v127.l row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0xfe,0x1c,0x7d,0x7f,0x5f,0x01,0x01]
+# GFX11-FAKE16: v_cmpx_nlt_f16 v127, v127 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0xfe,0x1c,0x7d,0x7f,0x5f,0x01,0x01]
+
+0xfa,0x04,0x1d,0x7d,0x81,0x60,0x01,0x13
+# GFX11-REAL16: v_cmpx_nlt_f16 v1.h, v2.h row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x1d,0x7d,0x81,0x60,0x01,0x13]
+# GFX11-FAKE16: v_cmpx_nlt_f16 v129/*Invalid register, operand has 'VGPR_32_Lo128' register class*/, v130/*Invalid register, operand has 'VGPR_32_Lo128' register class*/ row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x1d,0x7d,0x81,0x60,0x01,0x13]
+
+0xfa,0xfe,0x1d,0x7d,0xff,0x6f,0xfd,0x30
+# GFX11-REAL16: v_cmpx_nlt_f16 -|v127.h|, -|v127.h| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0x1d,0x7d,0xff,0x6f,0xfd,0x30]
+# GFX11-FAKE16: v_cmpx_nlt_f16 -|v255/*Invalid register, operand has 'VGPR_32_Lo128' register class*/|, -|v255/*Invalid register, operand has 'VGPR_32_Lo128' register class*/| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0x1d,0x7d,0xff,0x6f,0xfd,0x30]
 
 0xfa,0x04,0x3c,0x7d,0x01,0x1b,0x00,0xff
 # GFX11: v_cmpx_nlt_f32 v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x3c,0x7d,0x01,0x1b,0x00,0xff]
@@ -2637,46 +2949,72 @@
 # GFX11: v_cmpx_nlt_f32 -|v255|, -|v255| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0x3d,0x7d,0xff,0x6f,0xfd,0x30]
 
 0xfa,0x04,0x0e,0x7d,0x01,0x1b,0x00,0xff
-# GFX11: v_cmpx_o_f16 v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0e,0x7d,0x01,0x1b,0x00,0xff]
+# GFX11-REAL16: v_cmpx_o_f16 v1.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0e,0x7d,0x01,0x1b,0x00,0xff]
+# GFX11-FAKE16: v_cmpx_o_f16 v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0e,0x7d,0x01,0x1b,0x00,0xff]
 
 0xfa,0x04,0x0e,0x7d,0x01,0xe4,0x00,0xff
-# GFX11: v_cmpx_o_f16 v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0e,0x7d,0x01,0xe4,0x00,0xff]
+# GFX11-REAL16: v_cmpx_o_f16 v1.l, v2.l quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0e,0x7d,0x01,0xe4,0x00,0xff]
+# GFX11-FAKE16: v_cmpx_o_f16 v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0e,0x7d,0x01,0xe4,0x00,0xff]
 
 0xfa,0x04,0x0e,0x7d,0x01,0x40,0x01,0xff
-# GFX11: v_cmpx_o_f16 v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0e,0x7d,0x01,0x40,0x01,0xff]
+# GFX11-REAL16: v_cmpx_o_f16 v1.l, v2.l row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0e,0x7d,0x01,0x40,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_o_f16 v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0e,0x7d,0x01,0x40,0x01,0xff]
 
 0xfa,0x04,0x0e,0x7d,0x01,0x41,0x01,0xff
-# GFX11: v_cmpx_o_f16 v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0e,0x7d,0x01,0x41,0x01,0xff]
+# GFX11-REAL16: v_cmpx_o_f16 v1.l, v2.l row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0e,0x7d,0x01,0x41,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_o_f16 v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0e,0x7d,0x01,0x41,0x01,0xff]
 
 0xfa,0x04,0x0e,0x7d,0x01,0x01,0x01,0xff
-# GFX11: v_cmpx_o_f16 v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0e,0x7d,0x01,0x01,0x01,0xff]
+# GFX11-REAL16: v_cmpx_o_f16 v1.l, v2.l row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0e,0x7d,0x01,0x01,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_o_f16 v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0e,0x7d,0x01,0x01,0x01,0xff]
 
 0xfa,0x04,0x0e,0x7d,0x01,0x0f,0x01,0xff
-# GFX11: v_cmpx_o_f16 v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0e,0x7d,0x01,0x0f,0x01,0xff]
+# GFX11-REAL16: v_cmpx_o_f16 v1.l, v2.l row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0e,0x7d,0x01,0x0f,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_o_f16 v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0e,0x7d,0x01,0x0f,0x01,0xff]
 
 0xfa,0x04,0x0e,0x7d,0x01,0x11,0x01,0xff
-# GFX11: v_cmpx_o_f16 v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0e,0x7d,0x01,0x11,0x01,0xff]
+# GFX11-REAL16: v_cmpx_o_f16 v1.l, v2.l row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0e,0x7d,0x01,0x11,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_o_f16 v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0e,0x7d,0x01,0x11,0x01,0xff]
 
 0xfa,0x04,0x0e,0x7d,0x01,0x1f,0x01,0xff
-# GFX11: v_cmpx_o_f16 v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0e,0x7d,0x01,0x1f,0x01,0xff]
+# GFX11-REAL16: v_cmpx_o_f16 v1.l, v2.l row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0e,0x7d,0x01,0x1f,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_o_f16 v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0e,0x7d,0x01,0x1f,0x01,0xff]
 
 0xfa,0x04,0x0e,0x7d,0x01,0x21,0x01,0xff
-# GFX11: v_cmpx_o_f16 v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0e,0x7d,0x01,0x21,0x01,0xff]
+# GFX11-REAL16: v_cmpx_o_f16 v1.l, v2.l row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0e,0x7d,0x01,0x21,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_o_f16 v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0e,0x7d,0x01,0x21,0x01,0xff]
 
 0xfa,0x04,0x0e,0x7d,0x01,0x2f,0x01,0xff
-# GFX11: v_cmpx_o_f16 v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0e,0x7d,0x01,0x2f,0x01,0xff]
+# GFX11-REAL16: v_cmpx_o_f16 v1.l, v2.l row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0e,0x7d,0x01,0x2f,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_o_f16 v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0e,0x7d,0x01,0x2f,0x01,0xff]
 
 0xfa,0x04,0x0e,0x7d,0x01,0x50,0x01,0xff
-# GFX11: v_cmpx_o_f16 v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0e,0x7d,0x01,0x50,0x01,0xff]
+# GFX11-REAL16: v_cmpx_o_f16 v1.l, v2.l row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0e,0x7d,0x01,0x50,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_o_f16 v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0e,0x7d,0x01,0x50,0x01,0xff]
 
 0xfa,0x04,0x0e,0x7d,0x01,0x5f,0x01,0x01
-# GFX11: v_cmpx_o_f16 v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x0e,0x7d,0x01,0x5f,0x01,0x01]
+# GFX11-REAL16: v_cmpx_o_f16 v1.l, v2.l row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x0e,0x7d,0x01,0x5f,0x01,0x01]
+# GFX11-FAKE16: v_cmpx_o_f16 v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x0e,0x7d,0x01,0x5f,0x01,0x01]
 
 0xfa,0x04,0x0e,0x7d,0x01,0x60,0x01,0x13
-# GFX11: v_cmpx_o_f16 v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x0e,0x7d,0x01,0x60,0x01,0x13]
+# GFX11-REAL16: v_cmpx_o_f16 v1.l, v2.l row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x0e,0x7d,0x01,0x60,0x01,0x13]
+# GFX11-FAKE16: v_cmpx_o_f16 v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x0e,0x7d,0x01,0x60,0x01,0x13]
 
 0xfa,0xfe,0x0e,0x7d,0x7f,0x6f,0xfd,0x30
-# GFX11: v_cmpx_o_f16 -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0x0e,0x7d,0x7f,0x6f,0xfd,0x30]
+# GFX11-REAL16: v_cmpx_o_f16 -|v127.l|, -|v127.l| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0x0e,0x7d,0x7f,0x6f,0xfd,0x30]
+# GFX11-FAKE16: v_cmpx_o_f16 -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0x0e,0x7d,0x7f,0x6f,0xfd,0x30]
+
+0xfa,0xfe,0x0e,0x7d,0x7f,0x5f,0x01,0x01
+# GFX11-REAL16: v_cmpx_o_f16 v127.l, v127.l row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0xfe,0x0e,0x7d,0x7f,0x5f,0x01,0x01]
+# GFX11-FAKE16: v_cmpx_o_f16 v127, v127 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0xfe,0x0e,0x7d,0x7f,0x5f,0x01,0x01]
+
+0xfa,0x04,0x0f,0x7d,0x81,0x60,0x01,0x13
+# GFX11-REAL16: v_cmpx_o_f16 v1.h, v2.h row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x0f,0x7d,0x81,0x60,0x01,0x13]
+# GFX11-FAKE16: v_cmpx_o_f16 v129/*Invalid register, operand has 'VGPR_32_Lo128' register class*/, v130/*Invalid register, operand has 'VGPR_32_Lo128' register class*/ row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x0f,0x7d,0x81,0x60,0x01,0x13]
+
+0xfa,0xfe,0x0f,0x7d,0xff,0x6f,0xfd,0x30
+# GFX11-REAL16: v_cmpx_o_f16 -|v127.h|, -|v127.h| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0x0f,0x7d,0xff,0x6f,0xfd,0x30]
+# GFX11-FAKE16: v_cmpx_o_f16 -|v255/*Invalid register, operand has 'VGPR_32_Lo128' register class*/|, -|v255/*Invalid register, operand has 'VGPR_32_Lo128' register class*/| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0x0f,0x7d,0xff,0x6f,0xfd,0x30]
 
 0xfa,0x04,0x2e,0x7d,0x01,0x1b,0x00,0xff
 # GFX11: v_cmpx_o_f32 v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x2e,0x7d,0x01,0x1b,0x00,0xff]
@@ -2721,46 +3059,72 @@
 # GFX11: v_cmpx_o_f32 -|v255|, -|v255| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0x2f,0x7d,0xff,0x6f,0xfd,0x30]
 
 0xfa,0x04,0x1e,0x7d,0x01,0x1b,0x00,0xff
-# GFX11: v_cmpx_t_f16 v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1e,0x7d,0x01,0x1b,0x00,0xff]
+# GFX11-REAL16: v_cmpx_t_f16 v1.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1e,0x7d,0x01,0x1b,0x00,0xff]
+# GFX11-FAKE16: v_cmpx_t_f16 v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1e,0x7d,0x01,0x1b,0x00,0xff]
 
 0xfa,0x04,0x1e,0x7d,0x01,0xe4,0x00,0xff
-# GFX11: v_cmpx_t_f16 v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1e,0x7d,0x01,0xe4,0x00,0xff]
+# GFX11-REAL16: v_cmpx_t_f16 v1.l, v2.l quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1e,0x7d,0x01,0xe4,0x00,0xff]
+# GFX11-FAKE16: v_cmpx_t_f16 v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1e,0x7d,0x01,0xe4,0x00,0xff]
 
 0xfa,0x04,0x1e,0x7d,0x01,0x40,0x01,0xff
-# GFX11: v_cmpx_t_f16 v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1e,0x7d,0x01,0x40,0x01,0xff]
+# GFX11-REAL16: v_cmpx_t_f16 v1.l, v2.l row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1e,0x7d,0x01,0x40,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_t_f16 v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1e,0x7d,0x01,0x40,0x01,0xff]
 
 0xfa,0x04,0x1e,0x7d,0x01,0x41,0x01,0xff
-# GFX11: v_cmpx_t_f16 v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1e,0x7d,0x01,0x41,0x01,0xff]
+# GFX11-REAL16: v_cmpx_t_f16 v1.l, v2.l row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1e,0x7d,0x01,0x41,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_t_f16 v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1e,0x7d,0x01,0x41,0x01,0xff]
 
 0xfa,0x04,0x1e,0x7d,0x01,0x01,0x01,0xff
-# GFX11: v_cmpx_t_f16 v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1e,0x7d,0x01,0x01,0x01,0xff]
+# GFX11-REAL16: v_cmpx_t_f16 v1.l, v2.l row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1e,0x7d,0x01,0x01,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_t_f16 v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1e,0x7d,0x01,0x01,0x01,0xff]
 
 0xfa,0x04,0x1e,0x7d,0x01,0x0f,0x01,0xff
-# GFX11: v_cmpx_t_f16 v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1e,0x7d,0x01,0x0f,0x01,0xff]
+# GFX11-REAL16: v_cmpx_t_f16 v1.l, v2.l row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1e,0x7d,0x01,0x0f,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_t_f16 v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1e,0x7d,0x01,0x0f,0x01,0xff]
 
 0xfa,0x04,0x1e,0x7d,0x01,0x11,0x01,0xff
-# GFX11: v_cmpx_t_f16 v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1e,0x7d,0x01,0x11,0x01,0xff]
+# GFX11-REAL16: v_cmpx_t_f16 v1.l, v2.l row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1e,0x7d,0x01,0x11,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_t_f16 v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1e,0x7d,0x01,0x11,0x01,0xff]
 
 0xfa,0x04,0x1e,0x7d,0x01,0x1f,0x01,0xff
-# GFX11: v_cmpx_t_f16 v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1e,0x7d,0x01,0x1f,0x01,0xff]
+# GFX11-REAL16: v_cmpx_t_f16 v1.l, v2.l row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1e,0x7d,0x01,0x1f,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_t_f16 v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1e,0x7d,0x01,0x1f,0x01,0xff]
 
 0xfa,0x04,0x1e,0x7d,0x01,0x21,0x01,0xff
-# GFX11: v_cmpx_t_f16 v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1e,0x7d,0x01,0x21,0x01,0xff]
+# GFX11-REAL16: v_cmpx_t_f16 v1.l, v2.l row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1e,0x7d,0x01,0x21,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_t_f16 v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1e,0x7d,0x01,0x21,0x01,0xff]
 
 0xfa,0x04,0x1e,0x7d,0x01,0x2f,0x01,0xff
-# GFX11: v_cmpx_t_f16 v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1e,0x7d,0x01,0x2f,0x01,0xff]
+# GFX11-REAL16: v_cmpx_t_f16 v1.l, v2.l row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1e,0x7d,0x01,0x2f,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_t_f16 v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1e,0x7d,0x01,0x2f,0x01,0xff]
 
 0xfa,0x04,0x1e,0x7d,0x01,0x50,0x01,0xff
-# GFX11: v_cmpx_t_f16 v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1e,0x7d,0x01,0x50,0x01,0xff]
+# GFX11-REAL16: v_cmpx_t_f16 v1.l, v2.l row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1e,0x7d,0x01,0x50,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_t_f16 v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1e,0x7d,0x01,0x50,0x01,0xff]
 
 0xfa,0x04,0x1e,0x7d,0x01,0x5f,0x01,0x01
-# GFX11: v_cmpx_t_f16 v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x1e,0x7d,0x01,0x5f,0x01,0x01]
+# GFX11-REAL16: v_cmpx_t_f16 v1.l, v2.l row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x1e,0x7d,0x01,0x5f,0x01,0x01]
+# GFX11-FAKE16: v_cmpx_t_f16 v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x1e,0x7d,0x01,0x5f,0x01,0x01]
 
 0xfa,0x04,0x1e,0x7d,0x01,0x60,0x01,0x13
-# GFX11: v_cmpx_t_f16 v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x1e,0x7d,0x01,0x60,0x01,0x13]
+# GFX11-REAL16: v_cmpx_t_f16 v1.l, v2.l row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x1e,0x7d,0x01,0x60,0x01,0x13]
+# GFX11-FAKE16: v_cmpx_t_f16 v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x1e,0x7d,0x01,0x60,0x01,0x13]
 
 0xfa,0xfe,0x1e,0x7d,0x7f,0x6f,0xfd,0x30
-# GFX11: v_cmpx_t_f16 -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0x1e,0x7d,0x7f,0x6f,0xfd,0x30]
+# GFX11-REAL16: v_cmpx_t_f16 -|v127.l|, -|v127.l| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0x1e,0x7d,0x7f,0x6f,0xfd,0x30]
+# GFX11-FAKE16: v_cmpx_t_f16 -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0x1e,0x7d,0x7f,0x6f,0xfd,0x30]
+
+0xfa,0xfe,0x1e,0x7d,0x7f,0x5f,0x01,0x01
+# GFX11-REAL16: v_cmpx_t_f16 v127.l, v127.l row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0xfe,0x1e,0x7d,0x7f,0x5f,0x01,0x01]
+# GFX11-FAKE16: v_cmpx_t_f16 v127, v127 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0xfe,0x1e,0x7d,0x7f,0x5f,0x01,0x01]
+
+0xfa,0x04,0x1f,0x7d,0x81,0x60,0x01,0x13
+# GFX11-REAL16: v_cmpx_t_f16 v1.h, v2.h row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x1f,0x7d,0x81,0x60,0x01,0x13]
+# GFX11-FAKE16: v_cmpx_t_f16 v129/*Invalid register, operand has 'VGPR_32_Lo128' register class*/, v130/*Invalid register, operand has 'VGPR_32_Lo128' register class*/ row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x1f,0x7d,0x81,0x60,0x01,0x13]
+
+0xfa,0xfe,0x1f,0x7d,0xff,0x6f,0xfd,0x30
+# GFX11-REAL16: v_cmpx_t_f16 -|v127.h|, -|v127.h| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0x1f,0x7d,0xff,0x6f,0xfd,0x30]
+# GFX11-FAKE16: v_cmpx_t_f16 -|v255/*Invalid register, operand has 'VGPR_32_Lo128' register class*/|, -|v255/*Invalid register, operand has 'VGPR_32_Lo128' register class*/| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0x1f,0x7d,0xff,0x6f,0xfd,0x30]
 
 0xfa,0x04,0x3e,0x7d,0x01,0x1b,0x00,0xff
 # GFX11: v_cmpx_t_f32 v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x3e,0x7d,0x01,0x1b,0x00,0xff]
@@ -2889,46 +3253,72 @@
 # GFX11: v_cmpx_t_u32 v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0x9f,0x7d,0xff,0x6f,0x0d,0x30]
 
 0xfa,0x04,0x10,0x7d,0x01,0x1b,0x00,0xff
-# GFX11: v_cmpx_u_f16 v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x10,0x7d,0x01,0x1b,0x00,0xff]
+# GFX11-REAL16: v_cmpx_u_f16 v1.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x10,0x7d,0x01,0x1b,0x00,0xff]
+# GFX11-FAKE16: v_cmpx_u_f16 v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x10,0x7d,0x01,0x1b,0x00,0xff]
 
 0xfa,0x04,0x10,0x7d,0x01,0xe4,0x00,0xff
-# GFX11: v_cmpx_u_f16 v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x10,0x7d,0x01,0xe4,0x00,0xff]
+# GFX11-REAL16: v_cmpx_u_f16 v1.l, v2.l quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x10,0x7d,0x01,0xe4,0x00,0xff]
+# GFX11-FAKE16: v_cmpx_u_f16 v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x10,0x7d,0x01,0xe4,0x00,0xff]
 
 0xfa,0x04,0x10,0x7d,0x01,0x40,0x01,0xff
-# GFX11: v_cmpx_u_f16 v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x10,0x7d,0x01,0x40,0x01,0xff]
+# GFX11-REAL16: v_cmpx_u_f16 v1.l, v2.l row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x10,0x7d,0x01,0x40,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_u_f16 v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x10,0x7d,0x01,0x40,0x01,0xff]
 
 0xfa,0x04,0x10,0x7d,0x01,0x41,0x01,0xff
-# GFX11: v_cmpx_u_f16 v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x10,0x7d,0x01,0x41,0x01,0xff]
+# GFX11-REAL16: v_cmpx_u_f16 v1.l, v2.l row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x10,0x7d,0x01,0x41,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_u_f16 v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x10,0x7d,0x01,0x41,0x01,0xff]
 
 0xfa,0x04,0x10,0x7d,0x01,0x01,0x01,0xff
-# GFX11: v_cmpx_u_f16 v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x10,0x7d,0x01,0x01,0x01,0xff]
+# GFX11-REAL16: v_cmpx_u_f16 v1.l, v2.l row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x10,0x7d,0x01,0x01,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_u_f16 v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x10,0x7d,0x01,0x01,0x01,0xff]
 
 0xfa,0x04,0x10,0x7d,0x01,0x0f,0x01,0xff
-# GFX11: v_cmpx_u_f16 v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x10,0x7d,0x01,0x0f,0x01,0xff]
+# GFX11-REAL16: v_cmpx_u_f16 v1.l, v2.l row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x10,0x7d,0x01,0x0f,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_u_f16 v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x10,0x7d,0x01,0x0f,0x01,0xff]
 
 0xfa,0x04,0x10,0x7d,0x01,0x11,0x01,0xff
-# GFX11: v_cmpx_u_f16 v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x10,0x7d,0x01,0x11,0x01,0xff]
+# GFX11-REAL16: v_cmpx_u_f16 v1.l, v2.l row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x10,0x7d,0x01,0x11,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_u_f16 v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x10,0x7d,0x01,0x11,0x01,0xff]
 
 0xfa,0x04,0x10,0x7d,0x01,0x1f,0x01,0xff
-# GFX11: v_cmpx_u_f16 v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x10,0x7d,0x01,0x1f,0x01,0xff]
+# GFX11-REAL16: v_cmpx_u_f16 v1.l, v2.l row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x10,0x7d,0x01,0x1f,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_u_f16 v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x10,0x7d,0x01,0x1f,0x01,0xff]
 
 0xfa,0x04,0x10,0x7d,0x01,0x21,0x01,0xff
-# GFX11: v_cmpx_u_f16 v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x10,0x7d,0x01,0x21,0x01,0xff]
+# GFX11-REAL16: v_cmpx_u_f16 v1.l, v2.l row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x10,0x7d,0x01,0x21,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_u_f16 v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x10,0x7d,0x01,0x21,0x01,0xff]
 
 0xfa,0x04,0x10,0x7d,0x01,0x2f,0x01,0xff
-# GFX11: v_cmpx_u_f16 v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x10,0x7d,0x01,0x2f,0x01,0xff]
+# GFX11-REAL16: v_cmpx_u_f16 v1.l, v2.l row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x10,0x7d,0x01,0x2f,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_u_f16 v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x10,0x7d,0x01,0x2f,0x01,0xff]
 
 0xfa,0x04,0x10,0x7d,0x01,0x50,0x01,0xff
-# GFX11: v_cmpx_u_f16 v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x10,0x7d,0x01,0x50,0x01,0xff]
+# GFX11-REAL16: v_cmpx_u_f16 v1.l, v2.l row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x10,0x7d,0x01,0x50,0x01,0xff]
+# GFX11-FAKE16: v_cmpx_u_f16 v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x10,0x7d,0x01,0x50,0x01,0xff]
 
 0xfa,0x04,0x10,0x7d,0x01,0x5f,0x01,0x01
-# GFX11: v_cmpx_u_f16 v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x10,0x7d,0x01,0x5f,0x01,0x01]
+# GFX11-REAL16: v_cmpx_u_f16 v1.l, v2.l row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x10,0x7d,0x01,0x5f,0x01,0x01]
+# GFX11-FAKE16: v_cmpx_u_f16 v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x10,0x7d,0x01,0x5f,0x01,0x01]
 
 0xfa,0x04,0x10,0x7d,0x01,0x60,0x01,0x13
-# GFX11: v_cmpx_u_f16 v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x10,0x7d,0x01,0x60,0x01,0x13]
+# GFX11-REAL16: v_cmpx_u_f16 v1.l, v2.l row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x10,0x7d,0x01,0x60,0x01,0x13]
+# GFX11-FAKE16: v_cmpx_u_f16 v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x10,0x7d,0x01,0x60,0x01,0x13]
 
 0xfa,0xfe,0x10,0x7d,0x7f,0x6f,0xfd,0x30
-# GFX11: v_cmpx_u_f16 -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0x10,0x7d,0x7f,0x6f,0xfd,0x30]
+# GFX11-REAL16: v_cmpx_u_f16 -|v127.l|, -|v127.l| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0x10,0x7d,0x7f,0x6f,0xfd,0x30]
+# GFX11-FAKE16: v_cmpx_u_f16 -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0x10,0x7d,0x7f,0x6f,0xfd,0x30]
+
+0xfa,0xfe,0x10,0x7d,0x7f,0x5f,0x01,0x01
+# GFX11-REAL16: v_cmpx_u_f16 v127.l, v127.l row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0xfe,0x10,0x7d,0x7f,0x5f,0x01,0x01]
+# GFX11-FAKE16: v_cmpx_u_f16 v127, v127 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0xfe,0x10,0x7d,0x7f,0x5f,0x01,0x01]
+
+0xfa,0x04,0x11,0x7d,0x81,0x60,0x01,0x13
+# GFX11-REAL16: v_cmpx_u_f16 v1.h, v2.h row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x11,0x7d,0x81,0x60,0x01,0x13]
+# GFX11-FAKE16: v_cmpx_u_f16 v129/*Invalid register, operand has 'VGPR_32_Lo128' register class*/, v130/*Invalid register, operand has 'VGPR_32_Lo128' register class*/ row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x11,0x7d,0x81,0x60,0x01,0x13]
+
+0xfa,0xfe,0x11,0x7d,0xff,0x6f,0xfd,0x30
+# GFX11-REAL16: v_cmpx_u_f16 -|v127.h|, -|v127.h| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0x11,0x7d,0xff,0x6f,0xfd,0x30]
+# GFX11-FAKE16: v_cmpx_u_f16 -|v255/*Invalid register, operand has 'VGPR_32_Lo128' register class*/|, -|v255/*Invalid register, operand has 'VGPR_32_Lo128' register class*/| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0x11,0x7d,0xff,0x6f,0xfd,0x30]
 
 0xfa,0x04,0x30,0x7d,0x01,0x1b,0x00,0xff
 # GFX11: v_cmpx_u_f32 v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x30,0x7d,0x01,0x1b,0x00,0xff]
diff --git a/llvm/test/MC/Disassembler/AMDGPU/gfx11_dasm_vopcx_dpp8.txt b/llvm/test/MC/Disassembler/AMDGPU/gfx11_dasm_vopcx_dpp8.txt
index 8350088ca95a5..72fb40a7f22a0 100644
--- a/llvm/test/MC/Disassembler/AMDGPU/gfx11_dasm_vopcx_dpp8.txt
+++ b/llvm/test/MC/Disassembler/AMDGPU/gfx11_dasm_vopcx_dpp8.txt
@@ -31,10 +31,30 @@
 # GFX11: v_cmpx_class_f32 v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0xfd,0x7d,0xff,0x00,0x00,0x00]
 
 0xe9,0x04,0x04,0x7d,0x01,0x77,0x39,0x05
-# GFX11: v_cmpx_eq_f16 v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x04,0x7d,0x01,0x77,0x39,0x05]
+# GFX11-REAL16: v_cmpx_eq_f16 v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x04,0x7d,0x01,0x77,0x39,0x05]
+# GFX11-FAKE16: v_cmpx_eq_f16 v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x04,0x7d,0x01,0x77,0x39,0x05]
 
 0xea,0xfe,0x04,0x7d,0x7f,0x00,0x00,0x00
-# GFX11: v_cmpx_eq_f16 v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0x04,0x7d,0x7f,0x00,0x00,0x00]
+# GFX11-REAL16: v_cmpx_eq_f16 v127.l, v127.l dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0x04,0x7d,0x7f,0x00,0x00,0x00]
+# GFX11-FAKE16: v_cmpx_eq_f16 v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0x04,0x7d,0x7f,0x00,0x00,0x00]
+
+0xe9,0xfe,0x04,0x7d,0x7f,0x77,0x39,0x05
+# GFX11-REAL16: v_cmpx_eq_f16 v127.l, v127.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0xfe,0x04,0x7d,0x7f,0x77,0x39,0x05]
+# GFX11-FAKE16: v_cmpx_eq_f16 v127, v127 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0xfe,0x04,0x7d,0x7f,0x77,0x39,0x05]
+# GFX12-REAL16: v_cmpx_eq_f16 v127.l, v127.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0xfe,0x04,0x7d,0x7f,0x77,0x39,0x05]
+# GFX12-FAKE16: v_cmpx_eq_f16 v127, v127 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0xfe,0x04,0x7d,0x7f,0x77,0x39,0x05]
+
+0xe9,0x04,0x05,0x7d,0x81,0x77,0x39,0x05
+# GFX11-REAL16: v_cmpx_eq_f16 v1.h, v2.h dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x05,0x7d,0x81,0x77,0x39,0x05]
+# GFX11-FAKE16: v_cmpx_eq_f16 v129/*Invalid register, operand has 'VGPR_32_Lo128' register class*/, v130/*Invalid register, operand has 'VGPR_32_Lo128' register class*/ dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x05,0x7d,0x81,0x77,0x39,0x05]
+# GFX12-REAL16: v_cmpx_eq_f16 v1.h, v2.h dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x05,0x7d,0x81,0x77,0x39,0x05]
+# GFX12-FAKE16: v_cmpx_eq_f16 v129/*Invalid register, operand has 'VGPR_32_Lo128' register class*/, v130/*Invalid register, operand has 'VGPR_32_Lo128' register class*/ dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x05,0x7d,0x81,0x77,0x39,0x05]
+
+0xea,0xfe,0x05,0x7d,0xff,0x00,0x00,0x00
+# GFX11-REAL16: v_cmpx_eq_f16 v127.h, v127.h dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0x05,0x7d,0xff,0x00,0x00,0x00]
+# GFX11-FAKE16: v_cmpx_eq_f16 v255/*Invalid register, operand has 'VGPR_32_Lo128' register class*/, v255/*Invalid register, operand has 'VGPR_32_Lo128' register class*/ dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0x05,0x7d,0xff,0x00,0x00,0x00]
+# GFX12-REAL16: v_cmpx_eq_f16 v127.h, v127.h dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0x05,0x7d,0xff,0x00,0x00,0x00]
+# GFX12-FAKE16: v_cmpx_eq_f16 v255/*Invalid register, operand has 'VGPR_32_Lo128' register class*/, v255/*Invalid register, operand has 'VGPR_32_Lo128' register class*/ dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0x05,0x7d,0xff,0x00,0x00,0x00]
 
 0xe9,0x04,0x24,0x7d,0x01,0x77,0x39,0x05
 # GFX11: v_cmpx_eq_f32 v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x24,0x7d,0x01,0x77,0x39,0x05]
@@ -107,10 +127,30 @@
 # GFX11: v_cmpx_eq_u32 v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0x95,0x7d,0xff,0x00,0x00,0x00]
 
 0xe9,0x04,0x00,0x7d,0x01,0x77,0x39,0x05
-# GFX11: v_cmpx_f_f16 v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x00,0x7d,0x01,0x77,0x39,0x05]
+# GFX11-REAL16: v_cmpx_f_f16 v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x00,0x7d,0x01,0x77,0x39,0x05]
+# GFX11-FAKE16: v_cmpx_f_f16 v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x00,0x7d,0x01,0x77,0x39,0x05]
 
 0xea,0xfe,0x00,0x7d,0x7f,0x00,0x00,0x00
-# GFX11: v_cmpx_f_f16 v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0x00,0x7d,0x7f,0x00,0x00,0x00]
+# GFX11-REAL16: v_cmpx_f_f16 v127.l, v127.l dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0x00,0x7d,0x7f,0x00,0x00,0x00]
+# GFX11-FAKE16: v_cmpx_f_f16 v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0x00,0x7d,0x7f,0x00,0x00,0x00]
+
+0xe9,0xfe,0x00,0x7d,0x7f,0x77,0x39,0x05
+# GFX11-REAL16: v_cmpx_f_f16 v127.l, v127.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0xfe,0x00,0x7d,0x7f,0x77,0x39,0x05]
+# GFX11-FAKE16: v_cmpx_f_f16 v127, v127 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0xfe,0x00,0x7d,0x7f,0x77,0x39,0x05]
+# GFX12-REAL16: v_cmpx_f_f16 v127.l, v127.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0xfe,0x00,0x7d,0x7f,0x77,0x39,0x05]
+# GFX12-FAKE16: v_cmpx_f_f16 v127, v127 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0xfe,0x00,0x7d,0x7f,0x77,0x39,0x05]
+
+0xe9,0x04,0x01,0x7d,0x81,0x77,0x39,0x05
+# GFX11-REAL16: v_cmpx_f_f16 v1.h, v2.h dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x01,0x7d,0x81,0x77,0x39,0x05]
+# GFX11-FAKE16: v_cmpx_f_f16 v129/*Invalid register, operand has 'VGPR_32_Lo128' register class*/, v130/*Invalid register, operand has 'VGPR_32_Lo128' register class*/ dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x01,0x7d,0x81,0x77,0x39,0x05]
+# GFX12-REAL16: v_cmpx_f_f16 v1.h, v2.h dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x01,0x7d,0x81,0x77,0x39,0x05]
+# GFX12-FAKE16: v_cmpx_f_f16 v129/*Invalid register, operand has 'VGPR_32_Lo128' register class*/, v130/*Invalid register, operand has 'VGPR_32_Lo128' register class*/ dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x01,0x7d,0x81,0x77,0x39,0x05]
+
+0xea,0xfe,0x01,0x7d,0xff,0x00,0x00,0x00
+# GFX11-REAL16: v_cmpx_f_f16 v127.h, v127.h dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0x01,0x7d,0xff,0x00,0x00,0x00]
+# GFX11-FAKE16: v_cmpx_f_f16 v255/*Invalid register, operand has 'VGPR_32_Lo128' register class*/, v255/*Invalid register, operand has 'VGPR_32_Lo128' register class*/ dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0x01,0x7d,0xff,0x00,0x00,0x00]
+# GFX12-REAL16: v_cmpx_f_f16 v127.h, v127.h dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0x01,0x7d,0xff,0x00,0x00,0x00]
+# GFX12-FAKE16: v_cmpx_f_f16 v255/*Invalid register, operand has 'VGPR_32_Lo128' register class*/, v255/*Invalid register, operand has 'VGPR_32_Lo128' register class*/ dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0x01,0x7d,0xff,0x00,0x00,0x00]
 
 0xe9,0x04,0x20,0x7d,0x01,0x77,0x39,0x05
 # GFX11: v_cmpx_f_f32 v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x20,0x7d,0x01,0x77,0x39,0x05]
@@ -131,10 +171,30 @@
 # GFX11: v_cmpx_f_u32 v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0x91,0x7d,0xff,0x00,0x00,0x00]
 
 0xe9,0x04,0x0c,0x7d,0x01,0x77,0x39,0x05
-# GFX11: v_cmpx_ge_f16 v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x0c,0x7d,0x01,0x77,0x39,0x05]
+# GFX11-REAL16: v_cmpx_ge_f16 v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x0c,0x7d,0x01,0x77,0x39,0x05]
+# GFX11-FAKE16: v_cmpx_ge_f16 v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x0c,0x7d,0x01,0x77,0x39,0x05]
 
 0xea,0xfe,0x0c,0x7d,0x7f,0x00,0x00,0x00
-# GFX11: v_cmpx_ge_f16 v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0x0c,0x7d,0x7f,0x00,0x00,0x00]
+# GFX11-REAL16: v_cmpx_ge_f16 v127.l, v127.l dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0x0c,0x7d,0x7f,0x00,0x00,0x00]
+# GFX11-FAKE16: v_cmpx_ge_f16 v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0x0c,0x7d,0x7f,0x00,0x00,0x00]
+
+0xe9,0xfe,0x0c,0x7d,0x7f,0x77,0x39,0x05
+# GFX11-REAL16: v_cmpx_ge_f16 v127.l, v127.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0xfe,0x0c,0x7d,0x7f,0x77,0x39,0x05]
+# GFX11-FAKE16: v_cmpx_ge_f16 v127, v127 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0xfe,0x0c,0x7d,0x7f,0x77,0x39,0x05]
+# GFX12-REAL16: v_cmpx_ge_f16 v127.l, v127.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0xfe,0x0c,0x7d,0x7f,0x77,0x39,0x05]
+# GFX12-FAKE16: v_cmpx_ge_f16 v127, v127 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0xfe,0x0c,0x7d,0x7f,0x77,0x39,0x05]
+
+0xe9,0x04,0x0d,0x7d,0x81,0x77,0x39,0x05
+# GFX11-REAL16: v_cmpx_ge_f16 v1.h, v2.h dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x0d,0x7d,0x81,0x77,0x39,0x05]
+# GFX11-FAKE16: v_cmpx_ge_f16 v129/*Invalid register, operand has 'VGPR_32_Lo128' register class*/, v130/*Invalid register, operand has 'VGPR_32_Lo128' register class*/ dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x0d,0x7d,0x81,0x77,0x39,0x05]
+# GFX12-REAL16: v_cmpx_ge_f16 v1.h, v2.h dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x0d,0x7d,0x81,0x77,0x39,0x05]
+# GFX12-FAKE16: v_cmpx_ge_f16 v129/*Invalid register, operand has 'VGPR_32_Lo128' register class*/, v130/*Invalid register, operand has 'VGPR_32_Lo128' register class*/ dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x0d,0x7d,0x81,0x77,0x39,0x05]
+
+0xea,0xfe,0x0d,0x7d,0xff,0x00,0x00,0x00
+# GFX11-REAL16: v_cmpx_ge_f16 v127.h, v127.h dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0x0d,0x7d,0xff,0x00,0x00,0x00]
+# GFX11-FAKE16: v_cmpx_ge_f16 v255/*Invalid register, operand has 'VGPR_32_Lo128' register class*/, v255/*Invalid register, operand has 'VGPR_32_Lo128' register class*/ dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0x0d,0x7d,0xff,0x00,0x00,0x00]
+# GFX12-REAL16: v_cmpx_ge_f16 v127.h, v127.h dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0x0d,0x7d,0xff,0x00,0x00,0x00]
+# GFX12-FAKE16: v_cmpx_ge_f16 v255/*Invalid register, operand has 'VGPR_32_Lo128' register class*/, v255/*Invalid register, operand has 'VGPR_32_Lo128' register class*/ dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0x0d,0x7d,0xff,0x00,0x00,0x00]
 
 0xe9,0x04,0x2c,0x7d,0x01,0x77,0x39,0x05
 # GFX11: v_cmpx_ge_f32 v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x2c,0x7d,0x01,0x77,0x39,0x05]
@@ -207,10 +267,30 @@
 # GFX11: v_cmpx_ge_u32 v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0x9d,0x7d,0xff,0x00,0x00,0x00]
 
 0xe9,0x04,0x08,0x7d,0x01,0x77,0x39,0x05
-# GFX11: v_cmpx_gt_f16 v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x08,0x7d,0x01,0x77,0x39,0x05]
+# GFX11-REAL16: v_cmpx_gt_f16 v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x08,0x7d,0x01,0x77,0x39,0x05]
+# GFX11-FAKE16: v_cmpx_gt_f16 v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x08,0x7d,0x01,0x77,0x39,0x05]
 
 0xea,0xfe,0x08,0x7d,0x7f,0x00,0x00,0x00
-# GFX11: v_cmpx_gt_f16 v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0x08,0x7d,0x7f,0x00,0x00,0x00]
+# GFX11-REAL16: v_cmpx_gt_f16 v127.l, v127.l dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0x08,0x7d,0x7f,0x00,0x00,0x00]
+# GFX11-FAKE16: v_cmpx_gt_f16 v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0x08,0x7d,0x7f,0x00,0x00,0x00]
+
+0xe9,0xfe,0x08,0x7d,0x7f,0x77,0x39,0x05
+# GFX11-REAL16: v_cmpx_gt_f16 v127.l, v127.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0xfe,0x08,0x7d,0x7f,0x77,0x39,0x05]
+# GFX11-FAKE16: v_cmpx_gt_f16 v127, v127 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0xfe,0x08,0x7d,0x7f,0x77,0x39,0x05]
+# GFX12-REAL16: v_cmpx_gt_f16 v127.l, v127.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0xfe,0x08,0x7d,0x7f,0x77,0x39,0x05]
+# GFX12-FAKE16: v_cmpx_gt_f16 v127, v127 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0xfe,0x08,0x7d,0x7f,0x77,0x39,0x05]
+
+0xe9,0x04,0x09,0x7d,0x81,0x77,0x39,0x05
+# GFX11-REAL16: v_cmpx_gt_f16 v1.h, v2.h dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x09,0x7d,0x81,0x77,0x39,0x05]
+# GFX11-FAKE16: v_cmpx_gt_f16 v129/*Invalid register, operand has 'VGPR_32_Lo128' register class*/, v130/*Invalid register, operand has 'VGPR_32_Lo128' register class*/ dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x09,0x7d,0x81,0x77,0x39,0x05]
+# GFX12-REAL16: v_cmpx_gt_f16 v1.h, v2.h dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x09,0x7d,0x81,0x77,0x39,0x05]
+# GFX12-FAKE16: v_cmpx_gt_f16 v129/*Invalid register, operand has 'VGPR_32_Lo128' register class*/, v130/*Invalid register, operand has 'VGPR_32_Lo128' register class*/ dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x09,0x7d,0x81,0x77,0x39,0x05]
+
+0xea,0xfe,0x09,0x7d,0xff,0x00,0x00,0x00
+# GFX11-REAL16: v_cmpx_gt_f16 v127.h, v127.h dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0x09,0x7d,0xff,0x00,0x00,0x00]
+# GFX11-FAKE16: v_cmpx_gt_f16 v255/*Invalid register, operand has 'VGPR_32_Lo128' register class*/, v255/*Invalid register, operand has 'VGPR_32_Lo128' register class*/ dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0x09,0x7d,0xff,0x00,0x00,0x00]
+# GFX12-REAL16: v_cmpx_gt_f16 v127.h, v127.h dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0x09,0x7d,0xff,0x00,0x00,0x00]
+# GFX12-FAKE16: v_cmpx_gt_f16 v255/*Invalid register, operand has 'VGPR_32_Lo128' register class*/, v255/*Invalid register, operand has 'VGPR_32_Lo128' register class*/ dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0x09,0x7d,0xff,0x00,0x00,0x00]
 
 0xe9,0x04,0x28,0x7d,0x01,0x77,0x39,0x05
 # GFX11: v_cmpx_gt_f32 v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x28,0x7d,0x01,0x77,0x39,0x05]
@@ -283,10 +363,30 @@
 # GFX11: v_cmpx_gt_u32 v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0x99,0x7d,0xff,0x00,0x00,0x00]
 
 0xe9,0x04,0x06,0x7d,0x01,0x77,0x39,0x05
-# GFX11: v_cmpx_le_f16 v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x06,0x7d,0x01,0x77,0x39,0x05]
+# GFX11-REAL16: v_cmpx_le_f16 v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x06,0x7d,0x01,0x77,0x39,0x05]
+# GFX11-FAKE16: v_cmpx_le_f16 v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x06,0x7d,0x01,0x77,0x39,0x05]
 
 0xea,0xfe,0x06,0x7d,0x7f,0x00,0x00,0x00
-# GFX11: v_cmpx_le_f16 v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0x06,0x7d,0x7f,0x00,0x00,0x00]
+# GFX11-REAL16: v_cmpx_le_f16 v127.l, v127.l dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0x06,0x7d,0x7f,0x00,0x00,0x00]
+# GFX11-FAKE16: v_cmpx_le_f16 v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0x06,0x7d,0x7f,0x00,0x00,0x00]
+
+0xe9,0xfe,0x06,0x7d,0x7f,0x77,0x39,0x05
+# GFX11-REAL16: v_cmpx_le_f16 v127.l, v127.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0xfe,0x06,0x7d,0x7f,0x77,0x39,0x05]
+# GFX11-FAKE16: v_cmpx_le_f16 v127, v127 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0xfe,0x06,0x7d,0x7f,0x77,0x39,0x05]
+# GFX12-REAL16: v_cmpx_le_f16 v127.l, v127.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0xfe,0x06,0x7d,0x7f,0x77,0x39,0x05]
+# GFX12-FAKE16: v_cmpx_le_f16 v127, v127 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0xfe,0x06,0x7d,0x7f,0x77,0x39,0x05]
+
+0xe9,0x04,0x07,0x7d,0x81,0x77,0x39,0x05
+# GFX11-REAL16: v_cmpx_le_f16 v1.h, v2.h dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x07,0x7d,0x81,0x77,0x39,0x05]
+# GFX11-FAKE16: v_cmpx_le_f16 v129/*Invalid register, operand has 'VGPR_32_Lo128' register class*/, v130/*Invalid register, operand has 'VGPR_32_Lo128' register class*/ dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x07,0x7d,0x81,0x77,0x39,0x05]
+# GFX12-REAL16: v_cmpx_le_f16 v1.h, v2.h dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x07,0x7d,0x81,0x77,0x39,0x05]
+# GFX12-FAKE16: v_cmpx_le_f16 v129/*Invalid register, operand has 'VGPR_32_Lo128' register class*/, v130/*Invalid register, operand has 'VGPR_32_Lo128' register class*/ dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x07,0x7d,0x81,0x77,0x39,0x05]
+
+0xea,0xfe,0x07,0x7d,0xff,0x00,0x00,0x00
+# GFX11-REAL16: v_cmpx_le_f16 v127.h, v127.h dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0x07,0x7d,0xff,0x00,0x00,0x00]
+# GFX11-FAKE16: v_cmpx_le_f16 v255/*Invalid register, operand has 'VGPR_32_Lo128' register class*/, v255/*Invalid register, operand has 'VGPR_32_Lo128' register class*/ dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0x07,0x7d,0xff,0x00,0x00,0x00]
+# GFX12-REAL16: v_cmpx_le_f16 v127.h, v127.h dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0x07,0x7d,0xff,0x00,0x00,0x00]
+# GFX12-FAKE16: v_cmpx_le_f16 v255/*Invalid register, operand has 'VGPR_32_Lo128' register class*/, v255/*Invalid register, operand has 'VGPR_32_Lo128' register class*/ dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0x07,0x7d,0xff,0x00,0x00,0x00]
 
 0xe9,0x04,0x26,0x7d,0x01,0x77,0x39,0x05
 # GFX11: v_cmpx_le_f32 v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x26,0x7d,0x01,0x77,0x39,0x05]
@@ -359,10 +459,30 @@
 # GFX11: v_cmpx_le_u32 v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0x97,0x7d,0xff,0x00,0x00,0x00]
 
 0xe9,0x04,0x0a,0x7d,0x01,0x77,0x39,0x05
-# GFX11: v_cmpx_lg_f16 v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x0a,0x7d,0x01,0x77,0x39,0x05]
+# GFX11-REAL16: v_cmpx_lg_f16 v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x0a,0x7d,0x01,0x77,0x39,0x05]
+# GFX11-FAKE16: v_cmpx_lg_f16 v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x0a,0x7d,0x01,0x77,0x39,0x05]
 
 0xea,0xfe,0x0a,0x7d,0x7f,0x00,0x00,0x00
-# GFX11: v_cmpx_lg_f16 v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0x0a,0x7d,0x7f,0x00,0x00,0x00]
+# GFX11-REAL16: v_cmpx_lg_f16 v127.l, v127.l dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0x0a,0x7d,0x7f,0x00,0x00,0x00]
+# GFX11-FAKE16: v_cmpx_lg_f16 v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0x0a,0x7d,0x7f,0x00,0x00,0x00]
+
+0xe9,0xfe,0x0a,0x7d,0x7f,0x77,0x39,0x05
+# GFX11-REAL16: v_cmpx_lg_f16 v127.l, v127.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0xfe,0x0a,0x7d,0x7f,0x77,0x39,0x05]
+# GFX11-FAKE16: v_cmpx_lg_f16 v127, v127 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0xfe,0x0a,0x7d,0x7f,0x77,0x39,0x05]
+# GFX12-REAL16: v_cmpx_lg_f16 v127.l, v127.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0xfe,0x0a,0x7d,0x7f,0x77,0x39,0x05]
+# GFX12-FAKE16: v_cmpx_lg_f16 v127, v127 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0xfe,0x0a,0x7d,0x7f,0x77,0x39,0x05]
+
+0xe9,0x04,0x0b,0x7d,0x81,0x77,0x39,0x05
+# GFX11-REAL16: v_cmpx_lg_f16 v1.h, v2.h dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x0b,0x7d,0x81,0x77,0x39,0x05]
+# GFX11-FAKE16: v_cmpx_lg_f16 v129/*Invalid register, operand has 'VGPR_32_Lo128' register class*/, v130/*Invalid register, operand has 'VGPR_32_Lo128' register class*/ dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x0b,0x7d,0x81,0x77,0x39,0x05]
+# GFX12-REAL16: v_cmpx_lg_f16 v1.h, v2.h dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x0b,0x7d,0x81,0x77,0x39,0x05]
+# GFX12-FAKE16: v_cmpx_lg_f16 v129/*Invalid register, operand has 'VGPR_32_Lo128' register class*/, v130/*Invalid register, operand has 'VGPR_32_Lo128' register class*/ dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x0b,0x7d,0x81,0x77,0x39,0x05]
+
+0xea,0xfe,0x0b,0x7d,0xff,0x00,0x00,0x00
+# GFX11-REAL16: v_cmpx_lg_f16 v127.h, v127.h dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0x0b,0x7d,0xff,0x00,0x00,0x00]
+# GFX11-FAKE16: v_cmpx_lg_f16 v255/*Invalid register, operand has 'VGPR_32_Lo128' register class*/, v255/*Invalid register, operand has 'VGPR_32_Lo128' register class*/ dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0x0b,0x7d,0xff,0x00,0x00,0x00]
+# GFX12-REAL16: v_cmpx_lg_f16 v127.h, v127.h dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0x0b,0x7d,0xff,0x00,0x00,0x00]
+# GFX12-FAKE16: v_cmpx_lg_f16 v255/*Invalid register, operand has 'VGPR_32_Lo128' register class*/, v255/*Invalid register, operand has 'VGPR_32_Lo128' register class*/ dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0x0b,0x7d,0xff,0x00,0x00,0x00]
 
 0xe9,0x04,0x2a,0x7d,0x01,0x77,0x39,0x05
 # GFX11: v_cmpx_lg_f32 v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x2a,0x7d,0x01,0x77,0x39,0x05]
@@ -525,10 +645,30 @@
 # GFX11: v_cmpx_ne_u32 v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0x9b,0x7d,0xff,0x00,0x00,0x00]
 
 0xe9,0x04,0x1a,0x7d,0x01,0x77,0x39,0x05
-# GFX11: v_cmpx_neq_f16 v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x1a,0x7d,0x01,0x77,0x39,0x05]
+# GFX11-REAL16: v_cmpx_neq_f16 v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x1a,0x7d,0x01,0x77,0x39,0x05]
+# GFX11-FAKE16: v_cmpx_neq_f16 v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x1a,0x7d,0x01,0x77,0x39,0x05]
 
 0xea,0xfe,0x1a,0x7d,0x7f,0x00,0x00,0x00
-# GFX11: v_cmpx_neq_f16 v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0x1a,0x7d,0x7f,0x00,0x00,0x00]
+# GFX11-REAL16: v_cmpx_neq_f16 v127.l, v127.l dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0x1a,0x7d,0x7f,0x00,0x00,0x00]
+# GFX11-FAKE16: v_cmpx_neq_f16 v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0x1a,0x7d,0x7f,0x00,0x00,0x00]
+
+0xe9,0xfe,0x1a,0x7d,0x7f,0x77,0x39,0x05
+# GFX11-REAL16: v_cmpx_neq_f16 v127.l, v127.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0xfe,0x1a,0x7d,0x7f,0x77,0x39,0x05]
+# GFX11-FAKE16: v_cmpx_neq_f16 v127, v127 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0xfe,0x1a,0x7d,0x7f,0x77,0x39,0x05]
+# GFX12-REAL16: v_cmpx_neq_f16 v127.l, v127.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0xfe,0x1a,0x7d,0x7f,0x77,0x39,0x05]
+# GFX12-FAKE16: v_cmpx_neq_f16 v127, v127 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0xfe,0x1a,0x7d,0x7f,0x77,0x39,0x05]
+
+0xe9,0x04,0x1b,0x7d,0x81,0x77,0x39,0x05
+# GFX11-REAL16: v_cmpx_neq_f16 v1.h, v2.h dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x1b,0x7d,0x81,0x77,0x39,0x05]
+# GFX11-FAKE16: v_cmpx_neq_f16 v129/*Invalid register, operand has 'VGPR_32_Lo128' register class*/, v130/*Invalid register, operand has 'VGPR_32_Lo128' register class*/ dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x1b,0x7d,0x81,0x77,0x39,0x05]
+# GFX12-REAL16: v_cmpx_neq_f16 v1.h, v2.h dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x1b,0x7d,0x81,0x77,0x39,0x05]
+# GFX12-FAKE16: v_cmpx_neq_f16 v129/*Invalid register, operand has 'VGPR_32_Lo128' register class*/, v130/*Invalid register, operand has 'VGPR_32_Lo128' register class*/ dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x1b,0x7d,0x81,0x77,0x39,0x05]
+
+0xea,0xfe,0x1b,0x7d,0xff,0x00,0x00,0x00
+# GFX11-REAL16: v_cmpx_neq_f16 v127.h, v127.h dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0x1b,0x7d,0xff,0x00,0x00,0x00]
+# GFX11-FAKE16: v_cmpx_neq_f16 v255/*Invalid register, operand has 'VGPR_32_Lo128' register class*/, v255/*Invalid register, operand has 'VGPR_32_Lo128' register class*/ dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0x1b,0x7d,0xff,0x00,0x00,0x00]
+# GFX12-REAL16: v_cmpx_neq_f16 v127.h, v127.h dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0x1b,0x7d,0xff,0x00,0x00,0x00]
+# GFX12-FAKE16: v_cmpx_neq_f16 v255/*Invalid register, operand has 'VGPR_32_Lo128' register class*/, v255/*Invalid register, operand has 'VGPR_32_Lo128' register class*/ dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0x1b,0x7d,0xff,0x00,0x00,0x00]
 
 0xe9,0x04,0x3a,0x7d,0x01,0x77,0x39,0x05
 # GFX11: v_cmpx_neq_f32 v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x3a,0x7d,0x01,0x77,0x39,0x05]
@@ -537,10 +677,30 @@
 # GFX11: v_cmpx_neq_f32 v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0x3b,0x7d,0xff,0x00,0x00,0x00]
 
 0xe9,0x04,0x12,0x7d,0x01,0x77,0x39,0x05
-# GFX11: v_cmpx_nge_f16 v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x12,0x7d,0x01,0x77,0x39,0x05]
+# GFX11-REAL16: v_cmpx_nge_f16 v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x12,0x7d,0x01,0x77,0x39,0x05]
+# GFX11-FAKE16: v_cmpx_nge_f16 v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x12,0x7d,0x01,0x77,0x39,0x05]
 
 0xea,0xfe,0x12,0x7d,0x7f,0x00,0x00,0x00
-# GFX11: v_cmpx_nge_f16 v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0x12,0x7d,0x7f,0x00,0x00,0x00]
+# GFX11-REAL16: v_cmpx_nge_f16 v127.l, v127.l dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0x12,0x7d,0x7f,0x00,0x00,0x00]
+# GFX11-FAKE16: v_cmpx_nge_f16 v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0x12,0x7d,0x7f,0x00,0x00,0x00]
+
+0xe9,0xfe,0x12,0x7d,0x7f,0x77,0x39,0x05
+# GFX11-REAL16: v_cmpx_nge_f16 v127.l, v127.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0xfe,0x12,0x7d,0x7f,0x77,0x39,0x05]
+# GFX11-FAKE16: v_cmpx_nge_f16 v127, v127 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0xfe,0x12,0x7d,0x7f,0x77,0x39,0x05]
+# GFX12-REAL16: v_cmpx_nge_f16 v127.l, v127.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0xfe,0x12,0x7d,0x7f,0x77,0x39,0x05]
+# GFX12-FAKE16: v_cmpx_nge_f16 v127, v127 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0xfe,0x12,0x7d,0x7f,0x77,0x39,0x05]
+
+0xe9,0x04,0x13,0x7d,0x81,0x77,0x39,0x05
+# GFX11-REAL16: v_cmpx_nge_f16 v1.h, v2.h dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x13,0x7d,0x81,0x77,0x39,0x05]
+# GFX11-FAKE16: v_cmpx_nge_f16 v129/*Invalid register, operand has 'VGPR_32_Lo128' register class*/, v130/*Invalid register, operand has 'VGPR_32_Lo128' register class*/ dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x13,0x7d,0x81,0x77,0x39,0x05]
+# GFX12-REAL16: v_cmpx_nge_f16 v1.h, v2.h dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x13,0x7d,0x81,0x77,0x39,0x05]
+# GFX12-FAKE16: v_cmpx_nge_f16 v129/*Invalid register, operand has 'VGPR_32_Lo128' register class*/, v130/*Invalid register, operand has 'VGPR_32_Lo128' register class*/ dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x13,0x7d,0x81,0x77,0x39,0x05]
+
+0xea,0xfe,0x13,0x7d,0xff,0x00,0x00,0x00
+# GFX11-REAL16: v_cmpx_nge_f16 v127.h, v127.h dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0x13,0x7d,0xff,0x00,0x00,0x00]
+# GFX11-FAKE16: v_cmpx_nge_f16 v255/*Invalid register, operand has 'VGPR_32_Lo128' register class*/, v255/*Invalid register, operand has 'VGPR_32_Lo128' register class*/ dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0x13,0x7d,0xff,0x00,0x00,0x00]
+# GFX12-REAL16: v_cmpx_nge_f16 v127.h, v127.h dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0x13,0x7d,0xff,0x00,0x00,0x00]
+# GFX12-FAKE16: v_cmpx_nge_f16 v255/*Invalid register, operand has 'VGPR_32_Lo128' register class*/, v255/*Invalid register, operand has 'VGPR_32_Lo128' register class*/ dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0x13,0x7d,0xff,0x00,0x00,0x00]
 
 0xe9,0x04,0x32,0x7d,0x01,0x77,0x39,0x05
 # GFX11: v_cmpx_nge_f32 v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x32,0x7d,0x01,0x77,0x39,0x05]
@@ -549,10 +709,30 @@
 # GFX11: v_cmpx_nge_f32 v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0x33,0x7d,0xff,0x00,0x00,0x00]
 
 0xe9,0x04,0x16,0x7d,0x01,0x77,0x39,0x05
-# GFX11: v_cmpx_ngt_f16 v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x16,0x7d,0x01,0x77,0x39,0x05]
+# GFX11-REAL16: v_cmpx_ngt_f16 v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x16,0x7d,0x01,0x77,0x39,0x05]
+# GFX11-FAKE16: v_cmpx_ngt_f16 v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x16,0x7d,0x01,0x77,0x39,0x05]
 
 0xea,0xfe,0x16,0x7d,0x7f,0x00,0x00,0x00
-# GFX11: v_cmpx_ngt_f16 v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0x16,0x7d,0x7f,0x00,0x00,0x00]
+# GFX11-REAL16: v_cmpx_ngt_f16 v127.l, v127.l dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0x16,0x7d,0x7f,0x00,0x00,0x00]
+# GFX11-FAKE16: v_cmpx_ngt_f16 v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0x16,0x7d,0x7f,0x00,0x00,0x00]
+
+0xe9,0xfe,0x16,0x7d,0x7f,0x77,0x39,0x05
+# GFX11-REAL16: v_cmpx_ngt_f16 v127.l, v127.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0xfe,0x16,0x7d,0x7f,0x77,0x39,0x05]
+# GFX11-FAKE16: v_cmpx_ngt_f16 v127, v127 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0xfe,0x16,0x7d,0x7f,0x77,0x39,0x05]
+# GFX12-REAL16: v_cmpx_ngt_f16 v127.l, v127.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0xfe,0x16,0x7d,0x7f,0x77,0x39,0x05]
+# GFX12-FAKE16: v_cmpx_ngt_f16 v127, v127 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0xfe,0x16,0x7d,0x7f,0x77,0x39,0x05]
+
+0xe9,0x04,0x17,0x7d,0x81,0x77,0x39,0x05
+# GFX11-REAL16: v_cmpx_ngt_f16 v1.h, v2.h dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x17,0x7d,0x81,0x77,0x39,0x05]
+# GFX11-FAKE16: v_cmpx_ngt_f16 v129/*Invalid register, operand has 'VGPR_32_Lo128' register class*/, v130/*Invalid register, operand has 'VGPR_32_Lo128' register class*/ dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x17,0x7d,0x81,0x77,0x39,0x05]
+# GFX12-REAL16: v_cmpx_ngt_f16 v1.h, v2.h dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x17,0x7d,0x81,0x77,0x39,0x05]
+# GFX12-FAKE16: v_cmpx_ngt_f16 v129/*Invalid register, operand has 'VGPR_32_Lo128' register class*/, v130/*Invalid register, operand has 'VGPR_32_Lo128' register class*/ dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x17,0x7d,0x81,0x77,0x39,0x05]
+
+0xea,0xfe,0x17,0x7d,0xff,0x00,0x00,0x00
+# GFX11-REAL16: v_cmpx_ngt_f16 v127.h, v127.h dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0x17,0x7d,0xff,0x00,0x00,0x00]
+# GFX11-FAKE16: v_cmpx_ngt_f16 v255/*Invalid register, operand has 'VGPR_32_Lo128' register class*/, v255/*Invalid register, operand has 'VGPR_32_Lo128' register class*/ dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0x17,0x7d,0xff,0x00,0x00,0x00]
+# GFX12-REAL16: v_cmpx_ngt_f16 v127.h, v127.h dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0x17,0x7d,0xff,0x00,0x00,0x00]
+# GFX12-FAKE16: v_cmpx_ngt_f16 v255/*Invalid register, operand has 'VGPR_32_Lo128' register class*/, v255/*Invalid register, operand has 'VGPR_32_Lo128' register class*/ dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0x17,0x7d,0xff,0x00,0x00,0x00]
 
 0xe9,0x04,0x36,0x7d,0x01,0x77,0x39,0x05
 # GFX11: v_cmpx_ngt_f32 v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x36,0x7d,0x01,0x77,0x39,0x05]
@@ -561,10 +741,30 @@
 # GFX11: v_cmpx_ngt_f32 v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0x37,0x7d,0xff,0x00,0x00,0x00]
 
 0xe9,0x04,0x18,0x7d,0x01,0x77,0x39,0x05
-# GFX11: v_cmpx_nle_f16 v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x18,0x7d,0x01,0x77,0x39,0x05]
+# GFX11-REAL16: v_cmpx_nle_f16 v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x18,0x7d,0x01,0x77,0x39,0x05]
+# GFX11-FAKE16: v_cmpx_nle_f16 v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x18,0x7d,0x01,0x77,0x39,0x05]
 
 0xea,0xfe,0x18,0x7d,0x7f,0x00,0x00,0x00
-# GFX11: v_cmpx_nle_f16 v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0x18,0x7d,0x7f,0x00,0x00,0x00]
+# GFX11-REAL16: v_cmpx_nle_f16 v127.l, v127.l dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0x18,0x7d,0x7f,0x00,0x00,0x00]
+# GFX11-FAKE16: v_cmpx_nle_f16 v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0x18,0x7d,0x7f,0x00,0x00,0x00]
+
+0xe9,0xfe,0x18,0x7d,0x7f,0x77,0x39,0x05
+# GFX11-REAL16: v_cmpx_nle_f16 v127.l, v127.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0xfe,0x18,0x7d,0x7f,0x77,0x39,0x05]
+# GFX11-FAKE16: v_cmpx_nle_f16 v127, v127 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0xfe,0x18,0x7d,0x7f,0x77,0x39,0x05]
+# GFX12-REAL16: v_cmpx_nle_f16 v127.l, v127.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0xfe,0x18,0x7d,0x7f,0x77,0x39,0x05]
+# GFX12-FAKE16: v_cmpx_nle_f16 v127, v127 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0xfe,0x18,0x7d,0x7f,0x77,0x39,0x05]
+
+0xe9,0x04,0x19,0x7d,0x81,0x77,0x39,0x05
+# GFX11-REAL16: v_cmpx_nle_f16 v1.h, v2.h dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x19,0x7d,0x81,0x77,0x39,0x05]
+# GFX11-FAKE16: v_cmpx_nle_f16 v129/*Invalid register, operand has 'VGPR_32_Lo128' register class*/, v130/*Invalid register, operand has 'VGPR_32_Lo128' register class*/ dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x19,0x7d,0x81,0x77,0x39,0x05]
+# GFX12-REAL16: v_cmpx_nle_f16 v1.h, v2.h dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x19,0x7d,0x81,0x77,0x39,0x05]
+# GFX12-FAKE16: v_cmpx_nle_f16 v129/*Invalid register, operand has 'VGPR_32_Lo128' register class*/, v130/*Invalid register, operand has 'VGPR_32_Lo128' register class*/ dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x19,0x7d,0x81,0x77,0x39,0x05]
+
+0xea,0xfe,0x19,0x7d,0xff,0x00,0x00,0x00
+# GFX11-REAL16: v_cmpx_nle_f16 v127.h, v127.h dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0x19,0x7d,0xff,0x00,0x00,0x00]
+# GFX11-FAKE16: v_cmpx_nle_f16 v255/*Invalid register, operand has 'VGPR_32_Lo128' register class*/, v255/*Invalid register, operand has 'VGPR_32_Lo128' register class*/ dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0x19,0x7d,0xff,0x00,0x00,0x00]
+# GFX12-REAL16: v_cmpx_nle_f16 v127.h, v127.h dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0x19,0x7d,0xff,0x00,0x00,0x00]
+# GFX12-FAKE16: v_cmpx_nle_f16 v255/*Invalid register, operand has 'VGPR_32_Lo128' register class*/, v255/*Invalid register, operand has 'VGPR_32_Lo128' register class*/ dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0x19,0x7d,0xff,0x00,0x00,0x00]
 
 0xe9,0x04,0x38,0x7d,0x01,0x77,0x39,0x05
 # GFX11: v_cmpx_nle_f32 v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x38,0x7d,0x01,0x77,0x39,0x05]
@@ -573,10 +773,30 @@
 # GFX11: v_cmpx_nle_f32 v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0x39,0x7d,0xff,0x00,0x00,0x00]
 
 0xe9,0x04,0x14,0x7d,0x01,0x77,0x39,0x05
-# GFX11: v_cmpx_nlg_f16 v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x14,0x7d,0x01,0x77,0x39,0x05]
+# GFX11-REAL16: v_cmpx_nlg_f16 v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x14,0x7d,0x01,0x77,0x39,0x05]
+# GFX11-FAKE16: v_cmpx_nlg_f16 v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x14,0x7d,0x01,0x77,0x39,0x05]
 
 0xea,0xfe,0x14,0x7d,0x7f,0x00,0x00,0x00
-# GFX11: v_cmpx_nlg_f16 v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0x14,0x7d,0x7f,0x00,0x00,0x00]
+# GFX11-REAL16: v_cmpx_nlg_f16 v127.l, v127.l dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0x14,0x7d,0x7f,0x00,0x00,0x00]
+# GFX11-FAKE16: v_cmpx_nlg_f16 v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0x14,0x7d,0x7f,0x00,0x00,0x00]
+
+0xe9,0xfe,0x14,0x7d,0x7f,0x77,0x39,0x05
+# GFX11-REAL16: v_cmpx_nlg_f16 v127.l, v127.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0xfe,0x14,0x7d,0x7f,0x77,0x39,0x05]
+# GFX11-FAKE16: v_cmpx_nlg_f16 v127, v127 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0xfe,0x14,0x7d,0x7f,0x77,0x39,0x05]
+# GFX12-REAL16: v_cmpx_nlg_f16 v127.l, v127.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0xfe,0x14,0x7d,0x7f,0x77,0x39,0x05]
+# GFX12-FAKE16: v_cmpx_nlg_f16 v127, v127 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0xfe,0x14,0x7d,0x7f,0x77,0x39,0x05]
+
+0xe9,0x04,0x15,0x7d,0x81,0x77,0x39,0x05
+# GFX11-REAL16: v_cmpx_nlg_f16 v1.h, v2.h dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x15,0x7d,0x81,0x77,0x39,0x05]
+# GFX11-FAKE16: v_cmpx_nlg_f16 v129/*Invalid register, operand has 'VGPR_32_Lo128' register class*/, v130/*Invalid register, operand has 'VGPR_32_Lo128' register class*/ dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x15,0x7d,0x81,0x77,0x39,0x05]
+# GFX12-REAL16: v_cmpx_nlg_f16 v1.h, v2.h dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x15,0x7d,0x81,0x77,0x39,0x05]
+# GFX12-FAKE16: v_cmpx_nlg_f16 v129/*Invalid register, operand has 'VGPR_32_Lo128' register class*/, v130/*Invalid register, operand has 'VGPR_32_Lo128' register class*/ dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x15,0x7d,0x81,0x77,0x39,0x05]
+
+0xea,0xfe,0x15,0x7d,0xff,0x00,0x00,0x00
+# GFX11-REAL16: v_cmpx_nlg_f16 v127.h, v127.h dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0x15,0x7d,0xff,0x00,0x00,0x00]
+# GFX11-FAKE16: v_cmpx_nlg_f16 v255/*Invalid register, operand has 'VGPR_32_Lo128' register class*/, v255/*Invalid register, operand has 'VGPR_32_Lo128' register class*/ dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0x15,0x7d,0xff,0x00,0x00,0x00]
+# GFX12-REAL16: v_cmpx_nlg_f16 v127.h, v127.h dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0x15,0x7d,0xff,0x00,0x00,0x00]
+# GFX12-FAKE16: v_cmpx_nlg_f16 v255/*Invalid register, operand has 'VGPR_32_Lo128' register class*/, v255/*Invalid register, operand has 'VGPR_32_Lo128' register class*/ dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0x15,0x7d,0xff,0x00,0x00,0x00]
 
 0xe9,0x04,0x34,0x7d,0x01,0x77,0x39,0x05
 # GFX11: v_cmpx_nlg_f32 v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x34,0x7d,0x01,0x77,0x39,0x05]
@@ -585,10 +805,30 @@
 # GFX11: v_cmpx_nlg_f32 v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0x35,0x7d,0xff,0x00,0x00,0x00]
 
 0xe9,0x04,0x1c,0x7d,0x01,0x77,0x39,0x05
-# GFX11: v_cmpx_nlt_f16 v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x1c,0x7d,0x01,0x77,0x39,0x05]
+# GFX11-REAL16: v_cmpx_nlt_f16 v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x1c,0x7d,0x01,0x77,0x39,0x05]
+# GFX11-FAKE16: v_cmpx_nlt_f16 v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x1c,0x7d,0x01,0x77,0x39,0x05]
 
 0xea,0xfe,0x1c,0x7d,0x7f,0x00,0x00,0x00
-# GFX11: v_cmpx_nlt_f16 v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0x1c,0x7d,0x7f,0x00,0x00,0x00]
+# GFX11-REAL16: v_cmpx_nlt_f16 v127.l, v127.l dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0x1c,0x7d,0x7f,0x00,0x00,0x00]
+# GFX11-FAKE16: v_cmpx_nlt_f16 v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0x1c,0x7d,0x7f,0x00,0x00,0x00]
+
+0xe9,0xfe,0x1c,0x7d,0x7f,0x77,0x39,0x05
+# GFX11-REAL16: v_cmpx_nlt_f16 v127.l, v127.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0xfe,0x1c,0x7d,0x7f,0x77,0x39,0x05]
+# GFX11-FAKE16: v_cmpx_nlt_f16 v127, v127 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0xfe,0x1c,0x7d,0x7f,0x77,0x39,0x05]
+# GFX12-REAL16: v_cmpx_nlt_f16 v127.l, v127.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0xfe,0x1c,0x7d,0x7f,0x77,0x39,0x05]
+# GFX12-FAKE16: v_cmpx_nlt_f16 v127, v127 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0xfe,0x1c,0x7d,0x7f,0x77,0x39,0x05]
+
+0xe9,0x04,0x1d,0x7d,0x81,0x77,0x39,0x05
+# GFX11-REAL16: v_cmpx_nlt_f16 v1.h, v2.h dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x1d,0x7d,0x81,0x77,0x39,0x05]
+# GFX11-FAKE16: v_cmpx_nlt_f16 v129/*Invalid register, operand has 'VGPR_32_Lo128' register class*/, v130/*Invalid register, operand has 'VGPR_32_Lo128' register class*/ dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x1d,0x7d,0x81,0x77,0x39,0x05]
+# GFX12-REAL16: v_cmpx_nlt_f16 v1.h, v2.h dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x1d,0x7d,0x81,0x77,0x39,0x05]
+# GFX12-FAKE16: v_cmpx_nlt_f16 v129/*Invalid register, operand has 'VGPR_32_Lo128' register class*/, v130/*Invalid register, operand has 'VGPR_32_Lo128' register class*/ dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x1d,0x7d,0x81,0x77,0x39,0x05]
+
+0xea,0xfe,0x1d,0x7d,0xff,0x00,0x00,0x00
+# GFX11-REAL16: v_cmpx_nlt_f16 v127.h, v127.h dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0x1d,0x7d,0xff,0x00,0x00,0x00]
+# GFX11-FAKE16: v_cmpx_nlt_f16 v255/*Invalid register, operand has 'VGPR_32_Lo128' register class*/, v255/*Invalid register, operand has 'VGPR_32_Lo128' register class*/ dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0x1d,0x7d,0xff,0x00,0x00,0x00]
+# GFX12-REAL16: v_cmpx_nlt_f16 v127.h, v127.h dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0x1d,0x7d,0xff,0x00,0x00,0x00]
+# GFX12-FAKE16: v_cmpx_nlt_f16 v255/*Invalid register, operand has 'VGPR_32_Lo128' register class*/, v255/*Invalid register, operand has 'VGPR_32_Lo128' register class*/ dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0x1d,0x7d,0xff,0x00,0x00,0x00]
 
 0xe9,0x04,0x3c,0x7d,0x01,0x77,0x39,0x05
 # GFX11: v_cmpx_nlt_f32 v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x3c,0x7d,0x01,0x77,0x39,0x05]
@@ -597,10 +837,30 @@
 # GFX11: v_cmpx_nlt_f32 v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0x3d,0x7d,0xff,0x00,0x00,0x00]
 
 0xe9,0x04,0x0e,0x7d,0x01,0x77,0x39,0x05
-# GFX11: v_cmpx_o_f16 v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x0e,0x7d,0x01,0x77,0x39,0x05]
+# GFX11-REAL16: v_cmpx_o_f16 v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x0e,0x7d,0x01,0x77,0x39,0x05]
+# GFX11-FAKE16: v_cmpx_o_f16 v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x0e,0x7d,0x01,0x77,0x39,0x05]
 
 0xea,0xfe,0x0e,0x7d,0x7f,0x00,0x00,0x00
-# GFX11: v_cmpx_o_f16 v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0x0e,0x7d,0x7f,0x00,0x00,0x00]
+# GFX11-REAL16: v_cmpx_o_f16 v127.l, v127.l dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0x0e,0x7d,0x7f,0x00,0x00,0x00]
+# GFX11-FAKE16: v_cmpx_o_f16 v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0x0e,0x7d,0x7f,0x00,0x00,0x00]
+
+0xe9,0xfe,0x0e,0x7d,0x7f,0x77,0x39,0x05
+# GFX11-REAL16: v_cmpx_o_f16 v127.l, v127.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0xfe,0x0e,0x7d,0x7f,0x77,0x39,0x05]
+# GFX11-FAKE16: v_cmpx_o_f16 v127, v127 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0xfe,0x0e,0x7d,0x7f,0x77,0x39,0x05]
+# GFX12-REAL16: v_cmpx_o_f16 v127.l, v127.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0xfe,0x0e,0x7d,0x7f,0x77,0x39,0x05]
+# GFX12-FAKE16: v_cmpx_o_f16 v127, v127 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0xfe,0x0e,0x7d,0x7f,0x77,0x39,0x05]
+
+0xe9,0x04,0x0f,0x7d,0x81,0x77,0x39,0x05
+# GFX11-REAL16: v_cmpx_o_f16 v1.h, v2.h dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x0f,0x7d,0x81,0x77,0x39,0x05]
+# GFX11-FAKE16: v_cmpx_o_f16 v129/*Invalid register, operand has 'VGPR_32_Lo128' register class*/, v130/*Invalid register, operand has 'VGPR_32_Lo128' register class*/ dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x0f,0x7d,0x81,0x77,0x39,0x05]
+# GFX12-REAL16: v_cmpx_o_f16 v1.h, v2.h dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x0f,0x7d,0x81,0x77,0x39,0x05]
+# GFX12-FAKE16: v_cmpx_o_f16 v129/*Invalid register, operand has 'VGPR_32_Lo128' register class*/, v130/*Invalid register, operand has 'VGPR_32_Lo128' register class*/ dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x0f,0x7d,0x81,0x77,0x39,0x05]
+
+0xea,0xfe,0x0f,0x7d,0xff,0x00,0x00,0x00
+# GFX11-REAL16: v_cmpx_o_f16 v127.h, v127.h dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0x0f,0x7d,0xff,0x00,0x00,0x00]
+# GFX11-FAKE16: v_cmpx_o_f16 v255/*Invalid register, operand has 'VGPR_32_Lo128' register class*/, v255/*Invalid register, operand has 'VGPR_32_Lo128' register class*/ dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0x0f,0x7d,0xff,0x00,0x00,0x00]
+# GFX12-REAL16: v_cmpx_o_f16 v127.h, v127.h dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0x0f,0x7d,0xff,0x00,0x00,0x00]
+# GFX12-FAKE16: v_cmpx_o_f16 v255/*Invalid register, operand has 'VGPR_32_Lo128' register class*/, v255/*Invalid register, operand has 'VGPR_32_Lo128' register class*/ dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0x0f,0x7d,0xff,0x00,0x00,0x00]
 
 0xe9,0x04,0x2e,0x7d,0x01,0x77,0x39,0x05
 # GFX11: v_cmpx_o_f32 v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x2e,0x7d,0x01,0x77,0x39,0x05]
@@ -609,10 +869,30 @@
 # GFX11: v_cmpx_o_f32 v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0x2f,0x7d,0xff,0x00,0x00,0x00]
 
 0xe9,0x04,0x1e,0x7d,0x01,0x77,0x39,0x05
-# GFX11: v_cmpx_t_f16 v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x1e,0x7d,0x01,0x77,0x39,0x05]
+# GFX11-REAL16: v_cmpx_t_f16 v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x1e,0x7d,0x01,0x77,0x39,0x05]
+# GFX11-FAKE16: v_cmpx_t_f16 v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x1e,0x7d,0x01,0x77,0x39,0x05]
 
 0xea,0xfe,0x1e,0x7d,0x7f,0x00,0x00,0x00
-# GFX11: v_cmpx_t_f16 v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0x1e,0x7d,0x7f,0x00,0x00,0x00]
+# GFX11-REAL16: v_cmpx_t_f16 v127.l, v127.l dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0x1e,0x7d,0x7f,0x00,0x00,0x00]
+# GFX11-FAKE16: v_cmpx_t_f16 v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0x1e,0x7d,0x7f,0x00,0x00,0x00]
+
+0xe9,0xfe,0x1e,0x7d,0x7f,0x77,0x39,0x05
+# GFX11-REAL16: v_cmpx_t_f16 v127.l, v127.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0xfe,0x1e,0x7d,0x7f,0x77,0x39,0x05]
+# GFX11-FAKE16: v_cmpx_t_f16 v127, v127 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0xfe,0x1e,0x7d,0x7f,0x77,0x39,0x05]
+# GFX12-REAL16: v_cmpx_t_f16 v127.l, v127.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0xfe,0x1e,0x7d,0x7f,0x77,0x39,0x05]
+# GFX12-FAKE16: v_cmpx_t_f16 v127, v127 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0xfe,0x1e,0x7d,0x7f,0x77,0x39,0x05]
+
+0xe9,0x04,0x1f,0x7d,0x81,0x77,0x39,0x05
+# GFX11-REAL16: v_cmpx_t_f16 v1.h, v2.h dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x1f,0x7d,0x81,0x77,0x39,0x05]
+# GFX11-FAKE16: v_cmpx_t_f16 v129/*Invalid register, operand has 'VGPR_32_Lo128' register class*/, v130/*Invalid register, operand has 'VGPR_32_Lo128' register class*/ dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x1f,0x7d,0x81,0x77,0x39,0x05]
+# GFX12-REAL16: v_cmpx_t_f16 v1.h, v2.h dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x1f,0x7d,0x81,0x77,0x39,0x05]
+# GFX12-FAKE16: v_cmpx_t_f16 v129/*Invalid register, operand has 'VGPR_32_Lo128' register class*/, v130/*Invalid register, operand has 'VGPR_32_Lo128' register class*/ dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x1f,0x7d,0x81,0x77,0x39,0x05]
+
+0xea,0xfe,0x1f,0x7d,0xff,0x00,0x00,0x00
+# GFX11-REAL16: v_cmpx_t_f16 v127.h, v127.h dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0x1f,0x7d,0xff,0x00,0x00,0x00]
+# GFX11-FAKE16: v_cmpx_t_f16 v255/*Invalid register, operand has 'VGPR_32_Lo128' register class*/, v255/*Invalid register, operand has 'VGPR_32_Lo128' register class*/ dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0x1f,0x7d,0xff,0x00,0x00,0x00]
+# GFX12-REAL16: v_cmpx_t_f16 v127.h, v127.h dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0x1f,0x7d,0xff,0x00,0x00,0x00]
+# GFX12-FAKE16: v_cmpx_t_f16 v255/*Invalid register, operand has 'VGPR_32_Lo128' register class*/, v255/*Invalid register, operand has 'VGPR_32_Lo128' register class*/ dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0x1f,0x7d,0xff,0x00,0x00,0x00]
 
 0xe9,0x04,0x3e,0x7d,0x01,0x77,0x39,0x05
 # GFX11: v_cmpx_t_f32 v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x3e,0x7d,0x01,0x77,0x39,0x05]
@@ -633,10 +913,30 @@
 # GFX11: v_cmpx_t_u32 v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0x9f,0x7d,0xff,0x00,0x00,0x00]
 
 0xe9,0x04,0x10,0x7d,0x01,0x77,0x39,0x05
-# GFX11: v_cmpx_u_f16 v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x10,0x7d,0x01,0x77,0x39,0x05]
+# GFX11-REAL16: v_cmpx_u_f16 v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x10,0x7d,0x01,0x77,0x39,0x05]
+# GFX11-FAKE16: v_cmpx_u_f16 v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x10,0x7d,0x01,0x77,0x39,0x05]
 
 0xea,0xfe,0x10,0x7d,0x7f,0x00,0x00,0x00
-# GFX11: v_cmpx_u_f16 v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0x10,0x7d,0x7f,0x00,0x00,0x00]
+# GFX11-REAL16: v_cmpx_u_f16 v127.l, v127.l dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0x10,0x7d,0x7f,0x00,0x00,0x00]
+# GFX11-FAKE16: v_cmpx_u_f16 v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0x10,0x7d,0x7f,0x00,0x00,0x00]
+
+0xe9,0xfe,0x10,0x7d,0x7f,0x77,0x39,0x05
+# GFX11-REAL16: v_cmpx_u_f16 v127.l, v127.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0xfe,0x10,0x7d,0x7f,0x77,0x39,0x05]
+# GFX11-FAKE16: v_cmpx_u_f16 v127, v127 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0xfe,0x10,0x7d,0x7f,0x77,0x39,0x05]
+# GFX12-REAL16: v_cmpx_u_f16 v127.l, v127.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0xfe,0x10,0x7d,0x7f,0x77,0x39,0x05]
+# GFX12-FAKE16: v_cmpx_u_f16 v127, v127 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0xfe,0x10,0x7d,0x7f,0x77,0x39,0x05]
+
+0xe9,0x04,0x11,0x7d,0x81,0x77,0x39,0x05
+# GFX11-REAL16: v_cmpx_u_f16 v1.h, v2.h dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x11,0x7d,0x81,0x77,0x39,0x05]
+# GFX11-FAKE16: v_cmpx_u_f16 v129/*Invalid register, operand has 'VGPR_32_Lo128' register class*/, v130/*Invalid register, operand has 'VGPR_32_Lo128' register class*/ dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x11,0x7d,0x81,0x77,0x39,0x05]
+# GFX12-REAL16: v_cmpx_u_f16 v1.h, v2.h dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x11,0x7d,0x81,0x77,0x39,0x05]
+# GFX12-FAKE16: v_cmpx_u_f16 v129/*Invalid register, operand has 'VGPR_32_Lo128' register class*/, v130/*Invalid register, operand has 'VGPR_32_Lo128' register class*/ dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x11,0x7d,0x81,0x77,0x39,0x05]
+
+0xea,0xfe,0x11,0x7d,0xff,0x00,0x00,0x00
+# GFX11-REAL16: v_cmpx_u_f16 v127.h, v127.h dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0x11,0x7d,0xff,0x00,0x00,0x00]
+# GFX11-FAKE16: v_cmpx_u_f16 v255/*Invalid register, operand has 'VGPR_32_Lo128' register class*/, v255/*Invalid register, operand has 'VGPR_32_Lo128' register class*/ dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0x11,0x7d,0xff,0x00,0x00,0x00]
+# GFX12-REAL16: v_cmpx_u_f16 v127.h, v127.h dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0x11,0x7d,0xff,0x00,0x00,0x00]
+# GFX12-FAKE16: v_cmpx_u_f16 v255/*Invalid register, operand has 'VGPR_32_Lo128' register class*/, v255/*Invalid register, operand has 'VGPR_32_Lo128' register class*/ dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0x11,0x7d,0xff,0x00,0x00,0x00]
 
 0xe9,0x04,0x30,0x7d,0x01,0x77,0x39,0x05
 # GFX11: v_cmpx_u_f32 v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x30,0x7d,0x01,0x77,0x39,0x05]
diff --git a/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vop3.txt b/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vop3.txt
index 4108fd9c8be62..3d5e78c86bc22 100644
--- a/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vop3.txt
+++ b/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vop3.txt
@@ -472,10 +472,16 @@
 # GFX12: v_alignbyte_b32 v5, s105, s105, s105    ; encoding: [0x05,0x00,0x17,0xd6,0x69,0xd2,0xa4,0x01]
 
 0x05,0x00,0x17,0xd6,0x6a,0xf6,0x0c,0x04
-# GFX12: v_alignbyte_b32 v5, vcc_lo, ttmp15, v3  ; encoding: [0x05,0x00,0x17,0xd6,0x6a,0xf6,0x0c,0x04]
+# W32-REAL16: v_alignbyte_b32 v5, vcc_lo, ttmp15, v3.l ; encoding: [0x05,0x00,0x17,0xd6,0x6a,0xf6,0x0c,0x04]
+# W32-FAKE16: v_alignbyte_b32 v5, vcc_lo, ttmp15, v3  ; encoding: [0x05,0x00,0x17,0xd6,0x6a,0xf6,0x0c,0x04]
+# W64-REAL16: v_alignbyte_b32 v5, vcc_lo, ttmp15, v3.l ; encoding: [0x05,0x00,0x17,0xd6,0x6a,0xf6,0x0c,0x04]
+# W64-FAKE16: v_alignbyte_b32 v5, vcc_lo, ttmp15, v3  ; encoding: [0x05,0x00,0x17,0xd6,0x6a,0xf6,0x0c,0x04]
 
 0x05,0x00,0x17,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf
-# GFX12: v_alignbyte_b32 v5, vcc_hi, 0xaf123456, v255 ; encoding: [0x05,0x00,0x17,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf]
+# W32-REAL16: v_alignbyte_b32 v5, vcc_hi, 0xaf123456, v255.l ; encoding: [0x05,0x00,0x17,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf]
+# W32-FAKE16: v_alignbyte_b32 v5, vcc_hi, 0xaf123456, v255 ; encoding: [0x05,0x00,0x17,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf]
+# W64-REAL16: v_alignbyte_b32 v5, vcc_hi, 0xaf123456, v255.l ; encoding: [0x05,0x00,0x17,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf]
+# W64-FAKE16: v_alignbyte_b32 v5, vcc_hi, 0xaf123456, v255 ; encoding: [0x05,0x00,0x17,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf]
 
 0x05,0x00,0x17,0xd6,0x7b,0xfa,0xed,0x01
 # GFX12: v_alignbyte_b32 v5, ttmp15, src_scc, ttmp15 ; encoding: [0x05,0x00,0x17,0xd6,0x7b,0xfa,0xed,0x01]
@@ -504,6 +510,12 @@
 0xff,0x00,0x17,0xd6,0xff,0xd6,0xf0,0x01,0x56,0x34,0x12,0xaf
 # GFX12: v_alignbyte_b32 v255, 0xaf123456, vcc_hi, null ; encoding: [0xff,0x00,0x17,0xd6,0xff,0xd6,0xf0,0x01,0x56,0x34,0x12,0xaf]
 
+0x05,0x20,0x17,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf
+# W32-REAL16: v_alignbyte_b32 v5, vcc_hi, 0xaf123456, v255.h op_sel:[0,0,1,0] ; encoding: [0x05,0x20,0x17,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf]
+# W32-FAKE16: v_alignbyte_b32 v5, vcc_hi, 0xaf123456, v255 op_sel:[0,0,1,0] ; encoding: [0x05,0x20,0x17,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf]
+# W64-REAL16: v_alignbyte_b32 v5, vcc_hi, 0xaf123456, v255.h op_sel:[0,0,1,0] ; encoding: [0x05,0x20,0x17,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf]
+# W64-FAKE16: v_alignbyte_b32 v5, vcc_hi, 0xaf123456, v255 op_sel:[0,0,1,0] ; encoding: [0x05,0x20,0x17,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf]
+
 0x05,0x00,0x62,0xd7,0x01,0x05,0x02,0x00
 # W32-REAL16: v_and_b16 v5.l, v1.l, v2.l              ; encoding: [0x05,0x00,0x62,0xd7,0x01,0x05,0x02,0x00]
 # W32-FAKE16: v_and_b16 v5, v1, v2                    ; encoding: [0x05,0x00,0x62,0xd7,0x01,0x05,0x02,0x00]
@@ -3205,10 +3217,16 @@
 # W64-FAKE16: v_mad_i16 v255, 0xfe0b, vcc_hi, null op_sel:[0,0,0,1] clamp ; encoding: [0xff,0xc0,0x53,0xd6,0xff,0xd6,0xf0,0x01,0x0b,0xfe,0x00,0x00]
 
 0x05,0x00,0x5a,0xd6,0x01,0x05,0x0e,0x04
-# GFX12: v_mad_i32_i16 v5, v1, v2, v3            ; encoding: [0x05,0x00,0x5a,0xd6,0x01,0x05,0x0e,0x04]
+# W32-REAL16: v_mad_i32_i16 v5, v1.l, v2.l, v3        ; encoding: [0x05,0x00,0x5a,0xd6,0x01,0x05,0x0e,0x04]
+# W32-FAKE16: v_mad_i32_i16 v5, v1, v2, v3            ; encoding: [0x05,0x00,0x5a,0xd6,0x01,0x05,0x0e,0x04]
+# W64-REAL16: v_mad_i32_i16 v5, v1.l, v2.l, v3        ; encoding: [0x05,0x00,0x5a,0xd6,0x01,0x05,0x0e,0x04]
+# W64-FAKE16: v_mad_i32_i16 v5, v1, v2, v3            ; encoding: [0x05,0x00,0x5a,0xd6,0x01,0x05,0x0e,0x04]
 
 0x05,0x00,0x5a,0xd6,0xff,0xff,0x0f,0x00
-# GFX12: v_mad_i32_i16 v5, v255, v255, s3        ; encoding: [0x05,0x00,0x5a,0xd6,0xff,0xff,0x0f,0x00]
+# W32-REAL16: v_mad_i32_i16 v5, v255.l, v255.l, s3    ; encoding: [0x05,0x00,0x5a,0xd6,0xff,0xff,0x0f,0x00]
+# W32-FAKE16: v_mad_i32_i16 v5, v255, v255, s3        ; encoding: [0x05,0x00,0x5a,0xd6,0xff,0xff,0x0f,0x00]
+# W64-REAL16: v_mad_i32_i16 v5, v255.l, v255.l, s3    ; encoding: [0x05,0x00,0x5a,0xd6,0xff,0xff,0x0f,0x00]
+# W64-FAKE16: v_mad_i32_i16 v5, v255, v255, s3        ; encoding: [0x05,0x00,0x5a,0xd6,0xff,0xff,0x0f,0x00]
 
 0x05,0x00,0x5a,0xd6,0x01,0x04,0xfc,0x07
 # GFX12: v_mad_i32_i16 v5, s1, s2, v255          ; encoding: [0x05,0x00,0x5a,0xd6,0x01,0x04,0xfc,0x07]
@@ -3249,6 +3267,18 @@
 0xff,0x90,0x5a,0xd6,0xff,0xd6,0xc0,0x03,0x0b,0xfe,0x00,0x00
 # GFX12: v_mad_i32_i16 v255, 0xfe0b, vcc_hi, 0.5 op_sel:[0,1,0,0] clamp ; encoding: [0xff,0x90,0x5a,0xd6,0xff,0xd6,0xc0,0x03,0x0b,0xfe,0x00,0x00]
 
+0x05,0x08,0x5a,0xd6,0x01,0x05,0x0e,0x04
+# W32-REAL16: v_mad_i32_i16 v5, v1.h, v2.l, v3 op_sel:[1,0,0,0] ; encoding: [0x05,0x08,0x5a,0xd6,0x01,0x05,0x0e,0x04]
+# W32-FAKE16: v_mad_i32_i16 v5, v1, v2, v3 op_sel:[1,0,0,0] ; encoding: [0x05,0x08,0x5a,0xd6,0x01,0x05,0x0e,0x04]
+# W64-REAL16: v_mad_i32_i16 v5, v1.h, v2.l, v3 op_sel:[1,0,0,0] ; encoding: [0x05,0x08,0x5a,0xd6,0x01,0x05,0x0e,0x04]
+# W64-FAKE16: v_mad_i32_i16 v5, v1, v2, v3 op_sel:[1,0,0,0] ; encoding: [0x05,0x08,0x5a,0xd6,0x01,0x05,0x0e,0x04]
+
+0x05,0x10,0x5a,0xd6,0xff,0xff,0x0f,0x00
+# W32-REAL16: v_mad_i32_i16 v5, v255.l, v255.h, s3 op_sel:[0,1,0,0] ; encoding: [0x05,0x10,0x5a,0xd6,0xff,0xff,0x0f,0x00]
+# W32-FAKE16: v_mad_i32_i16 v5, v255, v255, s3 op_sel:[0,1,0,0] ; encoding: [0x05,0x10,0x5a,0xd6,0xff,0xff,0x0f,0x00]
+# W64-REAL16: v_mad_i32_i16 v5, v255.l, v255.h, s3 op_sel:[0,1,0,0] ; encoding: [0x05,0x10,0x5a,0xd6,0xff,0xff,0x0f,0x00]
+# W64-FAKE16: v_mad_i32_i16 v5, v255, v255, s3 op_sel:[0,1,0,0] ; encoding: [0x05,0x10,0x5a,0xd6,0xff,0xff,0x0f,0x00]
+
 0x05,0x00,0x0a,0xd6,0x01,0x05,0x0e,0x00
 # GFX12: v_mad_i32_i24 v5, v1, v2, s3            ; encoding: [0x05,0x00,0x0a,0xd6,0x01,0x05,0x0e,0x00]
 
@@ -3474,10 +3504,16 @@
 # W64-FAKE16: v_mad_u16 v255, 0xfe0b, vcc_hi, null op_sel:[0,0,0,1] clamp ; encoding: [0xff,0xc0,0x41,0xd6,0xff,0xd6,0xf0,0x01,0x0b,0xfe,0x00,0x00]
 
 0x05,0x00,0x59,0xd6,0x01,0x05,0x0e,0x04
-# GFX12: v_mad_u32_u16 v5, v1, v2, v3            ; encoding: [0x05,0x00,0x59,0xd6,0x01,0x05,0x0e,0x04]
+# W32-REAL16: v_mad_u32_u16 v5, v1.l, v2.l, v3        ; encoding: [0x05,0x00,0x59,0xd6,0x01,0x05,0x0e,0x04]
+# W32-FAKE16: v_mad_u32_u16 v5, v1, v2, v3            ; encoding: [0x05,0x00,0x59,0xd6,0x01,0x05,0x0e,0x04]
+# W64-REAL16: v_mad_u32_u16 v5, v1.l, v2.l, v3        ; encoding: [0x05,0x00,0x59,0xd6,0x01,0x05,0x0e,0x04]
+# W64-FAKE16: v_mad_u32_u16 v5, v1, v2, v3            ; encoding: [0x05,0x00,0x59,0xd6,0x01,0x05,0x0e,0x04]
 
 0x05,0x00,0x59,0xd6,0xff,0xff,0x0f,0x00
-# GFX12: v_mad_u32_u16 v5, v255, v255, s3        ; encoding: [0x05,0x00,0x59,0xd6,0xff,0xff,0x0f,0x00]
+# W32-REAL16: v_mad_u32_u16 v5, v255.l, v255.l, s3    ; encoding: [0x05,0x00,0x59,0xd6,0xff,0xff,0x0f,0x00]
+# W32-FAKE16: v_mad_u32_u16 v5, v255, v255, s3        ; encoding: [0x05,0x00,0x59,0xd6,0xff,0xff,0x0f,0x00]
+# W64-REAL16: v_mad_u32_u16 v5, v255.l, v255.l, s3    ; encoding: [0x05,0x00,0x59,0xd6,0xff,0xff,0x0f,0x00]
+# W64-FAKE16: v_mad_u32_u16 v5, v255, v255, s3        ; encoding: [0x05,0x00,0x59,0xd6,0xff,0xff,0x0f,0x00]
 
 0x05,0x00,0x59,0xd6,0x01,0x04,0xfc,0x07
 # GFX12: v_mad_u32_u16 v5, s1, s2, v255          ; encoding: [0x05,0x00,0x59,0xd6,0x01,0x04,0xfc,0x07]
@@ -3518,6 +3554,18 @@
 0xff,0x90,0x59,0xd6,0xff,0xd6,0xc0,0x03,0x0b,0xfe,0x00,0x00
 # GFX12: v_mad_u32_u16 v255, 0xfe0b, vcc_hi, 0.5 op_sel:[0,1,0,0] clamp ; encoding: [0xff,0x90,0x59,0xd6,0xff,0xd6,0xc0,0x03,0x0b,0xfe,0x00,0x00]
 
+0x05,0x08,0x59,0xd6,0x01,0x05,0x0e,0x04
+# W32-REAL16: v_mad_u32_u16 v5, v1.h, v2.l, v3 op_sel:[1,0,0,0] ; encoding: [0x05,0x08,0x59,0xd6,0x01,0x05,0x0e,0x04]
+# W32-FAKE16: v_mad_u32_u16 v5, v1, v2, v3 op_sel:[1,0,0,0] ; encoding: [0x05,0x08,0x59,0xd6,0x01,0x05,0x0e,0x04]
+# W64-REAL16: v_mad_u32_u16 v5, v1.h, v2.l, v3 op_sel:[1,0,0,0] ; encoding: [0x05,0x08,0x59,0xd6,0x01,0x05,0x0e,0x04]
+# W64-FAKE16: v_mad_u32_u16 v5, v1, v2, v3 op_sel:[1,0,0,0] ; encoding: [0x05,0x08,0x59,0xd6,0x01,0x05,0x0e,0x04]
+
+0x05,0x10,0x59,0xd6,0xff,0xff,0x0f,0x00
+# W32-REAL16: v_mad_u32_u16 v5, v255.l, v255.h, s3 op_sel:[0,1,0,0] ; encoding: [0x05,0x10,0x59,0xd6,0xff,0xff,0x0f,0x00]
+# W32-FAKE16: v_mad_u32_u16 v5, v255, v255, s3 op_sel:[0,1,0,0] ; encoding: [0x05,0x10,0x59,0xd6,0xff,0xff,0x0f,0x00]
+# W64-REAL16: v_mad_u32_u16 v5, v255.l, v255.h, s3 op_sel:[0,1,0,0] ; encoding: [0x05,0x10,0x59,0xd6,0xff,0xff,0x0f,0x00]
+# W64-FAKE16: v_mad_u32_u16 v5, v255, v255, s3 op_sel:[0,1,0,0] ; encoding: [0x05,0x10,0x59,0xd6,0xff,0xff,0x0f,0x00]
+
 0x05,0x00,0x0b,0xd6,0x01,0x05,0x0e,0x00
 # GFX12: v_mad_u32_u24 v5, v1, v2, s3            ; encoding: [0x05,0x00,0x0b,0xd6,0x01,0x05,0x0e,0x00]
 
@@ -7920,100 +7968,232 @@
 # GFX12: v_maximum_f32 v5, |exec_hi|, null       ; encoding: [0x05,0x01,0x66,0xd7,0x7f,0xf8,0x00,0x00]
 
 0xcd,0x00,0x67,0xd7,0xc9,0x91,0x03,0x00
-# GFX12: v_minimum_f16 v205, v201, v200          ; encoding: [0xcd,0x00,0x67,0xd7,0xc9,0x91,0x03,0x00]
+# W32-REAL16: v_minimum_f16 v205.l, v201.l, v200.l    ; encoding: [0xcd,0x00,0x67,0xd7,0xc9,0x91,0x03,0x00]
+# W32-FAKE16: v_minimum_f16 v205, v201, v200          ; encoding: [0xcd,0x00,0x67,0xd7,0xc9,0x91,0x03,0x00]
+# W64-REAL16: v_minimum_f16 v205.l, v201.l, v200.l    ; encoding: [0xcd,0x00,0x67,0xd7,0xc9,0x91,0x03,0x00]
+# W64-FAKE16: v_minimum_f16 v205, v201, v200          ; encoding: [0xcd,0x00,0x67,0xd7,0xc9,0x91,0x03,0x00]
 
 0xff,0x03,0x67,0xd7,0xff,0xd6,0x00,0x60,0x12,0xaf,0x00,0x00
-# GFX12: v_minimum_f16 v255, -|0xaf12|, -|vcc_hi| ; encoding: [0xff,0x03,0x67,0xd7,0xff,0xd6,0x00,0x60,0x12,0xaf,0x00,0x00]
+# W32-REAL16: v_minimum_f16 v255.l, -|0xaf12|, -|vcc_hi| ; encoding: [0xff,0x03,0x67,0xd7,0xff,0xd6,0x00,0x60,0x12,0xaf,0x00,0x00]
+# W32-FAKE16: v_minimum_f16 v255, -|0xaf12|, -|vcc_hi| ; encoding: [0xff,0x03,0x67,0xd7,0xff,0xd6,0x00,0x60,0x12,0xaf,0x00,0x00]
+# W64-REAL16: v_minimum_f16 v255.l, -|0xaf12|, -|vcc_hi| ; encoding: [0xff,0x03,0x67,0xd7,0xff,0xd6,0x00,0x60,0x12,0xaf,0x00,0x00]
+# W64-FAKE16: v_minimum_f16 v255, -|0xaf12|, -|vcc_hi| ; encoding: [0xff,0x03,0x67,0xd7,0xff,0xd6,0x00,0x60,0x12,0xaf,0x00,0x00]
 
 0x05,0x00,0x67,0xd7,0xc1,0xfe,0x00,0x00
-# GFX12: v_minimum_f16 v5, -1, exec_hi           ; encoding: [0x05,0x00,0x67,0xd7,0xc1,0xfe,0x00,0x00]
+# W32-REAL16: v_minimum_f16 v5.l, -1, exec_hi         ; encoding: [0x05,0x00,0x67,0xd7,0xc1,0xfe,0x00,0x00]
+# W32-FAKE16: v_minimum_f16 v5, -1, exec_hi           ; encoding: [0x05,0x00,0x67,0xd7,0xc1,0xfe,0x00,0x00]
+# W64-REAL16: v_minimum_f16 v5.l, -1, exec_hi         ; encoding: [0x05,0x00,0x67,0xd7,0xc1,0xfe,0x00,0x00]
+# W64-FAKE16: v_minimum_f16 v5, -1, exec_hi           ; encoding: [0x05,0x00,0x67,0xd7,0xc1,0xfe,0x00,0x00]
 
 0x05,0x02,0x67,0xd7,0xfd,0xd4,0x00,0x20
-# GFX12: v_minimum_f16 v5, -src_scc, |vcc_lo|    ; encoding: [0x05,0x02,0x67,0xd7,0xfd,0xd4,0x00,0x20]
+# W32-REAL16: v_minimum_f16 v5.l, -src_scc, |vcc_lo|  ; encoding: [0x05,0x02,0x67,0xd7,0xfd,0xd4,0x00,0x20]
+# W32-FAKE16: v_minimum_f16 v5, -src_scc, |vcc_lo|    ; encoding: [0x05,0x02,0x67,0xd7,0xfd,0xd4,0x00,0x20]
+# W64-REAL16: v_minimum_f16 v5.l, -src_scc, |vcc_lo|  ; encoding: [0x05,0x02,0x67,0xd7,0xfd,0xd4,0x00,0x20]
+# W64-FAKE16: v_minimum_f16 v5, -src_scc, |vcc_lo|    ; encoding: [0x05,0x02,0x67,0xd7,0xfd,0xd4,0x00,0x20]
 
 0x05,0x00,0x67,0xd7,0xf0,0xfa,0x00,0x40
-# GFX12: v_minimum_f16 v5, 0.5, -m0              ; encoding: [0x05,0x00,0x67,0xd7,0xf0,0xfa,0x00,0x40]
+# W32-REAL16: v_minimum_f16 v5.l, 0.5, -m0            ; encoding: [0x05,0x00,0x67,0xd7,0xf0,0xfa,0x00,0x40]
+# W32-FAKE16: v_minimum_f16 v5, 0.5, -m0              ; encoding: [0x05,0x00,0x67,0xd7,0xf0,0xfa,0x00,0x40]
+# W64-REAL16: v_minimum_f16 v5.l, 0.5, -m0            ; encoding: [0x05,0x00,0x67,0xd7,0xf0,0xfa,0x00,0x40]
+# W64-FAKE16: v_minimum_f16 v5, 0.5, -m0              ; encoding: [0x05,0x00,0x67,0xd7,0xf0,0xfa,0x00,0x40]
 
 0x05,0x00,0x67,0xd7,0x7e,0x82,0x01,0x00
-# GFX12: v_minimum_f16 v5, exec_lo, -1           ; encoding: [0x05,0x00,0x67,0xd7,0x7e,0x82,0x01,0x00]
+# W32-REAL16: v_minimum_f16 v5.l, exec_lo, -1         ; encoding: [0x05,0x00,0x67,0xd7,0x7e,0x82,0x01,0x00]
+# W32-FAKE16: v_minimum_f16 v5, exec_lo, -1           ; encoding: [0x05,0x00,0x67,0xd7,0x7e,0x82,0x01,0x00]
+# W64-REAL16: v_minimum_f16 v5.l, exec_lo, -1         ; encoding: [0x05,0x00,0x67,0xd7,0x7e,0x82,0x01,0x00]
+# W64-FAKE16: v_minimum_f16 v5, exec_lo, -1           ; encoding: [0x05,0x00,0x67,0xd7,0x7e,0x82,0x01,0x00]
 
 0x05,0x00,0x67,0xd7,0x7d,0xe0,0x01,0x00
-# GFX12: v_minimum_f16 v5, m0, 0.5               ; encoding: [0x05,0x00,0x67,0xd7,0x7d,0xe0,0x01,0x00]
+# W32-REAL16: v_minimum_f16 v5.l, m0, 0.5             ; encoding: [0x05,0x00,0x67,0xd7,0x7d,0xe0,0x01,0x00]
+# W32-FAKE16: v_minimum_f16 v5, m0, 0.5               ; encoding: [0x05,0x00,0x67,0xd7,0x7d,0xe0,0x01,0x00]
+# W64-REAL16: v_minimum_f16 v5.l, m0, 0.5             ; encoding: [0x05,0x00,0x67,0xd7,0x7d,0xe0,0x01,0x00]
+# W64-FAKE16: v_minimum_f16 v5, m0, 0.5               ; encoding: [0x05,0x00,0x67,0xd7,0x7d,0xe0,0x01,0x00]
 
 0x05,0x00,0x67,0xd7,0x7c,0xfc,0x00,0x00
-# GFX12: v_minimum_f16 v5, null, exec_lo         ; encoding: [0x05,0x00,0x67,0xd7,0x7c,0xfc,0x00,0x00]
+# W32-REAL16: v_minimum_f16 v5.l, null, exec_lo       ; encoding: [0x05,0x00,0x67,0xd7,0x7c,0xfc,0x00,0x00]
+# W32-FAKE16: v_minimum_f16 v5, null, exec_lo         ; encoding: [0x05,0x00,0x67,0xd7,0x7c,0xfc,0x00,0x00]
+# W64-REAL16: v_minimum_f16 v5.l, null, exec_lo       ; encoding: [0x05,0x00,0x67,0xd7,0x7c,0xfc,0x00,0x00]
+# W64-FAKE16: v_minimum_f16 v5, null, exec_lo         ; encoding: [0x05,0x00,0x67,0xd7,0x7c,0xfc,0x00,0x00]
 
 0x05,0x00,0x67,0xd7,0x01,0x04,0x00,0x00
-# GFX12: v_minimum_f16 v5, s1, s2                ; encoding: [0x05,0x00,0x67,0xd7,0x01,0x04,0x00,0x00]
+# W32-REAL16: v_minimum_f16 v5.l, s1, s2              ; encoding: [0x05,0x00,0x67,0xd7,0x01,0x04,0x00,0x00]
+# W32-FAKE16: v_minimum_f16 v5, s1, s2                ; encoding: [0x05,0x00,0x67,0xd7,0x01,0x04,0x00,0x00]
+# W64-REAL16: v_minimum_f16 v5.l, s1, s2              ; encoding: [0x05,0x00,0x67,0xd7,0x01,0x04,0x00,0x00]
+# W64-FAKE16: v_minimum_f16 v5, s1, s2                ; encoding: [0x05,0x00,0x67,0xd7,0x01,0x04,0x00,0x00]
 
 0x05,0x00,0x67,0xd7,0x69,0xd2,0x00,0x00
-# GFX12: v_minimum_f16 v5, s105, s105            ; encoding: [0x05,0x00,0x67,0xd7,0x69,0xd2,0x00,0x00]
+# W32-REAL16: v_minimum_f16 v5.l, s105, s105          ; encoding: [0x05,0x00,0x67,0xd7,0x69,0xd2,0x00,0x00]
+# W32-FAKE16: v_minimum_f16 v5, s105, s105            ; encoding: [0x05,0x00,0x67,0xd7,0x69,0xd2,0x00,0x00]
+# W64-REAL16: v_minimum_f16 v5.l, s105, s105          ; encoding: [0x05,0x00,0x67,0xd7,0x69,0xd2,0x00,0x00]
+# W64-FAKE16: v_minimum_f16 v5, s105, s105            ; encoding: [0x05,0x00,0x67,0xd7,0x69,0xd2,0x00,0x00]
 
 0x05,0x00,0x67,0xd7,0x7b,0xfa,0x01,0x00
-# GFX12: v_minimum_f16 v5, ttmp15, src_scc       ; encoding: [0x05,0x00,0x67,0xd7,0x7b,0xfa,0x01,0x00]
+# W32-REAL16: v_minimum_f16 v5.l, ttmp15, src_scc     ; encoding: [0x05,0x00,0x67,0xd7,0x7b,0xfa,0x01,0x00]
+# W32-FAKE16: v_minimum_f16 v5, ttmp15, src_scc       ; encoding: [0x05,0x00,0x67,0xd7,0x7b,0xfa,0x01,0x00]
+# W64-REAL16: v_minimum_f16 v5.l, ttmp15, src_scc     ; encoding: [0x05,0x00,0x67,0xd7,0x7b,0xfa,0x01,0x00]
+# W64-FAKE16: v_minimum_f16 v5, ttmp15, src_scc       ; encoding: [0x05,0x00,0x67,0xd7,0x7b,0xfa,0x01,0x00]
 
 0x05,0x00,0x67,0xd7,0x01,0x05,0x02,0x00
-# GFX12: v_minimum_f16 v5, v1, v2                ; encoding: [0x05,0x00,0x67,0xd7,0x01,0x05,0x02,0x00]
+# W32-REAL16: v_minimum_f16 v5.l, v1.l, v2.l          ; encoding: [0x05,0x00,0x67,0xd7,0x01,0x05,0x02,0x00]
+# W32-FAKE16: v_minimum_f16 v5, v1, v2                ; encoding: [0x05,0x00,0x67,0xd7,0x01,0x05,0x02,0x00]
+# W64-REAL16: v_minimum_f16 v5.l, v1.l, v2.l          ; encoding: [0x05,0x00,0x67,0xd7,0x01,0x05,0x02,0x00]
+# W64-FAKE16: v_minimum_f16 v5, v1, v2                ; encoding: [0x05,0x00,0x67,0xd7,0x01,0x05,0x02,0x00]
 
 0x05,0x00,0x67,0xd7,0xff,0xff,0x03,0x00
-# GFX12: v_minimum_f16 v5, v255, v255            ; encoding: [0x05,0x00,0x67,0xd7,0xff,0xff,0x03,0x00]
+# W32-REAL16: v_minimum_f16 v5.l, v255.l, v255.l      ; encoding: [0x05,0x00,0x67,0xd7,0xff,0xff,0x03,0x00]
+# W32-FAKE16: v_minimum_f16 v5, v255, v255            ; encoding: [0x05,0x00,0x67,0xd7,0xff,0xff,0x03,0x00]
+# W64-REAL16: v_minimum_f16 v5.l, v255.l, v255.l      ; encoding: [0x05,0x00,0x67,0xd7,0xff,0xff,0x03,0x00]
+# W64-FAKE16: v_minimum_f16 v5, v255, v255            ; encoding: [0x05,0x00,0x67,0xd7,0xff,0xff,0x03,0x00]
 
 0x05,0x00,0x67,0xd7,0x6b,0xfe,0x01,0x00,0x12,0xaf,0x00,0x00
-# GFX12: v_minimum_f16 v5, vcc_hi, 0xaf12        ; encoding: [0x05,0x00,0x67,0xd7,0x6b,0xfe,0x01,0x00,0x12,0xaf,0x00,0x00]
+# W32-REAL16: v_minimum_f16 v5.l, vcc_hi, 0xaf12      ; encoding: [0x05,0x00,0x67,0xd7,0x6b,0xfe,0x01,0x00,0x12,0xaf,0x00,0x00]
+# W32-FAKE16: v_minimum_f16 v5, vcc_hi, 0xaf12        ; encoding: [0x05,0x00,0x67,0xd7,0x6b,0xfe,0x01,0x00,0x12,0xaf,0x00,0x00]
+# W64-REAL16: v_minimum_f16 v5.l, vcc_hi, 0xaf12      ; encoding: [0x05,0x00,0x67,0xd7,0x6b,0xfe,0x01,0x00,0x12,0xaf,0x00,0x00]
+# W64-FAKE16: v_minimum_f16 v5, vcc_hi, 0xaf12        ; encoding: [0x05,0x00,0x67,0xd7,0x6b,0xfe,0x01,0x00,0x12,0xaf,0x00,0x00]
 
 0x05,0x00,0x67,0xd7,0x6a,0xf6,0x00,0x00
-# GFX12: v_minimum_f16 v5, vcc_lo, ttmp15        ; encoding: [0x05,0x00,0x67,0xd7,0x6a,0xf6,0x00,0x00]
+# W32-REAL16: v_minimum_f16 v5.l, vcc_lo, ttmp15      ; encoding: [0x05,0x00,0x67,0xd7,0x6a,0xf6,0x00,0x00]
+# W32-FAKE16: v_minimum_f16 v5, vcc_lo, ttmp15        ; encoding: [0x05,0x00,0x67,0xd7,0x6a,0xf6,0x00,0x00]
+# W64-REAL16: v_minimum_f16 v5.l, vcc_lo, ttmp15      ; encoding: [0x05,0x00,0x67,0xd7,0x6a,0xf6,0x00,0x00]
+# W64-FAKE16: v_minimum_f16 v5, vcc_lo, ttmp15        ; encoding: [0x05,0x00,0x67,0xd7,0x6a,0xf6,0x00,0x00]
 
 0x05,0x01,0x67,0xd7,0x7f,0xf8,0x00,0x00
-# GFX12: v_minimum_f16 v5, |exec_hi|, null       ; encoding: [0x05,0x01,0x67,0xd7,0x7f,0xf8,0x00,0x00]
+# W32-REAL16: v_minimum_f16 v5.l, |exec_hi|, null     ; encoding: [0x05,0x01,0x67,0xd7,0x7f,0xf8,0x00,0x00]
+# W32-FAKE16: v_minimum_f16 v5, |exec_hi|, null       ; encoding: [0x05,0x01,0x67,0xd7,0x7f,0xf8,0x00,0x00]
+# W64-REAL16: v_minimum_f16 v5.l, |exec_hi|, null     ; encoding: [0x05,0x01,0x67,0xd7,0x7f,0xf8,0x00,0x00]
+# W64-FAKE16: v_minimum_f16 v5, |exec_hi|, null       ; encoding: [0x05,0x01,0x67,0xd7,0x7f,0xf8,0x00,0x00]
+
+0x05,0x08,0x67,0xd7,0x01,0x05,0x02,0x00
+# W32-REAL16: v_minimum_f16 v5.l, v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x05,0x08,0x67,0xd7,0x01,0x05,0x02,0x00]
+# W32-FAKE16: v_minimum_f16 v5, v1, v2 op_sel:[1,0,0] ; encoding: [0x05,0x08,0x67,0xd7,0x01,0x05,0x02,0x00]
+# W64-REAL16: v_minimum_f16 v5.l, v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x05,0x08,0x67,0xd7,0x01,0x05,0x02,0x00]
+# W64-FAKE16: v_minimum_f16 v5, v1, v2 op_sel:[1,0,0] ; encoding: [0x05,0x08,0x67,0xd7,0x01,0x05,0x02,0x00]
+
+0x05,0x10,0x67,0xd7,0xff,0xff,0x03,0x00
+# W32-REAL16: v_minimum_f16 v5.l, v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x05,0x10,0x67,0xd7,0xff,0xff,0x03,0x00]
+# W32-FAKE16: v_minimum_f16 v5, v255, v255 op_sel:[0,1,0] ; encoding: [0x05,0x10,0x67,0xd7,0xff,0xff,0x03,0x00]
+# W64-REAL16: v_minimum_f16 v5.l, v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x05,0x10,0x67,0xd7,0xff,0xff,0x03,0x00]
+# W64-FAKE16: v_minimum_f16 v5, v255, v255 op_sel:[0,1,0] ; encoding: [0x05,0x10,0x67,0xd7,0xff,0xff,0x03,0x00]
+
+0xff,0x40,0x67,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00
+# W32-REAL16: v_minimum_f16 v255.h, 0xfe0b, vcc_hi op_sel:[0,0,1] ; encoding: [0xff,0x40,0x67,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
+# W32-FAKE16: v_minimum_f16 v255, 0xfe0b, vcc_hi op_sel:[0,0,1] ; encoding: [0xff,0x40,0x67,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
+# W64-REAL16: v_minimum_f16 v255.h, 0xfe0b, vcc_hi op_sel:[0,0,1] ; encoding: [0xff,0x40,0x67,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
+# W64-FAKE16: v_minimum_f16 v255, 0xfe0b, vcc_hi op_sel:[0,0,1] ; encoding: [0xff,0x40,0x67,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
 
 0xcd,0x00,0x68,0xd7,0xc9,0x91,0x03,0x00
-# GFX12: v_maximum_f16 v205, v201, v200          ; encoding: [0xcd,0x00,0x68,0xd7,0xc9,0x91,0x03,0x00]
+# W32-REAL16: v_maximum_f16 v205.l, v201.l, v200.l    ; encoding: [0xcd,0x00,0x68,0xd7,0xc9,0x91,0x03,0x00]
+# W32-FAKE16: v_maximum_f16 v205, v201, v200          ; encoding: [0xcd,0x00,0x68,0xd7,0xc9,0x91,0x03,0x00]
+# W64-REAL16: v_maximum_f16 v205.l, v201.l, v200.l    ; encoding: [0xcd,0x00,0x68,0xd7,0xc9,0x91,0x03,0x00]
+# W64-FAKE16: v_maximum_f16 v205, v201, v200          ; encoding: [0xcd,0x00,0x68,0xd7,0xc9,0x91,0x03,0x00]
 
 0xff,0x03,0x68,0xd7,0xff,0xd6,0x00,0x60,0x12,0xaf,0x00,0x00
-# GFX12: v_maximum_f16 v255, -|0xaf12|, -|vcc_hi| ; encoding: [0xff,0x03,0x68,0xd7,0xff,0xd6,0x00,0x60,0x12,0xaf,0x00,0x00]
+# W32-REAL16: v_maximum_f16 v255.l, -|0xaf12|, -|vcc_hi| ; encoding: [0xff,0x03,0x68,0xd7,0xff,0xd6,0x00,0x60,0x12,0xaf,0x00,0x00]
+# W32-FAKE16: v_maximum_f16 v255, -|0xaf12|, -|vcc_hi| ; encoding: [0xff,0x03,0x68,0xd7,0xff,0xd6,0x00,0x60,0x12,0xaf,0x00,0x00]
+# W64-REAL16: v_maximum_f16 v255.l, -|0xaf12|, -|vcc_hi| ; encoding: [0xff,0x03,0x68,0xd7,0xff,0xd6,0x00,0x60,0x12,0xaf,0x00,0x00]
+# W64-FAKE16: v_maximum_f16 v255, -|0xaf12|, -|vcc_hi| ; encoding: [0xff,0x03,0x68,0xd7,0xff,0xd6,0x00,0x60,0x12,0xaf,0x00,0x00]
 
 0x05,0x00,0x68,0xd7,0xc1,0xfe,0x00,0x00
-# GFX12: v_maximum_f16 v5, -1, exec_hi           ; encoding: [0x05,0x00,0x68,0xd7,0xc1,0xfe,0x00,0x00]
+# W32-REAL16: v_maximum_f16 v5.l, -1, exec_hi         ; encoding: [0x05,0x00,0x68,0xd7,0xc1,0xfe,0x00,0x00]
+# W32-FAKE16: v_maximum_f16 v5, -1, exec_hi           ; encoding: [0x05,0x00,0x68,0xd7,0xc1,0xfe,0x00,0x00]
+# W64-REAL16: v_maximum_f16 v5.l, -1, exec_hi         ; encoding: [0x05,0x00,0x68,0xd7,0xc1,0xfe,0x00,0x00]
+# W64-FAKE16: v_maximum_f16 v5, -1, exec_hi           ; encoding: [0x05,0x00,0x68,0xd7,0xc1,0xfe,0x00,0x00]
 
 0x05,0x02,0x68,0xd7,0xfd,0xd4,0x00,0x20
-# GFX12: v_maximum_f16 v5, -src_scc, |vcc_lo|    ; encoding: [0x05,0x02,0x68,0xd7,0xfd,0xd4,0x00,0x20]
+# W32-REAL16: v_maximum_f16 v5.l, -src_scc, |vcc_lo|  ; encoding: [0x05,0x02,0x68,0xd7,0xfd,0xd4,0x00,0x20]
+# W32-FAKE16: v_maximum_f16 v5, -src_scc, |vcc_lo|    ; encoding: [0x05,0x02,0x68,0xd7,0xfd,0xd4,0x00,0x20]
+# W64-REAL16: v_maximum_f16 v5.l, -src_scc, |vcc_lo|  ; encoding: [0x05,0x02,0x68,0xd7,0xfd,0xd4,0x00,0x20]
+# W64-FAKE16: v_maximum_f16 v5, -src_scc, |vcc_lo|    ; encoding: [0x05,0x02,0x68,0xd7,0xfd,0xd4,0x00,0x20]
 
 0x05,0x00,0x68,0xd7,0xf0,0xfa,0x00,0x40
-# GFX12: v_maximum_f16 v5, 0.5, -m0              ; encoding: [0x05,0x00,0x68,0xd7,0xf0,0xfa,0x00,0x40]
+# W32-REAL16: v_maximum_f16 v5.l, 0.5, -m0            ; encoding: [0x05,0x00,0x68,0xd7,0xf0,0xfa,0x00,0x40]
+# W32-FAKE16: v_maximum_f16 v5, 0.5, -m0              ; encoding: [0x05,0x00,0x68,0xd7,0xf0,0xfa,0x00,0x40]
+# W64-REAL16: v_maximum_f16 v5.l, 0.5, -m0            ; encoding: [0x05,0x00,0x68,0xd7,0xf0,0xfa,0x00,0x40]
+# W64-FAKE16: v_maximum_f16 v5, 0.5, -m0              ; encoding: [0x05,0x00,0x68,0xd7,0xf0,0xfa,0x00,0x40]
 
 0x05,0x00,0x68,0xd7,0x7e,0x82,0x01,0x00
-# GFX12: v_maximum_f16 v5, exec_lo, -1           ; encoding: [0x05,0x00,0x68,0xd7,0x7e,0x82,0x01,0x00]
+# W32-REAL16: v_maximum_f16 v5.l, exec_lo, -1         ; encoding: [0x05,0x00,0x68,0xd7,0x7e,0x82,0x01,0x00]
+# W32-FAKE16: v_maximum_f16 v5, exec_lo, -1           ; encoding: [0x05,0x00,0x68,0xd7,0x7e,0x82,0x01,0x00]
+# W64-REAL16: v_maximum_f16 v5.l, exec_lo, -1         ; encoding: [0x05,0x00,0x68,0xd7,0x7e,0x82,0x01,0x00]
+# W64-FAKE16: v_maximum_f16 v5, exec_lo, -1           ; encoding: [0x05,0x00,0x68,0xd7,0x7e,0x82,0x01,0x00]
 
 0x05,0x00,0x68,0xd7,0x7d,0xe0,0x01,0x00
-# GFX12: v_maximum_f16 v5, m0, 0.5               ; encoding: [0x05,0x00,0x68,0xd7,0x7d,0xe0,0x01,0x00]
+# W32-REAL16: v_maximum_f16 v5.l, m0, 0.5             ; encoding: [0x05,0x00,0x68,0xd7,0x7d,0xe0,0x01,0x00]
+# W32-FAKE16: v_maximum_f16 v5, m0, 0.5               ; encoding: [0x05,0x00,0x68,0xd7,0x7d,0xe0,0x01,0x00]
+# W64-REAL16: v_maximum_f16 v5.l, m0, 0.5             ; encoding: [0x05,0x00,0x68,0xd7,0x7d,0xe0,0x01,0x00]
+# W64-FAKE16: v_maximum_f16 v5, m0, 0.5               ; encoding: [0x05,0x00,0x68,0xd7,0x7d,0xe0,0x01,0x00]
 
 0x05,0x00,0x68,0xd7,0x7c,0xfc,0x00,0x00
-# GFX12: v_maximum_f16 v5, null, exec_lo         ; encoding: [0x05,0x00,0x68,0xd7,0x7c,0xfc,0x00,0x00]
+# W32-REAL16: v_maximum_f16 v5.l, null, exec_lo       ; encoding: [0x05,0x00,0x68,0xd7,0x7c,0xfc,0x00,0x00]
+# W32-FAKE16: v_maximum_f16 v5, null, exec_lo         ; encoding: [0x05,0x00,0x68,0xd7,0x7c,0xfc,0x00,0x00]
+# W64-REAL16: v_maximum_f16 v5.l, null, exec_lo       ; encoding: [0x05,0x00,0x68,0xd7,0x7c,0xfc,0x00,0x00]
+# W64-FAKE16: v_maximum_f16 v5, null, exec_lo         ; encoding: [0x05,0x00,0x68,0xd7,0x7c,0xfc,0x00,0x00]
 
 0x05,0x00,0x68,0xd7,0x01,0x04,0x00,0x00
-# GFX12: v_maximum_f16 v5, s1, s2                ; encoding: [0x05,0x00,0x68,0xd7,0x01,0x04,0x00,0x00]
+# W32-REAL16: v_maximum_f16 v5.l, s1, s2              ; encoding: [0x05,0x00,0x68,0xd7,0x01,0x04,0x00,0x00]
+# W32-FAKE16: v_maximum_f16 v5, s1, s2                ; encoding: [0x05,0x00,0x68,0xd7,0x01,0x04,0x00,0x00]
+# W64-REAL16: v_maximum_f16 v5.l, s1, s2              ; encoding: [0x05,0x00,0x68,0xd7,0x01,0x04,0x00,0x00]
+# W64-FAKE16: v_maximum_f16 v5, s1, s2                ; encoding: [0x05,0x00,0x68,0xd7,0x01,0x04,0x00,0x00]
 
 0x05,0x00,0x68,0xd7,0x69,0xd2,0x00,0x00
-# GFX12: v_maximum_f16 v5, s105, s105            ; encoding: [0x05,0x00,0x68,0xd7,0x69,0xd2,0x00,0x00]
+# W32-REAL16: v_maximum_f16 v5.l, s105, s105          ; encoding: [0x05,0x00,0x68,0xd7,0x69,0xd2,0x00,0x00]
+# W32-FAKE16: v_maximum_f16 v5, s105, s105            ; encoding: [0x05,0x00,0x68,0xd7,0x69,0xd2,0x00,0x00]
+# W64-REAL16: v_maximum_f16 v5.l, s105, s105          ; encoding: [0x05,0x00,0x68,0xd7,0x69,0xd2,0x00,0x00]
+# W64-FAKE16: v_maximum_f16 v5, s105, s105            ; encoding: [0x05,0x00,0x68,0xd7,0x69,0xd2,0x00,0x00]
 
 0x05,0x00,0x68,0xd7,0x7b,0xfa,0x01,0x00
-# GFX12: v_maximum_f16 v5, ttmp15, src_scc       ; encoding: [0x05,0x00,0x68,0xd7,0x7b,0xfa,0x01,0x00]
+# W32-REAL16: v_maximum_f16 v5.l, ttmp15, src_scc     ; encoding: [0x05,0x00,0x68,0xd7,0x7b,0xfa,0x01,0x00]
+# W32-FAKE16: v_maximum_f16 v5, ttmp15, src_scc       ; encoding: [0x05,0x00,0x68,0xd7,0x7b,0xfa,0x01,0x00]
+# W64-REAL16: v_maximum_f16 v5.l, ttmp15, src_scc     ; encoding: [0x05,0x00,0x68,0xd7,0x7b,0xfa,0x01,0x00]
+# W64-FAKE16: v_maximum_f16 v5, ttmp15, src_scc       ; encoding: [0x05,0x00,0x68,0xd7,0x7b,0xfa,0x01,0x00]
 
 0x05,0x00,0x68,0xd7,0x01,0x05,0x02,0x00
-# GFX12: v_maximum_f16 v5, v1, v2                ; encoding: [0x05,0x00,0x68,0xd7,0x01,0x05,0x02,0x00]
+# W32-REAL16: v_maximum_f16 v5.l, v1.l, v2.l          ; encoding: [0x05,0x00,0x68,0xd7,0x01,0x05,0x02,0x00]
+# W32-FAKE16: v_maximum_f16 v5, v1, v2                ; encoding: [0x05,0x00,0x68,0xd7,0x01,0x05,0x02,0x00]
+# W64-REAL16: v_maximum_f16 v5.l, v1.l, v2.l          ; encoding: [0x05,0x00,0x68,0xd7,0x01,0x05,0x02,0x00]
+# W64-FAKE16: v_maximum_f16 v5, v1, v2                ; encoding: [0x05,0x00,0x68,0xd7,0x01,0x05,0x02,0x00]
 
 0x05,0x00,0x68,0xd7,0xff,0xff,0x03,0x00
-# GFX12: v_maximum_f16 v5, v255, v255            ; encoding: [0x05,0x00,0x68,0xd7,0xff,0xff,0x03,0x00]
+# W32-REAL16: v_maximum_f16 v5.l, v255.l, v255.l      ; encoding: [0x05,0x00,0x68,0xd7,0xff,0xff,0x03,0x00]
+# W32-FAKE16: v_maximum_f16 v5, v255, v255            ; encoding: [0x05,0x00,0x68,0xd7,0xff,0xff,0x03,0x00]
+# W64-REAL16: v_maximum_f16 v5.l, v255.l, v255.l      ; encoding: [0x05,0x00,0x68,0xd7,0xff,0xff,0x03,0x00]
+# W64-FAKE16: v_maximum_f16 v5, v255, v255            ; encoding: [0x05,0x00,0x68,0xd7,0xff,0xff,0x03,0x00]
 
 0x05,0x00,0x68,0xd7,0x6b,0xfe,0x01,0x00,0x12,0xaf,0x00,0x00
-# GFX12: v_maximum_f16 v5, vcc_hi, 0xaf12        ; encoding: [0x05,0x00,0x68,0xd7,0x6b,0xfe,0x01,0x00,0x12,0xaf,0x00,0x00]
+# W32-REAL16: v_maximum_f16 v5.l, vcc_hi, 0xaf12      ; encoding: [0x05,0x00,0x68,0xd7,0x6b,0xfe,0x01,0x00,0x12,0xaf,0x00,0x00]
+# W32-FAKE16: v_maximum_f16 v5, vcc_hi, 0xaf12        ; encoding: [0x05,0x00,0x68,0xd7,0x6b,0xfe,0x01,0x00,0x12,0xaf,0x00,0x00]
+# W64-REAL16: v_maximum_f16 v5.l, vcc_hi, 0xaf12      ; encoding: [0x05,0x00,0x68,0xd7,0x6b,0xfe,0x01,0x00,0x12,0xaf,0x00,0x00]
+# W64-FAKE16: v_maximum_f16 v5, vcc_hi, 0xaf12        ; encoding: [0x05,0x00,0x68,0xd7,0x6b,0xfe,0x01,0x00,0x12,0xaf,0x00,0x00]
 
 0x05,0x00,0x68,0xd7,0x6a,0xf6,0x00,0x00
-# GFX12: v_maximum_f16 v5, vcc_lo, ttmp15        ; encoding: [0x05,0x00,0x68,0xd7,0x6a,0xf6,0x00,0x00]
+# W32-REAL16: v_maximum_f16 v5.l, vcc_lo, ttmp15      ; encoding: [0x05,0x00,0x68,0xd7,0x6a,0xf6,0x00,0x00]
+# W32-FAKE16: v_maximum_f16 v5, vcc_lo, ttmp15        ; encoding: [0x05,0x00,0x68,0xd7,0x6a,0xf6,0x00,0x00]
+# W64-REAL16: v_maximum_f16 v5.l, vcc_lo, ttmp15      ; encoding: [0x05,0x00,0x68,0xd7,0x6a,0xf6,0x00,0x00]
+# W64-FAKE16: v_maximum_f16 v5, vcc_lo, ttmp15        ; encoding: [0x05,0x00,0x68,0xd7,0x6a,0xf6,0x00,0x00]
 
 0x05,0x01,0x68,0xd7,0x7f,0xf8,0x00,0x00
-# GFX12: v_maximum_f16 v5, |exec_hi|, null       ; encoding: [0x05,0x01,0x68,0xd7,0x7f,0xf8,0x00,0x00]
+# W32-REAL16: v_maximum_f16 v5.l, |exec_hi|, null     ; encoding: [0x05,0x01,0x68,0xd7,0x7f,0xf8,0x00,0x00]
+# W32-FAKE16: v_maximum_f16 v5, |exec_hi|, null       ; encoding: [0x05,0x01,0x68,0xd7,0x7f,0xf8,0x00,0x00]
+# W64-REAL16: v_maximum_f16 v5.l, |exec_hi|, null     ; encoding: [0x05,0x01,0x68,0xd7,0x7f,0xf8,0x00,0x00]
+# W64-FAKE16: v_maximum_f16 v5, |exec_hi|, null       ; encoding: [0x05,0x01,0x68,0xd7,0x7f,0xf8,0x00,0x00]
+
+0x05,0x08,0x68,0xd7,0x01,0x05,0x02,0x00
+# W32-REAL16: v_maximum_f16 v5.l, v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x05,0x08,0x68,0xd7,0x01,0x05,0x02,0x00]
+# W32-FAKE16: v_maximum_f16 v5, v1, v2 op_sel:[1,0,0] ; encoding: [0x05,0x08,0x68,0xd7,0x01,0x05,0x02,0x00]
+# W64-REAL16: v_maximum_f16 v5.l, v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x05,0x08,0x68,0xd7,0x01,0x05,0x02,0x00]
+# W64-FAKE16: v_maximum_f16 v5, v1, v2 op_sel:[1,0,0] ; encoding: [0x05,0x08,0x68,0xd7,0x01,0x05,0x02,0x00]
+
+0x05,0x10,0x68,0xd7,0xff,0xff,0x03,0x00
+# W32-REAL16: v_maximum_f16 v5.l, v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x05,0x10,0x68,0xd7,0xff,0xff,0x03,0x00]
+# W32-FAKE16: v_maximum_f16 v5, v255, v255 op_sel:[0,1,0] ; encoding: [0x05,0x10,0x68,0xd7,0xff,0xff,0x03,0x00]
+# W64-REAL16: v_maximum_f16 v5.l, v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x05,0x10,0x68,0xd7,0xff,0xff,0x03,0x00]
+# W64-FAKE16: v_maximum_f16 v5, v255, v255 op_sel:[0,1,0] ; encoding: [0x05,0x10,0x68,0xd7,0xff,0xff,0x03,0x00]
+
+0xff,0x40,0x68,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00
+# W32-REAL16: v_maximum_f16 v255.h, 0xfe0b, vcc_hi op_sel:[0,0,1] ; encoding: [0xff,0x40,0x68,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
+# W32-FAKE16: v_maximum_f16 v255, 0xfe0b, vcc_hi op_sel:[0,0,1] ; encoding: [0xff,0x40,0x68,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
+# W64-REAL16: v_maximum_f16 v255.h, 0xfe0b, vcc_hi op_sel:[0,0,1] ; encoding: [0xff,0x40,0x68,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
+# W64-FAKE16: v_maximum_f16 v255, 0xfe0b, vcc_hi op_sel:[0,0,1] ; encoding: [0xff,0x40,0x68,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
 
 0xfe,0x03,0x41,0xd7,0x82,0xd4,0x00,0x60
 # GFX12: v_minimum_f64 v[254:255], -|2|, -|vcc|  ; encoding: [0xfe,0x03,0x41,0xd7,0x82,0xd4,0x00,0x60]
@@ -8196,91 +8376,226 @@
 # GFX12: v_minimum3_f32 v5, |exec_lo|, -1, vcc_hi ; encoding: [0x05,0x01,0x2d,0xd6,0x7e,0x82,0xad,0x01]
 
 0x05,0x06,0x30,0xd6,0xc1,0xfe,0xf4,0xc3
-# GFX12: v_maximum3_f16 v5, -1, -|exec_hi|, -|src_scc| ; encoding: [0x05,0x06,0x30,0xd6,0xc1,0xfe,0xf4,0xc3]
+# W32-REAL16: v_maximum3_f16 v5.l, -1, -|exec_hi|, -|src_scc| ; encoding: [0x05,0x06,0x30,0xd6,0xc1,0xfe,0xf4,0xc3]
+# W32-FAKE16: v_maximum3_f16 v5, -1, -|exec_hi|, -|src_scc| ; encoding: [0x05,0x06,0x30,0xd6,0xc1,0xfe,0xf4,0xc3]
+# W64-REAL16: v_maximum3_f16 v5.l, -1, -|exec_hi|, -|src_scc| ; encoding: [0x05,0x06,0x30,0xd6,0xc1,0xfe,0xf4,0xc3]
+# W64-FAKE16: v_maximum3_f16 v5, -1, -|exec_hi|, -|src_scc| ; encoding: [0x05,0x06,0x30,0xd6,0xc1,0xfe,0xf4,0xc3]
 
 0x05,0x02,0x30,0xd6,0xfd,0xd4,0x04,0x23
-# GFX12: v_maximum3_f16 v5, -src_scc, |vcc_lo|, -1 ; encoding: [0x05,0x02,0x30,0xd6,0xfd,0xd4,0x04,0x23]
+# W32-REAL16: v_maximum3_f16 v5.l, -src_scc, |vcc_lo|, -1 ; encoding: [0x05,0x02,0x30,0xd6,0xfd,0xd4,0x04,0x23]
+# W32-FAKE16: v_maximum3_f16 v5, -src_scc, |vcc_lo|, -1 ; encoding: [0x05,0x02,0x30,0xd6,0xfd,0xd4,0x04,0x23]
+# W64-REAL16: v_maximum3_f16 v5.l, -src_scc, |vcc_lo|, -1 ; encoding: [0x05,0x02,0x30,0xd6,0xfd,0xd4,0x04,0x23]
+# W64-FAKE16: v_maximum3_f16 v5, -src_scc, |vcc_lo|, -1 ; encoding: [0x05,0x02,0x30,0xd6,0xfd,0xd4,0x04,0x23]
 
 0x05,0x05,0x30,0xd6,0x7f,0xf8,0xa8,0xa1
-# GFX12: v_maximum3_f16 v5, -|exec_hi|, null, -|vcc_lo| ; encoding: [0x05,0x05,0x30,0xd6,0x7f,0xf8,0xa8,0xa1]
+# W32-REAL16: v_maximum3_f16 v5.l, -|exec_hi|, null, -|vcc_lo| ; encoding: [0x05,0x05,0x30,0xd6,0x7f,0xf8,0xa8,0xa1]
+# W32-FAKE16: v_maximum3_f16 v5, -|exec_hi|, null, -|vcc_lo| ; encoding: [0x05,0x05,0x30,0xd6,0x7f,0xf8,0xa8,0xa1]
+# W64-REAL16: v_maximum3_f16 v5.l, -|exec_hi|, null, -|vcc_lo| ; encoding: [0x05,0x05,0x30,0xd6,0x7f,0xf8,0xa8,0xa1]
+# W64-FAKE16: v_maximum3_f16 v5, -|exec_hi|, null, -|vcc_lo| ; encoding: [0x05,0x05,0x30,0xd6,0x7f,0xf8,0xa8,0xa1]
 
 0x05,0x07,0x30,0xd6,0x7b,0xfa,0xed,0xe1
-# GFX12: v_maximum3_f16 v5, -|ttmp15|, -|src_scc|, -|ttmp15| ; encoding: [0x05,0x07,0x30,0xd6,0x7b,0xfa,0xed,0xe1]
+# W32-REAL16: v_maximum3_f16 v5.l, -|ttmp15|, -|src_scc|, -|ttmp15| ; encoding: [0x05,0x07,0x30,0xd6,0x7b,0xfa,0xed,0xe1]
+# W32-FAKE16: v_maximum3_f16 v5, -|ttmp15|, -|src_scc|, -|ttmp15| ; encoding: [0x05,0x07,0x30,0xd6,0x7b,0xfa,0xed,0xe1]
+# W64-REAL16: v_maximum3_f16 v5.l, -|ttmp15|, -|src_scc|, -|ttmp15| ; encoding: [0x05,0x07,0x30,0xd6,0x7b,0xfa,0xed,0xe1]
+# W64-FAKE16: v_maximum3_f16 v5, -|ttmp15|, -|src_scc|, -|ttmp15| ; encoding: [0x05,0x07,0x30,0xd6,0x7b,0xfa,0xed,0xe1]
 
 0x05,0x00,0x30,0xd6,0xf0,0xfa,0xc0,0x43
-# GFX12: v_maximum3_f16 v5, 0.5, -m0, 0.5        ; encoding: [0x05,0x00,0x30,0xd6,0xf0,0xfa,0xc0,0x43]
+# W32-REAL16: v_maximum3_f16 v5.l, 0.5, -m0, 0.5      ; encoding: [0x05,0x00,0x30,0xd6,0xf0,0xfa,0xc0,0x43]
+# W32-FAKE16: v_maximum3_f16 v5, 0.5, -m0, 0.5        ; encoding: [0x05,0x00,0x30,0xd6,0xf0,0xfa,0xc0,0x43]
+# W64-REAL16: v_maximum3_f16 v5.l, 0.5, -m0, 0.5      ; encoding: [0x05,0x00,0x30,0xd6,0xf0,0xfa,0xc0,0x43]
+# W64-FAKE16: v_maximum3_f16 v5, 0.5, -m0, 0.5        ; encoding: [0x05,0x00,0x30,0xd6,0xf0,0xfa,0xc0,0x43]
 
 0x05,0x00,0x30,0xd6,0x7d,0xe0,0xf5,0x01
-# GFX12: v_maximum3_f16 v5, m0, 0.5, m0          ; encoding: [0x05,0x00,0x30,0xd6,0x7d,0xe0,0xf5,0x01]
+# W32-REAL16: v_maximum3_f16 v5.l, m0, 0.5, m0        ; encoding: [0x05,0x00,0x30,0xd6,0x7d,0xe0,0xf5,0x01]
+# W32-FAKE16: v_maximum3_f16 v5, m0, 0.5, m0          ; encoding: [0x05,0x00,0x30,0xd6,0x7d,0xe0,0xf5,0x01]
+# W64-REAL16: v_maximum3_f16 v5.l, m0, 0.5, m0        ; encoding: [0x05,0x00,0x30,0xd6,0x7d,0xe0,0xf5,0x01]
+# W64-FAKE16: v_maximum3_f16 v5, m0, 0.5, m0          ; encoding: [0x05,0x00,0x30,0xd6,0x7d,0xe0,0xf5,0x01]
 
 0x05,0x04,0x30,0xd6,0x7c,0xfc,0xfc,0x83,0x12,0xaf,0x00,0x00
-# GFX12: v_maximum3_f16 v5, null, exec_lo, -|0xaf12| ; encoding: [0x05,0x04,0x30,0xd6,0x7c,0xfc,0xfc,0x83,0x12,0xaf,0x00,0x00]
+# W32-REAL16: v_maximum3_f16 v5.l, null, exec_lo, -|0xaf12| ; encoding: [0x05,0x04,0x30,0xd6,0x7c,0xfc,0xfc,0x83,0x12,0xaf,0x00,0x00]
+# W32-FAKE16: v_maximum3_f16 v5, null, exec_lo, -|0xaf12| ; encoding: [0x05,0x04,0x30,0xd6,0x7c,0xfc,0xfc,0x83,0x12,0xaf,0x00,0x00]
+# W64-REAL16: v_maximum3_f16 v5.l, null, exec_lo, -|0xaf12| ; encoding: [0x05,0x04,0x30,0xd6,0x7c,0xfc,0xfc,0x83,0x12,0xaf,0x00,0x00]
+# W64-FAKE16: v_maximum3_f16 v5, null, exec_lo, -|0xaf12| ; encoding: [0x05,0x04,0x30,0xd6,0x7c,0xfc,0xfc,0x83,0x12,0xaf,0x00,0x00]
 
 0x05,0x00,0x30,0xd6,0x01,0xfe,0xff,0x01
-# GFX12: v_maximum3_f16 v5, s1, v255, exec_hi    ; encoding: [0x05,0x00,0x30,0xd6,0x01,0xfe,0xff,0x01]
+# W32-REAL16: v_maximum3_f16 v5.l, s1, v255.l, exec_hi ; encoding: [0x05,0x00,0x30,0xd6,0x01,0xfe,0xff,0x01]
+# W32-FAKE16: v_maximum3_f16 v5, s1, v255, exec_hi    ; encoding: [0x05,0x00,0x30,0xd6,0x01,0xfe,0xff,0x01]
+# W64-REAL16: v_maximum3_f16 v5.l, s1, v255.l, exec_hi ; encoding: [0x05,0x00,0x30,0xd6,0x01,0xfe,0xff,0x01]
+# W64-FAKE16: v_maximum3_f16 v5, s1, v255, exec_hi    ; encoding: [0x05,0x00,0x30,0xd6,0x01,0xfe,0xff,0x01]
 
 0x05,0x00,0x30,0xd6,0x69,0xd2,0xf8,0x01
-# GFX12: v_maximum3_f16 v5, s105, s105, exec_lo  ; encoding: [0x05,0x00,0x30,0xd6,0x69,0xd2,0xf8,0x01]
+# W32-REAL16: v_maximum3_f16 v5.l, s105, s105, exec_lo ; encoding: [0x05,0x00,0x30,0xd6,0x69,0xd2,0xf8,0x01]
+# W32-FAKE16: v_maximum3_f16 v5, s105, s105, exec_lo  ; encoding: [0x05,0x00,0x30,0xd6,0x69,0xd2,0xf8,0x01]
+# W64-REAL16: v_maximum3_f16 v5.l, s105, s105, exec_lo ; encoding: [0x05,0x00,0x30,0xd6,0x69,0xd2,0xf8,0x01]
+# W64-FAKE16: v_maximum3_f16 v5, s105, s105, exec_lo  ; encoding: [0x05,0x00,0x30,0xd6,0x69,0xd2,0xf8,0x01]
 
 0x05,0x00,0x30,0xd6,0x01,0x05,0x0e,0x00
-# GFX12: v_maximum3_f16 v5, v1, v2, s3           ; encoding: [0x05,0x00,0x30,0xd6,0x01,0x05,0x0e,0x00]
+# W32-REAL16: v_maximum3_f16 v5.l, v1.l, v2.l, s3     ; encoding: [0x05,0x00,0x30,0xd6,0x01,0x05,0x0e,0x00]
+# W32-FAKE16: v_maximum3_f16 v5, v1, v2, s3           ; encoding: [0x05,0x00,0x30,0xd6,0x01,0x05,0x0e,0x00]
+# W64-REAL16: v_maximum3_f16 v5.l, v1.l, v2.l, s3     ; encoding: [0x05,0x00,0x30,0xd6,0x01,0x05,0x0e,0x00]
+# W64-FAKE16: v_maximum3_f16 v5, v1, v2, s3           ; encoding: [0x05,0x00,0x30,0xd6,0x01,0x05,0x0e,0x00]
 
 0x05,0x00,0x30,0xd6,0xff,0x05,0xa4,0x01
-# GFX12: v_maximum3_f16 v5, v255, s2, s105       ; encoding: [0x05,0x00,0x30,0xd6,0xff,0x05,0xa4,0x01]
+# W32-REAL16: v_maximum3_f16 v5.l, v255.l, s2, s105   ; encoding: [0x05,0x00,0x30,0xd6,0xff,0x05,0xa4,0x01]
+# W32-FAKE16: v_maximum3_f16 v5, v255, s2, s105       ; encoding: [0x05,0x00,0x30,0xd6,0xff,0x05,0xa4,0x01]
+# W64-REAL16: v_maximum3_f16 v5.l, v255.l, s2, s105   ; encoding: [0x05,0x00,0x30,0xd6,0xff,0x05,0xa4,0x01]
+# W64-FAKE16: v_maximum3_f16 v5, v255, s2, s105       ; encoding: [0x05,0x00,0x30,0xd6,0xff,0x05,0xa4,0x01]
 
 0x05,0x00,0x30,0xd6,0x6b,0xfe,0xfd,0x07,0x12,0xaf,0x00,0x00
-# GFX12: v_maximum3_f16 v5, vcc_hi, 0xaf12, v255 ; encoding: [0x05,0x00,0x30,0xd6,0x6b,0xfe,0xfd,0x07,0x12,0xaf,0x00,0x00]
+# W32-REAL16: v_maximum3_f16 v5.l, vcc_hi, 0xaf12, v255.l ; encoding: [0x05,0x00,0x30,0xd6,0x6b,0xfe,0xfd,0x07,0x12,0xaf,0x00,0x00]
+# W32-FAKE16: v_maximum3_f16 v5, vcc_hi, 0xaf12, v255 ; encoding: [0x05,0x00,0x30,0xd6,0x6b,0xfe,0xfd,0x07,0x12,0xaf,0x00,0x00]
+# W64-REAL16: v_maximum3_f16 v5.l, vcc_hi, 0xaf12, v255.l ; encoding: [0x05,0x00,0x30,0xd6,0x6b,0xfe,0xfd,0x07,0x12,0xaf,0x00,0x00]
+# W64-FAKE16: v_maximum3_f16 v5, vcc_hi, 0xaf12, v255 ; encoding: [0x05,0x00,0x30,0xd6,0x6b,0xfe,0xfd,0x07,0x12,0xaf,0x00,0x00]
 
 0x05,0x00,0x30,0xd6,0x6a,0xf6,0x0c,0x04
-# GFX12: v_maximum3_f16 v5, vcc_lo, ttmp15, v3   ; encoding: [0x05,0x00,0x30,0xd6,0x6a,0xf6,0x0c,0x04]
+# W32-REAL16: v_maximum3_f16 v5.l, vcc_lo, ttmp15, v3.l ; encoding: [0x05,0x00,0x30,0xd6,0x6a,0xf6,0x0c,0x04]
+# W32-FAKE16: v_maximum3_f16 v5, vcc_lo, ttmp15, v3   ; encoding: [0x05,0x00,0x30,0xd6,0x6a,0xf6,0x0c,0x04]
+# W64-REAL16: v_maximum3_f16 v5.l, vcc_lo, ttmp15, v3.l ; encoding: [0x05,0x00,0x30,0xd6,0x6a,0xf6,0x0c,0x04]
+# W64-FAKE16: v_maximum3_f16 v5, vcc_lo, ttmp15, v3   ; encoding: [0x05,0x00,0x30,0xd6,0x6a,0xf6,0x0c,0x04]
 
 0x05,0x01,0x30,0xd6,0x7e,0x82,0xad,0x01
-# GFX12: v_maximum3_f16 v5, |exec_lo|, -1, vcc_hi ; encoding: [0x05,0x01,0x30,0xd6,0x7e,0x82,0xad,0x01]
+# W32-REAL16: v_maximum3_f16 v5.l, |exec_lo|, -1, vcc_hi ; encoding: [0x05,0x01,0x30,0xd6,0x7e,0x82,0xad,0x01]
+# W32-FAKE16: v_maximum3_f16 v5, |exec_lo|, -1, vcc_hi ; encoding: [0x05,0x01,0x30,0xd6,0x7e,0x82,0xad,0x01]
+# W64-REAL16: v_maximum3_f16 v5.l, |exec_lo|, -1, vcc_hi ; encoding: [0x05,0x01,0x30,0xd6,0x7e,0x82,0xad,0x01]
+# W64-FAKE16: v_maximum3_f16 v5, |exec_lo|, -1, vcc_hi ; encoding: [0x05,0x01,0x30,0xd6,0x7e,0x82,0xad,0x01]
+
+0x05,0x58,0x30,0xd6,0x01,0x05,0x0e,0x00
+# W32-REAL16: v_maximum3_f16 v5.h, v1.h, v2.h, s3 op_sel:[1,1,0,1] ; encoding: [0x05,0x58,0x30,0xd6,0x01,0x05,0x0e,0x00]
+# W32-FAKE16: v_maximum3_f16 v5, v1, v2, s3 op_sel:[1,1,0,1] ; encoding: [0x05,0x58,0x30,0xd6,0x01,0x05,0x0e,0x00]
+# W64-REAL16: v_maximum3_f16 v5.h, v1.h, v2.h, s3 op_sel:[1,1,0,1] ; encoding: [0x05,0x58,0x30,0xd6,0x01,0x05,0x0e,0x00]
+# W64-FAKE16: v_maximum3_f16 v5, v1, v2, s3 op_sel:[1,1,0,1] ; encoding: [0x05,0x58,0x30,0xd6,0x01,0x05,0x0e,0x00]
+
+0x05,0x08,0x30,0xd6,0xff,0x05,0xa4,0x01
+# W32-REAL16: v_maximum3_f16 v5.l, v255.h, s2, s105 op_sel:[1,0,0,0] ; encoding: [0x05,0x08,0x30,0xd6,0xff,0x05,0xa4,0x01]
+# W32-FAKE16: v_maximum3_f16 v5, v255, s2, s105 op_sel:[1,0,0,0] ; encoding: [0x05,0x08,0x30,0xd6,0xff,0x05,0xa4,0x01]
+# W64-REAL16: v_maximum3_f16 v5.l, v255.h, s2, s105 op_sel:[1,0,0,0] ; encoding: [0x05,0x08,0x30,0xd6,0xff,0x05,0xa4,0x01]
+# W64-FAKE16: v_maximum3_f16 v5, v255, s2, s105 op_sel:[1,0,0,0] ; encoding: [0x05,0x08,0x30,0xd6,0xff,0x05,0xa4,0x01]
+
+0x05,0x42,0x30,0xd6,0xfd,0xd6,0x04,0x23
+# W32-REAL16: v_maximum3_f16 v5.h, -src_scc, |vcc_hi|, -1 op_sel:[0,0,0,1] ; encoding: [0x05,0x42,0x30,0xd6,0xfd,0xd6,0x04,0x23]
+# W32-FAKE16: v_maximum3_f16 v5, -src_scc, |vcc_hi|, -1 op_sel:[0,0,0,1] ; encoding: [0x05,0x42,0x30,0xd6,0xfd,0xd6,0x04,0x23]
+# W64-REAL16: v_maximum3_f16 v5.h, -src_scc, |vcc_hi|, -1 op_sel:[0,0,0,1] ; encoding: [0x05,0x42,0x30,0xd6,0xfd,0xd6,0x04,0x23]
+# W64-FAKE16: v_maximum3_f16 v5, -src_scc, |vcc_hi|, -1 op_sel:[0,0,0,1] ; encoding: [0x05,0x42,0x30,0xd6,0xfd,0xd6,0x04,0x23]
+
+0x05,0x60,0x30,0xd6,0x6b,0xfe,0xfd,0x07,0x12,0xaf,0x00,0x00
+# W32-REAL16: v_maximum3_f16 v5.h, vcc_hi, 0xaf12, v255.h op_sel:[0,0,1,1] ; encoding: [0x05,0x60,0x30,0xd6,0x6b,0xfe,0xfd,0x07,0x12,0xaf,0x00,0x00]
+# W32-FAKE16: v_maximum3_f16 v5, vcc_hi, 0xaf12, v255 op_sel:[0,0,1,1] ; encoding: [0x05,0x60,0x30,0xd6,0x6b,0xfe,0xfd,0x07,0x12,0xaf,0x00,0x00]
+# W64-REAL16: v_maximum3_f16 v5.h, vcc_hi, 0xaf12, v255.h op_sel:[0,0,1,1] ; encoding: [0x05,0x60,0x30,0xd6,0x6b,0xfe,0xfd,0x07,0x12,0xaf,0x00,0x00]
+# W64-FAKE16: v_maximum3_f16 v5, vcc_hi, 0xaf12, v255 op_sel:[0,0,1,1] ; encoding: [0x05,0x60,0x30,0xd6,0x6b,0xfe,0xfd,0x07,0x12,0xaf,0x00,0x00]
 
 0xff,0x83,0x2f,0xd6,0xff,0xd6,0xf0,0x61,0x12,0xaf,0x00,0x00
-# GFX12: v_minimum3_f16 v255, -|0xaf12|, -|vcc_hi|, null clamp ; encoding: [0xff,0x83,0x2f,0xd6,0xff,0xd6,0xf0,0x61,0x12,0xaf,0x00,0x00]
+# W32-REAL16: v_minimum3_f16 v255.l, -|0xaf12|, -|vcc_hi|, null clamp ; encoding: [0xff,0x83,0x2f,0xd6,0xff,0xd6,0xf0,0x61,0x12,0xaf,0x00,0x00]
+# W32-FAKE16: v_minimum3_f16 v255, -|0xaf12|, -|vcc_hi|, null clamp ; encoding: [0xff,0x83,0x2f,0xd6,0xff,0xd6,0xf0,0x61,0x12,0xaf,0x00,0x00]
+# W64-REAL16: v_minimum3_f16 v255.l, -|0xaf12|, -|vcc_hi|, null clamp ; encoding: [0xff,0x83,0x2f,0xd6,0xff,0xd6,0xf0,0x61,0x12,0xaf,0x00,0x00]
+# W64-FAKE16: v_minimum3_f16 v255, -|0xaf12|, -|vcc_hi|, null clamp ; encoding: [0xff,0x83,0x2f,0xd6,0xff,0xd6,0xf0,0x61,0x12,0xaf,0x00,0x00]
 
 0x05,0x06,0x2f,0xd6,0xc1,0xfe,0xf4,0xc3
-# GFX12: v_minimum3_f16 v5, -1, -|exec_hi|, -|src_scc| ; encoding: [0x05,0x06,0x2f,0xd6,0xc1,0xfe,0xf4,0xc3]
+# W32-REAL16: v_minimum3_f16 v5.l, -1, -|exec_hi|, -|src_scc| ; encoding: [0x05,0x06,0x2f,0xd6,0xc1,0xfe,0xf4,0xc3]
+# W32-FAKE16: v_minimum3_f16 v5, -1, -|exec_hi|, -|src_scc| ; encoding: [0x05,0x06,0x2f,0xd6,0xc1,0xfe,0xf4,0xc3]
+# W64-REAL16: v_minimum3_f16 v5.l, -1, -|exec_hi|, -|src_scc| ; encoding: [0x05,0x06,0x2f,0xd6,0xc1,0xfe,0xf4,0xc3]
+# W64-FAKE16: v_minimum3_f16 v5, -1, -|exec_hi|, -|src_scc| ; encoding: [0x05,0x06,0x2f,0xd6,0xc1,0xfe,0xf4,0xc3]
 
 0x05,0x02,0x2f,0xd6,0xfd,0xd4,0x04,0x23
-# GFX12: v_minimum3_f16 v5, -src_scc, |vcc_lo|, -1 ; encoding: [0x05,0x02,0x2f,0xd6,0xfd,0xd4,0x04,0x23]
+# W32-REAL16: v_minimum3_f16 v5.l, -src_scc, |vcc_lo|, -1 ; encoding: [0x05,0x02,0x2f,0xd6,0xfd,0xd4,0x04,0x23]
+# W32-FAKE16: v_minimum3_f16 v5, -src_scc, |vcc_lo|, -1 ; encoding: [0x05,0x02,0x2f,0xd6,0xfd,0xd4,0x04,0x23]
+# W64-REAL16: v_minimum3_f16 v5.l, -src_scc, |vcc_lo|, -1 ; encoding: [0x05,0x02,0x2f,0xd6,0xfd,0xd4,0x04,0x23]
+# W64-FAKE16: v_minimum3_f16 v5, -src_scc, |vcc_lo|, -1 ; encoding: [0x05,0x02,0x2f,0xd6,0xfd,0xd4,0x04,0x23]
 
 0x05,0x05,0x2f,0xd6,0x7f,0xf8,0xa8,0xa1
-# GFX12: v_minimum3_f16 v5, -|exec_hi|, null, -|vcc_lo| ; encoding: [0x05,0x05,0x2f,0xd6,0x7f,0xf8,0xa8,0xa1]
+# W32-REAL16: v_minimum3_f16 v5.l, -|exec_hi|, null, -|vcc_lo| ; encoding: [0x05,0x05,0x2f,0xd6,0x7f,0xf8,0xa8,0xa1]
+# W32-FAKE16: v_minimum3_f16 v5, -|exec_hi|, null, -|vcc_lo| ; encoding: [0x05,0x05,0x2f,0xd6,0x7f,0xf8,0xa8,0xa1]
+# W64-REAL16: v_minimum3_f16 v5.l, -|exec_hi|, null, -|vcc_lo| ; encoding: [0x05,0x05,0x2f,0xd6,0x7f,0xf8,0xa8,0xa1]
+# W64-FAKE16: v_minimum3_f16 v5, -|exec_hi|, null, -|vcc_lo| ; encoding: [0x05,0x05,0x2f,0xd6,0x7f,0xf8,0xa8,0xa1]
 
 0x05,0x07,0x2f,0xd6,0x7b,0xfa,0xed,0xe1
-# GFX12: v_minimum3_f16 v5, -|ttmp15|, -|src_scc|, -|ttmp15| ; encoding: [0x05,0x07,0x2f,0xd6,0x7b,0xfa,0xed,0xe1]
+# W32-REAL16: v_minimum3_f16 v5.l, -|ttmp15|, -|src_scc|, -|ttmp15| ; encoding: [0x05,0x07,0x2f,0xd6,0x7b,0xfa,0xed,0xe1]
+# W32-FAKE16: v_minimum3_f16 v5, -|ttmp15|, -|src_scc|, -|ttmp15| ; encoding: [0x05,0x07,0x2f,0xd6,0x7b,0xfa,0xed,0xe1]
+# W64-REAL16: v_minimum3_f16 v5.l, -|ttmp15|, -|src_scc|, -|ttmp15| ; encoding: [0x05,0x07,0x2f,0xd6,0x7b,0xfa,0xed,0xe1]
+# W64-FAKE16: v_minimum3_f16 v5, -|ttmp15|, -|src_scc|, -|ttmp15| ; encoding: [0x05,0x07,0x2f,0xd6,0x7b,0xfa,0xed,0xe1]
 
 0x05,0x00,0x2f,0xd6,0xf0,0xfa,0xc0,0x43
-# GFX12: v_minimum3_f16 v5, 0.5, -m0, 0.5        ; encoding: [0x05,0x00,0x2f,0xd6,0xf0,0xfa,0xc0,0x43]
+# W32-REAL16: v_minimum3_f16 v5.l, 0.5, -m0, 0.5      ; encoding: [0x05,0x00,0x2f,0xd6,0xf0,0xfa,0xc0,0x43]
+# W32-FAKE16: v_minimum3_f16 v5, 0.5, -m0, 0.5        ; encoding: [0x05,0x00,0x2f,0xd6,0xf0,0xfa,0xc0,0x43]
+# W64-REAL16: v_minimum3_f16 v5.l, 0.5, -m0, 0.5      ; encoding: [0x05,0x00,0x2f,0xd6,0xf0,0xfa,0xc0,0x43]
+# W64-FAKE16: v_minimum3_f16 v5, 0.5, -m0, 0.5        ; encoding: [0x05,0x00,0x2f,0xd6,0xf0,0xfa,0xc0,0x43]
 
 0x05,0x00,0x2f,0xd6,0x7d,0xe0,0xf5,0x01
-# GFX12: v_minimum3_f16 v5, m0, 0.5, m0          ; encoding: [0x05,0x00,0x2f,0xd6,0x7d,0xe0,0xf5,0x01]
+# W32-REAL16: v_minimum3_f16 v5.l, m0, 0.5, m0        ; encoding: [0x05,0x00,0x2f,0xd6,0x7d,0xe0,0xf5,0x01]
+# W32-FAKE16: v_minimum3_f16 v5, m0, 0.5, m0          ; encoding: [0x05,0x00,0x2f,0xd6,0x7d,0xe0,0xf5,0x01]
+# W64-REAL16: v_minimum3_f16 v5.l, m0, 0.5, m0        ; encoding: [0x05,0x00,0x2f,0xd6,0x7d,0xe0,0xf5,0x01]
+# W64-FAKE16: v_minimum3_f16 v5, m0, 0.5, m0          ; encoding: [0x05,0x00,0x2f,0xd6,0x7d,0xe0,0xf5,0x01]
 
 0x05,0x04,0x2f,0xd6,0x7c,0xfc,0xfc,0x83,0x12,0xaf,0x00,0x00
-# GFX12: v_minimum3_f16 v5, null, exec_lo, -|0xaf12| ; encoding: [0x05,0x04,0x2f,0xd6,0x7c,0xfc,0xfc,0x83,0x12,0xaf,0x00,0x00]
+# W32-REAL16: v_minimum3_f16 v5.l, null, exec_lo, -|0xaf12| ; encoding: [0x05,0x04,0x2f,0xd6,0x7c,0xfc,0xfc,0x83,0x12,0xaf,0x00,0x00]
+# W32-FAKE16: v_minimum3_f16 v5, null, exec_lo, -|0xaf12| ; encoding: [0x05,0x04,0x2f,0xd6,0x7c,0xfc,0xfc,0x83,0x12,0xaf,0x00,0x00]
+# W64-REAL16: v_minimum3_f16 v5.l, null, exec_lo, -|0xaf12| ; encoding: [0x05,0x04,0x2f,0xd6,0x7c,0xfc,0xfc,0x83,0x12,0xaf,0x00,0x00]
+# W64-FAKE16: v_minimum3_f16 v5, null, exec_lo, -|0xaf12| ; encoding: [0x05,0x04,0x2f,0xd6,0x7c,0xfc,0xfc,0x83,0x12,0xaf,0x00,0x00]
 
 0x05,0x00,0x2f,0xd6,0x01,0xfe,0xff,0x01
-# GFX12: v_minimum3_f16 v5, s1, v255, exec_hi    ; encoding: [0x05,0x00,0x2f,0xd6,0x01,0xfe,0xff,0x01]
+# W32-REAL16: v_minimum3_f16 v5.l, s1, v255.l, exec_hi ; encoding: [0x05,0x00,0x2f,0xd6,0x01,0xfe,0xff,0x01]
+# W32-FAKE16: v_minimum3_f16 v5, s1, v255, exec_hi    ; encoding: [0x05,0x00,0x2f,0xd6,0x01,0xfe,0xff,0x01]
+# W64-REAL16: v_minimum3_f16 v5.l, s1, v255.l, exec_hi ; encoding: [0x05,0x00,0x2f,0xd6,0x01,0xfe,0xff,0x01]
+# W64-FAKE16: v_minimum3_f16 v5, s1, v255, exec_hi    ; encoding: [0x05,0x00,0x2f,0xd6,0x01,0xfe,0xff,0x01]
 
 0x05,0x00,0x2f,0xd6,0x69,0xd2,0xf8,0x01
-# GFX12: v_minimum3_f16 v5, s105, s105, exec_lo  ; encoding: [0x05,0x00,0x2f,0xd6,0x69,0xd2,0xf8,0x01]
+# W32-REAL16: v_minimum3_f16 v5.l, s105, s105, exec_lo ; encoding: [0x05,0x00,0x2f,0xd6,0x69,0xd2,0xf8,0x01]
+# W32-FAKE16: v_minimum3_f16 v5, s105, s105, exec_lo  ; encoding: [0x05,0x00,0x2f,0xd6,0x69,0xd2,0xf8,0x01]
+# W64-REAL16: v_minimum3_f16 v5.l, s105, s105, exec_lo ; encoding: [0x05,0x00,0x2f,0xd6,0x69,0xd2,0xf8,0x01]
+# W64-FAKE16: v_minimum3_f16 v5, s105, s105, exec_lo  ; encoding: [0x05,0x00,0x2f,0xd6,0x69,0xd2,0xf8,0x01]
 
 0x05,0x00,0x2f,0xd6,0x01,0x05,0x0e,0x00
-# GFX12: v_minimum3_f16 v5, v1, v2, s3           ; encoding: [0x05,0x00,0x2f,0xd6,0x01,0x05,0x0e,0x00]
+# W32-REAL16: v_minimum3_f16 v5.l, v1.l, v2.l, s3     ; encoding: [0x05,0x00,0x2f,0xd6,0x01,0x05,0x0e,0x00]
+# W32-FAKE16: v_minimum3_f16 v5, v1, v2, s3           ; encoding: [0x05,0x00,0x2f,0xd6,0x01,0x05,0x0e,0x00]
+# W64-REAL16: v_minimum3_f16 v5.l, v1.l, v2.l, s3     ; encoding: [0x05,0x00,0x2f,0xd6,0x01,0x05,0x0e,0x00]
+# W64-FAKE16: v_minimum3_f16 v5, v1, v2, s3           ; encoding: [0x05,0x00,0x2f,0xd6,0x01,0x05,0x0e,0x00]
 
 0x05,0x00,0x2f,0xd6,0xff,0x05,0xa4,0x01
-# GFX12: v_minimum3_f16 v5, v255, s2, s105       ; encoding: [0x05,0x00,0x2f,0xd6,0xff,0x05,0xa4,0x01]
+# W32-REAL16: v_minimum3_f16 v5.l, v255.l, s2, s105   ; encoding: [0x05,0x00,0x2f,0xd6,0xff,0x05,0xa4,0x01]
+# W32-FAKE16: v_minimum3_f16 v5, v255, s2, s105       ; encoding: [0x05,0x00,0x2f,0xd6,0xff,0x05,0xa4,0x01]
+# W64-REAL16: v_minimum3_f16 v5.l, v255.l, s2, s105   ; encoding: [0x05,0x00,0x2f,0xd6,0xff,0x05,0xa4,0x01]
+# W64-FAKE16: v_minimum3_f16 v5, v255, s2, s105       ; encoding: [0x05,0x00,0x2f,0xd6,0xff,0x05,0xa4,0x01]
 
 0x05,0x00,0x2f,0xd6,0x6b,0xfe,0xfd,0x07,0x12,0xaf,0x00,0x00
-# GFX12: v_minimum3_f16 v5, vcc_hi, 0xaf12, v255 ; encoding: [0x05,0x00,0x2f,0xd6,0x6b,0xfe,0xfd,0x07,0x12,0xaf,0x00,0x00]
+# W32-REAL16: v_minimum3_f16 v5.l, vcc_hi, 0xaf12, v255.l ; encoding: [0x05,0x00,0x2f,0xd6,0x6b,0xfe,0xfd,0x07,0x12,0xaf,0x00,0x00]
+# W32-FAKE16: v_minimum3_f16 v5, vcc_hi, 0xaf12, v255 ; encoding: [0x05,0x00,0x2f,0xd6,0x6b,0xfe,0xfd,0x07,0x12,0xaf,0x00,0x00]
+# W64-REAL16: v_minimum3_f16 v5.l, vcc_hi, 0xaf12, v255.l ; encoding: [0x05,0x00,0x2f,0xd6,0x6b,0xfe,0xfd,0x07,0x12,0xaf,0x00,0x00]
+# W64-FAKE16: v_minimum3_f16 v5, vcc_hi, 0xaf12, v255 ; encoding: [0x05,0x00,0x2f,0xd6,0x6b,0xfe,0xfd,0x07,0x12,0xaf,0x00,0x00]
 
 0x05,0x00,0x2f,0xd6,0x6a,0xf6,0x0c,0x04
-# GFX12: v_minimum3_f16 v5, vcc_lo, ttmp15, v3   ; encoding: [0x05,0x00,0x2f,0xd6,0x6a,0xf6,0x0c,0x04]
+# W32-REAL16: v_minimum3_f16 v5.l, vcc_lo, ttmp15, v3.l ; encoding: [0x05,0x00,0x2f,0xd6,0x6a,0xf6,0x0c,0x04]
+# W32-FAKE16: v_minimum3_f16 v5, vcc_lo, ttmp15, v3   ; encoding: [0x05,0x00,0x2f,0xd6,0x6a,0xf6,0x0c,0x04]
+# W64-REAL16: v_minimum3_f16 v5.l, vcc_lo, ttmp15, v3.l ; encoding: [0x05,0x00,0x2f,0xd6,0x6a,0xf6,0x0c,0x04]
+# W64-FAKE16: v_minimum3_f16 v5, vcc_lo, ttmp15, v3   ; encoding: [0x05,0x00,0x2f,0xd6,0x6a,0xf6,0x0c,0x04]
 
 0x05,0x01,0x2f,0xd6,0x7e,0x82,0xad,0x01
-# GFX12: v_minimum3_f16 v5, |exec_lo|, -1, vcc_hi ; encoding: [0x05,0x01,0x2f,0xd6,0x7e,0x82,0xad,0x01]
+# W32-REAL16: v_minimum3_f16 v5.l, |exec_lo|, -1, vcc_hi ; encoding: [0x05,0x01,0x2f,0xd6,0x7e,0x82,0xad,0x01]
+# W32-FAKE16: v_minimum3_f16 v5, |exec_lo|, -1, vcc_hi ; encoding: [0x05,0x01,0x2f,0xd6,0x7e,0x82,0xad,0x01]
+# W64-REAL16: v_minimum3_f16 v5.l, |exec_lo|, -1, vcc_hi ; encoding: [0x05,0x01,0x2f,0xd6,0x7e,0x82,0xad,0x01]
+# W64-FAKE16: v_minimum3_f16 v5, |exec_lo|, -1, vcc_hi ; encoding: [0x05,0x01,0x2f,0xd6,0x7e,0x82,0xad,0x01]
+
+0x05,0x58,0x2f,0xd6,0x01,0x05,0x0e,0x00
+# W32-REAL16: v_minimum3_f16 v5.h, v1.h, v2.h, s3 op_sel:[1,1,0,1] ; encoding: [0x05,0x58,0x2f,0xd6,0x01,0x05,0x0e,0x00]
+# W32-FAKE16: v_minimum3_f16 v5, v1, v2, s3 op_sel:[1,1,0,1] ; encoding: [0x05,0x58,0x2f,0xd6,0x01,0x05,0x0e,0x00]
+# W64-REAL16: v_minimum3_f16 v5.h, v1.h, v2.h, s3 op_sel:[1,1,0,1] ; encoding: [0x05,0x58,0x2f,0xd6,0x01,0x05,0x0e,0x00]
+# W64-FAKE16: v_minimum3_f16 v5, v1, v2, s3 op_sel:[1,1,0,1] ; encoding: [0x05,0x58,0x2f,0xd6,0x01,0x05,0x0e,0x00]
+
+0x05,0x08,0x2f,0xd6,0xff,0x05,0xa4,0x01
+# W32-REAL16: v_minimum3_f16 v5.l, v255.h, s2, s105 op_sel:[1,0,0,0] ; encoding: [0x05,0x08,0x2f,0xd6,0xff,0x05,0xa4,0x01]
+# W32-FAKE16: v_minimum3_f16 v5, v255, s2, s105 op_sel:[1,0,0,0] ; encoding: [0x05,0x08,0x2f,0xd6,0xff,0x05,0xa4,0x01]
+# W64-REAL16: v_minimum3_f16 v5.l, v255.h, s2, s105 op_sel:[1,0,0,0] ; encoding: [0x05,0x08,0x2f,0xd6,0xff,0x05,0xa4,0x01]
+# W64-FAKE16: v_minimum3_f16 v5, v255, s2, s105 op_sel:[1,0,0,0] ; encoding: [0x05,0x08,0x2f,0xd6,0xff,0x05,0xa4,0x01]
+
+0x05,0x42,0x2f,0xd6,0xfd,0xd6,0x04,0x23
+# W32-REAL16: v_minimum3_f16 v5.h, -src_scc, |vcc_hi|, -1 op_sel:[0,0,0,1] ; encoding: [0x05,0x42,0x2f,0xd6,0xfd,0xd6,0x04,0x23]
+# W32-FAKE16: v_minimum3_f16 v5, -src_scc, |vcc_hi|, -1 op_sel:[0,0,0,1] ; encoding: [0x05,0x42,0x2f,0xd6,0xfd,0xd6,0x04,0x23]
+# W64-REAL16: v_minimum3_f16 v5.h, -src_scc, |vcc_hi|, -1 op_sel:[0,0,0,1] ; encoding: [0x05,0x42,0x2f,0xd6,0xfd,0xd6,0x04,0x23]
+# W64-FAKE16: v_minimum3_f16 v5, -src_scc, |vcc_hi|, -1 op_sel:[0,0,0,1] ; encoding: [0x05,0x42,0x2f,0xd6,0xfd,0xd6,0x04,0x23]
+
+0x05,0x60,0x2f,0xd6,0x6b,0xfe,0xfd,0x07,0x12,0xaf,0x00,0x00
+# W32-REAL16: v_minimum3_f16 v5.h, vcc_hi, 0xaf12, v255.h op_sel:[0,0,1,1] ; encoding: [0x05,0x60,0x2f,0xd6,0x6b,0xfe,0xfd,0x07,0x12,0xaf,0x00,0x00]
+# W32-FAKE16: v_minimum3_f16 v5, vcc_hi, 0xaf12, v255 op_sel:[0,0,1,1] ; encoding: [0x05,0x60,0x2f,0xd6,0x6b,0xfe,0xfd,0x07,0x12,0xaf,0x00,0x00]
+# W64-REAL16: v_minimum3_f16 v5.h, vcc_hi, 0xaf12, v255.h op_sel:[0,0,1,1] ; encoding: [0x05,0x60,0x2f,0xd6,0x6b,0xfe,0xfd,0x07,0x12,0xaf,0x00,0x00]
+# W64-FAKE16: v_minimum3_f16 v5, vcc_hi, 0xaf12, v255 op_sel:[0,0,1,1] ; encoding: [0x05,0x60,0x2f,0xd6,0x6b,0xfe,0xfd,0x07,0x12,0xaf,0x00,0x00]
 
 0xff,0x83,0x6d,0xd6,0xff,0xd6,0xf0,0x79,0x56,0x34,0x12,0xaf
 # GFX12: v_maximumminimum_f32 v255, -|0xaf123456|, -|vcc_hi|, null clamp div:2 ; encoding: [0xff,0x83,0x6d,0xd6,0xff,0xd6,0xf0,0x79,0x56,0x34,0x12,0xaf]
@@ -8373,91 +8688,226 @@
 # GFX12: v_minimummaximum_f32 v5, |exec_lo|, -1, vcc_hi ; encoding: [0x05,0x01,0x6c,0xd6,0x7e,0x82,0xad,0x01]
 
 0xff,0x83,0x6f,0xd6,0xff,0xd6,0xf0,0x61,0x12,0xaf,0x00,0x00
-# GFX12: v_maximumminimum_f16 v255, -|0xaf12|, -|vcc_hi|, null clamp ; encoding: [0xff,0x83,0x6f,0xd6,0xff,0xd6,0xf0,0x61,0x12,0xaf,0x00,0x00]
+# W32-REAL16: v_maximumminimum_f16 v255.l, -|0xaf12|, -|vcc_hi|, null clamp ; encoding: [0xff,0x83,0x6f,0xd6,0xff,0xd6,0xf0,0x61,0x12,0xaf,0x00,0x00]
+# W32-FAKE16: v_maximumminimum_f16 v255, -|0xaf12|, -|vcc_hi|, null clamp ; encoding: [0xff,0x83,0x6f,0xd6,0xff,0xd6,0xf0,0x61,0x12,0xaf,0x00,0x00]
+# W64-REAL16: v_maximumminimum_f16 v255.l, -|0xaf12|, -|vcc_hi|, null clamp ; encoding: [0xff,0x83,0x6f,0xd6,0xff,0xd6,0xf0,0x61,0x12,0xaf,0x00,0x00]
+# W64-FAKE16: v_maximumminimum_f16 v255, -|0xaf12|, -|vcc_hi|, null clamp ; encoding: [0xff,0x83,0x6f,0xd6,0xff,0xd6,0xf0,0x61,0x12,0xaf,0x00,0x00]
 
 0x05,0x06,0x6f,0xd6,0xc1,0xfe,0xf4,0xc3
-# GFX12: v_maximumminimum_f16 v5, -1, -|exec_hi|, -|src_scc| ; encoding: [0x05,0x06,0x6f,0xd6,0xc1,0xfe,0xf4,0xc3]
+# W32-REAL16: v_maximumminimum_f16 v5.l, -1, -|exec_hi|, -|src_scc| ; encoding: [0x05,0x06,0x6f,0xd6,0xc1,0xfe,0xf4,0xc3]
+# W32-FAKE16: v_maximumminimum_f16 v5, -1, -|exec_hi|, -|src_scc| ; encoding: [0x05,0x06,0x6f,0xd6,0xc1,0xfe,0xf4,0xc3]
+# W64-REAL16: v_maximumminimum_f16 v5.l, -1, -|exec_hi|, -|src_scc| ; encoding: [0x05,0x06,0x6f,0xd6,0xc1,0xfe,0xf4,0xc3]
+# W64-FAKE16: v_maximumminimum_f16 v5, -1, -|exec_hi|, -|src_scc| ; encoding: [0x05,0x06,0x6f,0xd6,0xc1,0xfe,0xf4,0xc3]
 
 0x05,0x02,0x6f,0xd6,0xfd,0xd4,0x04,0x23
-# GFX12: v_maximumminimum_f16 v5, -src_scc, |vcc_lo|, -1 ; encoding: [0x05,0x02,0x6f,0xd6,0xfd,0xd4,0x04,0x23]
+# W32-REAL16: v_maximumminimum_f16 v5.l, -src_scc, |vcc_lo|, -1 ; encoding: [0x05,0x02,0x6f,0xd6,0xfd,0xd4,0x04,0x23]
+# W32-FAKE16: v_maximumminimum_f16 v5, -src_scc, |vcc_lo|, -1 ; encoding: [0x05,0x02,0x6f,0xd6,0xfd,0xd4,0x04,0x23]
+# W64-REAL16: v_maximumminimum_f16 v5.l, -src_scc, |vcc_lo|, -1 ; encoding: [0x05,0x02,0x6f,0xd6,0xfd,0xd4,0x04,0x23]
+# W64-FAKE16: v_maximumminimum_f16 v5, -src_scc, |vcc_lo|, -1 ; encoding: [0x05,0x02,0x6f,0xd6,0xfd,0xd4,0x04,0x23]
 
 0x05,0x05,0x6f,0xd6,0x7f,0xf8,0xa8,0xa1
-# GFX12: v_maximumminimum_f16 v5, -|exec_hi|, null, -|vcc_lo| ; encoding: [0x05,0x05,0x6f,0xd6,0x7f,0xf8,0xa8,0xa1]
+# W32-REAL16: v_maximumminimum_f16 v5.l, -|exec_hi|, null, -|vcc_lo| ; encoding: [0x05,0x05,0x6f,0xd6,0x7f,0xf8,0xa8,0xa1]
+# W32-FAKE16: v_maximumminimum_f16 v5, -|exec_hi|, null, -|vcc_lo| ; encoding: [0x05,0x05,0x6f,0xd6,0x7f,0xf8,0xa8,0xa1]
+# W64-REAL16: v_maximumminimum_f16 v5.l, -|exec_hi|, null, -|vcc_lo| ; encoding: [0x05,0x05,0x6f,0xd6,0x7f,0xf8,0xa8,0xa1]
+# W64-FAKE16: v_maximumminimum_f16 v5, -|exec_hi|, null, -|vcc_lo| ; encoding: [0x05,0x05,0x6f,0xd6,0x7f,0xf8,0xa8,0xa1]
 
 0x05,0x07,0x6f,0xd6,0x7b,0xfa,0xed,0xe1
-# GFX12: v_maximumminimum_f16 v5, -|ttmp15|, -|src_scc|, -|ttmp15| ; encoding: [0x05,0x07,0x6f,0xd6,0x7b,0xfa,0xed,0xe1]
+# W32-REAL16: v_maximumminimum_f16 v5.l, -|ttmp15|, -|src_scc|, -|ttmp15| ; encoding: [0x05,0x07,0x6f,0xd6,0x7b,0xfa,0xed,0xe1]
+# W32-FAKE16: v_maximumminimum_f16 v5, -|ttmp15|, -|src_scc|, -|ttmp15| ; encoding: [0x05,0x07,0x6f,0xd6,0x7b,0xfa,0xed,0xe1]
+# W64-REAL16: v_maximumminimum_f16 v5.l, -|ttmp15|, -|src_scc|, -|ttmp15| ; encoding: [0x05,0x07,0x6f,0xd6,0x7b,0xfa,0xed,0xe1]
+# W64-FAKE16: v_maximumminimum_f16 v5, -|ttmp15|, -|src_scc|, -|ttmp15| ; encoding: [0x05,0x07,0x6f,0xd6,0x7b,0xfa,0xed,0xe1]
 
 0x05,0x00,0x6f,0xd6,0xf0,0xfa,0xc0,0x43
-# GFX12: v_maximumminimum_f16 v5, 0.5, -m0, 0.5  ; encoding: [0x05,0x00,0x6f,0xd6,0xf0,0xfa,0xc0,0x43]
+# W32-REAL16: v_maximumminimum_f16 v5.l, 0.5, -m0, 0.5 ; encoding: [0x05,0x00,0x6f,0xd6,0xf0,0xfa,0xc0,0x43]
+# W32-FAKE16: v_maximumminimum_f16 v5, 0.5, -m0, 0.5  ; encoding: [0x05,0x00,0x6f,0xd6,0xf0,0xfa,0xc0,0x43]
+# W64-REAL16: v_maximumminimum_f16 v5.l, 0.5, -m0, 0.5 ; encoding: [0x05,0x00,0x6f,0xd6,0xf0,0xfa,0xc0,0x43]
+# W64-FAKE16: v_maximumminimum_f16 v5, 0.5, -m0, 0.5  ; encoding: [0x05,0x00,0x6f,0xd6,0xf0,0xfa,0xc0,0x43]
 
 0x05,0x00,0x6f,0xd6,0x7d,0xe0,0xf5,0x01
-# GFX12: v_maximumminimum_f16 v5, m0, 0.5, m0    ; encoding: [0x05,0x00,0x6f,0xd6,0x7d,0xe0,0xf5,0x01]
+# W32-REAL16: v_maximumminimum_f16 v5.l, m0, 0.5, m0  ; encoding: [0x05,0x00,0x6f,0xd6,0x7d,0xe0,0xf5,0x01]
+# W32-FAKE16: v_maximumminimum_f16 v5, m0, 0.5, m0    ; encoding: [0x05,0x00,0x6f,0xd6,0x7d,0xe0,0xf5,0x01]
+# W64-REAL16: v_maximumminimum_f16 v5.l, m0, 0.5, m0  ; encoding: [0x05,0x00,0x6f,0xd6,0x7d,0xe0,0xf5,0x01]
+# W64-FAKE16: v_maximumminimum_f16 v5, m0, 0.5, m0    ; encoding: [0x05,0x00,0x6f,0xd6,0x7d,0xe0,0xf5,0x01]
 
 0x05,0x04,0x6f,0xd6,0x7c,0xfc,0xfc,0x83,0x12,0xaf,0x00,0x00
-# GFX12: v_maximumminimum_f16 v5, null, exec_lo, -|0xaf12| ; encoding: [0x05,0x04,0x6f,0xd6,0x7c,0xfc,0xfc,0x83,0x12,0xaf,0x00,0x00]
+# W32-REAL16: v_maximumminimum_f16 v5.l, null, exec_lo, -|0xaf12| ; encoding: [0x05,0x04,0x6f,0xd6,0x7c,0xfc,0xfc,0x83,0x12,0xaf,0x00,0x00]
+# W32-FAKE16: v_maximumminimum_f16 v5, null, exec_lo, -|0xaf12| ; encoding: [0x05,0x04,0x6f,0xd6,0x7c,0xfc,0xfc,0x83,0x12,0xaf,0x00,0x00]
+# W64-REAL16: v_maximumminimum_f16 v5.l, null, exec_lo, -|0xaf12| ; encoding: [0x05,0x04,0x6f,0xd6,0x7c,0xfc,0xfc,0x83,0x12,0xaf,0x00,0x00]
+# W64-FAKE16: v_maximumminimum_f16 v5, null, exec_lo, -|0xaf12| ; encoding: [0x05,0x04,0x6f,0xd6,0x7c,0xfc,0xfc,0x83,0x12,0xaf,0x00,0x00]
 
 0x05,0x00,0x6f,0xd6,0x01,0xfe,0xff,0x01
-# GFX12: v_maximumminimum_f16 v5, s1, v255, exec_hi ; encoding: [0x05,0x00,0x6f,0xd6,0x01,0xfe,0xff,0x01]
+# W32-REAL16: v_maximumminimum_f16 v5.l, s1, v255.l, exec_hi ; encoding: [0x05,0x00,0x6f,0xd6,0x01,0xfe,0xff,0x01]
+# W32-FAKE16: v_maximumminimum_f16 v5, s1, v255, exec_hi ; encoding: [0x05,0x00,0x6f,0xd6,0x01,0xfe,0xff,0x01]
+# W64-REAL16: v_maximumminimum_f16 v5.l, s1, v255.l, exec_hi ; encoding: [0x05,0x00,0x6f,0xd6,0x01,0xfe,0xff,0x01]
+# W64-FAKE16: v_maximumminimum_f16 v5, s1, v255, exec_hi ; encoding: [0x05,0x00,0x6f,0xd6,0x01,0xfe,0xff,0x01]
 
 0x05,0x00,0x6f,0xd6,0x69,0xd2,0xf8,0x01
-# GFX12: v_maximumminimum_f16 v5, s105, s105, exec_lo ; encoding: [0x05,0x00,0x6f,0xd6,0x69,0xd2,0xf8,0x01]
+# W32-REAL16: v_maximumminimum_f16 v5.l, s105, s105, exec_lo ; encoding: [0x05,0x00,0x6f,0xd6,0x69,0xd2,0xf8,0x01]
+# W32-FAKE16: v_maximumminimum_f16 v5, s105, s105, exec_lo ; encoding: [0x05,0x00,0x6f,0xd6,0x69,0xd2,0xf8,0x01]
+# W64-REAL16: v_maximumminimum_f16 v5.l, s105, s105, exec_lo ; encoding: [0x05,0x00,0x6f,0xd6,0x69,0xd2,0xf8,0x01]
+# W64-FAKE16: v_maximumminimum_f16 v5, s105, s105, exec_lo ; encoding: [0x05,0x00,0x6f,0xd6,0x69,0xd2,0xf8,0x01]
 
 0x05,0x00,0x6f,0xd6,0x01,0x05,0x0e,0x00
-# GFX12: v_maximumminimum_f16 v5, v1, v2, s3     ; encoding: [0x05,0x00,0x6f,0xd6,0x01,0x05,0x0e,0x00]
+# W32-REAL16: v_maximumminimum_f16 v5.l, v1.l, v2.l, s3 ; encoding: [0x05,0x00,0x6f,0xd6,0x01,0x05,0x0e,0x00]
+# W32-FAKE16: v_maximumminimum_f16 v5, v1, v2, s3     ; encoding: [0x05,0x00,0x6f,0xd6,0x01,0x05,0x0e,0x00]
+# W64-REAL16: v_maximumminimum_f16 v5.l, v1.l, v2.l, s3 ; encoding: [0x05,0x00,0x6f,0xd6,0x01,0x05,0x0e,0x00]
+# W64-FAKE16: v_maximumminimum_f16 v5, v1, v2, s3     ; encoding: [0x05,0x00,0x6f,0xd6,0x01,0x05,0x0e,0x00]
 
 0x05,0x00,0x6f,0xd6,0xff,0x05,0xa4,0x01
-# GFX12: v_maximumminimum_f16 v5, v255, s2, s105 ; encoding: [0x05,0x00,0x6f,0xd6,0xff,0x05,0xa4,0x01]
+# W32-REAL16: v_maximumminimum_f16 v5.l, v255.l, s2, s105 ; encoding: [0x05,0x00,0x6f,0xd6,0xff,0x05,0xa4,0x01]
+# W32-FAKE16: v_maximumminimum_f16 v5, v255, s2, s105 ; encoding: [0x05,0x00,0x6f,0xd6,0xff,0x05,0xa4,0x01]
+# W64-REAL16: v_maximumminimum_f16 v5.l, v255.l, s2, s105 ; encoding: [0x05,0x00,0x6f,0xd6,0xff,0x05,0xa4,0x01]
+# W64-FAKE16: v_maximumminimum_f16 v5, v255, s2, s105 ; encoding: [0x05,0x00,0x6f,0xd6,0xff,0x05,0xa4,0x01]
 
 0x05,0x00,0x6f,0xd6,0x6b,0xfe,0xfd,0x07,0x12,0xaf,0x00,0x00
-# GFX12: v_maximumminimum_f16 v5, vcc_hi, 0xaf12, v255 ; encoding: [0x05,0x00,0x6f,0xd6,0x6b,0xfe,0xfd,0x07,0x12,0xaf,0x00,0x00]
+# W32-REAL16: v_maximumminimum_f16 v5.l, vcc_hi, 0xaf12, v255.l ; encoding: [0x05,0x00,0x6f,0xd6,0x6b,0xfe,0xfd,0x07,0x12,0xaf,0x00,0x00]
+# W32-FAKE16: v_maximumminimum_f16 v5, vcc_hi, 0xaf12, v255 ; encoding: [0x05,0x00,0x6f,0xd6,0x6b,0xfe,0xfd,0x07,0x12,0xaf,0x00,0x00]
+# W64-REAL16: v_maximumminimum_f16 v5.l, vcc_hi, 0xaf12, v255.l ; encoding: [0x05,0x00,0x6f,0xd6,0x6b,0xfe,0xfd,0x07,0x12,0xaf,0x00,0x00]
+# W64-FAKE16: v_maximumminimum_f16 v5, vcc_hi, 0xaf12, v255 ; encoding: [0x05,0x00,0x6f,0xd6,0x6b,0xfe,0xfd,0x07,0x12,0xaf,0x00,0x00]
 
 0x05,0x00,0x6f,0xd6,0x6a,0xf6,0x0c,0x04
-# GFX12: v_maximumminimum_f16 v5, vcc_lo, ttmp15, v3 ; encoding: [0x05,0x00,0x6f,0xd6,0x6a,0xf6,0x0c,0x04]
+# W32-REAL16: v_maximumminimum_f16 v5.l, vcc_lo, ttmp15, v3.l ; encoding: [0x05,0x00,0x6f,0xd6,0x6a,0xf6,0x0c,0x04]
+# W32-FAKE16: v_maximumminimum_f16 v5, vcc_lo, ttmp15, v3 ; encoding: [0x05,0x00,0x6f,0xd6,0x6a,0xf6,0x0c,0x04]
+# W64-REAL16: v_maximumminimum_f16 v5.l, vcc_lo, ttmp15, v3.l ; encoding: [0x05,0x00,0x6f,0xd6,0x6a,0xf6,0x0c,0x04]
+# W64-FAKE16: v_maximumminimum_f16 v5, vcc_lo, ttmp15, v3 ; encoding: [0x05,0x00,0x6f,0xd6,0x6a,0xf6,0x0c,0x04]
 
 0x05,0x01,0x6f,0xd6,0x7e,0x82,0xad,0x01
-# GFX12: v_maximumminimum_f16 v5, |exec_lo|, -1, vcc_hi ; encoding: [0x05,0x01,0x6f,0xd6,0x7e,0x82,0xad,0x01]
+# W32-REAL16: v_maximumminimum_f16 v5.l, |exec_lo|, -1, vcc_hi ; encoding: [0x05,0x01,0x6f,0xd6,0x7e,0x82,0xad,0x01]
+# W32-FAKE16: v_maximumminimum_f16 v5, |exec_lo|, -1, vcc_hi ; encoding: [0x05,0x01,0x6f,0xd6,0x7e,0x82,0xad,0x01]
+# W64-REAL16: v_maximumminimum_f16 v5.l, |exec_lo|, -1, vcc_hi ; encoding: [0x05,0x01,0x6f,0xd6,0x7e,0x82,0xad,0x01]
+# W64-FAKE16: v_maximumminimum_f16 v5, |exec_lo|, -1, vcc_hi ; encoding: [0x05,0x01,0x6f,0xd6,0x7e,0x82,0xad,0x01]
+
+0x05,0x58,0x6f,0xd6,0x01,0x05,0x0e,0x00
+# W32-REAL16: v_maximumminimum_f16 v5.h, v1.h, v2.h, s3 op_sel:[1,1,0,1] ; encoding: [0x05,0x58,0x6f,0xd6,0x01,0x05,0x0e,0x00]
+# W32-FAKE16: v_maximumminimum_f16 v5, v1, v2, s3 op_sel:[1,1,0,1] ; encoding: [0x05,0x58,0x6f,0xd6,0x01,0x05,0x0e,0x00]
+# W64-REAL16: v_maximumminimum_f16 v5.h, v1.h, v2.h, s3 op_sel:[1,1,0,1] ; encoding: [0x05,0x58,0x6f,0xd6,0x01,0x05,0x0e,0x00]
+# W64-FAKE16: v_maximumminimum_f16 v5, v1, v2, s3 op_sel:[1,1,0,1] ; encoding: [0x05,0x58,0x6f,0xd6,0x01,0x05,0x0e,0x00]
+
+0x05,0x50,0x6f,0xd6,0x01,0xfe,0xff,0x01
+# W32-REAL16: v_maximumminimum_f16 v5.h, s1, v255.h, exec_hi op_sel:[0,1,0,1] ; encoding: [0x05,0x50,0x6f,0xd6,0x01,0xfe,0xff,0x01]
+# W32-FAKE16: v_maximumminimum_f16 v5, s1, v255, exec_hi op_sel:[0,1,0,1] ; encoding: [0x05,0x50,0x6f,0xd6,0x01,0xfe,0xff,0x01]
+# W64-REAL16: v_maximumminimum_f16 v5.h, s1, v255.h, exec_hi op_sel:[0,1,0,1] ; encoding: [0x05,0x50,0x6f,0xd6,0x01,0xfe,0xff,0x01]
+# W64-FAKE16: v_maximumminimum_f16 v5, s1, v255, exec_hi op_sel:[0,1,0,1] ; encoding: [0x05,0x50,0x6f,0xd6,0x01,0xfe,0xff,0x01]
+
+0x05,0x42,0x6f,0xd6,0xfd,0xd6,0x04,0x23
+# W32-REAL16: v_maximumminimum_f16 v5.h, -src_scc, |vcc_hi|, -1 op_sel:[0,0,0,1] ; encoding: [0x05,0x42,0x6f,0xd6,0xfd,0xd6,0x04,0x23]
+# W32-FAKE16: v_maximumminimum_f16 v5, -src_scc, |vcc_hi|, -1 op_sel:[0,0,0,1] ; encoding: [0x05,0x42,0x6f,0xd6,0xfd,0xd6,0x04,0x23]
+# W64-REAL16: v_maximumminimum_f16 v5.h, -src_scc, |vcc_hi|, -1 op_sel:[0,0,0,1] ; encoding: [0x05,0x42,0x6f,0xd6,0xfd,0xd6,0x04,0x23]
+# W64-FAKE16: v_maximumminimum_f16 v5, -src_scc, |vcc_hi|, -1 op_sel:[0,0,0,1] ; encoding: [0x05,0x42,0x6f,0xd6,0xfd,0xd6,0x04,0x23]
+
+0xff,0xc3,0x6f,0xd6,0xff,0xd6,0xf0,0x61,0x12,0xaf,0x00,0x00
+# W32-REAL16: v_maximumminimum_f16 v255.h, -|0xaf12|, -|vcc_hi|, null op_sel:[0,0,0,1] clamp ; encoding: [0xff,0xc3,0x6f,0xd6,0xff,0xd6,0xf0,0x61,0x12,0xaf,0x00,0x00]
+# W32-FAKE16: v_maximumminimum_f16 v255, -|0xaf12|, -|vcc_hi|, null op_sel:[0,0,0,1] clamp ; encoding: [0xff,0xc3,0x6f,0xd6,0xff,0xd6,0xf0,0x61,0x12,0xaf,0x00,0x00]
+# W64-REAL16: v_maximumminimum_f16 v255.h, -|0xaf12|, -|vcc_hi|, null op_sel:[0,0,0,1] clamp ; encoding: [0xff,0xc3,0x6f,0xd6,0xff,0xd6,0xf0,0x61,0x12,0xaf,0x00,0x00]
+# W64-FAKE16: v_maximumminimum_f16 v255, -|0xaf12|, -|vcc_hi|, null op_sel:[0,0,0,1] clamp ; encoding: [0xff,0xc3,0x6f,0xd6,0xff,0xd6,0xf0,0x61,0x12,0xaf,0x00,0x00]
 
 0x05,0x06,0x6e,0xd6,0xc1,0xfe,0xf4,0xc3
-# GFX12: v_minimummaximum_f16 v5, -1, -|exec_hi|, -|src_scc| ; encoding: [0x05,0x06,0x6e,0xd6,0xc1,0xfe,0xf4,0xc3]
+# W32-REAL16: v_minimummaximum_f16 v5.l, -1, -|exec_hi|, -|src_scc| ; encoding: [0x05,0x06,0x6e,0xd6,0xc1,0xfe,0xf4,0xc3]
+# W32-FAKE16: v_minimummaximum_f16 v5, -1, -|exec_hi|, -|src_scc| ; encoding: [0x05,0x06,0x6e,0xd6,0xc1,0xfe,0xf4,0xc3]
+# W64-REAL16: v_minimummaximum_f16 v5.l, -1, -|exec_hi|, -|src_scc| ; encoding: [0x05,0x06,0x6e,0xd6,0xc1,0xfe,0xf4,0xc3]
+# W64-FAKE16: v_minimummaximum_f16 v5, -1, -|exec_hi|, -|src_scc| ; encoding: [0x05,0x06,0x6e,0xd6,0xc1,0xfe,0xf4,0xc3]
 
 0x05,0x02,0x6e,0xd6,0xfd,0xd4,0x04,0x23
-# GFX12: v_minimummaximum_f16 v5, -src_scc, |vcc_lo|, -1 ; encoding: [0x05,0x02,0x6e,0xd6,0xfd,0xd4,0x04,0x23]
+# W32-REAL16: v_minimummaximum_f16 v5.l, -src_scc, |vcc_lo|, -1 ; encoding: [0x05,0x02,0x6e,0xd6,0xfd,0xd4,0x04,0x23]
+# W32-FAKE16: v_minimummaximum_f16 v5, -src_scc, |vcc_lo|, -1 ; encoding: [0x05,0x02,0x6e,0xd6,0xfd,0xd4,0x04,0x23]
+# W64-REAL16: v_minimummaximum_f16 v5.l, -src_scc, |vcc_lo|, -1 ; encoding: [0x05,0x02,0x6e,0xd6,0xfd,0xd4,0x04,0x23]
+# W64-FAKE16: v_minimummaximum_f16 v5, -src_scc, |vcc_lo|, -1 ; encoding: [0x05,0x02,0x6e,0xd6,0xfd,0xd4,0x04,0x23]
 
 0x05,0x05,0x6e,0xd6,0x7f,0xf8,0xa8,0xa1
-# GFX12: v_minimummaximum_f16 v5, -|exec_hi|, null, -|vcc_lo| ; encoding: [0x05,0x05,0x6e,0xd6,0x7f,0xf8,0xa8,0xa1]
+# W32-REAL16: v_minimummaximum_f16 v5.l, -|exec_hi|, null, -|vcc_lo| ; encoding: [0x05,0x05,0x6e,0xd6,0x7f,0xf8,0xa8,0xa1]
+# W32-FAKE16: v_minimummaximum_f16 v5, -|exec_hi|, null, -|vcc_lo| ; encoding: [0x05,0x05,0x6e,0xd6,0x7f,0xf8,0xa8,0xa1]
+# W64-REAL16: v_minimummaximum_f16 v5.l, -|exec_hi|, null, -|vcc_lo| ; encoding: [0x05,0x05,0x6e,0xd6,0x7f,0xf8,0xa8,0xa1]
+# W64-FAKE16: v_minimummaximum_f16 v5, -|exec_hi|, null, -|vcc_lo| ; encoding: [0x05,0x05,0x6e,0xd6,0x7f,0xf8,0xa8,0xa1]
 
 0x05,0x07,0x6e,0xd6,0x7b,0xfa,0xed,0xe1
-# GFX12: v_minimummaximum_f16 v5, -|ttmp15|, -|src_scc|, -|ttmp15| ; encoding: [0x05,0x07,0x6e,0xd6,0x7b,0xfa,0xed,0xe1]
+# W32-REAL16: v_minimummaximum_f16 v5.l, -|ttmp15|, -|src_scc|, -|ttmp15| ; encoding: [0x05,0x07,0x6e,0xd6,0x7b,0xfa,0xed,0xe1]
+# W32-FAKE16: v_minimummaximum_f16 v5, -|ttmp15|, -|src_scc|, -|ttmp15| ; encoding: [0x05,0x07,0x6e,0xd6,0x7b,0xfa,0xed,0xe1]
+# W64-REAL16: v_minimummaximum_f16 v5.l, -|ttmp15|, -|src_scc|, -|ttmp15| ; encoding: [0x05,0x07,0x6e,0xd6,0x7b,0xfa,0xed,0xe1]
+# W64-FAKE16: v_minimummaximum_f16 v5, -|ttmp15|, -|src_scc|, -|ttmp15| ; encoding: [0x05,0x07,0x6e,0xd6,0x7b,0xfa,0xed,0xe1]
 
 0x05,0x00,0x6e,0xd6,0xf0,0xfa,0xc0,0x43
-# GFX12: v_minimummaximum_f16 v5, 0.5, -m0, 0.5  ; encoding: [0x05,0x00,0x6e,0xd6,0xf0,0xfa,0xc0,0x43]
+# W32-REAL16: v_minimummaximum_f16 v5.l, 0.5, -m0, 0.5 ; encoding: [0x05,0x00,0x6e,0xd6,0xf0,0xfa,0xc0,0x43]
+# W32-FAKE16: v_minimummaximum_f16 v5, 0.5, -m0, 0.5  ; encoding: [0x05,0x00,0x6e,0xd6,0xf0,0xfa,0xc0,0x43]
+# W64-REAL16: v_minimummaximum_f16 v5.l, 0.5, -m0, 0.5 ; encoding: [0x05,0x00,0x6e,0xd6,0xf0,0xfa,0xc0,0x43]
+# W64-FAKE16: v_minimummaximum_f16 v5, 0.5, -m0, 0.5  ; encoding: [0x05,0x00,0x6e,0xd6,0xf0,0xfa,0xc0,0x43]
 
 0x05,0x00,0x6e,0xd6,0x7d,0xe0,0xf5,0x01
-# GFX12: v_minimummaximum_f16 v5, m0, 0.5, m0    ; encoding: [0x05,0x00,0x6e,0xd6,0x7d,0xe0,0xf5,0x01]
+# W32-REAL16: v_minimummaximum_f16 v5.l, m0, 0.5, m0  ; encoding: [0x05,0x00,0x6e,0xd6,0x7d,0xe0,0xf5,0x01]
+# W32-FAKE16: v_minimummaximum_f16 v5, m0, 0.5, m0    ; encoding: [0x05,0x00,0x6e,0xd6,0x7d,0xe0,0xf5,0x01]
+# W64-REAL16: v_minimummaximum_f16 v5.l, m0, 0.5, m0  ; encoding: [0x05,0x00,0x6e,0xd6,0x7d,0xe0,0xf5,0x01]
+# W64-FAKE16: v_minimummaximum_f16 v5, m0, 0.5, m0    ; encoding: [0x05,0x00,0x6e,0xd6,0x7d,0xe0,0xf5,0x01]
 
 0x05,0x04,0x6e,0xd6,0x7c,0xfc,0xfc,0x83,0x12,0xaf,0x00,0x00
-# GFX12: v_minimummaximum_f16 v5, null, exec_lo, -|0xaf12| ; encoding: [0x05,0x04,0x6e,0xd6,0x7c,0xfc,0xfc,0x83,0x12,0xaf,0x00,0x00]
+# W32-REAL16: v_minimummaximum_f16 v5.l, null, exec_lo, -|0xaf12| ; encoding: [0x05,0x04,0x6e,0xd6,0x7c,0xfc,0xfc,0x83,0x12,0xaf,0x00,0x00]
+# W32-FAKE16: v_minimummaximum_f16 v5, null, exec_lo, -|0xaf12| ; encoding: [0x05,0x04,0x6e,0xd6,0x7c,0xfc,0xfc,0x83,0x12,0xaf,0x00,0x00]
+# W64-REAL16: v_minimummaximum_f16 v5.l, null, exec_lo, -|0xaf12| ; encoding: [0x05,0x04,0x6e,0xd6,0x7c,0xfc,0xfc,0x83,0x12,0xaf,0x00,0x00]
+# W64-FAKE16: v_minimummaximum_f16 v5, null, exec_lo, -|0xaf12| ; encoding: [0x05,0x04,0x6e,0xd6,0x7c,0xfc,0xfc,0x83,0x12,0xaf,0x00,0x00]
 
 0x05,0x00,0x6e,0xd6,0x01,0xfe,0xff,0x01
-# GFX12: v_minimummaximum_f16 v5, s1, v255, exec_hi ; encoding: [0x05,0x00,0x6e,0xd6,0x01,0xfe,0xff,0x01]
+# W32-REAL16: v_minimummaximum_f16 v5.l, s1, v255.l, exec_hi ; encoding: [0x05,0x00,0x6e,0xd6,0x01,0xfe,0xff,0x01]
+# W32-FAKE16: v_minimummaximum_f16 v5, s1, v255, exec_hi ; encoding: [0x05,0x00,0x6e,0xd6,0x01,0xfe,0xff,0x01]
+# W64-REAL16: v_minimummaximum_f16 v5.l, s1, v255.l, exec_hi ; encoding: [0x05,0x00,0x6e,0xd6,0x01,0xfe,0xff,0x01]
+# W64-FAKE16: v_minimummaximum_f16 v5, s1, v255, exec_hi ; encoding: [0x05,0x00,0x6e,0xd6,0x01,0xfe,0xff,0x01]
 
 0x05,0x00,0x6e,0xd6,0x69,0xd2,0xf8,0x01
-# GFX12: v_minimummaximum_f16 v5, s105, s105, exec_lo ; encoding: [0x05,0x00,0x6e,0xd6,0x69,0xd2,0xf8,0x01]
+# W32-REAL16: v_minimummaximum_f16 v5.l, s105, s105, exec_lo ; encoding: [0x05,0x00,0x6e,0xd6,0x69,0xd2,0xf8,0x01]
+# W32-FAKE16: v_minimummaximum_f16 v5, s105, s105, exec_lo ; encoding: [0x05,0x00,0x6e,0xd6,0x69,0xd2,0xf8,0x01]
+# W64-REAL16: v_minimummaximum_f16 v5.l, s105, s105, exec_lo ; encoding: [0x05,0x00,0x6e,0xd6,0x69,0xd2,0xf8,0x01]
+# W64-FAKE16: v_minimummaximum_f16 v5, s105, s105, exec_lo ; encoding: [0x05,0x00,0x6e,0xd6,0x69,0xd2,0xf8,0x01]
 
 0x05,0x00,0x6e,0xd6,0x01,0x05,0x0e,0x00
-# GFX12: v_minimummaximum_f16 v5, v1, v2, s3     ; encoding: [0x05,0x00,0x6e,0xd6,0x01,0x05,0x0e,0x00]
+# W32-REAL16: v_minimummaximum_f16 v5.l, v1.l, v2.l, s3 ; encoding: [0x05,0x00,0x6e,0xd6,0x01,0x05,0x0e,0x00]
+# W32-FAKE16: v_minimummaximum_f16 v5, v1, v2, s3     ; encoding: [0x05,0x00,0x6e,0xd6,0x01,0x05,0x0e,0x00]
+# W64-REAL16: v_minimummaximum_f16 v5.l, v1.l, v2.l, s3 ; encoding: [0x05,0x00,0x6e,0xd6,0x01,0x05,0x0e,0x00]
+# W64-FAKE16: v_minimummaximum_f16 v5, v1, v2, s3     ; encoding: [0x05,0x00,0x6e,0xd6,0x01,0x05,0x0e,0x00]
 
 0x05,0x00,0x6e,0xd6,0xff,0x05,0xa4,0x01
-# GFX12: v_minimummaximum_f16 v5, v255, s2, s105 ; encoding: [0x05,0x00,0x6e,0xd6,0xff,0x05,0xa4,0x01]
+# W32-REAL16: v_minimummaximum_f16 v5.l, v255.l, s2, s105 ; encoding: [0x05,0x00,0x6e,0xd6,0xff,0x05,0xa4,0x01]
+# W32-FAKE16: v_minimummaximum_f16 v5, v255, s2, s105 ; encoding: [0x05,0x00,0x6e,0xd6,0xff,0x05,0xa4,0x01]
+# W64-REAL16: v_minimummaximum_f16 v5.l, v255.l, s2, s105 ; encoding: [0x05,0x00,0x6e,0xd6,0xff,0x05,0xa4,0x01]
+# W64-FAKE16: v_minimummaximum_f16 v5, v255, s2, s105 ; encoding: [0x05,0x00,0x6e,0xd6,0xff,0x05,0xa4,0x01]
 
 0x05,0x00,0x6e,0xd6,0x6b,0xfe,0xfd,0x07,0x12,0xaf,0x00,0x00
-# GFX12: v_minimummaximum_f16 v5, vcc_hi, 0xaf12, v255 ; encoding: [0x05,0x00,0x6e,0xd6,0x6b,0xfe,0xfd,0x07,0x12,0xaf,0x00,0x00]
+# W32-REAL16: v_minimummaximum_f16 v5.l, vcc_hi, 0xaf12, v255.l ; encoding: [0x05,0x00,0x6e,0xd6,0x6b,0xfe,0xfd,0x07,0x12,0xaf,0x00,0x00]
+# W32-FAKE16: v_minimummaximum_f16 v5, vcc_hi, 0xaf12, v255 ; encoding: [0x05,0x00,0x6e,0xd6,0x6b,0xfe,0xfd,0x07,0x12,0xaf,0x00,0x00]
+# W64-REAL16: v_minimummaximum_f16 v5.l, vcc_hi, 0xaf12, v255.l ; encoding: [0x05,0x00,0x6e,0xd6,0x6b,0xfe,0xfd,0x07,0x12,0xaf,0x00,0x00]
+# W64-FAKE16: v_minimummaximum_f16 v5, vcc_hi, 0xaf12, v255 ; encoding: [0x05,0x00,0x6e,0xd6,0x6b,0xfe,0xfd,0x07,0x12,0xaf,0x00,0x00]
 
 0x05,0x00,0x6e,0xd6,0x6a,0xf6,0x0c,0x04
-# GFX12: v_minimummaximum_f16 v5, vcc_lo, ttmp15, v3 ; encoding: [0x05,0x00,0x6e,0xd6,0x6a,0xf6,0x0c,0x04]
+# W32-REAL16: v_minimummaximum_f16 v5.l, vcc_lo, ttmp15, v3.l ; encoding: [0x05,0x00,0x6e,0xd6,0x6a,0xf6,0x0c,0x04]
+# W32-FAKE16: v_minimummaximum_f16 v5, vcc_lo, ttmp15, v3 ; encoding: [0x05,0x00,0x6e,0xd6,0x6a,0xf6,0x0c,0x04]
+# W64-REAL16: v_minimummaximum_f16 v5.l, vcc_lo, ttmp15, v3.l ; encoding: [0x05,0x00,0x6e,0xd6,0x6a,0xf6,0x0c,0x04]
+# W64-FAKE16: v_minimummaximum_f16 v5, vcc_lo, ttmp15, v3 ; encoding: [0x05,0x00,0x6e,0xd6,0x6a,0xf6,0x0c,0x04]
 
 0x05,0x01,0x6e,0xd6,0x7e,0x82,0xad,0x01
-# GFX12: v_minimummaximum_f16 v5, |exec_lo|, -1, vcc_hi ; encoding: [0x05,0x01,0x6e,0xd6,0x7e,0x82,0xad,0x01]
+# W32-REAL16: v_minimummaximum_f16 v5.l, |exec_lo|, -1, vcc_hi ; encoding: [0x05,0x01,0x6e,0xd6,0x7e,0x82,0xad,0x01]
+# W32-FAKE16: v_minimummaximum_f16 v5, |exec_lo|, -1, vcc_hi ; encoding: [0x05,0x01,0x6e,0xd6,0x7e,0x82,0xad,0x01]
+# W64-REAL16: v_minimummaximum_f16 v5.l, |exec_lo|, -1, vcc_hi ; encoding: [0x05,0x01,0x6e,0xd6,0x7e,0x82,0xad,0x01]
+# W64-FAKE16: v_minimummaximum_f16 v5, |exec_lo|, -1, vcc_hi ; encoding: [0x05,0x01,0x6e,0xd6,0x7e,0x82,0xad,0x01]
+
+0x05,0x58,0x6e,0xd6,0x01,0x05,0x0e,0x00
+# W32-REAL16: v_minimummaximum_f16 v5.h, v1.h, v2.h, s3 op_sel:[1,1,0,1] ; encoding: [0x05,0x58,0x6e,0xd6,0x01,0x05,0x0e,0x00]
+# W32-FAKE16: v_minimummaximum_f16 v5, v1, v2, s3 op_sel:[1,1,0,1] ; encoding: [0x05,0x58,0x6e,0xd6,0x01,0x05,0x0e,0x00]
+# W64-REAL16: v_minimummaximum_f16 v5.h, v1.h, v2.h, s3 op_sel:[1,1,0,1] ; encoding: [0x05,0x58,0x6e,0xd6,0x01,0x05,0x0e,0x00]
+# W64-FAKE16: v_minimummaximum_f16 v5, v1, v2, s3 op_sel:[1,1,0,1] ; encoding: [0x05,0x58,0x6e,0xd6,0x01,0x05,0x0e,0x00]
+
+0x05,0x50,0x6e,0xd6,0x01,0xfe,0xff,0x01
+# W32-REAL16: v_minimummaximum_f16 v5.h, s1, v255.h, exec_hi op_sel:[0,1,0,1] ; encoding: [0x05,0x50,0x6e,0xd6,0x01,0xfe,0xff,0x01]
+# W32-FAKE16: v_minimummaximum_f16 v5, s1, v255, exec_hi op_sel:[0,1,0,1] ; encoding: [0x05,0x50,0x6e,0xd6,0x01,0xfe,0xff,0x01]
+# W64-REAL16: v_minimummaximum_f16 v5.h, s1, v255.h, exec_hi op_sel:[0,1,0,1] ; encoding: [0x05,0x50,0x6e,0xd6,0x01,0xfe,0xff,0x01]
+# W64-FAKE16: v_minimummaximum_f16 v5, s1, v255, exec_hi op_sel:[0,1,0,1] ; encoding: [0x05,0x50,0x6e,0xd6,0x01,0xfe,0xff,0x01]
+
+0x05,0x42,0x6e,0xd6,0xfd,0xd6,0x04,0x23
+# W32-REAL16: v_minimummaximum_f16 v5.h, -src_scc, |vcc_hi|, -1 op_sel:[0,0,0,1] ; encoding: [0x05,0x42,0x6e,0xd6,0xfd,0xd6,0x04,0x23]
+# W32-FAKE16: v_minimummaximum_f16 v5, -src_scc, |vcc_hi|, -1 op_sel:[0,0,0,1] ; encoding: [0x05,0x42,0x6e,0xd6,0xfd,0xd6,0x04,0x23]
+# W64-REAL16: v_minimummaximum_f16 v5.h, -src_scc, |vcc_hi|, -1 op_sel:[0,0,0,1] ; encoding: [0x05,0x42,0x6e,0xd6,0xfd,0xd6,0x04,0x23]
+# W64-FAKE16: v_minimummaximum_f16 v5, -src_scc, |vcc_hi|, -1 op_sel:[0,0,0,1] ; encoding: [0x05,0x42,0x6e,0xd6,0xfd,0xd6,0x04,0x23]
+
+0xff,0xc3,0x6e,0xd6,0xff,0xd6,0xf0,0x61,0x12,0xaf,0x00,0x00
+# W32-REAL16: v_minimummaximum_f16 v255.h, -|0xaf12|, -|vcc_hi|, null op_sel:[0,0,0,1] clamp ; encoding: [0xff,0xc3,0x6e,0xd6,0xff,0xd6,0xf0,0x61,0x12,0xaf,0x00,0x00]
+# W32-FAKE16: v_minimummaximum_f16 v255, -|0xaf12|, -|vcc_hi|, null op_sel:[0,0,0,1] clamp ; encoding: [0xff,0xc3,0x6e,0xd6,0xff,0xd6,0xf0,0x61,0x12,0xaf,0x00,0x00]
+# W64-REAL16: v_minimummaximum_f16 v255.h, -|0xaf12|, -|vcc_hi|, null op_sel:[0,0,0,1] clamp ; encoding: [0xff,0xc3,0x6e,0xd6,0xff,0xd6,0xf0,0x61,0x12,0xaf,0x00,0x00]
+# W64-FAKE16: v_minimummaximum_f16 v255, -|0xaf12|, -|vcc_hi|, null op_sel:[0,0,0,1] clamp ; encoding: [0xff,0xc3,0x6e,0xd6,0xff,0xd6,0xf0,0x61,0x12,0xaf,0x00,0x00]
 
 0x05,0x00,0x80,0xd6,0x01,0x00,0x00,0x00
 # GFX12: v_s_exp_f32 s5, s1                      ; encoding: [0x05,0x00,0x80,0xd6,0x01,0x00,0x00,0x00]
diff --git a/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vop3_dpp16.txt b/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vop3_dpp16.txt
index 0be540da8287b..c63c09b6b24c5 100644
--- a/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vop3_dpp16.txt
+++ b/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vop3_dpp16.txt
@@ -240,22 +240,40 @@
 # GFX12: v_alignbit_b32_e64_dpp v255, v255, v255, src_scc row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x00,0x16,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30]
 
 0x05,0x00,0x17,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff
-# GFX12: v_alignbyte_b32_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x17,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
+# W32-REAL16: v_alignbyte_b32_e64_dpp v5, v1, v2, v3.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x17,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
+# W32-FAKE16: v_alignbyte_b32_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x17,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
+# W64-REAL16: v_alignbyte_b32_e64_dpp v5, v1, v2, v3.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x17,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
+# W64-FAKE16: v_alignbyte_b32_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x17,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 
 0x05,0x00,0x17,0xd6,0xfa,0x06,0x0c,0x04,0x01,0x1b,0x00,0xff
-# GFX12: v_alignbyte_b32_e64_dpp v5, v1, s3, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x17,0xd6,0xfa,0x06,0x0c,0x04,0x01,0x1b,0x00,0xff]
+# W32-REAL16: v_alignbyte_b32_e64_dpp v5, v1, s3, v3.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x17,0xd6,0xfa,0x06,0x0c,0x04,0x01,0x1b,0x00,0xff]
+# W32-FAKE16: v_alignbyte_b32_e64_dpp v5, v1, s3, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x17,0xd6,0xfa,0x06,0x0c,0x04,0x01,0x1b,0x00,0xff]
+# W64-REAL16: v_alignbyte_b32_e64_dpp v5, v1, s3, v3.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x17,0xd6,0xfa,0x06,0x0c,0x04,0x01,0x1b,0x00,0xff]
+# W64-FAKE16: v_alignbyte_b32_e64_dpp v5, v1, s3, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x17,0xd6,0xfa,0x06,0x0c,0x04,0x01,0x1b,0x00,0xff]
 
 0x05,0x00,0x17,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff
-# GFX12: v_alignbyte_b32_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x17,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
+# W32-REAL16: v_alignbyte_b32_e64_dpp v5, v1, v2, v3.l quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x17,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
+# W32-FAKE16: v_alignbyte_b32_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x17,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
+# W64-REAL16: v_alignbyte_b32_e64_dpp v5, v1, v2, v3.l quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x17,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
+# W64-FAKE16: v_alignbyte_b32_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x17,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
 
 0x05,0x00,0x17,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff
-# GFX12: v_alignbyte_b32_e64_dpp v5, v1, v2, v3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x17,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
+# W32-REAL16: v_alignbyte_b32_e64_dpp v5, v1, v2, v3.l row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x17,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
+# W32-FAKE16: v_alignbyte_b32_e64_dpp v5, v1, v2, v3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x17,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
+# W64-REAL16: v_alignbyte_b32_e64_dpp v5, v1, v2, v3.l row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x17,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
+# W64-FAKE16: v_alignbyte_b32_e64_dpp v5, v1, v2, v3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x17,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
 
 0x05,0x00,0x17,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x41,0x01,0xff
-# GFX12: v_alignbyte_b32_e64_dpp v5, v1, v2, v3 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x17,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x41,0x01,0xff]
+# W32-REAL16: v_alignbyte_b32_e64_dpp v5, v1, v2, v3.l row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x17,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x41,0x01,0xff]
+# W32-FAKE16: v_alignbyte_b32_e64_dpp v5, v1, v2, v3 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x17,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x41,0x01,0xff]
+# W64-REAL16: v_alignbyte_b32_e64_dpp v5, v1, v2, v3.l row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x17,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x41,0x01,0xff]
+# W64-FAKE16: v_alignbyte_b32_e64_dpp v5, v1, v2, v3 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x17,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x41,0x01,0xff]
 
 0x05,0x00,0x17,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x01,0x01,0xff
-# GFX12: v_alignbyte_b32_e64_dpp v5, v1, v2, v255 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x17,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x01,0x01,0xff]
+# W32-REAL16: v_alignbyte_b32_e64_dpp v5, v1, v2, v255.l row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x17,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x01,0x01,0xff]
+# W32-FAKE16: v_alignbyte_b32_e64_dpp v5, v1, v2, v255 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x17,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x01,0x01,0xff]
+# W64-REAL16: v_alignbyte_b32_e64_dpp v5, v1, v2, v255.l row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x17,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x01,0x01,0xff]
+# W64-FAKE16: v_alignbyte_b32_e64_dpp v5, v1, v2, v255 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x17,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x01,0x01,0xff]
 
 0x05,0x00,0x17,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x0f,0x01,0xff
 # GFX12: v_alignbyte_b32_e64_dpp v5, v1, v2, s105 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x17,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x0f,0x01,0xff]
@@ -284,6 +302,12 @@
 0xff,0x00,0x17,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30
 # GFX12: v_alignbyte_b32_e64_dpp v255, v255, v255, src_scc row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x00,0x17,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30]
 
+0x05,0x20,0x17,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x40,0x01,0xff
+# W32-REAL16: v_alignbyte_b32_e64_dpp v5, v1, v2, v255.h op_sel:[0,0,1,0] row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x20,0x17,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x40,0x01,0xff]
+# W32-FAKE16: v_alignbyte_b32_e64_dpp v5, v1, v2, v255 op_sel:[0,0,1,0] row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x20,0x17,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x40,0x01,0xff]
+# W64-REAL16: v_alignbyte_b32_e64_dpp v5, v1, v2, v255.h op_sel:[0,0,1,0] row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x20,0x17,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x40,0x01,0xff]
+# W64-FAKE16: v_alignbyte_b32_e64_dpp v5, v1, v2, v255 op_sel:[0,0,1,0] row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x20,0x17,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x40,0x01,0xff]
+
 0x05,0x00,0x62,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff
 # W32-REAL16: v_and_b16_e64_dpp v5.l, v1.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x62,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
 # W32-FAKE16: v_and_b16_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x62,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
@@ -5236,49 +5260,94 @@
 # W64-FAKE16: v_mad_i16_e64_dpp v255, v255, v255, src_scc op_sel:[0,0,0,1] clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0xc0,0x53,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30]
 
 0x05,0x00,0x5a,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff
-# GFX12: v_mad_i32_i16_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5a,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
+# W32-REAL16: v_mad_i32_i16_e64_dpp v5, v1.l, v2.l, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5a,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
+# W32-FAKE16: v_mad_i32_i16_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5a,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
+# W64-REAL16: v_mad_i32_i16_e64_dpp v5, v1.l, v2.l, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5a,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
+# W64-FAKE16: v_mad_i32_i16_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5a,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 
 0x05,0x00,0x5a,0xd6,0xfa,0x06,0x0c,0x04,0x01,0x1b,0x00,0xff
-# GFX12: v_mad_i32_i16_e64_dpp v5, v1, s3, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5a,0xd6,0xfa,0x06,0x0c,0x04,0x01,0x1b,0x00,0xff]
+# W32-REAL16: v_mad_i32_i16_e64_dpp v5, v1.l, s3, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5a,0xd6,0xfa,0x06,0x0c,0x04,0x01,0x1b,0x00,0xff]
+# W32-FAKE16: v_mad_i32_i16_e64_dpp v5, v1, s3, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5a,0xd6,0xfa,0x06,0x0c,0x04,0x01,0x1b,0x00,0xff]
+# W64-REAL16: v_mad_i32_i16_e64_dpp v5, v1.l, s3, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5a,0xd6,0xfa,0x06,0x0c,0x04,0x01,0x1b,0x00,0xff]
+# W64-FAKE16: v_mad_i32_i16_e64_dpp v5, v1, s3, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5a,0xd6,0xfa,0x06,0x0c,0x04,0x01,0x1b,0x00,0xff]
 
 0x05,0x00,0x5a,0xd6,0xfa,0x04,0xfe,0x07,0x01,0xe4,0x00,0xff
-# GFX12: v_mad_i32_i16_e64_dpp v5, v1, v2, v255 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5a,0xd6,0xfa,0x04,0xfe,0x07,0x01,0xe4,0x00,0xff]
+# W32-REAL16: v_mad_i32_i16_e64_dpp v5, v1.l, v2.l, v255 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5a,0xd6,0xfa,0x04,0xfe,0x07,0x01,0xe4,0x00,0xff]
+# W32-FAKE16: v_mad_i32_i16_e64_dpp v5, v1, v2, v255 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5a,0xd6,0xfa,0x04,0xfe,0x07,0x01,0xe4,0x00,0xff]
+# W64-REAL16: v_mad_i32_i16_e64_dpp v5, v1.l, v2.l, v255 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5a,0xd6,0xfa,0x04,0xfe,0x07,0x01,0xe4,0x00,0xff]
+# W64-FAKE16: v_mad_i32_i16_e64_dpp v5, v1, v2, v255 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5a,0xd6,0xfa,0x04,0xfe,0x07,0x01,0xe4,0x00,0xff]
 
 0x05,0x00,0x5a,0xd6,0xfa,0x04,0x0e,0x00,0x01,0x40,0x01,0xff
-# GFX12: v_mad_i32_i16_e64_dpp v5, v1, v2, s3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5a,0xd6,0xfa,0x04,0x0e,0x00,0x01,0x40,0x01,0xff]
+# W32-REAL16: v_mad_i32_i16_e64_dpp v5, v1.l, v2.l, s3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5a,0xd6,0xfa,0x04,0x0e,0x00,0x01,0x40,0x01,0xff]
+# W32-FAKE16: v_mad_i32_i16_e64_dpp v5, v1, v2, s3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5a,0xd6,0xfa,0x04,0x0e,0x00,0x01,0x40,0x01,0xff]
+# W64-REAL16: v_mad_i32_i16_e64_dpp v5, v1.l, v2.l, s3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5a,0xd6,0xfa,0x04,0x0e,0x00,0x01,0x40,0x01,0xff]
+# W64-FAKE16: v_mad_i32_i16_e64_dpp v5, v1, v2, s3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5a,0xd6,0xfa,0x04,0x0e,0x00,0x01,0x40,0x01,0xff]
 
 0x05,0x00,0x5a,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x41,0x01,0xff
-# GFX12: v_mad_i32_i16_e64_dpp v5, v1, v2, s105 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5a,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x41,0x01,0xff]
+# W32-REAL16: v_mad_i32_i16_e64_dpp v5, v1.l, v2.l, s105 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5a,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x41,0x01,0xff]
+# W32-FAKE16: v_mad_i32_i16_e64_dpp v5, v1, v2, s105 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5a,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x41,0x01,0xff]
+# W64-REAL16: v_mad_i32_i16_e64_dpp v5, v1.l, v2.l, s105 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5a,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x41,0x01,0xff]
+# W64-FAKE16: v_mad_i32_i16_e64_dpp v5, v1, v2, s105 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5a,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x41,0x01,0xff]
 
 0x05,0x00,0x5a,0xd6,0xfa,0x04,0xee,0x01,0x01,0x01,0x01,0xff
-# GFX12: v_mad_i32_i16_e64_dpp v5, v1, v2, ttmp15 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5a,0xd6,0xfa,0x04,0xee,0x01,0x01,0x01,0x01,0xff]
+# W32-REAL16: v_mad_i32_i16_e64_dpp v5, v1.l, v2.l, ttmp15 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5a,0xd6,0xfa,0x04,0xee,0x01,0x01,0x01,0x01,0xff]
+# W32-FAKE16: v_mad_i32_i16_e64_dpp v5, v1, v2, ttmp15 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5a,0xd6,0xfa,0x04,0xee,0x01,0x01,0x01,0x01,0xff]
+# W64-REAL16: v_mad_i32_i16_e64_dpp v5, v1.l, v2.l, ttmp15 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5a,0xd6,0xfa,0x04,0xee,0x01,0x01,0x01,0x01,0xff]
+# W64-FAKE16: v_mad_i32_i16_e64_dpp v5, v1, v2, ttmp15 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5a,0xd6,0xfa,0x04,0xee,0x01,0x01,0x01,0x01,0xff]
 
 0x05,0x00,0x5a,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff
-# GFX12: v_mad_i32_i16_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5a,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
+# W32-REAL16: v_mad_i32_i16_e64_dpp v5, v1.l, v2.l, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5a,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
+# W32-FAKE16: v_mad_i32_i16_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5a,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
+# W64-REAL16: v_mad_i32_i16_e64_dpp v5, v1.l, v2.l, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5a,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
+# W64-FAKE16: v_mad_i32_i16_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5a,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
 
 0x05,0x00,0x5a,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff
-# GFX12: v_mad_i32_i16_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5a,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
+# W32-REAL16: v_mad_i32_i16_e64_dpp v5, v1.l, v2.l, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5a,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
+# W32-FAKE16: v_mad_i32_i16_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5a,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
+# W64-REAL16: v_mad_i32_i16_e64_dpp v5, v1.l, v2.l, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5a,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
+# W64-FAKE16: v_mad_i32_i16_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5a,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
 
 0x05,0x00,0x5a,0xd6,0xfa,0x04,0xf6,0x01,0x01,0x1f,0x01,0xff
-# GFX12: v_mad_i32_i16_e64_dpp v5, v1, v2, m0 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5a,0xd6,0xfa,0x04,0xf6,0x01,0x01,0x1f,0x01,0xff]
+# W32-REAL16: v_mad_i32_i16_e64_dpp v5, v1.l, v2.l, m0 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5a,0xd6,0xfa,0x04,0xf6,0x01,0x01,0x1f,0x01,0xff]
+# W32-FAKE16: v_mad_i32_i16_e64_dpp v5, v1, v2, m0 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5a,0xd6,0xfa,0x04,0xf6,0x01,0x01,0x1f,0x01,0xff]
+# W64-REAL16: v_mad_i32_i16_e64_dpp v5, v1.l, v2.l, m0 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5a,0xd6,0xfa,0x04,0xf6,0x01,0x01,0x1f,0x01,0xff]
+# W64-FAKE16: v_mad_i32_i16_e64_dpp v5, v1, v2, m0 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5a,0xd6,0xfa,0x04,0xf6,0x01,0x01,0x1f,0x01,0xff]
 
 0x05,0x00,0x5a,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x21,0x01,0xff
-# GFX12: v_mad_i32_i16_e64_dpp v5, v1, v2, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5a,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x21,0x01,0xff]
+# W32-REAL16: v_mad_i32_i16_e64_dpp v5, v1.l, v2.l, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5a,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x21,0x01,0xff]
+# W32-FAKE16: v_mad_i32_i16_e64_dpp v5, v1, v2, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5a,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x21,0x01,0xff]
+# W64-REAL16: v_mad_i32_i16_e64_dpp v5, v1.l, v2.l, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5a,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x21,0x01,0xff]
+# W64-FAKE16: v_mad_i32_i16_e64_dpp v5, v1, v2, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5a,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x21,0x01,0xff]
 
 0x05,0x00,0x5a,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x2f,0x01,0xff
-# GFX12: v_mad_i32_i16_e64_dpp v5, v1, v2, exec_lo row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5a,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x2f,0x01,0xff]
+# W32-REAL16: v_mad_i32_i16_e64_dpp v5, v1.l, v2.l, exec_lo row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5a,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x2f,0x01,0xff]
+# W32-FAKE16: v_mad_i32_i16_e64_dpp v5, v1, v2, exec_lo row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5a,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x2f,0x01,0xff]
+# W64-REAL16: v_mad_i32_i16_e64_dpp v5, v1.l, v2.l, exec_lo row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5a,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x2f,0x01,0xff]
+# W64-FAKE16: v_mad_i32_i16_e64_dpp v5, v1, v2, exec_lo row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5a,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x2f,0x01,0xff]
 
 0x05,0x00,0x5a,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x50,0x01,0xff
-# GFX12: v_mad_i32_i16_e64_dpp v5, v1, v2, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5a,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x50,0x01,0xff]
+# W32-REAL16: v_mad_i32_i16_e64_dpp v5, v1.l, v2.l, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5a,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x50,0x01,0xff]
+# W32-FAKE16: v_mad_i32_i16_e64_dpp v5, v1, v2, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5a,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x50,0x01,0xff]
+# W64-REAL16: v_mad_i32_i16_e64_dpp v5, v1.l, v2.l, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5a,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x50,0x01,0xff]
+# W64-FAKE16: v_mad_i32_i16_e64_dpp v5, v1, v2, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5a,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x50,0x01,0xff]
 
 0x05,0x00,0x5a,0xd6,0xfa,0x04,0x06,0x03,0x01,0x5f,0x01,0x01
-# GFX12: v_mad_i32_i16_e64_dpp v5, v1, v2, -1 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x5a,0xd6,0xfa,0x04,0x06,0x03,0x01,0x5f,0x01,0x01]
+# W32-REAL16: v_mad_i32_i16_e64_dpp v5, v1.l, v2.l, -1 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x5a,0xd6,0xfa,0x04,0x06,0x03,0x01,0x5f,0x01,0x01]
+# W32-FAKE16: v_mad_i32_i16_e64_dpp v5, v1, v2, -1 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x5a,0xd6,0xfa,0x04,0x06,0x03,0x01,0x5f,0x01,0x01]
+# W64-REAL16: v_mad_i32_i16_e64_dpp v5, v1.l, v2.l, -1 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x5a,0xd6,0xfa,0x04,0x06,0x03,0x01,0x5f,0x01,0x01]
+# W64-FAKE16: v_mad_i32_i16_e64_dpp v5, v1, v2, -1 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x5a,0xd6,0xfa,0x04,0x06,0x03,0x01,0x5f,0x01,0x01]
 
 0x05,0x08,0x5a,0xd6,0xfa,0x04,0xc2,0x03,0x01,0x60,0x01,0x13
-# GFX12: v_mad_i32_i16_e64_dpp v5, v1, v2, 0.5 op_sel:[1,0,0,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x08,0x5a,0xd6,0xfa,0x04,0xc2,0x03,0x01,0x60,0x01,0x13]
+# W32-REAL16: v_mad_i32_i16_e64_dpp v5, v1.h, v2.l, 0.5 op_sel:[1,0,0,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x08,0x5a,0xd6,0xfa,0x04,0xc2,0x03,0x01,0x60,0x01,0x13]
+# W32-FAKE16: v_mad_i32_i16_e64_dpp v5, v1, v2, 0.5 op_sel:[1,0,0,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x08,0x5a,0xd6,0xfa,0x04,0xc2,0x03,0x01,0x60,0x01,0x13]
+# W64-REAL16: v_mad_i32_i16_e64_dpp v5, v1.h, v2.l, 0.5 op_sel:[1,0,0,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x08,0x5a,0xd6,0xfa,0x04,0xc2,0x03,0x01,0x60,0x01,0x13]
+# W64-FAKE16: v_mad_i32_i16_e64_dpp v5, v1, v2, 0.5 op_sel:[1,0,0,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x08,0x5a,0xd6,0xfa,0x04,0xc2,0x03,0x01,0x60,0x01,0x13]
 
 0xff,0x90,0x5a,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30
-# GFX12: v_mad_i32_i16_e64_dpp v255, v255, v255, src_scc op_sel:[0,1,0,0] clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x90,0x5a,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30]
+# W32-REAL16: v_mad_i32_i16_e64_dpp v255, v255.l, v255.h, src_scc op_sel:[0,1,0,0] clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x90,0x5a,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30]
+# W32-FAKE16: v_mad_i32_i16_e64_dpp v255, v255, v255, src_scc op_sel:[0,1,0,0] clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x90,0x5a,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30]
+# W64-REAL16: v_mad_i32_i16_e64_dpp v255, v255.l, v255.h, src_scc op_sel:[0,1,0,0] clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x90,0x5a,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30]
+# W64-FAKE16: v_mad_i32_i16_e64_dpp v255, v255, v255, src_scc op_sel:[0,1,0,0] clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x90,0x5a,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30]
 
 0x05,0x00,0x41,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff
 # W32-REAL16: v_mad_u16_e64_dpp v5.l, v1.l, v2.l, v3.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x41,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
@@ -5431,49 +5500,94 @@
 # W64-FAKE16: v_mad_u16_e64_dpp v255, v255, v255, src_scc op_sel:[0,0,0,1] clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0xc0,0x41,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30]
 
 0x05,0x00,0x59,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff
-# GFX12: v_mad_u32_u16_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x59,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
+# W32-REAL16: v_mad_u32_u16_e64_dpp v5, v1.l, v2.l, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x59,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
+# W32-FAKE16: v_mad_u32_u16_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x59,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
+# W64-REAL16: v_mad_u32_u16_e64_dpp v5, v1.l, v2.l, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x59,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
+# W64-FAKE16: v_mad_u32_u16_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x59,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 
 0x05,0x00,0x59,0xd6,0xfa,0x06,0x0c,0x04,0x01,0x1b,0x00,0xff
-# GFX12: v_mad_u32_u16_e64_dpp v5, v1, s3, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x59,0xd6,0xfa,0x06,0x0c,0x04,0x01,0x1b,0x00,0xff]
+# W32-REAL16: v_mad_u32_u16_e64_dpp v5, v1.l, s3, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x59,0xd6,0xfa,0x06,0x0c,0x04,0x01,0x1b,0x00,0xff]
+# W32-FAKE16: v_mad_u32_u16_e64_dpp v5, v1, s3, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x59,0xd6,0xfa,0x06,0x0c,0x04,0x01,0x1b,0x00,0xff]
+# W64-REAL16: v_mad_u32_u16_e64_dpp v5, v1.l, s3, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x59,0xd6,0xfa,0x06,0x0c,0x04,0x01,0x1b,0x00,0xff]
+# W64-FAKE16: v_mad_u32_u16_e64_dpp v5, v1, s3, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x59,0xd6,0xfa,0x06,0x0c,0x04,0x01,0x1b,0x00,0xff]
 
 0x05,0x00,0x59,0xd6,0xfa,0x04,0xfe,0x07,0x01,0xe4,0x00,0xff
-# GFX12: v_mad_u32_u16_e64_dpp v5, v1, v2, v255 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x59,0xd6,0xfa,0x04,0xfe,0x07,0x01,0xe4,0x00,0xff]
+# W32-REAL16: v_mad_u32_u16_e64_dpp v5, v1.l, v2.l, v255 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x59,0xd6,0xfa,0x04,0xfe,0x07,0x01,0xe4,0x00,0xff]
+# W32-FAKE16: v_mad_u32_u16_e64_dpp v5, v1, v2, v255 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x59,0xd6,0xfa,0x04,0xfe,0x07,0x01,0xe4,0x00,0xff]
+# W64-REAL16: v_mad_u32_u16_e64_dpp v5, v1.l, v2.l, v255 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x59,0xd6,0xfa,0x04,0xfe,0x07,0x01,0xe4,0x00,0xff]
+# W64-FAKE16: v_mad_u32_u16_e64_dpp v5, v1, v2, v255 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x59,0xd6,0xfa,0x04,0xfe,0x07,0x01,0xe4,0x00,0xff]
 
 0x05,0x00,0x59,0xd6,0xfa,0x04,0x0e,0x00,0x01,0x40,0x01,0xff
-# GFX12: v_mad_u32_u16_e64_dpp v5, v1, v2, s3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x59,0xd6,0xfa,0x04,0x0e,0x00,0x01,0x40,0x01,0xff]
+# W32-REAL16: v_mad_u32_u16_e64_dpp v5, v1.l, v2.l, s3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x59,0xd6,0xfa,0x04,0x0e,0x00,0x01,0x40,0x01,0xff]
+# W32-FAKE16: v_mad_u32_u16_e64_dpp v5, v1, v2, s3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x59,0xd6,0xfa,0x04,0x0e,0x00,0x01,0x40,0x01,0xff]
+# W64-REAL16: v_mad_u32_u16_e64_dpp v5, v1.l, v2.l, s3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x59,0xd6,0xfa,0x04,0x0e,0x00,0x01,0x40,0x01,0xff]
+# W64-FAKE16: v_mad_u32_u16_e64_dpp v5, v1, v2, s3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x59,0xd6,0xfa,0x04,0x0e,0x00,0x01,0x40,0x01,0xff]
 
 0x05,0x00,0x59,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x41,0x01,0xff
-# GFX12: v_mad_u32_u16_e64_dpp v5, v1, v2, s105 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x59,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x41,0x01,0xff]
+# W32-REAL16: v_mad_u32_u16_e64_dpp v5, v1.l, v2.l, s105 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x59,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x41,0x01,0xff]
+# W32-FAKE16: v_mad_u32_u16_e64_dpp v5, v1, v2, s105 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x59,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x41,0x01,0xff]
+# W64-REAL16: v_mad_u32_u16_e64_dpp v5, v1.l, v2.l, s105 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x59,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x41,0x01,0xff]
+# W64-FAKE16: v_mad_u32_u16_e64_dpp v5, v1, v2, s105 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x59,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x41,0x01,0xff]
 
 0x05,0x00,0x59,0xd6,0xfa,0x04,0xee,0x01,0x01,0x01,0x01,0xff
-# GFX12: v_mad_u32_u16_e64_dpp v5, v1, v2, ttmp15 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x59,0xd6,0xfa,0x04,0xee,0x01,0x01,0x01,0x01,0xff]
+# W32-REAL16: v_mad_u32_u16_e64_dpp v5, v1.l, v2.l, ttmp15 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x59,0xd6,0xfa,0x04,0xee,0x01,0x01,0x01,0x01,0xff]
+# W32-FAKE16: v_mad_u32_u16_e64_dpp v5, v1, v2, ttmp15 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x59,0xd6,0xfa,0x04,0xee,0x01,0x01,0x01,0x01,0xff]
+# W64-REAL16: v_mad_u32_u16_e64_dpp v5, v1.l, v2.l, ttmp15 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x59,0xd6,0xfa,0x04,0xee,0x01,0x01,0x01,0x01,0xff]
+# W64-FAKE16: v_mad_u32_u16_e64_dpp v5, v1, v2, ttmp15 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x59,0xd6,0xfa,0x04,0xee,0x01,0x01,0x01,0x01,0xff]
 
 0x05,0x00,0x59,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff
-# GFX12: v_mad_u32_u16_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x59,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
+# W32-REAL16: v_mad_u32_u16_e64_dpp v5, v1.l, v2.l, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x59,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
+# W32-FAKE16: v_mad_u32_u16_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x59,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
+# W64-REAL16: v_mad_u32_u16_e64_dpp v5, v1.l, v2.l, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x59,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
+# W64-FAKE16: v_mad_u32_u16_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x59,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
 
 0x05,0x00,0x59,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff
-# GFX12: v_mad_u32_u16_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x59,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
+# W32-REAL16: v_mad_u32_u16_e64_dpp v5, v1.l, v2.l, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x59,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
+# W32-FAKE16: v_mad_u32_u16_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x59,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
+# W64-REAL16: v_mad_u32_u16_e64_dpp v5, v1.l, v2.l, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x59,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
+# W64-FAKE16: v_mad_u32_u16_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x59,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
 
 0x05,0x00,0x59,0xd6,0xfa,0x04,0xf6,0x01,0x01,0x1f,0x01,0xff
-# GFX12: v_mad_u32_u16_e64_dpp v5, v1, v2, m0 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x59,0xd6,0xfa,0x04,0xf6,0x01,0x01,0x1f,0x01,0xff]
+# W32-REAL16: v_mad_u32_u16_e64_dpp v5, v1.l, v2.l, m0 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x59,0xd6,0xfa,0x04,0xf6,0x01,0x01,0x1f,0x01,0xff]
+# W32-FAKE16: v_mad_u32_u16_e64_dpp v5, v1, v2, m0 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x59,0xd6,0xfa,0x04,0xf6,0x01,0x01,0x1f,0x01,0xff]
+# W64-REAL16: v_mad_u32_u16_e64_dpp v5, v1.l, v2.l, m0 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x59,0xd6,0xfa,0x04,0xf6,0x01,0x01,0x1f,0x01,0xff]
+# W64-FAKE16: v_mad_u32_u16_e64_dpp v5, v1, v2, m0 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x59,0xd6,0xfa,0x04,0xf6,0x01,0x01,0x1f,0x01,0xff]
 
 0x05,0x00,0x59,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x21,0x01,0xff
-# GFX12: v_mad_u32_u16_e64_dpp v5, v1, v2, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x59,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x21,0x01,0xff]
+# W32-REAL16: v_mad_u32_u16_e64_dpp v5, v1.l, v2.l, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x59,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x21,0x01,0xff]
+# W32-FAKE16: v_mad_u32_u16_e64_dpp v5, v1, v2, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x59,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x21,0x01,0xff]
+# W64-REAL16: v_mad_u32_u16_e64_dpp v5, v1.l, v2.l, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x59,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x21,0x01,0xff]
+# W64-FAKE16: v_mad_u32_u16_e64_dpp v5, v1, v2, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x59,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x21,0x01,0xff]
 
 0x05,0x00,0x59,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x2f,0x01,0xff
-# GFX12: v_mad_u32_u16_e64_dpp v5, v1, v2, exec_lo row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x59,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x2f,0x01,0xff]
+# W32-REAL16: v_mad_u32_u16_e64_dpp v5, v1.l, v2.l, exec_lo row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x59,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x2f,0x01,0xff]
+# W32-FAKE16: v_mad_u32_u16_e64_dpp v5, v1, v2, exec_lo row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x59,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x2f,0x01,0xff]
+# W64-REAL16: v_mad_u32_u16_e64_dpp v5, v1.l, v2.l, exec_lo row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x59,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x2f,0x01,0xff]
+# W64-FAKE16: v_mad_u32_u16_e64_dpp v5, v1, v2, exec_lo row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x59,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x2f,0x01,0xff]
 
 0x05,0x00,0x59,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x50,0x01,0xff
-# GFX12: v_mad_u32_u16_e64_dpp v5, v1, v2, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x59,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x50,0x01,0xff]
+# W32-REAL16: v_mad_u32_u16_e64_dpp v5, v1.l, v2.l, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x59,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x50,0x01,0xff]
+# W32-FAKE16: v_mad_u32_u16_e64_dpp v5, v1, v2, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x59,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x50,0x01,0xff]
+# W64-REAL16: v_mad_u32_u16_e64_dpp v5, v1.l, v2.l, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x59,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x50,0x01,0xff]
+# W64-FAKE16: v_mad_u32_u16_e64_dpp v5, v1, v2, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x59,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x50,0x01,0xff]
 
 0x05,0x00,0x59,0xd6,0xfa,0x04,0x06,0x03,0x01,0x5f,0x01,0x01
-# GFX12: v_mad_u32_u16_e64_dpp v5, v1, v2, -1 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x59,0xd6,0xfa,0x04,0x06,0x03,0x01,0x5f,0x01,0x01]
+# W32-REAL16: v_mad_u32_u16_e64_dpp v5, v1.l, v2.l, -1 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x59,0xd6,0xfa,0x04,0x06,0x03,0x01,0x5f,0x01,0x01]
+# W32-FAKE16: v_mad_u32_u16_e64_dpp v5, v1, v2, -1 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x59,0xd6,0xfa,0x04,0x06,0x03,0x01,0x5f,0x01,0x01]
+# W64-REAL16: v_mad_u32_u16_e64_dpp v5, v1.l, v2.l, -1 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x59,0xd6,0xfa,0x04,0x06,0x03,0x01,0x5f,0x01,0x01]
+# W64-FAKE16: v_mad_u32_u16_e64_dpp v5, v1, v2, -1 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x59,0xd6,0xfa,0x04,0x06,0x03,0x01,0x5f,0x01,0x01]
 
 0x05,0x08,0x59,0xd6,0xfa,0x04,0xc2,0x03,0x01,0x60,0x01,0x13
-# GFX12: v_mad_u32_u16_e64_dpp v5, v1, v2, 0.5 op_sel:[1,0,0,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x08,0x59,0xd6,0xfa,0x04,0xc2,0x03,0x01,0x60,0x01,0x13]
+# W32-REAL16: v_mad_u32_u16_e64_dpp v5, v1.h, v2.l, 0.5 op_sel:[1,0,0,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x08,0x59,0xd6,0xfa,0x04,0xc2,0x03,0x01,0x60,0x01,0x13]
+# W32-FAKE16: v_mad_u32_u16_e64_dpp v5, v1, v2, 0.5 op_sel:[1,0,0,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x08,0x59,0xd6,0xfa,0x04,0xc2,0x03,0x01,0x60,0x01,0x13]
+# W64-REAL16: v_mad_u32_u16_e64_dpp v5, v1.h, v2.l, 0.5 op_sel:[1,0,0,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x08,0x59,0xd6,0xfa,0x04,0xc2,0x03,0x01,0x60,0x01,0x13]
+# W64-FAKE16: v_mad_u32_u16_e64_dpp v5, v1, v2, 0.5 op_sel:[1,0,0,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x08,0x59,0xd6,0xfa,0x04,0xc2,0x03,0x01,0x60,0x01,0x13]
 
 0xff,0x90,0x59,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30
-# GFX12: v_mad_u32_u16_e64_dpp v255, v255, v255, src_scc op_sel:[0,1,0,0] clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x90,0x59,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30]
+# W32-REAL16: v_mad_u32_u16_e64_dpp v255, v255.l, v255.h, src_scc op_sel:[0,1,0,0] clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x90,0x59,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30]
+# W32-FAKE16: v_mad_u32_u16_e64_dpp v255, v255, v255, src_scc op_sel:[0,1,0,0] clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x90,0x59,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30]
+# W64-REAL16: v_mad_u32_u16_e64_dpp v255, v255.l, v255.h, src_scc op_sel:[0,1,0,0] clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x90,0x59,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30]
+# W64-FAKE16: v_mad_u32_u16_e64_dpp v255, v255, v255, src_scc op_sel:[0,1,0,0] clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x90,0x59,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30]
 
 0x05,0x00,0x2c,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff
 # W32-REAL16: v_max3_num_f16_e64_dpp v5.l, v1.l, v2.l, v3.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x2c,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
@@ -7268,88 +7382,208 @@
 # GFX12: v_maximum_f32_e64_dpp v5, |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x01,0x66,0xd7,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
 
 0xff,0x03,0x67,0xd7,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30
-# GFX12: v_minimum_f16_e64_dpp v255, -|v255|, -|v255| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x03,0x67,0xd7,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30]
+# W32-REAL16: v_minimum_f16_e64_dpp v255.l, -|v255.l|, -|v255.l| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x03,0x67,0xd7,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30]
+# W32-FAKE16: v_minimum_f16_e64_dpp v255, -|v255|, -|v255| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x03,0x67,0xd7,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30]
+# W64-REAL16: v_minimum_f16_e64_dpp v255.l, -|v255.l|, -|v255.l| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x03,0x67,0xd7,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30]
+# W64-FAKE16: v_minimum_f16_e64_dpp v255, -|v255|, -|v255| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x03,0x67,0xd7,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30]
 
 0x05,0x02,0x67,0xd7,0xfa,0x04,0x02,0x20,0x01,0x60,0x09,0x13
-# GFX12: v_minimum_f16_e64_dpp v5, -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x05,0x02,0x67,0xd7,0xfa,0x04,0x02,0x20,0x01,0x60,0x09,0x13]
+# W32-REAL16: v_minimum_f16_e64_dpp v5.l, -v1.l, |v2.l| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x05,0x02,0x67,0xd7,0xfa,0x04,0x02,0x20,0x01,0x60,0x09,0x13]
+# W32-FAKE16: v_minimum_f16_e64_dpp v5, -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x05,0x02,0x67,0xd7,0xfa,0x04,0x02,0x20,0x01,0x60,0x09,0x13]
+# W64-REAL16: v_minimum_f16_e64_dpp v5.l, -v1.l, |v2.l| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x05,0x02,0x67,0xd7,0xfa,0x04,0x02,0x20,0x01,0x60,0x09,0x13]
+# W64-FAKE16: v_minimum_f16_e64_dpp v5, -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x05,0x02,0x67,0xd7,0xfa,0x04,0x02,0x20,0x01,0x60,0x09,0x13]
 
 0x05,0x00,0x67,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff
-# GFX12: v_minimum_f16_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x67,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+# W32-REAL16: v_minimum_f16_e64_dpp v5.l, v1.l, v2.l quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x67,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+# W32-FAKE16: v_minimum_f16_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x67,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+# W64-REAL16: v_minimum_f16_e64_dpp v5.l, v1.l, v2.l quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x67,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+# W64-FAKE16: v_minimum_f16_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x67,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
 
 0x05,0x00,0x67,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff
-# GFX12: v_minimum_f16_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x67,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+# W32-REAL16: v_minimum_f16_e64_dpp v5.l, v1.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x67,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+# W32-FAKE16: v_minimum_f16_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x67,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+# W64-REAL16: v_minimum_f16_e64_dpp v5.l, v1.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x67,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+# W64-FAKE16: v_minimum_f16_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x67,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
 
 0x05,0x00,0x67,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff
-# GFX12: v_minimum_f16_e64_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x67,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+# W32-REAL16: v_minimum_f16_e64_dpp v5.l, v1.l, v2.l row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x67,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+# W32-FAKE16: v_minimum_f16_e64_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x67,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+# W64-REAL16: v_minimum_f16_e64_dpp v5.l, v1.l, v2.l row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x67,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+# W64-FAKE16: v_minimum_f16_e64_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x67,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
 
 0x05,0x00,0x67,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff
-# GFX12: v_minimum_f16_e64_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x67,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+# W32-REAL16: v_minimum_f16_e64_dpp v5.l, v1.l, v2.l row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x67,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+# W32-FAKE16: v_minimum_f16_e64_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x67,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+# W64-REAL16: v_minimum_f16_e64_dpp v5.l, v1.l, v2.l row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x67,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+# W64-FAKE16: v_minimum_f16_e64_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x67,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
 
 0x05,0x00,0x67,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff
-# GFX12: v_minimum_f16_e64_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x67,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+# W32-REAL16: v_minimum_f16_e64_dpp v5.l, v1.l, v2.l row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x67,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+# W32-FAKE16: v_minimum_f16_e64_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x67,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+# W64-REAL16: v_minimum_f16_e64_dpp v5.l, v1.l, v2.l row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x67,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+# W64-FAKE16: v_minimum_f16_e64_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x67,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
 
 0x05,0x00,0x67,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff
-# GFX12: v_minimum_f16_e64_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x67,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+# W32-REAL16: v_minimum_f16_e64_dpp v5.l, v1.l, v2.l row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x67,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+# W32-FAKE16: v_minimum_f16_e64_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x67,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+# W64-REAL16: v_minimum_f16_e64_dpp v5.l, v1.l, v2.l row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x67,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+# W64-FAKE16: v_minimum_f16_e64_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x67,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
 
 0x05,0x00,0x67,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff
-# GFX12: v_minimum_f16_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x67,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+# W32-REAL16: v_minimum_f16_e64_dpp v5.l, v1.l, v2.l row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x67,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+# W32-FAKE16: v_minimum_f16_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x67,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+# W64-REAL16: v_minimum_f16_e64_dpp v5.l, v1.l, v2.l row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x67,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+# W64-FAKE16: v_minimum_f16_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x67,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 
 0x05,0x00,0x67,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff
-# GFX12: v_minimum_f16_e64_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x67,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+# W32-REAL16: v_minimum_f16_e64_dpp v5.l, v1.l, v2.l row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x67,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+# W32-FAKE16: v_minimum_f16_e64_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x67,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+# W64-REAL16: v_minimum_f16_e64_dpp v5.l, v1.l, v2.l row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x67,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+# W64-FAKE16: v_minimum_f16_e64_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x67,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
 
 0x05,0x00,0x67,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff
-# GFX12: v_minimum_f16_e64_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x67,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+# W32-REAL16: v_minimum_f16_e64_dpp v5.l, v1.l, v2.l row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x67,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+# W32-FAKE16: v_minimum_f16_e64_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x67,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+# W64-REAL16: v_minimum_f16_e64_dpp v5.l, v1.l, v2.l row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x67,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+# W64-FAKE16: v_minimum_f16_e64_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x67,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
 
 0x05,0x00,0x67,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff
-# GFX12: v_minimum_f16_e64_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x67,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+# W32-REAL16: v_minimum_f16_e64_dpp v5.l, v1.l, v2.l row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x67,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+# W32-FAKE16: v_minimum_f16_e64_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x67,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+# W64-REAL16: v_minimum_f16_e64_dpp v5.l, v1.l, v2.l row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x67,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+# W64-FAKE16: v_minimum_f16_e64_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x67,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
 
 0x05,0x00,0x67,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff
-# GFX12: v_minimum_f16_e64_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x67,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+# W32-REAL16: v_minimum_f16_e64_dpp v5.l, v1.l, v2.l row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x67,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+# W32-FAKE16: v_minimum_f16_e64_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x67,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+# W64-REAL16: v_minimum_f16_e64_dpp v5.l, v1.l, v2.l row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x67,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+# W64-FAKE16: v_minimum_f16_e64_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x67,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
 
 0x05,0x01,0x67,0xd7,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01
-# GFX12: v_minimum_f16_e64_dpp v5, |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x01,0x67,0xd7,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+# W32-REAL16: v_minimum_f16_e64_dpp v5.l, |v1.l|, -v2.l row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x01,0x67,0xd7,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+# W32-FAKE16: v_minimum_f16_e64_dpp v5, |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x01,0x67,0xd7,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+# W64-REAL16: v_minimum_f16_e64_dpp v5.l, |v1.l|, -v2.l row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x01,0x67,0xd7,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+# W64-FAKE16: v_minimum_f16_e64_dpp v5, |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x01,0x67,0xd7,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+
+0x05,0x48,0x67,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff
+# W32-REAL16: v_minimum_f16_e64_dpp v5.h, v1.h, v2.l op_sel:[1,0,1] quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x48,0x67,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+# W32-FAKE16: v_minimum_f16_e64_dpp v5, v1, v2 op_sel:[1,0,1] quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x48,0x67,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+# W64-REAL16: v_minimum_f16_e64_dpp v5.h, v1.h, v2.l op_sel:[1,0,1] quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x48,0x67,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+# W64-FAKE16: v_minimum_f16_e64_dpp v5, v1, v2 op_sel:[1,0,1] quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x48,0x67,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+
+0x05,0x48,0x67,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff
+# W32-REAL16: v_minimum_f16_e64_dpp v5.h, v1.h, v2.l op_sel:[1,0,1] row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x48,0x67,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+# W32-FAKE16: v_minimum_f16_e64_dpp v5, v1, v2 op_sel:[1,0,1] row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x48,0x67,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+# W64-REAL16: v_minimum_f16_e64_dpp v5.h, v1.h, v2.l op_sel:[1,0,1] row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x48,0x67,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+# W64-FAKE16: v_minimum_f16_e64_dpp v5, v1, v2 op_sel:[1,0,1] row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x48,0x67,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+
+0xff,0x5b,0x67,0xd7,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x05,0x30
+# W32-REAL16: v_minimum_f16_e64_dpp v255.h, -|v255.h|, -|v255.h| op_sel:[1,1,1] row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xff,0x5b,0x67,0xd7,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x05,0x30]
+# W32-FAKE16: v_minimum_f16_e64_dpp v255, -|v255|, -|v255| op_sel:[1,1,1] row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xff,0x5b,0x67,0xd7,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x05,0x30]
+# W64-REAL16: v_minimum_f16_e64_dpp v255.h, -|v255.h|, -|v255.h| op_sel:[1,1,1] row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xff,0x5b,0x67,0xd7,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x05,0x30]
+# W64-FAKE16: v_minimum_f16_e64_dpp v255, -|v255|, -|v255| op_sel:[1,1,1] row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xff,0x5b,0x67,0xd7,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x05,0x30]
 
 0xff,0x03,0x68,0xd7,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30
-# GFX12: v_maximum_f16_e64_dpp v255, -|v255|, -|v255| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x03,0x68,0xd7,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30]
+# W32-REAL16: v_maximum_f16_e64_dpp v255.l, -|v255.l|, -|v255.l| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x03,0x68,0xd7,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30]
+# W32-FAKE16: v_maximum_f16_e64_dpp v255, -|v255|, -|v255| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x03,0x68,0xd7,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30]
+# W64-REAL16: v_maximum_f16_e64_dpp v255.l, -|v255.l|, -|v255.l| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x03,0x68,0xd7,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30]
+# W64-FAKE16: v_maximum_f16_e64_dpp v255, -|v255|, -|v255| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x03,0x68,0xd7,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30]
 
 0x05,0x02,0x68,0xd7,0xfa,0x04,0x02,0x20,0x01,0x60,0x09,0x13
-# GFX12: v_maximum_f16_e64_dpp v5, -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x05,0x02,0x68,0xd7,0xfa,0x04,0x02,0x20,0x01,0x60,0x09,0x13]
+# W32-REAL16: v_maximum_f16_e64_dpp v5.l, -v1.l, |v2.l| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x05,0x02,0x68,0xd7,0xfa,0x04,0x02,0x20,0x01,0x60,0x09,0x13]
+# W32-FAKE16: v_maximum_f16_e64_dpp v5, -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x05,0x02,0x68,0xd7,0xfa,0x04,0x02,0x20,0x01,0x60,0x09,0x13]
+# W64-REAL16: v_maximum_f16_e64_dpp v5.l, -v1.l, |v2.l| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x05,0x02,0x68,0xd7,0xfa,0x04,0x02,0x20,0x01,0x60,0x09,0x13]
+# W64-FAKE16: v_maximum_f16_e64_dpp v5, -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x05,0x02,0x68,0xd7,0xfa,0x04,0x02,0x20,0x01,0x60,0x09,0x13]
 
 0x05,0x00,0x68,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff
-# GFX12: v_maximum_f16_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x68,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+# W32-REAL16: v_maximum_f16_e64_dpp v5.l, v1.l, v2.l quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x68,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+# W32-FAKE16: v_maximum_f16_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x68,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+# W64-REAL16: v_maximum_f16_e64_dpp v5.l, v1.l, v2.l quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x68,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+# W64-FAKE16: v_maximum_f16_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x68,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
 
 0x05,0x00,0x68,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff
-# GFX12: v_maximum_f16_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x68,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+# W32-REAL16: v_maximum_f16_e64_dpp v5.l, v1.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x68,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+# W32-FAKE16: v_maximum_f16_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x68,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+# W64-REAL16: v_maximum_f16_e64_dpp v5.l, v1.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x68,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+# W64-FAKE16: v_maximum_f16_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x68,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
 
 0x05,0x00,0x68,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff
-# GFX12: v_maximum_f16_e64_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x68,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+# W32-REAL16: v_maximum_f16_e64_dpp v5.l, v1.l, v2.l row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x68,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+# W32-FAKE16: v_maximum_f16_e64_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x68,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+# W64-REAL16: v_maximum_f16_e64_dpp v5.l, v1.l, v2.l row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x68,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+# W64-FAKE16: v_maximum_f16_e64_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x68,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
 
 0x05,0x00,0x68,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff
-# GFX12: v_maximum_f16_e64_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x68,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+# W32-REAL16: v_maximum_f16_e64_dpp v5.l, v1.l, v2.l row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x68,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+# W32-FAKE16: v_maximum_f16_e64_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x68,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+# W64-REAL16: v_maximum_f16_e64_dpp v5.l, v1.l, v2.l row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x68,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+# W64-FAKE16: v_maximum_f16_e64_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x68,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
 
 0x05,0x00,0x68,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff
-# GFX12: v_maximum_f16_e64_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x68,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+# W32-REAL16: v_maximum_f16_e64_dpp v5.l, v1.l, v2.l row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x68,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+# W32-FAKE16: v_maximum_f16_e64_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x68,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+# W64-REAL16: v_maximum_f16_e64_dpp v5.l, v1.l, v2.l row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x68,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+# W64-FAKE16: v_maximum_f16_e64_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x68,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
 
 0x05,0x00,0x68,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff
-# GFX12: v_maximum_f16_e64_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x68,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+# W32-REAL16: v_maximum_f16_e64_dpp v5.l, v1.l, v2.l row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x68,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+# W32-FAKE16: v_maximum_f16_e64_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x68,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+# W64-REAL16: v_maximum_f16_e64_dpp v5.l, v1.l, v2.l row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x68,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+# W64-FAKE16: v_maximum_f16_e64_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x68,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
 
 0x05,0x00,0x68,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff
-# GFX12: v_maximum_f16_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x68,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+# W32-REAL16: v_maximum_f16_e64_dpp v5.l, v1.l, v2.l row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x68,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+# W32-FAKE16: v_maximum_f16_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x68,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+# W64-REAL16: v_maximum_f16_e64_dpp v5.l, v1.l, v2.l row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x68,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+# W64-FAKE16: v_maximum_f16_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x68,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 
 0x05,0x00,0x68,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff
-# GFX12: v_maximum_f16_e64_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x68,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+# W32-REAL16: v_maximum_f16_e64_dpp v5.l, v1.l, v2.l row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x68,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+# W32-FAKE16: v_maximum_f16_e64_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x68,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+# W64-REAL16: v_maximum_f16_e64_dpp v5.l, v1.l, v2.l row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x68,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+# W64-FAKE16: v_maximum_f16_e64_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x68,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
 
 0x05,0x00,0x68,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff
-# GFX12: v_maximum_f16_e64_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x68,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+# W32-REAL16: v_maximum_f16_e64_dpp v5.l, v1.l, v2.l row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x68,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+# W32-FAKE16: v_maximum_f16_e64_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x68,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+# W64-REAL16: v_maximum_f16_e64_dpp v5.l, v1.l, v2.l row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x68,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+# W64-FAKE16: v_maximum_f16_e64_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x68,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
 
 0x05,0x00,0x68,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff
-# GFX12: v_maximum_f16_e64_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x68,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+# W32-REAL16: v_maximum_f16_e64_dpp v5.l, v1.l, v2.l row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x68,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+# W32-FAKE16: v_maximum_f16_e64_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x68,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+# W64-REAL16: v_maximum_f16_e64_dpp v5.l, v1.l, v2.l row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x68,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+# W64-FAKE16: v_maximum_f16_e64_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x68,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
 
 0x05,0x00,0x68,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff
-# GFX12: v_maximum_f16_e64_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x68,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+# W32-REAL16: v_maximum_f16_e64_dpp v5.l, v1.l, v2.l row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x68,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+# W32-FAKE16: v_maximum_f16_e64_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x68,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+# W64-REAL16: v_maximum_f16_e64_dpp v5.l, v1.l, v2.l row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x68,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+# W64-FAKE16: v_maximum_f16_e64_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x68,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
 
 0x05,0x01,0x68,0xd7,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01
-# GFX12: v_maximum_f16_e64_dpp v5, |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x01,0x68,0xd7,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+# W32-REAL16: v_maximum_f16_e64_dpp v5.l, |v1.l|, -v2.l row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x01,0x68,0xd7,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+# W32-FAKE16: v_maximum_f16_e64_dpp v5, |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x01,0x68,0xd7,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+# W64-REAL16: v_maximum_f16_e64_dpp v5.l, |v1.l|, -v2.l row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x01,0x68,0xd7,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+# W64-FAKE16: v_maximum_f16_e64_dpp v5, |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x01,0x68,0xd7,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+
+0x05,0x48,0x68,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff
+# W32-REAL16: v_maximum_f16_e64_dpp v5.h, v1.h, v2.l op_sel:[1,0,1] quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x48,0x68,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+# W32-FAKE16: v_maximum_f16_e64_dpp v5, v1, v2 op_sel:[1,0,1] quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x48,0x68,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+# W64-REAL16: v_maximum_f16_e64_dpp v5.h, v1.h, v2.l op_sel:[1,0,1] quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x48,0x68,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+# W64-FAKE16: v_maximum_f16_e64_dpp v5, v1, v2 op_sel:[1,0,1] quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x48,0x68,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+
+0x05,0x48,0x68,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff
+# W32-REAL16: v_maximum_f16_e64_dpp v5.h, v1.h, v2.l op_sel:[1,0,1] row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x48,0x68,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+# W32-FAKE16: v_maximum_f16_e64_dpp v5, v1, v2 op_sel:[1,0,1] row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x48,0x68,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+# W64-REAL16: v_maximum_f16_e64_dpp v5.h, v1.h, v2.l op_sel:[1,0,1] row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x48,0x68,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+# W64-FAKE16: v_maximum_f16_e64_dpp v5, v1, v2 op_sel:[1,0,1] row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x48,0x68,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+
+0xff,0x5b,0x68,0xd7,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x05,0x30
+# W32-REAL16: v_maximum_f16_e64_dpp v255.h, -|v255.h|, -|v255.h| op_sel:[1,1,1] row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xff,0x5b,0x68,0xd7,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x05,0x30]
+# W32-FAKE16: v_maximum_f16_e64_dpp v255, -|v255|, -|v255| op_sel:[1,1,1] row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xff,0x5b,0x68,0xd7,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x05,0x30]
+# W64-REAL16: v_maximum_f16_e64_dpp v255.h, -|v255.h|, -|v255.h| op_sel:[1,1,1] row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xff,0x5b,0x68,0xd7,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x05,0x30]
+# W64-FAKE16: v_maximum_f16_e64_dpp v255, -|v255|, -|v255| op_sel:[1,1,1] row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xff,0x5b,0x68,0xd7,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x05,0x30]
 
 0xff,0x87,0x2e,0xd6,0xfa,0xfe,0xf7,0xfb,0xff,0x6f,0x0d,0x30
 # GFX12: v_maximum3_f32_e64_dpp v255, -|v255|, -|v255|, -|src_scc| clamp div:2 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x87,0x2e,0xd6,0xfa,0xfe,0xf7,0xfb,0xff,0x6f,0x0d,0x30]
@@ -7442,94 +7676,220 @@
 # GFX12: v_minimum3_f32_e64_dpp v5, |v1|, v2, -ttmp15 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x01,0x2d,0xd6,0xfa,0x04,0xee,0x81,0x01,0x1f,0x01,0xff]
 
 0xff,0x87,0x30,0xd6,0xfa,0xfe,0xf7,0xe3,0xff,0x6f,0x0d,0x30
-# GFX12: v_maximum3_f16_e64_dpp v255, -|v255|, -|v255|, -|src_scc| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x87,0x30,0xd6,0xfa,0xfe,0xf7,0xe3,0xff,0x6f,0x0d,0x30]
+# W32-REAL16: v_maximum3_f16_e64_dpp v255.l, -|v255.l|, -|v255.l|, -|src_scc| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x87,0x30,0xd6,0xfa,0xfe,0xf7,0xe3,0xff,0x6f,0x0d,0x30]
+# W32-FAKE16: v_maximum3_f16_e64_dpp v255, -|v255|, -|v255|, -|src_scc| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x87,0x30,0xd6,0xfa,0xfe,0xf7,0xe3,0xff,0x6f,0x0d,0x30]
+# W64-REAL16: v_maximum3_f16_e64_dpp v255.l, -|v255.l|, -|v255.l|, -|src_scc| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x87,0x30,0xd6,0xfa,0xfe,0xf7,0xe3,0xff,0x6f,0x0d,0x30]
+# W64-FAKE16: v_maximum3_f16_e64_dpp v255, -|v255|, -|v255|, -|src_scc| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x87,0x30,0xd6,0xfa,0xfe,0xf7,0xe3,0xff,0x6f,0x0d,0x30]
 
 0x05,0x04,0x30,0xd6,0xfa,0x04,0xfa,0x21,0x01,0x2f,0x01,0xff
-# GFX12: v_maximum3_f16_e64_dpp v5, -v1, v2, |exec_lo| row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x04,0x30,0xd6,0xfa,0x04,0xfa,0x21,0x01,0x2f,0x01,0xff]
+# W32-REAL16: v_maximum3_f16_e64_dpp v5.l, -v1.l, v2.l, |exec_lo| row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x04,0x30,0xd6,0xfa,0x04,0xfa,0x21,0x01,0x2f,0x01,0xff]
+# W32-FAKE16: v_maximum3_f16_e64_dpp v5, -v1, v2, |exec_lo| row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x04,0x30,0xd6,0xfa,0x04,0xfa,0x21,0x01,0x2f,0x01,0xff]
+# W64-REAL16: v_maximum3_f16_e64_dpp v5.l, -v1.l, v2.l, |exec_lo| row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x04,0x30,0xd6,0xfa,0x04,0xfa,0x21,0x01,0x2f,0x01,0xff]
+# W64-FAKE16: v_maximum3_f16_e64_dpp v5, -v1, v2, |exec_lo| row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x04,0x30,0xd6,0xfa,0x04,0xfa,0x21,0x01,0x2f,0x01,0xff]
 
 0x05,0x03,0x30,0xd6,0xfa,0x04,0xf2,0x61,0x01,0x50,0x01,0xff
-# GFX12: v_maximum3_f16_e64_dpp v5, -|v1|, -|v2|, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x03,0x30,0xd6,0xfa,0x04,0xf2,0x61,0x01,0x50,0x01,0xff]
+# W32-REAL16: v_maximum3_f16_e64_dpp v5.l, -|v1.l|, -|v2.l|, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x03,0x30,0xd6,0xfa,0x04,0xf2,0x61,0x01,0x50,0x01,0xff]
+# W32-FAKE16: v_maximum3_f16_e64_dpp v5, -|v1|, -|v2|, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x03,0x30,0xd6,0xfa,0x04,0xf2,0x61,0x01,0x50,0x01,0xff]
+# W64-REAL16: v_maximum3_f16_e64_dpp v5.l, -|v1.l|, -|v2.l|, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x03,0x30,0xd6,0xfa,0x04,0xf2,0x61,0x01,0x50,0x01,0xff]
+# W64-FAKE16: v_maximum3_f16_e64_dpp v5, -|v1|, -|v2|, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x03,0x30,0xd6,0xfa,0x04,0xf2,0x61,0x01,0x50,0x01,0xff]
 
 0x05,0x05,0x30,0xd6,0xfa,0x04,0x06,0xa3,0x01,0x5f,0x01,0x01
-# GFX12: v_maximum3_f16_e64_dpp v5, -|v1|, v2, -|-1| row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x05,0x30,0xd6,0xfa,0x04,0x06,0xa3,0x01,0x5f,0x01,0x01]
+# W32-REAL16: v_maximum3_f16_e64_dpp v5.l, -|v1.l|, v2.l, -|-1| row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x05,0x30,0xd6,0xfa,0x04,0x06,0xa3,0x01,0x5f,0x01,0x01]
+# W32-FAKE16: v_maximum3_f16_e64_dpp v5, -|v1|, v2, -|-1| row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x05,0x30,0xd6,0xfa,0x04,0x06,0xa3,0x01,0x5f,0x01,0x01]
+# W64-REAL16: v_maximum3_f16_e64_dpp v5.l, -|v1.l|, v2.l, -|-1| row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x05,0x30,0xd6,0xfa,0x04,0x06,0xa3,0x01,0x5f,0x01,0x01]
+# W64-FAKE16: v_maximum3_f16_e64_dpp v5, -|v1|, v2, -|-1| row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x05,0x30,0xd6,0xfa,0x04,0x06,0xa3,0x01,0x5f,0x01,0x01]
 
 0x05,0x06,0x30,0xd6,0xfa,0x04,0xc2,0xc3,0x01,0x60,0x09,0x13
-# GFX12: v_maximum3_f16_e64_dpp v5, v1, -|v2|, -|0.5| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x05,0x06,0x30,0xd6,0xfa,0x04,0xc2,0xc3,0x01,0x60,0x09,0x13]
+# W32-REAL16: v_maximum3_f16_e64_dpp v5.l, v1.l, -|v2.l|, -|0.5| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x05,0x06,0x30,0xd6,0xfa,0x04,0xc2,0xc3,0x01,0x60,0x09,0x13]
+# W32-FAKE16: v_maximum3_f16_e64_dpp v5, v1, -|v2|, -|0.5| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x05,0x06,0x30,0xd6,0xfa,0x04,0xc2,0xc3,0x01,0x60,0x09,0x13]
+# W64-REAL16: v_maximum3_f16_e64_dpp v5.l, v1.l, -|v2.l|, -|0.5| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x05,0x06,0x30,0xd6,0xfa,0x04,0xc2,0xc3,0x01,0x60,0x09,0x13]
+# W64-FAKE16: v_maximum3_f16_e64_dpp v5, v1, -|v2|, -|0.5| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x05,0x06,0x30,0xd6,0xfa,0x04,0xc2,0xc3,0x01,0x60,0x09,0x13]
 
 0x05,0x02,0x30,0xd6,0xfa,0x04,0xfe,0x41,0x01,0x21,0x01,0xff
-# GFX12: v_maximum3_f16_e64_dpp v5, v1, -|v2|, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x02,0x30,0xd6,0xfa,0x04,0xfe,0x41,0x01,0x21,0x01,0xff]
+# W32-REAL16: v_maximum3_f16_e64_dpp v5.l, v1.l, -|v2.l|, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x02,0x30,0xd6,0xfa,0x04,0xfe,0x41,0x01,0x21,0x01,0xff]
+# W32-FAKE16: v_maximum3_f16_e64_dpp v5, v1, -|v2|, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x02,0x30,0xd6,0xfa,0x04,0xfe,0x41,0x01,0x21,0x01,0xff]
+# W64-REAL16: v_maximum3_f16_e64_dpp v5.l, v1.l, -|v2.l|, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x02,0x30,0xd6,0xfa,0x04,0xfe,0x41,0x01,0x21,0x01,0xff]
+# W64-FAKE16: v_maximum3_f16_e64_dpp v5, v1, -|v2|, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x02,0x30,0xd6,0xfa,0x04,0xfe,0x41,0x01,0x21,0x01,0xff]
 
 0x05,0x00,0x30,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff
-# GFX12: v_maximum3_f16_e64_dpp v5, v1, v2, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x30,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
+# W32-REAL16: v_maximum3_f16_e64_dpp v5.l, v1.l, v2.l, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x30,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
+# W32-FAKE16: v_maximum3_f16_e64_dpp v5, v1, v2, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x30,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
+# W64-REAL16: v_maximum3_f16_e64_dpp v5.l, v1.l, v2.l, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x30,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
+# W64-FAKE16: v_maximum3_f16_e64_dpp v5, v1, v2, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x30,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
 
 0x05,0x00,0x30,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff
-# GFX12: v_maximum3_f16_e64_dpp v5, v1, v2, v255 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x30,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff]
+# W32-REAL16: v_maximum3_f16_e64_dpp v5.l, v1.l, v2.l, v255.l row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x30,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff]
+# W32-FAKE16: v_maximum3_f16_e64_dpp v5, v1, v2, v255 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x30,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff]
+# W64-REAL16: v_maximum3_f16_e64_dpp v5.l, v1.l, v2.l, v255.l row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x30,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff]
+# W64-FAKE16: v_maximum3_f16_e64_dpp v5, v1, v2, v255 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x30,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff]
 
 0x05,0x00,0x30,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff
-# GFX12: v_maximum3_f16_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x30,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
+# W32-REAL16: v_maximum3_f16_e64_dpp v5.l, v1.l, v2.l, v3.l quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x30,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
+# W32-FAKE16: v_maximum3_f16_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x30,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
+# W64-REAL16: v_maximum3_f16_e64_dpp v5.l, v1.l, v2.l, v3.l quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x30,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
+# W64-FAKE16: v_maximum3_f16_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x30,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
 
 0x05,0x00,0x30,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff
-# GFX12: v_maximum3_f16_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x30,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
+# W32-REAL16: v_maximum3_f16_e64_dpp v5.l, v1.l, v2.l, v3.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x30,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
+# W32-FAKE16: v_maximum3_f16_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x30,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
+# W64-REAL16: v_maximum3_f16_e64_dpp v5.l, v1.l, v2.l, v3.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x30,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
+# W64-FAKE16: v_maximum3_f16_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x30,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 
 0x05,0x00,0x30,0xd6,0xfa,0x06,0x0c,0x04,0x01,0x1b,0x00,0xff
-# GFX12: v_maximum3_f16_e64_dpp v5, v1, s3, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x30,0xd6,0xfa,0x06,0x0c,0x04,0x01,0x1b,0x00,0xff]
+# W32-REAL16: v_maximum3_f16_e64_dpp v5.l, v1.l, s3, v3.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x30,0xd6,0xfa,0x06,0x0c,0x04,0x01,0x1b,0x00,0xff]
+# W32-FAKE16: v_maximum3_f16_e64_dpp v5, v1, s3, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x30,0xd6,0xfa,0x06,0x0c,0x04,0x01,0x1b,0x00,0xff]
+# W64-REAL16: v_maximum3_f16_e64_dpp v5.l, v1.l, s3, v3.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x30,0xd6,0xfa,0x06,0x0c,0x04,0x01,0x1b,0x00,0xff]
+# W64-FAKE16: v_maximum3_f16_e64_dpp v5, v1, s3, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x30,0xd6,0xfa,0x06,0x0c,0x04,0x01,0x1b,0x00,0xff]
 
 0x05,0x00,0x30,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff
-# GFX12: v_maximum3_f16_e64_dpp v5, v1, v2, v3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x30,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
+# W32-REAL16: v_maximum3_f16_e64_dpp v5.l, v1.l, v2.l, v3.l row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x30,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
+# W32-FAKE16: v_maximum3_f16_e64_dpp v5, v1, v2, v3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x30,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
+# W64-REAL16: v_maximum3_f16_e64_dpp v5.l, v1.l, v2.l, v3.l row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x30,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
+# W64-FAKE16: v_maximum3_f16_e64_dpp v5, v1, v2, v3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x30,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
 
 0x05,0x00,0x30,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff
-# GFX12: v_maximum3_f16_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x30,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
+# W32-REAL16: v_maximum3_f16_e64_dpp v5.l, v1.l, v2.l, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x30,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
+# W32-FAKE16: v_maximum3_f16_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x30,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
+# W64-REAL16: v_maximum3_f16_e64_dpp v5.l, v1.l, v2.l, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x30,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
+# W64-FAKE16: v_maximum3_f16_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x30,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
 
 0x05,0x00,0x30,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff
-# GFX12: v_maximum3_f16_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x30,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
+# W32-REAL16: v_maximum3_f16_e64_dpp v5.l, v1.l, v2.l, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x30,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
+# W32-FAKE16: v_maximum3_f16_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x30,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
+# W64-REAL16: v_maximum3_f16_e64_dpp v5.l, v1.l, v2.l, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x30,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
+# W64-FAKE16: v_maximum3_f16_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x30,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
 
 0x05,0x01,0x30,0xd6,0xfa,0x04,0xee,0x81,0x01,0x1f,0x01,0xff
-# GFX12: v_maximum3_f16_e64_dpp v5, |v1|, v2, -ttmp15 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x01,0x30,0xd6,0xfa,0x04,0xee,0x81,0x01,0x1f,0x01,0xff]
+# W32-REAL16: v_maximum3_f16_e64_dpp v5.l, |v1.l|, v2.l, -ttmp15 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x01,0x30,0xd6,0xfa,0x04,0xee,0x81,0x01,0x1f,0x01,0xff]
+# W32-FAKE16: v_maximum3_f16_e64_dpp v5, |v1|, v2, -ttmp15 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x01,0x30,0xd6,0xfa,0x04,0xee,0x81,0x01,0x1f,0x01,0xff]
+# W64-REAL16: v_maximum3_f16_e64_dpp v5.l, |v1.l|, v2.l, -ttmp15 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x01,0x30,0xd6,0xfa,0x04,0xee,0x81,0x01,0x1f,0x01,0xff]
+# W64-FAKE16: v_maximum3_f16_e64_dpp v5, |v1|, v2, -ttmp15 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x01,0x30,0xd6,0xfa,0x04,0xee,0x81,0x01,0x1f,0x01,0xff]
+
+0x05,0x78,0x30,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff
+# W32-REAL16: v_maximum3_f16_e64_dpp v5.h, v1.h, v2.h, v3.h op_sel:[1,1,1,1] quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x78,0x30,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
+# W32-FAKE16: v_maximum3_f16_e64_dpp v5, v1, v2, v3 op_sel:[1,1,1,1] quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x78,0x30,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
+# W64-REAL16: v_maximum3_f16_e64_dpp v5.h, v1.h, v2.h, v3.h op_sel:[1,1,1,1] quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x78,0x30,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
+# W64-FAKE16: v_maximum3_f16_e64_dpp v5, v1, v2, v3 op_sel:[1,1,1,1] quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x78,0x30,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
+
+0x05,0x0a,0x30,0xd6,0xfa,0x04,0x06,0x23,0x01,0x5f,0x01,0x01
+# W32-REAL16: v_maximum3_f16_e64_dpp v5.l, -v1.h, |v2.l|, -1 op_sel:[1,0,0,0] row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x0a,0x30,0xd6,0xfa,0x04,0x06,0x23,0x01,0x5f,0x01,0x01]
+# W32-FAKE16: v_maximum3_f16_e64_dpp v5, -v1, |v2|, -1 op_sel:[1,0,0,0] row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x0a,0x30,0xd6,0xfa,0x04,0x06,0x23,0x01,0x5f,0x01,0x01]
+# W64-REAL16: v_maximum3_f16_e64_dpp v5.l, -v1.h, |v2.l|, -1 op_sel:[1,0,0,0] row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x0a,0x30,0xd6,0xfa,0x04,0x06,0x23,0x01,0x5f,0x01,0x01]
+# W64-FAKE16: v_maximum3_f16_e64_dpp v5, -v1, |v2|, -1 op_sel:[1,0,0,0] row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x0a,0x30,0xd6,0xfa,0x04,0x06,0x23,0x01,0x5f,0x01,0x01]
+
+0xff,0xc7,0x30,0xd6,0xfa,0xfe,0xf7,0xe3,0xff,0x6f,0x05,0x30
+# W32-REAL16: v_maximum3_f16_e64_dpp v255.h, -|v255.l|, -|v255.l|, -|src_scc| op_sel:[0,0,0,1] clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xff,0xc7,0x30,0xd6,0xfa,0xfe,0xf7,0xe3,0xff,0x6f,0x05,0x30]
+# W32-FAKE16: v_maximum3_f16_e64_dpp v255, -|v255|, -|v255|, -|src_scc| op_sel:[0,0,0,1] clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xff,0xc7,0x30,0xd6,0xfa,0xfe,0xf7,0xe3,0xff,0x6f,0x05,0x30]
+# W64-REAL16: v_maximum3_f16_e64_dpp v255.h, -|v255.l|, -|v255.l|, -|src_scc| op_sel:[0,0,0,1] clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xff,0xc7,0x30,0xd6,0xfa,0xfe,0xf7,0xe3,0xff,0x6f,0x05,0x30]
+# W64-FAKE16: v_maximum3_f16_e64_dpp v255, -|v255|, -|v255|, -|src_scc| op_sel:[0,0,0,1] clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xff,0xc7,0x30,0xd6,0xfa,0xfe,0xf7,0xe3,0xff,0x6f,0x05,0x30]
 
 0xff,0x87,0x2f,0xd6,0xfa,0xfe,0xf7,0xe3,0xff,0x6f,0x0d,0x30
-# GFX12: v_minimum3_f16_e64_dpp v255, -|v255|, -|v255|, -|src_scc| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x87,0x2f,0xd6,0xfa,0xfe,0xf7,0xe3,0xff,0x6f,0x0d,0x30]
+# W32-REAL16: v_minimum3_f16_e64_dpp v255.l, -|v255.l|, -|v255.l|, -|src_scc| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x87,0x2f,0xd6,0xfa,0xfe,0xf7,0xe3,0xff,0x6f,0x0d,0x30]
+# W32-FAKE16: v_minimum3_f16_e64_dpp v255, -|v255|, -|v255|, -|src_scc| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x87,0x2f,0xd6,0xfa,0xfe,0xf7,0xe3,0xff,0x6f,0x0d,0x30]
+# W64-REAL16: v_minimum3_f16_e64_dpp v255.l, -|v255.l|, -|v255.l|, -|src_scc| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x87,0x2f,0xd6,0xfa,0xfe,0xf7,0xe3,0xff,0x6f,0x0d,0x30]
+# W64-FAKE16: v_minimum3_f16_e64_dpp v255, -|v255|, -|v255|, -|src_scc| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x87,0x2f,0xd6,0xfa,0xfe,0xf7,0xe3,0xff,0x6f,0x0d,0x30]
 
 0x05,0x04,0x2f,0xd6,0xfa,0x04,0xfa,0x21,0x01,0x2f,0x01,0xff
-# GFX12: v_minimum3_f16_e64_dpp v5, -v1, v2, |exec_lo| row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x04,0x2f,0xd6,0xfa,0x04,0xfa,0x21,0x01,0x2f,0x01,0xff]
+# W32-REAL16: v_minimum3_f16_e64_dpp v5.l, -v1.l, v2.l, |exec_lo| row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x04,0x2f,0xd6,0xfa,0x04,0xfa,0x21,0x01,0x2f,0x01,0xff]
+# W32-FAKE16: v_minimum3_f16_e64_dpp v5, -v1, v2, |exec_lo| row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x04,0x2f,0xd6,0xfa,0x04,0xfa,0x21,0x01,0x2f,0x01,0xff]
+# W64-REAL16: v_minimum3_f16_e64_dpp v5.l, -v1.l, v2.l, |exec_lo| row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x04,0x2f,0xd6,0xfa,0x04,0xfa,0x21,0x01,0x2f,0x01,0xff]
+# W64-FAKE16: v_minimum3_f16_e64_dpp v5, -v1, v2, |exec_lo| row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x04,0x2f,0xd6,0xfa,0x04,0xfa,0x21,0x01,0x2f,0x01,0xff]
 
 0x05,0x03,0x2f,0xd6,0xfa,0x04,0xf2,0x61,0x01,0x50,0x01,0xff
-# GFX12: v_minimum3_f16_e64_dpp v5, -|v1|, -|v2|, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x03,0x2f,0xd6,0xfa,0x04,0xf2,0x61,0x01,0x50,0x01,0xff]
+# W32-REAL16: v_minimum3_f16_e64_dpp v5.l, -|v1.l|, -|v2.l|, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x03,0x2f,0xd6,0xfa,0x04,0xf2,0x61,0x01,0x50,0x01,0xff]
+# W32-FAKE16: v_minimum3_f16_e64_dpp v5, -|v1|, -|v2|, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x03,0x2f,0xd6,0xfa,0x04,0xf2,0x61,0x01,0x50,0x01,0xff]
+# W64-REAL16: v_minimum3_f16_e64_dpp v5.l, -|v1.l|, -|v2.l|, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x03,0x2f,0xd6,0xfa,0x04,0xf2,0x61,0x01,0x50,0x01,0xff]
+# W64-FAKE16: v_minimum3_f16_e64_dpp v5, -|v1|, -|v2|, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x03,0x2f,0xd6,0xfa,0x04,0xf2,0x61,0x01,0x50,0x01,0xff]
 
 0x05,0x05,0x2f,0xd6,0xfa,0x04,0x06,0xa3,0x01,0x5f,0x01,0x01
-# GFX12: v_minimum3_f16_e64_dpp v5, -|v1|, v2, -|-1| row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x05,0x2f,0xd6,0xfa,0x04,0x06,0xa3,0x01,0x5f,0x01,0x01]
+# W32-REAL16: v_minimum3_f16_e64_dpp v5.l, -|v1.l|, v2.l, -|-1| row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x05,0x2f,0xd6,0xfa,0x04,0x06,0xa3,0x01,0x5f,0x01,0x01]
+# W32-FAKE16: v_minimum3_f16_e64_dpp v5, -|v1|, v2, -|-1| row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x05,0x2f,0xd6,0xfa,0x04,0x06,0xa3,0x01,0x5f,0x01,0x01]
+# W64-REAL16: v_minimum3_f16_e64_dpp v5.l, -|v1.l|, v2.l, -|-1| row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x05,0x2f,0xd6,0xfa,0x04,0x06,0xa3,0x01,0x5f,0x01,0x01]
+# W64-FAKE16: v_minimum3_f16_e64_dpp v5, -|v1|, v2, -|-1| row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x05,0x2f,0xd6,0xfa,0x04,0x06,0xa3,0x01,0x5f,0x01,0x01]
 
 0x05,0x06,0x2f,0xd6,0xfa,0x04,0xc2,0xc3,0x01,0x60,0x09,0x13
-# GFX12: v_minimum3_f16_e64_dpp v5, v1, -|v2|, -|0.5| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x05,0x06,0x2f,0xd6,0xfa,0x04,0xc2,0xc3,0x01,0x60,0x09,0x13]
+# W32-REAL16: v_minimum3_f16_e64_dpp v5.l, v1.l, -|v2.l|, -|0.5| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x05,0x06,0x2f,0xd6,0xfa,0x04,0xc2,0xc3,0x01,0x60,0x09,0x13]
+# W32-FAKE16: v_minimum3_f16_e64_dpp v5, v1, -|v2|, -|0.5| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x05,0x06,0x2f,0xd6,0xfa,0x04,0xc2,0xc3,0x01,0x60,0x09,0x13]
+# W64-REAL16: v_minimum3_f16_e64_dpp v5.l, v1.l, -|v2.l|, -|0.5| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x05,0x06,0x2f,0xd6,0xfa,0x04,0xc2,0xc3,0x01,0x60,0x09,0x13]
+# W64-FAKE16: v_minimum3_f16_e64_dpp v5, v1, -|v2|, -|0.5| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x05,0x06,0x2f,0xd6,0xfa,0x04,0xc2,0xc3,0x01,0x60,0x09,0x13]
 
 0x05,0x02,0x2f,0xd6,0xfa,0x04,0xfe,0x41,0x01,0x21,0x01,0xff
-# GFX12: v_minimum3_f16_e64_dpp v5, v1, -|v2|, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x02,0x2f,0xd6,0xfa,0x04,0xfe,0x41,0x01,0x21,0x01,0xff]
+# W32-REAL16: v_minimum3_f16_e64_dpp v5.l, v1.l, -|v2.l|, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x02,0x2f,0xd6,0xfa,0x04,0xfe,0x41,0x01,0x21,0x01,0xff]
+# W32-FAKE16: v_minimum3_f16_e64_dpp v5, v1, -|v2|, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x02,0x2f,0xd6,0xfa,0x04,0xfe,0x41,0x01,0x21,0x01,0xff]
+# W64-REAL16: v_minimum3_f16_e64_dpp v5.l, v1.l, -|v2.l|, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x02,0x2f,0xd6,0xfa,0x04,0xfe,0x41,0x01,0x21,0x01,0xff]
+# W64-FAKE16: v_minimum3_f16_e64_dpp v5, v1, -|v2|, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x02,0x2f,0xd6,0xfa,0x04,0xfe,0x41,0x01,0x21,0x01,0xff]
 
 0x05,0x00,0x2f,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff
-# GFX12: v_minimum3_f16_e64_dpp v5, v1, v2, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x2f,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
+# W32-REAL16: v_minimum3_f16_e64_dpp v5.l, v1.l, v2.l, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x2f,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
+# W32-FAKE16: v_minimum3_f16_e64_dpp v5, v1, v2, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x2f,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
+# W64-REAL16: v_minimum3_f16_e64_dpp v5.l, v1.l, v2.l, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x2f,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
+# W64-FAKE16: v_minimum3_f16_e64_dpp v5, v1, v2, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x2f,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
 
 0x05,0x00,0x2f,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff
-# GFX12: v_minimum3_f16_e64_dpp v5, v1, v2, v255 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x2f,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff]
+# W32-REAL16: v_minimum3_f16_e64_dpp v5.l, v1.l, v2.l, v255.l row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x2f,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff]
+# W32-FAKE16: v_minimum3_f16_e64_dpp v5, v1, v2, v255 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x2f,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff]
+# W64-REAL16: v_minimum3_f16_e64_dpp v5.l, v1.l, v2.l, v255.l row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x2f,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff]
+# W64-FAKE16: v_minimum3_f16_e64_dpp v5, v1, v2, v255 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x2f,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff]
 
 0x05,0x00,0x2f,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff
-# GFX12: v_minimum3_f16_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x2f,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
+# W32-REAL16: v_minimum3_f16_e64_dpp v5.l, v1.l, v2.l, v3.l quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x2f,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
+# W32-FAKE16: v_minimum3_f16_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x2f,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
+# W64-REAL16: v_minimum3_f16_e64_dpp v5.l, v1.l, v2.l, v3.l quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x2f,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
+# W64-FAKE16: v_minimum3_f16_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x2f,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
 
 0x05,0x00,0x2f,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff
-# GFX12: v_minimum3_f16_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x2f,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
+# W32-REAL16: v_minimum3_f16_e64_dpp v5.l, v1.l, v2.l, v3.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x2f,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
+# W32-FAKE16: v_minimum3_f16_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x2f,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
+# W64-REAL16: v_minimum3_f16_e64_dpp v5.l, v1.l, v2.l, v3.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x2f,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
+# W64-FAKE16: v_minimum3_f16_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x2f,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 
 0x05,0x00,0x2f,0xd6,0xfa,0x06,0x0c,0x04,0x01,0x1b,0x00,0xff
-# GFX12: v_minimum3_f16_e64_dpp v5, v1, s3, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x2f,0xd6,0xfa,0x06,0x0c,0x04,0x01,0x1b,0x00,0xff]
+# W32-REAL16: v_minimum3_f16_e64_dpp v5.l, v1.l, s3, v3.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x2f,0xd6,0xfa,0x06,0x0c,0x04,0x01,0x1b,0x00,0xff]
+# W32-FAKE16: v_minimum3_f16_e64_dpp v5, v1, s3, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x2f,0xd6,0xfa,0x06,0x0c,0x04,0x01,0x1b,0x00,0xff]
+# W64-REAL16: v_minimum3_f16_e64_dpp v5.l, v1.l, s3, v3.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x2f,0xd6,0xfa,0x06,0x0c,0x04,0x01,0x1b,0x00,0xff]
+# W64-FAKE16: v_minimum3_f16_e64_dpp v5, v1, s3, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x2f,0xd6,0xfa,0x06,0x0c,0x04,0x01,0x1b,0x00,0xff]
 
 0x05,0x00,0x2f,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff
-# GFX12: v_minimum3_f16_e64_dpp v5, v1, v2, v3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x2f,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
+# W32-REAL16: v_minimum3_f16_e64_dpp v5.l, v1.l, v2.l, v3.l row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x2f,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
+# W32-FAKE16: v_minimum3_f16_e64_dpp v5, v1, v2, v3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x2f,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
+# W64-REAL16: v_minimum3_f16_e64_dpp v5.l, v1.l, v2.l, v3.l row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x2f,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
+# W64-FAKE16: v_minimum3_f16_e64_dpp v5, v1, v2, v3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x2f,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
 
 0x05,0x00,0x2f,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff
-# GFX12: v_minimum3_f16_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x2f,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
+# W32-REAL16: v_minimum3_f16_e64_dpp v5.l, v1.l, v2.l, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x2f,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
+# W32-FAKE16: v_minimum3_f16_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x2f,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
+# W64-REAL16: v_minimum3_f16_e64_dpp v5.l, v1.l, v2.l, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x2f,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
+# W64-FAKE16: v_minimum3_f16_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x2f,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
 
 0x05,0x00,0x2f,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff
-# GFX12: v_minimum3_f16_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x2f,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
+# W32-REAL16: v_minimum3_f16_e64_dpp v5.l, v1.l, v2.l, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x2f,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
+# W32-FAKE16: v_minimum3_f16_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x2f,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
+# W64-REAL16: v_minimum3_f16_e64_dpp v5.l, v1.l, v2.l, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x2f,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
+# W64-FAKE16: v_minimum3_f16_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x2f,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
 
 0x05,0x01,0x2f,0xd6,0xfa,0x04,0xee,0x81,0x01,0x1f,0x01,0xff
-# GFX12: v_minimum3_f16_e64_dpp v5, |v1|, v2, -ttmp15 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x01,0x2f,0xd6,0xfa,0x04,0xee,0x81,0x01,0x1f,0x01,0xff]
+# W32-REAL16: v_minimum3_f16_e64_dpp v5.l, |v1.l|, v2.l, -ttmp15 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x01,0x2f,0xd6,0xfa,0x04,0xee,0x81,0x01,0x1f,0x01,0xff]
+# W32-FAKE16: v_minimum3_f16_e64_dpp v5, |v1|, v2, -ttmp15 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x01,0x2f,0xd6,0xfa,0x04,0xee,0x81,0x01,0x1f,0x01,0xff]
+# W64-REAL16: v_minimum3_f16_e64_dpp v5.l, |v1.l|, v2.l, -ttmp15 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x01,0x2f,0xd6,0xfa,0x04,0xee,0x81,0x01,0x1f,0x01,0xff]
+# W64-FAKE16: v_minimum3_f16_e64_dpp v5, |v1|, v2, -ttmp15 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x01,0x2f,0xd6,0xfa,0x04,0xee,0x81,0x01,0x1f,0x01,0xff]
+
+0x05,0x78,0x2f,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff
+# W32-REAL16: v_minimum3_f16_e64_dpp v5.h, v1.h, v2.h, v3.h op_sel:[1,1,1,1] quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x78,0x2f,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
+# W32-FAKE16: v_minimum3_f16_e64_dpp v5, v1, v2, v3 op_sel:[1,1,1,1] quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x78,0x2f,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
+# W64-REAL16: v_minimum3_f16_e64_dpp v5.h, v1.h, v2.h, v3.h op_sel:[1,1,1,1] quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x78,0x2f,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
+# W64-FAKE16: v_minimum3_f16_e64_dpp v5, v1, v2, v3 op_sel:[1,1,1,1] quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x78,0x2f,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
+
+0x05,0x0a,0x2f,0xd6,0xfa,0x04,0x06,0x23,0x01,0x5f,0x01,0x01
+# W32-REAL16: v_minimum3_f16_e64_dpp v5.l, -v1.h, |v2.l|, -1 op_sel:[1,0,0,0] row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x0a,0x2f,0xd6,0xfa,0x04,0x06,0x23,0x01,0x5f,0x01,0x01]
+# W32-FAKE16: v_minimum3_f16_e64_dpp v5, -v1, |v2|, -1 op_sel:[1,0,0,0] row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x0a,0x2f,0xd6,0xfa,0x04,0x06,0x23,0x01,0x5f,0x01,0x01]
+# W64-REAL16: v_minimum3_f16_e64_dpp v5.l, -v1.h, |v2.l|, -1 op_sel:[1,0,0,0] row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x0a,0x2f,0xd6,0xfa,0x04,0x06,0x23,0x01,0x5f,0x01,0x01]
+# W64-FAKE16: v_minimum3_f16_e64_dpp v5, -v1, |v2|, -1 op_sel:[1,0,0,0] row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x0a,0x2f,0xd6,0xfa,0x04,0x06,0x23,0x01,0x5f,0x01,0x01]
+
+0xff,0xc7,0x2f,0xd6,0xfa,0xfe,0xf7,0xe3,0xff,0x6f,0x05,0x30
+# W32-REAL16: v_minimum3_f16_e64_dpp v255.h, -|v255.l|, -|v255.l|, -|src_scc| op_sel:[0,0,0,1] clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xff,0xc7,0x2f,0xd6,0xfa,0xfe,0xf7,0xe3,0xff,0x6f,0x05,0x30]
+# W32-FAKE16: v_minimum3_f16_e64_dpp v255, -|v255|, -|v255|, -|src_scc| op_sel:[0,0,0,1] clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xff,0xc7,0x2f,0xd6,0xfa,0xfe,0xf7,0xe3,0xff,0x6f,0x05,0x30]
+# W64-REAL16: v_minimum3_f16_e64_dpp v255.h, -|v255.l|, -|v255.l|, -|src_scc| op_sel:[0,0,0,1] clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xff,0xc7,0x2f,0xd6,0xfa,0xfe,0xf7,0xe3,0xff,0x6f,0x05,0x30]
+# W64-FAKE16: v_minimum3_f16_e64_dpp v255, -|v255|, -|v255|, -|src_scc| op_sel:[0,0,0,1] clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xff,0xc7,0x2f,0xd6,0xfa,0xfe,0xf7,0xe3,0xff,0x6f,0x05,0x30]
 
 0xff,0x87,0x6d,0xd6,0xfa,0xfe,0xf7,0xfb,0xff,0x6f,0x0d,0x30
 # GFX12: v_maximumminimum_f32_e64_dpp v255, -|v255|, -|v255|, -|src_scc| clamp div:2 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x87,0x6d,0xd6,0xfa,0xfe,0xf7,0xfb,0xff,0x6f,0x0d,0x30]
@@ -7622,91 +7982,218 @@
 # GFX12: v_minimummaximum_f32_e64_dpp v5, |v1|, v2, -ttmp15 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x01,0x6c,0xd6,0xfa,0x04,0xee,0x81,0x01,0x1f,0x01,0xff]
 
 0xff,0x87,0x6f,0xd6,0xfa,0xfe,0xf7,0xe3,0xff,0x6f,0x0d,0x30
-# GFX12: v_maximumminimum_f16_e64_dpp v255, -|v255|, -|v255|, -|src_scc| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x87,0x6f,0xd6,0xfa,0xfe,0xf7,0xe3,0xff,0x6f,0x0d,0x30]
+# W32-REAL16: v_maximumminimum_f16_e64_dpp v255.l, -|v255.l|, -|v255.l|, -|src_scc| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x87,0x6f,0xd6,0xfa,0xfe,0xf7,0xe3,0xff,0x6f,0x0d,0x30]
+# W32-FAKE16: v_maximumminimum_f16_e64_dpp v255, -|v255|, -|v255|, -|src_scc| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x87,0x6f,0xd6,0xfa,0xfe,0xf7,0xe3,0xff,0x6f,0x0d,0x30]
+# W64-REAL16: v_maximumminimum_f16_e64_dpp v255.l, -|v255.l|, -|v255.l|, -|src_scc| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x87,0x6f,0xd6,0xfa,0xfe,0xf7,0xe3,0xff,0x6f,0x0d,0x30]
+# W64-FAKE16: v_maximumminimum_f16_e64_dpp v255, -|v255|, -|v255|, -|src_scc| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x87,0x6f,0xd6,0xfa,0xfe,0xf7,0xe3,0xff,0x6f,0x0d,0x30]
 
 0x05,0x04,0x6f,0xd6,0xfa,0x04,0xfa,0x21,0x01,0x2f,0x01,0xff
-# GFX12: v_maximumminimum_f16_e64_dpp v5, -v1, v2, |exec_lo| row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x04,0x6f,0xd6,0xfa,0x04,0xfa,0x21,0x01,0x2f,0x01,0xff]
+# W32-REAL16: v_maximumminimum_f16_e64_dpp v5.l, -v1.l, v2.l, |exec_lo| row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x04,0x6f,0xd6,0xfa,0x04,0xfa,0x21,0x01,0x2f,0x01,0xff]
+# W32-FAKE16: v_maximumminimum_f16_e64_dpp v5, -v1, v2, |exec_lo| row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x04,0x6f,0xd6,0xfa,0x04,0xfa,0x21,0x01,0x2f,0x01,0xff]
+# W64-REAL16: v_maximumminimum_f16_e64_dpp v5.l, -v1.l, v2.l, |exec_lo| row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x04,0x6f,0xd6,0xfa,0x04,0xfa,0x21,0x01,0x2f,0x01,0xff]
+# W64-FAKE16: v_maximumminimum_f16_e64_dpp v5, -v1, v2, |exec_lo| row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x04,0x6f,0xd6,0xfa,0x04,0xfa,0x21,0x01,0x2f,0x01,0xff]
 
 0x05,0x03,0x6f,0xd6,0xfa,0x04,0xf2,0x61,0x01,0x50,0x01,0xff
-# GFX12: v_maximumminimum_f16_e64_dpp v5, -|v1|, -|v2|, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x03,0x6f,0xd6,0xfa,0x04,0xf2,0x61,0x01,0x50,0x01,0xff]
+# W32-REAL16: v_maximumminimum_f16_e64_dpp v5.l, -|v1.l|, -|v2.l|, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x03,0x6f,0xd6,0xfa,0x04,0xf2,0x61,0x01,0x50,0x01,0xff]
+# W32-FAKE16: v_maximumminimum_f16_e64_dpp v5, -|v1|, -|v2|, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x03,0x6f,0xd6,0xfa,0x04,0xf2,0x61,0x01,0x50,0x01,0xff]
+# W64-REAL16: v_maximumminimum_f16_e64_dpp v5.l, -|v1.l|, -|v2.l|, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x03,0x6f,0xd6,0xfa,0x04,0xf2,0x61,0x01,0x50,0x01,0xff]
+# W64-FAKE16: v_maximumminimum_f16_e64_dpp v5, -|v1|, -|v2|, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x03,0x6f,0xd6,0xfa,0x04,0xf2,0x61,0x01,0x50,0x01,0xff]
 
 0x05,0x05,0x6f,0xd6,0xfa,0x04,0x06,0xa3,0x01,0x5f,0x01,0x01
-# GFX12: v_maximumminimum_f16_e64_dpp v5, -|v1|, v2, -|-1| row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x05,0x6f,0xd6,0xfa,0x04,0x06,0xa3,0x01,0x5f,0x01,0x01]
+# W32-REAL16: v_maximumminimum_f16_e64_dpp v5.l, -|v1.l|, v2.l, -|-1| row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x05,0x6f,0xd6,0xfa,0x04,0x06,0xa3,0x01,0x5f,0x01,0x01]
+# W32-FAKE16: v_maximumminimum_f16_e64_dpp v5, -|v1|, v2, -|-1| row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x05,0x6f,0xd6,0xfa,0x04,0x06,0xa3,0x01,0x5f,0x01,0x01]
+# W64-REAL16: v_maximumminimum_f16_e64_dpp v5.l, -|v1.l|, v2.l, -|-1| row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x05,0x6f,0xd6,0xfa,0x04,0x06,0xa3,0x01,0x5f,0x01,0x01]
+# W64-FAKE16: v_maximumminimum_f16_e64_dpp v5, -|v1|, v2, -|-1| row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x05,0x6f,0xd6,0xfa,0x04,0x06,0xa3,0x01,0x5f,0x01,0x01]
 
 0x05,0x06,0x6f,0xd6,0xfa,0x04,0xc2,0xc3,0x01,0x60,0x09,0x13
-# GFX12: v_maximumminimum_f16_e64_dpp v5, v1, -|v2|, -|0.5| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x05,0x06,0x6f,0xd6,0xfa,0x04,0xc2,0xc3,0x01,0x60,0x09,0x13]
+# W32-REAL16: v_maximumminimum_f16_e64_dpp v5.l, v1.l, -|v2.l|, -|0.5| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x05,0x06,0x6f,0xd6,0xfa,0x04,0xc2,0xc3,0x01,0x60,0x09,0x13]
+# W32-FAKE16: v_maximumminimum_f16_e64_dpp v5, v1, -|v2|, -|0.5| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x05,0x06,0x6f,0xd6,0xfa,0x04,0xc2,0xc3,0x01,0x60,0x09,0x13]
+# W64-REAL16: v_maximumminimum_f16_e64_dpp v5.l, v1.l, -|v2.l|, -|0.5| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x05,0x06,0x6f,0xd6,0xfa,0x04,0xc2,0xc3,0x01,0x60,0x09,0x13]
+# W64-FAKE16: v_maximumminimum_f16_e64_dpp v5, v1, -|v2|, -|0.5| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x05,0x06,0x6f,0xd6,0xfa,0x04,0xc2,0xc3,0x01,0x60,0x09,0x13]
 
 0x05,0x02,0x6f,0xd6,0xfa,0x04,0xfe,0x41,0x01,0x21,0x01,0xff
-# GFX12: v_maximumminimum_f16_e64_dpp v5, v1, -|v2|, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x02,0x6f,0xd6,0xfa,0x04,0xfe,0x41,0x01,0x21,0x01,0xff]
+# W32-REAL16: v_maximumminimum_f16_e64_dpp v5.l, v1.l, -|v2.l|, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x02,0x6f,0xd6,0xfa,0x04,0xfe,0x41,0x01,0x21,0x01,0xff]
+# W32-FAKE16: v_maximumminimum_f16_e64_dpp v5, v1, -|v2|, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x02,0x6f,0xd6,0xfa,0x04,0xfe,0x41,0x01,0x21,0x01,0xff]
+# W64-REAL16: v_maximumminimum_f16_e64_dpp v5.l, v1.l, -|v2.l|, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x02,0x6f,0xd6,0xfa,0x04,0xfe,0x41,0x01,0x21,0x01,0xff]
+# W64-FAKE16: v_maximumminimum_f16_e64_dpp v5, v1, -|v2|, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x02,0x6f,0xd6,0xfa,0x04,0xfe,0x41,0x01,0x21,0x01,0xff]
 
 0x05,0x00,0x6f,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff
-# GFX12: v_maximumminimum_f16_e64_dpp v5, v1, v2, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x6f,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
+# W32-REAL16: v_maximumminimum_f16_e64_dpp v5.l, v1.l, v2.l, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x6f,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
+# W32-FAKE16: v_maximumminimum_f16_e64_dpp v5, v1, v2, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x6f,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
+# W64-REAL16: v_maximumminimum_f16_e64_dpp v5.l, v1.l, v2.l, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x6f,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
+# W64-FAKE16: v_maximumminimum_f16_e64_dpp v5, v1, v2, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x6f,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
 
 0x05,0x00,0x6f,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff
-# GFX12: v_maximumminimum_f16_e64_dpp v5, v1, v2, v255 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x6f,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff]
+# W32-REAL16: v_maximumminimum_f16_e64_dpp v5.l, v1.l, v2.l, v255.l row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x6f,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff]
+# W32-FAKE16: v_maximumminimum_f16_e64_dpp v5, v1, v2, v255 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x6f,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff]
+# W64-REAL16: v_maximumminimum_f16_e64_dpp v5.l, v1.l, v2.l, v255.l row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x6f,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff]
+# W64-FAKE16: v_maximumminimum_f16_e64_dpp v5, v1, v2, v255 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x6f,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff]
 
 0x05,0x00,0x6f,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff
-# GFX12: v_maximumminimum_f16_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x6f,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
+# W32-REAL16: v_maximumminimum_f16_e64_dpp v5.l, v1.l, v2.l, v3.l quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x6f,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
+# W32-FAKE16: v_maximumminimum_f16_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x6f,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
+# W64-REAL16: v_maximumminimum_f16_e64_dpp v5.l, v1.l, v2.l, v3.l quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x6f,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
+# W64-FAKE16: v_maximumminimum_f16_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x6f,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
 
 0x05,0x00,0x6f,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff
-# GFX12: v_maximumminimum_f16_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x6f,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
+# W32-REAL16: v_maximumminimum_f16_e64_dpp v5.l, v1.l, v2.l, v3.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x6f,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
+# W32-FAKE16: v_maximumminimum_f16_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x6f,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
+# W64-REAL16: v_maximumminimum_f16_e64_dpp v5.l, v1.l, v2.l, v3.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x6f,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
+# W64-FAKE16: v_maximumminimum_f16_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x6f,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 
 0x05,0x00,0x6f,0xd6,0xfa,0x06,0x0c,0x04,0x01,0x1b,0x00,0xff
-# GFX12: v_maximumminimum_f16_e64_dpp v5, v1, s3, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x6f,0xd6,0xfa,0x06,0x0c,0x04,0x01,0x1b,0x00,0xff]
+# W32-REAL16: v_maximumminimum_f16_e64_dpp v5.l, v1.l, s3, v3.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x6f,0xd6,0xfa,0x06,0x0c,0x04,0x01,0x1b,0x00,0xff]
+# W32-FAKE16: v_maximumminimum_f16_e64_dpp v5, v1, s3, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x6f,0xd6,0xfa,0x06,0x0c,0x04,0x01,0x1b,0x00,0xff]
+# W64-REAL16: v_maximumminimum_f16_e64_dpp v5.l, v1.l, s3, v3.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x6f,0xd6,0xfa,0x06,0x0c,0x04,0x01,0x1b,0x00,0xff]
+# W64-FAKE16: v_maximumminimum_f16_e64_dpp v5, v1, s3, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x6f,0xd6,0xfa,0x06,0x0c,0x04,0x01,0x1b,0x00,0xff]
 
 0x05,0x00,0x6f,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff
-# GFX12: v_maximumminimum_f16_e64_dpp v5, v1, v2, v3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x6f,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
+# W32-REAL16: v_maximumminimum_f16_e64_dpp v5.l, v1.l, v2.l, v3.l row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x6f,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
+# W32-FAKE16: v_maximumminimum_f16_e64_dpp v5, v1, v2, v3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x6f,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
+# W64-REAL16: v_maximumminimum_f16_e64_dpp v5.l, v1.l, v2.l, v3.l row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x6f,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
+# W64-FAKE16: v_maximumminimum_f16_e64_dpp v5, v1, v2, v3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x6f,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
 
 0x05,0x00,0x6f,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff
-# GFX12: v_maximumminimum_f16_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x6f,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
+# W32-REAL16: v_maximumminimum_f16_e64_dpp v5.l, v1.l, v2.l, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x6f,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
+# W32-FAKE16: v_maximumminimum_f16_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x6f,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
+# W64-REAL16: v_maximumminimum_f16_e64_dpp v5.l, v1.l, v2.l, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x6f,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
+# W64-FAKE16: v_maximumminimum_f16_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x6f,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
 
 0x05,0x00,0x6f,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff
-# GFX12: v_maximumminimum_f16_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x6f,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
+# W32-REAL16: v_maximumminimum_f16_e64_dpp v5.l, v1.l, v2.l, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x6f,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
+# W32-FAKE16: v_maximumminimum_f16_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x6f,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
+# W64-REAL16: v_maximumminimum_f16_e64_dpp v5.l, v1.l, v2.l, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x6f,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
+# W64-FAKE16: v_maximumminimum_f16_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x6f,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
 
 0x05,0x01,0x6f,0xd6,0xfa,0x04,0xee,0x81,0x01,0x1f,0x01,0xff
-# GFX12: v_maximumminimum_f16_e64_dpp v5, |v1|, v2, -ttmp15 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x01,0x6f,0xd6,0xfa,0x04,0xee,0x81,0x01,0x1f,0x01,0xff]
+# W32-REAL16: v_maximumminimum_f16_e64_dpp v5.l, |v1.l|, v2.l, -ttmp15 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x01,0x6f,0xd6,0xfa,0x04,0xee,0x81,0x01,0x1f,0x01,0xff]
+# W32-FAKE16: v_maximumminimum_f16_e64_dpp v5, |v1|, v2, -ttmp15 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x01,0x6f,0xd6,0xfa,0x04,0xee,0x81,0x01,0x1f,0x01,0xff]
+# W64-REAL16: v_maximumminimum_f16_e64_dpp v5.l, |v1.l|, v2.l, -ttmp15 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x01,0x6f,0xd6,0xfa,0x04,0xee,0x81,0x01,0x1f,0x01,0xff]
+# W64-FAKE16: v_maximumminimum_f16_e64_dpp v5, |v1|, v2, -ttmp15 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x01,0x6f,0xd6,0xfa,0x04,0xee,0x81,0x01,0x1f,0x01,0xff]
+
+0x05,0x78,0x6f,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff
+# W32-REAL16: v_maximumminimum_f16_e64_dpp v5.h, v1.h, v2.h, v3.h op_sel:[1,1,1,1] quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x78,0x6f,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
+# W32-FAKE16: v_maximumminimum_f16_e64_dpp v5, v1, v2, v3 op_sel:[1,1,1,1] quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x78,0x6f,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
+# W64-REAL16: v_maximumminimum_f16_e64_dpp v5.h, v1.h, v2.h, v3.h op_sel:[1,1,1,1] quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x78,0x6f,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
+# W64-FAKE16: v_maximumminimum_f16_e64_dpp v5, v1, v2, v3 op_sel:[1,1,1,1] quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x78,0x6f,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
+
+0x05,0x0a,0x6f,0xd6,0xfa,0x04,0x06,0x2b,0x01,0x5f,0x01,0x01
+# W32-REAL16: v_maximumminimum_f16_e64_dpp v5.l, -v1.h, |v2.l|, -1 op_sel:[1,0,0,0] mul:2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x0a,0x6f,0xd6,0xfa,0x04,0x06,0x2b,0x01,0x5f,0x01,0x01]
+# W32-FAKE16: v_maximumminimum_f16_e64_dpp v5, -v1, |v2|, -1 op_sel:[1,0,0,0] mul:2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x0a,0x6f,0xd6,0xfa,0x04,0x06,0x2b,0x01,0x5f,0x01,0x01]
+# W64-REAL16: v_maximumminimum_f16_e64_dpp v5.l, -v1.h, |v2.l|, -1 op_sel:[1,0,0,0] mul:2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x0a,0x6f,0xd6,0xfa,0x04,0x06,0x2b,0x01,0x5f,0x01,0x01]
+# W64-FAKE16: v_maximumminimum_f16_e64_dpp v5, -v1, |v2|, -1 op_sel:[1,0,0,0] mul:2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x0a,0x6f,0xd6,0xfa,0x04,0x06,0x2b,0x01,0x5f,0x01,0x01]
+
+0xff,0xc7,0x6f,0xd6,0xfa,0xfe,0xf7,0xfb,0xff,0x6f,0x05,0x30
+# W32-REAL16: v_maximumminimum_f16_e64_dpp v255.h, -|v255.l|, -|v255.l|, -|src_scc| op_sel:[0,0,0,1] clamp div:2 row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xff,0xc7,0x6f,0xd6,0xfa,0xfe,0xf7,0xfb,0xff,0x6f,0x05,0x30]
+# W32-FAKE16: v_maximumminimum_f16_e64_dpp v255, -|v255|, -|v255|, -|src_scc| op_sel:[0,0,0,1] clamp div:2 row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xff,0xc7,0x6f,0xd6,0xfa,0xfe,0xf7,0xfb,0xff,0x6f,0x05,0x30]
+# W64-REAL16: v_maximumminimum_f16_e64_dpp v255.h, -|v255.l|, -|v255.l|, -|src_scc| op_sel:[0,0,0,1] clamp div:2 row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xff,0xc7,0x6f,0xd6,0xfa,0xfe,0xf7,0xfb,0xff,0x6f,0x05,0x30]
+# W64-FAKE16: v_maximumminimum_f16_e64_dpp v255, -|v255|, -|v255|, -|src_scc| op_sel:[0,0,0,1] clamp div:2 row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xff,0xc7,0x6f,0xd6,0xfa,0xfe,0xf7,0xfb,0xff,0x6f,0x05,0x30]
 
 0xff,0x87,0x6e,0xd6,0xfa,0xfe,0xf7,0xe3,0xff,0x6f,0x0d,0x30
-# GFX12: v_minimummaximum_f16_e64_dpp v255, -|v255|, -|v255|, -|src_scc| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x87,0x6e,0xd6,0xfa,0xfe,0xf7,0xe3,0xff,0x6f,0x0d,0x30]
+# W32-REAL16: v_minimummaximum_f16_e64_dpp v255.l, -|v255.l|, -|v255.l|, -|src_scc| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x87,0x6e,0xd6,0xfa,0xfe,0xf7,0xe3,0xff,0x6f,0x0d,0x30]
+# W32-FAKE16: v_minimummaximum_f16_e64_dpp v255, -|v255|, -|v255|, -|src_scc| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x87,0x6e,0xd6,0xfa,0xfe,0xf7,0xe3,0xff,0x6f,0x0d,0x30]
+# W64-REAL16: v_minimummaximum_f16_e64_dpp v255.l, -|v255.l|, -|v255.l|, -|src_scc| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x87,0x6e,0xd6,0xfa,0xfe,0xf7,0xe3,0xff,0x6f,0x0d,0x30]
+# W64-FAKE16: v_minimummaximum_f16_e64_dpp v255, -|v255|, -|v255|, -|src_scc| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x87,0x6e,0xd6,0xfa,0xfe,0xf7,0xe3,0xff,0x6f,0x0d,0x30]
 
 0x05,0x04,0x6e,0xd6,0xfa,0x04,0xfa,0x21,0x01,0x2f,0x01,0xff
-# GFX12: v_minimummaximum_f16_e64_dpp v5, -v1, v2, |exec_lo| row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x04,0x6e,0xd6,0xfa,0x04,0xfa,0x21,0x01,0x2f,0x01,0xff]
+# W32-REAL16: v_minimummaximum_f16_e64_dpp v5.l, -v1.l, v2.l, |exec_lo| row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x04,0x6e,0xd6,0xfa,0x04,0xfa,0x21,0x01,0x2f,0x01,0xff]
+# W32-FAKE16: v_minimummaximum_f16_e64_dpp v5, -v1, v2, |exec_lo| row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x04,0x6e,0xd6,0xfa,0x04,0xfa,0x21,0x01,0x2f,0x01,0xff]
+# W64-REAL16: v_minimummaximum_f16_e64_dpp v5.l, -v1.l, v2.l, |exec_lo| row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x04,0x6e,0xd6,0xfa,0x04,0xfa,0x21,0x01,0x2f,0x01,0xff]
+# W64-FAKE16: v_minimummaximum_f16_e64_dpp v5, -v1, v2, |exec_lo| row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x04,0x6e,0xd6,0xfa,0x04,0xfa,0x21,0x01,0x2f,0x01,0xff]
 
 0x05,0x03,0x6e,0xd6,0xfa,0x04,0xf2,0x61,0x01,0x50,0x01,0xff
-# GFX12: v_minimummaximum_f16_e64_dpp v5, -|v1|, -|v2|, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x03,0x6e,0xd6,0xfa,0x04,0xf2,0x61,0x01,0x50,0x01,0xff]
+# W32-REAL16: v_minimummaximum_f16_e64_dpp v5.l, -|v1.l|, -|v2.l|, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x03,0x6e,0xd6,0xfa,0x04,0xf2,0x61,0x01,0x50,0x01,0xff]
+# W32-FAKE16: v_minimummaximum_f16_e64_dpp v5, -|v1|, -|v2|, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x03,0x6e,0xd6,0xfa,0x04,0xf2,0x61,0x01,0x50,0x01,0xff]
+# W64-REAL16: v_minimummaximum_f16_e64_dpp v5.l, -|v1.l|, -|v2.l|, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x03,0x6e,0xd6,0xfa,0x04,0xf2,0x61,0x01,0x50,0x01,0xff]
+# W64-FAKE16: v_minimummaximum_f16_e64_dpp v5, -|v1|, -|v2|, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x03,0x6e,0xd6,0xfa,0x04,0xf2,0x61,0x01,0x50,0x01,0xff]
 
 0x05,0x05,0x6e,0xd6,0xfa,0x04,0x06,0xa3,0x01,0x5f,0x01,0x01
-# GFX12: v_minimummaximum_f16_e64_dpp v5, -|v1|, v2, -|-1| row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x05,0x6e,0xd6,0xfa,0x04,0x06,0xa3,0x01,0x5f,0x01,0x01]
+# W32-REAL16: v_minimummaximum_f16_e64_dpp v5.l, -|v1.l|, v2.l, -|-1| row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x05,0x6e,0xd6,0xfa,0x04,0x06,0xa3,0x01,0x5f,0x01,0x01]
+# W32-FAKE16: v_minimummaximum_f16_e64_dpp v5, -|v1|, v2, -|-1| row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x05,0x6e,0xd6,0xfa,0x04,0x06,0xa3,0x01,0x5f,0x01,0x01]
+# W64-REAL16: v_minimummaximum_f16_e64_dpp v5.l, -|v1.l|, v2.l, -|-1| row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x05,0x6e,0xd6,0xfa,0x04,0x06,0xa3,0x01,0x5f,0x01,0x01]
+# W64-FAKE16: v_minimummaximum_f16_e64_dpp v5, -|v1|, v2, -|-1| row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x05,0x6e,0xd6,0xfa,0x04,0x06,0xa3,0x01,0x5f,0x01,0x01]
 
 0x05,0x06,0x6e,0xd6,0xfa,0x04,0xc2,0xc3,0x01,0x60,0x09,0x13
-# GFX12: v_minimummaximum_f16_e64_dpp v5, v1, -|v2|, -|0.5| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x05,0x06,0x6e,0xd6,0xfa,0x04,0xc2,0xc3,0x01,0x60,0x09,0x13]
+# W32-REAL16: v_minimummaximum_f16_e64_dpp v5.l, v1.l, -|v2.l|, -|0.5| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x05,0x06,0x6e,0xd6,0xfa,0x04,0xc2,0xc3,0x01,0x60,0x09,0x13]
+# W32-FAKE16: v_minimummaximum_f16_e64_dpp v5, v1, -|v2|, -|0.5| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x05,0x06,0x6e,0xd6,0xfa,0x04,0xc2,0xc3,0x01,0x60,0x09,0x13]
+# W64-REAL16: v_minimummaximum_f16_e64_dpp v5.l, v1.l, -|v2.l|, -|0.5| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x05,0x06,0x6e,0xd6,0xfa,0x04,0xc2,0xc3,0x01,0x60,0x09,0x13]
+# W64-FAKE16: v_minimummaximum_f16_e64_dpp v5, v1, -|v2|, -|0.5| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x05,0x06,0x6e,0xd6,0xfa,0x04,0xc2,0xc3,0x01,0x60,0x09,0x13]
 
 0x05,0x02,0x6e,0xd6,0xfa,0x04,0xfe,0x41,0x01,0x21,0x01,0xff
-# GFX12: v_minimummaximum_f16_e64_dpp v5, v1, -|v2|, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x02,0x6e,0xd6,0xfa,0x04,0xfe,0x41,0x01,0x21,0x01,0xff]
+# W32-REAL16: v_minimummaximum_f16_e64_dpp v5.l, v1.l, -|v2.l|, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x02,0x6e,0xd6,0xfa,0x04,0xfe,0x41,0x01,0x21,0x01,0xff]
+# W32-FAKE16: v_minimummaximum_f16_e64_dpp v5, v1, -|v2|, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x02,0x6e,0xd6,0xfa,0x04,0xfe,0x41,0x01,0x21,0x01,0xff]
+# W64-REAL16: v_minimummaximum_f16_e64_dpp v5.l, v1.l, -|v2.l|, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x02,0x6e,0xd6,0xfa,0x04,0xfe,0x41,0x01,0x21,0x01,0xff]
+# W64-FAKE16: v_minimummaximum_f16_e64_dpp v5, v1, -|v2|, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x02,0x6e,0xd6,0xfa,0x04,0xfe,0x41,0x01,0x21,0x01,0xff]
 
 0x05,0x00,0x6e,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff
-# GFX12: v_minimummaximum_f16_e64_dpp v5, v1, v2, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x6e,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
+# W32-REAL16: v_minimummaximum_f16_e64_dpp v5.l, v1.l, v2.l, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x6e,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
+# W32-FAKE16: v_minimummaximum_f16_e64_dpp v5, v1, v2, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x6e,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
+# W64-REAL16: v_minimummaximum_f16_e64_dpp v5.l, v1.l, v2.l, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x6e,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
+# W64-FAKE16: v_minimummaximum_f16_e64_dpp v5, v1, v2, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x6e,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
 
 0x05,0x00,0x6e,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff
-# GFX12: v_minimummaximum_f16_e64_dpp v5, v1, v2, v255 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x6e,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff]
+# W32-REAL16: v_minimummaximum_f16_e64_dpp v5.l, v1.l, v2.l, v255.l row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x6e,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff]
+# W32-FAKE16: v_minimummaximum_f16_e64_dpp v5, v1, v2, v255 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x6e,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff]
+# W64-REAL16: v_minimummaximum_f16_e64_dpp v5.l, v1.l, v2.l, v255.l row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x6e,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff]
+# W64-FAKE16: v_minimummaximum_f16_e64_dpp v5, v1, v2, v255 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x6e,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff]
 
 0x05,0x00,0x6e,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff
-# GFX12: v_minimummaximum_f16_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x6e,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
+# W32-REAL16: v_minimummaximum_f16_e64_dpp v5.l, v1.l, v2.l, v3.l quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x6e,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
+# W32-FAKE16: v_minimummaximum_f16_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x6e,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
+# W64-REAL16: v_minimummaximum_f16_e64_dpp v5.l, v1.l, v2.l, v3.l quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x6e,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
+# W64-FAKE16: v_minimummaximum_f16_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x6e,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
 
 0x05,0x00,0x6e,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff
-# GFX12: v_minimummaximum_f16_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x6e,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
+# W32-REAL16: v_minimummaximum_f16_e64_dpp v5.l, v1.l, v2.l, v3.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x6e,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
+# W32-FAKE16: v_minimummaximum_f16_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x6e,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
+# W64-REAL16: v_minimummaximum_f16_e64_dpp v5.l, v1.l, v2.l, v3.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x6e,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
+# W64-FAKE16: v_minimummaximum_f16_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x6e,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 
 0x05,0x00,0x6e,0xd6,0xfa,0x06,0x0c,0x04,0x01,0x1b,0x00,0xff
-# GFX12: v_minimummaximum_f16_e64_dpp v5, v1, s3, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x6e,0xd6,0xfa,0x06,0x0c,0x04,0x01,0x1b,0x00,0xff]
+# W32-REAL16: v_minimummaximum_f16_e64_dpp v5.l, v1.l, s3, v3.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x6e,0xd6,0xfa,0x06,0x0c,0x04,0x01,0x1b,0x00,0xff]
+# W32-FAKE16: v_minimummaximum_f16_e64_dpp v5, v1, s3, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x6e,0xd6,0xfa,0x06,0x0c,0x04,0x01,0x1b,0x00,0xff]
+# W64-REAL16: v_minimummaximum_f16_e64_dpp v5.l, v1.l, s3, v3.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x6e,0xd6,0xfa,0x06,0x0c,0x04,0x01,0x1b,0x00,0xff]
+# W64-FAKE16: v_minimummaximum_f16_e64_dpp v5, v1, s3, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x6e,0xd6,0xfa,0x06,0x0c,0x04,0x01,0x1b,0x00,0xff]
 
 0x05,0x00,0x6e,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff
-# GFX12: v_minimummaximum_f16_e64_dpp v5, v1, v2, v3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x6e,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
+# W32-REAL16: v_minimummaximum_f16_e64_dpp v5.l, v1.l, v2.l, v3.l row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x6e,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
+# W32-FAKE16: v_minimummaximum_f16_e64_dpp v5, v1, v2, v3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x6e,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
+# W64-REAL16: v_minimummaximum_f16_e64_dpp v5.l, v1.l, v2.l, v3.l row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x6e,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
+# W64-FAKE16: v_minimummaximum_f16_e64_dpp v5, v1, v2, v3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x6e,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
 
 0x05,0x00,0x6e,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff
-# GFX12: v_minimummaximum_f16_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x6e,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
+# W32-REAL16: v_minimummaximum_f16_e64_dpp v5.l, v1.l, v2.l, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x6e,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
+# W32-FAKE16: v_minimummaximum_f16_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x6e,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
+# W64-REAL16: v_minimummaximum_f16_e64_dpp v5.l, v1.l, v2.l, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x6e,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
+# W64-FAKE16: v_minimummaximum_f16_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x6e,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
 
 0x05,0x00,0x6e,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff
-# GFX12: v_minimummaximum_f16_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x6e,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
+# W32-REAL16: v_minimummaximum_f16_e64_dpp v5.l, v1.l, v2.l, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x6e,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
+# W32-FAKE16: v_minimummaximum_f16_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x6e,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
+# W64-REAL16: v_minimummaximum_f16_e64_dpp v5.l, v1.l, v2.l, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x6e,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
+# W64-FAKE16: v_minimummaximum_f16_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x6e,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
 
 0x05,0x01,0x6e,0xd6,0xfa,0x04,0xee,0x81,0x01,0x1f,0x01,0xff
-# GFX12: v_minimummaximum_f16_e64_dpp v5, |v1|, v2, -ttmp15 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x01,0x6e,0xd6,0xfa,0x04,0xee,0x81,0x01,0x1f,0x01,0xff]
+# W32-REAL16: v_minimummaximum_f16_e64_dpp v5.l, |v1.l|, v2.l, -ttmp15 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x01,0x6e,0xd6,0xfa,0x04,0xee,0x81,0x01,0x1f,0x01,0xff]
+# W32-FAKE16: v_minimummaximum_f16_e64_dpp v5, |v1|, v2, -ttmp15 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x01,0x6e,0xd6,0xfa,0x04,0xee,0x81,0x01,0x1f,0x01,0xff]
+# W64-REAL16: v_minimummaximum_f16_e64_dpp v5.l, |v1.l|, v2.l, -ttmp15 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x01,0x6e,0xd6,0xfa,0x04,0xee,0x81,0x01,0x1f,0x01,0xff]
+# W64-FAKE16: v_minimummaximum_f16_e64_dpp v5, |v1|, v2, -ttmp15 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x01,0x6e,0xd6,0xfa,0x04,0xee,0x81,0x01,0x1f,0x01,0xff]
+
+0x05,0x78,0x6e,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff
+# W32-REAL16: v_minimummaximum_f16_e64_dpp v5.h, v1.h, v2.h, v3.h op_sel:[1,1,1,1] quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x78,0x6e,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
+# W32-FAKE16: v_minimummaximum_f16_e64_dpp v5, v1, v2, v3 op_sel:[1,1,1,1] quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x78,0x6e,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
+# W64-REAL16: v_minimummaximum_f16_e64_dpp v5.h, v1.h, v2.h, v3.h op_sel:[1,1,1,1] quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x78,0x6e,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
+# W64-FAKE16: v_minimummaximum_f16_e64_dpp v5, v1, v2, v3 op_sel:[1,1,1,1] quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x78,0x6e,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
+
+0x05,0x0a,0x6e,0xd6,0xfa,0x04,0x06,0x2b,0x01,0x5f,0x01,0x01
+# W32-REAL16: v_minimummaximum_f16_e64_dpp v5.l, -v1.h, |v2.l|, -1 op_sel:[1,0,0,0] mul:2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x0a,0x6e,0xd6,0xfa,0x04,0x06,0x2b,0x01,0x5f,0x01,0x01]
+# W32-FAKE16: v_minimummaximum_f16_e64_dpp v5, -v1, |v2|, -1 op_sel:[1,0,0,0] mul:2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x0a,0x6e,0xd6,0xfa,0x04,0x06,0x2b,0x01,0x5f,0x01,0x01]
+# W64-REAL16: v_minimummaximum_f16_e64_dpp v5.l, -v1.h, |v2.l|, -1 op_sel:[1,0,0,0] mul:2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x0a,0x6e,0xd6,0xfa,0x04,0x06,0x2b,0x01,0x5f,0x01,0x01]
+# W64-FAKE16: v_minimummaximum_f16_e64_dpp v5, -v1, |v2|, -1 op_sel:[1,0,0,0] mul:2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x0a,0x6e,0xd6,0xfa,0x04,0x06,0x2b,0x01,0x5f,0x01,0x01]
+
+0xff,0xc7,0x6e,0xd6,0xfa,0xfe,0xf7,0xfb,0xff,0x6f,0x05,0x30
+# W32-REAL16: v_minimummaximum_f16_e64_dpp v255.h, -|v255.l|, -|v255.l|, -|src_scc| op_sel:[0,0,0,1] clamp div:2 row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xff,0xc7,0x6e,0xd6,0xfa,0xfe,0xf7,0xfb,0xff,0x6f,0x05,0x30]
+# W32-FAKE16: v_minimummaximum_f16_e64_dpp v255, -|v255|, -|v255|, -|src_scc| op_sel:[0,0,0,1] clamp div:2 row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xff,0xc7,0x6e,0xd6,0xfa,0xfe,0xf7,0xfb,0xff,0x6f,0x05,0x30]
+# W64-REAL16: v_minimummaximum_f16_e64_dpp v255.h, -|v255.l|, -|v255.l|, -|src_scc| op_sel:[0,0,0,1] clamp div:2 row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xff,0xc7,0x6e,0xd6,0xfa,0xfe,0xf7,0xfb,0xff,0x6f,0x05,0x30]
+# W64-FAKE16: v_minimummaximum_f16_e64_dpp v255, -|v255|, -|v255|, -|src_scc| op_sel:[0,0,0,1] clamp div:2 row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xff,0xc7,0x6e,0xd6,0xfa,0xfe,0xf7,0xfb,0xff,0x6f,0x05,0x30]
+
diff --git a/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vop3_dpp8.txt b/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vop3_dpp8.txt
index 343a71abb27d0..4182cd93f813c 100644
--- a/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vop3_dpp8.txt
+++ b/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vop3_dpp8.txt
@@ -147,13 +147,22 @@
 # GFX12: v_alignbit_b32_e64_dpp v255, v255, v255, src_scc dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x00,0x16,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00]
 
 0x05,0x00,0x17,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05
-# GFX12: v_alignbyte_b32_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x17,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
+# W32-REAL16: v_alignbyte_b32_e64_dpp v5, v1, v2, v3.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x17,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
+# W32-FAKE16: v_alignbyte_b32_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x17,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
+# W64-REAL16: v_alignbyte_b32_e64_dpp v5, v1, v2, v3.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x17,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
+# W64-FAKE16: v_alignbyte_b32_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x17,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 
 0x05,0x00,0x17,0xd6,0xe9,0x06,0x0c,0x04,0x01,0x77,0x39,0x05
-# GFX12: v_alignbyte_b32_e64_dpp v5, v1, s3, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x17,0xd6,0xe9,0x06,0x0c,0x04,0x01,0x77,0x39,0x05]
+# W32-REAL16: v_alignbyte_b32_e64_dpp v5, v1, s3, v3.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x17,0xd6,0xe9,0x06,0x0c,0x04,0x01,0x77,0x39,0x05]
+# W32-FAKE16: v_alignbyte_b32_e64_dpp v5, v1, s3, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x17,0xd6,0xe9,0x06,0x0c,0x04,0x01,0x77,0x39,0x05]
+# W64-REAL16: v_alignbyte_b32_e64_dpp v5, v1, s3, v3.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x17,0xd6,0xe9,0x06,0x0c,0x04,0x01,0x77,0x39,0x05]
+# W64-FAKE16: v_alignbyte_b32_e64_dpp v5, v1, s3, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x17,0xd6,0xe9,0x06,0x0c,0x04,0x01,0x77,0x39,0x05]
 
 0x05,0x00,0x17,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05
-# GFX12: v_alignbyte_b32_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x17,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
+# W32-REAL16: v_alignbyte_b32_e64_dpp v5, v1, v2, v255.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x17,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
+# W32-FAKE16: v_alignbyte_b32_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x17,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
+# W64-REAL16: v_alignbyte_b32_e64_dpp v5, v1, v2, v255.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x17,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
+# W64-FAKE16: v_alignbyte_b32_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x17,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 
 0x05,0x00,0x17,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05
 # GFX12: v_alignbyte_b32_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x17,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
@@ -182,6 +191,12 @@
 0xff,0x00,0x17,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00
 # GFX12: v_alignbyte_b32_e64_dpp v255, v255, v255, src_scc dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x00,0x17,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00]
 
+0x05,0x20,0x17,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05
+# W32-REAL16: v_alignbyte_b32_e64_dpp v5, v1, v2, v255.h op_sel:[0,0,1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x20,0x17,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
+# W32-FAKE16: v_alignbyte_b32_e64_dpp v5, v1, v2, v255 op_sel:[0,0,1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x20,0x17,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
+# W64-REAL16: v_alignbyte_b32_e64_dpp v5, v1, v2, v255.h op_sel:[0,0,1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x20,0x17,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
+# W64-FAKE16: v_alignbyte_b32_e64_dpp v5, v1, v2, v255 op_sel:[0,0,1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x20,0x17,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
+
 0x05,0x00,0x62,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 # W32-REAL16: v_and_b16_e64_dpp v5.l, v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x62,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W32-FAKE16: v_and_b16_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x62,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
@@ -3251,49 +3266,94 @@
 # W64-FAKE16: v_mad_i16_e64_dpp v255, v255, v255, src_scc op_sel:[0,0,0,1] clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0xc0,0x53,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00]
 
 0x05,0x00,0x5a,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05
-# GFX12: v_mad_i32_i16_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x5a,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
+# W32-REAL16: v_mad_i32_i16_e64_dpp v5, v1.l, v2.l, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x5a,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
+# W32-FAKE16: v_mad_i32_i16_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x5a,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
+# W64-REAL16: v_mad_i32_i16_e64_dpp v5, v1.l, v2.l, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x5a,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
+# W64-FAKE16: v_mad_i32_i16_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x5a,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 
 0x05,0x00,0x5a,0xd6,0xe9,0x06,0x0c,0x04,0x01,0x77,0x39,0x05
-# GFX12: v_mad_i32_i16_e64_dpp v5, v1, s3, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x5a,0xd6,0xe9,0x06,0x0c,0x04,0x01,0x77,0x39,0x05]
+# W32-REAL16: v_mad_i32_i16_e64_dpp v5, v1.l, s3, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x5a,0xd6,0xe9,0x06,0x0c,0x04,0x01,0x77,0x39,0x05]
+# W32-FAKE16: v_mad_i32_i16_e64_dpp v5, v1, s3, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x5a,0xd6,0xe9,0x06,0x0c,0x04,0x01,0x77,0x39,0x05]
+# W64-REAL16: v_mad_i32_i16_e64_dpp v5, v1.l, s3, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x5a,0xd6,0xe9,0x06,0x0c,0x04,0x01,0x77,0x39,0x05]
+# W64-FAKE16: v_mad_i32_i16_e64_dpp v5, v1, s3, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x5a,0xd6,0xe9,0x06,0x0c,0x04,0x01,0x77,0x39,0x05]
 
 0x05,0x00,0x5a,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05
-# GFX12: v_mad_i32_i16_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x5a,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
+# W32-REAL16: v_mad_i32_i16_e64_dpp v5, v1.l, v2.l, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x5a,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
+# W32-FAKE16: v_mad_i32_i16_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x5a,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
+# W64-REAL16: v_mad_i32_i16_e64_dpp v5, v1.l, v2.l, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x5a,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
+# W64-FAKE16: v_mad_i32_i16_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x5a,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 
 0x05,0x00,0x5a,0xd6,0xe9,0x04,0x0e,0x00,0x01,0x77,0x39,0x05
-# GFX12: v_mad_i32_i16_e64_dpp v5, v1, v2, s3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x5a,0xd6,0xe9,0x04,0x0e,0x00,0x01,0x77,0x39,0x05]
+# W32-REAL16: v_mad_i32_i16_e64_dpp v5, v1.l, v2.l, s3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x5a,0xd6,0xe9,0x04,0x0e,0x00,0x01,0x77,0x39,0x05]
+# W32-FAKE16: v_mad_i32_i16_e64_dpp v5, v1, v2, s3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x5a,0xd6,0xe9,0x04,0x0e,0x00,0x01,0x77,0x39,0x05]
+# W64-REAL16: v_mad_i32_i16_e64_dpp v5, v1.l, v2.l, s3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x5a,0xd6,0xe9,0x04,0x0e,0x00,0x01,0x77,0x39,0x05]
+# W64-FAKE16: v_mad_i32_i16_e64_dpp v5, v1, v2, s3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x5a,0xd6,0xe9,0x04,0x0e,0x00,0x01,0x77,0x39,0x05]
 
 0x05,0x00,0x5a,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05
-# GFX12: v_mad_i32_i16_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x5a,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
+# W32-REAL16: v_mad_i32_i16_e64_dpp v5, v1.l, v2.l, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x5a,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
+# W32-FAKE16: v_mad_i32_i16_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x5a,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
+# W64-REAL16: v_mad_i32_i16_e64_dpp v5, v1.l, v2.l, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x5a,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
+# W64-FAKE16: v_mad_i32_i16_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x5a,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 
 0x05,0x00,0x5a,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05
-# GFX12: v_mad_i32_i16_e64_dpp v5, v1, v2, ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x5a,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05]
+# W32-REAL16: v_mad_i32_i16_e64_dpp v5, v1.l, v2.l, ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x5a,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05]
+# W32-FAKE16: v_mad_i32_i16_e64_dpp v5, v1, v2, ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x5a,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05]
+# W64-REAL16: v_mad_i32_i16_e64_dpp v5, v1.l, v2.l, ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x5a,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05]
+# W64-FAKE16: v_mad_i32_i16_e64_dpp v5, v1, v2, ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x5a,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05]
 
 0x05,0x00,0x5a,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05
-# GFX12: v_mad_i32_i16_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x5a,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
+# W32-REAL16: v_mad_i32_i16_e64_dpp v5, v1.l, v2.l, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x5a,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
+# W32-FAKE16: v_mad_i32_i16_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x5a,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
+# W64-REAL16: v_mad_i32_i16_e64_dpp v5, v1.l, v2.l, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x5a,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
+# W64-FAKE16: v_mad_i32_i16_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x5a,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 
 0x05,0x00,0x5a,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05
-# GFX12: v_mad_i32_i16_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x5a,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
+# W32-REAL16: v_mad_i32_i16_e64_dpp v5, v1.l, v2.l, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x5a,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
+# W32-FAKE16: v_mad_i32_i16_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x5a,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
+# W64-REAL16: v_mad_i32_i16_e64_dpp v5, v1.l, v2.l, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x5a,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
+# W64-FAKE16: v_mad_i32_i16_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x5a,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 
 0x05,0x00,0x5a,0xd6,0xe9,0x04,0xf6,0x01,0x01,0x77,0x39,0x05
-# GFX12: v_mad_i32_i16_e64_dpp v5, v1, v2, m0 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x5a,0xd6,0xe9,0x04,0xf6,0x01,0x01,0x77,0x39,0x05]
+# W32-REAL16: v_mad_i32_i16_e64_dpp v5, v1.l, v2.l, m0 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x5a,0xd6,0xe9,0x04,0xf6,0x01,0x01,0x77,0x39,0x05]
+# W32-FAKE16: v_mad_i32_i16_e64_dpp v5, v1, v2, m0 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x5a,0xd6,0xe9,0x04,0xf6,0x01,0x01,0x77,0x39,0x05]
+# W64-REAL16: v_mad_i32_i16_e64_dpp v5, v1.l, v2.l, m0 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x5a,0xd6,0xe9,0x04,0xf6,0x01,0x01,0x77,0x39,0x05]
+# W64-FAKE16: v_mad_i32_i16_e64_dpp v5, v1, v2, m0 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x5a,0xd6,0xe9,0x04,0xf6,0x01,0x01,0x77,0x39,0x05]
 
 0x05,0x00,0x5a,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05
-# GFX12: v_mad_i32_i16_e64_dpp v5, v1, v2, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x5a,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05]
+# W32-REAL16: v_mad_i32_i16_e64_dpp v5, v1.l, v2.l, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x5a,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05]
+# W32-FAKE16: v_mad_i32_i16_e64_dpp v5, v1, v2, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x5a,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05]
+# W64-REAL16: v_mad_i32_i16_e64_dpp v5, v1.l, v2.l, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x5a,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05]
+# W64-FAKE16: v_mad_i32_i16_e64_dpp v5, v1, v2, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x5a,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05]
 
 0x05,0x00,0x5a,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05
-# GFX12: v_mad_i32_i16_e64_dpp v5, v1, v2, exec_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x5a,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05]
+# W32-REAL16: v_mad_i32_i16_e64_dpp v5, v1.l, v2.l, exec_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x5a,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05]
+# W32-FAKE16: v_mad_i32_i16_e64_dpp v5, v1, v2, exec_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x5a,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05]
+# W64-REAL16: v_mad_i32_i16_e64_dpp v5, v1.l, v2.l, exec_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x5a,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05]
+# W64-FAKE16: v_mad_i32_i16_e64_dpp v5, v1, v2, exec_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x5a,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05]
 
 0x05,0x00,0x5a,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05
-# GFX12: v_mad_i32_i16_e64_dpp v5, v1, v2, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x5a,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05]
+# W32-REAL16: v_mad_i32_i16_e64_dpp v5, v1.l, v2.l, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x5a,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05]
+# W32-FAKE16: v_mad_i32_i16_e64_dpp v5, v1, v2, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x5a,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05]
+# W64-REAL16: v_mad_i32_i16_e64_dpp v5, v1.l, v2.l, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x5a,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05]
+# W64-FAKE16: v_mad_i32_i16_e64_dpp v5, v1, v2, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x5a,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05]
 
 0x05,0x00,0x5a,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05
-# GFX12: v_mad_i32_i16_e64_dpp v5, v1, v2, -1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x5a,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05]
+# W32-REAL16: v_mad_i32_i16_e64_dpp v5, v1.l, v2.l, -1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x5a,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05]
+# W32-FAKE16: v_mad_i32_i16_e64_dpp v5, v1, v2, -1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x5a,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05]
+# W64-REAL16: v_mad_i32_i16_e64_dpp v5, v1.l, v2.l, -1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x5a,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05]
+# W64-FAKE16: v_mad_i32_i16_e64_dpp v5, v1, v2, -1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x5a,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05]
 
 0x05,0x08,0x5a,0xd6,0xe9,0x04,0xc2,0x03,0x01,0x77,0x39,0x05
-# GFX12: v_mad_i32_i16_e64_dpp v5, v1, v2, 0.5 op_sel:[1,0,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x08,0x5a,0xd6,0xe9,0x04,0xc2,0x03,0x01,0x77,0x39,0x05]
+# W32-REAL16: v_mad_i32_i16_e64_dpp v5, v1.h, v2.l, 0.5 op_sel:[1,0,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x08,0x5a,0xd6,0xe9,0x04,0xc2,0x03,0x01,0x77,0x39,0x05]
+# W32-FAKE16: v_mad_i32_i16_e64_dpp v5, v1, v2, 0.5 op_sel:[1,0,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x08,0x5a,0xd6,0xe9,0x04,0xc2,0x03,0x01,0x77,0x39,0x05]
+# W64-REAL16: v_mad_i32_i16_e64_dpp v5, v1.h, v2.l, 0.5 op_sel:[1,0,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x08,0x5a,0xd6,0xe9,0x04,0xc2,0x03,0x01,0x77,0x39,0x05]
+# W64-FAKE16: v_mad_i32_i16_e64_dpp v5, v1, v2, 0.5 op_sel:[1,0,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x08,0x5a,0xd6,0xe9,0x04,0xc2,0x03,0x01,0x77,0x39,0x05]
 
 0xff,0x90,0x5a,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00
-# GFX12: v_mad_i32_i16_e64_dpp v255, v255, v255, src_scc op_sel:[0,1,0,0] clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x90,0x5a,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00]
+# W32-REAL16: v_mad_i32_i16_e64_dpp v255, v255.l, v255.h, src_scc op_sel:[0,1,0,0] clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x90,0x5a,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00]
+# W32-FAKE16: v_mad_i32_i16_e64_dpp v255, v255, v255, src_scc op_sel:[0,1,0,0] clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x90,0x5a,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00]
+# W64-REAL16: v_mad_i32_i16_e64_dpp v255, v255.l, v255.h, src_scc op_sel:[0,1,0,0] clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x90,0x5a,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00]
+# W64-FAKE16: v_mad_i32_i16_e64_dpp v255, v255, v255, src_scc op_sel:[0,1,0,0] clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x90,0x5a,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00]
 
 0x05,0x00,0x41,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05
 # W32-REAL16: v_mad_u16_e64_dpp v5.l, v1.l, v2.l, v3.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x41,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
@@ -3440,49 +3500,94 @@
 # W64-FAKE16: v_mad_u16_e64_dpp v255, v255, v255, src_scc op_sel:[0,0,0,1] clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0xc0,0x41,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00]
 
 0x05,0x00,0x59,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05
-# GFX12: v_mad_u32_u16_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x59,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
+# W32-REAL16: v_mad_u32_u16_e64_dpp v5, v1.l, v2.l, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x59,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
+# W32-FAKE16: v_mad_u32_u16_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x59,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
+# W64-REAL16: v_mad_u32_u16_e64_dpp v5, v1.l, v2.l, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x59,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
+# W64-FAKE16: v_mad_u32_u16_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x59,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 
 0x05,0x00,0x59,0xd6,0xe9,0x06,0x0c,0x04,0x01,0x77,0x39,0x05
-# GFX12: v_mad_u32_u16_e64_dpp v5, v1, s3, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x59,0xd6,0xe9,0x06,0x0c,0x04,0x01,0x77,0x39,0x05]
+# W32-REAL16: v_mad_u32_u16_e64_dpp v5, v1.l, s3, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x59,0xd6,0xe9,0x06,0x0c,0x04,0x01,0x77,0x39,0x05]
+# W32-FAKE16: v_mad_u32_u16_e64_dpp v5, v1, s3, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x59,0xd6,0xe9,0x06,0x0c,0x04,0x01,0x77,0x39,0x05]
+# W64-REAL16: v_mad_u32_u16_e64_dpp v5, v1.l, s3, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x59,0xd6,0xe9,0x06,0x0c,0x04,0x01,0x77,0x39,0x05]
+# W64-FAKE16: v_mad_u32_u16_e64_dpp v5, v1, s3, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x59,0xd6,0xe9,0x06,0x0c,0x04,0x01,0x77,0x39,0x05]
 
 0x05,0x00,0x59,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05
-# GFX12: v_mad_u32_u16_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x59,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
+# W32-REAL16: v_mad_u32_u16_e64_dpp v5, v1.l, v2.l, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x59,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
+# W32-FAKE16: v_mad_u32_u16_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x59,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
+# W64-REAL16: v_mad_u32_u16_e64_dpp v5, v1.l, v2.l, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x59,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
+# W64-FAKE16: v_mad_u32_u16_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x59,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 
 0x05,0x00,0x59,0xd6,0xe9,0x04,0x0e,0x00,0x01,0x77,0x39,0x05
-# GFX12: v_mad_u32_u16_e64_dpp v5, v1, v2, s3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x59,0xd6,0xe9,0x04,0x0e,0x00,0x01,0x77,0x39,0x05]
+# W32-REAL16: v_mad_u32_u16_e64_dpp v5, v1.l, v2.l, s3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x59,0xd6,0xe9,0x04,0x0e,0x00,0x01,0x77,0x39,0x05]
+# W32-FAKE16: v_mad_u32_u16_e64_dpp v5, v1, v2, s3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x59,0xd6,0xe9,0x04,0x0e,0x00,0x01,0x77,0x39,0x05]
+# W64-REAL16: v_mad_u32_u16_e64_dpp v5, v1.l, v2.l, s3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x59,0xd6,0xe9,0x04,0x0e,0x00,0x01,0x77,0x39,0x05]
+# W64-FAKE16: v_mad_u32_u16_e64_dpp v5, v1, v2, s3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x59,0xd6,0xe9,0x04,0x0e,0x00,0x01,0x77,0x39,0x05]
 
 0x05,0x00,0x59,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05
-# GFX12: v_mad_u32_u16_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x59,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
+# W32-REAL16: v_mad_u32_u16_e64_dpp v5, v1.l, v2.l, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x59,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
+# W32-FAKE16: v_mad_u32_u16_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x59,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
+# W64-REAL16: v_mad_u32_u16_e64_dpp v5, v1.l, v2.l, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x59,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
+# W64-FAKE16: v_mad_u32_u16_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x59,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 
 0x05,0x00,0x59,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05
-# GFX12: v_mad_u32_u16_e64_dpp v5, v1, v2, ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x59,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05]
+# W32-REAL16: v_mad_u32_u16_e64_dpp v5, v1.l, v2.l, ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x59,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05]
+# W32-FAKE16: v_mad_u32_u16_e64_dpp v5, v1, v2, ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x59,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05]
+# W64-REAL16: v_mad_u32_u16_e64_dpp v5, v1.l, v2.l, ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x59,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05]
+# W64-FAKE16: v_mad_u32_u16_e64_dpp v5, v1, v2, ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x59,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05]
 
 0x05,0x00,0x59,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05
-# GFX12: v_mad_u32_u16_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x59,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
+# W32-REAL16: v_mad_u32_u16_e64_dpp v5, v1.l, v2.l, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x59,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
+# W32-FAKE16: v_mad_u32_u16_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x59,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
+# W64-REAL16: v_mad_u32_u16_e64_dpp v5, v1.l, v2.l, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x59,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
+# W64-FAKE16: v_mad_u32_u16_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x59,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 
 0x05,0x00,0x59,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05
-# GFX12: v_mad_u32_u16_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x59,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
+# W32-REAL16: v_mad_u32_u16_e64_dpp v5, v1.l, v2.l, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x59,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
+# W32-FAKE16: v_mad_u32_u16_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x59,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
+# W64-REAL16: v_mad_u32_u16_e64_dpp v5, v1.l, v2.l, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x59,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
+# W64-FAKE16: v_mad_u32_u16_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x59,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 
 0x05,0x00,0x59,0xd6,0xe9,0x04,0xf6,0x01,0x01,0x77,0x39,0x05
-# GFX12: v_mad_u32_u16_e64_dpp v5, v1, v2, m0 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x59,0xd6,0xe9,0x04,0xf6,0x01,0x01,0x77,0x39,0x05]
+# W32-REAL16: v_mad_u32_u16_e64_dpp v5, v1.l, v2.l, m0 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x59,0xd6,0xe9,0x04,0xf6,0x01,0x01,0x77,0x39,0x05]
+# W32-FAKE16: v_mad_u32_u16_e64_dpp v5, v1, v2, m0 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x59,0xd6,0xe9,0x04,0xf6,0x01,0x01,0x77,0x39,0x05]
+# W64-REAL16: v_mad_u32_u16_e64_dpp v5, v1.l, v2.l, m0 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x59,0xd6,0xe9,0x04,0xf6,0x01,0x01,0x77,0x39,0x05]
+# W64-FAKE16: v_mad_u32_u16_e64_dpp v5, v1, v2, m0 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x59,0xd6,0xe9,0x04,0xf6,0x01,0x01,0x77,0x39,0x05]
 
 0x05,0x00,0x59,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05
-# GFX12: v_mad_u32_u16_e64_dpp v5, v1, v2, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x59,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05]
+# W32-REAL16: v_mad_u32_u16_e64_dpp v5, v1.l, v2.l, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x59,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05]
+# W32-FAKE16: v_mad_u32_u16_e64_dpp v5, v1, v2, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x59,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05]
+# W64-REAL16: v_mad_u32_u16_e64_dpp v5, v1.l, v2.l, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x59,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05]
+# W64-FAKE16: v_mad_u32_u16_e64_dpp v5, v1, v2, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x59,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05]
 
 0x05,0x00,0x59,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05
-# GFX12: v_mad_u32_u16_e64_dpp v5, v1, v2, exec_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x59,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05]
+# W32-REAL16: v_mad_u32_u16_e64_dpp v5, v1.l, v2.l, exec_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x59,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05]
+# W32-FAKE16: v_mad_u32_u16_e64_dpp v5, v1, v2, exec_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x59,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05]
+# W64-REAL16: v_mad_u32_u16_e64_dpp v5, v1.l, v2.l, exec_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x59,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05]
+# W64-FAKE16: v_mad_u32_u16_e64_dpp v5, v1, v2, exec_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x59,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05]
 
 0x05,0x00,0x59,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05
-# GFX12: v_mad_u32_u16_e64_dpp v5, v1, v2, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x59,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05]
+# W32-REAL16: v_mad_u32_u16_e64_dpp v5, v1.l, v2.l, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x59,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05]
+# W32-FAKE16: v_mad_u32_u16_e64_dpp v5, v1, v2, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x59,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05]
+# W64-REAL16: v_mad_u32_u16_e64_dpp v5, v1.l, v2.l, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x59,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05]
+# W64-FAKE16: v_mad_u32_u16_e64_dpp v5, v1, v2, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x59,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05]
 
 0x05,0x00,0x59,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05
-# GFX12: v_mad_u32_u16_e64_dpp v5, v1, v2, -1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x59,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05]
+# W32-REAL16: v_mad_u32_u16_e64_dpp v5, v1.l, v2.l, -1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x59,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05]
+# W32-FAKE16: v_mad_u32_u16_e64_dpp v5, v1, v2, -1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x59,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05]
+# W64-REAL16: v_mad_u32_u16_e64_dpp v5, v1.l, v2.l, -1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x59,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05]
+# W64-FAKE16: v_mad_u32_u16_e64_dpp v5, v1, v2, -1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x59,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05]
 
 0x05,0x08,0x59,0xd6,0xe9,0x04,0xc2,0x03,0x01,0x77,0x39,0x05
-# GFX12: v_mad_u32_u16_e64_dpp v5, v1, v2, 0.5 op_sel:[1,0,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x08,0x59,0xd6,0xe9,0x04,0xc2,0x03,0x01,0x77,0x39,0x05]
+# W32-REAL16: v_mad_u32_u16_e64_dpp v5, v1.h, v2.l, 0.5 op_sel:[1,0,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x08,0x59,0xd6,0xe9,0x04,0xc2,0x03,0x01,0x77,0x39,0x05]
+# W32-FAKE16: v_mad_u32_u16_e64_dpp v5, v1, v2, 0.5 op_sel:[1,0,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x08,0x59,0xd6,0xe9,0x04,0xc2,0x03,0x01,0x77,0x39,0x05]
+# W64-REAL16: v_mad_u32_u16_e64_dpp v5, v1.h, v2.l, 0.5 op_sel:[1,0,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x08,0x59,0xd6,0xe9,0x04,0xc2,0x03,0x01,0x77,0x39,0x05]
+# W64-FAKE16: v_mad_u32_u16_e64_dpp v5, v1, v2, 0.5 op_sel:[1,0,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x08,0x59,0xd6,0xe9,0x04,0xc2,0x03,0x01,0x77,0x39,0x05]
 
 0xff,0x90,0x59,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00
-# GFX12: v_mad_u32_u16_e64_dpp v255, v255, v255, src_scc op_sel:[0,1,0,0] clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x90,0x59,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00]
+# W32-REAL16: v_mad_u32_u16_e64_dpp v255, v255.l, v255.h, src_scc op_sel:[0,1,0,0] clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x90,0x59,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00]
+# W32-FAKE16: v_mad_u32_u16_e64_dpp v255, v255, v255, src_scc op_sel:[0,1,0,0] clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x90,0x59,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00]
+# W64-REAL16: v_mad_u32_u16_e64_dpp v255, v255.l, v255.h, src_scc op_sel:[0,1,0,0] clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x90,0x59,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00]
+# W64-FAKE16: v_mad_u32_u16_e64_dpp v255, v255, v255, src_scc op_sel:[0,1,0,0] clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x90,0x59,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00]
 
 0x05,0x00,0x2c,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05
 # W32-REAL16: v_max3_num_f16_e64_dpp v5.l, v1.l, v2.l, v3.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x2c,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
@@ -5013,28 +5118,64 @@
 # GFX12: v_minimum_f32_e64_dpp v5, |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x65,0xd7,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
 
 0xff,0x03,0x68,0xd7,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00
-# GFX12: v_maximum_f16_e64_dpp v255, -|v255|, -|v255| dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x03,0x68,0xd7,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+# W32-REAL16: v_maximum_f16_e64_dpp v255.l, -|v255.l|, -|v255.l| dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x03,0x68,0xd7,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+# W32-FAKE16: v_maximum_f16_e64_dpp v255, -|v255|, -|v255| dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x03,0x68,0xd7,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+# W64-REAL16: v_maximum_f16_e64_dpp v255.l, -|v255.l|, -|v255.l| dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x03,0x68,0xd7,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+# W64-FAKE16: v_maximum_f16_e64_dpp v255, -|v255|, -|v255| dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x03,0x68,0xd7,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
 
 0x05,0x02,0x68,0xd7,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05
-# GFX12: v_maximum_f16_e64_dpp v5, -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x05,0x02,0x68,0xd7,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+# W32-REAL16: v_maximum_f16_e64_dpp v5.l, -v1.l, |v2.l| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x05,0x02,0x68,0xd7,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+# W32-FAKE16: v_maximum_f16_e64_dpp v5, -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x05,0x02,0x68,0xd7,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+# W64-REAL16: v_maximum_f16_e64_dpp v5.l, -v1.l, |v2.l| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x05,0x02,0x68,0xd7,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+# W64-FAKE16: v_maximum_f16_e64_dpp v5, -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x05,0x02,0x68,0xd7,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
 
 0x05,0x00,0x68,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
-# GFX12: v_maximum_f16_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x68,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+# W32-REAL16: v_maximum_f16_e64_dpp v5.l, v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x68,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+# W32-FAKE16: v_maximum_f16_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x68,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+# W64-REAL16: v_maximum_f16_e64_dpp v5.l, v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x68,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+# W64-FAKE16: v_maximum_f16_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x68,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 
 0x05,0x01,0x68,0xd7,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05
-# GFX12: v_maximum_f16_e64_dpp v5, |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x68,0xd7,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+# W32-REAL16: v_maximum_f16_e64_dpp v5.l, |v1.l|, -v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x68,0xd7,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+# W32-FAKE16: v_maximum_f16_e64_dpp v5, |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x68,0xd7,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+# W64-REAL16: v_maximum_f16_e64_dpp v5.l, |v1.l|, -v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x68,0xd7,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+# W64-FAKE16: v_maximum_f16_e64_dpp v5, |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x68,0xd7,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+
+0xff,0x5b,0x68,0xd7,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00
+# W32-REAL16: v_maximum_f16_e64_dpp v255.h, -|v255.h|, -|v255.h| op_sel:[1,1,1] dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x5b,0x68,0xd7,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+# W32-FAKE16: v_maximum_f16_e64_dpp v255, -|v255|, -|v255| op_sel:[1,1,1] dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x5b,0x68,0xd7,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+# W64-REAL16: v_maximum_f16_e64_dpp v255.h, -|v255.h|, -|v255.h| op_sel:[1,1,1] dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x5b,0x68,0xd7,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+# W64-FAKE16: v_maximum_f16_e64_dpp v255, -|v255|, -|v255| op_sel:[1,1,1] dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x5b,0x68,0xd7,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
 
 0xff,0x03,0x67,0xd7,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00
-# GFX12: v_minimum_f16_e64_dpp v255, -|v255|, -|v255| dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x03,0x67,0xd7,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+# W32-REAL16: v_minimum_f16_e64_dpp v255.l, -|v255.l|, -|v255.l| dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x03,0x67,0xd7,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+# W32-FAKE16: v_minimum_f16_e64_dpp v255, -|v255|, -|v255| dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x03,0x67,0xd7,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+# W64-REAL16: v_minimum_f16_e64_dpp v255.l, -|v255.l|, -|v255.l| dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x03,0x67,0xd7,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+# W64-FAKE16: v_minimum_f16_e64_dpp v255, -|v255|, -|v255| dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x03,0x67,0xd7,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
 
 0x05,0x02,0x67,0xd7,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05
-# GFX12: v_minimum_f16_e64_dpp v5, -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x05,0x02,0x67,0xd7,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+# W32-REAL16: v_minimum_f16_e64_dpp v5.l, -v1.l, |v2.l| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x05,0x02,0x67,0xd7,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+# W32-FAKE16: v_minimum_f16_e64_dpp v5, -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x05,0x02,0x67,0xd7,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+# W64-REAL16: v_minimum_f16_e64_dpp v5.l, -v1.l, |v2.l| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x05,0x02,0x67,0xd7,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+# W64-FAKE16: v_minimum_f16_e64_dpp v5, -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x05,0x02,0x67,0xd7,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
 
 0x05,0x00,0x67,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
-# GFX12: v_minimum_f16_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x67,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+# W32-REAL16: v_minimum_f16_e64_dpp v5.l, v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x67,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+# W32-FAKE16: v_minimum_f16_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x67,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+# W64-REAL16: v_minimum_f16_e64_dpp v5.l, v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x67,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+# W64-FAKE16: v_minimum_f16_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x67,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 
 0x05,0x01,0x67,0xd7,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05
-# GFX12: v_minimum_f16_e64_dpp v5, |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x67,0xd7,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+# W32-REAL16: v_minimum_f16_e64_dpp v5.l, |v1.l|, -v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x67,0xd7,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+# W32-FAKE16: v_minimum_f16_e64_dpp v5, |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x67,0xd7,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+# W64-REAL16: v_minimum_f16_e64_dpp v5.l, |v1.l|, -v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x67,0xd7,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+# W64-FAKE16: v_minimum_f16_e64_dpp v5, |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x67,0xd7,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+
+0xff,0x5b,0x67,0xd7,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00
+# W32-REAL16: v_minimum_f16_e64_dpp v255.h, -|v255.h|, -|v255.h| op_sel:[1,1,1] dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x5b,0x67,0xd7,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+# W32-FAKE16: v_minimum_f16_e64_dpp v255, -|v255|, -|v255| op_sel:[1,1,1] dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x5b,0x67,0xd7,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+# W64-REAL16: v_minimum_f16_e64_dpp v255.h, -|v255.h|, -|v255.h| op_sel:[1,1,1] dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x5b,0x67,0xd7,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+# W64-FAKE16: v_minimum_f16_e64_dpp v255, -|v255|, -|v255| op_sel:[1,1,1] dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x5b,0x67,0xd7,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
 
 0xff,0x87,0x2e,0xd6,0xe9,0xfe,0xf7,0xfb,0xff,0x00,0x00,0x00
 # GFX12: v_maximum3_f32_e64_dpp v255, -|v255|, -|v255|, -|src_scc| clamp div:2 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x87,0x2e,0xd6,0xe9,0xfe,0xf7,0xfb,0xff,0x00,0x00,0x00]
@@ -5115,82 +5256,172 @@
 # GFX12: v_minimum3_f32_e64_dpp v5, |v1|, v2, -ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x2d,0xd6,0xe9,0x04,0xee,0x81,0x01,0x77,0x39,0x05]
 
 0xff,0x87,0x30,0xd6,0xe9,0xfe,0xf7,0xe3,0xff,0x00,0x00,0x00
-# GFX12: v_maximum3_f16_e64_dpp v255, -|v255|, -|v255|, -|src_scc| clamp dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x87,0x30,0xd6,0xe9,0xfe,0xf7,0xe3,0xff,0x00,0x00,0x00]
+# W32-REAL16: v_maximum3_f16_e64_dpp v255.l, -|v255.l|, -|v255.l|, -|src_scc| clamp dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x87,0x30,0xd6,0xe9,0xfe,0xf7,0xe3,0xff,0x00,0x00,0x00]
+# W32-FAKE16: v_maximum3_f16_e64_dpp v255, -|v255|, -|v255|, -|src_scc| clamp dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x87,0x30,0xd6,0xe9,0xfe,0xf7,0xe3,0xff,0x00,0x00,0x00]
+# W64-REAL16: v_maximum3_f16_e64_dpp v255.l, -|v255.l|, -|v255.l|, -|src_scc| clamp dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x87,0x30,0xd6,0xe9,0xfe,0xf7,0xe3,0xff,0x00,0x00,0x00]
+# W64-FAKE16: v_maximum3_f16_e64_dpp v255, -|v255|, -|v255|, -|src_scc| clamp dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x87,0x30,0xd6,0xe9,0xfe,0xf7,0xe3,0xff,0x00,0x00,0x00]
 
 0x05,0x04,0x30,0xd6,0xe9,0x04,0xfa,0x21,0x01,0x77,0x39,0x05
-# GFX12: v_maximum3_f16_e64_dpp v5, -v1, v2, |exec_lo| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x04,0x30,0xd6,0xe9,0x04,0xfa,0x21,0x01,0x77,0x39,0x05]
+# W32-REAL16: v_maximum3_f16_e64_dpp v5.l, -v1.l, v2.l, |exec_lo| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x04,0x30,0xd6,0xe9,0x04,0xfa,0x21,0x01,0x77,0x39,0x05]
+# W32-FAKE16: v_maximum3_f16_e64_dpp v5, -v1, v2, |exec_lo| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x04,0x30,0xd6,0xe9,0x04,0xfa,0x21,0x01,0x77,0x39,0x05]
+# W64-REAL16: v_maximum3_f16_e64_dpp v5.l, -v1.l, v2.l, |exec_lo| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x04,0x30,0xd6,0xe9,0x04,0xfa,0x21,0x01,0x77,0x39,0x05]
+# W64-FAKE16: v_maximum3_f16_e64_dpp v5, -v1, v2, |exec_lo| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x04,0x30,0xd6,0xe9,0x04,0xfa,0x21,0x01,0x77,0x39,0x05]
 
 0x05,0x03,0x30,0xd6,0xe9,0x04,0xf2,0x61,0x01,0x77,0x39,0x05
-# GFX12: v_maximum3_f16_e64_dpp v5, -|v1|, -|v2|, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x03,0x30,0xd6,0xe9,0x04,0xf2,0x61,0x01,0x77,0x39,0x05]
+# W32-REAL16: v_maximum3_f16_e64_dpp v5.l, -|v1.l|, -|v2.l|, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x03,0x30,0xd6,0xe9,0x04,0xf2,0x61,0x01,0x77,0x39,0x05]
+# W32-FAKE16: v_maximum3_f16_e64_dpp v5, -|v1|, -|v2|, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x03,0x30,0xd6,0xe9,0x04,0xf2,0x61,0x01,0x77,0x39,0x05]
+# W64-REAL16: v_maximum3_f16_e64_dpp v5.l, -|v1.l|, -|v2.l|, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x03,0x30,0xd6,0xe9,0x04,0xf2,0x61,0x01,0x77,0x39,0x05]
+# W64-FAKE16: v_maximum3_f16_e64_dpp v5, -|v1|, -|v2|, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x03,0x30,0xd6,0xe9,0x04,0xf2,0x61,0x01,0x77,0x39,0x05]
 
 0x05,0x05,0x30,0xd6,0xe9,0x04,0x06,0xa3,0x01,0x77,0x39,0x05
-# GFX12: v_maximum3_f16_e64_dpp v5, -|v1|, v2, -|-1| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x05,0x30,0xd6,0xe9,0x04,0x06,0xa3,0x01,0x77,0x39,0x05]
+# W32-REAL16: v_maximum3_f16_e64_dpp v5.l, -|v1.l|, v2.l, -|-1| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x05,0x30,0xd6,0xe9,0x04,0x06,0xa3,0x01,0x77,0x39,0x05]
+# W32-FAKE16: v_maximum3_f16_e64_dpp v5, -|v1|, v2, -|-1| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x05,0x30,0xd6,0xe9,0x04,0x06,0xa3,0x01,0x77,0x39,0x05]
+# W64-REAL16: v_maximum3_f16_e64_dpp v5.l, -|v1.l|, v2.l, -|-1| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x05,0x30,0xd6,0xe9,0x04,0x06,0xa3,0x01,0x77,0x39,0x05]
+# W64-FAKE16: v_maximum3_f16_e64_dpp v5, -|v1|, v2, -|-1| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x05,0x30,0xd6,0xe9,0x04,0x06,0xa3,0x01,0x77,0x39,0x05]
 
 0x05,0x06,0x30,0xd6,0xea,0x04,0xc2,0xc3,0x01,0x77,0x39,0x05
-# GFX12: v_maximum3_f16_e64_dpp v5, v1, -|v2|, -|0.5| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x05,0x06,0x30,0xd6,0xea,0x04,0xc2,0xc3,0x01,0x77,0x39,0x05]
+# W32-REAL16: v_maximum3_f16_e64_dpp v5.l, v1.l, -|v2.l|, -|0.5| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x05,0x06,0x30,0xd6,0xea,0x04,0xc2,0xc3,0x01,0x77,0x39,0x05]
+# W32-FAKE16: v_maximum3_f16_e64_dpp v5, v1, -|v2|, -|0.5| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x05,0x06,0x30,0xd6,0xea,0x04,0xc2,0xc3,0x01,0x77,0x39,0x05]
+# W64-REAL16: v_maximum3_f16_e64_dpp v5.l, v1.l, -|v2.l|, -|0.5| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x05,0x06,0x30,0xd6,0xea,0x04,0xc2,0xc3,0x01,0x77,0x39,0x05]
+# W64-FAKE16: v_maximum3_f16_e64_dpp v5, v1, -|v2|, -|0.5| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x05,0x06,0x30,0xd6,0xea,0x04,0xc2,0xc3,0x01,0x77,0x39,0x05]
 
 0x05,0x02,0x30,0xd6,0xe9,0x04,0xfe,0x41,0x01,0x77,0x39,0x05
-# GFX12: v_maximum3_f16_e64_dpp v5, v1, -|v2|, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x02,0x30,0xd6,0xe9,0x04,0xfe,0x41,0x01,0x77,0x39,0x05]
+# W32-REAL16: v_maximum3_f16_e64_dpp v5.l, v1.l, -|v2.l|, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x02,0x30,0xd6,0xe9,0x04,0xfe,0x41,0x01,0x77,0x39,0x05]
+# W32-FAKE16: v_maximum3_f16_e64_dpp v5, v1, -|v2|, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x02,0x30,0xd6,0xe9,0x04,0xfe,0x41,0x01,0x77,0x39,0x05]
+# W64-REAL16: v_maximum3_f16_e64_dpp v5.l, v1.l, -|v2.l|, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x02,0x30,0xd6,0xe9,0x04,0xfe,0x41,0x01,0x77,0x39,0x05]
+# W64-FAKE16: v_maximum3_f16_e64_dpp v5, v1, -|v2|, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x02,0x30,0xd6,0xe9,0x04,0xfe,0x41,0x01,0x77,0x39,0x05]
 
 0x05,0x00,0x30,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05
-# GFX12: v_maximum3_f16_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x30,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
+# W32-REAL16: v_maximum3_f16_e64_dpp v5.l, v1.l, v2.l, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x30,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
+# W32-FAKE16: v_maximum3_f16_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x30,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
+# W64-REAL16: v_maximum3_f16_e64_dpp v5.l, v1.l, v2.l, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x30,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
+# W64-FAKE16: v_maximum3_f16_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x30,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 
 0x05,0x00,0x30,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05
-# GFX12: v_maximum3_f16_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x30,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
+# W32-REAL16: v_maximum3_f16_e64_dpp v5.l, v1.l, v2.l, v255.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x30,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
+# W32-FAKE16: v_maximum3_f16_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x30,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
+# W64-REAL16: v_maximum3_f16_e64_dpp v5.l, v1.l, v2.l, v255.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x30,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
+# W64-FAKE16: v_maximum3_f16_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x30,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 
 0x05,0x00,0x30,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05
-# GFX12: v_maximum3_f16_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x30,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
+# W32-REAL16: v_maximum3_f16_e64_dpp v5.l, v1.l, v2.l, v3.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x30,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
+# W32-FAKE16: v_maximum3_f16_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x30,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
+# W64-REAL16: v_maximum3_f16_e64_dpp v5.l, v1.l, v2.l, v3.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x30,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
+# W64-FAKE16: v_maximum3_f16_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x30,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 
 0x05,0x00,0x30,0xd6,0xe9,0x06,0x0c,0x04,0x01,0x77,0x39,0x05
-# GFX12: v_maximum3_f16_e64_dpp v5, v1, s3, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x30,0xd6,0xe9,0x06,0x0c,0x04,0x01,0x77,0x39,0x05]
+# W32-REAL16: v_maximum3_f16_e64_dpp v5.l, v1.l, s3, v3.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x30,0xd6,0xe9,0x06,0x0c,0x04,0x01,0x77,0x39,0x05]
+# W32-FAKE16: v_maximum3_f16_e64_dpp v5, v1, s3, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x30,0xd6,0xe9,0x06,0x0c,0x04,0x01,0x77,0x39,0x05]
+# W64-REAL16: v_maximum3_f16_e64_dpp v5.l, v1.l, s3, v3.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x30,0xd6,0xe9,0x06,0x0c,0x04,0x01,0x77,0x39,0x05]
+# W64-FAKE16: v_maximum3_f16_e64_dpp v5, v1, s3, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x30,0xd6,0xe9,0x06,0x0c,0x04,0x01,0x77,0x39,0x05]
 
 0x05,0x00,0x30,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05
-# GFX12: v_maximum3_f16_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x30,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
+# W32-REAL16: v_maximum3_f16_e64_dpp v5.l, v1.l, v2.l, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x30,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
+# W32-FAKE16: v_maximum3_f16_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x30,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
+# W64-REAL16: v_maximum3_f16_e64_dpp v5.l, v1.l, v2.l, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x30,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
+# W64-FAKE16: v_maximum3_f16_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x30,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 
 0x05,0x00,0x30,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05
-# GFX12: v_maximum3_f16_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x30,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
+# W32-REAL16: v_maximum3_f16_e64_dpp v5.l, v1.l, v2.l, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x30,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
+# W32-FAKE16: v_maximum3_f16_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x30,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
+# W64-REAL16: v_maximum3_f16_e64_dpp v5.l, v1.l, v2.l, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x30,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
+# W64-FAKE16: v_maximum3_f16_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x30,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 
 0x05,0x01,0x30,0xd6,0xe9,0x04,0xee,0x81,0x01,0x77,0x39,0x05
-# GFX12: v_maximum3_f16_e64_dpp v5, |v1|, v2, -ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x30,0xd6,0xe9,0x04,0xee,0x81,0x01,0x77,0x39,0x05]
+# W32-REAL16: v_maximum3_f16_e64_dpp v5.l, |v1.l|, v2.l, -ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x30,0xd6,0xe9,0x04,0xee,0x81,0x01,0x77,0x39,0x05]
+# W32-FAKE16: v_maximum3_f16_e64_dpp v5, |v1|, v2, -ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x30,0xd6,0xe9,0x04,0xee,0x81,0x01,0x77,0x39,0x05]
+# W64-REAL16: v_maximum3_f16_e64_dpp v5.l, |v1.l|, v2.l, -ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x30,0xd6,0xe9,0x04,0xee,0x81,0x01,0x77,0x39,0x05]
+# W64-FAKE16: v_maximum3_f16_e64_dpp v5, |v1|, v2, -ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x30,0xd6,0xe9,0x04,0xee,0x81,0x01,0x77,0x39,0x05]
+
+0xff,0xc7,0x30,0xd6,0xe9,0xfe,0xf7,0xe3,0xff,0x00,0x00,0x00
+# W32-REAL16: v_maximum3_f16_e64_dpp v255.h, -|v255.l|, -|v255.l|, -|src_scc| op_sel:[0,0,0,1] clamp dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0xc7,0x30,0xd6,0xe9,0xfe,0xf7,0xe3,0xff,0x00,0x00,0x00]
+# W32-FAKE16: v_maximum3_f16_e64_dpp v255, -|v255|, -|v255|, -|src_scc| op_sel:[0,0,0,1] clamp dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0xc7,0x30,0xd6,0xe9,0xfe,0xf7,0xe3,0xff,0x00,0x00,0x00]
+# W64-REAL16: v_maximum3_f16_e64_dpp v255.h, -|v255.l|, -|v255.l|, -|src_scc| op_sel:[0,0,0,1] clamp dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0xc7,0x30,0xd6,0xe9,0xfe,0xf7,0xe3,0xff,0x00,0x00,0x00]
+# W64-FAKE16: v_maximum3_f16_e64_dpp v255, -|v255|, -|v255|, -|src_scc| op_sel:[0,0,0,1] clamp dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0xc7,0x30,0xd6,0xe9,0xfe,0xf7,0xe3,0xff,0x00,0x00,0x00]
 
 0xff,0x87,0x2f,0xd6,0xe9,0xfe,0xf7,0xe3,0xff,0x00,0x00,0x00
-# GFX12: v_minimum3_f16_e64_dpp v255, -|v255|, -|v255|, -|src_scc| clamp dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x87,0x2f,0xd6,0xe9,0xfe,0xf7,0xe3,0xff,0x00,0x00,0x00]
+# W32-REAL16: v_minimum3_f16_e64_dpp v255.l, -|v255.l|, -|v255.l|, -|src_scc| clamp dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x87,0x2f,0xd6,0xe9,0xfe,0xf7,0xe3,0xff,0x00,0x00,0x00]
+# W32-FAKE16: v_minimum3_f16_e64_dpp v255, -|v255|, -|v255|, -|src_scc| clamp dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x87,0x2f,0xd6,0xe9,0xfe,0xf7,0xe3,0xff,0x00,0x00,0x00]
+# W64-REAL16: v_minimum3_f16_e64_dpp v255.l, -|v255.l|, -|v255.l|, -|src_scc| clamp dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x87,0x2f,0xd6,0xe9,0xfe,0xf7,0xe3,0xff,0x00,0x00,0x00]
+# W64-FAKE16: v_minimum3_f16_e64_dpp v255, -|v255|, -|v255|, -|src_scc| clamp dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x87,0x2f,0xd6,0xe9,0xfe,0xf7,0xe3,0xff,0x00,0x00,0x00]
 
 0x05,0x04,0x2f,0xd6,0xe9,0x04,0xfa,0x21,0x01,0x77,0x39,0x05
-# GFX12: v_minimum3_f16_e64_dpp v5, -v1, v2, |exec_lo| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x04,0x2f,0xd6,0xe9,0x04,0xfa,0x21,0x01,0x77,0x39,0x05]
+# W32-REAL16: v_minimum3_f16_e64_dpp v5.l, -v1.l, v2.l, |exec_lo| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x04,0x2f,0xd6,0xe9,0x04,0xfa,0x21,0x01,0x77,0x39,0x05]
+# W32-FAKE16: v_minimum3_f16_e64_dpp v5, -v1, v2, |exec_lo| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x04,0x2f,0xd6,0xe9,0x04,0xfa,0x21,0x01,0x77,0x39,0x05]
+# W64-REAL16: v_minimum3_f16_e64_dpp v5.l, -v1.l, v2.l, |exec_lo| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x04,0x2f,0xd6,0xe9,0x04,0xfa,0x21,0x01,0x77,0x39,0x05]
+# W64-FAKE16: v_minimum3_f16_e64_dpp v5, -v1, v2, |exec_lo| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x04,0x2f,0xd6,0xe9,0x04,0xfa,0x21,0x01,0x77,0x39,0x05]
 
 0x05,0x03,0x2f,0xd6,0xe9,0x04,0xf2,0x61,0x01,0x77,0x39,0x05
-# GFX12: v_minimum3_f16_e64_dpp v5, -|v1|, -|v2|, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x03,0x2f,0xd6,0xe9,0x04,0xf2,0x61,0x01,0x77,0x39,0x05]
+# W32-REAL16: v_minimum3_f16_e64_dpp v5.l, -|v1.l|, -|v2.l|, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x03,0x2f,0xd6,0xe9,0x04,0xf2,0x61,0x01,0x77,0x39,0x05]
+# W32-FAKE16: v_minimum3_f16_e64_dpp v5, -|v1|, -|v2|, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x03,0x2f,0xd6,0xe9,0x04,0xf2,0x61,0x01,0x77,0x39,0x05]
+# W64-REAL16: v_minimum3_f16_e64_dpp v5.l, -|v1.l|, -|v2.l|, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x03,0x2f,0xd6,0xe9,0x04,0xf2,0x61,0x01,0x77,0x39,0x05]
+# W64-FAKE16: v_minimum3_f16_e64_dpp v5, -|v1|, -|v2|, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x03,0x2f,0xd6,0xe9,0x04,0xf2,0x61,0x01,0x77,0x39,0x05]
 
 0x05,0x05,0x2f,0xd6,0xe9,0x04,0x06,0xa3,0x01,0x77,0x39,0x05
-# GFX12: v_minimum3_f16_e64_dpp v5, -|v1|, v2, -|-1| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x05,0x2f,0xd6,0xe9,0x04,0x06,0xa3,0x01,0x77,0x39,0x05]
+# W32-REAL16: v_minimum3_f16_e64_dpp v5.l, -|v1.l|, v2.l, -|-1| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x05,0x2f,0xd6,0xe9,0x04,0x06,0xa3,0x01,0x77,0x39,0x05]
+# W32-FAKE16: v_minimum3_f16_e64_dpp v5, -|v1|, v2, -|-1| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x05,0x2f,0xd6,0xe9,0x04,0x06,0xa3,0x01,0x77,0x39,0x05]
+# W64-REAL16: v_minimum3_f16_e64_dpp v5.l, -|v1.l|, v2.l, -|-1| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x05,0x2f,0xd6,0xe9,0x04,0x06,0xa3,0x01,0x77,0x39,0x05]
+# W64-FAKE16: v_minimum3_f16_e64_dpp v5, -|v1|, v2, -|-1| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x05,0x2f,0xd6,0xe9,0x04,0x06,0xa3,0x01,0x77,0x39,0x05]
 
 0x05,0x06,0x2f,0xd6,0xea,0x04,0xc2,0xc3,0x01,0x77,0x39,0x05
-# GFX12: v_minimum3_f16_e64_dpp v5, v1, -|v2|, -|0.5| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x05,0x06,0x2f,0xd6,0xea,0x04,0xc2,0xc3,0x01,0x77,0x39,0x05]
+# W32-REAL16: v_minimum3_f16_e64_dpp v5.l, v1.l, -|v2.l|, -|0.5| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x05,0x06,0x2f,0xd6,0xea,0x04,0xc2,0xc3,0x01,0x77,0x39,0x05]
+# W32-FAKE16: v_minimum3_f16_e64_dpp v5, v1, -|v2|, -|0.5| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x05,0x06,0x2f,0xd6,0xea,0x04,0xc2,0xc3,0x01,0x77,0x39,0x05]
+# W64-REAL16: v_minimum3_f16_e64_dpp v5.l, v1.l, -|v2.l|, -|0.5| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x05,0x06,0x2f,0xd6,0xea,0x04,0xc2,0xc3,0x01,0x77,0x39,0x05]
+# W64-FAKE16: v_minimum3_f16_e64_dpp v5, v1, -|v2|, -|0.5| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x05,0x06,0x2f,0xd6,0xea,0x04,0xc2,0xc3,0x01,0x77,0x39,0x05]
 
 0x05,0x02,0x2f,0xd6,0xe9,0x04,0xfe,0x41,0x01,0x77,0x39,0x05
-# GFX12: v_minimum3_f16_e64_dpp v5, v1, -|v2|, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x02,0x2f,0xd6,0xe9,0x04,0xfe,0x41,0x01,0x77,0x39,0x05]
+# W32-REAL16: v_minimum3_f16_e64_dpp v5.l, v1.l, -|v2.l|, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x02,0x2f,0xd6,0xe9,0x04,0xfe,0x41,0x01,0x77,0x39,0x05]
+# W32-FAKE16: v_minimum3_f16_e64_dpp v5, v1, -|v2|, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x02,0x2f,0xd6,0xe9,0x04,0xfe,0x41,0x01,0x77,0x39,0x05]
+# W64-REAL16: v_minimum3_f16_e64_dpp v5.l, v1.l, -|v2.l|, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x02,0x2f,0xd6,0xe9,0x04,0xfe,0x41,0x01,0x77,0x39,0x05]
+# W64-FAKE16: v_minimum3_f16_e64_dpp v5, v1, -|v2|, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x02,0x2f,0xd6,0xe9,0x04,0xfe,0x41,0x01,0x77,0x39,0x05]
 
 0x05,0x00,0x2f,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05
-# GFX12: v_minimum3_f16_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x2f,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
+# W32-REAL16: v_minimum3_f16_e64_dpp v5.l, v1.l, v2.l, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x2f,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
+# W32-FAKE16: v_minimum3_f16_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x2f,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
+# W64-REAL16: v_minimum3_f16_e64_dpp v5.l, v1.l, v2.l, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x2f,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
+# W64-FAKE16: v_minimum3_f16_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x2f,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 
 0x05,0x00,0x2f,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05
-# GFX12: v_minimum3_f16_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x2f,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
+# W32-REAL16: v_minimum3_f16_e64_dpp v5.l, v1.l, v2.l, v255.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x2f,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
+# W32-FAKE16: v_minimum3_f16_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x2f,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
+# W64-REAL16: v_minimum3_f16_e64_dpp v5.l, v1.l, v2.l, v255.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x2f,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
+# W64-FAKE16: v_minimum3_f16_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x2f,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 
 0x05,0x00,0x2f,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05
-# GFX12: v_minimum3_f16_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x2f,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
+# W32-REAL16: v_minimum3_f16_e64_dpp v5.l, v1.l, v2.l, v3.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x2f,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
+# W32-FAKE16: v_minimum3_f16_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x2f,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
+# W64-REAL16: v_minimum3_f16_e64_dpp v5.l, v1.l, v2.l, v3.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x2f,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
+# W64-FAKE16: v_minimum3_f16_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x2f,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 
 0x05,0x00,0x2f,0xd6,0xe9,0x06,0x0c,0x04,0x01,0x77,0x39,0x05
-# GFX12: v_minimum3_f16_e64_dpp v5, v1, s3, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x2f,0xd6,0xe9,0x06,0x0c,0x04,0x01,0x77,0x39,0x05]
+# W32-REAL16: v_minimum3_f16_e64_dpp v5.l, v1.l, s3, v3.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x2f,0xd6,0xe9,0x06,0x0c,0x04,0x01,0x77,0x39,0x05]
+# W32-FAKE16: v_minimum3_f16_e64_dpp v5, v1, s3, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x2f,0xd6,0xe9,0x06,0x0c,0x04,0x01,0x77,0x39,0x05]
+# W64-REAL16: v_minimum3_f16_e64_dpp v5.l, v1.l, s3, v3.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x2f,0xd6,0xe9,0x06,0x0c,0x04,0x01,0x77,0x39,0x05]
+# W64-FAKE16: v_minimum3_f16_e64_dpp v5, v1, s3, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x2f,0xd6,0xe9,0x06,0x0c,0x04,0x01,0x77,0x39,0x05]
 
 0x05,0x00,0x2f,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05
-# GFX12: v_minimum3_f16_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x2f,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
+# W32-REAL16: v_minimum3_f16_e64_dpp v5.l, v1.l, v2.l, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x2f,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
+# W32-FAKE16: v_minimum3_f16_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x2f,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
+# W64-REAL16: v_minimum3_f16_e64_dpp v5.l, v1.l, v2.l, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x2f,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
+# W64-FAKE16: v_minimum3_f16_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x2f,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 
 0x05,0x00,0x2f,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05
-# GFX12: v_minimum3_f16_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x2f,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
+# W32-REAL16: v_minimum3_f16_e64_dpp v5.l, v1.l, v2.l, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x2f,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
+# W32-FAKE16: v_minimum3_f16_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x2f,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
+# W64-REAL16: v_minimum3_f16_e64_dpp v5.l, v1.l, v2.l, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x2f,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
+# W64-FAKE16: v_minimum3_f16_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x2f,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 
 0x05,0x01,0x2f,0xd6,0xe9,0x04,0xee,0x81,0x01,0x77,0x39,0x05
-# GFX12: v_minimum3_f16_e64_dpp v5, |v1|, v2, -ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x2f,0xd6,0xe9,0x04,0xee,0x81,0x01,0x77,0x39,0x05]
+# W32-REAL16: v_minimum3_f16_e64_dpp v5.l, |v1.l|, v2.l, -ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x2f,0xd6,0xe9,0x04,0xee,0x81,0x01,0x77,0x39,0x05]
+# W32-FAKE16: v_minimum3_f16_e64_dpp v5, |v1|, v2, -ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x2f,0xd6,0xe9,0x04,0xee,0x81,0x01,0x77,0x39,0x05]
+# W64-REAL16: v_minimum3_f16_e64_dpp v5.l, |v1.l|, v2.l, -ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x2f,0xd6,0xe9,0x04,0xee,0x81,0x01,0x77,0x39,0x05]
+# W64-FAKE16: v_minimum3_f16_e64_dpp v5, |v1|, v2, -ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x2f,0xd6,0xe9,0x04,0xee,0x81,0x01,0x77,0x39,0x05]
+
+0xff,0xc7,0x2f,0xd6,0xe9,0xfe,0xf7,0xe3,0xff,0x00,0x00,0x00
+# W32-REAL16: v_minimum3_f16_e64_dpp v255.h, -|v255.l|, -|v255.l|, -|src_scc| op_sel:[0,0,0,1] clamp dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0xc7,0x2f,0xd6,0xe9,0xfe,0xf7,0xe3,0xff,0x00,0x00,0x00]
+# W32-FAKE16: v_minimum3_f16_e64_dpp v255, -|v255|, -|v255|, -|src_scc| op_sel:[0,0,0,1] clamp dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0xc7,0x2f,0xd6,0xe9,0xfe,0xf7,0xe3,0xff,0x00,0x00,0x00]
+# W64-REAL16: v_minimum3_f16_e64_dpp v255.h, -|v255.l|, -|v255.l|, -|src_scc| op_sel:[0,0,0,1] clamp dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0xc7,0x2f,0xd6,0xe9,0xfe,0xf7,0xe3,0xff,0x00,0x00,0x00]
+# W64-FAKE16: v_minimum3_f16_e64_dpp v255, -|v255|, -|v255|, -|src_scc| op_sel:[0,0,0,1] clamp dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0xc7,0x2f,0xd6,0xe9,0xfe,0xf7,0xe3,0xff,0x00,0x00,0x00]
 
 0xff,0x87,0x6d,0xd6,0xe9,0xfe,0xf7,0xfb,0xff,0x00,0x00,0x00
 # GFX12: v_maximumminimum_f32_e64_dpp v255, -|v255|, -|v255|, -|src_scc| clamp div:2 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x87,0x6d,0xd6,0xe9,0xfe,0xf7,0xfb,0xff,0x00,0x00,0x00]
@@ -5271,79 +5502,169 @@
 # GFX12: v_minimummaximum_f32_e64_dpp v5, |v1|, v2, -ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x6c,0xd6,0xe9,0x04,0xee,0x81,0x01,0x77,0x39,0x05]
 
 0xff,0x87,0x6f,0xd6,0xe9,0xfe,0xf7,0xe3,0xff,0x00,0x00,0x00
-# GFX12: v_maximumminimum_f16_e64_dpp v255, -|v255|, -|v255|, -|src_scc| clamp dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x87,0x6f,0xd6,0xe9,0xfe,0xf7,0xe3,0xff,0x00,0x00,0x00]
+# W32-REAL16: v_maximumminimum_f16_e64_dpp v255.l, -|v255.l|, -|v255.l|, -|src_scc| clamp dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x87,0x6f,0xd6,0xe9,0xfe,0xf7,0xe3,0xff,0x00,0x00,0x00]
+# W32-FAKE16: v_maximumminimum_f16_e64_dpp v255, -|v255|, -|v255|, -|src_scc| clamp dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x87,0x6f,0xd6,0xe9,0xfe,0xf7,0xe3,0xff,0x00,0x00,0x00]
+# W64-REAL16: v_maximumminimum_f16_e64_dpp v255.l, -|v255.l|, -|v255.l|, -|src_scc| clamp dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x87,0x6f,0xd6,0xe9,0xfe,0xf7,0xe3,0xff,0x00,0x00,0x00]
+# W64-FAKE16: v_maximumminimum_f16_e64_dpp v255, -|v255|, -|v255|, -|src_scc| clamp dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x87,0x6f,0xd6,0xe9,0xfe,0xf7,0xe3,0xff,0x00,0x00,0x00]
 
 0x05,0x04,0x6f,0xd6,0xe9,0x04,0xfa,0x21,0x01,0x77,0x39,0x05
-# GFX12: v_maximumminimum_f16_e64_dpp v5, -v1, v2, |exec_lo| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x04,0x6f,0xd6,0xe9,0x04,0xfa,0x21,0x01,0x77,0x39,0x05]
+# W32-REAL16: v_maximumminimum_f16_e64_dpp v5.l, -v1.l, v2.l, |exec_lo| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x04,0x6f,0xd6,0xe9,0x04,0xfa,0x21,0x01,0x77,0x39,0x05]
+# W32-FAKE16: v_maximumminimum_f16_e64_dpp v5, -v1, v2, |exec_lo| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x04,0x6f,0xd6,0xe9,0x04,0xfa,0x21,0x01,0x77,0x39,0x05]
+# W64-REAL16: v_maximumminimum_f16_e64_dpp v5.l, -v1.l, v2.l, |exec_lo| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x04,0x6f,0xd6,0xe9,0x04,0xfa,0x21,0x01,0x77,0x39,0x05]
+# W64-FAKE16: v_maximumminimum_f16_e64_dpp v5, -v1, v2, |exec_lo| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x04,0x6f,0xd6,0xe9,0x04,0xfa,0x21,0x01,0x77,0x39,0x05]
 
 0x05,0x03,0x6f,0xd6,0xe9,0x04,0xf2,0x61,0x01,0x77,0x39,0x05
-# GFX12: v_maximumminimum_f16_e64_dpp v5, -|v1|, -|v2|, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x03,0x6f,0xd6,0xe9,0x04,0xf2,0x61,0x01,0x77,0x39,0x05]
+# W32-REAL16: v_maximumminimum_f16_e64_dpp v5.l, -|v1.l|, -|v2.l|, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x03,0x6f,0xd6,0xe9,0x04,0xf2,0x61,0x01,0x77,0x39,0x05]
+# W32-FAKE16: v_maximumminimum_f16_e64_dpp v5, -|v1|, -|v2|, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x03,0x6f,0xd6,0xe9,0x04,0xf2,0x61,0x01,0x77,0x39,0x05]
+# W64-REAL16: v_maximumminimum_f16_e64_dpp v5.l, -|v1.l|, -|v2.l|, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x03,0x6f,0xd6,0xe9,0x04,0xf2,0x61,0x01,0x77,0x39,0x05]
+# W64-FAKE16: v_maximumminimum_f16_e64_dpp v5, -|v1|, -|v2|, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x03,0x6f,0xd6,0xe9,0x04,0xf2,0x61,0x01,0x77,0x39,0x05]
 
 0x05,0x05,0x6f,0xd6,0xe9,0x04,0x06,0xa3,0x01,0x77,0x39,0x05
-# GFX12: v_maximumminimum_f16_e64_dpp v5, -|v1|, v2, -|-1| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x05,0x6f,0xd6,0xe9,0x04,0x06,0xa3,0x01,0x77,0x39,0x05]
+# W32-REAL16: v_maximumminimum_f16_e64_dpp v5.l, -|v1.l|, v2.l, -|-1| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x05,0x6f,0xd6,0xe9,0x04,0x06,0xa3,0x01,0x77,0x39,0x05]
+# W32-FAKE16: v_maximumminimum_f16_e64_dpp v5, -|v1|, v2, -|-1| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x05,0x6f,0xd6,0xe9,0x04,0x06,0xa3,0x01,0x77,0x39,0x05]
+# W64-REAL16: v_maximumminimum_f16_e64_dpp v5.l, -|v1.l|, v2.l, -|-1| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x05,0x6f,0xd6,0xe9,0x04,0x06,0xa3,0x01,0x77,0x39,0x05]
+# W64-FAKE16: v_maximumminimum_f16_e64_dpp v5, -|v1|, v2, -|-1| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x05,0x6f,0xd6,0xe9,0x04,0x06,0xa3,0x01,0x77,0x39,0x05]
 
 0x05,0x06,0x6f,0xd6,0xea,0x04,0xc2,0xc3,0x01,0x77,0x39,0x05
-# GFX12: v_maximumminimum_f16_e64_dpp v5, v1, -|v2|, -|0.5| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x05,0x06,0x6f,0xd6,0xea,0x04,0xc2,0xc3,0x01,0x77,0x39,0x05]
+# W32-REAL16: v_maximumminimum_f16_e64_dpp v5.l, v1.l, -|v2.l|, -|0.5| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x05,0x06,0x6f,0xd6,0xea,0x04,0xc2,0xc3,0x01,0x77,0x39,0x05]
+# W32-FAKE16: v_maximumminimum_f16_e64_dpp v5, v1, -|v2|, -|0.5| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x05,0x06,0x6f,0xd6,0xea,0x04,0xc2,0xc3,0x01,0x77,0x39,0x05]
+# W64-REAL16: v_maximumminimum_f16_e64_dpp v5.l, v1.l, -|v2.l|, -|0.5| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x05,0x06,0x6f,0xd6,0xea,0x04,0xc2,0xc3,0x01,0x77,0x39,0x05]
+# W64-FAKE16: v_maximumminimum_f16_e64_dpp v5, v1, -|v2|, -|0.5| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x05,0x06,0x6f,0xd6,0xea,0x04,0xc2,0xc3,0x01,0x77,0x39,0x05]
 
 0x05,0x02,0x6f,0xd6,0xe9,0x04,0xfe,0x41,0x01,0x77,0x39,0x05
-# GFX12: v_maximumminimum_f16_e64_dpp v5, v1, -|v2|, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x02,0x6f,0xd6,0xe9,0x04,0xfe,0x41,0x01,0x77,0x39,0x05]
+# W32-REAL16: v_maximumminimum_f16_e64_dpp v5.l, v1.l, -|v2.l|, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x02,0x6f,0xd6,0xe9,0x04,0xfe,0x41,0x01,0x77,0x39,0x05]
+# W32-FAKE16: v_maximumminimum_f16_e64_dpp v5, v1, -|v2|, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x02,0x6f,0xd6,0xe9,0x04,0xfe,0x41,0x01,0x77,0x39,0x05]
+# W64-REAL16: v_maximumminimum_f16_e64_dpp v5.l, v1.l, -|v2.l|, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x02,0x6f,0xd6,0xe9,0x04,0xfe,0x41,0x01,0x77,0x39,0x05]
+# W64-FAKE16: v_maximumminimum_f16_e64_dpp v5, v1, -|v2|, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x02,0x6f,0xd6,0xe9,0x04,0xfe,0x41,0x01,0x77,0x39,0x05]
 
 0x05,0x00,0x6f,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05
-# GFX12: v_maximumminimum_f16_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x6f,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
+# W32-REAL16: v_maximumminimum_f16_e64_dpp v5.l, v1.l, v2.l, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x6f,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
+# W32-FAKE16: v_maximumminimum_f16_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x6f,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
+# W64-REAL16: v_maximumminimum_f16_e64_dpp v5.l, v1.l, v2.l, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x6f,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
+# W64-FAKE16: v_maximumminimum_f16_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x6f,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 
 0x05,0x00,0x6f,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05
-# GFX12: v_maximumminimum_f16_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x6f,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
+# W32-REAL16: v_maximumminimum_f16_e64_dpp v5.l, v1.l, v2.l, v255.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x6f,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
+# W32-FAKE16: v_maximumminimum_f16_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x6f,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
+# W64-REAL16: v_maximumminimum_f16_e64_dpp v5.l, v1.l, v2.l, v255.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x6f,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
+# W64-FAKE16: v_maximumminimum_f16_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x6f,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 
 0x05,0x00,0x6f,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05
-# GFX12: v_maximumminimum_f16_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x6f,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
+# W32-REAL16: v_maximumminimum_f16_e64_dpp v5.l, v1.l, v2.l, v3.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x6f,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
+# W32-FAKE16: v_maximumminimum_f16_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x6f,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
+# W64-REAL16: v_maximumminimum_f16_e64_dpp v5.l, v1.l, v2.l, v3.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x6f,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
+# W64-FAKE16: v_maximumminimum_f16_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x6f,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 
 0x05,0x00,0x6f,0xd6,0xe9,0x06,0x0c,0x04,0x01,0x77,0x39,0x05
-# GFX12: v_maximumminimum_f16_e64_dpp v5, v1, s3, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x6f,0xd6,0xe9,0x06,0x0c,0x04,0x01,0x77,0x39,0x05]
+# W32-REAL16: v_maximumminimum_f16_e64_dpp v5.l, v1.l, s3, v3.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x6f,0xd6,0xe9,0x06,0x0c,0x04,0x01,0x77,0x39,0x05]
+# W32-FAKE16: v_maximumminimum_f16_e64_dpp v5, v1, s3, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x6f,0xd6,0xe9,0x06,0x0c,0x04,0x01,0x77,0x39,0x05]
+# W64-REAL16: v_maximumminimum_f16_e64_dpp v5.l, v1.l, s3, v3.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x6f,0xd6,0xe9,0x06,0x0c,0x04,0x01,0x77,0x39,0x05]
+# W64-FAKE16: v_maximumminimum_f16_e64_dpp v5, v1, s3, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x6f,0xd6,0xe9,0x06,0x0c,0x04,0x01,0x77,0x39,0x05]
 
 0x05,0x00,0x6f,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05
-# GFX12: v_maximumminimum_f16_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x6f,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
+# W32-REAL16: v_maximumminimum_f16_e64_dpp v5.l, v1.l, v2.l, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x6f,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
+# W32-FAKE16: v_maximumminimum_f16_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x6f,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
+# W64-REAL16: v_maximumminimum_f16_e64_dpp v5.l, v1.l, v2.l, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x6f,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
+# W64-FAKE16: v_maximumminimum_f16_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x6f,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 
 0x05,0x00,0x6f,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05
-# GFX12: v_maximumminimum_f16_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x6f,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
+# W32-REAL16: v_maximumminimum_f16_e64_dpp v5.l, v1.l, v2.l, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x6f,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
+# W32-FAKE16: v_maximumminimum_f16_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x6f,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
+# W64-REAL16: v_maximumminimum_f16_e64_dpp v5.l, v1.l, v2.l, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x6f,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
+# W64-FAKE16: v_maximumminimum_f16_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x6f,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 
 0x05,0x01,0x6f,0xd6,0xe9,0x04,0xee,0x81,0x01,0x77,0x39,0x05
-# GFX12: v_maximumminimum_f16_e64_dpp v5, |v1|, v2, -ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x6f,0xd6,0xe9,0x04,0xee,0x81,0x01,0x77,0x39,0x05]
+# W32-REAL16: v_maximumminimum_f16_e64_dpp v5.l, |v1.l|, v2.l, -ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x6f,0xd6,0xe9,0x04,0xee,0x81,0x01,0x77,0x39,0x05]
+# W32-FAKE16: v_maximumminimum_f16_e64_dpp v5, |v1|, v2, -ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x6f,0xd6,0xe9,0x04,0xee,0x81,0x01,0x77,0x39,0x05]
+# W64-REAL16: v_maximumminimum_f16_e64_dpp v5.l, |v1.l|, v2.l, -ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x6f,0xd6,0xe9,0x04,0xee,0x81,0x01,0x77,0x39,0x05]
+# W64-FAKE16: v_maximumminimum_f16_e64_dpp v5, |v1|, v2, -ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x6f,0xd6,0xe9,0x04,0xee,0x81,0x01,0x77,0x39,0x05]
+
+0xff,0xc7,0x6f,0xd6,0xe9,0xfe,0xf7,0xfb,0xff,0x00,0x00,0x00
+# W32-REAL16: v_maximumminimum_f16_e64_dpp v255.h, -|v255.l|, -|v255.l|, -|src_scc| op_sel:[0,0,0,1] clamp div:2 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0xc7,0x6f,0xd6,0xe9,0xfe,0xf7,0xfb,0xff,0x00,0x00,0x00]
+# W32-FAKE16: v_maximumminimum_f16_e64_dpp v255, -|v255|, -|v255|, -|src_scc| op_sel:[0,0,0,1] clamp div:2 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0xc7,0x6f,0xd6,0xe9,0xfe,0xf7,0xfb,0xff,0x00,0x00,0x00]
+# W64-REAL16: v_maximumminimum_f16_e64_dpp v255.h, -|v255.l|, -|v255.l|, -|src_scc| op_sel:[0,0,0,1] clamp div:2 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0xc7,0x6f,0xd6,0xe9,0xfe,0xf7,0xfb,0xff,0x00,0x00,0x00]
+# W64-FAKE16: v_maximumminimum_f16_e64_dpp v255, -|v255|, -|v255|, -|src_scc| op_sel:[0,0,0,1] clamp div:2 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0xc7,0x6f,0xd6,0xe9,0xfe,0xf7,0xfb,0xff,0x00,0x00,0x00]
 
 0xff,0x87,0x6e,0xd6,0xe9,0xfe,0xf7,0xe3,0xff,0x00,0x00,0x00
-# GFX12: v_minimummaximum_f16_e64_dpp v255, -|v255|, -|v255|, -|src_scc| clamp dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x87,0x6e,0xd6,0xe9,0xfe,0xf7,0xe3,0xff,0x00,0x00,0x00]
+# W32-REAL16: v_minimummaximum_f16_e64_dpp v255.l, -|v255.l|, -|v255.l|, -|src_scc| clamp dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x87,0x6e,0xd6,0xe9,0xfe,0xf7,0xe3,0xff,0x00,0x00,0x00]
+# W32-FAKE16: v_minimummaximum_f16_e64_dpp v255, -|v255|, -|v255|, -|src_scc| clamp dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x87,0x6e,0xd6,0xe9,0xfe,0xf7,0xe3,0xff,0x00,0x00,0x00]
+# W64-REAL16: v_minimummaximum_f16_e64_dpp v255.l, -|v255.l|, -|v255.l|, -|src_scc| clamp dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x87,0x6e,0xd6,0xe9,0xfe,0xf7,0xe3,0xff,0x00,0x00,0x00]
+# W64-FAKE16: v_minimummaximum_f16_e64_dpp v255, -|v255|, -|v255|, -|src_scc| clamp dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x87,0x6e,0xd6,0xe9,0xfe,0xf7,0xe3,0xff,0x00,0x00,0x00]
 
 0x05,0x04,0x6e,0xd6,0xe9,0x04,0xfa,0x21,0x01,0x77,0x39,0x05
-# GFX12: v_minimummaximum_f16_e64_dpp v5, -v1, v2, |exec_lo| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x04,0x6e,0xd6,0xe9,0x04,0xfa,0x21,0x01,0x77,0x39,0x05]
+# W32-REAL16: v_minimummaximum_f16_e64_dpp v5.l, -v1.l, v2.l, |exec_lo| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x04,0x6e,0xd6,0xe9,0x04,0xfa,0x21,0x01,0x77,0x39,0x05]
+# W32-FAKE16: v_minimummaximum_f16_e64_dpp v5, -v1, v2, |exec_lo| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x04,0x6e,0xd6,0xe9,0x04,0xfa,0x21,0x01,0x77,0x39,0x05]
+# W64-REAL16: v_minimummaximum_f16_e64_dpp v5.l, -v1.l, v2.l, |exec_lo| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x04,0x6e,0xd6,0xe9,0x04,0xfa,0x21,0x01,0x77,0x39,0x05]
+# W64-FAKE16: v_minimummaximum_f16_e64_dpp v5, -v1, v2, |exec_lo| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x04,0x6e,0xd6,0xe9,0x04,0xfa,0x21,0x01,0x77,0x39,0x05]
 
 0x05,0x03,0x6e,0xd6,0xe9,0x04,0xf2,0x61,0x01,0x77,0x39,0x05
-# GFX12: v_minimummaximum_f16_e64_dpp v5, -|v1|, -|v2|, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x03,0x6e,0xd6,0xe9,0x04,0xf2,0x61,0x01,0x77,0x39,0x05]
+# W32-REAL16: v_minimummaximum_f16_e64_dpp v5.l, -|v1.l|, -|v2.l|, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x03,0x6e,0xd6,0xe9,0x04,0xf2,0x61,0x01,0x77,0x39,0x05]
+# W32-FAKE16: v_minimummaximum_f16_e64_dpp v5, -|v1|, -|v2|, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x03,0x6e,0xd6,0xe9,0x04,0xf2,0x61,0x01,0x77,0x39,0x05]
+# W64-REAL16: v_minimummaximum_f16_e64_dpp v5.l, -|v1.l|, -|v2.l|, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x03,0x6e,0xd6,0xe9,0x04,0xf2,0x61,0x01,0x77,0x39,0x05]
+# W64-FAKE16: v_minimummaximum_f16_e64_dpp v5, -|v1|, -|v2|, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x03,0x6e,0xd6,0xe9,0x04,0xf2,0x61,0x01,0x77,0x39,0x05]
 
 0x05,0x05,0x6e,0xd6,0xe9,0x04,0x06,0xa3,0x01,0x77,0x39,0x05
-# GFX12: v_minimummaximum_f16_e64_dpp v5, -|v1|, v2, -|-1| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x05,0x6e,0xd6,0xe9,0x04,0x06,0xa3,0x01,0x77,0x39,0x05]
+# W32-REAL16: v_minimummaximum_f16_e64_dpp v5.l, -|v1.l|, v2.l, -|-1| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x05,0x6e,0xd6,0xe9,0x04,0x06,0xa3,0x01,0x77,0x39,0x05]
+# W32-FAKE16: v_minimummaximum_f16_e64_dpp v5, -|v1|, v2, -|-1| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x05,0x6e,0xd6,0xe9,0x04,0x06,0xa3,0x01,0x77,0x39,0x05]
+# W64-REAL16: v_minimummaximum_f16_e64_dpp v5.l, -|v1.l|, v2.l, -|-1| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x05,0x6e,0xd6,0xe9,0x04,0x06,0xa3,0x01,0x77,0x39,0x05]
+# W64-FAKE16: v_minimummaximum_f16_e64_dpp v5, -|v1|, v2, -|-1| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x05,0x6e,0xd6,0xe9,0x04,0x06,0xa3,0x01,0x77,0x39,0x05]
 
 0x05,0x06,0x6e,0xd6,0xea,0x04,0xc2,0xc3,0x01,0x77,0x39,0x05
-# GFX12: v_minimummaximum_f16_e64_dpp v5, v1, -|v2|, -|0.5| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x05,0x06,0x6e,0xd6,0xea,0x04,0xc2,0xc3,0x01,0x77,0x39,0x05]
+# W32-REAL16: v_minimummaximum_f16_e64_dpp v5.l, v1.l, -|v2.l|, -|0.5| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x05,0x06,0x6e,0xd6,0xea,0x04,0xc2,0xc3,0x01,0x77,0x39,0x05]
+# W32-FAKE16: v_minimummaximum_f16_e64_dpp v5, v1, -|v2|, -|0.5| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x05,0x06,0x6e,0xd6,0xea,0x04,0xc2,0xc3,0x01,0x77,0x39,0x05]
+# W64-REAL16: v_minimummaximum_f16_e64_dpp v5.l, v1.l, -|v2.l|, -|0.5| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x05,0x06,0x6e,0xd6,0xea,0x04,0xc2,0xc3,0x01,0x77,0x39,0x05]
+# W64-FAKE16: v_minimummaximum_f16_e64_dpp v5, v1, -|v2|, -|0.5| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x05,0x06,0x6e,0xd6,0xea,0x04,0xc2,0xc3,0x01,0x77,0x39,0x05]
 
 0x05,0x02,0x6e,0xd6,0xe9,0x04,0xfe,0x41,0x01,0x77,0x39,0x05
-# GFX12: v_minimummaximum_f16_e64_dpp v5, v1, -|v2|, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x02,0x6e,0xd6,0xe9,0x04,0xfe,0x41,0x01,0x77,0x39,0x05]
+# W32-REAL16: v_minimummaximum_f16_e64_dpp v5.l, v1.l, -|v2.l|, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x02,0x6e,0xd6,0xe9,0x04,0xfe,0x41,0x01,0x77,0x39,0x05]
+# W32-FAKE16: v_minimummaximum_f16_e64_dpp v5, v1, -|v2|, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x02,0x6e,0xd6,0xe9,0x04,0xfe,0x41,0x01,0x77,0x39,0x05]
+# W64-REAL16: v_minimummaximum_f16_e64_dpp v5.l, v1.l, -|v2.l|, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x02,0x6e,0xd6,0xe9,0x04,0xfe,0x41,0x01,0x77,0x39,0x05]
+# W64-FAKE16: v_minimummaximum_f16_e64_dpp v5, v1, -|v2|, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x02,0x6e,0xd6,0xe9,0x04,0xfe,0x41,0x01,0x77,0x39,0x05]
 
 0x05,0x00,0x6e,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05
-# GFX12: v_minimummaximum_f16_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x6e,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
+# W32-REAL16: v_minimummaximum_f16_e64_dpp v5.l, v1.l, v2.l, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x6e,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
+# W32-FAKE16: v_minimummaximum_f16_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x6e,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
+# W64-REAL16: v_minimummaximum_f16_e64_dpp v5.l, v1.l, v2.l, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x6e,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
+# W64-FAKE16: v_minimummaximum_f16_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x6e,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 
 0x05,0x00,0x6e,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05
-# GFX12: v_minimummaximum_f16_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x6e,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
+# W32-REAL16: v_minimummaximum_f16_e64_dpp v5.l, v1.l, v2.l, v255.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x6e,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
+# W32-FAKE16: v_minimummaximum_f16_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x6e,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
+# W64-REAL16: v_minimummaximum_f16_e64_dpp v5.l, v1.l, v2.l, v255.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x6e,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
+# W64-FAKE16: v_minimummaximum_f16_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x6e,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 
 0x05,0x00,0x6e,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05
-# GFX12: v_minimummaximum_f16_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x6e,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
+# W32-REAL16: v_minimummaximum_f16_e64_dpp v5.l, v1.l, v2.l, v3.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x6e,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
+# W32-FAKE16: v_minimummaximum_f16_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x6e,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
+# W64-REAL16: v_minimummaximum_f16_e64_dpp v5.l, v1.l, v2.l, v3.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x6e,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
+# W64-FAKE16: v_minimummaximum_f16_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x6e,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 
 0x05,0x00,0x6e,0xd6,0xe9,0x06,0x0c,0x04,0x01,0x77,0x39,0x05
-# GFX12: v_minimummaximum_f16_e64_dpp v5, v1, s3, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x6e,0xd6,0xe9,0x06,0x0c,0x04,0x01,0x77,0x39,0x05]
+# W32-REAL16: v_minimummaximum_f16_e64_dpp v5.l, v1.l, s3, v3.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x6e,0xd6,0xe9,0x06,0x0c,0x04,0x01,0x77,0x39,0x05]
+# W32-FAKE16: v_minimummaximum_f16_e64_dpp v5, v1, s3, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x6e,0xd6,0xe9,0x06,0x0c,0x04,0x01,0x77,0x39,0x05]
+# W64-REAL16: v_minimummaximum_f16_e64_dpp v5.l, v1.l, s3, v3.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x6e,0xd6,0xe9,0x06,0x0c,0x04,0x01,0x77,0x39,0x05]
+# W64-FAKE16: v_minimummaximum_f16_e64_dpp v5, v1, s3, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x6e,0xd6,0xe9,0x06,0x0c,0x04,0x01,0x77,0x39,0x05]
 
 0x05,0x00,0x6e,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05
-# GFX12: v_minimummaximum_f16_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x6e,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
+# W32-REAL16: v_minimummaximum_f16_e64_dpp v5.l, v1.l, v2.l, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x6e,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
+# W32-FAKE16: v_minimummaximum_f16_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x6e,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
+# W64-REAL16: v_minimummaximum_f16_e64_dpp v5.l, v1.l, v2.l, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x6e,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
+# W64-FAKE16: v_minimummaximum_f16_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x6e,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 
 0x05,0x00,0x6e,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05
-# GFX12: v_minimummaximum_f16_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x6e,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
+# W32-REAL16: v_minimummaximum_f16_e64_dpp v5.l, v1.l, v2.l, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x6e,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
+# W32-FAKE16: v_minimummaximum_f16_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x6e,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
+# W64-REAL16: v_minimummaximum_f16_e64_dpp v5.l, v1.l, v2.l, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x6e,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
+# W64-FAKE16: v_minimummaximum_f16_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x6e,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 
 0x05,0x01,0x6e,0xd6,0xe9,0x04,0xee,0x81,0x01,0x77,0x39,0x05
-# GFX12: v_minimummaximum_f16_e64_dpp v5, |v1|, v2, -ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x6e,0xd6,0xe9,0x04,0xee,0x81,0x01,0x77,0x39,0x05]
+# W32-REAL16: v_minimummaximum_f16_e64_dpp v5.l, |v1.l|, v2.l, -ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x6e,0xd6,0xe9,0x04,0xee,0x81,0x01,0x77,0x39,0x05]
+# W32-FAKE16: v_minimummaximum_f16_e64_dpp v5, |v1|, v2, -ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x6e,0xd6,0xe9,0x04,0xee,0x81,0x01,0x77,0x39,0x05]
+# W64-REAL16: v_minimummaximum_f16_e64_dpp v5.l, |v1.l|, v2.l, -ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x6e,0xd6,0xe9,0x04,0xee,0x81,0x01,0x77,0x39,0x05]
+# W64-FAKE16: v_minimummaximum_f16_e64_dpp v5, |v1|, v2, -ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x6e,0xd6,0xe9,0x04,0xee,0x81,0x01,0x77,0x39,0x05]
+
+0xff,0xc7,0x6e,0xd6,0xe9,0xfe,0xf7,0xfb,0xff,0x00,0x00,0x00
+# W32-REAL16: v_minimummaximum_f16_e64_dpp v255.h, -|v255.l|, -|v255.l|, -|src_scc| op_sel:[0,0,0,1] clamp div:2 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0xc7,0x6e,0xd6,0xe9,0xfe,0xf7,0xfb,0xff,0x00,0x00,0x00]
+# W32-FAKE16: v_minimummaximum_f16_e64_dpp v255, -|v255|, -|v255|, -|src_scc| op_sel:[0,0,0,1] clamp div:2 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0xc7,0x6e,0xd6,0xe9,0xfe,0xf7,0xfb,0xff,0x00,0x00,0x00]
+# W64-REAL16: v_minimummaximum_f16_e64_dpp v255.h, -|v255.l|, -|v255.l|, -|src_scc| op_sel:[0,0,0,1] clamp div:2 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0xc7,0x6e,0xd6,0xe9,0xfe,0xf7,0xfb,0xff,0x00,0x00,0x00]
+# W64-FAKE16: v_minimummaximum_f16_e64_dpp v255, -|v255|, -|v255|, -|src_scc| op_sel:[0,0,0,1] clamp div:2 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0xc7,0x6e,0xd6,0xe9,0xfe,0xf7,0xfb,0xff,0x00,0x00,0x00]
diff --git a/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vop3cx.txt b/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vop3cx.txt
index 80235451fec6f..ab2d154e9ef9f 100644
--- a/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vop3cx.txt
+++ b/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vop3cx.txt
@@ -156,10 +156,12 @@
 # GFX12: v_cmpx_class_f64_e64 0xaf123456, 0xaf123456 ; encoding: [0x7e,0x00,0xff,0xd4,0xff,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
 
 0x7e,0x00,0x82,0xd4,0x01,0x05,0x02,0x00
-# GFX12: v_cmpx_eq_f16_e64 v1, v2                ; encoding: [0x7e,0x00,0x82,0xd4,0x01,0x05,0x02,0x00]
+# GFX12-REAL16: v_cmpx_eq_f16_e64 v1.l, v2.l            ; encoding: [0x7e,0x00,0x82,0xd4,0x01,0x05,0x02,0x00]
+# GFX12-FAKE16: v_cmpx_eq_f16_e64 v1, v2                ; encoding: [0x7e,0x00,0x82,0xd4,0x01,0x05,0x02,0x00]
 
 0x7e,0x00,0x82,0xd4,0xff,0xff,0x03,0x00
-# GFX12: v_cmpx_eq_f16_e64 v255, v255            ; encoding: [0x7e,0x00,0x82,0xd4,0xff,0xff,0x03,0x00]
+# GFX12-REAL16: v_cmpx_eq_f16_e64 v255.l, v255.l        ; encoding: [0x7e,0x00,0x82,0xd4,0xff,0xff,0x03,0x00]
+# GFX12-FAKE16: v_cmpx_eq_f16_e64 v255, v255            ; encoding: [0x7e,0x00,0x82,0xd4,0xff,0xff,0x03,0x00]
 
 0x7e,0x00,0x82,0xd4,0x01,0x04,0x00,0x00
 # GFX12: v_cmpx_eq_f16_e64 s1, s2                ; encoding: [0x7e,0x00,0x82,0xd4,0x01,0x04,0x00,0x00]
@@ -200,6 +202,15 @@
 0x7e,0x83,0x82,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00
 # GFX12: v_cmpx_eq_f16_e64 -|0xfe0b|, -|vcc_hi| clamp ; encoding: [0x7e,0x83,0x82,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00]
 
+0x7e,0x08,0x82,0xd4,0x01,0x05,0x02,0x00
+# GFX12-REAL16: v_cmpx_eq_f16_e64 v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x7e,0x08,0x82,0xd4,0x01,0x05,0x02,0x00]
+# GFX12-FAKE16: v_cmpx_eq_f16_e64 v1, v2                ; encoding: [0x7e,0x00,0x82,0xd4,0x01,0x05,0x02,0x00]
+
+0x7e,0x10,0x82,0xd4,0xff,0xff,0x03,0x00
+# GFX12-REAL16: v_cmpx_eq_f16_e64 v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x7e,0x10,0x82,0xd4,0xff,0xff,0x03,0x00]
+# GFX12-FAKE16: v_cmpx_eq_f16_e64 v255, v255            ; encoding: [0x7e,0x00,0x82,0xd4,0xff,0xff,0x03,0x00]
+
+
 0x7e,0x00,0x92,0xd4,0x01,0x05,0x02,0x00
 # GFX12: v_cmpx_eq_f32_e64 v1, v2                ; encoding: [0x7e,0x00,0x92,0xd4,0x01,0x05,0x02,0x00]
 
@@ -336,7 +347,6 @@
 # GFX12-REAL16: v_cmpx_eq_i16_e64 v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x7e,0x10,0xb2,0xd4,0xff,0xff,0x03,0x00]
 # GFX12-FAKE16: v_cmpx_eq_i16_e64 v255, v255            ; encoding: [0x7e,0x00,0xb2,0xd4,0xff,0xff,0x03,0x00]
 
-# GFX11: v_cmpx_eq_i16_e64 v1.h, v2.l op_sel:[1,0] ; encoding: [0x7e,0x08,0xb2,0xd4,0x01,0x05,0x02,0x00]
 
 0x7e,0x00,0xc2,0xd4,0x01,0x05,0x02,0x00
 # GFX12: v_cmpx_eq_i32_e64 v1, v2                ; encoding: [0x7e,0x00,0xc2,0xd4,0x01,0x05,0x02,0x00]
@@ -474,7 +484,6 @@
 # GFX12-REAL16: v_cmpx_eq_u16_e64 v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x7e,0x10,0xba,0xd4,0xff,0xff,0x03,0x00]
 # GFX12-FAKE16: v_cmpx_eq_u16_e64 v255, v255            ; encoding: [0x7e,0x00,0xba,0xd4,0xff,0xff,0x03,0x00]
 
-# GFX11: v_cmpx_eq_u16_e64 v1.h, v2.l op_sel:[1,0] ; encoding: [0x7e,0x08,0xba,0xd4,0x01,0x05,0x02,0x00]
 
 0x7e,0x00,0xca,0xd4,0x01,0x05,0x02,0x00
 # GFX12: v_cmpx_eq_u32_e64 v1, v2                ; encoding: [0x7e,0x00,0xca,0xd4,0x01,0x05,0x02,0x00]
@@ -558,10 +567,12 @@
 # GFX12: v_cmpx_eq_u64_e64 0xaf123456, vcc       ; encoding: [0x7e,0x00,0xda,0xd4,0xff,0xd4,0x00,0x00,0x56,0x34,0x12,0xaf]
 
 0x7e,0x00,0x86,0xd4,0x01,0x05,0x02,0x00
-# GFX12: v_cmpx_ge_f16_e64 v1, v2                ; encoding: [0x7e,0x00,0x86,0xd4,0x01,0x05,0x02,0x00]
+# GFX12-REAL16: v_cmpx_ge_f16_e64 v1.l, v2.l            ; encoding: [0x7e,0x00,0x86,0xd4,0x01,0x05,0x02,0x00]
+# GFX12-FAKE16: v_cmpx_ge_f16_e64 v1, v2                ; encoding: [0x7e,0x00,0x86,0xd4,0x01,0x05,0x02,0x00]
 
 0x7e,0x00,0x86,0xd4,0xff,0xff,0x03,0x00
-# GFX12: v_cmpx_ge_f16_e64 v255, v255            ; encoding: [0x7e,0x00,0x86,0xd4,0xff,0xff,0x03,0x00]
+# GFX12-REAL16: v_cmpx_ge_f16_e64 v255.l, v255.l        ; encoding: [0x7e,0x00,0x86,0xd4,0xff,0xff,0x03,0x00]
+# GFX12-FAKE16: v_cmpx_ge_f16_e64 v255, v255            ; encoding: [0x7e,0x00,0x86,0xd4,0xff,0xff,0x03,0x00]
 
 0x7e,0x00,0x86,0xd4,0x01,0x04,0x00,0x00
 # GFX12: v_cmpx_ge_f16_e64 s1, s2                ; encoding: [0x7e,0x00,0x86,0xd4,0x01,0x04,0x00,0x00]
@@ -602,6 +613,15 @@
 0x7e,0x83,0x86,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00
 # GFX12: v_cmpx_ge_f16_e64 -|0xfe0b|, -|vcc_hi| clamp ; encoding: [0x7e,0x83,0x86,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00]
 
+0x7e,0x08,0x86,0xd4,0x01,0x05,0x02,0x00
+# GFX12-REAL16: v_cmpx_ge_f16_e64 v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x7e,0x08,0x86,0xd4,0x01,0x05,0x02,0x00]
+# GFX12-FAKE16: v_cmpx_ge_f16_e64 v1, v2                ; encoding: [0x7e,0x00,0x86,0xd4,0x01,0x05,0x02,0x00]
+
+0x7e,0x10,0x86,0xd4,0xff,0xff,0x03,0x00
+# GFX12-REAL16: v_cmpx_ge_f16_e64 v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x7e,0x10,0x86,0xd4,0xff,0xff,0x03,0x00]
+# GFX12-FAKE16: v_cmpx_ge_f16_e64 v255, v255            ; encoding: [0x7e,0x00,0x86,0xd4,0xff,0xff,0x03,0x00]
+
+
 0x7e,0x00,0x96,0xd4,0x01,0x05,0x02,0x00
 # GFX12: v_cmpx_ge_f32_e64 v1, v2                ; encoding: [0x7e,0x00,0x96,0xd4,0x01,0x05,0x02,0x00]
 
@@ -738,7 +758,6 @@
 # GFX12-REAL16: v_cmpx_ge_i16_e64 v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x7e,0x10,0xb6,0xd4,0xff,0xff,0x03,0x00]
 # GFX12-FAKE16: v_cmpx_ge_i16_e64 v255, v255            ; encoding: [0x7e,0x00,0xb6,0xd4,0xff,0xff,0x03,0x00]
 
-# GFX11: v_cmpx_ge_i16_e64 v1.h, v2.l op_sel:[1,0] ; encoding: [0x7e,0x08,0xb6,0xd4,0x01,0x05,0x02,0x00]
 
 0x7e,0x00,0xc6,0xd4,0x01,0x05,0x02,0x00
 # GFX12: v_cmpx_ge_i32_e64 v1, v2                ; encoding: [0x7e,0x00,0xc6,0xd4,0x01,0x05,0x02,0x00]
@@ -876,7 +895,6 @@
 # GFX12-REAL16: v_cmpx_ge_u16_e64 v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x7e,0x10,0xbe,0xd4,0xff,0xff,0x03,0x00]
 # GFX12-FAKE16: v_cmpx_ge_u16_e64 v255, v255            ; encoding: [0x7e,0x00,0xbe,0xd4,0xff,0xff,0x03,0x00]
 
-# GFX11: v_cmpx_ge_u16_e64 v1.h, v2.l op_sel:[1,0] ; encoding: [0x7e,0x08,0xbe,0xd4,0x01,0x05,0x02,0x00]
 
 0x7e,0x00,0xce,0xd4,0x01,0x05,0x02,0x00
 # GFX12: v_cmpx_ge_u32_e64 v1, v2                ; encoding: [0x7e,0x00,0xce,0xd4,0x01,0x05,0x02,0x00]
@@ -960,10 +978,12 @@
 # GFX12: v_cmpx_ge_u64_e64 0xaf123456, vcc       ; encoding: [0x7e,0x00,0xde,0xd4,0xff,0xd4,0x00,0x00,0x56,0x34,0x12,0xaf]
 
 0x7e,0x00,0x84,0xd4,0x01,0x05,0x02,0x00
-# GFX12: v_cmpx_gt_f16_e64 v1, v2                ; encoding: [0x7e,0x00,0x84,0xd4,0x01,0x05,0x02,0x00]
+# GFX12-REAL16: v_cmpx_gt_f16_e64 v1.l, v2.l            ; encoding: [0x7e,0x00,0x84,0xd4,0x01,0x05,0x02,0x00]
+# GFX12-FAKE16: v_cmpx_gt_f16_e64 v1, v2                ; encoding: [0x7e,0x00,0x84,0xd4,0x01,0x05,0x02,0x00]
 
 0x7e,0x00,0x84,0xd4,0xff,0xff,0x03,0x00
-# GFX12: v_cmpx_gt_f16_e64 v255, v255            ; encoding: [0x7e,0x00,0x84,0xd4,0xff,0xff,0x03,0x00]
+# GFX12-REAL16: v_cmpx_gt_f16_e64 v255.l, v255.l        ; encoding: [0x7e,0x00,0x84,0xd4,0xff,0xff,0x03,0x00]
+# GFX12-FAKE16: v_cmpx_gt_f16_e64 v255, v255            ; encoding: [0x7e,0x00,0x84,0xd4,0xff,0xff,0x03,0x00]
 
 0x7e,0x00,0x84,0xd4,0x01,0x04,0x00,0x00
 # GFX12: v_cmpx_gt_f16_e64 s1, s2                ; encoding: [0x7e,0x00,0x84,0xd4,0x01,0x04,0x00,0x00]
@@ -1004,6 +1024,15 @@
 0x7e,0x83,0x84,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00
 # GFX12: v_cmpx_gt_f16_e64 -|0xfe0b|, -|vcc_hi| clamp ; encoding: [0x7e,0x83,0x84,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00]
 
+0x7e,0x08,0x84,0xd4,0x01,0x05,0x02,0x00
+# GFX12-REAL16: v_cmpx_gt_f16_e64 v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x7e,0x08,0x84,0xd4,0x01,0x05,0x02,0x00]
+# GFX12-FAKE16: v_cmpx_gt_f16_e64 v1, v2                ; encoding: [0x7e,0x00,0x84,0xd4,0x01,0x05,0x02,0x00]
+
+0x7e,0x10,0x84,0xd4,0xff,0xff,0x03,0x00
+# GFX12-REAL16: v_cmpx_gt_f16_e64 v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x7e,0x10,0x84,0xd4,0xff,0xff,0x03,0x00]
+# GFX12-FAKE16: v_cmpx_gt_f16_e64 v255, v255            ; encoding: [0x7e,0x00,0x84,0xd4,0xff,0xff,0x03,0x00]
+
+
 0x7e,0x00,0x94,0xd4,0x01,0x05,0x02,0x00
 # GFX12: v_cmpx_gt_f32_e64 v1, v2                ; encoding: [0x7e,0x00,0x94,0xd4,0x01,0x05,0x02,0x00]
 
@@ -1140,7 +1169,6 @@
 # GFX12-REAL16: v_cmpx_gt_i16_e64 v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x7e,0x10,0xb4,0xd4,0xff,0xff,0x03,0x00]
 # GFX12-FAKE16: v_cmpx_gt_i16_e64 v255, v255            ; encoding: [0x7e,0x00,0xb4,0xd4,0xff,0xff,0x03,0x00]
 
-# GFX11: v_cmpx_gt_i16_e64 v1.h, v2.l op_sel:[1,0] ; encoding: [0x7e,0x08,0xb4,0xd4,0x01,0x05,0x02,0x00]
 
 0x7e,0x00,0xc4,0xd4,0x01,0x05,0x02,0x00
 # GFX12: v_cmpx_gt_i32_e64 v1, v2                ; encoding: [0x7e,0x00,0xc4,0xd4,0x01,0x05,0x02,0x00]
@@ -1278,7 +1306,6 @@
 # GFX12-REAL16: v_cmpx_gt_u16_e64 v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x7e,0x10,0xbc,0xd4,0xff,0xff,0x03,0x00]
 # GFX12-FAKE16: v_cmpx_gt_u16_e64 v255, v255            ; encoding: [0x7e,0x00,0xbc,0xd4,0xff,0xff,0x03,0x00]
 
-# GFX11: v_cmpx_gt_u16_e64 v1.h, v2.l op_sel:[1,0] ; encoding: [0x7e,0x08,0xbc,0xd4,0x01,0x05,0x02,0x00]
 
 0x7e,0x00,0xcc,0xd4,0x01,0x05,0x02,0x00
 # GFX12: v_cmpx_gt_u32_e64 v1, v2                ; encoding: [0x7e,0x00,0xcc,0xd4,0x01,0x05,0x02,0x00]
@@ -1362,10 +1389,12 @@
 # GFX12: v_cmpx_gt_u64_e64 0xaf123456, vcc       ; encoding: [0x7e,0x00,0xdc,0xd4,0xff,0xd4,0x00,0x00,0x56,0x34,0x12,0xaf]
 
 0x7e,0x00,0x83,0xd4,0x01,0x05,0x02,0x00
-# GFX12: v_cmpx_le_f16_e64 v1, v2                ; encoding: [0x7e,0x00,0x83,0xd4,0x01,0x05,0x02,0x00]
+# GFX12-REAL16: v_cmpx_le_f16_e64 v1.l, v2.l            ; encoding: [0x7e,0x00,0x83,0xd4,0x01,0x05,0x02,0x00]
+# GFX12-FAKE16: v_cmpx_le_f16_e64 v1, v2                ; encoding: [0x7e,0x00,0x83,0xd4,0x01,0x05,0x02,0x00]
 
 0x7e,0x00,0x83,0xd4,0xff,0xff,0x03,0x00
-# GFX12: v_cmpx_le_f16_e64 v255, v255            ; encoding: [0x7e,0x00,0x83,0xd4,0xff,0xff,0x03,0x00]
+# GFX12-REAL16: v_cmpx_le_f16_e64 v255.l, v255.l        ; encoding: [0x7e,0x00,0x83,0xd4,0xff,0xff,0x03,0x00]
+# GFX12-FAKE16: v_cmpx_le_f16_e64 v255, v255            ; encoding: [0x7e,0x00,0x83,0xd4,0xff,0xff,0x03,0x00]
 
 0x7e,0x00,0x83,0xd4,0x01,0x04,0x00,0x00
 # GFX12: v_cmpx_le_f16_e64 s1, s2                ; encoding: [0x7e,0x00,0x83,0xd4,0x01,0x04,0x00,0x00]
@@ -1406,6 +1435,15 @@
 0x7e,0x83,0x83,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00
 # GFX12: v_cmpx_le_f16_e64 -|0xfe0b|, -|vcc_hi| clamp ; encoding: [0x7e,0x83,0x83,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00]
 
+0x7e,0x08,0x83,0xd4,0x01,0x05,0x02,0x00
+# GFX12-REAL16: v_cmpx_le_f16_e64 v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x7e,0x08,0x83,0xd4,0x01,0x05,0x02,0x00]
+# GFX12-FAKE16: v_cmpx_le_f16_e64 v1, v2                ; encoding: [0x7e,0x00,0x83,0xd4,0x01,0x05,0x02,0x00]
+
+0x7e,0x10,0x83,0xd4,0xff,0xff,0x03,0x00
+# GFX12-REAL16: v_cmpx_le_f16_e64 v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x7e,0x10,0x83,0xd4,0xff,0xff,0x03,0x00]
+# GFX12-FAKE16: v_cmpx_le_f16_e64 v255, v255            ; encoding: [0x7e,0x00,0x83,0xd4,0xff,0xff,0x03,0x00]
+
+
 0x7e,0x00,0x93,0xd4,0x01,0x05,0x02,0x00
 # GFX12: v_cmpx_le_f32_e64 v1, v2                ; encoding: [0x7e,0x00,0x93,0xd4,0x01,0x05,0x02,0x00]
 
@@ -1542,7 +1580,6 @@
 # GFX12-REAL16: v_cmpx_le_i16_e64 v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x7e,0x10,0xb3,0xd4,0xff,0xff,0x03,0x00]
 # GFX12-FAKE16: v_cmpx_le_i16_e64 v255, v255            ; encoding: [0x7e,0x00,0xb3,0xd4,0xff,0xff,0x03,0x00]
 
-# GFX11: v_cmpx_le_i16_e64 v1.h, v2.l op_sel:[1,0] ; encoding: [0x7e,0x08,0xb3,0xd4,0x01,0x05,0x02,0x00]
 
 0x7e,0x00,0xc3,0xd4,0x01,0x05,0x02,0x00
 # GFX12: v_cmpx_le_i32_e64 v1, v2                ; encoding: [0x7e,0x00,0xc3,0xd4,0x01,0x05,0x02,0x00]
@@ -1680,7 +1717,6 @@
 # GFX12-REAL16: v_cmpx_le_u16_e64 v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x7e,0x10,0xbb,0xd4,0xff,0xff,0x03,0x00]
 # GFX12-FAKE16: v_cmpx_le_u16_e64 v255, v255            ; encoding: [0x7e,0x00,0xbb,0xd4,0xff,0xff,0x03,0x00]
 
-# GFX11: v_cmpx_le_u16_e64 v1.h, v2.l op_sel:[1,0] ; encoding: [0x7e,0x08,0xbb,0xd4,0x01,0x05,0x02,0x00]
 
 0x7e,0x00,0xcb,0xd4,0x01,0x05,0x02,0x00
 # GFX12: v_cmpx_le_u32_e64 v1, v2                ; encoding: [0x7e,0x00,0xcb,0xd4,0x01,0x05,0x02,0x00]
@@ -1764,10 +1800,12 @@
 # GFX12: v_cmpx_le_u64_e64 0xaf123456, vcc       ; encoding: [0x7e,0x00,0xdb,0xd4,0xff,0xd4,0x00,0x00,0x56,0x34,0x12,0xaf]
 
 0x7e,0x00,0x85,0xd4,0x01,0x05,0x02,0x00
-# GFX12: v_cmpx_lg_f16_e64 v1, v2                ; encoding: [0x7e,0x00,0x85,0xd4,0x01,0x05,0x02,0x00]
+# GFX12-REAL16: v_cmpx_lg_f16_e64 v1.l, v2.l            ; encoding: [0x7e,0x00,0x85,0xd4,0x01,0x05,0x02,0x00]
+# GFX12-FAKE16: v_cmpx_lg_f16_e64 v1, v2                ; encoding: [0x7e,0x00,0x85,0xd4,0x01,0x05,0x02,0x00]
 
 0x7e,0x00,0x85,0xd4,0xff,0xff,0x03,0x00
-# GFX12: v_cmpx_lg_f16_e64 v255, v255            ; encoding: [0x7e,0x00,0x85,0xd4,0xff,0xff,0x03,0x00]
+# GFX12-REAL16: v_cmpx_lg_f16_e64 v255.l, v255.l        ; encoding: [0x7e,0x00,0x85,0xd4,0xff,0xff,0x03,0x00]
+# GFX12-FAKE16: v_cmpx_lg_f16_e64 v255, v255            ; encoding: [0x7e,0x00,0x85,0xd4,0xff,0xff,0x03,0x00]
 
 0x7e,0x00,0x85,0xd4,0x01,0x04,0x00,0x00
 # GFX12: v_cmpx_lg_f16_e64 s1, s2                ; encoding: [0x7e,0x00,0x85,0xd4,0x01,0x04,0x00,0x00]
@@ -1808,6 +1846,15 @@
 0x7e,0x83,0x85,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00
 # GFX12: v_cmpx_lg_f16_e64 -|0xfe0b|, -|vcc_hi| clamp ; encoding: [0x7e,0x83,0x85,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00]
 
+0x7e,0x08,0x85,0xd4,0x01,0x05,0x02,0x00
+# GFX12-REAL16: v_cmpx_lg_f16_e64 v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x7e,0x08,0x85,0xd4,0x01,0x05,0x02,0x00]
+# GFX12-FAKE16: v_cmpx_lg_f16_e64 v1, v2                ; encoding: [0x7e,0x00,0x85,0xd4,0x01,0x05,0x02,0x00]
+
+0x7e,0x10,0x85,0xd4,0xff,0xff,0x03,0x00
+# GFX12-REAL16: v_cmpx_lg_f16_e64 v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x7e,0x10,0x85,0xd4,0xff,0xff,0x03,0x00]
+# GFX12-FAKE16: v_cmpx_lg_f16_e64 v255, v255            ; encoding: [0x7e,0x00,0x85,0xd4,0xff,0xff,0x03,0x00]
+
+
 0x7e,0x00,0x95,0xd4,0x01,0x05,0x02,0x00
 # GFX12: v_cmpx_lg_f32_e64 v1, v2                ; encoding: [0x7e,0x00,0x95,0xd4,0x01,0x05,0x02,0x00]
 
@@ -2080,7 +2127,6 @@
 # GFX12-REAL16: v_cmpx_lt_i16_e64 v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x7e,0x10,0xb1,0xd4,0xff,0xff,0x03,0x00]
 # GFX12-FAKE16: v_cmpx_lt_i16_e64 v255, v255            ; encoding: [0x7e,0x00,0xb1,0xd4,0xff,0xff,0x03,0x00]
 
-# GFX11: v_cmpx_lt_i16_e64 v1.h, v2.l op_sel:[1,0] ; encoding: [0x7e,0x08,0xb1,0xd4,0x01,0x05,0x02,0x00]
 
 0x7e,0x00,0xc1,0xd4,0x01,0x05,0x02,0x00
 # GFX12: v_cmpx_lt_i32_e64 v1, v2                ; encoding: [0x7e,0x00,0xc1,0xd4,0x01,0x05,0x02,0x00]
@@ -2218,7 +2264,6 @@
 # GFX12-REAL16: v_cmpx_lt_u16_e64 v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x7e,0x10,0xb9,0xd4,0xff,0xff,0x03,0x00]
 # GFX12-FAKE16: v_cmpx_lt_u16_e64 v255, v255            ; encoding: [0x7e,0x00,0xb9,0xd4,0xff,0xff,0x03,0x00]
 
-# GFX11: v_cmpx_lt_u16_e64 v1.h, v2.l op_sel:[1,0] ; encoding: [0x7e,0x08,0xb9,0xd4,0x01,0x05,0x02,0x00]
 
 0x7e,0x00,0xc9,0xd4,0x01,0x05,0x02,0x00
 # GFX12: v_cmpx_lt_u32_e64 v1, v2                ; encoding: [0x7e,0x00,0xc9,0xd4,0x01,0x05,0x02,0x00]
@@ -2356,7 +2401,6 @@
 # GFX12-REAL16: v_cmpx_ne_i16_e64 v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x7e,0x10,0xb5,0xd4,0xff,0xff,0x03,0x00]
 # GFX12-FAKE16: v_cmpx_ne_i16_e64 v255, v255            ; encoding: [0x7e,0x00,0xb5,0xd4,0xff,0xff,0x03,0x00]
 
-# GFX11: v_cmpx_ne_i16_e64 v1.h, v2.l op_sel:[1,0] ; encoding: [0x7e,0x08,0xb5,0xd4,0x01,0x05,0x02,0x00]
 
 0x7e,0x00,0xc5,0xd4,0x01,0x05,0x02,0x00
 # GFX12: v_cmpx_ne_i32_e64 v1, v2                ; encoding: [0x7e,0x00,0xc5,0xd4,0x01,0x05,0x02,0x00]
@@ -2494,8 +2538,6 @@
 # GFX12-REAL16: v_cmpx_ne_u16_e64 v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x7e,0x10,0xbd,0xd4,0xff,0xff,0x03,0x00]
 # GFX12-FAKE16: v_cmpx_ne_u16_e64 v255, v255            ; encoding: [0x7e,0x00,0xbd,0xd4,0xff,0xff,0x03,0x00]
 
-# GFX11: v_cmpx_ne_u16_e64 v1.h, v2.l op_sel:[1,0] ; encoding: [0x7e,0x08,0xbd,0xd4,0x01,0x05,0x02,0x00]
-
 0x7e,0x00,0xcd,0xd4,0x01,0x05,0x02,0x00
 # GFX12: v_cmpx_ne_u32_e64 v1, v2                ; encoding: [0x7e,0x00,0xcd,0xd4,0x01,0x05,0x02,0x00]
 
@@ -2578,10 +2620,12 @@
 # GFX12: v_cmpx_ne_u64_e64 0xaf123456, vcc       ; encoding: [0x7e,0x00,0xdd,0xd4,0xff,0xd4,0x00,0x00,0x56,0x34,0x12,0xaf]
 
 0x7e,0x00,0x8d,0xd4,0x01,0x05,0x02,0x00
-# GFX12: v_cmpx_neq_f16_e64 v1, v2               ; encoding: [0x7e,0x00,0x8d,0xd4,0x01,0x05,0x02,0x00]
+# GFX12-REAL16: v_cmpx_neq_f16_e64 v1.l, v2.l           ; encoding: [0x7e,0x00,0x8d,0xd4,0x01,0x05,0x02,0x00]
+# GFX12-FAKE16: v_cmpx_neq_f16_e64 v1, v2               ; encoding: [0x7e,0x00,0x8d,0xd4,0x01,0x05,0x02,0x00]
 
 0x7e,0x00,0x8d,0xd4,0xff,0xff,0x03,0x00
-# GFX12: v_cmpx_neq_f16_e64 v255, v255           ; encoding: [0x7e,0x00,0x8d,0xd4,0xff,0xff,0x03,0x00]
+# GFX12-REAL16: v_cmpx_neq_f16_e64 v255.l, v255.l       ; encoding: [0x7e,0x00,0x8d,0xd4,0xff,0xff,0x03,0x00]
+# GFX12-FAKE16: v_cmpx_neq_f16_e64 v255, v255           ; encoding: [0x7e,0x00,0x8d,0xd4,0xff,0xff,0x03,0x00]
 
 0x7e,0x00,0x8d,0xd4,0x01,0x04,0x00,0x00
 # GFX12: v_cmpx_neq_f16_e64 s1, s2               ; encoding: [0x7e,0x00,0x8d,0xd4,0x01,0x04,0x00,0x00]
@@ -2622,6 +2666,15 @@
 0x7e,0x83,0x8d,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00
 # GFX12: v_cmpx_neq_f16_e64 -|0xfe0b|, -|vcc_hi| clamp ; encoding: [0x7e,0x83,0x8d,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00]
 
+0x7e,0x08,0x8d,0xd4,0x01,0x05,0x02,0x00
+# GFX12-REAL16: v_cmpx_neq_f16_e64 v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x7e,0x08,0x8d,0xd4,0x01,0x05,0x02,0x00]
+# GFX12-FAKE16: v_cmpx_neq_f16_e64 v1, v2               ; encoding: [0x7e,0x00,0x8d,0xd4,0x01,0x05,0x02,0x00]
+
+0x7e,0x10,0x8d,0xd4,0xff,0xff,0x03,0x00
+# GFX12-REAL16: v_cmpx_neq_f16_e64 v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x7e,0x10,0x8d,0xd4,0xff,0xff,0x03,0x00]
+# GFX12-FAKE16: v_cmpx_neq_f16_e64 v255, v255           ; encoding: [0x7e,0x00,0x8d,0xd4,0xff,0xff,0x03,0x00]
+
+
 0x7e,0x00,0x9d,0xd4,0x01,0x05,0x02,0x00
 # GFX12: v_cmpx_neq_f32_e64 v1, v2               ; encoding: [0x7e,0x00,0x9d,0xd4,0x01,0x05,0x02,0x00]
 
@@ -2704,10 +2757,12 @@
 # GFX12: v_cmpx_neq_f64_e64 0xaf123456, -|vcc| clamp ; encoding: [0x7e,0x82,0xad,0xd4,0xff,0xd4,0x00,0x40,0x56,0x34,0x12,0xaf]
 
 0x7e,0x00,0x89,0xd4,0x01,0x05,0x02,0x00
-# GFX12: v_cmpx_nge_f16_e64 v1, v2               ; encoding: [0x7e,0x00,0x89,0xd4,0x01,0x05,0x02,0x00]
+# GFX12-REAL16: v_cmpx_nge_f16_e64 v1.l, v2.l           ; encoding: [0x7e,0x00,0x89,0xd4,0x01,0x05,0x02,0x00]
+# GFX12-FAKE16: v_cmpx_nge_f16_e64 v1, v2               ; encoding: [0x7e,0x00,0x89,0xd4,0x01,0x05,0x02,0x00]
 
 0x7e,0x00,0x89,0xd4,0xff,0xff,0x03,0x00
-# GFX12: v_cmpx_nge_f16_e64 v255, v255           ; encoding: [0x7e,0x00,0x89,0xd4,0xff,0xff,0x03,0x00]
+# GFX12-REAL16: v_cmpx_nge_f16_e64 v255.l, v255.l       ; encoding: [0x7e,0x00,0x89,0xd4,0xff,0xff,0x03,0x00]
+# GFX12-FAKE16: v_cmpx_nge_f16_e64 v255, v255           ; encoding: [0x7e,0x00,0x89,0xd4,0xff,0xff,0x03,0x00]
 
 0x7e,0x00,0x89,0xd4,0x01,0x04,0x00,0x00
 # GFX12: v_cmpx_nge_f16_e64 s1, s2               ; encoding: [0x7e,0x00,0x89,0xd4,0x01,0x04,0x00,0x00]
@@ -2748,6 +2803,15 @@
 0x7e,0x83,0x89,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00
 # GFX12: v_cmpx_nge_f16_e64 -|0xfe0b|, -|vcc_hi| clamp ; encoding: [0x7e,0x83,0x89,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00]
 
+0x7e,0x08,0x89,0xd4,0x01,0x05,0x02,0x00
+# GFX12-REAL16: v_cmpx_nge_f16_e64 v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x7e,0x08,0x89,0xd4,0x01,0x05,0x02,0x00]
+# GFX12-FAKE16: v_cmpx_nge_f16_e64 v1, v2               ; encoding: [0x7e,0x00,0x89,0xd4,0x01,0x05,0x02,0x00]
+
+0x7e,0x10,0x89,0xd4,0xff,0xff,0x03,0x00
+# GFX12-REAL16: v_cmpx_nge_f16_e64 v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x7e,0x10,0x89,0xd4,0xff,0xff,0x03,0x00]
+# GFX12-FAKE16: v_cmpx_nge_f16_e64 v255, v255           ; encoding: [0x7e,0x00,0x89,0xd4,0xff,0xff,0x03,0x00]
+
+
 0x7e,0x00,0x99,0xd4,0x01,0x05,0x02,0x00
 # GFX12: v_cmpx_nge_f32_e64 v1, v2               ; encoding: [0x7e,0x00,0x99,0xd4,0x01,0x05,0x02,0x00]
 
@@ -2830,10 +2894,12 @@
 # GFX12: v_cmpx_nge_f64_e64 0xaf123456, -|vcc| clamp ; encoding: [0x7e,0x82,0xa9,0xd4,0xff,0xd4,0x00,0x40,0x56,0x34,0x12,0xaf]
 
 0x7e,0x00,0x8b,0xd4,0x01,0x05,0x02,0x00
-# GFX12: v_cmpx_ngt_f16_e64 v1, v2               ; encoding: [0x7e,0x00,0x8b,0xd4,0x01,0x05,0x02,0x00]
+# GFX12-REAL16: v_cmpx_ngt_f16_e64 v1.l, v2.l           ; encoding: [0x7e,0x00,0x8b,0xd4,0x01,0x05,0x02,0x00]
+# GFX12-FAKE16: v_cmpx_ngt_f16_e64 v1, v2               ; encoding: [0x7e,0x00,0x8b,0xd4,0x01,0x05,0x02,0x00]
 
 0x7e,0x00,0x8b,0xd4,0xff,0xff,0x03,0x00
-# GFX12: v_cmpx_ngt_f16_e64 v255, v255           ; encoding: [0x7e,0x00,0x8b,0xd4,0xff,0xff,0x03,0x00]
+# GFX12-REAL16: v_cmpx_ngt_f16_e64 v255.l, v255.l       ; encoding: [0x7e,0x00,0x8b,0xd4,0xff,0xff,0x03,0x00]
+# GFX12-FAKE16: v_cmpx_ngt_f16_e64 v255, v255           ; encoding: [0x7e,0x00,0x8b,0xd4,0xff,0xff,0x03,0x00]
 
 0x7e,0x00,0x8b,0xd4,0x01,0x04,0x00,0x00
 # GFX12: v_cmpx_ngt_f16_e64 s1, s2               ; encoding: [0x7e,0x00,0x8b,0xd4,0x01,0x04,0x00,0x00]
@@ -2874,6 +2940,15 @@
 0x7e,0x83,0x8b,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00
 # GFX12: v_cmpx_ngt_f16_e64 -|0xfe0b|, -|vcc_hi| clamp ; encoding: [0x7e,0x83,0x8b,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00]
 
+0x7e,0x08,0x8b,0xd4,0x01,0x05,0x02,0x00
+# GFX12-REAL16: v_cmpx_ngt_f16_e64 v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x7e,0x08,0x8b,0xd4,0x01,0x05,0x02,0x00]
+# GFX12-FAKE16: v_cmpx_ngt_f16_e64 v1, v2               ; encoding: [0x7e,0x00,0x8b,0xd4,0x01,0x05,0x02,0x00]
+
+0x7e,0x10,0x8b,0xd4,0xff,0xff,0x03,0x00
+# GFX12-REAL16: v_cmpx_ngt_f16_e64 v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x7e,0x10,0x8b,0xd4,0xff,0xff,0x03,0x00]
+# GFX12-FAKE16: v_cmpx_ngt_f16_e64 v255, v255           ; encoding: [0x7e,0x00,0x8b,0xd4,0xff,0xff,0x03,0x00]
+
+
 0x7e,0x00,0x9b,0xd4,0x01,0x05,0x02,0x00
 # GFX12: v_cmpx_ngt_f32_e64 v1, v2               ; encoding: [0x7e,0x00,0x9b,0xd4,0x01,0x05,0x02,0x00]
 
@@ -2956,10 +3031,12 @@
 # GFX12: v_cmpx_ngt_f64_e64 0xaf123456, -|vcc| clamp ; encoding: [0x7e,0x82,0xab,0xd4,0xff,0xd4,0x00,0x40,0x56,0x34,0x12,0xaf]
 
 0x7e,0x00,0x8c,0xd4,0x01,0x05,0x02,0x00
-# GFX12: v_cmpx_nle_f16_e64 v1, v2               ; encoding: [0x7e,0x00,0x8c,0xd4,0x01,0x05,0x02,0x00]
+# GFX12-REAL16: v_cmpx_nle_f16_e64 v1.l, v2.l           ; encoding: [0x7e,0x00,0x8c,0xd4,0x01,0x05,0x02,0x00]
+# GFX12-FAKE16: v_cmpx_nle_f16_e64 v1, v2               ; encoding: [0x7e,0x00,0x8c,0xd4,0x01,0x05,0x02,0x00]
 
 0x7e,0x00,0x8c,0xd4,0xff,0xff,0x03,0x00
-# GFX12: v_cmpx_nle_f16_e64 v255, v255           ; encoding: [0x7e,0x00,0x8c,0xd4,0xff,0xff,0x03,0x00]
+# GFX12-REAL16: v_cmpx_nle_f16_e64 v255.l, v255.l       ; encoding: [0x7e,0x00,0x8c,0xd4,0xff,0xff,0x03,0x00]
+# GFX12-FAKE16: v_cmpx_nle_f16_e64 v255, v255           ; encoding: [0x7e,0x00,0x8c,0xd4,0xff,0xff,0x03,0x00]
 
 0x7e,0x00,0x8c,0xd4,0x01,0x04,0x00,0x00
 # GFX12: v_cmpx_nle_f16_e64 s1, s2               ; encoding: [0x7e,0x00,0x8c,0xd4,0x01,0x04,0x00,0x00]
@@ -3000,6 +3077,15 @@
 0x7e,0x83,0x8c,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00
 # GFX12: v_cmpx_nle_f16_e64 -|0xfe0b|, -|vcc_hi| clamp ; encoding: [0x7e,0x83,0x8c,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00]
 
+0x7e,0x08,0x8c,0xd4,0x01,0x05,0x02,0x00
+# GFX12-REAL16: v_cmpx_nle_f16_e64 v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x7e,0x08,0x8c,0xd4,0x01,0x05,0x02,0x00]
+# GFX12-FAKE16: v_cmpx_nle_f16_e64 v1, v2               ; encoding: [0x7e,0x00,0x8c,0xd4,0x01,0x05,0x02,0x00]
+
+0x7e,0x10,0x8c,0xd4,0xff,0xff,0x03,0x00
+# GFX12-REAL16: v_cmpx_nle_f16_e64 v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x7e,0x10,0x8c,0xd4,0xff,0xff,0x03,0x00]
+# GFX12-FAKE16: v_cmpx_nle_f16_e64 v255, v255           ; encoding: [0x7e,0x00,0x8c,0xd4,0xff,0xff,0x03,0x00]
+
+
 0x7e,0x00,0x9c,0xd4,0x01,0x05,0x02,0x00
 # GFX12: v_cmpx_nle_f32_e64 v1, v2               ; encoding: [0x7e,0x00,0x9c,0xd4,0x01,0x05,0x02,0x00]
 
@@ -3082,10 +3168,12 @@
 # GFX12: v_cmpx_nle_f64_e64 0xaf123456, -|vcc| clamp ; encoding: [0x7e,0x82,0xac,0xd4,0xff,0xd4,0x00,0x40,0x56,0x34,0x12,0xaf]
 
 0x7e,0x00,0x8a,0xd4,0x01,0x05,0x02,0x00
-# GFX12: v_cmpx_nlg_f16_e64 v1, v2               ; encoding: [0x7e,0x00,0x8a,0xd4,0x01,0x05,0x02,0x00]
+# GFX12-REAL16: v_cmpx_nlg_f16_e64 v1.l, v2.l           ; encoding: [0x7e,0x00,0x8a,0xd4,0x01,0x05,0x02,0x00]
+# GFX12-FAKE16: v_cmpx_nlg_f16_e64 v1, v2               ; encoding: [0x7e,0x00,0x8a,0xd4,0x01,0x05,0x02,0x00]
 
 0x7e,0x00,0x8a,0xd4,0xff,0xff,0x03,0x00
-# GFX12: v_cmpx_nlg_f16_e64 v255, v255           ; encoding: [0x7e,0x00,0x8a,0xd4,0xff,0xff,0x03,0x00]
+# GFX12-REAL16: v_cmpx_nlg_f16_e64 v255.l, v255.l       ; encoding: [0x7e,0x00,0x8a,0xd4,0xff,0xff,0x03,0x00]
+# GFX12-FAKE16: v_cmpx_nlg_f16_e64 v255, v255           ; encoding: [0x7e,0x00,0x8a,0xd4,0xff,0xff,0x03,0x00]
 
 0x7e,0x00,0x8a,0xd4,0x01,0x04,0x00,0x00
 # GFX12: v_cmpx_nlg_f16_e64 s1, s2               ; encoding: [0x7e,0x00,0x8a,0xd4,0x01,0x04,0x00,0x00]
@@ -3126,6 +3214,15 @@
 0x7e,0x83,0x8a,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00
 # GFX12: v_cmpx_nlg_f16_e64 -|0xfe0b|, -|vcc_hi| clamp ; encoding: [0x7e,0x83,0x8a,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00]
 
+0x7e,0x08,0x8a,0xd4,0x01,0x05,0x02,0x00
+# GFX12-REAL16: v_cmpx_nlg_f16_e64 v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x7e,0x08,0x8a,0xd4,0x01,0x05,0x02,0x00]
+# GFX12-FAKE16: v_cmpx_nlg_f16_e64 v1, v2               ; encoding: [0x7e,0x00,0x8a,0xd4,0x01,0x05,0x02,0x00]
+
+0x7e,0x10,0x8a,0xd4,0xff,0xff,0x03,0x00
+# GFX12-REAL16: v_cmpx_nlg_f16_e64 v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x7e,0x10,0x8a,0xd4,0xff,0xff,0x03,0x00]
+# GFX12-FAKE16: v_cmpx_nlg_f16_e64 v255, v255           ; encoding: [0x7e,0x00,0x8a,0xd4,0xff,0xff,0x03,0x00]
+
+
 0x7e,0x00,0x9a,0xd4,0x01,0x05,0x02,0x00
 # GFX12: v_cmpx_nlg_f32_e64 v1, v2               ; encoding: [0x7e,0x00,0x9a,0xd4,0x01,0x05,0x02,0x00]
 
@@ -3208,10 +3305,12 @@
 # GFX12: v_cmpx_nlg_f64_e64 0xaf123456, -|vcc| clamp ; encoding: [0x7e,0x82,0xaa,0xd4,0xff,0xd4,0x00,0x40,0x56,0x34,0x12,0xaf]
 
 0x7e,0x00,0x8e,0xd4,0x01,0x05,0x02,0x00
-# GFX12: v_cmpx_nlt_f16_e64 v1, v2               ; encoding: [0x7e,0x00,0x8e,0xd4,0x01,0x05,0x02,0x00]
+# GFX12-REAL16: v_cmpx_nlt_f16_e64 v1.l, v2.l           ; encoding: [0x7e,0x00,0x8e,0xd4,0x01,0x05,0x02,0x00]
+# GFX12-FAKE16: v_cmpx_nlt_f16_e64 v1, v2               ; encoding: [0x7e,0x00,0x8e,0xd4,0x01,0x05,0x02,0x00]
 
 0x7e,0x00,0x8e,0xd4,0xff,0xff,0x03,0x00
-# GFX12: v_cmpx_nlt_f16_e64 v255, v255           ; encoding: [0x7e,0x00,0x8e,0xd4,0xff,0xff,0x03,0x00]
+# GFX12-REAL16: v_cmpx_nlt_f16_e64 v255.l, v255.l       ; encoding: [0x7e,0x00,0x8e,0xd4,0xff,0xff,0x03,0x00]
+# GFX12-FAKE16: v_cmpx_nlt_f16_e64 v255, v255           ; encoding: [0x7e,0x00,0x8e,0xd4,0xff,0xff,0x03,0x00]
 
 0x7e,0x00,0x8e,0xd4,0x01,0x04,0x00,0x00
 # GFX12: v_cmpx_nlt_f16_e64 s1, s2               ; encoding: [0x7e,0x00,0x8e,0xd4,0x01,0x04,0x00,0x00]
@@ -3252,6 +3351,15 @@
 0x7e,0x83,0x8e,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00
 # GFX12: v_cmpx_nlt_f16_e64 -|0xfe0b|, -|vcc_hi| clamp ; encoding: [0x7e,0x83,0x8e,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00]
 
+0x7e,0x08,0x8e,0xd4,0x01,0x05,0x02,0x00
+# GFX12-REAL16: v_cmpx_nlt_f16_e64 v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x7e,0x08,0x8e,0xd4,0x01,0x05,0x02,0x00]
+# GFX12-FAKE16: v_cmpx_nlt_f16_e64 v1, v2               ; encoding: [0x7e,0x00,0x8e,0xd4,0x01,0x05,0x02,0x00]
+
+0x7e,0x10,0x8e,0xd4,0xff,0xff,0x03,0x00
+# GFX12-REAL16: v_cmpx_nlt_f16_e64 v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x7e,0x10,0x8e,0xd4,0xff,0xff,0x03,0x00]
+# GFX12-FAKE16: v_cmpx_nlt_f16_e64 v255, v255           ; encoding: [0x7e,0x00,0x8e,0xd4,0xff,0xff,0x03,0x00]
+
+
 0x7e,0x00,0x9e,0xd4,0x01,0x05,0x02,0x00
 # GFX12: v_cmpx_nlt_f32_e64 v1, v2               ; encoding: [0x7e,0x00,0x9e,0xd4,0x01,0x05,0x02,0x00]
 
@@ -3334,10 +3442,12 @@
 # GFX12: v_cmpx_nlt_f64_e64 0xaf123456, -|vcc| clamp ; encoding: [0x7e,0x82,0xae,0xd4,0xff,0xd4,0x00,0x40,0x56,0x34,0x12,0xaf]
 
 0x7e,0x00,0x87,0xd4,0x01,0x05,0x02,0x00
-# GFX12: v_cmpx_o_f16_e64 v1, v2                 ; encoding: [0x7e,0x00,0x87,0xd4,0x01,0x05,0x02,0x00]
+# GFX12-REAL16: v_cmpx_o_f16_e64 v1.l, v2.l             ; encoding: [0x7e,0x00,0x87,0xd4,0x01,0x05,0x02,0x00]
+# GFX12-FAKE16: v_cmpx_o_f16_e64 v1, v2                 ; encoding: [0x7e,0x00,0x87,0xd4,0x01,0x05,0x02,0x00]
 
 0x7e,0x00,0x87,0xd4,0xff,0xff,0x03,0x00
-# GFX12: v_cmpx_o_f16_e64 v255, v255             ; encoding: [0x7e,0x00,0x87,0xd4,0xff,0xff,0x03,0x00]
+# GFX12-REAL16: v_cmpx_o_f16_e64 v255.l, v255.l         ; encoding: [0x7e,0x00,0x87,0xd4,0xff,0xff,0x03,0x00]
+# GFX12-FAKE16: v_cmpx_o_f16_e64 v255, v255             ; encoding: [0x7e,0x00,0x87,0xd4,0xff,0xff,0x03,0x00]
 
 0x7e,0x00,0x87,0xd4,0x01,0x04,0x00,0x00
 # GFX12: v_cmpx_o_f16_e64 s1, s2                 ; encoding: [0x7e,0x00,0x87,0xd4,0x01,0x04,0x00,0x00]
@@ -3378,6 +3488,15 @@
 0x7e,0x83,0x87,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00
 # GFX12: v_cmpx_o_f16_e64 -|0xfe0b|, -|vcc_hi| clamp ; encoding: [0x7e,0x83,0x87,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00]
 
+0x7e,0x08,0x87,0xd4,0x01,0x05,0x02,0x00
+# GFX12-REAL16: v_cmpx_o_f16_e64 v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x7e,0x08,0x87,0xd4,0x01,0x05,0x02,0x00]
+# GFX12-FAKE16: v_cmpx_o_f16_e64 v1, v2                 ; encoding: [0x7e,0x00,0x87,0xd4,0x01,0x05,0x02,0x00]
+
+0x7e,0x10,0x87,0xd4,0xff,0xff,0x03,0x00
+# GFX12-REAL16: v_cmpx_o_f16_e64 v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x7e,0x10,0x87,0xd4,0xff,0xff,0x03,0x00]
+# GFX12-FAKE16: v_cmpx_o_f16_e64 v255, v255             ; encoding: [0x7e,0x00,0x87,0xd4,0xff,0xff,0x03,0x00]
+
+
 0x7e,0x00,0x97,0xd4,0x01,0x05,0x02,0x00
 # GFX12: v_cmpx_o_f32_e64 v1, v2                 ; encoding: [0x7e,0x00,0x97,0xd4,0x01,0x05,0x02,0x00]
 
@@ -3460,10 +3579,12 @@
 # GFX12: v_cmpx_o_f64_e64 0xaf123456, -|vcc| clamp ; encoding: [0x7e,0x82,0xa7,0xd4,0xff,0xd4,0x00,0x40,0x56,0x34,0x12,0xaf]
 
 0x7e,0x00,0x88,0xd4,0x01,0x05,0x02,0x00
-# GFX12: v_cmpx_u_f16_e64 v1, v2                 ; encoding: [0x7e,0x00,0x88,0xd4,0x01,0x05,0x02,0x00]
+# GFX12-REAL16: v_cmpx_u_f16_e64 v1.l, v2.l             ; encoding: [0x7e,0x00,0x88,0xd4,0x01,0x05,0x02,0x00]
+# GFX12-FAKE16: v_cmpx_u_f16_e64 v1, v2                 ; encoding: [0x7e,0x00,0x88,0xd4,0x01,0x05,0x02,0x00]
 
 0x7e,0x00,0x88,0xd4,0xff,0xff,0x03,0x00
-# GFX12: v_cmpx_u_f16_e64 v255, v255             ; encoding: [0x7e,0x00,0x88,0xd4,0xff,0xff,0x03,0x00]
+# GFX12-REAL16: v_cmpx_u_f16_e64 v255.l, v255.l         ; encoding: [0x7e,0x00,0x88,0xd4,0xff,0xff,0x03,0x00]
+# GFX12-FAKE16: v_cmpx_u_f16_e64 v255, v255             ; encoding: [0x7e,0x00,0x88,0xd4,0xff,0xff,0x03,0x00]
 
 0x7e,0x00,0x88,0xd4,0x01,0x04,0x00,0x00
 # GFX12: v_cmpx_u_f16_e64 s1, s2                 ; encoding: [0x7e,0x00,0x88,0xd4,0x01,0x04,0x00,0x00]
@@ -3504,6 +3625,14 @@
 0x7e,0x83,0x88,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00
 # GFX12: v_cmpx_u_f16_e64 -|0xfe0b|, -|vcc_hi| clamp ; encoding: [0x7e,0x83,0x88,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00]
 
+0x7e,0x08,0x88,0xd4,0x01,0x05,0x02,0x00
+# GFX12-REAL16: v_cmpx_u_f16_e64 v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x7e,0x08,0x88,0xd4,0x01,0x05,0x02,0x00]
+# GFX12-FAKE16: v_cmpx_u_f16_e64 v1, v2                 ; encoding: [0x7e,0x00,0x88,0xd4,0x01,0x05,0x02,0x00]
+
+0x7e,0x10,0x88,0xd4,0xff,0xff,0x03,0x00
+# GFX12-REAL16: v_cmpx_u_f16_e64 v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x7e,0x10,0x88,0xd4,0xff,0xff,0x03,0x00]
+# GFX12-FAKE16: v_cmpx_u_f16_e64 v255, v255             ; encoding: [0x7e,0x00,0x88,0xd4,0xff,0xff,0x03,0x00]
+
 0x7e,0x00,0x98,0xd4,0x01,0x05,0x02,0x00
 # GFX12: v_cmpx_u_f32_e64 v1, v2                 ; encoding: [0x7e,0x00,0x98,0xd4,0x01,0x05,0x02,0x00]
 
diff --git a/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vop3cx_dpp16.txt b/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vop3cx_dpp16.txt
index 2dc231a4220f1..f8ce4fafc0252 100644
--- a/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vop3cx_dpp16.txt
+++ b/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vop3cx_dpp16.txt
@@ -171,49 +171,123 @@
 # GFX12: v_cmpx_class_f32_e64_dpp -|v255|, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0x7e,0x01,0xfe,0xd4,0xfa,0xfe,0x03,0x20,0xff,0x6f,0x0d,0x30]
 
 0x7e,0x00,0x82,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff
-# GFX12: v_cmpx_eq_f16_e64_dpp v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x82,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+# GFX12-REAL16: v_cmpx_eq_f16_e64_dpp v1.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x82,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+# GFX12-FAKE16: v_cmpx_eq_f16_e64_dpp v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x82,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
 
 0x7e,0x00,0x82,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff
-# GFX12: v_cmpx_eq_f16_e64_dpp v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x82,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+# GFX12-REAL16: v_cmpx_eq_f16_e64_dpp v1.l, v2.l quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x82,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+# GFX12-FAKE16: v_cmpx_eq_f16_e64_dpp v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x82,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
 
 0x7e,0x00,0x82,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff
-# GFX12: v_cmpx_eq_f16_e64_dpp v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x82,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+# GFX12-REAL16: v_cmpx_eq_f16_e64_dpp v1.l, v2.l row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x82,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+# GFX12-FAKE16: v_cmpx_eq_f16_e64_dpp v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x82,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
 
 0x7e,0x00,0x82,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff
-# GFX12: v_cmpx_eq_f16_e64_dpp v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x82,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+# GFX12-REAL16: v_cmpx_eq_f16_e64_dpp v1.l, v2.l row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x82,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+# GFX12-FAKE16: v_cmpx_eq_f16_e64_dpp v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x82,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
 
 0x7e,0x00,0x82,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff
-# GFX12: v_cmpx_eq_f16_e64_dpp v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x82,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+# GFX12-REAL16: v_cmpx_eq_f16_e64_dpp v1.l, v2.l row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x82,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+# GFX12-FAKE16: v_cmpx_eq_f16_e64_dpp v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x82,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
 
 0x7e,0x00,0x82,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff
-# GFX12: v_cmpx_eq_f16_e64_dpp v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x82,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+# GFX12-REAL16: v_cmpx_eq_f16_e64_dpp v1.l, v2.l row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x82,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+# GFX12-FAKE16: v_cmpx_eq_f16_e64_dpp v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x82,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
 
 0x7e,0x00,0x82,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff
-# GFX12: v_cmpx_eq_f16_e64_dpp v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x82,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+# GFX12-REAL16: v_cmpx_eq_f16_e64_dpp v1.l, v2.l row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x82,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+# GFX12-FAKE16: v_cmpx_eq_f16_e64_dpp v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x82,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
 
 0x7e,0x00,0x82,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff
-# GFX12: v_cmpx_eq_f16_e64_dpp v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x82,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+# GFX12-REAL16: v_cmpx_eq_f16_e64_dpp v1.l, v2.l row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x82,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+# GFX12-FAKE16: v_cmpx_eq_f16_e64_dpp v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x82,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
 
 0x7e,0x00,0x82,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff
-# GFX12: v_cmpx_eq_f16_e64_dpp v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x82,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+# GFX12-REAL16: v_cmpx_eq_f16_e64_dpp v1.l, v2.l row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x82,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+# GFX12-FAKE16: v_cmpx_eq_f16_e64_dpp v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x82,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
 
 0x7e,0x00,0x82,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff
-# GFX12: v_cmpx_eq_f16_e64_dpp v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x82,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+# GFX12-REAL16: v_cmpx_eq_f16_e64_dpp v1.l, v2.l row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x82,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+# GFX12-FAKE16: v_cmpx_eq_f16_e64_dpp v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x82,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
 
 0x7e,0x00,0x82,0xd4,0xfa,0x06,0x00,0x00,0x01,0x2f,0x01,0xff
-# GFX12: v_cmpx_eq_f16_e64_dpp v1, s3 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x82,0xd4,0xfa,0x06,0x00,0x00,0x01,0x2f,0x01,0xff]
+# GFX12-REAL16: v_cmpx_eq_f16_e64_dpp v1.l, s3 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x82,0xd4,0xfa,0x06,0x00,0x00,0x01,0x2f,0x01,0xff]
+# GFX12-FAKE16: v_cmpx_eq_f16_e64_dpp v1, s3 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x82,0xd4,0xfa,0x06,0x00,0x00,0x01,0x2f,0x01,0xff]
 
 0x7e,0x00,0x82,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff
-# GFX12: v_cmpx_eq_f16_e64_dpp v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x82,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+# GFX12-REAL16: v_cmpx_eq_f16_e64_dpp v1.l, v2.l row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x82,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+# GFX12-FAKE16: v_cmpx_eq_f16_e64_dpp v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x82,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 
 0x7e,0x01,0x82,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01
-# GFX12: v_cmpx_eq_f16_e64_dpp |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x01,0x82,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+# GFX12-REAL16: v_cmpx_eq_f16_e64_dpp |v1.l|, -v2.l row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x01,0x82,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+# GFX12-FAKE16: v_cmpx_eq_f16_e64_dpp |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x01,0x82,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
 
 0x7e,0x02,0x82,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x01,0x13
-# GFX12: v_cmpx_eq_f16_e64_dpp -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x7e,0x02,0x82,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x01,0x13]
+# GFX12-REAL16: v_cmpx_eq_f16_e64_dpp -v1.l, |v2.l| row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x7e,0x02,0x82,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x01,0x13]
+# GFX12-FAKE16: v_cmpx_eq_f16_e64_dpp -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x7e,0x02,0x82,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x01,0x13]
 
 0x7e,0x83,0x82,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30
-# GFX12: v_cmpx_eq_f16_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0x7e,0x83,0x82,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30]
+# GFX12-REAL16: v_cmpx_eq_f16_e64_dpp -|v255.l|, -|v255.l| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0x7e,0x83,0x82,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30]
+# GFX12-FAKE16: v_cmpx_eq_f16_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0x7e,0x83,0x82,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30]
+
+0x7e,0x08,0x82,0xd4,0x01,0x05,0x02,0x00
+# GFX12-REAL16: v_cmpx_eq_f16_e64 v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x7e,0x08,0x82,0xd4,0x01,0x05,0x02,0x00]
+# GFX12-FAKE16: v_cmpx_eq_f16_e64 v1, v2                ; encoding: [0x7e,0x00,0x82,0xd4,0x01,0x05,0x02,0x00]
+
+0x7e,0x10,0x82,0xd4,0xff,0xff,0x03,0x00
+# GFX12-REAL16: v_cmpx_eq_f16_e64 v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x7e,0x10,0x82,0xd4,0xff,0xff,0x03,0x00]
+# GFX12-FAKE16: v_cmpx_eq_f16_e64 v255, v255            ; encoding: [0x7e,0x00,0x82,0xd4,0xff,0xff,0x03,0x00]
+
+0x7e,0x00,0x82,0xd4,0x01,0x04,0x00,0x00
+# GFX12: v_cmpx_eq_f16_e64 s1, s2                ; encoding: [0x7e,0x00,0x82,0xd4,0x01,0x04,0x00,0x00]
+
+0x7e,0x00,0x82,0xd4,0x69,0xd2,0x00,0x00
+# GFX12: v_cmpx_eq_f16_e64 s105, s105            ; encoding: [0x7e,0x00,0x82,0xd4,0x69,0xd2,0x00,0x00]
+
+0x7e,0x00,0x82,0xd4,0x6a,0xf6,0x00,0x00
+# GFX12: v_cmpx_eq_f16_e64 vcc_lo, ttmp15        ; encoding: [0x7e,0x00,0x82,0xd4,0x6a,0xf6,0x00,0x00]
+
+0x7e,0x00,0x82,0xd4,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00
+# GFX12: v_cmpx_eq_f16_e64 vcc_hi, 0xfe0b        ; encoding: [0x7e,0x00,0x82,0xd4,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
+
+0x7e,0x00,0x82,0xd4,0x7b,0xfa,0x01,0x00
+# GFX12: v_cmpx_eq_f16_e64 ttmp15, src_scc       ; encoding: [0x7e,0x00,0x82,0xd4,0x7b,0xfa,0x01,0x00]
+
+0x7e,0x00,0x82,0xd4,0x7d,0xe0,0x01,0x00
+# GFX12: v_cmpx_eq_f16_e64 m0, 0.5               ; encoding: [0x7e,0x00,0x82,0xd4,0x7d,0xe0,0x01,0x00]
+
+0x7e,0x00,0x82,0xd4,0x7e,0x82,0x01,0x00
+# GFX12: v_cmpx_eq_f16_e64 exec_lo, -1           ; encoding: [0x7e,0x00,0x82,0xd4,0x7e,0x82,0x01,0x00]
+
+0x7e,0x01,0x82,0xd4,0x7f,0xf8,0x00,0x00
+# GFX12: v_cmpx_eq_f16_e64 |exec_hi|, null       ; encoding: [0x7e,0x01,0x82,0xd4,0x7f,0xf8,0x00,0x00]
+
+0x7e,0x00,0x82,0xd4,0x7c,0xfc,0x00,0x00
+# GFX12: v_cmpx_eq_f16_e64 null, exec_lo         ; encoding: [0x7e,0x00,0x82,0xd4,0x7c,0xfc,0x00,0x00]
+
+0x7e,0x00,0x82,0xd4,0xc1,0xfe,0x00,0x00
+# GFX12: v_cmpx_eq_f16_e64 -1, exec_hi           ; encoding: [0x7e,0x00,0x82,0xd4,0xc1,0xfe,0x00,0x00]
+
+0x7e,0x00,0x82,0xd4,0xf0,0xfa,0x00,0x40
+# GFX12: v_cmpx_eq_f16_e64 0.5, -m0              ; encoding: [0x7e,0x00,0x82,0xd4,0xf0,0xfa,0x00,0x40]
+
+0x7e,0x02,0x82,0xd4,0xfd,0xd4,0x00,0x20
+# GFX12: v_cmpx_eq_f16_e64 -src_scc, |vcc_lo|    ; encoding: [0x7e,0x02,0x82,0xd4,0xfd,0xd4,0x00,0x20]
+
+0x7e,0x83,0x82,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00
+# GFX12: v_cmpx_eq_f16_e64 -|0xfe0b|, -|vcc_hi| clamp ; encoding: [0x7e,0x83,0x82,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00]
+
+0x7e,0x19,0x82,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01
+# GFX12-REAL16: v_cmpx_eq_f16_e64_dpp |v1.h|, -v2.h op_sel:[1,1] row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x19,0x82,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+# GFX12-FAKE16: v_cmpx_eq_f16_e64_dpp |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x01,0x82,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+
+0x7e,0x0a,0x82,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x01,0x13
+# GFX12-REAL16: v_cmpx_eq_f16_e64_dpp -v1.h, |v2.l| op_sel:[1,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x7e,0x0a,0x82,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x01,0x13]
+# GFX12-FAKE16: v_cmpx_eq_f16_e64_dpp -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x7e,0x02,0x82,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x01,0x13]
+
+0x7e,0x93,0x82,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30
+# GFX12-REAL16: v_cmpx_eq_f16_e64_dpp -|v255.l|, -|v255.h| op_sel:[0,1] clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0x7e,0x93,0x82,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30]
+# GFX12-FAKE16: v_cmpx_eq_f16_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0x7e,0x83,0x82,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30]
 
 0x7e,0x00,0x92,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff
 # GFX12: v_cmpx_eq_f32_e64_dpp v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x92,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
@@ -367,17 +441,14 @@
 0x7e,0x00,0xb2,0xd4,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00
 # GFX12: v_cmpx_eq_i16_e64 0xfe0b, vcc_hi        ; encoding: [0x7e,0x00,0xb2,0xd4,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
 
-# GFX11: v_cmpx_eq_i16_e64_dpp v1.h, v2.h op_sel:[1,1] row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x18,0xb2,0xd4,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
 0x7e,0x18,0xb2,0xd4,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01
 # GFX12-REAL16: v_cmpx_eq_i16_e64_dpp v1.h, v2.h op_sel:[1,1] row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x18,0xb2,0xd4,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
 # GFX12-FAKE16: v_cmpx_eq_i16_e64_dpp v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x00,0xb2,0xd4,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
 
-# GFX11: v_cmpx_eq_i16_e64_dpp v1.h, v2.l op_sel:[1,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x7e,0x08,0xb2,0xd4,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
 0x7e,0x08,0xb2,0xd4,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13
 # GFX12-REAL16: v_cmpx_eq_i16_e64_dpp v1.h, v2.l op_sel:[1,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x7e,0x08,0xb2,0xd4,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
 # GFX12-FAKE16: v_cmpx_eq_i16_e64_dpp v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x7e,0x00,0xb2,0xd4,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
 
-# GFX11: v_cmpx_eq_i16_e64_dpp v255.l, v255.h op_sel:[0,1] row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0x7e,0x10,0xb2,0xd4,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 0x7e,0x10,0xb2,0xd4,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30
 # GFX12-REAL16: v_cmpx_eq_i16_e64_dpp v255.l, v255.h op_sel:[0,1] row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0x7e,0x10,0xb2,0xd4,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 # GFX12-FAKE16: v_cmpx_eq_i16_e64_dpp v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0x7e,0x00,0xb2,0xd4,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
@@ -537,17 +608,14 @@
 0x7e,0x00,0xba,0xd4,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00
 # GFX12: v_cmpx_eq_u16_e64 0xfe0b, vcc_hi        ; encoding: [0x7e,0x00,0xba,0xd4,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
 
-# GFX11: v_cmpx_eq_u16_e64_dpp v1.h, v2.h op_sel:[1,1] row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x18,0xba,0xd4,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
 0x7e,0x18,0xba,0xd4,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01
 # GFX12-REAL16: v_cmpx_eq_u16_e64_dpp v1.h, v2.h op_sel:[1,1] row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x18,0xba,0xd4,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
 # GFX12-FAKE16: v_cmpx_eq_u16_e64_dpp v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x00,0xba,0xd4,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
 
-# GFX11: v_cmpx_eq_u16_e64_dpp v1.h, v2.l op_sel:[1,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x7e,0x08,0xba,0xd4,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
 0x7e,0x08,0xba,0xd4,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13
 # GFX12-REAL16: v_cmpx_eq_u16_e64_dpp v1.h, v2.l op_sel:[1,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x7e,0x08,0xba,0xd4,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
 # GFX12-FAKE16: v_cmpx_eq_u16_e64_dpp v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x7e,0x00,0xba,0xd4,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
 
-# GFX11: v_cmpx_eq_u16_e64_dpp v255.l, v255.h op_sel:[0,1] row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0x7e,0x10,0xba,0xd4,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 0x7e,0x10,0xba,0xd4,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30
 # GFX12-REAL16: v_cmpx_eq_u16_e64_dpp v255.l, v255.h op_sel:[0,1] row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0x7e,0x10,0xba,0xd4,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 # GFX12-FAKE16: v_cmpx_eq_u16_e64_dpp v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0x7e,0x00,0xba,0xd4,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
@@ -598,49 +666,123 @@
 # GFX12: v_cmpx_eq_u32_e64_dpp v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0x7e,0x00,0xca,0xd4,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 
 0x7e,0x00,0x86,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff
-# GFX12: v_cmpx_ge_f16_e64_dpp v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x86,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+# GFX12-REAL16: v_cmpx_ge_f16_e64_dpp v1.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x86,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+# GFX12-FAKE16: v_cmpx_ge_f16_e64_dpp v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x86,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
 
 0x7e,0x00,0x86,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff
-# GFX12: v_cmpx_ge_f16_e64_dpp v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x86,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+# GFX12-REAL16: v_cmpx_ge_f16_e64_dpp v1.l, v2.l quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x86,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+# GFX12-FAKE16: v_cmpx_ge_f16_e64_dpp v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x86,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
 
 0x7e,0x00,0x86,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff
-# GFX12: v_cmpx_ge_f16_e64_dpp v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x86,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+# GFX12-REAL16: v_cmpx_ge_f16_e64_dpp v1.l, v2.l row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x86,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+# GFX12-FAKE16: v_cmpx_ge_f16_e64_dpp v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x86,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
 
 0x7e,0x00,0x86,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff
-# GFX12: v_cmpx_ge_f16_e64_dpp v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x86,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+# GFX12-REAL16: v_cmpx_ge_f16_e64_dpp v1.l, v2.l row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x86,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+# GFX12-FAKE16: v_cmpx_ge_f16_e64_dpp v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x86,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
 
 0x7e,0x00,0x86,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff
-# GFX12: v_cmpx_ge_f16_e64_dpp v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x86,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+# GFX12-REAL16: v_cmpx_ge_f16_e64_dpp v1.l, v2.l row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x86,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+# GFX12-FAKE16: v_cmpx_ge_f16_e64_dpp v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x86,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
 
 0x7e,0x00,0x86,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff
-# GFX12: v_cmpx_ge_f16_e64_dpp v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x86,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+# GFX12-REAL16: v_cmpx_ge_f16_e64_dpp v1.l, v2.l row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x86,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+# GFX12-FAKE16: v_cmpx_ge_f16_e64_dpp v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x86,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
 
 0x7e,0x00,0x86,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff
-# GFX12: v_cmpx_ge_f16_e64_dpp v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x86,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+# GFX12-REAL16: v_cmpx_ge_f16_e64_dpp v1.l, v2.l row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x86,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+# GFX12-FAKE16: v_cmpx_ge_f16_e64_dpp v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x86,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
 
 0x7e,0x00,0x86,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff
-# GFX12: v_cmpx_ge_f16_e64_dpp v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x86,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+# GFX12-REAL16: v_cmpx_ge_f16_e64_dpp v1.l, v2.l row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x86,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+# GFX12-FAKE16: v_cmpx_ge_f16_e64_dpp v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x86,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
 
 0x7e,0x00,0x86,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff
-# GFX12: v_cmpx_ge_f16_e64_dpp v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x86,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+# GFX12-REAL16: v_cmpx_ge_f16_e64_dpp v1.l, v2.l row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x86,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+# GFX12-FAKE16: v_cmpx_ge_f16_e64_dpp v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x86,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
 
 0x7e,0x00,0x86,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff
-# GFX12: v_cmpx_ge_f16_e64_dpp v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x86,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+# GFX12-REAL16: v_cmpx_ge_f16_e64_dpp v1.l, v2.l row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x86,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+# GFX12-FAKE16: v_cmpx_ge_f16_e64_dpp v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x86,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
 
 0x7e,0x00,0x86,0xd4,0xfa,0x06,0x00,0x00,0x01,0x2f,0x01,0xff
-# GFX12: v_cmpx_ge_f16_e64_dpp v1, s3 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x86,0xd4,0xfa,0x06,0x00,0x00,0x01,0x2f,0x01,0xff]
+# GFX12-REAL16: v_cmpx_ge_f16_e64_dpp v1.l, s3 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x86,0xd4,0xfa,0x06,0x00,0x00,0x01,0x2f,0x01,0xff]
+# GFX12-FAKE16: v_cmpx_ge_f16_e64_dpp v1, s3 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x86,0xd4,0xfa,0x06,0x00,0x00,0x01,0x2f,0x01,0xff]
 
 0x7e,0x00,0x86,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff
-# GFX12: v_cmpx_ge_f16_e64_dpp v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x86,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+# GFX12-REAL16: v_cmpx_ge_f16_e64_dpp v1.l, v2.l row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x86,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+# GFX12-FAKE16: v_cmpx_ge_f16_e64_dpp v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x86,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 
 0x7e,0x01,0x86,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01
-# GFX12: v_cmpx_ge_f16_e64_dpp |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x01,0x86,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+# GFX12-REAL16: v_cmpx_ge_f16_e64_dpp |v1.l|, -v2.l row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x01,0x86,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+# GFX12-FAKE16: v_cmpx_ge_f16_e64_dpp |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x01,0x86,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
 
 0x7e,0x02,0x86,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x01,0x13
-# GFX12: v_cmpx_ge_f16_e64_dpp -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x7e,0x02,0x86,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x01,0x13]
+# GFX12-REAL16: v_cmpx_ge_f16_e64_dpp -v1.l, |v2.l| row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x7e,0x02,0x86,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x01,0x13]
+# GFX12-FAKE16: v_cmpx_ge_f16_e64_dpp -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x7e,0x02,0x86,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x01,0x13]
 
 0x7e,0x83,0x86,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30
-# GFX12: v_cmpx_ge_f16_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0x7e,0x83,0x86,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30]
+# GFX12-REAL16: v_cmpx_ge_f16_e64_dpp -|v255.l|, -|v255.l| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0x7e,0x83,0x86,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30]
+# GFX12-FAKE16: v_cmpx_ge_f16_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0x7e,0x83,0x86,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30]
+
+0x7e,0x08,0x86,0xd4,0x01,0x05,0x02,0x00
+# GFX12-REAL16: v_cmpx_ge_f16_e64 v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x7e,0x08,0x86,0xd4,0x01,0x05,0x02,0x00]
+# GFX12-FAKE16: v_cmpx_ge_f16_e64 v1, v2                ; encoding: [0x7e,0x00,0x86,0xd4,0x01,0x05,0x02,0x00]
+
+0x7e,0x10,0x86,0xd4,0xff,0xff,0x03,0x00
+# GFX12-REAL16: v_cmpx_ge_f16_e64 v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x7e,0x10,0x86,0xd4,0xff,0xff,0x03,0x00]
+# GFX12-FAKE16: v_cmpx_ge_f16_e64 v255, v255            ; encoding: [0x7e,0x00,0x86,0xd4,0xff,0xff,0x03,0x00]
+
+0x7e,0x00,0x86,0xd4,0x01,0x04,0x00,0x00
+# GFX12: v_cmpx_ge_f16_e64 s1, s2                ; encoding: [0x7e,0x00,0x86,0xd4,0x01,0x04,0x00,0x00]
+
+0x7e,0x00,0x86,0xd4,0x69,0xd2,0x00,0x00
+# GFX12: v_cmpx_ge_f16_e64 s105, s105            ; encoding: [0x7e,0x00,0x86,0xd4,0x69,0xd2,0x00,0x00]
+
+0x7e,0x00,0x86,0xd4,0x6a,0xf6,0x00,0x00
+# GFX12: v_cmpx_ge_f16_e64 vcc_lo, ttmp15        ; encoding: [0x7e,0x00,0x86,0xd4,0x6a,0xf6,0x00,0x00]
+
+0x7e,0x00,0x86,0xd4,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00
+# GFX12: v_cmpx_ge_f16_e64 vcc_hi, 0xfe0b        ; encoding: [0x7e,0x00,0x86,0xd4,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
+
+0x7e,0x00,0x86,0xd4,0x7b,0xfa,0x01,0x00
+# GFX12: v_cmpx_ge_f16_e64 ttmp15, src_scc       ; encoding: [0x7e,0x00,0x86,0xd4,0x7b,0xfa,0x01,0x00]
+
+0x7e,0x00,0x86,0xd4,0x7d,0xe0,0x01,0x00
+# GFX12: v_cmpx_ge_f16_e64 m0, 0.5               ; encoding: [0x7e,0x00,0x86,0xd4,0x7d,0xe0,0x01,0x00]
+
+0x7e,0x00,0x86,0xd4,0x7e,0x82,0x01,0x00
+# GFX12: v_cmpx_ge_f16_e64 exec_lo, -1           ; encoding: [0x7e,0x00,0x86,0xd4,0x7e,0x82,0x01,0x00]
+
+0x7e,0x01,0x86,0xd4,0x7f,0xf8,0x00,0x00
+# GFX12: v_cmpx_ge_f16_e64 |exec_hi|, null       ; encoding: [0x7e,0x01,0x86,0xd4,0x7f,0xf8,0x00,0x00]
+
+0x7e,0x00,0x86,0xd4,0x7c,0xfc,0x00,0x00
+# GFX12: v_cmpx_ge_f16_e64 null, exec_lo         ; encoding: [0x7e,0x00,0x86,0xd4,0x7c,0xfc,0x00,0x00]
+
+0x7e,0x00,0x86,0xd4,0xc1,0xfe,0x00,0x00
+# GFX12: v_cmpx_ge_f16_e64 -1, exec_hi           ; encoding: [0x7e,0x00,0x86,0xd4,0xc1,0xfe,0x00,0x00]
+
+0x7e,0x00,0x86,0xd4,0xf0,0xfa,0x00,0x40
+# GFX12: v_cmpx_ge_f16_e64 0.5, -m0              ; encoding: [0x7e,0x00,0x86,0xd4,0xf0,0xfa,0x00,0x40]
+
+0x7e,0x02,0x86,0xd4,0xfd,0xd4,0x00,0x20
+# GFX12: v_cmpx_ge_f16_e64 -src_scc, |vcc_lo|    ; encoding: [0x7e,0x02,0x86,0xd4,0xfd,0xd4,0x00,0x20]
+
+0x7e,0x83,0x86,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00
+# GFX12: v_cmpx_ge_f16_e64 -|0xfe0b|, -|vcc_hi| clamp ; encoding: [0x7e,0x83,0x86,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00]
+
+0x7e,0x19,0x86,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01
+# GFX12-REAL16: v_cmpx_ge_f16_e64_dpp |v1.h|, -v2.h op_sel:[1,1] row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x19,0x86,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+# GFX12-FAKE16: v_cmpx_ge_f16_e64_dpp |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x01,0x86,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+
+0x7e,0x0a,0x86,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x01,0x13
+# GFX12-REAL16: v_cmpx_ge_f16_e64_dpp -v1.h, |v2.l| op_sel:[1,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x7e,0x0a,0x86,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x01,0x13]
+# GFX12-FAKE16: v_cmpx_ge_f16_e64_dpp -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x7e,0x02,0x86,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x01,0x13]
+
+0x7e,0x93,0x86,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30
+# GFX12-REAL16: v_cmpx_ge_f16_e64_dpp -|v255.l|, -|v255.h| op_sel:[0,1] clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0x7e,0x93,0x86,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30]
+# GFX12-FAKE16: v_cmpx_ge_f16_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0x7e,0x83,0x86,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30]
 
 0x7e,0x00,0x96,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff
 # GFX12: v_cmpx_ge_f32_e64_dpp v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x96,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
@@ -794,17 +936,14 @@
 0x7e,0x00,0xb6,0xd4,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00
 # GFX12: v_cmpx_ge_i16_e64 0xfe0b, vcc_hi        ; encoding: [0x7e,0x00,0xb6,0xd4,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
 
-# GFX11: v_cmpx_ge_i16_e64_dpp v1.h, v2.h op_sel:[1,1] row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x18,0xb6,0xd4,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
 0x7e,0x18,0xb6,0xd4,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01
 # GFX12-REAL16: v_cmpx_ge_i16_e64_dpp v1.h, v2.h op_sel:[1,1] row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x18,0xb6,0xd4,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
 # GFX12-FAKE16: v_cmpx_ge_i16_e64_dpp v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x00,0xb6,0xd4,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
 
-# GFX11: v_cmpx_ge_i16_e64_dpp v1.h, v2.l op_sel:[1,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x7e,0x08,0xb6,0xd4,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
 0x7e,0x08,0xb6,0xd4,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13
 # GFX12-REAL16: v_cmpx_ge_i16_e64_dpp v1.h, v2.l op_sel:[1,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x7e,0x08,0xb6,0xd4,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
 # GFX12-FAKE16: v_cmpx_ge_i16_e64_dpp v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x7e,0x00,0xb6,0xd4,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
 
-# GFX11: v_cmpx_ge_i16_e64_dpp v255.l, v255.h op_sel:[0,1] row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0x7e,0x10,0xb6,0xd4,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 0x7e,0x10,0xb6,0xd4,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30
 # GFX12-REAL16: v_cmpx_ge_i16_e64_dpp v255.l, v255.h op_sel:[0,1] row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0x7e,0x10,0xb6,0xd4,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 # GFX12-FAKE16: v_cmpx_ge_i16_e64_dpp v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0x7e,0x00,0xb6,0xd4,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
@@ -961,17 +1100,14 @@
 0x7e,0x00,0xbe,0xd4,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00
 # GFX12: v_cmpx_ge_u16_e64 0xfe0b, vcc_hi        ; encoding: [0x7e,0x00,0xbe,0xd4,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
 
-# GFX11: v_cmpx_ge_u16_e64_dpp v1.h, v2.h op_sel:[1,1] row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x18,0xbe,0xd4,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
 0x7e,0x18,0xbe,0xd4,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01
 # GFX12-REAL16: v_cmpx_ge_u16_e64_dpp v1.h, v2.h op_sel:[1,1] row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x18,0xbe,0xd4,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
 # GFX12-FAKE16: v_cmpx_ge_u16_e64_dpp v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x00,0xbe,0xd4,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
 
-# GFX11: v_cmpx_ge_u16_e64_dpp v1.h, v2.l op_sel:[1,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x7e,0x08,0xbe,0xd4,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
 0x7e,0x08,0xbe,0xd4,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13
 # GFX12-REAL16: v_cmpx_ge_u16_e64_dpp v1.h, v2.l op_sel:[1,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x7e,0x08,0xbe,0xd4,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
 # GFX12-FAKE16: v_cmpx_ge_u16_e64_dpp v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x7e,0x00,0xbe,0xd4,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
 
-# GFX11: v_cmpx_ge_u16_e64_dpp v255.l, v255.h op_sel:[0,1] row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0x7e,0x10,0xbe,0xd4,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 0x7e,0x10,0xbe,0xd4,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30
 # GFX12-REAL16: v_cmpx_ge_u16_e64_dpp v255.l, v255.h op_sel:[0,1] row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0x7e,0x10,0xbe,0xd4,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 # GFX12-FAKE16: v_cmpx_ge_u16_e64_dpp v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0x7e,0x00,0xbe,0xd4,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
@@ -1022,49 +1158,123 @@
 # GFX12: v_cmpx_ge_u32_e64_dpp v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0x7e,0x00,0xce,0xd4,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 
 0x7e,0x00,0x84,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff
-# GFX12: v_cmpx_gt_f16_e64_dpp v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x84,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+# GFX12-REAL16: v_cmpx_gt_f16_e64_dpp v1.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x84,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+# GFX12-FAKE16: v_cmpx_gt_f16_e64_dpp v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x84,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
 
 0x7e,0x00,0x84,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff
-# GFX12: v_cmpx_gt_f16_e64_dpp v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x84,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+# GFX12-REAL16: v_cmpx_gt_f16_e64_dpp v1.l, v2.l quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x84,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+# GFX12-FAKE16: v_cmpx_gt_f16_e64_dpp v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x84,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
 
 0x7e,0x00,0x84,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff
-# GFX12: v_cmpx_gt_f16_e64_dpp v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x84,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+# GFX12-REAL16: v_cmpx_gt_f16_e64_dpp v1.l, v2.l row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x84,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+# GFX12-FAKE16: v_cmpx_gt_f16_e64_dpp v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x84,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
 
 0x7e,0x00,0x84,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff
-# GFX12: v_cmpx_gt_f16_e64_dpp v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x84,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+# GFX12-REAL16: v_cmpx_gt_f16_e64_dpp v1.l, v2.l row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x84,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+# GFX12-FAKE16: v_cmpx_gt_f16_e64_dpp v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x84,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
 
 0x7e,0x00,0x84,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff
-# GFX12: v_cmpx_gt_f16_e64_dpp v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x84,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+# GFX12-REAL16: v_cmpx_gt_f16_e64_dpp v1.l, v2.l row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x84,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+# GFX12-FAKE16: v_cmpx_gt_f16_e64_dpp v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x84,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
 
 0x7e,0x00,0x84,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff
-# GFX12: v_cmpx_gt_f16_e64_dpp v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x84,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+# GFX12-REAL16: v_cmpx_gt_f16_e64_dpp v1.l, v2.l row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x84,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+# GFX12-FAKE16: v_cmpx_gt_f16_e64_dpp v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x84,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
 
 0x7e,0x00,0x84,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff
-# GFX12: v_cmpx_gt_f16_e64_dpp v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x84,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+# GFX12-REAL16: v_cmpx_gt_f16_e64_dpp v1.l, v2.l row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x84,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+# GFX12-FAKE16: v_cmpx_gt_f16_e64_dpp v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x84,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
 
 0x7e,0x00,0x84,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff
-# GFX12: v_cmpx_gt_f16_e64_dpp v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x84,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+# GFX12-REAL16: v_cmpx_gt_f16_e64_dpp v1.l, v2.l row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x84,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+# GFX12-FAKE16: v_cmpx_gt_f16_e64_dpp v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x84,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
 
 0x7e,0x00,0x84,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff
-# GFX12: v_cmpx_gt_f16_e64_dpp v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x84,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+# GFX12-REAL16: v_cmpx_gt_f16_e64_dpp v1.l, v2.l row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x84,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+# GFX12-FAKE16: v_cmpx_gt_f16_e64_dpp v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x84,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
 
 0x7e,0x00,0x84,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff
-# GFX12: v_cmpx_gt_f16_e64_dpp v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x84,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+# GFX12-REAL16: v_cmpx_gt_f16_e64_dpp v1.l, v2.l row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x84,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+# GFX12-FAKE16: v_cmpx_gt_f16_e64_dpp v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x84,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
 
 0x7e,0x00,0x84,0xd4,0xfa,0x06,0x00,0x00,0x01,0x2f,0x01,0xff
-# GFX12: v_cmpx_gt_f16_e64_dpp v1, s3 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x84,0xd4,0xfa,0x06,0x00,0x00,0x01,0x2f,0x01,0xff]
+# GFX12-REAL16: v_cmpx_gt_f16_e64_dpp v1.l, s3 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x84,0xd4,0xfa,0x06,0x00,0x00,0x01,0x2f,0x01,0xff]
+# GFX12-FAKE16: v_cmpx_gt_f16_e64_dpp v1, s3 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x84,0xd4,0xfa,0x06,0x00,0x00,0x01,0x2f,0x01,0xff]
 
 0x7e,0x00,0x84,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff
-# GFX12: v_cmpx_gt_f16_e64_dpp v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x84,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+# GFX12-REAL16: v_cmpx_gt_f16_e64_dpp v1.l, v2.l row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x84,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+# GFX12-FAKE16: v_cmpx_gt_f16_e64_dpp v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x84,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 
 0x7e,0x01,0x84,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01
-# GFX12: v_cmpx_gt_f16_e64_dpp |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x01,0x84,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+# GFX12-REAL16: v_cmpx_gt_f16_e64_dpp |v1.l|, -v2.l row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x01,0x84,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+# GFX12-FAKE16: v_cmpx_gt_f16_e64_dpp |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x01,0x84,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
 
 0x7e,0x02,0x84,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x01,0x13
-# GFX12: v_cmpx_gt_f16_e64_dpp -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x7e,0x02,0x84,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x01,0x13]
+# GFX12-REAL16: v_cmpx_gt_f16_e64_dpp -v1.l, |v2.l| row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x7e,0x02,0x84,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x01,0x13]
+# GFX12-FAKE16: v_cmpx_gt_f16_e64_dpp -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x7e,0x02,0x84,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x01,0x13]
 
 0x7e,0x83,0x84,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30
-# GFX12: v_cmpx_gt_f16_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0x7e,0x83,0x84,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30]
+# GFX12-REAL16: v_cmpx_gt_f16_e64_dpp -|v255.l|, -|v255.l| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0x7e,0x83,0x84,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30]
+# GFX12-FAKE16: v_cmpx_gt_f16_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0x7e,0x83,0x84,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30]
+
+0x7e,0x08,0x84,0xd4,0x01,0x05,0x02,0x00
+# GFX12-REAL16: v_cmpx_gt_f16_e64 v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x7e,0x08,0x84,0xd4,0x01,0x05,0x02,0x00]
+# GFX12-FAKE16: v_cmpx_gt_f16_e64 v1, v2                ; encoding: [0x7e,0x00,0x84,0xd4,0x01,0x05,0x02,0x00]
+
+0x7e,0x10,0x84,0xd4,0xff,0xff,0x03,0x00
+# GFX12-REAL16: v_cmpx_gt_f16_e64 v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x7e,0x10,0x84,0xd4,0xff,0xff,0x03,0x00]
+# GFX12-FAKE16: v_cmpx_gt_f16_e64 v255, v255            ; encoding: [0x7e,0x00,0x84,0xd4,0xff,0xff,0x03,0x00]
+
+0x7e,0x00,0x84,0xd4,0x01,0x04,0x00,0x00
+# GFX12: v_cmpx_gt_f16_e64 s1, s2                ; encoding: [0x7e,0x00,0x84,0xd4,0x01,0x04,0x00,0x00]
+
+0x7e,0x00,0x84,0xd4,0x69,0xd2,0x00,0x00
+# GFX12: v_cmpx_gt_f16_e64 s105, s105            ; encoding: [0x7e,0x00,0x84,0xd4,0x69,0xd2,0x00,0x00]
+
+0x7e,0x00,0x84,0xd4,0x6a,0xf6,0x00,0x00
+# GFX12: v_cmpx_gt_f16_e64 vcc_lo, ttmp15        ; encoding: [0x7e,0x00,0x84,0xd4,0x6a,0xf6,0x00,0x00]
+
+0x7e,0x00,0x84,0xd4,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00
+# GFX12: v_cmpx_gt_f16_e64 vcc_hi, 0xfe0b        ; encoding: [0x7e,0x00,0x84,0xd4,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
+
+0x7e,0x00,0x84,0xd4,0x7b,0xfa,0x01,0x00
+# GFX12: v_cmpx_gt_f16_e64 ttmp15, src_scc       ; encoding: [0x7e,0x00,0x84,0xd4,0x7b,0xfa,0x01,0x00]
+
+0x7e,0x00,0x84,0xd4,0x7d,0xe0,0x01,0x00
+# GFX12: v_cmpx_gt_f16_e64 m0, 0.5               ; encoding: [0x7e,0x00,0x84,0xd4,0x7d,0xe0,0x01,0x00]
+
+0x7e,0x00,0x84,0xd4,0x7e,0x82,0x01,0x00
+# GFX12: v_cmpx_gt_f16_e64 exec_lo, -1           ; encoding: [0x7e,0x00,0x84,0xd4,0x7e,0x82,0x01,0x00]
+
+0x7e,0x01,0x84,0xd4,0x7f,0xf8,0x00,0x00
+# GFX12: v_cmpx_gt_f16_e64 |exec_hi|, null       ; encoding: [0x7e,0x01,0x84,0xd4,0x7f,0xf8,0x00,0x00]
+
+0x7e,0x00,0x84,0xd4,0x7c,0xfc,0x00,0x00
+# GFX12: v_cmpx_gt_f16_e64 null, exec_lo         ; encoding: [0x7e,0x00,0x84,0xd4,0x7c,0xfc,0x00,0x00]
+
+0x7e,0x00,0x84,0xd4,0xc1,0xfe,0x00,0x00
+# GFX12: v_cmpx_gt_f16_e64 -1, exec_hi           ; encoding: [0x7e,0x00,0x84,0xd4,0xc1,0xfe,0x00,0x00]
+
+0x7e,0x00,0x84,0xd4,0xf0,0xfa,0x00,0x40
+# GFX12: v_cmpx_gt_f16_e64 0.5, -m0              ; encoding: [0x7e,0x00,0x84,0xd4,0xf0,0xfa,0x00,0x40]
+
+0x7e,0x02,0x84,0xd4,0xfd,0xd4,0x00,0x20
+# GFX12: v_cmpx_gt_f16_e64 -src_scc, |vcc_lo|    ; encoding: [0x7e,0x02,0x84,0xd4,0xfd,0xd4,0x00,0x20]
+
+0x7e,0x83,0x84,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00
+# GFX12: v_cmpx_gt_f16_e64 -|0xfe0b|, -|vcc_hi| clamp ; encoding: [0x7e,0x83,0x84,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00]
+
+0x7e,0x19,0x84,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01
+# GFX12-REAL16: v_cmpx_gt_f16_e64_dpp |v1.h|, -v2.h op_sel:[1,1] row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x19,0x84,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+# GFX12-FAKE16: v_cmpx_gt_f16_e64_dpp |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x01,0x84,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+
+0x7e,0x0a,0x84,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x01,0x13
+# GFX12-REAL16: v_cmpx_gt_f16_e64_dpp -v1.h, |v2.l| op_sel:[1,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x7e,0x0a,0x84,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x01,0x13]
+# GFX12-FAKE16: v_cmpx_gt_f16_e64_dpp -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x7e,0x02,0x84,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x01,0x13]
+
+0x7e,0x93,0x84,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30
+# GFX12-REAL16: v_cmpx_gt_f16_e64_dpp -|v255.l|, -|v255.h| op_sel:[0,1] clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0x7e,0x93,0x84,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30]
+# GFX12-FAKE16: v_cmpx_gt_f16_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0x7e,0x83,0x84,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30]
 
 0x7e,0x00,0x94,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff
 # GFX12: v_cmpx_gt_f32_e64_dpp v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x94,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
@@ -1218,17 +1428,14 @@
 0x7e,0x00,0xb4,0xd4,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00
 # GFX12: v_cmpx_gt_i16_e64 0xfe0b, vcc_hi        ; encoding: [0x7e,0x00,0xb4,0xd4,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
 
-# GFX11: v_cmpx_gt_i16_e64_dpp v1.h, v2.h op_sel:[1,1] row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x18,0xb4,0xd4,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
 0x7e,0x18,0xb4,0xd4,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01
 # GFX12-REAL16: v_cmpx_gt_i16_e64_dpp v1.h, v2.h op_sel:[1,1] row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x18,0xb4,0xd4,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
 # GFX12-FAKE16: v_cmpx_gt_i16_e64_dpp v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x00,0xb4,0xd4,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
 
-# GFX11: v_cmpx_gt_i16_e64_dpp v1.h, v2.l op_sel:[1,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x7e,0x08,0xb4,0xd4,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
 0x7e,0x08,0xb4,0xd4,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13
 # GFX12-REAL16: v_cmpx_gt_i16_e64_dpp v1.h, v2.l op_sel:[1,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x7e,0x08,0xb4,0xd4,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
 # GFX12-FAKE16: v_cmpx_gt_i16_e64_dpp v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x7e,0x00,0xb4,0xd4,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
 
-# GFX11: v_cmpx_gt_i16_e64_dpp v255.l, v255.h op_sel:[0,1] row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0x7e,0x10,0xb4,0xd4,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 0x7e,0x10,0xb4,0xd4,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30
 # GFX12-REAL16: v_cmpx_gt_i16_e64_dpp v255.l, v255.h op_sel:[0,1] row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0x7e,0x10,0xb4,0xd4,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 # GFX12-FAKE16: v_cmpx_gt_i16_e64_dpp v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0x7e,0x00,0xb4,0xd4,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
@@ -1385,17 +1592,14 @@
 0x7e,0x00,0xbc,0xd4,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00
 # GFX12: v_cmpx_gt_u16_e64 0xfe0b, vcc_hi        ; encoding: [0x7e,0x00,0xbc,0xd4,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
 
-# GFX11: v_cmpx_gt_u16_e64_dpp v1.h, v2.h op_sel:[1,1] row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x18,0xbc,0xd4,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
 0x7e,0x18,0xbc,0xd4,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01
 # GFX12-REAL16: v_cmpx_gt_u16_e64_dpp v1.h, v2.h op_sel:[1,1] row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x18,0xbc,0xd4,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
 # GFX12-FAKE16: v_cmpx_gt_u16_e64_dpp v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x00,0xbc,0xd4,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
 
-# GFX11: v_cmpx_gt_u16_e64_dpp v1.h, v2.l op_sel:[1,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x7e,0x08,0xbc,0xd4,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
 0x7e,0x08,0xbc,0xd4,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13
 # GFX12-REAL16: v_cmpx_gt_u16_e64_dpp v1.h, v2.l op_sel:[1,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x7e,0x08,0xbc,0xd4,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
 # GFX12-FAKE16: v_cmpx_gt_u16_e64_dpp v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x7e,0x00,0xbc,0xd4,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
 
-# GFX11: v_cmpx_gt_u16_e64_dpp v255.l, v255.h op_sel:[0,1] row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0x7e,0x10,0xbc,0xd4,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 0x7e,0x10,0xbc,0xd4,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30
 # GFX12-REAL16: v_cmpx_gt_u16_e64_dpp v255.l, v255.h op_sel:[0,1] row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0x7e,0x10,0xbc,0xd4,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 # GFX12-FAKE16: v_cmpx_gt_u16_e64_dpp v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0x7e,0x00,0xbc,0xd4,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
@@ -1446,49 +1650,123 @@
 # GFX12: v_cmpx_gt_u32_e64_dpp v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0x7e,0x00,0xcc,0xd4,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 
 0x7e,0x00,0x83,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff
-# GFX12: v_cmpx_le_f16_e64_dpp v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x83,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+# GFX12-REAL16: v_cmpx_le_f16_e64_dpp v1.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x83,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+# GFX12-FAKE16: v_cmpx_le_f16_e64_dpp v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x83,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
 
 0x7e,0x00,0x83,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff
-# GFX12: v_cmpx_le_f16_e64_dpp v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x83,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+# GFX12-REAL16: v_cmpx_le_f16_e64_dpp v1.l, v2.l quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x83,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+# GFX12-FAKE16: v_cmpx_le_f16_e64_dpp v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x83,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
 
 0x7e,0x00,0x83,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff
-# GFX12: v_cmpx_le_f16_e64_dpp v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x83,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+# GFX12-REAL16: v_cmpx_le_f16_e64_dpp v1.l, v2.l row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x83,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+# GFX12-FAKE16: v_cmpx_le_f16_e64_dpp v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x83,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
 
 0x7e,0x00,0x83,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff
-# GFX12: v_cmpx_le_f16_e64_dpp v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x83,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+# GFX12-REAL16: v_cmpx_le_f16_e64_dpp v1.l, v2.l row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x83,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+# GFX12-FAKE16: v_cmpx_le_f16_e64_dpp v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x83,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
 
 0x7e,0x00,0x83,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff
-# GFX12: v_cmpx_le_f16_e64_dpp v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x83,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+# GFX12-REAL16: v_cmpx_le_f16_e64_dpp v1.l, v2.l row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x83,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+# GFX12-FAKE16: v_cmpx_le_f16_e64_dpp v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x83,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
 
 0x7e,0x00,0x83,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff
-# GFX12: v_cmpx_le_f16_e64_dpp v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x83,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+# GFX12-REAL16: v_cmpx_le_f16_e64_dpp v1.l, v2.l row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x83,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+# GFX12-FAKE16: v_cmpx_le_f16_e64_dpp v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x83,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
 
 0x7e,0x00,0x83,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff
-# GFX12: v_cmpx_le_f16_e64_dpp v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x83,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+# GFX12-REAL16: v_cmpx_le_f16_e64_dpp v1.l, v2.l row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x83,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+# GFX12-FAKE16: v_cmpx_le_f16_e64_dpp v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x83,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
 
 0x7e,0x00,0x83,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff
-# GFX12: v_cmpx_le_f16_e64_dpp v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x83,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+# GFX12-REAL16: v_cmpx_le_f16_e64_dpp v1.l, v2.l row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x83,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+# GFX12-FAKE16: v_cmpx_le_f16_e64_dpp v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x83,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
 
 0x7e,0x00,0x83,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff
-# GFX12: v_cmpx_le_f16_e64_dpp v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x83,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+# GFX12-REAL16: v_cmpx_le_f16_e64_dpp v1.l, v2.l row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x83,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+# GFX12-FAKE16: v_cmpx_le_f16_e64_dpp v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x83,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
 
 0x7e,0x00,0x83,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff
-# GFX12: v_cmpx_le_f16_e64_dpp v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x83,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+# GFX12-REAL16: v_cmpx_le_f16_e64_dpp v1.l, v2.l row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x83,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+# GFX12-FAKE16: v_cmpx_le_f16_e64_dpp v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x83,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
 
 0x7e,0x00,0x83,0xd4,0xfa,0x06,0x00,0x00,0x01,0x2f,0x01,0xff
-# GFX12: v_cmpx_le_f16_e64_dpp v1, s3 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x83,0xd4,0xfa,0x06,0x00,0x00,0x01,0x2f,0x01,0xff]
+# GFX12-REAL16: v_cmpx_le_f16_e64_dpp v1.l, s3 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x83,0xd4,0xfa,0x06,0x00,0x00,0x01,0x2f,0x01,0xff]
+# GFX12-FAKE16: v_cmpx_le_f16_e64_dpp v1, s3 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x83,0xd4,0xfa,0x06,0x00,0x00,0x01,0x2f,0x01,0xff]
 
 0x7e,0x00,0x83,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff
-# GFX12: v_cmpx_le_f16_e64_dpp v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x83,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+# GFX12-REAL16: v_cmpx_le_f16_e64_dpp v1.l, v2.l row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x83,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+# GFX12-FAKE16: v_cmpx_le_f16_e64_dpp v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x83,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 
 0x7e,0x01,0x83,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01
-# GFX12: v_cmpx_le_f16_e64_dpp |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x01,0x83,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+# GFX12-REAL16: v_cmpx_le_f16_e64_dpp |v1.l|, -v2.l row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x01,0x83,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+# GFX12-FAKE16: v_cmpx_le_f16_e64_dpp |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x01,0x83,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
 
 0x7e,0x02,0x83,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x01,0x13
-# GFX12: v_cmpx_le_f16_e64_dpp -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x7e,0x02,0x83,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x01,0x13]
+# GFX12-REAL16: v_cmpx_le_f16_e64_dpp -v1.l, |v2.l| row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x7e,0x02,0x83,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x01,0x13]
+# GFX12-FAKE16: v_cmpx_le_f16_e64_dpp -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x7e,0x02,0x83,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x01,0x13]
 
 0x7e,0x83,0x83,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30
-# GFX12: v_cmpx_le_f16_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0x7e,0x83,0x83,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30]
+# GFX12-REAL16: v_cmpx_le_f16_e64_dpp -|v255.l|, -|v255.l| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0x7e,0x83,0x83,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30]
+# GFX12-FAKE16: v_cmpx_le_f16_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0x7e,0x83,0x83,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30]
+
+0x7e,0x08,0x83,0xd4,0x01,0x05,0x02,0x00
+# GFX12-REAL16: v_cmpx_le_f16_e64 v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x7e,0x08,0x83,0xd4,0x01,0x05,0x02,0x00]
+# GFX12-FAKE16: v_cmpx_le_f16_e64 v1, v2                ; encoding: [0x7e,0x00,0x83,0xd4,0x01,0x05,0x02,0x00]
+
+0x7e,0x10,0x83,0xd4,0xff,0xff,0x03,0x00
+# GFX12-REAL16: v_cmpx_le_f16_e64 v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x7e,0x10,0x83,0xd4,0xff,0xff,0x03,0x00]
+# GFX12-FAKE16: v_cmpx_le_f16_e64 v255, v255            ; encoding: [0x7e,0x00,0x83,0xd4,0xff,0xff,0x03,0x00]
+
+0x7e,0x00,0x83,0xd4,0x01,0x04,0x00,0x00
+# GFX12: v_cmpx_le_f16_e64 s1, s2                ; encoding: [0x7e,0x00,0x83,0xd4,0x01,0x04,0x00,0x00]
+
+0x7e,0x00,0x83,0xd4,0x69,0xd2,0x00,0x00
+# GFX12: v_cmpx_le_f16_e64 s105, s105            ; encoding: [0x7e,0x00,0x83,0xd4,0x69,0xd2,0x00,0x00]
+
+0x7e,0x00,0x83,0xd4,0x6a,0xf6,0x00,0x00
+# GFX12: v_cmpx_le_f16_e64 vcc_lo, ttmp15        ; encoding: [0x7e,0x00,0x83,0xd4,0x6a,0xf6,0x00,0x00]
+
+0x7e,0x00,0x83,0xd4,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00
+# GFX12: v_cmpx_le_f16_e64 vcc_hi, 0xfe0b        ; encoding: [0x7e,0x00,0x83,0xd4,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
+
+0x7e,0x00,0x83,0xd4,0x7b,0xfa,0x01,0x00
+# GFX12: v_cmpx_le_f16_e64 ttmp15, src_scc       ; encoding: [0x7e,0x00,0x83,0xd4,0x7b,0xfa,0x01,0x00]
+
+0x7e,0x00,0x83,0xd4,0x7d,0xe0,0x01,0x00
+# GFX12: v_cmpx_le_f16_e64 m0, 0.5               ; encoding: [0x7e,0x00,0x83,0xd4,0x7d,0xe0,0x01,0x00]
+
+0x7e,0x00,0x83,0xd4,0x7e,0x82,0x01,0x00
+# GFX12: v_cmpx_le_f16_e64 exec_lo, -1           ; encoding: [0x7e,0x00,0x83,0xd4,0x7e,0x82,0x01,0x00]
+
+0x7e,0x01,0x83,0xd4,0x7f,0xf8,0x00,0x00
+# GFX12: v_cmpx_le_f16_e64 |exec_hi|, null       ; encoding: [0x7e,0x01,0x83,0xd4,0x7f,0xf8,0x00,0x00]
+
+0x7e,0x00,0x83,0xd4,0x7c,0xfc,0x00,0x00
+# GFX12: v_cmpx_le_f16_e64 null, exec_lo         ; encoding: [0x7e,0x00,0x83,0xd4,0x7c,0xfc,0x00,0x00]
+
+0x7e,0x00,0x83,0xd4,0xc1,0xfe,0x00,0x00
+# GFX12: v_cmpx_le_f16_e64 -1, exec_hi           ; encoding: [0x7e,0x00,0x83,0xd4,0xc1,0xfe,0x00,0x00]
+
+0x7e,0x00,0x83,0xd4,0xf0,0xfa,0x00,0x40
+# GFX12: v_cmpx_le_f16_e64 0.5, -m0              ; encoding: [0x7e,0x00,0x83,0xd4,0xf0,0xfa,0x00,0x40]
+
+0x7e,0x02,0x83,0xd4,0xfd,0xd4,0x00,0x20
+# GFX12: v_cmpx_le_f16_e64 -src_scc, |vcc_lo|    ; encoding: [0x7e,0x02,0x83,0xd4,0xfd,0xd4,0x00,0x20]
+
+0x7e,0x83,0x83,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00
+# GFX12: v_cmpx_le_f16_e64 -|0xfe0b|, -|vcc_hi| clamp ; encoding: [0x7e,0x83,0x83,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00]
+
+0x7e,0x19,0x83,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01
+# GFX12-REAL16: v_cmpx_le_f16_e64_dpp |v1.h|, -v2.h op_sel:[1,1] row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x19,0x83,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+# GFX12-FAKE16: v_cmpx_le_f16_e64_dpp |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x01,0x83,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+
+0x7e,0x0a,0x83,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x01,0x13
+# GFX12-REAL16: v_cmpx_le_f16_e64_dpp -v1.h, |v2.l| op_sel:[1,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x7e,0x0a,0x83,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x01,0x13]
+# GFX12-FAKE16: v_cmpx_le_f16_e64_dpp -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x7e,0x02,0x83,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x01,0x13]
+
+0x7e,0x93,0x83,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30
+# GFX12-REAL16: v_cmpx_le_f16_e64_dpp -|v255.l|, -|v255.h| op_sel:[0,1] clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0x7e,0x93,0x83,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30]
+# GFX12-FAKE16: v_cmpx_le_f16_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0x7e,0x83,0x83,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30]
 
 0x7e,0x00,0x93,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff
 # GFX12: v_cmpx_le_f32_e64_dpp v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x93,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
@@ -1642,17 +1920,14 @@
 0x7e,0x00,0xb3,0xd4,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00
 # GFX12: v_cmpx_le_i16_e64 0xfe0b, vcc_hi        ; encoding: [0x7e,0x00,0xb3,0xd4,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
 
-# GFX11: v_cmpx_le_i16_e64_dpp v1.h, v2.h op_sel:[1,1] row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x18,0xb3,0xd4,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
 0x7e,0x18,0xb3,0xd4,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01
 # GFX12-REAL16: v_cmpx_le_i16_e64_dpp v1.h, v2.h op_sel:[1,1] row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x18,0xb3,0xd4,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
 # GFX12-FAKE16: v_cmpx_le_i16_e64_dpp v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x00,0xb3,0xd4,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
 
-# GFX11: v_cmpx_le_i16_e64_dpp v1.h, v2.l op_sel:[1,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x7e,0x08,0xb3,0xd4,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
 0x7e,0x08,0xb3,0xd4,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13
 # GFX12-REAL16: v_cmpx_le_i16_e64_dpp v1.h, v2.l op_sel:[1,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x7e,0x08,0xb3,0xd4,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
 # GFX12-FAKE16: v_cmpx_le_i16_e64_dpp v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x7e,0x00,0xb3,0xd4,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
 
-# GFX11: v_cmpx_le_i16_e64_dpp v255.l, v255.h op_sel:[0,1] row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0x7e,0x10,0xb3,0xd4,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 0x7e,0x10,0xb3,0xd4,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30
 # GFX12-REAL16: v_cmpx_le_i16_e64_dpp v255.l, v255.h op_sel:[0,1] row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0x7e,0x10,0xb3,0xd4,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 # GFX12-FAKE16: v_cmpx_le_i16_e64_dpp v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0x7e,0x00,0xb3,0xd4,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
@@ -1809,17 +2084,14 @@
 0x7e,0x00,0xbb,0xd4,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00
 # GFX12: v_cmpx_le_u16_e64 0xfe0b, vcc_hi        ; encoding: [0x7e,0x00,0xbb,0xd4,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
 
-# GFX11: v_cmpx_le_u16_e64_dpp v1.h, v2.h op_sel:[1,1] row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x18,0xbb,0xd4,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
 0x7e,0x18,0xbb,0xd4,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01
 # GFX12-REAL16: v_cmpx_le_u16_e64_dpp v1.h, v2.h op_sel:[1,1] row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x18,0xbb,0xd4,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
 # GFX12-FAKE16: v_cmpx_le_u16_e64_dpp v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x00,0xbb,0xd4,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
 
-# GFX11: v_cmpx_le_u16_e64_dpp v1.h, v2.l op_sel:[1,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x7e,0x08,0xbb,0xd4,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
 0x7e,0x08,0xbb,0xd4,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13
 # GFX12-REAL16: v_cmpx_le_u16_e64_dpp v1.h, v2.l op_sel:[1,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x7e,0x08,0xbb,0xd4,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
 # GFX12-FAKE16: v_cmpx_le_u16_e64_dpp v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x7e,0x00,0xbb,0xd4,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
 
-# GFX11: v_cmpx_le_u16_e64_dpp v255.l, v255.h op_sel:[0,1] row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0x7e,0x10,0xbb,0xd4,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 0x7e,0x10,0xbb,0xd4,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30
 # GFX12-REAL16: v_cmpx_le_u16_e64_dpp v255.l, v255.h op_sel:[0,1] row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0x7e,0x10,0xbb,0xd4,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 # GFX12-FAKE16: v_cmpx_le_u16_e64_dpp v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0x7e,0x00,0xbb,0xd4,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
@@ -1870,49 +2142,123 @@
 # GFX12: v_cmpx_le_u32_e64_dpp v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0x7e,0x00,0xcb,0xd4,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 
 0x7e,0x00,0x85,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff
-# GFX12: v_cmpx_lg_f16_e64_dpp v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x85,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+# GFX12-REAL16: v_cmpx_lg_f16_e64_dpp v1.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x85,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+# GFX12-FAKE16: v_cmpx_lg_f16_e64_dpp v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x85,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
 
 0x7e,0x00,0x85,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff
-# GFX12: v_cmpx_lg_f16_e64_dpp v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x85,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+# GFX12-REAL16: v_cmpx_lg_f16_e64_dpp v1.l, v2.l quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x85,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+# GFX12-FAKE16: v_cmpx_lg_f16_e64_dpp v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x85,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
 
 0x7e,0x00,0x85,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff
-# GFX12: v_cmpx_lg_f16_e64_dpp v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x85,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+# GFX12-REAL16: v_cmpx_lg_f16_e64_dpp v1.l, v2.l row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x85,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+# GFX12-FAKE16: v_cmpx_lg_f16_e64_dpp v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x85,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
 
 0x7e,0x00,0x85,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff
-# GFX12: v_cmpx_lg_f16_e64_dpp v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x85,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+# GFX12-REAL16: v_cmpx_lg_f16_e64_dpp v1.l, v2.l row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x85,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+# GFX12-FAKE16: v_cmpx_lg_f16_e64_dpp v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x85,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
 
 0x7e,0x00,0x85,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff
-# GFX12: v_cmpx_lg_f16_e64_dpp v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x85,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+# GFX12-REAL16: v_cmpx_lg_f16_e64_dpp v1.l, v2.l row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x85,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+# GFX12-FAKE16: v_cmpx_lg_f16_e64_dpp v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x85,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
 
 0x7e,0x00,0x85,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff
-# GFX12: v_cmpx_lg_f16_e64_dpp v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x85,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+# GFX12-REAL16: v_cmpx_lg_f16_e64_dpp v1.l, v2.l row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x85,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+# GFX12-FAKE16: v_cmpx_lg_f16_e64_dpp v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x85,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
 
 0x7e,0x00,0x85,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff
-# GFX12: v_cmpx_lg_f16_e64_dpp v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x85,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+# GFX12-REAL16: v_cmpx_lg_f16_e64_dpp v1.l, v2.l row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x85,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+# GFX12-FAKE16: v_cmpx_lg_f16_e64_dpp v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x85,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
 
 0x7e,0x00,0x85,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff
-# GFX12: v_cmpx_lg_f16_e64_dpp v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x85,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+# GFX12-REAL16: v_cmpx_lg_f16_e64_dpp v1.l, v2.l row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x85,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+# GFX12-FAKE16: v_cmpx_lg_f16_e64_dpp v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x85,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
 
 0x7e,0x00,0x85,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff
-# GFX12: v_cmpx_lg_f16_e64_dpp v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x85,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+# GFX12-REAL16: v_cmpx_lg_f16_e64_dpp v1.l, v2.l row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x85,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+# GFX12-FAKE16: v_cmpx_lg_f16_e64_dpp v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x85,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
 
 0x7e,0x00,0x85,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff
-# GFX12: v_cmpx_lg_f16_e64_dpp v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x85,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+# GFX12-REAL16: v_cmpx_lg_f16_e64_dpp v1.l, v2.l row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x85,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+# GFX12-FAKE16: v_cmpx_lg_f16_e64_dpp v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x85,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
 
 0x7e,0x00,0x85,0xd4,0xfa,0x06,0x00,0x00,0x01,0x2f,0x01,0xff
-# GFX12: v_cmpx_lg_f16_e64_dpp v1, s3 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x85,0xd4,0xfa,0x06,0x00,0x00,0x01,0x2f,0x01,0xff]
+# GFX12-REAL16: v_cmpx_lg_f16_e64_dpp v1.l, s3 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x85,0xd4,0xfa,0x06,0x00,0x00,0x01,0x2f,0x01,0xff]
+# GFX12-FAKE16: v_cmpx_lg_f16_e64_dpp v1, s3 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x85,0xd4,0xfa,0x06,0x00,0x00,0x01,0x2f,0x01,0xff]
 
 0x7e,0x00,0x85,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff
-# GFX12: v_cmpx_lg_f16_e64_dpp v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x85,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+# GFX12-REAL16: v_cmpx_lg_f16_e64_dpp v1.l, v2.l row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x85,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+# GFX12-FAKE16: v_cmpx_lg_f16_e64_dpp v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x85,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 
 0x7e,0x01,0x85,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01
-# GFX12: v_cmpx_lg_f16_e64_dpp |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x01,0x85,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+# GFX12-REAL16: v_cmpx_lg_f16_e64_dpp |v1.l|, -v2.l row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x01,0x85,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+# GFX12-FAKE16: v_cmpx_lg_f16_e64_dpp |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x01,0x85,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
 
 0x7e,0x02,0x85,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x01,0x13
-# GFX12: v_cmpx_lg_f16_e64_dpp -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x7e,0x02,0x85,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x01,0x13]
+# GFX12-REAL16: v_cmpx_lg_f16_e64_dpp -v1.l, |v2.l| row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x7e,0x02,0x85,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x01,0x13]
+# GFX12-FAKE16: v_cmpx_lg_f16_e64_dpp -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x7e,0x02,0x85,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x01,0x13]
 
 0x7e,0x83,0x85,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30
-# GFX12: v_cmpx_lg_f16_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0x7e,0x83,0x85,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30]
+# GFX12-REAL16: v_cmpx_lg_f16_e64_dpp -|v255.l|, -|v255.l| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0x7e,0x83,0x85,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30]
+# GFX12-FAKE16: v_cmpx_lg_f16_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0x7e,0x83,0x85,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30]
+
+0x7e,0x08,0x85,0xd4,0x01,0x05,0x02,0x00
+# GFX12-REAL16: v_cmpx_lg_f16_e64 v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x7e,0x08,0x85,0xd4,0x01,0x05,0x02,0x00]
+# GFX12-FAKE16: v_cmpx_lg_f16_e64 v1, v2                ; encoding: [0x7e,0x00,0x85,0xd4,0x01,0x05,0x02,0x00]
+
+0x7e,0x10,0x85,0xd4,0xff,0xff,0x03,0x00
+# GFX12-REAL16: v_cmpx_lg_f16_e64 v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x7e,0x10,0x85,0xd4,0xff,0xff,0x03,0x00]
+# GFX12-FAKE16: v_cmpx_lg_f16_e64 v255, v255            ; encoding: [0x7e,0x00,0x85,0xd4,0xff,0xff,0x03,0x00]
+
+0x7e,0x00,0x85,0xd4,0x01,0x04,0x00,0x00
+# GFX12: v_cmpx_lg_f16_e64 s1, s2                ; encoding: [0x7e,0x00,0x85,0xd4,0x01,0x04,0x00,0x00]
+
+0x7e,0x00,0x85,0xd4,0x69,0xd2,0x00,0x00
+# GFX12: v_cmpx_lg_f16_e64 s105, s105            ; encoding: [0x7e,0x00,0x85,0xd4,0x69,0xd2,0x00,0x00]
+
+0x7e,0x00,0x85,0xd4,0x6a,0xf6,0x00,0x00
+# GFX12: v_cmpx_lg_f16_e64 vcc_lo, ttmp15        ; encoding: [0x7e,0x00,0x85,0xd4,0x6a,0xf6,0x00,0x00]
+
+0x7e,0x00,0x85,0xd4,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00
+# GFX12: v_cmpx_lg_f16_e64 vcc_hi, 0xfe0b        ; encoding: [0x7e,0x00,0x85,0xd4,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
+
+0x7e,0x00,0x85,0xd4,0x7b,0xfa,0x01,0x00
+# GFX12: v_cmpx_lg_f16_e64 ttmp15, src_scc       ; encoding: [0x7e,0x00,0x85,0xd4,0x7b,0xfa,0x01,0x00]
+
+0x7e,0x00,0x85,0xd4,0x7d,0xe0,0x01,0x00
+# GFX12: v_cmpx_lg_f16_e64 m0, 0.5               ; encoding: [0x7e,0x00,0x85,0xd4,0x7d,0xe0,0x01,0x00]
+
+0x7e,0x00,0x85,0xd4,0x7e,0x82,0x01,0x00
+# GFX12: v_cmpx_lg_f16_e64 exec_lo, -1           ; encoding: [0x7e,0x00,0x85,0xd4,0x7e,0x82,0x01,0x00]
+
+0x7e,0x01,0x85,0xd4,0x7f,0xf8,0x00,0x00
+# GFX12: v_cmpx_lg_f16_e64 |exec_hi|, null       ; encoding: [0x7e,0x01,0x85,0xd4,0x7f,0xf8,0x00,0x00]
+
+0x7e,0x00,0x85,0xd4,0x7c,0xfc,0x00,0x00
+# GFX12: v_cmpx_lg_f16_e64 null, exec_lo         ; encoding: [0x7e,0x00,0x85,0xd4,0x7c,0xfc,0x00,0x00]
+
+0x7e,0x00,0x85,0xd4,0xc1,0xfe,0x00,0x00
+# GFX12: v_cmpx_lg_f16_e64 -1, exec_hi           ; encoding: [0x7e,0x00,0x85,0xd4,0xc1,0xfe,0x00,0x00]
+
+0x7e,0x00,0x85,0xd4,0xf0,0xfa,0x00,0x40
+# GFX12: v_cmpx_lg_f16_e64 0.5, -m0              ; encoding: [0x7e,0x00,0x85,0xd4,0xf0,0xfa,0x00,0x40]
+
+0x7e,0x02,0x85,0xd4,0xfd,0xd4,0x00,0x20
+# GFX12: v_cmpx_lg_f16_e64 -src_scc, |vcc_lo|    ; encoding: [0x7e,0x02,0x85,0xd4,0xfd,0xd4,0x00,0x20]
+
+0x7e,0x83,0x85,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00
+# GFX12: v_cmpx_lg_f16_e64 -|0xfe0b|, -|vcc_hi| clamp ; encoding: [0x7e,0x83,0x85,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00]
+
+0x7e,0x19,0x85,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01
+# GFX12-REAL16: v_cmpx_lg_f16_e64_dpp |v1.h|, -v2.h op_sel:[1,1] row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x19,0x85,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+# GFX12-FAKE16: v_cmpx_lg_f16_e64_dpp |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x01,0x85,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+
+0x7e,0x0a,0x85,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x01,0x13
+# GFX12-REAL16: v_cmpx_lg_f16_e64_dpp -v1.h, |v2.l| op_sel:[1,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x7e,0x0a,0x85,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x01,0x13]
+# GFX12-FAKE16: v_cmpx_lg_f16_e64_dpp -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x7e,0x02,0x85,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x01,0x13]
+
+0x7e,0x93,0x85,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30
+# GFX12-REAL16: v_cmpx_lg_f16_e64_dpp -|v255.l|, -|v255.h| op_sel:[0,1] clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0x7e,0x93,0x85,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30]
+# GFX12-FAKE16: v_cmpx_lg_f16_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0x7e,0x83,0x85,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30]
 
 0x7e,0x00,0x95,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff
 # GFX12: v_cmpx_lg_f32_e64_dpp v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x95,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
@@ -2230,17 +2576,14 @@
 0x7e,0x00,0xb1,0xd4,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00
 # GFX12: v_cmpx_lt_i16_e64 0xfe0b, vcc_hi        ; encoding: [0x7e,0x00,0xb1,0xd4,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
 
-# GFX11: v_cmpx_lt_i16_e64_dpp v1.h, v2.h op_sel:[1,1] row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x18,0xb1,0xd4,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
 0x7e,0x18,0xb1,0xd4,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01
 # GFX12-REAL16: v_cmpx_lt_i16_e64_dpp v1.h, v2.h op_sel:[1,1] row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x18,0xb1,0xd4,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
 # GFX12-FAKE16: v_cmpx_lt_i16_e64_dpp v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x00,0xb1,0xd4,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
 
-# GFX11: v_cmpx_lt_i16_e64_dpp v1.h, v2.l op_sel:[1,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x7e,0x08,0xb1,0xd4,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
 0x7e,0x08,0xb1,0xd4,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13
 # GFX12-REAL16: v_cmpx_lt_i16_e64_dpp v1.h, v2.l op_sel:[1,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x7e,0x08,0xb1,0xd4,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
 # GFX12-FAKE16: v_cmpx_lt_i16_e64_dpp v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x7e,0x00,0xb1,0xd4,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
 
-# GFX11: v_cmpx_lt_i16_e64_dpp v255.l, v255.h op_sel:[0,1] row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0x7e,0x10,0xb1,0xd4,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 0x7e,0x10,0xb1,0xd4,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30
 # GFX12-REAL16: v_cmpx_lt_i16_e64_dpp v255.l, v255.h op_sel:[0,1] row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0x7e,0x10,0xb1,0xd4,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 # GFX12-FAKE16: v_cmpx_lt_i16_e64_dpp v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0x7e,0x00,0xb1,0xd4,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
@@ -2397,17 +2740,14 @@
 0x7e,0x00,0xb9,0xd4,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00
 # GFX12: v_cmpx_lt_u16_e64 0xfe0b, vcc_hi        ; encoding: [0x7e,0x00,0xb9,0xd4,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
 
-# GFX11: v_cmpx_lt_u16_e64_dpp v1.h, v2.h op_sel:[1,1] row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x18,0xb9,0xd4,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
 0x7e,0x18,0xb9,0xd4,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01
 # GFX12-REAL16: v_cmpx_lt_u16_e64_dpp v1.h, v2.h op_sel:[1,1] row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x18,0xb9,0xd4,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
 # GFX12-FAKE16: v_cmpx_lt_u16_e64_dpp v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x00,0xb9,0xd4,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
 
-# GFX11: v_cmpx_lt_u16_e64_dpp v1.h, v2.l op_sel:[1,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x7e,0x08,0xb9,0xd4,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
 0x7e,0x08,0xb9,0xd4,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13
 # GFX12-REAL16: v_cmpx_lt_u16_e64_dpp v1.h, v2.l op_sel:[1,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x7e,0x08,0xb9,0xd4,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
 # GFX12-FAKE16: v_cmpx_lt_u16_e64_dpp v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x7e,0x00,0xb9,0xd4,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
 
-# GFX11: v_cmpx_lt_u16_e64_dpp v255.l, v255.h op_sel:[0,1] row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0x7e,0x10,0xb9,0xd4,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 0x7e,0x10,0xb9,0xd4,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30
 # GFX12-REAL16: v_cmpx_lt_u16_e64_dpp v255.l, v255.h op_sel:[0,1] row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0x7e,0x10,0xb9,0xd4,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 # GFX12-FAKE16: v_cmpx_lt_u16_e64_dpp v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0x7e,0x00,0xb9,0xd4,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
@@ -2564,17 +2904,14 @@
 0x7e,0x00,0xb5,0xd4,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00
 # GFX12: v_cmpx_ne_i16_e64 0xfe0b, vcc_hi        ; encoding: [0x7e,0x00,0xb5,0xd4,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
 
-# GFX11: v_cmpx_ne_i16_e64_dpp v1.h, v2.h op_sel:[1,1] row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x18,0xb5,0xd4,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
 0x7e,0x18,0xb5,0xd4,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01
 # GFX12-REAL16: v_cmpx_ne_i16_e64_dpp v1.h, v2.h op_sel:[1,1] row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x18,0xb5,0xd4,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
 # GFX12-FAKE16: v_cmpx_ne_i16_e64_dpp v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x00,0xb5,0xd4,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
 
-# GFX11: v_cmpx_ne_i16_e64_dpp v1.h, v2.l op_sel:[1,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x7e,0x08,0xb5,0xd4,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
 0x7e,0x08,0xb5,0xd4,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13
 # GFX12-REAL16: v_cmpx_ne_i16_e64_dpp v1.h, v2.l op_sel:[1,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x7e,0x08,0xb5,0xd4,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
 # GFX12-FAKE16: v_cmpx_ne_i16_e64_dpp v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x7e,0x00,0xb5,0xd4,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
 
-# GFX11: v_cmpx_ne_i16_e64_dpp v255.l, v255.h op_sel:[0,1] row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0x7e,0x10,0xb5,0xd4,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 0x7e,0x10,0xb5,0xd4,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30
 # GFX12-REAL16: v_cmpx_ne_i16_e64_dpp v255.l, v255.h op_sel:[0,1] row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0x7e,0x10,0xb5,0xd4,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 # GFX12-FAKE16: v_cmpx_ne_i16_e64_dpp v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0x7e,0x00,0xb5,0xd4,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
@@ -2731,17 +3068,14 @@
 0x7e,0x00,0xbd,0xd4,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00
 # GFX12: v_cmpx_ne_u16_e64 0xfe0b, vcc_hi        ; encoding: [0x7e,0x00,0xbd,0xd4,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
 
-# GFX11: v_cmpx_ne_u16_e64_dpp v1.h, v2.h op_sel:[1,1] row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x18,0xbd,0xd4,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
 0x7e,0x18,0xbd,0xd4,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01
 # GFX12-REAL16: v_cmpx_ne_u16_e64_dpp v1.h, v2.h op_sel:[1,1] row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x18,0xbd,0xd4,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
 # GFX12-FAKE16: v_cmpx_ne_u16_e64_dpp v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x00,0xbd,0xd4,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
 
-# GFX11: v_cmpx_ne_u16_e64_dpp v1.h, v2.l op_sel:[1,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x7e,0x08,0xbd,0xd4,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
 0x7e,0x08,0xbd,0xd4,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13
 # GFX12-REAL16: v_cmpx_ne_u16_e64_dpp v1.h, v2.l op_sel:[1,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x7e,0x08,0xbd,0xd4,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
 # GFX12-FAKE16: v_cmpx_ne_u16_e64_dpp v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x7e,0x00,0xbd,0xd4,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
 
-# GFX11: v_cmpx_ne_u16_e64_dpp v255.l, v255.h op_sel:[0,1] row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0x7e,0x10,0xbd,0xd4,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 0x7e,0x10,0xbd,0xd4,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30
 # GFX12-REAL16: v_cmpx_ne_u16_e64_dpp v255.l, v255.h op_sel:[0,1] row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0x7e,0x10,0xbd,0xd4,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 # GFX12-FAKE16: v_cmpx_ne_u16_e64_dpp v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0x7e,0x00,0xbd,0xd4,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
@@ -2792,49 +3126,123 @@
 # GFX12: v_cmpx_ne_u32_e64_dpp v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0x7e,0x00,0xcd,0xd4,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 
 0x7e,0x00,0x8d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff
-# GFX12: v_cmpx_neq_f16_e64_dpp v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+# GFX12-REAL16: v_cmpx_neq_f16_e64_dpp v1.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+# GFX12-FAKE16: v_cmpx_neq_f16_e64_dpp v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
 
 0x7e,0x00,0x8d,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff
-# GFX12: v_cmpx_neq_f16_e64_dpp v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8d,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+# GFX12-REAL16: v_cmpx_neq_f16_e64_dpp v1.l, v2.l quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8d,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+# GFX12-FAKE16: v_cmpx_neq_f16_e64_dpp v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8d,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
 
 0x7e,0x00,0x8d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff
-# GFX12: v_cmpx_neq_f16_e64_dpp v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+# GFX12-REAL16: v_cmpx_neq_f16_e64_dpp v1.l, v2.l row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+# GFX12-FAKE16: v_cmpx_neq_f16_e64_dpp v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
 
 0x7e,0x00,0x8d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff
-# GFX12: v_cmpx_neq_f16_e64_dpp v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+# GFX12-REAL16: v_cmpx_neq_f16_e64_dpp v1.l, v2.l row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+# GFX12-FAKE16: v_cmpx_neq_f16_e64_dpp v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
 
 0x7e,0x00,0x8d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff
-# GFX12: v_cmpx_neq_f16_e64_dpp v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+# GFX12-REAL16: v_cmpx_neq_f16_e64_dpp v1.l, v2.l row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+# GFX12-FAKE16: v_cmpx_neq_f16_e64_dpp v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
 
 0x7e,0x00,0x8d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff
-# GFX12: v_cmpx_neq_f16_e64_dpp v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+# GFX12-REAL16: v_cmpx_neq_f16_e64_dpp v1.l, v2.l row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+# GFX12-FAKE16: v_cmpx_neq_f16_e64_dpp v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
 
 0x7e,0x00,0x8d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff
-# GFX12: v_cmpx_neq_f16_e64_dpp v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+# GFX12-REAL16: v_cmpx_neq_f16_e64_dpp v1.l, v2.l row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+# GFX12-FAKE16: v_cmpx_neq_f16_e64_dpp v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
 
 0x7e,0x00,0x8d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff
-# GFX12: v_cmpx_neq_f16_e64_dpp v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+# GFX12-REAL16: v_cmpx_neq_f16_e64_dpp v1.l, v2.l row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+# GFX12-FAKE16: v_cmpx_neq_f16_e64_dpp v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
 
 0x7e,0x00,0x8d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff
-# GFX12: v_cmpx_neq_f16_e64_dpp v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+# GFX12-REAL16: v_cmpx_neq_f16_e64_dpp v1.l, v2.l row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+# GFX12-FAKE16: v_cmpx_neq_f16_e64_dpp v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
 
 0x7e,0x00,0x8d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff
-# GFX12: v_cmpx_neq_f16_e64_dpp v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+# GFX12-REAL16: v_cmpx_neq_f16_e64_dpp v1.l, v2.l row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+# GFX12-FAKE16: v_cmpx_neq_f16_e64_dpp v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
 
 0x7e,0x00,0x8d,0xd4,0xfa,0x06,0x00,0x00,0x01,0x2f,0x01,0xff
-# GFX12: v_cmpx_neq_f16_e64_dpp v1, s3 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8d,0xd4,0xfa,0x06,0x00,0x00,0x01,0x2f,0x01,0xff]
+# GFX12-REAL16: v_cmpx_neq_f16_e64_dpp v1.l, s3 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8d,0xd4,0xfa,0x06,0x00,0x00,0x01,0x2f,0x01,0xff]
+# GFX12-FAKE16: v_cmpx_neq_f16_e64_dpp v1, s3 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8d,0xd4,0xfa,0x06,0x00,0x00,0x01,0x2f,0x01,0xff]
 
 0x7e,0x00,0x8d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff
-# GFX12: v_cmpx_neq_f16_e64_dpp v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+# GFX12-REAL16: v_cmpx_neq_f16_e64_dpp v1.l, v2.l row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+# GFX12-FAKE16: v_cmpx_neq_f16_e64_dpp v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 
 0x7e,0x01,0x8d,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01
-# GFX12: v_cmpx_neq_f16_e64_dpp |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x01,0x8d,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+# GFX12-REAL16: v_cmpx_neq_f16_e64_dpp |v1.l|, -v2.l row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x01,0x8d,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+# GFX12-FAKE16: v_cmpx_neq_f16_e64_dpp |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x01,0x8d,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
 
 0x7e,0x02,0x8d,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x01,0x13
-# GFX12: v_cmpx_neq_f16_e64_dpp -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x7e,0x02,0x8d,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x01,0x13]
+# GFX12-REAL16: v_cmpx_neq_f16_e64_dpp -v1.l, |v2.l| row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x7e,0x02,0x8d,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x01,0x13]
+# GFX12-FAKE16: v_cmpx_neq_f16_e64_dpp -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x7e,0x02,0x8d,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x01,0x13]
 
 0x7e,0x83,0x8d,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30
-# GFX12: v_cmpx_neq_f16_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0x7e,0x83,0x8d,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30]
+# GFX12-REAL16: v_cmpx_neq_f16_e64_dpp -|v255.l|, -|v255.l| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0x7e,0x83,0x8d,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30]
+# GFX12-FAKE16: v_cmpx_neq_f16_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0x7e,0x83,0x8d,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30]
+
+0x7e,0x08,0x8d,0xd4,0x01,0x05,0x02,0x00
+# GFX12-REAL16: v_cmpx_neq_f16_e64 v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x7e,0x08,0x8d,0xd4,0x01,0x05,0x02,0x00]
+# GFX12-FAKE16: v_cmpx_neq_f16_e64 v1, v2               ; encoding: [0x7e,0x00,0x8d,0xd4,0x01,0x05,0x02,0x00]
+
+0x7e,0x10,0x8d,0xd4,0xff,0xff,0x03,0x00
+# GFX12-REAL16: v_cmpx_neq_f16_e64 v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x7e,0x10,0x8d,0xd4,0xff,0xff,0x03,0x00]
+# GFX12-FAKE16: v_cmpx_neq_f16_e64 v255, v255           ; encoding: [0x7e,0x00,0x8d,0xd4,0xff,0xff,0x03,0x00]
+
+0x7e,0x00,0x8d,0xd4,0x01,0x04,0x00,0x00
+# GFX12: v_cmpx_neq_f16_e64 s1, s2               ; encoding: [0x7e,0x00,0x8d,0xd4,0x01,0x04,0x00,0x00]
+
+0x7e,0x00,0x8d,0xd4,0x69,0xd2,0x00,0x00
+# GFX12: v_cmpx_neq_f16_e64 s105, s105           ; encoding: [0x7e,0x00,0x8d,0xd4,0x69,0xd2,0x00,0x00]
+
+0x7e,0x00,0x8d,0xd4,0x6a,0xf6,0x00,0x00
+# GFX12: v_cmpx_neq_f16_e64 vcc_lo, ttmp15       ; encoding: [0x7e,0x00,0x8d,0xd4,0x6a,0xf6,0x00,0x00]
+
+0x7e,0x00,0x8d,0xd4,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00
+# GFX12: v_cmpx_neq_f16_e64 vcc_hi, 0xfe0b       ; encoding: [0x7e,0x00,0x8d,0xd4,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
+
+0x7e,0x00,0x8d,0xd4,0x7b,0xfa,0x01,0x00
+# GFX12: v_cmpx_neq_f16_e64 ttmp15, src_scc      ; encoding: [0x7e,0x00,0x8d,0xd4,0x7b,0xfa,0x01,0x00]
+
+0x7e,0x00,0x8d,0xd4,0x7d,0xe0,0x01,0x00
+# GFX12: v_cmpx_neq_f16_e64 m0, 0.5              ; encoding: [0x7e,0x00,0x8d,0xd4,0x7d,0xe0,0x01,0x00]
+
+0x7e,0x00,0x8d,0xd4,0x7e,0x82,0x01,0x00
+# GFX12: v_cmpx_neq_f16_e64 exec_lo, -1          ; encoding: [0x7e,0x00,0x8d,0xd4,0x7e,0x82,0x01,0x00]
+
+0x7e,0x01,0x8d,0xd4,0x7f,0xf8,0x00,0x00
+# GFX12: v_cmpx_neq_f16_e64 |exec_hi|, null      ; encoding: [0x7e,0x01,0x8d,0xd4,0x7f,0xf8,0x00,0x00]
+
+0x7e,0x00,0x8d,0xd4,0x7c,0xfc,0x00,0x00
+# GFX12: v_cmpx_neq_f16_e64 null, exec_lo        ; encoding: [0x7e,0x00,0x8d,0xd4,0x7c,0xfc,0x00,0x00]
+
+0x7e,0x00,0x8d,0xd4,0xc1,0xfe,0x00,0x00
+# GFX12: v_cmpx_neq_f16_e64 -1, exec_hi          ; encoding: [0x7e,0x00,0x8d,0xd4,0xc1,0xfe,0x00,0x00]
+
+0x7e,0x00,0x8d,0xd4,0xf0,0xfa,0x00,0x40
+# GFX12: v_cmpx_neq_f16_e64 0.5, -m0             ; encoding: [0x7e,0x00,0x8d,0xd4,0xf0,0xfa,0x00,0x40]
+
+0x7e,0x02,0x8d,0xd4,0xfd,0xd4,0x00,0x20
+# GFX12: v_cmpx_neq_f16_e64 -src_scc, |vcc_lo|   ; encoding: [0x7e,0x02,0x8d,0xd4,0xfd,0xd4,0x00,0x20]
+
+0x7e,0x83,0x8d,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00
+# GFX12: v_cmpx_neq_f16_e64 -|0xfe0b|, -|vcc_hi| clamp ; encoding: [0x7e,0x83,0x8d,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00]
+
+0x7e,0x19,0x8d,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01
+# GFX12-REAL16: v_cmpx_neq_f16_e64_dpp |v1.h|, -v2.h op_sel:[1,1] row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x19,0x8d,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+# GFX12-FAKE16: v_cmpx_neq_f16_e64_dpp |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x01,0x8d,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+
+0x7e,0x0a,0x8d,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x01,0x13
+# GFX12-REAL16: v_cmpx_neq_f16_e64_dpp -v1.h, |v2.l| op_sel:[1,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x7e,0x0a,0x8d,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x01,0x13]
+# GFX12-FAKE16: v_cmpx_neq_f16_e64_dpp -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x7e,0x02,0x8d,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x01,0x13]
+
+0x7e,0x93,0x8d,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30
+# GFX12-REAL16: v_cmpx_neq_f16_e64_dpp -|v255.l|, -|v255.h| op_sel:[0,1] clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0x7e,0x93,0x8d,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30]
+# GFX12-FAKE16: v_cmpx_neq_f16_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0x7e,0x83,0x8d,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30]
 
 0x7e,0x00,0x9d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff
 # GFX12: v_cmpx_neq_f32_e64_dpp v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x9d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
@@ -2882,49 +3290,123 @@
 # GFX12: v_cmpx_neq_f32_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0x7e,0x83,0x9d,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30]
 
 0x7e,0x00,0x89,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff
-# GFX12: v_cmpx_nge_f16_e64_dpp v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x89,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+# GFX12-REAL16: v_cmpx_nge_f16_e64_dpp v1.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x89,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+# GFX12-FAKE16: v_cmpx_nge_f16_e64_dpp v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x89,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
 
 0x7e,0x00,0x89,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff
-# GFX12: v_cmpx_nge_f16_e64_dpp v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x89,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+# GFX12-REAL16: v_cmpx_nge_f16_e64_dpp v1.l, v2.l quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x89,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+# GFX12-FAKE16: v_cmpx_nge_f16_e64_dpp v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x89,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
 
 0x7e,0x00,0x89,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff
-# GFX12: v_cmpx_nge_f16_e64_dpp v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x89,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+# GFX12-REAL16: v_cmpx_nge_f16_e64_dpp v1.l, v2.l row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x89,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+# GFX12-FAKE16: v_cmpx_nge_f16_e64_dpp v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x89,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
 
 0x7e,0x00,0x89,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff
-# GFX12: v_cmpx_nge_f16_e64_dpp v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x89,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+# GFX12-REAL16: v_cmpx_nge_f16_e64_dpp v1.l, v2.l row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x89,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+# GFX12-FAKE16: v_cmpx_nge_f16_e64_dpp v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x89,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
 
 0x7e,0x00,0x89,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff
-# GFX12: v_cmpx_nge_f16_e64_dpp v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x89,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+# GFX12-REAL16: v_cmpx_nge_f16_e64_dpp v1.l, v2.l row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x89,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+# GFX12-FAKE16: v_cmpx_nge_f16_e64_dpp v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x89,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
 
 0x7e,0x00,0x89,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff
-# GFX12: v_cmpx_nge_f16_e64_dpp v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x89,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+# GFX12-REAL16: v_cmpx_nge_f16_e64_dpp v1.l, v2.l row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x89,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+# GFX12-FAKE16: v_cmpx_nge_f16_e64_dpp v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x89,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
 
 0x7e,0x00,0x89,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff
-# GFX12: v_cmpx_nge_f16_e64_dpp v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x89,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+# GFX12-REAL16: v_cmpx_nge_f16_e64_dpp v1.l, v2.l row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x89,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+# GFX12-FAKE16: v_cmpx_nge_f16_e64_dpp v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x89,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
 
 0x7e,0x00,0x89,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff
-# GFX12: v_cmpx_nge_f16_e64_dpp v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x89,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+# GFX12-REAL16: v_cmpx_nge_f16_e64_dpp v1.l, v2.l row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x89,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+# GFX12-FAKE16: v_cmpx_nge_f16_e64_dpp v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x89,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
 
 0x7e,0x00,0x89,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff
-# GFX12: v_cmpx_nge_f16_e64_dpp v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x89,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+# GFX12-REAL16: v_cmpx_nge_f16_e64_dpp v1.l, v2.l row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x89,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+# GFX12-FAKE16: v_cmpx_nge_f16_e64_dpp v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x89,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
 
 0x7e,0x00,0x89,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff
-# GFX12: v_cmpx_nge_f16_e64_dpp v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x89,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+# GFX12-REAL16: v_cmpx_nge_f16_e64_dpp v1.l, v2.l row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x89,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+# GFX12-FAKE16: v_cmpx_nge_f16_e64_dpp v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x89,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
 
 0x7e,0x00,0x89,0xd4,0xfa,0x06,0x00,0x00,0x01,0x2f,0x01,0xff
-# GFX12: v_cmpx_nge_f16_e64_dpp v1, s3 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x89,0xd4,0xfa,0x06,0x00,0x00,0x01,0x2f,0x01,0xff]
+# GFX12-REAL16: v_cmpx_nge_f16_e64_dpp v1.l, s3 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x89,0xd4,0xfa,0x06,0x00,0x00,0x01,0x2f,0x01,0xff]
+# GFX12-FAKE16: v_cmpx_nge_f16_e64_dpp v1, s3 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x89,0xd4,0xfa,0x06,0x00,0x00,0x01,0x2f,0x01,0xff]
 
 0x7e,0x00,0x89,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff
-# GFX12: v_cmpx_nge_f16_e64_dpp v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x89,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+# GFX12-REAL16: v_cmpx_nge_f16_e64_dpp v1.l, v2.l row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x89,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+# GFX12-FAKE16: v_cmpx_nge_f16_e64_dpp v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x89,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 
 0x7e,0x01,0x89,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01
-# GFX12: v_cmpx_nge_f16_e64_dpp |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x01,0x89,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+# GFX12-REAL16: v_cmpx_nge_f16_e64_dpp |v1.l|, -v2.l row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x01,0x89,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+# GFX12-FAKE16: v_cmpx_nge_f16_e64_dpp |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x01,0x89,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
 
 0x7e,0x02,0x89,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x01,0x13
-# GFX12: v_cmpx_nge_f16_e64_dpp -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x7e,0x02,0x89,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x01,0x13]
+# GFX12-REAL16: v_cmpx_nge_f16_e64_dpp -v1.l, |v2.l| row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x7e,0x02,0x89,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x01,0x13]
+# GFX12-FAKE16: v_cmpx_nge_f16_e64_dpp -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x7e,0x02,0x89,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x01,0x13]
 
 0x7e,0x83,0x89,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30
-# GFX12: v_cmpx_nge_f16_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0x7e,0x83,0x89,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30]
+# GFX12-REAL16: v_cmpx_nge_f16_e64_dpp -|v255.l|, -|v255.l| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0x7e,0x83,0x89,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30]
+# GFX12-FAKE16: v_cmpx_nge_f16_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0x7e,0x83,0x89,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30]
+
+0x7e,0x08,0x89,0xd4,0x01,0x05,0x02,0x00
+# GFX12-REAL16: v_cmpx_nge_f16_e64 v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x7e,0x08,0x89,0xd4,0x01,0x05,0x02,0x00]
+# GFX12-FAKE16: v_cmpx_nge_f16_e64 v1, v2               ; encoding: [0x7e,0x00,0x89,0xd4,0x01,0x05,0x02,0x00]
+
+0x7e,0x10,0x89,0xd4,0xff,0xff,0x03,0x00
+# GFX12-REAL16: v_cmpx_nge_f16_e64 v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x7e,0x10,0x89,0xd4,0xff,0xff,0x03,0x00]
+# GFX12-FAKE16: v_cmpx_nge_f16_e64 v255, v255           ; encoding: [0x7e,0x00,0x89,0xd4,0xff,0xff,0x03,0x00]
+
+0x7e,0x00,0x89,0xd4,0x01,0x04,0x00,0x00
+# GFX12: v_cmpx_nge_f16_e64 s1, s2               ; encoding: [0x7e,0x00,0x89,0xd4,0x01,0x04,0x00,0x00]
+
+0x7e,0x00,0x89,0xd4,0x69,0xd2,0x00,0x00
+# GFX12: v_cmpx_nge_f16_e64 s105, s105           ; encoding: [0x7e,0x00,0x89,0xd4,0x69,0xd2,0x00,0x00]
+
+0x7e,0x00,0x89,0xd4,0x6a,0xf6,0x00,0x00
+# GFX12: v_cmpx_nge_f16_e64 vcc_lo, ttmp15       ; encoding: [0x7e,0x00,0x89,0xd4,0x6a,0xf6,0x00,0x00]
+
+0x7e,0x00,0x89,0xd4,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00
+# GFX12: v_cmpx_nge_f16_e64 vcc_hi, 0xfe0b       ; encoding: [0x7e,0x00,0x89,0xd4,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
+
+0x7e,0x00,0x89,0xd4,0x7b,0xfa,0x01,0x00
+# GFX12: v_cmpx_nge_f16_e64 ttmp15, src_scc      ; encoding: [0x7e,0x00,0x89,0xd4,0x7b,0xfa,0x01,0x00]
+
+0x7e,0x00,0x89,0xd4,0x7d,0xe0,0x01,0x00
+# GFX12: v_cmpx_nge_f16_e64 m0, 0.5              ; encoding: [0x7e,0x00,0x89,0xd4,0x7d,0xe0,0x01,0x00]
+
+0x7e,0x00,0x89,0xd4,0x7e,0x82,0x01,0x00
+# GFX12: v_cmpx_nge_f16_e64 exec_lo, -1          ; encoding: [0x7e,0x00,0x89,0xd4,0x7e,0x82,0x01,0x00]
+
+0x7e,0x01,0x89,0xd4,0x7f,0xf8,0x00,0x00
+# GFX12: v_cmpx_nge_f16_e64 |exec_hi|, null      ; encoding: [0x7e,0x01,0x89,0xd4,0x7f,0xf8,0x00,0x00]
+
+0x7e,0x00,0x89,0xd4,0x7c,0xfc,0x00,0x00
+# GFX12: v_cmpx_nge_f16_e64 null, exec_lo        ; encoding: [0x7e,0x00,0x89,0xd4,0x7c,0xfc,0x00,0x00]
+
+0x7e,0x00,0x89,0xd4,0xc1,0xfe,0x00,0x00
+# GFX12: v_cmpx_nge_f16_e64 -1, exec_hi          ; encoding: [0x7e,0x00,0x89,0xd4,0xc1,0xfe,0x00,0x00]
+
+0x7e,0x00,0x89,0xd4,0xf0,0xfa,0x00,0x40
+# GFX12: v_cmpx_nge_f16_e64 0.5, -m0             ; encoding: [0x7e,0x00,0x89,0xd4,0xf0,0xfa,0x00,0x40]
+
+0x7e,0x02,0x89,0xd4,0xfd,0xd4,0x00,0x20
+# GFX12: v_cmpx_nge_f16_e64 -src_scc, |vcc_lo|   ; encoding: [0x7e,0x02,0x89,0xd4,0xfd,0xd4,0x00,0x20]
+
+0x7e,0x83,0x89,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00
+# GFX12: v_cmpx_nge_f16_e64 -|0xfe0b|, -|vcc_hi| clamp ; encoding: [0x7e,0x83,0x89,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00]
+
+0x7e,0x19,0x89,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01
+# GFX12-REAL16: v_cmpx_nge_f16_e64_dpp |v1.h|, -v2.h op_sel:[1,1] row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x19,0x89,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+# GFX12-FAKE16: v_cmpx_nge_f16_e64_dpp |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x01,0x89,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+
+0x7e,0x0a,0x89,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x01,0x13
+# GFX12-REAL16: v_cmpx_nge_f16_e64_dpp -v1.h, |v2.l| op_sel:[1,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x7e,0x0a,0x89,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x01,0x13]
+# GFX12-FAKE16: v_cmpx_nge_f16_e64_dpp -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x7e,0x02,0x89,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x01,0x13]
+
+0x7e,0x93,0x89,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30
+# GFX12-REAL16: v_cmpx_nge_f16_e64_dpp -|v255.l|, -|v255.h| op_sel:[0,1] clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0x7e,0x93,0x89,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30]
+# GFX12-FAKE16: v_cmpx_nge_f16_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0x7e,0x83,0x89,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30]
 
 0x7e,0x00,0x99,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff
 # GFX12: v_cmpx_nge_f32_e64_dpp v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x99,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
@@ -2972,49 +3454,123 @@
 # GFX12: v_cmpx_nge_f32_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0x7e,0x83,0x99,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30]
 
 0x7e,0x00,0x8b,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff
-# GFX12: v_cmpx_ngt_f16_e64_dpp v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8b,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+# GFX12-REAL16: v_cmpx_ngt_f16_e64_dpp v1.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8b,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+# GFX12-FAKE16: v_cmpx_ngt_f16_e64_dpp v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8b,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
 
 0x7e,0x00,0x8b,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff
-# GFX12: v_cmpx_ngt_f16_e64_dpp v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8b,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+# GFX12-REAL16: v_cmpx_ngt_f16_e64_dpp v1.l, v2.l quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8b,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+# GFX12-FAKE16: v_cmpx_ngt_f16_e64_dpp v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8b,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
 
 0x7e,0x00,0x8b,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff
-# GFX12: v_cmpx_ngt_f16_e64_dpp v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8b,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+# GFX12-REAL16: v_cmpx_ngt_f16_e64_dpp v1.l, v2.l row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8b,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+# GFX12-FAKE16: v_cmpx_ngt_f16_e64_dpp v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8b,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
 
 0x7e,0x00,0x8b,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff
-# GFX12: v_cmpx_ngt_f16_e64_dpp v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8b,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+# GFX12-REAL16: v_cmpx_ngt_f16_e64_dpp v1.l, v2.l row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8b,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+# GFX12-FAKE16: v_cmpx_ngt_f16_e64_dpp v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8b,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
 
 0x7e,0x00,0x8b,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff
-# GFX12: v_cmpx_ngt_f16_e64_dpp v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8b,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+# GFX12-REAL16: v_cmpx_ngt_f16_e64_dpp v1.l, v2.l row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8b,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+# GFX12-FAKE16: v_cmpx_ngt_f16_e64_dpp v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8b,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
 
 0x7e,0x00,0x8b,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff
-# GFX12: v_cmpx_ngt_f16_e64_dpp v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8b,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+# GFX12-REAL16: v_cmpx_ngt_f16_e64_dpp v1.l, v2.l row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8b,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+# GFX12-FAKE16: v_cmpx_ngt_f16_e64_dpp v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8b,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
 
 0x7e,0x00,0x8b,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff
-# GFX12: v_cmpx_ngt_f16_e64_dpp v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8b,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+# GFX12-REAL16: v_cmpx_ngt_f16_e64_dpp v1.l, v2.l row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8b,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+# GFX12-FAKE16: v_cmpx_ngt_f16_e64_dpp v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8b,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
 
 0x7e,0x00,0x8b,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff
-# GFX12: v_cmpx_ngt_f16_e64_dpp v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8b,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+# GFX12-REAL16: v_cmpx_ngt_f16_e64_dpp v1.l, v2.l row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8b,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+# GFX12-FAKE16: v_cmpx_ngt_f16_e64_dpp v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8b,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
 
 0x7e,0x00,0x8b,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff
-# GFX12: v_cmpx_ngt_f16_e64_dpp v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8b,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+# GFX12-REAL16: v_cmpx_ngt_f16_e64_dpp v1.l, v2.l row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8b,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+# GFX12-FAKE16: v_cmpx_ngt_f16_e64_dpp v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8b,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
 
 0x7e,0x00,0x8b,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff
-# GFX12: v_cmpx_ngt_f16_e64_dpp v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8b,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+# GFX12-REAL16: v_cmpx_ngt_f16_e64_dpp v1.l, v2.l row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8b,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+# GFX12-FAKE16: v_cmpx_ngt_f16_e64_dpp v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8b,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
 
 0x7e,0x00,0x8b,0xd4,0xfa,0x06,0x00,0x00,0x01,0x2f,0x01,0xff
-# GFX12: v_cmpx_ngt_f16_e64_dpp v1, s3 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8b,0xd4,0xfa,0x06,0x00,0x00,0x01,0x2f,0x01,0xff]
+# GFX12-REAL16: v_cmpx_ngt_f16_e64_dpp v1.l, s3 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8b,0xd4,0xfa,0x06,0x00,0x00,0x01,0x2f,0x01,0xff]
+# GFX12-FAKE16: v_cmpx_ngt_f16_e64_dpp v1, s3 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8b,0xd4,0xfa,0x06,0x00,0x00,0x01,0x2f,0x01,0xff]
 
 0x7e,0x00,0x8b,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff
-# GFX12: v_cmpx_ngt_f16_e64_dpp v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8b,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+# GFX12-REAL16: v_cmpx_ngt_f16_e64_dpp v1.l, v2.l row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8b,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+# GFX12-FAKE16: v_cmpx_ngt_f16_e64_dpp v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8b,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 
 0x7e,0x01,0x8b,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01
-# GFX12: v_cmpx_ngt_f16_e64_dpp |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x01,0x8b,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+# GFX12-REAL16: v_cmpx_ngt_f16_e64_dpp |v1.l|, -v2.l row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x01,0x8b,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+# GFX12-FAKE16: v_cmpx_ngt_f16_e64_dpp |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x01,0x8b,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
 
 0x7e,0x02,0x8b,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x01,0x13
-# GFX12: v_cmpx_ngt_f16_e64_dpp -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x7e,0x02,0x8b,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x01,0x13]
+# GFX12-REAL16: v_cmpx_ngt_f16_e64_dpp -v1.l, |v2.l| row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x7e,0x02,0x8b,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x01,0x13]
+# GFX12-FAKE16: v_cmpx_ngt_f16_e64_dpp -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x7e,0x02,0x8b,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x01,0x13]
 
 0x7e,0x83,0x8b,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30
-# GFX12: v_cmpx_ngt_f16_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0x7e,0x83,0x8b,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30]
+# GFX12-REAL16: v_cmpx_ngt_f16_e64_dpp -|v255.l|, -|v255.l| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0x7e,0x83,0x8b,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30]
+# GFX12-FAKE16: v_cmpx_ngt_f16_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0x7e,0x83,0x8b,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30]
+
+0x7e,0x08,0x8b,0xd4,0x01,0x05,0x02,0x00
+# GFX12-REAL16: v_cmpx_ngt_f16_e64 v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x7e,0x08,0x8b,0xd4,0x01,0x05,0x02,0x00]
+# GFX12-FAKE16: v_cmpx_ngt_f16_e64 v1, v2               ; encoding: [0x7e,0x00,0x8b,0xd4,0x01,0x05,0x02,0x00]
+
+0x7e,0x10,0x8b,0xd4,0xff,0xff,0x03,0x00
+# GFX12-REAL16: v_cmpx_ngt_f16_e64 v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x7e,0x10,0x8b,0xd4,0xff,0xff,0x03,0x00]
+# GFX12-FAKE16: v_cmpx_ngt_f16_e64 v255, v255           ; encoding: [0x7e,0x00,0x8b,0xd4,0xff,0xff,0x03,0x00]
+
+0x7e,0x00,0x8b,0xd4,0x01,0x04,0x00,0x00
+# GFX12: v_cmpx_ngt_f16_e64 s1, s2               ; encoding: [0x7e,0x00,0x8b,0xd4,0x01,0x04,0x00,0x00]
+
+0x7e,0x00,0x8b,0xd4,0x69,0xd2,0x00,0x00
+# GFX12: v_cmpx_ngt_f16_e64 s105, s105           ; encoding: [0x7e,0x00,0x8b,0xd4,0x69,0xd2,0x00,0x00]
+
+0x7e,0x00,0x8b,0xd4,0x6a,0xf6,0x00,0x00
+# GFX12: v_cmpx_ngt_f16_e64 vcc_lo, ttmp15       ; encoding: [0x7e,0x00,0x8b,0xd4,0x6a,0xf6,0x00,0x00]
+
+0x7e,0x00,0x8b,0xd4,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00
+# GFX12: v_cmpx_ngt_f16_e64 vcc_hi, 0xfe0b       ; encoding: [0x7e,0x00,0x8b,0xd4,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
+
+0x7e,0x00,0x8b,0xd4,0x7b,0xfa,0x01,0x00
+# GFX12: v_cmpx_ngt_f16_e64 ttmp15, src_scc      ; encoding: [0x7e,0x00,0x8b,0xd4,0x7b,0xfa,0x01,0x00]
+
+0x7e,0x00,0x8b,0xd4,0x7d,0xe0,0x01,0x00
+# GFX12: v_cmpx_ngt_f16_e64 m0, 0.5              ; encoding: [0x7e,0x00,0x8b,0xd4,0x7d,0xe0,0x01,0x00]
+
+0x7e,0x00,0x8b,0xd4,0x7e,0x82,0x01,0x00
+# GFX12: v_cmpx_ngt_f16_e64 exec_lo, -1          ; encoding: [0x7e,0x00,0x8b,0xd4,0x7e,0x82,0x01,0x00]
+
+0x7e,0x01,0x8b,0xd4,0x7f,0xf8,0x00,0x00
+# GFX12: v_cmpx_ngt_f16_e64 |exec_hi|, null      ; encoding: [0x7e,0x01,0x8b,0xd4,0x7f,0xf8,0x00,0x00]
+
+0x7e,0x00,0x8b,0xd4,0x7c,0xfc,0x00,0x00
+# GFX12: v_cmpx_ngt_f16_e64 null, exec_lo        ; encoding: [0x7e,0x00,0x8b,0xd4,0x7c,0xfc,0x00,0x00]
+
+0x7e,0x00,0x8b,0xd4,0xc1,0xfe,0x00,0x00
+# GFX12: v_cmpx_ngt_f16_e64 -1, exec_hi          ; encoding: [0x7e,0x00,0x8b,0xd4,0xc1,0xfe,0x00,0x00]
+
+0x7e,0x00,0x8b,0xd4,0xf0,0xfa,0x00,0x40
+# GFX12: v_cmpx_ngt_f16_e64 0.5, -m0             ; encoding: [0x7e,0x00,0x8b,0xd4,0xf0,0xfa,0x00,0x40]
+
+0x7e,0x02,0x8b,0xd4,0xfd,0xd4,0x00,0x20
+# GFX12: v_cmpx_ngt_f16_e64 -src_scc, |vcc_lo|   ; encoding: [0x7e,0x02,0x8b,0xd4,0xfd,0xd4,0x00,0x20]
+
+0x7e,0x83,0x8b,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00
+# GFX12: v_cmpx_ngt_f16_e64 -|0xfe0b|, -|vcc_hi| clamp ; encoding: [0x7e,0x83,0x8b,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00]
+
+0x7e,0x19,0x8b,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01
+# GFX12-REAL16: v_cmpx_ngt_f16_e64_dpp |v1.h|, -v2.h op_sel:[1,1] row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x19,0x8b,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+# GFX12-FAKE16: v_cmpx_ngt_f16_e64_dpp |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x01,0x8b,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+
+0x7e,0x0a,0x8b,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x01,0x13
+# GFX12-REAL16: v_cmpx_ngt_f16_e64_dpp -v1.h, |v2.l| op_sel:[1,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x7e,0x0a,0x8b,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x01,0x13]
+# GFX12-FAKE16: v_cmpx_ngt_f16_e64_dpp -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x7e,0x02,0x8b,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x01,0x13]
+
+0x7e,0x93,0x8b,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30
+# GFX12-REAL16: v_cmpx_ngt_f16_e64_dpp -|v255.l|, -|v255.h| op_sel:[0,1] clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0x7e,0x93,0x8b,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30]
+# GFX12-FAKE16: v_cmpx_ngt_f16_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0x7e,0x83,0x8b,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30]
 
 0x7e,0x00,0x9b,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff
 # GFX12: v_cmpx_ngt_f32_e64_dpp v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x9b,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
@@ -3062,49 +3618,123 @@
 # GFX12: v_cmpx_ngt_f32_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0x7e,0x83,0x9b,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30]
 
 0x7e,0x00,0x8c,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff
-# GFX12: v_cmpx_nle_f16_e64_dpp v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8c,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+# GFX12-REAL16: v_cmpx_nle_f16_e64_dpp v1.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8c,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+# GFX12-FAKE16: v_cmpx_nle_f16_e64_dpp v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8c,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
 
 0x7e,0x00,0x8c,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff
-# GFX12: v_cmpx_nle_f16_e64_dpp v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8c,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+# GFX12-REAL16: v_cmpx_nle_f16_e64_dpp v1.l, v2.l quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8c,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+# GFX12-FAKE16: v_cmpx_nle_f16_e64_dpp v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8c,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
 
 0x7e,0x00,0x8c,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff
-# GFX12: v_cmpx_nle_f16_e64_dpp v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8c,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+# GFX12-REAL16: v_cmpx_nle_f16_e64_dpp v1.l, v2.l row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8c,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+# GFX12-FAKE16: v_cmpx_nle_f16_e64_dpp v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8c,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
 
 0x7e,0x00,0x8c,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff
-# GFX12: v_cmpx_nle_f16_e64_dpp v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8c,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+# GFX12-REAL16: v_cmpx_nle_f16_e64_dpp v1.l, v2.l row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8c,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+# GFX12-FAKE16: v_cmpx_nle_f16_e64_dpp v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8c,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
 
 0x7e,0x00,0x8c,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff
-# GFX12: v_cmpx_nle_f16_e64_dpp v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8c,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+# GFX12-REAL16: v_cmpx_nle_f16_e64_dpp v1.l, v2.l row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8c,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+# GFX12-FAKE16: v_cmpx_nle_f16_e64_dpp v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8c,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
 
 0x7e,0x00,0x8c,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff
-# GFX12: v_cmpx_nle_f16_e64_dpp v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8c,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+# GFX12-REAL16: v_cmpx_nle_f16_e64_dpp v1.l, v2.l row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8c,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+# GFX12-FAKE16: v_cmpx_nle_f16_e64_dpp v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8c,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
 
 0x7e,0x00,0x8c,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff
-# GFX12: v_cmpx_nle_f16_e64_dpp v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8c,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+# GFX12-REAL16: v_cmpx_nle_f16_e64_dpp v1.l, v2.l row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8c,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+# GFX12-FAKE16: v_cmpx_nle_f16_e64_dpp v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8c,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
 
 0x7e,0x00,0x8c,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff
-# GFX12: v_cmpx_nle_f16_e64_dpp v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8c,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+# GFX12-REAL16: v_cmpx_nle_f16_e64_dpp v1.l, v2.l row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8c,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+# GFX12-FAKE16: v_cmpx_nle_f16_e64_dpp v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8c,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
 
 0x7e,0x00,0x8c,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff
-# GFX12: v_cmpx_nle_f16_e64_dpp v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8c,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+# GFX12-REAL16: v_cmpx_nle_f16_e64_dpp v1.l, v2.l row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8c,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+# GFX12-FAKE16: v_cmpx_nle_f16_e64_dpp v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8c,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
 
 0x7e,0x00,0x8c,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff
-# GFX12: v_cmpx_nle_f16_e64_dpp v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8c,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+# GFX12-REAL16: v_cmpx_nle_f16_e64_dpp v1.l, v2.l row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8c,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+# GFX12-FAKE16: v_cmpx_nle_f16_e64_dpp v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8c,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
 
 0x7e,0x00,0x8c,0xd4,0xfa,0x06,0x00,0x00,0x01,0x2f,0x01,0xff
-# GFX12: v_cmpx_nle_f16_e64_dpp v1, s3 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8c,0xd4,0xfa,0x06,0x00,0x00,0x01,0x2f,0x01,0xff]
+# GFX12-REAL16: v_cmpx_nle_f16_e64_dpp v1.l, s3 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8c,0xd4,0xfa,0x06,0x00,0x00,0x01,0x2f,0x01,0xff]
+# GFX12-FAKE16: v_cmpx_nle_f16_e64_dpp v1, s3 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8c,0xd4,0xfa,0x06,0x00,0x00,0x01,0x2f,0x01,0xff]
 
 0x7e,0x00,0x8c,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff
-# GFX12: v_cmpx_nle_f16_e64_dpp v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8c,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+# GFX12-REAL16: v_cmpx_nle_f16_e64_dpp v1.l, v2.l row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8c,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+# GFX12-FAKE16: v_cmpx_nle_f16_e64_dpp v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8c,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 
 0x7e,0x01,0x8c,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01
-# GFX12: v_cmpx_nle_f16_e64_dpp |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x01,0x8c,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+# GFX12-REAL16: v_cmpx_nle_f16_e64_dpp |v1.l|, -v2.l row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x01,0x8c,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+# GFX12-FAKE16: v_cmpx_nle_f16_e64_dpp |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x01,0x8c,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
 
 0x7e,0x02,0x8c,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x01,0x13
-# GFX12: v_cmpx_nle_f16_e64_dpp -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x7e,0x02,0x8c,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x01,0x13]
+# GFX12-REAL16: v_cmpx_nle_f16_e64_dpp -v1.l, |v2.l| row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x7e,0x02,0x8c,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x01,0x13]
+# GFX12-FAKE16: v_cmpx_nle_f16_e64_dpp -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x7e,0x02,0x8c,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x01,0x13]
 
 0x7e,0x83,0x8c,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30
-# GFX12: v_cmpx_nle_f16_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0x7e,0x83,0x8c,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30]
+# GFX12-REAL16: v_cmpx_nle_f16_e64_dpp -|v255.l|, -|v255.l| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0x7e,0x83,0x8c,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30]
+# GFX12-FAKE16: v_cmpx_nle_f16_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0x7e,0x83,0x8c,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30]
+
+0x7e,0x08,0x8c,0xd4,0x01,0x05,0x02,0x00
+# GFX12-REAL16: v_cmpx_nle_f16_e64 v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x7e,0x08,0x8c,0xd4,0x01,0x05,0x02,0x00]
+# GFX12-FAKE16: v_cmpx_nle_f16_e64 v1, v2               ; encoding: [0x7e,0x00,0x8c,0xd4,0x01,0x05,0x02,0x00]
+
+0x7e,0x10,0x8c,0xd4,0xff,0xff,0x03,0x00
+# GFX12-REAL16: v_cmpx_nle_f16_e64 v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x7e,0x10,0x8c,0xd4,0xff,0xff,0x03,0x00]
+# GFX12-FAKE16: v_cmpx_nle_f16_e64 v255, v255           ; encoding: [0x7e,0x00,0x8c,0xd4,0xff,0xff,0x03,0x00]
+
+0x7e,0x00,0x8c,0xd4,0x01,0x04,0x00,0x00
+# GFX12: v_cmpx_nle_f16_e64 s1, s2               ; encoding: [0x7e,0x00,0x8c,0xd4,0x01,0x04,0x00,0x00]
+
+0x7e,0x00,0x8c,0xd4,0x69,0xd2,0x00,0x00
+# GFX12: v_cmpx_nle_f16_e64 s105, s105           ; encoding: [0x7e,0x00,0x8c,0xd4,0x69,0xd2,0x00,0x00]
+
+0x7e,0x00,0x8c,0xd4,0x6a,0xf6,0x00,0x00
+# GFX12: v_cmpx_nle_f16_e64 vcc_lo, ttmp15       ; encoding: [0x7e,0x00,0x8c,0xd4,0x6a,0xf6,0x00,0x00]
+
+0x7e,0x00,0x8c,0xd4,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00
+# GFX12: v_cmpx_nle_f16_e64 vcc_hi, 0xfe0b       ; encoding: [0x7e,0x00,0x8c,0xd4,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
+
+0x7e,0x00,0x8c,0xd4,0x7b,0xfa,0x01,0x00
+# GFX12: v_cmpx_nle_f16_e64 ttmp15, src_scc      ; encoding: [0x7e,0x00,0x8c,0xd4,0x7b,0xfa,0x01,0x00]
+
+0x7e,0x00,0x8c,0xd4,0x7d,0xe0,0x01,0x00
+# GFX12: v_cmpx_nle_f16_e64 m0, 0.5              ; encoding: [0x7e,0x00,0x8c,0xd4,0x7d,0xe0,0x01,0x00]
+
+0x7e,0x00,0x8c,0xd4,0x7e,0x82,0x01,0x00
+# GFX12: v_cmpx_nle_f16_e64 exec_lo, -1          ; encoding: [0x7e,0x00,0x8c,0xd4,0x7e,0x82,0x01,0x00]
+
+0x7e,0x01,0x8c,0xd4,0x7f,0xf8,0x00,0x00
+# GFX12: v_cmpx_nle_f16_e64 |exec_hi|, null      ; encoding: [0x7e,0x01,0x8c,0xd4,0x7f,0xf8,0x00,0x00]
+
+0x7e,0x00,0x8c,0xd4,0x7c,0xfc,0x00,0x00
+# GFX12: v_cmpx_nle_f16_e64 null, exec_lo        ; encoding: [0x7e,0x00,0x8c,0xd4,0x7c,0xfc,0x00,0x00]
+
+0x7e,0x00,0x8c,0xd4,0xc1,0xfe,0x00,0x00
+# GFX12: v_cmpx_nle_f16_e64 -1, exec_hi          ; encoding: [0x7e,0x00,0x8c,0xd4,0xc1,0xfe,0x00,0x00]
+
+0x7e,0x00,0x8c,0xd4,0xf0,0xfa,0x00,0x40
+# GFX12: v_cmpx_nle_f16_e64 0.5, -m0             ; encoding: [0x7e,0x00,0x8c,0xd4,0xf0,0xfa,0x00,0x40]
+
+0x7e,0x02,0x8c,0xd4,0xfd,0xd4,0x00,0x20
+# GFX12: v_cmpx_nle_f16_e64 -src_scc, |vcc_lo|   ; encoding: [0x7e,0x02,0x8c,0xd4,0xfd,0xd4,0x00,0x20]
+
+0x7e,0x83,0x8c,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00
+# GFX12: v_cmpx_nle_f16_e64 -|0xfe0b|, -|vcc_hi| clamp ; encoding: [0x7e,0x83,0x8c,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00]
+
+0x7e,0x19,0x8c,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01
+# GFX12-REAL16: v_cmpx_nle_f16_e64_dpp |v1.h|, -v2.h op_sel:[1,1] row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x19,0x8c,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+# GFX12-FAKE16: v_cmpx_nle_f16_e64_dpp |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x01,0x8c,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+
+0x7e,0x0a,0x8c,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x01,0x13
+# GFX12-REAL16: v_cmpx_nle_f16_e64_dpp -v1.h, |v2.l| op_sel:[1,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x7e,0x0a,0x8c,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x01,0x13]
+# GFX12-FAKE16: v_cmpx_nle_f16_e64_dpp -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x7e,0x02,0x8c,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x01,0x13]
+
+0x7e,0x93,0x8c,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30
+# GFX12-REAL16: v_cmpx_nle_f16_e64_dpp -|v255.l|, -|v255.h| op_sel:[0,1] clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0x7e,0x93,0x8c,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30]
+# GFX12-FAKE16: v_cmpx_nle_f16_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0x7e,0x83,0x8c,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30]
 
 0x7e,0x00,0x9c,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff
 # GFX12: v_cmpx_nle_f32_e64_dpp v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x9c,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
@@ -3152,49 +3782,123 @@
 # GFX12: v_cmpx_nle_f32_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0x7e,0x83,0x9c,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30]
 
 0x7e,0x00,0x8a,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff
-# GFX12: v_cmpx_nlg_f16_e64_dpp v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8a,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+# GFX12-REAL16: v_cmpx_nlg_f16_e64_dpp v1.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8a,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+# GFX12-FAKE16: v_cmpx_nlg_f16_e64_dpp v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8a,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
 
 0x7e,0x00,0x8a,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff
-# GFX12: v_cmpx_nlg_f16_e64_dpp v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8a,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+# GFX12-REAL16: v_cmpx_nlg_f16_e64_dpp v1.l, v2.l quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8a,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+# GFX12-FAKE16: v_cmpx_nlg_f16_e64_dpp v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8a,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
 
 0x7e,0x00,0x8a,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff
-# GFX12: v_cmpx_nlg_f16_e64_dpp v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8a,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+# GFX12-REAL16: v_cmpx_nlg_f16_e64_dpp v1.l, v2.l row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8a,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+# GFX12-FAKE16: v_cmpx_nlg_f16_e64_dpp v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8a,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
 
 0x7e,0x00,0x8a,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff
-# GFX12: v_cmpx_nlg_f16_e64_dpp v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8a,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+# GFX12-REAL16: v_cmpx_nlg_f16_e64_dpp v1.l, v2.l row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8a,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+# GFX12-FAKE16: v_cmpx_nlg_f16_e64_dpp v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8a,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
 
 0x7e,0x00,0x8a,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff
-# GFX12: v_cmpx_nlg_f16_e64_dpp v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8a,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+# GFX12-REAL16: v_cmpx_nlg_f16_e64_dpp v1.l, v2.l row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8a,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+# GFX12-FAKE16: v_cmpx_nlg_f16_e64_dpp v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8a,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
 
 0x7e,0x00,0x8a,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff
-# GFX12: v_cmpx_nlg_f16_e64_dpp v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8a,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+# GFX12-REAL16: v_cmpx_nlg_f16_e64_dpp v1.l, v2.l row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8a,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+# GFX12-FAKE16: v_cmpx_nlg_f16_e64_dpp v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8a,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
 
 0x7e,0x00,0x8a,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff
-# GFX12: v_cmpx_nlg_f16_e64_dpp v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8a,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+# GFX12-REAL16: v_cmpx_nlg_f16_e64_dpp v1.l, v2.l row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8a,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+# GFX12-FAKE16: v_cmpx_nlg_f16_e64_dpp v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8a,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
 
 0x7e,0x00,0x8a,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff
-# GFX12: v_cmpx_nlg_f16_e64_dpp v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8a,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+# GFX12-REAL16: v_cmpx_nlg_f16_e64_dpp v1.l, v2.l row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8a,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+# GFX12-FAKE16: v_cmpx_nlg_f16_e64_dpp v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8a,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
 
 0x7e,0x00,0x8a,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff
-# GFX12: v_cmpx_nlg_f16_e64_dpp v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8a,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+# GFX12-REAL16: v_cmpx_nlg_f16_e64_dpp v1.l, v2.l row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8a,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+# GFX12-FAKE16: v_cmpx_nlg_f16_e64_dpp v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8a,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
 
 0x7e,0x00,0x8a,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff
-# GFX12: v_cmpx_nlg_f16_e64_dpp v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8a,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+# GFX12-REAL16: v_cmpx_nlg_f16_e64_dpp v1.l, v2.l row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8a,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+# GFX12-FAKE16: v_cmpx_nlg_f16_e64_dpp v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8a,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
 
 0x7e,0x00,0x8a,0xd4,0xfa,0x06,0x00,0x00,0x01,0x2f,0x01,0xff
-# GFX12: v_cmpx_nlg_f16_e64_dpp v1, s3 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8a,0xd4,0xfa,0x06,0x00,0x00,0x01,0x2f,0x01,0xff]
+# GFX12-REAL16: v_cmpx_nlg_f16_e64_dpp v1.l, s3 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8a,0xd4,0xfa,0x06,0x00,0x00,0x01,0x2f,0x01,0xff]
+# GFX12-FAKE16: v_cmpx_nlg_f16_e64_dpp v1, s3 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8a,0xd4,0xfa,0x06,0x00,0x00,0x01,0x2f,0x01,0xff]
 
 0x7e,0x00,0x8a,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff
-# GFX12: v_cmpx_nlg_f16_e64_dpp v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8a,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+# GFX12-REAL16: v_cmpx_nlg_f16_e64_dpp v1.l, v2.l row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8a,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+# GFX12-FAKE16: v_cmpx_nlg_f16_e64_dpp v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8a,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 
 0x7e,0x01,0x8a,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01
-# GFX12: v_cmpx_nlg_f16_e64_dpp |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x01,0x8a,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+# GFX12-REAL16: v_cmpx_nlg_f16_e64_dpp |v1.l|, -v2.l row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x01,0x8a,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+# GFX12-FAKE16: v_cmpx_nlg_f16_e64_dpp |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x01,0x8a,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
 
 0x7e,0x02,0x8a,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x01,0x13
-# GFX12: v_cmpx_nlg_f16_e64_dpp -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x7e,0x02,0x8a,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x01,0x13]
+# GFX12-REAL16: v_cmpx_nlg_f16_e64_dpp -v1.l, |v2.l| row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x7e,0x02,0x8a,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x01,0x13]
+# GFX12-FAKE16: v_cmpx_nlg_f16_e64_dpp -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x7e,0x02,0x8a,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x01,0x13]
 
 0x7e,0x83,0x8a,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30
-# GFX12: v_cmpx_nlg_f16_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0x7e,0x83,0x8a,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30]
+# GFX12-REAL16: v_cmpx_nlg_f16_e64_dpp -|v255.l|, -|v255.l| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0x7e,0x83,0x8a,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30]
+# GFX12-FAKE16: v_cmpx_nlg_f16_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0x7e,0x83,0x8a,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30]
+
+0x7e,0x08,0x8a,0xd4,0x01,0x05,0x02,0x00
+# GFX12-REAL16: v_cmpx_nlg_f16_e64 v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x7e,0x08,0x8a,0xd4,0x01,0x05,0x02,0x00]
+# GFX12-FAKE16: v_cmpx_nlg_f16_e64 v1, v2               ; encoding: [0x7e,0x00,0x8a,0xd4,0x01,0x05,0x02,0x00]
+
+0x7e,0x10,0x8a,0xd4,0xff,0xff,0x03,0x00
+# GFX12-REAL16: v_cmpx_nlg_f16_e64 v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x7e,0x10,0x8a,0xd4,0xff,0xff,0x03,0x00]
+# GFX12-FAKE16: v_cmpx_nlg_f16_e64 v255, v255           ; encoding: [0x7e,0x00,0x8a,0xd4,0xff,0xff,0x03,0x00]
+
+0x7e,0x00,0x8a,0xd4,0x01,0x04,0x00,0x00
+# GFX12: v_cmpx_nlg_f16_e64 s1, s2               ; encoding: [0x7e,0x00,0x8a,0xd4,0x01,0x04,0x00,0x00]
+
+0x7e,0x00,0x8a,0xd4,0x69,0xd2,0x00,0x00
+# GFX12: v_cmpx_nlg_f16_e64 s105, s105           ; encoding: [0x7e,0x00,0x8a,0xd4,0x69,0xd2,0x00,0x00]
+
+0x7e,0x00,0x8a,0xd4,0x6a,0xf6,0x00,0x00
+# GFX12: v_cmpx_nlg_f16_e64 vcc_lo, ttmp15       ; encoding: [0x7e,0x00,0x8a,0xd4,0x6a,0xf6,0x00,0x00]
+
+0x7e,0x00,0x8a,0xd4,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00
+# GFX12: v_cmpx_nlg_f16_e64 vcc_hi, 0xfe0b       ; encoding: [0x7e,0x00,0x8a,0xd4,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
+
+0x7e,0x00,0x8a,0xd4,0x7b,0xfa,0x01,0x00
+# GFX12: v_cmpx_nlg_f16_e64 ttmp15, src_scc      ; encoding: [0x7e,0x00,0x8a,0xd4,0x7b,0xfa,0x01,0x00]
+
+0x7e,0x00,0x8a,0xd4,0x7d,0xe0,0x01,0x00
+# GFX12: v_cmpx_nlg_f16_e64 m0, 0.5              ; encoding: [0x7e,0x00,0x8a,0xd4,0x7d,0xe0,0x01,0x00]
+
+0x7e,0x00,0x8a,0xd4,0x7e,0x82,0x01,0x00
+# GFX12: v_cmpx_nlg_f16_e64 exec_lo, -1          ; encoding: [0x7e,0x00,0x8a,0xd4,0x7e,0x82,0x01,0x00]
+
+0x7e,0x01,0x8a,0xd4,0x7f,0xf8,0x00,0x00
+# GFX12: v_cmpx_nlg_f16_e64 |exec_hi|, null      ; encoding: [0x7e,0x01,0x8a,0xd4,0x7f,0xf8,0x00,0x00]
+
+0x7e,0x00,0x8a,0xd4,0x7c,0xfc,0x00,0x00
+# GFX12: v_cmpx_nlg_f16_e64 null, exec_lo        ; encoding: [0x7e,0x00,0x8a,0xd4,0x7c,0xfc,0x00,0x00]
+
+0x7e,0x00,0x8a,0xd4,0xc1,0xfe,0x00,0x00
+# GFX12: v_cmpx_nlg_f16_e64 -1, exec_hi          ; encoding: [0x7e,0x00,0x8a,0xd4,0xc1,0xfe,0x00,0x00]
+
+0x7e,0x00,0x8a,0xd4,0xf0,0xfa,0x00,0x40
+# GFX12: v_cmpx_nlg_f16_e64 0.5, -m0             ; encoding: [0x7e,0x00,0x8a,0xd4,0xf0,0xfa,0x00,0x40]
+
+0x7e,0x02,0x8a,0xd4,0xfd,0xd4,0x00,0x20
+# GFX12: v_cmpx_nlg_f16_e64 -src_scc, |vcc_lo|   ; encoding: [0x7e,0x02,0x8a,0xd4,0xfd,0xd4,0x00,0x20]
+
+0x7e,0x83,0x8a,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00
+# GFX12: v_cmpx_nlg_f16_e64 -|0xfe0b|, -|vcc_hi| clamp ; encoding: [0x7e,0x83,0x8a,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00]
+
+0x7e,0x19,0x8a,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01
+# GFX12-REAL16: v_cmpx_nlg_f16_e64_dpp |v1.h|, -v2.h op_sel:[1,1] row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x19,0x8a,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+# GFX12-FAKE16: v_cmpx_nlg_f16_e64_dpp |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x01,0x8a,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+
+0x7e,0x0a,0x8a,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x01,0x13
+# GFX12-REAL16: v_cmpx_nlg_f16_e64_dpp -v1.h, |v2.l| op_sel:[1,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x7e,0x0a,0x8a,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x01,0x13]
+# GFX12-FAKE16: v_cmpx_nlg_f16_e64_dpp -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x7e,0x02,0x8a,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x01,0x13]
+
+0x7e,0x93,0x8a,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30
+# GFX12-REAL16: v_cmpx_nlg_f16_e64_dpp -|v255.l|, -|v255.h| op_sel:[0,1] clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0x7e,0x93,0x8a,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30]
+# GFX12-FAKE16: v_cmpx_nlg_f16_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0x7e,0x83,0x8a,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30]
 
 0x7e,0x00,0x9a,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff
 # GFX12: v_cmpx_nlg_f32_e64_dpp v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x9a,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
@@ -3242,49 +3946,123 @@
 # GFX12: v_cmpx_nlg_f32_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0x7e,0x83,0x9a,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30]
 
 0x7e,0x00,0x8e,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff
-# GFX12: v_cmpx_nlt_f16_e64_dpp v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8e,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+# GFX12-REAL16: v_cmpx_nlt_f16_e64_dpp v1.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8e,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+# GFX12-FAKE16: v_cmpx_nlt_f16_e64_dpp v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8e,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
 
 0x7e,0x00,0x8e,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff
-# GFX12: v_cmpx_nlt_f16_e64_dpp v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8e,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+# GFX12-REAL16: v_cmpx_nlt_f16_e64_dpp v1.l, v2.l quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8e,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+# GFX12-FAKE16: v_cmpx_nlt_f16_e64_dpp v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8e,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
 
 0x7e,0x00,0x8e,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff
-# GFX12: v_cmpx_nlt_f16_e64_dpp v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8e,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+# GFX12-REAL16: v_cmpx_nlt_f16_e64_dpp v1.l, v2.l row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8e,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+# GFX12-FAKE16: v_cmpx_nlt_f16_e64_dpp v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8e,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
 
 0x7e,0x00,0x8e,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff
-# GFX12: v_cmpx_nlt_f16_e64_dpp v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8e,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+# GFX12-REAL16: v_cmpx_nlt_f16_e64_dpp v1.l, v2.l row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8e,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+# GFX12-FAKE16: v_cmpx_nlt_f16_e64_dpp v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8e,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
 
 0x7e,0x00,0x8e,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff
-# GFX12: v_cmpx_nlt_f16_e64_dpp v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8e,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+# GFX12-REAL16: v_cmpx_nlt_f16_e64_dpp v1.l, v2.l row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8e,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+# GFX12-FAKE16: v_cmpx_nlt_f16_e64_dpp v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8e,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
 
 0x7e,0x00,0x8e,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff
-# GFX12: v_cmpx_nlt_f16_e64_dpp v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8e,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+# GFX12-REAL16: v_cmpx_nlt_f16_e64_dpp v1.l, v2.l row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8e,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+# GFX12-FAKE16: v_cmpx_nlt_f16_e64_dpp v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8e,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
 
 0x7e,0x00,0x8e,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff
-# GFX12: v_cmpx_nlt_f16_e64_dpp v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8e,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+# GFX12-REAL16: v_cmpx_nlt_f16_e64_dpp v1.l, v2.l row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8e,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+# GFX12-FAKE16: v_cmpx_nlt_f16_e64_dpp v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8e,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
 
 0x7e,0x00,0x8e,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff
-# GFX12: v_cmpx_nlt_f16_e64_dpp v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8e,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+# GFX12-REAL16: v_cmpx_nlt_f16_e64_dpp v1.l, v2.l row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8e,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+# GFX12-FAKE16: v_cmpx_nlt_f16_e64_dpp v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8e,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
 
 0x7e,0x00,0x8e,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff
-# GFX12: v_cmpx_nlt_f16_e64_dpp v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8e,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+# GFX12-REAL16: v_cmpx_nlt_f16_e64_dpp v1.l, v2.l row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8e,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+# GFX12-FAKE16: v_cmpx_nlt_f16_e64_dpp v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8e,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
 
 0x7e,0x00,0x8e,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff
-# GFX12: v_cmpx_nlt_f16_e64_dpp v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8e,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+# GFX12-REAL16: v_cmpx_nlt_f16_e64_dpp v1.l, v2.l row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8e,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+# GFX12-FAKE16: v_cmpx_nlt_f16_e64_dpp v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8e,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
 
 0x7e,0x00,0x8e,0xd4,0xfa,0x06,0x00,0x00,0x01,0x2f,0x01,0xff
-# GFX12: v_cmpx_nlt_f16_e64_dpp v1, s3 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8e,0xd4,0xfa,0x06,0x00,0x00,0x01,0x2f,0x01,0xff]
+# GFX12-REAL16: v_cmpx_nlt_f16_e64_dpp v1.l, s3 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8e,0xd4,0xfa,0x06,0x00,0x00,0x01,0x2f,0x01,0xff]
+# GFX12-FAKE16: v_cmpx_nlt_f16_e64_dpp v1, s3 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8e,0xd4,0xfa,0x06,0x00,0x00,0x01,0x2f,0x01,0xff]
 
 0x7e,0x00,0x8e,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff
-# GFX12: v_cmpx_nlt_f16_e64_dpp v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8e,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+# GFX12-REAL16: v_cmpx_nlt_f16_e64_dpp v1.l, v2.l row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8e,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+# GFX12-FAKE16: v_cmpx_nlt_f16_e64_dpp v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8e,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 
 0x7e,0x01,0x8e,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01
-# GFX12: v_cmpx_nlt_f16_e64_dpp |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x01,0x8e,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+# GFX12-REAL16: v_cmpx_nlt_f16_e64_dpp |v1.l|, -v2.l row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x01,0x8e,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+# GFX12-FAKE16: v_cmpx_nlt_f16_e64_dpp |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x01,0x8e,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
 
 0x7e,0x02,0x8e,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x01,0x13
-# GFX12: v_cmpx_nlt_f16_e64_dpp -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x7e,0x02,0x8e,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x01,0x13]
+# GFX12-REAL16: v_cmpx_nlt_f16_e64_dpp -v1.l, |v2.l| row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x7e,0x02,0x8e,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x01,0x13]
+# GFX12-FAKE16: v_cmpx_nlt_f16_e64_dpp -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x7e,0x02,0x8e,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x01,0x13]
 
 0x7e,0x83,0x8e,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30
-# GFX12: v_cmpx_nlt_f16_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0x7e,0x83,0x8e,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30]
+# GFX12-REAL16: v_cmpx_nlt_f16_e64_dpp -|v255.l|, -|v255.l| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0x7e,0x83,0x8e,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30]
+# GFX12-FAKE16: v_cmpx_nlt_f16_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0x7e,0x83,0x8e,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30]
+
+0x7e,0x08,0x8e,0xd4,0x01,0x05,0x02,0x00
+# GFX12-REAL16: v_cmpx_nlt_f16_e64 v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x7e,0x08,0x8e,0xd4,0x01,0x05,0x02,0x00]
+# GFX12-FAKE16: v_cmpx_nlt_f16_e64 v1, v2               ; encoding: [0x7e,0x00,0x8e,0xd4,0x01,0x05,0x02,0x00]
+
+0x7e,0x10,0x8e,0xd4,0xff,0xff,0x03,0x00
+# GFX12-REAL16: v_cmpx_nlt_f16_e64 v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x7e,0x10,0x8e,0xd4,0xff,0xff,0x03,0x00]
+# GFX12-FAKE16: v_cmpx_nlt_f16_e64 v255, v255           ; encoding: [0x7e,0x00,0x8e,0xd4,0xff,0xff,0x03,0x00]
+
+0x7e,0x00,0x8e,0xd4,0x01,0x04,0x00,0x00
+# GFX12: v_cmpx_nlt_f16_e64 s1, s2               ; encoding: [0x7e,0x00,0x8e,0xd4,0x01,0x04,0x00,0x00]
+
+0x7e,0x00,0x8e,0xd4,0x69,0xd2,0x00,0x00
+# GFX12: v_cmpx_nlt_f16_e64 s105, s105           ; encoding: [0x7e,0x00,0x8e,0xd4,0x69,0xd2,0x00,0x00]
+
+0x7e,0x00,0x8e,0xd4,0x6a,0xf6,0x00,0x00
+# GFX12: v_cmpx_nlt_f16_e64 vcc_lo, ttmp15       ; encoding: [0x7e,0x00,0x8e,0xd4,0x6a,0xf6,0x00,0x00]
+
+0x7e,0x00,0x8e,0xd4,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00
+# GFX12: v_cmpx_nlt_f16_e64 vcc_hi, 0xfe0b       ; encoding: [0x7e,0x00,0x8e,0xd4,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
+
+0x7e,0x00,0x8e,0xd4,0x7b,0xfa,0x01,0x00
+# GFX12: v_cmpx_nlt_f16_e64 ttmp15, src_scc      ; encoding: [0x7e,0x00,0x8e,0xd4,0x7b,0xfa,0x01,0x00]
+
+0x7e,0x00,0x8e,0xd4,0x7d,0xe0,0x01,0x00
+# GFX12: v_cmpx_nlt_f16_e64 m0, 0.5              ; encoding: [0x7e,0x00,0x8e,0xd4,0x7d,0xe0,0x01,0x00]
+
+0x7e,0x00,0x8e,0xd4,0x7e,0x82,0x01,0x00
+# GFX12: v_cmpx_nlt_f16_e64 exec_lo, -1          ; encoding: [0x7e,0x00,0x8e,0xd4,0x7e,0x82,0x01,0x00]
+
+0x7e,0x01,0x8e,0xd4,0x7f,0xf8,0x00,0x00
+# GFX12: v_cmpx_nlt_f16_e64 |exec_hi|, null      ; encoding: [0x7e,0x01,0x8e,0xd4,0x7f,0xf8,0x00,0x00]
+
+0x7e,0x00,0x8e,0xd4,0x7c,0xfc,0x00,0x00
+# GFX12: v_cmpx_nlt_f16_e64 null, exec_lo        ; encoding: [0x7e,0x00,0x8e,0xd4,0x7c,0xfc,0x00,0x00]
+
+0x7e,0x00,0x8e,0xd4,0xc1,0xfe,0x00,0x00
+# GFX12: v_cmpx_nlt_f16_e64 -1, exec_hi          ; encoding: [0x7e,0x00,0x8e,0xd4,0xc1,0xfe,0x00,0x00]
+
+0x7e,0x00,0x8e,0xd4,0xf0,0xfa,0x00,0x40
+# GFX12: v_cmpx_nlt_f16_e64 0.5, -m0             ; encoding: [0x7e,0x00,0x8e,0xd4,0xf0,0xfa,0x00,0x40]
+
+0x7e,0x02,0x8e,0xd4,0xfd,0xd4,0x00,0x20
+# GFX12: v_cmpx_nlt_f16_e64 -src_scc, |vcc_lo|   ; encoding: [0x7e,0x02,0x8e,0xd4,0xfd,0xd4,0x00,0x20]
+
+0x7e,0x83,0x8e,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00
+# GFX12: v_cmpx_nlt_f16_e64 -|0xfe0b|, -|vcc_hi| clamp ; encoding: [0x7e,0x83,0x8e,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00]
+
+0x7e,0x19,0x8e,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01
+# GFX12-REAL16: v_cmpx_nlt_f16_e64_dpp |v1.h|, -v2.h op_sel:[1,1] row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x19,0x8e,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+# GFX12-FAKE16: v_cmpx_nlt_f16_e64_dpp |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x01,0x8e,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+
+0x7e,0x0a,0x8e,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x01,0x13
+# GFX12-REAL16: v_cmpx_nlt_f16_e64_dpp -v1.h, |v2.l| op_sel:[1,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x7e,0x0a,0x8e,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x01,0x13]
+# GFX12-FAKE16: v_cmpx_nlt_f16_e64_dpp -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x7e,0x02,0x8e,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x01,0x13]
+
+0x7e,0x93,0x8e,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30
+# GFX12-REAL16: v_cmpx_nlt_f16_e64_dpp -|v255.l|, -|v255.h| op_sel:[0,1] clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0x7e,0x93,0x8e,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30]
+# GFX12-FAKE16: v_cmpx_nlt_f16_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0x7e,0x83,0x8e,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30]
 
 0x7e,0x00,0x9e,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff
 # GFX12: v_cmpx_nlt_f32_e64_dpp v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x9e,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
@@ -3332,49 +4110,123 @@
 # GFX12: v_cmpx_nlt_f32_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0x7e,0x83,0x9e,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30]
 
 0x7e,0x00,0x87,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff
-# GFX12: v_cmpx_o_f16_e64_dpp v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x87,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+# GFX12-REAL16: v_cmpx_o_f16_e64_dpp v1.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x87,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+# GFX12-FAKE16: v_cmpx_o_f16_e64_dpp v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x87,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
 
 0x7e,0x00,0x87,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff
-# GFX12: v_cmpx_o_f16_e64_dpp v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x87,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+# GFX12-REAL16: v_cmpx_o_f16_e64_dpp v1.l, v2.l quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x87,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+# GFX12-FAKE16: v_cmpx_o_f16_e64_dpp v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x87,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
 
 0x7e,0x00,0x87,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff
-# GFX12: v_cmpx_o_f16_e64_dpp v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x87,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+# GFX12-REAL16: v_cmpx_o_f16_e64_dpp v1.l, v2.l row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x87,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+# GFX12-FAKE16: v_cmpx_o_f16_e64_dpp v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x87,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
 
 0x7e,0x00,0x87,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff
-# GFX12: v_cmpx_o_f16_e64_dpp v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x87,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+# GFX12-REAL16: v_cmpx_o_f16_e64_dpp v1.l, v2.l row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x87,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+# GFX12-FAKE16: v_cmpx_o_f16_e64_dpp v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x87,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
 
 0x7e,0x00,0x87,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff
-# GFX12: v_cmpx_o_f16_e64_dpp v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x87,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+# GFX12-REAL16: v_cmpx_o_f16_e64_dpp v1.l, v2.l row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x87,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+# GFX12-FAKE16: v_cmpx_o_f16_e64_dpp v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x87,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
 
 0x7e,0x00,0x87,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff
-# GFX12: v_cmpx_o_f16_e64_dpp v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x87,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+# GFX12-REAL16: v_cmpx_o_f16_e64_dpp v1.l, v2.l row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x87,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+# GFX12-FAKE16: v_cmpx_o_f16_e64_dpp v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x87,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
 
 0x7e,0x00,0x87,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff
-# GFX12: v_cmpx_o_f16_e64_dpp v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x87,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+# GFX12-REAL16: v_cmpx_o_f16_e64_dpp v1.l, v2.l row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x87,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+# GFX12-FAKE16: v_cmpx_o_f16_e64_dpp v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x87,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
 
 0x7e,0x00,0x87,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff
-# GFX12: v_cmpx_o_f16_e64_dpp v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x87,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+# GFX12-REAL16: v_cmpx_o_f16_e64_dpp v1.l, v2.l row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x87,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+# GFX12-FAKE16: v_cmpx_o_f16_e64_dpp v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x87,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
 
 0x7e,0x00,0x87,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff
-# GFX12: v_cmpx_o_f16_e64_dpp v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x87,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+# GFX12-REAL16: v_cmpx_o_f16_e64_dpp v1.l, v2.l row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x87,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+# GFX12-FAKE16: v_cmpx_o_f16_e64_dpp v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x87,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
 
 0x7e,0x00,0x87,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff
-# GFX12: v_cmpx_o_f16_e64_dpp v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x87,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+# GFX12-REAL16: v_cmpx_o_f16_e64_dpp v1.l, v2.l row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x87,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+# GFX12-FAKE16: v_cmpx_o_f16_e64_dpp v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x87,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
 
 0x7e,0x00,0x87,0xd4,0xfa,0x06,0x00,0x00,0x01,0x2f,0x01,0xff
-# GFX12: v_cmpx_o_f16_e64_dpp v1, s3 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x87,0xd4,0xfa,0x06,0x00,0x00,0x01,0x2f,0x01,0xff]
+# GFX12-REAL16: v_cmpx_o_f16_e64_dpp v1.l, s3 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x87,0xd4,0xfa,0x06,0x00,0x00,0x01,0x2f,0x01,0xff]
+# GFX12-FAKE16: v_cmpx_o_f16_e64_dpp v1, s3 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x87,0xd4,0xfa,0x06,0x00,0x00,0x01,0x2f,0x01,0xff]
 
 0x7e,0x00,0x87,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff
-# GFX12: v_cmpx_o_f16_e64_dpp v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x87,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+# GFX12-REAL16: v_cmpx_o_f16_e64_dpp v1.l, v2.l row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x87,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+# GFX12-FAKE16: v_cmpx_o_f16_e64_dpp v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x87,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 
 0x7e,0x01,0x87,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01
-# GFX12: v_cmpx_o_f16_e64_dpp |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x01,0x87,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+# GFX12-REAL16: v_cmpx_o_f16_e64_dpp |v1.l|, -v2.l row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x01,0x87,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+# GFX12-FAKE16: v_cmpx_o_f16_e64_dpp |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x01,0x87,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
 
 0x7e,0x02,0x87,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x01,0x13
-# GFX12: v_cmpx_o_f16_e64_dpp -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x7e,0x02,0x87,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x01,0x13]
+# GFX12-REAL16: v_cmpx_o_f16_e64_dpp -v1.l, |v2.l| row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x7e,0x02,0x87,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x01,0x13]
+# GFX12-FAKE16: v_cmpx_o_f16_e64_dpp -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x7e,0x02,0x87,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x01,0x13]
 
 0x7e,0x83,0x87,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30
-# GFX12: v_cmpx_o_f16_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0x7e,0x83,0x87,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30]
+# GFX12-REAL16: v_cmpx_o_f16_e64_dpp -|v255.l|, -|v255.l| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0x7e,0x83,0x87,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30]
+# GFX12-FAKE16: v_cmpx_o_f16_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0x7e,0x83,0x87,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30]
+
+0x7e,0x08,0x87,0xd4,0x01,0x05,0x02,0x00
+# GFX12-REAL16: v_cmpx_o_f16_e64 v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x7e,0x08,0x87,0xd4,0x01,0x05,0x02,0x00]
+# GFX12-FAKE16: v_cmpx_o_f16_e64 v1, v2                 ; encoding: [0x7e,0x00,0x87,0xd4,0x01,0x05,0x02,0x00]
+
+0x7e,0x10,0x87,0xd4,0xff,0xff,0x03,0x00
+# GFX12-REAL16: v_cmpx_o_f16_e64 v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x7e,0x10,0x87,0xd4,0xff,0xff,0x03,0x00]
+# GFX12-FAKE16: v_cmpx_o_f16_e64 v255, v255             ; encoding: [0x7e,0x00,0x87,0xd4,0xff,0xff,0x03,0x00]
+
+0x7e,0x00,0x87,0xd4,0x01,0x04,0x00,0x00
+# GFX12: v_cmpx_o_f16_e64 s1, s2                 ; encoding: [0x7e,0x00,0x87,0xd4,0x01,0x04,0x00,0x00]
+
+0x7e,0x00,0x87,0xd4,0x69,0xd2,0x00,0x00
+# GFX12: v_cmpx_o_f16_e64 s105, s105             ; encoding: [0x7e,0x00,0x87,0xd4,0x69,0xd2,0x00,0x00]
+
+0x7e,0x00,0x87,0xd4,0x6a,0xf6,0x00,0x00
+# GFX12: v_cmpx_o_f16_e64 vcc_lo, ttmp15         ; encoding: [0x7e,0x00,0x87,0xd4,0x6a,0xf6,0x00,0x00]
+
+0x7e,0x00,0x87,0xd4,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00
+# GFX12: v_cmpx_o_f16_e64 vcc_hi, 0xfe0b         ; encoding: [0x7e,0x00,0x87,0xd4,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
+
+0x7e,0x00,0x87,0xd4,0x7b,0xfa,0x01,0x00
+# GFX12: v_cmpx_o_f16_e64 ttmp15, src_scc        ; encoding: [0x7e,0x00,0x87,0xd4,0x7b,0xfa,0x01,0x00]
+
+0x7e,0x00,0x87,0xd4,0x7d,0xe0,0x01,0x00
+# GFX12: v_cmpx_o_f16_e64 m0, 0.5                ; encoding: [0x7e,0x00,0x87,0xd4,0x7d,0xe0,0x01,0x00]
+
+0x7e,0x00,0x87,0xd4,0x7e,0x82,0x01,0x00
+# GFX12: v_cmpx_o_f16_e64 exec_lo, -1            ; encoding: [0x7e,0x00,0x87,0xd4,0x7e,0x82,0x01,0x00]
+
+0x7e,0x01,0x87,0xd4,0x7f,0xf8,0x00,0x00
+# GFX12: v_cmpx_o_f16_e64 |exec_hi|, null        ; encoding: [0x7e,0x01,0x87,0xd4,0x7f,0xf8,0x00,0x00]
+
+0x7e,0x00,0x87,0xd4,0x7c,0xfc,0x00,0x00
+# GFX12: v_cmpx_o_f16_e64 null, exec_lo          ; encoding: [0x7e,0x00,0x87,0xd4,0x7c,0xfc,0x00,0x00]
+
+0x7e,0x00,0x87,0xd4,0xc1,0xfe,0x00,0x00
+# GFX12: v_cmpx_o_f16_e64 -1, exec_hi            ; encoding: [0x7e,0x00,0x87,0xd4,0xc1,0xfe,0x00,0x00]
+
+0x7e,0x00,0x87,0xd4,0xf0,0xfa,0x00,0x40
+# GFX12: v_cmpx_o_f16_e64 0.5, -m0               ; encoding: [0x7e,0x00,0x87,0xd4,0xf0,0xfa,0x00,0x40]
+
+0x7e,0x02,0x87,0xd4,0xfd,0xd4,0x00,0x20
+# GFX12: v_cmpx_o_f16_e64 -src_scc, |vcc_lo|     ; encoding: [0x7e,0x02,0x87,0xd4,0xfd,0xd4,0x00,0x20]
+
+0x7e,0x83,0x87,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00
+# GFX12: v_cmpx_o_f16_e64 -|0xfe0b|, -|vcc_hi| clamp ; encoding: [0x7e,0x83,0x87,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00]
+
+0x7e,0x19,0x87,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01
+# GFX12-REAL16: v_cmpx_o_f16_e64_dpp |v1.h|, -v2.h op_sel:[1,1] row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x19,0x87,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+# GFX12-FAKE16: v_cmpx_o_f16_e64_dpp |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x01,0x87,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+
+0x7e,0x0a,0x87,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x01,0x13
+# GFX12-REAL16: v_cmpx_o_f16_e64_dpp -v1.h, |v2.l| op_sel:[1,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x7e,0x0a,0x87,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x01,0x13]
+# GFX12-FAKE16: v_cmpx_o_f16_e64_dpp -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x7e,0x02,0x87,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x01,0x13]
+
+0x7e,0x93,0x87,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30
+# GFX12-REAL16: v_cmpx_o_f16_e64_dpp -|v255.l|, -|v255.h| op_sel:[0,1] clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0x7e,0x93,0x87,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30]
+# GFX12-FAKE16: v_cmpx_o_f16_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0x7e,0x83,0x87,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30]
 
 0x7e,0x00,0x97,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff
 # GFX12: v_cmpx_o_f32_e64_dpp v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x97,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
@@ -3422,49 +4274,123 @@
 # GFX12: v_cmpx_o_f32_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0x7e,0x83,0x97,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30]
 
 0x7e,0x00,0x88,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff
-# GFX12: v_cmpx_u_f16_e64_dpp v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x88,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+# GFX12-REAL16: v_cmpx_u_f16_e64_dpp v1.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x88,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+# GFX12-FAKE16: v_cmpx_u_f16_e64_dpp v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x88,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
 
 0x7e,0x00,0x88,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff
-# GFX12: v_cmpx_u_f16_e64_dpp v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x88,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+# GFX12-REAL16: v_cmpx_u_f16_e64_dpp v1.l, v2.l quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x88,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+# GFX12-FAKE16: v_cmpx_u_f16_e64_dpp v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x88,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
 
 0x7e,0x00,0x88,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff
-# GFX12: v_cmpx_u_f16_e64_dpp v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x88,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+# GFX12-REAL16: v_cmpx_u_f16_e64_dpp v1.l, v2.l row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x88,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+# GFX12-FAKE16: v_cmpx_u_f16_e64_dpp v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x88,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
 
 0x7e,0x00,0x88,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff
-# GFX12: v_cmpx_u_f16_e64_dpp v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x88,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+# GFX12-REAL16: v_cmpx_u_f16_e64_dpp v1.l, v2.l row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x88,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+# GFX12-FAKE16: v_cmpx_u_f16_e64_dpp v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x88,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
 
 0x7e,0x00,0x88,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff
-# GFX12: v_cmpx_u_f16_e64_dpp v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x88,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+# GFX12-REAL16: v_cmpx_u_f16_e64_dpp v1.l, v2.l row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x88,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+# GFX12-FAKE16: v_cmpx_u_f16_e64_dpp v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x88,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
 
 0x7e,0x00,0x88,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff
-# GFX12: v_cmpx_u_f16_e64_dpp v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x88,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+# GFX12-REAL16: v_cmpx_u_f16_e64_dpp v1.l, v2.l row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x88,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+# GFX12-FAKE16: v_cmpx_u_f16_e64_dpp v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x88,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
 
 0x7e,0x00,0x88,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff
-# GFX12: v_cmpx_u_f16_e64_dpp v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x88,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+# GFX12-REAL16: v_cmpx_u_f16_e64_dpp v1.l, v2.l row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x88,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+# GFX12-FAKE16: v_cmpx_u_f16_e64_dpp v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x88,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
 
 0x7e,0x00,0x88,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff
-# GFX12: v_cmpx_u_f16_e64_dpp v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x88,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+# GFX12-REAL16: v_cmpx_u_f16_e64_dpp v1.l, v2.l row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x88,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+# GFX12-FAKE16: v_cmpx_u_f16_e64_dpp v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x88,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
 
 0x7e,0x00,0x88,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff
-# GFX12: v_cmpx_u_f16_e64_dpp v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x88,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+# GFX12-REAL16: v_cmpx_u_f16_e64_dpp v1.l, v2.l row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x88,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+# GFX12-FAKE16: v_cmpx_u_f16_e64_dpp v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x88,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
 
 0x7e,0x00,0x88,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff
-# GFX12: v_cmpx_u_f16_e64_dpp v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x88,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+# GFX12-REAL16: v_cmpx_u_f16_e64_dpp v1.l, v2.l row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x88,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+# GFX12-FAKE16: v_cmpx_u_f16_e64_dpp v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x88,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
 
 0x7e,0x00,0x88,0xd4,0xfa,0x06,0x00,0x00,0x01,0x2f,0x01,0xff
-# GFX12: v_cmpx_u_f16_e64_dpp v1, s3 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x88,0xd4,0xfa,0x06,0x00,0x00,0x01,0x2f,0x01,0xff]
+# GFX12-REAL16: v_cmpx_u_f16_e64_dpp v1.l, s3 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x88,0xd4,0xfa,0x06,0x00,0x00,0x01,0x2f,0x01,0xff]
+# GFX12-FAKE16: v_cmpx_u_f16_e64_dpp v1, s3 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x88,0xd4,0xfa,0x06,0x00,0x00,0x01,0x2f,0x01,0xff]
 
 0x7e,0x00,0x88,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff
-# GFX12: v_cmpx_u_f16_e64_dpp v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x88,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+# GFX12-REAL16: v_cmpx_u_f16_e64_dpp v1.l, v2.l row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x88,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+# GFX12-FAKE16: v_cmpx_u_f16_e64_dpp v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x88,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 
 0x7e,0x01,0x88,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01
-# GFX12: v_cmpx_u_f16_e64_dpp |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x01,0x88,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+# GFX12-REAL16: v_cmpx_u_f16_e64_dpp |v1.l|, -v2.l row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x01,0x88,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+# GFX12-FAKE16: v_cmpx_u_f16_e64_dpp |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x01,0x88,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
 
 0x7e,0x02,0x88,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x01,0x13
-# GFX12: v_cmpx_u_f16_e64_dpp -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x7e,0x02,0x88,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x01,0x13]
+# GFX12-REAL16: v_cmpx_u_f16_e64_dpp -v1.l, |v2.l| row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x7e,0x02,0x88,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x01,0x13]
+# GFX12-FAKE16: v_cmpx_u_f16_e64_dpp -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x7e,0x02,0x88,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x01,0x13]
 
 0x7e,0x83,0x88,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30
-# GFX12: v_cmpx_u_f16_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0x7e,0x83,0x88,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30]
+# GFX12-REAL16: v_cmpx_u_f16_e64_dpp -|v255.l|, -|v255.l| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0x7e,0x83,0x88,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30]
+# GFX12-FAKE16: v_cmpx_u_f16_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0x7e,0x83,0x88,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30]
+
+0x7e,0x08,0x88,0xd4,0x01,0x05,0x02,0x00
+# GFX12-REAL16: v_cmpx_u_f16_e64 v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x7e,0x08,0x88,0xd4,0x01,0x05,0x02,0x00]
+# GFX12-FAKE16: v_cmpx_u_f16_e64 v1, v2                 ; encoding: [0x7e,0x00,0x88,0xd4,0x01,0x05,0x02,0x00]
+
+0x7e,0x10,0x88,0xd4,0xff,0xff,0x03,0x00
+# GFX12-REAL16: v_cmpx_u_f16_e64 v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x7e,0x10,0x88,0xd4,0xff,0xff,0x03,0x00]
+# GFX12-FAKE16: v_cmpx_u_f16_e64 v255, v255             ; encoding: [0x7e,0x00,0x88,0xd4,0xff,0xff,0x03,0x00]
+
+0x7e,0x00,0x88,0xd4,0x01,0x04,0x00,0x00
+# GFX12: v_cmpx_u_f16_e64 s1, s2                 ; encoding: [0x7e,0x00,0x88,0xd4,0x01,0x04,0x00,0x00]
+
+0x7e,0x00,0x88,0xd4,0x69,0xd2,0x00,0x00
+# GFX12: v_cmpx_u_f16_e64 s105, s105             ; encoding: [0x7e,0x00,0x88,0xd4,0x69,0xd2,0x00,0x00]
+
+0x7e,0x00,0x88,0xd4,0x6a,0xf6,0x00,0x00
+# GFX12: v_cmpx_u_f16_e64 vcc_lo, ttmp15         ; encoding: [0x7e,0x00,0x88,0xd4,0x6a,0xf6,0x00,0x00]
+
+0x7e,0x00,0x88,0xd4,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00
+# GFX12: v_cmpx_u_f16_e64 vcc_hi, 0xfe0b         ; encoding: [0x7e,0x00,0x88,0xd4,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
+
+0x7e,0x00,0x88,0xd4,0x7b,0xfa,0x01,0x00
+# GFX12: v_cmpx_u_f16_e64 ttmp15, src_scc        ; encoding: [0x7e,0x00,0x88,0xd4,0x7b,0xfa,0x01,0x00]
+
+0x7e,0x00,0x88,0xd4,0x7d,0xe0,0x01,0x00
+# GFX12: v_cmpx_u_f16_e64 m0, 0.5                ; encoding: [0x7e,0x00,0x88,0xd4,0x7d,0xe0,0x01,0x00]
+
+0x7e,0x00,0x88,0xd4,0x7e,0x82,0x01,0x00
+# GFX12: v_cmpx_u_f16_e64 exec_lo, -1            ; encoding: [0x7e,0x00,0x88,0xd4,0x7e,0x82,0x01,0x00]
+
+0x7e,0x01,0x88,0xd4,0x7f,0xf8,0x00,0x00
+# GFX12: v_cmpx_u_f16_e64 |exec_hi|, null        ; encoding: [0x7e,0x01,0x88,0xd4,0x7f,0xf8,0x00,0x00]
+
+0x7e,0x00,0x88,0xd4,0x7c,0xfc,0x00,0x00
+# GFX12: v_cmpx_u_f16_e64 null, exec_lo          ; encoding: [0x7e,0x00,0x88,0xd4,0x7c,0xfc,0x00,0x00]
+
+0x7e,0x00,0x88,0xd4,0xc1,0xfe,0x00,0x00
+# GFX12: v_cmpx_u_f16_e64 -1, exec_hi            ; encoding: [0x7e,0x00,0x88,0xd4,0xc1,0xfe,0x00,0x00]
+
+0x7e,0x00,0x88,0xd4,0xf0,0xfa,0x00,0x40
+# GFX12: v_cmpx_u_f16_e64 0.5, -m0               ; encoding: [0x7e,0x00,0x88,0xd4,0xf0,0xfa,0x00,0x40]
+
+0x7e,0x02,0x88,0xd4,0xfd,0xd4,0x00,0x20
+# GFX12: v_cmpx_u_f16_e64 -src_scc, |vcc_lo|     ; encoding: [0x7e,0x02,0x88,0xd4,0xfd,0xd4,0x00,0x20]
+
+0x7e,0x83,0x88,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00
+# GFX12: v_cmpx_u_f16_e64 -|0xfe0b|, -|vcc_hi| clamp ; encoding: [0x7e,0x83,0x88,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00]
+
+0x7e,0x19,0x88,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01
+# GFX12-REAL16: v_cmpx_u_f16_e64_dpp |v1.h|, -v2.h op_sel:[1,1] row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x19,0x88,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+# GFX12-FAKE16: v_cmpx_u_f16_e64_dpp |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x01,0x88,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+
+0x7e,0x0a,0x88,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x01,0x13
+# GFX12-REAL16: v_cmpx_u_f16_e64_dpp -v1.h, |v2.l| op_sel:[1,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x7e,0x0a,0x88,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x01,0x13]
+# GFX12-FAKE16: v_cmpx_u_f16_e64_dpp -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x7e,0x02,0x88,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x01,0x13]
+
+0x7e,0x93,0x88,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30
+# GFX12-REAL16: v_cmpx_u_f16_e64_dpp -|v255.l|, -|v255.h| op_sel:[0,1] clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0x7e,0x93,0x88,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30]
+# GFX12-FAKE16: v_cmpx_u_f16_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0x7e,0x83,0x88,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30]
 
 0x7e,0x00,0x98,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff
 # GFX12: v_cmpx_u_f32_e64_dpp v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x98,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
diff --git a/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vop3cx_dpp8.txt b/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vop3cx_dpp8.txt
index cff9497778265..83d9623c5458f 100644
--- a/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vop3cx_dpp8.txt
+++ b/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vop3cx_dpp8.txt
@@ -38,19 +38,37 @@
 # GFX12: v_cmpx_class_f32_e64_dpp -|v255|, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0x7e,0x01,0xfe,0xd4,0xea,0xfe,0x03,0x20,0xff,0x00,0x00,0x00]
 
 0x7e,0x00,0x82,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
-# GFX12: v_cmpx_eq_f16_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x82,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+# GFX12-REAL16: v_cmpx_eq_f16_e64_dpp v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x82,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+# GFX12-FAKE16: v_cmpx_eq_f16_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x82,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 
 0x7e,0x00,0x82,0xd4,0xe9,0x06,0x00,0x00,0x01,0x77,0x39,0x05
-# GFX12: v_cmpx_eq_f16_e64_dpp v1, s3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x82,0xd4,0xe9,0x06,0x00,0x00,0x01,0x77,0x39,0x05]
+# GFX12-REAL16: v_cmpx_eq_f16_e64_dpp v1.l, s3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x82,0xd4,0xe9,0x06,0x00,0x00,0x01,0x77,0x39,0x05]
+# GFX12-FAKE16: v_cmpx_eq_f16_e64_dpp v1, s3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x82,0xd4,0xe9,0x06,0x00,0x00,0x01,0x77,0x39,0x05]
 
 0x7e,0x01,0x82,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05
-# GFX12: v_cmpx_eq_f16_e64_dpp |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x01,0x82,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+# GFX12-REAL16: v_cmpx_eq_f16_e64_dpp |v1.l|, -v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x01,0x82,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+# GFX12-FAKE16: v_cmpx_eq_f16_e64_dpp |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x01,0x82,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
 
 0x7e,0x02,0x82,0xd4,0xe9,0x04,0x02,0x20,0x01,0x77,0x39,0x05
-# GFX12: v_cmpx_eq_f16_e64_dpp -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x02,0x82,0xd4,0xe9,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+# GFX12-REAL16: v_cmpx_eq_f16_e64_dpp -v1.l, |v2.l| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x02,0x82,0xd4,0xe9,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+# GFX12-FAKE16: v_cmpx_eq_f16_e64_dpp -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x02,0x82,0xd4,0xe9,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
 
 0x7e,0x83,0x82,0xd4,0xea,0xfe,0x03,0x60,0xff,0x00,0x00,0x00
-# GFX12: v_cmpx_eq_f16_e64_dpp -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0x7e,0x83,0x82,0xd4,0xea,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+# GFX12-REAL16: v_cmpx_eq_f16_e64_dpp -|v255.l|, -|v255.l| clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0x7e,0x83,0x82,0xd4,0xea,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+# GFX12-FAKE16: v_cmpx_eq_f16_e64_dpp -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0x7e,0x83,0x82,0xd4,0xea,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+
+0x7e,0x19,0x82,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05
+# GFX12-REAL16: v_cmpx_eq_f16_e64_dpp |v1.h|, -v2.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x19,0x82,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+# GFX12-FAKE16: v_cmpx_eq_f16_e64_dpp |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x01,0x82,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+
+0x7e,0x0a,0x82,0xd4,0xe9,0x04,0x02,0x20,0x01,0x77,0x39,0x05
+# GFX12-REAL16: v_cmpx_eq_f16_e64_dpp -v1.h, |v2.l| op_sel:[1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x0a,0x82,0xd4,0xe9,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+# GFX12-FAKE16: v_cmpx_eq_f16_e64_dpp -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x02,0x82,0xd4,0xe9,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+
+0x7e,0x93,0x82,0xd4,0xea,0xfe,0x03,0x60,0xff,0x00,0x00,0x00
+# GFX12-REAL16: v_cmpx_eq_f16_e64_dpp -|v255.l|, -|v255.h| op_sel:[0,1] clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0x7e,0x93,0x82,0xd4,0xea,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+# GFX12-FAKE16: v_cmpx_eq_f16_e64_dpp -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0x7e,0x83,0x82,0xd4,0xea,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+
 
 0x7e,0x00,0x92,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 # GFX12: v_cmpx_eq_f32_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x92,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
@@ -82,17 +100,14 @@
 # GFX12-REAL16: v_cmpx_eq_i16_e64_dpp v255.l, v255.l dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0x7e,0x00,0xb2,0xd4,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 # GFX12-FAKE16: v_cmpx_eq_i16_e64_dpp v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0x7e,0x00,0xb2,0xd4,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 
-# GFX11: v_cmpx_eq_i16_e64_dpp v1.h, v2.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x18,0xb2,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 0x7e,0x18,0xb2,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 # GFX12-REAL16: v_cmpx_eq_i16_e64_dpp v1.h, v2.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x18,0xb2,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # GFX12-FAKE16: v_cmpx_eq_i16_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0xb2,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 
-# GFX11: v_cmpx_eq_i16_e64_dpp v1.h, v2.l op_sel:[1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x08,0xb2,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 0x7e,0x08,0xb2,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 # GFX12-REAL16: v_cmpx_eq_i16_e64_dpp v1.h, v2.l op_sel:[1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x08,0xb2,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # GFX12-FAKE16: v_cmpx_eq_i16_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0xb2,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 
-# GFX11: v_cmpx_eq_i16_e64_dpp v255.l, v255.h op_sel:[0,1] dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0x7e,0x10,0xb2,0xd4,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 0x7e,0x10,0xb2,0xd4,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00
 # GFX12-REAL16: v_cmpx_eq_i16_e64_dpp v255.l, v255.h op_sel:[0,1] dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0x7e,0x10,0xb2,0xd4,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 # GFX12-FAKE16: v_cmpx_eq_i16_e64_dpp v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0x7e,0x00,0xb2,0xd4,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
@@ -118,17 +133,14 @@
 # GFX12-REAL16: v_cmpx_eq_u16_e64_dpp v255.l, v255.l dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0x7e,0x00,0xba,0xd4,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 # GFX12-FAKE16: v_cmpx_eq_u16_e64_dpp v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0x7e,0x00,0xba,0xd4,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 
-# GFX11: v_cmpx_eq_u16_e64_dpp v1.h, v2.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x18,0xba,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 0x7e,0x18,0xba,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 # GFX12-REAL16: v_cmpx_eq_u16_e64_dpp v1.h, v2.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x18,0xba,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # GFX12-FAKE16: v_cmpx_eq_u16_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0xba,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 
-# GFX11: v_cmpx_eq_u16_e64_dpp v1.h, v2.l op_sel:[1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x08,0xba,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 0x7e,0x08,0xba,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 # GFX12-REAL16: v_cmpx_eq_u16_e64_dpp v1.h, v2.l op_sel:[1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x08,0xba,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # GFX12-FAKE16: v_cmpx_eq_u16_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0xba,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 
-# GFX11: v_cmpx_eq_u16_e64_dpp v255.l, v255.h op_sel:[0,1] dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0x7e,0x10,0xba,0xd4,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 0x7e,0x10,0xba,0xd4,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00
 # GFX12-REAL16: v_cmpx_eq_u16_e64_dpp v255.l, v255.h op_sel:[0,1] dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0x7e,0x10,0xba,0xd4,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 # GFX12-FAKE16: v_cmpx_eq_u16_e64_dpp v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0x7e,0x00,0xba,0xd4,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
@@ -143,22 +155,41 @@
 # GFX12: v_cmpx_eq_u32_e64_dpp v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0x7e,0x00,0xca,0xd4,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 
 0x7e,0x00,0x86,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
-# GFX12: v_cmpx_ge_f16_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x86,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+# GFX12-REAL16: v_cmpx_ge_f16_e64_dpp v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x86,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+# GFX12-FAKE16: v_cmpx_ge_f16_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x86,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 
 0x7e,0x00,0x86,0xd4,0xe9,0xe8,0x01,0x00,0x01,0x77,0x39,0x05
-# GFX12: v_cmpx_ge_f16_e64_dpp v1, 2.0 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x86,0xd4,0xe9,0xe8,0x01,0x00,0x01,0x77,0x39,0x05]
+# GFX12-REAL16: v_cmpx_ge_f16_e64_dpp v1.l, 2.0 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x86,0xd4,0xe9,0xe8,0x01,0x00,0x01,0x77,0x39,0x05]
+# GFX12-FAKE16: v_cmpx_ge_f16_e64_dpp v1, 2.0 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x86,0xd4,0xe9,0xe8,0x01,0x00,0x01,0x77,0x39,0x05]
 
 0x7e,0x00,0x86,0xd4,0xe9,0x06,0x00,0x00,0x01,0x77,0x39,0x05
-# GFX12: v_cmpx_ge_f16_e64_dpp v1, s3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x86,0xd4,0xe9,0x06,0x00,0x00,0x01,0x77,0x39,0x05]
+# GFX12-REAL16: v_cmpx_ge_f16_e64_dpp v1.l, s3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x86,0xd4,0xe9,0x06,0x00,0x00,0x01,0x77,0x39,0x05]
+# GFX12-FAKE16: v_cmpx_ge_f16_e64_dpp v1, s3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x86,0xd4,0xe9,0x06,0x00,0x00,0x01,0x77,0x39,0x05]
 
 0x7e,0x01,0x86,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05
-# GFX12: v_cmpx_ge_f16_e64_dpp |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x01,0x86,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+# GFX12-REAL16: v_cmpx_ge_f16_e64_dpp |v1.l|, -v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x01,0x86,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+# GFX12-FAKE16: v_cmpx_ge_f16_e64_dpp |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x01,0x86,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
 
 0x7e,0x02,0x86,0xd4,0xe9,0x04,0x02,0x20,0x01,0x77,0x39,0x05
-# GFX12: v_cmpx_ge_f16_e64_dpp -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x02,0x86,0xd4,0xe9,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+# GFX12-REAL16: v_cmpx_ge_f16_e64_dpp -v1.l, |v2.l| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x02,0x86,0xd4,0xe9,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+# GFX12-FAKE16: v_cmpx_ge_f16_e64_dpp -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x02,0x86,0xd4,0xe9,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
 
 0x7e,0x83,0x86,0xd4,0xea,0xfe,0x03,0x60,0xff,0x00,0x00,0x00
-# GFX12: v_cmpx_ge_f16_e64_dpp -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0x7e,0x83,0x86,0xd4,0xea,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+# GFX12-REAL16: v_cmpx_ge_f16_e64_dpp -|v255.l|, -|v255.l| clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0x7e,0x83,0x86,0xd4,0xea,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+# GFX12-FAKE16: v_cmpx_ge_f16_e64_dpp -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0x7e,0x83,0x86,0xd4,0xea,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+
+0x7e,0x19,0x86,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05
+# GFX12-REAL16: v_cmpx_ge_f16_e64_dpp |v1.h|, -v2.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x19,0x86,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+# GFX12-FAKE16: v_cmpx_ge_f16_e64_dpp |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x01,0x86,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+
+0x7e,0x0a,0x86,0xd4,0xe9,0x04,0x02,0x20,0x01,0x77,0x39,0x05
+# GFX12-REAL16: v_cmpx_ge_f16_e64_dpp -v1.h, |v2.l| op_sel:[1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x0a,0x86,0xd4,0xe9,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+# GFX12-FAKE16: v_cmpx_ge_f16_e64_dpp -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x02,0x86,0xd4,0xe9,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+
+0x7e,0x93,0x86,0xd4,0xea,0xfe,0x03,0x60,0xff,0x00,0x00,0x00
+# GFX12-REAL16: v_cmpx_ge_f16_e64_dpp -|v255.l|, -|v255.h| op_sel:[0,1] clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0x7e,0x93,0x86,0xd4,0xea,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+# GFX12-FAKE16: v_cmpx_ge_f16_e64_dpp -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0x7e,0x83,0x86,0xd4,0xea,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+
 
 0x7e,0x00,0x96,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 # GFX12: v_cmpx_ge_f32_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x96,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
@@ -187,17 +218,14 @@
 # GFX12-REAL16: v_cmpx_ge_i16_e64_dpp v255.l, v255.l dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0x7e,0x00,0xb6,0xd4,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 # GFX12-FAKE16: v_cmpx_ge_i16_e64_dpp v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0x7e,0x00,0xb6,0xd4,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 
-# GFX11: v_cmpx_ge_i16_e64_dpp v1.h, v2.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x18,0xb6,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 0x7e,0x18,0xb6,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 # GFX12-REAL16: v_cmpx_ge_i16_e64_dpp v1.h, v2.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x18,0xb6,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # GFX12-FAKE16: v_cmpx_ge_i16_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0xb6,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 
-# GFX11: v_cmpx_ge_i16_e64_dpp v1.h, v2.l op_sel:[1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x08,0xb6,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 0x7e,0x08,0xb6,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 # GFX12-REAL16: v_cmpx_ge_i16_e64_dpp v1.h, v2.l op_sel:[1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x08,0xb6,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # GFX12-FAKE16: v_cmpx_ge_i16_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0xb6,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 
-# GFX11: v_cmpx_ge_i16_e64_dpp v255.l, v255.h op_sel:[0,1] dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0x7e,0x10,0xb6,0xd4,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 0x7e,0x10,0xb6,0xd4,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00
 # GFX12-REAL16: v_cmpx_ge_i16_e64_dpp v255.l, v255.h op_sel:[0,1] dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0x7e,0x10,0xb6,0xd4,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 # GFX12-FAKE16: v_cmpx_ge_i16_e64_dpp v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0x7e,0x00,0xb6,0xd4,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
@@ -223,17 +251,14 @@
 # GFX12-REAL16: v_cmpx_ge_u16_e64_dpp v255.l, v255.l dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0x7e,0x00,0xbe,0xd4,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 # GFX12-FAKE16: v_cmpx_ge_u16_e64_dpp v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0x7e,0x00,0xbe,0xd4,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 
-# GFX11: v_cmpx_ge_u16_e64_dpp v1.h, v2.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x18,0xbe,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 0x7e,0x18,0xbe,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 # GFX12-REAL16: v_cmpx_ge_u16_e64_dpp v1.h, v2.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x18,0xbe,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # GFX12-FAKE16: v_cmpx_ge_u16_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0xbe,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 
-# GFX11: v_cmpx_ge_u16_e64_dpp v1.h, v2.l op_sel:[1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x08,0xbe,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 0x7e,0x08,0xbe,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 # GFX12-REAL16: v_cmpx_ge_u16_e64_dpp v1.h, v2.l op_sel:[1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x08,0xbe,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # GFX12-FAKE16: v_cmpx_ge_u16_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0xbe,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 
-# GFX11: v_cmpx_ge_u16_e64_dpp v255.l, v255.h op_sel:[0,1] dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0x7e,0x10,0xbe,0xd4,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 0x7e,0x10,0xbe,0xd4,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00
 # GFX12-REAL16: v_cmpx_ge_u16_e64_dpp v255.l, v255.h op_sel:[0,1] dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0x7e,0x10,0xbe,0xd4,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 # GFX12-FAKE16: v_cmpx_ge_u16_e64_dpp v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0x7e,0x00,0xbe,0xd4,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
@@ -248,19 +273,37 @@
 # GFX12: v_cmpx_ge_u32_e64_dpp v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0x7e,0x00,0xce,0xd4,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 
 0x7e,0x00,0x84,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
-# GFX12: v_cmpx_gt_f16_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x84,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+# GFX12-REAL16: v_cmpx_gt_f16_e64_dpp v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x84,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+# GFX12-FAKE16: v_cmpx_gt_f16_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x84,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 
 0x7e,0x00,0x84,0xd4,0xe9,0x06,0x00,0x00,0x01,0x77,0x39,0x05
-# GFX12: v_cmpx_gt_f16_e64_dpp v1, s3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x84,0xd4,0xe9,0x06,0x00,0x00,0x01,0x77,0x39,0x05]
+# GFX12-REAL16: v_cmpx_gt_f16_e64_dpp v1.l, s3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x84,0xd4,0xe9,0x06,0x00,0x00,0x01,0x77,0x39,0x05]
+# GFX12-FAKE16: v_cmpx_gt_f16_e64_dpp v1, s3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x84,0xd4,0xe9,0x06,0x00,0x00,0x01,0x77,0x39,0x05]
 
 0x7e,0x01,0x84,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05
-# GFX12: v_cmpx_gt_f16_e64_dpp |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x01,0x84,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+# GFX12-REAL16: v_cmpx_gt_f16_e64_dpp |v1.l|, -v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x01,0x84,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+# GFX12-FAKE16: v_cmpx_gt_f16_e64_dpp |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x01,0x84,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
 
 0x7e,0x02,0x84,0xd4,0xe9,0x04,0x02,0x20,0x01,0x77,0x39,0x05
-# GFX12: v_cmpx_gt_f16_e64_dpp -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x02,0x84,0xd4,0xe9,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+# GFX12-REAL16: v_cmpx_gt_f16_e64_dpp -v1.l, |v2.l| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x02,0x84,0xd4,0xe9,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+# GFX12-FAKE16: v_cmpx_gt_f16_e64_dpp -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x02,0x84,0xd4,0xe9,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
 
 0x7e,0x83,0x84,0xd4,0xea,0xfe,0x03,0x60,0xff,0x00,0x00,0x00
-# GFX12: v_cmpx_gt_f16_e64_dpp -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0x7e,0x83,0x84,0xd4,0xea,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+# GFX12-REAL16: v_cmpx_gt_f16_e64_dpp -|v255.l|, -|v255.l| clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0x7e,0x83,0x84,0xd4,0xea,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+# GFX12-FAKE16: v_cmpx_gt_f16_e64_dpp -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0x7e,0x83,0x84,0xd4,0xea,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+
+0x7e,0x19,0x84,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05
+# GFX12-REAL16: v_cmpx_gt_f16_e64_dpp |v1.h|, -v2.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x19,0x84,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+# GFX12-FAKE16: v_cmpx_gt_f16_e64_dpp |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x01,0x84,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+
+0x7e,0x0a,0x84,0xd4,0xe9,0x04,0x02,0x20,0x01,0x77,0x39,0x05
+# GFX12-REAL16: v_cmpx_gt_f16_e64_dpp -v1.h, |v2.l| op_sel:[1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x0a,0x84,0xd4,0xe9,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+# GFX12-FAKE16: v_cmpx_gt_f16_e64_dpp -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x02,0x84,0xd4,0xe9,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+
+0x7e,0x93,0x84,0xd4,0xea,0xfe,0x03,0x60,0xff,0x00,0x00,0x00
+# GFX12-REAL16: v_cmpx_gt_f16_e64_dpp -|v255.l|, -|v255.h| op_sel:[0,1] clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0x7e,0x93,0x84,0xd4,0xea,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+# GFX12-FAKE16: v_cmpx_gt_f16_e64_dpp -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0x7e,0x83,0x84,0xd4,0xea,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+
 
 0x7e,0x00,0x94,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 # GFX12: v_cmpx_gt_f32_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x94,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
@@ -289,17 +332,14 @@
 # GFX12-REAL16: v_cmpx_gt_i16_e64_dpp v255.l, v255.l dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0x7e,0x00,0xb4,0xd4,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 # GFX12-FAKE16: v_cmpx_gt_i16_e64_dpp v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0x7e,0x00,0xb4,0xd4,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 
-# GFX11: v_cmpx_gt_i16_e64_dpp v1.h, v2.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x18,0xb4,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 0x7e,0x18,0xb4,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 # GFX12-REAL16: v_cmpx_gt_i16_e64_dpp v1.h, v2.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x18,0xb4,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # GFX12-FAKE16: v_cmpx_gt_i16_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0xb4,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 
-# GFX11: v_cmpx_gt_i16_e64_dpp v1.h, v2.l op_sel:[1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x08,0xb4,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 0x7e,0x08,0xb4,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 # GFX12-REAL16: v_cmpx_gt_i16_e64_dpp v1.h, v2.l op_sel:[1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x08,0xb4,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # GFX12-FAKE16: v_cmpx_gt_i16_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0xb4,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 
-# GFX11: v_cmpx_gt_i16_e64_dpp v255.l, v255.h op_sel:[0,1] dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0x7e,0x10,0xb4,0xd4,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 0x7e,0x10,0xb4,0xd4,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00
 # GFX12-REAL16: v_cmpx_gt_i16_e64_dpp v255.l, v255.h op_sel:[0,1] dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0x7e,0x10,0xb4,0xd4,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 # GFX12-FAKE16: v_cmpx_gt_i16_e64_dpp v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0x7e,0x00,0xb4,0xd4,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
@@ -325,17 +365,14 @@
 # GFX12-REAL16: v_cmpx_gt_u16_e64_dpp v255.l, v255.l dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0x7e,0x00,0xbc,0xd4,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 # GFX12-FAKE16: v_cmpx_gt_u16_e64_dpp v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0x7e,0x00,0xbc,0xd4,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 
-# GFX11: v_cmpx_gt_u16_e64_dpp v1.h, v2.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x18,0xbc,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 0x7e,0x18,0xbc,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 # GFX12-REAL16: v_cmpx_gt_u16_e64_dpp v1.h, v2.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x18,0xbc,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # GFX12-FAKE16: v_cmpx_gt_u16_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0xbc,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 
-# GFX11: v_cmpx_gt_u16_e64_dpp v1.h, v2.l op_sel:[1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x08,0xbc,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 0x7e,0x08,0xbc,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 # GFX12-REAL16: v_cmpx_gt_u16_e64_dpp v1.h, v2.l op_sel:[1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x08,0xbc,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # GFX12-FAKE16: v_cmpx_gt_u16_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0xbc,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 
-# GFX11: v_cmpx_gt_u16_e64_dpp v255.l, v255.h op_sel:[0,1] dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0x7e,0x10,0xbc,0xd4,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 0x7e,0x10,0xbc,0xd4,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00
 # GFX12-REAL16: v_cmpx_gt_u16_e64_dpp v255.l, v255.h op_sel:[0,1] dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0x7e,0x10,0xbc,0xd4,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 # GFX12-FAKE16: v_cmpx_gt_u16_e64_dpp v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0x7e,0x00,0xbc,0xd4,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
@@ -350,19 +387,37 @@
 # GFX12: v_cmpx_gt_u32_e64_dpp v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0x7e,0x00,0xcc,0xd4,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 
 0x7e,0x00,0x83,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
-# GFX12: v_cmpx_le_f16_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x83,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+# GFX12-REAL16: v_cmpx_le_f16_e64_dpp v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x83,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+# GFX12-FAKE16: v_cmpx_le_f16_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x83,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 
 0x7e,0x00,0x83,0xd4,0xe9,0x06,0x00,0x00,0x01,0x77,0x39,0x05
-# GFX12: v_cmpx_le_f16_e64_dpp v1, s3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x83,0xd4,0xe9,0x06,0x00,0x00,0x01,0x77,0x39,0x05]
+# GFX12-REAL16: v_cmpx_le_f16_e64_dpp v1.l, s3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x83,0xd4,0xe9,0x06,0x00,0x00,0x01,0x77,0x39,0x05]
+# GFX12-FAKE16: v_cmpx_le_f16_e64_dpp v1, s3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x83,0xd4,0xe9,0x06,0x00,0x00,0x01,0x77,0x39,0x05]
 
 0x7e,0x01,0x83,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05
-# GFX12: v_cmpx_le_f16_e64_dpp |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x01,0x83,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+# GFX12-REAL16: v_cmpx_le_f16_e64_dpp |v1.l|, -v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x01,0x83,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+# GFX12-FAKE16: v_cmpx_le_f16_e64_dpp |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x01,0x83,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
 
 0x7e,0x02,0x83,0xd4,0xe9,0x04,0x02,0x20,0x01,0x77,0x39,0x05
-# GFX12: v_cmpx_le_f16_e64_dpp -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x02,0x83,0xd4,0xe9,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+# GFX12-REAL16: v_cmpx_le_f16_e64_dpp -v1.l, |v2.l| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x02,0x83,0xd4,0xe9,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+# GFX12-FAKE16: v_cmpx_le_f16_e64_dpp -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x02,0x83,0xd4,0xe9,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
 
 0x7e,0x83,0x83,0xd4,0xea,0xfe,0x03,0x60,0xff,0x00,0x00,0x00
-# GFX12: v_cmpx_le_f16_e64_dpp -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0x7e,0x83,0x83,0xd4,0xea,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+# GFX12-REAL16: v_cmpx_le_f16_e64_dpp -|v255.l|, -|v255.l| clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0x7e,0x83,0x83,0xd4,0xea,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+# GFX12-FAKE16: v_cmpx_le_f16_e64_dpp -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0x7e,0x83,0x83,0xd4,0xea,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+
+0x7e,0x19,0x83,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05
+# GFX12-REAL16: v_cmpx_le_f16_e64_dpp |v1.h|, -v2.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x19,0x83,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+# GFX12-FAKE16: v_cmpx_le_f16_e64_dpp |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x01,0x83,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+
+0x7e,0x0a,0x83,0xd4,0xe9,0x04,0x02,0x20,0x01,0x77,0x39,0x05
+# GFX12-REAL16: v_cmpx_le_f16_e64_dpp -v1.h, |v2.l| op_sel:[1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x0a,0x83,0xd4,0xe9,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+# GFX12-FAKE16: v_cmpx_le_f16_e64_dpp -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x02,0x83,0xd4,0xe9,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+
+0x7e,0x93,0x83,0xd4,0xea,0xfe,0x03,0x60,0xff,0x00,0x00,0x00
+# GFX12-REAL16: v_cmpx_le_f16_e64_dpp -|v255.l|, -|v255.h| op_sel:[0,1] clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0x7e,0x93,0x83,0xd4,0xea,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+# GFX12-FAKE16: v_cmpx_le_f16_e64_dpp -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0x7e,0x83,0x83,0xd4,0xea,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+
 
 0x7e,0x00,0x93,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 # GFX12: v_cmpx_le_f32_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x93,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
@@ -391,17 +446,14 @@
 # GFX12-REAL16: v_cmpx_le_i16_e64_dpp v255.l, v255.l dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0x7e,0x00,0xb3,0xd4,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 # GFX12-FAKE16: v_cmpx_le_i16_e64_dpp v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0x7e,0x00,0xb3,0xd4,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 
-# GFX11: v_cmpx_le_i16_e64_dpp v1.h, v2.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x18,0xb3,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 0x7e,0x18,0xb3,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 # GFX12-REAL16: v_cmpx_le_i16_e64_dpp v1.h, v2.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x18,0xb3,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # GFX12-FAKE16: v_cmpx_le_i16_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0xb3,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 
-# GFX11: v_cmpx_le_i16_e64_dpp v1.h, v2.l op_sel:[1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x08,0xb3,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 0x7e,0x08,0xb3,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 # GFX12-REAL16: v_cmpx_le_i16_e64_dpp v1.h, v2.l op_sel:[1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x08,0xb3,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # GFX12-FAKE16: v_cmpx_le_i16_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0xb3,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 
-# GFX11: v_cmpx_le_i16_e64_dpp v255.l, v255.h op_sel:[0,1] dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0x7e,0x10,0xb3,0xd4,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 0x7e,0x10,0xb3,0xd4,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00
 # GFX12-REAL16: v_cmpx_le_i16_e64_dpp v255.l, v255.h op_sel:[0,1] dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0x7e,0x10,0xb3,0xd4,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 # GFX12-FAKE16: v_cmpx_le_i16_e64_dpp v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0x7e,0x00,0xb3,0xd4,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
@@ -427,17 +479,14 @@
 # GFX12-REAL16: v_cmpx_le_u16_e64_dpp v255.l, v255.l dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0x7e,0x00,0xbb,0xd4,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 # GFX12-FAKE16: v_cmpx_le_u16_e64_dpp v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0x7e,0x00,0xbb,0xd4,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 
-# GFX11: v_cmpx_le_u16_e64_dpp v1.h, v2.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x18,0xbb,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 0x7e,0x18,0xbb,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 # GFX12-REAL16: v_cmpx_le_u16_e64_dpp v1.h, v2.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x18,0xbb,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # GFX12-FAKE16: v_cmpx_le_u16_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0xbb,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 
-# GFX11: v_cmpx_le_u16_e64_dpp v1.h, v2.l op_sel:[1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x08,0xbb,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 0x7e,0x08,0xbb,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 # GFX12-REAL16: v_cmpx_le_u16_e64_dpp v1.h, v2.l op_sel:[1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x08,0xbb,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # GFX12-FAKE16: v_cmpx_le_u16_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0xbb,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 
-# GFX11: v_cmpx_le_u16_e64_dpp v255.l, v255.h op_sel:[0,1] dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0x7e,0x10,0xbb,0xd4,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 0x7e,0x10,0xbb,0xd4,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00
 # GFX12-REAL16: v_cmpx_le_u16_e64_dpp v255.l, v255.h op_sel:[0,1] dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0x7e,0x10,0xbb,0xd4,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 # GFX12-FAKE16: v_cmpx_le_u16_e64_dpp v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0x7e,0x00,0xbb,0xd4,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
@@ -452,19 +501,37 @@
 # GFX12: v_cmpx_le_u32_e64_dpp v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0x7e,0x00,0xcb,0xd4,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 
 0x7e,0x00,0x85,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
-# GFX12: v_cmpx_lg_f16_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x85,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+# GFX12-REAL16: v_cmpx_lg_f16_e64_dpp v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x85,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+# GFX12-FAKE16: v_cmpx_lg_f16_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x85,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 
 0x7e,0x00,0x85,0xd4,0xe9,0x06,0x00,0x00,0x01,0x77,0x39,0x05
-# GFX12: v_cmpx_lg_f16_e64_dpp v1, s3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x85,0xd4,0xe9,0x06,0x00,0x00,0x01,0x77,0x39,0x05]
+# GFX12-REAL16: v_cmpx_lg_f16_e64_dpp v1.l, s3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x85,0xd4,0xe9,0x06,0x00,0x00,0x01,0x77,0x39,0x05]
+# GFX12-FAKE16: v_cmpx_lg_f16_e64_dpp v1, s3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x85,0xd4,0xe9,0x06,0x00,0x00,0x01,0x77,0x39,0x05]
 
 0x7e,0x01,0x85,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05
-# GFX12: v_cmpx_lg_f16_e64_dpp |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x01,0x85,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+# GFX12-REAL16: v_cmpx_lg_f16_e64_dpp |v1.l|, -v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x01,0x85,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+# GFX12-FAKE16: v_cmpx_lg_f16_e64_dpp |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x01,0x85,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
 
 0x7e,0x02,0x85,0xd4,0xe9,0x04,0x02,0x20,0x01,0x77,0x39,0x05
-# GFX12: v_cmpx_lg_f16_e64_dpp -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x02,0x85,0xd4,0xe9,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+# GFX12-REAL16: v_cmpx_lg_f16_e64_dpp -v1.l, |v2.l| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x02,0x85,0xd4,0xe9,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+# GFX12-FAKE16: v_cmpx_lg_f16_e64_dpp -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x02,0x85,0xd4,0xe9,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
 
 0x7e,0x83,0x85,0xd4,0xea,0xfe,0x03,0x60,0xff,0x00,0x00,0x00
-# GFX12: v_cmpx_lg_f16_e64_dpp -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0x7e,0x83,0x85,0xd4,0xea,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+# GFX12-REAL16: v_cmpx_lg_f16_e64_dpp -|v255.l|, -|v255.l| clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0x7e,0x83,0x85,0xd4,0xea,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+# GFX12-FAKE16: v_cmpx_lg_f16_e64_dpp -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0x7e,0x83,0x85,0xd4,0xea,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+
+0x7e,0x19,0x85,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05
+# GFX12-REAL16: v_cmpx_lg_f16_e64_dpp |v1.h|, -v2.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x19,0x85,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+# GFX12-FAKE16: v_cmpx_lg_f16_e64_dpp |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x01,0x85,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+
+0x7e,0x0a,0x85,0xd4,0xe9,0x04,0x02,0x20,0x01,0x77,0x39,0x05
+# GFX12-REAL16: v_cmpx_lg_f16_e64_dpp -v1.h, |v2.l| op_sel:[1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x0a,0x85,0xd4,0xe9,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+# GFX12-FAKE16: v_cmpx_lg_f16_e64_dpp -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x02,0x85,0xd4,0xe9,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+
+0x7e,0x93,0x85,0xd4,0xea,0xfe,0x03,0x60,0xff,0x00,0x00,0x00
+# GFX12-REAL16: v_cmpx_lg_f16_e64_dpp -|v255.l|, -|v255.h| op_sel:[0,1] clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0x7e,0x93,0x85,0xd4,0xea,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+# GFX12-FAKE16: v_cmpx_lg_f16_e64_dpp -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0x7e,0x83,0x85,0xd4,0xea,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+
 
 0x7e,0x00,0x95,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 # GFX12: v_cmpx_lg_f32_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x95,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
@@ -540,17 +607,14 @@
 # GFX12-REAL16: v_cmpx_lt_i16_e64_dpp v255.l, v255.l dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0x7e,0x00,0xb1,0xd4,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 # GFX12-FAKE16: v_cmpx_lt_i16_e64_dpp v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0x7e,0x00,0xb1,0xd4,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 
-# GFX11: v_cmpx_lt_i16_e64_dpp v1.h, v2.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x18,0xb1,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 0x7e,0x18,0xb1,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 # GFX12-REAL16: v_cmpx_lt_i16_e64_dpp v1.h, v2.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x18,0xb1,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # GFX12-FAKE16: v_cmpx_lt_i16_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0xb1,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 
-# GFX11: v_cmpx_lt_i16_e64_dpp v1.h, v2.l op_sel:[1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x08,0xb1,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 0x7e,0x08,0xb1,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 # GFX12-REAL16: v_cmpx_lt_i16_e64_dpp v1.h, v2.l op_sel:[1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x08,0xb1,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # GFX12-FAKE16: v_cmpx_lt_i16_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0xb1,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 
-# GFX11: v_cmpx_lt_i16_e64_dpp v255.l, v255.h op_sel:[0,1] dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0x7e,0x10,0xb1,0xd4,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 0x7e,0x10,0xb1,0xd4,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00
 # GFX12-REAL16: v_cmpx_lt_i16_e64_dpp v255.l, v255.h op_sel:[0,1] dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0x7e,0x10,0xb1,0xd4,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 # GFX12-FAKE16: v_cmpx_lt_i16_e64_dpp v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0x7e,0x00,0xb1,0xd4,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
@@ -576,17 +640,14 @@
 # GFX12-REAL16: v_cmpx_lt_u16_e64_dpp v255.l, v255.l dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0x7e,0x00,0xb9,0xd4,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 # GFX12-FAKE16: v_cmpx_lt_u16_e64_dpp v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0x7e,0x00,0xb9,0xd4,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 
-# GFX11: v_cmpx_lt_u16_e64_dpp v1.h, v2.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x18,0xb9,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 0x7e,0x18,0xb9,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 # GFX12-REAL16: v_cmpx_lt_u16_e64_dpp v1.h, v2.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x18,0xb9,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # GFX12-FAKE16: v_cmpx_lt_u16_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0xb9,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 
-# GFX11: v_cmpx_lt_u16_e64_dpp v1.h, v2.l op_sel:[1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x08,0xb9,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 0x7e,0x08,0xb9,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 # GFX12-REAL16: v_cmpx_lt_u16_e64_dpp v1.h, v2.l op_sel:[1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x08,0xb9,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # GFX12-FAKE16: v_cmpx_lt_u16_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0xb9,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 
-# GFX11: v_cmpx_lt_u16_e64_dpp v255.l, v255.h op_sel:[0,1] dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0x7e,0x10,0xb9,0xd4,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 0x7e,0x10,0xb9,0xd4,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00
 # GFX12-REAL16: v_cmpx_lt_u16_e64_dpp v255.l, v255.h op_sel:[0,1] dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0x7e,0x10,0xb9,0xd4,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 # GFX12-FAKE16: v_cmpx_lt_u16_e64_dpp v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0x7e,0x00,0xb9,0xd4,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
@@ -612,17 +673,14 @@
 # GFX12-REAL16: v_cmpx_ne_i16_e64_dpp v255.l, v255.l dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0x7e,0x00,0xb5,0xd4,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 # GFX12-FAKE16: v_cmpx_ne_i16_e64_dpp v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0x7e,0x00,0xb5,0xd4,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 
-# GFX11: v_cmpx_ne_i16_e64_dpp v1.h, v2.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x18,0xb5,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 0x7e,0x18,0xb5,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 # GFX12-REAL16: v_cmpx_ne_i16_e64_dpp v1.h, v2.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x18,0xb5,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # GFX12-FAKE16: v_cmpx_ne_i16_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0xb5,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 
-# GFX11: v_cmpx_ne_i16_e64_dpp v1.h, v2.l op_sel:[1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x08,0xb5,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 0x7e,0x08,0xb5,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 # GFX12-REAL16: v_cmpx_ne_i16_e64_dpp v1.h, v2.l op_sel:[1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x08,0xb5,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # GFX12-FAKE16: v_cmpx_ne_i16_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0xb5,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 
-# GFX11: v_cmpx_ne_i16_e64_dpp v255.l, v255.h op_sel:[0,1] dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0x7e,0x10,0xb5,0xd4,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 0x7e,0x10,0xb5,0xd4,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00
 # GFX12-REAL16: v_cmpx_ne_i16_e64_dpp v255.l, v255.h op_sel:[0,1] dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0x7e,0x10,0xb5,0xd4,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 # GFX12-FAKE16: v_cmpx_ne_i16_e64_dpp v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0x7e,0x00,0xb5,0xd4,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
@@ -651,17 +709,14 @@
 # GFX12-REAL16: v_cmpx_ne_u16_e64_dpp v255.l, v255.l dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0x7e,0x00,0xbd,0xd4,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 # GFX12-FAKE16: v_cmpx_ne_u16_e64_dpp v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0x7e,0x00,0xbd,0xd4,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 
-# GFX11: v_cmpx_ne_u16_e64_dpp v1.h, v2.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x18,0xbd,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 0x7e,0x18,0xbd,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 # GFX12-REAL16: v_cmpx_ne_u16_e64_dpp v1.h, v2.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x18,0xbd,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # GFX12-FAKE16: v_cmpx_ne_u16_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0xbd,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 
-# GFX11: v_cmpx_ne_u16_e64_dpp v1.h, v2.l op_sel:[1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x08,0xbd,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 0x7e,0x08,0xbd,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 # GFX12-REAL16: v_cmpx_ne_u16_e64_dpp v1.h, v2.l op_sel:[1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x08,0xbd,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # GFX12-FAKE16: v_cmpx_ne_u16_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0xbd,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 
-# GFX11: v_cmpx_ne_u16_e64_dpp v255.l, v255.h op_sel:[0,1] dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0x7e,0x10,0xbd,0xd4,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 0x7e,0x10,0xbd,0xd4,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00
 # GFX12-REAL16: v_cmpx_ne_u16_e64_dpp v255.l, v255.h op_sel:[0,1] dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0x7e,0x10,0xbd,0xd4,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 # GFX12-FAKE16: v_cmpx_ne_u16_e64_dpp v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0x7e,0x00,0xbd,0xd4,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
@@ -676,19 +731,37 @@
 # GFX12: v_cmpx_ne_u32_e64_dpp v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0x7e,0x00,0xcd,0xd4,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 
 0x7e,0x00,0x8d,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
-# GFX12: v_cmpx_neq_f16_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x8d,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+# GFX12-REAL16: v_cmpx_neq_f16_e64_dpp v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x8d,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+# GFX12-FAKE16: v_cmpx_neq_f16_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x8d,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 
 0x7e,0x00,0x8d,0xd4,0xe9,0x06,0x00,0x00,0x01,0x77,0x39,0x05
-# GFX12: v_cmpx_neq_f16_e64_dpp v1, s3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x8d,0xd4,0xe9,0x06,0x00,0x00,0x01,0x77,0x39,0x05]
+# GFX12-REAL16: v_cmpx_neq_f16_e64_dpp v1.l, s3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x8d,0xd4,0xe9,0x06,0x00,0x00,0x01,0x77,0x39,0x05]
+# GFX12-FAKE16: v_cmpx_neq_f16_e64_dpp v1, s3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x8d,0xd4,0xe9,0x06,0x00,0x00,0x01,0x77,0x39,0x05]
 
 0x7e,0x01,0x8d,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05
-# GFX12: v_cmpx_neq_f16_e64_dpp |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x01,0x8d,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+# GFX12-REAL16: v_cmpx_neq_f16_e64_dpp |v1.l|, -v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x01,0x8d,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+# GFX12-FAKE16: v_cmpx_neq_f16_e64_dpp |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x01,0x8d,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
 
 0x7e,0x02,0x8d,0xd4,0xe9,0x04,0x02,0x20,0x01,0x77,0x39,0x05
-# GFX12: v_cmpx_neq_f16_e64_dpp -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x02,0x8d,0xd4,0xe9,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+# GFX12-REAL16: v_cmpx_neq_f16_e64_dpp -v1.l, |v2.l| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x02,0x8d,0xd4,0xe9,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+# GFX12-FAKE16: v_cmpx_neq_f16_e64_dpp -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x02,0x8d,0xd4,0xe9,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
 
 0x7e,0x83,0x8d,0xd4,0xea,0xfe,0x03,0x60,0xff,0x00,0x00,0x00
-# GFX12: v_cmpx_neq_f16_e64_dpp -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0x7e,0x83,0x8d,0xd4,0xea,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+# GFX12-REAL16: v_cmpx_neq_f16_e64_dpp -|v255.l|, -|v255.l| clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0x7e,0x83,0x8d,0xd4,0xea,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+# GFX12-FAKE16: v_cmpx_neq_f16_e64_dpp -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0x7e,0x83,0x8d,0xd4,0xea,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+
+0x7e,0x19,0x8d,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05
+# GFX12-REAL16: v_cmpx_neq_f16_e64_dpp |v1.h|, -v2.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x19,0x8d,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+# GFX12-FAKE16: v_cmpx_neq_f16_e64_dpp |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x01,0x8d,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+
+0x7e,0x0a,0x8d,0xd4,0xe9,0x04,0x02,0x20,0x01,0x77,0x39,0x05
+# GFX12-REAL16: v_cmpx_neq_f16_e64_dpp -v1.h, |v2.l| op_sel:[1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x0a,0x8d,0xd4,0xe9,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+# GFX12-FAKE16: v_cmpx_neq_f16_e64_dpp -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x02,0x8d,0xd4,0xe9,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+
+0x7e,0x93,0x8d,0xd4,0xea,0xfe,0x03,0x60,0xff,0x00,0x00,0x00
+# GFX12-REAL16: v_cmpx_neq_f16_e64_dpp -|v255.l|, -|v255.h| op_sel:[0,1] clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0x7e,0x93,0x8d,0xd4,0xea,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+# GFX12-FAKE16: v_cmpx_neq_f16_e64_dpp -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0x7e,0x83,0x8d,0xd4,0xea,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+
 
 0x7e,0x00,0x9d,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 # GFX12: v_cmpx_neq_f32_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x9d,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
@@ -706,22 +779,41 @@
 # GFX12: v_cmpx_neq_f32_e64_dpp -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0x7e,0x83,0x9d,0xd4,0xea,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
 
 0x7e,0x00,0x89,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
-# GFX12: v_cmpx_nge_f16_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x89,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+# GFX12-REAL16: v_cmpx_nge_f16_e64_dpp v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x89,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+# GFX12-FAKE16: v_cmpx_nge_f16_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x89,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 
 0x7e,0x00,0x89,0xd4,0xe9,0xe8,0x01,0x00,0x01,0x77,0x39,0x05
-# GFX12: v_cmpx_nge_f16_e64_dpp v1, 2.0 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x89,0xd4,0xe9,0xe8,0x01,0x00,0x01,0x77,0x39,0x05]
+# GFX12-REAL16: v_cmpx_nge_f16_e64_dpp v1.l, 2.0 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x89,0xd4,0xe9,0xe8,0x01,0x00,0x01,0x77,0x39,0x05]
+# GFX12-FAKE16: v_cmpx_nge_f16_e64_dpp v1, 2.0 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x89,0xd4,0xe9,0xe8,0x01,0x00,0x01,0x77,0x39,0x05]
 
 0x7e,0x00,0x89,0xd4,0xe9,0x06,0x00,0x00,0x01,0x77,0x39,0x05
-# GFX12: v_cmpx_nge_f16_e64_dpp v1, s3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x89,0xd4,0xe9,0x06,0x00,0x00,0x01,0x77,0x39,0x05]
+# GFX12-REAL16: v_cmpx_nge_f16_e64_dpp v1.l, s3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x89,0xd4,0xe9,0x06,0x00,0x00,0x01,0x77,0x39,0x05]
+# GFX12-FAKE16: v_cmpx_nge_f16_e64_dpp v1, s3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x89,0xd4,0xe9,0x06,0x00,0x00,0x01,0x77,0x39,0x05]
 
 0x7e,0x01,0x89,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05
-# GFX12: v_cmpx_nge_f16_e64_dpp |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x01,0x89,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+# GFX12-REAL16: v_cmpx_nge_f16_e64_dpp |v1.l|, -v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x01,0x89,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+# GFX12-FAKE16: v_cmpx_nge_f16_e64_dpp |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x01,0x89,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
 
 0x7e,0x02,0x89,0xd4,0xe9,0x04,0x02,0x20,0x01,0x77,0x39,0x05
-# GFX12: v_cmpx_nge_f16_e64_dpp -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x02,0x89,0xd4,0xe9,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+# GFX12-REAL16: v_cmpx_nge_f16_e64_dpp -v1.l, |v2.l| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x02,0x89,0xd4,0xe9,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+# GFX12-FAKE16: v_cmpx_nge_f16_e64_dpp -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x02,0x89,0xd4,0xe9,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
 
 0x7e,0x83,0x89,0xd4,0xea,0xfe,0x03,0x60,0xff,0x00,0x00,0x00
-# GFX12: v_cmpx_nge_f16_e64_dpp -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0x7e,0x83,0x89,0xd4,0xea,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+# GFX12-REAL16: v_cmpx_nge_f16_e64_dpp -|v255.l|, -|v255.l| clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0x7e,0x83,0x89,0xd4,0xea,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+# GFX12-FAKE16: v_cmpx_nge_f16_e64_dpp -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0x7e,0x83,0x89,0xd4,0xea,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+
+0x7e,0x19,0x89,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05
+# GFX12-REAL16: v_cmpx_nge_f16_e64_dpp |v1.h|, -v2.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x19,0x89,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+# GFX12-FAKE16: v_cmpx_nge_f16_e64_dpp |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x01,0x89,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+
+0x7e,0x0a,0x89,0xd4,0xe9,0x04,0x02,0x20,0x01,0x77,0x39,0x05
+# GFX12-REAL16: v_cmpx_nge_f16_e64_dpp -v1.h, |v2.l| op_sel:[1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x0a,0x89,0xd4,0xe9,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+# GFX12-FAKE16: v_cmpx_nge_f16_e64_dpp -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x02,0x89,0xd4,0xe9,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+
+0x7e,0x93,0x89,0xd4,0xea,0xfe,0x03,0x60,0xff,0x00,0x00,0x00
+# GFX12-REAL16: v_cmpx_nge_f16_e64_dpp -|v255.l|, -|v255.h| op_sel:[0,1] clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0x7e,0x93,0x89,0xd4,0xea,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+# GFX12-FAKE16: v_cmpx_nge_f16_e64_dpp -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0x7e,0x83,0x89,0xd4,0xea,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+
 
 0x7e,0x00,0x99,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 # GFX12: v_cmpx_nge_f32_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x99,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
@@ -739,19 +831,37 @@
 # GFX12: v_cmpx_nge_f32_e64_dpp -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0x7e,0x83,0x99,0xd4,0xea,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
 
 0x7e,0x00,0x8b,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
-# GFX12: v_cmpx_ngt_f16_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x8b,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+# GFX12-REAL16: v_cmpx_ngt_f16_e64_dpp v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x8b,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+# GFX12-FAKE16: v_cmpx_ngt_f16_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x8b,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 
 0x7e,0x00,0x8b,0xd4,0xe9,0x06,0x00,0x00,0x01,0x77,0x39,0x05
-# GFX12: v_cmpx_ngt_f16_e64_dpp v1, s3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x8b,0xd4,0xe9,0x06,0x00,0x00,0x01,0x77,0x39,0x05]
+# GFX12-REAL16: v_cmpx_ngt_f16_e64_dpp v1.l, s3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x8b,0xd4,0xe9,0x06,0x00,0x00,0x01,0x77,0x39,0x05]
+# GFX12-FAKE16: v_cmpx_ngt_f16_e64_dpp v1, s3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x8b,0xd4,0xe9,0x06,0x00,0x00,0x01,0x77,0x39,0x05]
 
 0x7e,0x01,0x8b,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05
-# GFX12: v_cmpx_ngt_f16_e64_dpp |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x01,0x8b,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+# GFX12-REAL16: v_cmpx_ngt_f16_e64_dpp |v1.l|, -v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x01,0x8b,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+# GFX12-FAKE16: v_cmpx_ngt_f16_e64_dpp |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x01,0x8b,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
 
 0x7e,0x02,0x8b,0xd4,0xe9,0x04,0x02,0x20,0x01,0x77,0x39,0x05
-# GFX12: v_cmpx_ngt_f16_e64_dpp -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x02,0x8b,0xd4,0xe9,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+# GFX12-REAL16: v_cmpx_ngt_f16_e64_dpp -v1.l, |v2.l| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x02,0x8b,0xd4,0xe9,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+# GFX12-FAKE16: v_cmpx_ngt_f16_e64_dpp -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x02,0x8b,0xd4,0xe9,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
 
 0x7e,0x83,0x8b,0xd4,0xea,0xfe,0x03,0x60,0xff,0x00,0x00,0x00
-# GFX12: v_cmpx_ngt_f16_e64_dpp -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0x7e,0x83,0x8b,0xd4,0xea,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+# GFX12-REAL16: v_cmpx_ngt_f16_e64_dpp -|v255.l|, -|v255.l| clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0x7e,0x83,0x8b,0xd4,0xea,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+# GFX12-FAKE16: v_cmpx_ngt_f16_e64_dpp -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0x7e,0x83,0x8b,0xd4,0xea,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+
+0x7e,0x19,0x8b,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05
+# GFX12-REAL16: v_cmpx_ngt_f16_e64_dpp |v1.h|, -v2.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x19,0x8b,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+# GFX12-FAKE16: v_cmpx_ngt_f16_e64_dpp |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x01,0x8b,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+
+0x7e,0x0a,0x8b,0xd4,0xe9,0x04,0x02,0x20,0x01,0x77,0x39,0x05
+# GFX12-REAL16: v_cmpx_ngt_f16_e64_dpp -v1.h, |v2.l| op_sel:[1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x0a,0x8b,0xd4,0xe9,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+# GFX12-FAKE16: v_cmpx_ngt_f16_e64_dpp -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x02,0x8b,0xd4,0xe9,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+
+0x7e,0x93,0x8b,0xd4,0xea,0xfe,0x03,0x60,0xff,0x00,0x00,0x00
+# GFX12-REAL16: v_cmpx_ngt_f16_e64_dpp -|v255.l|, -|v255.h| op_sel:[0,1] clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0x7e,0x93,0x8b,0xd4,0xea,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+# GFX12-FAKE16: v_cmpx_ngt_f16_e64_dpp -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0x7e,0x83,0x8b,0xd4,0xea,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+
 
 0x7e,0x00,0x9b,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 # GFX12: v_cmpx_ngt_f32_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x9b,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
@@ -769,19 +879,37 @@
 # GFX12: v_cmpx_ngt_f32_e64_dpp -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0x7e,0x83,0x9b,0xd4,0xea,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
 
 0x7e,0x00,0x8c,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
-# GFX12: v_cmpx_nle_f16_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x8c,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+# GFX12-REAL16: v_cmpx_nle_f16_e64_dpp v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x8c,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+# GFX12-FAKE16: v_cmpx_nle_f16_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x8c,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 
 0x7e,0x00,0x8c,0xd4,0xe9,0x06,0x00,0x00,0x01,0x77,0x39,0x05
-# GFX12: v_cmpx_nle_f16_e64_dpp v1, s3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x8c,0xd4,0xe9,0x06,0x00,0x00,0x01,0x77,0x39,0x05]
+# GFX12-REAL16: v_cmpx_nle_f16_e64_dpp v1.l, s3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x8c,0xd4,0xe9,0x06,0x00,0x00,0x01,0x77,0x39,0x05]
+# GFX12-FAKE16: v_cmpx_nle_f16_e64_dpp v1, s3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x8c,0xd4,0xe9,0x06,0x00,0x00,0x01,0x77,0x39,0x05]
 
 0x7e,0x01,0x8c,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05
-# GFX12: v_cmpx_nle_f16_e64_dpp |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x01,0x8c,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+# GFX12-REAL16: v_cmpx_nle_f16_e64_dpp |v1.l|, -v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x01,0x8c,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+# GFX12-FAKE16: v_cmpx_nle_f16_e64_dpp |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x01,0x8c,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
 
 0x7e,0x02,0x8c,0xd4,0xe9,0x04,0x02,0x20,0x01,0x77,0x39,0x05
-# GFX12: v_cmpx_nle_f16_e64_dpp -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x02,0x8c,0xd4,0xe9,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+# GFX12-REAL16: v_cmpx_nle_f16_e64_dpp -v1.l, |v2.l| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x02,0x8c,0xd4,0xe9,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+# GFX12-FAKE16: v_cmpx_nle_f16_e64_dpp -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x02,0x8c,0xd4,0xe9,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
 
 0x7e,0x83,0x8c,0xd4,0xea,0xfe,0x03,0x60,0xff,0x00,0x00,0x00
-# GFX12: v_cmpx_nle_f16_e64_dpp -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0x7e,0x83,0x8c,0xd4,0xea,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+# GFX12-REAL16: v_cmpx_nle_f16_e64_dpp -|v255.l|, -|v255.l| clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0x7e,0x83,0x8c,0xd4,0xea,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+# GFX12-FAKE16: v_cmpx_nle_f16_e64_dpp -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0x7e,0x83,0x8c,0xd4,0xea,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+
+0x7e,0x19,0x8c,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05
+# GFX12-REAL16: v_cmpx_nle_f16_e64_dpp |v1.h|, -v2.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x19,0x8c,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+# GFX12-FAKE16: v_cmpx_nle_f16_e64_dpp |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x01,0x8c,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+
+0x7e,0x0a,0x8c,0xd4,0xe9,0x04,0x02,0x20,0x01,0x77,0x39,0x05
+# GFX12-REAL16: v_cmpx_nle_f16_e64_dpp -v1.h, |v2.l| op_sel:[1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x0a,0x8c,0xd4,0xe9,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+# GFX12-FAKE16: v_cmpx_nle_f16_e64_dpp -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x02,0x8c,0xd4,0xe9,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+
+0x7e,0x93,0x8c,0xd4,0xea,0xfe,0x03,0x60,0xff,0x00,0x00,0x00
+# GFX12-REAL16: v_cmpx_nle_f16_e64_dpp -|v255.l|, -|v255.h| op_sel:[0,1] clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0x7e,0x93,0x8c,0xd4,0xea,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+# GFX12-FAKE16: v_cmpx_nle_f16_e64_dpp -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0x7e,0x83,0x8c,0xd4,0xea,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+
 
 0x7e,0x00,0x9c,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 # GFX12: v_cmpx_nle_f32_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x9c,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
@@ -799,19 +927,37 @@
 # GFX12: v_cmpx_nle_f32_e64_dpp -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0x7e,0x83,0x9c,0xd4,0xea,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
 
 0x7e,0x00,0x8a,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
-# GFX12: v_cmpx_nlg_f16_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x8a,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+# GFX12-REAL16: v_cmpx_nlg_f16_e64_dpp v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x8a,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+# GFX12-FAKE16: v_cmpx_nlg_f16_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x8a,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 
 0x7e,0x00,0x8a,0xd4,0xe9,0x06,0x00,0x00,0x01,0x77,0x39,0x05
-# GFX12: v_cmpx_nlg_f16_e64_dpp v1, s3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x8a,0xd4,0xe9,0x06,0x00,0x00,0x01,0x77,0x39,0x05]
+# GFX12-REAL16: v_cmpx_nlg_f16_e64_dpp v1.l, s3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x8a,0xd4,0xe9,0x06,0x00,0x00,0x01,0x77,0x39,0x05]
+# GFX12-FAKE16: v_cmpx_nlg_f16_e64_dpp v1, s3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x8a,0xd4,0xe9,0x06,0x00,0x00,0x01,0x77,0x39,0x05]
 
 0x7e,0x01,0x8a,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05
-# GFX12: v_cmpx_nlg_f16_e64_dpp |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x01,0x8a,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+# GFX12-REAL16: v_cmpx_nlg_f16_e64_dpp |v1.l|, -v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x01,0x8a,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+# GFX12-FAKE16: v_cmpx_nlg_f16_e64_dpp |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x01,0x8a,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
 
 0x7e,0x02,0x8a,0xd4,0xe9,0x04,0x02,0x20,0x01,0x77,0x39,0x05
-# GFX12: v_cmpx_nlg_f16_e64_dpp -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x02,0x8a,0xd4,0xe9,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+# GFX12-REAL16: v_cmpx_nlg_f16_e64_dpp -v1.l, |v2.l| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x02,0x8a,0xd4,0xe9,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+# GFX12-FAKE16: v_cmpx_nlg_f16_e64_dpp -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x02,0x8a,0xd4,0xe9,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
 
 0x7e,0x83,0x8a,0xd4,0xea,0xfe,0x03,0x60,0xff,0x00,0x00,0x00
-# GFX12: v_cmpx_nlg_f16_e64_dpp -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0x7e,0x83,0x8a,0xd4,0xea,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+# GFX12-REAL16: v_cmpx_nlg_f16_e64_dpp -|v255.l|, -|v255.l| clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0x7e,0x83,0x8a,0xd4,0xea,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+# GFX12-FAKE16: v_cmpx_nlg_f16_e64_dpp -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0x7e,0x83,0x8a,0xd4,0xea,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+
+0x7e,0x19,0x8a,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05
+# GFX12-REAL16: v_cmpx_nlg_f16_e64_dpp |v1.h|, -v2.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x19,0x8a,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+# GFX12-FAKE16: v_cmpx_nlg_f16_e64_dpp |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x01,0x8a,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+
+0x7e,0x0a,0x8a,0xd4,0xe9,0x04,0x02,0x20,0x01,0x77,0x39,0x05
+# GFX12-REAL16: v_cmpx_nlg_f16_e64_dpp -v1.h, |v2.l| op_sel:[1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x0a,0x8a,0xd4,0xe9,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+# GFX12-FAKE16: v_cmpx_nlg_f16_e64_dpp -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x02,0x8a,0xd4,0xe9,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+
+0x7e,0x93,0x8a,0xd4,0xea,0xfe,0x03,0x60,0xff,0x00,0x00,0x00
+# GFX12-REAL16: v_cmpx_nlg_f16_e64_dpp -|v255.l|, -|v255.h| op_sel:[0,1] clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0x7e,0x93,0x8a,0xd4,0xea,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+# GFX12-FAKE16: v_cmpx_nlg_f16_e64_dpp -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0x7e,0x83,0x8a,0xd4,0xea,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+
 
 0x7e,0x00,0x9a,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 # GFX12: v_cmpx_nlg_f32_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x9a,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
@@ -829,19 +975,37 @@
 # GFX12: v_cmpx_nlg_f32_e64_dpp -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0x7e,0x83,0x9a,0xd4,0xea,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
 
 0x7e,0x00,0x8e,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
-# GFX12: v_cmpx_nlt_f16_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x8e,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+# GFX12-REAL16: v_cmpx_nlt_f16_e64_dpp v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x8e,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+# GFX12-FAKE16: v_cmpx_nlt_f16_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x8e,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 
 0x7e,0x00,0x8e,0xd4,0xe9,0x06,0x00,0x00,0x01,0x77,0x39,0x05
-# GFX12: v_cmpx_nlt_f16_e64_dpp v1, s3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x8e,0xd4,0xe9,0x06,0x00,0x00,0x01,0x77,0x39,0x05]
+# GFX12-REAL16: v_cmpx_nlt_f16_e64_dpp v1.l, s3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x8e,0xd4,0xe9,0x06,0x00,0x00,0x01,0x77,0x39,0x05]
+# GFX12-FAKE16: v_cmpx_nlt_f16_e64_dpp v1, s3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x8e,0xd4,0xe9,0x06,0x00,0x00,0x01,0x77,0x39,0x05]
 
 0x7e,0x01,0x8e,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05
-# GFX12: v_cmpx_nlt_f16_e64_dpp |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x01,0x8e,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+# GFX12-REAL16: v_cmpx_nlt_f16_e64_dpp |v1.l|, -v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x01,0x8e,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+# GFX12-FAKE16: v_cmpx_nlt_f16_e64_dpp |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x01,0x8e,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
 
 0x7e,0x02,0x8e,0xd4,0xe9,0x04,0x02,0x20,0x01,0x77,0x39,0x05
-# GFX12: v_cmpx_nlt_f16_e64_dpp -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x02,0x8e,0xd4,0xe9,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+# GFX12-REAL16: v_cmpx_nlt_f16_e64_dpp -v1.l, |v2.l| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x02,0x8e,0xd4,0xe9,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+# GFX12-FAKE16: v_cmpx_nlt_f16_e64_dpp -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x02,0x8e,0xd4,0xe9,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
 
 0x7e,0x83,0x8e,0xd4,0xea,0xfe,0x03,0x60,0xff,0x00,0x00,0x00
-# GFX12: v_cmpx_nlt_f16_e64_dpp -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0x7e,0x83,0x8e,0xd4,0xea,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+# GFX12-REAL16: v_cmpx_nlt_f16_e64_dpp -|v255.l|, -|v255.l| clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0x7e,0x83,0x8e,0xd4,0xea,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+# GFX12-FAKE16: v_cmpx_nlt_f16_e64_dpp -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0x7e,0x83,0x8e,0xd4,0xea,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+
+0x7e,0x19,0x8e,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05
+# GFX12-REAL16: v_cmpx_nlt_f16_e64_dpp |v1.h|, -v2.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x19,0x8e,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+# GFX12-FAKE16: v_cmpx_nlt_f16_e64_dpp |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x01,0x8e,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+
+0x7e,0x0a,0x8e,0xd4,0xe9,0x04,0x02,0x20,0x01,0x77,0x39,0x05
+# GFX12-REAL16: v_cmpx_nlt_f16_e64_dpp -v1.h, |v2.l| op_sel:[1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x0a,0x8e,0xd4,0xe9,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+# GFX12-FAKE16: v_cmpx_nlt_f16_e64_dpp -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x02,0x8e,0xd4,0xe9,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+
+0x7e,0x93,0x8e,0xd4,0xea,0xfe,0x03,0x60,0xff,0x00,0x00,0x00
+# GFX12-REAL16: v_cmpx_nlt_f16_e64_dpp -|v255.l|, -|v255.h| op_sel:[0,1] clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0x7e,0x93,0x8e,0xd4,0xea,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+# GFX12-FAKE16: v_cmpx_nlt_f16_e64_dpp -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0x7e,0x83,0x8e,0xd4,0xea,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+
 
 0x7e,0x00,0x9e,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 # GFX12: v_cmpx_nlt_f32_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x9e,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
@@ -859,19 +1023,37 @@
 # GFX12: v_cmpx_nlt_f32_e64_dpp -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0x7e,0x83,0x9e,0xd4,0xea,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
 
 0x7e,0x00,0x87,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
-# GFX12: v_cmpx_o_f16_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x87,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+# GFX12-REAL16: v_cmpx_o_f16_e64_dpp v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x87,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+# GFX12-FAKE16: v_cmpx_o_f16_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x87,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 
 0x7e,0x00,0x87,0xd4,0xe9,0x06,0x00,0x00,0x01,0x77,0x39,0x05
-# GFX12: v_cmpx_o_f16_e64_dpp v1, s3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x87,0xd4,0xe9,0x06,0x00,0x00,0x01,0x77,0x39,0x05]
+# GFX12-REAL16: v_cmpx_o_f16_e64_dpp v1.l, s3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x87,0xd4,0xe9,0x06,0x00,0x00,0x01,0x77,0x39,0x05]
+# GFX12-FAKE16: v_cmpx_o_f16_e64_dpp v1, s3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x87,0xd4,0xe9,0x06,0x00,0x00,0x01,0x77,0x39,0x05]
 
 0x7e,0x01,0x87,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05
-# GFX12: v_cmpx_o_f16_e64_dpp |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x01,0x87,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+# GFX12-REAL16: v_cmpx_o_f16_e64_dpp |v1.l|, -v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x01,0x87,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+# GFX12-FAKE16: v_cmpx_o_f16_e64_dpp |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x01,0x87,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
 
 0x7e,0x02,0x87,0xd4,0xe9,0x04,0x02,0x20,0x01,0x77,0x39,0x05
-# GFX12: v_cmpx_o_f16_e64_dpp -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x02,0x87,0xd4,0xe9,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+# GFX12-REAL16: v_cmpx_o_f16_e64_dpp -v1.l, |v2.l| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x02,0x87,0xd4,0xe9,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+# GFX12-FAKE16: v_cmpx_o_f16_e64_dpp -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x02,0x87,0xd4,0xe9,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
 
 0x7e,0x83,0x87,0xd4,0xea,0xfe,0x03,0x60,0xff,0x00,0x00,0x00
-# GFX12: v_cmpx_o_f16_e64_dpp -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0x7e,0x83,0x87,0xd4,0xea,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+# GFX12-REAL16: v_cmpx_o_f16_e64_dpp -|v255.l|, -|v255.l| clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0x7e,0x83,0x87,0xd4,0xea,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+# GFX12-FAKE16: v_cmpx_o_f16_e64_dpp -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0x7e,0x83,0x87,0xd4,0xea,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+
+0x7e,0x19,0x87,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05
+# GFX12-REAL16: v_cmpx_o_f16_e64_dpp |v1.h|, -v2.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x19,0x87,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+# GFX12-FAKE16: v_cmpx_o_f16_e64_dpp |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x01,0x87,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+
+0x7e,0x0a,0x87,0xd4,0xe9,0x04,0x02,0x20,0x01,0x77,0x39,0x05
+# GFX12-REAL16: v_cmpx_o_f16_e64_dpp -v1.h, |v2.l| op_sel:[1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x0a,0x87,0xd4,0xe9,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+# GFX12-FAKE16: v_cmpx_o_f16_e64_dpp -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x02,0x87,0xd4,0xe9,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+
+0x7e,0x93,0x87,0xd4,0xea,0xfe,0x03,0x60,0xff,0x00,0x00,0x00
+# GFX12-REAL16: v_cmpx_o_f16_e64_dpp -|v255.l|, -|v255.h| op_sel:[0,1] clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0x7e,0x93,0x87,0xd4,0xea,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+# GFX12-FAKE16: v_cmpx_o_f16_e64_dpp -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0x7e,0x83,0x87,0xd4,0xea,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+
 
 0x7e,0x00,0x97,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 # GFX12: v_cmpx_o_f32_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x97,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
@@ -889,19 +1071,36 @@
 # GFX12: v_cmpx_o_f32_e64_dpp -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0x7e,0x83,0x97,0xd4,0xea,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
 
 0x7e,0x00,0x88,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
-# GFX12: v_cmpx_u_f16_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x88,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+# GFX12-REAL16: v_cmpx_u_f16_e64_dpp v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x88,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+# GFX12-FAKE16: v_cmpx_u_f16_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x88,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 
 0x7e,0x00,0x88,0xd4,0xe9,0x06,0x00,0x00,0x01,0x77,0x39,0x05
-# GFX12: v_cmpx_u_f16_e64_dpp v1, s3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x88,0xd4,0xe9,0x06,0x00,0x00,0x01,0x77,0x39,0x05]
+# GFX12-REAL16: v_cmpx_u_f16_e64_dpp v1.l, s3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x88,0xd4,0xe9,0x06,0x00,0x00,0x01,0x77,0x39,0x05]
+# GFX12-FAKE16: v_cmpx_u_f16_e64_dpp v1, s3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x88,0xd4,0xe9,0x06,0x00,0x00,0x01,0x77,0x39,0x05]
 
 0x7e,0x01,0x88,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05
-# GFX12: v_cmpx_u_f16_e64_dpp |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x01,0x88,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+# GFX12-REAL16: v_cmpx_u_f16_e64_dpp |v1.l|, -v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x01,0x88,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+# GFX12-FAKE16: v_cmpx_u_f16_e64_dpp |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x01,0x88,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
 
 0x7e,0x02,0x88,0xd4,0xe9,0x04,0x02,0x20,0x01,0x77,0x39,0x05
-# GFX12: v_cmpx_u_f16_e64_dpp -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x02,0x88,0xd4,0xe9,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+# GFX12-REAL16: v_cmpx_u_f16_e64_dpp -v1.l, |v2.l| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x02,0x88,0xd4,0xe9,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+# GFX12-FAKE16: v_cmpx_u_f16_e64_dpp -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x02,0x88,0xd4,0xe9,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
 
 0x7e,0x83,0x88,0xd4,0xea,0xfe,0x03,0x60,0xff,0x00,0x00,0x00
-# GFX12: v_cmpx_u_f16_e64_dpp -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0x7e,0x83,0x88,0xd4,0xea,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+# GFX12-REAL16: v_cmpx_u_f16_e64_dpp -|v255.l|, -|v255.l| clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0x7e,0x83,0x88,0xd4,0xea,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+# GFX12-FAKE16: v_cmpx_u_f16_e64_dpp -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0x7e,0x83,0x88,0xd4,0xea,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+
+0x7e,0x19,0x88,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05
+# GFX12-REAL16: v_cmpx_u_f16_e64_dpp |v1.h|, -v2.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x19,0x88,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+# GFX12-FAKE16: v_cmpx_u_f16_e64_dpp |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x01,0x88,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+
+0x7e,0x0a,0x88,0xd4,0xe9,0x04,0x02,0x20,0x01,0x77,0x39,0x05
+# GFX12-REAL16: v_cmpx_u_f16_e64_dpp -v1.h, |v2.l| op_sel:[1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x0a,0x88,0xd4,0xe9,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+# GFX12-FAKE16: v_cmpx_u_f16_e64_dpp -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x02,0x88,0xd4,0xe9,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+
+0x7e,0x93,0x88,0xd4,0xea,0xfe,0x03,0x60,0xff,0x00,0x00,0x00
+# GFX12-REAL16: v_cmpx_u_f16_e64_dpp -|v255.l|, -|v255.h| op_sel:[0,1] clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0x7e,0x93,0x88,0xd4,0xea,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+# GFX12-FAKE16: v_cmpx_u_f16_e64_dpp -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0x7e,0x83,0x88,0xd4,0xea,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
 
 0x7e,0x00,0x98,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 # GFX12: v_cmpx_u_f32_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x98,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
diff --git a/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vopcx.txt b/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vopcx.txt
index 6ca815a1c88d3..373cc6b6e5878 100644
--- a/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vopcx.txt
+++ b/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vopcx.txt
@@ -162,49 +162,80 @@
 # GFX12: v_cmpx_class_f64_e32 0xaf123456, v255   ; encoding: [0xff,0xfe,0xff,0x7d,0x56,0x34,0x12,0xaf]
 
 0x01,0x05,0x04,0x7d
-# GFX12: v_cmpx_eq_f16_e32 v1, v2                ; encoding: [0x01,0x05,0x04,0x7d]
+# GFX12-REAL16: v_cmpx_eq_f16_e32 v1.l, v2.l            ; encoding: [0x01,0x05,0x04,0x7d]
+# GFX12-FAKE16: v_cmpx_eq_f16_e32 v1, v2                ; encoding: [0x01,0x05,0x04,0x7d]
 
 0x7f,0x05,0x04,0x7d
-# GFX12: v_cmpx_eq_f16_e32 v127, v2              ; encoding: [0x7f,0x05,0x04,0x7d]
+# GFX12-REAL16: v_cmpx_eq_f16_e32 v127.l, v2.l          ; encoding: [0x7f,0x05,0x04,0x7d]
+# GFX12-FAKE16: v_cmpx_eq_f16_e32 v127, v2              ; encoding: [0x7f,0x05,0x04,0x7d]
 
 0x01,0x04,0x04,0x7d
-# GFX12: v_cmpx_eq_f16_e32 s1, v2                ; encoding: [0x01,0x04,0x04,0x7d]
+# GFX12-REAL16: v_cmpx_eq_f16_e32 s1, v2.l              ; encoding: [0x01,0x04,0x04,0x7d]
+# GFX12-FAKE16: v_cmpx_eq_f16_e32 s1, v2                ; encoding: [0x01,0x04,0x04,0x7d]
 
 0x69,0x04,0x04,0x7d
-# GFX12: v_cmpx_eq_f16_e32 s105, v2              ; encoding: [0x69,0x04,0x04,0x7d]
+# GFX12-REAL16: v_cmpx_eq_f16_e32 s105, v2.l            ; encoding: [0x69,0x04,0x04,0x7d]
+# GFX12-FAKE16: v_cmpx_eq_f16_e32 s105, v2              ; encoding: [0x69,0x04,0x04,0x7d]
 
 0x6a,0x04,0x04,0x7d
-# GFX12: v_cmpx_eq_f16_e32 vcc_lo, v2            ; encoding: [0x6a,0x04,0x04,0x7d]
+# GFX12-REAL16: v_cmpx_eq_f16_e32 vcc_lo, v2.l          ; encoding: [0x6a,0x04,0x04,0x7d]
+# GFX12-FAKE16: v_cmpx_eq_f16_e32 vcc_lo, v2            ; encoding: [0x6a,0x04,0x04,0x7d]
 
 0x6b,0x04,0x04,0x7d
-# GFX12: v_cmpx_eq_f16_e32 vcc_hi, v2            ; encoding: [0x6b,0x04,0x04,0x7d]
+# GFX12-REAL16: v_cmpx_eq_f16_e32 vcc_hi, v2.l          ; encoding: [0x6b,0x04,0x04,0x7d]
+# GFX12-FAKE16: v_cmpx_eq_f16_e32 vcc_hi, v2            ; encoding: [0x6b,0x04,0x04,0x7d]
 
 0x7b,0x04,0x04,0x7d
-# GFX12: v_cmpx_eq_f16_e32 ttmp15, v2            ; encoding: [0x7b,0x04,0x04,0x7d]
+# GFX12-REAL16: v_cmpx_eq_f16_e32 ttmp15, v2.l          ; encoding: [0x7b,0x04,0x04,0x7d]
+# GFX12-FAKE16: v_cmpx_eq_f16_e32 ttmp15, v2            ; encoding: [0x7b,0x04,0x04,0x7d]
 
 0x7d,0x04,0x04,0x7d
-# GFX12: v_cmpx_eq_f16_e32 m0, v2                ; encoding: [0x7d,0x04,0x04,0x7d]
+# GFX12-REAL16: v_cmpx_eq_f16_e32 m0, v2.l              ; encoding: [0x7d,0x04,0x04,0x7d]
+# GFX12-FAKE16: v_cmpx_eq_f16_e32 m0, v2                ; encoding: [0x7d,0x04,0x04,0x7d]
 
 0x7e,0x04,0x04,0x7d
-# GFX12: v_cmpx_eq_f16_e32 exec_lo, v2           ; encoding: [0x7e,0x04,0x04,0x7d]
+# GFX12-REAL16: v_cmpx_eq_f16_e32 exec_lo, v2.l         ; encoding: [0x7e,0x04,0x04,0x7d]
+# GFX12-FAKE16: v_cmpx_eq_f16_e32 exec_lo, v2           ; encoding: [0x7e,0x04,0x04,0x7d]
 
 0x7f,0x04,0x04,0x7d
-# GFX12: v_cmpx_eq_f16_e32 exec_hi, v2           ; encoding: [0x7f,0x04,0x04,0x7d]
+# GFX12-REAL16: v_cmpx_eq_f16_e32 exec_hi, v2.l         ; encoding: [0x7f,0x04,0x04,0x7d]
+# GFX12-FAKE16: v_cmpx_eq_f16_e32 exec_hi, v2           ; encoding: [0x7f,0x04,0x04,0x7d]
 
 0x7c,0x04,0x04,0x7d
-# GFX12: v_cmpx_eq_f16_e32 null, v2              ; encoding: [0x7c,0x04,0x04,0x7d]
+# GFX12-REAL16: v_cmpx_eq_f16_e32 null, v2.l            ; encoding: [0x7c,0x04,0x04,0x7d]
+# GFX12-FAKE16: v_cmpx_eq_f16_e32 null, v2              ; encoding: [0x7c,0x04,0x04,0x7d]
 
 0xc1,0x04,0x04,0x7d
-# GFX12: v_cmpx_eq_f16_e32 -1, v2                ; encoding: [0xc1,0x04,0x04,0x7d]
+# GFX12-REAL16: v_cmpx_eq_f16_e32 -1, v2.l              ; encoding: [0xc1,0x04,0x04,0x7d]
+# GFX12-FAKE16: v_cmpx_eq_f16_e32 -1, v2                ; encoding: [0xc1,0x04,0x04,0x7d]
 
 0xf0,0x04,0x04,0x7d
-# GFX12: v_cmpx_eq_f16_e32 0.5, v2               ; encoding: [0xf0,0x04,0x04,0x7d]
+# GFX12-REAL16: v_cmpx_eq_f16_e32 0.5, v2.l             ; encoding: [0xf0,0x04,0x04,0x7d]
+# GFX12-FAKE16: v_cmpx_eq_f16_e32 0.5, v2               ; encoding: [0xf0,0x04,0x04,0x7d]
 
 0xfd,0x04,0x04,0x7d
-# GFX12: v_cmpx_eq_f16_e32 src_scc, v2           ; encoding: [0xfd,0x04,0x04,0x7d]
+# GFX12-REAL16: v_cmpx_eq_f16_e32 src_scc, v2.l         ; encoding: [0xfd,0x04,0x04,0x7d]
+# GFX12-FAKE16: v_cmpx_eq_f16_e32 src_scc, v2           ; encoding: [0xfd,0x04,0x04,0x7d]
 
 0xff,0xfe,0x04,0x7d,0x0b,0xfe,0x00,0x00
-# GFX12: v_cmpx_eq_f16_e32 0xfe0b, v127          ; encoding: [0xff,0xfe,0x04,0x7d,0x0b,0xfe,0x00,0x00]
+# GFX12-REAL16: v_cmpx_eq_f16_e32 0xfe0b, v127.l        ; encoding: [0xff,0xfe,0x04,0x7d,0x0b,0xfe,0x00,0x00]
+# GFX12-FAKE16: v_cmpx_eq_f16_e32 0xfe0b, v127          ; encoding: [0xff,0xfe,0x04,0x7d,0x0b,0xfe,0x00,0x00]
+
+0x81,0x05,0x04,0x7d
+# GFX12-REAL16: v_cmpx_eq_f16_e32 v1.h, v2.l            ; encoding: [0x81,0x05,0x04,0x7d]
+# GFX12-FAKE16: v_cmpx_eq_f16_e32 v129/*Invalid register, operand has 'VS_32_Lo128' register class*/, v2 ; encoding: [0x81,0x05,0x04,0x7d]
+
+0xff,0x05,0x04,0x7d
+# GFX12-REAL16: v_cmpx_eq_f16_e32 v127.h, v2.l          ; encoding: [0xff,0x05,0x04,0x7d]
+# GFX12-FAKE16: v_cmpx_eq_f16_e32 v255/*Invalid register, operand has 'VS_32_Lo128' register class*/, v2 ; encoding: [0xff,0x05,0x04,0x7d]
+
+0xfd,0x04,0x05,0x7d
+# GFX12-REAL16: v_cmpx_eq_f16_e32 src_scc, v2.h         ; encoding: [0xfd,0x04,0x05,0x7d]
+# GFX12-FAKE16: v_cmpx_eq_f16_e32 src_scc, v130/*Invalid register, operand has 'VGPR_32_Lo128' register class*/ ; encoding: [0xfd,0x04,0x05,0x7d]
+
+0xff,0xfe,0x05,0x7d,0x0b,0xfe,0x00,0x00
+# GFX12-REAL16: v_cmpx_eq_f16_e32 0xfe0b, v127.h        ; encoding: [0xff,0xfe,0x05,0x7d,0x0b,0xfe,0x00,0x00]
+# GFX12-FAKE16: v_cmpx_eq_f16_e32 0xfe0b, v255/*Invalid register, operand has 'VGPR_32_Lo128' register class*/ ; encoding: [0xff,0xfe,0x05,0x7d,0x0b,0xfe,0x00,0x00]
 
 0x01,0x05,0x24,0x7d
 # GFX12: v_cmpx_eq_f32_e32 v1, v2                ; encoding: [0x01,0x05,0x24,0x7d]
@@ -602,49 +633,80 @@
 # GFX12: v_cmpx_eq_u64_e32 0xaf123456, v[254:255] ; encoding: [0xff,0xfc,0xb5,0x7d,0x56,0x34,0x12,0xaf]
 
 0x01,0x05,0x0c,0x7d
-# GFX12: v_cmpx_ge_f16_e32 v1, v2                ; encoding: [0x01,0x05,0x0c,0x7d]
+# GFX12-REAL16: v_cmpx_ge_f16_e32 v1.l, v2.l            ; encoding: [0x01,0x05,0x0c,0x7d]
+# GFX12-FAKE16: v_cmpx_ge_f16_e32 v1, v2                ; encoding: [0x01,0x05,0x0c,0x7d]
 
 0x7f,0x05,0x0c,0x7d
-# GFX12: v_cmpx_ge_f16_e32 v127, v2              ; encoding: [0x7f,0x05,0x0c,0x7d]
+# GFX12-REAL16: v_cmpx_ge_f16_e32 v127.l, v2.l          ; encoding: [0x7f,0x05,0x0c,0x7d]
+# GFX12-FAKE16: v_cmpx_ge_f16_e32 v127, v2              ; encoding: [0x7f,0x05,0x0c,0x7d]
 
 0x01,0x04,0x0c,0x7d
-# GFX12: v_cmpx_ge_f16_e32 s1, v2                ; encoding: [0x01,0x04,0x0c,0x7d]
+# GFX12-REAL16: v_cmpx_ge_f16_e32 s1, v2.l              ; encoding: [0x01,0x04,0x0c,0x7d]
+# GFX12-FAKE16: v_cmpx_ge_f16_e32 s1, v2                ; encoding: [0x01,0x04,0x0c,0x7d]
 
 0x69,0x04,0x0c,0x7d
-# GFX12: v_cmpx_ge_f16_e32 s105, v2              ; encoding: [0x69,0x04,0x0c,0x7d]
+# GFX12-REAL16: v_cmpx_ge_f16_e32 s105, v2.l            ; encoding: [0x69,0x04,0x0c,0x7d]
+# GFX12-FAKE16: v_cmpx_ge_f16_e32 s105, v2              ; encoding: [0x69,0x04,0x0c,0x7d]
 
 0x6a,0x04,0x0c,0x7d
-# GFX12: v_cmpx_ge_f16_e32 vcc_lo, v2            ; encoding: [0x6a,0x04,0x0c,0x7d]
+# GFX12-REAL16: v_cmpx_ge_f16_e32 vcc_lo, v2.l          ; encoding: [0x6a,0x04,0x0c,0x7d]
+# GFX12-FAKE16: v_cmpx_ge_f16_e32 vcc_lo, v2            ; encoding: [0x6a,0x04,0x0c,0x7d]
 
 0x6b,0x04,0x0c,0x7d
-# GFX12: v_cmpx_ge_f16_e32 vcc_hi, v2            ; encoding: [0x6b,0x04,0x0c,0x7d]
+# GFX12-REAL16: v_cmpx_ge_f16_e32 vcc_hi, v2.l          ; encoding: [0x6b,0x04,0x0c,0x7d]
+# GFX12-FAKE16: v_cmpx_ge_f16_e32 vcc_hi, v2            ; encoding: [0x6b,0x04,0x0c,0x7d]
 
 0x7b,0x04,0x0c,0x7d
-# GFX12: v_cmpx_ge_f16_e32 ttmp15, v2            ; encoding: [0x7b,0x04,0x0c,0x7d]
+# GFX12-REAL16: v_cmpx_ge_f16_e32 ttmp15, v2.l          ; encoding: [0x7b,0x04,0x0c,0x7d]
+# GFX12-FAKE16: v_cmpx_ge_f16_e32 ttmp15, v2            ; encoding: [0x7b,0x04,0x0c,0x7d]
 
 0x7d,0x04,0x0c,0x7d
-# GFX12: v_cmpx_ge_f16_e32 m0, v2                ; encoding: [0x7d,0x04,0x0c,0x7d]
+# GFX12-REAL16: v_cmpx_ge_f16_e32 m0, v2.l              ; encoding: [0x7d,0x04,0x0c,0x7d]
+# GFX12-FAKE16: v_cmpx_ge_f16_e32 m0, v2                ; encoding: [0x7d,0x04,0x0c,0x7d]
 
 0x7e,0x04,0x0c,0x7d
-# GFX12: v_cmpx_ge_f16_e32 exec_lo, v2           ; encoding: [0x7e,0x04,0x0c,0x7d]
+# GFX12-REAL16: v_cmpx_ge_f16_e32 exec_lo, v2.l         ; encoding: [0x7e,0x04,0x0c,0x7d]
+# GFX12-FAKE16: v_cmpx_ge_f16_e32 exec_lo, v2           ; encoding: [0x7e,0x04,0x0c,0x7d]
 
 0x7f,0x04,0x0c,0x7d
-# GFX12: v_cmpx_ge_f16_e32 exec_hi, v2           ; encoding: [0x7f,0x04,0x0c,0x7d]
+# GFX12-REAL16: v_cmpx_ge_f16_e32 exec_hi, v2.l         ; encoding: [0x7f,0x04,0x0c,0x7d]
+# GFX12-FAKE16: v_cmpx_ge_f16_e32 exec_hi, v2           ; encoding: [0x7f,0x04,0x0c,0x7d]
 
 0x7c,0x04,0x0c,0x7d
-# GFX12: v_cmpx_ge_f16_e32 null, v2              ; encoding: [0x7c,0x04,0x0c,0x7d]
+# GFX12-REAL16: v_cmpx_ge_f16_e32 null, v2.l            ; encoding: [0x7c,0x04,0x0c,0x7d]
+# GFX12-FAKE16: v_cmpx_ge_f16_e32 null, v2              ; encoding: [0x7c,0x04,0x0c,0x7d]
 
 0xc1,0x04,0x0c,0x7d
-# GFX12: v_cmpx_ge_f16_e32 -1, v2                ; encoding: [0xc1,0x04,0x0c,0x7d]
+# GFX12-REAL16: v_cmpx_ge_f16_e32 -1, v2.l              ; encoding: [0xc1,0x04,0x0c,0x7d]
+# GFX12-FAKE16: v_cmpx_ge_f16_e32 -1, v2                ; encoding: [0xc1,0x04,0x0c,0x7d]
 
 0xf0,0x04,0x0c,0x7d
-# GFX12: v_cmpx_ge_f16_e32 0.5, v2               ; encoding: [0xf0,0x04,0x0c,0x7d]
+# GFX12-REAL16: v_cmpx_ge_f16_e32 0.5, v2.l             ; encoding: [0xf0,0x04,0x0c,0x7d]
+# GFX12-FAKE16: v_cmpx_ge_f16_e32 0.5, v2               ; encoding: [0xf0,0x04,0x0c,0x7d]
 
 0xfd,0x04,0x0c,0x7d
-# GFX12: v_cmpx_ge_f16_e32 src_scc, v2           ; encoding: [0xfd,0x04,0x0c,0x7d]
+# GFX12-REAL16: v_cmpx_ge_f16_e32 src_scc, v2.l         ; encoding: [0xfd,0x04,0x0c,0x7d]
+# GFX12-FAKE16: v_cmpx_ge_f16_e32 src_scc, v2           ; encoding: [0xfd,0x04,0x0c,0x7d]
 
 0xff,0xfe,0x0c,0x7d,0x0b,0xfe,0x00,0x00
-# GFX12: v_cmpx_ge_f16_e32 0xfe0b, v127          ; encoding: [0xff,0xfe,0x0c,0x7d,0x0b,0xfe,0x00,0x00]
+# GFX12-REAL16: v_cmpx_ge_f16_e32 0xfe0b, v127.l        ; encoding: [0xff,0xfe,0x0c,0x7d,0x0b,0xfe,0x00,0x00]
+# GFX12-FAKE16: v_cmpx_ge_f16_e32 0xfe0b, v127          ; encoding: [0xff,0xfe,0x0c,0x7d,0x0b,0xfe,0x00,0x00]
+
+0x81,0x05,0x0c,0x7d
+# GFX12-REAL16: v_cmpx_ge_f16_e32 v1.h, v2.l            ; encoding: [0x81,0x05,0x0c,0x7d]
+# GFX12-FAKE16: v_cmpx_ge_f16_e32 v129/*Invalid register, operand has 'VS_32_Lo128' register class*/, v2 ; encoding: [0x81,0x05,0x0c,0x7d]
+
+0xff,0x05,0x0c,0x7d
+# GFX12-REAL16: v_cmpx_ge_f16_e32 v127.h, v2.l          ; encoding: [0xff,0x05,0x0c,0x7d]
+# GFX12-FAKE16: v_cmpx_ge_f16_e32 v255/*Invalid register, operand has 'VS_32_Lo128' register class*/, v2 ; encoding: [0xff,0x05,0x0c,0x7d]
+
+0xfd,0x04,0x0d,0x7d
+# GFX12-REAL16: v_cmpx_ge_f16_e32 src_scc, v2.h         ; encoding: [0xfd,0x04,0x0d,0x7d]
+# GFX12-FAKE16: v_cmpx_ge_f16_e32 src_scc, v130/*Invalid register, operand has 'VGPR_32_Lo128' register class*/ ; encoding: [0xfd,0x04,0x0d,0x7d]
+
+0xff,0xfe,0x0d,0x7d,0x0b,0xfe,0x00,0x00
+# GFX12-REAL16: v_cmpx_ge_f16_e32 0xfe0b, v127.h        ; encoding: [0xff,0xfe,0x0d,0x7d,0x0b,0xfe,0x00,0x00]
+# GFX12-FAKE16: v_cmpx_ge_f16_e32 0xfe0b, v255/*Invalid register, operand has 'VGPR_32_Lo128' register class*/ ; encoding: [0xff,0xfe,0x0d,0x7d,0x0b,0xfe,0x00,0x00]
 
 0x01,0x05,0x2c,0x7d
 # GFX12: v_cmpx_ge_f32_e32 v1, v2                ; encoding: [0x01,0x05,0x2c,0x7d]
@@ -1042,49 +1104,80 @@
 # GFX12: v_cmpx_ge_u64_e32 0xaf123456, v[254:255] ; encoding: [0xff,0xfc,0xbd,0x7d,0x56,0x34,0x12,0xaf]
 
 0x01,0x05,0x08,0x7d
-# GFX12: v_cmpx_gt_f16_e32 v1, v2                ; encoding: [0x01,0x05,0x08,0x7d]
+# GFX12-REAL16: v_cmpx_gt_f16_e32 v1.l, v2.l            ; encoding: [0x01,0x05,0x08,0x7d]
+# GFX12-FAKE16: v_cmpx_gt_f16_e32 v1, v2                ; encoding: [0x01,0x05,0x08,0x7d]
 
 0x7f,0x05,0x08,0x7d
-# GFX12: v_cmpx_gt_f16_e32 v127, v2              ; encoding: [0x7f,0x05,0x08,0x7d]
+# GFX12-REAL16: v_cmpx_gt_f16_e32 v127.l, v2.l          ; encoding: [0x7f,0x05,0x08,0x7d]
+# GFX12-FAKE16: v_cmpx_gt_f16_e32 v127, v2              ; encoding: [0x7f,0x05,0x08,0x7d]
 
 0x01,0x04,0x08,0x7d
-# GFX12: v_cmpx_gt_f16_e32 s1, v2                ; encoding: [0x01,0x04,0x08,0x7d]
+# GFX12-REAL16: v_cmpx_gt_f16_e32 s1, v2.l              ; encoding: [0x01,0x04,0x08,0x7d]
+# GFX12-FAKE16: v_cmpx_gt_f16_e32 s1, v2                ; encoding: [0x01,0x04,0x08,0x7d]
 
 0x69,0x04,0x08,0x7d
-# GFX12: v_cmpx_gt_f16_e32 s105, v2              ; encoding: [0x69,0x04,0x08,0x7d]
+# GFX12-REAL16: v_cmpx_gt_f16_e32 s105, v2.l            ; encoding: [0x69,0x04,0x08,0x7d]
+# GFX12-FAKE16: v_cmpx_gt_f16_e32 s105, v2              ; encoding: [0x69,0x04,0x08,0x7d]
 
 0x6a,0x04,0x08,0x7d
-# GFX12: v_cmpx_gt_f16_e32 vcc_lo, v2            ; encoding: [0x6a,0x04,0x08,0x7d]
+# GFX12-REAL16: v_cmpx_gt_f16_e32 vcc_lo, v2.l          ; encoding: [0x6a,0x04,0x08,0x7d]
+# GFX12-FAKE16: v_cmpx_gt_f16_e32 vcc_lo, v2            ; encoding: [0x6a,0x04,0x08,0x7d]
 
 0x6b,0x04,0x08,0x7d
-# GFX12: v_cmpx_gt_f16_e32 vcc_hi, v2            ; encoding: [0x6b,0x04,0x08,0x7d]
+# GFX12-REAL16: v_cmpx_gt_f16_e32 vcc_hi, v2.l          ; encoding: [0x6b,0x04,0x08,0x7d]
+# GFX12-FAKE16: v_cmpx_gt_f16_e32 vcc_hi, v2            ; encoding: [0x6b,0x04,0x08,0x7d]
 
 0x7b,0x04,0x08,0x7d
-# GFX12: v_cmpx_gt_f16_e32 ttmp15, v2            ; encoding: [0x7b,0x04,0x08,0x7d]
+# GFX12-REAL16: v_cmpx_gt_f16_e32 ttmp15, v2.l          ; encoding: [0x7b,0x04,0x08,0x7d]
+# GFX12-FAKE16: v_cmpx_gt_f16_e32 ttmp15, v2            ; encoding: [0x7b,0x04,0x08,0x7d]
 
 0x7d,0x04,0x08,0x7d
-# GFX12: v_cmpx_gt_f16_e32 m0, v2                ; encoding: [0x7d,0x04,0x08,0x7d]
+# GFX12-REAL16: v_cmpx_gt_f16_e32 m0, v2.l              ; encoding: [0x7d,0x04,0x08,0x7d]
+# GFX12-FAKE16: v_cmpx_gt_f16_e32 m0, v2                ; encoding: [0x7d,0x04,0x08,0x7d]
 
 0x7e,0x04,0x08,0x7d
-# GFX12: v_cmpx_gt_f16_e32 exec_lo, v2           ; encoding: [0x7e,0x04,0x08,0x7d]
+# GFX12-REAL16: v_cmpx_gt_f16_e32 exec_lo, v2.l         ; encoding: [0x7e,0x04,0x08,0x7d]
+# GFX12-FAKE16: v_cmpx_gt_f16_e32 exec_lo, v2           ; encoding: [0x7e,0x04,0x08,0x7d]
 
 0x7f,0x04,0x08,0x7d
-# GFX12: v_cmpx_gt_f16_e32 exec_hi, v2           ; encoding: [0x7f,0x04,0x08,0x7d]
+# GFX12-REAL16: v_cmpx_gt_f16_e32 exec_hi, v2.l         ; encoding: [0x7f,0x04,0x08,0x7d]
+# GFX12-FAKE16: v_cmpx_gt_f16_e32 exec_hi, v2           ; encoding: [0x7f,0x04,0x08,0x7d]
 
 0x7c,0x04,0x08,0x7d
-# GFX12: v_cmpx_gt_f16_e32 null, v2              ; encoding: [0x7c,0x04,0x08,0x7d]
+# GFX12-REAL16: v_cmpx_gt_f16_e32 null, v2.l            ; encoding: [0x7c,0x04,0x08,0x7d]
+# GFX12-FAKE16: v_cmpx_gt_f16_e32 null, v2              ; encoding: [0x7c,0x04,0x08,0x7d]
 
 0xc1,0x04,0x08,0x7d
-# GFX12: v_cmpx_gt_f16_e32 -1, v2                ; encoding: [0xc1,0x04,0x08,0x7d]
+# GFX12-REAL16: v_cmpx_gt_f16_e32 -1, v2.l              ; encoding: [0xc1,0x04,0x08,0x7d]
+# GFX12-FAKE16: v_cmpx_gt_f16_e32 -1, v2                ; encoding: [0xc1,0x04,0x08,0x7d]
 
 0xf0,0x04,0x08,0x7d
-# GFX12: v_cmpx_gt_f16_e32 0.5, v2               ; encoding: [0xf0,0x04,0x08,0x7d]
+# GFX12-REAL16: v_cmpx_gt_f16_e32 0.5, v2.l             ; encoding: [0xf0,0x04,0x08,0x7d]
+# GFX12-FAKE16: v_cmpx_gt_f16_e32 0.5, v2               ; encoding: [0xf0,0x04,0x08,0x7d]
 
 0xfd,0x04,0x08,0x7d
-# GFX12: v_cmpx_gt_f16_e32 src_scc, v2           ; encoding: [0xfd,0x04,0x08,0x7d]
+# GFX12-REAL16: v_cmpx_gt_f16_e32 src_scc, v2.l         ; encoding: [0xfd,0x04,0x08,0x7d]
+# GFX12-FAKE16: v_cmpx_gt_f16_e32 src_scc, v2           ; encoding: [0xfd,0x04,0x08,0x7d]
 
 0xff,0xfe,0x08,0x7d,0x0b,0xfe,0x00,0x00
-# GFX12: v_cmpx_gt_f16_e32 0xfe0b, v127          ; encoding: [0xff,0xfe,0x08,0x7d,0x0b,0xfe,0x00,0x00]
+# GFX12-REAL16: v_cmpx_gt_f16_e32 0xfe0b, v127.l        ; encoding: [0xff,0xfe,0x08,0x7d,0x0b,0xfe,0x00,0x00]
+# GFX12-FAKE16: v_cmpx_gt_f16_e32 0xfe0b, v127          ; encoding: [0xff,0xfe,0x08,0x7d,0x0b,0xfe,0x00,0x00]
+
+0x81,0x05,0x08,0x7d
+# GFX12-REAL16: v_cmpx_gt_f16_e32 v1.h, v2.l            ; encoding: [0x81,0x05,0x08,0x7d]
+# GFX12-FAKE16: v_cmpx_gt_f16_e32 v129/*Invalid register, operand has 'VS_32_Lo128' register class*/, v2 ; encoding: [0x81,0x05,0x08,0x7d]
+
+0xff,0x05,0x08,0x7d
+# GFX12-REAL16: v_cmpx_gt_f16_e32 v127.h, v2.l          ; encoding: [0xff,0x05,0x08,0x7d]
+# GFX12-FAKE16: v_cmpx_gt_f16_e32 v255/*Invalid register, operand has 'VS_32_Lo128' register class*/, v2 ; encoding: [0xff,0x05,0x08,0x7d]
+
+0xfd,0x04,0x09,0x7d
+# GFX12-REAL16: v_cmpx_gt_f16_e32 src_scc, v2.h         ; encoding: [0xfd,0x04,0x09,0x7d]
+# GFX12-FAKE16: v_cmpx_gt_f16_e32 src_scc, v130/*Invalid register, operand has 'VGPR_32_Lo128' register class*/ ; encoding: [0xfd,0x04,0x09,0x7d]
+
+0xff,0xfe,0x09,0x7d,0x0b,0xfe,0x00,0x00
+# GFX12-REAL16: v_cmpx_gt_f16_e32 0xfe0b, v127.h        ; encoding: [0xff,0xfe,0x09,0x7d,0x0b,0xfe,0x00,0x00]
+# GFX12-FAKE16: v_cmpx_gt_f16_e32 0xfe0b, v255/*Invalid register, operand has 'VGPR_32_Lo128' register class*/ ; encoding: [0xff,0xfe,0x09,0x7d,0x0b,0xfe,0x00,0x00]
 
 0x01,0x05,0x28,0x7d
 # GFX12: v_cmpx_gt_f32_e32 v1, v2                ; encoding: [0x01,0x05,0x28,0x7d]
@@ -1482,49 +1575,80 @@
 # GFX12: v_cmpx_gt_u64_e32 0xaf123456, v[254:255] ; encoding: [0xff,0xfc,0xb9,0x7d,0x56,0x34,0x12,0xaf]
 
 0x01,0x05,0x06,0x7d
-# GFX12: v_cmpx_le_f16_e32 v1, v2                ; encoding: [0x01,0x05,0x06,0x7d]
+# GFX12-REAL16: v_cmpx_le_f16_e32 v1.l, v2.l            ; encoding: [0x01,0x05,0x06,0x7d]
+# GFX12-FAKE16: v_cmpx_le_f16_e32 v1, v2                ; encoding: [0x01,0x05,0x06,0x7d]
 
 0x7f,0x05,0x06,0x7d
-# GFX12: v_cmpx_le_f16_e32 v127, v2              ; encoding: [0x7f,0x05,0x06,0x7d]
+# GFX12-REAL16: v_cmpx_le_f16_e32 v127.l, v2.l          ; encoding: [0x7f,0x05,0x06,0x7d]
+# GFX12-FAKE16: v_cmpx_le_f16_e32 v127, v2              ; encoding: [0x7f,0x05,0x06,0x7d]
 
 0x01,0x04,0x06,0x7d
-# GFX12: v_cmpx_le_f16_e32 s1, v2                ; encoding: [0x01,0x04,0x06,0x7d]
+# GFX12-REAL16: v_cmpx_le_f16_e32 s1, v2.l              ; encoding: [0x01,0x04,0x06,0x7d]
+# GFX12-FAKE16: v_cmpx_le_f16_e32 s1, v2                ; encoding: [0x01,0x04,0x06,0x7d]
 
 0x69,0x04,0x06,0x7d
-# GFX12: v_cmpx_le_f16_e32 s105, v2              ; encoding: [0x69,0x04,0x06,0x7d]
+# GFX12-REAL16: v_cmpx_le_f16_e32 s105, v2.l            ; encoding: [0x69,0x04,0x06,0x7d]
+# GFX12-FAKE16: v_cmpx_le_f16_e32 s105, v2              ; encoding: [0x69,0x04,0x06,0x7d]
 
 0x6a,0x04,0x06,0x7d
-# GFX12: v_cmpx_le_f16_e32 vcc_lo, v2            ; encoding: [0x6a,0x04,0x06,0x7d]
+# GFX12-REAL16: v_cmpx_le_f16_e32 vcc_lo, v2.l          ; encoding: [0x6a,0x04,0x06,0x7d]
+# GFX12-FAKE16: v_cmpx_le_f16_e32 vcc_lo, v2            ; encoding: [0x6a,0x04,0x06,0x7d]
 
 0x6b,0x04,0x06,0x7d
-# GFX12: v_cmpx_le_f16_e32 vcc_hi, v2            ; encoding: [0x6b,0x04,0x06,0x7d]
+# GFX12-REAL16: v_cmpx_le_f16_e32 vcc_hi, v2.l          ; encoding: [0x6b,0x04,0x06,0x7d]
+# GFX12-FAKE16: v_cmpx_le_f16_e32 vcc_hi, v2            ; encoding: [0x6b,0x04,0x06,0x7d]
 
 0x7b,0x04,0x06,0x7d
-# GFX12: v_cmpx_le_f16_e32 ttmp15, v2            ; encoding: [0x7b,0x04,0x06,0x7d]
+# GFX12-REAL16: v_cmpx_le_f16_e32 ttmp15, v2.l          ; encoding: [0x7b,0x04,0x06,0x7d]
+# GFX12-FAKE16: v_cmpx_le_f16_e32 ttmp15, v2            ; encoding: [0x7b,0x04,0x06,0x7d]
 
 0x7d,0x04,0x06,0x7d
-# GFX12: v_cmpx_le_f16_e32 m0, v2                ; encoding: [0x7d,0x04,0x06,0x7d]
+# GFX12-REAL16: v_cmpx_le_f16_e32 m0, v2.l              ; encoding: [0x7d,0x04,0x06,0x7d]
+# GFX12-FAKE16: v_cmpx_le_f16_e32 m0, v2                ; encoding: [0x7d,0x04,0x06,0x7d]
 
 0x7e,0x04,0x06,0x7d
-# GFX12: v_cmpx_le_f16_e32 exec_lo, v2           ; encoding: [0x7e,0x04,0x06,0x7d]
+# GFX12-REAL16: v_cmpx_le_f16_e32 exec_lo, v2.l         ; encoding: [0x7e,0x04,0x06,0x7d]
+# GFX12-FAKE16: v_cmpx_le_f16_e32 exec_lo, v2           ; encoding: [0x7e,0x04,0x06,0x7d]
 
 0x7f,0x04,0x06,0x7d
-# GFX12: v_cmpx_le_f16_e32 exec_hi, v2           ; encoding: [0x7f,0x04,0x06,0x7d]
+# GFX12-REAL16: v_cmpx_le_f16_e32 exec_hi, v2.l         ; encoding: [0x7f,0x04,0x06,0x7d]
+# GFX12-FAKE16: v_cmpx_le_f16_e32 exec_hi, v2           ; encoding: [0x7f,0x04,0x06,0x7d]
 
 0x7c,0x04,0x06,0x7d
-# GFX12: v_cmpx_le_f16_e32 null, v2              ; encoding: [0x7c,0x04,0x06,0x7d]
+# GFX12-REAL16: v_cmpx_le_f16_e32 null, v2.l            ; encoding: [0x7c,0x04,0x06,0x7d]
+# GFX12-FAKE16: v_cmpx_le_f16_e32 null, v2              ; encoding: [0x7c,0x04,0x06,0x7d]
 
 0xc1,0x04,0x06,0x7d
-# GFX12: v_cmpx_le_f16_e32 -1, v2                ; encoding: [0xc1,0x04,0x06,0x7d]
+# GFX12-REAL16: v_cmpx_le_f16_e32 -1, v2.l              ; encoding: [0xc1,0x04,0x06,0x7d]
+# GFX12-FAKE16: v_cmpx_le_f16_e32 -1, v2                ; encoding: [0xc1,0x04,0x06,0x7d]
 
 0xf0,0x04,0x06,0x7d
-# GFX12: v_cmpx_le_f16_e32 0.5, v2               ; encoding: [0xf0,0x04,0x06,0x7d]
+# GFX12-REAL16: v_cmpx_le_f16_e32 0.5, v2.l             ; encoding: [0xf0,0x04,0x06,0x7d]
+# GFX12-FAKE16: v_cmpx_le_f16_e32 0.5, v2               ; encoding: [0xf0,0x04,0x06,0x7d]
 
 0xfd,0x04,0x06,0x7d
-# GFX12: v_cmpx_le_f16_e32 src_scc, v2           ; encoding: [0xfd,0x04,0x06,0x7d]
+# GFX12-REAL16: v_cmpx_le_f16_e32 src_scc, v2.l         ; encoding: [0xfd,0x04,0x06,0x7d]
+# GFX12-FAKE16: v_cmpx_le_f16_e32 src_scc, v2           ; encoding: [0xfd,0x04,0x06,0x7d]
 
 0xff,0xfe,0x06,0x7d,0x0b,0xfe,0x00,0x00
-# GFX12: v_cmpx_le_f16_e32 0xfe0b, v127          ; encoding: [0xff,0xfe,0x06,0x7d,0x0b,0xfe,0x00,0x00]
+# GFX12-REAL16: v_cmpx_le_f16_e32 0xfe0b, v127.l        ; encoding: [0xff,0xfe,0x06,0x7d,0x0b,0xfe,0x00,0x00]
+# GFX12-FAKE16: v_cmpx_le_f16_e32 0xfe0b, v127          ; encoding: [0xff,0xfe,0x06,0x7d,0x0b,0xfe,0x00,0x00]
+
+0x81,0x05,0x06,0x7d
+# GFX12-REAL16: v_cmpx_le_f16_e32 v1.h, v2.l            ; encoding: [0x81,0x05,0x06,0x7d]
+# GFX12-FAKE16: v_cmpx_le_f16_e32 v129/*Invalid register, operand has 'VS_32_Lo128' register class*/, v2 ; encoding: [0x81,0x05,0x06,0x7d]
+
+0xff,0x05,0x06,0x7d
+# GFX12-REAL16: v_cmpx_le_f16_e32 v127.h, v2.l          ; encoding: [0xff,0x05,0x06,0x7d]
+# GFX12-FAKE16: v_cmpx_le_f16_e32 v255/*Invalid register, operand has 'VS_32_Lo128' register class*/, v2 ; encoding: [0xff,0x05,0x06,0x7d]
+
+0xfd,0x04,0x07,0x7d
+# GFX12-REAL16: v_cmpx_le_f16_e32 src_scc, v2.h         ; encoding: [0xfd,0x04,0x07,0x7d]
+# GFX12-FAKE16: v_cmpx_le_f16_e32 src_scc, v130/*Invalid register, operand has 'VGPR_32_Lo128' register class*/ ; encoding: [0xfd,0x04,0x07,0x7d]
+
+0xff,0xfe,0x07,0x7d,0x0b,0xfe,0x00,0x00
+# GFX12-REAL16: v_cmpx_le_f16_e32 0xfe0b, v127.h        ; encoding: [0xff,0xfe,0x07,0x7d,0x0b,0xfe,0x00,0x00]
+# GFX12-FAKE16: v_cmpx_le_f16_e32 0xfe0b, v255/*Invalid register, operand has 'VGPR_32_Lo128' register class*/ ; encoding: [0xff,0xfe,0x07,0x7d,0x0b,0xfe,0x00,0x00]
 
 0x01,0x05,0x26,0x7d
 # GFX12: v_cmpx_le_f32_e32 v1, v2                ; encoding: [0x01,0x05,0x26,0x7d]
@@ -1922,49 +2046,80 @@
 # GFX12: v_cmpx_le_u64_e32 0xaf123456, v[254:255] ; encoding: [0xff,0xfc,0xb7,0x7d,0x56,0x34,0x12,0xaf]
 
 0x01,0x05,0x0a,0x7d
-# GFX12: v_cmpx_lg_f16_e32 v1, v2                ; encoding: [0x01,0x05,0x0a,0x7d]
+# GFX12-REAL16: v_cmpx_lg_f16_e32 v1.l, v2.l            ; encoding: [0x01,0x05,0x0a,0x7d]
+# GFX12-FAKE16: v_cmpx_lg_f16_e32 v1, v2                ; encoding: [0x01,0x05,0x0a,0x7d]
 
 0x7f,0x05,0x0a,0x7d
-# GFX12: v_cmpx_lg_f16_e32 v127, v2              ; encoding: [0x7f,0x05,0x0a,0x7d]
+# GFX12-REAL16: v_cmpx_lg_f16_e32 v127.l, v2.l          ; encoding: [0x7f,0x05,0x0a,0x7d]
+# GFX12-FAKE16: v_cmpx_lg_f16_e32 v127, v2              ; encoding: [0x7f,0x05,0x0a,0x7d]
 
 0x01,0x04,0x0a,0x7d
-# GFX12: v_cmpx_lg_f16_e32 s1, v2                ; encoding: [0x01,0x04,0x0a,0x7d]
+# GFX12-REAL16: v_cmpx_lg_f16_e32 s1, v2.l              ; encoding: [0x01,0x04,0x0a,0x7d]
+# GFX12-FAKE16: v_cmpx_lg_f16_e32 s1, v2                ; encoding: [0x01,0x04,0x0a,0x7d]
 
 0x69,0x04,0x0a,0x7d
-# GFX12: v_cmpx_lg_f16_e32 s105, v2              ; encoding: [0x69,0x04,0x0a,0x7d]
+# GFX12-REAL16: v_cmpx_lg_f16_e32 s105, v2.l            ; encoding: [0x69,0x04,0x0a,0x7d]
+# GFX12-FAKE16: v_cmpx_lg_f16_e32 s105, v2              ; encoding: [0x69,0x04,0x0a,0x7d]
 
 0x6a,0x04,0x0a,0x7d
-# GFX12: v_cmpx_lg_f16_e32 vcc_lo, v2            ; encoding: [0x6a,0x04,0x0a,0x7d]
+# GFX12-REAL16: v_cmpx_lg_f16_e32 vcc_lo, v2.l          ; encoding: [0x6a,0x04,0x0a,0x7d]
+# GFX12-FAKE16: v_cmpx_lg_f16_e32 vcc_lo, v2            ; encoding: [0x6a,0x04,0x0a,0x7d]
 
 0x6b,0x04,0x0a,0x7d
-# GFX12: v_cmpx_lg_f16_e32 vcc_hi, v2            ; encoding: [0x6b,0x04,0x0a,0x7d]
+# GFX12-REAL16: v_cmpx_lg_f16_e32 vcc_hi, v2.l          ; encoding: [0x6b,0x04,0x0a,0x7d]
+# GFX12-FAKE16: v_cmpx_lg_f16_e32 vcc_hi, v2            ; encoding: [0x6b,0x04,0x0a,0x7d]
 
 0x7b,0x04,0x0a,0x7d
-# GFX12: v_cmpx_lg_f16_e32 ttmp15, v2            ; encoding: [0x7b,0x04,0x0a,0x7d]
+# GFX12-REAL16: v_cmpx_lg_f16_e32 ttmp15, v2.l          ; encoding: [0x7b,0x04,0x0a,0x7d]
+# GFX12-FAKE16: v_cmpx_lg_f16_e32 ttmp15, v2            ; encoding: [0x7b,0x04,0x0a,0x7d]
 
 0x7d,0x04,0x0a,0x7d
-# GFX12: v_cmpx_lg_f16_e32 m0, v2                ; encoding: [0x7d,0x04,0x0a,0x7d]
+# GFX12-REAL16: v_cmpx_lg_f16_e32 m0, v2.l              ; encoding: [0x7d,0x04,0x0a,0x7d]
+# GFX12-FAKE16: v_cmpx_lg_f16_e32 m0, v2                ; encoding: [0x7d,0x04,0x0a,0x7d]
 
 0x7e,0x04,0x0a,0x7d
-# GFX12: v_cmpx_lg_f16_e32 exec_lo, v2           ; encoding: [0x7e,0x04,0x0a,0x7d]
+# GFX12-REAL16: v_cmpx_lg_f16_e32 exec_lo, v2.l         ; encoding: [0x7e,0x04,0x0a,0x7d]
+# GFX12-FAKE16: v_cmpx_lg_f16_e32 exec_lo, v2           ; encoding: [0x7e,0x04,0x0a,0x7d]
 
 0x7f,0x04,0x0a,0x7d
-# GFX12: v_cmpx_lg_f16_e32 exec_hi, v2           ; encoding: [0x7f,0x04,0x0a,0x7d]
+# GFX12-REAL16: v_cmpx_lg_f16_e32 exec_hi, v2.l         ; encoding: [0x7f,0x04,0x0a,0x7d]
+# GFX12-FAKE16: v_cmpx_lg_f16_e32 exec_hi, v2           ; encoding: [0x7f,0x04,0x0a,0x7d]
 
 0x7c,0x04,0x0a,0x7d
-# GFX12: v_cmpx_lg_f16_e32 null, v2              ; encoding: [0x7c,0x04,0x0a,0x7d]
+# GFX12-REAL16: v_cmpx_lg_f16_e32 null, v2.l            ; encoding: [0x7c,0x04,0x0a,0x7d]
+# GFX12-FAKE16: v_cmpx_lg_f16_e32 null, v2              ; encoding: [0x7c,0x04,0x0a,0x7d]
 
 0xc1,0x04,0x0a,0x7d
-# GFX12: v_cmpx_lg_f16_e32 -1, v2                ; encoding: [0xc1,0x04,0x0a,0x7d]
+# GFX12-REAL16: v_cmpx_lg_f16_e32 -1, v2.l              ; encoding: [0xc1,0x04,0x0a,0x7d]
+# GFX12-FAKE16: v_cmpx_lg_f16_e32 -1, v2                ; encoding: [0xc1,0x04,0x0a,0x7d]
 
 0xf0,0x04,0x0a,0x7d
-# GFX12: v_cmpx_lg_f16_e32 0.5, v2               ; encoding: [0xf0,0x04,0x0a,0x7d]
+# GFX12-REAL16: v_cmpx_lg_f16_e32 0.5, v2.l             ; encoding: [0xf0,0x04,0x0a,0x7d]
+# GFX12-FAKE16: v_cmpx_lg_f16_e32 0.5, v2               ; encoding: [0xf0,0x04,0x0a,0x7d]
 
 0xfd,0x04,0x0a,0x7d
-# GFX12: v_cmpx_lg_f16_e32 src_scc, v2           ; encoding: [0xfd,0x04,0x0a,0x7d]
+# GFX12-REAL16: v_cmpx_lg_f16_e32 src_scc, v2.l         ; encoding: [0xfd,0x04,0x0a,0x7d]
+# GFX12-FAKE16: v_cmpx_lg_f16_e32 src_scc, v2           ; encoding: [0xfd,0x04,0x0a,0x7d]
 
 0xff,0xfe,0x0a,0x7d,0x0b,0xfe,0x00,0x00
-# GFX12: v_cmpx_lg_f16_e32 0xfe0b, v127          ; encoding: [0xff,0xfe,0x0a,0x7d,0x0b,0xfe,0x00,0x00]
+# GFX12-REAL16: v_cmpx_lg_f16_e32 0xfe0b, v127.l        ; encoding: [0xff,0xfe,0x0a,0x7d,0x0b,0xfe,0x00,0x00]
+# GFX12-FAKE16: v_cmpx_lg_f16_e32 0xfe0b, v127          ; encoding: [0xff,0xfe,0x0a,0x7d,0x0b,0xfe,0x00,0x00]
+
+0x81,0x05,0x0a,0x7d
+# GFX12-REAL16: v_cmpx_lg_f16_e32 v1.h, v2.l            ; encoding: [0x81,0x05,0x0a,0x7d]
+# GFX12-FAKE16: v_cmpx_lg_f16_e32 v129/*Invalid register, operand has 'VS_32_Lo128' register class*/, v2 ; encoding: [0x81,0x05,0x0a,0x7d]
+
+0xff,0x05,0x0a,0x7d
+# GFX12-REAL16: v_cmpx_lg_f16_e32 v127.h, v2.l          ; encoding: [0xff,0x05,0x0a,0x7d]
+# GFX12-FAKE16: v_cmpx_lg_f16_e32 v255/*Invalid register, operand has 'VS_32_Lo128' register class*/, v2 ; encoding: [0xff,0x05,0x0a,0x7d]
+
+0xfd,0x04,0x0b,0x7d
+# GFX12-REAL16: v_cmpx_lg_f16_e32 src_scc, v2.h         ; encoding: [0xfd,0x04,0x0b,0x7d]
+# GFX12-FAKE16: v_cmpx_lg_f16_e32 src_scc, v130/*Invalid register, operand has 'VGPR_32_Lo128' register class*/ ; encoding: [0xfd,0x04,0x0b,0x7d]
+
+0xff,0xfe,0x0b,0x7d,0x0b,0xfe,0x00,0x00
+# GFX12-REAL16: v_cmpx_lg_f16_e32 0xfe0b, v127.h        ; encoding: [0xff,0xfe,0x0b,0x7d,0x0b,0xfe,0x00,0x00]
+# GFX12-FAKE16: v_cmpx_lg_f16_e32 0xfe0b, v255/*Invalid register, operand has 'VGPR_32_Lo128' register class*/ ; encoding: [0xff,0xfe,0x0b,0x7d,0x0b,0xfe,0x00,0x00]
 
 0x01,0x05,0x2a,0x7d
 # GFX12: v_cmpx_lg_f32_e32 v1, v2                ; encoding: [0x01,0x05,0x2a,0x7d]
@@ -2833,49 +2988,80 @@
 # GFX12: v_cmpx_ne_u64_e32 0xaf123456, v[254:255] ; encoding: [0xff,0xfc,0xbb,0x7d,0x56,0x34,0x12,0xaf]
 
 0x01,0x05,0x1a,0x7d
-# GFX12: v_cmpx_neq_f16_e32 v1, v2               ; encoding: [0x01,0x05,0x1a,0x7d]
+# GFX12-REAL16: v_cmpx_neq_f16_e32 v1.l, v2.l           ; encoding: [0x01,0x05,0x1a,0x7d]
+# GFX12-FAKE16: v_cmpx_neq_f16_e32 v1, v2               ; encoding: [0x01,0x05,0x1a,0x7d]
 
 0x7f,0x05,0x1a,0x7d
-# GFX12: v_cmpx_neq_f16_e32 v127, v2             ; encoding: [0x7f,0x05,0x1a,0x7d]
+# GFX12-REAL16: v_cmpx_neq_f16_e32 v127.l, v2.l         ; encoding: [0x7f,0x05,0x1a,0x7d]
+# GFX12-FAKE16: v_cmpx_neq_f16_e32 v127, v2             ; encoding: [0x7f,0x05,0x1a,0x7d]
 
 0x01,0x04,0x1a,0x7d
-# GFX12: v_cmpx_neq_f16_e32 s1, v2               ; encoding: [0x01,0x04,0x1a,0x7d]
+# GFX12-REAL16: v_cmpx_neq_f16_e32 s1, v2.l             ; encoding: [0x01,0x04,0x1a,0x7d]
+# GFX12-FAKE16: v_cmpx_neq_f16_e32 s1, v2               ; encoding: [0x01,0x04,0x1a,0x7d]
 
 0x69,0x04,0x1a,0x7d
-# GFX12: v_cmpx_neq_f16_e32 s105, v2             ; encoding: [0x69,0x04,0x1a,0x7d]
+# GFX12-REAL16: v_cmpx_neq_f16_e32 s105, v2.l           ; encoding: [0x69,0x04,0x1a,0x7d]
+# GFX12-FAKE16: v_cmpx_neq_f16_e32 s105, v2             ; encoding: [0x69,0x04,0x1a,0x7d]
 
 0x6a,0x04,0x1a,0x7d
-# GFX12: v_cmpx_neq_f16_e32 vcc_lo, v2           ; encoding: [0x6a,0x04,0x1a,0x7d]
+# GFX12-REAL16: v_cmpx_neq_f16_e32 vcc_lo, v2.l         ; encoding: [0x6a,0x04,0x1a,0x7d]
+# GFX12-FAKE16: v_cmpx_neq_f16_e32 vcc_lo, v2           ; encoding: [0x6a,0x04,0x1a,0x7d]
 
 0x6b,0x04,0x1a,0x7d
-# GFX12: v_cmpx_neq_f16_e32 vcc_hi, v2           ; encoding: [0x6b,0x04,0x1a,0x7d]
+# GFX12-REAL16: v_cmpx_neq_f16_e32 vcc_hi, v2.l         ; encoding: [0x6b,0x04,0x1a,0x7d]
+# GFX12-FAKE16: v_cmpx_neq_f16_e32 vcc_hi, v2           ; encoding: [0x6b,0x04,0x1a,0x7d]
 
 0x7b,0x04,0x1a,0x7d
-# GFX12: v_cmpx_neq_f16_e32 ttmp15, v2           ; encoding: [0x7b,0x04,0x1a,0x7d]
+# GFX12-REAL16: v_cmpx_neq_f16_e32 ttmp15, v2.l         ; encoding: [0x7b,0x04,0x1a,0x7d]
+# GFX12-FAKE16: v_cmpx_neq_f16_e32 ttmp15, v2           ; encoding: [0x7b,0x04,0x1a,0x7d]
 
 0x7d,0x04,0x1a,0x7d
-# GFX12: v_cmpx_neq_f16_e32 m0, v2               ; encoding: [0x7d,0x04,0x1a,0x7d]
+# GFX12-REAL16: v_cmpx_neq_f16_e32 m0, v2.l             ; encoding: [0x7d,0x04,0x1a,0x7d]
+# GFX12-FAKE16: v_cmpx_neq_f16_e32 m0, v2               ; encoding: [0x7d,0x04,0x1a,0x7d]
 
 0x7e,0x04,0x1a,0x7d
-# GFX12: v_cmpx_neq_f16_e32 exec_lo, v2          ; encoding: [0x7e,0x04,0x1a,0x7d]
+# GFX12-REAL16: v_cmpx_neq_f16_e32 exec_lo, v2.l        ; encoding: [0x7e,0x04,0x1a,0x7d]
+# GFX12-FAKE16: v_cmpx_neq_f16_e32 exec_lo, v2          ; encoding: [0x7e,0x04,0x1a,0x7d]
 
 0x7f,0x04,0x1a,0x7d
-# GFX12: v_cmpx_neq_f16_e32 exec_hi, v2          ; encoding: [0x7f,0x04,0x1a,0x7d]
+# GFX12-REAL16: v_cmpx_neq_f16_e32 exec_hi, v2.l        ; encoding: [0x7f,0x04,0x1a,0x7d]
+# GFX12-FAKE16: v_cmpx_neq_f16_e32 exec_hi, v2          ; encoding: [0x7f,0x04,0x1a,0x7d]
 
 0x7c,0x04,0x1a,0x7d
-# GFX12: v_cmpx_neq_f16_e32 null, v2             ; encoding: [0x7c,0x04,0x1a,0x7d]
+# GFX12-REAL16: v_cmpx_neq_f16_e32 null, v2.l           ; encoding: [0x7c,0x04,0x1a,0x7d]
+# GFX12-FAKE16: v_cmpx_neq_f16_e32 null, v2             ; encoding: [0x7c,0x04,0x1a,0x7d]
 
 0xc1,0x04,0x1a,0x7d
-# GFX12: v_cmpx_neq_f16_e32 -1, v2               ; encoding: [0xc1,0x04,0x1a,0x7d]
+# GFX12-REAL16: v_cmpx_neq_f16_e32 -1, v2.l             ; encoding: [0xc1,0x04,0x1a,0x7d]
+# GFX12-FAKE16: v_cmpx_neq_f16_e32 -1, v2               ; encoding: [0xc1,0x04,0x1a,0x7d]
 
 0xf0,0x04,0x1a,0x7d
-# GFX12: v_cmpx_neq_f16_e32 0.5, v2              ; encoding: [0xf0,0x04,0x1a,0x7d]
+# GFX12-REAL16: v_cmpx_neq_f16_e32 0.5, v2.l            ; encoding: [0xf0,0x04,0x1a,0x7d]
+# GFX12-FAKE16: v_cmpx_neq_f16_e32 0.5, v2              ; encoding: [0xf0,0x04,0x1a,0x7d]
 
 0xfd,0x04,0x1a,0x7d
-# GFX12: v_cmpx_neq_f16_e32 src_scc, v2          ; encoding: [0xfd,0x04,0x1a,0x7d]
+# GFX12-REAL16: v_cmpx_neq_f16_e32 src_scc, v2.l        ; encoding: [0xfd,0x04,0x1a,0x7d]
+# GFX12-FAKE16: v_cmpx_neq_f16_e32 src_scc, v2          ; encoding: [0xfd,0x04,0x1a,0x7d]
 
 0xff,0xfe,0x1a,0x7d,0x0b,0xfe,0x00,0x00
-# GFX12: v_cmpx_neq_f16_e32 0xfe0b, v127         ; encoding: [0xff,0xfe,0x1a,0x7d,0x0b,0xfe,0x00,0x00]
+# GFX12-REAL16: v_cmpx_neq_f16_e32 0xfe0b, v127.l       ; encoding: [0xff,0xfe,0x1a,0x7d,0x0b,0xfe,0x00,0x00]
+# GFX12-FAKE16: v_cmpx_neq_f16_e32 0xfe0b, v127         ; encoding: [0xff,0xfe,0x1a,0x7d,0x0b,0xfe,0x00,0x00]
+
+0x81,0x05,0x1a,0x7d
+# GFX12-REAL16: v_cmpx_neq_f16_e32 v1.h, v2.l           ; encoding: [0x81,0x05,0x1a,0x7d]
+# GFX12-FAKE16: v_cmpx_neq_f16_e32 v129/*Invalid register, operand has 'VS_32_Lo128' register class*/, v2 ; encoding: [0x81,0x05,0x1a,0x7d]
+
+0xff,0x05,0x1a,0x7d
+# GFX12-REAL16: v_cmpx_neq_f16_e32 v127.h, v2.l         ; encoding: [0xff,0x05,0x1a,0x7d]
+# GFX12-FAKE16: v_cmpx_neq_f16_e32 v255/*Invalid register, operand has 'VS_32_Lo128' register class*/, v2 ; encoding: [0xff,0x05,0x1a,0x7d]
+
+0xfd,0x04,0x1b,0x7d
+# GFX12-REAL16: v_cmpx_neq_f16_e32 src_scc, v2.h        ; encoding: [0xfd,0x04,0x1b,0x7d]
+# GFX12-FAKE16: v_cmpx_neq_f16_e32 src_scc, v130/*Invalid register, operand has 'VGPR_32_Lo128' register class*/ ; encoding: [0xfd,0x04,0x1b,0x7d]
+
+0xff,0xfe,0x1b,0x7d,0x0b,0xfe,0x00,0x00
+# GFX12-REAL16: v_cmpx_neq_f16_e32 0xfe0b, v127.h       ; encoding: [0xff,0xfe,0x1b,0x7d,0x0b,0xfe,0x00,0x00]
+# GFX12-FAKE16: v_cmpx_neq_f16_e32 0xfe0b, v255/*Invalid register, operand has 'VGPR_32_Lo128' register class*/ ; encoding: [0xff,0xfe,0x1b,0x7d,0x0b,0xfe,0x00,0x00]
 
 0x01,0x05,0x3a,0x7d
 # GFX12: v_cmpx_neq_f32_e32 v1, v2               ; encoding: [0x01,0x05,0x3a,0x7d]
@@ -2959,49 +3145,80 @@
 # GFX12: v_cmpx_neq_f64_e32 0xaf123456, v[254:255] ; encoding: [0xff,0xfc,0x5b,0x7d,0x56,0x34,0x12,0xaf]
 
 0x01,0x05,0x12,0x7d
-# GFX12: v_cmpx_nge_f16_e32 v1, v2               ; encoding: [0x01,0x05,0x12,0x7d]
+# GFX12-REAL16: v_cmpx_nge_f16_e32 v1.l, v2.l           ; encoding: [0x01,0x05,0x12,0x7d]
+# GFX12-FAKE16: v_cmpx_nge_f16_e32 v1, v2               ; encoding: [0x01,0x05,0x12,0x7d]
 
 0x7f,0x05,0x12,0x7d
-# GFX12: v_cmpx_nge_f16_e32 v127, v2             ; encoding: [0x7f,0x05,0x12,0x7d]
+# GFX12-REAL16: v_cmpx_nge_f16_e32 v127.l, v2.l         ; encoding: [0x7f,0x05,0x12,0x7d]
+# GFX12-FAKE16: v_cmpx_nge_f16_e32 v127, v2             ; encoding: [0x7f,0x05,0x12,0x7d]
 
 0x01,0x04,0x12,0x7d
-# GFX12: v_cmpx_nge_f16_e32 s1, v2               ; encoding: [0x01,0x04,0x12,0x7d]
+# GFX12-REAL16: v_cmpx_nge_f16_e32 s1, v2.l             ; encoding: [0x01,0x04,0x12,0x7d]
+# GFX12-FAKE16: v_cmpx_nge_f16_e32 s1, v2               ; encoding: [0x01,0x04,0x12,0x7d]
 
 0x69,0x04,0x12,0x7d
-# GFX12: v_cmpx_nge_f16_e32 s105, v2             ; encoding: [0x69,0x04,0x12,0x7d]
+# GFX12-REAL16: v_cmpx_nge_f16_e32 s105, v2.l           ; encoding: [0x69,0x04,0x12,0x7d]
+# GFX12-FAKE16: v_cmpx_nge_f16_e32 s105, v2             ; encoding: [0x69,0x04,0x12,0x7d]
 
 0x6a,0x04,0x12,0x7d
-# GFX12: v_cmpx_nge_f16_e32 vcc_lo, v2           ; encoding: [0x6a,0x04,0x12,0x7d]
+# GFX12-REAL16: v_cmpx_nge_f16_e32 vcc_lo, v2.l         ; encoding: [0x6a,0x04,0x12,0x7d]
+# GFX12-FAKE16: v_cmpx_nge_f16_e32 vcc_lo, v2           ; encoding: [0x6a,0x04,0x12,0x7d]
 
 0x6b,0x04,0x12,0x7d
-# GFX12: v_cmpx_nge_f16_e32 vcc_hi, v2           ; encoding: [0x6b,0x04,0x12,0x7d]
+# GFX12-REAL16: v_cmpx_nge_f16_e32 vcc_hi, v2.l         ; encoding: [0x6b,0x04,0x12,0x7d]
+# GFX12-FAKE16: v_cmpx_nge_f16_e32 vcc_hi, v2           ; encoding: [0x6b,0x04,0x12,0x7d]
 
 0x7b,0x04,0x12,0x7d
-# GFX12: v_cmpx_nge_f16_e32 ttmp15, v2           ; encoding: [0x7b,0x04,0x12,0x7d]
+# GFX12-REAL16: v_cmpx_nge_f16_e32 ttmp15, v2.l         ; encoding: [0x7b,0x04,0x12,0x7d]
+# GFX12-FAKE16: v_cmpx_nge_f16_e32 ttmp15, v2           ; encoding: [0x7b,0x04,0x12,0x7d]
 
 0x7d,0x04,0x12,0x7d
-# GFX12: v_cmpx_nge_f16_e32 m0, v2               ; encoding: [0x7d,0x04,0x12,0x7d]
+# GFX12-REAL16: v_cmpx_nge_f16_e32 m0, v2.l             ; encoding: [0x7d,0x04,0x12,0x7d]
+# GFX12-FAKE16: v_cmpx_nge_f16_e32 m0, v2               ; encoding: [0x7d,0x04,0x12,0x7d]
 
 0x7e,0x04,0x12,0x7d
-# GFX12: v_cmpx_nge_f16_e32 exec_lo, v2          ; encoding: [0x7e,0x04,0x12,0x7d]
+# GFX12-REAL16: v_cmpx_nge_f16_e32 exec_lo, v2.l        ; encoding: [0x7e,0x04,0x12,0x7d]
+# GFX12-FAKE16: v_cmpx_nge_f16_e32 exec_lo, v2          ; encoding: [0x7e,0x04,0x12,0x7d]
 
 0x7f,0x04,0x12,0x7d
-# GFX12: v_cmpx_nge_f16_e32 exec_hi, v2          ; encoding: [0x7f,0x04,0x12,0x7d]
+# GFX12-REAL16: v_cmpx_nge_f16_e32 exec_hi, v2.l        ; encoding: [0x7f,0x04,0x12,0x7d]
+# GFX12-FAKE16: v_cmpx_nge_f16_e32 exec_hi, v2          ; encoding: [0x7f,0x04,0x12,0x7d]
 
 0x7c,0x04,0x12,0x7d
-# GFX12: v_cmpx_nge_f16_e32 null, v2             ; encoding: [0x7c,0x04,0x12,0x7d]
+# GFX12-REAL16: v_cmpx_nge_f16_e32 null, v2.l           ; encoding: [0x7c,0x04,0x12,0x7d]
+# GFX12-FAKE16: v_cmpx_nge_f16_e32 null, v2             ; encoding: [0x7c,0x04,0x12,0x7d]
 
 0xc1,0x04,0x12,0x7d
-# GFX12: v_cmpx_nge_f16_e32 -1, v2               ; encoding: [0xc1,0x04,0x12,0x7d]
+# GFX12-REAL16: v_cmpx_nge_f16_e32 -1, v2.l             ; encoding: [0xc1,0x04,0x12,0x7d]
+# GFX12-FAKE16: v_cmpx_nge_f16_e32 -1, v2               ; encoding: [0xc1,0x04,0x12,0x7d]
 
 0xf0,0x04,0x12,0x7d
-# GFX12: v_cmpx_nge_f16_e32 0.5, v2              ; encoding: [0xf0,0x04,0x12,0x7d]
+# GFX12-REAL16: v_cmpx_nge_f16_e32 0.5, v2.l            ; encoding: [0xf0,0x04,0x12,0x7d]
+# GFX12-FAKE16: v_cmpx_nge_f16_e32 0.5, v2              ; encoding: [0xf0,0x04,0x12,0x7d]
 
 0xfd,0x04,0x12,0x7d
-# GFX12: v_cmpx_nge_f16_e32 src_scc, v2          ; encoding: [0xfd,0x04,0x12,0x7d]
+# GFX12-REAL16: v_cmpx_nge_f16_e32 src_scc, v2.l        ; encoding: [0xfd,0x04,0x12,0x7d]
+# GFX12-FAKE16: v_cmpx_nge_f16_e32 src_scc, v2          ; encoding: [0xfd,0x04,0x12,0x7d]
 
 0xff,0xfe,0x12,0x7d,0x0b,0xfe,0x00,0x00
-# GFX12: v_cmpx_nge_f16_e32 0xfe0b, v127         ; encoding: [0xff,0xfe,0x12,0x7d,0x0b,0xfe,0x00,0x00]
+# GFX12-REAL16: v_cmpx_nge_f16_e32 0xfe0b, v127.l       ; encoding: [0xff,0xfe,0x12,0x7d,0x0b,0xfe,0x00,0x00]
+# GFX12-FAKE16: v_cmpx_nge_f16_e32 0xfe0b, v127         ; encoding: [0xff,0xfe,0x12,0x7d,0x0b,0xfe,0x00,0x00]
+
+0x81,0x05,0x12,0x7d
+# GFX12-REAL16: v_cmpx_nge_f16_e32 v1.h, v2.l           ; encoding: [0x81,0x05,0x12,0x7d]
+# GFX12-FAKE16: v_cmpx_nge_f16_e32 v129/*Invalid register, operand has 'VS_32_Lo128' register class*/, v2 ; encoding: [0x81,0x05,0x12,0x7d]
+
+0xff,0x05,0x12,0x7d
+# GFX12-REAL16: v_cmpx_nge_f16_e32 v127.h, v2.l         ; encoding: [0xff,0x05,0x12,0x7d]
+# GFX12-FAKE16: v_cmpx_nge_f16_e32 v255/*Invalid register, operand has 'VS_32_Lo128' register class*/, v2 ; encoding: [0xff,0x05,0x12,0x7d]
+
+0xfd,0x04,0x13,0x7d
+# GFX12-REAL16: v_cmpx_nge_f16_e32 src_scc, v2.h        ; encoding: [0xfd,0x04,0x13,0x7d]
+# GFX12-FAKE16: v_cmpx_nge_f16_e32 src_scc, v130/*Invalid register, operand has 'VGPR_32_Lo128' register class*/ ; encoding: [0xfd,0x04,0x13,0x7d]
+
+0xff,0xfe,0x13,0x7d,0x0b,0xfe,0x00,0x00
+# GFX12-REAL16: v_cmpx_nge_f16_e32 0xfe0b, v127.h       ; encoding: [0xff,0xfe,0x13,0x7d,0x0b,0xfe,0x00,0x00]
+# GFX12-FAKE16: v_cmpx_nge_f16_e32 0xfe0b, v255/*Invalid register, operand has 'VGPR_32_Lo128' register class*/ ; encoding: [0xff,0xfe,0x13,0x7d,0x0b,0xfe,0x00,0x00]
 
 0x01,0x05,0x32,0x7d
 # GFX12: v_cmpx_nge_f32_e32 v1, v2               ; encoding: [0x01,0x05,0x32,0x7d]
@@ -3085,49 +3302,80 @@
 # GFX12: v_cmpx_nge_f64_e32 0xaf123456, v[254:255] ; encoding: [0xff,0xfc,0x53,0x7d,0x56,0x34,0x12,0xaf]
 
 0x01,0x05,0x16,0x7d
-# GFX12: v_cmpx_ngt_f16_e32 v1, v2               ; encoding: [0x01,0x05,0x16,0x7d]
+# GFX12-REAL16: v_cmpx_ngt_f16_e32 v1.l, v2.l           ; encoding: [0x01,0x05,0x16,0x7d]
+# GFX12-FAKE16: v_cmpx_ngt_f16_e32 v1, v2               ; encoding: [0x01,0x05,0x16,0x7d]
 
 0x7f,0x05,0x16,0x7d
-# GFX12: v_cmpx_ngt_f16_e32 v127, v2             ; encoding: [0x7f,0x05,0x16,0x7d]
+# GFX12-REAL16: v_cmpx_ngt_f16_e32 v127.l, v2.l         ; encoding: [0x7f,0x05,0x16,0x7d]
+# GFX12-FAKE16: v_cmpx_ngt_f16_e32 v127, v2             ; encoding: [0x7f,0x05,0x16,0x7d]
 
 0x01,0x04,0x16,0x7d
-# GFX12: v_cmpx_ngt_f16_e32 s1, v2               ; encoding: [0x01,0x04,0x16,0x7d]
+# GFX12-REAL16: v_cmpx_ngt_f16_e32 s1, v2.l             ; encoding: [0x01,0x04,0x16,0x7d]
+# GFX12-FAKE16: v_cmpx_ngt_f16_e32 s1, v2               ; encoding: [0x01,0x04,0x16,0x7d]
 
 0x69,0x04,0x16,0x7d
-# GFX12: v_cmpx_ngt_f16_e32 s105, v2             ; encoding: [0x69,0x04,0x16,0x7d]
+# GFX12-REAL16: v_cmpx_ngt_f16_e32 s105, v2.l           ; encoding: [0x69,0x04,0x16,0x7d]
+# GFX12-FAKE16: v_cmpx_ngt_f16_e32 s105, v2             ; encoding: [0x69,0x04,0x16,0x7d]
 
 0x6a,0x04,0x16,0x7d
-# GFX12: v_cmpx_ngt_f16_e32 vcc_lo, v2           ; encoding: [0x6a,0x04,0x16,0x7d]
+# GFX12-REAL16: v_cmpx_ngt_f16_e32 vcc_lo, v2.l         ; encoding: [0x6a,0x04,0x16,0x7d]
+# GFX12-FAKE16: v_cmpx_ngt_f16_e32 vcc_lo, v2           ; encoding: [0x6a,0x04,0x16,0x7d]
 
 0x6b,0x04,0x16,0x7d
-# GFX12: v_cmpx_ngt_f16_e32 vcc_hi, v2           ; encoding: [0x6b,0x04,0x16,0x7d]
+# GFX12-REAL16: v_cmpx_ngt_f16_e32 vcc_hi, v2.l         ; encoding: [0x6b,0x04,0x16,0x7d]
+# GFX12-FAKE16: v_cmpx_ngt_f16_e32 vcc_hi, v2           ; encoding: [0x6b,0x04,0x16,0x7d]
 
 0x7b,0x04,0x16,0x7d
-# GFX12: v_cmpx_ngt_f16_e32 ttmp15, v2           ; encoding: [0x7b,0x04,0x16,0x7d]
+# GFX12-REAL16: v_cmpx_ngt_f16_e32 ttmp15, v2.l         ; encoding: [0x7b,0x04,0x16,0x7d]
+# GFX12-FAKE16: v_cmpx_ngt_f16_e32 ttmp15, v2           ; encoding: [0x7b,0x04,0x16,0x7d]
 
 0x7d,0x04,0x16,0x7d
-# GFX12: v_cmpx_ngt_f16_e32 m0, v2               ; encoding: [0x7d,0x04,0x16,0x7d]
+# GFX12-REAL16: v_cmpx_ngt_f16_e32 m0, v2.l             ; encoding: [0x7d,0x04,0x16,0x7d]
+# GFX12-FAKE16: v_cmpx_ngt_f16_e32 m0, v2               ; encoding: [0x7d,0x04,0x16,0x7d]
 
 0x7e,0x04,0x16,0x7d
-# GFX12: v_cmpx_ngt_f16_e32 exec_lo, v2          ; encoding: [0x7e,0x04,0x16,0x7d]
+# GFX12-REAL16: v_cmpx_ngt_f16_e32 exec_lo, v2.l        ; encoding: [0x7e,0x04,0x16,0x7d]
+# GFX12-FAKE16: v_cmpx_ngt_f16_e32 exec_lo, v2          ; encoding: [0x7e,0x04,0x16,0x7d]
 
 0x7f,0x04,0x16,0x7d
-# GFX12: v_cmpx_ngt_f16_e32 exec_hi, v2          ; encoding: [0x7f,0x04,0x16,0x7d]
+# GFX12-REAL16: v_cmpx_ngt_f16_e32 exec_hi, v2.l        ; encoding: [0x7f,0x04,0x16,0x7d]
+# GFX12-FAKE16: v_cmpx_ngt_f16_e32 exec_hi, v2          ; encoding: [0x7f,0x04,0x16,0x7d]
 
 0x7c,0x04,0x16,0x7d
-# GFX12: v_cmpx_ngt_f16_e32 null, v2             ; encoding: [0x7c,0x04,0x16,0x7d]
+# GFX12-REAL16: v_cmpx_ngt_f16_e32 null, v2.l           ; encoding: [0x7c,0x04,0x16,0x7d]
+# GFX12-FAKE16: v_cmpx_ngt_f16_e32 null, v2             ; encoding: [0x7c,0x04,0x16,0x7d]
 
 0xc1,0x04,0x16,0x7d
-# GFX12: v_cmpx_ngt_f16_e32 -1, v2               ; encoding: [0xc1,0x04,0x16,0x7d]
+# GFX12-REAL16: v_cmpx_ngt_f16_e32 -1, v2.l             ; encoding: [0xc1,0x04,0x16,0x7d]
+# GFX12-FAKE16: v_cmpx_ngt_f16_e32 -1, v2               ; encoding: [0xc1,0x04,0x16,0x7d]
 
 0xf0,0x04,0x16,0x7d
-# GFX12: v_cmpx_ngt_f16_e32 0.5, v2              ; encoding: [0xf0,0x04,0x16,0x7d]
+# GFX12-REAL16: v_cmpx_ngt_f16_e32 0.5, v2.l            ; encoding: [0xf0,0x04,0x16,0x7d]
+# GFX12-FAKE16: v_cmpx_ngt_f16_e32 0.5, v2              ; encoding: [0xf0,0x04,0x16,0x7d]
 
 0xfd,0x04,0x16,0x7d
-# GFX12: v_cmpx_ngt_f16_e32 src_scc, v2          ; encoding: [0xfd,0x04,0x16,0x7d]
+# GFX12-REAL16: v_cmpx_ngt_f16_e32 src_scc, v2.l        ; encoding: [0xfd,0x04,0x16,0x7d]
+# GFX12-FAKE16: v_cmpx_ngt_f16_e32 src_scc, v2          ; encoding: [0xfd,0x04,0x16,0x7d]
 
 0xff,0xfe,0x16,0x7d,0x0b,0xfe,0x00,0x00
-# GFX12: v_cmpx_ngt_f16_e32 0xfe0b, v127         ; encoding: [0xff,0xfe,0x16,0x7d,0x0b,0xfe,0x00,0x00]
+# GFX12-REAL16: v_cmpx_ngt_f16_e32 0xfe0b, v127.l       ; encoding: [0xff,0xfe,0x16,0x7d,0x0b,0xfe,0x00,0x00]
+# GFX12-FAKE16: v_cmpx_ngt_f16_e32 0xfe0b, v127         ; encoding: [0xff,0xfe,0x16,0x7d,0x0b,0xfe,0x00,0x00]
+
+0x81,0x05,0x16,0x7d
+# GFX12-REAL16: v_cmpx_ngt_f16_e32 v1.h, v2.l           ; encoding: [0x81,0x05,0x16,0x7d]
+# GFX12-FAKE16: v_cmpx_ngt_f16_e32 v129/*Invalid register, operand has 'VS_32_Lo128' register class*/, v2 ; encoding: [0x81,0x05,0x16,0x7d]
+
+0xff,0x05,0x16,0x7d
+# GFX12-REAL16: v_cmpx_ngt_f16_e32 v127.h, v2.l         ; encoding: [0xff,0x05,0x16,0x7d]
+# GFX12-FAKE16: v_cmpx_ngt_f16_e32 v255/*Invalid register, operand has 'VS_32_Lo128' register class*/, v2 ; encoding: [0xff,0x05,0x16,0x7d]
+
+0xfd,0x04,0x17,0x7d
+# GFX12-REAL16: v_cmpx_ngt_f16_e32 src_scc, v2.h        ; encoding: [0xfd,0x04,0x17,0x7d]
+# GFX12-FAKE16: v_cmpx_ngt_f16_e32 src_scc, v130/*Invalid register, operand has 'VGPR_32_Lo128' register class*/ ; encoding: [0xfd,0x04,0x17,0x7d]
+
+0xff,0xfe,0x17,0x7d,0x0b,0xfe,0x00,0x00
+# GFX12-REAL16: v_cmpx_ngt_f16_e32 0xfe0b, v127.h       ; encoding: [0xff,0xfe,0x17,0x7d,0x0b,0xfe,0x00,0x00]
+# GFX12-FAKE16: v_cmpx_ngt_f16_e32 0xfe0b, v255/*Invalid register, operand has 'VGPR_32_Lo128' register class*/ ; encoding: [0xff,0xfe,0x17,0x7d,0x0b,0xfe,0x00,0x00]
 
 0x01,0x05,0x36,0x7d
 # GFX12: v_cmpx_ngt_f32_e32 v1, v2               ; encoding: [0x01,0x05,0x36,0x7d]
@@ -3211,49 +3459,80 @@
 # GFX12: v_cmpx_ngt_f64_e32 0xaf123456, v[254:255] ; encoding: [0xff,0xfc,0x57,0x7d,0x56,0x34,0x12,0xaf]
 
 0x01,0x05,0x18,0x7d
-# GFX12: v_cmpx_nle_f16_e32 v1, v2               ; encoding: [0x01,0x05,0x18,0x7d]
+# GFX12-REAL16: v_cmpx_nle_f16_e32 v1.l, v2.l           ; encoding: [0x01,0x05,0x18,0x7d]
+# GFX12-FAKE16: v_cmpx_nle_f16_e32 v1, v2               ; encoding: [0x01,0x05,0x18,0x7d]
 
 0x7f,0x05,0x18,0x7d
-# GFX12: v_cmpx_nle_f16_e32 v127, v2             ; encoding: [0x7f,0x05,0x18,0x7d]
+# GFX12-REAL16: v_cmpx_nle_f16_e32 v127.l, v2.l         ; encoding: [0x7f,0x05,0x18,0x7d]
+# GFX12-FAKE16: v_cmpx_nle_f16_e32 v127, v2             ; encoding: [0x7f,0x05,0x18,0x7d]
 
 0x01,0x04,0x18,0x7d
-# GFX12: v_cmpx_nle_f16_e32 s1, v2               ; encoding: [0x01,0x04,0x18,0x7d]
+# GFX12-REAL16: v_cmpx_nle_f16_e32 s1, v2.l             ; encoding: [0x01,0x04,0x18,0x7d]
+# GFX12-FAKE16: v_cmpx_nle_f16_e32 s1, v2               ; encoding: [0x01,0x04,0x18,0x7d]
 
 0x69,0x04,0x18,0x7d
-# GFX12: v_cmpx_nle_f16_e32 s105, v2             ; encoding: [0x69,0x04,0x18,0x7d]
+# GFX12-REAL16: v_cmpx_nle_f16_e32 s105, v2.l           ; encoding: [0x69,0x04,0x18,0x7d]
+# GFX12-FAKE16: v_cmpx_nle_f16_e32 s105, v2             ; encoding: [0x69,0x04,0x18,0x7d]
 
 0x6a,0x04,0x18,0x7d
-# GFX12: v_cmpx_nle_f16_e32 vcc_lo, v2           ; encoding: [0x6a,0x04,0x18,0x7d]
+# GFX12-REAL16: v_cmpx_nle_f16_e32 vcc_lo, v2.l         ; encoding: [0x6a,0x04,0x18,0x7d]
+# GFX12-FAKE16: v_cmpx_nle_f16_e32 vcc_lo, v2           ; encoding: [0x6a,0x04,0x18,0x7d]
 
 0x6b,0x04,0x18,0x7d
-# GFX12: v_cmpx_nle_f16_e32 vcc_hi, v2           ; encoding: [0x6b,0x04,0x18,0x7d]
+# GFX12-REAL16: v_cmpx_nle_f16_e32 vcc_hi, v2.l         ; encoding: [0x6b,0x04,0x18,0x7d]
+# GFX12-FAKE16: v_cmpx_nle_f16_e32 vcc_hi, v2           ; encoding: [0x6b,0x04,0x18,0x7d]
 
 0x7b,0x04,0x18,0x7d
-# GFX12: v_cmpx_nle_f16_e32 ttmp15, v2           ; encoding: [0x7b,0x04,0x18,0x7d]
+# GFX12-REAL16: v_cmpx_nle_f16_e32 ttmp15, v2.l         ; encoding: [0x7b,0x04,0x18,0x7d]
+# GFX12-FAKE16: v_cmpx_nle_f16_e32 ttmp15, v2           ; encoding: [0x7b,0x04,0x18,0x7d]
 
 0x7d,0x04,0x18,0x7d
-# GFX12: v_cmpx_nle_f16_e32 m0, v2               ; encoding: [0x7d,0x04,0x18,0x7d]
+# GFX12-REAL16: v_cmpx_nle_f16_e32 m0, v2.l             ; encoding: [0x7d,0x04,0x18,0x7d]
+# GFX12-FAKE16: v_cmpx_nle_f16_e32 m0, v2               ; encoding: [0x7d,0x04,0x18,0x7d]
 
 0x7e,0x04,0x18,0x7d
-# GFX12: v_cmpx_nle_f16_e32 exec_lo, v2          ; encoding: [0x7e,0x04,0x18,0x7d]
+# GFX12-REAL16: v_cmpx_nle_f16_e32 exec_lo, v2.l        ; encoding: [0x7e,0x04,0x18,0x7d]
+# GFX12-FAKE16: v_cmpx_nle_f16_e32 exec_lo, v2          ; encoding: [0x7e,0x04,0x18,0x7d]
 
 0x7f,0x04,0x18,0x7d
-# GFX12: v_cmpx_nle_f16_e32 exec_hi, v2          ; encoding: [0x7f,0x04,0x18,0x7d]
+# GFX12-REAL16: v_cmpx_nle_f16_e32 exec_hi, v2.l        ; encoding: [0x7f,0x04,0x18,0x7d]
+# GFX12-FAKE16: v_cmpx_nle_f16_e32 exec_hi, v2          ; encoding: [0x7f,0x04,0x18,0x7d]
 
 0x7c,0x04,0x18,0x7d
-# GFX12: v_cmpx_nle_f16_e32 null, v2             ; encoding: [0x7c,0x04,0x18,0x7d]
+# GFX12-REAL16: v_cmpx_nle_f16_e32 null, v2.l           ; encoding: [0x7c,0x04,0x18,0x7d]
+# GFX12-FAKE16: v_cmpx_nle_f16_e32 null, v2             ; encoding: [0x7c,0x04,0x18,0x7d]
 
 0xc1,0x04,0x18,0x7d
-# GFX12: v_cmpx_nle_f16_e32 -1, v2               ; encoding: [0xc1,0x04,0x18,0x7d]
+# GFX12-REAL16: v_cmpx_nle_f16_e32 -1, v2.l             ; encoding: [0xc1,0x04,0x18,0x7d]
+# GFX12-FAKE16: v_cmpx_nle_f16_e32 -1, v2               ; encoding: [0xc1,0x04,0x18,0x7d]
 
 0xf0,0x04,0x18,0x7d
-# GFX12: v_cmpx_nle_f16_e32 0.5, v2              ; encoding: [0xf0,0x04,0x18,0x7d]
+# GFX12-REAL16: v_cmpx_nle_f16_e32 0.5, v2.l            ; encoding: [0xf0,0x04,0x18,0x7d]
+# GFX12-FAKE16: v_cmpx_nle_f16_e32 0.5, v2              ; encoding: [0xf0,0x04,0x18,0x7d]
 
 0xfd,0x04,0x18,0x7d
-# GFX12: v_cmpx_nle_f16_e32 src_scc, v2          ; encoding: [0xfd,0x04,0x18,0x7d]
+# GFX12-REAL16: v_cmpx_nle_f16_e32 src_scc, v2.l        ; encoding: [0xfd,0x04,0x18,0x7d]
+# GFX12-FAKE16: v_cmpx_nle_f16_e32 src_scc, v2          ; encoding: [0xfd,0x04,0x18,0x7d]
 
 0xff,0xfe,0x18,0x7d,0x0b,0xfe,0x00,0x00
-# GFX12: v_cmpx_nle_f16_e32 0xfe0b, v127         ; encoding: [0xff,0xfe,0x18,0x7d,0x0b,0xfe,0x00,0x00]
+# GFX12-REAL16: v_cmpx_nle_f16_e32 0xfe0b, v127.l       ; encoding: [0xff,0xfe,0x18,0x7d,0x0b,0xfe,0x00,0x00]
+# GFX12-FAKE16: v_cmpx_nle_f16_e32 0xfe0b, v127         ; encoding: [0xff,0xfe,0x18,0x7d,0x0b,0xfe,0x00,0x00]
+
+0x81,0x05,0x18,0x7d
+# GFX12-REAL16: v_cmpx_nle_f16_e32 v1.h, v2.l           ; encoding: [0x81,0x05,0x18,0x7d]
+# GFX12-FAKE16: v_cmpx_nle_f16_e32 v129/*Invalid register, operand has 'VS_32_Lo128' register class*/, v2 ; encoding: [0x81,0x05,0x18,0x7d]
+
+0xff,0x05,0x18,0x7d
+# GFX12-REAL16: v_cmpx_nle_f16_e32 v127.h, v2.l         ; encoding: [0xff,0x05,0x18,0x7d]
+# GFX12-FAKE16: v_cmpx_nle_f16_e32 v255/*Invalid register, operand has 'VS_32_Lo128' register class*/, v2 ; encoding: [0xff,0x05,0x18,0x7d]
+
+0xfd,0x04,0x19,0x7d
+# GFX12-REAL16: v_cmpx_nle_f16_e32 src_scc, v2.h        ; encoding: [0xfd,0x04,0x19,0x7d]
+# GFX12-FAKE16: v_cmpx_nle_f16_e32 src_scc, v130/*Invalid register, operand has 'VGPR_32_Lo128' register class*/ ; encoding: [0xfd,0x04,0x19,0x7d]
+
+0xff,0xfe,0x19,0x7d,0x0b,0xfe,0x00,0x00
+# GFX12-REAL16: v_cmpx_nle_f16_e32 0xfe0b, v127.h       ; encoding: [0xff,0xfe,0x19,0x7d,0x0b,0xfe,0x00,0x00]
+# GFX12-FAKE16: v_cmpx_nle_f16_e32 0xfe0b, v255/*Invalid register, operand has 'VGPR_32_Lo128' register class*/ ; encoding: [0xff,0xfe,0x19,0x7d,0x0b,0xfe,0x00,0x00]
 
 0x01,0x05,0x38,0x7d
 # GFX12: v_cmpx_nle_f32_e32 v1, v2               ; encoding: [0x01,0x05,0x38,0x7d]
@@ -3337,49 +3616,80 @@
 # GFX12: v_cmpx_nle_f64_e32 0xaf123456, v[254:255] ; encoding: [0xff,0xfc,0x59,0x7d,0x56,0x34,0x12,0xaf]
 
 0x01,0x05,0x14,0x7d
-# GFX12: v_cmpx_nlg_f16_e32 v1, v2               ; encoding: [0x01,0x05,0x14,0x7d]
+# GFX12-REAL16: v_cmpx_nlg_f16_e32 v1.l, v2.l           ; encoding: [0x01,0x05,0x14,0x7d]
+# GFX12-FAKE16: v_cmpx_nlg_f16_e32 v1, v2               ; encoding: [0x01,0x05,0x14,0x7d]
 
 0x7f,0x05,0x14,0x7d
-# GFX12: v_cmpx_nlg_f16_e32 v127, v2             ; encoding: [0x7f,0x05,0x14,0x7d]
+# GFX12-REAL16: v_cmpx_nlg_f16_e32 v127.l, v2.l         ; encoding: [0x7f,0x05,0x14,0x7d]
+# GFX12-FAKE16: v_cmpx_nlg_f16_e32 v127, v2             ; encoding: [0x7f,0x05,0x14,0x7d]
 
 0x01,0x04,0x14,0x7d
-# GFX12: v_cmpx_nlg_f16_e32 s1, v2               ; encoding: [0x01,0x04,0x14,0x7d]
+# GFX12-REAL16: v_cmpx_nlg_f16_e32 s1, v2.l             ; encoding: [0x01,0x04,0x14,0x7d]
+# GFX12-FAKE16: v_cmpx_nlg_f16_e32 s1, v2               ; encoding: [0x01,0x04,0x14,0x7d]
 
 0x69,0x04,0x14,0x7d
-# GFX12: v_cmpx_nlg_f16_e32 s105, v2             ; encoding: [0x69,0x04,0x14,0x7d]
+# GFX12-REAL16: v_cmpx_nlg_f16_e32 s105, v2.l           ; encoding: [0x69,0x04,0x14,0x7d]
+# GFX12-FAKE16: v_cmpx_nlg_f16_e32 s105, v2             ; encoding: [0x69,0x04,0x14,0x7d]
 
 0x6a,0x04,0x14,0x7d
-# GFX12: v_cmpx_nlg_f16_e32 vcc_lo, v2           ; encoding: [0x6a,0x04,0x14,0x7d]
+# GFX12-REAL16: v_cmpx_nlg_f16_e32 vcc_lo, v2.l         ; encoding: [0x6a,0x04,0x14,0x7d]
+# GFX12-FAKE16: v_cmpx_nlg_f16_e32 vcc_lo, v2           ; encoding: [0x6a,0x04,0x14,0x7d]
 
 0x6b,0x04,0x14,0x7d
-# GFX12: v_cmpx_nlg_f16_e32 vcc_hi, v2           ; encoding: [0x6b,0x04,0x14,0x7d]
+# GFX12-REAL16: v_cmpx_nlg_f16_e32 vcc_hi, v2.l         ; encoding: [0x6b,0x04,0x14,0x7d]
+# GFX12-FAKE16: v_cmpx_nlg_f16_e32 vcc_hi, v2           ; encoding: [0x6b,0x04,0x14,0x7d]
 
 0x7b,0x04,0x14,0x7d
-# GFX12: v_cmpx_nlg_f16_e32 ttmp15, v2           ; encoding: [0x7b,0x04,0x14,0x7d]
+# GFX12-REAL16: v_cmpx_nlg_f16_e32 ttmp15, v2.l         ; encoding: [0x7b,0x04,0x14,0x7d]
+# GFX12-FAKE16: v_cmpx_nlg_f16_e32 ttmp15, v2           ; encoding: [0x7b,0x04,0x14,0x7d]
 
 0x7d,0x04,0x14,0x7d
-# GFX12: v_cmpx_nlg_f16_e32 m0, v2               ; encoding: [0x7d,0x04,0x14,0x7d]
+# GFX12-REAL16: v_cmpx_nlg_f16_e32 m0, v2.l             ; encoding: [0x7d,0x04,0x14,0x7d]
+# GFX12-FAKE16: v_cmpx_nlg_f16_e32 m0, v2               ; encoding: [0x7d,0x04,0x14,0x7d]
 
 0x7e,0x04,0x14,0x7d
-# GFX12: v_cmpx_nlg_f16_e32 exec_lo, v2          ; encoding: [0x7e,0x04,0x14,0x7d]
+# GFX12-REAL16: v_cmpx_nlg_f16_e32 exec_lo, v2.l        ; encoding: [0x7e,0x04,0x14,0x7d]
+# GFX12-FAKE16: v_cmpx_nlg_f16_e32 exec_lo, v2          ; encoding: [0x7e,0x04,0x14,0x7d]
 
 0x7f,0x04,0x14,0x7d
-# GFX12: v_cmpx_nlg_f16_e32 exec_hi, v2          ; encoding: [0x7f,0x04,0x14,0x7d]
+# GFX12-REAL16: v_cmpx_nlg_f16_e32 exec_hi, v2.l        ; encoding: [0x7f,0x04,0x14,0x7d]
+# GFX12-FAKE16: v_cmpx_nlg_f16_e32 exec_hi, v2          ; encoding: [0x7f,0x04,0x14,0x7d]
 
 0x7c,0x04,0x14,0x7d
-# GFX12: v_cmpx_nlg_f16_e32 null, v2             ; encoding: [0x7c,0x04,0x14,0x7d]
+# GFX12-REAL16: v_cmpx_nlg_f16_e32 null, v2.l           ; encoding: [0x7c,0x04,0x14,0x7d]
+# GFX12-FAKE16: v_cmpx_nlg_f16_e32 null, v2             ; encoding: [0x7c,0x04,0x14,0x7d]
 
 0xc1,0x04,0x14,0x7d
-# GFX12: v_cmpx_nlg_f16_e32 -1, v2               ; encoding: [0xc1,0x04,0x14,0x7d]
+# GFX12-REAL16: v_cmpx_nlg_f16_e32 -1, v2.l             ; encoding: [0xc1,0x04,0x14,0x7d]
+# GFX12-FAKE16: v_cmpx_nlg_f16_e32 -1, v2               ; encoding: [0xc1,0x04,0x14,0x7d]
 
 0xf0,0x04,0x14,0x7d
-# GFX12: v_cmpx_nlg_f16_e32 0.5, v2              ; encoding: [0xf0,0x04,0x14,0x7d]
+# GFX12-REAL16: v_cmpx_nlg_f16_e32 0.5, v2.l            ; encoding: [0xf0,0x04,0x14,0x7d]
+# GFX12-FAKE16: v_cmpx_nlg_f16_e32 0.5, v2              ; encoding: [0xf0,0x04,0x14,0x7d]
 
 0xfd,0x04,0x14,0x7d
-# GFX12: v_cmpx_nlg_f16_e32 src_scc, v2          ; encoding: [0xfd,0x04,0x14,0x7d]
+# GFX12-REAL16: v_cmpx_nlg_f16_e32 src_scc, v2.l        ; encoding: [0xfd,0x04,0x14,0x7d]
+# GFX12-FAKE16: v_cmpx_nlg_f16_e32 src_scc, v2          ; encoding: [0xfd,0x04,0x14,0x7d]
 
 0xff,0xfe,0x14,0x7d,0x0b,0xfe,0x00,0x00
-# GFX12: v_cmpx_nlg_f16_e32 0xfe0b, v127         ; encoding: [0xff,0xfe,0x14,0x7d,0x0b,0xfe,0x00,0x00]
+# GFX12-REAL16: v_cmpx_nlg_f16_e32 0xfe0b, v127.l       ; encoding: [0xff,0xfe,0x14,0x7d,0x0b,0xfe,0x00,0x00]
+# GFX12-FAKE16: v_cmpx_nlg_f16_e32 0xfe0b, v127         ; encoding: [0xff,0xfe,0x14,0x7d,0x0b,0xfe,0x00,0x00]
+
+0x81,0x05,0x14,0x7d
+# GFX12-REAL16: v_cmpx_nlg_f16_e32 v1.h, v2.l           ; encoding: [0x81,0x05,0x14,0x7d]
+# GFX12-FAKE16: v_cmpx_nlg_f16_e32 v129/*Invalid register, operand has 'VS_32_Lo128' register class*/, v2 ; encoding: [0x81,0x05,0x14,0x7d]
+
+0xff,0x05,0x14,0x7d
+# GFX12-REAL16: v_cmpx_nlg_f16_e32 v127.h, v2.l         ; encoding: [0xff,0x05,0x14,0x7d]
+# GFX12-FAKE16: v_cmpx_nlg_f16_e32 v255/*Invalid register, operand has 'VS_32_Lo128' register class*/, v2 ; encoding: [0xff,0x05,0x14,0x7d]
+
+0xfd,0x04,0x15,0x7d
+# GFX12-REAL16: v_cmpx_nlg_f16_e32 src_scc, v2.h        ; encoding: [0xfd,0x04,0x15,0x7d]
+# GFX12-FAKE16: v_cmpx_nlg_f16_e32 src_scc, v130/*Invalid register, operand has 'VGPR_32_Lo128' register class*/ ; encoding: [0xfd,0x04,0x15,0x7d]
+
+0xff,0xfe,0x15,0x7d,0x0b,0xfe,0x00,0x00
+# GFX12-REAL16: v_cmpx_nlg_f16_e32 0xfe0b, v127.h       ; encoding: [0xff,0xfe,0x15,0x7d,0x0b,0xfe,0x00,0x00]
+# GFX12-FAKE16: v_cmpx_nlg_f16_e32 0xfe0b, v255/*Invalid register, operand has 'VGPR_32_Lo128' register class*/ ; encoding: [0xff,0xfe,0x15,0x7d,0x0b,0xfe,0x00,0x00]
 
 0x01,0x05,0x34,0x7d
 # GFX12: v_cmpx_nlg_f32_e32 v1, v2               ; encoding: [0x01,0x05,0x34,0x7d]
@@ -3463,49 +3773,80 @@
 # GFX12: v_cmpx_nlg_f64_e32 0xaf123456, v[254:255] ; encoding: [0xff,0xfc,0x55,0x7d,0x56,0x34,0x12,0xaf]
 
 0x01,0x05,0x1c,0x7d
-# GFX12: v_cmpx_nlt_f16_e32 v1, v2               ; encoding: [0x01,0x05,0x1c,0x7d]
+# GFX12-REAL16: v_cmpx_nlt_f16_e32 v1.l, v2.l           ; encoding: [0x01,0x05,0x1c,0x7d]
+# GFX12-FAKE16: v_cmpx_nlt_f16_e32 v1, v2               ; encoding: [0x01,0x05,0x1c,0x7d]
 
 0x7f,0x05,0x1c,0x7d
-# GFX12: v_cmpx_nlt_f16_e32 v127, v2             ; encoding: [0x7f,0x05,0x1c,0x7d]
+# GFX12-REAL16: v_cmpx_nlt_f16_e32 v127.l, v2.l         ; encoding: [0x7f,0x05,0x1c,0x7d]
+# GFX12-FAKE16: v_cmpx_nlt_f16_e32 v127, v2             ; encoding: [0x7f,0x05,0x1c,0x7d]
 
 0x01,0x04,0x1c,0x7d
-# GFX12: v_cmpx_nlt_f16_e32 s1, v2               ; encoding: [0x01,0x04,0x1c,0x7d]
+# GFX12-REAL16: v_cmpx_nlt_f16_e32 s1, v2.l             ; encoding: [0x01,0x04,0x1c,0x7d]
+# GFX12-FAKE16: v_cmpx_nlt_f16_e32 s1, v2               ; encoding: [0x01,0x04,0x1c,0x7d]
 
 0x69,0x04,0x1c,0x7d
-# GFX12: v_cmpx_nlt_f16_e32 s105, v2             ; encoding: [0x69,0x04,0x1c,0x7d]
+# GFX12-REAL16: v_cmpx_nlt_f16_e32 s105, v2.l           ; encoding: [0x69,0x04,0x1c,0x7d]
+# GFX12-FAKE16: v_cmpx_nlt_f16_e32 s105, v2             ; encoding: [0x69,0x04,0x1c,0x7d]
 
 0x6a,0x04,0x1c,0x7d
-# GFX12: v_cmpx_nlt_f16_e32 vcc_lo, v2           ; encoding: [0x6a,0x04,0x1c,0x7d]
+# GFX12-REAL16: v_cmpx_nlt_f16_e32 vcc_lo, v2.l         ; encoding: [0x6a,0x04,0x1c,0x7d]
+# GFX12-FAKE16: v_cmpx_nlt_f16_e32 vcc_lo, v2           ; encoding: [0x6a,0x04,0x1c,0x7d]
 
 0x6b,0x04,0x1c,0x7d
-# GFX12: v_cmpx_nlt_f16_e32 vcc_hi, v2           ; encoding: [0x6b,0x04,0x1c,0x7d]
+# GFX12-REAL16: v_cmpx_nlt_f16_e32 vcc_hi, v2.l         ; encoding: [0x6b,0x04,0x1c,0x7d]
+# GFX12-FAKE16: v_cmpx_nlt_f16_e32 vcc_hi, v2           ; encoding: [0x6b,0x04,0x1c,0x7d]
 
 0x7b,0x04,0x1c,0x7d
-# GFX12: v_cmpx_nlt_f16_e32 ttmp15, v2           ; encoding: [0x7b,0x04,0x1c,0x7d]
+# GFX12-REAL16: v_cmpx_nlt_f16_e32 ttmp15, v2.l         ; encoding: [0x7b,0x04,0x1c,0x7d]
+# GFX12-FAKE16: v_cmpx_nlt_f16_e32 ttmp15, v2           ; encoding: [0x7b,0x04,0x1c,0x7d]
 
 0x7d,0x04,0x1c,0x7d
-# GFX12: v_cmpx_nlt_f16_e32 m0, v2               ; encoding: [0x7d,0x04,0x1c,0x7d]
+# GFX12-REAL16: v_cmpx_nlt_f16_e32 m0, v2.l             ; encoding: [0x7d,0x04,0x1c,0x7d]
+# GFX12-FAKE16: v_cmpx_nlt_f16_e32 m0, v2               ; encoding: [0x7d,0x04,0x1c,0x7d]
 
 0x7e,0x04,0x1c,0x7d
-# GFX12: v_cmpx_nlt_f16_e32 exec_lo, v2          ; encoding: [0x7e,0x04,0x1c,0x7d]
+# GFX12-REAL16: v_cmpx_nlt_f16_e32 exec_lo, v2.l        ; encoding: [0x7e,0x04,0x1c,0x7d]
+# GFX12-FAKE16: v_cmpx_nlt_f16_e32 exec_lo, v2          ; encoding: [0x7e,0x04,0x1c,0x7d]
 
 0x7f,0x04,0x1c,0x7d
-# GFX12: v_cmpx_nlt_f16_e32 exec_hi, v2          ; encoding: [0x7f,0x04,0x1c,0x7d]
+# GFX12-REAL16: v_cmpx_nlt_f16_e32 exec_hi, v2.l        ; encoding: [0x7f,0x04,0x1c,0x7d]
+# GFX12-FAKE16: v_cmpx_nlt_f16_e32 exec_hi, v2          ; encoding: [0x7f,0x04,0x1c,0x7d]
 
 0x7c,0x04,0x1c,0x7d
-# GFX12: v_cmpx_nlt_f16_e32 null, v2             ; encoding: [0x7c,0x04,0x1c,0x7d]
+# GFX12-REAL16: v_cmpx_nlt_f16_e32 null, v2.l           ; encoding: [0x7c,0x04,0x1c,0x7d]
+# GFX12-FAKE16: v_cmpx_nlt_f16_e32 null, v2             ; encoding: [0x7c,0x04,0x1c,0x7d]
 
 0xc1,0x04,0x1c,0x7d
-# GFX12: v_cmpx_nlt_f16_e32 -1, v2               ; encoding: [0xc1,0x04,0x1c,0x7d]
+# GFX12-REAL16: v_cmpx_nlt_f16_e32 -1, v2.l             ; encoding: [0xc1,0x04,0x1c,0x7d]
+# GFX12-FAKE16: v_cmpx_nlt_f16_e32 -1, v2               ; encoding: [0xc1,0x04,0x1c,0x7d]
 
 0xf0,0x04,0x1c,0x7d
-# GFX12: v_cmpx_nlt_f16_e32 0.5, v2              ; encoding: [0xf0,0x04,0x1c,0x7d]
+# GFX12-REAL16: v_cmpx_nlt_f16_e32 0.5, v2.l            ; encoding: [0xf0,0x04,0x1c,0x7d]
+# GFX12-FAKE16: v_cmpx_nlt_f16_e32 0.5, v2              ; encoding: [0xf0,0x04,0x1c,0x7d]
 
 0xfd,0x04,0x1c,0x7d
-# GFX12: v_cmpx_nlt_f16_e32 src_scc, v2          ; encoding: [0xfd,0x04,0x1c,0x7d]
+# GFX12-REAL16: v_cmpx_nlt_f16_e32 src_scc, v2.l        ; encoding: [0xfd,0x04,0x1c,0x7d]
+# GFX12-FAKE16: v_cmpx_nlt_f16_e32 src_scc, v2          ; encoding: [0xfd,0x04,0x1c,0x7d]
 
 0xff,0xfe,0x1c,0x7d,0x0b,0xfe,0x00,0x00
-# GFX12: v_cmpx_nlt_f16_e32 0xfe0b, v127         ; encoding: [0xff,0xfe,0x1c,0x7d,0x0b,0xfe,0x00,0x00]
+# GFX12-REAL16: v_cmpx_nlt_f16_e32 0xfe0b, v127.l       ; encoding: [0xff,0xfe,0x1c,0x7d,0x0b,0xfe,0x00,0x00]
+# GFX12-FAKE16: v_cmpx_nlt_f16_e32 0xfe0b, v127         ; encoding: [0xff,0xfe,0x1c,0x7d,0x0b,0xfe,0x00,0x00]
+
+0x81,0x05,0x1c,0x7d
+# GFX12-REAL16: v_cmpx_nlt_f16_e32 v1.h, v2.l           ; encoding: [0x81,0x05,0x1c,0x7d]
+# GFX12-FAKE16: v_cmpx_nlt_f16_e32 v129/*Invalid register, operand has 'VS_32_Lo128' register class*/, v2 ; encoding: [0x81,0x05,0x1c,0x7d]
+
+0xff,0x05,0x1c,0x7d
+# GFX12-REAL16: v_cmpx_nlt_f16_e32 v127.h, v2.l         ; encoding: [0xff,0x05,0x1c,0x7d]
+# GFX12-FAKE16: v_cmpx_nlt_f16_e32 v255/*Invalid register, operand has 'VS_32_Lo128' register class*/, v2 ; encoding: [0xff,0x05,0x1c,0x7d]
+
+0xfd,0x04,0x1d,0x7d
+# GFX12-REAL16: v_cmpx_nlt_f16_e32 src_scc, v2.h        ; encoding: [0xfd,0x04,0x1d,0x7d]
+# GFX12-FAKE16: v_cmpx_nlt_f16_e32 src_scc, v130/*Invalid register, operand has 'VGPR_32_Lo128' register class*/ ; encoding: [0xfd,0x04,0x1d,0x7d]
+
+0xff,0xfe,0x1d,0x7d,0x0b,0xfe,0x00,0x00
+# GFX12-REAL16: v_cmpx_nlt_f16_e32 0xfe0b, v127.h       ; encoding: [0xff,0xfe,0x1d,0x7d,0x0b,0xfe,0x00,0x00]
+# GFX12-FAKE16: v_cmpx_nlt_f16_e32 0xfe0b, v255/*Invalid register, operand has 'VGPR_32_Lo128' register class*/ ; encoding: [0xff,0xfe,0x1d,0x7d,0x0b,0xfe,0x00,0x00]
 
 0x01,0x05,0x3c,0x7d
 # GFX12: v_cmpx_nlt_f32_e32 v1, v2               ; encoding: [0x01,0x05,0x3c,0x7d]
@@ -3589,49 +3930,80 @@
 # GFX12: v_cmpx_nlt_f64_e32 0xaf123456, v[254:255] ; encoding: [0xff,0xfc,0x5d,0x7d,0x56,0x34,0x12,0xaf]
 
 0x01,0x05,0x0e,0x7d
-# GFX12: v_cmpx_o_f16_e32 v1, v2                 ; encoding: [0x01,0x05,0x0e,0x7d]
+# GFX12-REAL16: v_cmpx_o_f16_e32 v1.l, v2.l             ; encoding: [0x01,0x05,0x0e,0x7d]
+# GFX12-FAKE16: v_cmpx_o_f16_e32 v1, v2                 ; encoding: [0x01,0x05,0x0e,0x7d]
 
 0x7f,0x05,0x0e,0x7d
-# GFX12: v_cmpx_o_f16_e32 v127, v2               ; encoding: [0x7f,0x05,0x0e,0x7d]
+# GFX12-REAL16: v_cmpx_o_f16_e32 v127.l, v2.l           ; encoding: [0x7f,0x05,0x0e,0x7d]
+# GFX12-FAKE16: v_cmpx_o_f16_e32 v127, v2               ; encoding: [0x7f,0x05,0x0e,0x7d]
 
 0x01,0x04,0x0e,0x7d
-# GFX12: v_cmpx_o_f16_e32 s1, v2                 ; encoding: [0x01,0x04,0x0e,0x7d]
+# GFX12-REAL16: v_cmpx_o_f16_e32 s1, v2.l               ; encoding: [0x01,0x04,0x0e,0x7d]
+# GFX12-FAKE16: v_cmpx_o_f16_e32 s1, v2                 ; encoding: [0x01,0x04,0x0e,0x7d]
 
 0x69,0x04,0x0e,0x7d
-# GFX12: v_cmpx_o_f16_e32 s105, v2               ; encoding: [0x69,0x04,0x0e,0x7d]
+# GFX12-REAL16: v_cmpx_o_f16_e32 s105, v2.l             ; encoding: [0x69,0x04,0x0e,0x7d]
+# GFX12-FAKE16: v_cmpx_o_f16_e32 s105, v2               ; encoding: [0x69,0x04,0x0e,0x7d]
 
 0x6a,0x04,0x0e,0x7d
-# GFX12: v_cmpx_o_f16_e32 vcc_lo, v2             ; encoding: [0x6a,0x04,0x0e,0x7d]
+# GFX12-REAL16: v_cmpx_o_f16_e32 vcc_lo, v2.l           ; encoding: [0x6a,0x04,0x0e,0x7d]
+# GFX12-FAKE16: v_cmpx_o_f16_e32 vcc_lo, v2             ; encoding: [0x6a,0x04,0x0e,0x7d]
 
 0x6b,0x04,0x0e,0x7d
-# GFX12: v_cmpx_o_f16_e32 vcc_hi, v2             ; encoding: [0x6b,0x04,0x0e,0x7d]
+# GFX12-REAL16: v_cmpx_o_f16_e32 vcc_hi, v2.l           ; encoding: [0x6b,0x04,0x0e,0x7d]
+# GFX12-FAKE16: v_cmpx_o_f16_e32 vcc_hi, v2             ; encoding: [0x6b,0x04,0x0e,0x7d]
 
 0x7b,0x04,0x0e,0x7d
-# GFX12: v_cmpx_o_f16_e32 ttmp15, v2             ; encoding: [0x7b,0x04,0x0e,0x7d]
+# GFX12-REAL16: v_cmpx_o_f16_e32 ttmp15, v2.l           ; encoding: [0x7b,0x04,0x0e,0x7d]
+# GFX12-FAKE16: v_cmpx_o_f16_e32 ttmp15, v2             ; encoding: [0x7b,0x04,0x0e,0x7d]
 
 0x7d,0x04,0x0e,0x7d
-# GFX12: v_cmpx_o_f16_e32 m0, v2                 ; encoding: [0x7d,0x04,0x0e,0x7d]
+# GFX12-REAL16: v_cmpx_o_f16_e32 m0, v2.l               ; encoding: [0x7d,0x04,0x0e,0x7d]
+# GFX12-FAKE16: v_cmpx_o_f16_e32 m0, v2                 ; encoding: [0x7d,0x04,0x0e,0x7d]
 
 0x7e,0x04,0x0e,0x7d
-# GFX12: v_cmpx_o_f16_e32 exec_lo, v2            ; encoding: [0x7e,0x04,0x0e,0x7d]
+# GFX12-REAL16: v_cmpx_o_f16_e32 exec_lo, v2.l          ; encoding: [0x7e,0x04,0x0e,0x7d]
+# GFX12-FAKE16: v_cmpx_o_f16_e32 exec_lo, v2            ; encoding: [0x7e,0x04,0x0e,0x7d]
 
 0x7f,0x04,0x0e,0x7d
-# GFX12: v_cmpx_o_f16_e32 exec_hi, v2            ; encoding: [0x7f,0x04,0x0e,0x7d]
+# GFX12-REAL16: v_cmpx_o_f16_e32 exec_hi, v2.l          ; encoding: [0x7f,0x04,0x0e,0x7d]
+# GFX12-FAKE16: v_cmpx_o_f16_e32 exec_hi, v2            ; encoding: [0x7f,0x04,0x0e,0x7d]
 
 0x7c,0x04,0x0e,0x7d
-# GFX12: v_cmpx_o_f16_e32 null, v2               ; encoding: [0x7c,0x04,0x0e,0x7d]
+# GFX12-REAL16: v_cmpx_o_f16_e32 null, v2.l             ; encoding: [0x7c,0x04,0x0e,0x7d]
+# GFX12-FAKE16: v_cmpx_o_f16_e32 null, v2               ; encoding: [0x7c,0x04,0x0e,0x7d]
 
 0xc1,0x04,0x0e,0x7d
-# GFX12: v_cmpx_o_f16_e32 -1, v2                 ; encoding: [0xc1,0x04,0x0e,0x7d]
+# GFX12-REAL16: v_cmpx_o_f16_e32 -1, v2.l               ; encoding: [0xc1,0x04,0x0e,0x7d]
+# GFX12-FAKE16: v_cmpx_o_f16_e32 -1, v2                 ; encoding: [0xc1,0x04,0x0e,0x7d]
 
 0xf0,0x04,0x0e,0x7d
-# GFX12: v_cmpx_o_f16_e32 0.5, v2                ; encoding: [0xf0,0x04,0x0e,0x7d]
+# GFX12-REAL16: v_cmpx_o_f16_e32 0.5, v2.l              ; encoding: [0xf0,0x04,0x0e,0x7d]
+# GFX12-FAKE16: v_cmpx_o_f16_e32 0.5, v2                ; encoding: [0xf0,0x04,0x0e,0x7d]
 
 0xfd,0x04,0x0e,0x7d
-# GFX12: v_cmpx_o_f16_e32 src_scc, v2            ; encoding: [0xfd,0x04,0x0e,0x7d]
+# GFX12-REAL16: v_cmpx_o_f16_e32 src_scc, v2.l          ; encoding: [0xfd,0x04,0x0e,0x7d]
+# GFX12-FAKE16: v_cmpx_o_f16_e32 src_scc, v2            ; encoding: [0xfd,0x04,0x0e,0x7d]
 
 0xff,0xfe,0x0e,0x7d,0x0b,0xfe,0x00,0x00
-# GFX12: v_cmpx_o_f16_e32 0xfe0b, v127           ; encoding: [0xff,0xfe,0x0e,0x7d,0x0b,0xfe,0x00,0x00]
+# GFX12-REAL16: v_cmpx_o_f16_e32 0xfe0b, v127.l         ; encoding: [0xff,0xfe,0x0e,0x7d,0x0b,0xfe,0x00,0x00]
+# GFX12-FAKE16: v_cmpx_o_f16_e32 0xfe0b, v127           ; encoding: [0xff,0xfe,0x0e,0x7d,0x0b,0xfe,0x00,0x00]
+
+0x81,0x05,0x0e,0x7d
+# GFX12-REAL16: v_cmpx_o_f16_e32 v1.h, v2.l             ; encoding: [0x81,0x05,0x0e,0x7d]
+# GFX12-FAKE16: v_cmpx_o_f16_e32 v129/*Invalid register, operand has 'VS_32_Lo128' register class*/, v2 ; encoding: [0x81,0x05,0x0e,0x7d]
+
+0xff,0x05,0x0e,0x7d
+# GFX12-REAL16: v_cmpx_o_f16_e32 v127.h, v2.l           ; encoding: [0xff,0x05,0x0e,0x7d]
+# GFX12-FAKE16: v_cmpx_o_f16_e32 v255/*Invalid register, operand has 'VS_32_Lo128' register class*/, v2 ; encoding: [0xff,0x05,0x0e,0x7d]
+
+0xfd,0x04,0x0f,0x7d
+# GFX12-REAL16: v_cmpx_o_f16_e32 src_scc, v2.h          ; encoding: [0xfd,0x04,0x0f,0x7d]
+# GFX12-FAKE16: v_cmpx_o_f16_e32 src_scc, v130/*Invalid register, operand has 'VGPR_32_Lo128' register class*/ ; encoding: [0xfd,0x04,0x0f,0x7d]
+
+0xff,0xfe,0x0f,0x7d,0x0b,0xfe,0x00,0x00
+# GFX12-REAL16: v_cmpx_o_f16_e32 0xfe0b, v127.h         ; encoding: [0xff,0xfe,0x0f,0x7d,0x0b,0xfe,0x00,0x00]
+# GFX12-FAKE16: v_cmpx_o_f16_e32 0xfe0b, v255/*Invalid register, operand has 'VGPR_32_Lo128' register class*/ ; encoding: [0xff,0xfe,0x0f,0x7d,0x0b,0xfe,0x00,0x00]
 
 0x01,0x05,0x2e,0x7d
 # GFX12: v_cmpx_o_f32_e32 v1, v2                 ; encoding: [0x01,0x05,0x2e,0x7d]
@@ -3715,49 +4087,80 @@
 # GFX12: v_cmpx_o_f64_e32 0xaf123456, v[254:255] ; encoding: [0xff,0xfc,0x4f,0x7d,0x56,0x34,0x12,0xaf]
 
 0x01,0x05,0x10,0x7d
-# GFX12: v_cmpx_u_f16_e32 v1, v2                 ; encoding: [0x01,0x05,0x10,0x7d]
+# GFX12-REAL16: v_cmpx_u_f16_e32 v1.l, v2.l             ; encoding: [0x01,0x05,0x10,0x7d]
+# GFX12-FAKE16: v_cmpx_u_f16_e32 v1, v2                 ; encoding: [0x01,0x05,0x10,0x7d]
 
 0x7f,0x05,0x10,0x7d
-# GFX12: v_cmpx_u_f16_e32 v127, v2               ; encoding: [0x7f,0x05,0x10,0x7d]
+# GFX12-REAL16: v_cmpx_u_f16_e32 v127.l, v2.l           ; encoding: [0x7f,0x05,0x10,0x7d]
+# GFX12-FAKE16: v_cmpx_u_f16_e32 v127, v2               ; encoding: [0x7f,0x05,0x10,0x7d]
 
 0x01,0x04,0x10,0x7d
-# GFX12: v_cmpx_u_f16_e32 s1, v2                 ; encoding: [0x01,0x04,0x10,0x7d]
+# GFX12-REAL16: v_cmpx_u_f16_e32 s1, v2.l               ; encoding: [0x01,0x04,0x10,0x7d]
+# GFX12-FAKE16: v_cmpx_u_f16_e32 s1, v2                 ; encoding: [0x01,0x04,0x10,0x7d]
 
 0x69,0x04,0x10,0x7d
-# GFX12: v_cmpx_u_f16_e32 s105, v2               ; encoding: [0x69,0x04,0x10,0x7d]
+# GFX12-REAL16: v_cmpx_u_f16_e32 s105, v2.l             ; encoding: [0x69,0x04,0x10,0x7d]
+# GFX12-FAKE16: v_cmpx_u_f16_e32 s105, v2               ; encoding: [0x69,0x04,0x10,0x7d]
 
 0x6a,0x04,0x10,0x7d
-# GFX12: v_cmpx_u_f16_e32 vcc_lo, v2             ; encoding: [0x6a,0x04,0x10,0x7d]
+# GFX12-REAL16: v_cmpx_u_f16_e32 vcc_lo, v2.l           ; encoding: [0x6a,0x04,0x10,0x7d]
+# GFX12-FAKE16: v_cmpx_u_f16_e32 vcc_lo, v2             ; encoding: [0x6a,0x04,0x10,0x7d]
 
 0x6b,0x04,0x10,0x7d
-# GFX12: v_cmpx_u_f16_e32 vcc_hi, v2             ; encoding: [0x6b,0x04,0x10,0x7d]
+# GFX12-REAL16: v_cmpx_u_f16_e32 vcc_hi, v2.l           ; encoding: [0x6b,0x04,0x10,0x7d]
+# GFX12-FAKE16: v_cmpx_u_f16_e32 vcc_hi, v2             ; encoding: [0x6b,0x04,0x10,0x7d]
 
 0x7b,0x04,0x10,0x7d
-# GFX12: v_cmpx_u_f16_e32 ttmp15, v2             ; encoding: [0x7b,0x04,0x10,0x7d]
+# GFX12-REAL16: v_cmpx_u_f16_e32 ttmp15, v2.l           ; encoding: [0x7b,0x04,0x10,0x7d]
+# GFX12-FAKE16: v_cmpx_u_f16_e32 ttmp15, v2             ; encoding: [0x7b,0x04,0x10,0x7d]
 
 0x7d,0x04,0x10,0x7d
-# GFX12: v_cmpx_u_f16_e32 m0, v2                 ; encoding: [0x7d,0x04,0x10,0x7d]
+# GFX12-REAL16: v_cmpx_u_f16_e32 m0, v2.l               ; encoding: [0x7d,0x04,0x10,0x7d]
+# GFX12-FAKE16: v_cmpx_u_f16_e32 m0, v2                 ; encoding: [0x7d,0x04,0x10,0x7d]
 
 0x7e,0x04,0x10,0x7d
-# GFX12: v_cmpx_u_f16_e32 exec_lo, v2            ; encoding: [0x7e,0x04,0x10,0x7d]
+# GFX12-REAL16: v_cmpx_u_f16_e32 exec_lo, v2.l          ; encoding: [0x7e,0x04,0x10,0x7d]
+# GFX12-FAKE16: v_cmpx_u_f16_e32 exec_lo, v2            ; encoding: [0x7e,0x04,0x10,0x7d]
 
 0x7f,0x04,0x10,0x7d
-# GFX12: v_cmpx_u_f16_e32 exec_hi, v2            ; encoding: [0x7f,0x04,0x10,0x7d]
+# GFX12-REAL16: v_cmpx_u_f16_e32 exec_hi, v2.l          ; encoding: [0x7f,0x04,0x10,0x7d]
+# GFX12-FAKE16: v_cmpx_u_f16_e32 exec_hi, v2            ; encoding: [0x7f,0x04,0x10,0x7d]
 
 0x7c,0x04,0x10,0x7d
-# GFX12: v_cmpx_u_f16_e32 null, v2               ; encoding: [0x7c,0x04,0x10,0x7d]
+# GFX12-REAL16: v_cmpx_u_f16_e32 null, v2.l             ; encoding: [0x7c,0x04,0x10,0x7d]
+# GFX12-FAKE16: v_cmpx_u_f16_e32 null, v2               ; encoding: [0x7c,0x04,0x10,0x7d]
 
 0xc1,0x04,0x10,0x7d
-# GFX12: v_cmpx_u_f16_e32 -1, v2                 ; encoding: [0xc1,0x04,0x10,0x7d]
+# GFX12-REAL16: v_cmpx_u_f16_e32 -1, v2.l               ; encoding: [0xc1,0x04,0x10,0x7d]
+# GFX12-FAKE16: v_cmpx_u_f16_e32 -1, v2                 ; encoding: [0xc1,0x04,0x10,0x7d]
 
 0xf0,0x04,0x10,0x7d
-# GFX12: v_cmpx_u_f16_e32 0.5, v2                ; encoding: [0xf0,0x04,0x10,0x7d]
+# GFX12-REAL16: v_cmpx_u_f16_e32 0.5, v2.l              ; encoding: [0xf0,0x04,0x10,0x7d]
+# GFX12-FAKE16: v_cmpx_u_f16_e32 0.5, v2                ; encoding: [0xf0,0x04,0x10,0x7d]
 
 0xfd,0x04,0x10,0x7d
-# GFX12: v_cmpx_u_f16_e32 src_scc, v2            ; encoding: [0xfd,0x04,0x10,0x7d]
+# GFX12-REAL16: v_cmpx_u_f16_e32 src_scc, v2.l          ; encoding: [0xfd,0x04,0x10,0x7d]
+# GFX12-FAKE16: v_cmpx_u_f16_e32 src_scc, v2            ; encoding: [0xfd,0x04,0x10,0x7d]
 
 0xff,0xfe,0x10,0x7d,0x0b,0xfe,0x00,0x00
-# GFX12: v_cmpx_u_f16_e32 0xfe0b, v127           ; encoding: [0xff,0xfe,0x10,0x7d,0x0b,0xfe,0x00,0x00]
+# GFX12-REAL16: v_cmpx_u_f16_e32 0xfe0b, v127.l         ; encoding: [0xff,0xfe,0x10,0x7d,0x0b,0xfe,0x00,0x00]
+# GFX12-FAKE16: v_cmpx_u_f16_e32 0xfe0b, v127           ; encoding: [0xff,0xfe,0x10,0x7d,0x0b,0xfe,0x00,0x00]
+
+0x81,0x05,0x10,0x7d
+# GFX12-REAL16: v_cmpx_u_f16_e32 v1.h, v2.l             ; encoding: [0x81,0x05,0x10,0x7d]
+# GFX12-FAKE16: v_cmpx_u_f16_e32 v129/*Invalid register, operand has 'VS_32_Lo128' register class*/, v2 ; encoding: [0x81,0x05,0x10,0x7d]
+
+0xff,0x05,0x10,0x7d
+# GFX12-REAL16: v_cmpx_u_f16_e32 v127.h, v2.l           ; encoding: [0xff,0x05,0x10,0x7d]
+# GFX12-FAKE16: v_cmpx_u_f16_e32 v255/*Invalid register, operand has 'VS_32_Lo128' register class*/, v2 ; encoding: [0xff,0x05,0x10,0x7d]
+
+0xfd,0x04,0x11,0x7d
+# GFX12-REAL16: v_cmpx_u_f16_e32 src_scc, v2.h          ; encoding: [0xfd,0x04,0x11,0x7d]
+# GFX12-FAKE16: v_cmpx_u_f16_e32 src_scc, v130/*Invalid register, operand has 'VGPR_32_Lo128' register class*/ ; encoding: [0xfd,0x04,0x11,0x7d]
+
+0xff,0xfe,0x11,0x7d,0x0b,0xfe,0x00,0x00
+# GFX12-REAL16: v_cmpx_u_f16_e32 0xfe0b, v127.h         ; encoding: [0xff,0xfe,0x11,0x7d,0x0b,0xfe,0x00,0x00]
+# GFX12-FAKE16: v_cmpx_u_f16_e32 0xfe0b, v255/*Invalid register, operand has 'VGPR_32_Lo128' register class*/ ; encoding: [0xff,0xfe,0x11,0x7d,0x0b,0xfe,0x00,0x00]
 
 0x01,0x05,0x30,0x7d
 # GFX12: v_cmpx_u_f32_e32 v1, v2                 ; encoding: [0x01,0x05,0x30,0x7d]
diff --git a/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vopcx_dpp16.txt b/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vopcx_dpp16.txt
index f1fca29120490..61655c0aa2001 100644
--- a/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vopcx_dpp16.txt
+++ b/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vopcx_dpp16.txt
@@ -111,46 +111,68 @@
 # GFX12: v_cmpx_class_f32 -|v255|, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0xfd,0x7d,0xff,0x6f,0x3d,0x30]
 
 0xfa,0x04,0x04,0x7d,0x01,0x1b,0x00,0xff
-# GFX12: v_cmpx_eq_f16 v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x04,0x7d,0x01,0x1b,0x00,0xff]
+# GFX12-REAL16: v_cmpx_eq_f16 v1.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x04,0x7d,0x01,0x1b,0x00,0xff]
+# GFX12-FAKE16: v_cmpx_eq_f16 v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x04,0x7d,0x01,0x1b,0x00,0xff]
 
 0xfa,0x04,0x04,0x7d,0x01,0xe4,0x00,0xff
-# GFX12: v_cmpx_eq_f16 v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x04,0x7d,0x01,0xe4,0x00,0xff]
+# GFX12-REAL16: v_cmpx_eq_f16 v1.l, v2.l quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x04,0x7d,0x01,0xe4,0x00,0xff]
+# GFX12-FAKE16: v_cmpx_eq_f16 v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x04,0x7d,0x01,0xe4,0x00,0xff]
 
 0xfa,0x04,0x04,0x7d,0x01,0x40,0x01,0xff
-# GFX12: v_cmpx_eq_f16 v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x04,0x7d,0x01,0x40,0x01,0xff]
+# GFX12-REAL16: v_cmpx_eq_f16 v1.l, v2.l row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x04,0x7d,0x01,0x40,0x01,0xff]
+# GFX12-FAKE16: v_cmpx_eq_f16 v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x04,0x7d,0x01,0x40,0x01,0xff]
 
 0xfa,0x04,0x04,0x7d,0x01,0x41,0x01,0xff
-# GFX12: v_cmpx_eq_f16 v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x04,0x7d,0x01,0x41,0x01,0xff]
+# GFX12-REAL16: v_cmpx_eq_f16 v1.l, v2.l row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x04,0x7d,0x01,0x41,0x01,0xff]
+# GFX12-FAKE16: v_cmpx_eq_f16 v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x04,0x7d,0x01,0x41,0x01,0xff]
 
 0xfa,0x04,0x04,0x7d,0x01,0x01,0x01,0xff
-# GFX12: v_cmpx_eq_f16 v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x04,0x7d,0x01,0x01,0x01,0xff]
+# GFX12-REAL16: v_cmpx_eq_f16 v1.l, v2.l row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x04,0x7d,0x01,0x01,0x01,0xff]
+# GFX12-FAKE16: v_cmpx_eq_f16 v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x04,0x7d,0x01,0x01,0x01,0xff]
 
 0xfa,0x04,0x04,0x7d,0x01,0x0f,0x01,0xff
-# GFX12: v_cmpx_eq_f16 v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x04,0x7d,0x01,0x0f,0x01,0xff]
+# GFX12-REAL16: v_cmpx_eq_f16 v1.l, v2.l row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x04,0x7d,0x01,0x0f,0x01,0xff]
+# GFX12-FAKE16: v_cmpx_eq_f16 v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x04,0x7d,0x01,0x0f,0x01,0xff]
 
 0xfa,0x04,0x04,0x7d,0x01,0x11,0x01,0xff
-# GFX12: v_cmpx_eq_f16 v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x04,0x7d,0x01,0x11,0x01,0xff]
+# GFX12-REAL16: v_cmpx_eq_f16 v1.l, v2.l row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x04,0x7d,0x01,0x11,0x01,0xff]
+# GFX12-FAKE16: v_cmpx_eq_f16 v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x04,0x7d,0x01,0x11,0x01,0xff]
 
 0xfa,0x04,0x04,0x7d,0x01,0x1f,0x01,0xff
-# GFX12: v_cmpx_eq_f16 v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x04,0x7d,0x01,0x1f,0x01,0xff]
+# GFX12-REAL16: v_cmpx_eq_f16 v1.l, v2.l row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x04,0x7d,0x01,0x1f,0x01,0xff]
+# GFX12-FAKE16: v_cmpx_eq_f16 v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x04,0x7d,0x01,0x1f,0x01,0xff]
 
 0xfa,0x04,0x04,0x7d,0x01,0x21,0x01,0xff
-# GFX12: v_cmpx_eq_f16 v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x04,0x7d,0x01,0x21,0x01,0xff]
+# GFX12-REAL16: v_cmpx_eq_f16 v1.l, v2.l row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x04,0x7d,0x01,0x21,0x01,0xff]
+# GFX12-FAKE16: v_cmpx_eq_f16 v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x04,0x7d,0x01,0x21,0x01,0xff]
 
 0xfa,0x04,0x04,0x7d,0x01,0x2f,0x01,0xff
-# GFX12: v_cmpx_eq_f16 v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x04,0x7d,0x01,0x2f,0x01,0xff]
+# GFX12-REAL16: v_cmpx_eq_f16 v1.l, v2.l row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x04,0x7d,0x01,0x2f,0x01,0xff]
+# GFX12-FAKE16: v_cmpx_eq_f16 v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x04,0x7d,0x01,0x2f,0x01,0xff]
 
 0xfa,0x04,0x04,0x7d,0x01,0x50,0x01,0xff
-# GFX12: v_cmpx_eq_f16 v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x04,0x7d,0x01,0x50,0x01,0xff]
+# GFX12-REAL16: v_cmpx_eq_f16 v1.l, v2.l row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x04,0x7d,0x01,0x50,0x01,0xff]
+# GFX12-FAKE16: v_cmpx_eq_f16 v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x04,0x7d,0x01,0x50,0x01,0xff]
 
 0xfa,0x04,0x04,0x7d,0x01,0x5f,0x01,0x01
-# GFX12: v_cmpx_eq_f16 v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x04,0x7d,0x01,0x5f,0x01,0x01]
+# GFX12-REAL16: v_cmpx_eq_f16 v1.l, v2.l row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x04,0x7d,0x01,0x5f,0x01,0x01]
+# GFX12-FAKE16: v_cmpx_eq_f16 v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x04,0x7d,0x01,0x5f,0x01,0x01]
 
 0xfa,0x04,0x04,0x7d,0x01,0x60,0x01,0x13
-# GFX12: v_cmpx_eq_f16 v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x04,0x7d,0x01,0x60,0x01,0x13]
+# GFX12-REAL16: v_cmpx_eq_f16 v1.l, v2.l row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x04,0x7d,0x01,0x60,0x01,0x13]
+# GFX12-FAKE16: v_cmpx_eq_f16 v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x04,0x7d,0x01,0x60,0x01,0x13]
 
 0xfa,0xfe,0x04,0x7d,0x7f,0x6f,0xfd,0x30
-# GFX12: v_cmpx_eq_f16 -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0x04,0x7d,0x7f,0x6f,0xfd,0x30]
+# GFX12-REAL16: v_cmpx_eq_f16 -|v127.l|, -|v127.l| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0x04,0x7d,0x7f,0x6f,0xfd,0x30]
+# GFX12-FAKE16: v_cmpx_eq_f16 -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0x04,0x7d,0x7f,0x6f,0xfd,0x30]
+
+0xfa,0x04,0x05,0x7d,0x81,0x60,0x01,0x13
+# GFX12-REAL16: v_cmpx_eq_f16 v1.h, v2.h row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x05,0x7d,0x81,0x60,0x01,0x13]
+# GFX12-FAKE16: v_cmpx_eq_f16 v129/*Invalid register, operand has 'VGPR_32_Lo128' register class*/, v130/*Invalid register, operand has 'VGPR_32_Lo128' register class*/ row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x05,0x7d,0x81,0x60,0x01,0x13]
+
+0xfa,0xfe,0x05,0x7d,0xff,0x6f,0xfd,0x30
+# GFX12-REAL16: v_cmpx_eq_f16 -|v127.h|, -|v127.h| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0x05,0x7d,0xff,0x6f,0xfd,0x30]
+# GFX12-FAKE16: v_cmpx_eq_f16 -|v255/*Invalid register, operand has 'VGPR_32_Lo128' register class*/|, -|v255/*Invalid register, operand has 'VGPR_32_Lo128' register class*/| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0x05,0x7d,0xff,0x6f,0xfd,0x30]
 
 0xfa,0x04,0x24,0x7d,0x01,0x1b,0x00,0xff
 # GFX12: v_cmpx_eq_f32 v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x24,0x7d,0x01,0x1b,0x00,0xff]
@@ -407,46 +429,68 @@
 # GFX12: v_cmpx_eq_u32 v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0x95,0x7d,0xff,0x6f,0x0d,0x30]
 
 0xfa,0x04,0x0c,0x7d,0x01,0x1b,0x00,0xff
-# GFX12: v_cmpx_ge_f16 v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0c,0x7d,0x01,0x1b,0x00,0xff]
+# GFX12-REAL16: v_cmpx_ge_f16 v1.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0c,0x7d,0x01,0x1b,0x00,0xff]
+# GFX12-FAKE16: v_cmpx_ge_f16 v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0c,0x7d,0x01,0x1b,0x00,0xff]
 
 0xfa,0x04,0x0c,0x7d,0x01,0xe4,0x00,0xff
-# GFX12: v_cmpx_ge_f16 v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0c,0x7d,0x01,0xe4,0x00,0xff]
+# GFX12-REAL16: v_cmpx_ge_f16 v1.l, v2.l quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0c,0x7d,0x01,0xe4,0x00,0xff]
+# GFX12-FAKE16: v_cmpx_ge_f16 v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0c,0x7d,0x01,0xe4,0x00,0xff]
 
 0xfa,0x04,0x0c,0x7d,0x01,0x40,0x01,0xff
-# GFX12: v_cmpx_ge_f16 v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0c,0x7d,0x01,0x40,0x01,0xff]
+# GFX12-REAL16: v_cmpx_ge_f16 v1.l, v2.l row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0c,0x7d,0x01,0x40,0x01,0xff]
+# GFX12-FAKE16: v_cmpx_ge_f16 v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0c,0x7d,0x01,0x40,0x01,0xff]
 
 0xfa,0x04,0x0c,0x7d,0x01,0x41,0x01,0xff
-# GFX12: v_cmpx_ge_f16 v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0c,0x7d,0x01,0x41,0x01,0xff]
+# GFX12-REAL16: v_cmpx_ge_f16 v1.l, v2.l row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0c,0x7d,0x01,0x41,0x01,0xff]
+# GFX12-FAKE16: v_cmpx_ge_f16 v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0c,0x7d,0x01,0x41,0x01,0xff]
 
 0xfa,0x04,0x0c,0x7d,0x01,0x01,0x01,0xff
-# GFX12: v_cmpx_ge_f16 v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0c,0x7d,0x01,0x01,0x01,0xff]
+# GFX12-REAL16: v_cmpx_ge_f16 v1.l, v2.l row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0c,0x7d,0x01,0x01,0x01,0xff]
+# GFX12-FAKE16: v_cmpx_ge_f16 v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0c,0x7d,0x01,0x01,0x01,0xff]
 
 0xfa,0x04,0x0c,0x7d,0x01,0x0f,0x01,0xff
-# GFX12: v_cmpx_ge_f16 v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0c,0x7d,0x01,0x0f,0x01,0xff]
+# GFX12-REAL16: v_cmpx_ge_f16 v1.l, v2.l row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0c,0x7d,0x01,0x0f,0x01,0xff]
+# GFX12-FAKE16: v_cmpx_ge_f16 v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0c,0x7d,0x01,0x0f,0x01,0xff]
 
 0xfa,0x04,0x0c,0x7d,0x01,0x11,0x01,0xff
-# GFX12: v_cmpx_ge_f16 v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0c,0x7d,0x01,0x11,0x01,0xff]
+# GFX12-REAL16: v_cmpx_ge_f16 v1.l, v2.l row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0c,0x7d,0x01,0x11,0x01,0xff]
+# GFX12-FAKE16: v_cmpx_ge_f16 v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0c,0x7d,0x01,0x11,0x01,0xff]
 
 0xfa,0x04,0x0c,0x7d,0x01,0x1f,0x01,0xff
-# GFX12: v_cmpx_ge_f16 v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0c,0x7d,0x01,0x1f,0x01,0xff]
+# GFX12-REAL16: v_cmpx_ge_f16 v1.l, v2.l row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0c,0x7d,0x01,0x1f,0x01,0xff]
+# GFX12-FAKE16: v_cmpx_ge_f16 v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0c,0x7d,0x01,0x1f,0x01,0xff]
 
 0xfa,0x04,0x0c,0x7d,0x01,0x21,0x01,0xff
-# GFX12: v_cmpx_ge_f16 v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0c,0x7d,0x01,0x21,0x01,0xff]
+# GFX12-REAL16: v_cmpx_ge_f16 v1.l, v2.l row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0c,0x7d,0x01,0x21,0x01,0xff]
+# GFX12-FAKE16: v_cmpx_ge_f16 v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0c,0x7d,0x01,0x21,0x01,0xff]
 
 0xfa,0x04,0x0c,0x7d,0x01,0x2f,0x01,0xff
-# GFX12: v_cmpx_ge_f16 v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0c,0x7d,0x01,0x2f,0x01,0xff]
+# GFX12-REAL16: v_cmpx_ge_f16 v1.l, v2.l row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0c,0x7d,0x01,0x2f,0x01,0xff]
+# GFX12-FAKE16: v_cmpx_ge_f16 v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0c,0x7d,0x01,0x2f,0x01,0xff]
 
 0xfa,0x04,0x0c,0x7d,0x01,0x50,0x01,0xff
-# GFX12: v_cmpx_ge_f16 v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0c,0x7d,0x01,0x50,0x01,0xff]
+# GFX12-REAL16: v_cmpx_ge_f16 v1.l, v2.l row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0c,0x7d,0x01,0x50,0x01,0xff]
+# GFX12-FAKE16: v_cmpx_ge_f16 v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0c,0x7d,0x01,0x50,0x01,0xff]
 
 0xfa,0x04,0x0c,0x7d,0x01,0x5f,0x01,0x01
-# GFX12: v_cmpx_ge_f16 v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x0c,0x7d,0x01,0x5f,0x01,0x01]
+# GFX12-REAL16: v_cmpx_ge_f16 v1.l, v2.l row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x0c,0x7d,0x01,0x5f,0x01,0x01]
+# GFX12-FAKE16: v_cmpx_ge_f16 v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x0c,0x7d,0x01,0x5f,0x01,0x01]
 
 0xfa,0x04,0x0c,0x7d,0x01,0x60,0x01,0x13
-# GFX12: v_cmpx_ge_f16 v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x0c,0x7d,0x01,0x60,0x01,0x13]
+# GFX12-REAL16: v_cmpx_ge_f16 v1.l, v2.l row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x0c,0x7d,0x01,0x60,0x01,0x13]
+# GFX12-FAKE16: v_cmpx_ge_f16 v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x0c,0x7d,0x01,0x60,0x01,0x13]
 
 0xfa,0xfe,0x0c,0x7d,0x7f,0x6f,0xfd,0x30
-# GFX12: v_cmpx_ge_f16 -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0x0c,0x7d,0x7f,0x6f,0xfd,0x30]
+# GFX12-REAL16: v_cmpx_ge_f16 -|v127.l|, -|v127.l| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0x0c,0x7d,0x7f,0x6f,0xfd,0x30]
+# GFX12-FAKE16: v_cmpx_ge_f16 -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0x0c,0x7d,0x7f,0x6f,0xfd,0x30]
+
+0xfa,0x04,0x0d,0x7d,0x81,0x60,0x01,0x13
+# GFX12-REAL16: v_cmpx_ge_f16 v1.h, v2.h row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x0d,0x7d,0x81,0x60,0x01,0x13]
+# GFX12-FAKE16: v_cmpx_ge_f16 v129/*Invalid register, operand has 'VGPR_32_Lo128' register class*/, v130/*Invalid register, operand has 'VGPR_32_Lo128' register class*/ row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x0d,0x7d,0x81,0x60,0x01,0x13]
+
+0xfa,0xfe,0x0d,0x7d,0xff,0x6f,0xfd,0x30
+# GFX12-REAL16: v_cmpx_ge_f16 -|v127.h|, -|v127.h| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0x0d,0x7d,0xff,0x6f,0xfd,0x30]
+# GFX12-FAKE16: v_cmpx_ge_f16 -|v255/*Invalid register, operand has 'VGPR_32_Lo128' register class*/|, -|v255/*Invalid register, operand has 'VGPR_32_Lo128' register class*/| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0x0d,0x7d,0xff,0x6f,0xfd,0x30]
 
 0xfa,0x04,0x2c,0x7d,0x01,0x1b,0x00,0xff
 # GFX12: v_cmpx_ge_f32 v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x2c,0x7d,0x01,0x1b,0x00,0xff]
@@ -703,46 +747,68 @@
 # GFX12: v_cmpx_ge_u32 v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0x9d,0x7d,0xff,0x6f,0x0d,0x30]
 
 0xfa,0x04,0x08,0x7d,0x01,0x1b,0x00,0xff
-# GFX12: v_cmpx_gt_f16 v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x08,0x7d,0x01,0x1b,0x00,0xff]
+# GFX12-REAL16: v_cmpx_gt_f16 v1.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x08,0x7d,0x01,0x1b,0x00,0xff]
+# GFX12-FAKE16: v_cmpx_gt_f16 v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x08,0x7d,0x01,0x1b,0x00,0xff]
 
 0xfa,0x04,0x08,0x7d,0x01,0xe4,0x00,0xff
-# GFX12: v_cmpx_gt_f16 v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x08,0x7d,0x01,0xe4,0x00,0xff]
+# GFX12-REAL16: v_cmpx_gt_f16 v1.l, v2.l quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x08,0x7d,0x01,0xe4,0x00,0xff]
+# GFX12-FAKE16: v_cmpx_gt_f16 v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x08,0x7d,0x01,0xe4,0x00,0xff]
 
 0xfa,0x04,0x08,0x7d,0x01,0x40,0x01,0xff
-# GFX12: v_cmpx_gt_f16 v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x08,0x7d,0x01,0x40,0x01,0xff]
+# GFX12-REAL16: v_cmpx_gt_f16 v1.l, v2.l row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x08,0x7d,0x01,0x40,0x01,0xff]
+# GFX12-FAKE16: v_cmpx_gt_f16 v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x08,0x7d,0x01,0x40,0x01,0xff]
 
 0xfa,0x04,0x08,0x7d,0x01,0x41,0x01,0xff
-# GFX12: v_cmpx_gt_f16 v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x08,0x7d,0x01,0x41,0x01,0xff]
+# GFX12-REAL16: v_cmpx_gt_f16 v1.l, v2.l row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x08,0x7d,0x01,0x41,0x01,0xff]
+# GFX12-FAKE16: v_cmpx_gt_f16 v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x08,0x7d,0x01,0x41,0x01,0xff]
 
 0xfa,0x04,0x08,0x7d,0x01,0x01,0x01,0xff
-# GFX12: v_cmpx_gt_f16 v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x08,0x7d,0x01,0x01,0x01,0xff]
+# GFX12-REAL16: v_cmpx_gt_f16 v1.l, v2.l row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x08,0x7d,0x01,0x01,0x01,0xff]
+# GFX12-FAKE16: v_cmpx_gt_f16 v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x08,0x7d,0x01,0x01,0x01,0xff]
 
 0xfa,0x04,0x08,0x7d,0x01,0x0f,0x01,0xff
-# GFX12: v_cmpx_gt_f16 v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x08,0x7d,0x01,0x0f,0x01,0xff]
+# GFX12-REAL16: v_cmpx_gt_f16 v1.l, v2.l row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x08,0x7d,0x01,0x0f,0x01,0xff]
+# GFX12-FAKE16: v_cmpx_gt_f16 v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x08,0x7d,0x01,0x0f,0x01,0xff]
 
 0xfa,0x04,0x08,0x7d,0x01,0x11,0x01,0xff
-# GFX12: v_cmpx_gt_f16 v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x08,0x7d,0x01,0x11,0x01,0xff]
+# GFX12-REAL16: v_cmpx_gt_f16 v1.l, v2.l row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x08,0x7d,0x01,0x11,0x01,0xff]
+# GFX12-FAKE16: v_cmpx_gt_f16 v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x08,0x7d,0x01,0x11,0x01,0xff]
 
 0xfa,0x04,0x08,0x7d,0x01,0x1f,0x01,0xff
-# GFX12: v_cmpx_gt_f16 v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x08,0x7d,0x01,0x1f,0x01,0xff]
+# GFX12-REAL16: v_cmpx_gt_f16 v1.l, v2.l row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x08,0x7d,0x01,0x1f,0x01,0xff]
+# GFX12-FAKE16: v_cmpx_gt_f16 v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x08,0x7d,0x01,0x1f,0x01,0xff]
 
 0xfa,0x04,0x08,0x7d,0x01,0x21,0x01,0xff
-# GFX12: v_cmpx_gt_f16 v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x08,0x7d,0x01,0x21,0x01,0xff]
+# GFX12-REAL16: v_cmpx_gt_f16 v1.l, v2.l row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x08,0x7d,0x01,0x21,0x01,0xff]
+# GFX12-FAKE16: v_cmpx_gt_f16 v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x08,0x7d,0x01,0x21,0x01,0xff]
 
 0xfa,0x04,0x08,0x7d,0x01,0x2f,0x01,0xff
-# GFX12: v_cmpx_gt_f16 v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x08,0x7d,0x01,0x2f,0x01,0xff]
+# GFX12-REAL16: v_cmpx_gt_f16 v1.l, v2.l row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x08,0x7d,0x01,0x2f,0x01,0xff]
+# GFX12-FAKE16: v_cmpx_gt_f16 v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x08,0x7d,0x01,0x2f,0x01,0xff]
 
 0xfa,0x04,0x08,0x7d,0x01,0x50,0x01,0xff
-# GFX12: v_cmpx_gt_f16 v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x08,0x7d,0x01,0x50,0x01,0xff]
+# GFX12-REAL16: v_cmpx_gt_f16 v1.l, v2.l row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x08,0x7d,0x01,0x50,0x01,0xff]
+# GFX12-FAKE16: v_cmpx_gt_f16 v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x08,0x7d,0x01,0x50,0x01,0xff]
 
 0xfa,0x04,0x08,0x7d,0x01,0x5f,0x01,0x01
-# GFX12: v_cmpx_gt_f16 v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x08,0x7d,0x01,0x5f,0x01,0x01]
+# GFX12-REAL16: v_cmpx_gt_f16 v1.l, v2.l row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x08,0x7d,0x01,0x5f,0x01,0x01]
+# GFX12-FAKE16: v_cmpx_gt_f16 v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x08,0x7d,0x01,0x5f,0x01,0x01]
 
 0xfa,0x04,0x08,0x7d,0x01,0x60,0x01,0x13
-# GFX12: v_cmpx_gt_f16 v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x08,0x7d,0x01,0x60,0x01,0x13]
+# GFX12-REAL16: v_cmpx_gt_f16 v1.l, v2.l row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x08,0x7d,0x01,0x60,0x01,0x13]
+# GFX12-FAKE16: v_cmpx_gt_f16 v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x08,0x7d,0x01,0x60,0x01,0x13]
 
 0xfa,0xfe,0x08,0x7d,0x7f,0x6f,0xfd,0x30
-# GFX12: v_cmpx_gt_f16 -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0x08,0x7d,0x7f,0x6f,0xfd,0x30]
+# GFX12-REAL16: v_cmpx_gt_f16 -|v127.l|, -|v127.l| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0x08,0x7d,0x7f,0x6f,0xfd,0x30]
+# GFX12-FAKE16: v_cmpx_gt_f16 -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0x08,0x7d,0x7f,0x6f,0xfd,0x30]
+
+0xfa,0x04,0x09,0x7d,0x81,0x60,0x01,0x13
+# GFX12-REAL16: v_cmpx_gt_f16 v1.h, v2.h row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x09,0x7d,0x81,0x60,0x01,0x13]
+# GFX12-FAKE16: v_cmpx_gt_f16 v129/*Invalid register, operand has 'VGPR_32_Lo128' register class*/, v130/*Invalid register, operand has 'VGPR_32_Lo128' register class*/ row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x09,0x7d,0x81,0x60,0x01,0x13]
+
+0xfa,0xfe,0x09,0x7d,0xff,0x6f,0xfd,0x30
+# GFX12-REAL16: v_cmpx_gt_f16 -|v127.h|, -|v127.h| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0x09,0x7d,0xff,0x6f,0xfd,0x30]
+# GFX12-FAKE16: v_cmpx_gt_f16 -|v255/*Invalid register, operand has 'VGPR_32_Lo128' register class*/|, -|v255/*Invalid register, operand has 'VGPR_32_Lo128' register class*/| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0x09,0x7d,0xff,0x6f,0xfd,0x30]
 
 0xfa,0x04,0x28,0x7d,0x01,0x1b,0x00,0xff
 # GFX12: v_cmpx_gt_f32 v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x28,0x7d,0x01,0x1b,0x00,0xff]
@@ -999,46 +1065,68 @@
 # GFX12: v_cmpx_gt_u32 v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0x99,0x7d,0xff,0x6f,0x0d,0x30]
 
 0xfa,0x04,0x06,0x7d,0x01,0x1b,0x00,0xff
-# GFX12: v_cmpx_le_f16 v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x06,0x7d,0x01,0x1b,0x00,0xff]
+# GFX12-REAL16: v_cmpx_le_f16 v1.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x06,0x7d,0x01,0x1b,0x00,0xff]
+# GFX12-FAKE16: v_cmpx_le_f16 v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x06,0x7d,0x01,0x1b,0x00,0xff]
 
 0xfa,0x04,0x06,0x7d,0x01,0xe4,0x00,0xff
-# GFX12: v_cmpx_le_f16 v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x06,0x7d,0x01,0xe4,0x00,0xff]
+# GFX12-REAL16: v_cmpx_le_f16 v1.l, v2.l quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x06,0x7d,0x01,0xe4,0x00,0xff]
+# GFX12-FAKE16: v_cmpx_le_f16 v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x06,0x7d,0x01,0xe4,0x00,0xff]
 
 0xfa,0x04,0x06,0x7d,0x01,0x40,0x01,0xff
-# GFX12: v_cmpx_le_f16 v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x06,0x7d,0x01,0x40,0x01,0xff]
+# GFX12-REAL16: v_cmpx_le_f16 v1.l, v2.l row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x06,0x7d,0x01,0x40,0x01,0xff]
+# GFX12-FAKE16: v_cmpx_le_f16 v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x06,0x7d,0x01,0x40,0x01,0xff]
 
 0xfa,0x04,0x06,0x7d,0x01,0x41,0x01,0xff
-# GFX12: v_cmpx_le_f16 v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x06,0x7d,0x01,0x41,0x01,0xff]
+# GFX12-REAL16: v_cmpx_le_f16 v1.l, v2.l row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x06,0x7d,0x01,0x41,0x01,0xff]
+# GFX12-FAKE16: v_cmpx_le_f16 v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x06,0x7d,0x01,0x41,0x01,0xff]
 
 0xfa,0x04,0x06,0x7d,0x01,0x01,0x01,0xff
-# GFX12: v_cmpx_le_f16 v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x06,0x7d,0x01,0x01,0x01,0xff]
+# GFX12-REAL16: v_cmpx_le_f16 v1.l, v2.l row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x06,0x7d,0x01,0x01,0x01,0xff]
+# GFX12-FAKE16: v_cmpx_le_f16 v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x06,0x7d,0x01,0x01,0x01,0xff]
 
 0xfa,0x04,0x06,0x7d,0x01,0x0f,0x01,0xff
-# GFX12: v_cmpx_le_f16 v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x06,0x7d,0x01,0x0f,0x01,0xff]
+# GFX12-REAL16: v_cmpx_le_f16 v1.l, v2.l row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x06,0x7d,0x01,0x0f,0x01,0xff]
+# GFX12-FAKE16: v_cmpx_le_f16 v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x06,0x7d,0x01,0x0f,0x01,0xff]
 
 0xfa,0x04,0x06,0x7d,0x01,0x11,0x01,0xff
-# GFX12: v_cmpx_le_f16 v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x06,0x7d,0x01,0x11,0x01,0xff]
+# GFX12-REAL16: v_cmpx_le_f16 v1.l, v2.l row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x06,0x7d,0x01,0x11,0x01,0xff]
+# GFX12-FAKE16: v_cmpx_le_f16 v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x06,0x7d,0x01,0x11,0x01,0xff]
 
 0xfa,0x04,0x06,0x7d,0x01,0x1f,0x01,0xff
-# GFX12: v_cmpx_le_f16 v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x06,0x7d,0x01,0x1f,0x01,0xff]
+# GFX12-REAL16: v_cmpx_le_f16 v1.l, v2.l row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x06,0x7d,0x01,0x1f,0x01,0xff]
+# GFX12-FAKE16: v_cmpx_le_f16 v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x06,0x7d,0x01,0x1f,0x01,0xff]
 
 0xfa,0x04,0x06,0x7d,0x01,0x21,0x01,0xff
-# GFX12: v_cmpx_le_f16 v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x06,0x7d,0x01,0x21,0x01,0xff]
+# GFX12-REAL16: v_cmpx_le_f16 v1.l, v2.l row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x06,0x7d,0x01,0x21,0x01,0xff]
+# GFX12-FAKE16: v_cmpx_le_f16 v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x06,0x7d,0x01,0x21,0x01,0xff]
 
 0xfa,0x04,0x06,0x7d,0x01,0x2f,0x01,0xff
-# GFX12: v_cmpx_le_f16 v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x06,0x7d,0x01,0x2f,0x01,0xff]
+# GFX12-REAL16: v_cmpx_le_f16 v1.l, v2.l row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x06,0x7d,0x01,0x2f,0x01,0xff]
+# GFX12-FAKE16: v_cmpx_le_f16 v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x06,0x7d,0x01,0x2f,0x01,0xff]
 
 0xfa,0x04,0x06,0x7d,0x01,0x50,0x01,0xff
-# GFX12: v_cmpx_le_f16 v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x06,0x7d,0x01,0x50,0x01,0xff]
+# GFX12-REAL16: v_cmpx_le_f16 v1.l, v2.l row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x06,0x7d,0x01,0x50,0x01,0xff]
+# GFX12-FAKE16: v_cmpx_le_f16 v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x06,0x7d,0x01,0x50,0x01,0xff]
 
 0xfa,0x04,0x06,0x7d,0x01,0x5f,0x01,0x01
-# GFX12: v_cmpx_le_f16 v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x06,0x7d,0x01,0x5f,0x01,0x01]
+# GFX12-REAL16: v_cmpx_le_f16 v1.l, v2.l row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x06,0x7d,0x01,0x5f,0x01,0x01]
+# GFX12-FAKE16: v_cmpx_le_f16 v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x06,0x7d,0x01,0x5f,0x01,0x01]
 
 0xfa,0x04,0x06,0x7d,0x01,0x60,0x01,0x13
-# GFX12: v_cmpx_le_f16 v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x06,0x7d,0x01,0x60,0x01,0x13]
+# GFX12-REAL16: v_cmpx_le_f16 v1.l, v2.l row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x06,0x7d,0x01,0x60,0x01,0x13]
+# GFX12-FAKE16: v_cmpx_le_f16 v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x06,0x7d,0x01,0x60,0x01,0x13]
 
 0xfa,0xfe,0x06,0x7d,0x7f,0x6f,0xfd,0x30
-# GFX12: v_cmpx_le_f16 -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0x06,0x7d,0x7f,0x6f,0xfd,0x30]
+# GFX12-REAL16: v_cmpx_le_f16 -|v127.l|, -|v127.l| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0x06,0x7d,0x7f,0x6f,0xfd,0x30]
+# GFX12-FAKE16: v_cmpx_le_f16 -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0x06,0x7d,0x7f,0x6f,0xfd,0x30]
+
+0xfa,0x04,0x07,0x7d,0x81,0x60,0x01,0x13
+# GFX12-REAL16: v_cmpx_le_f16 v1.h, v2.h row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x07,0x7d,0x81,0x60,0x01,0x13]
+# GFX12-FAKE16: v_cmpx_le_f16 v129/*Invalid register, operand has 'VGPR_32_Lo128' register class*/, v130/*Invalid register, operand has 'VGPR_32_Lo128' register class*/ row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x07,0x7d,0x81,0x60,0x01,0x13]
+
+0xfa,0xfe,0x07,0x7d,0xff,0x6f,0xfd,0x30
+# GFX12-REAL16: v_cmpx_le_f16 -|v127.h|, -|v127.h| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0x07,0x7d,0xff,0x6f,0xfd,0x30]
+# GFX12-FAKE16: v_cmpx_le_f16 -|v255/*Invalid register, operand has 'VGPR_32_Lo128' register class*/|, -|v255/*Invalid register, operand has 'VGPR_32_Lo128' register class*/| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0x07,0x7d,0xff,0x6f,0xfd,0x30]
 
 0xfa,0x04,0x26,0x7d,0x01,0x1b,0x00,0xff
 # GFX12: v_cmpx_le_f32 v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x26,0x7d,0x01,0x1b,0x00,0xff]
@@ -1295,46 +1383,68 @@
 # GFX12: v_cmpx_le_u32 v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0x97,0x7d,0xff,0x6f,0x0d,0x30]
 
 0xfa,0x04,0x0a,0x7d,0x01,0x1b,0x00,0xff
-# GFX12: v_cmpx_lg_f16 v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x7d,0x01,0x1b,0x00,0xff]
+# GFX12-REAL16: v_cmpx_lg_f16 v1.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x7d,0x01,0x1b,0x00,0xff]
+# GFX12-FAKE16: v_cmpx_lg_f16 v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x7d,0x01,0x1b,0x00,0xff]
 
 0xfa,0x04,0x0a,0x7d,0x01,0xe4,0x00,0xff
-# GFX12: v_cmpx_lg_f16 v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x7d,0x01,0xe4,0x00,0xff]
+# GFX12-REAL16: v_cmpx_lg_f16 v1.l, v2.l quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x7d,0x01,0xe4,0x00,0xff]
+# GFX12-FAKE16: v_cmpx_lg_f16 v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x7d,0x01,0xe4,0x00,0xff]
 
 0xfa,0x04,0x0a,0x7d,0x01,0x40,0x01,0xff
-# GFX12: v_cmpx_lg_f16 v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x7d,0x01,0x40,0x01,0xff]
+# GFX12-REAL16: v_cmpx_lg_f16 v1.l, v2.l row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x7d,0x01,0x40,0x01,0xff]
+# GFX12-FAKE16: v_cmpx_lg_f16 v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x7d,0x01,0x40,0x01,0xff]
 
 0xfa,0x04,0x0a,0x7d,0x01,0x41,0x01,0xff
-# GFX12: v_cmpx_lg_f16 v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x7d,0x01,0x41,0x01,0xff]
+# GFX12-REAL16: v_cmpx_lg_f16 v1.l, v2.l row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x7d,0x01,0x41,0x01,0xff]
+# GFX12-FAKE16: v_cmpx_lg_f16 v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x7d,0x01,0x41,0x01,0xff]
 
 0xfa,0x04,0x0a,0x7d,0x01,0x01,0x01,0xff
-# GFX12: v_cmpx_lg_f16 v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x7d,0x01,0x01,0x01,0xff]
+# GFX12-REAL16: v_cmpx_lg_f16 v1.l, v2.l row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x7d,0x01,0x01,0x01,0xff]
+# GFX12-FAKE16: v_cmpx_lg_f16 v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x7d,0x01,0x01,0x01,0xff]
 
 0xfa,0x04,0x0a,0x7d,0x01,0x0f,0x01,0xff
-# GFX12: v_cmpx_lg_f16 v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x7d,0x01,0x0f,0x01,0xff]
+# GFX12-REAL16: v_cmpx_lg_f16 v1.l, v2.l row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x7d,0x01,0x0f,0x01,0xff]
+# GFX12-FAKE16: v_cmpx_lg_f16 v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x7d,0x01,0x0f,0x01,0xff]
 
 0xfa,0x04,0x0a,0x7d,0x01,0x11,0x01,0xff
-# GFX12: v_cmpx_lg_f16 v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x7d,0x01,0x11,0x01,0xff]
+# GFX12-REAL16: v_cmpx_lg_f16 v1.l, v2.l row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x7d,0x01,0x11,0x01,0xff]
+# GFX12-FAKE16: v_cmpx_lg_f16 v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x7d,0x01,0x11,0x01,0xff]
 
 0xfa,0x04,0x0a,0x7d,0x01,0x1f,0x01,0xff
-# GFX12: v_cmpx_lg_f16 v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x7d,0x01,0x1f,0x01,0xff]
+# GFX12-REAL16: v_cmpx_lg_f16 v1.l, v2.l row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x7d,0x01,0x1f,0x01,0xff]
+# GFX12-FAKE16: v_cmpx_lg_f16 v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x7d,0x01,0x1f,0x01,0xff]
 
 0xfa,0x04,0x0a,0x7d,0x01,0x21,0x01,0xff
-# GFX12: v_cmpx_lg_f16 v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x7d,0x01,0x21,0x01,0xff]
+# GFX12-REAL16: v_cmpx_lg_f16 v1.l, v2.l row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x7d,0x01,0x21,0x01,0xff]
+# GFX12-FAKE16: v_cmpx_lg_f16 v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x7d,0x01,0x21,0x01,0xff]
 
 0xfa,0x04,0x0a,0x7d,0x01,0x2f,0x01,0xff
-# GFX12: v_cmpx_lg_f16 v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x7d,0x01,0x2f,0x01,0xff]
+# GFX12-REAL16: v_cmpx_lg_f16 v1.l, v2.l row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x7d,0x01,0x2f,0x01,0xff]
+# GFX12-FAKE16: v_cmpx_lg_f16 v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x7d,0x01,0x2f,0x01,0xff]
 
 0xfa,0x04,0x0a,0x7d,0x01,0x50,0x01,0xff
-# GFX12: v_cmpx_lg_f16 v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x7d,0x01,0x50,0x01,0xff]
+# GFX12-REAL16: v_cmpx_lg_f16 v1.l, v2.l row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x7d,0x01,0x50,0x01,0xff]
+# GFX12-FAKE16: v_cmpx_lg_f16 v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x7d,0x01,0x50,0x01,0xff]
 
 0xfa,0x04,0x0a,0x7d,0x01,0x5f,0x01,0x01
-# GFX12: v_cmpx_lg_f16 v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x0a,0x7d,0x01,0x5f,0x01,0x01]
+# GFX12-REAL16: v_cmpx_lg_f16 v1.l, v2.l row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x0a,0x7d,0x01,0x5f,0x01,0x01]
+# GFX12-FAKE16: v_cmpx_lg_f16 v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x0a,0x7d,0x01,0x5f,0x01,0x01]
 
 0xfa,0x04,0x0a,0x7d,0x01,0x60,0x01,0x13
-# GFX12: v_cmpx_lg_f16 v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x0a,0x7d,0x01,0x60,0x01,0x13]
+# GFX12-REAL16: v_cmpx_lg_f16 v1.l, v2.l row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x0a,0x7d,0x01,0x60,0x01,0x13]
+# GFX12-FAKE16: v_cmpx_lg_f16 v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x0a,0x7d,0x01,0x60,0x01,0x13]
 
 0xfa,0xfe,0x0a,0x7d,0x7f,0x6f,0xfd,0x30
-# GFX12: v_cmpx_lg_f16 -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0x0a,0x7d,0x7f,0x6f,0xfd,0x30]
+# GFX12-REAL16: v_cmpx_lg_f16 -|v127.l|, -|v127.l| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0x0a,0x7d,0x7f,0x6f,0xfd,0x30]
+# GFX12-FAKE16: v_cmpx_lg_f16 -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0x0a,0x7d,0x7f,0x6f,0xfd,0x30]
+
+0xfa,0x04,0x0b,0x7d,0x81,0x60,0x01,0x13
+# GFX12-REAL16: v_cmpx_lg_f16 v1.h, v2.h row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x0b,0x7d,0x81,0x60,0x01,0x13]
+# GFX12-FAKE16: v_cmpx_lg_f16 v129/*Invalid register, operand has 'VGPR_32_Lo128' register class*/, v130/*Invalid register, operand has 'VGPR_32_Lo128' register class*/ row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x0b,0x7d,0x81,0x60,0x01,0x13]
+
+0xfa,0xfe,0x0b,0x7d,0xff,0x6f,0xfd,0x30
+# GFX12-REAL16: v_cmpx_lg_f16 -|v127.h|, -|v127.h| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0x0b,0x7d,0xff,0x6f,0xfd,0x30]
+# GFX12-FAKE16: v_cmpx_lg_f16 -|v255/*Invalid register, operand has 'VGPR_32_Lo128' register class*/|, -|v255/*Invalid register, operand has 'VGPR_32_Lo128' register class*/| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0x0b,0x7d,0xff,0x6f,0xfd,0x30]
 
 0xfa,0x04,0x2a,0x7d,0x01,0x1b,0x00,0xff
 # GFX12: v_cmpx_lg_f32 v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x2a,0x7d,0x01,0x1b,0x00,0xff]
@@ -1909,46 +2019,68 @@
 # GFX12: v_cmpx_ne_u32 v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0x9b,0x7d,0xff,0x6f,0x0d,0x30]
 
 0xfa,0x04,0x1a,0x7d,0x01,0x1b,0x00,0xff
-# GFX12: v_cmpx_neq_f16 v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1a,0x7d,0x01,0x1b,0x00,0xff]
+# GFX12-REAL16: v_cmpx_neq_f16 v1.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1a,0x7d,0x01,0x1b,0x00,0xff]
+# GFX12-FAKE16: v_cmpx_neq_f16 v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1a,0x7d,0x01,0x1b,0x00,0xff]
 
 0xfa,0x04,0x1a,0x7d,0x01,0xe4,0x00,0xff
-# GFX12: v_cmpx_neq_f16 v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1a,0x7d,0x01,0xe4,0x00,0xff]
+# GFX12-REAL16: v_cmpx_neq_f16 v1.l, v2.l quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1a,0x7d,0x01,0xe4,0x00,0xff]
+# GFX12-FAKE16: v_cmpx_neq_f16 v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1a,0x7d,0x01,0xe4,0x00,0xff]
 
 0xfa,0x04,0x1a,0x7d,0x01,0x40,0x01,0xff
-# GFX12: v_cmpx_neq_f16 v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1a,0x7d,0x01,0x40,0x01,0xff]
+# GFX12-REAL16: v_cmpx_neq_f16 v1.l, v2.l row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1a,0x7d,0x01,0x40,0x01,0xff]
+# GFX12-FAKE16: v_cmpx_neq_f16 v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1a,0x7d,0x01,0x40,0x01,0xff]
 
 0xfa,0x04,0x1a,0x7d,0x01,0x41,0x01,0xff
-# GFX12: v_cmpx_neq_f16 v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1a,0x7d,0x01,0x41,0x01,0xff]
+# GFX12-REAL16: v_cmpx_neq_f16 v1.l, v2.l row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1a,0x7d,0x01,0x41,0x01,0xff]
+# GFX12-FAKE16: v_cmpx_neq_f16 v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1a,0x7d,0x01,0x41,0x01,0xff]
 
 0xfa,0x04,0x1a,0x7d,0x01,0x01,0x01,0xff
-# GFX12: v_cmpx_neq_f16 v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1a,0x7d,0x01,0x01,0x01,0xff]
+# GFX12-REAL16: v_cmpx_neq_f16 v1.l, v2.l row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1a,0x7d,0x01,0x01,0x01,0xff]
+# GFX12-FAKE16: v_cmpx_neq_f16 v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1a,0x7d,0x01,0x01,0x01,0xff]
 
 0xfa,0x04,0x1a,0x7d,0x01,0x0f,0x01,0xff
-# GFX12: v_cmpx_neq_f16 v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1a,0x7d,0x01,0x0f,0x01,0xff]
+# GFX12-REAL16: v_cmpx_neq_f16 v1.l, v2.l row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1a,0x7d,0x01,0x0f,0x01,0xff]
+# GFX12-FAKE16: v_cmpx_neq_f16 v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1a,0x7d,0x01,0x0f,0x01,0xff]
 
 0xfa,0x04,0x1a,0x7d,0x01,0x11,0x01,0xff
-# GFX12: v_cmpx_neq_f16 v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1a,0x7d,0x01,0x11,0x01,0xff]
+# GFX12-REAL16: v_cmpx_neq_f16 v1.l, v2.l row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1a,0x7d,0x01,0x11,0x01,0xff]
+# GFX12-FAKE16: v_cmpx_neq_f16 v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1a,0x7d,0x01,0x11,0x01,0xff]
 
 0xfa,0x04,0x1a,0x7d,0x01,0x1f,0x01,0xff
-# GFX12: v_cmpx_neq_f16 v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1a,0x7d,0x01,0x1f,0x01,0xff]
+# GFX12-REAL16: v_cmpx_neq_f16 v1.l, v2.l row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1a,0x7d,0x01,0x1f,0x01,0xff]
+# GFX12-FAKE16: v_cmpx_neq_f16 v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1a,0x7d,0x01,0x1f,0x01,0xff]
 
 0xfa,0x04,0x1a,0x7d,0x01,0x21,0x01,0xff
-# GFX12: v_cmpx_neq_f16 v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1a,0x7d,0x01,0x21,0x01,0xff]
+# GFX12-REAL16: v_cmpx_neq_f16 v1.l, v2.l row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1a,0x7d,0x01,0x21,0x01,0xff]
+# GFX12-FAKE16: v_cmpx_neq_f16 v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1a,0x7d,0x01,0x21,0x01,0xff]
 
 0xfa,0x04,0x1a,0x7d,0x01,0x2f,0x01,0xff
-# GFX12: v_cmpx_neq_f16 v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1a,0x7d,0x01,0x2f,0x01,0xff]
+# GFX12-REAL16: v_cmpx_neq_f16 v1.l, v2.l row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1a,0x7d,0x01,0x2f,0x01,0xff]
+# GFX12-FAKE16: v_cmpx_neq_f16 v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1a,0x7d,0x01,0x2f,0x01,0xff]
 
 0xfa,0x04,0x1a,0x7d,0x01,0x50,0x01,0xff
-# GFX12: v_cmpx_neq_f16 v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1a,0x7d,0x01,0x50,0x01,0xff]
+# GFX12-REAL16: v_cmpx_neq_f16 v1.l, v2.l row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1a,0x7d,0x01,0x50,0x01,0xff]
+# GFX12-FAKE16: v_cmpx_neq_f16 v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1a,0x7d,0x01,0x50,0x01,0xff]
 
 0xfa,0x04,0x1a,0x7d,0x01,0x5f,0x01,0x01
-# GFX12: v_cmpx_neq_f16 v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x1a,0x7d,0x01,0x5f,0x01,0x01]
+# GFX12-REAL16: v_cmpx_neq_f16 v1.l, v2.l row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x1a,0x7d,0x01,0x5f,0x01,0x01]
+# GFX12-FAKE16: v_cmpx_neq_f16 v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x1a,0x7d,0x01,0x5f,0x01,0x01]
 
 0xfa,0x04,0x1a,0x7d,0x01,0x60,0x01,0x13
-# GFX12: v_cmpx_neq_f16 v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x1a,0x7d,0x01,0x60,0x01,0x13]
+# GFX12-REAL16: v_cmpx_neq_f16 v1.l, v2.l row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x1a,0x7d,0x01,0x60,0x01,0x13]
+# GFX12-FAKE16: v_cmpx_neq_f16 v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x1a,0x7d,0x01,0x60,0x01,0x13]
 
 0xfa,0xfe,0x1a,0x7d,0x7f,0x6f,0xfd,0x30
-# GFX12: v_cmpx_neq_f16 -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0x1a,0x7d,0x7f,0x6f,0xfd,0x30]
+# GFX12-REAL16: v_cmpx_neq_f16 -|v127.l|, -|v127.l| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0x1a,0x7d,0x7f,0x6f,0xfd,0x30]
+# GFX12-FAKE16: v_cmpx_neq_f16 -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0x1a,0x7d,0x7f,0x6f,0xfd,0x30]
+
+0xfa,0x04,0x1b,0x7d,0x81,0x60,0x01,0x13
+# GFX12-REAL16: v_cmpx_neq_f16 v1.h, v2.h row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x1b,0x7d,0x81,0x60,0x01,0x13]
+# GFX12-FAKE16: v_cmpx_neq_f16 v129/*Invalid register, operand has 'VGPR_32_Lo128' register class*/, v130/*Invalid register, operand has 'VGPR_32_Lo128' register class*/ row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x1b,0x7d,0x81,0x60,0x01,0x13]
+
+0xfa,0xfe,0x1b,0x7d,0xff,0x6f,0xfd,0x30
+# GFX12-REAL16: v_cmpx_neq_f16 -|v127.h|, -|v127.h| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0x1b,0x7d,0xff,0x6f,0xfd,0x30]
+# GFX12-FAKE16: v_cmpx_neq_f16 -|v255/*Invalid register, operand has 'VGPR_32_Lo128' register class*/|, -|v255/*Invalid register, operand has 'VGPR_32_Lo128' register class*/| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0x1b,0x7d,0xff,0x6f,0xfd,0x30]
 
 0xfa,0x04,0x3a,0x7d,0x01,0x1b,0x00,0xff
 # GFX12: v_cmpx_neq_f32 v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x3a,0x7d,0x01,0x1b,0x00,0xff]
@@ -1993,46 +2125,68 @@
 # GFX12: v_cmpx_neq_f32 -|v255|, -|v255| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0x3b,0x7d,0xff,0x6f,0xfd,0x30]
 
 0xfa,0x04,0x12,0x7d,0x01,0x1b,0x00,0xff
-# GFX12: v_cmpx_nge_f16 v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x12,0x7d,0x01,0x1b,0x00,0xff]
+# GFX12-REAL16: v_cmpx_nge_f16 v1.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x12,0x7d,0x01,0x1b,0x00,0xff]
+# GFX12-FAKE16: v_cmpx_nge_f16 v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x12,0x7d,0x01,0x1b,0x00,0xff]
 
 0xfa,0x04,0x12,0x7d,0x01,0xe4,0x00,0xff
-# GFX12: v_cmpx_nge_f16 v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x12,0x7d,0x01,0xe4,0x00,0xff]
+# GFX12-REAL16: v_cmpx_nge_f16 v1.l, v2.l quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x12,0x7d,0x01,0xe4,0x00,0xff]
+# GFX12-FAKE16: v_cmpx_nge_f16 v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x12,0x7d,0x01,0xe4,0x00,0xff]
 
 0xfa,0x04,0x12,0x7d,0x01,0x40,0x01,0xff
-# GFX12: v_cmpx_nge_f16 v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x12,0x7d,0x01,0x40,0x01,0xff]
+# GFX12-REAL16: v_cmpx_nge_f16 v1.l, v2.l row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x12,0x7d,0x01,0x40,0x01,0xff]
+# GFX12-FAKE16: v_cmpx_nge_f16 v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x12,0x7d,0x01,0x40,0x01,0xff]
 
 0xfa,0x04,0x12,0x7d,0x01,0x41,0x01,0xff
-# GFX12: v_cmpx_nge_f16 v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x12,0x7d,0x01,0x41,0x01,0xff]
+# GFX12-REAL16: v_cmpx_nge_f16 v1.l, v2.l row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x12,0x7d,0x01,0x41,0x01,0xff]
+# GFX12-FAKE16: v_cmpx_nge_f16 v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x12,0x7d,0x01,0x41,0x01,0xff]
 
 0xfa,0x04,0x12,0x7d,0x01,0x01,0x01,0xff
-# GFX12: v_cmpx_nge_f16 v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x12,0x7d,0x01,0x01,0x01,0xff]
+# GFX12-REAL16: v_cmpx_nge_f16 v1.l, v2.l row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x12,0x7d,0x01,0x01,0x01,0xff]
+# GFX12-FAKE16: v_cmpx_nge_f16 v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x12,0x7d,0x01,0x01,0x01,0xff]
 
 0xfa,0x04,0x12,0x7d,0x01,0x0f,0x01,0xff
-# GFX12: v_cmpx_nge_f16 v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x12,0x7d,0x01,0x0f,0x01,0xff]
+# GFX12-REAL16: v_cmpx_nge_f16 v1.l, v2.l row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x12,0x7d,0x01,0x0f,0x01,0xff]
+# GFX12-FAKE16: v_cmpx_nge_f16 v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x12,0x7d,0x01,0x0f,0x01,0xff]
 
 0xfa,0x04,0x12,0x7d,0x01,0x11,0x01,0xff
-# GFX12: v_cmpx_nge_f16 v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x12,0x7d,0x01,0x11,0x01,0xff]
+# GFX12-REAL16: v_cmpx_nge_f16 v1.l, v2.l row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x12,0x7d,0x01,0x11,0x01,0xff]
+# GFX12-FAKE16: v_cmpx_nge_f16 v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x12,0x7d,0x01,0x11,0x01,0xff]
 
 0xfa,0x04,0x12,0x7d,0x01,0x1f,0x01,0xff
-# GFX12: v_cmpx_nge_f16 v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x12,0x7d,0x01,0x1f,0x01,0xff]
+# GFX12-REAL16: v_cmpx_nge_f16 v1.l, v2.l row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x12,0x7d,0x01,0x1f,0x01,0xff]
+# GFX12-FAKE16: v_cmpx_nge_f16 v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x12,0x7d,0x01,0x1f,0x01,0xff]
 
 0xfa,0x04,0x12,0x7d,0x01,0x21,0x01,0xff
-# GFX12: v_cmpx_nge_f16 v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x12,0x7d,0x01,0x21,0x01,0xff]
+# GFX12-REAL16: v_cmpx_nge_f16 v1.l, v2.l row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x12,0x7d,0x01,0x21,0x01,0xff]
+# GFX12-FAKE16: v_cmpx_nge_f16 v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x12,0x7d,0x01,0x21,0x01,0xff]
 
 0xfa,0x04,0x12,0x7d,0x01,0x2f,0x01,0xff
-# GFX12: v_cmpx_nge_f16 v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x12,0x7d,0x01,0x2f,0x01,0xff]
+# GFX12-REAL16: v_cmpx_nge_f16 v1.l, v2.l row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x12,0x7d,0x01,0x2f,0x01,0xff]
+# GFX12-FAKE16: v_cmpx_nge_f16 v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x12,0x7d,0x01,0x2f,0x01,0xff]
 
 0xfa,0x04,0x12,0x7d,0x01,0x50,0x01,0xff
-# GFX12: v_cmpx_nge_f16 v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x12,0x7d,0x01,0x50,0x01,0xff]
+# GFX12-REAL16: v_cmpx_nge_f16 v1.l, v2.l row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x12,0x7d,0x01,0x50,0x01,0xff]
+# GFX12-FAKE16: v_cmpx_nge_f16 v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x12,0x7d,0x01,0x50,0x01,0xff]
 
 0xfa,0x04,0x12,0x7d,0x01,0x5f,0x01,0x01
-# GFX12: v_cmpx_nge_f16 v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x12,0x7d,0x01,0x5f,0x01,0x01]
+# GFX12-REAL16: v_cmpx_nge_f16 v1.l, v2.l row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x12,0x7d,0x01,0x5f,0x01,0x01]
+# GFX12-FAKE16: v_cmpx_nge_f16 v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x12,0x7d,0x01,0x5f,0x01,0x01]
 
 0xfa,0x04,0x12,0x7d,0x01,0x60,0x01,0x13
-# GFX12: v_cmpx_nge_f16 v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x12,0x7d,0x01,0x60,0x01,0x13]
+# GFX12-REAL16: v_cmpx_nge_f16 v1.l, v2.l row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x12,0x7d,0x01,0x60,0x01,0x13]
+# GFX12-FAKE16: v_cmpx_nge_f16 v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x12,0x7d,0x01,0x60,0x01,0x13]
 
 0xfa,0xfe,0x12,0x7d,0x7f,0x6f,0xfd,0x30
-# GFX12: v_cmpx_nge_f16 -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0x12,0x7d,0x7f,0x6f,0xfd,0x30]
+# GFX12-REAL16: v_cmpx_nge_f16 -|v127.l|, -|v127.l| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0x12,0x7d,0x7f,0x6f,0xfd,0x30]
+# GFX12-FAKE16: v_cmpx_nge_f16 -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0x12,0x7d,0x7f,0x6f,0xfd,0x30]
+
+0xfa,0x04,0x13,0x7d,0x81,0x60,0x01,0x13
+# GFX12-REAL16: v_cmpx_nge_f16 v1.h, v2.h row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x13,0x7d,0x81,0x60,0x01,0x13]
+# GFX12-FAKE16: v_cmpx_nge_f16 v129/*Invalid register, operand has 'VGPR_32_Lo128' register class*/, v130/*Invalid register, operand has 'VGPR_32_Lo128' register class*/ row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x13,0x7d,0x81,0x60,0x01,0x13]
+
+0xfa,0xfe,0x13,0x7d,0xff,0x6f,0xfd,0x30
+# GFX12-REAL16: v_cmpx_nge_f16 -|v127.h|, -|v127.h| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0x13,0x7d,0xff,0x6f,0xfd,0x30]
+# GFX12-FAKE16: v_cmpx_nge_f16 -|v255/*Invalid register, operand has 'VGPR_32_Lo128' register class*/|, -|v255/*Invalid register, operand has 'VGPR_32_Lo128' register class*/| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0x13,0x7d,0xff,0x6f,0xfd,0x30]
 
 0xfa,0x04,0x32,0x7d,0x01,0x1b,0x00,0xff
 # GFX12: v_cmpx_nge_f32 v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x32,0x7d,0x01,0x1b,0x00,0xff]
@@ -2077,46 +2231,68 @@
 # GFX12: v_cmpx_nge_f32 -|v255|, -|v255| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0x33,0x7d,0xff,0x6f,0xfd,0x30]
 
 0xfa,0x04,0x16,0x7d,0x01,0x1b,0x00,0xff
-# GFX12: v_cmpx_ngt_f16 v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x16,0x7d,0x01,0x1b,0x00,0xff]
+# GFX12-REAL16: v_cmpx_ngt_f16 v1.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x16,0x7d,0x01,0x1b,0x00,0xff]
+# GFX12-FAKE16: v_cmpx_ngt_f16 v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x16,0x7d,0x01,0x1b,0x00,0xff]
 
 0xfa,0x04,0x16,0x7d,0x01,0xe4,0x00,0xff
-# GFX12: v_cmpx_ngt_f16 v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x16,0x7d,0x01,0xe4,0x00,0xff]
+# GFX12-REAL16: v_cmpx_ngt_f16 v1.l, v2.l quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x16,0x7d,0x01,0xe4,0x00,0xff]
+# GFX12-FAKE16: v_cmpx_ngt_f16 v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x16,0x7d,0x01,0xe4,0x00,0xff]
 
 0xfa,0x04,0x16,0x7d,0x01,0x40,0x01,0xff
-# GFX12: v_cmpx_ngt_f16 v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x16,0x7d,0x01,0x40,0x01,0xff]
+# GFX12-REAL16: v_cmpx_ngt_f16 v1.l, v2.l row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x16,0x7d,0x01,0x40,0x01,0xff]
+# GFX12-FAKE16: v_cmpx_ngt_f16 v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x16,0x7d,0x01,0x40,0x01,0xff]
 
 0xfa,0x04,0x16,0x7d,0x01,0x41,0x01,0xff
-# GFX12: v_cmpx_ngt_f16 v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x16,0x7d,0x01,0x41,0x01,0xff]
+# GFX12-REAL16: v_cmpx_ngt_f16 v1.l, v2.l row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x16,0x7d,0x01,0x41,0x01,0xff]
+# GFX12-FAKE16: v_cmpx_ngt_f16 v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x16,0x7d,0x01,0x41,0x01,0xff]
 
 0xfa,0x04,0x16,0x7d,0x01,0x01,0x01,0xff
-# GFX12: v_cmpx_ngt_f16 v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x16,0x7d,0x01,0x01,0x01,0xff]
+# GFX12-REAL16: v_cmpx_ngt_f16 v1.l, v2.l row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x16,0x7d,0x01,0x01,0x01,0xff]
+# GFX12-FAKE16: v_cmpx_ngt_f16 v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x16,0x7d,0x01,0x01,0x01,0xff]
 
 0xfa,0x04,0x16,0x7d,0x01,0x0f,0x01,0xff
-# GFX12: v_cmpx_ngt_f16 v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x16,0x7d,0x01,0x0f,0x01,0xff]
+# GFX12-REAL16: v_cmpx_ngt_f16 v1.l, v2.l row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x16,0x7d,0x01,0x0f,0x01,0xff]
+# GFX12-FAKE16: v_cmpx_ngt_f16 v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x16,0x7d,0x01,0x0f,0x01,0xff]
 
 0xfa,0x04,0x16,0x7d,0x01,0x11,0x01,0xff
-# GFX12: v_cmpx_ngt_f16 v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x16,0x7d,0x01,0x11,0x01,0xff]
+# GFX12-REAL16: v_cmpx_ngt_f16 v1.l, v2.l row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x16,0x7d,0x01,0x11,0x01,0xff]
+# GFX12-FAKE16: v_cmpx_ngt_f16 v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x16,0x7d,0x01,0x11,0x01,0xff]
 
 0xfa,0x04,0x16,0x7d,0x01,0x1f,0x01,0xff
-# GFX12: v_cmpx_ngt_f16 v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x16,0x7d,0x01,0x1f,0x01,0xff]
+# GFX12-REAL16: v_cmpx_ngt_f16 v1.l, v2.l row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x16,0x7d,0x01,0x1f,0x01,0xff]
+# GFX12-FAKE16: v_cmpx_ngt_f16 v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x16,0x7d,0x01,0x1f,0x01,0xff]
 
 0xfa,0x04,0x16,0x7d,0x01,0x21,0x01,0xff
-# GFX12: v_cmpx_ngt_f16 v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x16,0x7d,0x01,0x21,0x01,0xff]
+# GFX12-REAL16: v_cmpx_ngt_f16 v1.l, v2.l row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x16,0x7d,0x01,0x21,0x01,0xff]
+# GFX12-FAKE16: v_cmpx_ngt_f16 v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x16,0x7d,0x01,0x21,0x01,0xff]
 
 0xfa,0x04,0x16,0x7d,0x01,0x2f,0x01,0xff
-# GFX12: v_cmpx_ngt_f16 v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x16,0x7d,0x01,0x2f,0x01,0xff]
+# GFX12-REAL16: v_cmpx_ngt_f16 v1.l, v2.l row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x16,0x7d,0x01,0x2f,0x01,0xff]
+# GFX12-FAKE16: v_cmpx_ngt_f16 v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x16,0x7d,0x01,0x2f,0x01,0xff]
 
 0xfa,0x04,0x16,0x7d,0x01,0x50,0x01,0xff
-# GFX12: v_cmpx_ngt_f16 v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x16,0x7d,0x01,0x50,0x01,0xff]
+# GFX12-REAL16: v_cmpx_ngt_f16 v1.l, v2.l row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x16,0x7d,0x01,0x50,0x01,0xff]
+# GFX12-FAKE16: v_cmpx_ngt_f16 v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x16,0x7d,0x01,0x50,0x01,0xff]
 
 0xfa,0x04,0x16,0x7d,0x01,0x5f,0x01,0x01
-# GFX12: v_cmpx_ngt_f16 v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x16,0x7d,0x01,0x5f,0x01,0x01]
+# GFX12-REAL16: v_cmpx_ngt_f16 v1.l, v2.l row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x16,0x7d,0x01,0x5f,0x01,0x01]
+# GFX12-FAKE16: v_cmpx_ngt_f16 v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x16,0x7d,0x01,0x5f,0x01,0x01]
 
 0xfa,0x04,0x16,0x7d,0x01,0x60,0x01,0x13
-# GFX12: v_cmpx_ngt_f16 v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x16,0x7d,0x01,0x60,0x01,0x13]
+# GFX12-REAL16: v_cmpx_ngt_f16 v1.l, v2.l row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x16,0x7d,0x01,0x60,0x01,0x13]
+# GFX12-FAKE16: v_cmpx_ngt_f16 v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x16,0x7d,0x01,0x60,0x01,0x13]
 
 0xfa,0xfe,0x16,0x7d,0x7f,0x6f,0xfd,0x30
-# GFX12: v_cmpx_ngt_f16 -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0x16,0x7d,0x7f,0x6f,0xfd,0x30]
+# GFX12-REAL16: v_cmpx_ngt_f16 -|v127.l|, -|v127.l| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0x16,0x7d,0x7f,0x6f,0xfd,0x30]
+# GFX12-FAKE16: v_cmpx_ngt_f16 -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0x16,0x7d,0x7f,0x6f,0xfd,0x30]
+
+0xfa,0x04,0x17,0x7d,0x81,0x60,0x01,0x13
+# GFX12-REAL16: v_cmpx_ngt_f16 v1.h, v2.h row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x17,0x7d,0x81,0x60,0x01,0x13]
+# GFX12-FAKE16: v_cmpx_ngt_f16 v129/*Invalid register, operand has 'VGPR_32_Lo128' register class*/, v130/*Invalid register, operand has 'VGPR_32_Lo128' register class*/ row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x17,0x7d,0x81,0x60,0x01,0x13]
+
+0xfa,0xfe,0x17,0x7d,0xff,0x6f,0xfd,0x30
+# GFX12-REAL16: v_cmpx_ngt_f16 -|v127.h|, -|v127.h| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0x17,0x7d,0xff,0x6f,0xfd,0x30]
+# GFX12-FAKE16: v_cmpx_ngt_f16 -|v255/*Invalid register, operand has 'VGPR_32_Lo128' register class*/|, -|v255/*Invalid register, operand has 'VGPR_32_Lo128' register class*/| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0x17,0x7d,0xff,0x6f,0xfd,0x30]
 
 0xfa,0x04,0x36,0x7d,0x01,0x1b,0x00,0xff
 # GFX12: v_cmpx_ngt_f32 v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x36,0x7d,0x01,0x1b,0x00,0xff]
@@ -2161,46 +2337,68 @@
 # GFX12: v_cmpx_ngt_f32 -|v255|, -|v255| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0x37,0x7d,0xff,0x6f,0xfd,0x30]
 
 0xfa,0x04,0x18,0x7d,0x01,0x1b,0x00,0xff
-# GFX12: v_cmpx_nle_f16 v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x18,0x7d,0x01,0x1b,0x00,0xff]
+# GFX12-REAL16: v_cmpx_nle_f16 v1.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x18,0x7d,0x01,0x1b,0x00,0xff]
+# GFX12-FAKE16: v_cmpx_nle_f16 v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x18,0x7d,0x01,0x1b,0x00,0xff]
 
 0xfa,0x04,0x18,0x7d,0x01,0xe4,0x00,0xff
-# GFX12: v_cmpx_nle_f16 v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x18,0x7d,0x01,0xe4,0x00,0xff]
+# GFX12-REAL16: v_cmpx_nle_f16 v1.l, v2.l quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x18,0x7d,0x01,0xe4,0x00,0xff]
+# GFX12-FAKE16: v_cmpx_nle_f16 v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x18,0x7d,0x01,0xe4,0x00,0xff]
 
 0xfa,0x04,0x18,0x7d,0x01,0x40,0x01,0xff
-# GFX12: v_cmpx_nle_f16 v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x18,0x7d,0x01,0x40,0x01,0xff]
+# GFX12-REAL16: v_cmpx_nle_f16 v1.l, v2.l row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x18,0x7d,0x01,0x40,0x01,0xff]
+# GFX12-FAKE16: v_cmpx_nle_f16 v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x18,0x7d,0x01,0x40,0x01,0xff]
 
 0xfa,0x04,0x18,0x7d,0x01,0x41,0x01,0xff
-# GFX12: v_cmpx_nle_f16 v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x18,0x7d,0x01,0x41,0x01,0xff]
+# GFX12-REAL16: v_cmpx_nle_f16 v1.l, v2.l row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x18,0x7d,0x01,0x41,0x01,0xff]
+# GFX12-FAKE16: v_cmpx_nle_f16 v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x18,0x7d,0x01,0x41,0x01,0xff]
 
 0xfa,0x04,0x18,0x7d,0x01,0x01,0x01,0xff
-# GFX12: v_cmpx_nle_f16 v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x18,0x7d,0x01,0x01,0x01,0xff]
+# GFX12-REAL16: v_cmpx_nle_f16 v1.l, v2.l row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x18,0x7d,0x01,0x01,0x01,0xff]
+# GFX12-FAKE16: v_cmpx_nle_f16 v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x18,0x7d,0x01,0x01,0x01,0xff]
 
 0xfa,0x04,0x18,0x7d,0x01,0x0f,0x01,0xff
-# GFX12: v_cmpx_nle_f16 v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x18,0x7d,0x01,0x0f,0x01,0xff]
+# GFX12-REAL16: v_cmpx_nle_f16 v1.l, v2.l row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x18,0x7d,0x01,0x0f,0x01,0xff]
+# GFX12-FAKE16: v_cmpx_nle_f16 v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x18,0x7d,0x01,0x0f,0x01,0xff]
 
 0xfa,0x04,0x18,0x7d,0x01,0x11,0x01,0xff
-# GFX12: v_cmpx_nle_f16 v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x18,0x7d,0x01,0x11,0x01,0xff]
+# GFX12-REAL16: v_cmpx_nle_f16 v1.l, v2.l row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x18,0x7d,0x01,0x11,0x01,0xff]
+# GFX12-FAKE16: v_cmpx_nle_f16 v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x18,0x7d,0x01,0x11,0x01,0xff]
 
 0xfa,0x04,0x18,0x7d,0x01,0x1f,0x01,0xff
-# GFX12: v_cmpx_nle_f16 v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x18,0x7d,0x01,0x1f,0x01,0xff]
+# GFX12-REAL16: v_cmpx_nle_f16 v1.l, v2.l row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x18,0x7d,0x01,0x1f,0x01,0xff]
+# GFX12-FAKE16: v_cmpx_nle_f16 v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x18,0x7d,0x01,0x1f,0x01,0xff]
 
 0xfa,0x04,0x18,0x7d,0x01,0x21,0x01,0xff
-# GFX12: v_cmpx_nle_f16 v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x18,0x7d,0x01,0x21,0x01,0xff]
+# GFX12-REAL16: v_cmpx_nle_f16 v1.l, v2.l row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x18,0x7d,0x01,0x21,0x01,0xff]
+# GFX12-FAKE16: v_cmpx_nle_f16 v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x18,0x7d,0x01,0x21,0x01,0xff]
 
 0xfa,0x04,0x18,0x7d,0x01,0x2f,0x01,0xff
-# GFX12: v_cmpx_nle_f16 v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x18,0x7d,0x01,0x2f,0x01,0xff]
+# GFX12-REAL16: v_cmpx_nle_f16 v1.l, v2.l row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x18,0x7d,0x01,0x2f,0x01,0xff]
+# GFX12-FAKE16: v_cmpx_nle_f16 v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x18,0x7d,0x01,0x2f,0x01,0xff]
 
 0xfa,0x04,0x18,0x7d,0x01,0x50,0x01,0xff
-# GFX12: v_cmpx_nle_f16 v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x18,0x7d,0x01,0x50,0x01,0xff]
+# GFX12-REAL16: v_cmpx_nle_f16 v1.l, v2.l row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x18,0x7d,0x01,0x50,0x01,0xff]
+# GFX12-FAKE16: v_cmpx_nle_f16 v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x18,0x7d,0x01,0x50,0x01,0xff]
 
 0xfa,0x04,0x18,0x7d,0x01,0x5f,0x01,0x01
-# GFX12: v_cmpx_nle_f16 v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x18,0x7d,0x01,0x5f,0x01,0x01]
+# GFX12-REAL16: v_cmpx_nle_f16 v1.l, v2.l row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x18,0x7d,0x01,0x5f,0x01,0x01]
+# GFX12-FAKE16: v_cmpx_nle_f16 v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x18,0x7d,0x01,0x5f,0x01,0x01]
 
 0xfa,0x04,0x18,0x7d,0x01,0x60,0x01,0x13
-# GFX12: v_cmpx_nle_f16 v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x18,0x7d,0x01,0x60,0x01,0x13]
+# GFX12-REAL16: v_cmpx_nle_f16 v1.l, v2.l row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x18,0x7d,0x01,0x60,0x01,0x13]
+# GFX12-FAKE16: v_cmpx_nle_f16 v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x18,0x7d,0x01,0x60,0x01,0x13]
 
 0xfa,0xfe,0x18,0x7d,0x7f,0x6f,0xfd,0x30
-# GFX12: v_cmpx_nle_f16 -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0x18,0x7d,0x7f,0x6f,0xfd,0x30]
+# GFX12-REAL16: v_cmpx_nle_f16 -|v127.l|, -|v127.l| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0x18,0x7d,0x7f,0x6f,0xfd,0x30]
+# GFX12-FAKE16: v_cmpx_nle_f16 -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0x18,0x7d,0x7f,0x6f,0xfd,0x30]
+
+0xfa,0x04,0x19,0x7d,0x81,0x60,0x01,0x13
+# GFX12-REAL16: v_cmpx_nle_f16 v1.h, v2.h row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x19,0x7d,0x81,0x60,0x01,0x13]
+# GFX12-FAKE16: v_cmpx_nle_f16 v129/*Invalid register, operand has 'VGPR_32_Lo128' register class*/, v130/*Invalid register, operand has 'VGPR_32_Lo128' register class*/ row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x19,0x7d,0x81,0x60,0x01,0x13]
+
+0xfa,0xfe,0x19,0x7d,0xff,0x6f,0xfd,0x30
+# GFX12-REAL16: v_cmpx_nle_f16 -|v127.h|, -|v127.h| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0x19,0x7d,0xff,0x6f,0xfd,0x30]
+# GFX12-FAKE16: v_cmpx_nle_f16 -|v255/*Invalid register, operand has 'VGPR_32_Lo128' register class*/|, -|v255/*Invalid register, operand has 'VGPR_32_Lo128' register class*/| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0x19,0x7d,0xff,0x6f,0xfd,0x30]
 
 0xfa,0x04,0x38,0x7d,0x01,0x1b,0x00,0xff
 # GFX12: v_cmpx_nle_f32 v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x38,0x7d,0x01,0x1b,0x00,0xff]
@@ -2245,46 +2443,68 @@
 # GFX12: v_cmpx_nle_f32 -|v255|, -|v255| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0x39,0x7d,0xff,0x6f,0xfd,0x30]
 
 0xfa,0x04,0x14,0x7d,0x01,0x1b,0x00,0xff
-# GFX12: v_cmpx_nlg_f16 v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x14,0x7d,0x01,0x1b,0x00,0xff]
+# GFX12-REAL16: v_cmpx_nlg_f16 v1.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x14,0x7d,0x01,0x1b,0x00,0xff]
+# GFX12-FAKE16: v_cmpx_nlg_f16 v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x14,0x7d,0x01,0x1b,0x00,0xff]
 
 0xfa,0x04,0x14,0x7d,0x01,0xe4,0x00,0xff
-# GFX12: v_cmpx_nlg_f16 v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x14,0x7d,0x01,0xe4,0x00,0xff]
+# GFX12-REAL16: v_cmpx_nlg_f16 v1.l, v2.l quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x14,0x7d,0x01,0xe4,0x00,0xff]
+# GFX12-FAKE16: v_cmpx_nlg_f16 v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x14,0x7d,0x01,0xe4,0x00,0xff]
 
 0xfa,0x04,0x14,0x7d,0x01,0x40,0x01,0xff
-# GFX12: v_cmpx_nlg_f16 v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x14,0x7d,0x01,0x40,0x01,0xff]
+# GFX12-REAL16: v_cmpx_nlg_f16 v1.l, v2.l row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x14,0x7d,0x01,0x40,0x01,0xff]
+# GFX12-FAKE16: v_cmpx_nlg_f16 v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x14,0x7d,0x01,0x40,0x01,0xff]
 
 0xfa,0x04,0x14,0x7d,0x01,0x41,0x01,0xff
-# GFX12: v_cmpx_nlg_f16 v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x14,0x7d,0x01,0x41,0x01,0xff]
+# GFX12-REAL16: v_cmpx_nlg_f16 v1.l, v2.l row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x14,0x7d,0x01,0x41,0x01,0xff]
+# GFX12-FAKE16: v_cmpx_nlg_f16 v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x14,0x7d,0x01,0x41,0x01,0xff]
 
 0xfa,0x04,0x14,0x7d,0x01,0x01,0x01,0xff
-# GFX12: v_cmpx_nlg_f16 v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x14,0x7d,0x01,0x01,0x01,0xff]
+# GFX12-REAL16: v_cmpx_nlg_f16 v1.l, v2.l row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x14,0x7d,0x01,0x01,0x01,0xff]
+# GFX12-FAKE16: v_cmpx_nlg_f16 v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x14,0x7d,0x01,0x01,0x01,0xff]
 
 0xfa,0x04,0x14,0x7d,0x01,0x0f,0x01,0xff
-# GFX12: v_cmpx_nlg_f16 v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x14,0x7d,0x01,0x0f,0x01,0xff]
+# GFX12-REAL16: v_cmpx_nlg_f16 v1.l, v2.l row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x14,0x7d,0x01,0x0f,0x01,0xff]
+# GFX12-FAKE16: v_cmpx_nlg_f16 v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x14,0x7d,0x01,0x0f,0x01,0xff]
 
 0xfa,0x04,0x14,0x7d,0x01,0x11,0x01,0xff
-# GFX12: v_cmpx_nlg_f16 v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x14,0x7d,0x01,0x11,0x01,0xff]
+# GFX12-REAL16: v_cmpx_nlg_f16 v1.l, v2.l row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x14,0x7d,0x01,0x11,0x01,0xff]
+# GFX12-FAKE16: v_cmpx_nlg_f16 v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x14,0x7d,0x01,0x11,0x01,0xff]
 
 0xfa,0x04,0x14,0x7d,0x01,0x1f,0x01,0xff
-# GFX12: v_cmpx_nlg_f16 v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x14,0x7d,0x01,0x1f,0x01,0xff]
+# GFX12-REAL16: v_cmpx_nlg_f16 v1.l, v2.l row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x14,0x7d,0x01,0x1f,0x01,0xff]
+# GFX12-FAKE16: v_cmpx_nlg_f16 v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x14,0x7d,0x01,0x1f,0x01,0xff]
 
 0xfa,0x04,0x14,0x7d,0x01,0x21,0x01,0xff
-# GFX12: v_cmpx_nlg_f16 v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x14,0x7d,0x01,0x21,0x01,0xff]
+# GFX12-REAL16: v_cmpx_nlg_f16 v1.l, v2.l row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x14,0x7d,0x01,0x21,0x01,0xff]
+# GFX12-FAKE16: v_cmpx_nlg_f16 v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x14,0x7d,0x01,0x21,0x01,0xff]
 
 0xfa,0x04,0x14,0x7d,0x01,0x2f,0x01,0xff
-# GFX12: v_cmpx_nlg_f16 v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x14,0x7d,0x01,0x2f,0x01,0xff]
+# GFX12-REAL16: v_cmpx_nlg_f16 v1.l, v2.l row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x14,0x7d,0x01,0x2f,0x01,0xff]
+# GFX12-FAKE16: v_cmpx_nlg_f16 v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x14,0x7d,0x01,0x2f,0x01,0xff]
 
 0xfa,0x04,0x14,0x7d,0x01,0x50,0x01,0xff
-# GFX12: v_cmpx_nlg_f16 v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x14,0x7d,0x01,0x50,0x01,0xff]
+# GFX12-REAL16: v_cmpx_nlg_f16 v1.l, v2.l row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x14,0x7d,0x01,0x50,0x01,0xff]
+# GFX12-FAKE16: v_cmpx_nlg_f16 v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x14,0x7d,0x01,0x50,0x01,0xff]
 
 0xfa,0x04,0x14,0x7d,0x01,0x5f,0x01,0x01
-# GFX12: v_cmpx_nlg_f16 v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x14,0x7d,0x01,0x5f,0x01,0x01]
+# GFX12-REAL16: v_cmpx_nlg_f16 v1.l, v2.l row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x14,0x7d,0x01,0x5f,0x01,0x01]
+# GFX12-FAKE16: v_cmpx_nlg_f16 v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x14,0x7d,0x01,0x5f,0x01,0x01]
 
 0xfa,0x04,0x14,0x7d,0x01,0x60,0x01,0x13
-# GFX12: v_cmpx_nlg_f16 v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x14,0x7d,0x01,0x60,0x01,0x13]
+# GFX12-REAL16: v_cmpx_nlg_f16 v1.l, v2.l row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x14,0x7d,0x01,0x60,0x01,0x13]
+# GFX12-FAKE16: v_cmpx_nlg_f16 v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x14,0x7d,0x01,0x60,0x01,0x13]
 
 0xfa,0xfe,0x14,0x7d,0x7f,0x6f,0xfd,0x30
-# GFX12: v_cmpx_nlg_f16 -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0x14,0x7d,0x7f,0x6f,0xfd,0x30]
+# GFX12-REAL16: v_cmpx_nlg_f16 -|v127.l|, -|v127.l| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0x14,0x7d,0x7f,0x6f,0xfd,0x30]
+# GFX12-FAKE16: v_cmpx_nlg_f16 -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0x14,0x7d,0x7f,0x6f,0xfd,0x30]
+
+0xfa,0x04,0x15,0x7d,0x81,0x60,0x01,0x13
+# GFX12-REAL16: v_cmpx_nlg_f16 v1.h, v2.h row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x15,0x7d,0x81,0x60,0x01,0x13]
+# GFX12-FAKE16: v_cmpx_nlg_f16 v129/*Invalid register, operand has 'VGPR_32_Lo128' register class*/, v130/*Invalid register, operand has 'VGPR_32_Lo128' register class*/ row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x15,0x7d,0x81,0x60,0x01,0x13]
+
+0xfa,0xfe,0x15,0x7d,0xff,0x6f,0xfd,0x30
+# GFX12-REAL16: v_cmpx_nlg_f16 -|v127.h|, -|v127.h| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0x15,0x7d,0xff,0x6f,0xfd,0x30]
+# GFX12-FAKE16: v_cmpx_nlg_f16 -|v255/*Invalid register, operand has 'VGPR_32_Lo128' register class*/|, -|v255/*Invalid register, operand has 'VGPR_32_Lo128' register class*/| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0x15,0x7d,0xff,0x6f,0xfd,0x30]
 
 0xfa,0x04,0x34,0x7d,0x01,0x1b,0x00,0xff
 # GFX12: v_cmpx_nlg_f32 v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x34,0x7d,0x01,0x1b,0x00,0xff]
@@ -2329,46 +2549,68 @@
 # GFX12: v_cmpx_nlg_f32 -|v255|, -|v255| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0x35,0x7d,0xff,0x6f,0xfd,0x30]
 
 0xfa,0x04,0x1c,0x7d,0x01,0x1b,0x00,0xff
-# GFX12: v_cmpx_nlt_f16 v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1c,0x7d,0x01,0x1b,0x00,0xff]
+# GFX12-REAL16: v_cmpx_nlt_f16 v1.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1c,0x7d,0x01,0x1b,0x00,0xff]
+# GFX12-FAKE16: v_cmpx_nlt_f16 v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1c,0x7d,0x01,0x1b,0x00,0xff]
 
 0xfa,0x04,0x1c,0x7d,0x01,0xe4,0x00,0xff
-# GFX12: v_cmpx_nlt_f16 v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1c,0x7d,0x01,0xe4,0x00,0xff]
+# GFX12-REAL16: v_cmpx_nlt_f16 v1.l, v2.l quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1c,0x7d,0x01,0xe4,0x00,0xff]
+# GFX12-FAKE16: v_cmpx_nlt_f16 v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1c,0x7d,0x01,0xe4,0x00,0xff]
 
 0xfa,0x04,0x1c,0x7d,0x01,0x40,0x01,0xff
-# GFX12: v_cmpx_nlt_f16 v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1c,0x7d,0x01,0x40,0x01,0xff]
+# GFX12-REAL16: v_cmpx_nlt_f16 v1.l, v2.l row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1c,0x7d,0x01,0x40,0x01,0xff]
+# GFX12-FAKE16: v_cmpx_nlt_f16 v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1c,0x7d,0x01,0x40,0x01,0xff]
 
 0xfa,0x04,0x1c,0x7d,0x01,0x41,0x01,0xff
-# GFX12: v_cmpx_nlt_f16 v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1c,0x7d,0x01,0x41,0x01,0xff]
+# GFX12-REAL16: v_cmpx_nlt_f16 v1.l, v2.l row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1c,0x7d,0x01,0x41,0x01,0xff]
+# GFX12-FAKE16: v_cmpx_nlt_f16 v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1c,0x7d,0x01,0x41,0x01,0xff]
 
 0xfa,0x04,0x1c,0x7d,0x01,0x01,0x01,0xff
-# GFX12: v_cmpx_nlt_f16 v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1c,0x7d,0x01,0x01,0x01,0xff]
+# GFX12-REAL16: v_cmpx_nlt_f16 v1.l, v2.l row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1c,0x7d,0x01,0x01,0x01,0xff]
+# GFX12-FAKE16: v_cmpx_nlt_f16 v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1c,0x7d,0x01,0x01,0x01,0xff]
 
 0xfa,0x04,0x1c,0x7d,0x01,0x0f,0x01,0xff
-# GFX12: v_cmpx_nlt_f16 v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1c,0x7d,0x01,0x0f,0x01,0xff]
+# GFX12-REAL16: v_cmpx_nlt_f16 v1.l, v2.l row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1c,0x7d,0x01,0x0f,0x01,0xff]
+# GFX12-FAKE16: v_cmpx_nlt_f16 v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1c,0x7d,0x01,0x0f,0x01,0xff]
 
 0xfa,0x04,0x1c,0x7d,0x01,0x11,0x01,0xff
-# GFX12: v_cmpx_nlt_f16 v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1c,0x7d,0x01,0x11,0x01,0xff]
+# GFX12-REAL16: v_cmpx_nlt_f16 v1.l, v2.l row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1c,0x7d,0x01,0x11,0x01,0xff]
+# GFX12-FAKE16: v_cmpx_nlt_f16 v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1c,0x7d,0x01,0x11,0x01,0xff]
 
 0xfa,0x04,0x1c,0x7d,0x01,0x1f,0x01,0xff
-# GFX12: v_cmpx_nlt_f16 v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1c,0x7d,0x01,0x1f,0x01,0xff]
+# GFX12-REAL16: v_cmpx_nlt_f16 v1.l, v2.l row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1c,0x7d,0x01,0x1f,0x01,0xff]
+# GFX12-FAKE16: v_cmpx_nlt_f16 v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1c,0x7d,0x01,0x1f,0x01,0xff]
 
 0xfa,0x04,0x1c,0x7d,0x01,0x21,0x01,0xff
-# GFX12: v_cmpx_nlt_f16 v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1c,0x7d,0x01,0x21,0x01,0xff]
+# GFX12-REAL16: v_cmpx_nlt_f16 v1.l, v2.l row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1c,0x7d,0x01,0x21,0x01,0xff]
+# GFX12-FAKE16: v_cmpx_nlt_f16 v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1c,0x7d,0x01,0x21,0x01,0xff]
 
 0xfa,0x04,0x1c,0x7d,0x01,0x2f,0x01,0xff
-# GFX12: v_cmpx_nlt_f16 v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1c,0x7d,0x01,0x2f,0x01,0xff]
+# GFX12-REAL16: v_cmpx_nlt_f16 v1.l, v2.l row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1c,0x7d,0x01,0x2f,0x01,0xff]
+# GFX12-FAKE16: v_cmpx_nlt_f16 v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1c,0x7d,0x01,0x2f,0x01,0xff]
 
 0xfa,0x04,0x1c,0x7d,0x01,0x50,0x01,0xff
-# GFX12: v_cmpx_nlt_f16 v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1c,0x7d,0x01,0x50,0x01,0xff]
+# GFX12-REAL16: v_cmpx_nlt_f16 v1.l, v2.l row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1c,0x7d,0x01,0x50,0x01,0xff]
+# GFX12-FAKE16: v_cmpx_nlt_f16 v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x1c,0x7d,0x01,0x50,0x01,0xff]
 
 0xfa,0x04,0x1c,0x7d,0x01,0x5f,0x01,0x01
-# GFX12: v_cmpx_nlt_f16 v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x1c,0x7d,0x01,0x5f,0x01,0x01]
+# GFX12-REAL16: v_cmpx_nlt_f16 v1.l, v2.l row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x1c,0x7d,0x01,0x5f,0x01,0x01]
+# GFX12-FAKE16: v_cmpx_nlt_f16 v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x1c,0x7d,0x01,0x5f,0x01,0x01]
 
 0xfa,0x04,0x1c,0x7d,0x01,0x60,0x01,0x13
-# GFX12: v_cmpx_nlt_f16 v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x1c,0x7d,0x01,0x60,0x01,0x13]
+# GFX12-REAL16: v_cmpx_nlt_f16 v1.l, v2.l row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x1c,0x7d,0x01,0x60,0x01,0x13]
+# GFX12-FAKE16: v_cmpx_nlt_f16 v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x1c,0x7d,0x01,0x60,0x01,0x13]
 
 0xfa,0xfe,0x1c,0x7d,0x7f,0x6f,0xfd,0x30
-# GFX12: v_cmpx_nlt_f16 -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0x1c,0x7d,0x7f,0x6f,0xfd,0x30]
+# GFX12-REAL16: v_cmpx_nlt_f16 -|v127.l|, -|v127.l| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0x1c,0x7d,0x7f,0x6f,0xfd,0x30]
+# GFX12-FAKE16: v_cmpx_nlt_f16 -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0x1c,0x7d,0x7f,0x6f,0xfd,0x30]
+
+0xfa,0x04,0x1d,0x7d,0x81,0x60,0x01,0x13
+# GFX12-REAL16: v_cmpx_nlt_f16 v1.h, v2.h row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x1d,0x7d,0x81,0x60,0x01,0x13]
+# GFX12-FAKE16: v_cmpx_nlt_f16 v129/*Invalid register, operand has 'VGPR_32_Lo128' register class*/, v130/*Invalid register, operand has 'VGPR_32_Lo128' register class*/ row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x1d,0x7d,0x81,0x60,0x01,0x13]
+
+0xfa,0xfe,0x1d,0x7d,0xff,0x6f,0xfd,0x30
+# GFX12-REAL16: v_cmpx_nlt_f16 -|v127.h|, -|v127.h| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0x1d,0x7d,0xff,0x6f,0xfd,0x30]
+# GFX12-FAKE16: v_cmpx_nlt_f16 -|v255/*Invalid register, operand has 'VGPR_32_Lo128' register class*/|, -|v255/*Invalid register, operand has 'VGPR_32_Lo128' register class*/| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0x1d,0x7d,0xff,0x6f,0xfd,0x30]
 
 0xfa,0x04,0x3c,0x7d,0x01,0x1b,0x00,0xff
 # GFX12: v_cmpx_nlt_f32 v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x3c,0x7d,0x01,0x1b,0x00,0xff]
@@ -2413,46 +2655,68 @@
 # GFX12: v_cmpx_nlt_f32 -|v255|, -|v255| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0x3d,0x7d,0xff,0x6f,0xfd,0x30]
 
 0xfa,0x04,0x0e,0x7d,0x01,0x1b,0x00,0xff
-# GFX12: v_cmpx_o_f16 v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0e,0x7d,0x01,0x1b,0x00,0xff]
+# GFX12-REAL16: v_cmpx_o_f16 v1.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0e,0x7d,0x01,0x1b,0x00,0xff]
+# GFX12-FAKE16: v_cmpx_o_f16 v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0e,0x7d,0x01,0x1b,0x00,0xff]
 
 0xfa,0x04,0x0e,0x7d,0x01,0xe4,0x00,0xff
-# GFX12: v_cmpx_o_f16 v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0e,0x7d,0x01,0xe4,0x00,0xff]
+# GFX12-REAL16: v_cmpx_o_f16 v1.l, v2.l quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0e,0x7d,0x01,0xe4,0x00,0xff]
+# GFX12-FAKE16: v_cmpx_o_f16 v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0e,0x7d,0x01,0xe4,0x00,0xff]
 
 0xfa,0x04,0x0e,0x7d,0x01,0x40,0x01,0xff
-# GFX12: v_cmpx_o_f16 v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0e,0x7d,0x01,0x40,0x01,0xff]
+# GFX12-REAL16: v_cmpx_o_f16 v1.l, v2.l row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0e,0x7d,0x01,0x40,0x01,0xff]
+# GFX12-FAKE16: v_cmpx_o_f16 v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0e,0x7d,0x01,0x40,0x01,0xff]
 
 0xfa,0x04,0x0e,0x7d,0x01,0x41,0x01,0xff
-# GFX12: v_cmpx_o_f16 v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0e,0x7d,0x01,0x41,0x01,0xff]
+# GFX12-REAL16: v_cmpx_o_f16 v1.l, v2.l row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0e,0x7d,0x01,0x41,0x01,0xff]
+# GFX12-FAKE16: v_cmpx_o_f16 v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0e,0x7d,0x01,0x41,0x01,0xff]
 
 0xfa,0x04,0x0e,0x7d,0x01,0x01,0x01,0xff
-# GFX12: v_cmpx_o_f16 v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0e,0x7d,0x01,0x01,0x01,0xff]
+# GFX12-REAL16: v_cmpx_o_f16 v1.l, v2.l row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0e,0x7d,0x01,0x01,0x01,0xff]
+# GFX12-FAKE16: v_cmpx_o_f16 v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0e,0x7d,0x01,0x01,0x01,0xff]
 
 0xfa,0x04,0x0e,0x7d,0x01,0x0f,0x01,0xff
-# GFX12: v_cmpx_o_f16 v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0e,0x7d,0x01,0x0f,0x01,0xff]
+# GFX12-REAL16: v_cmpx_o_f16 v1.l, v2.l row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0e,0x7d,0x01,0x0f,0x01,0xff]
+# GFX12-FAKE16: v_cmpx_o_f16 v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0e,0x7d,0x01,0x0f,0x01,0xff]
 
 0xfa,0x04,0x0e,0x7d,0x01,0x11,0x01,0xff
-# GFX12: v_cmpx_o_f16 v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0e,0x7d,0x01,0x11,0x01,0xff]
+# GFX12-REAL16: v_cmpx_o_f16 v1.l, v2.l row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0e,0x7d,0x01,0x11,0x01,0xff]
+# GFX12-FAKE16: v_cmpx_o_f16 v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0e,0x7d,0x01,0x11,0x01,0xff]
 
 0xfa,0x04,0x0e,0x7d,0x01,0x1f,0x01,0xff
-# GFX12: v_cmpx_o_f16 v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0e,0x7d,0x01,0x1f,0x01,0xff]
+# GFX12-REAL16: v_cmpx_o_f16 v1.l, v2.l row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0e,0x7d,0x01,0x1f,0x01,0xff]
+# GFX12-FAKE16: v_cmpx_o_f16 v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0e,0x7d,0x01,0x1f,0x01,0xff]
 
 0xfa,0x04,0x0e,0x7d,0x01,0x21,0x01,0xff
-# GFX12: v_cmpx_o_f16 v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0e,0x7d,0x01,0x21,0x01,0xff]
+# GFX12-REAL16: v_cmpx_o_f16 v1.l, v2.l row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0e,0x7d,0x01,0x21,0x01,0xff]
+# GFX12-FAKE16: v_cmpx_o_f16 v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0e,0x7d,0x01,0x21,0x01,0xff]
 
 0xfa,0x04,0x0e,0x7d,0x01,0x2f,0x01,0xff
-# GFX12: v_cmpx_o_f16 v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0e,0x7d,0x01,0x2f,0x01,0xff]
+# GFX12-REAL16: v_cmpx_o_f16 v1.l, v2.l row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0e,0x7d,0x01,0x2f,0x01,0xff]
+# GFX12-FAKE16: v_cmpx_o_f16 v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0e,0x7d,0x01,0x2f,0x01,0xff]
 
 0xfa,0x04,0x0e,0x7d,0x01,0x50,0x01,0xff
-# GFX12: v_cmpx_o_f16 v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0e,0x7d,0x01,0x50,0x01,0xff]
+# GFX12-REAL16: v_cmpx_o_f16 v1.l, v2.l row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0e,0x7d,0x01,0x50,0x01,0xff]
+# GFX12-FAKE16: v_cmpx_o_f16 v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0e,0x7d,0x01,0x50,0x01,0xff]
 
 0xfa,0x04,0x0e,0x7d,0x01,0x5f,0x01,0x01
-# GFX12: v_cmpx_o_f16 v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x0e,0x7d,0x01,0x5f,0x01,0x01]
+# GFX12-REAL16: v_cmpx_o_f16 v1.l, v2.l row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x0e,0x7d,0x01,0x5f,0x01,0x01]
+# GFX12-FAKE16: v_cmpx_o_f16 v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x0e,0x7d,0x01,0x5f,0x01,0x01]
 
 0xfa,0x04,0x0e,0x7d,0x01,0x60,0x01,0x13
-# GFX12: v_cmpx_o_f16 v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x0e,0x7d,0x01,0x60,0x01,0x13]
+# GFX12-REAL16: v_cmpx_o_f16 v1.l, v2.l row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x0e,0x7d,0x01,0x60,0x01,0x13]
+# GFX12-FAKE16: v_cmpx_o_f16 v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x0e,0x7d,0x01,0x60,0x01,0x13]
 
 0xfa,0xfe,0x0e,0x7d,0x7f,0x6f,0xfd,0x30
-# GFX12: v_cmpx_o_f16 -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0x0e,0x7d,0x7f,0x6f,0xfd,0x30]
+# GFX12-REAL16: v_cmpx_o_f16 -|v127.l|, -|v127.l| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0x0e,0x7d,0x7f,0x6f,0xfd,0x30]
+# GFX12-FAKE16: v_cmpx_o_f16 -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0x0e,0x7d,0x7f,0x6f,0xfd,0x30]
+
+0xfa,0x04,0x0f,0x7d,0x81,0x60,0x01,0x13
+# GFX12-REAL16: v_cmpx_o_f16 v1.h, v2.h row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x0f,0x7d,0x81,0x60,0x01,0x13]
+# GFX12-FAKE16: v_cmpx_o_f16 v129/*Invalid register, operand has 'VGPR_32_Lo128' register class*/, v130/*Invalid register, operand has 'VGPR_32_Lo128' register class*/ row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x0f,0x7d,0x81,0x60,0x01,0x13]
+
+0xfa,0xfe,0x0f,0x7d,0xff,0x6f,0xfd,0x30
+# GFX12-REAL16: v_cmpx_o_f16 -|v127.h|, -|v127.h| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0x0f,0x7d,0xff,0x6f,0xfd,0x30]
+# GFX12-FAKE16: v_cmpx_o_f16 -|v255/*Invalid register, operand has 'VGPR_32_Lo128' register class*/|, -|v255/*Invalid register, operand has 'VGPR_32_Lo128' register class*/| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0x0f,0x7d,0xff,0x6f,0xfd,0x30]
 
 0xfa,0x04,0x2e,0x7d,0x01,0x1b,0x00,0xff
 # GFX12: v_cmpx_o_f32 v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x2e,0x7d,0x01,0x1b,0x00,0xff]
@@ -2497,46 +2761,68 @@
 # GFX12: v_cmpx_o_f32 -|v255|, -|v255| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0x2f,0x7d,0xff,0x6f,0xfd,0x30]
 
 0xfa,0x04,0x10,0x7d,0x01,0x1b,0x00,0xff
-# GFX12: v_cmpx_u_f16 v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x10,0x7d,0x01,0x1b,0x00,0xff]
+# GFX12-REAL16: v_cmpx_u_f16 v1.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x10,0x7d,0x01,0x1b,0x00,0xff]
+# GFX12-FAKE16: v_cmpx_u_f16 v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x10,0x7d,0x01,0x1b,0x00,0xff]
 
 0xfa,0x04,0x10,0x7d,0x01,0xe4,0x00,0xff
-# GFX12: v_cmpx_u_f16 v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x10,0x7d,0x01,0xe4,0x00,0xff]
+# GFX12-REAL16: v_cmpx_u_f16 v1.l, v2.l quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x10,0x7d,0x01,0xe4,0x00,0xff]
+# GFX12-FAKE16: v_cmpx_u_f16 v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x10,0x7d,0x01,0xe4,0x00,0xff]
 
 0xfa,0x04,0x10,0x7d,0x01,0x40,0x01,0xff
-# GFX12: v_cmpx_u_f16 v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x10,0x7d,0x01,0x40,0x01,0xff]
+# GFX12-REAL16: v_cmpx_u_f16 v1.l, v2.l row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x10,0x7d,0x01,0x40,0x01,0xff]
+# GFX12-FAKE16: v_cmpx_u_f16 v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x10,0x7d,0x01,0x40,0x01,0xff]
 
 0xfa,0x04,0x10,0x7d,0x01,0x41,0x01,0xff
-# GFX12: v_cmpx_u_f16 v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x10,0x7d,0x01,0x41,0x01,0xff]
+# GFX12-REAL16: v_cmpx_u_f16 v1.l, v2.l row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x10,0x7d,0x01,0x41,0x01,0xff]
+# GFX12-FAKE16: v_cmpx_u_f16 v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x10,0x7d,0x01,0x41,0x01,0xff]
 
 0xfa,0x04,0x10,0x7d,0x01,0x01,0x01,0xff
-# GFX12: v_cmpx_u_f16 v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x10,0x7d,0x01,0x01,0x01,0xff]
+# GFX12-REAL16: v_cmpx_u_f16 v1.l, v2.l row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x10,0x7d,0x01,0x01,0x01,0xff]
+# GFX12-FAKE16: v_cmpx_u_f16 v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x10,0x7d,0x01,0x01,0x01,0xff]
 
 0xfa,0x04,0x10,0x7d,0x01,0x0f,0x01,0xff
-# GFX12: v_cmpx_u_f16 v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x10,0x7d,0x01,0x0f,0x01,0xff]
+# GFX12-REAL16: v_cmpx_u_f16 v1.l, v2.l row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x10,0x7d,0x01,0x0f,0x01,0xff]
+# GFX12-FAKE16: v_cmpx_u_f16 v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x10,0x7d,0x01,0x0f,0x01,0xff]
 
 0xfa,0x04,0x10,0x7d,0x01,0x11,0x01,0xff
-# GFX12: v_cmpx_u_f16 v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x10,0x7d,0x01,0x11,0x01,0xff]
+# GFX12-REAL16: v_cmpx_u_f16 v1.l, v2.l row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x10,0x7d,0x01,0x11,0x01,0xff]
+# GFX12-FAKE16: v_cmpx_u_f16 v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x10,0x7d,0x01,0x11,0x01,0xff]
 
 0xfa,0x04,0x10,0x7d,0x01,0x1f,0x01,0xff
-# GFX12: v_cmpx_u_f16 v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x10,0x7d,0x01,0x1f,0x01,0xff]
+# GFX12-REAL16: v_cmpx_u_f16 v1.l, v2.l row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x10,0x7d,0x01,0x1f,0x01,0xff]
+# GFX12-FAKE16: v_cmpx_u_f16 v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x10,0x7d,0x01,0x1f,0x01,0xff]
 
 0xfa,0x04,0x10,0x7d,0x01,0x21,0x01,0xff
-# GFX12: v_cmpx_u_f16 v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x10,0x7d,0x01,0x21,0x01,0xff]
+# GFX12-REAL16: v_cmpx_u_f16 v1.l, v2.l row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x10,0x7d,0x01,0x21,0x01,0xff]
+# GFX12-FAKE16: v_cmpx_u_f16 v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x10,0x7d,0x01,0x21,0x01,0xff]
 
 0xfa,0x04,0x10,0x7d,0x01,0x2f,0x01,0xff
-# GFX12: v_cmpx_u_f16 v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x10,0x7d,0x01,0x2f,0x01,0xff]
+# GFX12-REAL16: v_cmpx_u_f16 v1.l, v2.l row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x10,0x7d,0x01,0x2f,0x01,0xff]
+# GFX12-FAKE16: v_cmpx_u_f16 v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x10,0x7d,0x01,0x2f,0x01,0xff]
 
 0xfa,0x04,0x10,0x7d,0x01,0x50,0x01,0xff
-# GFX12: v_cmpx_u_f16 v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x10,0x7d,0x01,0x50,0x01,0xff]
+# GFX12-REAL16: v_cmpx_u_f16 v1.l, v2.l row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x10,0x7d,0x01,0x50,0x01,0xff]
+# GFX12-FAKE16: v_cmpx_u_f16 v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x10,0x7d,0x01,0x50,0x01,0xff]
 
 0xfa,0x04,0x10,0x7d,0x01,0x5f,0x01,0x01
-# GFX12: v_cmpx_u_f16 v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x10,0x7d,0x01,0x5f,0x01,0x01]
+# GFX12-REAL16: v_cmpx_u_f16 v1.l, v2.l row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x10,0x7d,0x01,0x5f,0x01,0x01]
+# GFX12-FAKE16: v_cmpx_u_f16 v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x10,0x7d,0x01,0x5f,0x01,0x01]
 
 0xfa,0x04,0x10,0x7d,0x01,0x60,0x01,0x13
-# GFX12: v_cmpx_u_f16 v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x10,0x7d,0x01,0x60,0x01,0x13]
+# GFX12-REAL16: v_cmpx_u_f16 v1.l, v2.l row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x10,0x7d,0x01,0x60,0x01,0x13]
+# GFX12-FAKE16: v_cmpx_u_f16 v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x10,0x7d,0x01,0x60,0x01,0x13]
 
 0xfa,0xfe,0x10,0x7d,0x7f,0x6f,0xfd,0x30
-# GFX12: v_cmpx_u_f16 -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0x10,0x7d,0x7f,0x6f,0xfd,0x30]
+# GFX12-REAL16: v_cmpx_u_f16 -|v127.l|, -|v127.l| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0x10,0x7d,0x7f,0x6f,0xfd,0x30]
+# GFX12-FAKE16: v_cmpx_u_f16 -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0x10,0x7d,0x7f,0x6f,0xfd,0x30]
+
+0xfa,0x04,0x11,0x7d,0x81,0x60,0x01,0x13
+# GFX12-REAL16: v_cmpx_u_f16 v1.h, v2.h row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x11,0x7d,0x81,0x60,0x01,0x13]
+# GFX12-FAKE16: v_cmpx_u_f16 v129/*Invalid register, operand has 'VGPR_32_Lo128' register class*/, v130/*Invalid register, operand has 'VGPR_32_Lo128' register class*/ row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x11,0x7d,0x81,0x60,0x01,0x13]
+
+0xfa,0xfe,0x11,0x7d,0xff,0x6f,0xfd,0x30
+# GFX12-REAL16: v_cmpx_u_f16 -|v127.h|, -|v127.h| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0x11,0x7d,0xff,0x6f,0xfd,0x30]
+# GFX12-FAKE16: v_cmpx_u_f16 -|v255/*Invalid register, operand has 'VGPR_32_Lo128' register class*/|, -|v255/*Invalid register, operand has 'VGPR_32_Lo128' register class*/| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0x11,0x7d,0xff,0x6f,0xfd,0x30]
 
 0xfa,0x04,0x30,0x7d,0x01,0x1b,0x00,0xff
 # GFX12: v_cmpx_u_f32 v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x30,0x7d,0x01,0x1b,0x00,0xff]
diff --git a/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vopcx_dpp8.txt b/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vopcx_dpp8.txt
index b2539ad5a49e7..bcd75b7bfa087 100644
--- a/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vopcx_dpp8.txt
+++ b/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vopcx_dpp8.txt
@@ -27,10 +27,20 @@
 # GFX12: v_cmpx_class_f32 v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0xfd,0x7d,0xff,0x00,0x00,0x00]
 
 0xe9,0x04,0x04,0x7d,0x01,0x77,0x39,0x05
-# GFX12: v_cmpx_eq_f16 v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x04,0x7d,0x01,0x77,0x39,0x05]
+# GFX12-REAL16: v_cmpx_eq_f16 v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x04,0x7d,0x01,0x77,0x39,0x05]
+# GFX12-FAKE16: v_cmpx_eq_f16 v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x04,0x7d,0x01,0x77,0x39,0x05]
 
 0xea,0xfe,0x04,0x7d,0x7f,0x00,0x00,0x00
-# GFX12: v_cmpx_eq_f16 v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0x04,0x7d,0x7f,0x00,0x00,0x00]
+# GFX12-REAL16: v_cmpx_eq_f16 v127.l, v127.l dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0x04,0x7d,0x7f,0x00,0x00,0x00]
+# GFX12-FAKE16: v_cmpx_eq_f16 v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0x04,0x7d,0x7f,0x00,0x00,0x00]
+
+0xe9,0x04,0x05,0x7d,0x81,0x77,0x39,0x05
+# GFX12-REAL16: v_cmpx_eq_f16 v1.h, v2.h dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x05,0x7d,0x81,0x77,0x39,0x05]
+# GFX12-FAKE16: v_cmpx_eq_f16 v129/*Invalid register, operand has 'VGPR_32_Lo128' register class*/, v130/*Invalid register, operand has 'VGPR_32_Lo128' register class*/ dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x05,0x7d,0x81,0x77,0x39,0x05]
+
+0xea,0xfe,0x05,0x7d,0xff,0x00,0x00,0x00
+# GFX12-REAL16: v_cmpx_eq_f16 v127.h, v127.h dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0x05,0x7d,0xff,0x00,0x00,0x00]
+# GFX12-FAKE16: v_cmpx_eq_f16 v255/*Invalid register, operand has 'VGPR_32_Lo128' register class*/, v255/*Invalid register, operand has 'VGPR_32_Lo128' register class*/ dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0x05,0x7d,0xff,0x00,0x00,0x00]
 
 0xe9,0x04,0x24,0x7d,0x01,0x77,0x39,0x05
 # GFX12: v_cmpx_eq_f32 v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x24,0x7d,0x01,0x77,0x39,0x05]
@@ -83,10 +93,20 @@
 # GFX12: v_cmpx_eq_u32 v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0x95,0x7d,0xff,0x00,0x00,0x00]
 
 0xe9,0x04,0x0c,0x7d,0x01,0x77,0x39,0x05
-# GFX12: v_cmpx_ge_f16 v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x0c,0x7d,0x01,0x77,0x39,0x05]
+# GFX12-REAL16: v_cmpx_ge_f16 v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x0c,0x7d,0x01,0x77,0x39,0x05]
+# GFX12-FAKE16: v_cmpx_ge_f16 v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x0c,0x7d,0x01,0x77,0x39,0x05]
 
 0xea,0xfe,0x0c,0x7d,0x7f,0x00,0x00,0x00
-# GFX12: v_cmpx_ge_f16 v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0x0c,0x7d,0x7f,0x00,0x00,0x00]
+# GFX12-REAL16: v_cmpx_ge_f16 v127.l, v127.l dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0x0c,0x7d,0x7f,0x00,0x00,0x00]
+# GFX12-FAKE16: v_cmpx_ge_f16 v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0x0c,0x7d,0x7f,0x00,0x00,0x00]
+
+0xe9,0x04,0x0d,0x7d,0x81,0x77,0x39,0x05
+# GFX12-REAL16: v_cmpx_ge_f16 v1.h, v2.h dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x0d,0x7d,0x81,0x77,0x39,0x05]
+# GFX12-FAKE16: v_cmpx_ge_f16 v129/*Invalid register, operand has 'VGPR_32_Lo128' register class*/, v130/*Invalid register, operand has 'VGPR_32_Lo128' register class*/ dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x0d,0x7d,0x81,0x77,0x39,0x05]
+
+0xea,0xfe,0x0d,0x7d,0xff,0x00,0x00,0x00
+# GFX12-REAL16: v_cmpx_ge_f16 v127.h, v127.h dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0x0d,0x7d,0xff,0x00,0x00,0x00]
+# GFX12-FAKE16: v_cmpx_ge_f16 v255/*Invalid register, operand has 'VGPR_32_Lo128' register class*/, v255/*Invalid register, operand has 'VGPR_32_Lo128' register class*/ dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0x0d,0x7d,0xff,0x00,0x00,0x00]
 
 0xe9,0x04,0x2c,0x7d,0x01,0x77,0x39,0x05
 # GFX12: v_cmpx_ge_f32 v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x2c,0x7d,0x01,0x77,0x39,0x05]
@@ -139,10 +159,20 @@
 # GFX12: v_cmpx_ge_u32 v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0x9d,0x7d,0xff,0x00,0x00,0x00]
 
 0xe9,0x04,0x08,0x7d,0x01,0x77,0x39,0x05
-# GFX12: v_cmpx_gt_f16 v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x08,0x7d,0x01,0x77,0x39,0x05]
+# GFX12-REAL16: v_cmpx_gt_f16 v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x08,0x7d,0x01,0x77,0x39,0x05]
+# GFX12-FAKE16: v_cmpx_gt_f16 v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x08,0x7d,0x01,0x77,0x39,0x05]
 
 0xea,0xfe,0x08,0x7d,0x7f,0x00,0x00,0x00
-# GFX12: v_cmpx_gt_f16 v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0x08,0x7d,0x7f,0x00,0x00,0x00]
+# GFX12-REAL16: v_cmpx_gt_f16 v127.l, v127.l dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0x08,0x7d,0x7f,0x00,0x00,0x00]
+# GFX12-FAKE16: v_cmpx_gt_f16 v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0x08,0x7d,0x7f,0x00,0x00,0x00]
+
+0xe9,0x04,0x09,0x7d,0x81,0x77,0x39,0x05
+# GFX12-REAL16: v_cmpx_gt_f16 v1.h, v2.h dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x09,0x7d,0x81,0x77,0x39,0x05]
+# GFX12-FAKE16: v_cmpx_gt_f16 v129/*Invalid register, operand has 'VGPR_32_Lo128' register class*/, v130/*Invalid register, operand has 'VGPR_32_Lo128' register class*/ dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x09,0x7d,0x81,0x77,0x39,0x05]
+
+0xea,0xfe,0x09,0x7d,0xff,0x00,0x00,0x00
+# GFX12-REAL16: v_cmpx_gt_f16 v127.h, v127.h dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0x09,0x7d,0xff,0x00,0x00,0x00]
+# GFX12-FAKE16: v_cmpx_gt_f16 v255/*Invalid register, operand has 'VGPR_32_Lo128' register class*/, v255/*Invalid register, operand has 'VGPR_32_Lo128' register class*/ dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0x09,0x7d,0xff,0x00,0x00,0x00]
 
 0xe9,0x04,0x28,0x7d,0x01,0x77,0x39,0x05
 # GFX12: v_cmpx_gt_f32 v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x28,0x7d,0x01,0x77,0x39,0x05]
@@ -195,10 +225,20 @@
 # GFX12: v_cmpx_gt_u32 v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0x99,0x7d,0xff,0x00,0x00,0x00]
 
 0xe9,0x04,0x06,0x7d,0x01,0x77,0x39,0x05
-# GFX12: v_cmpx_le_f16 v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x06,0x7d,0x01,0x77,0x39,0x05]
+# GFX12-REAL16: v_cmpx_le_f16 v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x06,0x7d,0x01,0x77,0x39,0x05]
+# GFX12-FAKE16: v_cmpx_le_f16 v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x06,0x7d,0x01,0x77,0x39,0x05]
 
 0xea,0xfe,0x06,0x7d,0x7f,0x00,0x00,0x00
-# GFX12: v_cmpx_le_f16 v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0x06,0x7d,0x7f,0x00,0x00,0x00]
+# GFX12-REAL16: v_cmpx_le_f16 v127.l, v127.l dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0x06,0x7d,0x7f,0x00,0x00,0x00]
+# GFX12-FAKE16: v_cmpx_le_f16 v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0x06,0x7d,0x7f,0x00,0x00,0x00]
+
+0xe9,0x04,0x07,0x7d,0x81,0x77,0x39,0x05
+# GFX12-REAL16: v_cmpx_le_f16 v1.h, v2.h dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x07,0x7d,0x81,0x77,0x39,0x05]
+# GFX12-FAKE16: v_cmpx_le_f16 v129/*Invalid register, operand has 'VGPR_32_Lo128' register class*/, v130/*Invalid register, operand has 'VGPR_32_Lo128' register class*/ dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x07,0x7d,0x81,0x77,0x39,0x05]
+
+0xea,0xfe,0x07,0x7d,0xff,0x00,0x00,0x00
+# GFX12-REAL16: v_cmpx_le_f16 v127.h, v127.h dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0x07,0x7d,0xff,0x00,0x00,0x00]
+# GFX12-FAKE16: v_cmpx_le_f16 v255/*Invalid register, operand has 'VGPR_32_Lo128' register class*/, v255/*Invalid register, operand has 'VGPR_32_Lo128' register class*/ dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0x07,0x7d,0xff,0x00,0x00,0x00]
 
 0xe9,0x04,0x26,0x7d,0x01,0x77,0x39,0x05
 # GFX12: v_cmpx_le_f32 v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x26,0x7d,0x01,0x77,0x39,0x05]
@@ -251,10 +291,20 @@
 # GFX12: v_cmpx_le_u32 v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0x97,0x7d,0xff,0x00,0x00,0x00]
 
 0xe9,0x04,0x0a,0x7d,0x01,0x77,0x39,0x05
-# GFX12: v_cmpx_lg_f16 v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x0a,0x7d,0x01,0x77,0x39,0x05]
+# GFX12-REAL16: v_cmpx_lg_f16 v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x0a,0x7d,0x01,0x77,0x39,0x05]
+# GFX12-FAKE16: v_cmpx_lg_f16 v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x0a,0x7d,0x01,0x77,0x39,0x05]
 
 0xea,0xfe,0x0a,0x7d,0x7f,0x00,0x00,0x00
-# GFX12: v_cmpx_lg_f16 v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0x0a,0x7d,0x7f,0x00,0x00,0x00]
+# GFX12-REAL16: v_cmpx_lg_f16 v127.l, v127.l dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0x0a,0x7d,0x7f,0x00,0x00,0x00]
+# GFX12-FAKE16: v_cmpx_lg_f16 v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0x0a,0x7d,0x7f,0x00,0x00,0x00]
+
+0xe9,0x04,0x0b,0x7d,0x81,0x77,0x39,0x05
+# GFX12-REAL16: v_cmpx_lg_f16 v1.h, v2.h dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x0b,0x7d,0x81,0x77,0x39,0x05]
+# GFX12-FAKE16: v_cmpx_lg_f16 v129/*Invalid register, operand has 'VGPR_32_Lo128' register class*/, v130/*Invalid register, operand has 'VGPR_32_Lo128' register class*/ dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x0b,0x7d,0x81,0x77,0x39,0x05]
+
+0xea,0xfe,0x0b,0x7d,0xff,0x00,0x00,0x00
+# GFX12-REAL16: v_cmpx_lg_f16 v127.h, v127.h dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0x0b,0x7d,0xff,0x00,0x00,0x00]
+# GFX12-FAKE16: v_cmpx_lg_f16 v255/*Invalid register, operand has 'VGPR_32_Lo128' register class*/, v255/*Invalid register, operand has 'VGPR_32_Lo128' register class*/ dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0x0b,0x7d,0xff,0x00,0x00,0x00]
 
 0xe9,0x04,0x2a,0x7d,0x01,0x77,0x39,0x05
 # GFX12: v_cmpx_lg_f32 v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x2a,0x7d,0x01,0x77,0x39,0x05]
@@ -373,10 +423,20 @@
 # GFX12: v_cmpx_ne_u32 v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0x9b,0x7d,0xff,0x00,0x00,0x00]
 
 0xe9,0x04,0x1a,0x7d,0x01,0x77,0x39,0x05
-# GFX12: v_cmpx_neq_f16 v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x1a,0x7d,0x01,0x77,0x39,0x05]
+# GFX12-REAL16: v_cmpx_neq_f16 v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x1a,0x7d,0x01,0x77,0x39,0x05]
+# GFX12-FAKE16: v_cmpx_neq_f16 v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x1a,0x7d,0x01,0x77,0x39,0x05]
 
 0xea,0xfe,0x1a,0x7d,0x7f,0x00,0x00,0x00
-# GFX12: v_cmpx_neq_f16 v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0x1a,0x7d,0x7f,0x00,0x00,0x00]
+# GFX12-REAL16: v_cmpx_neq_f16 v127.l, v127.l dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0x1a,0x7d,0x7f,0x00,0x00,0x00]
+# GFX12-FAKE16: v_cmpx_neq_f16 v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0x1a,0x7d,0x7f,0x00,0x00,0x00]
+
+0xe9,0x04,0x1b,0x7d,0x81,0x77,0x39,0x05
+# GFX12-REAL16: v_cmpx_neq_f16 v1.h, v2.h dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x1b,0x7d,0x81,0x77,0x39,0x05]
+# GFX12-FAKE16: v_cmpx_neq_f16 v129/*Invalid register, operand has 'VGPR_32_Lo128' register class*/, v130/*Invalid register, operand has 'VGPR_32_Lo128' register class*/ dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x1b,0x7d,0x81,0x77,0x39,0x05]
+
+0xea,0xfe,0x1b,0x7d,0xff,0x00,0x00,0x00
+# GFX12-REAL16: v_cmpx_neq_f16 v127.h, v127.h dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0x1b,0x7d,0xff,0x00,0x00,0x00]
+# GFX12-FAKE16: v_cmpx_neq_f16 v255/*Invalid register, operand has 'VGPR_32_Lo128' register class*/, v255/*Invalid register, operand has 'VGPR_32_Lo128' register class*/ dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0x1b,0x7d,0xff,0x00,0x00,0x00]
 
 0xe9,0x04,0x3a,0x7d,0x01,0x77,0x39,0x05
 # GFX12: v_cmpx_neq_f32 v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x3a,0x7d,0x01,0x77,0x39,0x05]
@@ -385,10 +445,20 @@
 # GFX12: v_cmpx_neq_f32 v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0x3b,0x7d,0xff,0x00,0x00,0x00]
 
 0xe9,0x04,0x12,0x7d,0x01,0x77,0x39,0x05
-# GFX12: v_cmpx_nge_f16 v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x12,0x7d,0x01,0x77,0x39,0x05]
+# GFX12-REAL16: v_cmpx_nge_f16 v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x12,0x7d,0x01,0x77,0x39,0x05]
+# GFX12-FAKE16: v_cmpx_nge_f16 v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x12,0x7d,0x01,0x77,0x39,0x05]
 
 0xea,0xfe,0x12,0x7d,0x7f,0x00,0x00,0x00
-# GFX12: v_cmpx_nge_f16 v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0x12,0x7d,0x7f,0x00,0x00,0x00]
+# GFX12-REAL16: v_cmpx_nge_f16 v127.l, v127.l dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0x12,0x7d,0x7f,0x00,0x00,0x00]
+# GFX12-FAKE16: v_cmpx_nge_f16 v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0x12,0x7d,0x7f,0x00,0x00,0x00]
+
+0xe9,0x04,0x13,0x7d,0x81,0x77,0x39,0x05
+# GFX12-REAL16: v_cmpx_nge_f16 v1.h, v2.h dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x13,0x7d,0x81,0x77,0x39,0x05]
+# GFX12-FAKE16: v_cmpx_nge_f16 v129/*Invalid register, operand has 'VGPR_32_Lo128' register class*/, v130/*Invalid register, operand has 'VGPR_32_Lo128' register class*/ dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x13,0x7d,0x81,0x77,0x39,0x05]
+
+0xea,0xfe,0x13,0x7d,0xff,0x00,0x00,0x00
+# GFX12-REAL16: v_cmpx_nge_f16 v127.h, v127.h dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0x13,0x7d,0xff,0x00,0x00,0x00]
+# GFX12-FAKE16: v_cmpx_nge_f16 v255/*Invalid register, operand has 'VGPR_32_Lo128' register class*/, v255/*Invalid register, operand has 'VGPR_32_Lo128' register class*/ dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0x13,0x7d,0xff,0x00,0x00,0x00]
 
 0xe9,0x04,0x32,0x7d,0x01,0x77,0x39,0x05
 # GFX12: v_cmpx_nge_f32 v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x32,0x7d,0x01,0x77,0x39,0x05]
@@ -397,10 +467,20 @@
 # GFX12: v_cmpx_nge_f32 v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0x33,0x7d,0xff,0x00,0x00,0x00]
 
 0xe9,0x04,0x16,0x7d,0x01,0x77,0x39,0x05
-# GFX12: v_cmpx_ngt_f16 v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x16,0x7d,0x01,0x77,0x39,0x05]
+# GFX12-REAL16: v_cmpx_ngt_f16 v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x16,0x7d,0x01,0x77,0x39,0x05]
+# GFX12-FAKE16: v_cmpx_ngt_f16 v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x16,0x7d,0x01,0x77,0x39,0x05]
 
 0xea,0xfe,0x16,0x7d,0x7f,0x00,0x00,0x00
-# GFX12: v_cmpx_ngt_f16 v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0x16,0x7d,0x7f,0x00,0x00,0x00]
+# GFX12-REAL16: v_cmpx_ngt_f16 v127.l, v127.l dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0x16,0x7d,0x7f,0x00,0x00,0x00]
+# GFX12-FAKE16: v_cmpx_ngt_f16 v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0x16,0x7d,0x7f,0x00,0x00,0x00]
+
+0xe9,0x04,0x17,0x7d,0x81,0x77,0x39,0x05
+# GFX12-REAL16: v_cmpx_ngt_f16 v1.h, v2.h dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x17,0x7d,0x81,0x77,0x39,0x05]
+# GFX12-FAKE16: v_cmpx_ngt_f16 v129/*Invalid register, operand has 'VGPR_32_Lo128' register class*/, v130/*Invalid register, operand has 'VGPR_32_Lo128' register class*/ dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x17,0x7d,0x81,0x77,0x39,0x05]
+
+0xea,0xfe,0x17,0x7d,0xff,0x00,0x00,0x00
+# GFX12-REAL16: v_cmpx_ngt_f16 v127.h, v127.h dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0x17,0x7d,0xff,0x00,0x00,0x00]
+# GFX12-FAKE16: v_cmpx_ngt_f16 v255/*Invalid register, operand has 'VGPR_32_Lo128' register class*/, v255/*Invalid register, operand has 'VGPR_32_Lo128' register class*/ dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0x17,0x7d,0xff,0x00,0x00,0x00]
 
 0xe9,0x04,0x36,0x7d,0x01,0x77,0x39,0x05
 # GFX12: v_cmpx_ngt_f32 v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x36,0x7d,0x01,0x77,0x39,0x05]
@@ -409,10 +489,20 @@
 # GFX12: v_cmpx_ngt_f32 v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0x37,0x7d,0xff,0x00,0x00,0x00]
 
 0xe9,0x04,0x18,0x7d,0x01,0x77,0x39,0x05
-# GFX12: v_cmpx_nle_f16 v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x18,0x7d,0x01,0x77,0x39,0x05]
+# GFX12-REAL16: v_cmpx_nle_f16 v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x18,0x7d,0x01,0x77,0x39,0x05]
+# GFX12-FAKE16: v_cmpx_nle_f16 v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x18,0x7d,0x01,0x77,0x39,0x05]
 
 0xea,0xfe,0x18,0x7d,0x7f,0x00,0x00,0x00
-# GFX12: v_cmpx_nle_f16 v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0x18,0x7d,0x7f,0x00,0x00,0x00]
+# GFX12-REAL16: v_cmpx_nle_f16 v127.l, v127.l dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0x18,0x7d,0x7f,0x00,0x00,0x00]
+# GFX12-FAKE16: v_cmpx_nle_f16 v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0x18,0x7d,0x7f,0x00,0x00,0x00]
+
+0xe9,0x04,0x19,0x7d,0x81,0x77,0x39,0x05
+# GFX12-REAL16: v_cmpx_nle_f16 v1.h, v2.h dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x19,0x7d,0x81,0x77,0x39,0x05]
+# GFX12-FAKE16: v_cmpx_nle_f16 v129/*Invalid register, operand has 'VGPR_32_Lo128' register class*/, v130/*Invalid register, operand has 'VGPR_32_Lo128' register class*/ dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x19,0x7d,0x81,0x77,0x39,0x05]
+
+0xea,0xfe,0x19,0x7d,0xff,0x00,0x00,0x00
+# GFX12-REAL16: v_cmpx_nle_f16 v127.h, v127.h dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0x19,0x7d,0xff,0x00,0x00,0x00]
+# GFX12-FAKE16: v_cmpx_nle_f16 v255/*Invalid register, operand has 'VGPR_32_Lo128' register class*/, v255/*Invalid register, operand has 'VGPR_32_Lo128' register class*/ dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0x19,0x7d,0xff,0x00,0x00,0x00]
 
 0xe9,0x04,0x38,0x7d,0x01,0x77,0x39,0x05
 # GFX12: v_cmpx_nle_f32 v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x38,0x7d,0x01,0x77,0x39,0x05]
@@ -421,10 +511,20 @@
 # GFX12: v_cmpx_nle_f32 v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0x39,0x7d,0xff,0x00,0x00,0x00]
 
 0xe9,0x04,0x14,0x7d,0x01,0x77,0x39,0x05
-# GFX12: v_cmpx_nlg_f16 v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x14,0x7d,0x01,0x77,0x39,0x05]
+# GFX12-REAL16: v_cmpx_nlg_f16 v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x14,0x7d,0x01,0x77,0x39,0x05]
+# GFX12-FAKE16: v_cmpx_nlg_f16 v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x14,0x7d,0x01,0x77,0x39,0x05]
 
 0xea,0xfe,0x14,0x7d,0x7f,0x00,0x00,0x00
-# GFX12: v_cmpx_nlg_f16 v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0x14,0x7d,0x7f,0x00,0x00,0x00]
+# GFX12-REAL16: v_cmpx_nlg_f16 v127.l, v127.l dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0x14,0x7d,0x7f,0x00,0x00,0x00]
+# GFX12-FAKE16: v_cmpx_nlg_f16 v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0x14,0x7d,0x7f,0x00,0x00,0x00]
+
+0xe9,0x04,0x15,0x7d,0x81,0x77,0x39,0x05
+# GFX12-REAL16: v_cmpx_nlg_f16 v1.h, v2.h dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x15,0x7d,0x81,0x77,0x39,0x05]
+# GFX12-FAKE16: v_cmpx_nlg_f16 v129/*Invalid register, operand has 'VGPR_32_Lo128' register class*/, v130/*Invalid register, operand has 'VGPR_32_Lo128' register class*/ dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x15,0x7d,0x81,0x77,0x39,0x05]
+
+0xea,0xfe,0x15,0x7d,0xff,0x00,0x00,0x00
+# GFX12-REAL16: v_cmpx_nlg_f16 v127.h, v127.h dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0x15,0x7d,0xff,0x00,0x00,0x00]
+# GFX12-FAKE16: v_cmpx_nlg_f16 v255/*Invalid register, operand has 'VGPR_32_Lo128' register class*/, v255/*Invalid register, operand has 'VGPR_32_Lo128' register class*/ dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0x15,0x7d,0xff,0x00,0x00,0x00]
 
 0xe9,0x04,0x34,0x7d,0x01,0x77,0x39,0x05
 # GFX12: v_cmpx_nlg_f32 v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x34,0x7d,0x01,0x77,0x39,0x05]
@@ -433,10 +533,20 @@
 # GFX12: v_cmpx_nlg_f32 v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0x35,0x7d,0xff,0x00,0x00,0x00]
 
 0xe9,0x04,0x1c,0x7d,0x01,0x77,0x39,0x05
-# GFX12: v_cmpx_nlt_f16 v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x1c,0x7d,0x01,0x77,0x39,0x05]
+# GFX12-REAL16: v_cmpx_nlt_f16 v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x1c,0x7d,0x01,0x77,0x39,0x05]
+# GFX12-FAKE16: v_cmpx_nlt_f16 v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x1c,0x7d,0x01,0x77,0x39,0x05]
 
 0xea,0xfe,0x1c,0x7d,0x7f,0x00,0x00,0x00
-# GFX12: v_cmpx_nlt_f16 v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0x1c,0x7d,0x7f,0x00,0x00,0x00]
+# GFX12-REAL16: v_cmpx_nlt_f16 v127.l, v127.l dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0x1c,0x7d,0x7f,0x00,0x00,0x00]
+# GFX12-FAKE16: v_cmpx_nlt_f16 v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0x1c,0x7d,0x7f,0x00,0x00,0x00]
+
+0xe9,0x04,0x1d,0x7d,0x81,0x77,0x39,0x05
+# GFX12-REAL16: v_cmpx_nlt_f16 v1.h, v2.h dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x1d,0x7d,0x81,0x77,0x39,0x05]
+# GFX12-FAKE16: v_cmpx_nlt_f16 v129/*Invalid register, operand has 'VGPR_32_Lo128' register class*/, v130/*Invalid register, operand has 'VGPR_32_Lo128' register class*/ dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x1d,0x7d,0x81,0x77,0x39,0x05]
+
+0xea,0xfe,0x1d,0x7d,0xff,0x00,0x00,0x00
+# GFX12-REAL16: v_cmpx_nlt_f16 v127.h, v127.h dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0x1d,0x7d,0xff,0x00,0x00,0x00]
+# GFX12-FAKE16: v_cmpx_nlt_f16 v255/*Invalid register, operand has 'VGPR_32_Lo128' register class*/, v255/*Invalid register, operand has 'VGPR_32_Lo128' register class*/ dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0x1d,0x7d,0xff,0x00,0x00,0x00]
 
 0xe9,0x04,0x3c,0x7d,0x01,0x77,0x39,0x05
 # GFX12: v_cmpx_nlt_f32 v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x3c,0x7d,0x01,0x77,0x39,0x05]
@@ -445,10 +555,20 @@
 # GFX12: v_cmpx_nlt_f32 v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0x3d,0x7d,0xff,0x00,0x00,0x00]
 
 0xe9,0x04,0x0e,0x7d,0x01,0x77,0x39,0x05
-# GFX12: v_cmpx_o_f16 v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x0e,0x7d,0x01,0x77,0x39,0x05]
+# GFX12-REAL16: v_cmpx_o_f16 v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x0e,0x7d,0x01,0x77,0x39,0x05]
+# GFX12-FAKE16: v_cmpx_o_f16 v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x0e,0x7d,0x01,0x77,0x39,0x05]
 
 0xea,0xfe,0x0e,0x7d,0x7f,0x00,0x00,0x00
-# GFX12: v_cmpx_o_f16 v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0x0e,0x7d,0x7f,0x00,0x00,0x00]
+# GFX12-REAL16: v_cmpx_o_f16 v127.l, v127.l dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0x0e,0x7d,0x7f,0x00,0x00,0x00]
+# GFX12-FAKE16: v_cmpx_o_f16 v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0x0e,0x7d,0x7f,0x00,0x00,0x00]
+
+0xe9,0x04,0x0f,0x7d,0x81,0x77,0x39,0x05
+# GFX12-REAL16: v_cmpx_o_f16 v1.h, v2.h dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x0f,0x7d,0x81,0x77,0x39,0x05]
+# GFX12-FAKE16: v_cmpx_o_f16 v129/*Invalid register, operand has 'VGPR_32_Lo128' register class*/, v130/*Invalid register, operand has 'VGPR_32_Lo128' register class*/ dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x0f,0x7d,0x81,0x77,0x39,0x05]
+
+0xea,0xfe,0x0f,0x7d,0xff,0x00,0x00,0x00
+# GFX12-REAL16: v_cmpx_o_f16 v127.h, v127.h dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0x0f,0x7d,0xff,0x00,0x00,0x00]
+# GFX12-FAKE16: v_cmpx_o_f16 v255/*Invalid register, operand has 'VGPR_32_Lo128' register class*/, v255/*Invalid register, operand has 'VGPR_32_Lo128' register class*/ dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0x0f,0x7d,0xff,0x00,0x00,0x00]
 
 0xe9,0x04,0x2e,0x7d,0x01,0x77,0x39,0x05
 # GFX12: v_cmpx_o_f32 v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x2e,0x7d,0x01,0x77,0x39,0x05]
@@ -457,10 +577,20 @@
 # GFX12: v_cmpx_o_f32 v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0x2f,0x7d,0xff,0x00,0x00,0x00]
 
 0xe9,0x04,0x10,0x7d,0x01,0x77,0x39,0x05
-# GFX12: v_cmpx_u_f16 v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x10,0x7d,0x01,0x77,0x39,0x05]
+# GFX12-REAL16: v_cmpx_u_f16 v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x10,0x7d,0x01,0x77,0x39,0x05]
+# GFX12-FAKE16: v_cmpx_u_f16 v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x10,0x7d,0x01,0x77,0x39,0x05]
 
 0xea,0xfe,0x10,0x7d,0x7f,0x00,0x00,0x00
-# GFX12: v_cmpx_u_f16 v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0x10,0x7d,0x7f,0x00,0x00,0x00]
+# GFX12-REAL16: v_cmpx_u_f16 v127.l, v127.l dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0x10,0x7d,0x7f,0x00,0x00,0x00]
+# GFX12-FAKE16: v_cmpx_u_f16 v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0x10,0x7d,0x7f,0x00,0x00,0x00]
+
+0xe9,0x04,0x11,0x7d,0x81,0x77,0x39,0x05
+# GFX12-REAL16: v_cmpx_u_f16 v1.h, v2.h dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x11,0x7d,0x81,0x77,0x39,0x05]
+# GFX12-FAKE16: v_cmpx_u_f16 v129/*Invalid register, operand has 'VGPR_32_Lo128' register class*/, v130/*Invalid register, operand has 'VGPR_32_Lo128' register class*/ dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x11,0x7d,0x81,0x77,0x39,0x05]
+
+0xea,0xfe,0x11,0x7d,0xff,0x00,0x00,0x00
+# GFX12-REAL16: v_cmpx_u_f16 v127.h, v127.h dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0x11,0x7d,0xff,0x00,0x00,0x00]
+# GFX12-FAKE16: v_cmpx_u_f16 v255/*Invalid register, operand has 'VGPR_32_Lo128' register class*/, v255/*Invalid register, operand has 'VGPR_32_Lo128' register class*/ dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0x11,0x7d,0xff,0x00,0x00,0x00]
 
 0xe9,0x04,0x30,0x7d,0x01,0x77,0x39,0x05
 # GFX12: v_cmpx_u_f32 v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x30,0x7d,0x01,0x77,0x39,0x05]
diff --git a/llvm/test/MC/Disassembler/AMDGPU/gfx950_dasm_ds_read_tr.txt b/llvm/test/MC/Disassembler/AMDGPU/gfx950_dasm_ds_read_tr.txt
index 10310f7ad1f3d..0b5dba24612ca 100644
--- a/llvm/test/MC/Disassembler/AMDGPU/gfx950_dasm_ds_read_tr.txt
+++ b/llvm/test/MC/Disassembler/AMDGPU/gfx950_dasm_ds_read_tr.txt
@@ -35,3 +35,9 @@
 
 # GFX950: ds_read_b96_tr_b6 v[2:4], v2 offset:64  ; encoding: [0x40,0x00,0xc2,0xd9,0x02,0x00,0x00,0x02]
 0x40,0x00,0xc2,0xd9,0x02,0x00,0x00,0x02
+
+# GFX950: ds_read_b96_tr_b6 v[1:3], v0            ; encoding: [0x00,0x00,0xc2,0xd9,0x00,0x00,0x00,0x01]
+0x00,0x00,0xc2,0xd9,0x00,0x00,0x00,0x01
+
+# GFX950: ds_read_b96_tr_b6 v[1:3], v2 offset:64  ; encoding: [0x40,0x00,0xc2,0xd9,0x02,0x00,0x00,0x01]
+0x40,0x00,0xc2,0xd9,0x02,0x00,0x00,0x01
diff --git a/llvm/test/MC/Disassembler/RISCV/vmask-carry-in.txt b/llvm/test/MC/Disassembler/RISCV/vmask-carry-in.txt
new file mode 100644
index 0000000000000..e9af01ac60b43
--- /dev/null
+++ b/llvm/test/MC/Disassembler/RISCV/vmask-carry-in.txt
@@ -0,0 +1,69 @@
+# RUN: llvm-mc -triple=riscv64 -disassemble -show-inst --mattr=+v %s \
+# RUN:   --M no-aliases | FileCheck %s
+
+# Check if there is a MCOperand for the carry-in mask.
+
+[0x57,0x04,0x4a,0x5c]
+# CHECK: <MCInst #{{[0-9]+}} VMERGE_VVM
+# CHECK-COUNT-4: MCOperand Reg
+
+[0x57,0x44,0x45,0x5c]
+# CHECK: <MCInst #{{[0-9]+}} VMERGE_VXM
+# CHECK-COUNT-4: MCOperand Reg
+
+[0x57,0xb4,0x47,0x5c]
+# CHECK: <MCInst #{{[0-9]+}} VMERGE_VIM
+# CHECK-NEXT: MCOperand Reg
+# CHECK-NEXT: MCOperand Reg
+# CHECK-NEXT: MCOperand Imm
+# CHECK-NEXT: MCOperand Reg
+
+[0x57,0x04,0x4a,0x40]
+# CHECK: <MCInst #{{[0-9]+}} VADC_VVM
+# CHECK-COUNT-4: MCOperand Reg
+
+[0x57,0x44,0x45,0x40]
+# CHECK: <MCInst #{{[0-9]+}} VADC_VXM
+# CHECK-COUNT-4: MCOperand Reg
+
+[0x57,0xb4,0x47,0x40]
+# CHECK: <MCInst #{{[0-9]+}} VADC_VIM
+# CHECK-NEXT: MCOperand Reg
+# CHECK-NEXT: MCOperand Reg
+# CHECK-NEXT: MCOperand Imm
+# CHECK-NEXT: MCOperand Reg
+
+[0x57,0x04,0x4a,0x44]
+# CHECK: <MCInst #{{[0-9]+}} VMADC_VVM
+# CHECK-COUNT-4: MCOperand Reg
+
+[0x57,0x44,0x45,0x44]
+# CHECK: <MCInst #{{[0-9]+}} VMADC_VXM
+# CHECK-COUNT-4: MCOperand Reg
+
+[0x57,0xb4,0x47,0x44]
+# CHECK: <MCInst #{{[0-9]+}} VMADC_VIM
+# CHECK-NEXT: MCOperand Reg
+# CHECK-NEXT: MCOperand Reg
+# CHECK-NEXT: MCOperand Imm
+# CHECK-NEXT: MCOperand Reg
+
+[0x57,0x04,0x4a,0x48]
+# CHECK: <MCInst #{{[0-9]+}} VSBC_VVM
+# CHECK-COUNT-4: MCOperand Reg
+
+[0x57,0x44,0x45,0x48]
+# CHECK: <MCInst #{{[0-9]+}} VSBC_VXM
+# CHECK-COUNT-4: MCOperand Reg
+
+[0x57,0x04,0x4a,0x4c]
+# CHECK: <MCInst #{{[0-9]+}} VMSBC_VVM
+# CHECK-COUNT-4: MCOperand Reg
+
+[0x57,0x44,0x45,0x4c]
+# CHECK: <MCInst #{{[0-9]+}} VMSBC_VXM
+# CHECK-COUNT-4: MCOperand Reg
+
+[0x57,0x54,0x45,0x5c]
+# CHECK: <MCInst #{{[0-9]+}} VFMERGE_VFM
+# CHECK-COUNT-4: MCOperand Reg
diff --git a/llvm/test/MC/Disassembler/Xtensa/windowed.txt b/llvm/test/MC/Disassembler/Xtensa/windowed.txt
new file mode 100644
index 0000000000000..34c51dd32b6be
--- /dev/null
+++ b/llvm/test/MC/Disassembler/Xtensa/windowed.txt
@@ -0,0 +1,71 @@
+# NOTE: Assertions have been autogenerated by utils/update_mc_test_checks.py UTC_ARGS: --version 5
+# RUN: llvm-mc -triple=xtensa -mattr=+windowed -disassemble %s | FileCheck -check-prefixes=CHECK-WINDOWED %s
+# RUN: not llvm-mc -triple=xtensa -disassemble %s 2>&1 | FileCheck --implicit-check-not=warning: -check-prefixes=CHECK-CORE %s
+
+## Verify that binary code is correctly disassembled with
+## windowed register option enabled. Also verify that dissasembling without
+## windowed register option generates warnings.
+
+[0x36,0x03,0x01]
+# CHECK-WINDOWED: entry	a3, 128
+# CHECK-CORE: :[[@LINE-2]]:2: warning: invalid instruction encoding
+
+[0x30,0x14,0x00]
+# CHECK-WINDOWED: movsp	a3, a4
+# CHECK-CORE: :[[@LINE-2]]:2: warning: invalid instruction encoding
+
+[0x15,0x10,0x00]
+# CHECK-WINDOWED: call4	. +260
+# CHECK-CORE: :[[@LINE-2]]:2: warning: invalid instruction encoding
+
+[0x25,0x10,0x00]
+# CHECK-WINDOWED: call8	. +260
+# CHECK-CORE: :[[@LINE-2]]:2: warning: invalid instruction encoding
+
+[0x35,0x10,0x00]
+# CHECK-WINDOWED: call12	. +260
+# CHECK-CORE: :[[@LINE-2]]:2: warning: invalid instruction encoding
+
+[0xd0,0x03,0x00]
+# CHECK-WINDOWED: callx4	a3
+# CHECK-CORE: :[[@LINE-2]]:2: warning: invalid instruction encoding
+
+[0xe0,0x03,0x00]
+# CHECK-WINDOWED: callx8	a3
+# CHECK-CORE: :[[@LINE-2]]:2: warning: invalid instruction encoding
+
+[0xf0,0x03,0x00]
+# CHECK-WINDOWED: callx12	a3
+# CHECK-CORE: :[[@LINE-2]]:2: warning: invalid instruction encoding
+
+[0x90,0x00,0x00]
+# CHECK-WINDOWED: retw
+# CHECK-CORE: :[[@LINE-2]]:2: warning: invalid instruction encoding
+
+[0x20,0x80,0x40]
+# CHECK-WINDOWED: rotw	2
+# CHECK-CORE: :[[@LINE-2]]:2: warning: invalid instruction encoding
+
+[0x30,0xd4,0x09]
+# CHECK-WINDOWED: l32e	a3, a4, -12
+# CHECK-CORE: :[[@LINE-2]]:2: warning: invalid instruction encoding
+
+[0x30,0xd4,0x49]
+# CHECK-WINDOWED: s32e	a3, a4, -12
+# CHECK-CORE: :[[@LINE-2]]:2: warning: invalid instruction encoding
+
+[0x00,0x34,0x00]
+# CHECK-WINDOWED: rfwo
+# CHECK-CORE: :[[@LINE-2]]:2: warning: invalid instruction encoding
+
+[0x00,0x35,0x00]
+# CHECK-WINDOWED: rfwu
+# CHECK-CORE: :[[@LINE-2]]:2: warning: invalid instruction encoding
+
+[0x30,0x48,0x61]
+# CHECK-WINDOWED: xsr	a3, windowbase
+# CHECK-CORE: :[[@LINE-2]]:2: warning: invalid instruction encoding
+
+[0x30,0x49,0x61]
+# CHECK-WINDOWED: xsr	a3, windowstart
+# CHECK-CORE: :[[@LINE-2]]:2: warning: invalid instruction encoding
diff --git a/llvm/test/MC/Disassembler/Xtensa/windowed_code_density.txt b/llvm/test/MC/Disassembler/Xtensa/windowed_code_density.txt
new file mode 100644
index 0000000000000..69d7d424d9682
--- /dev/null
+++ b/llvm/test/MC/Disassembler/Xtensa/windowed_code_density.txt
@@ -0,0 +1,11 @@
+# NOTE: Assertions have been autogenerated by utils/update_mc_test_checks.py UTC_ARGS: --version 5
+# RUN: llvm-mc -triple=xtensa -mattr=+windowed,+density -disassemble %s | FileCheck -check-prefixes=CHECK-WINDOWED-DENSITY %s
+# RUN: not llvm-mc -triple=xtensa -disassemble %s 2>&1 | FileCheck --implicit-check-not=warning: -check-prefixes=CHECK-CORE %s
+
+## Verify that binary code is correctly disassembled with windowed register
+## and code density options enabled. Also verify that
+## dissasembling without these options generates warnings.
+
+[0x1d,0xf0]
+# CHECK-WINDOWED-DENSITY: retw.n
+# CHECK-CORE: :[[@LINE-2]]:2: warning: invalid instruction encoding
diff --git a/llvm/test/MC/RISCV/rv32c-invalid.s b/llvm/test/MC/RISCV/rv32c-invalid.s
index 9fbd5e9f117f9..1027c08505d50 100644
--- a/llvm/test/MC/RISCV/rv32c-invalid.s
+++ b/llvm/test/MC/RISCV/rv32c-invalid.s
@@ -19,25 +19,25 @@ c.xor  t2, a0 # CHECK: :[[@LINE]]:8: error: invalid operand for instruction
 c.sub  a0, s8 # CHECK: :[[@LINE]]:12: error: invalid operand for instruction
 
 ## GPRNoX0
-c.lwsp  x0, 4(sp) # CHECK: :[[@LINE]]:9: error: invalid operand for instruction
-c.lwsp  zero, 4(sp) # CHECK: :[[@LINE]]:9: error: invalid operand for instruction
-c.jr  x0 # CHECK: :[[@LINE]]:7: error: invalid operand for instruction
-c.jalr  zero # CHECK: :[[@LINE]]:9: error: invalid operand for instruction
+c.lwsp  x0, 4(sp) # CHECK: :[[@LINE]]:9: error: register must be a GPR excluding zero (x0)
+c.lwsp  zero, 4(sp) # CHECK: :[[@LINE]]:9: error: register must be a GPR excluding zero (x0)
+c.jr  x0 # CHECK: :[[@LINE]]:7: error: register must be a GPR excluding zero (x0)
+c.jalr  zero # CHECK: :[[@LINE]]:9: error: register must be a GPR excluding zero (x0)
 c.addi  x0, x0, 1 # CHECK: :[[@LINE]]:13: error: immediate must be zero
 c.li  zero, 2 # CHECK: :[[@LINE]]:1: error: instruction requires the following: RVC Hint Instructions{{$}}
 c.slli  zero, zero, 4 # CHECK: :[[@LINE]]:15: error: invalid operand for instruction
 c.mv  zero, s0 # CHECK: :[[@LINE]]:1: error: instruction requires the following: RVC Hint Instructions{{$}}
-c.mv  ra, x0 # CHECK: :[[@LINE]]:11: error: invalid operand for instruction
+c.mv  ra, x0 # CHECK: :[[@LINE]]:11: error: register must be a GPR excluding zero (x0)
 c.add  ra, ra, x0 # CHECK: :[[@LINE]]:16: error: invalid operand for instruction
 c.add  zero, zero, sp # CHECK: :[[@LINE]]:14: error: invalid operand for instruction
 
 ## GPRNoX0X2
 c.lui x0, 4 # CHECK: :[[@LINE]]:1: error: instruction requires the following: RVC Hint Instructions{{$}}
-c.lui x2, 4 # CHECK: :[[@LINE]]:7: error: invalid operand for instruction
+c.lui x2, 4 # CHECK: :[[@LINE]]:7: error: register must be a GPR excluding zero (x0) and sp (x2){{$}}
 
 ## SP
-c.addi4spn  a0, a0, 12 # CHECK: :[[@LINE]]:17: error: invalid operand for instruction
-c.addi16sp  t0, 16 # CHECK: :[[@LINE]]:13: error: invalid operand for instruction
+c.addi4spn  a0, a0, 12 # CHECK: :[[@LINE]]:17: error: register must be sp (x2)
+c.addi16sp  t0, 16 # CHECK: :[[@LINE]]:13: error: register must be sp (x2)
 
 # Out of range immediates
 
diff --git a/llvm/test/MC/RISCV/rv32zicfiss-invalid.s b/llvm/test/MC/RISCV/rv32zicfiss-invalid.s
deleted file mode 100644
index 048df67e8a646..0000000000000
--- a/llvm/test/MC/RISCV/rv32zicfiss-invalid.s
+++ /dev/null
@@ -1,17 +0,0 @@
-# RUN: not llvm-mc %s -triple=riscv32 -mattr=+experimental-zicfiss,+c -M no-aliases -show-encoding \
-# RUN:     2>&1 | FileCheck -check-prefixes=CHECK-ERR %s
-
-# CHECK-ERR: error: invalid operand for instruction
-sspopchk a1
-
-# CHECK-ERR: error: invalid operand for instruction
-c.sspush t0
-
-# CHECK-ERR: error: invalid operand for instruction
-c.sspopchk ra
-
-# CHECK-ERR: error: invalid operand for instruction
-sspush a0
-
-# CHECK-ERR: error: invalid operand for instruction
-ssrdp zero
diff --git a/llvm/test/MC/RISCV/rv64c-invalid.s b/llvm/test/MC/RISCV/rv64c-invalid.s
index aecc3b4d664ff..9b0a3244f3aac 100644
--- a/llvm/test/MC/RISCV/rv64c-invalid.s
+++ b/llvm/test/MC/RISCV/rv64c-invalid.s
@@ -8,8 +8,8 @@ c.addw   a0, a7 # CHECK: :[[@LINE]]:14: error: invalid operand for instruction
 c.subw   a0, a6 # CHECK: :[[@LINE]]:14: error: invalid operand for instruction
 
 ## GPRNoX0
-c.ldsp  x0, 4(sp) # CHECK: :[[@LINE]]:9: error: invalid operand for instruction
-c.ldsp  zero, 4(sp) # CHECK: :[[@LINE]]:9: error: invalid operand for instruction
+c.ldsp  x0, 4(sp) # CHECK: :[[@LINE]]:9: error: register must be a GPR excluding zero (x0)
+c.ldsp  zero, 4(sp) # CHECK: :[[@LINE]]:9: error: register must be a GPR excluding zero (x0)
 
 # Out of range immediates
 
diff --git a/llvm/test/MC/RISCV/rv64zicfiss-invalid.s b/llvm/test/MC/RISCV/rv64zicfiss-invalid.s
deleted file mode 100644
index fc69c68a477d6..0000000000000
--- a/llvm/test/MC/RISCV/rv64zicfiss-invalid.s
+++ /dev/null
@@ -1,17 +0,0 @@
-# RUN: not llvm-mc %s -triple=riscv64 -mattr=+experimental-zicfiss,+c -M no-aliases -show-encoding \
-# RUN:     2>&1 | FileCheck -check-prefixes=CHECK-ERR %s
-
-# CHECK-ERR: error: invalid operand for instruction
-sspopchk a1
-
-# CHECK-ERR: error: invalid operand for instruction
-c.sspush t0
-
-# CHECK-ERR: error: invalid operand for instruction
-c.sspopchk ra
-
-# CHECK-ERR: error: invalid operand for instruction
-sspush a0
-
-# CHECK-ERR: error: invalid operand for instruction
-ssrdp zero
diff --git a/llvm/test/MC/RISCV/rvc-hints-invalid.s b/llvm/test/MC/RISCV/rvc-hints-invalid.s
index 50704010fc82a..2a7a6addd31ab 100644
--- a/llvm/test/MC/RISCV/rvc-hints-invalid.s
+++ b/llvm/test/MC/RISCV/rvc-hints-invalid.s
@@ -11,9 +11,9 @@ c.li x0, 42 # CHECK: :[[@LINE]]:10: error: immediate must be an integer in the r
 
 c.lui x0, 0 # CHECK: :[[@LINE]]:11: error: immediate must be in [0xfffe0, 0xfffff] or [1, 31]
 
-c.mv x0, x0 # CHECK: :[[@LINE]]:10: error: invalid operand for instruction
+c.mv x0, x0 # CHECK: :[[@LINE]]:10: error: register must be a GPR excluding zero (x0)
 
-c.add x0, x0 # CHECK: :[[@LINE]]:11: error: invalid operand for instruction
+c.add x0, x0 # CHECK: :[[@LINE]]:11: error: register must be a GPR excluding zero (x0)
 
 c.slli x0, 0 # CHECK-RV32: :[[@LINE]]:12: error: immediate must be an integer in the range [1, 31]
 c.slli x0, 32 # CHECK-RV32: :[[@LINE]]:12: error: immediate must be an integer in the range [1, 31]
diff --git a/llvm/test/MC/RISCV/rvv/vmask-carry-in.s b/llvm/test/MC/RISCV/rvv/vmask-carry-in.s
new file mode 100644
index 0000000000000..2eb0a2bd0ef85
--- /dev/null
+++ b/llvm/test/MC/RISCV/rvv/vmask-carry-in.s
@@ -0,0 +1,69 @@
+# RUN: llvm-mc -triple=riscv64 -show-inst --mattr=+v %s \
+# RUN:   --M no-aliases | FileCheck %s
+
+# Check if there is a MCOperand for the carry-in mask.
+
+vmerge.vvm v8, v4, v20, v0
+# CHECK: <MCInst #{{[0-9]+}} VMERGE_VVM
+# CHECK-COUNT-4: MCOperand Reg
+
+vmerge.vxm v8, v4, a0, v0
+# CHECK: <MCInst #{{[0-9]+}} VMERGE_VXM
+# CHECK-COUNT-4: MCOperand Reg
+
+vmerge.vim v8, v4, 15, v0
+# CHECK: <MCInst #{{[0-9]+}} VMERGE_VIM
+# CHECK-NEXT: MCOperand Reg
+# CHECK-NEXT: MCOperand Reg
+# CHECK-NEXT: MCOperand Imm
+# CHECK-NEXT: MCOperand Reg
+
+vadc.vvm v8, v4, v20, v0
+# CHECK: <MCInst #{{[0-9]+}} VADC_VVM
+# CHECK-COUNT-4: MCOperand Reg
+
+vadc.vxm v8, v4, a0, v0
+# CHECK: <MCInst #{{[0-9]+}} VADC_VXM
+# CHECK-COUNT-4: MCOperand Reg
+
+vadc.vim v8, v4, 15, v0
+# CHECK: <MCInst #{{[0-9]+}} VADC_VIM
+# CHECK-NEXT: MCOperand Reg
+# CHECK-NEXT: MCOperand Reg
+# CHECK-NEXT: MCOperand Imm
+# CHECK-NEXT: MCOperand Reg
+
+vmadc.vvm v8, v4, v20, v0
+# CHECK: <MCInst #{{[0-9]+}} VMADC_VVM
+# CHECK-COUNT-4: MCOperand Reg
+
+vmadc.vxm v8, v4, a0, v0
+# CHECK: <MCInst #{{[0-9]+}} VMADC_VXM
+# CHECK-COUNT-4: MCOperand Reg
+
+vmadc.vim v8, v4, 15, v0
+# CHECK: <MCInst #{{[0-9]+}} VMADC_VIM
+# CHECK-NEXT: MCOperand Reg
+# CHECK-NEXT: MCOperand Reg
+# CHECK-NEXT: MCOperand Imm
+# CHECK-NEXT: MCOperand Reg
+
+vsbc.vvm v8, v4, v20, v0
+# CHECK: <MCInst #{{[0-9]+}} VSBC_VVM
+# CHECK-COUNT-4: MCOperand Reg
+
+vsbc.vxm v8, v4, a0, v0
+# CHECK: <MCInst #{{[0-9]+}} VSBC_VXM
+# CHECK-COUNT-4: MCOperand Reg
+
+vmsbc.vvm v8, v4, v20, v0
+# CHECK: <MCInst #{{[0-9]+}} VMSBC_VVM
+# CHECK-COUNT-4: MCOperand Reg
+
+vmsbc.vxm v8, v4, a0, v0
+# CHECK: <MCInst #{{[0-9]+}} VMSBC_VXM
+# CHECK-COUNT-4: MCOperand Reg
+
+vfmerge.vfm v8, v4, fa0, v0
+# CHECK: <MCInst #{{[0-9]+}} VFMERGE_VFM
+# CHECK-COUNT-4: MCOperand Reg
diff --git a/llvm/test/MC/RISCV/xmips-invalid.s b/llvm/test/MC/RISCV/xmips-invalid.s
new file mode 100644
index 0000000000000..a1c1fd0666e0a
--- /dev/null
+++ b/llvm/test/MC/RISCV/xmips-invalid.s
@@ -0,0 +1,26 @@
+# RUN: not llvm-mc -triple=riscv64 < %s 2>&1 | FileCheck %s -check-prefixes=CHECK-FEATURE
+# RUN: not llvm-mc -triple=riscv64 -mattr=+xmipslsp,+xmipscmove < %s 2>&1 | FileCheck %s
+
+mips.ccmov x0, x1, 0x10
+# CHECK: error: invalid operand for instruction
+
+mips.ccmov x10
+# CHECK: error: too few operands for instruction
+
+mips.ccmov	s0, s1, s2, s3
+# CHECK-FEATURE: error: instruction requires the following: 'Xmipscmove' ('mips.ccmov' instruction)
+
+mips.lwp x10, x11
+# CHECK: error: too few operands for instruction
+
+mips.ldp x9, 0x20
+# CHECK: error: invalid operand for instruction
+
+mips.lwp x11, x12, 0(x13)
+# CHECK-FEATURE: error: instruction requires the following: 'Xmipslsp' (load and store pair instructions)
+
+mips.swp x18, x19, 8(x2)
+# CHECK-FEATURE: error: instruction requires the following: 'Xmipslsp' (load and store pair instructions)
+
+mips.sdp 0x10, x3, 12(x4)
+# CHECK: error: invalid operand for instruction
diff --git a/llvm/test/MC/RISCV/xmips-valid.s b/llvm/test/MC/RISCV/xmips-valid.s
new file mode 100644
index 0000000000000..ba256a823f511
--- /dev/null
+++ b/llvm/test/MC/RISCV/xmips-valid.s
@@ -0,0 +1,35 @@
+# RUN: llvm-mc %s -triple=riscv64 -mattr=+xmipslsp,+xmipscmove -M no-aliases -show-encoding \
+# RUN:   | FileCheck -check-prefixes=CHECK-INST,CHECK-ENC %s
+# RUN: llvm-mc -filetype=obj -triple=riscv64 -mattr=+xmipslsp,+xmipscmove < %s \
+# RUN:   | llvm-objdump --mattr=+xmipslsp,+xmipscmove -M no-aliases -d - \
+# RUN:   | FileCheck -check-prefix=CHECK-DIS %s
+
+# CHECK-INST: mips.ccmov	s0, s1, s2, s3
+# CHECK-ENC:  encoding: [0x0b,0x34,0x99,0x9e]
+mips.ccmov s0, s1, s2, s3
+
+# CHECK-DIS: mips.ccmov	s0, s1, s2, s3
+
+# CHECK-INST: mips.swp s3, s2, 0(sp)
+# CHECK-ENC: encoding: [0x8b,0x50,0x31,0x91]
+mips.swp s3, s2, 0(sp)
+
+# CHECK-DIS: mips.swp s3, s2, 0x0(sp)
+
+# CHECK-INST: mips.sdp s5, s6, 16(s7)
+# CHECK-ENC: encoding: [0x0b,0xd8,0x5b,0xb1]
+mips.sdp s5, s6, 16(s7)
+
+# CHECK-DIS: mips.sdp s5, s6, 0x10(s7)
+
+# CHECK-INST: mips.ldp s1, s2, 8(sp)
+# CHECK-ENC: encoding: [0x8b,0x44,0x81,0x90]
+mips.ldp s1, s2, 8(sp)
+
+# CHECK-DIS: mips.ldp s1, s2, 0x8(sp)
+
+# CHECK-INST: mips.lwp a0, a1, 20(a2)
+# CHECK-ENC: encoding: [0x0b,0x45,0x56,0x59]
+mips.lwp x10, x11, 20(x12)
+
+# CHECK-DIS: mips.lwp a0, a1, 0x14(a2)
diff --git a/llvm/test/MC/RISCV/xqcia-invalid.s b/llvm/test/MC/RISCV/xqcia-invalid.s
index a410fb63fad9c..aa4d7143a4919 100644
--- a/llvm/test/MC/RISCV/xqcia-invalid.s
+++ b/llvm/test/MC/RISCV/xqcia-invalid.s
@@ -4,143 +4,173 @@
 # RUN: not llvm-mc -triple riscv32 -mattr=-experimental-xqcia < %s 2>&1 \
 # RUN:     | FileCheck -check-prefixes=CHECK,CHECK-MINUS %s
 
-# CHECK: :[[@LINE+1]]:20: error: invalid operand for instruction
+# CHECK-PLUS: :[[@LINE+2]]:20: error: register must be a GPR excluding zero (x0)
+# CHECK-MINUS: :[[@LINE+1]]:20: error: invalid operand for instruction
 qc.slasat x10, x3, 17
 
 # CHECK: :[[@LINE+1]]:1: error: too few operands for instruction
 qc.slasat x10, x3
 
-# CHECK: :[[@LINE+1]]:11: error: invalid operand for instruction
+# CHECK-PLUS: :[[@LINE+2]]:11: error: register must be a GPR excluding zero (x0)
+# CHECK-MINUS: :[[@LINE+1]]:11: error: invalid operand for instruction
 qc.slasat x0, x3, x17
 
-# CHECK: :[[@LINE+1]]:16: error: invalid operand for instruction
+# CHECK-PLUS: :[[@LINE+2]]:16: error: register must be a GPR excluding zero (x0)
+# CHECK-MINUS: :[[@LINE+1]]:16: error: invalid operand for instruction
 qc.slasat x10, x0, x17
 
-# CHECK: :[[@LINE+1]]:20: error: invalid operand for instruction
+# CHECK-PLUS: :[[@LINE+2]]:20: error: register must be a GPR excluding zero (x0)
+# CHECK-MINUS: :[[@LINE+1]]:20: error: invalid operand for instruction
 qc.slasat x10, x3, x0
 
 # CHECK-MINUS: :[[@LINE+1]]:1: error: instruction requires the following: 'Xqcia' (Qualcomm uC Arithmetic Extension)
 qc.slasat x10, x3, x17
 
 
-# CHECK: :[[@LINE+1]]:21: error: invalid operand for instruction
+# CHECK-PLUS: :[[@LINE+2]]:21: error: register must be a GPR excluding zero (x0)
+# CHECK-MINUS: :[[@LINE+1]]:21: error: invalid operand for instruction
 qc.sllsat x23, x25, 27
 
 # CHECK: :[[@LINE+1]]:1: error: too few operands for instruction
 qc.sllsat x23, x25
 
-# CHECK: :[[@LINE+1]]:11: error: invalid operand for instruction
+# CHECK-PLUS: :[[@LINE+2]]:11: error: register must be a GPR excluding zero (x0)
+# CHECK-MINUS: :[[@LINE+1]]:11: error: invalid operand for instruction
 qc.sllsat x0, x25, x27
 
-# CHECK: :[[@LINE+1]]:16: error: invalid operand for instruction
+# CHECK-PLUS: :[[@LINE+2]]:16: error: register must be a GPR excluding zero (x0)
+# CHECK-MINUS: :[[@LINE+1]]:16: error: invalid operand for instruction
 qc.sllsat x23, x0, x27
 
-# CHECK: :[[@LINE+1]]:21: error: invalid operand for instruction
+# CHECK-PLUS: :[[@LINE+2]]:21: error: register must be a GPR excluding zero (x0)
+# CHECK-MINUS: :[[@LINE+1]]:21: error: invalid operand for instruction
 qc.sllsat x23, x25, x0
 
 # CHECK-MINUS: :[[@LINE+1]]:1: error: instruction requires the following: 'Xqcia' (Qualcomm uC Arithmetic Extension)
 qc.sllsat x23, x25, x27
 
 
-# CHECK: :[[@LINE+1]]:21: error: invalid operand for instruction
+# CHECK-PLUS: :[[@LINE+2]]:21: error: register must be a GPR excluding zero (x0)
+# CHECK-MINUS: :[[@LINE+1]]:21: error: invalid operand for instruction
 qc.addsat x17, x14, 7
 
 # CHECK: :[[@LINE+1]]:1: error: too few operands for instruction
 qc.addsat x17, x14
 
-# CHECK: :[[@LINE+1]]:11: error: invalid operand for instruction
+# CHECK-PLUS: :[[@LINE+2]]:11: error: register must be a GPR excluding zero (x0)
+# CHECK-MINUS: :[[@LINE+1]]:11: error: invalid operand for instruction
 qc.addsat x0, x14, x7
 
-# CHECK: :[[@LINE+1]]:16: error: invalid operand for instruction
+# CHECK-PLUS: :[[@LINE+2]]:16: error: register must be a GPR excluding zero (x0)
+# CHECK-MINUS: :[[@LINE+1]]:16: error: invalid operand for instruction
 qc.addsat x17, x0, x7
 
-# CHECK: :[[@LINE+1]]:21: error: invalid operand for instruction
+# CHECK-PLUS: :[[@LINE+2]]:21: error: register must be a GPR excluding zero (x0)
+# CHECK-MINUS: :[[@LINE+1]]:21: error: invalid operand for instruction
 qc.addsat x17, x14, x0
 
 # CHECK-MINUS: :[[@LINE+1]]:1: error: instruction requires the following: 'Xqcia' (Qualcomm uC Arithmetic Extension)
 qc.addsat x17, x14, x7
 
 
-# CHECK: :[[@LINE+1]]:21: error: invalid operand for instruction
+# CHECK-PLUS: :[[@LINE+2]]:21: error: register must be a GPR excluding zero (x0)
+# CHECK-MINUS: :[[@LINE+1]]:21: error: invalid operand for instruction
 qc.addusat x8, x18, 28
 
 # CHECK: :[[@LINE+1]]:1: error: too few operands for instruction
 qc.addusat x8, x18
 
-# CHECK: :[[@LINE+1]]:12: error: invalid operand for instruction
+# CHECK-PLUS: :[[@LINE+2]]:12: error: register must be a GPR excluding zero (x0)
+# CHECK-MINUS: :[[@LINE+1]]:12: error: invalid operand for instruction
 qc.addusat x0, x18, x28
 
-# CHECK: :[[@LINE+1]]:16: error: invalid operand for instruction
+# CHECK-PLUS: :[[@LINE+2]]:16: error: register must be a GPR excluding zero (x0)
+# CHECK-MINUS: :[[@LINE+1]]:16: error: invalid operand for instruction
 qc.addusat x8, x0, x28
 
-# CHECK: :[[@LINE+1]]:21: error: invalid operand for instruction
+# CHECK-PLUS: :[[@LINE+2]]:21: error: register must be a GPR excluding zero (x0)
+# CHECK-MINUS: :[[@LINE+1]]:21: error: invalid operand for instruction
 qc.addusat x8, x18, x0
 
 # CHECK-MINUS: :[[@LINE+1]]:1: error: instruction requires the following: 'Xqcia' (Qualcomm uC Arithmetic Extension)
 qc.addusat x8, x18, x28
 
 
-# CHECK: :[[@LINE+1]]:20: error: invalid operand for instruction
+# CHECK-PLUS: :[[@LINE+2]]:20: error: register must be a GPR excluding zero (x0)
+# CHECK-MINUS: :[[@LINE+1]]:20: error: invalid operand for instruction
 qc.subsat x22, x2, 12
 
 # CHECK: :[[@LINE+1]]:1: error: too few operands for instruction
 qc.subsat x22, x2
 
-# CHECK: :[[@LINE+1]]:11: error: invalid operand for instruction
+# CHECK-PLUS: :[[@LINE+2]]:11: error: register must be a GPR excluding zero (x0)
+# CHECK-MINUS: :[[@LINE+1]]:11: error: invalid operand for instruction
 qc.subsat x0, x2, x12
 
-# CHECK: :[[@LINE+1]]:16: error: invalid operand for instruction
+# CHECK-PLUS: :[[@LINE+2]]:16: error: register must be a GPR excluding zero (x0)
+# CHECK-MINUS: :[[@LINE+1]]:16: error: invalid operand for instruction
 qc.subsat x22, x0, x12
 
-# CHECK: :[[@LINE+1]]:20: error: invalid operand for instruction
+# CHECK-PLUS: :[[@LINE+2]]:20: error: register must be a GPR excluding zero (x0)
+# CHECK-MINUS: :[[@LINE+1]]:20: error: invalid operand for instruction
 qc.subsat x22, x2, x0
 
 # CHECK-MINUS: :[[@LINE+1]]:1: error: instruction requires the following: 'Xqcia' (Qualcomm uC Arithmetic Extension)
 qc.subsat x22, x2, x12
 
 
-# CHECK: :[[@LINE+1]]:21: error: invalid operand for instruction
+# CHECK-PLUS: :[[@LINE+2]]:21: error: register must be a GPR excluding zero (x0)
+# CHECK-MINUS: :[[@LINE+1]]:21: error: invalid operand for instruction
 qc.subusat x9, x14, 17
 
 # CHECK: :[[@LINE+1]]:1: error: too few operands for instruction
 qc.subusat x9, x14
 
-# CHECK: :[[@LINE+1]]:12: error: invalid operand for instruction
+# CHECK-PLUS: :[[@LINE+2]]:12: error: register must be a GPR excluding zero (x0)
+# CHECK-MINUS: :[[@LINE+1]]:12: error: invalid operand for instruction
 qc.subusat x0, x14, x17
 
-# CHECK: :[[@LINE+1]]:16: error: invalid operand for instruction
+# CHECK-PLUS: :[[@LINE+2]]:16: error: register must be a GPR excluding zero (x0)
+# CHECK-MINUS: :[[@LINE+1]]:16: error: invalid operand for instruction
 qc.subusat x9, x0, x17
 
-# CHECK: :[[@LINE+1]]:21: error: invalid operand for instruction
+# CHECK-PLUS: :[[@LINE+2]]:21: error: register must be a GPR excluding zero (x0)
+# CHECK-MINUS: :[[@LINE+1]]:21: error: invalid operand for instruction
 qc.subusat x9, x14, x0
 
 # CHECK-MINUS: :[[@LINE+1]]:1: error: instruction requires the following: 'Xqcia' (Qualcomm uC Arithmetic Extension)
 qc.subusat x9, x14, x17
 
 
-# CHECK: :[[@LINE+1]]:18: error: invalid operand for instruction
+# CHECK-PLUS: :[[@LINE+2]]:18: error: register must be a GPR excluding zero (x0)
+# CHECK-MINUS: :[[@LINE+1]]:18: error: invalid operand for instruction
 qc.wrap x3, x30, 23
 
 # CHECK: :[[@LINE+1]]:1: error: too few operands for instruction
 qc.wrap x3, x30
 
-# CHECK: :[[@LINE+1]]:9: error: invalid operand for instruction
+# CHECK-PLUS: :[[@LINE+2]]:9: error: register must be a GPR excluding zero (x0)
+# CHECK-MINUS: :[[@LINE+1]]:9: error: invalid operand for instruction
 qc.wrap x0, x30, x23
 
-# CHECK: :[[@LINE+1]]:18: error: invalid operand for instruction
+# CHECK-PLUS: :[[@LINE+2]]:18: error: register must be a GPR excluding zero (x0)
+# CHECK-MINUS: :[[@LINE+1]]:18: error: invalid operand for instruction
 qc.wrap x3, x30, x0
 
 # CHECK-MINUS: :[[@LINE+1]]:1: error: instruction requires the following: 'Xqcia' (Qualcomm uC Arithmetic Extension)
 qc.wrap x3, x30, x23
 
 
-# CHECK: :[[@LINE+1]]:10: error: invalid operand for instruction
+# CHECK-PLUS: :[[@LINE+2]]:10: error: register must be a GPR excluding zero (x0)
+# CHECK-MINUS: :[[@LINE+1]]:10: error: invalid operand for instruction
 qc.wrapi x0, 12, 2047
 
-# CHECK: :[[@LINE+1]]:10: error: invalid operand for instruction
+# CHECK-PLUS: :[[@LINE+2]]:10: error: register must be a GPR excluding zero (x0)
+# CHECK-MINUS: :[[@LINE+1]]:10: error: invalid operand for instruction
 qc.wrapi x0, x12, 2047
 
-# CHECK: :[[@LINE+1]]:14: error: invalid operand for instruction
+# CHECK-PLUS: :[[@LINE+2]]:14: error: register must be a GPR excluding zero (x0)
+# CHECK-MINUS: :[[@LINE+1]]:14: error: invalid operand for instruction
 qc.wrapi x6, x0, 2047
 
 # CHECK: :[[@LINE+1]]:1: error: too few operands for instruction
@@ -153,48 +183,57 @@ qc.wrapi x6, x12, 2048
 qc.wrapi x6, x12, 2047
 
 
-# CHECK: :[[@LINE+1]]:13: error: invalid operand for instruction
+# CHECK-PLUS: :[[@LINE+2]]:13: error: register must be a GPR excluding zero (x0)
+# CHECK-MINUS: :[[@LINE+1]]:13: error: invalid operand for instruction
 qc.norm x3, 7
 
 # CHECK: :[[@LINE+1]]:1: error: too few operands for instruction
 qc.norm x3
 
-# CHECK: :[[@LINE+1]]:9: error: invalid operand for instruction
+# CHECK-PLUS: :[[@LINE+2]]:9: error: register must be a GPR excluding zero (x0)
+# CHECK-MINUS: :[[@LINE+1]]:9: error: invalid operand for instruction
 qc.norm x0, x7
 
-# CHECK: :[[@LINE+1]]:13: error: invalid operand for instruction
+# CHECK-PLUS: :[[@LINE+2]]:13: error: register must be a GPR excluding zero (x0)
+# CHECK-MINUS: :[[@LINE+1]]:13: error: invalid operand for instruction
 qc.norm x3, x0
 
 # CHECK-MINUS: :[[@LINE+1]]:1: error: instruction requires the following: 'Xqcia' (Qualcomm uC Arithmetic Extension)
 qc.norm x3, x7
 
 
-# CHECK: :[[@LINE+1]]:15: error: invalid operand for instruction
+# CHECK-PLUS: :[[@LINE+2]]:15: error: register must be a GPR excluding zero (x0)
+# CHECK-MINUS: :[[@LINE+1]]:15: error: invalid operand for instruction
 qc.normu x11, 17
 
 # CHECK: :[[@LINE+1]]:1: error: too few operands for instruction
 qc.normu x11
 
-# CHECK: :[[@LINE+1]]:10: error: invalid operand for instruction
+# CHECK-PLUS: :[[@LINE+2]]:10: error: register must be a GPR excluding zero (x0)
+# CHECK-MINUS: :[[@LINE+1]]:10: error: invalid operand for instruction
 qc.normu x0, x17
 
-# CHECK: :[[@LINE+1]]:15: error: invalid operand for instruction
+# CHECK-PLUS: :[[@LINE+2]]:15: error: register must be a GPR excluding zero (x0)
+# CHECK-MINUS: :[[@LINE+1]]:15: error: invalid operand for instruction
 qc.normu x11, x0
 
 # CHECK-MINUS: :[[@LINE+1]]:1: error: instruction requires the following: 'Xqcia' (Qualcomm uC Arithmetic Extension)
 qc.normu x11, x17
 
 
-# CHECK: :[[@LINE+1]]:16: error: invalid operand for instruction
+# CHECK-PLUS: :[[@LINE+2]]:16: error: register must be a GPR excluding zero (x0)
+# CHECK-MINUS: :[[@LINE+1]]:16: error: invalid operand for instruction
 qc.normeu x26, 31
 
 # CHECK: :[[@LINE+1]]:1: error: too few operands for instruction
 qc.normeu x26
 
-# CHECK: :[[@LINE+1]]:11: error: invalid operand for instruction
+# CHECK-PLUS: :[[@LINE+2]]:11: error: register must be a GPR excluding zero (x0)
+# CHECK-MINUS: :[[@LINE+1]]:11: error: invalid operand for instruction
 qc.normeu x0, x31
 
-# CHECK: :[[@LINE+1]]:16: error: invalid operand for instruction
+# CHECK-PLUS: :[[@LINE+2]]:16: error: register must be a GPR excluding zero (x0)
+# CHECK-MINUS: :[[@LINE+1]]:16: error: invalid operand for instruction
 qc.normeu x26, x0
 
 # CHECK-MINUS: :[[@LINE+1]]:1: error: instruction requires the following: 'Xqcia' (Qualcomm uC Arithmetic Extension)
diff --git a/llvm/test/MC/RISCV/xqciac-invalid.s b/llvm/test/MC/RISCV/xqciac-invalid.s
index 4e0182aff9cc2..3f2d8733b7032 100644
--- a/llvm/test/MC/RISCV/xqciac-invalid.s
+++ b/llvm/test/MC/RISCV/xqciac-invalid.s
@@ -4,33 +4,36 @@
 # RUN: not llvm-mc -triple riscv32 -mattr=-experimental-xqciac < %s 2>&1 \
 # RUN:     | FileCheck -check-prefixes=CHECK,CHECK-EXT %s
 
-# CHECK: :[[@LINE+1]]:14: error: invalid operand for instruction
-qc.c.muladdi x5, x10, 4
+# CHECK-PLUS: :[[@LINE+2]]:14: error: register must be a GPR excluding zero (x0)
+# CHECK-MINUS: :[[@LINE+1]]:14: error: invalid operand for instruction
+qc.c.muliadd x5, x10, 4
 
 # CHECK: :[[@LINE+1]]:1: error: too few operands for instruction
-qc.c.muladdi x15
+qc.c.muliadd x15
 
 # CHECK-IMM: :[[@LINE+1]]:24: error: immediate must be an integer in the range [0, 31]
-qc.c.muladdi x10, x15, 32
+qc.c.muliadd x10, x15, 32
 
 # CHECK-EXT: :[[@LINE+1]]:1: error: instruction requires the following: 'Xqciac' (Qualcomm uC Load-Store Address Calculation Extension)
-qc.c.muladdi x10, x15, 20
+qc.c.muliadd x10, x15, 20
 
 
-# CHECK: :[[@LINE+1]]:12: error: invalid operand for instruction
-qc.muladdi x0, x10, 1048577
+# CHECK-PLUS: :[[@LINE+2]]:12: error: register must be a GPR excluding zero (x0)
+# CHECK-MINUS: :[[@LINE+1]]:12: error: invalid operand for instruction
+qc.muliadd x0, x10, 1048577
 
 # CHECK: :[[@LINE+1]]:1: error: too few operands for instruction
-qc.muladdi x10
+qc.muliadd x10
 
 # CHECK-IMM: :[[@LINE+1]]:22: error: operand must be a symbol with %lo/%pcrel_lo/%tprel_lo modifier or an integer in the range [-2048, 2047]
-qc.muladdi x10, x15, 8589934592
+qc.muliadd x10, x15, 8589934592
 
 # CHECK-EXT: :[[@LINE+1]]:1: error: instruction requires the following: 'Xqciac' (Qualcomm uC Load-Store Address Calculation Extension)
-qc.muladdi x10, x15, 577
+qc.muliadd x10, x15, 577
 
 
-# CHECK: :[[@LINE+1]]:11: error: invalid operand for instruction
+# CHECK-PLUS: :[[@LINE+2]]:11: error: register must be a GPR excluding zero (x0)
+# CHECK-MINUS: :[[@LINE+1]]:11: error: invalid operand for instruction
 qc.shladd 0, x10, 1048577
 
 # CHECK: :[[@LINE+1]]:1: error: too few operands for instruction
diff --git a/llvm/test/MC/RISCV/xqciac-valid.s b/llvm/test/MC/RISCV/xqciac-valid.s
index 6e97d8cc447e1..c786d7c4ea51d 100644
--- a/llvm/test/MC/RISCV/xqciac-valid.s
+++ b/llvm/test/MC/RISCV/xqciac-valid.s
@@ -10,30 +10,30 @@
 # RUN:     | llvm-objdump --mattr=+experimental-xqciac --no-print-imm-hex -d - \
 # RUN:     | FileCheck -check-prefix=CHECK-INST %s
 
-# CHECK-INST: qc.c.muladdi    a0, a1, 0
+# CHECK-INST: qc.c.muliadd    a0, a1, 0
 # CHECK-ENC: encoding: [0x8a,0x21]
-qc.c.muladdi x10, x11, 0
+qc.c.muliadd x10, x11, 0
 
-# CHECK-INST: qc.c.muladdi    a0, a1, 31
+# CHECK-INST: qc.c.muliadd    a0, a1, 31
 # CHECK-ENC: encoding: [0xea,0x3d]
-qc.c.muladdi x10, x11, 31
+qc.c.muliadd x10, x11, 31
 
-# CHECK-INST: qc.c.muladdi    a0, a1, 16
+# CHECK-INST: qc.c.muliadd    a0, a1, 16
 # CHECK-ENC: encoding: [0xaa,0x21]
-qc.c.muladdi x10, x11, 16
+qc.c.muliadd x10, x11, 16
 
 
-# CHECK-INST: qc.muladdi      tp, t0, 1234
+# CHECK-INST: qc.muliadd      tp, t0, 1234
 # CHECK-ENC: encoding: [0x0b,0xe2,0x22,0x4d]
-qc.muladdi x4, x5, 1234
+qc.muliadd x4, x5, 1234
 
-# CHECK-INST: qc.muladdi      a0, a1, -2048
+# CHECK-INST: qc.muliadd      a0, a1, -2048
 # CHECK-ENC: encoding: [0x0b,0xe5,0x05,0x80]
-qc.muladdi x10, x11, -2048
+qc.muliadd x10, x11, -2048
 
-# CHECK-INST: qc.muladdi      a0, a1, 2047
+# CHECK-INST: qc.muliadd      a0, a1, 2047
 # CHECK-ENC: encoding: [0x0b,0xe5,0xf5,0x7f]
-qc.muladdi x10, x11, 2047
+qc.muliadd x10, x11, 2047
 
 
 # CHECK-INST: qc.shladd       tp, t0, t1, 12
diff --git a/llvm/test/MC/RISCV/xqcicli-invalid.s b/llvm/test/MC/RISCV/xqcicli-invalid.s
index 7ee92ec4cbc01..a2f5549a394f4 100644
--- a/llvm/test/MC/RISCV/xqcicli-invalid.s
+++ b/llvm/test/MC/RISCV/xqcicli-invalid.s
@@ -4,13 +4,16 @@
 # RUN: not llvm-mc -triple riscv32 -mattr=-experimental-xqcicli < %s 2>&1 \
 # RUN:     | FileCheck -check-prefixes=CHECK,CHECK-MINUS %s
 
-# CHECK: :[[@LINE+1]]:9: error: invalid operand for instruction
+# CHECK-PLUS: :[[@LINE+2]]:9: error: register must be a GPR excluding zero (x0)
+# CHECK-MINUS: :[[@LINE+1]]:9: error: invalid operand for instruction
 qc.lieq x0, x4, x6, 10
 
-# CHECK: :[[@LINE+1]]:13: error: invalid operand for instruction
+# CHECK-PLUS: :[[@LINE+2]]:13: error: register must be a GPR excluding zero (x0)
+# CHECK-MINUS: :[[@LINE+1]]:13: error: invalid operand for instruction
 qc.lieq x2, x0, x6, 10
 
-# CHECK: :[[@LINE+1]]:17: error: invalid operand for instruction
+# CHECK-PLUS: :[[@LINE+2]]:17: error: register must be a GPR excluding zero (x0)
+# CHECK-MINUS: :[[@LINE+1]]:17: error: invalid operand for instruction
 qc.lieq x2, x4, x0, 10
 
 # CHECK: :[[@LINE+1]]:1: error: too few operands for instruction
@@ -23,13 +26,16 @@ qc.lieq x2, x4, x6, 40
 qc.lieq x2, x4, x6, 10
 
 
-# CHECK: :[[@LINE+1]]:9: error: invalid operand for instruction
+# CHECK-PLUS: :[[@LINE+2]]:9: error: register must be a GPR excluding zero (x0)
+# CHECK-MINUS: :[[@LINE+1]]:9: error: invalid operand for instruction
 qc.lige x0, x8, x20, 2
 
-# CHECK: :[[@LINE+1]]:13: error: invalid operand for instruction
+# CHECK-PLUS: :[[@LINE+2]]:13: error: register must be a GPR excluding zero (x0)
+# CHECK-MINUS: :[[@LINE+1]]:13: error: invalid operand for instruction
 qc.lige x4, x0, x20, 2
 
-# CHECK: :[[@LINE+1]]:17: error: invalid operand for instruction
+# CHECK-PLUS: :[[@LINE+2]]:17: error: register must be a GPR excluding zero (x0)
+# CHECK-MINUS: :[[@LINE+1]]:17: error: invalid operand for instruction
 qc.lige x4, x8, x0, 2
 
 # CHECK: :[[@LINE+1]]:1: error: too few operands for instruction
@@ -42,13 +48,16 @@ qc.lige x4, x8, x20, -18
 qc.lige x4, x8, x20, 2
 
 
-# CHECK: :[[@LINE+1]]:9: error: invalid operand for instruction
+# CHECK-PLUS: :[[@LINE+2]]:9: error: register must be a GPR excluding zero (x0)
+# CHECK-MINUS: :[[@LINE+1]]:9: error: invalid operand for instruction
 qc.lilt x0, x9, x10, 3
 
-# CHECK: :[[@LINE+1]]:14: error: invalid operand for instruction
+# CHECK-PLUS: :[[@LINE+2]]:14: error: register must be a GPR excluding zero (x0)
+# CHECK-MINUS: :[[@LINE+1]]:14: error: invalid operand for instruction
 qc.lilt x19, x0, x10, 3
 
-# CHECK: :[[@LINE+1]]:18: error: invalid operand for instruction
+# CHECK-PLUS: :[[@LINE+2]]:18: error: register must be a GPR excluding zero (x0)
+# CHECK-MINUS: :[[@LINE+1]]:18: error: invalid operand for instruction
 qc.lilt x19, x9, x0, 3
 
 # CHECK: :[[@LINE+1]]:1: error: too few operands for instruction
@@ -61,13 +70,16 @@ qc.lilt x19, x9, x10, 39
 qc.lilt x19, x9, x10, 3
 
 
-# CHECK: :[[@LINE+1]]:9: error: invalid operand for instruction
+# CHECK-PLUS: :[[@LINE+2]]:9: error: register must be a GPR excluding zero (x0)
+# CHECK-MINUS: :[[@LINE+1]]:9: error: invalid operand for instruction
 qc.line x0, x14, x6, 10
 
-# CHECK: :[[@LINE+1]]:14: error: invalid operand for instruction
+# CHECK-PLUS: :[[@LINE+2]]:14: error: register must be a GPR excluding zero (x0)
+# CHECK-MINUS: :[[@LINE+1]]:14: error: invalid operand for instruction
 qc.line x18, x0, x6, 10
 
-# CHECK: :[[@LINE+1]]:19: error: invalid operand for instruction
+# CHECK-PLUS: :[[@LINE+2]]:19: error: register must be a GPR excluding zero (x0)
+# CHECK-MINUS: :[[@LINE+1]]:19: error: invalid operand for instruction
 qc.line x18, x14, x0, 10
 
 # CHECK: :[[@LINE+1]]:1: error: too few operands for instruction
@@ -80,13 +92,16 @@ qc.line x18, x14, x6, 100
 qc.line x18, x14, x6, 10
 
 
-# CHECK: :[[@LINE+1]]:10: error: invalid operand for instruction
+# CHECK-PLUS: :[[@LINE+2]]:10: error: register must be a GPR excluding zero (x0)
+# CHECK-MINUS: :[[@LINE+1]]:10: error: invalid operand for instruction
 qc.ligeu x0, x4, x6, 10
 
-# CHECK: :[[@LINE+1]]:14: error: invalid operand for instruction
+# CHECK-PLUS: :[[@LINE+2]]:14: error: register must be a GPR excluding zero (x0)
+# CHECK-MINUS: :[[@LINE+1]]:14: error: invalid operand for instruction
 qc.ligeu x2, x0, x6, 10
 
-# CHECK: :[[@LINE+1]]:18: error: invalid operand for instruction
+# CHECK-PLUS: :[[@LINE+2]]:18: error: register must be a GPR excluding zero (x0)
+# CHECK-MINUS: :[[@LINE+1]]:18: error: invalid operand for instruction
 qc.ligeu x2, x4, x0, 10
 
 # CHECK: :[[@LINE+1]]:1: error: too few operands for instruction
@@ -99,13 +114,16 @@ qc.ligeu x2, x4, x6, 70
 qc.ligeu x2, x4, x6, 10
 
 
-# CHECK: :[[@LINE+1]]:10: error: invalid operand for instruction
+# CHECK-PLUS: :[[@LINE+2]]:10: error: register must be a GPR excluding zero (x0)
+# CHECK-MINUS: :[[@LINE+1]]:10: error: invalid operand for instruction
 qc.liltu x0, x19, x12, 13
 
-# CHECK: :[[@LINE+1]]:14: error: invalid operand for instruction
+# CHECK-PLUS: :[[@LINE+2]]:14: error: register must be a GPR excluding zero (x0)
+# CHECK-MINUS: :[[@LINE+1]]:14: error: invalid operand for instruction
 qc.liltu x1, x0, x12, 13
 
-# CHECK: :[[@LINE+1]]:19: error: invalid operand for instruction
+# CHECK-PLUS: :[[@LINE+2]]:19: error: register must be a GPR excluding zero (x0)
+# CHECK-MINUS: :[[@LINE+1]]:19: error: invalid operand for instruction
 qc.liltu x1, x19, x0, 13
 
 # CHECK: :[[@LINE+1]]:1: error: too few operands for instruction
@@ -118,10 +136,12 @@ qc.liltu x1, x19, x12, 73
 qc.liltu x1, x19, x12, 13
 
 
-# CHECK: :[[@LINE+1]]:10: error: invalid operand for instruction
+# CHECK-PLUS: :[[@LINE+2]]:10: error: register must be a GPR excluding zero (x0)
+# CHECK-MINUS: :[[@LINE+1]]:10: error: invalid operand for instruction
 qc.lieqi x0, x1, 15, 12
 
-# CHECK: :[[@LINE+1]]:14: error: invalid operand for instruction
+# CHECK-PLUS: :[[@LINE+2]]:14: error: register must be a GPR excluding zero (x0)
+# CHECK-MINUS: :[[@LINE+1]]:14: error: invalid operand for instruction
 qc.lieqi x7, x0, 15, 12
 
 # CHECK: :[[@LINE+1]]:1: error: too few operands for instruction
@@ -137,10 +157,12 @@ qc.lieqi x7, x1, 15, -22
 qc.lieqi x7, x1, 15, 12
 
 
-# CHECK: :[[@LINE+1]]:10: error: invalid operand for instruction
+# CHECK-PLUS: :[[@LINE+2]]:10: error: register must be a GPR excluding zero (x0)
+# CHECK-MINUS: :[[@LINE+1]]:10: error: invalid operand for instruction
 qc.ligei x0, x11, -4, 9
 
-# CHECK: :[[@LINE+1]]:15: error: invalid operand for instruction
+# CHECK-PLUS: :[[@LINE+2]]:15: error: register must be a GPR excluding zero (x0)
+# CHECK-MINUS: :[[@LINE+1]]:15: error: invalid operand for instruction
 qc.ligei x17, x0, -4, 9
 
 # CHECK: :[[@LINE+1]]:1: error: too few operands for instruction
@@ -156,10 +178,12 @@ qc.ligei x17, x11, -4, 59
 qc.ligei x17, x11, -4, 9
 
 
-# CHECK: :[[@LINE+1]]:10: error: invalid operand for instruction
+# CHECK-PLUS: :[[@LINE+2]]:10: error: register must be a GPR excluding zero (x0)
+# CHECK-MINUS: :[[@LINE+1]]:10: error: invalid operand for instruction
 qc.lilti x0, x11, -14, 2
 
-# CHECK: :[[@LINE+1]]:14: error: invalid operand for instruction
+# CHECK-PLUS: :[[@LINE+2]]:14: error: register must be a GPR excluding zero (x0)
+# CHECK-MINUS: :[[@LINE+1]]:14: error: invalid operand for instruction
 qc.lilti x9, x0, -14, 2
 
 # CHECK: :[[@LINE+1]]:1: error: too few operands for instruction
@@ -175,10 +199,12 @@ qc.lilti x9, x11, -14, 52
 qc.lilti x9, x11, -14, 2
 
 
-# CHECK: :[[@LINE+1]]:10: error: invalid operand for instruction
+# CHECK-PLUS: :[[@LINE+2]]:10: error: register must be a GPR excluding zero (x0)
+# CHECK-MINUS: :[[@LINE+1]]:10: error: invalid operand for instruction
 qc.linei x0, x1, 10, 12
 
-# CHECK: :[[@LINE+1]]:14: error: invalid operand for instruction
+# CHECK-PLUS: :[[@LINE+2]]:14: error: register must be a GPR excluding zero (x0)
+# CHECK-MINUS: :[[@LINE+1]]:14: error: invalid operand for instruction
 qc.linei x5, x0, 10, 12
 
 # CHECK: :[[@LINE+1]]:1: error: too few operands for instruction
@@ -194,10 +220,12 @@ qc.linei x5, x1, 10, 124
 qc.linei x5, x1, 10, 12
 
 
-# CHECK: :[[@LINE+1]]:11: error: invalid operand for instruction
+# CHECK-PLUS: :[[@LINE+2]]:11: error: register must be a GPR excluding zero (x0)
+# CHECK-MINUS: :[[@LINE+1]]:11: error: invalid operand for instruction
 qc.ligeui x0, x12, 7, -12
 
-# CHECK: :[[@LINE+1]]:15: error: invalid operand for instruction
+# CHECK-PLUS: :[[@LINE+2]]:15: error: register must be a GPR excluding zero (x0)
+# CHECK-MINUS: :[[@LINE+1]]:15: error: invalid operand for instruction
 qc.ligeui x2, x0, 7, -12
 
 # CHECK: :[[@LINE+1]]:1: error: too few operands for instruction
@@ -213,10 +241,12 @@ qc.ligeui x2, x12, 7, -17
 qc.ligeui x2, x12, 7, -12
 
 
-# CHECK: :[[@LINE+1]]:11: error: invalid operand for instruction
+# CHECK-PLUS: :[[@LINE+2]]:11: error: register must be a GPR excluding zero (x0)
+# CHECK-MINUS: :[[@LINE+1]]:11: error: invalid operand for instruction
 qc.liltui x0, x25, 31, 12
 
-# CHECK: :[[@LINE+1]]:15: error: invalid operand for instruction
+# CHECK-PLUS: :[[@LINE+2]]:15: error: register must be a GPR excluding zero (x0)
+# CHECK-MINUS: :[[@LINE+1]]:15: error: invalid operand for instruction
 qc.liltui x3, x0, 31, 12
 
 # CHECK: :[[@LINE+1]]:1: error: too few operands for instruction
diff --git a/llvm/test/MC/RISCV/xqcicm-invalid.s b/llvm/test/MC/RISCV/xqcicm-invalid.s
index 8b37ed4edeb00..c1ee53fb21793 100644
--- a/llvm/test/MC/RISCV/xqcicm-invalid.s
+++ b/llvm/test/MC/RISCV/xqcicm-invalid.s
@@ -4,7 +4,8 @@
 # RUN: not llvm-mc -triple riscv32 -mattr=-experimental-xqcicm < %s 2>&1 \
 # RUN:     | FileCheck -check-prefixes=CHECK,CHECK-EXT %s
 
-# CHECK: :[[@LINE+1]]:12: error: invalid operand for instruction
+# CHECK-PLUS: :[[@LINE+2]]:12: error: register must be a GPR excluding zero (x0)
+# CHECK-MINUS: :[[@LINE+1]]:12: error: invalid operand for instruction
 qc.c.mveqz 9, x10
 
 # CHECK: :[[@LINE+1]]:1: error: too few operands for instruction
@@ -14,7 +15,8 @@ qc.c.mveqz x9
 qc.c.mveqz x9, x10
 
 
-# CHECK: :[[@LINE+1]]:9: error: invalid operand for instruction
+# CHECK-PLUS: :[[@LINE+2]]:9: error: register must be a GPR excluding zero (x0)
+# CHECK-MINUS: :[[@LINE+1]]:9: error: invalid operand for instruction
 qc.mveq 9, x10, x11, x12
 
 # CHECK: :[[@LINE+1]]:1: error: too few operands for instruction
@@ -24,7 +26,8 @@ qc.mveq x9
 qc.mveq x9, x10, x11, x12
 
 
-# CHECK: :[[@LINE+1]]:9: error: invalid operand for instruction
+# CHECK-PLUS: :[[@LINE+2]]:9: error: register must be a GPR excluding zero (x0)
+# CHECK-MINUS: :[[@LINE+1]]:9: error: invalid operand for instruction
 qc.mvge 9, x10, x11, x12
 
 # CHECK: :[[@LINE+1]]:1: error: too few operands for instruction
@@ -34,7 +37,8 @@ qc.mvge x9
 qc.mvge x9, x10, x11, x12
 
 
-# CHECK: :[[@LINE+1]]:10: error: invalid operand for instruction
+# CHECK-PLUS: :[[@LINE+2]]:10: error: register must be a GPR excluding zero (x0)
+# CHECK-MINUS: :[[@LINE+1]]:10: error: invalid operand for instruction
 qc.mvgeu 9, x10, x11, x12
 
 # CHECK: :[[@LINE+1]]:1: error: too few operands for instruction
@@ -44,7 +48,8 @@ qc.mvgeu x9
 qc.mvgeu x9, x10, x11, x12
 
 
-# CHECK: :[[@LINE+1]]:9: error: invalid operand for instruction
+# CHECK-PLUS: :[[@LINE+2]]:9: error: register must be a GPR excluding zero (x0)
+# CHECK-MINUS: :[[@LINE+1]]:9: error: invalid operand for instruction
 qc.mvlt 9, x10, x11, x12
 
 # CHECK: :[[@LINE+1]]:1: error: too few operands for instruction
@@ -54,7 +59,8 @@ qc.mvlt x9
 qc.mvlt x9, x10, x11, x12
 
 
-# CHECK: :[[@LINE+1]]:10: error: invalid operand for instruction
+# CHECK-PLUS: :[[@LINE+2]]:10: error: register must be a GPR excluding zero (x0)
+# CHECK-MINUS: :[[@LINE+1]]:10: error: invalid operand for instruction
 qc.mvltu 9, x10, x11, x12
 
 # CHECK: :[[@LINE+1]]:1: error: too few operands for instruction
@@ -64,7 +70,8 @@ qc.mvltu x9
 qc.mvltu x9, x10, x11, x12
 
 
-# CHECK: :[[@LINE+1]]:9: error: invalid operand for instruction
+# CHECK-PLUS: :[[@LINE+2]]:9: error: register must be a GPR excluding zero (x0)
+# CHECK-MINUS: :[[@LINE+1]]:9: error: invalid operand for instruction
 qc.mvne 9, x10, x11, x12
 
 # CHECK: :[[@LINE+1]]:1: error: too few operands for instruction
@@ -74,7 +81,8 @@ qc.mvne x9
 qc.mvne x9, x10, x11, x12
 
 
-# CHECK: :[[@LINE+1]]:10: error: invalid operand for instruction
+# CHECK-PLUS: :[[@LINE+2]]:10: error: register must be a GPR excluding zero (x0)
+# CHECK-MINUS: :[[@LINE+1]]:10: error: invalid operand for instruction
 qc.mveqi 9, x10, 5, x12
 
 # CHECK: :[[@LINE+1]]:1: error: too few operands for instruction
@@ -87,7 +95,8 @@ qc.mveqi x9, x10, 17, x12
 qc.mveqi x9, x10, 5, x12
 
 
-# CHECK: :[[@LINE+1]]:10: error: invalid operand for instruction
+# CHECK-PLUS: :[[@LINE+2]]:10: error: register must be a GPR excluding zero (x0)
+# CHECK-MINUS: :[[@LINE+1]]:10: error: invalid operand for instruction
 qc.mvgei 9, x10, 5, x12
 
 # CHECK: :[[@LINE+1]]:1: error: too few operands for instruction
@@ -100,7 +109,8 @@ qc.mvgei x9, x10, 17, x12
 qc.mvgei x9, x10, 5, x12
 
 
-# CHECK: :[[@LINE+1]]:10: error: invalid operand for instruction
+# CHECK-PLUS: :[[@LINE+2]]:10: error: register must be a GPR excluding zero (x0)
+# CHECK-MINUS: :[[@LINE+1]]:10: error: invalid operand for instruction
 qc.mvlti 9, x10, 5, x12
 
 # CHECK: :[[@LINE+1]]:1: error: too few operands for instruction
@@ -113,7 +123,8 @@ qc.mvlti x9, x10, 17, x12
 qc.mvlti x9, x10, 5, x12
 
 
-# CHECK: :[[@LINE+1]]:10: error: invalid operand for instruction
+# CHECK-PLUS: :[[@LINE+2]]:10: error: register must be a GPR excluding zero (x0)
+# CHECK-MINUS: :[[@LINE+1]]:10: error: invalid operand for instruction
 qc.mvnei 9, x10, 5, x12
 
 # CHECK: :[[@LINE+1]]:1: error: too few operands for instruction
@@ -126,7 +137,8 @@ qc.mvnei x9, x10, 17, x12
 qc.mvnei x9, x10, 5, x12
 
 
-# CHECK: :[[@LINE+1]]:11: error: invalid operand for instruction
+# CHECK-PLUS: :[[@LINE+2]]:11: error: register must be a GPR excluding zero (x0)
+# CHECK-MINUS: :[[@LINE+1]]:11: error: invalid operand for instruction
 qc.mvltui 9, x10, 5, x12
 
 # CHECK: :[[@LINE+1]]:1: error: too few operands for instruction
@@ -139,7 +151,8 @@ qc.mvltui x9, x10, 37, x12
 qc.mvltui x9, x10, 5, x12
 
 
-# CHECK: :[[@LINE+1]]:11: error: invalid operand for instruction
+# CHECK-PLUS: :[[@LINE+2]]:11: error: register must be a GPR excluding zero (x0)
+# CHECK-MINUS: :[[@LINE+1]]:11: error: invalid operand for instruction
 qc.mvgeui 9, x10, 5, x12
 
 # CHECK: :[[@LINE+1]]:1: error: too few operands for instruction
diff --git a/llvm/test/MC/RISCV/xqcics-invalid.s b/llvm/test/MC/RISCV/xqcics-invalid.s
index e7effdfd263b6..e3eda3fcdc193 100644
--- a/llvm/test/MC/RISCV/xqcics-invalid.s
+++ b/llvm/test/MC/RISCV/xqcics-invalid.s
@@ -4,7 +4,8 @@
 # RUN: not llvm-mc -triple riscv32 -mattr=-experimental-xqcics < %s 2>&1 \
 # RUN:     | FileCheck -check-prefixes=CHECK,CHECK-EXT %s
 
-# CHECK: :[[@LINE+1]]:14: error: invalid operand for instruction
+# CHECK-PLUS: :[[@LINE+2]]:14: error: register must be a GPR excluding zero (x0)
+# CHECK-MINUS: :[[@LINE+1]]:14: error: invalid operand for instruction
 qc.selecteqi 9, 15, x4, x3
 
 # CHECK: :[[@LINE+1]]:1: error: too few operands for instruction
@@ -17,7 +18,8 @@ qc.selecteqi x9, 16, x4, x3
 qc.selecteqi x9, 15, x4, x3
 
 
-# CHECK: :[[@LINE+1]]:14: error: invalid operand for instruction
+# CHECK-PLUS: :[[@LINE+2]]:14: error: register must be a GPR excluding zero (x0)
+# CHECK-MINUS: :[[@LINE+1]]:14: error: invalid operand for instruction
 qc.selectieq 8, x4, x3, 12
 
 # CHECK: :[[@LINE+1]]:1: error: too few operands for instruction
@@ -30,7 +32,8 @@ qc.selectieq x8, x4, x3, 17
 qc.selectieq x8, x4, x3, 12
 
 
-# CHECK: :[[@LINE+1]]:15: error: invalid operand for instruction
+# CHECK-PLUS: :[[@LINE+2]]:15: error: register must be a GPR excluding zero (x0)
+# CHECK-MINUS: :[[@LINE+1]]:15: error: invalid operand for instruction
 qc.selectieqi 9, 11, x3, 12
 
 # CHECK: :[[@LINE+1]]:1: error: too few operands for instruction
@@ -46,7 +49,8 @@ qc.selectieqi x9, 11, x3, 18
 qc.selectieqi x9, 11, x3, 12
 
 
-# CHECK: :[[@LINE+1]]:15: error: invalid operand for instruction
+# CHECK-PLUS: :[[@LINE+2]]:15: error: register must be a GPR excluding zero (x0)
+# CHECK-MINUS: :[[@LINE+1]]:15: error: invalid operand for instruction
 qc.selectiieq 9, x3, 11, 12
 
 # CHECK: :[[@LINE+1]]:1: error: too few operands for instruction
@@ -62,7 +66,8 @@ qc.selectiieq x9, x3, 11, 17
 qc.selectiieq x9, x3, 11, 12
 
 
-# CHECK: :[[@LINE+1]]:15: error: invalid operand for instruction
+# CHECK-PLUS: :[[@LINE+2]]:15: error: register must be a GPR excluding zero (x0)
+# CHECK-MINUS: :[[@LINE+1]]:15: error: invalid operand for instruction
 qc.selectiine 8, x3, 10, 11
 
 # CHECK: :[[@LINE+1]]:1: error: too few operands for instruction
@@ -78,7 +83,8 @@ qc.selectiine x8, x3, 12, 18
 qc.selectiine x8, x3, 10, 11
 
 
-# CHECK: :[[@LINE+1]]:14: error: invalid operand for instruction
+# CHECK-PLUS: :[[@LINE+2]]:14: error: register must be a GPR excluding zero (x0)
+# CHECK-MINUS: :[[@LINE+1]]:14: error: invalid operand for instruction
 qc.selectine 8, x3, x4, 11
 
 # CHECK: :[[@LINE+1]]:1: error: too few operands for instruction
@@ -91,7 +97,8 @@ qc.selectine x8, x3, x4, 16
 qc.selectine x8, x3, x4, 11
 
 
-# CHECK: :[[@LINE+1]]:15: error: invalid operand for instruction
+# CHECK-PLUS: :[[@LINE+2]]:15: error: register must be a GPR excluding zero (x0)
+# CHECK-MINUS: :[[@LINE+1]]:15: error: invalid operand for instruction
 qc.selectinei 8, 11, x3, 12
 
 # CHECK: :[[@LINE+1]]:1: error: too few operands for instruction
@@ -107,7 +114,8 @@ qc.selectinei x8, 11, x3, 18
 qc.selectinei x8, 11, x3, 12
 
 
-# CHECK: :[[@LINE+1]]:14: error: invalid operand for instruction
+# CHECK-PLUS: :[[@LINE+2]]:14: error: register must be a GPR excluding zero (x0)
+# CHECK-MINUS: :[[@LINE+1]]:14: error: invalid operand for instruction
 qc.selectnei 8, 11, x3, x5
 
 # CHECK: :[[@LINE+1]]:1: error: too few operands for instruction
diff --git a/llvm/test/MC/RISCV/xqcicsr-invalid.s b/llvm/test/MC/RISCV/xqcicsr-invalid.s
index 6d3d62d4b4b41..b4119dbf88272 100644
--- a/llvm/test/MC/RISCV/xqcicsr-invalid.s
+++ b/llvm/test/MC/RISCV/xqcicsr-invalid.s
@@ -4,7 +4,8 @@
 # RUN: not llvm-mc -triple riscv32 -mattr=-experimental-xqcicsr < %s 2>&1 \
 # RUN:     | FileCheck -check-prefixes=CHECK,CHECK-MINUS %s
 
-# CHECK: :[[@LINE+1]]:20: error: invalid operand for instruction
+# CHECK-PLUS: :[[@LINE+2]]:20: error: register must be a GPR excluding zero (x0)
+# CHECK-MINUS: :[[@LINE+1]]:20: error: invalid operand for instruction
 qc.csrrwr x10, x5, x0
 
 # CHECK: :[[@LINE+1]]:1: error: too few operands for instruction
@@ -14,7 +15,8 @@ qc.csrrwr x10, x5
 qc.csrrwr x10, x5, x20
 
 
-# CHECK: :[[@LINE+1]]:21: error: invalid operand for instruction
+# CHECK-PLUS: :[[@LINE+2]]:21: error: register must be a GPR excluding zero (x0)
+# CHECK-MINUS: :[[@LINE+1]]:21: error: invalid operand for instruction
 qc.csrrwri x20, 31, x0
 
 # CHECK-PLUS: :[[@LINE+1]]:17: error: immediate must be an integer in the range [0, 31]
diff --git a/llvm/test/MC/RISCV/xqciint-invalid.s b/llvm/test/MC/RISCV/xqciint-invalid.s
index e748109f41d82..34da463cb49c9 100644
--- a/llvm/test/MC/RISCV/xqciint-invalid.s
+++ b/llvm/test/MC/RISCV/xqciint-invalid.s
@@ -1,10 +1,11 @@
 # Xqciint - Qualcomm uC Interrupts extension
 # RUN: not llvm-mc -triple riscv32 -mattr=+experimental-xqciint < %s 2>&1 \
-# RUN:     | FileCheck -check-prefixes=CHECK,CHECK-IMM %s
+# RUN:     | FileCheck -check-prefixes=CHECK,CHECK-PLUS %s
 # RUN: not llvm-mc -triple riscv32 -mattr=-experimental-xqciint < %s 2>&1 \
-# RUN:     | FileCheck -check-prefixes=CHECK,CHECK-EXT %s
+# RUN:     | FileCheck -check-prefixes=CHECK,CHECK-MINUS %s
 
-# CHECK-IMM: :[[@LINE+1]]:12: error: immediate must be an integer in the range [0, 1023]
+# CHECK-PLUS: :[[@LINE+2]]:12: error: immediate must be an integer in the range [0, 1023]
+# CHECK-MINUS: :[[@LINE+1]]:12: error: invalid operand for instruction
 qc.setinti 1025
 
 # CHECK: :[[@LINE+1]]:16: error: invalid operand for instruction
@@ -13,11 +14,11 @@ qc.setinti 11, 12
 # CHECK: :[[@LINE+1]]:1: error: too few operands for instruction
 qc.setinti
 
-# CHECK-EXT: :[[@LINE+1]]:1: error: instruction requires the following: 'Xqciint' (Qualcomm uC Interrupts Extension)
+# CHECK-MINUS: :[[@LINE+1]]:1: error: instruction requires the following: 'Xqciint' (Qualcomm uC Interrupts Extension)
 qc.setinti   10
 
 
-# CHECK-IMM: :[[@LINE+1]]:12: error: immediate must be an integer in the range [0, 1023]
+# CHECK-PLUS: :[[@LINE+1]]:12: error: immediate must be an integer in the range [0, 1023]
 qc.clrinti 2000
 
 # CHECK: :[[@LINE+1]]:16: error: invalid operand for instruction
@@ -26,80 +27,84 @@ qc.clrinti 22, x4
 # CHECK: :[[@LINE+1]]:1: error: too few operands for instruction
 qc.clrinti
 
-# CHECK-EXT: :[[@LINE+1]]:1: error: instruction requires the following: 'Xqciint' (Qualcomm uC Interrupts Extension)
+# CHECK-MINUS: :[[@LINE+1]]:1: error: instruction requires the following: 'Xqciint' (Qualcomm uC Interrupts Extension)
 qc.clrinti 8
 
 
-# CHECK: :[[@LINE+1]]:13: error: invalid operand for instruction
+# CHECK-PLUS: :[[@LINE+2]]:13: error: register must be a GPR excluding zero (x0)
+# CHECK-MINUS: :[[@LINE+1]]:13: error: invalid operand for instruction
 qc.c.clrint 22
 
 # CHECK: :[[@LINE+1]]:1: error: too few operands for instruction
 qc.c.clrint
 
-# CHECK-EXT: :[[@LINE+1]]:1: error: instruction requires the following: 'Xqciint' (Qualcomm uC Interrupts Extension)
+# CHECK-MINUS: :[[@LINE+1]]:1: error: instruction requires the following: 'Xqciint' (Qualcomm uC Interrupts Extension)
 qc.c.clrint x8
 
 
 # CHECK: :[[@LINE+1]]:9: error: invalid operand for instruction
 qc.c.di 22
 
-# CHECK-EXT: :[[@LINE+1]]:1: error: instruction requires the following: 'Xqciint' (Qualcomm uC Interrupts Extension)
+# CHECK-MINUS: :[[@LINE+1]]:1: error: instruction requires the following: 'Xqciint' (Qualcomm uC Interrupts Extension)
 qc.c.di
 
 
-# CHECK: :[[@LINE+1]]:10: error: invalid operand for instruction
+# CHECK-PLUS: :[[@LINE+2]]:10: error: register must be a GPR excluding zero (x0)
+# CHECK-MINUS: :[[@LINE+1]]:10: error: invalid operand for instruction
 qc.c.dir 22
 
 # CHECK: :[[@LINE+1]]:1: error: too few operands for instruction
 qc.c.dir
 
-# CHECK-EXT: :[[@LINE+1]]:1: error: instruction requires the following: 'Xqciint' (Qualcomm uC Interrupts Extension)
+# CHECK-MINUS: :[[@LINE+1]]:1: error: instruction requires the following: 'Xqciint' (Qualcomm uC Interrupts Extension)
 qc.c.dir x8
 
 
 # CHECK: :[[@LINE+1]]:9: error: invalid operand for instruction
 qc.c.ei 22
 
-# CHECK-EXT: :[[@LINE+1]]:1: error: instruction requires the following: 'Xqciint' (Qualcomm uC Interrupts Extension)
+# CHECK-MINUS: :[[@LINE+1]]:1: error: instruction requires the following: 'Xqciint' (Qualcomm uC Interrupts Extension)
 qc.c.ei
 
 
-# CHECK: :[[@LINE+1]]:10: error: invalid operand for instruction
+# CHECK-PLUS: :[[@LINE+2]]:10: error: register must be a GPR excluding zero (x0)
+# CHECK-MINUS: :[[@LINE+1]]:10: error: invalid operand for instruction
 qc.c.eir 22
 
 # CHECK: :[[@LINE+1]]:1: error: too few operands for instruction
 qc.c.eir
 
-# CHECK-EXT: :[[@LINE+1]]:1: error: instruction requires the following: 'Xqciint' (Qualcomm uC Interrupts Extension)
+# CHECK-MINUS: :[[@LINE+1]]:1: error: instruction requires the following: 'Xqciint' (Qualcomm uC Interrupts Extension)
 qc.c.eir x8
 
 
 # CHECK: :[[@LINE+1]]:19: error: invalid operand for instruction
 qc.c.mienter.nest 22
 
-# CHECK-EXT: :[[@LINE+1]]:1: error: instruction requires the following: 'Xqciint' (Qualcomm uC Interrupts Extension)
+# CHECK-MINUS: :[[@LINE+1]]:1: error: instruction requires the following: 'Xqciint' (Qualcomm uC Interrupts Extension)
 qc.c.mienter.nest
 
 
 # CHECK: :[[@LINE+1]]:14: error: invalid operand for instruction
 qc.c.mienter 22
 
-# CHECK-EXT: :[[@LINE+1]]:1: error: instruction requires the following: 'Xqciint' (Qualcomm uC Interrupts Extension)
+# CHECK-MINUS: :[[@LINE+1]]:1: error: instruction requires the following: 'Xqciint' (Qualcomm uC Interrupts Extension)
 qc.c.mienter
 
 
 # CHECK: :[[@LINE+1]]:17: error: invalid operand for instruction
 qc.c.mileaveret 22
 
-# CHECK-EXT: :[[@LINE+1]]:1: error: instruction requires the following: 'Xqciint' (Qualcomm uC Interrupts Extension)
+# CHECK-MINUS: :[[@LINE+1]]:1: error: instruction requires the following: 'Xqciint' (Qualcomm uC Interrupts Extension)
 qc.c.mileaveret
 
 
-# CHECK: :[[@LINE+1]]:13: error: invalid operand for instruction
+# CHECK-PLUS: :[[@LINE+2]]:13: error: register must be a GPR excluding zero (x0)
+# CHECK-MINUS: :[[@LINE+1]]:13: error: invalid operand for instruction
 qc.c.setint 22
 
 # CHECK: :[[@LINE+1]]:1: error: too few operands for instruction
 qc.c.setint
 
-# CHECK-EXT: :[[@LINE+1]]:1: error: instruction requires the following: 'Xqciint' (Qualcomm uC Interrupts Extension)
+# CHECK-MINUS: :[[@LINE+1]]:1: error: instruction requires the following: 'Xqciint' (Qualcomm uC Interrupts Extension)
 qc.c.setint x8
diff --git a/llvm/test/MC/RISCV/xqcilsm-invalid.s b/llvm/test/MC/RISCV/xqcilsm-invalid.s
index c59260b5718d0..15d55021d64e4 100644
--- a/llvm/test/MC/RISCV/xqcilsm-invalid.s
+++ b/llvm/test/MC/RISCV/xqcilsm-invalid.s
@@ -7,7 +7,8 @@
 # CHECK: :[[@LINE+1]]:20: error: expected register
 qc.swm x5, x20, 12(20)
 
-# CHECK: :[[@LINE+1]]:12: error: invalid operand for instruction
+# CHECK-PLUS: :[[@LINE+2]]:12: error: register must be a GPR excluding zero (x0)
+# CHECK-MINUS: :[[@LINE+1]]:12: error: invalid operand for instruction
 qc.swm x5, x0, 12(x3)
 
 # CHECK: :[[@LINE+1]]:1: error: too few operands for instruction
@@ -42,7 +43,8 @@ qc.swmi x10, 4, 20(x4)
 # CHECK: :[[@LINE+1]]:23: error: expected register
 qc.setwm x4, x30, 124(2)
 
-# CHECK: :[[@LINE+1]]:14: error: invalid operand for instruction
+# CHECK-PLUS: :[[@LINE+2]]:14: error: register must be a GPR excluding zero (x0)
+# CHECK-MINUS: :[[@LINE+1]]:14: error: invalid operand for instruction
 qc.setwm x4, x0, 124(x2)
 
 # CHECK: :[[@LINE+1]]:1: error: too few operands for instruction
@@ -80,7 +82,8 @@ qc.lwm x7, x1, 24(20)
 # CHECK: :[[@LINE+1]]:1: error: too few operands for instruction
 qc.lwm x7, x1, 24
 
-# CHECK: :[[@LINE+1]]:8: error: invalid operand for instruction
+# CHECK-PLUS: :[[@LINE+2]]:8: error: register must be a GPR excluding zero (x0)
+# CHECK-MINUS: :[[@LINE+1]]:8: error: invalid operand for instruction
 qc.lwm x0, x1, 24(x20)
 
 # CHECK-PLUS: :[[@LINE+1]]:16: error: immediate must be a multiple of 4 bytes in the range [0, 124]
@@ -96,7 +99,8 @@ qc.lwmi x13, 9, 4(23)
 # CHECK: :[[@LINE+1]]:1: error: too few operands for instruction
 qc.lwmi x13, 9, 4
 
-# CHECK: :[[@LINE+1]]:9: error: invalid operand for instruction
+# CHECK-PLUS: :[[@LINE+2]]:9: error: register must be a GPR excluding zero (x0)
+# CHECK-MINUS: :[[@LINE+1]]:9: error: invalid operand for instruction
 qc.lwmi x0, 9, 4(x23)
 
 # CHECK-PLUS: :[[@LINE+1]]:14: error: immediate must be an integer in the range [1, 31]
diff --git a/llvm/test/MC/RISCV/xqcisls-invalid.s b/llvm/test/MC/RISCV/xqcisls-invalid.s
index 5e9e54812fee8..58a0f8ceaab35 100644
--- a/llvm/test/MC/RISCV/xqcisls-invalid.s
+++ b/llvm/test/MC/RISCV/xqcisls-invalid.s
@@ -4,13 +4,15 @@
 # RUN: not llvm-mc -triple riscv32 -mattr=-experimental-xqcisls < %s 2>&1 \
 # RUN:     | FileCheck -check-prefixes=CHECK,CHECK-MINUS %s
 
-# CHECK: :[[@LINE+1]]:16: error: invalid operand for instruction
+# CHECK-PLUS: :[[@LINE+2]]:16: error: register must be a GPR excluding zero (x0)
+# CHECK-MINUS: :[[@LINE+1]]:16: error: invalid operand for instruction
 qc.lrb x5, x2, x0, 4
 
 # CHECK: :[[@LINE+1]]:1: error: too few operands for instruction
 qc.lrb x5, x2, x4
 
-# CHECK-PLUS: :[[@LINE+1]]:20: error: immediate must be an integer in the range [0, 7]
+# CHECK-PLUS: :[[@LINE+2]]:20: error: immediate must be an integer in the range [0, 7]
+# CHECK-MINUS: :[[@LINE+1]]:20: error: invalid operand for instruction
 qc.lrb x5, x2, x4, 12
 
 # CHECK: :[[@LINE+1]]:12: error: invalid operand for instruction
@@ -20,13 +22,15 @@ qc.lrb x5, 2, x4, 4
 qc.lrb x5, x2, x4, 4
 
 
-# CHECK: :[[@LINE+1]]:17: error: invalid operand for instruction
+# CHECK-PLUS: :[[@LINE+2]]:17: error: register must be a GPR excluding zero (x0)
+# CHECK-MINUS: :[[@LINE+1]]:17: error: invalid operand for instruction
 qc.lrh x1, x12, x0, 2
 
 # CHECK: :[[@LINE+1]]:1: error: too few operands for instruction
 qc.lrh x1, x12, x6
 
-# CHECK-PLUS: :[[@LINE+1]]:21: error: immediate must be an integer in the range [0, 7]
+# CHECK-PLUS: :[[@LINE+2]]:21: error: immediate must be an integer in the range [0, 7]
+# CHECK-MINUS: :[[@LINE+1]]:21: error: invalid operand for instruction
 qc.lrh x1, x12, x6, 22
 
 # CHECK: :[[@LINE+1]]:12: error: invalid operand for instruction
@@ -36,13 +40,15 @@ qc.lrh x1, 12, x6, 2
 qc.lrh x1, x12, x6, 2
 
 
-# CHECK: :[[@LINE+1]]:17: error: invalid operand for instruction
+# CHECK-PLUS: :[[@LINE+2]]:17: error: register must be a GPR excluding zero (x0)
+# CHECK-MINUS: :[[@LINE+1]]:17: error: invalid operand for instruction
 qc.lrw x15, x7, x0, 1
 
 # CHECK: :[[@LINE+1]]:1: error: too few operands for instruction
 qc.lrw x15, x7, x14
 
-# CHECK-PLUS: :[[@LINE+1]]:22: error: immediate must be an integer in the range [0, 7]
+# CHECK-PLUS: :[[@LINE+2]]:22: error: immediate must be an integer in the range [0, 7]
+# CHECK-MINUS: :[[@LINE+1]]:22: error: invalid operand for instruction
 qc.lrw x15, x7, x14, 11
 
 # CHECK: :[[@LINE+1]]:13: error: invalid operand for instruction
@@ -52,13 +58,15 @@ qc.lrw x15, 7, x14, 1
 qc.lrw x15, x7, x14, 1
 
 
-# CHECK: :[[@LINE+1]]:18: error: invalid operand for instruction
+# CHECK-PLUS: :[[@LINE+2]]:18: error: register must be a GPR excluding zero (x0)
+# CHECK-MINUS: :[[@LINE+1]]:18: error: invalid operand for instruction
 qc.lrbu x9, x11, x0, 7
 
 # CHECK: :[[@LINE+1]]:1: error: too few operands for instruction
 qc.lrbu x9, x11, x4
 
-# CHECK-PLUS: :[[@LINE+1]]:22: error: immediate must be an integer in the range [0, 7]
+# CHECK-PLUS: :[[@LINE+2]]:22: error: immediate must be an integer in the range [0, 7]
+# CHECK-MINUS: :[[@LINE+1]]:22: error: invalid operand for instruction
 qc.lrbu x9, x11, x4, 37
 
 # CHECK: :[[@LINE+1]]:13: error: invalid operand for instruction
@@ -68,13 +76,15 @@ qc.lrbu x9, 11, x4, 7
 qc.lrbu x9, x11, x4, 7
 
 
-# CHECK: :[[@LINE+1]]:18: error: invalid operand for instruction
+# CHECK-PLUS: :[[@LINE+2]]:18: error: register must be a GPR excluding zero (x0)
+# CHECK-MINUS: :[[@LINE+1]]:18: error: invalid operand for instruction
 qc.lrhu x16, x6, x0, 4
 
 # CHECK: :[[@LINE+1]]:1: error: too few operands for instruction
 qc.lrhu x16, x6, x10
 
-# CHECK-PLUS: :[[@LINE+1]]:23: error: immediate must be an integer in the range [0, 7]
+# CHECK-PLUS: :[[@LINE+2]]:23: error: immediate must be an integer in the range [0, 7]
+# CHECK-MINUS: :[[@LINE+1]]:23: error: invalid operand for instruction
 qc.lrhu x16, x6, x10, 44
 
 # CHECK: :[[@LINE+1]]:14: error: invalid operand for instruction
@@ -84,13 +94,15 @@ qc.lrhu x16, 6, x10, 4
 qc.lrhu x16, x6, x10, 4
 
 
-# CHECK: :[[@LINE+1]]:16: error: invalid operand for instruction
+# CHECK-PLUS: :[[@LINE+2]]:16: error: register must be a GPR excluding zero (x0)
+# CHECK-MINUS: :[[@LINE+1]]:16: error: invalid operand for instruction
 qc.srb x0, x2, x0, 3
 
 # CHECK: :[[@LINE+1]]:1: error: too few operands for instruction
 qc.srb x0, x2, x8
 
-# CHECK-PLUS: :[[@LINE+1]]:20: error: immediate must be an integer in the range [0, 7]
+# CHECK-PLUS: :[[@LINE+2]]:20: error: immediate must be an integer in the range [0, 7]
+# CHECK-MINUS: :[[@LINE+1]]:20: error: invalid operand for instruction
 qc.srb x0, x2, x8, 93
 
 # CHECK: :[[@LINE+1]]:12: error: invalid operand for instruction
@@ -100,13 +112,15 @@ qc.srb x0, 2, x8, 3
 qc.srb x0, x2, x8, 3
 
 
-# CHECK: :[[@LINE+1]]:17: error: invalid operand for instruction
+# CHECK-PLUS: :[[@LINE+2]]:17: error: register must be a GPR excluding zero (x0)
+# CHECK-MINUS: :[[@LINE+1]]:17: error: invalid operand for instruction
 qc.srh x13, x0, x0, 6
 
 # CHECK: :[[@LINE+1]]:1: error: too few operands for instruction
 qc.srh x13, x0, x20
 
-# CHECK-PLUS: :[[@LINE+1]]:22: error: immediate must be an integer in the range [0, 7]
+# CHECK-PLUS: :[[@LINE+2]]:22: error: immediate must be an integer in the range [0, 7]
+# CHECK-MINUS: :[[@LINE+1]]:22: error: invalid operand for instruction
 qc.srh x13, x0, x20, 76
 
 # CHECK: :[[@LINE+1]]:13: error: invalid operand for instruction
@@ -116,13 +130,15 @@ qc.srh x13, 0, x20, 6
 qc.srh x13, x0, x20, 6
 
 
-# CHECK: :[[@LINE+1]]:18: error: invalid operand for instruction
+# CHECK-PLUS: :[[@LINE+2]]:18: error: register must be a GPR excluding zero (x0)
+# CHECK-MINUS: :[[@LINE+1]]:18: error: invalid operand for instruction
 qc.srw x17, x18, x0, 0
 
 # CHECK: :[[@LINE+1]]:1: error: too few operands for instruction
 qc.srw x17, x18, x19
 
-# CHECK-PLUS: :[[@LINE+1]]:23: error: immediate must be an integer in the range [0, 7]
+# CHECK-PLUS: :[[@LINE+2]]:23: error: immediate must be an integer in the range [0, 7]
+# CHECK-MINUS: :[[@LINE+1]]:23: error: invalid operand for instruction
 qc.srw x17, x18, x19, 10
 
 # CHECK: :[[@LINE+1]]:13: error: invalid operand for instruction
diff --git a/llvm/test/MC/RISCV/xwchc-invalid.s b/llvm/test/MC/RISCV/xwchc-invalid.s
index 99cc519573895..a399e1cb66271 100644
--- a/llvm/test/MC/RISCV/xwchc-invalid.s
+++ b/llvm/test/MC/RISCV/xwchc-invalid.s
@@ -8,13 +8,13 @@ qk.c.sh x8, 1(x8) # CHECK: :[[@LINE]]:{{[0-9]+}}: error: immediate must be a mul
 qk.c.lhu x8, 64(x8) # CHECK: :[[@LINE]]:{{[0-9]+}}: error: immediate must be a multiple of 2 bytes in the range [0, 62]
 qk.c.sh x8, 64(x8) # CHECK: :[[@LINE]]:{{[0-9]+}}: error: immediate must be a multiple of 2 bytes in the range [0, 62]
 
-qk.c.lbusp x8, 0(x8) # CHECK: :[[@LINE]]:{{[0-9]+}}: error: invalid operand for instruction
-qk.c.sbsp x8, 0(x8) # CHECK: :[[@LINE]]:{{[0-9]+}}: error: invalid operand for instruction
+qk.c.lbusp x8, 0(x8) # CHECK: :[[@LINE]]:{{[0-9]+}}: error: register must be sp (x2)
+qk.c.sbsp x8, 0(x8) # CHECK: :[[@LINE]]:{{[0-9]+}}: error: register must be sp (x2)
 qk.c.lbusp x8, 32(sp) # CHECK: :[[@LINE]]:{{[0-9]+}}: error: immediate must be an integer in the range [0, 15]
 qk.c.sbsp x8, 32(sp) # CHECK: :[[@LINE]]:{{[0-9]+}}: error: immediate must be an integer in the range [0, 15]
 
-qk.c.lhusp x8, 0(x8) # CHECK: :[[@LINE]]:{{[0-9]+}}: error: invalid operand for instruction
-qk.c.shsp x8, 0(x8) # CHECK: :[[@LINE]]:{{[0-9]+}}: error: invalid operand for instruction
+qk.c.lhusp x8, 0(x8) # CHECK: :[[@LINE]]:{{[0-9]+}}: error: register must be sp (x2)
+qk.c.shsp x8, 0(x8) # CHECK: :[[@LINE]]:{{[0-9]+}}: error: register must be sp (x2)
 qk.c.lhusp x8, 1(sp) # CHECK: :[[@LINE]]:{{[0-9]+}}: error: immediate must be a multiple of 2 bytes in the range [0, 30]
 qk.c.shsp x8, 1(sp) # CHECK: :[[@LINE]]:{{[0-9]+}}: error: immediate must be a multiple of 2 bytes in the range [0, 30]
 qk.c.lhusp x8, 32(sp) # CHECK: :[[@LINE]]:{{[0-9]+}}: error: immediate must be a multiple of 2 bytes in the range [0, 30]
diff --git a/llvm/test/MC/RISCV/zicfiss-invalid.s b/llvm/test/MC/RISCV/zicfiss-invalid.s
new file mode 100644
index 0000000000000..f978d8c2c5ce8
--- /dev/null
+++ b/llvm/test/MC/RISCV/zicfiss-invalid.s
@@ -0,0 +1,19 @@
+# RUN: not llvm-mc %s -triple=riscv32 -mattr=+experimental-zicfiss,+zcmop,+c -M no-aliases -show-encoding \
+# RUN:     2>&1 | FileCheck -check-prefixes=CHECK-ERR %s
+# RUN: not llvm-mc %s -triple=riscv64 -mattr=+experimental-zicfiss,+zcmop,+c -M no-aliases -show-encoding \
+# RUN:     2>&1 | FileCheck -check-prefixes=CHECK-ERR %s
+
+# CHECK-ERR: error: register must be ra or t0 (x1 or x5)
+sspopchk a1
+
+# CHECK-ERR: error: register must be ra (x1)
+c.sspush t0
+
+# CHECK-ERR: error: register must be t0 (x5)
+c.sspopchk ra
+
+# CHECK-ERR: error: register must be ra or t0 (x1 or x5)
+sspush a0
+
+# CHECK-ERR: error: register must be a GPR excluding zero (x0)
+ssrdp zero
diff --git a/llvm/test/MC/Sparc/sparcv9-synthetic-instructions.s b/llvm/test/MC/Sparc/sparcv9-synthetic-instructions.s
index d387e56bf02a3..0d14ff558b314 100644
--- a/llvm/test/MC/Sparc/sparcv9-synthetic-instructions.s
+++ b/llvm/test/MC/Sparc/sparcv9-synthetic-instructions.s
@@ -78,3 +78,47 @@ setx (0x0123456700000000+0x0000000089abcdef), %g1, %o0
 ! V9: sllx %g1, 32, %g1                       ! encoding: [0x83,0x28,0x70,0x20]
 ! V9: or %g1, %o0, %o0                        ! encoding: [0x90,0x10,0x40,0x08]
 setx (.BB1-.BB0), %g1, %o0
+
+!! In V9, setuw is a mnemonic alias for set.
+! V8:      error: invalid instruction mnemonic
+! V9: sethi %hi(32768), %g1            ! encoding: [0x03,0b00AAAAAA,A,A]
+! V9:                                  !   fixup A - offset: 0, value: %hi(32768), kind: fixup_sparc_hi22
+setuw 32768, %g1
+! V9: mov 1, %g1 ! encoding: [0x82,0x10,0x20,0x01]
+setuw 1, %g1
+
+! V8:      error: instruction requires a CPU feature not currently enabled
+! V9: mov 4095, %g1                   ! encoding: [0x82,0x10,0x2f,0xff]
+setsw 4095, %g1
+! V8:      error: instruction requires a CPU feature not currently enabled
+! V9: mov -4096, %g1                  ! encoding: [0x82,0x10,0x30,0x00]
+setsw -4096, %g1
+! V8:      error: instruction requires a CPU feature not currently enabled
+! V9: sethi %hi(4096), %g1            ! encoding: [0x03,0b00AAAAAA,A,A]
+! V9:                                 !   fixup A - offset: 0, value: %hi(4096), kind: fixup_sparc_hi22
+setsw 4096, %g1
+! V8:      error: instruction requires a CPU feature not currently enabled
+! V9: sethi %hi(-4097), %g1           ! encoding: [0x03,0b00AAAAAA,A,A]
+! V9:                                 !   fixup A - offset: 0, value: %hi(-4097), kind: fixup_sparc_hi22
+! V9: sra %g1, %g0, %g1               ! encoding: [0x83,0x38,0x40,0x00]
+setsw -4097, %g1
+! V8:      error: instruction requires a CPU feature not currently enabled
+! V9: sethi %hi(2147483647), %o1      ! encoding: [0x13,0b00AAAAAA,A,A]
+! V9:                                 !   fixup A - offset: 0, value: %hi(2147483647), kind: fixup_sparc_hi22
+! V9: or %o1, %lo(2147483647), %o1    ! encoding: [0x92,0x12,0b011000AA,A]
+! V9:                                 !   fixup A - offset: 0, value: %lo(2147483647), kind: fixup_sparc_lo10
+setsw 2147483647, %o1
+! V8:      error: instruction requires a CPU feature not currently enabled
+! V9: sethi %hi(-2147483647), %o1     ! encoding: [0x13,0b00AAAAAA,A,A]
+! V9:                                 !   fixup A - offset: 0, value: %hi(-2147483647), kind: fixup_sparc_hi22
+! V9: or %o1, %lo(-2147483647), %o1   ! encoding: [0x92,0x12,0b011000AA,A]
+! V9:                                 !   fixup A - offset: 0, value: %lo(-2147483647), kind: fixup_sparc_lo10
+! V9: sra %o1, %g0, %o1               ! encoding: [0x93,0x3a,0x40,0x00]
+setsw -2147483647, %o1
+! V8:      error: instruction requires a CPU feature not currently enabled
+! V9: sethi %hi(.Ltmp0), %o1          ! encoding: [0x13,0b00AAAAAA,A,A]
+! V9:                                 !   fixup A - offset: 0, value: %hi(.Ltmp0), kind: fixup_sparc_hi22
+! V9: or %o1, %lo(.Ltmp0), %o1        ! encoding: [0x92,0x12,0b011000AA,A]
+! V9:                                 !   fixup A - offset: 0, value: %lo(.Ltmp0), kind: fixup_sparc_lo10
+! V9: sra %o1, %g0, %o1               ! encoding: [0x93,0x3a,0x40,0x00]
+setsw ., %o1
diff --git a/llvm/test/MC/Xtensa/windowed.s b/llvm/test/MC/Xtensa/windowed.s
new file mode 100644
index 0000000000000..3376469e3085b
--- /dev/null
+++ b/llvm/test/MC/Xtensa/windowed.s
@@ -0,0 +1,105 @@
+# RUN: llvm-mc %s -triple=xtensa -show-encoding --mattr=+windowed \
+# RUN:     | FileCheck -check-prefixes=CHECK,CHECK-INST %s
+
+.align	4
+LBL0:
+
+# Instruction format BRI12
+# CHECK-INST: entry a3, 128
+# CHECK: encoding: [0x36,0x03,0x01]
+entry a3, 128
+
+# Instruction format RRR
+# CHECK-INST: movsp a3, a4
+# CHECK: encoding: [0x30,0x14,0x00]
+movsp a3, a4
+
+# Instruction format CALL
+# CHECK-INST: call4	LBL0
+# CHECK: encoding: [0bAA010101,A,A]
+call4 LBL0
+
+# Instruction format CALL
+# CHECK-INST: call8 LBL0
+# CHECK: encoding: [0bAA100101,A,A]
+call8 LBL0
+
+# Instruction format CALL
+# CHECK-INST: call12 LBL0
+# CHECK: encoding: [0bAA110101,A,A]
+call12 LBL0
+
+# Instruction format CALLX
+# CHECK-INST: callx4 a3
+# CHECK: encoding: [0xd0,0x03,0x00]
+callx4 a3
+
+# Instruction format CALLX
+# CHECK-INST: callx8 a3
+# CHECK: encoding: [0xe0,0x03,0x00]
+callx8 a3
+
+# Instruction format CALLX
+# CHECK-INST: callx12 a3
+# CHECK: encoding: [0xf0,0x03,0x00]
+callx12 a3
+
+# Instruction format CALLX
+# CHECK-INST: retw
+# CHECK: encoding: [0x90,0x00,0x00]
+retw
+
+# Instruction format CALLX
+# CHECK-INST: retw
+# CHECK: encoding: [0x90,0x00,0x00]
+_retw
+
+# Instruction format RRR
+# CHECK-INST: rotw 2
+# CHECK: encoding: [0x20,0x80,0x40]
+rotw 2
+
+# Instruction format RRI4
+# CHECK-INST: l32e a3, a4, -12
+# CHECK: encoding: [0x30,0xd4,0x09]
+l32e a3, a4, -12
+
+# Instruction format RRI4
+# CHECK-INST: s32e a3, a4, -12
+# CHECK: encoding: [0x30,0xd4,0x49]
+s32e a3, a4, -12
+
+# Instruction format RRR
+# CHECK-INST: rfwo
+# CHECK: encoding: [0x00,0x34,0x00]
+rfwo
+
+# Instruction format RRR
+# CHECK-INST: rfwu
+# CHECK: encoding: [0x00,0x35,0x00]
+rfwu
+
+# Instruction format RSR
+# CHECK-INST: xsr a3, windowbase
+# CHECK: # encoding: [0x30,0x48,0x61]
+xsr a3, windowbase
+
+# CHECK-INST: xsr a3, windowbase
+# CHECK: # encoding: [0x30,0x48,0x61]
+xsr.windowbase a3
+
+# CHECK-INST: xsr a3, windowbase
+# CHECK: # encoding: [0x30,0x48,0x61]
+xsr a3, 72
+
+# CHECK-INST: xsr a3, windowstart
+# CHECK: # encoding: [0x30,0x49,0x61]
+xsr a3, windowstart
+
+# CHECK-INST: xsr a3, windowstart
+# CHECK: # encoding: [0x30,0x49,0x61]
+xsr.windowstart a3
+
+# CHECK-INST: xsr a3, windowstart
+# CHECK: # encoding: [0x30,0x49,0x61]
+xsr a3, 73
diff --git a/llvm/test/MC/Xtensa/windowed_code_density.s b/llvm/test/MC/Xtensa/windowed_code_density.s
new file mode 100644
index 0000000000000..e532657466fba
--- /dev/null
+++ b/llvm/test/MC/Xtensa/windowed_code_density.s
@@ -0,0 +1,14 @@
+# RUN: llvm-mc %s -triple=xtensa -show-encoding --mattr=+windowed,+density \
+# RUN:     | FileCheck -check-prefixes=CHECK,CHECK-INST %s
+
+.align	4
+
+# Instruction format RRRN
+# CHECK-INST: retw.n
+# CHECK: encoding: [0x1d,0xf0]
+retw.n
+
+# Instruction format RRRN
+# CHECK-INST: retw.n
+# CHECK: encoding: [0x1d,0xf0]
+_retw.n
diff --git a/llvm/test/MC/Xtensa/windowed_invalid.s b/llvm/test/MC/Xtensa/windowed_invalid.s
new file mode 100644
index 0000000000000..958363093dd8e
--- /dev/null
+++ b/llvm/test/MC/Xtensa/windowed_invalid.s
@@ -0,0 +1,35 @@
+// NOTE: Assertions have been autogenerated by utils/update_mc_test_checks.py UTC_ARGS: --version 5
+# RUN: not llvm-mc -triple xtensa --mattr=+windowed %s 2>&1 | FileCheck %s
+
+## Out of range immediates
+
+
+# entry_imm12
+
+
+entry a3, -1
+// CHECK: :[[@LINE-1]]:11: error: expected immediate in range [0, 32760], first 3 bits should be zero
+
+# entry_imm12
+
+
+entry a3, 32764
+// CHECK: :[[@LINE-1]]:11: error: expected immediate in range [0, 32760], first 3 bits should be zero
+
+# entry_imm12
+
+
+entry a3, 4
+// CHECK: :[[@LINE-1]]:11: error: expected immediate in range [0, 32760], first 3 bits should be zero
+
+# imm8n_7
+
+
+rotw 100
+// CHECK: :[[@LINE-1]]:6: error: expected immediate in range [-8, 7]
+
+# imm64n_4n
+
+
+l32e a3, a4, -100
+// CHECK: :[[@LINE-1]]:14: error: expected immediate in range [-64, -4]
diff --git a/llvm/test/MachineVerifier/AMDGPU/test_g_incompatible_range.mir b/llvm/test/MachineVerifier/AMDGPU/test_g_incompatible_range.mir
new file mode 100644
index 0000000000000..0038173a07788
--- /dev/null
+++ b/llvm/test/MachineVerifier/AMDGPU/test_g_incompatible_range.mir
@@ -0,0 +1,39 @@
+# RUN: not --crash llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx942 -run-pass=none -filetype=null %s 2>&1 | FileCheck -implicit-check-not="Bad machine code" %s
+--- |
+  define void @mismatched_range_type() {
+    ret void
+  }
+
+  !0 = !{i64 -4294967295, i64 4294967296}
+
+...
+---
+name:            mismatched_range_type
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    liveins: $vgpr0, $vgpr1
+
+    %1:_(s32) = COPY $vgpr0
+    %2:_(s32) = COPY $vgpr1
+    %0:_(p1) = G_MERGE_VALUES %1(s32), %2(s32)
+
+    ; CHECK: Bad machine code: range is incompatible with the result type
+    %3:_(<2 x s32>) = G_LOAD %0(p1) :: (volatile load (s64), align 4, !range !0, addrspace 1)
+
+    ; CHECK: Bad machine code: range is incompatible with the result type
+    %4:_(p0) = G_LOAD %0(p1) :: (volatile load (s64), align 4, !range !0, addrspace 1)
+
+    ; CHECK: Bad machine code: range is incompatible with the result type
+    %5:_(<2 x p0>) = G_LOAD %0(p1) :: (volatile load (s64), align 4, !range !0, addrspace 1)
+
+    ; CHECK: Bad machine code: range is incompatible with the result type
+    %6:_(<3 x s64>) = G_LOAD %0(p1) :: (volatile load (s64), align 4, !range !0, addrspace 1)
+
+    ; CHECK: Bad machine code: range is incompatible with the result type
+    %7:_(<3 x s64>) = G_LOAD %0(p1) :: (volatile load (<2 x s64>), align 4, !range !0, addrspace 1)
+
+    $vgpr0_vgpr1 = COPY %3
+    SI_RETURN implicit $vgpr0_vgpr1
+
+...
diff --git a/llvm/test/ObjectYAML/DXContainer/RootSignature-Flags.yaml b/llvm/test/ObjectYAML/DXContainer/RootSignature-Flags.yaml
new file mode 100644
index 0000000000000..b0a3e6945f454
--- /dev/null
+++ b/llvm/test/ObjectYAML/DXContainer/RootSignature-Flags.yaml
@@ -0,0 +1,33 @@
+# RUN: yaml2obj %s | obj2yaml | FileCheck %s
+
+--- !dxcontainer
+Header:
+  Hash:            [ 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 
+                     0x0, 0x0, 0x0, 0x0, 0x0, 0x0 ]
+  Version:
+    Major:           1
+    Minor:           0
+  PartCount:       1
+  PartOffsets:     [ 60 ]
+Parts:
+  - Name:            RTS0
+    Size:            24
+    RootSignature:
+      Version: 2
+      NumParameters: 1
+      RootParametersOffset: 3
+      NumStaticSamplers: 4
+      StaticSamplersOffset: 5
+      AllowInputAssemblerInputLayout: true
+      DenyGeometryShaderRootAccess: true
+
+# CHECK:  - Name:            RTS0
+# CHECK-NEXT:    Size:            24
+# CHECK-NEXT:    RootSignature:
+# CHECK-NEXT:      Version: 2
+# CHECK-NEXT:      NumParameters: 1
+# CHECK-NEXT:      RootParametersOffset: 3
+# CHECK-NEXT:      NumStaticSamplers: 4
+# CHECK-NEXT:      StaticSamplersOffset: 5
+# CHECK-NEXT:      AllowInputAssemblerInputLayout: true
+# CHECK-NEXT:      DenyGeometryShaderRootAccess: true
diff --git a/llvm/test/Other/cgscc-devirt-iteration.ll b/llvm/test/Other/cgscc-devirt-iteration.ll
index 54229461cb394..2ee016a134baa 100644
--- a/llvm/test/Other/cgscc-devirt-iteration.ll
+++ b/llvm/test/Other/cgscc-devirt-iteration.ll
@@ -112,7 +112,7 @@ define void @test3(ptr %src, ptr %dest, i64 %size) noinline {
 ; CHECK-NOT: read
 ; CHECK-SAME: noinline
 ; BEFORE-LABEL: define void @test3(ptr %src, ptr %dest, i64 %size)
-; AFTER-LABEL: define void @test3(ptr nocapture readonly %src, ptr nocapture writeonly %dest, i64 %size)
+; AFTER-LABEL: define void @test3(ptr readonly captures(none) %src, ptr writeonly captures(none) %dest, i64 %size)
   %fptr = alloca ptr
   store ptr @memcpy, ptr %fptr
   %f = load ptr, ptr %fptr
diff --git a/llvm/test/Other/new-pm-O0-ep-callbacks.ll b/llvm/test/Other/new-pm-O0-ep-callbacks.ll
index 2184a2ded5920..77ccddba56a62 100644
--- a/llvm/test/Other/new-pm-O0-ep-callbacks.ll
+++ b/llvm/test/Other/new-pm-O0-ep-callbacks.ll
@@ -3,6 +3,7 @@
 ; RUN: opt -disable-output -debug-pass-manager -passes-ep-scalar-optimizer-late=no-op-function -passes='default<O0>' 2>&1 < %s | FileCheck %s
 ; RUN: opt -disable-output -debug-pass-manager -passes-ep-cgscc-optimizer-late=no-op-cgscc -passes='default<O0>' 2>&1 < %s | FileCheck %s
 ; RUN: opt -disable-output -debug-pass-manager -passes-ep-vectorizer-start=no-op-function -passes='default<O0>' 2>&1 < %s | FileCheck %s
+; RUN: opt -disable-output -debug-pass-manager -passes-ep-vectorizer-end=no-op-function -passes='default<O0>' 2>&1 < %s | FileCheck %s
 ; RUN: opt -disable-output -debug-pass-manager -passes-ep-pipeline-start=no-op-module -passes='default<O0>' 2>&1 < %s | FileCheck %s
 ; RUN: opt -disable-output -debug-pass-manager -passes-ep-pipeline-early-simplification=no-op-module -passes='default<O0>' 2>&1 < %s | FileCheck %s
 ; RUN: opt -disable-output -debug-pass-manager -passes-ep-optimizer-early=no-op-module -passes='default<O0>' 2>&1 < %s | FileCheck %s
diff --git a/llvm/test/Other/new-pm-defaults.ll b/llvm/test/Other/new-pm-defaults.ll
index 7cf035b0c6f37..c554fdbf4c799 100644
--- a/llvm/test/Other/new-pm-defaults.ll
+++ b/llvm/test/Other/new-pm-defaults.ll
@@ -51,6 +51,10 @@
 ; RUN:     -passes='default<O3>' -S  %s 2>&1 \
 ; RUN:     | FileCheck %s --check-prefixes=CHECK-O,CHECK-DEFAULT,CHECK-O3,%llvmcheckext,CHECK-EP-VECTORIZER-START,CHECK-O23SZ
 ; RUN: opt -disable-verify -verify-analysis-invalidation=0 -eagerly-invalidate-analyses=0 -debug-pass-manager \
+; RUN:     -passes-ep-vectorizer-end='no-op-function' \
+; RUN:     -passes='default<O3>' -S  %s 2>&1 \
+; RUN:     | FileCheck %s --check-prefixes=CHECK-O,CHECK-DEFAULT,CHECK-O3,%llvmcheckext,CHECK-EP-VECTORIZER-END,CHECK-O23SZ
+; RUN: opt -disable-verify -verify-analysis-invalidation=0 -eagerly-invalidate-analyses=0 -debug-pass-manager \
 ; RUN:     -passes-ep-pipeline-start='no-op-module' \
 ; RUN:     -passes='default<O3>' -S  %s 2>&1 \
 ; RUN:     | FileCheck %s --check-prefixes=CHECK-O,CHECK-DEFAULT,CHECK-O3,%llvmcheckext,CHECK-EP-PIPELINE-START,CHECK-O23SZ
@@ -274,6 +278,7 @@
 ; CHECK-O-NEXT: Running pass: LCSSAPass
 ; CHECK-O-NEXT: Running pass: LICMPass
 ; CHECK-O-NEXT: Running pass: AlignmentFromAssumptionsPass
+; CHECK-EP-VECTORIZER-END-NEXT: Running pass: NoOpFunctionPass
 ; CHECK-O-NEXT: Running pass: LoopSinkPass
 ; CHECK-O-NEXT: Running pass: InstSimplifyPass
 ; CHECK-O-NEXT: Running pass: DivRemPairsPass
diff --git a/llvm/test/Other/new-pm-lto-defaults.ll b/llvm/test/Other/new-pm-lto-defaults.ll
index f788db1e338a1..86480c5115748 100644
--- a/llvm/test/Other/new-pm-lto-defaults.ll
+++ b/llvm/test/Other/new-pm-lto-defaults.ll
@@ -18,6 +18,12 @@
 ; RUN:     -passes='lto<O3>' -S  %s 2>&1 \
 ; RUN:     | FileCheck %s --check-prefixes=CHECK-O,CHECK-O3,CHECK-O23SZ
 ; RUN: opt -disable-verify -verify-analysis-invalidation=0 -eagerly-invalidate-analyses=0 -debug-pass-manager \
+; RUN:     -passes='lto<O3>' -S  %s -passes-ep-vectorizer-start='no-op-function' 2>&1 \
+; RUN:     | FileCheck %s --check-prefixes=CHECK-O,CHECK-O3,CHECK-O23SZ,CHECK-EP-VECTORIZER-START
+; RUN: opt -disable-verify -verify-analysis-invalidation=0 -eagerly-invalidate-analyses=0 -debug-pass-manager \
+; RUN:     -passes='lto<O3>' -S  %s -passes-ep-vectorizer-end='no-op-function' 2>&1 \
+; RUN:     | FileCheck %s --check-prefixes=CHECK-O,CHECK-O3,CHECK-O23SZ,CHECK-EP-VECTORIZER-END
+; RUN: opt -disable-verify -verify-analysis-invalidation=0 -eagerly-invalidate-analyses=0 -debug-pass-manager \
 ; RUN:     -passes='lto<Os>' -S %s 2>&1 \
 ; RUN:     | FileCheck %s --check-prefixes=CHECK-O,CHECK-OS,CHECK-O23SZ
 ; RUN: opt -disable-verify -verify-analysis-invalidation=0 -eagerly-invalidate-analyses=0 -debug-pass-manager \
@@ -25,7 +31,7 @@
 ; RUN:     | FileCheck %s --check-prefixes=CHECK-O,CHECK-O23SZ
 ; RUN: opt -disable-verify -verify-analysis-invalidation=0 -eagerly-invalidate-analyses=0 -debug-pass-manager \
 ; RUN:     -passes='lto<O3>' -S  %s -passes-ep-peephole='no-op-function' 2>&1 \
-; RUN:     | FileCheck %s --check-prefixes=CHECK-O,CHECK-O3,CHECK-O23SZ,CHECK-EP-Peephole
+; RUN:     | FileCheck %s --check-prefixes=CHECK-O,CHECK-O3,CHECK-O23SZ,CHECK-EP-PEEPHOLE
 
 ; CHECK-EP: Running pass: NoOpModulePass
 ; CHECK-O: Running pass: CrossDSOCFIPass
@@ -69,7 +75,7 @@
 ; CHECK-O23SZ-NEXT: Running pass: InstCombinePass
 ; CHECK-O23SZ-NEXT: Running analysis: LastRunTrackingAnalysis
 ; CHECK-O23SZ-NEXT: Running pass: AggressiveInstCombinePass
-; CHECK-EP-Peephole-NEXT: Running pass: NoOpFunctionPass
+; CHECK-EP-PEEPHOLE-NEXT: Running pass: NoOpFunctionPass
 ; CHECK-O23SZ-NEXT: Running pass: ExpandVariadicsPass
 ; CHECK-O23SZ-NEXT: Running pass: ModuleInlinerWrapperPass
 ; CHECK-O23SZ-NEXT: Running analysis: InlineAdvisorAnalysis
@@ -81,7 +87,7 @@
 ; CHECK-O23SZ-NEXT: Running pass: GlobalDCEPass
 ; CHECK-O23SZ-NEXT: Running pass: ArgumentPromotionPass
 ; CHECK-O23SZ-NEXT: Running pass: InstCombinePass
-; CHECK-EP-Peephole-NEXT: Running pass: NoOpFunctionPass
+; CHECK-EP-PEEPHOLE-NEXT: Running pass: NoOpFunctionPass
 ; CHECK-O23SZ-NEXT: Running pass: ConstraintEliminationPass
 ; CHECK-O23SZ-NEXT: Running analysis: LoopAnalysis on foo
 ; CHECK-O23SZ-NEXT: Running analysis: ScalarEvolutionAnalysis on foo
@@ -109,6 +115,7 @@
 ; CHECK-O23SZ-NEXT: Running pass: DSEPass on foo
 ; CHECK-O23SZ-NEXT: Running pass: MoveAutoInitPass on foo
 ; CHECK-O23SZ-NEXT: Running pass: MergedLoadStoreMotionPass on foo
+; CHECK-EP-VECTORIZER-START-NEXT: Running pass: NoOpFunctionPass on foo
 ; CHECK-O23SZ-NEXT: Running pass: LoopSimplifyPass on foo
 ; CHECK-O23SZ-NEXT: Running pass: LCSSAPass on foo
 ; CHECK-O23SZ-NEXT: Running pass: IndVarSimplifyPass on loop
@@ -137,7 +144,8 @@
 ; CHECK-O23SZ-NEXT: Running pass: LCSSAPass
 ; CHECK-O23SZ-NEXT: Running pass: LICMPass
 ; CHECK-O23SZ-NEXT: Running pass: AlignmentFromAssumptionsPass on foo
-; CHECK-EP-Peephole-NEXT: Running pass: NoOpFunctionPass on foo
+; CHECK-EP-VECTORIZER-END-NEXT: Running pass: NoOpFunctionPass on foo
+; CHECK-EP-PEEPHOLE-NEXT: Running pass: NoOpFunctionPass on foo
 ; CHECK-O23SZ-NEXT: Running pass: JumpThreadingPass on foo
 ; CHECK-O23SZ-NEXT: Running pass: LowerTypeTestsPass
 ; CHECK-O-NEXT: Running pass: LowerTypeTestsPass
@@ -146,6 +154,7 @@
 ; CHECK-O23SZ-NEXT: Running pass: SimplifyCFGPass
 ; CHECK-O23SZ-NEXT: Running pass: EliminateAvailableExternallyPass
 ; CHECK-O23SZ-NEXT: Running pass: GlobalDCEPass
+; CHECK-O23SZ-NEXT: Running pass: RelLookupTableConverterPass
 ; CHECK-O23SZ-NEXT: Running pass: CGProfilePass
 ; CHECK-EP-NEXT: Running pass: NoOpModulePass
 ; CHECK-O-NEXT: Running pass: AnnotationRemarksPass on foo
diff --git a/llvm/test/Other/optimize-inrange-gep.ll b/llvm/test/Other/optimize-inrange-gep.ll
index 6dea44005353c..1eab1dfd563c7 100644
--- a/llvm/test/Other/optimize-inrange-gep.ll
+++ b/llvm/test/Other/optimize-inrange-gep.ll
@@ -19,7 +19,7 @@ define void @foo(ptr %p) {
 ; O0-NEXT:    ret void
 ;
 ; CHECK-LABEL: define void @foo(
-; CHECK-SAME: ptr nocapture writeonly initializes((0, 8)) [[P:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
+; CHECK-SAME: ptr writeonly captures(none) initializes((0, 8)) [[P:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
 ; CHECK-NEXT:    store ptr getelementptr inbounds nuw inrange(-24, 0) (i8, ptr @vtable, i64 24), ptr [[P]], align 8
 ; CHECK-NEXT:    ret void
 ;
diff --git a/llvm/test/Other/pass-pipeline-parsing.ll b/llvm/test/Other/pass-pipeline-parsing.ll
index fd65224b1ef0d..4253b646b69b0 100644
--- a/llvm/test/Other/pass-pipeline-parsing.ll
+++ b/llvm/test/Other/pass-pipeline-parsing.ll
@@ -264,6 +264,9 @@
 ; RUN: opt -passes-ep-vectorizer-start=bad -passes=no-op-function \
 ; RUN:       /dev/null -disable-output 2>&1 | FileCheck %s -check-prefix=PASSES-EP-VECTORIZERSTART-ERR
 ; PASSES-EP-VECTORIZERSTART-ERR: Could not parse -passes-ep-vectorizer-start pipeline: unknown function pass 'bad'
+; RUN: opt -passes-ep-vectorizer-end=bad -passes=no-op-function \
+; RUN:       /dev/null -disable-output 2>&1 | FileCheck %s -check-prefix=PASSES-EP-VECTORIZEREND-ERR
+; PASSES-EP-VECTORIZEREND-ERR: Could not parse -passes-ep-vectorizer-end pipeline: unknown function pass 'bad'
 ; RUN: opt -passes-ep-pipeline-start=bad -passes=no-op-function \
 ; RUN:       /dev/null -disable-output 2>&1 | FileCheck %s -check-prefix=PASSES-EP-PIPELINESTART-ERR
 ; PASSES-EP-PIPELINESTART-ERR: Could not parse -passes-ep-pipeline-start pipeline: unknown pass name 'bad'
diff --git a/llvm/test/TableGen/BitsInit.td b/llvm/test/TableGen/BitsInit.td
index c5527aebb9417..0e7fa7acb722f 100644
--- a/llvm/test/TableGen/BitsInit.td
+++ b/llvm/test/TableGen/BitsInit.td
@@ -1,12 +1,10 @@
 
-// RUN: not llvm-tblgen %s 2>&1 > %t
-// RUN: FileCheck %s < %t
+// RUN: llvm-tblgen %s | FileCheck %s
 
 def a {
   bits<2> opc = { 0, 1 };
   bits<2> opc2 = { 1, 0 };
   bits<1> opc3 = { 1 };
-  bits<2> a = { opc, opc2 }; // error!
   bits<2> b = { opc{0}, opc2{0} };
   bits<2> c = { opc{1}, opc2{1} };
   bits<2> c = { opc3{0}, opc3 };
@@ -16,34 +14,25 @@ def a {
 // CHECK:   bits<2> opc = { 0, 1 };
 // CHECK:   bits<2> opc2 = { 1, 0 };
 // CHECK:   bits<1> opc3 = { 1 };
-// CHECK:   bits<2> a;
 // CHECK:   bits<2> b = { 1, 0 };
 // CHECK:   bits<2> c = { 1, 1 };
 // CHECK: }
 
 def {
-  bits<2> B1 = 0b011;  // bitfield is too small, reject
   bits<3> B2 = 0b011;  // ok
 
-  bits<2> C1 = 0b111;  // bitfield is too small, reject
   bits<3> C2 = 0b111;  // ok
 
   bits<2> D1 = { 0, 0 }; // ok
   bits<2> D2 = { 0b00 }; // ok
-  bits<3> D3 = { 0, 0 }; // type mismatch.  RHS doesn't have enough bits
-  bits<3> D4 = { 0b00 }; // type mismatch.  RHS doesn't have enough bits
   bits<1> D5 = { 0 };    // ok
   bits<1> D6 = { 1 };    // ok
-  bits<1> D7 = { 3 };    // type mismatch.  LHS doesn't have enough bits
-  bits<2> D8 = { 0 };    // type mismatch.  RHS doesn't have enough bits
 
   bits<8> E;
   let E{7...0} = {0,0,1,?,?,?,?,?};
   let E{3...0} = 0b0010;
 
   bits<8> F1 = { 0, 1, 0b1001, 0, 0b0 }; // ok
-  bits<7> F2 = { 0, 1, 0b1001, 0, 0b0 }; // LHS doesn't have enough bits
-  bits<9> F3 = { 0, 1, 0b1001, 0, 0b0 }; // RHS doesn't have enough bits
 
   bits<8> G1 = { 0, { 1, 0b1001, 0 }, 0b0 }; // ok
   bits<8> G2 = { 0, { 1, 0b1001 }, 0, 0b0 }; // ok
@@ -63,22 +52,14 @@ def {
 }
 
 // CHECK: def {{.*}} {
-// CHECK: bits<2> B1;
 // CHECK: bits<3> B2 = { 0, 1, 1 };
-// CHECK: bits<2> C1;
 // CHECK: bits<3> C2 = { 1, 1, 1 };
 // CHECK: bits<2> D1 = { 0, 0 };
 // CHECK: bits<2> D2 = { 0, 0 };
-// CHECK: bits<3> D3;
-// CHECK: bits<3> D4;
 // CHECK: bits<1> D5 = { 0 };
 // CHECK: bits<1> D6 = { 1 };
-// CHECK: bits<1> D7 = { !cast<bit>(3) };
-// CHECK: bits<2> D8;
 // CHECK: bits<8> E = { 0, 0, 1, ?, 0, 0, 1, 0 };
 // CHECK: bits<8> F1 = { 0, 1, 1, 0, 0, 1, 0, 0 };
-// CHECK: bits<7> F2;
-// CHECK: bits<9> F3;
 // CHECK: bits<8> G1 = { 0, 1, 1, 0, 0, 1, 0, 0 };
 // CHECK: bits<8> G2 = { 0, 1, 1, 0, 0, 1, 0, 0 };
 // CHECK: bits<8> G3 = { 0, 1, 1, 0, 0, 1, 0, 0 };
diff --git a/llvm/test/TableGen/BitsInitErrors.td b/llvm/test/TableGen/BitsInitErrors.td
new file mode 100644
index 0000000000000..8e8e8bf179e2c
--- /dev/null
+++ b/llvm/test/TableGen/BitsInitErrors.td
@@ -0,0 +1,37 @@
+
+// RUN: not llvm-tblgen %s 2>&1 | FileCheck %s -DFILE=%s
+
+def a {
+  bits<2> opc = { 0, 1 };
+  bits<2> opc2 = { 1, 0 };
+  bits<1> opc3 = { 1 };
+  // CHECK: [[FILE]]:[[@LINE+1]]:15: error: Field 'a' of type 'bits<2>' is incompatible with value '{ opc{1}, opc{0}, opc2{1}, opc2{0} }' of type bit initializer with length 4
+  bits<2> a = { opc, opc2 }; // error!
+}
+
+def {
+  // CHECK: [[FILE]]:[[@LINE+1]]:16: error: Field 'B1' of type 'bits<2>' is incompatible with value '{ 0, 1, 1 }' of type bit initializer with length 3
+  bits<2> B1 = 0b011;  // bitfield is too small, reject
+
+  // CHECK: [[FILE]]:[[@LINE+1]]:16: error: Field 'C1' of type 'bits<2>' is incompatible with value '{ 1, 1, 1 }' of type bit initializer with length 3
+  bits<2> C1 = 0b111;  // bitfield is too small, reject
+
+  // CHECK: [[FILE]]:[[@LINE+1]]:16: error: Field 'D3' of type 'bits<3>' is incompatible with value '{ 0, 0 }' of type bit initializer with length 2
+  bits<3> D3 = { 0, 0 }; // type mismatch.  RHS doesn't have enough bits
+
+  // CHECK: [[FILE]]:[[@LINE+1]]:16: error: Field 'D4' of type 'bits<3>' is incompatible with value '{ 0, 0 }' of type bit initializer with length 2
+  bits<3> D4 = { 0b00 }; // type mismatch.  RHS doesn't have enough bits
+
+  bits<1> D7 = { 3 };    // type mismatch.  LHS doesn't have enough bits
+
+  // CHECK: [[FILE]]:[[@LINE+1]]:16: error: Field 'D8' of type 'bits<2>' is incompatible with value '{ 0 }' of type bit initializer with length 1
+  bits<2> D8 = { 0 };    // type mismatch.  RHS doesn't have enough bits
+
+  // CHECK: [[FILE]]:[[@LINE+1]]:16: error: Field 'F2' of type 'bits<7>' is incompatible with value '{ 0, 1, 1, 0, 0, 1, 0, 0 }' of type bit initializer with length 8
+  bits<7> F2 = { 0, 1, 0b1001, 0, 0b0 }; // LHS doesn't have enough bits
+
+  // CHECK: [[FILE]]:[[@LINE+1]]:16: error: Field 'F3' of type 'bits<9>' is incompatible with value '{ 0, 1, 1, 0, 0, 1, 0, 0 }' of type bit initializer with length 8
+  bits<9> F3 = { 0, 1, 0b1001, 0, 0b0 }; // RHS doesn't have enough bits
+
+  // CHECK: Initializer of 'D7' in 'anonymous_0' could not be fully resolved: { !cast<bit>(3) }
+}
diff --git a/llvm/test/TableGen/HwModeSelect.td b/llvm/test/TableGen/HwModeSelect.td
index e849febe0c4cb..0bac59a92304d 100644
--- a/llvm/test/TableGen/HwModeSelect.td
+++ b/llvm/test/TableGen/HwModeSelect.td
@@ -1,4 +1,4 @@
-// RUN: not --crash llvm-tblgen -gen-dag-isel -I %p/../../include %s 2>&1 | FileCheck %s
+// RUN: not llvm-tblgen -gen-dag-isel -I %p/../../include %s 2>&1 | FileCheck %s -DFILE=%s
 
 // The HwModeSelect class is intended to serve as a base class for other
 // classes that are then used to select a value based on the HW mode.
@@ -24,7 +24,8 @@ def HasFeat2 : Predicate<"Subtarget->hasFeat2()">;
 def TestMode1 : HwMode<"+feat1", [HasFeat1]>;
 def TestMode2 : HwMode<"+feat2", [HasFeat2]>;
 
+// CHECK: error: assertion failed: The Objects and Modes lists must be the same length
+// CHECK: [[FILE]]:[[@LINE+1]]:5: error: assertion failed in this record
 def BadDef : ValueTypeByHwMode<[TestMode1, TestMode2, DefaultMode],
                                [i8, i16, i32, i64]>;
 
-// CHECK: error: in record BadDef derived from HwModeSelect: the lists Modes and Objects should have the same size
diff --git a/llvm/test/TableGen/MixedCasedMnemonic.td b/llvm/test/TableGen/MixedCasedMnemonic.td
index cb224ac59c6de..14ab104c7e120 100644
--- a/llvm/test/TableGen/MixedCasedMnemonic.td
+++ b/llvm/test/TableGen/MixedCasedMnemonic.td
@@ -41,7 +41,7 @@ def :MnemonicAlias<"InstB", "BInst">;
 
 // Check that the matcher lower()s the mnemonics it matches.
 // MATCHER: static const char MnemonicTable[] =
-// MATCHER-NEXT: "\005ainst\005binst";
+// MATCHER-NEXT: "\000\005ainst\005binst";
 
 // Check that aInst appears before BInst in the match table.
 // This shows that the mnemonics are sorted in a case-insensitive way,
diff --git a/llvm/test/TableGen/SDNodeInfoEmitter/ambiguous-constraints.td b/llvm/test/TableGen/SDNodeInfoEmitter/ambiguous-constraints.td
index 668464190e6d8..c09e2198dbeba 100644
--- a/llvm/test/TableGen/SDNodeInfoEmitter/ambiguous-constraints.td
+++ b/llvm/test/TableGen/SDNodeInfoEmitter/ambiguous-constraints.td
@@ -14,16 +14,17 @@ def my_node_b : SDNode<"MyTargetISD::NODE", SDTypeProfile<1, 0, [SDTCisVT<0, f32
 // CHECK-NEXT:    NODE = ISD::BUILTIN_OP_END,
 // CHECK-NEXT:  };
 
-// CHECK:       static const char MyTargetSDNodeNames[] =
+// CHECK:       static constexpr char MyTargetSDNodeNamesStorage[] =
+// CHECK-NEXT:    "\0"
 // CHECK-NEXT:    "MyTargetISD::NODE\0"
-// CHECK-NEXT:    "\0";
+// CHECK-NEXT:    ;
 
 // CHECK:       static const SDTypeConstraint MyTargetSDTypeConstraints[] = {
 // CHECK-NEXT:    /* dummy */ {SDTCisVT, 0, 0, MVT::INVALID_SIMPLE_VALUE_TYPE}
 // CHECK-NEXT:  };
 // CHECK-EMPTY:
 // CHECK-NEXT:  static const SDNodeDesc MyTargetSDNodeDescs[] = {
-// CHECK-NEXT:      {1, 0, 0, 0, 0, 0, 0, 0}, // NODE
+// CHECK-NEXT:      {1, 0, 0, 0, 0, 1, 0, 0}, // NODE
 // CHECK-NEXT:  };
 // CHECK-EMPTY:
 // CHECK-NEXT:  static const SDNodeInfo MyTargetGenSDNodeInfo(
@@ -54,18 +55,19 @@ def my_node_2b : SDNode<"MyTargetISD::NODE_2", SDTypeProfile<1, 0, [SDTCisVT<0,
 // CHECK-EMPTY:
 // CHECK-NEXT:  } // namespace llvm::MyTargetISD
 
-// CHECK:       static const char MyTargetSDNodeNames[] =
+// CHECK:       static constexpr char MyTargetSDNodeNamesStorage[] =
+// CHECK-NEXT:    "\0"
 // CHECK-NEXT:    "MyTargetISD::NODE_1\0"
 // CHECK-NEXT:    "MyTargetISD::NODE_2\0"
-// CHECK-NEXT:    "\0";
+// CHECK-NEXT:    ;
 
 // CHECK:       static const SDTypeConstraint MyTargetSDTypeConstraints[] = {
 // CHECK-NEXT:    /* 0 */ {SDTCisVT, 0, 0, MVT::i32},
 // CHECK-NEXT:  };
 // CHECK-EMPTY:
 // CHECK-NEXT:  static const SDNodeDesc MyTargetSDNodeDescs[] = {
-// CHECK-NEXT:      {1, 0, 0, 0, 0, 0, 0, 1}, // NODE_1
-// CHECK-NEXT:      {1, 0, 0, 0, 0, 20, 0, 0}, // NODE_2
+// CHECK-NEXT:      {1, 0, 0, 0, 0, 1, 0, 1}, // NODE_1
+// CHECK-NEXT:      {1, 0, 0, 0, 0, 21, 0, 0}, // NODE_2
 // CHECK-NEXT:  };
 // CHECK-EMPTY:
 // CHECK-NEXT:  static const SDNodeInfo MyTargetGenSDNodeInfo(
diff --git a/llvm/test/TableGen/SDNodeInfoEmitter/basic.td b/llvm/test/TableGen/SDNodeInfoEmitter/basic.td
index 5332b4f458dfd..b8bff520bbcaa 100644
--- a/llvm/test/TableGen/SDNodeInfoEmitter/basic.td
+++ b/llvm/test/TableGen/SDNodeInfoEmitter/basic.td
@@ -28,12 +28,16 @@ def MyTarget : Target;
 // CHECK-NEXT:  #pragma GCC diagnostic push
 // CHECK-NEXT:  #pragma GCC diagnostic ignored "-Woverlength-strings"
 // CHECK-NEXT:  #endif
-// CHECK-NEXT:  static const char MyTargetSDNodeNames[] =
-// CHECK-NEXT:    "\0";
+// CHECK-NEXT:  static constexpr char MyTargetSDNodeNamesStorage[] =
+// CHECK-NEXT:    "\0"
+// CHECK-NEXT:    ;
 // CHECK-NEXT:  #ifdef __GNUC__
 // CHECK-NEXT:  #pragma GCC diagnostic pop
 // CHECK-NEXT:  #endif
 // CHECK-EMPTY:
+// CHECK-NEXT:  static constexpr llvm::StringTable MyTargetSDNodeNames =
+// CHECK-NEXT:      MyTargetSDNodeNamesStorage;
+// CHECK-EMPTY:
 // CHECK-NEXT:  static const SDTypeConstraint MyTargetSDTypeConstraints[] = {
 // CHECK-NEXT:    /* dummy */ {SDTCisVT, 0, 0, MVT::INVALID_SIMPLE_VALUE_TYPE}
 // CHECK-NEXT:  };
@@ -70,16 +74,17 @@ def my_noop : SDNode<"MyTargetISD::NOOP", SDTypeProfile<0, 0, []>>;
 // CHECK-EMPTY:
 // CHECK-NEXT:  } // namespace llvm::MyTargetISD
 
-// CHECK:       static const char MyTargetSDNodeNames[] =
+// CHECK:       static constexpr char MyTargetSDNodeNamesStorage[] =
+// CHECK-NEXT:    "\0"
 // CHECK-NEXT:    "MyTargetISD::NOOP\0"
-// CHECK-NEXT:    "\0";
+// CHECK-NEXT:    ;
 
 // CHECK:       static const SDTypeConstraint MyTargetSDTypeConstraints[] = {
 // CHECK-NEXT:    /* dummy */ {SDTCisVT, 0, 0, MVT::INVALID_SIMPLE_VALUE_TYPE}
 // CHECK-NEXT:  };
 // CHECK-EMPTY:
 // CHECK-NEXT:  static const SDNodeDesc MyTargetSDNodeDescs[] = {
-// CHECK-NEXT:      {0, 0, 0, 0, 0, 0, 0, 0}, // NOOP
+// CHECK-NEXT:      {0, 0, 0, 0, 0, 1, 0, 0}, // NOOP
 // CHECK-NEXT:  };
 // CHECK-EMPTY:
 // CHECK-NEXT:  static const SDNodeInfo MyTargetGenSDNodeInfo(
@@ -148,11 +153,12 @@ def my_node_3 : SDNode<
 // CHECK-EMPTY:
 // CHECK-NEXT:  } // namespace llvm::MyTargetISD
 
-// CHECK:       static const char MyTargetSDNodeNames[] =
+// CHECK:       static constexpr char MyTargetSDNodeNamesStorage[] =
+// CHECK-NEXT:    "\0"
 // CHECK-NEXT:    "MyTargetISD::NODE_1\0"
 // CHECK-NEXT:    "MyTargetISD::NODE_2\0"
 // CHECK-NEXT:    "MyTargetISD::NODE_3\0"
-// CHECK-NEXT:    "\0";
+// CHECK-NEXT:    ;
 
 // CHECK:       static const SDTypeConstraint MyTargetSDTypeConstraints[] = {
 // CHECK-NEXT:    /* 0 */ {SDTCisVT, 1, 0, MVT::i2},
@@ -173,9 +179,9 @@ def my_node_3 : SDNode<
 // CHECK-NEXT:  };
 // CHECK-EMPTY:
 // CHECK-NEXT:  static const SDNodeDesc MyTargetSDNodeDescs[] = {
-// CHECK-NEXT:          {1, 1, 0|1<<SDNPHasChain, 0, 0, 0, 0, 2}, // NODE_1
-// CHECK-NEXT:          {3, 1, 0|1<<SDNPVariadic|1<<SDNPMemOperand, 0, 42, 20, 11, 4}, // NODE_2
-// CHECK-NEXT:          {2, -1, 0|1<<SDNPHasChain|1<<SDNPOutGlue|1<<SDNPInGlue|1<<SDNPOptInGlue, 0|1<<SDNFIsStrictFP, 24, 40, 2, 13}, // NODE_3
+// CHECK-NEXT:          {1, 1, 0|1<<SDNPHasChain, 0, 0, 1, 0, 2}, // NODE_1
+// CHECK-NEXT:          {3, 1, 0|1<<SDNPVariadic|1<<SDNPMemOperand, 0, 42, 21, 11, 4}, // NODE_2
+// CHECK-NEXT:          {2, -1, 0|1<<SDNPHasChain|1<<SDNPOutGlue|1<<SDNPInGlue|1<<SDNPOptInGlue, 0|1<<SDNFIsStrictFP, 24, 41, 2, 13}, // NODE_3
 // CHECK-NEXT:  };
 // CHECK-EMPTY:
 // CHECK-NEXT: static const SDNodeInfo MyTargetGenSDNodeInfo(
diff --git a/llvm/test/TableGen/SDNodeInfoEmitter/namespace.td b/llvm/test/TableGen/SDNodeInfoEmitter/namespace.td
index 844c12bd182fc..217fb7c9fd475 100644
--- a/llvm/test/TableGen/SDNodeInfoEmitter/namespace.td
+++ b/llvm/test/TableGen/SDNodeInfoEmitter/namespace.td
@@ -19,8 +19,9 @@ def node_2 : SDNode<"MyCustomISD::NODE", SDTypeProfile<0, 1, [SDTCisVT<0, i2>]>>
 // EMPTY-EMPTY:
 // EMPTY-NEXT:   } // namespace llvm::EmptyISD
 
-// EMPTY:        static const char MyTargetSDNodeNames[] =
-// EMPTY-NEXT:     "\0";
+// EMPTY:        static constexpr char MyTargetSDNodeNamesStorage[] =
+// EMPTY-NEXT:     "\0"
+// EMPTY-NEXT:     ;
 
 // EMPTY:        static const SDTypeConstraint MyTargetSDTypeConstraints[] = {
 // EMPTY-NEXT:     /* dummy */ {SDTCisVT, 0, 0, MVT::INVALID_SIMPLE_VALUE_TYPE}
@@ -43,9 +44,10 @@ def node_2 : SDNode<"MyCustomISD::NODE", SDTypeProfile<0, 1, [SDTCisVT<0, i2>]>>
 // COMMON-EMPTY:
 // COMMON-NEXT:  } // namespace llvm::[[NS]]
 
-// COMMON:       static const char MyTargetSDNodeNames[] =
+// COMMON:       static constexpr char MyTargetSDNodeNamesStorage[] =
+// COMMON-NEXT:    "\0"
 // COMMON-NEXT:    "[[NS]]::NODE\0"
-// COMMON-NEXT:    "\0";
+// COMMON-NEXT:    ;
 
 // COMMON:       static const SDTypeConstraint MyTargetSDTypeConstraints[] = {
 // TARGET-NEXT:    /* 0 */ {SDTCisVT, 0, 0, MVT::i1},
@@ -53,8 +55,8 @@ def node_2 : SDNode<"MyCustomISD::NODE", SDTypeProfile<0, 1, [SDTCisVT<0, i2>]>>
 // COMMON-NEXT:  };
 // COMMON-EMPTY:
 // COMMON-NEXT:  static const SDNodeDesc MyTargetSDNodeDescs[] = {
-// TARGET-NEXT:      {1, 0, 0, 0, 0, 0, 0, 1}, // NODE
-// CUSTOM-NEXT:      {0, 1, 0, 0, 0, 0, 0, 1}, // NODE
+// TARGET-NEXT:      {1, 0, 0, 0, 0, 1, 0, 1}, // NODE
+// CUSTOM-NEXT:      {0, 1, 0, 0, 0, 1, 0, 1}, // NODE
 // COMMON-NEXT:  };
 // COMMON-EMPTY:
 // COMMON-NEXT:  static const SDNodeInfo MyTargetGenSDNodeInfo(
diff --git a/llvm/test/TableGen/SDNodeInfoEmitter/skipped-nodes.td b/llvm/test/TableGen/SDNodeInfoEmitter/skipped-nodes.td
index ed278f262ca8f..abd6ad3bda3bc 100644
--- a/llvm/test/TableGen/SDNodeInfoEmitter/skipped-nodes.td
+++ b/llvm/test/TableGen/SDNodeInfoEmitter/skipped-nodes.td
@@ -69,16 +69,17 @@ def node_5b : SDNode<"MyTargetISD::NODE_5", SDTypeProfile<0, 0, []>, [SDNPHasCha
 // CHECK-NEXT:    COMPAT = ISD::BUILTIN_OP_END,
 // CHECK-NEXT:  };
 
-// CHECK:       static const char MyTargetSDNodeNames[] =
+// CHECK:       static constexpr char MyTargetSDNodeNamesStorage[] =
+// CHECK-NEXT:    "\0"
 // CHECK-NEXT:    "MyTargetISD::COMPAT\0"
-// CHECK-NEXT:    "\0";
+// CHECK-NEXT:    ;
 
 // CHECK:       static const SDTypeConstraint MyTargetSDTypeConstraints[] = {
 // CHECK-NEXT:    /* dummy */ {SDTCisVT, 0, 0, MVT::INVALID_SIMPLE_VALUE_TYPE}
 // CHECK-NEXT:  };
 // CHECK-EMPTY:
 // CHECK-NEXT:  static const SDNodeDesc MyTargetSDNodeDescs[] = {
-// CHECK-NEXT:      {1, -1, 0, 0, 0, 0, 0, 0}, // COMPAT
+// CHECK-NEXT:      {1, -1, 0, 0, 0, 1, 0, 0}, // COMPAT
 // CHECK-NEXT:  };
 // CHECK-EMPTY:
 // CHECK-NEXT:  static const SDNodeInfo MyTargetGenSDNodeInfo(
diff --git a/llvm/test/TableGen/get-operand-type-no-expand.td b/llvm/test/TableGen/get-operand-type-no-expand.td
index 9dfcbfaec76af..a0a8fa957f9b6 100644
--- a/llvm/test/TableGen/get-operand-type-no-expand.td
+++ b/llvm/test/TableGen/get-operand-type-no-expand.td
@@ -5,7 +5,7 @@ include "llvm/Target/Target.td"
 
 def archInstrInfo : InstrInfo { }
 
-def arch : Target {
+def X86 : Target {
   let InstructionSet = archInstrInfo;
 }
 
@@ -26,7 +26,7 @@ def InstA : Instruction {
   let InOperandList = (ins i8complex:$b, i32imm:$c);
   field bits<8> Inst;
   field bits<8> SoftFail = 0;
-  let Namespace = "MyNamespace";
+  let Namespace = "X86";
 }
 
 // RUN: llvm-tblgen -gen-instr-info -I %p/../../include %s \
diff --git a/llvm/test/TableGen/get-operand-type.td b/llvm/test/TableGen/get-operand-type.td
index 6ebda5cffe8af..b2f63cafd6a89 100644
--- a/llvm/test/TableGen/get-operand-type.td
+++ b/llvm/test/TableGen/get-operand-type.td
@@ -1,12 +1,12 @@
 // RUN: llvm-tblgen -gen-instr-info -I %p/../../include %s | FileCheck %s
 
-// Check that getOperandType has the expected info in it
+// Check that getOperandType has the expected info in it.
 
 include "llvm/Target/Target.td"
 
 def archInstrInfo : InstrInfo { }
 
-def arch : Target {
+def X86 : Target {
   let InstructionSet = archInstrInfo;
 }
 
@@ -24,7 +24,7 @@ def InstA : Instruction {
   let InOperandList = (ins OpB:$b, i32imm:$c);
   field bits<8> Inst;
   field bits<8> SoftFail = 0;
-  let Namespace = "MyNamespace";
+  let Namespace = "X86";
 }
 
 def InstB : Instruction {
@@ -33,7 +33,7 @@ def InstB : Instruction {
   let InOperandList = (ins unknown:$x);
   field bits<8> Inst;
   field bits<8> SoftFail = 0;
-  let Namespace = "MyNamespace";
+  let Namespace = "X86";
 }
 
 def InstC : Instruction {
@@ -42,12 +42,12 @@ def InstC : Instruction {
   let InOperandList = (ins RegOp:$x);
   field bits<8> Inst;
   field bits<8> SoftFail = 0;
-  let Namespace = "MyNamespace";
+  let Namespace = "X86";
 }
 
 // CHECK: #ifdef GET_INSTRINFO_OPERAND_TYPE
-// CHECK: static const uint{{.*}}_t Offsets[] = {
-// CHECK: static const {{.*}} OpcodeOperandTypes[] = {
+// CHECK: static constexpr uint{{.*}}_t Offsets[] = {
+// CHECK: static constexpr {{.*}} OpcodeOperandTypes[] = {
 // CHECK:        /* InstA */
 // CHECK-NEXT:   OpA, OpB, i32imm,
 // CHECK-NEXT:   /* InstB */
diff --git a/llvm/test/ThinLTO/X86/funcattrs-prop-unknown.ll b/llvm/test/ThinLTO/X86/funcattrs-prop-unknown.ll
index 48fa8f6bbffed..564c99bd9ad62 100644
--- a/llvm/test/ThinLTO/X86/funcattrs-prop-unknown.ll
+++ b/llvm/test/ThinLTO/X86/funcattrs-prop-unknown.ll
@@ -8,8 +8,8 @@ target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
 ; CHECK-NOT: ; Function Attrs:
-; CHECK: define i32 @indirect(ptr nocapture %0) {
-define i32 @indirect(ptr nocapture) {
+; CHECK: define i32 @indirect(ptr captures(none) %0) {
+define i32 @indirect(ptr captures(none)) {
   %2 = tail call i32 %0()
   ret i32 %2
 }
diff --git a/llvm/test/ThinLTO/X86/memprof-icp-recursive.ll b/llvm/test/ThinLTO/X86/memprof-icp-recursive.ll
new file mode 100644
index 0000000000000..f8dcd80d4e141
--- /dev/null
+++ b/llvm/test/ThinLTO/X86/memprof-icp-recursive.ll
@@ -0,0 +1,260 @@
+;; This test case is the same as memprof-icp.ll but manually modified to
+;; add recursive cycles into the allocation contexts, and it has
+;; -memprof-allow-recursive-callsites enabled to test that handling.
+;; The call contexts are somewhat nonsensical, but designed to provoke
+;; several scenarios that were seen in large applications and caused
+;; failures until they were addressed. In particular, converting mutual
+;; recursion to direct recursion when stack nodes are matched onto inlined
+;; callsites; edge partitioning due to multiple calls with the same debug
+;; info (here because it is an indirect call) requiring updating/moving
+;; of recursive contexts; and the necessary updates of recursive contexts
+;; and edges during cloning.
+
+;; The checks are however not modified from the original memprof-icp.ll test:
+;; when basic recursive callsite handling is enabled (which currently doesn't
+;; clone through recursive cycles but is designed to at least allow cloning
+;; across them for the the non-recursive call sequence of the context) we
+;; should successfully get the same cloning as without any recursion.
+
+;; -stats requires asserts
+; REQUIRES: asserts
+
+; RUN: split-file %s %t
+
+;; For now explicitly turn on this handling, which is off by default.
+; RUN: opt -thinlto-bc %t/main.ll -enable-memprof-indirect-call-support=true >%t/main.o
+; RUN: opt -thinlto-bc %t/foo.ll -enable-memprof-indirect-call-support=true >%t/foo.o
+
+;; First perform in-process ThinLTO
+; RUN: llvm-lto2 run %t/main.o %t/foo.o -enable-memprof-context-disambiguation \
+; RUN:	-enable-memprof-indirect-call-support=true \
+; RUN:  -memprof-allow-recursive-callsites \
+; RUN:  -supports-hot-cold-new \
+; RUN:  -r=%t/foo.o,_Z3fooR2B0j,plx \
+; RUN:  -r=%t/foo.o,_ZN2B03barEj, \
+; RUN:  -r=%t/foo.o,_ZN1B3barEj, \
+; RUN:  -r=%t/main.o,_Z3fooR2B0j, \
+; RUN:  -r=%t/main.o,_Znwm, \
+; RUN:  -r=%t/main.o,_ZdlPvm, \
+; RUN:  -r=%t/main.o,_Z8externalPi, \
+; RUN:  -r=%t/main.o,main,plx \
+; RUN:  -r=%t/main.o,_ZN2B03barEj,plx \
+; RUN:  -r=%t/main.o,_ZN1B3barEj,plx \
+; RUN:  -r=%t/main.o,_ZTV1B,plx \
+; RUN:  -r=%t/main.o,_ZTVN10__cxxabiv120__si_class_type_infoE,plx \
+; RUN:  -r=%t/main.o,_ZTS1B,plx \
+; RUN:  -r=%t/main.o,_ZTVN10__cxxabiv117__class_type_infoE,plx \
+; RUN:  -r=%t/main.o,_ZTS2B0,plx \
+; RUN:  -r=%t/main.o,_ZTI2B0,plx \
+; RUN:  -r=%t/main.o,_ZTI1B,plx \
+; RUN:  -r=%t/main.o,_ZTV2B0,plx \
+; RUN:	-thinlto-threads=1 \
+; RUN:  -memprof-verify-ccg -memprof-verify-nodes -stats \
+; RUN:  -pass-remarks=. -save-temps \
+; RUN:  -o %t.out 2>&1 | FileCheck %s --check-prefix=STATS \
+; RUN:  --check-prefix=REMARKS
+
+; RUN: llvm-dis %t.out.2.4.opt.bc -o - | FileCheck %s --check-prefix=IR
+
+; REMARKS: call in clone main assigned to call function clone _Z3fooR2B0j.memprof.1
+; REMARKS: call in clone main assigned to call function clone _Z3fooR2B0j.memprof.1
+; REMARKS: created clone _ZN2B03barEj.memprof.1
+; REMARKS: call in clone _ZN2B03barEj marked with memprof allocation attribute notcold
+; REMARKS: call in clone _ZN2B03barEj.memprof.1 marked with memprof allocation attribute cold
+; REMARKS: call in clone _ZN2B03barEj marked with memprof allocation attribute notcold
+; REMARKS: call in clone _ZN2B03barEj.memprof.1 marked with memprof allocation attribute cold
+; REMARKS: created clone _ZN1B3barEj.memprof.1
+; REMARKS: call in clone _ZN1B3barEj marked with memprof allocation attribute notcold
+; REMARKS: call in clone _ZN1B3barEj.memprof.1 marked with memprof allocation attribute cold
+; REMARKS: call in clone _ZN1B3barEj marked with memprof allocation attribute notcold
+; REMARKS: call in clone _ZN1B3barEj.memprof.1 marked with memprof allocation attribute cold
+; REMARKS: created clone _Z3fooR2B0j.memprof.1
+;; In each version of foo we should have promoted the indirect call to two conditional
+;; direct calls, one to B::bar and one to B0::bar. The cloned version of foo should call
+;; the cloned versions of bar for both promotions.
+; REMARKS: Promote indirect call to _ZN1B3barEj with count 2 out of 4
+; REMARKS: call in clone _Z3fooR2B0j promoted and assigned to call function clone _ZN1B3barEj
+; REMARKS: Promote indirect call to _ZN1B3barEj with count 2 out of 4
+; REMARKS: call in clone _Z3fooR2B0j.memprof.1 promoted and assigned to call function clone _ZN1B3barEj.memprof.1
+; REMARKS: Promote indirect call to _ZN2B03barEj with count 2 out of 2
+; REMARKS: call in clone _Z3fooR2B0j promoted and assigned to call function clone _ZN2B03barEj
+; REMARKS: Promote indirect call to _ZN2B03barEj with count 2 out of 2
+; REMARKS: call in clone _Z3fooR2B0j.memprof.1 promoted and assigned to call function clone _ZN2B03barEj.memprof.1
+; REMARKS: created clone _ZN2B03barEj.memprof.1
+; REMARKS: call in clone _ZN2B03barEj marked with memprof allocation attribute notcold
+; REMARKS: call in clone _ZN2B03barEj.memprof.1 marked with memprof allocation attribute cold
+; REMARKS: created clone _ZN1B3barEj.memprof.1
+; REMARKS: call in clone _ZN1B3barEj marked with memprof allocation attribute notcold
+; REMARKS: call in clone _ZN1B3barEj.memprof.1 marked with memprof allocation attribute cold
+
+; STATS: 4 memprof-context-disambiguation - Number of cold static allocations (possibly cloned) during whole program analysis
+; STATS: 8 memprof-context-disambiguation - Number of cold static allocations (possibly cloned) during ThinLTO backend
+; STATS: 4 memprof-context-disambiguation - Number of not cold static allocations (possibly cloned) during whole program analysis
+; STATS: 8 memprof-context-disambiguation - Number of not cold static allocations (possibly cloned) during ThinLTO backend
+; STATS: 3 memprof-context-disambiguation - Number of function clones created during whole program analysis
+; STATS: 5 memprof-context-disambiguation - Number of function clones created during ThinLTO backend
+
+; IR: define {{.*}} @_Z3fooR2B0j(
+; IR:   %[[R1:[0-9]+]] = icmp eq ptr %0, @_ZN1B3barEj
+; IR:   br i1 %[[R1]], label %if.true.direct_targ, label %if.false.orig_indirect
+; IR: if.true.direct_targ:
+; IR:   call {{.*}} @_Znwm(i64 noundef 4) #[[NOTCOLD:[0-9]+]]
+; IR: if.false.orig_indirect:
+; IR:   %[[R2:[0-9]+]] = icmp eq ptr %0, @_ZN2B03barEj
+; IR:   br i1 %[[R2]], label %if.true.direct_targ1, label %if.false.orig_indirect2
+; IR: if.true.direct_targ1:
+; IR:   call {{.*}} @_Znwm(i64 noundef 4) #[[NOTCOLD]]
+; IR: if.false.orig_indirect2:
+; IR:   call {{.*}} %0
+
+; IR: define {{.*}} @_Z3fooR2B0j.memprof.1(
+;; We should still compare against the original versions of bar since that is
+;; what is in the vtable. However, we should have called the cloned versions
+;; that perform cold allocations, which were subsequently inlined.
+; IR:   %[[R3:[0-9]+]] = icmp eq ptr %0, @_ZN1B3barEj
+; IR:   br i1 %[[R3]], label %if.true.direct_targ, label %if.false.orig_indirect
+; IR: if.true.direct_targ:
+; IR:   call {{.*}} @_Znwm(i64 noundef 4) #[[COLD:[0-9]+]]
+; IR: if.false.orig_indirect:
+; IR:   %[[R4:[0-9]+]] = icmp eq ptr %0, @_ZN2B03barEj
+; IR:   br i1 %[[R4]], label %if.true.direct_targ1, label %if.false.orig_indirect2
+; IR: if.true.direct_targ1:
+; IR:   call {{.*}} @_Znwm(i64 noundef 4) #[[COLD]]
+; IR: if.false.orig_indirect2:
+; IR:   call {{.*}} %0
+
+; IR: attributes #[[NOTCOLD]] = {{.*}} "memprof"="notcold"
+; IR: attributes #[[COLD]] = {{.*}} "memprof"="cold"
+
+;--- foo.ll
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+declare i32 @_ZN2B03barEj(ptr %this, i32 %s)
+declare i32 @_ZN1B3barEj(ptr %this, i32 %s)
+
+define i32 @_Z3fooR2B0j(ptr %b) {
+entry:
+  %0 = load ptr, ptr %b, align 8
+  %call = tail call i32 %0(ptr null, i32 0), !prof !0, !callsite !1
+  ret i32 0
+}
+
+!0 = !{!"VP", i32 0, i64 4, i64 4445083295448962937, i64 2, i64 -2718743882639408571, i64 2}
+!1 = !{i64 -2101080423462424381, i64 3456}
+
+;--- main.ll
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+@_ZTV1B = external constant { [3 x ptr] }
+@_ZTVN10__cxxabiv120__si_class_type_infoE = external global [0 x ptr]
+@_ZTS1B = external constant [3 x i8]
+@_ZTVN10__cxxabiv117__class_type_infoE = external global [0 x ptr]
+@_ZTS2B0 = external constant [4 x i8]
+@_ZTI2B0 = external constant { ptr, ptr }
+@_ZTI1B = external constant { ptr, ptr, ptr }
+@_ZTV2B0 = external constant { [3 x ptr] }
+
+define i32 @main() !prof !29 {
+entry:
+  %call2 = call i32 @_Z3fooR2B0j(ptr null, i32 0), !callsite !30
+  %call4 = call i32 @_Z3fooR2B0j(ptr null, i32 0), !callsite !31
+  %call6 = call i32 @_Z3fooR2B0j(ptr null, i32 0), !callsite !32
+  %call8 = call i32 @_Z3fooR2B0j(ptr null, i32 0), !callsite !57
+  ret i32 0
+}
+
+declare i32 @_Z3fooR2B0j(ptr, i32)
+
+define i32 @_ZN2B03barEj(ptr %this, i32 %s) {
+entry:
+  %call = tail call ptr @_Znwm(i64 noundef 4) #0, !memprof !33, !callsite !38
+  ;; Second allocation in this function, to ensure that indirect edges to the
+  ;; same callee are partitioned correctly.
+  %call2 = tail call ptr @_Znwm(i64 noundef 4) #0, !memprof !45, !callsite !50
+  store volatile i32 0, ptr %call, align 4
+  ret i32 0
+}
+
+declare ptr @_Znwm(i64)
+
+declare void @_Z8externalPi()
+
+declare void @_ZdlPvm()
+
+define i32 @_ZN1B3barEj(ptr %this, i32 %s) {
+entry:
+  %call = tail call ptr @_Znwm(i64 noundef 4) #0, !memprof !39, !callsite !44
+  ;; Second allocation in this function, to ensure that indirect edges to the
+  ;; same callee are partitioned correctly.
+  %call2 = tail call ptr @_Znwm(i64 noundef 4) #0, !memprof !51, !callsite !56
+  store volatile i32 0, ptr %call, align 4
+  ret i32 0
+}
+
+; uselistorder directives
+uselistorder ptr @_Z3fooR2B0j, { 3, 2, 1, 0 }
+
+attributes #0 = { builtin allocsize(0) }
+
+!llvm.module.flags = !{!0}
+
+!0 = !{i32 1, !"ProfileSummary", !1}
+!1 = !{!2, !3, !4, !5, !6, !7, !8, !9, !10, !11}
+!2 = !{!"ProfileFormat", !"InstrProf"}
+!3 = !{!"TotalCount", i64 13}
+!4 = !{!"MaxCount", i64 4}
+!5 = !{!"MaxInternalCount", i64 0}
+!6 = !{!"MaxFunctionCount", i64 4}
+!7 = !{!"NumCounts", i64 5}
+!8 = !{!"NumFunctions", i64 5}
+!9 = !{!"IsPartialProfile", i64 0}
+!10 = !{!"PartialProfileRatio", double 0.000000e+00}
+!11 = !{!"DetailedSummary", !12}
+!12 = !{!13, !14, !15, !16, !17, !18, !19, !20, !21, !22, !23, !24, !25, !26, !27, !28}
+!13 = !{i32 10000, i64 0, i32 0}
+!14 = !{i32 100000, i64 4, i32 2}
+!15 = !{i32 200000, i64 4, i32 2}
+!16 = !{i32 300000, i64 4, i32 2}
+!17 = !{i32 400000, i64 4, i32 2}
+!18 = !{i32 500000, i64 4, i32 2}
+!19 = !{i32 600000, i64 4, i32 2}
+!20 = !{i32 700000, i64 2, i32 4}
+!21 = !{i32 800000, i64 2, i32 4}
+!22 = !{i32 900000, i64 2, i32 4}
+!23 = !{i32 950000, i64 2, i32 4}
+!24 = !{i32 990000, i64 2, i32 4}
+!25 = !{i32 999000, i64 2, i32 4}
+!26 = !{i32 999900, i64 2, i32 4}
+!27 = !{i32 999990, i64 2, i32 4}
+!28 = !{i32 999999, i64 2, i32 4}
+!29 = !{!"function_entry_count", i64 1}
+!30 = !{i64 -6490791336773930154}
+!31 = !{i64 5188446645037944434}
+!32 = !{i64 5583420417449503557}
+!57 = !{i64 132626519179914298}
+!33 = !{!34, !36}
+!34 = !{!35, !"notcold"}
+!35 = !{i64 -852997907418798798, i64 -2101080423462424381, i64 3456, i64 -2101080423462424381, i64 3456, i64 5188446645037944434}
+!36 = !{!37, !"cold"}
+!37 = !{i64 -852997907418798798, i64 -2101080423462424381, i64 3456, i64 -2101080423462424381, i64 3456, i64 5583420417449503557}
+!38 = !{i64 -852997907418798798}
+!39 = !{!40, !42}
+!40 = !{!41, !"notcold"}
+!41 = !{i64 4457553070050523782, i64 -2101080423462424381, i64 3456, i64 -2101080423462424381, i64 3456, i64 132626519179914298}
+!42 = !{!43, !"cold"}
+!43 = !{i64 4457553070050523782, i64 -2101080423462424381, i64 3456, i64 -2101080423462424381, i64 3456, i64 -6490791336773930154}
+!44 = !{i64 4457553070050523782}
+!45 = !{!46, !48}
+!46 = !{!47, !"notcold"}
+!47 = !{i64 456, i64 -2101080423462424381, i64 3456, i64 5188446645037944434}
+!48 = !{!49, !"cold"}
+!49 = !{i64 456, i64 -2101080423462424381, i64 3456, i64 5583420417449503557}
+!50 = !{i64 456}
+!51 = !{!52, !54}
+!52 = !{!53, !"notcold"}
+!53 = !{i64 789, i64 -2101080423462424381, i64 3456, i64 132626519179914298}
+!54 = !{!55, !"cold"}
+!55 = !{i64 789, i64 -2101080423462424381, i64 3456, i64 -6490791336773930154}
+!56 = !{i64 789}
diff --git a/llvm/test/ThinLTO/X86/memprof-recursive.ll b/llvm/test/ThinLTO/X86/memprof-recursive.ll
index 1f4f75460b070..5be27c3e380fc 100644
--- a/llvm/test/ThinLTO/X86/memprof-recursive.ll
+++ b/llvm/test/ThinLTO/X86/memprof-recursive.ll
@@ -39,7 +39,7 @@
 ; RUN:  --implicit-check-not="created clone" \
 ; RUN:	--implicit-check-not="marked with memprof allocation attribute cold"
 
-;; Check the default behavior (disabled recursive callsites).
+;; Check the default behavior (enabled recursive callsites).
 ; RUN: llvm-lto2 run %t.o -enable-memprof-context-disambiguation \
 ; RUN:  -supports-hot-cold-new \
 ; RUN:  -r=%t.o,_Z1Dv,plx \
@@ -49,10 +49,9 @@
 ; RUN:  -r=%t.o,_Znam, \
 ; RUN:  -memprof-verify-ccg -memprof-verify-nodes \
 ; RUN:  -pass-remarks=memprof-context-disambiguation \
-; RUN:  -o %t.out 2>&1 | FileCheck %s --allow-empty \
+; RUN:  -o %t.out 2>&1 | FileCheck %s \
 ; RUN:  --implicit-check-not "memprof_recursive3.cc:12:10: call in clone _Z1Ci.memprof.1 assigned" \
-; RUN:  --implicit-check-not="created clone" \
-; RUN:	--implicit-check-not="marked with memprof allocation attribute cold"
+; RUN:  --check-prefix=ALLOW-RECUR-CALLSITES --check-prefix=ALLOW-RECUR-CONTEXTS
 
 ;; Skipping recursive contexts should prevent spurious call to cloned version of
 ;; B from the context starting at memprof_recursive.cc:19:13, which is actually
diff --git a/llvm/test/ThinLTO/X86/memprof_imported_internal.ll b/llvm/test/ThinLTO/X86/memprof_imported_internal.ll
new file mode 100644
index 0000000000000..a6e254ca4d586
--- /dev/null
+++ b/llvm/test/ThinLTO/X86/memprof_imported_internal.ll
@@ -0,0 +1,152 @@
+;; Test to make sure that the memprof ThinLTO backend finds the correct summary
+;; for an imported promoted local, so that we can perform the correct cloning.
+;; In particular, we should be able to use the thinlto_src_file metadata to
+;; recreate its original GUID. In particular, this test contains promoted
+;; internal functions with the same original name as those that were imported,
+;; and we want to ensure we don't use those by mistake.
+
+;; The original code looks something like:
+;;
+;; src1.cc:
+;; extern void external1();
+;; extern void external2();
+;; static void internal1() {
+;;   external2();
+;; }
+;; static void internal2() {
+;;   external2();
+;; }
+;; int main() {
+;;   internal1();
+;;   internal2();
+;;   external1();
+;;   return 0;
+;; }
+;;
+;; src2.cc:
+;; extern void external2();
+;; static void internal1() {
+;;   external2();
+;; }
+;; static void internal2() {
+;;   external2();
+;; }
+;; void external1() {
+;;   internal1();
+;;   internal2();
+;; }
+;;
+;; The assembly for src1 shown below was dumped after function importing, with
+;; some hand modification to ensure we import the definitions of src2.cc's
+;; external1 and internal1 functions, and the declaration only for its
+;; internal2 function. I also hand modified it to add !callsite metadata
+;; to a few calls, and the distributed ThinLTO summary in src1.o.thinlto.ll to
+;; contain callsite metadata records with cloning results.
+
+; RUN: rm -rf %t && split-file %s %t && cd %t
+; RUN: llvm-as src1.ll -o src1.o
+; RUN: llvm-as src1.o.thinlto.ll -o src1.o.thinlto.bc
+
+; RUN: opt -passes=memprof-context-disambiguation src1.o -S -memprof-import-summary=src1.o.thinlto.bc | FileCheck %s
+
+;; Per the cloning results in the summary, none of the original functions should
+;; call any memprof clones.
+; CHECK-NOT: memprof
+;; We should have one clone of src1.cc's internal1 that calls a clone of
+;; external2.
+; CHECK-LABEL: define void @_ZL9internal1v.llvm.5985484347676238233.memprof.1()
+; CHECK:  tail call void @_Z9external2v.memprof.1()
+; CHECK-LABEL: declare void @_Z9external2v.memprof.1()
+;; We should have one clone of external1 that calls a clone of internal2 from
+;; a synthesized callsite record (for a tail call with a missing frame).
+; CHECK-LABEL: define available_externally {{.*}} void @_Z9external1v.memprof.1()
+; CHECK:  tail call void @_ZL9internal1v.llvm.3267420853450984672()
+; CHECK:  tail call void @_ZL9internal2v.llvm.3267420853450984672.memprof.1()
+; CHECK-LABEL: declare void @_ZL9internal2v.llvm.3267420853450984672.memprof.1()
+;; We should have 2 clones of src2.cc's internal1 function, calling a single
+;; clone of external2.
+; CHECK-LABEL: define available_externally void @_ZL9internal1v.llvm.3267420853450984672.memprof.1()
+; CHECK:  tail call void @_Z9external2v.memprof.1()
+; CHECK:  tail call void @_Z9external2v.memprof.1()
+; CHECK-LABEL: define available_externally void @_ZL9internal1v.llvm.3267420853450984672.memprof.2()
+; CHECK:  tail call void @_Z9external2v.memprof.1()
+; CHECK:  tail call void @_Z9external2v.memprof.1()
+; CHECK-NOT: memprof
+
+;--- src1.ll
+; ModuleID = 'src1.o'
+source_filename = "src1.cc"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+define dso_local noundef i32 @main() {
+entry:
+  tail call void @_ZL9internal1v.llvm.5985484347676238233()
+  tail call void @_ZL9internal2v.llvm.5985484347676238233()
+  tail call void @_Z9external1v()
+  ret i32 0
+}
+
+define void @_ZL9internal1v.llvm.5985484347676238233() {
+entry:
+  tail call void @_Z9external2v(), !callsite !8
+  ret void
+}
+
+define void @_ZL9internal2v.llvm.5985484347676238233() {
+entry:
+  tail call void @_Z9external2v()
+  ret void
+}
+
+declare void @_Z9external2v()
+
+define available_externally dso_local void @_Z9external1v() !thinlto_src_module !6 !thinlto_src_file !7 {
+entry:
+  tail call void @_ZL9internal1v.llvm.3267420853450984672()
+  tail call void @_ZL9internal2v.llvm.3267420853450984672()
+  ret void
+}
+
+define available_externally void @_ZL9internal1v.llvm.3267420853450984672() !thinlto_src_module !6 !thinlto_src_file !7 {
+entry:
+  ;; This one has more callsite records than the other version of internal1,
+  ;; which would cause the code to iterate past the end of the callsite
+  ;; records if we incorrectly got the other internal1's summary.
+  tail call void @_Z9external2v(), !callsite !9
+  tail call void @_Z9external2v(), !callsite !10
+  ret void
+}
+
+declare void @_ZL9internal2v.llvm.3267420853450984672()
+
+!6 = !{!"src2.o"}
+!7 = !{!"src2.cc"}
+!8 = !{i64 12345}
+!9 = !{i64 23456}
+!10 = !{i64 34567}
+
+;--- src1.o.thinlto.ll
+; ModuleID = 'src1.o.thinlto.bc'
+source_filename = "src1.o.thinlto.bc"
+
+^0 = module: (path: "src1.o", hash: (1393604173, 1072112025, 2857473630, 2016801496, 3238735916))
+^1 = module: (path: "src2.o", hash: (760755700, 1705397472, 4198605753, 677969311, 2408738824))
+;; src2.o:internal1. It specifies that we should have 3 clones total (including
+;; original).
+^3 = gv: (guid: 1143217136900127394, summaries: (function: (module: ^1, flags: (linkage: available_externally, visibility: default, notEligibleToImport: 0, live: 1, dsoLocal: 1, canAutoHide: 0, importType: definition), insts: 2, funcFlags: (readNone: 0, readOnly: 0, noRecurse: 0, returnDoesNotAlias: 0, noInline: 1, alwaysInline: 0, noUnwind: 0, mayThrow: 0, hasUnknownCall: 0, mustBeUnreachable: 0), calls: ((callee: ^6, tail: 1), (callee: ^6, tail: 1)), callsites: ((callee: ^6, clones: (0, 1, 1), stackIds: (23456)), (callee: ^6, clones: (0, 1, 1), stackIds: (34567))))))
+;; src2.o:internal2. It was manually modified to have importType = declaration.
+^4 = gv: (guid: 3599593882704738259, summaries: (function: (module: ^1, flags: (linkage: available_externally, visibility: default, notEligibleToImport: 0, live: 1, dsoLocal: 1, canAutoHide: 0, importType: declaration), insts: 2, funcFlags: (readNone: 0, readOnly: 0, noRecurse: 0, returnDoesNotAlias: 0, noInline: 1, alwaysInline: 0, noUnwind: 0, mayThrow: 0, hasUnknownCall: 0, mustBeUnreachable: 0), calls: ((callee: ^6, tail: 1)))))
+;; src1.o:internal1.
+^5 = gv: (guid: 6084810090198994915, summaries: (function: (module: ^0, flags: (linkage: internal, visibility: default, notEligibleToImport: 0, live: 1, dsoLocal: 1, canAutoHide: 0, importType: definition), insts: 2, funcFlags: (readNone: 0, readOnly: 0, noRecurse: 0, returnDoesNotAlias: 0, noInline: 1, alwaysInline: 0, noUnwind: 0, mayThrow: 0, hasUnknownCall: 0, mustBeUnreachable: 0), calls: ((callee: ^6, tail: 1)), callsites: ((callee: ^6, clones: (0, 1), stackIds: (12345))))))
+^6 = gv: (guid: 8596367375252297795)
+;; src1.o:internal2.
+^7 = gv: (guid: 11092151021205906565, summaries: (function: (module: ^0, flags: (linkage: internal, visibility: default, notEligibleToImport: 0, live: 1, dsoLocal: 1, canAutoHide: 0, importType: definition), insts: 2, funcFlags: (readNone: 0, readOnly: 0, noRecurse: 0, returnDoesNotAlias: 0, noInline: 1, alwaysInline: 0, noUnwind: 0, mayThrow: 0, hasUnknownCall: 0, mustBeUnreachable: 0), calls: ((callee: ^6, tail: 1)))))
+;; src2.o:external1. It contains a synthesized callsite record for the tail call
+;; to internal2 (the empty stackId list indicates it is synthesized for a
+;; discovered missing tail call frame.
+^8 = gv: (guid: 12313225385227428720, summaries: (function: (module: ^1, flags: (linkage: external, visibility: default, notEligibleToImport: 0, live: 1, dsoLocal: 1, canAutoHide: 0, importType: definition), insts: 3, calls: ((callee: ^3, tail: 1), (callee: ^4, tail: 1)), callsites: ((callee: ^4, clones: (0, 1), stackIds: ())))))
+;; src1.o:main.
+^9 = gv: (guid: 15822663052811949562, summaries: (function: (module: ^0, flags: (linkage: external, visibility: default, notEligibleToImport: 0, live: 1, dsoLocal: 1, canAutoHide: 0, importType: definition), insts: 4, funcFlags: (readNone: 0, readOnly: 0, noRecurse: 1, returnDoesNotAlias: 0, noInline: 0, alwaysInline: 0, noUnwind: 0, mayThrow: 0, hasUnknownCall: 0, mustBeUnreachable: 0), calls: ((callee: ^5, tail: 1), (callee: ^7, tail: 1), (callee: ^8, tail: 1)))))
+^10 = flags: 97
+^11 = blockcount: 0
diff --git a/llvm/test/ThinLTO/X86/memprof_imported_internal2.ll b/llvm/test/ThinLTO/X86/memprof_imported_internal2.ll
new file mode 100644
index 0000000000000..b431747fc7306
--- /dev/null
+++ b/llvm/test/ThinLTO/X86/memprof_imported_internal2.ll
@@ -0,0 +1,93 @@
+;; Test to make sure that the memprof ThinLTO backend finds the correct summary
+;; when there was a naming conflict between an internal function in the original
+;; module and an imported external function with the same name. The IR linking
+;; will automatically append a "." followed by a numbered suffix to the existing
+;; local name in that case. Note this can happen with C where the mangling would
+;; be the same for the internal and external functions of the same name (C++
+;; would have different mangling).
+
+;; Note we don't need any MemProf related metadata for this to fail to find a
+;; ValueInfo and crash if the wrong GUID is computed for the renamed local.
+
+;; The original code looks something like:
+;;
+;; src1.c:
+;; extern void external1();
+;; extern void external2();
+;; static void foo() {
+;;   external2();
+;; }
+;; int main() {
+;;   external1();
+;;   foo();
+;;   return 0;
+;; }
+;;
+;; src2.c:
+;; extern void external2();
+;; void foo() {
+;;   external2();
+;; }
+;; void external1() {
+;;   foo();
+;; }
+;;
+;; The assembly for src1 shown below was dumped after function importing.
+
+; RUN: rm -rf %t && split-file %s %t && cd %t
+; RUN: llvm-as src1.ll -o src1.o
+; RUN: llvm-as src1.o.thinlto.ll -o src1.o.thinlto.bc
+
+;; Simply check that we don't crash when trying to find the ValueInfo for each
+;; function in the IR.
+; RUN: opt -passes=memprof-context-disambiguation src1.o -S -memprof-import-summary=src1.o.thinlto.bc
+
+;--- src1.ll
+; ModuleID = 'src1.o'
+source_filename = "src1.c"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+define dso_local noundef i32 @main() {
+entry:
+  tail call void @external1()
+  tail call void @foo.2()
+  ret i32 0
+}
+
+define internal void @foo.2() {
+entry:
+  tail call void @external2()
+  ret void
+}
+
+declare void @external2()
+
+define available_externally dso_local void @foo() !thinlto_src_module !1 !thinlto_src_file !2 {
+entry:
+  tail call void @external2()
+  ret void
+}
+
+define available_externally dso_local void @external1() !thinlto_src_module !1 !thinlto_src_file !2 {
+entry:
+  tail call void @foo()
+  ret void
+}
+
+!1 = !{!"src2.o"}
+!2 = !{!"src2.c"}
+
+;--- src1.o.thinlto.ll
+; ModuleID = 'src1.o.thinlto.bc'
+source_filename = "src1.o.thinlto.bc"
+
+^0 = module: (path: "src1.o", hash: (2435941910, 498944982, 2551913764, 2759430100, 3918124321))
+^1 = module: (path: "src2.o", hash: (1826286437, 1557684621, 1220464477, 2734102338, 1025249503))
+^2 = module: (path: "src3.o", hash: (1085916433, 503665945, 2163560042, 340524, 2255774964))
+^3 = gv: (guid: 1456206394295721279, summaries: (function: (module: ^0, flags: (linkage: internal, visibility: default, notEligibleToImport: 0, live: 1, dsoLocal: 1, canAutoHide: 0, importType: definition), insts: 2, funcFlags: (readNone: 0, readOnly: 0, noRecurse: 0, returnDoesNotAlias: 0, noInline: 1, alwaysInline: 0, noUnwind: 1, mayThrow: 0, hasUnknownCall: 0, mustBeUnreachable: 0)))) ;; src1.c:foo
+^4 = gv: (guid: 6699318081062747564, summaries: (function: (module: ^1, flags: (linkage: external, visibility: default, notEligibleToImport: 0, live: 1, dsoLocal: 1, canAutoHide: 0, importType: definition), insts: 2, funcFlags: (readNone: 0, readOnly: 0, noRecurse: 0, returnDoesNotAlias: 0, noInline: 1, alwaysInline: 0, noUnwind: 1, mayThrow: 0, hasUnknownCall: 0, mustBeUnreachable: 0)))) ;; src2.c:foo
+^5 = gv: (guid: 13087145834073153720, summaries: (function: (module: ^1, flags: (linkage: external, visibility: default, notEligibleToImport: 0, live: 1, dsoLocal: 1, canAutoHide: 0, importType: definition), insts: 2, funcFlags: (readNone: 0, readOnly: 0, noRecurse: 0, returnDoesNotAlias: 0, noInline: 0, alwaysInline: 0, noUnwind: 1, mayThrow: 0, hasUnknownCall: 0, mustBeUnreachable: 0), calls: ((callee: ^4, tail: 1))))) ;; src1.c:external1
+^6 = gv: (guid: 15822663052811949562, summaries: (function: (module: ^0, flags: (linkage: external, visibility: default, notEligibleToImport: 0, live: 1, dsoLocal: 1, canAutoHide: 0, importType: definition), insts: 3, funcFlags: (readNone: 0, readOnly: 0, noRecurse: 0, returnDoesNotAlias: 0, noInline: 0, alwaysInline: 0, noUnwind: 1, mayThrow: 0, hasUnknownCall: 0, mustBeUnreachable: 0), calls: ((callee: ^5, tail: 1), (callee: ^3, tail: 1))))) ;; src1.c:main
+^8 = flags: 97
+^9 = blockcount: 0
diff --git a/llvm/test/Transforms/AggressiveInstCombine/memchr.ll b/llvm/test/Transforms/AggressiveInstCombine/memchr.ll
index 2601b9f05a97f..b26320be634b8 100644
--- a/llvm/test/Transforms/AggressiveInstCombine/memchr.ll
+++ b/llvm/test/Transforms/AggressiveInstCombine/memchr.ll
@@ -161,3 +161,16 @@ entry:
   %memchr = call ptr @memchr(ptr @str_long, i32 %x, i64 8)
   ret ptr %memchr
 }
+
+; We want to check that the compiler still calls memchr if the length is non-constant:
+define ptr @test_memchr_non_constant_length2(i32 %x, i64 %len) {
+; CHECK-LABEL: define ptr @test_memchr_non_constant_length2(
+; CHECK-SAME: i32 [[X:%.*]], i64 [[LEN:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[MEMCHR:%.*]] = call ptr @memchr(ptr @str, i32 [[X]], i64 [[LEN]])
+; CHECK-NEXT:    ret ptr [[MEMCHR]]
+;
+entry:
+  %memchr = call ptr @memchr(ptr @str, i32 %x, i64 %len)
+  ret ptr %memchr
+}
diff --git a/llvm/test/Transforms/AlignmentFromAssumptions/alignment-from-assumptions-track-users.ll b/llvm/test/Transforms/AlignmentFromAssumptions/alignment-from-assumptions-track-users.ll
index 1c388528ae4b1..79d9aa815e010 100644
--- a/llvm/test/Transforms/AlignmentFromAssumptions/alignment-from-assumptions-track-users.ll
+++ b/llvm/test/Transforms/AlignmentFromAssumptions/alignment-from-assumptions-track-users.ll
@@ -3,7 +3,7 @@
 
 define void @widget(ptr addrspace(1) nocapture readonly %arg, ptr addrspace(3) nocapture %arg1) {
 ; CHECK-LABEL: define void @widget(
-; CHECK-SAME: ptr addrspace(1) nocapture readonly [[ARG:%.*]], ptr addrspace(3) nocapture [[ARG1:%.*]]) {
+; CHECK-SAME: ptr addrspace(1) readonly captures(none) [[ARG:%.*]], ptr addrspace(3) captures(none) [[ARG1:%.*]]) {
 ; CHECK-NEXT:  bb:
 ; CHECK-NEXT:    [[GETELEMENTPTR:%.*]] = getelementptr inbounds i32, ptr addrspace(1) [[ARG]], i64 1
 ; CHECK-NEXT:    call void @llvm.assume(i1 true) [ "align"(ptr addrspace(1) [[GETELEMENTPTR]], i64 4) ]
@@ -23,7 +23,7 @@ bb:
 
 define void @wibble(ptr addrspace(1) nocapture readonly %arg, i32 %arg2, ptr addrspace(3) nocapture %arg3) {
 ; CHECK-LABEL: define void @wibble(
-; CHECK-SAME: ptr addrspace(1) nocapture readonly [[ARG:%.*]], i32 [[ARG2:%.*]], ptr addrspace(3) nocapture [[ARG3:%.*]]) {
+; CHECK-SAME: ptr addrspace(1) readonly captures(none) [[ARG:%.*]], i32 [[ARG2:%.*]], ptr addrspace(3) captures(none) [[ARG3:%.*]]) {
 ; CHECK-NEXT:  bb:
 ; CHECK-NEXT:    [[ICMP:%.*]] = icmp ugt i32 [[ARG2]], 10
 ; CHECK-NEXT:    br i1 [[ICMP]], label [[BB4:%.*]], label [[BB5:%.*]]
@@ -66,7 +66,7 @@ bb7:                                              ; preds = %bb5, %bb4
 
 define void @ham(ptr addrspace(1) nocapture readonly %arg, i32 %arg2, ptr addrspace(3) nocapture %arg3) {
 ; CHECK-LABEL: define void @ham(
-; CHECK-SAME: ptr addrspace(1) nocapture readonly [[ARG:%.*]], i32 [[ARG2:%.*]], ptr addrspace(3) nocapture [[ARG3:%.*]]) {
+; CHECK-SAME: ptr addrspace(1) readonly captures(none) [[ARG:%.*]], i32 [[ARG2:%.*]], ptr addrspace(3) captures(none) [[ARG3:%.*]]) {
 ; CHECK-NEXT:  bb:
 ; CHECK-NEXT:    [[GETELEMENTPTR:%.*]] = getelementptr i32, ptr addrspace(1) [[ARG]], i32 0
 ; CHECK-NEXT:    [[GETELEMENTPTR4:%.*]] = getelementptr i32, ptr addrspace(1) [[ARG]], i32 10
@@ -114,7 +114,7 @@ bb10:                                             ; preds = %bb5, %bb
 
 define void @quux(ptr addrspace(1) nocapture readonly %arg, i32 %arg2, ptr addrspace(3) nocapture %arg3) {
 ; CHECK-LABEL: define void @quux(
-; CHECK-SAME: ptr addrspace(1) nocapture readonly [[ARG:%.*]], i32 [[ARG2:%.*]], ptr addrspace(3) nocapture [[ARG3:%.*]]) {
+; CHECK-SAME: ptr addrspace(1) readonly captures(none) [[ARG:%.*]], i32 [[ARG2:%.*]], ptr addrspace(3) captures(none) [[ARG3:%.*]]) {
 ; CHECK-NEXT:  bb:
 ; CHECK-NEXT:    [[ICMP:%.*]] = icmp ugt i32 [[ARG2]], 10
 ; CHECK-NEXT:    [[GETELEMENTPTR:%.*]] = getelementptr i32, ptr addrspace(1) [[ARG]], i32 6
@@ -140,7 +140,7 @@ bb:
 
 define void @widget.1(ptr addrspace(1) nocapture readonly %arg, i32 %arg2, ptr addrspace(3) nocapture %arg3) {
 ; CHECK-LABEL: define void @widget.1(
-; CHECK-SAME: ptr addrspace(1) nocapture readonly [[ARG:%.*]], i32 [[ARG2:%.*]], ptr addrspace(3) nocapture [[ARG3:%.*]]) {
+; CHECK-SAME: ptr addrspace(1) readonly captures(none) [[ARG:%.*]], i32 [[ARG2:%.*]], ptr addrspace(3) captures(none) [[ARG3:%.*]]) {
 ; CHECK-NEXT:  bb:
 ; CHECK-NEXT:    [[ADDRSPACECAST:%.*]] = addrspacecast ptr addrspace(3) [[ARG3]] to ptr addrspace(1)
 ; CHECK-NEXT:    [[GETELEMENTPTR:%.*]] = getelementptr i32, ptr addrspace(1) [[ADDRSPACECAST]]
@@ -162,7 +162,7 @@ bb:
 
 define void @baz(ptr addrspace(1) nocapture readonly %arg, ptr addrspace(3) nocapture %arg1) {
 ; CHECK-LABEL: define void @baz(
-; CHECK-SAME: ptr addrspace(1) nocapture readonly [[ARG:%.*]], ptr addrspace(3) nocapture [[ARG1:%.*]]) {
+; CHECK-SAME: ptr addrspace(1) readonly captures(none) [[ARG:%.*]], ptr addrspace(3) captures(none) [[ARG1:%.*]]) {
 ; CHECK-NEXT:  bb:
 ; CHECK-NEXT:    [[GETELEMENTPTR:%.*]] = getelementptr ptr addrspace(1), ptr addrspace(1) [[ARG]], i64 16
 ; CHECK-NEXT:    call void @llvm.assume(i1 true) [ "align"(ptr addrspace(1) [[ARG]], i64 4) ]
@@ -182,7 +182,7 @@ bb:
 
 define void @foo(ptr addrspace(1) nocapture readonly %arg, i32 %arg1) {
 ; CHECK-LABEL: define void @foo(
-; CHECK-SAME: ptr addrspace(1) nocapture readonly [[ARG:%.*]], i32 [[ARG1:%.*]]) {
+; CHECK-SAME: ptr addrspace(1) readonly captures(none) [[ARG:%.*]], i32 [[ARG1:%.*]]) {
 ; CHECK-NEXT:  bb:
 ; CHECK-NEXT:    [[GETELEMENTPTR:%.*]] = getelementptr ptr addrspace(3), ptr addrspace(1) [[ARG]], i64 16
 ; CHECK-NEXT:    call void @llvm.assume(i1 true) [ "align"(ptr addrspace(1) [[ARG]], i64 4) ]
diff --git a/llvm/test/Transforms/AlignmentFromAssumptions/simple.ll b/llvm/test/Transforms/AlignmentFromAssumptions/simple.ll
index 42f36f1abcef1..c2f2d21fafc40 100644
--- a/llvm/test/Transforms/AlignmentFromAssumptions/simple.ll
+++ b/llvm/test/Transforms/AlignmentFromAssumptions/simple.ll
@@ -4,7 +4,7 @@ target datalayout = "e-i64:64-f80:128-n8:16:32:64-S128"
 
 define i32 @foo(ptr nocapture %a) {
 ; CHECK-LABEL: define i32 @foo
-; CHECK-SAME: (ptr nocapture [[A:%.*]]) {
+; CHECK-SAME: (ptr captures(none) [[A:%.*]]) {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    tail call void @llvm.assume(i1 true) [ "align"(ptr [[A]], i32 32) ]
 ; CHECK-NEXT:    [[TMP0:%.*]] = load i32, ptr [[A]], align 32
@@ -18,7 +18,7 @@ entry:
 
 define i32 @foo2(ptr nocapture %a) {
 ; CHECK-LABEL: define i32 @foo2
-; CHECK-SAME: (ptr nocapture [[A:%.*]]) {
+; CHECK-SAME: (ptr captures(none) [[A:%.*]]) {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    tail call void @llvm.assume(i1 true) [ "align"(ptr [[A]], i32 32, i32 24) ]
 ; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 -2
@@ -34,7 +34,7 @@ entry:
 
 define i32 @foo2a(ptr nocapture %a) {
 ; CHECK-LABEL: define i32 @foo2a
-; CHECK-SAME: (ptr nocapture [[A:%.*]]) {
+; CHECK-SAME: (ptr captures(none) [[A:%.*]]) {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    tail call void @llvm.assume(i1 true) [ "align"(ptr [[A]], i32 32, i32 28) ]
 ; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 1
@@ -51,7 +51,7 @@ entry:
 ; TODO: this can be 8-bytes aligned
 define i32 @foo2b(ptr nocapture %a) {
 ; CHECK-LABEL: define i32 @foo2b
-; CHECK-SAME: (ptr nocapture [[A:%.*]]) {
+; CHECK-SAME: (ptr captures(none) [[A:%.*]]) {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    tail call void @llvm.assume(i1 true) [ "align"(ptr [[A]], i32 32, i32 28) ]
 ; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 -1
@@ -67,7 +67,7 @@ entry:
 
 define i32 @goo(ptr nocapture %a) {
 ; CHECK-LABEL: define i32 @goo
-; CHECK-SAME: (ptr nocapture [[A:%.*]]) {
+; CHECK-SAME: (ptr captures(none) [[A:%.*]]) {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    tail call void @llvm.assume(i1 true) [ "align"(ptr [[A]], i32 32, i32 0) ]
 ; CHECK-NEXT:    [[TMP0:%.*]] = load i32, ptr [[A]], align 32
@@ -81,7 +81,7 @@ entry:
 
 define i32 @hoo(ptr nocapture %a) {
 ; CHECK-LABEL: define i32 @hoo
-; CHECK-SAME: (ptr nocapture [[A:%.*]]) {
+; CHECK-SAME: (ptr captures(none) [[A:%.*]]) {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    tail call void @llvm.assume(i1 true) [ "align"(ptr [[A]], i64 32, i32 0) ]
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
@@ -127,7 +127,7 @@ for.end:                                          ; preds = %for.body
 ;         load(a, i0+i1+i2+32)
 define void @hoo2(ptr nocapture %a, i64 %id, i64 %num) {
 ; CHECK-LABEL: define void @hoo2
-; CHECK-SAME: (ptr nocapture [[A:%.*]], i64 [[ID:%.*]], i64 [[NUM:%.*]]) {
+; CHECK-SAME: (ptr captures(none) [[A:%.*]], i64 [[ID:%.*]], i64 [[NUM:%.*]]) {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    tail call void @llvm.assume(i1 true) [ "align"(ptr [[A]], i8 32, i64 0) ]
 ; CHECK-NEXT:    [[ID_MUL:%.*]] = shl nsw i64 [[ID]], 6
@@ -203,7 +203,7 @@ return:
 
 define i32 @joo(ptr nocapture %a) {
 ; CHECK-LABEL: define i32 @joo
-; CHECK-SAME: (ptr nocapture [[A:%.*]]) {
+; CHECK-SAME: (ptr captures(none) [[A:%.*]]) {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    tail call void @llvm.assume(i1 true) [ "align"(ptr [[A]], i8 32, i8 0) ]
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
@@ -243,7 +243,7 @@ for.end:                                          ; preds = %for.body
 
 define i32 @koo(ptr nocapture %a) {
 ; CHECK-LABEL: define i32 @koo
-; CHECK-SAME: (ptr nocapture [[A:%.*]]) {
+; CHECK-SAME: (ptr captures(none) [[A:%.*]]) {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK:       for.body:
@@ -283,7 +283,7 @@ for.end:                                          ; preds = %for.body
 
 define i32 @koo2(ptr nocapture %a) {
 ; CHECK-LABEL: define i32 @koo2
-; CHECK-SAME: (ptr nocapture [[A:%.*]]) {
+; CHECK-SAME: (ptr captures(none) [[A:%.*]]) {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    tail call void @llvm.assume(i1 true) [ "align"(ptr [[A]], i128 32, i128 0) ]
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
@@ -323,7 +323,7 @@ for.end:                                          ; preds = %for.body
 
 define i32 @moo(ptr nocapture %a) {
 ; CHECK-LABEL: define i32 @moo
-; CHECK-SAME: (ptr nocapture [[A:%.*]]) {
+; CHECK-SAME: (ptr captures(none) [[A:%.*]]) {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    tail call void @llvm.assume(i1 true) [ "align"(ptr [[A]], i16 32) ]
 ; CHECK-NEXT:    tail call void @llvm.memset.p0.i64(ptr align 32 [[A]], i8 0, i64 64, i1 false)
@@ -337,7 +337,7 @@ entry:
 
 define i32 @moo2(ptr nocapture %a, ptr nocapture %b) {
 ; CHECK-LABEL: define i32 @moo2
-; CHECK-SAME: (ptr nocapture [[A:%.*]], ptr nocapture [[B:%.*]]) {
+; CHECK-SAME: (ptr captures(none) [[A:%.*]], ptr captures(none) [[B:%.*]]) {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    tail call void @llvm.assume(i1 true) [ "align"(ptr [[B]], i32 128) ]
 ; CHECK-NEXT:    tail call void @llvm.assume(i1 true) [ "align"(ptr [[A]], i16 32) ]
@@ -353,7 +353,7 @@ entry:
 
 define i32 @moo3(ptr nocapture %a, ptr nocapture %b) {
 ; CHECK-LABEL: define i32 @moo3
-; CHECK-SAME: (ptr nocapture [[A:%.*]], ptr nocapture [[B:%.*]]) {
+; CHECK-SAME: (ptr captures(none) [[A:%.*]], ptr captures(none) [[B:%.*]]) {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    tail call void @llvm.assume(i1 true) [ "align"(ptr [[A]], i16 32), "align"(ptr [[B]], i32 128) ]
 ; CHECK-NEXT:    tail call void @llvm.memcpy.p0.p0.i64(ptr align 32 [[A]], ptr align 128 [[B]], i64 64, i1 false)
@@ -369,7 +369,7 @@ entry:
 ; Variable alignments appear to be legal, don't crash
 define i32 @pr51680(ptr nocapture %a, i32 %align) {
 ; CHECK-LABEL: define i32 @pr51680
-; CHECK-SAME: (ptr nocapture [[A:%.*]], i32 [[ALIGN:%.*]]) {
+; CHECK-SAME: (ptr captures(none) [[A:%.*]], i32 [[ALIGN:%.*]]) {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    tail call void @llvm.assume(i1 true) [ "align"(ptr [[A]], i32 [[ALIGN]]) ]
 ; CHECK-NEXT:    [[TMP0:%.*]] = load i32, ptr [[A]], align 4
@@ -383,7 +383,7 @@ entry:
 
 define i32 @pr64687(ptr nocapture %a) {
 ; CHECK-LABEL: define i32 @pr64687
-; CHECK-SAME: (ptr nocapture [[A:%.*]]) {
+; CHECK-SAME: (ptr captures(none) [[A:%.*]]) {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    tail call void @llvm.assume(i1 true) [ "align"(ptr [[A]], i32 123) ]
 ; CHECK-NEXT:    [[TMP0:%.*]] = load i32, ptr [[A]], align 4
diff --git a/llvm/test/Transforms/AlignmentFromAssumptions/simple32.ll b/llvm/test/Transforms/AlignmentFromAssumptions/simple32.ll
index 4b20673c1ed77..407d620704188 100644
--- a/llvm/test/Transforms/AlignmentFromAssumptions/simple32.ll
+++ b/llvm/test/Transforms/AlignmentFromAssumptions/simple32.ll
@@ -4,7 +4,7 @@ target datalayout = "e-m:e-p:32:32-i64:64-v128:64:128-n32-S64"
 
 define i32 @foo(ptr nocapture %a) nounwind uwtable readonly {
 ; CHECK-LABEL: define {{[^@]+}}@foo
-; CHECK-SAME: (ptr nocapture [[A:%.*]]) #0
+; CHECK-SAME: (ptr captures(none) [[A:%.*]]) #[[ATTR0:[0-9]+]] {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    call void @llvm.assume(i1 true) [ "align"(ptr [[A]], i64 32) ]
 ; CHECK-NEXT:    [[TMP0:%.*]] = load i32, ptr [[A]], align 32
@@ -19,7 +19,7 @@ entry:
 
 define i32 @foo2(ptr nocapture %a) nounwind uwtable readonly {
 ; CHECK-LABEL: define {{[^@]+}}@foo2
-; CHECK-SAME: (ptr nocapture [[A:%.*]]) #0
+; CHECK-SAME: (ptr captures(none) [[A:%.*]]) #[[ATTR0]] {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    call void @llvm.assume(i1 true) [ "align"(ptr [[A]], i64 32, i64 24) ]
 ; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 -2
@@ -36,7 +36,7 @@ entry:
 
 define i32 @foo2a(ptr nocapture %a) nounwind uwtable readonly {
 ; CHECK-LABEL: define {{[^@]+}}@foo2a
-; CHECK-SAME: (ptr nocapture [[A:%.*]]) #0
+; CHECK-SAME: (ptr captures(none) [[A:%.*]]) #[[ATTR0]] {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    call void @llvm.assume(i1 true) [ "align"(ptr [[A]], i64 32, i64 28) ]
 ; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 1
@@ -54,7 +54,7 @@ entry:
 ; TODO: this can be 8-bytes aligned
 define i32 @foo2b(ptr nocapture %a) nounwind uwtable readonly {
 ; CHECK-LABEL: define {{[^@]+}}@foo2b
-; CHECK-SAME: (ptr nocapture [[A:%.*]]) #0
+; CHECK-SAME: (ptr captures(none) [[A:%.*]]) #[[ATTR0]] {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    call void @llvm.assume(i1 true) [ "align"(ptr [[A]], i64 32, i64 28) ]
 ; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 -1
@@ -71,7 +71,7 @@ entry:
 
 define i32 @goo(ptr nocapture %a) nounwind uwtable readonly {
 ; CHECK-LABEL: define {{[^@]+}}@goo
-; CHECK-SAME: (ptr nocapture [[A:%.*]]) #0
+; CHECK-SAME: (ptr captures(none) [[A:%.*]]) #[[ATTR0]] {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    call void @llvm.assume(i1 true) [ "align"(ptr [[A]], i64 32) ]
 ; CHECK-NEXT:    [[TMP0:%.*]] = load i32, ptr [[A]], align 32
@@ -86,7 +86,7 @@ entry:
 
 define i32 @hoo(ptr nocapture %a) nounwind uwtable readonly {
 ; CHECK-LABEL: define {{[^@]+}}@hoo
-; CHECK-SAME: (ptr nocapture [[A:%.*]]) #0
+; CHECK-SAME: (ptr captures(none) [[A:%.*]]) #[[ATTR0]] {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    call void @llvm.assume(i1 true) [ "align"(ptr [[A]], i64 32) ]
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
@@ -127,7 +127,7 @@ for.end:                                          ; preds = %for.body
 
 define i32 @joo(ptr nocapture %a) nounwind uwtable readonly {
 ; CHECK-LABEL: define {{[^@]+}}@joo
-; CHECK-SAME: (ptr nocapture [[A:%.*]]) #0
+; CHECK-SAME: (ptr captures(none) [[A:%.*]]) #[[ATTR0]] {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    call void @llvm.assume(i1 true) [ "align"(ptr [[A]], i64 32) ]
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
@@ -168,7 +168,7 @@ for.end:                                          ; preds = %for.body
 
 define i32 @koo(ptr nocapture %a) nounwind uwtable readonly {
 ; CHECK-LABEL: define {{[^@]+}}@koo
-; CHECK-SAME: (ptr nocapture [[A:%.*]]) #0
+; CHECK-SAME: (ptr captures(none) [[A:%.*]]) #[[ATTR0]] {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    call void @llvm.assume(i1 true) [ "align"(ptr [[A]], i64 32) ]
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
@@ -209,7 +209,7 @@ for.end:                                          ; preds = %for.body
 
 define i32 @koo2(ptr nocapture %a) nounwind uwtable readonly {
 ; CHECK-LABEL: define {{[^@]+}}@koo2
-; CHECK-SAME: (ptr nocapture [[A:%.*]]) #0
+; CHECK-SAME: (ptr captures(none) [[A:%.*]]) #[[ATTR0]] {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    call void @llvm.assume(i1 true) [ "align"(ptr [[A]], i64 32) ]
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
@@ -250,7 +250,7 @@ for.end:                                          ; preds = %for.body
 
 define i32 @moo(ptr nocapture %a) nounwind uwtable {
 ; CHECK-LABEL: define {{[^@]+}}@moo
-; CHECK-SAME: (ptr nocapture [[A:%.*]]) #1
+; CHECK-SAME: (ptr captures(none) [[A:%.*]]) #[[ATTR1:[0-9]+]] {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    call void @llvm.assume(i1 true) [ "align"(ptr [[A]], i64 32) ]
 ; CHECK-NEXT:    tail call void @llvm.memset.p0.i64(ptr align 32 [[A]], i8 0, i64 64, i1 false)
@@ -265,7 +265,7 @@ entry:
 
 define i32 @moo2(ptr nocapture %a, ptr nocapture %b) nounwind uwtable {
 ; CHECK-LABEL: define {{[^@]+}}@moo2
-; CHECK-SAME: (ptr nocapture [[A:%.*]], ptr nocapture [[B:%.*]]) #1
+; CHECK-SAME: (ptr captures(none) [[A:%.*]], ptr captures(none) [[B:%.*]]) #[[ATTR1]] {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    call void @llvm.assume(i1 true) [ "align"(ptr [[A]], i64 32) ]
 ; CHECK-NEXT:    call void @llvm.assume(i1 true) [ "align"(ptr [[B]], i64 128) ]
diff --git a/llvm/test/Transforms/AlignmentFromAssumptions/start-unk.ll b/llvm/test/Transforms/AlignmentFromAssumptions/start-unk.ll
index 349906e8c5db5..f6c8b09fd3d96 100644
--- a/llvm/test/Transforms/AlignmentFromAssumptions/start-unk.ll
+++ b/llvm/test/Transforms/AlignmentFromAssumptions/start-unk.ll
@@ -74,17 +74,17 @@ if.end123:                                        ; preds = %for.end
   br i1 %arg, label %if.end150, label %if.then126
 
 if.then126:                                       ; preds = %if.end123
-  %ptrint.i.i185 = ptrtoint ptr undef to i64
+  %ptrint.i.i185 = ptrtoint ptr poison to i64
   %maskedptr.i.i186 = and i64 %ptrint.i.i185, 1
   %maskcond.i.i187 = icmp eq i64 %maskedptr.i.i186, 0
   tail call void @llvm.assume(i1 %maskcond.i.i187) #0
-  %ret.0.copyload.i.i189 = load i32, ptr undef, align 2
+  %ret.0.copyload.i.i189 = load i32, ptr poison, align 2
 
 ; CHECK: load {{.*}} align 2
 
   %0 = tail call i32 @llvm.bswap.i32(i32 %ret.0.copyload.i.i189) #0
   %conv131 = zext i32 %0 to i64
-  %add.ptr132 = getelementptr inbounds i8, ptr undef, i64 %conv131
+  %add.ptr132 = getelementptr inbounds i8, ptr poison, i64 %conv131
   br i1 %arg, label %if.end150, label %if.end.i173
 
 if.end.i173:                                      ; preds = %if.then126
diff --git a/llvm/test/Transforms/ArgumentPromotion/BPF/argpromotion.ll b/llvm/test/Transforms/ArgumentPromotion/BPF/argpromotion.ll
index 6c39f27115ada..0620198b63adc 100644
--- a/llvm/test/Transforms/ArgumentPromotion/BPF/argpromotion.ll
+++ b/llvm/test/Transforms/ArgumentPromotion/BPF/argpromotion.ll
@@ -62,7 +62,7 @@ entry:
 ; Without number-of-argument constraint, argpromotion will create a function signature with 6 arguments. Since
 ; bpf target only supports maximum 5 arguments, so no argpromotion here.
 ;
-; CHECK:  i32 @foo1(ptr nocapture noundef readonly %p1, ptr nocapture noundef readonly %p2, ptr nocapture noundef readonly %p3)
+; CHECK:  i32 @foo1(ptr noundef readonly captures(none) %p1, ptr noundef readonly captures(none) %p2, ptr noundef readonly captures(none) %p3)
 
 define internal i32 @foo2(ptr noundef %p1, ptr noundef %p2, ptr noundef %p3) {
 entry:
diff --git a/llvm/test/Transforms/ArgumentPromotion/actual-arguments.ll b/llvm/test/Transforms/ArgumentPromotion/actual-arguments.ll
index af8da57484541..65874d9a7cf36 100644
--- a/llvm/test/Transforms/ArgumentPromotion/actual-arguments.ll
+++ b/llvm/test/Transforms/ArgumentPromotion/actual-arguments.ll
@@ -10,7 +10,7 @@
 ;
 define internal i32 @test_cannot_promote_1(ptr %p, ptr nocapture readonly %test_c) {
 ; CHECK-LABEL: define {{[^@]+}}@test_cannot_promote_1
-; CHECK-SAME: (ptr [[P:%.*]], ptr nocapture readonly [[TEST_C:%.*]]) {
+; CHECK-SAME: (ptr [[P:%.*]], ptr readonly captures(none) [[TEST_C:%.*]]) {
 ; CHECK-NEXT:    [[TEST_C_VAL:%.*]] = load i32, ptr [[TEST_C]], align 4
 ; CHECK-NEXT:    [[RES:%.*]] = call i32 @callee(ptr [[P]], i32 [[TEST_C_VAL]])
 ; CHECK-NEXT:    [[LTEST_C:%.*]] = load i32, ptr [[TEST_C]], align 4
@@ -31,7 +31,7 @@ define internal i32 @test_cannot_promote_1(ptr %p, ptr nocapture readonly %test_
 ;
 define internal i32 @test_cannot_promote_2(ptr %p, ptr nocapture readonly %test_c) {
 ; CHECK-LABEL: define {{[^@]+}}@test_cannot_promote_2
-; CHECK-SAME: (ptr [[P:%.*]], ptr nocapture readonly [[TEST_C:%.*]]) {
+; CHECK-SAME: (ptr [[P:%.*]], ptr readonly captures(none) [[TEST_C:%.*]]) {
 ; CHECK-NEXT:    [[TEST_C_VAL:%.*]] = load i32, ptr [[TEST_C]], align 4
 ; CHECK-NEXT:    [[RES:%.*]] = call i32 @callee(ptr [[P]], i32 [[TEST_C_VAL]])
 ; CHECK-NEXT:    [[LTEST_C:%.*]] = load i32, ptr [[TEST_C]], align 4
@@ -52,7 +52,7 @@ define internal i32 @test_cannot_promote_2(ptr %p, ptr nocapture readonly %test_
 ;
 define internal i32 @test_cannot_promote_3(ptr %p, ptr nocapture readonly %test_c) {
 ; CHECK-LABEL: define {{[^@]+}}@test_cannot_promote_3
-; CHECK-SAME: (ptr [[P:%.*]], ptr nocapture readonly [[TEST_C:%.*]]) {
+; CHECK-SAME: (ptr [[P:%.*]], ptr readonly captures(none) [[TEST_C:%.*]]) {
 ; CHECK-NEXT:    [[TEST_C_VAL:%.*]] = load i32, ptr [[TEST_C]], align 4
 ; CHECK-NEXT:    [[RES:%.*]] = call i32 @callee(ptr [[P]], i32 [[TEST_C_VAL]])
 ; CHECK-NEXT:    [[LTEST_C:%.*]] = load i32, ptr [[TEST_C]], align 4
diff --git a/llvm/test/Transforms/ArgumentPromotion/aliasing-and-non-aliasing-loads-with-clobber.ll b/llvm/test/Transforms/ArgumentPromotion/aliasing-and-non-aliasing-loads-with-clobber.ll
index 1e1669b29b0db..a02e2c36d6482 100644
--- a/llvm/test/Transforms/ArgumentPromotion/aliasing-and-non-aliasing-loads-with-clobber.ll
+++ b/llvm/test/Transforms/ArgumentPromotion/aliasing-and-non-aliasing-loads-with-clobber.ll
@@ -24,7 +24,7 @@ entry:
 
 define internal void @callee1(ptr nocapture noundef readonly %q, i1 %c) {
 ; CHECK-LABEL: define internal void @callee1(
-; CHECK-SAME: ptr nocapture noundef readonly [[Q:%.*]], i1 [[C:%.*]]) {
+; CHECK-SAME: ptr noundef readonly captures(none) [[Q:%.*]], i1 [[C:%.*]]) {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    br i1 [[C]], label [[THEN:%.*]], label [[EXIT:%.*]]
 ; CHECK:       then:
@@ -69,7 +69,7 @@ entry:
 
 define internal void @callee2(ptr nocapture noundef readonly %q, i1 %c) {
 ; CHECK-LABEL: define internal void @callee2(
-; CHECK-SAME: ptr nocapture noundef readonly [[Q:%.*]], i1 [[C:%.*]]) {
+; CHECK-SAME: ptr noundef readonly captures(none) [[Q:%.*]], i1 [[C:%.*]]) {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    br i1 [[C]], label [[THEN:%.*]], label [[EXIT:%.*]]
 ; CHECK:       then:
diff --git a/llvm/test/Transforms/ArgumentPromotion/fp80.ll b/llvm/test/Transforms/ArgumentPromotion/fp80.ll
index 1e3d01a2361b9..de611e1bcf433 100644
--- a/llvm/test/Transforms/ArgumentPromotion/fp80.ll
+++ b/llvm/test/Transforms/ArgumentPromotion/fp80.ll
@@ -7,7 +7,7 @@ target triple = "x86_64-unknown-linux-gnu"
 %union.u = type { x86_fp80 }
 %struct.s = type { double, i16, i8, [5 x i8] }
 
-@b = internal global %struct.s { double 3.14, i16 9439, i8 25, [5 x i8] undef }, align 16
+@b = internal global %struct.s { double 3.14, i16 9439, i8 25, [5 x i8] poison }, align 16
 
 %struct.Foo = type { i32, i64 }
 @a = internal global %struct.Foo { i32 1, i64 2 }, align 8
diff --git a/llvm/test/Transforms/ArgumentPromotion/musttail.ll b/llvm/test/Transforms/ArgumentPromotion/musttail.ll
index 3947af8ecf979..b0cae3940d716 100644
--- a/llvm/test/Transforms/ArgumentPromotion/musttail.ll
+++ b/llvm/test/Transforms/ArgumentPromotion/musttail.ll
@@ -52,7 +52,7 @@ define internal i32 @test2(ptr %p, i32 %p2) {
 ; CHECK-NEXT:    [[A:%.*]] = load i32, ptr [[A_GEP]], align 4
 ; CHECK-NEXT:    [[B:%.*]] = load i32, ptr [[B_GEP]], align 4
 ; CHECK-NEXT:    [[V:%.*]] = add i32 [[A]], [[B]]
-; CHECK-NEXT:    [[CA:%.*]] = musttail call i32 @foo(ptr undef, i32 [[V]])
+; CHECK-NEXT:    [[CA:%.*]] = musttail call i32 @foo(ptr poison, i32 [[V]])
 ; CHECK-NEXT:    ret i32 [[CA]]
 ;
   %a.gep = getelementptr %T, ptr %p, i64 0, i32 3
@@ -60,7 +60,7 @@ define internal i32 @test2(ptr %p, i32 %p2) {
   %a = load i32, ptr %a.gep
   %b = load i32, ptr %b.gep
   %v = add i32 %a, %b
-  %ca = musttail call i32 @foo(ptr undef, i32 %v)
+  %ca = musttail call i32 @foo(ptr poison, i32 %v)
   ret i32 %ca
 }
 
diff --git a/llvm/test/Transforms/ArgumentPromotion/variadic.ll b/llvm/test/Transforms/ArgumentPromotion/variadic.ll
index 360a7a94f8eaa..2b5345bf78ca3 100644
--- a/llvm/test/Transforms/ArgumentPromotion/variadic.ll
+++ b/llvm/test/Transforms/ArgumentPromotion/variadic.ll
@@ -16,20 +16,20 @@ target triple = "x86_64-unknown-linux-gnu"
 ; Function Attrs: nounwind uwtable
 define i32 @main(i32 %argc, ptr nocapture readnone %argv) #0 {
 ; CHECK-LABEL: define {{[^@]+}}@main
-; CHECK-SAME: (i32 [[ARGC:%.*]], ptr nocapture readnone [[ARGV:%.*]]) {
+; CHECK-SAME: (i32 [[ARGC:%.*]], ptr readnone captures(none) [[ARGV:%.*]]) {
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    tail call void (ptr, ptr, ptr, ptr, ptr, ...) @callee_t0f(ptr undef, ptr undef, ptr undef, ptr undef, ptr undef, ptr byval([[STRUCT_TT0:%.*]]) align 8 @t45)
+; CHECK-NEXT:    tail call void (ptr, ptr, ptr, ptr, ptr, ...) @callee_t0f(ptr poison, ptr poison, ptr poison, ptr poison, ptr poison, ptr byval([[STRUCT_TT0:%.*]]) align 8 @t45)
 ; CHECK-NEXT:    ret i32 0
 ;
 entry:
-  tail call void (ptr, ptr, ptr, ptr, ptr, ...) @callee_t0f(ptr undef, ptr undef, ptr undef, ptr undef, ptr undef, ptr byval(%struct.tt0) align 8 @t45)
+  tail call void (ptr, ptr, ptr, ptr, ptr, ...) @callee_t0f(ptr poison, ptr poison, ptr poison, ptr poison, ptr poison, ptr byval(%struct.tt0) align 8 @t45)
   ret i32 0
 }
 
 ; Function Attrs: nounwind uwtable
 define internal void @callee_t0f(ptr nocapture readnone %tp13, ptr nocapture readnone %tp14, ptr nocapture readnone %tp15, ptr nocapture readnone %tp16, ptr nocapture readnone %tp17, ...) {
 ; CHECK-LABEL: define {{[^@]+}}@callee_t0f
-; CHECK-SAME: (ptr nocapture readnone [[TP13:%.*]], ptr nocapture readnone [[TP14:%.*]], ptr nocapture readnone [[TP15:%.*]], ptr nocapture readnone [[TP16:%.*]], ptr nocapture readnone [[TP17:%.*]], ...) {
+; CHECK-SAME: (ptr readnone captures(none) [[TP13:%.*]], ptr readnone captures(none) [[TP14:%.*]], ptr readnone captures(none) [[TP15:%.*]], ptr readnone captures(none) [[TP16:%.*]], ptr readnone captures(none) [[TP17:%.*]], ...) {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    ret void
 ;
diff --git a/llvm/test/Transforms/Attributor/ArgumentPromotion/2008-07-02-array-indexing.ll b/llvm/test/Transforms/Attributor/ArgumentPromotion/2008-07-02-array-indexing.ll
index efbd1937db636..dca1a1edc1e3a 100644
--- a/llvm/test/Transforms/Attributor/ArgumentPromotion/2008-07-02-array-indexing.ll
+++ b/llvm/test/Transforms/Attributor/ArgumentPromotion/2008-07-02-array-indexing.ll
@@ -9,7 +9,7 @@ define internal i32 @callee(i1 %C, ptr %A) {
 ;
 ; CHECK: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: read)
 ; CHECK-LABEL: define {{[^@]+}}@callee
-; CHECK-SAME: (ptr nocapture nofree noundef nonnull readonly align 4 dereferenceable(4) [[A:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-SAME: (ptr nofree noundef nonnull readonly align 4 captures(none) dereferenceable(4) [[A:%.*]]) #[[ATTR0:[0-9]+]] {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[A_0:%.*]] = load i32, ptr [[A]], align 4
 ; CHECK-NEXT:    br label [[F:%.*]]
@@ -38,14 +38,14 @@ F:
 define i32 @foo(ptr %A) {
 ; TUNIT: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: read)
 ; TUNIT-LABEL: define {{[^@]+}}@foo
-; TUNIT-SAME: (ptr nocapture nofree readonly [[A:%.*]]) #[[ATTR0]] {
-; TUNIT-NEXT:    [[X:%.*]] = call i32 @callee(ptr nocapture nofree noundef readonly align 4 [[A]]) #[[ATTR1:[0-9]+]]
+; TUNIT-SAME: (ptr nofree readonly captures(none) [[A:%.*]]) #[[ATTR0]] {
+; TUNIT-NEXT:    [[X:%.*]] = call i32 @callee(ptr nofree noundef readonly align 4 captures(none) [[A]]) #[[ATTR1:[0-9]+]]
 ; TUNIT-NEXT:    ret i32 [[X]]
 ;
 ; CGSCC: Function Attrs: mustprogress nofree nosync nounwind willreturn memory(argmem: read)
 ; CGSCC-LABEL: define {{[^@]+}}@foo
-; CGSCC-SAME: (ptr nocapture nofree noundef nonnull readonly align 4 dereferenceable(4) [[A:%.*]]) #[[ATTR1:[0-9]+]] {
-; CGSCC-NEXT:    [[X:%.*]] = call i32 @callee(ptr nocapture nofree noundef nonnull readonly align 4 dereferenceable(4) [[A]]) #[[ATTR2:[0-9]+]]
+; CGSCC-SAME: (ptr nofree noundef nonnull readonly align 4 captures(none) dereferenceable(4) [[A:%.*]]) #[[ATTR1:[0-9]+]] {
+; CGSCC-NEXT:    [[X:%.*]] = call i32 @callee(ptr nofree noundef nonnull readonly align 4 captures(none) dereferenceable(4) [[A]]) #[[ATTR2:[0-9]+]]
 ; CGSCC-NEXT:    ret i32 [[X]]
 ;
   %X = call i32 @callee(i1 false, ptr %A)             ; <i32> [#uses=1]
diff --git a/llvm/test/Transforms/Attributor/ArgumentPromotion/2008-09-07-CGUpdate.ll b/llvm/test/Transforms/Attributor/ArgumentPromotion/2008-09-07-CGUpdate.ll
index 50b51ab8c25aa..24f40d625a460 100644
--- a/llvm/test/Transforms/Attributor/ArgumentPromotion/2008-09-07-CGUpdate.ll
+++ b/llvm/test/Transforms/Attributor/ArgumentPromotion/2008-09-07-CGUpdate.ll
@@ -16,13 +16,13 @@ entry:
 define void @encode(ptr %m, ptr %ts, ptr %new) nounwind {
 ; TUNIT: Function Attrs: mustprogress nofree norecurse noreturn nosync nounwind willreturn memory(none)
 ; TUNIT-LABEL: define {{[^@]+}}@encode
-; TUNIT-SAME: (ptr nocapture nofree readnone [[M:%.*]], ptr nocapture nofree readnone [[TS:%.*]], ptr nocapture nofree readnone [[NEW:%.*]]) #[[ATTR0:[0-9]+]] {
+; TUNIT-SAME: (ptr nofree readnone captures(none) [[M:%.*]], ptr nofree readnone captures(none) [[TS:%.*]], ptr nofree readnone captures(none) [[NEW:%.*]]) #[[ATTR0:[0-9]+]] {
 ; TUNIT-NEXT:  entry:
 ; TUNIT-NEXT:    unreachable
 ;
 ; CGSCC: Function Attrs: mustprogress nofree noreturn nosync nounwind willreturn memory(none)
 ; CGSCC-LABEL: define {{[^@]+}}@encode
-; CGSCC-SAME: (ptr nocapture nofree readnone [[M:%.*]], ptr nocapture nofree readnone [[TS:%.*]], ptr nocapture nofree readnone [[NEW:%.*]]) #[[ATTR1:[0-9]+]] {
+; CGSCC-SAME: (ptr nofree readnone captures(none) [[M:%.*]], ptr nofree readnone captures(none) [[TS:%.*]], ptr nofree readnone captures(none) [[NEW:%.*]]) #[[ATTR1:[0-9]+]] {
 ; CGSCC-NEXT:  entry:
 ; CGSCC-NEXT:    unreachable
 ;
diff --git a/llvm/test/Transforms/Attributor/ArgumentPromotion/2008-09-08-CGUpdateSelfEdge.ll b/llvm/test/Transforms/Attributor/ArgumentPromotion/2008-09-08-CGUpdateSelfEdge.ll
index c5f7aacab57bd..dca1f782ea381 100644
--- a/llvm/test/Transforms/Attributor/ArgumentPromotion/2008-09-08-CGUpdateSelfEdge.ll
+++ b/llvm/test/Transforms/Attributor/ArgumentPromotion/2008-09-08-CGUpdateSelfEdge.ll
@@ -27,7 +27,7 @@ bb5:		; preds = %entry
 define i32 @term_Sharing(ptr %Term) nounwind {
 ; CHECK: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(none)
 ; CHECK-LABEL: define {{[^@]+}}@term_Sharing
-; CHECK-SAME: (ptr nocapture nofree readnone [[TERM:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-SAME: (ptr nofree readnone captures(none) [[TERM:%.*]]) #[[ATTR0:[0-9]+]] {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    br i1 false, label [[BB_I:%.*]], label [[BB14:%.*]]
 ; CHECK:       bb.i:
diff --git a/llvm/test/Transforms/Attributor/ArgumentPromotion/X86/attributes.ll b/llvm/test/Transforms/Attributor/ArgumentPromotion/X86/attributes.ll
index 23415d3d3262b..73231cc4524fe 100644
--- a/llvm/test/Transforms/Attributor/ArgumentPromotion/X86/attributes.ll
+++ b/llvm/test/Transforms/Attributor/ArgumentPromotion/X86/attributes.ll
@@ -9,7 +9,7 @@ target triple = "x86_64-unknown-linux-gnu"
 define internal fastcc void @no_promote_avx2(ptr %arg, ptr readonly %arg1) #0 {
 ; CHECK: Function Attrs: inlinehint mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: readwrite) uwtable
 ; CHECK-LABEL: define {{[^@]+}}@no_promote_avx2
-; CHECK-SAME: (ptr noalias nocapture nofree noundef nonnull writeonly align 32 dereferenceable(32) [[ARG:%.*]], ptr noalias nocapture nofree noundef nonnull readonly align 32 dereferenceable(32) [[ARG1:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-SAME: (ptr noalias nofree noundef nonnull writeonly align 32 captures(none) dereferenceable(32) [[ARG:%.*]], ptr noalias nofree noundef nonnull readonly align 32 captures(none) dereferenceable(32) [[ARG1:%.*]]) #[[ATTR0:[0-9]+]] {
 ; CHECK-NEXT:  bb:
 ; CHECK-NEXT:    [[TMP:%.*]] = load <4 x i64>, ptr [[ARG1]], align 32
 ; CHECK-NEXT:    store <4 x i64> [[TMP]], ptr [[ARG]], align 32
@@ -24,24 +24,24 @@ bb:
 define void @no_promote(ptr %arg) #1 {
 ; TUNIT: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: readwrite) uwtable
 ; TUNIT-LABEL: define {{[^@]+}}@no_promote
-; TUNIT-SAME: (ptr nocapture nofree writeonly [[ARG:%.*]]) #[[ATTR1:[0-9]+]] {
+; TUNIT-SAME: (ptr nofree writeonly captures(none) [[ARG:%.*]]) #[[ATTR1:[0-9]+]] {
 ; TUNIT-NEXT:  bb:
 ; TUNIT-NEXT:    [[TMP:%.*]] = alloca <4 x i64>, align 32
 ; TUNIT-NEXT:    [[TMP2:%.*]] = alloca <4 x i64>, align 32
-; TUNIT-NEXT:    call void @llvm.memset.p0.i64(ptr noalias nocapture nofree noundef nonnull writeonly align 32 dereferenceable(32) [[TMP]], i8 noundef 0, i64 noundef 32, i1 noundef false) #[[ATTR3:[0-9]+]]
-; TUNIT-NEXT:    call fastcc void @no_promote_avx2(ptr noalias nocapture nofree noundef nonnull writeonly align 32 dereferenceable(32) [[TMP2]], ptr noalias nocapture nofree noundef nonnull readonly align 32 dereferenceable(32) [[TMP]]) #[[ATTR4:[0-9]+]]
+; TUNIT-NEXT:    call void @llvm.memset.p0.i64(ptr noalias nofree noundef nonnull writeonly align 32 captures(none) dereferenceable(32) [[TMP]], i8 noundef 0, i64 noundef 32, i1 noundef false) #[[ATTR3:[0-9]+]]
+; TUNIT-NEXT:    call fastcc void @no_promote_avx2(ptr noalias nofree noundef nonnull writeonly align 32 captures(none) dereferenceable(32) [[TMP2]], ptr noalias nofree noundef nonnull readonly align 32 captures(none) dereferenceable(32) [[TMP]]) #[[ATTR4:[0-9]+]]
 ; TUNIT-NEXT:    [[TMP4:%.*]] = load <4 x i64>, ptr [[TMP2]], align 32
 ; TUNIT-NEXT:    store <4 x i64> [[TMP4]], ptr [[ARG]], align 2
 ; TUNIT-NEXT:    ret void
 ;
 ; CGSCC: Function Attrs: mustprogress nofree nosync nounwind willreturn memory(argmem: readwrite) uwtable
 ; CGSCC-LABEL: define {{[^@]+}}@no_promote
-; CGSCC-SAME: (ptr nocapture nofree noundef nonnull writeonly align 2 dereferenceable(32) [[ARG:%.*]]) #[[ATTR1:[0-9]+]] {
+; CGSCC-SAME: (ptr nofree noundef nonnull writeonly align 2 captures(none) dereferenceable(32) [[ARG:%.*]]) #[[ATTR1:[0-9]+]] {
 ; CGSCC-NEXT:  bb:
 ; CGSCC-NEXT:    [[TMP:%.*]] = alloca <4 x i64>, align 32
 ; CGSCC-NEXT:    [[TMP2:%.*]] = alloca <4 x i64>, align 32
-; CGSCC-NEXT:    call void @llvm.memset.p0.i64(ptr noalias nocapture nofree noundef nonnull writeonly align 32 dereferenceable(32) [[TMP]], i8 noundef 0, i64 noundef 32, i1 noundef false) #[[ATTR3:[0-9]+]]
-; CGSCC-NEXT:    call fastcc void @no_promote_avx2(ptr noalias nocapture nofree noundef nonnull writeonly align 32 dereferenceable(32) [[TMP2]], ptr noalias nocapture nofree noundef nonnull readonly align 32 dereferenceable(32) [[TMP]]) #[[ATTR4:[0-9]+]]
+; CGSCC-NEXT:    call void @llvm.memset.p0.i64(ptr noalias nofree noundef nonnull writeonly align 32 captures(none) dereferenceable(32) [[TMP]], i8 noundef 0, i64 noundef 32, i1 noundef false) #[[ATTR3:[0-9]+]]
+; CGSCC-NEXT:    call fastcc void @no_promote_avx2(ptr noalias nofree noundef nonnull writeonly align 32 captures(none) dereferenceable(32) [[TMP2]], ptr noalias nofree noundef nonnull readonly align 32 captures(none) dereferenceable(32) [[TMP]]) #[[ATTR4:[0-9]+]]
 ; CGSCC-NEXT:    [[TMP4:%.*]] = load <4 x i64>, ptr [[TMP2]], align 32
 ; CGSCC-NEXT:    store <4 x i64> [[TMP4]], ptr [[ARG]], align 2
 ; CGSCC-NEXT:    ret void
@@ -59,7 +59,7 @@ bb:
 define internal fastcc void @promote_avx2(ptr %arg, ptr readonly %arg1) #0 {
 ; CHECK: Function Attrs: inlinehint mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: readwrite) uwtable
 ; CHECK-LABEL: define {{[^@]+}}@promote_avx2
-; CHECK-SAME: (ptr noalias nocapture nofree noundef nonnull writeonly align 32 dereferenceable(32) [[ARG:%.*]], <4 x i64> [[TMP0:%.*]]) #[[ATTR0]] {
+; CHECK-SAME: (ptr noalias nofree noundef nonnull writeonly align 32 captures(none) dereferenceable(32) [[ARG:%.*]], <4 x i64> [[TMP0:%.*]]) #[[ATTR0]] {
 ; CHECK-NEXT:  bb:
 ; CHECK-NEXT:    [[ARG1_PRIV:%.*]] = alloca <4 x i64>, align 32
 ; CHECK-NEXT:    store <4 x i64> [[TMP0]], ptr [[ARG1_PRIV]], align 32
@@ -76,26 +76,26 @@ bb:
 define void @promote(ptr %arg) #0 {
 ; TUNIT: Function Attrs: inlinehint mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: readwrite) uwtable
 ; TUNIT-LABEL: define {{[^@]+}}@promote
-; TUNIT-SAME: (ptr nocapture nofree writeonly [[ARG:%.*]]) #[[ATTR0]] {
+; TUNIT-SAME: (ptr nofree writeonly captures(none) [[ARG:%.*]]) #[[ATTR0]] {
 ; TUNIT-NEXT:  bb:
 ; TUNIT-NEXT:    [[TMP:%.*]] = alloca <4 x i64>, align 32
 ; TUNIT-NEXT:    [[TMP2:%.*]] = alloca <4 x i64>, align 32
-; TUNIT-NEXT:    call void @llvm.memset.p0.i64(ptr noalias nocapture nofree noundef nonnull writeonly align 32 dereferenceable(32) [[TMP]], i8 noundef 0, i64 noundef 32, i1 noundef false) #[[ATTR3]]
+; TUNIT-NEXT:    call void @llvm.memset.p0.i64(ptr noalias nofree noundef nonnull writeonly align 32 captures(none) dereferenceable(32) [[TMP]], i8 noundef 0, i64 noundef 32, i1 noundef false) #[[ATTR3]]
 ; TUNIT-NEXT:    [[TMP0:%.*]] = load <4 x i64>, ptr [[TMP]], align 32
-; TUNIT-NEXT:    call fastcc void @promote_avx2(ptr noalias nocapture nofree noundef nonnull writeonly align 32 dereferenceable(32) [[TMP2]], <4 x i64> [[TMP0]]) #[[ATTR4]]
+; TUNIT-NEXT:    call fastcc void @promote_avx2(ptr noalias nofree noundef nonnull writeonly align 32 captures(none) dereferenceable(32) [[TMP2]], <4 x i64> [[TMP0]]) #[[ATTR4]]
 ; TUNIT-NEXT:    [[TMP4:%.*]] = load <4 x i64>, ptr [[TMP2]], align 32
 ; TUNIT-NEXT:    store <4 x i64> [[TMP4]], ptr [[ARG]], align 2
 ; TUNIT-NEXT:    ret void
 ;
 ; CGSCC: Function Attrs: inlinehint mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: readwrite) uwtable
 ; CGSCC-LABEL: define {{[^@]+}}@promote
-; CGSCC-SAME: (ptr nocapture nofree noundef nonnull writeonly align 2 dereferenceable(32) [[ARG:%.*]]) #[[ATTR0]] {
+; CGSCC-SAME: (ptr nofree noundef nonnull writeonly align 2 captures(none) dereferenceable(32) [[ARG:%.*]]) #[[ATTR0]] {
 ; CGSCC-NEXT:  bb:
 ; CGSCC-NEXT:    [[TMP:%.*]] = alloca <4 x i64>, align 32
 ; CGSCC-NEXT:    [[TMP2:%.*]] = alloca <4 x i64>, align 32
-; CGSCC-NEXT:    call void @llvm.memset.p0.i64(ptr noalias nocapture nofree noundef nonnull writeonly align 32 dereferenceable(32) [[TMP]], i8 noundef 0, i64 noundef 32, i1 noundef false) #[[ATTR3]]
+; CGSCC-NEXT:    call void @llvm.memset.p0.i64(ptr noalias nofree noundef nonnull writeonly align 32 captures(none) dereferenceable(32) [[TMP]], i8 noundef 0, i64 noundef 32, i1 noundef false) #[[ATTR3]]
 ; CGSCC-NEXT:    [[TMP0:%.*]] = load <4 x i64>, ptr [[TMP]], align 32
-; CGSCC-NEXT:    call fastcc void @promote_avx2(ptr noalias nocapture nofree noundef nonnull writeonly align 32 dereferenceable(32) [[TMP2]], <4 x i64> [[TMP0]]) #[[ATTR4]]
+; CGSCC-NEXT:    call fastcc void @promote_avx2(ptr noalias nofree noundef nonnull writeonly align 32 captures(none) dereferenceable(32) [[TMP2]], <4 x i64> [[TMP0]]) #[[ATTR4]]
 ; CGSCC-NEXT:    [[TMP4:%.*]] = load <4 x i64>, ptr [[TMP2]], align 32
 ; CGSCC-NEXT:    store <4 x i64> [[TMP4]], ptr [[ARG]], align 2
 ; CGSCC-NEXT:    ret void
diff --git a/llvm/test/Transforms/Attributor/ArgumentPromotion/X86/min-legal-vector-width.ll b/llvm/test/Transforms/Attributor/ArgumentPromotion/X86/min-legal-vector-width.ll
index f0bcf68b6444b..649e9467c0318 100644
--- a/llvm/test/Transforms/Attributor/ArgumentPromotion/X86/min-legal-vector-width.ll
+++ b/llvm/test/Transforms/Attributor/ArgumentPromotion/X86/min-legal-vector-width.ll
@@ -11,7 +11,7 @@ define internal fastcc void @callee_avx512_legal512_prefer512_call_avx512_legal5
 ;
 ; CHECK: Function Attrs: inlinehint mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: readwrite) uwtable
 ; CHECK-LABEL: define {{[^@]+}}@callee_avx512_legal512_prefer512_call_avx512_legal512_prefer512
-; CHECK-SAME: (ptr noalias nocapture nofree noundef nonnull writeonly align 64 dereferenceable(64) [[ARG:%.*]], <8 x i64> [[TMP0:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-SAME: (ptr noalias nofree noundef nonnull writeonly align 64 captures(none) dereferenceable(64) [[ARG:%.*]], <8 x i64> [[TMP0:%.*]]) #[[ATTR0:[0-9]+]] {
 ; CHECK-NEXT:  bb:
 ; CHECK-NEXT:    [[ARG1_PRIV:%.*]] = alloca <8 x i64>, align 64
 ; CHECK-NEXT:    store <8 x i64> [[TMP0]], ptr [[ARG1_PRIV]], align 64
@@ -29,26 +29,26 @@ define void @avx512_legal512_prefer512_call_avx512_legal512_prefer512(ptr %arg)
 ;
 ; TUNIT: Function Attrs: inlinehint mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: readwrite) uwtable
 ; TUNIT-LABEL: define {{[^@]+}}@avx512_legal512_prefer512_call_avx512_legal512_prefer512
-; TUNIT-SAME: (ptr nocapture nofree writeonly [[ARG:%.*]]) #[[ATTR0]] {
+; TUNIT-SAME: (ptr nofree writeonly captures(none) [[ARG:%.*]]) #[[ATTR0]] {
 ; TUNIT-NEXT:  bb:
 ; TUNIT-NEXT:    [[TMP:%.*]] = alloca <8 x i64>, align 32
 ; TUNIT-NEXT:    [[TMP2:%.*]] = alloca <8 x i64>, align 32
-; TUNIT-NEXT:    call void @llvm.memset.p0.i64(ptr noalias nocapture nofree noundef nonnull writeonly align 32 dereferenceable(64) [[TMP]], i8 noundef 0, i64 noundef 32, i1 noundef false) #[[ATTR5:[0-9]+]]
+; TUNIT-NEXT:    call void @llvm.memset.p0.i64(ptr noalias nofree noundef nonnull writeonly align 32 captures(none) dereferenceable(64) [[TMP]], i8 noundef 0, i64 noundef 32, i1 noundef false) #[[ATTR5:[0-9]+]]
 ; TUNIT-NEXT:    [[TMP0:%.*]] = load <8 x i64>, ptr [[TMP]], align 64
-; TUNIT-NEXT:    call fastcc void @callee_avx512_legal512_prefer512_call_avx512_legal512_prefer512(ptr noalias nocapture nofree noundef nonnull writeonly align 64 dereferenceable(64) [[TMP2]], <8 x i64> [[TMP0]]) #[[ATTR6:[0-9]+]]
+; TUNIT-NEXT:    call fastcc void @callee_avx512_legal512_prefer512_call_avx512_legal512_prefer512(ptr noalias nofree noundef nonnull writeonly align 64 captures(none) dereferenceable(64) [[TMP2]], <8 x i64> [[TMP0]]) #[[ATTR6:[0-9]+]]
 ; TUNIT-NEXT:    [[TMP4:%.*]] = load <8 x i64>, ptr [[TMP2]], align 64
 ; TUNIT-NEXT:    store <8 x i64> [[TMP4]], ptr [[ARG]], align 2
 ; TUNIT-NEXT:    ret void
 ;
 ; CGSCC: Function Attrs: inlinehint mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: readwrite) uwtable
 ; CGSCC-LABEL: define {{[^@]+}}@avx512_legal512_prefer512_call_avx512_legal512_prefer512
-; CGSCC-SAME: (ptr nocapture nofree noundef nonnull writeonly align 2 dereferenceable(64) [[ARG:%.*]]) #[[ATTR0]] {
+; CGSCC-SAME: (ptr nofree noundef nonnull writeonly align 2 captures(none) dereferenceable(64) [[ARG:%.*]]) #[[ATTR0]] {
 ; CGSCC-NEXT:  bb:
 ; CGSCC-NEXT:    [[TMP:%.*]] = alloca <8 x i64>, align 32
 ; CGSCC-NEXT:    [[TMP2:%.*]] = alloca <8 x i64>, align 32
-; CGSCC-NEXT:    call void @llvm.memset.p0.i64(ptr noalias nocapture nofree noundef nonnull writeonly align 64 dereferenceable(64) [[TMP]], i8 noundef 0, i64 noundef 32, i1 noundef false) #[[ATTR5:[0-9]+]]
+; CGSCC-NEXT:    call void @llvm.memset.p0.i64(ptr noalias nofree noundef nonnull writeonly align 64 captures(none) dereferenceable(64) [[TMP]], i8 noundef 0, i64 noundef 32, i1 noundef false) #[[ATTR5:[0-9]+]]
 ; CGSCC-NEXT:    [[TMP0:%.*]] = load <8 x i64>, ptr [[TMP]], align 64
-; CGSCC-NEXT:    call fastcc void @callee_avx512_legal512_prefer512_call_avx512_legal512_prefer512(ptr noalias nocapture nofree noundef nonnull writeonly align 64 dereferenceable(64) [[TMP2]], <8 x i64> [[TMP0]]) #[[ATTR6:[0-9]+]]
+; CGSCC-NEXT:    call fastcc void @callee_avx512_legal512_prefer512_call_avx512_legal512_prefer512(ptr noalias nofree noundef nonnull writeonly align 64 captures(none) dereferenceable(64) [[TMP2]], <8 x i64> [[TMP0]]) #[[ATTR6:[0-9]+]]
 ; CGSCC-NEXT:    [[TMP4:%.*]] = load <8 x i64>, ptr [[TMP2]], align 64
 ; CGSCC-NEXT:    store <8 x i64> [[TMP4]], ptr [[ARG]], align 2
 ; CGSCC-NEXT:    ret void
@@ -68,7 +68,7 @@ define internal fastcc void @callee_avx512_legal512_prefer256_call_avx512_legal5
 ;
 ; CHECK: Function Attrs: inlinehint mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: readwrite) uwtable
 ; CHECK-LABEL: define {{[^@]+}}@callee_avx512_legal512_prefer256_call_avx512_legal512_prefer256
-; CHECK-SAME: (ptr noalias nocapture nofree noundef nonnull writeonly align 64 dereferenceable(64) [[ARG:%.*]], <8 x i64> [[TMP0:%.*]]) #[[ATTR1:[0-9]+]] {
+; CHECK-SAME: (ptr noalias nofree noundef nonnull writeonly align 64 captures(none) dereferenceable(64) [[ARG:%.*]], <8 x i64> [[TMP0:%.*]]) #[[ATTR1:[0-9]+]] {
 ; CHECK-NEXT:  bb:
 ; CHECK-NEXT:    [[ARG1_PRIV:%.*]] = alloca <8 x i64>, align 64
 ; CHECK-NEXT:    store <8 x i64> [[TMP0]], ptr [[ARG1_PRIV]], align 64
@@ -86,26 +86,26 @@ define void @avx512_legal512_prefer256_call_avx512_legal512_prefer256(ptr %arg)
 ;
 ; TUNIT: Function Attrs: inlinehint mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: readwrite) uwtable
 ; TUNIT-LABEL: define {{[^@]+}}@avx512_legal512_prefer256_call_avx512_legal512_prefer256
-; TUNIT-SAME: (ptr nocapture nofree writeonly [[ARG:%.*]]) #[[ATTR1]] {
+; TUNIT-SAME: (ptr nofree writeonly captures(none) [[ARG:%.*]]) #[[ATTR1]] {
 ; TUNIT-NEXT:  bb:
 ; TUNIT-NEXT:    [[TMP:%.*]] = alloca <8 x i64>, align 32
 ; TUNIT-NEXT:    [[TMP2:%.*]] = alloca <8 x i64>, align 32
-; TUNIT-NEXT:    call void @llvm.memset.p0.i64(ptr noalias nocapture nofree noundef nonnull writeonly align 32 dereferenceable(64) [[TMP]], i8 noundef 0, i64 noundef 32, i1 noundef false) #[[ATTR5]]
+; TUNIT-NEXT:    call void @llvm.memset.p0.i64(ptr noalias nofree noundef nonnull writeonly align 32 captures(none) dereferenceable(64) [[TMP]], i8 noundef 0, i64 noundef 32, i1 noundef false) #[[ATTR5]]
 ; TUNIT-NEXT:    [[TMP0:%.*]] = load <8 x i64>, ptr [[TMP]], align 64
-; TUNIT-NEXT:    call fastcc void @callee_avx512_legal512_prefer256_call_avx512_legal512_prefer256(ptr noalias nocapture nofree noundef nonnull writeonly align 64 dereferenceable(64) [[TMP2]], <8 x i64> [[TMP0]]) #[[ATTR6]]
+; TUNIT-NEXT:    call fastcc void @callee_avx512_legal512_prefer256_call_avx512_legal512_prefer256(ptr noalias nofree noundef nonnull writeonly align 64 captures(none) dereferenceable(64) [[TMP2]], <8 x i64> [[TMP0]]) #[[ATTR6]]
 ; TUNIT-NEXT:    [[TMP4:%.*]] = load <8 x i64>, ptr [[TMP2]], align 64
 ; TUNIT-NEXT:    store <8 x i64> [[TMP4]], ptr [[ARG]], align 2
 ; TUNIT-NEXT:    ret void
 ;
 ; CGSCC: Function Attrs: inlinehint mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: readwrite) uwtable
 ; CGSCC-LABEL: define {{[^@]+}}@avx512_legal512_prefer256_call_avx512_legal512_prefer256
-; CGSCC-SAME: (ptr nocapture nofree noundef nonnull writeonly align 2 dereferenceable(64) [[ARG:%.*]]) #[[ATTR1]] {
+; CGSCC-SAME: (ptr nofree noundef nonnull writeonly align 2 captures(none) dereferenceable(64) [[ARG:%.*]]) #[[ATTR1]] {
 ; CGSCC-NEXT:  bb:
 ; CGSCC-NEXT:    [[TMP:%.*]] = alloca <8 x i64>, align 32
 ; CGSCC-NEXT:    [[TMP2:%.*]] = alloca <8 x i64>, align 32
-; CGSCC-NEXT:    call void @llvm.memset.p0.i64(ptr noalias nocapture nofree noundef nonnull writeonly align 64 dereferenceable(64) [[TMP]], i8 noundef 0, i64 noundef 32, i1 noundef false) #[[ATTR5]]
+; CGSCC-NEXT:    call void @llvm.memset.p0.i64(ptr noalias nofree noundef nonnull writeonly align 64 captures(none) dereferenceable(64) [[TMP]], i8 noundef 0, i64 noundef 32, i1 noundef false) #[[ATTR5]]
 ; CGSCC-NEXT:    [[TMP0:%.*]] = load <8 x i64>, ptr [[TMP]], align 64
-; CGSCC-NEXT:    call fastcc void @callee_avx512_legal512_prefer256_call_avx512_legal512_prefer256(ptr noalias nocapture nofree noundef nonnull writeonly align 64 dereferenceable(64) [[TMP2]], <8 x i64> [[TMP0]]) #[[ATTR6]]
+; CGSCC-NEXT:    call fastcc void @callee_avx512_legal512_prefer256_call_avx512_legal512_prefer256(ptr noalias nofree noundef nonnull writeonly align 64 captures(none) dereferenceable(64) [[TMP2]], <8 x i64> [[TMP0]]) #[[ATTR6]]
 ; CGSCC-NEXT:    [[TMP4:%.*]] = load <8 x i64>, ptr [[TMP2]], align 64
 ; CGSCC-NEXT:    store <8 x i64> [[TMP4]], ptr [[ARG]], align 2
 ; CGSCC-NEXT:    ret void
@@ -125,7 +125,7 @@ define internal fastcc void @callee_avx512_legal512_prefer512_call_avx512_legal5
 ;
 ; CHECK: Function Attrs: inlinehint mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: readwrite) uwtable
 ; CHECK-LABEL: define {{[^@]+}}@callee_avx512_legal512_prefer512_call_avx512_legal512_prefer256
-; CHECK-SAME: (ptr noalias nocapture nofree noundef nonnull writeonly align 64 dereferenceable(64) [[ARG:%.*]], <8 x i64> [[TMP0:%.*]]) #[[ATTR1]] {
+; CHECK-SAME: (ptr noalias nofree noundef nonnull writeonly align 64 captures(none) dereferenceable(64) [[ARG:%.*]], <8 x i64> [[TMP0:%.*]]) #[[ATTR1]] {
 ; CHECK-NEXT:  bb:
 ; CHECK-NEXT:    [[ARG1_PRIV:%.*]] = alloca <8 x i64>, align 64
 ; CHECK-NEXT:    store <8 x i64> [[TMP0]], ptr [[ARG1_PRIV]], align 64
@@ -143,26 +143,26 @@ define void @avx512_legal512_prefer512_call_avx512_legal512_prefer256(ptr %arg)
 ;
 ; TUNIT: Function Attrs: inlinehint mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: readwrite) uwtable
 ; TUNIT-LABEL: define {{[^@]+}}@avx512_legal512_prefer512_call_avx512_legal512_prefer256
-; TUNIT-SAME: (ptr nocapture nofree writeonly [[ARG:%.*]]) #[[ATTR0]] {
+; TUNIT-SAME: (ptr nofree writeonly captures(none) [[ARG:%.*]]) #[[ATTR0]] {
 ; TUNIT-NEXT:  bb:
 ; TUNIT-NEXT:    [[TMP:%.*]] = alloca <8 x i64>, align 32
 ; TUNIT-NEXT:    [[TMP2:%.*]] = alloca <8 x i64>, align 32
-; TUNIT-NEXT:    call void @llvm.memset.p0.i64(ptr noalias nocapture nofree noundef nonnull writeonly align 32 dereferenceable(64) [[TMP]], i8 noundef 0, i64 noundef 32, i1 noundef false) #[[ATTR5]]
+; TUNIT-NEXT:    call void @llvm.memset.p0.i64(ptr noalias nofree noundef nonnull writeonly align 32 captures(none) dereferenceable(64) [[TMP]], i8 noundef 0, i64 noundef 32, i1 noundef false) #[[ATTR5]]
 ; TUNIT-NEXT:    [[TMP0:%.*]] = load <8 x i64>, ptr [[TMP]], align 64
-; TUNIT-NEXT:    call fastcc void @callee_avx512_legal512_prefer512_call_avx512_legal512_prefer256(ptr noalias nocapture nofree noundef nonnull writeonly align 64 dereferenceable(64) [[TMP2]], <8 x i64> [[TMP0]]) #[[ATTR6]]
+; TUNIT-NEXT:    call fastcc void @callee_avx512_legal512_prefer512_call_avx512_legal512_prefer256(ptr noalias nofree noundef nonnull writeonly align 64 captures(none) dereferenceable(64) [[TMP2]], <8 x i64> [[TMP0]]) #[[ATTR6]]
 ; TUNIT-NEXT:    [[TMP4:%.*]] = load <8 x i64>, ptr [[TMP2]], align 64
 ; TUNIT-NEXT:    store <8 x i64> [[TMP4]], ptr [[ARG]], align 2
 ; TUNIT-NEXT:    ret void
 ;
 ; CGSCC: Function Attrs: inlinehint mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: readwrite) uwtable
 ; CGSCC-LABEL: define {{[^@]+}}@avx512_legal512_prefer512_call_avx512_legal512_prefer256
-; CGSCC-SAME: (ptr nocapture nofree noundef nonnull writeonly align 2 dereferenceable(64) [[ARG:%.*]]) #[[ATTR0]] {
+; CGSCC-SAME: (ptr nofree noundef nonnull writeonly align 2 captures(none) dereferenceable(64) [[ARG:%.*]]) #[[ATTR0]] {
 ; CGSCC-NEXT:  bb:
 ; CGSCC-NEXT:    [[TMP:%.*]] = alloca <8 x i64>, align 32
 ; CGSCC-NEXT:    [[TMP2:%.*]] = alloca <8 x i64>, align 32
-; CGSCC-NEXT:    call void @llvm.memset.p0.i64(ptr noalias nocapture nofree noundef nonnull writeonly align 64 dereferenceable(64) [[TMP]], i8 noundef 0, i64 noundef 32, i1 noundef false) #[[ATTR5]]
+; CGSCC-NEXT:    call void @llvm.memset.p0.i64(ptr noalias nofree noundef nonnull writeonly align 64 captures(none) dereferenceable(64) [[TMP]], i8 noundef 0, i64 noundef 32, i1 noundef false) #[[ATTR5]]
 ; CGSCC-NEXT:    [[TMP0:%.*]] = load <8 x i64>, ptr [[TMP]], align 64
-; CGSCC-NEXT:    call fastcc void @callee_avx512_legal512_prefer512_call_avx512_legal512_prefer256(ptr noalias nocapture nofree noundef nonnull writeonly align 64 dereferenceable(64) [[TMP2]], <8 x i64> [[TMP0]]) #[[ATTR6]]
+; CGSCC-NEXT:    call fastcc void @callee_avx512_legal512_prefer512_call_avx512_legal512_prefer256(ptr noalias nofree noundef nonnull writeonly align 64 captures(none) dereferenceable(64) [[TMP2]], <8 x i64> [[TMP0]]) #[[ATTR6]]
 ; CGSCC-NEXT:    [[TMP4:%.*]] = load <8 x i64>, ptr [[TMP2]], align 64
 ; CGSCC-NEXT:    store <8 x i64> [[TMP4]], ptr [[ARG]], align 2
 ; CGSCC-NEXT:    ret void
@@ -182,7 +182,7 @@ define internal fastcc void @callee_avx512_legal512_prefer256_call_avx512_legal5
 ;
 ; CHECK: Function Attrs: inlinehint mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: readwrite) uwtable
 ; CHECK-LABEL: define {{[^@]+}}@callee_avx512_legal512_prefer256_call_avx512_legal512_prefer512
-; CHECK-SAME: (ptr noalias nocapture nofree noundef nonnull writeonly align 64 dereferenceable(64) [[ARG:%.*]], <8 x i64> [[TMP0:%.*]]) #[[ATTR0]] {
+; CHECK-SAME: (ptr noalias nofree noundef nonnull writeonly align 64 captures(none) dereferenceable(64) [[ARG:%.*]], <8 x i64> [[TMP0:%.*]]) #[[ATTR0]] {
 ; CHECK-NEXT:  bb:
 ; CHECK-NEXT:    [[ARG1_PRIV:%.*]] = alloca <8 x i64>, align 64
 ; CHECK-NEXT:    store <8 x i64> [[TMP0]], ptr [[ARG1_PRIV]], align 64
@@ -200,26 +200,26 @@ define void @avx512_legal512_prefer256_call_avx512_legal512_prefer512(ptr %arg)
 ;
 ; TUNIT: Function Attrs: inlinehint mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: readwrite) uwtable
 ; TUNIT-LABEL: define {{[^@]+}}@avx512_legal512_prefer256_call_avx512_legal512_prefer512
-; TUNIT-SAME: (ptr nocapture nofree writeonly [[ARG:%.*]]) #[[ATTR1]] {
+; TUNIT-SAME: (ptr nofree writeonly captures(none) [[ARG:%.*]]) #[[ATTR1]] {
 ; TUNIT-NEXT:  bb:
 ; TUNIT-NEXT:    [[TMP:%.*]] = alloca <8 x i64>, align 32
 ; TUNIT-NEXT:    [[TMP2:%.*]] = alloca <8 x i64>, align 32
-; TUNIT-NEXT:    call void @llvm.memset.p0.i64(ptr noalias nocapture nofree noundef nonnull writeonly align 32 dereferenceable(64) [[TMP]], i8 noundef 0, i64 noundef 32, i1 noundef false) #[[ATTR5]]
+; TUNIT-NEXT:    call void @llvm.memset.p0.i64(ptr noalias nofree noundef nonnull writeonly align 32 captures(none) dereferenceable(64) [[TMP]], i8 noundef 0, i64 noundef 32, i1 noundef false) #[[ATTR5]]
 ; TUNIT-NEXT:    [[TMP0:%.*]] = load <8 x i64>, ptr [[TMP]], align 64
-; TUNIT-NEXT:    call fastcc void @callee_avx512_legal512_prefer256_call_avx512_legal512_prefer512(ptr noalias nocapture nofree noundef nonnull writeonly align 64 dereferenceable(64) [[TMP2]], <8 x i64> [[TMP0]]) #[[ATTR6]]
+; TUNIT-NEXT:    call fastcc void @callee_avx512_legal512_prefer256_call_avx512_legal512_prefer512(ptr noalias nofree noundef nonnull writeonly align 64 captures(none) dereferenceable(64) [[TMP2]], <8 x i64> [[TMP0]]) #[[ATTR6]]
 ; TUNIT-NEXT:    [[TMP4:%.*]] = load <8 x i64>, ptr [[TMP2]], align 64
 ; TUNIT-NEXT:    store <8 x i64> [[TMP4]], ptr [[ARG]], align 2
 ; TUNIT-NEXT:    ret void
 ;
 ; CGSCC: Function Attrs: inlinehint mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: readwrite) uwtable
 ; CGSCC-LABEL: define {{[^@]+}}@avx512_legal512_prefer256_call_avx512_legal512_prefer512
-; CGSCC-SAME: (ptr nocapture nofree noundef nonnull writeonly align 2 dereferenceable(64) [[ARG:%.*]]) #[[ATTR1]] {
+; CGSCC-SAME: (ptr nofree noundef nonnull writeonly align 2 captures(none) dereferenceable(64) [[ARG:%.*]]) #[[ATTR1]] {
 ; CGSCC-NEXT:  bb:
 ; CGSCC-NEXT:    [[TMP:%.*]] = alloca <8 x i64>, align 32
 ; CGSCC-NEXT:    [[TMP2:%.*]] = alloca <8 x i64>, align 32
-; CGSCC-NEXT:    call void @llvm.memset.p0.i64(ptr noalias nocapture nofree noundef nonnull writeonly align 64 dereferenceable(64) [[TMP]], i8 noundef 0, i64 noundef 32, i1 noundef false) #[[ATTR5]]
+; CGSCC-NEXT:    call void @llvm.memset.p0.i64(ptr noalias nofree noundef nonnull writeonly align 64 captures(none) dereferenceable(64) [[TMP]], i8 noundef 0, i64 noundef 32, i1 noundef false) #[[ATTR5]]
 ; CGSCC-NEXT:    [[TMP0:%.*]] = load <8 x i64>, ptr [[TMP]], align 64
-; CGSCC-NEXT:    call fastcc void @callee_avx512_legal512_prefer256_call_avx512_legal512_prefer512(ptr noalias nocapture nofree noundef nonnull writeonly align 64 dereferenceable(64) [[TMP2]], <8 x i64> [[TMP0]]) #[[ATTR6]]
+; CGSCC-NEXT:    call fastcc void @callee_avx512_legal512_prefer256_call_avx512_legal512_prefer512(ptr noalias nofree noundef nonnull writeonly align 64 captures(none) dereferenceable(64) [[TMP2]], <8 x i64> [[TMP0]]) #[[ATTR6]]
 ; CGSCC-NEXT:    [[TMP4:%.*]] = load <8 x i64>, ptr [[TMP2]], align 64
 ; CGSCC-NEXT:    store <8 x i64> [[TMP4]], ptr [[ARG]], align 2
 ; CGSCC-NEXT:    ret void
@@ -239,7 +239,7 @@ define internal fastcc void @callee_avx512_legal256_prefer256_call_avx512_legal5
 ;
 ; CHECK: Function Attrs: inlinehint mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: readwrite) uwtable
 ; CHECK-LABEL: define {{[^@]+}}@callee_avx512_legal256_prefer256_call_avx512_legal512_prefer256
-; CHECK-SAME: (ptr noalias nocapture nofree noundef nonnull writeonly align 64 dereferenceable(64) [[ARG:%.*]], ptr noalias nocapture nofree noundef nonnull readonly align 64 dereferenceable(64) [[ARG1:%.*]]) #[[ATTR1]] {
+; CHECK-SAME: (ptr noalias nofree noundef nonnull writeonly align 64 captures(none) dereferenceable(64) [[ARG:%.*]], ptr noalias nofree noundef nonnull readonly align 64 captures(none) dereferenceable(64) [[ARG1:%.*]]) #[[ATTR1]] {
 ; CHECK-NEXT:  bb:
 ; CHECK-NEXT:    [[TMP:%.*]] = load <8 x i64>, ptr [[ARG1]], align 64
 ; CHECK-NEXT:    store <8 x i64> [[TMP]], ptr [[ARG]], align 64
@@ -255,24 +255,24 @@ define void @avx512_legal256_prefer256_call_avx512_legal512_prefer256(ptr %arg)
 ;
 ; TUNIT: Function Attrs: inlinehint mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: readwrite) uwtable
 ; TUNIT-LABEL: define {{[^@]+}}@avx512_legal256_prefer256_call_avx512_legal512_prefer256
-; TUNIT-SAME: (ptr nocapture nofree writeonly [[ARG:%.*]]) #[[ATTR2:[0-9]+]] {
+; TUNIT-SAME: (ptr nofree writeonly captures(none) [[ARG:%.*]]) #[[ATTR2:[0-9]+]] {
 ; TUNIT-NEXT:  bb:
 ; TUNIT-NEXT:    [[TMP:%.*]] = alloca <8 x i64>, align 32
 ; TUNIT-NEXT:    [[TMP2:%.*]] = alloca <8 x i64>, align 32
-; TUNIT-NEXT:    call void @llvm.memset.p0.i64(ptr noalias nocapture nofree noundef nonnull writeonly align 32 dereferenceable(64) [[TMP]], i8 noundef 0, i64 noundef 32, i1 noundef false) #[[ATTR5]]
-; TUNIT-NEXT:    call fastcc void @callee_avx512_legal256_prefer256_call_avx512_legal512_prefer256(ptr noalias nocapture nofree noundef nonnull writeonly align 64 dereferenceable(64) [[TMP2]], ptr noalias nocapture nofree noundef nonnull readonly align 64 dereferenceable(64) [[TMP]]) #[[ATTR6]]
+; TUNIT-NEXT:    call void @llvm.memset.p0.i64(ptr noalias nofree noundef nonnull writeonly align 32 captures(none) dereferenceable(64) [[TMP]], i8 noundef 0, i64 noundef 32, i1 noundef false) #[[ATTR5]]
+; TUNIT-NEXT:    call fastcc void @callee_avx512_legal256_prefer256_call_avx512_legal512_prefer256(ptr noalias nofree noundef nonnull writeonly align 64 captures(none) dereferenceable(64) [[TMP2]], ptr noalias nofree noundef nonnull readonly align 64 captures(none) dereferenceable(64) [[TMP]]) #[[ATTR6]]
 ; TUNIT-NEXT:    [[TMP4:%.*]] = load <8 x i64>, ptr [[TMP2]], align 64
 ; TUNIT-NEXT:    store <8 x i64> [[TMP4]], ptr [[ARG]], align 2
 ; TUNIT-NEXT:    ret void
 ;
 ; CGSCC: Function Attrs: inlinehint mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: readwrite) uwtable
 ; CGSCC-LABEL: define {{[^@]+}}@avx512_legal256_prefer256_call_avx512_legal512_prefer256
-; CGSCC-SAME: (ptr nocapture nofree noundef nonnull writeonly align 2 dereferenceable(64) [[ARG:%.*]]) #[[ATTR2:[0-9]+]] {
+; CGSCC-SAME: (ptr nofree noundef nonnull writeonly align 2 captures(none) dereferenceable(64) [[ARG:%.*]]) #[[ATTR2:[0-9]+]] {
 ; CGSCC-NEXT:  bb:
 ; CGSCC-NEXT:    [[TMP:%.*]] = alloca <8 x i64>, align 32
 ; CGSCC-NEXT:    [[TMP2:%.*]] = alloca <8 x i64>, align 32
-; CGSCC-NEXT:    call void @llvm.memset.p0.i64(ptr noalias nocapture nofree noundef nonnull writeonly align 64 dereferenceable(64) [[TMP]], i8 noundef 0, i64 noundef 32, i1 noundef false) #[[ATTR5]]
-; CGSCC-NEXT:    call fastcc void @callee_avx512_legal256_prefer256_call_avx512_legal512_prefer256(ptr noalias nocapture nofree noundef nonnull writeonly align 64 dereferenceable(64) [[TMP2]], ptr noalias nocapture nofree noundef nonnull readonly align 64 dereferenceable(64) [[TMP]]) #[[ATTR6]]
+; CGSCC-NEXT:    call void @llvm.memset.p0.i64(ptr noalias nofree noundef nonnull writeonly align 64 captures(none) dereferenceable(64) [[TMP]], i8 noundef 0, i64 noundef 32, i1 noundef false) #[[ATTR5]]
+; CGSCC-NEXT:    call fastcc void @callee_avx512_legal256_prefer256_call_avx512_legal512_prefer256(ptr noalias nofree noundef nonnull writeonly align 64 captures(none) dereferenceable(64) [[TMP2]], ptr noalias nofree noundef nonnull readonly align 64 captures(none) dereferenceable(64) [[TMP]]) #[[ATTR6]]
 ; CGSCC-NEXT:    [[TMP4:%.*]] = load <8 x i64>, ptr [[TMP2]], align 64
 ; CGSCC-NEXT:    store <8 x i64> [[TMP4]], ptr [[ARG]], align 2
 ; CGSCC-NEXT:    ret void
@@ -292,7 +292,7 @@ define internal fastcc void @callee_avx512_legal512_prefer256_call_avx512_legal2
 ;
 ; CHECK: Function Attrs: inlinehint mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: readwrite) uwtable
 ; CHECK-LABEL: define {{[^@]+}}@callee_avx512_legal512_prefer256_call_avx512_legal256_prefer256
-; CHECK-SAME: (ptr noalias nocapture nofree noundef nonnull writeonly align 64 dereferenceable(64) [[ARG:%.*]], ptr noalias nocapture nofree noundef nonnull readonly align 64 dereferenceable(64) [[ARG1:%.*]]) #[[ATTR2:[0-9]+]] {
+; CHECK-SAME: (ptr noalias nofree noundef nonnull writeonly align 64 captures(none) dereferenceable(64) [[ARG:%.*]], ptr noalias nofree noundef nonnull readonly align 64 captures(none) dereferenceable(64) [[ARG1:%.*]]) #[[ATTR2:[0-9]+]] {
 ; CHECK-NEXT:  bb:
 ; CHECK-NEXT:    [[TMP:%.*]] = load <8 x i64>, ptr [[ARG1]], align 64
 ; CHECK-NEXT:    store <8 x i64> [[TMP]], ptr [[ARG]], align 64
@@ -308,24 +308,24 @@ define void @avx512_legal512_prefer256_call_avx512_legal256_prefer256(ptr %arg)
 ;
 ; TUNIT: Function Attrs: inlinehint mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: readwrite) uwtable
 ; TUNIT-LABEL: define {{[^@]+}}@avx512_legal512_prefer256_call_avx512_legal256_prefer256
-; TUNIT-SAME: (ptr nocapture nofree writeonly [[ARG:%.*]]) #[[ATTR1]] {
+; TUNIT-SAME: (ptr nofree writeonly captures(none) [[ARG:%.*]]) #[[ATTR1]] {
 ; TUNIT-NEXT:  bb:
 ; TUNIT-NEXT:    [[TMP:%.*]] = alloca <8 x i64>, align 32
 ; TUNIT-NEXT:    [[TMP2:%.*]] = alloca <8 x i64>, align 32
-; TUNIT-NEXT:    call void @llvm.memset.p0.i64(ptr noalias nocapture nofree noundef nonnull writeonly align 32 dereferenceable(64) [[TMP]], i8 noundef 0, i64 noundef 32, i1 noundef false) #[[ATTR5]]
-; TUNIT-NEXT:    call fastcc void @callee_avx512_legal512_prefer256_call_avx512_legal256_prefer256(ptr noalias nocapture nofree noundef nonnull writeonly align 64 dereferenceable(64) [[TMP2]], ptr noalias nocapture nofree noundef nonnull readonly align 64 dereferenceable(64) [[TMP]]) #[[ATTR6]]
+; TUNIT-NEXT:    call void @llvm.memset.p0.i64(ptr noalias nofree noundef nonnull writeonly align 32 captures(none) dereferenceable(64) [[TMP]], i8 noundef 0, i64 noundef 32, i1 noundef false) #[[ATTR5]]
+; TUNIT-NEXT:    call fastcc void @callee_avx512_legal512_prefer256_call_avx512_legal256_prefer256(ptr noalias nofree noundef nonnull writeonly align 64 captures(none) dereferenceable(64) [[TMP2]], ptr noalias nofree noundef nonnull readonly align 64 captures(none) dereferenceable(64) [[TMP]]) #[[ATTR6]]
 ; TUNIT-NEXT:    [[TMP4:%.*]] = load <8 x i64>, ptr [[TMP2]], align 64
 ; TUNIT-NEXT:    store <8 x i64> [[TMP4]], ptr [[ARG]], align 2
 ; TUNIT-NEXT:    ret void
 ;
 ; CGSCC: Function Attrs: inlinehint mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: readwrite) uwtable
 ; CGSCC-LABEL: define {{[^@]+}}@avx512_legal512_prefer256_call_avx512_legal256_prefer256
-; CGSCC-SAME: (ptr nocapture nofree noundef nonnull writeonly align 2 dereferenceable(64) [[ARG:%.*]]) #[[ATTR1]] {
+; CGSCC-SAME: (ptr nofree noundef nonnull writeonly align 2 captures(none) dereferenceable(64) [[ARG:%.*]]) #[[ATTR1]] {
 ; CGSCC-NEXT:  bb:
 ; CGSCC-NEXT:    [[TMP:%.*]] = alloca <8 x i64>, align 32
 ; CGSCC-NEXT:    [[TMP2:%.*]] = alloca <8 x i64>, align 32
-; CGSCC-NEXT:    call void @llvm.memset.p0.i64(ptr noalias nocapture nofree noundef nonnull writeonly align 64 dereferenceable(64) [[TMP]], i8 noundef 0, i64 noundef 32, i1 noundef false) #[[ATTR5]]
-; CGSCC-NEXT:    call fastcc void @callee_avx512_legal512_prefer256_call_avx512_legal256_prefer256(ptr noalias nocapture nofree noundef nonnull writeonly align 64 dereferenceable(64) [[TMP2]], ptr noalias nocapture nofree noundef nonnull readonly align 64 dereferenceable(64) [[TMP]]) #[[ATTR6]]
+; CGSCC-NEXT:    call void @llvm.memset.p0.i64(ptr noalias nofree noundef nonnull writeonly align 64 captures(none) dereferenceable(64) [[TMP]], i8 noundef 0, i64 noundef 32, i1 noundef false) #[[ATTR5]]
+; CGSCC-NEXT:    call fastcc void @callee_avx512_legal512_prefer256_call_avx512_legal256_prefer256(ptr noalias nofree noundef nonnull writeonly align 64 captures(none) dereferenceable(64) [[TMP2]], ptr noalias nofree noundef nonnull readonly align 64 captures(none) dereferenceable(64) [[TMP]]) #[[ATTR6]]
 ; CGSCC-NEXT:    [[TMP4:%.*]] = load <8 x i64>, ptr [[TMP2]], align 64
 ; CGSCC-NEXT:    store <8 x i64> [[TMP4]], ptr [[ARG]], align 2
 ; CGSCC-NEXT:    ret void
@@ -345,7 +345,7 @@ define internal fastcc void @callee_avx2_legal256_prefer256_call_avx2_legal512_p
 ;
 ; CHECK: Function Attrs: inlinehint mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: readwrite) uwtable
 ; CHECK-LABEL: define {{[^@]+}}@callee_avx2_legal256_prefer256_call_avx2_legal512_prefer256
-; CHECK-SAME: (ptr noalias nocapture nofree noundef nonnull writeonly align 64 dereferenceable(64) [[ARG:%.*]], <8 x i64> [[TMP0:%.*]]) #[[ATTR3:[0-9]+]] {
+; CHECK-SAME: (ptr noalias nofree noundef nonnull writeonly align 64 captures(none) dereferenceable(64) [[ARG:%.*]], <8 x i64> [[TMP0:%.*]]) #[[ATTR3:[0-9]+]] {
 ; CHECK-NEXT:  bb:
 ; CHECK-NEXT:    [[ARG1_PRIV:%.*]] = alloca <8 x i64>, align 64
 ; CHECK-NEXT:    store <8 x i64> [[TMP0]], ptr [[ARG1_PRIV]], align 64
@@ -363,26 +363,26 @@ define void @avx2_legal256_prefer256_call_avx2_legal512_prefer256(ptr %arg) #4 {
 ;
 ; TUNIT: Function Attrs: inlinehint mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: readwrite) uwtable
 ; TUNIT-LABEL: define {{[^@]+}}@avx2_legal256_prefer256_call_avx2_legal512_prefer256
-; TUNIT-SAME: (ptr nocapture nofree writeonly [[ARG:%.*]]) #[[ATTR3]] {
+; TUNIT-SAME: (ptr nofree writeonly captures(none) [[ARG:%.*]]) #[[ATTR3]] {
 ; TUNIT-NEXT:  bb:
 ; TUNIT-NEXT:    [[TMP:%.*]] = alloca <8 x i64>, align 32
 ; TUNIT-NEXT:    [[TMP2:%.*]] = alloca <8 x i64>, align 32
-; TUNIT-NEXT:    call void @llvm.memset.p0.i64(ptr noalias nocapture nofree noundef nonnull writeonly align 32 dereferenceable(64) [[TMP]], i8 noundef 0, i64 noundef 32, i1 noundef false) #[[ATTR5]]
+; TUNIT-NEXT:    call void @llvm.memset.p0.i64(ptr noalias nofree noundef nonnull writeonly align 32 captures(none) dereferenceable(64) [[TMP]], i8 noundef 0, i64 noundef 32, i1 noundef false) #[[ATTR5]]
 ; TUNIT-NEXT:    [[TMP0:%.*]] = load <8 x i64>, ptr [[TMP]], align 64
-; TUNIT-NEXT:    call fastcc void @callee_avx2_legal256_prefer256_call_avx2_legal512_prefer256(ptr noalias nocapture nofree noundef nonnull writeonly align 64 dereferenceable(64) [[TMP2]], <8 x i64> [[TMP0]]) #[[ATTR6]]
+; TUNIT-NEXT:    call fastcc void @callee_avx2_legal256_prefer256_call_avx2_legal512_prefer256(ptr noalias nofree noundef nonnull writeonly align 64 captures(none) dereferenceable(64) [[TMP2]], <8 x i64> [[TMP0]]) #[[ATTR6]]
 ; TUNIT-NEXT:    [[TMP4:%.*]] = load <8 x i64>, ptr [[TMP2]], align 64
 ; TUNIT-NEXT:    store <8 x i64> [[TMP4]], ptr [[ARG]], align 2
 ; TUNIT-NEXT:    ret void
 ;
 ; CGSCC: Function Attrs: inlinehint mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: readwrite) uwtable
 ; CGSCC-LABEL: define {{[^@]+}}@avx2_legal256_prefer256_call_avx2_legal512_prefer256
-; CGSCC-SAME: (ptr nocapture nofree noundef nonnull writeonly align 2 dereferenceable(64) [[ARG:%.*]]) #[[ATTR3]] {
+; CGSCC-SAME: (ptr nofree noundef nonnull writeonly align 2 captures(none) dereferenceable(64) [[ARG:%.*]]) #[[ATTR3]] {
 ; CGSCC-NEXT:  bb:
 ; CGSCC-NEXT:    [[TMP:%.*]] = alloca <8 x i64>, align 32
 ; CGSCC-NEXT:    [[TMP2:%.*]] = alloca <8 x i64>, align 32
-; CGSCC-NEXT:    call void @llvm.memset.p0.i64(ptr noalias nocapture nofree noundef nonnull writeonly align 64 dereferenceable(64) [[TMP]], i8 noundef 0, i64 noundef 32, i1 noundef false) #[[ATTR5]]
+; CGSCC-NEXT:    call void @llvm.memset.p0.i64(ptr noalias nofree noundef nonnull writeonly align 64 captures(none) dereferenceable(64) [[TMP]], i8 noundef 0, i64 noundef 32, i1 noundef false) #[[ATTR5]]
 ; CGSCC-NEXT:    [[TMP0:%.*]] = load <8 x i64>, ptr [[TMP]], align 64
-; CGSCC-NEXT:    call fastcc void @callee_avx2_legal256_prefer256_call_avx2_legal512_prefer256(ptr noalias nocapture nofree noundef nonnull writeonly align 64 dereferenceable(64) [[TMP2]], <8 x i64> [[TMP0]]) #[[ATTR6]]
+; CGSCC-NEXT:    call fastcc void @callee_avx2_legal256_prefer256_call_avx2_legal512_prefer256(ptr noalias nofree noundef nonnull writeonly align 64 captures(none) dereferenceable(64) [[TMP2]], <8 x i64> [[TMP0]]) #[[ATTR6]]
 ; CGSCC-NEXT:    [[TMP4:%.*]] = load <8 x i64>, ptr [[TMP2]], align 64
 ; CGSCC-NEXT:    store <8 x i64> [[TMP4]], ptr [[ARG]], align 2
 ; CGSCC-NEXT:    ret void
@@ -402,7 +402,7 @@ define internal fastcc void @callee_avx2_legal512_prefer256_call_avx2_legal256_p
 ;
 ; CHECK: Function Attrs: inlinehint mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: readwrite) uwtable
 ; CHECK-LABEL: define {{[^@]+}}@callee_avx2_legal512_prefer256_call_avx2_legal256_prefer256
-; CHECK-SAME: (ptr noalias nocapture nofree noundef nonnull writeonly align 64 dereferenceable(64) [[ARG:%.*]], <8 x i64> [[TMP0:%.*]]) #[[ATTR3]] {
+; CHECK-SAME: (ptr noalias nofree noundef nonnull writeonly align 64 captures(none) dereferenceable(64) [[ARG:%.*]], <8 x i64> [[TMP0:%.*]]) #[[ATTR3]] {
 ; CHECK-NEXT:  bb:
 ; CHECK-NEXT:    [[ARG1_PRIV:%.*]] = alloca <8 x i64>, align 64
 ; CHECK-NEXT:    store <8 x i64> [[TMP0]], ptr [[ARG1_PRIV]], align 64
@@ -420,26 +420,26 @@ define void @avx2_legal512_prefer256_call_avx2_legal256_prefer256(ptr %arg) #3 {
 ;
 ; TUNIT: Function Attrs: inlinehint mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: readwrite) uwtable
 ; TUNIT-LABEL: define {{[^@]+}}@avx2_legal512_prefer256_call_avx2_legal256_prefer256
-; TUNIT-SAME: (ptr nocapture nofree writeonly [[ARG:%.*]]) #[[ATTR3]] {
+; TUNIT-SAME: (ptr nofree writeonly captures(none) [[ARG:%.*]]) #[[ATTR3]] {
 ; TUNIT-NEXT:  bb:
 ; TUNIT-NEXT:    [[TMP:%.*]] = alloca <8 x i64>, align 32
 ; TUNIT-NEXT:    [[TMP2:%.*]] = alloca <8 x i64>, align 32
-; TUNIT-NEXT:    call void @llvm.memset.p0.i64(ptr noalias nocapture nofree noundef nonnull writeonly align 32 dereferenceable(64) [[TMP]], i8 noundef 0, i64 noundef 32, i1 noundef false) #[[ATTR5]]
+; TUNIT-NEXT:    call void @llvm.memset.p0.i64(ptr noalias nofree noundef nonnull writeonly align 32 captures(none) dereferenceable(64) [[TMP]], i8 noundef 0, i64 noundef 32, i1 noundef false) #[[ATTR5]]
 ; TUNIT-NEXT:    [[TMP0:%.*]] = load <8 x i64>, ptr [[TMP]], align 64
-; TUNIT-NEXT:    call fastcc void @callee_avx2_legal512_prefer256_call_avx2_legal256_prefer256(ptr noalias nocapture nofree noundef nonnull writeonly align 64 dereferenceable(64) [[TMP2]], <8 x i64> [[TMP0]]) #[[ATTR6]]
+; TUNIT-NEXT:    call fastcc void @callee_avx2_legal512_prefer256_call_avx2_legal256_prefer256(ptr noalias nofree noundef nonnull writeonly align 64 captures(none) dereferenceable(64) [[TMP2]], <8 x i64> [[TMP0]]) #[[ATTR6]]
 ; TUNIT-NEXT:    [[TMP4:%.*]] = load <8 x i64>, ptr [[TMP2]], align 64
 ; TUNIT-NEXT:    store <8 x i64> [[TMP4]], ptr [[ARG]], align 2
 ; TUNIT-NEXT:    ret void
 ;
 ; CGSCC: Function Attrs: inlinehint mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: readwrite) uwtable
 ; CGSCC-LABEL: define {{[^@]+}}@avx2_legal512_prefer256_call_avx2_legal256_prefer256
-; CGSCC-SAME: (ptr nocapture nofree noundef nonnull writeonly align 2 dereferenceable(64) [[ARG:%.*]]) #[[ATTR3]] {
+; CGSCC-SAME: (ptr nofree noundef nonnull writeonly align 2 captures(none) dereferenceable(64) [[ARG:%.*]]) #[[ATTR3]] {
 ; CGSCC-NEXT:  bb:
 ; CGSCC-NEXT:    [[TMP:%.*]] = alloca <8 x i64>, align 32
 ; CGSCC-NEXT:    [[TMP2:%.*]] = alloca <8 x i64>, align 32
-; CGSCC-NEXT:    call void @llvm.memset.p0.i64(ptr noalias nocapture nofree noundef nonnull writeonly align 64 dereferenceable(64) [[TMP]], i8 noundef 0, i64 noundef 32, i1 noundef false) #[[ATTR5]]
+; CGSCC-NEXT:    call void @llvm.memset.p0.i64(ptr noalias nofree noundef nonnull writeonly align 64 captures(none) dereferenceable(64) [[TMP]], i8 noundef 0, i64 noundef 32, i1 noundef false) #[[ATTR5]]
 ; CGSCC-NEXT:    [[TMP0:%.*]] = load <8 x i64>, ptr [[TMP]], align 64
-; CGSCC-NEXT:    call fastcc void @callee_avx2_legal512_prefer256_call_avx2_legal256_prefer256(ptr noalias nocapture nofree noundef nonnull writeonly align 64 dereferenceable(64) [[TMP2]], <8 x i64> [[TMP0]]) #[[ATTR6]]
+; CGSCC-NEXT:    call fastcc void @callee_avx2_legal512_prefer256_call_avx2_legal256_prefer256(ptr noalias nofree noundef nonnull writeonly align 64 captures(none) dereferenceable(64) [[TMP2]], <8 x i64> [[TMP0]]) #[[ATTR6]]
 ; CGSCC-NEXT:    [[TMP4:%.*]] = load <8 x i64>, ptr [[TMP2]], align 64
 ; CGSCC-NEXT:    store <8 x i64> [[TMP4]], ptr [[ARG]], align 2
 ; CGSCC-NEXT:    ret void
diff --git a/llvm/test/Transforms/Attributor/ArgumentPromotion/X86/thiscall.ll b/llvm/test/Transforms/Attributor/ArgumentPromotion/X86/thiscall.ll
index 71fd087c8ac15..4795b424b0d2d 100644
--- a/llvm/test/Transforms/Attributor/ArgumentPromotion/X86/thiscall.ll
+++ b/llvm/test/Transforms/Attributor/ArgumentPromotion/X86/thiscall.ll
@@ -14,7 +14,7 @@ target triple = "i386-pc-windows-msvc19.11.0"
 
 define internal x86_thiscallcc void @internalfun(ptr %this, ptr inalloca(<{ %struct.a }>)) {
 ; CHECK-LABEL: define {{[^@]+}}@internalfun
-; CHECK-SAME: (ptr noalias nocapture nofree readnone [[THIS:%.*]], ptr noundef nonnull inalloca(<{ [[STRUCT_A:%.*]] }>) align 4 dereferenceable(1) [[TMP0:%.*]]) {
+; CHECK-SAME: (ptr noalias nofree readnone captures(none) [[THIS:%.*]], ptr noundef nonnull inalloca(<{ [[STRUCT_A:%.*]] }>) align 4 dereferenceable(1) [[TMP0:%.*]]) {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[ARGMEM:%.*]] = alloca inalloca <{ [[STRUCT_A]] }>, align 4
 ; CHECK-NEXT:    [[CALL:%.*]] = call x86_thiscallcc ptr @copy_ctor(ptr noundef nonnull align 4 dereferenceable(1) [[ARGMEM]], ptr noundef nonnull align 4 dereferenceable(1) [[TMP0]])
@@ -31,18 +31,18 @@ entry:
 ; This is here to ensure @internalfun is live.
 define void @exportedfun(ptr %a) {
 ; TUNIT-LABEL: define {{[^@]+}}@exportedfun
-; TUNIT-SAME: (ptr nocapture nofree readnone [[A:%.*]]) {
+; TUNIT-SAME: (ptr nofree readnone captures(none) [[A:%.*]]) {
 ; TUNIT-NEXT:    [[INALLOCA_SAVE:%.*]] = tail call ptr @llvm.stacksave.p0() #[[ATTR1:[0-9]+]]
 ; TUNIT-NEXT:    [[ARGMEM:%.*]] = alloca inalloca <{ [[STRUCT_A:%.*]] }>, align 4
-; TUNIT-NEXT:    call x86_thiscallcc void @internalfun(ptr noalias nocapture nofree readnone undef, ptr noundef nonnull inalloca(<{ [[STRUCT_A]] }>) align 4 dereferenceable(1) [[ARGMEM]])
+; TUNIT-NEXT:    call x86_thiscallcc void @internalfun(ptr noalias nofree readnone captures(none) undef, ptr noundef nonnull inalloca(<{ [[STRUCT_A]] }>) align 4 dereferenceable(1) [[ARGMEM]])
 ; TUNIT-NEXT:    call void @llvm.stackrestore.p0(ptr nofree [[INALLOCA_SAVE]])
 ; TUNIT-NEXT:    ret void
 ;
 ; CGSCC-LABEL: define {{[^@]+}}@exportedfun
-; CGSCC-SAME: (ptr nocapture nofree readnone [[A:%.*]]) {
+; CGSCC-SAME: (ptr nofree readnone captures(none) [[A:%.*]]) {
 ; CGSCC-NEXT:    [[INALLOCA_SAVE:%.*]] = tail call ptr @llvm.stacksave.p0() #[[ATTR1:[0-9]+]]
 ; CGSCC-NEXT:    [[ARGMEM:%.*]] = alloca inalloca <{ [[STRUCT_A:%.*]] }>, align 4
-; CGSCC-NEXT:    call x86_thiscallcc void @internalfun(ptr noalias nocapture nofree readnone [[A]], ptr noundef nonnull inalloca(<{ [[STRUCT_A]] }>) align 4 dereferenceable(1) [[ARGMEM]])
+; CGSCC-NEXT:    call x86_thiscallcc void @internalfun(ptr noalias nofree readnone captures(none) [[A]], ptr noundef nonnull inalloca(<{ [[STRUCT_A]] }>) align 4 dereferenceable(1) [[ARGMEM]])
 ; CGSCC-NEXT:    call void @llvm.stackrestore.p0(ptr nofree [[INALLOCA_SAVE]])
 ; CGSCC-NEXT:    ret void
 ;
diff --git a/llvm/test/Transforms/Attributor/ArgumentPromotion/alignment.ll b/llvm/test/Transforms/Attributor/ArgumentPromotion/alignment.ll
index 54a5b8c564077..590ff6bebf9b0 100644
--- a/llvm/test/Transforms/Attributor/ArgumentPromotion/alignment.ll
+++ b/llvm/test/Transforms/Attributor/ArgumentPromotion/alignment.ll
@@ -47,7 +47,7 @@ declare void @z(i32)
 define internal i32 @test(ptr %X, ptr %Y) {
 ; CGSCC: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: read)
 ; CGSCC-LABEL: define {{[^@]+}}@test
-; CGSCC-SAME: (ptr nocapture nofree noundef nonnull readonly align 4 dereferenceable(4) [[X:%.*]], i64 [[TMP0:%.*]]) #[[ATTR1:[0-9]+]] {
+; CGSCC-SAME: (ptr nofree noundef nonnull readonly align 4 captures(none) dereferenceable(4) [[X:%.*]], i64 [[TMP0:%.*]]) #[[ATTR1:[0-9]+]] {
 ; CGSCC-NEXT:    [[Y_PRIV:%.*]] = alloca i64, align 8
 ; CGSCC-NEXT:    store i64 [[TMP0]], ptr [[Y_PRIV]], align 4
 ; CGSCC-NEXT:    [[A:%.*]] = load i32, ptr [[X]], align 4
@@ -79,7 +79,7 @@ define internal i32 @caller(ptr %A) {
 ; CGSCC-SAME: (i32 [[TMP0:%.*]]) #[[ATTR2:[0-9]+]] {
 ; CGSCC-NEXT:    [[A_PRIV:%.*]] = alloca i32, align 4
 ; CGSCC-NEXT:    store i32 [[TMP0]], ptr [[A_PRIV]], align 4
-; CGSCC-NEXT:    [[C:%.*]] = call i32 @test(ptr noalias nocapture nofree noundef nonnull readonly align 4 dereferenceable(4) [[A_PRIV]], i64 noundef 1) #[[ATTR3:[0-9]+]]
+; CGSCC-NEXT:    [[C:%.*]] = call i32 @test(ptr noalias nofree noundef nonnull readonly align 4 captures(none) dereferenceable(4) [[A_PRIV]], i64 noundef 1) #[[ATTR3:[0-9]+]]
 ; CGSCC-NEXT:    ret i32 [[C]]
 ;
   %B = alloca i64
diff --git a/llvm/test/Transforms/Attributor/ArgumentPromotion/array.ll b/llvm/test/Transforms/Attributor/ArgumentPromotion/array.ll
index a52bbfbe1a346..81497d8d2bb8e 100644
--- a/llvm/test/Transforms/Attributor/ArgumentPromotion/array.ll
+++ b/llvm/test/Transforms/Attributor/ArgumentPromotion/array.ll
@@ -40,7 +40,7 @@ define internal void @callee(ptr noalias %arg) {
 ; CHECK-NEXT:    store i32 [[TMP1]], ptr [[ARG_PRIV_B4]], align 4
 ; CHECK-NEXT:    [[ARG_PRIV_B8:%.*]] = getelementptr i8, ptr [[ARG_PRIV]], i64 8
 ; CHECK-NEXT:    store i32 [[TMP2]], ptr [[ARG_PRIV_B8]], align 4
-; CHECK-NEXT:    call void @use(ptr noalias nocapture nofree noundef nonnull readonly align 4 dereferenceable(12) [[ARG_PRIV]])
+; CHECK-NEXT:    call void @use(ptr noalias nofree noundef nonnull readonly align 4 captures(none) dereferenceable(12) [[ARG_PRIV]])
 ; CHECK-NEXT:    ret void
 ;
 entry:
diff --git a/llvm/test/Transforms/Attributor/ArgumentPromotion/attrs.ll b/llvm/test/Transforms/Attributor/ArgumentPromotion/attrs.ll
index 9ce752aa95ee8..f8e0543ce2596 100644
--- a/llvm/test/Transforms/Attributor/ArgumentPromotion/attrs.ll
+++ b/llvm/test/Transforms/Attributor/ArgumentPromotion/attrs.ll
@@ -42,7 +42,7 @@ define i32 @test(ptr %X) {
 ;
 ; TUNIT: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: readwrite)
 ; TUNIT-LABEL: define {{[^@]+}}@test
-; TUNIT-SAME: (ptr nocapture nofree nonnull readonly [[X:%.*]]) #[[ATTR1:[0-9]+]] {
+; TUNIT-SAME: (ptr nofree nonnull readonly captures(none) [[X:%.*]]) #[[ATTR1:[0-9]+]] {
 ; TUNIT-NEXT:  entry:
 ; TUNIT-NEXT:    [[S:%.*]] = alloca [[STRUCT_SS:%.*]], align 8
 ; TUNIT-NEXT:    store i32 1, ptr [[S]], align 8
@@ -56,7 +56,7 @@ define i32 @test(ptr %X) {
 ;
 ; CGSCC: Function Attrs: mustprogress nofree nosync nounwind willreturn memory(argmem: readwrite)
 ; CGSCC-LABEL: define {{[^@]+}}@test
-; CGSCC-SAME: (ptr nocapture nofree noundef nonnull readonly align 4 dereferenceable(4) [[X:%.*]]) #[[ATTR1:[0-9]+]] {
+; CGSCC-SAME: (ptr nofree noundef nonnull readonly align 4 captures(none) dereferenceable(4) [[X:%.*]]) #[[ATTR1:[0-9]+]] {
 ; CGSCC-NEXT:  entry:
 ; CGSCC-NEXT:    [[S:%.*]] = alloca [[STRUCT_SS:%.*]], align 8
 ; CGSCC-NEXT:    [[VAL4:%.*]] = getelementptr [[STRUCT_SS]], ptr [[S]], i32 0, i32 1
diff --git a/llvm/test/Transforms/Attributor/ArgumentPromotion/basictest.ll b/llvm/test/Transforms/Attributor/ArgumentPromotion/basictest.ll
index 7f8aeb59fd151..21dedc73952df 100644
--- a/llvm/test/Transforms/Attributor/ArgumentPromotion/basictest.ll
+++ b/llvm/test/Transforms/Attributor/ArgumentPromotion/basictest.ll
@@ -6,7 +6,7 @@ target datalayout = "E-p:64:64:64-a0:0:8-f32:32:32-f64:64:64-i1:8:8-i8:8:8-i16:1
 define internal i32 @test(ptr %X, ptr %Y) {
 ; CGSCC: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: read)
 ; CGSCC-LABEL: define {{[^@]+}}@test
-; CGSCC-SAME: (i32 [[TMP0:%.*]], ptr nocapture nofree noundef nonnull readonly align 4 dereferenceable(4) [[Y:%.*]]) #[[ATTR0:[0-9]+]] {
+; CGSCC-SAME: (i32 [[TMP0:%.*]], ptr nofree noundef nonnull readonly align 4 captures(none) dereferenceable(4) [[Y:%.*]]) #[[ATTR0:[0-9]+]] {
 ; CGSCC-NEXT:    [[X_PRIV:%.*]] = alloca i32, align 4
 ; CGSCC-NEXT:    store i32 [[TMP0]], ptr [[X_PRIV]], align 4
 ; CGSCC-NEXT:    [[A:%.*]] = load i32, ptr [[X_PRIV]], align 4
@@ -26,7 +26,7 @@ define internal i32 @caller(ptr %B) {
 ; CGSCC-SAME: (i32 [[TMP0:%.*]]) #[[ATTR1:[0-9]+]] {
 ; CGSCC-NEXT:    [[B_PRIV:%.*]] = alloca i32, align 4
 ; CGSCC-NEXT:    store i32 [[TMP0]], ptr [[B_PRIV]], align 4
-; CGSCC-NEXT:    [[C:%.*]] = call i32 @test(i32 noundef 1, ptr noalias nocapture nofree noundef nonnull readonly align 4 dereferenceable(4) [[B_PRIV]]) #[[ATTR2:[0-9]+]]
+; CGSCC-NEXT:    [[C:%.*]] = call i32 @test(i32 noundef 1, ptr noalias nofree noundef nonnull readonly align 4 captures(none) dereferenceable(4) [[B_PRIV]]) #[[ATTR2:[0-9]+]]
 ; CGSCC-NEXT:    ret i32 [[C]]
 ;
   %A = alloca i32
diff --git a/llvm/test/Transforms/Attributor/ArgumentPromotion/byval-2.ll b/llvm/test/Transforms/Attributor/ArgumentPromotion/byval-2.ll
index 9f7acd579b274..82626e58d26dd 100644
--- a/llvm/test/Transforms/Attributor/ArgumentPromotion/byval-2.ll
+++ b/llvm/test/Transforms/Attributor/ArgumentPromotion/byval-2.ll
@@ -34,7 +34,7 @@ define i32 @test(ptr %X) {
 ;
 ; TUNIT: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: readwrite)
 ; TUNIT-LABEL: define {{[^@]+}}@test
-; TUNIT-SAME: (ptr nocapture nofree nonnull readonly [[X:%.*]]) #[[ATTR1:[0-9]+]] {
+; TUNIT-SAME: (ptr nofree nonnull readonly captures(none) [[X:%.*]]) #[[ATTR1:[0-9]+]] {
 ; TUNIT-NEXT:  entry:
 ; TUNIT-NEXT:    [[S:%.*]] = alloca [[STRUCT_SS:%.*]], align 8
 ; TUNIT-NEXT:    store i32 1, ptr [[S]], align 8
@@ -48,7 +48,7 @@ define i32 @test(ptr %X) {
 ;
 ; CGSCC: Function Attrs: mustprogress nofree nosync nounwind willreturn memory(argmem: readwrite)
 ; CGSCC-LABEL: define {{[^@]+}}@test
-; CGSCC-SAME: (ptr nocapture nofree noundef nonnull readonly align 4 dereferenceable(4) [[X:%.*]]) #[[ATTR1:[0-9]+]] {
+; CGSCC-SAME: (ptr nofree noundef nonnull readonly align 4 captures(none) dereferenceable(4) [[X:%.*]]) #[[ATTR1:[0-9]+]] {
 ; CGSCC-NEXT:  entry:
 ; CGSCC-NEXT:    [[S:%.*]] = alloca [[STRUCT_SS:%.*]], align 8
 ; CGSCC-NEXT:    [[VAL4:%.*]] = getelementptr [[STRUCT_SS]], ptr [[S]], i32 0, i32 1
diff --git a/llvm/test/Transforms/Attributor/ArgumentPromotion/control-flow.ll b/llvm/test/Transforms/Attributor/ArgumentPromotion/control-flow.ll
index d414926e95beb..2a1d0de62784b 100644
--- a/llvm/test/Transforms/Attributor/ArgumentPromotion/control-flow.ll
+++ b/llvm/test/Transforms/Attributor/ArgumentPromotion/control-flow.ll
@@ -6,7 +6,7 @@
 define internal i32 @callee(i1 %C, ptr %P) {
 ; CHECK: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: read)
 ; CHECK-LABEL: define {{[^@]+}}@callee
-; CHECK-SAME: (i1 noundef [[C:%.*]], ptr nocapture nofree readonly [[P:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-SAME: (i1 noundef [[C:%.*]], ptr nofree readonly captures(none) [[P:%.*]]) #[[ATTR0:[0-9]+]] {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    br i1 [[C]], label [[T:%.*]], label [[F:%.*]]
 ; CHECK:       T:
@@ -29,16 +29,16 @@ F:
 define i32 @foo(i1 %C, ptr %P) {
 ; TUNIT: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: read)
 ; TUNIT-LABEL: define {{[^@]+}}@foo
-; TUNIT-SAME: (i1 [[C:%.*]], ptr nocapture nofree readonly [[P:%.*]]) #[[ATTR0]] {
+; TUNIT-SAME: (i1 [[C:%.*]], ptr nofree readonly captures(none) [[P:%.*]]) #[[ATTR0]] {
 ; TUNIT-NEXT:  entry:
-; TUNIT-NEXT:    [[X:%.*]] = call i32 @callee(i1 noundef [[C]], ptr nocapture nofree readonly [[P]]) #[[ATTR1:[0-9]+]]
+; TUNIT-NEXT:    [[X:%.*]] = call i32 @callee(i1 noundef [[C]], ptr nofree readonly captures(none) [[P]]) #[[ATTR1:[0-9]+]]
 ; TUNIT-NEXT:    ret i32 [[X]]
 ;
 ; CGSCC: Function Attrs: mustprogress nofree nosync nounwind willreturn memory(argmem: read)
 ; CGSCC-LABEL: define {{[^@]+}}@foo
-; CGSCC-SAME: (i1 noundef [[C:%.*]], ptr nocapture nofree readonly [[P:%.*]]) #[[ATTR1:[0-9]+]] {
+; CGSCC-SAME: (i1 noundef [[C:%.*]], ptr nofree readonly captures(none) [[P:%.*]]) #[[ATTR1:[0-9]+]] {
 ; CGSCC-NEXT:  entry:
-; CGSCC-NEXT:    [[X:%.*]] = call i32 @callee(i1 noundef [[C]], ptr nocapture nofree readonly [[P]]) #[[ATTR2:[0-9]+]]
+; CGSCC-NEXT:    [[X:%.*]] = call i32 @callee(i1 noundef [[C]], ptr nofree readonly captures(none) [[P]]) #[[ATTR2:[0-9]+]]
 ; CGSCC-NEXT:    ret i32 [[X]]
 ;
 entry:
diff --git a/llvm/test/Transforms/Attributor/ArgumentPromotion/dbg.ll b/llvm/test/Transforms/Attributor/ArgumentPromotion/dbg.ll
index 67d783351aa09..c6f377c87f881 100644
--- a/llvm/test/Transforms/Attributor/ArgumentPromotion/dbg.ll
+++ b/llvm/test/Transforms/Attributor/ArgumentPromotion/dbg.ll
@@ -6,7 +6,7 @@ declare void @sink(i32)
 
 define internal void @test(ptr %X) !dbg !2 {
 ; CHECK-LABEL: define {{[^@]+}}@test
-; CHECK-SAME: (ptr nocapture nofree noundef nonnull readonly align 8 dereferenceable(8) [[X:%.*]]) !dbg [[DBG3:![0-9]+]] {
+; CHECK-SAME: (ptr nofree noundef nonnull readonly align 8 captures(none) dereferenceable(8) [[X:%.*]]) !dbg [[DBG3:![0-9]+]] {
 ; CHECK-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[X]], align 8
 ; CHECK-NEXT:    [[TMP2:%.*]] = load i32, ptr [[TMP1]], align 8
 ; CHECK-NEXT:    call void @sink(i32 [[TMP2]])
@@ -33,14 +33,14 @@ define internal void @test_byval(ptr byval(%struct.pair) %P) {
 
 define void @caller(ptr %Y, ptr %P) {
 ; TUNIT-LABEL: define {{[^@]+}}@caller
-; TUNIT-SAME: (ptr nocapture nofree readonly [[Y:%.*]], ptr nocapture nofree readnone [[P:%.*]]) {
-; TUNIT-NEXT:    call void @test(ptr nocapture nofree noundef readonly align 8 [[Y]]), !dbg [[DBG4:![0-9]+]]
+; TUNIT-SAME: (ptr nofree readonly captures(none) [[Y:%.*]], ptr nofree readnone captures(none) [[P:%.*]]) {
+; TUNIT-NEXT:    call void @test(ptr nofree noundef readonly align 8 captures(none) [[Y]]), !dbg [[DBG4:![0-9]+]]
 ; TUNIT-NEXT:    call void @test_byval(), !dbg [[DBG5:![0-9]+]]
 ; TUNIT-NEXT:    ret void
 ;
 ; CGSCC-LABEL: define {{[^@]+}}@caller
-; CGSCC-SAME: (ptr nocapture nofree noundef nonnull readonly align 8 dereferenceable(8) [[Y:%.*]], ptr nocapture nofree readnone [[P:%.*]]) {
-; CGSCC-NEXT:    call void @test(ptr nocapture nofree noundef nonnull readonly align 8 dereferenceable(8) [[Y]]), !dbg [[DBG4:![0-9]+]]
+; CGSCC-SAME: (ptr nofree noundef nonnull readonly align 8 captures(none) dereferenceable(8) [[Y:%.*]], ptr nofree readnone captures(none) [[P:%.*]]) {
+; CGSCC-NEXT:    call void @test(ptr nofree noundef nonnull readonly align 8 captures(none) dereferenceable(8) [[Y]]), !dbg [[DBG4:![0-9]+]]
 ; CGSCC-NEXT:    call void @test_byval(), !dbg [[DBG5:![0-9]+]]
 ; CGSCC-NEXT:    ret void
 ;
diff --git a/llvm/test/Transforms/Attributor/ArgumentPromotion/inalloca.ll b/llvm/test/Transforms/Attributor/ArgumentPromotion/inalloca.ll
index 957a26ff7176b..46a0819baeabf 100644
--- a/llvm/test/Transforms/Attributor/ArgumentPromotion/inalloca.ll
+++ b/llvm/test/Transforms/Attributor/ArgumentPromotion/inalloca.ll
@@ -10,7 +10,7 @@ target datalayout = "E-p:64:64:64-a0:0:8-f32:32:32-f64:64:64-i1:8:8-i8:8:8-i16:1
 define internal i32 @f(ptr inalloca(%struct.ss) %s) {
 ; CHECK: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: read)
 ; CHECK-LABEL: define {{[^@]+}}@f
-; CHECK-SAME: (ptr noalias nocapture nofree noundef nonnull inalloca([[STRUCT_SS:%.*]]) align 4 dereferenceable(8) [[S:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-SAME: (ptr noalias nofree noundef nonnull inalloca([[STRUCT_SS:%.*]]) align 4 captures(none) dereferenceable(8) [[S:%.*]]) #[[ATTR0:[0-9]+]] {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[F1:%.*]] = getelementptr [[STRUCT_SS]], ptr [[S]], i32 0, i32 1
 ; CHECK-NEXT:    [[A:%.*]] = load i32, ptr [[S]], align 4
@@ -35,7 +35,7 @@ define i32 @main() {
 ; TUNIT-NEXT:    [[F1:%.*]] = getelementptr [[STRUCT_SS]], ptr [[S]], i32 0, i32 1
 ; TUNIT-NEXT:    store i32 1, ptr [[S]], align 4
 ; TUNIT-NEXT:    store i32 2, ptr [[F1]], align 4
-; TUNIT-NEXT:    [[R:%.*]] = call i32 @f(ptr noalias nocapture nofree noundef nonnull inalloca([[STRUCT_SS]]) align 4 dereferenceable(8) [[S]]) #[[ATTR2:[0-9]+]]
+; TUNIT-NEXT:    [[R:%.*]] = call i32 @f(ptr noalias nofree noundef nonnull inalloca([[STRUCT_SS]]) align 4 captures(none) dereferenceable(8) [[S]]) #[[ATTR2:[0-9]+]]
 ; TUNIT-NEXT:    ret i32 [[R]]
 ;
 ; CGSCC: Function Attrs: mustprogress nofree nosync nounwind willreturn memory(none)
@@ -46,7 +46,7 @@ define i32 @main() {
 ; CGSCC-NEXT:    [[F1:%.*]] = getelementptr [[STRUCT_SS]], ptr [[S]], i32 0, i32 1
 ; CGSCC-NEXT:    store i32 1, ptr [[S]], align 4
 ; CGSCC-NEXT:    store i32 2, ptr [[F1]], align 4
-; CGSCC-NEXT:    [[R:%.*]] = call i32 @f(ptr noalias nocapture nofree noundef nonnull inalloca([[STRUCT_SS]]) align 4 dereferenceable(8) [[S]]) #[[ATTR3:[0-9]+]]
+; CGSCC-NEXT:    [[R:%.*]] = call i32 @f(ptr noalias nofree noundef nonnull inalloca([[STRUCT_SS]]) align 4 captures(none) dereferenceable(8) [[S]]) #[[ATTR3:[0-9]+]]
 ; CGSCC-NEXT:    ret i32 [[R]]
 ;
 entry:
@@ -62,7 +62,7 @@ entry:
 define internal i1 @g(ptr %a, ptr inalloca(%struct.ss) %b) nounwind  {
 ; CGSCC: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(none)
 ; CGSCC-LABEL: define {{[^@]+}}@g
-; CGSCC-SAME: (ptr noalias nocapture nofree noundef nonnull readnone align 4 dereferenceable(8) [[A:%.*]], ptr noalias nocapture nofree noundef nonnull writeonly inalloca([[STRUCT_SS:%.*]]) align 4 dereferenceable(8) [[B:%.*]]) #[[ATTR2:[0-9]+]] {
+; CGSCC-SAME: (ptr noalias nofree noundef nonnull readnone align 4 captures(none) dereferenceable(8) [[A:%.*]], ptr noalias nofree noundef nonnull writeonly inalloca([[STRUCT_SS:%.*]]) align 4 captures(none) dereferenceable(8) [[B:%.*]]) #[[ATTR2:[0-9]+]] {
 ; CGSCC-NEXT:  entry:
 ; CGSCC-NEXT:    ret i1 undef
 ;
diff --git a/llvm/test/Transforms/Attributor/ArgumentPromotion/live_called_from_dead_2.ll b/llvm/test/Transforms/Attributor/ArgumentPromotion/live_called_from_dead_2.ll
index b42647840f7cf..6a8605ed19546 100644
--- a/llvm/test/Transforms/Attributor/ArgumentPromotion/live_called_from_dead_2.ll
+++ b/llvm/test/Transforms/Attributor/ArgumentPromotion/live_called_from_dead_2.ll
@@ -15,7 +15,7 @@ define internal void @dead() {
 define internal i32 @test(ptr %X, ptr %Y) {
 ; TUNIT: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: write)
 ; TUNIT-LABEL: define {{[^@]+}}@test
-; TUNIT-SAME: (ptr noalias nocapture nofree noundef nonnull writeonly align 4 dereferenceable(4) [[X:%.*]]) #[[ATTR0:[0-9]+]] {
+; TUNIT-SAME: (ptr noalias nofree noundef nonnull writeonly align 4 captures(none) dereferenceable(4) [[X:%.*]]) #[[ATTR0:[0-9]+]] {
 ; TUNIT-NEXT:    br i1 true, label [[LIVE:%.*]], label [[DEAD:%.*]]
 ; TUNIT:       live:
 ; TUNIT-NEXT:    ret i32 undef
@@ -24,7 +24,7 @@ define internal i32 @test(ptr %X, ptr %Y) {
 ;
 ; CGSCC: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: write)
 ; CGSCC-LABEL: define {{[^@]+}}@test
-; CGSCC-SAME: (ptr noalias nocapture nofree noundef nonnull writeonly align 4 dereferenceable(4) [[X:%.*]]) #[[ATTR0:[0-9]+]] {
+; CGSCC-SAME: (ptr noalias nofree noundef nonnull writeonly align 4 captures(none) dereferenceable(4) [[X:%.*]]) #[[ATTR0:[0-9]+]] {
 ; CGSCC-NEXT:    br i1 true, label [[LIVE:%.*]], label [[DEAD:%.*]]
 ; CGSCC:       live:
 ; CGSCC-NEXT:    store i32 0, ptr [[X]], align 4
@@ -45,17 +45,17 @@ dead:
 define internal i32 @caller(ptr %B) {
 ; TUNIT: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: write)
 ; TUNIT-LABEL: define {{[^@]+}}@caller
-; TUNIT-SAME: (ptr noalias nocapture nofree noundef nonnull writeonly align 4 dereferenceable(4) [[B:%.*]]) #[[ATTR0]] {
+; TUNIT-SAME: (ptr noalias nofree noundef nonnull writeonly align 4 captures(none) dereferenceable(4) [[B:%.*]]) #[[ATTR0]] {
 ; TUNIT-NEXT:    [[A:%.*]] = alloca i32, align 4
-; TUNIT-NEXT:    [[C:%.*]] = call i32 @test(ptr noalias nocapture nofree noundef nonnull writeonly align 4 dereferenceable(4) [[B]]) #[[ATTR2:[0-9]+]]
+; TUNIT-NEXT:    [[C:%.*]] = call i32 @test(ptr noalias nofree noundef nonnull writeonly align 4 captures(none) dereferenceable(4) [[B]]) #[[ATTR2:[0-9]+]]
 ; TUNIT-NEXT:    ret i32 undef
 ;
 ; CGSCC: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: write)
 ; CGSCC-LABEL: define {{[^@]+}}@caller
-; CGSCC-SAME: (ptr noalias nocapture nofree noundef nonnull writeonly align 4 dereferenceable(4) [[B:%.*]]) #[[ATTR0]] {
+; CGSCC-SAME: (ptr noalias nofree noundef nonnull writeonly align 4 captures(none) dereferenceable(4) [[B:%.*]]) #[[ATTR0]] {
 ; CGSCC-NEXT:    [[A:%.*]] = alloca i32, align 4
 ; CGSCC-NEXT:    [[A1:%.*]] = alloca i8, i32 0, align 4
-; CGSCC-NEXT:    [[C:%.*]] = call i32 @test(ptr noalias nocapture nofree noundef nonnull writeonly align 4 dereferenceable(4) [[B]]) #[[ATTR2:[0-9]+]]
+; CGSCC-NEXT:    [[C:%.*]] = call i32 @test(ptr noalias nofree noundef nonnull writeonly align 4 captures(none) dereferenceable(4) [[B]]) #[[ATTR2:[0-9]+]]
 ; CGSCC-NEXT:    ret i32 0
 ;
   %A = alloca i32
@@ -69,7 +69,7 @@ define i32 @callercaller() {
 ; TUNIT-LABEL: define {{[^@]+}}@callercaller
 ; TUNIT-SAME: () #[[ATTR1:[0-9]+]] {
 ; TUNIT-NEXT:    [[B:%.*]] = alloca i32, align 4
-; TUNIT-NEXT:    [[X:%.*]] = call i32 @caller(ptr noalias nocapture nofree noundef nonnull writeonly align 4 dereferenceable(4) [[B]]) #[[ATTR2]]
+; TUNIT-NEXT:    [[X:%.*]] = call i32 @caller(ptr noalias nofree noundef nonnull writeonly align 4 captures(none) dereferenceable(4) [[B]]) #[[ATTR2]]
 ; TUNIT-NEXT:    ret i32 0
 ;
 ; CGSCC: Function Attrs: mustprogress nofree nosync nounwind willreturn memory(none)
@@ -77,7 +77,7 @@ define i32 @callercaller() {
 ; CGSCC-SAME: () #[[ATTR1:[0-9]+]] {
 ; CGSCC-NEXT:    [[B:%.*]] = alloca i32, align 4
 ; CGSCC-NEXT:    store i32 2, ptr [[B]], align 4
-; CGSCC-NEXT:    [[X:%.*]] = call noundef i32 @caller(ptr noalias nocapture nofree noundef nonnull writeonly align 4 dereferenceable(4) [[B]]) #[[ATTR3:[0-9]+]]
+; CGSCC-NEXT:    [[X:%.*]] = call noundef i32 @caller(ptr noalias nofree noundef nonnull writeonly align 4 captures(none) dereferenceable(4) [[B]]) #[[ATTR3:[0-9]+]]
 ; CGSCC-NEXT:    ret i32 [[X]]
 ;
   %B = alloca i32
diff --git a/llvm/test/Transforms/Attributor/ArgumentPromotion/musttail.ll b/llvm/test/Transforms/Attributor/ArgumentPromotion/musttail.ll
index ca931de587b39..bb9200c24e6c2 100644
--- a/llvm/test/Transforms/Attributor/ArgumentPromotion/musttail.ll
+++ b/llvm/test/Transforms/Attributor/ArgumentPromotion/musttail.ll
@@ -10,7 +10,7 @@
 define internal i32 @test(ptr %p) {
 ; CHECK: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: read)
 ; CHECK-LABEL: define {{[^@]+}}@test
-; CHECK-SAME: (ptr nocapture nofree readonly [[P:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-SAME: (ptr nofree readonly captures(none) [[P:%.*]]) #[[ATTR0:[0-9]+]] {
 ; CHECK-NEXT:    [[A_GEP:%.*]] = getelementptr [[T:%.*]], ptr [[P]], i64 0, i32 3
 ; CHECK-NEXT:    [[B_GEP:%.*]] = getelementptr [[T]], ptr [[P]], i64 0, i32 2
 ; CHECK-NEXT:    [[A:%.*]] = load i32, ptr [[A_GEP]], align 4
@@ -29,14 +29,14 @@ define internal i32 @test(ptr %p) {
 define i32 @caller(ptr %p) {
 ; TUNIT: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: read)
 ; TUNIT-LABEL: define {{[^@]+}}@caller
-; TUNIT-SAME: (ptr nocapture nofree readonly [[P:%.*]]) #[[ATTR0]] {
-; TUNIT-NEXT:    [[V:%.*]] = musttail call i32 @test(ptr nocapture nofree readonly [[P]]) #[[ATTR4:[0-9]+]]
+; TUNIT-SAME: (ptr nofree readonly captures(none) [[P:%.*]]) #[[ATTR0]] {
+; TUNIT-NEXT:    [[V:%.*]] = musttail call i32 @test(ptr nofree readonly captures(none) [[P]]) #[[ATTR4:[0-9]+]]
 ; TUNIT-NEXT:    ret i32 [[V]]
 ;
 ; CGSCC: Function Attrs: mustprogress nofree nosync nounwind willreturn memory(argmem: read)
 ; CGSCC-LABEL: define {{[^@]+}}@caller
-; CGSCC-SAME: (ptr nocapture nofree readonly [[P:%.*]]) #[[ATTR1:[0-9]+]] {
-; CGSCC-NEXT:    [[V:%.*]] = musttail call i32 @test(ptr nocapture nofree readonly [[P]]) #[[ATTR5:[0-9]+]]
+; CGSCC-SAME: (ptr nofree readonly captures(none) [[P:%.*]]) #[[ATTR1:[0-9]+]] {
+; CGSCC-NEXT:    [[V:%.*]] = musttail call i32 @test(ptr nofree readonly captures(none) [[P]]) #[[ATTR5:[0-9]+]]
 ; CGSCC-NEXT:    ret i32 [[V]]
 ;
   %v = musttail call i32 @test(ptr %p)
@@ -48,12 +48,12 @@ define i32 @caller(ptr %p) {
 define i32 @foo(ptr %p, i32 %v) {
 ; TUNIT: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(none)
 ; TUNIT-LABEL: define {{[^@]+}}@foo
-; TUNIT-SAME: (ptr nocapture nofree readnone [[P:%.*]], i32 [[V:%.*]]) #[[ATTR1:[0-9]+]] {
+; TUNIT-SAME: (ptr nofree readnone captures(none) [[P:%.*]], i32 [[V:%.*]]) #[[ATTR1:[0-9]+]] {
 ; TUNIT-NEXT:    ret i32 0
 ;
 ; CGSCC: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(none)
 ; CGSCC-LABEL: define {{[^@]+}}@foo
-; CGSCC-SAME: (ptr nocapture nofree readnone [[P:%.*]], i32 [[V:%.*]]) #[[ATTR2:[0-9]+]] {
+; CGSCC-SAME: (ptr nofree readnone captures(none) [[P:%.*]], i32 [[V:%.*]]) #[[ATTR2:[0-9]+]] {
 ; CGSCC-NEXT:    ret i32 0
 ;
   ret i32 0
@@ -62,7 +62,7 @@ define i32 @foo(ptr %p, i32 %v) {
 define internal i32 @test2(ptr %p, i32 %p2) {
 ; CGSCC: Function Attrs: mustprogress nofree nosync nounwind willreturn memory(argmem: read)
 ; CGSCC-LABEL: define {{[^@]+}}@test2
-; CGSCC-SAME: (ptr nocapture nofree readonly [[P:%.*]], i32 [[P2:%.*]]) #[[ATTR1]] {
+; CGSCC-SAME: (ptr nofree readonly captures(none) [[P:%.*]], i32 [[P2:%.*]]) #[[ATTR1]] {
 ; CGSCC-NEXT:    [[A_GEP:%.*]] = getelementptr [[T:%.*]], ptr [[P]], i64 0, i32 3
 ; CGSCC-NEXT:    [[B_GEP:%.*]] = getelementptr [[T]], ptr [[P]], i64 0, i32 2
 ; CGSCC-NEXT:    [[A:%.*]] = load i32, ptr [[A_GEP]], align 4
@@ -83,13 +83,13 @@ define internal i32 @test2(ptr %p, i32 %p2) {
 define i32 @caller2(ptr %g) {
 ; TUNIT: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(none)
 ; TUNIT-LABEL: define {{[^@]+}}@caller2
-; TUNIT-SAME: (ptr nocapture nofree readnone [[G:%.*]]) #[[ATTR1]] {
+; TUNIT-SAME: (ptr nofree readnone captures(none) [[G:%.*]]) #[[ATTR1]] {
 ; TUNIT-NEXT:    ret i32 0
 ;
 ; CGSCC: Function Attrs: mustprogress nofree nosync nounwind willreturn memory(argmem: read)
 ; CGSCC-LABEL: define {{[^@]+}}@caller2
-; CGSCC-SAME: (ptr nocapture nofree readonly align 4 [[G:%.*]]) #[[ATTR1]] {
-; CGSCC-NEXT:    [[V:%.*]] = call noundef i32 @test2(ptr nocapture nofree readonly [[G]], i32 noundef 0) #[[ATTR5]]
+; CGSCC-SAME: (ptr nofree readonly align 4 captures(none) [[G:%.*]]) #[[ATTR1]] {
+; CGSCC-NEXT:    [[V:%.*]] = call noundef i32 @test2(ptr nofree readonly captures(none) [[G]], i32 noundef 0) #[[ATTR5]]
 ; CGSCC-NEXT:    ret i32 [[V]]
 ;
   %v = call i32 @test2(ptr %g, i32 0)
@@ -103,13 +103,13 @@ define i32 @caller2(ptr %g) {
 define i32 @bar(ptr %p, i32 %v) {
 ; TUNIT: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: write)
 ; TUNIT-LABEL: define {{[^@]+}}@bar
-; TUNIT-SAME: (ptr nocapture nofree nonnull writeonly dereferenceable(4) [[P:%.*]], i32 [[V:%.*]]) #[[ATTR2:[0-9]+]] {
+; TUNIT-SAME: (ptr nofree nonnull writeonly captures(none) dereferenceable(4) [[P:%.*]], i32 [[V:%.*]]) #[[ATTR2:[0-9]+]] {
 ; TUNIT-NEXT:    store i32 [[V]], ptr [[P]], align 4
 ; TUNIT-NEXT:    ret i32 0
 ;
 ; CGSCC: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: write)
 ; CGSCC-LABEL: define {{[^@]+}}@bar
-; CGSCC-SAME: (ptr nocapture nofree nonnull writeonly dereferenceable(4) [[P:%.*]], i32 [[V:%.*]]) #[[ATTR3:[0-9]+]] {
+; CGSCC-SAME: (ptr nofree nonnull writeonly captures(none) dereferenceable(4) [[P:%.*]], i32 [[V:%.*]]) #[[ATTR3:[0-9]+]] {
 ; CGSCC-NEXT:    store i32 [[V]], ptr [[P]], align 4
 ; CGSCC-NEXT:    ret i32 0
 ;
@@ -121,7 +121,7 @@ define i32 @bar(ptr %p, i32 %v) {
 define internal i32 @test2b(ptr %p, i32 %p2) {
 ; TUNIT: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: readwrite)
 ; TUNIT-LABEL: define {{[^@]+}}@test2b
-; TUNIT-SAME: (ptr nocapture nofree readonly [[P:%.*]], i32 [[P2:%.*]]) #[[ATTR3:[0-9]+]] {
+; TUNIT-SAME: (ptr nofree readonly captures(none) [[P:%.*]], i32 [[P2:%.*]]) #[[ATTR3:[0-9]+]] {
 ; TUNIT-NEXT:    [[A_GEP:%.*]] = getelementptr [[T:%.*]], ptr [[P]], i64 0, i32 3
 ; TUNIT-NEXT:    [[B_GEP:%.*]] = getelementptr [[T]], ptr [[P]], i64 0, i32 2
 ; TUNIT-NEXT:    [[A:%.*]] = load i32, ptr [[A_GEP]], align 4
@@ -132,7 +132,7 @@ define internal i32 @test2b(ptr %p, i32 %p2) {
 ;
 ; CGSCC: Function Attrs: mustprogress nofree nosync nounwind willreturn memory(argmem: readwrite)
 ; CGSCC-LABEL: define {{[^@]+}}@test2b
-; CGSCC-SAME: (ptr nocapture nofree readonly [[P:%.*]], i32 [[P2:%.*]]) #[[ATTR4:[0-9]+]] {
+; CGSCC-SAME: (ptr nofree readonly captures(none) [[P:%.*]], i32 [[P2:%.*]]) #[[ATTR4:[0-9]+]] {
 ; CGSCC-NEXT:    [[A_GEP:%.*]] = getelementptr [[T:%.*]], ptr [[P]], i64 0, i32 3
 ; CGSCC-NEXT:    [[B_GEP:%.*]] = getelementptr [[T]], ptr [[P]], i64 0, i32 2
 ; CGSCC-NEXT:    [[A:%.*]] = load i32, ptr [[A_GEP]], align 4
@@ -153,14 +153,14 @@ define internal i32 @test2b(ptr %p, i32 %p2) {
 define i32 @caller2b(ptr %g) {
 ; TUNIT: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: readwrite)
 ; TUNIT-LABEL: define {{[^@]+}}@caller2b
-; TUNIT-SAME: (ptr nocapture nofree readonly [[G:%.*]]) #[[ATTR3]] {
-; TUNIT-NEXT:    [[V:%.*]] = call i32 @test2b(ptr nocapture nofree readonly [[G]], i32 undef) #[[ATTR6:[0-9]+]]
+; TUNIT-SAME: (ptr nofree readonly captures(none) [[G:%.*]]) #[[ATTR3]] {
+; TUNIT-NEXT:    [[V:%.*]] = call i32 @test2b(ptr nofree readonly captures(none) [[G]], i32 undef) #[[ATTR6:[0-9]+]]
 ; TUNIT-NEXT:    ret i32 0
 ;
 ; CGSCC: Function Attrs: mustprogress nofree nosync nounwind willreturn memory(argmem: readwrite)
 ; CGSCC-LABEL: define {{[^@]+}}@caller2b
-; CGSCC-SAME: (ptr nocapture nofree readonly align 4 [[G:%.*]]) #[[ATTR4]] {
-; CGSCC-NEXT:    [[V:%.*]] = call noundef i32 @test2b(ptr nocapture nofree readonly [[G]], i32 noundef 0) #[[ATTR8:[0-9]+]]
+; CGSCC-SAME: (ptr nofree readonly align 4 captures(none) [[G:%.*]]) #[[ATTR4]] {
+; CGSCC-NEXT:    [[V:%.*]] = call noundef i32 @test2b(ptr nofree readonly captures(none) [[G]], i32 noundef 0) #[[ATTR8:[0-9]+]]
 ; CGSCC-NEXT:    ret i32 [[V]]
 ;
   %v = call i32 @test2b(ptr %g, i32 0)
diff --git a/llvm/test/Transforms/Attributor/ArgumentPromotion/pr32917.ll b/llvm/test/Transforms/Attributor/ArgumentPromotion/pr32917.ll
index a8639be2fa904..11467f23245fe 100644
--- a/llvm/test/Transforms/Attributor/ArgumentPromotion/pr32917.ll
+++ b/llvm/test/Transforms/Attributor/ArgumentPromotion/pr32917.ll
@@ -17,7 +17,7 @@ define i32 @fn2() local_unnamed_addr {
 ; TUNIT-NEXT:    [[TMP1:%.*]] = load i32, ptr @b, align 4
 ; TUNIT-NEXT:    [[TMP2:%.*]] = sext i32 [[TMP1]] to i64
 ; TUNIT-NEXT:    [[TMP3:%.*]] = inttoptr i64 [[TMP2]] to ptr
-; TUNIT-NEXT:    call fastcc void @fn1(ptr nocapture nofree readonly align 4 [[TMP3]]) #[[ATTR1:[0-9]+]]
+; TUNIT-NEXT:    call fastcc void @fn1(ptr nofree readonly align 4 captures(none) [[TMP3]]) #[[ATTR1:[0-9]+]]
 ; TUNIT-NEXT:    ret i32 undef
 ;
 ; CGSCC: Function Attrs: mustprogress nofree nosync nounwind willreturn
@@ -26,7 +26,7 @@ define i32 @fn2() local_unnamed_addr {
 ; CGSCC-NEXT:    [[TMP1:%.*]] = load i32, ptr @b, align 4
 ; CGSCC-NEXT:    [[TMP2:%.*]] = sext i32 [[TMP1]] to i64
 ; CGSCC-NEXT:    [[TMP3:%.*]] = inttoptr i64 [[TMP2]] to ptr
-; CGSCC-NEXT:    call fastcc void @fn1(ptr nocapture nofree nonnull readonly align 4 [[TMP3]]) #[[ATTR2:[0-9]+]]
+; CGSCC-NEXT:    call fastcc void @fn1(ptr nofree nonnull readonly align 4 captures(none) [[TMP3]]) #[[ATTR2:[0-9]+]]
 ; CGSCC-NEXT:    ret i32 undef
 ;
   %1 = load i32, ptr @b, align 4
@@ -39,7 +39,7 @@ define i32 @fn2() local_unnamed_addr {
 define internal fastcc void @fn1(ptr nocapture readonly) unnamed_addr {
 ; TUNIT: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn
 ; TUNIT-LABEL: define {{[^@]+}}@fn1
-; TUNIT-SAME: (ptr nocapture nofree nonnull readonly align 4 [[TMP0:%.*]]) unnamed_addr #[[ATTR0]] {
+; TUNIT-SAME: (ptr nofree nonnull readonly align 4 captures(none) [[TMP0:%.*]]) unnamed_addr #[[ATTR0]] {
 ; TUNIT-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[TMP0]], i64 -1
 ; TUNIT-NEXT:    [[TMP3:%.*]] = load i32, ptr [[TMP2]], align 4
 ; TUNIT-NEXT:    store i32 [[TMP3]], ptr @a, align 4
@@ -47,7 +47,7 @@ define internal fastcc void @fn1(ptr nocapture readonly) unnamed_addr {
 ;
 ; CGSCC: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn
 ; CGSCC-LABEL: define {{[^@]+}}@fn1
-; CGSCC-SAME: (ptr nocapture nofree nonnull readonly align 4 [[TMP0:%.*]]) unnamed_addr #[[ATTR1:[0-9]+]] {
+; CGSCC-SAME: (ptr nofree nonnull readonly align 4 captures(none) [[TMP0:%.*]]) unnamed_addr #[[ATTR1:[0-9]+]] {
 ; CGSCC-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[TMP0]], i64 -1
 ; CGSCC-NEXT:    [[TMP3:%.*]] = load i32, ptr [[TMP2]], align 4
 ; CGSCC-NEXT:    store i32 [[TMP3]], ptr @a, align 4
diff --git a/llvm/test/Transforms/Attributor/ArgumentPromotion/pr33641_remove_arg_dbgvalue.ll b/llvm/test/Transforms/Attributor/ArgumentPromotion/pr33641_remove_arg_dbgvalue.ll
index a4797a093abc0..d6fcc6eaf458a 100644
--- a/llvm/test/Transforms/Attributor/ArgumentPromotion/pr33641_remove_arg_dbgvalue.ll
+++ b/llvm/test/Transforms/Attributor/ArgumentPromotion/pr33641_remove_arg_dbgvalue.ll
@@ -29,7 +29,7 @@ define void @foo() {
 define internal void @bar(%p_t %p)  {
 ; CGSCC: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(none)
 ; CGSCC-LABEL: define {{[^@]+}}@bar
-; CGSCC-SAME: (ptr nocapture nofree readnone [[P:%.*]]) #[[ATTR0]] {
+; CGSCC-SAME: (ptr nofree readnone captures(none) [[P:%.*]]) #[[ATTR0]] {
 ; CGSCC-NEXT:      #dbg_value(ptr [[P]], [[META3:![0-9]+]], !DIExpression(), [[META5:![0-9]+]])
 ; CGSCC-NEXT:    ret void
 ;
diff --git a/llvm/test/Transforms/Attributor/ArgumentPromotion/sret.ll b/llvm/test/Transforms/Attributor/ArgumentPromotion/sret.ll
index cb5b01750aaf0..a7369e5fd23a1 100644
--- a/llvm/test/Transforms/Attributor/ArgumentPromotion/sret.ll
+++ b/llvm/test/Transforms/Attributor/ArgumentPromotion/sret.ll
@@ -9,13 +9,13 @@ define internal void @add(ptr %this, ptr sret(i32) %r) {
 ;
 ; TUNIT: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: write)
 ; TUNIT-LABEL: define {{[^@]+}}@add
-; TUNIT-SAME: (ptr noalias nocapture nofree nonnull readnone align 8 dereferenceable(8) [[THIS:%.*]], ptr noalias nocapture nofree noundef nonnull writeonly sret(i32) align 4 dereferenceable(4) [[R:%.*]]) #[[ATTR0:[0-9]+]] {
+; TUNIT-SAME: (ptr noalias nofree nonnull readnone align 8 captures(none) dereferenceable(8) [[THIS:%.*]], ptr noalias nofree noundef nonnull writeonly sret(i32) align 4 captures(none) dereferenceable(4) [[R:%.*]]) #[[ATTR0:[0-9]+]] {
 ; TUNIT-NEXT:    store i32 undef, ptr [[R]], align 4
 ; TUNIT-NEXT:    ret void
 ;
 ; CGSCC: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: readwrite)
 ; CGSCC-LABEL: define {{[^@]+}}@add
-; CGSCC-SAME: (ptr noalias nocapture nofree noundef nonnull readonly align 8 dereferenceable(8) [[THIS:%.*]], ptr noalias nocapture nofree noundef nonnull writeonly sret(i32) align 4 dereferenceable(4) [[R:%.*]]) #[[ATTR0:[0-9]+]] {
+; CGSCC-SAME: (ptr noalias nofree noundef nonnull readonly align 8 captures(none) dereferenceable(8) [[THIS:%.*]], ptr noalias nofree noundef nonnull writeonly sret(i32) align 4 captures(none) dereferenceable(4) [[R:%.*]]) #[[ATTR0:[0-9]+]] {
 ; CGSCC-NEXT:    [[BP:%.*]] = getelementptr { i32, i32 }, ptr [[THIS]], i32 0, i32 1
 ; CGSCC-NEXT:    [[A:%.*]] = load i32, ptr [[THIS]], align 8
 ; CGSCC-NEXT:    [[B:%.*]] = load i32, ptr [[BP]], align 4
@@ -37,7 +37,7 @@ define void @f() {
 ; TUNIT-LABEL: define {{[^@]+}}@f
 ; TUNIT-SAME: () #[[ATTR1:[0-9]+]] {
 ; TUNIT-NEXT:    [[R:%.*]] = alloca i32, align 4
-; TUNIT-NEXT:    call void @add(ptr noalias nocapture nofree nonnull readnone align 8 dereferenceable(8) undef, ptr noalias nocapture nofree noundef nonnull writeonly sret(i32) align 4 dereferenceable(4) [[R]]) #[[ATTR2:[0-9]+]]
+; TUNIT-NEXT:    call void @add(ptr noalias nofree nonnull readnone align 8 captures(none) dereferenceable(8) undef, ptr noalias nofree noundef nonnull writeonly sret(i32) align 4 captures(none) dereferenceable(4) [[R]]) #[[ATTR2:[0-9]+]]
 ; TUNIT-NEXT:    ret void
 ;
 ; CGSCC: Function Attrs: mustprogress nofree nosync nounwind willreturn memory(none)
@@ -45,7 +45,7 @@ define void @f() {
 ; CGSCC-SAME: () #[[ATTR1:[0-9]+]] {
 ; CGSCC-NEXT:    [[R:%.*]] = alloca i32, align 4
 ; CGSCC-NEXT:    [[PAIR:%.*]] = alloca { i32, i32 }, align 8
-; CGSCC-NEXT:    call void @add(ptr noalias nocapture nofree noundef nonnull readonly align 8 dereferenceable(8) [[PAIR]], ptr noalias nocapture nofree noundef nonnull writeonly sret(i32) align 4 dereferenceable(4) [[R]]) #[[ATTR2:[0-9]+]]
+; CGSCC-NEXT:    call void @add(ptr noalias nofree noundef nonnull readonly align 8 captures(none) dereferenceable(8) [[PAIR]], ptr noalias nofree noundef nonnull writeonly sret(i32) align 4 captures(none) dereferenceable(4) [[R]]) #[[ATTR2:[0-9]+]]
 ; CGSCC-NEXT:    ret void
 ;
   %r = alloca i32
diff --git a/llvm/test/Transforms/Attributor/ArgumentPromotion/tail.ll b/llvm/test/Transforms/Attributor/ArgumentPromotion/tail.ll
index c1e92bde0236d..00e44ea53797d 100644
--- a/llvm/test/Transforms/Attributor/ArgumentPromotion/tail.ll
+++ b/llvm/test/Transforms/Attributor/ArgumentPromotion/tail.ll
@@ -36,7 +36,7 @@ define internal void @bar(ptr byval(%pair) %Data) {
 
 define void @zed(ptr byval(%pair) %Data) {
 ; TUNIT-LABEL: define {{[^@]+}}@zed
-; TUNIT-SAME: (ptr noalias nocapture nonnull readonly byval([[PAIR:%.*]]) dereferenceable(8) [[DATA:%.*]]) {
+; TUNIT-SAME: (ptr noalias nonnull readonly byval([[PAIR:%.*]]) captures(none) dereferenceable(8) [[DATA:%.*]]) {
 ; TUNIT-NEXT:    [[TMP1:%.*]] = load i32, ptr [[DATA]], align 1
 ; TUNIT-NEXT:    [[DATA_B4:%.*]] = getelementptr i8, ptr [[DATA]], i64 4
 ; TUNIT-NEXT:    [[TMP2:%.*]] = load i32, ptr [[DATA_B4]], align 1
@@ -44,7 +44,7 @@ define void @zed(ptr byval(%pair) %Data) {
 ; TUNIT-NEXT:    ret void
 ;
 ; CGSCC-LABEL: define {{[^@]+}}@zed
-; CGSCC-SAME: (ptr noalias nocapture nofree noundef nonnull readonly byval([[PAIR:%.*]]) dereferenceable(8) [[DATA:%.*]]) {
+; CGSCC-SAME: (ptr noalias nofree noundef nonnull readonly byval([[PAIR:%.*]]) captures(none) dereferenceable(8) [[DATA:%.*]]) {
 ; CGSCC-NEXT:    [[TMP1:%.*]] = load i32, ptr [[DATA]], align 1
 ; CGSCC-NEXT:    [[DATA_B4:%.*]] = getelementptr i8, ptr [[DATA]], i64 4
 ; CGSCC-NEXT:    [[TMP2:%.*]] = load i32, ptr [[DATA_B4]], align 1
diff --git a/llvm/test/Transforms/Attributor/ArgumentPromotion/variadic.ll b/llvm/test/Transforms/Attributor/ArgumentPromotion/variadic.ll
index f260b0192cf19..70f860b62434b 100644
--- a/llvm/test/Transforms/Attributor/ArgumentPromotion/variadic.ll
+++ b/llvm/test/Transforms/Attributor/ArgumentPromotion/variadic.ll
@@ -20,7 +20,7 @@ target triple = "x86_64-unknown-linux-gnu"
 ;.
 define i32 @main(i32 %argc, ptr nocapture readnone %argv) #0 {
 ; CHECK-LABEL: define {{[^@]+}}@main
-; CHECK-SAME: (i32 [[ARGC:%.*]], ptr nocapture nofree readnone [[ARGV:%.*]]) {
+; CHECK-SAME: (i32 [[ARGC:%.*]], ptr nofree readnone captures(none) [[ARGV:%.*]]) {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    tail call void (ptr, ptr, ptr, ptr, ptr, ...) @callee_t0f(ptr nofree undef, ptr nofree undef, ptr nofree undef, ptr nofree undef, ptr nofree undef, ptr noalias noundef nonnull byval([[STRUCT_TT0:%.*]]) align 8 dereferenceable(16) @t45)
 ; CHECK-NEXT:    ret i32 0
@@ -33,7 +33,7 @@ entry:
 ; Function Attrs: nounwind uwtable
 define internal void @callee_t0f(ptr nocapture readnone %tp13, ptr nocapture readnone %tp14, ptr nocapture readnone %tp15, ptr nocapture readnone %tp16, ptr nocapture readnone %tp17, ...) {
 ; CHECK-LABEL: define {{[^@]+}}@callee_t0f
-; CHECK-SAME: (ptr noalias nocapture nofree readnone align 4294967296 [[TP13:%.*]], ptr noalias nocapture nofree readnone align 4294967296 [[TP14:%.*]], ptr noalias nocapture nofree readnone align 4294967296 [[TP15:%.*]], ptr noalias nocapture nofree readnone align 4294967296 [[TP16:%.*]], ptr noalias nocapture nofree readnone align 4294967296 [[TP17:%.*]], ...) {
+; CHECK-SAME: (ptr noalias nofree readnone align 4294967296 captures(none) [[TP13:%.*]], ptr noalias nofree readnone align 4294967296 captures(none) [[TP14:%.*]], ptr noalias nofree readnone align 4294967296 captures(none) [[TP15:%.*]], ptr noalias nofree readnone align 4294967296 captures(none) [[TP16:%.*]], ptr noalias nofree readnone align 4294967296 captures(none) [[TP17:%.*]], ...) {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    call void @sink(i32 noundef 0)
 ; CHECK-NEXT:    ret void
diff --git a/llvm/test/Transforms/Attributor/IPConstantProp/PR26044.ll b/llvm/test/Transforms/Attributor/IPConstantProp/PR26044.ll
index f673ffc3bfac6..52e355cf43ddb 100644
--- a/llvm/test/Transforms/Attributor/IPConstantProp/PR26044.ll
+++ b/llvm/test/Transforms/Attributor/IPConstantProp/PR26044.ll
@@ -8,7 +8,7 @@ define void @fn2(ptr %P, i1 %C) {
 ;
 ; TUNIT: Function Attrs: nofree norecurse nosync nounwind memory(argmem: readwrite)
 ; TUNIT-LABEL: define {{[^@]+}}@fn2
-; TUNIT-SAME: (ptr nocapture nofree [[P:%.*]], i1 [[C:%.*]]) #[[ATTR0:[0-9]+]] {
+; TUNIT-SAME: (ptr nofree captures(none) [[P:%.*]], i1 [[C:%.*]]) #[[ATTR0:[0-9]+]] {
 ; TUNIT-NEXT:  entry:
 ; TUNIT-NEXT:    br label [[IF_END:%.*]]
 ; TUNIT:       for.cond1:
@@ -23,7 +23,7 @@ define void @fn2(ptr %P, i1 %C) {
 ;
 ; CGSCC: Function Attrs: nofree nosync nounwind memory(argmem: readwrite)
 ; CGSCC-LABEL: define {{[^@]+}}@fn2
-; CGSCC-SAME: (ptr nocapture nofree nonnull align 4 dereferenceable(4) [[P:%.*]], i1 [[C:%.*]]) #[[ATTR0:[0-9]+]] {
+; CGSCC-SAME: (ptr nofree nonnull align 4 captures(none) dereferenceable(4) [[P:%.*]], i1 [[C:%.*]]) #[[ATTR0:[0-9]+]] {
 ; CGSCC-NEXT:  entry:
 ; CGSCC-NEXT:    br label [[IF_END:%.*]]
 ; CGSCC:       for.cond1:
@@ -70,7 +70,7 @@ define void @fn_no_null_opt(ptr %P, i1 %C) null_pointer_is_valid {
 ;
 ; TUNIT: Function Attrs: nofree norecurse nosync nounwind null_pointer_is_valid
 ; TUNIT-LABEL: define {{[^@]+}}@fn_no_null_opt
-; TUNIT-SAME: (ptr nocapture nofree writeonly [[P:%.*]], i1 [[C:%.*]]) #[[ATTR1:[0-9]+]] {
+; TUNIT-SAME: (ptr nofree writeonly captures(none) [[P:%.*]], i1 [[C:%.*]]) #[[ATTR1:[0-9]+]] {
 ; TUNIT-NEXT:  entry:
 ; TUNIT-NEXT:    br label [[IF_END:%.*]]
 ; TUNIT:       for.cond1:
@@ -85,7 +85,7 @@ define void @fn_no_null_opt(ptr %P, i1 %C) null_pointer_is_valid {
 ;
 ; CGSCC: Function Attrs: nofree nosync nounwind null_pointer_is_valid
 ; CGSCC-LABEL: define {{[^@]+}}@fn_no_null_opt
-; CGSCC-SAME: (ptr nocapture nofree writeonly align 4 dereferenceable_or_null(4) [[P:%.*]], i1 [[C:%.*]]) #[[ATTR2:[0-9]+]] {
+; CGSCC-SAME: (ptr nofree writeonly align 4 captures(none) dereferenceable_or_null(4) [[P:%.*]], i1 [[C:%.*]]) #[[ATTR2:[0-9]+]] {
 ; CGSCC-NEXT:  entry:
 ; CGSCC-NEXT:    br label [[IF_END:%.*]]
 ; CGSCC:       for.cond1:
diff --git a/llvm/test/Transforms/Attributor/IPConstantProp/dangling-block-address.ll b/llvm/test/Transforms/Attributor/IPConstantProp/dangling-block-address.ll
index d76ea5e9290b1..b224b5e238e57 100644
--- a/llvm/test/Transforms/Attributor/IPConstantProp/dangling-block-address.ll
+++ b/llvm/test/Transforms/Attributor/IPConstantProp/dangling-block-address.ll
@@ -34,7 +34,7 @@ entry:
 define internal void @bar(ptr nocapture %pc) nounwind readonly {
 ; CGSCC: Function Attrs: nounwind memory(read)
 ; CGSCC-LABEL: define {{[^@]+}}@bar
-; CGSCC-SAME: (ptr nocapture [[PC:%.*]]) #[[ATTR1:[0-9]+]] {
+; CGSCC-SAME: (ptr captures(none) [[PC:%.*]]) #[[ATTR1:[0-9]+]] {
 ; CGSCC-NEXT:  entry:
 ; CGSCC-NEXT:    br label [[INDIRECTGOTO:%.*]]
 ; CGSCC:       lab0:
diff --git a/llvm/test/Transforms/Attributor/IPConstantProp/openmp_parallel_for.ll b/llvm/test/Transforms/Attributor/IPConstantProp/openmp_parallel_for.ll
index 683da42b64811..c0d778b9d72b1 100644
--- a/llvm/test/Transforms/Attributor/IPConstantProp/openmp_parallel_for.ll
+++ b/llvm/test/Transforms/Attributor/IPConstantProp/openmp_parallel_for.ll
@@ -36,7 +36,7 @@ define dso_local void @foo(i32 %N) {
 ; TUNIT-NEXT:  entry:
 ; TUNIT-NEXT:    [[N_ADDR:%.*]] = alloca i32, align 4
 ; TUNIT-NEXT:    [[P:%.*]] = alloca float, align 4
-; TUNIT-NEXT:    call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr noundef nonnull align 8 dereferenceable(24) @[[GLOB1]], i32 noundef 3, ptr noundef nonnull @.omp_outlined., ptr noalias nocapture nofree nonnull readnone align 4 dereferenceable(4) undef, ptr noalias nocapture nofree nonnull readnone align 4 dereferenceable(4) undef, i64 undef)
+; TUNIT-NEXT:    call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr noundef nonnull align 8 dereferenceable(24) @[[GLOB1]], i32 noundef 3, ptr noundef nonnull @.omp_outlined., ptr noalias nofree nonnull readnone align 4 captures(none) dereferenceable(4) undef, ptr noalias nofree nonnull readnone align 4 captures(none) dereferenceable(4) undef, i64 undef)
 ; TUNIT-NEXT:    ret void
 ;
 ; CGSCC-LABEL: define {{[^@]+}}@foo
@@ -47,7 +47,7 @@ define dso_local void @foo(i32 %N) {
 ; CGSCC-NEXT:    store i32 [[N]], ptr [[N_ADDR]], align 4
 ; CGSCC-NEXT:    store float 3.000000e+00, ptr [[P]], align 4
 ; CGSCC-NEXT:    store i32 7, ptr [[N_ADDR]], align 4
-; CGSCC-NEXT:    call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr noundef nonnull align 8 dereferenceable(24) @[[GLOB1]], i32 noundef 3, ptr noundef nonnull @.omp_outlined., ptr noalias nocapture nofree noundef nonnull readonly align 4 dereferenceable(4) [[N_ADDR]], ptr noalias nocapture nofree noundef nonnull readonly align 4 dereferenceable(4) [[P]], i64 noundef 4617315517961601024)
+; CGSCC-NEXT:    call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr noundef nonnull align 8 dereferenceable(24) @[[GLOB1]], i32 noundef 3, ptr noundef nonnull @.omp_outlined., ptr noalias nofree noundef nonnull readonly align 4 captures(none) dereferenceable(4) [[N_ADDR]], ptr noalias nofree noundef nonnull readonly align 4 captures(none) dereferenceable(4) [[P]], i64 noundef 4617315517961601024)
 ; CGSCC-NEXT:    ret void
 ;
 entry:
@@ -62,7 +62,7 @@ entry:
 
 define internal void @.omp_outlined.(ptr noalias %.global_tid., ptr noalias %.bound_tid., ptr dereferenceable(4) %N, ptr dereferenceable(4) %p, i64 %q) {
 ; TUNIT-LABEL: define {{[^@]+}}@.omp_outlined.
-; TUNIT-SAME: (ptr noalias nocapture nofree readonly [[DOTGLOBAL_TID_:%.*]], ptr noalias nocapture nofree readnone [[DOTBOUND_TID_:%.*]], ptr noalias nocapture nofree noundef nonnull readnone align 4 dereferenceable(4) [[N:%.*]], ptr noalias nocapture nofree noundef nonnull readnone align 4 dereferenceable(4) [[P:%.*]], i64 [[Q:%.*]]) {
+; TUNIT-SAME: (ptr noalias nofree readonly captures(none) [[DOTGLOBAL_TID_:%.*]], ptr noalias nofree readnone captures(none) [[DOTBOUND_TID_:%.*]], ptr noalias nofree noundef nonnull readnone align 4 captures(none) dereferenceable(4) [[N:%.*]], ptr noalias nofree noundef nonnull readnone align 4 captures(none) dereferenceable(4) [[P:%.*]], i64 [[Q:%.*]]) {
 ; TUNIT-NEXT:  entry:
 ; TUNIT-NEXT:    [[Q_ADDR:%.*]] = alloca i64, align 8
 ; TUNIT-NEXT:    [[DOTOMP_LB:%.*]] = alloca i32, align 4
@@ -118,7 +118,7 @@ define internal void @.omp_outlined.(ptr noalias %.global_tid., ptr noalias %.bo
 ; TUNIT-NEXT:    ret void
 ;
 ; CGSCC-LABEL: define {{[^@]+}}@.omp_outlined.
-; CGSCC-SAME: (ptr noalias nocapture nofree readonly [[DOTGLOBAL_TID_:%.*]], ptr noalias nocapture nofree readnone [[DOTBOUND_TID_:%.*]], ptr noalias nocapture nofree noundef nonnull readonly align 4 dereferenceable(4) [[N:%.*]], ptr noalias nocapture nofree noundef nonnull readonly align 4 dereferenceable(4) [[P:%.*]], i64 [[Q:%.*]]) {
+; CGSCC-SAME: (ptr noalias nofree readonly captures(none) [[DOTGLOBAL_TID_:%.*]], ptr noalias nofree readnone captures(none) [[DOTBOUND_TID_:%.*]], ptr noalias nofree noundef nonnull readonly align 4 captures(none) dereferenceable(4) [[N:%.*]], ptr noalias nofree noundef nonnull readonly align 4 captures(none) dereferenceable(4) [[P:%.*]], i64 [[Q:%.*]]) {
 ; CGSCC-NEXT:  entry:
 ; CGSCC-NEXT:    [[Q_ADDR:%.*]] = alloca i64, align 8
 ; CGSCC-NEXT:    [[DOTOMP_LB:%.*]] = alloca i32, align 4
diff --git a/llvm/test/Transforms/Attributor/IPConstantProp/pthreads.ll b/llvm/test/Transforms/Attributor/IPConstantProp/pthreads.ll
index 01a97821140ec..5accfc9b5ce6c 100644
--- a/llvm/test/Transforms/Attributor/IPConstantProp/pthreads.ll
+++ b/llvm/test/Transforms/Attributor/IPConstantProp/pthreads.ll
@@ -38,8 +38,8 @@ define dso_local i32 @main() {
 ; TUNIT-NEXT:    [[ALLOC2:%.*]] = alloca i8, align 8
 ; TUNIT-NEXT:    [[THREAD:%.*]] = alloca i64, align 8
 ; TUNIT-NEXT:    [[CALL:%.*]] = call i32 @pthread_create(ptr noundef nonnull align 8 dereferenceable(8) [[THREAD]], ptr noundef align 4294967296 null, ptr noundef nonnull @foo, ptr nofree readnone align 4294967296 undef)
-; TUNIT-NEXT:    [[CALL1:%.*]] = call i32 @pthread_create(ptr noundef nonnull align 8 dereferenceable(8) [[THREAD]], ptr noundef align 4294967296 null, ptr noundef nonnull @bar, ptr noalias nocapture nofree nonnull readnone align 8 dereferenceable(8) undef)
-; TUNIT-NEXT:    [[CALL2:%.*]] = call i32 @pthread_create(ptr noundef nonnull align 8 dereferenceable(8) [[THREAD]], ptr noundef align 4294967296 null, ptr noundef nonnull @baz, ptr noalias nocapture nofree noundef nonnull readnone align 8 dereferenceable(1) [[ALLOC1]])
+; TUNIT-NEXT:    [[CALL1:%.*]] = call i32 @pthread_create(ptr noundef nonnull align 8 dereferenceable(8) [[THREAD]], ptr noundef align 4294967296 null, ptr noundef nonnull @bar, ptr noalias nofree nonnull readnone align 8 captures(none) dereferenceable(8) undef)
+; TUNIT-NEXT:    [[CALL2:%.*]] = call i32 @pthread_create(ptr noundef nonnull align 8 dereferenceable(8) [[THREAD]], ptr noundef align 4294967296 null, ptr noundef nonnull @baz, ptr noalias nofree noundef nonnull readnone align 8 captures(none) dereferenceable(1) [[ALLOC1]])
 ; TUNIT-NEXT:    [[CALL3:%.*]] = call i32 @pthread_create(ptr noundef nonnull align 8 dereferenceable(8) [[THREAD]], ptr noundef align 4294967296 null, ptr noundef nonnull @buz, ptr noalias nofree noundef nonnull readnone align 8 dereferenceable(1) "no-capture-maybe-returned" [[ALLOC2]])
 ; TUNIT-NEXT:    ret i32 0
 ;
@@ -49,8 +49,8 @@ define dso_local i32 @main() {
 ; CGSCC-NEXT:    [[ALLOC2:%.*]] = alloca i8, align 8
 ; CGSCC-NEXT:    [[THREAD:%.*]] = alloca i64, align 8
 ; CGSCC-NEXT:    [[CALL:%.*]] = call i32 @pthread_create(ptr noundef nonnull align 8 dereferenceable(8) [[THREAD]], ptr noundef align 4294967296 null, ptr noundef nonnull @foo, ptr nofree noundef readnone align 4294967296 null)
-; CGSCC-NEXT:    [[CALL1:%.*]] = call i32 @pthread_create(ptr noundef nonnull align 8 dereferenceable(8) [[THREAD]], ptr noundef align 4294967296 null, ptr noundef nonnull @bar, ptr noalias nocapture nofree noundef nonnull readnone align 8 dereferenceable(8) @GlobalVPtr)
-; CGSCC-NEXT:    [[CALL2:%.*]] = call i32 @pthread_create(ptr noundef nonnull align 8 dereferenceable(8) [[THREAD]], ptr noundef align 4294967296 null, ptr noundef nonnull @baz, ptr noalias nocapture nofree noundef nonnull readnone align 8 dereferenceable(1) [[ALLOC1]])
+; CGSCC-NEXT:    [[CALL1:%.*]] = call i32 @pthread_create(ptr noundef nonnull align 8 dereferenceable(8) [[THREAD]], ptr noundef align 4294967296 null, ptr noundef nonnull @bar, ptr noalias nofree noundef nonnull readnone align 8 captures(none) dereferenceable(8) @GlobalVPtr)
+; CGSCC-NEXT:    [[CALL2:%.*]] = call i32 @pthread_create(ptr noundef nonnull align 8 dereferenceable(8) [[THREAD]], ptr noundef align 4294967296 null, ptr noundef nonnull @baz, ptr noalias nofree noundef nonnull readnone align 8 captures(none) dereferenceable(1) [[ALLOC1]])
 ; CGSCC-NEXT:    [[CALL3:%.*]] = call i32 @pthread_create(ptr noundef nonnull align 8 dereferenceable(8) [[THREAD]], ptr noundef align 4294967296 null, ptr noundef nonnull @buz, ptr noalias nofree noundef nonnull readnone align 8 dereferenceable(1) [[ALLOC2]])
 ; CGSCC-NEXT:    ret i32 0
 ;
@@ -70,7 +70,7 @@ declare !callback !0 dso_local i32 @pthread_create(ptr, ptr, ptr, ptr)
 define internal ptr @foo(ptr %arg) {
 ; CHECK: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(none)
 ; CHECK-LABEL: define {{[^@]+}}@foo
-; CHECK-SAME: (ptr noalias nocapture nofree readnone align 4294967296 [[ARG:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-SAME: (ptr noalias nofree readnone align 4294967296 captures(none) [[ARG:%.*]]) #[[ATTR0:[0-9]+]] {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    ret ptr null
 ;
@@ -81,7 +81,7 @@ entry:
 define internal ptr @bar(ptr %arg) {
 ; CHECK: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(none)
 ; CHECK-LABEL: define {{[^@]+}}@bar
-; CHECK-SAME: (ptr noalias nocapture nofree nonnull readnone align 8 dereferenceable(8) [[ARG:%.*]]) #[[ATTR0]] {
+; CHECK-SAME: (ptr noalias nofree nonnull readnone align 8 captures(none) dereferenceable(8) [[ARG:%.*]]) #[[ATTR0]] {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    ret ptr @GlobalVPtr
 ;
diff --git a/llvm/test/Transforms/Attributor/IPConstantProp/thread_local_acs.ll b/llvm/test/Transforms/Attributor/IPConstantProp/thread_local_acs.ll
index fd787d5b5d271..21ca6cc53260b 100644
--- a/llvm/test/Transforms/Attributor/IPConstantProp/thread_local_acs.ll
+++ b/llvm/test/Transforms/Attributor/IPConstantProp/thread_local_acs.ll
@@ -28,7 +28,7 @@ target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 define internal i32 @callee(ptr %thread_local_ptr, ptr %shared_ptr) {
 ; CHECK: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(read)
 ; CHECK-LABEL: define {{[^@]+}}@callee
-; CHECK-SAME: (ptr nocapture nofree noundef nonnull readonly align 4 dereferenceable(4) [[THREAD_LOCAL_PTR:%.*]], ptr nocapture nofree noundef nonnull readonly align 4 dereferenceable(4) [[SHARED_PTR:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-SAME: (ptr nofree noundef nonnull readonly align 4 captures(none) dereferenceable(4) [[THREAD_LOCAL_PTR:%.*]], ptr nofree noundef nonnull readonly align 4 captures(none) dereferenceable(4) [[SHARED_PTR:%.*]]) #[[ATTR0:[0-9]+]] {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[TMP:%.*]] = load i32, ptr @gtl, align 4
 ; CHECK-NEXT:    [[TMP1:%.*]] = load i32, ptr @gsh, align 4
@@ -45,12 +45,12 @@ entry:
 define dso_local void @caller() {
 ; TUNIT-LABEL: define {{[^@]+}}@caller() {
 ; TUNIT-NEXT:  entry:
-; TUNIT-NEXT:    call void @broker(ptr nocapture nofree nonnull readonly align 4 dereferenceable(4) undef, ptr noundef nonnull @callee, ptr nocapture nofree nonnull readonly align 4 dereferenceable(4) undef)
+; TUNIT-NEXT:    call void @broker(ptr nofree nonnull readonly align 4 captures(none) dereferenceable(4) undef, ptr noundef nonnull @callee, ptr nofree nonnull readonly align 4 captures(none) dereferenceable(4) undef)
 ; TUNIT-NEXT:    ret void
 ;
 ; CGSCC-LABEL: define {{[^@]+}}@caller() {
 ; CGSCC-NEXT:  entry:
-; CGSCC-NEXT:    call void @broker(ptr nocapture nofree noundef nonnull readonly align 4 dereferenceable(4) @gtl, ptr noundef nonnull @callee, ptr nocapture nofree noundef nonnull readonly align 4 dereferenceable(4) @gsh)
+; CGSCC-NEXT:    call void @broker(ptr nofree noundef nonnull readonly align 4 captures(none) dereferenceable(4) @gtl, ptr noundef nonnull @callee, ptr nofree noundef nonnull readonly align 4 captures(none) dereferenceable(4) @gsh)
 ; CGSCC-NEXT:    ret void
 ;
 entry:
diff --git a/llvm/test/Transforms/Attributor/address_space_info.ll b/llvm/test/Transforms/Attributor/address_space_info.ll
index 0c8b06ac6666a..2768aad960081 100644
--- a/llvm/test/Transforms/Attributor/address_space_info.ll
+++ b/llvm/test/Transforms/Attributor/address_space_info.ll
@@ -34,7 +34,7 @@ entry:
 define internal void @_Z13unknown_writePi(ptr noundef %p) #0 {
 ; CHECK: Function Attrs: mustprogress nofree noinline norecurse nosync nounwind willreturn memory(write)
 ; CHECK-LABEL: define {{[^@]+}}@_Z13unknown_writePi
-; CHECK-SAME: (ptr nocapture nofree noundef nonnull writeonly align 4 dereferenceable(4) [[P:%.*]]) #[[ATTR0]] {
+; CHECK-SAME: (ptr nofree noundef nonnull writeonly align 4 captures(none) dereferenceable(4) [[P:%.*]]) #[[ATTR0]] {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    store i32 2, ptr [[P]], align 4
 ; CHECK-NEXT:    ret void
@@ -48,7 +48,7 @@ entry:
 define internal void @_Z12shared_writePi(ptr noundef %p) #0 {
 ; CHECK: Function Attrs: mustprogress nofree noinline norecurse nosync nounwind willreturn memory(write)
 ; CHECK-LABEL: define {{[^@]+}}@_Z12shared_writePi
-; CHECK-SAME: (ptr nocapture nofree noundef nonnull writeonly align 4 dereferenceable(4) [[P:%.*]]) #[[ATTR0]] {
+; CHECK-SAME: (ptr nofree noundef nonnull writeonly align 4 captures(none) dereferenceable(4) [[P:%.*]]) #[[ATTR0]] {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[TMP0:%.*]] = addrspacecast ptr [[P]] to ptr addrspace(3)
 ; CHECK-NEXT:    store i32 3, ptr addrspace(3) [[TMP0]], align 4
@@ -63,7 +63,7 @@ entry:
 define internal void @_Z11global_readPi(ptr noundef %p) #0 {
 ; CHECK: Function Attrs: mustprogress nofree noinline norecurse nosync nounwind willreturn
 ; CHECK-LABEL: define {{[^@]+}}@_Z11global_readPi
-; CHECK-SAME: (ptr nocapture nofree noundef nonnull readonly align 4 dereferenceable(4) [[P:%.*]]) #[[ATTR1:[0-9]+]] {
+; CHECK-SAME: (ptr nofree noundef nonnull readonly align 4 captures(none) dereferenceable(4) [[P:%.*]]) #[[ATTR1:[0-9]+]] {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[TMP0:%.*]] = addrspacecast ptr [[P]] to ptr addrspace(1)
 ; CHECK-NEXT:    [[TMP1:%.*]] = load i32, ptr addrspace(1) [[TMP0]], align 4
@@ -80,7 +80,7 @@ entry:
 define internal void @_Z12unknown_readPi(ptr noundef %p) #0 {
 ; CHECK: Function Attrs: mustprogress nofree noinline norecurse nosync nounwind willreturn
 ; CHECK-LABEL: define {{[^@]+}}@_Z12unknown_readPi
-; CHECK-SAME: (ptr nocapture nofree noundef nonnull readonly align 4 dereferenceable(4) [[P:%.*]]) #[[ATTR1]] {
+; CHECK-SAME: (ptr nofree noundef nonnull readonly align 4 captures(none) dereferenceable(4) [[P:%.*]]) #[[ATTR1]] {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[TMP0:%.*]] = load i32, ptr [[P]], align 4
 ; CHECK-NEXT:    store i32 [[TMP0]], ptr addrspace(1) @dst, align 4
@@ -96,7 +96,7 @@ entry:
 define internal void @_Z11shared_readPi(ptr noundef %p) #0 {
 ; CHECK: Function Attrs: mustprogress nofree noinline norecurse nosync nounwind willreturn
 ; CHECK-LABEL: define {{[^@]+}}@_Z11shared_readPi
-; CHECK-SAME: (ptr nocapture nofree noundef nonnull readonly align 4 dereferenceable(4) [[P:%.*]]) #[[ATTR1]] {
+; CHECK-SAME: (ptr nofree noundef nonnull readonly align 4 captures(none) dereferenceable(4) [[P:%.*]]) #[[ATTR1]] {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[TMP0:%.*]] = addrspacecast ptr [[P]] to ptr addrspace(3)
 ; CHECK-NEXT:    [[TMP1:%.*]] = load i32, ptr addrspace(3) [[TMP0]], align 4
@@ -117,16 +117,16 @@ define dso_local void @_Z3bazv() #0 {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    call void @_Z12global_writePi(ptr nofree noundef nonnull writeonly align 4 dereferenceable(8) addrspacecast (ptr addrspace(1) @g1 to ptr)) #[[ATTR2:[0-9]+]]
 ; CHECK-NEXT:    call void @_Z12global_writePi(ptr nofree noundef nonnull writeonly align 4 dereferenceable(4) addrspacecast (ptr addrspace(1) @g2 to ptr)) #[[ATTR2]]
-; CHECK-NEXT:    call void @_Z13unknown_writePi(ptr nocapture nofree noundef nonnull writeonly align 4 dereferenceable(8) addrspacecast (ptr addrspace(1) @g1 to ptr)) #[[ATTR2]]
-; CHECK-NEXT:    call void @_Z13unknown_writePi(ptr nocapture nofree noundef nonnull writeonly align 4 dereferenceable(4) addrspacecast (ptr addrspace(3) @s1 to ptr)) #[[ATTR2]]
-; CHECK-NEXT:    call void @_Z12shared_writePi(ptr nocapture nofree noundef nonnull writeonly align 4 dereferenceable(4) addrspacecast (ptr addrspace(3) @s1 to ptr)) #[[ATTR2]]
-; CHECK-NEXT:    call void @_Z12shared_writePi(ptr nocapture nofree noundef nonnull writeonly align 4 dereferenceable(4) addrspacecast (ptr addrspace(3) @s2 to ptr)) #[[ATTR2]]
-; CHECK-NEXT:    call void @_Z11global_readPi(ptr nocapture nofree noundef nonnull readonly align 4 dereferenceable(8) addrspacecast (ptr addrspace(1) @g1 to ptr)) #[[ATTR3:[0-9]+]]
-; CHECK-NEXT:    call void @_Z11global_readPi(ptr nocapture nofree noundef nonnull readonly align 4 dereferenceable(4) addrspacecast (ptr addrspace(1) @g2 to ptr)) #[[ATTR3]]
-; CHECK-NEXT:    call void @_Z12unknown_readPi(ptr nocapture nofree noundef nonnull readonly align 4 dereferenceable(8) addrspacecast (ptr addrspace(1) @g1 to ptr)) #[[ATTR3]]
-; CHECK-NEXT:    call void @_Z12unknown_readPi(ptr nocapture nofree noundef nonnull readonly align 4 dereferenceable(4) addrspacecast (ptr addrspace(3) @s1 to ptr)) #[[ATTR3]]
-; CHECK-NEXT:    call void @_Z11shared_readPi(ptr nocapture nofree noundef nonnull readonly align 4 dereferenceable(4) addrspacecast (ptr addrspace(3) @s1 to ptr)) #[[ATTR3]]
-; CHECK-NEXT:    call void @_Z11shared_readPi(ptr nocapture nofree noundef nonnull readonly align 4 dereferenceable(4) addrspacecast (ptr addrspace(3) @s2 to ptr)) #[[ATTR3]]
+; CHECK-NEXT:    call void @_Z13unknown_writePi(ptr nofree noundef nonnull writeonly align 4 captures(none) dereferenceable(8) addrspacecast (ptr addrspace(1) @g1 to ptr)) #[[ATTR2]]
+; CHECK-NEXT:    call void @_Z13unknown_writePi(ptr nofree noundef nonnull writeonly align 4 captures(none) dereferenceable(4) addrspacecast (ptr addrspace(3) @s1 to ptr)) #[[ATTR2]]
+; CHECK-NEXT:    call void @_Z12shared_writePi(ptr nofree noundef nonnull writeonly align 4 captures(none) dereferenceable(4) addrspacecast (ptr addrspace(3) @s1 to ptr)) #[[ATTR2]]
+; CHECK-NEXT:    call void @_Z12shared_writePi(ptr nofree noundef nonnull writeonly align 4 captures(none) dereferenceable(4) addrspacecast (ptr addrspace(3) @s2 to ptr)) #[[ATTR2]]
+; CHECK-NEXT:    call void @_Z11global_readPi(ptr nofree noundef nonnull readonly align 4 captures(none) dereferenceable(8) addrspacecast (ptr addrspace(1) @g1 to ptr)) #[[ATTR3:[0-9]+]]
+; CHECK-NEXT:    call void @_Z11global_readPi(ptr nofree noundef nonnull readonly align 4 captures(none) dereferenceable(4) addrspacecast (ptr addrspace(1) @g2 to ptr)) #[[ATTR3]]
+; CHECK-NEXT:    call void @_Z12unknown_readPi(ptr nofree noundef nonnull readonly align 4 captures(none) dereferenceable(8) addrspacecast (ptr addrspace(1) @g1 to ptr)) #[[ATTR3]]
+; CHECK-NEXT:    call void @_Z12unknown_readPi(ptr nofree noundef nonnull readonly align 4 captures(none) dereferenceable(4) addrspacecast (ptr addrspace(3) @s1 to ptr)) #[[ATTR3]]
+; CHECK-NEXT:    call void @_Z11shared_readPi(ptr nofree noundef nonnull readonly align 4 captures(none) dereferenceable(4) addrspacecast (ptr addrspace(3) @s1 to ptr)) #[[ATTR3]]
+; CHECK-NEXT:    call void @_Z11shared_readPi(ptr nofree noundef nonnull readonly align 4 captures(none) dereferenceable(4) addrspacecast (ptr addrspace(3) @s2 to ptr)) #[[ATTR3]]
 ; CHECK-NEXT:    ret void
 ;
 entry:
diff --git a/llvm/test/Transforms/Attributor/align.ll b/llvm/test/Transforms/Attributor/align.ll
index 0a0cd415a2ab7..6bd1a2a65bb4a 100644
--- a/llvm/test/Transforms/Attributor/align.ll
+++ b/llvm/test/Transforms/Attributor/align.ll
@@ -348,18 +348,18 @@ define void @test8_helper() {
 ; TUNIT-NEXT:    [[PTR0:%.*]] = tail call ptr @unknown()
 ; TUNIT-NEXT:    [[PTR1:%.*]] = tail call align 4 ptr @unknown()
 ; TUNIT-NEXT:    [[PTR2:%.*]] = tail call align 8 ptr @unknown()
-; TUNIT-NEXT:    tail call void @test8(ptr noalias nocapture nofree readnone align 4 [[PTR1]], ptr noalias nocapture nofree readnone align 4 [[PTR1]], ptr noalias nocapture nofree readnone [[PTR0]]) #[[ATTR2:[0-9]+]]
-; TUNIT-NEXT:    tail call void @test8(ptr noalias nocapture nofree readnone align 8 [[PTR2]], ptr noalias nocapture nofree readnone align 4 [[PTR1]], ptr noalias nocapture nofree readnone align 4 [[PTR1]]) #[[ATTR2]]
-; TUNIT-NEXT:    tail call void @test8(ptr noalias nocapture nofree readnone align 8 [[PTR2]], ptr noalias nocapture nofree readnone align 4 [[PTR1]], ptr noalias nocapture nofree readnone align 4 [[PTR1]]) #[[ATTR2]]
+; TUNIT-NEXT:    tail call void @test8(ptr noalias nofree readnone align 4 captures(none) [[PTR1]], ptr noalias nofree readnone align 4 captures(none) [[PTR1]], ptr noalias nofree readnone captures(none) [[PTR0]]) #[[ATTR2:[0-9]+]]
+; TUNIT-NEXT:    tail call void @test8(ptr noalias nofree readnone align 8 captures(none) [[PTR2]], ptr noalias nofree readnone align 4 captures(none) [[PTR1]], ptr noalias nofree readnone align 4 captures(none) [[PTR1]]) #[[ATTR2]]
+; TUNIT-NEXT:    tail call void @test8(ptr noalias nofree readnone align 8 captures(none) [[PTR2]], ptr noalias nofree readnone align 4 captures(none) [[PTR1]], ptr noalias nofree readnone align 4 captures(none) [[PTR1]]) #[[ATTR2]]
 ; TUNIT-NEXT:    ret void
 ;
 ; CGSCC-LABEL: define {{[^@]+}}@test8_helper() {
 ; CGSCC-NEXT:    [[PTR0:%.*]] = tail call ptr @unknown()
 ; CGSCC-NEXT:    [[PTR1:%.*]] = tail call align 4 ptr @unknown()
 ; CGSCC-NEXT:    [[PTR2:%.*]] = tail call align 8 ptr @unknown()
-; CGSCC-NEXT:    tail call void @test8(ptr noalias nocapture nofree readnone align 4 [[PTR1]], ptr noalias nocapture nofree readnone align 4 [[PTR1]], ptr noalias nocapture nofree readnone [[PTR0]]) #[[ATTR3:[0-9]+]]
-; CGSCC-NEXT:    tail call void @test8(ptr noalias nocapture nofree readnone align 8 [[PTR2]], ptr noalias nocapture nofree readnone align 4 [[PTR1]], ptr noalias nocapture nofree readnone align 4 [[PTR1]]) #[[ATTR3]]
-; CGSCC-NEXT:    tail call void @test8(ptr noalias nocapture nofree readnone align 8 [[PTR2]], ptr noalias nocapture nofree readnone align 4 [[PTR1]], ptr noalias nocapture nofree readnone align 4 [[PTR1]]) #[[ATTR3]]
+; CGSCC-NEXT:    tail call void @test8(ptr noalias nofree readnone align 4 captures(none) [[PTR1]], ptr noalias nofree readnone align 4 captures(none) [[PTR1]], ptr noalias nofree readnone captures(none) [[PTR0]]) #[[ATTR3:[0-9]+]]
+; CGSCC-NEXT:    tail call void @test8(ptr noalias nofree readnone align 8 captures(none) [[PTR2]], ptr noalias nofree readnone align 4 captures(none) [[PTR1]], ptr noalias nofree readnone align 4 captures(none) [[PTR1]]) #[[ATTR3]]
+; CGSCC-NEXT:    tail call void @test8(ptr noalias nofree readnone align 8 captures(none) [[PTR2]], ptr noalias nofree readnone align 4 captures(none) [[PTR1]], ptr noalias nofree readnone align 4 captures(none) [[PTR1]]) #[[ATTR3]]
 ; CGSCC-NEXT:    ret void
 ;
   %ptr0 = tail call ptr @unknown()
@@ -376,18 +376,18 @@ declare void @user_i32_ptr(ptr nocapture readnone) nounwind
 define internal void @test8(ptr %a, ptr %b, ptr %c) {
 ; TUNIT: Function Attrs: nounwind
 ; TUNIT-LABEL: define {{[^@]+}}@test8
-; TUNIT-SAME: (ptr noalias nocapture nofree readnone align 4 [[A:%.*]], ptr noalias nocapture nofree readnone align 4 [[B:%.*]], ptr noalias nocapture nofree readnone [[C:%.*]]) #[[ATTR2]] {
-; TUNIT-NEXT:    call void @user_i32_ptr(ptr noalias nocapture nofree readnone align 4 [[A]]) #[[ATTR2]]
-; TUNIT-NEXT:    call void @user_i32_ptr(ptr noalias nocapture nofree readnone align 4 [[B]]) #[[ATTR2]]
-; TUNIT-NEXT:    call void @user_i32_ptr(ptr noalias nocapture nofree readnone [[C]]) #[[ATTR2]]
+; TUNIT-SAME: (ptr noalias nofree readnone align 4 captures(none) [[A:%.*]], ptr noalias nofree readnone align 4 captures(none) [[B:%.*]], ptr noalias nofree readnone captures(none) [[C:%.*]]) #[[ATTR2]] {
+; TUNIT-NEXT:    call void @user_i32_ptr(ptr noalias nofree readnone align 4 captures(none) [[A]]) #[[ATTR2]]
+; TUNIT-NEXT:    call void @user_i32_ptr(ptr noalias nofree readnone align 4 captures(none) [[B]]) #[[ATTR2]]
+; TUNIT-NEXT:    call void @user_i32_ptr(ptr noalias nofree readnone captures(none) [[C]]) #[[ATTR2]]
 ; TUNIT-NEXT:    ret void
 ;
 ; CGSCC: Function Attrs: nounwind
 ; CGSCC-LABEL: define {{[^@]+}}@test8
-; CGSCC-SAME: (ptr noalias nocapture nofree readnone align 4 [[A:%.*]], ptr noalias nocapture nofree readnone align 4 [[B:%.*]], ptr noalias nocapture nofree readnone [[C:%.*]]) #[[ATTR3]] {
-; CGSCC-NEXT:    call void @user_i32_ptr(ptr noalias nocapture nofree readnone align 4 [[A]]) #[[ATTR3]]
-; CGSCC-NEXT:    call void @user_i32_ptr(ptr noalias nocapture nofree readnone align 4 [[B]]) #[[ATTR3]]
-; CGSCC-NEXT:    call void @user_i32_ptr(ptr noalias nocapture nofree readnone [[C]]) #[[ATTR3]]
+; CGSCC-SAME: (ptr noalias nofree readnone align 4 captures(none) [[A:%.*]], ptr noalias nofree readnone align 4 captures(none) [[B:%.*]], ptr noalias nofree readnone captures(none) [[C:%.*]]) #[[ATTR3]] {
+; CGSCC-NEXT:    call void @user_i32_ptr(ptr noalias nofree readnone align 4 captures(none) [[A]]) #[[ATTR3]]
+; CGSCC-NEXT:    call void @user_i32_ptr(ptr noalias nofree readnone align 4 captures(none) [[B]]) #[[ATTR3]]
+; CGSCC-NEXT:    call void @user_i32_ptr(ptr noalias nofree readnone captures(none) [[C]]) #[[ATTR3]]
 ; CGSCC-NEXT:    ret void
 ;
   call void @user_i32_ptr(ptr %a)
@@ -537,13 +537,13 @@ e:
 define i64 @test11(ptr %p) {
 ; TUNIT: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: read)
 ; TUNIT-LABEL: define {{[^@]+}}@test11
-; TUNIT-SAME: (ptr nocapture nofree nonnull readonly align 8 dereferenceable(8) [[P:%.*]]) #[[ATTR4:[0-9]+]] {
+; TUNIT-SAME: (ptr nofree nonnull readonly align 8 captures(none) dereferenceable(8) [[P:%.*]]) #[[ATTR4:[0-9]+]] {
 ; TUNIT-NEXT:    [[RET:%.*]] = load i64, ptr [[P]], align 8
 ; TUNIT-NEXT:    ret i64 [[RET]]
 ;
 ; CGSCC: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: read)
 ; CGSCC-LABEL: define {{[^@]+}}@test11
-; CGSCC-SAME: (ptr nocapture nofree nonnull readonly align 8 dereferenceable(8) [[P:%.*]]) #[[ATTR5:[0-9]+]] {
+; CGSCC-SAME: (ptr nofree nonnull readonly align 8 captures(none) dereferenceable(8) [[P:%.*]]) #[[ATTR5:[0-9]+]] {
 ; CGSCC-NEXT:    [[RET:%.*]] = load i64, ptr [[P]], align 8
 ; CGSCC-NEXT:    ret i64 [[RET]]
 ;
@@ -559,7 +559,7 @@ define i64 @test11(ptr %p) {
 define i64 @test12-1(ptr align 4 %p) {
 ; TUNIT: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: read)
 ; TUNIT-LABEL: define {{[^@]+}}@test12-1
-; TUNIT-SAME: (ptr nocapture nofree readonly align 16 [[P:%.*]]) #[[ATTR4]] {
+; TUNIT-SAME: (ptr nofree readonly align 16 captures(none) [[P:%.*]]) #[[ATTR4]] {
 ; TUNIT-NEXT:    [[ARRAYIDX0:%.*]] = getelementptr i64, ptr [[P]], i64 1
 ; TUNIT-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr i64, ptr [[ARRAYIDX0]], i64 3
 ; TUNIT-NEXT:    [[RET:%.*]] = load i64, ptr [[ARRAYIDX1]], align 16
@@ -567,7 +567,7 @@ define i64 @test12-1(ptr align 4 %p) {
 ;
 ; CGSCC: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: read)
 ; CGSCC-LABEL: define {{[^@]+}}@test12-1
-; CGSCC-SAME: (ptr nocapture nofree readonly align 16 [[P:%.*]]) #[[ATTR5]] {
+; CGSCC-SAME: (ptr nofree readonly align 16 captures(none) [[P:%.*]]) #[[ATTR5]] {
 ; CGSCC-NEXT:    [[ARRAYIDX0:%.*]] = getelementptr i64, ptr [[P]], i64 1
 ; CGSCC-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr i64, ptr [[ARRAYIDX0]], i64 3
 ; CGSCC-NEXT:    [[RET:%.*]] = load i64, ptr [[ARRAYIDX1]], align 16
@@ -583,13 +583,13 @@ define i64 @test12-1(ptr align 4 %p) {
 define i64 @test12-2(ptr align 4 %p) {
 ; TUNIT: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: read)
 ; TUNIT-LABEL: define {{[^@]+}}@test12-2
-; TUNIT-SAME: (ptr nocapture nofree nonnull readonly align 16 dereferenceable(8) [[P:%.*]]) #[[ATTR4]] {
+; TUNIT-SAME: (ptr nofree nonnull readonly align 16 captures(none) dereferenceable(8) [[P:%.*]]) #[[ATTR4]] {
 ; TUNIT-NEXT:    [[RET:%.*]] = load i64, ptr [[P]], align 16
 ; TUNIT-NEXT:    ret i64 [[RET]]
 ;
 ; CGSCC: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: read)
 ; CGSCC-LABEL: define {{[^@]+}}@test12-2
-; CGSCC-SAME: (ptr nocapture nofree nonnull readonly align 16 dereferenceable(8) [[P:%.*]]) #[[ATTR5]] {
+; CGSCC-SAME: (ptr nofree nonnull readonly align 16 captures(none) dereferenceable(8) [[P:%.*]]) #[[ATTR5]] {
 ; CGSCC-NEXT:    [[RET:%.*]] = load i64, ptr [[P]], align 16
 ; CGSCC-NEXT:    ret i64 [[RET]]
 ;
@@ -602,7 +602,7 @@ define i64 @test12-2(ptr align 4 %p) {
 define void @test12-3(ptr align 4 %p) {
 ; TUNIT: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: write)
 ; TUNIT-LABEL: define {{[^@]+}}@test12-3
-; TUNIT-SAME: (ptr nocapture nofree writeonly align 16 [[P:%.*]]) #[[ATTR5:[0-9]+]] {
+; TUNIT-SAME: (ptr nofree writeonly align 16 captures(none) [[P:%.*]]) #[[ATTR5:[0-9]+]] {
 ; TUNIT-NEXT:    [[ARRAYIDX0:%.*]] = getelementptr i64, ptr [[P]], i64 1
 ; TUNIT-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr i64, ptr [[ARRAYIDX0]], i64 3
 ; TUNIT-NEXT:    store i64 0, ptr [[ARRAYIDX1]], align 16
@@ -610,7 +610,7 @@ define void @test12-3(ptr align 4 %p) {
 ;
 ; CGSCC: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: write)
 ; CGSCC-LABEL: define {{[^@]+}}@test12-3
-; CGSCC-SAME: (ptr nocapture nofree writeonly align 16 [[P:%.*]]) #[[ATTR6:[0-9]+]] {
+; CGSCC-SAME: (ptr nofree writeonly align 16 captures(none) [[P:%.*]]) #[[ATTR6:[0-9]+]] {
 ; CGSCC-NEXT:    [[ARRAYIDX0:%.*]] = getelementptr i64, ptr [[P]], i64 1
 ; CGSCC-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr i64, ptr [[ARRAYIDX0]], i64 3
 ; CGSCC-NEXT:    store i64 0, ptr [[ARRAYIDX1]], align 16
@@ -626,13 +626,13 @@ define void @test12-3(ptr align 4 %p) {
 define void @test12-4(ptr align 4 %p) {
 ; TUNIT: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: write)
 ; TUNIT-LABEL: define {{[^@]+}}@test12-4
-; TUNIT-SAME: (ptr nocapture nofree nonnull writeonly align 16 dereferenceable(8) [[P:%.*]]) #[[ATTR5]] {
+; TUNIT-SAME: (ptr nofree nonnull writeonly align 16 captures(none) dereferenceable(8) [[P:%.*]]) #[[ATTR5]] {
 ; TUNIT-NEXT:    store i64 0, ptr [[P]], align 16
 ; TUNIT-NEXT:    ret void
 ;
 ; CGSCC: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: write)
 ; CGSCC-LABEL: define {{[^@]+}}@test12-4
-; CGSCC-SAME: (ptr nocapture nofree nonnull writeonly align 16 dereferenceable(8) [[P:%.*]]) #[[ATTR6]] {
+; CGSCC-SAME: (ptr nofree nonnull writeonly align 16 captures(none) dereferenceable(8) [[P:%.*]]) #[[ATTR6]] {
 ; CGSCC-NEXT:    store i64 0, ptr [[P]], align 16
 ; CGSCC-NEXT:    ret void
 ;
@@ -688,7 +688,7 @@ define void @test12-6(ptr align 4 %p) {
 define void @test13(i1 %c, ptr align 32 %dst) #0 {
 ; TUNIT: Function Attrs: mustprogress nofree noinline norecurse nosync nounwind willreturn memory(argmem: write) uwtable
 ; TUNIT-LABEL: define {{[^@]+}}@test13
-; TUNIT-SAME: (i1 noundef [[C:%.*]], ptr nocapture nofree writeonly align 32 [[DST:%.*]]) #[[ATTR8:[0-9]+]] {
+; TUNIT-SAME: (i1 noundef [[C:%.*]], ptr nofree writeonly align 32 captures(none) [[DST:%.*]]) #[[ATTR8:[0-9]+]] {
 ; TUNIT-NEXT:    br i1 [[C]], label [[TRUEBB:%.*]], label [[FALSEBB:%.*]]
 ; TUNIT:       truebb:
 ; TUNIT-NEXT:    br label [[END:%.*]]
@@ -701,7 +701,7 @@ define void @test13(i1 %c, ptr align 32 %dst) #0 {
 ;
 ; CGSCC: Function Attrs: mustprogress nofree noinline norecurse nosync nounwind willreturn memory(argmem: write) uwtable
 ; CGSCC-LABEL: define {{[^@]+}}@test13
-; CGSCC-SAME: (i1 noundef [[C:%.*]], ptr nocapture nofree writeonly align 32 [[DST:%.*]]) #[[ATTR9:[0-9]+]] {
+; CGSCC-SAME: (i1 noundef [[C:%.*]], ptr nofree writeonly align 32 captures(none) [[DST:%.*]]) #[[ATTR9:[0-9]+]] {
 ; CGSCC-NEXT:    br i1 [[C]], label [[TRUEBB:%.*]], label [[FALSEBB:%.*]]
 ; CGSCC:       truebb:
 ; CGSCC-NEXT:    br label [[END:%.*]]
@@ -726,7 +726,7 @@ end:
 define void @test13-1(i1 %c, ptr align 32 %dst) {
 ; TUNIT: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(write)
 ; TUNIT-LABEL: define {{[^@]+}}@test13-1
-; TUNIT-SAME: (i1 noundef [[C:%.*]], ptr nocapture nofree writeonly align 32 [[DST:%.*]]) #[[ATTR9:[0-9]+]] {
+; TUNIT-SAME: (i1 noundef [[C:%.*]], ptr nofree writeonly align 32 captures(none) [[DST:%.*]]) #[[ATTR9:[0-9]+]] {
 ; TUNIT-NEXT:    br i1 [[C]], label [[TRUEBB:%.*]], label [[FALSEBB:%.*]]
 ; TUNIT:       truebb:
 ; TUNIT-NEXT:    br label [[END:%.*]]
@@ -739,7 +739,7 @@ define void @test13-1(i1 %c, ptr align 32 %dst) {
 ;
 ; CGSCC: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(write)
 ; CGSCC-LABEL: define {{[^@]+}}@test13-1
-; CGSCC-SAME: (i1 noundef [[C:%.*]], ptr nocapture nofree writeonly align 32 [[DST:%.*]]) #[[ATTR10:[0-9]+]] {
+; CGSCC-SAME: (i1 noundef [[C:%.*]], ptr nofree writeonly align 32 captures(none) [[DST:%.*]]) #[[ATTR10:[0-9]+]] {
 ; CGSCC-NEXT:    br i1 [[C]], label [[TRUEBB:%.*]], label [[FALSEBB:%.*]]
 ; CGSCC:       truebb:
 ; CGSCC-NEXT:    br label [[END:%.*]]
@@ -764,7 +764,7 @@ end:
 define void @test13-2(i1 %c, ptr align 32 %dst) {
 ; TUNIT: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(write)
 ; TUNIT-LABEL: define {{[^@]+}}@test13-2
-; TUNIT-SAME: (i1 noundef [[C:%.*]], ptr nocapture nofree writeonly align 32 [[DST:%.*]]) #[[ATTR9]] {
+; TUNIT-SAME: (i1 noundef [[C:%.*]], ptr nofree writeonly align 32 captures(none) [[DST:%.*]]) #[[ATTR9]] {
 ; TUNIT-NEXT:    br i1 [[C]], label [[TRUEBB:%.*]], label [[FALSEBB:%.*]]
 ; TUNIT:       truebb:
 ; TUNIT-NEXT:    br label [[END:%.*]]
@@ -777,7 +777,7 @@ define void @test13-2(i1 %c, ptr align 32 %dst) {
 ;
 ; CGSCC: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(write)
 ; CGSCC-LABEL: define {{[^@]+}}@test13-2
-; CGSCC-SAME: (i1 noundef [[C:%.*]], ptr nocapture nofree writeonly align 32 [[DST:%.*]]) #[[ATTR10]] {
+; CGSCC-SAME: (i1 noundef [[C:%.*]], ptr nofree writeonly align 32 captures(none) [[DST:%.*]]) #[[ATTR10]] {
 ; CGSCC-NEXT:    br i1 [[C]], label [[TRUEBB:%.*]], label [[FALSEBB:%.*]]
 ; CGSCC:       truebb:
 ; CGSCC-NEXT:    br label [[END:%.*]]
@@ -802,7 +802,7 @@ end:
 define void @test13-3(i1 %c, ptr align 32 %dst) {
 ; TUNIT: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(write)
 ; TUNIT-LABEL: define {{[^@]+}}@test13-3
-; TUNIT-SAME: (i1 noundef [[C:%.*]], ptr nocapture nofree writeonly align 32 [[DST:%.*]]) #[[ATTR9]] {
+; TUNIT-SAME: (i1 noundef [[C:%.*]], ptr nofree writeonly align 32 captures(none) [[DST:%.*]]) #[[ATTR9]] {
 ; TUNIT-NEXT:    br i1 [[C]], label [[TRUEBB:%.*]], label [[FALSEBB:%.*]]
 ; TUNIT:       truebb:
 ; TUNIT-NEXT:    br label [[END:%.*]]
@@ -815,7 +815,7 @@ define void @test13-3(i1 %c, ptr align 32 %dst) {
 ;
 ; CGSCC: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(write)
 ; CGSCC-LABEL: define {{[^@]+}}@test13-3
-; CGSCC-SAME: (i1 noundef [[C:%.*]], ptr nocapture nofree writeonly align 32 [[DST:%.*]]) #[[ATTR10]] {
+; CGSCC-SAME: (i1 noundef [[C:%.*]], ptr nofree writeonly align 32 captures(none) [[DST:%.*]]) #[[ATTR10]] {
 ; CGSCC-NEXT:    br i1 [[C]], label [[TRUEBB:%.*]], label [[FALSEBB:%.*]]
 ; CGSCC:       truebb:
 ; CGSCC-NEXT:    br label [[END:%.*]]
@@ -875,13 +875,13 @@ define ptr @int2ptr(i64 %i) {
 define void @aligned_store(ptr %Value, ptr %Ptr) {
 ; TUNIT: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: write)
 ; TUNIT-LABEL: define {{[^@]+}}@aligned_store
-; TUNIT-SAME: (ptr nofree writeonly [[VALUE:%.*]], ptr nocapture nofree noundef nonnull writeonly align 32 dereferenceable(8) [[PTR:%.*]]) #[[ATTR5]] {
+; TUNIT-SAME: (ptr nofree writeonly [[VALUE:%.*]], ptr nofree noundef nonnull writeonly align 32 captures(none) dereferenceable(8) [[PTR:%.*]]) #[[ATTR5]] {
 ; TUNIT-NEXT:    store ptr [[VALUE]], ptr [[PTR]], align 32
 ; TUNIT-NEXT:    ret void
 ;
 ; CGSCC: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: write)
 ; CGSCC-LABEL: define {{[^@]+}}@aligned_store
-; CGSCC-SAME: (ptr nofree writeonly [[VALUE:%.*]], ptr nocapture nofree noundef nonnull writeonly align 32 dereferenceable(8) [[PTR:%.*]]) #[[ATTR6]] {
+; CGSCC-SAME: (ptr nofree writeonly [[VALUE:%.*]], ptr nofree noundef nonnull writeonly align 32 captures(none) dereferenceable(8) [[PTR:%.*]]) #[[ATTR6]] {
 ; CGSCC-NEXT:    store ptr [[VALUE]], ptr [[PTR]], align 32
 ; CGSCC-NEXT:    ret void
 ;
@@ -905,13 +905,13 @@ define void @align_store_after_bc(ptr align 2048 %arg) {
 ;
 ; TUNIT: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: write)
 ; TUNIT-LABEL: define {{[^@]+}}@align_store_after_bc
-; TUNIT-SAME: (ptr nocapture nofree nonnull writeonly align 2048 dereferenceable(1) [[ARG:%.*]]) #[[ATTR5]] {
+; TUNIT-SAME: (ptr nofree nonnull writeonly align 2048 captures(none) dereferenceable(1) [[ARG:%.*]]) #[[ATTR5]] {
 ; TUNIT-NEXT:    store i8 0, ptr [[ARG]], align 2048
 ; TUNIT-NEXT:    ret void
 ;
 ; CGSCC: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: write)
 ; CGSCC-LABEL: define {{[^@]+}}@align_store_after_bc
-; CGSCC-SAME: (ptr nocapture nofree nonnull writeonly align 2048 dereferenceable(1) [[ARG:%.*]]) #[[ATTR6]] {
+; CGSCC-SAME: (ptr nofree nonnull writeonly align 2048 captures(none) dereferenceable(1) [[ARG:%.*]]) #[[ATTR6]] {
 ; CGSCC-NEXT:    store i8 0, ptr [[ARG]], align 2048
 ; CGSCC-NEXT:    ret void
 ;
@@ -926,13 +926,13 @@ define void @align_store_after_bc(ptr align 2048 %arg) {
 define i32 @musttail_callee_1(ptr %p) {
 ; TUNIT: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: read)
 ; TUNIT-LABEL: define {{[^@]+}}@musttail_callee_1
-; TUNIT-SAME: (ptr nocapture nofree noundef nonnull readonly dereferenceable(4) [[P:%.*]]) #[[ATTR4]] {
+; TUNIT-SAME: (ptr nofree noundef nonnull readonly captures(none) dereferenceable(4) [[P:%.*]]) #[[ATTR4]] {
 ; TUNIT-NEXT:    [[V:%.*]] = load i32, ptr [[P]], align 32
 ; TUNIT-NEXT:    ret i32 [[V]]
 ;
 ; CGSCC: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: read)
 ; CGSCC-LABEL: define {{[^@]+}}@musttail_callee_1
-; CGSCC-SAME: (ptr nocapture nofree noundef nonnull readonly dereferenceable(4) [[P:%.*]]) #[[ATTR5]] {
+; CGSCC-SAME: (ptr nofree noundef nonnull readonly captures(none) dereferenceable(4) [[P:%.*]]) #[[ATTR5]] {
 ; CGSCC-NEXT:    [[V:%.*]] = load i32, ptr [[P]], align 32
 ; CGSCC-NEXT:    ret i32 [[V]]
 ;
@@ -942,22 +942,22 @@ define i32 @musttail_callee_1(ptr %p) {
 define i32 @musttail_caller_1(ptr %p) {
 ; TUNIT: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(read)
 ; TUNIT-LABEL: define {{[^@]+}}@musttail_caller_1
-; TUNIT-SAME: (ptr nocapture nofree readonly [[P:%.*]]) #[[ATTR11:[0-9]+]] {
+; TUNIT-SAME: (ptr nofree readonly captures(none) [[P:%.*]]) #[[ATTR11:[0-9]+]] {
 ; TUNIT-NEXT:    [[C:%.*]] = load i1, ptr @cnd, align 1
 ; TUNIT-NEXT:    br i1 [[C]], label [[MT:%.*]], label [[EXIT:%.*]]
 ; TUNIT:       mt:
-; TUNIT-NEXT:    [[V:%.*]] = musttail call i32 @musttail_callee_1(ptr nocapture nofree noundef readonly [[P]]) #[[ATTR14:[0-9]+]]
+; TUNIT-NEXT:    [[V:%.*]] = musttail call i32 @musttail_callee_1(ptr nofree noundef readonly captures(none) [[P]]) #[[ATTR14:[0-9]+]]
 ; TUNIT-NEXT:    ret i32 [[V]]
 ; TUNIT:       exit:
 ; TUNIT-NEXT:    ret i32 0
 ;
 ; CGSCC: Function Attrs: mustprogress nofree nosync nounwind willreturn memory(read)
 ; CGSCC-LABEL: define {{[^@]+}}@musttail_caller_1
-; CGSCC-SAME: (ptr nocapture nofree readonly [[P:%.*]]) #[[ATTR12:[0-9]+]] {
+; CGSCC-SAME: (ptr nofree readonly captures(none) [[P:%.*]]) #[[ATTR12:[0-9]+]] {
 ; CGSCC-NEXT:    [[C:%.*]] = load i1, ptr @cnd, align 1
 ; CGSCC-NEXT:    br i1 [[C]], label [[MT:%.*]], label [[EXIT:%.*]]
 ; CGSCC:       mt:
-; CGSCC-NEXT:    [[V:%.*]] = musttail call i32 @musttail_callee_1(ptr nocapture nofree noundef nonnull readonly dereferenceable(4) [[P]]) #[[ATTR17:[0-9]+]]
+; CGSCC-NEXT:    [[V:%.*]] = musttail call i32 @musttail_callee_1(ptr nofree noundef nonnull readonly captures(none) dereferenceable(4) [[P]]) #[[ATTR17:[0-9]+]]
 ; CGSCC-NEXT:    ret i32 [[V]]
 ; CGSCC:       exit:
 ; CGSCC-NEXT:    ret i32 0
@@ -985,7 +985,7 @@ define ptr @checkAndAdvance(ptr align(16) %p) {
 ; TUNIT-NEXT:    br label [[RETURN]]
 ; TUNIT:       return:
 ; TUNIT-NEXT:    [[RETVAL_0:%.*]] = phi ptr [ [[ADD_PTR]], [[IF_THEN]] ], [ [[P]], [[ENTRY:%.*]] ]
-; TUNIT-NEXT:    call void @user_i32_ptr(ptr noalias nocapture nofree nonnull readnone align 16 [[RETVAL_0]]) #[[ATTR2]]
+; TUNIT-NEXT:    call void @user_i32_ptr(ptr noalias nofree nonnull readnone align 16 captures(none) [[RETVAL_0]]) #[[ATTR2]]
 ; TUNIT-NEXT:    ret ptr [[RETVAL_0]]
 ;
 ; CGSCC: Function Attrs: nounwind
@@ -1001,7 +1001,7 @@ define ptr @checkAndAdvance(ptr align(16) %p) {
 ; CGSCC-NEXT:    br label [[RETURN]]
 ; CGSCC:       return:
 ; CGSCC-NEXT:    [[RETVAL_0:%.*]] = phi ptr [ [[ADD_PTR]], [[IF_THEN]] ], [ [[P]], [[ENTRY:%.*]] ]
-; CGSCC-NEXT:    call void @user_i32_ptr(ptr noalias nocapture nofree nonnull readnone align 16 [[RETVAL_0]]) #[[ATTR3]]
+; CGSCC-NEXT:    call void @user_i32_ptr(ptr noalias nofree nonnull readnone align 16 captures(none) [[RETVAL_0]]) #[[ATTR3]]
 ; CGSCC-NEXT:    ret ptr [[RETVAL_0]]
 ;
 entry:
@@ -1117,7 +1117,7 @@ entry:
 define i64 @infer_align_atomicrmw(ptr align 4 %p) {
 ; TUNIT: Function Attrs: mustprogress nofree norecurse nounwind willreturn memory(argmem: readwrite)
 ; TUNIT-LABEL: define {{[^@]+}}@infer_align_atomicrmw
-; TUNIT-SAME: (ptr nocapture nofree align 16 [[P:%.*]]) #[[ATTR12:[0-9]+]] {
+; TUNIT-SAME: (ptr nofree align 16 captures(none) [[P:%.*]]) #[[ATTR12:[0-9]+]] {
 ; TUNIT-NEXT:    [[ARRAYIDX0:%.*]] = getelementptr i64, ptr [[P]], i64 1
 ; TUNIT-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr i64, ptr [[ARRAYIDX0]], i64 3
 ; TUNIT-NEXT:    [[RET:%.*]] = atomicrmw add ptr [[ARRAYIDX1]], i64 4 seq_cst, align 16
@@ -1125,7 +1125,7 @@ define i64 @infer_align_atomicrmw(ptr align 4 %p) {
 ;
 ; CGSCC: Function Attrs: mustprogress nofree norecurse nounwind willreturn memory(argmem: readwrite)
 ; CGSCC-LABEL: define {{[^@]+}}@infer_align_atomicrmw
-; CGSCC-SAME: (ptr nocapture nofree align 16 [[P:%.*]]) #[[ATTR14:[0-9]+]] {
+; CGSCC-SAME: (ptr nofree align 16 captures(none) [[P:%.*]]) #[[ATTR14:[0-9]+]] {
 ; CGSCC-NEXT:    [[ARRAYIDX0:%.*]] = getelementptr i64, ptr [[P]], i64 1
 ; CGSCC-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr i64, ptr [[ARRAYIDX0]], i64 3
 ; CGSCC-NEXT:    [[RET:%.*]] = atomicrmw add ptr [[ARRAYIDX1]], i64 4 seq_cst, align 16
@@ -1140,7 +1140,7 @@ define i64 @infer_align_atomicrmw(ptr align 4 %p) {
 define ptr @infer_align_atomicrmw_ptr(ptr align 4 %p, ptr %val) {
 ; TUNIT: Function Attrs: mustprogress nofree norecurse nounwind willreturn memory(argmem: readwrite)
 ; TUNIT-LABEL: define {{[^@]+}}@infer_align_atomicrmw_ptr
-; TUNIT-SAME: (ptr nocapture nofree align 16 [[P:%.*]], ptr nofree [[VAL:%.*]]) #[[ATTR12]] {
+; TUNIT-SAME: (ptr nofree align 16 captures(none) [[P:%.*]], ptr nofree [[VAL:%.*]]) #[[ATTR12]] {
 ; TUNIT-NEXT:    [[ARRAYIDX0:%.*]] = getelementptr i64, ptr [[P]], i64 1
 ; TUNIT-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr i64, ptr [[ARRAYIDX0]], i64 3
 ; TUNIT-NEXT:    [[RET:%.*]] = atomicrmw xchg ptr [[ARRAYIDX1]], ptr [[VAL]] seq_cst, align 16
@@ -1148,7 +1148,7 @@ define ptr @infer_align_atomicrmw_ptr(ptr align 4 %p, ptr %val) {
 ;
 ; CGSCC: Function Attrs: mustprogress nofree norecurse nounwind willreturn memory(argmem: readwrite)
 ; CGSCC-LABEL: define {{[^@]+}}@infer_align_atomicrmw_ptr
-; CGSCC-SAME: (ptr nocapture nofree align 16 [[P:%.*]], ptr nofree [[VAL:%.*]]) #[[ATTR14]] {
+; CGSCC-SAME: (ptr nofree align 16 captures(none) [[P:%.*]], ptr nofree [[VAL:%.*]]) #[[ATTR14]] {
 ; CGSCC-NEXT:    [[ARRAYIDX0:%.*]] = getelementptr i64, ptr [[P]], i64 1
 ; CGSCC-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr i64, ptr [[ARRAYIDX0]], i64 3
 ; CGSCC-NEXT:    [[RET:%.*]] = atomicrmw xchg ptr [[ARRAYIDX1]], ptr [[VAL]] seq_cst, align 16
@@ -1163,7 +1163,7 @@ define ptr @infer_align_atomicrmw_ptr(ptr align 4 %p, ptr %val) {
 define i64 @infer_align_cmpxchg(ptr align 4 %p) {
 ; TUNIT: Function Attrs: mustprogress nofree norecurse nounwind willreturn memory(argmem: readwrite)
 ; TUNIT-LABEL: define {{[^@]+}}@infer_align_cmpxchg
-; TUNIT-SAME: (ptr nocapture nofree align 16 [[P:%.*]]) #[[ATTR12]] {
+; TUNIT-SAME: (ptr nofree align 16 captures(none) [[P:%.*]]) #[[ATTR12]] {
 ; TUNIT-NEXT:    [[ARRAYIDX0:%.*]] = getelementptr i64, ptr [[P]], i64 1
 ; TUNIT-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr i64, ptr [[ARRAYIDX0]], i64 3
 ; TUNIT-NEXT:    [[CMPX:%.*]] = cmpxchg ptr [[ARRAYIDX1]], i64 4, i64 1 seq_cst seq_cst, align 16
@@ -1172,7 +1172,7 @@ define i64 @infer_align_cmpxchg(ptr align 4 %p) {
 ;
 ; CGSCC: Function Attrs: mustprogress nofree norecurse nounwind willreturn memory(argmem: readwrite)
 ; CGSCC-LABEL: define {{[^@]+}}@infer_align_cmpxchg
-; CGSCC-SAME: (ptr nocapture nofree align 16 [[P:%.*]]) #[[ATTR14]] {
+; CGSCC-SAME: (ptr nofree align 16 captures(none) [[P:%.*]]) #[[ATTR14]] {
 ; CGSCC-NEXT:    [[ARRAYIDX0:%.*]] = getelementptr i64, ptr [[P]], i64 1
 ; CGSCC-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr i64, ptr [[ARRAYIDX0]], i64 3
 ; CGSCC-NEXT:    [[CMPX:%.*]] = cmpxchg ptr [[ARRAYIDX1]], i64 4, i64 1 seq_cst seq_cst, align 16
@@ -1189,7 +1189,7 @@ define i64 @infer_align_cmpxchg(ptr align 4 %p) {
 define ptr @infer_align_cmpxchg_ptr(ptr align 4 %p, ptr %cmp0, ptr %cmp1) {
 ; TUNIT: Function Attrs: mustprogress nofree norecurse nounwind willreturn memory(argmem: readwrite)
 ; TUNIT-LABEL: define {{[^@]+}}@infer_align_cmpxchg_ptr
-; TUNIT-SAME: (ptr nocapture nofree align 16 [[P:%.*]], ptr nofree [[CMP0:%.*]], ptr nofree [[CMP1:%.*]]) #[[ATTR12]] {
+; TUNIT-SAME: (ptr nofree align 16 captures(none) [[P:%.*]], ptr nofree [[CMP0:%.*]], ptr nofree [[CMP1:%.*]]) #[[ATTR12]] {
 ; TUNIT-NEXT:    [[ARRAYIDX0:%.*]] = getelementptr i64, ptr [[P]], i64 1
 ; TUNIT-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr i64, ptr [[ARRAYIDX0]], i64 3
 ; TUNIT-NEXT:    [[CMPX:%.*]] = cmpxchg ptr [[ARRAYIDX1]], ptr [[CMP0]], ptr [[CMP1]] seq_cst seq_cst, align 16
@@ -1198,7 +1198,7 @@ define ptr @infer_align_cmpxchg_ptr(ptr align 4 %p, ptr %cmp0, ptr %cmp1) {
 ;
 ; CGSCC: Function Attrs: mustprogress nofree norecurse nounwind willreturn memory(argmem: readwrite)
 ; CGSCC-LABEL: define {{[^@]+}}@infer_align_cmpxchg_ptr
-; CGSCC-SAME: (ptr nocapture nofree align 16 [[P:%.*]], ptr nofree [[CMP0:%.*]], ptr nofree [[CMP1:%.*]]) #[[ATTR14]] {
+; CGSCC-SAME: (ptr nofree align 16 captures(none) [[P:%.*]], ptr nofree [[CMP0:%.*]], ptr nofree [[CMP1:%.*]]) #[[ATTR14]] {
 ; CGSCC-NEXT:    [[ARRAYIDX0:%.*]] = getelementptr i64, ptr [[P]], i64 1
 ; CGSCC-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr i64, ptr [[ARRAYIDX0]], i64 3
 ; CGSCC-NEXT:    [[CMPX:%.*]] = cmpxchg ptr [[ARRAYIDX1]], ptr [[CMP0]], ptr [[CMP1]] seq_cst seq_cst, align 16
diff --git a/llvm/test/Transforms/Attributor/allocator.ll b/llvm/test/Transforms/Attributor/allocator.ll
index f2d9ecd1d8fa4..4f74a59176cbb 100644
--- a/llvm/test/Transforms/Attributor/allocator.ll
+++ b/llvm/test/Transforms/Attributor/allocator.ll
@@ -47,7 +47,7 @@ entry:
 ; Function Attrs: noinline nounwind uwtable
 define dso_local void @positive_malloc_1(ptr noundef %val) #0 {
 ; CHECK-LABEL: define dso_local void @positive_malloc_1
-; CHECK-SAME: (ptr nocapture nofree noundef readonly [[VAL:%.*]]) {
+; CHECK-SAME: (ptr nofree noundef readonly captures(none) [[VAL:%.*]]) {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[VAL_ADDR:%.*]] = alloca ptr, align 8
 ; CHECK-NEXT:    [[F:%.*]] = alloca ptr, align 8
@@ -84,7 +84,7 @@ entry:
 ; Function Attrs: noinline nounwind uwtable
 define dso_local void @positive_malloc_2(ptr noundef %val) #0 {
 ; CHECK-LABEL: define dso_local void @positive_malloc_2
-; CHECK-SAME: (ptr nocapture nofree noundef readonly [[VAL:%.*]]) {
+; CHECK-SAME: (ptr nofree noundef readonly captures(none) [[VAL:%.*]]) {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[VAL_ADDR:%.*]] = alloca ptr, align 8
 ; CHECK-NEXT:    [[F:%.*]] = alloca ptr, align 8
@@ -229,7 +229,7 @@ entry:
 ; Should the optimization reduce the allocation size regardless? Based on AAPointerInfo.
 define dso_local void @baz(ptr noundef %val, i32 noundef %arrayLength) #0 {
 ; CHECK-LABEL: define dso_local void @baz
-; CHECK-SAME: (ptr nocapture nofree noundef readonly [[VAL:%.*]], i32 noundef [[ARRAYLENGTH:%.*]]) {
+; CHECK-SAME: (ptr nofree noundef readonly captures(none) [[VAL:%.*]], i32 noundef [[ARRAYLENGTH:%.*]]) {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[VAL_ADDR:%.*]] = alloca ptr, align 8
 ; CHECK-NEXT:    [[ARRAYLENGTH_ADDR:%.*]] = alloca i32, align 4
@@ -427,7 +427,7 @@ define dso_local void @pthread_test(){
 ; TUNIT-NEXT:    [[THREAD:%.*]] = alloca i64, align 8
 ; TUNIT-NEXT:    [[CALL1:%.*]] = call i32 @pthread_create(ptr noundef nonnull align 8 dereferenceable(8) [[THREAD]], ptr noundef align 4294967296 null, ptr noundef nonnull @pthread_allocation_should_remain_same, ptr noundef nonnull align 8 dereferenceable(1) [[ARG1]])
 ; TUNIT-NEXT:    [[F1:%.*]] = alloca i8, i32 4, align 4
-; TUNIT-NEXT:    [[CALL2:%.*]] = call i32 @pthread_create(ptr noundef nonnull align 8 dereferenceable(8) [[THREAD]], ptr noundef align 4294967296 null, ptr noundef nonnull @pthread_allocation_should_be_reduced, ptr noalias nocapture nofree nonnull readnone align 4 dereferenceable(12) undef)
+; TUNIT-NEXT:    [[CALL2:%.*]] = call i32 @pthread_create(ptr noundef nonnull align 8 dereferenceable(8) [[THREAD]], ptr noundef align 4294967296 null, ptr noundef nonnull @pthread_allocation_should_be_reduced, ptr noalias nofree nonnull readnone align 4 captures(none) dereferenceable(12) undef)
 ; TUNIT-NEXT:    [[F2:%.*]] = alloca [[STRUCT_FOO:%.*]], align 4
 ; TUNIT-NEXT:    [[CALL3:%.*]] = call i32 @pthread_create(ptr noundef nonnull align 8 dereferenceable(8) [[THREAD]], ptr noundef align 4294967296 null, ptr noundef nonnull @pthread_check_captured_pointer, ptr noundef nonnull align 4 dereferenceable(12) [[F2]])
 ; TUNIT-NEXT:    ret void
@@ -437,7 +437,7 @@ define dso_local void @pthread_test(){
 ; CGSCC-NEXT:    [[THREAD:%.*]] = alloca i64, align 8
 ; CGSCC-NEXT:    [[CALL1:%.*]] = call i32 @pthread_create(ptr noundef nonnull align 8 dereferenceable(8) [[THREAD]], ptr noundef align 4294967296 null, ptr noundef nonnull @pthread_allocation_should_remain_same, ptr noundef nonnull align 8 dereferenceable(1) [[ARG1]])
 ; CGSCC-NEXT:    [[F:%.*]] = alloca [[STRUCT_FOO:%.*]], align 4
-; CGSCC-NEXT:    [[CALL2:%.*]] = call i32 @pthread_create(ptr noundef nonnull align 8 dereferenceable(8) [[THREAD]], ptr noundef align 4294967296 null, ptr noundef nonnull @pthread_allocation_should_be_reduced, ptr noalias nocapture nofree noundef nonnull readonly align 4 dereferenceable(12) [[F]])
+; CGSCC-NEXT:    [[CALL2:%.*]] = call i32 @pthread_create(ptr noundef nonnull align 8 dereferenceable(8) [[THREAD]], ptr noundef align 4294967296 null, ptr noundef nonnull @pthread_allocation_should_be_reduced, ptr noalias nofree noundef nonnull readonly align 4 captures(none) dereferenceable(12) [[F]])
 ; CGSCC-NEXT:    [[F2:%.*]] = alloca [[STRUCT_FOO]], align 4
 ; CGSCC-NEXT:    [[CALL3:%.*]] = call i32 @pthread_create(ptr noundef nonnull align 8 dereferenceable(8) [[THREAD]], ptr noundef align 4294967296 null, ptr noundef nonnull @pthread_check_captured_pointer, ptr noundef nonnull align 4 dereferenceable(12) [[F2]])
 ; CGSCC-NEXT:    ret void
@@ -467,13 +467,13 @@ entry:
 define internal void @pthread_allocation_should_be_reduced(ptr %arg) {
 ;
 ; TUNIT-LABEL: define internal void @pthread_allocation_should_be_reduced
-; TUNIT-SAME: (ptr noalias nocapture nofree nonnull readnone align 4 dereferenceable(12) [[ARG:%.*]]) {
+; TUNIT-SAME: (ptr noalias nofree nonnull readnone align 4 captures(none) dereferenceable(12) [[ARG:%.*]]) {
 ; TUNIT-NEXT:  entry:
 ; TUNIT-NEXT:    [[CALL:%.*]] = call i32 (ptr, ...) @printf(ptr noundef nonnull dereferenceable(17) @.str, i32 undef)
 ; TUNIT-NEXT:    ret void
 ;
 ; CGSCC-LABEL: define internal void @pthread_allocation_should_be_reduced
-; CGSCC-SAME: (ptr noalias nocapture nofree noundef nonnull readonly align 4 dereferenceable(12) [[ARG:%.*]]) {
+; CGSCC-SAME: (ptr noalias nofree noundef nonnull readonly align 4 captures(none) dereferenceable(12) [[ARG:%.*]]) {
 ; CGSCC-NEXT:  entry:
 ; CGSCC-NEXT:    [[TMP0:%.*]] = load i32, ptr [[ARG]], align 4
 ; CGSCC-NEXT:    [[CALL:%.*]] = call i32 (ptr, ...) @printf(ptr noundef nonnull dereferenceable(17) @.str, i32 noundef [[TMP0]])
diff --git a/llvm/test/Transforms/Attributor/call-simplify-pointer-info.ll b/llvm/test/Transforms/Attributor/call-simplify-pointer-info.ll
index 9162f9c251894..1d435815d89e3 100644
--- a/llvm/test/Transforms/Attributor/call-simplify-pointer-info.ll
+++ b/llvm/test/Transforms/Attributor/call-simplify-pointer-info.ll
@@ -6,7 +6,7 @@
 define internal i8 @read_arg(ptr %p) {
 ; CGSCC: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: read)
 ; CGSCC-LABEL: define {{[^@]+}}@read_arg
-; CGSCC-SAME: (ptr nocapture nofree noundef nonnull readonly dereferenceable(1022) [[P:%.*]]) #[[ATTR0:[0-9]+]] {
+; CGSCC-SAME: (ptr nofree noundef nonnull readonly captures(none) dereferenceable(1022) [[P:%.*]]) #[[ATTR0:[0-9]+]] {
 ; CGSCC-NEXT:  entry:
 ; CGSCC-NEXT:    [[L:%.*]] = load i8, ptr [[P]], align 1
 ; CGSCC-NEXT:    ret i8 [[L]]
@@ -19,7 +19,7 @@ entry:
 define internal i8 @read_arg_index(ptr %p, i64 %index) {
 ; CGSCC: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: read)
 ; CGSCC-LABEL: define {{[^@]+}}@read_arg_index
-; CGSCC-SAME: (ptr nocapture nofree noundef nonnull readonly align 16 dereferenceable(1024) [[P:%.*]]) #[[ATTR0]] {
+; CGSCC-SAME: (ptr nofree noundef nonnull readonly align 16 captures(none) dereferenceable(1024) [[P:%.*]]) #[[ATTR0]] {
 ; CGSCC-NEXT:  entry:
 ; CGSCC-NEXT:    [[G:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 2
 ; CGSCC-NEXT:    [[L:%.*]] = load i8, ptr [[G]], align 1
@@ -47,7 +47,7 @@ define i8 @call_simplifiable_1() {
 ; CGSCC-NEXT:    [[BYTES:%.*]] = alloca [1024 x i8], align 16
 ; CGSCC-NEXT:    [[I0:%.*]] = getelementptr inbounds [1024 x i8], ptr [[BYTES]], i64 0, i64 2
 ; CGSCC-NEXT:    store i8 2, ptr [[I0]], align 2
-; CGSCC-NEXT:    [[R:%.*]] = call i8 @read_arg(ptr nocapture nofree noundef nonnull readonly align 2 dereferenceable(1022) [[I0]]) #[[ATTR3:[0-9]+]]
+; CGSCC-NEXT:    [[R:%.*]] = call i8 @read_arg(ptr nofree noundef nonnull readonly align 2 captures(none) dereferenceable(1022) [[I0]]) #[[ATTR3:[0-9]+]]
 ; CGSCC-NEXT:    ret i8 [[R]]
 ;
 entry:
@@ -63,7 +63,7 @@ entry:
 define internal i8 @read_arg_1(ptr %p) {
 ; CGSCC: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: read)
 ; CGSCC-LABEL: define {{[^@]+}}@read_arg_1
-; CGSCC-SAME: (ptr nocapture nofree noundef nonnull readonly dereferenceable(1) [[P:%.*]]) #[[ATTR0]] {
+; CGSCC-SAME: (ptr nofree noundef nonnull readonly captures(none) dereferenceable(1) [[P:%.*]]) #[[ATTR0]] {
 ; CGSCC-NEXT:  entry:
 ; CGSCC-NEXT:    [[L:%.*]] = load i8, ptr [[P]], align 1
 ; CGSCC-NEXT:    ret i8 [[L]]
@@ -76,9 +76,9 @@ entry:
 define internal i8 @sum_two_same_loads(ptr %p) {
 ; CGSCC: Function Attrs: mustprogress nofree nosync nounwind willreturn memory(argmem: read)
 ; CGSCC-LABEL: define {{[^@]+}}@sum_two_same_loads
-; CGSCC-SAME: (ptr nocapture nofree noundef nonnull readonly dereferenceable(1022) [[P:%.*]]) #[[ATTR2:[0-9]+]] {
-; CGSCC-NEXT:    [[X:%.*]] = call i8 @read_arg_1(ptr nocapture nofree noundef nonnull readonly dereferenceable(1022) [[P]]) #[[ATTR3]]
-; CGSCC-NEXT:    [[Y:%.*]] = call i8 @read_arg_1(ptr nocapture nofree noundef nonnull readonly dereferenceable(1022) [[P]]) #[[ATTR3]]
+; CGSCC-SAME: (ptr nofree noundef nonnull readonly captures(none) dereferenceable(1022) [[P:%.*]]) #[[ATTR2:[0-9]+]] {
+; CGSCC-NEXT:    [[X:%.*]] = call i8 @read_arg_1(ptr nofree noundef nonnull readonly captures(none) dereferenceable(1022) [[P]]) #[[ATTR3]]
+; CGSCC-NEXT:    [[Y:%.*]] = call i8 @read_arg_1(ptr nofree noundef nonnull readonly captures(none) dereferenceable(1022) [[P]]) #[[ATTR3]]
 ; CGSCC-NEXT:    [[Z:%.*]] = add nsw i8 [[X]], [[Y]]
 ; CGSCC-NEXT:    ret i8 [[Z]]
 ;
@@ -107,7 +107,7 @@ define i8 @call_simplifiable_2() {
 ; CGSCC-NEXT:    store i8 2, ptr [[I0]], align 2
 ; CGSCC-NEXT:    [[I1:%.*]] = getelementptr inbounds [1024 x i8], ptr [[BYTES]], i64 0, i64 3
 ; CGSCC-NEXT:    store i8 3, ptr [[I1]], align 1
-; CGSCC-NEXT:    [[R:%.*]] = call i8 @sum_two_same_loads(ptr nocapture nofree noundef nonnull readonly align 2 dereferenceable(1022) [[I0]]) #[[ATTR3]]
+; CGSCC-NEXT:    [[R:%.*]] = call i8 @sum_two_same_loads(ptr nofree noundef nonnull readonly align 2 captures(none) dereferenceable(1022) [[I0]]) #[[ATTR3]]
 ; CGSCC-NEXT:    ret i8 [[R]]
 ;
 entry:
@@ -136,7 +136,7 @@ define i8 @call_simplifiable_3() {
 ; CGSCC-NEXT:    [[BYTES:%.*]] = alloca [1024 x i8], align 16
 ; CGSCC-NEXT:    [[I2:%.*]] = getelementptr inbounds [1024 x i8], ptr [[BYTES]], i64 0, i64 2
 ; CGSCC-NEXT:    store i8 2, ptr [[I2]], align 2
-; CGSCC-NEXT:    [[R:%.*]] = call i8 @read_arg_index(ptr nocapture nofree noundef nonnull readonly align 16 dereferenceable(1024) [[BYTES]]) #[[ATTR3]]
+; CGSCC-NEXT:    [[R:%.*]] = call i8 @read_arg_index(ptr nofree noundef nonnull readonly align 16 captures(none) dereferenceable(1024) [[BYTES]]) #[[ATTR3]]
 ; CGSCC-NEXT:    ret i8 [[R]]
 ;
 entry:
@@ -153,14 +153,14 @@ entry:
 define internal i8 @read_arg_2(ptr %p) {
 ; TUNIT: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: read)
 ; TUNIT-LABEL: define {{[^@]+}}@read_arg_2
-; TUNIT-SAME: (ptr nocapture nofree noundef nonnull readonly dereferenceable(971) [[P:%.*]]) #[[ATTR1:[0-9]+]] {
+; TUNIT-SAME: (ptr nofree noundef nonnull readonly captures(none) dereferenceable(971) [[P:%.*]]) #[[ATTR1:[0-9]+]] {
 ; TUNIT-NEXT:  entry:
 ; TUNIT-NEXT:    [[L:%.*]] = load i8, ptr [[P]], align 1
 ; TUNIT-NEXT:    ret i8 [[L]]
 ;
 ; CGSCC: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: read)
 ; CGSCC-LABEL: define {{[^@]+}}@read_arg_2
-; CGSCC-SAME: (ptr nocapture nofree noundef nonnull readonly dereferenceable(1) [[P:%.*]]) #[[ATTR0]] {
+; CGSCC-SAME: (ptr nofree noundef nonnull readonly captures(none) dereferenceable(1) [[P:%.*]]) #[[ATTR0]] {
 ; CGSCC-NEXT:  entry:
 ; CGSCC-NEXT:    [[L:%.*]] = load i8, ptr [[P]], align 1
 ; CGSCC-NEXT:    ret i8 [[L]]
@@ -173,17 +173,17 @@ entry:
 define internal i8 @sum_two_different_loads(ptr %p, ptr %q) {
 ; TUNIT: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: read)
 ; TUNIT-LABEL: define {{[^@]+}}@sum_two_different_loads
-; TUNIT-SAME: (ptr nocapture nofree nonnull readonly dereferenceable(972) [[P:%.*]], ptr nocapture nofree noundef nonnull readonly dereferenceable(971) [[Q:%.*]]) #[[ATTR1]] {
-; TUNIT-NEXT:    [[X:%.*]] = call i8 @read_arg_2(ptr nocapture nofree noundef nonnull readonly dereferenceable(972) [[P]]) #[[ATTR2:[0-9]+]]
-; TUNIT-NEXT:    [[Y:%.*]] = call i8 @read_arg_2(ptr nocapture nofree noundef nonnull readonly dereferenceable(971) [[Q]]) #[[ATTR2]]
+; TUNIT-SAME: (ptr nofree nonnull readonly captures(none) dereferenceable(972) [[P:%.*]], ptr nofree noundef nonnull readonly captures(none) dereferenceable(971) [[Q:%.*]]) #[[ATTR1]] {
+; TUNIT-NEXT:    [[X:%.*]] = call i8 @read_arg_2(ptr nofree noundef nonnull readonly captures(none) dereferenceable(972) [[P]]) #[[ATTR2:[0-9]+]]
+; TUNIT-NEXT:    [[Y:%.*]] = call i8 @read_arg_2(ptr nofree noundef nonnull readonly captures(none) dereferenceable(971) [[Q]]) #[[ATTR2]]
 ; TUNIT-NEXT:    [[Z:%.*]] = add nsw i8 [[X]], [[Y]]
 ; TUNIT-NEXT:    ret i8 [[Z]]
 ;
 ; CGSCC: Function Attrs: mustprogress nofree nosync nounwind willreturn memory(argmem: read)
 ; CGSCC-LABEL: define {{[^@]+}}@sum_two_different_loads
-; CGSCC-SAME: (ptr nocapture nofree noundef nonnull readonly dereferenceable(972) [[P:%.*]], ptr nocapture nofree noundef nonnull readonly dereferenceable(971) [[Q:%.*]]) #[[ATTR2]] {
-; CGSCC-NEXT:    [[X:%.*]] = call i8 @read_arg_2(ptr nocapture nofree noundef nonnull readonly dereferenceable(972) [[P]]) #[[ATTR3]]
-; CGSCC-NEXT:    [[Y:%.*]] = call i8 @read_arg_2(ptr nocapture nofree noundef nonnull readonly dereferenceable(971) [[Q]]) #[[ATTR3]]
+; CGSCC-SAME: (ptr nofree noundef nonnull readonly captures(none) dereferenceable(972) [[P:%.*]], ptr nofree noundef nonnull readonly captures(none) dereferenceable(971) [[Q:%.*]]) #[[ATTR2]] {
+; CGSCC-NEXT:    [[X:%.*]] = call i8 @read_arg_2(ptr nofree noundef nonnull readonly captures(none) dereferenceable(972) [[P]]) #[[ATTR3]]
+; CGSCC-NEXT:    [[Y:%.*]] = call i8 @read_arg_2(ptr nofree noundef nonnull readonly captures(none) dereferenceable(971) [[Q]]) #[[ATTR3]]
 ; CGSCC-NEXT:    [[Z:%.*]] = add nsw i8 [[X]], [[Y]]
 ; CGSCC-NEXT:    ret i8 [[Z]]
 ;
@@ -204,7 +204,7 @@ define i8 @call_partially_simplifiable_1() {
 ; TUNIT-NEXT:    [[I3:%.*]] = getelementptr inbounds [1024 x i8], ptr [[BYTES]], i64 0, i64 3
 ; TUNIT-NEXT:    store i8 3, ptr [[I3]], align 1
 ; TUNIT-NEXT:    [[I4:%.*]] = getelementptr inbounds [1024 x i8], ptr [[BYTES]], i64 0, i64 4
-; TUNIT-NEXT:    [[R:%.*]] = call i8 @sum_two_different_loads(ptr nocapture nofree noundef nonnull readonly align 2 dereferenceable(1022) [[I2]], ptr nocapture nofree noundef nonnull readonly dereferenceable(1021) [[I3]]) #[[ATTR3:[0-9]+]]
+; TUNIT-NEXT:    [[R:%.*]] = call i8 @sum_two_different_loads(ptr nofree noundef nonnull readonly align 2 captures(none) dereferenceable(1022) [[I2]], ptr nofree noundef nonnull readonly captures(none) dereferenceable(1021) [[I3]]) #[[ATTR3:[0-9]+]]
 ; TUNIT-NEXT:    ret i8 [[R]]
 ;
 ; CGSCC: Function Attrs: mustprogress nofree nosync nounwind willreturn memory(none)
@@ -218,7 +218,7 @@ define i8 @call_partially_simplifiable_1() {
 ; CGSCC-NEXT:    store i8 3, ptr [[I3]], align 1
 ; CGSCC-NEXT:    [[I4:%.*]] = getelementptr inbounds [1024 x i8], ptr [[BYTES]], i64 0, i64 4
 ; CGSCC-NEXT:    store i8 4, ptr [[I4]], align 4
-; CGSCC-NEXT:    [[R:%.*]] = call i8 @sum_two_different_loads(ptr nocapture nofree noundef nonnull readonly align 2 dereferenceable(1022) [[I2]], ptr nocapture nofree noundef nonnull readonly dereferenceable(1021) [[I3]]) #[[ATTR3]]
+; CGSCC-NEXT:    [[R:%.*]] = call i8 @sum_two_different_loads(ptr nofree noundef nonnull readonly align 2 captures(none) dereferenceable(1022) [[I2]], ptr nofree noundef nonnull readonly captures(none) dereferenceable(1021) [[I3]]) #[[ATTR3]]
 ; CGSCC-NEXT:    ret i8 [[R]]
 ;
 entry:
@@ -247,7 +247,7 @@ define i8 @call_partially_simplifiable_2(i1 %cond) {
 ; TUNIT-NEXT:    store i8 3, ptr [[I53]], align 1
 ; TUNIT-NEXT:    [[I54:%.*]] = getelementptr inbounds [1024 x i8], ptr [[BYTES]], i64 0, i64 54
 ; TUNIT-NEXT:    [[SEL:%.*]] = select i1 [[COND]], ptr [[I51]], ptr [[I52]]
-; TUNIT-NEXT:    [[R:%.*]] = call i8 @sum_two_different_loads(ptr nocapture nofree nonnull readonly dereferenceable(972) [[SEL]], ptr nocapture nofree noundef nonnull readonly dereferenceable(971) [[I53]]) #[[ATTR3]]
+; TUNIT-NEXT:    [[R:%.*]] = call i8 @sum_two_different_loads(ptr nofree nonnull readonly captures(none) dereferenceable(972) [[SEL]], ptr nofree noundef nonnull readonly captures(none) dereferenceable(971) [[I53]]) #[[ATTR3]]
 ; TUNIT-NEXT:    ret i8 [[R]]
 ;
 ; CGSCC: Function Attrs: mustprogress nofree nosync nounwind willreturn memory(none)
@@ -263,7 +263,7 @@ define i8 @call_partially_simplifiable_2(i1 %cond) {
 ; CGSCC-NEXT:    [[I54:%.*]] = getelementptr inbounds [1024 x i8], ptr [[BYTES]], i64 0, i64 54
 ; CGSCC-NEXT:    store i8 4, ptr [[I54]], align 2
 ; CGSCC-NEXT:    [[SEL:%.*]] = select i1 [[COND]], ptr [[I51]], ptr [[I52]]
-; CGSCC-NEXT:    [[R:%.*]] = call i8 @sum_two_different_loads(ptr nocapture nofree noundef nonnull readonly dereferenceable(972) [[SEL]], ptr nocapture nofree noundef nonnull readonly dereferenceable(971) [[I53]]) #[[ATTR3]]
+; CGSCC-NEXT:    [[R:%.*]] = call i8 @sum_two_different_loads(ptr nofree noundef nonnull readonly captures(none) dereferenceable(972) [[SEL]], ptr nofree noundef nonnull readonly captures(none) dereferenceable(971) [[I53]]) #[[ATTR3]]
 ; CGSCC-NEXT:    ret i8 [[R]]
 ;
 entry:
diff --git a/llvm/test/Transforms/Attributor/callbacks.ll b/llvm/test/Transforms/Attributor/callbacks.ll
index dd5cbbc9e271e..9e1db3f56cbed 100644
--- a/llvm/test/Transforms/Attributor/callbacks.ll
+++ b/llvm/test/Transforms/Attributor/callbacks.ll
@@ -22,7 +22,7 @@ define void @t0_caller(ptr %a) {
 ; TUNIT-NEXT:    [[PTR:%.*]] = alloca i32, align 128
 ; TUNIT-NEXT:    store i32 42, ptr [[B]], align 32
 ; TUNIT-NEXT:    store ptr [[B]], ptr [[C]], align 64
-; TUNIT-NEXT:    call void (ptr, ptr, ptr, ...) @t0_callback_broker(ptr noundef align 4294967296 null, ptr noundef nonnull align 128 dereferenceable(4) [[PTR]], ptr noundef nonnull @t0_callback_callee, ptr align 256 [[A]], i64 undef, ptr noalias nocapture nofree noundef nonnull readonly align 64 dereferenceable(8) [[C]])
+; TUNIT-NEXT:    call void (ptr, ptr, ptr, ...) @t0_callback_broker(ptr noundef align 4294967296 null, ptr noundef nonnull align 128 dereferenceable(4) [[PTR]], ptr noundef nonnull @t0_callback_callee, ptr align 256 [[A]], i64 undef, ptr noalias nofree noundef nonnull readonly align 64 captures(none) dereferenceable(8) [[C]])
 ; TUNIT-NEXT:    ret void
 ;
 ; CGSCC-LABEL: define {{[^@]+}}@t0_caller
@@ -33,7 +33,7 @@ define void @t0_caller(ptr %a) {
 ; CGSCC-NEXT:    [[PTR:%.*]] = alloca i32, align 128
 ; CGSCC-NEXT:    store i32 42, ptr [[B]], align 32
 ; CGSCC-NEXT:    store ptr [[B]], ptr [[C]], align 64
-; CGSCC-NEXT:    call void (ptr, ptr, ptr, ...) @t0_callback_broker(ptr noundef align 4294967296 null, ptr noundef nonnull align 128 dereferenceable(4) [[PTR]], ptr noundef nonnull @t0_callback_callee, ptr align 256 [[A]], i64 noundef 99, ptr noalias nocapture nofree noundef nonnull readonly align 64 dereferenceable(8) [[C]])
+; CGSCC-NEXT:    call void (ptr, ptr, ptr, ...) @t0_callback_broker(ptr noundef align 4294967296 null, ptr noundef nonnull align 128 dereferenceable(4) [[PTR]], ptr noundef nonnull @t0_callback_callee, ptr align 256 [[A]], i64 noundef 99, ptr noalias nofree noundef nonnull readonly align 64 captures(none) dereferenceable(8) [[C]])
 ; CGSCC-NEXT:    ret void
 ;
 entry:
@@ -51,7 +51,7 @@ entry:
 define internal void @t0_callback_callee(ptr %is_not_null, ptr %ptr, ptr %a, i64 %b, ptr %c) {
 ;
 ; TUNIT-LABEL: define {{[^@]+}}@t0_callback_callee
-; TUNIT-SAME: (ptr nocapture nofree noundef nonnull writeonly align 4 dereferenceable(4) [[IS_NOT_NULL:%.*]], ptr nocapture nofree noundef nonnull readonly align 8 dereferenceable(4) [[PTR:%.*]], ptr align 256 [[A:%.*]], i64 [[B:%.*]], ptr noalias nocapture nofree noundef nonnull readonly align 64 dereferenceable(8) [[C:%.*]]) {
+; TUNIT-SAME: (ptr nofree noundef nonnull writeonly align 4 captures(none) dereferenceable(4) [[IS_NOT_NULL:%.*]], ptr nofree noundef nonnull readonly align 8 captures(none) dereferenceable(4) [[PTR:%.*]], ptr align 256 [[A:%.*]], i64 [[B:%.*]], ptr noalias nofree noundef nonnull readonly align 64 captures(none) dereferenceable(8) [[C:%.*]]) {
 ; TUNIT-NEXT:  entry:
 ; TUNIT-NEXT:    [[PTR_VAL:%.*]] = load i32, ptr [[PTR]], align 8
 ; TUNIT-NEXT:    store i32 [[PTR_VAL]], ptr [[IS_NOT_NULL]], align 4
@@ -60,7 +60,7 @@ define internal void @t0_callback_callee(ptr %is_not_null, ptr %ptr, ptr %a, i64
 ; TUNIT-NEXT:    ret void
 ;
 ; CGSCC-LABEL: define {{[^@]+}}@t0_callback_callee
-; CGSCC-SAME: (ptr nocapture nofree noundef nonnull writeonly align 4 dereferenceable(4) [[IS_NOT_NULL:%.*]], ptr nocapture nofree noundef nonnull readonly align 8 dereferenceable(4) [[PTR:%.*]], ptr align 256 [[A:%.*]], i64 [[B:%.*]], ptr noalias nocapture nofree noundef nonnull readonly align 64 dereferenceable(8) [[C:%.*]]) {
+; CGSCC-SAME: (ptr nofree noundef nonnull writeonly align 4 captures(none) dereferenceable(4) [[IS_NOT_NULL:%.*]], ptr nofree noundef nonnull readonly align 8 captures(none) dereferenceable(4) [[PTR:%.*]], ptr align 256 [[A:%.*]], i64 [[B:%.*]], ptr noalias nofree noundef nonnull readonly align 64 captures(none) dereferenceable(8) [[C:%.*]]) {
 ; CGSCC-NEXT:  entry:
 ; CGSCC-NEXT:    [[PTR_VAL:%.*]] = load i32, ptr [[PTR]], align 8
 ; CGSCC-NEXT:    store i32 [[PTR_VAL]], ptr [[IS_NOT_NULL]], align 4
@@ -88,25 +88,25 @@ declare !callback !0 void @t0_callback_broker(ptr, ptr, ptr, ...)
 define void @t1_caller(ptr noalias %a) {
 ;
 ; TUNIT-LABEL: define {{[^@]+}}@t1_caller
-; TUNIT-SAME: (ptr noalias nocapture align 256 [[A:%.*]]) {
+; TUNIT-SAME: (ptr noalias align 256 captures(none) [[A:%.*]]) {
 ; TUNIT-NEXT:  entry:
 ; TUNIT-NEXT:    [[B:%.*]] = alloca i32, align 32
 ; TUNIT-NEXT:    [[C:%.*]] = alloca ptr, align 64
 ; TUNIT-NEXT:    [[PTR:%.*]] = alloca i32, align 128
 ; TUNIT-NEXT:    store i32 42, ptr [[B]], align 32
 ; TUNIT-NEXT:    store ptr [[B]], ptr [[C]], align 64
-; TUNIT-NEXT:    call void (ptr, ptr, ptr, ...) @t1_callback_broker(ptr noundef align 4294967296 null, ptr noalias nocapture noundef nonnull align 128 dereferenceable(4) [[PTR]], ptr nocapture noundef nonnull @t1_callback_callee, ptr nocapture align 256 [[A]], i64 undef, ptr noalias nocapture nofree noundef nonnull readonly align 64 dereferenceable(8) [[C]])
+; TUNIT-NEXT:    call void (ptr, ptr, ptr, ...) @t1_callback_broker(ptr noundef align 4294967296 null, ptr noalias noundef nonnull align 128 captures(none) dereferenceable(4) [[PTR]], ptr noundef nonnull captures(none) @t1_callback_callee, ptr align 256 captures(none) [[A]], i64 undef, ptr noalias nofree noundef nonnull readonly align 64 captures(none) dereferenceable(8) [[C]])
 ; TUNIT-NEXT:    ret void
 ;
 ; CGSCC-LABEL: define {{[^@]+}}@t1_caller
-; CGSCC-SAME: (ptr noalias nocapture align 256 [[A:%.*]]) {
+; CGSCC-SAME: (ptr noalias align 256 captures(none) [[A:%.*]]) {
 ; CGSCC-NEXT:  entry:
 ; CGSCC-NEXT:    [[B:%.*]] = alloca i32, align 32
 ; CGSCC-NEXT:    [[C:%.*]] = alloca ptr, align 64
 ; CGSCC-NEXT:    [[PTR:%.*]] = alloca i32, align 128
 ; CGSCC-NEXT:    store i32 42, ptr [[B]], align 32
 ; CGSCC-NEXT:    store ptr [[B]], ptr [[C]], align 64
-; CGSCC-NEXT:    call void (ptr, ptr, ptr, ...) @t1_callback_broker(ptr noundef align 4294967296 null, ptr noalias nocapture noundef nonnull align 128 dereferenceable(4) [[PTR]], ptr nocapture noundef nonnull @t1_callback_callee, ptr nocapture align 256 [[A]], i64 noundef 99, ptr noalias nocapture nofree noundef nonnull readonly align 64 dereferenceable(8) [[C]])
+; CGSCC-NEXT:    call void (ptr, ptr, ptr, ...) @t1_callback_broker(ptr noundef align 4294967296 null, ptr noalias noundef nonnull align 128 captures(none) dereferenceable(4) [[PTR]], ptr noundef nonnull captures(none) @t1_callback_callee, ptr align 256 captures(none) [[A]], i64 noundef 99, ptr noalias nofree noundef nonnull readonly align 64 captures(none) dereferenceable(8) [[C]])
 ; CGSCC-NEXT:    ret void
 ;
 entry:
@@ -125,22 +125,22 @@ define internal void @t1_callback_callee(ptr %is_not_null, ptr %ptr, ptr %a, i64
 ;
 ; TUNIT: Function Attrs: nosync
 ; TUNIT-LABEL: define {{[^@]+}}@t1_callback_callee
-; TUNIT-SAME: (ptr nocapture nofree noundef nonnull writeonly align 4 dereferenceable(4) [[IS_NOT_NULL:%.*]], ptr nocapture nofree noundef nonnull readonly align 8 dereferenceable(4) [[PTR:%.*]], ptr nocapture align 256 [[A:%.*]], i64 [[B:%.*]], ptr noalias nocapture nofree noundef nonnull readonly align 64 dereferenceable(8) [[C:%.*]]) #[[ATTR0:[0-9]+]] {
+; TUNIT-SAME: (ptr nofree noundef nonnull writeonly align 4 captures(none) dereferenceable(4) [[IS_NOT_NULL:%.*]], ptr nofree noundef nonnull readonly align 8 captures(none) dereferenceable(4) [[PTR:%.*]], ptr align 256 captures(none) [[A:%.*]], i64 [[B:%.*]], ptr noalias nofree noundef nonnull readonly align 64 captures(none) dereferenceable(8) [[C:%.*]]) #[[ATTR0:[0-9]+]] {
 ; TUNIT-NEXT:  entry:
 ; TUNIT-NEXT:    [[PTR_VAL:%.*]] = load i32, ptr [[PTR]], align 8
 ; TUNIT-NEXT:    store i32 [[PTR_VAL]], ptr [[IS_NOT_NULL]], align 4
 ; TUNIT-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[C]], align 64
-; TUNIT-NEXT:    tail call void @t1_check(ptr nocapture align 256 [[A]], i64 noundef 99, ptr nocapture align 32 [[TMP0]])
+; TUNIT-NEXT:    tail call void @t1_check(ptr align 256 captures(none) [[A]], i64 noundef 99, ptr align 32 captures(none) [[TMP0]])
 ; TUNIT-NEXT:    ret void
 ;
 ; CGSCC: Function Attrs: nosync
 ; CGSCC-LABEL: define {{[^@]+}}@t1_callback_callee
-; CGSCC-SAME: (ptr nocapture nofree noundef nonnull writeonly align 4 dereferenceable(4) [[IS_NOT_NULL:%.*]], ptr nocapture nofree noundef nonnull readonly align 8 dereferenceable(4) [[PTR:%.*]], ptr nocapture align 256 [[A:%.*]], i64 [[B:%.*]], ptr noalias nocapture nofree noundef nonnull readonly align 64 dereferenceable(8) [[C:%.*]]) #[[ATTR0:[0-9]+]] {
+; CGSCC-SAME: (ptr nofree noundef nonnull writeonly align 4 captures(none) dereferenceable(4) [[IS_NOT_NULL:%.*]], ptr nofree noundef nonnull readonly align 8 captures(none) dereferenceable(4) [[PTR:%.*]], ptr align 256 captures(none) [[A:%.*]], i64 [[B:%.*]], ptr noalias nofree noundef nonnull readonly align 64 captures(none) dereferenceable(8) [[C:%.*]]) #[[ATTR0:[0-9]+]] {
 ; CGSCC-NEXT:  entry:
 ; CGSCC-NEXT:    [[PTR_VAL:%.*]] = load i32, ptr [[PTR]], align 8
 ; CGSCC-NEXT:    store i32 [[PTR_VAL]], ptr [[IS_NOT_NULL]], align 4
 ; CGSCC-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[C]], align 64
-; CGSCC-NEXT:    tail call void @t1_check(ptr nocapture align 256 [[A]], i64 noundef 99, ptr nocapture [[TMP0]])
+; CGSCC-NEXT:    tail call void @t1_check(ptr align 256 captures(none) [[A]], i64 noundef 99, ptr captures(none) [[TMP0]])
 ; CGSCC-NEXT:    ret void
 ;
 entry:
@@ -161,25 +161,25 @@ declare !callback !0 void @t1_callback_broker(ptr nocapture , ptr nocapture , pt
 
 define void @t2_caller(ptr noalias %a) {
 ; TUNIT-LABEL: define {{[^@]+}}@t2_caller
-; TUNIT-SAME: (ptr noalias nocapture align 256 [[A:%.*]]) {
+; TUNIT-SAME: (ptr noalias align 256 captures(none) [[A:%.*]]) {
 ; TUNIT-NEXT:  entry:
 ; TUNIT-NEXT:    [[B:%.*]] = alloca i32, align 32
 ; TUNIT-NEXT:    [[C:%.*]] = alloca ptr, align 64
 ; TUNIT-NEXT:    [[PTR:%.*]] = alloca i32, align 128
 ; TUNIT-NEXT:    store i32 42, ptr [[B]], align 32
 ; TUNIT-NEXT:    store ptr [[B]], ptr [[C]], align 64
-; TUNIT-NEXT:    call void (ptr, ptr, ptr, ...) @t2_callback_broker(ptr noundef align 4294967296 null, ptr noalias nocapture noundef nonnull align 128 dereferenceable(4) [[PTR]], ptr nocapture noundef nonnull @t2_callback_callee, ptr nocapture align 256 [[A]], i64 undef, ptr noalias nocapture nofree noundef nonnull readonly align 64 dereferenceable(8) [[C]])
+; TUNIT-NEXT:    call void (ptr, ptr, ptr, ...) @t2_callback_broker(ptr noundef align 4294967296 null, ptr noalias noundef nonnull align 128 captures(none) dereferenceable(4) [[PTR]], ptr noundef nonnull captures(none) @t2_callback_callee, ptr align 256 captures(none) [[A]], i64 undef, ptr noalias nofree noundef nonnull readonly align 64 captures(none) dereferenceable(8) [[C]])
 ; TUNIT-NEXT:    ret void
 ;
 ; CGSCC-LABEL: define {{[^@]+}}@t2_caller
-; CGSCC-SAME: (ptr noalias nocapture align 256 [[A:%.*]]) {
+; CGSCC-SAME: (ptr noalias align 256 captures(none) [[A:%.*]]) {
 ; CGSCC-NEXT:  entry:
 ; CGSCC-NEXT:    [[B:%.*]] = alloca i32, align 32
 ; CGSCC-NEXT:    [[C:%.*]] = alloca ptr, align 64
 ; CGSCC-NEXT:    [[PTR:%.*]] = alloca i32, align 128
 ; CGSCC-NEXT:    store i32 42, ptr [[B]], align 32
 ; CGSCC-NEXT:    store ptr [[B]], ptr [[C]], align 64
-; CGSCC-NEXT:    call void (ptr, ptr, ptr, ...) @t2_callback_broker(ptr noundef align 4294967296 null, ptr noalias nocapture noundef nonnull align 128 dereferenceable(4) [[PTR]], ptr nocapture noundef nonnull @t2_callback_callee, ptr nocapture align 256 [[A]], i64 noundef 99, ptr noalias nocapture nofree noundef nonnull readonly align 64 dereferenceable(8) [[C]])
+; CGSCC-NEXT:    call void (ptr, ptr, ptr, ...) @t2_callback_broker(ptr noundef align 4294967296 null, ptr noalias noundef nonnull align 128 captures(none) dereferenceable(4) [[PTR]], ptr noundef nonnull captures(none) @t2_callback_callee, ptr align 256 captures(none) [[A]], i64 noundef 99, ptr noalias nofree noundef nonnull readonly align 64 captures(none) dereferenceable(8) [[C]])
 ; CGSCC-NEXT:    ret void
 ;
 entry:
@@ -199,21 +199,21 @@ entry:
 define internal void @t2_callback_callee(ptr %is_not_null, ptr %ptr, ptr %a, i64 %b, ptr %c) {
 ;
 ; TUNIT-LABEL: define {{[^@]+}}@t2_callback_callee
-; TUNIT-SAME: (ptr nocapture nofree noundef nonnull writeonly align 4 dereferenceable(4) [[IS_NOT_NULL:%.*]], ptr nocapture nofree noundef nonnull readonly align 8 dereferenceable(4) [[PTR:%.*]], ptr nocapture align 256 [[A:%.*]], i64 [[B:%.*]], ptr noalias nocapture nofree noundef nonnull readonly align 64 dereferenceable(8) [[C:%.*]]) {
+; TUNIT-SAME: (ptr nofree noundef nonnull writeonly align 4 captures(none) dereferenceable(4) [[IS_NOT_NULL:%.*]], ptr nofree noundef nonnull readonly align 8 captures(none) dereferenceable(4) [[PTR:%.*]], ptr align 256 captures(none) [[A:%.*]], i64 [[B:%.*]], ptr noalias nofree noundef nonnull readonly align 64 captures(none) dereferenceable(8) [[C:%.*]]) {
 ; TUNIT-NEXT:  entry:
 ; TUNIT-NEXT:    [[PTR_VAL:%.*]] = load i32, ptr [[PTR]], align 8
 ; TUNIT-NEXT:    store i32 [[PTR_VAL]], ptr [[IS_NOT_NULL]], align 4
 ; TUNIT-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[C]], align 64
-; TUNIT-NEXT:    tail call void @t2_check(ptr nocapture align 256 [[A]], i64 noundef 99, ptr nocapture align 32 [[TMP0]])
+; TUNIT-NEXT:    tail call void @t2_check(ptr align 256 captures(none) [[A]], i64 noundef 99, ptr align 32 captures(none) [[TMP0]])
 ; TUNIT-NEXT:    ret void
 ;
 ; CGSCC-LABEL: define {{[^@]+}}@t2_callback_callee
-; CGSCC-SAME: (ptr nocapture nofree noundef nonnull writeonly align 4 dereferenceable(4) [[IS_NOT_NULL:%.*]], ptr nocapture nofree noundef nonnull readonly align 8 dereferenceable(4) [[PTR:%.*]], ptr nocapture align 256 [[A:%.*]], i64 [[B:%.*]], ptr noalias nocapture nofree noundef nonnull readonly align 64 dereferenceable(8) [[C:%.*]]) {
+; CGSCC-SAME: (ptr nofree noundef nonnull writeonly align 4 captures(none) dereferenceable(4) [[IS_NOT_NULL:%.*]], ptr nofree noundef nonnull readonly align 8 captures(none) dereferenceable(4) [[PTR:%.*]], ptr align 256 captures(none) [[A:%.*]], i64 [[B:%.*]], ptr noalias nofree noundef nonnull readonly align 64 captures(none) dereferenceable(8) [[C:%.*]]) {
 ; CGSCC-NEXT:  entry:
 ; CGSCC-NEXT:    [[PTR_VAL:%.*]] = load i32, ptr [[PTR]], align 8
 ; CGSCC-NEXT:    store i32 [[PTR_VAL]], ptr [[IS_NOT_NULL]], align 4
 ; CGSCC-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[C]], align 64
-; CGSCC-NEXT:    tail call void @t2_check(ptr nocapture align 256 [[A]], i64 noundef 99, ptr nocapture [[TMP0]])
+; CGSCC-NEXT:    tail call void @t2_check(ptr align 256 captures(none) [[A]], i64 noundef 99, ptr captures(none) [[TMP0]])
 ; CGSCC-NEXT:    ret void
 ;
 entry:
@@ -234,27 +234,27 @@ declare !callback !0 void @t2_callback_broker(ptr nocapture , ptr nocapture , pt
 
 define void @t3_caller(ptr noalias %a) {
 ; TUNIT-LABEL: define {{[^@]+}}@t3_caller
-; TUNIT-SAME: (ptr noalias nocapture align 256 [[A:%.*]]) {
+; TUNIT-SAME: (ptr noalias align 256 captures(none) [[A:%.*]]) {
 ; TUNIT-NEXT:  entry:
 ; TUNIT-NEXT:    [[B:%.*]] = alloca i32, align 32
 ; TUNIT-NEXT:    [[C:%.*]] = alloca ptr, align 64
 ; TUNIT-NEXT:    [[PTR:%.*]] = alloca i32, align 128
 ; TUNIT-NEXT:    store i32 42, ptr [[B]], align 32
 ; TUNIT-NEXT:    store ptr [[B]], ptr [[C]], align 64
-; TUNIT-NEXT:    call void (ptr, ptr, ptr, ...) @t3_callback_broker(ptr noundef align 4294967296 null, ptr noalias nocapture noundef nonnull align 128 dereferenceable(4) [[PTR]], ptr nocapture noundef nonnull @t3_callback_callee, ptr nocapture align 256 [[A]], i64 undef, ptr noalias nocapture nofree noundef nonnull readonly align 64 dereferenceable(8) [[C]])
-; TUNIT-NEXT:    call void (ptr, ptr, ptr, ...) @t3_callback_broker(ptr noundef align 4294967296 null, ptr noalias nocapture noundef nonnull align 128 dereferenceable(4) [[PTR]], ptr nocapture noundef nonnull @t3_callback_callee, ptr nocapture align 256 [[A]], i64 undef, ptr noalias nocapture nofree noundef nonnull readonly align 64 dereferenceable(8) [[C]])
+; TUNIT-NEXT:    call void (ptr, ptr, ptr, ...) @t3_callback_broker(ptr noundef align 4294967296 null, ptr noalias noundef nonnull align 128 captures(none) dereferenceable(4) [[PTR]], ptr noundef nonnull captures(none) @t3_callback_callee, ptr align 256 captures(none) [[A]], i64 undef, ptr noalias nofree noundef nonnull readonly align 64 captures(none) dereferenceable(8) [[C]])
+; TUNIT-NEXT:    call void (ptr, ptr, ptr, ...) @t3_callback_broker(ptr noundef align 4294967296 null, ptr noalias noundef nonnull align 128 captures(none) dereferenceable(4) [[PTR]], ptr noundef nonnull captures(none) @t3_callback_callee, ptr align 256 captures(none) [[A]], i64 undef, ptr noalias nofree noundef nonnull readonly align 64 captures(none) dereferenceable(8) [[C]])
 ; TUNIT-NEXT:    ret void
 ;
 ; CGSCC-LABEL: define {{[^@]+}}@t3_caller
-; CGSCC-SAME: (ptr noalias nocapture align 256 [[A:%.*]]) {
+; CGSCC-SAME: (ptr noalias align 256 captures(none) [[A:%.*]]) {
 ; CGSCC-NEXT:  entry:
 ; CGSCC-NEXT:    [[B:%.*]] = alloca i32, align 32
 ; CGSCC-NEXT:    [[C:%.*]] = alloca ptr, align 64
 ; CGSCC-NEXT:    [[PTR:%.*]] = alloca i32, align 128
 ; CGSCC-NEXT:    store i32 42, ptr [[B]], align 32
 ; CGSCC-NEXT:    store ptr [[B]], ptr [[C]], align 64
-; CGSCC-NEXT:    call void (ptr, ptr, ptr, ...) @t3_callback_broker(ptr noundef align 4294967296 null, ptr noalias nocapture noundef nonnull align 128 dereferenceable(4) [[PTR]], ptr nocapture noundef nonnull @t3_callback_callee, ptr nocapture align 256 [[A]], i64 noundef 99, ptr noalias nocapture nofree noundef nonnull readonly align 64 dereferenceable(8) [[C]])
-; CGSCC-NEXT:    call void (ptr, ptr, ptr, ...) @t3_callback_broker(ptr noundef align 4294967296 null, ptr noalias nocapture noundef nonnull align 128 dereferenceable(4) [[PTR]], ptr nocapture noundef nonnull @t3_callback_callee, ptr nocapture align 256 [[A]], i64 noundef 99, ptr noalias nocapture nofree noundef nonnull readonly align 64 dereferenceable(8) [[C]])
+; CGSCC-NEXT:    call void (ptr, ptr, ptr, ...) @t3_callback_broker(ptr noundef align 4294967296 null, ptr noalias noundef nonnull align 128 captures(none) dereferenceable(4) [[PTR]], ptr noundef nonnull captures(none) @t3_callback_callee, ptr align 256 captures(none) [[A]], i64 noundef 99, ptr noalias nofree noundef nonnull readonly align 64 captures(none) dereferenceable(8) [[C]])
+; CGSCC-NEXT:    call void (ptr, ptr, ptr, ...) @t3_callback_broker(ptr noundef align 4294967296 null, ptr noalias noundef nonnull align 128 captures(none) dereferenceable(4) [[PTR]], ptr noundef nonnull captures(none) @t3_callback_callee, ptr align 256 captures(none) [[A]], i64 noundef 99, ptr noalias nofree noundef nonnull readonly align 64 captures(none) dereferenceable(8) [[C]])
 ; CGSCC-NEXT:    ret void
 ;
 entry:
@@ -275,21 +275,21 @@ entry:
 define internal void @t3_callback_callee(ptr %is_not_null, ptr %ptr, ptr %a, i64 %b, ptr %c) {
 ;
 ; TUNIT-LABEL: define {{[^@]+}}@t3_callback_callee
-; TUNIT-SAME: (ptr nocapture nofree noundef nonnull writeonly align 4 dereferenceable(4) [[IS_NOT_NULL:%.*]], ptr nocapture nofree noundef nonnull readonly align 8 dereferenceable(4) [[PTR:%.*]], ptr nocapture align 256 [[A:%.*]], i64 [[B:%.*]], ptr noalias nocapture nofree noundef nonnull readonly align 64 dereferenceable(8) [[C:%.*]]) {
+; TUNIT-SAME: (ptr nofree noundef nonnull writeonly align 4 captures(none) dereferenceable(4) [[IS_NOT_NULL:%.*]], ptr nofree noundef nonnull readonly align 8 captures(none) dereferenceable(4) [[PTR:%.*]], ptr align 256 captures(none) [[A:%.*]], i64 [[B:%.*]], ptr noalias nofree noundef nonnull readonly align 64 captures(none) dereferenceable(8) [[C:%.*]]) {
 ; TUNIT-NEXT:  entry:
 ; TUNIT-NEXT:    [[PTR_VAL:%.*]] = load i32, ptr [[PTR]], align 8
 ; TUNIT-NEXT:    store i32 [[PTR_VAL]], ptr [[IS_NOT_NULL]], align 4
 ; TUNIT-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[C]], align 64
-; TUNIT-NEXT:    tail call void @t3_check(ptr nocapture align 256 [[A]], i64 noundef 99, ptr nocapture align 32 [[TMP0]])
+; TUNIT-NEXT:    tail call void @t3_check(ptr align 256 captures(none) [[A]], i64 noundef 99, ptr align 32 captures(none) [[TMP0]])
 ; TUNIT-NEXT:    ret void
 ;
 ; CGSCC-LABEL: define {{[^@]+}}@t3_callback_callee
-; CGSCC-SAME: (ptr nocapture nofree noundef nonnull writeonly align 4 dereferenceable(4) [[IS_NOT_NULL:%.*]], ptr nocapture nofree noundef nonnull readonly align 8 dereferenceable(4) [[PTR:%.*]], ptr nocapture align 256 [[A:%.*]], i64 [[B:%.*]], ptr noalias nocapture nofree noundef nonnull readonly align 64 dereferenceable(8) [[C:%.*]]) {
+; CGSCC-SAME: (ptr nofree noundef nonnull writeonly align 4 captures(none) dereferenceable(4) [[IS_NOT_NULL:%.*]], ptr nofree noundef nonnull readonly align 8 captures(none) dereferenceable(4) [[PTR:%.*]], ptr align 256 captures(none) [[A:%.*]], i64 [[B:%.*]], ptr noalias nofree noundef nonnull readonly align 64 captures(none) dereferenceable(8) [[C:%.*]]) {
 ; CGSCC-NEXT:  entry:
 ; CGSCC-NEXT:    [[PTR_VAL:%.*]] = load i32, ptr [[PTR]], align 8
 ; CGSCC-NEXT:    store i32 [[PTR_VAL]], ptr [[IS_NOT_NULL]], align 4
 ; CGSCC-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[C]], align 64
-; CGSCC-NEXT:    tail call void @t3_check(ptr nocapture align 256 [[A]], i64 noundef 99, ptr nocapture [[TMP0]])
+; CGSCC-NEXT:    tail call void @t3_check(ptr align 256 captures(none) [[A]], i64 noundef 99, ptr captures(none) [[TMP0]])
 ; CGSCC-NEXT:    ret void
 ;
 entry:
diff --git a/llvm/test/Transforms/Attributor/callgraph.ll b/llvm/test/Transforms/Attributor/callgraph.ll
index 929b89105c240..84e2c54bd832d 100644
--- a/llvm/test/Transforms/Attributor/callgraph.ll
+++ b/llvm/test/Transforms/Attributor/callgraph.ll
@@ -540,7 +540,7 @@ define void @broker(ptr %unknown) !callback !0 {
 
 define void @func6() {
 ; CHECK-LABEL: @func6(
-; CHECK-NEXT:    call void @broker(ptr nocapture nofree noundef nonnull @func3)
+; CHECK-NEXT:    call void @broker(ptr nofree noundef nonnull captures(none) @func3)
 ; CHECK-NEXT:    ret void
 ;
   call void @broker(ptr @func3)
diff --git a/llvm/test/Transforms/Attributor/convergent.ll b/llvm/test/Transforms/Attributor/convergent.ll
index cd81c4d6f27b8..702d0bb0a9e63 100644
--- a/llvm/test/Transforms/Attributor/convergent.ll
+++ b/llvm/test/Transforms/Attributor/convergent.ll
@@ -111,14 +111,14 @@ declare void @llvm.memcpy.p0.p0.i64(ptr %dest, ptr %src, i64 %size, i1 %isVolati
 define void @calls_intrinsic(ptr %dest, ptr %src, i64 %size) convergent {
 ; TUNIT: Function Attrs: convergent mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: readwrite)
 ; TUNIT-LABEL: define {{[^@]+}}@calls_intrinsic
-; TUNIT-SAME: (ptr nocapture nofree writeonly [[DEST:%.*]], ptr nocapture nofree readonly [[SRC:%.*]], i64 [[SIZE:%.*]]) #[[ATTR2:[0-9]+]] {
-; TUNIT-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr nocapture nofree writeonly [[DEST]], ptr nocapture nofree readonly [[SRC]], i64 [[SIZE]], i1 noundef false) #[[ATTR5]]
+; TUNIT-SAME: (ptr nofree writeonly captures(none) [[DEST:%.*]], ptr nofree readonly captures(none) [[SRC:%.*]], i64 [[SIZE:%.*]]) #[[ATTR2:[0-9]+]] {
+; TUNIT-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr nofree writeonly captures(none) [[DEST]], ptr nofree readonly captures(none) [[SRC]], i64 [[SIZE]], i1 noundef false) #[[ATTR5]]
 ; TUNIT-NEXT:    ret void
 ;
 ; CGSCC: Function Attrs: convergent mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: readwrite)
 ; CGSCC-LABEL: define {{[^@]+}}@calls_intrinsic
-; CGSCC-SAME: (ptr nocapture nofree writeonly [[DEST:%.*]], ptr nocapture nofree readonly [[SRC:%.*]], i64 [[SIZE:%.*]]) #[[ATTR3:[0-9]+]] {
-; CGSCC-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr nocapture nofree writeonly [[DEST]], ptr nocapture nofree readonly [[SRC]], i64 [[SIZE]], i1 noundef false) #[[ATTR7]]
+; CGSCC-SAME: (ptr nofree writeonly captures(none) [[DEST:%.*]], ptr nofree readonly captures(none) [[SRC:%.*]], i64 [[SIZE:%.*]]) #[[ATTR3:[0-9]+]] {
+; CGSCC-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr nofree writeonly captures(none) [[DEST]], ptr nofree readonly captures(none) [[SRC]], i64 [[SIZE]], i1 noundef false) #[[ATTR7]]
 ; CGSCC-NEXT:    ret void
 ;
   call void @llvm.memcpy.p0.p0.i64(ptr %dest, ptr %src, i64 %size, i1 false)
diff --git a/llvm/test/Transforms/Attributor/dereferenceable-1.ll b/llvm/test/Transforms/Attributor/dereferenceable-1.ll
index f6a2df9eecc58..07e2d5ea15752 100644
--- a/llvm/test/Transforms/Attributor/dereferenceable-1.ll
+++ b/llvm/test/Transforms/Attributor/dereferenceable-1.ll
@@ -295,7 +295,7 @@ define ptr @test_for_minus_index(ptr %p) {
 define void @deref_or_null_and_nonnull(ptr dereferenceable_or_null(100) %0) {
 ; CHECK: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: write)
 ; CHECK-LABEL: define {{[^@]+}}@deref_or_null_and_nonnull
-; CHECK-SAME: (ptr nocapture nofree noundef nonnull writeonly align 4 dereferenceable(100) [[TMP0:%.*]]) #[[ATTR3]] {
+; CHECK-SAME: (ptr nofree noundef nonnull writeonly align 4 captures(none) dereferenceable(100) [[TMP0:%.*]]) #[[ATTR3]] {
 ; CHECK-NEXT:    store i32 1, ptr [[TMP0]], align 4
 ; CHECK-NEXT:    ret void
 ;
@@ -314,7 +314,7 @@ define void @deref_or_null_and_nonnull(ptr dereferenceable_or_null(100) %0) {
 define void @test8(ptr %ptr) #0 {
 ; CHECK: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: write)
 ; CHECK-LABEL: define {{[^@]+}}@test8
-; CHECK-SAME: (ptr nocapture nofree nonnull writeonly dereferenceable(21) [[PTR:%.*]]) #[[ATTR3]] {
+; CHECK-SAME: (ptr nofree nonnull writeonly captures(none) dereferenceable(21) [[PTR:%.*]]) #[[ATTR3]] {
 ; CHECK-NEXT:    br label [[TMP1:%.*]]
 ; CHECK:       1:
 ; CHECK-NEXT:    [[I_0:%.*]] = phi i32 [ 20, [[TMP0:%.*]] ], [ [[TMP4:%.*]], [[TMP5:%.*]] ]
@@ -349,7 +349,7 @@ define void @test8(ptr %ptr) #0 {
 define void @test8_neg(i32 %i, ptr %ptr) #0 {
 ; CHECK: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: write)
 ; CHECK-LABEL: define {{[^@]+}}@test8_neg
-; CHECK-SAME: (i32 [[I:%.*]], ptr nocapture nofree nonnull writeonly [[PTR:%.*]]) #[[ATTR3]] {
+; CHECK-SAME: (i32 [[I:%.*]], ptr nofree nonnull writeonly captures(none) [[PTR:%.*]]) #[[ATTR3]] {
 ; CHECK-NEXT:    [[TMP1:%.*]] = sext i32 [[I]] to i64
 ; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i8, ptr [[PTR]], i64 [[TMP1]]
 ; CHECK-NEXT:    store i8 65, ptr [[TMP2]], align 1
@@ -372,7 +372,7 @@ define void @test8_neg(i32 %i, ptr %ptr) #0 {
 define internal void @fill_range_not_inbounds(ptr %p, i64 %start){
 ; CHECK: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: write)
 ; CHECK-LABEL: define {{[^@]+}}@fill_range_not_inbounds
-; CHECK-SAME: (ptr nocapture nofree writeonly [[P:%.*]], i64 [[START:%.*]]) #[[ATTR3]] {
+; CHECK-SAME: (ptr nofree writeonly captures(none) [[P:%.*]], i64 [[START:%.*]]) #[[ATTR3]] {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[TMP0:%.*]] = add nsw i64 [[START]], 9
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
@@ -408,7 +408,7 @@ for.body:                                         ; preds = %entry, %for.body
 define internal void @fill_range_inbounds(ptr %p, i64 %start){
 ; CHECK: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: write)
 ; CHECK-LABEL: define {{[^@]+}}@fill_range_inbounds
-; CHECK-SAME: (ptr nocapture nofree writeonly [[P:%.*]], i64 [[START:%.*]]) #[[ATTR3]] {
+; CHECK-SAME: (ptr nofree writeonly captures(none) [[P:%.*]], i64 [[START:%.*]]) #[[ATTR3]] {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[TMP0:%.*]] = add nsw i64 [[START]], 9
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
@@ -443,20 +443,20 @@ for.body:                                         ; preds = %entry, %for.body
 define void @call_fill_range(ptr nocapture %p, ptr nocapture readonly %range) {
 ; TUNIT: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: readwrite)
 ; TUNIT-LABEL: define {{[^@]+}}@call_fill_range
-; TUNIT-SAME: (ptr nocapture nofree writeonly [[P:%.*]], ptr nocapture nofree noundef nonnull readonly align 8 dereferenceable(8) [[RANGE:%.*]]) #[[ATTR4:[0-9]+]] {
+; TUNIT-SAME: (ptr nofree writeonly captures(none) [[P:%.*]], ptr nofree noundef nonnull readonly align 8 captures(none) dereferenceable(8) [[RANGE:%.*]]) #[[ATTR4:[0-9]+]] {
 ; TUNIT-NEXT:  entry:
 ; TUNIT-NEXT:    [[TMP0:%.*]] = load i64, ptr [[RANGE]], align 8, !range [[RNG0:![0-9]+]]
-; TUNIT-NEXT:    tail call void @fill_range_inbounds(ptr nocapture nofree writeonly [[P]], i64 [[TMP0]]) #[[ATTR7:[0-9]+]]
-; TUNIT-NEXT:    tail call void @fill_range_not_inbounds(ptr nocapture nofree writeonly [[P]], i64 [[TMP0]]) #[[ATTR7]]
+; TUNIT-NEXT:    tail call void @fill_range_inbounds(ptr nofree writeonly captures(none) [[P]], i64 [[TMP0]]) #[[ATTR7:[0-9]+]]
+; TUNIT-NEXT:    tail call void @fill_range_not_inbounds(ptr nofree writeonly captures(none) [[P]], i64 [[TMP0]]) #[[ATTR7]]
 ; TUNIT-NEXT:    ret void
 ;
 ; CGSCC: Function Attrs: mustprogress nofree nosync nounwind willreturn memory(argmem: readwrite)
 ; CGSCC-LABEL: define {{[^@]+}}@call_fill_range
-; CGSCC-SAME: (ptr nocapture nofree writeonly [[P:%.*]], ptr nocapture nofree noundef nonnull readonly align 8 dereferenceable(8) [[RANGE:%.*]]) #[[ATTR4:[0-9]+]] {
+; CGSCC-SAME: (ptr nofree writeonly captures(none) [[P:%.*]], ptr nofree noundef nonnull readonly align 8 captures(none) dereferenceable(8) [[RANGE:%.*]]) #[[ATTR4:[0-9]+]] {
 ; CGSCC-NEXT:  entry:
 ; CGSCC-NEXT:    [[TMP0:%.*]] = load i64, ptr [[RANGE]], align 8, !range [[RNG0:![0-9]+]]
-; CGSCC-NEXT:    tail call void @fill_range_inbounds(ptr nocapture nofree writeonly [[P]], i64 [[TMP0]]) #[[ATTR7:[0-9]+]]
-; CGSCC-NEXT:    tail call void @fill_range_not_inbounds(ptr nocapture nofree writeonly [[P]], i64 [[TMP0]]) #[[ATTR7]]
+; CGSCC-NEXT:    tail call void @fill_range_inbounds(ptr nofree writeonly captures(none) [[P]], i64 [[TMP0]]) #[[ATTR7:[0-9]+]]
+; CGSCC-NEXT:    tail call void @fill_range_not_inbounds(ptr nofree writeonly captures(none) [[P]], i64 [[TMP0]]) #[[ATTR7]]
 ; CGSCC-NEXT:    ret void
 ;
 entry:
@@ -515,7 +515,7 @@ if.else:
 define void @complex-path(ptr %a, ptr %b, i8 %c) {
 ; CHECK: Function Attrs: mustprogress nounwind willreturn
 ; CHECK-LABEL: define {{[^@]+}}@complex-path
-; CHECK-SAME: (ptr noundef nonnull dereferenceable(12) [[A:%.*]], ptr nocapture nofree readnone [[B:%.*]], i8 [[C:%.*]]) #[[ATTR2]] {
+; CHECK-SAME: (ptr noundef nonnull dereferenceable(12) [[A:%.*]], ptr nofree readnone captures(none) [[B:%.*]], i8 [[C:%.*]]) #[[ATTR2]] {
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i8 [[C]], 0
 ; CHECK-NEXT:    tail call void @use1(ptr noundef nonnull dereferenceable(12) [[A]]) #[[ATTR1]]
 ; CHECK-NEXT:    br i1 [[CMP]], label [[CONT_THEN:%.*]], label [[CONT_ELSE:%.*]]
@@ -561,7 +561,7 @@ cont2:
 define dso_local void @rec-branch-1(i32 %a, i32 %b, i32 %c, ptr %ptr) {
 ; CHECK: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: write)
 ; CHECK-LABEL: define {{[^@]+}}@rec-branch-1
-; CHECK-SAME: (i32 [[A:%.*]], i32 [[B:%.*]], i32 [[C:%.*]], ptr nocapture nofree writeonly [[PTR:%.*]]) #[[ATTR3]] {
+; CHECK-SAME: (i32 [[A:%.*]], i32 [[B:%.*]], i32 [[C:%.*]], ptr nofree writeonly captures(none) [[PTR:%.*]]) #[[ATTR3]] {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[TOBOOL:%.*]] = icmp eq i32 [[A]], 0
 ; CHECK-NEXT:    br i1 [[TOBOOL]], label [[IF_ELSE3:%.*]], label [[IF_THEN:%.*]]
@@ -635,7 +635,7 @@ if.end8:                                          ; preds = %if.then5, %if.else6
 define dso_local void @rec-branch-2(i32 %a, i32 %b, i32 %c, ptr %ptr) {
 ; CHECK: Function Attrs: nofree nosync nounwind memory(argmem: write)
 ; CHECK-LABEL: define {{[^@]+}}@rec-branch-2
-; CHECK-SAME: (i32 [[A:%.*]], i32 [[B:%.*]], i32 [[C:%.*]], ptr nocapture nofree writeonly [[PTR:%.*]]) #[[ATTR5:[0-9]+]] {
+; CHECK-SAME: (i32 [[A:%.*]], i32 [[B:%.*]], i32 [[C:%.*]], ptr nofree writeonly captures(none) [[PTR:%.*]]) #[[ATTR5:[0-9]+]] {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[TOBOOL:%.*]] = icmp eq i32 [[A]], 0
 ; CHECK-NEXT:    br i1 [[TOBOOL]], label [[IF_ELSE3:%.*]], label [[IF_THEN:%.*]]
@@ -655,7 +655,7 @@ define dso_local void @rec-branch-2(i32 %a, i32 %b, i32 %c, ptr %ptr) {
 ; CHECK-NEXT:    store i32 3, ptr [[PTR]], align 4
 ; CHECK-NEXT:    br label [[IF_END8]]
 ; CHECK:       if.else6:
-; CHECK-NEXT:    tail call void @rec-branch-2(i32 noundef 1, i32 noundef 1, i32 noundef 1, ptr nocapture nofree writeonly [[PTR]]) #[[ATTR8:[0-9]+]]
+; CHECK-NEXT:    tail call void @rec-branch-2(i32 noundef 1, i32 noundef 1, i32 noundef 1, ptr nofree writeonly captures(none) [[PTR]]) #[[ATTR8:[0-9]+]]
 ; CHECK-NEXT:    br label [[IF_END8]]
 ; CHECK:       if.end8:
 ; CHECK-NEXT:    ret void
@@ -701,7 +701,7 @@ define void @nonnull_assume_pos(ptr %arg1, ptr %arg2, ptr %arg3, ptr %arg4) {
 ; ATTRIBUTOR-NEXT:    ret void
 ;
 ; CHECK-LABEL: define {{[^@]+}}@nonnull_assume_pos
-; CHECK-SAME: (ptr nocapture nofree nonnull readnone dereferenceable(101) [[ARG1:%.*]], ptr nocapture nofree readnone dereferenceable_or_null(31) [[ARG2:%.*]], ptr nocapture nofree nonnull readnone [[ARG3:%.*]], ptr nocapture nofree readnone dereferenceable_or_null(42) [[ARG4:%.*]]) {
+; CHECK-SAME: (ptr nofree nonnull readnone captures(none) dereferenceable(101) [[ARG1:%.*]], ptr nofree readnone captures(none) dereferenceable_or_null(31) [[ARG2:%.*]], ptr nofree nonnull readnone captures(none) [[ARG3:%.*]], ptr nofree readnone captures(none) dereferenceable_or_null(42) [[ARG4:%.*]]) {
 ; CHECK-NEXT:    call void @llvm.assume(i1 noundef true) #[[ATTR9:[0-9]+]] [ "nonnull"(ptr [[ARG3]]), "dereferenceable"(ptr [[ARG1]], i64 1), "dereferenceable"(ptr [[ARG1]], i64 2), "dereferenceable"(ptr [[ARG1]], i64 101), "dereferenceable_or_null"(ptr [[ARG2]], i64 31), "dereferenceable_or_null"(ptr [[ARG4]], i64 42) ]
 ; CHECK-NEXT:    call void @unknown()
 ; CHECK-NEXT:    ret void
@@ -718,7 +718,7 @@ define void @nonnull_assume_neg(ptr %arg1, ptr %arg2, ptr %arg3) {
 ; ATTRIBUTOR-NEXT:    ret void
 ;
 ; CHECK-LABEL: define {{[^@]+}}@nonnull_assume_neg
-; CHECK-SAME: (ptr nocapture nofree readnone [[ARG1:%.*]], ptr nocapture nofree readnone [[ARG2:%.*]], ptr nocapture nofree readnone [[ARG3:%.*]]) {
+; CHECK-SAME: (ptr nofree readnone captures(none) [[ARG1:%.*]], ptr nofree readnone captures(none) [[ARG2:%.*]], ptr nofree readnone captures(none) [[ARG3:%.*]]) {
 ; CHECK-NEXT:    call void @unknown()
 ; CHECK-NEXT:    call void @llvm.assume(i1 noundef true) [ "dereferenceable"(ptr [[ARG1]], i64 101), "dereferenceable"(ptr [[ARG2]], i64 -2), "dereferenceable_or_null"(ptr [[ARG3]], i64 31) ]
 ; CHECK-NEXT:    ret void
diff --git a/llvm/test/Transforms/Attributor/dereferenceable-2-inseltpoison.ll b/llvm/test/Transforms/Attributor/dereferenceable-2-inseltpoison.ll
index c7f76a291b836..3a904e5fe7d01 100644
--- a/llvm/test/Transforms/Attributor/dereferenceable-2-inseltpoison.ll
+++ b/llvm/test/Transforms/Attributor/dereferenceable-2-inseltpoison.ll
@@ -8,7 +8,7 @@
 define <4 x double> @PR21780(ptr %ptr) {
 ; CHECK: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: read)
 ; CHECK-LABEL: define {{[^@]+}}@PR21780
-; CHECK-SAME: (ptr nocapture nofree noundef nonnull readonly align 8 dereferenceable(32) [[PTR:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-SAME: (ptr nofree noundef nonnull readonly align 8 captures(none) dereferenceable(32) [[PTR:%.*]]) #[[ATTR0:[0-9]+]] {
 ; CHECK-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds double, ptr [[PTR]], i64 1
 ; CHECK-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds double, ptr [[PTR]], i64 2
 ; CHECK-NEXT:    [[ARRAYIDX3:%.*]] = getelementptr inbounds double, ptr [[PTR]], i64 3
@@ -46,7 +46,7 @@ define <4 x double> @PR21780(ptr %ptr) {
 define double @PR21780_only_access3_with_inbounds(ptr %ptr) {
 ; CHECK: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: read)
 ; CHECK-LABEL: define {{[^@]+}}@PR21780_only_access3_with_inbounds
-; CHECK-SAME: (ptr nocapture nofree nonnull readonly align 8 dereferenceable(32) [[PTR:%.*]]) #[[ATTR0]] {
+; CHECK-SAME: (ptr nofree nonnull readonly align 8 captures(none) dereferenceable(32) [[PTR:%.*]]) #[[ATTR0]] {
 ; CHECK-NEXT:    [[ARRAYIDX3:%.*]] = getelementptr inbounds double, ptr [[PTR]], i64 3
 ; CHECK-NEXT:    [[T3:%.*]] = load double, ptr [[ARRAYIDX3]], align 8
 ; CHECK-NEXT:    ret double [[T3]]
@@ -60,7 +60,7 @@ define double @PR21780_only_access3_with_inbounds(ptr %ptr) {
 define double @PR21780_only_access3_without_inbounds(ptr %ptr) {
 ; CHECK: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: read)
 ; CHECK-LABEL: define {{[^@]+}}@PR21780_only_access3_without_inbounds
-; CHECK-SAME: (ptr nocapture nofree readonly align 8 [[PTR:%.*]]) #[[ATTR0]] {
+; CHECK-SAME: (ptr nofree readonly align 8 captures(none) [[PTR:%.*]]) #[[ATTR0]] {
 ; CHECK-NEXT:    [[ARRAYIDX3:%.*]] = getelementptr double, ptr [[PTR]], i64 3
 ; CHECK-NEXT:    [[T3:%.*]] = load double, ptr [[ARRAYIDX3]], align 8
 ; CHECK-NEXT:    ret double [[T3]]
@@ -73,7 +73,7 @@ define double @PR21780_only_access3_without_inbounds(ptr %ptr) {
 define double @PR21780_without_inbounds(ptr %ptr) {
 ; CHECK: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: read)
 ; CHECK-LABEL: define {{[^@]+}}@PR21780_without_inbounds
-; CHECK-SAME: (ptr nocapture nofree noundef nonnull readonly align 8 dereferenceable(32) [[PTR:%.*]]) #[[ATTR0]] {
+; CHECK-SAME: (ptr nofree noundef nonnull readonly align 8 captures(none) dereferenceable(32) [[PTR:%.*]]) #[[ATTR0]] {
 ; CHECK-NEXT:    [[ARRAYIDX3:%.*]] = getelementptr double, ptr [[PTR]], i64 3
 ; CHECK-NEXT:    [[T3:%.*]] = load double, ptr [[ARRAYIDX3]], align 8
 ; CHECK-NEXT:    ret double [[T3]]
@@ -96,7 +96,7 @@ define double @PR21780_without_inbounds(ptr %ptr) {
 define void @gep0(ptr %unused, ptr %other, ptr %ptr) {
 ; CHECK: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: readwrite)
 ; CHECK-LABEL: define {{[^@]+}}@gep0
-; CHECK-SAME: (ptr nocapture nofree readnone [[UNUSED:%.*]], ptr nocapture nofree noundef nonnull writeonly dereferenceable(1) [[OTHER:%.*]], ptr nocapture nofree nonnull readonly dereferenceable(3) [[PTR:%.*]]) #[[ATTR1:[0-9]+]] {
+; CHECK-SAME: (ptr nofree readnone captures(none) [[UNUSED:%.*]], ptr nofree noundef nonnull writeonly captures(none) dereferenceable(1) [[OTHER:%.*]], ptr nofree nonnull readonly captures(none) dereferenceable(3) [[PTR:%.*]]) #[[ATTR1:[0-9]+]] {
 ; CHECK-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr i8, ptr [[PTR]], i64 2
 ; CHECK-NEXT:    [[T2:%.*]] = load i8, ptr [[ARRAYIDX2]], align 1
 ; CHECK-NEXT:    store i8 [[T2]], ptr [[OTHER]], align 1
@@ -118,7 +118,7 @@ define void @gep0(ptr %unused, ptr %other, ptr %ptr) {
 define void @ordering(ptr %ptr1, ptr %ptr2) {
 ; CHECK: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(none)
 ; CHECK-LABEL: define {{[^@]+}}@ordering
-; CHECK-SAME: (ptr nocapture nofree noundef nonnull readnone dereferenceable(3) [[PTR1:%.*]], ptr nocapture nofree nonnull readnone align 4 dereferenceable(8) [[PTR2:%.*]]) #[[ATTR2:[0-9]+]] {
+; CHECK-SAME: (ptr nofree noundef nonnull readnone captures(none) dereferenceable(3) [[PTR1:%.*]], ptr nofree nonnull readnone align 4 captures(none) dereferenceable(8) [[PTR2:%.*]]) #[[ATTR2:[0-9]+]] {
 ; CHECK-NEXT:    ret void
 ;
   %a20 = getelementptr i32, ptr %ptr2, i64 0
@@ -138,7 +138,7 @@ define void @ordering(ptr %ptr1, ptr %ptr2) {
 define void @not_entry_but_guaranteed_to_execute(ptr %ptr) {
 ; CHECK: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(none)
 ; CHECK-LABEL: define {{[^@]+}}@not_entry_but_guaranteed_to_execute
-; CHECK-SAME: (ptr nocapture nofree nonnull readnone dereferenceable(3) [[PTR:%.*]]) #[[ATTR2]] {
+; CHECK-SAME: (ptr nofree nonnull readnone captures(none) dereferenceable(3) [[PTR:%.*]]) #[[ATTR2]] {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    br label [[EXIT:%.*]]
 ; CHECK:       exit:
@@ -160,7 +160,7 @@ exit:
 define void @not_entry_not_guaranteed_to_execute(ptr %ptr, i1 %cond) {
 ; CHECK: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(none)
 ; CHECK-LABEL: define {{[^@]+}}@not_entry_not_guaranteed_to_execute
-; CHECK-SAME: (ptr nocapture nofree readnone [[PTR:%.*]], i1 noundef [[COND:%.*]]) #[[ATTR2]] {
+; CHECK-SAME: (ptr nofree readnone captures(none) [[PTR:%.*]], i1 noundef [[COND:%.*]]) #[[ATTR2]] {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    br i1 [[COND]], label [[LOADS:%.*]], label [[EXIT:%.*]]
 ; CHECK:       loads:
@@ -186,7 +186,7 @@ exit:
 define void @partial_in_entry(ptr %ptr, i1 %cond) {
 ; CHECK: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(none)
 ; CHECK-LABEL: define {{[^@]+}}@partial_in_entry
-; CHECK-SAME: (ptr nocapture nofree noundef nonnull readnone align 2 dereferenceable(4) [[PTR:%.*]], i1 noundef [[COND:%.*]]) #[[ATTR2]] {
+; CHECK-SAME: (ptr nofree noundef nonnull readnone align 2 captures(none) dereferenceable(4) [[PTR:%.*]], i1 noundef [[COND:%.*]]) #[[ATTR2]] {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    br i1 [[COND]], label [[LOADS:%.*]], label [[EXIT:%.*]]
 ; CHECK:       loads:
@@ -231,7 +231,7 @@ define void @volatile_is_not_dereferenceable(ptr %ptr) {
 define void @atomic_is_alright(ptr %ptr) {
 ; CHECK: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(none)
 ; CHECK-LABEL: define {{[^@]+}}@atomic_is_alright
-; CHECK-SAME: (ptr nocapture nofree nonnull readnone align 2 dereferenceable(6) [[PTR:%.*]]) #[[ATTR2]] {
+; CHECK-SAME: (ptr nofree nonnull readnone align 2 captures(none) dereferenceable(6) [[PTR:%.*]]) #[[ATTR2]] {
 ; CHECK-NEXT:    ret void
 ;
   %arrayidx0 = getelementptr i16, ptr %ptr, i64 0
@@ -247,7 +247,7 @@ declare void @may_not_return()
 
 define void @not_guaranteed_to_transfer_execution(ptr %ptr) {
 ; CHECK-LABEL: define {{[^@]+}}@not_guaranteed_to_transfer_execution
-; CHECK-SAME: (ptr nocapture nofree nonnull readnone align 2 dereferenceable(2) [[PTR:%.*]]) {
+; CHECK-SAME: (ptr nofree nonnull readnone align 2 captures(none) dereferenceable(2) [[PTR:%.*]]) {
 ; CHECK-NEXT:    call void @may_not_return()
 ; CHECK-NEXT:    ret void
 ;
@@ -266,7 +266,7 @@ define void @not_guaranteed_to_transfer_execution(ptr %ptr) {
 define void @variable_gep_index(ptr %unused, ptr %ptr, i64 %variable_index) {
 ; CHECK: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(none)
 ; CHECK-LABEL: define {{[^@]+}}@variable_gep_index
-; CHECK-SAME: (ptr nocapture nofree readnone [[UNUSED:%.*]], ptr nocapture nofree noundef nonnull readnone dereferenceable(1) [[PTR:%.*]], i64 [[VARIABLE_INDEX:%.*]]) #[[ATTR2]] {
+; CHECK-SAME: (ptr nofree readnone captures(none) [[UNUSED:%.*]], ptr nofree noundef nonnull readnone captures(none) dereferenceable(1) [[PTR:%.*]], i64 [[VARIABLE_INDEX:%.*]]) #[[ATTR2]] {
 ; CHECK-NEXT:    ret void
 ;
   %arrayidx1 = getelementptr i8, ptr %ptr, i64 %variable_index
@@ -283,7 +283,7 @@ define void @multi_index_gep(ptr %ptr) {
 ; FIXME: %ptr should be dereferenceable(4)
 ; CHECK: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(none)
 ; CHECK-LABEL: define {{[^@]+}}@multi_index_gep
-; CHECK-SAME: (ptr nocapture nofree nonnull readnone dereferenceable(1) [[PTR:%.*]]) #[[ATTR2]] {
+; CHECK-SAME: (ptr nofree nonnull readnone captures(none) dereferenceable(1) [[PTR:%.*]]) #[[ATTR2]] {
 ; CHECK-NEXT:    ret void
 ;
   %arrayidx00 = getelementptr <4 x i8>, ptr %ptr, i64 0, i64 0
@@ -296,7 +296,7 @@ define void @multi_index_gep(ptr %ptr) {
 define void @not_byte_multiple(ptr %ptr) {
 ; CHECK: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(none)
 ; CHECK-LABEL: define {{[^@]+}}@not_byte_multiple
-; CHECK-SAME: (ptr nocapture nofree nonnull readnone align 2 dereferenceable(2) [[PTR:%.*]]) #[[ATTR2]] {
+; CHECK-SAME: (ptr nofree nonnull readnone align 2 captures(none) dereferenceable(2) [[PTR:%.*]]) #[[ATTR2]] {
 ; CHECK-NEXT:    ret void
 ;
   %arrayidx0 = getelementptr i9, ptr %ptr, i64 0
@@ -309,7 +309,7 @@ define void @not_byte_multiple(ptr %ptr) {
 define void @no_pointer_deref(ptr %ptr) {
 ; CHECK: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(none)
 ; CHECK-LABEL: define {{[^@]+}}@no_pointer_deref
-; CHECK-SAME: (ptr nocapture nofree readnone align 2 [[PTR:%.*]]) #[[ATTR2]] {
+; CHECK-SAME: (ptr nofree readnone align 2 captures(none) [[PTR:%.*]]) #[[ATTR2]] {
 ; CHECK-NEXT:    ret void
 ;
   %arrayidx1 = getelementptr i16, ptr %ptr, i64 1
@@ -324,7 +324,7 @@ define void @no_pointer_deref(ptr %ptr) {
 define void @non_consecutive(ptr %ptr) {
 ; CHECK: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(none)
 ; CHECK-LABEL: define {{[^@]+}}@non_consecutive
-; CHECK-SAME: (ptr nocapture nofree noundef nonnull readnone align 4 dereferenceable(8) [[PTR:%.*]]) #[[ATTR2]] {
+; CHECK-SAME: (ptr nofree noundef nonnull readnone align 4 captures(none) dereferenceable(8) [[PTR:%.*]]) #[[ATTR2]] {
 ; CHECK-NEXT:    ret void
 ;
   %arrayidx1 = getelementptr i32, ptr %ptr, i64 1
@@ -340,7 +340,7 @@ define void @non_consecutive(ptr %ptr) {
 define void @more_bytes(ptr dereferenceable(8) %ptr) {
 ; CHECK: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(none)
 ; CHECK-LABEL: define {{[^@]+}}@more_bytes
-; CHECK-SAME: (ptr nocapture nofree noundef nonnull readnone align 4 dereferenceable(16) [[PTR:%.*]]) #[[ATTR2]] {
+; CHECK-SAME: (ptr nofree noundef nonnull readnone align 4 captures(none) dereferenceable(16) [[PTR:%.*]]) #[[ATTR2]] {
 ; CHECK-NEXT:    ret void
 ;
   %arrayidx3 = getelementptr i32, ptr %ptr, i64 3
@@ -358,7 +358,7 @@ define void @more_bytes(ptr dereferenceable(8) %ptr) {
 define void @more_bytes_and_not_null(ptr dereferenceable_or_null(8) %ptr) {
 ; CHECK: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(none)
 ; CHECK-LABEL: define {{[^@]+}}@more_bytes_and_not_null
-; CHECK-SAME: (ptr nocapture nofree noundef nonnull readnone align 4 dereferenceable(16) [[PTR:%.*]]) #[[ATTR2]] {
+; CHECK-SAME: (ptr nofree noundef nonnull readnone align 4 captures(none) dereferenceable(16) [[PTR:%.*]]) #[[ATTR2]] {
 ; CHECK-NEXT:    ret void
 ;
   %arrayidx3 = getelementptr i32, ptr %ptr, i64 3
@@ -376,7 +376,7 @@ define void @more_bytes_and_not_null(ptr dereferenceable_or_null(8) %ptr) {
 define void @better_bytes(ptr dereferenceable(100) %ptr) {
 ; CHECK: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(none)
 ; CHECK-LABEL: define {{[^@]+}}@better_bytes
-; CHECK-SAME: (ptr nocapture nofree noundef nonnull readnone align 4 dereferenceable(100) [[PTR:%.*]]) #[[ATTR2]] {
+; CHECK-SAME: (ptr nofree noundef nonnull readnone align 4 captures(none) dereferenceable(100) [[PTR:%.*]]) #[[ATTR2]] {
 ; CHECK-NEXT:    ret void
 ;
   %arrayidx3 = getelementptr i32, ptr %ptr, i64 3
@@ -392,7 +392,7 @@ define void @better_bytes(ptr dereferenceable(100) %ptr) {
 define void @bitcast(ptr %arg) {
 ; CHECK: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(none)
 ; CHECK-LABEL: define {{[^@]+}}@bitcast
-; CHECK-SAME: (ptr nocapture nofree nonnull readnone align 4 dereferenceable(8) [[ARG:%.*]]) #[[ATTR2]] {
+; CHECK-SAME: (ptr nofree nonnull readnone align 4 captures(none) dereferenceable(8) [[ARG:%.*]]) #[[ATTR2]] {
 ; CHECK-NEXT:    ret void
 ;
   %ptr = bitcast ptr %arg to ptr
@@ -405,7 +405,7 @@ define void @bitcast(ptr %arg) {
 define void @bitcast_different_sizes(ptr %arg1, ptr %arg2) {
 ; CHECK: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(none)
 ; CHECK-LABEL: define {{[^@]+}}@bitcast_different_sizes
-; CHECK-SAME: (ptr nocapture nofree nonnull readnone align 4 dereferenceable(12) [[ARG1:%.*]], ptr nocapture nofree noundef nonnull readnone align 4 dereferenceable(16) [[ARG2:%.*]]) #[[ATTR2]] {
+; CHECK-SAME: (ptr nofree nonnull readnone align 4 captures(none) dereferenceable(12) [[ARG1:%.*]], ptr nofree noundef nonnull readnone align 4 captures(none) dereferenceable(16) [[ARG2:%.*]]) #[[ATTR2]] {
 ; CHECK-NEXT:    ret void
 ;
   %ptr1 = bitcast ptr %arg1 to ptr
@@ -424,7 +424,7 @@ define void @bitcast_different_sizes(ptr %arg1, ptr %arg2) {
 define void @negative_offset(ptr %arg) {
 ; CHECK: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(none)
 ; CHECK-LABEL: define {{[^@]+}}@negative_offset
-; CHECK-SAME: (ptr nocapture nofree nonnull readnone align 4 dereferenceable(4) [[ARG:%.*]]) #[[ATTR2]] {
+; CHECK-SAME: (ptr nofree nonnull readnone align 4 captures(none) dereferenceable(4) [[ARG:%.*]]) #[[ATTR2]] {
 ; CHECK-NEXT:    ret void
 ;
   %ptr = bitcast ptr %arg to ptr
@@ -437,7 +437,7 @@ define void @negative_offset(ptr %arg) {
 define void @stores(ptr %arg) {
 ; CHECK: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: write)
 ; CHECK-LABEL: define {{[^@]+}}@stores
-; CHECK-SAME: (ptr nocapture nofree nonnull writeonly align 4 dereferenceable(8) [[ARG:%.*]]) #[[ATTR4:[0-9]+]] {
+; CHECK-SAME: (ptr nofree nonnull writeonly align 4 captures(none) dereferenceable(8) [[ARG:%.*]]) #[[ATTR4:[0-9]+]] {
 ; CHECK-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr float, ptr [[ARG]], i64 1
 ; CHECK-NEXT:    store float 1.000000e+00, ptr [[ARG]], align 4
 ; CHECK-NEXT:    store float 2.000000e+00, ptr [[ARRAYIDX1]], align 4
@@ -453,7 +453,7 @@ define void @stores(ptr %arg) {
 define void @load_store(ptr %arg) {
 ; CHECK: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: write)
 ; CHECK-LABEL: define {{[^@]+}}@load_store
-; CHECK-SAME: (ptr nocapture nofree nonnull writeonly align 4 dereferenceable(8) [[ARG:%.*]]) #[[ATTR4]] {
+; CHECK-SAME: (ptr nofree nonnull writeonly align 4 captures(none) dereferenceable(8) [[ARG:%.*]]) #[[ATTR4]] {
 ; CHECK-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr float, ptr [[ARG]], i64 1
 ; CHECK-NEXT:    store float 2.000000e+00, ptr [[ARRAYIDX1]], align 4
 ; CHECK-NEXT:    ret void
@@ -468,7 +468,7 @@ define void @load_store(ptr %arg) {
 define void @different_size1(ptr %arg) {
 ; CHECK: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: write)
 ; CHECK-LABEL: define {{[^@]+}}@different_size1
-; CHECK-SAME: (ptr nocapture nofree noundef nonnull writeonly align 8 dereferenceable(8) [[ARG:%.*]]) #[[ATTR4]] {
+; CHECK-SAME: (ptr nofree noundef nonnull writeonly align 8 captures(none) dereferenceable(8) [[ARG:%.*]]) #[[ATTR4]] {
 ; CHECK-NEXT:    store double 0.000000e+00, ptr [[ARG]], align 8
 ; CHECK-NEXT:    store i32 0, ptr [[ARG]], align 8
 ; CHECK-NEXT:    ret void
@@ -482,7 +482,7 @@ define void @different_size1(ptr %arg) {
 define void @different_size2(ptr %arg) {
 ; CHECK: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: write)
 ; CHECK-LABEL: define {{[^@]+}}@different_size2
-; CHECK-SAME: (ptr nocapture nofree noundef nonnull writeonly align 8 dereferenceable(8) [[ARG:%.*]]) #[[ATTR4]] {
+; CHECK-SAME: (ptr nofree noundef nonnull writeonly align 8 captures(none) dereferenceable(8) [[ARG:%.*]]) #[[ATTR4]] {
 ; CHECK-NEXT:    store i32 0, ptr [[ARG]], align 8
 ; CHECK-NEXT:    store double 0.000000e+00, ptr [[ARG]], align 8
 ; CHECK-NEXT:    ret void
@@ -513,7 +513,7 @@ define void @different_size2(ptr %arg) {
 define i32 @require_cfg_analysis(i32 %c, ptr %p) {
 ; CHECK: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: write)
 ; CHECK-LABEL: define {{[^@]+}}@require_cfg_analysis
-; CHECK-SAME: (i32 [[C:%.*]], ptr nocapture nofree nonnull writeonly align 4 dereferenceable(4) [[P:%.*]]) #[[ATTR4]] {
+; CHECK-SAME: (i32 [[C:%.*]], ptr nofree nonnull writeonly align 4 captures(none) dereferenceable(4) [[P:%.*]]) #[[ATTR4]] {
 ; CHECK-NEXT:    [[TOBOOL1:%.*]] = icmp eq i32 [[C]], 0
 ; CHECK-NEXT:    br i1 [[TOBOOL1]], label [[L1:%.*]], label [[L2:%.*]]
 ; CHECK:       l1:
diff --git a/llvm/test/Transforms/Attributor/dereferenceable-2.ll b/llvm/test/Transforms/Attributor/dereferenceable-2.ll
index 435544dc8e844..52129e8261a2c 100644
--- a/llvm/test/Transforms/Attributor/dereferenceable-2.ll
+++ b/llvm/test/Transforms/Attributor/dereferenceable-2.ll
@@ -8,7 +8,7 @@
 define <4 x double> @PR21780(ptr %ptr) {
 ; CHECK: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: read)
 ; CHECK-LABEL: define {{[^@]+}}@PR21780
-; CHECK-SAME: (ptr nocapture nofree noundef nonnull readonly align 8 dereferenceable(32) [[PTR:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-SAME: (ptr nofree noundef nonnull readonly align 8 captures(none) dereferenceable(32) [[PTR:%.*]]) #[[ATTR0:[0-9]+]] {
 ; CHECK-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds double, ptr [[PTR]], i64 1
 ; CHECK-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds double, ptr [[PTR]], i64 2
 ; CHECK-NEXT:    [[ARRAYIDX3:%.*]] = getelementptr inbounds double, ptr [[PTR]], i64 3
@@ -46,7 +46,7 @@ define <4 x double> @PR21780(ptr %ptr) {
 define double @PR21780_only_access3_with_inbounds(ptr %ptr) {
 ; CHECK: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: read)
 ; CHECK-LABEL: define {{[^@]+}}@PR21780_only_access3_with_inbounds
-; CHECK-SAME: (ptr nocapture nofree nonnull readonly align 8 dereferenceable(32) [[PTR:%.*]]) #[[ATTR0]] {
+; CHECK-SAME: (ptr nofree nonnull readonly align 8 captures(none) dereferenceable(32) [[PTR:%.*]]) #[[ATTR0]] {
 ; CHECK-NEXT:    [[ARRAYIDX3:%.*]] = getelementptr inbounds double, ptr [[PTR]], i64 3
 ; CHECK-NEXT:    [[T3:%.*]] = load double, ptr [[ARRAYIDX3]], align 8
 ; CHECK-NEXT:    ret double [[T3]]
@@ -60,7 +60,7 @@ define double @PR21780_only_access3_with_inbounds(ptr %ptr) {
 define double @PR21780_only_access3_without_inbounds(ptr %ptr) {
 ; CHECK: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: read)
 ; CHECK-LABEL: define {{[^@]+}}@PR21780_only_access3_without_inbounds
-; CHECK-SAME: (ptr nocapture nofree readonly align 8 [[PTR:%.*]]) #[[ATTR0]] {
+; CHECK-SAME: (ptr nofree readonly align 8 captures(none) [[PTR:%.*]]) #[[ATTR0]] {
 ; CHECK-NEXT:    [[ARRAYIDX3:%.*]] = getelementptr double, ptr [[PTR]], i64 3
 ; CHECK-NEXT:    [[T3:%.*]] = load double, ptr [[ARRAYIDX3]], align 8
 ; CHECK-NEXT:    ret double [[T3]]
@@ -73,7 +73,7 @@ define double @PR21780_only_access3_without_inbounds(ptr %ptr) {
 define double @PR21780_without_inbounds(ptr %ptr) {
 ; CHECK: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: read)
 ; CHECK-LABEL: define {{[^@]+}}@PR21780_without_inbounds
-; CHECK-SAME: (ptr nocapture nofree noundef nonnull readonly align 8 dereferenceable(32) [[PTR:%.*]]) #[[ATTR0]] {
+; CHECK-SAME: (ptr nofree noundef nonnull readonly align 8 captures(none) dereferenceable(32) [[PTR:%.*]]) #[[ATTR0]] {
 ; CHECK-NEXT:    [[ARRAYIDX3:%.*]] = getelementptr double, ptr [[PTR]], i64 3
 ; CHECK-NEXT:    [[T3:%.*]] = load double, ptr [[ARRAYIDX3]], align 8
 ; CHECK-NEXT:    ret double [[T3]]
@@ -96,7 +96,7 @@ define double @PR21780_without_inbounds(ptr %ptr) {
 define void @gep0(ptr %unused, ptr %other, ptr %ptr) {
 ; CHECK: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: readwrite)
 ; CHECK-LABEL: define {{[^@]+}}@gep0
-; CHECK-SAME: (ptr nocapture nofree readnone [[UNUSED:%.*]], ptr nocapture nofree noundef nonnull writeonly dereferenceable(1) [[OTHER:%.*]], ptr nocapture nofree nonnull readonly dereferenceable(3) [[PTR:%.*]]) #[[ATTR1:[0-9]+]] {
+; CHECK-SAME: (ptr nofree readnone captures(none) [[UNUSED:%.*]], ptr nofree noundef nonnull writeonly captures(none) dereferenceable(1) [[OTHER:%.*]], ptr nofree nonnull readonly captures(none) dereferenceable(3) [[PTR:%.*]]) #[[ATTR1:[0-9]+]] {
 ; CHECK-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr i8, ptr [[PTR]], i64 2
 ; CHECK-NEXT:    [[T2:%.*]] = load i8, ptr [[ARRAYIDX2]], align 1
 ; CHECK-NEXT:    store i8 [[T2]], ptr [[OTHER]], align 1
@@ -118,7 +118,7 @@ define void @gep0(ptr %unused, ptr %other, ptr %ptr) {
 define void @ordering(ptr %ptr1, ptr %ptr2) {
 ; CHECK: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(none)
 ; CHECK-LABEL: define {{[^@]+}}@ordering
-; CHECK-SAME: (ptr nocapture nofree noundef nonnull readnone dereferenceable(3) [[PTR1:%.*]], ptr nocapture nofree nonnull readnone align 4 dereferenceable(8) [[PTR2:%.*]]) #[[ATTR2:[0-9]+]] {
+; CHECK-SAME: (ptr nofree noundef nonnull readnone captures(none) dereferenceable(3) [[PTR1:%.*]], ptr nofree nonnull readnone align 4 captures(none) dereferenceable(8) [[PTR2:%.*]]) #[[ATTR2:[0-9]+]] {
 ; CHECK-NEXT:    ret void
 ;
   %a20 = getelementptr i32, ptr %ptr2, i64 0
@@ -138,7 +138,7 @@ define void @ordering(ptr %ptr1, ptr %ptr2) {
 define void @not_entry_but_guaranteed_to_execute(ptr %ptr) {
 ; CHECK: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(none)
 ; CHECK-LABEL: define {{[^@]+}}@not_entry_but_guaranteed_to_execute
-; CHECK-SAME: (ptr nocapture nofree nonnull readnone dereferenceable(3) [[PTR:%.*]]) #[[ATTR2]] {
+; CHECK-SAME: (ptr nofree nonnull readnone captures(none) dereferenceable(3) [[PTR:%.*]]) #[[ATTR2]] {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    br label [[EXIT:%.*]]
 ; CHECK:       exit:
@@ -160,7 +160,7 @@ exit:
 define void @not_entry_not_guaranteed_to_execute(ptr %ptr, i1 %cond) {
 ; CHECK: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(none)
 ; CHECK-LABEL: define {{[^@]+}}@not_entry_not_guaranteed_to_execute
-; CHECK-SAME: (ptr nocapture nofree readnone [[PTR:%.*]], i1 noundef [[COND:%.*]]) #[[ATTR2]] {
+; CHECK-SAME: (ptr nofree readnone captures(none) [[PTR:%.*]], i1 noundef [[COND:%.*]]) #[[ATTR2]] {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    br i1 [[COND]], label [[LOADS:%.*]], label [[EXIT:%.*]]
 ; CHECK:       loads:
@@ -186,7 +186,7 @@ exit:
 define void @partial_in_entry(ptr %ptr, i1 %cond) {
 ; CHECK: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(none)
 ; CHECK-LABEL: define {{[^@]+}}@partial_in_entry
-; CHECK-SAME: (ptr nocapture nofree noundef nonnull readnone align 2 dereferenceable(4) [[PTR:%.*]], i1 noundef [[COND:%.*]]) #[[ATTR2]] {
+; CHECK-SAME: (ptr nofree noundef nonnull readnone align 2 captures(none) dereferenceable(4) [[PTR:%.*]], i1 noundef [[COND:%.*]]) #[[ATTR2]] {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    br i1 [[COND]], label [[LOADS:%.*]], label [[EXIT:%.*]]
 ; CHECK:       loads:
@@ -231,7 +231,7 @@ define void @volatile_is_not_dereferenceable(ptr %ptr) {
 define void @atomic_is_alright(ptr %ptr) {
 ; CHECK: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(none)
 ; CHECK-LABEL: define {{[^@]+}}@atomic_is_alright
-; CHECK-SAME: (ptr nocapture nofree nonnull readnone align 2 dereferenceable(6) [[PTR:%.*]]) #[[ATTR2]] {
+; CHECK-SAME: (ptr nofree nonnull readnone align 2 captures(none) dereferenceable(6) [[PTR:%.*]]) #[[ATTR2]] {
 ; CHECK-NEXT:    ret void
 ;
   %arrayidx0 = getelementptr i16, ptr %ptr, i64 0
@@ -247,7 +247,7 @@ declare void @may_not_return()
 
 define void @not_guaranteed_to_transfer_execution(ptr %ptr) {
 ; CHECK-LABEL: define {{[^@]+}}@not_guaranteed_to_transfer_execution
-; CHECK-SAME: (ptr nocapture nofree nonnull readnone align 2 dereferenceable(2) [[PTR:%.*]]) {
+; CHECK-SAME: (ptr nofree nonnull readnone align 2 captures(none) dereferenceable(2) [[PTR:%.*]]) {
 ; CHECK-NEXT:    call void @may_not_return()
 ; CHECK-NEXT:    ret void
 ;
@@ -266,7 +266,7 @@ define void @not_guaranteed_to_transfer_execution(ptr %ptr) {
 define void @variable_gep_index(ptr %unused, ptr %ptr, i64 %variable_index) {
 ; CHECK: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(none)
 ; CHECK-LABEL: define {{[^@]+}}@variable_gep_index
-; CHECK-SAME: (ptr nocapture nofree readnone [[UNUSED:%.*]], ptr nocapture nofree noundef nonnull readnone dereferenceable(1) [[PTR:%.*]], i64 [[VARIABLE_INDEX:%.*]]) #[[ATTR2]] {
+; CHECK-SAME: (ptr nofree readnone captures(none) [[UNUSED:%.*]], ptr nofree noundef nonnull readnone captures(none) dereferenceable(1) [[PTR:%.*]], i64 [[VARIABLE_INDEX:%.*]]) #[[ATTR2]] {
 ; CHECK-NEXT:    ret void
 ;
   %arrayidx1 = getelementptr i8, ptr %ptr, i64 %variable_index
@@ -283,7 +283,7 @@ define void @multi_index_gep(ptr %ptr) {
 ; FIXME: %ptr should be dereferenceable(4)
 ; CHECK: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(none)
 ; CHECK-LABEL: define {{[^@]+}}@multi_index_gep
-; CHECK-SAME: (ptr nocapture nofree nonnull readnone dereferenceable(1) [[PTR:%.*]]) #[[ATTR2]] {
+; CHECK-SAME: (ptr nofree nonnull readnone captures(none) dereferenceable(1) [[PTR:%.*]]) #[[ATTR2]] {
 ; CHECK-NEXT:    ret void
 ;
   %arrayidx00 = getelementptr <4 x i8>, ptr %ptr, i64 0, i64 0
@@ -296,7 +296,7 @@ define void @multi_index_gep(ptr %ptr) {
 define void @not_byte_multiple(ptr %ptr) {
 ; CHECK: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(none)
 ; CHECK-LABEL: define {{[^@]+}}@not_byte_multiple
-; CHECK-SAME: (ptr nocapture nofree nonnull readnone align 2 dereferenceable(2) [[PTR:%.*]]) #[[ATTR2]] {
+; CHECK-SAME: (ptr nofree nonnull readnone align 2 captures(none) dereferenceable(2) [[PTR:%.*]]) #[[ATTR2]] {
 ; CHECK-NEXT:    ret void
 ;
   %arrayidx0 = getelementptr i9, ptr %ptr, i64 0
@@ -309,7 +309,7 @@ define void @not_byte_multiple(ptr %ptr) {
 define void @no_pointer_deref(ptr %ptr) {
 ; CHECK: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(none)
 ; CHECK-LABEL: define {{[^@]+}}@no_pointer_deref
-; CHECK-SAME: (ptr nocapture nofree readnone align 2 [[PTR:%.*]]) #[[ATTR2]] {
+; CHECK-SAME: (ptr nofree readnone align 2 captures(none) [[PTR:%.*]]) #[[ATTR2]] {
 ; CHECK-NEXT:    ret void
 ;
   %arrayidx1 = getelementptr i16, ptr %ptr, i64 1
@@ -324,7 +324,7 @@ define void @no_pointer_deref(ptr %ptr) {
 define void @non_consecutive(ptr %ptr) {
 ; CHECK: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(none)
 ; CHECK-LABEL: define {{[^@]+}}@non_consecutive
-; CHECK-SAME: (ptr nocapture nofree noundef nonnull readnone align 4 dereferenceable(8) [[PTR:%.*]]) #[[ATTR2]] {
+; CHECK-SAME: (ptr nofree noundef nonnull readnone align 4 captures(none) dereferenceable(8) [[PTR:%.*]]) #[[ATTR2]] {
 ; CHECK-NEXT:    ret void
 ;
   %arrayidx1 = getelementptr i32, ptr %ptr, i64 1
@@ -340,7 +340,7 @@ define void @non_consecutive(ptr %ptr) {
 define void @more_bytes(ptr dereferenceable(8) %ptr) {
 ; CHECK: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(none)
 ; CHECK-LABEL: define {{[^@]+}}@more_bytes
-; CHECK-SAME: (ptr nocapture nofree noundef nonnull readnone align 4 dereferenceable(16) [[PTR:%.*]]) #[[ATTR2]] {
+; CHECK-SAME: (ptr nofree noundef nonnull readnone align 4 captures(none) dereferenceable(16) [[PTR:%.*]]) #[[ATTR2]] {
 ; CHECK-NEXT:    ret void
 ;
   %arrayidx3 = getelementptr i32, ptr %ptr, i64 3
@@ -358,7 +358,7 @@ define void @more_bytes(ptr dereferenceable(8) %ptr) {
 define void @more_bytes_and_not_null(ptr dereferenceable_or_null(8) %ptr) {
 ; CHECK: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(none)
 ; CHECK-LABEL: define {{[^@]+}}@more_bytes_and_not_null
-; CHECK-SAME: (ptr nocapture nofree noundef nonnull readnone align 4 dereferenceable(16) [[PTR:%.*]]) #[[ATTR2]] {
+; CHECK-SAME: (ptr nofree noundef nonnull readnone align 4 captures(none) dereferenceable(16) [[PTR:%.*]]) #[[ATTR2]] {
 ; CHECK-NEXT:    ret void
 ;
   %arrayidx3 = getelementptr i32, ptr %ptr, i64 3
@@ -376,7 +376,7 @@ define void @more_bytes_and_not_null(ptr dereferenceable_or_null(8) %ptr) {
 define void @better_bytes(ptr dereferenceable(100) %ptr) {
 ; CHECK: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(none)
 ; CHECK-LABEL: define {{[^@]+}}@better_bytes
-; CHECK-SAME: (ptr nocapture nofree noundef nonnull readnone align 4 dereferenceable(100) [[PTR:%.*]]) #[[ATTR2]] {
+; CHECK-SAME: (ptr nofree noundef nonnull readnone align 4 captures(none) dereferenceable(100) [[PTR:%.*]]) #[[ATTR2]] {
 ; CHECK-NEXT:    ret void
 ;
   %arrayidx3 = getelementptr i32, ptr %ptr, i64 3
@@ -392,7 +392,7 @@ define void @better_bytes(ptr dereferenceable(100) %ptr) {
 define void @bitcast(ptr %arg) {
 ; CHECK: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(none)
 ; CHECK-LABEL: define {{[^@]+}}@bitcast
-; CHECK-SAME: (ptr nocapture nofree nonnull readnone align 4 dereferenceable(8) [[ARG:%.*]]) #[[ATTR2]] {
+; CHECK-SAME: (ptr nofree nonnull readnone align 4 captures(none) dereferenceable(8) [[ARG:%.*]]) #[[ATTR2]] {
 ; CHECK-NEXT:    ret void
 ;
   %ptr = bitcast ptr %arg to ptr
@@ -405,7 +405,7 @@ define void @bitcast(ptr %arg) {
 define void @bitcast_different_sizes(ptr %arg1, ptr %arg2) {
 ; CHECK: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(none)
 ; CHECK-LABEL: define {{[^@]+}}@bitcast_different_sizes
-; CHECK-SAME: (ptr nocapture nofree nonnull readnone align 4 dereferenceable(12) [[ARG1:%.*]], ptr nocapture nofree noundef nonnull readnone align 4 dereferenceable(16) [[ARG2:%.*]]) #[[ATTR2]] {
+; CHECK-SAME: (ptr nofree nonnull readnone align 4 captures(none) dereferenceable(12) [[ARG1:%.*]], ptr nofree noundef nonnull readnone align 4 captures(none) dereferenceable(16) [[ARG2:%.*]]) #[[ATTR2]] {
 ; CHECK-NEXT:    ret void
 ;
   %ptr1 = bitcast ptr %arg1 to ptr
@@ -424,7 +424,7 @@ define void @bitcast_different_sizes(ptr %arg1, ptr %arg2) {
 define void @negative_offset(ptr %arg) {
 ; CHECK: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(none)
 ; CHECK-LABEL: define {{[^@]+}}@negative_offset
-; CHECK-SAME: (ptr nocapture nofree nonnull readnone align 4 dereferenceable(4) [[ARG:%.*]]) #[[ATTR2]] {
+; CHECK-SAME: (ptr nofree nonnull readnone align 4 captures(none) dereferenceable(4) [[ARG:%.*]]) #[[ATTR2]] {
 ; CHECK-NEXT:    ret void
 ;
   %ptr = bitcast ptr %arg to ptr
@@ -437,7 +437,7 @@ define void @negative_offset(ptr %arg) {
 define void @stores(ptr %arg) {
 ; CHECK: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: write)
 ; CHECK-LABEL: define {{[^@]+}}@stores
-; CHECK-SAME: (ptr nocapture nofree nonnull writeonly align 4 dereferenceable(8) [[ARG:%.*]]) #[[ATTR4:[0-9]+]] {
+; CHECK-SAME: (ptr nofree nonnull writeonly align 4 captures(none) dereferenceable(8) [[ARG:%.*]]) #[[ATTR4:[0-9]+]] {
 ; CHECK-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr float, ptr [[ARG]], i64 1
 ; CHECK-NEXT:    store float 1.000000e+00, ptr [[ARG]], align 4
 ; CHECK-NEXT:    store float 2.000000e+00, ptr [[ARRAYIDX1]], align 4
@@ -453,7 +453,7 @@ define void @stores(ptr %arg) {
 define void @load_store(ptr %arg) {
 ; CHECK: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: write)
 ; CHECK-LABEL: define {{[^@]+}}@load_store
-; CHECK-SAME: (ptr nocapture nofree nonnull writeonly align 4 dereferenceable(8) [[ARG:%.*]]) #[[ATTR4]] {
+; CHECK-SAME: (ptr nofree nonnull writeonly align 4 captures(none) dereferenceable(8) [[ARG:%.*]]) #[[ATTR4]] {
 ; CHECK-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr float, ptr [[ARG]], i64 1
 ; CHECK-NEXT:    store float 2.000000e+00, ptr [[ARRAYIDX1]], align 4
 ; CHECK-NEXT:    ret void
@@ -468,7 +468,7 @@ define void @load_store(ptr %arg) {
 define void @different_size1(ptr %arg) {
 ; CHECK: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: write)
 ; CHECK-LABEL: define {{[^@]+}}@different_size1
-; CHECK-SAME: (ptr nocapture nofree noundef nonnull writeonly align 8 dereferenceable(8) [[ARG:%.*]]) #[[ATTR4]] {
+; CHECK-SAME: (ptr nofree noundef nonnull writeonly align 8 captures(none) dereferenceable(8) [[ARG:%.*]]) #[[ATTR4]] {
 ; CHECK-NEXT:    store double 0.000000e+00, ptr [[ARG]], align 8
 ; CHECK-NEXT:    store i32 0, ptr [[ARG]], align 8
 ; CHECK-NEXT:    ret void
@@ -482,7 +482,7 @@ define void @different_size1(ptr %arg) {
 define void @different_size2(ptr %arg) {
 ; CHECK: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: write)
 ; CHECK-LABEL: define {{[^@]+}}@different_size2
-; CHECK-SAME: (ptr nocapture nofree noundef nonnull writeonly align 8 dereferenceable(8) [[ARG:%.*]]) #[[ATTR4]] {
+; CHECK-SAME: (ptr nofree noundef nonnull writeonly align 8 captures(none) dereferenceable(8) [[ARG:%.*]]) #[[ATTR4]] {
 ; CHECK-NEXT:    store i32 0, ptr [[ARG]], align 8
 ; CHECK-NEXT:    store double 0.000000e+00, ptr [[ARG]], align 8
 ; CHECK-NEXT:    ret void
@@ -513,7 +513,7 @@ define void @different_size2(ptr %arg) {
 define i32 @require_cfg_analysis(i32 %c, ptr %p) {
 ; CHECK: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: write)
 ; CHECK-LABEL: define {{[^@]+}}@require_cfg_analysis
-; CHECK-SAME: (i32 [[C:%.*]], ptr nocapture nofree nonnull writeonly align 4 dereferenceable(4) [[P:%.*]]) #[[ATTR4]] {
+; CHECK-SAME: (i32 [[C:%.*]], ptr nofree nonnull writeonly align 4 captures(none) dereferenceable(4) [[P:%.*]]) #[[ATTR4]] {
 ; CHECK-NEXT:    [[TOBOOL1:%.*]] = icmp eq i32 [[C]], 0
 ; CHECK-NEXT:    br i1 [[TOBOOL1]], label [[L1:%.*]], label [[L2:%.*]]
 ; CHECK:       l1:
diff --git a/llvm/test/Transforms/Attributor/heap_to_stack.ll b/llvm/test/Transforms/Attributor/heap_to_stack.ll
index 33ac066e43d09..fad4acb86e101 100644
--- a/llvm/test/Transforms/Attributor/heap_to_stack.ll
+++ b/llvm/test/Transforms/Attributor/heap_to_stack.ll
@@ -34,7 +34,7 @@ declare void @llvm.lifetime.start.p0(i64, ptr nocapture) nounwind
 ;.
 define void @h2s_value_simplify_interaction(i1 %c, ptr %A) {
 ; CHECK-LABEL: define {{[^@]+}}@h2s_value_simplify_interaction
-; CHECK-SAME: (i1 [[C:%.*]], ptr nocapture nofree readnone [[A:%.*]]) {
+; CHECK-SAME: (i1 [[C:%.*]], ptr nofree readnone captures(none) [[A:%.*]]) {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[ADD:%.*]] = add i64 2, 2
 ; CHECK-NEXT:    [[M:%.*]] = tail call noalias align 16 ptr @malloc(i64 noundef [[ADD]])
@@ -46,13 +46,13 @@ define void @h2s_value_simplify_interaction(i1 %c, ptr %A) {
 ; CHECK:       f2:
 ; CHECK-NEXT:    [[L:%.*]] = load i8, ptr [[M]], align 16
 ; CHECK-NEXT:    call void @usei8(i8 [[L]])
-; CHECK-NEXT:    call void @no_sync_func(ptr noalias nocapture nofree noundef nonnull align 16 dereferenceable(1) [[M]]) #[[ATTR11:[0-9]+]]
+; CHECK-NEXT:    call void @no_sync_func(ptr noalias nofree noundef nonnull align 16 captures(none) dereferenceable(1) [[M]]) #[[ATTR11:[0-9]+]]
 ; CHECK-NEXT:    br label [[J]]
 ; CHECK:       dead:
 ; CHECK-NEXT:    unreachable
 ; CHECK:       j:
 ; CHECK-NEXT:    [[PHI:%.*]] = phi ptr [ [[M]], [[F]] ], [ null, [[F2]] ]
-; CHECK-NEXT:    tail call void @no_sync_func(ptr nocapture nofree noundef align 16 [[PHI]]) #[[ATTR11]]
+; CHECK-NEXT:    tail call void @no_sync_func(ptr nofree noundef align 16 captures(none) [[PHI]]) #[[ATTR11]]
 ; CHECK-NEXT:    ret void
 ;
 entry:
@@ -83,10 +83,10 @@ j:
 
 define void @nofree_arg_only(ptr %p1, ptr %p2) {
 ; CHECK-LABEL: define {{[^@]+}}@nofree_arg_only
-; CHECK-SAME: (ptr nocapture nofree [[P1:%.*]], ptr nocapture [[P2:%.*]]) {
+; CHECK-SAME: (ptr nofree captures(none) [[P1:%.*]], ptr captures(none) [[P2:%.*]]) {
 ; CHECK-NEXT:  bb:
-; CHECK-NEXT:    tail call void @free(ptr nocapture [[P2]])
-; CHECK-NEXT:    tail call void @nofree_func(ptr nocapture nofree [[P1]])
+; CHECK-NEXT:    tail call void @free(ptr captures(none) [[P2]])
+; CHECK-NEXT:    tail call void @nofree_func(ptr nofree captures(none) [[P1]])
 ; CHECK-NEXT:    ret void
 ;
 bb:
@@ -101,9 +101,9 @@ define void @test1() {
 ; CHECK-LABEL: define {{[^@]+}}@test1() {
 ; CHECK-NEXT:  bb:
 ; CHECK-NEXT:    [[I:%.*]] = tail call noalias ptr @malloc(i64 noundef 4)
-; CHECK-NEXT:    tail call void @nocapture_func_frees_pointer(ptr noalias nocapture [[I]])
+; CHECK-NEXT:    tail call void @nocapture_func_frees_pointer(ptr noalias captures(none) [[I]])
 ; CHECK-NEXT:    tail call void (...) @func_throws()
-; CHECK-NEXT:    tail call void @free(ptr noalias nocapture [[I]])
+; CHECK-NEXT:    tail call void @free(ptr noalias captures(none) [[I]])
 ; CHECK-NEXT:    ret void
 ;
 bb:
@@ -121,7 +121,7 @@ define void @test2() {
 ; CHECK-NEXT:  bb:
 ; CHECK-NEXT:    [[I:%.*]] = tail call noalias ptr @malloc(i64 noundef 4)
 ; CHECK-NEXT:    tail call void @sync_func(ptr [[I]])
-; CHECK-NEXT:    tail call void @free(ptr nocapture [[I]])
+; CHECK-NEXT:    tail call void @free(ptr captures(none) [[I]])
 ; CHECK-NEXT:    ret void
 ;
 bb:
@@ -137,7 +137,7 @@ define void @test3() {
 ; CHECK-LABEL: define {{[^@]+}}@test3() {
 ; CHECK-NEXT:  bb:
 ; CHECK-NEXT:    [[I_H2S:%.*]] = alloca i8, i64 4, align 1
-; CHECK-NEXT:    tail call void @no_sync_func(ptr noalias nocapture nofree [[I_H2S]])
+; CHECK-NEXT:    tail call void @no_sync_func(ptr noalias nofree captures(none) [[I_H2S]])
 ; CHECK-NEXT:    ret void
 ;
 bb:
@@ -149,10 +149,10 @@ bb:
 
 define void @test3a(ptr %p) {
 ; CHECK-LABEL: define {{[^@]+}}@test3a
-; CHECK-SAME: (ptr nocapture [[P:%.*]]) {
+; CHECK-SAME: (ptr captures(none) [[P:%.*]]) {
 ; CHECK-NEXT:  bb:
 ; CHECK-NEXT:    [[I_H2S:%.*]] = alloca i8, i64 4, align 1
-; CHECK-NEXT:    tail call void @nofree_arg_only(ptr noalias nocapture nofree [[I_H2S]], ptr nocapture [[P]])
+; CHECK-NEXT:    tail call void @nofree_arg_only(ptr noalias nofree captures(none) [[I_H2S]], ptr captures(none) [[P]])
 ; CHECK-NEXT:    ret void
 ;
 bb:
@@ -166,10 +166,10 @@ declare noalias ptr @aligned_alloc(i64 allocalign, i64) allockind("alloc,uniniti
 
 define void @test3b(ptr %p) {
 ; CHECK-LABEL: define {{[^@]+}}@test3b
-; CHECK-SAME: (ptr nocapture [[P:%.*]]) {
+; CHECK-SAME: (ptr captures(none) [[P:%.*]]) {
 ; CHECK-NEXT:  bb:
 ; CHECK-NEXT:    [[I_H2S:%.*]] = alloca i8, i64 128, align 32
-; CHECK-NEXT:    tail call void @nofree_arg_only(ptr noalias nocapture nofree [[I_H2S]], ptr nocapture [[P]])
+; CHECK-NEXT:    tail call void @nofree_arg_only(ptr noalias nofree captures(none) [[I_H2S]], ptr captures(none) [[P]])
 ; CHECK-NEXT:    ret void
 ;
 bb:
@@ -185,7 +185,7 @@ define void @test3c(i64 %alignment) {
 ; CHECK-SAME: (i64 [[ALIGNMENT:%.*]]) {
 ; CHECK-NEXT:  bb:
 ; CHECK-NEXT:    [[I:%.*]] = tail call noalias ptr @aligned_alloc(i64 [[ALIGNMENT]], i64 noundef 128)
-; CHECK-NEXT:    tail call void @free(ptr noalias nocapture [[I]])
+; CHECK-NEXT:    tail call void @free(ptr noalias captures(none) [[I]])
 ; CHECK-NEXT:    ret void
 ;
 bb:
@@ -197,11 +197,11 @@ bb:
 ; leave alone a constant-but-invalid alignment
 define void @test3d(ptr %p) {
 ; CHECK-LABEL: define {{[^@]+}}@test3d
-; CHECK-SAME: (ptr nocapture [[P:%.*]]) {
+; CHECK-SAME: (ptr captures(none) [[P:%.*]]) {
 ; CHECK-NEXT:  bb:
 ; CHECK-NEXT:    [[I:%.*]] = tail call noalias ptr @aligned_alloc(i64 noundef 33, i64 noundef 128)
-; CHECK-NEXT:    tail call void @nofree_arg_only(ptr noalias nocapture nofree [[I]], ptr nocapture [[P]])
-; CHECK-NEXT:    tail call void @free(ptr noalias nocapture [[I]])
+; CHECK-NEXT:    tail call void @nofree_arg_only(ptr noalias nofree captures(none) [[I]], ptr captures(none) [[P]])
+; CHECK-NEXT:    tail call void @free(ptr noalias captures(none) [[I]])
 ; CHECK-NEXT:    ret void
 ;
 bb:
@@ -218,7 +218,7 @@ define void @test0() {
 ; CHECK-NEXT:  bb:
 ; CHECK-NEXT:    [[I_H2S:%.*]] = alloca i8, i64 8, align 1
 ; CHECK-NEXT:    call void @llvm.memset.p0.i64(ptr [[I_H2S]], i8 0, i64 8, i1 false)
-; CHECK-NEXT:    tail call void @no_sync_func(ptr noalias nocapture nofree [[I_H2S]])
+; CHECK-NEXT:    tail call void @no_sync_func(ptr noalias nofree captures(none) [[I_H2S]])
 ; CHECK-NEXT:    ret void
 ;
 bb:
@@ -233,7 +233,7 @@ define void @test4() {
 ; CHECK-LABEL: define {{[^@]+}}@test4() {
 ; CHECK-NEXT:  bb:
 ; CHECK-NEXT:    [[I_H2S:%.*]] = alloca i8, i64 4, align 1
-; CHECK-NEXT:    tail call void @nofree_func(ptr noalias nocapture nofree [[I_H2S]])
+; CHECK-NEXT:    tail call void @nofree_func(ptr noalias nofree captures(none) [[I_H2S]])
 ; CHECK-NEXT:    ret void
 ;
 bb:
@@ -247,16 +247,16 @@ bb:
 
 define void @test5(i32 %arg, ptr %p) {
 ; CHECK-LABEL: define {{[^@]+}}@test5
-; CHECK-SAME: (i32 [[ARG:%.*]], ptr nocapture [[P:%.*]]) {
+; CHECK-SAME: (i32 [[ARG:%.*]], ptr captures(none) [[P:%.*]]) {
 ; CHECK-NEXT:  bb:
 ; CHECK-NEXT:    [[I_H2S:%.*]] = alloca i8, i64 4, align 1
 ; CHECK-NEXT:    [[I1:%.*]] = icmp eq i32 [[ARG]], 0
 ; CHECK-NEXT:    br i1 [[I1]], label [[BB3:%.*]], label [[BB2:%.*]]
 ; CHECK:       bb2:
-; CHECK-NEXT:    tail call void @nofree_func(ptr noalias nocapture nofree [[I_H2S]])
+; CHECK-NEXT:    tail call void @nofree_func(ptr noalias nofree captures(none) [[I_H2S]])
 ; CHECK-NEXT:    br label [[BB4:%.*]]
 ; CHECK:       bb3:
-; CHECK-NEXT:    tail call void @nofree_arg_only(ptr noalias nocapture nofree [[I_H2S]], ptr nocapture [[P]])
+; CHECK-NEXT:    tail call void @nofree_arg_only(ptr noalias nofree captures(none) [[I_H2S]], ptr captures(none) [[P]])
 ; CHECK-NEXT:    br label [[BB4]]
 ; CHECK:       bb4:
 ; CHECK-NEXT:    ret void
@@ -289,7 +289,7 @@ define void @test6(i32 %arg) {
 ; CHECK-NEXT:    [[I1:%.*]] = icmp eq i32 [[ARG]], 0
 ; CHECK-NEXT:    br i1 [[I1]], label [[BB3:%.*]], label [[BB2:%.*]]
 ; CHECK:       bb2:
-; CHECK-NEXT:    tail call void @nofree_func(ptr noalias nocapture nofree [[I_H2S]])
+; CHECK-NEXT:    tail call void @nofree_func(ptr noalias nofree captures(none) [[I_H2S]])
 ; CHECK-NEXT:    br label [[BB4:%.*]]
 ; CHECK:       bb3:
 ; CHECK-NEXT:    br label [[BB4]]
@@ -336,10 +336,10 @@ define void @test8() {
 ; CHECK-LABEL: define {{[^@]+}}@test8() {
 ; CHECK-NEXT:  bb:
 ; CHECK-NEXT:    [[I:%.*]] = tail call noalias ptr @malloc(i64 noundef 4)
-; CHECK-NEXT:    tail call void @no_sync_func(ptr nocapture nofree [[I]])
+; CHECK-NEXT:    tail call void @no_sync_func(ptr nofree captures(none) [[I]])
 ; CHECK-NEXT:    store i32 10, ptr [[I]], align 4
 ; CHECK-NEXT:    tail call void @foo(ptr nonnull align 4 dereferenceable(4) [[I]])
-; CHECK-NEXT:    tail call void @free(ptr nocapture nonnull align 4 dereferenceable(4) [[I]])
+; CHECK-NEXT:    tail call void @free(ptr nonnull align 4 captures(none) dereferenceable(4) [[I]])
 ; CHECK-NEXT:    ret void
 ;
 bb:
@@ -357,10 +357,10 @@ define void @test9() {
 ; CHECK-LABEL: define {{[^@]+}}@test9() {
 ; CHECK-NEXT:  bb:
 ; CHECK-NEXT:    [[I:%.*]] = tail call noalias ptr @malloc(i64 noundef 4)
-; CHECK-NEXT:    tail call void @no_sync_func(ptr nocapture nofree [[I]])
+; CHECK-NEXT:    tail call void @no_sync_func(ptr nofree captures(none) [[I]])
 ; CHECK-NEXT:    store i32 10, ptr [[I]], align 4
 ; CHECK-NEXT:    tail call void @foo_nounw(ptr nofree nonnull align 4 dereferenceable(4) [[I]]) #[[ATTR11]]
-; CHECK-NEXT:    tail call void @free(ptr nocapture nonnull align 4 dereferenceable(4) [[I]])
+; CHECK-NEXT:    tail call void @free(ptr nonnull align 4 captures(none) dereferenceable(4) [[I]])
 ; CHECK-NEXT:    ret void
 ;
 bb:
@@ -379,7 +379,7 @@ define i32 @test10() {
 ; CHECK-LABEL: define {{[^@]+}}@test10() {
 ; CHECK-NEXT:  bb:
 ; CHECK-NEXT:    [[I_H2S:%.*]] = alloca i8, i64 4, align 1
-; CHECK-NEXT:    tail call void @no_sync_func(ptr noalias nocapture nofree [[I_H2S]])
+; CHECK-NEXT:    tail call void @no_sync_func(ptr noalias nofree captures(none) [[I_H2S]])
 ; CHECK-NEXT:    store i32 10, ptr [[I_H2S]], align 4
 ; CHECK-NEXT:    [[I2:%.*]] = load i32, ptr [[I_H2S]], align 4
 ; CHECK-NEXT:    ret i32 [[I2]]
@@ -397,8 +397,8 @@ define i32 @test_lifetime() {
 ; CHECK-LABEL: define {{[^@]+}}@test_lifetime() {
 ; CHECK-NEXT:  bb:
 ; CHECK-NEXT:    [[I_H2S:%.*]] = alloca i8, i64 4, align 1
-; CHECK-NEXT:    tail call void @no_sync_func(ptr noalias nocapture nofree [[I_H2S]])
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 noundef 4, ptr noalias nocapture nofree nonnull align 4 dereferenceable(4) [[I_H2S]])
+; CHECK-NEXT:    tail call void @no_sync_func(ptr noalias nofree captures(none) [[I_H2S]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 noundef 4, ptr noalias nofree nonnull align 4 captures(none) dereferenceable(4) [[I_H2S]])
 ; CHECK-NEXT:    store i32 10, ptr [[I_H2S]], align 4
 ; CHECK-NEXT:    [[I2:%.*]] = load i32, ptr [[I_H2S]], align 4
 ; CHECK-NEXT:    ret i32 [[I2]]
@@ -545,10 +545,10 @@ define i32 @test13() {
 ; CHECK-LABEL: define {{[^@]+}}@test13() {
 ; CHECK-NEXT:  bb:
 ; CHECK-NEXT:    [[I:%.*]] = tail call noalias ptr @malloc(i64 noundef 256)
-; CHECK-NEXT:    tail call void @no_sync_func(ptr noalias nocapture nofree [[I]])
+; CHECK-NEXT:    tail call void @no_sync_func(ptr noalias nofree captures(none) [[I]])
 ; CHECK-NEXT:    store i32 10, ptr [[I]], align 4
 ; CHECK-NEXT:    [[I2:%.*]] = load i32, ptr [[I]], align 4
-; CHECK-NEXT:    tail call void @free(ptr noalias nocapture nonnull align 4 dereferenceable(4) [[I]])
+; CHECK-NEXT:    tail call void @free(ptr noalias nonnull align 4 captures(none) dereferenceable(4) [[I]])
 ; CHECK-NEXT:    ret i32 [[I2]]
 ;
 bb:
@@ -564,10 +564,10 @@ define i32 @test_sle() {
 ; CHECK-LABEL: define {{[^@]+}}@test_sle() {
 ; CHECK-NEXT:  bb:
 ; CHECK-NEXT:    [[I:%.*]] = tail call noalias ptr @malloc(i64 noundef -1)
-; CHECK-NEXT:    tail call void @no_sync_func(ptr noalias nocapture nofree [[I]])
+; CHECK-NEXT:    tail call void @no_sync_func(ptr noalias nofree captures(none) [[I]])
 ; CHECK-NEXT:    store i32 10, ptr [[I]], align 4
 ; CHECK-NEXT:    [[I2:%.*]] = load i32, ptr [[I]], align 4
-; CHECK-NEXT:    tail call void @free(ptr noalias nocapture nonnull align 4 dereferenceable(4) [[I]])
+; CHECK-NEXT:    tail call void @free(ptr noalias nonnull align 4 captures(none) dereferenceable(4) [[I]])
 ; CHECK-NEXT:    ret i32 [[I2]]
 ;
 bb:
@@ -583,10 +583,10 @@ define i32 @test_overflow() {
 ; CHECK-LABEL: define {{[^@]+}}@test_overflow() {
 ; CHECK-NEXT:  bb:
 ; CHECK-NEXT:    [[I:%.*]] = tail call noalias ptr @calloc(i64 noundef 65537, i64 noundef 65537)
-; CHECK-NEXT:    tail call void @no_sync_func(ptr noalias nocapture nofree [[I]])
+; CHECK-NEXT:    tail call void @no_sync_func(ptr noalias nofree captures(none) [[I]])
 ; CHECK-NEXT:    store i32 10, ptr [[I]], align 4
 ; CHECK-NEXT:    [[I2:%.*]] = load i32, ptr [[I]], align 4
-; CHECK-NEXT:    tail call void @free(ptr noalias nocapture nonnull align 4 dereferenceable(4) [[I]])
+; CHECK-NEXT:    tail call void @free(ptr noalias nonnull align 4 captures(none) dereferenceable(4) [[I]])
 ; CHECK-NEXT:    ret i32 [[I2]]
 ;
 bb:
@@ -602,8 +602,8 @@ define void @test14() {
 ; CHECK-LABEL: define {{[^@]+}}@test14() {
 ; CHECK-NEXT:  bb:
 ; CHECK-NEXT:    [[I:%.*]] = tail call noalias ptr @calloc(i64 noundef 64, i64 noundef 4)
-; CHECK-NEXT:    tail call void @no_sync_func(ptr noalias nocapture nofree [[I]])
-; CHECK-NEXT:    tail call void @free(ptr noalias nocapture [[I]])
+; CHECK-NEXT:    tail call void @no_sync_func(ptr noalias nofree captures(none) [[I]])
+; CHECK-NEXT:    tail call void @free(ptr noalias captures(none) [[I]])
 ; CHECK-NEXT:    ret void
 ;
 bb:
@@ -618,8 +618,8 @@ define void @test15(i64 %S) {
 ; CHECK-SAME: (i64 [[S:%.*]]) {
 ; CHECK-NEXT:  bb:
 ; CHECK-NEXT:    [[I:%.*]] = tail call noalias ptr @malloc(i64 [[S]])
-; CHECK-NEXT:    tail call void @no_sync_func(ptr noalias nocapture nofree [[I]])
-; CHECK-NEXT:    tail call void @free(ptr noalias nocapture [[I]])
+; CHECK-NEXT:    tail call void @no_sync_func(ptr noalias nofree captures(none) [[I]])
+; CHECK-NEXT:    tail call void @free(ptr noalias captures(none) [[I]])
 ; CHECK-NEXT:    ret void
 ;
 bb:
@@ -631,11 +631,11 @@ bb:
 
 define void @test16a(i8 %v, ptr %P) {
 ; CHECK-LABEL: define {{[^@]+}}@test16a
-; CHECK-SAME: (i8 [[V:%.*]], ptr nocapture nofree readnone [[P:%.*]]) {
+; CHECK-SAME: (i8 [[V:%.*]], ptr nofree readnone captures(none) [[P:%.*]]) {
 ; CHECK-NEXT:  bb:
 ; CHECK-NEXT:    [[I_H2S:%.*]] = alloca i8, i64 4, align 1
 ; CHECK-NEXT:    store i8 [[V]], ptr [[I_H2S]], align 1
-; CHECK-NEXT:    tail call void @no_sync_func(ptr noalias nocapture nofree noundef nonnull dereferenceable(1) [[I_H2S]])
+; CHECK-NEXT:    tail call void @no_sync_func(ptr noalias nofree noundef nonnull captures(none) dereferenceable(1) [[I_H2S]])
 ; CHECK-NEXT:    ret void
 ;
 bb:
@@ -648,12 +648,12 @@ bb:
 
 define void @test16b(i8 %v, ptr %P) {
 ; CHECK-LABEL: define {{[^@]+}}@test16b
-; CHECK-SAME: (i8 [[V:%.*]], ptr nocapture nofree writeonly [[P:%.*]]) {
+; CHECK-SAME: (i8 [[V:%.*]], ptr nofree writeonly captures(none) [[P:%.*]]) {
 ; CHECK-NEXT:  bb:
 ; CHECK-NEXT:    [[I:%.*]] = tail call noalias ptr @malloc(i64 noundef 4)
 ; CHECK-NEXT:    store ptr [[I]], ptr [[P]], align 8
-; CHECK-NEXT:    tail call void @no_sync_func(ptr nocapture nofree [[I]])
-; CHECK-NEXT:    tail call void @free(ptr nocapture [[I]])
+; CHECK-NEXT:    tail call void @no_sync_func(ptr nofree captures(none) [[I]])
+; CHECK-NEXT:    tail call void @free(ptr captures(none) [[I]])
 ; CHECK-NEXT:    ret void
 ;
 bb:
@@ -666,11 +666,11 @@ bb:
 
 define void @test16c(i8 %v, ptr %P) {
 ; CHECK-LABEL: define {{[^@]+}}@test16c
-; CHECK-SAME: (i8 [[V:%.*]], ptr nocapture nofree writeonly [[P:%.*]]) {
+; CHECK-SAME: (i8 [[V:%.*]], ptr nofree writeonly captures(none) [[P:%.*]]) {
 ; CHECK-NEXT:  bb:
 ; CHECK-NEXT:    [[I_H2S:%.*]] = alloca i8, i64 4, align 1
 ; CHECK-NEXT:    store ptr [[I_H2S]], ptr [[P]], align 8
-; CHECK-NEXT:    tail call void @no_sync_func(ptr nocapture nofree [[I_H2S]]) #[[ATTR11]]
+; CHECK-NEXT:    tail call void @no_sync_func(ptr nofree captures(none) [[I_H2S]]) #[[ATTR11]]
 ; CHECK-NEXT:    ret void
 ;
 bb:
@@ -683,7 +683,7 @@ bb:
 
 define void @test16d(i8 %v, ptr %P) {
 ; CHECK-LABEL: define {{[^@]+}}@test16d
-; CHECK-SAME: (i8 [[V:%.*]], ptr nocapture nofree writeonly [[P:%.*]]) {
+; CHECK-SAME: (i8 [[V:%.*]], ptr nofree writeonly captures(none) [[P:%.*]]) {
 ; CHECK-NEXT:  bb:
 ; CHECK-NEXT:    [[I:%.*]] = tail call noalias ptr @malloc(i64 noundef 4)
 ; CHECK-NEXT:    store ptr [[I]], ptr [[P]], align 8
@@ -703,7 +703,7 @@ define void @test16e(i8 %v) norecurse {
 ; CHECK-NEXT:  bb:
 ; CHECK-NEXT:    [[I_H2S:%.*]] = alloca i8, i64 4, align 1
 ; CHECK-NEXT:    store ptr [[I_H2S]], ptr @G, align 8
-; CHECK-NEXT:    call void @usei8p(ptr nocapture nofree [[I_H2S]]) #[[ATTR12:[0-9]+]]
+; CHECK-NEXT:    call void @usei8p(ptr nofree captures(none) [[I_H2S]]) #[[ATTR12:[0-9]+]]
 ; CHECK-NEXT:    ret void
 ;
 bb:
diff --git a/llvm/test/Transforms/Attributor/heap_to_stack_gpu.ll b/llvm/test/Transforms/Attributor/heap_to_stack_gpu.ll
index 9143d44a3c178..c7a9ec8edc0f6 100644
--- a/llvm/test/Transforms/Attributor/heap_to_stack_gpu.ll
+++ b/llvm/test/Transforms/Attributor/heap_to_stack_gpu.ll
@@ -40,10 +40,10 @@ declare void @llvm.lifetime.start.p0(i64, ptr nocapture) nounwind
 ;.
 define void @nofree_arg_only(ptr %p1, ptr %p2) {
 ; CHECK-LABEL: define {{[^@]+}}@nofree_arg_only
-; CHECK-SAME: (ptr nocapture nofree [[P1:%.*]], ptr nocapture [[P2:%.*]]) {
+; CHECK-SAME: (ptr nofree captures(none) [[P1:%.*]], ptr captures(none) [[P2:%.*]]) {
 ; CHECK-NEXT:  bb:
-; CHECK-NEXT:    tail call void @free(ptr nocapture [[P2]])
-; CHECK-NEXT:    tail call void @nofree_func(ptr nocapture nofree [[P1]])
+; CHECK-NEXT:    tail call void @free(ptr captures(none) [[P2]])
+; CHECK-NEXT:    tail call void @nofree_func(ptr nofree captures(none) [[P1]])
 ; CHECK-NEXT:    ret void
 ;
 bb:
@@ -58,9 +58,9 @@ define void @test1() {
 ; CHECK-LABEL: define {{[^@]+}}@test1() {
 ; CHECK-NEXT:  bb:
 ; CHECK-NEXT:    [[I:%.*]] = tail call noalias ptr @malloc(i64 noundef 4)
-; CHECK-NEXT:    tail call void @nocapture_func_frees_pointer(ptr noalias nocapture [[I]])
+; CHECK-NEXT:    tail call void @nocapture_func_frees_pointer(ptr noalias captures(none) [[I]])
 ; CHECK-NEXT:    tail call void (...) @func_throws()
-; CHECK-NEXT:    tail call void @free(ptr noalias nocapture [[I]])
+; CHECK-NEXT:    tail call void @free(ptr noalias captures(none) [[I]])
 ; CHECK-NEXT:    ret void
 ;
 bb:
@@ -78,7 +78,7 @@ define void @test2() {
 ; CHECK-NEXT:  bb:
 ; CHECK-NEXT:    [[I:%.*]] = tail call noalias ptr @malloc(i64 noundef 4)
 ; CHECK-NEXT:    tail call void @sync_func(ptr [[I]])
-; CHECK-NEXT:    tail call void @free(ptr nocapture [[I]])
+; CHECK-NEXT:    tail call void @free(ptr captures(none) [[I]])
 ; CHECK-NEXT:    ret void
 ;
 bb:
@@ -94,8 +94,8 @@ define void @test3() {
 ; CHECK-LABEL: define {{[^@]+}}@test3() {
 ; CHECK-NEXT:  bb:
 ; CHECK-NEXT:    [[I:%.*]] = tail call noalias ptr @malloc(i64 noundef 4)
-; CHECK-NEXT:    tail call void @no_sync_func(ptr noalias nocapture nofree [[I]])
-; CHECK-NEXT:    tail call void @free(ptr noalias nocapture [[I]])
+; CHECK-NEXT:    tail call void @no_sync_func(ptr noalias nofree captures(none) [[I]])
+; CHECK-NEXT:    tail call void @free(ptr noalias captures(none) [[I]])
 ; CHECK-NEXT:    ret void
 ;
 bb:
@@ -107,11 +107,11 @@ bb:
 
 define void @test3a(ptr %p) {
 ; CHECK-LABEL: define {{[^@]+}}@test3a
-; CHECK-SAME: (ptr nocapture [[P:%.*]]) {
+; CHECK-SAME: (ptr captures(none) [[P:%.*]]) {
 ; CHECK-NEXT:  bb:
 ; CHECK-NEXT:    [[I:%.*]] = tail call noalias ptr @malloc(i64 noundef 4)
-; CHECK-NEXT:    tail call void @nofree_arg_only(ptr noalias nocapture nofree [[I]], ptr nocapture [[P]])
-; CHECK-NEXT:    tail call void @free(ptr noalias nocapture [[I]])
+; CHECK-NEXT:    tail call void @nofree_arg_only(ptr noalias nofree captures(none) [[I]], ptr captures(none) [[P]])
+; CHECK-NEXT:    tail call void @free(ptr noalias captures(none) [[I]])
 ; CHECK-NEXT:    ret void
 ;
 bb:
@@ -125,11 +125,11 @@ declare noalias ptr @aligned_alloc(i64, i64)
 
 define void @test3b(ptr %p) {
 ; CHECK-LABEL: define {{[^@]+}}@test3b
-; CHECK-SAME: (ptr nocapture [[P:%.*]]) {
+; CHECK-SAME: (ptr captures(none) [[P:%.*]]) {
 ; CHECK-NEXT:  bb:
 ; CHECK-NEXT:    [[I:%.*]] = tail call noalias ptr @aligned_alloc(i64 noundef 32, i64 noundef 128)
-; CHECK-NEXT:    tail call void @nofree_arg_only(ptr noalias nocapture nofree [[I]], ptr nocapture [[P]])
-; CHECK-NEXT:    tail call void @free(ptr noalias nocapture [[I]])
+; CHECK-NEXT:    tail call void @nofree_arg_only(ptr noalias nofree captures(none) [[I]], ptr captures(none) [[P]])
+; CHECK-NEXT:    tail call void @free(ptr noalias captures(none) [[I]])
 ; CHECK-NEXT:    ret void
 ;
 bb:
@@ -145,7 +145,7 @@ define void @test3c(i64 %alignment) {
 ; CHECK-SAME: (i64 [[ALIGNMENT:%.*]]) {
 ; CHECK-NEXT:  bb:
 ; CHECK-NEXT:    [[I:%.*]] = tail call noalias ptr @aligned_alloc(i64 [[ALIGNMENT]], i64 noundef 128)
-; CHECK-NEXT:    tail call void @free(ptr noalias nocapture [[I]])
+; CHECK-NEXT:    tail call void @free(ptr noalias captures(none) [[I]])
 ; CHECK-NEXT:    ret void
 ;
 bb:
@@ -160,8 +160,8 @@ define void @test0() {
 ; CHECK-LABEL: define {{[^@]+}}@test0() {
 ; CHECK-NEXT:  bb:
 ; CHECK-NEXT:    [[I:%.*]] = tail call noalias ptr @calloc(i64 noundef 2, i64 noundef 4)
-; CHECK-NEXT:    tail call void @no_sync_func(ptr noalias nocapture nofree [[I]])
-; CHECK-NEXT:    tail call void @free(ptr noalias nocapture [[I]])
+; CHECK-NEXT:    tail call void @no_sync_func(ptr noalias nofree captures(none) [[I]])
+; CHECK-NEXT:    tail call void @free(ptr noalias captures(none) [[I]])
 ; CHECK-NEXT:    ret void
 ;
 bb:
@@ -176,7 +176,7 @@ define void @test4() {
 ; CHECK-LABEL: define {{[^@]+}}@test4() {
 ; CHECK-NEXT:  bb:
 ; CHECK-NEXT:    [[I:%.*]] = tail call noalias ptr @malloc(i64 noundef 4)
-; CHECK-NEXT:    tail call void @nofree_func(ptr noalias nocapture nofree [[I]])
+; CHECK-NEXT:    tail call void @nofree_func(ptr noalias nofree captures(none) [[I]])
 ; CHECK-NEXT:    ret void
 ;
 bb:
@@ -190,17 +190,17 @@ bb:
 
 define void @test5(i32 %arg, ptr %p) {
 ; CHECK-LABEL: define {{[^@]+}}@test5
-; CHECK-SAME: (i32 [[ARG:%.*]], ptr nocapture [[P:%.*]]) {
+; CHECK-SAME: (i32 [[ARG:%.*]], ptr captures(none) [[P:%.*]]) {
 ; CHECK-NEXT:  bb:
 ; CHECK-NEXT:    [[I:%.*]] = tail call noalias ptr @malloc(i64 noundef 4)
 ; CHECK-NEXT:    [[I1:%.*]] = icmp eq i32 [[ARG]], 0
 ; CHECK-NEXT:    br i1 [[I1]], label [[BB3:%.*]], label [[BB2:%.*]]
 ; CHECK:       bb2:
-; CHECK-NEXT:    tail call void @nofree_func(ptr noalias nocapture nofree [[I]])
+; CHECK-NEXT:    tail call void @nofree_func(ptr noalias nofree captures(none) [[I]])
 ; CHECK-NEXT:    br label [[BB4:%.*]]
 ; CHECK:       bb3:
-; CHECK-NEXT:    tail call void @nofree_arg_only(ptr noalias nocapture nofree [[I]], ptr nocapture [[P]])
-; CHECK-NEXT:    tail call void @free(ptr noalias nocapture [[I]])
+; CHECK-NEXT:    tail call void @nofree_arg_only(ptr noalias nofree captures(none) [[I]], ptr captures(none) [[P]])
+; CHECK-NEXT:    tail call void @free(ptr noalias captures(none) [[I]])
 ; CHECK-NEXT:    br label [[BB4]]
 ; CHECK:       bb4:
 ; CHECK-NEXT:    ret void
@@ -233,11 +233,11 @@ define void @test6(i32 %arg) {
 ; CHECK-NEXT:    [[I1:%.*]] = icmp eq i32 [[ARG]], 0
 ; CHECK-NEXT:    br i1 [[I1]], label [[BB3:%.*]], label [[BB2:%.*]]
 ; CHECK:       bb2:
-; CHECK-NEXT:    tail call void @nofree_func(ptr noalias nocapture nofree [[I]])
-; CHECK-NEXT:    tail call void @free(ptr noalias nocapture [[I]])
+; CHECK-NEXT:    tail call void @nofree_func(ptr noalias nofree captures(none) [[I]])
+; CHECK-NEXT:    tail call void @free(ptr noalias captures(none) [[I]])
 ; CHECK-NEXT:    br label [[BB4:%.*]]
 ; CHECK:       bb3:
-; CHECK-NEXT:    tail call void @free(ptr noalias nocapture [[I]])
+; CHECK-NEXT:    tail call void @free(ptr noalias captures(none) [[I]])
 ; CHECK-NEXT:    br label [[BB4]]
 ; CHECK:       bb4:
 ; CHECK-NEXT:    ret void
@@ -282,10 +282,10 @@ define void @test8() {
 ; CHECK-LABEL: define {{[^@]+}}@test8() {
 ; CHECK-NEXT:  bb:
 ; CHECK-NEXT:    [[I:%.*]] = tail call noalias ptr @malloc(i64 noundef 4)
-; CHECK-NEXT:    tail call void @no_sync_func(ptr nocapture nofree [[I]])
+; CHECK-NEXT:    tail call void @no_sync_func(ptr nofree captures(none) [[I]])
 ; CHECK-NEXT:    store i32 10, ptr [[I]], align 4
 ; CHECK-NEXT:    tail call void @foo(ptr nonnull align 4 dereferenceable(4) [[I]])
-; CHECK-NEXT:    tail call void @free(ptr nocapture nonnull align 4 dereferenceable(4) [[I]])
+; CHECK-NEXT:    tail call void @free(ptr nonnull align 4 captures(none) dereferenceable(4) [[I]])
 ; CHECK-NEXT:    ret void
 ;
 bb:
@@ -303,10 +303,10 @@ define void @test9() {
 ; CHECK-LABEL: define {{[^@]+}}@test9() {
 ; CHECK-NEXT:  bb:
 ; CHECK-NEXT:    [[I:%.*]] = tail call noalias ptr @malloc(i64 noundef 4)
-; CHECK-NEXT:    tail call void @no_sync_func(ptr nocapture nofree [[I]])
+; CHECK-NEXT:    tail call void @no_sync_func(ptr nofree captures(none) [[I]])
 ; CHECK-NEXT:    store i32 10, ptr [[I]], align 4
 ; CHECK-NEXT:    tail call void @foo_nounw(ptr nofree nonnull align 4 dereferenceable(4) [[I]]) #[[ATTR6:[0-9]+]]
-; CHECK-NEXT:    tail call void @free(ptr nocapture nonnull align 4 dereferenceable(4) [[I]])
+; CHECK-NEXT:    tail call void @free(ptr nonnull align 4 captures(none) dereferenceable(4) [[I]])
 ; CHECK-NEXT:    ret void
 ;
 bb:
@@ -325,10 +325,10 @@ define i32 @test10() {
 ; CHECK-LABEL: define {{[^@]+}}@test10() {
 ; CHECK-NEXT:  bb:
 ; CHECK-NEXT:    [[I:%.*]] = tail call noalias ptr @malloc(i64 noundef 4)
-; CHECK-NEXT:    tail call void @no_sync_func(ptr noalias nocapture nofree [[I]])
+; CHECK-NEXT:    tail call void @no_sync_func(ptr noalias nofree captures(none) [[I]])
 ; CHECK-NEXT:    store i32 10, ptr [[I]], align 4
 ; CHECK-NEXT:    [[I2:%.*]] = load i32, ptr [[I]], align 4
-; CHECK-NEXT:    tail call void @free(ptr noalias nocapture nonnull align 4 dereferenceable(4) [[I]])
+; CHECK-NEXT:    tail call void @free(ptr noalias nonnull align 4 captures(none) dereferenceable(4) [[I]])
 ; CHECK-NEXT:    ret i32 [[I2]]
 ;
 bb:
@@ -344,11 +344,11 @@ define i32 @test_lifetime() {
 ; CHECK-LABEL: define {{[^@]+}}@test_lifetime() {
 ; CHECK-NEXT:  bb:
 ; CHECK-NEXT:    [[I:%.*]] = tail call noalias ptr @malloc(i64 noundef 4)
-; CHECK-NEXT:    tail call void @no_sync_func(ptr noalias nocapture nofree [[I]])
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 noundef 4, ptr noalias nocapture nofree nonnull align 4 dereferenceable(4) [[I]])
+; CHECK-NEXT:    tail call void @no_sync_func(ptr noalias nofree captures(none) [[I]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 noundef 4, ptr noalias nofree nonnull align 4 captures(none) dereferenceable(4) [[I]])
 ; CHECK-NEXT:    store i32 10, ptr [[I]], align 4
 ; CHECK-NEXT:    [[I2:%.*]] = load i32, ptr [[I]], align 4
-; CHECK-NEXT:    tail call void @free(ptr noalias nocapture nonnull align 4 dereferenceable(4) [[I]])
+; CHECK-NEXT:    tail call void @free(ptr noalias nonnull align 4 captures(none) dereferenceable(4) [[I]])
 ; CHECK-NEXT:    ret i32 [[I2]]
 ;
 bb:
@@ -368,7 +368,7 @@ define void @test11() {
 ; CHECK-NEXT:  bb:
 ; CHECK-NEXT:    [[I:%.*]] = tail call noalias ptr @malloc(i64 noundef 4)
 ; CHECK-NEXT:    tail call void @sync_will_return(ptr [[I]]) #[[ATTR6]]
-; CHECK-NEXT:    tail call void @free(ptr nocapture [[I]])
+; CHECK-NEXT:    tail call void @free(ptr captures(none) [[I]])
 ; CHECK-NEXT:    ret void
 ;
 bb:
@@ -406,7 +406,7 @@ define i32 @irreducible_cfg(i32 %arg) {
 ; CHECK-NEXT:    [[I12]] = add nsw i32 [[DOT1]], 1
 ; CHECK-NEXT:    br label [[BB6]]
 ; CHECK:       bb13:
-; CHECK-NEXT:    call void @free(ptr noalias nocapture noundef nonnull align 4 dereferenceable(4) [[I]])
+; CHECK-NEXT:    call void @free(ptr noalias noundef nonnull align 4 captures(none) dereferenceable(4) [[I]])
 ; CHECK-NEXT:    [[I16:%.*]] = load i32, ptr [[I]], align 4
 ; CHECK-NEXT:    ret i32 [[I16]]
 ;
@@ -495,10 +495,10 @@ define i32 @test13() {
 ; CHECK-LABEL: define {{[^@]+}}@test13() {
 ; CHECK-NEXT:  bb:
 ; CHECK-NEXT:    [[I:%.*]] = tail call noalias ptr @malloc(i64 noundef 256)
-; CHECK-NEXT:    tail call void @no_sync_func(ptr noalias nocapture nofree [[I]])
+; CHECK-NEXT:    tail call void @no_sync_func(ptr noalias nofree captures(none) [[I]])
 ; CHECK-NEXT:    store i32 10, ptr [[I]], align 4
 ; CHECK-NEXT:    [[I2:%.*]] = load i32, ptr [[I]], align 4
-; CHECK-NEXT:    tail call void @free(ptr noalias nocapture nonnull align 4 dereferenceable(4) [[I]])
+; CHECK-NEXT:    tail call void @free(ptr noalias nonnull align 4 captures(none) dereferenceable(4) [[I]])
 ; CHECK-NEXT:    ret i32 [[I2]]
 ;
 bb:
@@ -514,10 +514,10 @@ define i32 @test_sle() {
 ; CHECK-LABEL: define {{[^@]+}}@test_sle() {
 ; CHECK-NEXT:  bb:
 ; CHECK-NEXT:    [[I:%.*]] = tail call noalias ptr @malloc(i64 noundef -1)
-; CHECK-NEXT:    tail call void @no_sync_func(ptr noalias nocapture nofree [[I]])
+; CHECK-NEXT:    tail call void @no_sync_func(ptr noalias nofree captures(none) [[I]])
 ; CHECK-NEXT:    store i32 10, ptr [[I]], align 4
 ; CHECK-NEXT:    [[I2:%.*]] = load i32, ptr [[I]], align 4
-; CHECK-NEXT:    tail call void @free(ptr noalias nocapture nonnull align 4 dereferenceable(4) [[I]])
+; CHECK-NEXT:    tail call void @free(ptr noalias nonnull align 4 captures(none) dereferenceable(4) [[I]])
 ; CHECK-NEXT:    ret i32 [[I2]]
 ;
 bb:
@@ -533,10 +533,10 @@ define i32 @test_overflow() {
 ; CHECK-LABEL: define {{[^@]+}}@test_overflow() {
 ; CHECK-NEXT:  bb:
 ; CHECK-NEXT:    [[I:%.*]] = tail call noalias ptr @calloc(i64 noundef 65537, i64 noundef 65537)
-; CHECK-NEXT:    tail call void @no_sync_func(ptr noalias nocapture nofree [[I]])
+; CHECK-NEXT:    tail call void @no_sync_func(ptr noalias nofree captures(none) [[I]])
 ; CHECK-NEXT:    store i32 10, ptr [[I]], align 4
 ; CHECK-NEXT:    [[I2:%.*]] = load i32, ptr [[I]], align 4
-; CHECK-NEXT:    tail call void @free(ptr noalias nocapture nonnull align 4 dereferenceable(4) [[I]])
+; CHECK-NEXT:    tail call void @free(ptr noalias nonnull align 4 captures(none) dereferenceable(4) [[I]])
 ; CHECK-NEXT:    ret i32 [[I2]]
 ;
 bb:
@@ -552,8 +552,8 @@ define void @test14() {
 ; CHECK-LABEL: define {{[^@]+}}@test14() {
 ; CHECK-NEXT:  bb:
 ; CHECK-NEXT:    [[I:%.*]] = tail call noalias ptr @calloc(i64 noundef 64, i64 noundef 4)
-; CHECK-NEXT:    tail call void @no_sync_func(ptr noalias nocapture nofree [[I]])
-; CHECK-NEXT:    tail call void @free(ptr noalias nocapture [[I]])
+; CHECK-NEXT:    tail call void @no_sync_func(ptr noalias nofree captures(none) [[I]])
+; CHECK-NEXT:    tail call void @free(ptr noalias captures(none) [[I]])
 ; CHECK-NEXT:    ret void
 ;
 bb:
@@ -568,8 +568,8 @@ define void @test15(i64 %S) {
 ; CHECK-SAME: (i64 [[S:%.*]]) {
 ; CHECK-NEXT:  bb:
 ; CHECK-NEXT:    [[I:%.*]] = tail call noalias ptr @malloc(i64 [[S]])
-; CHECK-NEXT:    tail call void @no_sync_func(ptr noalias nocapture nofree [[I]])
-; CHECK-NEXT:    tail call void @free(ptr noalias nocapture [[I]])
+; CHECK-NEXT:    tail call void @no_sync_func(ptr noalias nofree captures(none) [[I]])
+; CHECK-NEXT:    tail call void @free(ptr noalias captures(none) [[I]])
 ; CHECK-NEXT:    ret void
 ;
 bb:
@@ -581,12 +581,12 @@ bb:
 
 define void @test16a(i8 %v, ptr %P) {
 ; CHECK-LABEL: define {{[^@]+}}@test16a
-; CHECK-SAME: (i8 [[V:%.*]], ptr nocapture nofree readnone [[P:%.*]]) {
+; CHECK-SAME: (i8 [[V:%.*]], ptr nofree readnone captures(none) [[P:%.*]]) {
 ; CHECK-NEXT:  bb:
 ; CHECK-NEXT:    [[I:%.*]] = tail call noalias ptr @malloc(i64 noundef 4)
 ; CHECK-NEXT:    store i8 [[V]], ptr [[I]], align 1
-; CHECK-NEXT:    tail call void @no_sync_func(ptr noalias nocapture nofree noundef nonnull dereferenceable(1) [[I]])
-; CHECK-NEXT:    tail call void @free(ptr noalias nocapture noundef nonnull dereferenceable(1) [[I]])
+; CHECK-NEXT:    tail call void @no_sync_func(ptr noalias nofree noundef nonnull captures(none) dereferenceable(1) [[I]])
+; CHECK-NEXT:    tail call void @free(ptr noalias noundef nonnull captures(none) dereferenceable(1) [[I]])
 ; CHECK-NEXT:    ret void
 ;
 bb:
@@ -599,12 +599,12 @@ bb:
 
 define void @test16b(i8 %v, ptr %P) {
 ; CHECK-LABEL: define {{[^@]+}}@test16b
-; CHECK-SAME: (i8 [[V:%.*]], ptr nocapture nofree writeonly [[P:%.*]]) {
+; CHECK-SAME: (i8 [[V:%.*]], ptr nofree writeonly captures(none) [[P:%.*]]) {
 ; CHECK-NEXT:  bb:
 ; CHECK-NEXT:    [[I:%.*]] = tail call noalias ptr @malloc(i64 noundef 4)
 ; CHECK-NEXT:    store ptr [[I]], ptr [[P]], align 8
-; CHECK-NEXT:    tail call void @no_sync_func(ptr nocapture nofree [[I]])
-; CHECK-NEXT:    tail call void @free(ptr nocapture [[I]])
+; CHECK-NEXT:    tail call void @no_sync_func(ptr nofree captures(none) [[I]])
+; CHECK-NEXT:    tail call void @free(ptr captures(none) [[I]])
 ; CHECK-NEXT:    ret void
 ;
 bb:
@@ -617,12 +617,12 @@ bb:
 
 define void @test16c(i8 %v, ptr %P) {
 ; CHECK-LABEL: define {{[^@]+}}@test16c
-; CHECK-SAME: (i8 [[V:%.*]], ptr nocapture nofree writeonly [[P:%.*]]) {
+; CHECK-SAME: (i8 [[V:%.*]], ptr nofree writeonly captures(none) [[P:%.*]]) {
 ; CHECK-NEXT:  bb:
 ; CHECK-NEXT:    [[I:%.*]] = tail call noalias ptr @malloc(i64 noundef 4)
 ; CHECK-NEXT:    store ptr [[I]], ptr [[P]], align 8
-; CHECK-NEXT:    tail call void @no_sync_func(ptr nocapture nofree [[I]]) #[[ATTR6]]
-; CHECK-NEXT:    tail call void @free(ptr nocapture [[I]])
+; CHECK-NEXT:    tail call void @no_sync_func(ptr nofree captures(none) [[I]]) #[[ATTR6]]
+; CHECK-NEXT:    tail call void @free(ptr captures(none) [[I]])
 ; CHECK-NEXT:    ret void
 ;
 bb:
@@ -635,7 +635,7 @@ bb:
 
 define void @test16d(i8 %v, ptr %P) {
 ; CHECK-LABEL: define {{[^@]+}}@test16d
-; CHECK-SAME: (i8 [[V:%.*]], ptr nocapture nofree writeonly [[P:%.*]]) {
+; CHECK-SAME: (i8 [[V:%.*]], ptr nofree writeonly captures(none) [[P:%.*]]) {
 ; CHECK-NEXT:  bb:
 ; CHECK-NEXT:    [[I:%.*]] = tail call noalias ptr @malloc(i64 noundef 4)
 ; CHECK-NEXT:    store ptr [[I]], ptr [[P]], align 8
@@ -655,7 +655,7 @@ define void @test17() {
 ; CHECK-NEXT:  bb:
 ; CHECK-NEXT:    [[I_H2S:%.*]] = alloca i8, i64 4, align 1, addrspace(5)
 ; CHECK-NEXT:    [[MALLOC_CAST:%.*]] = addrspacecast ptr addrspace(5) [[I_H2S]] to ptr
-; CHECK-NEXT:    tail call void @usei8(ptr noalias nocapture nofree [[MALLOC_CAST]]) #[[ATTR7:[0-9]+]]
+; CHECK-NEXT:    tail call void @usei8(ptr noalias nofree captures(none) [[MALLOC_CAST]]) #[[ATTR7:[0-9]+]]
 ; CHECK-NEXT:    ret void
 ;
 bb:
@@ -670,7 +670,7 @@ define void @test17b() {
 ; CHECK-NEXT:  bb:
 ; CHECK-NEXT:    [[I:%.*]] = tail call noalias ptr @__kmpc_alloc_shared(i64 noundef 4)
 ; CHECK-NEXT:    tail call void @usei8(ptr nofree [[I]]) #[[ATTR7]]
-; CHECK-NEXT:    tail call void @__kmpc_free_shared(ptr nocapture [[I]], i64 noundef 4)
+; CHECK-NEXT:    tail call void @__kmpc_free_shared(ptr captures(none) [[I]], i64 noundef 4)
 ; CHECK-NEXT:    ret void
 ;
 bb:
@@ -687,7 +687,7 @@ define void @move_alloca() {
 ; CHECK-NEXT:    br label [[NOT_ENTRY:%.*]]
 ; CHECK:       not_entry:
 ; CHECK-NEXT:    [[MALLOC_CAST:%.*]] = addrspacecast ptr addrspace(5) [[I_H2S]] to ptr
-; CHECK-NEXT:    tail call void @usei8(ptr noalias nocapture nofree [[MALLOC_CAST]]) #[[ATTR7]]
+; CHECK-NEXT:    tail call void @usei8(ptr noalias nofree captures(none) [[MALLOC_CAST]]) #[[ATTR7]]
 ; CHECK-NEXT:    ret void
 ;
 entry:
@@ -708,8 +708,8 @@ define void @test16e(i8 %v) norecurse {
 ; CHECK-NEXT:  bb:
 ; CHECK-NEXT:    [[I:%.*]] = tail call noalias ptr @__kmpc_alloc_shared(i64 noundef 4)
 ; CHECK-NEXT:    store ptr [[I]], ptr @G, align 8
-; CHECK-NEXT:    call void @usei8(ptr nocapture nofree [[I]]) #[[ATTR8:[0-9]+]]
-; CHECK-NEXT:    tail call void @__kmpc_free_shared(ptr noalias nocapture [[I]], i64 noundef 4)
+; CHECK-NEXT:    call void @usei8(ptr nofree captures(none) [[I]]) #[[ATTR8:[0-9]+]]
+; CHECK-NEXT:    tail call void @__kmpc_free_shared(ptr noalias captures(none) [[I]], i64 noundef 4)
 ; CHECK-NEXT:    ret void
 ;
 bb:
@@ -730,7 +730,7 @@ define void @test16f(i8 %v) norecurse {
 ; CHECK-NEXT:    [[I_H2S:%.*]] = alloca i8, i64 4, align 1, addrspace(5)
 ; CHECK-NEXT:    [[MALLOC_CAST:%.*]] = addrspacecast ptr addrspace(5) [[I_H2S]] to ptr
 ; CHECK-NEXT:    store ptr [[MALLOC_CAST]], ptr @Gtl, align 8
-; CHECK-NEXT:    call void @usei8(ptr nocapture nofree [[MALLOC_CAST]]) #[[ATTR8]]
+; CHECK-NEXT:    call void @usei8(ptr nofree captures(none) [[MALLOC_CAST]]) #[[ATTR8]]
 ; CHECK-NEXT:    ret void
 ;
 bb:
@@ -747,7 +747,7 @@ define void @convert_large_kmpc_alloc_shared() {
 ; CHECK-NEXT:  bb:
 ; CHECK-NEXT:    [[I_H2S:%.*]] = alloca i8, i64 256, align 1, addrspace(5)
 ; CHECK-NEXT:    [[MALLOC_CAST:%.*]] = addrspacecast ptr addrspace(5) [[I_H2S]] to ptr
-; CHECK-NEXT:    tail call void @usei8(ptr noalias nocapture nofree [[MALLOC_CAST]]) #[[ATTR7]]
+; CHECK-NEXT:    tail call void @usei8(ptr noalias nofree captures(none) [[MALLOC_CAST]]) #[[ATTR7]]
 ; CHECK-NEXT:    ret void
 ;
 bb:
diff --git a/llvm/test/Transforms/Attributor/internal-noalias.ll b/llvm/test/Transforms/Attributor/internal-noalias.ll
index 0dba6853e0c8f..28dd5b21898d7 100644
--- a/llvm/test/Transforms/Attributor/internal-noalias.ll
+++ b/llvm/test/Transforms/Attributor/internal-noalias.ll
@@ -5,19 +5,19 @@
 define dso_local i32 @visible(ptr noalias %A, ptr noalias %B) #0 {
 ; TUNIT: Function Attrs: mustprogress nofree noinline norecurse nosync nounwind willreturn memory(argmem: read) uwtable
 ; TUNIT-LABEL: define {{[^@]+}}@visible
-; TUNIT-SAME: (ptr noalias nocapture nofree readonly [[A:%.*]], ptr noalias nocapture nofree readonly [[B:%.*]]) #[[ATTR0:[0-9]+]] {
+; TUNIT-SAME: (ptr noalias nofree readonly captures(none) [[A:%.*]], ptr noalias nofree readonly captures(none) [[B:%.*]]) #[[ATTR0:[0-9]+]] {
 ; TUNIT-NEXT:  entry:
-; TUNIT-NEXT:    [[CALL1:%.*]] = call i32 @noalias_args(ptr noalias nocapture nofree noundef readonly align 4 [[A]], ptr noalias nocapture nofree noundef readonly align 4 [[B]]) #[[ATTR4:[0-9]+]]
-; TUNIT-NEXT:    [[CALL2:%.*]] = call i32 @noalias_args_argmem(ptr noalias nocapture nofree noundef readonly align 4 [[A]], ptr noalias nocapture nofree noundef readonly align 4 [[B]]) #[[ATTR4]]
+; TUNIT-NEXT:    [[CALL1:%.*]] = call i32 @noalias_args(ptr noalias nofree noundef readonly align 4 captures(none) [[A]], ptr noalias nofree noundef readonly align 4 captures(none) [[B]]) #[[ATTR4:[0-9]+]]
+; TUNIT-NEXT:    [[CALL2:%.*]] = call i32 @noalias_args_argmem(ptr noalias nofree noundef readonly align 4 captures(none) [[A]], ptr noalias nofree noundef readonly align 4 captures(none) [[B]]) #[[ATTR4]]
 ; TUNIT-NEXT:    [[ADD:%.*]] = add nsw i32 [[CALL1]], [[CALL2]]
 ; TUNIT-NEXT:    ret i32 [[ADD]]
 ;
 ; CGSCC: Function Attrs: mustprogress nofree noinline nosync nounwind willreturn memory(argmem: read) uwtable
 ; CGSCC-LABEL: define {{[^@]+}}@visible
-; CGSCC-SAME: (ptr noalias nocapture nofree noundef nonnull readonly align 4 dereferenceable(4) [[A:%.*]], ptr noalias nocapture nofree noundef nonnull readonly align 4 dereferenceable(4) [[B:%.*]]) #[[ATTR0:[0-9]+]] {
+; CGSCC-SAME: (ptr noalias nofree noundef nonnull readonly align 4 captures(none) dereferenceable(4) [[A:%.*]], ptr noalias nofree noundef nonnull readonly align 4 captures(none) dereferenceable(4) [[B:%.*]]) #[[ATTR0:[0-9]+]] {
 ; CGSCC-NEXT:  entry:
-; CGSCC-NEXT:    [[CALL1:%.*]] = call i32 @noalias_args(ptr noalias nocapture nofree noundef nonnull readonly align 4 dereferenceable(4) [[A]], ptr noalias nocapture nofree noundef nonnull readonly align 4 dereferenceable(4) [[B]]) #[[ATTR6:[0-9]+]]
-; CGSCC-NEXT:    [[CALL2:%.*]] = call i32 @noalias_args_argmem(ptr noalias nocapture nofree noundef nonnull readonly align 4 dereferenceable(4) [[A]], ptr noalias nocapture nofree noundef nonnull readonly align 4 dereferenceable(4) [[B]]) #[[ATTR6]]
+; CGSCC-NEXT:    [[CALL1:%.*]] = call i32 @noalias_args(ptr noalias nofree noundef nonnull readonly align 4 captures(none) dereferenceable(4) [[A]], ptr noalias nofree noundef nonnull readonly align 4 captures(none) dereferenceable(4) [[B]]) #[[ATTR6:[0-9]+]]
+; CGSCC-NEXT:    [[CALL2:%.*]] = call i32 @noalias_args_argmem(ptr noalias nofree noundef nonnull readonly align 4 captures(none) dereferenceable(4) [[A]], ptr noalias nofree noundef nonnull readonly align 4 captures(none) dereferenceable(4) [[B]]) #[[ATTR6]]
 ; CGSCC-NEXT:    [[ADD:%.*]] = add nsw i32 [[CALL1]], [[CALL2]]
 ; CGSCC-NEXT:    ret i32 [[ADD]]
 ;
@@ -31,23 +31,23 @@ entry:
 define private i32 @noalias_args(ptr %A, ptr %B) #0 {
 ; TUNIT: Function Attrs: mustprogress nofree noinline norecurse nosync nounwind willreturn memory(argmem: read) uwtable
 ; TUNIT-LABEL: define {{[^@]+}}@noalias_args
-; TUNIT-SAME: (ptr nocapture nofree noundef nonnull readonly align 4 dereferenceable(4) [[A:%.*]], ptr noalias nocapture nofree noundef nonnull readonly align 4 dereferenceable(4) [[B:%.*]]) #[[ATTR0]] {
+; TUNIT-SAME: (ptr nofree noundef nonnull readonly align 4 captures(none) dereferenceable(4) [[A:%.*]], ptr noalias nofree noundef nonnull readonly align 4 captures(none) dereferenceable(4) [[B:%.*]]) #[[ATTR0]] {
 ; TUNIT-NEXT:  entry:
 ; TUNIT-NEXT:    [[TMP0:%.*]] = load i32, ptr [[A]], align 4
 ; TUNIT-NEXT:    [[TMP1:%.*]] = load i32, ptr [[B]], align 4
 ; TUNIT-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP0]], [[TMP1]]
-; TUNIT-NEXT:    [[CALL:%.*]] = call i32 @noalias_args_argmem(ptr nocapture nofree noundef nonnull readonly align 4 dereferenceable(4) [[A]], ptr noalias nocapture nofree noundef nonnull readonly align 4 dereferenceable(4) [[B]]) #[[ATTR4]]
+; TUNIT-NEXT:    [[CALL:%.*]] = call i32 @noalias_args_argmem(ptr nofree noundef nonnull readonly align 4 captures(none) dereferenceable(4) [[A]], ptr noalias nofree noundef nonnull readonly align 4 captures(none) dereferenceable(4) [[B]]) #[[ATTR4]]
 ; TUNIT-NEXT:    [[ADD2:%.*]] = add nsw i32 [[ADD]], [[CALL]]
 ; TUNIT-NEXT:    ret i32 [[ADD2]]
 ;
 ; CGSCC: Function Attrs: mustprogress nofree noinline nosync nounwind willreturn memory(argmem: read) uwtable
 ; CGSCC-LABEL: define {{[^@]+}}@noalias_args
-; CGSCC-SAME: (ptr nocapture nofree noundef nonnull readonly align 4 dereferenceable(4) [[A:%.*]], ptr noalias nocapture nofree noundef nonnull readonly align 4 dereferenceable(4) [[B:%.*]]) #[[ATTR0]] {
+; CGSCC-SAME: (ptr nofree noundef nonnull readonly align 4 captures(none) dereferenceable(4) [[A:%.*]], ptr noalias nofree noundef nonnull readonly align 4 captures(none) dereferenceable(4) [[B:%.*]]) #[[ATTR0]] {
 ; CGSCC-NEXT:  entry:
 ; CGSCC-NEXT:    [[TMP0:%.*]] = load i32, ptr [[A]], align 4
 ; CGSCC-NEXT:    [[TMP1:%.*]] = load i32, ptr [[B]], align 4
 ; CGSCC-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP0]], [[TMP1]]
-; CGSCC-NEXT:    [[CALL:%.*]] = call i32 @noalias_args_argmem(ptr nocapture nofree noundef nonnull readonly align 4 dereferenceable(4) [[A]], ptr noalias nocapture nofree noundef nonnull readonly align 4 dereferenceable(4) [[B]]) #[[ATTR7:[0-9]+]]
+; CGSCC-NEXT:    [[CALL:%.*]] = call i32 @noalias_args_argmem(ptr nofree noundef nonnull readonly align 4 captures(none) dereferenceable(4) [[A]], ptr noalias nofree noundef nonnull readonly align 4 captures(none) dereferenceable(4) [[B]]) #[[ATTR7:[0-9]+]]
 ; CGSCC-NEXT:    [[ADD2:%.*]] = add nsw i32 [[ADD]], [[CALL]]
 ; CGSCC-NEXT:    ret i32 [[ADD2]]
 ;
@@ -64,7 +64,7 @@ entry:
 define internal i32 @noalias_args_argmem(ptr %A, ptr %B) #1 {
 ; TUNIT: Function Attrs: mustprogress nofree noinline norecurse nosync nounwind willreturn memory(argmem: read) uwtable
 ; TUNIT-LABEL: define {{[^@]+}}@noalias_args_argmem
-; TUNIT-SAME: (ptr nocapture nofree noundef nonnull readonly align 4 dereferenceable(4) [[A:%.*]], ptr noalias nocapture nofree noundef nonnull readonly align 4 dereferenceable(4) [[B:%.*]]) #[[ATTR0]] {
+; TUNIT-SAME: (ptr nofree noundef nonnull readonly align 4 captures(none) dereferenceable(4) [[A:%.*]], ptr noalias nofree noundef nonnull readonly align 4 captures(none) dereferenceable(4) [[B:%.*]]) #[[ATTR0]] {
 ; TUNIT-NEXT:  entry:
 ; TUNIT-NEXT:    [[TMP0:%.*]] = load i32, ptr [[A]], align 4
 ; TUNIT-NEXT:    [[TMP1:%.*]] = load i32, ptr [[B]], align 4
@@ -73,7 +73,7 @@ define internal i32 @noalias_args_argmem(ptr %A, ptr %B) #1 {
 ;
 ; CGSCC: Function Attrs: mustprogress nofree noinline norecurse nosync nounwind willreturn memory(argmem: read) uwtable
 ; CGSCC-LABEL: define {{[^@]+}}@noalias_args_argmem
-; CGSCC-SAME: (ptr nocapture nofree noundef nonnull readonly align 4 dereferenceable(4) [[A:%.*]], ptr nocapture nofree noundef nonnull readonly align 4 dereferenceable(4) [[B:%.*]]) #[[ATTR1:[0-9]+]] {
+; CGSCC-SAME: (ptr nofree noundef nonnull readonly align 4 captures(none) dereferenceable(4) [[A:%.*]], ptr nofree noundef nonnull readonly align 4 captures(none) dereferenceable(4) [[B:%.*]]) #[[ATTR1:[0-9]+]] {
 ; CGSCC-NEXT:  entry:
 ; CGSCC-NEXT:    [[TMP0:%.*]] = load i32, ptr [[A]], align 4
 ; CGSCC-NEXT:    [[TMP1:%.*]] = load i32, ptr [[B]], align 4
@@ -90,23 +90,23 @@ entry:
 define dso_local i32 @visible_local(ptr %A) #0 {
 ; TUNIT: Function Attrs: mustprogress nofree noinline norecurse nosync nounwind willreturn memory(argmem: readwrite) uwtable
 ; TUNIT-LABEL: define {{[^@]+}}@visible_local
-; TUNIT-SAME: (ptr nocapture nofree readonly [[A:%.*]]) #[[ATTR1:[0-9]+]] {
+; TUNIT-SAME: (ptr nofree readonly captures(none) [[A:%.*]]) #[[ATTR1:[0-9]+]] {
 ; TUNIT-NEXT:  entry:
 ; TUNIT-NEXT:    [[B:%.*]] = alloca i32, align 4
 ; TUNIT-NEXT:    store i32 5, ptr [[B]], align 4
-; TUNIT-NEXT:    [[CALL1:%.*]] = call i32 @noalias_args(ptr nocapture nofree noundef readonly align 4 [[A]], ptr noalias nocapture nofree noundef nonnull readonly align 4 dereferenceable(4) [[B]]) #[[ATTR4]]
-; TUNIT-NEXT:    [[CALL2:%.*]] = call i32 @noalias_args_argmem(ptr nocapture nofree noundef readonly align 4 [[A]], ptr noalias nocapture nofree noundef nonnull readonly align 4 dereferenceable(4) [[B]]) #[[ATTR4]]
+; TUNIT-NEXT:    [[CALL1:%.*]] = call i32 @noalias_args(ptr nofree noundef readonly align 4 captures(none) [[A]], ptr noalias nofree noundef nonnull readonly align 4 captures(none) dereferenceable(4) [[B]]) #[[ATTR4]]
+; TUNIT-NEXT:    [[CALL2:%.*]] = call i32 @noalias_args_argmem(ptr nofree noundef readonly align 4 captures(none) [[A]], ptr noalias nofree noundef nonnull readonly align 4 captures(none) dereferenceable(4) [[B]]) #[[ATTR4]]
 ; TUNIT-NEXT:    [[ADD:%.*]] = add nsw i32 [[CALL1]], [[CALL2]]
 ; TUNIT-NEXT:    ret i32 [[ADD]]
 ;
 ; CGSCC: Function Attrs: mustprogress nofree noinline nosync nounwind willreturn memory(argmem: readwrite) uwtable
 ; CGSCC-LABEL: define {{[^@]+}}@visible_local
-; CGSCC-SAME: (ptr nocapture nofree noundef nonnull readonly align 4 dereferenceable(4) [[A:%.*]]) #[[ATTR2:[0-9]+]] {
+; CGSCC-SAME: (ptr nofree noundef nonnull readonly align 4 captures(none) dereferenceable(4) [[A:%.*]]) #[[ATTR2:[0-9]+]] {
 ; CGSCC-NEXT:  entry:
 ; CGSCC-NEXT:    [[B:%.*]] = alloca i32, align 4
 ; CGSCC-NEXT:    store i32 5, ptr [[B]], align 4
-; CGSCC-NEXT:    [[CALL1:%.*]] = call i32 @noalias_args(ptr nocapture nofree noundef nonnull readonly align 4 dereferenceable(4) [[A]], ptr noalias nocapture nofree noundef nonnull readonly align 4 dereferenceable(4) [[B]]) #[[ATTR6]]
-; CGSCC-NEXT:    [[CALL2:%.*]] = call i32 @noalias_args_argmem(ptr nocapture nofree noundef nonnull readonly align 4 dereferenceable(4) [[A]], ptr noalias nocapture nofree noundef nonnull readonly align 4 dereferenceable(4) [[B]]) #[[ATTR8:[0-9]+]]
+; CGSCC-NEXT:    [[CALL1:%.*]] = call i32 @noalias_args(ptr nofree noundef nonnull readonly align 4 captures(none) dereferenceable(4) [[A]], ptr noalias nofree noundef nonnull readonly align 4 captures(none) dereferenceable(4) [[B]]) #[[ATTR6]]
+; CGSCC-NEXT:    [[CALL2:%.*]] = call i32 @noalias_args_argmem(ptr nofree noundef nonnull readonly align 4 captures(none) dereferenceable(4) [[A]], ptr noalias nofree noundef nonnull readonly align 4 captures(none) dereferenceable(4) [[B]]) #[[ATTR8:[0-9]+]]
 ; CGSCC-NEXT:    [[ADD:%.*]] = add nsw i32 [[CALL1]], [[CALL2]]
 ; CGSCC-NEXT:    ret i32 [[ADD]]
 ;
@@ -160,12 +160,12 @@ define i32 @visible_local_2() {
 define internal i32 @noalias_args_argmem_rn(ptr %A, ptr %B) #1 {
 ; TUNIT: Function Attrs: mustprogress nofree noinline norecurse nosync nounwind willreturn memory(argmem: write) uwtable
 ; TUNIT-LABEL: define {{[^@]+}}@noalias_args_argmem_rn
-; TUNIT-SAME: (ptr noalias nocapture nofree noundef nonnull writeonly align 4 dereferenceable(4) [[B:%.*]]) #[[ATTR3:[0-9]+]] {
+; TUNIT-SAME: (ptr noalias nofree noundef nonnull writeonly align 4 captures(none) dereferenceable(4) [[B:%.*]]) #[[ATTR3:[0-9]+]] {
 ; TUNIT-NEXT:    ret i32 undef
 ;
 ; CGSCC: Function Attrs: mustprogress nofree noinline norecurse nosync nounwind willreturn memory(argmem: readwrite) uwtable
 ; CGSCC-LABEL: define {{[^@]+}}@noalias_args_argmem_rn
-; CGSCC-SAME: (ptr noalias nocapture nofree noundef nonnull align 4 dereferenceable(4) [[B:%.*]]) #[[ATTR5:[0-9]+]] {
+; CGSCC-SAME: (ptr noalias nofree noundef nonnull align 4 captures(none) dereferenceable(4) [[B:%.*]]) #[[ATTR5:[0-9]+]] {
 ; CGSCC-NEXT:    [[T0:%.*]] = load i32, ptr [[B]], align 4
 ; CGSCC-NEXT:    store i32 0, ptr [[B]], align 4
 ; CGSCC-NEXT:    ret i32 [[T0]]
@@ -180,7 +180,7 @@ define i32 @visible_local_3() {
 ; TUNIT-LABEL: define {{[^@]+}}@visible_local_3
 ; TUNIT-SAME: () #[[ATTR2]] {
 ; TUNIT-NEXT:    [[B:%.*]] = alloca i32, align 4
-; TUNIT-NEXT:    [[CALL:%.*]] = call i32 @noalias_args_argmem_rn(ptr noalias nocapture nofree noundef nonnull writeonly align 4 dereferenceable(4) [[B]]) #[[ATTR5:[0-9]+]]
+; TUNIT-NEXT:    [[CALL:%.*]] = call i32 @noalias_args_argmem_rn(ptr noalias nofree noundef nonnull writeonly align 4 captures(none) dereferenceable(4) [[B]]) #[[ATTR5:[0-9]+]]
 ; TUNIT-NEXT:    ret i32 5
 ;
 ; CGSCC: Function Attrs: mustprogress nofree nosync nounwind willreturn memory(none)
@@ -188,7 +188,7 @@ define i32 @visible_local_3() {
 ; CGSCC-SAME: () #[[ATTR4]] {
 ; CGSCC-NEXT:    [[B:%.*]] = alloca i32, align 4
 ; CGSCC-NEXT:    store i32 5, ptr [[B]], align 4
-; CGSCC-NEXT:    [[CALL:%.*]] = call i32 @noalias_args_argmem_rn(ptr noalias nocapture nofree noundef nonnull align 4 dereferenceable(4) [[B]]) #[[ATTR10:[0-9]+]]
+; CGSCC-NEXT:    [[CALL:%.*]] = call i32 @noalias_args_argmem_rn(ptr noalias nofree noundef nonnull align 4 captures(none) dereferenceable(4) [[B]]) #[[ATTR10:[0-9]+]]
 ; CGSCC-NEXT:    ret i32 [[CALL]]
 ;
   %B = alloca i32, align 4
diff --git a/llvm/test/Transforms/Attributor/issue87856.ll b/llvm/test/Transforms/Attributor/issue87856.ll
index 4da29cc4448d4..aa7072b335b40 100644
--- a/llvm/test/Transforms/Attributor/issue87856.ll
+++ b/llvm/test/Transforms/Attributor/issue87856.ll
@@ -23,7 +23,7 @@ define void @null_ptr_is_valid_call_with_undef() #0 {
 
 define void @store_as0(ptr %0) {
 ; CHECK-LABEL: define void @store_as0(
-; CHECK-SAME: ptr nocapture nofree noundef nonnull writeonly align 2 dereferenceable(2) [[TMP0:%.*]]) #[[ATTR2:[0-9]+]] {
+; CHECK-SAME: ptr nofree noundef nonnull writeonly align 2 captures(none) dereferenceable(2) [[TMP0:%.*]]) #[[ATTR2:[0-9]+]] {
 ; CHECK-NEXT:    store i16 0, ptr [[TMP0]], align 2
 ; CHECK-NEXT:    ret void
 ;
@@ -34,7 +34,7 @@ define void @store_as0(ptr %0) {
 define void @call_store_as1() {
 ; CHECK-LABEL: define void @call_store_as1(
 ; CHECK-SAME: ) #[[ATTR3:[0-9]+]] {
-; CHECK-NEXT:    call void @store_as1(ptr addrspace(1) nocapture nofree noundef writeonly align 4294967296 null) #[[ATTR4]]
+; CHECK-NEXT:    call void @store_as1(ptr addrspace(1) nofree noundef writeonly align 4294967296 captures(none) null) #[[ATTR4]]
 ; CHECK-NEXT:    ret void
 ;
   call void @store_as1(ptr addrspace(1) null)
@@ -43,7 +43,7 @@ define void @call_store_as1() {
 
 define void @store_as1(ptr addrspace(1) %arg) {
 ; CHECK-LABEL: define void @store_as1(
-; CHECK-SAME: ptr addrspace(1) nocapture nofree noundef writeonly align 2 dereferenceable_or_null(2) [[ARG:%.*]]) #[[ATTR2]] {
+; CHECK-SAME: ptr addrspace(1) nofree noundef writeonly align 2 captures(none) dereferenceable_or_null(2) [[ARG:%.*]]) #[[ATTR2]] {
 ; CHECK-NEXT:    store i16 0, ptr addrspace(1) [[ARG]], align 2
 ; CHECK-NEXT:    ret void
 ;
diff --git a/llvm/test/Transforms/Attributor/liveness.ll b/llvm/test/Transforms/Attributor/liveness.ll
index f17bd5795a174..874eff661f053 100644
--- a/llvm/test/Transforms/Attributor/liveness.ll
+++ b/llvm/test/Transforms/Attributor/liveness.ll
@@ -98,7 +98,7 @@ define internal i32 @internal_load(ptr) norecurse nounwind uwtable {
 define i32 @first_block_no_return(i32 %a, ptr nonnull %ptr1, ptr %ptr2) #0 {
 ; TUNIT: Function Attrs: nofree noreturn nosync nounwind
 ; TUNIT-LABEL: define {{[^@]+}}@first_block_no_return
-; TUNIT-SAME: (i32 [[A:%.*]], ptr nocapture nofree nonnull readnone [[PTR1:%.*]], ptr nocapture nofree readnone [[PTR2:%.*]]) #[[ATTR0:[0-9]+]] {
+; TUNIT-SAME: (i32 [[A:%.*]], ptr nofree nonnull readnone captures(none) [[PTR1:%.*]], ptr nofree readnone captures(none) [[PTR2:%.*]]) #[[ATTR0:[0-9]+]] {
 ; TUNIT-NEXT:  entry:
 ; TUNIT-NEXT:    call void @no_return_call() #[[ATTR15:[0-9]+]]
 ; TUNIT-NEXT:    unreachable
@@ -111,7 +111,7 @@ define i32 @first_block_no_return(i32 %a, ptr nonnull %ptr1, ptr %ptr2) #0 {
 ;
 ; CGSCC: Function Attrs: nofree noreturn nosync nounwind
 ; CGSCC-LABEL: define {{[^@]+}}@first_block_no_return
-; CGSCC-SAME: (i32 [[A:%.*]], ptr nocapture nofree nonnull readnone [[PTR1:%.*]], ptr nocapture nofree readnone [[PTR2:%.*]]) #[[ATTR0:[0-9]+]] {
+; CGSCC-SAME: (i32 [[A:%.*]], ptr nofree nonnull readnone captures(none) [[PTR1:%.*]], ptr nofree readnone captures(none) [[PTR2:%.*]]) #[[ATTR0:[0-9]+]] {
 ; CGSCC-NEXT:  entry:
 ; CGSCC-NEXT:    call void @no_return_call() #[[ATTR17:[0-9]+]]
 ; CGSCC-NEXT:    unreachable
@@ -543,7 +543,7 @@ cleanup:
 define void @ub(ptr %0) {
 ; TUNIT: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: write)
 ; TUNIT-LABEL: define {{[^@]+}}@ub
-; TUNIT-SAME: (ptr nocapture nofree writeonly [[TMP0:%.*]]) #[[ATTR7:[0-9]+]] {
+; TUNIT-SAME: (ptr nofree writeonly captures(none) [[TMP0:%.*]]) #[[ATTR7:[0-9]+]] {
 ; TUNIT-NEXT:    [[POISON:%.*]] = sub nuw i32 0, 1
 ; TUNIT-NEXT:    [[STILL_POISON:%.*]] = and i32 [[POISON]], 0
 ; TUNIT-NEXT:    [[POISON_YET_AGAIN:%.*]] = getelementptr i32, ptr [[TMP0]], i32 [[STILL_POISON]]
@@ -552,7 +552,7 @@ define void @ub(ptr %0) {
 ;
 ; CGSCC: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: write)
 ; CGSCC-LABEL: define {{[^@]+}}@ub
-; CGSCC-SAME: (ptr nocapture nofree writeonly [[TMP0:%.*]]) #[[ATTR9:[0-9]+]] {
+; CGSCC-SAME: (ptr nofree writeonly captures(none) [[TMP0:%.*]]) #[[ATTR9:[0-9]+]] {
 ; CGSCC-NEXT:    [[POISON:%.*]] = sub nuw i32 0, 1
 ; CGSCC-NEXT:    [[STILL_POISON:%.*]] = and i32 [[POISON]], 0
 ; CGSCC-NEXT:    [[POISON_YET_AGAIN:%.*]] = getelementptr i32, ptr [[TMP0]], i32 [[STILL_POISON]]
@@ -2348,7 +2348,7 @@ define i32 @switch_default_dead_caller() {
 
 define void @call_via_pointer_with_dead_args(ptr %a, ptr %b, ptr %fp) {
 ; CHECK-LABEL: define {{[^@]+}}@call_via_pointer_with_dead_args
-; CHECK-SAME: (ptr [[A:%.*]], ptr [[B:%.*]], ptr nocapture nofree noundef nonnull [[FP:%.*]]) {
+; CHECK-SAME: (ptr [[A:%.*]], ptr [[B:%.*]], ptr nofree noundef nonnull captures(none) [[FP:%.*]]) {
 ; CHECK-NEXT:    call void [[FP]](ptr [[A]], ptr [[B]], ptr [[A]], i64 -1, ptr null)
 ; CHECK-NEXT:    ret void
 ;
@@ -2364,7 +2364,7 @@ define internal void @call_via_pointer_with_dead_args_internal_a(ptr %a, ptr %b,
 ;
 ; CGSCC-LABEL: define {{[^@]+}}@call_via_pointer_with_dead_args_internal_a
 ; CGSCC-SAME: (ptr [[A:%.*]], ptr noundef nonnull align 128 dereferenceable(4) [[B:%.*]]) {
-; CGSCC-NEXT:    call void @called_via_pointer(ptr [[A]], ptr nocapture nofree noundef nonnull align 128 dereferenceable(4) [[B]], ptr nocapture nofree [[A]], i64 noundef -1, ptr nofree noundef null)
+; CGSCC-NEXT:    call void @called_via_pointer(ptr [[A]], ptr nofree noundef nonnull align 128 captures(none) dereferenceable(4) [[B]], ptr nofree captures(none) [[A]], i64 noundef -1, ptr nofree noundef null)
 ; CGSCC-NEXT:    ret void
 ;
   call void %fp(ptr %a, ptr %b, ptr %a, i64 -1, ptr null)
@@ -2391,8 +2391,8 @@ define void @call_via_pointer_with_dead_args_caller(ptr %a, ptr %b) {
 ; CHECK-NEXT:    [[PTR2:%.*]] = alloca i32, align 128
 ; CHECK-NEXT:    [[PTR3:%.*]] = alloca i32, align 128
 ; CHECK-NEXT:    [[PTR4:%.*]] = alloca i32, align 128
-; CHECK-NEXT:    call void @call_via_pointer_with_dead_args(ptr [[A]], ptr noundef nonnull align 128 dereferenceable(4) [[PTR1]], ptr nocapture nofree noundef nonnull @called_via_pointer)
-; CHECK-NEXT:    call void @call_via_pointer_with_dead_args(ptr [[A]], ptr noundef nonnull align 128 dereferenceable(4) [[PTR2]], ptr nocapture nofree noundef nonnull @called_via_pointer_internal_1)
+; CHECK-NEXT:    call void @call_via_pointer_with_dead_args(ptr [[A]], ptr noundef nonnull align 128 dereferenceable(4) [[PTR1]], ptr nofree noundef nonnull captures(none) @called_via_pointer)
+; CHECK-NEXT:    call void @call_via_pointer_with_dead_args(ptr [[A]], ptr noundef nonnull align 128 dereferenceable(4) [[PTR2]], ptr nofree noundef nonnull captures(none) @called_via_pointer_internal_1)
 ; CHECK-NEXT:    call void @call_via_pointer_with_dead_args_internal_a(ptr [[B]], ptr noundef nonnull align 128 dereferenceable(4) [[PTR3]])
 ; CHECK-NEXT:    call void @call_via_pointer_with_dead_args_internal_b(ptr [[B]], ptr noundef nonnull align 128 dereferenceable(4) [[PTR4]])
 ; CHECK-NEXT:    ret void
@@ -2409,7 +2409,7 @@ define void @call_via_pointer_with_dead_args_caller(ptr %a, ptr %b) {
 }
 define void @called_via_pointer(ptr %a, ptr %b, ptr %c, i64 %d, ptr %e) {
 ; CHECK-LABEL: define {{[^@]+}}@called_via_pointer
-; CHECK-SAME: (ptr [[A:%.*]], ptr nocapture nofree readnone [[B:%.*]], ptr nocapture nofree readnone [[C:%.*]], i64 [[D:%.*]], ptr nocapture nofree readnone [[E:%.*]]) {
+; CHECK-SAME: (ptr [[A:%.*]], ptr nofree readnone captures(none) [[B:%.*]], ptr nofree readnone captures(none) [[C:%.*]], i64 [[D:%.*]], ptr nofree readnone captures(none) [[E:%.*]]) {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    tail call void @use_i32p(ptr [[A]])
 ; CHECK-NEXT:    tail call void @use_i32p(ptr [[A]])
@@ -2422,7 +2422,7 @@ entry:
 }
 define internal void @called_via_pointer_internal_1(ptr %a, ptr %b, ptr %c, i64 %d, ptr %e) {
 ; CHECK-LABEL: define {{[^@]+}}@called_via_pointer_internal_1
-; CHECK-SAME: (ptr [[A:%.*]], ptr nocapture nofree readnone [[B:%.*]], ptr nocapture nofree readnone [[C:%.*]], i64 [[D:%.*]], ptr nocapture nofree readnone [[E:%.*]]) {
+; CHECK-SAME: (ptr [[A:%.*]], ptr nofree readnone captures(none) [[B:%.*]], ptr nofree readnone captures(none) [[C:%.*]], i64 [[D:%.*]], ptr nofree readnone captures(none) [[E:%.*]]) {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    tail call void @use_i32p(ptr [[A]])
 ; CHECK-NEXT:    tail call void @use_i32p(ptr [[A]])
@@ -2436,7 +2436,7 @@ entry:
 ; FIXME: Figure out why the MODULE has the unused arguments still
 define internal void @called_via_pointer_internal_2(ptr %a, ptr %b, ptr %c, i64 %d, ptr %e) {
 ; TUNIT-LABEL: define {{[^@]+}}@called_via_pointer_internal_2
-; TUNIT-SAME: (ptr [[A:%.*]], ptr nocapture nofree readnone [[B:%.*]], ptr nocapture nofree readnone [[C:%.*]], i64 [[D:%.*]], ptr nocapture nofree readnone [[E:%.*]]) {
+; TUNIT-SAME: (ptr [[A:%.*]], ptr nofree readnone captures(none) [[B:%.*]], ptr nofree readnone captures(none) [[C:%.*]], i64 [[D:%.*]], ptr nofree readnone captures(none) [[E:%.*]]) {
 ; TUNIT-NEXT:  entry:
 ; TUNIT-NEXT:    tail call void @use_i32p(ptr [[A]])
 ; TUNIT-NEXT:    tail call void @use_i32p(ptr [[A]])
@@ -2460,7 +2460,7 @@ declare void @use_i32p(ptr)
 define internal void @dead_with_blockaddress_users(ptr nocapture %pc) nounwind readonly {
 ; CGSCC: Function Attrs: nounwind memory(read)
 ; CGSCC-LABEL: define {{[^@]+}}@dead_with_blockaddress_users
-; CGSCC-SAME: (ptr nocapture [[PC:%.*]]) #[[ATTR15:[0-9]+]] {
+; CGSCC-SAME: (ptr captures(none) [[PC:%.*]]) #[[ATTR15:[0-9]+]] {
 ; CGSCC-NEXT:  entry:
 ; CGSCC-NEXT:    br label [[INDIRECTGOTO:%.*]]
 ; CGSCC:       lab0:
@@ -2589,7 +2589,7 @@ define void @bad_gep() {
 ; TUNIT-NEXT:  entry:
 ; TUNIT-NEXT:    [[N1:%.*]] = alloca i8, i32 0, align 1
 ; TUNIT-NEXT:    [[M2:%.*]] = alloca i8, i32 0, align 1
-; TUNIT-NEXT:    call void @llvm.lifetime.start.p0(i64 noundef 1, ptr noalias nocapture nofree noundef nonnull dereferenceable(1) [[N1]]) #[[ATTR18:[0-9]+]]
+; TUNIT-NEXT:    call void @llvm.lifetime.start.p0(i64 noundef 1, ptr noalias nofree noundef nonnull captures(none) dereferenceable(1) [[N1]]) #[[ATTR18:[0-9]+]]
 ; TUNIT-NEXT:    br label [[EXIT:%.*]]
 ; TUNIT:       while.body:
 ; TUNIT-NEXT:    unreachable
@@ -2598,7 +2598,7 @@ define void @bad_gep() {
 ; TUNIT:       if.end:
 ; TUNIT-NEXT:    unreachable
 ; TUNIT:       exit:
-; TUNIT-NEXT:    call void @llvm.lifetime.end.p0(i64 noundef 1, ptr noalias nocapture nofree noundef nonnull dereferenceable(1) [[N1]]) #[[ATTR18]]
+; TUNIT-NEXT:    call void @llvm.lifetime.end.p0(i64 noundef 1, ptr noalias nofree noundef nonnull captures(none) dereferenceable(1) [[N1]]) #[[ATTR18]]
 ; TUNIT-NEXT:    ret void
 ;
 ; CGSCC: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(none)
@@ -2607,7 +2607,7 @@ define void @bad_gep() {
 ; CGSCC-NEXT:  entry:
 ; CGSCC-NEXT:    [[N1:%.*]] = alloca i8, i32 0, align 1
 ; CGSCC-NEXT:    [[M2:%.*]] = alloca i8, i32 0, align 1
-; CGSCC-NEXT:    call void @llvm.lifetime.start.p0(i64 noundef 1, ptr noalias nocapture nofree noundef nonnull dereferenceable(1) [[N1]]) #[[ATTR21:[0-9]+]]
+; CGSCC-NEXT:    call void @llvm.lifetime.start.p0(i64 noundef 1, ptr noalias nofree noundef nonnull captures(none) dereferenceable(1) [[N1]]) #[[ATTR21:[0-9]+]]
 ; CGSCC-NEXT:    br label [[EXIT:%.*]]
 ; CGSCC:       while.body:
 ; CGSCC-NEXT:    unreachable
@@ -2616,7 +2616,7 @@ define void @bad_gep() {
 ; CGSCC:       if.end:
 ; CGSCC-NEXT:    unreachable
 ; CGSCC:       exit:
-; CGSCC-NEXT:    call void @llvm.lifetime.end.p0(i64 noundef 1, ptr noalias nocapture nofree noundef nonnull dereferenceable(1) [[N1]]) #[[ATTR21]]
+; CGSCC-NEXT:    call void @llvm.lifetime.end.p0(i64 noundef 1, ptr noalias nofree noundef nonnull captures(none) dereferenceable(1) [[N1]]) #[[ATTR21]]
 ; CGSCC-NEXT:    ret void
 ;
 entry:
diff --git a/llvm/test/Transforms/Attributor/memory_locations.ll b/llvm/test/Transforms/Attributor/memory_locations.ll
index a7d3fba9cf9b8..22cdf299bfc3e 100644
--- a/llvm/test/Transforms/Attributor/memory_locations.ll
+++ b/llvm/test/Transforms/Attributor/memory_locations.ll
@@ -175,7 +175,7 @@ return:                                           ; preds = %if.end, %if.then
 define dso_local ptr @internal_argmem_only_read(ptr %arg) {
 ; CHECK: Function Attrs: memory(argmem: readwrite, inaccessiblemem: readwrite)
 ; CHECK-LABEL: define {{[^@]+}}@internal_argmem_only_read
-; CHECK-SAME: (ptr nocapture nofree noundef nonnull readonly align 4 dereferenceable(4) [[ARG:%.*]]) #[[ATTR1:[0-9]+]] {
+; CHECK-SAME: (ptr nofree noundef nonnull readonly align 4 captures(none) dereferenceable(4) [[ARG:%.*]]) #[[ATTR1:[0-9]+]] {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[TMP:%.*]] = load i32, ptr [[ARG]], align 4
 ; CHECK-NEXT:    [[CONV:%.*]] = sext i32 [[TMP]] to i64
@@ -192,7 +192,7 @@ entry:
 define dso_local ptr @internal_argmem_only_write(ptr %arg) {
 ; CHECK: Function Attrs: memory(argmem: readwrite, inaccessiblemem: readwrite)
 ; CHECK-LABEL: define {{[^@]+}}@internal_argmem_only_write
-; CHECK-SAME: (ptr nocapture nofree noundef nonnull writeonly align 4 dereferenceable(4) [[ARG:%.*]]) #[[ATTR1]] {
+; CHECK-SAME: (ptr nofree noundef nonnull writeonly align 4 captures(none) dereferenceable(4) [[ARG:%.*]]) #[[ATTR1]] {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    store i32 10, ptr [[ARG]], align 4
 ; CHECK-NEXT:    [[CALL:%.*]] = call noalias noundef dereferenceable_or_null(10) ptr @malloc(i64 noundef 10)
@@ -207,16 +207,16 @@ entry:
 define dso_local ptr @internal_argmem_only_rec(ptr %arg) {
 ; TUNIT: Function Attrs: memory(argmem: readwrite, inaccessiblemem: readwrite)
 ; TUNIT-LABEL: define {{[^@]+}}@internal_argmem_only_rec
-; TUNIT-SAME: (ptr nocapture nofree [[ARG:%.*]]) #[[ATTR1]] {
+; TUNIT-SAME: (ptr nofree captures(none) [[ARG:%.*]]) #[[ATTR1]] {
 ; TUNIT-NEXT:  entry:
-; TUNIT-NEXT:    [[CALL:%.*]] = call noalias ptr @internal_argmem_only_rec_1(ptr nocapture nofree noundef align 4 [[ARG]])
+; TUNIT-NEXT:    [[CALL:%.*]] = call noalias ptr @internal_argmem_only_rec_1(ptr nofree noundef align 4 captures(none) [[ARG]])
 ; TUNIT-NEXT:    ret ptr [[CALL]]
 ;
 ; CGSCC: Function Attrs: memory(argmem: readwrite, inaccessiblemem: readwrite)
 ; CGSCC-LABEL: define {{[^@]+}}@internal_argmem_only_rec
-; CGSCC-SAME: (ptr nocapture nofree noundef nonnull align 4 dereferenceable(4) [[ARG:%.*]]) #[[ATTR1]] {
+; CGSCC-SAME: (ptr nofree noundef nonnull align 4 captures(none) dereferenceable(4) [[ARG:%.*]]) #[[ATTR1]] {
 ; CGSCC-NEXT:  entry:
-; CGSCC-NEXT:    [[CALL:%.*]] = call noalias ptr @internal_argmem_only_rec_1(ptr nocapture nofree noundef nonnull align 4 dereferenceable(4) [[ARG]])
+; CGSCC-NEXT:    [[CALL:%.*]] = call noalias ptr @internal_argmem_only_rec_1(ptr nofree noundef nonnull align 4 captures(none) dereferenceable(4) [[ARG]])
 ; CGSCC-NEXT:    ret ptr [[CALL]]
 ;
 entry:
@@ -227,7 +227,7 @@ entry:
 define internal ptr @internal_argmem_only_rec_1(ptr %arg) {
 ; CHECK: Function Attrs: memory(argmem: readwrite, inaccessiblemem: readwrite)
 ; CHECK-LABEL: define {{[^@]+}}@internal_argmem_only_rec_1
-; CHECK-SAME: (ptr nocapture nofree noundef nonnull align 4 dereferenceable(4) [[ARG:%.*]]) #[[ATTR1]] {
+; CHECK-SAME: (ptr nofree noundef nonnull align 4 captures(none) dereferenceable(4) [[ARG:%.*]]) #[[ATTR1]] {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[TMP:%.*]] = load i32, ptr [[ARG]], align 4
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[TMP]], 0
@@ -240,7 +240,7 @@ define internal ptr @internal_argmem_only_rec_1(ptr %arg) {
 ; CHECK-NEXT:    br i1 [[CMP1]], label [[IF_THEN2:%.*]], label [[IF_END3:%.*]]
 ; CHECK:       if.then2:
 ; CHECK-NEXT:    [[ADD_PTR:%.*]] = getelementptr inbounds i32, ptr [[ARG]], i64 -1
-; CHECK-NEXT:    [[CALL:%.*]] = call noalias ptr @internal_argmem_only_rec_2(ptr nocapture nofree noundef nonnull align 4 dereferenceable(4) [[ADD_PTR]])
+; CHECK-NEXT:    [[CALL:%.*]] = call noalias ptr @internal_argmem_only_rec_2(ptr nofree noundef nonnull align 4 captures(none) dereferenceable(4) [[ADD_PTR]])
 ; CHECK-NEXT:    br label [[RETURN]]
 ; CHECK:       if.end3:
 ; CHECK-NEXT:    [[TMP2:%.*]] = load i32, ptr [[ARG]], align 4
@@ -283,11 +283,11 @@ return:                                           ; preds = %if.end3, %if.then2,
 define internal ptr @internal_argmem_only_rec_2(ptr %arg) {
 ; CHECK: Function Attrs: memory(argmem: readwrite, inaccessiblemem: readwrite)
 ; CHECK-LABEL: define {{[^@]+}}@internal_argmem_only_rec_2
-; CHECK-SAME: (ptr nocapture nofree noundef nonnull align 4 dereferenceable(4) [[ARG:%.*]]) #[[ATTR1]] {
+; CHECK-SAME: (ptr nofree noundef nonnull align 4 captures(none) dereferenceable(4) [[ARG:%.*]]) #[[ATTR1]] {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    store i32 0, ptr [[ARG]], align 4
 ; CHECK-NEXT:    [[ADD_PTR:%.*]] = getelementptr inbounds i32, ptr [[ARG]], i64 -1
-; CHECK-NEXT:    [[CALL:%.*]] = call noalias ptr @internal_argmem_only_rec_1(ptr nocapture nofree noundef nonnull align 4 dereferenceable(4) [[ADD_PTR]])
+; CHECK-NEXT:    [[CALL:%.*]] = call noalias ptr @internal_argmem_only_rec_1(ptr nofree noundef nonnull align 4 captures(none) dereferenceable(4) [[ADD_PTR]])
 ; CHECK-NEXT:    ret ptr [[CALL]]
 ;
 entry:
@@ -390,7 +390,7 @@ define void @callerD2() {
 define void @callerE(ptr %arg) {
 ; CHECK: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(none)
 ; CHECK-LABEL: define {{[^@]+}}@callerE
-; CHECK-SAME: (ptr nocapture nofree readnone [[ARG:%.*]]) #[[ATTR5:[0-9]+]] {
+; CHECK-SAME: (ptr nofree readnone captures(none) [[ARG:%.*]]) #[[ATTR5:[0-9]+]] {
 ; CHECK-NEXT:    ret void
 ;
   call void @llvm.lifetime.start.p0(i64 4, ptr %arg)
@@ -411,7 +411,7 @@ define void @write_global() {
 define void @write_global_via_arg(ptr %GPtr) {
 ; CHECK: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: write)
 ; CHECK-LABEL: define {{[^@]+}}@write_global_via_arg
-; CHECK-SAME: (ptr nocapture nofree noundef nonnull writeonly align 4 dereferenceable(4) [[GPTR:%.*]]) #[[ATTR7:[0-9]+]] {
+; CHECK-SAME: (ptr nofree noundef nonnull writeonly align 4 captures(none) dereferenceable(4) [[GPTR:%.*]]) #[[ATTR7:[0-9]+]] {
 ; CHECK-NEXT:    store i32 0, ptr [[GPTR]], align 4
 ; CHECK-NEXT:    ret void
 ;
@@ -449,13 +449,13 @@ define void @writeonly_global_via_arg() {
 ; TUNIT: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(write)
 ; TUNIT-LABEL: define {{[^@]+}}@writeonly_global_via_arg
 ; TUNIT-SAME: () #[[ATTR6]] {
-; TUNIT-NEXT:    call void @write_global_via_arg(ptr nocapture nofree noundef nonnull writeonly align 4 dereferenceable(4) @G) #[[ATTR12]]
+; TUNIT-NEXT:    call void @write_global_via_arg(ptr nofree noundef nonnull writeonly align 4 captures(none) dereferenceable(4) @G) #[[ATTR12]]
 ; TUNIT-NEXT:    ret void
 ;
 ; CGSCC: Function Attrs: mustprogress nofree nosync nounwind willreturn memory(write)
 ; CGSCC-LABEL: define {{[^@]+}}@writeonly_global_via_arg
 ; CGSCC-SAME: () #[[ATTR9]] {
-; CGSCC-NEXT:    call void @write_global_via_arg(ptr nocapture nofree noundef nonnull writeonly align 4 dereferenceable(4) @G) #[[ATTR13]]
+; CGSCC-NEXT:    call void @write_global_via_arg(ptr nofree noundef nonnull writeonly align 4 captures(none) dereferenceable(4) @G) #[[ATTR13]]
 ; CGSCC-NEXT:    ret void
 ;
   call void @write_global_via_arg(ptr @G)
@@ -483,11 +483,11 @@ define void @writeonly_global_via_arg_internal() {
 define i8 @recursive_not_readnone(ptr %ptr, i1 %c) {
 ; TUNIT: Function Attrs: nofree nosync nounwind memory(argmem: write)
 ; TUNIT-LABEL: define {{[^@]+}}@recursive_not_readnone
-; TUNIT-SAME: (ptr nocapture nofree writeonly [[PTR:%.*]], i1 noundef [[C:%.*]]) #[[ATTR9:[0-9]+]] {
+; TUNIT-SAME: (ptr nofree writeonly captures(none) [[PTR:%.*]], i1 noundef [[C:%.*]]) #[[ATTR9:[0-9]+]] {
 ; TUNIT-NEXT:    [[ALLOC:%.*]] = alloca i8, align 1
 ; TUNIT-NEXT:    br i1 [[C]], label [[T:%.*]], label [[F:%.*]]
 ; TUNIT:       t:
-; TUNIT-NEXT:    [[TMP1:%.*]] = call i8 @recursive_not_readnone(ptr noalias nocapture nofree noundef nonnull writeonly dereferenceable(1) [[ALLOC]], i1 noundef false) #[[ATTR13:[0-9]+]]
+; TUNIT-NEXT:    [[TMP1:%.*]] = call i8 @recursive_not_readnone(ptr noalias nofree noundef nonnull writeonly captures(none) dereferenceable(1) [[ALLOC]], i1 noundef false) #[[ATTR13:[0-9]+]]
 ; TUNIT-NEXT:    ret i8 1
 ; TUNIT:       f:
 ; TUNIT-NEXT:    store i8 1, ptr [[PTR]], align 1
@@ -495,11 +495,11 @@ define i8 @recursive_not_readnone(ptr %ptr, i1 %c) {
 ;
 ; CGSCC: Function Attrs: nofree nosync nounwind memory(argmem: write)
 ; CGSCC-LABEL: define {{[^@]+}}@recursive_not_readnone
-; CGSCC-SAME: (ptr nocapture nofree writeonly [[PTR:%.*]], i1 noundef [[C:%.*]]) #[[ATTR10:[0-9]+]] {
+; CGSCC-SAME: (ptr nofree writeonly captures(none) [[PTR:%.*]], i1 noundef [[C:%.*]]) #[[ATTR10:[0-9]+]] {
 ; CGSCC-NEXT:    [[ALLOC:%.*]] = alloca i8, align 1
 ; CGSCC-NEXT:    br i1 [[C]], label [[T:%.*]], label [[F:%.*]]
 ; CGSCC:       t:
-; CGSCC-NEXT:    [[TMP1:%.*]] = call i8 @recursive_not_readnone(ptr noalias nocapture nofree noundef nonnull writeonly dereferenceable(1) [[ALLOC]], i1 noundef false) #[[ATTR14:[0-9]+]]
+; CGSCC-NEXT:    [[TMP1:%.*]] = call i8 @recursive_not_readnone(ptr noalias nofree noundef nonnull writeonly captures(none) dereferenceable(1) [[ALLOC]], i1 noundef false) #[[ATTR14:[0-9]+]]
 ; CGSCC-NEXT:    ret i8 1
 ; CGSCC:       f:
 ; CGSCC-NEXT:    store i8 1, ptr [[PTR]], align 1
@@ -519,11 +519,11 @@ f:
 define internal i8 @recursive_not_readnone_internal(ptr %ptr, i1 %c) {
 ; TUNIT: Function Attrs: nofree nosync nounwind memory(argmem: write)
 ; TUNIT-LABEL: define {{[^@]+}}@recursive_not_readnone_internal
-; TUNIT-SAME: (ptr noalias nocapture nofree noundef nonnull writeonly dereferenceable(1) [[PTR:%.*]], i1 noundef [[C:%.*]]) #[[ATTR9]] {
+; TUNIT-SAME: (ptr noalias nofree noundef nonnull writeonly captures(none) dereferenceable(1) [[PTR:%.*]], i1 noundef [[C:%.*]]) #[[ATTR9]] {
 ; TUNIT-NEXT:    [[ALLOC:%.*]] = alloca i8, align 1
 ; TUNIT-NEXT:    br i1 [[C]], label [[T:%.*]], label [[F:%.*]]
 ; TUNIT:       t:
-; TUNIT-NEXT:    [[TMP1:%.*]] = call i8 @recursive_not_readnone_internal(ptr noalias nocapture nofree noundef nonnull writeonly dereferenceable(1) [[ALLOC]], i1 noundef false) #[[ATTR13]]
+; TUNIT-NEXT:    [[TMP1:%.*]] = call i8 @recursive_not_readnone_internal(ptr noalias nofree noundef nonnull writeonly captures(none) dereferenceable(1) [[ALLOC]], i1 noundef false) #[[ATTR13]]
 ; TUNIT-NEXT:    ret i8 1
 ; TUNIT:       f:
 ; TUNIT-NEXT:    store i8 1, ptr [[PTR]], align 1
@@ -531,11 +531,11 @@ define internal i8 @recursive_not_readnone_internal(ptr %ptr, i1 %c) {
 ;
 ; CGSCC: Function Attrs: nofree nosync nounwind memory(argmem: write)
 ; CGSCC-LABEL: define {{[^@]+}}@recursive_not_readnone_internal
-; CGSCC-SAME: (ptr noalias nocapture nofree noundef nonnull writeonly dereferenceable(1) [[PTR:%.*]], i1 noundef [[C:%.*]]) #[[ATTR10]] {
+; CGSCC-SAME: (ptr noalias nofree noundef nonnull writeonly captures(none) dereferenceable(1) [[PTR:%.*]], i1 noundef [[C:%.*]]) #[[ATTR10]] {
 ; CGSCC-NEXT:    [[ALLOC:%.*]] = alloca i8, align 1
 ; CGSCC-NEXT:    br i1 [[C]], label [[T:%.*]], label [[F:%.*]]
 ; CGSCC:       t:
-; CGSCC-NEXT:    [[TMP1:%.*]] = call i8 @recursive_not_readnone_internal(ptr noalias nocapture nofree noundef nonnull writeonly dereferenceable(1) [[ALLOC]], i1 noundef false) #[[ATTR14]]
+; CGSCC-NEXT:    [[TMP1:%.*]] = call i8 @recursive_not_readnone_internal(ptr noalias nofree noundef nonnull writeonly captures(none) dereferenceable(1) [[ALLOC]], i1 noundef false) #[[ATTR14]]
 ; CGSCC-NEXT:    ret i8 1
 ; CGSCC:       f:
 ; CGSCC-NEXT:    store i8 1, ptr [[PTR]], align 1
@@ -557,14 +557,14 @@ define i8 @readnone_caller(i1 %c) {
 ; TUNIT-LABEL: define {{[^@]+}}@readnone_caller
 ; TUNIT-SAME: (i1 [[C:%.*]]) #[[ATTR10:[0-9]+]] {
 ; TUNIT-NEXT:    [[A:%.*]] = alloca i8, align 1
-; TUNIT-NEXT:    [[R:%.*]] = call i8 @recursive_not_readnone_internal(ptr noalias nocapture nofree noundef nonnull writeonly dereferenceable(1) [[A]], i1 noundef [[C]]) #[[ATTR13]]
+; TUNIT-NEXT:    [[R:%.*]] = call i8 @recursive_not_readnone_internal(ptr noalias nofree noundef nonnull writeonly captures(none) dereferenceable(1) [[A]], i1 noundef [[C]]) #[[ATTR13]]
 ; TUNIT-NEXT:    ret i8 [[R]]
 ;
 ; CGSCC: Function Attrs: nofree nosync nounwind memory(none)
 ; CGSCC-LABEL: define {{[^@]+}}@readnone_caller
 ; CGSCC-SAME: (i1 noundef [[C:%.*]]) #[[ATTR11:[0-9]+]] {
 ; CGSCC-NEXT:    [[A:%.*]] = alloca i8, align 1
-; CGSCC-NEXT:    [[R:%.*]] = call i8 @recursive_not_readnone_internal(ptr noalias nocapture nofree noundef nonnull writeonly dereferenceable(1) [[A]], i1 noundef [[C]]) #[[ATTR15:[0-9]+]]
+; CGSCC-NEXT:    [[R:%.*]] = call i8 @recursive_not_readnone_internal(ptr noalias nofree noundef nonnull writeonly captures(none) dereferenceable(1) [[A]], i1 noundef [[C]]) #[[ATTR15:[0-9]+]]
 ; CGSCC-NEXT:    ret i8 [[R]]
 ;
   %a = alloca i8
@@ -575,11 +575,11 @@ define i8 @readnone_caller(i1 %c) {
 define internal i8 @recursive_readnone_internal2(ptr %ptr, i1 %c) {
 ; TUNIT: Function Attrs: nofree nosync nounwind memory(argmem: write)
 ; TUNIT-LABEL: define {{[^@]+}}@recursive_readnone_internal2
-; TUNIT-SAME: (ptr noalias nocapture nofree writeonly [[PTR:%.*]], i1 noundef [[C:%.*]]) #[[ATTR9]] {
+; TUNIT-SAME: (ptr noalias nofree writeonly captures(none) [[PTR:%.*]], i1 noundef [[C:%.*]]) #[[ATTR9]] {
 ; TUNIT-NEXT:    [[ALLOC:%.*]] = alloca i8, align 1
 ; TUNIT-NEXT:    br i1 [[C]], label [[T:%.*]], label [[F:%.*]]
 ; TUNIT:       t:
-; TUNIT-NEXT:    [[TMP1:%.*]] = call i8 @recursive_readnone_internal2(ptr noalias nocapture nofree noundef nonnull writeonly dereferenceable(1) [[ALLOC]], i1 noundef false) #[[ATTR13]]
+; TUNIT-NEXT:    [[TMP1:%.*]] = call i8 @recursive_readnone_internal2(ptr noalias nofree noundef nonnull writeonly captures(none) dereferenceable(1) [[ALLOC]], i1 noundef false) #[[ATTR13]]
 ; TUNIT-NEXT:    ret i8 1
 ; TUNIT:       f:
 ; TUNIT-NEXT:    store i8 1, ptr [[PTR]], align 1
@@ -587,11 +587,11 @@ define internal i8 @recursive_readnone_internal2(ptr %ptr, i1 %c) {
 ;
 ; CGSCC: Function Attrs: nofree nosync nounwind memory(argmem: write)
 ; CGSCC-LABEL: define {{[^@]+}}@recursive_readnone_internal2
-; CGSCC-SAME: (ptr noalias nocapture nofree writeonly [[PTR:%.*]], i1 noundef [[C:%.*]]) #[[ATTR10]] {
+; CGSCC-SAME: (ptr noalias nofree writeonly captures(none) [[PTR:%.*]], i1 noundef [[C:%.*]]) #[[ATTR10]] {
 ; CGSCC-NEXT:    [[ALLOC:%.*]] = alloca i8, align 1
 ; CGSCC-NEXT:    br i1 [[C]], label [[T:%.*]], label [[F:%.*]]
 ; CGSCC:       t:
-; CGSCC-NEXT:    [[TMP1:%.*]] = call i8 @recursive_readnone_internal2(ptr noalias nocapture nofree noundef nonnull writeonly dereferenceable(1) [[ALLOC]], i1 noundef false) #[[ATTR14]]
+; CGSCC-NEXT:    [[TMP1:%.*]] = call i8 @recursive_readnone_internal2(ptr noalias nofree noundef nonnull writeonly captures(none) dereferenceable(1) [[ALLOC]], i1 noundef false) #[[ATTR14]]
 ; CGSCC-NEXT:    ret i8 1
 ; CGSCC:       f:
 ; CGSCC-NEXT:    store i8 1, ptr [[PTR]], align 1
@@ -628,11 +628,11 @@ define i8 @readnone_caller2(i1 %c) {
 define internal i8 @recursive_not_readnone_internal3(ptr %ptr, i1 %c) {
 ; TUNIT: Function Attrs: nofree nosync nounwind memory(argmem: write)
 ; TUNIT-LABEL: define {{[^@]+}}@recursive_not_readnone_internal3
-; TUNIT-SAME: (ptr noalias nocapture nofree noundef nonnull writeonly dereferenceable(1) [[PTR:%.*]], i1 noundef [[C:%.*]]) #[[ATTR9]] {
+; TUNIT-SAME: (ptr noalias nofree noundef nonnull writeonly captures(none) dereferenceable(1) [[PTR:%.*]], i1 noundef [[C:%.*]]) #[[ATTR9]] {
 ; TUNIT-NEXT:    [[ALLOC:%.*]] = alloca i8, align 1
 ; TUNIT-NEXT:    br i1 [[C]], label [[T:%.*]], label [[F:%.*]]
 ; TUNIT:       t:
-; TUNIT-NEXT:    [[TMP1:%.*]] = call i8 @recursive_not_readnone_internal3(ptr noalias nocapture nofree noundef nonnull writeonly dereferenceable(1) [[ALLOC]], i1 noundef false) #[[ATTR13]]
+; TUNIT-NEXT:    [[TMP1:%.*]] = call i8 @recursive_not_readnone_internal3(ptr noalias nofree noundef nonnull writeonly captures(none) dereferenceable(1) [[ALLOC]], i1 noundef false) #[[ATTR13]]
 ; TUNIT-NEXT:    ret i8 1
 ; TUNIT:       f:
 ; TUNIT-NEXT:    store i8 1, ptr [[PTR]], align 1
@@ -640,11 +640,11 @@ define internal i8 @recursive_not_readnone_internal3(ptr %ptr, i1 %c) {
 ;
 ; CGSCC: Function Attrs: nofree nosync nounwind memory(argmem: write)
 ; CGSCC-LABEL: define {{[^@]+}}@recursive_not_readnone_internal3
-; CGSCC-SAME: (ptr noalias nocapture nofree noundef nonnull writeonly dereferenceable(1) [[PTR:%.*]], i1 noundef [[C:%.*]]) #[[ATTR10]] {
+; CGSCC-SAME: (ptr noalias nofree noundef nonnull writeonly captures(none) dereferenceable(1) [[PTR:%.*]], i1 noundef [[C:%.*]]) #[[ATTR10]] {
 ; CGSCC-NEXT:    [[ALLOC:%.*]] = alloca i8, align 1
 ; CGSCC-NEXT:    br i1 [[C]], label [[T:%.*]], label [[F:%.*]]
 ; CGSCC:       t:
-; CGSCC-NEXT:    [[TMP1:%.*]] = call i8 @recursive_not_readnone_internal3(ptr noalias nocapture nofree noundef nonnull writeonly dereferenceable(1) [[ALLOC]], i1 noundef false) #[[ATTR14]]
+; CGSCC-NEXT:    [[TMP1:%.*]] = call i8 @recursive_not_readnone_internal3(ptr noalias nofree noundef nonnull writeonly captures(none) dereferenceable(1) [[ALLOC]], i1 noundef false) #[[ATTR14]]
 ; CGSCC-NEXT:    ret i8 1
 ; CGSCC:       f:
 ; CGSCC-NEXT:    store i8 1, ptr [[PTR]], align 1
@@ -666,14 +666,14 @@ define i8 @readnone_caller3(i1 %c) {
 ; TUNIT-LABEL: define {{[^@]+}}@readnone_caller3
 ; TUNIT-SAME: (i1 [[C:%.*]]) #[[ATTR10]] {
 ; TUNIT-NEXT:    [[ALLOC:%.*]] = alloca i8, align 1
-; TUNIT-NEXT:    [[R:%.*]] = call i8 @recursive_not_readnone_internal3(ptr noalias nocapture nofree noundef nonnull writeonly dereferenceable(1) [[ALLOC]], i1 noundef [[C]]) #[[ATTR13]]
+; TUNIT-NEXT:    [[R:%.*]] = call i8 @recursive_not_readnone_internal3(ptr noalias nofree noundef nonnull writeonly captures(none) dereferenceable(1) [[ALLOC]], i1 noundef [[C]]) #[[ATTR13]]
 ; TUNIT-NEXT:    ret i8 [[R]]
 ;
 ; CGSCC: Function Attrs: nofree nosync nounwind memory(none)
 ; CGSCC-LABEL: define {{[^@]+}}@readnone_caller3
 ; CGSCC-SAME: (i1 noundef [[C:%.*]]) #[[ATTR11]] {
 ; CGSCC-NEXT:    [[ALLOC:%.*]] = alloca i8, align 1
-; CGSCC-NEXT:    [[R:%.*]] = call i8 @recursive_not_readnone_internal3(ptr noalias nocapture nofree noundef nonnull writeonly dereferenceable(1) [[ALLOC]], i1 noundef [[C]]) #[[ATTR15]]
+; CGSCC-NEXT:    [[R:%.*]] = call i8 @recursive_not_readnone_internal3(ptr noalias nofree noundef nonnull writeonly captures(none) dereferenceable(1) [[ALLOC]], i1 noundef [[C]]) #[[ATTR15]]
 ; CGSCC-NEXT:    ret i8 [[R]]
 ;
   %alloc = alloca i8
diff --git a/llvm/test/Transforms/Attributor/misc.ll b/llvm/test/Transforms/Attributor/misc.ll
index 8b1ac70909639..c3458894fe84e 100644
--- a/llvm/test/Transforms/Attributor/misc.ll
+++ b/llvm/test/Transforms/Attributor/misc.ll
@@ -11,7 +11,7 @@ define internal void @internal(ptr %fp) {
 ; TUNIT-SAME: (ptr nonnull [[FP:%.*]]) {
 ; TUNIT-NEXT:  entry:
 ; TUNIT-NEXT:    [[A:%.*]] = alloca i32, align 4
-; TUNIT-NEXT:    call void @foo(ptr nocapture nofree noundef nonnull writeonly align 4 dereferenceable(4) [[A]]) #[[ATTR1:[0-9]+]]
+; TUNIT-NEXT:    call void @foo(ptr nofree noundef nonnull writeonly align 4 captures(none) dereferenceable(4) [[A]]) #[[ATTR1:[0-9]+]]
 ; TUNIT-NEXT:    call void [[FP]](ptr nonnull @foo)
 ; TUNIT-NEXT:    call void @callback1(ptr noundef nonnull @foo)
 ; TUNIT-NEXT:    call void @callback2(ptr noundef nonnull @foo)
@@ -23,7 +23,7 @@ define internal void @internal(ptr %fp) {
 ; CGSCC-SAME: (ptr noundef nonnull [[FP:%.*]]) {
 ; CGSCC-NEXT:  entry:
 ; CGSCC-NEXT:    [[A:%.*]] = alloca i32, align 4
-; CGSCC-NEXT:    call void @foo(ptr nocapture nofree noundef nonnull writeonly align 4 dereferenceable(4) [[A]]) #[[ATTR1:[0-9]+]]
+; CGSCC-NEXT:    call void @foo(ptr nofree noundef nonnull writeonly align 4 captures(none) dereferenceable(4) [[A]]) #[[ATTR1:[0-9]+]]
 ; CGSCC-NEXT:    call void [[FP]](ptr nonnull @foo)
 ; CGSCC-NEXT:    call void @callback1(ptr noundef nonnull @foo)
 ; CGSCC-NEXT:    call void @callback2(ptr noundef nonnull @foo)
@@ -49,7 +49,7 @@ define void @external(ptr %fp) {
 ; TUNIT-SAME: (ptr [[FP:%.*]]) {
 ; TUNIT-NEXT:  entry:
 ; TUNIT-NEXT:    [[A:%.*]] = alloca i32, align 4
-; TUNIT-NEXT:    call void @foo(ptr nocapture nofree noundef nonnull writeonly align 4 dereferenceable(4) [[A]]) #[[ATTR1]]
+; TUNIT-NEXT:    call void @foo(ptr nofree noundef nonnull writeonly align 4 captures(none) dereferenceable(4) [[A]]) #[[ATTR1]]
 ; TUNIT-NEXT:    call void @callback1(ptr noundef nonnull @foo)
 ; TUNIT-NEXT:    call void @callback2(ptr noundef nonnull @foo)
 ; TUNIT-NEXT:    call void @callback2(ptr [[FP]])
@@ -62,7 +62,7 @@ define void @external(ptr %fp) {
 ; CGSCC-SAME: (ptr [[FP:%.*]]) {
 ; CGSCC-NEXT:  entry:
 ; CGSCC-NEXT:    [[A:%.*]] = alloca i32, align 4
-; CGSCC-NEXT:    call void @foo(ptr nocapture nofree noundef nonnull writeonly align 4 dereferenceable(4) [[A]]) #[[ATTR1]]
+; CGSCC-NEXT:    call void @foo(ptr nofree noundef nonnull writeonly align 4 captures(none) dereferenceable(4) [[A]]) #[[ATTR1]]
 ; CGSCC-NEXT:    call void @callback1(ptr noundef nonnull @foo)
 ; CGSCC-NEXT:    call void @callback2(ptr noundef nonnull @foo)
 ; CGSCC-NEXT:    call void @callback2(ptr [[FP]])
@@ -87,7 +87,7 @@ define internal void @foo(ptr %a) {
 ;
 ; CHECK: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: write)
 ; CHECK-LABEL: define {{[^@]+}}@foo
-; CHECK-SAME: (ptr nocapture nofree noundef nonnull writeonly align 4 dereferenceable(4) [[A:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-SAME: (ptr nofree noundef nonnull writeonly align 4 captures(none) dereferenceable(4) [[A:%.*]]) #[[ATTR0:[0-9]+]] {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    store i32 0, ptr [[A]], align 4
 ; CHECK-NEXT:    ret void
diff --git a/llvm/test/Transforms/Attributor/misc_crash.ll b/llvm/test/Transforms/Attributor/misc_crash.ll
index 68e0e22f29cf0..21838097ae7db 100644
--- a/llvm/test/Transforms/Attributor/misc_crash.ll
+++ b/llvm/test/Transforms/Attributor/misc_crash.ll
@@ -38,7 +38,7 @@ define internal ptr @func1a(ptr %arg) {
 define internal void @func2a(ptr %0) {
 ; CHECK: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(write)
 ; CHECK-LABEL: define {{[^@]+}}@func2a
-; CHECK-SAME: (ptr nocapture nofree nonnull writeonly align 4 dereferenceable(4) [[TMP0:%.*]]) #[[ATTR1:[0-9]+]] {
+; CHECK-SAME: (ptr nofree nonnull writeonly align 4 captures(none) dereferenceable(4) [[TMP0:%.*]]) #[[ATTR1:[0-9]+]] {
 ; CHECK-NEXT:    ret void
 ;
   store i32 0, ptr %0
@@ -49,7 +49,7 @@ define i32 @func2() {
 ; CHECK: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(write)
 ; CHECK-LABEL: define {{[^@]+}}@func2
 ; CHECK-SAME: () #[[ATTR1]] {
-; CHECK-NEXT:    [[TMP1:%.*]] = tail call i32 (ptr, ...) @func2a(ptr nocapture nofree nonnull writeonly align 4 dereferenceable(4) undef) #[[ATTR3:[0-9]+]]
+; CHECK-NEXT:    [[TMP1:%.*]] = tail call i32 (ptr, ...) @func2a(ptr nofree nonnull writeonly align 4 captures(none) dereferenceable(4) undef) #[[ATTR3:[0-9]+]]
 ; CHECK-NEXT:    ret i32 0
 ;
   %1 = tail call i32 (ptr, ...) @func2a(ptr @var2)
@@ -61,7 +61,7 @@ define i32 @func3(i1 %false) {
 ; CHECK: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(write)
 ; CHECK-LABEL: define {{[^@]+}}@func3
 ; CHECK-SAME: (i1 [[FALSE:%.*]]) #[[ATTR1]] {
-; CHECK-NEXT:    [[TMP1:%.*]] = tail call i32 (ptr, ...) @func2a(ptr nocapture nofree nonnull writeonly align 4 dereferenceable(4) undef) #[[ATTR3]]
+; CHECK-NEXT:    [[TMP1:%.*]] = tail call i32 (ptr, ...) @func2a(ptr nofree nonnull writeonly align 4 captures(none) dereferenceable(4) undef) #[[ATTR3]]
 ; CHECK-NEXT:    br i1 [[FALSE]], label [[USE_BB:%.*]], label [[RET_BB:%.*]]
 ; CHECK:       use_bb:
 ; CHECK-NEXT:    ret i32 [[TMP1]]
@@ -118,7 +118,7 @@ define i16 @foo3() {
 define internal i16 @bar3(ptr %p1, i16 %p2) {
 ; CHECK: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(none)
 ; CHECK-LABEL: define {{[^@]+}}@bar3
-; CHECK-SAME: (ptr nocapture nofree readnone [[P1:%.*]], i16 returned [[P2:%.*]]) #[[ATTR0]] {
+; CHECK-SAME: (ptr nofree readnone captures(none) [[P1:%.*]], i16 returned [[P2:%.*]]) #[[ATTR0]] {
 ; CHECK-NEXT:    ret i16 [[P2]]
 ;
   ret i16 %p2
diff --git a/llvm/test/Transforms/Attributor/noalias.ll b/llvm/test/Transforms/Attributor/noalias.ll
index c63d81878f535..c22be2297ff25 100644
--- a/llvm/test/Transforms/Attributor/noalias.ll
+++ b/llvm/test/Transforms/Attributor/noalias.ll
@@ -45,7 +45,7 @@ define ptr @return_noalias(){
 define void @nocapture(ptr %a){
 ; CHECK: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(none)
 ; CHECK-LABEL: define {{[^@]+}}@nocapture
-; CHECK-SAME: (ptr nocapture nofree readnone [[A:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-SAME: (ptr nofree readnone captures(none) [[A:%.*]]) #[[ATTR0:[0-9]+]] {
 ; CHECK-NEXT:    ret void
 ;
   ret void
@@ -193,7 +193,7 @@ define ptr @test6() nounwind uwtable ssp {
 ; TUNIT-NEXT:    store i8 97, ptr [[X]], align 1
 ; TUNIT-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds [2 x i8], ptr [[X]], i64 0, i64 1
 ; TUNIT-NEXT:    store i8 0, ptr [[ARRAYIDX1]], align 1
-; TUNIT-NEXT:    [[CALL:%.*]] = call noalias ptr @strdup(ptr noalias nocapture noundef nonnull dereferenceable(2) [[X]]) #[[ATTR2]]
+; TUNIT-NEXT:    [[CALL:%.*]] = call noalias ptr @strdup(ptr noalias noundef nonnull captures(none) dereferenceable(2) [[X]]) #[[ATTR2]]
 ; TUNIT-NEXT:    ret ptr [[CALL]]
 ;
 ; CGSCC: Function Attrs: nounwind ssp uwtable
@@ -203,7 +203,7 @@ define ptr @test6() nounwind uwtable ssp {
 ; CGSCC-NEXT:    store i8 97, ptr [[X]], align 1
 ; CGSCC-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds [2 x i8], ptr [[X]], i64 0, i64 1
 ; CGSCC-NEXT:    store i8 0, ptr [[ARRAYIDX1]], align 1
-; CGSCC-NEXT:    [[CALL:%.*]] = call noalias ptr @strdup(ptr noalias nocapture noundef nonnull dereferenceable(2) [[X]]) #[[ATTR3]]
+; CGSCC-NEXT:    [[CALL:%.*]] = call noalias ptr @strdup(ptr noalias noundef nonnull captures(none) dereferenceable(2) [[X]]) #[[ATTR3]]
 ; CGSCC-NEXT:    ret ptr [[CALL]]
 ;
   %x = alloca [2 x i8], align 1
@@ -308,9 +308,9 @@ define internal void @test9a(ptr %a, ptr %b) {
 define internal void @test9b(ptr %a, ptr %b) {
 ; FIXME: %b should be noalias
 ; CHECK-LABEL: define {{[^@]+}}@test9b
-; CHECK-SAME: (ptr noalias nocapture [[A:%.*]], ptr nocapture [[B:%.*]]) {
-; CHECK-NEXT:    call void @use_i8(ptr noalias nocapture [[A]])
-; CHECK-NEXT:    call void @use_i8(ptr nocapture [[B]])
+; CHECK-SAME: (ptr noalias captures(none) [[A:%.*]], ptr captures(none) [[B:%.*]]) {
+; CHECK-NEXT:    call void @use_i8(ptr noalias captures(none) [[A]])
+; CHECK-NEXT:    call void @use_i8(ptr captures(none) [[B]])
 ; CHECK-NEXT:    ret void
 ;
   call void @use_i8(ptr %a)
@@ -319,10 +319,10 @@ define internal void @test9b(ptr %a, ptr %b) {
 }
 define internal void @test9c(ptr %a, ptr %b, ptr %c) {
 ; CHECK-LABEL: define {{[^@]+}}@test9c
-; CHECK-SAME: (ptr noalias nocapture [[A:%.*]], ptr nocapture [[B:%.*]], ptr nocapture [[C:%.*]]) {
-; CHECK-NEXT:    call void @use_i8(ptr noalias nocapture [[A]])
-; CHECK-NEXT:    call void @use_i8(ptr nocapture [[B]])
-; CHECK-NEXT:    call void @use_i8(ptr nocapture [[C]])
+; CHECK-SAME: (ptr noalias captures(none) [[A:%.*]], ptr captures(none) [[B:%.*]], ptr captures(none) [[C:%.*]]) {
+; CHECK-NEXT:    call void @use_i8(ptr noalias captures(none) [[A]])
+; CHECK-NEXT:    call void @use_i8(ptr captures(none) [[B]])
+; CHECK-NEXT:    call void @use_i8(ptr captures(none) [[C]])
 ; CHECK-NEXT:    ret void
 ;
   call void @use_i8(ptr %a)
@@ -332,13 +332,13 @@ define internal void @test9c(ptr %a, ptr %b, ptr %c) {
 }
 define void @test9_helper(ptr %a, ptr %b) {
 ; CHECK-LABEL: define {{[^@]+}}@test9_helper
-; CHECK-SAME: (ptr nocapture [[A:%.*]], ptr nocapture [[B:%.*]]) {
+; CHECK-SAME: (ptr captures(none) [[A:%.*]], ptr captures(none) [[B:%.*]]) {
 ; CHECK-NEXT:    tail call void @test9a()
 ; CHECK-NEXT:    tail call void @test9a()
-; CHECK-NEXT:    tail call void @test9b(ptr noalias nocapture [[A]], ptr nocapture [[B]])
-; CHECK-NEXT:    tail call void @test9b(ptr noalias nocapture [[B]], ptr noalias nocapture [[A]])
-; CHECK-NEXT:    tail call void @test9c(ptr noalias nocapture [[A]], ptr nocapture [[B]], ptr nocapture [[B]])
-; CHECK-NEXT:    tail call void @test9c(ptr noalias nocapture [[B]], ptr noalias nocapture [[A]], ptr noalias nocapture [[A]])
+; CHECK-NEXT:    tail call void @test9b(ptr noalias captures(none) [[A]], ptr captures(none) [[B]])
+; CHECK-NEXT:    tail call void @test9b(ptr noalias captures(none) [[B]], ptr noalias captures(none) [[A]])
+; CHECK-NEXT:    tail call void @test9c(ptr noalias captures(none) [[A]], ptr captures(none) [[B]], ptr captures(none) [[B]])
+; CHECK-NEXT:    tail call void @test9c(ptr noalias captures(none) [[B]], ptr noalias captures(none) [[A]], ptr noalias captures(none) [[A]])
 ; CHECK-NEXT:    ret void
 ;
   tail call void @test9a(ptr noalias %a, ptr %b)
@@ -401,10 +401,10 @@ define void @test12_1() {
 ; CHECK-LABEL: define {{[^@]+}}@test12_1() {
 ; CHECK-NEXT:    [[A:%.*]] = alloca i8, align 4
 ; CHECK-NEXT:    [[B:%.*]] = tail call noalias ptr @malloc(i64 noundef 4)
-; CHECK-NEXT:    tail call void @use_nocapture(ptr noalias nocapture noundef nonnull align 4 dereferenceable(1) [[A]])
-; CHECK-NEXT:    tail call void @use_nocapture(ptr noalias nocapture noundef nonnull align 4 dereferenceable(1) [[A]])
-; CHECK-NEXT:    tail call void @use_nocapture(ptr noalias nocapture [[B]])
-; CHECK-NEXT:    tail call void @use_nocapture(ptr noalias nocapture [[B]])
+; CHECK-NEXT:    tail call void @use_nocapture(ptr noalias noundef nonnull align 4 captures(none) dereferenceable(1) [[A]])
+; CHECK-NEXT:    tail call void @use_nocapture(ptr noalias noundef nonnull align 4 captures(none) dereferenceable(1) [[A]])
+; CHECK-NEXT:    tail call void @use_nocapture(ptr noalias captures(none) [[B]])
+; CHECK-NEXT:    tail call void @use_nocapture(ptr noalias captures(none) [[B]])
 ; CHECK-NEXT:    ret void
 ;
   %A = alloca i8, align 4
@@ -419,10 +419,10 @@ define void @test12_1() {
 define void @test12_2(){
 ; CHECK-LABEL: define {{[^@]+}}@test12_2() {
 ; CHECK-NEXT:    [[A:%.*]] = tail call noalias ptr @malloc(i64 noundef 4)
-; CHECK-NEXT:    tail call void @use_nocapture(ptr nocapture [[A]])
-; CHECK-NEXT:    tail call void @use_nocapture(ptr nocapture [[A]])
+; CHECK-NEXT:    tail call void @use_nocapture(ptr captures(none) [[A]])
+; CHECK-NEXT:    tail call void @use_nocapture(ptr captures(none) [[A]])
 ; CHECK-NEXT:    tail call void @use(ptr [[A]])
-; CHECK-NEXT:    tail call void @use_nocapture(ptr nocapture [[A]])
+; CHECK-NEXT:    tail call void @use_nocapture(ptr captures(none) [[A]])
 ; CHECK-NEXT:    ret void
 ;
 ; FIXME: This should be @use_nocapture(ptr noalias [[A]])
@@ -439,7 +439,7 @@ declare void @two_args(ptr nocapture , ptr nocapture)
 define void @test12_3(){
 ; CHECK-LABEL: define {{[^@]+}}@test12_3() {
 ; CHECK-NEXT:    [[A:%.*]] = tail call noalias ptr @malloc(i64 noundef 4)
-; CHECK-NEXT:    tail call void @two_args(ptr nocapture [[A]], ptr nocapture [[A]])
+; CHECK-NEXT:    tail call void @two_args(ptr captures(none) [[A]], ptr captures(none) [[A]])
 ; CHECK-NEXT:    ret void
 ;
   %A = tail call noalias ptr @malloc(i64 4)
@@ -452,10 +452,10 @@ define void @test12_4(){
 ; CHECK-NEXT:    [[A:%.*]] = tail call noalias ptr @malloc(i64 noundef 4)
 ; CHECK-NEXT:    [[B:%.*]] = tail call noalias ptr @malloc(i64 noundef 4)
 ; CHECK-NEXT:    [[A_1:%.*]] = getelementptr i8, ptr [[A]], i64 1
-; CHECK-NEXT:    tail call void @two_args(ptr noalias nocapture [[A]], ptr noalias nocapture [[B]])
-; CHECK-NEXT:    tail call void @two_args(ptr nocapture [[A]], ptr nocapture [[A]])
-; CHECK-NEXT:    tail call void @two_args(ptr nocapture [[A]], ptr nocapture [[A_1]])
-; CHECK-NEXT:    tail call void @two_args(ptr noalias nocapture [[A]], ptr noalias nocapture [[B]])
+; CHECK-NEXT:    tail call void @two_args(ptr noalias captures(none) [[A]], ptr noalias captures(none) [[B]])
+; CHECK-NEXT:    tail call void @two_args(ptr captures(none) [[A]], ptr captures(none) [[A]])
+; CHECK-NEXT:    tail call void @two_args(ptr captures(none) [[A]], ptr captures(none) [[A_1]])
+; CHECK-NEXT:    tail call void @two_args(ptr noalias captures(none) [[A]], ptr noalias captures(none) [[B]])
 ; CHECK-NEXT:    ret void
 ;
   %A = tail call noalias ptr @malloc(i64 4)
@@ -476,8 +476,8 @@ define void @test12_4(){
 ; TEST 13
 define void @use_i8_internal(ptr %a) {
 ; CHECK-LABEL: define {{[^@]+}}@use_i8_internal
-; CHECK-SAME: (ptr nocapture [[A:%.*]]) {
-; CHECK-NEXT:    call void @use_i8(ptr nocapture [[A]])
+; CHECK-SAME: (ptr captures(none) [[A:%.*]]) {
+; CHECK-NEXT:    call void @use_i8(ptr captures(none) [[A]])
 ; CHECK-NEXT:    ret void
 ;
   call void @use_i8(ptr %a)
@@ -487,7 +487,7 @@ define void @use_i8_internal(ptr %a) {
 define void @test13_use_noalias(){
 ; CHECK-LABEL: define {{[^@]+}}@test13_use_noalias() {
 ; CHECK-NEXT:    [[M1:%.*]] = tail call noalias ptr @malloc(i64 noundef 4)
-; CHECK-NEXT:    call void @use_i8_internal(ptr noalias nocapture [[M1]])
+; CHECK-NEXT:    call void @use_i8_internal(ptr noalias captures(none) [[M1]])
 ; CHECK-NEXT:    ret void
 ;
 ; IS__CGSCC_OPM-LABEL: define {{[^@]+}}@test13_use_noalias()
@@ -502,8 +502,8 @@ define void @test13_use_noalias(){
 define void @test13_use_alias(){
 ; CHECK-LABEL: define {{[^@]+}}@test13_use_alias() {
 ; CHECK-NEXT:    [[M1:%.*]] = tail call noalias ptr @malloc(i64 noundef 4)
-; CHECK-NEXT:    call void @use_i8_internal(ptr noalias nocapture [[M1]])
-; CHECK-NEXT:    call void @use_i8_internal(ptr noalias nocapture [[M1]])
+; CHECK-NEXT:    call void @use_i8_internal(ptr noalias captures(none) [[M1]])
+; CHECK-NEXT:    call void @use_i8_internal(ptr noalias captures(none) [[M1]])
 ; CHECK-NEXT:    ret void
 ;
   %m1 = tail call noalias ptr @malloc(i64 4)
@@ -530,7 +530,7 @@ define i32 @i2p(ptr %arg) {
 ; TUNIT-SAME: (ptr nofree readonly [[ARG:%.*]]) #[[ATTR5:[0-9]+]] {
 ; TUNIT-NEXT:    [[C:%.*]] = call i32 @p2i(ptr noalias nofree readnone [[ARG]]) #[[ATTR11:[0-9]+]]
 ; TUNIT-NEXT:    [[I2P:%.*]] = inttoptr i32 [[C]] to ptr
-; TUNIT-NEXT:    [[CALL:%.*]] = call i32 @ret(ptr nocapture nofree noundef readonly align 4 [[I2P]]) #[[ATTR12:[0-9]+]]
+; TUNIT-NEXT:    [[CALL:%.*]] = call i32 @ret(ptr nofree noundef readonly align 4 captures(none) [[I2P]]) #[[ATTR12:[0-9]+]]
 ; TUNIT-NEXT:    ret i32 [[CALL]]
 ;
 ; CGSCC: Function Attrs: mustprogress nofree nosync nounwind willreturn memory(read)
@@ -538,7 +538,7 @@ define i32 @i2p(ptr %arg) {
 ; CGSCC-SAME: (ptr nofree readonly [[ARG:%.*]]) #[[ATTR6:[0-9]+]] {
 ; CGSCC-NEXT:    [[C:%.*]] = call i32 @p2i(ptr noalias nofree readnone [[ARG]]) #[[ATTR12]]
 ; CGSCC-NEXT:    [[I2P:%.*]] = inttoptr i32 [[C]] to ptr
-; CGSCC-NEXT:    [[CALL:%.*]] = call i32 @ret(ptr nocapture nofree noundef nonnull readonly align 4 dereferenceable(4) [[I2P]]) #[[ATTR13:[0-9]+]]
+; CGSCC-NEXT:    [[CALL:%.*]] = call i32 @ret(ptr nofree noundef nonnull readonly align 4 captures(none) dereferenceable(4) [[I2P]]) #[[ATTR13:[0-9]+]]
 ; CGSCC-NEXT:    ret i32 [[CALL]]
 ;
   %c = call i32 @p2i(ptr %arg)
@@ -549,13 +549,13 @@ define i32 @i2p(ptr %arg) {
 define internal i32 @ret(ptr %arg) {
 ; TUNIT: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: read)
 ; TUNIT-LABEL: define {{[^@]+}}@ret
-; TUNIT-SAME: (ptr nocapture nofree noundef nonnull readonly align 4 dereferenceable(4) [[ARG:%.*]]) #[[ATTR6:[0-9]+]] {
+; TUNIT-SAME: (ptr nofree noundef nonnull readonly align 4 captures(none) dereferenceable(4) [[ARG:%.*]]) #[[ATTR6:[0-9]+]] {
 ; TUNIT-NEXT:    [[L:%.*]] = load i32, ptr [[ARG]], align 4
 ; TUNIT-NEXT:    ret i32 [[L]]
 ;
 ; CGSCC: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: read)
 ; CGSCC-LABEL: define {{[^@]+}}@ret
-; CGSCC-SAME: (ptr nocapture nofree noundef nonnull readonly align 4 dereferenceable(4) [[ARG:%.*]]) #[[ATTR7:[0-9]+]] {
+; CGSCC-SAME: (ptr nofree noundef nonnull readonly align 4 captures(none) dereferenceable(4) [[ARG:%.*]]) #[[ATTR7:[0-9]+]] {
 ; CGSCC-NEXT:    [[L:%.*]] = load i32, ptr [[ARG]], align 4
 ; CGSCC-NEXT:    ret i32 [[L]]
 ;
@@ -577,22 +577,22 @@ define internal fastcc double @strtox(ptr %s, ptr %p, i32 %prec) unnamed_addr {
 ; TUNIT-SAME: (ptr [[S:%.*]]) unnamed_addr {
 ; TUNIT-NEXT:  entry:
 ; TUNIT-NEXT:    [[F:%.*]] = alloca [[STRUCT__IO_FILE:%.*]], align 8
-; TUNIT-NEXT:    call void @llvm.lifetime.start.p0(i64 noundef 144, ptr nocapture nofree noundef nonnull align 8 dereferenceable(240) [[F]]) #[[ATTR13:[0-9]+]]
+; TUNIT-NEXT:    call void @llvm.lifetime.start.p0(i64 noundef 144, ptr nofree noundef nonnull align 8 captures(none) dereferenceable(240) [[F]]) #[[ATTR13:[0-9]+]]
 ; TUNIT-NEXT:    [[CALL:%.*]] = call i32 @sh_fromstring(ptr noundef nonnull align 8 dereferenceable(240) [[F]], ptr [[S]])
 ; TUNIT-NEXT:    call void @__shlim(ptr noundef nonnull align 8 dereferenceable(240) [[F]], i64 noundef 0)
 ; TUNIT-NEXT:    [[CALL1:%.*]] = call double @__floatscan(ptr noundef nonnull align 8 dereferenceable(240) [[F]], i32 noundef 1, i32 noundef 1)
-; TUNIT-NEXT:    call void @llvm.lifetime.end.p0(i64 noundef 144, ptr nocapture nofree noundef nonnull align 8 dereferenceable(240) [[F]])
+; TUNIT-NEXT:    call void @llvm.lifetime.end.p0(i64 noundef 144, ptr nofree noundef nonnull align 8 captures(none) dereferenceable(240) [[F]])
 ; TUNIT-NEXT:    ret double [[CALL1]]
 ;
 ; CGSCC-LABEL: define {{[^@]+}}@strtox
 ; CGSCC-SAME: (ptr [[S:%.*]]) unnamed_addr {
 ; CGSCC-NEXT:  entry:
 ; CGSCC-NEXT:    [[F:%.*]] = alloca [[STRUCT__IO_FILE:%.*]], align 8
-; CGSCC-NEXT:    call void @llvm.lifetime.start.p0(i64 noundef 144, ptr nocapture nofree noundef nonnull align 8 dereferenceable(240) [[F]]) #[[ATTR14:[0-9]+]]
+; CGSCC-NEXT:    call void @llvm.lifetime.start.p0(i64 noundef 144, ptr nofree noundef nonnull align 8 captures(none) dereferenceable(240) [[F]]) #[[ATTR14:[0-9]+]]
 ; CGSCC-NEXT:    [[CALL:%.*]] = call i32 @sh_fromstring(ptr noundef nonnull align 8 dereferenceable(240) [[F]], ptr [[S]])
 ; CGSCC-NEXT:    call void @__shlim(ptr noundef nonnull align 8 dereferenceable(240) [[F]], i64 noundef 0)
 ; CGSCC-NEXT:    [[CALL1:%.*]] = call double @__floatscan(ptr noundef nonnull align 8 dereferenceable(240) [[F]], i32 noundef 1, i32 noundef 1)
-; CGSCC-NEXT:    call void @llvm.lifetime.end.p0(i64 noundef 144, ptr nocapture nofree noundef nonnull align 8 dereferenceable(240) [[F]])
+; CGSCC-NEXT:    call void @llvm.lifetime.end.p0(i64 noundef 144, ptr nofree noundef nonnull align 8 captures(none) dereferenceable(240) [[F]])
 ; CGSCC-NEXT:    ret double [[CALL1]]
 ;
 entry:
@@ -609,7 +609,7 @@ entry:
 ; Function Attrs: nounwind optsize
 define dso_local double @strtod(ptr noalias %s, ptr noalias %p) {
 ; CHECK-LABEL: define {{[^@]+}}@strtod
-; CHECK-SAME: (ptr noalias [[S:%.*]], ptr noalias nocapture nofree readnone [[P:%.*]]) {
+; CHECK-SAME: (ptr noalias [[S:%.*]], ptr noalias nofree readnone captures(none) [[P:%.*]]) {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[CALL:%.*]] = tail call fastcc double @strtox(ptr [[S]])
 ; CHECK-NEXT:    ret double [[CALL]]
@@ -659,13 +659,13 @@ define void @make_alias(ptr %p) {
 define void @only_store(ptr %p) {
 ; TUNIT: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: write)
 ; TUNIT-LABEL: define {{[^@]+}}@only_store
-; TUNIT-SAME: (ptr nocapture nofree noundef nonnull writeonly align 4 dereferenceable(4) [[P:%.*]]) #[[ATTR9:[0-9]+]] {
+; TUNIT-SAME: (ptr nofree noundef nonnull writeonly align 4 captures(none) dereferenceable(4) [[P:%.*]]) #[[ATTR9:[0-9]+]] {
 ; TUNIT-NEXT:    store i32 0, ptr [[P]], align 4
 ; TUNIT-NEXT:    ret void
 ;
 ; CGSCC: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: write)
 ; CGSCC-LABEL: define {{[^@]+}}@only_store
-; CGSCC-SAME: (ptr nocapture nofree noundef nonnull writeonly align 4 dereferenceable(4) [[P:%.*]]) #[[ATTR10:[0-9]+]] {
+; CGSCC-SAME: (ptr nofree noundef nonnull writeonly align 4 captures(none) dereferenceable(4) [[P:%.*]]) #[[ATTR10:[0-9]+]] {
 ; CGSCC-NEXT:    store i32 0, ptr [[P]], align 4
 ; CGSCC-NEXT:    ret void
 ;
@@ -680,7 +680,7 @@ define void @test15_caller(ptr noalias %p, i32 %c) {
 ; TUNIT-NEXT:    [[TOBOOL:%.*]] = icmp eq i32 [[C]], 0
 ; TUNIT-NEXT:    br i1 [[TOBOOL]], label [[IF_END:%.*]], label [[IF_THEN:%.*]]
 ; TUNIT:       if.then:
-; TUNIT-NEXT:    tail call void @only_store(ptr noalias nocapture nofree noundef writeonly align 4 [[P]]) #[[ATTR14:[0-9]+]]
+; TUNIT-NEXT:    tail call void @only_store(ptr noalias nofree noundef writeonly align 4 captures(none) [[P]]) #[[ATTR14:[0-9]+]]
 ; TUNIT-NEXT:    br label [[IF_END]]
 ; TUNIT:       if.end:
 ; TUNIT-NEXT:    tail call void @make_alias(ptr nofree writeonly [[P]]) #[[ATTR14]]
@@ -692,7 +692,7 @@ define void @test15_caller(ptr noalias %p, i32 %c) {
 ; CGSCC-NEXT:    [[TOBOOL:%.*]] = icmp eq i32 [[C]], 0
 ; CGSCC-NEXT:    br i1 [[TOBOOL]], label [[IF_END:%.*]], label [[IF_THEN:%.*]]
 ; CGSCC:       if.then:
-; CGSCC-NEXT:    tail call void @only_store(ptr nocapture nofree noundef nonnull writeonly align 4 dereferenceable(4) [[P]]) #[[ATTR15:[0-9]+]]
+; CGSCC-NEXT:    tail call void @only_store(ptr nofree noundef nonnull writeonly align 4 captures(none) dereferenceable(4) [[P]]) #[[ATTR15:[0-9]+]]
 ; CGSCC-NEXT:    br label [[IF_END]]
 ; CGSCC:       if.end:
 ; CGSCC-NEXT:    tail call void @make_alias(ptr nofree writeonly [[P]]) #[[ATTR15]]
@@ -737,14 +737,14 @@ define internal void @test16_sub(ptr noalias %p, i32 %c1, i32 %c2) {
 ; TUNIT-NEXT:    [[TOBOOL:%.*]] = icmp eq i32 [[C1]], 0
 ; TUNIT-NEXT:    br i1 [[TOBOOL]], label [[IF_END:%.*]], label [[IF_THEN:%.*]]
 ; TUNIT:       if.then:
-; TUNIT-NEXT:    tail call void @only_store(ptr nocapture nofree noundef writeonly align 4 [[P]]) #[[ATTR14]]
+; TUNIT-NEXT:    tail call void @only_store(ptr nofree noundef writeonly align 4 captures(none) [[P]]) #[[ATTR14]]
 ; TUNIT-NEXT:    tail call void @make_alias(ptr nofree writeonly align 4 [[P]]) #[[ATTR14]]
 ; TUNIT-NEXT:    br label [[IF_END]]
 ; TUNIT:       if.end:
 ; TUNIT-NEXT:    [[TOBOOL1:%.*]] = icmp eq i32 [[C2]], 0
 ; TUNIT-NEXT:    br i1 [[TOBOOL1]], label [[IF_THEN2:%.*]], label [[IF_END3:%.*]]
 ; TUNIT:       if.then2:
-; TUNIT-NEXT:    tail call void @only_store(ptr nocapture nofree noundef writeonly align 4 [[P]]) #[[ATTR14]]
+; TUNIT-NEXT:    tail call void @only_store(ptr nofree noundef writeonly align 4 captures(none) [[P]]) #[[ATTR14]]
 ; TUNIT-NEXT:    br label [[IF_END3]]
 ; TUNIT:       if.end3:
 ; TUNIT-NEXT:    ret void
@@ -755,14 +755,14 @@ define internal void @test16_sub(ptr noalias %p, i32 %c1, i32 %c2) {
 ; CGSCC-NEXT:    [[TOBOOL:%.*]] = icmp eq i32 [[C1]], 0
 ; CGSCC-NEXT:    br i1 [[TOBOOL]], label [[IF_END:%.*]], label [[IF_THEN:%.*]]
 ; CGSCC:       if.then:
-; CGSCC-NEXT:    tail call void @only_store(ptr nocapture nofree noundef nonnull writeonly align 4 dereferenceable(4) [[P]]) #[[ATTR15]]
+; CGSCC-NEXT:    tail call void @only_store(ptr nofree noundef nonnull writeonly align 4 captures(none) dereferenceable(4) [[P]]) #[[ATTR15]]
 ; CGSCC-NEXT:    tail call void @make_alias(ptr nofree nonnull writeonly align 4 dereferenceable(4) [[P]]) #[[ATTR15]]
 ; CGSCC-NEXT:    br label [[IF_END]]
 ; CGSCC:       if.end:
 ; CGSCC-NEXT:    [[TOBOOL1:%.*]] = icmp eq i32 [[C2]], 0
 ; CGSCC-NEXT:    br i1 [[TOBOOL1]], label [[IF_THEN2:%.*]], label [[IF_END3:%.*]]
 ; CGSCC:       if.then2:
-; CGSCC-NEXT:    tail call void @only_store(ptr nocapture nofree noundef nonnull writeonly align 4 dereferenceable(4) [[P]]) #[[ATTR15]]
+; CGSCC-NEXT:    tail call void @only_store(ptr nofree noundef nonnull writeonly align 4 captures(none) dereferenceable(4) [[P]]) #[[ATTR15]]
 ; CGSCC-NEXT:    br label [[IF_END3]]
 ; CGSCC:       if.end3:
 ; CGSCC-NEXT:    ret void
@@ -834,7 +834,7 @@ define void @test17_caller(ptr noalias %p, i32 %c) {
 ; TUNIT-NEXT:    tail call void @make_alias(ptr nofree writeonly [[P]]) #[[ATTR14]]
 ; TUNIT-NEXT:    br label [[L3:%.*]]
 ; TUNIT:       l2:
-; TUNIT-NEXT:    tail call void @only_store(ptr noalias nocapture nofree noundef writeonly align 4 [[P]]) #[[ATTR14]]
+; TUNIT-NEXT:    tail call void @only_store(ptr noalias nofree noundef writeonly align 4 captures(none) [[P]]) #[[ATTR14]]
 ; TUNIT-NEXT:    br label [[L3]]
 ; TUNIT:       l3:
 ; TUNIT-NEXT:    ret void
@@ -849,7 +849,7 @@ define void @test17_caller(ptr noalias %p, i32 %c) {
 ; CGSCC-NEXT:    tail call void @make_alias(ptr nofree writeonly [[P]]) #[[ATTR15]]
 ; CGSCC-NEXT:    br label [[L3:%.*]]
 ; CGSCC:       l2:
-; CGSCC-NEXT:    tail call void @only_store(ptr nocapture nofree noundef nonnull writeonly align 4 dereferenceable(4) [[P]]) #[[ATTR15]]
+; CGSCC-NEXT:    tail call void @only_store(ptr nofree noundef nonnull writeonly align 4 captures(none) dereferenceable(4) [[P]]) #[[ATTR15]]
 ; CGSCC-NEXT:    br label [[L3]]
 ; CGSCC:       l3:
 ; CGSCC-NEXT:    ret void
@@ -907,7 +907,7 @@ define void @test18_caller(ptr noalias %p, i32 %c) {
 ; TUNIT-NEXT:    tail call void @make_alias(ptr nofree writeonly [[P]]) #[[ATTR14]]
 ; TUNIT-NEXT:    br label [[L2]]
 ; TUNIT:       l2:
-; TUNIT-NEXT:    tail call void @only_store(ptr nocapture nofree noundef writeonly align 4 [[P]]) #[[ATTR14]]
+; TUNIT-NEXT:    tail call void @only_store(ptr nofree noundef writeonly align 4 captures(none) [[P]]) #[[ATTR14]]
 ; TUNIT-NEXT:    ret void
 ;
 ; CGSCC: Function Attrs: mustprogress nofree nosync nounwind willreturn memory(write)
@@ -920,7 +920,7 @@ define void @test18_caller(ptr noalias %p, i32 %c) {
 ; CGSCC-NEXT:    tail call void @make_alias(ptr nofree nonnull writeonly align 4 dereferenceable(4) [[P]]) #[[ATTR15]]
 ; CGSCC-NEXT:    br label [[L2]]
 ; CGSCC:       l2:
-; CGSCC-NEXT:    tail call void @only_store(ptr nocapture nofree noundef nonnull writeonly align 4 dereferenceable(4) [[P]]) #[[ATTR15]]
+; CGSCC-NEXT:    tail call void @only_store(ptr nofree noundef nonnull writeonly align 4 captures(none) dereferenceable(4) [[P]]) #[[ATTR15]]
 ; CGSCC-NEXT:    ret void
 ;
 entry:
diff --git a/llvm/test/Transforms/Attributor/nocapture-1.ll b/llvm/test/Transforms/Attributor/nocapture-1.ll
index de5f31e470edf..2e67b637eebf6 100644
--- a/llvm/test/Transforms/Attributor/nocapture-1.ll
+++ b/llvm/test/Transforms/Attributor/nocapture-1.ll
@@ -75,7 +75,7 @@ l1:
 define i1 @c4b(ptr %q, i32 %bitno) {
 ; CHECK: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(none)
 ; CHECK-LABEL: define {{[^@]+}}@c4b
-; CHECK-SAME: (ptr nocapture nofree readnone [[Q:%.*]], i32 [[BITNO:%.*]]) #[[ATTR0]] {
+; CHECK-SAME: (ptr nofree readnone captures(none) [[Q:%.*]], i32 [[BITNO:%.*]]) #[[ATTR0]] {
 ; CHECK-NEXT:    [[TMP:%.*]] = ptrtoint ptr [[Q]] to i32
 ; CHECK-NEXT:    [[TMP2:%.*]] = lshr i32 [[TMP]], [[BITNO]]
 ; CHECK-NEXT:    [[BIT:%.*]] = trunc i32 [[TMP2]] to i1
@@ -207,7 +207,7 @@ define i1 @c7(ptr %q, i32 %bitno) {
 define i32 @nc1(ptr %q, ptr %p, i1 %b) {
 ; TUNIT: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn
 ; TUNIT-LABEL: define {{[^@]+}}@nc1
-; TUNIT-SAME: (ptr nofree [[Q:%.*]], ptr nocapture nofree [[P:%.*]], i1 [[B:%.*]]) #[[ATTR5:[0-9]+]] {
+; TUNIT-SAME: (ptr nofree [[Q:%.*]], ptr nofree captures(none) [[P:%.*]], i1 [[B:%.*]]) #[[ATTR5:[0-9]+]] {
 ; TUNIT-NEXT:  e:
 ; TUNIT-NEXT:    br label [[L:%.*]]
 ; TUNIT:       l:
@@ -221,7 +221,7 @@ define i32 @nc1(ptr %q, ptr %p, i1 %b) {
 ;
 ; CGSCC: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn
 ; CGSCC-LABEL: define {{[^@]+}}@nc1
-; CGSCC-SAME: (ptr nofree [[Q:%.*]], ptr nocapture nofree [[P:%.*]], i1 [[B:%.*]]) #[[ATTR7:[0-9]+]] {
+; CGSCC-SAME: (ptr nofree [[Q:%.*]], ptr nofree captures(none) [[P:%.*]], i1 [[B:%.*]]) #[[ATTR7:[0-9]+]] {
 ; CGSCC-NEXT:  e:
 ; CGSCC-NEXT:    br label [[L:%.*]]
 ; CGSCC:       l:
@@ -248,7 +248,7 @@ l:
 define i32 @nc1_addrspace(ptr %q, ptr addrspace(1) %p, i1 %b) {
 ; TUNIT: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn
 ; TUNIT-LABEL: define {{[^@]+}}@nc1_addrspace
-; TUNIT-SAME: (ptr nofree [[Q:%.*]], ptr addrspace(1) nocapture nofree [[P:%.*]], i1 [[B:%.*]]) #[[ATTR5]] {
+; TUNIT-SAME: (ptr nofree [[Q:%.*]], ptr addrspace(1) nofree captures(none) [[P:%.*]], i1 [[B:%.*]]) #[[ATTR5]] {
 ; TUNIT-NEXT:  e:
 ; TUNIT-NEXT:    br label [[L:%.*]]
 ; TUNIT:       l:
@@ -263,7 +263,7 @@ define i32 @nc1_addrspace(ptr %q, ptr addrspace(1) %p, i1 %b) {
 ;
 ; CGSCC: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn
 ; CGSCC-LABEL: define {{[^@]+}}@nc1_addrspace
-; CGSCC-SAME: (ptr nofree [[Q:%.*]], ptr addrspace(1) nocapture nofree [[P:%.*]], i1 [[B:%.*]]) #[[ATTR7]] {
+; CGSCC-SAME: (ptr nofree [[Q:%.*]], ptr addrspace(1) nofree captures(none) [[P:%.*]], i1 [[B:%.*]]) #[[ATTR7]] {
 ; CGSCC-NEXT:  e:
 ; CGSCC-NEXT:    br label [[L:%.*]]
 ; CGSCC:       l:
@@ -292,14 +292,14 @@ l:
 define void @nc2(ptr %p, ptr %q) {
 ; TUNIT: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn
 ; TUNIT-LABEL: define {{[^@]+}}@nc2
-; TUNIT-SAME: (ptr nocapture nofree [[P:%.*]], ptr nofree [[Q:%.*]]) #[[ATTR5]] {
-; TUNIT-NEXT:    [[TMP1:%.*]] = call i32 @nc1(ptr nofree [[Q]], ptr nocapture nofree [[P]], i1 noundef false) #[[ATTR19:[0-9]+]]
+; TUNIT-SAME: (ptr nofree captures(none) [[P:%.*]], ptr nofree [[Q:%.*]]) #[[ATTR5]] {
+; TUNIT-NEXT:    [[TMP1:%.*]] = call i32 @nc1(ptr nofree [[Q]], ptr nofree captures(none) [[P]], i1 noundef false) #[[ATTR19:[0-9]+]]
 ; TUNIT-NEXT:    ret void
 ;
 ; CGSCC: Function Attrs: mustprogress nofree nosync nounwind willreturn
 ; CGSCC-LABEL: define {{[^@]+}}@nc2
-; CGSCC-SAME: (ptr nocapture nofree align 4 [[P:%.*]], ptr nofree [[Q:%.*]]) #[[ATTR8:[0-9]+]] {
-; CGSCC-NEXT:    [[TMP1:%.*]] = call i32 @nc1(ptr nofree [[Q]], ptr nocapture nofree align 4 [[P]], i1 noundef false) #[[ATTR22:[0-9]+]]
+; CGSCC-SAME: (ptr nofree align 4 captures(none) [[P:%.*]], ptr nofree [[Q:%.*]]) #[[ATTR8:[0-9]+]] {
+; CGSCC-NEXT:    [[TMP1:%.*]] = call i32 @nc1(ptr nofree [[Q]], ptr nofree align 4 captures(none) [[P]], i1 noundef false) #[[ATTR22:[0-9]+]]
 ; CGSCC-NEXT:    ret void
 ;
   %1 = call i32 @nc1(ptr %q, ptr %p, i1 0)		; <i32> [#uses=0]
@@ -309,7 +309,7 @@ define void @nc2(ptr %p, ptr %q) {
 
 define void @nc3(ptr %p) {
 ; CHECK-LABEL: define {{[^@]+}}@nc3
-; CHECK-SAME: (ptr nocapture nofree noundef nonnull [[P:%.*]]) {
+; CHECK-SAME: (ptr nofree noundef nonnull captures(none) [[P:%.*]]) {
 ; CHECK-NEXT:    call void [[P]]()
 ; CHECK-NEXT:    ret void
 ;
@@ -339,8 +339,8 @@ define void @nc4(ptr %p) {
 
 define void @nc5(ptr %f, ptr %p) {
 ; CHECK-LABEL: define {{[^@]+}}@nc5
-; CHECK-SAME: (ptr nocapture nofree noundef nonnull [[F:%.*]], ptr nocapture [[P:%.*]]) {
-; CHECK-NEXT:    call void [[F]](ptr nocapture [[P]])
+; CHECK-SAME: (ptr nofree noundef nonnull captures(none) [[F:%.*]], ptr captures(none) [[P:%.*]]) {
+; CHECK-NEXT:    call void [[F]](ptr captures(none) [[P]])
 ; CHECK-NEXT:    ret void
 ;
   call void %f(ptr %p) readonly nounwind
@@ -352,15 +352,15 @@ define void @nc5(ptr %f, ptr %p) {
 define void @test1_1(ptr %x1_1, ptr %y1_1, i1 %c) {
 ; TUNIT: Function Attrs: nofree nosync nounwind memory(write)
 ; TUNIT-LABEL: define {{[^@]+}}@test1_1
-; TUNIT-SAME: (ptr nocapture nofree readnone [[X1_1:%.*]], ptr nocapture nofree readnone [[Y1_1:%.*]], i1 [[C:%.*]]) #[[ATTR7:[0-9]+]] {
-; TUNIT-NEXT:    [[TMP1:%.*]] = call ptr @test1_2(ptr noalias nocapture nofree readnone undef, ptr noalias nofree readnone "no-capture-maybe-returned" [[Y1_1]], i1 noundef [[C]]) #[[ATTR7]]
+; TUNIT-SAME: (ptr nofree readnone captures(none) [[X1_1:%.*]], ptr nofree readnone captures(none) [[Y1_1:%.*]], i1 [[C:%.*]]) #[[ATTR7:[0-9]+]] {
+; TUNIT-NEXT:    [[TMP1:%.*]] = call ptr @test1_2(ptr noalias nofree readnone captures(none) undef, ptr noalias nofree readnone "no-capture-maybe-returned" [[Y1_1]], i1 noundef [[C]]) #[[ATTR7]]
 ; TUNIT-NEXT:    store ptr null, ptr @g, align 8
 ; TUNIT-NEXT:    ret void
 ;
 ; CGSCC: Function Attrs: nofree nosync nounwind memory(write)
 ; CGSCC-LABEL: define {{[^@]+}}@test1_1
-; CGSCC-SAME: (ptr nocapture nofree readnone [[X1_1:%.*]], ptr nocapture nofree readnone [[Y1_1:%.*]], i1 [[C:%.*]]) #[[ATTR10:[0-9]+]] {
-; CGSCC-NEXT:    [[TMP1:%.*]] = call ptr @test1_2(ptr noalias nocapture nofree readnone undef, ptr noalias nofree readnone "no-capture-maybe-returned" [[Y1_1]], i1 noundef [[C]]) #[[ATTR10]]
+; CGSCC-SAME: (ptr nofree readnone captures(none) [[X1_1:%.*]], ptr nofree readnone captures(none) [[Y1_1:%.*]], i1 [[C:%.*]]) #[[ATTR10:[0-9]+]] {
+; CGSCC-NEXT:    [[TMP1:%.*]] = call ptr @test1_2(ptr noalias nofree readnone captures(none) undef, ptr noalias nofree readnone "no-capture-maybe-returned" [[Y1_1]], i1 noundef [[C]]) #[[ATTR10]]
 ; CGSCC-NEXT:    store ptr null, ptr @g, align 8
 ; CGSCC-NEXT:    ret void
 ;
@@ -372,10 +372,10 @@ define void @test1_1(ptr %x1_1, ptr %y1_1, i1 %c) {
 define ptr @test1_2(ptr %x1_2, ptr %y1_2, i1 %c) {
 ; TUNIT: Function Attrs: nofree nosync nounwind memory(write)
 ; TUNIT-LABEL: define {{[^@]+}}@test1_2
-; TUNIT-SAME: (ptr nocapture nofree readnone [[X1_2:%.*]], ptr nofree readnone returned "no-capture-maybe-returned" [[Y1_2:%.*]], i1 noundef [[C:%.*]]) #[[ATTR7]] {
+; TUNIT-SAME: (ptr nofree readnone captures(none) [[X1_2:%.*]], ptr nofree readnone returned "no-capture-maybe-returned" [[Y1_2:%.*]], i1 noundef [[C:%.*]]) #[[ATTR7]] {
 ; TUNIT-NEXT:    br i1 [[C]], label [[T:%.*]], label [[F:%.*]]
 ; TUNIT:       t:
-; TUNIT-NEXT:    call void @test1_1(ptr noalias nocapture nofree readnone undef, ptr noalias nocapture nofree readnone [[Y1_2]], i1 noundef [[C]]) #[[ATTR7]]
+; TUNIT-NEXT:    call void @test1_1(ptr noalias nofree readnone captures(none) undef, ptr noalias nofree readnone captures(none) [[Y1_2]], i1 noundef [[C]]) #[[ATTR7]]
 ; TUNIT-NEXT:    store ptr null, ptr @g, align 8
 ; TUNIT-NEXT:    br label [[F]]
 ; TUNIT:       f:
@@ -383,10 +383,10 @@ define ptr @test1_2(ptr %x1_2, ptr %y1_2, i1 %c) {
 ;
 ; CGSCC: Function Attrs: nofree nosync nounwind memory(write)
 ; CGSCC-LABEL: define {{[^@]+}}@test1_2
-; CGSCC-SAME: (ptr nocapture nofree readnone [[X1_2:%.*]], ptr nofree readnone returned "no-capture-maybe-returned" [[Y1_2:%.*]], i1 noundef [[C:%.*]]) #[[ATTR10]] {
+; CGSCC-SAME: (ptr nofree readnone captures(none) [[X1_2:%.*]], ptr nofree readnone returned "no-capture-maybe-returned" [[Y1_2:%.*]], i1 noundef [[C:%.*]]) #[[ATTR10]] {
 ; CGSCC-NEXT:    br i1 [[C]], label [[T:%.*]], label [[F:%.*]]
 ; CGSCC:       t:
-; CGSCC-NEXT:    call void @test1_1(ptr noalias nocapture nofree readnone undef, ptr noalias nocapture nofree readnone [[Y1_2]], i1 noundef [[C]]) #[[ATTR10]]
+; CGSCC-NEXT:    call void @test1_1(ptr noalias nofree readnone captures(none) undef, ptr noalias nofree readnone captures(none) [[Y1_2]], i1 noundef [[C]]) #[[ATTR10]]
 ; CGSCC-NEXT:    store ptr null, ptr @g, align 8
 ; CGSCC-NEXT:    br label [[F]]
 ; CGSCC:       f:
@@ -404,15 +404,15 @@ f:
 define void @test2(ptr %x2) {
 ; TUNIT: Function Attrs: nofree nosync nounwind memory(write)
 ; TUNIT-LABEL: define {{[^@]+}}@test2
-; TUNIT-SAME: (ptr nocapture nofree readnone [[X2:%.*]]) #[[ATTR7]] {
-; TUNIT-NEXT:    call void @test2(ptr noalias nocapture nofree readnone undef) #[[ATTR7]]
+; TUNIT-SAME: (ptr nofree readnone captures(none) [[X2:%.*]]) #[[ATTR7]] {
+; TUNIT-NEXT:    call void @test2(ptr noalias nofree readnone captures(none) undef) #[[ATTR7]]
 ; TUNIT-NEXT:    store ptr null, ptr @g, align 8
 ; TUNIT-NEXT:    ret void
 ;
 ; CGSCC: Function Attrs: nofree nosync nounwind memory(write)
 ; CGSCC-LABEL: define {{[^@]+}}@test2
-; CGSCC-SAME: (ptr nocapture nofree readnone [[X2:%.*]]) #[[ATTR10]] {
-; CGSCC-NEXT:    call void @test2(ptr noalias nocapture nofree readnone undef) #[[ATTR10]]
+; CGSCC-SAME: (ptr nofree readnone captures(none) [[X2:%.*]]) #[[ATTR10]] {
+; CGSCC-NEXT:    call void @test2(ptr noalias nofree readnone captures(none) undef) #[[ATTR10]]
 ; CGSCC-NEXT:    store ptr null, ptr @g, align 8
 ; CGSCC-NEXT:    ret void
 ;
@@ -424,15 +424,15 @@ define void @test2(ptr %x2) {
 define void @test3(ptr %x3, ptr %y3, ptr %z3) {
 ; TUNIT: Function Attrs: nofree nosync nounwind memory(write)
 ; TUNIT-LABEL: define {{[^@]+}}@test3
-; TUNIT-SAME: (ptr nocapture nofree readnone [[X3:%.*]], ptr nocapture nofree readnone [[Y3:%.*]], ptr nocapture nofree readnone [[Z3:%.*]]) #[[ATTR7]] {
-; TUNIT-NEXT:    call void @test3(ptr noalias nocapture nofree readnone undef, ptr noalias nocapture nofree readnone undef, ptr noalias nocapture nofree readnone undef) #[[ATTR7]]
+; TUNIT-SAME: (ptr nofree readnone captures(none) [[X3:%.*]], ptr nofree readnone captures(none) [[Y3:%.*]], ptr nofree readnone captures(none) [[Z3:%.*]]) #[[ATTR7]] {
+; TUNIT-NEXT:    call void @test3(ptr noalias nofree readnone captures(none) undef, ptr noalias nofree readnone captures(none) undef, ptr noalias nofree readnone captures(none) undef) #[[ATTR7]]
 ; TUNIT-NEXT:    store ptr null, ptr @g, align 8
 ; TUNIT-NEXT:    ret void
 ;
 ; CGSCC: Function Attrs: nofree nosync nounwind memory(write)
 ; CGSCC-LABEL: define {{[^@]+}}@test3
-; CGSCC-SAME: (ptr nocapture nofree readnone [[X3:%.*]], ptr nocapture nofree readnone [[Y3:%.*]], ptr nocapture nofree readnone [[Z3:%.*]]) #[[ATTR10]] {
-; CGSCC-NEXT:    call void @test3(ptr noalias nocapture nofree readnone undef, ptr noalias nocapture nofree readnone undef, ptr noalias nocapture nofree readnone undef) #[[ATTR10]]
+; CGSCC-SAME: (ptr nofree readnone captures(none) [[X3:%.*]], ptr nofree readnone captures(none) [[Y3:%.*]], ptr nofree readnone captures(none) [[Z3:%.*]]) #[[ATTR10]] {
+; CGSCC-NEXT:    call void @test3(ptr noalias nofree readnone captures(none) undef, ptr noalias nofree readnone captures(none) undef, ptr noalias nofree readnone captures(none) undef) #[[ATTR10]]
 ; CGSCC-NEXT:    store ptr null, ptr @g, align 8
 ; CGSCC-NEXT:    ret void
 ;
@@ -444,15 +444,15 @@ define void @test3(ptr %x3, ptr %y3, ptr %z3) {
 define void @test4_1(ptr %x4_1, i1 %c) {
 ; TUNIT: Function Attrs: nofree nosync nounwind memory(write)
 ; TUNIT-LABEL: define {{[^@]+}}@test4_1
-; TUNIT-SAME: (ptr nocapture nofree readnone [[X4_1:%.*]], i1 [[C:%.*]]) #[[ATTR7]] {
-; TUNIT-NEXT:    [[TMP1:%.*]] = call ptr @test4_2(ptr noalias nocapture nofree readnone undef, ptr noalias nofree readnone "no-capture-maybe-returned" [[X4_1]], ptr noalias nocapture nofree readnone undef, i1 noundef [[C]]) #[[ATTR7]]
+; TUNIT-SAME: (ptr nofree readnone captures(none) [[X4_1:%.*]], i1 [[C:%.*]]) #[[ATTR7]] {
+; TUNIT-NEXT:    [[TMP1:%.*]] = call ptr @test4_2(ptr noalias nofree readnone captures(none) undef, ptr noalias nofree readnone "no-capture-maybe-returned" [[X4_1]], ptr noalias nofree readnone captures(none) undef, i1 noundef [[C]]) #[[ATTR7]]
 ; TUNIT-NEXT:    store ptr null, ptr @g, align 8
 ; TUNIT-NEXT:    ret void
 ;
 ; CGSCC: Function Attrs: nofree nosync nounwind memory(write)
 ; CGSCC-LABEL: define {{[^@]+}}@test4_1
-; CGSCC-SAME: (ptr nocapture nofree readnone [[X4_1:%.*]], i1 [[C:%.*]]) #[[ATTR10]] {
-; CGSCC-NEXT:    [[TMP1:%.*]] = call ptr @test4_2(ptr noalias nocapture nofree readnone undef, ptr noalias nofree readnone "no-capture-maybe-returned" [[X4_1]], ptr noalias nocapture nofree readnone undef, i1 noundef [[C]]) #[[ATTR10]]
+; CGSCC-SAME: (ptr nofree readnone captures(none) [[X4_1:%.*]], i1 [[C:%.*]]) #[[ATTR10]] {
+; CGSCC-NEXT:    [[TMP1:%.*]] = call ptr @test4_2(ptr noalias nofree readnone captures(none) undef, ptr noalias nofree readnone "no-capture-maybe-returned" [[X4_1]], ptr noalias nofree readnone captures(none) undef, i1 noundef [[C]]) #[[ATTR10]]
 ; CGSCC-NEXT:    store ptr null, ptr @g, align 8
 ; CGSCC-NEXT:    ret void
 ;
@@ -464,7 +464,7 @@ define void @test4_1(ptr %x4_1, i1 %c) {
 define ptr @test4_2(ptr %x4_2, ptr %y4_2, ptr %z4_2, i1 %c) {
 ; TUNIT: Function Attrs: nofree nosync nounwind memory(write)
 ; TUNIT-LABEL: define {{[^@]+}}@test4_2
-; TUNIT-SAME: (ptr nocapture nofree readnone [[X4_2:%.*]], ptr nofree readnone returned "no-capture-maybe-returned" [[Y4_2:%.*]], ptr nocapture nofree readnone [[Z4_2:%.*]], i1 noundef [[C:%.*]]) #[[ATTR7]] {
+; TUNIT-SAME: (ptr nofree readnone captures(none) [[X4_2:%.*]], ptr nofree readnone returned "no-capture-maybe-returned" [[Y4_2:%.*]], ptr nofree readnone captures(none) [[Z4_2:%.*]], i1 noundef [[C:%.*]]) #[[ATTR7]] {
 ; TUNIT-NEXT:    br i1 [[C]], label [[T:%.*]], label [[F:%.*]]
 ; TUNIT:       t:
 ; TUNIT-NEXT:    call void @test4_1(ptr nofree noundef readnone align 4294967296 null, i1 noundef [[C]]) #[[ATTR7]]
@@ -475,7 +475,7 @@ define ptr @test4_2(ptr %x4_2, ptr %y4_2, ptr %z4_2, i1 %c) {
 ;
 ; CGSCC: Function Attrs: nofree nosync nounwind memory(write)
 ; CGSCC-LABEL: define {{[^@]+}}@test4_2
-; CGSCC-SAME: (ptr nocapture nofree readnone [[X4_2:%.*]], ptr nofree readnone returned "no-capture-maybe-returned" [[Y4_2:%.*]], ptr nocapture nofree readnone [[Z4_2:%.*]], i1 noundef [[C:%.*]]) #[[ATTR10]] {
+; CGSCC-SAME: (ptr nofree readnone captures(none) [[X4_2:%.*]], ptr nofree readnone returned "no-capture-maybe-returned" [[Y4_2:%.*]], ptr nofree readnone captures(none) [[Z4_2:%.*]], i1 noundef [[C:%.*]]) #[[ATTR10]] {
 ; CGSCC-NEXT:    br i1 [[C]], label [[T:%.*]], label [[F:%.*]]
 ; CGSCC:       t:
 ; CGSCC-NEXT:    call void @test4_1(ptr nofree noundef readnone align 4294967296 null, i1 noundef [[C]]) #[[ATTR10]]
@@ -511,8 +511,8 @@ declare void @test6_1(ptr %x6_1, ptr nocapture %y6_1, ...)
 
 define void @test6_2(ptr %x6_2, ptr %y6_2, ptr %z6_2) {
 ; CHECK-LABEL: define {{[^@]+}}@test6_2
-; CHECK-SAME: (ptr [[X6_2:%.*]], ptr nocapture [[Y6_2:%.*]], ptr [[Z6_2:%.*]]) {
-; CHECK-NEXT:    call void (ptr, ptr, ...) @test6_1(ptr [[X6_2]], ptr nocapture [[Y6_2]], ptr [[Z6_2]])
+; CHECK-SAME: (ptr [[X6_2:%.*]], ptr captures(none) [[Y6_2:%.*]], ptr [[Z6_2:%.*]]) {
+; CHECK-NEXT:    call void (ptr, ptr, ...) @test6_1(ptr [[X6_2]], ptr captures(none) [[Y6_2]], ptr [[Z6_2]])
 ; CHECK-NEXT:    store ptr null, ptr @g, align 8
 ; CHECK-NEXT:    ret void
 ;
@@ -524,13 +524,13 @@ define void @test6_2(ptr %x6_2, ptr %y6_2, ptr %z6_2) {
 define void @test_cmpxchg(ptr %p) {
 ; TUNIT: Function Attrs: mustprogress nofree norecurse nounwind willreturn memory(argmem: readwrite)
 ; TUNIT-LABEL: define {{[^@]+}}@test_cmpxchg
-; TUNIT-SAME: (ptr nocapture nofree noundef nonnull align 4 dereferenceable(4) [[P:%.*]]) #[[ATTR8:[0-9]+]] {
+; TUNIT-SAME: (ptr nofree noundef nonnull align 4 captures(none) dereferenceable(4) [[P:%.*]]) #[[ATTR8:[0-9]+]] {
 ; TUNIT-NEXT:    [[TMP1:%.*]] = cmpxchg ptr [[P]], i32 0, i32 1 acquire monotonic, align 4
 ; TUNIT-NEXT:    ret void
 ;
 ; CGSCC: Function Attrs: mustprogress nofree norecurse nounwind willreturn memory(argmem: readwrite)
 ; CGSCC-LABEL: define {{[^@]+}}@test_cmpxchg
-; CGSCC-SAME: (ptr nocapture nofree noundef nonnull align 4 dereferenceable(4) [[P:%.*]]) #[[ATTR11:[0-9]+]] {
+; CGSCC-SAME: (ptr nofree noundef nonnull align 4 captures(none) dereferenceable(4) [[P:%.*]]) #[[ATTR11:[0-9]+]] {
 ; CGSCC-NEXT:    [[TMP1:%.*]] = cmpxchg ptr [[P]], i32 0, i32 1 acquire monotonic, align 4
 ; CGSCC-NEXT:    ret void
 ;
@@ -541,13 +541,13 @@ define void @test_cmpxchg(ptr %p) {
 define void @test_cmpxchg_ptr(ptr %p, ptr %q) {
 ; TUNIT: Function Attrs: mustprogress nofree norecurse nounwind willreturn memory(argmem: readwrite)
 ; TUNIT-LABEL: define {{[^@]+}}@test_cmpxchg_ptr
-; TUNIT-SAME: (ptr nocapture nofree noundef nonnull align 8 dereferenceable(8) [[P:%.*]], ptr nofree [[Q:%.*]]) #[[ATTR8]] {
+; TUNIT-SAME: (ptr nofree noundef nonnull align 8 captures(none) dereferenceable(8) [[P:%.*]], ptr nofree [[Q:%.*]]) #[[ATTR8]] {
 ; TUNIT-NEXT:    [[TMP1:%.*]] = cmpxchg ptr [[P]], ptr null, ptr [[Q]] acquire monotonic, align 8
 ; TUNIT-NEXT:    ret void
 ;
 ; CGSCC: Function Attrs: mustprogress nofree norecurse nounwind willreturn memory(argmem: readwrite)
 ; CGSCC-LABEL: define {{[^@]+}}@test_cmpxchg_ptr
-; CGSCC-SAME: (ptr nocapture nofree noundef nonnull align 8 dereferenceable(8) [[P:%.*]], ptr nofree [[Q:%.*]]) #[[ATTR11]] {
+; CGSCC-SAME: (ptr nofree noundef nonnull align 8 captures(none) dereferenceable(8) [[P:%.*]], ptr nofree [[Q:%.*]]) #[[ATTR11]] {
 ; CGSCC-NEXT:    [[TMP1:%.*]] = cmpxchg ptr [[P]], ptr null, ptr [[Q]] acquire monotonic, align 8
 ; CGSCC-NEXT:    ret void
 ;
@@ -558,13 +558,13 @@ define void @test_cmpxchg_ptr(ptr %p, ptr %q) {
 define void @test_atomicrmw(ptr %p) {
 ; TUNIT: Function Attrs: mustprogress nofree norecurse nounwind willreturn memory(argmem: readwrite)
 ; TUNIT-LABEL: define {{[^@]+}}@test_atomicrmw
-; TUNIT-SAME: (ptr nocapture nofree noundef nonnull align 4 dereferenceable(4) [[P:%.*]]) #[[ATTR8]] {
+; TUNIT-SAME: (ptr nofree noundef nonnull align 4 captures(none) dereferenceable(4) [[P:%.*]]) #[[ATTR8]] {
 ; TUNIT-NEXT:    [[TMP1:%.*]] = atomicrmw add ptr [[P]], i32 1 seq_cst, align 4
 ; TUNIT-NEXT:    ret void
 ;
 ; CGSCC: Function Attrs: mustprogress nofree norecurse nounwind willreturn memory(argmem: readwrite)
 ; CGSCC-LABEL: define {{[^@]+}}@test_atomicrmw
-; CGSCC-SAME: (ptr nocapture nofree noundef nonnull align 4 dereferenceable(4) [[P:%.*]]) #[[ATTR11]] {
+; CGSCC-SAME: (ptr nofree noundef nonnull align 4 captures(none) dereferenceable(4) [[P:%.*]]) #[[ATTR11]] {
 ; CGSCC-NEXT:    [[TMP1:%.*]] = atomicrmw add ptr [[P]], i32 1 seq_cst, align 4
 ; CGSCC-NEXT:    ret void
 ;
@@ -598,7 +598,7 @@ entry:
 define void @nocaptureLaunder(ptr %p) {
 ; TUNIT: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: readwrite, inaccessiblemem: readwrite)
 ; TUNIT-LABEL: define {{[^@]+}}@nocaptureLaunder
-; TUNIT-SAME: (ptr nocapture nofree [[P:%.*]]) #[[ATTR9:[0-9]+]] {
+; TUNIT-SAME: (ptr nofree captures(none) [[P:%.*]]) #[[ATTR9:[0-9]+]] {
 ; TUNIT-NEXT:  entry:
 ; TUNIT-NEXT:    [[B:%.*]] = call ptr @llvm.launder.invariant.group.p0(ptr nofree [[P]]) #[[ATTR21:[0-9]+]]
 ; TUNIT-NEXT:    store i8 42, ptr [[B]], align 1
@@ -606,7 +606,7 @@ define void @nocaptureLaunder(ptr %p) {
 ;
 ; CGSCC: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: readwrite, inaccessiblemem: readwrite)
 ; CGSCC-LABEL: define {{[^@]+}}@nocaptureLaunder
-; CGSCC-SAME: (ptr nocapture nofree [[P:%.*]]) #[[ATTR12:[0-9]+]] {
+; CGSCC-SAME: (ptr nofree captures(none) [[P:%.*]]) #[[ATTR12:[0-9]+]] {
 ; CGSCC-NEXT:  entry:
 ; CGSCC-NEXT:    [[B:%.*]] = call ptr @llvm.launder.invariant.group.p0(ptr nofree [[P]]) #[[ATTR24:[0-9]+]]
 ; CGSCC-NEXT:    store i8 42, ptr [[B]], align 1
@@ -642,7 +642,7 @@ define void @captureLaunder(ptr %p) {
 define void @nocaptureStrip(ptr %p) {
 ; TUNIT: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: write)
 ; TUNIT-LABEL: define {{[^@]+}}@nocaptureStrip
-; TUNIT-SAME: (ptr nocapture nofree writeonly [[P:%.*]]) #[[ATTR10:[0-9]+]] {
+; TUNIT-SAME: (ptr nofree writeonly captures(none) [[P:%.*]]) #[[ATTR10:[0-9]+]] {
 ; TUNIT-NEXT:  entry:
 ; TUNIT-NEXT:    [[B:%.*]] = call ptr @llvm.strip.invariant.group.p0(ptr noalias nofree readnone [[P]]) #[[ATTR22:[0-9]+]]
 ; TUNIT-NEXT:    store i8 42, ptr [[B]], align 1
@@ -650,7 +650,7 @@ define void @nocaptureStrip(ptr %p) {
 ;
 ; CGSCC: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: write)
 ; CGSCC-LABEL: define {{[^@]+}}@nocaptureStrip
-; CGSCC-SAME: (ptr nocapture nofree writeonly [[P:%.*]]) #[[ATTR13:[0-9]+]] {
+; CGSCC-SAME: (ptr nofree writeonly captures(none) [[P:%.*]]) #[[ATTR13:[0-9]+]] {
 ; CGSCC-NEXT:  entry:
 ; CGSCC-NEXT:    [[B:%.*]] = call ptr @llvm.strip.invariant.group.p0(ptr noalias nofree readnone [[P]]) #[[ATTR21]]
 ; CGSCC-NEXT:    store i8 42, ptr [[B]], align 1
@@ -708,7 +708,7 @@ define i1 @captureICmpRev(ptr %x) {
 define i1 @nocaptureInboundsGEPICmp(ptr %x) {
 ; CHECK: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(none)
 ; CHECK-LABEL: define {{[^@]+}}@nocaptureInboundsGEPICmp
-; CHECK-SAME: (ptr nocapture nofree readnone [[X:%.*]]) #[[ATTR0]] {
+; CHECK-SAME: (ptr nofree readnone captures(none) [[X:%.*]]) #[[ATTR0]] {
 ; CHECK-NEXT:    ret i1 false
 ;
   %1 = getelementptr inbounds i32, ptr %x, i32 5
@@ -719,7 +719,7 @@ define i1 @nocaptureInboundsGEPICmp(ptr %x) {
 define i1 @nocaptureInboundsGEPICmpRev(ptr %x) {
 ; CHECK: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(none)
 ; CHECK-LABEL: define {{[^@]+}}@nocaptureInboundsGEPICmpRev
-; CHECK-SAME: (ptr nocapture nofree readnone [[X:%.*]]) #[[ATTR0]] {
+; CHECK-SAME: (ptr nofree readnone captures(none) [[X:%.*]]) #[[ATTR0]] {
 ; CHECK-NEXT:    ret i1 true
 ;
   %1 = getelementptr inbounds i32, ptr %x, i32 5
@@ -730,7 +730,7 @@ define i1 @nocaptureInboundsGEPICmpRev(ptr %x) {
 define i1 @nocaptureDereferenceableOrNullICmp(ptr dereferenceable_or_null(4) %x) {
 ; CHECK: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(none)
 ; CHECK-LABEL: define {{[^@]+}}@nocaptureDereferenceableOrNullICmp
-; CHECK-SAME: (ptr nocapture nofree noundef readnone dereferenceable_or_null(4) [[X:%.*]]) #[[ATTR0]] {
+; CHECK-SAME: (ptr nofree noundef readnone captures(none) dereferenceable_or_null(4) [[X:%.*]]) #[[ATTR0]] {
 ; CHECK-NEXT:    [[TMP1:%.*]] = icmp eq ptr [[X]], null
 ; CHECK-NEXT:    ret i1 [[TMP1]]
 ;
@@ -772,14 +772,14 @@ declare ptr @unknownpi8pi8(ptr,ptr returned)
 define ptr @test_returned1(ptr %A, ptr returned %B) nounwind readonly {
 ; TUNIT: Function Attrs: nosync nounwind memory(read)
 ; TUNIT-LABEL: define {{[^@]+}}@test_returned1
-; TUNIT-SAME: (ptr nocapture [[A:%.*]], ptr returned [[B:%.*]]) #[[ATTR12:[0-9]+]] {
+; TUNIT-SAME: (ptr captures(none) [[A:%.*]], ptr returned [[B:%.*]]) #[[ATTR12:[0-9]+]] {
 ; TUNIT-NEXT:  entry:
 ; TUNIT-NEXT:    [[P:%.*]] = call ptr @unknownpi8pi8(ptr [[A]], ptr [[B]])
 ; TUNIT-NEXT:    ret ptr [[P]]
 ;
 ; CGSCC: Function Attrs: nosync nounwind memory(read)
 ; CGSCC-LABEL: define {{[^@]+}}@test_returned1
-; CGSCC-SAME: (ptr nocapture [[A:%.*]], ptr returned [[B:%.*]]) #[[ATTR15:[0-9]+]] {
+; CGSCC-SAME: (ptr captures(none) [[A:%.*]], ptr returned [[B:%.*]]) #[[ATTR15:[0-9]+]] {
 ; CGSCC-NEXT:  entry:
 ; CGSCC-NEXT:    [[P:%.*]] = call ptr @unknownpi8pi8(ptr [[A]], ptr [[B]])
 ; CGSCC-NEXT:    ret ptr [[P]]
@@ -817,13 +817,13 @@ declare void @val_use(i8 %ptr) readonly nounwind willreturn
 define void @ptr_uses(ptr %ptr, ptr %wptr) {
 ; TUNIT: Function Attrs: mustprogress nosync nounwind willreturn
 ; TUNIT-LABEL: define {{[^@]+}}@ptr_uses
-; TUNIT-SAME: (ptr nofree [[PTR:%.*]], ptr nocapture nofree noundef nonnull writeonly dereferenceable(1) [[WPTR:%.*]]) #[[ATTR14:[0-9]+]] {
+; TUNIT-SAME: (ptr nofree [[PTR:%.*]], ptr nofree noundef nonnull writeonly captures(none) dereferenceable(1) [[WPTR:%.*]]) #[[ATTR14:[0-9]+]] {
 ; TUNIT-NEXT:    store i8 0, ptr [[WPTR]], align 1
 ; TUNIT-NEXT:    ret void
 ;
 ; CGSCC: Function Attrs: mustprogress nosync nounwind willreturn
 ; CGSCC-LABEL: define {{[^@]+}}@ptr_uses
-; CGSCC-SAME: (ptr nofree [[PTR:%.*]], ptr nocapture nofree noundef nonnull writeonly dereferenceable(1) [[WPTR:%.*]]) #[[ATTR17:[0-9]+]] {
+; CGSCC-SAME: (ptr nofree [[PTR:%.*]], ptr nofree noundef nonnull writeonly captures(none) dereferenceable(1) [[WPTR:%.*]]) #[[ATTR17:[0-9]+]] {
 ; CGSCC-NEXT:    store i8 0, ptr [[WPTR]], align 1
 ; CGSCC-NEXT:    ret void
 ;
diff --git a/llvm/test/Transforms/Attributor/nocapture-2.ll b/llvm/test/Transforms/Attributor/nocapture-2.ll
index 1e5e9da19909f..475121f6bd357 100644
--- a/llvm/test/Transforms/Attributor/nocapture-2.ll
+++ b/llvm/test/Transforms/Attributor/nocapture-2.ll
@@ -99,7 +99,7 @@ return:                                           ; preds = %if.end3, %if.then2,
 define ptr @srec0(ptr %a) #0 {
 ; CHECK: Function Attrs: mustprogress nofree noinline nosync nounwind willreturn memory(none) uwtable
 ; CHECK-LABEL: define noalias noundef align 4294967296 ptr @srec0
-; CHECK-SAME: (ptr nocapture nofree readnone [[A:%.*]]) #[[ATTR1:[0-9]+]] {
+; CHECK-SAME: (ptr nofree readnone captures(none) [[A:%.*]]) #[[ATTR1:[0-9]+]] {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    ret ptr null
 ;
@@ -124,7 +124,7 @@ entry:
 define ptr @srec16(ptr %a) #0 {
 ; CHECK: Function Attrs: mustprogress nofree noinline nosync nounwind willreturn memory(none) uwtable
 ; CHECK-LABEL: define noalias nonnull align 4294967296 dereferenceable(4294967295) ptr @srec16
-; CHECK-SAME: (ptr nocapture nofree readnone [[A:%.*]]) #[[ATTR1]] {
+; CHECK-SAME: (ptr nofree readnone captures(none) [[A:%.*]]) #[[ATTR1]] {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    ret ptr undef
 ;
@@ -285,9 +285,9 @@ declare void @external_no_capture(ptr nocapture)
 define void @test_external_no_capture(ptr %p) #0 {
 ; CHECK: Function Attrs: noinline nounwind uwtable
 ; CHECK-LABEL: define void @test_external_no_capture
-; CHECK-SAME: (ptr nocapture [[P:%.*]]) #[[ATTR3:[0-9]+]] {
+; CHECK-SAME: (ptr captures(none) [[P:%.*]]) #[[ATTR3:[0-9]+]] {
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    call void @external_no_capture(ptr nocapture [[P]])
+; CHECK-NEXT:    call void @external_no_capture(ptr captures(none) [[P]])
 ; CHECK-NEXT:    ret void
 ;
 entry:
@@ -304,9 +304,9 @@ entry:
 define void @test_var_arg_call(ptr %p, i32 %a) #0 {
 ; CHECK: Function Attrs: noinline nounwind uwtable
 ; CHECK-LABEL: define void @test_var_arg_call
-; CHECK-SAME: (ptr nocapture [[P:%.*]], i32 [[A:%.*]]) #[[ATTR3]] {
+; CHECK-SAME: (ptr captures(none) [[P:%.*]], i32 [[A:%.*]]) #[[ATTR3]] {
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[CALL:%.*]] = call i32 (ptr, ...) @printf(ptr nocapture [[P]], i32 [[A]])
+; CHECK-NEXT:    [[CALL:%.*]] = call i32 (ptr, ...) @printf(ptr captures(none) [[P]], i32 [[A]])
 ; CHECK-NEXT:    ret void
 ;
 entry:
@@ -371,7 +371,7 @@ entry:
 define void @test_not_captured_but_returned_calls(ptr %a) #0 {
 ; TUNIT: Function Attrs: mustprogress nofree noinline norecurse nosync nounwind willreturn memory(argmem: write) uwtable
 ; TUNIT-LABEL: define void @test_not_captured_but_returned_calls
-; TUNIT-SAME: (ptr nocapture nofree writeonly align 8 [[A:%.*]]) #[[ATTR4]] {
+; TUNIT-SAME: (ptr nofree writeonly align 8 captures(none) [[A:%.*]]) #[[ATTR4]] {
 ; TUNIT-NEXT:  entry:
 ; TUNIT-NEXT:    [[CALL:%.*]] = call ptr @not_captured_but_returned_0(ptr nofree noundef writeonly align 8 "no-capture-maybe-returned" [[A]]) #[[ATTR12:[0-9]+]]
 ; TUNIT-NEXT:    [[CALL1:%.*]] = call ptr @not_captured_but_returned_1(ptr nofree writeonly align 8 "no-capture-maybe-returned" [[A]]) #[[ATTR12]]
@@ -587,14 +587,14 @@ declare ptr @readonly_unknown(ptr, ptr) readonly
 define void @not_captured_by_readonly_call(ptr %b) #0 {
 ; TUNIT: Function Attrs: noinline nosync nounwind memory(read) uwtable
 ; TUNIT-LABEL: define void @not_captured_by_readonly_call
-; TUNIT-SAME: (ptr nocapture readonly [[B:%.*]]) #[[ATTR7:[0-9]+]] {
+; TUNIT-SAME: (ptr readonly captures(none) [[B:%.*]]) #[[ATTR7:[0-9]+]] {
 ; TUNIT-NEXT:  entry:
 ; TUNIT-NEXT:    [[CALL:%.*]] = call ptr @readonly_unknown(ptr readonly [[B]], ptr readonly [[B]]) #[[ATTR13:[0-9]+]]
 ; TUNIT-NEXT:    ret void
 ;
 ; CGSCC: Function Attrs: noinline nosync nounwind memory(read) uwtable
 ; CGSCC-LABEL: define void @not_captured_by_readonly_call
-; CGSCC-SAME: (ptr nocapture readonly [[B:%.*]]) #[[ATTR8:[0-9]+]] {
+; CGSCC-SAME: (ptr readonly captures(none) [[B:%.*]]) #[[ATTR8:[0-9]+]] {
 ; CGSCC-NEXT:  entry:
 ; CGSCC-NEXT:    [[CALL:%.*]] = call ptr @readonly_unknown(ptr readonly [[B]], ptr readonly [[B]]) #[[ATTR15:[0-9]+]]
 ; CGSCC-NEXT:    ret void
@@ -612,14 +612,14 @@ entry:
 define ptr @not_captured_by_readonly_call_not_returned_either1(ptr %b, ptr returned %r) {
 ; TUNIT: Function Attrs: nosync nounwind memory(read)
 ; TUNIT-LABEL: define ptr @not_captured_by_readonly_call_not_returned_either1
-; TUNIT-SAME: (ptr nocapture readonly [[B:%.*]], ptr readonly returned [[R:%.*]]) #[[ATTR8:[0-9]+]] {
+; TUNIT-SAME: (ptr readonly captures(none) [[B:%.*]], ptr readonly returned [[R:%.*]]) #[[ATTR8:[0-9]+]] {
 ; TUNIT-NEXT:  entry:
 ; TUNIT-NEXT:    [[CALL:%.*]] = call ptr @readonly_unknown(ptr readonly [[B]], ptr readonly [[R]]) #[[ATTR8]]
 ; TUNIT-NEXT:    ret ptr [[CALL]]
 ;
 ; CGSCC: Function Attrs: nosync nounwind memory(read)
 ; CGSCC-LABEL: define ptr @not_captured_by_readonly_call_not_returned_either1
-; CGSCC-SAME: (ptr nocapture readonly [[B:%.*]], ptr readonly returned [[R:%.*]]) #[[ATTR9:[0-9]+]] {
+; CGSCC-SAME: (ptr readonly captures(none) [[B:%.*]], ptr readonly returned [[R:%.*]]) #[[ATTR9:[0-9]+]] {
 ; CGSCC-NEXT:  entry:
 ; CGSCC-NEXT:    [[CALL:%.*]] = call ptr @readonly_unknown(ptr readonly [[B]], ptr readonly [[R]]) #[[ATTR9]]
 ; CGSCC-NEXT:    ret ptr [[CALL]]
@@ -654,16 +654,16 @@ declare ptr @readonly_unknown_r1b(ptr, ptr returned) readonly nounwind
 define ptr @not_captured_by_readonly_call_not_returned_either3(ptr %b, ptr %r) {
 ; TUNIT: Function Attrs: nosync nounwind memory(read)
 ; TUNIT-LABEL: define ptr @not_captured_by_readonly_call_not_returned_either3
-; TUNIT-SAME: (ptr nocapture readonly [[B:%.*]], ptr readonly [[R:%.*]]) #[[ATTR8]] {
+; TUNIT-SAME: (ptr readonly captures(none) [[B:%.*]], ptr readonly [[R:%.*]]) #[[ATTR8]] {
 ; TUNIT-NEXT:  entry:
-; TUNIT-NEXT:    [[CALL:%.*]] = call ptr @readonly_unknown_r1b(ptr nocapture readonly [[B]], ptr readonly [[R]]) #[[ATTR8]]
+; TUNIT-NEXT:    [[CALL:%.*]] = call ptr @readonly_unknown_r1b(ptr readonly captures(none) [[B]], ptr readonly [[R]]) #[[ATTR8]]
 ; TUNIT-NEXT:    ret ptr [[CALL]]
 ;
 ; CGSCC: Function Attrs: nosync nounwind memory(read)
 ; CGSCC-LABEL: define ptr @not_captured_by_readonly_call_not_returned_either3
-; CGSCC-SAME: (ptr nocapture readonly [[B:%.*]], ptr readonly [[R:%.*]]) #[[ATTR9]] {
+; CGSCC-SAME: (ptr readonly captures(none) [[B:%.*]], ptr readonly [[R:%.*]]) #[[ATTR9]] {
 ; CGSCC-NEXT:  entry:
-; CGSCC-NEXT:    [[CALL:%.*]] = call ptr @readonly_unknown_r1b(ptr nocapture readonly [[B]], ptr readonly [[R]]) #[[ATTR9]]
+; CGSCC-NEXT:    [[CALL:%.*]] = call ptr @readonly_unknown_r1b(ptr readonly captures(none) [[B]], ptr readonly [[R]]) #[[ATTR9]]
 ; CGSCC-NEXT:    ret ptr [[CALL]]
 ;
 entry:
@@ -695,7 +695,7 @@ entry:
 declare ptr @unknown_i32p(ptr)
 define void @nocapture_is_not_subsumed_1(ptr nocapture %b) {
 ; CHECK-LABEL: define void @nocapture_is_not_subsumed_1
-; CHECK-SAME: (ptr nocapture [[B:%.*]]) {
+; CHECK-SAME: (ptr captures(none) [[B:%.*]]) {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[CALL:%.*]] = call ptr @unknown_i32p(ptr [[B]])
 ; CHECK-NEXT:    store i32 0, ptr [[CALL]], align 4
@@ -711,7 +711,7 @@ declare ptr @readonly_i32p(ptr) readonly
 define void @nocapture_is_not_subsumed_2(ptr nocapture %b) {
 ; TUNIT: Function Attrs: nosync
 ; TUNIT-LABEL: define void @nocapture_is_not_subsumed_2
-; TUNIT-SAME: (ptr nocapture [[B:%.*]]) #[[ATTR10:[0-9]+]] {
+; TUNIT-SAME: (ptr captures(none) [[B:%.*]]) #[[ATTR10:[0-9]+]] {
 ; TUNIT-NEXT:  entry:
 ; TUNIT-NEXT:    [[CALL:%.*]] = call ptr @readonly_i32p(ptr readonly [[B]]) #[[ATTR13]]
 ; TUNIT-NEXT:    store i32 0, ptr [[CALL]], align 4
@@ -719,7 +719,7 @@ define void @nocapture_is_not_subsumed_2(ptr nocapture %b) {
 ;
 ; CGSCC: Function Attrs: nosync
 ; CGSCC-LABEL: define void @nocapture_is_not_subsumed_2
-; CGSCC-SAME: (ptr nocapture [[B:%.*]]) #[[ATTR11:[0-9]+]] {
+; CGSCC-SAME: (ptr captures(none) [[B:%.*]]) #[[ATTR11:[0-9]+]] {
 ; CGSCC-NEXT:  entry:
 ; CGSCC-NEXT:    [[CALL:%.*]] = call ptr @readonly_i32p(ptr readonly [[B]]) #[[ATTR15]]
 ; CGSCC-NEXT:    store i32 0, ptr [[CALL]], align 4
@@ -781,7 +781,7 @@ define ptr @b64613_b(ptr noundef %p, i32 %i) {
 define void @b64613_positive(ptr noundef %p, i32 %i) {
 ; TUNIT: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(none)
 ; TUNIT-LABEL: define void @b64613_positive
-; TUNIT-SAME: (ptr nocapture nofree noundef [[P:%.*]], i32 [[I:%.*]]) #[[ATTR11]] {
+; TUNIT-SAME: (ptr nofree noundef captures(none) [[P:%.*]], i32 [[I:%.*]]) #[[ATTR11]] {
 ; TUNIT-NEXT:    [[P_ADDR:%.*]] = alloca <2 x ptr>, align 1
 ; TUNIT-NEXT:    [[G:%.*]] = getelementptr i8, ptr [[P_ADDR]], i32 [[I]]
 ; TUNIT-NEXT:    store ptr [[P]], ptr [[G]], align 1
@@ -789,7 +789,7 @@ define void @b64613_positive(ptr noundef %p, i32 %i) {
 ;
 ; CGSCC: Function Attrs: mustprogress nofree nosync nounwind willreturn memory(none)
 ; CGSCC-LABEL: define void @b64613_positive
-; CGSCC-SAME: (ptr nocapture nofree noundef [[P:%.*]], i32 [[I:%.*]]) #[[ATTR13:[0-9]+]] {
+; CGSCC-SAME: (ptr nofree noundef captures(none) [[P:%.*]], i32 [[I:%.*]]) #[[ATTR13:[0-9]+]] {
 ; CGSCC-NEXT:    [[P_ADDR:%.*]] = alloca <2 x ptr>, align 1
 ; CGSCC-NEXT:    [[G:%.*]] = getelementptr i8, ptr [[P_ADDR]], i32 [[I]]
 ; CGSCC-NEXT:    store ptr [[P]], ptr [[G]], align 1
diff --git a/llvm/test/Transforms/Attributor/nofpclass.ll b/llvm/test/Transforms/Attributor/nofpclass.ll
index b97454a29d513..cfc7877383881 100644
--- a/llvm/test/Transforms/Attributor/nofpclass.ll
+++ b/llvm/test/Transforms/Attributor/nofpclass.ll
@@ -675,7 +675,7 @@ B:
 define float @returned_load(ptr %ptr) {
 ; CHECK: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: read)
 ; CHECK-LABEL: define float @returned_load
-; CHECK-SAME: (ptr nocapture nofree noundef nonnull readonly align 4 dereferenceable(4) [[PTR:%.*]]) #[[ATTR5:[0-9]+]] {
+; CHECK-SAME: (ptr nofree noundef nonnull readonly align 4 captures(none) dereferenceable(4) [[PTR:%.*]]) #[[ATTR5:[0-9]+]] {
 ; CHECK-NEXT:    [[LOAD:%.*]] = load float, ptr [[PTR]], align 4
 ; CHECK-NEXT:    ret float [[LOAD]]
 ;
@@ -689,7 +689,7 @@ define float @pass_nofpclass_inf_through_memory(float nofpclass(inf) %arg) {
 ; TUNIT-SAME: (float nofpclass(inf) [[ARG:%.*]]) #[[ATTR3]] {
 ; TUNIT-NEXT:    [[ALLOCA:%.*]] = alloca float, align 4
 ; TUNIT-NEXT:    store float [[ARG]], ptr [[ALLOCA]], align 4
-; TUNIT-NEXT:    [[RET:%.*]] = call float @returned_load(ptr noalias nocapture nofree noundef nonnull readonly align 4 dereferenceable(4) [[ALLOCA]]) #[[ATTR21:[0-9]+]]
+; TUNIT-NEXT:    [[RET:%.*]] = call float @returned_load(ptr noalias nofree noundef nonnull readonly align 4 captures(none) dereferenceable(4) [[ALLOCA]]) #[[ATTR21:[0-9]+]]
 ; TUNIT-NEXT:    ret float [[RET]]
 ;
 ; CGSCC: Function Attrs: mustprogress nofree nosync nounwind willreturn memory(none)
@@ -697,7 +697,7 @@ define float @pass_nofpclass_inf_through_memory(float nofpclass(inf) %arg) {
 ; CGSCC-SAME: (float nofpclass(inf) [[ARG:%.*]]) #[[ATTR4]] {
 ; CGSCC-NEXT:    [[ALLOCA:%.*]] = alloca float, align 4
 ; CGSCC-NEXT:    store float [[ARG]], ptr [[ALLOCA]], align 4
-; CGSCC-NEXT:    [[RET:%.*]] = call float @returned_load(ptr noalias nocapture nofree noundef nonnull readonly align 4 dereferenceable(4) [[ALLOCA]]) #[[ATTR21:[0-9]+]]
+; CGSCC-NEXT:    [[RET:%.*]] = call float @returned_load(ptr noalias nofree noundef nonnull readonly align 4 captures(none) dereferenceable(4) [[ALLOCA]]) #[[ATTR21:[0-9]+]]
 ; CGSCC-NEXT:    ret float [[RET]]
 ;
   %alloca = alloca float
@@ -1880,7 +1880,7 @@ define double @fpext(float nofpclass(inf nan) %arg) {
 define float @atomicrmw_fadd(ptr %ptr, float nofpclass(inf nan) %val) {
 ; CHECK: Function Attrs: mustprogress nofree norecurse nounwind willreturn memory(argmem: readwrite)
 ; CHECK-LABEL: define float @atomicrmw_fadd
-; CHECK-SAME: (ptr nocapture nofree noundef nonnull align 4 dereferenceable(4) [[PTR:%.*]], float nofpclass(nan inf) [[VAL:%.*]]) #[[ATTR6:[0-9]+]] {
+; CHECK-SAME: (ptr nofree noundef nonnull align 4 captures(none) dereferenceable(4) [[PTR:%.*]], float nofpclass(nan inf) [[VAL:%.*]]) #[[ATTR6:[0-9]+]] {
 ; CHECK-NEXT:    [[RESULT:%.*]] = atomicrmw fadd ptr [[PTR]], float [[VAL]] seq_cst, align 4
 ; CHECK-NEXT:    ret float [[RESULT]]
 ;
@@ -1891,7 +1891,7 @@ define float @atomicrmw_fadd(ptr %ptr, float nofpclass(inf nan) %val) {
 define float @load(ptr %ptr, float nofpclass(nan inf) %val) {
 ; CHECK: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: readwrite)
 ; CHECK-LABEL: define nofpclass(ninf nzero nsub nnorm) float @load
-; CHECK-SAME: (ptr nocapture nofree noundef nonnull align 4 dereferenceable(4) [[PTR:%.*]], float nofpclass(nan inf) [[VAL:%.*]]) #[[ATTR7:[0-9]+]] {
+; CHECK-SAME: (ptr nofree noundef nonnull align 4 captures(none) dereferenceable(4) [[PTR:%.*]], float nofpclass(nan inf) [[VAL:%.*]]) #[[ATTR7:[0-9]+]] {
 ; CHECK-NEXT:    store float [[VAL]], ptr [[PTR]], align 4
 ; CHECK-NEXT:    [[LOAD:%.*]] = load float, ptr [[PTR]], align 4
 ; CHECK-NEXT:    [[MUL:%.*]] = fmul float [[LOAD]], [[LOAD]]
@@ -1906,7 +1906,7 @@ define float @load(ptr %ptr, float nofpclass(nan inf) %val) {
 define float @load_atomic(ptr %ptr, float nofpclass(nan inf) %val) {
 ; CHECK: Function Attrs: mustprogress nofree norecurse nounwind willreturn memory(argmem: readwrite)
 ; CHECK-LABEL: define nofpclass(ninf nzero nsub nnorm) float @load_atomic
-; CHECK-SAME: (ptr nocapture nofree noundef nonnull align 4 dereferenceable(4) [[PTR:%.*]], float nofpclass(nan inf) [[VAL:%.*]]) #[[ATTR6]] {
+; CHECK-SAME: (ptr nofree noundef nonnull align 4 captures(none) dereferenceable(4) [[PTR:%.*]], float nofpclass(nan inf) [[VAL:%.*]]) #[[ATTR6]] {
 ; CHECK-NEXT:    store atomic float [[VAL]], ptr [[PTR]] seq_cst, align 4
 ; CHECK-NEXT:    [[LOAD:%.*]] = load atomic float, ptr [[PTR]] seq_cst, align 4
 ; CHECK-NEXT:    [[MUL:%.*]] = fmul float [[LOAD]], [[LOAD]]
@@ -2586,7 +2586,7 @@ define float @call_through_memory2(float nofpclass(nan) %val) {
 
 define amdgpu_kernel void @fast_pow_kernel(ptr addrspace(1) nocapture noundef writeonly align 4 %out, ptr addrspace(1) nocapture noundef readonly align 4 %in1, ptr addrspace(1) nocapture noundef readonly align 4 %in2)  {
 ; CHECK-LABEL: define amdgpu_kernel void @fast_pow_kernel
-; CHECK-SAME: (ptr addrspace(1) nocapture nofree noundef writeonly align 4 [[OUT:%.*]], ptr addrspace(1) nocapture nofree noundef readonly align 4 [[IN1:%.*]], ptr addrspace(1) nocapture nofree noundef readonly align 4 [[IN2:%.*]]) {
+; CHECK-SAME: (ptr addrspace(1) nofree noundef writeonly align 4 captures(none) [[OUT:%.*]], ptr addrspace(1) nofree noundef readonly align 4 captures(none) [[IN1:%.*]], ptr addrspace(1) nofree noundef readonly align 4 captures(none) [[IN2:%.*]]) {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[CALL:%.*]] = tail call i64 @_Z13get_global_idj(i32 noundef 0)
 ; CHECK-NEXT:    [[SEXT:%.*]] = shl i64 [[CALL]], 32
diff --git a/llvm/test/Transforms/Attributor/nofree.ll b/llvm/test/Transforms/Attributor/nofree.ll
index 10753916dabc6..2a9d5d91ae053 100644
--- a/llvm/test/Transforms/Attributor/nofree.ll
+++ b/llvm/test/Transforms/Attributor/nofree.ll
@@ -33,8 +33,8 @@ define void @only_return() #0 {
 define void @only_free(ptr nocapture %0) local_unnamed_addr #0 {
 ; CHECK: Function Attrs: noinline nounwind uwtable
 ; CHECK-LABEL: define {{[^@]+}}@only_free
-; CHECK-SAME: (ptr nocapture [[TMP0:%.*]]) local_unnamed_addr #[[ATTR1:[0-9]+]] {
-; CHECK-NEXT:    tail call void @free(ptr nocapture [[TMP0]]) #[[ATTR0:[0-9]+]]
+; CHECK-SAME: (ptr captures(none) [[TMP0:%.*]]) local_unnamed_addr #[[ATTR1:[0-9]+]] {
+; CHECK-NEXT:    tail call void @free(ptr captures(none) [[TMP0]]) #[[ATTR0:[0-9]+]]
 ; CHECK-NEXT:    ret void
 ;
   tail call void @free(ptr %0) #1
@@ -56,8 +56,8 @@ define void @only_free(ptr nocapture %0) local_unnamed_addr #0 {
 define void @free_in_scc1(ptr nocapture %0) local_unnamed_addr #0 {
 ; CHECK: Function Attrs: noinline nounwind uwtable
 ; CHECK-LABEL: define {{[^@]+}}@free_in_scc1
-; CHECK-SAME: (ptr nocapture [[TMP0:%.*]]) local_unnamed_addr #[[ATTR1]] {
-; CHECK-NEXT:    tail call void @free_in_scc2(ptr nocapture [[TMP0]]) #[[ATTR0]]
+; CHECK-SAME: (ptr captures(none) [[TMP0:%.*]]) local_unnamed_addr #[[ATTR1]] {
+; CHECK-NEXT:    tail call void @free_in_scc2(ptr captures(none) [[TMP0]]) #[[ATTR0]]
 ; CHECK-NEXT:    ret void
 ;
   tail call void @free_in_scc2(ptr %0) #1
@@ -68,14 +68,14 @@ define void @free_in_scc1(ptr nocapture %0) local_unnamed_addr #0 {
 define void @free_in_scc2(ptr nocapture %0) local_unnamed_addr #0 {
 ; CHECK: Function Attrs: noinline nounwind uwtable
 ; CHECK-LABEL: define {{[^@]+}}@free_in_scc2
-; CHECK-SAME: (ptr nocapture [[TMP0:%.*]]) local_unnamed_addr #[[ATTR1]] {
+; CHECK-SAME: (ptr captures(none) [[TMP0:%.*]]) local_unnamed_addr #[[ATTR1]] {
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp eq ptr [[TMP0]], null
 ; CHECK-NEXT:    br i1 [[CMP]], label [[REC:%.*]], label [[CALL:%.*]]
 ; CHECK:       call:
-; CHECK-NEXT:    tail call void @free(ptr nocapture nonnull [[TMP0]]) #[[ATTR0]]
+; CHECK-NEXT:    tail call void @free(ptr nonnull captures(none) [[TMP0]]) #[[ATTR0]]
 ; CHECK-NEXT:    br label [[END:%.*]]
 ; CHECK:       rec:
-; CHECK-NEXT:    tail call void @free_in_scc1(ptr nocapture [[TMP0]]) #[[ATTR0]]
+; CHECK-NEXT:    tail call void @free_in_scc1(ptr captures(none) [[TMP0]]) #[[ATTR0]]
 ; CHECK-NEXT:    br label [[END]]
 ; CHECK:       end:
 ; CHECK-NEXT:    ret void
@@ -169,8 +169,8 @@ define void @_Z9delete_opPc(ptr %0) local_unnamed_addr #0 {
 define noalias ptr @call_realloc(ptr nocapture %0, i64 %1) local_unnamed_addr #0 {
 ; CHECK: Function Attrs: noinline nounwind uwtable
 ; CHECK-LABEL: define {{[^@]+}}@call_realloc
-; CHECK-SAME: (ptr nocapture [[TMP0:%.*]], i64 [[TMP1:%.*]]) local_unnamed_addr #[[ATTR1]] {
-; CHECK-NEXT:    [[RET:%.*]] = tail call ptr @realloc(ptr nocapture [[TMP0]], i64 [[TMP1]]) #[[ATTR2]]
+; CHECK-SAME: (ptr captures(none) [[TMP0:%.*]], i64 [[TMP1:%.*]]) local_unnamed_addr #[[ATTR1]] {
+; CHECK-NEXT:    [[RET:%.*]] = tail call ptr @realloc(ptr captures(none) [[TMP0]], i64 [[TMP1]]) #[[ATTR2]]
 ; CHECK-NEXT:    ret ptr [[RET]]
 ;
   %ret = tail call ptr @realloc(ptr %0, i64 %1) #2
@@ -299,7 +299,7 @@ define void @f2() #0 {
 define double @test12(ptr nocapture readonly %a) {
 ; CHECK: Function Attrs: nofree nounwind
 ; CHECK-LABEL: define {{[^@]+}}@test12
-; CHECK-SAME: (ptr nocapture nofree noundef nonnull readonly align 8 dereferenceable(8) [[A:%.*]]) #[[ATTR7:[0-9]+]] {
+; CHECK-SAME: (ptr nofree noundef nonnull readonly align 8 captures(none) dereferenceable(8) [[A:%.*]]) #[[ATTR7:[0-9]+]] {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[TMP0:%.*]] = load double, ptr [[A]], align 8
 ; CHECK-NEXT:    [[CALL:%.*]] = tail call double @cos(double [[TMP0]]) #[[ATTR8:[0-9]+]]
@@ -318,7 +318,7 @@ declare double @cos(double) nobuiltin nounwind nofree
 define noalias ptr @test13(ptr nocapture readonly %a) {
 ; CHECK: Function Attrs: nounwind
 ; CHECK-LABEL: define {{[^@]+}}@test13
-; CHECK-SAME: (ptr nocapture nofree noundef nonnull readonly align 8 dereferenceable(8) [[A:%.*]]) #[[ATTR0]] {
+; CHECK-SAME: (ptr nofree noundef nonnull readonly align 8 captures(none) dereferenceable(8) [[A:%.*]]) #[[ATTR0]] {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[TMP0:%.*]] = load i64, ptr [[A]], align 8
 ; CHECK-NEXT:    [[CALL:%.*]] = tail call noalias ptr @malloc(i64 [[TMP0]]) #[[ATTR2]]
@@ -333,8 +333,8 @@ entry:
 define void @test14(ptr nocapture %0, ptr nocapture %1) {
 ; CHECK: Function Attrs: nounwind
 ; CHECK-LABEL: define {{[^@]+}}@test14
-; CHECK-SAME: (ptr nocapture [[TMP0:%.*]], ptr nocapture nofree readnone [[TMP1:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT:    tail call void @free(ptr nocapture [[TMP0]]) #[[ATTR0]]
+; CHECK-SAME: (ptr captures(none) [[TMP0:%.*]], ptr nofree readnone captures(none) [[TMP1:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    tail call void @free(ptr captures(none) [[TMP0]]) #[[ATTR0]]
 ; CHECK-NEXT:    ret void
 ;
   tail call void @free(ptr %0) #1
@@ -396,16 +396,16 @@ define void @nonnull_assume_call(ptr %arg1, ptr %arg2, ptr %arg3, ptr %arg4) {
 ; CHECK-LABEL: define {{[^@]+}}@nonnull_assume_call
 ; CHECK-SAME: (ptr [[ARG1:%.*]], ptr [[ARG2:%.*]], ptr [[ARG3:%.*]], ptr [[ARG4:%.*]]) {
 ; CHECK-NEXT:    call void @unknown(ptr [[ARG1]], ptr [[ARG2]], ptr [[ARG3]], ptr [[ARG4]])
-; CHECK-NEXT:    call void @use_i8_ptr(ptr noalias nocapture nofree readnone [[ARG1]]) #[[ATTR0]]
-; CHECK-NEXT:    call void @use_i8_ptr(ptr noalias nocapture nofree readnone [[ARG2]]) #[[ATTR0]]
+; CHECK-NEXT:    call void @use_i8_ptr(ptr noalias nofree readnone captures(none) [[ARG1]]) #[[ATTR0]]
+; CHECK-NEXT:    call void @use_i8_ptr(ptr noalias nofree readnone captures(none) [[ARG2]]) #[[ATTR0]]
 ; CHECK-NEXT:    call void @llvm.assume(i1 noundef true) [ "nofree"(ptr [[ARG1]]), "nofree"(ptr [[ARG3]]) ]
-; CHECK-NEXT:    call void @use_i8_ptr(ptr noalias nocapture nofree readnone [[ARG3]]) #[[ATTR0]]
-; CHECK-NEXT:    call void @use_i8_ptr(ptr noalias nocapture nofree readnone [[ARG4]]) #[[ATTR0]]
-; CHECK-NEXT:    call void @use_i8_ptr_ret(ptr noalias nocapture nofree readnone [[ARG1]]) #[[ATTR0]]
-; CHECK-NEXT:    call void @use_i8_ptr_ret(ptr noalias nocapture nofree readnone [[ARG2]]) #[[ATTR0]]
+; CHECK-NEXT:    call void @use_i8_ptr(ptr noalias nofree readnone captures(none) [[ARG3]]) #[[ATTR0]]
+; CHECK-NEXT:    call void @use_i8_ptr(ptr noalias nofree readnone captures(none) [[ARG4]]) #[[ATTR0]]
+; CHECK-NEXT:    call void @use_i8_ptr_ret(ptr noalias nofree readnone captures(none) [[ARG1]]) #[[ATTR0]]
+; CHECK-NEXT:    call void @use_i8_ptr_ret(ptr noalias nofree readnone captures(none) [[ARG2]]) #[[ATTR0]]
 ; CHECK-NEXT:    call void @llvm.assume(i1 noundef true) [ "nofree"(ptr [[ARG1]]), "nofree"(ptr [[ARG4]]) ]
-; CHECK-NEXT:    call void @use_i8_ptr_ret(ptr noalias nocapture nofree readnone [[ARG3]]) #[[ATTR0]]
-; CHECK-NEXT:    call void @use_i8_ptr_ret(ptr noalias nocapture nofree readnone [[ARG4]]) #[[ATTR0]]
+; CHECK-NEXT:    call void @use_i8_ptr_ret(ptr noalias nofree readnone captures(none) [[ARG3]]) #[[ATTR0]]
+; CHECK-NEXT:    call void @use_i8_ptr_ret(ptr noalias nofree readnone captures(none) [[ARG4]]) #[[ATTR0]]
 ; CHECK-NEXT:    ret void
 ;
   call void @unknown(ptr %arg1, ptr %arg2, ptr %arg3, ptr %arg4)
diff --git a/llvm/test/Transforms/Attributor/nonnull.ll b/llvm/test/Transforms/Attributor/nonnull.ll
index 990695954a9f6..2ff8a3fa3a688 100644
--- a/llvm/test/Transforms/Attributor/nonnull.ll
+++ b/llvm/test/Transforms/Attributor/nonnull.ll
@@ -212,7 +212,7 @@ entry:
   br label %loop
 loop:
   %phi = phi ptr [%ret, %entry], [%phi, %loop]
-  br i1 undef, label %loop, label %exit
+  br i1 poison, label %loop, label %exit
 exit:
   ret ptr %phi
 }
@@ -336,15 +336,15 @@ define void @test13_helper() {
 ; TUNIT-LABEL: define {{[^@]+}}@test13_helper() {
 ; TUNIT-NEXT:    [[NONNULLPTR:%.*]] = tail call nonnull ptr @ret_nonnull()
 ; TUNIT-NEXT:    [[MAYBENULLPTR:%.*]] = tail call ptr @unknown()
-; TUNIT-NEXT:    tail call void @test13(ptr noalias nocapture nofree nonnull readnone [[NONNULLPTR]], ptr noalias nocapture nofree nonnull readnone [[NONNULLPTR]], ptr noalias nocapture nofree readnone [[MAYBENULLPTR]]) #[[ATTR5:[0-9]+]]
-; TUNIT-NEXT:    tail call void @test13(ptr noalias nocapture nofree nonnull readnone [[NONNULLPTR]], ptr noalias nocapture nofree readnone [[MAYBENULLPTR]], ptr noalias nocapture nofree nonnull readnone [[NONNULLPTR]]) #[[ATTR5]]
+; TUNIT-NEXT:    tail call void @test13(ptr noalias nofree nonnull readnone captures(none) [[NONNULLPTR]], ptr noalias nofree nonnull readnone captures(none) [[NONNULLPTR]], ptr noalias nofree readnone captures(none) [[MAYBENULLPTR]]) #[[ATTR5:[0-9]+]]
+; TUNIT-NEXT:    tail call void @test13(ptr noalias nofree nonnull readnone captures(none) [[NONNULLPTR]], ptr noalias nofree readnone captures(none) [[MAYBENULLPTR]], ptr noalias nofree nonnull readnone captures(none) [[NONNULLPTR]]) #[[ATTR5]]
 ; TUNIT-NEXT:    ret void
 ;
 ; CGSCC-LABEL: define {{[^@]+}}@test13_helper() {
 ; CGSCC-NEXT:    [[NONNULLPTR:%.*]] = tail call nonnull ptr @ret_nonnull()
 ; CGSCC-NEXT:    [[MAYBENULLPTR:%.*]] = tail call ptr @unknown()
-; CGSCC-NEXT:    tail call void @test13(ptr noalias nocapture nofree nonnull readnone [[NONNULLPTR]], ptr noalias nocapture nofree nonnull readnone [[NONNULLPTR]], ptr noalias nocapture nofree readnone [[MAYBENULLPTR]]) #[[ATTR4:[0-9]+]]
-; CGSCC-NEXT:    tail call void @test13(ptr noalias nocapture nofree nonnull readnone [[NONNULLPTR]], ptr noalias nocapture nofree readnone [[MAYBENULLPTR]], ptr noalias nocapture nofree nonnull readnone [[NONNULLPTR]]) #[[ATTR4]]
+; CGSCC-NEXT:    tail call void @test13(ptr noalias nofree nonnull readnone captures(none) [[NONNULLPTR]], ptr noalias nofree nonnull readnone captures(none) [[NONNULLPTR]], ptr noalias nofree readnone captures(none) [[MAYBENULLPTR]]) #[[ATTR4:[0-9]+]]
+; CGSCC-NEXT:    tail call void @test13(ptr noalias nofree nonnull readnone captures(none) [[NONNULLPTR]], ptr noalias nofree readnone captures(none) [[MAYBENULLPTR]], ptr noalias nofree nonnull readnone captures(none) [[NONNULLPTR]]) #[[ATTR4]]
 ; CGSCC-NEXT:    ret void
 ;
   %nonnullptr = tail call ptr @ret_nonnull()
@@ -357,18 +357,18 @@ define internal void @test13(ptr %a, ptr %b, ptr %c) {
 ;
 ; TUNIT: Function Attrs: nounwind
 ; TUNIT-LABEL: define {{[^@]+}}@test13
-; TUNIT-SAME: (ptr noalias nocapture nofree nonnull readnone [[A:%.*]], ptr noalias nocapture nofree readnone [[B:%.*]], ptr noalias nocapture nofree readnone [[C:%.*]]) #[[ATTR5]] {
-; TUNIT-NEXT:    call void @use_i8_ptr(ptr noalias nocapture nofree nonnull readnone [[A]]) #[[ATTR5]]
-; TUNIT-NEXT:    call void @use_i8_ptr(ptr noalias nocapture nofree readnone [[B]]) #[[ATTR5]]
-; TUNIT-NEXT:    call void @use_i8_ptr(ptr noalias nocapture nofree readnone [[C]]) #[[ATTR5]]
+; TUNIT-SAME: (ptr noalias nofree nonnull readnone captures(none) [[A:%.*]], ptr noalias nofree readnone captures(none) [[B:%.*]], ptr noalias nofree readnone captures(none) [[C:%.*]]) #[[ATTR5]] {
+; TUNIT-NEXT:    call void @use_i8_ptr(ptr noalias nofree nonnull readnone captures(none) [[A]]) #[[ATTR5]]
+; TUNIT-NEXT:    call void @use_i8_ptr(ptr noalias nofree readnone captures(none) [[B]]) #[[ATTR5]]
+; TUNIT-NEXT:    call void @use_i8_ptr(ptr noalias nofree readnone captures(none) [[C]]) #[[ATTR5]]
 ; TUNIT-NEXT:    ret void
 ;
 ; CGSCC: Function Attrs: nounwind
 ; CGSCC-LABEL: define {{[^@]+}}@test13
-; CGSCC-SAME: (ptr noalias nocapture nofree nonnull readnone [[A:%.*]], ptr noalias nocapture nofree readnone [[B:%.*]], ptr noalias nocapture nofree readnone [[C:%.*]]) #[[ATTR4]] {
-; CGSCC-NEXT:    call void @use_i8_ptr(ptr noalias nocapture nofree nonnull readnone [[A]]) #[[ATTR4]]
-; CGSCC-NEXT:    call void @use_i8_ptr(ptr noalias nocapture nofree readnone [[B]]) #[[ATTR4]]
-; CGSCC-NEXT:    call void @use_i8_ptr(ptr noalias nocapture nofree readnone [[C]]) #[[ATTR4]]
+; CGSCC-SAME: (ptr noalias nofree nonnull readnone captures(none) [[A:%.*]], ptr noalias nofree readnone captures(none) [[B:%.*]], ptr noalias nofree readnone captures(none) [[C:%.*]]) #[[ATTR4]] {
+; CGSCC-NEXT:    call void @use_i8_ptr(ptr noalias nofree nonnull readnone captures(none) [[A]]) #[[ATTR4]]
+; CGSCC-NEXT:    call void @use_i8_ptr(ptr noalias nofree readnone captures(none) [[B]]) #[[ATTR4]]
+; CGSCC-NEXT:    call void @use_i8_ptr(ptr noalias nofree readnone captures(none) [[C]]) #[[ATTR4]]
 ; CGSCC-NEXT:    ret void
 ;
   call void @use_i8_ptr(ptr %a)
@@ -877,7 +877,7 @@ declare i32 @esfp(...)
 define i1 @parent8(ptr %a, ptr %bogus1, ptr %b) personality ptr @esfp{
 ; TUNIT: Function Attrs: nounwind
 ; TUNIT-LABEL: define {{[^@]+}}@parent8
-; TUNIT-SAME: (ptr nonnull [[A:%.*]], ptr nocapture nofree readnone [[BOGUS1:%.*]], ptr nonnull [[B:%.*]]) #[[ATTR5]] personality ptr @esfp {
+; TUNIT-SAME: (ptr nonnull [[A:%.*]], ptr nofree readnone captures(none) [[BOGUS1:%.*]], ptr nonnull [[B:%.*]]) #[[ATTR5]] personality ptr @esfp {
 ; TUNIT-NEXT:  entry:
 ; TUNIT-NEXT:    invoke void @use2nonnull(ptr nonnull [[A]], ptr nonnull [[B]])
 ; TUNIT-NEXT:            to label [[CONT:%.*]] unwind label [[EXC:%.*]]
@@ -890,7 +890,7 @@ define i1 @parent8(ptr %a, ptr %bogus1, ptr %b) personality ptr @esfp{
 ;
 ; CGSCC: Function Attrs: nounwind
 ; CGSCC-LABEL: define {{[^@]+}}@parent8
-; CGSCC-SAME: (ptr nonnull [[A:%.*]], ptr nocapture nofree readnone [[BOGUS1:%.*]], ptr nonnull [[B:%.*]]) #[[ATTR4]] personality ptr @esfp {
+; CGSCC-SAME: (ptr nonnull [[A:%.*]], ptr nofree readnone captures(none) [[BOGUS1:%.*]], ptr nonnull [[B:%.*]]) #[[ATTR4]] personality ptr @esfp {
 ; CGSCC-NEXT:  entry:
 ; CGSCC-NEXT:    invoke void @use2nonnull(ptr nonnull [[A]], ptr nonnull [[B]])
 ; CGSCC-NEXT:            to label [[CONT:%.*]] unwind label [[EXC:%.*]]
@@ -995,14 +995,14 @@ declare void @use_i32_ptr(ptr readnone nocapture) nounwind
 define internal void @called_by_weak(ptr %a) {
 ; TUNIT: Function Attrs: nounwind
 ; TUNIT-LABEL: define {{[^@]+}}@called_by_weak
-; TUNIT-SAME: (ptr noalias nocapture nofree nonnull readnone [[A:%.*]]) #[[ATTR5]] {
-; TUNIT-NEXT:    call void @use_i32_ptr(ptr noalias nocapture nofree nonnull readnone [[A]]) #[[ATTR5]]
+; TUNIT-SAME: (ptr noalias nofree nonnull readnone captures(none) [[A:%.*]]) #[[ATTR5]] {
+; TUNIT-NEXT:    call void @use_i32_ptr(ptr noalias nofree nonnull readnone captures(none) [[A]]) #[[ATTR5]]
 ; TUNIT-NEXT:    ret void
 ;
 ; CGSCC: Function Attrs: nounwind
 ; CGSCC-LABEL: define {{[^@]+}}@called_by_weak
-; CGSCC-SAME: (ptr noalias nocapture nofree nonnull readnone [[A:%.*]]) #[[ATTR4]] {
-; CGSCC-NEXT:    call void @use_i32_ptr(ptr noalias nocapture nofree nonnull readnone [[A]]) #[[ATTR4]]
+; CGSCC-SAME: (ptr noalias nofree nonnull readnone captures(none) [[A:%.*]]) #[[ATTR4]] {
+; CGSCC-NEXT:    call void @use_i32_ptr(ptr noalias nofree nonnull readnone captures(none) [[A]]) #[[ATTR4]]
 ; CGSCC-NEXT:    ret void
 ;
   call void @use_i32_ptr(ptr %a)
@@ -1014,12 +1014,12 @@ define weak_odr void @weak_caller(ptr nonnull %a) {
 ;
 ; TUNIT-LABEL: define {{[^@]+}}@weak_caller
 ; TUNIT-SAME: (ptr nonnull [[A:%.*]]) {
-; TUNIT-NEXT:    call void @called_by_weak(ptr noalias nocapture nofree nonnull readnone [[A]]) #[[ATTR5]]
+; TUNIT-NEXT:    call void @called_by_weak(ptr noalias nofree nonnull readnone captures(none) [[A]]) #[[ATTR5]]
 ; TUNIT-NEXT:    ret void
 ;
 ; CGSCC-LABEL: define {{[^@]+}}@weak_caller
 ; CGSCC-SAME: (ptr nonnull [[A:%.*]]) {
-; CGSCC-NEXT:    call void @called_by_weak(ptr noalias nocapture nofree nonnull readnone [[A]]) #[[ATTR4]]
+; CGSCC-NEXT:    call void @called_by_weak(ptr noalias nofree nonnull readnone captures(none) [[A]]) #[[ATTR4]]
 ; CGSCC-NEXT:    ret void
 ;
   call void @called_by_weak(ptr %a)
@@ -1030,14 +1030,14 @@ define weak_odr void @weak_caller(ptr nonnull %a) {
 define internal void @control(ptr dereferenceable(4) %a) {
 ; TUNIT: Function Attrs: nounwind
 ; TUNIT-LABEL: define {{[^@]+}}@control
-; TUNIT-SAME: (ptr noalias nocapture nofree noundef nonnull readnone align 16 dereferenceable(8) [[A:%.*]]) #[[ATTR5]] {
-; TUNIT-NEXT:    call void @use_i32_ptr(ptr noalias nocapture nofree noundef nonnull readnone align 16 dereferenceable(8) [[A]]) #[[ATTR5]]
+; TUNIT-SAME: (ptr noalias nofree noundef nonnull readnone align 16 captures(none) dereferenceable(8) [[A:%.*]]) #[[ATTR5]] {
+; TUNIT-NEXT:    call void @use_i32_ptr(ptr noalias nofree noundef nonnull readnone align 16 captures(none) dereferenceable(8) [[A]]) #[[ATTR5]]
 ; TUNIT-NEXT:    ret void
 ;
 ; CGSCC: Function Attrs: nounwind
 ; CGSCC-LABEL: define {{[^@]+}}@control
-; CGSCC-SAME: (ptr noalias nocapture nofree noundef nonnull readnone align 16 dereferenceable(8) [[A:%.*]]) #[[ATTR4]] {
-; CGSCC-NEXT:    call void @use_i32_ptr(ptr noalias nocapture nofree noundef nonnull readnone align 16 dereferenceable(8) [[A]]) #[[ATTR4]]
+; CGSCC-SAME: (ptr noalias nofree noundef nonnull readnone align 16 captures(none) dereferenceable(8) [[A:%.*]]) #[[ATTR4]] {
+; CGSCC-NEXT:    call void @use_i32_ptr(ptr noalias nofree noundef nonnull readnone align 16 captures(none) dereferenceable(8) [[A]]) #[[ATTR4]]
 ; CGSCC-NEXT:    ret void
 ;
   call void @use_i32_ptr(ptr %a)
@@ -1058,7 +1058,7 @@ define internal void @optnone(ptr dereferenceable(4) %a) optnone noinline {
 ; CHECK: Function Attrs: noinline optnone
 ; CHECK-LABEL: define {{[^@]+}}@optnone
 ; CHECK-SAME: (ptr noundef nonnull dereferenceable(4) [[A:%.*]]) #[[ATTR12:[0-9]+]] {
-; CHECK-NEXT:    call void @use_i32_ptr(ptr nocapture nofree noundef nonnull [[A]])
+; CHECK-NEXT:    call void @use_i32_ptr(ptr nofree noundef nonnull captures(none) [[A]])
 ; CHECK-NEXT:    ret void
 ;
   call void @use_i32_ptr(ptr %a)
@@ -1068,14 +1068,14 @@ define void @make_live(ptr nonnull dereferenceable(8) %a) {
 ; TUNIT-LABEL: define {{[^@]+}}@make_live
 ; TUNIT-SAME: (ptr noundef nonnull align 16 dereferenceable(8) [[A:%.*]]) {
 ; TUNIT-NEXT:    call void @naked(ptr noundef nonnull align 16 dereferenceable(8) [[A]])
-; TUNIT-NEXT:    call void @control(ptr noalias nocapture nofree noundef nonnull readnone align 16 dereferenceable(8) [[A]]) #[[ATTR5]]
+; TUNIT-NEXT:    call void @control(ptr noalias nofree noundef nonnull readnone align 16 captures(none) dereferenceable(8) [[A]]) #[[ATTR5]]
 ; TUNIT-NEXT:    call void @optnone(ptr noundef nonnull align 16 dereferenceable(8) [[A]])
 ; TUNIT-NEXT:    ret void
 ;
 ; CGSCC-LABEL: define {{[^@]+}}@make_live
 ; CGSCC-SAME: (ptr noundef nonnull align 16 dereferenceable(8) [[A:%.*]]) {
 ; CGSCC-NEXT:    call void @naked(ptr noundef nonnull align 16 dereferenceable(8) [[A]])
-; CGSCC-NEXT:    call void @control(ptr noalias nocapture nofree noundef nonnull readnone align 16 dereferenceable(8) [[A]]) #[[ATTR4]]
+; CGSCC-NEXT:    call void @control(ptr noalias nofree noundef nonnull readnone align 16 captures(none) dereferenceable(8) [[A]]) #[[ATTR4]]
 ; CGSCC-NEXT:    call void @optnone(ptr noundef nonnull align 16 dereferenceable(8) [[A]])
 ; CGSCC-NEXT:    ret void
 ;
@@ -1424,16 +1424,16 @@ define void @nonnull_assume_pos(ptr %arg) {
 ; ATTRIBUTOR-NEXT:    ret void
 ;
 ; TUNIT-LABEL: define {{[^@]+}}@nonnull_assume_pos
-; TUNIT-SAME: (ptr nocapture nofree nonnull readnone [[ARG:%.*]]) {
+; TUNIT-SAME: (ptr nofree nonnull readnone captures(none) [[ARG:%.*]]) {
 ; TUNIT-NEXT:    call void @llvm.assume(i1 noundef true) #[[ATTR16]] [ "nonnull"(ptr [[ARG]]) ]
-; TUNIT-NEXT:    call void @use_i8_ptr(ptr noalias nocapture nofree nonnull readnone [[ARG]]) #[[ATTR5]]
+; TUNIT-NEXT:    call void @use_i8_ptr(ptr noalias nofree nonnull readnone captures(none) [[ARG]]) #[[ATTR5]]
 ; TUNIT-NEXT:    [[TMP1:%.*]] = call ptr @unknown()
 ; TUNIT-NEXT:    ret void
 ;
 ; CGSCC-LABEL: define {{[^@]+}}@nonnull_assume_pos
-; CGSCC-SAME: (ptr nocapture nofree nonnull readnone [[ARG:%.*]]) {
+; CGSCC-SAME: (ptr nofree nonnull readnone captures(none) [[ARG:%.*]]) {
 ; CGSCC-NEXT:    call void @llvm.assume(i1 noundef true) #[[ATTR16]] [ "nonnull"(ptr [[ARG]]) ]
-; CGSCC-NEXT:    call void @use_i8_ptr(ptr noalias nocapture nofree nonnull readnone [[ARG]]) #[[ATTR4]]
+; CGSCC-NEXT:    call void @use_i8_ptr(ptr noalias nofree nonnull readnone captures(none) [[ARG]]) #[[ATTR4]]
 ; CGSCC-NEXT:    [[TMP1:%.*]] = call ptr @unknown()
 ; CGSCC-NEXT:    ret void
 ;
@@ -1457,27 +1457,27 @@ define void @nonnull_assume_neg(ptr %arg) {
 ;
 ;
 ; TUNIT-LABEL: define {{[^@]+}}@nonnull_assume_neg
-; TUNIT-SAME: (ptr nocapture nofree readnone [[ARG:%.*]]) {
+; TUNIT-SAME: (ptr nofree readnone captures(none) [[ARG:%.*]]) {
 ; TUNIT-NEXT:    [[TMP1:%.*]] = call ptr @unknown()
-; TUNIT-NEXT:    call void @use_i8_ptr(ptr noalias nocapture nofree readnone [[ARG]]) #[[ATTR5]]
+; TUNIT-NEXT:    call void @use_i8_ptr(ptr noalias nofree readnone captures(none) [[ARG]]) #[[ATTR5]]
 ; TUNIT-NEXT:    call void @llvm.assume(i1 noundef true) [ "nonnull"(ptr [[ARG]]) ]
-; TUNIT-NEXT:    call void @use_i8_ptr(ptr noalias nocapture nofree nonnull readnone [[ARG]]) #[[ATTR5]]
+; TUNIT-NEXT:    call void @use_i8_ptr(ptr noalias nofree nonnull readnone captures(none) [[ARG]]) #[[ATTR5]]
 ; TUNIT-NEXT:    [[TMP2:%.*]] = call ptr @unknown()
-; TUNIT-NEXT:    call void @use_i8_ptr_ret(ptr noalias nocapture nofree nonnull readnone [[ARG]]) #[[ATTR5]]
+; TUNIT-NEXT:    call void @use_i8_ptr_ret(ptr noalias nofree nonnull readnone captures(none) [[ARG]]) #[[ATTR5]]
 ; TUNIT-NEXT:    call void @llvm.assume(i1 noundef true) [ "nonnull"(ptr [[ARG]]) ]
-; TUNIT-NEXT:    call void @use_i8_ptr_ret(ptr noalias nocapture nofree nonnull readnone [[ARG]]) #[[ATTR5]]
+; TUNIT-NEXT:    call void @use_i8_ptr_ret(ptr noalias nofree nonnull readnone captures(none) [[ARG]]) #[[ATTR5]]
 ; TUNIT-NEXT:    ret void
 ;
 ; CGSCC-LABEL: define {{[^@]+}}@nonnull_assume_neg
-; CGSCC-SAME: (ptr nocapture nofree readnone [[ARG:%.*]]) {
+; CGSCC-SAME: (ptr nofree readnone captures(none) [[ARG:%.*]]) {
 ; CGSCC-NEXT:    [[TMP1:%.*]] = call ptr @unknown()
-; CGSCC-NEXT:    call void @use_i8_ptr(ptr noalias nocapture nofree readnone [[ARG]]) #[[ATTR4]]
+; CGSCC-NEXT:    call void @use_i8_ptr(ptr noalias nofree readnone captures(none) [[ARG]]) #[[ATTR4]]
 ; CGSCC-NEXT:    call void @llvm.assume(i1 noundef true) [ "nonnull"(ptr [[ARG]]) ]
-; CGSCC-NEXT:    call void @use_i8_ptr(ptr noalias nocapture nofree nonnull readnone [[ARG]]) #[[ATTR4]]
+; CGSCC-NEXT:    call void @use_i8_ptr(ptr noalias nofree nonnull readnone captures(none) [[ARG]]) #[[ATTR4]]
 ; CGSCC-NEXT:    [[TMP2:%.*]] = call ptr @unknown()
-; CGSCC-NEXT:    call void @use_i8_ptr_ret(ptr noalias nocapture nofree nonnull readnone [[ARG]]) #[[ATTR4]]
+; CGSCC-NEXT:    call void @use_i8_ptr_ret(ptr noalias nofree nonnull readnone captures(none) [[ARG]]) #[[ATTR4]]
 ; CGSCC-NEXT:    call void @llvm.assume(i1 noundef true) [ "nonnull"(ptr [[ARG]]) ]
-; CGSCC-NEXT:    call void @use_i8_ptr_ret(ptr noalias nocapture nofree nonnull readnone [[ARG]]) #[[ATTR4]]
+; CGSCC-NEXT:    call void @use_i8_ptr_ret(ptr noalias nofree nonnull readnone captures(none) [[ARG]]) #[[ATTR4]]
 ; CGSCC-NEXT:    ret void
 ;
   call ptr @unknown()
@@ -1555,14 +1555,14 @@ define void @phi_caller(ptr %p) {
 ; TUNIT-LABEL: define {{[^@]+}}@phi_caller
 ; TUNIT-SAME: (ptr nofree [[P:%.*]]) #[[ATTR5]] {
 ; TUNIT-NEXT:    [[C:%.*]] = call nonnull ptr @phi(ptr noalias nofree readnone [[P]]) #[[ATTR20:[0-9]+]]
-; TUNIT-NEXT:    call void @use_i8_ptr(ptr noalias nocapture nofree nonnull readnone [[C]]) #[[ATTR5]]
+; TUNIT-NEXT:    call void @use_i8_ptr(ptr noalias nofree nonnull readnone captures(none) [[C]]) #[[ATTR5]]
 ; TUNIT-NEXT:    ret void
 ;
 ; CGSCC: Function Attrs: nounwind
 ; CGSCC-LABEL: define {{[^@]+}}@phi_caller
 ; CGSCC-SAME: (ptr nofree [[P:%.*]]) #[[ATTR4]] {
 ; CGSCC-NEXT:    [[C:%.*]] = call nonnull ptr @phi(ptr noalias nofree readnone [[P]]) #[[ATTR21:[0-9]+]]
-; CGSCC-NEXT:    call void @use_i8_ptr(ptr noalias nocapture nofree nonnull readnone [[C]]) #[[ATTR4]]
+; CGSCC-NEXT:    call void @use_i8_ptr(ptr noalias nofree nonnull readnone captures(none) [[C]]) #[[ATTR4]]
 ; CGSCC-NEXT:    ret void
 ;
   %c = call ptr @phi(ptr %p)
@@ -1595,14 +1595,14 @@ define void @multi_ret_caller(ptr %p) {
 ; TUNIT-LABEL: define {{[^@]+}}@multi_ret_caller
 ; TUNIT-SAME: (ptr nofree [[P:%.*]]) #[[ATTR5]] {
 ; TUNIT-NEXT:    [[C:%.*]] = call nonnull ptr @multi_ret(ptr noalias nofree readnone [[P]]) #[[ATTR20]]
-; TUNIT-NEXT:    call void @use_i8_ptr(ptr noalias nocapture nofree nonnull readnone [[C]]) #[[ATTR5]]
+; TUNIT-NEXT:    call void @use_i8_ptr(ptr noalias nofree nonnull readnone captures(none) [[C]]) #[[ATTR5]]
 ; TUNIT-NEXT:    ret void
 ;
 ; CGSCC: Function Attrs: nounwind
 ; CGSCC-LABEL: define {{[^@]+}}@multi_ret_caller
 ; CGSCC-SAME: (ptr nofree [[P:%.*]]) #[[ATTR4]] {
 ; CGSCC-NEXT:    [[C:%.*]] = call nonnull ptr @multi_ret(ptr noalias nofree readnone [[P]]) #[[ATTR21]]
-; CGSCC-NEXT:    call void @use_i8_ptr(ptr noalias nocapture nofree nonnull readnone [[C]]) #[[ATTR4]]
+; CGSCC-NEXT:    call void @use_i8_ptr(ptr noalias nofree nonnull readnone captures(none) [[C]]) #[[ATTR4]]
 ; CGSCC-NEXT:    ret void
 ;
   %c = call ptr @multi_ret(ptr %p)
diff --git a/llvm/test/Transforms/Attributor/norecurse.ll b/llvm/test/Transforms/Attributor/norecurse.ll
index f139f193f10de..df3a93f300971 100644
--- a/llvm/test/Transforms/Attributor/norecurse.ll
+++ b/llvm/test/Transforms/Attributor/norecurse.ll
@@ -67,8 +67,8 @@ declare i32 @k() readnone
 define void @intrinsic(ptr %dest, ptr %src, i32 %len) {
 ; CHECK: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: readwrite)
 ; CHECK-LABEL: define {{[^@]+}}@intrinsic
-; CHECK-SAME: (ptr nocapture nofree writeonly [[DEST:%.*]], ptr nocapture nofree readonly [[SRC:%.*]], i32 [[LEN:%.*]]) #[[ATTR4:[0-9]+]] {
-; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr nocapture nofree writeonly [[DEST]], ptr nocapture nofree readonly [[SRC]], i32 [[LEN]], i1 noundef false) #[[ATTR10:[0-9]+]]
+; CHECK-SAME: (ptr nofree writeonly captures(none) [[DEST:%.*]], ptr nofree readonly captures(none) [[SRC:%.*]], i32 [[LEN:%.*]]) #[[ATTR4:[0-9]+]] {
+; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr nofree writeonly captures(none) [[DEST]], ptr nofree readonly captures(none) [[SRC]], i32 [[LEN]], i1 noundef false) #[[ATTR10:[0-9]+]]
 ; CHECK-NEXT:    ret void
 ;
   call void @llvm.memcpy.p0.p0.i32(ptr %dest, ptr %src, i32 %len, i1 false)
@@ -217,7 +217,7 @@ define linkonce_odr i32 @leaf_redefinable() {
 ; Call through a function pointer
 define i32 @eval_func1(ptr , i32) local_unnamed_addr {
 ; CHECK-LABEL: define {{[^@]+}}@eval_func1
-; CHECK-SAME: (ptr nocapture nofree noundef nonnull [[TMP0:%.*]], i32 [[TMP1:%.*]]) local_unnamed_addr {
+; CHECK-SAME: (ptr nofree noundef nonnull captures(none) [[TMP0:%.*]], i32 [[TMP1:%.*]]) local_unnamed_addr {
 ; CHECK-NEXT:    [[TMP3:%.*]] = tail call i32 [[TMP0]](i32 [[TMP1]])
 ; CHECK-NEXT:    ret i32 [[TMP3]]
 ;
@@ -228,7 +228,7 @@ define i32 @eval_func1(ptr , i32) local_unnamed_addr {
 define i32 @eval_func2(ptr , i32) local_unnamed_addr null_pointer_is_valid{
 ; CHECK: Function Attrs: null_pointer_is_valid
 ; CHECK-LABEL: define {{[^@]+}}@eval_func2
-; CHECK-SAME: (ptr nocapture nofree noundef [[TMP0:%.*]], i32 [[TMP1:%.*]]) local_unnamed_addr #[[ATTR7:[0-9]+]] {
+; CHECK-SAME: (ptr nofree noundef captures(none) [[TMP0:%.*]], i32 [[TMP1:%.*]]) local_unnamed_addr #[[ATTR7:[0-9]+]] {
 ; CHECK-NEXT:    [[TMP3:%.*]] = tail call i32 [[TMP0]](i32 [[TMP1]])
 ; CHECK-NEXT:    ret i32 [[TMP3]]
 ;
diff --git a/llvm/test/Transforms/Attributor/nosync.ll b/llvm/test/Transforms/Attributor/nosync.ll
index 1fbb334b9e47c..7ef46e8e94c9e 100644
--- a/llvm/test/Transforms/Attributor/nosync.ll
+++ b/llvm/test/Transforms/Attributor/nosync.ll
@@ -52,7 +52,7 @@ entry:
 define i32 @load_monotonic(ptr nocapture readonly %arg) norecurse nounwind uwtable {
 ; CHECK: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: readwrite) uwtable
 ; CHECK-LABEL: define {{[^@]+}}@load_monotonic
-; CHECK-SAME: (ptr nocapture nofree noundef nonnull readonly align 4 dereferenceable(4) [[ARG:%.*]]) #[[ATTR1:[0-9]+]] {
+; CHECK-SAME: (ptr nofree noundef nonnull readonly align 4 captures(none) dereferenceable(4) [[ARG:%.*]]) #[[ATTR1:[0-9]+]] {
 ; CHECK-NEXT:    [[I:%.*]] = load atomic i32, ptr [[ARG]] monotonic, align 4
 ; CHECK-NEXT:    ret i32 [[I]]
 ;
@@ -70,7 +70,7 @@ define i32 @load_monotonic(ptr nocapture readonly %arg) norecurse nounwind uwtab
 define void @store_monotonic(ptr nocapture %arg) norecurse nounwind uwtable {
 ; CHECK: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: readwrite) uwtable
 ; CHECK-LABEL: define {{[^@]+}}@store_monotonic
-; CHECK-SAME: (ptr nocapture nofree noundef nonnull writeonly align 4 dereferenceable(4) [[ARG:%.*]]) #[[ATTR1]] {
+; CHECK-SAME: (ptr nofree noundef nonnull writeonly align 4 captures(none) dereferenceable(4) [[ARG:%.*]]) #[[ATTR1]] {
 ; CHECK-NEXT:    store atomic i32 10, ptr [[ARG]] monotonic, align 4
 ; CHECK-NEXT:    ret void
 ;
@@ -88,7 +88,7 @@ define void @store_monotonic(ptr nocapture %arg) norecurse nounwind uwtable {
 define i32 @load_acquire(ptr nocapture readonly %arg) norecurse nounwind uwtable {
 ; CHECK: Function Attrs: mustprogress nofree norecurse nounwind willreturn memory(argmem: readwrite) uwtable
 ; CHECK-LABEL: define {{[^@]+}}@load_acquire
-; CHECK-SAME: (ptr nocapture nofree noundef nonnull readonly align 4 dereferenceable(4) [[ARG:%.*]]) #[[ATTR2:[0-9]+]] {
+; CHECK-SAME: (ptr nofree noundef nonnull readonly align 4 captures(none) dereferenceable(4) [[ARG:%.*]]) #[[ATTR2:[0-9]+]] {
 ; CHECK-NEXT:    [[I:%.*]] = load atomic i32, ptr [[ARG]] acquire, align 4
 ; CHECK-NEXT:    ret i32 [[I]]
 ;
@@ -105,7 +105,7 @@ define i32 @load_acquire(ptr nocapture readonly %arg) norecurse nounwind uwtable
 define void @load_release(ptr nocapture %arg) norecurse nounwind uwtable {
 ; CHECK: Function Attrs: mustprogress nofree norecurse nounwind willreturn memory(argmem: readwrite) uwtable
 ; CHECK-LABEL: define {{[^@]+}}@load_release
-; CHECK-SAME: (ptr nocapture nofree noundef writeonly align 4 [[ARG:%.*]]) #[[ATTR2]] {
+; CHECK-SAME: (ptr nofree noundef writeonly align 4 captures(none) [[ARG:%.*]]) #[[ATTR2]] {
 ; CHECK-NEXT:    store atomic volatile i32 10, ptr [[ARG]] release, align 4
 ; CHECK-NEXT:    ret void
 ;
@@ -118,7 +118,7 @@ define void @load_release(ptr nocapture %arg) norecurse nounwind uwtable {
 define void @load_volatile_release(ptr nocapture %arg) norecurse nounwind uwtable {
 ; CHECK: Function Attrs: mustprogress nofree norecurse nounwind willreturn memory(argmem: readwrite) uwtable
 ; CHECK-LABEL: define {{[^@]+}}@load_volatile_release
-; CHECK-SAME: (ptr nocapture nofree noundef writeonly align 4 [[ARG:%.*]]) #[[ATTR2]] {
+; CHECK-SAME: (ptr nofree noundef writeonly align 4 captures(none) [[ARG:%.*]]) #[[ATTR2]] {
 ; CHECK-NEXT:    store atomic volatile i32 10, ptr [[ARG]] release, align 4
 ; CHECK-NEXT:    ret void
 ;
@@ -242,7 +242,7 @@ define void @scc2(ptr %arg) noinline nounwind uwtable {
 define void @foo1(ptr %arg, ptr %arg1) {
 ; CHECK: Function Attrs: mustprogress nofree norecurse nounwind willreturn
 ; CHECK-LABEL: define {{[^@]+}}@foo1
-; CHECK-SAME: (ptr nocapture nofree noundef nonnull writeonly align 4 dereferenceable(4) [[ARG:%.*]], ptr nocapture nofree noundef nonnull writeonly dereferenceable(1) [[ARG1:%.*]]) #[[ATTR6:[0-9]+]] {
+; CHECK-SAME: (ptr nofree noundef nonnull writeonly align 4 captures(none) dereferenceable(4) [[ARG:%.*]], ptr nofree noundef nonnull writeonly captures(none) dereferenceable(1) [[ARG1:%.*]]) #[[ATTR6:[0-9]+]] {
 ; CHECK-NEXT:    store i32 100, ptr [[ARG]], align 4
 ; CHECK-NEXT:    fence release
 ; CHECK-NEXT:    store atomic i8 1, ptr [[ARG1]] monotonic, align 1
@@ -257,7 +257,7 @@ define void @foo1(ptr %arg, ptr %arg1) {
 define void @bar(ptr %arg, ptr %arg1) {
 ; CHECK: Function Attrs: nofree norecurse nounwind
 ; CHECK-LABEL: define {{[^@]+}}@bar
-; CHECK-SAME: (ptr nocapture nofree readnone [[ARG:%.*]], ptr nocapture nofree nonnull readonly dereferenceable(1) [[ARG1:%.*]]) #[[ATTR7:[0-9]+]] {
+; CHECK-SAME: (ptr nofree readnone captures(none) [[ARG:%.*]], ptr nofree nonnull readonly captures(none) dereferenceable(1) [[ARG1:%.*]]) #[[ATTR7:[0-9]+]] {
 ; CHECK-NEXT:    br label [[BB2:%.*]]
 ; CHECK:       bb2:
 ; CHECK-NEXT:    [[I3:%.*]] = load atomic i8, ptr [[ARG1]] monotonic, align 1
@@ -285,7 +285,7 @@ bb6:
 define void @foo1_singlethread(ptr %arg, ptr %arg1) {
 ; CHECK: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn
 ; CHECK-LABEL: define {{[^@]+}}@foo1_singlethread
-; CHECK-SAME: (ptr nocapture nofree noundef nonnull writeonly align 4 dereferenceable(4) [[ARG:%.*]], ptr nocapture nofree noundef nonnull writeonly dereferenceable(1) [[ARG1:%.*]]) #[[ATTR8:[0-9]+]] {
+; CHECK-SAME: (ptr nofree noundef nonnull writeonly align 4 captures(none) dereferenceable(4) [[ARG:%.*]], ptr nofree noundef nonnull writeonly captures(none) dereferenceable(1) [[ARG1:%.*]]) #[[ATTR8:[0-9]+]] {
 ; CHECK-NEXT:    store i32 100, ptr [[ARG]], align 4
 ; CHECK-NEXT:    fence syncscope("singlethread") release
 ; CHECK-NEXT:    store atomic i8 1, ptr [[ARG1]] monotonic, align 1
@@ -300,7 +300,7 @@ define void @foo1_singlethread(ptr %arg, ptr %arg1) {
 define void @bar_singlethread(ptr %arg, ptr %arg1) {
 ; CHECK: Function Attrs: nofree norecurse nosync nounwind
 ; CHECK-LABEL: define {{[^@]+}}@bar_singlethread
-; CHECK-SAME: (ptr nocapture nofree readnone [[ARG:%.*]], ptr nocapture nofree nonnull readonly dereferenceable(1) [[ARG1:%.*]]) #[[ATTR9:[0-9]+]] {
+; CHECK-SAME: (ptr nofree readnone captures(none) [[ARG:%.*]], ptr nofree nonnull readonly captures(none) dereferenceable(1) [[ARG1:%.*]]) #[[ATTR9:[0-9]+]] {
 ; CHECK-NEXT:    br label [[BB2:%.*]]
 ; CHECK:       bb2:
 ; CHECK-NEXT:    [[I3:%.*]] = load atomic i8, ptr [[ARG1]] monotonic, align 1
@@ -334,8 +334,8 @@ declare void @llvm.memset.p0.i32(ptr %dest, i8 %val, i32 %len, i1 %isvolatile)
 define i32 @memcpy_volatile(ptr %ptr1, ptr %ptr2) {
 ; CHECK: Function Attrs: mustprogress nofree norecurse nounwind willreturn memory(argmem: readwrite)
 ; CHECK-LABEL: define {{[^@]+}}@memcpy_volatile
-; CHECK-SAME: (ptr nocapture nofree writeonly [[PTR1:%.*]], ptr nocapture nofree readonly [[PTR2:%.*]]) #[[ATTR12:[0-9]+]] {
-; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr nocapture nofree writeonly [[PTR1]], ptr nocapture nofree readonly [[PTR2]], i32 noundef 8, i1 noundef true) #[[ATTR21:[0-9]+]]
+; CHECK-SAME: (ptr nofree writeonly captures(none) [[PTR1:%.*]], ptr nofree readonly captures(none) [[PTR2:%.*]]) #[[ATTR12:[0-9]+]] {
+; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr nofree writeonly captures(none) [[PTR1]], ptr nofree readonly captures(none) [[PTR2]], i32 noundef 8, i1 noundef true) #[[ATTR21:[0-9]+]]
 ; CHECK-NEXT:    ret i32 4
 ;
   call void @llvm.memcpy.p0.p0.i32(ptr %ptr1, ptr %ptr2, i32 8, i1 true)
@@ -349,8 +349,8 @@ define i32 @memcpy_volatile(ptr %ptr1, ptr %ptr2) {
 define i32 @memset_non_volatile(ptr %ptr1, i8 %val) {
 ; CHECK: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: write)
 ; CHECK-LABEL: define {{[^@]+}}@memset_non_volatile
-; CHECK-SAME: (ptr nocapture nofree writeonly [[PTR1:%.*]], i8 [[VAL:%.*]]) #[[ATTR13:[0-9]+]] {
-; CHECK-NEXT:    call void @llvm.memset.p0.i32(ptr nocapture nofree writeonly [[PTR1]], i8 [[VAL]], i32 noundef 8, i1 noundef false) #[[ATTR22:[0-9]+]]
+; CHECK-SAME: (ptr nofree writeonly captures(none) [[PTR1:%.*]], i8 [[VAL:%.*]]) #[[ATTR13:[0-9]+]] {
+; CHECK-NEXT:    call void @llvm.memset.p0.i32(ptr nofree writeonly captures(none) [[PTR1]], i8 [[VAL]], i32 noundef 8, i1 noundef false) #[[ATTR22:[0-9]+]]
 ; CHECK-NEXT:    ret i32 4
 ;
   call void @llvm.memset.p0.i32(ptr %ptr1, i8 %val, i32 8, i1 false)
diff --git a/llvm/test/Transforms/Attributor/noundef.ll b/llvm/test/Transforms/Attributor/noundef.ll
index 67dcf2680c64a..1855a93449a44 100644
--- a/llvm/test/Transforms/Attributor/noundef.ll
+++ b/llvm/test/Transforms/Attributor/noundef.ll
@@ -39,7 +39,7 @@ define void @caller1() {
 
 define internal void @argument_dead_callback_callee(ptr %c) {
 ; CHECK-LABEL: define {{[^@]+}}@argument_dead_callback_callee
-; CHECK-SAME: (ptr noalias nocapture nofree readnone align 4294967296 [[C:%.*]]) {
+; CHECK-SAME: (ptr noalias nofree readnone align 4294967296 captures(none) [[C:%.*]]) {
 ; CHECK-NEXT:    call void @unknown()
 ; CHECK-NEXT:    ret void
 ;
diff --git a/llvm/test/Transforms/Attributor/openmp_parallel.ll b/llvm/test/Transforms/Attributor/openmp_parallel.ll
index 02636ab926dde..d7b194dd64071 100644
--- a/llvm/test/Transforms/Attributor/openmp_parallel.ll
+++ b/llvm/test/Transforms/Attributor/openmp_parallel.ll
@@ -21,19 +21,19 @@ target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16
 define dso_local void @func(ptr nocapture %a, ptr %b, i32 %N) local_unnamed_addr #0 {
 ; TUNIT: Function Attrs: nounwind uwtable
 ; TUNIT-LABEL: define {{[^@]+}}@func
-; TUNIT-SAME: (ptr nocapture nofree writeonly [[A:%.*]], ptr nocapture nofree readonly [[B:%.*]], i32 [[N:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
+; TUNIT-SAME: (ptr nofree writeonly captures(none) [[A:%.*]], ptr nofree readonly captures(none) [[B:%.*]], i32 [[N:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
 ; TUNIT-NEXT:  entry:
 ; TUNIT-NEXT:    [[A_ADDR:%.*]] = alloca ptr, align 8
 ; TUNIT-NEXT:    [[B_ADDR:%.*]] = alloca ptr, align 8
 ; TUNIT-NEXT:    [[N_ADDR:%.*]] = alloca i32, align 4
 ; TUNIT-NEXT:    store ptr [[A]], ptr [[A_ADDR]], align 8
 ; TUNIT-NEXT:    store ptr [[B]], ptr [[B_ADDR]], align 8
-; TUNIT-NEXT:    call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr noundef nonnull align 8 dereferenceable(24) @[[GLOB2]], i32 noundef 3, ptr noundef nonnull @.omp_outlined., ptr noalias nocapture nofree nonnull readnone align 4 dereferenceable(4) undef, ptr noalias nocapture nofree noundef nonnull readonly align 8 dereferenceable(8) [[A_ADDR]], ptr noalias nocapture nofree noundef nonnull readonly align 8 dereferenceable(8) [[B_ADDR]])
+; TUNIT-NEXT:    call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr noundef nonnull align 8 dereferenceable(24) @[[GLOB2]], i32 noundef 3, ptr noundef nonnull @.omp_outlined., ptr noalias nofree nonnull readnone align 4 captures(none) dereferenceable(4) undef, ptr noalias nofree noundef nonnull readonly align 8 captures(none) dereferenceable(8) [[A_ADDR]], ptr noalias nofree noundef nonnull readonly align 8 captures(none) dereferenceable(8) [[B_ADDR]])
 ; TUNIT-NEXT:    ret void
 ;
 ; CGSCC: Function Attrs: nounwind uwtable
 ; CGSCC-LABEL: define {{[^@]+}}@func
-; CGSCC-SAME: (ptr nocapture nofree [[A:%.*]], ptr nofree [[B:%.*]], i32 [[N:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
+; CGSCC-SAME: (ptr nofree captures(none) [[A:%.*]], ptr nofree [[B:%.*]], i32 [[N:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
 ; CGSCC-NEXT:  entry:
 ; CGSCC-NEXT:    [[A_ADDR:%.*]] = alloca ptr, align 8
 ; CGSCC-NEXT:    [[B_ADDR:%.*]] = alloca ptr, align 8
@@ -41,7 +41,7 @@ define dso_local void @func(ptr nocapture %a, ptr %b, i32 %N) local_unnamed_addr
 ; CGSCC-NEXT:    store ptr [[A]], ptr [[A_ADDR]], align 8
 ; CGSCC-NEXT:    store ptr [[B]], ptr [[B_ADDR]], align 8
 ; CGSCC-NEXT:    store i32 199, ptr [[N_ADDR]], align 4
-; CGSCC-NEXT:    call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr noundef nonnull align 8 dereferenceable(24) @[[GLOB2]], i32 noundef 3, ptr noundef nonnull @.omp_outlined., ptr noalias nocapture nofree noundef nonnull readonly align 4 dereferenceable(4) [[N_ADDR]], ptr noalias nocapture nofree noundef nonnull readonly align 8 dereferenceable(8) [[A_ADDR]], ptr noalias nocapture nofree noundef nonnull readonly align 8 dereferenceable(8) [[B_ADDR]])
+; CGSCC-NEXT:    call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr noundef nonnull align 8 dereferenceable(24) @[[GLOB2]], i32 noundef 3, ptr noundef nonnull @.omp_outlined., ptr noalias nofree noundef nonnull readonly align 4 captures(none) dereferenceable(4) [[N_ADDR]], ptr noalias nofree noundef nonnull readonly align 8 captures(none) dereferenceable(8) [[A_ADDR]], ptr noalias nofree noundef nonnull readonly align 8 captures(none) dereferenceable(8) [[B_ADDR]])
 ; CGSCC-NEXT:    ret void
 ;
 entry:
@@ -60,7 +60,7 @@ entry:
 define internal void @.omp_outlined.(ptr noalias nocapture readonly %.global_tid., ptr noalias nocapture readnone %.bound_tid., ptr nocapture nonnull readonly align 4 dereferenceable(4) %N, ptr nocapture nonnull readonly align 8 dereferenceable(8) %a, ptr nocapture nonnull readonly align 8 dereferenceable(8) %b) #1 {
 ; TUNIT: Function Attrs: alwaysinline nofree norecurse nounwind uwtable
 ; TUNIT-LABEL: define {{[^@]+}}@.omp_outlined.
-; TUNIT-SAME: (ptr noalias nocapture nofree readonly [[DOTGLOBAL_TID_:%.*]], ptr noalias nocapture nofree readnone [[DOTBOUND_TID_:%.*]], ptr noalias nocapture nofree noundef nonnull readnone align 4 dereferenceable(4) [[N:%.*]], ptr noalias nocapture nofree noundef nonnull readonly align 8 dereferenceable(8) [[A:%.*]], ptr noalias nocapture nofree noundef nonnull readonly align 8 dereferenceable(8) [[B:%.*]]) #[[ATTR1:[0-9]+]] {
+; TUNIT-SAME: (ptr noalias nofree readonly captures(none) [[DOTGLOBAL_TID_:%.*]], ptr noalias nofree readnone captures(none) [[DOTBOUND_TID_:%.*]], ptr noalias nofree noundef nonnull readnone align 4 captures(none) dereferenceable(4) [[N:%.*]], ptr noalias nofree noundef nonnull readonly align 8 captures(none) dereferenceable(8) [[A:%.*]], ptr noalias nofree noundef nonnull readonly align 8 captures(none) dereferenceable(8) [[B:%.*]]) #[[ATTR1:[0-9]+]] {
 ; TUNIT-NEXT:  entry:
 ; TUNIT-NEXT:    [[DOTOMP_LB:%.*]] = alloca i32, align 4
 ; TUNIT-NEXT:    [[DOTOMP_UB:%.*]] = alloca i32, align 4
@@ -68,13 +68,13 @@ define internal void @.omp_outlined.(ptr noalias nocapture readonly %.global_tid
 ; TUNIT-NEXT:    [[DOTOMP_IS_LAST:%.*]] = alloca i32, align 4
 ; TUNIT-NEXT:    br label [[OMP_PRECOND_THEN:%.*]]
 ; TUNIT:       omp.precond.then:
-; TUNIT-NEXT:    call void @llvm.lifetime.start.p0(i64 noundef 4, ptr nocapture nofree noundef nonnull align 4 dereferenceable(4) [[DOTOMP_LB]]) #[[ATTR3:[0-9]+]]
+; TUNIT-NEXT:    call void @llvm.lifetime.start.p0(i64 noundef 4, ptr nofree noundef nonnull align 4 captures(none) dereferenceable(4) [[DOTOMP_LB]]) #[[ATTR3:[0-9]+]]
 ; TUNIT-NEXT:    store i32 0, ptr [[DOTOMP_LB]], align 4
-; TUNIT-NEXT:    call void @llvm.lifetime.start.p0(i64 noundef 4, ptr nocapture nofree noundef nonnull align 4 dereferenceable(4) [[DOTOMP_UB]]) #[[ATTR3]]
+; TUNIT-NEXT:    call void @llvm.lifetime.start.p0(i64 noundef 4, ptr nofree noundef nonnull align 4 captures(none) dereferenceable(4) [[DOTOMP_UB]]) #[[ATTR3]]
 ; TUNIT-NEXT:    store i32 197, ptr [[DOTOMP_UB]], align 4
-; TUNIT-NEXT:    call void @llvm.lifetime.start.p0(i64 noundef 4, ptr nocapture nofree noundef nonnull align 4 dereferenceable(4) [[DOTOMP_STRIDE]]) #[[ATTR3]]
+; TUNIT-NEXT:    call void @llvm.lifetime.start.p0(i64 noundef 4, ptr nofree noundef nonnull align 4 captures(none) dereferenceable(4) [[DOTOMP_STRIDE]]) #[[ATTR3]]
 ; TUNIT-NEXT:    store i32 1, ptr [[DOTOMP_STRIDE]], align 4
-; TUNIT-NEXT:    call void @llvm.lifetime.start.p0(i64 noundef 4, ptr nocapture nofree noundef nonnull align 4 dereferenceable(4) [[DOTOMP_IS_LAST]]) #[[ATTR3]]
+; TUNIT-NEXT:    call void @llvm.lifetime.start.p0(i64 noundef 4, ptr nofree noundef nonnull align 4 captures(none) dereferenceable(4) [[DOTOMP_IS_LAST]]) #[[ATTR3]]
 ; TUNIT-NEXT:    store i32 0, ptr [[DOTOMP_IS_LAST]], align 4
 ; TUNIT-NEXT:    [[TMP0:%.*]] = load i32, ptr [[DOTGLOBAL_TID_]], align 4
 ; TUNIT-NEXT:    call void @__kmpc_for_static_init_4(ptr noundef nonnull align 8 dereferenceable(24) @[[GLOB1]], i32 [[TMP0]], i32 noundef 34, ptr noundef nonnull align 4 dereferenceable(4) [[DOTOMP_IS_LAST]], ptr noundef nonnull align 4 dereferenceable(4) [[DOTOMP_LB]], ptr noundef nonnull align 4 dereferenceable(4) [[DOTOMP_UB]], ptr noundef nonnull align 4 dereferenceable(4) [[DOTOMP_STRIDE]], i32 noundef 1, i32 noundef 1)
@@ -103,17 +103,17 @@ define internal void @.omp_outlined.(ptr noalias nocapture readonly %.global_tid
 ; TUNIT-NEXT:    br i1 [[CMP5]], label [[OMP_INNER_FOR_BODY]], label [[OMP_LOOP_EXIT]]
 ; TUNIT:       omp.loop.exit:
 ; TUNIT-NEXT:    call void @__kmpc_for_static_fini(ptr noundef nonnull align 8 dereferenceable(24) @[[GLOB1]], i32 [[TMP0]])
-; TUNIT-NEXT:    call void @llvm.lifetime.end.p0(i64 noundef 4, ptr nocapture nofree noundef nonnull align 4 dereferenceable(4) [[DOTOMP_IS_LAST]])
-; TUNIT-NEXT:    call void @llvm.lifetime.end.p0(i64 noundef 4, ptr nocapture nofree noundef nonnull align 4 dereferenceable(4) [[DOTOMP_STRIDE]])
-; TUNIT-NEXT:    call void @llvm.lifetime.end.p0(i64 noundef 4, ptr nocapture nofree noundef nonnull align 4 dereferenceable(4) [[DOTOMP_UB]])
-; TUNIT-NEXT:    call void @llvm.lifetime.end.p0(i64 noundef 4, ptr nocapture nofree noundef nonnull align 4 dereferenceable(4) [[DOTOMP_LB]])
+; TUNIT-NEXT:    call void @llvm.lifetime.end.p0(i64 noundef 4, ptr nofree noundef nonnull align 4 captures(none) dereferenceable(4) [[DOTOMP_IS_LAST]])
+; TUNIT-NEXT:    call void @llvm.lifetime.end.p0(i64 noundef 4, ptr nofree noundef nonnull align 4 captures(none) dereferenceable(4) [[DOTOMP_STRIDE]])
+; TUNIT-NEXT:    call void @llvm.lifetime.end.p0(i64 noundef 4, ptr nofree noundef nonnull align 4 captures(none) dereferenceable(4) [[DOTOMP_UB]])
+; TUNIT-NEXT:    call void @llvm.lifetime.end.p0(i64 noundef 4, ptr nofree noundef nonnull align 4 captures(none) dereferenceable(4) [[DOTOMP_LB]])
 ; TUNIT-NEXT:    br label [[OMP_PRECOND_END:%.*]]
 ; TUNIT:       omp.precond.end:
 ; TUNIT-NEXT:    ret void
 ;
 ; CGSCC: Function Attrs: alwaysinline nofree norecurse nounwind uwtable
 ; CGSCC-LABEL: define {{[^@]+}}@.omp_outlined.
-; CGSCC-SAME: (ptr noalias nocapture nofree readonly [[DOTGLOBAL_TID_:%.*]], ptr noalias nocapture nofree readnone [[DOTBOUND_TID_:%.*]], ptr noalias nocapture nofree noundef nonnull readonly align 4 dereferenceable(4) [[N:%.*]], ptr noalias nocapture nofree noundef nonnull readonly align 8 dereferenceable(8) [[A:%.*]], ptr noalias nocapture nofree noundef nonnull readonly align 8 dereferenceable(8) [[B:%.*]]) #[[ATTR1:[0-9]+]] {
+; CGSCC-SAME: (ptr noalias nofree readonly captures(none) [[DOTGLOBAL_TID_:%.*]], ptr noalias nofree readnone captures(none) [[DOTBOUND_TID_:%.*]], ptr noalias nofree noundef nonnull readonly align 4 captures(none) dereferenceable(4) [[N:%.*]], ptr noalias nofree noundef nonnull readonly align 8 captures(none) dereferenceable(8) [[A:%.*]], ptr noalias nofree noundef nonnull readonly align 8 captures(none) dereferenceable(8) [[B:%.*]]) #[[ATTR1:[0-9]+]] {
 ; CGSCC-NEXT:  entry:
 ; CGSCC-NEXT:    [[DOTOMP_LB:%.*]] = alloca i32, align 4
 ; CGSCC-NEXT:    [[DOTOMP_UB:%.*]] = alloca i32, align 4
@@ -124,13 +124,13 @@ define internal void @.omp_outlined.(ptr noalias nocapture readonly %.global_tid
 ; CGSCC-NEXT:    [[CMP:%.*]] = icmp sgt i32 [[TMP0]], 1
 ; CGSCC-NEXT:    br i1 [[CMP]], label [[OMP_PRECOND_THEN:%.*]], label [[OMP_PRECOND_END:%.*]]
 ; CGSCC:       omp.precond.then:
-; CGSCC-NEXT:    call void @llvm.lifetime.start.p0(i64 noundef 4, ptr nocapture nofree noundef nonnull align 4 dereferenceable(4) [[DOTOMP_LB]]) #[[ATTR3:[0-9]+]]
+; CGSCC-NEXT:    call void @llvm.lifetime.start.p0(i64 noundef 4, ptr nofree noundef nonnull align 4 captures(none) dereferenceable(4) [[DOTOMP_LB]]) #[[ATTR3:[0-9]+]]
 ; CGSCC-NEXT:    store i32 0, ptr [[DOTOMP_LB]], align 4
-; CGSCC-NEXT:    call void @llvm.lifetime.start.p0(i64 noundef 4, ptr nocapture nofree noundef nonnull align 4 dereferenceable(4) [[DOTOMP_UB]]) #[[ATTR3]]
+; CGSCC-NEXT:    call void @llvm.lifetime.start.p0(i64 noundef 4, ptr nofree noundef nonnull align 4 captures(none) dereferenceable(4) [[DOTOMP_UB]]) #[[ATTR3]]
 ; CGSCC-NEXT:    store i32 [[SUB2]], ptr [[DOTOMP_UB]], align 4
-; CGSCC-NEXT:    call void @llvm.lifetime.start.p0(i64 noundef 4, ptr nocapture nofree noundef nonnull align 4 dereferenceable(4) [[DOTOMP_STRIDE]]) #[[ATTR3]]
+; CGSCC-NEXT:    call void @llvm.lifetime.start.p0(i64 noundef 4, ptr nofree noundef nonnull align 4 captures(none) dereferenceable(4) [[DOTOMP_STRIDE]]) #[[ATTR3]]
 ; CGSCC-NEXT:    store i32 1, ptr [[DOTOMP_STRIDE]], align 4
-; CGSCC-NEXT:    call void @llvm.lifetime.start.p0(i64 noundef 4, ptr nocapture nofree noundef nonnull align 4 dereferenceable(4) [[DOTOMP_IS_LAST]]) #[[ATTR3]]
+; CGSCC-NEXT:    call void @llvm.lifetime.start.p0(i64 noundef 4, ptr nofree noundef nonnull align 4 captures(none) dereferenceable(4) [[DOTOMP_IS_LAST]]) #[[ATTR3]]
 ; CGSCC-NEXT:    store i32 0, ptr [[DOTOMP_IS_LAST]], align 4
 ; CGSCC-NEXT:    [[TMP1:%.*]] = load i32, ptr [[DOTGLOBAL_TID_]], align 4
 ; CGSCC-NEXT:    call void @__kmpc_for_static_init_4(ptr noundef nonnull align 8 dereferenceable(24) @[[GLOB1]], i32 [[TMP1]], i32 noundef 34, ptr noundef nonnull align 4 dereferenceable(4) [[DOTOMP_IS_LAST]], ptr noundef nonnull align 4 dereferenceable(4) [[DOTOMP_LB]], ptr noundef nonnull align 4 dereferenceable(4) [[DOTOMP_UB]], ptr noundef nonnull align 4 dereferenceable(4) [[DOTOMP_STRIDE]], i32 noundef 1, i32 noundef 1)
@@ -159,10 +159,10 @@ define internal void @.omp_outlined.(ptr noalias nocapture readonly %.global_tid
 ; CGSCC-NEXT:    br i1 [[CMP5]], label [[OMP_INNER_FOR_BODY]], label [[OMP_LOOP_EXIT]]
 ; CGSCC:       omp.loop.exit:
 ; CGSCC-NEXT:    call void @__kmpc_for_static_fini(ptr noundef nonnull align 8 dereferenceable(24) @[[GLOB1]], i32 [[TMP1]])
-; CGSCC-NEXT:    call void @llvm.lifetime.end.p0(i64 noundef 4, ptr nocapture nofree noundef nonnull align 4 dereferenceable(4) [[DOTOMP_IS_LAST]])
-; CGSCC-NEXT:    call void @llvm.lifetime.end.p0(i64 noundef 4, ptr nocapture nofree noundef nonnull align 4 dereferenceable(4) [[DOTOMP_STRIDE]])
-; CGSCC-NEXT:    call void @llvm.lifetime.end.p0(i64 noundef 4, ptr nocapture nofree noundef nonnull align 4 dereferenceable(4) [[DOTOMP_UB]])
-; CGSCC-NEXT:    call void @llvm.lifetime.end.p0(i64 noundef 4, ptr nocapture nofree noundef nonnull align 4 dereferenceable(4) [[DOTOMP_LB]])
+; CGSCC-NEXT:    call void @llvm.lifetime.end.p0(i64 noundef 4, ptr nofree noundef nonnull align 4 captures(none) dereferenceable(4) [[DOTOMP_IS_LAST]])
+; CGSCC-NEXT:    call void @llvm.lifetime.end.p0(i64 noundef 4, ptr nofree noundef nonnull align 4 captures(none) dereferenceable(4) [[DOTOMP_STRIDE]])
+; CGSCC-NEXT:    call void @llvm.lifetime.end.p0(i64 noundef 4, ptr nofree noundef nonnull align 4 captures(none) dereferenceable(4) [[DOTOMP_UB]])
+; CGSCC-NEXT:    call void @llvm.lifetime.end.p0(i64 noundef 4, ptr nofree noundef nonnull align 4 captures(none) dereferenceable(4) [[DOTOMP_LB]])
 ; CGSCC-NEXT:    br label [[OMP_PRECOND_END]]
 ; CGSCC:       omp.precond.end:
 ; CGSCC-NEXT:    ret void
diff --git a/llvm/test/Transforms/Attributor/phi_bug_pointer_info.ll b/llvm/test/Transforms/Attributor/phi_bug_pointer_info.ll
index b4a2192c25faa..17262378a5b4d 100644
--- a/llvm/test/Transforms/Attributor/phi_bug_pointer_info.ll
+++ b/llvm/test/Transforms/Attributor/phi_bug_pointer_info.ll
@@ -19,11 +19,23 @@
 ; CHECK:      - c: <unknown>
 ;.
 ; CHECK: @globalBytes = internal global [1024 x i8] zeroinitializer
+; CHECK: @globalBytes = internal global [1024 x i8] zeroinitializer
+; CHECK: @globalBytes = internal global [1024 x i8] zeroinitializer
+; CHECK: @globalBytes = internal global [1024 x i8] zeroinitializer
+; CHECK: @globalBytes = internal global [1024 x i8] zeroinitializer
+; CHECK: @globalBytes = internal global [1024 x i8] zeroinitializer
+; CHECK: @globalBytes = internal global [1024 x i8] zeroinitializer
+; CHECK: @globalBytes = internal global [1024 x i8] zeroinitializer
+; CHECK: @globalBytes = internal global [1024 x i8] zeroinitializer
+; CHECK: @globalBytes = internal global [1024 x i8] zeroinitializer
+; CHECK: @globalBytes = internal global [1024 x i8] zeroinitializer
+; CHECK: @globalBytes = internal global [1024 x i8] zeroinitializer
+; CHECK: @globalBytes = internal global [1024 x i8] zeroinitializer
 ;.
 define dso_local i32 @phi_different_offsets(ptr nocapture %val, ptr nocapture %val2, i1 %cmp) {
 ; CHECK: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn
 ; CHECK-LABEL: define dso_local i32 @phi_different_offsets
-; CHECK-SAME: (ptr nocapture nofree readonly [[VAL:%.*]], ptr nocapture nofree readonly [[VAL2:%.*]], i1 noundef [[CMP:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-SAME: (ptr nofree readonly captures(none) [[VAL:%.*]], ptr nofree readonly captures(none) [[VAL2:%.*]], i1 noundef [[CMP:%.*]]) #[[ATTR0:[0-9]+]] {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    br i1 [[CMP]], label [[THEN:%.*]], label [[ELSE:%.*]]
 ; CHECK:       then:
diff --git a/llvm/test/Transforms/Attributor/pointer-info.ll b/llvm/test/Transforms/Attributor/pointer-info.ll
index 6afdbdaee317c..cd7fd1a4c1123 100644
--- a/llvm/test/Transforms/Attributor/pointer-info.ll
+++ b/llvm/test/Transforms/Attributor/pointer-info.ll
@@ -8,25 +8,25 @@
 define void @foo(ptr %ptr) {
 ; TUNIT: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(none)
 ; TUNIT-LABEL: define {{[^@]+}}@foo
-; TUNIT-SAME: (ptr nocapture nofree readnone [[PTR:%.*]]) #[[ATTR0:[0-9]+]] {
+; TUNIT-SAME: (ptr nofree readnone captures(none) [[PTR:%.*]]) #[[ATTR0:[0-9]+]] {
 ; TUNIT-NEXT:  entry:
 ; TUNIT-NEXT:    [[TMP0:%.*]] = alloca [[STRUCT_TEST_A:%.*]], align 8
 ; TUNIT-NEXT:    br label [[CALL_BR:%.*]]
 ; TUNIT:       call.br:
 ; TUNIT-NEXT:    [[TMP1:%.*]] = getelementptr inbounds [[STRUCT_TEST_A]], ptr [[TMP0]], i64 0, i32 2
-; TUNIT-NEXT:    tail call void @bar(ptr noalias nocapture nofree noundef nonnull readonly byval([[STRUCT_TEST_A]]) align 8 dereferenceable(24) [[TMP0]]) #[[ATTR2:[0-9]+]]
+; TUNIT-NEXT:    tail call void @bar(ptr noalias nofree noundef nonnull readonly byval([[STRUCT_TEST_A]]) align 8 captures(none) dereferenceable(24) [[TMP0]]) #[[ATTR2:[0-9]+]]
 ; TUNIT-NEXT:    ret void
 ;
 ; CGSCC: Function Attrs: mustprogress nofree nosync nounwind willreturn memory(none)
 ; CGSCC-LABEL: define {{[^@]+}}@foo
-; CGSCC-SAME: (ptr nocapture nofree writeonly [[PTR:%.*]]) #[[ATTR0:[0-9]+]] {
+; CGSCC-SAME: (ptr nofree writeonly captures(none) [[PTR:%.*]]) #[[ATTR0:[0-9]+]] {
 ; CGSCC-NEXT:  entry:
 ; CGSCC-NEXT:    [[TMP0:%.*]] = alloca [[STRUCT_TEST_A:%.*]], align 8
 ; CGSCC-NEXT:    br label [[CALL_BR:%.*]]
 ; CGSCC:       call.br:
 ; CGSCC-NEXT:    [[TMP1:%.*]] = getelementptr inbounds [[STRUCT_TEST_A]], ptr [[TMP0]], i64 0, i32 2
 ; CGSCC-NEXT:    store ptr [[PTR]], ptr [[TMP1]], align 8
-; CGSCC-NEXT:    tail call void @bar(ptr noalias nocapture nofree noundef nonnull readnone byval([[STRUCT_TEST_A]]) align 8 dereferenceable(24) [[TMP0]]) #[[ATTR2:[0-9]+]]
+; CGSCC-NEXT:    tail call void @bar(ptr noalias nofree noundef nonnull readnone byval([[STRUCT_TEST_A]]) align 8 captures(none) dereferenceable(24) [[TMP0]]) #[[ATTR2:[0-9]+]]
 ; CGSCC-NEXT:    ret void
 ;
 entry:
@@ -43,7 +43,7 @@ call.br:
 define void @bar(ptr noundef byval(%struct.test.a) align 8 %dev) {
 ; CHECK: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: write)
 ; CHECK-LABEL: define {{[^@]+}}@bar
-; CHECK-SAME: (ptr noalias nocapture nofree noundef nonnull writeonly byval([[STRUCT_TEST_A:%.*]]) align 8 dereferenceable(24) [[DEV:%.*]]) #[[ATTR1:[0-9]+]] {
+; CHECK-SAME: (ptr noalias nofree noundef nonnull writeonly byval([[STRUCT_TEST_A:%.*]]) align 8 captures(none) dereferenceable(24) [[DEV:%.*]]) #[[ATTR1:[0-9]+]] {
 ; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds [[STRUCT_TEST_B:%.*]], ptr [[DEV]], i64 0, i32 1
 ; CHECK-NEXT:    store i32 1, ptr [[TMP1]], align 4
 ; CHECK-NEXT:    ret void
diff --git a/llvm/test/Transforms/Attributor/range.ll b/llvm/test/Transforms/Attributor/range.ll
index 48040fec772dc..1b27fc5f16f56 100644
--- a/llvm/test/Transforms/Attributor/range.ll
+++ b/llvm/test/Transforms/Attributor/range.ll
@@ -7,7 +7,7 @@
 define i32 @test0(ptr %p) {
 ; CHECK: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: read)
 ; CHECK-LABEL: define {{[^@]+}}@test0
-; CHECK-SAME: (ptr nocapture nofree noundef nonnull readonly align 4 dereferenceable(4) [[P:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-SAME: (ptr nofree noundef nonnull readonly align 4 captures(none) dereferenceable(4) [[P:%.*]]) #[[ATTR0:[0-9]+]] {
 ; CHECK-NEXT:    [[A:%.*]] = load i32, ptr [[P]], align 4, !range [[RNG0:![0-9]+]]
 ; CHECK-NEXT:    ret i32 [[A]]
 ;
@@ -18,14 +18,14 @@ define i32 @test0(ptr %p) {
 define i32 @test0-range-check(ptr %p) {
 ; TUNIT: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: read)
 ; TUNIT-LABEL: define {{[^@]+}}@test0-range-check
-; TUNIT-SAME: (ptr nocapture nofree readonly align 4 [[P:%.*]]) #[[ATTR0]] {
-; TUNIT-NEXT:    [[A:%.*]] = tail call i32 @test0(ptr nocapture nofree noundef readonly align 4 [[P]]) #[[ATTR3:[0-9]+]], !range [[RNG0]]
+; TUNIT-SAME: (ptr nofree readonly align 4 captures(none) [[P:%.*]]) #[[ATTR0]] {
+; TUNIT-NEXT:    [[A:%.*]] = tail call i32 @test0(ptr nofree noundef readonly align 4 captures(none) [[P]]) #[[ATTR3:[0-9]+]], !range [[RNG0]]
 ; TUNIT-NEXT:    ret i32 [[A]]
 ;
 ; CGSCC: Function Attrs: mustprogress nofree nosync nounwind willreturn memory(argmem: read)
 ; CGSCC-LABEL: define {{[^@]+}}@test0-range-check
-; CGSCC-SAME: (ptr nocapture nofree noundef nonnull readonly align 4 dereferenceable(4) [[P:%.*]]) #[[ATTR1:[0-9]+]] {
-; CGSCC-NEXT:    [[A:%.*]] = tail call i32 @test0(ptr nocapture nofree noundef nonnull readonly align 4 dereferenceable(4) [[P]]) #[[ATTR5:[0-9]+]]
+; CGSCC-SAME: (ptr nofree noundef nonnull readonly align 4 captures(none) dereferenceable(4) [[P:%.*]]) #[[ATTR1:[0-9]+]] {
+; CGSCC-NEXT:    [[A:%.*]] = tail call i32 @test0(ptr nofree noundef nonnull readonly align 4 captures(none) dereferenceable(4) [[P]]) #[[ATTR5:[0-9]+]]
 ; CGSCC-NEXT:    ret i32 [[A]]
 ;
   %a = tail call i32 @test0(ptr %p)
@@ -47,8 +47,8 @@ define void @use3(i1, i1, i1) {
 define void @test0-icmp-check(ptr %p){
   ; ret = [0, 10)
 ; TUNIT-LABEL: define {{[^@]+}}@test0-icmp-check
-; TUNIT-SAME: (ptr nocapture nofree readonly align 4 [[P:%.*]]) {
-; TUNIT-NEXT:    [[RET:%.*]] = tail call i32 @test0(ptr nocapture nofree noundef readonly align 4 [[P]]) #[[ATTR3]]
+; TUNIT-SAME: (ptr nofree readonly align 4 captures(none) [[P:%.*]]) {
+; TUNIT-NEXT:    [[RET:%.*]] = tail call i32 @test0(ptr nofree noundef readonly align 4 captures(none) [[P]]) #[[ATTR3]], !range [[RNG0]]
 ; TUNIT-NEXT:    [[CMP_EQ_1:%.*]] = icmp eq i32 [[RET]], 10
 ; TUNIT-NEXT:    [[CMP_EQ_2:%.*]] = icmp eq i32 [[RET]], 9
 ; TUNIT-NEXT:    [[CMP_EQ_3:%.*]] = icmp eq i32 [[RET]], 8
@@ -114,8 +114,8 @@ define void @test0-icmp-check(ptr %p){
 ; TUNIT-NEXT:    ret void
 ;
 ; CGSCC-LABEL: define {{[^@]+}}@test0-icmp-check
-; CGSCC-SAME: (ptr nocapture nofree noundef nonnull readonly align 4 dereferenceable(4) [[P:%.*]]) {
-; CGSCC-NEXT:    [[RET:%.*]] = tail call i32 @test0(ptr nocapture nofree noundef nonnull readonly align 4 dereferenceable(4) [[P]]) #[[ATTR5]]
+; CGSCC-SAME: (ptr nofree noundef nonnull readonly align 4 captures(none) dereferenceable(4) [[P:%.*]]) {
+; CGSCC-NEXT:    [[RET:%.*]] = tail call i32 @test0(ptr nofree noundef nonnull readonly align 4 captures(none) dereferenceable(4) [[P]]) #[[ATTR5]]
 ; CGSCC-NEXT:    [[CMP_EQ_1:%.*]] = icmp eq i32 [[RET]], 10
 ; CGSCC-NEXT:    [[CMP_EQ_2:%.*]] = icmp eq i32 [[RET]], 9
 ; CGSCC-NEXT:    [[CMP_EQ_3:%.*]] = icmp eq i32 [[RET]], 8
@@ -267,7 +267,7 @@ define void @test0-icmp-check(ptr %p){
 define i32 @test1(ptr %p) {
 ; CHECK: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: read)
 ; CHECK-LABEL: define {{[^@]+}}@test1
-; CHECK-SAME: (ptr nocapture nofree noundef nonnull readonly align 4 dereferenceable(4) [[P:%.*]]) #[[ATTR0]] {
+; CHECK-SAME: (ptr nofree noundef nonnull readonly align 4 captures(none) dereferenceable(4) [[P:%.*]]) #[[ATTR0]] {
 ; CHECK-NEXT:    [[LOAD_10_100:%.*]] = load i32, ptr [[P]], align 4, !range [[RNG1:![0-9]+]]
 ; CHECK-NEXT:    [[ADD_10_THEN_20_110:%.*]] = add i32 [[LOAD_10_100]], 10
 ; CHECK-NEXT:    [[MUL_10_THEN_200_1091:%.*]] = mul i32 [[ADD_10_THEN_20_110]], 10
@@ -283,15 +283,15 @@ define i1 @test1-check(ptr %p) {
 ;
 ; TUNIT: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: read)
 ; TUNIT-LABEL: define {{[^@]+}}@test1-check
-; TUNIT-SAME: (ptr nocapture nofree readonly align 4 [[P:%.*]]) #[[ATTR0]] {
-; TUNIT-NEXT:    [[RES:%.*]] = tail call i32 @test1(ptr nocapture nofree noundef readonly align 4 [[P]]) #[[ATTR3]], !range [[RNG2:![0-9]+]]
+; TUNIT-SAME: (ptr nofree readonly align 4 captures(none) [[P:%.*]]) #[[ATTR0]] {
+; TUNIT-NEXT:    [[RES:%.*]] = tail call i32 @test1(ptr nofree noundef readonly align 4 captures(none) [[P]]) #[[ATTR3]], !range [[RNG2:![0-9]+]]
 ; TUNIT-NEXT:    [[CMP:%.*]] = icmp eq i32 [[RES]], 500
 ; TUNIT-NEXT:    ret i1 [[CMP]]
 ;
 ; CGSCC: Function Attrs: mustprogress nofree nosync nounwind willreturn memory(argmem: read)
 ; CGSCC-LABEL: define {{[^@]+}}@test1-check
-; CGSCC-SAME: (ptr nocapture nofree noundef nonnull readonly align 4 dereferenceable(4) [[P:%.*]]) #[[ATTR1]] {
-; CGSCC-NEXT:    [[RES:%.*]] = tail call i32 @test1(ptr nocapture nofree noundef nonnull readonly align 4 dereferenceable(4) [[P]]) #[[ATTR5]]
+; CGSCC-SAME: (ptr nofree noundef nonnull readonly align 4 captures(none) dereferenceable(4) [[P:%.*]]) #[[ATTR1]] {
+; CGSCC-NEXT:    [[RES:%.*]] = tail call i32 @test1(ptr nofree noundef nonnull readonly align 4 captures(none) dereferenceable(4) [[P]]) #[[ATTR5]]
 ; CGSCC-NEXT:    [[CMP:%.*]] = icmp eq i32 [[RES]], 500
 ; CGSCC-NEXT:    ret i1 [[CMP]]
 ;
@@ -315,7 +315,7 @@ define i1 @test1-check(ptr %p) {
 define i32 @test2(ptr %p) {
 ; CHECK: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: read)
 ; CHECK-LABEL: define {{[^@]+}}@test2
-; CHECK-SAME: (ptr nocapture nofree noundef nonnull readonly align 4 dereferenceable(4) [[P:%.*]]) #[[ATTR0]] {
+; CHECK-SAME: (ptr nofree noundef nonnull readonly align 4 captures(none) dereferenceable(4) [[P:%.*]]) #[[ATTR0]] {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[TMP0:%.*]] = load i32, ptr [[P]], align 4
 ; CHECK-NEXT:    [[TOBOOL:%.*]] = icmp eq i32 [[TMP0]], 0
@@ -332,7 +332,7 @@ entry:
 define i32 @test2_check(ptr %p) {
 ; TUNIT: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(none)
 ; TUNIT-LABEL: define {{[^@]+}}@test2_check
-; TUNIT-SAME: (ptr nocapture nofree readnone align 4 [[P:%.*]]) #[[ATTR1:[0-9]+]] {
+; TUNIT-SAME: (ptr nofree readnone align 4 captures(none) [[P:%.*]]) #[[ATTR1:[0-9]+]] {
 ; TUNIT-NEXT:  entry:
 ; TUNIT-NEXT:    br label [[IF_THEN:%.*]]
 ; TUNIT:       if.then:
@@ -344,9 +344,9 @@ define i32 @test2_check(ptr %p) {
 ;
 ; CGSCC: Function Attrs: mustprogress nofree nosync nounwind willreturn memory(argmem: read)
 ; CGSCC-LABEL: define {{[^@]+}}@test2_check
-; CGSCC-SAME: (ptr nocapture nofree noundef nonnull readonly align 4 dereferenceable(4) [[P:%.*]]) #[[ATTR1]] {
+; CGSCC-SAME: (ptr nofree noundef nonnull readonly align 4 captures(none) dereferenceable(4) [[P:%.*]]) #[[ATTR1]] {
 ; CGSCC-NEXT:  entry:
-; CGSCC-NEXT:    [[CALL:%.*]] = tail call i32 @test2(ptr nocapture nofree noundef nonnull readonly align 4 dereferenceable(4) [[P]]) #[[ATTR5]]
+; CGSCC-NEXT:    [[CALL:%.*]] = tail call i32 @test2(ptr nofree noundef nonnull readonly align 4 captures(none) dereferenceable(4) [[P]]) #[[ATTR5]]
 ; CGSCC-NEXT:    [[CMP:%.*]] = icmp slt i32 [[CALL]], 5
 ; CGSCC-NEXT:    br i1 [[CMP]], label [[IF_THEN:%.*]], label [[IF_END:%.*]]
 ; CGSCC:       if.then:
@@ -1614,7 +1614,7 @@ define internal i1 @non_zero(i8 %v) {
 define i1 @context(ptr %p) {
 ; TUNIT: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: read)
 ; TUNIT-LABEL: define {{[^@]+}}@context
-; TUNIT-SAME: (ptr nocapture nofree noundef nonnull readonly dereferenceable(1) [[P:%.*]]) #[[ATTR0]] {
+; TUNIT-SAME: (ptr nofree noundef nonnull readonly captures(none) dereferenceable(1) [[P:%.*]]) #[[ATTR0]] {
 ; TUNIT-NEXT:    [[L:%.*]] = load i8, ptr [[P]], align 1
 ; TUNIT-NEXT:    [[C:%.*]] = icmp slt i8 0, [[L]]
 ; TUNIT-NEXT:    br i1 [[C]], label [[T:%.*]], label [[F:%.*]]
@@ -1626,7 +1626,7 @@ define i1 @context(ptr %p) {
 ;
 ; CGSCC: Function Attrs: mustprogress nofree nosync nounwind willreturn memory(argmem: read)
 ; CGSCC-LABEL: define {{[^@]+}}@context
-; CGSCC-SAME: (ptr nocapture nofree noundef nonnull readonly dereferenceable(1) [[P:%.*]]) #[[ATTR1]] {
+; CGSCC-SAME: (ptr nofree noundef nonnull readonly captures(none) dereferenceable(1) [[P:%.*]]) #[[ATTR1]] {
 ; CGSCC-NEXT:    [[L:%.*]] = load i8, ptr [[P]], align 1
 ; CGSCC-NEXT:    [[C:%.*]] = icmp slt i8 0, [[L]]
 ; CGSCC-NEXT:    br i1 [[C]], label [[T:%.*]], label [[F:%.*]]
@@ -1649,7 +1649,7 @@ f:
 
 define void @spam(ptr %arg, ptr %arg1) {
 ; CHECK-LABEL: define {{[^@]+}}@spam
-; CHECK-SAME: (ptr nocapture nofree noundef nonnull readonly align 8 dereferenceable(4) [[ARG:%.*]], ptr nocapture nofree readnone [[ARG1:%.*]]) {
+; CHECK-SAME: (ptr nofree noundef nonnull readonly align 8 captures(none) dereferenceable(4) [[ARG:%.*]], ptr nofree readnone captures(none) [[ARG1:%.*]]) {
 ; CHECK-NEXT:  bb:
 ; CHECK-NEXT:    [[TMP:%.*]] = load i32, ptr [[ARG]], align 8
 ; CHECK-NEXT:    [[TMP2:%.*]] = icmp ult i32 [[TMP]], 4
diff --git a/llvm/test/Transforms/Attributor/read_write_returned_arguments_scc.ll b/llvm/test/Transforms/Attributor/read_write_returned_arguments_scc.ll
index 775c949ca8939..d4bade75b6cbe 100644
--- a/llvm/test/Transforms/Attributor/read_write_returned_arguments_scc.ll
+++ b/llvm/test/Transforms/Attributor/read_write_returned_arguments_scc.ll
@@ -39,7 +39,7 @@ define ptr @external_ret2_nrw(ptr %n0, ptr %r0, ptr %w0) {
 ; TUNIT-NEXT:  entry:
 ; TUNIT-NEXT:    [[CALL:%.*]] = call ptr @internal_ret0_nw(ptr nofree [[N0]], ptr nofree [[W0]]) #[[ATTR3:[0-9]+]]
 ; TUNIT-NEXT:    [[CALL1:%.*]] = call ptr @internal_ret1_rrw(ptr nofree noundef align 4 [[R0]], ptr nofree align 4 [[R0]], ptr nofree [[W0]]) #[[ATTR3]]
-; TUNIT-NEXT:    [[CALL2:%.*]] = call ptr @external_sink_ret2_nrw(ptr nofree [[N0]], ptr nocapture nofree readonly align 4 [[R0]], ptr nofree writeonly "no-capture-maybe-returned" [[W0]]) #[[ATTR3]]
+; TUNIT-NEXT:    [[CALL2:%.*]] = call ptr @external_sink_ret2_nrw(ptr nofree [[N0]], ptr nofree readonly align 4 captures(none) [[R0]], ptr nofree writeonly "no-capture-maybe-returned" [[W0]]) #[[ATTR3]]
 ; TUNIT-NEXT:    [[CALL3:%.*]] = call ptr @internal_ret1_rw(ptr nofree noundef align 4 [[R0]], ptr nofree [[W0]]) #[[ATTR3]]
 ; TUNIT-NEXT:    ret ptr [[W0]]
 ;
@@ -49,7 +49,7 @@ define ptr @external_ret2_nrw(ptr %n0, ptr %r0, ptr %w0) {
 ; CGSCC-NEXT:  entry:
 ; CGSCC-NEXT:    [[CALL:%.*]] = call ptr @internal_ret0_nw(ptr nofree [[N0]], ptr nofree [[W0]]) #[[ATTR2:[0-9]+]]
 ; CGSCC-NEXT:    [[CALL1:%.*]] = call ptr @internal_ret1_rrw(ptr nofree noundef align 4 [[R0]], ptr nofree align 4 [[R0]], ptr nofree [[W0]]) #[[ATTR2]]
-; CGSCC-NEXT:    [[CALL2:%.*]] = call ptr @external_sink_ret2_nrw(ptr nofree [[N0]], ptr nocapture nofree readonly align 4 [[R0]], ptr nofree writeonly [[W0]]) #[[ATTR3:[0-9]+]]
+; CGSCC-NEXT:    [[CALL2:%.*]] = call ptr @external_sink_ret2_nrw(ptr nofree [[N0]], ptr nofree readonly align 4 captures(none) [[R0]], ptr nofree writeonly [[W0]]) #[[ATTR3:[0-9]+]]
 ; CGSCC-NEXT:    [[CALL3:%.*]] = call ptr @internal_ret1_rw(ptr nofree noundef align 4 [[R0]], ptr nofree [[W0]]) #[[ATTR2]]
 ; CGSCC-NEXT:    ret ptr [[W0]]
 ;
@@ -79,8 +79,8 @@ define internal ptr @internal_ret0_nw(ptr %n0, ptr %w0) {
 ; TUNIT-NEXT:    [[CALL:%.*]] = call ptr @internal_ret1_rrw(ptr nofree noundef nonnull align 4 dereferenceable(4) [[R0]], ptr nofree noundef nonnull align 4 dereferenceable(4) [[R1]], ptr nofree nonnull align 4 dereferenceable(4) [[W0]]) #[[ATTR3]]
 ; TUNIT-NEXT:    [[CALL1:%.*]] = call ptr @external_ret2_nrw(ptr nofree nonnull [[N0]], ptr nofree noundef nonnull align 4 dereferenceable(4) [[R0]], ptr nofree nonnull align 4 dereferenceable(4) [[W0]]) #[[ATTR3]]
 ; TUNIT-NEXT:    [[CALL2:%.*]] = call ptr @external_ret2_nrw(ptr nofree nonnull [[N0]], ptr nofree noundef nonnull align 4 dereferenceable(4) [[R1]], ptr nofree nonnull align 4 dereferenceable(4) [[W0]]) #[[ATTR3]]
-; TUNIT-NEXT:    [[CALL3:%.*]] = call ptr @external_sink_ret2_nrw(ptr nofree nonnull [[N0]], ptr nocapture nofree noundef nonnull readonly align 4 dereferenceable(4) [[R0]], ptr nofree nonnull writeonly align 4 dereferenceable(4) "no-capture-maybe-returned" [[W0]]) #[[ATTR3]]
-; TUNIT-NEXT:    [[CALL4:%.*]] = call ptr @external_sink_ret2_nrw(ptr nofree nonnull [[N0]], ptr nocapture nofree noundef nonnull readonly align 4 dereferenceable(4) [[R1]], ptr nofree nonnull writeonly align 4 dereferenceable(4) "no-capture-maybe-returned" [[W0]]) #[[ATTR3]]
+; TUNIT-NEXT:    [[CALL3:%.*]] = call ptr @external_sink_ret2_nrw(ptr nofree nonnull [[N0]], ptr nofree noundef nonnull readonly align 4 captures(none) dereferenceable(4) [[R0]], ptr nofree nonnull writeonly align 4 dereferenceable(4) "no-capture-maybe-returned" [[W0]]) #[[ATTR3]]
+; TUNIT-NEXT:    [[CALL4:%.*]] = call ptr @external_sink_ret2_nrw(ptr nofree nonnull [[N0]], ptr nofree noundef nonnull readonly align 4 captures(none) dereferenceable(4) [[R1]], ptr nofree nonnull writeonly align 4 dereferenceable(4) "no-capture-maybe-returned" [[W0]]) #[[ATTR3]]
 ; TUNIT-NEXT:    [[CALL5:%.*]] = call ptr @internal_ret0_nw(ptr nofree nonnull [[N0]], ptr nofree nonnull align 4 dereferenceable(4) [[W0]]) #[[ATTR3]]
 ; TUNIT-NEXT:    br label [[RETURN]]
 ; TUNIT:       return:
@@ -104,8 +104,8 @@ define internal ptr @internal_ret0_nw(ptr %n0, ptr %w0) {
 ; CGSCC-NEXT:    [[CALL:%.*]] = call ptr @internal_ret1_rrw(ptr nofree noundef nonnull align 4 dereferenceable(4) [[R0]], ptr nofree noundef nonnull align 4 dereferenceable(4) [[R1]], ptr nofree nonnull align 4 dereferenceable(4) [[W0]]) #[[ATTR2]]
 ; CGSCC-NEXT:    [[CALL1:%.*]] = call ptr @external_ret2_nrw(ptr nofree nonnull [[N0]], ptr nofree noundef nonnull align 4 dereferenceable(4) [[R0]], ptr nofree nonnull align 4 dereferenceable(4) [[W0]]) #[[ATTR2]]
 ; CGSCC-NEXT:    [[CALL2:%.*]] = call ptr @external_ret2_nrw(ptr nofree nonnull [[N0]], ptr nofree noundef nonnull align 4 dereferenceable(4) [[R1]], ptr nofree nonnull align 4 dereferenceable(4) [[W0]]) #[[ATTR2]]
-; CGSCC-NEXT:    [[CALL3:%.*]] = call ptr @external_sink_ret2_nrw(ptr nofree nonnull [[N0]], ptr nocapture nofree noundef nonnull readonly align 4 dereferenceable(4) [[R0]], ptr nofree nonnull writeonly align 4 dereferenceable(4) [[W0]]) #[[ATTR4:[0-9]+]]
-; CGSCC-NEXT:    [[CALL4:%.*]] = call ptr @external_sink_ret2_nrw(ptr nofree nonnull [[N0]], ptr nocapture nofree noundef nonnull readonly align 4 dereferenceable(4) [[R1]], ptr nofree nonnull writeonly align 4 dereferenceable(4) [[W0]]) #[[ATTR4]]
+; CGSCC-NEXT:    [[CALL3:%.*]] = call ptr @external_sink_ret2_nrw(ptr nofree nonnull [[N0]], ptr nofree noundef nonnull readonly align 4 captures(none) dereferenceable(4) [[R0]], ptr nofree nonnull writeonly align 4 dereferenceable(4) [[W0]]) #[[ATTR4:[0-9]+]]
+; CGSCC-NEXT:    [[CALL4:%.*]] = call ptr @external_sink_ret2_nrw(ptr nofree nonnull [[N0]], ptr nofree noundef nonnull readonly align 4 captures(none) dereferenceable(4) [[R1]], ptr nofree nonnull writeonly align 4 dereferenceable(4) [[W0]]) #[[ATTR4]]
 ; CGSCC-NEXT:    [[CALL5:%.*]] = call ptr @internal_ret0_nw(ptr nofree nonnull [[N0]], ptr nofree nonnull align 4 dereferenceable(4) [[W0]]) #[[ATTR2]]
 ; CGSCC-NEXT:    br label [[RETURN]]
 ; CGSCC:       return:
@@ -159,8 +159,8 @@ define internal ptr @internal_ret1_rrw(ptr %r0, ptr %r1, ptr %w0) {
 ; TUNIT-NEXT:    [[CALL3:%.*]] = call ptr @internal_ret0_nw(ptr nofree nonnull align 4 dereferenceable(4) [[W0]], ptr nofree nonnull align 4 dereferenceable(4) [[W0]]) #[[ATTR3]]
 ; TUNIT-NEXT:    [[CALL4:%.*]] = call ptr @external_ret2_nrw(ptr nofree noundef nonnull align 4 dereferenceable(4) [[R0]], ptr nofree nonnull align 4 dereferenceable(4) [[R1]], ptr nofree nonnull align 4 dereferenceable(4) [[W0]]) #[[ATTR3]]
 ; TUNIT-NEXT:    [[CALL5:%.*]] = call ptr @external_ret2_nrw(ptr nofree nonnull align 4 dereferenceable(4) [[R1]], ptr nofree noundef nonnull align 4 dereferenceable(4) [[R0]], ptr nofree nonnull align 4 dereferenceable(4) [[W0]]) #[[ATTR3]]
-; TUNIT-NEXT:    [[CALL6:%.*]] = call ptr @external_sink_ret2_nrw(ptr nofree noundef nonnull align 4 dereferenceable(4) [[R0]], ptr nocapture nofree nonnull readonly align 4 dereferenceable(4) [[R1]], ptr nofree nonnull writeonly align 4 dereferenceable(4) "no-capture-maybe-returned" [[W0]]) #[[ATTR3]]
-; TUNIT-NEXT:    [[CALL7:%.*]] = call ptr @external_sink_ret2_nrw(ptr nofree nonnull align 4 dereferenceable(4) [[R1]], ptr nocapture nofree noundef nonnull readonly align 4 dereferenceable(4) [[R0]], ptr nofree nonnull writeonly align 4 dereferenceable(4) "no-capture-maybe-returned" [[W0]]) #[[ATTR3]]
+; TUNIT-NEXT:    [[CALL6:%.*]] = call ptr @external_sink_ret2_nrw(ptr nofree noundef nonnull align 4 dereferenceable(4) [[R0]], ptr nofree nonnull readonly align 4 captures(none) dereferenceable(4) [[R1]], ptr nofree nonnull writeonly align 4 dereferenceable(4) "no-capture-maybe-returned" [[W0]]) #[[ATTR3]]
+; TUNIT-NEXT:    [[CALL7:%.*]] = call ptr @external_sink_ret2_nrw(ptr nofree nonnull align 4 dereferenceable(4) [[R1]], ptr nofree noundef nonnull readonly align 4 captures(none) dereferenceable(4) [[R0]], ptr nofree nonnull writeonly align 4 dereferenceable(4) "no-capture-maybe-returned" [[W0]]) #[[ATTR3]]
 ; TUNIT-NEXT:    [[CALL8:%.*]] = call ptr @internal_ret0_nw(ptr nofree nonnull align 4 dereferenceable(4) [[R1]], ptr nofree nonnull align 4 dereferenceable(4) [[W0]]) #[[ATTR3]]
 ; TUNIT-NEXT:    br label [[RETURN]]
 ; TUNIT:       return:
@@ -187,8 +187,8 @@ define internal ptr @internal_ret1_rrw(ptr %r0, ptr %r1, ptr %w0) {
 ; CGSCC-NEXT:    [[CALL3:%.*]] = call ptr @internal_ret0_nw(ptr nofree nonnull align 4 dereferenceable(4) [[W0]], ptr nofree nonnull align 4 dereferenceable(4) [[W0]]) #[[ATTR2]]
 ; CGSCC-NEXT:    [[CALL4:%.*]] = call ptr @external_ret2_nrw(ptr nofree noundef nonnull align 4 dereferenceable(4) [[R0]], ptr nofree nonnull align 4 dereferenceable(4) [[R1]], ptr nofree nonnull align 4 dereferenceable(4) [[W0]]) #[[ATTR2]]
 ; CGSCC-NEXT:    [[CALL5:%.*]] = call ptr @external_ret2_nrw(ptr nofree nonnull align 4 dereferenceable(4) [[R1]], ptr nofree noundef nonnull align 4 dereferenceable(4) [[R0]], ptr nofree nonnull align 4 dereferenceable(4) [[W0]]) #[[ATTR2]]
-; CGSCC-NEXT:    [[CALL6:%.*]] = call ptr @external_sink_ret2_nrw(ptr nofree noundef nonnull align 4 dereferenceable(4) [[R0]], ptr nocapture nofree nonnull readonly align 4 dereferenceable(4) [[R1]], ptr nofree nonnull writeonly align 4 dereferenceable(4) [[W0]]) #[[ATTR4]]
-; CGSCC-NEXT:    [[CALL7:%.*]] = call ptr @external_sink_ret2_nrw(ptr nofree nonnull align 4 dereferenceable(4) [[R1]], ptr nocapture nofree noundef nonnull readonly align 4 dereferenceable(4) [[R0]], ptr nofree nonnull writeonly align 4 dereferenceable(4) [[W0]]) #[[ATTR4]]
+; CGSCC-NEXT:    [[CALL6:%.*]] = call ptr @external_sink_ret2_nrw(ptr nofree noundef nonnull align 4 dereferenceable(4) [[R0]], ptr nofree nonnull readonly align 4 captures(none) dereferenceable(4) [[R1]], ptr nofree nonnull writeonly align 4 dereferenceable(4) [[W0]]) #[[ATTR4]]
+; CGSCC-NEXT:    [[CALL7:%.*]] = call ptr @external_sink_ret2_nrw(ptr nofree nonnull align 4 dereferenceable(4) [[R1]], ptr nofree noundef nonnull readonly align 4 captures(none) dereferenceable(4) [[R0]], ptr nofree nonnull writeonly align 4 dereferenceable(4) [[W0]]) #[[ATTR4]]
 ; CGSCC-NEXT:    [[CALL8:%.*]] = call ptr @internal_ret0_nw(ptr nofree nonnull align 4 dereferenceable(4) [[R1]], ptr nofree nonnull align 4 dereferenceable(4) [[W0]]) #[[ATTR2]]
 ; CGSCC-NEXT:    br label [[RETURN]]
 ; CGSCC:       return:
@@ -227,7 +227,7 @@ return:                                           ; preds = %if.end, %if.then
 define ptr @external_sink_ret2_nrw(ptr %n0, ptr %r0, ptr %w0) {
 ; CHECK: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: readwrite)
 ; CHECK-LABEL: define {{[^@]+}}@external_sink_ret2_nrw
-; CHECK-SAME: (ptr nofree [[N0:%.*]], ptr nocapture nofree readonly [[R0:%.*]], ptr nofree returned writeonly "no-capture-maybe-returned" [[W0:%.*]]) #[[ATTR1:[0-9]+]] {
+; CHECK-SAME: (ptr nofree [[N0:%.*]], ptr nofree readonly captures(none) [[R0:%.*]], ptr nofree returned writeonly "no-capture-maybe-returned" [[W0:%.*]]) #[[ATTR1:[0-9]+]] {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[TOBOOL:%.*]] = icmp ne ptr [[N0]], null
 ; CHECK-NEXT:    br i1 [[TOBOOL]], label [[IF_END:%.*]], label [[IF_THEN:%.*]]
@@ -272,7 +272,7 @@ define internal ptr @internal_ret1_rw(ptr %r0, ptr %w0) {
 ; TUNIT-NEXT:    store i32 [[TMP1]], ptr [[W0]], align 4
 ; TUNIT-NEXT:    [[CALL1:%.*]] = call ptr @internal_ret0_nw(ptr nofree noundef nonnull align 4 dereferenceable(4) [[R0]], ptr nofree nonnull align 4 dereferenceable(4) [[W0]]) #[[ATTR3]]
 ; TUNIT-NEXT:    [[CALL2:%.*]] = call ptr @internal_ret0_nw(ptr nofree nonnull align 4 dereferenceable(4) [[W0]], ptr nofree nonnull align 4 dereferenceable(4) [[W0]]) #[[ATTR3]]
-; TUNIT-NEXT:    [[CALL3:%.*]] = call ptr @external_sink_ret2_nrw(ptr nofree noundef nonnull align 4 dereferenceable(4) [[R0]], ptr nocapture nofree noundef nonnull readonly align 4 dereferenceable(4) [[R0]], ptr nofree nonnull writeonly align 4 dereferenceable(4) "no-capture-maybe-returned" [[W0]]) #[[ATTR3]]
+; TUNIT-NEXT:    [[CALL3:%.*]] = call ptr @external_sink_ret2_nrw(ptr nofree noundef nonnull align 4 dereferenceable(4) [[R0]], ptr nofree noundef nonnull readonly align 4 captures(none) dereferenceable(4) [[R0]], ptr nofree nonnull writeonly align 4 dereferenceable(4) "no-capture-maybe-returned" [[W0]]) #[[ATTR3]]
 ; TUNIT-NEXT:    [[CALL4:%.*]] = call ptr @external_ret2_nrw(ptr nofree noundef nonnull align 4 dereferenceable(4) [[R0]], ptr nofree noundef nonnull align 4 dereferenceable(4) [[R0]], ptr nofree nonnull align 4 dereferenceable(4) [[W0]]) #[[ATTR3]]
 ; TUNIT-NEXT:    br label [[RETURN]]
 ; TUNIT:       return:
@@ -294,7 +294,7 @@ define internal ptr @internal_ret1_rw(ptr %r0, ptr %w0) {
 ; CGSCC-NEXT:    store i32 [[TMP1]], ptr [[W0]], align 4
 ; CGSCC-NEXT:    [[CALL1:%.*]] = call ptr @internal_ret0_nw(ptr nofree noundef nonnull align 4 dereferenceable(4) [[R0]], ptr nofree nonnull align 4 dereferenceable(4) [[W0]]) #[[ATTR2]]
 ; CGSCC-NEXT:    [[CALL2:%.*]] = call ptr @internal_ret0_nw(ptr nofree nonnull align 4 dereferenceable(4) [[W0]], ptr nofree nonnull align 4 dereferenceable(4) [[W0]]) #[[ATTR2]]
-; CGSCC-NEXT:    [[CALL3:%.*]] = call ptr @external_sink_ret2_nrw(ptr nofree noundef nonnull align 4 dereferenceable(4) [[R0]], ptr nocapture nofree noundef nonnull readonly align 4 dereferenceable(4) [[R0]], ptr nofree nonnull writeonly align 4 dereferenceable(4) [[W0]]) #[[ATTR4]]
+; CGSCC-NEXT:    [[CALL3:%.*]] = call ptr @external_sink_ret2_nrw(ptr nofree noundef nonnull align 4 dereferenceable(4) [[R0]], ptr nofree noundef nonnull readonly align 4 captures(none) dereferenceable(4) [[R0]], ptr nofree nonnull writeonly align 4 dereferenceable(4) [[W0]]) #[[ATTR4]]
 ; CGSCC-NEXT:    [[CALL4:%.*]] = call ptr @external_ret2_nrw(ptr nofree noundef nonnull align 4 dereferenceable(4) [[R0]], ptr nofree noundef nonnull align 4 dereferenceable(4) [[R0]], ptr nofree nonnull align 4 dereferenceable(4) [[W0]]) #[[ATTR2]]
 ; CGSCC-NEXT:    br label [[RETURN]]
 ; CGSCC:       return:
@@ -329,7 +329,7 @@ define ptr @external_source_ret2_nrw(ptr %n0, ptr %r0, ptr %w0) {
 ; TUNIT-LABEL: define {{[^@]+}}@external_source_ret2_nrw
 ; TUNIT-SAME: (ptr nofree [[N0:%.*]], ptr nofree [[R0:%.*]], ptr nofree returned [[W0:%.*]]) #[[ATTR2:[0-9]+]] {
 ; TUNIT-NEXT:  entry:
-; TUNIT-NEXT:    [[CALL:%.*]] = call ptr @external_sink_ret2_nrw(ptr nofree [[N0]], ptr nocapture nofree readonly [[R0]], ptr nofree writeonly "no-capture-maybe-returned" [[W0]]) #[[ATTR4:[0-9]+]]
+; TUNIT-NEXT:    [[CALL:%.*]] = call ptr @external_sink_ret2_nrw(ptr nofree [[N0]], ptr nofree readonly captures(none) [[R0]], ptr nofree writeonly "no-capture-maybe-returned" [[W0]]) #[[ATTR4:[0-9]+]]
 ; TUNIT-NEXT:    [[CALL1:%.*]] = call ptr @external_ret2_nrw(ptr nofree [[N0]], ptr nofree [[R0]], ptr nofree [[W0]]) #[[ATTR3]]
 ; TUNIT-NEXT:    ret ptr [[W0]]
 ;
@@ -337,7 +337,7 @@ define ptr @external_source_ret2_nrw(ptr %n0, ptr %r0, ptr %w0) {
 ; CGSCC-LABEL: define {{[^@]+}}@external_source_ret2_nrw
 ; CGSCC-SAME: (ptr nofree [[N0:%.*]], ptr nofree [[R0:%.*]], ptr nofree [[W0:%.*]]) #[[ATTR0]] {
 ; CGSCC-NEXT:  entry:
-; CGSCC-NEXT:    [[CALL:%.*]] = call ptr @external_sink_ret2_nrw(ptr nofree [[N0]], ptr nocapture nofree readonly [[R0]], ptr nofree writeonly [[W0]]) #[[ATTR5:[0-9]+]]
+; CGSCC-NEXT:    [[CALL:%.*]] = call ptr @external_sink_ret2_nrw(ptr nofree [[N0]], ptr nofree readonly captures(none) [[R0]], ptr nofree writeonly [[W0]]) #[[ATTR5:[0-9]+]]
 ; CGSCC-NEXT:    [[CALL1:%.*]] = call ptr @external_ret2_nrw(ptr nofree [[N0]], ptr nofree [[R0]], ptr nofree [[W0]]) #[[ATTR3]]
 ; CGSCC-NEXT:    ret ptr [[CALL1]]
 ;
diff --git a/llvm/test/Transforms/Attributor/readattrs.ll b/llvm/test/Transforms/Attributor/readattrs.ll
index 54afdebb3a896..30cb0854d6478 100644
--- a/llvm/test/Transforms/Attributor/readattrs.ll
+++ b/llvm/test/Transforms/Attributor/readattrs.ll
@@ -52,8 +52,8 @@ declare void @test4_1(ptr nocapture) readonly
 define void @test4_2(ptr %p) {
 ; CHECK: Function Attrs: nosync memory(read)
 ; CHECK-LABEL: define {{[^@]+}}@test4_2
-; CHECK-SAME: (ptr nocapture readonly [[P:%.*]]) #[[ATTR3:[0-9]+]] {
-; CHECK-NEXT:    call void @test4_1(ptr nocapture readonly [[P]]) #[[ATTR3]]
+; CHECK-SAME: (ptr readonly captures(none) [[P:%.*]]) #[[ATTR3:[0-9]+]] {
+; CHECK-NEXT:    call void @test4_1(ptr readonly captures(none) [[P]]) #[[ATTR3]]
 ; CHECK-NEXT:    ret void
 ;
   call void @test4_1(ptr %p)
@@ -64,7 +64,7 @@ define void @test4_2(ptr %p) {
 define void @test5(ptr %p, ptr %q) {
 ; CHECK: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: write)
 ; CHECK-LABEL: define {{[^@]+}}@test5
-; CHECK-SAME: (ptr nocapture nofree noundef nonnull writeonly align 8 dereferenceable(8) [[P:%.*]], ptr nofree writeonly [[Q:%.*]]) #[[ATTR4:[0-9]+]] {
+; CHECK-SAME: (ptr nofree noundef nonnull writeonly align 8 captures(none) dereferenceable(8) [[P:%.*]], ptr nofree writeonly [[Q:%.*]]) #[[ATTR4:[0-9]+]] {
 ; CHECK-NEXT:    store ptr [[Q]], ptr [[P]], align 8
 ; CHECK-NEXT:    ret void
 ;
@@ -76,7 +76,7 @@ declare void @test6_1()
 ; This is not a missed optz'n.
 define void @test6_2(ptr %p, ptr %q) {
 ; CHECK-LABEL: define {{[^@]+}}@test6_2
-; CHECK-SAME: (ptr nocapture nofree noundef nonnull writeonly align 8 dereferenceable(8) [[P:%.*]], ptr nofree [[Q:%.*]]) {
+; CHECK-SAME: (ptr nofree noundef nonnull writeonly align 8 captures(none) dereferenceable(8) [[P:%.*]], ptr nofree [[Q:%.*]]) {
 ; CHECK-NEXT:    store ptr [[Q]], ptr [[P]], align 8
 ; CHECK-NEXT:    call void @test6_1()
 ; CHECK-NEXT:    ret void
@@ -90,7 +90,7 @@ define void @test6_2(ptr %p, ptr %q) {
 define void @test7_1(ptr inalloca(i32) %a) {
 ; CHECK: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(none)
 ; CHECK-LABEL: define {{[^@]+}}@test7_1
-; CHECK-SAME: (ptr nocapture nofree nonnull writeonly inalloca(i32) dereferenceable(4) [[A:%.*]]) #[[ATTR1]] {
+; CHECK-SAME: (ptr nofree nonnull writeonly inalloca(i32) captures(none) dereferenceable(4) [[A:%.*]]) #[[ATTR1]] {
 ; CHECK-NEXT:    ret void
 ;
   ret void
@@ -110,7 +110,7 @@ entry:
 define void @test8_2(ptr %p) {
 ; TUNIT: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: write)
 ; TUNIT-LABEL: define {{[^@]+}}@test8_2
-; TUNIT-SAME: (ptr nocapture nofree writeonly [[P:%.*]]) #[[ATTR4]] {
+; TUNIT-SAME: (ptr nofree writeonly captures(none) [[P:%.*]]) #[[ATTR4]] {
 ; TUNIT-NEXT:  entry:
 ; TUNIT-NEXT:    store i32 10, ptr [[P]], align 4
 ; TUNIT-NEXT:    ret void
@@ -232,7 +232,7 @@ declare void @escape_readonly_ptr(ptr %addr, ptr readonly %ptr)
 ;
 define void @unsound_readnone(ptr %ignored, ptr %escaped_then_written) {
 ; CHECK-LABEL: define {{[^@]+}}@unsound_readnone
-; CHECK-SAME: (ptr nocapture nofree readnone [[IGNORED:%.*]], ptr nofree [[ESCAPED_THEN_WRITTEN:%.*]]) {
+; CHECK-SAME: (ptr nofree readnone captures(none) [[IGNORED:%.*]], ptr nofree [[ESCAPED_THEN_WRITTEN:%.*]]) {
 ; CHECK-NEXT:    [[ADDR:%.*]] = alloca ptr, align 8
 ; CHECK-NEXT:    call void @escape_readnone_ptr(ptr noundef nonnull align 8 dereferenceable(8) [[ADDR]], ptr noalias nofree readnone [[ESCAPED_THEN_WRITTEN]])
 ; CHECK-NEXT:    [[ADDR_LD:%.*]] = load ptr, ptr [[ADDR]], align 8
@@ -248,7 +248,7 @@ define void @unsound_readnone(ptr %ignored, ptr %escaped_then_written) {
 
 define void @unsound_readonly(ptr %ignored, ptr %escaped_then_written) {
 ; CHECK-LABEL: define {{[^@]+}}@unsound_readonly
-; CHECK-SAME: (ptr nocapture nofree readnone [[IGNORED:%.*]], ptr nofree [[ESCAPED_THEN_WRITTEN:%.*]]) {
+; CHECK-SAME: (ptr nofree readnone captures(none) [[IGNORED:%.*]], ptr nofree [[ESCAPED_THEN_WRITTEN:%.*]]) {
 ; CHECK-NEXT:    [[ADDR:%.*]] = alloca ptr, align 8
 ; CHECK-NEXT:    call void @escape_readonly_ptr(ptr noundef nonnull align 8 dereferenceable(8) [[ADDR]], ptr nofree readonly [[ESCAPED_THEN_WRITTEN]])
 ; CHECK-NEXT:    [[ADDR_LD:%.*]] = load ptr, ptr [[ADDR]], align 8
@@ -281,7 +281,7 @@ define void @byval_not_readonly_1(ptr byval(i8) %written) readonly {
 define void @byval_not_readonly_2(ptr byval(i8) %written) readonly {
 ; CHECK: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(none)
 ; CHECK-LABEL: define {{[^@]+}}@byval_not_readonly_2
-; CHECK-SAME: (ptr noalias nocapture nofree noundef nonnull writeonly byval(i8) dereferenceable(1) [[WRITTEN:%.*]]) #[[ATTR1]] {
+; CHECK-SAME: (ptr noalias nofree noundef nonnull writeonly byval(i8) captures(none) dereferenceable(1) [[WRITTEN:%.*]]) #[[ATTR1]] {
 ; CHECK-NEXT:    store i8 0, ptr [[WRITTEN]], align 1
 ; CHECK-NEXT:    ret void
 ;
@@ -309,7 +309,7 @@ define void @byval_not_readnone_1(ptr byval(i8) %written) readnone {
 define void @byval_not_readnone_2(ptr byval(i8) %written) readnone {
 ; CHECK: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(none)
 ; CHECK-LABEL: define {{[^@]+}}@byval_not_readnone_2
-; CHECK-SAME: (ptr noalias nocapture nofree noundef nonnull writeonly byval(i8) dereferenceable(1) [[WRITTEN:%.*]]) #[[ATTR1]] {
+; CHECK-SAME: (ptr noalias nofree noundef nonnull writeonly byval(i8) captures(none) dereferenceable(1) [[WRITTEN:%.*]]) #[[ATTR1]] {
 ; CHECK-NEXT:    store i8 0, ptr [[WRITTEN]], align 1
 ; CHECK-NEXT:    ret void
 ;
@@ -320,7 +320,7 @@ define void @byval_not_readnone_2(ptr byval(i8) %written) readnone {
 define void @byval_no_fnarg(ptr byval(i8) %written) {
 ; CHECK: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: write)
 ; CHECK-LABEL: define {{[^@]+}}@byval_no_fnarg
-; CHECK-SAME: (ptr noalias nocapture nofree noundef nonnull writeonly byval(i8) dereferenceable(1) [[WRITTEN:%.*]]) #[[ATTR4]] {
+; CHECK-SAME: (ptr noalias nofree noundef nonnull writeonly byval(i8) captures(none) dereferenceable(1) [[WRITTEN:%.*]]) #[[ATTR4]] {
 ; CHECK-NEXT:    store i8 0, ptr [[WRITTEN]], align 1
 ; CHECK-NEXT:    ret void
 ;
@@ -331,18 +331,18 @@ define void @byval_no_fnarg(ptr byval(i8) %written) {
 define void @testbyval(ptr %read_only) {
 ; TUNIT: Function Attrs: nosync
 ; TUNIT-LABEL: define {{[^@]+}}@testbyval
-; TUNIT-SAME: (ptr nocapture nonnull readonly [[READ_ONLY:%.*]]) #[[ATTR13:[0-9]+]] {
-; TUNIT-NEXT:    call void @byval_not_readonly_1(ptr noalias nocapture nonnull readonly byval(i8) [[READ_ONLY]]) #[[ATTR3]]
-; TUNIT-NEXT:    call void @byval_not_readnone_1(ptr noalias nocapture nonnull readnone byval(i8) [[READ_ONLY]]) #[[ATTR13]]
-; TUNIT-NEXT:    call void @byval_no_fnarg(ptr noalias nocapture nofree noundef nonnull readonly byval(i8) [[READ_ONLY]]) #[[ATTR19:[0-9]+]]
+; TUNIT-SAME: (ptr nonnull readonly captures(none) [[READ_ONLY:%.*]]) #[[ATTR13:[0-9]+]] {
+; TUNIT-NEXT:    call void @byval_not_readonly_1(ptr noalias nonnull readonly byval(i8) captures(none) [[READ_ONLY]]) #[[ATTR3]]
+; TUNIT-NEXT:    call void @byval_not_readnone_1(ptr noalias nonnull readnone byval(i8) captures(none) [[READ_ONLY]]) #[[ATTR13]]
+; TUNIT-NEXT:    call void @byval_no_fnarg(ptr noalias nofree noundef nonnull readonly byval(i8) captures(none) [[READ_ONLY]]) #[[ATTR19:[0-9]+]]
 ; TUNIT-NEXT:    ret void
 ;
 ; CGSCC: Function Attrs: nosync
 ; CGSCC-LABEL: define {{[^@]+}}@testbyval
-; CGSCC-SAME: (ptr nocapture noundef nonnull readonly dereferenceable(1) [[READ_ONLY:%.*]]) #[[ATTR14:[0-9]+]] {
-; CGSCC-NEXT:    call void @byval_not_readonly_1(ptr noalias nocapture noundef nonnull readonly byval(i8) dereferenceable(1) [[READ_ONLY]]) #[[ATTR2:[0-9]+]]
-; CGSCC-NEXT:    call void @byval_not_readnone_1(ptr noalias nocapture noundef nonnull readnone byval(i8) dereferenceable(1) [[READ_ONLY]]) #[[ATTR14]]
-; CGSCC-NEXT:    call void @byval_no_fnarg(ptr noalias nocapture nofree noundef nonnull readnone byval(i8) dereferenceable(1) [[READ_ONLY]]) #[[ATTR21:[0-9]+]]
+; CGSCC-SAME: (ptr noundef nonnull readonly captures(none) dereferenceable(1) [[READ_ONLY:%.*]]) #[[ATTR14:[0-9]+]] {
+; CGSCC-NEXT:    call void @byval_not_readonly_1(ptr noalias noundef nonnull readonly byval(i8) captures(none) dereferenceable(1) [[READ_ONLY]]) #[[ATTR2:[0-9]+]]
+; CGSCC-NEXT:    call void @byval_not_readnone_1(ptr noalias noundef nonnull readnone byval(i8) captures(none) dereferenceable(1) [[READ_ONLY]]) #[[ATTR14]]
+; CGSCC-NEXT:    call void @byval_no_fnarg(ptr noalias nofree noundef nonnull readnone byval(i8) captures(none) dereferenceable(1) [[READ_ONLY]]) #[[ATTR21:[0-9]+]]
 ; CGSCC-NEXT:    ret void
 ;
   call void @byval_not_readonly_1(ptr byval(i8) %read_only)
@@ -361,14 +361,14 @@ declare void @val_use(i8 %ptr) readonly nounwind
 define void @ptr_uses(ptr %ptr) {
 ; TUNIT: Function Attrs: nosync nounwind memory(read)
 ; TUNIT-LABEL: define {{[^@]+}}@ptr_uses
-; TUNIT-SAME: (ptr nocapture nofree readonly [[PTR:%.*]]) #[[ATTR15]] {
+; TUNIT-SAME: (ptr nofree readonly captures(none) [[PTR:%.*]]) #[[ATTR15]] {
 ; TUNIT-NEXT:    [[CALL_PTR:%.*]] = call ptr @maybe_returned_ptr(ptr nofree readonly [[PTR]]) #[[ATTR15]]
 ; TUNIT-NEXT:    [[CALL_VAL:%.*]] = call i8 @maybe_returned_val(ptr readonly [[CALL_PTR]]) #[[ATTR15]]
 ; TUNIT-NEXT:    ret void
 ;
 ; CGSCC: Function Attrs: nosync nounwind memory(read)
 ; CGSCC-LABEL: define {{[^@]+}}@ptr_uses
-; CGSCC-SAME: (ptr nocapture nofree readonly [[PTR:%.*]]) #[[ATTR16]] {
+; CGSCC-SAME: (ptr nofree readonly captures(none) [[PTR:%.*]]) #[[ATTR16]] {
 ; CGSCC-NEXT:    [[CALL_PTR:%.*]] = call ptr @maybe_returned_ptr(ptr nofree readonly [[PTR]]) #[[ATTR16]]
 ; CGSCC-NEXT:    [[CALL_VAL:%.*]] = call i8 @maybe_returned_val(ptr readonly [[CALL_PTR]]) #[[ATTR16]]
 ; CGSCC-NEXT:    ret void
diff --git a/llvm/test/Transforms/Attributor/recursive_globals.ll b/llvm/test/Transforms/Attributor/recursive_globals.ll
index b2b27b15c7eea..f06b65baef9d2 100644
--- a/llvm/test/Transforms/Attributor/recursive_globals.ll
+++ b/llvm/test/Transforms/Attributor/recursive_globals.ll
@@ -27,7 +27,7 @@ define internal void @fnc2() {
 
 define dso_local void @indr_caller(ptr %0) {
 ; CHECK-LABEL: define dso_local void @indr_caller(
-; CHECK-SAME: ptr nocapture nofree noundef nonnull [[TMP0:%.*]]) {
+; CHECK-SAME: ptr nofree noundef nonnull captures(none) [[TMP0:%.*]]) {
 ; CHECK-NEXT:    call void [[TMP0]]()
 ; CHECK-NEXT:    ret void
 ;
@@ -37,8 +37,8 @@ define dso_local void @indr_caller(ptr %0) {
 
 define void @main() {
 ; CHECK-LABEL: define void @main() {
-; CHECK-NEXT:    call void @indr_caller(ptr nocapture nofree noundef nonnull @fnc1)
-; CHECK-NEXT:    call void @indr_caller(ptr nocapture nofree noundef nonnull @fnc2)
+; CHECK-NEXT:    call void @indr_caller(ptr nofree noundef nonnull captures(none) @fnc1)
+; CHECK-NEXT:    call void @indr_caller(ptr nofree noundef nonnull captures(none) @fnc2)
 ; CHECK-NEXT:    ret void
 ;
   call void @indr_caller(ptr @fnc1)
diff --git a/llvm/test/Transforms/Attributor/reduced/aapointer_info_map_invalidation.ll b/llvm/test/Transforms/Attributor/reduced/aapointer_info_map_invalidation.ll
index a7622143c2e73..9a5f789e88fa9 100644
--- a/llvm/test/Transforms/Attributor/reduced/aapointer_info_map_invalidation.ll
+++ b/llvm/test/Transforms/Attributor/reduced/aapointer_info_map_invalidation.ll
@@ -8,7 +8,7 @@ define amdgpu_kernel void @__omp_offloading_fd00_2c00523__ZN11qmcplusplus7ompBLA
 ; CHECK-NEXT:    [[TMP1:%.*]] = alloca [0 x [0 x float]], i32 0, align 8, addrspace(5)
 ; CHECK-NEXT:    [[TMP2:%.*]] = addrspacecast ptr addrspace(5) [[TMP1]] to ptr
 ; CHECK-NEXT:    store ptr [[TMP2]], ptr addrspace(5) [[TMP1]], align 8
-; CHECK-NEXT:    [[TMP3:%.*]] = call fastcc i32 @__kmpc_nvptx_parallel_reduce_nowait_v2(ptr nocapture nofree noundef readonly align 8 dereferenceable_or_null(8) [[TMP2]], i1 noundef false)
+; CHECK-NEXT:    [[TMP3:%.*]] = call fastcc i32 @__kmpc_nvptx_parallel_reduce_nowait_v2(ptr nofree noundef readonly align 8 captures(none) dereferenceable_or_null(8) [[TMP2]], i1 noundef false)
 ; CHECK-NEXT:    ret void
 ;
   %1 = alloca [0 x [0 x float]], i32 0, align 8, addrspace(5)
@@ -21,7 +21,7 @@ define amdgpu_kernel void @__omp_offloading_fd00_2c00523__ZN11qmcplusplus7ompBLA
 define fastcc i32 @__kmpc_nvptx_parallel_reduce_nowait_v2(ptr %0, i1 %1) {
 ; CHECK: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn
 ; CHECK-LABEL: define {{[^@]+}}@__kmpc_nvptx_parallel_reduce_nowait_v2
-; CHECK-SAME: (ptr nocapture nofree nonnull readonly align 8 dereferenceable(8) [[TMP0:%.*]], i1 noundef [[TMP1:%.*]]) #[[ATTR0]] {
+; CHECK-SAME: (ptr nofree nonnull readonly align 8 captures(none) dereferenceable(8) [[TMP0:%.*]], i1 noundef [[TMP1:%.*]]) #[[ATTR0]] {
 ; CHECK-NEXT:    br i1 [[TMP1]], label [[TMP3:%.*]], label [[TMP79:%.*]]
 ; CHECK:       3:
 ; CHECK-NEXT:    br i1 [[TMP1]], label [[TMP4:%.*]], label [[TMP6:%.*]]
diff --git a/llvm/test/Transforms/Attributor/reduced/clear_cached_analysis_for_deleted_functions.ll b/llvm/test/Transforms/Attributor/reduced/clear_cached_analysis_for_deleted_functions.ll
index b3e3b2497194c..03149f2652a6e 100644
--- a/llvm/test/Transforms/Attributor/reduced/clear_cached_analysis_for_deleted_functions.ll
+++ b/llvm/test/Transforms/Attributor/reduced/clear_cached_analysis_for_deleted_functions.ll
@@ -12,7 +12,7 @@ target triple = "x86_64-unknown-linux-gnu"
 define i32 @clause_LiteralComputeWeight(ptr %call23) {
 ; CGSCC: Function Attrs: nofree noreturn nounwind
 ; CGSCC-LABEL: define {{[^@]+}}@clause_LiteralComputeWeight
-; CGSCC-SAME: (ptr nocapture nofree nonnull readonly align 8 dereferenceable(8) [[CALL23:%.*]]) #[[ATTR0:[0-9]+]] {
+; CGSCC-SAME: (ptr nofree nonnull readonly align 8 captures(none) dereferenceable(8) [[CALL23:%.*]]) #[[ATTR0:[0-9]+]] {
 ; CGSCC-NEXT:  entry:
 ; CGSCC-NEXT:    br label [[DO_BODY:%.*]]
 ; CGSCC:       do.body:
diff --git a/llvm/test/Transforms/Attributor/reduced/missed_cached_entry_for_intra_reachability.ll b/llvm/test/Transforms/Attributor/reduced/missed_cached_entry_for_intra_reachability.ll
index 188939db5074f..bc56865338f6a 100644
--- a/llvm/test/Transforms/Attributor/reduced/missed_cached_entry_for_intra_reachability.ll
+++ b/llvm/test/Transforms/Attributor/reduced/missed_cached_entry_for_intra_reachability.ll
@@ -15,7 +15,7 @@
 define void @widget(ptr %arg1, float %arg2, i64 %idx1, i64 %idx2, i32 %limit) {
 ; CHECK: Function Attrs: nofree norecurse nounwind
 ; CHECK-LABEL: define {{[^@]+}}@widget
-; CHECK-SAME: (ptr nocapture nofree writeonly [[ARG1:%.*]], float [[ARG2:%.*]], i64 [[IDX1:%.*]], i64 [[IDX2:%.*]], i32 [[LIMIT:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-SAME: (ptr nofree writeonly captures(none) [[ARG1:%.*]], float [[ARG2:%.*]], i64 [[IDX1:%.*]], i64 [[IDX2:%.*]], i32 [[LIMIT:%.*]]) #[[ATTR0:[0-9]+]] {
 ; CHECK-NEXT:  bb:
 ; CHECK-NEXT:    [[ALLOCA:%.*]] = alloca [128 x float], i32 0, align 4
 ; CHECK-NEXT:    [[ALLOCA3:%.*]] = alloca [81 x float], i32 0, align 4
diff --git a/llvm/test/Transforms/Attributor/reduced/openmp_opt_constant_type_crash.ll b/llvm/test/Transforms/Attributor/reduced/openmp_opt_constant_type_crash.ll
index 518ed97f42bc1..f54fb45793df2 100644
--- a/llvm/test/Transforms/Attributor/reduced/openmp_opt_constant_type_crash.ll
+++ b/llvm/test/Transforms/Attributor/reduced/openmp_opt_constant_type_crash.ll
@@ -114,7 +114,7 @@ cond.end:                                         ; preds = %cond.true, %entry
 ;
 ; CHECK: Function Attrs: norecurse nosync nounwind memory(argmem: write)
 ; CHECK-LABEL: define {{[^@]+}}@_ZN6Kokkos4Impl14SubviewExtentsILj2ELj1EE3setIJLm0ELm0EEJiEEEbjjRKNS0_13ViewDimensionIJXspT_EEEENS0_5ALL_tEDpT0_.internalized
-; CHECK-SAME: (ptr nocapture writeonly [[THIS:%.*]]) #[[ATTR1]] {
+; CHECK-SAME: (ptr writeonly captures(none) [[THIS:%.*]]) #[[ATTR1]] {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[TMP0:%.*]] = addrspacecast ptr [[THIS]] to ptr addrspace(5)
 ; CHECK-NEXT:    ret i1 false
diff --git a/llvm/test/Transforms/Attributor/returned.ll b/llvm/test/Transforms/Attributor/returned.ll
index bc55a50f0e6f7..32a8a16b75883 100644
--- a/llvm/test/Transforms/Attributor/returned.ll
+++ b/llvm/test/Transforms/Attributor/returned.ll
@@ -338,16 +338,16 @@ entry:
 define ptr @ptr_scc_r1(ptr %a, ptr %r, ptr %b) #0 {
 ; TUNIT: Function Attrs: nofree noinline nosync nounwind memory(none) uwtable
 ; TUNIT-LABEL: define {{[^@]+}}@ptr_scc_r1
-; TUNIT-SAME: (ptr nocapture nofree readnone [[A:%.*]], ptr nofree readnone returned "no-capture-maybe-returned" [[R:%.*]], ptr nocapture nofree readnone [[B:%.*]]) #[[ATTR1]] {
+; TUNIT-SAME: (ptr nofree readnone captures(none) [[A:%.*]], ptr nofree readnone returned "no-capture-maybe-returned" [[R:%.*]], ptr nofree readnone captures(none) [[B:%.*]]) #[[ATTR1]] {
 ; TUNIT-NEXT:  entry:
-; TUNIT-NEXT:    [[CALL1:%.*]] = call ptr @ptr_scc_r2(ptr noalias nocapture nofree readnone [[R]], ptr noalias nocapture nofree readnone [[A]], ptr noalias nofree readnone "no-capture-maybe-returned" [[R]]) #[[ATTR8]]
+; TUNIT-NEXT:    [[CALL1:%.*]] = call ptr @ptr_scc_r2(ptr noalias nofree readnone captures(none) [[R]], ptr noalias nofree readnone captures(none) [[A]], ptr noalias nofree readnone "no-capture-maybe-returned" [[R]]) #[[ATTR8]]
 ; TUNIT-NEXT:    ret ptr [[R]]
 ;
 ; CGSCC: Function Attrs: nofree noinline nosync nounwind memory(none) uwtable
 ; CGSCC-LABEL: define {{[^@]+}}@ptr_scc_r1
-; CGSCC-SAME: (ptr nocapture nofree readnone [[A:%.*]], ptr nofree readnone returned [[R:%.*]], ptr nocapture nofree readnone [[B:%.*]]) #[[ATTR1]] {
+; CGSCC-SAME: (ptr nofree readnone captures(none) [[A:%.*]], ptr nofree readnone returned [[R:%.*]], ptr nofree readnone captures(none) [[B:%.*]]) #[[ATTR1]] {
 ; CGSCC-NEXT:  entry:
-; CGSCC-NEXT:    [[CALL1:%.*]] = call ptr @ptr_scc_r2(ptr noalias nocapture nofree readnone [[R]], ptr noalias nocapture nofree readnone [[A]], ptr noalias nofree readnone [[R]]) #[[ATTR7]]
+; CGSCC-NEXT:    [[CALL1:%.*]] = call ptr @ptr_scc_r2(ptr noalias nofree readnone captures(none) [[R]], ptr noalias nofree readnone captures(none) [[A]], ptr noalias nofree readnone [[R]]) #[[ATTR7]]
 ; CGSCC-NEXT:    ret ptr [[R]]
 ;
 entry:
@@ -359,23 +359,23 @@ entry:
 define ptr @ptr_scc_r2(ptr %a, ptr %b, ptr %r) #0 {
 ; TUNIT: Function Attrs: nofree noinline nosync nounwind memory(none) uwtable
 ; TUNIT-LABEL: define {{[^@]+}}@ptr_scc_r2
-; TUNIT-SAME: (ptr nocapture nofree readnone [[A:%.*]], ptr nocapture nofree readnone [[B:%.*]], ptr nofree readnone returned "no-capture-maybe-returned" [[R:%.*]]) #[[ATTR1]] {
+; TUNIT-SAME: (ptr nofree readnone captures(none) [[A:%.*]], ptr nofree readnone captures(none) [[B:%.*]], ptr nofree readnone returned "no-capture-maybe-returned" [[R:%.*]]) #[[ATTR1]] {
 ; TUNIT-NEXT:  entry:
 ; TUNIT-NEXT:    [[CMP:%.*]] = icmp ugt ptr [[A]], [[B]]
 ; TUNIT-NEXT:    br i1 [[CMP]], label [[IF_THEN:%.*]], label [[IF_END:%.*]]
 ; TUNIT:       if.then:
-; TUNIT-NEXT:    [[CALL1:%.*]] = call ptr @ptr_scc_r2(ptr noalias nocapture nofree readnone [[B]], ptr noalias nocapture nofree nonnull readnone [[A]], ptr noalias nofree readnone "no-capture-maybe-returned" [[R]]) #[[ATTR8]]
+; TUNIT-NEXT:    [[CALL1:%.*]] = call ptr @ptr_scc_r2(ptr noalias nofree readnone captures(none) [[B]], ptr noalias nofree nonnull readnone captures(none) [[A]], ptr noalias nofree readnone "no-capture-maybe-returned" [[R]]) #[[ATTR8]]
 ; TUNIT-NEXT:    br label [[RETURN:%.*]]
 ; TUNIT:       if.end:
 ; TUNIT-NEXT:    [[CMP2:%.*]] = icmp ult ptr [[A]], [[B]]
 ; TUNIT-NEXT:    br i1 [[CMP2]], label [[IF_THEN3:%.*]], label [[IF_END12:%.*]]
 ; TUNIT:       if.then3:
-; TUNIT-NEXT:    [[CALL5:%.*]] = call ptr @ptr_scc_r1(ptr noalias nocapture nofree readnone [[A]], ptr noalias nofree nonnull readnone "no-capture-maybe-returned" [[B]], ptr noalias nocapture nofree readnone undef) #[[ATTR8]]
-; TUNIT-NEXT:    [[CALL6:%.*]] = call ptr @ptr_scc_r2(ptr noalias nocapture nofree readnone [[R]], ptr noalias nocapture nofree readnone [[R]], ptr noalias nofree readnone "no-capture-maybe-returned" [[R]]) #[[ATTR8]]
-; TUNIT-NEXT:    [[CALL7:%.*]] = call ptr @ptr_scc_r1(ptr noalias nocapture nofree readnone [[A]], ptr noalias nofree readnone "no-capture-maybe-returned" [[R]], ptr noalias nocapture nofree readnone undef) #[[ATTR8]]
-; TUNIT-NEXT:    [[CALL8:%.*]] = call ptr @ptr_scc_r2(ptr noalias nocapture nofree readnone [[A]], ptr noalias nocapture nofree nonnull readnone [[B]], ptr noalias nofree readnone "no-capture-maybe-returned" [[R]]) #[[ATTR8]]
-; TUNIT-NEXT:    [[CALL9:%.*]] = call ptr @ptr_scc_r2(ptr noalias nocapture nofree readnone [[B]], ptr noalias nocapture nofree readnone [[R]], ptr noalias nofree readnone "no-capture-maybe-returned" [[R]]) #[[ATTR8]]
-; TUNIT-NEXT:    [[CALL11:%.*]] = call ptr @ptr_scc_r1(ptr noalias nocapture nofree readnone [[B]], ptr noalias nofree readnone "no-capture-maybe-returned" [[R]], ptr noalias nocapture nofree readnone undef) #[[ATTR8]]
+; TUNIT-NEXT:    [[CALL5:%.*]] = call ptr @ptr_scc_r1(ptr noalias nofree readnone captures(none) [[A]], ptr noalias nofree nonnull readnone "no-capture-maybe-returned" [[B]], ptr noalias nofree readnone captures(none) undef) #[[ATTR8]]
+; TUNIT-NEXT:    [[CALL6:%.*]] = call ptr @ptr_scc_r2(ptr noalias nofree readnone captures(none) [[R]], ptr noalias nofree readnone captures(none) [[R]], ptr noalias nofree readnone "no-capture-maybe-returned" [[R]]) #[[ATTR8]]
+; TUNIT-NEXT:    [[CALL7:%.*]] = call ptr @ptr_scc_r1(ptr noalias nofree readnone captures(none) [[A]], ptr noalias nofree readnone "no-capture-maybe-returned" [[R]], ptr noalias nofree readnone captures(none) undef) #[[ATTR8]]
+; TUNIT-NEXT:    [[CALL8:%.*]] = call ptr @ptr_scc_r2(ptr noalias nofree readnone captures(none) [[A]], ptr noalias nofree nonnull readnone captures(none) [[B]], ptr noalias nofree readnone "no-capture-maybe-returned" [[R]]) #[[ATTR8]]
+; TUNIT-NEXT:    [[CALL9:%.*]] = call ptr @ptr_scc_r2(ptr noalias nofree readnone captures(none) [[B]], ptr noalias nofree readnone captures(none) [[R]], ptr noalias nofree readnone "no-capture-maybe-returned" [[R]]) #[[ATTR8]]
+; TUNIT-NEXT:    [[CALL11:%.*]] = call ptr @ptr_scc_r1(ptr noalias nofree readnone captures(none) [[B]], ptr noalias nofree readnone "no-capture-maybe-returned" [[R]], ptr noalias nofree readnone captures(none) undef) #[[ATTR8]]
 ; TUNIT-NEXT:    br label [[RETURN]]
 ; TUNIT:       if.end12:
 ; TUNIT-NEXT:    [[CMP13:%.*]] = icmp eq ptr [[A]], [[B]]
@@ -383,7 +383,7 @@ define ptr @ptr_scc_r2(ptr %a, ptr %b, ptr %r) #0 {
 ; TUNIT:       cond.true:
 ; TUNIT-NEXT:    br label [[COND_END:%.*]]
 ; TUNIT:       cond.false:
-; TUNIT-NEXT:    [[CALL14:%.*]] = call ptr @ptr_scc_r2(ptr noalias nocapture nofree readnone [[A]], ptr noalias nocapture nofree readnone [[B]], ptr noalias nofree readnone "no-capture-maybe-returned" [[R]]) #[[ATTR8]]
+; TUNIT-NEXT:    [[CALL14:%.*]] = call ptr @ptr_scc_r2(ptr noalias nofree readnone captures(none) [[A]], ptr noalias nofree readnone captures(none) [[B]], ptr noalias nofree readnone "no-capture-maybe-returned" [[R]]) #[[ATTR8]]
 ; TUNIT-NEXT:    br label [[COND_END]]
 ; TUNIT:       cond.end:
 ; TUNIT-NEXT:    [[COND:%.*]] = phi ptr [ [[R]], [[COND_TRUE]] ], [ [[R]], [[COND_FALSE]] ]
@@ -394,23 +394,23 @@ define ptr @ptr_scc_r2(ptr %a, ptr %b, ptr %r) #0 {
 ;
 ; CGSCC: Function Attrs: nofree noinline nosync nounwind memory(none) uwtable
 ; CGSCC-LABEL: define {{[^@]+}}@ptr_scc_r2
-; CGSCC-SAME: (ptr nocapture nofree readnone [[A:%.*]], ptr nocapture nofree readnone [[B:%.*]], ptr nofree readnone returned [[R:%.*]]) #[[ATTR1]] {
+; CGSCC-SAME: (ptr nofree readnone captures(none) [[A:%.*]], ptr nofree readnone captures(none) [[B:%.*]], ptr nofree readnone returned [[R:%.*]]) #[[ATTR1]] {
 ; CGSCC-NEXT:  entry:
 ; CGSCC-NEXT:    [[CMP:%.*]] = icmp ugt ptr [[A]], [[B]]
 ; CGSCC-NEXT:    br i1 [[CMP]], label [[IF_THEN:%.*]], label [[IF_END:%.*]]
 ; CGSCC:       if.then:
-; CGSCC-NEXT:    [[CALL1:%.*]] = call ptr @ptr_scc_r2(ptr noalias nocapture nofree readnone [[B]], ptr noalias nocapture nofree nonnull readnone [[A]], ptr noalias nofree readnone [[R]]) #[[ATTR7]]
+; CGSCC-NEXT:    [[CALL1:%.*]] = call ptr @ptr_scc_r2(ptr noalias nofree readnone captures(none) [[B]], ptr noalias nofree nonnull readnone captures(none) [[A]], ptr noalias nofree readnone [[R]]) #[[ATTR7]]
 ; CGSCC-NEXT:    br label [[RETURN:%.*]]
 ; CGSCC:       if.end:
 ; CGSCC-NEXT:    [[CMP2:%.*]] = icmp ult ptr [[A]], [[B]]
 ; CGSCC-NEXT:    br i1 [[CMP2]], label [[IF_THEN3:%.*]], label [[IF_END12:%.*]]
 ; CGSCC:       if.then3:
-; CGSCC-NEXT:    [[CALL5:%.*]] = call ptr @ptr_scc_r1(ptr noalias nocapture nofree readnone [[A]], ptr noalias nofree nonnull readnone [[B]], ptr noalias nocapture nofree readnone undef) #[[ATTR7]]
-; CGSCC-NEXT:    [[CALL6:%.*]] = call ptr @ptr_scc_r2(ptr noalias nocapture nofree readnone [[R]], ptr noalias nocapture nofree readnone [[R]], ptr noalias nofree readnone [[R]]) #[[ATTR7]]
-; CGSCC-NEXT:    [[CALL7:%.*]] = call ptr @ptr_scc_r1(ptr noalias nocapture nofree readnone [[A]], ptr noalias nofree readnone [[R]], ptr noalias nocapture nofree readnone undef) #[[ATTR7]]
-; CGSCC-NEXT:    [[CALL8:%.*]] = call ptr @ptr_scc_r2(ptr noalias nocapture nofree readnone [[A]], ptr noalias nocapture nofree nonnull readnone [[B]], ptr noalias nofree readnone [[R]]) #[[ATTR7]]
-; CGSCC-NEXT:    [[CALL9:%.*]] = call ptr @ptr_scc_r2(ptr noalias nocapture nofree readnone [[B]], ptr noalias nocapture nofree readnone [[R]], ptr noalias nofree readnone [[R]]) #[[ATTR7]]
-; CGSCC-NEXT:    [[CALL11:%.*]] = call ptr @ptr_scc_r1(ptr noalias nocapture nofree nonnull readnone [[B]], ptr noalias nofree readnone [[R]], ptr noalias nocapture nofree readnone undef) #[[ATTR7]]
+; CGSCC-NEXT:    [[CALL5:%.*]] = call ptr @ptr_scc_r1(ptr noalias nofree readnone captures(none) [[A]], ptr noalias nofree nonnull readnone [[B]], ptr noalias nofree readnone captures(none) undef) #[[ATTR7]]
+; CGSCC-NEXT:    [[CALL6:%.*]] = call ptr @ptr_scc_r2(ptr noalias nofree readnone captures(none) [[R]], ptr noalias nofree readnone captures(none) [[R]], ptr noalias nofree readnone [[R]]) #[[ATTR7]]
+; CGSCC-NEXT:    [[CALL7:%.*]] = call ptr @ptr_scc_r1(ptr noalias nofree readnone captures(none) [[A]], ptr noalias nofree readnone [[R]], ptr noalias nofree readnone captures(none) undef) #[[ATTR7]]
+; CGSCC-NEXT:    [[CALL8:%.*]] = call ptr @ptr_scc_r2(ptr noalias nofree readnone captures(none) [[A]], ptr noalias nofree nonnull readnone captures(none) [[B]], ptr noalias nofree readnone [[R]]) #[[ATTR7]]
+; CGSCC-NEXT:    [[CALL9:%.*]] = call ptr @ptr_scc_r2(ptr noalias nofree readnone captures(none) [[B]], ptr noalias nofree readnone captures(none) [[R]], ptr noalias nofree readnone [[R]]) #[[ATTR7]]
+; CGSCC-NEXT:    [[CALL11:%.*]] = call ptr @ptr_scc_r1(ptr noalias nofree nonnull readnone captures(none) [[B]], ptr noalias nofree readnone [[R]], ptr noalias nofree readnone captures(none) undef) #[[ATTR7]]
 ; CGSCC-NEXT:    br label [[RETURN]]
 ; CGSCC:       if.end12:
 ; CGSCC-NEXT:    [[CMP13:%.*]] = icmp eq ptr [[A]], [[B]]
@@ -418,7 +418,7 @@ define ptr @ptr_scc_r2(ptr %a, ptr %b, ptr %r) #0 {
 ; CGSCC:       cond.true:
 ; CGSCC-NEXT:    br label [[COND_END:%.*]]
 ; CGSCC:       cond.false:
-; CGSCC-NEXT:    [[CALL14:%.*]] = call ptr @ptr_scc_r2(ptr noalias nocapture nofree readnone [[A]], ptr noalias nocapture nofree readnone [[B]], ptr noalias nofree readnone [[R]]) #[[ATTR7]]
+; CGSCC-NEXT:    [[CALL14:%.*]] = call ptr @ptr_scc_r2(ptr noalias nofree readnone captures(none) [[A]], ptr noalias nofree readnone captures(none) [[B]], ptr noalias nofree readnone [[R]]) #[[ATTR7]]
 ; CGSCC-NEXT:    br label [[COND_END]]
 ; CGSCC:       cond.end:
 ; CGSCC-NEXT:    [[COND:%.*]] = phi ptr [ [[R]], [[COND_TRUE]] ], [ [[R]], [[COND_FALSE]] ]
@@ -510,13 +510,13 @@ entry:
 define ptr @rt1(ptr %a) #0 {
 ; TUNIT: Function Attrs: mustprogress nofree noinline nosync nounwind willreturn memory(none) uwtable
 ; TUNIT-LABEL: define {{[^@]+}}@rt1
-; TUNIT-SAME: (ptr nocapture nofree nonnull readnone align 4 dereferenceable(4) [[A:%.*]]) #[[ATTR4:[0-9]+]] {
+; TUNIT-SAME: (ptr nofree nonnull readnone align 4 captures(none) dereferenceable(4) [[A:%.*]]) #[[ATTR4:[0-9]+]] {
 ; TUNIT-NEXT:  entry:
 ; TUNIT-NEXT:    ret ptr undef
 ;
 ; CGSCC: Function Attrs: mustprogress nofree noinline nosync nounwind willreturn memory(none) uwtable
 ; CGSCC-LABEL: define {{[^@]+}}@rt1
-; CGSCC-SAME: (ptr nocapture nofree nonnull readnone align 4 dereferenceable(4) [[A:%.*]]) #[[ATTR3:[0-9]+]] {
+; CGSCC-SAME: (ptr nofree nonnull readnone align 4 captures(none) dereferenceable(4) [[A:%.*]]) #[[ATTR3:[0-9]+]] {
 ; CGSCC-NEXT:  entry:
 ; CGSCC-NEXT:    ret ptr undef
 ;
@@ -595,16 +595,16 @@ if.end:
 define ptr @rt3_helper(ptr %a, ptr %b) #0 {
 ; TUNIT: Function Attrs: nofree noinline nosync nounwind memory(none) uwtable
 ; TUNIT-LABEL: define {{[^@]+}}@rt3_helper
-; TUNIT-SAME: (ptr nocapture nofree readnone [[A:%.*]], ptr nofree readnone returned "no-capture-maybe-returned" [[B:%.*]]) #[[ATTR1]] {
+; TUNIT-SAME: (ptr nofree readnone captures(none) [[A:%.*]], ptr nofree readnone returned "no-capture-maybe-returned" [[B:%.*]]) #[[ATTR1]] {
 ; TUNIT-NEXT:  entry:
-; TUNIT-NEXT:    [[CALL:%.*]] = call ptr @rt3(ptr noalias nocapture nofree readnone [[A]], ptr noalias nofree readnone "no-capture-maybe-returned" [[B]]) #[[ATTR8]]
+; TUNIT-NEXT:    [[CALL:%.*]] = call ptr @rt3(ptr noalias nofree readnone captures(none) [[A]], ptr noalias nofree readnone "no-capture-maybe-returned" [[B]]) #[[ATTR8]]
 ; TUNIT-NEXT:    ret ptr [[B]]
 ;
 ; CGSCC: Function Attrs: nofree noinline nosync nounwind memory(none) uwtable
 ; CGSCC-LABEL: define {{[^@]+}}@rt3_helper
-; CGSCC-SAME: (ptr nocapture nofree readnone [[A:%.*]], ptr nofree readnone returned "no-capture-maybe-returned" [[B:%.*]]) #[[ATTR1]] {
+; CGSCC-SAME: (ptr nofree readnone captures(none) [[A:%.*]], ptr nofree readnone returned "no-capture-maybe-returned" [[B:%.*]]) #[[ATTR1]] {
 ; CGSCC-NEXT:  entry:
-; CGSCC-NEXT:    [[CALL:%.*]] = call ptr @rt3(ptr noalias nocapture nofree readnone [[A]], ptr noalias nofree readnone "no-capture-maybe-returned" [[B]]) #[[ATTR7]]
+; CGSCC-NEXT:    [[CALL:%.*]] = call ptr @rt3(ptr noalias nofree readnone captures(none) [[A]], ptr noalias nofree readnone "no-capture-maybe-returned" [[B]]) #[[ATTR7]]
 ; CGSCC-NEXT:    ret ptr [[B]]
 ;
 entry:
@@ -615,12 +615,12 @@ entry:
 define ptr @rt3(ptr %a, ptr %b) #0 {
 ; TUNIT: Function Attrs: nofree noinline nosync nounwind memory(none) uwtable
 ; TUNIT-LABEL: define {{[^@]+}}@rt3
-; TUNIT-SAME: (ptr nocapture nofree readnone [[A:%.*]], ptr nofree readnone returned "no-capture-maybe-returned" [[B:%.*]]) #[[ATTR1]] {
+; TUNIT-SAME: (ptr nofree readnone captures(none) [[A:%.*]], ptr nofree readnone returned "no-capture-maybe-returned" [[B:%.*]]) #[[ATTR1]] {
 ; TUNIT-NEXT:  entry:
 ; TUNIT-NEXT:    [[CMP:%.*]] = icmp eq ptr [[A]], null
 ; TUNIT-NEXT:    br i1 [[CMP]], label [[IF_THEN:%.*]], label [[IF_END:%.*]]
 ; TUNIT:       if.then:
-; TUNIT-NEXT:    [[CALL:%.*]] = call ptr @rt3_helper(ptr noalias nocapture nofree readnone [[A]], ptr noalias nofree readnone "no-capture-maybe-returned" [[B]]) #[[ATTR8]]
+; TUNIT-NEXT:    [[CALL:%.*]] = call ptr @rt3_helper(ptr noalias nofree readnone captures(none) [[A]], ptr noalias nofree readnone "no-capture-maybe-returned" [[B]]) #[[ATTR8]]
 ; TUNIT-NEXT:    br label [[IF_END]]
 ; TUNIT:       if.end:
 ; TUNIT-NEXT:    [[SEL:%.*]] = phi ptr [ [[B]], [[ENTRY:%.*]] ], [ [[B]], [[IF_THEN]] ]
@@ -628,12 +628,12 @@ define ptr @rt3(ptr %a, ptr %b) #0 {
 ;
 ; CGSCC: Function Attrs: nofree noinline nosync nounwind memory(none) uwtable
 ; CGSCC-LABEL: define {{[^@]+}}@rt3
-; CGSCC-SAME: (ptr nocapture nofree readnone [[A:%.*]], ptr nofree readnone returned "no-capture-maybe-returned" [[B:%.*]]) #[[ATTR1]] {
+; CGSCC-SAME: (ptr nofree readnone captures(none) [[A:%.*]], ptr nofree readnone returned "no-capture-maybe-returned" [[B:%.*]]) #[[ATTR1]] {
 ; CGSCC-NEXT:  entry:
 ; CGSCC-NEXT:    [[CMP:%.*]] = icmp eq ptr [[A]], null
 ; CGSCC-NEXT:    br i1 [[CMP]], label [[IF_THEN:%.*]], label [[IF_END:%.*]]
 ; CGSCC:       if.then:
-; CGSCC-NEXT:    [[CALL:%.*]] = call ptr @rt3_helper(ptr noalias nocapture nofree readnone [[A]], ptr noalias nofree readnone "no-capture-maybe-returned" [[B]]) #[[ATTR7]]
+; CGSCC-NEXT:    [[CALL:%.*]] = call ptr @rt3_helper(ptr noalias nofree readnone captures(none) [[A]], ptr noalias nofree readnone "no-capture-maybe-returned" [[B]]) #[[ATTR7]]
 ; CGSCC-NEXT:    br label [[IF_END]]
 ; CGSCC:       if.end:
 ; CGSCC-NEXT:    [[SEL:%.*]] = phi ptr [ [[B]], [[ENTRY:%.*]] ], [ [[B]], [[IF_THEN]] ]
diff --git a/llvm/test/Transforms/Attributor/undefined_behavior.ll b/llvm/test/Transforms/Attributor/undefined_behavior.ll
index 4ca6ab2c73435..9839bd5521329 100644
--- a/llvm/test/Transforms/Attributor/undefined_behavior.ll
+++ b/llvm/test/Transforms/Attributor/undefined_behavior.ll
@@ -585,13 +585,13 @@ define i32 @foo() {
 define void @arg_nonnull_1(ptr nonnull %a) {
 ; TUNIT: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: write)
 ; TUNIT-LABEL: define {{[^@]+}}@arg_nonnull_1
-; TUNIT-SAME: (ptr nocapture nofree noundef nonnull writeonly align 4 dereferenceable(4) [[A:%.*]]) #[[ATTR6:[0-9]+]] {
+; TUNIT-SAME: (ptr nofree noundef nonnull writeonly align 4 captures(none) dereferenceable(4) [[A:%.*]]) #[[ATTR6:[0-9]+]] {
 ; TUNIT-NEXT:    store i32 0, ptr [[A]], align 4
 ; TUNIT-NEXT:    ret void
 ;
 ; CGSCC: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: write)
 ; CGSCC-LABEL: define {{[^@]+}}@arg_nonnull_1
-; CGSCC-SAME: (ptr nocapture nofree noundef nonnull writeonly align 4 dereferenceable(4) [[A:%.*]]) #[[ATTR9:[0-9]+]] {
+; CGSCC-SAME: (ptr nofree noundef nonnull writeonly align 4 captures(none) dereferenceable(4) [[A:%.*]]) #[[ATTR9:[0-9]+]] {
 ; CGSCC-NEXT:    store i32 0, ptr [[A]], align 4
 ; CGSCC-NEXT:    ret void
 ;
@@ -602,13 +602,13 @@ define void @arg_nonnull_1(ptr nonnull %a) {
 define void @arg_nonnull_1_noundef_1(ptr nonnull noundef %a) {
 ; TUNIT: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: write)
 ; TUNIT-LABEL: define {{[^@]+}}@arg_nonnull_1_noundef_1
-; TUNIT-SAME: (ptr nocapture nofree noundef nonnull writeonly align 4 dereferenceable(4) [[A:%.*]]) #[[ATTR6]] {
+; TUNIT-SAME: (ptr nofree noundef nonnull writeonly align 4 captures(none) dereferenceable(4) [[A:%.*]]) #[[ATTR6]] {
 ; TUNIT-NEXT:    store i32 0, ptr [[A]], align 4
 ; TUNIT-NEXT:    ret void
 ;
 ; CGSCC: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: write)
 ; CGSCC-LABEL: define {{[^@]+}}@arg_nonnull_1_noundef_1
-; CGSCC-SAME: (ptr nocapture nofree noundef nonnull writeonly align 4 dereferenceable(4) [[A:%.*]]) #[[ATTR9]] {
+; CGSCC-SAME: (ptr nofree noundef nonnull writeonly align 4 captures(none) dereferenceable(4) [[A:%.*]]) #[[ATTR9]] {
 ; CGSCC-NEXT:    store i32 0, ptr [[A]], align 4
 ; CGSCC-NEXT:    ret void
 ;
@@ -619,7 +619,7 @@ define void @arg_nonnull_1_noundef_1(ptr nonnull noundef %a) {
 define void @arg_nonnull_12(ptr nonnull %a, ptr nonnull %b, ptr %c) {
 ; TUNIT: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: write)
 ; TUNIT-LABEL: define {{[^@]+}}@arg_nonnull_12
-; TUNIT-SAME: (ptr nocapture nofree nonnull writeonly [[A:%.*]], ptr nocapture nofree nonnull writeonly [[B:%.*]], ptr nofree writeonly [[C:%.*]]) #[[ATTR6]] {
+; TUNIT-SAME: (ptr nofree nonnull writeonly captures(none) [[A:%.*]], ptr nofree nonnull writeonly captures(none) [[B:%.*]], ptr nofree writeonly [[C:%.*]]) #[[ATTR6]] {
 ; TUNIT-NEXT:    [[D:%.*]] = icmp eq ptr [[C]], null
 ; TUNIT-NEXT:    br i1 [[D]], label [[T:%.*]], label [[F:%.*]]
 ; TUNIT:       t:
@@ -633,7 +633,7 @@ define void @arg_nonnull_12(ptr nonnull %a, ptr nonnull %b, ptr %c) {
 ;
 ; CGSCC: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: write)
 ; CGSCC-LABEL: define {{[^@]+}}@arg_nonnull_12
-; CGSCC-SAME: (ptr nocapture nofree nonnull writeonly [[A:%.*]], ptr nocapture nofree nonnull writeonly [[B:%.*]], ptr nofree writeonly [[C:%.*]]) #[[ATTR9]] {
+; CGSCC-SAME: (ptr nofree nonnull writeonly captures(none) [[A:%.*]], ptr nofree nonnull writeonly captures(none) [[B:%.*]], ptr nofree writeonly [[C:%.*]]) #[[ATTR9]] {
 ; CGSCC-NEXT:    [[D:%.*]] = icmp eq ptr [[C]], null
 ; CGSCC-NEXT:    br i1 [[D]], label [[T:%.*]], label [[F:%.*]]
 ; CGSCC:       t:
@@ -660,7 +660,7 @@ ret:
 define void @arg_nonnull_12_noundef_2(ptr nonnull %a, ptr noundef nonnull %b, ptr %c) {
 ; TUNIT: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: write)
 ; TUNIT-LABEL: define {{[^@]+}}@arg_nonnull_12_noundef_2
-; TUNIT-SAME: (ptr nocapture nofree nonnull writeonly [[A:%.*]], ptr nocapture nofree noundef nonnull writeonly [[B:%.*]], ptr nofree writeonly [[C:%.*]]) #[[ATTR6]] {
+; TUNIT-SAME: (ptr nofree nonnull writeonly captures(none) [[A:%.*]], ptr nofree noundef nonnull writeonly captures(none) [[B:%.*]], ptr nofree writeonly [[C:%.*]]) #[[ATTR6]] {
 ; TUNIT-NEXT:    [[D:%.*]] = icmp eq ptr [[C]], null
 ; TUNIT-NEXT:    br i1 [[D]], label [[T:%.*]], label [[F:%.*]]
 ; TUNIT:       t:
@@ -674,7 +674,7 @@ define void @arg_nonnull_12_noundef_2(ptr nonnull %a, ptr noundef nonnull %b, pt
 ;
 ; CGSCC: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: write)
 ; CGSCC-LABEL: define {{[^@]+}}@arg_nonnull_12_noundef_2
-; CGSCC-SAME: (ptr nocapture nofree nonnull writeonly [[A:%.*]], ptr nocapture nofree noundef nonnull writeonly [[B:%.*]], ptr nofree writeonly [[C:%.*]]) #[[ATTR9]] {
+; CGSCC-SAME: (ptr nofree nonnull writeonly captures(none) [[A:%.*]], ptr nofree noundef nonnull writeonly captures(none) [[B:%.*]], ptr nofree writeonly [[C:%.*]]) #[[ATTR9]] {
 ; CGSCC-NEXT:    [[D:%.*]] = icmp eq ptr [[C]], null
 ; CGSCC-NEXT:    br i1 [[D]], label [[T:%.*]], label [[F:%.*]]
 ; CGSCC:       t:
@@ -770,8 +770,8 @@ define void @arg_nonnull_violation3_1(i1 %c) {
 ; TUNIT-NEXT:    [[PTR:%.*]] = alloca i32, align 4
 ; TUNIT-NEXT:    br i1 [[C]], label [[T:%.*]], label [[F:%.*]]
 ; TUNIT:       t:
-; TUNIT-NEXT:    call void @arg_nonnull_12(ptr nocapture nofree noundef nonnull writeonly align 4 dereferenceable(4) [[PTR]], ptr nocapture nofree noundef nonnull writeonly align 4 dereferenceable(4) [[PTR]], ptr nofree noundef nonnull writeonly align 4 dereferenceable(4) [[PTR]]) #[[ATTR7:[0-9]+]]
-; TUNIT-NEXT:    call void @arg_nonnull_12(ptr nocapture nofree noundef nonnull writeonly align 4 dereferenceable(4) [[PTR]], ptr nocapture nofree noundef nonnull writeonly align 4 dereferenceable(4) [[PTR]], ptr nofree noundef writeonly align 4294967296 null) #[[ATTR7]]
+; TUNIT-NEXT:    call void @arg_nonnull_12(ptr nofree noundef nonnull writeonly align 4 captures(none) dereferenceable(4) [[PTR]], ptr nofree noundef nonnull writeonly align 4 captures(none) dereferenceable(4) [[PTR]], ptr nofree noundef nonnull writeonly align 4 dereferenceable(4) [[PTR]]) #[[ATTR7:[0-9]+]]
+; TUNIT-NEXT:    call void @arg_nonnull_12(ptr nofree noundef nonnull writeonly align 4 captures(none) dereferenceable(4) [[PTR]], ptr nofree noundef nonnull writeonly align 4 captures(none) dereferenceable(4) [[PTR]], ptr nofree noundef writeonly align 4294967296 null) #[[ATTR7]]
 ; TUNIT-NEXT:    unreachable
 ; TUNIT:       f:
 ; TUNIT-NEXT:    unreachable
@@ -784,8 +784,8 @@ define void @arg_nonnull_violation3_1(i1 %c) {
 ; CGSCC-NEXT:    [[PTR:%.*]] = alloca i32, align 4
 ; CGSCC-NEXT:    br i1 [[C]], label [[T:%.*]], label [[F:%.*]]
 ; CGSCC:       t:
-; CGSCC-NEXT:    call void @arg_nonnull_12(ptr nocapture nofree noundef nonnull writeonly align 4 dereferenceable(4) [[PTR]], ptr nocapture nofree noundef nonnull writeonly align 4 dereferenceable(4) [[PTR]], ptr nofree noundef nonnull writeonly align 4 dereferenceable(4) [[PTR]]) #[[ATTR12:[0-9]+]]
-; CGSCC-NEXT:    call void @arg_nonnull_12(ptr nocapture nofree noundef nonnull writeonly align 4 dereferenceable(4) [[PTR]], ptr nocapture nofree noundef nonnull writeonly align 4 dereferenceable(4) [[PTR]], ptr nofree noundef writeonly align 4294967296 null) #[[ATTR12]]
+; CGSCC-NEXT:    call void @arg_nonnull_12(ptr nofree noundef nonnull writeonly align 4 captures(none) dereferenceable(4) [[PTR]], ptr nofree noundef nonnull writeonly align 4 captures(none) dereferenceable(4) [[PTR]], ptr nofree noundef nonnull writeonly align 4 dereferenceable(4) [[PTR]]) #[[ATTR12:[0-9]+]]
+; CGSCC-NEXT:    call void @arg_nonnull_12(ptr nofree noundef nonnull writeonly align 4 captures(none) dereferenceable(4) [[PTR]], ptr nofree noundef nonnull writeonly align 4 captures(none) dereferenceable(4) [[PTR]], ptr nofree noundef writeonly align 4294967296 null) #[[ATTR12]]
 ; CGSCC-NEXT:    unreachable
 ; CGSCC:       f:
 ; CGSCC-NEXT:    unreachable
@@ -817,8 +817,8 @@ define void @arg_nonnull_violation3_2(i1 %c) {
 ; TUNIT-NEXT:    [[PTR:%.*]] = alloca i32, align 4
 ; TUNIT-NEXT:    br i1 [[C]], label [[T:%.*]], label [[F:%.*]]
 ; TUNIT:       t:
-; TUNIT-NEXT:    call void @arg_nonnull_12_noundef_2(ptr nocapture nofree noundef nonnull writeonly align 4 dereferenceable(4) [[PTR]], ptr nocapture nofree noundef nonnull writeonly align 4 dereferenceable(4) [[PTR]], ptr nofree noundef nonnull writeonly align 4 dereferenceable(4) [[PTR]]) #[[ATTR7]]
-; TUNIT-NEXT:    call void @arg_nonnull_12_noundef_2(ptr nocapture nofree noundef nonnull writeonly align 4 dereferenceable(4) [[PTR]], ptr nocapture nofree noundef nonnull writeonly align 4 dereferenceable(4) [[PTR]], ptr nofree noundef writeonly align 4294967296 null) #[[ATTR7]]
+; TUNIT-NEXT:    call void @arg_nonnull_12_noundef_2(ptr nofree noundef nonnull writeonly align 4 captures(none) dereferenceable(4) [[PTR]], ptr nofree noundef nonnull writeonly align 4 captures(none) dereferenceable(4) [[PTR]], ptr nofree noundef nonnull writeonly align 4 dereferenceable(4) [[PTR]]) #[[ATTR7]]
+; TUNIT-NEXT:    call void @arg_nonnull_12_noundef_2(ptr nofree noundef nonnull writeonly align 4 captures(none) dereferenceable(4) [[PTR]], ptr nofree noundef nonnull writeonly align 4 captures(none) dereferenceable(4) [[PTR]], ptr nofree noundef writeonly align 4294967296 null) #[[ATTR7]]
 ; TUNIT-NEXT:    unreachable
 ; TUNIT:       f:
 ; TUNIT-NEXT:    unreachable
@@ -831,8 +831,8 @@ define void @arg_nonnull_violation3_2(i1 %c) {
 ; CGSCC-NEXT:    [[PTR:%.*]] = alloca i32, align 4
 ; CGSCC-NEXT:    br i1 [[C]], label [[T:%.*]], label [[F:%.*]]
 ; CGSCC:       t:
-; CGSCC-NEXT:    call void @arg_nonnull_12_noundef_2(ptr nocapture nofree noundef nonnull writeonly align 4 dereferenceable(4) [[PTR]], ptr nocapture nofree noundef nonnull writeonly align 4 dereferenceable(4) [[PTR]], ptr nofree noundef nonnull writeonly align 4 dereferenceable(4) [[PTR]]) #[[ATTR12]]
-; CGSCC-NEXT:    call void @arg_nonnull_12_noundef_2(ptr nocapture nofree noundef nonnull writeonly align 4 dereferenceable(4) [[PTR]], ptr nocapture nofree noundef nonnull writeonly align 4 dereferenceable(4) [[PTR]], ptr nofree noundef writeonly align 4294967296 null) #[[ATTR12]]
+; CGSCC-NEXT:    call void @arg_nonnull_12_noundef_2(ptr nofree noundef nonnull writeonly align 4 captures(none) dereferenceable(4) [[PTR]], ptr nofree noundef nonnull writeonly align 4 captures(none) dereferenceable(4) [[PTR]], ptr nofree noundef nonnull writeonly align 4 dereferenceable(4) [[PTR]]) #[[ATTR12]]
+; CGSCC-NEXT:    call void @arg_nonnull_12_noundef_2(ptr nofree noundef nonnull writeonly align 4 captures(none) dereferenceable(4) [[PTR]], ptr nofree noundef nonnull writeonly align 4 captures(none) dereferenceable(4) [[PTR]], ptr nofree noundef writeonly align 4294967296 null) #[[ATTR12]]
 ; CGSCC-NEXT:    unreachable
 ; CGSCC:       f:
 ; CGSCC-NEXT:    unreachable
diff --git a/llvm/test/Transforms/Attributor/value-simplify-assume.ll b/llvm/test/Transforms/Attributor/value-simplify-assume.ll
index b01a43e3ec758..c1ffb599febf6 100644
--- a/llvm/test/Transforms/Attributor/value-simplify-assume.ll
+++ b/llvm/test/Transforms/Attributor/value-simplify-assume.ll
@@ -16,7 +16,7 @@ declare void @unknown()
 define i1 @readI1p(ptr %p) {
 ; CHECK: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: read)
 ; CHECK-LABEL: define {{[^@]+}}@readI1p
-; CHECK-SAME: (ptr nocapture nofree noundef nonnull readonly dereferenceable(1) [[P:%.*]]) #[[ATTR1:[0-9]+]] {
+; CHECK-SAME: (ptr nofree noundef nonnull readonly captures(none) dereferenceable(1) [[P:%.*]]) #[[ATTR1:[0-9]+]] {
 ; CHECK-NEXT:    [[L:%.*]] = load i1, ptr [[P]], align 1
 ; CHECK-NEXT:    ret i1 [[L]]
 ;
@@ -123,7 +123,7 @@ define i1 @keep_assume_4c_nr() norecurse {
 ; TUNIT-NEXT:    [[STACK:%.*]] = alloca i1, align 1
 ; TUNIT-NEXT:    store i1 true, ptr [[STACK]], align 1
 ; TUNIT-NEXT:    call void @llvm.assume(i1 noundef true) #[[ATTR7]]
-; TUNIT-NEXT:    call void @useI1p(ptr noalias nocapture noundef nonnull dereferenceable(1) [[STACK]])
+; TUNIT-NEXT:    call void @useI1p(ptr noalias noundef nonnull captures(none) dereferenceable(1) [[STACK]])
 ; TUNIT-NEXT:    ret i1 true
 ;
 ; CGSCC: Function Attrs: norecurse
@@ -132,7 +132,7 @@ define i1 @keep_assume_4c_nr() norecurse {
 ; CGSCC-NEXT:    [[STACK:%.*]] = alloca i1, align 1
 ; CGSCC-NEXT:    store i1 true, ptr [[STACK]], align 1
 ; CGSCC-NEXT:    call void @llvm.assume(i1 noundef true) #[[ATTR8]]
-; CGSCC-NEXT:    call void @useI1p(ptr noalias nocapture noundef nonnull dereferenceable(1) [[STACK]])
+; CGSCC-NEXT:    call void @useI1p(ptr noalias noundef nonnull captures(none) dereferenceable(1) [[STACK]])
 ; CGSCC-NEXT:    ret i1 true
 ;
   %stack = alloca i1
@@ -247,7 +247,7 @@ define i1 @keep_assume_4_nr(i1 %arg) norecurse {
 ; TUNIT-NEXT:    [[STACK:%.*]] = alloca i1, align 1
 ; TUNIT-NEXT:    store i1 [[ARG]], ptr [[STACK]], align 1
 ; TUNIT-NEXT:    call void @llvm.assume(i1 noundef [[ARG]]) #[[ATTR7]]
-; TUNIT-NEXT:    call void @useI1p(ptr noalias nocapture noundef nonnull dereferenceable(1) [[STACK]])
+; TUNIT-NEXT:    call void @useI1p(ptr noalias noundef nonnull captures(none) dereferenceable(1) [[STACK]])
 ; TUNIT-NEXT:    ret i1 [[ARG]]
 ;
 ; CGSCC: Function Attrs: norecurse
@@ -256,7 +256,7 @@ define i1 @keep_assume_4_nr(i1 %arg) norecurse {
 ; CGSCC-NEXT:    [[STACK:%.*]] = alloca i1, align 1
 ; CGSCC-NEXT:    store i1 [[ARG]], ptr [[STACK]], align 1
 ; CGSCC-NEXT:    call void @llvm.assume(i1 noundef [[ARG]]) #[[ATTR8]]
-; CGSCC-NEXT:    call void @useI1p(ptr noalias nocapture noundef nonnull dereferenceable(1) [[STACK]])
+; CGSCC-NEXT:    call void @useI1p(ptr noalias noundef nonnull captures(none) dereferenceable(1) [[STACK]])
 ; CGSCC-NEXT:    ret i1 [[ARG]]
 ;
   %stack = alloca i1
@@ -428,7 +428,7 @@ define i1 @assume_3_nr(i1 %arg, i1 %cond) norecurse {
 ; TUNIT-NEXT:    store i1 false, ptr [[STACK]], align 1
 ; TUNIT-NEXT:    br label [[M]]
 ; TUNIT:       m:
-; TUNIT-NEXT:    [[R:%.*]] = call i1 @readI1p(ptr noalias nocapture nofree noundef nonnull readonly dereferenceable(1) [[STACK]]) #[[ATTR8:[0-9]+]]
+; TUNIT-NEXT:    [[R:%.*]] = call i1 @readI1p(ptr noalias nofree noundef nonnull readonly captures(none) dereferenceable(1) [[STACK]]) #[[ATTR8:[0-9]+]]
 ; TUNIT-NEXT:    ret i1 [[R]]
 ;
 ; CGSCC: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(inaccessiblemem: readwrite)
@@ -446,7 +446,7 @@ define i1 @assume_3_nr(i1 %arg, i1 %cond) norecurse {
 ; CGSCC-NEXT:    store i1 false, ptr [[STACK]], align 1
 ; CGSCC-NEXT:    br label [[M]]
 ; CGSCC:       m:
-; CGSCC-NEXT:    [[R:%.*]] = call i1 @readI1p(ptr noalias nocapture nofree noundef nonnull readonly dereferenceable(1) [[STACK]]) #[[ATTR9:[0-9]+]]
+; CGSCC-NEXT:    [[R:%.*]] = call i1 @readI1p(ptr noalias nofree noundef nonnull readonly captures(none) dereferenceable(1) [[STACK]]) #[[ATTR9:[0-9]+]]
 ; CGSCC-NEXT:    ret i1 [[R]]
 ;
   %stack = alloca i1
@@ -480,7 +480,7 @@ define i1 @assume_4_nr(i1 %arg, i1 %cond) norecurse {
 ; TUNIT:       m:
 ; TUNIT-NEXT:    [[L:%.*]] = load i1, ptr [[STACK]], align 1
 ; TUNIT-NEXT:    call void @llvm.assume(i1 noundef [[L]]) #[[ATTR7]]
-; TUNIT-NEXT:    [[R:%.*]] = call i1 @readI1p(ptr noalias nocapture nofree noundef nonnull readonly dereferenceable(1) [[STACK]]) #[[ATTR8]]
+; TUNIT-NEXT:    [[R:%.*]] = call i1 @readI1p(ptr noalias nofree noundef nonnull readonly captures(none) dereferenceable(1) [[STACK]]) #[[ATTR8]]
 ; TUNIT-NEXT:    ret i1 [[R]]
 ;
 ; CGSCC: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(inaccessiblemem: readwrite)
@@ -498,7 +498,7 @@ define i1 @assume_4_nr(i1 %arg, i1 %cond) norecurse {
 ; CGSCC:       m:
 ; CGSCC-NEXT:    [[L:%.*]] = load i1, ptr [[STACK]], align 1
 ; CGSCC-NEXT:    call void @llvm.assume(i1 noundef [[L]]) #[[ATTR8]]
-; CGSCC-NEXT:    [[R:%.*]] = call i1 @readI1p(ptr noalias nocapture nofree noundef nonnull readonly dereferenceable(1) [[STACK]]) #[[ATTR9]]
+; CGSCC-NEXT:    [[R:%.*]] = call i1 @readI1p(ptr noalias nofree noundef nonnull readonly captures(none) dereferenceable(1) [[STACK]]) #[[ATTR9]]
 ; CGSCC-NEXT:    ret i1 [[R]]
 ;
   %stack = alloca i1
@@ -536,7 +536,7 @@ define i1 @assume_5_nr(i1 %arg, i1 %cond) norecurse {
 ; TUNIT:       m:
 ; TUNIT-NEXT:    [[L4:%.*]] = load i1, ptr [[STACK]], align 1
 ; TUNIT-NEXT:    call void @llvm.assume(i1 noundef [[L4]]) #[[ATTR9:[0-9]+]]
-; TUNIT-NEXT:    [[R:%.*]] = call i1 @readI1p(ptr noalias nocapture nofree noundef nonnull readonly dereferenceable(1) [[STACK]]) #[[ATTR8]]
+; TUNIT-NEXT:    [[R:%.*]] = call i1 @readI1p(ptr noalias nofree noundef nonnull readonly captures(none) dereferenceable(1) [[STACK]]) #[[ATTR8]]
 ; TUNIT-NEXT:    ret i1 [[R]]
 ;
 ; CGSCC: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(inaccessiblemem: readwrite)
@@ -560,7 +560,7 @@ define i1 @assume_5_nr(i1 %arg, i1 %cond) norecurse {
 ; CGSCC:       m:
 ; CGSCC-NEXT:    [[L4:%.*]] = load i1, ptr [[STACK]], align 1
 ; CGSCC-NEXT:    call void @llvm.assume(i1 noundef [[L4]]) #[[ATTR10]]
-; CGSCC-NEXT:    [[R:%.*]] = call i1 @readI1p(ptr noalias nocapture nofree noundef nonnull readonly dereferenceable(1) [[STACK]]) #[[ATTR10]]
+; CGSCC-NEXT:    [[R:%.*]] = call i1 @readI1p(ptr noalias nofree noundef nonnull readonly captures(none) dereferenceable(1) [[STACK]]) #[[ATTR10]]
 ; CGSCC-NEXT:    ret i1 [[R]]
 ;
   %stack = alloca i1
@@ -605,7 +605,7 @@ define i1 @assume_5c_nr(i1 %cond) norecurse {
 ; TUNIT:       m:
 ; TUNIT-NEXT:    [[L4:%.*]] = load i1, ptr [[STACK]], align 1
 ; TUNIT-NEXT:    call void @llvm.assume(i1 noundef [[L4]]) #[[ATTR9]]
-; TUNIT-NEXT:    [[R:%.*]] = call i1 @readI1p(ptr noalias nocapture nofree noundef nonnull readonly dereferenceable(1) [[STACK]]) #[[ATTR8]]
+; TUNIT-NEXT:    [[R:%.*]] = call i1 @readI1p(ptr noalias nofree noundef nonnull readonly captures(none) dereferenceable(1) [[STACK]]) #[[ATTR8]]
 ; TUNIT-NEXT:    ret i1 [[R]]
 ;
 ; CGSCC: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(inaccessiblemem: readwrite)
@@ -629,7 +629,7 @@ define i1 @assume_5c_nr(i1 %cond) norecurse {
 ; CGSCC:       m:
 ; CGSCC-NEXT:    [[L4:%.*]] = load i1, ptr [[STACK]], align 1
 ; CGSCC-NEXT:    call void @llvm.assume(i1 noundef [[L4]]) #[[ATTR10]]
-; CGSCC-NEXT:    [[R:%.*]] = call i1 @readI1p(ptr noalias nocapture nofree noundef nonnull readonly dereferenceable(1) [[STACK]]) #[[ATTR10]]
+; CGSCC-NEXT:    [[R:%.*]] = call i1 @readI1p(ptr noalias nofree noundef nonnull readonly captures(none) dereferenceable(1) [[STACK]]) #[[ATTR10]]
 ; CGSCC-NEXT:    ret i1 [[R]]
 ;
   %stack = alloca i1
@@ -745,7 +745,7 @@ define i1 @keep_assume_4c() {
 ; TUNIT-NEXT:    store i1 true, ptr [[STACK]], align 1
 ; TUNIT-NEXT:    [[L4:%.*]] = load i1, ptr [[STACK]], align 1
 ; TUNIT-NEXT:    call void @llvm.assume(i1 noundef [[L4]]) #[[ATTR7]]
-; TUNIT-NEXT:    call void @useI1p(ptr noalias nocapture noundef nonnull dereferenceable(1) [[STACK]])
+; TUNIT-NEXT:    call void @useI1p(ptr noalias noundef nonnull captures(none) dereferenceable(1) [[STACK]])
 ; TUNIT-NEXT:    ret i1 [[L4]]
 ;
 ; CGSCC-LABEL: define {{[^@]+}}@keep_assume_4c() {
@@ -753,7 +753,7 @@ define i1 @keep_assume_4c() {
 ; CGSCC-NEXT:    store i1 true, ptr [[STACK]], align 1
 ; CGSCC-NEXT:    [[L4:%.*]] = load i1, ptr [[STACK]], align 1
 ; CGSCC-NEXT:    call void @llvm.assume(i1 noundef [[L4]]) #[[ATTR8]]
-; CGSCC-NEXT:    call void @useI1p(ptr noalias nocapture noundef nonnull dereferenceable(1) [[STACK]])
+; CGSCC-NEXT:    call void @useI1p(ptr noalias noundef nonnull captures(none) dereferenceable(1) [[STACK]])
 ; CGSCC-NEXT:    ret i1 [[L4]]
 ;
   %stack = alloca i1
@@ -864,7 +864,7 @@ define i1 @keep_assume_4(i1 %arg) {
 ; TUNIT-NEXT:    store i1 [[ARG]], ptr [[STACK]], align 1
 ; TUNIT-NEXT:    [[L:%.*]] = load i1, ptr [[STACK]], align 1
 ; TUNIT-NEXT:    call void @llvm.assume(i1 noundef [[L]]) #[[ATTR7]]
-; TUNIT-NEXT:    call void @useI1p(ptr noalias nocapture noundef nonnull dereferenceable(1) [[STACK]])
+; TUNIT-NEXT:    call void @useI1p(ptr noalias noundef nonnull captures(none) dereferenceable(1) [[STACK]])
 ; TUNIT-NEXT:    ret i1 [[L]]
 ;
 ; CGSCC-LABEL: define {{[^@]+}}@keep_assume_4
@@ -873,7 +873,7 @@ define i1 @keep_assume_4(i1 %arg) {
 ; CGSCC-NEXT:    store i1 [[ARG]], ptr [[STACK]], align 1
 ; CGSCC-NEXT:    [[L:%.*]] = load i1, ptr [[STACK]], align 1
 ; CGSCC-NEXT:    call void @llvm.assume(i1 noundef [[L]]) #[[ATTR8]]
-; CGSCC-NEXT:    call void @useI1p(ptr noalias nocapture noundef nonnull dereferenceable(1) [[STACK]])
+; CGSCC-NEXT:    call void @useI1p(ptr noalias noundef nonnull captures(none) dereferenceable(1) [[STACK]])
 ; CGSCC-NEXT:    ret i1 [[L]]
 ;
   %stack = alloca i1
@@ -1045,7 +1045,7 @@ define i1 @assume_3(i1 %arg, i1 %cond) {
 ; TUNIT-NEXT:    store i1 false, ptr [[STACK]], align 1
 ; TUNIT-NEXT:    br label [[M]]
 ; TUNIT:       m:
-; TUNIT-NEXT:    [[R:%.*]] = call i1 @readI1p(ptr noalias nocapture nofree noundef nonnull readonly dereferenceable(1) [[STACK]]) #[[ATTR8]]
+; TUNIT-NEXT:    [[R:%.*]] = call i1 @readI1p(ptr noalias nofree noundef nonnull readonly captures(none) dereferenceable(1) [[STACK]]) #[[ATTR8]]
 ; TUNIT-NEXT:    ret i1 [[R]]
 ;
 ; CGSCC: Function Attrs: mustprogress nofree nosync nounwind willreturn memory(inaccessiblemem: readwrite)
@@ -1063,7 +1063,7 @@ define i1 @assume_3(i1 %arg, i1 %cond) {
 ; CGSCC-NEXT:    store i1 false, ptr [[STACK]], align 1
 ; CGSCC-NEXT:    br label [[M]]
 ; CGSCC:       m:
-; CGSCC-NEXT:    [[R:%.*]] = call i1 @readI1p(ptr noalias nocapture nofree noundef nonnull readonly dereferenceable(1) [[STACK]]) #[[ATTR9]]
+; CGSCC-NEXT:    [[R:%.*]] = call i1 @readI1p(ptr noalias nofree noundef nonnull readonly captures(none) dereferenceable(1) [[STACK]]) #[[ATTR9]]
 ; CGSCC-NEXT:    ret i1 [[R]]
 ;
   %stack = alloca i1
@@ -1097,7 +1097,7 @@ define i1 @assume_4(i1 %arg, i1 %cond) {
 ; TUNIT:       m:
 ; TUNIT-NEXT:    [[L:%.*]] = load i1, ptr [[STACK]], align 1
 ; TUNIT-NEXT:    call void @llvm.assume(i1 noundef [[L]]) #[[ATTR7]]
-; TUNIT-NEXT:    [[R:%.*]] = call i1 @readI1p(ptr noalias nocapture nofree noundef nonnull readonly dereferenceable(1) [[STACK]]) #[[ATTR8]]
+; TUNIT-NEXT:    [[R:%.*]] = call i1 @readI1p(ptr noalias nofree noundef nonnull readonly captures(none) dereferenceable(1) [[STACK]]) #[[ATTR8]]
 ; TUNIT-NEXT:    ret i1 [[R]]
 ;
 ; CGSCC: Function Attrs: mustprogress nofree nosync nounwind willreturn memory(inaccessiblemem: readwrite)
@@ -1115,7 +1115,7 @@ define i1 @assume_4(i1 %arg, i1 %cond) {
 ; CGSCC:       m:
 ; CGSCC-NEXT:    [[L:%.*]] = load i1, ptr [[STACK]], align 1
 ; CGSCC-NEXT:    call void @llvm.assume(i1 noundef [[L]]) #[[ATTR8]]
-; CGSCC-NEXT:    [[R:%.*]] = call i1 @readI1p(ptr noalias nocapture nofree noundef nonnull readonly dereferenceable(1) [[STACK]]) #[[ATTR9]]
+; CGSCC-NEXT:    [[R:%.*]] = call i1 @readI1p(ptr noalias nofree noundef nonnull readonly captures(none) dereferenceable(1) [[STACK]]) #[[ATTR9]]
 ; CGSCC-NEXT:    ret i1 [[R]]
 ;
   %stack = alloca i1
@@ -1153,7 +1153,7 @@ define i1 @assume_5(i1 %arg, i1 %cond) {
 ; TUNIT:       m:
 ; TUNIT-NEXT:    [[L4:%.*]] = load i1, ptr [[STACK]], align 1
 ; TUNIT-NEXT:    call void @llvm.assume(i1 noundef [[L4]]) #[[ATTR9]]
-; TUNIT-NEXT:    [[R:%.*]] = call i1 @readI1p(ptr noalias nocapture nofree noundef nonnull readonly dereferenceable(1) [[STACK]]) #[[ATTR8]]
+; TUNIT-NEXT:    [[R:%.*]] = call i1 @readI1p(ptr noalias nofree noundef nonnull readonly captures(none) dereferenceable(1) [[STACK]]) #[[ATTR8]]
 ; TUNIT-NEXT:    ret i1 [[R]]
 ;
 ; CGSCC: Function Attrs: mustprogress nofree nosync nounwind willreturn memory(inaccessiblemem: readwrite)
@@ -1177,7 +1177,7 @@ define i1 @assume_5(i1 %arg, i1 %cond) {
 ; CGSCC:       m:
 ; CGSCC-NEXT:    [[L4:%.*]] = load i1, ptr [[STACK]], align 1
 ; CGSCC-NEXT:    call void @llvm.assume(i1 noundef [[L4]]) #[[ATTR10]]
-; CGSCC-NEXT:    [[R:%.*]] = call i1 @readI1p(ptr noalias nocapture nofree noundef nonnull readonly dereferenceable(1) [[STACK]]) #[[ATTR10]]
+; CGSCC-NEXT:    [[R:%.*]] = call i1 @readI1p(ptr noalias nofree noundef nonnull readonly captures(none) dereferenceable(1) [[STACK]]) #[[ATTR10]]
 ; CGSCC-NEXT:    ret i1 [[R]]
 ;
   %stack = alloca i1
@@ -1222,7 +1222,7 @@ define i1 @assume_5c(i1 %cond) {
 ; TUNIT:       m:
 ; TUNIT-NEXT:    [[L4:%.*]] = load i1, ptr [[STACK]], align 1
 ; TUNIT-NEXT:    call void @llvm.assume(i1 noundef [[L4]]) #[[ATTR9]]
-; TUNIT-NEXT:    [[R:%.*]] = call i1 @readI1p(ptr noalias nocapture nofree noundef nonnull readonly dereferenceable(1) [[STACK]]) #[[ATTR8]]
+; TUNIT-NEXT:    [[R:%.*]] = call i1 @readI1p(ptr noalias nofree noundef nonnull readonly captures(none) dereferenceable(1) [[STACK]]) #[[ATTR8]]
 ; TUNIT-NEXT:    ret i1 [[R]]
 ;
 ; CGSCC: Function Attrs: mustprogress nofree nosync nounwind willreturn memory(inaccessiblemem: readwrite)
@@ -1246,7 +1246,7 @@ define i1 @assume_5c(i1 %cond) {
 ; CGSCC:       m:
 ; CGSCC-NEXT:    [[L4:%.*]] = load i1, ptr [[STACK]], align 1
 ; CGSCC-NEXT:    call void @llvm.assume(i1 noundef [[L4]]) #[[ATTR10]]
-; CGSCC-NEXT:    [[R:%.*]] = call i1 @readI1p(ptr noalias nocapture nofree noundef nonnull readonly dereferenceable(1) [[STACK]]) #[[ATTR10]]
+; CGSCC-NEXT:    [[R:%.*]] = call i1 @readI1p(ptr noalias nofree noundef nonnull readonly captures(none) dereferenceable(1) [[STACK]]) #[[ATTR10]]
 ; CGSCC-NEXT:    ret i1 [[R]]
 ;
   %stack = alloca i1
@@ -1314,7 +1314,7 @@ define i32 @assume_read_global_bad(ptr %p) {
 ;
 ; TUNIT: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn
 ; TUNIT-LABEL: define {{[^@]+}}@assume_read_global_bad
-; TUNIT-SAME: (ptr nocapture nofree noundef nonnull writeonly align 4 dereferenceable(4) [[P:%.*]]) #[[ATTR5]] {
+; TUNIT-SAME: (ptr nofree noundef nonnull writeonly align 4 captures(none) dereferenceable(4) [[P:%.*]]) #[[ATTR5]] {
 ; TUNIT-NEXT:    [[LGS1:%.*]] = load i32, ptr @Gstatic_int2, align 4
 ; TUNIT-NEXT:    [[C:%.*]] = icmp eq i32 [[LGS1]], 42
 ; TUNIT-NEXT:    store i32 13, ptr [[P]], align 4
@@ -1325,7 +1325,7 @@ define i32 @assume_read_global_bad(ptr %p) {
 ;
 ; CGSCC: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn
 ; CGSCC-LABEL: define {{[^@]+}}@assume_read_global_bad
-; CGSCC-SAME: (ptr nocapture nofree noundef nonnull writeonly align 4 dereferenceable(4) [[P:%.*]]) #[[ATTR6]] {
+; CGSCC-SAME: (ptr nofree noundef nonnull writeonly align 4 captures(none) dereferenceable(4) [[P:%.*]]) #[[ATTR6]] {
 ; CGSCC-NEXT:    [[LGS1:%.*]] = load i32, ptr @Gstatic_int2, align 4
 ; CGSCC-NEXT:    [[C:%.*]] = icmp eq i32 [[LGS1]], 42
 ; CGSCC-NEXT:    store i32 13, ptr [[P]], align 4
diff --git a/llvm/test/Transforms/Attributor/value-simplify-dominance.ll b/llvm/test/Transforms/Attributor/value-simplify-dominance.ll
index 7d95f35b24fd0..00891cf41d400 100644
--- a/llvm/test/Transforms/Attributor/value-simplify-dominance.ll
+++ b/llvm/test/Transforms/Attributor/value-simplify-dominance.ll
@@ -141,13 +141,13 @@ declare void @usei32(i32) nocallback
 define internal i32 @remote_write_and_read(ptr %p) norecurse {
 ; TUNIT: Function Attrs: norecurse
 ; TUNIT-LABEL: define {{[^@]+}}@remote_write_and_read
-; TUNIT-SAME: (ptr noalias nocapture nofree noundef nonnull writeonly align 4 dereferenceable(4) [[P:%.*]]) #[[ATTR2]] {
+; TUNIT-SAME: (ptr noalias nofree noundef nonnull writeonly align 4 captures(none) dereferenceable(4) [[P:%.*]]) #[[ATTR2]] {
 ; TUNIT-NEXT:    call void @usei32(i32 noundef 42)
 ; TUNIT-NEXT:    ret i32 undef
 ;
 ; CGSCC: Function Attrs: norecurse
 ; CGSCC-LABEL: define {{[^@]+}}@remote_write_and_read
-; CGSCC-SAME: (ptr noalias nocapture nofree noundef nonnull align 4 dereferenceable(4) [[P:%.*]]) #[[ATTR2]] {
+; CGSCC-SAME: (ptr noalias nofree noundef nonnull align 4 captures(none) dereferenceable(4) [[P:%.*]]) #[[ATTR2]] {
 ; CGSCC-NEXT:    store i32 42, ptr [[P]], align 4
 ; CGSCC-NEXT:    [[L:%.*]] = load i32, ptr [[P]], align 4
 ; CGSCC-NEXT:    call void @usei32(i32 [[L]])
@@ -164,14 +164,14 @@ define i32 @local_stack_remote_write_and_read() norecurse {
 ; TUNIT-LABEL: define {{[^@]+}}@local_stack_remote_write_and_read
 ; TUNIT-SAME: () #[[ATTR2]] {
 ; TUNIT-NEXT:    [[A:%.*]] = alloca i32, align 4
-; TUNIT-NEXT:    [[R:%.*]] = call i32 @remote_write_and_read(ptr noalias nocapture nofree noundef nonnull writeonly align 4 dereferenceable(4) [[A]])
+; TUNIT-NEXT:    [[R:%.*]] = call i32 @remote_write_and_read(ptr noalias nofree noundef nonnull writeonly align 4 captures(none) dereferenceable(4) [[A]])
 ; TUNIT-NEXT:    ret i32 42
 ;
 ; CGSCC: Function Attrs: norecurse
 ; CGSCC-LABEL: define {{[^@]+}}@local_stack_remote_write_and_read
 ; CGSCC-SAME: () #[[ATTR2]] {
 ; CGSCC-NEXT:    [[A:%.*]] = alloca i32, align 4
-; CGSCC-NEXT:    [[R:%.*]] = call i32 @remote_write_and_read(ptr noalias nocapture nofree noundef nonnull align 4 dereferenceable(4) [[A]])
+; CGSCC-NEXT:    [[R:%.*]] = call i32 @remote_write_and_read(ptr noalias nofree noundef nonnull align 4 captures(none) dereferenceable(4) [[A]])
 ; CGSCC-NEXT:    ret i32 [[R]]
 ;
   %a = alloca i32
diff --git a/llvm/test/Transforms/Attributor/value-simplify-gpu.ll b/llvm/test/Transforms/Attributor/value-simplify-gpu.ll
index dc2d92facf8a3..31df85a0a75dd 100644
--- a/llvm/test/Transforms/Attributor/value-simplify-gpu.ll
+++ b/llvm/test/Transforms/Attributor/value-simplify-gpu.ll
@@ -207,7 +207,7 @@ define internal void @level1(i32 %C) {
 ; TUNIT-NEXT:  entry:
 ; TUNIT-NEXT:    [[LOCAL_ALLOCA:%.*]] = alloca i32, align 4, addrspace(5)
 ; TUNIT-NEXT:    [[LOCAL:%.*]] = addrspacecast ptr addrspace(5) [[LOCAL_ALLOCA]] to ptr
-; TUNIT-NEXT:    call void @level2all_early(ptr nocapture nofree noundef nonnull writeonly align 4 dereferenceable(4) [[LOCAL]]) #[[ATTR4]]
+; TUNIT-NEXT:    call void @level2all_early(ptr nofree noundef nonnull writeonly align 4 captures(none) dereferenceable(4) [[LOCAL]]) #[[ATTR4]]
 ; TUNIT-NEXT:    [[TOBOOL:%.*]] = icmp ne i32 [[C]], 0
 ; TUNIT-NEXT:    br i1 [[TOBOOL]], label [[IF_THEN:%.*]], label [[IF_ELSE:%.*]]
 ; TUNIT:       if.then:
@@ -217,7 +217,7 @@ define internal void @level1(i32 %C) {
 ; TUNIT-NEXT:    call void @level2b() #[[ATTR5]]
 ; TUNIT-NEXT:    br label [[IF_END]]
 ; TUNIT:       if.end:
-; TUNIT-NEXT:    call void @level2all_late(ptr nocapture nofree noundef writeonly align 4 dereferenceable_or_null(4) [[LOCAL]]) #[[ATTR6]]
+; TUNIT-NEXT:    call void @level2all_late(ptr nofree noundef writeonly align 4 captures(none) dereferenceable_or_null(4) [[LOCAL]]) #[[ATTR6]]
 ; TUNIT-NEXT:    ret void
 ;
 ; CGSCC: Function Attrs: norecurse nosync nounwind
@@ -226,17 +226,17 @@ define internal void @level1(i32 %C) {
 ; CGSCC-NEXT:  entry:
 ; CGSCC-NEXT:    [[LOCAL_ALLOCA:%.*]] = alloca i32, align 4, addrspace(5)
 ; CGSCC-NEXT:    [[LOCAL:%.*]] = addrspacecast ptr addrspace(5) [[LOCAL_ALLOCA]] to ptr
-; CGSCC-NEXT:    call void @level2all_early(ptr nocapture nofree noundef nonnull writeonly align 4 dereferenceable(4) [[LOCAL]]) #[[ATTR5]]
+; CGSCC-NEXT:    call void @level2all_early(ptr nofree noundef nonnull writeonly align 4 captures(none) dereferenceable(4) [[LOCAL]]) #[[ATTR5]]
 ; CGSCC-NEXT:    [[TOBOOL:%.*]] = icmp ne i32 [[C]], 0
 ; CGSCC-NEXT:    br i1 [[TOBOOL]], label [[IF_THEN:%.*]], label [[IF_ELSE:%.*]]
 ; CGSCC:       if.then:
-; CGSCC-NEXT:    call void @level2a(ptr nocapture nofree noundef nonnull readonly align 4 dereferenceable(4) [[LOCAL]]) #[[ATTR4]]
+; CGSCC-NEXT:    call void @level2a(ptr nofree noundef nonnull readonly align 4 captures(none) dereferenceable(4) [[LOCAL]]) #[[ATTR4]]
 ; CGSCC-NEXT:    br label [[IF_END:%.*]]
 ; CGSCC:       if.else:
-; CGSCC-NEXT:    call void @level2b(ptr nocapture nofree noundef nonnull readonly align 4 dereferenceable(4) [[LOCAL]]) #[[ATTR4]]
+; CGSCC-NEXT:    call void @level2b(ptr nofree noundef nonnull readonly align 4 captures(none) dereferenceable(4) [[LOCAL]]) #[[ATTR4]]
 ; CGSCC-NEXT:    br label [[IF_END]]
 ; CGSCC:       if.end:
-; CGSCC-NEXT:    call void @level2all_late(ptr nocapture nofree noundef nonnull writeonly align 4 dereferenceable(4) [[LOCAL]]) #[[ATTR6]]
+; CGSCC-NEXT:    call void @level2all_late(ptr nofree noundef nonnull writeonly align 4 captures(none) dereferenceable(4) [[LOCAL]]) #[[ATTR6]]
 ; CGSCC-NEXT:    ret void
 ;
 entry:
@@ -262,7 +262,7 @@ if.end:                                           ; preds = %if.else, %if.then
 define internal void @level2all_early(ptr %addr) {
 ; TUNIT: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(write)
 ; TUNIT-LABEL: define {{[^@]+}}@level2all_early
-; TUNIT-SAME: (ptr nocapture nofree noundef nonnull writeonly align 4 dereferenceable(4) [[ADDR:%.*]]) #[[ATTR2]] {
+; TUNIT-SAME: (ptr nofree noundef nonnull writeonly align 4 captures(none) dereferenceable(4) [[ADDR:%.*]]) #[[ATTR2]] {
 ; TUNIT-NEXT:  entry:
 ; TUNIT-NEXT:    store i32 1, ptr addrspace(3) @ReachableNonKernel, align 4
 ; TUNIT-NEXT:    [[TMP0:%.*]] = addrspacecast ptr [[ADDR]] to ptr addrspace(5)
@@ -270,7 +270,7 @@ define internal void @level2all_early(ptr %addr) {
 ;
 ; CGSCC: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(write)
 ; CGSCC-LABEL: define {{[^@]+}}@level2all_early
-; CGSCC-SAME: (ptr nocapture nofree noundef nonnull writeonly align 4 dereferenceable(4) [[ADDR:%.*]]) #[[ATTR2]] {
+; CGSCC-SAME: (ptr nofree noundef nonnull writeonly align 4 captures(none) dereferenceable(4) [[ADDR:%.*]]) #[[ATTR2]] {
 ; CGSCC-NEXT:  entry:
 ; CGSCC-NEXT:    store i32 1, ptr addrspace(3) @ReachableNonKernel, align 4
 ; CGSCC-NEXT:    store i32 17, ptr [[ADDR]], align 4
@@ -294,7 +294,7 @@ define internal void @level2a(ptr %addr) {
 ;
 ; CGSCC: Function Attrs: nosync nounwind
 ; CGSCC-LABEL: define {{[^@]+}}@level2a
-; CGSCC-SAME: (ptr nocapture nofree noundef nonnull readonly align 4 dereferenceable(4) [[ADDR:%.*]]) #[[ATTR3]] {
+; CGSCC-SAME: (ptr nofree noundef nonnull readonly align 4 captures(none) dereferenceable(4) [[ADDR:%.*]]) #[[ATTR3]] {
 ; CGSCC-NEXT:  entry:
 ; CGSCC-NEXT:    [[TMP0:%.*]] = load i32, ptr addrspace(3) @ReachableNonKernel, align 4
 ; CGSCC-NEXT:    [[TMP1:%.*]] = load i32, ptr addrspace(3) @UnreachableNonKernel, align 4
@@ -322,7 +322,7 @@ define internal void @level2b(ptr %addr) {
 ;
 ; CGSCC: Function Attrs: nosync nounwind
 ; CGSCC-LABEL: define {{[^@]+}}@level2b
-; CGSCC-SAME: (ptr nocapture nofree noundef nonnull readonly align 4 dereferenceable(4) [[ADDR:%.*]]) #[[ATTR3]] {
+; CGSCC-SAME: (ptr nofree noundef nonnull readonly align 4 captures(none) dereferenceable(4) [[ADDR:%.*]]) #[[ATTR3]] {
 ; CGSCC-NEXT:  entry:
 ; CGSCC-NEXT:    [[TMP0:%.*]] = load i32, ptr addrspace(3) @ReachableNonKernel, align 4
 ; CGSCC-NEXT:    [[TMP1:%.*]] = load i32, ptr addrspace(3) @UnreachableNonKernel, align 4
@@ -341,7 +341,7 @@ entry:
 define internal void @level2all_late(ptr %addr) {
 ; TUNIT: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(write)
 ; TUNIT-LABEL: define {{[^@]+}}@level2all_late
-; TUNIT-SAME: (ptr nocapture nofree noundef nonnull writeonly align 4 dereferenceable(4) [[ADDR:%.*]]) #[[ATTR2]] {
+; TUNIT-SAME: (ptr nofree noundef nonnull writeonly align 4 captures(none) dereferenceable(4) [[ADDR:%.*]]) #[[ATTR2]] {
 ; TUNIT-NEXT:  entry:
 ; TUNIT-NEXT:    store i32 1, ptr addrspace(3) @UnreachableNonKernel, align 4
 ; TUNIT-NEXT:    [[TMP0:%.*]] = addrspacecast ptr [[ADDR]] to ptr addrspace(5)
@@ -349,7 +349,7 @@ define internal void @level2all_late(ptr %addr) {
 ;
 ; CGSCC: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(write)
 ; CGSCC-LABEL: define {{[^@]+}}@level2all_late
-; CGSCC-SAME: (ptr nocapture nofree noundef nonnull writeonly align 4 dereferenceable(4) [[ADDR:%.*]]) #[[ATTR2]] {
+; CGSCC-SAME: (ptr nofree noundef nonnull writeonly align 4 captures(none) dereferenceable(4) [[ADDR:%.*]]) #[[ATTR2]] {
 ; CGSCC-NEXT:  entry:
 ; CGSCC-NEXT:    store i32 1, ptr addrspace(3) @UnreachableNonKernel, align 4
 ; CGSCC-NEXT:    store i32 5, ptr [[ADDR]], align 4
diff --git a/llvm/test/Transforms/Attributor/value-simplify-instances.ll b/llvm/test/Transforms/Attributor/value-simplify-instances.ll
index d1675fdc9a06f..f1dbaf0563e77 100644
--- a/llvm/test/Transforms/Attributor/value-simplify-instances.ll
+++ b/llvm/test/Transforms/Attributor/value-simplify-instances.ll
@@ -152,7 +152,7 @@ define i1 @recursive_alloca_compare_caller(i1 %c) {
 define internal i8 @recursive_alloca_load_return(i1 %c, ptr %p, i8 %v) {
 ; TUNIT: Function Attrs: nofree nosync nounwind memory(argmem: readwrite)
 ; TUNIT-LABEL: define {{[^@]+}}@recursive_alloca_load_return
-; TUNIT-SAME: (i1 noundef [[C:%.*]], ptr noalias nocapture nofree readonly [[P:%.*]], i8 noundef [[V:%.*]]) #[[ATTR3:[0-9]+]] {
+; TUNIT-SAME: (i1 noundef [[C:%.*]], ptr noalias nofree readonly captures(none) [[P:%.*]], i8 noundef [[V:%.*]]) #[[ATTR3:[0-9]+]] {
 ; TUNIT-NEXT:    [[A:%.*]] = alloca i8, align 1
 ; TUNIT-NEXT:    store i8 [[V]], ptr [[A]], align 1
 ; TUNIT-NEXT:    br i1 [[C]], label [[T:%.*]], label [[F:%.*]]
@@ -161,12 +161,12 @@ define internal i8 @recursive_alloca_load_return(i1 %c, ptr %p, i8 %v) {
 ; TUNIT-NEXT:    [[L:%.*]] = load i8, ptr [[P]], align 1
 ; TUNIT-NEXT:    ret i8 [[L]]
 ; TUNIT:       f:
-; TUNIT-NEXT:    [[CALL:%.*]] = call i8 @recursive_alloca_load_return(i1 noundef true, ptr noalias nocapture nofree noundef nonnull readonly dereferenceable(1) [[A]], i8 noundef 1) #[[ATTR4:[0-9]+]]
+; TUNIT-NEXT:    [[CALL:%.*]] = call i8 @recursive_alloca_load_return(i1 noundef true, ptr noalias nofree noundef nonnull readonly captures(none) dereferenceable(1) [[A]], i8 noundef 1) #[[ATTR4:[0-9]+]]
 ; TUNIT-NEXT:    ret i8 [[CALL]]
 ;
 ; CGSCC: Function Attrs: nofree nosync nounwind memory(argmem: readwrite)
 ; CGSCC-LABEL: define {{[^@]+}}@recursive_alloca_load_return
-; CGSCC-SAME: (i1 noundef [[C:%.*]], ptr noalias nocapture nofree readonly [[P:%.*]], i8 noundef [[V:%.*]]) #[[ATTR2:[0-9]+]] {
+; CGSCC-SAME: (i1 noundef [[C:%.*]], ptr noalias nofree readonly captures(none) [[P:%.*]], i8 noundef [[V:%.*]]) #[[ATTR2:[0-9]+]] {
 ; CGSCC-NEXT:    [[A:%.*]] = alloca i8, align 1
 ; CGSCC-NEXT:    store i8 [[V]], ptr [[A]], align 1
 ; CGSCC-NEXT:    br i1 [[C]], label [[T:%.*]], label [[F:%.*]]
@@ -175,7 +175,7 @@ define internal i8 @recursive_alloca_load_return(i1 %c, ptr %p, i8 %v) {
 ; CGSCC-NEXT:    [[L:%.*]] = load i8, ptr [[P]], align 1
 ; CGSCC-NEXT:    ret i8 [[L]]
 ; CGSCC:       f:
-; CGSCC-NEXT:    [[CALL:%.*]] = call i8 @recursive_alloca_load_return(i1 noundef true, ptr noalias nocapture nofree noundef nonnull readonly dereferenceable(1) [[A]], i8 noundef 1) #[[ATTR3:[0-9]+]]
+; CGSCC-NEXT:    [[CALL:%.*]] = call i8 @recursive_alloca_load_return(i1 noundef true, ptr noalias nofree noundef nonnull readonly captures(none) dereferenceable(1) [[A]], i8 noundef 1) #[[ATTR3:[0-9]+]]
 ; CGSCC-NEXT:    ret i8 [[CALL]]
 ;
   %a = alloca i8
@@ -389,7 +389,7 @@ define i1 @recursive_inst_compare_caller_global3(i1 %c) {
 define i32 @non_unique_phi_ops(ptr %ptr) {
 ; TUNIT: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: read)
 ; TUNIT-LABEL: define {{[^@]+}}@non_unique_phi_ops
-; TUNIT-SAME: (ptr nocapture nofree readonly [[PTR:%.*]]) #[[ATTR6:[0-9]+]] {
+; TUNIT-SAME: (ptr nofree readonly captures(none) [[PTR:%.*]]) #[[ATTR6:[0-9]+]] {
 ; TUNIT-NEXT:  entry:
 ; TUNIT-NEXT:    br label [[HEADER:%.*]]
 ; TUNIT:       header:
@@ -411,7 +411,7 @@ define i32 @non_unique_phi_ops(ptr %ptr) {
 ;
 ; CGSCC: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: read)
 ; CGSCC-LABEL: define {{[^@]+}}@non_unique_phi_ops
-; CGSCC-SAME: (ptr nocapture nofree readonly [[PTR:%.*]]) #[[ATTR4:[0-9]+]] {
+; CGSCC-SAME: (ptr nofree readonly captures(none) [[PTR:%.*]]) #[[ATTR4:[0-9]+]] {
 ; CGSCC-NEXT:  entry:
 ; CGSCC-NEXT:    br label [[HEADER:%.*]]
 ; CGSCC:       header:
diff --git a/llvm/test/Transforms/Attributor/value-simplify-local-remote.ll b/llvm/test/Transforms/Attributor/value-simplify-local-remote.ll
index 9ff51e97d6e15..e349d927611d9 100644
--- a/llvm/test/Transforms/Attributor/value-simplify-local-remote.ll
+++ b/llvm/test/Transforms/Attributor/value-simplify-local-remote.ll
@@ -15,7 +15,7 @@ target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16
 define i64 @t1(ptr %first, ptr %first.addr, ptr %0) {
 ; TUNIT: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: write)
 ; TUNIT-LABEL: define {{[^@]+}}@t1
-; TUNIT-SAME: (ptr nofree noundef nonnull writeonly align 8 dereferenceable(8) [[FIRST:%.*]], ptr nocapture nofree readnone [[FIRST_ADDR:%.*]], ptr nocapture nofree readnone [[TMP0:%.*]]) #[[ATTR0:[0-9]+]] {
+; TUNIT-SAME: (ptr nofree noundef nonnull writeonly align 8 dereferenceable(8) [[FIRST:%.*]], ptr nofree readnone captures(none) [[FIRST_ADDR:%.*]], ptr nofree readnone captures(none) [[TMP0:%.*]]) #[[ATTR0:[0-9]+]] {
 ; TUNIT-NEXT:  entry:
 ; TUNIT-NEXT:    [[FIRST_ADDR1:%.*]] = alloca ptr, i32 0, align 8
 ; TUNIT-NEXT:    store ptr [[FIRST]], ptr [[FIRST]], align 8
@@ -26,7 +26,7 @@ define i64 @t1(ptr %first, ptr %first.addr, ptr %0) {
 ;
 ; CGSCC: Function Attrs: mustprogress nofree nosync nounwind willreturn memory(argmem: write)
 ; CGSCC-LABEL: define {{[^@]+}}@t1
-; CGSCC-SAME: (ptr nofree noundef nonnull writeonly align 8 dereferenceable(8) [[FIRST:%.*]], ptr nocapture nofree readnone [[FIRST_ADDR:%.*]], ptr nocapture nofree readnone [[TMP0:%.*]]) #[[ATTR0:[0-9]+]] {
+; CGSCC-SAME: (ptr nofree noundef nonnull writeonly align 8 dereferenceable(8) [[FIRST:%.*]], ptr nofree readnone captures(none) [[FIRST_ADDR:%.*]], ptr nofree readnone captures(none) [[TMP0:%.*]]) #[[ATTR0:[0-9]+]] {
 ; CGSCC-NEXT:  entry:
 ; CGSCC-NEXT:    [[FIRST_ADDR1:%.*]] = alloca ptr, i32 0, align 8
 ; CGSCC-NEXT:    store ptr [[FIRST]], ptr [[FIRST]], align 8
@@ -103,7 +103,7 @@ while.end:                                        ; preds = %while.cond
 define ptr @t2(ptr %this, ptr %this.addr, ptr %this1) {
 ; TUNIT: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: readwrite)
 ; TUNIT-LABEL: define {{[^@]+}}@t2
-; TUNIT-SAME: (ptr nofree noundef nonnull align 8 dereferenceable(8) [[THIS:%.*]], ptr nocapture nofree readnone [[THIS_ADDR:%.*]], ptr nocapture nofree readnone [[THIS1:%.*]]) #[[ATTR1:[0-9]+]] {
+; TUNIT-SAME: (ptr nofree noundef nonnull align 8 dereferenceable(8) [[THIS:%.*]], ptr nofree readnone captures(none) [[THIS_ADDR:%.*]], ptr nofree readnone captures(none) [[THIS1:%.*]]) #[[ATTR1:[0-9]+]] {
 ; TUNIT-NEXT:  entry:
 ; TUNIT-NEXT:    store ptr [[THIS]], ptr [[THIS]], align 8
 ; TUNIT-NEXT:    [[CALL:%.*]] = call [[S:%.*]] @[[FOO_1:[a-zA-Z0-9_$\"\\.-]*[a-zA-Z_$\"\\.-][a-zA-Z0-9_$\"\\.-]*]](ptr nofree noundef nonnull align 8 dereferenceable(8) [[THIS]]) #[[ATTR4:[0-9]+]]
@@ -112,7 +112,7 @@ define ptr @t2(ptr %this, ptr %this.addr, ptr %this1) {
 ;
 ; CGSCC: Function Attrs: mustprogress nofree nosync nounwind willreturn memory(argmem: readwrite)
 ; CGSCC-LABEL: define {{[^@]+}}@t2
-; CGSCC-SAME: (ptr nofree noundef nonnull align 8 dereferenceable(8) [[THIS:%.*]], ptr nocapture nofree readnone [[THIS_ADDR:%.*]], ptr nocapture nofree readnone [[THIS1:%.*]]) #[[ATTR2:[0-9]+]] {
+; CGSCC-SAME: (ptr nofree noundef nonnull align 8 dereferenceable(8) [[THIS:%.*]], ptr nofree readnone captures(none) [[THIS_ADDR:%.*]], ptr nofree readnone captures(none) [[THIS1:%.*]]) #[[ATTR2:[0-9]+]] {
 ; CGSCC-NEXT:  entry:
 ; CGSCC-NEXT:    store ptr [[THIS]], ptr [[THIS]], align 8
 ; CGSCC-NEXT:    [[CALL:%.*]] = call [[S:%.*]] @[[FOO_1:[a-zA-Z0-9_$\"\\.-]*[a-zA-Z_$\"\\.-][a-zA-Z0-9_$\"\\.-]*]](ptr nofree noundef nonnull align 8 dereferenceable(8) [[THIS]]) #[[ATTR8:[0-9]+]]
@@ -134,7 +134,7 @@ define internal %S @foo.1(ptr %foo.this) {
 ; TUNIT-NEXT:  entry:
 ; TUNIT-NEXT:    [[RETVAL:%.*]] = alloca [[S:%.*]], i32 0, align 8
 ; TUNIT-NEXT:    store ptr [[FOO_THIS]], ptr [[FOO_THIS]], align 8
-; TUNIT-NEXT:    call void @bar.2(ptr noalias nocapture nofree noundef nonnull writeonly align 8 [[RETVAL]], ptr nofree noundef nonnull writeonly align 8 dereferenceable(8) [[FOO_THIS]]) #[[ATTR5:[0-9]+]]
+; TUNIT-NEXT:    call void @bar.2(ptr noalias nofree noundef nonnull writeonly align 8 captures(none) [[RETVAL]], ptr nofree noundef nonnull writeonly align 8 dereferenceable(8) [[FOO_THIS]]) #[[ATTR5:[0-9]+]]
 ; TUNIT-NEXT:    [[FOO_RET:%.*]] = load [[S]], ptr [[RETVAL]], align 8
 ; TUNIT-NEXT:    ret [[S]] [[FOO_RET]]
 ;
@@ -144,7 +144,7 @@ define internal %S @foo.1(ptr %foo.this) {
 ; CGSCC-NEXT:  entry:
 ; CGSCC-NEXT:    [[RETVAL:%.*]] = alloca [[S:%.*]], i32 0, align 8
 ; CGSCC-NEXT:    store ptr [[FOO_THIS]], ptr [[FOO_THIS]], align 8
-; CGSCC-NEXT:    call void @bar.2(ptr noalias nocapture nofree noundef nonnull writeonly align 8 dereferenceable(8) [[RETVAL]], ptr nofree noundef nonnull writeonly align 8 dereferenceable(8) [[FOO_THIS]]) #[[ATTR6]]
+; CGSCC-NEXT:    call void @bar.2(ptr noalias nofree noundef nonnull writeonly align 8 captures(none) dereferenceable(8) [[RETVAL]], ptr nofree noundef nonnull writeonly align 8 dereferenceable(8) [[FOO_THIS]]) #[[ATTR6]]
 ; CGSCC-NEXT:    [[FOO_RET:%.*]] = load [[S]], ptr [[RETVAL]], align 8
 ; CGSCC-NEXT:    ret [[S]] [[FOO_RET]]
 ;
@@ -159,18 +159,18 @@ entry:
 define internal void @bar.2(ptr %bar.this, ptr %bar.data) {
 ; TUNIT: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: write)
 ; TUNIT-LABEL: define {{[^@]+}}@bar.2
-; TUNIT-SAME: (ptr noalias nocapture nofree noundef nonnull writeonly align 8 dereferenceable(8) [[BAR_THIS:%.*]], ptr nofree noundef nonnull writeonly align 8 dereferenceable(8) [[BAR_DATA:%.*]]) #[[ATTR0]] {
+; TUNIT-SAME: (ptr noalias nofree noundef nonnull writeonly align 8 captures(none) dereferenceable(8) [[BAR_THIS:%.*]], ptr nofree noundef nonnull writeonly align 8 dereferenceable(8) [[BAR_DATA:%.*]]) #[[ATTR0]] {
 ; TUNIT-NEXT:  entry:
 ; TUNIT-NEXT:    store ptr [[BAR_DATA]], ptr [[BAR_THIS]], align 8
-; TUNIT-NEXT:    call void @baz(ptr nocapture nofree noundef nonnull writeonly align 8 dereferenceable(8) [[BAR_THIS]], ptr nofree noundef nonnull writeonly align 8 dereferenceable(8) [[BAR_DATA]]) #[[ATTR5]]
+; TUNIT-NEXT:    call void @baz(ptr nofree noundef nonnull writeonly align 8 captures(none) dereferenceable(8) [[BAR_THIS]], ptr nofree noundef nonnull writeonly align 8 dereferenceable(8) [[BAR_DATA]]) #[[ATTR5]]
 ; TUNIT-NEXT:    ret void
 ;
 ; CGSCC: Function Attrs: mustprogress nofree nosync nounwind willreturn memory(argmem: write)
 ; CGSCC-LABEL: define {{[^@]+}}@bar.2
-; CGSCC-SAME: (ptr noalias nocapture nofree noundef nonnull writeonly align 8 dereferenceable(8) [[BAR_THIS:%.*]], ptr nofree noundef nonnull writeonly align 8 dereferenceable(8) [[BAR_DATA:%.*]]) #[[ATTR0]] {
+; CGSCC-SAME: (ptr noalias nofree noundef nonnull writeonly align 8 captures(none) dereferenceable(8) [[BAR_THIS:%.*]], ptr nofree noundef nonnull writeonly align 8 dereferenceable(8) [[BAR_DATA:%.*]]) #[[ATTR0]] {
 ; CGSCC-NEXT:  entry:
 ; CGSCC-NEXT:    store ptr [[BAR_DATA]], ptr [[BAR_THIS]], align 8
-; CGSCC-NEXT:    call void @baz(ptr nocapture nofree noundef nonnull writeonly align 8 dereferenceable(8) [[BAR_THIS]], ptr nofree noundef nonnull writeonly align 8 dereferenceable(8) [[BAR_DATA]]) #[[ATTR6]]
+; CGSCC-NEXT:    call void @baz(ptr nofree noundef nonnull writeonly align 8 captures(none) dereferenceable(8) [[BAR_THIS]], ptr nofree noundef nonnull writeonly align 8 dereferenceable(8) [[BAR_DATA]]) #[[ATTR6]]
 ; CGSCC-NEXT:    ret void
 ;
 entry:
@@ -182,14 +182,14 @@ entry:
 define internal void @baz(ptr %baz.this, ptr %baz.data) {
 ; TUNIT: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: write)
 ; TUNIT-LABEL: define {{[^@]+}}@baz
-; TUNIT-SAME: (ptr nocapture nofree noundef nonnull writeonly align 8 dereferenceable(8) [[BAZ_THIS:%.*]], ptr nofree noundef nonnull writeonly align 8 dereferenceable(8) [[BAZ_DATA:%.*]]) #[[ATTR0]] {
+; TUNIT-SAME: (ptr nofree noundef nonnull writeonly align 8 captures(none) dereferenceable(8) [[BAZ_THIS:%.*]], ptr nofree noundef nonnull writeonly align 8 dereferenceable(8) [[BAZ_DATA:%.*]]) #[[ATTR0]] {
 ; TUNIT-NEXT:  entry:
 ; TUNIT-NEXT:    store ptr [[BAZ_DATA]], ptr [[BAZ_THIS]], align 8
 ; TUNIT-NEXT:    ret void
 ;
 ; CGSCC: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: write)
 ; CGSCC-LABEL: define {{[^@]+}}@baz
-; CGSCC-SAME: (ptr nocapture nofree noundef nonnull writeonly align 8 dereferenceable(8) [[BAZ_THIS:%.*]], ptr nofree writeonly [[BAZ_DATA:%.*]]) #[[ATTR3:[0-9]+]] {
+; CGSCC-SAME: (ptr nofree noundef nonnull writeonly align 8 captures(none) dereferenceable(8) [[BAZ_THIS:%.*]], ptr nofree writeonly [[BAZ_DATA:%.*]]) #[[ATTR3:[0-9]+]] {
 ; CGSCC-NEXT:  entry:
 ; CGSCC-NEXT:    store ptr [[BAZ_DATA]], ptr [[BAZ_THIS]], align 8
 ; CGSCC-NEXT:    ret void
@@ -202,7 +202,7 @@ entry:
 define ptr @foo(ptr %this, ptr %this.addr, ptr %this1) {
 ; TUNIT: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: readwrite)
 ; TUNIT-LABEL: define {{[^@]+}}@foo
-; TUNIT-SAME: (ptr nofree noundef nonnull align 8 dereferenceable(8) [[THIS:%.*]], ptr nocapture nofree readnone [[THIS_ADDR:%.*]], ptr nocapture nofree readnone [[THIS1:%.*]]) #[[ATTR1]] {
+; TUNIT-SAME: (ptr nofree noundef nonnull align 8 dereferenceable(8) [[THIS:%.*]], ptr nofree readnone captures(none) [[THIS_ADDR:%.*]], ptr nofree readnone captures(none) [[THIS1:%.*]]) #[[ATTR1]] {
 ; TUNIT-NEXT:  entry:
 ; TUNIT-NEXT:    store ptr [[THIS]], ptr [[THIS]], align 8
 ; TUNIT-NEXT:    [[CALL:%.*]] = call [[S:%.*]] @[[BAR_5:[a-zA-Z0-9_$\"\\.-]*[a-zA-Z_$\"\\.-][a-zA-Z0-9_$\"\\.-]*]](ptr nofree noundef nonnull align 8 dereferenceable(8) [[THIS]]) #[[ATTR4]]
@@ -211,7 +211,7 @@ define ptr @foo(ptr %this, ptr %this.addr, ptr %this1) {
 ;
 ; CGSCC: Function Attrs: mustprogress nofree nosync nounwind willreturn memory(argmem: readwrite)
 ; CGSCC-LABEL: define {{[^@]+}}@foo
-; CGSCC-SAME: (ptr nofree noundef nonnull align 8 dereferenceable(8) [[THIS:%.*]], ptr nocapture nofree readnone [[THIS_ADDR:%.*]], ptr nocapture nofree readnone [[THIS1:%.*]]) #[[ATTR2]] {
+; CGSCC-SAME: (ptr nofree noundef nonnull align 8 dereferenceable(8) [[THIS:%.*]], ptr nofree readnone captures(none) [[THIS_ADDR:%.*]], ptr nofree readnone captures(none) [[THIS1:%.*]]) #[[ATTR2]] {
 ; CGSCC-NEXT:  entry:
 ; CGSCC-NEXT:    store ptr [[THIS]], ptr [[THIS]], align 8
 ; CGSCC-NEXT:    [[CALL:%.*]] = call [[S:%.*]] @[[BAR_5:[a-zA-Z0-9_$\"\\.-]*[a-zA-Z_$\"\\.-][a-zA-Z0-9_$\"\\.-]*]](ptr nofree noundef nonnull align 8 dereferenceable(8) [[THIS]]) #[[ATTR8]]
@@ -233,7 +233,7 @@ define internal %S @bar.5(ptr %this) {
 ; TUNIT-NEXT:  entry:
 ; TUNIT-NEXT:    [[RETVAL:%.*]] = alloca [[S:%.*]], i32 0, align 8
 ; TUNIT-NEXT:    store ptr [[THIS]], ptr [[THIS]], align 8
-; TUNIT-NEXT:    call void @baz.6(ptr noalias nocapture nofree noundef nonnull writeonly align 8 [[RETVAL]], ptr nofree noundef nonnull align 8 dereferenceable(8) [[THIS]]) #[[ATTR4]]
+; TUNIT-NEXT:    call void @baz.6(ptr noalias nofree noundef nonnull writeonly align 8 captures(none) [[RETVAL]], ptr nofree noundef nonnull align 8 dereferenceable(8) [[THIS]]) #[[ATTR4]]
 ; TUNIT-NEXT:    [[BAR_RET:%.*]] = load [[S]], ptr [[RETVAL]], align 8
 ; TUNIT-NEXT:    ret [[S]] [[BAR_RET]]
 ;
@@ -243,7 +243,7 @@ define internal %S @bar.5(ptr %this) {
 ; CGSCC-NEXT:  entry:
 ; CGSCC-NEXT:    [[RETVAL:%.*]] = alloca [[S:%.*]], i32 0, align 8
 ; CGSCC-NEXT:    store ptr [[THIS]], ptr [[THIS]], align 8
-; CGSCC-NEXT:    call void @baz.6(ptr noalias nocapture nofree noundef nonnull writeonly align 8 dereferenceable(8) [[RETVAL]], ptr nofree noundef nonnull align 8 dereferenceable(8) [[THIS]]) #[[ATTR9:[0-9]+]]
+; CGSCC-NEXT:    call void @baz.6(ptr noalias nofree noundef nonnull writeonly align 8 captures(none) dereferenceable(8) [[RETVAL]], ptr nofree noundef nonnull align 8 dereferenceable(8) [[THIS]]) #[[ATTR9:[0-9]+]]
 ; CGSCC-NEXT:    [[BAR_RET:%.*]] = load [[S]], ptr [[RETVAL]], align 8
 ; CGSCC-NEXT:    ret [[S]] [[BAR_RET]]
 ;
@@ -259,18 +259,18 @@ entry:
 define internal void @baz.6(ptr %this, ptr %data) {
 ; TUNIT: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: readwrite)
 ; TUNIT-LABEL: define {{[^@]+}}@baz.6
-; TUNIT-SAME: (ptr noalias nocapture nofree noundef nonnull writeonly align 8 dereferenceable(8) [[THIS:%.*]], ptr nofree noundef nonnull align 8 dereferenceable(8) [[DATA:%.*]]) #[[ATTR1]] {
+; TUNIT-SAME: (ptr noalias nofree noundef nonnull writeonly align 8 captures(none) dereferenceable(8) [[THIS:%.*]], ptr nofree noundef nonnull align 8 dereferenceable(8) [[DATA:%.*]]) #[[ATTR1]] {
 ; TUNIT-NEXT:  entry:
 ; TUNIT-NEXT:    store ptr [[DATA]], ptr [[THIS]], align 8
-; TUNIT-NEXT:    call void @boom(ptr nocapture nofree noundef nonnull writeonly align 8 dereferenceable(8) [[THIS]], ptr nofree noundef nonnull align 8 dereferenceable(8) [[DATA]]) #[[ATTR4]]
+; TUNIT-NEXT:    call void @boom(ptr nofree noundef nonnull writeonly align 8 captures(none) dereferenceable(8) [[THIS]], ptr nofree noundef nonnull align 8 dereferenceable(8) [[DATA]]) #[[ATTR4]]
 ; TUNIT-NEXT:    ret void
 ;
 ; CGSCC: Function Attrs: mustprogress nofree nosync nounwind willreturn memory(argmem: readwrite)
 ; CGSCC-LABEL: define {{[^@]+}}@baz.6
-; CGSCC-SAME: (ptr noalias nocapture nofree noundef nonnull writeonly align 8 dereferenceable(8) [[THIS:%.*]], ptr nofree noundef nonnull align 8 dereferenceable(8) [[DATA:%.*]]) #[[ATTR2]] {
+; CGSCC-SAME: (ptr noalias nofree noundef nonnull writeonly align 8 captures(none) dereferenceable(8) [[THIS:%.*]], ptr nofree noundef nonnull align 8 dereferenceable(8) [[DATA:%.*]]) #[[ATTR2]] {
 ; CGSCC-NEXT:  entry:
 ; CGSCC-NEXT:    store ptr [[DATA]], ptr [[THIS]], align 8
-; CGSCC-NEXT:    call void @boom(ptr nocapture nofree noundef nonnull writeonly align 8 dereferenceable(8) [[THIS]], ptr nofree noundef nonnull align 8 dereferenceable(8) [[DATA]]) #[[ATTR9]]
+; CGSCC-NEXT:    call void @boom(ptr nofree noundef nonnull writeonly align 8 captures(none) dereferenceable(8) [[THIS]], ptr nofree noundef nonnull align 8 dereferenceable(8) [[DATA]]) #[[ATTR9]]
 ; CGSCC-NEXT:    ret void
 ;
 entry:
@@ -282,7 +282,7 @@ entry:
 define internal void @boom(ptr %this, ptr %data) {
 ; TUNIT: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: readwrite)
 ; TUNIT-LABEL: define {{[^@]+}}@boom
-; TUNIT-SAME: (ptr nocapture nofree noundef nonnull writeonly align 8 dereferenceable(8) [[THIS:%.*]], ptr nofree noundef nonnull align 8 dereferenceable(8) [[DATA:%.*]]) #[[ATTR1]] {
+; TUNIT-SAME: (ptr nofree noundef nonnull writeonly align 8 captures(none) dereferenceable(8) [[THIS:%.*]], ptr nofree noundef nonnull align 8 dereferenceable(8) [[DATA:%.*]]) #[[ATTR1]] {
 ; TUNIT-NEXT:  entry:
 ; TUNIT-NEXT:    [[DATA_ADDR:%.*]] = alloca ptr, i32 0, align 8
 ; TUNIT-NEXT:    store ptr [[DATA]], ptr [[DATA_ADDR]], align 8
@@ -292,7 +292,7 @@ define internal void @boom(ptr %this, ptr %data) {
 ;
 ; CGSCC: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: readwrite)
 ; CGSCC-LABEL: define {{[^@]+}}@boom
-; CGSCC-SAME: (ptr nocapture nofree noundef nonnull writeonly align 8 dereferenceable(8) [[THIS:%.*]], ptr nofree [[DATA:%.*]]) #[[ATTR4:[0-9]+]] {
+; CGSCC-SAME: (ptr nofree noundef nonnull writeonly align 8 captures(none) dereferenceable(8) [[THIS:%.*]], ptr nofree [[DATA:%.*]]) #[[ATTR4:[0-9]+]] {
 ; CGSCC-NEXT:  entry:
 ; CGSCC-NEXT:    [[DATA_ADDR:%.*]] = alloca ptr, i32 0, align 8
 ; CGSCC-NEXT:    store ptr [[DATA]], ptr [[DATA_ADDR]], align 8
@@ -463,7 +463,7 @@ bb35:                                             ; preds = %bb16
 define double @t4(ptr %this, ptr %this.addr, ptr %this1) {
 ; TUNIT: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: write)
 ; TUNIT-LABEL: define {{[^@]+}}@t4
-; TUNIT-SAME: (ptr nofree noundef nonnull writeonly align 8 dereferenceable(8) [[THIS:%.*]], ptr nocapture nofree readnone [[THIS_ADDR:%.*]], ptr nocapture nofree readnone [[THIS1:%.*]]) #[[ATTR0]] {
+; TUNIT-SAME: (ptr nofree noundef nonnull writeonly align 8 dereferenceable(8) [[THIS:%.*]], ptr nofree readnone captures(none) [[THIS_ADDR:%.*]], ptr nofree readnone captures(none) [[THIS1:%.*]]) #[[ATTR0]] {
 ; TUNIT-NEXT:  entry:
 ; TUNIT-NEXT:    [[THIS_ADDR1:%.*]] = alloca ptr, i32 0, align 8
 ; TUNIT-NEXT:    store ptr [[THIS]], ptr [[THIS]], align 8
@@ -472,7 +472,7 @@ define double @t4(ptr %this, ptr %this.addr, ptr %this1) {
 ;
 ; CGSCC: Function Attrs: mustprogress nofree nosync nounwind willreturn memory(argmem: readwrite)
 ; CGSCC-LABEL: define {{[^@]+}}@t4
-; CGSCC-SAME: (ptr nofree noundef nonnull align 8 dereferenceable(8) [[THIS:%.*]], ptr nocapture nofree readnone [[THIS_ADDR:%.*]], ptr nocapture nofree readnone [[THIS1:%.*]]) #[[ATTR2]] {
+; CGSCC-SAME: (ptr nofree noundef nonnull align 8 dereferenceable(8) [[THIS:%.*]], ptr nofree readnone captures(none) [[THIS_ADDR:%.*]], ptr nofree readnone captures(none) [[THIS1:%.*]]) #[[ATTR2]] {
 ; CGSCC-NEXT:  entry:
 ; CGSCC-NEXT:    [[THIS_ADDR1:%.*]] = alloca ptr, i32 0, align 8
 ; CGSCC-NEXT:    store ptr [[THIS]], ptr [[THIS]], align 8
@@ -497,7 +497,7 @@ define internal %S @t4a(ptr %this) {
 ; TUNIT-NEXT:    [[RETVAL:%.*]] = alloca [[S:%.*]], i32 0, align 8
 ; TUNIT-NEXT:    [[THIS_ADDR:%.*]] = alloca ptr, i32 0, align 8
 ; TUNIT-NEXT:    store ptr [[THIS]], ptr [[THIS]], align 8
-; TUNIT-NEXT:    call void @t4b(ptr noalias nocapture nofree noundef nonnull writeonly align 8 [[RETVAL]]) #[[ATTR5]]
+; TUNIT-NEXT:    call void @t4b(ptr noalias nofree noundef nonnull writeonly align 8 captures(none) [[RETVAL]]) #[[ATTR5]]
 ; TUNIT-NEXT:    ret [[S]] undef
 ;
 ; CGSCC: Function Attrs: mustprogress nofree nosync nounwind willreturn memory(argmem: readwrite)
@@ -507,7 +507,7 @@ define internal %S @t4a(ptr %this) {
 ; CGSCC-NEXT:    [[RETVAL:%.*]] = alloca [[S:%.*]], i32 0, align 8
 ; CGSCC-NEXT:    [[THIS_ADDR:%.*]] = alloca ptr, i32 0, align 8
 ; CGSCC-NEXT:    store ptr [[THIS]], ptr [[THIS]], align 8
-; CGSCC-NEXT:    call void @t4b(ptr noalias nocapture nofree noundef nonnull writeonly align 8 dereferenceable(8) [[RETVAL]], ptr nofree noundef nonnull writeonly align 8 dereferenceable(8) [[THIS]]) #[[ATTR6]]
+; CGSCC-NEXT:    call void @t4b(ptr noalias nofree noundef nonnull writeonly align 8 captures(none) dereferenceable(8) [[RETVAL]], ptr nofree noundef nonnull writeonly align 8 dereferenceable(8) [[THIS]]) #[[ATTR6]]
 ; CGSCC-NEXT:    [[TMP0:%.*]] = load [[S]], ptr [[RETVAL]], align 8
 ; CGSCC-NEXT:    ret [[S]] [[TMP0]]
 ;
@@ -527,21 +527,21 @@ entry:
 define internal void @t4b(ptr %this, ptr %data) {
 ; TUNIT: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: write)
 ; TUNIT-LABEL: define {{[^@]+}}@t4b
-; TUNIT-SAME: (ptr noalias nocapture nofree noundef nonnull writeonly align 8 dereferenceable(8) [[THIS:%.*]]) #[[ATTR0]] {
+; TUNIT-SAME: (ptr noalias nofree noundef nonnull writeonly align 8 captures(none) dereferenceable(8) [[THIS:%.*]]) #[[ATTR0]] {
 ; TUNIT-NEXT:  entry:
 ; TUNIT-NEXT:    [[THIS_ADDR:%.*]] = alloca ptr, i32 0, align 8
 ; TUNIT-NEXT:    [[DATA_ADDR:%.*]] = alloca ptr, i32 0, align 8
-; TUNIT-NEXT:    call void @t4c(ptr noalias nocapture nofree noundef nonnull writeonly align 8 dereferenceable(8) [[THIS]]) #[[ATTR5]]
+; TUNIT-NEXT:    call void @t4c(ptr noalias nofree noundef nonnull writeonly align 8 captures(none) dereferenceable(8) [[THIS]]) #[[ATTR5]]
 ; TUNIT-NEXT:    ret void
 ;
 ; CGSCC: Function Attrs: mustprogress nofree nosync nounwind willreturn memory(argmem: write)
 ; CGSCC-LABEL: define {{[^@]+}}@t4b
-; CGSCC-SAME: (ptr noalias nocapture nofree noundef nonnull writeonly align 8 dereferenceable(8) [[THIS:%.*]], ptr nofree noundef nonnull writeonly align 8 dereferenceable(8) [[DATA:%.*]]) #[[ATTR0]] {
+; CGSCC-SAME: (ptr noalias nofree noundef nonnull writeonly align 8 captures(none) dereferenceable(8) [[THIS:%.*]], ptr nofree noundef nonnull writeonly align 8 dereferenceable(8) [[DATA:%.*]]) #[[ATTR0]] {
 ; CGSCC-NEXT:  entry:
 ; CGSCC-NEXT:    [[THIS_ADDR:%.*]] = alloca ptr, i32 0, align 8
 ; CGSCC-NEXT:    [[DATA_ADDR:%.*]] = alloca ptr, i32 0, align 8
 ; CGSCC-NEXT:    store ptr [[DATA]], ptr [[THIS]], align 8
-; CGSCC-NEXT:    call void @t4c(ptr nocapture nofree noundef nonnull writeonly align 8 dereferenceable(8) [[THIS]], ptr nofree noundef nonnull writeonly align 8 dereferenceable(8) [[DATA]]) #[[ATTR6]]
+; CGSCC-NEXT:    call void @t4c(ptr nofree noundef nonnull writeonly align 8 captures(none) dereferenceable(8) [[THIS]], ptr nofree noundef nonnull writeonly align 8 dereferenceable(8) [[DATA]]) #[[ATTR6]]
 ; CGSCC-NEXT:    ret void
 ;
 entry:
@@ -558,7 +558,7 @@ entry:
 define internal void @t4c(ptr %this, ptr %data) {
 ; TUNIT: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: write)
 ; TUNIT-LABEL: define {{[^@]+}}@t4c
-; TUNIT-SAME: (ptr noalias nocapture nofree noundef nonnull writeonly align 8 dereferenceable(8) [[THIS:%.*]]) #[[ATTR0]] {
+; TUNIT-SAME: (ptr noalias nofree noundef nonnull writeonly align 8 captures(none) dereferenceable(8) [[THIS:%.*]]) #[[ATTR0]] {
 ; TUNIT-NEXT:  entry:
 ; TUNIT-NEXT:    [[THIS_ADDR:%.*]] = alloca ptr, i32 0, align 8
 ; TUNIT-NEXT:    [[DATA_ADDR:%.*]] = alloca ptr, i32 0, align 8
@@ -566,7 +566,7 @@ define internal void @t4c(ptr %this, ptr %data) {
 ;
 ; CGSCC: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: write)
 ; CGSCC-LABEL: define {{[^@]+}}@t4c
-; CGSCC-SAME: (ptr nocapture nofree noundef nonnull writeonly align 8 dereferenceable(8) [[THIS:%.*]], ptr nofree writeonly [[DATA:%.*]]) #[[ATTR3]] {
+; CGSCC-SAME: (ptr nofree noundef nonnull writeonly align 8 captures(none) dereferenceable(8) [[THIS:%.*]], ptr nofree writeonly [[DATA:%.*]]) #[[ATTR3]] {
 ; CGSCC-NEXT:  entry:
 ; CGSCC-NEXT:    [[THIS_ADDR:%.*]] = alloca ptr, i32 0, align 8
 ; CGSCC-NEXT:    [[DATA_ADDR:%.*]] = alloca ptr, i32 0, align 8
diff --git a/llvm/test/Transforms/Attributor/value-simplify-pointer-info.ll b/llvm/test/Transforms/Attributor/value-simplify-pointer-info.ll
index adcee750cae6e..fa942c99fb36f 100644
--- a/llvm/test/Transforms/Attributor/value-simplify-pointer-info.ll
+++ b/llvm/test/Transforms/Attributor/value-simplify-pointer-info.ll
@@ -67,7 +67,7 @@
 define void @write_arg(ptr %p, i32 %v) {
 ; CHECK: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: write)
 ; CHECK-LABEL: define {{[^@]+}}@write_arg
-; CHECK-SAME: (ptr nocapture nofree noundef nonnull writeonly align 4 dereferenceable(4) [[P:%.*]], i32 [[V:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-SAME: (ptr nofree noundef nonnull writeonly align 4 captures(none) dereferenceable(4) [[P:%.*]], i32 [[V:%.*]]) #[[ATTR0:[0-9]+]] {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    store i32 [[V]], ptr [[P]], align 4, !tbaa [[TBAA3:![0-9]+]]
 ; CHECK-NEXT:    ret void
@@ -79,7 +79,7 @@ entry:
 
 define void @write_random(ptr %p) {
 ; CHECK-LABEL: define {{[^@]+}}@write_random
-; CHECK-SAME: (ptr nocapture nofree writeonly [[P:%.*]]) {
+; CHECK-SAME: (ptr nofree writeonly captures(none) [[P:%.*]]) {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[CALL:%.*]] = call i32 (...) @random()
 ; CHECK-NEXT:    store i32 [[CALL]], ptr [[P]], align 4, !tbaa [[TBAA3]]
@@ -113,18 +113,18 @@ declare i32 @random(...)
 define void @local_alloca_simplifiable_1(ptr noalias sret(%struct.S) align 4 %agg.result) {
 ; TUNIT: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: readwrite)
 ; TUNIT-LABEL: define {{[^@]+}}@local_alloca_simplifiable_1
-; TUNIT-SAME: (ptr noalias nocapture nofree writeonly sret([[STRUCT_S:%.*]]) align 4 dereferenceable_or_null(24) [[AGG_RESULT:%.*]]) #[[ATTR1:[0-9]+]] {
+; TUNIT-SAME: (ptr noalias nofree writeonly sret([[STRUCT_S:%.*]]) align 4 captures(none) dereferenceable_or_null(24) [[AGG_RESULT:%.*]]) #[[ATTR1:[0-9]+]] {
 ; TUNIT-NEXT:  entry:
 ; TUNIT-NEXT:    [[S:%.*]] = alloca [[STRUCT_S]], align 4
-; TUNIT-NEXT:    call void @llvm.lifetime.start.p0(i64 noundef 24, ptr noalias nocapture nofree noundef nonnull align 4 dereferenceable(24) [[S]]) #[[ATTR17:[0-9]+]]
+; TUNIT-NEXT:    call void @llvm.lifetime.start.p0(i64 noundef 24, ptr noalias nofree noundef nonnull align 4 captures(none) dereferenceable(24) [[S]]) #[[ATTR17:[0-9]+]]
 ; TUNIT-NEXT:    [[F1:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[S]], i64 0, i32 3
 ; TUNIT-NEXT:    [[F2:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[S]], i64 0, i32 4
 ; TUNIT-NEXT:    [[F3:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[S]], i64 0, i32 5
-; TUNIT-NEXT:    call void @write_arg(ptr noalias nocapture nofree noundef nonnull writeonly align 4 dereferenceable(24) [[S]], i32 noundef 1) #[[ATTR18:[0-9]+]]
+; TUNIT-NEXT:    call void @write_arg(ptr noalias nofree noundef nonnull writeonly align 4 captures(none) dereferenceable(24) [[S]], i32 noundef 1) #[[ATTR18:[0-9]+]]
 ; TUNIT-NEXT:    [[I2:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[S]], i64 0, i32 1
-; TUNIT-NEXT:    call void @write_arg(ptr nocapture nofree noundef nonnull writeonly align 4 dereferenceable(20) [[I2]], i32 noundef 2) #[[ATTR18]]
+; TUNIT-NEXT:    call void @write_arg(ptr nofree noundef nonnull writeonly align 4 captures(none) dereferenceable(20) [[I2]], i32 noundef 2) #[[ATTR18]]
 ; TUNIT-NEXT:    [[I3:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[S]], i64 0, i32 2
-; TUNIT-NEXT:    call void @write_arg(ptr nocapture nofree noundef nonnull writeonly align 4 dereferenceable(16) [[I3]], i32 noundef 3) #[[ATTR18]]
+; TUNIT-NEXT:    call void @write_arg(ptr nofree noundef nonnull writeonly align 4 captures(none) dereferenceable(16) [[I3]], i32 noundef 3) #[[ATTR18]]
 ; TUNIT-NEXT:    [[F12:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[AGG_RESULT]], i64 0, i32 3
 ; TUNIT-NEXT:    store float 0x3FF19999A0000000, ptr [[F12]], align 4, !tbaa [[TBAA7:![0-9]+]]
 ; TUNIT-NEXT:    [[F24:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[AGG_RESULT]], i64 0, i32 4
@@ -136,26 +136,26 @@ define void @local_alloca_simplifiable_1(ptr noalias sret(%struct.S) align 4 %ag
 ; TUNIT-NEXT:    store i32 4, ptr [[I212]], align 4, !tbaa [[TBAA13:![0-9]+]]
 ; TUNIT-NEXT:    [[I316:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[AGG_RESULT]], i64 0, i32 2
 ; TUNIT-NEXT:    store i32 4, ptr [[I316]], align 4, !tbaa [[TBAA14:![0-9]+]]
-; TUNIT-NEXT:    call void @llvm.lifetime.end.p0(i64 noundef 24, ptr noalias nocapture nofree noundef nonnull align 4 dereferenceable(24) [[S]]) #[[ATTR17]]
+; TUNIT-NEXT:    call void @llvm.lifetime.end.p0(i64 noundef 24, ptr noalias nofree noundef nonnull align 4 captures(none) dereferenceable(24) [[S]]) #[[ATTR17]]
 ; TUNIT-NEXT:    ret void
 ;
 ; CGSCC: Function Attrs: mustprogress nofree nosync nounwind willreturn memory(argmem: readwrite)
 ; CGSCC-LABEL: define {{[^@]+}}@local_alloca_simplifiable_1
-; CGSCC-SAME: (ptr noalias nocapture nofree noundef nonnull writeonly sret([[STRUCT_S:%.*]]) align 4 dereferenceable(24) [[AGG_RESULT:%.*]]) #[[ATTR1:[0-9]+]] {
+; CGSCC-SAME: (ptr noalias nofree noundef nonnull writeonly sret([[STRUCT_S:%.*]]) align 4 captures(none) dereferenceable(24) [[AGG_RESULT:%.*]]) #[[ATTR1:[0-9]+]] {
 ; CGSCC-NEXT:  entry:
 ; CGSCC-NEXT:    [[S:%.*]] = alloca [[STRUCT_S]], align 4
-; CGSCC-NEXT:    call void @llvm.lifetime.start.p0(i64 noundef 24, ptr noalias nocapture nofree noundef nonnull align 4 dereferenceable(24) [[S]]) #[[ATTR20:[0-9]+]]
+; CGSCC-NEXT:    call void @llvm.lifetime.start.p0(i64 noundef 24, ptr noalias nofree noundef nonnull align 4 captures(none) dereferenceable(24) [[S]]) #[[ATTR20:[0-9]+]]
 ; CGSCC-NEXT:    [[F1:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[S]], i64 0, i32 3
 ; CGSCC-NEXT:    store float 0x3FF19999A0000000, ptr [[F1]], align 4, !tbaa [[TBAA7:![0-9]+]]
 ; CGSCC-NEXT:    [[F2:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[S]], i64 0, i32 4
 ; CGSCC-NEXT:    store float 0x40019999A0000000, ptr [[F2]], align 4, !tbaa [[TBAA10:![0-9]+]]
 ; CGSCC-NEXT:    [[F3:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[S]], i64 0, i32 5
 ; CGSCC-NEXT:    store float 0x400A666660000000, ptr [[F3]], align 4, !tbaa [[TBAA11:![0-9]+]]
-; CGSCC-NEXT:    call void @write_arg(ptr noalias nocapture nofree noundef nonnull writeonly align 4 dereferenceable(24) [[S]], i32 noundef 1) #[[ATTR21:[0-9]+]]
+; CGSCC-NEXT:    call void @write_arg(ptr noalias nofree noundef nonnull writeonly align 4 captures(none) dereferenceable(24) [[S]], i32 noundef 1) #[[ATTR21:[0-9]+]]
 ; CGSCC-NEXT:    [[I2:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[S]], i64 0, i32 1
-; CGSCC-NEXT:    call void @write_arg(ptr nocapture nofree noundef nonnull writeonly align 4 dereferenceable(20) [[I2]], i32 noundef 2) #[[ATTR21]]
+; CGSCC-NEXT:    call void @write_arg(ptr nofree noundef nonnull writeonly align 4 captures(none) dereferenceable(20) [[I2]], i32 noundef 2) #[[ATTR21]]
 ; CGSCC-NEXT:    [[I3:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[S]], i64 0, i32 2
-; CGSCC-NEXT:    call void @write_arg(ptr nocapture nofree noundef nonnull writeonly align 4 dereferenceable(16) [[I3]], i32 noundef 3) #[[ATTR21]]
+; CGSCC-NEXT:    call void @write_arg(ptr nofree noundef nonnull writeonly align 4 captures(none) dereferenceable(16) [[I3]], i32 noundef 3) #[[ATTR21]]
 ; CGSCC-NEXT:    [[F11:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[S]], i64 0, i32 3
 ; CGSCC-NEXT:    [[I4:%.*]] = load float, ptr [[F11]], align 4, !tbaa [[TBAA7]]
 ; CGSCC-NEXT:    [[F12:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[AGG_RESULT]], i64 0, i32 3
@@ -185,7 +185,7 @@ define void @local_alloca_simplifiable_1(ptr noalias sret(%struct.S) align 4 %ag
 ; CGSCC-NEXT:    [[ADD15:%.*]] = add nsw i32 [[I10]], [[I11]]
 ; CGSCC-NEXT:    [[I316:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[AGG_RESULT]], i64 0, i32 2
 ; CGSCC-NEXT:    store i32 [[ADD15]], ptr [[I316]], align 4, !tbaa [[TBAA14]]
-; CGSCC-NEXT:    call void @llvm.lifetime.end.p0(i64 noundef 24, ptr noalias nocapture nofree noundef nonnull align 4 dereferenceable(24) [[S]]) #[[ATTR20]]
+; CGSCC-NEXT:    call void @llvm.lifetime.end.p0(i64 noundef 24, ptr noalias nofree noundef nonnull align 4 captures(none) dereferenceable(24) [[S]]) #[[ATTR20]]
 ; CGSCC-NEXT:    ret void
 ;
 entry:
@@ -260,7 +260,7 @@ define void @local_alloca_simplifiable_2() {
 ; TUNIT-SAME: () #[[ATTR3:[0-9]+]] {
 ; TUNIT-NEXT:  entry:
 ; TUNIT-NEXT:    [[BYTES:%.*]] = alloca [1024 x i8], align 16
-; TUNIT-NEXT:    call void @llvm.lifetime.start.p0(i64 noundef 1024, ptr noalias nocapture nofree noundef nonnull align 16 dereferenceable(1024) [[BYTES]]) #[[ATTR17]]
+; TUNIT-NEXT:    call void @llvm.lifetime.start.p0(i64 noundef 1024, ptr noalias nofree noundef nonnull align 16 captures(none) dereferenceable(1024) [[BYTES]]) #[[ATTR17]]
 ; TUNIT-NEXT:    br label [[FOR_COND:%.*]]
 ; TUNIT:       for.cond:
 ; TUNIT-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], [[FOR_INC:%.*]] ], [ 0, [[ENTRY:%.*]] ]
@@ -310,7 +310,7 @@ define void @local_alloca_simplifiable_2() {
 ; TUNIT:       for.end24:
 ; TUNIT-NEXT:    [[ARRAYIDX25:%.*]] = getelementptr inbounds [1024 x i8], ptr [[BYTES]], i64 0, i64 1023
 ; TUNIT-NEXT:    [[ARRAYIDX26:%.*]] = getelementptr inbounds [1024 x i8], ptr [[BYTES]], i64 0, i64 500
-; TUNIT-NEXT:    call void @write_arg(ptr nocapture nofree noundef nonnull writeonly align 4 dereferenceable(524) [[ARRAYIDX26]], i32 noundef 0) #[[ATTR18]]
+; TUNIT-NEXT:    call void @write_arg(ptr nofree noundef nonnull writeonly align 4 captures(none) dereferenceable(524) [[ARRAYIDX26]], i32 noundef 0) #[[ATTR18]]
 ; TUNIT-NEXT:    br label [[FOR_COND28:%.*]]
 ; TUNIT:       for.cond28:
 ; TUNIT-NEXT:    [[INDVARS_IV12:%.*]] = phi i64 [ [[INDVARS_IV_NEXT13:%.*]], [[FOR_INC36:%.*]] ], [ 0, [[FOR_END24]] ]
@@ -326,7 +326,7 @@ define void @local_alloca_simplifiable_2() {
 ; TUNIT-NEXT:    [[INDVARS_IV_NEXT13]] = add nuw nsw i64 [[INDVARS_IV12]], 1
 ; TUNIT-NEXT:    br label [[FOR_COND28]], !llvm.loop [[LOOP20:![0-9]+]]
 ; TUNIT:       for.end38:
-; TUNIT-NEXT:    call void @llvm.lifetime.end.p0(i64 noundef 1024, ptr noalias nocapture nofree noundef nonnull align 16 dereferenceable(1024) [[BYTES]]) #[[ATTR17]]
+; TUNIT-NEXT:    call void @llvm.lifetime.end.p0(i64 noundef 1024, ptr noalias nofree noundef nonnull align 16 captures(none) dereferenceable(1024) [[BYTES]]) #[[ATTR17]]
 ; TUNIT-NEXT:    ret void
 ;
 ; CGSCC: Function Attrs: mustprogress nofree nosync nounwind willreturn
@@ -334,7 +334,7 @@ define void @local_alloca_simplifiable_2() {
 ; CGSCC-SAME: () #[[ATTR3:[0-9]+]] {
 ; CGSCC-NEXT:  entry:
 ; CGSCC-NEXT:    [[BYTES:%.*]] = alloca [1024 x i8], align 16
-; CGSCC-NEXT:    call void @llvm.lifetime.start.p0(i64 noundef 1024, ptr noalias nocapture nofree noundef nonnull align 16 dereferenceable(1024) [[BYTES]]) #[[ATTR20]]
+; CGSCC-NEXT:    call void @llvm.lifetime.start.p0(i64 noundef 1024, ptr noalias nofree noundef nonnull align 16 captures(none) dereferenceable(1024) [[BYTES]]) #[[ATTR20]]
 ; CGSCC-NEXT:    br label [[FOR_COND:%.*]]
 ; CGSCC:       for.cond:
 ; CGSCC-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], [[FOR_INC:%.*]] ], [ 0, [[ENTRY:%.*]] ]
@@ -388,7 +388,7 @@ define void @local_alloca_simplifiable_2() {
 ; CGSCC-NEXT:    [[ARRAYIDX25:%.*]] = getelementptr inbounds [1024 x i8], ptr [[BYTES]], i64 0, i64 1023
 ; CGSCC-NEXT:    store i8 0, ptr [[ARRAYIDX25]], align 1, !tbaa [[TBAA15]]
 ; CGSCC-NEXT:    [[ARRAYIDX26:%.*]] = getelementptr inbounds [1024 x i8], ptr [[BYTES]], i64 0, i64 500
-; CGSCC-NEXT:    call void @write_arg(ptr nocapture nofree noundef nonnull writeonly align 4 dereferenceable(524) [[ARRAYIDX26]], i32 noundef 0) #[[ATTR21]]
+; CGSCC-NEXT:    call void @write_arg(ptr nofree noundef nonnull writeonly align 4 captures(none) dereferenceable(524) [[ARRAYIDX26]], i32 noundef 0) #[[ATTR21]]
 ; CGSCC-NEXT:    br label [[FOR_COND28:%.*]]
 ; CGSCC:       for.cond28:
 ; CGSCC-NEXT:    [[INDVARS_IV12:%.*]] = phi i64 [ [[INDVARS_IV_NEXT13:%.*]], [[FOR_INC36:%.*]] ], [ 0, [[FOR_END24]] ]
@@ -406,7 +406,7 @@ define void @local_alloca_simplifiable_2() {
 ; CGSCC-NEXT:    [[INDVARS_IV_NEXT13]] = add nuw nsw i64 [[INDVARS_IV12]], 1
 ; CGSCC-NEXT:    br label [[FOR_COND28]], !llvm.loop [[LOOP23:![0-9]+]]
 ; CGSCC:       for.end38:
-; CGSCC-NEXT:    call void @llvm.lifetime.end.p0(i64 noundef 1024, ptr noalias nocapture nofree noundef nonnull align 16 dereferenceable(1024) [[BYTES]]) #[[ATTR20]]
+; CGSCC-NEXT:    call void @llvm.lifetime.end.p0(i64 noundef 1024, ptr noalias nofree noundef nonnull align 16 captures(none) dereferenceable(1024) [[BYTES]]) #[[ATTR20]]
 ; CGSCC-NEXT:    ret void
 ;
 entry:
@@ -558,7 +558,7 @@ define i32 @multi_obj_simplifiable_1(i32 %cnd) {
 ; TUNIT-SAME: (i32 [[CND:%.*]]) #[[ATTR3]] {
 ; TUNIT-NEXT:  entry:
 ; TUNIT-NEXT:    [[L:%.*]] = alloca i32, align 4
-; TUNIT-NEXT:    call void @llvm.lifetime.start.p0(i64 noundef 4, ptr noalias nocapture nofree noundef nonnull align 4 dereferenceable(4) [[L]]) #[[ATTR17]]
+; TUNIT-NEXT:    call void @llvm.lifetime.start.p0(i64 noundef 4, ptr noalias nofree noundef nonnull align 4 captures(none) dereferenceable(4) [[L]]) #[[ATTR17]]
 ; TUNIT-NEXT:    [[TOBOOL_NOT:%.*]] = icmp eq i32 [[CND]], 0
 ; TUNIT-NEXT:    br i1 [[TOBOOL_NOT]], label [[COND_FALSE:%.*]], label [[COND_TRUE:%.*]]
 ; TUNIT:       cond.true:
@@ -566,7 +566,7 @@ define i32 @multi_obj_simplifiable_1(i32 %cnd) {
 ; TUNIT:       cond.false:
 ; TUNIT-NEXT:    br label [[COND_END]]
 ; TUNIT:       cond.end:
-; TUNIT-NEXT:    call void @llvm.lifetime.end.p0(i64 noundef 4, ptr noalias nocapture nofree noundef nonnull align 4 dereferenceable(4) [[L]]) #[[ATTR17]]
+; TUNIT-NEXT:    call void @llvm.lifetime.end.p0(i64 noundef 4, ptr noalias nofree noundef nonnull align 4 captures(none) dereferenceable(4) [[L]]) #[[ATTR17]]
 ; TUNIT-NEXT:    ret i32 5
 ;
 ; CGSCC: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn
@@ -574,7 +574,7 @@ define i32 @multi_obj_simplifiable_1(i32 %cnd) {
 ; CGSCC-SAME: (i32 [[CND:%.*]]) #[[ATTR5:[0-9]+]] {
 ; CGSCC-NEXT:  entry:
 ; CGSCC-NEXT:    [[L:%.*]] = alloca i32, align 4
-; CGSCC-NEXT:    call void @llvm.lifetime.start.p0(i64 noundef 4, ptr noalias nocapture nofree noundef nonnull align 4 dereferenceable(4) [[L]]) #[[ATTR20]]
+; CGSCC-NEXT:    call void @llvm.lifetime.start.p0(i64 noundef 4, ptr noalias nofree noundef nonnull align 4 captures(none) dereferenceable(4) [[L]]) #[[ATTR20]]
 ; CGSCC-NEXT:    [[TOBOOL_NOT:%.*]] = icmp eq i32 [[CND]], 0
 ; CGSCC-NEXT:    br i1 [[TOBOOL_NOT]], label [[COND_FALSE:%.*]], label [[COND_TRUE:%.*]]
 ; CGSCC:       cond.true:
@@ -582,7 +582,7 @@ define i32 @multi_obj_simplifiable_1(i32 %cnd) {
 ; CGSCC:       cond.false:
 ; CGSCC-NEXT:    br label [[COND_END]]
 ; CGSCC:       cond.end:
-; CGSCC-NEXT:    call void @llvm.lifetime.end.p0(i64 noundef 4, ptr noalias nocapture nofree noundef nonnull align 4 dereferenceable(4) [[L]]) #[[ATTR20]]
+; CGSCC-NEXT:    call void @llvm.lifetime.end.p0(i64 noundef 4, ptr noalias nofree noundef nonnull align 4 captures(none) dereferenceable(4) [[L]]) #[[ATTR20]]
 ; CGSCC-NEXT:    ret i32 5
 ;
 entry:
@@ -620,7 +620,7 @@ define i32 @multi_obj_simplifiable_2(i32 %cnd) {
 ; TUNIT-SAME: (i32 [[CND:%.*]]) #[[ATTR3]] {
 ; TUNIT-NEXT:  entry:
 ; TUNIT-NEXT:    [[L:%.*]] = alloca i32, align 4
-; TUNIT-NEXT:    call void @llvm.lifetime.start.p0(i64 noundef 4, ptr noalias nocapture nofree noundef nonnull align 4 dereferenceable(4) [[L]]) #[[ATTR17]]
+; TUNIT-NEXT:    call void @llvm.lifetime.start.p0(i64 noundef 4, ptr noalias nofree noundef nonnull align 4 captures(none) dereferenceable(4) [[L]]) #[[ATTR17]]
 ; TUNIT-NEXT:    [[TOBOOL_NOT:%.*]] = icmp eq i32 [[CND]], 0
 ; TUNIT-NEXT:    br i1 [[TOBOOL_NOT]], label [[COND_FALSE:%.*]], label [[COND_TRUE:%.*]]
 ; TUNIT:       cond.true:
@@ -628,7 +628,7 @@ define i32 @multi_obj_simplifiable_2(i32 %cnd) {
 ; TUNIT:       cond.false:
 ; TUNIT-NEXT:    br label [[COND_END]]
 ; TUNIT:       cond.end:
-; TUNIT-NEXT:    call void @llvm.lifetime.end.p0(i64 noundef 4, ptr noalias nocapture nofree noundef nonnull align 4 dereferenceable(4) [[L]]) #[[ATTR17]]
+; TUNIT-NEXT:    call void @llvm.lifetime.end.p0(i64 noundef 4, ptr noalias nofree noundef nonnull align 4 captures(none) dereferenceable(4) [[L]]) #[[ATTR17]]
 ; TUNIT-NEXT:    ret i32 5
 ;
 ; CGSCC: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn
@@ -636,7 +636,7 @@ define i32 @multi_obj_simplifiable_2(i32 %cnd) {
 ; CGSCC-SAME: (i32 [[CND:%.*]]) #[[ATTR5]] {
 ; CGSCC-NEXT:  entry:
 ; CGSCC-NEXT:    [[L:%.*]] = alloca i32, align 4
-; CGSCC-NEXT:    call void @llvm.lifetime.start.p0(i64 noundef 4, ptr noalias nocapture nofree noundef nonnull align 4 dereferenceable(4) [[L]]) #[[ATTR20]]
+; CGSCC-NEXT:    call void @llvm.lifetime.start.p0(i64 noundef 4, ptr noalias nofree noundef nonnull align 4 captures(none) dereferenceable(4) [[L]]) #[[ATTR20]]
 ; CGSCC-NEXT:    [[TOBOOL_NOT:%.*]] = icmp eq i32 [[CND]], 0
 ; CGSCC-NEXT:    br i1 [[TOBOOL_NOT]], label [[COND_FALSE:%.*]], label [[COND_TRUE:%.*]]
 ; CGSCC:       cond.true:
@@ -644,7 +644,7 @@ define i32 @multi_obj_simplifiable_2(i32 %cnd) {
 ; CGSCC:       cond.false:
 ; CGSCC-NEXT:    br label [[COND_END]]
 ; CGSCC:       cond.end:
-; CGSCC-NEXT:    call void @llvm.lifetime.end.p0(i64 noundef 4, ptr noalias nocapture nofree noundef nonnull align 4 dereferenceable(4) [[L]]) #[[ATTR20]]
+; CGSCC-NEXT:    call void @llvm.lifetime.end.p0(i64 noundef 4, ptr noalias nofree noundef nonnull align 4 captures(none) dereferenceable(4) [[L]]) #[[ATTR20]]
 ; CGSCC-NEXT:    ret i32 5
 ;
 entry:
@@ -688,11 +688,11 @@ cond.end:                                         ; preds = %cond.false, %cond.t
 define void @static_global_simplifiable_1(ptr noalias sret(%struct.S) align 4 %agg.result) {
 ; TUNIT: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(write)
 ; TUNIT-LABEL: define {{[^@]+}}@static_global_simplifiable_1
-; TUNIT-SAME: (ptr noalias nocapture nofree writeonly sret([[STRUCT_S:%.*]]) align 4 dereferenceable_or_null(24) [[AGG_RESULT:%.*]]) #[[ATTR5:[0-9]+]] {
+; TUNIT-SAME: (ptr noalias nofree writeonly sret([[STRUCT_S:%.*]]) align 4 captures(none) dereferenceable_or_null(24) [[AGG_RESULT:%.*]]) #[[ATTR5:[0-9]+]] {
 ; TUNIT-NEXT:  entry:
-; TUNIT-NEXT:    call void @write_arg(ptr nocapture nofree noundef nonnull writeonly align 4 dereferenceable(24) @Gs1, i32 noundef 1) #[[ATTR18]]
-; TUNIT-NEXT:    call void @write_arg(ptr nocapture nofree noundef nonnull writeonly align 4 dereferenceable(20) getelementptr inbounds ([[STRUCT_S]], ptr @Gs1, i64 0, i32 1), i32 noundef 2) #[[ATTR18]]
-; TUNIT-NEXT:    call void @write_arg(ptr nocapture nofree noundef nonnull writeonly align 4 dereferenceable(16) getelementptr inbounds ([[STRUCT_S]], ptr @Gs1, i64 0, i32 2), i32 noundef 3) #[[ATTR18]]
+; TUNIT-NEXT:    call void @write_arg(ptr nofree noundef nonnull writeonly align 4 captures(none) dereferenceable(24) @Gs1, i32 noundef 1) #[[ATTR18]]
+; TUNIT-NEXT:    call void @write_arg(ptr nofree noundef nonnull writeonly align 4 captures(none) dereferenceable(20) getelementptr inbounds ([[STRUCT_S]], ptr @Gs1, i64 0, i32 1), i32 noundef 2) #[[ATTR18]]
+; TUNIT-NEXT:    call void @write_arg(ptr nofree noundef nonnull writeonly align 4 captures(none) dereferenceable(16) getelementptr inbounds ([[STRUCT_S]], ptr @Gs1, i64 0, i32 2), i32 noundef 3) #[[ATTR18]]
 ; TUNIT-NEXT:    [[F1:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[AGG_RESULT]], i64 0, i32 3
 ; TUNIT-NEXT:    store float 0x3FF19999A0000000, ptr [[F1]], align 4, !tbaa [[TBAA7]]
 ; TUNIT-NEXT:    [[F2:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[AGG_RESULT]], i64 0, i32 4
@@ -708,14 +708,14 @@ define void @static_global_simplifiable_1(ptr noalias sret(%struct.S) align 4 %a
 ;
 ; CGSCC: Function Attrs: mustprogress nofree nosync nounwind willreturn
 ; CGSCC-LABEL: define {{[^@]+}}@static_global_simplifiable_1
-; CGSCC-SAME: (ptr noalias nocapture nofree noundef nonnull writeonly sret([[STRUCT_S:%.*]]) align 4 dereferenceable(24) [[AGG_RESULT:%.*]]) #[[ATTR3]] {
+; CGSCC-SAME: (ptr noalias nofree noundef nonnull writeonly sret([[STRUCT_S:%.*]]) align 4 captures(none) dereferenceable(24) [[AGG_RESULT:%.*]]) #[[ATTR3]] {
 ; CGSCC-NEXT:  entry:
 ; CGSCC-NEXT:    store float 0x3FF19999A0000000, ptr getelementptr inbounds ([[STRUCT_S]], ptr @Gs1, i64 0, i32 3), align 4, !tbaa [[TBAA7]]
 ; CGSCC-NEXT:    store float 0x40019999A0000000, ptr getelementptr inbounds ([[STRUCT_S]], ptr @Gs1, i64 0, i32 4), align 4, !tbaa [[TBAA10]]
 ; CGSCC-NEXT:    store float 0x400A666660000000, ptr getelementptr inbounds ([[STRUCT_S]], ptr @Gs1, i64 0, i32 5), align 4, !tbaa [[TBAA11]]
-; CGSCC-NEXT:    call void @write_arg(ptr nocapture nofree noundef nonnull writeonly align 4 dereferenceable(24) @Gs1, i32 noundef 1) #[[ATTR21]]
-; CGSCC-NEXT:    call void @write_arg(ptr nocapture nofree noundef nonnull writeonly align 4 dereferenceable(20) getelementptr inbounds ([[STRUCT_S]], ptr @Gs1, i64 0, i32 1), i32 noundef 2) #[[ATTR21]]
-; CGSCC-NEXT:    call void @write_arg(ptr nocapture nofree noundef nonnull writeonly align 4 dereferenceable(16) getelementptr inbounds ([[STRUCT_S]], ptr @Gs1, i64 0, i32 2), i32 noundef 3) #[[ATTR21]]
+; CGSCC-NEXT:    call void @write_arg(ptr nofree noundef nonnull writeonly align 4 captures(none) dereferenceable(24) @Gs1, i32 noundef 1) #[[ATTR21]]
+; CGSCC-NEXT:    call void @write_arg(ptr nofree noundef nonnull writeonly align 4 captures(none) dereferenceable(20) getelementptr inbounds ([[STRUCT_S]], ptr @Gs1, i64 0, i32 1), i32 noundef 2) #[[ATTR21]]
+; CGSCC-NEXT:    call void @write_arg(ptr nofree noundef nonnull writeonly align 4 captures(none) dereferenceable(16) getelementptr inbounds ([[STRUCT_S]], ptr @Gs1, i64 0, i32 2), i32 noundef 3) #[[ATTR21]]
 ; CGSCC-NEXT:    [[I:%.*]] = load float, ptr getelementptr inbounds ([[STRUCT_S]], ptr @Gs1, i64 0, i32 3), align 4, !tbaa [[TBAA7]]
 ; CGSCC-NEXT:    [[F1:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[AGG_RESULT]], i64 0, i32 3
 ; CGSCC-NEXT:    store float [[I]], ptr [[F1]], align 4, !tbaa [[TBAA7]]
@@ -887,7 +887,7 @@ define void @static_global_simplifiable_2() {
 ; TUNIT-NEXT:    [[INDVARS_IV_NEXT8]] = add nuw nsw i64 [[INDVARS_IV7]], 1
 ; TUNIT-NEXT:    br label [[FOR_COND13]], !llvm.loop [[LOOP23:![0-9]+]]
 ; TUNIT:       for.end23:
-; TUNIT-NEXT:    call void @write_arg(ptr nocapture nofree noundef nonnull writeonly align 4 dereferenceable(524) getelementptr inbounds ([1024 x i8], ptr @GBytes, i64 0, i64 500), i32 noundef 0) #[[ATTR18]]
+; TUNIT-NEXT:    call void @write_arg(ptr nofree noundef nonnull writeonly align 4 captures(none) dereferenceable(524) getelementptr inbounds ([1024 x i8], ptr @GBytes, i64 0, i64 500), i32 noundef 0) #[[ATTR18]]
 ; TUNIT-NEXT:    br label [[FOR_COND25:%.*]]
 ; TUNIT:       for.cond25:
 ; TUNIT-NEXT:    [[INDVARS_IV12:%.*]] = phi i64 [ [[INDVARS_IV_NEXT13:%.*]], [[FOR_INC33:%.*]] ], [ 0, [[FOR_END23]] ]
@@ -960,7 +960,7 @@ define void @static_global_simplifiable_2() {
 ; CGSCC-NEXT:    br label [[FOR_COND13]], !llvm.loop [[LOOP26:![0-9]+]]
 ; CGSCC:       for.end23:
 ; CGSCC-NEXT:    store i8 0, ptr getelementptr inbounds ([1024 x i8], ptr @GBytes, i64 0, i64 1023), align 1, !tbaa [[TBAA15]]
-; CGSCC-NEXT:    call void @write_arg(ptr nocapture nofree noundef nonnull writeonly align 4 dereferenceable(524) getelementptr inbounds ([1024 x i8], ptr @GBytes, i64 0, i64 500), i32 noundef 0) #[[ATTR21]]
+; CGSCC-NEXT:    call void @write_arg(ptr nofree noundef nonnull writeonly align 4 captures(none) dereferenceable(524) getelementptr inbounds ([1024 x i8], ptr @GBytes, i64 0, i64 500), i32 noundef 0) #[[ATTR21]]
 ; CGSCC-NEXT:    br label [[FOR_COND25:%.*]]
 ; CGSCC:       for.cond25:
 ; CGSCC-NEXT:    [[INDVARS_IV12:%.*]] = phi i64 [ [[INDVARS_IV_NEXT13:%.*]], [[FOR_INC33:%.*]] ], [ 0, [[FOR_END23]] ]
@@ -1116,7 +1116,7 @@ define i32 @static_global_simplifiable_3() {
 define void @noalias_arg_simplifiable_1(ptr noalias sret(%struct.S) align 4 %agg.result, ptr byval(%struct.S) align 8 %s) {
 ; TUNIT: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: readwrite)
 ; TUNIT-LABEL: define {{[^@]+}}@noalias_arg_simplifiable_1
-; TUNIT-SAME: (ptr noalias nocapture nofree writeonly sret([[STRUCT_S:%.*]]) align 4 dereferenceable_or_null(24) [[AGG_RESULT:%.*]], ptr noalias nocapture nofree nonnull byval([[STRUCT_S]]) align 8 dereferenceable(24) [[S:%.*]]) #[[ATTR1]] {
+; TUNIT-SAME: (ptr noalias nofree writeonly sret([[STRUCT_S:%.*]]) align 4 captures(none) dereferenceable_or_null(24) [[AGG_RESULT:%.*]], ptr noalias nofree nonnull byval([[STRUCT_S]]) align 8 captures(none) dereferenceable(24) [[S:%.*]]) #[[ATTR1]] {
 ; TUNIT-NEXT:  entry:
 ; TUNIT-NEXT:    [[F1:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[S]], i64 0, i32 3
 ; TUNIT-NEXT:    store float 0x3FF19999A0000000, ptr [[F1]], align 4, !tbaa [[TBAA7]]
@@ -1124,11 +1124,11 @@ define void @noalias_arg_simplifiable_1(ptr noalias sret(%struct.S) align 4 %agg
 ; TUNIT-NEXT:    store float 0x40019999A0000000, ptr [[F2]], align 8, !tbaa [[TBAA10]]
 ; TUNIT-NEXT:    [[F3:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[S]], i64 0, i32 5
 ; TUNIT-NEXT:    store float 0x400A666660000000, ptr [[F3]], align 4, !tbaa [[TBAA11]]
-; TUNIT-NEXT:    call void @write_arg(ptr noalias nocapture nofree noundef nonnull writeonly align 8 dereferenceable(24) [[S]], i32 noundef 1) #[[ATTR18]]
+; TUNIT-NEXT:    call void @write_arg(ptr noalias nofree noundef nonnull writeonly align 8 captures(none) dereferenceable(24) [[S]], i32 noundef 1) #[[ATTR18]]
 ; TUNIT-NEXT:    [[I2:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[S]], i64 0, i32 1
-; TUNIT-NEXT:    call void @write_arg(ptr nocapture nofree noundef nonnull writeonly align 4 dereferenceable(20) [[I2]], i32 noundef 2) #[[ATTR18]]
+; TUNIT-NEXT:    call void @write_arg(ptr nofree noundef nonnull writeonly align 4 captures(none) dereferenceable(20) [[I2]], i32 noundef 2) #[[ATTR18]]
 ; TUNIT-NEXT:    [[I3:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[S]], i64 0, i32 2
-; TUNIT-NEXT:    call void @write_arg(ptr nocapture nofree noundef nonnull writeonly align 8 dereferenceable(16) [[I3]], i32 noundef 3) #[[ATTR18]]
+; TUNIT-NEXT:    call void @write_arg(ptr nofree noundef nonnull writeonly align 8 captures(none) dereferenceable(16) [[I3]], i32 noundef 3) #[[ATTR18]]
 ; TUNIT-NEXT:    [[F11:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[S]], i64 0, i32 3
 ; TUNIT-NEXT:    [[I:%.*]] = load float, ptr [[F11]], align 4, !tbaa [[TBAA7]]
 ; TUNIT-NEXT:    [[F12:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[AGG_RESULT]], i64 0, i32 3
@@ -1162,7 +1162,7 @@ define void @noalias_arg_simplifiable_1(ptr noalias sret(%struct.S) align 4 %agg
 ;
 ; CGSCC: Function Attrs: mustprogress nofree nosync nounwind willreturn memory(argmem: readwrite)
 ; CGSCC-LABEL: define {{[^@]+}}@noalias_arg_simplifiable_1
-; CGSCC-SAME: (ptr noalias nocapture nofree noundef nonnull writeonly sret([[STRUCT_S:%.*]]) align 4 dereferenceable(24) [[AGG_RESULT:%.*]], ptr noalias nocapture nofree noundef nonnull byval([[STRUCT_S]]) align 8 dereferenceable(24) [[S:%.*]]) #[[ATTR1]] {
+; CGSCC-SAME: (ptr noalias nofree noundef nonnull writeonly sret([[STRUCT_S:%.*]]) align 4 captures(none) dereferenceable(24) [[AGG_RESULT:%.*]], ptr noalias nofree noundef nonnull byval([[STRUCT_S]]) align 8 captures(none) dereferenceable(24) [[S:%.*]]) #[[ATTR1]] {
 ; CGSCC-NEXT:  entry:
 ; CGSCC-NEXT:    [[F1:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[S]], i64 0, i32 3
 ; CGSCC-NEXT:    store float 0x3FF19999A0000000, ptr [[F1]], align 4, !tbaa [[TBAA7]]
@@ -1170,11 +1170,11 @@ define void @noalias_arg_simplifiable_1(ptr noalias sret(%struct.S) align 4 %agg
 ; CGSCC-NEXT:    store float 0x40019999A0000000, ptr [[F2]], align 8, !tbaa [[TBAA10]]
 ; CGSCC-NEXT:    [[F3:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[S]], i64 0, i32 5
 ; CGSCC-NEXT:    store float 0x400A666660000000, ptr [[F3]], align 4, !tbaa [[TBAA11]]
-; CGSCC-NEXT:    call void @write_arg(ptr noalias nocapture nofree noundef nonnull writeonly align 8 dereferenceable(24) [[S]], i32 noundef 1) #[[ATTR21]]
+; CGSCC-NEXT:    call void @write_arg(ptr noalias nofree noundef nonnull writeonly align 8 captures(none) dereferenceable(24) [[S]], i32 noundef 1) #[[ATTR21]]
 ; CGSCC-NEXT:    [[I2:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[S]], i64 0, i32 1
-; CGSCC-NEXT:    call void @write_arg(ptr nocapture nofree noundef nonnull writeonly align 4 dereferenceable(20) [[I2]], i32 noundef 2) #[[ATTR21]]
+; CGSCC-NEXT:    call void @write_arg(ptr nofree noundef nonnull writeonly align 4 captures(none) dereferenceable(20) [[I2]], i32 noundef 2) #[[ATTR21]]
 ; CGSCC-NEXT:    [[I3:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[S]], i64 0, i32 2
-; CGSCC-NEXT:    call void @write_arg(ptr nocapture nofree noundef nonnull writeonly align 8 dereferenceable(16) [[I3]], i32 noundef 3) #[[ATTR21]]
+; CGSCC-NEXT:    call void @write_arg(ptr nofree noundef nonnull writeonly align 8 captures(none) dereferenceable(16) [[I3]], i32 noundef 3) #[[ATTR21]]
 ; CGSCC-NEXT:    [[F11:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[S]], i64 0, i32 3
 ; CGSCC-NEXT:    [[I:%.*]] = load float, ptr [[F11]], align 4, !tbaa [[TBAA7]]
 ; CGSCC-NEXT:    [[F12:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[AGG_RESULT]], i64 0, i32 3
@@ -1267,7 +1267,7 @@ entry:
 define void @noalias_arg_simplifiable_2(ptr %Bytes) {
 ; TUNIT: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn
 ; TUNIT-LABEL: define {{[^@]+}}@noalias_arg_simplifiable_2
-; TUNIT-SAME: (ptr nocapture nofree [[BYTES:%.*]]) #[[ATTR3]] {
+; TUNIT-SAME: (ptr nofree captures(none) [[BYTES:%.*]]) #[[ATTR3]] {
 ; TUNIT-NEXT:  entry:
 ; TUNIT-NEXT:    br label [[FOR_COND:%.*]]
 ; TUNIT:       for.cond:
@@ -1322,7 +1322,7 @@ define void @noalias_arg_simplifiable_2(ptr %Bytes) {
 ; TUNIT-NEXT:    [[ARRAYIDX24:%.*]] = getelementptr inbounds i8, ptr [[BYTES]], i64 1023
 ; TUNIT-NEXT:    store i8 0, ptr [[ARRAYIDX24]], align 1, !tbaa [[TBAA19]]
 ; TUNIT-NEXT:    [[ARRAYIDX25:%.*]] = getelementptr inbounds i8, ptr [[BYTES]], i64 500
-; TUNIT-NEXT:    call void @write_arg(ptr nocapture nofree noundef nonnull writeonly align 4 [[ARRAYIDX25]], i32 noundef 0) #[[ATTR18]]
+; TUNIT-NEXT:    call void @write_arg(ptr nofree noundef nonnull writeonly align 4 captures(none) [[ARRAYIDX25]], i32 noundef 0) #[[ATTR18]]
 ; TUNIT-NEXT:    br label [[FOR_COND27:%.*]]
 ; TUNIT:       for.cond27:
 ; TUNIT-NEXT:    [[INDVARS_IV12:%.*]] = phi i64 [ [[INDVARS_IV_NEXT13:%.*]], [[FOR_INC35:%.*]] ], [ 0, [[FOR_END23]] ]
@@ -1344,7 +1344,7 @@ define void @noalias_arg_simplifiable_2(ptr %Bytes) {
 ;
 ; CGSCC: Function Attrs: mustprogress nofree nosync nounwind willreturn
 ; CGSCC-LABEL: define {{[^@]+}}@noalias_arg_simplifiable_2
-; CGSCC-SAME: (ptr nocapture nofree [[BYTES:%.*]]) #[[ATTR3]] {
+; CGSCC-SAME: (ptr nofree captures(none) [[BYTES:%.*]]) #[[ATTR3]] {
 ; CGSCC-NEXT:  entry:
 ; CGSCC-NEXT:    br label [[FOR_COND:%.*]]
 ; CGSCC:       for.cond:
@@ -1399,7 +1399,7 @@ define void @noalias_arg_simplifiable_2(ptr %Bytes) {
 ; CGSCC-NEXT:    [[ARRAYIDX24:%.*]] = getelementptr inbounds i8, ptr [[BYTES]], i64 1023
 ; CGSCC-NEXT:    store i8 0, ptr [[ARRAYIDX24]], align 1, !tbaa [[TBAA15]]
 ; CGSCC-NEXT:    [[ARRAYIDX25:%.*]] = getelementptr inbounds i8, ptr [[BYTES]], i64 500
-; CGSCC-NEXT:    call void @write_arg(ptr nocapture nofree noundef nonnull writeonly align 4 dereferenceable(4) [[ARRAYIDX25]], i32 noundef 0) #[[ATTR21]]
+; CGSCC-NEXT:    call void @write_arg(ptr nofree noundef nonnull writeonly align 4 captures(none) dereferenceable(4) [[ARRAYIDX25]], i32 noundef 0) #[[ATTR21]]
 ; CGSCC-NEXT:    br label [[FOR_COND27:%.*]]
 ; CGSCC:       for.cond27:
 ; CGSCC-NEXT:    [[INDVARS_IV12:%.*]] = phi i64 [ [[INDVARS_IV_NEXT13:%.*]], [[FOR_INC35:%.*]] ], [ 0, [[FOR_END23]] ]
@@ -1528,40 +1528,40 @@ define i32 @local_alloca_not_simplifiable_1() {
 ; TUNIT-NEXT:  entry:
 ; TUNIT-NEXT:    [[X:%.*]] = alloca i32, align 4
 ; TUNIT-NEXT:    [[Y:%.*]] = alloca i32, align 4
-; TUNIT-NEXT:    call void @llvm.lifetime.start.p0(i64 noundef 4, ptr nocapture nofree noundef nonnull align 4 dereferenceable(4) [[X]]) #[[ATTR17]]
-; TUNIT-NEXT:    call void @llvm.lifetime.start.p0(i64 noundef 4, ptr noalias nocapture nofree noundef nonnull align 4 dereferenceable(4) [[Y]]) #[[ATTR17]]
+; TUNIT-NEXT:    call void @llvm.lifetime.start.p0(i64 noundef 4, ptr nofree noundef nonnull align 4 captures(none) dereferenceable(4) [[X]]) #[[ATTR17]]
+; TUNIT-NEXT:    call void @llvm.lifetime.start.p0(i64 noundef 4, ptr noalias nofree noundef nonnull align 4 captures(none) dereferenceable(4) [[Y]]) #[[ATTR17]]
 ; TUNIT-NEXT:    store i32 1, ptr [[Y]], align 4, !tbaa [[TBAA3]]
 ; TUNIT-NEXT:    store i32 1, ptr [[X]], align 4, !tbaa [[TBAA3]]
 ; TUNIT-NEXT:    call void @escape(ptr noundef nonnull align 4 dereferenceable(4) [[X]])
-; TUNIT-NEXT:    call void @write_random(ptr noalias nocapture nofree noundef nonnull writeonly align 4 dereferenceable(4) [[Y]])
+; TUNIT-NEXT:    call void @write_random(ptr noalias nofree noundef nonnull writeonly align 4 captures(none) dereferenceable(4) [[Y]])
 ; TUNIT-NEXT:    [[I3:%.*]] = load i32, ptr [[X]], align 4, !tbaa [[TBAA3]]
 ; TUNIT-NEXT:    [[TOBOOL_NOT:%.*]] = icmp eq i32 [[I3]], 0
 ; TUNIT-NEXT:    [[COND:%.*]] = select i1 [[TOBOOL_NOT]], i32 2, i32 1
 ; TUNIT-NEXT:    [[I4:%.*]] = load i32, ptr [[Y]], align 4, !tbaa [[TBAA3]]
 ; TUNIT-NEXT:    [[ADD:%.*]] = add nsw i32 [[I3]], [[I4]]
 ; TUNIT-NEXT:    [[ADD1:%.*]] = add nsw i32 [[ADD]], [[COND]]
-; TUNIT-NEXT:    call void @llvm.lifetime.end.p0(i64 noundef 4, ptr noalias nocapture nofree noundef nonnull align 4 dereferenceable(4) [[Y]])
-; TUNIT-NEXT:    call void @llvm.lifetime.end.p0(i64 noundef 4, ptr nocapture nofree noundef nonnull align 4 dereferenceable(4) [[X]])
+; TUNIT-NEXT:    call void @llvm.lifetime.end.p0(i64 noundef 4, ptr noalias nofree noundef nonnull align 4 captures(none) dereferenceable(4) [[Y]])
+; TUNIT-NEXT:    call void @llvm.lifetime.end.p0(i64 noundef 4, ptr nofree noundef nonnull align 4 captures(none) dereferenceable(4) [[X]])
 ; TUNIT-NEXT:    ret i32 [[ADD1]]
 ;
 ; CGSCC-LABEL: define {{[^@]+}}@local_alloca_not_simplifiable_1() {
 ; CGSCC-NEXT:  entry:
 ; CGSCC-NEXT:    [[X:%.*]] = alloca i32, align 4
 ; CGSCC-NEXT:    [[Y:%.*]] = alloca i32, align 4
-; CGSCC-NEXT:    call void @llvm.lifetime.start.p0(i64 noundef 4, ptr nocapture nofree noundef nonnull align 4 dereferenceable(4) [[X]]) #[[ATTR20]]
-; CGSCC-NEXT:    call void @llvm.lifetime.start.p0(i64 noundef 4, ptr noalias nocapture nofree noundef nonnull align 4 dereferenceable(4) [[Y]]) #[[ATTR20]]
+; CGSCC-NEXT:    call void @llvm.lifetime.start.p0(i64 noundef 4, ptr nofree noundef nonnull align 4 captures(none) dereferenceable(4) [[X]]) #[[ATTR20]]
+; CGSCC-NEXT:    call void @llvm.lifetime.start.p0(i64 noundef 4, ptr noalias nofree noundef nonnull align 4 captures(none) dereferenceable(4) [[Y]]) #[[ATTR20]]
 ; CGSCC-NEXT:    store i32 1, ptr [[Y]], align 4, !tbaa [[TBAA3]]
 ; CGSCC-NEXT:    store i32 1, ptr [[X]], align 4, !tbaa [[TBAA3]]
 ; CGSCC-NEXT:    call void @escape(ptr noundef nonnull align 4 dereferenceable(4) [[X]])
-; CGSCC-NEXT:    call void @write_random(ptr noalias nocapture nofree noundef nonnull writeonly align 4 dereferenceable(4) [[Y]])
+; CGSCC-NEXT:    call void @write_random(ptr noalias nofree noundef nonnull writeonly align 4 captures(none) dereferenceable(4) [[Y]])
 ; CGSCC-NEXT:    [[I3:%.*]] = load i32, ptr [[X]], align 4, !tbaa [[TBAA3]]
 ; CGSCC-NEXT:    [[TOBOOL_NOT:%.*]] = icmp eq i32 [[I3]], 0
 ; CGSCC-NEXT:    [[COND:%.*]] = select i1 [[TOBOOL_NOT]], i32 2, i32 1
 ; CGSCC-NEXT:    [[I4:%.*]] = load i32, ptr [[Y]], align 4, !tbaa [[TBAA3]]
 ; CGSCC-NEXT:    [[ADD:%.*]] = add nsw i32 [[I3]], [[I4]]
 ; CGSCC-NEXT:    [[ADD1:%.*]] = add nsw i32 [[ADD]], [[COND]]
-; CGSCC-NEXT:    call void @llvm.lifetime.end.p0(i64 noundef 4, ptr noalias nocapture nofree noundef nonnull align 4 dereferenceable(4) [[Y]])
-; CGSCC-NEXT:    call void @llvm.lifetime.end.p0(i64 noundef 4, ptr nocapture nofree noundef nonnull align 4 dereferenceable(4) [[X]])
+; CGSCC-NEXT:    call void @llvm.lifetime.end.p0(i64 noundef 4, ptr noalias nofree noundef nonnull align 4 captures(none) dereferenceable(4) [[Y]])
+; CGSCC-NEXT:    call void @llvm.lifetime.end.p0(i64 noundef 4, ptr nofree noundef nonnull align 4 captures(none) dereferenceable(4) [[X]])
 ; CGSCC-NEXT:    ret i32 [[ADD1]]
 ;
 entry:
@@ -1852,7 +1852,7 @@ define void @static_global_not_simplifiable_2_helper() {
 define i32 @static_global_not_simplifiable_3(i1 %c, ptr %p) {
 ; TUNIT: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn
 ; TUNIT-LABEL: define {{[^@]+}}@static_global_not_simplifiable_3
-; TUNIT-SAME: (i1 [[C:%.*]], ptr nocapture nofree writeonly [[P:%.*]]) #[[ATTR3]] {
+; TUNIT-SAME: (i1 [[C:%.*]], ptr nofree writeonly captures(none) [[P:%.*]]) #[[ATTR3]] {
 ; TUNIT-NEXT:    [[SEL:%.*]] = select i1 [[C]], ptr @Flag3, ptr [[P]]
 ; TUNIT-NEXT:    store i32 1, ptr [[SEL]], align 4, !tbaa [[TBAA3]]
 ; TUNIT-NEXT:    [[I:%.*]] = load i32, ptr @Flag3, align 4, !tbaa [[TBAA3]]
@@ -1860,7 +1860,7 @@ define i32 @static_global_not_simplifiable_3(i1 %c, ptr %p) {
 ;
 ; CGSCC: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn
 ; CGSCC-LABEL: define {{[^@]+}}@static_global_not_simplifiable_3
-; CGSCC-SAME: (i1 [[C:%.*]], ptr nocapture nofree writeonly [[P:%.*]]) #[[ATTR5]] {
+; CGSCC-SAME: (i1 [[C:%.*]], ptr nofree writeonly captures(none) [[P:%.*]]) #[[ATTR5]] {
 ; CGSCC-NEXT:    [[SEL:%.*]] = select i1 [[C]], ptr @Flag3, ptr [[P]]
 ; CGSCC-NEXT:    store i32 1, ptr [[SEL]], align 4, !tbaa [[TBAA3]]
 ; CGSCC-NEXT:    [[I:%.*]] = load i32, ptr @Flag3, align 4, !tbaa [[TBAA3]]
@@ -2644,7 +2644,7 @@ if.end:                                           ; preds = %if.then, %entry
 ; We mark %dst as writeonly and %src as readonly, that is (for now) all we can expect.
 define dso_local void @test_nested_memory(ptr %dst, ptr %src) {
 ; TUNIT-LABEL: define {{[^@]+}}@test_nested_memory
-; TUNIT-SAME: (ptr nocapture nofree writeonly [[DST:%.*]], ptr nocapture nofree readonly [[SRC:%.*]]) {
+; TUNIT-SAME: (ptr nofree writeonly captures(none) [[DST:%.*]], ptr nofree readonly captures(none) [[SRC:%.*]]) {
 ; TUNIT-NEXT:  entry:
 ; TUNIT-NEXT:    [[CALL_H2S:%.*]] = alloca i8, i64 24, align 1
 ; TUNIT-NEXT:    [[LOCAL:%.*]] = alloca [[STRUCT_STY:%.*]], align 8
@@ -2752,10 +2752,10 @@ entry:
 define hidden void @no_propagation_of_unknown_index_access(ptr %in, ptr %out, i32 %idx) #0 {
 ; TUNIT: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: readwrite)
 ; TUNIT-LABEL: define {{[^@]+}}@no_propagation_of_unknown_index_access
-; TUNIT-SAME: (ptr nocapture nofree readonly [[IN:%.*]], ptr nocapture nofree writeonly [[OUT:%.*]], i32 [[IDX:%.*]]) #[[ATTR1]] {
+; TUNIT-SAME: (ptr nofree readonly captures(none) [[IN:%.*]], ptr nofree writeonly captures(none) [[OUT:%.*]], i32 [[IDX:%.*]]) #[[ATTR1]] {
 ; TUNIT-NEXT:  entry:
 ; TUNIT-NEXT:    [[BUF:%.*]] = alloca [128 x i32], align 16
-; TUNIT-NEXT:    call void @llvm.lifetime.start.p0(i64 noundef 512, ptr noalias nocapture nofree noundef nonnull align 16 dereferenceable(512) [[BUF]]) #[[ATTR17]]
+; TUNIT-NEXT:    call void @llvm.lifetime.start.p0(i64 noundef 512, ptr noalias nofree noundef nonnull align 16 captures(none) dereferenceable(512) [[BUF]]) #[[ATTR17]]
 ; TUNIT-NEXT:    br label [[FOR_COND:%.*]]
 ; TUNIT:       for.cond:
 ; TUNIT-NEXT:    [[I_0:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[INC:%.*]], [[FOR_BODY:%.*]] ]
@@ -2776,7 +2776,7 @@ define hidden void @no_propagation_of_unknown_index_access(ptr %in, ptr %out, i3
 ; TUNIT-NEXT:    [[CMP5:%.*]] = icmp slt i32 [[I3_0]], 128
 ; TUNIT-NEXT:    br i1 [[CMP5]], label [[FOR_BODY7]], label [[FOR_COND_CLEANUP6:%.*]]
 ; TUNIT:       for.cond.cleanup6:
-; TUNIT-NEXT:    call void @llvm.lifetime.end.p0(i64 noundef 512, ptr noalias nocapture nofree noundef nonnull align 16 dereferenceable(512) [[BUF]]) #[[ATTR17]]
+; TUNIT-NEXT:    call void @llvm.lifetime.end.p0(i64 noundef 512, ptr noalias nofree noundef nonnull align 16 captures(none) dereferenceable(512) [[BUF]]) #[[ATTR17]]
 ; TUNIT-NEXT:    ret void
 ; TUNIT:       for.body7:
 ; TUNIT-NEXT:    [[IDXPROM8:%.*]] = sext i32 [[I3_0]] to i64
@@ -2794,10 +2794,10 @@ define hidden void @no_propagation_of_unknown_index_access(ptr %in, ptr %out, i3
 ;
 ; CGSCC: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: readwrite)
 ; CGSCC-LABEL: define {{[^@]+}}@no_propagation_of_unknown_index_access
-; CGSCC-SAME: (ptr nocapture nofree readonly [[IN:%.*]], ptr nocapture nofree writeonly [[OUT:%.*]], i32 [[IDX:%.*]]) #[[ATTR13:[0-9]+]] {
+; CGSCC-SAME: (ptr nofree readonly captures(none) [[IN:%.*]], ptr nofree writeonly captures(none) [[OUT:%.*]], i32 [[IDX:%.*]]) #[[ATTR13:[0-9]+]] {
 ; CGSCC-NEXT:  entry:
 ; CGSCC-NEXT:    [[BUF:%.*]] = alloca [128 x i32], align 16
-; CGSCC-NEXT:    call void @llvm.lifetime.start.p0(i64 noundef 512, ptr noalias nocapture nofree noundef nonnull align 16 dereferenceable(512) [[BUF]]) #[[ATTR20]]
+; CGSCC-NEXT:    call void @llvm.lifetime.start.p0(i64 noundef 512, ptr noalias nofree noundef nonnull align 16 captures(none) dereferenceable(512) [[BUF]]) #[[ATTR20]]
 ; CGSCC-NEXT:    br label [[FOR_COND:%.*]]
 ; CGSCC:       for.cond:
 ; CGSCC-NEXT:    [[I_0:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[INC:%.*]], [[FOR_BODY:%.*]] ]
@@ -2818,7 +2818,7 @@ define hidden void @no_propagation_of_unknown_index_access(ptr %in, ptr %out, i3
 ; CGSCC-NEXT:    [[CMP5:%.*]] = icmp slt i32 [[I3_0]], 128
 ; CGSCC-NEXT:    br i1 [[CMP5]], label [[FOR_BODY7]], label [[FOR_COND_CLEANUP6:%.*]]
 ; CGSCC:       for.cond.cleanup6:
-; CGSCC-NEXT:    call void @llvm.lifetime.end.p0(i64 noundef 512, ptr noalias nocapture nofree noundef nonnull align 16 dereferenceable(512) [[BUF]]) #[[ATTR20]]
+; CGSCC-NEXT:    call void @llvm.lifetime.end.p0(i64 noundef 512, ptr noalias nofree noundef nonnull align 16 captures(none) dereferenceable(512) [[BUF]]) #[[ATTR20]]
 ; CGSCC-NEXT:    ret void
 ; CGSCC:       for.body7:
 ; CGSCC-NEXT:    [[IDXPROM8:%.*]] = sext i32 [[I3_0]] to i64
@@ -2884,12 +2884,12 @@ for.body7:                                        ; preds = %for.cond4
 define internal i1 @alloca_non_unique(ptr %p, i32 %in, i1 %c) {
 ; TUNIT: Function Attrs: nofree nosync nounwind memory(argmem: readwrite)
 ; TUNIT-LABEL: define {{[^@]+}}@alloca_non_unique
-; TUNIT-SAME: (ptr noalias nocapture nofree readonly align 4 [[P:%.*]], i32 [[IN:%.*]], i1 noundef [[C:%.*]]) #[[ATTR12:[0-9]+]] {
+; TUNIT-SAME: (ptr noalias nofree readonly align 4 captures(none) [[P:%.*]], i32 [[IN:%.*]], i1 noundef [[C:%.*]]) #[[ATTR12:[0-9]+]] {
 ; TUNIT-NEXT:    [[A:%.*]] = alloca i32, align 4
 ; TUNIT-NEXT:    store i32 [[IN]], ptr [[A]], align 4
 ; TUNIT-NEXT:    br i1 [[C]], label [[T:%.*]], label [[F:%.*]]
 ; TUNIT:       t:
-; TUNIT-NEXT:    [[R:%.*]] = call i1 @alloca_non_unique(ptr noalias nocapture nofree noundef nonnull readonly align 4 dereferenceable(4) [[A]], i32 noundef 42, i1 noundef false) #[[ATTR14:[0-9]+]]
+; TUNIT-NEXT:    [[R:%.*]] = call i1 @alloca_non_unique(ptr noalias nofree noundef nonnull readonly align 4 captures(none) dereferenceable(4) [[A]], i32 noundef 42, i1 noundef false) #[[ATTR14:[0-9]+]]
 ; TUNIT-NEXT:    ret i1 [[R]]
 ; TUNIT:       f:
 ; TUNIT-NEXT:    [[L:%.*]] = load i32, ptr [[P]], align 4
@@ -2898,12 +2898,12 @@ define internal i1 @alloca_non_unique(ptr %p, i32 %in, i1 %c) {
 ;
 ; CGSCC: Function Attrs: nofree nosync nounwind memory(argmem: readwrite)
 ; CGSCC-LABEL: define {{[^@]+}}@alloca_non_unique
-; CGSCC-SAME: (ptr noalias nocapture nofree readonly align 4 [[P:%.*]], i32 [[IN:%.*]], i1 noundef [[C:%.*]]) #[[ATTR14:[0-9]+]] {
+; CGSCC-SAME: (ptr noalias nofree readonly align 4 captures(none) [[P:%.*]], i32 [[IN:%.*]], i1 noundef [[C:%.*]]) #[[ATTR14:[0-9]+]] {
 ; CGSCC-NEXT:    [[A:%.*]] = alloca i32, align 4
 ; CGSCC-NEXT:    store i32 [[IN]], ptr [[A]], align 4
 ; CGSCC-NEXT:    br i1 [[C]], label [[T:%.*]], label [[F:%.*]]
 ; CGSCC:       t:
-; CGSCC-NEXT:    [[R:%.*]] = call i1 @alloca_non_unique(ptr noalias nocapture nofree noundef nonnull readonly align 4 dereferenceable(4) [[A]], i32 noundef 42, i1 noundef false) #[[ATTR17:[0-9]+]]
+; CGSCC-NEXT:    [[R:%.*]] = call i1 @alloca_non_unique(ptr noalias nofree noundef nonnull readonly align 4 captures(none) dereferenceable(4) [[A]], i32 noundef 42, i1 noundef false) #[[ATTR17:[0-9]+]]
 ; CGSCC-NEXT:    ret i1 [[R]]
 ; CGSCC:       f:
 ; CGSCC-NEXT:    [[L:%.*]] = load i32, ptr [[P]], align 4
@@ -2947,7 +2947,7 @@ define i32 @scope_value_traversal(i32 %bad, i1 %c, i1 %c2) {
 ; TUNIT-SAME: (i32 [[BAD:%.*]], i1 [[C:%.*]], i1 [[C2:%.*]]) #[[ATTR4]] {
 ; TUNIT-NEXT:    [[A:%.*]] = alloca i32, align 4
 ; TUNIT-NEXT:    store i32 [[BAD]], ptr [[A]], align 4
-; TUNIT-NEXT:    call void @scope_value_traversal_helper(ptr noalias nocapture nofree noundef nonnull align 4 dereferenceable(4) [[A]], i1 [[C2]]) #[[ATTR22:[0-9]+]]
+; TUNIT-NEXT:    call void @scope_value_traversal_helper(ptr noalias nofree noundef nonnull align 4 captures(none) dereferenceable(4) [[A]], i1 [[C2]]) #[[ATTR22:[0-9]+]]
 ; TUNIT-NEXT:    [[L:%.*]] = load i32, ptr [[A]], align 4
 ; TUNIT-NEXT:    [[SEL:%.*]] = select i1 [[C]], i32 [[BAD]], i32 [[L]]
 ; TUNIT-NEXT:    ret i32 [[SEL]]
@@ -2957,7 +2957,7 @@ define i32 @scope_value_traversal(i32 %bad, i1 %c, i1 %c2) {
 ; CGSCC-SAME: (i32 [[BAD:%.*]], i1 [[C:%.*]], i1 [[C2:%.*]]) #[[ATTR16:[0-9]+]] {
 ; CGSCC-NEXT:    [[A:%.*]] = alloca i32, align 4
 ; CGSCC-NEXT:    store i32 [[BAD]], ptr [[A]], align 4
-; CGSCC-NEXT:    call void @scope_value_traversal_helper(ptr noalias nocapture nofree noundef nonnull align 4 dereferenceable(4) [[A]], i1 [[C2]]) #[[ATTR26:[0-9]+]]
+; CGSCC-NEXT:    call void @scope_value_traversal_helper(ptr noalias nofree noundef nonnull align 4 captures(none) dereferenceable(4) [[A]], i1 [[C2]]) #[[ATTR26:[0-9]+]]
 ; CGSCC-NEXT:    [[L:%.*]] = load i32, ptr [[A]], align 4
 ; CGSCC-NEXT:    [[SEL:%.*]] = select i1 [[C]], i32 [[BAD]], i32 [[L]]
 ; CGSCC-NEXT:    ret i32 [[SEL]]
@@ -2973,7 +2973,7 @@ define i32 @scope_value_traversal(i32 %bad, i1 %c, i1 %c2) {
 define void @scope_value_traversal_helper(ptr %a, i1 %c) {
 ; TUNIT: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: readwrite)
 ; TUNIT-LABEL: define {{[^@]+}}@scope_value_traversal_helper
-; TUNIT-SAME: (ptr nocapture nofree noundef nonnull align 4 dereferenceable(4) [[A:%.*]], i1 [[C:%.*]]) #[[ATTR1]] {
+; TUNIT-SAME: (ptr nofree noundef nonnull align 4 captures(none) dereferenceable(4) [[A:%.*]], i1 [[C:%.*]]) #[[ATTR1]] {
 ; TUNIT-NEXT:    [[L:%.*]] = load i32, ptr [[A]], align 4
 ; TUNIT-NEXT:    [[SEL:%.*]] = select i1 [[C]], i32 [[L]], i32 42
 ; TUNIT-NEXT:    store i32 [[SEL]], ptr [[A]], align 4
@@ -2981,7 +2981,7 @@ define void @scope_value_traversal_helper(ptr %a, i1 %c) {
 ;
 ; CGSCC: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: readwrite)
 ; CGSCC-LABEL: define {{[^@]+}}@scope_value_traversal_helper
-; CGSCC-SAME: (ptr nocapture nofree noundef nonnull align 4 dereferenceable(4) [[A:%.*]], i1 [[C:%.*]]) #[[ATTR13]] {
+; CGSCC-SAME: (ptr nofree noundef nonnull align 4 captures(none) dereferenceable(4) [[A:%.*]], i1 [[C:%.*]]) #[[ATTR13]] {
 ; CGSCC-NEXT:    [[L:%.*]] = load i32, ptr [[A]], align 4
 ; CGSCC-NEXT:    [[SEL:%.*]] = select i1 [[C]], i32 [[L]], i32 42
 ; CGSCC-NEXT:    store i32 [[SEL]], ptr [[A]], align 4
@@ -3184,7 +3184,7 @@ define i32 @may_access_after_return(i32 noundef %N, i32 noundef %M) {
 ; TUNIT-NEXT:  entry:
 ; TUNIT-NEXT:    [[A:%.*]] = alloca i32, align 4
 ; TUNIT-NEXT:    [[B:%.*]] = alloca i32, align 4
-; TUNIT-NEXT:    call void @write_both(ptr nocapture nofree noundef nonnull writeonly align 4 dereferenceable(4) [[A]], ptr nocapture nofree noundef nonnull writeonly align 4 dereferenceable(4) [[B]]) #[[ATTR18]]
+; TUNIT-NEXT:    call void @write_both(ptr nofree noundef nonnull writeonly align 4 captures(none) dereferenceable(4) [[A]], ptr nofree noundef nonnull writeonly align 4 captures(none) dereferenceable(4) [[B]]) #[[ATTR18]]
 ; TUNIT-NEXT:    ret i32 8
 ;
 ; CGSCC: Function Attrs: mustprogress nofree nosync nounwind willreturn memory(none)
@@ -3193,7 +3193,7 @@ define i32 @may_access_after_return(i32 noundef %N, i32 noundef %M) {
 ; CGSCC-NEXT:  entry:
 ; CGSCC-NEXT:    [[A:%.*]] = alloca i32, align 4
 ; CGSCC-NEXT:    [[B:%.*]] = alloca i32, align 4
-; CGSCC-NEXT:    call void @write_both(ptr noalias nocapture nofree noundef nonnull writeonly align 4 dereferenceable(4) [[A]], ptr noalias nocapture nofree noundef nonnull writeonly align 4 dereferenceable(4) [[B]]) #[[ATTR21]]
+; CGSCC-NEXT:    call void @write_both(ptr noalias nofree noundef nonnull writeonly align 4 captures(none) dereferenceable(4) [[A]], ptr noalias nofree noundef nonnull writeonly align 4 captures(none) dereferenceable(4) [[B]]) #[[ATTR21]]
 ; CGSCC-NEXT:    [[TMP0:%.*]] = load i32, ptr [[A]], align 4
 ; CGSCC-NEXT:    [[TMP1:%.*]] = load i32, ptr [[B]], align 4
 ; CGSCC-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP0]], [[TMP1]]
@@ -3214,7 +3214,7 @@ entry:
 define internal void @write_both(ptr noundef %Q, ptr noundef %R) {
 ; CHECK: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: write)
 ; CHECK-LABEL: define {{[^@]+}}@write_both
-; CHECK-SAME: (ptr nocapture nofree noundef nonnull writeonly align 4 dereferenceable(4) [[Q:%.*]], ptr nocapture nofree noundef nonnull writeonly align 4 dereferenceable(4) [[R:%.*]]) #[[ATTR0]] {
+; CHECK-SAME: (ptr nofree noundef nonnull writeonly align 4 captures(none) dereferenceable(4) [[Q:%.*]], ptr nofree noundef nonnull writeonly align 4 captures(none) dereferenceable(4) [[R:%.*]]) #[[ATTR0]] {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    store i32 3, ptr [[Q]], align 4
 ; CHECK-NEXT:    store i32 5, ptr [[R]], align 4
@@ -3247,7 +3247,7 @@ define i32 @may_access_after_return_choice(i32 noundef %N, i32 noundef %M, i1 %c
 ; TUNIT-NEXT:    [[B:%.*]] = alloca i32, align 4
 ; TUNIT-NEXT:    [[CALL:%.*]] = call nonnull align 4 dereferenceable(4) ptr @passthrough_choice(i1 [[C]], ptr noalias nofree noundef nonnull readnone align 4 dereferenceable(4) "no-capture-maybe-returned" [[A]], ptr noalias nofree noundef nonnull readnone align 4 dereferenceable(4) "no-capture-maybe-returned" [[B]]) #[[ATTR23:[0-9]+]]
 ; TUNIT-NEXT:    [[CALL1:%.*]] = call nonnull align 4 dereferenceable(4) ptr @passthrough_choice(i1 [[C]], ptr noalias nofree noundef nonnull readnone align 4 dereferenceable(4) "no-capture-maybe-returned" [[B]], ptr noalias nofree noundef nonnull readnone align 4 dereferenceable(4) "no-capture-maybe-returned" [[A]]) #[[ATTR23]]
-; TUNIT-NEXT:    call void @write_both(ptr nocapture nofree noundef nonnull writeonly align 4 dereferenceable(4) [[CALL]], ptr nocapture nofree noundef nonnull writeonly align 4 dereferenceable(4) [[CALL1]]) #[[ATTR18]]
+; TUNIT-NEXT:    call void @write_both(ptr nofree noundef nonnull writeonly align 4 captures(none) dereferenceable(4) [[CALL]], ptr nofree noundef nonnull writeonly align 4 captures(none) dereferenceable(4) [[CALL1]]) #[[ATTR18]]
 ; TUNIT-NEXT:    [[TMP0:%.*]] = load i32, ptr [[A]], align 4
 ; TUNIT-NEXT:    [[TMP1:%.*]] = load i32, ptr [[B]], align 4
 ; TUNIT-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP0]], [[TMP1]]
@@ -3261,7 +3261,7 @@ define i32 @may_access_after_return_choice(i32 noundef %N, i32 noundef %M, i1 %c
 ; CGSCC-NEXT:    [[B:%.*]] = alloca i32, align 4
 ; CGSCC-NEXT:    [[CALL:%.*]] = call nonnull align 4 dereferenceable(4) ptr @passthrough_choice(i1 [[C]], ptr noalias nofree noundef nonnull readnone align 4 dereferenceable(4) [[A]], ptr noalias nofree noundef nonnull readnone align 4 dereferenceable(4) [[B]]) #[[ATTR28:[0-9]+]]
 ; CGSCC-NEXT:    [[CALL1:%.*]] = call nonnull align 4 dereferenceable(4) ptr @passthrough_choice(i1 [[C]], ptr noalias nofree noundef nonnull readnone align 4 dereferenceable(4) [[B]], ptr noalias nofree noundef nonnull readnone align 4 dereferenceable(4) [[A]]) #[[ATTR28]]
-; CGSCC-NEXT:    call void @write_both(ptr nocapture nofree noundef nonnull writeonly align 4 dereferenceable(4) [[CALL]], ptr nocapture nofree noundef nonnull writeonly align 4 dereferenceable(4) [[CALL1]]) #[[ATTR21]]
+; CGSCC-NEXT:    call void @write_both(ptr nofree noundef nonnull writeonly align 4 captures(none) dereferenceable(4) [[CALL]], ptr nofree noundef nonnull writeonly align 4 captures(none) dereferenceable(4) [[CALL1]]) #[[ATTR21]]
 ; CGSCC-NEXT:    [[TMP0:%.*]] = load i32, ptr [[A]], align 4
 ; CGSCC-NEXT:    [[TMP1:%.*]] = load i32, ptr [[B]], align 4
 ; CGSCC-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP0]], [[TMP1]]
@@ -3300,7 +3300,7 @@ define i32 @may_access_after_return_no_choice1(i32 noundef %N, i32 noundef %M) {
 ; TUNIT-NEXT:  entry:
 ; TUNIT-NEXT:    [[A:%.*]] = alloca i32, align 4
 ; TUNIT-NEXT:    [[B:%.*]] = alloca i32, align 4
-; TUNIT-NEXT:    call void @write_both(ptr nocapture nofree noundef nonnull writeonly align 4 dereferenceable(4) [[A]], ptr nocapture nofree noundef nonnull writeonly align 4 dereferenceable(4) [[B]]) #[[ATTR18]]
+; TUNIT-NEXT:    call void @write_both(ptr nofree noundef nonnull writeonly align 4 captures(none) dereferenceable(4) [[A]], ptr nofree noundef nonnull writeonly align 4 captures(none) dereferenceable(4) [[B]]) #[[ATTR18]]
 ; TUNIT-NEXT:    ret i32 8
 ;
 ; CGSCC: Function Attrs: mustprogress nofree nosync nounwind willreturn memory(none)
@@ -3309,7 +3309,7 @@ define i32 @may_access_after_return_no_choice1(i32 noundef %N, i32 noundef %M) {
 ; CGSCC-NEXT:  entry:
 ; CGSCC-NEXT:    [[A:%.*]] = alloca i32, align 4
 ; CGSCC-NEXT:    [[B:%.*]] = alloca i32, align 4
-; CGSCC-NEXT:    call void @write_both(ptr noalias nocapture nofree noundef nonnull writeonly align 4 dereferenceable(4) [[A]], ptr noalias nocapture nofree noundef nonnull writeonly align 4 dereferenceable(4) [[B]]) #[[ATTR21]]
+; CGSCC-NEXT:    call void @write_both(ptr noalias nofree noundef nonnull writeonly align 4 captures(none) dereferenceable(4) [[A]], ptr noalias nofree noundef nonnull writeonly align 4 captures(none) dereferenceable(4) [[B]]) #[[ATTR21]]
 ; CGSCC-NEXT:    [[TMP0:%.*]] = load i32, ptr [[A]], align 4
 ; CGSCC-NEXT:    [[TMP1:%.*]] = load i32, ptr [[B]], align 4
 ; CGSCC-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP0]], [[TMP1]]
@@ -3335,7 +3335,7 @@ define i32 @may_access_after_return_no_choice2(i32 noundef %N, i32 noundef %M) {
 ; TUNIT-NEXT:  entry:
 ; TUNIT-NEXT:    [[A:%.*]] = alloca i32, align 4
 ; TUNIT-NEXT:    [[B:%.*]] = alloca i32, align 4
-; TUNIT-NEXT:    call void @write_both(ptr nocapture nofree noundef nonnull writeonly align 4 dereferenceable(4) [[B]], ptr nocapture nofree noundef nonnull writeonly align 4 dereferenceable(4) [[A]]) #[[ATTR18]]
+; TUNIT-NEXT:    call void @write_both(ptr nofree noundef nonnull writeonly align 4 captures(none) dereferenceable(4) [[B]], ptr nofree noundef nonnull writeonly align 4 captures(none) dereferenceable(4) [[A]]) #[[ATTR18]]
 ; TUNIT-NEXT:    ret i32 8
 ;
 ; CGSCC: Function Attrs: mustprogress nofree nosync nounwind willreturn memory(none)
@@ -3344,7 +3344,7 @@ define i32 @may_access_after_return_no_choice2(i32 noundef %N, i32 noundef %M) {
 ; CGSCC-NEXT:  entry:
 ; CGSCC-NEXT:    [[A:%.*]] = alloca i32, align 4
 ; CGSCC-NEXT:    [[B:%.*]] = alloca i32, align 4
-; CGSCC-NEXT:    call void @write_both(ptr noalias nocapture nofree noundef nonnull writeonly align 4 dereferenceable(4) [[B]], ptr noalias nocapture nofree noundef nonnull writeonly align 4 dereferenceable(4) [[A]]) #[[ATTR21]]
+; CGSCC-NEXT:    call void @write_both(ptr noalias nofree noundef nonnull writeonly align 4 captures(none) dereferenceable(4) [[B]], ptr noalias nofree noundef nonnull writeonly align 4 captures(none) dereferenceable(4) [[A]]) #[[ATTR21]]
 ; CGSCC-NEXT:    [[TMP0:%.*]] = load i32, ptr [[A]], align 4
 ; CGSCC-NEXT:    [[TMP1:%.*]] = load i32, ptr [[B]], align 4
 ; CGSCC-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP0]], [[TMP1]]
diff --git a/llvm/test/Transforms/Attributor/value-simplify-reachability.ll b/llvm/test/Transforms/Attributor/value-simplify-reachability.ll
index 2dc2ec203692a..a1b9d9adc4aab 100644
--- a/llvm/test/Transforms/Attributor/value-simplify-reachability.ll
+++ b/llvm/test/Transforms/Attributor/value-simplify-reachability.ll
@@ -692,7 +692,7 @@ declare void @usei32(i32) nocallback nosync
 define internal void @exclusion_set3_helper(i1 %c, ptr %p) {
 ; TUNIT: Function Attrs: nosync
 ; TUNIT-LABEL: define {{[^@]+}}@exclusion_set3_helper
-; TUNIT-SAME: (i1 noundef [[C:%.*]], ptr noalias nocapture nofree noundef nonnull align 4 dereferenceable(4) [[P:%.*]]) #[[ATTR8:[0-9]+]] {
+; TUNIT-SAME: (i1 noundef [[C:%.*]], ptr noalias nofree noundef nonnull align 4 captures(none) dereferenceable(4) [[P:%.*]]) #[[ATTR8:[0-9]+]] {
 ; TUNIT-NEXT:    br i1 [[C]], label [[T:%.*]], label [[F:%.*]]
 ; TUNIT:       t:
 ; TUNIT-NEXT:    store i32 42, ptr [[P]], align 4
@@ -707,7 +707,7 @@ define internal void @exclusion_set3_helper(i1 %c, ptr %p) {
 ; TUNIT-NEXT:    [[USE1:%.*]] = load i32, ptr [[P]], align 4
 ; TUNIT-NEXT:    call void @usei32(i32 [[USE1]])
 ; TUNIT-NEXT:    store i32 77, ptr [[P]], align 4
-; TUNIT-NEXT:    call void @exclusion_set3_helper(i1 noundef true, ptr noalias nocapture nofree noundef nonnull align 4 dereferenceable(4) [[P]]) #[[ATTR8]]
+; TUNIT-NEXT:    call void @exclusion_set3_helper(i1 noundef true, ptr noalias nofree noundef nonnull align 4 captures(none) dereferenceable(4) [[P]]) #[[ATTR8]]
 ; TUNIT-NEXT:    [[USE2:%.*]] = load i32, ptr [[P]], align 4
 ; TUNIT-NEXT:    call void @usei32(i32 noundef [[USE2]])
 ; TUNIT-NEXT:    br label [[T]]
@@ -717,7 +717,7 @@ define internal void @exclusion_set3_helper(i1 %c, ptr %p) {
 ;
 ; CGSCC: Function Attrs: nosync
 ; CGSCC-LABEL: define {{[^@]+}}@exclusion_set3_helper
-; CGSCC-SAME: (i1 noundef [[C:%.*]], ptr noalias nocapture nofree noundef nonnull align 4 dereferenceable(4) [[P:%.*]]) #[[ATTR5]] {
+; CGSCC-SAME: (i1 noundef [[C:%.*]], ptr noalias nofree noundef nonnull align 4 captures(none) dereferenceable(4) [[P:%.*]]) #[[ATTR5]] {
 ; CGSCC-NEXT:    br i1 [[C]], label [[T:%.*]], label [[F:%.*]]
 ; CGSCC:       t:
 ; CGSCC-NEXT:    store i32 42, ptr [[P]], align 4
@@ -732,7 +732,7 @@ define internal void @exclusion_set3_helper(i1 %c, ptr %p) {
 ; CGSCC-NEXT:    [[USE1:%.*]] = load i32, ptr [[P]], align 4
 ; CGSCC-NEXT:    call void @usei32(i32 [[USE1]])
 ; CGSCC-NEXT:    store i32 77, ptr [[P]], align 4
-; CGSCC-NEXT:    call void @exclusion_set3_helper(i1 noundef true, ptr noalias nocapture nofree noundef nonnull align 4 dereferenceable(4) [[P]]) #[[ATTR5]]
+; CGSCC-NEXT:    call void @exclusion_set3_helper(i1 noundef true, ptr noalias nofree noundef nonnull align 4 captures(none) dereferenceable(4) [[P]]) #[[ATTR5]]
 ; CGSCC-NEXT:    [[USE2:%.*]] = load i32, ptr [[P]], align 4
 ; CGSCC-NEXT:    call void @usei32(i32 [[USE2]])
 ; CGSCC-NEXT:    br label [[T]]
@@ -771,7 +771,7 @@ define i32 @exclusion_set3(i1 %c) {
 ; TUNIT-SAME: (i1 [[C:%.*]]) #[[ATTR5]] {
 ; TUNIT-NEXT:    [[A:%.*]] = alloca i32, align 4
 ; TUNIT-NEXT:    store i32 3, ptr [[A]], align 4
-; TUNIT-NEXT:    call void @exclusion_set3_helper(i1 noundef [[C]], ptr noalias nocapture nofree noundef nonnull align 4 dereferenceable(4) [[A]]) #[[ATTR8]]
+; TUNIT-NEXT:    call void @exclusion_set3_helper(i1 noundef [[C]], ptr noalias nofree noundef nonnull align 4 captures(none) dereferenceable(4) [[A]]) #[[ATTR8]]
 ; TUNIT-NEXT:    [[FINAL:%.*]] = load i32, ptr [[A]], align 4
 ; TUNIT-NEXT:    ret i32 [[FINAL]]
 ;
@@ -780,7 +780,7 @@ define i32 @exclusion_set3(i1 %c) {
 ; CGSCC-SAME: (i1 noundef [[C:%.*]]) #[[ATTR5]] {
 ; CGSCC-NEXT:    [[A:%.*]] = alloca i32, align 4
 ; CGSCC-NEXT:    store i32 3, ptr [[A]], align 4
-; CGSCC-NEXT:    call void @exclusion_set3_helper(i1 noundef [[C]], ptr noalias nocapture nofree noundef nonnull align 4 dereferenceable(4) [[A]])
+; CGSCC-NEXT:    call void @exclusion_set3_helper(i1 noundef [[C]], ptr noalias nofree noundef nonnull align 4 captures(none) dereferenceable(4) [[A]])
 ; CGSCC-NEXT:    [[FINAL:%.*]] = load i32, ptr [[A]], align 4
 ; CGSCC-NEXT:    ret i32 [[FINAL]]
 ;
@@ -805,7 +805,7 @@ define internal i32 @readI32(ptr %a) {
 ;
 ; CGSCC: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn
 ; CGSCC-LABEL: define {{[^@]+}}@readI32
-; CGSCC-SAME: (ptr nocapture nofree noundef nonnull readonly align 4 dereferenceable(4) [[A:%.*]]) #[[ATTR9:[0-9]+]] {
+; CGSCC-SAME: (ptr nofree noundef nonnull readonly align 4 captures(none) dereferenceable(4) [[A:%.*]]) #[[ATTR9:[0-9]+]] {
 ; CGSCC-NEXT:    [[R:%.*]] = load i32, ptr [[A]], align 4
 ; CGSCC-NEXT:    store i32 1, ptr @B, align 4
 ; CGSCC-NEXT:    ret i32 [[R]]
@@ -830,7 +830,7 @@ define internal i32 @broker(ptr %a) {
 ; CGSCC-SAME: (i32 [[TMP0:%.*]]) #[[ATTR10:[0-9]+]] {
 ; CGSCC-NEXT:    [[A_PRIV:%.*]] = alloca i32, align 4
 ; CGSCC-NEXT:    store i32 [[TMP0]], ptr [[A_PRIV]], align 4
-; CGSCC-NEXT:    [[R:%.*]] = call i32 @readI32(ptr noalias nocapture nofree noundef nonnull readonly align 4 dereferenceable(4) [[A_PRIV]]) #[[ATTR14:[0-9]+]]
+; CGSCC-NEXT:    [[R:%.*]] = call i32 @readI32(ptr noalias nofree noundef nonnull readonly align 4 captures(none) dereferenceable(4) [[A_PRIV]]) #[[ATTR14:[0-9]+]]
 ; CGSCC-NEXT:    ret i32 [[R]]
 ;
   %r = call i32 @readI32(ptr %a)
diff --git a/llvm/test/Transforms/Attributor/value-simplify.ll b/llvm/test/Transforms/Attributor/value-simplify.ll
index a5789790cc92a..a90fb54fbe74a 100644
--- a/llvm/test/Transforms/Attributor/value-simplify.ll
+++ b/llvm/test/Transforms/Attributor/value-simplify.ll
@@ -519,13 +519,13 @@ define internal void @test_sret(ptr sret(%struct.X) %a, ptr %b) {
 ;
 ; TUNIT: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: write)
 ; TUNIT-LABEL: define {{[^@]+}}@test_sret
-; TUNIT-SAME: (ptr noalias nofree noundef writeonly sret([[STRUCT_X:%.*]]) align 4294967296 dereferenceable_or_null(8) [[A:%.*]], ptr nocapture nofree noundef nonnull writeonly align 8 dereferenceable(8) [[B:%.*]]) #[[ATTR4:[0-9]+]] {
+; TUNIT-SAME: (ptr noalias nofree noundef writeonly sret([[STRUCT_X:%.*]]) align 4294967296 dereferenceable_or_null(8) [[A:%.*]], ptr nofree noundef nonnull writeonly align 8 captures(none) dereferenceable(8) [[B:%.*]]) #[[ATTR4:[0-9]+]] {
 ; TUNIT-NEXT:    store ptr [[A]], ptr [[B]], align 8
 ; TUNIT-NEXT:    ret void
 ;
 ; CGSCC: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: write)
 ; CGSCC-LABEL: define {{[^@]+}}@test_sret
-; CGSCC-SAME: (ptr noalias nofree noundef writeonly sret([[STRUCT_X:%.*]]) align 4294967296 dereferenceable_or_null(8) [[A:%.*]], ptr nocapture nofree noundef nonnull writeonly align 8 dereferenceable(8) [[B:%.*]]) #[[ATTR5:[0-9]+]] {
+; CGSCC-SAME: (ptr noalias nofree noundef writeonly sret([[STRUCT_X:%.*]]) align 4294967296 dereferenceable_or_null(8) [[A:%.*]], ptr nofree noundef nonnull writeonly align 8 captures(none) dereferenceable(8) [[B:%.*]]) #[[ATTR5:[0-9]+]] {
 ; CGSCC-NEXT:    store ptr [[A]], ptr [[B]], align 8
 ; CGSCC-NEXT:    ret void
 ;
@@ -538,14 +538,14 @@ define void @complicated_args_sret(ptr %b) {
 ;
 ; TUNIT: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: write)
 ; TUNIT-LABEL: define {{[^@]+}}@complicated_args_sret
-; TUNIT-SAME: (ptr nocapture nofree writeonly [[B:%.*]]) #[[ATTR4]] {
-; TUNIT-NEXT:    call void @test_sret(ptr nofree noundef writeonly sret([[STRUCT_X:%.*]]) align 4294967296 null, ptr nocapture nofree noundef writeonly align 8 [[B]]) #[[ATTR11:[0-9]+]]
+; TUNIT-SAME: (ptr nofree writeonly captures(none) [[B:%.*]]) #[[ATTR4]] {
+; TUNIT-NEXT:    call void @test_sret(ptr nofree noundef writeonly sret([[STRUCT_X:%.*]]) align 4294967296 null, ptr nofree noundef writeonly align 8 captures(none) [[B]]) #[[ATTR11:[0-9]+]]
 ; TUNIT-NEXT:    ret void
 ;
 ; CGSCC: Function Attrs: mustprogress nofree nosync nounwind willreturn memory(argmem: write)
 ; CGSCC-LABEL: define {{[^@]+}}@complicated_args_sret
-; CGSCC-SAME: (ptr nocapture nofree noundef nonnull writeonly align 8 dereferenceable(8) [[B:%.*]]) #[[ATTR6:[0-9]+]] {
-; CGSCC-NEXT:    call void @test_sret(ptr nofree noundef writeonly sret([[STRUCT_X:%.*]]) align 4294967296 dereferenceable_or_null(8) null, ptr nocapture nofree noundef nonnull writeonly align 8 dereferenceable(8) [[B]]) #[[ATTR15:[0-9]+]]
+; CGSCC-SAME: (ptr nofree noundef nonnull writeonly align 8 captures(none) dereferenceable(8) [[B:%.*]]) #[[ATTR6:[0-9]+]] {
+; CGSCC-NEXT:    call void @test_sret(ptr nofree noundef writeonly sret([[STRUCT_X:%.*]]) align 4294967296 dereferenceable_or_null(8) null, ptr nofree noundef nonnull writeonly align 8 captures(none) dereferenceable(8) [[B]]) #[[ATTR15:[0-9]+]]
 ; CGSCC-NEXT:    ret void
 ;
   call void @test_sret(ptr sret(%struct.X) null, ptr %b)
@@ -555,7 +555,7 @@ define void @complicated_args_sret(ptr %b) {
 define internal ptr @test_nest(ptr nest %a) {
 ; CGSCC: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(none)
 ; CGSCC-LABEL: define {{[^@]+}}@test_nest
-; CGSCC-SAME: (ptr nest noalias nocapture nofree readnone align 4294967296 [[A:%.*]]) #[[ATTR1]] {
+; CGSCC-SAME: (ptr nest noalias nofree readnone align 4294967296 captures(none) [[A:%.*]]) #[[ATTR1]] {
 ; CGSCC-NEXT:    ret ptr null
 ;
   ret ptr %a
@@ -645,7 +645,7 @@ define ptr @complicated_args_byval2() {
 define void @fixpoint_changed(ptr %p) {
 ; TUNIT: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: write)
 ; TUNIT-LABEL: define {{[^@]+}}@fixpoint_changed
-; TUNIT-SAME: (ptr nocapture nofree writeonly [[P:%.*]]) #[[ATTR4]] {
+; TUNIT-SAME: (ptr nofree writeonly captures(none) [[P:%.*]]) #[[ATTR4]] {
 ; TUNIT-NEXT:  entry:
 ; TUNIT-NEXT:    br label [[FOR_COND:%.*]]
 ; TUNIT:       for.cond:
@@ -668,7 +668,7 @@ define void @fixpoint_changed(ptr %p) {
 ;
 ; CGSCC: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: write)
 ; CGSCC-LABEL: define {{[^@]+}}@fixpoint_changed
-; CGSCC-SAME: (ptr nocapture nofree writeonly [[P:%.*]]) #[[ATTR5]] {
+; CGSCC-SAME: (ptr nofree writeonly captures(none) [[P:%.*]]) #[[ATTR5]] {
 ; CGSCC-NEXT:  entry:
 ; CGSCC-NEXT:    br label [[FOR_COND:%.*]]
 ; CGSCC:       for.cond:
@@ -972,13 +972,13 @@ define i1 @icmp() {
 
 define void @test_callee_is_undef(ptr %fn) {
 ; TUNIT-LABEL: define {{[^@]+}}@test_callee_is_undef
-; TUNIT-SAME: (ptr nocapture nofree [[FN:%.*]]) {
-; TUNIT-NEXT:    call void @unknown_calle_arg_is_undef(ptr nocapture nofree noundef [[FN]])
+; TUNIT-SAME: (ptr nofree captures(none) [[FN:%.*]]) {
+; TUNIT-NEXT:    call void @unknown_calle_arg_is_undef(ptr nofree noundef captures(none) [[FN]])
 ; TUNIT-NEXT:    ret void
 ;
 ; CGSCC-LABEL: define {{[^@]+}}@test_callee_is_undef
-; CGSCC-SAME: (ptr nocapture nofree noundef nonnull [[FN:%.*]]) {
-; CGSCC-NEXT:    call void @unknown_calle_arg_is_undef(ptr nocapture nofree noundef nonnull [[FN]])
+; CGSCC-SAME: (ptr nofree noundef nonnull captures(none) [[FN:%.*]]) {
+; CGSCC-NEXT:    call void @unknown_calle_arg_is_undef(ptr nofree noundef nonnull captures(none) [[FN]])
 ; CGSCC-NEXT:    ret void
 ;
   call void @callee_is_undef(ptr undef)
@@ -998,7 +998,7 @@ define internal void @callee_is_undef(ptr %fn) {
 define internal void @unknown_calle_arg_is_undef(ptr %fn, i32 %arg) {
 ;
 ; CHECK-LABEL: define {{[^@]+}}@unknown_calle_arg_is_undef
-; CHECK-SAME: (ptr nocapture nofree noundef nonnull [[FN:%.*]]) {
+; CHECK-SAME: (ptr nofree noundef nonnull captures(none) [[FN:%.*]]) {
 ; CHECK-NEXT:    call void [[FN]](i32 undef)
 ; CHECK-NEXT:    ret void
 ;
@@ -1014,14 +1014,14 @@ define internal void @unknown_calle_arg_is_undef(ptr %fn, i32 %arg) {
 define internal void @f1(ptr %a) {
 ; TUNIT: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: write)
 ; TUNIT-LABEL: define {{[^@]+}}@f1
-; TUNIT-SAME: (ptr nocapture nofree noundef nonnull writeonly align 8 dereferenceable(8) [[A:%.*]]) #[[ATTR4]] {
+; TUNIT-SAME: (ptr nofree noundef nonnull writeonly align 8 captures(none) dereferenceable(8) [[A:%.*]]) #[[ATTR4]] {
 ; TUNIT-NEXT:  entry:
 ; TUNIT-NEXT:    store ptr @g, ptr [[A]], align 8
 ; TUNIT-NEXT:    ret void
 ;
 ; CGSCC: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: write)
 ; CGSCC-LABEL: define {{[^@]+}}@f1
-; CGSCC-SAME: (ptr nocapture nofree noundef nonnull writeonly align 8 dereferenceable(8) [[A:%.*]]) #[[ATTR5]] {
+; CGSCC-SAME: (ptr nofree noundef nonnull writeonly align 8 captures(none) dereferenceable(8) [[A:%.*]]) #[[ATTR5]] {
 ; CGSCC-NEXT:  entry:
 ; CGSCC-NEXT:    store ptr @g, ptr [[A]], align 8
 ; CGSCC-NEXT:    ret void
@@ -1035,7 +1035,7 @@ define internal void @f2(ptr %a) {
 ; CHECK-LABEL: define {{[^@]+}}@f2
 ; CHECK-SAME: (ptr [[A:%.*]]) {
 ; CHECK-NEXT:  cont461:
-; CHECK-NEXT:    call void @f3(ptr [[A]], ptr nocapture nofree [[A]])
+; CHECK-NEXT:    call void @f3(ptr [[A]], ptr nofree captures(none) [[A]])
 ; CHECK-NEXT:    ret void
 ;
 cont461:
@@ -1045,7 +1045,7 @@ cont461:
 
 define internal void @f3(ptr %a1, ptr %a) {
 ; CHECK-LABEL: define {{[^@]+}}@f3
-; CHECK-SAME: (ptr [[A1:%.*]], ptr nocapture nofree [[A:%.*]]) {
+; CHECK-SAME: (ptr [[A1:%.*]], ptr nofree captures(none) [[A:%.*]]) {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[CALL20:%.*]] = call i1 @f9()
 ; CHECK-NEXT:    br i1 [[CALL20]], label [[LAND_LHS_TRUE:%.*]], label [[IF_END40:%.*]]
@@ -1269,7 +1269,7 @@ define internal i8 @memcpy_uses_store(i8 %arg) {
 ; TUNIT-NEXT:    [[SRC:%.*]] = alloca i8, align 1
 ; TUNIT-NEXT:    [[DST:%.*]] = alloca i8, align 1
 ; TUNIT-NEXT:    store i8 [[ARG]], ptr [[SRC]], align 1
-; TUNIT-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr noalias nocapture nofree noundef nonnull writeonly dereferenceable(1) [[DST]], ptr noalias nocapture nofree noundef nonnull readonly dereferenceable(1) [[SRC]], i32 noundef 1, i1 noundef false) #[[ATTR12:[0-9]+]]
+; TUNIT-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr noalias nofree noundef nonnull writeonly captures(none) dereferenceable(1) [[DST]], ptr noalias nofree noundef nonnull readonly captures(none) dereferenceable(1) [[SRC]], i32 noundef 1, i1 noundef false) #[[ATTR12:[0-9]+]]
 ; TUNIT-NEXT:    [[L:%.*]] = load i8, ptr [[DST]], align 1
 ; TUNIT-NEXT:    ret i8 [[L]]
 ;
@@ -1279,7 +1279,7 @@ define internal i8 @memcpy_uses_store(i8 %arg) {
 ; CGSCC-NEXT:    [[SRC:%.*]] = alloca i8, align 1
 ; CGSCC-NEXT:    [[DST:%.*]] = alloca i8, align 1
 ; CGSCC-NEXT:    store i8 [[ARG]], ptr [[SRC]], align 1
-; CGSCC-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr noalias nocapture nofree noundef nonnull writeonly dereferenceable(1) [[DST]], ptr noalias nocapture nofree noundef nonnull readonly dereferenceable(1) [[SRC]], i32 noundef 1, i1 noundef false) #[[ATTR16:[0-9]+]]
+; CGSCC-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr noalias nofree noundef nonnull writeonly captures(none) dereferenceable(1) [[DST]], ptr noalias nofree noundef nonnull readonly captures(none) dereferenceable(1) [[SRC]], i32 noundef 1, i1 noundef false) #[[ATTR16:[0-9]+]]
 ; CGSCC-NEXT:    [[L:%.*]] = load i8, ptr [[DST]], align 1
 ; CGSCC-NEXT:    ret i8 [[L]]
 ;
diff --git a/llvm/test/Transforms/Attributor/willreturn.ll b/llvm/test/Transforms/Attributor/willreturn.ll
index 4a1f8788a60a5..b7ac7fc2970b0 100644
--- a/llvm/test/Transforms/Attributor/willreturn.ll
+++ b/llvm/test/Transforms/Attributor/willreturn.ll
@@ -238,7 +238,7 @@ define void @only_exit() local_unnamed_addr #0 {
 define void @conditional_exit(i32 %0, ptr nocapture readonly %1) local_unnamed_addr #0 {
 ; CHECK: Function Attrs: noinline nounwind uwtable
 ; CHECK-LABEL: define {{[^@]+}}@conditional_exit
-; CHECK-SAME: (i32 [[TMP0:%.*]], ptr nocapture nofree readonly [[TMP1:%.*]]) local_unnamed_addr #[[ATTR7:[0-9]+]] {
+; CHECK-SAME: (i32 [[TMP0:%.*]], ptr nofree readonly captures(none) [[TMP1:%.*]]) local_unnamed_addr #[[ATTR7:[0-9]+]] {
 ; CHECK-NEXT:    [[TMP3:%.*]] = icmp eq i32 [[TMP0]], 0
 ; CHECK-NEXT:    br i1 [[TMP3]], label [[TMP5:%.*]], label [[TMP4:%.*]]
 ; CHECK:       4:
@@ -417,7 +417,7 @@ declare i32 @__gxx_personality_v0(...)
 define i32 @loop_constant_trip_count(ptr nocapture readonly %0) #0 {
 ; CHECK: Function Attrs: mustprogress nofree noinline norecurse nosync nounwind willreturn memory(argmem: read) uwtable
 ; CHECK-LABEL: define {{[^@]+}}@loop_constant_trip_count
-; CHECK-SAME: (ptr nocapture nofree nonnull readonly dereferenceable(4) [[TMP0:%.*]]) #[[ATTR13:[0-9]+]] {
+; CHECK-SAME: (ptr nofree nonnull readonly captures(none) dereferenceable(4) [[TMP0:%.*]]) #[[ATTR13:[0-9]+]] {
 ; CHECK-NEXT:    br label [[TMP3:%.*]]
 ; CHECK:       2:
 ; CHECK-NEXT:    ret i32 [[TMP8:%.*]]
@@ -461,7 +461,7 @@ define i32 @loop_constant_trip_count(ptr nocapture readonly %0) #0 {
 define i32 @loop_trip_count_unbound(i32 %0, i32 %1, ptr nocapture readonly %2, i32 %3) local_unnamed_addr #0 {
 ; CHECK: Function Attrs: nofree noinline norecurse nosync nounwind memory(argmem: read) uwtable
 ; CHECK-LABEL: define {{[^@]+}}@loop_trip_count_unbound
-; CHECK-SAME: (i32 [[TMP0:%.*]], i32 [[TMP1:%.*]], ptr nocapture nofree readonly [[TMP2:%.*]], i32 [[TMP3:%.*]]) local_unnamed_addr #[[ATTR14:[0-9]+]] {
+; CHECK-SAME: (i32 [[TMP0:%.*]], i32 [[TMP1:%.*]], ptr nofree readonly captures(none) [[TMP2:%.*]], i32 [[TMP3:%.*]]) local_unnamed_addr #[[ATTR14:[0-9]+]] {
 ; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq i32 [[TMP0]], [[TMP1]]
 ; CHECK-NEXT:    br i1 [[TMP5]], label [[TMP6:%.*]], label [[TMP8:%.*]]
 ; CHECK:       6:
@@ -512,7 +512,7 @@ define i32 @loop_trip_count_unbound(i32 %0, i32 %1, ptr nocapture readonly %2, i
 define i32 @loop_trip_dec(i32 %0, ptr nocapture readonly %1) local_unnamed_addr #0 {
 ; CHECK: Function Attrs: mustprogress nofree noinline norecurse nosync nounwind willreturn memory(argmem: read) uwtable
 ; CHECK-LABEL: define {{[^@]+}}@loop_trip_dec
-; CHECK-SAME: (i32 [[TMP0:%.*]], ptr nocapture nofree readonly [[TMP1:%.*]]) local_unnamed_addr #[[ATTR13]] {
+; CHECK-SAME: (i32 [[TMP0:%.*]], ptr nofree readonly captures(none) [[TMP1:%.*]]) local_unnamed_addr #[[ATTR13]] {
 ; CHECK-NEXT:    [[TMP3:%.*]] = icmp sgt i32 [[TMP0]], -1
 ; CHECK-NEXT:    br i1 [[TMP3]], label [[TMP4:%.*]], label [[TMP14:%.*]]
 ; CHECK:       4:
@@ -685,7 +685,7 @@ declare void @llvm.eh.sjlj.longjmp(ptr)
 define void @call_longjmp(ptr nocapture readnone %0) local_unnamed_addr #0 {
 ; CHECK: Function Attrs: noinline nounwind uwtable
 ; CHECK-LABEL: define {{[^@]+}}@call_longjmp
-; CHECK-SAME: (ptr nocapture nofree readnone [[TMP0:%.*]]) local_unnamed_addr #[[ATTR7]] {
+; CHECK-SAME: (ptr nofree readnone captures(none) [[TMP0:%.*]]) local_unnamed_addr #[[ATTR7]] {
 ; CHECK-NEXT:    tail call void @llvm.eh.sjlj.longjmp(ptr noalias nofree readnone [[TMP0]]) #[[ATTR5]]
 ; CHECK-NEXT:    unreachable
 ;
diff --git a/llvm/test/Transforms/CodeGenPrepare/AArch64/reduce-or-opt.ll b/llvm/test/Transforms/CodeGenPrepare/AArch64/reduce-or-opt.ll
index 52257c10b0bf6..14a6b2f6e9c4f 100644
--- a/llvm/test/Transforms/CodeGenPrepare/AArch64/reduce-or-opt.ll
+++ b/llvm/test/Transforms/CodeGenPrepare/AArch64/reduce-or-opt.ll
@@ -3,7 +3,7 @@
 
 define i64 @select_or_reduce_v2i1(ptr nocapture noundef readonly %src) {
 ; CHECK-LABEL: define i64 @select_or_reduce_v2i1(
-; CHECK-SAME: ptr nocapture noundef readonly [[SRC:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-SAME: ptr noundef readonly captures(none) [[SRC:%.*]]) #[[ATTR0:[0-9]+]] {
 ; CHECK-NEXT:  [[ENTRY:.*]]:
 ; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; CHECK:       [[VECTOR_BODY]]:
@@ -41,7 +41,7 @@ middle.split:
 
 define i64 @br_or_reduce_v2i1(ptr nocapture noundef readonly %src, ptr noundef readnone %p) {
 ; CHECK-LABEL: define i64 @br_or_reduce_v2i1(
-; CHECK-SAME: ptr nocapture noundef readonly [[SRC:%.*]], ptr noundef readnone [[P:%.*]]) #[[ATTR0]] {
+; CHECK-SAME: ptr noundef readonly captures(none) [[SRC:%.*]], ptr noundef readnone [[P:%.*]]) #[[ATTR0]] {
 ; CHECK-NEXT:  [[ENTRY:.*]]:
 ; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; CHECK:       [[VECTOR_BODY]]:
@@ -89,7 +89,7 @@ notfound:
 
 define i64 @select_or_reduce_nxv2i1(ptr nocapture noundef readonly %src) {
 ; CHECK-LABEL: define i64 @select_or_reduce_nxv2i1(
-; CHECK-SAME: ptr nocapture noundef readonly [[SRC:%.*]]) #[[ATTR0]] {
+; CHECK-SAME: ptr noundef readonly captures(none) [[SRC:%.*]]) #[[ATTR0]] {
 ; CHECK-NEXT:  [[ENTRY:.*]]:
 ; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; CHECK:       [[VECTOR_BODY]]:
@@ -133,7 +133,7 @@ middle.split:
 
 define i64 @br_or_reduce_nxv2i1(ptr nocapture noundef readonly %src, ptr noundef readnone %p) {
 ; CHECK-LABEL: define i64 @br_or_reduce_nxv2i1(
-; CHECK-SAME: ptr nocapture noundef readonly [[SRC:%.*]], ptr noundef readnone [[P:%.*]]) #[[ATTR0]] {
+; CHECK-SAME: ptr noundef readonly captures(none) [[SRC:%.*]], ptr noundef readnone [[P:%.*]]) #[[ATTR0]] {
 ; CHECK-NEXT:  [[ENTRY:.*]]:
 ; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; CHECK:       [[VECTOR_BODY]]:
diff --git a/llvm/test/Transforms/CodeGenPrepare/X86/fake-use-split-ret.ll b/llvm/test/Transforms/CodeGenPrepare/X86/fake-use-split-ret.ll
index b2cf89f6f2dd8..9f356b5e3cb42 100644
--- a/llvm/test/Transforms/CodeGenPrepare/X86/fake-use-split-ret.ll
+++ b/llvm/test/Transforms/CodeGenPrepare/X86/fake-use-split-ret.ll
@@ -2,7 +2,7 @@
 ;
 ; Ensure return instruction splitting ignores fake uses.
 ;
-; IR Generated with clang -O2 -S -emit-llvm -fextend-lifetimes test.cpp
+; IR Generated with clang -O2 -S -emit-llvm -fextend-variable-liveness test.cpp
 ;
 ;// test.cpp
 ;extern int bar(int);
diff --git a/llvm/test/Transforms/CodeGenPrepare/X86/section-samplepgo.ll b/llvm/test/Transforms/CodeGenPrepare/X86/section-samplepgo.ll
index 58af88d8cf365..48d02e5cebc69 100644
--- a/llvm/test/Transforms/CodeGenPrepare/X86/section-samplepgo.ll
+++ b/llvm/test/Transforms/CodeGenPrepare/X86/section-samplepgo.ll
@@ -34,8 +34,8 @@ define void @cold_func() !prof !16 {
   ret void
 }
 
-; CHECK: ![[HOT_ID]] = !{!"function_section_prefix", !"hot"}
-; CHECK: ![[COLD_ID]] = !{!"function_section_prefix", !"unlikely"}
+; CHECK: ![[HOT_ID]] = !{!"section_prefix", !"hot"}
+; CHECK: ![[COLD_ID]] = !{!"section_prefix", !"unlikely"}
 !llvm.module.flags = !{!1}
 !1 = !{i32 1, !"ProfileSummary", !2}
 !2 = !{!3, !4, !5, !6, !7, !8, !9, !10}
diff --git a/llvm/test/Transforms/CodeGenPrepare/X86/section.ll b/llvm/test/Transforms/CodeGenPrepare/X86/section.ll
index 6dad1122e4294..4baa0b5baa4be 100644
--- a/llvm/test/Transforms/CodeGenPrepare/X86/section.ll
+++ b/llvm/test/Transforms/CodeGenPrepare/X86/section.ll
@@ -66,8 +66,8 @@ define void @cold_func3() !prof !16 {
   ret void
 }
 
-; CHECK: ![[HOT_ID]] = !{!"function_section_prefix", !"hot"}
-; CHECK: ![[COLD_ID]] = !{!"function_section_prefix", !"unlikely"}
+; CHECK: ![[HOT_ID]] = !{!"section_prefix", !"hot"}
+; CHECK: ![[COLD_ID]] = !{!"section_prefix", !"unlikely"}
 !llvm.module.flags = !{!1}
 !1 = !{i32 1, !"ProfileSummary", !2}
 !2 = !{!3, !4, !5, !6, !7, !8, !9, !10}
diff --git a/llvm/test/Transforms/Coroutines/coro-async.ll b/llvm/test/Transforms/Coroutines/coro-async.ll
index acc448559c7fa..e5d2e6c2c42c6 100644
--- a/llvm/test/Transforms/Coroutines/coro-async.ll
+++ b/llvm/test/Transforms/Coroutines/coro-async.ll
@@ -143,7 +143,7 @@ define void @my_async_function_pa(ptr %ctxt, ptr %task, ptr %actor) {
 ; CHECK:   ret void
 ; CHECK: }
 
-; CHECK-LABEL: define internal swiftcc void @my_async_functionTQ0_(ptr nocapture readonly swiftasync %0, ptr %1, ptr nocapture readnone %2)
+; CHECK-LABEL: define internal swiftcc void @my_async_functionTQ0_(ptr readonly swiftasync captures(none) %0, ptr %1, ptr readnone captures(none) %2)
 ; CHECK-O0-LABEL: define internal swiftcc void @my_async_functionTQ0_(ptr swiftasync %0, ptr %1, ptr %2)
 ; CHECK-SAME: !dbg ![[SP2:[0-9]+]] {
 ; CHECK: entryresume.0:
@@ -228,7 +228,7 @@ entry:
 ; CHECK: tail call swiftcc void @asyncSuspend(ptr nonnull [[CALLEE_CTXT]], ptr %task, ptr %actor)
 ; CHECK: ret void
 
-; CHECK-LABEL: define internal swiftcc void @my_async_function2.resume.0(ptr %0, ptr nocapture readnone %1, ptr nocapture readonly %2)
+; CHECK-LABEL: define internal swiftcc void @my_async_function2.resume.0(ptr %0, ptr readnone captures(none) %1, ptr readonly captures(none) %2)
 ; CHECK-SAME: #[[FRAMEPOINTER]]
 ; CHECK-SAME: !dbg ![[SP4:[0-9]+]]
 ; CHECK: [[CALLEE_CTXT:%.*]] = load ptr, ptr %2
@@ -238,7 +238,7 @@ entry:
 ; CHECK: tail call swiftcc void @asyncSuspend(ptr [[CALLEE_CTXT_RELOAD]]
 ; CHECK: ret void
 
-; CHECK-LABEL: define internal swiftcc void @my_async_function2.resume.1(ptr nocapture readonly %0, ptr %1, ptr nocapture readnone %2)
+; CHECK-LABEL: define internal swiftcc void @my_async_function2.resume.1(ptr readonly captures(none) %0, ptr %1, ptr readnone captures(none) %2)
 ; CHECK-SAME: #[[FRAMEPOINTER]]
 ; CHECK: tail call swiftcc void @asyncReturn({{.*}}%1)
 ; CHECK: ret void
diff --git a/llvm/test/Transforms/Coroutines/coro-noalias-param.ll b/llvm/test/Transforms/Coroutines/coro-noalias-param.ll
index 943d2a67978fa..e48ad8fddd5fe 100644
--- a/llvm/test/Transforms/Coroutines/coro-noalias-param.ll
+++ b/llvm/test/Transforms/Coroutines/coro-noalias-param.ll
@@ -24,7 +24,7 @@ suspend:
 }
 
 ; check that the noalias attribute is removed from the argument
-; CHECK: define void @f(ptr nocapture readonly align 8 %a)
+; CHECK: define void @f(ptr readonly align 8 captures(none) %a)
 
 declare token @llvm.coro.id(i32, ptr, ptr, ptr)
 declare ptr @llvm.coro.begin(token, ptr)
diff --git a/llvm/test/Transforms/CorrelatedValuePropagation/cond-at-use.ll b/llvm/test/Transforms/CorrelatedValuePropagation/cond-at-use.ll
index cca648bb6ea23..fa9323df4dff7 100644
--- a/llvm/test/Transforms/CorrelatedValuePropagation/cond-at-use.ll
+++ b/llvm/test/Transforms/CorrelatedValuePropagation/cond-at-use.ll
@@ -555,6 +555,30 @@ define i16 @and_elide(i16 noundef %x) {
   ret i16 %sel
 }
 
+define i16 @and_elide_trunc_cond(i16 noundef %x) {
+; CHECK-LABEL: @and_elide_trunc_cond(
+; CHECK-NEXT:    [[CMP:%.*]] = trunc nuw i16 [[X:%.*]] to i1
+; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[CMP]], i16 [[X]], i16 24
+; CHECK-NEXT:    ret i16 [[SEL]]
+;
+  %and = and i16 %x, 1
+  %cmp = trunc nuw i16 %x to i1
+  %sel = select i1 %cmp, i16 %and, i16 24
+  ret i16 %sel
+}
+
+define <2 x i8> @and_elide_trunc_cond_vec(<2 x i8> noundef %x) {
+; CHECK-LABEL: @and_elide_trunc_cond_vec(
+; CHECK-NEXT:    [[CMP:%.*]] = trunc nuw <2 x i8> [[X:%.*]] to <2 x i1>
+; CHECK-NEXT:    [[SEL:%.*]] = select <2 x i1> [[CMP]], <2 x i8> [[X]], <2 x i8> splat (i8 24)
+; CHECK-NEXT:    ret <2 x i8> [[SEL]]
+;
+  %and = and <2 x i8> %x, splat (i8 1)
+  %cmp = trunc nuw <2 x i8> %x to <2 x i1>
+  %sel = select <2 x i1> %cmp, <2 x i8> %and, <2 x i8> splat (i8 24)
+  ret <2 x i8> %sel
+}
+
 define i16 @cond_value_may_not_well_defined(i16 %x) {
 ; CHECK-LABEL: @cond_value_may_not_well_defined(
 ; CHECK-NEXT:    [[AND:%.*]] = and i16 [[X:%.*]], 7
diff --git a/llvm/test/Transforms/CorrelatedValuePropagation/icmp.ll b/llvm/test/Transforms/CorrelatedValuePropagation/icmp.ll
index 72f09a949a060..5fa8f4e1b9f29 100644
--- a/llvm/test/Transforms/CorrelatedValuePropagation/icmp.ll
+++ b/llvm/test/Transforms/CorrelatedValuePropagation/icmp.ll
@@ -1509,3 +1509,151 @@ end:
   ; %arg is within [-16, -8).
   ret void
 }
+
+define void @test_trunc_bittest(i8 %a) {
+; CHECK-LABEL: @test_trunc_bittest(
+; CHECK-NEXT:    [[TRUNC:%.*]] = trunc i8 [[A:%.*]] to i1
+; CHECK-NEXT:    br i1 [[TRUNC]], label [[IF_TRUE:%.*]], label [[IF_FALSE:%.*]]
+; CHECK:       if.true:
+; CHECK-NEXT:    call void @check1(i1 true)
+; CHECK-NEXT:    call void @check1(i1 false)
+; CHECK-NEXT:    [[CMP3:%.*]] = icmp ne i8 [[A]], 1
+; CHECK-NEXT:    call void @check1(i1 [[CMP3]])
+; CHECK-NEXT:    [[CMP4:%.*]] = icmp eq i8 [[A]], 1
+; CHECK-NEXT:    call void @check1(i1 [[CMP4]])
+; CHECK-NEXT:    [[CMP5:%.*]] = icmp ne i8 [[A]], -1
+; CHECK-NEXT:    call void @check1(i1 [[CMP5]])
+; CHECK-NEXT:    [[CMP6:%.*]] = icmp eq i8 [[A]], -1
+; CHECK-NEXT:    call void @check1(i1 [[CMP6]])
+; CHECK-NEXT:    ret void
+; CHECK:       if.false:
+; CHECK-NEXT:    ret void
+;
+  %trunc = trunc i8 %a to i1
+  br i1 %trunc, label %if.true, label %if.false
+
+if.true:
+  %cmp1 = icmp ne i8 %a, 0
+  call void @check1(i1 %cmp1)
+  %cmp2 = icmp eq i8 %a, 0
+  call void @check1(i1 %cmp2)
+  %cmp3 = icmp ne i8 %a, 1
+  call void @check1(i1 %cmp3)
+  %cmp4 = icmp eq i8 %a, 1
+  call void @check1(i1 %cmp4)
+  %cmp5 = icmp ne i8 %a, -1
+  call void @check1(i1 %cmp5)
+  %cmp6 = icmp eq i8 %a, -1
+  call void @check1(i1 %cmp6)
+  ret void
+
+if.false:
+  ret void
+}
+
+define void @test_trunc_not_bittest(i8 %a) {
+; CHECK-LABEL: @test_trunc_not_bittest(
+; CHECK-NEXT:    [[TRUNC:%.*]] = trunc i8 [[A:%.*]] to i1
+; CHECK-NEXT:    [[NOT:%.*]] = xor i1 [[TRUNC]], true
+; CHECK-NEXT:    br i1 [[NOT]], label [[IF_FALSE:%.*]], label [[IF_TRUE:%.*]]
+; CHECK:       if.true:
+; CHECK-NEXT:    call void @check1(i1 true)
+; CHECK-NEXT:    call void @check1(i1 false)
+; CHECK-NEXT:    [[CMP3:%.*]] = icmp ne i8 [[A]], 0
+; CHECK-NEXT:    call void @check1(i1 [[CMP3]])
+; CHECK-NEXT:    [[CMP4:%.*]] = icmp eq i8 [[A]], 0
+; CHECK-NEXT:    call void @check1(i1 [[CMP4]])
+; CHECK-NEXT:    [[CMP5:%.*]] = icmp ne i8 [[A]], -2
+; CHECK-NEXT:    call void @check1(i1 [[CMP5]])
+; CHECK-NEXT:    [[CMP6:%.*]] = icmp eq i8 [[A]], -2
+; CHECK-NEXT:    call void @check1(i1 [[CMP6]])
+; CHECK-NEXT:    ret void
+; CHECK:       if.false:
+; CHECK-NEXT:    ret void
+;
+  %trunc = trunc i8 %a to i1
+  %not = xor i1 %trunc, true
+  br i1 %not, label %if.true, label %if.false
+
+if.true:
+  %cmp1 = icmp ne i8 %a, -1
+  call void @check1(i1 %cmp1)
+  %cmp2 = icmp eq i8 %a, -1
+  call void @check1(i1 %cmp2)
+  %cmp3 = icmp ne i8 %a, 0
+  call void @check1(i1 %cmp3)
+  %cmp4 = icmp eq i8 %a, 0
+  call void @check1(i1 %cmp4)
+  %cmp5 = icmp ne i8 %a, -2
+  call void @check1(i1 %cmp5)
+  %cmp6 = icmp eq i8 %a, -2
+  call void @check1(i1 %cmp6)
+  ret void
+
+if.false:
+  ret void
+}
+
+define void @test_trunc_nuw_bittest(i8 %a) {
+; CHECK-LABEL: @test_trunc_nuw_bittest(
+; CHECK-NEXT:    [[TRUNC:%.*]] = trunc nuw i8 [[A:%.*]] to i1
+; CHECK-NEXT:    br i1 [[TRUNC]], label [[IF_TRUE:%.*]], label [[IF_FALSE:%.*]]
+; CHECK:       if.true:
+; CHECK-NEXT:    call void @check1(i1 true)
+; CHECK-NEXT:    call void @check1(i1 false)
+; CHECK-NEXT:    call void @check1(i1 false)
+; CHECK-NEXT:    call void @check1(i1 true)
+; CHECK-NEXT:    ret void
+; CHECK:       if.false:
+; CHECK-NEXT:    ret void
+;
+  %trunc = trunc nuw i8 %a to i1
+  br i1 %trunc, label %if.true, label %if.false
+
+if.true:
+  %cmp1 = icmp ne i8 %a, 0
+  call void @check1(i1 %cmp1)
+  %cmp2 = icmp eq i8 %a, 0
+  call void @check1(i1 %cmp2)
+  %cmp3 = icmp ne i8 %a, 1
+  call void @check1(i1 %cmp3)
+  %cmp4 = icmp eq i8 %a, 1
+  call void @check1(i1 %cmp4)
+  ret void
+
+if.false:
+  ret void
+}
+
+define void @test_trunc_nuw_not_bittest(i8 %a) {
+; CHECK-LABEL: @test_trunc_nuw_not_bittest(
+; CHECK-NEXT:    [[TRUNC:%.*]] = trunc nuw i8 [[A:%.*]] to i1
+; CHECK-NEXT:    [[NOT:%.*]] = xor i1 [[TRUNC]], true
+; CHECK-NEXT:    br i1 [[NOT]], label [[IF_TRUE:%.*]], label [[IF_FALSE:%.*]]
+; CHECK:       if.true:
+; CHECK-NEXT:    call void @check1(i1 false)
+; CHECK-NEXT:    call void @check1(i1 true)
+; CHECK-NEXT:    call void @check1(i1 true)
+; CHECK-NEXT:    call void @check1(i1 false)
+; CHECK-NEXT:    ret void
+; CHECK:       if.false:
+; CHECK-NEXT:    ret void
+;
+  %trunc = trunc nuw i8 %a to i1
+  %not = xor i1 %trunc, true
+  br i1 %not, label %if.true, label %if.false
+
+if.true:
+  %cmp1 = icmp ne i8 %a, 0
+  call void @check1(i1 %cmp1)
+  %cmp2 = icmp eq i8 %a, 0
+  call void @check1(i1 %cmp2)
+  %cmp3 = icmp ne i8 %a, 1
+  call void @check1(i1 %cmp3)
+  %cmp4 = icmp eq i8 %a, 1
+  call void @check1(i1 %cmp4)
+  ret void
+
+if.false:
+  ret void
+}
diff --git a/llvm/test/Transforms/CorrelatedValuePropagation/non-null.ll b/llvm/test/Transforms/CorrelatedValuePropagation/non-null.ll
index 35d866ed2d92e..53a94e13a1763 100644
--- a/llvm/test/Transforms/CorrelatedValuePropagation/non-null.ll
+++ b/llvm/test/Transforms/CorrelatedValuePropagation/non-null.ll
@@ -344,4 +344,105 @@ define i1 @test_store_same_block(ptr %arg) {
   ret i1 %cmp
 }
 
+
+define i1 @test_known_nonnull_at_callsite(ptr %src) {
+; CHECK-LABEL: @test_known_nonnull_at_callsite(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    call void @callee(ptr noundef nonnull [[SRC:%.*]])
+; CHECK-NEXT:    [[NONNULL:%.*]] = icmp eq ptr [[SRC]], null
+; CHECK-NEXT:    ret i1 false
+;
+entry:
+  call void @callee(ptr noundef nonnull %src)
+  %nonnull = icmp eq ptr %src, null
+  ret i1 %nonnull
+}
+
+define i1 @test_known_nonnull_mixed(ptr %src) {
+; CHECK-LABEL: @test_known_nonnull_mixed(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    call void @callee2(ptr nonnull [[SRC:%.*]])
+; CHECK-NEXT:    [[NONNULL:%.*]] = icmp eq ptr [[SRC]], null
+; CHECK-NEXT:    ret i1 false
+;
+entry:
+  call void @callee2(ptr nonnull %src)
+  %nonnull = icmp eq ptr %src, null
+  ret i1 %nonnull
+}
+
+define i1 @test_known_nonnull_at_callsite_dereferenceable(ptr %src) {
+; CHECK-LABEL: @test_known_nonnull_at_callsite_dereferenceable(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    call void @callee(ptr dereferenceable(1) [[SRC:%.*]])
+; CHECK-NEXT:    [[NONNULL:%.*]] = icmp eq ptr [[SRC]], null
+; CHECK-NEXT:    ret i1 false
+;
+entry:
+  call void @callee(ptr dereferenceable(1) %src)
+  %nonnull = icmp eq ptr %src, null
+  ret i1 %nonnull
+}
+
+define i1 @test_known_nonnull_at_callsite_gep_inbounds(ptr %src, i64 %x) {
+; CHECK-LABEL: @test_known_nonnull_at_callsite_gep_inbounds(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[GEP:%.*]] = getelementptr inbounds i8, ptr [[SRC:%.*]], i64 [[X:%.*]]
+; CHECK-NEXT:    call void @callee(ptr noundef nonnull [[GEP]])
+; CHECK-NEXT:    [[NONNULL:%.*]] = icmp eq ptr [[SRC]], null
+; CHECK-NEXT:    ret i1 false
+;
+entry:
+  %gep = getelementptr inbounds i8, ptr %src, i64 %x
+  call void @callee(ptr noundef nonnull %gep)
+  %nonnull = icmp eq ptr %src, null
+  ret i1 %nonnull
+}
+
+; Negative tests
+
+define i1 @test_known_nonnull_at_callsite_without_noundef(ptr %src) {
+; CHECK-LABEL: @test_known_nonnull_at_callsite_without_noundef(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    call void @callee(ptr nonnull [[SRC:%.*]])
+; CHECK-NEXT:    [[NONNULL:%.*]] = icmp eq ptr [[SRC]], null
+; CHECK-NEXT:    ret i1 [[NONNULL]]
+;
+entry:
+  call void @callee(ptr nonnull %src)
+  %nonnull = icmp eq ptr %src, null
+  ret i1 %nonnull
+}
+
+define i1 @test_known_nonnull_at_callsite_dereferenceable_null_is_defined(ptr %src) #0 {
+; CHECK-LABEL: @test_known_nonnull_at_callsite_dereferenceable_null_is_defined(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    call void @callee(ptr dereferenceable(1) [[SRC:%.*]])
+; CHECK-NEXT:    [[NONNULL:%.*]] = icmp eq ptr [[SRC]], null
+; CHECK-NEXT:    ret i1 [[NONNULL]]
+;
+entry:
+  call void @callee(ptr dereferenceable(1) %src)
+  %nonnull = icmp eq ptr %src, null
+  ret i1 %nonnull
+}
+
+define i1 @test_known_nonnull_at_callsite_gep_without_inbounds(ptr %src, i64 %x) {
+; CHECK-LABEL: @test_known_nonnull_at_callsite_gep_without_inbounds(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[GEP:%.*]] = getelementptr i8, ptr [[SRC:%.*]], i64 [[X:%.*]]
+; CHECK-NEXT:    call void @callee(ptr noundef nonnull [[GEP]])
+; CHECK-NEXT:    [[NONNULL:%.*]] = icmp eq ptr [[SRC]], null
+; CHECK-NEXT:    ret i1 [[NONNULL]]
+;
+entry:
+  %gep = getelementptr i8, ptr %src, i64 %x
+  call void @callee(ptr noundef nonnull %gep)
+  %nonnull = icmp eq ptr %src, null
+  ret i1 %nonnull
+}
+
+declare void @callee(ptr)
+declare void @callee2(ptr noundef)
+
 attributes #0 = { null_pointer_is_valid }
diff --git a/llvm/test/Transforms/DeadStoreElimination/inter-procedural.ll b/llvm/test/Transforms/DeadStoreElimination/inter-procedural.ll
index e590c5bf4004a..5f8ab56c22754 100644
--- a/llvm/test/Transforms/DeadStoreElimination/inter-procedural.ll
+++ b/llvm/test/Transforms/DeadStoreElimination/inter-procedural.ll
@@ -338,3 +338,17 @@ define i16 @global_var_alias() {
   ret i16 %l
 }
 
+declare void @byval_fn(ptr byval(i32) initializes((0, 4)) %am)
+
+define void @test_byval() {
+; CHECK-LABEL: @test_byval(
+; CHECK-NEXT:    [[A:%.*]] = alloca i32, align 4
+; CHECK-NEXT:    store i32 0, ptr [[A]], align 4
+; CHECK-NEXT:    call void @byval_fn(ptr [[A]])
+; CHECK-NEXT:    ret void
+;
+  %a = alloca i32
+  store i32 0, ptr %a
+  call void @byval_fn(ptr %a)
+  ret void
+}
diff --git a/llvm/test/Transforms/DeadStoreElimination/stores-of-existing-values.ll b/llvm/test/Transforms/DeadStoreElimination/stores-of-existing-values.ll
index 7193bb501c89d..4f38027f17213 100644
--- a/llvm/test/Transforms/DeadStoreElimination/stores-of-existing-values.ll
+++ b/llvm/test/Transforms/DeadStoreElimination/stores-of-existing-values.ll
@@ -746,3 +746,43 @@ define void @fixed_scalable_neg(ptr %ptr) vscale_range(1, 2) {
   store <4 x i64> zeroinitializer, ptr %ptr
   ret void
 }
+
+define void @memset_different_attributes_1(i1 %c, ptr %ptr) {
+; CHECK-LABEL: @memset_different_attributes_1(
+; CHECK-NEXT:    call void @llvm.memset.p0.i64(ptr [[PTR:%.*]], i8 0, i64 20, i1 false)
+; CHECK-NEXT:    br i1 [[C:%.*]], label [[IF:%.*]], label [[END:%.*]]
+; CHECK:       if:
+; CHECK-NEXT:    br label [[END]]
+; CHECK:       end:
+; CHECK-NEXT:    ret void
+;
+  call void @llvm.memset.p0.i64(ptr %ptr, i8 0, i64 20, i1 false)
+  br i1 %c, label %if, label %end
+
+if:
+  call void @llvm.memset.p0.i64(ptr dereferenceable(20) %ptr, i8 0, i64 20, i1 false)
+  br label %end
+
+end:
+  ret void
+}
+
+define void @memset_different_attributes_2(i1 %c, ptr %ptr) {
+; CHECK-LABEL: @memset_different_attributes_2(
+; CHECK-NEXT:    call void @llvm.memset.p0.i64(ptr dereferenceable(20) [[PTR:%.*]], i8 0, i64 20, i1 false)
+; CHECK-NEXT:    br i1 [[C:%.*]], label [[IF:%.*]], label [[END:%.*]]
+; CHECK:       if:
+; CHECK-NEXT:    br label [[END]]
+; CHECK:       end:
+; CHECK-NEXT:    ret void
+;
+  call void @llvm.memset.p0.i64(ptr dereferenceable(20) %ptr, i8 0, i64 20, i1 false)
+  br i1 %c, label %if, label %end
+
+if:
+  call void @llvm.memset.p0.i64(ptr %ptr, i8 0, i64 20, i1 false)
+  br label %end
+
+end:
+  ret void
+}
diff --git a/llvm/test/Transforms/DeadStoreElimination/trivial-dse-calls.ll b/llvm/test/Transforms/DeadStoreElimination/trivial-dse-calls.ll
index 4e4f47a18e00b..030d315bfd925 100644
--- a/llvm/test/Transforms/DeadStoreElimination/trivial-dse-calls.ll
+++ b/llvm/test/Transforms/DeadStoreElimination/trivial-dse-calls.ll
@@ -68,7 +68,7 @@ define void @test_dead_readwrite() {
 define i32 @test_neg_read_after() {
 ; CHECK-LABEL: @test_neg_read_after(
 ; CHECK-NEXT:    [[A:%.*]] = alloca i32, align 4
-; CHECK-NEXT:    call void @f(ptr nocapture writeonly [[A]]) #[[ATTR1:[0-9]+]]
+; CHECK-NEXT:    call void @f(ptr writeonly captures(none) [[A]]) #[[ATTR1:[0-9]+]]
 ; CHECK-NEXT:    [[RES:%.*]] = load i32, ptr [[A]], align 4
 ; CHECK-NEXT:    ret i32 [[RES]]
 ;
@@ -82,7 +82,7 @@ define i32 @test_neg_read_after() {
 define void @test_neg_infinite_loop() {
 ; CHECK-LABEL: @test_neg_infinite_loop(
 ; CHECK-NEXT:    [[A:%.*]] = alloca i32, align 4
-; CHECK-NEXT:    call void @f(ptr nocapture writeonly [[A]]) #[[ATTR2:[0-9]+]]
+; CHECK-NEXT:    call void @f(ptr writeonly captures(none) [[A]]) #[[ATTR2:[0-9]+]]
 ; CHECK-NEXT:    ret void
 ;
   %a = alloca i32, align 4
@@ -93,7 +93,7 @@ define void @test_neg_infinite_loop() {
 define void @test_neg_throw() {
 ; CHECK-LABEL: @test_neg_throw(
 ; CHECK-NEXT:    [[A:%.*]] = alloca i32, align 4
-; CHECK-NEXT:    call void @f(ptr nocapture writeonly [[A]]) #[[ATTR3:[0-9]+]]
+; CHECK-NEXT:    call void @f(ptr writeonly captures(none) [[A]]) #[[ATTR3:[0-9]+]]
 ; CHECK-NEXT:    ret void
 ;
   %a = alloca i32, align 4
@@ -104,7 +104,7 @@ define void @test_neg_throw() {
 define void @test_neg_extra_write() {
 ; CHECK-LABEL: @test_neg_extra_write(
 ; CHECK-NEXT:    [[A:%.*]] = alloca i32, align 4
-; CHECK-NEXT:    call void @f(ptr nocapture writeonly [[A]]) #[[ATTR4:[0-9]+]]
+; CHECK-NEXT:    call void @f(ptr writeonly captures(none) [[A]]) #[[ATTR4:[0-9]+]]
 ; CHECK-NEXT:    ret void
 ;
   %a = alloca i32, align 4
@@ -118,7 +118,7 @@ define void @test_neg_unmodeled_write() {
 ; CHECK-LABEL: @test_neg_unmodeled_write(
 ; CHECK-NEXT:    [[A:%.*]] = alloca i32, align 4
 ; CHECK-NEXT:    [[A2:%.*]] = alloca i32, align 4
-; CHECK-NEXT:    call void @f2(ptr nocapture writeonly [[A]], ptr [[A2]]) #[[ATTR1]]
+; CHECK-NEXT:    call void @f2(ptr writeonly captures(none) [[A]], ptr [[A2]]) #[[ATTR1]]
 ; CHECK-NEXT:    ret void
 ;
   %a = alloca i32, align 4
@@ -149,7 +149,7 @@ define i32 @test_neg_captured_before() {
 ; CHECK-NEXT:    [[A:%.*]] = alloca i32, align 4
 ; CHECK-NEXT:    [[A2:%.*]] = alloca ptr, align 4
 ; CHECK-NEXT:    store ptr [[A]], ptr [[A2]], align 8
-; CHECK-NEXT:    call void @f(ptr nocapture writeonly [[A]]) #[[ATTR1]]
+; CHECK-NEXT:    call void @f(ptr writeonly captures(none) [[A]]) #[[ATTR1]]
 ; CHECK-NEXT:    [[A_COPY_CAST:%.*]] = load ptr, ptr [[A2]], align 8
 ; CHECK-NEXT:    [[RES:%.*]] = load i32, ptr [[A_COPY_CAST]], align 4
 ; CHECK-NEXT:    ret i32 [[RES]]
@@ -167,7 +167,7 @@ define i32 @test_neg_captured_before() {
 define void @test_new_op_bundle() {
 ; CHECK-LABEL: @test_new_op_bundle(
 ; CHECK-NEXT:    [[A:%.*]] = alloca i32, align 4
-; CHECK-NEXT:    call void @f(ptr nocapture writeonly [[A]]) #[[ATTR1]] [ "unknown"(ptr [[A]]) ]
+; CHECK-NEXT:    call void @f(ptr writeonly captures(none) [[A]]) #[[ATTR1]] [ "unknown"(ptr [[A]]) ]
 ; CHECK-NEXT:    ret void
 ;
   %a = alloca i32, align 4
@@ -203,7 +203,7 @@ define i8 @test_neg_unrelated_capture_used_via_return() {
 ; CHECK-LABEL: @test_neg_unrelated_capture_used_via_return(
 ; CHECK-NEXT:    [[A:%.*]] = alloca i32, align 4
 ; CHECK-NEXT:    [[A2:%.*]] = alloca i32, align 4
-; CHECK-NEXT:    [[CAPTURE:%.*]] = call ptr @f3(ptr nocapture writeonly [[A]], ptr readonly [[A2]]) #[[ATTR1]]
+; CHECK-NEXT:    [[CAPTURE:%.*]] = call ptr @f3(ptr writeonly captures(none) [[A]], ptr readonly [[A2]]) #[[ATTR1]]
 ; CHECK-NEXT:    [[V:%.*]] = load i8, ptr [[CAPTURE]], align 1
 ; CHECK-NEXT:    ret i8 [[V]]
 ;
@@ -246,7 +246,7 @@ define i32 @test_dse_overwrite() {
 define i32 @test_neg_dse_partial_overwrite() {
 ; CHECK-LABEL: @test_neg_dse_partial_overwrite(
 ; CHECK-NEXT:    [[A:%.*]] = alloca i32, align 4
-; CHECK-NEXT:    call void @f(ptr nocapture writeonly [[A]]) #[[ATTR1]]
+; CHECK-NEXT:    call void @f(ptr writeonly captures(none) [[A]]) #[[ATTR1]]
 ; CHECK-NEXT:    store i8 0, ptr [[A]], align 1
 ; CHECK-NEXT:    [[V:%.*]] = load i32, ptr [[A]], align 4
 ; CHECK-NEXT:    ret i32 [[V]]
@@ -262,7 +262,7 @@ define i32 @test_neg_dse_partial_overwrite() {
 ; full overwrite reasoning
 define i32 @test_neg_dse_unsized(ptr %a) {
 ; CHECK-LABEL: @test_neg_dse_unsized(
-; CHECK-NEXT:    call void @f(ptr nocapture writeonly [[A:%.*]]) #[[ATTR1]]
+; CHECK-NEXT:    call void @f(ptr writeonly captures(none) [[A:%.*]]) #[[ATTR1]]
 ; CHECK-NEXT:    store i32 0, ptr [[A]], align 4
 ; CHECK-NEXT:    [[V:%.*]] = load i32, ptr [[A]], align 4
 ; CHECK-NEXT:    ret i32 [[V]]
diff --git a/llvm/test/Transforms/ExpandMemCmp/AArch64/memcmp.ll b/llvm/test/Transforms/ExpandMemCmp/AArch64/memcmp.ll
index 179b5b0a3dbf5..7df48d878bd0b 100644
--- a/llvm/test/Transforms/ExpandMemCmp/AArch64/memcmp.ll
+++ b/llvm/test/Transforms/ExpandMemCmp/AArch64/memcmp.ll
@@ -6,7 +6,7 @@ declare i32 @memcmp(ptr nocapture, ptr nocapture, i64)
 
 define i32 @cmp2(ptr nocapture readonly %x, ptr nocapture readonly %y)  {
 ; CHECK-LABEL: define i32 @cmp2(
-; CHECK-SAME: ptr nocapture readonly [[X:%.*]], ptr nocapture readonly [[Y:%.*]]) {
+; CHECK-SAME: ptr readonly captures(none) [[X:%.*]], ptr readonly captures(none) [[Y:%.*]]) {
 ; CHECK-NEXT:    [[TMP1:%.*]] = load i16, ptr [[X]], align 1
 ; CHECK-NEXT:    [[TMP2:%.*]] = load i16, ptr [[Y]], align 1
 ; CHECK-NEXT:    [[TMP3:%.*]] = call i16 @llvm.bswap.i16(i16 [[TMP1]])
@@ -22,7 +22,7 @@ define i32 @cmp2(ptr nocapture readonly %x, ptr nocapture readonly %y)  {
 
 define i32 @cmp2_align2(ptr nocapture readonly align 2 %x, ptr nocapture readonly align 2 %y)  {
 ; CHECK-LABEL: define i32 @cmp2_align2(
-; CHECK-SAME: ptr nocapture readonly align 2 [[X:%.*]], ptr nocapture readonly align 2 [[Y:%.*]]) {
+; CHECK-SAME: ptr readonly align 2 captures(none) [[X:%.*]], ptr readonly align 2 captures(none) [[Y:%.*]]) {
 ; CHECK-NEXT:    [[TMP1:%.*]] = load i16, ptr [[X]], align 2
 ; CHECK-NEXT:    [[TMP2:%.*]] = load i16, ptr [[Y]], align 2
 ; CHECK-NEXT:    [[TMP3:%.*]] = call i16 @llvm.bswap.i16(i16 [[TMP1]])
@@ -38,7 +38,7 @@ define i32 @cmp2_align2(ptr nocapture readonly align 2 %x, ptr nocapture readonl
 
 define i32 @cmp3(ptr nocapture readonly %x, ptr nocapture readonly %y)  {
 ; CHECK-LABEL: define i32 @cmp3(
-; CHECK-SAME: ptr nocapture readonly [[X:%.*]], ptr nocapture readonly [[Y:%.*]]) {
+; CHECK-SAME: ptr readonly captures(none) [[X:%.*]], ptr readonly captures(none) [[Y:%.*]]) {
 ; CHECK-NEXT:    [[TMP1:%.*]] = load i24, ptr [[X]], align 1
 ; CHECK-NEXT:    [[TMP2:%.*]] = load i24, ptr [[Y]], align 1
 ; CHECK-NEXT:    [[TMP3:%.*]] = zext i24 [[TMP1]] to i32
@@ -54,7 +54,7 @@ define i32 @cmp3(ptr nocapture readonly %x, ptr nocapture readonly %y)  {
 
 define i32 @cmp4(ptr nocapture readonly %x, ptr nocapture readonly %y)  {
 ; CHECK-LABEL: define i32 @cmp4(
-; CHECK-SAME: ptr nocapture readonly [[X:%.*]], ptr nocapture readonly [[Y:%.*]]) {
+; CHECK-SAME: ptr readonly captures(none) [[X:%.*]], ptr readonly captures(none) [[Y:%.*]]) {
 ; CHECK-NEXT:    [[TMP1:%.*]] = load i32, ptr [[X]], align 1
 ; CHECK-NEXT:    [[TMP2:%.*]] = load i32, ptr [[Y]], align 1
 ; CHECK-NEXT:    [[TMP3:%.*]] = call i32 @llvm.bswap.i32(i32 [[TMP1]])
@@ -68,7 +68,7 @@ define i32 @cmp4(ptr nocapture readonly %x, ptr nocapture readonly %y)  {
 
 define i32 @cmp5(ptr nocapture readonly %x, ptr nocapture readonly %y)  {
 ; CHECK-LABEL: define i32 @cmp5(
-; CHECK-SAME: ptr nocapture readonly [[X:%.*]], ptr nocapture readonly [[Y:%.*]]) {
+; CHECK-SAME: ptr readonly captures(none) [[X:%.*]], ptr readonly captures(none) [[Y:%.*]]) {
 ; CHECK-NEXT:    [[TMP1:%.*]] = load i40, ptr [[X]], align 1
 ; CHECK-NEXT:    [[TMP2:%.*]] = load i40, ptr [[Y]], align 1
 ; CHECK-NEXT:    [[TMP3:%.*]] = zext i40 [[TMP1]] to i64
@@ -84,7 +84,7 @@ define i32 @cmp5(ptr nocapture readonly %x, ptr nocapture readonly %y)  {
 
 define i32 @cmp6(ptr nocapture readonly %x, ptr nocapture readonly %y)  {
 ; CHECK-LABEL: define i32 @cmp6(
-; CHECK-SAME: ptr nocapture readonly [[X:%.*]], ptr nocapture readonly [[Y:%.*]]) {
+; CHECK-SAME: ptr readonly captures(none) [[X:%.*]], ptr readonly captures(none) [[Y:%.*]]) {
 ; CHECK-NEXT:    [[TMP1:%.*]] = load i48, ptr [[X]], align 1
 ; CHECK-NEXT:    [[TMP2:%.*]] = load i48, ptr [[Y]], align 1
 ; CHECK-NEXT:    [[TMP3:%.*]] = zext i48 [[TMP1]] to i64
@@ -100,7 +100,7 @@ define i32 @cmp6(ptr nocapture readonly %x, ptr nocapture readonly %y)  {
 
 define i32 @cmp7(ptr nocapture readonly %x, ptr nocapture readonly %y)  {
 ; CHECK-LABEL: define i32 @cmp7(
-; CHECK-SAME: ptr nocapture readonly [[X:%.*]], ptr nocapture readonly [[Y:%.*]]) {
+; CHECK-SAME: ptr readonly captures(none) [[X:%.*]], ptr readonly captures(none) [[Y:%.*]]) {
 ; CHECK-NEXT:    br label [[LOADBB:%.*]]
 ; CHECK:       res_block:
 ; CHECK-NEXT:    [[PHI_SRC1:%.*]] = phi i32 [ [[TMP5:%.*]], [[LOADBB]] ], [ [[TMP12:%.*]], [[LOADBB1:%.*]] ]
@@ -134,7 +134,7 @@ define i32 @cmp7(ptr nocapture readonly %x, ptr nocapture readonly %y)  {
 
 define i32 @cmp8(ptr nocapture readonly %x, ptr nocapture readonly %y)  {
 ; CHECK-LABEL: define i32 @cmp8(
-; CHECK-SAME: ptr nocapture readonly [[X:%.*]], ptr nocapture readonly [[Y:%.*]]) {
+; CHECK-SAME: ptr readonly captures(none) [[X:%.*]], ptr readonly captures(none) [[Y:%.*]]) {
 ; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr [[X]], align 1
 ; CHECK-NEXT:    [[TMP2:%.*]] = load i64, ptr [[Y]], align 1
 ; CHECK-NEXT:    [[TMP3:%.*]] = call i64 @llvm.bswap.i64(i64 [[TMP1]])
@@ -148,7 +148,7 @@ define i32 @cmp8(ptr nocapture readonly %x, ptr nocapture readonly %y)  {
 
 define i32 @cmp9(ptr nocapture readonly %x, ptr nocapture readonly %y)  {
 ; CHECK-LABEL: define i32 @cmp9(
-; CHECK-SAME: ptr nocapture readonly [[X:%.*]], ptr nocapture readonly [[Y:%.*]]) {
+; CHECK-SAME: ptr readonly captures(none) [[X:%.*]], ptr readonly captures(none) [[Y:%.*]]) {
 ; CHECK-NEXT:    br label [[LOADBB:%.*]]
 ; CHECK:       res_block:
 ; CHECK-NEXT:    [[TMP1:%.*]] = icmp ult i64 [[TMP5:%.*]], [[TMP6:%.*]]
@@ -180,7 +180,7 @@ define i32 @cmp9(ptr nocapture readonly %x, ptr nocapture readonly %y)  {
 
 define i32 @cmp10(ptr nocapture readonly %x, ptr nocapture readonly %y)  {
 ; CHECK-LABEL: define i32 @cmp10(
-; CHECK-SAME: ptr nocapture readonly [[X:%.*]], ptr nocapture readonly [[Y:%.*]]) {
+; CHECK-SAME: ptr readonly captures(none) [[X:%.*]], ptr readonly captures(none) [[Y:%.*]]) {
 ; CHECK-NEXT:    br label [[LOADBB:%.*]]
 ; CHECK:       res_block:
 ; CHECK-NEXT:    [[PHI_SRC1:%.*]] = phi i64 [ [[TMP5:%.*]], [[LOADBB]] ], [ [[TMP14:%.*]], [[LOADBB1:%.*]] ]
@@ -216,7 +216,7 @@ define i32 @cmp10(ptr nocapture readonly %x, ptr nocapture readonly %y)  {
 
 define i32 @cmp11(ptr nocapture readonly %x, ptr nocapture readonly %y)  {
 ; CHECK-LABEL: define i32 @cmp11(
-; CHECK-SAME: ptr nocapture readonly [[X:%.*]], ptr nocapture readonly [[Y:%.*]]) {
+; CHECK-SAME: ptr readonly captures(none) [[X:%.*]], ptr readonly captures(none) [[Y:%.*]]) {
 ; CHECK-NEXT:    br label [[LOADBB:%.*]]
 ; CHECK:       res_block:
 ; CHECK-NEXT:    [[PHI_SRC1:%.*]] = phi i64 [ [[TMP5:%.*]], [[LOADBB]] ], [ [[TMP12:%.*]], [[LOADBB1:%.*]] ]
@@ -250,7 +250,7 @@ define i32 @cmp11(ptr nocapture readonly %x, ptr nocapture readonly %y)  {
 
 define i32 @cmp12(ptr nocapture readonly %x, ptr nocapture readonly %y)  {
 ; CHECK-LABEL: define i32 @cmp12(
-; CHECK-SAME: ptr nocapture readonly [[X:%.*]], ptr nocapture readonly [[Y:%.*]]) {
+; CHECK-SAME: ptr readonly captures(none) [[X:%.*]], ptr readonly captures(none) [[Y:%.*]]) {
 ; CHECK-NEXT:    br label [[LOADBB:%.*]]
 ; CHECK:       res_block:
 ; CHECK-NEXT:    [[PHI_SRC1:%.*]] = phi i64 [ [[TMP5:%.*]], [[LOADBB]] ], [ [[TMP14:%.*]], [[LOADBB1:%.*]] ]
@@ -286,7 +286,7 @@ define i32 @cmp12(ptr nocapture readonly %x, ptr nocapture readonly %y)  {
 
 define i32 @cmp13(ptr nocapture readonly %x, ptr nocapture readonly %y)  {
 ; CHECK-LABEL: define i32 @cmp13(
-; CHECK-SAME: ptr nocapture readonly [[X:%.*]], ptr nocapture readonly [[Y:%.*]]) {
+; CHECK-SAME: ptr readonly captures(none) [[X:%.*]], ptr readonly captures(none) [[Y:%.*]]) {
 ; CHECK-NEXT:    br label [[LOADBB:%.*]]
 ; CHECK:       res_block:
 ; CHECK-NEXT:    [[PHI_SRC1:%.*]] = phi i64 [ [[TMP5:%.*]], [[LOADBB]] ], [ [[TMP12:%.*]], [[LOADBB1:%.*]] ]
@@ -320,7 +320,7 @@ define i32 @cmp13(ptr nocapture readonly %x, ptr nocapture readonly %y)  {
 
 define i32 @cmp14(ptr nocapture readonly %x, ptr nocapture readonly %y)  {
 ; CHECK-LABEL: define i32 @cmp14(
-; CHECK-SAME: ptr nocapture readonly [[X:%.*]], ptr nocapture readonly [[Y:%.*]]) {
+; CHECK-SAME: ptr readonly captures(none) [[X:%.*]], ptr readonly captures(none) [[Y:%.*]]) {
 ; CHECK-NEXT:    br label [[LOADBB:%.*]]
 ; CHECK:       res_block:
 ; CHECK-NEXT:    [[PHI_SRC1:%.*]] = phi i64 [ [[TMP5:%.*]], [[LOADBB]] ], [ [[TMP12:%.*]], [[LOADBB1:%.*]] ]
@@ -354,7 +354,7 @@ define i32 @cmp14(ptr nocapture readonly %x, ptr nocapture readonly %y)  {
 
 define i32 @cmp15(ptr nocapture readonly %x, ptr nocapture readonly %y)  {
 ; CHECK-LABEL: define i32 @cmp15(
-; CHECK-SAME: ptr nocapture readonly [[X:%.*]], ptr nocapture readonly [[Y:%.*]]) {
+; CHECK-SAME: ptr readonly captures(none) [[X:%.*]], ptr readonly captures(none) [[Y:%.*]]) {
 ; CHECK-NEXT:    br label [[LOADBB:%.*]]
 ; CHECK:       res_block:
 ; CHECK-NEXT:    [[PHI_SRC1:%.*]] = phi i64 [ [[TMP5:%.*]], [[LOADBB]] ], [ [[TMP12:%.*]], [[LOADBB1:%.*]] ]
@@ -388,7 +388,7 @@ define i32 @cmp15(ptr nocapture readonly %x, ptr nocapture readonly %y)  {
 
 define i32 @cmp16(ptr nocapture readonly %x, ptr nocapture readonly %y)  {
 ; CHECK-LABEL: define i32 @cmp16(
-; CHECK-SAME: ptr nocapture readonly [[X:%.*]], ptr nocapture readonly [[Y:%.*]]) {
+; CHECK-SAME: ptr readonly captures(none) [[X:%.*]], ptr readonly captures(none) [[Y:%.*]]) {
 ; CHECK-NEXT:    br label [[LOADBB:%.*]]
 ; CHECK:       res_block:
 ; CHECK-NEXT:    [[PHI_SRC1:%.*]] = phi i64 [ [[TMP5:%.*]], [[LOADBB]] ], [ [[TMP12:%.*]], [[LOADBB1:%.*]] ]
@@ -422,7 +422,7 @@ define i32 @cmp16(ptr nocapture readonly %x, ptr nocapture readonly %y)  {
 
 define i32 @cmp_eq2(ptr nocapture readonly %x, ptr nocapture readonly %y)  {
 ; CHECK-LABEL: define i32 @cmp_eq2(
-; CHECK-SAME: ptr nocapture readonly [[X:%.*]], ptr nocapture readonly [[Y:%.*]]) {
+; CHECK-SAME: ptr readonly captures(none) [[X:%.*]], ptr readonly captures(none) [[Y:%.*]]) {
 ; CHECK-NEXT:    [[TMP1:%.*]] = load i16, ptr [[X]], align 1
 ; CHECK-NEXT:    [[TMP2:%.*]] = load i16, ptr [[Y]], align 1
 ; CHECK-NEXT:    [[TMP3:%.*]] = icmp ne i16 [[TMP1]], [[TMP2]]
@@ -439,7 +439,7 @@ define i32 @cmp_eq2(ptr nocapture readonly %x, ptr nocapture readonly %y)  {
 
 define i32 @cmp_eq3(ptr nocapture readonly %x, ptr nocapture readonly %y)  {
 ; CHECK-LABEL: define i32 @cmp_eq3(
-; CHECK-SAME: ptr nocapture readonly [[X:%.*]], ptr nocapture readonly [[Y:%.*]]) {
+; CHECK-SAME: ptr readonly captures(none) [[X:%.*]], ptr readonly captures(none) [[Y:%.*]]) {
 ; CHECK-NEXT:    br label [[LOADBB:%.*]]
 ; CHECK:       res_block:
 ; CHECK-NEXT:    br label [[ENDBLOCK:%.*]]
@@ -469,7 +469,7 @@ define i32 @cmp_eq3(ptr nocapture readonly %x, ptr nocapture readonly %y)  {
 
 define i32 @cmp_eq4(ptr nocapture readonly %x, ptr nocapture readonly %y)  {
 ; CHECK-LABEL: define i32 @cmp_eq4(
-; CHECK-SAME: ptr nocapture readonly [[X:%.*]], ptr nocapture readonly [[Y:%.*]]) {
+; CHECK-SAME: ptr readonly captures(none) [[X:%.*]], ptr readonly captures(none) [[Y:%.*]]) {
 ; CHECK-NEXT:    [[TMP1:%.*]] = load i32, ptr [[X]], align 1
 ; CHECK-NEXT:    [[TMP2:%.*]] = load i32, ptr [[Y]], align 1
 ; CHECK-NEXT:    [[TMP3:%.*]] = icmp ne i32 [[TMP1]], [[TMP2]]
@@ -486,7 +486,7 @@ define i32 @cmp_eq4(ptr nocapture readonly %x, ptr nocapture readonly %y)  {
 
 define i32 @cmp_eq5(ptr nocapture readonly %x, ptr nocapture readonly %y)  {
 ; CHECK-LABEL: define i32 @cmp_eq5(
-; CHECK-SAME: ptr nocapture readonly [[X:%.*]], ptr nocapture readonly [[Y:%.*]]) {
+; CHECK-SAME: ptr readonly captures(none) [[X:%.*]], ptr readonly captures(none) [[Y:%.*]]) {
 ; CHECK-NEXT:    br label [[LOADBB:%.*]]
 ; CHECK:       res_block:
 ; CHECK-NEXT:    br label [[ENDBLOCK:%.*]]
@@ -516,7 +516,7 @@ define i32 @cmp_eq5(ptr nocapture readonly %x, ptr nocapture readonly %y)  {
 
 define i32 @cmp_eq6(ptr nocapture readonly %x, ptr nocapture readonly %y)  {
 ; CHECK-LABEL: define i32 @cmp_eq6(
-; CHECK-SAME: ptr nocapture readonly [[X:%.*]], ptr nocapture readonly [[Y:%.*]]) {
+; CHECK-SAME: ptr readonly captures(none) [[X:%.*]], ptr readonly captures(none) [[Y:%.*]]) {
 ; CHECK-NEXT:    br label [[LOADBB:%.*]]
 ; CHECK:       res_block:
 ; CHECK-NEXT:    br label [[ENDBLOCK:%.*]]
@@ -546,7 +546,7 @@ define i32 @cmp_eq6(ptr nocapture readonly %x, ptr nocapture readonly %y)  {
 
 define i32 @cmp_eq6_align4(ptr nocapture readonly align 4 %x, ptr nocapture readonly align 4 %y)  {
 ; CHECK-LABEL: define i32 @cmp_eq6_align4(
-; CHECK-SAME: ptr nocapture readonly align 4 [[X:%.*]], ptr nocapture readonly align 4 [[Y:%.*]]) {
+; CHECK-SAME: ptr readonly align 4 captures(none) [[X:%.*]], ptr readonly align 4 captures(none) [[Y:%.*]]) {
 ; CHECK-NEXT:    br label [[LOADBB:%.*]]
 ; CHECK:       res_block:
 ; CHECK-NEXT:    br label [[ENDBLOCK:%.*]]
@@ -576,7 +576,7 @@ define i32 @cmp_eq6_align4(ptr nocapture readonly align 4 %x, ptr nocapture read
 
 define i32 @cmp_eq7(ptr nocapture readonly %x, ptr nocapture readonly %y)  {
 ; CHECK-LABEL: define i32 @cmp_eq7(
-; CHECK-SAME: ptr nocapture readonly [[X:%.*]], ptr nocapture readonly [[Y:%.*]]) {
+; CHECK-SAME: ptr readonly captures(none) [[X:%.*]], ptr readonly captures(none) [[Y:%.*]]) {
 ; CHECK-NEXT:    br label [[LOADBB:%.*]]
 ; CHECK:       res_block:
 ; CHECK-NEXT:    br label [[ENDBLOCK:%.*]]
@@ -606,7 +606,7 @@ define i32 @cmp_eq7(ptr nocapture readonly %x, ptr nocapture readonly %y)  {
 
 define i32 @cmp_eq8(ptr nocapture readonly %x, ptr nocapture readonly %y)  {
 ; CHECK-LABEL: define i32 @cmp_eq8(
-; CHECK-SAME: ptr nocapture readonly [[X:%.*]], ptr nocapture readonly [[Y:%.*]]) {
+; CHECK-SAME: ptr readonly captures(none) [[X:%.*]], ptr readonly captures(none) [[Y:%.*]]) {
 ; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr [[X]], align 1
 ; CHECK-NEXT:    [[TMP2:%.*]] = load i64, ptr [[Y]], align 1
 ; CHECK-NEXT:    [[TMP3:%.*]] = icmp ne i64 [[TMP1]], [[TMP2]]
@@ -623,7 +623,7 @@ define i32 @cmp_eq8(ptr nocapture readonly %x, ptr nocapture readonly %y)  {
 
 define i32 @cmp_eq9(ptr nocapture readonly %x, ptr nocapture readonly %y)  {
 ; CHECK-LABEL: define i32 @cmp_eq9(
-; CHECK-SAME: ptr nocapture readonly [[X:%.*]], ptr nocapture readonly [[Y:%.*]]) {
+; CHECK-SAME: ptr readonly captures(none) [[X:%.*]], ptr readonly captures(none) [[Y:%.*]]) {
 ; CHECK-NEXT:    br label [[LOADBB:%.*]]
 ; CHECK:       res_block:
 ; CHECK-NEXT:    br label [[ENDBLOCK:%.*]]
@@ -653,7 +653,7 @@ define i32 @cmp_eq9(ptr nocapture readonly %x, ptr nocapture readonly %y)  {
 
 define i32 @cmp_eq10(ptr nocapture readonly %x, ptr nocapture readonly %y)  {
 ; CHECK-LABEL: define i32 @cmp_eq10(
-; CHECK-SAME: ptr nocapture readonly [[X:%.*]], ptr nocapture readonly [[Y:%.*]]) {
+; CHECK-SAME: ptr readonly captures(none) [[X:%.*]], ptr readonly captures(none) [[Y:%.*]]) {
 ; CHECK-NEXT:    br label [[LOADBB:%.*]]
 ; CHECK:       res_block:
 ; CHECK-NEXT:    br label [[ENDBLOCK:%.*]]
@@ -683,7 +683,7 @@ define i32 @cmp_eq10(ptr nocapture readonly %x, ptr nocapture readonly %y)  {
 
 define i32 @cmp_eq11(ptr nocapture readonly %x, ptr nocapture readonly %y)  {
 ; CHECK-LABEL: define i32 @cmp_eq11(
-; CHECK-SAME: ptr nocapture readonly [[X:%.*]], ptr nocapture readonly [[Y:%.*]]) {
+; CHECK-SAME: ptr readonly captures(none) [[X:%.*]], ptr readonly captures(none) [[Y:%.*]]) {
 ; CHECK-NEXT:    br label [[LOADBB:%.*]]
 ; CHECK:       res_block:
 ; CHECK-NEXT:    br label [[ENDBLOCK:%.*]]
@@ -713,7 +713,7 @@ define i32 @cmp_eq11(ptr nocapture readonly %x, ptr nocapture readonly %y)  {
 
 define i32 @cmp_eq12(ptr nocapture readonly %x, ptr nocapture readonly %y)  {
 ; CHECK-LABEL: define i32 @cmp_eq12(
-; CHECK-SAME: ptr nocapture readonly [[X:%.*]], ptr nocapture readonly [[Y:%.*]]) {
+; CHECK-SAME: ptr readonly captures(none) [[X:%.*]], ptr readonly captures(none) [[Y:%.*]]) {
 ; CHECK-NEXT:    br label [[LOADBB:%.*]]
 ; CHECK:       res_block:
 ; CHECK-NEXT:    br label [[ENDBLOCK:%.*]]
@@ -743,7 +743,7 @@ define i32 @cmp_eq12(ptr nocapture readonly %x, ptr nocapture readonly %y)  {
 
 define i32 @cmp_eq13(ptr nocapture readonly %x, ptr nocapture readonly %y)  {
 ; CHECK-LABEL: define i32 @cmp_eq13(
-; CHECK-SAME: ptr nocapture readonly [[X:%.*]], ptr nocapture readonly [[Y:%.*]]) {
+; CHECK-SAME: ptr readonly captures(none) [[X:%.*]], ptr readonly captures(none) [[Y:%.*]]) {
 ; CHECK-NEXT:    br label [[LOADBB:%.*]]
 ; CHECK:       res_block:
 ; CHECK-NEXT:    br label [[ENDBLOCK:%.*]]
@@ -773,7 +773,7 @@ define i32 @cmp_eq13(ptr nocapture readonly %x, ptr nocapture readonly %y)  {
 
 define i32 @cmp_eq14(ptr nocapture readonly %x, ptr nocapture readonly %y)  {
 ; CHECK-LABEL: define i32 @cmp_eq14(
-; CHECK-SAME: ptr nocapture readonly [[X:%.*]], ptr nocapture readonly [[Y:%.*]]) {
+; CHECK-SAME: ptr readonly captures(none) [[X:%.*]], ptr readonly captures(none) [[Y:%.*]]) {
 ; CHECK-NEXT:    br label [[LOADBB:%.*]]
 ; CHECK:       res_block:
 ; CHECK-NEXT:    br label [[ENDBLOCK:%.*]]
@@ -803,7 +803,7 @@ define i32 @cmp_eq14(ptr nocapture readonly %x, ptr nocapture readonly %y)  {
 
 define i32 @cmp_eq15(ptr nocapture readonly %x, ptr nocapture readonly %y)  {
 ; CHECK-LABEL: define i32 @cmp_eq15(
-; CHECK-SAME: ptr nocapture readonly [[X:%.*]], ptr nocapture readonly [[Y:%.*]]) {
+; CHECK-SAME: ptr readonly captures(none) [[X:%.*]], ptr readonly captures(none) [[Y:%.*]]) {
 ; CHECK-NEXT:    br label [[LOADBB:%.*]]
 ; CHECK:       res_block:
 ; CHECK-NEXT:    br label [[ENDBLOCK:%.*]]
@@ -833,7 +833,7 @@ define i32 @cmp_eq15(ptr nocapture readonly %x, ptr nocapture readonly %y)  {
 
 define i32 @cmp_eq16(ptr nocapture readonly %x, ptr nocapture readonly %y)  {
 ; CHECK-LABEL: define i32 @cmp_eq16(
-; CHECK-SAME: ptr nocapture readonly [[X:%.*]], ptr nocapture readonly [[Y:%.*]]) {
+; CHECK-SAME: ptr readonly captures(none) [[X:%.*]], ptr readonly captures(none) [[Y:%.*]]) {
 ; CHECK-NEXT:    br label [[LOADBB:%.*]]
 ; CHECK:       res_block:
 ; CHECK-NEXT:    br label [[ENDBLOCK:%.*]]
diff --git a/llvm/test/Transforms/FunctionAttrs/2009-01-02-LocalStores.ll b/llvm/test/Transforms/FunctionAttrs/2009-01-02-LocalStores.ll
index 8d63045328fd7..f706184f9727e 100644
--- a/llvm/test/Transforms/FunctionAttrs/2009-01-02-LocalStores.ll
+++ b/llvm/test/Transforms/FunctionAttrs/2009-01-02-LocalStores.ll
@@ -1,6 +1,6 @@
 ; RUN: opt < %s -passes=function-attrs -S | FileCheck %s
 
-; CHECK: define ptr @a(ptr nocapture readonly %p)
+; CHECK: define ptr @a(ptr readonly captures(none) %p)
 define ptr @a(ptr %p) {
 	%tmp = load ptr, ptr %p
 	ret ptr %tmp
diff --git a/llvm/test/Transforms/FunctionAttrs/arg_returned.ll b/llvm/test/Transforms/FunctionAttrs/arg_returned.ll
index 1f8607eca6e53..13954694eefe0 100644
--- a/llvm/test/Transforms/FunctionAttrs/arg_returned.ll
+++ b/llvm/test/Transforms/FunctionAttrs/arg_returned.ll
@@ -146,7 +146,7 @@ return:                                           ; preds = %cond.end, %if.then3
 ; TEST SCC test returning a pointer value argument
 ;
 ; FNATTR: define ptr @ptr_sink_r0(ptr readnone returned %r)
-; FNATTR: define ptr @ptr_scc_r1(ptr %a, ptr readnone %r, ptr nocapture readnone %b)
+; FNATTR: define ptr @ptr_scc_r1(ptr %a, ptr readnone %r, ptr readnone captures(none) %b)
 ; FNATTR: define ptr @ptr_scc_r2(ptr readnone %a, ptr readnone %b, ptr readnone %r)
 ;
 ;
@@ -248,7 +248,7 @@ entry:
 ;   return *a ? undef : rt1(a);
 ; }
 ;
-; FNATTR:  define noalias ptr @rt1(ptr nocapture readonly %a)
+; FNATTR:  define noalias ptr @rt1(ptr readonly captures(none) %a)
 define ptr @rt1(ptr %a) #0 {
 entry:
   %v = load i32, ptr %a, align 4
diff --git a/llvm/test/Transforms/FunctionAttrs/argmemonly.ll b/llvm/test/Transforms/FunctionAttrs/argmemonly.ll
index 5bbe6fa7c27c2..42e0e94c1cee3 100644
--- a/llvm/test/Transforms/FunctionAttrs/argmemonly.ll
+++ b/llvm/test/Transforms/FunctionAttrs/argmemonly.ll
@@ -18,14 +18,14 @@ entry:
 define i32 @test_only_read_arg(ptr %ptr) {
 ; FNATTRS: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: read)
 ; FNATTRS-LABEL: define i32 @test_only_read_arg
-; FNATTRS-SAME: (ptr nocapture readonly [[PTR:%.*]]) #[[ATTR1:[0-9]+]] {
+; FNATTRS-SAME: (ptr readonly captures(none) [[PTR:%.*]]) #[[ATTR1:[0-9]+]] {
 ; FNATTRS-NEXT:  entry:
 ; FNATTRS-NEXT:    [[L:%.*]] = load i32, ptr [[PTR]], align 4
 ; FNATTRS-NEXT:    ret i32 [[L]]
 ;
 ; ATTRIBUTOR: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: read)
 ; ATTRIBUTOR-LABEL: define i32 @test_only_read_arg
-; ATTRIBUTOR-SAME: (ptr nocapture nofree nonnull readonly [[PTR:%.*]]) #[[ATTR1:[0-9]+]] {
+; ATTRIBUTOR-SAME: (ptr nofree nonnull readonly captures(none) [[PTR:%.*]]) #[[ATTR1:[0-9]+]] {
 ; ATTRIBUTOR-NEXT:  entry:
 ; ATTRIBUTOR-NEXT:    [[L:%.*]] = load i32, ptr [[PTR]], align 4
 ; ATTRIBUTOR-NEXT:    ret i32 [[L]]
@@ -38,14 +38,14 @@ entry:
 define i32 @test_only_read_arg_already_has_argmemonly(ptr %ptr) argmemonly {
 ; FNATTRS: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: read)
 ; FNATTRS-LABEL: define i32 @test_only_read_arg_already_has_argmemonly
-; FNATTRS-SAME: (ptr nocapture readonly [[PTR:%.*]]) #[[ATTR1]] {
+; FNATTRS-SAME: (ptr readonly captures(none) [[PTR:%.*]]) #[[ATTR1]] {
 ; FNATTRS-NEXT:  entry:
 ; FNATTRS-NEXT:    [[L:%.*]] = load i32, ptr [[PTR]], align 4
 ; FNATTRS-NEXT:    ret i32 [[L]]
 ;
 ; ATTRIBUTOR: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: read)
 ; ATTRIBUTOR-LABEL: define i32 @test_only_read_arg_already_has_argmemonly
-; ATTRIBUTOR-SAME: (ptr nocapture nofree nonnull readonly [[PTR:%.*]]) #[[ATTR1]] {
+; ATTRIBUTOR-SAME: (ptr nofree nonnull readonly captures(none) [[PTR:%.*]]) #[[ATTR1]] {
 ; ATTRIBUTOR-NEXT:  entry:
 ; ATTRIBUTOR-NEXT:    [[L:%.*]] = load i32, ptr [[PTR]], align 4
 ; ATTRIBUTOR-NEXT:    ret i32 [[L]]
@@ -78,7 +78,7 @@ entry:
 define i32 @test_read_loaded_ptr(ptr %ptr) {
 ; FNATTRS: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(read, inaccessiblemem: none)
 ; FNATTRS-LABEL: define i32 @test_read_loaded_ptr
-; FNATTRS-SAME: (ptr nocapture readonly [[PTR:%.*]]) #[[ATTR3:[0-9]+]] {
+; FNATTRS-SAME: (ptr readonly captures(none) [[PTR:%.*]]) #[[ATTR3:[0-9]+]] {
 ; FNATTRS-NEXT:  entry:
 ; FNATTRS-NEXT:    [[L:%.*]] = load ptr, ptr [[PTR]], align 8
 ; FNATTRS-NEXT:    [[L_2:%.*]] = load i32, ptr [[L]], align 4
@@ -86,7 +86,7 @@ define i32 @test_read_loaded_ptr(ptr %ptr) {
 ;
 ; ATTRIBUTOR: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(read)
 ; ATTRIBUTOR-LABEL: define i32 @test_read_loaded_ptr
-; ATTRIBUTOR-SAME: (ptr nocapture nofree nonnull readonly [[PTR:%.*]]) #[[ATTR2]] {
+; ATTRIBUTOR-SAME: (ptr nofree nonnull readonly captures(none) [[PTR:%.*]]) #[[ATTR2]] {
 ; ATTRIBUTOR-NEXT:  entry:
 ; ATTRIBUTOR-NEXT:    [[L:%.*]] = load ptr, ptr [[PTR]], align 8
 ; ATTRIBUTOR-NEXT:    [[L_2:%.*]] = load i32, ptr [[L]], align 4
@@ -101,14 +101,14 @@ entry:
 define void @test_only_write_arg(ptr %ptr) {
 ; FNATTRS: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: write)
 ; FNATTRS-LABEL: define void @test_only_write_arg
-; FNATTRS-SAME: (ptr nocapture writeonly initializes((0, 4)) [[PTR:%.*]]) #[[ATTR4:[0-9]+]] {
+; FNATTRS-SAME: (ptr writeonly captures(none) initializes((0, 4)) [[PTR:%.*]]) #[[ATTR4:[0-9]+]] {
 ; FNATTRS-NEXT:  entry:
 ; FNATTRS-NEXT:    store i32 0, ptr [[PTR]], align 4
 ; FNATTRS-NEXT:    ret void
 ;
 ; ATTRIBUTOR: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: write)
 ; ATTRIBUTOR-LABEL: define void @test_only_write_arg
-; ATTRIBUTOR-SAME: (ptr nocapture nofree nonnull writeonly [[PTR:%.*]]) #[[ATTR3:[0-9]+]] {
+; ATTRIBUTOR-SAME: (ptr nofree nonnull writeonly captures(none) [[PTR:%.*]]) #[[ATTR3:[0-9]+]] {
 ; ATTRIBUTOR-NEXT:  entry:
 ; ATTRIBUTOR-NEXT:    store i32 0, ptr [[PTR]], align 4
 ; ATTRIBUTOR-NEXT:    ret void
@@ -156,7 +156,7 @@ declare i32 @fn_readnone() readnone
 define void @test_call_readnone(ptr %ptr) {
 ; FNATTRS: Function Attrs: memory(argmem: write)
 ; FNATTRS-LABEL: define void @test_call_readnone
-; FNATTRS-SAME: (ptr nocapture writeonly initializes((0, 4)) [[PTR:%.*]]) #[[ATTR7:[0-9]+]] {
+; FNATTRS-SAME: (ptr writeonly captures(none) initializes((0, 4)) [[PTR:%.*]]) #[[ATTR7:[0-9]+]] {
 ; FNATTRS-NEXT:  entry:
 ; FNATTRS-NEXT:    [[C:%.*]] = call i32 @fn_readnone()
 ; FNATTRS-NEXT:    store i32 [[C]], ptr [[PTR]], align 4
@@ -164,7 +164,7 @@ define void @test_call_readnone(ptr %ptr) {
 ;
 ; ATTRIBUTOR: Function Attrs: nosync memory(argmem: write)
 ; ATTRIBUTOR-LABEL: define void @test_call_readnone
-; ATTRIBUTOR-SAME: (ptr nocapture nofree writeonly [[PTR:%.*]]) #[[ATTR6:[0-9]+]] {
+; ATTRIBUTOR-SAME: (ptr nofree writeonly captures(none) [[PTR:%.*]]) #[[ATTR6:[0-9]+]] {
 ; ATTRIBUTOR-NEXT:  entry:
 ; ATTRIBUTOR-NEXT:    [[C:%.*]] = call i32 @fn_readnone() #[[ATTR18:[0-9]+]]
 ; ATTRIBUTOR-NEXT:    store i32 [[C]], ptr [[PTR]], align 4
@@ -201,16 +201,16 @@ entry:
 define i32 @test_call_fn_where_argmemonly_can_be_inferred(ptr %ptr) {
 ; FNATTRS: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: read)
 ; FNATTRS-LABEL: define i32 @test_call_fn_where_argmemonly_can_be_inferred
-; FNATTRS-SAME: (ptr nocapture readonly [[PTR:%.*]]) #[[ATTR1]] {
+; FNATTRS-SAME: (ptr readonly captures(none) [[PTR:%.*]]) #[[ATTR1]] {
 ; FNATTRS-NEXT:  entry:
 ; FNATTRS-NEXT:    [[C:%.*]] = call i32 @test_only_read_arg(ptr [[PTR]])
 ; FNATTRS-NEXT:    ret i32 [[C]]
 ;
 ; ATTRIBUTOR: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: read)
 ; ATTRIBUTOR-LABEL: define i32 @test_call_fn_where_argmemonly_can_be_inferred
-; ATTRIBUTOR-SAME: (ptr nocapture nofree readonly [[PTR:%.*]]) #[[ATTR1]] {
+; ATTRIBUTOR-SAME: (ptr nofree readonly captures(none) [[PTR:%.*]]) #[[ATTR1]] {
 ; ATTRIBUTOR-NEXT:  entry:
-; ATTRIBUTOR-NEXT:    [[C:%.*]] = call i32 @test_only_read_arg(ptr nocapture nofree readonly [[PTR]]) #[[ATTR19:[0-9]+]]
+; ATTRIBUTOR-NEXT:    [[C:%.*]] = call i32 @test_only_read_arg(ptr nofree readonly captures(none) [[PTR]]) #[[ATTR19:[0-9]+]]
 ; ATTRIBUTOR-NEXT:    ret i32 [[C]]
 ;
 entry:
@@ -221,16 +221,16 @@ entry:
 define void @test_memcpy_argonly(ptr %dst, ptr %src) {
 ; FNATTRS: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: readwrite)
 ; FNATTRS-LABEL: define void @test_memcpy_argonly
-; FNATTRS-SAME: (ptr nocapture writeonly initializes((0, 32)) [[DST:%.*]], ptr nocapture readonly [[SRC:%.*]]) #[[ATTR9:[0-9]+]] {
+; FNATTRS-SAME: (ptr writeonly captures(none) initializes((0, 32)) [[DST:%.*]], ptr readonly captures(none) [[SRC:%.*]]) #[[ATTR9:[0-9]+]] {
 ; FNATTRS-NEXT:  entry:
 ; FNATTRS-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr [[DST]], ptr [[SRC]], i64 32, i1 false)
 ; FNATTRS-NEXT:    ret void
 ;
 ; ATTRIBUTOR: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: readwrite)
 ; ATTRIBUTOR-LABEL: define void @test_memcpy_argonly
-; ATTRIBUTOR-SAME: (ptr nocapture nofree writeonly [[DST:%.*]], ptr nocapture nofree readonly [[SRC:%.*]]) #[[ATTR8:[0-9]+]] {
+; ATTRIBUTOR-SAME: (ptr nofree writeonly captures(none) [[DST:%.*]], ptr nofree readonly captures(none) [[SRC:%.*]]) #[[ATTR8:[0-9]+]] {
 ; ATTRIBUTOR-NEXT:  entry:
-; ATTRIBUTOR-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr nocapture writeonly [[DST]], ptr nocapture readonly [[SRC]], i64 32, i1 false) #[[ATTR20:[0-9]+]]
+; ATTRIBUTOR-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr writeonly captures(none) [[DST]], ptr readonly captures(none) [[SRC]], i64 32, i1 false) #[[ATTR20:[0-9]+]]
 ; ATTRIBUTOR-NEXT:    ret void
 ;
 entry:
@@ -245,16 +245,16 @@ declare void @llvm.memcpy.p0.p0.i64(ptr, ptr, i64, i1)
 define void @test_memcpy_src_global(ptr %dst) {
 ; FNATTRS: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(readwrite, inaccessiblemem: none)
 ; FNATTRS-LABEL: define void @test_memcpy_src_global
-; FNATTRS-SAME: (ptr nocapture writeonly initializes((0, 32)) [[DST:%.*]]) #[[ATTR11:[0-9]+]] {
+; FNATTRS-SAME: (ptr writeonly captures(none) initializes((0, 32)) [[DST:%.*]]) #[[ATTR11:[0-9]+]] {
 ; FNATTRS-NEXT:  entry:
 ; FNATTRS-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr [[DST]], ptr @arr, i64 32, i1 false)
 ; FNATTRS-NEXT:    ret void
 ;
 ; ATTRIBUTOR: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn
 ; ATTRIBUTOR-LABEL: define void @test_memcpy_src_global
-; ATTRIBUTOR-SAME: (ptr nocapture nofree writeonly [[DST:%.*]]) #[[ATTR10:[0-9]+]] {
+; ATTRIBUTOR-SAME: (ptr nofree writeonly captures(none) [[DST:%.*]]) #[[ATTR10:[0-9]+]] {
 ; ATTRIBUTOR-NEXT:  entry:
-; ATTRIBUTOR-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr nocapture writeonly [[DST]], ptr readonly @arr, i64 32, i1 false) #[[ATTR20]]
+; ATTRIBUTOR-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr writeonly captures(none) [[DST]], ptr readonly @arr, i64 32, i1 false) #[[ATTR20]]
 ; ATTRIBUTOR-NEXT:    ret void
 ;
 entry:
@@ -265,16 +265,16 @@ entry:
 define void @test_memcpy_dst_global(ptr %src) {
 ; FNATTRS: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(readwrite, inaccessiblemem: none)
 ; FNATTRS-LABEL: define void @test_memcpy_dst_global
-; FNATTRS-SAME: (ptr nocapture readonly [[SRC:%.*]]) #[[ATTR11]] {
+; FNATTRS-SAME: (ptr readonly captures(none) [[SRC:%.*]]) #[[ATTR11]] {
 ; FNATTRS-NEXT:  entry:
 ; FNATTRS-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr @arr, ptr [[SRC]], i64 32, i1 false)
 ; FNATTRS-NEXT:    ret void
 ;
 ; ATTRIBUTOR: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn
 ; ATTRIBUTOR-LABEL: define void @test_memcpy_dst_global
-; ATTRIBUTOR-SAME: (ptr nocapture nofree readonly [[SRC:%.*]]) #[[ATTR10]] {
+; ATTRIBUTOR-SAME: (ptr nofree readonly captures(none) [[SRC:%.*]]) #[[ATTR10]] {
 ; ATTRIBUTOR-NEXT:  entry:
-; ATTRIBUTOR-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr writeonly @arr, ptr nocapture readonly [[SRC]], i64 32, i1 false) #[[ATTR20]]
+; ATTRIBUTOR-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr writeonly @arr, ptr readonly captures(none) [[SRC]], i64 32, i1 false) #[[ATTR20]]
 ; ATTRIBUTOR-NEXT:    ret void
 ;
 entry:
@@ -285,7 +285,7 @@ entry:
 define i32 @test_read_arg_access_alloca(ptr %ptr) {
 ; FNATTRS: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: read)
 ; FNATTRS-LABEL: define i32 @test_read_arg_access_alloca
-; FNATTRS-SAME: (ptr nocapture readonly [[PTR:%.*]]) #[[ATTR1]] {
+; FNATTRS-SAME: (ptr readonly captures(none) [[PTR:%.*]]) #[[ATTR1]] {
 ; FNATTRS-NEXT:  entry:
 ; FNATTRS-NEXT:    [[A:%.*]] = alloca i32, align 4
 ; FNATTRS-NEXT:    [[L:%.*]] = load i32, ptr [[PTR]], align 4
@@ -295,7 +295,7 @@ define i32 @test_read_arg_access_alloca(ptr %ptr) {
 ;
 ; ATTRIBUTOR: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: readwrite)
 ; ATTRIBUTOR-LABEL: define i32 @test_read_arg_access_alloca
-; ATTRIBUTOR-SAME: (ptr nocapture nofree nonnull readonly [[PTR:%.*]]) #[[ATTR8]] {
+; ATTRIBUTOR-SAME: (ptr nofree nonnull readonly captures(none) [[PTR:%.*]]) #[[ATTR8]] {
 ; ATTRIBUTOR-NEXT:  entry:
 ; ATTRIBUTOR-NEXT:    [[A:%.*]] = alloca i32, align 4
 ; ATTRIBUTOR-NEXT:    [[L:%.*]] = load i32, ptr [[PTR]], align 4
@@ -350,14 +350,14 @@ define void @test_inaccessiblememonly_readonly() {
 define void @test_inaccessibleorargmemonly_readonly(ptr %arg) {
 ; FNATTRS: Function Attrs: nofree memory(argmem: read, inaccessiblemem: read)
 ; FNATTRS-LABEL: define void @test_inaccessibleorargmemonly_readonly
-; FNATTRS-SAME: (ptr nocapture readonly [[ARG:%.*]]) #[[ATTR14:[0-9]+]] {
+; FNATTRS-SAME: (ptr readonly captures(none) [[ARG:%.*]]) #[[ATTR14:[0-9]+]] {
 ; FNATTRS-NEXT:    [[TMP1:%.*]] = load i32, ptr [[ARG]], align 4
 ; FNATTRS-NEXT:    call void @fn_inaccessiblememonly() #[[ATTR19]]
 ; FNATTRS-NEXT:    ret void
 ;
 ; ATTRIBUTOR: Function Attrs: nosync memory(argmem: read, inaccessiblemem: read)
 ; ATTRIBUTOR-LABEL: define void @test_inaccessibleorargmemonly_readonly
-; ATTRIBUTOR-SAME: (ptr nocapture nofree nonnull readonly [[ARG:%.*]]) #[[ATTR13:[0-9]+]] {
+; ATTRIBUTOR-SAME: (ptr nofree nonnull readonly captures(none) [[ARG:%.*]]) #[[ATTR13:[0-9]+]] {
 ; ATTRIBUTOR-NEXT:    [[TMP1:%.*]] = load i32, ptr [[ARG]], align 4
 ; ATTRIBUTOR-NEXT:    call void @fn_inaccessiblememonly() #[[ATTR21]]
 ; ATTRIBUTOR-NEXT:    ret void
@@ -370,14 +370,14 @@ define void @test_inaccessibleorargmemonly_readonly(ptr %arg) {
 define void @test_inaccessibleorargmemonly_readwrite(ptr %arg) {
 ; FNATTRS: Function Attrs: memory(argmem: write, inaccessiblemem: read)
 ; FNATTRS-LABEL: define void @test_inaccessibleorargmemonly_readwrite
-; FNATTRS-SAME: (ptr nocapture writeonly initializes((0, 4)) [[ARG:%.*]]) #[[ATTR15:[0-9]+]] {
+; FNATTRS-SAME: (ptr writeonly captures(none) initializes((0, 4)) [[ARG:%.*]]) #[[ATTR15:[0-9]+]] {
 ; FNATTRS-NEXT:    store i32 0, ptr [[ARG]], align 4
 ; FNATTRS-NEXT:    call void @fn_inaccessiblememonly() #[[ATTR19]]
 ; FNATTRS-NEXT:    ret void
 ;
 ; ATTRIBUTOR: Function Attrs: nosync memory(argmem: readwrite, inaccessiblemem: readwrite)
 ; ATTRIBUTOR-LABEL: define void @test_inaccessibleorargmemonly_readwrite
-; ATTRIBUTOR-SAME: (ptr nocapture nofree nonnull writeonly [[ARG:%.*]]) #[[ATTR14:[0-9]+]] {
+; ATTRIBUTOR-SAME: (ptr nofree nonnull writeonly captures(none) [[ARG:%.*]]) #[[ATTR14:[0-9]+]] {
 ; ATTRIBUTOR-NEXT:    store i32 0, ptr [[ARG]], align 4
 ; ATTRIBUTOR-NEXT:    call void @fn_inaccessiblememonly() #[[ATTR21]]
 ; ATTRIBUTOR-NEXT:    ret void
@@ -390,16 +390,16 @@ define void @test_inaccessibleorargmemonly_readwrite(ptr %arg) {
 define void @test_recursive_argmem_read(ptr %p) {
 ; FNATTRS: Function Attrs: nofree nosync nounwind memory(read, inaccessiblemem: none)
 ; FNATTRS-LABEL: define void @test_recursive_argmem_read
-; FNATTRS-SAME: (ptr nocapture readonly [[P:%.*]]) #[[ATTR16:[0-9]+]] {
+; FNATTRS-SAME: (ptr readonly captures(none) [[P:%.*]]) #[[ATTR16:[0-9]+]] {
 ; FNATTRS-NEXT:    [[PVAL:%.*]] = load ptr, ptr [[P]], align 8
 ; FNATTRS-NEXT:    call void @test_recursive_argmem_read(ptr [[PVAL]])
 ; FNATTRS-NEXT:    ret void
 ;
 ; ATTRIBUTOR: Function Attrs: nofree nosync nounwind memory(read)
 ; ATTRIBUTOR-LABEL: define void @test_recursive_argmem_read
-; ATTRIBUTOR-SAME: (ptr nocapture nofree nonnull readonly [[P:%.*]]) #[[ATTR15:[0-9]+]] {
+; ATTRIBUTOR-SAME: (ptr nofree nonnull readonly captures(none) [[P:%.*]]) #[[ATTR15:[0-9]+]] {
 ; ATTRIBUTOR-NEXT:    [[PVAL:%.*]] = load ptr, ptr [[P]], align 8
-; ATTRIBUTOR-NEXT:    call void @test_recursive_argmem_read(ptr nocapture nofree readonly [[PVAL]]) #[[ATTR15]]
+; ATTRIBUTOR-NEXT:    call void @test_recursive_argmem_read(ptr nofree readonly captures(none) [[PVAL]]) #[[ATTR15]]
 ; ATTRIBUTOR-NEXT:    ret void
 ;
   %pval = load ptr, ptr %p
@@ -410,7 +410,7 @@ define void @test_recursive_argmem_read(ptr %p) {
 define void @test_recursive_argmem_readwrite(ptr %p) {
 ; FNATTRS: Function Attrs: nofree nosync nounwind memory(readwrite, inaccessiblemem: none)
 ; FNATTRS-LABEL: define void @test_recursive_argmem_readwrite
-; FNATTRS-SAME: (ptr nocapture [[P:%.*]]) #[[ATTR17:[0-9]+]] {
+; FNATTRS-SAME: (ptr captures(none) [[P:%.*]]) #[[ATTR17:[0-9]+]] {
 ; FNATTRS-NEXT:    [[PVAL:%.*]] = load ptr, ptr [[P]], align 8
 ; FNATTRS-NEXT:    store i32 0, ptr [[P]], align 4
 ; FNATTRS-NEXT:    call void @test_recursive_argmem_readwrite(ptr [[PVAL]])
@@ -418,10 +418,10 @@ define void @test_recursive_argmem_readwrite(ptr %p) {
 ;
 ; ATTRIBUTOR: Function Attrs: nofree nosync nounwind
 ; ATTRIBUTOR-LABEL: define void @test_recursive_argmem_readwrite
-; ATTRIBUTOR-SAME: (ptr nocapture nofree nonnull [[P:%.*]]) #[[ATTR16:[0-9]+]] {
+; ATTRIBUTOR-SAME: (ptr nofree nonnull captures(none) [[P:%.*]]) #[[ATTR16:[0-9]+]] {
 ; ATTRIBUTOR-NEXT:    [[PVAL:%.*]] = load ptr, ptr [[P]], align 8
 ; ATTRIBUTOR-NEXT:    store i32 0, ptr [[P]], align 4
-; ATTRIBUTOR-NEXT:    call void @test_recursive_argmem_readwrite(ptr nocapture nofree [[PVAL]]) #[[ATTR16]]
+; ATTRIBUTOR-NEXT:    call void @test_recursive_argmem_readwrite(ptr nofree captures(none) [[PVAL]]) #[[ATTR16]]
 ; ATTRIBUTOR-NEXT:    ret void
 ;
   %pval = load ptr, ptr %p
@@ -433,7 +433,7 @@ define void @test_recursive_argmem_readwrite(ptr %p) {
 define void @test_recursive_argmem_read_alloca(ptr %p) {
 ; FNATTRS: Function Attrs: nofree nosync nounwind memory(argmem: read)
 ; FNATTRS-LABEL: define void @test_recursive_argmem_read_alloca
-; FNATTRS-SAME: (ptr nocapture readonly [[P:%.*]]) #[[ATTR18:[0-9]+]] {
+; FNATTRS-SAME: (ptr readonly captures(none) [[P:%.*]]) #[[ATTR18:[0-9]+]] {
 ; FNATTRS-NEXT:    [[A:%.*]] = alloca ptr, align 8
 ; FNATTRS-NEXT:    [[TMP1:%.*]] = load i32, ptr [[P]], align 4
 ; FNATTRS-NEXT:    call void @test_recursive_argmem_read_alloca(ptr [[A]])
@@ -441,10 +441,10 @@ define void @test_recursive_argmem_read_alloca(ptr %p) {
 ;
 ; ATTRIBUTOR: Function Attrs: nofree nosync nounwind memory(argmem: read)
 ; ATTRIBUTOR-LABEL: define void @test_recursive_argmem_read_alloca
-; ATTRIBUTOR-SAME: (ptr nocapture nofree nonnull readonly [[P:%.*]]) #[[ATTR17:[0-9]+]] {
+; ATTRIBUTOR-SAME: (ptr nofree nonnull readonly captures(none) [[P:%.*]]) #[[ATTR17:[0-9]+]] {
 ; ATTRIBUTOR-NEXT:    [[A:%.*]] = alloca ptr, align 8
 ; ATTRIBUTOR-NEXT:    [[TMP1:%.*]] = load i32, ptr [[P]], align 4
-; ATTRIBUTOR-NEXT:    call void @test_recursive_argmem_read_alloca(ptr nocapture nofree nonnull readonly [[A]]) #[[ATTR15]]
+; ATTRIBUTOR-NEXT:    call void @test_recursive_argmem_read_alloca(ptr nofree nonnull readonly captures(none) [[A]]) #[[ATTR15]]
 ; ATTRIBUTOR-NEXT:    ret void
 ;
   %a = alloca ptr
@@ -456,16 +456,16 @@ define void @test_recursive_argmem_read_alloca(ptr %p) {
 define void @test_scc_argmem_read_1(ptr %p) {
 ; FNATTRS: Function Attrs: nofree nosync nounwind memory(read, inaccessiblemem: none)
 ; FNATTRS-LABEL: define void @test_scc_argmem_read_1
-; FNATTRS-SAME: (ptr nocapture readonly [[P:%.*]]) #[[ATTR16]] {
+; FNATTRS-SAME: (ptr readonly captures(none) [[P:%.*]]) #[[ATTR16]] {
 ; FNATTRS-NEXT:    [[PVAL:%.*]] = load ptr, ptr [[P]], align 8
 ; FNATTRS-NEXT:    call void @test_scc_argmem_read_2(ptr [[PVAL]])
 ; FNATTRS-NEXT:    ret void
 ;
 ; ATTRIBUTOR: Function Attrs: nofree nosync nounwind memory(read)
 ; ATTRIBUTOR-LABEL: define void @test_scc_argmem_read_1
-; ATTRIBUTOR-SAME: (ptr nocapture nofree nonnull readonly [[P:%.*]]) #[[ATTR15]] {
+; ATTRIBUTOR-SAME: (ptr nofree nonnull readonly captures(none) [[P:%.*]]) #[[ATTR15]] {
 ; ATTRIBUTOR-NEXT:    [[PVAL:%.*]] = load ptr, ptr [[P]], align 8
-; ATTRIBUTOR-NEXT:    call void @test_scc_argmem_read_2(ptr nocapture nofree readonly [[PVAL]]) #[[ATTR15]]
+; ATTRIBUTOR-NEXT:    call void @test_scc_argmem_read_2(ptr nofree readonly captures(none) [[PVAL]]) #[[ATTR15]]
 ; ATTRIBUTOR-NEXT:    ret void
 ;
   %pval = load ptr, ptr %p
@@ -476,14 +476,14 @@ define void @test_scc_argmem_read_1(ptr %p) {
 define void @test_scc_argmem_read_2(ptr %p) {
 ; FNATTRS: Function Attrs: nofree nosync nounwind memory(read, inaccessiblemem: none)
 ; FNATTRS-LABEL: define void @test_scc_argmem_read_2
-; FNATTRS-SAME: (ptr nocapture readonly [[P:%.*]]) #[[ATTR16]] {
+; FNATTRS-SAME: (ptr readonly captures(none) [[P:%.*]]) #[[ATTR16]] {
 ; FNATTRS-NEXT:    call void @test_scc_argmem_read_1(ptr [[P]])
 ; FNATTRS-NEXT:    ret void
 ;
 ; ATTRIBUTOR: Function Attrs: nofree nosync nounwind memory(read)
 ; ATTRIBUTOR-LABEL: define void @test_scc_argmem_read_2
-; ATTRIBUTOR-SAME: (ptr nocapture nofree readonly [[P:%.*]]) #[[ATTR15]] {
-; ATTRIBUTOR-NEXT:    call void @test_scc_argmem_read_1(ptr nocapture nofree readonly [[P]]) #[[ATTR15]]
+; ATTRIBUTOR-SAME: (ptr nofree readonly captures(none) [[P:%.*]]) #[[ATTR15]] {
+; ATTRIBUTOR-NEXT:    call void @test_scc_argmem_read_1(ptr nofree readonly captures(none) [[P]]) #[[ATTR15]]
 ; ATTRIBUTOR-NEXT:    ret void
 ;
   call void @test_scc_argmem_read_1(ptr %p)
@@ -493,7 +493,7 @@ define void @test_scc_argmem_read_2(ptr %p) {
 define i64 @select_same_obj(i1 %c, ptr %p, i64 %x) {
 ; FNATTRS: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: read)
 ; FNATTRS-LABEL: define i64 @select_same_obj
-; FNATTRS-SAME: (i1 [[C:%.*]], ptr nocapture readonly [[P:%.*]], i64 [[X:%.*]]) #[[ATTR1]] {
+; FNATTRS-SAME: (i1 [[C:%.*]], ptr readonly captures(none) [[P:%.*]], i64 [[X:%.*]]) #[[ATTR1]] {
 ; FNATTRS-NEXT:  entry:
 ; FNATTRS-NEXT:    [[P2:%.*]] = getelementptr i8, ptr [[P]], i64 [[X]]
 ; FNATTRS-NEXT:    [[P3:%.*]] = select i1 [[C]], ptr [[P]], ptr [[P2]]
@@ -502,7 +502,7 @@ define i64 @select_same_obj(i1 %c, ptr %p, i64 %x) {
 ;
 ; ATTRIBUTOR: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(none)
 ; ATTRIBUTOR-LABEL: define i64 @select_same_obj
-; ATTRIBUTOR-SAME: (i1 [[C:%.*]], ptr nocapture nofree readonly [[P:%.*]], i64 [[X:%.*]]) #[[ATTR0]] {
+; ATTRIBUTOR-SAME: (i1 [[C:%.*]], ptr nofree readonly captures(none) [[P:%.*]], i64 [[X:%.*]]) #[[ATTR0]] {
 ; ATTRIBUTOR-NEXT:  entry:
 ; ATTRIBUTOR-NEXT:    [[P2:%.*]] = getelementptr i8, ptr [[P]], i64 [[X]]
 ; ATTRIBUTOR-NEXT:    [[P3:%.*]] = select i1 [[C]], ptr [[P]], ptr [[P2]]
@@ -520,7 +520,7 @@ entry:
 define i64 @select_different_obj(i1 %c, ptr %p, ptr %p2) {
 ; FNATTRS: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(read, inaccessiblemem: none)
 ; FNATTRS-LABEL: define i64 @select_different_obj
-; FNATTRS-SAME: (i1 [[C:%.*]], ptr nocapture readonly [[P:%.*]], ptr nocapture readonly [[P2:%.*]]) #[[ATTR3]] {
+; FNATTRS-SAME: (i1 [[C:%.*]], ptr readonly captures(none) [[P:%.*]], ptr readonly captures(none) [[P2:%.*]]) #[[ATTR3]] {
 ; FNATTRS-NEXT:  entry:
 ; FNATTRS-NEXT:    [[P3:%.*]] = select i1 [[C]], ptr [[P]], ptr [[P2]]
 ; FNATTRS-NEXT:    [[R:%.*]] = load i64, ptr [[P3]], align 4
@@ -528,7 +528,7 @@ define i64 @select_different_obj(i1 %c, ptr %p, ptr %p2) {
 ;
 ; ATTRIBUTOR: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(none)
 ; ATTRIBUTOR-LABEL: define i64 @select_different_obj
-; ATTRIBUTOR-SAME: (i1 [[C:%.*]], ptr nocapture nofree readonly [[P:%.*]], ptr nocapture nofree readonly [[P2:%.*]]) #[[ATTR0]] {
+; ATTRIBUTOR-SAME: (i1 [[C:%.*]], ptr nofree readonly captures(none) [[P:%.*]], ptr nofree readonly captures(none) [[P2:%.*]]) #[[ATTR0]] {
 ; ATTRIBUTOR-NEXT:  entry:
 ; ATTRIBUTOR-NEXT:    [[P3:%.*]] = select i1 [[C]], ptr [[P]], ptr [[P2]]
 ; ATTRIBUTOR-NEXT:    [[R:%.*]] = load i64, ptr [[P3]], align 4
@@ -543,7 +543,7 @@ entry:
 define i64 @phi_same_obj(i1 %c, ptr %p, i64 %x) {
 ; FNATTRS: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: read)
 ; FNATTRS-LABEL: define i64 @phi_same_obj
-; FNATTRS-SAME: (i1 [[C:%.*]], ptr nocapture readonly [[P:%.*]], i64 [[X:%.*]]) #[[ATTR1]] {
+; FNATTRS-SAME: (i1 [[C:%.*]], ptr readonly captures(none) [[P:%.*]], i64 [[X:%.*]]) #[[ATTR1]] {
 ; FNATTRS-NEXT:  entry:
 ; FNATTRS-NEXT:    [[P2:%.*]] = getelementptr i8, ptr [[P]], i64 [[X]]
 ; FNATTRS-NEXT:    br i1 [[C]], label [[IF:%.*]], label [[JOIN:%.*]]
@@ -556,7 +556,7 @@ define i64 @phi_same_obj(i1 %c, ptr %p, i64 %x) {
 ;
 ; ATTRIBUTOR: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: read)
 ; ATTRIBUTOR-LABEL: define i64 @phi_same_obj
-; ATTRIBUTOR-SAME: (i1 [[C:%.*]], ptr nocapture nofree readonly [[P:%.*]], i64 [[X:%.*]]) #[[ATTR1]] {
+; ATTRIBUTOR-SAME: (i1 [[C:%.*]], ptr nofree readonly captures(none) [[P:%.*]], i64 [[X:%.*]]) #[[ATTR1]] {
 ; ATTRIBUTOR-NEXT:  entry:
 ; ATTRIBUTOR-NEXT:    [[P2:%.*]] = getelementptr i8, ptr [[P]], i64 [[X]]
 ; ATTRIBUTOR-NEXT:    br i1 [[C]], label [[IF:%.*]], label [[JOIN:%.*]]
@@ -582,7 +582,7 @@ join:
 define i64 @phi_different_obj(i1 %c, ptr %p, ptr %p2) {
 ; FNATTRS: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(read, inaccessiblemem: none)
 ; FNATTRS-LABEL: define i64 @phi_different_obj
-; FNATTRS-SAME: (i1 [[C:%.*]], ptr nocapture readonly [[P:%.*]], ptr nocapture readonly [[P2:%.*]]) #[[ATTR3]] {
+; FNATTRS-SAME: (i1 [[C:%.*]], ptr readonly captures(none) [[P:%.*]], ptr readonly captures(none) [[P2:%.*]]) #[[ATTR3]] {
 ; FNATTRS-NEXT:  entry:
 ; FNATTRS-NEXT:    br i1 [[C]], label [[IF:%.*]], label [[JOIN:%.*]]
 ; FNATTRS:       if:
@@ -594,7 +594,7 @@ define i64 @phi_different_obj(i1 %c, ptr %p, ptr %p2) {
 ;
 ; ATTRIBUTOR: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: read)
 ; ATTRIBUTOR-LABEL: define i64 @phi_different_obj
-; ATTRIBUTOR-SAME: (i1 [[C:%.*]], ptr nocapture nofree readonly [[P:%.*]], ptr nocapture nofree readonly [[P2:%.*]]) #[[ATTR1]] {
+; ATTRIBUTOR-SAME: (i1 [[C:%.*]], ptr nofree readonly captures(none) [[P:%.*]], ptr nofree readonly captures(none) [[P2:%.*]]) #[[ATTR1]] {
 ; ATTRIBUTOR-NEXT:  entry:
 ; ATTRIBUTOR-NEXT:    br i1 [[C]], label [[IF:%.*]], label [[JOIN:%.*]]
 ; ATTRIBUTOR:       if:
diff --git a/llvm/test/Transforms/FunctionAttrs/convergent.ll b/llvm/test/Transforms/FunctionAttrs/convergent.ll
index a0f4c07e43371..fe8029d39d924 100644
--- a/llvm/test/Transforms/FunctionAttrs/convergent.ll
+++ b/llvm/test/Transforms/FunctionAttrs/convergent.ll
@@ -50,7 +50,7 @@ define i32 @extern_non_convergent_call() convergent {
 define i32 @indirect_convergent_call(ptr %f) convergent {
 ; CHECK: Function Attrs: convergent
 ; CHECK-LABEL: define {{[^@]+}}@indirect_convergent_call
-; CHECK-SAME: (ptr nocapture readonly [[F:%.*]]) #[[ATTR1]] {
+; CHECK-SAME: (ptr readonly captures(none) [[F:%.*]]) #[[ATTR1]] {
 ; CHECK-NEXT:    [[A:%.*]] = call i32 [[F]]() #[[ATTR1]]
 ; CHECK-NEXT:    ret i32 [[A]]
 ;
@@ -62,7 +62,7 @@ define i32 @indirect_convergent_call(ptr %f) convergent {
 define i32 @indirect_non_convergent_call(ptr %f) convergent norecurse {
 ; CHECK: Function Attrs: norecurse
 ; CHECK-LABEL: define {{[^@]+}}@indirect_non_convergent_call
-; CHECK-SAME: (ptr nocapture readonly [[F:%.*]]) #[[ATTR2:[0-9]+]] {
+; CHECK-SAME: (ptr readonly captures(none) [[F:%.*]]) #[[ATTR2:[0-9]+]] {
 ; CHECK-NEXT:    [[A:%.*]] = call i32 [[F]]()
 ; CHECK-NEXT:    ret i32 [[A]]
 ;
diff --git a/llvm/test/Transforms/FunctionAttrs/initializes.ll b/llvm/test/Transforms/FunctionAttrs/initializes.ll
index e04a095c8a43b..861c61d683ae0 100644
--- a/llvm/test/Transforms/FunctionAttrs/initializes.ll
+++ b/llvm/test/Transforms/FunctionAttrs/initializes.ll
@@ -4,7 +4,7 @@
 define void @basic(ptr %p) {
 ; CHECK: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: write)
 ; CHECK-LABEL: define void @basic(
-; CHECK-SAME: ptr nocapture writeonly initializes((0, 8)) [[P:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-SAME: ptr writeonly captures(none) initializes((0, 8)) [[P:%.*]]) #[[ATTR0:[0-9]+]] {
 ; CHECK-NEXT:    store i64 123, ptr [[P]], align 4
 ; CHECK-NEXT:    ret void
 ;
@@ -15,7 +15,7 @@ define void @basic(ptr %p) {
 define void @stores_on_both_paths(ptr %p, i1 %i) {
 ; CHECK: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: write)
 ; CHECK-LABEL: define void @stores_on_both_paths(
-; CHECK-SAME: ptr nocapture writeonly initializes((0, 8)) [[P:%.*]], i1 [[I:%.*]]) #[[ATTR0]] {
+; CHECK-SAME: ptr writeonly captures(none) initializes((0, 8)) [[P:%.*]], i1 [[I:%.*]]) #[[ATTR0]] {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    br i1 [[I]], label [[BB1:%.*]], label [[BB2:%.*]]
 ; CHECK:       bb1:
@@ -42,7 +42,7 @@ end:
 define void @store_pointer_to_pointer(ptr %p, ptr %p2) {
 ; CHECK: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: write)
 ; CHECK-LABEL: define void @store_pointer_to_pointer(
-; CHECK-SAME: ptr [[P:%.*]], ptr nocapture writeonly initializes((0, 8)) [[P2:%.*]]) #[[ATTR0]] {
+; CHECK-SAME: ptr [[P:%.*]], ptr writeonly captures(none) initializes((0, 8)) [[P2:%.*]]) #[[ATTR0]] {
 ; CHECK-NEXT:    store ptr [[P]], ptr [[P2]], align 8
 ; CHECK-NEXT:    ret void
 ;
@@ -65,7 +65,7 @@ define void @store_pointer_to_itself(ptr %p) {
 define void @load_before_store(ptr %p) {
 ; CHECK: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: readwrite)
 ; CHECK-LABEL: define void @load_before_store(
-; CHECK-SAME: ptr nocapture [[P:%.*]]) #[[ATTR1:[0-9]+]] {
+; CHECK-SAME: ptr captures(none) [[P:%.*]]) #[[ATTR1:[0-9]+]] {
 ; CHECK-NEXT:    [[A:%.*]] = load i32, ptr [[P]], align 4
 ; CHECK-NEXT:    store i32 123, ptr [[P]], align 4
 ; CHECK-NEXT:    ret void
@@ -78,7 +78,7 @@ define void @load_before_store(ptr %p) {
 define void @partial_load_before_store(ptr %p) {
 ; CHECK: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: readwrite)
 ; CHECK-LABEL: define void @partial_load_before_store(
-; CHECK-SAME: ptr nocapture initializes((4, 8)) [[P:%.*]]) #[[ATTR1]] {
+; CHECK-SAME: ptr captures(none) initializes((4, 8)) [[P:%.*]]) #[[ATTR1]] {
 ; CHECK-NEXT:    [[A:%.*]] = load i32, ptr [[P]], align 4
 ; CHECK-NEXT:    store i64 123, ptr [[P]], align 4
 ; CHECK-NEXT:    ret void
@@ -117,7 +117,7 @@ define void @call_clobber_after_store(ptr %p) {
 define void @store_offset(ptr %p) {
 ; CHECK: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: write)
 ; CHECK-LABEL: define void @store_offset(
-; CHECK-SAME: ptr nocapture writeonly initializes((8, 12)) [[P:%.*]]) #[[ATTR0]] {
+; CHECK-SAME: ptr writeonly captures(none) initializes((8, 12)) [[P:%.*]]) #[[ATTR0]] {
 ; CHECK-NEXT:    [[G:%.*]] = getelementptr i8, ptr [[P]], i64 8
 ; CHECK-NEXT:    store i32 123, ptr [[G]], align 4
 ; CHECK-NEXT:    ret void
@@ -143,7 +143,7 @@ define void @store_volatile(ptr %p) {
 define void @merge_store_ranges(ptr %p) {
 ; CHECK: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: write)
 ; CHECK-LABEL: define void @merge_store_ranges(
-; CHECK-SAME: ptr nocapture writeonly initializes((0, 8)) [[P:%.*]]) #[[ATTR0]] {
+; CHECK-SAME: ptr writeonly captures(none) initializes((0, 8)) [[P:%.*]]) #[[ATTR0]] {
 ; CHECK-NEXT:    [[G:%.*]] = getelementptr i8, ptr [[P]], i64 4
 ; CHECK-NEXT:    store i32 123, ptr [[G]], align 4
 ; CHECK-NEXT:    store i32 123, ptr [[P]], align 4
@@ -158,7 +158,7 @@ define void @merge_store_ranges(ptr %p) {
 define void @partially_overlapping_stores_branches(ptr %p, i1 %i) {
 ; CHECK: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: readwrite)
 ; CHECK-LABEL: define void @partially_overlapping_stores_branches(
-; CHECK-SAME: ptr nocapture initializes((4, 8)) [[P:%.*]], i1 [[I:%.*]]) #[[ATTR1]] {
+; CHECK-SAME: ptr captures(none) initializes((4, 8)) [[P:%.*]], i1 [[I:%.*]]) #[[ATTR1]] {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[A:%.*]] = load i32, ptr [[P]], align 4
 ; CHECK-NEXT:    [[G:%.*]] = getelementptr i8, ptr [[P]], i64 4
@@ -189,7 +189,7 @@ end:
 define void @non_overlapping_stores_branches(ptr %p, i1 %i) {
 ; CHECK: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: write)
 ; CHECK-LABEL: define void @non_overlapping_stores_branches(
-; CHECK-SAME: ptr nocapture writeonly [[P:%.*]], i1 [[I:%.*]]) #[[ATTR0]] {
+; CHECK-SAME: ptr writeonly captures(none) [[P:%.*]], i1 [[I:%.*]]) #[[ATTR0]] {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[G:%.*]] = getelementptr i8, ptr [[P]], i64 8
 ; CHECK-NEXT:    br i1 [[I]], label [[BB1:%.*]], label [[BB2:%.*]]
@@ -218,7 +218,7 @@ end:
 define void @dominating_store(ptr %p, i1 %i) {
 ; CHECK: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: write)
 ; CHECK-LABEL: define void @dominating_store(
-; CHECK-SAME: ptr nocapture writeonly initializes((0, 8)) [[P:%.*]], i1 [[I:%.*]]) #[[ATTR0]] {
+; CHECK-SAME: ptr writeonly captures(none) initializes((0, 8)) [[P:%.*]], i1 [[I:%.*]]) #[[ATTR0]] {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    br i1 [[I]], label [[BB1:%.*]], label [[BB2:%.*]]
 ; CHECK:       bb1:
@@ -269,7 +269,7 @@ end:
 define void @merge_existing_initializes(ptr initializes((33, 36)) %p) {
 ; CHECK: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: write)
 ; CHECK-LABEL: define void @merge_existing_initializes(
-; CHECK-SAME: ptr nocapture writeonly initializes((0, 8), (33, 36)) [[P:%.*]]) #[[ATTR0]] {
+; CHECK-SAME: ptr writeonly captures(none) initializes((0, 8), (33, 36)) [[P:%.*]]) #[[ATTR0]] {
 ; CHECK-NEXT:    store i64 123, ptr [[P]], align 4
 ; CHECK-NEXT:    ret void
 ;
@@ -280,7 +280,7 @@ define void @merge_existing_initializes(ptr initializes((33, 36)) %p) {
 define void @negative_offset(ptr %p) {
 ; CHECK: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: write)
 ; CHECK-LABEL: define void @negative_offset(
-; CHECK-SAME: ptr nocapture writeonly initializes((-5, 3)) [[P:%.*]]) #[[ATTR0]] {
+; CHECK-SAME: ptr writeonly captures(none) initializes((-5, 3)) [[P:%.*]]) #[[ATTR0]] {
 ; CHECK-NEXT:    [[G:%.*]] = getelementptr i8, ptr [[P]], i64 -5
 ; CHECK-NEXT:    store i64 123, ptr [[G]], align 4
 ; CHECK-NEXT:    ret void
@@ -293,7 +293,7 @@ define void @negative_offset(ptr %p) {
 define void @non_const_gep(ptr %p, i64 %i) {
 ; CHECK: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: write)
 ; CHECK-LABEL: define void @non_const_gep(
-; CHECK-SAME: ptr nocapture writeonly initializes((0, 8)) [[P:%.*]], i64 [[I:%.*]]) #[[ATTR0]] {
+; CHECK-SAME: ptr writeonly captures(none) initializes((0, 8)) [[P:%.*]], i64 [[I:%.*]]) #[[ATTR0]] {
 ; CHECK-NEXT:    [[G:%.*]] = getelementptr i8, ptr [[P]], i64 [[I]]
 ; CHECK-NEXT:    store i64 123, ptr [[G]], align 4
 ; CHECK-NEXT:    store i64 123, ptr [[P]], align 4
@@ -410,7 +410,7 @@ declare void @llvm.memset(ptr, i8, i64 ,i1)
 define void @memset(ptr %p) {
 ; CHECK: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: write)
 ; CHECK-LABEL: define void @memset(
-; CHECK-SAME: ptr nocapture writeonly initializes((0, 9)) [[P:%.*]]) #[[ATTR0]] {
+; CHECK-SAME: ptr writeonly captures(none) initializes((0, 9)) [[P:%.*]]) #[[ATTR0]] {
 ; CHECK-NEXT:    call void @llvm.memset.p0.i64(ptr [[P]], i8 2, i64 9, i1 false)
 ; CHECK-NEXT:    ret void
 ;
@@ -421,7 +421,7 @@ define void @memset(ptr %p) {
 define void @memset_offset(ptr %p) {
 ; CHECK: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: write)
 ; CHECK-LABEL: define void @memset_offset(
-; CHECK-SAME: ptr nocapture writeonly initializes((3, 12)) [[P:%.*]]) #[[ATTR0]] {
+; CHECK-SAME: ptr writeonly captures(none) initializes((3, 12)) [[P:%.*]]) #[[ATTR0]] {
 ; CHECK-NEXT:    [[G:%.*]] = getelementptr i8, ptr [[P]], i64 3
 ; CHECK-NEXT:    call void @llvm.memset.p0.i64(ptr [[G]], i8 2, i64 9, i1 false)
 ; CHECK-NEXT:    ret void
@@ -434,7 +434,7 @@ define void @memset_offset(ptr %p) {
 define void @memset_neg(ptr %p) {
 ; CHECK: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: write)
 ; CHECK-LABEL: define void @memset_neg(
-; CHECK-SAME: ptr nocapture writeonly [[P:%.*]]) #[[ATTR0]] {
+; CHECK-SAME: ptr writeonly captures(none) [[P:%.*]]) #[[ATTR0]] {
 ; CHECK-NEXT:    call void @llvm.memset.p0.i64(ptr [[P]], i8 2, i64 -1, i1 false)
 ; CHECK-NEXT:    ret void
 ;
@@ -456,7 +456,7 @@ define void @memset_volatile(ptr %p) {
 define void @memset_non_constant(ptr %p, i64 %i) {
 ; CHECK: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: write)
 ; CHECK-LABEL: define void @memset_non_constant(
-; CHECK-SAME: ptr nocapture writeonly [[P:%.*]], i64 [[I:%.*]]) #[[ATTR0]] {
+; CHECK-SAME: ptr writeonly captures(none) [[P:%.*]], i64 [[I:%.*]]) #[[ATTR0]] {
 ; CHECK-NEXT:    call void @llvm.memset.p0.i64(ptr [[P]], i8 2, i64 [[I]], i1 false)
 ; CHECK-NEXT:    ret void
 ;
@@ -469,7 +469,7 @@ declare void @llvm.memcpy(ptr, ptr, i64 ,i1)
 define void @memcpy(ptr %p, ptr %p2) {
 ; CHECK: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: readwrite)
 ; CHECK-LABEL: define void @memcpy(
-; CHECK-SAME: ptr nocapture writeonly initializes((0, 9)) [[P:%.*]], ptr nocapture readonly [[P2:%.*]]) #[[ATTR1]] {
+; CHECK-SAME: ptr writeonly captures(none) initializes((0, 9)) [[P:%.*]], ptr readonly captures(none) [[P2:%.*]]) #[[ATTR1]] {
 ; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr [[P]], ptr [[P2]], i64 9, i1 false)
 ; CHECK-NEXT:    ret void
 ;
@@ -491,7 +491,7 @@ define void @memcpy_volatile(ptr %p, ptr %p2) {
 define void @memcpy_offset(ptr %p, ptr %p2) {
 ; CHECK: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: readwrite)
 ; CHECK-LABEL: define void @memcpy_offset(
-; CHECK-SAME: ptr nocapture writeonly initializes((3, 12)) [[P:%.*]], ptr nocapture readonly [[P2:%.*]]) #[[ATTR1]] {
+; CHECK-SAME: ptr writeonly captures(none) initializes((3, 12)) [[P:%.*]], ptr readonly captures(none) [[P2:%.*]]) #[[ATTR1]] {
 ; CHECK-NEXT:    [[G:%.*]] = getelementptr i8, ptr [[P]], i64 3
 ; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr [[G]], ptr [[P2]], i64 9, i1 false)
 ; CHECK-NEXT:    ret void
@@ -504,7 +504,7 @@ define void @memcpy_offset(ptr %p, ptr %p2) {
 define void @memcpy_src(ptr %p, ptr %p2) {
 ; CHECK: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: readwrite)
 ; CHECK-LABEL: define void @memcpy_src(
-; CHECK-SAME: ptr nocapture initializes((96, 128)) [[P:%.*]], ptr nocapture initializes((0, 96)) [[P2:%.*]]) #[[ATTR1]] {
+; CHECK-SAME: ptr captures(none) initializes((96, 128)) [[P:%.*]], ptr captures(none) initializes((0, 96)) [[P2:%.*]]) #[[ATTR1]] {
 ; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr [[P2]], ptr [[P]], i64 96, i1 false)
 ; CHECK-NEXT:    [[G:%.*]] = getelementptr i8, ptr [[P]], i64 64
 ; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr [[G]], ptr [[P2]], i64 64, i1 false)
@@ -519,7 +519,7 @@ define void @memcpy_src(ptr %p, ptr %p2) {
 define void @memcpy_non_constant(ptr %p, ptr %p2, i64 %i) {
 ; CHECK: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: readwrite)
 ; CHECK-LABEL: define void @memcpy_non_constant(
-; CHECK-SAME: ptr nocapture writeonly [[P:%.*]], ptr nocapture readonly [[P2:%.*]], i64 [[I:%.*]]) #[[ATTR1]] {
+; CHECK-SAME: ptr writeonly captures(none) [[P:%.*]], ptr readonly captures(none) [[P2:%.*]], i64 [[I:%.*]]) #[[ATTR1]] {
 ; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr [[P]], ptr [[P2]], i64 [[I]], i1 false)
 ; CHECK-NEXT:    ret void
 ;
@@ -532,7 +532,7 @@ declare void @llvm.memmove(ptr, ptr, i64 ,i1)
 define void @memmove(ptr %p, ptr %p2) {
 ; CHECK: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: readwrite)
 ; CHECK-LABEL: define void @memmove(
-; CHECK-SAME: ptr nocapture writeonly initializes((0, 9)) [[P:%.*]], ptr nocapture readonly [[P2:%.*]]) #[[ATTR1]] {
+; CHECK-SAME: ptr writeonly captures(none) initializes((0, 9)) [[P:%.*]], ptr readonly captures(none) [[P2:%.*]]) #[[ATTR1]] {
 ; CHECK-NEXT:    call void @llvm.memmove.p0.p0.i64(ptr [[P]], ptr [[P2]], i64 9, i1 false)
 ; CHECK-NEXT:    ret void
 ;
@@ -554,7 +554,7 @@ define void @memmove_volatile(ptr %p, ptr %p2) {
 define void @memmove_offset(ptr %p, ptr %p2) {
 ; CHECK: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: readwrite)
 ; CHECK-LABEL: define void @memmove_offset(
-; CHECK-SAME: ptr nocapture writeonly initializes((3, 12)) [[P:%.*]], ptr nocapture readonly [[P2:%.*]]) #[[ATTR1]] {
+; CHECK-SAME: ptr writeonly captures(none) initializes((3, 12)) [[P:%.*]], ptr readonly captures(none) [[P2:%.*]]) #[[ATTR1]] {
 ; CHECK-NEXT:    [[G:%.*]] = getelementptr i8, ptr [[P]], i64 3
 ; CHECK-NEXT:    call void @llvm.memmove.p0.p0.i64(ptr [[G]], ptr [[P2]], i64 9, i1 false)
 ; CHECK-NEXT:    ret void
@@ -567,7 +567,7 @@ define void @memmove_offset(ptr %p, ptr %p2) {
 define void @memmove_src(ptr %p, ptr %p2) {
 ; CHECK: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: readwrite)
 ; CHECK-LABEL: define void @memmove_src(
-; CHECK-SAME: ptr nocapture initializes((96, 128)) [[P:%.*]], ptr nocapture initializes((0, 96)) [[P2:%.*]]) #[[ATTR1]] {
+; CHECK-SAME: ptr captures(none) initializes((96, 128)) [[P:%.*]], ptr captures(none) initializes((0, 96)) [[P2:%.*]]) #[[ATTR1]] {
 ; CHECK-NEXT:    call void @llvm.memmove.p0.p0.i64(ptr [[P2]], ptr [[P]], i64 96, i1 false)
 ; CHECK-NEXT:    [[G:%.*]] = getelementptr i8, ptr [[P]], i64 64
 ; CHECK-NEXT:    call void @llvm.memmove.p0.p0.i64(ptr [[G]], ptr [[P2]], i64 64, i1 false)
@@ -582,7 +582,7 @@ define void @memmove_src(ptr %p, ptr %p2) {
 define void @memmove_non_constant(ptr %p, ptr %p2, i64 %i) {
 ; CHECK: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: readwrite)
 ; CHECK-LABEL: define void @memmove_non_constant(
-; CHECK-SAME: ptr nocapture writeonly [[P:%.*]], ptr nocapture readonly [[P2:%.*]], i64 [[I:%.*]]) #[[ATTR1]] {
+; CHECK-SAME: ptr writeonly captures(none) [[P:%.*]], ptr readonly captures(none) [[P2:%.*]], i64 [[I:%.*]]) #[[ATTR1]] {
 ; CHECK-NEXT:    call void @llvm.memmove.p0.p0.i64(ptr [[P]], ptr [[P2]], i64 [[I]], i1 false)
 ; CHECK-NEXT:    ret void
 ;
@@ -593,7 +593,7 @@ define void @memmove_non_constant(ptr %p, ptr %p2, i64 %i) {
 define void @callee_byval(ptr byval(i32) %p) {
 ; CHECK: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: write)
 ; CHECK-LABEL: define void @callee_byval(
-; CHECK-SAME: ptr nocapture writeonly byval(i32) initializes((0, 4)) [[P:%.*]]) #[[ATTR0]] {
+; CHECK-SAME: ptr writeonly byval(i32) captures(none) initializes((0, 4)) [[P:%.*]]) #[[ATTR0]] {
 ; CHECK-NEXT:    store i32 0, ptr [[P]], align 4
 ; CHECK-NEXT:    ret void
 ;
@@ -604,7 +604,7 @@ define void @callee_byval(ptr byval(i32) %p) {
 define void @caller_byval(ptr %p) {
 ; CHECK: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: write)
 ; CHECK-LABEL: define void @caller_byval(
-; CHECK-SAME: ptr nocapture readonly [[P:%.*]]) #[[ATTR0]] {
+; CHECK-SAME: ptr readonly captures(none) [[P:%.*]]) #[[ATTR0]] {
 ; CHECK-NEXT:    call void @callee_byval(ptr byval(i32) [[P]])
 ; CHECK-NEXT:    ret void
 ;
@@ -615,7 +615,7 @@ define void @caller_byval(ptr %p) {
 define void @memset_offset_0_size_0(ptr %dst, ptr %src) {
 ; CHECK: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: readwrite)
 ; CHECK-LABEL: define void @memset_offset_0_size_0(
-; CHECK-SAME: ptr nocapture writeonly [[DST:%.*]], ptr nocapture readonly [[SRC:%.*]]) #[[ATTR1]] {
+; CHECK-SAME: ptr writeonly captures(none) [[DST:%.*]], ptr readonly captures(none) [[SRC:%.*]]) #[[ATTR1]] {
 ; CHECK-NEXT:    call void @llvm.memmove.p0.p0.i64(ptr [[DST]], ptr [[SRC]], i64 0, i1 false)
 ; CHECK-NEXT:    ret void
 ;
@@ -626,7 +626,7 @@ define void @memset_offset_0_size_0(ptr %dst, ptr %src) {
 define void @memset_offset_1_size_0(ptr %dst, ptr %src) {
 ; CHECK: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: readwrite)
 ; CHECK-LABEL: define void @memset_offset_1_size_0(
-; CHECK-SAME: ptr nocapture writeonly [[DST:%.*]], ptr nocapture readonly [[SRC:%.*]]) #[[ATTR1]] {
+; CHECK-SAME: ptr writeonly captures(none) [[DST:%.*]], ptr readonly captures(none) [[SRC:%.*]]) #[[ATTR1]] {
 ; CHECK-NEXT:    [[DST_1:%.*]] = getelementptr inbounds i8, ptr [[DST]], i64 1
 ; CHECK-NEXT:    call void @llvm.memmove.p0.p0.i64(ptr [[DST_1]], ptr [[SRC]], i64 0, i1 false)
 ; CHECK-NEXT:    ret void
diff --git a/llvm/test/Transforms/FunctionAttrs/make-buffer-rsrc.ll b/llvm/test/Transforms/FunctionAttrs/make-buffer-rsrc.ll
index bb9ef9156794e..59ec2d47bc72c 100644
--- a/llvm/test/Transforms/FunctionAttrs/make-buffer-rsrc.ll
+++ b/llvm/test/Transforms/FunctionAttrs/make-buffer-rsrc.ll
@@ -8,7 +8,7 @@ target datalayout = "e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:3
 define amdgpu_kernel void @test_make_buffer_rsrc(ptr %p, ptr %q) {
 ; FNATTRS: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: readwrite)
 ; FNATTRS-LABEL: define {{[^@]+}}@test_make_buffer_rsrc
-; FNATTRS-SAME: (ptr nocapture readonly [[P:%.*]], ptr nocapture writeonly [[Q:%.*]]) #[[ATTR0:[0-9]+]] {
+; FNATTRS-SAME: (ptr readonly captures(none) [[P:%.*]], ptr writeonly captures(none) [[Q:%.*]]) #[[ATTR0:[0-9]+]] {
 ; FNATTRS-NEXT:    [[P_RSRC:%.*]] = call ptr addrspace(8) @llvm.amdgcn.make.buffer.rsrc.p0(ptr [[P]], i16 0, i32 4, i32 822243328)
 ; FNATTRS-NEXT:    [[Q_RSRC:%.*]] = call ptr addrspace(8) @llvm.amdgcn.make.buffer.rsrc.p0(ptr [[Q]], i16 0, i32 4, i32 822243328)
 ; FNATTRS-NEXT:    [[V:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) [[P_RSRC]], i32 0, i32 0, i32 0)
@@ -17,11 +17,11 @@ define amdgpu_kernel void @test_make_buffer_rsrc(ptr %p, ptr %q) {
 ;
 ; ATTRIBUTOR: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: readwrite)
 ; ATTRIBUTOR-LABEL: define {{[^@]+}}@test_make_buffer_rsrc
-; ATTRIBUTOR-SAME: (ptr nocapture nofree readonly [[P:%.*]], ptr nocapture nofree writeonly [[Q:%.*]]) #[[ATTR0:[0-9]+]] {
+; ATTRIBUTOR-SAME: (ptr nofree readonly captures(none) [[P:%.*]], ptr nofree writeonly captures(none) [[Q:%.*]]) #[[ATTR0:[0-9]+]] {
 ; ATTRIBUTOR-NEXT:    [[P_RSRC:%.*]] = call ptr addrspace(8) @llvm.amdgcn.make.buffer.rsrc.p0(ptr [[P]], i16 0, i32 4, i32 822243328) #[[ATTR4:[0-9]+]]
 ; ATTRIBUTOR-NEXT:    [[Q_RSRC:%.*]] = call ptr addrspace(8) @llvm.amdgcn.make.buffer.rsrc.p0(ptr [[Q]], i16 0, i32 4, i32 822243328) #[[ATTR4]]
-; ATTRIBUTOR-NEXT:    [[V:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) nocapture readonly [[P_RSRC]], i32 0, i32 0, i32 0) #[[ATTR5:[0-9]+]]
-; ATTRIBUTOR-NEXT:    call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[V]], ptr addrspace(8) nocapture writeonly [[Q_RSRC]], i32 0, i32 0, i32 0) #[[ATTR6:[0-9]+]]
+; ATTRIBUTOR-NEXT:    [[V:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) readonly captures(none) [[P_RSRC]], i32 0, i32 0, i32 0) #[[ATTR5:[0-9]+]]
+; ATTRIBUTOR-NEXT:    call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[V]], ptr addrspace(8) writeonly captures(none) [[Q_RSRC]], i32 0, i32 0, i32 0) #[[ATTR6:[0-9]+]]
 ; ATTRIBUTOR-NEXT:    ret void
 ;
   %p.rsrc = call ptr addrspace(8) @llvm.amdgcn.make.buffer.rsrc.p0(ptr %p, i16 0, i32 4, i32 822243328)
diff --git a/llvm/test/Transforms/FunctionAttrs/nocapture.ll b/llvm/test/Transforms/FunctionAttrs/nocapture.ll
index 7df6132ac6a31..6164f2adbf5b9 100644
--- a/llvm/test/Transforms/FunctionAttrs/nocapture.ll
+++ b/llvm/test/Transforms/FunctionAttrs/nocapture.ll
@@ -245,7 +245,7 @@ define i1 @c7(ptr %q, i32 %bitno) {
 define i32 @nc1(ptr %q, ptr %p, i1 %b) {
 ; FNATTRS: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(readwrite, inaccessiblemem: none)
 ; FNATTRS-LABEL: define i32 @nc1
-; FNATTRS-SAME: (ptr [[Q:%.*]], ptr nocapture [[P:%.*]], i1 [[B:%.*]]) #[[ATTR7:[0-9]+]] {
+; FNATTRS-SAME: (ptr [[Q:%.*]], ptr captures(none) [[P:%.*]], i1 [[B:%.*]]) #[[ATTR7:[0-9]+]] {
 ; FNATTRS-NEXT:  e:
 ; FNATTRS-NEXT:    br label [[L:%.*]]
 ; FNATTRS:       l:
@@ -259,7 +259,7 @@ define i32 @nc1(ptr %q, ptr %p, i1 %b) {
 ;
 ; ATTRIBUTOR: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn
 ; ATTRIBUTOR-LABEL: define i32 @nc1
-; ATTRIBUTOR-SAME: (ptr nofree [[Q:%.*]], ptr nocapture nofree [[P:%.*]], i1 [[B:%.*]]) #[[ATTR5:[0-9]+]] {
+; ATTRIBUTOR-SAME: (ptr nofree [[Q:%.*]], ptr nofree captures(none) [[P:%.*]], i1 [[B:%.*]]) #[[ATTR5:[0-9]+]] {
 ; ATTRIBUTOR-NEXT:  e:
 ; ATTRIBUTOR-NEXT:    br label [[L:%.*]]
 ; ATTRIBUTOR:       l:
@@ -286,7 +286,7 @@ l:
 define i32 @nc1_addrspace(ptr %q, ptr addrspace(1) %p, i1 %b) {
 ; FNATTRS: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(readwrite, inaccessiblemem: none)
 ; FNATTRS-LABEL: define i32 @nc1_addrspace
-; FNATTRS-SAME: (ptr [[Q:%.*]], ptr addrspace(1) nocapture [[P:%.*]], i1 [[B:%.*]]) #[[ATTR7]] {
+; FNATTRS-SAME: (ptr [[Q:%.*]], ptr addrspace(1) captures(none) [[P:%.*]], i1 [[B:%.*]]) #[[ATTR7]] {
 ; FNATTRS-NEXT:  e:
 ; FNATTRS-NEXT:    br label [[L:%.*]]
 ; FNATTRS:       l:
@@ -301,7 +301,7 @@ define i32 @nc1_addrspace(ptr %q, ptr addrspace(1) %p, i1 %b) {
 ;
 ; ATTRIBUTOR: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn
 ; ATTRIBUTOR-LABEL: define i32 @nc1_addrspace
-; ATTRIBUTOR-SAME: (ptr nofree [[Q:%.*]], ptr addrspace(1) nocapture nofree [[P:%.*]], i1 [[B:%.*]]) #[[ATTR5]] {
+; ATTRIBUTOR-SAME: (ptr nofree [[Q:%.*]], ptr addrspace(1) nofree captures(none) [[P:%.*]], i1 [[B:%.*]]) #[[ATTR5]] {
 ; ATTRIBUTOR-NEXT:  e:
 ; ATTRIBUTOR-NEXT:    br label [[L:%.*]]
 ; ATTRIBUTOR:       l:
@@ -330,14 +330,14 @@ l:
 define void @nc2(ptr %p, ptr %q) {
 ; FNATTRS: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(readwrite, inaccessiblemem: none)
 ; FNATTRS-LABEL: define void @nc2
-; FNATTRS-SAME: (ptr nocapture [[P:%.*]], ptr [[Q:%.*]]) #[[ATTR7]] {
+; FNATTRS-SAME: (ptr captures(none) [[P:%.*]], ptr [[Q:%.*]]) #[[ATTR7]] {
 ; FNATTRS-NEXT:    [[TMP1:%.*]] = call i32 @nc1(ptr [[Q]], ptr [[P]], i1 false)
 ; FNATTRS-NEXT:    ret void
 ;
 ; ATTRIBUTOR: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn
 ; ATTRIBUTOR-LABEL: define void @nc2
-; ATTRIBUTOR-SAME: (ptr nocapture nofree [[P:%.*]], ptr nofree [[Q:%.*]]) #[[ATTR5]] {
-; ATTRIBUTOR-NEXT:    [[TMP1:%.*]] = call i32 @nc1(ptr nofree [[Q]], ptr nocapture nofree [[P]], i1 false) #[[ATTR18:[0-9]+]]
+; ATTRIBUTOR-SAME: (ptr nofree captures(none) [[P:%.*]], ptr nofree [[Q:%.*]]) #[[ATTR5]] {
+; ATTRIBUTOR-NEXT:    [[TMP1:%.*]] = call i32 @nc1(ptr nofree [[Q]], ptr nofree captures(none) [[P]], i1 false) #[[ATTR18:[0-9]+]]
 ; ATTRIBUTOR-NEXT:    ret void
 ;
   %1 = call i32 @nc1(ptr %q, ptr %p, i1 0)		; <i32> [#uses=0]
@@ -347,12 +347,12 @@ define void @nc2(ptr %p, ptr %q) {
 
 define void @nc3(ptr %p) {
 ; FNATTRS-LABEL: define void @nc3
-; FNATTRS-SAME: (ptr nocapture readonly [[P:%.*]]) {
+; FNATTRS-SAME: (ptr readonly captures(none) [[P:%.*]]) {
 ; FNATTRS-NEXT:    call void [[P]]()
 ; FNATTRS-NEXT:    ret void
 ;
 ; ATTRIBUTOR-LABEL: define void @nc3
-; ATTRIBUTOR-SAME: (ptr nocapture nofree nonnull [[P:%.*]]) {
+; ATTRIBUTOR-SAME: (ptr nofree nonnull captures(none) [[P:%.*]]) {
 ; ATTRIBUTOR-NEXT:    call void [[P]]()
 ; ATTRIBUTOR-NEXT:    ret void
 ;
@@ -364,14 +364,14 @@ declare void @external(ptr) readonly nounwind
 define void @nc4(ptr %p) {
 ; FNATTRS: Function Attrs: nofree nounwind memory(read)
 ; FNATTRS-LABEL: define void @nc4
-; FNATTRS-SAME: (ptr nocapture readonly [[P:%.*]]) #[[ATTR9:[0-9]+]] {
+; FNATTRS-SAME: (ptr readonly captures(none) [[P:%.*]]) #[[ATTR9:[0-9]+]] {
 ; FNATTRS-NEXT:    call void @external(ptr [[P]])
 ; FNATTRS-NEXT:    ret void
 ;
 ; ATTRIBUTOR: Function Attrs: nosync nounwind memory(read)
 ; ATTRIBUTOR-LABEL: define void @nc4
-; ATTRIBUTOR-SAME: (ptr nocapture readonly [[P:%.*]]) #[[ATTR7:[0-9]+]] {
-; ATTRIBUTOR-NEXT:    call void @external(ptr nocapture readonly [[P]]) #[[ATTR4]]
+; ATTRIBUTOR-SAME: (ptr readonly captures(none) [[P:%.*]]) #[[ATTR7:[0-9]+]] {
+; ATTRIBUTOR-NEXT:    call void @external(ptr readonly captures(none) [[P]]) #[[ATTR4]]
 ; ATTRIBUTOR-NEXT:    ret void
 ;
   call void @external(ptr %p)
@@ -380,15 +380,15 @@ define void @nc4(ptr %p) {
 
 define void @nc5(ptr %f, ptr %p) {
 ; FNATTRS-LABEL: define void @nc5
-; FNATTRS-SAME: (ptr nocapture readonly [[F:%.*]], ptr nocapture [[P:%.*]]) {
+; FNATTRS-SAME: (ptr readonly captures(none) [[F:%.*]], ptr captures(none) [[P:%.*]]) {
 ; FNATTRS-NEXT:    call void [[F]](ptr [[P]]) #[[ATTR8:[0-9]+]]
-; FNATTRS-NEXT:    call void [[F]](ptr nocapture [[P]])
+; FNATTRS-NEXT:    call void [[F]](ptr captures(none) [[P]])
 ; FNATTRS-NEXT:    ret void
 ;
 ; ATTRIBUTOR-LABEL: define void @nc5
-; ATTRIBUTOR-SAME: (ptr nocapture nofree nonnull [[F:%.*]], ptr nocapture [[P:%.*]]) {
+; ATTRIBUTOR-SAME: (ptr nofree nonnull captures(none) [[F:%.*]], ptr captures(none) [[P:%.*]]) {
 ; ATTRIBUTOR-NEXT:    call void [[F]](ptr [[P]]) #[[ATTR6:[0-9]+]]
-; ATTRIBUTOR-NEXT:    call void [[F]](ptr nocapture [[P]])
+; ATTRIBUTOR-NEXT:    call void [[F]](ptr captures(none) [[P]])
 ; ATTRIBUTOR-NEXT:    ret void
 ;
   call void %f(ptr %p) readonly nounwind
@@ -400,15 +400,15 @@ define void @nc5(ptr %f, ptr %p) {
 define void @test1_1(ptr %x1_1, ptr %y1_1, i1 %c) {
 ; FNATTRS: Function Attrs: nofree nosync nounwind memory(write, argmem: none, inaccessiblemem: none)
 ; FNATTRS-LABEL: define void @test1_1
-; FNATTRS-SAME: (ptr nocapture readnone [[X1_1:%.*]], ptr [[Y1_1:%.*]], i1 [[C:%.*]]) #[[ATTR10:[0-9]+]] {
+; FNATTRS-SAME: (ptr readnone captures(none) [[X1_1:%.*]], ptr [[Y1_1:%.*]], i1 [[C:%.*]]) #[[ATTR10:[0-9]+]] {
 ; FNATTRS-NEXT:    [[TMP1:%.*]] = call ptr @test1_2(ptr [[X1_1]], ptr [[Y1_1]], i1 [[C]])
 ; FNATTRS-NEXT:    store ptr null, ptr @g, align 8
 ; FNATTRS-NEXT:    ret void
 ;
 ; ATTRIBUTOR: Function Attrs: nofree nosync nounwind memory(write)
 ; ATTRIBUTOR-LABEL: define void @test1_1
-; ATTRIBUTOR-SAME: (ptr nocapture nofree readnone [[X1_1:%.*]], ptr nocapture nofree readnone [[Y1_1:%.*]], i1 [[C:%.*]]) #[[ATTR8:[0-9]+]] {
-; ATTRIBUTOR-NEXT:    [[TMP1:%.*]] = call ptr @test1_2(ptr nocapture nofree readnone [[X1_1]], ptr nofree readnone [[Y1_1]], i1 [[C]]) #[[ATTR8]]
+; ATTRIBUTOR-SAME: (ptr nofree readnone captures(none) [[X1_1:%.*]], ptr nofree readnone captures(none) [[Y1_1:%.*]], i1 [[C:%.*]]) #[[ATTR8:[0-9]+]] {
+; ATTRIBUTOR-NEXT:    [[TMP1:%.*]] = call ptr @test1_2(ptr nofree readnone captures(none) [[X1_1]], ptr nofree readnone [[Y1_1]], i1 [[C]]) #[[ATTR8]]
 ; ATTRIBUTOR-NEXT:    store ptr null, ptr @g, align 8
 ; ATTRIBUTOR-NEXT:    ret void
 ;
@@ -420,7 +420,7 @@ define void @test1_1(ptr %x1_1, ptr %y1_1, i1 %c) {
 define ptr @test1_2(ptr %x1_2, ptr %y1_2, i1 %c) {
 ; FNATTRS: Function Attrs: nofree nosync nounwind memory(write, argmem: none, inaccessiblemem: none)
 ; FNATTRS-LABEL: define ptr @test1_2
-; FNATTRS-SAME: (ptr nocapture readnone [[X1_2:%.*]], ptr returned [[Y1_2:%.*]], i1 [[C:%.*]]) #[[ATTR10]] {
+; FNATTRS-SAME: (ptr readnone captures(none) [[X1_2:%.*]], ptr returned [[Y1_2:%.*]], i1 [[C:%.*]]) #[[ATTR10]] {
 ; FNATTRS-NEXT:    br i1 [[C]], label [[T:%.*]], label [[F:%.*]]
 ; FNATTRS:       t:
 ; FNATTRS-NEXT:    call void @test1_1(ptr [[X1_2]], ptr [[Y1_2]], i1 [[C]])
@@ -431,10 +431,10 @@ define ptr @test1_2(ptr %x1_2, ptr %y1_2, i1 %c) {
 ;
 ; ATTRIBUTOR: Function Attrs: nofree nosync nounwind memory(write)
 ; ATTRIBUTOR-LABEL: define ptr @test1_2
-; ATTRIBUTOR-SAME: (ptr nocapture nofree readnone [[X1_2:%.*]], ptr nofree readnone [[Y1_2:%.*]], i1 [[C:%.*]]) #[[ATTR8]] {
+; ATTRIBUTOR-SAME: (ptr nofree readnone captures(none) [[X1_2:%.*]], ptr nofree readnone [[Y1_2:%.*]], i1 [[C:%.*]]) #[[ATTR8]] {
 ; ATTRIBUTOR-NEXT:    br i1 [[C]], label [[T:%.*]], label [[F:%.*]]
 ; ATTRIBUTOR:       t:
-; ATTRIBUTOR-NEXT:    call void @test1_1(ptr nocapture nofree readnone [[X1_2]], ptr nocapture nofree readnone [[Y1_2]], i1 [[C]]) #[[ATTR8]]
+; ATTRIBUTOR-NEXT:    call void @test1_1(ptr nofree readnone captures(none) [[X1_2]], ptr nofree readnone captures(none) [[Y1_2]], i1 [[C]]) #[[ATTR8]]
 ; ATTRIBUTOR-NEXT:    store ptr null, ptr @g, align 8
 ; ATTRIBUTOR-NEXT:    br label [[F]]
 ; ATTRIBUTOR:       f:
@@ -452,15 +452,15 @@ f:
 define void @test2(ptr %x2) {
 ; FNATTRS: Function Attrs: nofree nosync nounwind memory(write, argmem: none, inaccessiblemem: none)
 ; FNATTRS-LABEL: define void @test2
-; FNATTRS-SAME: (ptr nocapture readnone [[X2:%.*]]) #[[ATTR10]] {
+; FNATTRS-SAME: (ptr readnone captures(none) [[X2:%.*]]) #[[ATTR10]] {
 ; FNATTRS-NEXT:    call void @test2(ptr [[X2]])
 ; FNATTRS-NEXT:    store ptr null, ptr @g, align 8
 ; FNATTRS-NEXT:    ret void
 ;
 ; ATTRIBUTOR: Function Attrs: nofree nosync nounwind memory(write)
 ; ATTRIBUTOR-LABEL: define void @test2
-; ATTRIBUTOR-SAME: (ptr nocapture nofree readnone [[X2:%.*]]) #[[ATTR8]] {
-; ATTRIBUTOR-NEXT:    call void @test2(ptr nocapture nofree readnone [[X2]]) #[[ATTR8]]
+; ATTRIBUTOR-SAME: (ptr nofree readnone captures(none) [[X2:%.*]]) #[[ATTR8]] {
+; ATTRIBUTOR-NEXT:    call void @test2(ptr nofree readnone captures(none) [[X2]]) #[[ATTR8]]
 ; ATTRIBUTOR-NEXT:    store ptr null, ptr @g, align 8
 ; ATTRIBUTOR-NEXT:    ret void
 ;
@@ -472,15 +472,15 @@ define void @test2(ptr %x2) {
 define void @test3(ptr %x3, ptr %y3, ptr %z3) {
 ; FNATTRS: Function Attrs: nofree nosync nounwind memory(write, argmem: none, inaccessiblemem: none)
 ; FNATTRS-LABEL: define void @test3
-; FNATTRS-SAME: (ptr nocapture readnone [[X3:%.*]], ptr nocapture readnone [[Y3:%.*]], ptr nocapture readnone [[Z3:%.*]]) #[[ATTR10]] {
+; FNATTRS-SAME: (ptr readnone captures(none) [[X3:%.*]], ptr readnone captures(none) [[Y3:%.*]], ptr readnone captures(none) [[Z3:%.*]]) #[[ATTR10]] {
 ; FNATTRS-NEXT:    call void @test3(ptr [[Z3]], ptr [[Y3]], ptr [[X3]])
 ; FNATTRS-NEXT:    store ptr null, ptr @g, align 8
 ; FNATTRS-NEXT:    ret void
 ;
 ; ATTRIBUTOR: Function Attrs: nofree nosync nounwind memory(write)
 ; ATTRIBUTOR-LABEL: define void @test3
-; ATTRIBUTOR-SAME: (ptr nocapture nofree readnone [[X3:%.*]], ptr nocapture nofree readnone [[Y3:%.*]], ptr nocapture nofree readnone [[Z3:%.*]]) #[[ATTR8]] {
-; ATTRIBUTOR-NEXT:    call void @test3(ptr nocapture nofree readnone [[Z3]], ptr nocapture nofree readnone [[Y3]], ptr nocapture nofree readnone [[X3]]) #[[ATTR8]]
+; ATTRIBUTOR-SAME: (ptr nofree readnone captures(none) [[X3:%.*]], ptr nofree readnone captures(none) [[Y3:%.*]], ptr nofree readnone captures(none) [[Z3:%.*]]) #[[ATTR8]] {
+; ATTRIBUTOR-NEXT:    call void @test3(ptr nofree readnone captures(none) [[Z3]], ptr nofree readnone captures(none) [[Y3]], ptr nofree readnone captures(none) [[X3]]) #[[ATTR8]]
 ; ATTRIBUTOR-NEXT:    store ptr null, ptr @g, align 8
 ; ATTRIBUTOR-NEXT:    ret void
 ;
@@ -499,8 +499,8 @@ define void @test4_1(ptr %x4_1, i1 %c) {
 ;
 ; ATTRIBUTOR: Function Attrs: nofree nosync nounwind memory(write)
 ; ATTRIBUTOR-LABEL: define void @test4_1
-; ATTRIBUTOR-SAME: (ptr nocapture nofree readnone [[X4_1:%.*]], i1 [[C:%.*]]) #[[ATTR8]] {
-; ATTRIBUTOR-NEXT:    [[TMP1:%.*]] = call ptr @test4_2(ptr nocapture nofree readnone [[X4_1]], ptr nofree readnone [[X4_1]], ptr nocapture nofree readnone [[X4_1]], i1 [[C]]) #[[ATTR8]]
+; ATTRIBUTOR-SAME: (ptr nofree readnone captures(none) [[X4_1:%.*]], i1 [[C:%.*]]) #[[ATTR8]] {
+; ATTRIBUTOR-NEXT:    [[TMP1:%.*]] = call ptr @test4_2(ptr nofree readnone captures(none) [[X4_1]], ptr nofree readnone [[X4_1]], ptr nofree readnone captures(none) [[X4_1]], i1 [[C]]) #[[ATTR8]]
 ; ATTRIBUTOR-NEXT:    store ptr null, ptr @g, align 8
 ; ATTRIBUTOR-NEXT:    ret void
 ;
@@ -512,7 +512,7 @@ define void @test4_1(ptr %x4_1, i1 %c) {
 define ptr @test4_2(ptr %x4_2, ptr %y4_2, ptr %z4_2, i1 %c) {
 ; FNATTRS: Function Attrs: nofree nosync nounwind memory(write, argmem: none, inaccessiblemem: none)
 ; FNATTRS-LABEL: define ptr @test4_2
-; FNATTRS-SAME: (ptr nocapture readnone [[X4_2:%.*]], ptr readnone returned [[Y4_2:%.*]], ptr nocapture readnone [[Z4_2:%.*]], i1 [[C:%.*]]) #[[ATTR10]] {
+; FNATTRS-SAME: (ptr readnone captures(none) [[X4_2:%.*]], ptr readnone returned [[Y4_2:%.*]], ptr readnone captures(none) [[Z4_2:%.*]], i1 [[C:%.*]]) #[[ATTR10]] {
 ; FNATTRS-NEXT:    br i1 [[C]], label [[T:%.*]], label [[F:%.*]]
 ; FNATTRS:       t:
 ; FNATTRS-NEXT:    call void @test4_1(ptr null, i1 [[C]])
@@ -523,7 +523,7 @@ define ptr @test4_2(ptr %x4_2, ptr %y4_2, ptr %z4_2, i1 %c) {
 ;
 ; ATTRIBUTOR: Function Attrs: nofree nosync nounwind memory(write)
 ; ATTRIBUTOR-LABEL: define ptr @test4_2
-; ATTRIBUTOR-SAME: (ptr nocapture nofree readnone [[X4_2:%.*]], ptr nofree readnone [[Y4_2:%.*]], ptr nocapture nofree readnone [[Z4_2:%.*]], i1 [[C:%.*]]) #[[ATTR8]] {
+; ATTRIBUTOR-SAME: (ptr nofree readnone captures(none) [[X4_2:%.*]], ptr nofree readnone [[Y4_2:%.*]], ptr nofree readnone captures(none) [[Z4_2:%.*]], i1 [[C:%.*]]) #[[ATTR8]] {
 ; ATTRIBUTOR-NEXT:    br i1 [[C]], label [[T:%.*]], label [[F:%.*]]
 ; ATTRIBUTOR:       t:
 ; ATTRIBUTOR-NEXT:    call void @test4_1(ptr nofree readnone null, i1 [[C]]) #[[ATTR8]]
@@ -559,14 +559,14 @@ declare void @test6_1(ptr %x6_1, ptr nocapture %y6_1, ...)
 
 define void @test6_2(ptr %x6_2, ptr %y6_2, ptr %z6_2) {
 ; FNATTRS-LABEL: define void @test6_2
-; FNATTRS-SAME: (ptr [[X6_2:%.*]], ptr nocapture [[Y6_2:%.*]], ptr [[Z6_2:%.*]]) {
+; FNATTRS-SAME: (ptr [[X6_2:%.*]], ptr captures(none) [[Y6_2:%.*]], ptr [[Z6_2:%.*]]) {
 ; FNATTRS-NEXT:    call void (ptr, ptr, ...) @test6_1(ptr [[X6_2]], ptr [[Y6_2]], ptr [[Z6_2]])
 ; FNATTRS-NEXT:    store ptr null, ptr @g, align 8
 ; FNATTRS-NEXT:    ret void
 ;
 ; ATTRIBUTOR-LABEL: define void @test6_2
-; ATTRIBUTOR-SAME: (ptr [[X6_2:%.*]], ptr nocapture [[Y6_2:%.*]], ptr [[Z6_2:%.*]]) {
-; ATTRIBUTOR-NEXT:    call void (ptr, ptr, ...) @test6_1(ptr [[X6_2]], ptr nocapture [[Y6_2]], ptr [[Z6_2]])
+; ATTRIBUTOR-SAME: (ptr [[X6_2:%.*]], ptr captures(none) [[Y6_2:%.*]], ptr [[Z6_2:%.*]]) {
+; ATTRIBUTOR-NEXT:    call void (ptr, ptr, ...) @test6_1(ptr [[X6_2]], ptr captures(none) [[Y6_2]], ptr [[Z6_2]])
 ; ATTRIBUTOR-NEXT:    store ptr null, ptr @g, align 8
 ; ATTRIBUTOR-NEXT:    ret void
 ;
@@ -578,13 +578,13 @@ define void @test6_2(ptr %x6_2, ptr %y6_2, ptr %z6_2) {
 define void @test_cmpxchg(ptr %p) {
 ; FNATTRS: Function Attrs: mustprogress nofree norecurse nounwind willreturn memory(argmem: readwrite)
 ; FNATTRS-LABEL: define void @test_cmpxchg
-; FNATTRS-SAME: (ptr nocapture [[P:%.*]]) #[[ATTR11:[0-9]+]] {
+; FNATTRS-SAME: (ptr captures(none) [[P:%.*]]) #[[ATTR11:[0-9]+]] {
 ; FNATTRS-NEXT:    [[TMP1:%.*]] = cmpxchg ptr [[P]], i32 0, i32 1 acquire monotonic, align 4
 ; FNATTRS-NEXT:    ret void
 ;
 ; ATTRIBUTOR: Function Attrs: mustprogress nofree norecurse nounwind willreturn memory(argmem: readwrite)
 ; ATTRIBUTOR-LABEL: define void @test_cmpxchg
-; ATTRIBUTOR-SAME: (ptr nocapture nofree nonnull [[P:%.*]]) #[[ATTR9:[0-9]+]] {
+; ATTRIBUTOR-SAME: (ptr nofree nonnull captures(none) [[P:%.*]]) #[[ATTR9:[0-9]+]] {
 ; ATTRIBUTOR-NEXT:    [[TMP1:%.*]] = cmpxchg ptr [[P]], i32 0, i32 1 acquire monotonic, align 4
 ; ATTRIBUTOR-NEXT:    ret void
 ;
@@ -595,13 +595,13 @@ define void @test_cmpxchg(ptr %p) {
 define void @test_cmpxchg_ptr(ptr %p, ptr %q) {
 ; FNATTRS: Function Attrs: mustprogress nofree norecurse nounwind willreturn memory(argmem: readwrite)
 ; FNATTRS-LABEL: define void @test_cmpxchg_ptr
-; FNATTRS-SAME: (ptr nocapture [[P:%.*]], ptr [[Q:%.*]]) #[[ATTR11]] {
+; FNATTRS-SAME: (ptr captures(none) [[P:%.*]], ptr [[Q:%.*]]) #[[ATTR11]] {
 ; FNATTRS-NEXT:    [[TMP1:%.*]] = cmpxchg ptr [[P]], ptr null, ptr [[Q]] acquire monotonic, align 8
 ; FNATTRS-NEXT:    ret void
 ;
 ; ATTRIBUTOR: Function Attrs: mustprogress nofree norecurse nounwind willreturn memory(argmem: readwrite)
 ; ATTRIBUTOR-LABEL: define void @test_cmpxchg_ptr
-; ATTRIBUTOR-SAME: (ptr nocapture nofree nonnull [[P:%.*]], ptr nofree [[Q:%.*]]) #[[ATTR9]] {
+; ATTRIBUTOR-SAME: (ptr nofree nonnull captures(none) [[P:%.*]], ptr nofree [[Q:%.*]]) #[[ATTR9]] {
 ; ATTRIBUTOR-NEXT:    [[TMP1:%.*]] = cmpxchg ptr [[P]], ptr null, ptr [[Q]] acquire monotonic, align 8
 ; ATTRIBUTOR-NEXT:    ret void
 ;
@@ -612,13 +612,13 @@ define void @test_cmpxchg_ptr(ptr %p, ptr %q) {
 define void @test_atomicrmw(ptr %p) {
 ; FNATTRS: Function Attrs: mustprogress nofree norecurse nounwind willreturn memory(argmem: readwrite)
 ; FNATTRS-LABEL: define void @test_atomicrmw
-; FNATTRS-SAME: (ptr nocapture [[P:%.*]]) #[[ATTR11]] {
+; FNATTRS-SAME: (ptr captures(none) [[P:%.*]]) #[[ATTR11]] {
 ; FNATTRS-NEXT:    [[TMP1:%.*]] = atomicrmw add ptr [[P]], i32 1 seq_cst, align 4
 ; FNATTRS-NEXT:    ret void
 ;
 ; ATTRIBUTOR: Function Attrs: mustprogress nofree norecurse nounwind willreturn memory(argmem: readwrite)
 ; ATTRIBUTOR-LABEL: define void @test_atomicrmw
-; ATTRIBUTOR-SAME: (ptr nocapture nofree nonnull [[P:%.*]]) #[[ATTR9]] {
+; ATTRIBUTOR-SAME: (ptr nofree nonnull captures(none) [[P:%.*]]) #[[ATTR9]] {
 ; ATTRIBUTOR-NEXT:    [[TMP1:%.*]] = atomicrmw add ptr [[P]], i32 1 seq_cst, align 4
 ; ATTRIBUTOR-NEXT:    ret void
 ;
@@ -652,7 +652,7 @@ entry:
 define void @nocaptureLaunder(ptr %p) {
 ; FNATTRS: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: write, inaccessiblemem: readwrite)
 ; FNATTRS-LABEL: define void @nocaptureLaunder
-; FNATTRS-SAME: (ptr nocapture writeonly [[P:%.*]]) #[[ATTR13:[0-9]+]] {
+; FNATTRS-SAME: (ptr writeonly captures(none) [[P:%.*]]) #[[ATTR13:[0-9]+]] {
 ; FNATTRS-NEXT:  entry:
 ; FNATTRS-NEXT:    [[B:%.*]] = call ptr @llvm.launder.invariant.group.p0(ptr [[P]])
 ; FNATTRS-NEXT:    store i8 42, ptr [[B]], align 1
@@ -660,7 +660,7 @@ define void @nocaptureLaunder(ptr %p) {
 ;
 ; ATTRIBUTOR: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: readwrite, inaccessiblemem: readwrite)
 ; ATTRIBUTOR-LABEL: define void @nocaptureLaunder
-; ATTRIBUTOR-SAME: (ptr nocapture nofree [[P:%.*]]) #[[ATTR10:[0-9]+]] {
+; ATTRIBUTOR-SAME: (ptr nofree captures(none) [[P:%.*]]) #[[ATTR10:[0-9]+]] {
 ; ATTRIBUTOR-NEXT:  entry:
 ; ATTRIBUTOR-NEXT:    [[B:%.*]] = call ptr @llvm.launder.invariant.group.p0(ptr [[P]]) #[[ATTR19:[0-9]+]]
 ; ATTRIBUTOR-NEXT:    store i8 42, ptr [[B]], align 1
@@ -696,7 +696,7 @@ define void @captureLaunder(ptr %p) {
 define void @nocaptureStrip(ptr %p) {
 ; FNATTRS: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: write)
 ; FNATTRS-LABEL: define void @nocaptureStrip
-; FNATTRS-SAME: (ptr nocapture writeonly [[P:%.*]]) #[[ATTR15:[0-9]+]] {
+; FNATTRS-SAME: (ptr writeonly captures(none) [[P:%.*]]) #[[ATTR15:[0-9]+]] {
 ; FNATTRS-NEXT:  entry:
 ; FNATTRS-NEXT:    [[B:%.*]] = call ptr @llvm.strip.invariant.group.p0(ptr [[P]])
 ; FNATTRS-NEXT:    store i8 42, ptr [[B]], align 1
@@ -704,7 +704,7 @@ define void @nocaptureStrip(ptr %p) {
 ;
 ; ATTRIBUTOR: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: write)
 ; ATTRIBUTOR-LABEL: define void @nocaptureStrip
-; ATTRIBUTOR-SAME: (ptr nocapture nofree writeonly [[P:%.*]]) #[[ATTR11:[0-9]+]] {
+; ATTRIBUTOR-SAME: (ptr nofree writeonly captures(none) [[P:%.*]]) #[[ATTR11:[0-9]+]] {
 ; ATTRIBUTOR-NEXT:  entry:
 ; ATTRIBUTOR-NEXT:    [[B:%.*]] = call ptr @llvm.strip.invariant.group.p0(ptr [[P]]) #[[ATTR17]]
 ; ATTRIBUTOR-NEXT:    store i8 42, ptr [[B]], align 1
@@ -814,7 +814,7 @@ define i1 @nocaptureInboundsGEPICmpRev(ptr %x) {
 define i1 @nocaptureDereferenceableOrNullICmp(ptr dereferenceable_or_null(4) %x) {
 ; FNATTRS: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(none)
 ; FNATTRS-LABEL: define noundef i1 @nocaptureDereferenceableOrNullICmp
-; FNATTRS-SAME: (ptr nocapture readnone dereferenceable_or_null(4) [[X:%.*]]) #[[ATTR0]] {
+; FNATTRS-SAME: (ptr readnone captures(none) dereferenceable_or_null(4) [[X:%.*]]) #[[ATTR0]] {
 ; FNATTRS-NEXT:    [[TMP1:%.*]] = icmp eq ptr [[X]], null
 ; FNATTRS-NEXT:    ret i1 [[TMP1]]
 ;
@@ -849,13 +849,13 @@ declare void @capture(ptr)
 
 define void @nocapture_fptr(ptr %f, ptr %p) {
 ; FNATTRS-LABEL: define void @nocapture_fptr
-; FNATTRS-SAME: (ptr nocapture readonly [[F:%.*]], ptr [[P:%.*]]) {
+; FNATTRS-SAME: (ptr readonly captures(none) [[F:%.*]], ptr [[P:%.*]]) {
 ; FNATTRS-NEXT:    [[RES:%.*]] = call ptr [[F]](ptr [[P]])
 ; FNATTRS-NEXT:    call void @capture(ptr [[RES]])
 ; FNATTRS-NEXT:    ret void
 ;
 ; ATTRIBUTOR-LABEL: define void @nocapture_fptr
-; ATTRIBUTOR-SAME: (ptr nocapture nofree nonnull [[F:%.*]], ptr [[P:%.*]]) {
+; ATTRIBUTOR-SAME: (ptr nofree nonnull captures(none) [[F:%.*]], ptr [[P:%.*]]) {
 ; ATTRIBUTOR-NEXT:    [[RES:%.*]] = call ptr [[F]](ptr [[P]])
 ; ATTRIBUTOR-NEXT:    call void @capture(ptr [[RES]])
 ; ATTRIBUTOR-NEXT:    ret void
@@ -867,13 +867,13 @@ define void @nocapture_fptr(ptr %f, ptr %p) {
 
 define void @recurse_fptr(ptr %f, ptr %p) {
 ; FNATTRS-LABEL: define void @recurse_fptr
-; FNATTRS-SAME: (ptr nocapture readonly [[F:%.*]], ptr [[P:%.*]]) {
+; FNATTRS-SAME: (ptr readonly captures(none) [[F:%.*]], ptr [[P:%.*]]) {
 ; FNATTRS-NEXT:    [[RES:%.*]] = call ptr [[F]](ptr [[P]])
 ; FNATTRS-NEXT:    store i8 0, ptr [[RES]], align 1
 ; FNATTRS-NEXT:    ret void
 ;
 ; ATTRIBUTOR-LABEL: define void @recurse_fptr
-; ATTRIBUTOR-SAME: (ptr nocapture nofree nonnull [[F:%.*]], ptr [[P:%.*]]) {
+; ATTRIBUTOR-SAME: (ptr nofree nonnull captures(none) [[F:%.*]], ptr [[P:%.*]]) {
 ; ATTRIBUTOR-NEXT:    [[RES:%.*]] = call ptr [[F]](ptr [[P]])
 ; ATTRIBUTOR-NEXT:    store i8 0, ptr [[RES]], align 1
 ; ATTRIBUTOR-NEXT:    ret void
@@ -886,13 +886,13 @@ define void @recurse_fptr(ptr %f, ptr %p) {
 define void @readnone_indirec(ptr %f, ptr %p) {
 ; FNATTRS: Function Attrs: nofree nosync memory(none)
 ; FNATTRS-LABEL: define void @readnone_indirec
-; FNATTRS-SAME: (ptr nocapture readonly [[F:%.*]], ptr readnone [[P:%.*]]) #[[ATTR17:[0-9]+]] {
+; FNATTRS-SAME: (ptr readonly captures(none) [[F:%.*]], ptr readnone [[P:%.*]]) #[[ATTR17:[0-9]+]] {
 ; FNATTRS-NEXT:    call void [[F]](ptr [[P]]) #[[ATTR20:[0-9]+]]
 ; FNATTRS-NEXT:    ret void
 ;
 ; ATTRIBUTOR: Function Attrs: nosync memory(none)
 ; ATTRIBUTOR-LABEL: define void @readnone_indirec
-; ATTRIBUTOR-SAME: (ptr nocapture nofree nonnull readnone [[F:%.*]], ptr readnone [[P:%.*]]) #[[ATTR13:[0-9]+]] {
+; ATTRIBUTOR-SAME: (ptr nofree nonnull readnone captures(none) [[F:%.*]], ptr readnone [[P:%.*]]) #[[ATTR13:[0-9]+]] {
 ; ATTRIBUTOR-NEXT:    call void [[F]](ptr [[P]]) #[[ATTR20:[0-9]+]]
 ; ATTRIBUTOR-NEXT:    ret void
 ;
@@ -900,6 +900,104 @@ define void @readnone_indirec(ptr %f, ptr %p) {
   ret void
 }
 
+define ptr @captures_ret_only(ptr %p) {
+; FNATTRS: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(none)
+; FNATTRS-LABEL: define ptr @captures_ret_only
+; FNATTRS-SAME: (ptr readnone [[P:%.*]]) #[[ATTR0]] {
+; FNATTRS-NEXT:    [[GEP:%.*]] = getelementptr i8, ptr [[P]], i64 8
+; FNATTRS-NEXT:    ret ptr [[GEP]]
+;
+; ATTRIBUTOR: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(none)
+; ATTRIBUTOR-LABEL: define ptr @captures_ret_only
+; ATTRIBUTOR-SAME: (ptr nofree readnone [[P:%.*]]) #[[ATTR0]] {
+; ATTRIBUTOR-NEXT:    [[GEP:%.*]] = getelementptr i8, ptr [[P]], i64 8
+; ATTRIBUTOR-NEXT:    ret ptr [[GEP]]
+;
+  %gep = getelementptr i8, ptr %p, i64 8
+  ret ptr %gep
+}
+
+define i64 @captures_not_ret_only(ptr %p) {
+; FNATTRS: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(none)
+; FNATTRS-LABEL: define i64 @captures_not_ret_only
+; FNATTRS-SAME: (ptr [[P:%.*]]) #[[ATTR0]] {
+; FNATTRS-NEXT:    [[INT:%.*]] = ptrtoint ptr [[P]] to i64
+; FNATTRS-NEXT:    ret i64 [[INT]]
+;
+; ATTRIBUTOR: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(none)
+; ATTRIBUTOR-LABEL: define i64 @captures_not_ret_only
+; ATTRIBUTOR-SAME: (ptr nofree readnone [[P:%.*]]) #[[ATTR0]] {
+; ATTRIBUTOR-NEXT:    [[INT:%.*]] = ptrtoint ptr [[P]] to i64
+; ATTRIBUTOR-NEXT:    ret i64 [[INT]]
+;
+  %int = ptrtoint ptr %p to i64
+  ret i64 %int
+}
+
+define void @captures_read_provenance(ptr %p) {
+; COMMON-LABEL: define void @captures_read_provenance
+; COMMON-SAME: (ptr [[P:%.*]]) {
+; COMMON-NEXT:    call void @capture(ptr captures(address, read_provenance) [[P]])
+; COMMON-NEXT:    ret void
+;
+  call void @capture(ptr captures(address, read_provenance) %p)
+  ret void
+}
+
+define void @captures_unused_ret(ptr %p) {
+; COMMON-LABEL: define void @captures_unused_ret
+; COMMON-SAME: (ptr [[P:%.*]]) {
+; COMMON-NEXT:    [[TMP1:%.*]] = call ptr @capture(ptr captures(address_is_null, ret: address, read_provenance) [[P]])
+; COMMON-NEXT:    ret void
+;
+  call ptr @capture(ptr captures(address_is_null, ret: address, read_provenance) %p)
+  ret void
+}
+
+define ptr @captures_used_ret(ptr %p) {
+; COMMON-LABEL: define ptr @captures_used_ret
+; COMMON-SAME: (ptr [[P:%.*]]) {
+; COMMON-NEXT:    [[RET:%.*]] = call ptr @capture(ptr captures(address_is_null, ret: address, read_provenance) [[P]])
+; COMMON-NEXT:    ret ptr [[RET]]
+;
+  %ret = call ptr @capture(ptr captures(address_is_null, ret: address, read_provenance) %p)
+  ret ptr %ret
+}
+
+define ptr @scc_capture_via_ret(i1 %c, ptr %p) {
+; FNATTRS: Function Attrs: nofree nosync nounwind memory(write, argmem: none, inaccessiblemem: none)
+; FNATTRS-LABEL: define ptr @scc_capture_via_ret
+; FNATTRS-SAME: (i1 [[C:%.*]], ptr [[P:%.*]]) #[[ATTR10]] {
+; FNATTRS-NEXT:    br i1 [[C]], label [[IF:%.*]], label [[ELSE:%.*]]
+; FNATTRS:       if:
+; FNATTRS-NEXT:    [[C_NOT:%.*]] = xor i1 [[C]], true
+; FNATTRS-NEXT:    [[RET:%.*]] = call ptr @scc_capture_via_ret(i1 [[C_NOT]], ptr [[P]])
+; FNATTRS-NEXT:    store ptr [[RET]], ptr @g, align 8
+; FNATTRS-NEXT:    ret ptr [[RET]]
+; FNATTRS:       else:
+; FNATTRS-NEXT:    ret ptr [[P]]
+;
+; ATTRIBUTOR: Function Attrs: nofree nosync nounwind memory(write)
+; ATTRIBUTOR-LABEL: define ptr @scc_capture_via_ret
+; ATTRIBUTOR-SAME: (i1 [[C:%.*]], ptr nofree [[P:%.*]]) #[[ATTR8]] {
+; ATTRIBUTOR-NEXT:    br i1 [[C]], label [[IF:%.*]], label [[ELSE:%.*]]
+; ATTRIBUTOR:       if:
+; ATTRIBUTOR-NEXT:    [[C_NOT:%.*]] = xor i1 [[C]], true
+; ATTRIBUTOR-NEXT:    [[RET:%.*]] = call ptr @scc_capture_via_ret(i1 [[C_NOT]], ptr nofree [[P]]) #[[ATTR8]]
+; ATTRIBUTOR-NEXT:    store ptr [[RET]], ptr @g, align 8
+; ATTRIBUTOR-NEXT:    ret ptr [[RET]]
+; ATTRIBUTOR:       else:
+; ATTRIBUTOR-NEXT:    ret ptr [[P]]
+;
+  br i1 %c, label %if, label %else
+if:
+  %c.not = xor i1 %c, true
+  %ret = call ptr @scc_capture_via_ret(i1 %c.not, ptr %p)
+  store ptr %ret, ptr @g
+  ret ptr %ret
+else:
+  ret ptr %p
+}
 
 declare ptr @llvm.launder.invariant.group.p0(ptr)
 declare ptr @llvm.strip.invariant.group.p0(ptr)
diff --git a/llvm/test/Transforms/FunctionAttrs/nofree-attributor.ll b/llvm/test/Transforms/FunctionAttrs/nofree-attributor.ll
index ed5534a24cbe8..5a0b6ef1e3043 100644
--- a/llvm/test/Transforms/FunctionAttrs/nofree-attributor.ll
+++ b/llvm/test/Transforms/FunctionAttrs/nofree-attributor.ll
@@ -32,7 +32,7 @@ define void @only_return() #0 {
 define void @only_free(ptr nocapture %0) local_unnamed_addr #0 {
 ; FNATTR: Function Attrs: noinline nounwind uwtable
 ; FNATTR-LABEL: define {{[^@]+}}@only_free
-; FNATTR-SAME: (ptr nocapture [[TMP0:%.*]]) local_unnamed_addr #[[ATTR1:[0-9]+]] {
+; FNATTR-SAME: (ptr captures(none) [[TMP0:%.*]]) local_unnamed_addr #[[ATTR1:[0-9]+]] {
 ; FNATTR-NEXT:    tail call void @free(ptr [[TMP0]]) #[[ATTR0:[0-9]+]]
 ; FNATTR-NEXT:    ret void
 ;
@@ -54,7 +54,7 @@ define void @only_free(ptr nocapture %0) local_unnamed_addr #0 {
 define void @free_in_scc1(ptr nocapture %0) local_unnamed_addr #0 {
 ; FNATTR: Function Attrs: noinline nounwind uwtable
 ; FNATTR-LABEL: define {{[^@]+}}@free_in_scc1
-; FNATTR-SAME: (ptr nocapture [[TMP0:%.*]]) local_unnamed_addr #[[ATTR1]] {
+; FNATTR-SAME: (ptr captures(none) [[TMP0:%.*]]) local_unnamed_addr #[[ATTR1]] {
 ; FNATTR-NEXT:    tail call void @free_in_scc2(ptr [[TMP0]]) #[[ATTR0]]
 ; FNATTR-NEXT:    ret void
 ;
@@ -65,7 +65,7 @@ define void @free_in_scc1(ptr nocapture %0) local_unnamed_addr #0 {
 define void @free_in_scc2(ptr nocapture %0) local_unnamed_addr #0 {
 ; FNATTR: Function Attrs: noinline nounwind uwtable
 ; FNATTR-LABEL: define {{[^@]+}}@free_in_scc2
-; FNATTR-SAME: (ptr nocapture [[TMP0:%.*]]) local_unnamed_addr #[[ATTR1]] {
+; FNATTR-SAME: (ptr captures(none) [[TMP0:%.*]]) local_unnamed_addr #[[ATTR1]] {
 ; FNATTR-NEXT:    [[CMP:%.*]] = icmp eq ptr [[TMP0]], null
 ; FNATTR-NEXT:    br i1 [[CMP]], label [[REC:%.*]], label [[CALL:%.*]]
 ; FNATTR:       call:
@@ -158,7 +158,7 @@ define void @_Z9delete_opPc(ptr %0) local_unnamed_addr #0 {
 define noalias ptr @call_realloc(ptr nocapture %0, i64 %1) local_unnamed_addr #0 {
 ; FNATTR: Function Attrs: noinline nounwind uwtable
 ; FNATTR-LABEL: define {{[^@]+}}@call_realloc
-; FNATTR-SAME: (ptr nocapture [[TMP0:%.*]], i64 [[TMP1:%.*]]) local_unnamed_addr #[[ATTR1]] {
+; FNATTR-SAME: (ptr captures(none) [[TMP0:%.*]], i64 [[TMP1:%.*]]) local_unnamed_addr #[[ATTR1]] {
 ; FNATTR-NEXT:    [[RET:%.*]] = tail call ptr @realloc(ptr [[TMP0]], i64 [[TMP1]]) #[[ATTR2]]
 ; FNATTR-NEXT:    ret ptr [[RET]]
 ;
diff --git a/llvm/test/Transforms/FunctionAttrs/nonnull.ll b/llvm/test/Transforms/FunctionAttrs/nonnull.ll
index 6dd2399093b66..0f6762f0d4342 100644
--- a/llvm/test/Transforms/FunctionAttrs/nonnull.ll
+++ b/llvm/test/Transforms/FunctionAttrs/nonnull.ll
@@ -323,8 +323,8 @@ define void @test13_helper() {
 ; ATTRIBUTOR-LABEL: define void @test13_helper() {
 ; ATTRIBUTOR-NEXT:    [[NONNULLPTR:%.*]] = tail call ptr @ret_nonnull()
 ; ATTRIBUTOR-NEXT:    [[MAYBENULLPTR:%.*]] = tail call ptr @unknown()
-; ATTRIBUTOR-NEXT:    tail call void @test13(ptr nocapture nofree nonnull readnone [[NONNULLPTR]], ptr nocapture nofree nonnull readnone [[NONNULLPTR]], ptr nocapture nofree readnone [[MAYBENULLPTR]])
-; ATTRIBUTOR-NEXT:    tail call void @test13(ptr nocapture nofree nonnull readnone [[NONNULLPTR]], ptr nocapture nofree readnone [[MAYBENULLPTR]], ptr nocapture nofree nonnull readnone [[NONNULLPTR]])
+; ATTRIBUTOR-NEXT:    tail call void @test13(ptr nofree nonnull readnone captures(none) [[NONNULLPTR]], ptr nofree nonnull readnone captures(none) [[NONNULLPTR]], ptr nofree readnone captures(none) [[MAYBENULLPTR]])
+; ATTRIBUTOR-NEXT:    tail call void @test13(ptr nofree nonnull readnone captures(none) [[NONNULLPTR]], ptr nofree readnone captures(none) [[MAYBENULLPTR]], ptr nofree nonnull readnone captures(none) [[NONNULLPTR]])
 ; ATTRIBUTOR-NEXT:    ret void
 ;
   %nonnullptr = tail call ptr @ret_nonnull()
@@ -335,11 +335,11 @@ define void @test13_helper() {
 }
 define internal void @test13(ptr %a, ptr %b, ptr %c) {
 ; FNATTRS-LABEL: define internal void @test13(
-; FNATTRS-SAME: ptr nocapture readnone [[A:%.*]], ptr nocapture readnone [[B:%.*]], ptr nocapture readnone [[C:%.*]]) #[[ATTR0]] {
+; FNATTRS-SAME: ptr readnone captures(none) [[A:%.*]], ptr readnone captures(none) [[B:%.*]], ptr readnone captures(none) [[C:%.*]]) #[[ATTR0]] {
 ; FNATTRS-NEXT:    ret void
 ;
 ; ATTRIBUTOR-LABEL: define internal void @test13(
-; ATTRIBUTOR-SAME: ptr nocapture nofree nonnull readnone [[A:%.*]], ptr nocapture nofree readnone [[B:%.*]], ptr nocapture nofree readnone [[C:%.*]]) #[[ATTR0]] {
+; ATTRIBUTOR-SAME: ptr nofree nonnull readnone captures(none) [[A:%.*]], ptr nofree readnone captures(none) [[B:%.*]], ptr nofree readnone captures(none) [[C:%.*]]) #[[ATTR0]] {
 ; ATTRIBUTOR-NEXT:    ret void
 ;
   ret void
@@ -903,7 +903,7 @@ declare i32 @esfp(...)
 
 define i1 @parent8(ptr %a, ptr %bogus1, ptr %b) personality ptr @esfp{
 ; FNATTRS-LABEL: define noundef i1 @parent8(
-; FNATTRS-SAME: ptr nonnull [[A:%.*]], ptr nocapture readnone [[BOGUS1:%.*]], ptr nonnull [[B:%.*]]) #[[ATTR7]] personality ptr @esfp {
+; FNATTRS-SAME: ptr nonnull [[A:%.*]], ptr readnone captures(none) [[BOGUS1:%.*]], ptr nonnull [[B:%.*]]) #[[ATTR7]] personality ptr @esfp {
 ; FNATTRS-NEXT:  entry:
 ; FNATTRS-NEXT:    invoke void @use2nonnull(ptr [[A]], ptr [[B]])
 ; FNATTRS-NEXT:            to label [[CONT:%.*]] unwind label [[EXC:%.*]]
@@ -916,7 +916,7 @@ define i1 @parent8(ptr %a, ptr %bogus1, ptr %b) personality ptr @esfp{
 ; FNATTRS-NEXT:    unreachable
 ;
 ; ATTRIBUTOR-LABEL: define i1 @parent8(
-; ATTRIBUTOR-SAME: ptr nonnull [[A:%.*]], ptr nocapture nofree readnone [[BOGUS1:%.*]], ptr nonnull [[B:%.*]]) #[[ATTR7]] personality ptr @esfp {
+; ATTRIBUTOR-SAME: ptr nonnull [[A:%.*]], ptr nofree readnone captures(none) [[BOGUS1:%.*]], ptr nonnull [[B:%.*]]) #[[ATTR7]] personality ptr @esfp {
 ; ATTRIBUTOR-NEXT:  entry:
 ; ATTRIBUTOR-NEXT:    invoke void @use2nonnull(ptr nonnull [[A]], ptr nonnull [[B]])
 ; ATTRIBUTOR-NEXT:            to label [[CONT:%.*]] unwind label [[EXC:%.*]]
@@ -1032,12 +1032,12 @@ define  ptr @g1() {
 declare void @use_i32_ptr(ptr) readnone nounwind
 define internal void @called_by_weak(ptr %a) {
 ; FNATTRS-LABEL: define internal void @called_by_weak(
-; FNATTRS-SAME: ptr nocapture readnone [[A:%.*]]) #[[ATTR1]] {
+; FNATTRS-SAME: ptr readnone captures(none) [[A:%.*]]) #[[ATTR1]] {
 ; FNATTRS-NEXT:    call void @use_i32_ptr(ptr [[A]])
 ; FNATTRS-NEXT:    ret void
 ;
 ; ATTRIBUTOR-LABEL: define internal void @called_by_weak(
-; ATTRIBUTOR-SAME: ptr nocapture nonnull readnone [[A:%.*]]) #[[ATTR10:[0-9]+]] {
+; ATTRIBUTOR-SAME: ptr nonnull readnone captures(none) [[A:%.*]]) #[[ATTR10:[0-9]+]] {
 ; ATTRIBUTOR-NEXT:    call void @use_i32_ptr(ptr nonnull [[A]]) #[[ATTR17:[0-9]+]]
 ; ATTRIBUTOR-NEXT:    ret void
 ;
@@ -1054,7 +1054,7 @@ define weak_odr void @weak_caller(ptr nonnull %a) {
 ;
 ; ATTRIBUTOR-LABEL: define weak_odr void @weak_caller(
 ; ATTRIBUTOR-SAME: ptr nonnull [[A:%.*]]) {
-; ATTRIBUTOR-NEXT:    call void @called_by_weak(ptr nocapture nonnull readnone [[A]])
+; ATTRIBUTOR-NEXT:    call void @called_by_weak(ptr nonnull readnone captures(none) [[A]])
 ; ATTRIBUTOR-NEXT:    ret void
 ;
   call void @called_by_weak(ptr %a)
@@ -1064,12 +1064,12 @@ define weak_odr void @weak_caller(ptr nonnull %a) {
 ; Expect nonnull
 define internal void @control(ptr dereferenceable(4) %a) {
 ; FNATTRS-LABEL: define internal void @control(
-; FNATTRS-SAME: ptr nocapture readnone dereferenceable(4) [[A:%.*]]) #[[ATTR1]] {
+; FNATTRS-SAME: ptr readnone captures(none) dereferenceable(4) [[A:%.*]]) #[[ATTR1]] {
 ; FNATTRS-NEXT:    call void @use_i32_ptr(ptr [[A]])
 ; FNATTRS-NEXT:    ret void
 ;
 ; ATTRIBUTOR-LABEL: define internal void @control(
-; ATTRIBUTOR-SAME: ptr nocapture nonnull readnone dereferenceable(4) [[A:%.*]]) #[[ATTR10]] {
+; ATTRIBUTOR-SAME: ptr nonnull readnone captures(none) dereferenceable(4) [[A:%.*]]) #[[ATTR10]] {
 ; ATTRIBUTOR-NEXT:    call void @use_i32_ptr(ptr [[A]]) #[[ATTR17]]
 ; ATTRIBUTOR-NEXT:    ret void
 ;
@@ -1114,7 +1114,7 @@ define void @make_live(ptr nonnull dereferenceable(8) %a) {
 ; ATTRIBUTOR-LABEL: define void @make_live(
 ; ATTRIBUTOR-SAME: ptr nonnull dereferenceable(8) [[A:%.*]]) {
 ; ATTRIBUTOR-NEXT:    call void @naked(ptr nonnull align 16 dereferenceable(8) [[A]])
-; ATTRIBUTOR-NEXT:    call void @control(ptr nocapture nonnull readnone align 16 dereferenceable(8) [[A]])
+; ATTRIBUTOR-NEXT:    call void @control(ptr nonnull readnone align 16 captures(none) dereferenceable(8) [[A]])
 ; ATTRIBUTOR-NEXT:    call void @optnone(ptr nonnull align 16 dereferenceable(8) [[A]])
 ; ATTRIBUTOR-NEXT:    ret void
 ;
diff --git a/llvm/test/Transforms/FunctionAttrs/norecurse.ll b/llvm/test/Transforms/FunctionAttrs/norecurse.ll
index a902974fed282..7a089f69df12c 100644
--- a/llvm/test/Transforms/FunctionAttrs/norecurse.ll
+++ b/llvm/test/Transforms/FunctionAttrs/norecurse.ll
@@ -90,14 +90,14 @@ declare i32 @k() readnone
 define void @intrinsic(ptr %dest, ptr %src, i32 %len) {
 ; FNATTRS: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: readwrite)
 ; FNATTRS-LABEL: define {{[^@]+}}@intrinsic
-; FNATTRS-SAME: (ptr nocapture writeonly [[DEST:%.*]], ptr nocapture readonly [[SRC:%.*]], i32 [[LEN:%.*]]) #[[ATTR4:[0-9]+]] {
+; FNATTRS-SAME: (ptr writeonly captures(none) [[DEST:%.*]], ptr readonly captures(none) [[SRC:%.*]], i32 [[LEN:%.*]]) #[[ATTR4:[0-9]+]] {
 ; FNATTRS-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr [[DEST]], ptr [[SRC]], i32 [[LEN]], i1 false)
 ; FNATTRS-NEXT:    ret void
 ;
 ; ATTRIBUTOR: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: readwrite)
 ; ATTRIBUTOR-LABEL: define {{[^@]+}}@intrinsic
-; ATTRIBUTOR-SAME: (ptr nocapture nofree writeonly [[DEST:%.*]], ptr nocapture nofree readonly [[SRC:%.*]], i32 [[LEN:%.*]]) #[[ATTR4:[0-9]+]] {
-; ATTRIBUTOR-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr nocapture writeonly [[DEST]], ptr nocapture readonly [[SRC]], i32 [[LEN]], i1 false) #[[ATTR8:[0-9]+]]
+; ATTRIBUTOR-SAME: (ptr nofree writeonly captures(none) [[DEST:%.*]], ptr nofree readonly captures(none) [[SRC:%.*]], i32 [[LEN:%.*]]) #[[ATTR4:[0-9]+]] {
+; ATTRIBUTOR-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr writeonly captures(none) [[DEST]], ptr readonly captures(none) [[SRC]], i32 [[LEN]], i1 false) #[[ATTR8:[0-9]+]]
 ; ATTRIBUTOR-NEXT:    ret void
 ;
   call void @llvm.memcpy.p0.p0.i32(ptr %dest, ptr %src, i32 %len, i1 false)
@@ -194,13 +194,13 @@ define void @p() norecurse {
 define internal i32 @escapes_as_parameter(ptr %p) {
 ; FNATTRS: Function Attrs: nofree nosync memory(none)
 ; FNATTRS-LABEL: define {{[^@]+}}@escapes_as_parameter
-; FNATTRS-SAME: (ptr nocapture readnone [[P:%.*]]) #[[ATTR2]] {
+; FNATTRS-SAME: (ptr readnone captures(none) [[P:%.*]]) #[[ATTR2]] {
 ; FNATTRS-NEXT:    [[A:%.*]] = call i32 @k()
 ; FNATTRS-NEXT:    ret i32 [[A]]
 ;
 ; ATTRIBUTOR: Function Attrs: nosync memory(none)
 ; ATTRIBUTOR-LABEL: define {{[^@]+}}@escapes_as_parameter
-; ATTRIBUTOR-SAME: (ptr nocapture nofree readnone [[P:%.*]]) #[[ATTR2]] {
+; ATTRIBUTOR-SAME: (ptr nofree readnone captures(none) [[P:%.*]]) #[[ATTR2]] {
 ; ATTRIBUTOR-NEXT:    [[A:%.*]] = call i32 @k() #[[ATTR7]]
 ; ATTRIBUTOR-NEXT:    ret i32 [[A]]
 ;
@@ -218,7 +218,7 @@ define internal void @q() {
 ; ATTRIBUTOR: Function Attrs: norecurse nosync memory(none)
 ; ATTRIBUTOR-LABEL: define {{[^@]+}}@q
 ; ATTRIBUTOR-SAME: () #[[ATTR6]] {
-; ATTRIBUTOR-NEXT:    [[A:%.*]] = call i32 @escapes_as_parameter(ptr nocapture nofree nonnull readnone @escapes_as_parameter) #[[ATTR2]]
+; ATTRIBUTOR-NEXT:    [[A:%.*]] = call i32 @escapes_as_parameter(ptr nofree nonnull readnone captures(none) @escapes_as_parameter) #[[ATTR2]]
 ; ATTRIBUTOR-NEXT:    ret void
 ;
   %a = call i32 @escapes_as_parameter(ptr @escapes_as_parameter)
diff --git a/llvm/test/Transforms/FunctionAttrs/optnone.ll b/llvm/test/Transforms/FunctionAttrs/optnone.ll
index aab1ec1d29548..9b5054c4eff96 100644
--- a/llvm/test/Transforms/FunctionAttrs/optnone.ll
+++ b/llvm/test/Transforms/FunctionAttrs/optnone.ll
@@ -4,7 +4,7 @@
 
 define void @test_opt(ptr %p) {
 ; CHECK-LABEL: @test_opt
-; CHECK: (ptr nocapture readnone %p) #0 {
+; CHECK: (ptr readnone captures(none) %p) #0 {
   ret void
 }
 
diff --git a/llvm/test/Transforms/FunctionAttrs/out-of-bounds-iterator-bug.ll b/llvm/test/Transforms/FunctionAttrs/out-of-bounds-iterator-bug.ll
index 6209ddc8317e8..fd98b71cb5562 100644
--- a/llvm/test/Transforms/FunctionAttrs/out-of-bounds-iterator-bug.ll
+++ b/llvm/test/Transforms/FunctionAttrs/out-of-bounds-iterator-bug.ll
@@ -7,7 +7,7 @@ declare void @llvm.va_start(ptr)
 declare void @llvm.va_end(ptr)
 
 define void @va_func(ptr readonly %b, ...) readonly nounwind {
-; CHECK-LABEL: define void @va_func(ptr nocapture readonly %b, ...)
+; CHECK-LABEL: define void @va_func(ptr readonly captures(none) %b, ...)
  entry:
   %valist = alloca i8
   call void @llvm.va_start(ptr %valist)
@@ -17,14 +17,14 @@ define void @va_func(ptr readonly %b, ...) readonly nounwind {
 }
 
 define i32 @caller(ptr %x) {
-; CHECK-LABEL: define noundef i32 @caller(ptr nocapture readonly %x)
+; CHECK-LABEL: define noundef i32 @caller(ptr readonly captures(none) %x)
  entry:
   call void(ptr,...) @va_func(ptr null, i32 0, i32 0, i32 0, ptr %x)
   ret i32 42
 }
 
 define void @va_func2(ptr readonly %b, ...) {
-; CHECK-LABEL: define void @va_func2(ptr nocapture readonly %b, ...)
+; CHECK-LABEL: define void @va_func2(ptr readonly captures(none) %b, ...)
  entry:
   %valist = alloca i8
   call void @llvm.va_start(ptr %valist)
@@ -34,7 +34,7 @@ define void @va_func2(ptr readonly %b, ...) {
 }
 
 define i32 @caller2(ptr %x, ptr %y) {
-; CHECK-LABEL: define noundef i32 @caller2(ptr nocapture readonly %x, ptr %y)
+; CHECK-LABEL: define noundef i32 @caller2(ptr readonly captures(none) %x, ptr %y)
  entry:
   call void(ptr,...) @va_func2(ptr %x, i32 0, i32 0, i32 0, ptr %y)
   ret i32 42
diff --git a/llvm/test/Transforms/FunctionAttrs/readattrs.ll b/llvm/test/Transforms/FunctionAttrs/readattrs.ll
index e60954c9cd29a..b24c097ad54d0 100644
--- a/llvm/test/Transforms/FunctionAttrs/readattrs.ll
+++ b/llvm/test/Transforms/FunctionAttrs/readattrs.ll
@@ -9,20 +9,20 @@ declare void @test1_1(ptr %x1_1, ptr nocapture readonly %y1_1, ...)
 
 define void @test1_2(ptr %x1_2, ptr %y1_2, ptr %z1_2) {
 ; FNATTRS-LABEL: define {{[^@]+}}@test1_2
-; FNATTRS-SAME: (ptr [[X1_2:%.*]], ptr nocapture readonly [[Y1_2:%.*]], ptr [[Z1_2:%.*]]) {
+; FNATTRS-SAME: (ptr [[X1_2:%.*]], ptr readonly captures(none) [[Y1_2:%.*]], ptr [[Z1_2:%.*]]) {
 ; FNATTRS-NEXT:    call void (ptr, ptr, ...) @test1_1(ptr [[X1_2]], ptr [[Y1_2]], ptr [[Z1_2]])
 ; FNATTRS-NEXT:    store i32 0, ptr @x, align 4
 ; FNATTRS-NEXT:    ret void
 ;
 ; ATTRIBUTOR-LABEL: define {{[^@]+}}@test1_2
-; ATTRIBUTOR-SAME: (ptr [[X1_2:%.*]], ptr nocapture nofree readonly [[Y1_2:%.*]], ptr [[Z1_2:%.*]]) {
-; ATTRIBUTOR-NEXT:    call void (ptr, ptr, ...) @test1_1(ptr [[X1_2]], ptr nocapture nofree readonly [[Y1_2]], ptr [[Z1_2]])
+; ATTRIBUTOR-SAME: (ptr [[X1_2:%.*]], ptr nofree readonly captures(none) [[Y1_2:%.*]], ptr [[Z1_2:%.*]]) {
+; ATTRIBUTOR-NEXT:    call void (ptr, ptr, ...) @test1_1(ptr [[X1_2]], ptr nofree readonly captures(none) [[Y1_2]], ptr [[Z1_2]])
 ; ATTRIBUTOR-NEXT:    store i32 0, ptr @x, align 4
 ; ATTRIBUTOR-NEXT:    ret void
 ;
 ; ATTRIBUTOR-CGSCC-LABEL: define {{[^@]+}}@test1_2
-; ATTRIBUTOR-CGSCC-SAME: (ptr [[X1_2:%.*]], ptr nocapture nofree readonly [[Y1_2:%.*]], ptr [[Z1_2:%.*]]) {
-; ATTRIBUTOR-CGSCC-NEXT:    call void (ptr, ptr, ...) @test1_1(ptr [[X1_2]], ptr nocapture nofree readonly [[Y1_2]], ptr [[Z1_2]])
+; ATTRIBUTOR-CGSCC-SAME: (ptr [[X1_2:%.*]], ptr nofree readonly captures(none) [[Y1_2:%.*]], ptr [[Z1_2:%.*]]) {
+; ATTRIBUTOR-CGSCC-NEXT:    call void (ptr, ptr, ...) @test1_1(ptr [[X1_2]], ptr nofree readonly captures(none) [[Y1_2]], ptr [[Z1_2]])
 ; ATTRIBUTOR-CGSCC-NEXT:    store i32 0, ptr @x, align 4
 ; ATTRIBUTOR-CGSCC-NEXT:    ret void
 ;
@@ -83,20 +83,20 @@ declare void @test4_1(ptr nocapture) readonly
 define void @test4_2(ptr %p) {
 ; FNATTRS: Function Attrs: nofree memory(read)
 ; FNATTRS-LABEL: define {{[^@]+}}@test4_2
-; FNATTRS-SAME: (ptr nocapture readonly [[P:%.*]]) #[[ATTR3:[0-9]+]] {
+; FNATTRS-SAME: (ptr readonly captures(none) [[P:%.*]]) #[[ATTR3:[0-9]+]] {
 ; FNATTRS-NEXT:    call void @test4_1(ptr [[P]])
 ; FNATTRS-NEXT:    ret void
 ;
 ; ATTRIBUTOR: Function Attrs: nosync memory(read)
 ; ATTRIBUTOR-LABEL: define {{[^@]+}}@test4_2
-; ATTRIBUTOR-SAME: (ptr nocapture readonly [[P:%.*]]) #[[ATTR3:[0-9]+]] {
-; ATTRIBUTOR-NEXT:    call void @test4_1(ptr nocapture readonly [[P]]) #[[ATTR3]]
+; ATTRIBUTOR-SAME: (ptr readonly captures(none) [[P:%.*]]) #[[ATTR3:[0-9]+]] {
+; ATTRIBUTOR-NEXT:    call void @test4_1(ptr readonly captures(none) [[P]]) #[[ATTR3]]
 ; ATTRIBUTOR-NEXT:    ret void
 ;
 ; ATTRIBUTOR-CGSCC: Function Attrs: nosync memory(read)
 ; ATTRIBUTOR-CGSCC-LABEL: define {{[^@]+}}@test4_2
-; ATTRIBUTOR-CGSCC-SAME: (ptr nocapture readonly [[P:%.*]]) #[[ATTR3:[0-9]+]] {
-; ATTRIBUTOR-CGSCC-NEXT:    call void @test4_1(ptr nocapture readonly [[P]]) #[[ATTR3]]
+; ATTRIBUTOR-CGSCC-SAME: (ptr readonly captures(none) [[P:%.*]]) #[[ATTR3:[0-9]+]] {
+; ATTRIBUTOR-CGSCC-NEXT:    call void @test4_1(ptr readonly captures(none) [[P]]) #[[ATTR3]]
 ; ATTRIBUTOR-CGSCC-NEXT:    ret void
 ;
   call void @test4_1(ptr %p)
@@ -107,19 +107,19 @@ define void @test4_2(ptr %p) {
 define void @test5(ptr %p, ptr %q) {
 ; FNATTRS: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: write)
 ; FNATTRS-LABEL: define {{[^@]+}}@test5
-; FNATTRS-SAME: (ptr nocapture writeonly initializes((0, 8)) [[P:%.*]], ptr [[Q:%.*]]) #[[ATTR4:[0-9]+]] {
+; FNATTRS-SAME: (ptr writeonly captures(none) initializes((0, 8)) [[P:%.*]], ptr [[Q:%.*]]) #[[ATTR4:[0-9]+]] {
 ; FNATTRS-NEXT:    store ptr [[Q]], ptr [[P]], align 8
 ; FNATTRS-NEXT:    ret void
 ;
 ; ATTRIBUTOR: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: write)
 ; ATTRIBUTOR-LABEL: define {{[^@]+}}@test5
-; ATTRIBUTOR-SAME: (ptr nocapture nofree nonnull writeonly [[P:%.*]], ptr nofree writeonly [[Q:%.*]]) #[[ATTR4:[0-9]+]] {
+; ATTRIBUTOR-SAME: (ptr nofree nonnull writeonly captures(none) [[P:%.*]], ptr nofree writeonly [[Q:%.*]]) #[[ATTR4:[0-9]+]] {
 ; ATTRIBUTOR-NEXT:    store ptr [[Q]], ptr [[P]], align 8
 ; ATTRIBUTOR-NEXT:    ret void
 ;
 ; ATTRIBUTOR-CGSCC: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: write)
 ; ATTRIBUTOR-CGSCC-LABEL: define {{[^@]+}}@test5
-; ATTRIBUTOR-CGSCC-SAME: (ptr nocapture nofree nonnull writeonly [[P:%.*]], ptr nofree writeonly [[Q:%.*]]) #[[ATTR4:[0-9]+]] {
+; ATTRIBUTOR-CGSCC-SAME: (ptr nofree nonnull writeonly captures(none) [[P:%.*]], ptr nofree writeonly [[Q:%.*]]) #[[ATTR4:[0-9]+]] {
 ; ATTRIBUTOR-CGSCC-NEXT:    store ptr [[Q]], ptr [[P]], align 8
 ; ATTRIBUTOR-CGSCC-NEXT:    ret void
 ;
@@ -132,19 +132,19 @@ declare void @test6_1()
 ; This is not a missed optz'n.
 define void @test6_2(ptr %p, ptr %q) {
 ; FNATTRS-LABEL: define {{[^@]+}}@test6_2
-; FNATTRS-SAME: (ptr nocapture writeonly initializes((0, 8)) [[P:%.*]], ptr [[Q:%.*]]) {
+; FNATTRS-SAME: (ptr writeonly captures(none) initializes((0, 8)) [[P:%.*]], ptr [[Q:%.*]]) {
 ; FNATTRS-NEXT:    store ptr [[Q]], ptr [[P]], align 8
 ; FNATTRS-NEXT:    call void @test6_1()
 ; FNATTRS-NEXT:    ret void
 ;
 ; ATTRIBUTOR-LABEL: define {{[^@]+}}@test6_2
-; ATTRIBUTOR-SAME: (ptr nocapture nofree nonnull writeonly [[P:%.*]], ptr nofree [[Q:%.*]]) {
+; ATTRIBUTOR-SAME: (ptr nofree nonnull writeonly captures(none) [[P:%.*]], ptr nofree [[Q:%.*]]) {
 ; ATTRIBUTOR-NEXT:    store ptr [[Q]], ptr [[P]], align 8
 ; ATTRIBUTOR-NEXT:    call void @test6_1()
 ; ATTRIBUTOR-NEXT:    ret void
 ;
 ; ATTRIBUTOR-CGSCC-LABEL: define {{[^@]+}}@test6_2
-; ATTRIBUTOR-CGSCC-SAME: (ptr nocapture nofree nonnull writeonly [[P:%.*]], ptr nofree [[Q:%.*]]) {
+; ATTRIBUTOR-CGSCC-SAME: (ptr nofree nonnull writeonly captures(none) [[P:%.*]], ptr nofree [[Q:%.*]]) {
 ; ATTRIBUTOR-CGSCC-NEXT:    store ptr [[Q]], ptr [[P]], align 8
 ; ATTRIBUTOR-CGSCC-NEXT:    call void @test6_1()
 ; ATTRIBUTOR-CGSCC-NEXT:    ret void
@@ -158,17 +158,17 @@ define void @test6_2(ptr %p, ptr %q) {
 define void @test7_1(ptr inalloca(i32) %a) {
 ; FNATTRS: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: readwrite)
 ; FNATTRS-LABEL: define {{[^@]+}}@test7_1
-; FNATTRS-SAME: (ptr nocapture inalloca(i32) [[A:%.*]]) #[[ATTR5:[0-9]+]] {
+; FNATTRS-SAME: (ptr inalloca(i32) captures(none) [[A:%.*]]) #[[ATTR5:[0-9]+]] {
 ; FNATTRS-NEXT:    ret void
 ;
 ; ATTRIBUTOR: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(none)
 ; ATTRIBUTOR-LABEL: define {{[^@]+}}@test7_1
-; ATTRIBUTOR-SAME: (ptr nocapture nofree nonnull writeonly inalloca(i32) [[A:%.*]]) #[[ATTR1]] {
+; ATTRIBUTOR-SAME: (ptr nofree nonnull writeonly inalloca(i32) captures(none) [[A:%.*]]) #[[ATTR1]] {
 ; ATTRIBUTOR-NEXT:    ret void
 ;
 ; ATTRIBUTOR-CGSCC: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(none)
 ; ATTRIBUTOR-CGSCC-LABEL: define {{[^@]+}}@test7_1
-; ATTRIBUTOR-CGSCC-SAME: (ptr nocapture nofree nonnull writeonly inalloca(i32) [[A:%.*]]) #[[ATTR1]] {
+; ATTRIBUTOR-CGSCC-SAME: (ptr nofree nonnull writeonly inalloca(i32) captures(none) [[A:%.*]]) #[[ATTR1]] {
 ; ATTRIBUTOR-CGSCC-NEXT:    ret void
 ;
   ret void
@@ -178,17 +178,17 @@ define void @test7_1(ptr inalloca(i32) %a) {
 define void @test7_2(ptr preallocated(i32) %a) {
 ; FNATTRS: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: readwrite)
 ; FNATTRS-LABEL: define {{[^@]+}}@test7_2
-; FNATTRS-SAME: (ptr nocapture preallocated(i32) [[A:%.*]]) #[[ATTR5]] {
+; FNATTRS-SAME: (ptr preallocated(i32) captures(none) [[A:%.*]]) #[[ATTR5]] {
 ; FNATTRS-NEXT:    ret void
 ;
 ; ATTRIBUTOR: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(none)
 ; ATTRIBUTOR-LABEL: define {{[^@]+}}@test7_2
-; ATTRIBUTOR-SAME: (ptr nocapture nofree nonnull writeonly preallocated(i32) [[A:%.*]]) #[[ATTR1]] {
+; ATTRIBUTOR-SAME: (ptr nofree nonnull writeonly preallocated(i32) captures(none) [[A:%.*]]) #[[ATTR1]] {
 ; ATTRIBUTOR-NEXT:    ret void
 ;
 ; ATTRIBUTOR-CGSCC: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(none)
 ; ATTRIBUTOR-CGSCC-LABEL: define {{[^@]+}}@test7_2
-; ATTRIBUTOR-CGSCC-SAME: (ptr nocapture nofree nonnull writeonly preallocated(i32) [[A:%.*]]) #[[ATTR1]] {
+; ATTRIBUTOR-CGSCC-SAME: (ptr nofree nonnull writeonly preallocated(i32) captures(none) [[A:%.*]]) #[[ATTR1]] {
 ; ATTRIBUTOR-CGSCC-NEXT:    ret void
 ;
   ret void
@@ -228,7 +228,7 @@ define void @test8_2(ptr %p) {
 ;
 ; ATTRIBUTOR: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(write)
 ; ATTRIBUTOR-LABEL: define {{[^@]+}}@test8_2
-; ATTRIBUTOR-SAME: (ptr nocapture nofree writeonly [[P:%.*]]) #[[ATTR0]] {
+; ATTRIBUTOR-SAME: (ptr nofree writeonly captures(none) [[P:%.*]]) #[[ATTR0]] {
 ; ATTRIBUTOR-NEXT:  entry:
 ; ATTRIBUTOR-NEXT:    [[CALL:%.*]] = call ptr @test8_1(ptr nofree readnone [[P]]) #[[ATTR14:[0-9]+]]
 ; ATTRIBUTOR-NEXT:    store i32 10, ptr [[CALL]], align 4
@@ -377,7 +377,7 @@ declare void @escape_readonly_ptr(ptr %addr, ptr readonly %ptr)
 ; %addr, causing the store to write to %escaped_then_written.
 define void @unsound_readnone(ptr %ignored, ptr %escaped_then_written) {
 ; FNATTRS-LABEL: define {{[^@]+}}@unsound_readnone
-; FNATTRS-SAME: (ptr nocapture readnone [[IGNORED:%.*]], ptr [[ESCAPED_THEN_WRITTEN:%.*]]) {
+; FNATTRS-SAME: (ptr readnone captures(none) [[IGNORED:%.*]], ptr [[ESCAPED_THEN_WRITTEN:%.*]]) {
 ; FNATTRS-NEXT:    [[ADDR:%.*]] = alloca ptr, align 8
 ; FNATTRS-NEXT:    call void @escape_readnone_ptr(ptr [[ADDR]], ptr [[ESCAPED_THEN_WRITTEN]])
 ; FNATTRS-NEXT:    [[ADDR_LD:%.*]] = load ptr, ptr [[ADDR]], align 8
@@ -385,7 +385,7 @@ define void @unsound_readnone(ptr %ignored, ptr %escaped_then_written) {
 ; FNATTRS-NEXT:    ret void
 ;
 ; ATTRIBUTOR-LABEL: define {{[^@]+}}@unsound_readnone
-; ATTRIBUTOR-SAME: (ptr nocapture nofree readnone [[IGNORED:%.*]], ptr nofree [[ESCAPED_THEN_WRITTEN:%.*]]) {
+; ATTRIBUTOR-SAME: (ptr nofree readnone captures(none) [[IGNORED:%.*]], ptr nofree [[ESCAPED_THEN_WRITTEN:%.*]]) {
 ; ATTRIBUTOR-NEXT:    [[ADDR:%.*]] = alloca ptr, align 8
 ; ATTRIBUTOR-NEXT:    call void @escape_readnone_ptr(ptr [[ADDR]], ptr nofree [[ESCAPED_THEN_WRITTEN]])
 ; ATTRIBUTOR-NEXT:    [[ADDR_LD:%.*]] = load ptr, ptr [[ADDR]], align 8
@@ -393,7 +393,7 @@ define void @unsound_readnone(ptr %ignored, ptr %escaped_then_written) {
 ; ATTRIBUTOR-NEXT:    ret void
 ;
 ; ATTRIBUTOR-CGSCC-LABEL: define {{[^@]+}}@unsound_readnone
-; ATTRIBUTOR-CGSCC-SAME: (ptr nocapture nofree readnone [[IGNORED:%.*]], ptr nofree [[ESCAPED_THEN_WRITTEN:%.*]]) {
+; ATTRIBUTOR-CGSCC-SAME: (ptr nofree readnone captures(none) [[IGNORED:%.*]], ptr nofree [[ESCAPED_THEN_WRITTEN:%.*]]) {
 ; ATTRIBUTOR-CGSCC-NEXT:    [[ADDR:%.*]] = alloca ptr, align 8
 ; ATTRIBUTOR-CGSCC-NEXT:    call void @escape_readnone_ptr(ptr [[ADDR]], ptr nofree [[ESCAPED_THEN_WRITTEN]])
 ; ATTRIBUTOR-CGSCC-NEXT:    [[ADDR_LD:%.*]] = load ptr, ptr [[ADDR]], align 8
@@ -409,7 +409,7 @@ define void @unsound_readnone(ptr %ignored, ptr %escaped_then_written) {
 
 define void @unsound_readonly(ptr %ignored, ptr %escaped_then_written) {
 ; FNATTRS-LABEL: define {{[^@]+}}@unsound_readonly
-; FNATTRS-SAME: (ptr nocapture readnone [[IGNORED:%.*]], ptr [[ESCAPED_THEN_WRITTEN:%.*]]) {
+; FNATTRS-SAME: (ptr readnone captures(none) [[IGNORED:%.*]], ptr [[ESCAPED_THEN_WRITTEN:%.*]]) {
 ; FNATTRS-NEXT:    [[ADDR:%.*]] = alloca ptr, align 8
 ; FNATTRS-NEXT:    call void @escape_readonly_ptr(ptr [[ADDR]], ptr [[ESCAPED_THEN_WRITTEN]])
 ; FNATTRS-NEXT:    [[ADDR_LD:%.*]] = load ptr, ptr [[ADDR]], align 8
@@ -417,7 +417,7 @@ define void @unsound_readonly(ptr %ignored, ptr %escaped_then_written) {
 ; FNATTRS-NEXT:    ret void
 ;
 ; ATTRIBUTOR-LABEL: define {{[^@]+}}@unsound_readonly
-; ATTRIBUTOR-SAME: (ptr nocapture nofree readnone [[IGNORED:%.*]], ptr nofree [[ESCAPED_THEN_WRITTEN:%.*]]) {
+; ATTRIBUTOR-SAME: (ptr nofree readnone captures(none) [[IGNORED:%.*]], ptr nofree [[ESCAPED_THEN_WRITTEN:%.*]]) {
 ; ATTRIBUTOR-NEXT:    [[ADDR:%.*]] = alloca ptr, align 8
 ; ATTRIBUTOR-NEXT:    call void @escape_readonly_ptr(ptr [[ADDR]], ptr nofree [[ESCAPED_THEN_WRITTEN]])
 ; ATTRIBUTOR-NEXT:    [[ADDR_LD:%.*]] = load ptr, ptr [[ADDR]], align 8
@@ -425,7 +425,7 @@ define void @unsound_readonly(ptr %ignored, ptr %escaped_then_written) {
 ; ATTRIBUTOR-NEXT:    ret void
 ;
 ; ATTRIBUTOR-CGSCC-LABEL: define {{[^@]+}}@unsound_readonly
-; ATTRIBUTOR-CGSCC-SAME: (ptr nocapture nofree readnone [[IGNORED:%.*]], ptr nofree [[ESCAPED_THEN_WRITTEN:%.*]]) {
+; ATTRIBUTOR-CGSCC-SAME: (ptr nofree readnone captures(none) [[IGNORED:%.*]], ptr nofree [[ESCAPED_THEN_WRITTEN:%.*]]) {
 ; ATTRIBUTOR-CGSCC-NEXT:    [[ADDR:%.*]] = alloca ptr, align 8
 ; ATTRIBUTOR-CGSCC-NEXT:    call void @escape_readonly_ptr(ptr [[ADDR]], ptr nofree [[ESCAPED_THEN_WRITTEN]])
 ; ATTRIBUTOR-CGSCC-NEXT:    [[ADDR_LD:%.*]] = load ptr, ptr [[ADDR]], align 8
@@ -441,18 +441,18 @@ define void @unsound_readonly(ptr %ignored, ptr %escaped_then_written) {
 
 define void @fptr_test1a(ptr %p, ptr %f) {
 ; FNATTRS-LABEL: define {{[^@]+}}@fptr_test1a
-; FNATTRS-SAME: (ptr nocapture readnone [[P:%.*]], ptr nocapture readonly [[F:%.*]]) {
-; FNATTRS-NEXT:    call void [[F]](ptr nocapture readnone [[P]])
+; FNATTRS-SAME: (ptr readnone captures(none) [[P:%.*]], ptr readonly captures(none) [[F:%.*]]) {
+; FNATTRS-NEXT:    call void [[F]](ptr readnone captures(none) [[P]])
 ; FNATTRS-NEXT:    ret void
 ;
 ; ATTRIBUTOR-LABEL: define {{[^@]+}}@fptr_test1a
-; ATTRIBUTOR-SAME: (ptr nocapture nofree [[P:%.*]], ptr nocapture nofree nonnull [[F:%.*]]) {
-; ATTRIBUTOR-NEXT:    call void [[F]](ptr nocapture nofree readnone [[P]])
+; ATTRIBUTOR-SAME: (ptr nofree captures(none) [[P:%.*]], ptr nofree nonnull captures(none) [[F:%.*]]) {
+; ATTRIBUTOR-NEXT:    call void [[F]](ptr nofree readnone captures(none) [[P]])
 ; ATTRIBUTOR-NEXT:    ret void
 ;
 ; ATTRIBUTOR-CGSCC-LABEL: define {{[^@]+}}@fptr_test1a
-; ATTRIBUTOR-CGSCC-SAME: (ptr nocapture nofree [[P:%.*]], ptr nocapture nofree nonnull [[F:%.*]]) {
-; ATTRIBUTOR-CGSCC-NEXT:    call void [[F]](ptr nocapture nofree readnone [[P]])
+; ATTRIBUTOR-CGSCC-SAME: (ptr nofree captures(none) [[P:%.*]], ptr nofree nonnull captures(none) [[F:%.*]]) {
+; ATTRIBUTOR-CGSCC-NEXT:    call void [[F]](ptr nofree readnone captures(none) [[P]])
 ; ATTRIBUTOR-CGSCC-NEXT:    ret void
 ;
   call void %f(ptr nocapture readnone %p)
@@ -462,17 +462,17 @@ define void @fptr_test1a(ptr %p, ptr %f) {
 ; Can't infer readnone here because call might capture %p
 define void @fptr_test1b(ptr %p, ptr %f) {
 ; FNATTRS-LABEL: define {{[^@]+}}@fptr_test1b
-; FNATTRS-SAME: (ptr [[P:%.*]], ptr nocapture readonly [[F:%.*]]) {
+; FNATTRS-SAME: (ptr [[P:%.*]], ptr readonly captures(none) [[F:%.*]]) {
 ; FNATTRS-NEXT:    call void [[F]](ptr readnone [[P]])
 ; FNATTRS-NEXT:    ret void
 ;
 ; ATTRIBUTOR-LABEL: define {{[^@]+}}@fptr_test1b
-; ATTRIBUTOR-SAME: (ptr nofree [[P:%.*]], ptr nocapture nofree nonnull [[F:%.*]]) {
+; ATTRIBUTOR-SAME: (ptr nofree [[P:%.*]], ptr nofree nonnull captures(none) [[F:%.*]]) {
 ; ATTRIBUTOR-NEXT:    call void [[F]](ptr nofree readnone [[P]])
 ; ATTRIBUTOR-NEXT:    ret void
 ;
 ; ATTRIBUTOR-CGSCC-LABEL: define {{[^@]+}}@fptr_test1b
-; ATTRIBUTOR-CGSCC-SAME: (ptr nofree [[P:%.*]], ptr nocapture nofree nonnull [[F:%.*]]) {
+; ATTRIBUTOR-CGSCC-SAME: (ptr nofree [[P:%.*]], ptr nofree nonnull captures(none) [[F:%.*]]) {
 ; ATTRIBUTOR-CGSCC-NEXT:    call void [[F]](ptr nofree readnone [[P]])
 ; ATTRIBUTOR-CGSCC-NEXT:    ret void
 ;
@@ -483,19 +483,19 @@ define void @fptr_test1b(ptr %p, ptr %f) {
 define void @fptr_test1c(ptr %p, ptr %f) {
 ; FNATTRS: Function Attrs: nofree memory(read)
 ; FNATTRS-LABEL: define {{[^@]+}}@fptr_test1c
-; FNATTRS-SAME: (ptr readnone [[P:%.*]], ptr nocapture readonly [[F:%.*]]) #[[ATTR3]] {
+; FNATTRS-SAME: (ptr readnone [[P:%.*]], ptr readonly captures(none) [[F:%.*]]) #[[ATTR3]] {
 ; FNATTRS-NEXT:    call void [[F]](ptr readnone [[P]]) #[[ATTR2:[0-9]+]]
 ; FNATTRS-NEXT:    ret void
 ;
 ; ATTRIBUTOR: Function Attrs: memory(read)
 ; ATTRIBUTOR-LABEL: define {{[^@]+}}@fptr_test1c
-; ATTRIBUTOR-SAME: (ptr nofree readonly [[P:%.*]], ptr nocapture nofree nonnull readonly [[F:%.*]]) #[[ATTR2:[0-9]+]] {
+; ATTRIBUTOR-SAME: (ptr nofree readonly [[P:%.*]], ptr nofree nonnull readonly captures(none) [[F:%.*]]) #[[ATTR2:[0-9]+]] {
 ; ATTRIBUTOR-NEXT:    call void [[F]](ptr nofree readnone [[P]]) #[[ATTR2]]
 ; ATTRIBUTOR-NEXT:    ret void
 ;
 ; ATTRIBUTOR-CGSCC: Function Attrs: memory(read)
 ; ATTRIBUTOR-CGSCC-LABEL: define {{[^@]+}}@fptr_test1c
-; ATTRIBUTOR-CGSCC-SAME: (ptr nofree readonly [[P:%.*]], ptr nocapture nofree nonnull readonly [[F:%.*]]) #[[ATTR2:[0-9]+]] {
+; ATTRIBUTOR-CGSCC-SAME: (ptr nofree readonly [[P:%.*]], ptr nofree nonnull readonly captures(none) [[F:%.*]]) #[[ATTR2:[0-9]+]] {
 ; ATTRIBUTOR-CGSCC-NEXT:    call void [[F]](ptr nofree readnone [[P]]) #[[ATTR2]]
 ; ATTRIBUTOR-CGSCC-NEXT:    ret void
 ;
@@ -505,18 +505,18 @@ define void @fptr_test1c(ptr %p, ptr %f) {
 
 define void @fptr_test2a(ptr %p, ptr %f) {
 ; FNATTRS-LABEL: define {{[^@]+}}@fptr_test2a
-; FNATTRS-SAME: (ptr nocapture readonly [[P:%.*]], ptr nocapture readonly [[F:%.*]]) {
-; FNATTRS-NEXT:    call void [[F]](ptr nocapture readonly [[P]])
+; FNATTRS-SAME: (ptr readonly captures(none) [[P:%.*]], ptr readonly captures(none) [[F:%.*]]) {
+; FNATTRS-NEXT:    call void [[F]](ptr readonly captures(none) [[P]])
 ; FNATTRS-NEXT:    ret void
 ;
 ; ATTRIBUTOR-LABEL: define {{[^@]+}}@fptr_test2a
-; ATTRIBUTOR-SAME: (ptr nocapture nofree [[P:%.*]], ptr nocapture nofree nonnull [[F:%.*]]) {
-; ATTRIBUTOR-NEXT:    call void [[F]](ptr nocapture nofree readonly [[P]])
+; ATTRIBUTOR-SAME: (ptr nofree captures(none) [[P:%.*]], ptr nofree nonnull captures(none) [[F:%.*]]) {
+; ATTRIBUTOR-NEXT:    call void [[F]](ptr nofree readonly captures(none) [[P]])
 ; ATTRIBUTOR-NEXT:    ret void
 ;
 ; ATTRIBUTOR-CGSCC-LABEL: define {{[^@]+}}@fptr_test2a
-; ATTRIBUTOR-CGSCC-SAME: (ptr nocapture nofree [[P:%.*]], ptr nocapture nofree nonnull [[F:%.*]]) {
-; ATTRIBUTOR-CGSCC-NEXT:    call void [[F]](ptr nocapture nofree readonly [[P]])
+; ATTRIBUTOR-CGSCC-SAME: (ptr nofree captures(none) [[P:%.*]], ptr nofree nonnull captures(none) [[F:%.*]]) {
+; ATTRIBUTOR-CGSCC-NEXT:    call void [[F]](ptr nofree readonly captures(none) [[P]])
 ; ATTRIBUTOR-CGSCC-NEXT:    ret void
 ;
   call void %f(ptr nocapture readonly %p)
@@ -526,17 +526,17 @@ define void @fptr_test2a(ptr %p, ptr %f) {
 define void @fptr_test2b(ptr %p, ptr %f) {
   ; Can't infer readonly here because call might capture %p
 ; FNATTRS-LABEL: define {{[^@]+}}@fptr_test2b
-; FNATTRS-SAME: (ptr [[P:%.*]], ptr nocapture readonly [[F:%.*]]) {
+; FNATTRS-SAME: (ptr [[P:%.*]], ptr readonly captures(none) [[F:%.*]]) {
 ; FNATTRS-NEXT:    call void [[F]](ptr readonly [[P]])
 ; FNATTRS-NEXT:    ret void
 ;
 ; ATTRIBUTOR-LABEL: define {{[^@]+}}@fptr_test2b
-; ATTRIBUTOR-SAME: (ptr nofree [[P:%.*]], ptr nocapture nofree nonnull [[F:%.*]]) {
+; ATTRIBUTOR-SAME: (ptr nofree [[P:%.*]], ptr nofree nonnull captures(none) [[F:%.*]]) {
 ; ATTRIBUTOR-NEXT:    call void [[F]](ptr nofree readonly [[P]])
 ; ATTRIBUTOR-NEXT:    ret void
 ;
 ; ATTRIBUTOR-CGSCC-LABEL: define {{[^@]+}}@fptr_test2b
-; ATTRIBUTOR-CGSCC-SAME: (ptr nofree [[P:%.*]], ptr nocapture nofree nonnull [[F:%.*]]) {
+; ATTRIBUTOR-CGSCC-SAME: (ptr nofree [[P:%.*]], ptr nofree nonnull captures(none) [[F:%.*]]) {
 ; ATTRIBUTOR-CGSCC-NEXT:    call void [[F]](ptr nofree readonly [[P]])
 ; ATTRIBUTOR-CGSCC-NEXT:    ret void
 ;
@@ -547,19 +547,19 @@ define void @fptr_test2b(ptr %p, ptr %f) {
 define void @fptr_test2c(ptr %p, ptr %f) {
 ; FNATTRS: Function Attrs: nofree memory(read)
 ; FNATTRS-LABEL: define {{[^@]+}}@fptr_test2c
-; FNATTRS-SAME: (ptr readonly [[P:%.*]], ptr nocapture readonly [[F:%.*]]) #[[ATTR3]] {
+; FNATTRS-SAME: (ptr readonly [[P:%.*]], ptr readonly captures(none) [[F:%.*]]) #[[ATTR3]] {
 ; FNATTRS-NEXT:    call void [[F]](ptr readonly [[P]]) #[[ATTR2]]
 ; FNATTRS-NEXT:    ret void
 ;
 ; ATTRIBUTOR: Function Attrs: memory(read)
 ; ATTRIBUTOR-LABEL: define {{[^@]+}}@fptr_test2c
-; ATTRIBUTOR-SAME: (ptr nofree readonly [[P:%.*]], ptr nocapture nofree nonnull readonly [[F:%.*]]) #[[ATTR2]] {
+; ATTRIBUTOR-SAME: (ptr nofree readonly [[P:%.*]], ptr nofree nonnull readonly captures(none) [[F:%.*]]) #[[ATTR2]] {
 ; ATTRIBUTOR-NEXT:    call void [[F]](ptr nofree readonly [[P]]) #[[ATTR2]]
 ; ATTRIBUTOR-NEXT:    ret void
 ;
 ; ATTRIBUTOR-CGSCC: Function Attrs: memory(read)
 ; ATTRIBUTOR-CGSCC-LABEL: define {{[^@]+}}@fptr_test2c
-; ATTRIBUTOR-CGSCC-SAME: (ptr nofree readonly [[P:%.*]], ptr nocapture nofree nonnull readonly [[F:%.*]]) #[[ATTR2]] {
+; ATTRIBUTOR-CGSCC-SAME: (ptr nofree readonly [[P:%.*]], ptr nofree nonnull readonly captures(none) [[F:%.*]]) #[[ATTR2]] {
 ; ATTRIBUTOR-CGSCC-NEXT:    call void [[F]](ptr nofree readonly [[P]]) #[[ATTR2]]
 ; ATTRIBUTOR-CGSCC-NEXT:    ret void
 ;
@@ -642,18 +642,18 @@ declare void @readonly_param(ptr nocapture readonly %p)
 ; FIXME: While this can't be readnone, this could be readonly.
 define void @op_bundle_readnone_deopt(ptr %p) {
 ; FNATTRS-LABEL: define {{[^@]+}}@op_bundle_readnone_deopt
-; FNATTRS-SAME: (ptr nocapture [[P:%.*]]) {
+; FNATTRS-SAME: (ptr captures(none) [[P:%.*]]) {
 ; FNATTRS-NEXT:    call void @readnone_param(ptr [[P]]) [ "deopt"() ]
 ; FNATTRS-NEXT:    ret void
 ;
 ; ATTRIBUTOR-LABEL: define {{[^@]+}}@op_bundle_readnone_deopt
-; ATTRIBUTOR-SAME: (ptr nocapture nofree [[P:%.*]]) {
-; ATTRIBUTOR-NEXT:    call void @readnone_param(ptr nocapture nofree [[P]]) [ "deopt"() ]
+; ATTRIBUTOR-SAME: (ptr nofree captures(none) [[P:%.*]]) {
+; ATTRIBUTOR-NEXT:    call void @readnone_param(ptr nofree captures(none) [[P]]) [ "deopt"() ]
 ; ATTRIBUTOR-NEXT:    ret void
 ;
 ; ATTRIBUTOR-CGSCC-LABEL: define {{[^@]+}}@op_bundle_readnone_deopt
-; ATTRIBUTOR-CGSCC-SAME: (ptr nocapture nofree [[P:%.*]]) {
-; ATTRIBUTOR-CGSCC-NEXT:    call void @readnone_param(ptr nocapture nofree [[P]]) [ "deopt"() ]
+; ATTRIBUTOR-CGSCC-SAME: (ptr nofree captures(none) [[P:%.*]]) {
+; ATTRIBUTOR-CGSCC-NEXT:    call void @readnone_param(ptr nofree captures(none) [[P]]) [ "deopt"() ]
 ; ATTRIBUTOR-CGSCC-NEXT:    ret void
 ;
   call void @readnone_param(ptr %p) ["deopt"()]
@@ -662,18 +662,18 @@ define void @op_bundle_readnone_deopt(ptr %p) {
 
 define void @op_bundle_readnone_unknown(ptr %p) {
 ; FNATTRS-LABEL: define {{[^@]+}}@op_bundle_readnone_unknown
-; FNATTRS-SAME: (ptr nocapture [[P:%.*]]) {
+; FNATTRS-SAME: (ptr captures(none) [[P:%.*]]) {
 ; FNATTRS-NEXT:    call void @readnone_param(ptr [[P]]) [ "unknown"() ]
 ; FNATTRS-NEXT:    ret void
 ;
 ; ATTRIBUTOR-LABEL: define {{[^@]+}}@op_bundle_readnone_unknown
-; ATTRIBUTOR-SAME: (ptr nocapture nofree [[P:%.*]]) {
-; ATTRIBUTOR-NEXT:    call void @readnone_param(ptr nocapture nofree [[P]]) [ "unknown"() ]
+; ATTRIBUTOR-SAME: (ptr nofree captures(none) [[P:%.*]]) {
+; ATTRIBUTOR-NEXT:    call void @readnone_param(ptr nofree captures(none) [[P]]) [ "unknown"() ]
 ; ATTRIBUTOR-NEXT:    ret void
 ;
 ; ATTRIBUTOR-CGSCC-LABEL: define {{[^@]+}}@op_bundle_readnone_unknown
-; ATTRIBUTOR-CGSCC-SAME: (ptr nocapture nofree [[P:%.*]]) {
-; ATTRIBUTOR-CGSCC-NEXT:    call void @readnone_param(ptr nocapture nofree [[P]]) [ "unknown"() ]
+; ATTRIBUTOR-CGSCC-SAME: (ptr nofree captures(none) [[P:%.*]]) {
+; ATTRIBUTOR-CGSCC-NEXT:    call void @readnone_param(ptr nofree captures(none) [[P]]) [ "unknown"() ]
 ; ATTRIBUTOR-CGSCC-NEXT:    ret void
 ;
   call void @readnone_param(ptr %p) ["unknown"()]
@@ -682,18 +682,18 @@ define void @op_bundle_readnone_unknown(ptr %p) {
 
 define void @op_bundle_readonly_deopt(ptr %p) {
 ; FNATTRS-LABEL: define {{[^@]+}}@op_bundle_readonly_deopt
-; FNATTRS-SAME: (ptr nocapture readonly [[P:%.*]]) {
+; FNATTRS-SAME: (ptr readonly captures(none) [[P:%.*]]) {
 ; FNATTRS-NEXT:    call void @readonly_param(ptr [[P]]) [ "deopt"() ]
 ; FNATTRS-NEXT:    ret void
 ;
 ; ATTRIBUTOR-LABEL: define {{[^@]+}}@op_bundle_readonly_deopt
-; ATTRIBUTOR-SAME: (ptr nocapture nofree [[P:%.*]]) {
-; ATTRIBUTOR-NEXT:    call void @readonly_param(ptr nocapture nofree [[P]]) [ "deopt"() ]
+; ATTRIBUTOR-SAME: (ptr nofree captures(none) [[P:%.*]]) {
+; ATTRIBUTOR-NEXT:    call void @readonly_param(ptr nofree captures(none) [[P]]) [ "deopt"() ]
 ; ATTRIBUTOR-NEXT:    ret void
 ;
 ; ATTRIBUTOR-CGSCC-LABEL: define {{[^@]+}}@op_bundle_readonly_deopt
-; ATTRIBUTOR-CGSCC-SAME: (ptr nocapture nofree [[P:%.*]]) {
-; ATTRIBUTOR-CGSCC-NEXT:    call void @readonly_param(ptr nocapture nofree [[P]]) [ "deopt"() ]
+; ATTRIBUTOR-CGSCC-SAME: (ptr nofree captures(none) [[P:%.*]]) {
+; ATTRIBUTOR-CGSCC-NEXT:    call void @readonly_param(ptr nofree captures(none) [[P]]) [ "deopt"() ]
 ; ATTRIBUTOR-CGSCC-NEXT:    ret void
 ;
   call void @readonly_param(ptr %p) ["deopt"()]
@@ -702,18 +702,18 @@ define void @op_bundle_readonly_deopt(ptr %p) {
 
 define void @op_bundle_readonly_unknown(ptr %p) {
 ; FNATTRS-LABEL: define {{[^@]+}}@op_bundle_readonly_unknown
-; FNATTRS-SAME: (ptr nocapture [[P:%.*]]) {
+; FNATTRS-SAME: (ptr captures(none) [[P:%.*]]) {
 ; FNATTRS-NEXT:    call void @readonly_param(ptr [[P]]) [ "unknown"() ]
 ; FNATTRS-NEXT:    ret void
 ;
 ; ATTRIBUTOR-LABEL: define {{[^@]+}}@op_bundle_readonly_unknown
-; ATTRIBUTOR-SAME: (ptr nocapture nofree [[P:%.*]]) {
-; ATTRIBUTOR-NEXT:    call void @readonly_param(ptr nocapture nofree [[P]]) [ "unknown"() ]
+; ATTRIBUTOR-SAME: (ptr nofree captures(none) [[P:%.*]]) {
+; ATTRIBUTOR-NEXT:    call void @readonly_param(ptr nofree captures(none) [[P]]) [ "unknown"() ]
 ; ATTRIBUTOR-NEXT:    ret void
 ;
 ; ATTRIBUTOR-CGSCC-LABEL: define {{[^@]+}}@op_bundle_readonly_unknown
-; ATTRIBUTOR-CGSCC-SAME: (ptr nocapture nofree [[P:%.*]]) {
-; ATTRIBUTOR-CGSCC-NEXT:    call void @readonly_param(ptr nocapture nofree [[P]]) [ "unknown"() ]
+; ATTRIBUTOR-CGSCC-SAME: (ptr nofree captures(none) [[P:%.*]]) {
+; ATTRIBUTOR-CGSCC-NEXT:    call void @readonly_param(ptr nofree captures(none) [[P]]) [ "unknown"() ]
 ; ATTRIBUTOR-CGSCC-NEXT:    ret void
 ;
   call void @readonly_param(ptr %p) ["unknown"()]
@@ -723,19 +723,19 @@ define void @op_bundle_readonly_unknown(ptr %p) {
 define i32 @writable_readonly(ptr writable dereferenceable(4) %p) {
 ; FNATTRS: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: read)
 ; FNATTRS-LABEL: define {{[^@]+}}@writable_readonly
-; FNATTRS-SAME: (ptr nocapture readonly dereferenceable(4) [[P:%.*]]) #[[ATTR15:[0-9]+]] {
+; FNATTRS-SAME: (ptr readonly captures(none) dereferenceable(4) [[P:%.*]]) #[[ATTR15:[0-9]+]] {
 ; FNATTRS-NEXT:    [[V:%.*]] = load i32, ptr [[P]], align 4
 ; FNATTRS-NEXT:    ret i32 [[V]]
 ;
 ; ATTRIBUTOR: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: read)
 ; ATTRIBUTOR-LABEL: define {{[^@]+}}@writable_readonly
-; ATTRIBUTOR-SAME: (ptr nocapture nofree nonnull readonly dereferenceable(4) [[P:%.*]]) #[[ATTR13:[0-9]+]] {
+; ATTRIBUTOR-SAME: (ptr nofree nonnull readonly captures(none) dereferenceable(4) [[P:%.*]]) #[[ATTR13:[0-9]+]] {
 ; ATTRIBUTOR-NEXT:    [[V:%.*]] = load i32, ptr [[P]], align 4
 ; ATTRIBUTOR-NEXT:    ret i32 [[V]]
 ;
 ; ATTRIBUTOR-CGSCC: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: read)
 ; ATTRIBUTOR-CGSCC-LABEL: define {{[^@]+}}@writable_readonly
-; ATTRIBUTOR-CGSCC-SAME: (ptr nocapture nofree nonnull readonly dereferenceable(4) [[P:%.*]]) #[[ATTR13:[0-9]+]] {
+; ATTRIBUTOR-CGSCC-SAME: (ptr nofree nonnull readonly captures(none) dereferenceable(4) [[P:%.*]]) #[[ATTR13:[0-9]+]] {
 ; ATTRIBUTOR-CGSCC-NEXT:    [[V:%.*]] = load i32, ptr [[P]], align 4
 ; ATTRIBUTOR-CGSCC-NEXT:    ret i32 [[V]]
 ;
@@ -746,17 +746,17 @@ define i32 @writable_readonly(ptr writable dereferenceable(4) %p) {
 define void @writable_readnone(ptr writable dereferenceable(4) %p) {
 ; FNATTRS: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(none)
 ; FNATTRS-LABEL: define {{[^@]+}}@writable_readnone
-; FNATTRS-SAME: (ptr nocapture readnone dereferenceable(4) [[P:%.*]]) #[[ATTR1]] {
+; FNATTRS-SAME: (ptr readnone captures(none) dereferenceable(4) [[P:%.*]]) #[[ATTR1]] {
 ; FNATTRS-NEXT:    ret void
 ;
 ; ATTRIBUTOR: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(none)
 ; ATTRIBUTOR-LABEL: define {{[^@]+}}@writable_readnone
-; ATTRIBUTOR-SAME: (ptr nocapture nofree nonnull readnone dereferenceable(4) [[P:%.*]]) #[[ATTR1]] {
+; ATTRIBUTOR-SAME: (ptr nofree nonnull readnone captures(none) dereferenceable(4) [[P:%.*]]) #[[ATTR1]] {
 ; ATTRIBUTOR-NEXT:    ret void
 ;
 ; ATTRIBUTOR-CGSCC: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(none)
 ; ATTRIBUTOR-CGSCC-LABEL: define {{[^@]+}}@writable_readnone
-; ATTRIBUTOR-CGSCC-SAME: (ptr nocapture nofree nonnull readnone dereferenceable(4) [[P:%.*]]) #[[ATTR1]] {
+; ATTRIBUTOR-CGSCC-SAME: (ptr nofree nonnull readnone captures(none) dereferenceable(4) [[P:%.*]]) #[[ATTR1]] {
 ; ATTRIBUTOR-CGSCC-NEXT:    ret void
 ;
   ret void
@@ -766,18 +766,18 @@ declare void @byval_param(ptr byval(i32) %p)
 
 define void @call_byval_param(ptr %p) {
 ; FNATTRS-LABEL: define {{[^@]+}}@call_byval_param
-; FNATTRS-SAME: (ptr nocapture readonly [[P:%.*]]) {
+; FNATTRS-SAME: (ptr readonly captures(none) [[P:%.*]]) {
 ; FNATTRS-NEXT:    call void @byval_param(ptr byval(i32) [[P]])
 ; FNATTRS-NEXT:    ret void
 ;
 ; ATTRIBUTOR-LABEL: define {{[^@]+}}@call_byval_param
-; ATTRIBUTOR-SAME: (ptr nocapture readonly [[P:%.*]]) {
-; ATTRIBUTOR-NEXT:    call void @byval_param(ptr nocapture readonly byval(i32) [[P]])
+; ATTRIBUTOR-SAME: (ptr readonly captures(none) [[P:%.*]]) {
+; ATTRIBUTOR-NEXT:    call void @byval_param(ptr readonly byval(i32) captures(none) [[P]])
 ; ATTRIBUTOR-NEXT:    ret void
 ;
 ; ATTRIBUTOR-CGSCC-LABEL: define {{[^@]+}}@call_byval_param
-; ATTRIBUTOR-CGSCC-SAME: (ptr nocapture readonly [[P:%.*]]) {
-; ATTRIBUTOR-CGSCC-NEXT:    call void @byval_param(ptr nocapture readonly byval(i32) [[P]])
+; ATTRIBUTOR-CGSCC-SAME: (ptr readonly captures(none) [[P:%.*]]) {
+; ATTRIBUTOR-CGSCC-NEXT:    call void @byval_param(ptr readonly byval(i32) captures(none) [[P]])
 ; ATTRIBUTOR-CGSCC-NEXT:    ret void
 ;
   call void @byval_param(ptr byval(i32) %p)
diff --git a/llvm/test/Transforms/FunctionAttrs/readnone.ll b/llvm/test/Transforms/FunctionAttrs/readnone.ll
index 7a6d47cbabd8b..74361b7486fcc 100644
--- a/llvm/test/Transforms/FunctionAttrs/readnone.ll
+++ b/llvm/test/Transforms/FunctionAttrs/readnone.ll
@@ -4,13 +4,13 @@
 
 define void @bar(ptr readonly %0) {
 ; FNATTRS-LABEL: define void @bar(
-; FNATTRS-SAME: ptr nocapture readnone [[TMP0:%.*]]) #[[ATTR0:[0-9]+]] {
+; FNATTRS-SAME: ptr readnone captures(none) [[TMP0:%.*]]) #[[ATTR0:[0-9]+]] {
 ; FNATTRS-NEXT:    call void @foo(ptr [[TMP0]])
 ; FNATTRS-NEXT:    ret void
 ;
 ; ATTRIBUTOR-LABEL: define void @bar(
-; ATTRIBUTOR-SAME: ptr nocapture nofree readnone [[TMP0:%.*]]) #[[ATTR0:[0-9]+]] {
-; ATTRIBUTOR-NEXT:    call void @foo(ptr nocapture nofree readnone [[TMP0]]) #[[ATTR0]]
+; ATTRIBUTOR-SAME: ptr nofree readnone captures(none) [[TMP0:%.*]]) #[[ATTR0:[0-9]+]] {
+; ATTRIBUTOR-NEXT:    call void @foo(ptr nofree readnone captures(none) [[TMP0]]) #[[ATTR0]]
 ; ATTRIBUTOR-NEXT:    ret void
 ;
   call void @foo(ptr %0)
@@ -19,13 +19,13 @@ define void @bar(ptr readonly %0) {
 
 define void @foo(ptr readonly %0) {
 ; FNATTRS-LABEL: define void @foo(
-; FNATTRS-SAME: ptr nocapture readnone [[TMP0:%.*]]) #[[ATTR0]] {
+; FNATTRS-SAME: ptr readnone captures(none) [[TMP0:%.*]]) #[[ATTR0]] {
 ; FNATTRS-NEXT:    call void @bar(ptr [[TMP0]])
 ; FNATTRS-NEXT:    ret void
 ;
 ; ATTRIBUTOR-LABEL: define void @foo(
-; ATTRIBUTOR-SAME: ptr nocapture nofree readnone [[TMP0:%.*]]) #[[ATTR0]] {
-; ATTRIBUTOR-NEXT:    call void @bar(ptr nocapture nofree readnone [[TMP0]]) #[[ATTR0]]
+; ATTRIBUTOR-SAME: ptr nofree readnone captures(none) [[TMP0:%.*]]) #[[ATTR0]] {
+; ATTRIBUTOR-NEXT:    call void @bar(ptr nofree readnone captures(none) [[TMP0]]) #[[ATTR0]]
 ; ATTRIBUTOR-NEXT:    ret void
 ;
   call void @bar(ptr %0)
diff --git a/llvm/test/Transforms/FunctionAttrs/willreturn.ll b/llvm/test/Transforms/FunctionAttrs/willreturn.ll
index 70926345ce274..dc0280288b4fe 100644
--- a/llvm/test/Transforms/FunctionAttrs/willreturn.ll
+++ b/llvm/test/Transforms/FunctionAttrs/willreturn.ll
@@ -88,7 +88,7 @@ define i32 @mustprogress_call_known_functions(ptr %ptr) mustprogress {
 ; ATTRIBUTOR: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: read)
 ; ATTRIBUTOR-LABEL: @mustprogress_call_known_functions(
 ; ATTRIBUTOR-NEXT:    call void @mustprogress_readnone() #[[ATTR9:[0-9]+]]
-; ATTRIBUTOR-NEXT:    [[R:%.*]] = call i32 @mustprogress_load(ptr nocapture nofree readonly [[PTR:%.*]]) #[[ATTR12:[0-9]+]]
+; ATTRIBUTOR-NEXT:    [[R:%.*]] = call i32 @mustprogress_load(ptr nofree readonly captures(none) [[PTR:%.*]]) #[[ATTR12:[0-9]+]]
 ; ATTRIBUTOR-NEXT:    ret i32 [[R]]
 ;
   call void @mustprogress_readnone()
diff --git a/llvm/test/Transforms/FunctionAttrs/writeonly.ll b/llvm/test/Transforms/FunctionAttrs/writeonly.ll
index ba546aff6e621..88c6031613697 100644
--- a/llvm/test/Transforms/FunctionAttrs/writeonly.ll
+++ b/llvm/test/Transforms/FunctionAttrs/writeonly.ll
@@ -5,13 +5,13 @@
 define void @nouses-argworn-funrn(ptr writeonly %.aaa) {
 ; FNATTRS: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(none)
 ; FNATTRS-LABEL: define {{[^@]+}}@nouses-argworn-funrn
-; FNATTRS-SAME: (ptr nocapture readnone [[DOTAAA:%.*]]) #[[ATTR0:[0-9]+]] {
+; FNATTRS-SAME: (ptr readnone captures(none) [[DOTAAA:%.*]]) #[[ATTR0:[0-9]+]] {
 ; FNATTRS-NEXT:  nouses-argworn-funrn_entry:
 ; FNATTRS-NEXT:    ret void
 ;
 ; ATTRIBUTOR: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(none)
 ; ATTRIBUTOR-LABEL: define {{[^@]+}}@nouses-argworn-funrn
-; ATTRIBUTOR-SAME: (ptr nocapture nofree readnone [[DOTAAA:%.*]]) #[[ATTR0:[0-9]+]] {
+; ATTRIBUTOR-SAME: (ptr nofree readnone captures(none) [[DOTAAA:%.*]]) #[[ATTR0:[0-9]+]] {
 ; ATTRIBUTOR-NEXT:  nouses-argworn-funrn_entry:
 ; ATTRIBUTOR-NEXT:    ret void
 ;
@@ -22,14 +22,14 @@ nouses-argworn-funrn_entry:
 define void @nouses-argworn-funro(ptr writeonly %.aaa, ptr %.bbb) {
 ; FNATTRS: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: read)
 ; FNATTRS-LABEL: define {{[^@]+}}@nouses-argworn-funro
-; FNATTRS-SAME: (ptr nocapture readnone [[DOTAAA:%.*]], ptr nocapture readonly [[DOTBBB:%.*]]) #[[ATTR1:[0-9]+]] {
+; FNATTRS-SAME: (ptr readnone captures(none) [[DOTAAA:%.*]], ptr readonly captures(none) [[DOTBBB:%.*]]) #[[ATTR1:[0-9]+]] {
 ; FNATTRS-NEXT:  nouses-argworn-funro_entry:
 ; FNATTRS-NEXT:    [[VAL:%.*]] = load i32, ptr [[DOTBBB]], align 4
 ; FNATTRS-NEXT:    ret void
 ;
 ; ATTRIBUTOR: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: read)
 ; ATTRIBUTOR-LABEL: define {{[^@]+}}@nouses-argworn-funro
-; ATTRIBUTOR-SAME: (ptr nocapture nofree readnone [[DOTAAA:%.*]], ptr nocapture nofree nonnull readonly [[DOTBBB:%.*]]) #[[ATTR1:[0-9]+]] {
+; ATTRIBUTOR-SAME: (ptr nofree readnone captures(none) [[DOTAAA:%.*]], ptr nofree nonnull readonly captures(none) [[DOTBBB:%.*]]) #[[ATTR1:[0-9]+]] {
 ; ATTRIBUTOR-NEXT:  nouses-argworn-funro_entry:
 ; ATTRIBUTOR-NEXT:    [[VAL:%.*]] = load i32, ptr [[DOTBBB]], align 4
 ; ATTRIBUTOR-NEXT:    ret void
@@ -46,14 +46,14 @@ nouses-argworn-funro_entry:
 define void @nouses-argworn-funwo(ptr writeonly %.aaa) {
 ; FNATTRS: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(write, argmem: none, inaccessiblemem: none)
 ; FNATTRS-LABEL: define {{[^@]+}}@nouses-argworn-funwo
-; FNATTRS-SAME: (ptr nocapture readnone [[DOTAAA:%.*]]) #[[ATTR2:[0-9]+]] {
+; FNATTRS-SAME: (ptr readnone captures(none) [[DOTAAA:%.*]]) #[[ATTR2:[0-9]+]] {
 ; FNATTRS-NEXT:  nouses-argworn-funwo_entry:
 ; FNATTRS-NEXT:    store i8 0, ptr getelementptr inbounds ([[_TYPE_OF_D_CCC:%.*]], ptr @d-ccc, i32 0, i32 3), align 1
 ; FNATTRS-NEXT:    ret void
 ;
 ; ATTRIBUTOR: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(write)
 ; ATTRIBUTOR-LABEL: define {{[^@]+}}@nouses-argworn-funwo
-; ATTRIBUTOR-SAME: (ptr nocapture nofree readnone [[DOTAAA:%.*]]) #[[ATTR2:[0-9]+]] {
+; ATTRIBUTOR-SAME: (ptr nofree readnone captures(none) [[DOTAAA:%.*]]) #[[ATTR2:[0-9]+]] {
 ; ATTRIBUTOR-NEXT:  nouses-argworn-funwo_entry:
 ; ATTRIBUTOR-NEXT:    store i8 0, ptr getelementptr inbounds ([[_TYPE_OF_D_CCC:%.*]], ptr @d-ccc, i32 0, i32 3), align 1
 ; ATTRIBUTOR-NEXT:    ret void
@@ -66,13 +66,13 @@ nouses-argworn-funwo_entry:
 define void @test_store(ptr %p) {
 ; FNATTRS: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: write)
 ; FNATTRS-LABEL: define {{[^@]+}}@test_store
-; FNATTRS-SAME: (ptr nocapture writeonly initializes((0, 1)) [[P:%.*]]) #[[ATTR3:[0-9]+]] {
+; FNATTRS-SAME: (ptr writeonly captures(none) initializes((0, 1)) [[P:%.*]]) #[[ATTR3:[0-9]+]] {
 ; FNATTRS-NEXT:    store i8 0, ptr [[P]], align 1
 ; FNATTRS-NEXT:    ret void
 ;
 ; ATTRIBUTOR: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: write)
 ; ATTRIBUTOR-LABEL: define {{[^@]+}}@test_store
-; ATTRIBUTOR-SAME: (ptr nocapture nofree nonnull writeonly [[P:%.*]]) #[[ATTR3:[0-9]+]] {
+; ATTRIBUTOR-SAME: (ptr nofree nonnull writeonly captures(none) [[P:%.*]]) #[[ATTR3:[0-9]+]] {
 ; ATTRIBUTOR-NEXT:    store i8 0, ptr [[P]], align 1
 ; ATTRIBUTOR-NEXT:    ret void
 ;
@@ -107,14 +107,14 @@ define i8 @test_store_capture(ptr %p) {
 define void @test_addressing(ptr %p) {
 ; FNATTRS: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: write)
 ; FNATTRS-LABEL: define {{[^@]+}}@test_addressing
-; FNATTRS-SAME: (ptr nocapture writeonly initializes((8, 12)) [[P:%.*]]) #[[ATTR3]] {
+; FNATTRS-SAME: (ptr writeonly captures(none) initializes((8, 12)) [[P:%.*]]) #[[ATTR3]] {
 ; FNATTRS-NEXT:    [[GEP:%.*]] = getelementptr i8, ptr [[P]], i64 8
 ; FNATTRS-NEXT:    store i32 0, ptr [[GEP]], align 4
 ; FNATTRS-NEXT:    ret void
 ;
 ; ATTRIBUTOR: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: write)
 ; ATTRIBUTOR-LABEL: define {{[^@]+}}@test_addressing
-; ATTRIBUTOR-SAME: (ptr nocapture nofree writeonly [[P:%.*]]) #[[ATTR3]] {
+; ATTRIBUTOR-SAME: (ptr nofree writeonly captures(none) [[P:%.*]]) #[[ATTR3]] {
 ; ATTRIBUTOR-NEXT:    [[GEP:%.*]] = getelementptr i8, ptr [[P]], i64 8
 ; ATTRIBUTOR-NEXT:    store i32 0, ptr [[GEP]], align 4
 ; ATTRIBUTOR-NEXT:    ret void
@@ -127,14 +127,14 @@ define void @test_addressing(ptr %p) {
 define void @test_readwrite(ptr %p) {
 ; FNATTRS: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: readwrite)
 ; FNATTRS-LABEL: define {{[^@]+}}@test_readwrite
-; FNATTRS-SAME: (ptr nocapture [[P:%.*]]) #[[ATTR5:[0-9]+]] {
+; FNATTRS-SAME: (ptr captures(none) [[P:%.*]]) #[[ATTR5:[0-9]+]] {
 ; FNATTRS-NEXT:    [[V:%.*]] = load i8, ptr [[P]], align 1
 ; FNATTRS-NEXT:    store i8 [[V]], ptr [[P]], align 1
 ; FNATTRS-NEXT:    ret void
 ;
 ; ATTRIBUTOR: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: readwrite)
 ; ATTRIBUTOR-LABEL: define {{[^@]+}}@test_readwrite
-; ATTRIBUTOR-SAME: (ptr nocapture nofree nonnull [[P:%.*]]) #[[ATTR5:[0-9]+]] {
+; ATTRIBUTOR-SAME: (ptr nofree nonnull captures(none) [[P:%.*]]) #[[ATTR5:[0-9]+]] {
 ; ATTRIBUTOR-NEXT:    [[V:%.*]] = load i8, ptr [[P]], align 1
 ; ATTRIBUTOR-NEXT:    store i8 [[V]], ptr [[P]], align 1
 ; ATTRIBUTOR-NEXT:    ret void
@@ -164,13 +164,13 @@ define void @test_volatile(ptr %p) {
 define void @test_atomicrmw(ptr %p) {
 ; FNATTRS: Function Attrs: mustprogress nofree norecurse nounwind willreturn memory(argmem: readwrite)
 ; FNATTRS-LABEL: define {{[^@]+}}@test_atomicrmw
-; FNATTRS-SAME: (ptr nocapture [[P:%.*]]) #[[ATTR7:[0-9]+]] {
+; FNATTRS-SAME: (ptr captures(none) [[P:%.*]]) #[[ATTR7:[0-9]+]] {
 ; FNATTRS-NEXT:    [[TMP1:%.*]] = atomicrmw add ptr [[P]], i8 0 seq_cst, align 1
 ; FNATTRS-NEXT:    ret void
 ;
 ; ATTRIBUTOR: Function Attrs: mustprogress nofree norecurse nounwind willreturn memory(argmem: readwrite)
 ; ATTRIBUTOR-LABEL: define {{[^@]+}}@test_atomicrmw
-; ATTRIBUTOR-SAME: (ptr nocapture nofree nonnull [[P:%.*]]) #[[ATTR6]] {
+; ATTRIBUTOR-SAME: (ptr nofree nonnull captures(none) [[P:%.*]]) #[[ATTR6]] {
 ; ATTRIBUTOR-NEXT:    [[TMP1:%.*]] = atomicrmw add ptr [[P]], i8 0 seq_cst, align 1
 ; ATTRIBUTOR-NEXT:    ret void
 ;
@@ -236,14 +236,14 @@ define void @direct2(ptr %p) {
 define void @direct2b(ptr %p) {
 ; FNATTRS: Function Attrs: memory(write)
 ; FNATTRS-LABEL: define {{[^@]+}}@direct2b
-; FNATTRS-SAME: (ptr nocapture writeonly [[P:%.*]]) #[[ATTR9]] {
-; FNATTRS-NEXT:    call void @direct2_callee(ptr nocapture [[P]])
+; FNATTRS-SAME: (ptr writeonly captures(none) [[P:%.*]]) #[[ATTR9]] {
+; FNATTRS-NEXT:    call void @direct2_callee(ptr captures(none) [[P]])
 ; FNATTRS-NEXT:    ret void
 ;
 ; ATTRIBUTOR: Function Attrs: memory(write)
 ; ATTRIBUTOR-LABEL: define {{[^@]+}}@direct2b
-; ATTRIBUTOR-SAME: (ptr nocapture writeonly [[P:%.*]]) #[[ATTR8]] {
-; ATTRIBUTOR-NEXT:    call void @direct2_callee(ptr nocapture writeonly [[P]]) #[[ATTR8]]
+; ATTRIBUTOR-SAME: (ptr writeonly captures(none) [[P:%.*]]) #[[ATTR8]] {
+; ATTRIBUTOR-NEXT:    call void @direct2_callee(ptr writeonly captures(none) [[P]]) #[[ATTR8]]
 ; ATTRIBUTOR-NEXT:    ret void
 ;
   call void @direct2_callee(ptr nocapture %p)
@@ -254,13 +254,13 @@ declare void @direct3_callee(ptr nocapture writeonly %p)
 
 define void @direct3(ptr %p) {
 ; FNATTRS-LABEL: define {{[^@]+}}@direct3
-; FNATTRS-SAME: (ptr nocapture writeonly [[P:%.*]]) {
+; FNATTRS-SAME: (ptr writeonly captures(none) [[P:%.*]]) {
 ; FNATTRS-NEXT:    call void @direct3_callee(ptr [[P]])
 ; FNATTRS-NEXT:    ret void
 ;
 ; ATTRIBUTOR-LABEL: define {{[^@]+}}@direct3
-; ATTRIBUTOR-SAME: (ptr nocapture writeonly [[P:%.*]]) {
-; ATTRIBUTOR-NEXT:    call void @direct3_callee(ptr nocapture writeonly [[P]])
+; ATTRIBUTOR-SAME: (ptr writeonly captures(none) [[P:%.*]]) {
+; ATTRIBUTOR-NEXT:    call void @direct3_callee(ptr writeonly captures(none) [[P]])
 ; ATTRIBUTOR-NEXT:    ret void
 ;
   call void @direct3_callee(ptr %p)
@@ -279,13 +279,13 @@ define void @direct3b(ptr %p) {
 
 define void @direct3c(ptr %p) {
 ; FNATTRS-LABEL: define {{[^@]+}}@direct3c
-; FNATTRS-SAME: (ptr nocapture [[P:%.*]]) {
+; FNATTRS-SAME: (ptr captures(none) [[P:%.*]]) {
 ; FNATTRS-NEXT:    call void @direct3_callee(ptr [[P]]) [ "may-read"() ]
 ; FNATTRS-NEXT:    ret void
 ;
 ; ATTRIBUTOR-LABEL: define {{[^@]+}}@direct3c
-; ATTRIBUTOR-SAME: (ptr nocapture [[P:%.*]]) {
-; ATTRIBUTOR-NEXT:    call void @direct3_callee(ptr nocapture [[P]]) [ "may-read"() ]
+; ATTRIBUTOR-SAME: (ptr captures(none) [[P:%.*]]) {
+; ATTRIBUTOR-NEXT:    call void @direct3_callee(ptr captures(none) [[P]]) [ "may-read"() ]
 ; ATTRIBUTOR-NEXT:    ret void
 ;
   call void @direct3_callee(ptr %p) ["may-read"()]
@@ -294,12 +294,12 @@ define void @direct3c(ptr %p) {
 
 define void @fptr_test1(ptr %p, ptr %f) {
 ; FNATTRS-LABEL: define {{[^@]+}}@fptr_test1
-; FNATTRS-SAME: (ptr [[P:%.*]], ptr nocapture readonly [[F:%.*]]) {
+; FNATTRS-SAME: (ptr [[P:%.*]], ptr readonly captures(none) [[F:%.*]]) {
 ; FNATTRS-NEXT:    call void [[F]](ptr [[P]])
 ; FNATTRS-NEXT:    ret void
 ;
 ; ATTRIBUTOR-LABEL: define {{[^@]+}}@fptr_test1
-; ATTRIBUTOR-SAME: (ptr [[P:%.*]], ptr nocapture nofree nonnull [[F:%.*]]) {
+; ATTRIBUTOR-SAME: (ptr [[P:%.*]], ptr nofree nonnull captures(none) [[F:%.*]]) {
 ; ATTRIBUTOR-NEXT:    call void [[F]](ptr [[P]])
 ; ATTRIBUTOR-NEXT:    ret void
 ;
@@ -309,13 +309,13 @@ define void @fptr_test1(ptr %p, ptr %f) {
 
 define void @fptr_test2(ptr %p, ptr %f) {
 ; FNATTRS-LABEL: define {{[^@]+}}@fptr_test2
-; FNATTRS-SAME: (ptr nocapture writeonly [[P:%.*]], ptr nocapture readonly [[F:%.*]]) {
-; FNATTRS-NEXT:    call void [[F]](ptr nocapture writeonly [[P]])
+; FNATTRS-SAME: (ptr writeonly captures(none) [[P:%.*]], ptr readonly captures(none) [[F:%.*]]) {
+; FNATTRS-NEXT:    call void [[F]](ptr writeonly captures(none) [[P]])
 ; FNATTRS-NEXT:    ret void
 ;
 ; ATTRIBUTOR-LABEL: define {{[^@]+}}@fptr_test2
-; ATTRIBUTOR-SAME: (ptr nocapture [[P:%.*]], ptr nocapture nofree nonnull [[F:%.*]]) {
-; ATTRIBUTOR-NEXT:    call void [[F]](ptr nocapture writeonly [[P]])
+; ATTRIBUTOR-SAME: (ptr captures(none) [[P:%.*]], ptr nofree nonnull captures(none) [[F:%.*]]) {
+; ATTRIBUTOR-NEXT:    call void [[F]](ptr writeonly captures(none) [[P]])
 ; ATTRIBUTOR-NEXT:    ret void
 ;
   call void %f(ptr nocapture writeonly %p)
@@ -325,14 +325,14 @@ define void @fptr_test2(ptr %p, ptr %f) {
 define void @fptr_test3(ptr %p, ptr %f) {
 ; FNATTRS: Function Attrs: memory(write)
 ; FNATTRS-LABEL: define {{[^@]+}}@fptr_test3
-; FNATTRS-SAME: (ptr nocapture writeonly [[P:%.*]], ptr nocapture readonly [[F:%.*]]) #[[ATTR9]] {
-; FNATTRS-NEXT:    call void [[F]](ptr nocapture [[P]]) #[[ATTR9]]
+; FNATTRS-SAME: (ptr writeonly captures(none) [[P:%.*]], ptr readonly captures(none) [[F:%.*]]) #[[ATTR9]] {
+; FNATTRS-NEXT:    call void [[F]](ptr captures(none) [[P]]) #[[ATTR9]]
 ; FNATTRS-NEXT:    ret void
 ;
 ; ATTRIBUTOR: Function Attrs: memory(write)
 ; ATTRIBUTOR-LABEL: define {{[^@]+}}@fptr_test3
-; ATTRIBUTOR-SAME: (ptr nocapture writeonly [[P:%.*]], ptr nocapture nofree nonnull writeonly [[F:%.*]]) #[[ATTR8]] {
-; ATTRIBUTOR-NEXT:    call void [[F]](ptr nocapture [[P]]) #[[ATTR8]]
+; ATTRIBUTOR-SAME: (ptr writeonly captures(none) [[P:%.*]], ptr nofree nonnull writeonly captures(none) [[F:%.*]]) #[[ATTR8]] {
+; ATTRIBUTOR-NEXT:    call void [[F]](ptr captures(none) [[P]]) #[[ATTR8]]
 ; ATTRIBUTOR-NEXT:    ret void
 ;
   call void %f(ptr nocapture %p) writeonly
@@ -341,13 +341,13 @@ define void @fptr_test3(ptr %p, ptr %f) {
 
 define void @test_argmem_none_callee(ptr %p) {
 ; FNATTRS-LABEL: define {{[^@]+}}@test_argmem_none_callee
-; FNATTRS-SAME: (ptr nocapture readnone [[P:%.*]]) {
-; FNATTRS-NEXT:    call void @direct1_callee(ptr nocapture [[P]]) #[[ATTR10:[0-9]+]]
+; FNATTRS-SAME: (ptr readnone captures(none) [[P:%.*]]) {
+; FNATTRS-NEXT:    call void @direct1_callee(ptr captures(none) [[P]]) #[[ATTR10:[0-9]+]]
 ; FNATTRS-NEXT:    ret void
 ;
 ; ATTRIBUTOR-LABEL: define {{[^@]+}}@test_argmem_none_callee
-; ATTRIBUTOR-SAME: (ptr nocapture [[P:%.*]]) {
-; ATTRIBUTOR-NEXT:    call void @direct1_callee(ptr nocapture [[P]]) #[[ATTR10:[0-9]+]]
+; ATTRIBUTOR-SAME: (ptr captures(none) [[P:%.*]]) {
+; ATTRIBUTOR-NEXT:    call void @direct1_callee(ptr captures(none) [[P]]) #[[ATTR10:[0-9]+]]
 ; ATTRIBUTOR-NEXT:    ret void
 ;
   call void @direct1_callee(ptr nocapture %p) memory(readwrite, argmem: none)
@@ -356,13 +356,13 @@ define void @test_argmem_none_callee(ptr %p) {
 
 define void @test_argmem_read_callee(ptr %p) {
 ; FNATTRS-LABEL: define {{[^@]+}}@test_argmem_read_callee
-; FNATTRS-SAME: (ptr nocapture readonly [[P:%.*]]) {
-; FNATTRS-NEXT:    call void @direct1_callee(ptr nocapture [[P]]) #[[ATTR11:[0-9]+]]
+; FNATTRS-SAME: (ptr readonly captures(none) [[P:%.*]]) {
+; FNATTRS-NEXT:    call void @direct1_callee(ptr captures(none) [[P]]) #[[ATTR11:[0-9]+]]
 ; FNATTRS-NEXT:    ret void
 ;
 ; ATTRIBUTOR-LABEL: define {{[^@]+}}@test_argmem_read_callee
-; ATTRIBUTOR-SAME: (ptr nocapture [[P:%.*]]) {
-; ATTRIBUTOR-NEXT:    call void @direct1_callee(ptr nocapture [[P]]) #[[ATTR11:[0-9]+]]
+; ATTRIBUTOR-SAME: (ptr captures(none) [[P:%.*]]) {
+; ATTRIBUTOR-NEXT:    call void @direct1_callee(ptr captures(none) [[P]]) #[[ATTR11:[0-9]+]]
 ; ATTRIBUTOR-NEXT:    ret void
 ;
   call void @direct1_callee(ptr nocapture %p) memory(readwrite, argmem: read)
@@ -371,13 +371,13 @@ define void @test_argmem_read_callee(ptr %p) {
 
 define void @test_argmem_write_callee(ptr %p) {
 ; FNATTRS-LABEL: define {{[^@]+}}@test_argmem_write_callee
-; FNATTRS-SAME: (ptr nocapture writeonly [[P:%.*]]) {
-; FNATTRS-NEXT:    call void @direct1_callee(ptr nocapture [[P]]) #[[ATTR12:[0-9]+]]
+; FNATTRS-SAME: (ptr writeonly captures(none) [[P:%.*]]) {
+; FNATTRS-NEXT:    call void @direct1_callee(ptr captures(none) [[P]]) #[[ATTR12:[0-9]+]]
 ; FNATTRS-NEXT:    ret void
 ;
 ; ATTRIBUTOR-LABEL: define {{[^@]+}}@test_argmem_write_callee
-; ATTRIBUTOR-SAME: (ptr nocapture [[P:%.*]]) {
-; ATTRIBUTOR-NEXT:    call void @direct1_callee(ptr nocapture [[P]]) #[[ATTR12:[0-9]+]]
+; ATTRIBUTOR-SAME: (ptr captures(none) [[P:%.*]]) {
+; ATTRIBUTOR-NEXT:    call void @direct1_callee(ptr captures(none) [[P]]) #[[ATTR12:[0-9]+]]
 ; ATTRIBUTOR-NEXT:    ret void
 ;
   call void @direct1_callee(ptr nocapture %p) memory(readwrite, argmem: write)
diff --git a/llvm/test/Transforms/FunctionSpecialization/function-specialization2.ll b/llvm/test/Transforms/FunctionSpecialization/function-specialization2.ll
index 6f36a394979d8..eaec98d3522e5 100644
--- a/llvm/test/Transforms/FunctionSpecialization/function-specialization2.ll
+++ b/llvm/test/Transforms/FunctionSpecialization/function-specialization2.ll
@@ -46,7 +46,7 @@ define i32 @main(ptr %0, i32 %1) {
 }
 
 ; CHECK-LABEL: define internal void @increment(
-; CHECK-SAME: ptr nocapture [[TMP0:%.*]]) {
+; CHECK-SAME: ptr captures(none) [[TMP0:%.*]]) {
 ; CHECK-NEXT:    [[TMP2:%.*]] = load i32, ptr [[TMP0]], align 4
 ; CHECK-NEXT:    [[TMP3:%.*]] = add nsw i32 [[TMP2]], 1
 ; CHECK-NEXT:    store i32 [[TMP3]], ptr [[TMP0]], align 4
@@ -54,7 +54,7 @@ define i32 @main(ptr %0, i32 %1) {
 ;
 ;
 ; CHECK-LABEL: define internal void @decrement(
-; CHECK-SAME: ptr nocapture [[TMP0:%.*]]) {
+; CHECK-SAME: ptr captures(none) [[TMP0:%.*]]) {
 ; CHECK-NEXT:    [[TMP2:%.*]] = load i32, ptr [[TMP0]], align 4
 ; CHECK-NEXT:    [[TMP3:%.*]] = add nsw i32 [[TMP2]], -1
 ; CHECK-NEXT:    store i32 [[TMP3]], ptr [[TMP0]], align 4
@@ -109,7 +109,7 @@ define i32 @main(ptr %0, i32 %1) {
 ;
 ;
 ; ONE-ITER-LABEL: define internal void @increment(
-; ONE-ITER-SAME: ptr nocapture [[TMP0:%.*]]) {
+; ONE-ITER-SAME: ptr captures(none) [[TMP0:%.*]]) {
 ; ONE-ITER-NEXT:    [[TMP2:%.*]] = load i32, ptr [[TMP0]], align 4
 ; ONE-ITER-NEXT:    [[TMP3:%.*]] = add nsw i32 [[TMP2]], 1
 ; ONE-ITER-NEXT:    store i32 [[TMP3]], ptr [[TMP0]], align 4
@@ -117,7 +117,7 @@ define i32 @main(ptr %0, i32 %1) {
 ;
 ;
 ; ONE-ITER-LABEL: define internal void @decrement(
-; ONE-ITER-SAME: ptr nocapture [[TMP0:%.*]]) {
+; ONE-ITER-SAME: ptr captures(none) [[TMP0:%.*]]) {
 ; ONE-ITER-NEXT:    [[TMP2:%.*]] = load i32, ptr [[TMP0]], align 4
 ; ONE-ITER-NEXT:    [[TMP3:%.*]] = add nsw i32 [[TMP2]], -1
 ; ONE-ITER-NEXT:    store i32 [[TMP3]], ptr [[TMP0]], align 4
@@ -172,7 +172,7 @@ define i32 @main(ptr %0, i32 %1) {
 ;
 ;
 ; DISABLED-LABEL: define internal void @func(
-; DISABLED-SAME: ptr [[TMP0:%.*]], i32 [[TMP1:%.*]], ptr nocapture [[TMP2:%.*]]) {
+; DISABLED-SAME: ptr [[TMP0:%.*]], i32 [[TMP1:%.*]], ptr captures(none) [[TMP2:%.*]]) {
 ; DISABLED-NEXT:    [[TMP4:%.*]] = alloca i32, align 4
 ; DISABLED-NEXT:    store i32 [[TMP1]], ptr [[TMP4]], align 4
 ; DISABLED-NEXT:    [[TMP5:%.*]] = load i32, ptr [[TMP4]], align 4
@@ -192,7 +192,7 @@ define i32 @main(ptr %0, i32 %1) {
 ;
 ;
 ; DISABLED-LABEL: define internal void @increment(
-; DISABLED-SAME: ptr nocapture [[TMP0:%.*]]) {
+; DISABLED-SAME: ptr captures(none) [[TMP0:%.*]]) {
 ; DISABLED-NEXT:    [[TMP2:%.*]] = load i32, ptr [[TMP0]], align 4
 ; DISABLED-NEXT:    [[TMP3:%.*]] = add nsw i32 [[TMP2]], 1
 ; DISABLED-NEXT:    store i32 [[TMP3]], ptr [[TMP0]], align 4
@@ -200,7 +200,7 @@ define i32 @main(ptr %0, i32 %1) {
 ;
 ;
 ; DISABLED-LABEL: define internal void @decrement(
-; DISABLED-SAME: ptr nocapture [[TMP0:%.*]]) {
+; DISABLED-SAME: ptr captures(none) [[TMP0:%.*]]) {
 ; DISABLED-NEXT:    [[TMP2:%.*]] = load i32, ptr [[TMP0]], align 4
 ; DISABLED-NEXT:    [[TMP3:%.*]] = add nsw i32 [[TMP2]], -1
 ; DISABLED-NEXT:    store i32 [[TMP3]], ptr [[TMP0]], align 4
diff --git a/llvm/test/Transforms/FunctionSpecialization/solver-constant-strictfpmetadata.ll b/llvm/test/Transforms/FunctionSpecialization/solver-constant-strictfpmetadata.ll
new file mode 100644
index 0000000000000..99224b4efba6b
--- /dev/null
+++ b/llvm/test/Transforms/FunctionSpecialization/solver-constant-strictfpmetadata.ll
@@ -0,0 +1,17 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -passes=ipsccp -force-specialization -S < %s | FileCheck %s
+
+define float @test(ptr %this, float %cm, i1 %0) strictfp {
+; CHECK-LABEL: define float @test(
+; CHECK-SAME: ptr [[THIS:%.*]], float [[CM:%.*]], i1 [[TMP0:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[CMP:%.*]] = call i1 @llvm.experimental.constrained.fcmps.f32(float [[CM]], float 0.000000e+00, metadata !"ole", metadata !"fpexcept.strict")
+; CHECK-NEXT:    [[CALL295:%.*]] = call float @test.specialized.1(ptr null, float 0.000000e+00, i1 false)
+; CHECK-NEXT:    ret float 0.000000e+00
+;
+entry:
+  %cmp = call i1 @llvm.experimental.constrained.fcmps.f32(float %cm, float 0.000000e+00, metadata !"ole", metadata !"fpexcept.strict") #0
+  %call295 = call float @test(ptr null, float 0.000000e+00, i1 false) #0
+  ret float 0.000000e+00
+}
+
diff --git a/llvm/test/Transforms/GVN/PRE/invariant-load.ll b/llvm/test/Transforms/GVN/PRE/invariant-load.ll
index 79cf95155c70c..71332bf0e0bdb 100644
--- a/llvm/test/Transforms/GVN/PRE/invariant-load.ll
+++ b/llvm/test/Transforms/GVN/PRE/invariant-load.ll
@@ -4,9 +4,9 @@
 
 define i32 @test1(ptr nocapture %p, ptr nocapture %q) {
 ; CHECK-LABEL: define i32 @test1
-; CHECK-SAME: (ptr nocapture [[P:%.*]], ptr nocapture [[Q:%.*]]) {
+; CHECK-SAME: (ptr captures(none) [[P:%.*]], ptr captures(none) [[Q:%.*]]) {
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[X:%.*]] = load i32, ptr [[P]], align 4, !invariant.load !0
+; CHECK-NEXT:    [[X:%.*]] = load i32, ptr [[P]], align 4, !invariant.load [[META0:![0-9]+]]
 ; CHECK-NEXT:    [[CONV:%.*]] = trunc i32 [[X]] to i8
 ; CHECK-NEXT:    store i8 [[CONV]], ptr [[Q]], align 1
 ; CHECK-NEXT:    [[ADD:%.*]] = add i32 [[X]], 1
@@ -23,7 +23,7 @@ entry:
 
 define i32 @test2(ptr nocapture %p, ptr nocapture %q) {
 ; CHECK-LABEL: define i32 @test2
-; CHECK-SAME: (ptr nocapture [[P:%.*]], ptr nocapture [[Q:%.*]]) {
+; CHECK-SAME: (ptr captures(none) [[P:%.*]], ptr captures(none) [[Q:%.*]]) {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[X:%.*]] = load i32, ptr [[P]], align 4
 ; CHECK-NEXT:    [[CONV:%.*]] = trunc i32 [[X]] to i8
@@ -73,7 +73,7 @@ define i32 @test4(i1 %cnd, ptr %p, ptr %q) {
 ; CHECK-LABEL: define i32 @test4
 ; CHECK-SAME: (i1 [[CND:%.*]], ptr [[P:%.*]], ptr [[Q:%.*]]) {
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[V1:%.*]] = load i32, ptr [[P]], align 4, !invariant.load !0
+; CHECK-NEXT:    [[V1:%.*]] = load i32, ptr [[P]], align 4, !invariant.load [[META0]]
 ; CHECK-NEXT:    br i1 [[CND]], label [[BB1:%.*]], label [[BB2:%.*]]
 ; CHECK:       bb1:
 ; CHECK-NEXT:    store i32 5, ptr [[Q]], align 4
@@ -158,11 +158,11 @@ define i32 @test8(i1 %cnd, ptr %p) {
 ; CHECK-LABEL: define i32 @test8
 ; CHECK-SAME: (i1 [[CND:%.*]], ptr [[P:%.*]]) {
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[V1:%.*]] = load i32, ptr [[P]], align 4, !invariant.load !0
+; CHECK-NEXT:    [[V1:%.*]] = load i32, ptr [[P]], align 4, !invariant.load [[META0]]
 ; CHECK-NEXT:    br i1 [[CND]], label [[TAKEN:%.*]], label [[MERGE:%.*]]
 ; CHECK:       taken:
 ; CHECK-NEXT:    [[P2:%.*]] = call ptr (...) @bar(ptr [[P]])
-; CHECK-NEXT:    [[V2_PRE:%.*]] = load i32, ptr [[P2]], align 4, !invariant.load !0
+; CHECK-NEXT:    [[V2_PRE:%.*]] = load i32, ptr [[P2]], align 4, !invariant.load [[META0]]
 ; CHECK-NEXT:    br label [[MERGE]]
 ; CHECK:       merge:
 ; CHECK-NEXT:    [[V2:%.*]] = phi i32 [ [[V1]], [[ENTRY:%.*]] ], [ [[V2_PRE]], [[TAKEN]] ]
@@ -185,9 +185,9 @@ merge:
 
 define i32 @metadata_preservation(ptr nocapture %p, ptr nocapture %q) {
 ; CHECK-LABEL: define i32 @metadata_preservation
-; CHECK-SAME: (ptr nocapture [[P:%.*]], ptr nocapture [[Q:%.*]]) {
+; CHECK-SAME: (ptr captures(none) [[P:%.*]], ptr captures(none) [[Q:%.*]]) {
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[X:%.*]] = load i32, ptr [[P]], align 4, !invariant.load !0
+; CHECK-NEXT:    [[X:%.*]] = load i32, ptr [[P]], align 4, !invariant.load [[META0]]
 ; CHECK-NEXT:    [[ADD:%.*]] = add i32 [[X]], [[X]]
 ; CHECK-NEXT:    ret i32 [[ADD]]
 ;
diff --git a/llvm/test/Transforms/GVN/fake-use-constprop.ll b/llvm/test/Transforms/GVN/fake-use-constprop.ll
index 1466f9f9fca27..0e7ca101a2524 100644
--- a/llvm/test/Transforms/GVN/fake-use-constprop.ll
+++ b/llvm/test/Transforms/GVN/fake-use-constprop.ll
@@ -7,7 +7,7 @@
 ; with respect to keeping the variable live up until the fake.use.
 ; This test checks that we don't generate any fake.uses with constant 0.
 ;
-; Reduced from the following test case, generated with clang -O2 -S -emit-llvm -fextend-lifetimes test.c
+; Reduced from the following test case, generated with clang -O2 -S -emit-llvm -fextend-variable-liveness test.c
 ;
 ; extern void func1();
 ; extern int bar();
diff --git a/llvm/test/Transforms/GVN/vscale.ll b/llvm/test/Transforms/GVN/vscale.ll
index 67cbfc2f05ef8..d7b07b9891c41 100644
--- a/llvm/test/Transforms/GVN/vscale.ll
+++ b/llvm/test/Transforms/GVN/vscale.ll
@@ -641,3 +641,127 @@ entry:
   call void @llvm.lifetime.end.p0(i64 -1, ptr nonnull %ref.tmp)
   ret { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %15
 }
+
+define <vscale x 4 x float> @scalable_store_to_fixed_load(<vscale x 4 x float> %.coerce) vscale_range(4,4) {
+; CHECK-LABEL: @scalable_store_to_fixed_load(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[RETVAL:%.*]] = alloca { <16 x float> }, align 64
+; CHECK-NEXT:    [[TMP0:%.*]] = fadd <vscale x 4 x float> [[DOTCOERCE:%.*]], [[DOTCOERCE]]
+; CHECK-NEXT:    store <vscale x 4 x float> [[TMP0]], ptr [[RETVAL]], align 16
+; CHECK-NEXT:    ret <vscale x 4 x float> [[TMP0]]
+;
+entry:
+  %retval = alloca { <16 x float> }
+  %0 = fadd <vscale x 4 x float> %.coerce, %.coerce
+  store <vscale x 4 x float> %0, ptr %retval
+  %1 = load <16 x float>, ptr %retval
+  %cast.scalable = tail call <vscale x 4 x float> @llvm.vector.insert.nxv4f32.v16f32(<vscale x 4 x float> poison, <16 x float> %1, i64 0)
+  ret <vscale x 4 x float> %cast.scalable
+}
+
+; Here, only the lower bound for the vscale is known, but this is enough to allow a forward to a load to 16 elements.
+define <vscale x 4 x float> @scalable_store_to_fixed_load_only_lower_bound(<vscale x 4 x float> %a) vscale_range(4) {
+; CHECK-LABEL: @scalable_store_to_fixed_load_only_lower_bound(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[RETVAL:%.*]] = alloca { <vscale x 4 x float> }, align 16
+; CHECK-NEXT:    store <vscale x 4 x float> [[A:%.*]], ptr [[RETVAL]], align 16
+; CHECK-NEXT:    ret <vscale x 4 x float> [[A]]
+;
+entry:
+  %retval = alloca { <vscale x 4 x float> }
+  store <vscale x 4 x float> %a, ptr %retval
+  %1 = load <16 x float>, ptr %retval
+  %cast.scalable = tail call <vscale x 4 x float> @llvm.vector.insert.nxv4f32.v16f32(<vscale x 4 x float> poison, <16 x float> %1, i64 0)
+  ret <vscale x 4 x float> %cast.scalable
+}
+
+define <vscale x 4 x float> @scalable_store_to_fixed_load_with_offset(<vscale x 4 x float> %a) vscale_range(4,4) {
+; CHECK-LABEL: @scalable_store_to_fixed_load_with_offset(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[PTR:%.*]] = alloca { <32 x float> }, align 128
+; CHECK-NEXT:    store <vscale x 4 x float> [[A:%.*]], ptr [[PTR]], align 16
+; CHECK-NEXT:    [[GEP:%.*]] = getelementptr inbounds i8, ptr [[PTR]], i64 8
+; CHECK-NEXT:    [[TMP0:%.*]] = load <16 x float>, ptr [[GEP]], align 64
+; CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = tail call <vscale x 4 x float> @llvm.vector.insert.nxv4f32.v16f32(<vscale x 4 x float> poison, <16 x float> [[TMP0]], i64 0)
+; CHECK-NEXT:    ret <vscale x 4 x float> [[CAST_SCALABLE]]
+;
+entry:
+  %ptr = alloca { <32 x float> }
+  store <vscale x 4 x float> %a, ptr %ptr
+  %gep = getelementptr inbounds i8, ptr %ptr, i64 8
+  %1 = load <16 x float>, ptr %gep
+  %cast.scalable = tail call <vscale x 4 x float> @llvm.vector.insert.nxv4f32.v16f32(<vscale x 4 x float> poison, <16 x float> %1, i64 0)
+  ret <vscale x 4 x float> %cast.scalable
+}
+
+define <vscale x 4 x float> @scalable_store_to_fixed_load_unknown_vscale(<vscale x 4 x float> %.coerce) {
+; CHECK-LABEL: @scalable_store_to_fixed_load_unknown_vscale(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[RETVAL:%.*]] = alloca { <16 x float> }, align 64
+; CHECK-NEXT:    [[TMP0:%.*]] = fadd <vscale x 4 x float> [[DOTCOERCE:%.*]], [[DOTCOERCE]]
+; CHECK-NEXT:    store <vscale x 4 x float> [[TMP0]], ptr [[RETVAL]], align 16
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x float>, ptr [[RETVAL]], align 64
+; CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = tail call <vscale x 4 x float> @llvm.vector.insert.nxv4f32.v16f32(<vscale x 4 x float> poison, <16 x float> [[TMP1]], i64 0)
+; CHECK-NEXT:    ret <vscale x 4 x float> [[CAST_SCALABLE]]
+;
+entry:
+  %retval = alloca { <16 x float> }
+  %0 = fadd <vscale x 4 x float> %.coerce, %.coerce
+  store <vscale x 4 x float> %0, ptr %retval
+  %1 = load <16 x float>, ptr %retval
+  %cast.scalable = tail call <vscale x 4 x float> @llvm.vector.insert.nxv4f32.v16f32(<vscale x 4 x float> poison, <16 x float> %1, i64 0)
+  ret <vscale x 4 x float> %cast.scalable
+}
+
+define <vscale x 4 x float> @scalable_store_to_fixed_load_size_missmatch(<vscale x 4 x float> %.coerce) vscale_range(4,4) {
+; CHECK-LABEL: @scalable_store_to_fixed_load_size_missmatch(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[RETVAL:%.*]] = alloca { <32 x float> }, align 128
+; CHECK-NEXT:    [[TMP0:%.*]] = fadd <vscale x 4 x float> [[DOTCOERCE:%.*]], [[DOTCOERCE]]
+; CHECK-NEXT:    store <vscale x 4 x float> [[TMP0]], ptr [[RETVAL]], align 16
+; CHECK-NEXT:    [[TMP1:%.*]] = load <32 x float>, ptr [[RETVAL]], align 128
+; CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = tail call <vscale x 4 x float> @llvm.vector.insert.nxv4f32.v32f32(<vscale x 4 x float> poison, <32 x float> [[TMP1]], i64 0)
+; CHECK-NEXT:    ret <vscale x 4 x float> [[CAST_SCALABLE]]
+;
+entry:
+  %retval = alloca { <32 x float> }
+  %0 = fadd <vscale x 4 x float> %.coerce, %.coerce
+  store <vscale x 4 x float> %0, ptr %retval
+  %1 = load <32 x float>, ptr %retval
+  %cast.scalable = tail call <vscale x 4 x float> @llvm.vector.insert.nxv4f32.v32f32(<vscale x 4 x float> poison, <32 x float> %1, i64 0)
+  ret <vscale x 4 x float> %cast.scalable
+}
+
+define <vscale x 4 x i32> @scalable_store_to_fixed_load_different_types(<vscale x 4 x float> %a) vscale_range(4,4) {
+; CHECK-LABEL: @scalable_store_to_fixed_load_different_types(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[PTR:%.*]] = alloca { <16 x float> }, align 64
+; CHECK-NEXT:    store <vscale x 4 x float> [[A:%.*]], ptr [[PTR]], align 16
+; CHECK-NEXT:    [[TMP0:%.*]] = load <16 x i32>, ptr [[PTR]], align 64
+; CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = tail call <vscale x 4 x i32> @llvm.vector.insert.nxv4i32.v16i32(<vscale x 4 x i32> poison, <16 x i32> [[TMP0]], i64 0)
+; CHECK-NEXT:    ret <vscale x 4 x i32> [[CAST_SCALABLE]]
+;
+entry:
+  %ptr = alloca { <16 x float> }
+  store <vscale x 4 x float> %a, ptr %ptr
+  %1 = load <16 x i32>, ptr %ptr
+  %cast.scalable = tail call <vscale x 4 x i32> @llvm.vector.insert.nxv4i32.v16i32(<vscale x 4 x i32> poison, <16 x i32> %1, i64 0)
+  ret <vscale x 4 x i32> %cast.scalable
+}
+
+; This function does not have a fixed vscale, but the loaded vector is still known
+; to be smaller or equal in size compared to the stored vector.
+define <4 x float> @scalable_store_to_small_fixed_load(<vscale x 4 x float> %a) {
+; CHECK-LABEL: @scalable_store_to_small_fixed_load(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[PTR:%.*]] = alloca <vscale x 4 x float>, align 16
+; CHECK-NEXT:    store <vscale x 4 x float> [[A:%.*]], ptr [[PTR]], align 16
+; CHECK-NEXT:    [[TMP0:%.*]] = call <4 x float> @llvm.vector.extract.v4f32.nxv4f32(<vscale x 4 x float> [[A]], i64 0)
+; CHECK-NEXT:    ret <4 x float> [[TMP0]]
+;
+entry:
+  %ptr = alloca <vscale x 4 x float>
+  store <vscale x 4 x float> %a, ptr %ptr
+  %1 = load <4 x float>, ptr %ptr
+  ret <4 x float> %1
+}
diff --git a/llvm/test/Transforms/GVNHoist/infinite-loop-indirect.ll b/llvm/test/Transforms/GVNHoist/infinite-loop-indirect.ll
index a7e6ff30d8b2f..8aa9003f3d809 100644
--- a/llvm/test/Transforms/GVNHoist/infinite-loop-indirect.ll
+++ b/llvm/test/Transforms/GVNHoist/infinite-loop-indirect.ll
@@ -12,7 +12,7 @@
 ; Check that the bitcast is not hoisted because it is after an indirect call
 define i32 @foo(ptr nocapture readonly %i) {
 ; CHECK-LABEL: define i32 @foo
-; CHECK-SAME: (ptr nocapture readonly [[I:%.*]]) {
+; CHECK-SAME: (ptr readonly captures(none) [[I:%.*]]) {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[AGG_TMP:%.*]] = alloca [[CLASS_BAR:%.*]], align 8
 ; CHECK-NEXT:    [[X:%.*]] = getelementptr inbounds [[CLASS_BAR]], ptr [[AGG_TMP]], i64 0, i32 1
@@ -64,7 +64,7 @@ sw.default:                                       ; preds = %entry
 ; there is no path to exit of the function.
 define i32 @foo1(ptr nocapture readonly %i) {
 ; CHECK-LABEL: define i32 @foo1
-; CHECK-SAME: (ptr nocapture readonly [[I:%.*]]) {
+; CHECK-SAME: (ptr readonly captures(none) [[I:%.*]]) {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[AGG_TMP:%.*]] = alloca [[CLASS_BAR:%.*]], align 8
 ; CHECK-NEXT:    [[X:%.*]] = getelementptr inbounds [[CLASS_BAR]], ptr [[AGG_TMP]], i64 0, i32 1
@@ -118,7 +118,7 @@ sw.default:                                       ; preds = %entry
 ; Check that bitcast is hoisted even when one of them is partially redundant.
 define i32 @test13(ptr %P, ptr %Ptr, ptr nocapture readonly %i) {
 ; CHECK-LABEL: define i32 @test13
-; CHECK-SAME: (ptr [[P:%.*]], ptr [[PTR:%.*]], ptr nocapture readonly [[I:%.*]]) {
+; CHECK-SAME: (ptr [[P:%.*]], ptr [[PTR:%.*]], ptr readonly captures(none) [[I:%.*]]) {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[AGG_TMP:%.*]] = alloca [[CLASS_BAR:%.*]], align 8
 ; CHECK-NEXT:    [[X:%.*]] = getelementptr inbounds [[CLASS_BAR]], ptr [[AGG_TMP]], i64 0, i32 1
@@ -165,7 +165,7 @@ F:
 ; do not have the bitcast instruction.
 define i32 @test14(ptr %P, ptr %Ptr, ptr nocapture readonly %i) {
 ; CHECK-LABEL: define i32 @test14
-; CHECK-SAME: (ptr [[P:%.*]], ptr [[PTR:%.*]], ptr nocapture readonly [[I:%.*]]) {
+; CHECK-SAME: (ptr [[P:%.*]], ptr [[PTR:%.*]], ptr readonly captures(none) [[I:%.*]]) {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[AGG_TMP:%.*]] = alloca [[CLASS_BAR:%.*]], align 8
 ; CHECK-NEXT:    [[X:%.*]] = getelementptr inbounds [[CLASS_BAR]], ptr [[AGG_TMP]], i64 0, i32 1
@@ -217,7 +217,7 @@ F:
 ; due to indirect branches
 define i32 @test16(ptr %P, ptr %Ptr, ptr nocapture readonly %i) {
 ; CHECK-LABEL: define i32 @test16
-; CHECK-SAME: (ptr [[P:%.*]], ptr [[PTR:%.*]], ptr nocapture readonly [[I:%.*]]) {
+; CHECK-SAME: (ptr [[P:%.*]], ptr [[PTR:%.*]], ptr readonly captures(none) [[I:%.*]]) {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[AGG_TMP:%.*]] = alloca [[CLASS_BAR:%.*]], align 8
 ; CHECK-NEXT:    [[X:%.*]] = getelementptr inbounds [[CLASS_BAR]], ptr [[AGG_TMP]], i64 0, i32 1
@@ -274,7 +274,7 @@ F:
 
 define i32 @foo2(ptr nocapture readonly %i) local_unnamed_addr personality ptr @__gxx_personality_v0 {
 ; CHECK-LABEL: define i32 @foo2
-; CHECK-SAME: (ptr nocapture readonly [[I:%.*]]) local_unnamed_addr personality ptr @__gxx_personality_v0 {
+; CHECK-SAME: (ptr readonly captures(none) [[I:%.*]]) local_unnamed_addr personality ptr @__gxx_personality_v0 {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[TMP0:%.*]] = load i32, ptr [[I]], align 4
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[TMP0]], 0
@@ -284,11 +284,11 @@ define i32 @foo2(ptr nocapture readonly %i) local_unnamed_addr personality ptr @
 ; CHECK-NEXT:    [[TMP1:%.*]] = bitcast ptr [[EXCEPTION]] to ptr
 ; CHECK-NEXT:    store i32 [[TMP0]], ptr [[TMP1]], align 16
 ; CHECK-NEXT:    invoke void @__cxa_throw(ptr [[EXCEPTION]], ptr @_ZTIi, ptr null) #[[ATTR2:[0-9]+]]
-; CHECK-NEXT:    to label [[UNREACHABLE:%.*]] unwind label [[LPAD:%.*]]
+; CHECK-NEXT:            to label [[UNREACHABLE:%.*]] unwind label [[LPAD:%.*]]
 ; CHECK:       lpad:
 ; CHECK-NEXT:    [[TMP2:%.*]] = landingpad { ptr, i32 }
-; CHECK-NEXT:    catch ptr @_ZTIi
-; CHECK-NEXT:    catch ptr null
+; CHECK-NEXT:            catch ptr @_ZTIi
+; CHECK-NEXT:            catch ptr null
 ; CHECK-NEXT:    [[BC1:%.*]] = add i32 [[TMP0]], 10
 ; CHECK-NEXT:    [[TMP3:%.*]] = extractvalue { ptr, i32 } [[TMP2]], 0
 ; CHECK-NEXT:    [[TMP4:%.*]] = extractvalue { ptr, i32 } [[TMP2]], 1
@@ -300,7 +300,7 @@ define i32 @foo2(ptr nocapture readonly %i) local_unnamed_addr personality ptr @
 ; CHECK-NEXT:    br i1 [[MATCHES]], label [[CATCH1:%.*]], label [[CATCH:%.*]]
 ; CHECK:       catch1:
 ; CHECK-NEXT:    invoke void @__cxa_rethrow() #[[ATTR2]]
-; CHECK-NEXT:    to label [[UNREACHABLE]] unwind label [[LPAD4:%.*]]
+; CHECK-NEXT:            to label [[UNREACHABLE]] unwind label [[LPAD4:%.*]]
 ; CHECK:       catch:
 ; CHECK-NEXT:    [[TMP7:%.*]] = load i32, ptr [[I]], align 4
 ; CHECK-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP7]], 1
@@ -308,11 +308,11 @@ define i32 @foo2(ptr nocapture readonly %i) local_unnamed_addr personality ptr @
 ; CHECK-NEXT:    br label [[TRY_CONT]]
 ; CHECK:       lpad4:
 ; CHECK-NEXT:    [[TMP8:%.*]] = landingpad { ptr, i32 }
-; CHECK-NEXT:    cleanup
+; CHECK-NEXT:            cleanup
 ; CHECK-NEXT:    [[BC5:%.*]] = add i32 [[TMP0]], 10
 ; CHECK-NEXT:    tail call void @__cxa_end_catch() #[[ATTR1]]
 ; CHECK-NEXT:    invoke void @__cxa_throw(ptr [[EXCEPTION]], ptr @_ZTIi, ptr null) #[[ATTR2]]
-; CHECK-NEXT:    to label [[UNREACHABLE]] unwind label [[LPAD]]
+; CHECK-NEXT:            to label [[UNREACHABLE]] unwind label [[LPAD]]
 ; CHECK:       try.cont:
 ; CHECK-NEXT:    [[K_0:%.*]] = phi i32 [ [[ADD]], [[CATCH]] ], [ 0, [[ENTRY:%.*]] ]
 ; CHECK-NEXT:    [[BC6:%.*]] = add i32 [[TMP0]], 10
diff --git a/llvm/test/Transforms/GlobalOpt/read-with-constexpr-users.ll b/llvm/test/Transforms/GlobalOpt/read-with-constexpr-users.ll
new file mode 100644
index 0000000000000..9df31887102f2
--- /dev/null
+++ b/llvm/test/Transforms/GlobalOpt/read-with-constexpr-users.ll
@@ -0,0 +1,36 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals all --version 5
+; RUN: opt -p globalopt -S %s | FileCheck %s
+
+@G = internal global [2 x i64 ] zeroinitializer
+@H = internal global [2 x i64 ] zeroinitializer
+
+;.
+; CHECK: @G = internal global [2 x i64] zeroinitializer
+; CHECK: @H = internal global [2 x i64] zeroinitializer
+;.
+define i64 @G_used_by_gep_inttoptr_exprs() {
+; CHECK-LABEL: define i64 @G_used_by_gep_inttoptr_exprs() local_unnamed_addr {
+; CHECK-NEXT:    [[L:%.*]] = load i64, ptr @G, align 8
+; CHECK-NEXT:    [[GEP:%.*]] = getelementptr i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr getelementptr inbounds nuw (i8, ptr @G, i64 16) to i64), i64 8) to ptr), i64 1
+; CHECK-NEXT:    [[C:%.*]] = icmp eq ptr [[GEP]], @G
+; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[C]], i64 [[L]], i64 9
+; CHECK-NEXT:    ret i64 [[SEL]]
+;
+  %l = load i64, ptr @G, align 8
+  %gep = getelementptr i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr getelementptr inbounds nuw (i8, ptr @G, i64 16) to i64), i64 8) to ptr), i64 1
+  %c = icmp eq ptr %gep, @G
+  %sel = select i1 %c, i64 %l, i64 9
+  ret i64 %sel
+}
+
+define i64 @H_constexpr_stored(ptr %p) {
+; CHECK-LABEL: define i64 @H_constexpr_stored(
+; CHECK-SAME: ptr [[P:%.*]]) local_unnamed_addr {
+; CHECK-NEXT:    [[L:%.*]] = load i64, ptr @H, align 8
+; CHECK-NEXT:    store i64 ptrtoint (ptr getelementptr inbounds nuw (i8, ptr @H, i64 16) to i64), ptr [[P]], align 4
+; CHECK-NEXT:    ret i64 [[L]]
+;
+  %l = load i64, ptr @H, align 8
+  store i64 ptrtoint (ptr getelementptr inbounds nuw (i8, ptr @H, i64 16) to i64), ptr %p
+  ret i64 %l
+}
diff --git a/llvm/test/Transforms/GlobalOpt/resolve-fmv-ifunc.ll b/llvm/test/Transforms/GlobalOpt/resolve-fmv-ifunc.ll
index fa817a8cbf417..4b6a19d3f05cf 100644
--- a/llvm/test/Transforms/GlobalOpt/resolve-fmv-ifunc.ll
+++ b/llvm/test/Transforms/GlobalOpt/resolve-fmv-ifunc.ll
@@ -221,7 +221,7 @@ resolver_entry:
 define i32 @caller4() #8 {
 ; CHECK-LABEL: define i32 @caller4(
 ; CHECK-SAME: ) local_unnamed_addr #[[ATTR7:[0-9]+]] {
-; CHECK:    [[CALL:%.*]] = tail call i32 @test_non_fmv_caller()
+; CHECK:    [[CALL:%.*]] = tail call i32 @test_non_fmv_caller._Maes()
 ;
 entry:
   %call = tail call i32 @test_non_fmv_caller()
diff --git a/llvm/test/Transforms/HotColdSplit/coldentrycount.ll b/llvm/test/Transforms/HotColdSplit/coldentrycount.ll
index 6e5ef1aa25392..1e8825e651ec4 100644
--- a/llvm/test/Transforms/HotColdSplit/coldentrycount.ll
+++ b/llvm/test/Transforms/HotColdSplit/coldentrycount.ll
@@ -27,9 +27,9 @@ declare void @sink() cold
 ; CHECK: define {{.*}} @fun.cold.1{{.*}} ![[PROF:[0-9]+]] {{.*}}section_prefix ![[UNLIKELY:[0-9]+]]
 
 ; CHECK: ![[HOTPROF]] = !{!"function_entry_count", i64 100}
-; CHECK: ![[LIKELY]] = !{!"function_section_prefix", !"hot"}
+; CHECK: ![[LIKELY]] = !{!"section_prefix", !"hot"}
 ; CHECK: ![[PROF]] = !{!"function_entry_count", i64 0}
-; CHECK: ![[UNLIKELY]] = !{!"function_section_prefix", !"unlikely"}
+; CHECK: ![[UNLIKELY]] = !{!"section_prefix", !"unlikely"}
 
 !llvm.module.flags = !{!0}
 !0 = !{i32 1, !"ProfileSummary", !1}
diff --git a/llvm/test/Transforms/IRCE/variable-loop-bounds.ll b/llvm/test/Transforms/IRCE/variable-loop-bounds.ll
index a46bb0f570846..a8b5eff75769a 100644
--- a/llvm/test/Transforms/IRCE/variable-loop-bounds.ll
+++ b/llvm/test/Transforms/IRCE/variable-loop-bounds.ll
@@ -3,7 +3,7 @@
 
 define void @test_inc_eq(ptr nocapture %a, ptr nocapture readonly %b, ptr nocapture readonly %c, i32 %N) {
 ; CHECK-LABEL: define void @test_inc_eq(
-; CHECK-SAME: ptr nocapture [[A:%.*]], ptr nocapture readonly [[B:%.*]], ptr nocapture readonly [[C:%.*]], i32 [[N:%.*]]) {
+; CHECK-SAME: ptr captures(none) [[A:%.*]], ptr readonly captures(none) [[B:%.*]], ptr readonly captures(none) [[C:%.*]], i32 [[N:%.*]]) {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[CMP16:%.*]] = icmp sgt i32 [[N]], 0
 ; CHECK-NEXT:    br i1 [[CMP16]], label [[FOR_BODY_PREHEADER:%.*]], label [[FOR_COND_CLEANUP:%.*]]
@@ -78,7 +78,7 @@ define void @test_inc_eq(ptr nocapture %a, ptr nocapture readonly %b, ptr nocapt
 ; CHECK:       for.inc.postloop:
 ; CHECK-NEXT:    [[INC_POSTLOOP]] = add nuw nsw i32 [[I_017_POSTLOOP]], 1
 ; CHECK-NEXT:    [[EXITCOND_POSTLOOP:%.*]] = icmp eq i32 [[INC_POSTLOOP]], [[N]]
-; CHECK-NEXT:    br i1 [[EXITCOND_POSTLOOP]], label [[FOR_COND_CLEANUP_LOOPEXIT_LOOPEXIT:%.*]], label [[FOR_BODY_POSTLOOP]], !llvm.loop [[LOOP0:![0-9]+]], !loop_constrainer.loop.clone !5
+; CHECK-NEXT:    br i1 [[EXITCOND_POSTLOOP]], label [[FOR_COND_CLEANUP_LOOPEXIT_LOOPEXIT:%.*]], label [[FOR_BODY_POSTLOOP]], !llvm.loop [[LOOP0:![0-9]+]], !loop_constrainer.loop.clone [[META5:![0-9]+]]
 ;
 entry:
   %cmp16 = icmp sgt i32 %N, 0
@@ -118,7 +118,7 @@ for.inc:
 
 define void @test_inc_ne(ptr nocapture %a, ptr nocapture readonly %b, ptr nocapture readonly %c, i32 %N) {
 ; CHECK-LABEL: define void @test_inc_ne(
-; CHECK-SAME: ptr nocapture [[A:%.*]], ptr nocapture readonly [[B:%.*]], ptr nocapture readonly [[C:%.*]], i32 [[N:%.*]]) {
+; CHECK-SAME: ptr captures(none) [[A:%.*]], ptr readonly captures(none) [[B:%.*]], ptr readonly captures(none) [[C:%.*]], i32 [[N:%.*]]) {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[CMP16:%.*]] = icmp sgt i32 [[N]], 0
 ; CHECK-NEXT:    br i1 [[CMP16]], label [[FOR_BODY_PREHEADER:%.*]], label [[FOR_COND_CLEANUP:%.*]]
@@ -192,7 +192,7 @@ define void @test_inc_ne(ptr nocapture %a, ptr nocapture readonly %b, ptr nocapt
 ; CHECK:       for.inc.postloop:
 ; CHECK-NEXT:    [[INC_POSTLOOP]] = add nuw nsw i32 [[I_017_POSTLOOP]], 1
 ; CHECK-NEXT:    [[EXITCOND_POSTLOOP:%.*]] = icmp ne i32 [[INC_POSTLOOP]], [[N]]
-; CHECK-NEXT:    br i1 [[EXITCOND_POSTLOOP]], label [[FOR_BODY_POSTLOOP]], label [[FOR_COND_CLEANUP_LOOPEXIT_LOOPEXIT:%.*]], !llvm.loop [[LOOP6:![0-9]+]], !loop_constrainer.loop.clone !5
+; CHECK-NEXT:    br i1 [[EXITCOND_POSTLOOP]], label [[FOR_BODY_POSTLOOP]], label [[FOR_COND_CLEANUP_LOOPEXIT_LOOPEXIT:%.*]], !llvm.loop [[LOOP6:![0-9]+]], !loop_constrainer.loop.clone [[META5]]
 ;
 entry:
   %cmp16 = icmp sgt i32 %N, 0
@@ -232,7 +232,7 @@ for.inc:
 
 define void @test_inc_slt(ptr nocapture %a, ptr nocapture readonly %b, ptr nocapture readonly %c, i32 %N) {
 ; CHECK-LABEL: define void @test_inc_slt(
-; CHECK-SAME: ptr nocapture [[A:%.*]], ptr nocapture readonly [[B:%.*]], ptr nocapture readonly [[C:%.*]], i32 [[N:%.*]]) {
+; CHECK-SAME: ptr captures(none) [[A:%.*]], ptr readonly captures(none) [[B:%.*]], ptr readonly captures(none) [[C:%.*]], i32 [[N:%.*]]) {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[CMP16:%.*]] = icmp sgt i32 [[N]], 0
 ; CHECK-NEXT:    br i1 [[CMP16]], label [[FOR_BODY_PREHEADER:%.*]], label [[FOR_COND_CLEANUP:%.*]]
@@ -307,7 +307,7 @@ define void @test_inc_slt(ptr nocapture %a, ptr nocapture readonly %b, ptr nocap
 ; CHECK:       for.inc.postloop:
 ; CHECK-NEXT:    [[INC_POSTLOOP]] = add nuw nsw i32 [[I_017_POSTLOOP]], 1
 ; CHECK-NEXT:    [[EXITCOND_POSTLOOP:%.*]] = icmp slt i32 [[INC_POSTLOOP]], [[N]]
-; CHECK-NEXT:    br i1 [[EXITCOND_POSTLOOP]], label [[FOR_BODY_POSTLOOP]], label [[FOR_COND_CLEANUP_LOOPEXIT_LOOPEXIT:%.*]], !llvm.loop [[LOOP7:![0-9]+]], !loop_constrainer.loop.clone !5
+; CHECK-NEXT:    br i1 [[EXITCOND_POSTLOOP]], label [[FOR_BODY_POSTLOOP]], label [[FOR_COND_CLEANUP_LOOPEXIT_LOOPEXIT:%.*]], !llvm.loop [[LOOP7:![0-9]+]], !loop_constrainer.loop.clone [[META5]]
 ;
 entry:
   %cmp16 = icmp sgt i32 %N, 0
@@ -347,7 +347,7 @@ for.inc:
 
 define void @test_inc_ult(ptr nocapture %a, ptr nocapture readonly %b, ptr nocapture readonly %c, i32 %N) {
 ; CHECK-LABEL: define void @test_inc_ult(
-; CHECK-SAME: ptr nocapture [[A:%.*]], ptr nocapture readonly [[B:%.*]], ptr nocapture readonly [[C:%.*]], i32 [[N:%.*]]) {
+; CHECK-SAME: ptr captures(none) [[A:%.*]], ptr readonly captures(none) [[B:%.*]], ptr readonly captures(none) [[C:%.*]], i32 [[N:%.*]]) {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[CMP16:%.*]] = icmp ugt i32 [[N]], 0
 ; CHECK-NEXT:    br i1 [[CMP16]], label [[FOR_BODY_PREHEADER:%.*]], label [[FOR_COND_CLEANUP:%.*]]
@@ -421,7 +421,7 @@ define void @test_inc_ult(ptr nocapture %a, ptr nocapture readonly %b, ptr nocap
 ; CHECK:       for.inc.postloop:
 ; CHECK-NEXT:    [[INC_POSTLOOP]] = add nuw nsw i32 [[I_017_POSTLOOP]], 1
 ; CHECK-NEXT:    [[EXITCOND_POSTLOOP:%.*]] = icmp ult i32 [[INC_POSTLOOP]], [[N]]
-; CHECK-NEXT:    br i1 [[EXITCOND_POSTLOOP]], label [[FOR_BODY_POSTLOOP]], label [[FOR_COND_CLEANUP_LOOPEXIT_LOOPEXIT:%.*]], !llvm.loop [[LOOP8:![0-9]+]], !loop_constrainer.loop.clone !5
+; CHECK-NEXT:    br i1 [[EXITCOND_POSTLOOP]], label [[FOR_BODY_POSTLOOP]], label [[FOR_COND_CLEANUP_LOOPEXIT_LOOPEXIT:%.*]], !llvm.loop [[LOOP8:![0-9]+]], !loop_constrainer.loop.clone [[META5]]
 ;
 entry:
   %cmp16 = icmp ugt i32 %N, 0
@@ -461,7 +461,7 @@ for.inc:
 
 define void @signed_var_imm_dec_sgt(ptr nocapture %a, ptr nocapture readonly %b, ptr nocapture readonly %c, i32 %M) {
 ; CHECK-LABEL: define void @signed_var_imm_dec_sgt(
-; CHECK-SAME: ptr nocapture [[A:%.*]], ptr nocapture readonly [[B:%.*]], ptr nocapture readonly [[C:%.*]], i32 [[M:%.*]]) {
+; CHECK-SAME: ptr captures(none) [[A:%.*]], ptr readonly captures(none) [[B:%.*]], ptr readonly captures(none) [[C:%.*]], i32 [[M:%.*]]) {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[CMP14:%.*]] = icmp slt i32 [[M]], 1024
 ; CHECK-NEXT:    br i1 [[CMP14]], label [[FOR_BODY_PREHEADER:%.*]], label [[FOR_COND_CLEANUP:%.*]]
@@ -535,7 +535,7 @@ define void @signed_var_imm_dec_sgt(ptr nocapture %a, ptr nocapture readonly %b,
 ; CHECK-NEXT:    [[DEC_PRELOOP]] = add nsw i32 [[IV_PRELOOP]], -1
 ; CHECK-NEXT:    [[CMP_PRELOOP:%.*]] = icmp sgt i32 [[DEC_PRELOOP]], [[M]]
 ; CHECK-NEXT:    [[TMP11:%.*]] = icmp sgt i32 [[DEC_PRELOOP]], [[EXIT_PRELOOP_AT]]
-; CHECK-NEXT:    br i1 [[TMP11]], label [[FOR_BODY_PRELOOP]], label [[PRELOOP_EXIT_SELECTOR:%.*]], !llvm.loop [[LOOP9:![0-9]+]], !loop_constrainer.loop.clone !5
+; CHECK-NEXT:    br i1 [[TMP11]], label [[FOR_BODY_PRELOOP]], label [[PRELOOP_EXIT_SELECTOR:%.*]], !llvm.loop [[LOOP9:![0-9]+]], !loop_constrainer.loop.clone [[META5]]
 ; CHECK:       preloop.exit.selector:
 ; CHECK-NEXT:    [[DEC_PRELOOP_LCSSA:%.*]] = phi i32 [ [[DEC_PRELOOP]], [[FOR_INC_PRELOOP]] ]
 ; CHECK-NEXT:    [[TMP12:%.*]] = icmp sgt i32 [[DEC_PRELOOP_LCSSA]], [[M]]
@@ -565,7 +565,7 @@ define void @signed_var_imm_dec_sgt(ptr nocapture %a, ptr nocapture readonly %b,
 ; CHECK-NEXT:    store i32 [[STOREMERGE_POSTLOOP]], ptr [[ARRAYIDX3_POSTLOOP]], align 4
 ; CHECK-NEXT:    [[DEC_POSTLOOP]] = add nsw i32 [[IV_POSTLOOP]], -1
 ; CHECK-NEXT:    [[CMP_POSTLOOP:%.*]] = icmp sgt i32 [[DEC_POSTLOOP]], [[M]]
-; CHECK-NEXT:    br i1 [[CMP_POSTLOOP]], label [[FOR_BODY_POSTLOOP]], label [[FOR_COND_CLEANUP_LOOPEXIT_LOOPEXIT:%.*]], !llvm.loop [[LOOP10:![0-9]+]], !loop_constrainer.loop.clone !5
+; CHECK-NEXT:    br i1 [[CMP_POSTLOOP]], label [[FOR_BODY_POSTLOOP]], label [[FOR_COND_CLEANUP_LOOPEXIT_LOOPEXIT:%.*]], !llvm.loop [[LOOP10:![0-9]+]], !loop_constrainer.loop.clone [[META5]]
 ;
 entry:
   %cmp14 = icmp slt i32 %M, 1024
@@ -600,7 +600,7 @@ for.inc:                                          ; preds = %for.body, %if.else
 
 define void @signed_var_imm_dec_sge(ptr nocapture %a, ptr nocapture readonly %b, ptr nocapture readonly %c, i32 %M) {
 ; CHECK-LABEL: define void @signed_var_imm_dec_sge(
-; CHECK-SAME: ptr nocapture [[A:%.*]], ptr nocapture readonly [[B:%.*]], ptr nocapture readonly [[C:%.*]], i32 [[M:%.*]]) {
+; CHECK-SAME: ptr captures(none) [[A:%.*]], ptr readonly captures(none) [[B:%.*]], ptr readonly captures(none) [[C:%.*]], i32 [[M:%.*]]) {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[CMP14:%.*]] = icmp sgt i32 [[M]], 1024
 ; CHECK-NEXT:    br i1 [[CMP14]], label [[FOR_COND_CLEANUP:%.*]], label [[FOR_BODY_PREHEADER:%.*]]
@@ -675,7 +675,7 @@ define void @signed_var_imm_dec_sge(ptr nocapture %a, ptr nocapture readonly %b,
 ; CHECK-NEXT:    [[DEC_PRELOOP]] = add nsw i32 [[IV_PRELOOP]], -1
 ; CHECK-NEXT:    [[CMP_PRELOOP:%.*]] = icmp sgt i32 [[IV_PRELOOP]], [[M]]
 ; CHECK-NEXT:    [[TMP11:%.*]] = icmp sgt i32 [[IV_PRELOOP]], [[EXIT_PRELOOP_AT]]
-; CHECK-NEXT:    br i1 [[TMP11]], label [[FOR_BODY_PRELOOP]], label [[PRELOOP_EXIT_SELECTOR:%.*]], !llvm.loop [[LOOP11:![0-9]+]], !loop_constrainer.loop.clone !5
+; CHECK-NEXT:    br i1 [[TMP11]], label [[FOR_BODY_PRELOOP]], label [[PRELOOP_EXIT_SELECTOR:%.*]], !llvm.loop [[LOOP11:![0-9]+]], !loop_constrainer.loop.clone [[META5]]
 ; CHECK:       preloop.exit.selector:
 ; CHECK-NEXT:    [[DEC_PRELOOP_LCSSA:%.*]] = phi i32 [ [[DEC_PRELOOP]], [[FOR_INC_PRELOOP]] ]
 ; CHECK-NEXT:    [[IV_PRELOOP_LCSSA:%.*]] = phi i32 [ [[IV_PRELOOP]], [[FOR_INC_PRELOOP]] ]
@@ -706,7 +706,7 @@ define void @signed_var_imm_dec_sge(ptr nocapture %a, ptr nocapture readonly %b,
 ; CHECK-NEXT:    store i32 [[STOREMERGE_POSTLOOP]], ptr [[ARRAYIDX3_POSTLOOP]], align 4
 ; CHECK-NEXT:    [[DEC_POSTLOOP]] = add nsw i32 [[IV_POSTLOOP]], -1
 ; CHECK-NEXT:    [[CMP_POSTLOOP:%.*]] = icmp sgt i32 [[IV_POSTLOOP]], [[M]]
-; CHECK-NEXT:    br i1 [[CMP_POSTLOOP]], label [[FOR_BODY_POSTLOOP]], label [[FOR_COND_CLEANUP_LOOPEXIT_LOOPEXIT:%.*]], !llvm.loop [[LOOP12:![0-9]+]], !loop_constrainer.loop.clone !5
+; CHECK-NEXT:    br i1 [[CMP_POSTLOOP]], label [[FOR_BODY_POSTLOOP]], label [[FOR_COND_CLEANUP_LOOPEXIT_LOOPEXIT:%.*]], !llvm.loop [[LOOP12:![0-9]+]], !loop_constrainer.loop.clone [[META5]]
 ;
 entry:
   %cmp14 = icmp sgt i32 %M, 1024
@@ -741,7 +741,7 @@ for.inc:                                          ; preds = %for.body, %if.else
 
 define void @signed_var_imm_dec_slt(ptr nocapture %a, ptr nocapture readonly %b, ptr nocapture readonly %c, i32 %M) {
 ; CHECK-LABEL: define void @signed_var_imm_dec_slt(
-; CHECK-SAME: ptr nocapture [[A:%.*]], ptr nocapture readonly [[B:%.*]], ptr nocapture readonly [[C:%.*]], i32 [[M:%.*]]) {
+; CHECK-SAME: ptr captures(none) [[A:%.*]], ptr readonly captures(none) [[B:%.*]], ptr readonly captures(none) [[C:%.*]], i32 [[M:%.*]]) {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[CMP14:%.*]] = icmp sgt i32 [[M]], 1024
 ; CHECK-NEXT:    br i1 [[CMP14]], label [[FOR_COND_CLEANUP:%.*]], label [[FOR_BODY_PREHEADER:%.*]]
@@ -805,7 +805,7 @@ for.inc:                                          ; preds = %for.body, %if.else
 
 define void @signed_var_imm_dec_ne(ptr nocapture %a, ptr nocapture readonly %b, ptr nocapture readonly %c, i32 %M) {
 ; CHECK-LABEL: define void @signed_var_imm_dec_ne(
-; CHECK-SAME: ptr nocapture [[A:%.*]], ptr nocapture readonly [[B:%.*]], ptr nocapture readonly [[C:%.*]], i32 [[M:%.*]]) {
+; CHECK-SAME: ptr captures(none) [[A:%.*]], ptr readonly captures(none) [[B:%.*]], ptr readonly captures(none) [[C:%.*]], i32 [[M:%.*]]) {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[CMP14:%.*]] = icmp slt i32 [[M]], 1024
 ; CHECK-NEXT:    br i1 [[CMP14]], label [[FOR_BODY_PREHEADER:%.*]], label [[FOR_COND_CLEANUP:%.*]]
@@ -879,7 +879,7 @@ define void @signed_var_imm_dec_ne(ptr nocapture %a, ptr nocapture readonly %b,
 ; CHECK-NEXT:    [[DEC_PRELOOP]] = add nsw i32 [[IV_PRELOOP]], -1
 ; CHECK-NEXT:    [[CMP_PRELOOP:%.*]] = icmp ne i32 [[DEC_PRELOOP]], [[M]]
 ; CHECK-NEXT:    [[TMP11:%.*]] = icmp sgt i32 [[DEC_PRELOOP]], [[EXIT_PRELOOP_AT]]
-; CHECK-NEXT:    br i1 [[TMP11]], label [[FOR_BODY_PRELOOP]], label [[PRELOOP_EXIT_SELECTOR:%.*]], !llvm.loop [[LOOP13:![0-9]+]], !loop_constrainer.loop.clone !5
+; CHECK-NEXT:    br i1 [[TMP11]], label [[FOR_BODY_PRELOOP]], label [[PRELOOP_EXIT_SELECTOR:%.*]], !llvm.loop [[LOOP13:![0-9]+]], !loop_constrainer.loop.clone [[META5]]
 ; CHECK:       preloop.exit.selector:
 ; CHECK-NEXT:    [[DEC_PRELOOP_LCSSA:%.*]] = phi i32 [ [[DEC_PRELOOP]], [[FOR_INC_PRELOOP]] ]
 ; CHECK-NEXT:    [[TMP12:%.*]] = icmp sgt i32 [[DEC_PRELOOP_LCSSA]], [[M]]
@@ -909,7 +909,7 @@ define void @signed_var_imm_dec_ne(ptr nocapture %a, ptr nocapture readonly %b,
 ; CHECK-NEXT:    store i32 [[STOREMERGE_POSTLOOP]], ptr [[ARRAYIDX3_POSTLOOP]], align 4
 ; CHECK-NEXT:    [[DEC_POSTLOOP]] = add nsw i32 [[IV_POSTLOOP]], -1
 ; CHECK-NEXT:    [[CMP_POSTLOOP:%.*]] = icmp ne i32 [[DEC_POSTLOOP]], [[M]]
-; CHECK-NEXT:    br i1 [[CMP_POSTLOOP]], label [[FOR_BODY_POSTLOOP]], label [[FOR_COND_CLEANUP_LOOPEXIT_LOOPEXIT:%.*]], !llvm.loop [[LOOP14:![0-9]+]], !loop_constrainer.loop.clone !5
+; CHECK-NEXT:    br i1 [[CMP_POSTLOOP]], label [[FOR_BODY_POSTLOOP]], label [[FOR_COND_CLEANUP_LOOPEXIT_LOOPEXIT:%.*]], !llvm.loop [[LOOP14:![0-9]+]], !loop_constrainer.loop.clone [[META5]]
 ;
 entry:
   %cmp14 = icmp slt i32 %M, 1024
@@ -944,7 +944,7 @@ for.inc:                                          ; preds = %for.body, %if.else
 
 define void @signed_var_imm_dec_eq(ptr nocapture %a, ptr nocapture readonly %b, ptr nocapture readonly %c, i32 %M) {
 ; CHECK-LABEL: define void @signed_var_imm_dec_eq(
-; CHECK-SAME: ptr nocapture [[A:%.*]], ptr nocapture readonly [[B:%.*]], ptr nocapture readonly [[C:%.*]], i32 [[M:%.*]]) {
+; CHECK-SAME: ptr captures(none) [[A:%.*]], ptr readonly captures(none) [[B:%.*]], ptr readonly captures(none) [[C:%.*]], i32 [[M:%.*]]) {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[CMP14:%.*]] = icmp slt i32 [[M]], 1024
 ; CHECK-NEXT:    br i1 [[CMP14]], label [[FOR_BODY_PREHEADER:%.*]], label [[FOR_COND_CLEANUP:%.*]]
@@ -1020,7 +1020,7 @@ define void @signed_var_imm_dec_eq(ptr nocapture %a, ptr nocapture readonly %b,
 ; CHECK-NEXT:    [[CMP_PRELOOP:%.*]] = icmp eq i32 [[DEC_PRELOOP]], [[M]]
 ; CHECK-NEXT:    [[TMP12:%.*]] = icmp sgt i32 [[DEC_PRELOOP]], [[EXIT_PRELOOP_AT]]
 ; CHECK-NEXT:    [[TMP13:%.*]] = xor i1 [[TMP12]], true
-; CHECK-NEXT:    br i1 [[TMP13]], label [[PRELOOP_EXIT_SELECTOR:%.*]], label [[FOR_BODY_PRELOOP]], !llvm.loop [[LOOP15:![0-9]+]], !loop_constrainer.loop.clone !5
+; CHECK-NEXT:    br i1 [[TMP13]], label [[PRELOOP_EXIT_SELECTOR:%.*]], label [[FOR_BODY_PRELOOP]], !llvm.loop [[LOOP15:![0-9]+]], !loop_constrainer.loop.clone [[META5]]
 ; CHECK:       preloop.exit.selector:
 ; CHECK-NEXT:    [[DEC_PRELOOP_LCSSA:%.*]] = phi i32 [ [[DEC_PRELOOP]], [[FOR_INC_PRELOOP]] ]
 ; CHECK-NEXT:    [[TMP14:%.*]] = icmp sgt i32 [[DEC_PRELOOP_LCSSA]], [[M]]
@@ -1050,7 +1050,7 @@ define void @signed_var_imm_dec_eq(ptr nocapture %a, ptr nocapture readonly %b,
 ; CHECK-NEXT:    store i32 [[STOREMERGE_POSTLOOP]], ptr [[ARRAYIDX3_POSTLOOP]], align 4
 ; CHECK-NEXT:    [[DEC_POSTLOOP]] = add nsw i32 [[IV_POSTLOOP]], -1
 ; CHECK-NEXT:    [[CMP_POSTLOOP:%.*]] = icmp eq i32 [[DEC_POSTLOOP]], [[M]]
-; CHECK-NEXT:    br i1 [[CMP_POSTLOOP]], label [[FOR_COND_CLEANUP_LOOPEXIT_LOOPEXIT:%.*]], label [[FOR_BODY_POSTLOOP]], !llvm.loop [[LOOP16:![0-9]+]], !loop_constrainer.loop.clone !5
+; CHECK-NEXT:    br i1 [[CMP_POSTLOOP]], label [[FOR_COND_CLEANUP_LOOPEXIT_LOOPEXIT:%.*]], label [[FOR_BODY_POSTLOOP]], !llvm.loop [[LOOP16:![0-9]+]], !loop_constrainer.loop.clone [[META5]]
 ;
 entry:
   %cmp14 = icmp slt i32 %M, 1024
diff --git a/llvm/test/Transforms/IndVarSimplify/ada-loops.ll b/llvm/test/Transforms/IndVarSimplify/ada-loops.ll
index 0da00408e41a5..83f2f212a4385 100644
--- a/llvm/test/Transforms/IndVarSimplify/ada-loops.ll
+++ b/llvm/test/Transforms/IndVarSimplify/ada-loops.ll
@@ -18,7 +18,7 @@ target triple = "i686-pc-linux-gnu"
 
 define void @kinds__sbytezero(ptr nocapture %a) nounwind {
 ; CHECK-LABEL: define void @kinds__sbytezero(
-; CHECK-SAME: ptr nocapture [[A:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-SAME: ptr captures(none) [[A:%.*]]) #[[ATTR0:[0-9]+]] {
 ; CHECK-NEXT:  bb.thread:
 ; CHECK-NEXT:    [[TMP46:%.*]] = getelementptr [256 x i32], ptr [[A]], i32 0, i32 0
 ; CHECK-NEXT:    store i32 0, ptr [[TMP46]], align 4
@@ -56,7 +56,7 @@ return:		; preds = %bb
 
 define void @kinds__ubytezero(ptr nocapture %a) nounwind {
 ; CHECK-LABEL: define void @kinds__ubytezero(
-; CHECK-SAME: ptr nocapture [[A:%.*]]) #[[ATTR0]] {
+; CHECK-SAME: ptr captures(none) [[A:%.*]]) #[[ATTR0]] {
 ; CHECK-NEXT:  bb.thread:
 ; CHECK-NEXT:    [[TMP35:%.*]] = getelementptr [256 x i32], ptr [[A]], i32 0, i32 0
 ; CHECK-NEXT:    store i32 0, ptr [[TMP35]], align 4
@@ -91,7 +91,7 @@ return:		; preds = %bb
 
 define void @kinds__srangezero(ptr nocapture %a) nounwind {
 ; CHECK-LABEL: define void @kinds__srangezero(
-; CHECK-SAME: ptr nocapture [[A:%.*]]) #[[ATTR0]] {
+; CHECK-SAME: ptr captures(none) [[A:%.*]]) #[[ATTR0]] {
 ; CHECK-NEXT:  bb.thread:
 ; CHECK-NEXT:    br label [[BB:%.*]]
 ; CHECK:       bb:
@@ -124,7 +124,7 @@ return:		; preds = %bb
 
 define void @kinds__urangezero(ptr nocapture %a) nounwind {
 ; CHECK-LABEL: define void @kinds__urangezero(
-; CHECK-SAME: ptr nocapture [[A:%.*]]) #[[ATTR0]] {
+; CHECK-SAME: ptr captures(none) [[A:%.*]]) #[[ATTR0]] {
 ; CHECK-NEXT:  bb.thread:
 ; CHECK-NEXT:    br label [[BB:%.*]]
 ; CHECK:       bb:
diff --git a/llvm/test/Transforms/IndVarSimplify/infer-via-ranges.ll b/llvm/test/Transforms/IndVarSimplify/infer-via-ranges.ll
new file mode 100644
index 0000000000000..effae2322dba3
--- /dev/null
+++ b/llvm/test/Transforms/IndVarSimplify/infer-via-ranges.ll
@@ -0,0 +1,51 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -passes=indvars -S < %s | FileCheck %s
+
+define void @infer_via_ranges(ptr %arr, i32 %n) {
+; CHECK-LABEL: define void @infer_via_ranges(
+; CHECK-SAME: ptr [[ARR:%.*]], i32 [[N:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[FIRST_ITR_CHECK:%.*]] = icmp sgt i32 [[N]], 0
+; CHECK-NEXT:    [[START:%.*]] = sub i32 [[N]], 1
+; CHECK-NEXT:    br i1 [[FIRST_ITR_CHECK]], label %[[LOOP_PREHEADER:.*]], label %[[EXIT:.*]]
+; CHECK:       [[LOOP_PREHEADER]]:
+; CHECK-NEXT:    br label %[[LOOP:.*]]
+; CHECK:       [[LOOP]]:
+; CHECK-NEXT:    [[IDX:%.*]] = phi i32 [ [[IDX_DEC:%.*]], %[[IN_BOUNDS:.*]] ], [ [[START]], %[[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    [[IDX_DEC]] = sub nsw i32 [[IDX]], 1
+; CHECK-NEXT:    br i1 true, label %[[IN_BOUNDS]], label %[[OUT_OF_BOUNDS:.*]]
+; CHECK:       [[IN_BOUNDS]]:
+; CHECK-NEXT:    [[ADDR:%.*]] = getelementptr i32, ptr [[ARR]], i32 [[IDX]]
+; CHECK-NEXT:    store i32 0, ptr [[ADDR]], align 4
+; CHECK-NEXT:    [[NEXT:%.*]] = icmp sgt i32 [[IDX_DEC]], -1
+; CHECK-NEXT:    br i1 [[NEXT]], label %[[LOOP]], label %[[EXIT_LOOPEXIT:.*]]
+; CHECK:       [[OUT_OF_BOUNDS]]:
+; CHECK-NEXT:    ret void
+; CHECK:       [[EXIT_LOOPEXIT]]:
+; CHECK-NEXT:    br label %[[EXIT]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    ret void
+;
+  entry:
+  %first.itr.check = icmp sgt i32 %n, 0
+  %start = sub i32 %n, 1
+  br i1 %first.itr.check, label %loop, label %exit
+
+  loop:
+  %idx = phi i32 [ %start, %entry ] , [ %idx.dec, %in.bounds ]
+  %idx.dec = sub i32 %idx, 1
+  %abc = icmp sge i32 %idx, 0
+  br i1 %abc, label %in.bounds, label %out.of.bounds
+
+  in.bounds:
+  %addr = getelementptr i32, ptr %arr, i32 %idx
+  store i32 0, ptr %addr
+  %next = icmp sgt i32 %idx.dec, -1
+  br i1 %next, label %loop, label %exit
+
+  out.of.bounds:
+  ret void
+
+  exit:
+  ret void
+}
diff --git a/llvm/test/Transforms/IndVarSimplify/iv-ext-samesign.ll b/llvm/test/Transforms/IndVarSimplify/iv-ext-samesign.ll
new file mode 100644
index 0000000000000..1207f47c5e3c9
--- /dev/null
+++ b/llvm/test/Transforms/IndVarSimplify/iv-ext-samesign.ll
@@ -0,0 +1,716 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt < %s -passes=indvars -S | FileCheck %s
+
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128"
+
+define i32 @iv_zext_zext_sgt_slt(i32 %iter.count, ptr %ptr) {
+; CHECK-LABEL: define i32 @iv_zext_zext_sgt_slt(
+; CHECK-SAME: i32 [[ITER_COUNT:%.*]], ptr [[PTR:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    [[TMP0:%.*]] = add nsw i32 [[ITER_COUNT]], -1
+; CHECK-NEXT:    br label %[[OUTER_LOOP:.*]]
+; CHECK:       [[PH_LOOPEXIT:.*]]:
+; CHECK-NEXT:    br label %[[PH:.*]]
+; CHECK:       [[PH]]:
+; CHECK-NEXT:    [[INDVARS_IV_NEXT2:%.*]] = add i32 [[INDVARS_IV1:%.*]], -1
+; CHECK-NEXT:    br label %[[OUTER_LOOP]]
+; CHECK:       [[OUTER_LOOP]]:
+; CHECK-NEXT:    [[INDVARS_IV1]] = phi i32 [ [[INDVARS_IV_NEXT2]], %[[PH]] ], [ [[TMP0]], %[[ENTRY]] ]
+; CHECK-NEXT:    [[IV_OUTER:%.*]] = phi i32 [ [[IV_OUTER_1:%.*]], %[[PH]] ], [ [[ITER_COUNT]], %[[ENTRY]] ]
+; CHECK-NEXT:    [[IV_OUTER_1]] = add nsw i32 [[IV_OUTER]], -1
+; CHECK-NEXT:    [[EXT_OUTER:%.*]] = zext nneg i32 [[IV_OUTER_1]] to i64
+; CHECK-NEXT:    [[GEP_OUTER:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[EXT_OUTER]]
+; CHECK-NEXT:    store i8 0, ptr [[GEP_OUTER]], align 1
+; CHECK-NEXT:    [[EXIT_COND_OUTER:%.*]] = icmp sgt i32 [[IV_OUTER]], 1
+; CHECK-NEXT:    br i1 [[EXIT_COND_OUTER]], label %[[INNER_LOOP_PREHEADER:.*]], label %[[PH]]
+; CHECK:       [[INNER_LOOP_PREHEADER]]:
+; CHECK-NEXT:    [[WIDE_TRIP_COUNT:%.*]] = zext i32 [[INDVARS_IV1]] to i64
+; CHECK-NEXT:    br label %[[INNER_LOOP:.*]]
+; CHECK:       [[INNER_LOOP]]:
+; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ 0, %[[INNER_LOOP_PREHEADER]] ], [ [[INDVARS_IV_NEXT:%.*]], %[[INNER_LOOP]] ]
+; CHECK-NEXT:    [[GEP_INNER:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[INDVARS_IV]]
+; CHECK-NEXT:    store i8 0, ptr [[GEP_INNER]], align 1
+; CHECK-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
+; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp ne i64 [[INDVARS_IV_NEXT]], [[WIDE_TRIP_COUNT]]
+; CHECK-NEXT:    br i1 [[EXITCOND]], label %[[INNER_LOOP]], label %[[PH_LOOPEXIT]]
+; CHECK:       [[EXIT:.*:]]
+; CHECK-NEXT:    ret i32 0
+;
+entry:
+  br label %outer.loop
+
+ph:
+  br label %outer.loop
+
+outer.loop:
+  %iv.outer = phi i32 [ %iv.outer.1, %ph ], [ %iter.count, %entry ]
+  %iv.outer.1 = add nsw i32 %iv.outer, -1
+  %ext.outer = zext nneg i32 %iv.outer.1 to i64
+  %gep.outer = getelementptr i8, ptr %ptr, i64 %ext.outer
+  store i8 0, ptr %gep.outer
+  %exit.cond.outer = icmp sgt i32 %iv.outer, 1
+  br i1 %exit.cond.outer, label %inner.loop, label %ph
+
+inner.loop:
+  %iv.inner = phi i32 [ %iv.next, %inner.loop ], [ 0, %outer.loop ]
+  %ext.inner = zext nneg i32 %iv.inner to i64
+  %gep.inner = getelementptr i8, ptr %ptr, i64 %ext.inner
+  store i8 0, ptr %gep.inner
+  %iv.next = add nuw nsw i32 %iv.inner, 1
+  %exit.cond.inner = icmp slt i32 %iv.next, %iv.outer.1
+  br i1 %exit.cond.inner, label %inner.loop, label %ph
+
+exit:
+  ret i32 0
+}
+
+define i32 @iv_zext_zext_gt_slt(i32 %iter.count, ptr %ptr) {
+; CHECK-LABEL: define i32 @iv_zext_zext_gt_slt(
+; CHECK-SAME: i32 [[ITER_COUNT:%.*]], ptr [[PTR:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    [[TMP0:%.*]] = sext i32 [[ITER_COUNT]] to i64
+; CHECK-NEXT:    br label %[[OUTER_LOOP:.*]]
+; CHECK:       [[PH_LOOPEXIT:.*]]:
+; CHECK-NEXT:    br label %[[PH:.*]]
+; CHECK:       [[PH]]:
+; CHECK-NEXT:    br label %[[OUTER_LOOP]]
+; CHECK:       [[OUTER_LOOP]]:
+; CHECK-NEXT:    [[INDVARS_IV1:%.*]] = phi i64 [ [[INDVARS_IV_NEXT2:%.*]], %[[PH]] ], [ [[TMP0]], %[[ENTRY]] ]
+; CHECK-NEXT:    [[INDVARS_IV_NEXT2]] = add nsw i64 [[INDVARS_IV1]], -1
+; CHECK-NEXT:    [[GEP_OUTER:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[INDVARS_IV_NEXT2]]
+; CHECK-NEXT:    store i8 0, ptr [[GEP_OUTER]], align 1
+; CHECK-NEXT:    [[EXIT_COND_OUTER:%.*]] = icmp samesign ugt i64 [[INDVARS_IV1]], 1
+; CHECK-NEXT:    br i1 [[EXIT_COND_OUTER]], label %[[INNER_LOOP_PREHEADER:.*]], label %[[PH]]
+; CHECK:       [[INNER_LOOP_PREHEADER]]:
+; CHECK-NEXT:    br label %[[INNER_LOOP:.*]]
+; CHECK:       [[INNER_LOOP]]:
+; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ 0, %[[INNER_LOOP_PREHEADER]] ], [ [[INDVARS_IV_NEXT:%.*]], %[[INNER_LOOP]] ]
+; CHECK-NEXT:    [[GEP_INNER:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[INDVARS_IV]]
+; CHECK-NEXT:    store i8 0, ptr [[GEP_INNER]], align 1
+; CHECK-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
+; CHECK-NEXT:    [[EXIT_COND_INNER:%.*]] = icmp slt i64 [[INDVARS_IV_NEXT]], [[INDVARS_IV_NEXT2]]
+; CHECK-NEXT:    br i1 [[EXIT_COND_INNER]], label %[[INNER_LOOP]], label %[[PH_LOOPEXIT]]
+; CHECK:       [[EXIT:.*:]]
+; CHECK-NEXT:    ret i32 0
+;
+entry:
+  br label %outer.loop
+
+ph:
+  br label %outer.loop
+
+outer.loop:
+  %iv.outer = phi i32 [ %iv.outer.1, %ph ], [ %iter.count, %entry ]
+  %iv.outer.1 = add nsw i32 %iv.outer, -1
+  %ext.outer = zext nneg i32 %iv.outer.1 to i64
+  %gep.outer = getelementptr i8, ptr %ptr, i64 %ext.outer
+  store i8 0, ptr %gep.outer
+  %exit.cond.outer = icmp samesign ugt i32 %iv.outer, 1
+  br i1 %exit.cond.outer, label %inner.loop, label %ph
+
+inner.loop:
+  %iv.inner = phi i32 [ %iv.next, %inner.loop ], [ 0, %outer.loop ]
+  %ext.inner = zext nneg i32 %iv.inner to i64
+  %gep.inner = getelementptr i8, ptr %ptr, i64 %ext.inner
+  store i8 0, ptr %gep.inner
+  %iv.next = add nuw nsw i32 %iv.inner, 1
+  %exit.cond.inner = icmp slt i32 %iv.next, %iv.outer.1
+  br i1 %exit.cond.inner, label %inner.loop, label %ph
+
+exit:
+  ret i32 0
+}
+
+define i32 @iv_zext_zext_gt_slt_exitlimit(i32 %iter.count, i32 %exit.limit, ptr %ptr) {
+; CHECK-LABEL: define i32 @iv_zext_zext_gt_slt_exitlimit(
+; CHECK-SAME: i32 [[ITER_COUNT:%.*]], i32 [[EXIT_LIMIT:%.*]], ptr [[PTR:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    [[TMP0:%.*]] = sext i32 [[ITER_COUNT]] to i64
+; CHECK-NEXT:    [[TMP1:%.*]] = sext i32 [[EXIT_LIMIT]] to i64
+; CHECK-NEXT:    br label %[[OUTER_LOOP:.*]]
+; CHECK:       [[PH_LOOPEXIT:.*]]:
+; CHECK-NEXT:    br label %[[PH:.*]]
+; CHECK:       [[PH]]:
+; CHECK-NEXT:    br label %[[OUTER_LOOP]]
+; CHECK:       [[OUTER_LOOP]]:
+; CHECK-NEXT:    [[INDVARS_IV1:%.*]] = phi i64 [ [[INDVARS_IV_NEXT2:%.*]], %[[PH]] ], [ [[TMP0]], %[[ENTRY]] ]
+; CHECK-NEXT:    [[INDVARS_IV_NEXT2]] = add nsw i64 [[INDVARS_IV1]], -1
+; CHECK-NEXT:    [[GEP_OUTER:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[INDVARS_IV_NEXT2]]
+; CHECK-NEXT:    store i8 0, ptr [[GEP_OUTER]], align 1
+; CHECK-NEXT:    [[EXIT_COND_OUTER:%.*]] = icmp samesign ugt i64 [[INDVARS_IV1]], [[TMP1]]
+; CHECK-NEXT:    br i1 [[EXIT_COND_OUTER]], label %[[INNER_LOOP_PREHEADER:.*]], label %[[PH]]
+; CHECK:       [[INNER_LOOP_PREHEADER]]:
+; CHECK-NEXT:    br label %[[INNER_LOOP:.*]]
+; CHECK:       [[INNER_LOOP]]:
+; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ 0, %[[INNER_LOOP_PREHEADER]] ], [ [[INDVARS_IV_NEXT:%.*]], %[[INNER_LOOP]] ]
+; CHECK-NEXT:    [[GEP_INNER:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[INDVARS_IV]]
+; CHECK-NEXT:    store i8 0, ptr [[GEP_INNER]], align 1
+; CHECK-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
+; CHECK-NEXT:    [[EXIT_COND_INNER:%.*]] = icmp slt i64 [[INDVARS_IV_NEXT]], [[INDVARS_IV_NEXT2]]
+; CHECK-NEXT:    br i1 [[EXIT_COND_INNER]], label %[[INNER_LOOP]], label %[[PH_LOOPEXIT]]
+; CHECK:       [[EXIT:.*:]]
+; CHECK-NEXT:    ret i32 0
+;
+entry:
+  br label %outer.loop
+
+ph:
+  br label %outer.loop
+
+outer.loop:
+  %iv.outer = phi i32 [ %iv.outer.1, %ph ], [ %iter.count, %entry ]
+  %iv.outer.1 = add nsw i32 %iv.outer, -1
+  %ext.outer = zext nneg i32 %iv.outer.1 to i64
+  %gep.outer = getelementptr i8, ptr %ptr, i64 %ext.outer
+  store i8 0, ptr %gep.outer
+  %exit.cond.outer = icmp samesign ugt i32 %iv.outer, %exit.limit
+  br i1 %exit.cond.outer, label %inner.loop, label %ph
+
+inner.loop:
+  %iv.inner = phi i32 [ %iv.next, %inner.loop ], [ 0, %outer.loop ]
+  %ext.inner = zext nneg i32 %iv.inner to i64
+  %gep.inner = getelementptr i8, ptr %ptr, i64 %ext.inner
+  store i8 0, ptr %gep.inner
+  %iv.next = add nuw nsw i32 %iv.inner, 1
+  %exit.cond.inner = icmp slt i32 %iv.next, %iv.outer.1
+  br i1 %exit.cond.inner, label %inner.loop, label %ph
+
+exit:
+  ret i32 0
+}
+
+define i32 @iv_zext_zext_sgt_lt(i32 %iter.count, ptr %ptr) {
+; CHECK-LABEL: define i32 @iv_zext_zext_sgt_lt(
+; CHECK-SAME: i32 [[ITER_COUNT:%.*]], ptr [[PTR:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    [[TMP0:%.*]] = add nsw i32 [[ITER_COUNT]], -1
+; CHECK-NEXT:    br label %[[OUTER_LOOP:.*]]
+; CHECK:       [[PH_LOOPEXIT:.*]]:
+; CHECK-NEXT:    br label %[[PH:.*]]
+; CHECK:       [[PH]]:
+; CHECK-NEXT:    [[INDVARS_IV_NEXT2:%.*]] = add i32 [[INDVARS_IV1:%.*]], -1
+; CHECK-NEXT:    br label %[[OUTER_LOOP]]
+; CHECK:       [[OUTER_LOOP]]:
+; CHECK-NEXT:    [[INDVARS_IV1]] = phi i32 [ [[INDVARS_IV_NEXT2]], %[[PH]] ], [ [[TMP0]], %[[ENTRY]] ]
+; CHECK-NEXT:    [[IV_OUTER:%.*]] = phi i32 [ [[IV_OUTER_1:%.*]], %[[PH]] ], [ [[ITER_COUNT]], %[[ENTRY]] ]
+; CHECK-NEXT:    [[IV_OUTER_1]] = add nsw i32 [[IV_OUTER]], -1
+; CHECK-NEXT:    [[EXT_OUTER:%.*]] = zext nneg i32 [[IV_OUTER_1]] to i64
+; CHECK-NEXT:    [[GEP_OUTER:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[EXT_OUTER]]
+; CHECK-NEXT:    store i8 0, ptr [[GEP_OUTER]], align 1
+; CHECK-NEXT:    [[EXIT_COND_OUTER:%.*]] = icmp sgt i32 [[IV_OUTER]], 1
+; CHECK-NEXT:    br i1 [[EXIT_COND_OUTER]], label %[[INNER_LOOP_PREHEADER:.*]], label %[[PH]]
+; CHECK:       [[INNER_LOOP_PREHEADER]]:
+; CHECK-NEXT:    [[WIDE_TRIP_COUNT:%.*]] = zext i32 [[INDVARS_IV1]] to i64
+; CHECK-NEXT:    br label %[[INNER_LOOP:.*]]
+; CHECK:       [[INNER_LOOP]]:
+; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ 0, %[[INNER_LOOP_PREHEADER]] ], [ [[INDVARS_IV_NEXT:%.*]], %[[INNER_LOOP]] ]
+; CHECK-NEXT:    [[GEP_INNER:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[INDVARS_IV]]
+; CHECK-NEXT:    store i8 0, ptr [[GEP_INNER]], align 1
+; CHECK-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
+; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp ne i64 [[INDVARS_IV_NEXT]], [[WIDE_TRIP_COUNT]]
+; CHECK-NEXT:    br i1 [[EXITCOND]], label %[[INNER_LOOP]], label %[[PH_LOOPEXIT]]
+; CHECK:       [[EXIT:.*:]]
+; CHECK-NEXT:    ret i32 0
+;
+entry:
+  br label %outer.loop
+
+ph:
+  br label %outer.loop
+
+outer.loop:
+  %iv.outer = phi i32 [ %iv.outer.1, %ph ], [ %iter.count, %entry ]
+  %iv.outer.1 = add nsw i32 %iv.outer, -1
+  %ext.outer = zext nneg i32 %iv.outer.1 to i64
+  %gep.outer = getelementptr i8, ptr %ptr, i64 %ext.outer
+  store i8 0, ptr %gep.outer
+  %exit.cond.outer = icmp sgt i32 %iv.outer, 1
+  br i1 %exit.cond.outer, label %inner.loop, label %ph
+
+inner.loop:
+  %iv.inner = phi i32 [ %iv.next, %inner.loop ], [ 0, %outer.loop ]
+  %ext.inner = zext nneg i32 %iv.inner to i64
+  %gep.inner = getelementptr i8, ptr %ptr, i64 %ext.inner
+  store i8 0, ptr %gep.inner
+  %iv.next = add nuw nsw i32 %iv.inner, 1
+  %exit.cond.inner = icmp samesign ult i32 %iv.next, %iv.outer.1
+  br i1 %exit.cond.inner, label %inner.loop, label %ph
+
+exit:
+  ret i32 0
+}
+
+define i32 @iv_zext_zext_sgt_lt_exitlimit(i32 %iter.count, i32 %exit.limit, ptr %ptr) {
+; CHECK-LABEL: define i32 @iv_zext_zext_sgt_lt_exitlimit(
+; CHECK-SAME: i32 [[ITER_COUNT:%.*]], i32 [[EXIT_LIMIT:%.*]], ptr [[PTR:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[OUTER_LOOP:.*]]
+; CHECK:       [[PH_LOOPEXIT:.*]]:
+; CHECK-NEXT:    br label %[[PH:.*]]
+; CHECK:       [[PH]]:
+; CHECK-NEXT:    br label %[[OUTER_LOOP]]
+; CHECK:       [[OUTER_LOOP]]:
+; CHECK-NEXT:    [[IV_OUTER:%.*]] = phi i32 [ [[IV_OUTER_1:%.*]], %[[PH]] ], [ [[ITER_COUNT]], %[[ENTRY]] ]
+; CHECK-NEXT:    [[IV_OUTER_1]] = add nsw i32 [[IV_OUTER]], -1
+; CHECK-NEXT:    [[EXT_OUTER:%.*]] = zext nneg i32 [[IV_OUTER_1]] to i64
+; CHECK-NEXT:    [[GEP_OUTER:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[EXT_OUTER]]
+; CHECK-NEXT:    store i8 0, ptr [[GEP_OUTER]], align 1
+; CHECK-NEXT:    [[EXIT_COND_OUTER:%.*]] = icmp sgt i32 [[IV_OUTER]], [[EXIT_LIMIT]]
+; CHECK-NEXT:    br i1 [[EXIT_COND_OUTER]], label %[[INNER_LOOP_PREHEADER:.*]], label %[[PH]]
+; CHECK:       [[INNER_LOOP_PREHEADER]]:
+; CHECK-NEXT:    [[TMP1:%.*]] = zext i32 [[IV_OUTER_1]] to i64
+; CHECK-NEXT:    br label %[[INNER_LOOP:.*]]
+; CHECK:       [[INNER_LOOP]]:
+; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ 0, %[[INNER_LOOP_PREHEADER]] ], [ [[INDVARS_IV_NEXT:%.*]], %[[INNER_LOOP]] ]
+; CHECK-NEXT:    [[GEP_INNER:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[INDVARS_IV]]
+; CHECK-NEXT:    store i8 0, ptr [[GEP_INNER]], align 1
+; CHECK-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
+; CHECK-NEXT:    [[EXIT_COND_INNER:%.*]] = icmp samesign ult i64 [[INDVARS_IV_NEXT]], [[TMP1]]
+; CHECK-NEXT:    br i1 [[EXIT_COND_INNER]], label %[[INNER_LOOP]], label %[[PH_LOOPEXIT]]
+; CHECK:       [[EXIT:.*:]]
+; CHECK-NEXT:    ret i32 0
+;
+entry:
+  br label %outer.loop
+
+ph:
+  br label %outer.loop
+
+outer.loop:
+  %iv.outer = phi i32 [ %iv.outer.1, %ph ], [ %iter.count, %entry ]
+  %iv.outer.1 = add nsw i32 %iv.outer, -1
+  %ext.outer = zext nneg i32 %iv.outer.1 to i64
+  %gep.outer = getelementptr i8, ptr %ptr, i64 %ext.outer
+  store i8 0, ptr %gep.outer
+  %exit.cond.outer = icmp sgt i32 %iv.outer, %exit.limit
+  br i1 %exit.cond.outer, label %inner.loop, label %ph
+
+inner.loop:
+  %iv.inner = phi i32 [ %iv.next, %inner.loop ], [ 0, %outer.loop ]
+  %ext.inner = zext nneg i32 %iv.inner to i64
+  %gep.inner = getelementptr i8, ptr %ptr, i64 %ext.inner
+  store i8 0, ptr %gep.inner
+  %iv.next = add nuw nsw i32 %iv.inner, 1
+  %exit.cond.inner = icmp samesign ult i32 %iv.next, %iv.outer.1
+  br i1 %exit.cond.inner, label %inner.loop, label %ph
+
+exit:
+  ret i32 0
+}
+
+define i32 @iv_zext_zext_gt_lt(i32 %iter.count, ptr %ptr) {
+; CHECK-LABEL: define i32 @iv_zext_zext_gt_lt(
+; CHECK-SAME: i32 [[ITER_COUNT:%.*]], ptr [[PTR:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    [[TMP0:%.*]] = add nsw i32 [[ITER_COUNT]], -1
+; CHECK-NEXT:    br label %[[OUTER_LOOP:.*]]
+; CHECK:       [[PH_LOOPEXIT:.*]]:
+; CHECK-NEXT:    br label %[[PH:.*]]
+; CHECK:       [[PH]]:
+; CHECK-NEXT:    [[INDVARS_IV_NEXT2:%.*]] = add i32 [[INDVARS_IV1:%.*]], -1
+; CHECK-NEXT:    br label %[[OUTER_LOOP]]
+; CHECK:       [[OUTER_LOOP]]:
+; CHECK-NEXT:    [[INDVARS_IV1]] = phi i32 [ [[INDVARS_IV_NEXT2]], %[[PH]] ], [ [[TMP0]], %[[ENTRY]] ]
+; CHECK-NEXT:    [[IV_OUTER:%.*]] = phi i32 [ [[IV_OUTER_1:%.*]], %[[PH]] ], [ [[ITER_COUNT]], %[[ENTRY]] ]
+; CHECK-NEXT:    [[IV_OUTER_1]] = add nsw i32 [[IV_OUTER]], -1
+; CHECK-NEXT:    [[EXT_OUTER:%.*]] = zext nneg i32 [[IV_OUTER_1]] to i64
+; CHECK-NEXT:    [[GEP_OUTER:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[EXT_OUTER]]
+; CHECK-NEXT:    store i8 0, ptr [[GEP_OUTER]], align 1
+; CHECK-NEXT:    [[EXIT_COND_OUTER:%.*]] = icmp samesign ugt i32 [[IV_OUTER]], 1
+; CHECK-NEXT:    br i1 [[EXIT_COND_OUTER]], label %[[INNER_LOOP_PREHEADER:.*]], label %[[PH]]
+; CHECK:       [[INNER_LOOP_PREHEADER]]:
+; CHECK-NEXT:    [[WIDE_TRIP_COUNT:%.*]] = zext i32 [[INDVARS_IV1]] to i64
+; CHECK-NEXT:    br label %[[INNER_LOOP:.*]]
+; CHECK:       [[INNER_LOOP]]:
+; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ 0, %[[INNER_LOOP_PREHEADER]] ], [ [[INDVARS_IV_NEXT:%.*]], %[[INNER_LOOP]] ]
+; CHECK-NEXT:    [[GEP_INNER:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[INDVARS_IV]]
+; CHECK-NEXT:    store i8 0, ptr [[GEP_INNER]], align 1
+; CHECK-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
+; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp ne i64 [[INDVARS_IV_NEXT]], [[WIDE_TRIP_COUNT]]
+; CHECK-NEXT:    br i1 [[EXITCOND]], label %[[INNER_LOOP]], label %[[PH_LOOPEXIT]]
+; CHECK:       [[EXIT:.*:]]
+; CHECK-NEXT:    ret i32 0
+;
+entry:
+  br label %outer.loop
+
+ph:
+  br label %outer.loop
+
+outer.loop:
+  %iv.outer = phi i32 [ %iv.outer.1, %ph ], [ %iter.count, %entry ]
+  %iv.outer.1 = add nsw i32 %iv.outer, -1
+  %ext.outer = zext nneg i32 %iv.outer.1 to i64
+  %gep.outer = getelementptr i8, ptr %ptr, i64 %ext.outer
+  store i8 0, ptr %gep.outer
+  %exit.cond.outer = icmp samesign ugt i32 %iv.outer, 1
+  br i1 %exit.cond.outer, label %inner.loop, label %ph
+
+inner.loop:
+  %iv.inner = phi i32 [ %iv.next, %inner.loop ], [ 0, %outer.loop ]
+  %ext.inner = zext nneg i32 %iv.inner to i64
+  %gep.inner = getelementptr i8, ptr %ptr, i64 %ext.inner
+  store i8 0, ptr %gep.inner
+  %iv.next = add nuw nsw i32 %iv.inner, 1
+  %exit.cond.inner = icmp samesign ult i32 %iv.next, %iv.outer.1
+  br i1 %exit.cond.inner, label %inner.loop, label %ph
+
+exit:
+  ret i32 0
+}
+
+define i32 @iv_sext_sext_sgt_slt(i32 %iter.count, ptr %ptr) {
+; CHECK-LABEL: define i32 @iv_sext_sext_sgt_slt(
+; CHECK-SAME: i32 [[ITER_COUNT:%.*]], ptr [[PTR:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    [[TMP0:%.*]] = add nsw i32 [[ITER_COUNT]], -1
+; CHECK-NEXT:    [[TMP1:%.*]] = sext i32 [[ITER_COUNT]] to i64
+; CHECK-NEXT:    br label %[[OUTER_LOOP:.*]]
+; CHECK:       [[PH_LOOPEXIT:.*]]:
+; CHECK-NEXT:    br label %[[PH:.*]]
+; CHECK:       [[PH]]:
+; CHECK-NEXT:    [[INDVARS_IV_NEXT2:%.*]] = add i32 [[INDVARS_IV1:%.*]], -1
+; CHECK-NEXT:    br label %[[OUTER_LOOP]]
+; CHECK:       [[OUTER_LOOP]]:
+; CHECK-NEXT:    [[INDVARS_IV3:%.*]] = phi i64 [ [[INDVARS_IV_NEXT4:%.*]], %[[PH]] ], [ [[TMP1]], %[[ENTRY]] ]
+; CHECK-NEXT:    [[INDVARS_IV1]] = phi i32 [ [[INDVARS_IV_NEXT2]], %[[PH]] ], [ [[TMP0]], %[[ENTRY]] ]
+; CHECK-NEXT:    [[INDVARS_IV_NEXT4]] = add nsw i64 [[INDVARS_IV3]], -1
+; CHECK-NEXT:    [[GEP_OUTER:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[INDVARS_IV_NEXT4]]
+; CHECK-NEXT:    store i8 0, ptr [[GEP_OUTER]], align 1
+; CHECK-NEXT:    [[EXIT_COND_OUTER:%.*]] = icmp sgt i64 [[INDVARS_IV3]], 1
+; CHECK-NEXT:    br i1 [[EXIT_COND_OUTER]], label %[[INNER_LOOP_PREHEADER:.*]], label %[[PH]]
+; CHECK:       [[INNER_LOOP_PREHEADER]]:
+; CHECK-NEXT:    [[WIDE_TRIP_COUNT:%.*]] = zext i32 [[INDVARS_IV1]] to i64
+; CHECK-NEXT:    br label %[[INNER_LOOP:.*]]
+; CHECK:       [[INNER_LOOP]]:
+; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ 0, %[[INNER_LOOP_PREHEADER]] ], [ [[INDVARS_IV_NEXT:%.*]], %[[INNER_LOOP]] ]
+; CHECK-NEXT:    [[GEP_INNER:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[INDVARS_IV]]
+; CHECK-NEXT:    store i8 0, ptr [[GEP_INNER]], align 1
+; CHECK-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
+; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp ne i64 [[INDVARS_IV_NEXT]], [[WIDE_TRIP_COUNT]]
+; CHECK-NEXT:    br i1 [[EXITCOND]], label %[[INNER_LOOP]], label %[[PH_LOOPEXIT]]
+; CHECK:       [[EXIT:.*:]]
+; CHECK-NEXT:    ret i32 0
+;
+entry:
+  br label %outer.loop
+
+ph:
+  br label %outer.loop
+
+outer.loop:
+  %iv.outer = phi i32 [ %iv.outer.1, %ph ], [ %iter.count, %entry ]
+  %iv.outer.1 = add nsw i32 %iv.outer, -1
+  %ext.outer = sext i32 %iv.outer.1 to i64
+  %gep.outer = getelementptr i8, ptr %ptr, i64 %ext.outer
+  store i8 0, ptr %gep.outer
+  %exit.cond.outer = icmp sgt i32 %iv.outer, 1
+  br i1 %exit.cond.outer, label %inner.loop, label %ph
+
+inner.loop:
+  %iv.inner = phi i32 [ %iv.next, %inner.loop ], [ 0, %outer.loop ]
+  %ext.inner = sext i32 %iv.inner to i64
+  %gep.inner = getelementptr i8, ptr %ptr, i64 %ext.inner
+  store i8 0, ptr %gep.inner
+  %iv.next = add nuw nsw i32 %iv.inner, 1
+  %exit.cond.inner = icmp slt i32 %iv.next, %iv.outer.1
+  br i1 %exit.cond.inner, label %inner.loop, label %ph
+
+exit:
+  ret i32 0
+}
+
+define i32 @iv_sext_sext_gt_slt(i32 %iter.count, ptr %ptr) {
+; CHECK-LABEL: define i32 @iv_sext_sext_gt_slt(
+; CHECK-SAME: i32 [[ITER_COUNT:%.*]], ptr [[PTR:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    [[TMP0:%.*]] = sext i32 [[ITER_COUNT]] to i64
+; CHECK-NEXT:    br label %[[OUTER_LOOP:.*]]
+; CHECK:       [[PH_LOOPEXIT:.*]]:
+; CHECK-NEXT:    br label %[[PH:.*]]
+; CHECK:       [[PH]]:
+; CHECK-NEXT:    br label %[[OUTER_LOOP]]
+; CHECK:       [[OUTER_LOOP]]:
+; CHECK-NEXT:    [[INDVARS_IV1:%.*]] = phi i64 [ [[INDVARS_IV_NEXT2:%.*]], %[[PH]] ], [ [[TMP0]], %[[ENTRY]] ]
+; CHECK-NEXT:    [[INDVARS_IV_NEXT2]] = add nsw i64 [[INDVARS_IV1]], -1
+; CHECK-NEXT:    [[GEP_OUTER:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[INDVARS_IV_NEXT2]]
+; CHECK-NEXT:    store i8 0, ptr [[GEP_OUTER]], align 1
+; CHECK-NEXT:    [[EXIT_COND_OUTER:%.*]] = icmp samesign ugt i64 [[INDVARS_IV1]], 1
+; CHECK-NEXT:    br i1 [[EXIT_COND_OUTER]], label %[[INNER_LOOP_PREHEADER:.*]], label %[[PH]]
+; CHECK:       [[INNER_LOOP_PREHEADER]]:
+; CHECK-NEXT:    br label %[[INNER_LOOP:.*]]
+; CHECK:       [[INNER_LOOP]]:
+; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ 0, %[[INNER_LOOP_PREHEADER]] ], [ [[INDVARS_IV_NEXT:%.*]], %[[INNER_LOOP]] ]
+; CHECK-NEXT:    [[GEP_INNER:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[INDVARS_IV]]
+; CHECK-NEXT:    store i8 0, ptr [[GEP_INNER]], align 1
+; CHECK-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
+; CHECK-NEXT:    [[EXIT_COND_INNER:%.*]] = icmp slt i64 [[INDVARS_IV_NEXT]], [[INDVARS_IV_NEXT2]]
+; CHECK-NEXT:    br i1 [[EXIT_COND_INNER]], label %[[INNER_LOOP]], label %[[PH_LOOPEXIT]]
+; CHECK:       [[EXIT:.*:]]
+; CHECK-NEXT:    ret i32 0
+;
+entry:
+  br label %outer.loop
+
+ph:
+  br label %outer.loop
+
+outer.loop:
+  %iv.outer = phi i32 [ %iv.outer.1, %ph ], [ %iter.count, %entry ]
+  %iv.outer.1 = add nsw i32 %iv.outer, -1
+  %ext.outer = sext i32 %iv.outer.1 to i64
+  %gep.outer = getelementptr i8, ptr %ptr, i64 %ext.outer
+  store i8 0, ptr %gep.outer
+  %exit.cond.outer = icmp samesign ugt i32 %iv.outer, 1
+  br i1 %exit.cond.outer, label %inner.loop, label %ph
+
+inner.loop:
+  %iv.inner = phi i32 [ %iv.next, %inner.loop ], [ 0, %outer.loop ]
+  %ext.inner = sext i32 %iv.inner to i64
+  %gep.inner = getelementptr i8, ptr %ptr, i64 %ext.inner
+  store i8 0, ptr %gep.inner
+  %iv.next = add nuw nsw i32 %iv.inner, 1
+  %exit.cond.inner = icmp slt i32 %iv.next, %iv.outer.1
+  br i1 %exit.cond.inner, label %inner.loop, label %ph
+
+exit:
+  ret i32 0
+}
+
+define i32 @iv_sext_sext_gt_slt_exitlimit(i32 %iter.count, i32 %exit.limit, ptr %ptr) {
+; CHECK-LABEL: define i32 @iv_sext_sext_gt_slt_exitlimit(
+; CHECK-SAME: i32 [[ITER_COUNT:%.*]], i32 [[EXIT_LIMIT:%.*]], ptr [[PTR:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    [[TMP0:%.*]] = sext i32 [[ITER_COUNT]] to i64
+; CHECK-NEXT:    [[TMP1:%.*]] = sext i32 [[EXIT_LIMIT]] to i64
+; CHECK-NEXT:    br label %[[OUTER_LOOP:.*]]
+; CHECK:       [[PH_LOOPEXIT:.*]]:
+; CHECK-NEXT:    br label %[[PH:.*]]
+; CHECK:       [[PH]]:
+; CHECK-NEXT:    br label %[[OUTER_LOOP]]
+; CHECK:       [[OUTER_LOOP]]:
+; CHECK-NEXT:    [[INDVARS_IV1:%.*]] = phi i64 [ [[INDVARS_IV_NEXT2:%.*]], %[[PH]] ], [ [[TMP0]], %[[ENTRY]] ]
+; CHECK-NEXT:    [[INDVARS_IV_NEXT2]] = add nsw i64 [[INDVARS_IV1]], -1
+; CHECK-NEXT:    [[GEP_OUTER:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[INDVARS_IV_NEXT2]]
+; CHECK-NEXT:    store i8 0, ptr [[GEP_OUTER]], align 1
+; CHECK-NEXT:    [[EXIT_COND_OUTER:%.*]] = icmp samesign ugt i64 [[INDVARS_IV1]], [[TMP1]]
+; CHECK-NEXT:    br i1 [[EXIT_COND_OUTER]], label %[[INNER_LOOP_PREHEADER:.*]], label %[[PH]]
+; CHECK:       [[INNER_LOOP_PREHEADER]]:
+; CHECK-NEXT:    br label %[[INNER_LOOP:.*]]
+; CHECK:       [[INNER_LOOP]]:
+; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ 0, %[[INNER_LOOP_PREHEADER]] ], [ [[INDVARS_IV_NEXT:%.*]], %[[INNER_LOOP]] ]
+; CHECK-NEXT:    [[GEP_INNER:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[INDVARS_IV]]
+; CHECK-NEXT:    store i8 0, ptr [[GEP_INNER]], align 1
+; CHECK-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
+; CHECK-NEXT:    [[EXIT_COND_INNER:%.*]] = icmp slt i64 [[INDVARS_IV_NEXT]], [[INDVARS_IV_NEXT2]]
+; CHECK-NEXT:    br i1 [[EXIT_COND_INNER]], label %[[INNER_LOOP]], label %[[PH_LOOPEXIT]]
+; CHECK:       [[EXIT:.*:]]
+; CHECK-NEXT:    ret i32 0
+;
+entry:
+  br label %outer.loop
+
+ph:
+  br label %outer.loop
+
+outer.loop:
+  %iv.outer = phi i32 [ %iv.outer.1, %ph ], [ %iter.count, %entry ]
+  %iv.outer.1 = add nsw i32 %iv.outer, -1
+  %ext.outer = sext i32 %iv.outer.1 to i64
+  %gep.outer = getelementptr i8, ptr %ptr, i64 %ext.outer
+  store i8 0, ptr %gep.outer
+  %exit.cond.outer = icmp samesign ugt i32 %iv.outer, %exit.limit
+  br i1 %exit.cond.outer, label %inner.loop, label %ph
+
+inner.loop:
+  %iv.inner = phi i32 [ %iv.next, %inner.loop ], [ 0, %outer.loop ]
+  %ext.inner = sext i32 %iv.inner to i64
+  %gep.inner = getelementptr i8, ptr %ptr, i64 %ext.inner
+  store i8 0, ptr %gep.inner
+  %iv.next = add nuw nsw i32 %iv.inner, 1
+  %exit.cond.inner = icmp slt i32 %iv.next, %iv.outer.1
+  br i1 %exit.cond.inner, label %inner.loop, label %ph
+
+exit:
+  ret i32 0
+}
+
+define i32 @iv_sext_sext_sgt_lt(i32 %iter.count, ptr %ptr) {
+; CHECK-LABEL: define i32 @iv_sext_sext_sgt_lt(
+; CHECK-SAME: i32 [[ITER_COUNT:%.*]], ptr [[PTR:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    [[TMP0:%.*]] = add nsw i32 [[ITER_COUNT]], -1
+; CHECK-NEXT:    [[TMP1:%.*]] = sext i32 [[ITER_COUNT]] to i64
+; CHECK-NEXT:    br label %[[OUTER_LOOP:.*]]
+; CHECK:       [[PH_LOOPEXIT:.*]]:
+; CHECK-NEXT:    br label %[[PH:.*]]
+; CHECK:       [[PH]]:
+; CHECK-NEXT:    [[INDVARS_IV_NEXT2:%.*]] = add i32 [[INDVARS_IV1:%.*]], -1
+; CHECK-NEXT:    br label %[[OUTER_LOOP]]
+; CHECK:       [[OUTER_LOOP]]:
+; CHECK-NEXT:    [[INDVARS_IV3:%.*]] = phi i64 [ [[INDVARS_IV_NEXT4:%.*]], %[[PH]] ], [ [[TMP1]], %[[ENTRY]] ]
+; CHECK-NEXT:    [[INDVARS_IV1]] = phi i32 [ [[INDVARS_IV_NEXT2]], %[[PH]] ], [ [[TMP0]], %[[ENTRY]] ]
+; CHECK-NEXT:    [[INDVARS_IV_NEXT4]] = add nsw i64 [[INDVARS_IV3]], -1
+; CHECK-NEXT:    [[GEP_OUTER:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[INDVARS_IV_NEXT4]]
+; CHECK-NEXT:    store i8 0, ptr [[GEP_OUTER]], align 1
+; CHECK-NEXT:    [[EXIT_COND_OUTER:%.*]] = icmp sgt i64 [[INDVARS_IV3]], 1
+; CHECK-NEXT:    br i1 [[EXIT_COND_OUTER]], label %[[INNER_LOOP_PREHEADER:.*]], label %[[PH]]
+; CHECK:       [[INNER_LOOP_PREHEADER]]:
+; CHECK-NEXT:    [[WIDE_TRIP_COUNT:%.*]] = zext i32 [[INDVARS_IV1]] to i64
+; CHECK-NEXT:    br label %[[INNER_LOOP:.*]]
+; CHECK:       [[INNER_LOOP]]:
+; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ 0, %[[INNER_LOOP_PREHEADER]] ], [ [[INDVARS_IV_NEXT:%.*]], %[[INNER_LOOP]] ]
+; CHECK-NEXT:    [[GEP_INNER:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[INDVARS_IV]]
+; CHECK-NEXT:    store i8 0, ptr [[GEP_INNER]], align 1
+; CHECK-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
+; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp ne i64 [[INDVARS_IV_NEXT]], [[WIDE_TRIP_COUNT]]
+; CHECK-NEXT:    br i1 [[EXITCOND]], label %[[INNER_LOOP]], label %[[PH_LOOPEXIT]]
+; CHECK:       [[EXIT:.*:]]
+; CHECK-NEXT:    ret i32 0
+;
+entry:
+  br label %outer.loop
+
+ph:
+  br label %outer.loop
+
+outer.loop:
+  %iv.outer = phi i32 [ %iv.outer.1, %ph ], [ %iter.count, %entry ]
+  %iv.outer.1 = add nsw i32 %iv.outer, -1
+  %ext.outer = sext i32 %iv.outer.1 to i64
+  %gep.outer = getelementptr i8, ptr %ptr, i64 %ext.outer
+  store i8 0, ptr %gep.outer
+  %exit.cond.outer = icmp sgt i32 %iv.outer, 1
+  br i1 %exit.cond.outer, label %inner.loop, label %ph
+
+inner.loop:
+  %iv.inner = phi i32 [ %iv.next, %inner.loop ], [ 0, %outer.loop ]
+  %ext.inner = sext i32 %iv.inner to i64
+  %gep.inner = getelementptr i8, ptr %ptr, i64 %ext.inner
+  store i8 0, ptr %gep.inner
+  %iv.next = add nuw nsw i32 %iv.inner, 1
+  %exit.cond.inner = icmp samesign ult i32 %iv.next, %iv.outer.1
+  br i1 %exit.cond.inner, label %inner.loop, label %ph
+
+exit:
+  ret i32 0
+}
+
+define i32 @iv_sext_sext_sgt_lt_exitlimit(i32 %iter.count, i32 %exit.limit, ptr %ptr) {
+; CHECK-LABEL: define i32 @iv_sext_sext_sgt_lt_exitlimit(
+; CHECK-SAME: i32 [[ITER_COUNT:%.*]], i32 [[EXIT_LIMIT:%.*]], ptr [[PTR:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    [[TMP1:%.*]] = sext i32 [[ITER_COUNT]] to i64
+; CHECK-NEXT:    [[TMP4:%.*]] = sext i32 [[EXIT_LIMIT]] to i64
+; CHECK-NEXT:    br label %[[OUTER_LOOP:.*]]
+; CHECK:       [[PH_LOOPEXIT:.*]]:
+; CHECK-NEXT:    br label %[[PH:.*]]
+; CHECK:       [[PH]]:
+; CHECK-NEXT:    br label %[[OUTER_LOOP]]
+; CHECK:       [[OUTER_LOOP]]:
+; CHECK-NEXT:    [[INDVARS_IV1:%.*]] = phi i64 [ [[INDVARS_IV_NEXT2:%.*]], %[[PH]] ], [ [[TMP1]], %[[ENTRY]] ]
+; CHECK-NEXT:    [[INDVARS_IV_NEXT2]] = add nsw i64 [[INDVARS_IV1]], -1
+; CHECK-NEXT:    [[GEP_OUTER:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[INDVARS_IV_NEXT2]]
+; CHECK-NEXT:    store i8 0, ptr [[GEP_OUTER]], align 1
+; CHECK-NEXT:    [[EXIT_COND_OUTER:%.*]] = icmp sgt i64 [[INDVARS_IV1]], [[TMP4]]
+; CHECK-NEXT:    br i1 [[EXIT_COND_OUTER]], label %[[INNER_LOOP_PREHEADER:.*]], label %[[PH]]
+; CHECK:       [[INNER_LOOP_PREHEADER]]:
+; CHECK-NEXT:    br label %[[INNER_LOOP:.*]]
+; CHECK:       [[INNER_LOOP]]:
+; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ 0, %[[INNER_LOOP_PREHEADER]] ], [ [[INDVARS_IV_NEXT:%.*]], %[[INNER_LOOP]] ]
+; CHECK-NEXT:    [[GEP_INNER:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[INDVARS_IV]]
+; CHECK-NEXT:    store i8 0, ptr [[GEP_INNER]], align 1
+; CHECK-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
+; CHECK-NEXT:    [[EXIT_COND_INNER:%.*]] = icmp samesign ult i64 [[INDVARS_IV_NEXT]], [[INDVARS_IV_NEXT2]]
+; CHECK-NEXT:    br i1 [[EXIT_COND_INNER]], label %[[INNER_LOOP]], label %[[PH_LOOPEXIT]]
+; CHECK:       [[EXIT:.*:]]
+; CHECK-NEXT:    ret i32 0
+;
+entry:
+  br label %outer.loop
+
+ph:
+  br label %outer.loop
+
+outer.loop:
+  %iv.outer = phi i32 [ %iv.outer.1, %ph ], [ %iter.count, %entry ]
+  %iv.outer.1 = add nsw i32 %iv.outer, -1
+  %ext.outer = sext i32 %iv.outer.1 to i64
+  %gep.outer = getelementptr i8, ptr %ptr, i64 %ext.outer
+  store i8 0, ptr %gep.outer
+  %exit.cond.outer = icmp sgt i32 %iv.outer, %exit.limit
+  br i1 %exit.cond.outer, label %inner.loop, label %ph
+
+inner.loop:
+  %iv.inner = phi i32 [ %iv.next, %inner.loop ], [ 0, %outer.loop ]
+  %ext.inner = sext i32 %iv.inner to i64
+  %gep.inner = getelementptr i8, ptr %ptr, i64 %ext.inner
+  store i8 0, ptr %gep.inner
+  %iv.next = add nuw nsw i32 %iv.inner, 1
+  %exit.cond.inner = icmp samesign ult i32 %iv.next, %iv.outer.1
+  br i1 %exit.cond.inner, label %inner.loop, label %ph
+
+exit:
+  ret i32 0
+}
+
+define i32 @iv_sext_sext_gt_lt(i32 %iter.count, ptr %ptr) {
+; CHECK-LABEL: define i32 @iv_sext_sext_gt_lt(
+; CHECK-SAME: i32 [[ITER_COUNT:%.*]], ptr [[PTR:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    [[TMP0:%.*]] = add nsw i32 [[ITER_COUNT]], -1
+; CHECK-NEXT:    [[TMP1:%.*]] = sext i32 [[ITER_COUNT]] to i64
+; CHECK-NEXT:    br label %[[OUTER_LOOP:.*]]
+; CHECK:       [[PH_LOOPEXIT:.*]]:
+; CHECK-NEXT:    br label %[[PH:.*]]
+; CHECK:       [[PH]]:
+; CHECK-NEXT:    [[INDVARS_IV_NEXT2:%.*]] = add i32 [[INDVARS_IV1:%.*]], -1
+; CHECK-NEXT:    br label %[[OUTER_LOOP]]
+; CHECK:       [[OUTER_LOOP]]:
+; CHECK-NEXT:    [[INDVARS_IV3:%.*]] = phi i64 [ [[INDVARS_IV_NEXT4:%.*]], %[[PH]] ], [ [[TMP1]], %[[ENTRY]] ]
+; CHECK-NEXT:    [[INDVARS_IV1]] = phi i32 [ [[INDVARS_IV_NEXT2]], %[[PH]] ], [ [[TMP0]], %[[ENTRY]] ]
+; CHECK-NEXT:    [[INDVARS_IV_NEXT4]] = add nsw i64 [[INDVARS_IV3]], -1
+; CHECK-NEXT:    [[GEP_OUTER:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[INDVARS_IV_NEXT4]]
+; CHECK-NEXT:    store i8 0, ptr [[GEP_OUTER]], align 1
+; CHECK-NEXT:    [[EXIT_COND_OUTER:%.*]] = icmp samesign ugt i64 [[INDVARS_IV3]], 1
+; CHECK-NEXT:    br i1 [[EXIT_COND_OUTER]], label %[[INNER_LOOP_PREHEADER:.*]], label %[[PH]]
+; CHECK:       [[INNER_LOOP_PREHEADER]]:
+; CHECK-NEXT:    [[WIDE_TRIP_COUNT:%.*]] = zext i32 [[INDVARS_IV1]] to i64
+; CHECK-NEXT:    br label %[[INNER_LOOP:.*]]
+; CHECK:       [[INNER_LOOP]]:
+; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ 0, %[[INNER_LOOP_PREHEADER]] ], [ [[INDVARS_IV_NEXT:%.*]], %[[INNER_LOOP]] ]
+; CHECK-NEXT:    [[GEP_INNER:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[INDVARS_IV]]
+; CHECK-NEXT:    store i8 0, ptr [[GEP_INNER]], align 1
+; CHECK-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
+; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp ne i64 [[INDVARS_IV_NEXT]], [[WIDE_TRIP_COUNT]]
+; CHECK-NEXT:    br i1 [[EXITCOND]], label %[[INNER_LOOP]], label %[[PH_LOOPEXIT]]
+; CHECK:       [[EXIT:.*:]]
+; CHECK-NEXT:    ret i32 0
+;
+entry:
+  br label %outer.loop
+
+ph:
+  br label %outer.loop
+
+outer.loop:
+  %iv.outer = phi i32 [ %iv.outer.1, %ph ], [ %iter.count, %entry ]
+  %iv.outer.1 = add nsw i32 %iv.outer, -1
+  %ext.outer = sext i32 %iv.outer.1 to i64
+  %gep.outer = getelementptr i8, ptr %ptr, i64 %ext.outer
+  store i8 0, ptr %gep.outer
+  %exit.cond.outer = icmp samesign ugt i32 %iv.outer, 1
+  br i1 %exit.cond.outer, label %inner.loop, label %ph
+
+inner.loop:
+  %iv.inner = phi i32 [ %iv.next, %inner.loop ], [ 0, %outer.loop ]
+  %ext.inner = sext i32 %iv.inner to i64
+  %gep.inner = getelementptr i8, ptr %ptr, i64 %ext.inner
+  store i8 0, ptr %gep.inner
+  %iv.next = add nuw nsw i32 %iv.inner, 1
+  %exit.cond.inner = icmp samesign ult i32 %iv.next, %iv.outer.1
+  br i1 %exit.cond.inner, label %inner.loop, label %ph
+
+exit:
+  ret i32 0
+}
diff --git a/llvm/test/Transforms/IndVarSimplify/lftr.ll b/llvm/test/Transforms/IndVarSimplify/lftr.ll
index def2cd9b8c679..a67709829b2f4 100644
--- a/llvm/test/Transforms/IndVarSimplify/lftr.ll
+++ b/llvm/test/Transforms/IndVarSimplify/lftr.ll
@@ -309,7 +309,7 @@ define i64 @no_undef_counter() nounwind {
 ; CHECK-NEXT:    [[UNDEF:%.*]] = phi i64 [ [[NEXT_UNDEF:%.*]], [[BLOCK9]] ], [ undef, [[FUNC_START:%.*]] ]
 ; CHECK-NEXT:    [[ITER:%.*]] = phi i64 [ [[NEXT_ITER:%.*]], [[BLOCK9]] ], [ 1, [[FUNC_START]] ]
 ; CHECK-NEXT:    [[NEXT_ITER]] = add nuw nsw i64 [[ITER]], 1
-; CHECK-NEXT:    [[TMP0:%.*]] = tail call i32 (ptr, ...) @printf(ptr noalias nocapture @.str3, i64 [[NEXT_ITER]], i64 [[UNDEF]])
+; CHECK-NEXT:    [[TMP0:%.*]] = tail call i32 (ptr, ...) @printf(ptr noalias captures(none) @.str3, i64 [[NEXT_ITER]], i64 [[UNDEF]])
 ; CHECK-NEXT:    [[NEXT_UNDEF]] = add nsw i64 [[UNDEF]], 1
 ; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp ne i64 [[NEXT_ITER]], 100
 ; CHECK-NEXT:    br i1 [[EXITCOND]], label [[BLOCK9]], label [[EXIT:%.*]]
diff --git a/llvm/test/Transforms/IndVarSimplify/pr126012.ll b/llvm/test/Transforms/IndVarSimplify/pr126012.ll
new file mode 100644
index 0000000000000..5189fe020dd3b
--- /dev/null
+++ b/llvm/test/Transforms/IndVarSimplify/pr126012.ll
@@ -0,0 +1,53 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -S -passes=indvars < %s | FileCheck %s
+
+; Do not infer that %cmp is true. The %indvar3 input of %indvar2 comes from
+; a previous iteration, so we should not compare it to a value from the current
+; iteration.
+define i32 @test() {
+; CHECK-LABEL: define i32 @test() {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[FOR_PREHEADER:.*]]
+; CHECK:       [[FOR_PREHEADER]]:
+; CHECK-NEXT:    [[INDVAR1:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[PHI:%.*]], %[[FOR_INC:.*]] ]
+; CHECK-NEXT:    [[INDVAR2:%.*]] = phi i32 [ 1, %[[ENTRY]] ], [ [[INDVAR3:%.*]], %[[FOR_INC]] ]
+; CHECK-NEXT:    [[INDVAR3]] = phi i32 [ 0, %[[ENTRY]] ], [ [[INC:%.*]], %[[FOR_INC]] ]
+; CHECK-NEXT:    [[COND1:%.*]] = icmp eq i32 [[INDVAR3]], 0
+; CHECK-NEXT:    br i1 [[COND1]], label %[[FOR_INC]], label %[[FOR_END:.*]]
+; CHECK:       [[FOR_END]]:
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ugt i32 [[INDVAR2]], 0
+; CHECK-NEXT:    [[EXT:%.*]] = zext i1 [[CMP]] to i32
+; CHECK-NEXT:    br label %[[FOR_INC]]
+; CHECK:       [[FOR_INC]]:
+; CHECK-NEXT:    [[PHI]] = phi i32 [ [[EXT]], %[[FOR_END]] ], [ 0, %[[FOR_PREHEADER]] ]
+; CHECK-NEXT:    [[INC]] = add nuw nsw i32 [[INDVAR3]], 1
+; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i32 [[INDVAR3]], 2
+; CHECK-NEXT:    br i1 [[EXITCOND]], label %[[FOR_EXIT:.*]], label %[[FOR_PREHEADER]]
+; CHECK:       [[FOR_EXIT]]:
+; CHECK-NEXT:    [[INDVAR1_LCSSA:%.*]] = phi i32 [ [[INDVAR1]], %[[FOR_INC]] ]
+; CHECK-NEXT:    ret i32 [[INDVAR1_LCSSA]]
+;
+entry:
+  br label %for.preheader
+
+for.preheader:
+  %indvar1 = phi i32 [ 0, %entry ], [ %phi, %for.inc ]
+  %indvar2 = phi i32 [ 1, %entry ], [ %indvar3, %for.inc ]
+  %indvar3 = phi i32 [ 0, %entry ], [ %inc, %for.inc ]
+  %cond1 = icmp eq i32 %indvar3, 0
+  br i1 %cond1, label %for.inc, label %for.end
+
+for.end:
+  %cmp = icmp sgt i32 %indvar2, 0
+  %ext = zext i1 %cmp to i32
+  br label %for.inc
+
+for.inc:
+  %phi = phi i32 [ %ext, %for.end ], [ 0, %for.preheader ]
+  %inc = add i32 %indvar3, 1
+  %exitcond = icmp eq i32 %indvar3, 2
+  br i1 %exitcond, label %for.exit, label %for.preheader
+
+for.exit:
+  ret i32 %indvar1
+}
diff --git a/llvm/test/Transforms/IndVarSimplify/preserve-nsw-during-expansion.ll b/llvm/test/Transforms/IndVarSimplify/preserve-nsw-during-expansion.ll
index 080bc9b42bbed..f13dad0b87753 100644
--- a/llvm/test/Transforms/IndVarSimplify/preserve-nsw-during-expansion.ll
+++ b/llvm/test/Transforms/IndVarSimplify/preserve-nsw-during-expansion.ll
@@ -6,7 +6,7 @@ target datalayout = "e-m:o-i64:64-i128:128-n32:64-S128"
 ; Based on TSVC s172.
 define void @test_s172(i32 noundef %xa, i32 noundef %xb, ptr nocapture noundef %a, ptr nocapture noundef readonly %b) {
 ; CHECK-LABEL: define void @test_s172(
-; CHECK-SAME: i32 noundef [[XA:%.*]], i32 noundef [[XB:%.*]], ptr nocapture noundef [[A:%.*]], ptr nocapture noundef readonly [[B:%.*]]) {
+; CHECK-SAME: i32 noundef [[XA:%.*]], i32 noundef [[XB:%.*]], ptr noundef captures(none) [[A:%.*]], ptr noundef readonly captures(none) [[B:%.*]]) {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[SUB:%.*]] = add i32 [[XA]], -1
 ; CHECK-NEXT:    [[CMP1:%.*]] = icmp slt i32 [[XA]], 32001
diff --git a/llvm/test/Transforms/IndVarSimplify/rewrite-loop-exit-values-phi.ll b/llvm/test/Transforms/IndVarSimplify/rewrite-loop-exit-values-phi.ll
index 37bc67c23adb7..b1043b48d40ac 100644
--- a/llvm/test/Transforms/IndVarSimplify/rewrite-loop-exit-values-phi.ll
+++ b/llvm/test/Transforms/IndVarSimplify/rewrite-loop-exit-values-phi.ll
@@ -6,19 +6,17 @@
 ; status. This was caught by the expensive check introduced in D86589.
 
 target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
-target triple = "x86_64-unknown-linux-gnu"
 
-@ptr = external global i64
-
-define dso_local void @hoge() local_unnamed_addr {
+define void @hoge(i64 %x, i64 %idx.start, ptr %ptr) {
 ; CHECK-LABEL: @hoge(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[N:%.*]] = sdiv exact i64 undef, 40
-; CHECK-NEXT:    [[TMP0:%.*]] = sub i64 undef, [[N]]
+; CHECK-NEXT:    [[N:%.*]] = sdiv exact i64 [[X:%.*]], 40
+; CHECK-NEXT:    [[TMP0:%.*]] = add i64 [[IDX_START:%.*]], 1
+; CHECK-NEXT:    [[TMP1:%.*]] = sub i64 [[TMP0]], [[N]]
 ; CHECK-NEXT:    br label [[HEADER:%.*]]
 ; CHECK:       header:
-; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], [[LATCH:%.*]] ], [ [[TMP0]], [[ENTRY:%.*]] ]
-; CHECK-NEXT:    [[IDX:%.*]] = phi i64 [ [[IDX_NEXT:%.*]], [[LATCH]] ], [ undef, [[ENTRY]] ]
+; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], [[LATCH:%.*]] ], [ [[TMP1]], [[ENTRY:%.*]] ]
+; CHECK-NEXT:    [[IDX:%.*]] = phi i64 [ [[IDX_NEXT:%.*]], [[LATCH]] ], [ [[IDX_START]], [[ENTRY]] ]
 ; CHECK-NEXT:    [[COND:%.*]] = icmp sgt i64 [[N]], [[IDX]]
 ; CHECK-NEXT:    br i1 [[COND]], label [[END:%.*]], label [[INNER_PREHEADER:%.*]]
 ; CHECK:       inner.preheader:
@@ -26,7 +24,7 @@ define dso_local void @hoge() local_unnamed_addr {
 ; CHECK:       inner:
 ; CHECK-NEXT:    [[I:%.*]] = phi i64 [ [[I_NEXT:%.*]], [[INNER]] ], [ 0, [[INNER_PREHEADER]] ]
 ; CHECK-NEXT:    [[I_NEXT]] = add nuw i64 [[I]], 1
-; CHECK-NEXT:    store i64 undef, ptr @ptr, align 8
+; CHECK-NEXT:    store i64 0, ptr [[PTR:%.*]], align 8
 ; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp ne i64 [[I_NEXT]], [[INDVARS_IV]]
 ; CHECK-NEXT:    br i1 [[EXITCOND]], label [[INNER]], label [[INNER_EXIT:%.*]]
 ; CHECK:       inner_exit:
@@ -41,11 +39,11 @@ define dso_local void @hoge() local_unnamed_addr {
 ; CHECK-NEXT:    ret void
 ;
 entry:                                            ; preds = %entry
-  %n = sdiv exact i64 undef, 40
+  %n = sdiv exact i64 %x, 40
   br label %header
 
 header:                                           ; preds = %latch, %entry
-  %idx = phi i64 [ %idx.next, %latch ], [ undef, %entry ]
+  %idx = phi i64 [ %idx.next, %latch ], [ %idx.start, %entry ]
   %cond = icmp sgt i64 %n, %idx
   br i1 %cond, label %end, label %inner
 
@@ -54,7 +52,7 @@ inner:                                            ; preds = %inner, %header
   %j = phi i64 [ %j.next, %inner ], [ %n, %header ]
   %i.next = add nsw i64 %i, 1
   %j.next = add nsw i64 %j, 1
-  store i64 undef, ptr @ptr
+  store i64 0, ptr %ptr
   %cond1 = icmp slt i64 %j, %idx
   br i1 %cond1, label %inner, label %inner_exit
 
diff --git a/llvm/test/Transforms/InferAddressSpaces/AMDGPU/store-pointer-to-self.ll b/llvm/test/Transforms/InferAddressSpaces/AMDGPU/store-pointer-to-self.ll
index 5fcf1f6b31178..0b2c4744d4723 100644
--- a/llvm/test/Transforms/InferAddressSpaces/AMDGPU/store-pointer-to-self.ll
+++ b/llvm/test/Transforms/InferAddressSpaces/AMDGPU/store-pointer-to-self.ll
@@ -196,7 +196,7 @@ define void @mixed_volatile_multi_store_flat_pointer_to_self() {
 
 define amdgpu_kernel void @uselist_regression_skipped_load(ptr nocapture readonly %Arg, i32 %i) {
 ; CHECK-LABEL: define amdgpu_kernel void @uselist_regression_skipped_load(
-; CHECK-SAME: ptr nocapture readonly [[ARG:%.*]], i32 [[I:%.*]]) {
+; CHECK-SAME: ptr readonly captures(none) [[ARG:%.*]], i32 [[I:%.*]]) {
 ; CHECK-NEXT:  [[ENTRY:.*:]]
 ; CHECK-NEXT:    [[ARG_GLOBAL:%.*]] = addrspacecast ptr [[ARG]] to ptr addrspace(1)
 ; CHECK-NEXT:    [[P1:%.*]] = getelementptr inbounds ptr, ptr addrspace(1) [[ARG_GLOBAL]], i32 [[I]]
diff --git a/llvm/test/Transforms/InferAddressSpaces/NVPTX/bug31948.ll b/llvm/test/Transforms/InferAddressSpaces/NVPTX/bug31948.ll
index 753e7b505317d..5ead48aac18e1 100644
--- a/llvm/test/Transforms/InferAddressSpaces/NVPTX/bug31948.ll
+++ b/llvm/test/Transforms/InferAddressSpaces/NVPTX/bug31948.ll
@@ -9,7 +9,7 @@ target datalayout = "e-i64:64-v16:16-v32:32-n16:32:64"
 
 define void @bug31948(float %a, ptr nocapture readnone %x, ptr nocapture readnone %y) local_unnamed_addr #0 {
 ; CHECK-LABEL: define void @bug31948(
-; CHECK-SAME: float [[A:%.*]], ptr nocapture readnone [[X:%.*]], ptr nocapture readnone [[Y:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
+; CHECK-SAME: float [[A:%.*]], ptr readnone captures(none) [[X:%.*]], ptr readnone captures(none) [[Y:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
 ; CHECK-NEXT:  [[ENTRY:.*:]]
 ; CHECK-NEXT:    [[TMP:%.*]] = load ptr, ptr addrspace(3) getelementptr ([[STRUCT_BAR:%.*]], ptr addrspace(3) @var1, i64 0, i32 1), align 8
 ; CHECK-NEXT:    [[TMP1:%.*]] = load float, ptr [[TMP]], align 4
diff --git a/llvm/test/Transforms/InferFunctionAttrs/annotate.ll b/llvm/test/Transforms/InferFunctionAttrs/annotate.ll
index e63487de61ba2..e4b9a8d25d1bc 100644
--- a/llvm/test/Transforms/InferFunctionAttrs/annotate.ll
+++ b/llvm/test/Transforms/InferFunctionAttrs/annotate.ll
@@ -198,7 +198,7 @@ declare float @__sinpif(float)
 ; CHECK: declare i32 @abs(i32) [[NOFREE_NOUNWIND_WILLRETURN_WRITEONLY:#[0-9]+]]
 declare i32 @abs(i32)
 
-; CHECK: declare noundef i32 @access(ptr nocapture noundef readonly, i32 noundef) [[NOFREE_NOUNWIND:#[0-9]+]]
+; CHECK: declare noundef i32 @access(ptr noundef readonly captures(none), i32 noundef) [[NOFREE_NOUNWIND:#[0-9]+]]
 declare i32 @access(ptr, i32)
 
 ; CHECK: declare double @acos(double) [[NOFREE_NOUNWIND_WILLRETURN_WRITEONLY]]
@@ -267,26 +267,26 @@ declare x86_fp80 @atanhl(x86_fp80)
 ; CHECK: declare x86_fp80 @atanl(x86_fp80) [[NOFREE_NOUNWIND_WILLRETURN_WRITEONLY]]
 declare x86_fp80 @atanl(x86_fp80)
 
-; CHECK: declare double @atof(ptr nocapture) [[NOFREE_NOUNWIND_READONLY_WILLRETURN:#[0-9]+]]
+; CHECK: declare double @atof(ptr captures(none)) [[NOFREE_NOUNWIND_READONLY_WILLRETURN:#[0-9]+]]
 declare double @atof(ptr)
 
-; CHECK: declare i32 @atoi(ptr nocapture) [[NOFREE_NOUNWIND_READONLY_WILLRETURN]]
+; CHECK: declare i32 @atoi(ptr captures(none)) [[NOFREE_NOUNWIND_READONLY_WILLRETURN]]
 declare i32 @atoi(ptr)
 
-; CHECK: declare i64 @atol(ptr nocapture) [[NOFREE_NOUNWIND_READONLY_WILLRETURN]]
+; CHECK: declare i64 @atol(ptr captures(none)) [[NOFREE_NOUNWIND_READONLY_WILLRETURN]]
 declare i64 @atol(ptr)
 
-; CHECK: declare i64 @atoll(ptr nocapture) [[NOFREE_NOUNWIND_READONLY_WILLRETURN]]
+; CHECK: declare i64 @atoll(ptr captures(none)) [[NOFREE_NOUNWIND_READONLY_WILLRETURN]]
 declare i64 @atoll(ptr)
 
-; CHECK-LINUX: declare i32 @bcmp(ptr nocapture, ptr nocapture, i64) [[ARGMEMONLY_NOFREE_NOUNWIND_READONLY:#[0-9]+]]
+; CHECK-LINUX: declare i32 @bcmp(ptr captures(none), ptr captures(none), i64) [[ARGMEMONLY_NOFREE_NOUNWIND_READONLY:#[0-9]+]]
 ; CHECK-NOLINUX: declare i32 @bcmp(ptr, ptr, i64){{$}}
 declare i32 @bcmp(ptr, ptr, i64)
 
-; CHECK: declare void @bcopy(ptr nocapture readonly, ptr nocapture writeonly, i64)  [[ARGMEMONLY_NOFREE_NOUNWIND_WILLRETURN:#[0-9]+]]
+; CHECK: declare void @bcopy(ptr readonly captures(none), ptr writeonly captures(none), i64)  [[ARGMEMONLY_NOFREE_NOUNWIND_WILLRETURN:#[0-9]+]]
 declare void @bcopy(ptr, ptr, i64)
 
-; CHECK: declare void @bzero(ptr nocapture writeonly, i64)  [[ARGMEMONLY_NOFREE_NOUNWIND_WILLRETURN]]
+; CHECK: declare void @bzero(ptr writeonly captures(none), i64)  [[ARGMEMONLY_NOFREE_NOUNWIND_WILLRETURN]]
 declare void @bzero(ptr, i64)
 
 ; CHECK: declare noalias noundef ptr @calloc(i64 noundef, i64 noundef) [[INACCESSIBLEMEMONLY_NOFREE_NOUNWIND_WILLRETURN_ALLOCKIND_ALLOCZEROED_ALLOCSIZE01_FAMILY_MALLOC:#[0-9]+]]
@@ -318,16 +318,16 @@ declare x86_fp80 @ceill(x86_fp80)
 ; the function is still recognized.
 ; FIXME: this should be tightened up to verify that only the type with
 ; the right size for the target matches.
-; CHECK: declare noundef i32 @chmod(ptr nocapture noundef readonly, i16 noundef zeroext) [[NOFREE_NOUNWIND]]
+; CHECK: declare noundef i32 @chmod(ptr noundef readonly captures(none), i16 noundef zeroext) [[NOFREE_NOUNWIND]]
 declare i32 @chmod(ptr, i16 zeroext)
 
-; CHECK: declare noundef i32 @chown(ptr nocapture noundef readonly, i32 noundef, i32 noundef) [[NOFREE_NOUNWIND]]
+; CHECK: declare noundef i32 @chown(ptr noundef readonly captures(none), i32 noundef, i32 noundef) [[NOFREE_NOUNWIND]]
 declare i32 @chown(ptr, i32, i32)
 
-; CHECK: declare void @clearerr(ptr nocapture noundef) [[NOFREE_NOUNWIND]]
+; CHECK: declare void @clearerr(ptr noundef captures(none)) [[NOFREE_NOUNWIND]]
 declare void @clearerr(ptr)
 
-; CHECK: declare noundef i32 @closedir(ptr nocapture noundef) [[NOFREE_NOUNWIND]]
+; CHECK: declare noundef i32 @closedir(ptr noundef captures(none)) [[NOFREE_NOUNWIND]]
 declare i32 @closedir(ptr)
 
 ; CHECK: declare double @copysign(double, double) [[NOFREE_NOUNWIND_WILLRETURN_WRITEONLY]]
@@ -357,7 +357,7 @@ declare x86_fp80 @coshl(x86_fp80)
 ; CHECK: declare x86_fp80 @cosl(x86_fp80) [[NOFREE_NOUNWIND_WILLRETURN_WRITEONLY]]
 declare x86_fp80 @cosl(x86_fp80)
 
-; CHECK: declare noundef ptr @ctermid(ptr nocapture noundef) [[NOFREE_NOUNWIND]]
+; CHECK: declare noundef ptr @ctermid(ptr noundef captures(none)) [[NOFREE_NOUNWIND]]
 declare ptr @ctermid(ptr)
 
 ; CHECK: declare double @exp(double) [[NOFREE_NOUNWIND_WILLRETURN_WRITEONLY]]
@@ -396,19 +396,19 @@ declare float @fabsf(float)
 ; CHECK: declare x86_fp80 @fabsl(x86_fp80) [[NOFREE_NOUNWIND_WILLRETURN_WRITEONLY]]
 declare x86_fp80 @fabsl(x86_fp80)
 
-; CHECK: declare noundef i32 @fclose(ptr nocapture noundef) [[NOFREE_NOUNWIND]]
+; CHECK: declare noundef i32 @fclose(ptr noundef captures(none)) [[NOFREE_NOUNWIND]]
 declare i32 @fclose(ptr)
 
-; CHECK: declare noalias noundef ptr @fdopen(i32 noundef, ptr nocapture noundef readonly) [[NOFREE_NOUNWIND]]
+; CHECK: declare noalias noundef ptr @fdopen(i32 noundef, ptr noundef readonly captures(none)) [[NOFREE_NOUNWIND]]
 declare ptr @fdopen(i32, ptr)
 
-; CHECK: declare noundef i32 @feof(ptr nocapture noundef) [[NOFREE_NOUNWIND]]
+; CHECK: declare noundef i32 @feof(ptr noundef captures(none)) [[NOFREE_NOUNWIND]]
 declare i32 @feof(ptr)
 
-; CHECK: declare noundef i32 @ferror(ptr nocapture noundef) [[NOFREE_NOUNWIND_READONLY:#[0-9]+]]
+; CHECK: declare noundef i32 @ferror(ptr noundef captures(none)) [[NOFREE_NOUNWIND_READONLY:#[0-9]+]]
 declare i32 @ferror(ptr)
 
-; CHECK: declare noundef i32 @fflush(ptr nocapture noundef) [[NOFREE_NOUNWIND]]
+; CHECK: declare noundef i32 @fflush(ptr noundef captures(none)) [[NOFREE_NOUNWIND]]
 declare i32 @fflush(ptr)
 
 ; CHECK: declare i32 @ffs(i32) [[NOFREE_NOUNWIND_WILLRETURN_WRITEONLY]]
@@ -422,19 +422,19 @@ declare i32 @ffsl(i64)
 ; CHECK-UNKNOWN: declare i32 @ffsll(i64){{$}}
 declare i32 @ffsll(i64)
 
-; CHECK: declare noundef i32 @fgetc(ptr nocapture noundef) [[NOFREE_NOUNWIND]]
+; CHECK: declare noundef i32 @fgetc(ptr noundef captures(none)) [[NOFREE_NOUNWIND]]
 declare i32 @fgetc(ptr)
 
-; CHECK: declare noundef i32 @fgetpos(ptr nocapture noundef, ptr nocapture noundef) [[NOFREE_NOUNWIND]]
+; CHECK: declare noundef i32 @fgetpos(ptr noundef captures(none), ptr noundef captures(none)) [[NOFREE_NOUNWIND]]
 declare i32 @fgetpos(ptr, ptr)
 
-; CHECK: declare noundef ptr @fgets(ptr noundef, i32 noundef, ptr nocapture noundef) [[NOFREE_NOUNWIND]]
+; CHECK: declare noundef ptr @fgets(ptr noundef, i32 noundef, ptr noundef captures(none)) [[NOFREE_NOUNWIND]]
 declare ptr @fgets(ptr, i32, ptr)
 
-; CHECK: declare noundef i32 @fileno(ptr nocapture noundef) [[NOFREE_NOUNWIND]]
+; CHECK: declare noundef i32 @fileno(ptr noundef captures(none)) [[NOFREE_NOUNWIND]]
 declare i32 @fileno(ptr)
 
-; CHECK: declare void @flockfile(ptr nocapture noundef) [[NOFREE_NOUNWIND]]
+; CHECK: declare void @flockfile(ptr noundef captures(none)) [[NOFREE_NOUNWIND]]
 declare void @flockfile(ptr)
 
 ; CHECK: declare double @floor(double) [[NOFREE_NOUNWIND_WILLRETURN_WRITEONLY]]
@@ -482,85 +482,85 @@ declare float @fmodf(float, float)
 ; CHECK: declare x86_fp80 @fmodl(x86_fp80, x86_fp80) [[NOFREE_NOUNWIND_WILLRETURN_WRITEONLY]]
 declare x86_fp80 @fmodl(x86_fp80, x86_fp80)
 
-; CHECK: declare noalias noundef ptr @fopen(ptr nocapture noundef readonly, ptr nocapture noundef readonly) [[NOFREE_NOUNWIND]]
+; CHECK: declare noalias noundef ptr @fopen(ptr noundef readonly captures(none), ptr noundef readonly captures(none)) [[NOFREE_NOUNWIND]]
 declare ptr @fopen(ptr, ptr)
 
-; CHECK: declare noundef i32 @fprintf(ptr nocapture noundef, ptr nocapture noundef readonly, ...) [[NOFREE_NOUNWIND]]
+; CHECK: declare noundef i32 @fprintf(ptr noundef captures(none), ptr noundef readonly captures(none), ...) [[NOFREE_NOUNWIND]]
 declare i32 @fprintf(ptr, ptr, ...)
 
-; CHECK: declare noundef i32 @fputc(i32 noundef, ptr nocapture noundef) [[NOFREE_NOUNWIND]]
+; CHECK: declare noundef i32 @fputc(i32 noundef, ptr noundef captures(none)) [[NOFREE_NOUNWIND]]
 declare i32 @fputc(i32, ptr)
 
-; CHECK: declare noundef i32 @fputs(ptr nocapture noundef readonly, ptr nocapture noundef) [[NOFREE_NOUNWIND]]
+; CHECK: declare noundef i32 @fputs(ptr noundef readonly captures(none), ptr noundef captures(none)) [[NOFREE_NOUNWIND]]
 declare i32 @fputs(ptr, ptr)
 
-; CHECK: declare noundef i64 @fread(ptr nocapture noundef, i64 noundef, i64 noundef, ptr nocapture noundef) [[NOFREE_NOUNWIND]]
+; CHECK: declare noundef i64 @fread(ptr noundef captures(none), i64 noundef, i64 noundef, ptr noundef captures(none)) [[NOFREE_NOUNWIND]]
 declare i64 @fread(ptr, i64, i64, ptr)
 
-; CHECK: declare void @free(ptr allocptr nocapture noundef) [[INACCESSIBLEMEMORARGMEMONLY_NOUNWIND_WILLRETURN_ALLOCKIND_FREE_FAMILY_MALLOC:#[0-9]+]]
+; CHECK: declare void @free(ptr allocptr noundef captures(none)) [[INACCESSIBLEMEMORARGMEMONLY_NOUNWIND_WILLRETURN_ALLOCKIND_FREE_FAMILY_MALLOC:#[0-9]+]]
 declare void @free(ptr)
 
-; CHECK-AIX: declare void @vec_free(ptr allocptr nocapture noundef) [[INACCESSIBLEMEMORARGMEMONLY_NOUNWIND_WILLRETURN_FAMILY_VEC_MALLOC:#[0-9]+]]
+; CHECK-AIX: declare void @vec_free(ptr allocptr noundef captures(none)) [[INACCESSIBLEMEMORARGMEMONLY_NOUNWIND_WILLRETURN_FAMILY_VEC_MALLOC:#[0-9]+]]
 declare void @vec_free(ptr)
 
-; CHECK: declare double @frexp(double, ptr nocapture) [[ARGMEMONLY_NOFREE_NOUNWIND_WILLRETURN_WRITEONLY:#[0-9]+]]
+; CHECK: declare double @frexp(double, ptr captures(none)) [[ARGMEMONLY_NOFREE_NOUNWIND_WILLRETURN_WRITEONLY:#[0-9]+]]
 declare double @frexp(double, ptr)
 
-; CHECK: declare float @frexpf(float, ptr nocapture) [[ARGMEMONLY_NOFREE_NOUNWIND_WILLRETURN_WRITEONLY]]
+; CHECK: declare float @frexpf(float, ptr captures(none)) [[ARGMEMONLY_NOFREE_NOUNWIND_WILLRETURN_WRITEONLY]]
 declare float @frexpf(float, ptr)
 
-; CHECK: declare x86_fp80 @frexpl(x86_fp80, ptr nocapture) [[ARGMEMONLY_NOFREE_NOUNWIND_WILLRETURN_WRITEONLY]]
+; CHECK: declare x86_fp80 @frexpl(x86_fp80, ptr captures(none)) [[ARGMEMONLY_NOFREE_NOUNWIND_WILLRETURN_WRITEONLY]]
 declare x86_fp80 @frexpl(x86_fp80, ptr)
 
-; CHECK: declare noundef i32 @fscanf(ptr nocapture noundef, ptr nocapture noundef readonly, ...) [[NOFREE_NOUNWIND]]
+; CHECK: declare noundef i32 @fscanf(ptr noundef captures(none), ptr noundef readonly captures(none), ...) [[NOFREE_NOUNWIND]]
 declare i32 @fscanf(ptr, ptr, ...)
 
-; CHECK: declare noundef i32 @fseek(ptr nocapture noundef, i64 noundef, i32 noundef) [[NOFREE_NOUNWIND]]
+; CHECK: declare noundef i32 @fseek(ptr noundef captures(none), i64 noundef, i32 noundef) [[NOFREE_NOUNWIND]]
 declare i32 @fseek(ptr, i64, i32)
 
-; CHECK: declare noundef i32 @fseeko(ptr nocapture noundef, i64 noundef, i32 noundef) [[NOFREE_NOUNWIND]]
+; CHECK: declare noundef i32 @fseeko(ptr noundef captures(none), i64 noundef, i32 noundef) [[NOFREE_NOUNWIND]]
 declare i32 @fseeko(ptr, i64, i32)
 
-; CHECK-LINUX: declare noundef i32 @fseeko64(ptr nocapture noundef, i64 noundef, i32 noundef) [[NOFREE_NOUNWIND]]
+; CHECK-LINUX: declare noundef i32 @fseeko64(ptr noundef captures(none), i64 noundef, i32 noundef) [[NOFREE_NOUNWIND]]
 declare i32 @fseeko64(ptr, i64, i32)
 
-; CHECK: declare noundef i32 @fsetpos(ptr nocapture noundef, ptr noundef) [[NOFREE_NOUNWIND]]
+; CHECK: declare noundef i32 @fsetpos(ptr noundef captures(none), ptr noundef) [[NOFREE_NOUNWIND]]
 declare i32 @fsetpos(ptr, ptr)
 
-; CHECK: declare noundef i32 @fstat(i32 noundef, ptr nocapture noundef) [[NOFREE_NOUNWIND]]
+; CHECK: declare noundef i32 @fstat(i32 noundef, ptr noundef captures(none)) [[NOFREE_NOUNWIND]]
 declare i32 @fstat(i32, ptr)
 
-; CHECK-LINUX: declare noundef i32 @fstat64(i32 noundef, ptr nocapture noundef) [[NOFREE_NOUNWIND]]
+; CHECK-LINUX: declare noundef i32 @fstat64(i32 noundef, ptr noundef captures(none)) [[NOFREE_NOUNWIND]]
 declare i32 @fstat64(i32, ptr)
 
-; CHECK: declare noundef i32 @fstatvfs(i32 noundef, ptr nocapture noundef) [[NOFREE_NOUNWIND]]
+; CHECK: declare noundef i32 @fstatvfs(i32 noundef, ptr noundef captures(none)) [[NOFREE_NOUNWIND]]
 declare i32 @fstatvfs(i32, ptr)
 
-; CHECK-LINUX: declare noundef i32 @fstatvfs64(i32 noundef, ptr nocapture noundef) [[NOFREE_NOUNWIND]]
+; CHECK-LINUX: declare noundef i32 @fstatvfs64(i32 noundef, ptr noundef captures(none)) [[NOFREE_NOUNWIND]]
 declare i32 @fstatvfs64(i32, ptr)
 
-; CHECK: declare noundef i64 @ftell(ptr nocapture noundef) [[NOFREE_NOUNWIND]]
+; CHECK: declare noundef i64 @ftell(ptr noundef captures(none)) [[NOFREE_NOUNWIND]]
 declare i64 @ftell(ptr)
 
-; CHECK: declare noundef i64 @ftello(ptr nocapture noundef) [[NOFREE_NOUNWIND]]
+; CHECK: declare noundef i64 @ftello(ptr noundef captures(none)) [[NOFREE_NOUNWIND]]
 declare i64 @ftello(ptr)
 
-; CHECK-LINUX: declare noundef i64 @ftello64(ptr nocapture noundef) [[NOFREE_NOUNWIND]]
+; CHECK-LINUX: declare noundef i64 @ftello64(ptr noundef captures(none)) [[NOFREE_NOUNWIND]]
 declare i64 @ftello64(ptr)
 
-; CHECK: declare noundef i32 @ftrylockfile(ptr nocapture noundef) [[NOFREE_NOUNWIND]]
+; CHECK: declare noundef i32 @ftrylockfile(ptr noundef captures(none)) [[NOFREE_NOUNWIND]]
 declare i32 @ftrylockfile(ptr)
 
-; CHECK: declare void @funlockfile(ptr nocapture noundef) [[NOFREE_NOUNWIND]]
+; CHECK: declare void @funlockfile(ptr noundef captures(none)) [[NOFREE_NOUNWIND]]
 declare void @funlockfile(ptr)
 
-; CHECK: declare noundef i64 @fwrite(ptr nocapture noundef, i64 noundef, i64 noundef, ptr nocapture noundef) [[NOFREE_NOUNWIND]]
+; CHECK: declare noundef i64 @fwrite(ptr noundef captures(none), i64 noundef, i64 noundef, ptr noundef captures(none)) [[NOFREE_NOUNWIND]]
 declare i64 @fwrite(ptr, i64, i64, ptr)
 
-; CHECK: declare noundef i32 @getc(ptr nocapture noundef) [[NOFREE_NOUNWIND]]
+; CHECK: declare noundef i32 @getc(ptr noundef captures(none)) [[NOFREE_NOUNWIND]]
 declare i32 @getc(ptr)
 
-; CHECK-KNOWN: declare noundef i32 @getc_unlocked(ptr nocapture noundef) [[NOFREE_NOUNWIND]]
+; CHECK-KNOWN: declare noundef i32 @getc_unlocked(ptr noundef captures(none)) [[NOFREE_NOUNWIND]]
 ; CHECK-UNKNOWN: declare i32 @getc_unlocked(ptr){{$}}
 declare i32 @getc_unlocked(ptr)
 
@@ -571,22 +571,22 @@ declare i32 @getchar()
 ; CHECK-UNKNOWN: declare i32 @getchar_unlocked(){{$}}
 declare i32 @getchar_unlocked()
 
-; CHECK: declare noundef ptr @getenv(ptr nocapture noundef) [[NOFREE_NOUNWIND_READONLY]]
+; CHECK: declare noundef ptr @getenv(ptr noundef captures(none)) [[NOFREE_NOUNWIND_READONLY]]
 declare ptr @getenv(ptr)
 
-; CHECK: declare noundef i32 @getitimer(i32 noundef, ptr nocapture noundef) [[NOFREE_NOUNWIND]]
+; CHECK: declare noundef i32 @getitimer(i32 noundef, ptr noundef captures(none)) [[NOFREE_NOUNWIND]]
 declare i32 @getitimer(i32, ptr)
 
-; CHECK: declare noundef i32 @getlogin_r(ptr nocapture noundef, i64 noundef) [[NOFREE_NOUNWIND]]
+; CHECK: declare noundef i32 @getlogin_r(ptr noundef captures(none), i64 noundef) [[NOFREE_NOUNWIND]]
 declare i32 @getlogin_r(ptr, i64)
 
-; CHECK: declare noundef ptr @getpwnam(ptr nocapture noundef readonly) [[NOFREE_NOUNWIND]]
+; CHECK: declare noundef ptr @getpwnam(ptr noundef readonly captures(none)) [[NOFREE_NOUNWIND]]
 declare ptr @getpwnam(ptr)
 
 ; CHECK: declare noundef ptr @gets(ptr noundef) [[NOFREE_NOUNWIND]]
 declare ptr @gets(ptr)
 
-; CHECK: declare noundef i32 @gettimeofday(ptr nocapture noundef, ptr nocapture noundef) [[NOFREE_NOUNWIND]]
+; CHECK: declare noundef i32 @gettimeofday(ptr noundef captures(none), ptr noundef captures(none)) [[NOFREE_NOUNWIND]]
 declare i32 @gettimeofday(ptr, ptr)
 
 ; CHECK: declare double @hypot(double, double) [[NOFREE_NOUNWIND_WILLRETURN_WRITEONLY]]
@@ -607,7 +607,7 @@ declare i32 @isdigit(i32)
 ; CHECK: declare i64 @labs(i64) [[NOFREE_NOUNWIND_WILLRETURN_WRITEONLY]]
 declare i64 @labs(i64)
 
-; CHECK: declare noundef i32 @lchown(ptr nocapture noundef readonly, i32 noundef, i32 noundef) [[NOFREE_NOUNWIND]]
+; CHECK: declare noundef i32 @lchown(ptr noundef readonly captures(none), i32 noundef, i32 noundef) [[NOFREE_NOUNWIND]]
 declare i32 @lchown(ptr, i32, i32)
 
 ; CHECK: declare double @ldexp(double, i32) [[NOFREE_WILLRETURN:#[0-9]+]]
@@ -685,10 +685,10 @@ declare float @tgammaf(float)
 ; CHECK: declare x86_fp80 @tgammal(x86_fp80) [[NOFREE_NOUNWIND_WILLRETURN_WRITEONLY]]
 declare x86_fp80 @tgammal(x86_fp80)
 
-; CHECK: declare noundef i32 @lstat(ptr nocapture noundef readonly, ptr nocapture noundef) [[NOFREE_NOUNWIND]]
+; CHECK: declare noundef i32 @lstat(ptr noundef readonly captures(none), ptr noundef captures(none)) [[NOFREE_NOUNWIND]]
 declare i32 @lstat(ptr, ptr)
 
-; CHECK-LINUX: declare noundef i32 @lstat64(ptr nocapture noundef readonly, ptr nocapture noundef) [[NOFREE_NOUNWIND]]
+; CHECK-LINUX: declare noundef i32 @lstat64(ptr noundef readonly captures(none), ptr noundef captures(none)) [[NOFREE_NOUNWIND]]
 declare i32 @lstat64(ptr, ptr)
 
 ; CHECK: declare noalias noundef ptr @malloc(i64 noundef) [[INACCESSIBLEMEMONLY_NOFREE_NOUNWIND_WILLRETURN_ALLOCKIND_ALLOCUNINIT_ALLOCSIZE0_FAMILY_MALLOC:#[0-9]+]]
@@ -700,7 +700,7 @@ declare ptr @vec_malloc(i64)
 ; CHECK-LINUX: declare noalias noundef ptr @memalign(i64 allocalign, i64) [[INACCESSIBLEMEMONLY_NOFREE_NOUNWIND_WILLRETURN:#[0-9]+]]
 declare ptr @memalign(i64, i64)
 
-; CHECK: declare ptr @memccpy(ptr noalias writeonly, ptr noalias nocapture readonly, i32, i64) [[ARGMEMONLY_NOFREE_NOUNWIND_WILLRETURN]]
+; CHECK: declare ptr @memccpy(ptr noalias writeonly, ptr noalias readonly captures(none), i32, i64) [[ARGMEMONLY_NOFREE_NOUNWIND_WILLRETURN]]
 declare ptr @memccpy(ptr, ptr, i32, i64)
 
 ; CHECK-LINUX:   declare ptr @memchr(ptr, i32, i64) [[ARGMEMONLY_NOFREE_NOUNWIND_READONLY]]
@@ -708,19 +708,19 @@ declare ptr @memccpy(ptr, ptr, i32, i64)
 ; CHECK-UNKNOWN: declare ptr @memchr(ptr, i32, i64) [[ARGMEMONLY_NOFREE_NOUNWIND_READONLY:#[0-9]+]]
 declare ptr @memchr(ptr, i32, i64)
 
-; CHECK: declare i32 @memcmp(ptr nocapture, ptr nocapture, i64) [[ARGMEMONLY_NOFREE_NOUNWIND_READONLY]]
+; CHECK: declare i32 @memcmp(ptr captures(none), ptr captures(none), i64) [[ARGMEMONLY_NOFREE_NOUNWIND_READONLY]]
 declare i32 @memcmp(ptr, ptr, i64)
 
-; CHECK: declare ptr @memcpy(ptr noalias returned writeonly, ptr noalias nocapture readonly, i64) [[ARGMEMONLY_NOFREE_NOUNWIND_WILLRETURN]]
+; CHECK: declare ptr @memcpy(ptr noalias returned writeonly, ptr noalias readonly captures(none), i64) [[ARGMEMONLY_NOFREE_NOUNWIND_WILLRETURN]]
 declare ptr @memcpy(ptr, ptr, i64)
 
-; CHECK: declare ptr @__memcpy_chk(ptr noalias writeonly, ptr noalias nocapture readonly, i64, i64) [[ARGMEMONLY_NOFREE_NOUNWIND:#[0-9]+]]
+; CHECK: declare ptr @__memcpy_chk(ptr noalias writeonly, ptr noalias readonly captures(none), i64, i64) [[ARGMEMONLY_NOFREE_NOUNWIND:#[0-9]+]]
 declare ptr @__memcpy_chk(ptr, ptr, i64, i64)
 
-; CHECK: declare ptr @mempcpy(ptr noalias writeonly, ptr noalias nocapture readonly, i64) [[ARGMEMONLY_NOFREE_NOUNWIND_WILLRETURN]]
+; CHECK: declare ptr @mempcpy(ptr noalias writeonly, ptr noalias readonly captures(none), i64) [[ARGMEMONLY_NOFREE_NOUNWIND_WILLRETURN]]
 declare ptr @mempcpy(ptr, ptr, i64)
 
-; CHECK: declare ptr @memmove(ptr returned writeonly, ptr nocapture readonly, i64) [[ARGMEMONLY_NOFREE_NOUNWIND_WILLRETURN]]
+; CHECK: declare ptr @memmove(ptr returned writeonly, ptr readonly captures(none), i64) [[ARGMEMONLY_NOFREE_NOUNWIND_WILLRETURN]]
 declare ptr @memmove(ptr, ptr, i64)
 
 ; CHECK: declare ptr @memset(ptr writeonly, i32, i64) [[ARGMEMONLY_NOFREE_NOUNWIND_WILLRETURN:#[0-9]+]]
@@ -729,28 +729,28 @@ declare ptr @memset(ptr, i32, i64)
 ; CHECK: declare ptr @__memset_chk(ptr writeonly, i32, i64, i64) [[ARGMEMONLY_NOFREE_NOUNWIND]]
 declare ptr @__memset_chk(ptr, i32, i64, i64)
 
-; CHECK: declare noundef i32 @mkdir(ptr nocapture noundef readonly, i16 noundef zeroext) [[NOFREE_NOUNWIND]]
+; CHECK: declare noundef i32 @mkdir(ptr noundef readonly captures(none), i16 noundef zeroext) [[NOFREE_NOUNWIND]]
 declare i32 @mkdir(ptr, i16 zeroext)
 
-; CHECK: declare noundef i64 @mktime(ptr nocapture noundef) [[NOFREE_NOUNWIND_WILLRETURN:#[0-9]+]]
+; CHECK: declare noundef i64 @mktime(ptr noundef captures(none)) [[NOFREE_NOUNWIND_WILLRETURN:#[0-9]+]]
 declare i64 @mktime(ptr)
 
-; CHECK: declare double @modf(double, ptr nocapture) [[ARGMEMONLY_NOFREE_NOUNWIND_WILLRETURN_WRITEONLY]]
+; CHECK: declare double @modf(double, ptr captures(none)) [[ARGMEMONLY_NOFREE_NOUNWIND_WILLRETURN_WRITEONLY]]
 declare double @modf(double, ptr)
 
-; CHECK: declare float @modff(float, ptr nocapture) [[ARGMEMONLY_NOFREE_NOUNWIND_WILLRETURN_WRITEONLY]]
+; CHECK: declare float @modff(float, ptr captures(none)) [[ARGMEMONLY_NOFREE_NOUNWIND_WILLRETURN_WRITEONLY]]
 declare float @modff(float, ptr)
 
-; CHECK: declare x86_fp80 @modfl(x86_fp80, ptr nocapture) [[ARGMEMONLY_NOFREE_NOUNWIND_WILLRETURN_WRITEONLY]]
+; CHECK: declare x86_fp80 @modfl(x86_fp80, ptr captures(none)) [[ARGMEMONLY_NOFREE_NOUNWIND_WILLRETURN_WRITEONLY]]
 declare x86_fp80 @modfl(x86_fp80, ptr)
 
-; CHECK: declare double @nan(ptr nocapture) [[ARGMEMONLY_NOFREE_NOUNWIND_READONLY_WILLRETURN:#[0-9]+]]
+; CHECK: declare double @nan(ptr captures(none)) [[ARGMEMONLY_NOFREE_NOUNWIND_READONLY_WILLRETURN:#[0-9]+]]
 declare double @nan(ptr)
 
-; CHECK: declare float @nanf(ptr nocapture) [[ARGMEMONLY_NOFREE_NOUNWIND_READONLY_WILLRETURN]]
+; CHECK: declare float @nanf(ptr captures(none)) [[ARGMEMONLY_NOFREE_NOUNWIND_READONLY_WILLRETURN]]
 declare float @nanf(ptr)
 
-; CHECK: declare x86_fp80 @nanl(ptr nocapture) [[ARGMEMONLY_NOFREE_NOUNWIND_READONLY_WILLRETURN]]
+; CHECK: declare x86_fp80 @nanl(ptr captures(none)) [[ARGMEMONLY_NOFREE_NOUNWIND_READONLY_WILLRETURN]]
 declare x86_fp80 @nanl(ptr)
 
 ; CHECK: declare double @nearbyint(double) [[NOFREE_NOUNWIND_WILLRETURN_WRITEONLY]]
@@ -762,23 +762,23 @@ declare float @nearbyintf(float)
 ; CHECK: declare x86_fp80 @nearbyintl(x86_fp80) [[NOFREE_NOUNWIND_WILLRETURN_WRITEONLY]]
 declare x86_fp80 @nearbyintl(x86_fp80)
 
-; CHECK-LINUX: declare noundef i32 @open(ptr nocapture noundef readonly, i32 noundef, ...) [[NOFREE]]
-; CHECK-OPEN: declare noundef i32 @open(ptr nocapture noundef readonly, i32 noundef, ...) [[NOFREE:#[0-9]+]]
+; CHECK-LINUX: declare noundef i32 @open(ptr noundef readonly captures(none), i32 noundef, ...) [[NOFREE]]
+; CHECK-OPEN: declare noundef i32 @open(ptr noundef readonly captures(none), i32 noundef, ...) [[NOFREE:#[0-9]+]]
 declare i32 @open(ptr, i32, ...)
 
-; CHECK-LINUX: declare noundef i32 @open64(ptr nocapture noundef readonly, i32 noundef, ...) [[NOFREE]]
+; CHECK-LINUX: declare noundef i32 @open64(ptr noundef readonly captures(none), i32 noundef, ...) [[NOFREE]]
 declare i32 @open64(ptr, i32, ...)
 
-; CHECK: declare noalias noundef ptr @opendir(ptr nocapture noundef readonly) [[NOFREE_NOUNWIND]]
+; CHECK: declare noalias noundef ptr @opendir(ptr noundef readonly captures(none)) [[NOFREE_NOUNWIND]]
 declare ptr @opendir(ptr)
 
-; CHECK: declare noundef i32 @pclose(ptr nocapture noundef) [[NOFREE_NOUNWIND]]
+; CHECK: declare noundef i32 @pclose(ptr noundef captures(none)) [[NOFREE_NOUNWIND]]
 declare i32 @pclose(ptr)
 
-; CHECK: declare void @perror(ptr nocapture noundef readonly) [[NOFREE_NOUNWIND]]
+; CHECK: declare void @perror(ptr noundef readonly captures(none)) [[NOFREE_NOUNWIND]]
 declare void @perror(ptr)
 
-; CHECK: declare noalias noundef ptr @popen(ptr nocapture noundef readonly, ptr nocapture noundef readonly) [[NOFREE_NOUNWIND]]
+; CHECK: declare noalias noundef ptr @popen(ptr noundef readonly captures(none), ptr noundef readonly captures(none)) [[NOFREE_NOUNWIND]]
 declare ptr @popen(ptr, ptr)
 
 ; CHECK: declare i32 @posix_memalign(ptr, i64, i64) [[NOFREE]]
@@ -793,13 +793,13 @@ declare float @powf(float, float)
 ; CHECK: declare x86_fp80 @powl(x86_fp80, x86_fp80) [[NOFREE_NOUNWIND_WILLRETURN_WRITEONLY]]
 declare x86_fp80 @powl(x86_fp80, x86_fp80)
 
-; CHECK: declare noundef i64 @pread(i32 noundef, ptr nocapture noundef, i64 noundef, i64 noundef) [[NOFREE]]
+; CHECK: declare noundef i64 @pread(i32 noundef, ptr noundef captures(none), i64 noundef, i64 noundef) [[NOFREE]]
 declare i64 @pread(i32, ptr, i64, i64)
 
-; CHECK: declare noundef i32 @printf(ptr nocapture noundef readonly, ...) [[NOFREE_NOUNWIND]]
+; CHECK: declare noundef i32 @printf(ptr noundef readonly captures(none), ...) [[NOFREE_NOUNWIND]]
 declare i32 @printf(ptr, ...)
 
-; CHECK: declare noundef i32 @putc(i32 noundef, ptr nocapture noundef) [[NOFREE_NOUNWIND]]
+; CHECK: declare noundef i32 @putc(i32 noundef, ptr noundef captures(none)) [[NOFREE_NOUNWIND]]
 declare i32 @putc(i32, ptr)
 
 ; CHECK: declare noundef i32 @putchar(i32 noundef) [[NOFREE_NOUNWIND]]
@@ -809,34 +809,34 @@ declare i32 @putchar(i32)
 ; CHECK-UNKNOWN: declare i32 @putchar_unlocked(i32){{$}}
 declare i32 @putchar_unlocked(i32)
 
-; CHECK: declare noundef i32 @puts(ptr nocapture noundef readonly) [[NOFREE_NOUNWIND]]
+; CHECK: declare noundef i32 @puts(ptr noundef readonly captures(none)) [[NOFREE_NOUNWIND]]
 declare i32 @puts(ptr)
 
-; CHECK: declare noundef i64 @pwrite(i32 noundef, ptr nocapture noundef readonly, i64 noundef, i64 noundef) [[NOFREE]]
+; CHECK: declare noundef i64 @pwrite(i32 noundef, ptr noundef readonly captures(none), i64 noundef, i64 noundef) [[NOFREE]]
 declare i64 @pwrite(i32, ptr, i64, i64)
 
-; CHECK: declare void @qsort(ptr noundef, i64 noundef, i64 noundef, ptr nocapture noundef) [[NOFREE]]
+; CHECK: declare void @qsort(ptr noundef, i64 noundef, i64 noundef, ptr noundef captures(none)) [[NOFREE]]
 declare void @qsort(ptr, i64, i64, ptr)
 
-; CHECK: declare noundef i64 @read(i32 noundef, ptr nocapture noundef, i64 noundef) [[NOFREE]]
+; CHECK: declare noundef i64 @read(i32 noundef, ptr noundef captures(none), i64 noundef) [[NOFREE]]
 declare i64 @read(i32, ptr, i64)
 
-; CHECK: declare noundef i64 @readlink(ptr nocapture noundef readonly, ptr nocapture noundef, i64 noundef) [[NOFREE_NOUNWIND]]
+; CHECK: declare noundef i64 @readlink(ptr noundef readonly captures(none), ptr noundef captures(none), i64 noundef) [[NOFREE_NOUNWIND]]
 declare i64 @readlink(ptr, ptr, i64)
 
-; CHECK: declare noalias noundef ptr @realloc(ptr allocptr nocapture, i64 noundef) [[INACCESSIBLEMEMORARGMEMONLY_NOUNWIND_WILLRETURN_ALLOCKIND_REALLOC_ALLOCSIZE1_FAMILY_MALLOC:#[0-9]+]]
+; CHECK: declare noalias noundef ptr @realloc(ptr allocptr captures(none), i64 noundef) [[INACCESSIBLEMEMORARGMEMONLY_NOUNWIND_WILLRETURN_ALLOCKIND_REALLOC_ALLOCSIZE1_FAMILY_MALLOC:#[0-9]+]]
 declare ptr @realloc(ptr, i64)
 
-; CHECK: declare noalias noundef ptr @reallocarray(ptr allocptr nocapture, i64 noundef, i64 noundef) [[INACCESSIBLEMEMORARGMEMONLY_NOUNWIND_WILLRETURN_ALLOCKIND_REALLOC_ALLOCSIZE12_FAMILY_MALLOC:#[0-9]+]]
+; CHECK: declare noalias noundef ptr @reallocarray(ptr allocptr captures(none), i64 noundef, i64 noundef) [[INACCESSIBLEMEMORARGMEMONLY_NOUNWIND_WILLRETURN_ALLOCKIND_REALLOC_ALLOCSIZE12_FAMILY_MALLOC:#[0-9]+]]
 declare ptr @reallocarray(ptr, i64, i64)
 
-; CHECK: declare noalias noundef ptr @reallocf(ptr allocptr nocapture, i64 noundef) [[INACCESSIBLEMEMORARGMEMONLY_NOUNWIND_WILLRETURN_ALLOCKIND_REALLOC_ALLOCSIZE1_FAMILY_MALLOC]]
+; CHECK: declare noalias noundef ptr @reallocf(ptr allocptr captures(none), i64 noundef) [[INACCESSIBLEMEMORARGMEMONLY_NOUNWIND_WILLRETURN_ALLOCKIND_REALLOC_ALLOCSIZE1_FAMILY_MALLOC]]
 declare ptr @reallocf(ptr, i64)
 
-; CHECK-AIX: declare noalias noundef ptr @vec_realloc(ptr allocptr nocapture, i64 noundef) [[INACCESSIBLEMEMORARGMEMONLY_NOUNWIND_WILLRETURN_ALLOCSIZE_FAMILY_VEC_MALLOC:#[0-9]+]]
+; CHECK-AIX: declare noalias noundef ptr @vec_realloc(ptr allocptr captures(none), i64 noundef) [[INACCESSIBLEMEMORARGMEMONLY_NOUNWIND_WILLRETURN_ALLOCSIZE_FAMILY_VEC_MALLOC:#[0-9]+]]
 declare ptr @vec_realloc(ptr, i64)
 
-; CHECK: declare noundef ptr @realpath(ptr nocapture noundef readonly, ptr noundef) [[NOFREE_NOUNWIND]]
+; CHECK: declare noundef ptr @realpath(ptr noundef readonly captures(none), ptr noundef) [[NOFREE_NOUNWIND]]
 declare ptr @realpath(ptr, ptr)
 
 ; CHECK: declare double @remainder(double, double) [[NOFREE_NOUNWIND_WILLRETURN_WRITEONLY]]
@@ -848,16 +848,16 @@ declare float @remainderf(float, float)
 ; CHECK: declare x86_fp80 @remainderl(x86_fp80, x86_fp80) [[NOFREE_NOUNWIND_WILLRETURN_WRITEONLY]]
 declare x86_fp80 @remainderl(x86_fp80, x86_fp80)
 
-; CHECK: declare noundef i32 @remove(ptr nocapture noundef readonly) [[NOFREE_NOUNWIND]]
+; CHECK: declare noundef i32 @remove(ptr noundef readonly captures(none)) [[NOFREE_NOUNWIND]]
 declare i32 @remove(ptr)
 
-; CHECK: declare double @remquo(double, double, ptr nocapture) [[NOFREE_NOUNWIND_WILLRETURN_WRITEONLY]]
+; CHECK: declare double @remquo(double, double, ptr captures(none)) [[NOFREE_NOUNWIND_WILLRETURN_WRITEONLY]]
 declare double @remquo(double, double, ptr)
 
-; CHECK: declare float @remquof(float, float, ptr nocapture) [[NOFREE_NOUNWIND_WILLRETURN_WRITEONLY]]
+; CHECK: declare float @remquof(float, float, ptr captures(none)) [[NOFREE_NOUNWIND_WILLRETURN_WRITEONLY]]
 declare float @remquof(float, float, ptr)
 
-; CHECK: declare x86_fp80 @remquol(x86_fp80, x86_fp80, ptr nocapture) [[NOFREE_NOUNWIND_WILLRETURN_WRITEONLY]]
+; CHECK: declare x86_fp80 @remquol(x86_fp80, x86_fp80, ptr captures(none)) [[NOFREE_NOUNWIND_WILLRETURN_WRITEONLY]]
 declare x86_fp80 @remquol(x86_fp80, x86_fp80, ptr)
 
 
@@ -870,10 +870,10 @@ declare float @fdimf(float, float)
 ; CHECK: declare x86_fp80 @fdiml(x86_fp80, x86_fp80) [[NOFREE_NOUNWIND_WILLRETURN_WRITEONLY]]
 declare x86_fp80 @fdiml(x86_fp80, x86_fp80)
 
-; CHECK: declare noundef i32 @rename(ptr nocapture noundef readonly, ptr nocapture noundef readonly) [[NOFREE_NOUNWIND]]
+; CHECK: declare noundef i32 @rename(ptr noundef readonly captures(none), ptr noundef readonly captures(none)) [[NOFREE_NOUNWIND]]
 declare i32 @rename(ptr, ptr)
 
-; CHECK: declare void @rewind(ptr nocapture noundef) [[NOFREE_NOUNWIND]]
+; CHECK: declare void @rewind(ptr noundef captures(none)) [[NOFREE_NOUNWIND]]
 declare void @rewind(ptr)
 
 ; CHECK: declare double @rint(double) [[NOFREE_NOUNWIND_WILLRETURN_WRITEONLY]]
@@ -885,7 +885,7 @@ declare float @rintf(float)
 ; CHECK: declare x86_fp80 @rintl(x86_fp80) [[NOFREE_NOUNWIND_WILLRETURN_WRITEONLY]]
 declare x86_fp80 @rintl(x86_fp80)
 
-; CHECK: declare noundef i32 @rmdir(ptr nocapture noundef readonly) [[NOFREE_NOUNWIND]]
+; CHECK: declare noundef i32 @rmdir(ptr noundef readonly captures(none)) [[NOFREE_NOUNWIND]]
 declare i32 @rmdir(ptr)
 
 ; CHECK: declare double @round(double) [[NOFREE_NOUNWIND_WILLRETURN_WRITEONLY]]
@@ -915,16 +915,16 @@ declare float @scalbnf(float, i32)
 ; CHECK: declare x86_fp80 @scalbnl(x86_fp80, i32) [[NOFREE_NOUNWIND_WILLRETURN_WRITEONLY]]
 declare x86_fp80 @scalbnl(x86_fp80, i32)
 
-; CHECK: declare noundef i32 @scanf(ptr nocapture noundef readonly, ...) [[NOFREE_NOUNWIND]]
+; CHECK: declare noundef i32 @scanf(ptr noundef readonly captures(none), ...) [[NOFREE_NOUNWIND]]
 declare i32 @scanf(ptr, ...)
 
-; CHECK: declare void @setbuf(ptr nocapture noundef, ptr noundef) [[NOFREE_NOUNWIND]]
+; CHECK: declare void @setbuf(ptr noundef captures(none), ptr noundef) [[NOFREE_NOUNWIND]]
 declare void @setbuf(ptr, ptr)
 
-; CHECK: declare noundef i32 @setitimer(i32 noundef, ptr nocapture noundef readonly, ptr nocapture noundef) [[NOFREE_NOUNWIND_WILLRETURN]]
+; CHECK: declare noundef i32 @setitimer(i32 noundef, ptr noundef readonly captures(none), ptr noundef captures(none)) [[NOFREE_NOUNWIND_WILLRETURN]]
 declare i32 @setitimer(i32, ptr, ptr)
 
-; CHECK: declare noundef i32 @setvbuf(ptr nocapture noundef, ptr noundef, i32 noundef, i64 noundef) [[NOFREE_NOUNWIND]]
+; CHECK: declare noundef i32 @setvbuf(ptr noundef captures(none), ptr noundef, i32 noundef, i64 noundef) [[NOFREE_NOUNWIND]]
 declare i32 @setvbuf(ptr, ptr, i32, i64)
 
 ; CHECK: declare double @sin(double) [[NOFREE_NOUNWIND_WILLRETURN_WRITEONLY]]
@@ -945,10 +945,10 @@ declare x86_fp80 @sinhl(x86_fp80)
 ; CHECK: declare x86_fp80 @sinl(x86_fp80) [[NOFREE_NOUNWIND_WILLRETURN_WRITEONLY]]
 declare x86_fp80 @sinl(x86_fp80)
 
-; CHECK: declare noundef i32 @snprintf(ptr noalias nocapture noundef writeonly, i64 noundef, ptr nocapture noundef readonly, ...) [[NOFREE_NOUNWIND]]
+; CHECK: declare noundef i32 @snprintf(ptr noalias noundef writeonly captures(none), i64 noundef, ptr noundef readonly captures(none), ...) [[NOFREE_NOUNWIND]]
 declare i32 @snprintf(ptr, i64, ptr, ...)
 
-; CHECK: declare noundef i32 @sprintf(ptr noalias nocapture noundef writeonly, ptr nocapture noundef readonly, ...) [[NOFREE_NOUNWIND]]
+; CHECK: declare noundef i32 @sprintf(ptr noalias noundef writeonly captures(none), ptr noundef readonly captures(none), ...) [[NOFREE_NOUNWIND]]
 declare i32 @sprintf(ptr, ptr, ...)
 
 ; CHECK: declare double @sqrt(double) [[NOFREE_NOUNWIND_WILLRETURN_WRITEONLY]]
@@ -960,115 +960,115 @@ declare float @sqrtf(float)
 ; CHECK: declare x86_fp80 @sqrtl(x86_fp80) [[NOFREE_NOUNWIND_WILLRETURN_WRITEONLY]]
 declare x86_fp80 @sqrtl(x86_fp80)
 
-; CHECK: declare noundef i32 @sscanf(ptr nocapture noundef readonly, ptr nocapture noundef readonly, ...) [[NOFREE_NOUNWIND]]
+; CHECK: declare noundef i32 @sscanf(ptr noundef readonly captures(none), ptr noundef readonly captures(none), ...) [[NOFREE_NOUNWIND]]
 declare i32 @sscanf(ptr, ptr, ...)
 
-; CHECK: declare noundef i32 @stat(ptr nocapture noundef readonly, ptr nocapture noundef) [[NOFREE_NOUNWIND]]
+; CHECK: declare noundef i32 @stat(ptr noundef readonly captures(none), ptr noundef captures(none)) [[NOFREE_NOUNWIND]]
 declare i32 @stat(ptr, ptr)
 
-; CHECK-LINUX: declare noundef i32 @stat64(ptr nocapture noundef readonly, ptr nocapture noundef) [[NOFREE_NOUNWIND]]
+; CHECK-LINUX: declare noundef i32 @stat64(ptr noundef readonly captures(none), ptr noundef captures(none)) [[NOFREE_NOUNWIND]]
 declare i32 @stat64(ptr, ptr)
 
-; CHECK: declare noundef i32 @statvfs(ptr nocapture noundef readonly, ptr nocapture noundef) [[NOFREE_NOUNWIND]]
+; CHECK: declare noundef i32 @statvfs(ptr noundef readonly captures(none), ptr noundef captures(none)) [[NOFREE_NOUNWIND]]
 declare i32 @statvfs(ptr, ptr)
 
-; CHECK-LINUX: declare noundef i32 @statvfs64(ptr nocapture noundef readonly, ptr nocapture noundef) [[NOFREE_NOUNWIND]]
+; CHECK-LINUX: declare noundef i32 @statvfs64(ptr noundef readonly captures(none), ptr noundef captures(none)) [[NOFREE_NOUNWIND]]
 declare i32 @statvfs64(ptr, ptr)
 
-; CHECK: declare ptr @stpcpy(ptr noalias writeonly, ptr noalias nocapture readonly) [[ARGMEMONLY_NOFREE_NOUNWIND_WILLRETURN]]
+; CHECK: declare ptr @stpcpy(ptr noalias writeonly, ptr noalias readonly captures(none)) [[ARGMEMONLY_NOFREE_NOUNWIND_WILLRETURN]]
 declare ptr @stpcpy(ptr, ptr)
 
-; CHECK: declare ptr @stpncpy(ptr noalias writeonly, ptr noalias nocapture readonly, i64) [[ARGMEMONLY_NOFREE_NOUNWIND_WILLRETURN]]
+; CHECK: declare ptr @stpncpy(ptr noalias writeonly, ptr noalias readonly captures(none), i64) [[ARGMEMONLY_NOFREE_NOUNWIND_WILLRETURN]]
 declare ptr @stpncpy(ptr, ptr, i64)
 
-; CHECK: declare i32 @strcasecmp(ptr nocapture, ptr nocapture) [[NOFREE_NOUNWIND_READONLY_WILLRETURN:#[0-9]+]]
+; CHECK: declare i32 @strcasecmp(ptr captures(none), ptr captures(none)) [[NOFREE_NOUNWIND_READONLY_WILLRETURN:#[0-9]+]]
 declare i32 @strcasecmp(ptr, ptr)
 
-; CHECK: declare ptr @strcat(ptr noalias returned, ptr noalias nocapture readonly) [[ARGMEMONLY_NOFREE_NOUNWIND_WILLRETURN]]
+; CHECK: declare ptr @strcat(ptr noalias returned, ptr noalias readonly captures(none)) [[ARGMEMONLY_NOFREE_NOUNWIND_WILLRETURN]]
 declare ptr @strcat(ptr, ptr)
 
 ; CHECK: declare ptr @strchr(ptr, i32) [[ARGMEMONLY_NOFREE_NOUNWIND_READONLY]]
 declare ptr @strchr(ptr, i32)
 
-; CHECK: declare i32 @strcmp(ptr nocapture, ptr nocapture) [[ARGMEMONLY_NOFREE_NOUNWIND_READONLY]]
+; CHECK: declare i32 @strcmp(ptr captures(none), ptr captures(none)) [[ARGMEMONLY_NOFREE_NOUNWIND_READONLY]]
 declare i32 @strcmp(ptr, ptr)
 
-; CHECK: declare i32 @strcoll(ptr nocapture, ptr nocapture) [[NOFREE_NOUNWIND_READONLY_WILLRETURN]]
+; CHECK: declare i32 @strcoll(ptr captures(none), ptr captures(none)) [[NOFREE_NOUNWIND_READONLY_WILLRETURN]]
 declare i32 @strcoll(ptr, ptr)
 
-; CHECK: declare ptr @strcpy(ptr noalias returned writeonly, ptr noalias nocapture readonly) [[ARGMEMONLY_NOFREE_NOUNWIND_WILLRETURN]]
+; CHECK: declare ptr @strcpy(ptr noalias returned writeonly, ptr noalias readonly captures(none)) [[ARGMEMONLY_NOFREE_NOUNWIND_WILLRETURN]]
 declare ptr @strcpy(ptr, ptr)
 
-; CHECK: declare i64 @strcspn(ptr nocapture, ptr nocapture) [[ARGMEMONLY_NOFREE_NOUNWIND_READONLY]]
+; CHECK: declare i64 @strcspn(ptr captures(none), ptr captures(none)) [[ARGMEMONLY_NOFREE_NOUNWIND_READONLY]]
 declare i64 @strcspn(ptr, ptr)
 
-; CHECK: declare noalias ptr @strdup(ptr nocapture readonly) [[INACCESSIBLEMEMORARGONLY_NOFREE_NOUNWIND_WILLRETURN_FAMILY_MALLOC:#[0-9]+]]
+; CHECK: declare noalias ptr @strdup(ptr readonly captures(none)) [[INACCESSIBLEMEMORARGONLY_NOFREE_NOUNWIND_WILLRETURN_FAMILY_MALLOC:#[0-9]+]]
 declare ptr @strdup(ptr)
 
-; CHECK: declare i64 @strlen(ptr nocapture) [[ARGMEMONLY_NOFREE_NOUNWIND_READONLY]]
+; CHECK: declare i64 @strlen(ptr captures(none)) [[ARGMEMONLY_NOFREE_NOUNWIND_READONLY]]
 declare i64 @strlen(ptr)
 
-; CHECK: declare i32 @strncasecmp(ptr nocapture, ptr nocapture, i64) [[NOFREE_NOUNWIND_READONLY_WILLRETURN]]
+; CHECK: declare i32 @strncasecmp(ptr captures(none), ptr captures(none), i64) [[NOFREE_NOUNWIND_READONLY_WILLRETURN]]
 declare i32 @strncasecmp(ptr, ptr, i64)
 
-; CHECK: declare ptr @strncat(ptr noalias returned, ptr noalias nocapture readonly, i64) [[ARGMEMONLY_NOFREE_NOUNWIND_WILLRETURN]]
+; CHECK: declare ptr @strncat(ptr noalias returned, ptr noalias readonly captures(none), i64) [[ARGMEMONLY_NOFREE_NOUNWIND_WILLRETURN]]
 declare ptr @strncat(ptr, ptr, i64)
 
-; CHECK: declare i32 @strncmp(ptr nocapture, ptr nocapture, i64) [[ARGMEMONLY_NOFREE_NOUNWIND_READONLY]]
+; CHECK: declare i32 @strncmp(ptr captures(none), ptr captures(none), i64) [[ARGMEMONLY_NOFREE_NOUNWIND_READONLY]]
 declare i32 @strncmp(ptr, ptr, i64)
 
-; CHECK: declare ptr @strncpy(ptr noalias returned writeonly, ptr noalias nocapture readonly, i64) [[ARGMEMONLY_NOFREE_NOUNWIND_WILLRETURN]]
+; CHECK: declare ptr @strncpy(ptr noalias returned writeonly, ptr noalias readonly captures(none), i64) [[ARGMEMONLY_NOFREE_NOUNWIND_WILLRETURN]]
 declare ptr @strncpy(ptr, ptr, i64)
 
-; CHECK: declare noalias ptr @strndup(ptr nocapture readonly, i64 noundef) [[INACCESSIBLEMEMORARGONLY_NOFREE_NOUNWIND_WILLRETURN_FAMILY_MALLOC]]
+; CHECK: declare noalias ptr @strndup(ptr readonly captures(none), i64 noundef) [[INACCESSIBLEMEMORARGONLY_NOFREE_NOUNWIND_WILLRETURN_FAMILY_MALLOC]]
 declare ptr @strndup(ptr, i64)
 
-; CHECK: declare i64 @strnlen(ptr nocapture, i64) [[ARGMEMONLY_NOFREE_NOUNWIND_READONLY_WILLRETURN]]
+; CHECK: declare i64 @strnlen(ptr captures(none), i64) [[ARGMEMONLY_NOFREE_NOUNWIND_READONLY_WILLRETURN]]
 declare i64 @strnlen(ptr, i64)
 
-; CHECK: declare ptr @strpbrk(ptr, ptr nocapture) [[ARGMEMONLY_NOFREE_NOUNWIND_READONLY_WILLRETURN]]
+; CHECK: declare ptr @strpbrk(ptr, ptr captures(none)) [[ARGMEMONLY_NOFREE_NOUNWIND_READONLY_WILLRETURN]]
 declare ptr @strpbrk(ptr, ptr)
 
 ; CHECK: declare ptr @strrchr(ptr, i32) [[ARGMEMONLY_NOFREE_NOUNWIND_READONLY]]
 declare ptr @strrchr(ptr, i32)
 
-; CHECK: declare i64 @strspn(ptr nocapture, ptr nocapture) [[ARGMEMONLY_NOFREE_NOUNWIND_READONLY]]
+; CHECK: declare i64 @strspn(ptr captures(none), ptr captures(none)) [[ARGMEMONLY_NOFREE_NOUNWIND_READONLY]]
 declare i64 @strspn(ptr, ptr)
 
-; CHECK: declare ptr @strstr(ptr, ptr nocapture) [[ARGMEMONLY_NOFREE_NOUNWIND_READONLY_WILLRETURN]]
+; CHECK: declare ptr @strstr(ptr, ptr captures(none)) [[ARGMEMONLY_NOFREE_NOUNWIND_READONLY_WILLRETURN]]
 declare ptr @strstr(ptr, ptr)
 
-; CHECK: declare double @strtod(ptr readonly, ptr nocapture) [[NOFREE_NOUNWIND_WILLRETURN]]
+; CHECK: declare double @strtod(ptr readonly, ptr captures(none)) [[NOFREE_NOUNWIND_WILLRETURN]]
 declare double @strtod(ptr, ptr)
 
-; CHECK: declare float @strtof(ptr readonly, ptr nocapture) [[NOFREE_NOUNWIND_WILLRETURN]]
+; CHECK: declare float @strtof(ptr readonly, ptr captures(none)) [[NOFREE_NOUNWIND_WILLRETURN]]
 declare float @strtof(ptr, ptr)
 
-; CHECK: declare ptr @strtok(ptr, ptr nocapture readonly) [[NOFREE_NOUNWIND_WILLRETURN]]
+; CHECK: declare ptr @strtok(ptr, ptr readonly captures(none)) [[NOFREE_NOUNWIND_WILLRETURN]]
 declare ptr @strtok(ptr, ptr)
 
-; CHECK: declare ptr @strtok_r(ptr, ptr nocapture readonly, ptr) [[NOFREE_NOUNWIND_WILLRETURN]]
+; CHECK: declare ptr @strtok_r(ptr, ptr readonly captures(none), ptr) [[NOFREE_NOUNWIND_WILLRETURN]]
 declare ptr @strtok_r(ptr, ptr, ptr)
 
-; CHECK: declare i64 @strtol(ptr readonly, ptr nocapture, i32) [[NOFREE_NOUNWIND_WILLRETURN]]
+; CHECK: declare i64 @strtol(ptr readonly, ptr captures(none), i32) [[NOFREE_NOUNWIND_WILLRETURN]]
 declare i64 @strtol(ptr, ptr, i32)
 
-; CHECK: declare x86_fp80 @strtold(ptr readonly, ptr nocapture) [[NOFREE_NOUNWIND_WILLRETURN]]
+; CHECK: declare x86_fp80 @strtold(ptr readonly, ptr captures(none)) [[NOFREE_NOUNWIND_WILLRETURN]]
 declare x86_fp80 @strtold(ptr, ptr)
 
-; CHECK: declare i64 @strtoll(ptr readonly, ptr nocapture, i32) [[NOFREE_NOUNWIND_WILLRETURN]]
+; CHECK: declare i64 @strtoll(ptr readonly, ptr captures(none), i32) [[NOFREE_NOUNWIND_WILLRETURN]]
 declare i64 @strtoll(ptr, ptr, i32)
 
-; CHECK: declare i64 @strtoul(ptr readonly, ptr nocapture, i32) [[NOFREE_NOUNWIND_WILLRETURN]]
+; CHECK: declare i64 @strtoul(ptr readonly, ptr captures(none), i32) [[NOFREE_NOUNWIND_WILLRETURN]]
 declare i64 @strtoul(ptr, ptr, i32)
 
-; CHECK: declare i64 @strtoull(ptr readonly, ptr nocapture, i32) [[NOFREE_NOUNWIND_WILLRETURN]]
+; CHECK: declare i64 @strtoull(ptr readonly, ptr captures(none), i32) [[NOFREE_NOUNWIND_WILLRETURN]]
 declare i64 @strtoull(ptr, ptr, i32)
 
-; CHECK: declare i64 @strxfrm(ptr nocapture, ptr nocapture readonly, i64) [[NOFREE_NOUNWIND_WILLRETURN]]
+; CHECK: declare i64 @strxfrm(ptr captures(none), ptr readonly captures(none), i64) [[NOFREE_NOUNWIND_WILLRETURN]]
 declare i64 @strxfrm(ptr, ptr, i64)
 
-; CHECK: declare noundef i32 @system(ptr nocapture noundef readonly) [[NOFREE]]
+; CHECK: declare noundef i32 @system(ptr noundef readonly captures(none)) [[NOFREE]]
 declare i32 @system(ptr)
 
 ; CHECK: declare double @tan(double) [[NOFREE_NOUNWIND_WILLRETURN_WRITEONLY]]
@@ -1089,7 +1089,7 @@ declare x86_fp80 @tanhl(x86_fp80)
 ; CHECK: declare x86_fp80 @tanl(x86_fp80) [[NOFREE_NOUNWIND_WILLRETURN_WRITEONLY]]
 declare x86_fp80 @tanl(x86_fp80)
 
-; CHECK: declare noundef i64 @times(ptr nocapture noundef) [[NOFREE_NOUNWIND]]
+; CHECK: declare noundef i64 @times(ptr noundef captures(none)) [[NOFREE_NOUNWIND]]
 declare i64 @times(ptr)
 
 ; CHECK: declare noalias noundef ptr @tmpfile() [[NOFREE_NOUNWIND]]
@@ -1110,49 +1110,49 @@ declare float @truncf(float)
 ; CHECK: declare x86_fp80 @truncl(x86_fp80) [[NOFREE_NOUNWIND_WILLRETURN_WRITEONLY]]
 declare x86_fp80 @truncl(x86_fp80)
 
-; CHECK: declare noundef i32 @uname(ptr nocapture noundef) [[NOFREE_NOUNWIND]]
+; CHECK: declare noundef i32 @uname(ptr noundef captures(none)) [[NOFREE_NOUNWIND]]
 declare i32 @uname(ptr)
 
-; CHECK: declare noundef i32 @ungetc(i32 noundef, ptr nocapture noundef) [[NOFREE_NOUNWIND]]
+; CHECK: declare noundef i32 @ungetc(i32 noundef, ptr noundef captures(none)) [[NOFREE_NOUNWIND]]
 declare i32 @ungetc(i32, ptr)
 
-; CHECK: declare noundef i32 @unlink(ptr nocapture noundef readonly) [[NOFREE_NOUNWIND]]
+; CHECK: declare noundef i32 @unlink(ptr noundef readonly captures(none)) [[NOFREE_NOUNWIND]]
 declare i32 @unlink(ptr)
 
-; CHECK: declare noundef i32 @unsetenv(ptr nocapture noundef readonly) [[NOFREE_NOUNWIND]]
+; CHECK: declare noundef i32 @unsetenv(ptr noundef readonly captures(none)) [[NOFREE_NOUNWIND]]
 declare i32 @unsetenv(ptr)
 
-; CHECK: declare noundef i32 @utime(ptr nocapture noundef readonly, ptr nocapture noundef readonly) [[NOFREE_NOUNWIND]]
+; CHECK: declare noundef i32 @utime(ptr noundef readonly captures(none), ptr noundef readonly captures(none)) [[NOFREE_NOUNWIND]]
 declare i32 @utime(ptr, ptr)
 
-; CHECK: declare noundef i32 @utimes(ptr nocapture noundef readonly, ptr nocapture noundef readonly) [[NOFREE_NOUNWIND]]
+; CHECK: declare noundef i32 @utimes(ptr noundef readonly captures(none), ptr noundef readonly captures(none)) [[NOFREE_NOUNWIND]]
 declare i32 @utimes(ptr, ptr)
 
 ; CHECK: declare noalias noundef ptr @valloc(i64 noundef) [[INACCESSIBLEMEMONLY_NOFREE_NOUNWIND_WILLRETURN_ALLOCKIND_ALLOCUNINIT_ALLOCSIZE0_FAMILY_MALLOC]]
 declare ptr @valloc(i64)
 
-; CHECK: declare noundef i32 @vfprintf(ptr nocapture noundef, ptr nocapture noundef readonly, ptr noundef) [[NOFREE_NOUNWIND]]
+; CHECK: declare noundef i32 @vfprintf(ptr noundef captures(none), ptr noundef readonly captures(none), ptr noundef) [[NOFREE_NOUNWIND]]
 declare i32 @vfprintf(ptr, ptr, ptr)
 
-; CHECK: declare noundef i32 @vfscanf(ptr nocapture noundef, ptr nocapture noundef readonly, ptr noundef) [[NOFREE_NOUNWIND]]
+; CHECK: declare noundef i32 @vfscanf(ptr noundef captures(none), ptr noundef readonly captures(none), ptr noundef) [[NOFREE_NOUNWIND]]
 declare i32 @vfscanf(ptr, ptr, ptr)
 
-; CHECK: declare noundef i32 @vprintf(ptr nocapture noundef readonly, ptr noundef) [[NOFREE_NOUNWIND]]
+; CHECK: declare noundef i32 @vprintf(ptr noundef readonly captures(none), ptr noundef) [[NOFREE_NOUNWIND]]
 declare i32 @vprintf(ptr, ptr)
 
-; CHECK: declare noundef i32 @vscanf(ptr nocapture noundef readonly, ptr noundef) [[NOFREE_NOUNWIND]]
+; CHECK: declare noundef i32 @vscanf(ptr noundef readonly captures(none), ptr noundef) [[NOFREE_NOUNWIND]]
 declare i32 @vscanf(ptr, ptr)
 
-; CHECK: declare noundef i32 @vsnprintf(ptr nocapture noundef, i64 noundef, ptr nocapture noundef readonly, ptr noundef) [[NOFREE_NOUNWIND]]
+; CHECK: declare noundef i32 @vsnprintf(ptr noundef captures(none), i64 noundef, ptr noundef readonly captures(none), ptr noundef) [[NOFREE_NOUNWIND]]
 declare i32 @vsnprintf(ptr, i64, ptr, ptr)
 
-; CHECK: declare noundef i32 @vsprintf(ptr nocapture noundef, ptr nocapture noundef readonly, ptr noundef) [[NOFREE_NOUNWIND]]
+; CHECK: declare noundef i32 @vsprintf(ptr noundef captures(none), ptr noundef readonly captures(none), ptr noundef) [[NOFREE_NOUNWIND]]
 declare i32 @vsprintf(ptr, ptr, ptr)
 
-; CHECK: declare noundef i32 @vsscanf(ptr nocapture noundef readonly, ptr nocapture noundef readonly, ptr noundef) [[NOFREE_NOUNWIND]]
+; CHECK: declare noundef i32 @vsscanf(ptr noundef readonly captures(none), ptr noundef readonly captures(none), ptr noundef) [[NOFREE_NOUNWIND]]
 declare i32 @vsscanf(ptr, ptr, ptr)
 
-; CHECK: declare noundef i64 @write(i32 noundef, ptr nocapture noundef readonly, i64 noundef) [[NOFREE]]
+; CHECK: declare noundef i64 @write(i32 noundef, ptr noundef readonly captures(none), i64 noundef) [[NOFREE]]
 declare i64 @write(i32, ptr, i64)
 
 ; CHECK: declare void @abort() [[NOFREE_COLD:#[0-9]+]]
@@ -1164,21 +1164,21 @@ declare void @__cxa_throw(ptr, ptr, ptr)
 ; CHECK: declare void @_ZSt9terminatev() [[NOFREE_COLD_NORETURN:#[0-9]+]]
 declare void @_ZSt9terminatev()
 
-; CHECK: declare void @sincos(double, ptr nocapture writeonly, ptr nocapture writeonly) [[NOFREE_NOUNWIND_WILLRETURN_WRITEONLY]]
+; CHECK: declare void @sincos(double, ptr writeonly captures(none), ptr writeonly captures(none)) [[NOFREE_NOUNWIND_WILLRETURN_WRITEONLY]]
 declare void @sincos(double, ptr, ptr)
 
-; CHECK: declare void @sincosf(float, ptr nocapture writeonly, ptr nocapture writeonly) [[NOFREE_NOUNWIND_WILLRETURN_WRITEONLY]]
+; CHECK: declare void @sincosf(float, ptr writeonly captures(none), ptr writeonly captures(none)) [[NOFREE_NOUNWIND_WILLRETURN_WRITEONLY]]
 declare void @sincosf(float, ptr, ptr)
 
-; CHECK: declare void @sincosl(x86_fp80, ptr nocapture writeonly, ptr nocapture writeonly) [[NOFREE_NOUNWIND_WILLRETURN_WRITEONLY]]
+; CHECK: declare void @sincosl(x86_fp80, ptr writeonly captures(none), ptr writeonly captures(none)) [[NOFREE_NOUNWIND_WILLRETURN_WRITEONLY]]
 declare void @sincosl(x86_fp80, ptr, ptr)
 
 ; memset_pattern{4,8,16} aren't available everywhere.
-; CHECK-DARWIN: declare void @memset_pattern4(ptr nocapture writeonly, ptr nocapture readonly, i64) [[ARGMEMONLY_NOFREE_NOUNWIND_WILLRETURN]]
+; CHECK-DARWIN: declare void @memset_pattern4(ptr writeonly captures(none), ptr readonly captures(none), i64) [[ARGMEMONLY_NOFREE_NOUNWIND_WILLRETURN]]
 declare void @memset_pattern4(ptr, ptr, i64)
-; CHECK-DARWIN: declare void @memset_pattern8(ptr nocapture writeonly, ptr nocapture readonly, i64) [[ARGMEMONLY_NOFREE_NOUNWIND_WILLRETURN]]
+; CHECK-DARWIN: declare void @memset_pattern8(ptr writeonly captures(none), ptr readonly captures(none), i64) [[ARGMEMONLY_NOFREE_NOUNWIND_WILLRETURN]]
 declare void @memset_pattern8(ptr, ptr, i64)
-; CHECK-DARWIN: declare void @memset_pattern16(ptr nocapture writeonly, ptr nocapture readonly, i64) [[ARGMEMONLY_NOFREE_NOUNWIND_WILLRETURN]]
+; CHECK-DARWIN: declare void @memset_pattern16(ptr writeonly captures(none), ptr readonly captures(none), i64) [[ARGMEMONLY_NOFREE_NOUNWIND_WILLRETURN]]
 declare void @memset_pattern16(ptr, ptr, i64)
 
 ; CHECK-DAG: attributes [[NOFREE_NOUNWIND_WILLRETURN]] = { mustprogress nofree nounwind willreturn }
diff --git a/llvm/test/Transforms/Inline/align.ll b/llvm/test/Transforms/Inline/align.ll
index a85ba7e49c6b6..8bf94c91418f9 100644
--- a/llvm/test/Transforms/Inline/align.ll
+++ b/llvm/test/Transforms/Inline/align.ll
@@ -5,7 +5,7 @@ target triple = "x86_64-unknown-linux-gnu"
 
 define void @hello(ptr align 128 nocapture %a, ptr nocapture readonly %c) #0 {
 ; CHECK-LABEL: define {{[^@]+}}@hello
-; CHECK-SAME: (ptr nocapture align 128 [[A:%.*]], ptr nocapture readonly [[C:%.*]]) #0
+; CHECK-SAME: (ptr align 128 captures(none) [[A:%.*]], ptr readonly captures(none) [[C:%.*]]) #[[ATTR0:[0-9]+]] {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[TMP0:%.*]] = load float, ptr [[C]], align 4
 ; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds float, ptr [[A]], i64 5
@@ -21,7 +21,7 @@ entry:
 
 define void @foo(ptr nocapture %a, ptr nocapture readonly %c) #0 {
 ; CHECK-LABEL: define {{[^@]+}}@foo
-; CHECK-SAME: (ptr nocapture [[A:%.*]], ptr nocapture readonly [[C:%.*]]) #0
+; CHECK-SAME: (ptr captures(none) [[A:%.*]], ptr readonly captures(none) [[C:%.*]]) #[[ATTR0]] {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    call void @llvm.assume(i1 true) [ "align"(ptr [[A]], i64 128) ]
 ; CHECK-NEXT:    [[TMP0:%.*]] = load float, ptr [[C]], align 4
@@ -42,7 +42,7 @@ entry:
 
 define void @fooa(ptr nocapture align 128 %a, ptr nocapture readonly %c) #0 {
 ; CHECK-LABEL: define {{[^@]+}}@fooa
-; CHECK-SAME: (ptr nocapture align 128 [[A:%.*]], ptr nocapture readonly [[C:%.*]]) #0
+; CHECK-SAME: (ptr align 128 captures(none) [[A:%.*]], ptr readonly captures(none) [[C:%.*]]) #[[ATTR0]] {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[TMP0:%.*]] = load float, ptr [[C]], align 4
 ; CHECK-NEXT:    [[ARRAYIDX_I:%.*]] = getelementptr inbounds float, ptr [[A]], i64 5
@@ -62,7 +62,7 @@ entry:
 
 define void @hello2(ptr align 128 nocapture %a, ptr align 128 nocapture %b, ptr nocapture readonly %c) #0 {
 ; CHECK-LABEL: define {{[^@]+}}@hello2
-; CHECK-SAME: (ptr nocapture align 128 [[A:%.*]], ptr nocapture align 128 [[B:%.*]], ptr nocapture readonly [[C:%.*]]) #0
+; CHECK-SAME: (ptr align 128 captures(none) [[A:%.*]], ptr align 128 captures(none) [[B:%.*]], ptr readonly captures(none) [[C:%.*]]) #[[ATTR0]] {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[TMP0:%.*]] = load float, ptr [[C]], align 4
 ; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds float, ptr [[A]], i64 5
@@ -82,7 +82,7 @@ entry:
 
 define void @foo2(ptr nocapture %a, ptr nocapture %b, ptr nocapture readonly %c) #0 {
 ; CHECK-LABEL: define {{[^@]+}}@foo2
-; CHECK-SAME: (ptr nocapture [[A:%.*]], ptr nocapture [[B:%.*]], ptr nocapture readonly [[C:%.*]]) #0
+; CHECK-SAME: (ptr captures(none) [[A:%.*]], ptr captures(none) [[B:%.*]], ptr readonly captures(none) [[C:%.*]]) #[[ATTR0]] {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    call void @llvm.assume(i1 true) [ "align"(ptr [[A]], i64 128) ]
 ; CHECK-NEXT:    call void @llvm.assume(i1 true) [ "align"(ptr [[B]], i64 128) ]
diff --git a/llvm/test/Transforms/Inline/byref-align.ll b/llvm/test/Transforms/Inline/byref-align.ll
index 128a3c9148b75..1f2af078fe143 100644
--- a/llvm/test/Transforms/Inline/byref-align.ll
+++ b/llvm/test/Transforms/Inline/byref-align.ll
@@ -8,7 +8,7 @@ target triple = "x86_64-unknown-linux-gnu"
 ; should be inserted.
 define void @byref_callee(ptr align(128) byref(float) nocapture %a, ptr %b) #0 {
 ; CHECK-LABEL: define {{[^@]+}}@byref_callee
-; CHECK-SAME: (ptr nocapture byref(float) align 128 [[A:%.*]], ptr [[B:%.*]]) [[ATTR0:#.*]] {
+; CHECK-SAME: (ptr byref(float) align 128 captures(none) [[A:%.*]], ptr [[B:%.*]]) #[[ATTR0:[0-9]+]] {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[LOAD:%.*]] = load float, ptr [[A]], align 4
 ; CHECK-NEXT:    [[B_IDX:%.*]] = getelementptr inbounds float, ptr [[B]], i64 8
@@ -26,7 +26,7 @@ entry:
 
 define void @byref_caller(ptr nocapture align 64 %a, ptr %b) #0 {
 ; CHECK-LABEL: define {{[^@]+}}@byref_caller
-; CHECK-SAME: (ptr nocapture align 64 [[A:%.*]], ptr [[B:%.*]]) [[ATTR0]] {
+; CHECK-SAME: (ptr align 64 captures(none) [[A:%.*]], ptr [[B:%.*]]) #[[ATTR0]] {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    call void @llvm.assume(i1 true) [ "align"(ptr [[A]], i64 128) ]
 ; CHECK-NEXT:    [[LOAD_I:%.*]] = load float, ptr [[A]], align 4
diff --git a/llvm/test/Transforms/Inline/byval-align.ll b/llvm/test/Transforms/Inline/byval-align.ll
index 11a3330fc6d7a..766094f05be0c 100644
--- a/llvm/test/Transforms/Inline/byval-align.ll
+++ b/llvm/test/Transforms/Inline/byval-align.ll
@@ -7,7 +7,7 @@ target triple = "x86_64-unknown-linux-gnu"
 ; assume should be inserted.
 define void @byval_callee(ptr byval(float) align 128 nocapture %a, ptr %b) #0 {
 ; CHECK-LABEL: define {{[^@]+}}@byval_callee
-; CHECK-SAME: (ptr nocapture byval(float) align 128 [[A:%.*]], ptr [[B:%.*]]) #0
+; CHECK-SAME: (ptr byval(float) align 128 captures(none) [[A:%.*]], ptr [[B:%.*]]) #[[ATTR0:[0-9]+]] {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[LOAD:%.*]] = load float, ptr [[A]], align 4
 ; CHECK-NEXT:    [[B_IDX:%.*]] = getelementptr inbounds float, ptr [[B]], i64 8
@@ -25,7 +25,7 @@ entry:
 
 define void @byval_caller(ptr nocapture align 64 %a, ptr %b) #0 {
 ; CHECK-LABEL: define {{[^@]+}}@byval_caller
-; CHECK-SAME: (ptr nocapture align 64 [[A:%.*]], ptr [[B:%.*]]) #0
+; CHECK-SAME: (ptr align 64 captures(none) [[A:%.*]], ptr [[B:%.*]]) #[[ATTR0]] {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[A1:%.*]] = alloca float, align 128
 ; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr [[A1]])
diff --git a/llvm/test/Transforms/Inline/byval.ll b/llvm/test/Transforms/Inline/byval.ll
index 1a70da8472cb1..d98f6e8efa05d 100644
--- a/llvm/test/Transforms/Inline/byval.ll
+++ b/llvm/test/Transforms/Inline/byval.ll
@@ -184,7 +184,7 @@ entry:
 
 define internal void @f5_as1(ptr addrspace(1) byval(%struct.S1) nocapture readonly align 4 %p) {
 ; CHECK-LABEL: define internal void @f5_as1(
-; CHECK-SAME: ptr addrspace(1) nocapture readonly byval([[STRUCT_S1:%.*]]) align 4 [[P:%.*]]) {
+; CHECK-SAME: ptr addrspace(1) readonly byval([[STRUCT_S1:%.*]]) align 4 captures(none) [[P:%.*]]) {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    store i32 0, ptr addrspace(1) @d, align 4
 ; CHECK-NEXT:    [[TMP0:%.*]] = load i32, ptr addrspace(1) [[P]], align 4
diff --git a/llvm/test/Transforms/Inline/noalias-calls.ll b/llvm/test/Transforms/Inline/noalias-calls.ll
index 0dd9ec3498a93..fdbad60b0e1cd 100644
--- a/llvm/test/Transforms/Inline/noalias-calls.ll
+++ b/llvm/test/Transforms/Inline/noalias-calls.ll
@@ -9,7 +9,7 @@ declare void @hey() #0
 
 define void @hello(ptr noalias nocapture %a, ptr noalias nocapture readonly %c, ptr nocapture %b) #1 {
 ; CHECK-LABEL: define {{[^@]+}}@hello
-; CHECK-SAME: (ptr noalias nocapture [[A:%.*]], ptr noalias nocapture readonly [[C:%.*]], ptr nocapture [[B:%.*]]) #[[ATTR2:[0-9]+]] {
+; CHECK-SAME: (ptr noalias captures(none) [[A:%.*]], ptr noalias readonly captures(none) [[C:%.*]], ptr captures(none) [[B:%.*]]) #[[ATTR2:[0-9]+]] {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[L:%.*]] = alloca i8, i32 512, align 1
 ; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 16 [[A]], ptr align 16 [[B]], i64 16, i1 false)
@@ -31,7 +31,7 @@ entry:
 
 define void @foo(ptr nocapture %a, ptr nocapture readonly %c, ptr nocapture %b) #2 {
 ; CHECK-LABEL: define {{[^@]+}}@foo
-; CHECK-SAME: (ptr nocapture [[A:%.*]], ptr nocapture readonly [[C:%.*]], ptr nocapture [[B:%.*]]) #[[ATTR3:[0-9]+]] {
+; CHECK-SAME: (ptr captures(none) [[A:%.*]], ptr readonly captures(none) [[C:%.*]], ptr captures(none) [[B:%.*]]) #[[ATTR3:[0-9]+]] {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[L_I:%.*]] = alloca i8, i32 512, align 1
 ; CHECK-NEXT:    call void @llvm.experimental.noalias.scope.decl(metadata [[META0:![0-9]+]])
@@ -52,7 +52,7 @@ entry:
 
 define void @hello_cs(ptr nocapture %a, ptr nocapture readonly %c, ptr nocapture %b) #1 {
 ; CHECK-LABEL: define {{[^@]+}}@hello_cs
-; CHECK-SAME: (ptr nocapture [[A:%.*]], ptr nocapture readonly [[C:%.*]], ptr nocapture [[B:%.*]]) #[[ATTR2]] {
+; CHECK-SAME: (ptr captures(none) [[A:%.*]], ptr readonly captures(none) [[C:%.*]], ptr captures(none) [[B:%.*]]) #[[ATTR2]] {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[L:%.*]] = alloca i8, i32 512, align 1
 ; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 16 [[A]], ptr align 16 [[B]], i64 16, i1 false)
@@ -74,7 +74,7 @@ entry:
 
 define void @foo_cs(ptr nocapture %a, ptr nocapture readonly %c, ptr nocapture %b) #2 {
 ; CHECK-LABEL: define {{[^@]+}}@foo_cs
-; CHECK-SAME: (ptr nocapture [[A:%.*]], ptr nocapture readonly [[C:%.*]], ptr nocapture [[B:%.*]]) #[[ATTR3]] {
+; CHECK-SAME: (ptr captures(none) [[A:%.*]], ptr readonly captures(none) [[C:%.*]], ptr captures(none) [[B:%.*]]) #[[ATTR3]] {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[L_I:%.*]] = alloca i8, i32 512, align 1
 ; CHECK-NEXT:    call void @llvm.experimental.noalias.scope.decl(metadata [[META6:![0-9]+]])
diff --git a/llvm/test/Transforms/Inline/noalias.ll b/llvm/test/Transforms/Inline/noalias.ll
index e25513080bedd..974e87a53fd9a 100644
--- a/llvm/test/Transforms/Inline/noalias.ll
+++ b/llvm/test/Transforms/Inline/noalias.ll
@@ -19,7 +19,7 @@ entry:
   ret void
 }
 
-; CHECK-LABEL: define void @foo(ptr nocapture %a, ptr nocapture readonly %c) #0 {
+; CHECK-LABEL: define void @foo(ptr captures(none) %a, ptr readonly captures(none) %c) #0 {
 ; CHECK: entry:
 ; CHECK:   call void @llvm.experimental.noalias.scope.decl
 ; CHECK:   [[TMP0:%.+]] = load float, ptr %c, align 4, !noalias !0
@@ -50,7 +50,7 @@ entry:
   ret void
 }
 
-; CHECK-LABEL: define void @foo2(ptr nocapture %a, ptr nocapture %b, ptr nocapture readonly %c) #0 {
+; CHECK-LABEL: define void @foo2(ptr captures(none) %a, ptr captures(none) %b, ptr readonly captures(none) %c) #0 {
 ; CHECK: entry:
 ; CHECK:   call void @llvm.experimental.noalias.scope.decl(metadata !3)
 ; CHECK:   call void @llvm.experimental.noalias.scope.decl(metadata !6)
diff --git a/llvm/test/Transforms/Inline/noalias2.ll b/llvm/test/Transforms/Inline/noalias2.ll
index 7e43598f9502b..fcf1e3870afd0 100644
--- a/llvm/test/Transforms/Inline/noalias2.ll
+++ b/llvm/test/Transforms/Inline/noalias2.ll
@@ -7,7 +7,7 @@ target triple = "x86_64-unknown-linux-gnu"
 
 define void @hello(ptr noalias nocapture %a, ptr noalias nocapture readonly %c) #0 {
 ; CHECK-LABEL: define {{[^@]+}}@hello
-; CHECK-SAME: (ptr noalias nocapture [[A:%.*]], ptr noalias nocapture readonly [[C:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-SAME: (ptr noalias captures(none) [[A:%.*]], ptr noalias readonly captures(none) [[C:%.*]]) #[[ATTR0:[0-9]+]] {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[TMP0:%.*]] = load float, ptr [[C]], align 4
 ; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds float, ptr [[A]], i64 5
@@ -23,13 +23,13 @@ entry:
 
 define void @foo(ptr noalias nocapture %a, ptr noalias nocapture readonly %c) #0 {
 ; CHECK-LABEL: define {{[^@]+}}@foo
-; CHECK-SAME: (ptr noalias nocapture [[A:%.*]], ptr noalias nocapture readonly [[C:%.*]]) #[[ATTR0]] {
+; CHECK-SAME: (ptr noalias captures(none) [[A:%.*]], ptr noalias readonly captures(none) [[C:%.*]]) #[[ATTR0]] {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    call void @llvm.experimental.noalias.scope.decl(metadata [[META0:![0-9]+]])
 ; CHECK-NEXT:    call void @llvm.experimental.noalias.scope.decl(metadata [[META3:![0-9]+]])
-; CHECK-NEXT:    [[TMP0:%.*]] = load float, ptr [[C]], align 4, !alias.scope !3, !noalias !0
+; CHECK-NEXT:    [[TMP0:%.*]] = load float, ptr [[C]], align 4, !alias.scope [[META3]], !noalias [[META0]]
 ; CHECK-NEXT:    [[ARRAYIDX_I:%.*]] = getelementptr inbounds float, ptr [[A]], i64 5
-; CHECK-NEXT:    store float [[TMP0]], ptr [[ARRAYIDX_I]], align 4, !alias.scope !0, !noalias !3
+; CHECK-NEXT:    store float [[TMP0]], ptr [[ARRAYIDX_I]], align 4, !alias.scope [[META0]], !noalias [[META3]]
 ; CHECK-NEXT:    [[TMP1:%.*]] = load float, ptr [[C]], align 4
 ; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds float, ptr [[A]], i64 7
 ; CHECK-NEXT:    store float [[TMP1]], ptr [[ARRAYIDX]], align 4
@@ -45,7 +45,7 @@ entry:
 
 define void @hello2(ptr noalias nocapture %a, ptr noalias nocapture %b, ptr nocapture readonly %c) #0 {
 ; CHECK-LABEL: define {{[^@]+}}@hello2
-; CHECK-SAME: (ptr noalias nocapture [[A:%.*]], ptr noalias nocapture [[B:%.*]], ptr nocapture readonly [[C:%.*]]) #[[ATTR0]] {
+; CHECK-SAME: (ptr noalias captures(none) [[A:%.*]], ptr noalias captures(none) [[B:%.*]], ptr readonly captures(none) [[C:%.*]]) #[[ATTR0]] {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[TMP0:%.*]] = load float, ptr [[C]], align 4
 ; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds float, ptr [[A]], i64 6
@@ -67,25 +67,25 @@ entry:
 ; foo2(), the noalias scopes are properly concatenated.
 define void @foo2(ptr nocapture %a, ptr nocapture %b, ptr nocapture readonly %c) #0 {
 ; CHECK-LABEL: define {{[^@]+}}@foo2
-; CHECK-SAME: (ptr nocapture [[A:%.*]], ptr nocapture [[B:%.*]], ptr nocapture readonly [[C:%.*]]) #[[ATTR0]] {
+; CHECK-SAME: (ptr captures(none) [[A:%.*]], ptr captures(none) [[B:%.*]], ptr readonly captures(none) [[C:%.*]]) #[[ATTR0]] {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    call void @llvm.experimental.noalias.scope.decl(metadata [[META5:![0-9]+]])
 ; CHECK-NEXT:    call void @llvm.experimental.noalias.scope.decl(metadata [[META8:![0-9]+]])
 ; CHECK-NEXT:    call void @llvm.experimental.noalias.scope.decl(metadata [[META10:![0-9]+]])
 ; CHECK-NEXT:    call void @llvm.experimental.noalias.scope.decl(metadata [[META13:![0-9]+]])
-; CHECK-NEXT:    [[TMP0:%.*]] = load float, ptr [[C]], align 4, !alias.scope !15, !noalias !16
+; CHECK-NEXT:    [[TMP0:%.*]] = load float, ptr [[C]], align 4, !alias.scope [[META15:![0-9]+]], !noalias [[META16:![0-9]+]]
 ; CHECK-NEXT:    [[ARRAYIDX_I_I:%.*]] = getelementptr inbounds float, ptr [[A]], i64 5
-; CHECK-NEXT:    store float [[TMP0]], ptr [[ARRAYIDX_I_I]], align 4, !alias.scope !16, !noalias !15
-; CHECK-NEXT:    [[TMP1:%.*]] = load float, ptr [[C]], align 4, !alias.scope !8, !noalias !5
+; CHECK-NEXT:    store float [[TMP0]], ptr [[ARRAYIDX_I_I]], align 4, !alias.scope [[META16]], !noalias [[META15]]
+; CHECK-NEXT:    [[TMP1:%.*]] = load float, ptr [[C]], align 4, !alias.scope [[META8]], !noalias [[META5]]
 ; CHECK-NEXT:    [[ARRAYIDX_I:%.*]] = getelementptr inbounds float, ptr [[A]], i64 7
-; CHECK-NEXT:    store float [[TMP1]], ptr [[ARRAYIDX_I]], align 4, !alias.scope !5, !noalias !8
+; CHECK-NEXT:    store float [[TMP1]], ptr [[ARRAYIDX_I]], align 4, !alias.scope [[META5]], !noalias [[META8]]
 ; CHECK-NEXT:    call void @llvm.experimental.noalias.scope.decl(metadata [[META17:![0-9]+]])
 ; CHECK-NEXT:    call void @llvm.experimental.noalias.scope.decl(metadata [[META20:![0-9]+]])
-; CHECK-NEXT:    [[TMP2:%.*]] = load float, ptr [[C]], align 4, !noalias !22
+; CHECK-NEXT:    [[TMP2:%.*]] = load float, ptr [[C]], align 4, !noalias [[META22:![0-9]+]]
 ; CHECK-NEXT:    [[ARRAYIDX_I1:%.*]] = getelementptr inbounds float, ptr [[A]], i64 6
-; CHECK-NEXT:    store float [[TMP2]], ptr [[ARRAYIDX_I1]], align 4, !alias.scope !17, !noalias !20
+; CHECK-NEXT:    store float [[TMP2]], ptr [[ARRAYIDX_I1]], align 4, !alias.scope [[META17]], !noalias [[META20]]
 ; CHECK-NEXT:    [[ARRAYIDX1_I:%.*]] = getelementptr inbounds float, ptr [[B]], i64 8
-; CHECK-NEXT:    store float [[TMP2]], ptr [[ARRAYIDX1_I]], align 4, !alias.scope !20, !noalias !17
+; CHECK-NEXT:    store float [[TMP2]], ptr [[ARRAYIDX1_I]], align 4, !alias.scope [[META20]], !noalias [[META17]]
 ; CHECK-NEXT:    [[TMP3:%.*]] = load float, ptr [[C]], align 4
 ; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds float, ptr [[A]], i64 7
 ; CHECK-NEXT:    store float [[TMP3]], ptr [[ARRAYIDX]], align 4
diff --git a/llvm/test/Transforms/InstCombine/AMDGPU/llvm.amdgcn.wavefrontsize.ll b/llvm/test/Transforms/InstCombine/AMDGPU/llvm.amdgcn.wavefrontsize.ll
index d9c105f753e26..92f0af30b9e4d 100644
--- a/llvm/test/Transforms/InstCombine/AMDGPU/llvm.amdgcn.wavefrontsize.ll
+++ b/llvm/test/Transforms/InstCombine/AMDGPU/llvm.amdgcn.wavefrontsize.ll
@@ -10,20 +10,20 @@
 
 define amdgpu_kernel void @fold_wavefrontsize(ptr addrspace(1) nocapture %arg) {
 ; OPT-LABEL: define amdgpu_kernel void @fold_wavefrontsize(
-; OPT-SAME: ptr addrspace(1) nocapture [[ARG:%.*]]) {
+; OPT-SAME: ptr addrspace(1) captures(none) [[ARG:%.*]]) {
 ; OPT-NEXT:  [[BB:.*:]]
 ; OPT-NEXT:    [[TMP:%.*]] = tail call i32 @llvm.amdgcn.wavefrontsize() #[[ATTR1:[0-9]+]]
 ; OPT-NEXT:    store i32 [[TMP]], ptr addrspace(1) [[ARG]], align 4
 ; OPT-NEXT:    ret void
 ;
 ; OPT-W32-LABEL: define amdgpu_kernel void @fold_wavefrontsize(
-; OPT-W32-SAME: ptr addrspace(1) nocapture [[ARG:%.*]]) #[[ATTR0:[0-9]+]] {
+; OPT-W32-SAME: ptr addrspace(1) captures(none) [[ARG:%.*]]) #[[ATTR0:[0-9]+]] {
 ; OPT-W32-NEXT:  [[BB:.*:]]
 ; OPT-W32-NEXT:    store i32 32, ptr addrspace(1) [[ARG]], align 4
 ; OPT-W32-NEXT:    ret void
 ;
 ; OPT-W64-LABEL: define amdgpu_kernel void @fold_wavefrontsize(
-; OPT-W64-SAME: ptr addrspace(1) nocapture [[ARG:%.*]]) #[[ATTR0:[0-9]+]] {
+; OPT-W64-SAME: ptr addrspace(1) captures(none) [[ARG:%.*]]) #[[ATTR0:[0-9]+]] {
 ; OPT-W64-NEXT:  [[BB:.*:]]
 ; OPT-W64-NEXT:    store i32 64, ptr addrspace(1) [[ARG]], align 4
 ; OPT-W64-NEXT:    ret void
@@ -36,7 +36,7 @@ bb:
 
 define amdgpu_kernel void @fold_and_optimize_wavefrontsize(ptr addrspace(1) nocapture %arg) {
 ; OPT-LABEL: define amdgpu_kernel void @fold_and_optimize_wavefrontsize(
-; OPT-SAME: ptr addrspace(1) nocapture [[ARG:%.*]]) {
+; OPT-SAME: ptr addrspace(1) captures(none) [[ARG:%.*]]) {
 ; OPT-NEXT:  [[BB:.*:]]
 ; OPT-NEXT:    [[TMP:%.*]] = tail call i32 @llvm.amdgcn.wavefrontsize() #[[ATTR1]]
 ; OPT-NEXT:    [[TMP1:%.*]] = icmp ugt i32 [[TMP]], 32
@@ -45,13 +45,13 @@ define amdgpu_kernel void @fold_and_optimize_wavefrontsize(ptr addrspace(1) noca
 ; OPT-NEXT:    ret void
 ;
 ; OPT-W32-LABEL: define amdgpu_kernel void @fold_and_optimize_wavefrontsize(
-; OPT-W32-SAME: ptr addrspace(1) nocapture [[ARG:%.*]]) #[[ATTR0]] {
+; OPT-W32-SAME: ptr addrspace(1) captures(none) [[ARG:%.*]]) #[[ATTR0]] {
 ; OPT-W32-NEXT:  [[BB:.*:]]
 ; OPT-W32-NEXT:    store i32 1, ptr addrspace(1) [[ARG]], align 4
 ; OPT-W32-NEXT:    ret void
 ;
 ; OPT-W64-LABEL: define amdgpu_kernel void @fold_and_optimize_wavefrontsize(
-; OPT-W64-SAME: ptr addrspace(1) nocapture [[ARG:%.*]]) #[[ATTR0]] {
+; OPT-W64-SAME: ptr addrspace(1) captures(none) [[ARG:%.*]]) #[[ATTR0]] {
 ; OPT-W64-NEXT:  [[BB:.*:]]
 ; OPT-W64-NEXT:    store i32 2, ptr addrspace(1) [[ARG]], align 4
 ; OPT-W64-NEXT:    ret void
@@ -66,7 +66,7 @@ bb:
 
 define amdgpu_kernel void @fold_and_optimize_if_wavefrontsize(ptr addrspace(1) nocapture %arg) {
 ; OPT-LABEL: define amdgpu_kernel void @fold_and_optimize_if_wavefrontsize(
-; OPT-SAME: ptr addrspace(1) nocapture [[ARG:%.*]]) {
+; OPT-SAME: ptr addrspace(1) captures(none) [[ARG:%.*]]) {
 ; OPT-NEXT:  [[BB:.*:]]
 ; OPT-NEXT:    [[TMP:%.*]] = tail call i32 @llvm.amdgcn.wavefrontsize() #[[ATTR1]]
 ; OPT-NEXT:    [[TMP1:%.*]] = icmp ugt i32 [[TMP]], 32
@@ -78,7 +78,7 @@ define amdgpu_kernel void @fold_and_optimize_if_wavefrontsize(ptr addrspace(1) n
 ; OPT-NEXT:    ret void
 ;
 ; OPT-W32-LABEL: define amdgpu_kernel void @fold_and_optimize_if_wavefrontsize(
-; OPT-W32-SAME: ptr addrspace(1) nocapture [[ARG:%.*]]) #[[ATTR0]] {
+; OPT-W32-SAME: ptr addrspace(1) captures(none) [[ARG:%.*]]) #[[ATTR0]] {
 ; OPT-W32-NEXT:  [[BB:.*:]]
 ; OPT-W32-NEXT:    br i1 false, label %[[BB2:.*]], label %[[BB3:.*]]
 ; OPT-W32:       [[BB2]]:
@@ -87,7 +87,7 @@ define amdgpu_kernel void @fold_and_optimize_if_wavefrontsize(ptr addrspace(1) n
 ; OPT-W32-NEXT:    ret void
 ;
 ; OPT-W64-LABEL: define amdgpu_kernel void @fold_and_optimize_if_wavefrontsize(
-; OPT-W64-SAME: ptr addrspace(1) nocapture [[ARG:%.*]]) #[[ATTR0]] {
+; OPT-W64-SAME: ptr addrspace(1) captures(none) [[ARG:%.*]]) #[[ATTR0]] {
 ; OPT-W64-NEXT:  [[BB:.*:]]
 ; OPT-W64-NEXT:    br i1 true, label %[[BB2:.*]], label %[[BB3:.*]]
 ; OPT-W64:       [[BB2]]:
diff --git a/llvm/test/Transforms/InstCombine/NVPTX/isspacep.ll b/llvm/test/Transforms/InstCombine/NVPTX/isspacep.ll
index dedd85e1a8cda..01c0e222299bd 100644
--- a/llvm/test/Transforms/InstCombine/NVPTX/isspacep.ll
+++ b/llvm/test/Transforms/InstCombine/NVPTX/isspacep.ll
@@ -25,7 +25,7 @@ declare i1 @llvm.nvvm.isspacep.local(ptr nocapture)
 
 define dso_local void @check_global(ptr nocapture noundef readnone %out, ptr nocapture noundef readnone %genp,
 ; CHECK-LABEL: define dso_local void @check_global(
-; CHECK-SAME: ptr nocapture noundef readnone [[OUT:%.*]], ptr nocapture noundef readnone [[GENP:%.*]], ptr addrspace(1) [[GP:%.*]], ptr addrspace(3) [[SP:%.*]], ptr addrspace(4) [[CP:%.*]], ptr addrspace(5) [[LP:%.*]]) local_unnamed_addr {
+; CHECK-SAME: ptr noundef readnone captures(none) [[OUT:%.*]], ptr noundef readnone captures(none) [[GENP:%.*]], ptr addrspace(1) [[GP:%.*]], ptr addrspace(3) [[SP:%.*]], ptr addrspace(4) [[CP:%.*]], ptr addrspace(5) [[LP:%.*]]) local_unnamed_addr {
 ; CHECK-NEXT:  [[ENTRY:.*:]]
 ; CHECK-NEXT:    [[GEN0:%.*]] = tail call i1 @llvm.nvvm.isspacep.global(ptr [[GENP]])
 ; CHECK-NEXT:    [[STOREDV:%.*]] = zext i1 [[GEN0]] to i8
@@ -88,7 +88,7 @@ entry:
 
 define dso_local void @check_shared(ptr nocapture noundef readnone %out, ptr nocapture noundef readnone %genp,
 ; CHECK-LABEL: define dso_local void @check_shared(
-; CHECK-SAME: ptr nocapture noundef readnone [[OUT:%.*]], ptr nocapture noundef readnone [[GENP:%.*]], ptr addrspace(1) [[GP:%.*]], ptr addrspace(3) [[SP:%.*]], ptr addrspace(4) [[CP:%.*]], ptr addrspace(5) [[LP:%.*]]) local_unnamed_addr {
+; CHECK-SAME: ptr noundef readnone captures(none) [[OUT:%.*]], ptr noundef readnone captures(none) [[GENP:%.*]], ptr addrspace(1) [[GP:%.*]], ptr addrspace(3) [[SP:%.*]], ptr addrspace(4) [[CP:%.*]], ptr addrspace(5) [[LP:%.*]]) local_unnamed_addr {
 ; CHECK-NEXT:  [[ENTRY:.*:]]
 ; CHECK-NEXT:    [[GEN0:%.*]] = tail call i1 @llvm.nvvm.isspacep.shared(ptr [[GENP]])
 ; CHECK-NEXT:    [[STOREDV:%.*]] = zext i1 [[GEN0]] to i8
@@ -151,7 +151,7 @@ entry:
 
 define dso_local void @check_const(ptr nocapture noundef readnone %out, ptr nocapture noundef readnone %genp,
 ; CHECK-LABEL: define dso_local void @check_const(
-; CHECK-SAME: ptr nocapture noundef readnone [[OUT:%.*]], ptr nocapture noundef readnone [[GENP:%.*]], ptr addrspace(1) [[GP:%.*]], ptr addrspace(3) [[SP:%.*]], ptr addrspace(4) [[CP:%.*]], ptr addrspace(5) [[LP:%.*]]) local_unnamed_addr {
+; CHECK-SAME: ptr noundef readnone captures(none) [[OUT:%.*]], ptr noundef readnone captures(none) [[GENP:%.*]], ptr addrspace(1) [[GP:%.*]], ptr addrspace(3) [[SP:%.*]], ptr addrspace(4) [[CP:%.*]], ptr addrspace(5) [[LP:%.*]]) local_unnamed_addr {
 ; CHECK-NEXT:  [[ENTRY:.*:]]
 ; CHECK-NEXT:    [[GEN0:%.*]] = tail call i1 @llvm.nvvm.isspacep.const(ptr [[GENP]])
 ; CHECK-NEXT:    [[STOREDV:%.*]] = zext i1 [[GEN0]] to i8
@@ -214,7 +214,7 @@ entry:
 
 define dso_local void @check_local(ptr nocapture noundef readnone %out, ptr nocapture noundef readnone %genp,
 ; CHECK-LABEL: define dso_local void @check_local(
-; CHECK-SAME: ptr nocapture noundef readnone [[OUT:%.*]], ptr nocapture noundef readnone [[GENP:%.*]], ptr addrspace(1) [[GP:%.*]], ptr addrspace(3) [[SP:%.*]], ptr addrspace(4) [[CP:%.*]], ptr addrspace(5) [[LP:%.*]]) local_unnamed_addr {
+; CHECK-SAME: ptr noundef readnone captures(none) [[OUT:%.*]], ptr noundef readnone captures(none) [[GENP:%.*]], ptr addrspace(1) [[GP:%.*]], ptr addrspace(3) [[SP:%.*]], ptr addrspace(4) [[CP:%.*]], ptr addrspace(5) [[LP:%.*]]) local_unnamed_addr {
 ; CHECK-NEXT:  [[ENTRY:.*:]]
 ; CHECK-NEXT:    [[GEN0:%.*]] = tail call i1 @llvm.nvvm.isspacep.local(ptr [[GENP]])
 ; CHECK-NEXT:    [[STOREDV:%.*]] = zext i1 [[GEN0]] to i8
diff --git a/llvm/test/Transforms/InstCombine/RISCV/libcall-arg-exts.ll b/llvm/test/Transforms/InstCombine/RISCV/libcall-arg-exts.ll
index 559413d7973bb..fc4d2b5600a62 100644
--- a/llvm/test/Transforms/InstCombine/RISCV/libcall-arg-exts.ll
+++ b/llvm/test/Transforms/InstCombine/RISCV/libcall-arg-exts.ll
@@ -86,7 +86,7 @@ define void @fun8(i32 %chr) {
 ; CHECK: declare double @ldexp(double, i32 signext)
 ; CHECK: declare float @ldexpf(float, i32 signext)
 ; CHECK: declare fp128 @ldexpl(fp128, i32 signext)
-; CHECK: declare ptr @memccpy(ptr noalias writeonly, ptr noalias nocapture readonly, i32 signext, i64)
-; CHECK: declare noundef i32 @fputc(i32 noundef signext, ptr nocapture noundef)
+; CHECK: declare ptr @memccpy(ptr noalias writeonly, ptr noalias readonly captures(none), i32 signext, i64)
+; CHECK: declare noundef i32 @fputc(i32 noundef signext, ptr noundef captures(none))
 ; CHECK: declare noundef i32 @putchar(i32 noundef signext)
 ; CHECK: declare ptr @memchr(ptr, i32 signext, i64)
diff --git a/llvm/test/Transforms/InstCombine/RISCV/memcmp.ll b/llvm/test/Transforms/InstCombine/RISCV/memcmp.ll
index 7447677ea4e75..65421d5aa5362 100644
--- a/llvm/test/Transforms/InstCombine/RISCV/memcmp.ll
+++ b/llvm/test/Transforms/InstCombine/RISCV/memcmp.ll
@@ -16,4 +16,4 @@ define signext i32 @test_bcmp(ptr %mem1, ptr %mem2, i64 %size) {
   ret i32 %zext
 }
 
-; CHECK: declare signext i32 @bcmp(ptr nocapture, ptr nocapture, i64)
+; CHECK: declare signext i32 @bcmp(ptr captures(none), ptr captures(none), i64)
diff --git a/llvm/test/Transforms/InstCombine/SystemZ/libcall-arg-exts.ll b/llvm/test/Transforms/InstCombine/SystemZ/libcall-arg-exts.ll
index 7d3645681e2d8..dd905effca45d 100644
--- a/llvm/test/Transforms/InstCombine/SystemZ/libcall-arg-exts.ll
+++ b/llvm/test/Transforms/InstCombine/SystemZ/libcall-arg-exts.ll
@@ -86,7 +86,7 @@ define void @fun8(i32 %chr) {
 ; CHECK: declare double @ldexp(double, i32 signext)
 ; CHECK: declare float @ldexpf(float, i32 signext)
 ; CHECK: declare fp128 @ldexpl(fp128, i32 signext)
-; CHECK: declare ptr @memccpy(ptr noalias writeonly, ptr noalias nocapture readonly, i32 signext, i64)
-; CHECK: declare noundef i32 @fputc(i32 noundef signext, ptr nocapture noundef)
+; CHECK: declare ptr @memccpy(ptr noalias writeonly, ptr noalias readonly captures(none), i32 signext, i64)
+; CHECK: declare noundef i32 @fputc(i32 noundef signext, ptr noundef captures(none))
 ; CHECK: declare noundef i32 @putchar(i32 noundef signext)
 ; CHECK: declare ptr @memchr(ptr, i32 signext, i64)
diff --git a/llvm/test/Transforms/InstCombine/add2.ll b/llvm/test/Transforms/InstCombine/add2.ll
index ae80ab2e92ad1..c474a33c48a2b 100644
--- a/llvm/test/Transforms/InstCombine/add2.ll
+++ b/llvm/test/Transforms/InstCombine/add2.ll
@@ -321,6 +321,17 @@ define i16 @mul_add_to_mul_9(i16 %a) {
   ret i16 %add
 }
 
+@g = external global i8
+
+define i32 @shl_add_to_shl_constexpr() {
+; CHECK-LABEL: @shl_add_to_shl_constexpr(
+; CHECK-NEXT:    ret i32 mul (i32 ptrtoint (ptr @g to i32), i32 4)
+;
+  %shl = shl i32 ptrtoint (ptr @g to i32), 1
+  %add = add i32 %shl, %shl
+  ret i32 %add
+}
+
 ; This test and the next test verify that when a range metadata is attached to
 ; llvm.cttz, ValueTracking correctly intersects the range specified by the
 ; metadata and the range implied by the intrinsic.
diff --git a/llvm/test/Transforms/InstCombine/align-addr.ll b/llvm/test/Transforms/InstCombine/align-addr.ll
index 6ef4d85fe4e41..b77037e592b54 100644
--- a/llvm/test/Transforms/InstCombine/align-addr.ll
+++ b/llvm/test/Transforms/InstCombine/align-addr.ll
@@ -112,7 +112,7 @@ define void @test3(ptr sret(%struct.s) %a4) {
 ; Check that the alignment is bumped up the alignment of the sret type.
 ; CHECK-LABEL: @test3(
 ; CHECK-NEXT:    call void @llvm.memset.p0.i64(ptr noundef nonnull align 4 dereferenceable(16) [[A4:%.*]], i8 0, i64 16, i1 false)
-; CHECK-NEXT:    call void @use(ptr [[A4]])
+; CHECK-NEXT:    call void @use(ptr nonnull [[A4]])
 ; CHECK-NEXT:    ret void
 ;
   call void @llvm.memset.p0.i64(ptr %a4, i8 0, i64 16, i1 false)
diff --git a/llvm/test/Transforms/InstCombine/and-compare.ll b/llvm/test/Transforms/InstCombine/and-compare.ll
index c8f93410f9e9c..3a59aca0fa8e1 100644
--- a/llvm/test/Transforms/InstCombine/and-compare.ll
+++ b/llvm/test/Transforms/InstCombine/and-compare.ll
@@ -256,3 +256,27 @@ entry:
   %cmp = icmp ne i8 %and, 11
   ret i1 %cmp
 }
+
+define i1 @test_eq_16_and_15_add_10(i8 %a) {
+; CHECK-LABEL: @test_eq_16_and_15_add_10(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    ret i1 false
+;
+entry:
+  %add = add i8 %a, 10
+  %and = and i8 %add, 15
+  %cmp = icmp eq i8 %and, 16
+  ret i1 %cmp
+}
+
+define i1 @test_ne_16_and_15_add_10(i8 %a) {
+; CHECK-LABEL: @test_ne_16_and_15_add_10(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    ret i1 true
+;
+entry:
+  %add = add i8 %a, 10
+  %and = and i8 %add, 15
+  %cmp = icmp ne i8 %and, 16
+  ret i1 %cmp
+}
diff --git a/llvm/test/Transforms/InstCombine/bit_ceil.ll b/llvm/test/Transforms/InstCombine/bit_ceil.ll
index a2e27dfd6f64d..09f90ee05735d 100644
--- a/llvm/test/Transforms/InstCombine/bit_ceil.ll
+++ b/llvm/test/Transforms/InstCombine/bit_ceil.ll
@@ -302,9 +302,9 @@ define i32 @pr91691(i32 %0) {
   ret i32 %7
 }
 
-define i32 @pr91691_keep_nsw(i32 %0) {
-; CHECK-LABEL: @pr91691_keep_nsw(
-; CHECK-NEXT:    [[TMP2:%.*]] = sub nsw i32 -2, [[TMP0:%.*]]
+define i32 @pr91691_drop_nsw(i32 %0) {
+; CHECK-LABEL: @pr91691_drop_nsw(
+; CHECK-NEXT:    [[TMP2:%.*]] = sub i32 -2, [[TMP0:%.*]]
 ; CHECK-NEXT:    [[TMP3:%.*]] = tail call range(i32 0, 33) i32 @llvm.ctlz.i32(i32 [[TMP2]], i1 false)
 ; CHECK-NEXT:    [[TMP4:%.*]] = sub nsw i32 0, [[TMP3]]
 ; CHECK-NEXT:    [[TMP5:%.*]] = and i32 [[TMP4]], 31
@@ -337,6 +337,46 @@ define i32 @test_drop_range_attr(i32 %x) {
   ret i32 %sel
 }
 
+define i32 @bit_ceil_plus_nsw(i32 %x) {
+; CHECK-LABEL: @bit_ceil_plus_nsw(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[SUB:%.*]] = add i32 [[X:%.*]], 1
+; CHECK-NEXT:    [[CTLZ:%.*]] = tail call range(i32 0, 33) i32 @llvm.ctlz.i32(i32 [[SUB]], i1 false)
+; CHECK-NEXT:    [[TMP0:%.*]] = sub nsw i32 0, [[CTLZ]]
+; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[TMP0]], 31
+; CHECK-NEXT:    [[SEL:%.*]] = shl nuw i32 1, [[TMP1]]
+; CHECK-NEXT:    ret i32 [[SEL]]
+;
+entry:
+  %sub = add nsw i32 %x, 1
+  %ctlz = tail call i32 @llvm.ctlz.i32(i32 %sub, i1 false)
+  %sub2 = sub nuw nsw i32 32, %ctlz
+  %shl = shl nuw i32 1, %sub2
+  %ult = icmp ult i32 %x, 2147483647
+  %sel = select i1 %ult, i32 %shl, i32 1
+  ret i32 %sel
+}
+
+define i32 @bit_ceil_plus_nuw(i32 %x) {
+; CHECK-LABEL: @bit_ceil_plus_nuw(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[SUB:%.*]] = add i32 [[X:%.*]], 1
+; CHECK-NEXT:    [[CTLZ:%.*]] = tail call range(i32 0, 33) i32 @llvm.ctlz.i32(i32 [[SUB]], i1 false)
+; CHECK-NEXT:    [[TMP0:%.*]] = sub nsw i32 0, [[CTLZ]]
+; CHECK-NEXT:    [[SUB2:%.*]] = and i32 [[TMP0]], 31
+; CHECK-NEXT:    [[SHL:%.*]] = shl nuw i32 1, [[SUB2]]
+; CHECK-NEXT:    ret i32 [[SHL]]
+;
+entry:
+  %sub = add nuw i32 %x, 1
+  %ctlz = tail call i32 @llvm.ctlz.i32(i32 %sub, i1 false)
+  %sub2 = sub nuw nsw i32 32, %ctlz
+  %shl = shl nuw i32 1, %sub2
+  %ult = icmp ult i32 %x, 2147483647
+  %sel = select i1 %ult, i32 %shl, i32 1
+  ret i32 %sel
+}
+
 declare i32 @llvm.ctlz.i32(i32, i1 immarg)
 declare i64 @llvm.ctlz.i64(i64, i1 immarg)
 declare <4 x i32> @llvm.ctlz.v4i32(<4 x i32>, i1)
diff --git a/llvm/test/Transforms/InstCombine/call-cast-attrs.ll b/llvm/test/Transforms/InstCombine/call-cast-attrs.ll
index bb122b0e2c4aa..009ddaf6b5e37 100644
--- a/llvm/test/Transforms/InstCombine/call-cast-attrs.ll
+++ b/llvm/test/Transforms/InstCombine/call-cast-attrs.ll
@@ -26,8 +26,8 @@ define void @g(ptr %y) {
   call void @c(ptr %y)
   call void @c(ptr sret(i32) %y)
   call void @d(i32 0, ptr sret(i32) %y)
-  call void @d(i32 0, ptr nocapture %y)
-  call void @d(ptr nocapture noundef %y)
+  call void @d(i32 0, ptr captures(none) %y)
+  call void @d(ptr noundef captures(none) %y)
   call void @naked_func(i32 1)
   ret void
 }
@@ -36,8 +36,8 @@ define void @g(ptr %y) {
 ; CHECK:    call void (...) @c(ptr %y)
 ; CHECK:    call void @c(ptr sret(i32) %y)
 ; CHECK:    call void @d(i32 0, ptr sret(i32) %y)
-; CHECK:    call void (i32, ...) @d(i32 0, ptr nocapture %y)
+; CHECK:    call void (i32, ...) @d(i32 0, ptr captures(none) %y)
 ; CHECK32:  %2 = ptrtoint ptr %y to i32
 ; CHECK32:  call void (i32, ...) @d(i32 noundef %2)
-; CHECK64:  call void @d(ptr nocapture noundef %y)
+; CHECK64:  call void @d(ptr noundef captures(none) %y)
 ; CHECK:    call void @naked_func(i32 1)
diff --git a/llvm/test/Transforms/InstCombine/compare-alloca.ll b/llvm/test/Transforms/InstCombine/compare-alloca.ll
index e6342520c6fa9..a27cd70fe8bab 100644
--- a/llvm/test/Transforms/InstCombine/compare-alloca.ll
+++ b/llvm/test/Transforms/InstCombine/compare-alloca.ll
@@ -218,7 +218,7 @@ declare void @unknown(ptr)
 define i1 @consistent_nocapture_inttoptr() {
 ; CHECK-LABEL: @consistent_nocapture_inttoptr(
 ; CHECK-NEXT:    [[M1:%.*]] = alloca [4 x i8], align 1
-; CHECK-NEXT:    call void @unknown(ptr nocapture nonnull [[M1]])
+; CHECK-NEXT:    call void @unknown(ptr nonnull captures(none) [[M1]])
 ; CHECK-NEXT:    ret i1 false
 ;
   %m = alloca i8, i32 4
@@ -231,7 +231,7 @@ define i1 @consistent_nocapture_inttoptr() {
 define i1 @consistent_nocapture_offset() {
 ; CHECK-LABEL: @consistent_nocapture_offset(
 ; CHECK-NEXT:    [[M1:%.*]] = alloca [4 x i8], align 1
-; CHECK-NEXT:    call void @unknown(ptr nocapture nonnull [[M1]])
+; CHECK-NEXT:    call void @unknown(ptr nonnull captures(none) [[M1]])
 ; CHECK-NEXT:    ret i1 false
 ;
   %m = alloca i8, i32 4
@@ -247,7 +247,7 @@ define i1 @consistent_nocapture_offset() {
 define i1 @consistent_nocapture_through_global() {
 ; CHECK-LABEL: @consistent_nocapture_through_global(
 ; CHECK-NEXT:    [[M1:%.*]] = alloca [4 x i8], align 1
-; CHECK-NEXT:    call void @unknown(ptr nocapture nonnull [[M1]])
+; CHECK-NEXT:    call void @unknown(ptr nonnull captures(none) [[M1]])
 ; CHECK-NEXT:    ret i1 false
 ;
   %m = alloca i8, i32 4
diff --git a/llvm/test/Transforms/InstCombine/compare-unescaped.ll b/llvm/test/Transforms/InstCombine/compare-unescaped.ll
index c48c754e3941b..31e3e4581a2ef 100644
--- a/llvm/test/Transforms/InstCombine/compare-unescaped.ll
+++ b/llvm/test/Transforms/InstCombine/compare-unescaped.ll
@@ -311,7 +311,7 @@ declare void @unknown(ptr)
 define i1 @consistent_nocapture_inttoptr() {
 ; CHECK-LABEL: @consistent_nocapture_inttoptr(
 ; CHECK-NEXT:    [[M:%.*]] = call dereferenceable_or_null(4) ptr @malloc(i64 4)
-; CHECK-NEXT:    call void @unknown(ptr nocapture [[M]])
+; CHECK-NEXT:    call void @unknown(ptr captures(none) [[M]])
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp eq ptr [[M]], inttoptr (i64 2048 to ptr)
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
@@ -325,7 +325,7 @@ define i1 @consistent_nocapture_inttoptr() {
 define i1 @consistent_nocapture_offset() {
 ; CHECK-LABEL: @consistent_nocapture_offset(
 ; CHECK-NEXT:    [[M:%.*]] = call dereferenceable_or_null(4) ptr @malloc(i64 4)
-; CHECK-NEXT:    call void @unknown(ptr nocapture [[M]])
+; CHECK-NEXT:    call void @unknown(ptr captures(none) [[M]])
 ; CHECK-NEXT:    ret i1 false
 ;
   %m = call ptr @malloc(i64 4)
@@ -339,7 +339,7 @@ define i1 @consistent_nocapture_offset() {
 define i1 @consistent_nocapture_through_global() {
 ; CHECK-LABEL: @consistent_nocapture_through_global(
 ; CHECK-NEXT:    [[M:%.*]] = call dereferenceable_or_null(4) ptr @malloc(i64 4)
-; CHECK-NEXT:    call void @unknown(ptr nocapture [[M]])
+; CHECK-NEXT:    call void @unknown(ptr captures(none) [[M]])
 ; CHECK-NEXT:    ret i1 false
 ;
   %m = call ptr @malloc(i64 4)
diff --git a/llvm/test/Transforms/InstCombine/fabs.ll b/llvm/test/Transforms/InstCombine/fabs.ll
index cccf0f4457b6a..7b9a672f188ca 100644
--- a/llvm/test/Transforms/InstCombine/fabs.ll
+++ b/llvm/test/Transforms/InstCombine/fabs.ll
@@ -484,7 +484,7 @@ define double @select_fcmp_nnan_nsz_olt_zero(double %x) {
 ; CHECK-LABEL: @select_fcmp_nnan_nsz_olt_zero(
 ; CHECK-NEXT:    [[LTZERO:%.*]] = fcmp olt double [[X:%.*]], 0.000000e+00
 ; CHECK-NEXT:    [[NEGX:%.*]] = fneg nnan nsz double [[X]]
-; CHECK-NEXT:    [[FABS:%.*]] = select nnan i1 [[LTZERO]], double [[NEGX]], double [[X]]
+; CHECK-NEXT:    [[FABS:%.*]] = select i1 [[LTZERO]], double [[NEGX]], double [[X]]
 ; CHECK-NEXT:    ret double [[FABS]]
 ;
   %ltzero = fcmp olt double %x, 0.0
@@ -523,7 +523,7 @@ define double @select_fcmp_nnan_nsz_olt_zero_unary_fneg(double %x) {
 ; CHECK-LABEL: @select_fcmp_nnan_nsz_olt_zero_unary_fneg(
 ; CHECK-NEXT:    [[LTZERO:%.*]] = fcmp olt double [[X:%.*]], 0.000000e+00
 ; CHECK-NEXT:    [[NEGX:%.*]] = fneg nnan nsz double [[X]]
-; CHECK-NEXT:    [[FABS:%.*]] = select nnan i1 [[LTZERO]], double [[NEGX]], double [[X]]
+; CHECK-NEXT:    [[FABS:%.*]] = select i1 [[LTZERO]], double [[NEGX]], double [[X]]
 ; CHECK-NEXT:    ret double [[FABS]]
 ;
   %ltzero = fcmp olt double %x, 0.0
@@ -553,7 +553,7 @@ define float @select_fcmp_nnan_nsz_olt_negzero(float %x) {
 ; CHECK-LABEL: @select_fcmp_nnan_nsz_olt_negzero(
 ; CHECK-NEXT:    [[LTZERO:%.*]] = fcmp olt float [[X:%.*]], 0.000000e+00
 ; CHECK-NEXT:    [[NEGX:%.*]] = fneg nnan ninf nsz float [[X]]
-; CHECK-NEXT:    [[FABS:%.*]] = select nnan ninf i1 [[LTZERO]], float [[NEGX]], float [[X]]
+; CHECK-NEXT:    [[FABS:%.*]] = select i1 [[LTZERO]], float [[NEGX]], float [[X]]
 ; CHECK-NEXT:    ret float [[FABS]]
 ;
   %ltzero = fcmp olt float %x, -0.0
@@ -579,7 +579,7 @@ define float @select_fcmp_nnan_nsz_ult_negzero(float %x) {
 ; CHECK-LABEL: @select_fcmp_nnan_nsz_ult_negzero(
 ; CHECK-NEXT:    [[LTZERO:%.*]] = fcmp ult float [[X:%.*]], 0.000000e+00
 ; CHECK-NEXT:    [[NEGX:%.*]] = fneg nnan ninf nsz float [[X]]
-; CHECK-NEXT:    [[FABS:%.*]] = select nnan ninf i1 [[LTZERO]], float [[NEGX]], float [[X]]
+; CHECK-NEXT:    [[FABS:%.*]] = select nnan i1 [[LTZERO]], float [[NEGX]], float [[X]]
 ; CHECK-NEXT:    ret float [[FABS]]
 ;
   %ltzero = fcmp ult float %x, -0.0
@@ -592,7 +592,7 @@ define float @select_fcmp_nnan_nsz_olt_negzero_unary_fneg(float %x) {
 ; CHECK-LABEL: @select_fcmp_nnan_nsz_olt_negzero_unary_fneg(
 ; CHECK-NEXT:    [[LTZERO:%.*]] = fcmp olt float [[X:%.*]], 0.000000e+00
 ; CHECK-NEXT:    [[NEGX:%.*]] = fneg nnan ninf nsz float [[X]]
-; CHECK-NEXT:    [[FABS:%.*]] = select nnan ninf i1 [[LTZERO]], float [[NEGX]], float [[X]]
+; CHECK-NEXT:    [[FABS:%.*]] = select i1 [[LTZERO]], float [[NEGX]], float [[X]]
 ; CHECK-NEXT:    ret float [[FABS]]
 ;
   %ltzero = fcmp olt float %x, -0.0
@@ -607,7 +607,7 @@ define float @select_fcmp_nnan_nsz_ult_negzero_unary_fneg(float %x) {
 ; CHECK-LABEL: @select_fcmp_nnan_nsz_ult_negzero_unary_fneg(
 ; CHECK-NEXT:    [[LTZERO:%.*]] = fcmp ult float [[X:%.*]], 0.000000e+00
 ; CHECK-NEXT:    [[NEGX:%.*]] = fneg nnan ninf nsz float [[X]]
-; CHECK-NEXT:    [[FABS:%.*]] = select nnan ninf i1 [[LTZERO]], float [[NEGX]], float [[X]]
+; CHECK-NEXT:    [[FABS:%.*]] = select nnan i1 [[LTZERO]], float [[NEGX]], float [[X]]
 ; CHECK-NEXT:    ret float [[FABS]]
 ;
   %ltzero = fcmp ult float %x, -0.0
@@ -622,7 +622,7 @@ define double @select_fcmp_nnan_nsz_ole_zero(double %x) {
 ; CHECK-LABEL: @select_fcmp_nnan_nsz_ole_zero(
 ; CHECK-NEXT:    [[LEZERO:%.*]] = fcmp ole double [[X:%.*]], 0.000000e+00
 ; CHECK-NEXT:    [[NEGX:%.*]] = fneg fast double [[X]]
-; CHECK-NEXT:    [[FABS:%.*]] = select nnan ninf i1 [[LEZERO]], double [[NEGX]], double [[X]]
+; CHECK-NEXT:    [[FABS:%.*]] = select i1 [[LEZERO]], double [[NEGX]], double [[X]]
 ; CHECK-NEXT:    ret double [[FABS]]
 ;
   %lezero = fcmp ole double %x, 0.0
@@ -648,7 +648,7 @@ define double @select_fcmp_nnan_nsz_ule_zero(double %x) {
 ; CHECK-LABEL: @select_fcmp_nnan_nsz_ule_zero(
 ; CHECK-NEXT:    [[LEZERO:%.*]] = fcmp ule double [[X:%.*]], 0.000000e+00
 ; CHECK-NEXT:    [[NEGX:%.*]] = fneg fast double [[X]]
-; CHECK-NEXT:    [[FABS:%.*]] = select nnan ninf i1 [[LEZERO]], double [[NEGX]], double [[X]]
+; CHECK-NEXT:    [[FABS:%.*]] = select nnan i1 [[LEZERO]], double [[NEGX]], double [[X]]
 ; CHECK-NEXT:    ret double [[FABS]]
 ;
   %lezero = fcmp ule double %x, 0.0
@@ -661,7 +661,7 @@ define double @select_fcmp_nnan_nsz_ole_zero_unary_fneg(double %x) {
 ; CHECK-LABEL: @select_fcmp_nnan_nsz_ole_zero_unary_fneg(
 ; CHECK-NEXT:    [[LEZERO:%.*]] = fcmp ole double [[X:%.*]], 0.000000e+00
 ; CHECK-NEXT:    [[NEGX:%.*]] = fneg fast double [[X]]
-; CHECK-NEXT:    [[FABS:%.*]] = select nnan ninf i1 [[LEZERO]], double [[NEGX]], double [[X]]
+; CHECK-NEXT:    [[FABS:%.*]] = select i1 [[LEZERO]], double [[NEGX]], double [[X]]
 ; CHECK-NEXT:    ret double [[FABS]]
 ;
   %lezero = fcmp ole double %x, 0.0
@@ -676,7 +676,7 @@ define double @select_fcmp_nnan_nsz_ule_zero_unary_fneg(double %x) {
 ; CHECK-LABEL: @select_fcmp_nnan_nsz_ule_zero_unary_fneg(
 ; CHECK-NEXT:    [[LEZERO:%.*]] = fcmp ule double [[X:%.*]], 0.000000e+00
 ; CHECK-NEXT:    [[NEGX:%.*]] = fneg fast double [[X]]
-; CHECK-NEXT:    [[FABS:%.*]] = select nnan ninf i1 [[LEZERO]], double [[NEGX]], double [[X]]
+; CHECK-NEXT:    [[FABS:%.*]] = select nnan i1 [[LEZERO]], double [[NEGX]], double [[X]]
 ; CHECK-NEXT:    ret double [[FABS]]
 ;
   %lezero = fcmp ule double %x, 0.0
@@ -691,7 +691,7 @@ define float @select_fcmp_nnan_nsz_ole_negzero(float %x) {
 ; CHECK-LABEL: @select_fcmp_nnan_nsz_ole_negzero(
 ; CHECK-NEXT:    [[LEZERO:%.*]] = fcmp ole float [[X:%.*]], 0.000000e+00
 ; CHECK-NEXT:    [[NEGX:%.*]] = fneg nnan nsz float [[X]]
-; CHECK-NEXT:    [[FABS:%.*]] = select nnan i1 [[LEZERO]], float [[NEGX]], float [[X]]
+; CHECK-NEXT:    [[FABS:%.*]] = select i1 [[LEZERO]], float [[NEGX]], float [[X]]
 ; CHECK-NEXT:    ret float [[FABS]]
 ;
   %lezero = fcmp ole float %x, -0.0
@@ -730,7 +730,7 @@ define float @select_fcmp_nnan_nsz_ole_negzero_unary_fneg(float %x) {
 ; CHECK-LABEL: @select_fcmp_nnan_nsz_ole_negzero_unary_fneg(
 ; CHECK-NEXT:    [[LEZERO:%.*]] = fcmp ole float [[X:%.*]], 0.000000e+00
 ; CHECK-NEXT:    [[NEGX:%.*]] = fneg nnan nsz float [[X]]
-; CHECK-NEXT:    [[FABS:%.*]] = select nnan i1 [[LEZERO]], float [[NEGX]], float [[X]]
+; CHECK-NEXT:    [[FABS:%.*]] = select i1 [[LEZERO]], float [[NEGX]], float [[X]]
 ; CHECK-NEXT:    ret float [[FABS]]
 ;
   %lezero = fcmp ole float %x, -0.0
@@ -802,7 +802,7 @@ define <2 x float> @select_fcmp_nnan_nsz_ugt_zero(<2 x float> %x) {
 ; CHECK-LABEL: @select_fcmp_nnan_nsz_ugt_zero(
 ; CHECK-NEXT:    [[GTZERO:%.*]] = fcmp ugt <2 x float> [[X:%.*]], zeroinitializer
 ; CHECK-NEXT:    [[NEGX:%.*]] = fneg nnan nsz arcp <2 x float> [[X]]
-; CHECK-NEXT:    [[FABS:%.*]] = select nnan <2 x i1> [[GTZERO]], <2 x float> [[X]], <2 x float> [[NEGX]]
+; CHECK-NEXT:    [[FABS:%.*]] = select <2 x i1> [[GTZERO]], <2 x float> [[X]], <2 x float> [[NEGX]]
 ; CHECK-NEXT:    ret <2 x float> [[FABS]]
 ;
   %gtzero = fcmp ugt <2 x float> %x, zeroinitializer
@@ -830,7 +830,7 @@ define <2 x float> @select_fcmp_nnan_nsz_ugt_zero_unary_fneg(<2 x float> %x) {
 ; CHECK-LABEL: @select_fcmp_nnan_nsz_ugt_zero_unary_fneg(
 ; CHECK-NEXT:    [[GTZERO:%.*]] = fcmp ugt <2 x float> [[X:%.*]], zeroinitializer
 ; CHECK-NEXT:    [[NEGX:%.*]] = fneg nnan nsz arcp <2 x float> [[X]]
-; CHECK-NEXT:    [[FABS:%.*]] = select nnan <2 x i1> [[GTZERO]], <2 x float> [[X]], <2 x float> [[NEGX]]
+; CHECK-NEXT:    [[FABS:%.*]] = select <2 x i1> [[GTZERO]], <2 x float> [[X]], <2 x float> [[NEGX]]
 ; CHECK-NEXT:    ret <2 x float> [[FABS]]
 ;
   %gtzero = fcmp ugt <2 x float> %x, zeroinitializer
@@ -845,7 +845,7 @@ define half @select_fcmp_nnan_nsz_ogt_negzero(half %x) {
 ; CHECK-LABEL: @select_fcmp_nnan_nsz_ogt_negzero(
 ; CHECK-NEXT:    [[GTZERO:%.*]] = fcmp ogt half [[X:%.*]], 0xH0000
 ; CHECK-NEXT:    [[NEGX:%.*]] = fneg fast half [[X]]
-; CHECK-NEXT:    [[FABS:%.*]] = select nnan ninf i1 [[GTZERO]], half [[X]], half [[NEGX]]
+; CHECK-NEXT:    [[FABS:%.*]] = select nnan i1 [[GTZERO]], half [[X]], half [[NEGX]]
 ; CHECK-NEXT:    ret half [[FABS]]
 ;
   %gtzero = fcmp ogt half %x, -0.0
@@ -860,7 +860,7 @@ define half @select_fcmp_nnan_nsz_ugt_negzero(half %x) {
 ; CHECK-LABEL: @select_fcmp_nnan_nsz_ugt_negzero(
 ; CHECK-NEXT:    [[GTZERO:%.*]] = fcmp ugt half [[X:%.*]], 0xH0000
 ; CHECK-NEXT:    [[NEGX:%.*]] = fneg fast half [[X]]
-; CHECK-NEXT:    [[FABS:%.*]] = select nnan ninf i1 [[GTZERO]], half [[X]], half [[NEGX]]
+; CHECK-NEXT:    [[FABS:%.*]] = select i1 [[GTZERO]], half [[X]], half [[NEGX]]
 ; CHECK-NEXT:    ret half [[FABS]]
 ;
   %gtzero = fcmp ugt half %x, -0.0
@@ -890,7 +890,7 @@ define <2 x double> @select_fcmp_nnan_nsz_uge_zero(<2 x double> %x) {
 ; CHECK-LABEL: @select_fcmp_nnan_nsz_uge_zero(
 ; CHECK-NEXT:    [[GEZERO:%.*]] = fcmp uge <2 x double> [[X:%.*]], zeroinitializer
 ; CHECK-NEXT:    [[NEGX:%.*]] = fneg reassoc nnan nsz <2 x double> [[X]]
-; CHECK-NEXT:    [[FABS:%.*]] = select nnan <2 x i1> [[GEZERO]], <2 x double> [[X]], <2 x double> [[NEGX]]
+; CHECK-NEXT:    [[FABS:%.*]] = select <2 x i1> [[GEZERO]], <2 x double> [[X]], <2 x double> [[NEGX]]
 ; CHECK-NEXT:    ret <2 x double> [[FABS]]
 ;
   %gezero = fcmp uge <2 x double> %x, zeroinitializer
@@ -918,7 +918,7 @@ define <2 x double> @select_fcmp_nnan_nsz_uge_zero_unary_fneg(<2 x double> %x) {
 ; CHECK-LABEL: @select_fcmp_nnan_nsz_uge_zero_unary_fneg(
 ; CHECK-NEXT:    [[GEZERO:%.*]] = fcmp uge <2 x double> [[X:%.*]], zeroinitializer
 ; CHECK-NEXT:    [[NEGX:%.*]] = fneg reassoc nnan nsz <2 x double> [[X]]
-; CHECK-NEXT:    [[FABS:%.*]] = select nnan <2 x i1> [[GEZERO]], <2 x double> [[X]], <2 x double> [[NEGX]]
+; CHECK-NEXT:    [[FABS:%.*]] = select <2 x i1> [[GEZERO]], <2 x double> [[X]], <2 x double> [[NEGX]]
 ; CHECK-NEXT:    ret <2 x double> [[FABS]]
 ;
   %gezero = fcmp uge <2 x double> %x, zeroinitializer
@@ -948,7 +948,7 @@ define half @select_fcmp_nnan_nsz_uge_negzero(half %x) {
 ; CHECK-LABEL: @select_fcmp_nnan_nsz_uge_negzero(
 ; CHECK-NEXT:    [[GEZERO:%.*]] = fcmp uge half [[X:%.*]], 0xH0000
 ; CHECK-NEXT:    [[NEGX:%.*]] = fneg nnan nsz half [[X]]
-; CHECK-NEXT:    [[FABS:%.*]] = select nnan i1 [[GEZERO]], half [[X]], half [[NEGX]]
+; CHECK-NEXT:    [[FABS:%.*]] = select i1 [[GEZERO]], half [[X]], half [[NEGX]]
 ; CHECK-NEXT:    ret half [[FABS]]
 ;
   %gezero = fcmp uge half %x, -0.0
@@ -976,7 +976,7 @@ define half @select_fcmp_nnan_nsz_uge_negzero_unary_fneg(half %x) {
 ; CHECK-LABEL: @select_fcmp_nnan_nsz_uge_negzero_unary_fneg(
 ; CHECK-NEXT:    [[GEZERO:%.*]] = fcmp uge half [[X:%.*]], 0xH0000
 ; CHECK-NEXT:    [[NEGX:%.*]] = fneg nnan nsz half [[X]]
-; CHECK-NEXT:    [[FABS:%.*]] = select nnan i1 [[GEZERO]], half [[X]], half [[NEGX]]
+; CHECK-NEXT:    [[FABS:%.*]] = select i1 [[GEZERO]], half [[X]], half [[NEGX]]
 ; CHECK-NEXT:    ret half [[FABS]]
 ;
   %gezero = fcmp uge half %x, -0.0
diff --git a/llvm/test/Transforms/InstCombine/fneg-fabs.ll b/llvm/test/Transforms/InstCombine/fneg-fabs.ll
index fdcdfd123eefa..c013a2c9f5449 100644
--- a/llvm/test/Transforms/InstCombine/fneg-fabs.ll
+++ b/llvm/test/Transforms/InstCombine/fneg-fabs.ll
@@ -20,8 +20,8 @@ define double @select_noFMF_nfabs_lt(double %x) {
 ; One test where the neg has fmfs.
 define double @select_nsz_nfabs_lt_fmfProp(double %x) {
 ; CHECK-LABEL: @select_nsz_nfabs_lt_fmfProp(
-; CHECK-NEXT:    [[TMP1:%.*]] = call nnan ninf nsz double @llvm.fabs.f64(double [[X:%.*]])
-; CHECK-NEXT:    [[SEL:%.*]] = fneg nnan ninf nsz double [[TMP1]]
+; CHECK-NEXT:    [[TMP1:%.*]] = call nnan nsz double @llvm.fabs.f64(double [[X:%.*]])
+; CHECK-NEXT:    [[SEL:%.*]] = fneg nnan nsz double [[TMP1]]
 ; CHECK-NEXT:    ret double [[SEL]]
 ;
   %cmp = fcmp olt double %x, 0.000000e+00
@@ -32,8 +32,8 @@ define double @select_nsz_nfabs_lt_fmfProp(double %x) {
 
 define double @select_nsz_nnan_nfabs_lt_fmfProp(double %x) {
 ; CHECK-LABEL: @select_nsz_nnan_nfabs_lt_fmfProp(
-; CHECK-NEXT:    [[TMP1:%.*]] = call nnan ninf nsz double @llvm.fabs.f64(double [[X:%.*]])
-; CHECK-NEXT:    [[SEL:%.*]] = fneg nnan ninf nsz double [[TMP1]]
+; CHECK-NEXT:    [[TMP1:%.*]] = call nnan nsz double @llvm.fabs.f64(double [[X:%.*]])
+; CHECK-NEXT:    [[SEL:%.*]] = fneg nnan nsz double [[TMP1]]
 ; CHECK-NEXT:    ret double [[SEL]]
 ;
   %cmp = fcmp olt double %x, 0.000000e+00
@@ -147,8 +147,9 @@ define double @select_noFMF_nfabs_gt(double %x) {
 ; One test where the neg has fmfs.
 define double @select_nsz_nfabs_gt_fmfProp(double %x) {
 ; CHECK-LABEL: @select_nsz_nfabs_gt_fmfProp(
-; CHECK-NEXT:    [[TMP1:%.*]] = call nnan ninf nsz double @llvm.fabs.f64(double [[X:%.*]])
-; CHECK-NEXT:    [[SEL:%.*]] = fneg nnan ninf nsz double [[TMP1]]
+; CHECK-NEXT:    [[CMP:%.*]] = fcmp ogt double [[X:%.*]], 0.000000e+00
+; CHECK-NEXT:    [[NEGX:%.*]] = fneg fast double [[X]]
+; CHECK-NEXT:    [[SEL:%.*]] = select nsz i1 [[CMP]], double [[NEGX]], double [[X]]
 ; CHECK-NEXT:    ret double [[SEL]]
 ;
   %cmp = fcmp ogt double %x, 0.000000e+00
@@ -159,8 +160,8 @@ define double @select_nsz_nfabs_gt_fmfProp(double %x) {
 
 define double @select_nsz_nnan_nfabs_gt_fmfProp(double %x) {
 ; CHECK-LABEL: @select_nsz_nnan_nfabs_gt_fmfProp(
-; CHECK-NEXT:    [[TMP1:%.*]] = call nnan ninf nsz double @llvm.fabs.f64(double [[X:%.*]])
-; CHECK-NEXT:    [[SEL:%.*]] = fneg nnan ninf nsz double [[TMP1]]
+; CHECK-NEXT:    [[TMP1:%.*]] = call nnan nsz double @llvm.fabs.f64(double [[X:%.*]])
+; CHECK-NEXT:    [[SEL:%.*]] = fneg nnan nsz double [[TMP1]]
 ; CHECK-NEXT:    ret double [[SEL]]
 ;
   %cmp = fcmp ogt double %x, 0.000000e+00
diff --git a/llvm/test/Transforms/InstCombine/fpclass-check-idioms.ll b/llvm/test/Transforms/InstCombine/fpclass-check-idioms.ll
index 66970a9d48ddf..bb5713a1a56ea 100644
--- a/llvm/test/Transforms/InstCombine/fpclass-check-idioms.ll
+++ b/llvm/test/Transforms/InstCombine/fpclass-check-idioms.ll
@@ -566,6 +566,142 @@ define i1 @f32_fcnan_fcinf_noimplicitfloat_strictfp(float %a) strictfp #0 {
   ret i1 %cmp
 }
 
+define i1 @f32_fcsubnormal_fczero(float %a) {
+; CHECK-LABEL: define i1 @f32_fcsubnormal_fczero(
+; CHECK-SAME: float [[A:%.*]]) {
+; CHECK-NEXT:    [[CMP:%.*]] = call i1 @llvm.is.fpclass.f32(float [[A]], i32 240)
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %i32 = bitcast float %a to i32
+  %and = and i32 %i32, 2139095040
+  %cmp = icmp eq i32 %and, 0
+  ret i1 %cmp
+}
+
+define i1 @f32_not_fcsubnormal_fczero(float %a) {
+; CHECK-LABEL: define i1 @f32_not_fcsubnormal_fczero(
+; CHECK-SAME: float [[A:%.*]]) {
+; CHECK-NEXT:    [[CMP:%.*]] = call i1 @llvm.is.fpclass.f32(float [[A]], i32 783)
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %i32 = bitcast float %a to i32
+  %and = and i32 %i32, 2139095040
+  %cmp = icmp ne i32 %and, 0
+  ret i1 %cmp
+}
+
+define <2 x i1> @f64_fcsubnormal_fczero_vec(<2 x double> %a) {
+; CHECK-LABEL: define <2 x i1> @f64_fcsubnormal_fczero_vec(
+; CHECK-SAME: <2 x double> [[A:%.*]]) {
+; CHECK-NEXT:    [[CMP:%.*]] = call <2 x i1> @llvm.is.fpclass.v2f64(<2 x double> [[A]], i32 240)
+; CHECK-NEXT:    ret <2 x i1> [[CMP]]
+;
+  %i64 = bitcast <2 x double> %a to <2 x i64>
+  %and = and <2 x i64> %i64, splat(i64 9218868437227405312)
+  %cmp = icmp eq <2 x i64> %and, zeroinitializer
+  ret <2 x i1> %cmp
+}
+
+define <2 x i1> @f64_no_fcsubnormal_fczero_vec(<2 x double> %a) {
+; CHECK-LABEL: define <2 x i1> @f64_no_fcsubnormal_fczero_vec(
+; CHECK-SAME: <2 x double> [[A:%.*]]) {
+; CHECK-NEXT:    [[CMP:%.*]] = call <2 x i1> @llvm.is.fpclass.v2f64(<2 x double> [[A]], i32 783)
+; CHECK-NEXT:    ret <2 x i1> [[CMP]]
+;
+  %i64 = bitcast <2 x double> %a to <2 x i64>
+  %and = and <2 x i64> %i64, splat(i64 9218868437227405312)
+  %cmp = icmp ne <2 x i64> %and, zeroinitializer
+  ret <2 x i1> %cmp
+}
+
+define i1 @f32_fcsubnormal_fczero_no_implicit_fp(float %a) #0 {
+; CHECK-LABEL: define i1 @f32_fcsubnormal_fczero_no_implicit_fp(
+; CHECK-SAME: float [[A:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[I32:%.*]] = bitcast float [[A]] to i32
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[I32]], 2139095040
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[AND]], 0
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %i32 = bitcast float %a to i32
+  %and = and i32 %i32, 2139095040
+  %cmp = icmp eq i32 %and, 0
+  ret i1 %cmp
+}
+
+define i1 @f32_fcsubnormal_fczero_invalid_constant1(float %a) {
+; CHECK-LABEL: define i1 @f32_fcsubnormal_fczero_invalid_constant1(
+; CHECK-SAME: float [[A:%.*]]) {
+; CHECK-NEXT:    [[I32:%.*]] = bitcast float [[A]] to i32
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[I32]], 2139095039
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[AND]], 0
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %i32 = bitcast float %a to i32
+  %and = and i32 %i32, 2139095039
+  %cmp = icmp eq i32 %and, 0
+  ret i1 %cmp
+}
+
+define i1 @f32_fcsubnormal_fczero_invalid_constant2(float %a) {
+; CHECK-LABEL: define i1 @f32_fcsubnormal_fczero_invalid_constant2(
+; CHECK-SAME: float [[A:%.*]]) {
+; CHECK-NEXT:    [[I32:%.*]] = bitcast float [[A]] to i32
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[I32]], 2139095040
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[AND]], 2130706432
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %i32 = bitcast float %a to i32
+  %and = and i32 %i32, 2139095040
+  %cmp = icmp eq i32 %and, 2130706432
+  ret i1 %cmp
+}
+
+define i1 @ppc128_fcsubnormal_fczero(ppc_fp128 %a) {
+; CHECK-LABEL: define i1 @ppc128_fcsubnormal_fczero(
+; CHECK-SAME: ppc_fp128 [[A:%.*]]) {
+; CHECK-NEXT:    [[I128:%.*]] = bitcast ppc_fp128 [[A]] to i128
+; CHECK-NEXT:    [[AND:%.*]] = and i128 [[I128]], 170058106710732674489630815774616584192
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i128 [[AND]], 0
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %i128 = bitcast ppc_fp128 %a to i128
+  %and = and i128 %i128, 170058106710732674489630815774616584192
+  %cmp = icmp eq i128 %and, 0
+  ret i1 %cmp
+}
+
+define i1 @f32_fcsubnormal_fczero_multiuse1(float %a) {
+; CHECK-LABEL: define i1 @f32_fcsubnormal_fczero_multiuse1(
+; CHECK-SAME: float [[A:%.*]]) {
+; CHECK-NEXT:    [[I32:%.*]] = bitcast float [[A]] to i32
+; CHECK-NEXT:    call void @usei32(i32 [[I32]])
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[I32]], 2139095040
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[AND]], 0
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %i32 = bitcast float %a to i32
+  call void @usei32(i32 %i32)
+  %and = and i32 %i32, 2139095040
+  %cmp = icmp eq i32 %and, 0
+  ret i1 %cmp
+}
+
+define i1 @f32_fcsubnormal_fczero_multiuse2(float %a) {
+; CHECK-LABEL: define i1 @f32_fcsubnormal_fczero_multiuse2(
+; CHECK-SAME: float [[A:%.*]]) {
+; CHECK-NEXT:    [[I32:%.*]] = bitcast float [[A]] to i32
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[I32]], 2139095040
+; CHECK-NEXT:    call void @usei32(i32 [[AND]])
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[AND]], 0
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %i32 = bitcast float %a to i32
+  %and = and i32 %i32, 2139095040
+  call void @usei32(i32 %and)
+  %cmp = icmp eq i32 %and, 0
+  ret i1 %cmp
+}
+
 define i1 @f32_fcposinf_noimplicitfloat(float %a) #0 {
 ; CHECK-LABEL: define i1 @f32_fcposinf_noimplicitfloat(
 ; CHECK-SAME: float [[A:%.*]]) #[[ATTR1]] {
diff --git a/llvm/test/Transforms/InstCombine/fpclass-from-dom-cond.ll b/llvm/test/Transforms/InstCombine/fpclass-from-dom-cond.ll
index 78329faf34172..934852d1ca8bc 100644
--- a/llvm/test/Transforms/InstCombine/fpclass-from-dom-cond.ll
+++ b/llvm/test/Transforms/InstCombine/fpclass-from-dom-cond.ll
@@ -518,3 +518,31 @@ if.else:
 if.end:
   ret i1 false
 }
+
+define i1 @test_inv_and(float %x, i1 %cond2) {
+; CHECK-LABEL: define i1 @test_inv_and(
+; CHECK-SAME: float [[X:%.*]], i1 [[COND2:%.*]]) {
+; CHECK-NEXT:    [[COND:%.*]] = fcmp oge float [[X]], -1.000000e+00
+; CHECK-NEXT:    call void @use(i1 [[COND]])
+; CHECK-NEXT:    [[NOT:%.*]] = xor i1 [[COND]], true
+; CHECK-NEXT:    [[AND:%.*]] = and i1 [[COND2]], [[NOT]]
+; CHECK-NEXT:    br i1 [[AND]], label [[IF_THEN:%.*]], label [[IF_ELSE:%.*]]
+; CHECK:       if.then:
+; CHECK-NEXT:    ret i1 false
+; CHECK:       if.else:
+; CHECK-NEXT:    ret i1 false
+;
+  %cond = fcmp oge float %x, -1.0
+  %neg = fneg float %x
+  call void @use(i1 %cond)
+  %not = xor i1 %cond, true
+  %and = and i1 %not, %cond2
+  br i1 %and, label %if.then, label %if.else
+if.then:
+  %ret1 = fcmp oeq float %neg, 0xFFF0000000000000
+  ret i1 %ret1
+if.else:
+  ret i1 false
+}
+
+declare void @use(i1)
diff --git a/llvm/test/Transforms/InstCombine/free-inversion.ll b/llvm/test/Transforms/InstCombine/free-inversion.ll
index ebb9310ee0a78..4b69a5e77b4ce 100644
--- a/llvm/test/Transforms/InstCombine/free-inversion.ll
+++ b/llvm/test/Transforms/InstCombine/free-inversion.ll
@@ -740,3 +740,23 @@ exit:
 b4:
   ret i1 true
 }
+
+define i64 @PR71390(i64 %v) {
+; CHECK-LABEL: @PR71390(
+; CHECK-NEXT:    [[AND:%.*]] = and i64 [[V:%.*]], 9187201950435737471
+; CHECK-NEXT:    [[TMP1:%.*]] = sub nuw i64 -72340172838076674, [[AND]]
+; CHECK-NEXT:    [[TMP2:%.*]] = add nuw i64 [[AND]], 6872316419617283935
+; CHECK-NEXT:    [[OR_NOT:%.*]] = and i64 [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[NOT5:%.*]] = or i64 [[V]], [[OR_NOT]]
+; CHECK-NEXT:    ret i64 [[NOT5]]
+;
+  %and = and i64 %v, 9187201950435737471
+  %add = add nuw i64 %and, 72340172838076673
+  %add2 = add nuw i64 %and, 6872316419617283935
+  %not = xor i64 %v, -1
+  %not3 = xor i64 %add2, -1
+  %or = or i64 %add, %not3
+  %and4 = and i64 %not, %or
+  %not5 = xor i64 %and4, -1
+  ret i64 %not5
+}
diff --git a/llvm/test/Transforms/InstCombine/fsh.ll b/llvm/test/Transforms/InstCombine/fsh.ll
index 236c69e7a5bcb..3ff4f9a2abf33 100644
--- a/llvm/test/Transforms/InstCombine/fsh.ll
+++ b/llvm/test/Transforms/InstCombine/fsh.ll
@@ -1068,3 +1068,19 @@ entry:
   %res = call <2 x i31> @llvm.fshl.v2i31(<2 x i31> %x, <2 x i31> zeroinitializer, <2 x i31>  %y)
   ret <2 x i31>  %res
 }
+
+;; Issue #124387 Range attribute no longer holds after operands changed.
+define i8 @fshl_range_trunc(i1 %x) {
+; CHECK-LABEL: @fshl_range_trunc(
+; CHECK-NEXT:    [[ZEXT:%.*]] = zext i1 [[X:%.*]] to i32
+; CHECK-NEXT:    [[OR:%.*]] = or disjoint i32 [[ZEXT]], 126
+; CHECK-NEXT:    [[FSHL:%.*]] = call i32 @llvm.fshl.i32(i32 [[OR]], i32 -2, i32 1)
+; CHECK-NEXT:    [[TR:%.*]] = trunc nuw i32 [[FSHL]] to i8
+; CHECK-NEXT:    ret i8 [[TR]]
+;
+  %zext = zext i1 %x to i32
+  %or = or disjoint i32 %zext, -2
+  %fshl = call range(i32 -4, 2) i32 @llvm.fshl.i32(i32 %or, i32 %or, i32 1)
+  %tr = trunc nsw i32 %fshl to i8
+  ret i8 %tr
+}
diff --git a/llvm/test/Transforms/InstCombine/icmp-of-xor-x.ll b/llvm/test/Transforms/InstCombine/icmp-of-xor-x.ll
index 0635bb31c44fc..2260f77ea5f44 100644
--- a/llvm/test/Transforms/InstCombine/icmp-of-xor-x.ll
+++ b/llvm/test/Transforms/InstCombine/icmp-of-xor-x.ll
@@ -5,6 +5,349 @@ declare void @llvm.assume(i1)
 declare void @barrier()
 declare void @use.i8(i8)
 
+; Test case of X comp X^Neg_C, which have Transform to SLT.
+; X s< X^Neg_C  -->  X s< 0
+define i1 @src_slt(i8 %x) {
+; CHECK-LABEL: @src_slt(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i8 [[X:%.*]], 0
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %not = xor i8 %x, -1
+  %cmp = icmp slt i8 %x, %not
+  ret i1 %cmp
+}
+
+define <2 x i1> @src_slt_vec(<2 x i8> %x) {
+; CHECK-LABEL: @src_slt_vec(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt <2 x i8> [[X:%.*]], zeroinitializer
+; CHECK-NEXT:    ret <2 x i1> [[CMP]]
+;
+  %not = xor <2 x i8> %x, <i8 -1, i8 -1>
+  %cmp = icmp slt <2 x i8> %x, %not
+  ret <2 x i1> %cmp
+}
+
+; X s<= X^Neg_C  -->  X s< 0
+define i1 @src_sle(i8 %x) {
+; CHECK-LABEL: @src_sle(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i8 [[X:%.*]], 0
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %not = xor i8 %x, -1
+  %cmp = icmp sle i8 %x, %not
+  ret i1 %cmp
+}
+
+define <2 x i1> @src_sle_vec(<2 x i8> %x) {
+; CHECK-LABEL: @src_sle_vec(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt <2 x i8> [[X:%.*]], zeroinitializer
+; CHECK-NEXT:    ret <2 x i1> [[CMP]]
+;
+  %not = xor <2 x i8> %x, <i8 -1, i8 -1>
+  %cmp = icmp sle <2 x i8> %x, %not
+  ret <2 x i1> %cmp
+}
+
+; X u> X^Neg_C  -->  X s< 0
+define i1 @src_ugt(i8 %x) {
+; CHECK-LABEL: @src_ugt(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i8 [[X:%.*]], 0
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %not = xor i8 %x, -1
+  %cmp = icmp ugt i8 %x, %not
+  ret i1 %cmp
+}
+
+define <2 x i1> @src_ugt_vec(<2 x i8> %x) {
+; CHECK-LABEL: @src_ugt_vec(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt <2 x i8> [[X:%.*]], zeroinitializer
+; CHECK-NEXT:    ret <2 x i1> [[CMP]]
+;
+  %not = xor <2 x i8> %x, <i8 -1, i8 -1>
+  %cmp = icmp ugt <2 x i8> %x, %not
+  ret <2 x i1> %cmp
+}
+
+; X u>= X^Neg_C  -->  X s< 0
+define i1 @src_uge(i8 %x) {
+; CHECK-LABEL: @src_uge(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i8 [[X:%.*]], 0
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %not = xor i8 %x, -1
+  %cmp = icmp uge i8 %x, %not
+  ret i1 %cmp
+}
+
+define <2 x i1> @src_uge_vec(<2 x i8> %x) {
+; CHECK-LABEL: @src_uge_vec(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt <2 x i8> [[X:%.*]], zeroinitializer
+; CHECK-NEXT:    ret <2 x i1> [[CMP]]
+;
+  %not = xor <2 x i8> %x, <i8 -1, i8 -1>
+  %cmp = icmp uge <2 x i8> %x, %not
+  ret <2 x i1> %cmp
+}
+
+define <2 x i1> @src_uge_vec_min(<2 x i8> %x) {
+; CHECK-LABEL: @src_uge_vec_min(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt <2 x i8> [[X:%.*]], zeroinitializer
+; CHECK-NEXT:    ret <2 x i1> [[CMP]]
+;
+  %not = xor <2 x i8> %x, <i8 -128, i8 -128>
+  %cmp = icmp uge <2 x i8> %x, %not
+  ret <2 x i1> %cmp
+}
+
+define <2 x i1> @src_uge_vec_128(<2 x i128> %x) {
+; CHECK-LABEL: @src_uge_vec_128(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt <2 x i128> [[X:%.*]], zeroinitializer
+; CHECK-NEXT:    ret <2 x i1> [[CMP]]
+;
+  %not = xor <2 x i128> %x, <i128 -170141183460469231731687303715884105728, i128 -170141183460469231731687303715884105728>
+  %cmp = icmp uge <2 x i128> %x, %not
+  ret <2 x i1> %cmp
+}
+
+; Test case of X comp X^Neg_C, which have Transform to SGT.
+; X s> X^Neg_C  -->  X s> -1
+define i1 @src_sgt(i8 %x) {
+; CHECK-LABEL: @src_sgt(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i8 [[X:%.*]], -1
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %not = xor i8 %x, -1
+  %cmp = icmp sgt i8 %x, %not
+  ret i1 %cmp
+}
+
+define <2 x i1> @src_sgt_vec(<2 x i8> %x) {
+; CHECK-LABEL: @src_sgt_vec(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt <2 x i8> [[X:%.*]], splat (i8 -1)
+; CHECK-NEXT:    ret <2 x i1> [[CMP]]
+;
+  %not = xor <2 x i8> %x, <i8 -1, i8 -1>
+  %cmp = icmp sgt <2 x i8> %x, %not
+  ret <2 x i1> %cmp
+}
+
+; X s>= X^Neg_C  -->  X s> -1
+define i1 @src_sge(i8 %x) {
+; CHECK-LABEL: @src_sge(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i8 [[X:%.*]], -1
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %not = xor i8 %x, -1
+  %cmp = icmp sge i8 %x, %not
+  ret i1 %cmp
+}
+
+define <2 x i1> @src_sge_vec(<2 x i8> %x) {
+; CHECK-LABEL: @src_sge_vec(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt <2 x i8> [[X:%.*]], splat (i8 -1)
+; CHECK-NEXT:    ret <2 x i1> [[CMP]]
+;
+  %not = xor <2 x i8> %x, <i8 -1, i8 -1>
+  %cmp = icmp sge <2 x i8> %x, %not
+  ret <2 x i1> %cmp
+}
+
+; X u< X^Neg_C  -->  X s> -1
+define i1 @src_ult(i8 %x) {
+; CHECK-LABEL: @src_ult(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i8 [[X:%.*]], -1
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %not = xor i8 %x, -1
+  %cmp = icmp ult i8 %x, %not
+  ret i1 %cmp
+}
+
+define <2 x i1> @src_ult_vec(<2 x i8> %x) {
+; CHECK-LABEL: @src_ult_vec(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt <2 x i8> [[X:%.*]], splat (i8 -1)
+; CHECK-NEXT:    ret <2 x i1> [[CMP]]
+;
+  %not = xor <2 x i8> %x, <i8 -1, i8 -1>
+  %cmp = icmp ult <2 x i8> %x, %not
+  ret <2 x i1> %cmp
+}
+
+; X u<= X^Neg_C  -->  X s> -1
+define i1 @src_ule(i8 %x) {
+; CHECK-LABEL: @src_ule(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i8 [[X:%.*]], -1
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %not = xor i8 %x, -1
+  %cmp = icmp ule i8 %x, %not
+  ret i1 %cmp
+}
+
+define <2 x i1> @src_ule_vec(<2 x i8> %x) {
+; CHECK-LABEL: @src_ule_vec(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt <2 x i8> [[X:%.*]], splat (i8 -1)
+; CHECK-NEXT:    ret <2 x i1> [[CMP]]
+;
+  %not = xor <2 x i8> %x, <i8 -1, i8 -1>
+  %cmp = icmp ule <2 x i8> %x, %not
+  ret <2 x i1> %cmp
+}
+
+define <2 x i1> @src_ule_vec_min(<2 x i8> %x) {
+; CHECK-LABEL: @src_ule_vec_min(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt <2 x i8> [[X:%.*]], splat (i8 -1)
+; CHECK-NEXT:    ret <2 x i1> [[CMP]]
+;
+  %not = xor <2 x i8> %x, <i8 -128, i8 -128>
+  %cmp = icmp ule <2 x i8> %x, %not
+  ret <2 x i1> %cmp
+}
+
+define <2 x i1> @src_ule_vec_128(<2 x i128> %x) {
+; CHECK-LABEL: @src_ule_vec_128(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt <2 x i128> [[X:%.*]], splat (i128 -1)
+; CHECK-NEXT:    ret <2 x i1> [[CMP]]
+;
+  %not = xor <2 x i128> %x, <i128 -170141183460469231731687303715884105728, i128 -170141183460469231731687303715884105728>
+  %cmp = icmp ule <2 x i128> %x, %not
+  ret <2 x i1> %cmp
+}
+
+; X comp X^Neg_C tests. negative
+; X comp Y
+define i1 @src_sle_xny(i8 %x, i8 %y) {
+; CHECK-LABEL: @src_sle_xny(
+; CHECK-NEXT:    [[Y_NOT:%.*]] = xor i8 [[Y:%.*]], -1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sle i8 [[X:%.*]], [[Y_NOT]]
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %y.not = xor i8 %y, -1
+  %cmp = icmp sle i8 %x, %y.not
+  ret i1 %cmp
+}
+define i1 @src_sle_nyx(i8 %x, i8 %y) {
+; CHECK-LABEL: @src_sle_nyx(
+; CHECK-NEXT:    [[Y_NOT:%.*]] = xor i8 [[Y:%.*]], -1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sge i8 [[X:%.*]], [[Y_NOT]]
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %y.not = xor i8 %y, -1
+  %cmp = icmp sle i8 %y.not, %x
+  ret i1 %cmp
+}
+define i1 @src_sge_xny(i8 %x, i8 %y) {
+; CHECK-LABEL: @src_sge_xny(
+; CHECK-NEXT:    [[Y_NOT:%.*]] = xor i8 [[Y:%.*]], -1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sge i8 [[X:%.*]], [[Y_NOT]]
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %y.not = xor i8 %y, -1
+  %cmp = icmp sge i8 %x, %y.not
+  ret i1 %cmp
+}
+define i1 @src_sge_nyx(i8 %x, i8 %y) {
+; CHECK-LABEL: @src_sge_nyx(
+; CHECK-NEXT:    [[Y_NOT:%.*]] = xor i8 [[Y:%.*]], -1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sle i8 [[X:%.*]], [[Y_NOT]]
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %y.not = xor i8 %y, -1
+  %cmp = icmp sge i8 %y.not, %x
+  ret i1 %cmp
+}
+define i1 @src_ule_xny(i8 %x, i8 %y) {
+; CHECK-LABEL: @src_ule_xny(
+; CHECK-NEXT:    [[Y_NOT:%.*]] = xor i8 [[Y:%.*]], -1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ule i8 [[X:%.*]], [[Y_NOT]]
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %y.not = xor i8 %y, -1
+  %cmp = icmp ule i8 %x, %y.not
+  ret i1 %cmp
+}
+define i1 @src_ule_nyx(i8 %x, i8 %y) {
+; CHECK-LABEL: @src_ule_nyx(
+; CHECK-NEXT:    [[Y_NOT:%.*]] = xor i8 [[Y:%.*]], -1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp uge i8 [[X:%.*]], [[Y_NOT]]
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %y.not = xor i8 %y, -1
+  %cmp = icmp ule i8 %y.not, %x
+  ret i1 %cmp
+}
+define i1 @src_uge_xny(i8 %x, i8 %y) {
+; CHECK-LABEL: @src_uge_xny(
+; CHECK-NEXT:    [[Y_NOT:%.*]] = xor i8 [[Y:%.*]], -1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp uge i8 [[X:%.*]], [[Y_NOT]]
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %y.not = xor i8 %y, -1
+  %cmp = icmp uge i8 %x, %y.not
+  ret i1 %cmp
+}
+define i1 @src_uge_nyx(i8 %x, i8 %y) {
+; CHECK-LABEL: @src_uge_nyx(
+; CHECK-NEXT:    [[Y_NOT:%.*]] = xor i8 [[Y:%.*]], -1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ule i8 [[X:%.*]], [[Y_NOT]]
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %y.not = xor i8 %y, -1
+  %cmp = icmp uge i8 %y.not, %x
+  ret i1 %cmp
+}
+
+; X comp X^Neg_C tests. negative
+; (X+1) comp X^Neg_C
+define i1 @src_sle_incx_nx(i8 %x) {
+; CHECK-LABEL: @src_sle_incx_nx(
+; CHECK-NEXT:    [[TMP1:%.*]] = sub i8 -2, [[X:%.*]]
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sge i8 [[TMP1]], [[X]]
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %nx = xor i8 %x, -1
+  %inc.x = add i8 %x, 1
+  %cmp = icmp sle i8 %inc.x, %nx
+  ret i1 %cmp
+}
+; (X-1) comp X^Neg_C
+define i1 @src_sle_decx_nx(i8 %x) {
+; CHECK-LABEL: @src_sle_decx_nx(
+; CHECK-NEXT:    [[TMP1:%.*]] = sub i8 0, [[X:%.*]]
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sle i8 [[X]], [[TMP1]]
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %nx = xor i8 %x, -1
+  %dec.x = add i8 %x, -1
+  %cmp = icmp sle i8 %dec.x, %nx
+  ret i1 %cmp
+}
+; X comp (X+1)^Neg_C
+define i1 @src_sle_x_nincx(i8 %x) {
+; CHECK-LABEL: @src_sle_x_nincx(
+; CHECK-NEXT:    [[NOT_INC_X:%.*]] = sub i8 -2, [[X:%.*]]
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sle i8 [[X]], [[NOT_INC_X]]
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %inc.x = add i8 %x, 1
+  %not.inc.x = xor i8 %inc.x, -1
+  %cmp = icmp sle i8 %x, %not.inc.x
+  ret i1 %cmp
+}
+; X comp (X-1)^Neg_C
+define i1 @src_sle_x_ndecx(i8 %x) {
+; CHECK-LABEL: @src_sle_x_ndecx(
+; CHECK-NEXT:    [[NOT_DEC_X:%.*]] = sub i8 0, [[X:%.*]]
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sle i8 [[X]], [[NOT_DEC_X]]
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %dec.x = add i8 %x, -1
+  %not.dec.x = xor i8 %dec.x, -1
+  %cmp = icmp sle i8 %x, %not.dec.x
+  ret i1 %cmp
+}
+
 ; test for (~x ^ y) < ~z
 define i1 @test_xor1(i8 %x, i8 %y, i8 %z) {
 ; CHECK-LABEL: @test_xor1(
diff --git a/llvm/test/Transforms/InstCombine/intrinsic-select.ll b/llvm/test/Transforms/InstCombine/intrinsic-select.ll
index 4ce2908a63078..a8117ce663a6d 100644
--- a/llvm/test/Transforms/InstCombine/intrinsic-select.ll
+++ b/llvm/test/Transforms/InstCombine/intrinsic-select.ll
@@ -2,6 +2,7 @@
 ; RUN: opt -passes=instcombine -S < %s | FileCheck %s
 
 declare void @use(i32)
+declare void @usef32(float)
 
 declare i32 @llvm.ctlz.i32(i32, i1)
 declare <3 x i17> @llvm.ctlz.v3i17(<3 x i17>, i1)
@@ -344,3 +345,29 @@ define double @test_fabs_select_fmf2(i1 %cond, double %a) {
   %fabs = call nnan ninf nsz double @llvm.fabs.f64(double %sel1)
   ret double %fabs
 }
+
+define float @test_fabs_select_multiuse(i1 %cond, float %x) {
+; CHECK-LABEL: @test_fabs_select_multiuse(
+; CHECK-NEXT:    [[SELECT:%.*]] = select i1 [[COND:%.*]], float [[X:%.*]], float 0x7FF0000000000000
+; CHECK-NEXT:    call void @usef32(float [[SELECT]])
+; CHECK-NEXT:    [[FABS:%.*]] = call float @llvm.fabs.f32(float [[SELECT]])
+; CHECK-NEXT:    ret float [[FABS]]
+;
+  %select = select i1 %cond, float %x, float 0x7FF0000000000000
+  call void @usef32(float %select)
+  %fabs = call float @llvm.fabs.f32(float %select)
+  ret float %fabs
+}
+
+define float @test_fabs_select_multiuse_both_constant(i1 %cond, float %x) {
+; CHECK-LABEL: @test_fabs_select_multiuse_both_constant(
+; CHECK-NEXT:    [[SELECT:%.*]] = select i1 [[COND:%.*]], float -1.000000e+00, float -2.000000e+00
+; CHECK-NEXT:    call void @usef32(float [[SELECT]])
+; CHECK-NEXT:    [[FABS:%.*]] = select i1 [[COND]], float 1.000000e+00, float 2.000000e+00
+; CHECK-NEXT:    ret float [[FABS]]
+;
+  %select = select i1 %cond, float -1.0, float -2.0
+  call void @usef32(float %select)
+  %fabs = call float @llvm.fabs.f32(float %select)
+  ret float %fabs
+}
diff --git a/llvm/test/Transforms/InstCombine/known-bits.ll b/llvm/test/Transforms/InstCombine/known-bits.ll
index 9467f507cd630..9a9fec694ff0e 100644
--- a/llvm/test/Transforms/InstCombine/known-bits.ll
+++ b/llvm/test/Transforms/InstCombine/known-bits.ll
@@ -480,6 +480,55 @@ if.else:
   ret i64 13
 }
 
+define i64 @test_icmp_trunc_nuw(i64 %a) {
+; CHECK-LABEL: @test_icmp_trunc_nuw(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CAST:%.*]] = trunc nuw i64 [[A:%.*]] to i32
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i32 [[CAST]], 0
+; CHECK-NEXT:    br i1 [[CMP]], label [[IF_THEN:%.*]], label [[IF_ELSE:%.*]]
+; CHECK:       if.then:
+; CHECK-NEXT:    ret i64 [[A]]
+; CHECK:       if.else:
+; CHECK-NEXT:    ret i64 0
+;
+entry:
+  %cast = trunc nuw i64 %a to i32
+  %cmp = icmp sgt i32 %cast, 0
+  br i1 %cmp, label %if.then, label %if.else
+
+if.then:
+  %b = and i64 %a, 2147483647
+  ret i64 %b
+
+if.else:
+  ret i64 0
+}
+
+define i64 @test_icmp_trunc_no_nuw(i64 %a) {
+; CHECK-LABEL: @test_icmp_trunc_no_nuw(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CAST:%.*]] = trunc i64 [[A:%.*]] to i32
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i32 [[CAST]], 0
+; CHECK-NEXT:    br i1 [[CMP]], label [[IF_THEN:%.*]], label [[IF_ELSE:%.*]]
+; CHECK:       if.then:
+; CHECK-NEXT:    [[B:%.*]] = and i64 [[A]], 2147483647
+; CHECK-NEXT:    ret i64 [[B]]
+; CHECK:       if.else:
+; CHECK-NEXT:    ret i64 0
+;
+entry:
+  %cast = trunc i64 %a to i32
+  %cmp = icmp sgt i32 %cast, 0
+  br i1 %cmp, label %if.then, label %if.else
+
+if.then:
+  %b = and i64 %a, 2147483647
+  ret i64 %b
+
+if.else:
+  ret i64 0
+}
+
 define i1 @test_icmp_or_distjoint(i8 %n, i1 %other) {
 ; CHECK-LABEL: @test_icmp_or_distjoint(
 ; CHECK-NEXT:  entry:
@@ -1609,17 +1658,13 @@ if.else:
   ret i16 0
 }
 
-; TODO: %cmp always evaluates to false
-
 define i1 @test_simplify_icmp2(double %x) {
 ; CHECK-LABEL: @test_simplify_icmp2(
 ; CHECK-NEXT:    [[ABS:%.*]] = tail call double @llvm.fabs.f64(double [[X:%.*]])
 ; CHECK-NEXT:    [[COND:%.*]] = fcmp oeq double [[ABS]], 0x7FF0000000000000
 ; CHECK-NEXT:    br i1 [[COND]], label [[IF_THEN:%.*]], label [[IF_ELSE:%.*]]
 ; CHECK:       if.then:
-; CHECK-NEXT:    [[CAST:%.*]] = bitcast double [[X]] to i64
-; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i64 [[CAST]], 3458764513820540928
-; CHECK-NEXT:    ret i1 [[CMP]]
+; CHECK-NEXT:    ret i1 false
 ; CHECK:       if.else:
 ; CHECK-NEXT:    ret i1 false
 ;
@@ -2116,6 +2161,228 @@ define i1 @mul_nuw_nsw_nonneg_cant_be_one_commuted(i8 %x, i8 %y) {
   ret i1 %cmp
 }
 
+define i8 @test_trunc_and_1(i8 %a) {
+; CHECK-LABEL: @test_trunc_and_1(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CAST:%.*]] = trunc i8 [[A:%.*]] to i1
+; CHECK-NEXT:    br i1 [[CAST]], label [[IF_THEN:%.*]], label [[IF_ELSE:%.*]]
+; CHECK:       if.then:
+; CHECK-NEXT:    ret i8 1
+; CHECK:       if.else:
+; CHECK-NEXT:    ret i8 0
+;
+entry:
+  %cast = trunc i8 %a to i1
+  br i1 %cast, label %if.then, label %if.else
+
+if.then:
+  %b = and i8 %a, 1
+  ret i8 %b
+
+if.else:
+  %c = and i8 %a, 1
+  ret i8 %c
+}
+
+define i8 @test_not_trunc_and_1(i8 %a) {
+; CHECK-LABEL: @test_not_trunc_and_1(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CAST:%.*]] = trunc i8 [[A:%.*]] to i1
+; CHECK-NEXT:    br i1 [[CAST]], label [[IF_ELSE:%.*]], label [[IF_THEN:%.*]]
+; CHECK:       if.then:
+; CHECK-NEXT:    ret i8 0
+; CHECK:       if.else:
+; CHECK-NEXT:    ret i8 1
+;
+entry:
+  %cast = trunc i8 %a to i1
+  %not = xor i1 %cast, true
+  br i1 %not, label %if.then, label %if.else
+
+if.then:
+  %b = and i8 %a, 1
+  ret i8 %b
+
+if.else:
+  %c = and i8 %a, 1
+  ret i8 %c
+}
+
+define i8 @neg_test_trunc_or_2(i8 %a) {
+; CHECK-LABEL: @neg_test_trunc_or_2(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CAST:%.*]] = trunc i8 [[A:%.*]] to i1
+; CHECK-NEXT:    br i1 [[CAST]], label [[IF_THEN:%.*]], label [[IF_ELSE:%.*]]
+; CHECK:       if.then:
+; CHECK-NEXT:    [[B:%.*]] = or i8 [[A]], 2
+; CHECK-NEXT:    ret i8 [[B]]
+; CHECK:       if.else:
+; CHECK-NEXT:    [[C:%.*]] = or i8 [[A]], 2
+; CHECK-NEXT:    ret i8 [[C]]
+;
+entry:
+  %cast = trunc i8 %a to i1
+  br i1 %cast, label %if.then, label %if.else
+
+if.then:
+  %b = or i8 %a, 2
+  ret i8 %b
+
+if.else:
+  %c = or i8 %a, 2
+  ret i8 %c
+}
+
+define i8 @test_trunc_nuw_and_1(i8 %a) {
+; CHECK-LABEL: @test_trunc_nuw_and_1(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CAST:%.*]] = trunc nuw i8 [[A:%.*]] to i1
+; CHECK-NEXT:    br i1 [[CAST]], label [[IF_ELSE:%.*]], label [[IF_THEN:%.*]]
+; CHECK:       if.then:
+; CHECK-NEXT:    ret i8 0
+; CHECK:       if.else:
+; CHECK-NEXT:    ret i8 1
+;
+entry:
+  %cast = trunc nuw i8 %a to i1
+  br i1 %cast, label %if.else, label %if.then
+
+if.then:
+  %b = and i8 %a, 1
+  ret i8 %b
+
+if.else:
+  %c = and i8 %a, 1
+  ret i8 %c
+}
+
+define i8 @test_trunc_nuw_or_2(i8 %a) {
+; CHECK-LABEL: @test_trunc_nuw_or_2(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CAST:%.*]] = trunc nuw i8 [[A:%.*]] to i1
+; CHECK-NEXT:    br i1 [[CAST]], label [[IF_ELSE:%.*]], label [[IF_THEN:%.*]]
+; CHECK:       if.then:
+; CHECK-NEXT:    ret i8 2
+; CHECK:       if.else:
+; CHECK-NEXT:    ret i8 3
+;
+entry:
+  %cast = trunc nuw i8 %a to i1
+  br i1 %cast, label %if.else, label %if.then
+
+if.then:
+  %b = or i8 %a, 2
+  ret i8 %b
+
+if.else:
+  %c = or i8 %a, 2
+  ret i8 %c
+}
+
+define i8 @test_not_trunc_nuw_and_1(i8 %a) {
+; CHECK-LABEL: @test_not_trunc_nuw_and_1(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CAST:%.*]] = trunc nuw i8 [[A:%.*]] to i1
+; CHECK-NEXT:    br i1 [[CAST]], label [[IF_ELSE:%.*]], label [[IF_THEN:%.*]]
+; CHECK:       if.then:
+; CHECK-NEXT:    ret i8 0
+; CHECK:       if.else:
+; CHECK-NEXT:    ret i8 1
+;
+entry:
+  %cast = trunc nuw i8 %a to i1
+  %not = xor i1 %cast, true
+  br i1 %not, label %if.then, label %if.else
+
+if.then:
+  %b = and i8 %a, 1
+  ret i8 %b
+
+if.else:
+  %c = and i8 %a, 1
+  ret i8 %c
+}
+
+define i8 @test_trunc_cond_and(i8 %x, i1 %c) {
+; CHECK-LABEL: @test_trunc_cond_and(
+; CHECK-NEXT:    [[CMP:%.*]] = trunc i8 [[X:%.*]] to i1
+; CHECK-NEXT:    [[COND:%.*]] = and i1 [[C:%.*]], [[CMP]]
+; CHECK-NEXT:    br i1 [[COND]], label [[IF:%.*]], label [[EXIT:%.*]]
+; CHECK:       if:
+; CHECK-NEXT:    ret i8 -1
+; CHECK:       exit:
+; CHECK-NEXT:    [[OR2:%.*]] = or i8 [[X]], -2
+; CHECK-NEXT:    ret i8 [[OR2]]
+;
+  %cmp = trunc i8 %x to i1
+  %cond = and i1 %cmp, %c
+  br i1 %cond, label %if, label %exit
+
+if:
+  %or1 = or i8 %x, -2
+  ret i8 %or1
+
+exit:
+  %or2 = or i8 %x, -2
+  ret i8 %or2
+}
+
+define i8 @test_not_trunc_cond_and(i8 %x, i1 %c) {
+; CHECK-LABEL: @test_not_trunc_cond_and(
+; CHECK-NEXT:    [[CMP:%.*]] = trunc i8 [[X:%.*]] to i1
+; CHECK-NEXT:    [[NOT:%.*]] = xor i1 [[CMP]], true
+; CHECK-NEXT:    [[COND:%.*]] = and i1 [[C:%.*]], [[NOT]]
+; CHECK-NEXT:    br i1 [[COND]], label [[IF:%.*]], label [[EXIT:%.*]]
+; CHECK:       if:
+; CHECK-NEXT:    ret i8 -2
+; CHECK:       exit:
+; CHECK-NEXT:    [[OR2:%.*]] = or i8 [[X]], -2
+; CHECK-NEXT:    ret i8 [[OR2]]
+;
+  %cmp = trunc i8 %x to i1
+  %not = xor i1 %cmp, true
+  %cond = and i1 %not, %c
+  br i1 %cond, label %if, label %exit
+
+if:
+  %or1 = or i8 %x, -2
+  ret i8 %or1
+
+exit:
+  %or2 = or i8 %x, -2
+  ret i8 %or2
+}
+
+define i8 @test_inv_cond_and(i8 %x, i1 %c) {
+; CHECK-LABEL: @test_inv_cond_and(
+; CHECK-NEXT:    [[AND:%.*]] = and i8 [[X:%.*]], 3
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ne i8 [[AND]], 0
+; CHECK-NEXT:    call void @use(i1 [[CMP]])
+; CHECK-NEXT:    [[NOT:%.*]] = xor i1 [[CMP]], true
+; CHECK-NEXT:    [[COND:%.*]] = and i1 [[C:%.*]], [[NOT]]
+; CHECK-NEXT:    br i1 [[COND]], label [[IF:%.*]], label [[EXIT:%.*]]
+; CHECK:       if:
+; CHECK-NEXT:    ret i8 -4
+; CHECK:       exit:
+; CHECK-NEXT:    [[OR2:%.*]] = or i8 [[X]], -4
+; CHECK-NEXT:    ret i8 [[OR2]]
+;
+  %and = and i8 %x, 3
+  %cmp = icmp ne i8 %and, 0
+  call void @use(i1 %cmp)
+  %not = xor i1 %cmp, true
+  %cond = and i1 %not, %c
+  br i1 %cond, label %if, label %exit
+
+if:
+  %or1 = or i8 %x, -4
+  ret i8 %or1
+
+exit:
+  %or2 = or i8 %x, -4
+  ret i8 %or2
+}
+
 declare void @dummy()
 declare void @use(i1)
 declare void @sink(i8)
diff --git a/llvm/test/Transforms/InstCombine/memcpy-from-global.ll b/llvm/test/Transforms/InstCombine/memcpy-from-global.ll
index d28dd8f5758b8..9c9ba83425845 100644
--- a/llvm/test/Transforms/InstCombine/memcpy-from-global.ll
+++ b/llvm/test/Transforms/InstCombine/memcpy-from-global.ll
@@ -380,7 +380,7 @@ define void @volatile_memcpy() {
 ; and then forwarding it by readonly nocapture reference.
 define void @memcpy_to_nocapture_readonly() {
 ; CHECK-LABEL: @memcpy_to_nocapture_readonly(
-; CHECK-NEXT:    call void @bar(ptr nocapture nonnull readonly @H)
+; CHECK-NEXT:    call void @bar(ptr nonnull readonly captures(none) @H)
 ; CHECK-NEXT:    ret void
 ;
   %A = alloca %U, align 16
@@ -411,7 +411,7 @@ define void @memcpy_to_aliased_nocapture_readonly() {
 ; CHECK-LABEL: @memcpy_to_aliased_nocapture_readonly(
 ; CHECK-NEXT:    [[A:%.*]] = alloca [[U:%.*]], align 16
 ; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr noundef nonnull align 16 dereferenceable(20) [[A]], ptr noundef nonnull align 16 dereferenceable(20) @H, i64 20, i1 false)
-; CHECK-NEXT:    call void @two_params(ptr nocapture nonnull readonly [[A]], ptr nocapture nonnull [[A]])
+; CHECK-NEXT:    call void @two_params(ptr nonnull readonly captures(none) [[A]], ptr nonnull captures(none) [[A]])
 ; CHECK-NEXT:    ret void
 ;
   %A = alloca %U, align 16
diff --git a/llvm/test/Transforms/InstCombine/memset2.ll b/llvm/test/Transforms/InstCombine/memset2.ll
index 04a19d6c2f3d1..5fe5620d61166 100644
--- a/llvm/test/Transforms/InstCombine/memset2.ll
+++ b/llvm/test/Transforms/InstCombine/memset2.ll
@@ -7,7 +7,7 @@
 
 define i32 @test(ptr addrspace(1) nocapture %moves) {
 ; CHECK-LABEL: define i32 @test
-; CHECK-SAME: (ptr addrspace(1) nocapture [[MOVES:%.*]]) {
+; CHECK-SAME: (ptr addrspace(1) captures(none) [[MOVES:%.*]]) {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[GEP:%.*]] = getelementptr inbounds nuw i8, ptr addrspace(1) [[MOVES]], i64 26
 ; CHECK-NEXT:    store i64 0, ptr addrspace(1) [[GEP]], align 1
diff --git a/llvm/test/Transforms/InstCombine/memset_chk-1.ll b/llvm/test/Transforms/InstCombine/memset_chk-1.ll
index 44b549e400dd8..9020f174fb5b7 100644
--- a/llvm/test/Transforms/InstCombine/memset_chk-1.ll
+++ b/llvm/test/Transforms/InstCombine/memset_chk-1.ll
@@ -92,7 +92,7 @@ define i32 @test_rauw(ptr %a, ptr %b, ptr %c) {
 ; CHECK-NEXT:    [[CALL49:%.*]] = call i64 @strlen(ptr noundef nonnull dereferenceable(1) [[A:%.*]])
 ; CHECK-NEXT:    [[ADD180:%.*]] = add i64 [[CALL49]], 1
 ; CHECK-NEXT:    [[YO107:%.*]] = call i64 @llvm.objectsize.i64.p0(ptr [[B:%.*]], i1 false, i1 false, i1 false)
-; CHECK-NEXT:    [[CALL50:%.*]] = call ptr @__memmove_chk(ptr [[B]], ptr [[A]], i64 [[ADD180]], i64 [[YO107]])
+; CHECK-NEXT:    [[CALL50:%.*]] = call ptr @__memmove_chk(ptr [[B]], ptr nonnull [[A]], i64 [[ADD180]], i64 [[YO107]])
 ; CHECK-NEXT:    [[STRLEN:%.*]] = call i64 @strlen(ptr noundef nonnull dereferenceable(1) [[B]])
 ; CHECK-NEXT:    [[STRCHR1:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[STRLEN]]
 ; CHECK-NEXT:    [[D:%.*]] = load ptr, ptr [[C:%.*]], align 8
@@ -100,7 +100,7 @@ define i32 @test_rauw(ptr %a, ptr %b, ptr %c) {
 ; CHECK-NEXT:    [[SUB183:%.*]] = ptrtoint ptr [[B]] to i64
 ; CHECK-NEXT:    [[SUB184:%.*]] = sub i64 [[SUB182]], [[SUB183]]
 ; CHECK-NEXT:    [[ADD52_I_I:%.*]] = add nsw i64 [[SUB184]], 1
-; CHECK-NEXT:    call void @llvm.memset.p0.i64(ptr align 1 [[STRCHR1]], i8 0, i64 [[ADD52_I_I]], i1 false)
+; CHECK-NEXT:    call void @llvm.memset.p0.i64(ptr nonnull align 1 [[STRCHR1]], i8 0, i64 [[ADD52_I_I]], i1 false)
 ; CHECK-NEXT:    ret i32 4
 ;
 entry:
diff --git a/llvm/test/Transforms/InstCombine/opaque-ptr.ll b/llvm/test/Transforms/InstCombine/opaque-ptr.ll
index b05274658e812..be734243d14a1 100644
--- a/llvm/test/Transforms/InstCombine/opaque-ptr.ll
+++ b/llvm/test/Transforms/InstCombine/opaque-ptr.ll
@@ -467,6 +467,41 @@ define i1 @cmp_gep_same_base_same_type(ptr %ptr, i64 %idx1, i64 %idx2) {
   ret i1 %cmp
 }
 
+define i1 @cmp_gep_same_base_same_type_maywrap(ptr %ptr, i64 %idx1, i64 %idx2) {
+; CHECK-LABEL: @cmp_gep_same_base_same_type_maywrap(
+; CHECK-NEXT:    [[CMP_UNSHIFTED:%.*]] = xor i64 [[IDX1:%.*]], [[IDX2:%.*]]
+; CHECK-NEXT:    [[CMP_MASK:%.*]] = and i64 [[CMP_UNSHIFTED]], 4611686018427387903
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i64 [[CMP_MASK]], 0
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %gep1 = getelementptr i32, ptr %ptr, i64 %idx1
+  %gep2 = getelementptr i32, ptr %ptr, i64 %idx2
+  %cmp = icmp eq ptr %gep1, %gep2
+  ret i1 %cmp
+}
+
+define i1 @cmp_gep_same_base_same_type_nuw(ptr %ptr, i64 %idx1, i64 %idx2) {
+; CHECK-LABEL: @cmp_gep_same_base_same_type_nuw(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i64 [[IDX1:%.*]], [[IDX2:%.*]]
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %gep1 = getelementptr nuw i32, ptr %ptr, i64 %idx1
+  %gep2 = getelementptr nuw i32, ptr %ptr, i64 %idx2
+  %cmp = icmp eq ptr %gep1, %gep2
+  ret i1 %cmp
+}
+
+define i1 @cmp_gep_same_base_same_type_nusw(ptr %ptr, i64 %idx1, i64 %idx2) {
+; CHECK-LABEL: @cmp_gep_same_base_same_type_nusw(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i64 [[IDX1:%.*]], [[IDX2:%.*]]
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %gep1 = getelementptr nusw i32, ptr %ptr, i64 %idx1
+  %gep2 = getelementptr nusw i32, ptr %ptr, i64 %idx2
+  %cmp = icmp eq ptr %gep1, %gep2
+  ret i1 %cmp
+}
+
 define i1 @cmp_gep_same_base_different_type(ptr %ptr, i64 %idx1, i64 %idx2) {
 ; CHECK-LABEL: @cmp_gep_same_base_different_type(
 ; CHECK-NEXT:    [[GEP1_IDX:%.*]] = shl nsw i64 [[IDX1:%.*]], 2
diff --git a/llvm/test/Transforms/InstCombine/select-cmp.ll b/llvm/test/Transforms/InstCombine/select-cmp.ll
index f7505bd85f89e..b1bd7a0ecc8ac 100644
--- a/llvm/test/Transforms/InstCombine/select-cmp.ll
+++ b/llvm/test/Transforms/InstCombine/select-cmp.ll
@@ -23,6 +23,18 @@ define i1 @icmp_ne_common_op00(i1 %c, i6 %x, i6 %y, i6 %z) {
   ret i1 %r
 }
 
+define i1 @icmp_ne_samesign_common(i1 %c, i6 %x, i6 %y, i6 %z) {
+; CHECK-LABEL: @icmp_ne_samesign_common(
+; CHECK-NEXT:    [[R_V:%.*]] = select i1 [[C:%.*]], i6 [[Y:%.*]], i6 [[Z:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = icmp ne i6 [[X:%.*]], [[R_V]]
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %cmp1 = icmp samesign ne i6 %x, %y
+  %cmp2 = icmp ne i6 %x, %z
+  %r = select i1 %c, i1 %cmp1, i1 %cmp2
+  ret i1 %r
+}
+
 define i1 @icmp_ne_common_op01(i1 %c, i3 %x, i3 %y, i3 %z) {
 ; CHECK-LABEL: @icmp_ne_common_op01(
 ; CHECK-NEXT:    [[R_V:%.*]] = select i1 [[C:%.*]], i3 [[Y:%.*]], i3 [[Z:%.*]]
@@ -71,6 +83,18 @@ define i1 @icmp_eq_common_op00(i1 %c, i5 %x, i5 %y, i5 %z) {
   ret i1 %r
 }
 
+define i1 @icmp_eq_samesign_common(i1 %c, i5 %x, i5 %y, i5 %z) {
+; CHECK-LABEL: @icmp_eq_samesign_common(
+; CHECK-NEXT:    [[R_V:%.*]] = select i1 [[C:%.*]], i5 [[Y:%.*]], i5 [[Z:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = icmp eq i5 [[X:%.*]], [[R_V]]
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %cmp1 = icmp eq i5 %x, %y
+  %cmp2 = icmp samesign eq i5 %x, %z
+  %r = select i1 %c, i1 %cmp1, i1 %cmp2
+  ret i1 %r
+}
+
 define <5 x i1> @icmp_eq_common_op01(<5 x i1> %c, <5 x i7> %x, <5 x i7> %y, <5 x i7> %z) {
 ; CHECK-LABEL: @icmp_eq_common_op01(
 ; CHECK-NEXT:    [[R_V:%.*]] = select <5 x i1> [[C:%.*]], <5 x i7> [[Y:%.*]], <5 x i7> [[Z:%.*]]
@@ -134,6 +158,18 @@ define i1 @icmp_slt_common(i1 %c, i6 %x, i6 %y, i6 %z) {
   ret i1 %r
 }
 
+define i1 @icmp_slt_samesign_common(i1 %c, i6 %x, i6 %y, i6 %z) {
+; CHECK-LABEL: @icmp_slt_samesign_common(
+; CHECK-NEXT:    [[R_V:%.*]] = select i1 [[C:%.*]], i6 [[Y:%.*]], i6 [[Z:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = icmp slt i6 [[X:%.*]], [[R_V]]
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %cmp1 = icmp samesign ult i6 %x, %y
+  %cmp2 = icmp slt i6 %x, %z
+  %r = select i1 %c, i1 %cmp1, i1 %cmp2
+  ret i1 %r
+}
+
 define i1 @icmp_sgt_common(i1 %c, i6 %x, i6 %y, i6 %z) {
 ; CHECK-LABEL: @icmp_sgt_common(
 ; CHECK-NEXT:    [[R_V:%.*]] = select i1 [[C:%.*]], i6 [[Y:%.*]], i6 [[Z:%.*]]
@@ -146,6 +182,18 @@ define i1 @icmp_sgt_common(i1 %c, i6 %x, i6 %y, i6 %z) {
   ret i1 %r
 }
 
+define i1 @icmp_sgt_samesign_common(i1 %c, i6 %x, i6 %y, i6 %z) {
+; CHECK-LABEL: @icmp_sgt_samesign_common(
+; CHECK-NEXT:    [[R_V:%.*]] = select i1 [[C:%.*]], i6 [[Y:%.*]], i6 [[Z:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = icmp sgt i6 [[X:%.*]], [[R_V]]
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %cmp1 = icmp samesign ugt i6 %x, %y
+  %cmp2 = icmp sgt i6 %x, %z
+  %r = select i1 %c, i1 %cmp1, i1 %cmp2
+  ret i1 %r
+}
+
 define i1 @icmp_sle_common(i1 %c, i6 %x, i6 %y, i6 %z) {
 ; CHECK-LABEL: @icmp_sle_common(
 ; CHECK-NEXT:    [[R_V:%.*]] = select i1 [[C:%.*]], i6 [[Y:%.*]], i6 [[Z:%.*]]
@@ -158,6 +206,18 @@ define i1 @icmp_sle_common(i1 %c, i6 %x, i6 %y, i6 %z) {
   ret i1 %r
 }
 
+define i1 @icmp_sle_samesign_common(i1 %c, i6 %x, i6 %y, i6 %z) {
+; CHECK-LABEL: @icmp_sle_samesign_common(
+; CHECK-NEXT:    [[R_V:%.*]] = select i1 [[C:%.*]], i6 [[Y:%.*]], i6 [[Z:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = icmp sge i6 [[X:%.*]], [[R_V]]
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %cmp1 = icmp sle i6 %y, %x
+  %cmp2 = icmp samesign ule i6 %z, %x
+  %r = select i1 %c, i1 %cmp1, i1 %cmp2
+  ret i1 %r
+}
+
 define i1 @icmp_sge_common(i1 %c, i6 %x, i6 %y, i6 %z) {
 ; CHECK-LABEL: @icmp_sge_common(
 ; CHECK-NEXT:    [[R_V:%.*]] = select i1 [[C:%.*]], i6 [[Y:%.*]], i6 [[Z:%.*]]
@@ -170,6 +230,18 @@ define i1 @icmp_sge_common(i1 %c, i6 %x, i6 %y, i6 %z) {
   ret i1 %r
 }
 
+define i1 @icmp_sge_samesign_common(i1 %c, i6 %x, i6 %y, i6 %z) {
+; CHECK-LABEL: @icmp_sge_samesign_common(
+; CHECK-NEXT:    [[R_V:%.*]] = select i1 [[C:%.*]], i6 [[Y:%.*]], i6 [[Z:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = icmp sle i6 [[X:%.*]], [[R_V]]
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %cmp1 = icmp sge i6 %y, %x
+  %cmp2 = icmp samesign uge i6 %z, %x
+  %r = select i1 %c, i1 %cmp1, i1 %cmp2
+  ret i1 %r
+}
+
 define i1 @icmp_slt_sgt_common(i1 %c, i6 %x, i6 %y, i6 %z) {
 ; CHECK-LABEL: @icmp_slt_sgt_common(
 ; CHECK-NEXT:    [[R_V:%.*]] = select i1 [[C:%.*]], i6 [[Y:%.*]], i6 [[Z:%.*]]
@@ -182,6 +254,18 @@ define i1 @icmp_slt_sgt_common(i1 %c, i6 %x, i6 %y, i6 %z) {
   ret i1 %r
 }
 
+define i1 @icmp_slt_sgt_samesign_common(i1 %c, i6 %x, i6 %y, i6 %z) {
+; CHECK-LABEL: @icmp_slt_sgt_samesign_common(
+; CHECK-NEXT:    [[R_V:%.*]] = select i1 [[C:%.*]], i6 [[Y:%.*]], i6 [[Z:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = icmp slt i6 [[X:%.*]], [[R_V]]
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %cmp1 = icmp samesign ult i6 %x, %y
+  %cmp2 = icmp sgt i6 %z, %x
+  %r = select i1 %c, i1 %cmp1, i1 %cmp2
+  ret i1 %r
+}
+
 define i1 @icmp_sle_sge_common(i1 %c, i6 %x, i6 %y, i6 %z) {
 ; CHECK-LABEL: @icmp_sle_sge_common(
 ; CHECK-NEXT:    [[R_V:%.*]] = select i1 [[C:%.*]], i6 [[Y:%.*]], i6 [[Z:%.*]]
@@ -194,6 +278,18 @@ define i1 @icmp_sle_sge_common(i1 %c, i6 %x, i6 %y, i6 %z) {
   ret i1 %r
 }
 
+define i1 @icmp_sle_sge_samesign_common(i1 %c, i6 %x, i6 %y, i6 %z) {
+; CHECK-LABEL: @icmp_sle_sge_samesign_common(
+; CHECK-NEXT:    [[R_V:%.*]] = select i1 [[C:%.*]], i6 [[Y:%.*]], i6 [[Z:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = icmp sge i6 [[X:%.*]], [[R_V]]
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %cmp1 = icmp sle i6 %y, %x
+  %cmp2 = icmp samesign uge i6 %x, %z
+  %r = select i1 %c, i1 %cmp1, i1 %cmp2
+  ret i1 %r
+}
+
 define i1 @icmp_ult_common(i1 %c, i6 %x, i6 %y, i6 %z) {
 ; CHECK-LABEL: @icmp_ult_common(
 ; CHECK-NEXT:    [[R_V:%.*]] = select i1 [[C:%.*]], i6 [[Y:%.*]], i6 [[Z:%.*]]
@@ -700,5 +796,17 @@ define i1 @sel_icmp_cmp_and_no_simplify_comm(i1 %c, i32 %a1, i32 %a2, i8 %b) {
   ret i1 %cmp
 }
 
+define i1 @icmp_lt_slt(i1 %c, i32 %arg) {
+; CHECK-LABEL: @icmp_lt_slt(
+; CHECK-NEXT:    [[SELECT_V:%.*]] = select i1 [[C:%.*]], i32 131072, i32 0
+; CHECK-NEXT:    [[SELECT:%.*]] = icmp slt i32 [[ARG:%.*]], [[SELECT_V]]
+; CHECK-NEXT:    ret i1 [[SELECT]]
+;
+  %cmp1 = icmp samesign ult i32 %arg, 131072
+  %cmp2 = icmp slt i32 %arg, 0
+  %select = select i1 %c, i1 %cmp1, i1 %cmp2
+  ret i1 %select
+}
+
 declare void @use(i1)
 declare void @use.i8(i8)
diff --git a/llvm/test/Transforms/InstCombine/select-min-max.ll b/llvm/test/Transforms/InstCombine/select-min-max.ll
index f13223284e6a6..0430fcd5ad370 100644
--- a/llvm/test/Transforms/InstCombine/select-min-max.ll
+++ b/llvm/test/Transforms/InstCombine/select-min-max.ll
@@ -301,3 +301,66 @@ define i8 @not_smin_swap(i8 %i41, i8 %i43) {
   %spec.select = select i1 %i44, i8 %i46, i8 0
   ret i8 %spec.select
 }
+
+define i8 @sel_umin_constant(i8 %x) {
+; CHECK-LABEL: @sel_umin_constant(
+; CHECK-NEXT:    [[UMIN:%.*]] = call i8 @llvm.umin.i8(i8 [[X:%.*]], i8 16)
+; CHECK-NEXT:    ret i8 [[UMIN]]
+;
+  %cmp = icmp sgt i8 %x, -1
+  %umin = call i8 @llvm.umin.i8(i8 %x, i8 16)
+  %sel = select i1 %cmp, i8 %umin, i8 16
+  ret i8 %sel
+}
+
+define i8 @sel_constant_smax_with_range_attr(i8 %x) {
+; CHECK-LABEL: @sel_constant_smax_with_range_attr(
+; CHECK-NEXT:    [[SEL:%.*]] = call i8 @llvm.smax.i8(i8 [[X:%.*]], i8 16)
+; CHECK-NEXT:    ret i8 [[SEL]]
+;
+  %cmp = icmp slt i8 %x, 0
+  %smax = call range(i8 8, 16) i8 @llvm.smax.i8(i8 %x, i8 16)
+  %sel = select i1 %cmp, i8 16, i8 %smax
+  ret i8 %sel
+}
+
+; Negative tests
+
+define i8 @sel_umin_constant_mismatch(i8 %x) {
+; CHECK-LABEL: @sel_umin_constant_mismatch(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i8 [[X:%.*]], -1
+; CHECK-NEXT:    [[UMIN:%.*]] = call i8 @llvm.umin.i8(i8 [[X]], i8 16)
+; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[CMP]], i8 [[UMIN]], i8 15
+; CHECK-NEXT:    ret i8 [[SEL]]
+;
+  %cmp = icmp sgt i8 %x, -1
+  %umin = call i8 @llvm.umin.i8(i8 %x, i8 16)
+  %sel = select i1 %cmp, i8 %umin, i8 15
+  ret i8 %sel
+}
+
+define i8 @sel_umin_constant_op_mismatch(i8 %x, i8 %y) {
+; CHECK-LABEL: @sel_umin_constant_op_mismatch(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i8 [[X:%.*]], -1
+; CHECK-NEXT:    [[UMIN:%.*]] = call i8 @llvm.umin.i8(i8 [[Y:%.*]], i8 16)
+; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[CMP]], i8 [[UMIN]], i8 16
+; CHECK-NEXT:    ret i8 [[SEL]]
+;
+  %cmp = icmp sgt i8 %x, -1
+  %umin = call i8 @llvm.umin.i8(i8 %y, i8 16)
+  %sel = select i1 %cmp, i8 %umin, i8 16
+  ret i8 %sel
+}
+
+define i8 @sel_umin_non_constant(i8 %x, i8 %y) {
+; CHECK-LABEL: @sel_umin_non_constant(
+; CHECK-NEXT:    [[UMIN:%.*]] = call i8 @llvm.umin.i8(i8 [[X:%.*]], i8 16)
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp slt i8 [[X]], 0
+; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[CMP1]], i8 [[Y:%.*]], i8 [[UMIN]]
+; CHECK-NEXT:    ret i8 [[SEL]]
+;
+  %cmp = icmp sgt i8 %x, -1
+  %umin = call i8 @llvm.umin.i8(i8 %x, i8 16)
+  %sel = select i1 %cmp, i8 %umin, i8 %y
+  ret i8 %sel
+}
diff --git a/llvm/test/Transforms/InstCombine/select_frexp.ll b/llvm/test/Transforms/InstCombine/select_frexp.ll
new file mode 100644
index 0000000000000..d025aedda7170
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/select_frexp.ll
@@ -0,0 +1,191 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -passes=instcombine -S < %s | FileCheck %s
+
+declare { float, i32 } @llvm.frexp.f32.i32(float)
+declare void @use(float)
+
+; Basic test case - constant in true position
+define float @test_select_frexp_basic(float %x, i1 %cond) {
+; CHECK-LABEL: define float @test_select_frexp_basic(
+; CHECK-SAME: float [[X:%.*]], i1 [[COND:%.*]]) {
+; CHECK-NEXT:    [[FREXP:%.*]] = call { float, i32 } @llvm.frexp.f32.i32(float [[X]])
+; CHECK-NEXT:    [[FREXP_0:%.*]] = extractvalue { float, i32 } [[FREXP]], 0
+; CHECK-NEXT:    [[SELECT_FREXP:%.*]] = select i1 [[COND]], float 5.000000e-01, float [[FREXP_0]]
+; CHECK-NEXT:    ret float [[SELECT_FREXP]]
+;
+  %sel = select i1 %cond, float 1.000000e+00, float %x
+  %frexp = call { float, i32 } @llvm.frexp.f32.i32(float %sel)
+  %frexp.0 = extractvalue { float, i32 } %frexp, 0
+  ret float %frexp.0
+}
+
+; Test with constant in false position
+define float @test_select_frexp_const_false(float %x, i1 %cond) {
+; CHECK-LABEL: define float @test_select_frexp_const_false(
+; CHECK-SAME: float [[X:%.*]], i1 [[COND:%.*]]) {
+; CHECK-NEXT:    [[FREXP:%.*]] = call { float, i32 } @llvm.frexp.f32.i32(float [[X]])
+; CHECK-NEXT:    [[FREXP_0:%.*]] = extractvalue { float, i32 } [[FREXP]], 0
+; CHECK-NEXT:    [[SELECT_FREXP:%.*]] = select i1 [[COND]], float [[FREXP_0]], float 5.000000e-01
+; CHECK-NEXT:    ret float [[SELECT_FREXP]]
+;
+  %sel = select i1 %cond, float %x, float 1.000000e+00
+  %frexp = call { float, i32 } @llvm.frexp.f32.i32(float %sel)
+  %frexp.0 = extractvalue { float, i32 } %frexp, 0
+  ret float %frexp.0
+}
+
+; Multi-use test
+define float @test_select_frexp_multi_use(float %x, i1 %cond) {
+; CHECK-LABEL: define float @test_select_frexp_multi_use(
+; CHECK-SAME: float [[X:%.*]], i1 [[COND:%.*]]) {
+; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[COND]], float 1.000000e+00, float [[X]]
+; CHECK-NEXT:    call void @use(float [[SEL]])
+; CHECK-NEXT:    [[FREXP:%.*]] = call { float, i32 } @llvm.frexp.f32.i32(float [[SEL]])
+; CHECK-NEXT:    [[FREXP_0:%.*]] = extractvalue { float, i32 } [[FREXP]], 0
+; CHECK-NEXT:    ret float [[FREXP_0]]
+;
+  %sel = select i1 %cond, float 1.000000e+00, float %x
+  call void @use(float %sel)
+  %frexp = call { float, i32 } @llvm.frexp.f32.i32(float %sel)
+  %frexp.0 = extractvalue { float, i32 } %frexp, 0
+  ret float %frexp.0
+}
+
+; Vector test - splat constant
+define <2 x float> @test_select_frexp_vec_splat(<2 x float> %x, <2 x i1> %cond) {
+; CHECK-LABEL: define <2 x float> @test_select_frexp_vec_splat(
+; CHECK-SAME: <2 x float> [[X:%.*]], <2 x i1> [[COND:%.*]]) {
+; CHECK-NEXT:    [[FREXP:%.*]] = call { <2 x float>, <2 x i32> } @llvm.frexp.v2f32.v2i32(<2 x float> [[X]])
+; CHECK-NEXT:    [[FREXP_0:%.*]] = extractvalue { <2 x float>, <2 x i32> } [[FREXP]], 0
+; CHECK-NEXT:    [[SELECT_FREXP:%.*]] = select <2 x i1> [[COND]], <2 x float> splat (float 5.000000e-01), <2 x float> [[FREXP_0]]
+; CHECK-NEXT:    ret <2 x float> [[SELECT_FREXP]]
+;
+  %sel = select <2 x i1> %cond, <2 x float> <float 1.000000e+00, float 1.000000e+00>, <2 x float> %x
+  %frexp = call { <2 x float>, <2 x i32> } @llvm.frexp.v2f32.v2i32(<2 x float> %sel)
+  %frexp.0 = extractvalue { <2 x float>, <2 x i32> } %frexp, 0
+  ret <2 x float> %frexp.0
+}
+
+; Vector test with poison
+define <2 x float> @test_select_frexp_vec_poison(<2 x float> %x, <2 x i1> %cond) {
+; CHECK-LABEL: define <2 x float> @test_select_frexp_vec_poison(
+; CHECK-SAME: <2 x float> [[X:%.*]], <2 x i1> [[COND:%.*]]) {
+; CHECK-NEXT:    [[SEL:%.*]] = select <2 x i1> [[COND]], <2 x float> <float 1.000000e+00, float poison>, <2 x float> [[X]]
+; CHECK-NEXT:    [[FREXP:%.*]] = call { <2 x float>, <2 x i32> } @llvm.frexp.v2f32.v2i32(<2 x float> [[SEL]])
+; CHECK-NEXT:    [[FREXP_0:%.*]] = extractvalue { <2 x float>, <2 x i32> } [[FREXP]], 0
+; CHECK-NEXT:    ret <2 x float> [[FREXP_0]]
+;
+  %sel = select <2 x i1> %cond, <2 x float> <float 1.000000e+00, float poison>, <2 x float> %x
+  %frexp = call { <2 x float>, <2 x i32> } @llvm.frexp.v2f32.v2i32(<2 x float> %sel)
+  %frexp.0 = extractvalue { <2 x float>, <2 x i32> } %frexp, 0
+  ret <2 x float> %frexp.0
+}
+
+; Vector test - non-splat (should not fold)
+define <2 x float> @test_select_frexp_vec_nonsplat(<2 x float> %x, <2 x i1> %cond) {
+; CHECK-LABEL: define <2 x float> @test_select_frexp_vec_nonsplat(
+; CHECK-SAME: <2 x float> [[X:%.*]], <2 x i1> [[COND:%.*]]) {
+; CHECK-NEXT:    [[SEL:%.*]] = select <2 x i1> [[COND]], <2 x float> <float 1.000000e+00, float 2.000000e+00>, <2 x float> [[X]]
+; CHECK-NEXT:    [[FREXP:%.*]] = call { <2 x float>, <2 x i32> } @llvm.frexp.v2f32.v2i32(<2 x float> [[SEL]])
+; CHECK-NEXT:    [[FREXP_0:%.*]] = extractvalue { <2 x float>, <2 x i32> } [[FREXP]], 0
+; CHECK-NEXT:    ret <2 x float> [[FREXP_0]]
+;
+  %sel = select <2 x i1> %cond, <2 x float> <float 1.000000e+00, float 2.000000e+00>, <2 x float> %x
+  %frexp = call { <2 x float>, <2 x i32> } @llvm.frexp.v2f32.v2i32(<2 x float> %sel)
+  %frexp.0 = extractvalue { <2 x float>, <2 x i32> } %frexp, 0
+  ret <2 x float> %frexp.0
+}
+
+; Negative test - both operands non-constant
+define float @test_select_frexp_no_const(float %x, float %y, i1 %cond) {
+; CHECK-LABEL: define float @test_select_frexp_no_const(
+; CHECK-SAME: float [[X:%.*]], float [[Y:%.*]], i1 [[COND:%.*]]) {
+; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[COND]], float [[X]], float [[Y]]
+; CHECK-NEXT:    [[FREXP:%.*]] = call { float, i32 } @llvm.frexp.f32.i32(float [[SEL]])
+; CHECK-NEXT:    [[FREXP_0:%.*]] = extractvalue { float, i32 } [[FREXP]], 0
+; CHECK-NEXT:    ret float [[FREXP_0]]
+;
+  %sel = select i1 %cond, float %x, float %y
+  %frexp = call { float, i32 } @llvm.frexp.f32.i32(float %sel)
+  %frexp.0 = extractvalue { float, i32 } %frexp, 0
+  ret float %frexp.0
+}
+
+; Negative test - extracting exp instead of mantissa
+define i32 @test_select_frexp_extract_exp(float %x, i1 %cond) {
+; CHECK-LABEL: define i32 @test_select_frexp_extract_exp(
+; CHECK-SAME: float [[X:%.*]], i1 [[COND:%.*]]) {
+; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[COND]], float 1.000000e+00, float [[X]]
+; CHECK-NEXT:    [[FREXP:%.*]] = call { float, i32 } @llvm.frexp.f32.i32(float [[SEL]])
+; CHECK-NEXT:    [[FREXP_1:%.*]] = extractvalue { float, i32 } [[FREXP]], 1
+; CHECK-NEXT:    ret i32 [[FREXP_1]]
+;
+  %sel = select i1 %cond, float 1.000000e+00, float %x
+  %frexp = call { float, i32 } @llvm.frexp.f32.i32(float %sel)
+  %frexp.1 = extractvalue { float, i32 } %frexp, 1
+  ret i32 %frexp.1
+}
+
+; Test with fast math flags
+define float @test_select_frexp_fast_math_select(float %x, i1 %cond) {
+; CHECK-LABEL: define float @test_select_frexp_fast_math_select(
+; CHECK-SAME: float [[X:%.*]], i1 [[COND:%.*]]) {
+; CHECK-NEXT:    [[FREXP1:%.*]] = call { float, i32 } @llvm.frexp.f32.i32(float [[X]])
+; CHECK-NEXT:    [[MANTISSA:%.*]] = extractvalue { float, i32 } [[FREXP1]], 0
+; CHECK-NEXT:    [[SELECT_FREXP:%.*]] = select nnan ninf nsz i1 [[COND]], float 5.000000e-01, float [[MANTISSA]]
+; CHECK-NEXT:    ret float [[SELECT_FREXP]]
+;
+  %sel = select nnan ninf nsz i1 %cond, float 1.000000e+00, float %x
+  %frexp = call { float, i32 } @llvm.frexp.f32.i32(float %sel)
+  %frexp.0 = extractvalue { float, i32 } %frexp, 0
+  ret float %frexp.0
+}
+
+
+; Test vector case with fast math flags
+define <2 x float> @test_select_frexp_vec_fast_math(<2 x float> %x, <2 x i1> %cond) {
+; CHECK-LABEL: define <2 x float> @test_select_frexp_vec_fast_math(
+; CHECK-SAME: <2 x float> [[X:%.*]], <2 x i1> [[COND:%.*]]) {
+; CHECK-NEXT:    [[FREXP1:%.*]] = call { <2 x float>, <2 x i32> } @llvm.frexp.v2f32.v2i32(<2 x float> [[X]])
+; CHECK-NEXT:    [[MANTISSA:%.*]] = extractvalue { <2 x float>, <2 x i32> } [[FREXP1]], 0
+; CHECK-NEXT:    [[SELECT_FREXP:%.*]] = select nnan ninf nsz <2 x i1> [[COND]], <2 x float> splat (float 5.000000e-01), <2 x float> [[MANTISSA]]
+; CHECK-NEXT:    ret <2 x float> [[SELECT_FREXP]]
+;
+  %sel = select nnan ninf nsz <2 x i1> %cond, <2 x float> <float 1.000000e+00, float 1.000000e+00>, <2 x float> %x
+  %frexp = call { <2 x float>, <2 x i32> } @llvm.frexp.v2f32.v2i32(<2 x float> %sel)
+  %frexp.0 = extractvalue { <2 x float>, <2 x i32> } %frexp, 0
+  ret <2 x float> %frexp.0
+}
+
+; Test with scalable vectors with constant at True Position
+define <vscale x 2 x float> @test_select_frexp_scalable_vec0(<vscale x 2 x float> %x, <vscale x 2 x i1> %cond) {
+; CHECK-LABEL: define <vscale x 2 x float> @test_select_frexp_scalable_vec0(
+; CHECK-SAME: <vscale x 2 x float> [[X:%.*]], <vscale x 2 x i1> [[COND:%.*]]) {
+; CHECK-NEXT:    [[FREXP1:%.*]] = call { <vscale x 2 x float>, <vscale x 2 x i32> } @llvm.frexp.nxv2f32.nxv2i32(<vscale x 2 x float> [[X]])
+; CHECK-NEXT:    [[MANTISSA:%.*]] = extractvalue { <vscale x 2 x float>, <vscale x 2 x i32> } [[FREXP1]], 0
+; CHECK-NEXT:    [[SELECT_FREXP:%.*]] = select <vscale x 2 x i1> [[COND]], <vscale x 2 x float> splat (float 5.000000e-01), <vscale x 2 x float> [[MANTISSA]]
+; CHECK-NEXT:    ret <vscale x 2 x float> [[SELECT_FREXP]]
+;
+  %sel = select <vscale x 2 x i1> %cond, <vscale x 2 x float> splat (float 1.000000e+00), <vscale x 2 x float> %x
+  %frexp = call { <vscale x 2 x float>, <vscale x 2 x i32> } @llvm.frexp.nxv2f32.nxv2i32(<vscale x 2 x float> %sel)
+  %frexp.0 = extractvalue { <vscale x 2 x float>, <vscale x 2 x i32> } %frexp, 0
+  ret <vscale x 2 x float> %frexp.0
+}
+
+; Test with scalable vectors with constant at False Position
+define <vscale x 2 x float> @test_select_frexp_scalable_vec1(<vscale x 2 x float> %x, <vscale x 2 x i1> %cond) {
+; CHECK-LABEL: define <vscale x 2 x float> @test_select_frexp_scalable_vec1(
+; CHECK-SAME: <vscale x 2 x float> [[X:%.*]], <vscale x 2 x i1> [[COND:%.*]]) {
+; CHECK-NEXT:    [[FREXP1:%.*]] = call { <vscale x 2 x float>, <vscale x 2 x i32> } @llvm.frexp.nxv2f32.nxv2i32(<vscale x 2 x float> [[X]])
+; CHECK-NEXT:    [[MANTISSA:%.*]] = extractvalue { <vscale x 2 x float>, <vscale x 2 x i32> } [[FREXP1]], 0
+; CHECK-NEXT:    [[SELECT_FREXP:%.*]] = select <vscale x 2 x i1> [[COND]], <vscale x 2 x float> [[MANTISSA]], <vscale x 2 x float> splat (float 5.000000e-01)
+; CHECK-NEXT:    ret <vscale x 2 x float> [[SELECT_FREXP]]
+;
+  %sel = select <vscale x 2 x i1> %cond, <vscale x 2 x float> %x, <vscale x 2 x float> splat (float 1.000000e+00)
+  %frexp = call { <vscale x 2 x float>, <vscale x 2 x i32> } @llvm.frexp.nxv2f32.nxv2i32(<vscale x 2 x float> %sel)
+  %frexp.0 = extractvalue { <vscale x 2 x float>, <vscale x 2 x i32> } %frexp, 0
+  ret <vscale x 2 x float> %frexp.0
+}
+
+declare { <2 x float>, <2 x i32> } @llvm.frexp.v2f32.v2i32(<2 x float>)
+declare { <vscale x 2 x float>, <vscale x 2 x i32> } @llvm.frexp.nxv2f32.nxv2i32(<vscale x 2 x float>)
diff --git a/llvm/test/Transforms/InstCombine/sign-test-and-or.ll b/llvm/test/Transforms/InstCombine/sign-test-and-or.ll
index 65363620563be..3e9ff63869d64 100644
--- a/llvm/test/Transforms/InstCombine/sign-test-and-or.ll
+++ b/llvm/test/Transforms/InstCombine/sign-test-and-or.ll
@@ -349,6 +349,29 @@ define i1 @test9_logical(i32 %a) {
   ret i1 %or.cond
 }
 
+define i1 @test9_logical_samesign(i32 %a) {
+; CHECK-LABEL: @test9_logical_samesign(
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp sgt i32 [[A:%.*]], -1
+; CHECK-NEXT:    ret i1 [[CMP2]]
+;
+  %masked = and i32 %a, -1073741825
+  %cmp1 = icmp eq i32 %masked, 0
+  %cmp2 = icmp samesign sgt i32 %a, -1
+  %or.cond = select i1 %cmp1, i1 true, i1 %cmp2
+  ret i1 %or.cond
+}
+
+define i1 @test_logical_or_icmp_icmp_samesign(i32 %a) {
+; CHECK-LABEL: @test_logical_or_icmp_icmp_samesign(
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp sgt i32 [[A:%.*]], -1
+; CHECK-NEXT:    ret i1 [[CMP2]]
+;
+  %cmp1 = icmp eq i32 %a, 0
+  %cmp2 = icmp samesign sgt i32 %a, -1
+  %or = select i1 %cmp1, i1 true, i1 %cmp2
+  ret i1 %or
+}
+
 define i1 @test10(i32 %a) {
 ; CHECK-LABEL: @test10(
 ; CHECK-NEXT:    [[OR_COND:%.*]] = icmp ult i32 [[A:%.*]], 2
diff --git a/llvm/test/Transforms/InstCombine/simplify-libcalls-inreg.ll b/llvm/test/Transforms/InstCombine/simplify-libcalls-inreg.ll
index 8bbd972ef6e0f..9c38b867f5d93 100644
--- a/llvm/test/Transforms/InstCombine/simplify-libcalls-inreg.ll
+++ b/llvm/test/Transforms/InstCombine/simplify-libcalls-inreg.ll
@@ -7,7 +7,7 @@ target datalayout = "e-m:e-p:32:32-p270:32:32-p271:32:32-p272:64:64-f64:32:64-f8
 target triple = "i386-unknown-linux-gnu"
 
 declare ptr @foo()
-declare i32 @memcmp(ptr inreg nocapture noundef, ptr inreg nocapture noundef, i32 inreg noundef)
+declare i32 @memcmp(ptr inreg captures(none) noundef, ptr inreg captures(none) noundef, i32 inreg noundef)
 declare i32 @printf(ptr, ...)
 declare double @exp2(double)
 declare i32 @__sprintf_chk(ptr, i32, i32, ptr, ...)
@@ -15,8 +15,8 @@ declare i32 @__sprintf_chk(ptr, i32, i32, ptr, ...)
 @b = common global [60 x i8] zeroinitializer, align 1
 @h = constant [2 x i8] c"h\00"
 
-; CHECK:     declare i32 @bcmp(ptr inreg nocapture, ptr inreg nocapture, i32 inreg)
-; CHECK-NOT: declare i32 @bcmp(ptr nocapture, ptr nocapture, i32)
+; CHECK:     declare i32 @bcmp(ptr inreg captures(none), ptr inreg captures(none), i32 inreg)
+; CHECK-NOT: declare i32 @bcmp(ptr captures(none), ptr captures(none), i32)
 
 define i32 @baz(ptr inreg noundef %s2, i32 inreg noundef %n){
   %call = call ptr @foo()
@@ -43,8 +43,8 @@ define double @test_non_int_params(i16 signext %x) {
   ret double %ret
 }
 
-; CHECK:     declare noundef i32 @sprintf(ptr noalias nocapture noundef writeonly, ptr nocapture noundef readonly, ...)
-; CHECK-NOT: declare noundef i32 @sprintf(ptr inreg noalias nocapture noundef writeonly, ptr inreg nocapture noundef readonly, ...)
+; CHECK:     declare noundef i32 @sprintf(ptr noalias noundef writeonly captures(none), ptr noundef readonly captures(none), ...)
+; CHECK-NOT: declare noundef i32 @sprintf(ptr inreg noalias noundef writeonly captures(none), ptr inreg noundef readonly captures(none), ...)
 define i32 @test_variadic() {
   %ret = call i32 (ptr, i32, i32, ptr, ...) @__sprintf_chk(ptr @a, i32 0, i32 -1, ptr @b)
   ret i32 %ret
diff --git a/llvm/test/Transforms/InstCombine/sprintf-1.ll b/llvm/test/Transforms/InstCombine/sprintf-1.ll
index 0749015059415..1d87758340f71 100644
--- a/llvm/test/Transforms/InstCombine/sprintf-1.ll
+++ b/llvm/test/Transforms/InstCombine/sprintf-1.ll
@@ -103,7 +103,7 @@ define i32 @test_simplify7(ptr %dst, ptr %str) {
 ; NOSTPCPY-LABEL: @test_simplify7(
 ; NOSTPCPY-NEXT:    [[STRLEN:%.*]] = call i32 @strlen(ptr noundef nonnull dereferenceable(1) [[STR:%.*]])
 ; NOSTPCPY-NEXT:    [[LENINC:%.*]] = add i32 [[STRLEN]], 1
-; NOSTPCPY-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr align 1 [[DST:%.*]], ptr align 1 [[STR]], i32 [[LENINC]], i1 false)
+; NOSTPCPY-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr align 1 [[DST:%.*]], ptr nonnull align 1 [[STR]], i32 [[LENINC]], i1 false)
 ; NOSTPCPY-NEXT:    ret i32 [[STRLEN]]
 ;
   %r = call i32 (ptr, ptr, ...) @sprintf(ptr %dst, ptr @percent_s, ptr %str)
@@ -133,7 +133,7 @@ define i32 @test_simplify9(ptr %dst, ptr %str) {
 ; NOSTPCPY-LABEL: @test_simplify9(
 ; NOSTPCPY-NEXT:    [[STRLEN:%.*]] = call i32 @strlen(ptr noundef nonnull dereferenceable(1) [[STR:%.*]])
 ; NOSTPCPY-NEXT:    [[LENINC:%.*]] = add i32 [[STRLEN]], 1
-; NOSTPCPY-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr align 1 [[DST:%.*]], ptr align 1 [[STR]], i32 [[LENINC]], i1 false)
+; NOSTPCPY-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr align 1 [[DST:%.*]], ptr nonnull align 1 [[STR]], i32 [[LENINC]], i1 false)
 ; NOSTPCPY-NEXT:    ret i32 [[STRLEN]]
 ;
   %r = call i32 (ptr, ptr, ...) @sprintf(ptr %dst, ptr @percent_s, ptr %str)
diff --git a/llvm/test/Transforms/InstCombine/stpncpy-1.ll b/llvm/test/Transforms/InstCombine/stpncpy-1.ll
index 87f54918b7d25..6ef9b425ae9d9 100644
--- a/llvm/test/Transforms/InstCombine/stpncpy-1.ll
+++ b/llvm/test/Transforms/InstCombine/stpncpy-1.ll
@@ -70,11 +70,11 @@ define void @fold_stpncpy_overlap(ptr %dst, i64 %n) {
 define void @call_stpncpy_overlap(ptr %dst, i64 %n) {
 ; ANY-LABEL: @call_stpncpy_overlap(
 ; ANY-NEXT:    [[ES_2:%.*]] = call ptr @stpncpy(ptr noundef nonnull dereferenceable(1) [[DST:%.*]], ptr noundef nonnull dereferenceable(1) [[DST]], i64 2)
-; ANY-NEXT:    call void @sink(ptr [[DST]], ptr [[ES_2]])
+; ANY-NEXT:    call void @sink(ptr nonnull [[DST]], ptr [[ES_2]])
 ; ANY-NEXT:    [[ES_3:%.*]] = call ptr @stpncpy(ptr noundef nonnull dereferenceable(1) [[DST]], ptr noundef nonnull dereferenceable(1) [[DST]], i64 3)
-; ANY-NEXT:    call void @sink(ptr [[DST]], ptr [[ES_3]])
-; ANY-NEXT:    [[ES_N:%.*]] = call ptr @stpncpy(ptr [[DST]], ptr [[DST]], i64 [[N:%.*]])
-; ANY-NEXT:    call void @sink(ptr [[DST]], ptr [[ES_N]])
+; ANY-NEXT:    call void @sink(ptr nonnull [[DST]], ptr [[ES_3]])
+; ANY-NEXT:    [[ES_N:%.*]] = call ptr @stpncpy(ptr nonnull [[DST]], ptr nonnull [[DST]], i64 [[N:%.*]])
+; ANY-NEXT:    call void @sink(ptr nonnull [[DST]], ptr [[ES_N]])
 ; ANY-NEXT:    ret void
 ;
 ; Do not transform stpncpy(D, D, 2).
@@ -428,9 +428,9 @@ define void @fold_stpncpy_s(ptr %dst, ptr %src) {
 define void @call_stpncpy_s(ptr %dst, ptr %src, i64 %n) {
 ; ANY-LABEL: @call_stpncpy_s(
 ; ANY-NEXT:    [[ES_2:%.*]] = call ptr @stpncpy(ptr noundef nonnull dereferenceable(1) [[DST:%.*]], ptr noundef nonnull dereferenceable(1) [[SRC:%.*]], i64 2)
-; ANY-NEXT:    call void @sink(ptr [[DST]], ptr [[ES_2]])
-; ANY-NEXT:    [[ES_N:%.*]] = call ptr @stpncpy(ptr [[DST]], ptr [[SRC]], i64 [[N:%.*]])
-; ANY-NEXT:    call void @sink(ptr [[DST]], ptr [[ES_N]])
+; ANY-NEXT:    call void @sink(ptr nonnull [[DST]], ptr [[ES_2]])
+; ANY-NEXT:    [[ES_N:%.*]] = call ptr @stpncpy(ptr nonnull [[DST]], ptr nonnull [[SRC]], i64 [[N:%.*]])
+; ANY-NEXT:    call void @sink(ptr nonnull [[DST]], ptr [[ES_N]])
 ; ANY-NEXT:    ret void
 ;
 ; Do not transform stpncpy(D, S, 2).  Both *D and *S must be derefernceable
diff --git a/llvm/test/Transforms/InstCombine/str-int-2.ll b/llvm/test/Transforms/InstCombine/str-int-2.ll
index 1379b9b727bfa..1d9bd3211e038 100644
--- a/llvm/test/Transforms/InstCombine/str-int-2.ll
+++ b/llvm/test/Transforms/InstCombine/str-int-2.ll
@@ -72,7 +72,7 @@ define i32 @atoi_test() #0 {
 
 define i64 @strtol_not_const_str(ptr %s) #0 {
 ; CHECK-LABEL: @strtol_not_const_str(
-; CHECK-NEXT:    [[CALL:%.*]] = call i64 @strtol(ptr nocapture [[S:%.*]], ptr null, i32 10)
+; CHECK-NEXT:    [[CALL:%.*]] = call i64 @strtol(ptr captures(none) [[S:%.*]], ptr null, i32 10)
 ; CHECK-NEXT:    ret i64 [[CALL]]
 ;
   %call = call i64 @strtol(ptr %s, ptr null, i32 10) #3
@@ -90,7 +90,7 @@ define i32 @atoi_not_const_str(ptr %s) #0 {
 
 define i64 @strtol_not_const_base(i32 %b) #0 {
 ; CHECK-LABEL: @strtol_not_const_base(
-; CHECK-NEXT:    [[CALL:%.*]] = call i64 @strtol(ptr nocapture nonnull @.str, ptr null, i32 [[B:%.*]])
+; CHECK-NEXT:    [[CALL:%.*]] = call i64 @strtol(ptr nonnull captures(none) @.str, ptr null, i32 [[B:%.*]])
 ; CHECK-NEXT:    ret i64 [[CALL]]
 ;
   %call = call i64 @strtol(ptr @.str, ptr null, i32 %b) #2
@@ -108,7 +108,7 @@ define i64 @strtol_long_int() #0 {
 
 define i64 @strtol_big_overflow() #0 {
 ; CHECK-LABEL: @strtol_big_overflow(
-; CHECK-NEXT:    [[CALL:%.*]] = call i64 @strtol(ptr nocapture nonnull @.str.3, ptr null, i32 10)
+; CHECK-NEXT:    [[CALL:%.*]] = call i64 @strtol(ptr nonnull captures(none) @.str.3, ptr null, i32 10)
 ; CHECK-NEXT:    ret i64 [[CALL]]
 ;
   %call = call i64 @strtol(ptr nocapture @.str.3, ptr null, i32 10) #2
diff --git a/llvm/test/Transforms/InstCombine/str-int.ll b/llvm/test/Transforms/InstCombine/str-int.ll
index 2d22c1ae13049..d724f0ea156de 100644
--- a/llvm/test/Transforms/InstCombine/str-int.ll
+++ b/llvm/test/Transforms/InstCombine/str-int.ll
@@ -75,7 +75,7 @@ define i32 @atoi_test() #0 {
 
 define i32 @strtol_not_const_str(ptr %s) #0 {
 ; CHECK-LABEL: @strtol_not_const_str(
-; CHECK-NEXT:    [[CALL:%.*]] = call i32 @strtol(ptr nocapture [[S:%.*]], ptr null, i32 10)
+; CHECK-NEXT:    [[CALL:%.*]] = call i32 @strtol(ptr captures(none) [[S:%.*]], ptr null, i32 10)
 ; CHECK-NEXT:    ret i32 [[CALL]]
 ;
   %call = call i32 @strtol(ptr %s, ptr null, i32 10) #3
@@ -93,7 +93,7 @@ define i32 @atoi_not_const_str(ptr %s) #0 {
 
 define i32 @strtol_not_const_base(i32 %b) #0 {
 ; CHECK-LABEL: @strtol_not_const_base(
-; CHECK-NEXT:    [[CALL:%.*]] = call i32 @strtol(ptr nocapture nonnull @.str, ptr null, i32 [[B:%.*]])
+; CHECK-NEXT:    [[CALL:%.*]] = call i32 @strtol(ptr nonnull captures(none) @.str, ptr null, i32 [[B:%.*]])
 ; CHECK-NEXT:    ret i32 [[CALL]]
 ;
   %call = call i32 @strtol(ptr @.str, ptr null, i32 %b) #2
@@ -102,7 +102,7 @@ define i32 @strtol_not_const_base(i32 %b) #0 {
 
 define i32 @strtol_long_int() #0 {
 ; CHECK-LABEL: @strtol_long_int(
-; CHECK-NEXT:    [[CALL:%.*]] = call i32 @strtol(ptr nocapture nonnull @.str.2, ptr null, i32 10)
+; CHECK-NEXT:    [[CALL:%.*]] = call i32 @strtol(ptr nonnull captures(none) @.str.2, ptr null, i32 10)
 ; CHECK-NEXT:    ret i32 [[CALL]]
 ;
   %call = call i32 @strtol(ptr @.str.2, ptr null, i32 10) #3
@@ -112,7 +112,7 @@ define i32 @strtol_long_int() #0 {
 
 define i32 @strtol_big_overflow() #0 {
 ; CHECK-LABEL: @strtol_big_overflow(
-; CHECK-NEXT:    [[CALL:%.*]] = call i32 @strtol(ptr nocapture nonnull @.str.3, ptr null, i32 10)
+; CHECK-NEXT:    [[CALL:%.*]] = call i32 @strtol(ptr nonnull captures(none) @.str.3, ptr null, i32 10)
 ; CHECK-NEXT:    ret i32 [[CALL]]
 ;
   %call = call i32 @strtol(ptr nocapture @.str.3, ptr null, i32 10) #2
diff --git a/llvm/test/Transforms/InstCombine/strcall-no-nul.ll b/llvm/test/Transforms/InstCombine/strcall-no-nul.ll
index 0b6dde916da21..23e6329016908 100644
--- a/llvm/test/Transforms/InstCombine/strcall-no-nul.ll
+++ b/llvm/test/Transforms/InstCombine/strcall-no-nul.ll
@@ -287,16 +287,16 @@ define void @fold_atol_strtol_past_end(ptr %ps) {
 ; CHECK-NEXT:    [[I1:%.*]] = call i64 @atoll(ptr nonnull getelementptr inbounds nuw (i8, ptr @a5, i64 5))
 ; CHECK-NEXT:    [[P1:%.*]] = getelementptr i8, ptr [[PS]], i64 8
 ; CHECK-NEXT:    store i64 [[I1]], ptr [[P1]], align 4
-; CHECK-NEXT:    [[I2:%.*]] = call i64 @strtol(ptr nocapture nonnull getelementptr inbounds nuw (i8, ptr @a5, i64 5), ptr null, i32 0)
+; CHECK-NEXT:    [[I2:%.*]] = call i64 @strtol(ptr nonnull captures(none) getelementptr inbounds nuw (i8, ptr @a5, i64 5), ptr null, i32 0)
 ; CHECK-NEXT:    [[P2:%.*]] = getelementptr i8, ptr [[PS]], i64 16
 ; CHECK-NEXT:    store i64 [[I2]], ptr [[P2]], align 4
-; CHECK-NEXT:    [[I3:%.*]] = call i64 @strtoul(ptr nocapture nonnull getelementptr inbounds nuw (i8, ptr @a5, i64 5), ptr null, i32 8)
+; CHECK-NEXT:    [[I3:%.*]] = call i64 @strtoul(ptr nonnull captures(none) getelementptr inbounds nuw (i8, ptr @a5, i64 5), ptr null, i32 8)
 ; CHECK-NEXT:    [[P3:%.*]] = getelementptr i8, ptr [[PS]], i64 24
 ; CHECK-NEXT:    store i64 [[I3]], ptr [[P3]], align 4
-; CHECK-NEXT:    [[I4:%.*]] = call i64 @strtoll(ptr nocapture nonnull getelementptr inbounds nuw (i8, ptr @a5, i64 5), ptr null, i32 10)
+; CHECK-NEXT:    [[I4:%.*]] = call i64 @strtoll(ptr nonnull captures(none) getelementptr inbounds nuw (i8, ptr @a5, i64 5), ptr null, i32 10)
 ; CHECK-NEXT:    [[P4:%.*]] = getelementptr i8, ptr [[PS]], i64 32
 ; CHECK-NEXT:    store i64 [[I4]], ptr [[P4]], align 4
-; CHECK-NEXT:    [[I5:%.*]] = call i64 @strtoul(ptr nocapture nonnull getelementptr inbounds nuw (i8, ptr @a5, i64 5), ptr null, i32 16)
+; CHECK-NEXT:    [[I5:%.*]] = call i64 @strtoul(ptr nonnull captures(none) getelementptr inbounds nuw (i8, ptr @a5, i64 5), ptr null, i32 16)
 ; CHECK-NEXT:    [[P5:%.*]] = getelementptr i8, ptr [[PS]], i64 40
 ; CHECK-NEXT:    store i64 [[I5]], ptr [[P5]], align 4
 ; CHECK-NEXT:    ret void
diff --git a/llvm/test/Transforms/InstCombine/strlcpy-1.ll b/llvm/test/Transforms/InstCombine/strlcpy-1.ll
index fd9d0580426f0..ad538259ae962 100644
--- a/llvm/test/Transforms/InstCombine/strlcpy-1.ll
+++ b/llvm/test/Transforms/InstCombine/strlcpy-1.ll
@@ -229,18 +229,18 @@ define void @fold_strlcpy_s_0(ptr %dst, ptr %s, i64 %n) {
 define void @call_strlcpy_s0_n(ptr %dst, ptr %s, i64 %n) {
 ; ANY-LABEL: @call_strlcpy_s0_n(
 ; ANY-NEXT:    [[NS_2:%.*]] = call i64 @strlcpy(ptr noundef nonnull dereferenceable(1) [[DST:%.*]], ptr noundef nonnull dereferenceable(1) [[S:%.*]], i64 2)
-; ANY-NEXT:    call void @sink(ptr [[DST]], i64 [[NS_2]])
-; ANY-NEXT:    [[NS_N:%.*]] = call i64 @strlcpy(ptr [[DST]], ptr noundef nonnull dereferenceable(1) [[S]], i64 [[N:%.*]])
-; ANY-NEXT:    call void @sink(ptr [[DST]], i64 [[NS_N]])
+; ANY-NEXT:    call void @sink(ptr nonnull [[DST]], i64 [[NS_2]])
+; ANY-NEXT:    [[NS_N:%.*]] = call i64 @strlcpy(ptr nonnull [[DST]], ptr noundef nonnull dereferenceable(1) [[S]], i64 [[N:%.*]])
+; ANY-NEXT:    call void @sink(ptr nonnull [[DST]], i64 [[NS_N]])
 ; ANY-NEXT:    [[NZ:%.*]] = or i64 [[N]], 1
 ; ANY-NEXT:    [[NS_NZ:%.*]] = call i64 @strlcpy(ptr noundef nonnull dereferenceable(1) [[DST]], ptr noundef nonnull dereferenceable(1) [[S]], i64 [[NZ]])
-; ANY-NEXT:    call void @sink(ptr [[DST]], i64 [[NS_NZ]])
-; ANY-NEXT:    [[NS0_N:%.*]] = call i64 @strlcpy(ptr [[DST]], ptr noundef nonnull dereferenceable(1) getelementptr inbounds nuw (i8, ptr @s4, i64 4), i64 [[N]])
-; ANY-NEXT:    call void @sink(ptr [[DST]], i64 [[NS0_N]])
-; ANY-NEXT:    [[NS1_N:%.*]] = call i64 @strlcpy(ptr [[DST]], ptr noundef nonnull dereferenceable(1) getelementptr inbounds nuw (i8, ptr @s4, i64 3), i64 [[N]])
-; ANY-NEXT:    call void @sink(ptr [[DST]], i64 [[NS1_N]])
-; ANY-NEXT:    [[NS4_N:%.*]] = call i64 @strlcpy(ptr [[DST]], ptr noundef nonnull dereferenceable(1) @s4, i64 [[N]])
-; ANY-NEXT:    call void @sink(ptr [[DST]], i64 [[NS4_N]])
+; ANY-NEXT:    call void @sink(ptr nonnull [[DST]], i64 [[NS_NZ]])
+; ANY-NEXT:    [[NS0_N:%.*]] = call i64 @strlcpy(ptr nonnull [[DST]], ptr noundef nonnull dereferenceable(1) getelementptr inbounds nuw (i8, ptr @s4, i64 4), i64 [[N]])
+; ANY-NEXT:    call void @sink(ptr nonnull [[DST]], i64 [[NS0_N]])
+; ANY-NEXT:    [[NS1_N:%.*]] = call i64 @strlcpy(ptr nonnull [[DST]], ptr noundef nonnull dereferenceable(1) getelementptr inbounds nuw (i8, ptr @s4, i64 3), i64 [[N]])
+; ANY-NEXT:    call void @sink(ptr nonnull [[DST]], i64 [[NS1_N]])
+; ANY-NEXT:    [[NS4_N:%.*]] = call i64 @strlcpy(ptr nonnull [[DST]], ptr noundef nonnull dereferenceable(1) @s4, i64 [[N]])
+; ANY-NEXT:    call void @sink(ptr nonnull [[DST]], i64 [[NS4_N]])
 ; ANY-NEXT:    ret void
 ;
   %ns_2 = call i64 @strlcpy(ptr %dst, ptr %s, i64 2)
diff --git a/llvm/test/Transforms/InstCombine/strncpy-4.ll b/llvm/test/Transforms/InstCombine/strncpy-4.ll
index aa70e76e20ed9..0e1f800a9a357 100644
--- a/llvm/test/Transforms/InstCombine/strncpy-4.ll
+++ b/llvm/test/Transforms/InstCombine/strncpy-4.ll
@@ -45,11 +45,11 @@ define void @fold_strncpy_overlap(ptr %dst, i64 %n) {
 define void @call_strncpy_overlap(ptr %dst, i64 %n) {
 ; CHECK-LABEL: @call_strncpy_overlap(
 ; CHECK-NEXT:    [[ED_2:%.*]] = call ptr @strncpy(ptr noundef nonnull dereferenceable(1) [[DST:%.*]], ptr noundef nonnull dereferenceable(1) [[DST]], i64 2)
-; CHECK-NEXT:    call void @sink(ptr [[DST]], ptr [[ED_2]])
+; CHECK-NEXT:    call void @sink(ptr nonnull [[DST]], ptr [[ED_2]])
 ; CHECK-NEXT:    [[ED_3:%.*]] = call ptr @strncpy(ptr noundef nonnull dereferenceable(1) [[DST]], ptr noundef nonnull dereferenceable(1) [[DST]], i64 3)
-; CHECK-NEXT:    call void @sink(ptr [[DST]], ptr [[ED_3]])
-; CHECK-NEXT:    [[ED_N:%.*]] = call ptr @strncpy(ptr [[DST]], ptr [[DST]], i64 [[N:%.*]])
-; CHECK-NEXT:    call void @sink(ptr [[DST]], ptr [[ED_N]])
+; CHECK-NEXT:    call void @sink(ptr nonnull [[DST]], ptr [[ED_3]])
+; CHECK-NEXT:    [[ED_N:%.*]] = call ptr @strncpy(ptr nonnull [[DST]], ptr nonnull [[DST]], i64 [[N:%.*]])
+; CHECK-NEXT:    call void @sink(ptr nonnull [[DST]], ptr [[ED_N]])
 ; CHECK-NEXT:    ret void
 ;
 
@@ -141,11 +141,11 @@ define void @fold_strncpy_s(ptr %dst, ptr %src, i64 %n) {
 define void @call_strncpy_s(ptr %dst, ptr %src, i64 %n) {
 ; CHECK-LABEL: @call_strncpy_s(
 ; CHECK-NEXT:    [[ED_2:%.*]] = call ptr @strncpy(ptr noundef nonnull dereferenceable(1) [[DST:%.*]], ptr noundef nonnull dereferenceable(1) [[SRC:%.*]], i64 2)
-; CHECK-NEXT:    call void @sink(ptr [[DST]], ptr [[ED_2]])
+; CHECK-NEXT:    call void @sink(ptr nonnull [[DST]], ptr [[ED_2]])
 ; CHECK-NEXT:    [[ED_9:%.*]] = call ptr @strncpy(ptr noundef nonnull dereferenceable(1) [[DST]], ptr noundef nonnull dereferenceable(1) [[SRC]], i64 9)
-; CHECK-NEXT:    call void @sink(ptr [[DST]], ptr [[ED_9]])
-; CHECK-NEXT:    [[ED_N:%.*]] = call ptr @strncpy(ptr [[DST]], ptr [[SRC]], i64 [[N:%.*]])
-; CHECK-NEXT:    call void @sink(ptr [[DST]], ptr [[ED_N]])
+; CHECK-NEXT:    call void @sink(ptr nonnull [[DST]], ptr [[ED_9]])
+; CHECK-NEXT:    [[ED_N:%.*]] = call ptr @strncpy(ptr nonnull [[DST]], ptr nonnull [[SRC]], i64 [[N:%.*]])
+; CHECK-NEXT:    call void @sink(ptr nonnull [[DST]], ptr [[ED_N]])
 ; CHECK-NEXT:    ret void
 ;
 ; Do not transform strncpy(D, S, 2) when S is unknown.  Both *D and *S must
diff --git a/llvm/test/Transforms/InstCombine/strstr-1.ll b/llvm/test/Transforms/InstCombine/strstr-1.ll
index 68de7614aad2b..083ee47bb47d1 100644
--- a/llvm/test/Transforms/InstCombine/strstr-1.ll
+++ b/llvm/test/Transforms/InstCombine/strstr-1.ll
@@ -58,7 +58,7 @@ define ptr @test_simplify4(ptr %str) {
 define i1 @test_simplify5(ptr %str, ptr %pat) {
 ; CHECK-LABEL: @test_simplify5(
 ; CHECK-NEXT:    [[STRLEN:%.*]] = call i64 @strlen(ptr noundef nonnull dereferenceable(1) [[PAT:%.*]])
-; CHECK-NEXT:    [[STRNCMP:%.*]] = call i32 @strncmp(ptr [[STR:%.*]], ptr [[PAT]], i64 [[STRLEN]])
+; CHECK-NEXT:    [[STRNCMP:%.*]] = call i32 @strncmp(ptr [[STR:%.*]], ptr nonnull [[PAT]], i64 [[STRLEN]])
 ; CHECK-NEXT:    [[CMP1:%.*]] = icmp eq i32 [[STRNCMP]], 0
 ; CHECK-NEXT:    ret i1 [[CMP1]]
 ;
diff --git a/llvm/test/Transforms/InstCombine/strto-1.ll b/llvm/test/Transforms/InstCombine/strto-1.ll
index 131997dc4b53d..8d42c863f39e9 100644
--- a/llvm/test/Transforms/InstCombine/strto-1.ll
+++ b/llvm/test/Transforms/InstCombine/strto-1.ll
@@ -5,72 +5,72 @@
 target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:128:128"
 
 declare i32 @strtol(ptr %s, ptr %endptr, i32 %base)
-; CHECK: declare i32 @strtol(ptr readonly, ptr nocapture, i32)
+; CHECK: declare i32 @strtol(ptr readonly, ptr captures(none), i32)
 
 declare double @strtod(ptr %s, ptr %endptr)
-; CHECK: declare double @strtod(ptr readonly, ptr nocapture)
+; CHECK: declare double @strtod(ptr readonly, ptr captures(none))
 
 declare float @strtof(ptr %s, ptr %endptr)
-; CHECK: declare float @strtof(ptr readonly, ptr nocapture)
+; CHECK: declare float @strtof(ptr readonly, ptr captures(none))
 
 declare i64 @strtoul(ptr %s, ptr %endptr, i32 %base)
-; CHECK: declare i64 @strtoul(ptr readonly, ptr nocapture, i32)
+; CHECK: declare i64 @strtoul(ptr readonly, ptr captures(none), i32)
 
 declare i64 @strtoll(ptr %s, ptr %endptr, i32 %base)
-; CHECK: declare i64 @strtoll(ptr readonly, ptr nocapture, i32)
+; CHECK: declare i64 @strtoll(ptr readonly, ptr captures(none), i32)
 
 declare double @strtold(ptr %s, ptr %endptr)
-; CHECK: declare double @strtold(ptr readonly, ptr nocapture)
+; CHECK: declare double @strtold(ptr readonly, ptr captures(none))
 
 declare i64 @strtoull(ptr %s, ptr %endptr, i32 %base)
-; CHECK: declare i64 @strtoull(ptr readonly, ptr nocapture, i32)
+; CHECK: declare i64 @strtoull(ptr readonly, ptr captures(none), i32)
 
 define void @test_simplify1(ptr %x, ptr %endptr) {
 ; CHECK-LABEL: @test_simplify1(
   call i32 @strtol(ptr %x, ptr null, i32 10)
-; CHECK-NEXT: call i32 @strtol(ptr nocapture %x, ptr null, i32 10)
+; CHECK-NEXT: call i32 @strtol(ptr captures(none) %x, ptr null, i32 10)
   ret void
 }
 
 define void @test_simplify2(ptr %x, ptr %endptr) {
 ; CHECK-LABEL: @test_simplify2(
   call double @strtod(ptr %x, ptr null)
-; CHECK-NEXT: call double @strtod(ptr nocapture %x, ptr null)
+; CHECK-NEXT: call double @strtod(ptr captures(none) %x, ptr null)
   ret void
 }
 
 define void @test_simplify3(ptr %x, ptr %endptr) {
 ; CHECK-LABEL: @test_simplify3(
   call float @strtof(ptr %x, ptr null)
-; CHECK-NEXT: call float @strtof(ptr nocapture %x, ptr null)
+; CHECK-NEXT: call float @strtof(ptr captures(none) %x, ptr null)
   ret void
 }
 
 define void @test_simplify4(ptr %x, ptr %endptr) {
 ; CHECK-LABEL: @test_simplify4(
   call i64 @strtoul(ptr %x, ptr null, i32 10)
-; CHECK-NEXT: call i64 @strtoul(ptr nocapture %x, ptr null, i32 10)
+; CHECK-NEXT: call i64 @strtoul(ptr captures(none) %x, ptr null, i32 10)
   ret void
 }
 
 define void @test_simplify5(ptr %x, ptr %endptr) {
 ; CHECK-LABEL: @test_simplify5(
   call i64 @strtoll(ptr %x, ptr null, i32 10)
-; CHECK-NEXT: call i64 @strtoll(ptr nocapture %x, ptr null, i32 10)
+; CHECK-NEXT: call i64 @strtoll(ptr captures(none) %x, ptr null, i32 10)
   ret void
 }
 
 define void @test_simplify6(ptr %x, ptr %endptr) {
 ; CHECK-LABEL: @test_simplify6(
   call double @strtold(ptr %x, ptr null)
-; CHECK-NEXT: call double @strtold(ptr nocapture %x, ptr null)
+; CHECK-NEXT: call double @strtold(ptr captures(none) %x, ptr null)
   ret void
 }
 
 define void @test_simplify7(ptr %x, ptr %endptr) {
 ; CHECK-LABEL: @test_simplify7(
   call i64 @strtoull(ptr %x, ptr null, i32 10)
-; CHECK-NEXT: call i64 @strtoull(ptr nocapture %x, ptr null, i32 10)
+; CHECK-NEXT: call i64 @strtoull(ptr captures(none) %x, ptr null, i32 10)
   ret void
 }
 
diff --git a/llvm/test/Transforms/InstCombine/trivial-dse-calls.ll b/llvm/test/Transforms/InstCombine/trivial-dse-calls.ll
index feb98891035ba..128edffc1ffb0 100644
--- a/llvm/test/Transforms/InstCombine/trivial-dse-calls.ll
+++ b/llvm/test/Transforms/InstCombine/trivial-dse-calls.ll
@@ -62,7 +62,7 @@ define void @test_dead_readwrite() {
 define i32 @test_neg_read_after() {
 ; CHECK-LABEL: @test_neg_read_after(
 ; CHECK-NEXT:    [[A:%.*]] = alloca i32, align 4
-; CHECK-NEXT:    call void @f(ptr nocapture nonnull writeonly [[A]]) #[[ATTR3:[0-9]+]]
+; CHECK-NEXT:    call void @f(ptr nonnull writeonly captures(none) [[A]]) #[[ATTR3:[0-9]+]]
 ; CHECK-NEXT:    [[RES:%.*]] = load i32, ptr [[A]], align 4
 ; CHECK-NEXT:    ret i32 [[RES]]
 ;
@@ -76,7 +76,7 @@ define i32 @test_neg_read_after() {
 define void @test_neg_infinite_loop() {
 ; CHECK-LABEL: @test_neg_infinite_loop(
 ; CHECK-NEXT:    [[A:%.*]] = alloca i32, align 4
-; CHECK-NEXT:    call void @f(ptr nocapture nonnull writeonly [[A]]) #[[ATTR4:[0-9]+]]
+; CHECK-NEXT:    call void @f(ptr nonnull writeonly captures(none) [[A]]) #[[ATTR4:[0-9]+]]
 ; CHECK-NEXT:    ret void
 ;
   %a = alloca i32, align 4
@@ -87,7 +87,7 @@ define void @test_neg_infinite_loop() {
 define void @test_neg_throw() {
 ; CHECK-LABEL: @test_neg_throw(
 ; CHECK-NEXT:    [[A:%.*]] = alloca i32, align 4
-; CHECK-NEXT:    call void @f(ptr nocapture nonnull writeonly [[A]]) #[[ATTR5:[0-9]+]]
+; CHECK-NEXT:    call void @f(ptr nonnull writeonly captures(none) [[A]]) #[[ATTR5:[0-9]+]]
 ; CHECK-NEXT:    ret void
 ;
   %a = alloca i32, align 4
@@ -98,7 +98,7 @@ define void @test_neg_throw() {
 define void @test_neg_extra_write() {
 ; CHECK-LABEL: @test_neg_extra_write(
 ; CHECK-NEXT:    [[A:%.*]] = alloca i32, align 4
-; CHECK-NEXT:    call void @f(ptr nocapture nonnull writeonly [[A]]) #[[ATTR6:[0-9]+]]
+; CHECK-NEXT:    call void @f(ptr nonnull writeonly captures(none) [[A]]) #[[ATTR6:[0-9]+]]
 ; CHECK-NEXT:    ret void
 ;
   %a = alloca i32, align 4
@@ -112,7 +112,7 @@ define void @test_neg_unmodeled_write() {
 ; CHECK-LABEL: @test_neg_unmodeled_write(
 ; CHECK-NEXT:    [[A:%.*]] = alloca i32, align 4
 ; CHECK-NEXT:    [[A2:%.*]] = alloca i32, align 4
-; CHECK-NEXT:    call void @f2(ptr nocapture nonnull writeonly [[A]], ptr nonnull [[A2]]) #[[ATTR3]]
+; CHECK-NEXT:    call void @f2(ptr nonnull writeonly captures(none) [[A]], ptr nonnull [[A2]]) #[[ATTR3]]
 ; CHECK-NEXT:    ret void
 ;
   %a = alloca i32, align 4
@@ -141,7 +141,7 @@ define i32 @test_neg_captured_by_call() {
 define i32 @test_neg_captured_before() {
 ; CHECK-LABEL: @test_neg_captured_before(
 ; CHECK-NEXT:    [[A:%.*]] = alloca i32, align 4
-; CHECK-NEXT:    call void @f(ptr nocapture nonnull writeonly [[A]]) #[[ATTR3]]
+; CHECK-NEXT:    call void @f(ptr nonnull writeonly captures(none) [[A]]) #[[ATTR3]]
 ; CHECK-NEXT:    [[RES:%.*]] = load i32, ptr [[A]], align 4
 ; CHECK-NEXT:    ret i32 [[RES]]
 ;
@@ -182,7 +182,7 @@ define i8 @test_neg_unrelated_capture_used_via_return() {
 ; CHECK-LABEL: @test_neg_unrelated_capture_used_via_return(
 ; CHECK-NEXT:    [[A:%.*]] = alloca i32, align 4
 ; CHECK-NEXT:    [[A2:%.*]] = alloca i32, align 4
-; CHECK-NEXT:    [[CAPTURE:%.*]] = call ptr @f3(ptr nocapture nonnull writeonly [[A]], ptr nonnull readonly [[A2]]) #[[ATTR3]]
+; CHECK-NEXT:    [[CAPTURE:%.*]] = call ptr @f3(ptr nonnull writeonly captures(none) [[A]], ptr nonnull readonly [[A2]]) #[[ATTR3]]
 ; CHECK-NEXT:    [[V:%.*]] = load i8, ptr [[CAPTURE]], align 1
 ; CHECK-NEXT:    ret i8 [[V]]
 ;
diff --git a/llvm/test/Transforms/InstCombine/unused-nonnull.ll b/llvm/test/Transforms/InstCombine/unused-nonnull.ll
index 32dd65977ea86..af468593b887c 100644
--- a/llvm/test/Transforms/InstCombine/unused-nonnull.ll
+++ b/llvm/test/Transforms/InstCombine/unused-nonnull.ll
@@ -9,7 +9,7 @@ target triple = "x86_64-unknown-linux-gnu"
 
 define i32 @main(i32 %argc, ptr %argv) #0 {
 ; CHECK-LABEL: define {{[^@]+}}@main
-; CHECK-SAME: (i32 [[ARGC:%.*]], ptr nocapture readonly [[ARGV:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
+; CHECK-SAME: (i32 [[ARGC:%.*]], ptr readonly captures(none) [[ARGV:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[TMP0:%.*]] = icmp slt i32 [[ARGC]], 2
 ; CHECK-NEXT:    [[SPEC_SELECT:%.*]] = select i1 [[TMP0]], i32 0, i32 [[ARGC]]
@@ -36,7 +36,7 @@ done:
 
 define i32 @compute(ptr noundef nonnull %ptr, i32 %x) #1 {
 ; CHECK-LABEL: define {{[^@]+}}@compute
-; CHECK-SAME: (ptr nocapture noundef nonnull readnone [[PTR:%.*]], i32 returned [[X:%.*]]) local_unnamed_addr #[[ATTR1:[0-9]+]] {
+; CHECK-SAME: (ptr noundef nonnull readnone captures(none) [[PTR:%.*]], i32 returned [[X:%.*]]) local_unnamed_addr #[[ATTR1:[0-9]+]] {
 ; CHECK-NEXT:    ret i32 [[X]]
 ;
   ret i32 %x
diff --git a/llvm/test/Transforms/InstCombine/xor-icmps.ll b/llvm/test/Transforms/InstCombine/xor-icmps.ll
index 0384c1aa184b8..55af2116aacd1 100644
--- a/llvm/test/Transforms/InstCombine/xor-icmps.ll
+++ b/llvm/test/Transforms/InstCombine/xor-icmps.ll
@@ -319,3 +319,194 @@ define i1 @xor_icmp_to_icmp_add_multiuse2(i32 %a) {
   %cmp3 = xor i1 %cmp, %cmp1
   ret i1 %cmp3
 }
+
+define i1 @test_xor_of_bittest_ne_ne(i8 %x, i8 %y) {
+; CHECK-LABEL: @test_xor_of_bittest_ne_ne(
+; CHECK-NEXT:    [[Y:%.*]] = xor i8 [[X:%.*]], [[Y1:%.*]]
+; CHECK-NEXT:    [[MASK2:%.*]] = and i8 [[Y]], 2
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp ne i8 [[MASK2]], 0
+; CHECK-NEXT:    ret i1 [[CMP2]]
+;
+  %mask1 = and i8 %x, 2
+  %cmp1 = icmp ne i8 %mask1, 0
+  %mask2 = and i8 %y, 2
+  %cmp2 = icmp ne i8 %mask2, 0
+  %xor = xor i1 %cmp1, %cmp2
+  ret i1 %xor
+}
+
+define i1 @test_xor_of_bittest_ne_ne_var_pow2(i8 %x, i8 %y, i8 %shamt) {
+; CHECK-LABEL: @test_xor_of_bittest_ne_ne_var_pow2(
+; CHECK-NEXT:    [[POW2:%.*]] = shl nuw i8 1, [[SHAMT:%.*]]
+; CHECK-NEXT:    [[Y:%.*]] = xor i8 [[X:%.*]], [[Y1:%.*]]
+; CHECK-NEXT:    [[MASK2:%.*]] = and i8 [[Y]], [[POW2]]
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp ne i8 [[MASK2]], 0
+; CHECK-NEXT:    ret i1 [[CMP2]]
+;
+  %pow2 = shl nuw i8 1, %shamt
+  %mask1 = and i8 %x, %pow2
+  %cmp1 = icmp ne i8 %mask1, 0
+  %mask2 = and i8 %y, %pow2
+  %cmp2 = icmp ne i8 %mask2, 0
+  %xor = xor i1 %cmp1, %cmp2
+  ret i1 %xor
+}
+
+define i1 @test_xor_of_bittest_ne_ne_var_pow2_or_zero(i8 %x, i8 %y, i8 %z) {
+; CHECK-LABEL: @test_xor_of_bittest_ne_ne_var_pow2_or_zero(
+; CHECK-NEXT:    [[NZ:%.*]] = sub i8 0, [[Z:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = xor i8 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[TMP2:%.*]] = and i8 [[TMP1]], [[NZ]]
+; CHECK-NEXT:    [[MASK2:%.*]] = and i8 [[TMP2]], [[Z]]
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp ne i8 [[MASK2]], 0
+; CHECK-NEXT:    ret i1 [[CMP2]]
+;
+  %nz = sub i8 0, %z
+  %pow2 = and i8 %z, %nz
+  %mask1 = and i8 %x, %pow2
+  %cmp1 = icmp ne i8 %mask1, 0
+  %mask2 = and i8 %y, %pow2
+  %cmp2 = icmp ne i8 %mask2, 0
+  %xor = xor i1 %cmp1, %cmp2
+  ret i1 %xor
+}
+
+define i1 @test_xor_of_bittest_eq_eq(i8 %x, i8 %y) {
+; CHECK-LABEL: @test_xor_of_bittest_eq_eq(
+; CHECK-NEXT:    [[Y:%.*]] = xor i8 [[X:%.*]], [[Y1:%.*]]
+; CHECK-NEXT:    [[MASK2:%.*]] = and i8 [[Y]], 2
+; CHECK-NEXT:    [[XOR:%.*]] = icmp ne i8 [[MASK2]], 0
+; CHECK-NEXT:    ret i1 [[XOR]]
+;
+  %mask1 = and i8 %x, 2
+  %cmp1 = icmp eq i8 %mask1, 0
+  %mask2 = and i8 %y, 2
+  %cmp2 = icmp eq i8 %mask2, 0
+  %xor = xor i1 %cmp1, %cmp2
+  ret i1 %xor
+}
+
+define i1 @test_xor_of_bittest_ne_eq(i8 %x, i8 %y) {
+; CHECK-LABEL: @test_xor_of_bittest_ne_eq(
+; CHECK-NEXT:    [[Y:%.*]] = xor i8 [[X:%.*]], [[Y1:%.*]]
+; CHECK-NEXT:    [[MASK2:%.*]] = and i8 [[Y]], 2
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp eq i8 [[MASK2]], 0
+; CHECK-NEXT:    ret i1 [[CMP2]]
+;
+  %mask1 = and i8 %x, 2
+  %cmp1 = icmp ne i8 %mask1, 0
+  %mask2 = and i8 %y, 2
+  %cmp2 = icmp eq i8 %mask2, 0
+  %xor = xor i1 %cmp1, %cmp2
+  ret i1 %xor
+}
+
+define i1 @test_xor_of_bittest_eq_ne(i8 %x, i8 %y) {
+; CHECK-LABEL: @test_xor_of_bittest_eq_ne(
+; CHECK-NEXT:    [[X:%.*]] = xor i8 [[X1:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[MASK1:%.*]] = and i8 [[X]], 2
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp eq i8 [[MASK1]], 0
+; CHECK-NEXT:    ret i1 [[CMP1]]
+;
+  %mask1 = and i8 %x, 2
+  %cmp1 = icmp eq i8 %mask1, 0
+  %mask2 = and i8 %y, 2
+  %cmp2 = icmp ne i8 %mask2, 0
+  %xor = xor i1 %cmp1, %cmp2
+  ret i1 %xor
+}
+
+define i1 @test_xor_of_bittest_ne_ne_multiuse1(i8 %x, i8 %y) {
+; CHECK-LABEL: @test_xor_of_bittest_ne_ne_multiuse1(
+; CHECK-NEXT:    [[MASK1:%.*]] = and i8 [[X:%.*]], 2
+; CHECK-NEXT:    call void @usei8(i8 [[MASK1]])
+; CHECK-NEXT:    [[MASK2:%.*]] = and i8 [[Y:%.*]], 2
+; CHECK-NEXT:    call void @usei8(i8 [[MASK2]])
+; CHECK-NEXT:    [[TMP1:%.*]] = xor i8 [[X]], [[Y]]
+; CHECK-NEXT:    [[TMP2:%.*]] = and i8 [[TMP1]], 2
+; CHECK-NEXT:    [[XOR:%.*]] = icmp ne i8 [[TMP2]], 0
+; CHECK-NEXT:    ret i1 [[XOR]]
+;
+  %mask1 = and i8 %x, 2
+  call void @usei8(i8 %mask1)
+  %cmp1 = icmp ne i8 %mask1, 0
+  %mask2 = and i8 %y, 2
+  call void @usei8(i8 %mask2)
+  %cmp2 = icmp ne i8 %mask2, 0
+  %xor = xor i1 %cmp1, %cmp2
+  ret i1 %xor
+}
+
+; Negative tests
+
+define i1 @test_xor_of_bittest_ne_ne_type_mismatch(i8 %x, i16 %y) {
+; CHECK-LABEL: @test_xor_of_bittest_ne_ne_type_mismatch(
+; CHECK-NEXT:    [[MASK1:%.*]] = and i8 [[X:%.*]], 2
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp ne i8 [[MASK1]], 0
+; CHECK-NEXT:    [[MASK2:%.*]] = and i16 [[Y:%.*]], 2
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp ne i16 [[MASK2]], 0
+; CHECK-NEXT:    [[XOR:%.*]] = xor i1 [[CMP1]], [[CMP2]]
+; CHECK-NEXT:    ret i1 [[XOR]]
+;
+  %mask1 = and i8 %x, 2
+  %cmp1 = icmp ne i8 %mask1, 0
+  %mask2 = and i16 %y, 2
+  %cmp2 = icmp ne i16 %mask2, 0
+  %xor = xor i1 %cmp1, %cmp2
+  ret i1 %xor
+}
+
+define i1 @test_xor_of_bittest_ne_ne_mask_mismatch(i8 %x, i8 %y) {
+; CHECK-LABEL: @test_xor_of_bittest_ne_ne_mask_mismatch(
+; CHECK-NEXT:    [[MASK1:%.*]] = and i8 [[X:%.*]], 4
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp ne i8 [[MASK1]], 0
+; CHECK-NEXT:    [[MASK2:%.*]] = and i8 [[Y:%.*]], 2
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp ne i8 [[MASK2]], 0
+; CHECK-NEXT:    [[XOR:%.*]] = xor i1 [[CMP1]], [[CMP2]]
+; CHECK-NEXT:    ret i1 [[XOR]]
+;
+  %mask1 = and i8 %x, 4
+  %cmp1 = icmp ne i8 %mask1, 0
+  %mask2 = and i8 %y, 2
+  %cmp2 = icmp ne i8 %mask2, 0
+  %xor = xor i1 %cmp1, %cmp2
+  ret i1 %xor
+}
+
+define i1 @test_xor_of_bittest_ne_ne_nonpower2(i8 %x, i8 %y) {
+; CHECK-LABEL: @test_xor_of_bittest_ne_ne_nonpower2(
+; CHECK-NEXT:    [[MASK1:%.*]] = and i8 [[X:%.*]], 3
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp ne i8 [[MASK1]], 0
+; CHECK-NEXT:    [[MASK2:%.*]] = and i8 [[Y:%.*]], 3
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp ne i8 [[MASK2]], 0
+; CHECK-NEXT:    [[XOR:%.*]] = xor i1 [[CMP1]], [[CMP2]]
+; CHECK-NEXT:    ret i1 [[XOR]]
+;
+  %mask1 = and i8 %x, 3
+  %cmp1 = icmp ne i8 %mask1, 0
+  %mask2 = and i8 %y, 3
+  %cmp2 = icmp ne i8 %mask2, 0
+  %xor = xor i1 %cmp1, %cmp2
+  ret i1 %xor
+}
+
+define i1 @test_xor_of_bittest_ne_ne_multiuse2(i8 %x, i8 %y) {
+; CHECK-LABEL: @test_xor_of_bittest_ne_ne_multiuse2(
+; CHECK-NEXT:    [[MASK1:%.*]] = and i8 [[X:%.*]], 2
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp ne i8 [[MASK1]], 0
+; CHECK-NEXT:    call void @use(i1 [[CMP1]])
+; CHECK-NEXT:    [[MASK2:%.*]] = and i8 [[Y:%.*]], 2
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp ne i8 [[MASK2]], 0
+; CHECK-NEXT:    [[XOR:%.*]] = xor i1 [[CMP1]], [[CMP2]]
+; CHECK-NEXT:    ret i1 [[XOR]]
+;
+  %mask1 = and i8 %x, 2
+  %cmp1 = icmp ne i8 %mask1, 0
+  call void @use(i1 %cmp1)
+  %mask2 = and i8 %y, 2
+  %cmp2 = icmp ne i8 %mask2, 0
+  %xor = xor i1 %cmp1, %cmp2
+  ret i1 %xor
+}
+
+declare void @usei8(i8)
diff --git a/llvm/test/Transforms/InstSimplify/assume-non-zero.ll b/llvm/test/Transforms/InstSimplify/assume-non-zero.ll
index 9176b8101da65..22bdf374487da 100644
--- a/llvm/test/Transforms/InstSimplify/assume-non-zero.ll
+++ b/llvm/test/Transforms/InstSimplify/assume-non-zero.ll
@@ -231,3 +231,17 @@ define i1 @nonnull17_unknown(i8 %x) {
   %q = icmp ne i8 %x, 0
   ret i1 %q
 }
+
+define i1 @nonnull_trunc_true(i8 %x) {
+; CHECK-LABEL: @nonnull_trunc_true(
+; CHECK-NEXT:    [[A:%.*]] = trunc i8 [[X:%.*]] to i1
+; CHECK-NEXT:    call void @llvm.assume(i1 [[A]])
+; CHECK-NEXT:    [[Q:%.*]] = icmp ne i8 [[X]], 0
+; CHECK-NEXT:    ret i1 [[Q]]
+;
+  %a = trunc i8 %x to i1
+  call void @llvm.assume(i1 %a)
+  %q = icmp ne i8 %x, 0
+  ret i1 %q
+}
+
diff --git a/llvm/test/Transforms/InstSimplify/compare.ll b/llvm/test/Transforms/InstSimplify/compare.ll
index 222b25836e053..4192a59dc7134 100644
--- a/llvm/test/Transforms/InstSimplify/compare.ll
+++ b/llvm/test/Transforms/InstSimplify/compare.ll
@@ -3450,6 +3450,21 @@ define i1 @icmp_ult_vscale_false(i8 %x, i8 %y) {
   ret i1 %cmp
 }
 
+define i1 @icmp_eq_false_by_trunc(i8 %x) {
+; CHECK-LABEL: @icmp_eq_false_by_trunc(
+; CHECK-NEXT:    [[TRUNC:%.*]] = trunc i8 [[X:%.*]] to i1
+; CHECK-NEXT:    [[NOT:%.*]] = xor i1 [[TRUNC]], true
+; CHECK-NEXT:    call void @llvm.assume(i1 [[NOT]])
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i8 [[X]], 1
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %trunc = trunc i8 %x to i1
+  %not = xor i1 %trunc, true
+  call void @llvm.assume(i1 %not)
+  %cmp = icmp eq i8 %x, 1
+  ret i1 %cmp
+}
+
 declare i64 @llvm.vscale.i64()
 
 ; TODO: Add coverage for global aliases, link once, etc..
diff --git a/llvm/test/Transforms/InstSimplify/constant-fold-inttoptr-add.ll b/llvm/test/Transforms/InstSimplify/constant-fold-inttoptr-add.ll
new file mode 100644
index 0000000000000..34dee39463d9b
--- /dev/null
+++ b/llvm/test/Transforms/InstSimplify/constant-fold-inttoptr-add.ll
@@ -0,0 +1,114 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -p instsimplify -S %s | FileCheck %s
+
+@glob = external global [314 x i64]
+
+define i1 @known_constexpr_add_eq() {
+; CHECK-LABEL: define i1 @known_constexpr_add_eq() {
+; CHECK-NEXT:    ret i1 false
+;
+  %cond = icmp eq ptr getelementptr inbounds nuw (i8, ptr @glob, i64 80), inttoptr (i64 add (i64 ptrtoint (ptr @glob to i64), i64 -80) to ptr)
+  ret i1 %cond
+}
+
+define i1 @known_constexpr_add_eq_ops_swapped() {
+; CHECK-LABEL: define i1 @known_constexpr_add_eq_ops_swapped() {
+; CHECK-NEXT:    ret i1 false
+;
+  %cond = icmp eq ptr inttoptr (i64 add (i64 ptrtoint (ptr @glob to i64), i64 -80) to ptr), getelementptr inbounds nuw (i8, ptr @glob, i64 80)
+  ret i1 %cond
+}
+
+define i1 @known_constexpr_add_ne() {
+; CHECK-LABEL: define i1 @known_constexpr_add_ne() {
+; CHECK-NEXT:    ret i1 true
+;
+  %cond = icmp ne ptr getelementptr inbounds nuw (i8, ptr @glob, i64 80), inttoptr (i64 add (i64 ptrtoint (ptr @glob to i64), i64 -80) to ptr)
+  ret i1 %cond
+}
+
+define i1 @wrap_positive_to_negate() {
+; CHECK-LABEL: define i1 @wrap_positive_to_negate() {
+; CHECK-NEXT:    ret i1 false
+;
+  %cond = icmp eq ptr @glob, inttoptr (i64 add (i64 ptrtoint (ptr getelementptr nuw (i8, ptr @glob, i64 1)to i64), i64 9223372036854775808) to ptr)
+  ret i1 %cond
+}
+
+; 9223372036854775808 = 2^63
+define i1 @wrap_positive_to_zero() {
+; CHECK-LABEL: define i1 @wrap_positive_to_zero() {
+; CHECK-NEXT:    ret i1 true
+;
+  %cond = icmp eq ptr @glob, inttoptr (i64 add (i64 ptrtoint (ptr getelementptr nuw (i8, ptr @glob, i64 9223372036854775808)to i64), i64 9223372036854775808) to ptr)
+  ret i1 %cond
+}
+
+define i1 @not_known_constexpr_add_sle() {
+; CHECK-LABEL: define i1 @not_known_constexpr_add_sle() {
+; CHECK-NEXT:    [[COND:%.*]] = icmp sle ptr @glob, inttoptr (i64 add (i64 ptrtoint (ptr @glob to i64), i64 -80) to ptr)
+; CHECK-NEXT:    ret i1 [[COND]]
+;
+  %cond = icmp sle ptr @glob, inttoptr (i64 add (i64 ptrtoint (ptr @glob to i64), i64 -80) to ptr)
+  ret i1 %cond
+}
+
+define i1 @not_known_constexpr_add_ule() {
+; CHECK-LABEL: define i1 @not_known_constexpr_add_ule() {
+; CHECK-NEXT:    [[COND:%.*]] = icmp ule ptr @glob, inttoptr (i64 add (i64 ptrtoint (ptr @glob to i64), i64 -80) to ptr)
+; CHECK-NEXT:    ret i1 [[COND]]
+;
+  %cond = icmp ule ptr @glob, inttoptr (i64 add (i64 ptrtoint (ptr @glob to i64), i64 -80) to ptr)
+  ret i1 %cond
+}
+
+define i1 @not_known_constexpr_add_sge() {
+; CHECK-LABEL: define i1 @not_known_constexpr_add_sge() {
+; CHECK-NEXT:    [[COND:%.*]] = icmp sge ptr @glob, inttoptr (i64 add (i64 ptrtoint (ptr @glob to i64), i64 -80) to ptr)
+; CHECK-NEXT:    ret i1 [[COND]]
+;
+  %cond = icmp sge ptr @glob, inttoptr (i64 add (i64 ptrtoint (ptr @glob to i64), i64 -80) to ptr)
+  ret i1 %cond
+}
+
+define i1 @not_known_constexpr_add_uge() {
+; CHECK-LABEL: define i1 @not_known_constexpr_add_uge() {
+; CHECK-NEXT:    [[COND:%.*]] = icmp uge ptr @glob, inttoptr (i64 add (i64 ptrtoint (ptr @glob to i64), i64 -80) to ptr)
+; CHECK-NEXT:    ret i1 [[COND]]
+;
+  %cond = icmp uge ptr @glob, inttoptr (i64 add (i64 ptrtoint (ptr @glob to i64), i64 -80) to ptr)
+  ret i1 %cond
+}
+
+define i1 @not_known_integer_width_does_not_match_pointer_index_width() {
+; CHECK-LABEL: define i1 @not_known_integer_width_does_not_match_pointer_index_width() {
+; CHECK-NEXT:    [[COND:%.*]] = icmp eq ptr getelementptr inbounds nuw (i8, ptr @glob, i64 80), inttoptr (i32 add (i32 ptrtoint (ptr @glob to i32), i32 -80) to ptr)
+; CHECK-NEXT:    ret i1 [[COND]]
+;
+  %cond = icmp eq ptr getelementptr inbounds nuw (i8, ptr @glob, i64 80), inttoptr (i32 add (i32 ptrtoint (ptr @glob to i32), i32 -80) to ptr)
+  ret i1 %cond
+}
+
+define ptr @return_inttoptr() {
+; CHECK-LABEL: define ptr @return_inttoptr() {
+; CHECK-NEXT:    ret ptr inttoptr (i64 add (i64 ptrtoint (ptr @glob to i64), i64 8) to ptr)
+;
+  ret ptr inttoptr (i64 add (i64 ptrtoint (ptr @glob to i64), i64 8) to ptr)
+}
+
+define i1 @known_constexpr_add_nested_1() {
+; CHECK-LABEL: define i1 @known_constexpr_add_nested_1() {
+; CHECK-NEXT:    ret i1 true
+;
+  %cond = icmp eq ptr @glob, inttoptr (i64 add (i64 ptrtoint (ptr getelementptr inbounds nuw (i8, ptr @glob, i64 80) to i64), i64 -80) to ptr)
+  ret i1 %cond
+}
+
+define i1 @known_constexpr_add_nested_2() {
+; CHECK-LABEL: define i1 @known_constexpr_add_nested_2() {
+; CHECK-NEXT:    ret i1 true
+;
+  ;%cond = icmp eq ptr @glob, ptr getelementptr inbounds nuw (i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @glob to i64), i64 -80) to ptr), i64 80)
+  %cond = icmp eq ptr @glob, getelementptr inbounds nuw (i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @glob to i64), i64 -80) to ptr), i64 80)
+  ret i1 %cond
+}
diff --git a/llvm/test/Transforms/InstSimplify/known-non-zero-opaque-ptrs.ll b/llvm/test/Transforms/InstSimplify/known-non-zero-opaque-ptrs.ll
index 92084196d9553..941b31fd1198c 100644
--- a/llvm/test/Transforms/InstSimplify/known-non-zero-opaque-ptrs.ll
+++ b/llvm/test/Transforms/InstSimplify/known-non-zero-opaque-ptrs.ll
@@ -4,12 +4,10 @@
 declare void @zero_args()
 declare void @two_args(ptr, ptr)
 
-; TODO: Could be non-null based on call-site attributes.
 define i1 @test_zero_args_nonnull(ptr %p) {
 ; CHECK-LABEL: @test_zero_args_nonnull(
 ; CHECK-NEXT:    call void @zero_args(ptr noundef nonnull [[P:%.*]])
-; CHECK-NEXT:    [[C:%.*]] = icmp ne ptr [[P]], null
-; CHECK-NEXT:    ret i1 [[C]]
+; CHECK-NEXT:    ret i1 true
 ;
   call void @zero_args(ptr nonnull noundef %p)
   %c = icmp ne ptr %p, null
@@ -27,12 +25,10 @@ define i1 @test_zero_args_maybe_null(ptr %p) {
   ret i1 %c
 }
 
-; TODO: Could be non-null based on call-site attributes.
 define i1 @test_two_args_nonnull(ptr %p) {
 ; CHECK-LABEL: @test_two_args_nonnull(
 ; CHECK-NEXT:    call void @two_args(ptr noundef nonnull [[P:%.*]])
-; CHECK-NEXT:    [[C:%.*]] = icmp ne ptr [[P]], null
-; CHECK-NEXT:    ret i1 [[C]]
+; CHECK-NEXT:    ret i1 true
 ;
   call void @two_args(ptr nonnull noundef %p)
   %c = icmp ne ptr %p, null
diff --git a/llvm/test/Transforms/InstSimplify/select-icmp.ll b/llvm/test/Transforms/InstSimplify/select-icmp.ll
index a6ef937760a58..64c0d1d7553fe 100755
--- a/llvm/test/Transforms/InstSimplify/select-icmp.ll
+++ b/llvm/test/Transforms/InstSimplify/select-icmp.ll
@@ -244,3 +244,35 @@ cond.true:                                        ; preds = %entry
 cond.end:                                         ; preds = %entry, %cond.true
   ret i8 0
 }
+
+define ptr @icmp_ptr_eq_replace(ptr %a, ptr %b) {
+; CHECK-LABEL: @icmp_ptr_eq_replace(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq ptr [[A:%.*]], [[B1:%.*]]
+; CHECK-NEXT:    [[B:%.*]] = select i1 [[CMP]], ptr [[A]], ptr [[B1]]
+; CHECK-NEXT:    ret ptr [[B]]
+;
+  %cmp = icmp eq ptr %a, %b
+  %sel = select i1 %cmp, ptr %a, ptr %b
+  ret ptr %sel
+}
+
+define ptr @icmp_ptr_eq_replace_null(ptr %a) {
+; CHECK-LABEL: @icmp_ptr_eq_replace_null(
+; CHECK-NEXT:    ret ptr [[A:%.*]]
+;
+  %cmp = icmp eq ptr %a, null
+  %sel = select i1 %cmp, ptr null, ptr %a
+  ret ptr %sel
+}
+
+define ptr @ptr_eq_replace_same_underlying_object(ptr %st, i64 %i, i64 %j) {
+; CHECK-LABEL: @ptr_eq_replace_same_underlying_object(
+; CHECK-NEXT:    [[B:%.*]] = getelementptr inbounds i8, ptr [[ST:%.*]], i64 [[J:%.*]]
+; CHECK-NEXT:    ret ptr [[B]]
+;
+  %a = getelementptr inbounds i8, ptr %st, i64 %i
+  %b = getelementptr inbounds i8, ptr %st, i64 %j
+  %cmp = icmp eq ptr %a, %b
+  %sel = select i1 %cmp, ptr %a, ptr %b
+  ret ptr %sel
+}
diff --git a/llvm/test/Transforms/InstSimplify/select.ll b/llvm/test/Transforms/InstSimplify/select.ll
index 40539b8ade388..1b5703a46cf68 100644
--- a/llvm/test/Transforms/InstSimplify/select.ll
+++ b/llvm/test/Transforms/InstSimplify/select.ll
@@ -1752,10 +1752,7 @@ define <4 x i32> @select_vector_cmp_with_bitcasts(<2 x i64> %x, <4 x i32> %y) {
 
 define i8 @bittest_trunc_or(i8 %x) {
 ; CHECK-LABEL: @bittest_trunc_or(
-; CHECK-NEXT:    [[TRUNC:%.*]] = trunc i8 [[X1:%.*]] to i1
-; CHECK-NEXT:    [[OR:%.*]] = or i8 [[X1]], 1
-; CHECK-NEXT:    [[X:%.*]] = select i1 [[TRUNC]], i8 [[OR]], i8 [[X1]]
-; CHECK-NEXT:    ret i8 [[X]]
+; CHECK-NEXT:    ret i8 [[X:%.*]]
 ;
   %trunc = trunc i8 %x to i1
   %or = or i8 %x, 1
@@ -1765,11 +1762,8 @@ define i8 @bittest_trunc_or(i8 %x) {
 
 define i8 @bittest_trunc_not_or(i8 %x) {
 ; CHECK-LABEL: @bittest_trunc_not_or(
-; CHECK-NEXT:    [[TRUNC:%.*]] = trunc i8 [[X:%.*]] to i1
-; CHECK-NEXT:    [[NOT:%.*]] = xor i1 [[TRUNC]], true
-; CHECK-NEXT:    [[OR:%.*]] = or i8 [[X]], 1
-; CHECK-NEXT:    [[COND:%.*]] = select i1 [[NOT]], i8 [[OR]], i8 [[X]]
-; CHECK-NEXT:    ret i8 [[COND]]
+; CHECK-NEXT:    [[OR:%.*]] = or i8 [[X:%.*]], 1
+; CHECK-NEXT:    ret i8 [[OR]]
 ;
   %trunc = trunc i8 %x to i1
   %not = xor i1 %trunc, true
diff --git a/llvm/test/Transforms/InstSimplify/shr-nop.ll b/llvm/test/Transforms/InstSimplify/shr-nop.ll
index 29fe222faaae3..4b9292aab4bb3 100644
--- a/llvm/test/Transforms/InstSimplify/shr-nop.ll
+++ b/llvm/test/Transforms/InstSimplify/shr-nop.ll
@@ -381,6 +381,19 @@ define i32 @exact_lshr_lowbit(i32 %shiftval) {
   ret i32 %shr
 }
 
+define i8 @exact_lshr_lowbit_set_assume_trunc(i8 %x) {
+; CHECK-LABEL: @exact_lshr_lowbit_set_assume_trunc(
+; CHECK-NEXT:    [[COND:%.*]] = trunc i8 [[X:%.*]] to i1
+; CHECK-NEXT:    call void @llvm.assume(i1 [[COND]])
+; CHECK-NEXT:    [[SHR:%.*]] = lshr exact i8 [[X]], 1
+; CHECK-NEXT:    ret i8 [[SHR]]
+;
+  %cond = trunc i8 %x to i1
+  call void @llvm.assume(i1 %cond)
+  %shr = lshr exact i8 %x, 1
+  ret i8 %shr
+}
+
 define i32 @exact_ashr_lowbit(i32 %shiftval) {
 ; CHECK-LABEL: @exact_ashr_lowbit(
 ; CHECK-NEXT:    ret i32 7
diff --git a/llvm/test/Transforms/LICM/hoist-mustexec.ll b/llvm/test/Transforms/LICM/hoist-mustexec.ll
index a6f5a2be05ee4..9acb39147bf78 100644
--- a/llvm/test/Transforms/LICM/hoist-mustexec.ll
+++ b/llvm/test/Transforms/LICM/hoist-mustexec.ll
@@ -10,7 +10,7 @@ declare void @llvm.experimental.guard(i1,...)
 ; constant fold on first ieration
 define i32 @test1(ptr noalias nocapture readonly %a) nounwind uwtable {
 ; CHECK-LABEL: define i32 @test1(
-; CHECK-SAME: ptr noalias nocapture readonly [[A:%.*]]) #[[ATTR1:[0-9]+]] {
+; CHECK-SAME: ptr noalias readonly captures(none) [[A:%.*]]) #[[ATTR1:[0-9]+]] {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[I1:%.*]] = load i32, ptr [[A]], align 4
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
@@ -57,7 +57,7 @@ fail:
 ; Same as test1, but with a floating point IR and fcmp
 define i32 @test_fcmp(ptr noalias nocapture readonly %a) nounwind uwtable {
 ; CHECK-LABEL: define i32 @test_fcmp(
-; CHECK-SAME: ptr noalias nocapture readonly [[A:%.*]]) #[[ATTR1]] {
+; CHECK-SAME: ptr noalias readonly captures(none) [[A:%.*]]) #[[ATTR1]] {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[I1:%.*]] = load i32, ptr [[A]], align 4
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
@@ -106,7 +106,7 @@ fail:
 ; ule i32 (add nsw i32 %len, -1), %len where len is [0, 512]
 define i32 @test2(ptr noalias nocapture readonly %a) nounwind uwtable {
 ; CHECK-LABEL: define i32 @test2(
-; CHECK-SAME: ptr noalias nocapture readonly [[A:%.*]]) #[[ATTR1]] {
+; CHECK-SAME: ptr noalias readonly captures(none) [[A:%.*]]) #[[ATTR1]] {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[LEN:%.*]] = load i32, ptr [[A]], align 4, !range [[RNG0:![0-9]+]]
 ; CHECK-NEXT:    [[IS_NON_POS:%.*]] = icmp eq i32 [[LEN]], 0
@@ -164,7 +164,7 @@ fail:
 ; trivially true for zero
 define i32 @test3(ptr noalias nocapture readonly %a) nounwind uwtable {
 ; CHECK-LABEL: define i32 @test3(
-; CHECK-SAME: ptr noalias nocapture readonly [[A:%.*]]) #[[ATTR1]] {
+; CHECK-SAME: ptr noalias readonly captures(none) [[A:%.*]]) #[[ATTR1]] {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[LEN:%.*]] = load i32, ptr [[A]], align 4, !range [[RNG0]]
 ; CHECK-NEXT:    [[IS_ZERO:%.*]] = icmp eq i32 [[LEN]], 0
@@ -220,7 +220,7 @@ fail:
 ; Same as previous case, with commuted icmp.
 define i32 @test3_commuted(ptr noalias nocapture readonly %a) nounwind uwtable {
 ; CHECK-LABEL: define i32 @test3_commuted(
-; CHECK-SAME: ptr noalias nocapture readonly [[A:%.*]]) #[[ATTR1]] {
+; CHECK-SAME: ptr noalias readonly captures(none) [[A:%.*]]) #[[ATTR1]] {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[LEN:%.*]] = load i32, ptr [[A]], align 4, !range [[RNG0]]
 ; CHECK-NEXT:    [[IS_ZERO:%.*]] = icmp eq i32 [[LEN]], 0
@@ -276,7 +276,7 @@ fail:
 ; requires fact length is non-zero
 define i32 @test4(ptr noalias nocapture readonly %a) nounwind uwtable {
 ; CHECK-LABEL: define i32 @test4(
-; CHECK-SAME: ptr noalias nocapture readonly [[A:%.*]]) #[[ATTR1]] {
+; CHECK-SAME: ptr noalias readonly captures(none) [[A:%.*]]) #[[ATTR1]] {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[LEN:%.*]] = load i32, ptr [[A]], align 4, !range [[RNG0]]
 ; CHECK-NEXT:    [[IS_ZERO:%.*]] = icmp eq i32 [[LEN]], 0
@@ -332,7 +332,7 @@ fail:
 ; variation on test1 with branch swapped
 define i32 @test-brswap(ptr noalias nocapture readonly %a) nounwind uwtable {
 ; CHECK-LABEL: define i32 @test-brswap(
-; CHECK-SAME: ptr noalias nocapture readonly [[A:%.*]]) #[[ATTR1]] {
+; CHECK-SAME: ptr noalias readonly captures(none) [[A:%.*]]) #[[ATTR1]] {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[I1:%.*]] = load i32, ptr [[A]], align 4
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
@@ -378,7 +378,7 @@ fail:
 
 define i32 @test-nonphi(ptr noalias nocapture readonly %a) nounwind uwtable {
 ; CHECK-LABEL: define i32 @test-nonphi(
-; CHECK-SAME: ptr noalias nocapture readonly [[A:%.*]]) #[[ATTR1]] {
+; CHECK-SAME: ptr noalias readonly captures(none) [[A:%.*]]) #[[ATTR1]] {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK:       for.body:
@@ -426,7 +426,7 @@ fail:
 
 define i32 @test-wrongphi(ptr noalias nocapture readonly %a) nounwind uwtable {
 ; CHECK-LABEL: define i32 @test-wrongphi(
-; CHECK-SAME: ptr noalias nocapture readonly [[A:%.*]]) #[[ATTR1]] {
+; CHECK-SAME: ptr noalias readonly captures(none) [[A:%.*]]) #[[ATTR1]] {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK:       for.body:
@@ -487,7 +487,7 @@ fail:
 ; This works because loop-simplify is run implicitly, but test for it anyways
 define i32 @test-multiple-latch(ptr noalias nocapture readonly %a) nounwind uwtable {
 ; CHECK-LABEL: define i32 @test-multiple-latch(
-; CHECK-SAME: ptr noalias nocapture readonly [[A:%.*]]) #[[ATTR1]] {
+; CHECK-SAME: ptr noalias readonly captures(none) [[A:%.*]]) #[[ATTR1]] {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[I1:%.*]] = load i32, ptr [[A]], align 4
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
diff --git a/llvm/test/Transforms/LICM/strlen.ll b/llvm/test/Transforms/LICM/strlen.ll
index 5e9f09a0971fc..e758b2f86cdd4 100644
--- a/llvm/test/Transforms/LICM/strlen.ll
+++ b/llvm/test/Transforms/LICM/strlen.ll
@@ -12,7 +12,7 @@ loop:
   br label %loop
 }
 
-; CHECK: declare i64 @strlen(ptr nocapture) #0
+; CHECK: declare i64 @strlen(ptr captures(none)) #0
 ; CHECK: attributes #0 = { mustprogress nofree nounwind willreturn memory(argmem: read) }
 declare i64 @strlen(ptr)
 
diff --git a/llvm/test/Transforms/LoopDataPrefetch/RISCV/basic.ll b/llvm/test/Transforms/LoopDataPrefetch/RISCV/basic.ll
index 8e1123991a9cc..71d9953a03da1 100644
--- a/llvm/test/Transforms/LoopDataPrefetch/RISCV/basic.ll
+++ b/llvm/test/Transforms/LoopDataPrefetch/RISCV/basic.ll
@@ -4,7 +4,7 @@
 
 define void @foo(ptr nocapture %a, ptr nocapture readonly %b) {
 ; CHECK-LABEL: define void @foo(
-; CHECK-SAME: ptr nocapture [[A:%.*]], ptr nocapture readonly [[B:%.*]]) {
+; CHECK-SAME: ptr captures(none) [[A:%.*]], ptr readonly captures(none) [[B:%.*]]) {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK:       for.body:
diff --git a/llvm/test/Transforms/LoopDistribute/pointer-phi-in-loop.ll b/llvm/test/Transforms/LoopDistribute/pointer-phi-in-loop.ll
index 2ab9140baf866..b95551eb94f4c 100644
--- a/llvm/test/Transforms/LoopDistribute/pointer-phi-in-loop.ll
+++ b/llvm/test/Transforms/LoopDistribute/pointer-phi-in-loop.ll
@@ -3,26 +3,73 @@
 
 ; Testcases inspired by PR50296, PR50288.
 
-define void @phi_load_store_distribute(i1 %c, ptr %A, ptr %B, ptr %C) {
+define void @phi_load_store_distribute(i1 %cond, ptr %A, ptr %B, ptr %C) {
 ; CHECK-LABEL: @phi_load_store_distribute(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
-; CHECK:       for.body:
-; CHECK-NEXT:    [[IV:%.*]] = phi i16 [ 0, [[ENTRY:%.*]] ], [ [[IV_NEXT:%.*]], [[IF_END:%.*]] ]
-; CHECK-NEXT:    [[LV:%.*]] = load i16, ptr [[A:%.*]], align 1
+; CHECK:       for.body.lver.check:
+; CHECK-NEXT:    [[SCEVGEP:%.*]] = getelementptr i8, ptr [[A:%.*]], i64 2
+; CHECK-NEXT:    [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[C:%.*]], i64 2
+; CHECK-NEXT:    [[SCEVGEP2:%.*]] = getelementptr i8, ptr [[B:%.*]], i64 2
+; CHECK-NEXT:    [[BOUND0:%.*]] = icmp ult ptr [[A]], [[SCEVGEP1]]
+; CHECK-NEXT:    [[BOUND1:%.*]] = icmp ult ptr [[C]], [[SCEVGEP]]
+; CHECK-NEXT:    [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
+; CHECK-NEXT:    [[BOUND03:%.*]] = icmp ult ptr [[A]], [[SCEVGEP2]]
+; CHECK-NEXT:    [[BOUND14:%.*]] = icmp ult ptr [[B]], [[SCEVGEP]]
+; CHECK-NEXT:    [[FOUND_CONFLICT5:%.*]] = and i1 [[BOUND03]], [[BOUND14]]
+; CHECK-NEXT:    [[CONFLICT_RDX:%.*]] = or i1 [[FOUND_CONFLICT]], [[FOUND_CONFLICT5]]
+; CHECK-NEXT:    br i1 [[CONFLICT_RDX]], label [[ENTRY:%.*]], label [[FOR_BODY_PH_LDIST1:%.*]]
+; CHECK:       for.body.ph.lver.orig:
+; CHECK-NEXT:    br label [[FOR_BODY_LVER_ORIG:%.*]]
+; CHECK:       for.body.lver.orig:
+; CHECK-NEXT:    [[IV:%.*]] = phi i16 [ 0, [[ENTRY]] ], [ [[IV_NEXT:%.*]], [[IF_END:%.*]] ]
+; CHECK-NEXT:    [[LV:%.*]] = load i16, ptr [[A]], align 1
 ; CHECK-NEXT:    store i16 [[LV]], ptr [[A]], align 1
-; CHECK-NEXT:    br i1 [[C:%.*]], label [[IF_THEN:%.*]], label [[IF_END]]
-; CHECK:       if.then:
+; CHECK-NEXT:    br i1 [[COND:%.*]], label [[IF_THEN:%.*]], label [[IF_END]]
+; CHECK:       if.then.lver.orig:
 ; CHECK-NEXT:    [[LV2:%.*]] = load i16, ptr [[A]], align 1
 ; CHECK-NEXT:    br label [[IF_END]]
+; CHECK:       if.end.lver.orig:
+; CHECK-NEXT:    [[C_SINK_LVER_ORIG:%.*]] = phi ptr [ [[B]], [[IF_THEN]] ], [ [[C]], [[FOR_BODY_LVER_ORIG]] ]
+; CHECK-NEXT:    [[LV3_LVER_ORIG:%.*]] = load i16, ptr [[C_SINK_LVER_ORIG]], align 2
+; CHECK-NEXT:    [[ADD_LVER_ORIG:%.*]] = add i16 [[LV3_LVER_ORIG]], 10
+; CHECK-NEXT:    store i16 [[ADD_LVER_ORIG]], ptr [[C_SINK_LVER_ORIG]], align 1
+; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i16 [[IV]], 1
+; CHECK-NEXT:    [[TOBOOL_NOT_LVER_ORIG:%.*]] = icmp eq i16 [[IV_NEXT]], 1000
+; CHECK-NEXT:    br i1 [[TOBOOL_NOT_LVER_ORIG]], label [[FOR_END_LOOPEXIT_LOOPEXIT:%.*]], label [[FOR_BODY_LVER_ORIG]]
+; CHECK:       for.body.ph.ldist1:
+; CHECK-NEXT:    br label [[FOR_BODY_LDIST1:%.*]]
+; CHECK:       for.body.ldist1:
+; CHECK-NEXT:    [[IV_LDIST1:%.*]] = phi i16 [ 0, [[FOR_BODY_PH_LDIST1]] ], [ [[IV_NEXT_LDIST1:%.*]], [[IF_END_LDIST1:%.*]] ]
+; CHECK-NEXT:    [[LV_LDIST1:%.*]] = load i16, ptr [[A]], align 1, !alias.scope [[META0:![0-9]+]], !noalias [[META3:![0-9]+]]
+; CHECK-NEXT:    store i16 [[LV_LDIST1]], ptr [[A]], align 1, !alias.scope [[META0]], !noalias [[META3]]
+; CHECK-NEXT:    br i1 [[COND]], label [[IF_THEN_LDIST1:%.*]], label [[IF_END_LDIST1]]
+; CHECK:       if.then.ldist1:
+; CHECK-NEXT:    [[LV2_LDIST1:%.*]] = load i16, ptr [[A]], align 1, !alias.scope [[META0]], !noalias [[META3]]
+; CHECK-NEXT:    br label [[IF_END_LDIST1]]
+; CHECK:       if.end.ldist1:
+; CHECK-NEXT:    [[IV_NEXT_LDIST1]] = add nuw nsw i16 [[IV_LDIST1]], 1
+; CHECK-NEXT:    [[TOBOOL_NOT_LDIST1:%.*]] = icmp eq i16 [[IV_NEXT_LDIST1]], 1000
+; CHECK-NEXT:    br i1 [[TOBOOL_NOT_LDIST1]], label [[FOR_BODY_PH:%.*]], label [[FOR_BODY_LDIST1]]
+; CHECK:       for.body.ph:
+; CHECK-NEXT:    br label [[FOR_BODY1:%.*]]
+; CHECK:       for.body:
+; CHECK-NEXT:    [[IV1:%.*]] = phi i16 [ 0, [[FOR_BODY_PH]] ], [ [[IV_NEXT1:%.*]], [[IF_END1:%.*]] ]
+; CHECK-NEXT:    br i1 [[COND]], label [[IF_THEN1:%.*]], label [[IF_END1]]
+; CHECK:       if.then:
+; CHECK-NEXT:    br label [[IF_END1]]
 ; CHECK:       if.end:
-; CHECK-NEXT:    [[C_SINK:%.*]] = phi ptr [ [[B:%.*]], [[IF_THEN]] ], [ [[C:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT:    [[C_SINK:%.*]] = phi ptr [ [[B]], [[IF_THEN1]] ], [ [[C]], [[FOR_BODY1]] ]
 ; CHECK-NEXT:    [[LV3:%.*]] = load i16, ptr [[C_SINK]], align 2
 ; CHECK-NEXT:    [[ADD:%.*]] = add i16 [[LV3]], 10
 ; CHECK-NEXT:    store i16 [[ADD]], ptr [[C_SINK]], align 1
-; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i16 [[IV]], 1
-; CHECK-NEXT:    [[TOBOOL_NOT:%.*]] = icmp eq i16 [[IV_NEXT]], 1000
-; CHECK-NEXT:    br i1 [[TOBOOL_NOT]], label [[FOR_END_LOOPEXIT:%.*]], label [[FOR_BODY]]
+; CHECK-NEXT:    [[IV_NEXT1]] = add nuw nsw i16 [[IV1]], 1
+; CHECK-NEXT:    [[TOBOOL_NOT:%.*]] = icmp eq i16 [[IV_NEXT1]], 1000
+; CHECK-NEXT:    br i1 [[TOBOOL_NOT]], label [[FOR_END_LOOPEXIT_LOOPEXIT6:%.*]], label [[FOR_BODY1]]
+; CHECK:       for.end.loopexit.loopexit:
+; CHECK-NEXT:    br label [[FOR_END_LOOPEXIT:%.*]]
+; CHECK:       for.end.loopexit.loopexit6:
+; CHECK-NEXT:    br label [[FOR_END_LOOPEXIT]]
 ; CHECK:       for.end.loopexit:
 ; CHECK-NEXT:    ret void
 ;
@@ -33,7 +80,7 @@ for.body:                                         ; preds = %if.end, %entry
   %iv = phi i16 [ 0, %entry ], [ %iv.next, %if.end ]
   %lv = load i16, ptr %A, align 1
   store i16 %lv, ptr %A, align 1
-  br i1 %c, label %if.then, label %if.end
+  br i1 %cond, label %if.then, label %if.end
 
 if.then:                                          ; preds = %for.body
   %lv2 = load i16, ptr %A, align 1
@@ -55,66 +102,21 @@ for.end.loopexit:                                 ; preds = %if.end
 define void @phi_load_distribute(i1 %cond, ptr %A, ptr %B, ptr %C) {
 ; CHECK-LABEL: @phi_load_distribute(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    br label [[FOR_BODY_LVER_CHECK:%.*]]
-; CHECK:       for.body.lver.check:
-; CHECK-NEXT:    [[SCEVGEP:%.*]] = getelementptr i8, ptr [[A:%.*]], i64 2
-; CHECK-NEXT:    [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[C:%.*]], i64 2
-; CHECK-NEXT:    [[SCEVGEP2:%.*]] = getelementptr i8, ptr [[B:%.*]], i64 2
-; CHECK-NEXT:    [[BOUND0:%.*]] = icmp ult ptr [[A]], [[SCEVGEP1]]
-; CHECK-NEXT:    [[BOUND1:%.*]] = icmp ult ptr [[C]], [[SCEVGEP]]
-; CHECK-NEXT:    [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
-; CHECK-NEXT:    [[BOUND03:%.*]] = icmp ult ptr [[A]], [[SCEVGEP2]]
-; CHECK-NEXT:    [[BOUND14:%.*]] = icmp ult ptr [[B]], [[SCEVGEP]]
-; CHECK-NEXT:    [[FOUND_CONFLICT5:%.*]] = and i1 [[BOUND03]], [[BOUND14]]
-; CHECK-NEXT:    [[CONFLICT_RDX:%.*]] = or i1 [[FOUND_CONFLICT]], [[FOUND_CONFLICT5]]
-; CHECK-NEXT:    br i1 [[CONFLICT_RDX]], label [[FOR_BODY_PH_LVER_ORIG:%.*]], label [[FOR_BODY_PH_LDIST1:%.*]]
-; CHECK:       for.body.ph.lver.orig:
 ; CHECK-NEXT:    br label [[FOR_BODY_LVER_ORIG:%.*]]
-; CHECK:       for.body.lver.orig:
-; CHECK-NEXT:    [[IV_LVER_ORIG:%.*]] = phi i16 [ 0, [[FOR_BODY_PH_LVER_ORIG]] ], [ [[IV_NEXT_LVER_ORIG:%.*]], [[IF_END_LVER_ORIG:%.*]] ]
-; CHECK-NEXT:    [[LV_LVER_ORIG:%.*]] = load i16, ptr [[A]], align 1
+; CHECK:       for.body:
+; CHECK-NEXT:    [[IV_LVER_ORIG:%.*]] = phi i16 [ 0, [[FOR_BODY_PH_LVER_ORIG:%.*]] ], [ [[IV_NEXT_LVER_ORIG:%.*]], [[IF_END_LVER_ORIG:%.*]] ]
+; CHECK-NEXT:    [[LV_LVER_ORIG:%.*]] = load i16, ptr [[A:%.*]], align 1
 ; CHECK-NEXT:    store i16 [[LV_LVER_ORIG]], ptr [[A]], align 1
 ; CHECK-NEXT:    br i1 [[COND:%.*]], label [[IF_THEN_LVER_ORIG:%.*]], label [[IF_END_LVER_ORIG]]
-; CHECK:       if.then.lver.orig:
-; CHECK-NEXT:    [[LV2_LVER_ORIG:%.*]] = load i16, ptr [[A]], align 1
-; CHECK-NEXT:    br label [[IF_END_LVER_ORIG]]
-; CHECK:       if.end.lver.orig:
-; CHECK-NEXT:    [[C_SINK_LVER_ORIG:%.*]] = phi ptr [ [[B]], [[IF_THEN_LVER_ORIG]] ], [ [[C]], [[FOR_BODY_LVER_ORIG]] ]
-; CHECK-NEXT:    [[LV3_LVER_ORIG:%.*]] = load i16, ptr [[C_SINK_LVER_ORIG]], align 2
-; CHECK-NEXT:    [[IV_NEXT_LVER_ORIG]] = add nuw nsw i16 [[IV_LVER_ORIG]], 1
-; CHECK-NEXT:    [[TOBOOL_NOT_LVER_ORIG:%.*]] = icmp eq i16 [[IV_NEXT_LVER_ORIG]], 1000
-; CHECK-NEXT:    br i1 [[TOBOOL_NOT_LVER_ORIG]], label [[FOR_END_LOOPEXIT_LOOPEXIT:%.*]], label [[FOR_BODY_LVER_ORIG]]
-; CHECK:       for.body.ph.ldist1:
-; CHECK-NEXT:    br label [[FOR_BODY_LDIST1:%.*]]
-; CHECK:       for.body.ldist1:
-; CHECK-NEXT:    [[IV_LDIST1:%.*]] = phi i16 [ 0, [[FOR_BODY_PH_LDIST1]] ], [ [[IV_NEXT_LDIST1:%.*]], [[IF_END_LDIST1:%.*]] ]
-; CHECK-NEXT:    [[LV_LDIST1:%.*]] = load i16, ptr [[A]], align 1, !alias.scope [[META0:![0-9]+]], !noalias [[META3:![0-9]+]]
-; CHECK-NEXT:    store i16 [[LV_LDIST1]], ptr [[A]], align 1, !alias.scope [[META0]], !noalias [[META3]]
-; CHECK-NEXT:    br i1 [[COND]], label [[IF_THEN_LDIST1:%.*]], label [[IF_END_LDIST1]]
-; CHECK:       if.then.ldist1:
-; CHECK-NEXT:    [[LV2_LDIST1:%.*]] = load i16, ptr [[A]], align 1, !alias.scope [[META0]], !noalias [[META3]]
-; CHECK-NEXT:    br label [[IF_END_LDIST1]]
-; CHECK:       if.end.ldist1:
-; CHECK-NEXT:    [[IV_NEXT_LDIST1]] = add nuw nsw i16 [[IV_LDIST1]], 1
-; CHECK-NEXT:    [[TOBOOL_NOT_LDIST1:%.*]] = icmp eq i16 [[IV_NEXT_LDIST1]], 1000
-; CHECK-NEXT:    br i1 [[TOBOOL_NOT_LDIST1]], label [[FOR_BODY_PH:%.*]], label [[FOR_BODY_LDIST1]]
-; CHECK:       for.body.ph:
-; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
-; CHECK:       for.body:
-; CHECK-NEXT:    [[IV:%.*]] = phi i16 [ 0, [[FOR_BODY_PH]] ], [ [[IV_NEXT:%.*]], [[IF_END:%.*]] ]
-; CHECK-NEXT:    br i1 [[COND]], label [[IF_THEN:%.*]], label [[IF_END]]
 ; CHECK:       if.then:
-; CHECK-NEXT:    br label [[IF_END]]
+; CHECK-NEXT:    [[LV2:%.*]] = load i16, ptr [[A]], align 1
+; CHECK-NEXT:    br label [[IF_END_LVER_ORIG]]
 ; CHECK:       if.end:
-; CHECK-NEXT:    [[C_SINK:%.*]] = phi ptr [ [[B]], [[IF_THEN]] ], [ [[C]], [[FOR_BODY]] ]
+; CHECK-NEXT:    [[C_SINK:%.*]] = phi ptr [ [[B:%.*]], [[IF_THEN_LVER_ORIG]] ], [ [[C:%.*]], [[FOR_BODY_LVER_ORIG]] ]
 ; CHECK-NEXT:    [[LV3:%.*]] = load i16, ptr [[C_SINK]], align 2
-; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i16 [[IV]], 1
-; CHECK-NEXT:    [[TOBOOL_NOT:%.*]] = icmp eq i16 [[IV_NEXT]], 1000
-; CHECK-NEXT:    br i1 [[TOBOOL_NOT]], label [[FOR_END_LOOPEXIT_LOOPEXIT6:%.*]], label [[FOR_BODY]]
-; CHECK:       for.end.loopexit.loopexit:
-; CHECK-NEXT:    br label [[FOR_END_LOOPEXIT:%.*]]
-; CHECK:       for.end.loopexit.loopexit6:
-; CHECK-NEXT:    br label [[FOR_END_LOOPEXIT]]
+; CHECK-NEXT:    [[IV_NEXT_LVER_ORIG]] = add nuw nsw i16 [[IV_LVER_ORIG]], 1
+; CHECK-NEXT:    [[TOBOOL_NOT:%.*]] = icmp eq i16 [[IV_NEXT_LVER_ORIG]], 1000
+; CHECK-NEXT:    br i1 [[TOBOOL_NOT]], label [[FOR_END_LOOPEXIT:%.*]], label [[FOR_BODY_LVER_ORIG]]
 ; CHECK:       for.end.loopexit:
 ; CHECK-NEXT:    ret void
 ;
diff --git a/llvm/test/Transforms/LoopIdiom/AArch64/ctlz.ll b/llvm/test/Transforms/LoopIdiom/AArch64/ctlz.ll
index 5a182745399be..fdde7147d13d1 100644
--- a/llvm/test/Transforms/LoopIdiom/AArch64/ctlz.ll
+++ b/llvm/test/Transforms/LoopIdiom/AArch64/ctlz.ll
@@ -20,7 +20,7 @@
 ; Function Attrs: norecurse nounwind uwtable
 define i32 @ctlz_and_other(i32 %n, ptr nocapture %a) {
 ; CHECK-LABEL: define i32 @ctlz_and_other(
-; CHECK-SAME: i32 [[N:%.*]], ptr nocapture [[A:%.*]]) {
+; CHECK-SAME: i32 [[N:%.*]], ptr captures(none) [[A:%.*]]) {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[ABS_N:%.*]] = call i32 @llvm.abs.i32(i32 [[N]], i1 true)
 ; CHECK-NEXT:    [[SHR8:%.*]] = lshr i32 [[ABS_N]], 1
diff --git a/llvm/test/Transforms/LoopIdiom/AArch64/find-first-byte.ll b/llvm/test/Transforms/LoopIdiom/AArch64/find-first-byte.ll
new file mode 100644
index 0000000000000..8ef2a51506606
--- /dev/null
+++ b/llvm/test/Transforms/LoopIdiom/AArch64/find-first-byte.ll
@@ -0,0 +1,671 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -mtriple=aarch64 -mattr=+sve -passes='loop(loop-idiom-vectorize)' -verify-loop-info -verify-dom-info -S < %s | FileCheck %s
+; RUN: opt -mtriple=aarch64 -mattr=+sve -passes='loop(loop-idiom-vectorize)' -disable-loop-idiom-vectorize-find-first-byte -S < %s | FileCheck -check-prefix=DISABLE %s
+
+; Base case based on `libcxx/include/__algorithm/find_first_of.h':
+;   char* find_first_of(char *first, char *last, char *s_first, char *s_last) {
+;     for (; first != last; ++first)
+;       for (char *it = s_first; it != s_last; ++it)
+;         if (*first == *it)
+;           return first;
+;     return last;
+;   }
+define ptr @find_first_of_i8(ptr %search_start, ptr %search_end, ptr %needle_start, ptr %needle_end) #0 {
+; CHECK-LABEL: define ptr @find_first_of_i8(
+; CHECK-SAME: ptr [[SEARCH_START:%.*]], ptr [[SEARCH_END:%.*]], ptr [[NEEDLE_START:%.*]], ptr [[NEEDLE_END:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    [[SEARCH_TEST:%.*]] = icmp eq ptr [[SEARCH_START]], [[SEARCH_END]]
+; CHECK-NEXT:    [[NEEDLE_TEST:%.*]] = icmp eq ptr [[NEEDLE_START]], [[NEEDLE_END]]
+; CHECK-NEXT:    [[COMBINED_TEST:%.*]] = or i1 [[SEARCH_TEST]], [[NEEDLE_TEST]]
+; CHECK-NEXT:    br i1 [[COMBINED_TEST]], label %[[EXIT:.*]], label %[[HEADER_PREHEADER:.*]]
+; CHECK:       [[HEADER_PREHEADER]]:
+; CHECK-NEXT:    br label %[[MEM_CHECK:.*]]
+; CHECK:       [[MEM_CHECK]]:
+; CHECK-NEXT:    [[SEARCH_START_INT:%.*]] = ptrtoint ptr [[SEARCH_START]] to i64
+; CHECK-NEXT:    [[SEARCH_END_INT:%.*]] = ptrtoint ptr [[SEARCH_END]] to i64
+; CHECK-NEXT:    [[NEEDLE_START_INT:%.*]] = ptrtoint ptr [[NEEDLE_START]] to i64
+; CHECK-NEXT:    [[NEEDLE_END_INT:%.*]] = ptrtoint ptr [[NEEDLE_END]] to i64
+; CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i64(i64 0, i64 16)
+; CHECK-NEXT:    [[SEARCH_START_PAGE:%.*]] = lshr i64 [[SEARCH_START_INT]], 12
+; CHECK-NEXT:    [[SEARCH_END_PAGE:%.*]] = lshr i64 [[SEARCH_END_INT]], 12
+; CHECK-NEXT:    [[NEEDLE_START_PAGE:%.*]] = lshr i64 [[NEEDLE_START_INT]], 12
+; CHECK-NEXT:    [[NEEDLE_END_PAGE:%.*]] = lshr i64 [[NEEDLE_END_INT]], 12
+; CHECK-NEXT:    [[SEARCH_PAGE_CMP:%.*]] = icmp ne i64 [[SEARCH_START_PAGE]], [[SEARCH_END_PAGE]]
+; CHECK-NEXT:    [[NEEDLE_PAGE_CMP:%.*]] = icmp ne i64 [[NEEDLE_START_PAGE]], [[NEEDLE_END_PAGE]]
+; CHECK-NEXT:    [[COMBINED_PAGE_CMP:%.*]] = or i1 [[SEARCH_PAGE_CMP]], [[NEEDLE_PAGE_CMP]]
+; CHECK-NEXT:    br i1 [[COMBINED_PAGE_CMP]], label %[[SCALAR_PREHEADER:.*]], label %[[FIND_FIRST_VEC_HEADER:.*]], !prof [[PROF0:![0-9]+]]
+; CHECK:       [[FIND_FIRST_VEC_HEADER]]:
+; CHECK-NEXT:    [[PSEARCH:%.*]] = phi ptr [ [[SEARCH_START]], %[[MEM_CHECK]] ], [ [[SEARCH_NEXT_VEC:%.*]], %[[SEARCH_CHECK_VEC:.*]] ]
+; CHECK-NEXT:    [[TMP1:%.*]] = ptrtoint ptr [[PSEARCH]] to i64
+; CHECK-NEXT:    [[SEARCH_PRED:%.*]] = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i64(i64 [[TMP1]], i64 [[SEARCH_END_INT]])
+; CHECK-NEXT:    [[SEARCH_MASKED:%.*]] = and <vscale x 16 x i1> [[TMP0]], [[SEARCH_PRED]]
+; CHECK-NEXT:    [[SEARCH_LOAD_VEC:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr [[PSEARCH]], i32 1, <vscale x 16 x i1> [[SEARCH_MASKED]], <vscale x 16 x i8> zeroinitializer)
+; CHECK-NEXT:    br label %[[MATCH_CHECK_VEC:.*]]
+; CHECK:       [[MATCH_CHECK_VEC]]:
+; CHECK-NEXT:    [[PNEEDLE:%.*]] = phi ptr [ [[NEEDLE_START]], %[[FIND_FIRST_VEC_HEADER]] ], [ [[NEEDLE_NEXT_VEC:%.*]], %[[NEEDLE_CHECK_VEC:.*]] ]
+; CHECK-NEXT:    [[TMP2:%.*]] = ptrtoint ptr [[PNEEDLE]] to i64
+; CHECK-NEXT:    [[NEEDLE_PRED:%.*]] = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i64(i64 [[TMP2]], i64 [[NEEDLE_END_INT]])
+; CHECK-NEXT:    [[NEEDLE_MASKED:%.*]] = and <vscale x 16 x i1> [[TMP0]], [[NEEDLE_PRED]]
+; CHECK-NEXT:    [[NEEDLE_LOAD_VEC:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr [[PNEEDLE]], i32 1, <vscale x 16 x i1> [[NEEDLE_MASKED]], <vscale x 16 x i8> zeroinitializer)
+; CHECK-NEXT:    [[NEEDLE0:%.*]] = extractelement <vscale x 16 x i8> [[NEEDLE_LOAD_VEC]], i64 0
+; CHECK-NEXT:    [[NEEDLE0_SPLATINSERT:%.*]] = insertelement <vscale x 16 x i8> poison, i8 [[NEEDLE0]], i64 0
+; CHECK-NEXT:    [[NEEDLE0_SPLAT:%.*]] = shufflevector <vscale x 16 x i8> [[NEEDLE0_SPLATINSERT]], <vscale x 16 x i8> poison, <vscale x 16 x i32> zeroinitializer
+; CHECK-NEXT:    [[NEEDLE_SPLAT:%.*]] = select <vscale x 16 x i1> [[NEEDLE_MASKED]], <vscale x 16 x i8> [[NEEDLE_LOAD_VEC]], <vscale x 16 x i8> [[NEEDLE0_SPLAT]]
+; CHECK-NEXT:    [[NEEDLE_VEC:%.*]] = call <16 x i8> @llvm.vector.extract.v16i8.nxv16i8(<vscale x 16 x i8> [[NEEDLE_SPLAT]], i64 0)
+; CHECK-NEXT:    [[MATCH_PRED:%.*]] = call <vscale x 16 x i1> @llvm.experimental.vector.match.nxv16i8.v16i8(<vscale x 16 x i8> [[SEARCH_LOAD_VEC]], <16 x i8> [[NEEDLE_VEC]], <vscale x 16 x i1> [[SEARCH_MASKED]])
+; CHECK-NEXT:    [[TMP3:%.*]] = call i1 @llvm.vector.reduce.or.nxv16i1(<vscale x 16 x i1> [[MATCH_PRED]])
+; CHECK-NEXT:    br i1 [[TMP3]], label %[[CALCULATE_MATCH:.*]], label %[[NEEDLE_CHECK_VEC]]
+; CHECK:       [[CALCULATE_MATCH]]:
+; CHECK-NEXT:    [[MATCH_START:%.*]] = phi ptr [ [[PSEARCH]], %[[MATCH_CHECK_VEC]] ]
+; CHECK-NEXT:    [[MATCH_VEC:%.*]] = phi <vscale x 16 x i1> [ [[MATCH_PRED]], %[[MATCH_CHECK_VEC]] ]
+; CHECK-NEXT:    [[MATCH_IDX:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.nxv16i1(<vscale x 16 x i1> [[MATCH_VEC]], i1 true)
+; CHECK-NEXT:    [[MATCH_RES:%.*]] = getelementptr i8, ptr [[MATCH_START]], i64 [[MATCH_IDX]]
+; CHECK-NEXT:    br label %[[EXIT_LOOPEXIT:.*]]
+; CHECK:       [[NEEDLE_CHECK_VEC]]:
+; CHECK-NEXT:    [[NEEDLE_NEXT_VEC]] = getelementptr i8, ptr [[PNEEDLE]], i64 16
+; CHECK-NEXT:    [[TMP4:%.*]] = icmp ult ptr [[NEEDLE_NEXT_VEC]], [[NEEDLE_END]]
+; CHECK-NEXT:    br i1 [[TMP4]], label %[[MATCH_CHECK_VEC]], label %[[SEARCH_CHECK_VEC]]
+; CHECK:       [[SEARCH_CHECK_VEC]]:
+; CHECK-NEXT:    [[SEARCH_NEXT_VEC]] = getelementptr i8, ptr [[PSEARCH]], i64 16
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp ult ptr [[SEARCH_NEXT_VEC]], [[SEARCH_END]]
+; CHECK-NEXT:    br i1 [[TMP5]], label %[[FIND_FIRST_VEC_HEADER]], label %[[EXIT_LOOPEXIT1:.*]]
+; CHECK:       [[SCALAR_PREHEADER]]:
+; CHECK-NEXT:    br label %[[HEADER:.*]]
+; CHECK:       [[HEADER]]:
+; CHECK-NEXT:    [[SEARCH_PTR:%.*]] = phi ptr [ [[SEARCH_NEXT:%.*]], %[[SEARCH_CHECK:.*]] ], [ [[SEARCH_START]], %[[SCALAR_PREHEADER]] ]
+; CHECK-NEXT:    [[SEARCH_LOAD:%.*]] = load i8, ptr [[SEARCH_PTR]], align 1
+; CHECK-NEXT:    br label %[[MATCH_CHECK:.*]]
+; CHECK:       [[NEEDLE_CHECK:.*]]:
+; CHECK-NEXT:    [[NEEDLE_NEXT:%.*]] = getelementptr inbounds i8, ptr [[NEEDLE_PTR:%.*]], i64 1
+; CHECK-NEXT:    [[NEEDLE_CMP:%.*]] = icmp eq ptr [[NEEDLE_NEXT]], [[NEEDLE_END]]
+; CHECK-NEXT:    br i1 [[NEEDLE_CMP]], label %[[SEARCH_CHECK]], label %[[MATCH_CHECK]]
+; CHECK:       [[MATCH_CHECK]]:
+; CHECK-NEXT:    [[NEEDLE_PTR]] = phi ptr [ [[NEEDLE_START]], %[[HEADER]] ], [ [[NEEDLE_NEXT]], %[[NEEDLE_CHECK]] ]
+; CHECK-NEXT:    [[NEEDLE_LOAD:%.*]] = load i8, ptr [[NEEDLE_PTR]], align 1
+; CHECK-NEXT:    [[MATCH_CMP:%.*]] = icmp eq i8 [[SEARCH_LOAD]], [[NEEDLE_LOAD]]
+; CHECK-NEXT:    br i1 [[MATCH_CMP]], label %[[EXIT_LOOPEXIT]], label %[[NEEDLE_CHECK]]
+; CHECK:       [[SEARCH_CHECK]]:
+; CHECK-NEXT:    [[SEARCH_NEXT]] = getelementptr inbounds i8, ptr [[SEARCH_PTR]], i64 1
+; CHECK-NEXT:    [[SEARCH_CMP:%.*]] = icmp eq ptr [[SEARCH_NEXT]], [[SEARCH_END]]
+; CHECK-NEXT:    br i1 [[SEARCH_CMP]], label %[[EXIT_LOOPEXIT1]], label %[[HEADER]]
+; CHECK:       [[EXIT_LOOPEXIT]]:
+; CHECK-NEXT:    [[SEARCH_PTR_LCSSA:%.*]] = phi ptr [ [[SEARCH_PTR]], %[[MATCH_CHECK]] ], [ [[MATCH_RES]], %[[CALCULATE_MATCH]] ]
+; CHECK-NEXT:    br label %[[EXIT]]
+; CHECK:       [[EXIT_LOOPEXIT1]]:
+; CHECK-NEXT:    br label %[[EXIT]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    [[RES:%.*]] = phi ptr [ [[SEARCH_END]], %[[ENTRY]] ], [ [[SEARCH_PTR_LCSSA]], %[[EXIT_LOOPEXIT]] ], [ [[SEARCH_END]], %[[EXIT_LOOPEXIT1]] ]
+; CHECK-NEXT:    ret ptr [[RES]]
+;
+; DISABLE-LABEL: define ptr @find_first_of_i8(
+; DISABLE-SAME: ptr [[SEARCH_START:%.*]], ptr [[SEARCH_END:%.*]], ptr [[NEEDLE_START:%.*]], ptr [[NEEDLE_END:%.*]]) #[[ATTR0:[0-9]+]] {
+; DISABLE-NEXT:  [[ENTRY:.*]]:
+; DISABLE-NEXT:    [[SEARCH_TEST:%.*]] = icmp eq ptr [[SEARCH_START]], [[SEARCH_END]]
+; DISABLE-NEXT:    [[NEEDLE_TEST:%.*]] = icmp eq ptr [[NEEDLE_START]], [[NEEDLE_END]]
+; DISABLE-NEXT:    [[COMBINED_TEST:%.*]] = or i1 [[SEARCH_TEST]], [[NEEDLE_TEST]]
+; DISABLE-NEXT:    br i1 [[COMBINED_TEST]], label %[[EXIT:.*]], label %[[HEADER_PREHEADER:.*]]
+; DISABLE:       [[HEADER_PREHEADER]]:
+; DISABLE-NEXT:    br label %[[HEADER:.*]]
+; DISABLE:       [[HEADER]]:
+; DISABLE-NEXT:    [[SEARCH_PTR:%.*]] = phi ptr [ [[SEARCH_NEXT:%.*]], %[[SEARCH_CHECK:.*]] ], [ [[SEARCH_START]], %[[HEADER_PREHEADER]] ]
+; DISABLE-NEXT:    [[SEARCH_LOAD:%.*]] = load i8, ptr [[SEARCH_PTR]], align 1
+; DISABLE-NEXT:    br label %[[MATCH_CHECK:.*]]
+; DISABLE:       [[NEEDLE_CHECK:.*]]:
+; DISABLE-NEXT:    [[NEEDLE_NEXT:%.*]] = getelementptr inbounds i8, ptr [[NEEDLE_PTR:%.*]], i64 1
+; DISABLE-NEXT:    [[NEEDLE_CMP:%.*]] = icmp eq ptr [[NEEDLE_NEXT]], [[NEEDLE_END]]
+; DISABLE-NEXT:    br i1 [[NEEDLE_CMP]], label %[[SEARCH_CHECK]], label %[[MATCH_CHECK]]
+; DISABLE:       [[MATCH_CHECK]]:
+; DISABLE-NEXT:    [[NEEDLE_PTR]] = phi ptr [ [[NEEDLE_START]], %[[HEADER]] ], [ [[NEEDLE_NEXT]], %[[NEEDLE_CHECK]] ]
+; DISABLE-NEXT:    [[NEEDLE_LOAD:%.*]] = load i8, ptr [[NEEDLE_PTR]], align 1
+; DISABLE-NEXT:    [[MATCH_CMP:%.*]] = icmp eq i8 [[SEARCH_LOAD]], [[NEEDLE_LOAD]]
+; DISABLE-NEXT:    br i1 [[MATCH_CMP]], label %[[EXIT_LOOPEXIT:.*]], label %[[NEEDLE_CHECK]]
+; DISABLE:       [[SEARCH_CHECK]]:
+; DISABLE-NEXT:    [[SEARCH_NEXT]] = getelementptr inbounds i8, ptr [[SEARCH_PTR]], i64 1
+; DISABLE-NEXT:    [[SEARCH_CMP:%.*]] = icmp eq ptr [[SEARCH_NEXT]], [[SEARCH_END]]
+; DISABLE-NEXT:    br i1 [[SEARCH_CMP]], label %[[EXIT_LOOPEXIT1:.*]], label %[[HEADER]]
+; DISABLE:       [[EXIT_LOOPEXIT]]:
+; DISABLE-NEXT:    [[SEARCH_PTR_LCSSA:%.*]] = phi ptr [ [[SEARCH_PTR]], %[[MATCH_CHECK]] ]
+; DISABLE-NEXT:    br label %[[EXIT]]
+; DISABLE:       [[EXIT_LOOPEXIT1]]:
+; DISABLE-NEXT:    br label %[[EXIT]]
+; DISABLE:       [[EXIT]]:
+; DISABLE-NEXT:    [[RES:%.*]] = phi ptr [ [[SEARCH_END]], %[[ENTRY]] ], [ [[SEARCH_PTR_LCSSA]], %[[EXIT_LOOPEXIT]] ], [ [[SEARCH_END]], %[[EXIT_LOOPEXIT1]] ]
+; DISABLE-NEXT:    ret ptr [[RES]]
+;
+entry:
+  %search_test = icmp eq ptr %search_start, %search_end
+  %needle_test = icmp eq ptr %needle_start, %needle_end
+  %combined_test = or i1 %search_test, %needle_test
+  br i1 %combined_test, label %exit, label %header
+
+header:
+  %search_ptr = phi ptr [ %search_next, %search_check ], [ %search_start, %entry ]
+  %search_load = load i8, ptr %search_ptr, align 1
+  br label %match_check
+
+needle_check:
+  %needle_next = getelementptr inbounds i8, ptr %needle_ptr, i64 1
+  %needle_cmp = icmp eq ptr %needle_next, %needle_end
+  br i1 %needle_cmp, label %search_check, label %match_check
+
+match_check:
+  %needle_ptr = phi ptr [ %needle_start, %header ], [ %needle_next, %needle_check ]
+  %needle_load = load i8, ptr %needle_ptr, align 1
+  %match_cmp = icmp eq i8 %search_load, %needle_load
+  br i1 %match_cmp, label %exit, label %needle_check
+
+search_check:
+  %search_next = getelementptr inbounds i8, ptr %search_ptr, i64 1
+  %search_cmp = icmp eq ptr %search_next, %search_end
+  br i1 %search_cmp, label %exit, label %header
+
+exit:
+  %res = phi ptr [ %search_end, %entry ], [ %search_ptr, %match_check ], [ %search_end, %search_check ]
+  ret ptr %res
+}
+
+; Equivalent to @find_first_of_i8 but with i16.
+; This is accepted and generates a similar loop.
+define ptr @find_first_of_i16(ptr %search_start, ptr %search_end, ptr %needle_start, ptr %needle_end) #0 {
+; CHECK-LABEL: define ptr @find_first_of_i16(
+; CHECK-SAME: ptr [[SEARCH_START:%.*]], ptr [[SEARCH_END:%.*]], ptr [[NEEDLE_START:%.*]], ptr [[NEEDLE_END:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    [[SEARCH_TEST:%.*]] = icmp eq ptr [[SEARCH_START]], [[SEARCH_END]]
+; CHECK-NEXT:    [[NEEDLE_TEST:%.*]] = icmp eq ptr [[NEEDLE_START]], [[NEEDLE_END]]
+; CHECK-NEXT:    [[COMBINED_TEST:%.*]] = or i1 [[SEARCH_TEST]], [[NEEDLE_TEST]]
+; CHECK-NEXT:    br i1 [[COMBINED_TEST]], label %[[EXIT:.*]], label %[[HEADER_PREHEADER:.*]]
+; CHECK:       [[HEADER_PREHEADER]]:
+; CHECK-NEXT:    br label %[[MEM_CHECK:.*]]
+; CHECK:       [[MEM_CHECK]]:
+; CHECK-NEXT:    [[SEARCH_START_INT:%.*]] = ptrtoint ptr [[SEARCH_START]] to i64
+; CHECK-NEXT:    [[SEARCH_END_INT:%.*]] = ptrtoint ptr [[SEARCH_END]] to i64
+; CHECK-NEXT:    [[NEEDLE_START_INT:%.*]] = ptrtoint ptr [[NEEDLE_START]] to i64
+; CHECK-NEXT:    [[NEEDLE_END_INT:%.*]] = ptrtoint ptr [[NEEDLE_END]] to i64
+; CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 0, i64 8)
+; CHECK-NEXT:    [[SEARCH_START_PAGE:%.*]] = lshr i64 [[SEARCH_START_INT]], 12
+; CHECK-NEXT:    [[SEARCH_END_PAGE:%.*]] = lshr i64 [[SEARCH_END_INT]], 12
+; CHECK-NEXT:    [[NEEDLE_START_PAGE:%.*]] = lshr i64 [[NEEDLE_START_INT]], 12
+; CHECK-NEXT:    [[NEEDLE_END_PAGE:%.*]] = lshr i64 [[NEEDLE_END_INT]], 12
+; CHECK-NEXT:    [[SEARCH_PAGE_CMP:%.*]] = icmp ne i64 [[SEARCH_START_PAGE]], [[SEARCH_END_PAGE]]
+; CHECK-NEXT:    [[NEEDLE_PAGE_CMP:%.*]] = icmp ne i64 [[NEEDLE_START_PAGE]], [[NEEDLE_END_PAGE]]
+; CHECK-NEXT:    [[COMBINED_PAGE_CMP:%.*]] = or i1 [[SEARCH_PAGE_CMP]], [[NEEDLE_PAGE_CMP]]
+; CHECK-NEXT:    br i1 [[COMBINED_PAGE_CMP]], label %[[SCALAR_PREHEADER:.*]], label %[[FIND_FIRST_VEC_HEADER:.*]], !prof [[PROF0]]
+; CHECK:       [[FIND_FIRST_VEC_HEADER]]:
+; CHECK-NEXT:    [[PSEARCH:%.*]] = phi ptr [ [[SEARCH_START]], %[[MEM_CHECK]] ], [ [[SEARCH_NEXT_VEC:%.*]], %[[SEARCH_CHECK_VEC:.*]] ]
+; CHECK-NEXT:    [[TMP1:%.*]] = ptrtoint ptr [[PSEARCH]] to i64
+; CHECK-NEXT:    [[SEARCH_PRED:%.*]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[TMP1]], i64 [[SEARCH_END_INT]])
+; CHECK-NEXT:    [[SEARCH_MASKED:%.*]] = and <vscale x 8 x i1> [[TMP0]], [[SEARCH_PRED]]
+; CHECK-NEXT:    [[SEARCH_LOAD_VEC:%.*]] = call <vscale x 8 x i16> @llvm.masked.load.nxv8i16.p0(ptr [[PSEARCH]], i32 1, <vscale x 8 x i1> [[SEARCH_MASKED]], <vscale x 8 x i16> zeroinitializer)
+; CHECK-NEXT:    br label %[[MATCH_CHECK_VEC:.*]]
+; CHECK:       [[MATCH_CHECK_VEC]]:
+; CHECK-NEXT:    [[PNEEDLE:%.*]] = phi ptr [ [[NEEDLE_START]], %[[FIND_FIRST_VEC_HEADER]] ], [ [[NEEDLE_NEXT_VEC:%.*]], %[[NEEDLE_CHECK_VEC:.*]] ]
+; CHECK-NEXT:    [[TMP2:%.*]] = ptrtoint ptr [[PNEEDLE]] to i64
+; CHECK-NEXT:    [[NEEDLE_PRED:%.*]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[TMP2]], i64 [[NEEDLE_END_INT]])
+; CHECK-NEXT:    [[NEEDLE_MASKED:%.*]] = and <vscale x 8 x i1> [[TMP0]], [[NEEDLE_PRED]]
+; CHECK-NEXT:    [[NEEDLE_LOAD_VEC:%.*]] = call <vscale x 8 x i16> @llvm.masked.load.nxv8i16.p0(ptr [[PNEEDLE]], i32 1, <vscale x 8 x i1> [[NEEDLE_MASKED]], <vscale x 8 x i16> zeroinitializer)
+; CHECK-NEXT:    [[NEEDLE0:%.*]] = extractelement <vscale x 8 x i16> [[NEEDLE_LOAD_VEC]], i64 0
+; CHECK-NEXT:    [[NEEDLE0_SPLATINSERT:%.*]] = insertelement <vscale x 8 x i16> poison, i16 [[NEEDLE0]], i64 0
+; CHECK-NEXT:    [[NEEDLE0_SPLAT:%.*]] = shufflevector <vscale x 8 x i16> [[NEEDLE0_SPLATINSERT]], <vscale x 8 x i16> poison, <vscale x 8 x i32> zeroinitializer
+; CHECK-NEXT:    [[NEEDLE_SPLAT:%.*]] = select <vscale x 8 x i1> [[NEEDLE_MASKED]], <vscale x 8 x i16> [[NEEDLE_LOAD_VEC]], <vscale x 8 x i16> [[NEEDLE0_SPLAT]]
+; CHECK-NEXT:    [[NEEDLE_VEC:%.*]] = call <8 x i16> @llvm.vector.extract.v8i16.nxv8i16(<vscale x 8 x i16> [[NEEDLE_SPLAT]], i64 0)
+; CHECK-NEXT:    [[MATCH_PRED:%.*]] = call <vscale x 8 x i1> @llvm.experimental.vector.match.nxv8i16.v8i16(<vscale x 8 x i16> [[SEARCH_LOAD_VEC]], <8 x i16> [[NEEDLE_VEC]], <vscale x 8 x i1> [[SEARCH_MASKED]])
+; CHECK-NEXT:    [[TMP3:%.*]] = call i1 @llvm.vector.reduce.or.nxv8i1(<vscale x 8 x i1> [[MATCH_PRED]])
+; CHECK-NEXT:    br i1 [[TMP3]], label %[[CALCULATE_MATCH:.*]], label %[[NEEDLE_CHECK_VEC]]
+; CHECK:       [[CALCULATE_MATCH]]:
+; CHECK-NEXT:    [[MATCH_START:%.*]] = phi ptr [ [[PSEARCH]], %[[MATCH_CHECK_VEC]] ]
+; CHECK-NEXT:    [[MATCH_VEC:%.*]] = phi <vscale x 8 x i1> [ [[MATCH_PRED]], %[[MATCH_CHECK_VEC]] ]
+; CHECK-NEXT:    [[MATCH_IDX:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.nxv8i1(<vscale x 8 x i1> [[MATCH_VEC]], i1 true)
+; CHECK-NEXT:    [[MATCH_RES:%.*]] = getelementptr i16, ptr [[MATCH_START]], i64 [[MATCH_IDX]]
+; CHECK-NEXT:    br label %[[EXIT_LOOPEXIT:.*]]
+; CHECK:       [[NEEDLE_CHECK_VEC]]:
+; CHECK-NEXT:    [[NEEDLE_NEXT_VEC]] = getelementptr i16, ptr [[PNEEDLE]], i64 8
+; CHECK-NEXT:    [[TMP4:%.*]] = icmp ult ptr [[NEEDLE_NEXT_VEC]], [[NEEDLE_END]]
+; CHECK-NEXT:    br i1 [[TMP4]], label %[[MATCH_CHECK_VEC]], label %[[SEARCH_CHECK_VEC]]
+; CHECK:       [[SEARCH_CHECK_VEC]]:
+; CHECK-NEXT:    [[SEARCH_NEXT_VEC]] = getelementptr i16, ptr [[PSEARCH]], i64 8
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp ult ptr [[SEARCH_NEXT_VEC]], [[SEARCH_END]]
+; CHECK-NEXT:    br i1 [[TMP5]], label %[[FIND_FIRST_VEC_HEADER]], label %[[EXIT_LOOPEXIT1:.*]]
+; CHECK:       [[SCALAR_PREHEADER]]:
+; CHECK-NEXT:    br label %[[HEADER:.*]]
+; CHECK:       [[HEADER]]:
+; CHECK-NEXT:    [[SEARCH_PTR:%.*]] = phi ptr [ [[SEARCH_NEXT:%.*]], %[[SEARCH_CHECK:.*]] ], [ [[SEARCH_START]], %[[SCALAR_PREHEADER]] ]
+; CHECK-NEXT:    [[SEARCH_LOAD:%.*]] = load i16, ptr [[SEARCH_PTR]], align 1
+; CHECK-NEXT:    br label %[[MATCH_CHECK:.*]]
+; CHECK:       [[NEEDLE_CHECK:.*]]:
+; CHECK-NEXT:    [[NEEDLE_NEXT:%.*]] = getelementptr inbounds i16, ptr [[NEEDLE_PTR:%.*]], i64 1
+; CHECK-NEXT:    [[NEEDLE_CMP:%.*]] = icmp eq ptr [[NEEDLE_NEXT]], [[NEEDLE_END]]
+; CHECK-NEXT:    br i1 [[NEEDLE_CMP]], label %[[SEARCH_CHECK]], label %[[MATCH_CHECK]]
+; CHECK:       [[MATCH_CHECK]]:
+; CHECK-NEXT:    [[NEEDLE_PTR]] = phi ptr [ [[NEEDLE_START]], %[[HEADER]] ], [ [[NEEDLE_NEXT]], %[[NEEDLE_CHECK]] ]
+; CHECK-NEXT:    [[NEEDLE_LOAD:%.*]] = load i16, ptr [[NEEDLE_PTR]], align 1
+; CHECK-NEXT:    [[MATCH_CMP:%.*]] = icmp eq i16 [[SEARCH_LOAD]], [[NEEDLE_LOAD]]
+; CHECK-NEXT:    br i1 [[MATCH_CMP]], label %[[EXIT_LOOPEXIT]], label %[[NEEDLE_CHECK]]
+; CHECK:       [[SEARCH_CHECK]]:
+; CHECK-NEXT:    [[SEARCH_NEXT]] = getelementptr inbounds i16, ptr [[SEARCH_PTR]], i64 1
+; CHECK-NEXT:    [[SEARCH_CMP:%.*]] = icmp eq ptr [[SEARCH_NEXT]], [[SEARCH_END]]
+; CHECK-NEXT:    br i1 [[SEARCH_CMP]], label %[[EXIT_LOOPEXIT1]], label %[[HEADER]]
+; CHECK:       [[EXIT_LOOPEXIT]]:
+; CHECK-NEXT:    [[SEARCH_PTR_LCSSA:%.*]] = phi ptr [ [[SEARCH_PTR]], %[[MATCH_CHECK]] ], [ [[MATCH_RES]], %[[CALCULATE_MATCH]] ]
+; CHECK-NEXT:    br label %[[EXIT]]
+; CHECK:       [[EXIT_LOOPEXIT1]]:
+; CHECK-NEXT:    br label %[[EXIT]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    [[RES:%.*]] = phi ptr [ [[SEARCH_END]], %[[ENTRY]] ], [ [[SEARCH_PTR_LCSSA]], %[[EXIT_LOOPEXIT]] ], [ [[SEARCH_END]], %[[EXIT_LOOPEXIT1]] ]
+; CHECK-NEXT:    ret ptr [[RES]]
+;
+; DISABLE-LABEL: define ptr @find_first_of_i16(
+; DISABLE-SAME: ptr [[SEARCH_START:%.*]], ptr [[SEARCH_END:%.*]], ptr [[NEEDLE_START:%.*]], ptr [[NEEDLE_END:%.*]]) #[[ATTR0]] {
+; DISABLE-NEXT:  [[ENTRY:.*]]:
+; DISABLE-NEXT:    [[SEARCH_TEST:%.*]] = icmp eq ptr [[SEARCH_START]], [[SEARCH_END]]
+; DISABLE-NEXT:    [[NEEDLE_TEST:%.*]] = icmp eq ptr [[NEEDLE_START]], [[NEEDLE_END]]
+; DISABLE-NEXT:    [[COMBINED_TEST:%.*]] = or i1 [[SEARCH_TEST]], [[NEEDLE_TEST]]
+; DISABLE-NEXT:    br i1 [[COMBINED_TEST]], label %[[EXIT:.*]], label %[[HEADER_PREHEADER:.*]]
+; DISABLE:       [[HEADER_PREHEADER]]:
+; DISABLE-NEXT:    br label %[[HEADER:.*]]
+; DISABLE:       [[HEADER]]:
+; DISABLE-NEXT:    [[SEARCH_PTR:%.*]] = phi ptr [ [[SEARCH_NEXT:%.*]], %[[SEARCH_CHECK:.*]] ], [ [[SEARCH_START]], %[[HEADER_PREHEADER]] ]
+; DISABLE-NEXT:    [[SEARCH_LOAD:%.*]] = load i16, ptr [[SEARCH_PTR]], align 1
+; DISABLE-NEXT:    br label %[[MATCH_CHECK:.*]]
+; DISABLE:       [[NEEDLE_CHECK:.*]]:
+; DISABLE-NEXT:    [[NEEDLE_NEXT:%.*]] = getelementptr inbounds i16, ptr [[NEEDLE_PTR:%.*]], i64 1
+; DISABLE-NEXT:    [[NEEDLE_CMP:%.*]] = icmp eq ptr [[NEEDLE_NEXT]], [[NEEDLE_END]]
+; DISABLE-NEXT:    br i1 [[NEEDLE_CMP]], label %[[SEARCH_CHECK]], label %[[MATCH_CHECK]]
+; DISABLE:       [[MATCH_CHECK]]:
+; DISABLE-NEXT:    [[NEEDLE_PTR]] = phi ptr [ [[NEEDLE_START]], %[[HEADER]] ], [ [[NEEDLE_NEXT]], %[[NEEDLE_CHECK]] ]
+; DISABLE-NEXT:    [[NEEDLE_LOAD:%.*]] = load i16, ptr [[NEEDLE_PTR]], align 1
+; DISABLE-NEXT:    [[MATCH_CMP:%.*]] = icmp eq i16 [[SEARCH_LOAD]], [[NEEDLE_LOAD]]
+; DISABLE-NEXT:    br i1 [[MATCH_CMP]], label %[[EXIT_LOOPEXIT:.*]], label %[[NEEDLE_CHECK]]
+; DISABLE:       [[SEARCH_CHECK]]:
+; DISABLE-NEXT:    [[SEARCH_NEXT]] = getelementptr inbounds i16, ptr [[SEARCH_PTR]], i64 1
+; DISABLE-NEXT:    [[SEARCH_CMP:%.*]] = icmp eq ptr [[SEARCH_NEXT]], [[SEARCH_END]]
+; DISABLE-NEXT:    br i1 [[SEARCH_CMP]], label %[[EXIT_LOOPEXIT1:.*]], label %[[HEADER]]
+; DISABLE:       [[EXIT_LOOPEXIT]]:
+; DISABLE-NEXT:    [[SEARCH_PTR_LCSSA:%.*]] = phi ptr [ [[SEARCH_PTR]], %[[MATCH_CHECK]] ]
+; DISABLE-NEXT:    br label %[[EXIT]]
+; DISABLE:       [[EXIT_LOOPEXIT1]]:
+; DISABLE-NEXT:    br label %[[EXIT]]
+; DISABLE:       [[EXIT]]:
+; DISABLE-NEXT:    [[RES:%.*]] = phi ptr [ [[SEARCH_END]], %[[ENTRY]] ], [ [[SEARCH_PTR_LCSSA]], %[[EXIT_LOOPEXIT]] ], [ [[SEARCH_END]], %[[EXIT_LOOPEXIT1]] ]
+; DISABLE-NEXT:    ret ptr [[RES]]
+;
+entry:
+  %search_test = icmp eq ptr %search_start, %search_end
+  %needle_test = icmp eq ptr %needle_start, %needle_end
+  %combined_test = or i1 %search_test, %needle_test
+  br i1 %combined_test, label %exit, label %header
+
+header:
+  %search_ptr = phi ptr [ %search_next, %search_check ], [ %search_start, %entry ]
+  %search_load = load i16, ptr %search_ptr, align 1
+  br label %match_check
+
+needle_check:
+  %needle_next = getelementptr inbounds i16, ptr %needle_ptr, i64 1
+  %needle_cmp = icmp eq ptr %needle_next, %needle_end
+  br i1 %needle_cmp, label %search_check, label %match_check
+
+match_check:
+  %needle_ptr = phi ptr [ %needle_start, %header ], [ %needle_next, %needle_check ]
+  %needle_load = load i16, ptr %needle_ptr, align 1
+  %match_cmp = icmp eq i16 %search_load, %needle_load
+  br i1 %match_cmp, label %exit, label %needle_check
+
+search_check:
+  %search_next = getelementptr inbounds i16, ptr %search_ptr, i64 1
+  %search_cmp = icmp eq ptr %search_next, %search_end
+  br i1 %search_cmp, label %exit, label %header
+
+exit:
+  %res = phi ptr [ %search_end, %entry ], [ %search_ptr, %match_check ], [ %search_end, %search_check ]
+  ret ptr %res
+}
+
+; Same as @find_first_of_i8 but with two intermediate exit blocks for the
+; "success" (exit_succ) and "failure" (exit_fail) paths.
+define ptr @find_first_of_i8_multi_exit(ptr %search_start, ptr %search_end, ptr %needle_start, ptr %needle_end) #0 {
+; CHECK-LABEL: define ptr @find_first_of_i8_multi_exit(
+; CHECK-SAME: ptr [[SEARCH_START:%.*]], ptr [[SEARCH_END:%.*]], ptr [[NEEDLE_START:%.*]], ptr [[NEEDLE_END:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    [[SEARCH_TEST:%.*]] = icmp eq ptr [[SEARCH_START]], [[SEARCH_END]]
+; CHECK-NEXT:    [[NEEDLE_TEST:%.*]] = icmp eq ptr [[NEEDLE_START]], [[NEEDLE_END]]
+; CHECK-NEXT:    [[COMBINED_TEST:%.*]] = or i1 [[SEARCH_TEST]], [[NEEDLE_TEST]]
+; CHECK-NEXT:    br i1 [[COMBINED_TEST]], label %[[EXIT_FAIL:.*]], label %[[HEADER_PREHEADER:.*]]
+; CHECK:       [[HEADER_PREHEADER]]:
+; CHECK-NEXT:    br label %[[MEM_CHECK:.*]]
+; CHECK:       [[MEM_CHECK]]:
+; CHECK-NEXT:    [[SEARCH_START_INT:%.*]] = ptrtoint ptr [[SEARCH_START]] to i64
+; CHECK-NEXT:    [[SEARCH_END_INT:%.*]] = ptrtoint ptr [[SEARCH_END]] to i64
+; CHECK-NEXT:    [[NEEDLE_START_INT:%.*]] = ptrtoint ptr [[NEEDLE_START]] to i64
+; CHECK-NEXT:    [[NEEDLE_END_INT:%.*]] = ptrtoint ptr [[NEEDLE_END]] to i64
+; CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i64(i64 0, i64 16)
+; CHECK-NEXT:    [[SEARCH_START_PAGE:%.*]] = lshr i64 [[SEARCH_START_INT]], 12
+; CHECK-NEXT:    [[SEARCH_END_PAGE:%.*]] = lshr i64 [[SEARCH_END_INT]], 12
+; CHECK-NEXT:    [[NEEDLE_START_PAGE:%.*]] = lshr i64 [[NEEDLE_START_INT]], 12
+; CHECK-NEXT:    [[NEEDLE_END_PAGE:%.*]] = lshr i64 [[NEEDLE_END_INT]], 12
+; CHECK-NEXT:    [[SEARCH_PAGE_CMP:%.*]] = icmp ne i64 [[SEARCH_START_PAGE]], [[SEARCH_END_PAGE]]
+; CHECK-NEXT:    [[NEEDLE_PAGE_CMP:%.*]] = icmp ne i64 [[NEEDLE_START_PAGE]], [[NEEDLE_END_PAGE]]
+; CHECK-NEXT:    [[COMBINED_PAGE_CMP:%.*]] = or i1 [[SEARCH_PAGE_CMP]], [[NEEDLE_PAGE_CMP]]
+; CHECK-NEXT:    br i1 [[COMBINED_PAGE_CMP]], label %[[SCALAR_PREHEADER:.*]], label %[[FIND_FIRST_VEC_HEADER:.*]], !prof [[PROF0]]
+; CHECK:       [[FIND_FIRST_VEC_HEADER]]:
+; CHECK-NEXT:    [[PSEARCH:%.*]] = phi ptr [ [[SEARCH_START]], %[[MEM_CHECK]] ], [ [[SEARCH_NEXT_VEC:%.*]], %[[SEARCH_CHECK_VEC:.*]] ]
+; CHECK-NEXT:    [[TMP1:%.*]] = ptrtoint ptr [[PSEARCH]] to i64
+; CHECK-NEXT:    [[SEARCH_PRED:%.*]] = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i64(i64 [[TMP1]], i64 [[SEARCH_END_INT]])
+; CHECK-NEXT:    [[SEARCH_MASKED:%.*]] = and <vscale x 16 x i1> [[TMP0]], [[SEARCH_PRED]]
+; CHECK-NEXT:    [[SEARCH_LOAD_VEC:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr [[PSEARCH]], i32 1, <vscale x 16 x i1> [[SEARCH_MASKED]], <vscale x 16 x i8> zeroinitializer)
+; CHECK-NEXT:    br label %[[MATCH_CHECK_VEC:.*]]
+; CHECK:       [[MATCH_CHECK_VEC]]:
+; CHECK-NEXT:    [[PNEEDLE:%.*]] = phi ptr [ [[NEEDLE_START]], %[[FIND_FIRST_VEC_HEADER]] ], [ [[NEEDLE_NEXT_VEC:%.*]], %[[NEEDLE_CHECK_VEC:.*]] ]
+; CHECK-NEXT:    [[TMP2:%.*]] = ptrtoint ptr [[PNEEDLE]] to i64
+; CHECK-NEXT:    [[NEEDLE_PRED:%.*]] = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i64(i64 [[TMP2]], i64 [[NEEDLE_END_INT]])
+; CHECK-NEXT:    [[NEEDLE_MASKED:%.*]] = and <vscale x 16 x i1> [[TMP0]], [[NEEDLE_PRED]]
+; CHECK-NEXT:    [[NEEDLE_LOAD_VEC:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr [[PNEEDLE]], i32 1, <vscale x 16 x i1> [[NEEDLE_MASKED]], <vscale x 16 x i8> zeroinitializer)
+; CHECK-NEXT:    [[NEEDLE0:%.*]] = extractelement <vscale x 16 x i8> [[NEEDLE_LOAD_VEC]], i64 0
+; CHECK-NEXT:    [[NEEDLE0_SPLATINSERT:%.*]] = insertelement <vscale x 16 x i8> poison, i8 [[NEEDLE0]], i64 0
+; CHECK-NEXT:    [[NEEDLE0_SPLAT:%.*]] = shufflevector <vscale x 16 x i8> [[NEEDLE0_SPLATINSERT]], <vscale x 16 x i8> poison, <vscale x 16 x i32> zeroinitializer
+; CHECK-NEXT:    [[NEEDLE_SPLAT:%.*]] = select <vscale x 16 x i1> [[NEEDLE_MASKED]], <vscale x 16 x i8> [[NEEDLE_LOAD_VEC]], <vscale x 16 x i8> [[NEEDLE0_SPLAT]]
+; CHECK-NEXT:    [[NEEDLE_VEC:%.*]] = call <16 x i8> @llvm.vector.extract.v16i8.nxv16i8(<vscale x 16 x i8> [[NEEDLE_SPLAT]], i64 0)
+; CHECK-NEXT:    [[MATCH_PRED:%.*]] = call <vscale x 16 x i1> @llvm.experimental.vector.match.nxv16i8.v16i8(<vscale x 16 x i8> [[SEARCH_LOAD_VEC]], <16 x i8> [[NEEDLE_VEC]], <vscale x 16 x i1> [[SEARCH_MASKED]])
+; CHECK-NEXT:    [[TMP3:%.*]] = call i1 @llvm.vector.reduce.or.nxv16i1(<vscale x 16 x i1> [[MATCH_PRED]])
+; CHECK-NEXT:    br i1 [[TMP3]], label %[[CALCULATE_MATCH:.*]], label %[[NEEDLE_CHECK_VEC]]
+; CHECK:       [[CALCULATE_MATCH]]:
+; CHECK-NEXT:    [[MATCH_START:%.*]] = phi ptr [ [[PSEARCH]], %[[MATCH_CHECK_VEC]] ]
+; CHECK-NEXT:    [[MATCH_VEC:%.*]] = phi <vscale x 16 x i1> [ [[MATCH_PRED]], %[[MATCH_CHECK_VEC]] ]
+; CHECK-NEXT:    [[MATCH_IDX:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.nxv16i1(<vscale x 16 x i1> [[MATCH_VEC]], i1 true)
+; CHECK-NEXT:    [[MATCH_RES:%.*]] = getelementptr i8, ptr [[MATCH_START]], i64 [[MATCH_IDX]]
+; CHECK-NEXT:    br label %[[EXIT_SUCC:.*]]
+; CHECK:       [[NEEDLE_CHECK_VEC]]:
+; CHECK-NEXT:    [[NEEDLE_NEXT_VEC]] = getelementptr i8, ptr [[PNEEDLE]], i64 16
+; CHECK-NEXT:    [[TMP4:%.*]] = icmp ult ptr [[NEEDLE_NEXT_VEC]], [[NEEDLE_END]]
+; CHECK-NEXT:    br i1 [[TMP4]], label %[[MATCH_CHECK_VEC]], label %[[SEARCH_CHECK_VEC]]
+; CHECK:       [[SEARCH_CHECK_VEC]]:
+; CHECK-NEXT:    [[SEARCH_NEXT_VEC]] = getelementptr i8, ptr [[PSEARCH]], i64 16
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp ult ptr [[SEARCH_NEXT_VEC]], [[SEARCH_END]]
+; CHECK-NEXT:    br i1 [[TMP5]], label %[[FIND_FIRST_VEC_HEADER]], label %[[EXIT_FAIL_LOOPEXIT:.*]]
+; CHECK:       [[SCALAR_PREHEADER]]:
+; CHECK-NEXT:    br label %[[HEADER:.*]]
+; CHECK:       [[HEADER]]:
+; CHECK-NEXT:    [[SEARCH_PTR:%.*]] = phi ptr [ [[SEARCH_NEXT:%.*]], %[[SEARCH_CHECK:.*]] ], [ [[SEARCH_START]], %[[SCALAR_PREHEADER]] ]
+; CHECK-NEXT:    [[SEARCH_LOAD:%.*]] = load i8, ptr [[SEARCH_PTR]], align 1
+; CHECK-NEXT:    br label %[[MATCH_CHECK:.*]]
+; CHECK:       [[NEEDLE_CHECK:.*]]:
+; CHECK-NEXT:    [[NEEDLE_NEXT:%.*]] = getelementptr inbounds i8, ptr [[NEEDLE_PTR:%.*]], i64 1
+; CHECK-NEXT:    [[NEEDLE_CMP:%.*]] = icmp eq ptr [[NEEDLE_NEXT]], [[NEEDLE_END]]
+; CHECK-NEXT:    br i1 [[NEEDLE_CMP]], label %[[SEARCH_CHECK]], label %[[MATCH_CHECK]]
+; CHECK:       [[MATCH_CHECK]]:
+; CHECK-NEXT:    [[NEEDLE_PTR]] = phi ptr [ [[NEEDLE_START]], %[[HEADER]] ], [ [[NEEDLE_NEXT]], %[[NEEDLE_CHECK]] ]
+; CHECK-NEXT:    [[NEEDLE_LOAD:%.*]] = load i8, ptr [[NEEDLE_PTR]], align 1
+; CHECK-NEXT:    [[MATCH_CMP:%.*]] = icmp eq i8 [[SEARCH_LOAD]], [[NEEDLE_LOAD]]
+; CHECK-NEXT:    br i1 [[MATCH_CMP]], label %[[EXIT_SUCC]], label %[[NEEDLE_CHECK]]
+; CHECK:       [[SEARCH_CHECK]]:
+; CHECK-NEXT:    [[SEARCH_NEXT]] = getelementptr inbounds i8, ptr [[SEARCH_PTR]], i64 1
+; CHECK-NEXT:    [[SEARCH_CMP:%.*]] = icmp eq ptr [[SEARCH_NEXT]], [[SEARCH_END]]
+; CHECK-NEXT:    br i1 [[SEARCH_CMP]], label %[[EXIT_FAIL_LOOPEXIT]], label %[[HEADER]]
+; CHECK:       [[EXIT_SUCC]]:
+; CHECK-NEXT:    [[RES_SUCC:%.*]] = phi ptr [ [[SEARCH_PTR]], %[[MATCH_CHECK]] ], [ [[MATCH_RES]], %[[CALCULATE_MATCH]] ]
+; CHECK-NEXT:    br label %[[EXIT:.*]]
+; CHECK:       [[EXIT_FAIL_LOOPEXIT]]:
+; CHECK-NEXT:    br label %[[EXIT_FAIL]]
+; CHECK:       [[EXIT_FAIL]]:
+; CHECK-NEXT:    [[RES_FAIL:%.*]] = phi ptr [ [[SEARCH_END]], %[[ENTRY]] ], [ [[SEARCH_END]], %[[EXIT_FAIL_LOOPEXIT]] ]
+; CHECK-NEXT:    br label %[[EXIT]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    [[RES:%.*]] = phi ptr [ [[RES_SUCC]], %[[EXIT_SUCC]] ], [ [[RES_FAIL]], %[[EXIT_FAIL]] ]
+; CHECK-NEXT:    ret ptr [[RES]]
+;
+; DISABLE-LABEL: define ptr @find_first_of_i8_multi_exit(
+; DISABLE-SAME: ptr [[SEARCH_START:%.*]], ptr [[SEARCH_END:%.*]], ptr [[NEEDLE_START:%.*]], ptr [[NEEDLE_END:%.*]]) #[[ATTR0]] {
+; DISABLE-NEXT:  [[ENTRY:.*]]:
+; DISABLE-NEXT:    [[SEARCH_TEST:%.*]] = icmp eq ptr [[SEARCH_START]], [[SEARCH_END]]
+; DISABLE-NEXT:    [[NEEDLE_TEST:%.*]] = icmp eq ptr [[NEEDLE_START]], [[NEEDLE_END]]
+; DISABLE-NEXT:    [[COMBINED_TEST:%.*]] = or i1 [[SEARCH_TEST]], [[NEEDLE_TEST]]
+; DISABLE-NEXT:    br i1 [[COMBINED_TEST]], label %[[EXIT_FAIL:.*]], label %[[HEADER_PREHEADER:.*]]
+; DISABLE:       [[HEADER_PREHEADER]]:
+; DISABLE-NEXT:    br label %[[HEADER:.*]]
+; DISABLE:       [[HEADER]]:
+; DISABLE-NEXT:    [[SEARCH_PTR:%.*]] = phi ptr [ [[SEARCH_NEXT:%.*]], %[[SEARCH_CHECK:.*]] ], [ [[SEARCH_START]], %[[HEADER_PREHEADER]] ]
+; DISABLE-NEXT:    [[SEARCH_LOAD:%.*]] = load i8, ptr [[SEARCH_PTR]], align 1
+; DISABLE-NEXT:    br label %[[MATCH_CHECK:.*]]
+; DISABLE:       [[NEEDLE_CHECK:.*]]:
+; DISABLE-NEXT:    [[NEEDLE_NEXT:%.*]] = getelementptr inbounds i8, ptr [[NEEDLE_PTR:%.*]], i64 1
+; DISABLE-NEXT:    [[NEEDLE_CMP:%.*]] = icmp eq ptr [[NEEDLE_NEXT]], [[NEEDLE_END]]
+; DISABLE-NEXT:    br i1 [[NEEDLE_CMP]], label %[[SEARCH_CHECK]], label %[[MATCH_CHECK]]
+; DISABLE:       [[MATCH_CHECK]]:
+; DISABLE-NEXT:    [[NEEDLE_PTR]] = phi ptr [ [[NEEDLE_START]], %[[HEADER]] ], [ [[NEEDLE_NEXT]], %[[NEEDLE_CHECK]] ]
+; DISABLE-NEXT:    [[NEEDLE_LOAD:%.*]] = load i8, ptr [[NEEDLE_PTR]], align 1
+; DISABLE-NEXT:    [[MATCH_CMP:%.*]] = icmp eq i8 [[SEARCH_LOAD]], [[NEEDLE_LOAD]]
+; DISABLE-NEXT:    br i1 [[MATCH_CMP]], label %[[EXIT_SUCC:.*]], label %[[NEEDLE_CHECK]]
+; DISABLE:       [[SEARCH_CHECK]]:
+; DISABLE-NEXT:    [[SEARCH_NEXT]] = getelementptr inbounds i8, ptr [[SEARCH_PTR]], i64 1
+; DISABLE-NEXT:    [[SEARCH_CMP:%.*]] = icmp eq ptr [[SEARCH_NEXT]], [[SEARCH_END]]
+; DISABLE-NEXT:    br i1 [[SEARCH_CMP]], label %[[EXIT_FAIL_LOOPEXIT:.*]], label %[[HEADER]]
+; DISABLE:       [[EXIT_SUCC]]:
+; DISABLE-NEXT:    [[RES_SUCC:%.*]] = phi ptr [ [[SEARCH_PTR]], %[[MATCH_CHECK]] ]
+; DISABLE-NEXT:    br label %[[EXIT:.*]]
+; DISABLE:       [[EXIT_FAIL_LOOPEXIT]]:
+; DISABLE-NEXT:    br label %[[EXIT_FAIL]]
+; DISABLE:       [[EXIT_FAIL]]:
+; DISABLE-NEXT:    [[RES_FAIL:%.*]] = phi ptr [ [[SEARCH_END]], %[[ENTRY]] ], [ [[SEARCH_END]], %[[EXIT_FAIL_LOOPEXIT]] ]
+; DISABLE-NEXT:    br label %[[EXIT]]
+; DISABLE:       [[EXIT]]:
+; DISABLE-NEXT:    [[RES:%.*]] = phi ptr [ [[RES_SUCC]], %[[EXIT_SUCC]] ], [ [[RES_FAIL]], %[[EXIT_FAIL]] ]
+; DISABLE-NEXT:    ret ptr [[RES]]
+;
+entry:
+  %search_test = icmp eq ptr %search_start, %search_end
+  %needle_test = icmp eq ptr %needle_start, %needle_end
+  %combined_test = or i1 %search_test, %needle_test
+  br i1 %combined_test, label %exit_fail, label %header
+
+header:
+  %search_ptr = phi ptr [ %search_next, %search_check ], [ %search_start, %entry ]
+  %search_load = load i8, ptr %search_ptr, align 1
+  br label %match_check
+
+needle_check:
+  %needle_next = getelementptr inbounds i8, ptr %needle_ptr, i64 1
+  %needle_cmp = icmp eq ptr %needle_next, %needle_end
+  br i1 %needle_cmp, label %search_check, label %match_check
+
+match_check:
+  %needle_ptr = phi ptr [ %needle_start, %header ], [ %needle_next, %needle_check ]
+  %needle_load = load i8, ptr %needle_ptr, align 1
+  %match_cmp = icmp eq i8 %search_load, %needle_load
+  br i1 %match_cmp, label %exit_succ, label %needle_check
+
+search_check:
+  %search_next = getelementptr inbounds i8, ptr %search_ptr, i64 1
+  %search_cmp = icmp eq ptr %search_next, %search_end
+  br i1 %search_cmp, label %exit_fail, label %header
+
+exit_succ:
+  %res_succ = phi ptr [ %search_ptr, %match_check ]
+  br label %exit
+
+exit_fail:
+  %res_fail = phi ptr [ %search_end, %entry ], [ %search_end, %search_check ]
+  br label %exit
+
+exit:
+  %res = phi ptr [ %res_succ, %exit_succ ], [ %res_fail, %exit_fail ]
+  ret ptr %res
+}
+
+; From here on we only test for the presence/absence of the intrinsic.
+; UTC_ARGS: --disable
+
+; Same as @find_first_of_i8 but with `ne' comparison.
+; This is rejected for now, but should eventually be supported.
+define ptr @find_first_not_of_i8(ptr %search_start, ptr %search_end, ptr %needle_start, ptr %needle_end) #0 {
+; CHECK-LABEL: define ptr @find_first_not_of_i8(
+; CHECK-NOT: {{%.*}} @llvm.experimental.vector.match{{.*}}
+;
+; DISABLE-LABEL: define ptr @find_first_not_of_i8(
+; DISABLE-NOT: {{%.*}} @llvm.experimental.vector.match{{.*}}
+;
+entry:
+  %search_test = icmp eq ptr %search_start, %search_end
+  %needle_test = icmp eq ptr %needle_start, %needle_end
+  %combined_test = or i1 %search_test, %needle_test
+  br i1 %combined_test, label %exit, label %header
+
+header:
+  %search_ptr = phi ptr [ %search_next, %search_check ], [ %search_start, %entry ]
+  %search_load = load i8, ptr %search_ptr, align 1
+  br label %match_check
+
+needle_check:
+  %needle_next = getelementptr inbounds i8, ptr %needle_ptr, i64 1
+  %needle_cmp = icmp eq ptr %needle_next, %needle_end
+  br i1 %needle_cmp, label %search_check, label %match_check
+
+match_check:
+  %needle_ptr = phi ptr [ %needle_start, %header ], [ %needle_next, %needle_check ]
+  %needle_load = load i8, ptr %needle_ptr, align 1
+  %match_cmp = icmp ne i8 %search_load, %needle_load
+  br i1 %match_cmp, label %exit, label %needle_check
+
+search_check:
+  %search_next = getelementptr inbounds i8, ptr %search_ptr, i64 1
+  %search_cmp = icmp eq ptr %search_next, %search_end
+  br i1 %search_cmp, label %exit, label %header
+
+exit:
+  %res = phi ptr [ %search_end, %entry ], [ %search_ptr, %match_check ], [ %search_end, %search_check ]
+  ret ptr %res
+}
+
+; This is the same as @find_first_of_i8 but without SVE2, which we require to
+; perform the conversion.
+define ptr @find_first_of_i8_nosve2(ptr %search_start, ptr %search_end, ptr %needle_start, ptr %needle_end) {
+; CHECK-LABEL: define ptr @find_first_of_i8_nosve2(
+; CHECK-NOT: {{%.*}} @llvm.experimental.vector.match{{.*}}
+;
+; DISABLE-LABEL: define ptr @find_first_of_i8_nosve2(
+; DISABLE-NOT: {{%.*}} @llvm.experimental.vector.match{{.*}}
+;
+entry:
+  %search_test = icmp eq ptr %search_start, %search_end
+  %needle_test = icmp eq ptr %needle_start, %needle_end
+  %combined_test = or i1 %search_test, %needle_test
+  br i1 %combined_test, label %exit, label %header
+
+header:
+  %search_ptr = phi ptr [ %search_next, %search_check ], [ %search_start, %entry ]
+  %search_load = load i8, ptr %search_ptr, align 1
+  br label %match_check
+
+needle_check:
+  %needle_next = getelementptr inbounds i8, ptr %needle_ptr, i64 1
+  %needle_cmp = icmp eq ptr %needle_next, %needle_end
+  br i1 %needle_cmp, label %search_check, label %match_check
+
+match_check:
+  %needle_ptr = phi ptr [ %needle_start, %header ], [ %needle_next, %needle_check ]
+  %needle_load = load i8, ptr %needle_ptr, align 1
+  %match_cmp = icmp eq i8 %search_load, %needle_load
+  br i1 %match_cmp, label %exit, label %needle_check
+
+search_check:
+  %search_next = getelementptr inbounds i8, ptr %search_ptr, i64 1
+  %search_cmp = icmp eq ptr %search_next, %search_end
+  br i1 %search_cmp, label %exit, label %header
+
+exit:
+  %res = phi ptr [ %search_end, %entry ], [ %search_ptr, %match_check ], [ %search_end, %search_check ]
+  ret ptr %res
+}
+
+; Same as @find_first_of_i8 but here we use the inner PHI outside the loop nest.
+; This isn't supported.
+define ptr @find_first_of_i8_outside_use(ptr %search_start, ptr %search_end, ptr %needle_start, ptr %needle_end) #0 {
+; CHECK-LABEL: define ptr @find_first_of_i8_outside_use(
+; CHECK-NOT: {{%.*}} @llvm.experimental.vector.match{{.*}}
+;
+; DISABLE-LABEL: define ptr @find_first_of_i8_outside_use(
+; DISABLE-NOT: {{%.*}} @llvm.experimental.vector.match{{.*}}
+;
+entry:
+  %search_test = icmp eq ptr %search_start, %search_end
+  %needle_test = icmp eq ptr %needle_start, %needle_end
+  %combined_test = or i1 %search_test, %needle_test
+  br i1 %combined_test, label %exit, label %header
+
+header:
+  %search_ptr = phi ptr [ %search_next, %search_check ], [ %search_start, %entry ]
+  %search_load = load i8, ptr %search_ptr, align 1
+  br label %match_check
+
+needle_check:
+  %needle_next = getelementptr inbounds i8, ptr %needle_ptr, i64 1
+  %needle_cmp = icmp eq ptr %needle_next, %needle_end
+  br i1 %needle_cmp, label %search_check, label %match_check
+
+match_check:
+  %needle_ptr = phi ptr [ %needle_start, %header ], [ %needle_next, %needle_check ]
+  %needle_load = load i8, ptr %needle_ptr, align 1
+  %match_cmp = icmp eq i8 %search_load, %needle_load
+  br i1 %match_cmp, label %exit, label %needle_check
+
+search_check:
+  %search_next = getelementptr inbounds i8, ptr %search_ptr, i64 1
+  %search_cmp = icmp eq ptr %search_next, %search_end
+  br i1 %search_cmp, label %exit, label %header
+
+exit:
+  %res = phi ptr [ %search_end, %entry ], [ %search_ptr, %match_check ], [ %search_end, %search_check ]
+  %use = phi ptr [ %needle_end, %entry ], [ %needle_ptr, %match_check ], [ %needle_end, %search_check ]
+  ret ptr %res
+}
+
+; Same as @find_first_of_i8_multi_exit but `search_ptr' is used in `exit_fail'
+; which should block the transform.
+define ptr @find_first_of_i8_multi_exit_outside_use(ptr %search_start, ptr %search_end, ptr %needle_start, ptr %needle_end) #0 {
+; CHECK-LABEL: define ptr @find_first_of_i8_multi_exit_outside_use(
+; CHECK-NOT: {{%.*}} @llvm.experimental.vector.match{{.*}}
+;
+; DISABLE-LABEL: define ptr @find_first_of_i8_multi_exit_outside_use(
+; DISABLE-NOT: {{%.*}} @llvm.experimental.vector.match{{.*}}
+;
+entry:
+  %search_test = icmp eq ptr %search_start, %search_end
+  %needle_test = icmp eq ptr %needle_start, %needle_end
+  %combined_test = or i1 %search_test, %needle_test
+  br i1 %combined_test, label %exit_fail, label %header
+
+header:
+  %search_ptr = phi ptr [ %search_next, %search_check ], [ %search_start, %entry ]
+  %search_load = load i8, ptr %search_ptr, align 1
+  br label %match_check
+
+needle_check:
+  %needle_next = getelementptr inbounds i8, ptr %needle_ptr, i64 1
+  %needle_cmp = icmp eq ptr %needle_next, %needle_end
+  br i1 %needle_cmp, label %search_check, label %match_check
+
+match_check:
+  %needle_ptr = phi ptr [ %needle_start, %header ], [ %needle_next, %needle_check ]
+  %needle_load = load i8, ptr %needle_ptr, align 1
+  %match_cmp = icmp eq i8 %search_load, %needle_load
+  br i1 %match_cmp, label %exit_succ, label %needle_check
+
+search_check:
+  %search_next = getelementptr inbounds i8, ptr %search_ptr, i64 1
+  %search_cmp = icmp eq ptr %search_next, %search_end
+  br i1 %search_cmp, label %exit_fail, label %header
+
+exit_succ:
+  %res_succ = phi ptr [ %search_ptr, %match_check ]
+  br label %exit
+
+exit_fail:
+  %res_fail = phi ptr [ %search_end, %entry ], [ %search_ptr, %search_check ]
+  br label %exit
+
+exit:
+  %res = phi ptr [ %res_succ, %exit_succ ], [ %res_fail, %exit_fail ]
+  ret ptr %res
+}
+
+attributes #0 = { "target-features"="+sve2" }
+
+; CHECK: [[PROF0]] = !{!"branch_weights", i32 10, i32 90}
diff --git a/llvm/test/Transforms/LoopIdiom/memset-pattern-tbaa.ll b/llvm/test/Transforms/LoopIdiom/memset-pattern-tbaa.ll
index 6b7e9a8192c4c..57a91a3bf6e2c 100644
--- a/llvm/test/Transforms/LoopIdiom/memset-pattern-tbaa.ll
+++ b/llvm/test/Transforms/LoopIdiom/memset-pattern-tbaa.ll
@@ -1,4 +1,4 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals
 ; RUN: opt -passes="loop-idiom" < %s -S | FileCheck %s
 
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
@@ -6,6 +6,11 @@ target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f3
 target triple = "x86_64-apple-darwin10.0.0"
 
 
+;.
+; CHECK: @.memset_pattern = private unnamed_addr constant [2 x double] [double 3.141590e+00, double 3.141590e+00], align 16
+; CHECK: @.memset_pattern.1 = private unnamed_addr constant [2 x double] [double 3.141590e+00, double 3.141590e+00], align 16
+; CHECK: @.memset_pattern.2 = private unnamed_addr constant [2 x double] [double 3.141590e+00, double 3.141590e+00], align 16
+;.
 define dso_local void @double_memset(ptr nocapture %p) {
 ; CHECK-LABEL: @double_memset(
 ; CHECK-NEXT:  entry:
@@ -110,3 +115,15 @@ for.body:
 !20 = !{!"any pointer", !7, i64 0}
 !21 = !{!22, !20, i64 0}
 !22 = !{!"B", !20, i64 0}
+;.
+; CHECK: attributes #[[ATTR0:[0-9]+]] = { nofree nounwind willreturn memory(argmem: readwrite) }
+;.
+; CHECK: [[TBAA0]] = !{[[META1:![0-9]+]], [[META1]], i64 0}
+; CHECK: [[META1]] = !{!"double", [[META2:![0-9]+]], i64 0}
+; CHECK: [[META2]] = !{!"omnipotent char", [[META3:![0-9]+]], i64 0}
+; CHECK: [[META3]] = !{!"Simple C++ TBAA"}
+; CHECK: [[TBAA4]] = !{[[META5:![0-9]+]], [[META7:![0-9]+]], i64 0, i64 128}
+; CHECK: [[META5]] = !{[[META6:![0-9]+]], i64 32, !"_ZTS1A", [[META7]], i64 0, i64 8, [[META7]], i64 8, i64 8, [[META7]], i64 16, i64 8, [[META7]], i64 24, i64 8}
+; CHECK: [[META6]] = !{[[META3]], i64 0, !"omnipotent char"}
+; CHECK: [[META7]] = !{[[META6]], i64 8, !"double"}
+;.
diff --git a/llvm/test/Transforms/LoopIdiom/struct_pattern.ll b/llvm/test/Transforms/LoopIdiom/struct_pattern.ll
index 5335b12e63982..b65e95353ab3e 100644
--- a/llvm/test/Transforms/LoopIdiom/struct_pattern.ll
+++ b/llvm/test/Transforms/LoopIdiom/struct_pattern.ll
@@ -1,10 +1,7 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals
 ; RUN: opt -passes=loop-idiom < %s -S | FileCheck %s
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
 
-; CHECK: @.memset_pattern = private unnamed_addr constant [4 x i32] [i32 2, i32 2, i32 2, i32 2], align 16
-; CHECK: @.memset_pattern.1 = private unnamed_addr constant [4 x i32] [i32 2, i32 2, i32 2, i32 2], align 16
-; CHECK: @.memset_pattern.2 = private unnamed_addr constant [4 x i32] [i32 2, i32 2, i32 2, i32 2], align 16
 
 target triple = "x86_64-apple-darwin10.0.0"
 
@@ -17,6 +14,13 @@ target triple = "x86_64-apple-darwin10.0.0"
 ;    f[i].b = 2;
 ;  }
 ;}
+
+
+;.
+; CHECK: @.memset_pattern = private unnamed_addr constant [4 x i32] [i32 2, i32 2, i32 2, i32 2], align 16
+; CHECK: @.memset_pattern.1 = private unnamed_addr constant [4 x i32] [i32 2, i32 2, i32 2, i32 2], align 16
+; CHECK: @.memset_pattern.2 = private unnamed_addr constant [4 x i32] [i32 2, i32 2, i32 2, i32 2], align 16
+;.
 define void @bar1(ptr %f, i32 %n) nounwind ssp {
 ; CHECK-LABEL: @bar1(
 ; CHECK-NEXT:  entry:
@@ -286,3 +290,7 @@ for.end.loopexit:                                 ; preds = %for.body
 for.end:                                          ; preds = %for.end.loopexit, %entry
   ret void
 }
+;.
+; CHECK: attributes #[[ATTR0:[0-9]+]] = { nounwind ssp }
+; CHECK: attributes #[[ATTR1:[0-9]+]] = { nofree nounwind willreturn memory(argmem: readwrite) }
+;.
diff --git a/llvm/test/Transforms/LoopIdiom/unroll-custom-dl.ll b/llvm/test/Transforms/LoopIdiom/unroll-custom-dl.ll
index ac50c8716c73c..a3b421907885b 100644
--- a/llvm/test/Transforms/LoopIdiom/unroll-custom-dl.ll
+++ b/llvm/test/Transforms/LoopIdiom/unroll-custom-dl.ll
@@ -1,8 +1,7 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals
 ; RUN: opt -passes=loop-idiom < %s -S | FileCheck %s
 target datalayout = "e-p:64:64:64:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
 
-; CHECK: @.memset_pattern = private unnamed_addr constant [4 x i32] [i32 2, i32 2, i32 2, i32 2], align 16
 
 target triple = "x86_64-apple-darwin10.0.0"
 
@@ -12,6 +11,9 @@ target triple = "x86_64-apple-darwin10.0.0"
 ;    f[i+1] = 0;
 ;  }
 ;}
+;.
+; CHECK: @.memset_pattern = private unnamed_addr constant [4 x i32] [i32 2, i32 2, i32 2, i32 2], align 16
+;.
 define void @test(ptr %f, i32 %n) nounwind ssp {
 ; CHECK-LABEL: @test(
 ; CHECK-NEXT:  entry:
@@ -121,3 +123,8 @@ for.end.loopexit:                                 ; preds = %for.body
 for.end:                                          ; preds = %for.end.loopexit, %entry
   ret void
 }
+;.
+; CHECK: attributes #[[ATTR0:[0-9]+]] = { nounwind ssp }
+; CHECK: attributes #[[ATTR1:[0-9]+]] = { nocallback nofree nounwind willreturn memory(argmem: write) }
+; CHECK: attributes #[[ATTR2:[0-9]+]] = { nofree nounwind willreturn memory(argmem: readwrite) }
+;.
diff --git a/llvm/test/Transforms/LoopIdiom/unroll.ll b/llvm/test/Transforms/LoopIdiom/unroll.ll
index 7c41310abdfd8..c70eeefd6376d 100644
--- a/llvm/test/Transforms/LoopIdiom/unroll.ll
+++ b/llvm/test/Transforms/LoopIdiom/unroll.ll
@@ -1,8 +1,7 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals
 ; RUN: opt -passes=loop-idiom < %s -S | FileCheck %s
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
 
-; CHECK: @.memset_pattern = private unnamed_addr constant [4 x i32] [i32 2, i32 2, i32 2, i32 2], align 16
 
 target triple = "x86_64-apple-darwin10.0.0"
 
@@ -12,6 +11,9 @@ target triple = "x86_64-apple-darwin10.0.0"
 ;    f[i+1] = 0;
 ;  }
 ;}
+;.
+; CHECK: @.memset_pattern = private unnamed_addr constant [4 x i32] [i32 2, i32 2, i32 2, i32 2], align 16
+;.
 define void @test(ptr %f, i32 %n) nounwind ssp {
 ; CHECK-LABEL: @test(
 ; CHECK-NEXT:  entry:
@@ -125,3 +127,8 @@ for.end.loopexit:                                 ; preds = %for.body
 for.end:                                          ; preds = %for.end.loopexit, %entry
   ret void
 }
+;.
+; CHECK: attributes #[[ATTR0:[0-9]+]] = { nounwind ssp }
+; CHECK: attributes #[[ATTR1:[0-9]+]] = { nocallback nofree nounwind willreturn memory(argmem: write) }
+; CHECK: attributes #[[ATTR2:[0-9]+]] = { nofree nounwind willreturn memory(argmem: readwrite) }
+;.
diff --git a/llvm/test/Transforms/LoopInterchange/interchange-no-deps.ll b/llvm/test/Transforms/LoopInterchange/interchange-no-deps.ll
index 9e8e30d670c51..a0b2eab87945c 100644
--- a/llvm/test/Transforms/LoopInterchange/interchange-no-deps.ll
+++ b/llvm/test/Transforms/LoopInterchange/interchange-no-deps.ll
@@ -7,7 +7,7 @@
 ;
 define i32 @no_deps_interchange(ptr nocapture %Arr) {
 ; STATS-LABEL: define i32 @no_deps_interchange(
-; STATS-SAME: ptr nocapture [[ARR:%.*]]) {
+; STATS-SAME: ptr captures(none) [[ARR:%.*]]) {
 ; STATS-NEXT:  [[ENTRY:.*]]:
 ; STATS-NEXT:    br label %[[FOR2:.*]]
 ; STATS:       [[FOR1_HEADER:.*]]:
diff --git a/llvm/test/Transforms/LoopInterchange/legality-for-scalar-deps.ll b/llvm/test/Transforms/LoopInterchange/legality-for-scalar-deps.ll
new file mode 100644
index 0000000000000..c30f9a399fed8
--- /dev/null
+++ b/llvm/test/Transforms/LoopInterchange/legality-for-scalar-deps.ll
@@ -0,0 +1,322 @@
+; RUN: opt < %s -passes=loop-interchange -pass-remarks-output=%t -disable-output
+; RUN: FileCheck -input-file %t %s
+
+; This is a collection of cases where the loops were incorrectly interchanged
+; due to mishandling of scalar dependencies.
+
+;; The original code is as follows, with some simplifications from the one in
+;; the issue #46867. The interchange is illegal due to the dependency of `s`.
+;;
+;; void issue46867(int *s, int c, int ff[4][9]) {
+;;   for (int d = 0; d <= 2; d++) {
+;;     for (int e = 0; e <= 2; e++) {
+;;       if ((long)(ff[e][d] && (*s = 3), c) % 4073709551606)
+;;         ++*s;
+;;     }
+;;   }
+;; }
+
+; CHECK:      --- !Missed
+; CHECK-NEXT: Pass:            loop-interchange
+; CHECK-NEXT: Name:            Dependence
+; CHECK-NEXT: Function:        issue46867
+; CHECK-NEXT: Args:
+; CHECK-NEXT:  - String:       Cannot interchange loops due to dependences.
+; CHECK:      --- !Missed
+; CHECK-NEXT: Pass:            loop-interchange
+; CHECK-NEXT: Name:            Dependence
+; CHECK-NEXT: Function:        issue46867
+; CHECK-NEXT: Args:
+; CHECK-NEXT:  - String:       Cannot interchange loops due to dependences.
+define void @issue46867(ptr noundef captures(none) %s, i32 noundef %c, ptr noundef readonly captures(none) %ff) {
+entry:
+  %tobool7.not = icmp eq i32 %c, 0
+  br i1 %tobool7.not, label %for.cond1.preheader.us.preheader, label %entry.split
+
+for.cond1.preheader.us.preheader:
+  br label %for.cond1.preheader.us
+
+for.cond1.preheader.us:
+  %indvars.iv31 = phi i64 [ 0, %for.cond1.preheader.us.preheader ], [ %indvars.iv.next32, %for.cond.cleanup3.split.us.us ]
+  br label %for.body4.us.us
+
+for.body4.us.us:
+  %indvars.iv27 = phi i64 [ %indvars.iv.next28, %land.end.us.us ], [ 0, %for.cond1.preheader.us ]
+  %arrayidx6.us.us = getelementptr inbounds nuw [9 x i32], ptr %ff, i64 %indvars.iv27, i64 %indvars.iv31
+  %0 = load i32, ptr %arrayidx6.us.us, align 4
+  %tobool.not.us.us = icmp eq i32 %0, 0
+  br i1 %tobool.not.us.us, label %land.end.us.us, label %land.rhs.us.us
+
+land.rhs.us.us:
+  store i32 3, ptr %s, align 4
+  br label %land.end.us.us
+
+land.end.us.us:
+  %indvars.iv.next28 = add nuw nsw i64 %indvars.iv27, 1
+  %exitcond30 = icmp ne i64 %indvars.iv.next28, 3
+  br i1 %exitcond30, label %for.body4.us.us, label %for.cond.cleanup3.split.us.us
+
+for.cond.cleanup3.split.us.us:
+  %indvars.iv.next32 = add nuw nsw i64 %indvars.iv31, 1
+  %exitcond34 = icmp ne i64 %indvars.iv.next32, 3
+  br i1 %exitcond34, label %for.cond1.preheader.us, label %for.cond.cleanup.loopexit
+
+entry.split:
+  %s.promoted19 = load i32, ptr %s, align 4
+  br label %for.cond1.preheader
+
+for.cond1.preheader:
+  %indvars.iv23 = phi i64 [ 0, %entry.split ], [ %indvars.iv.next24, %for.cond.cleanup3.split ]
+  %s.promoted20 = phi i32 [ %s.promoted19, %entry.split ], [ %inc.lcssa, %for.cond.cleanup3.split ]
+  br label %for.body4
+
+for.cond.cleanup.loopexit:
+  br label %for.cond.cleanup
+
+for.cond.cleanup.loopexit21:
+  br label %for.cond.cleanup
+
+for.cond.cleanup:
+  ret void
+
+for.cond.cleanup3.split:
+  %inc.lcssa = phi i32 [ %inc, %land.end ]
+  %indvars.iv.next24 = add nuw nsw i64 %indvars.iv23, 1
+  %exitcond26 = icmp ne i64 %indvars.iv.next24, 3
+  br i1 %exitcond26, label %for.cond1.preheader, label %for.cond.cleanup.loopexit21
+
+for.body4:
+  %indvars.iv = phi i64 [ 0, %for.cond1.preheader ], [ %indvars.iv.next, %land.end ]
+  %1 = phi i32 [ %s.promoted20, %for.cond1.preheader ], [ %inc, %land.end ]
+  %arrayidx6 = getelementptr inbounds nuw [9 x i32], ptr %ff, i64 %indvars.iv, i64 %indvars.iv23
+  %2 = load i32, ptr %arrayidx6, align 4
+  %tobool.not = icmp eq i32 %2, 0
+  br i1 %tobool.not, label %land.end, label %land.rhs
+
+land.rhs:
+  store i32 3, ptr %s, align 4
+  br label %land.end
+
+land.end:
+  %3 = phi i32 [ 3, %land.rhs ], [ %1, %for.body4 ]
+  %inc = add nsw i32 %3, 1
+  store i32 %inc, ptr %s, align 4
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %exitcond = icmp ne i64 %indvars.iv.next, 3
+  br i1 %exitcond, label %for.body4, label %for.cond.cleanup3.split
+}
+
+
+;; The original code is as follows, with some simplifications from the one in
+;; the issue #47401. The interchange is illegal due to the dependency of `e`.
+;;
+;; void issue47401(int *e, int bb[][8]) {
+;;   for (int c = 0; c <= 7; c++)
+;;     for (int d = 4; d; d--)
+;;       bb[d + 2][c] && (*e = bb[d][0]);
+;; }
+
+; CHECK:      --- !Missed
+; CHECK-NEXT: Pass:            loop-interchange
+; CHECK-NEXT: Name:            Dependence
+; CHECK-NEXT: Function:        issue47401
+; CHECK-NEXT: Args:
+; CHECK-NEXT:  - String:       Cannot interchange loops due to dependences.
+define void @issue47401(ptr noundef writeonly captures(none) %e, ptr noundef readonly captures(none) %bb) {
+entry:
+  br label %for.cond1.preheader
+
+for.cond1.preheader:
+  %indvars.iv22 = phi i64 [ 0, %entry ], [ %indvars.iv.next23, %for.cond.cleanup2 ]
+  br label %for.body3
+
+for.cond.cleanup:
+  ret void
+
+for.cond.cleanup2:
+  %indvars.iv.next23 = add nuw nsw i64 %indvars.iv22, 1
+  %exitcond = icmp ne i64 %indvars.iv.next23, 8
+  br i1 %exitcond, label %for.cond1.preheader, label %for.cond.cleanup
+
+for.body3:
+  %indvars.iv = phi i64 [ 4, %for.cond1.preheader ], [ %indvars.iv.next, %land.end ]
+  %0 = getelementptr [8 x i32], ptr %bb, i64 %indvars.iv
+  %arrayidx = getelementptr i8, ptr %0, i64 64
+  %arrayidx5 = getelementptr inbounds nuw [8 x i32], ptr %arrayidx, i64 0, i64 %indvars.iv22
+  %1 = load i32, ptr %arrayidx5, align 4
+  %tobool6.not = icmp eq i32 %1, 0
+  br i1 %tobool6.not, label %land.end, label %land.rhs
+
+land.rhs:
+  %2 = load i32, ptr %0, align 4
+  store i32 %2, ptr %e, align 4
+  br label %land.end
+
+land.end:
+  %indvars.iv.next = add nsw i64 %indvars.iv, -1
+  %tobool.not = icmp eq i64 %indvars.iv.next, 0
+  br i1 %tobool.not, label %for.cond.cleanup2, label %for.body3
+}
+
+;; The original code is as follows, with some simplifications from the one in
+;; the issue #47295. The interchange is illegal due to the dependency of `f`.
+;;
+;; void issue47295(int *f, int cc[4][4]) {
+;;   for (int i = 0; i <= 3; i++) {
+;;     for (int j = 0; j <= 3; j++) {
+;;       *f ^= 0x1000;
+;;       cc[j][i] = *f;
+;;     }
+;;   }
+;; }
+
+; CHECK:      --- !Missed
+; CHECK-NEXT: Pass:            loop-interchange
+; CHECK-NEXT: Name:            Dependence
+; CHECK-NEXT: Function:        issue47295
+; CHECK-NEXT: Args:
+; CHECK-NEXT:  - String:       Cannot interchange loops due to dependences.
+define void @issue47295(ptr noundef captures(none) %f, ptr noundef writeonly captures(none) %cc) {
+entry:
+  br label %for.cond1.preheader
+
+for.cond1.preheader:
+  %indvars.iv18 = phi i64 [ 0, %entry ], [ %indvars.iv.next19, %for.cond.cleanup3 ]
+  br label %for.body4
+
+for.cond.cleanup:
+  ret void
+
+for.cond.cleanup3:
+  %indvars.iv.next19 = add nuw nsw i64 %indvars.iv18, 1
+  %exitcond21 = icmp ne i64 %indvars.iv.next19, 4
+  br i1 %exitcond21, label %for.cond1.preheader, label %for.cond.cleanup
+
+for.body4:
+  %indvars.iv = phi i64 [ 0, %for.cond1.preheader ], [ %indvars.iv.next, %for.body4 ]
+  %0 = load i32, ptr %f, align 4
+  %xor = xor i32 %0, 4096
+  store i32 %xor, ptr %f, align 4
+  %arrayidx6 = getelementptr inbounds nuw [4 x i32], ptr %cc, i64 %indvars.iv, i64 %indvars.iv18
+  store i32 %xor, ptr %arrayidx6, align 4
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %exitcond = icmp ne i64 %indvars.iv.next, 4
+  br i1 %exitcond, label %for.body4, label %for.cond.cleanup3
+}
+
+;; The original code is as follows, with some simplifications from the one in
+;; the issue #54176. The interchange is illegal due to the dependency of `aa`.
+;;
+;; void issue54176(int n, int m, float aa[1024][128], float bb[1024][128], float cc[1024][128]) {
+;;   for (int j = 1; j < 128; j++) {
+;;     for (int i = 1; i < 1024; i++) {
+;;       cc[i][j] = aa[1][j];
+;;       aa[1][j-1] += bb[i][j];
+;;     }
+;;   }
+;; }
+
+; CHECK:      --- !Missed
+; CHECK-NEXT: Pass:            loop-interchange
+; CHECK-NEXT: Name:            Dependence
+; CHECK-NEXT: Function:        issue54176
+; CHECK-NEXT: Args:
+; CHECK-NEXT:  - String:       Cannot interchange loops due to dependences.
+define void @issue54176(i32 noundef %n, i32 noundef %m, ptr noundef captures(none) %aa, ptr noundef readonly captures(none) %bb, ptr noundef writeonly captures(none) %cc) {
+
+entry:
+  %arrayidx = getelementptr inbounds nuw i8, ptr %aa, i64 512
+  br label %for.cond1.preheader
+
+for.cond1.preheader:
+  %indvars.iv32 = phi i64 [ 1, %entry ], [ %indvars.iv.next33, %for.cond.cleanup3 ]
+  %arrayidx5 = getelementptr inbounds nuw [128 x float], ptr %arrayidx, i64 0, i64 %indvars.iv32
+  %0 = add nsw i64 %indvars.iv32, -1
+  %arrayidx16 = getelementptr inbounds [128 x float], ptr %arrayidx, i64 0, i64 %0
+  br label %for.body4
+
+for.cond.cleanup3:
+  %indvars.iv.next33 = add nuw nsw i64 %indvars.iv32, 1
+  %exitcond36 = icmp ne i64 %indvars.iv.next33, 128
+  br i1 %exitcond36, label %for.cond1.preheader, label %for.cond.cleanup
+
+for.body4:
+  %indvars.iv = phi i64 [ 1, %for.cond1.preheader ], [ %indvars.iv.next, %for.body4 ]
+  %1 = load float, ptr %arrayidx5, align 4
+  %arrayidx9 = getelementptr inbounds nuw [128 x float], ptr %cc, i64 %indvars.iv, i64 %indvars.iv32
+  store float %1, ptr %arrayidx9, align 4
+  %arrayidx13 = getelementptr inbounds nuw [128 x float], ptr %bb, i64 %indvars.iv, i64 %indvars.iv32
+  %2 = load float, ptr %arrayidx13, align 4
+  %3 = load float, ptr %arrayidx16, align 4
+  %add = fadd float %2, %3
+  store float %add, ptr %arrayidx16, align 4
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %exitcond = icmp ne i64 %indvars.iv.next, 1024
+  br i1 %exitcond, label %for.body4, label %for.cond.cleanup3
+
+for.cond.cleanup:
+  ret void
+}
+
+;; The original code is as follows, with some simplifications from the one in
+;; the issue #116114. The interchange is illegal due to the dependency of `A`.
+;;
+;; void issue116114(int *A, int x, unsigned N, unsigned M) {
+;;   for (unsigned m = 0; m < M; ++m)
+;;     for (unsigned i = 0U; i < N - 1; ++i) {
+;;       A[i] = A[i + 1] + x;
+;;     }
+;; }
+
+; CHECK:      --- !Missed
+; CHECK-NEXT: Pass:            loop-interchange
+; CHECK-NEXT: Name:            Dependence
+; CHECK-NEXT: Function:        issue116114
+; CHECK-NEXT: Args:
+; CHECK-NEXT:  - String:       Cannot interchange loops due to dependences.
+define void @issue116114(ptr noundef captures(none) %A, i32 noundef %x, i32 noundef %N, i32 noundef %M) {
+entry:
+  %cmp18.not = icmp eq i32 %M, 0
+  br i1 %cmp18.not, label %for.cond.cleanup, label %for.cond1.preheader.lr.ph
+
+for.cond1.preheader.lr.ph:
+  %sub = add i32 %N, -1
+  %cmp216.not = icmp eq i32 %sub, 0
+  br i1 %cmp216.not, label %for.cond1.preheader.preheader, label %for.cond1.preheader.us.preheader
+
+for.cond1.preheader.us.preheader:
+  br label %for.cond1.preheader.us
+
+for.cond1.preheader.preheader:
+  br label %for.cond.cleanup.loopexit
+
+for.cond1.preheader.us:
+  %m.019.us = phi i32 [ %inc9.us, %for.cond1.for.cond.cleanup3_crit_edge.us ], [ 0, %for.cond1.preheader.us.preheader ]
+  %wide.trip.count = zext i32 %sub to i64
+  br label %for.body4.us
+
+for.body4.us:
+  %indvars.iv = phi i64 [ 0, %for.cond1.preheader.us ], [ %indvars.iv.next, %for.body4.us ]
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %arrayidx.us = getelementptr inbounds nuw i32, ptr %A, i64 %indvars.iv.next
+  %0 = load i32, ptr %arrayidx.us, align 4
+  %add5.us = add nsw i32 %0, %x
+  %arrayidx7.us = getelementptr inbounds nuw i32, ptr %A, i64 %indvars.iv
+  store i32 %add5.us, ptr %arrayidx7.us, align 4
+  %exitcond = icmp ne i64 %indvars.iv.next, %wide.trip.count
+  br i1 %exitcond, label %for.body4.us, label %for.cond1.for.cond.cleanup3_crit_edge.us
+
+for.cond1.for.cond.cleanup3_crit_edge.us:
+  %inc9.us = add nuw i32 %m.019.us, 1
+  %exitcond22 = icmp ne i32 %inc9.us, %M
+  br i1 %exitcond22, label %for.cond1.preheader.us, label %for.cond.cleanup.loopexit20
+
+for.cond.cleanup.loopexit:
+  br label %for.cond.cleanup
+
+for.cond.cleanup.loopexit20:
+  br label %for.cond.cleanup
+
+for.cond.cleanup:
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopInterchange/loop-interchange-optimization-remarks.ll b/llvm/test/Transforms/LoopInterchange/loop-interchange-optimization-remarks.ll
index 3c7828a49477c..73a566a310157 100644
--- a/llvm/test/Transforms/LoopInterchange/loop-interchange-optimization-remarks.ll
+++ b/llvm/test/Transforms/LoopInterchange/loop-interchange-optimization-remarks.ll
@@ -58,6 +58,14 @@ for.end19:
   ret void
 }
 
+; CHECK: --- !Analysis
+; CHECK-NEXT: Pass:            loop-interchange
+; CHECK-NEXT: Name:            Dependence
+; CHECK-NEXT: Function:        test01
+; CHECK-NEXT: Args:
+; CHECK-NEXT:   - String:          Computed dependence info, invoking the transform.
+; CHECK-NEXT: ...
+
 ; CHECK: --- !Missed
 ; CHECK-NEXT: Pass:            loop-interchange
 ; CHECK-NEXT: Name:            Dependence
@@ -66,6 +74,14 @@ for.end19:
 ; CHECK-NEXT:   - String:          Cannot interchange loops due to dependences.
 ; CHECK-NEXT: ...
 
+; DELIN: --- !Analysis
+; DELIN-NEXT: Pass:            loop-interchange
+; DELIN-NEXT: Name:            Dependence
+; DELIN-NEXT: Function:        test01
+; DELIN-NEXT: Args:
+; DELIN-NEXT:   - String:          Computed dependence info, invoking the transform.
+; DELIN-NEXT: ...
+
 ; DELIN: --- !Missed
 ; DELIN-NEXT: Pass:            loop-interchange
 ; DELIN-NEXT: Name:            InterchangeNotProfitable
@@ -118,6 +134,14 @@ define void @test02(i32 %k, i32 %N) {
    ret void
 }
 
+; CHECK: --- !Analysis
+; CHECK-NEXT: Pass:            loop-interchange
+; CHECK-NEXT: Name:            Dependence
+; CHECK-NEXT: Function:        test02
+; CHECK-NEXT: Args:
+; CHECK-NEXT:   - String:          Computed dependence info, invoking the transform.
+; CHECK-NEXT: ...
+
 ; CHECK: --- !Missed
 ; CHECK-NEXT: Pass:            loop-interchange
 ; CHECK-NEXT: Name:            Dependence
@@ -126,6 +150,14 @@ define void @test02(i32 %k, i32 %N) {
 ; CHECK-NEXT:   - String:          Cannot interchange loops due to dependences.
 ; CHECK-NEXT: ...
 
+; DELIN: --- !Analysis
+; DELIN-NEXT: Pass:            loop-interchange
+; DELIN-NEXT: Name:            Dependence
+; DELIN-NEXT: Function:        test02
+; DELIN-NEXT: Args:
+; DELIN-NEXT:   - String:          Computed dependence info, invoking the transform.
+; DELIN-NEXT: ...
+
 ; DELIN: --- !Passed
 ; DELIN-NEXT: Pass:            loop-interchange
 ; DELIN-NEXT: Name:            Interchanged
@@ -174,6 +206,14 @@ for.body4:                                        ; preds = %for.body4, %for.con
   br i1 %exitcond, label %for.body4, label %for.cond.loopexit
 }
 
+; CHECK: --- !Analysis
+; CHECK-NEXT: Pass:            loop-interchange
+; CHECK-NEXT: Name:            Dependence
+; CHECK-NEXT: Function:        test03
+; CHECK-NEXT: Args:
+; CHECK-NEXT:   - String:          Computed dependence info, invoking the transform.
+; CHECK-NEXT: ...
+
 ; CHECK: --- !Passed
 ; CHECK-NEXT: Pass:            loop-interchange
 ; CHECK-NEXT: Name:            Interchanged
@@ -182,6 +222,14 @@ for.body4:                                        ; preds = %for.body4, %for.con
 ; CHECK-NEXT:   - String:          Loop interchanged with enclosing loop.
 ; CHECK-NEXT: ...
 
+; DELIN: --- !Analysis
+; DELIN-NEXT: Pass:            loop-interchange
+; DELIN-NEXT: Name:            Dependence
+; DELIN-NEXT: Function:        test03
+; DELIN-NEXT: Args:
+; DELIN-NEXT:   - String:          Computed dependence info, invoking the transform.
+; DELIN-NEXT: ...
+
 ; DELIN: --- !Passed
 ; DELIN-NEXT: Pass:            loop-interchange
 ; DELIN-NEXT: Name:            Interchanged
diff --git a/llvm/test/Transforms/LoopInterchange/no-dependence-info.ll b/llvm/test/Transforms/LoopInterchange/no-dependence-info.ll
new file mode 100644
index 0000000000000..d37fb46fc5d68
--- /dev/null
+++ b/llvm/test/Transforms/LoopInterchange/no-dependence-info.ll
@@ -0,0 +1,52 @@
+; RUN: opt %s -passes='loop-interchange' -pass-remarks=loop-interchange -disable-output 2>&1 | FileCheck --allow-empty %s
+
+target triple = "aarch64-unknown-linux-gnu"
+
+; CHECK-NOT: Computed dependence info, invoking the transform.
+
+; For the below test, backedge count cannot be computed. 
+; Computing backedge count requires only SCEV and should
+; not require dependence info. 
+;
+; void bar(int m, int n) {
+; for (unsigned int i = 0; i < m; ++i) {
+;    for (unsigned int j = 0; j < m; ++j) {
+;     // dummy code
+;    }
+;  }
+;}
+
+define void @bar(i32 %m, i32 %n)
+{
+entry:
+  br label %outer.header
+
+outer.header:
+ %m_temp1 = phi i32 [%m, %entry], [%m_temp, %outer.latch]
+ br label %inner.header
+
+
+inner.header:
+ %n_temp1 = phi i32 [%n, %outer.header], [%n_temp, %inner.latch]
+
+ br label %body
+
+body:
+ ; dummy code
+
+br label %inner.latch 
+
+inner.latch:
+%n_temp = add i32 %n_temp1, 1
+%cmp2 = icmp eq i32 %n_temp, 1 
+br i1 %cmp2, label %outer.latch, label %inner.header
+
+outer.latch:
+%m_temp = add i32 %n, 1
+%cmp3 = icmp eq i32 %m_temp, 1 
+br i1 %cmp3, label %exit, label %outer.header
+
+exit:
+ret void
+}
+
diff --git a/llvm/test/Transforms/LoopInterchange/outer-dependency-lte.ll b/llvm/test/Transforms/LoopInterchange/outer-dependency-lte.ll
new file mode 100644
index 0000000000000..c17d78f7cfce6
--- /dev/null
+++ b/llvm/test/Transforms/LoopInterchange/outer-dependency-lte.ll
@@ -0,0 +1,75 @@
+; RUN: opt < %s -passes=loop-interchange -pass-remarks-missed='loop-interchange' -pass-remarks-output=%t \
+; RUN:     -verify-dom-info -verify-loop-info -verify-loop-lcssa
+; RUN: FileCheck --input-file=%t %s
+
+;; The original code:
+;;
+;; #define N 4
+;; int a[N*N][N*N][N*N];
+;; void f() {
+;;   for (int i = 0; i < N; i++)
+;;     for (int j = 1; j < 2*N; j++)
+;;       for (int k = 1; k < 2*N; k++)
+;;         a[2*i][k+1][j-1] -= a[i+N-1][k][j];
+;; }
+;;
+;; The entry of the direction vector for the outermost loop is `DVEntry::LE`.
+;; We need to treat this as `*`, not `<`. See issue #123920 for details.
+
+; CHECK: --- !Missed
+; CHECK-NEXT: Pass:            loop-interchange
+; CHECK-NEXT: Name:            Dependence
+; CHECK-NEXT: Function:        f
+; CHECK: --- !Missed
+; CHECK-NEXT: Pass:            loop-interchange
+; CHECK-NEXT: Name:            Dependence
+; CHECK-NEXT: Function:        f
+
+@a = dso_local global [16 x [16 x [16 x i32]]] zeroinitializer, align 4
+
+define dso_local void @f() {
+entry:
+  br label %for.cond1.preheader
+
+for.cond1.preheader:
+  %i.039 = phi i32 [ 0, %entry ], [ %inc26, %for.cond.cleanup3 ]
+  %sub = add nuw nsw i32 %i.039, 3
+  %idxprom = zext nneg i32 %sub to i64
+  %mul = shl nuw nsw i32 %i.039, 1
+  %idxprom13 = zext nneg i32 %mul to i64
+  br label %for.cond5.preheader
+
+for.cond.cleanup:
+  ret void
+
+for.cond5.preheader:
+  %j.038 = phi i32 [ 1, %for.cond1.preheader ], [ %inc23, %for.cond.cleanup7 ]
+  %idxprom11 = zext nneg i32 %j.038 to i64
+  %sub18 = add nsw i32 %j.038, -1
+  %idxprom19 = sext i32 %sub18 to i64
+  br label %for.body8
+
+for.cond.cleanup3:
+  %inc26 = add nuw nsw i32 %i.039, 1
+  %cmp = icmp samesign ult i32 %i.039, 3
+  br i1 %cmp, label %for.cond1.preheader, label %for.cond.cleanup
+
+for.cond.cleanup7:
+  %inc23 = add nuw nsw i32 %j.038, 1
+  %cmp2 = icmp samesign ult i32 %j.038, 7
+  br i1 %cmp2, label %for.cond5.preheader, label %for.cond.cleanup3
+
+for.body8:
+  %k.037 = phi i32 [ 1, %for.cond5.preheader ], [ %add15, %for.body8 ]
+  %idxprom9 = zext nneg i32 %k.037 to i64
+  %arrayidx12 = getelementptr inbounds nuw [16 x [16 x [16 x i32]]], ptr @a, i64 0, i64 %idxprom, i64 %idxprom9, i64 %idxprom11
+  %0 = load i32, ptr %arrayidx12, align 4
+  %add15 = add nuw nsw i32 %k.037, 1
+  %idxprom16 = zext nneg i32 %add15 to i64
+  %arrayidx20 = getelementptr inbounds [16 x [16 x [16 x i32]]], ptr @a, i64 0, i64 %idxprom13, i64 %idxprom16, i64 %idxprom19
+  %1 = load i32, ptr %arrayidx20, align 4
+  %sub21 = sub nsw i32 %1, %0
+  store i32 %sub21, ptr %arrayidx20, align 4
+  %cmp6 = icmp samesign ult i32 %k.037, 7
+  br i1 %cmp6, label %for.body8, label %for.cond.cleanup7
+}
diff --git a/llvm/test/Transforms/LoopInterchange/pr43326-ideal-access-pattern.ll b/llvm/test/Transforms/LoopInterchange/pr43326-ideal-access-pattern.ll
index def68ca3cd07e..520e1ee3506da 100644
--- a/llvm/test/Transforms/LoopInterchange/pr43326-ideal-access-pattern.ll
+++ b/llvm/test/Transforms/LoopInterchange/pr43326-ideal-access-pattern.ll
@@ -14,6 +14,14 @@
 ;   }
 ; }
 
+; REMARKS: --- !Analysis
+; REMARKS-NEXT: Pass:            loop-interchange
+; REMARKS-NEXT: Name:            Dependence
+; REMARKS-NEXT: Function:        pr43326-triply-nested
+; REMARKS-NEXT: Args:
+; REMARKS-NEXT:   - String:          Computed dependence info, invoking the transform.
+; REMARKS-NEXT: ...
+
 ; REMARKS: --- !Passed
 ; REMARKS-NEXT: Pass:            loop-interchange
 ; REMARKS-NEXT: Name:            Interchanged
diff --git a/llvm/test/Transforms/LoopInterchange/pr43326.ll b/llvm/test/Transforms/LoopInterchange/pr43326.ll
index 4dbb06780a898..c25c4fadd3042 100644
--- a/llvm/test/Transforms/LoopInterchange/pr43326.ll
+++ b/llvm/test/Transforms/LoopInterchange/pr43326.ll
@@ -8,6 +8,14 @@
 @d = global i32 0
 @e = global [1 x [1 x i32]] zeroinitializer
 
+; REMARKS: --- !Analysis
+; REMARKS-NEXT: Pass:            loop-interchange
+; REMARKS-NEXT: Name:            Dependence
+; REMARKS-NEXT: Function:        pr43326
+; REMARKS-NEXT: Args:
+; REMARKS-NEXT:   - String:          Computed dependence info, invoking the transform.
+; REMARKS-NEXT: ...
+
 ; REMARKS: --- !Passed
 ; REMARKS-NEXT: Pass:            loop-interchange
 ; REMARKS-NEXT: Name:            Interchanged
diff --git a/llvm/test/Transforms/LoopInterchange/pr48212.ll b/llvm/test/Transforms/LoopInterchange/pr48212.ll
index b580794ab7d36..936c53e217540 100644
--- a/llvm/test/Transforms/LoopInterchange/pr48212.ll
+++ b/llvm/test/Transforms/LoopInterchange/pr48212.ll
@@ -2,6 +2,14 @@
 ; RUN:     -verify-dom-info -verify-loop-info -verify-loop-lcssa 2>&1
 ; RUN: FileCheck --input-file=%t --check-prefix=REMARKS %s
 
+; REMARKS: --- !Analysis
+; REMARKS-NEXT: Pass:            loop-interchange
+; REMARKS-NEXT: Name:            Dependence
+; REMARKS-NEXT: Function:        pr48212
+; REMARKS-NEXT: Args:
+; REMARKS-NEXT:   - String:          Computed dependence info, invoking the transform.
+; REMARKS-NEXT: ...
+
 ; REMARKS: --- !Passed
 ; REMARKS-NEXT: Pass:            loop-interchange
 ; REMARKS-NEXT: Name:            Interchanged
diff --git a/llvm/test/Transforms/LoopInterchange/reductions-across-inner-and-outer-loop.ll b/llvm/test/Transforms/LoopInterchange/reductions-across-inner-and-outer-loop.ll
index eea0c2635d595..27d99e05e84ee 100644
--- a/llvm/test/Transforms/LoopInterchange/reductions-across-inner-and-outer-loop.ll
+++ b/llvm/test/Transforms/LoopInterchange/reductions-across-inner-and-outer-loop.ll
@@ -5,6 +5,14 @@
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 
+; REMARKS: --- !Analysis
+; REMARKS-NEXT: Pass:            loop-interchange
+; REMARKS-NEXT: Name:            Dependence
+; REMARKS-NEXT: Function:        test1
+; REMARKS-NEXT: Args:
+; REMARKS-NEXT:   - String:          Computed dependence info, invoking the transform.
+; REMARKS-NEXT: ...
+
 ; REMARKS: --- !Passed
 ; REMARKS-NEXT: Pass:            loop-interchange
 ; REMARKS-NEXT: Name:            Interchanged
@@ -77,6 +85,14 @@ for1.loopexit:                                 ; preds = %for1.inc
 
 ; In this test case, the inner reduction PHI %inner does not involve the outer
 ; reduction PHI %sum.outer, do not interchange.
+; REMARKS: --- !Analysis
+; REMARKS-NEXT: Pass:            loop-interchange
+; REMARKS-NEXT: Name:            Dependence
+; REMARKS-NEXT: Function:        test2
+; REMARKS-NEXT: Args:
+; REMARKS-NEXT:   - String:          Computed dependence info, invoking the transform.
+; REMARKS-NEXT: ...
+
 ; REMARKS: --- !Missed
 ; REMARKS-NEXT: Pass:            loop-interchange
 ; REMARKS-NEXT: Name:            UnsupportedPHIOuter
@@ -114,6 +130,14 @@ for1.loopexit:                                 ; preds = %for1.inc
 
 ; Check that we do not interchange if there is an additional instruction
 ; between the outer and inner reduction PHIs.
+; REMARKS: --- !Analysis
+; REMARKS-NEXT: Pass:            loop-interchange
+; REMARKS-NEXT: Name:            Dependence
+; REMARKS-NEXT: Function:        test3
+; REMARKS-NEXT: Args:
+; REMARKS-NEXT:   - String:          Computed dependence info, invoking the transform.
+; REMARKS-NEXT: ...
+
 ; REMARKS: --- !Missed
 ; REMARKS-NEXT: Pass:            loop-interchange
 ; REMARKS-NEXT: Name:            UnsupportedPHIOuter
@@ -151,6 +175,14 @@ for1.loopexit:                                 ; preds = %for1.inc
 }
 
 ; Check that we do not interchange if reduction is stored in an invariant address inside inner loop
+; REMARKS: --- !Analysis
+; REMARKS-NEXT: Pass:            loop-interchange
+; REMARKS-NEXT: Name:            Dependence
+; REMARKS-NEXT: Function:        test4
+; REMARKS-NEXT: Args:
+; REMARKS-NEXT:   - String:          Computed dependence info, invoking the transform.
+; REMARKS-NEXT: ...
+
 ; REMARKS: --- !Missed
 ; REMARKS-NEXT: Pass:            loop-interchange
 ; REMARKS-NEXT: Name:            Dependence
@@ -190,6 +222,14 @@ for1.loopexit:                                 ; preds = %for1.inc
 
 ; Check that we do not interchange or crash if the PHI in the outer loop gets a
 ; constant from the inner loop.
+; REMARKS: --- !Analysis
+; REMARKS-NEXT: Pass:            loop-interchange
+; REMARKS-NEXT: Name:            Dependence
+; REMARKS-NEXT: Function:        test_constant_inner_loop_res
+; REMARKS-NEXT: Args:
+; REMARKS-NEXT:   - String:          Computed dependence info, invoking the transform.
+; REMARKS-NEXT: ...
+
 ; REMARKS: --- !Missed
 ; REMARKS-NEXT: Pass:            loop-interchange
 ; REMARKS-NEXT: Name:            UnsupportedPHIOuter
@@ -229,6 +269,14 @@ for1.loopexit:                                 ; preds = %for1.inc
 
 ; Floating point reductions are interchanged if all the fp instructions
 ; involved allow reassociation.
+; REMARKS: --- !Analysis
+; REMARKS-NEXT: Pass:            loop-interchange
+; REMARKS-NEXT: Name:            Dependence
+; REMARKS-NEXT: Function:        test5
+; REMARKS-NEXT: Args:
+; REMARKS-NEXT:   - String:          Computed dependence info, invoking the transform.
+; REMARKS-NEXT: ...
+
 ; REMARKS: --- !Passed
 ; REMARKS-NEXT: Pass:            loop-interchange
 ; REMARKS-NEXT: Name:            Interchanged
@@ -269,6 +317,14 @@ for.exit:                                         ; preds = %outer.inc
 
 ; Floating point reductions are not interchanged if not all the fp instructions
 ; involved allow reassociation.
+; REMARKS: --- !Analysis
+; REMARKS-NEXT: Pass:            loop-interchange
+; REMARKS-NEXT: Name:            Dependence
+; REMARKS-NEXT: Function:        test6
+; REMARKS-NEXT: Args:
+; REMARKS-NEXT:   - String:          Computed dependence info, invoking the transform.
+; REMARKS-NEXT: ...
+
 ; REMARKS: --- !Missed
 ; REMARKS-NEXT: Pass:            loop-interchange
 ; REMARKS-NEXT: Name:            UnsupportedPHIOuter
diff --git a/llvm/test/Transforms/LoopPredication/scale.ll b/llvm/test/Transforms/LoopPredication/scale.ll
index 29e48cf579598..edf67a28b4c36 100644
--- a/llvm/test/Transforms/LoopPredication/scale.ll
+++ b/llvm/test/Transforms/LoopPredication/scale.ll
@@ -7,7 +7,7 @@
 ; Predicate is profitable when the scale factor is 2 and not profitable if it's less than 2.
 define i64 @predicate_eq_ones(ptr nocapture readonly %arg, i32 %length, ptr nocapture readonly %arg2, ptr nocapture readonly %n_addr, i64 %i) !prof !21 {
 ; CHECK-PROF-LABEL: define i64 @predicate_eq_ones(
-; CHECK-PROF-SAME: ptr nocapture readonly [[ARG:%.*]], i32 [[LENGTH:%.*]], ptr nocapture readonly [[ARG2:%.*]], ptr nocapture readonly [[N_ADDR:%.*]], i64 [[I:%.*]]) !prof [[PROF0:![0-9]+]] {
+; CHECK-PROF-SAME: ptr readonly captures(none) [[ARG:%.*]], i32 [[LENGTH:%.*]], ptr readonly captures(none) [[ARG2:%.*]], ptr readonly captures(none) [[N_ADDR:%.*]], i64 [[I:%.*]]) !prof [[PROF0:![0-9]+]] {
 ; CHECK-PROF-NEXT:  entry:
 ; CHECK-PROF-NEXT:    [[LENGTH_EXT:%.*]] = zext i32 [[LENGTH]] to i64
 ; CHECK-PROF-NEXT:    [[N_PRE:%.*]] = load i64, ptr [[N_ADDR]], align 4
@@ -36,7 +36,7 @@ define i64 @predicate_eq_ones(ptr nocapture readonly %arg, i32 %length, ptr noca
 ; CHECK-PROF-NEXT:    ret i64 [[RESULT_LE]]
 ;
 ; CHECK-NOTPROF-LABEL: define i64 @predicate_eq_ones(
-; CHECK-NOTPROF-SAME: ptr nocapture readonly [[ARG:%.*]], i32 [[LENGTH:%.*]], ptr nocapture readonly [[ARG2:%.*]], ptr nocapture readonly [[N_ADDR:%.*]], i64 [[I:%.*]]) !prof [[PROF0:![0-9]+]] {
+; CHECK-NOTPROF-SAME: ptr readonly captures(none) [[ARG:%.*]], i32 [[LENGTH:%.*]], ptr readonly captures(none) [[ARG2:%.*]], ptr readonly captures(none) [[N_ADDR:%.*]], i64 [[I:%.*]]) !prof [[PROF0:![0-9]+]] {
 ; CHECK-NOTPROF-NEXT:  entry:
 ; CHECK-NOTPROF-NEXT:    [[LENGTH_EXT:%.*]] = zext i32 [[LENGTH]] to i64
 ; CHECK-NOTPROF-NEXT:    [[N_PRE:%.*]] = load i64, ptr [[N_ADDR]], align 4
@@ -90,7 +90,7 @@ exit:                                             ; preds = %Header
 ; Same as the previous one, but with zero weights (should be treated as if no profile - equal probability)
 define i64 @predicate_eq_zeroes(ptr nocapture readonly %arg, i32 %length, ptr nocapture readonly %arg2, ptr nocapture readonly %n_addr, i64 %i) !prof !21 {
 ; CHECK-PROF-LABEL: define i64 @predicate_eq_zeroes(
-; CHECK-PROF-SAME: ptr nocapture readonly [[ARG:%.*]], i32 [[LENGTH:%.*]], ptr nocapture readonly [[ARG2:%.*]], ptr nocapture readonly [[N_ADDR:%.*]], i64 [[I:%.*]]) !prof [[PROF0]] {
+; CHECK-PROF-SAME: ptr readonly captures(none) [[ARG:%.*]], i32 [[LENGTH:%.*]], ptr readonly captures(none) [[ARG2:%.*]], ptr readonly captures(none) [[N_ADDR:%.*]], i64 [[I:%.*]]) !prof [[PROF0]] {
 ; CHECK-PROF-NEXT:  entry:
 ; CHECK-PROF-NEXT:    [[LENGTH_EXT:%.*]] = zext i32 [[LENGTH]] to i64
 ; CHECK-PROF-NEXT:    [[N_PRE:%.*]] = load i64, ptr [[N_ADDR]], align 4
@@ -119,7 +119,7 @@ define i64 @predicate_eq_zeroes(ptr nocapture readonly %arg, i32 %length, ptr no
 ; CHECK-PROF-NEXT:    ret i64 [[RESULT_LE]]
 ;
 ; CHECK-NOTPROF-LABEL: define i64 @predicate_eq_zeroes(
-; CHECK-NOTPROF-SAME: ptr nocapture readonly [[ARG:%.*]], i32 [[LENGTH:%.*]], ptr nocapture readonly [[ARG2:%.*]], ptr nocapture readonly [[N_ADDR:%.*]], i64 [[I:%.*]]) !prof [[PROF0]] {
+; CHECK-NOTPROF-SAME: ptr readonly captures(none) [[ARG:%.*]], i32 [[LENGTH:%.*]], ptr readonly captures(none) [[ARG2:%.*]], ptr readonly captures(none) [[N_ADDR:%.*]], i64 [[I:%.*]]) !prof [[PROF0]] {
 ; CHECK-NOTPROF-NEXT:  entry:
 ; CHECK-NOTPROF-NEXT:    [[LENGTH_EXT:%.*]] = zext i32 [[LENGTH]] to i64
 ; CHECK-NOTPROF-NEXT:    [[N_PRE:%.*]] = load i64, ptr [[N_ADDR]], align 4
@@ -173,7 +173,7 @@ exit:                                             ; preds = %Header
 ; No profile on br in Header
 define i64 @predicate_eq_none(ptr nocapture readonly %arg, i32 %length, ptr nocapture readonly %arg2, ptr nocapture readonly %n_addr, i64 %i) !prof !21 {
 ; CHECK-PROF-LABEL: define i64 @predicate_eq_none(
-; CHECK-PROF-SAME: ptr nocapture readonly [[ARG:%.*]], i32 [[LENGTH:%.*]], ptr nocapture readonly [[ARG2:%.*]], ptr nocapture readonly [[N_ADDR:%.*]], i64 [[I:%.*]]) !prof [[PROF0]] {
+; CHECK-PROF-SAME: ptr readonly captures(none) [[ARG:%.*]], i32 [[LENGTH:%.*]], ptr readonly captures(none) [[ARG2:%.*]], ptr readonly captures(none) [[N_ADDR:%.*]], i64 [[I:%.*]]) !prof [[PROF0]] {
 ; CHECK-PROF-NEXT:  entry:
 ; CHECK-PROF-NEXT:    [[LENGTH_EXT:%.*]] = zext i32 [[LENGTH]] to i64
 ; CHECK-PROF-NEXT:    [[N_PRE:%.*]] = load i64, ptr [[N_ADDR]], align 4
@@ -202,7 +202,7 @@ define i64 @predicate_eq_none(ptr nocapture readonly %arg, i32 %length, ptr noca
 ; CHECK-PROF-NEXT:    ret i64 [[RESULT_LE]]
 ;
 ; CHECK-NOTPROF-LABEL: define i64 @predicate_eq_none(
-; CHECK-NOTPROF-SAME: ptr nocapture readonly [[ARG:%.*]], i32 [[LENGTH:%.*]], ptr nocapture readonly [[ARG2:%.*]], ptr nocapture readonly [[N_ADDR:%.*]], i64 [[I:%.*]]) !prof [[PROF0]] {
+; CHECK-NOTPROF-SAME: ptr readonly captures(none) [[ARG:%.*]], i32 [[LENGTH:%.*]], ptr readonly captures(none) [[ARG2:%.*]], ptr readonly captures(none) [[N_ADDR:%.*]], i64 [[I:%.*]]) !prof [[PROF0]] {
 ; CHECK-NOTPROF-NEXT:  entry:
 ; CHECK-NOTPROF-NEXT:    [[LENGTH_EXT:%.*]] = zext i32 [[LENGTH]] to i64
 ; CHECK-NOTPROF-NEXT:    [[N_PRE:%.*]] = load i64, ptr [[N_ADDR]], align 4
diff --git a/llvm/test/Transforms/LoopStrengthReduce/2011-10-06-ReusePhi.ll b/llvm/test/Transforms/LoopStrengthReduce/2011-10-06-ReusePhi.ll
index f7b2ba30d15ca..6c5dcacbd471f 100644
--- a/llvm/test/Transforms/LoopStrengthReduce/2011-10-06-ReusePhi.ll
+++ b/llvm/test/Transforms/LoopStrengthReduce/2011-10-06-ReusePhi.ll
@@ -12,7 +12,7 @@ target datalayout = "n8:16:32:64"
 
 define float @test(ptr nocapture %A, ptr nocapture %B, i32 %N, i32 %IA, i32 %IB) nounwind uwtable readonly ssp {
 ; CHECK-LABEL: define float @test
-; CHECK-SAME: (ptr nocapture [[A:%.*]], ptr nocapture [[B:%.*]], i32 [[N:%.*]], i32 [[IA:%.*]], i32 [[IB:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-SAME: (ptr captures(none) [[A:%.*]], ptr captures(none) [[B:%.*]], i32 [[N:%.*]], i32 [[IA:%.*]], i32 [[IB:%.*]]) #[[ATTR0:[0-9]+]] {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[CMP1:%.*]] = icmp sgt i32 [[N]], 0
 ; CHECK-NEXT:    br i1 [[CMP1]], label [[WHILE_BODY_LR_PH:%.*]], label [[WHILE_END:%.*]]
diff --git a/llvm/test/Transforms/LoopStrengthReduce/ARM/illegal-addr-modes.ll b/llvm/test/Transforms/LoopStrengthReduce/ARM/illegal-addr-modes.ll
index 1709ec1086042..3844e00ae0a4f 100644
--- a/llvm/test/Transforms/LoopStrengthReduce/ARM/illegal-addr-modes.ll
+++ b/llvm/test/Transforms/LoopStrengthReduce/ARM/illegal-addr-modes.ll
@@ -19,7 +19,7 @@ target triple = "thumbv6m-arm-none-eabi"
 ; Test case 01: -1*reg is not free for the Thumb1 target.
 define ptr @negativeOneCase(ptr returned %a, ptr nocapture readonly %b, i32 %n) nounwind {
 ; CHECK-LABEL: define ptr @negativeOneCase(
-; CHECK-SAME: ptr returned [[A:%.*]], ptr nocapture readonly [[B:%.*]], i32 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-SAME: ptr returned [[A:%.*]], ptr readonly captures(none) [[B:%.*]], i32 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[ADD_PTR:%.*]] = getelementptr inbounds i8, ptr [[A]], i32 -1
 ; CHECK-NEXT:    br label [[WHILE_COND:%.*]]
diff --git a/llvm/test/Transforms/LoopStrengthReduce/ARM/ivchain-ARM.ll b/llvm/test/Transforms/LoopStrengthReduce/ARM/ivchain-ARM.ll
index 1614de8dbf558..eb076776ee743 100644
--- a/llvm/test/Transforms/LoopStrengthReduce/ARM/ivchain-ARM.ll
+++ b/llvm/test/Transforms/LoopStrengthReduce/ARM/ivchain-ARM.ll
@@ -23,9 +23,9 @@ define i32 @simple(ptr %a, ptr %b, i32 %x) nounwind {
 ; A9-NEXT:    add.w r4, lr, r2
 ; A9-NEXT:    ldr.w r6, [lr, r2]
 ; A9-NEXT:    add r0, r3
-; A9-NEXT:    adds r3, r4, r2
-; A9-NEXT:    add r0, r12
 ; A9-NEXT:    ldr r5, [r4, r2]
+; A9-NEXT:    add r0, r12
+; A9-NEXT:    adds r3, r4, r2
 ; A9-NEXT:    add r0, r6
 ; A9-NEXT:    add r3, r2
 ; A9-NEXT:    add r0, r5
diff --git a/llvm/test/Transforms/LoopStrengthReduce/addrec-gep-address-space.ll b/llvm/test/Transforms/LoopStrengthReduce/addrec-gep-address-space.ll
index 10ce8735ce9f6..c9637a32aa67f 100644
--- a/llvm/test/Transforms/LoopStrengthReduce/addrec-gep-address-space.ll
+++ b/llvm/test/Transforms/LoopStrengthReduce/addrec-gep-address-space.ll
@@ -11,8 +11,8 @@
 target datalayout = "e-p:64:64:64-p1:16:16:16-n16:32:64"
 
 define void @foo(i64 %n, i64 %m, i64 %o, i64 %q, ptr addrspace(1) nocapture %p) nounwind {
-; CHECK-LABEL: define void @foo
-; CHECK-SAME: (i64 [[N:%.*]], i64 [[M:%.*]], i64 [[O:%.*]], i64 [[Q:%.*]], ptr addrspace(1) nocapture [[P:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-LABEL: define void @foo(
+; CHECK-SAME: i64 [[N:%.*]], i64 [[M:%.*]], i64 [[O:%.*]], i64 [[Q:%.*]], ptr addrspace(1) captures(none) [[P:%.*]]) #[[ATTR0:[0-9]+]] {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[TMP:%.*]] = icmp sgt i64 [[N]], 0
 ; CHECK-NEXT:    br i1 [[TMP]], label [[BB_NPH3:%.*]], label [[RETURN:%.*]]
diff --git a/llvm/test/Transforms/LoopStrengthReduce/addrec-gep.ll b/llvm/test/Transforms/LoopStrengthReduce/addrec-gep.ll
index aba5ff20058ec..54c970c6ac13f 100644
--- a/llvm/test/Transforms/LoopStrengthReduce/addrec-gep.ll
+++ b/llvm/test/Transforms/LoopStrengthReduce/addrec-gep.ll
@@ -11,8 +11,8 @@
 target datalayout = "e-p:64:64:64-n32:64"
 
 define void @foo(i64 %n, i64 %m, i64 %o, i64 %q, ptr nocapture %p) nounwind {
-; CHECK-LABEL: define void @foo
-; CHECK-SAME: (i64 [[N:%.*]], i64 [[M:%.*]], i64 [[O:%.*]], i64 [[Q:%.*]], ptr nocapture [[P:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-LABEL: define void @foo(
+; CHECK-SAME: i64 [[N:%.*]], i64 [[M:%.*]], i64 [[O:%.*]], i64 [[Q:%.*]], ptr captures(none) [[P:%.*]]) #[[ATTR0:[0-9]+]] {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[TMP:%.*]] = icmp sgt i64 [[N]], 0
 ; CHECK-NEXT:    br i1 [[TMP]], label [[BB_NPH3:%.*]], label [[RETURN:%.*]]
diff --git a/llvm/test/Transforms/LoopStrengthReduce/shl.ll b/llvm/test/Transforms/LoopStrengthReduce/shl.ll
index 2a341e96d0cad..390dd8a0a89bf 100644
--- a/llvm/test/Transforms/LoopStrengthReduce/shl.ll
+++ b/llvm/test/Transforms/LoopStrengthReduce/shl.ll
@@ -6,7 +6,7 @@ target datalayout = "e-i64:64-v16:16-v32:32-n16:32:64"
 ; LoopStrengthReduce should reuse %mul as the stride.
 define void @_Z3fooPfll(ptr nocapture readonly %input, i64 %n, i64 %s) {
 ; CHECK-LABEL: define void @_Z3fooPfll
-; CHECK-SAME: (ptr nocapture readonly [[INPUT:%.*]], i64 [[N:%.*]], i64 [[S:%.*]]) {
+; CHECK-SAME: (ptr readonly captures(none) [[INPUT:%.*]], i64 [[N:%.*]], i64 [[S:%.*]]) {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[MUL:%.*]] = shl i64 [[S]], 2
 ; CHECK-NEXT:    tail call void @_Z3bazl(i64 [[MUL]])
diff --git a/llvm/test/Transforms/LoopUnroll/AArch64/apple-unrolling-multi-exit.ll b/llvm/test/Transforms/LoopUnroll/AArch64/apple-unrolling-multi-exit.ll
new file mode 100644
index 0000000000000..bfcd6f9e32a3b
--- /dev/null
+++ b/llvm/test/Transforms/LoopUnroll/AArch64/apple-unrolling-multi-exit.ll
@@ -0,0 +1,395 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -p loop-unroll -mcpu=apple-m1 -S %s | FileCheck --check-prefix=APPLE %s
+; RUN: opt -p loop-unroll -mcpu=apple-m2 -S %s | FileCheck --check-prefix=APPLE %s
+; RUN: opt -p loop-unroll -mcpu=apple-m3 -S %s | FileCheck --check-prefix=APPLE %s
+; RUN: opt -p loop-unroll -mcpu=apple-m4 -S %s | FileCheck --check-prefix=APPLE %s
+; RUN: opt -p loop-unroll -mcpu=cortex-a57 -S %s | FileCheck --check-prefix=OTHER %s
+
+target datalayout = "e-m:o-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-n32:64-S128-Fn32"
+target triple = "arm64-apple-macosx15.0.0"
+
+define i1 @multi_2_exit_find_i8_loop(ptr %vec, i8 %tgt) {
+; APPLE-LABEL: define i1 @multi_2_exit_find_i8_loop(
+; APPLE-SAME: ptr [[VEC:%.*]], i8 [[TGT:%.*]]) #[[ATTR0:[0-9]+]] {
+; APPLE-NEXT:  [[ENTRY:.*]]:
+; APPLE-NEXT:    [[START:%.*]] = load ptr, ptr [[VEC]], align 8
+; APPLE-NEXT:    [[GEP_END:%.*]] = getelementptr inbounds nuw i8, ptr [[VEC]], i64 1
+; APPLE-NEXT:    [[END:%.*]] = load ptr, ptr [[GEP_END]], align 8
+; APPLE-NEXT:    br label %[[LOOP_HEADER:.*]]
+; APPLE:       [[LOOP_HEADER]]:
+; APPLE-NEXT:    [[PTR_IV:%.*]] = phi ptr [ [[PTR_IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ], [ [[START]], %[[ENTRY]] ]
+; APPLE-NEXT:    [[L:%.*]] = load i8, ptr [[PTR_IV]], align 8
+; APPLE-NEXT:    [[C_1:%.*]] = icmp eq i8 [[L]], [[TGT]]
+; APPLE-NEXT:    br i1 [[C_1]], label %[[EXIT:.*]], label %[[LOOP_LATCH]]
+; APPLE:       [[LOOP_LATCH]]:
+; APPLE-NEXT:    [[PTR_IV_NEXT]] = getelementptr inbounds nuw i8, ptr [[PTR_IV]], i64 1
+; APPLE-NEXT:    [[C_2:%.*]] = icmp eq ptr [[PTR_IV_NEXT]], [[END]]
+; APPLE-NEXT:    br i1 [[C_2]], label %[[EXIT]], label %[[LOOP_HEADER]]
+; APPLE:       [[EXIT]]:
+; APPLE-NEXT:    [[RES:%.*]] = phi ptr [ [[PTR_IV]], %[[LOOP_HEADER]] ], [ [[END]], %[[LOOP_LATCH]] ]
+; APPLE-NEXT:    [[C_3:%.*]] = icmp eq ptr [[RES]], [[END]]
+; APPLE-NEXT:    ret i1 [[C_3]]
+;
+; OTHER-LABEL: define i1 @multi_2_exit_find_i8_loop(
+; OTHER-SAME: ptr [[VEC:%.*]], i8 [[TGT:%.*]]) #[[ATTR0:[0-9]+]] {
+; OTHER-NEXT:  [[ENTRY:.*]]:
+; OTHER-NEXT:    [[START:%.*]] = load ptr, ptr [[VEC]], align 8
+; OTHER-NEXT:    [[GEP_END:%.*]] = getelementptr inbounds nuw i8, ptr [[VEC]], i64 1
+; OTHER-NEXT:    [[END:%.*]] = load ptr, ptr [[GEP_END]], align 8
+; OTHER-NEXT:    br label %[[LOOP_HEADER:.*]]
+; OTHER:       [[LOOP_HEADER]]:
+; OTHER-NEXT:    [[PTR_IV:%.*]] = phi ptr [ [[PTR_IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ], [ [[START]], %[[ENTRY]] ]
+; OTHER-NEXT:    [[L:%.*]] = load i8, ptr [[PTR_IV]], align 8
+; OTHER-NEXT:    [[C_1:%.*]] = icmp eq i8 [[L]], [[TGT]]
+; OTHER-NEXT:    br i1 [[C_1]], label %[[EXIT:.*]], label %[[LOOP_LATCH]]
+; OTHER:       [[LOOP_LATCH]]:
+; OTHER-NEXT:    [[PTR_IV_NEXT]] = getelementptr inbounds nuw i8, ptr [[PTR_IV]], i64 1
+; OTHER-NEXT:    [[C_2:%.*]] = icmp eq ptr [[PTR_IV_NEXT]], [[END]]
+; OTHER-NEXT:    br i1 [[C_2]], label %[[EXIT]], label %[[LOOP_HEADER]]
+; OTHER:       [[EXIT]]:
+; OTHER-NEXT:    [[RES:%.*]] = phi ptr [ [[PTR_IV]], %[[LOOP_HEADER]] ], [ [[END]], %[[LOOP_LATCH]] ]
+; OTHER-NEXT:    [[C_3:%.*]] = icmp eq ptr [[RES]], [[END]]
+; OTHER-NEXT:    ret i1 [[C_3]]
+;
+entry:
+  %start = load ptr, ptr %vec, align 8
+  %gep.end = getelementptr inbounds nuw i8, ptr %vec, i64 1
+  %end = load ptr, ptr %gep.end, align 8
+  br label %loop.header
+
+loop.header:
+  %ptr.iv = phi ptr [ %ptr.iv.next, %loop.latch ], [ %start, %entry ]
+  %l = load i8, ptr %ptr.iv, align 8
+  %c.1 = icmp eq i8 %l, %tgt
+  br i1 %c.1, label %exit, label %loop.latch
+
+loop.latch:
+  %ptr.iv.next = getelementptr inbounds nuw i8, ptr %ptr.iv, i64 1
+  %c.2 = icmp eq ptr %ptr.iv.next, %end
+  br i1 %c.2, label %exit, label %loop.header
+
+exit:
+  %res = phi ptr [ %ptr.iv, %loop.header ], [ %end, %loop.latch ]
+  %c.3 = icmp eq ptr %res, %end
+  ret i1 %c.3
+}
+
+
+define i1 @multi_2_exit_find_ptr_loop(ptr %vec, ptr %tgt) {
+; APPLE-LABEL: define i1 @multi_2_exit_find_ptr_loop(
+; APPLE-SAME: ptr [[VEC:%.*]], ptr [[TGT:%.*]]) #[[ATTR0]] {
+; APPLE-NEXT:  [[ENTRY:.*]]:
+; APPLE-NEXT:    [[START:%.*]] = load ptr, ptr [[VEC]], align 8
+; APPLE-NEXT:    call void @llvm.assume(i1 true) [ "align"(ptr [[START]], i64 8) ]
+; APPLE-NEXT:    [[GEP_END:%.*]] = getelementptr inbounds nuw i8, ptr [[VEC]], i64 8
+; APPLE-NEXT:    [[END:%.*]] = load ptr, ptr [[GEP_END]], align 8
+; APPLE-NEXT:    call void @llvm.assume(i1 true) [ "align"(ptr [[END]], i64 8) ]
+; APPLE-NEXT:    br label %[[LOOP_HEADER:.*]]
+; APPLE:       [[LOOP_HEADER]]:
+; APPLE-NEXT:    [[PTR_IV:%.*]] = phi ptr [ [[PTR_IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ], [ [[START]], %[[ENTRY]] ]
+; APPLE-NEXT:    [[L:%.*]] = load ptr, ptr [[PTR_IV]], align 8
+; APPLE-NEXT:    [[C_1:%.*]] = icmp eq ptr [[L]], [[TGT]]
+; APPLE-NEXT:    br i1 [[C_1]], label %[[EXIT:.*]], label %[[LOOP_LATCH]]
+; APPLE:       [[LOOP_LATCH]]:
+; APPLE-NEXT:    [[PTR_IV_NEXT]] = getelementptr inbounds nuw i8, ptr [[PTR_IV]], i64 8
+; APPLE-NEXT:    [[C_2:%.*]] = icmp eq ptr [[PTR_IV_NEXT]], [[END]]
+; APPLE-NEXT:    br i1 [[C_2]], label %[[EXIT]], label %[[LOOP_HEADER]]
+; APPLE:       [[EXIT]]:
+; APPLE-NEXT:    [[RES:%.*]] = phi ptr [ [[PTR_IV]], %[[LOOP_HEADER]] ], [ [[END]], %[[LOOP_LATCH]] ]
+; APPLE-NEXT:    call void @llvm.assume(i1 true) [ "align"(ptr [[END]], i64 8) ]
+; APPLE-NEXT:    [[C_3:%.*]] = icmp eq ptr [[RES]], [[END]]
+; APPLE-NEXT:    ret i1 [[C_3]]
+;
+; OTHER-LABEL: define i1 @multi_2_exit_find_ptr_loop(
+; OTHER-SAME: ptr [[VEC:%.*]], ptr [[TGT:%.*]]) #[[ATTR0]] {
+; OTHER-NEXT:  [[ENTRY:.*]]:
+; OTHER-NEXT:    [[START:%.*]] = load ptr, ptr [[VEC]], align 8
+; OTHER-NEXT:    call void @llvm.assume(i1 true) [ "align"(ptr [[START]], i64 8) ]
+; OTHER-NEXT:    [[GEP_END:%.*]] = getelementptr inbounds nuw i8, ptr [[VEC]], i64 8
+; OTHER-NEXT:    [[END:%.*]] = load ptr, ptr [[GEP_END]], align 8
+; OTHER-NEXT:    call void @llvm.assume(i1 true) [ "align"(ptr [[END]], i64 8) ]
+; OTHER-NEXT:    br label %[[LOOP_HEADER:.*]]
+; OTHER:       [[LOOP_HEADER]]:
+; OTHER-NEXT:    [[PTR_IV:%.*]] = phi ptr [ [[PTR_IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ], [ [[START]], %[[ENTRY]] ]
+; OTHER-NEXT:    [[L:%.*]] = load ptr, ptr [[PTR_IV]], align 8
+; OTHER-NEXT:    [[C_1:%.*]] = icmp eq ptr [[L]], [[TGT]]
+; OTHER-NEXT:    br i1 [[C_1]], label %[[EXIT:.*]], label %[[LOOP_LATCH]]
+; OTHER:       [[LOOP_LATCH]]:
+; OTHER-NEXT:    [[PTR_IV_NEXT]] = getelementptr inbounds nuw i8, ptr [[PTR_IV]], i64 8
+; OTHER-NEXT:    [[C_2:%.*]] = icmp eq ptr [[PTR_IV_NEXT]], [[END]]
+; OTHER-NEXT:    br i1 [[C_2]], label %[[EXIT]], label %[[LOOP_HEADER]]
+; OTHER:       [[EXIT]]:
+; OTHER-NEXT:    [[RES:%.*]] = phi ptr [ [[PTR_IV]], %[[LOOP_HEADER]] ], [ [[END]], %[[LOOP_LATCH]] ]
+; OTHER-NEXT:    call void @llvm.assume(i1 true) [ "align"(ptr [[END]], i64 8) ]
+; OTHER-NEXT:    [[C_3:%.*]] = icmp eq ptr [[RES]], [[END]]
+; OTHER-NEXT:    ret i1 [[C_3]]
+;
+entry:
+  %start = load ptr, ptr %vec, align 8
+  call void @llvm.assume(i1 true) [ "align"(ptr %start, i64 8) ]
+  %gep.end = getelementptr inbounds nuw i8, ptr %vec, i64 8
+  %end = load ptr, ptr %gep.end, align 8
+  call void @llvm.assume(i1 true) [ "align"(ptr %end, i64 8) ]
+  br label %loop.header
+
+loop.header:
+  %ptr.iv = phi ptr [ %ptr.iv.next, %loop.latch ], [ %start, %entry ]
+  %l = load ptr, ptr %ptr.iv, align 8
+  %c.1 = icmp eq ptr %l, %tgt
+  br i1 %c.1, label %exit, label %loop.latch
+
+loop.latch:
+  %ptr.iv.next = getelementptr inbounds nuw i8, ptr %ptr.iv, i64 8
+  %c.2 = icmp eq ptr %ptr.iv.next, %end
+  br i1 %c.2, label %exit, label %loop.header
+
+exit:
+  %res = phi ptr [ %ptr.iv, %loop.header ], [ %end, %loop.latch ]
+  call void @llvm.assume(i1 true) [ "align"(ptr %end, i64 8) ]
+  %c.3 = icmp eq ptr %res, %end
+  ret i1 %c.3
+}
+
+define i1 @multi_2_exit_find_i8_loop_too_large(ptr %vec, i8 %tgt) {
+; APPLE-LABEL: define i1 @multi_2_exit_find_i8_loop_too_large(
+; APPLE-SAME: ptr [[VEC:%.*]], i8 [[TGT:%.*]]) #[[ATTR0]] {
+; APPLE-NEXT:  [[ENTRY:.*]]:
+; APPLE-NEXT:    [[START:%.*]] = load ptr, ptr [[VEC]], align 8
+; APPLE-NEXT:    [[GEP_END:%.*]] = getelementptr inbounds nuw i8, ptr [[VEC]], i64 1
+; APPLE-NEXT:    [[END:%.*]] = load ptr, ptr [[GEP_END]], align 8
+; APPLE-NEXT:    br label %[[LOOP_HEADER:.*]]
+; APPLE:       [[LOOP_HEADER]]:
+; APPLE-NEXT:    [[PTR_IV:%.*]] = phi ptr [ [[PTR_IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ], [ [[START]], %[[ENTRY]] ]
+; APPLE-NEXT:    [[L:%.*]] = load i8, ptr [[PTR_IV]], align 8
+; APPLE-NEXT:    [[UDIV:%.*]] = udiv i8 [[L]], [[TGT]]
+; APPLE-NEXT:    [[UDIV_2:%.*]] = udiv i8 [[UDIV]], 10
+; APPLE-NEXT:    [[C_1:%.*]] = icmp eq i8 [[UDIV_2]], 2
+; APPLE-NEXT:    br i1 [[C_1]], label %[[EXIT:.*]], label %[[LOOP_LATCH]]
+; APPLE:       [[LOOP_LATCH]]:
+; APPLE-NEXT:    [[PTR_IV_NEXT]] = getelementptr inbounds nuw i8, ptr [[PTR_IV]], i64 1
+; APPLE-NEXT:    [[C_2:%.*]] = icmp eq ptr [[PTR_IV_NEXT]], [[END]]
+; APPLE-NEXT:    br i1 [[C_2]], label %[[EXIT]], label %[[LOOP_HEADER]]
+; APPLE:       [[EXIT]]:
+; APPLE-NEXT:    [[RES:%.*]] = phi ptr [ [[PTR_IV]], %[[LOOP_HEADER]] ], [ [[END]], %[[LOOP_LATCH]] ]
+; APPLE-NEXT:    [[C_3:%.*]] = icmp eq ptr [[RES]], [[END]]
+; APPLE-NEXT:    ret i1 [[C_3]]
+;
+; OTHER-LABEL: define i1 @multi_2_exit_find_i8_loop_too_large(
+; OTHER-SAME: ptr [[VEC:%.*]], i8 [[TGT:%.*]]) #[[ATTR0]] {
+; OTHER-NEXT:  [[ENTRY:.*]]:
+; OTHER-NEXT:    [[START:%.*]] = load ptr, ptr [[VEC]], align 8
+; OTHER-NEXT:    [[GEP_END:%.*]] = getelementptr inbounds nuw i8, ptr [[VEC]], i64 1
+; OTHER-NEXT:    [[END:%.*]] = load ptr, ptr [[GEP_END]], align 8
+; OTHER-NEXT:    br label %[[LOOP_HEADER:.*]]
+; OTHER:       [[LOOP_HEADER]]:
+; OTHER-NEXT:    [[PTR_IV:%.*]] = phi ptr [ [[PTR_IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ], [ [[START]], %[[ENTRY]] ]
+; OTHER-NEXT:    [[L:%.*]] = load i8, ptr [[PTR_IV]], align 8
+; OTHER-NEXT:    [[UDIV:%.*]] = udiv i8 [[L]], [[TGT]]
+; OTHER-NEXT:    [[UDIV_2:%.*]] = udiv i8 [[UDIV]], 10
+; OTHER-NEXT:    [[C_1:%.*]] = icmp eq i8 [[UDIV_2]], 2
+; OTHER-NEXT:    br i1 [[C_1]], label %[[EXIT:.*]], label %[[LOOP_LATCH]]
+; OTHER:       [[LOOP_LATCH]]:
+; OTHER-NEXT:    [[PTR_IV_NEXT]] = getelementptr inbounds nuw i8, ptr [[PTR_IV]], i64 1
+; OTHER-NEXT:    [[C_2:%.*]] = icmp eq ptr [[PTR_IV_NEXT]], [[END]]
+; OTHER-NEXT:    br i1 [[C_2]], label %[[EXIT]], label %[[LOOP_HEADER]]
+; OTHER:       [[EXIT]]:
+; OTHER-NEXT:    [[RES:%.*]] = phi ptr [ [[PTR_IV]], %[[LOOP_HEADER]] ], [ [[END]], %[[LOOP_LATCH]] ]
+; OTHER-NEXT:    [[C_3:%.*]] = icmp eq ptr [[RES]], [[END]]
+; OTHER-NEXT:    ret i1 [[C_3]]
+;
+entry:
+  %start = load ptr, ptr %vec, align 8
+  %gep.end = getelementptr inbounds nuw i8, ptr %vec, i64 1
+  %end = load ptr, ptr %gep.end, align 8
+  br label %loop.header
+
+loop.header:
+  %ptr.iv = phi ptr [ %ptr.iv.next, %loop.latch ], [ %start, %entry ]
+  %l = load i8, ptr %ptr.iv, align 8
+  %udiv = udiv i8 %l, %tgt
+  %udiv.2 = udiv i8 %udiv, 10
+  %c.1 = icmp eq i8 %udiv.2, 2
+  br i1 %c.1, label %exit, label %loop.latch
+
+loop.latch:
+  %ptr.iv.next = getelementptr inbounds nuw i8, ptr %ptr.iv, i64 1
+  %c.2 = icmp eq ptr %ptr.iv.next, %end
+  br i1 %c.2, label %exit, label %loop.header
+
+exit:
+  %res = phi ptr [ %ptr.iv, %loop.header ], [ %end, %loop.latch ]
+  %c.3 = icmp eq ptr %res, %end
+  ret i1 %c.3
+}
+define i1 @multi_3_exit_find_ptr_loop(ptr %vec, ptr %tgt, ptr %tgt2) {
+; APPLE-LABEL: define i1 @multi_3_exit_find_ptr_loop(
+; APPLE-SAME: ptr [[VEC:%.*]], ptr [[TGT:%.*]], ptr [[TGT2:%.*]]) #[[ATTR0]] {
+; APPLE-NEXT:  [[ENTRY:.*]]:
+; APPLE-NEXT:    [[START:%.*]] = load ptr, ptr [[VEC]], align 8
+; APPLE-NEXT:    call void @llvm.assume(i1 true) [ "align"(ptr [[START]], i64 8) ]
+; APPLE-NEXT:    [[GEP_END:%.*]] = getelementptr inbounds nuw i8, ptr [[VEC]], i64 8
+; APPLE-NEXT:    [[END:%.*]] = load ptr, ptr [[GEP_END]], align 8
+; APPLE-NEXT:    call void @llvm.assume(i1 true) [ "align"(ptr [[END]], i64 8) ]
+; APPLE-NEXT:    br label %[[LOOP_HEADER:.*]]
+; APPLE:       [[LOOP_HEADER]]:
+; APPLE-NEXT:    [[PTR_IV:%.*]] = phi ptr [ [[PTR_IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ], [ [[START]], %[[ENTRY]] ]
+; APPLE-NEXT:    [[L:%.*]] = load ptr, ptr [[PTR_IV]], align 8
+; APPLE-NEXT:    [[C_1:%.*]] = icmp eq ptr [[L]], [[TGT]]
+; APPLE-NEXT:    [[C_2:%.*]] = icmp eq ptr [[L]], [[TGT2]]
+; APPLE-NEXT:    [[OR_COND:%.*]] = select i1 [[C_1]], i1 true, i1 [[C_2]]
+; APPLE-NEXT:    br i1 [[OR_COND]], label %[[EXIT:.*]], label %[[LOOP_LATCH]]
+; APPLE:       [[LOOP_LATCH]]:
+; APPLE-NEXT:    [[PTR_IV_NEXT]] = getelementptr inbounds nuw i8, ptr [[PTR_IV]], i64 8
+; APPLE-NEXT:    [[C_3:%.*]] = icmp eq ptr [[PTR_IV_NEXT]], [[END]]
+; APPLE-NEXT:    br i1 [[C_3]], label %[[EXIT]], label %[[LOOP_HEADER]]
+; APPLE:       [[EXIT]]:
+; APPLE-NEXT:    [[RES:%.*]] = phi ptr [ [[PTR_IV]], %[[LOOP_HEADER]] ], [ [[END]], %[[LOOP_LATCH]] ]
+; APPLE-NEXT:    call void @llvm.assume(i1 true) [ "align"(ptr [[END]], i64 8) ]
+; APPLE-NEXT:    [[C_4:%.*]] = icmp eq ptr [[RES]], [[END]]
+; APPLE-NEXT:    ret i1 [[C_4]]
+;
+; OTHER-LABEL: define i1 @multi_3_exit_find_ptr_loop(
+; OTHER-SAME: ptr [[VEC:%.*]], ptr [[TGT:%.*]], ptr [[TGT2:%.*]]) #[[ATTR0]] {
+; OTHER-NEXT:  [[ENTRY:.*]]:
+; OTHER-NEXT:    [[START:%.*]] = load ptr, ptr [[VEC]], align 8
+; OTHER-NEXT:    call void @llvm.assume(i1 true) [ "align"(ptr [[START]], i64 8) ]
+; OTHER-NEXT:    [[GEP_END:%.*]] = getelementptr inbounds nuw i8, ptr [[VEC]], i64 8
+; OTHER-NEXT:    [[END:%.*]] = load ptr, ptr [[GEP_END]], align 8
+; OTHER-NEXT:    call void @llvm.assume(i1 true) [ "align"(ptr [[END]], i64 8) ]
+; OTHER-NEXT:    br label %[[LOOP_HEADER:.*]]
+; OTHER:       [[LOOP_HEADER]]:
+; OTHER-NEXT:    [[PTR_IV:%.*]] = phi ptr [ [[PTR_IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ], [ [[START]], %[[ENTRY]] ]
+; OTHER-NEXT:    [[L:%.*]] = load ptr, ptr [[PTR_IV]], align 8
+; OTHER-NEXT:    [[C_1:%.*]] = icmp eq ptr [[L]], [[TGT]]
+; OTHER-NEXT:    [[C_2:%.*]] = icmp eq ptr [[L]], [[TGT2]]
+; OTHER-NEXT:    [[OR_COND:%.*]] = select i1 [[C_1]], i1 true, i1 [[C_2]]
+; OTHER-NEXT:    br i1 [[OR_COND]], label %[[EXIT:.*]], label %[[LOOP_LATCH]]
+; OTHER:       [[LOOP_LATCH]]:
+; OTHER-NEXT:    [[PTR_IV_NEXT]] = getelementptr inbounds nuw i8, ptr [[PTR_IV]], i64 8
+; OTHER-NEXT:    [[C_3:%.*]] = icmp eq ptr [[PTR_IV_NEXT]], [[END]]
+; OTHER-NEXT:    br i1 [[C_3]], label %[[EXIT]], label %[[LOOP_HEADER]]
+; OTHER:       [[EXIT]]:
+; OTHER-NEXT:    [[RES:%.*]] = phi ptr [ [[PTR_IV]], %[[LOOP_HEADER]] ], [ [[END]], %[[LOOP_LATCH]] ]
+; OTHER-NEXT:    call void @llvm.assume(i1 true) [ "align"(ptr [[END]], i64 8) ]
+; OTHER-NEXT:    [[C_4:%.*]] = icmp eq ptr [[RES]], [[END]]
+; OTHER-NEXT:    ret i1 [[C_4]]
+;
+entry:
+  %start = load ptr, ptr %vec, align 8
+  call void @llvm.assume(i1 true) [ "align"(ptr %start, i64 8) ]
+  %gep.end = getelementptr inbounds nuw i8, ptr %vec, i64 8
+  %end = load ptr, ptr %gep.end, align 8
+  call void @llvm.assume(i1 true) [ "align"(ptr %end, i64 8) ]
+  br label %loop.header
+
+loop.header:
+  %ptr.iv = phi ptr [ %ptr.iv.next, %loop.latch ], [ %start, %entry ]
+  %l = load ptr, ptr %ptr.iv, align 8
+  %c.1 = icmp eq ptr %l, %tgt
+  br i1 %c.1, label %exit, label %then
+
+then:
+  %c.2 = icmp eq ptr %l, %tgt2
+  br i1 %c.2, label %exit, label %loop.latch
+
+loop.latch:
+  %ptr.iv.next = getelementptr inbounds nuw i8, ptr %ptr.iv, i64 8
+  %c.3 = icmp eq ptr %ptr.iv.next, %end
+  br i1 %c.3, label %exit, label %loop.header
+
+exit:
+  %res = phi ptr [ %ptr.iv, %loop.header ], [ %ptr.iv, %then], [ %end, %loop.latch ]
+  call void @llvm.assume(i1 true) [ "align"(ptr %end, i64 8) ]
+  %c.4 = icmp eq ptr %res, %end
+  ret i1 %c.4
+}
+
+define i1 @multi_3_exit_find_i8_loop_switch(ptr %vec, i8 %tgt) {
+; APPLE-LABEL: define i1 @multi_3_exit_find_i8_loop_switch(
+; APPLE-SAME: ptr [[VEC:%.*]], i8 [[TGT:%.*]]) #[[ATTR0]] {
+; APPLE-NEXT:  [[ENTRY:.*]]:
+; APPLE-NEXT:    [[START:%.*]] = load ptr, ptr [[VEC]], align 8
+; APPLE-NEXT:    [[GEP_END:%.*]] = getelementptr inbounds nuw i8, ptr [[VEC]], i64 1
+; APPLE-NEXT:    [[END:%.*]] = load ptr, ptr [[GEP_END]], align 8
+; APPLE-NEXT:    br label %[[LOOP_HEADER:.*]]
+; APPLE:       [[LOOP_HEADER]]:
+; APPLE-NEXT:    [[PTR_IV:%.*]] = phi ptr [ [[PTR_IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ], [ [[START]], %[[ENTRY]] ]
+; APPLE-NEXT:    [[L:%.*]] = load i8, ptr [[PTR_IV]], align 8
+; APPLE-NEXT:    switch i8 [[L]], label %[[LOOP_LATCH]] [
+; APPLE-NEXT:      i8 0, label %[[EXIT_1:.*]]
+; APPLE-NEXT:      i8 1, label %[[EXIT_2:.*]]
+; APPLE-NEXT:      i8 2, label %[[EXIT:.*]]
+; APPLE-NEXT:    ]
+; APPLE:       [[LOOP_LATCH]]:
+; APPLE-NEXT:    [[PTR_IV_NEXT]] = getelementptr inbounds nuw i8, ptr [[PTR_IV]], i64 1
+; APPLE-NEXT:    [[C_2:%.*]] = icmp eq ptr [[PTR_IV_NEXT]], [[END]]
+; APPLE-NEXT:    br i1 [[C_2]], label %[[EXIT]], label %[[LOOP_HEADER]]
+; APPLE:       [[EXIT]]:
+; APPLE-NEXT:    [[RES:%.*]] = phi ptr [ [[PTR_IV]], %[[LOOP_HEADER]] ], [ [[END]], %[[LOOP_LATCH]] ]
+; APPLE-NEXT:    [[C_3:%.*]] = icmp eq ptr [[RES]], [[END]]
+; APPLE-NEXT:    ret i1 [[C_3]]
+; APPLE:       [[EXIT_1]]:
+; APPLE-NEXT:    ret i1 false
+; APPLE:       [[EXIT_2]]:
+; APPLE-NEXT:    ret i1 true
+;
+; OTHER-LABEL: define i1 @multi_3_exit_find_i8_loop_switch(
+; OTHER-SAME: ptr [[VEC:%.*]], i8 [[TGT:%.*]]) #[[ATTR0]] {
+; OTHER-NEXT:  [[ENTRY:.*]]:
+; OTHER-NEXT:    [[START:%.*]] = load ptr, ptr [[VEC]], align 8
+; OTHER-NEXT:    [[GEP_END:%.*]] = getelementptr inbounds nuw i8, ptr [[VEC]], i64 1
+; OTHER-NEXT:    [[END:%.*]] = load ptr, ptr [[GEP_END]], align 8
+; OTHER-NEXT:    br label %[[LOOP_HEADER:.*]]
+; OTHER:       [[LOOP_HEADER]]:
+; OTHER-NEXT:    [[PTR_IV:%.*]] = phi ptr [ [[PTR_IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ], [ [[START]], %[[ENTRY]] ]
+; OTHER-NEXT:    [[L:%.*]] = load i8, ptr [[PTR_IV]], align 8
+; OTHER-NEXT:    switch i8 [[L]], label %[[LOOP_LATCH]] [
+; OTHER-NEXT:      i8 0, label %[[EXIT_1:.*]]
+; OTHER-NEXT:      i8 1, label %[[EXIT_2:.*]]
+; OTHER-NEXT:      i8 2, label %[[EXIT:.*]]
+; OTHER-NEXT:    ]
+; OTHER:       [[LOOP_LATCH]]:
+; OTHER-NEXT:    [[PTR_IV_NEXT]] = getelementptr inbounds nuw i8, ptr [[PTR_IV]], i64 1
+; OTHER-NEXT:    [[C_2:%.*]] = icmp eq ptr [[PTR_IV_NEXT]], [[END]]
+; OTHER-NEXT:    br i1 [[C_2]], label %[[EXIT]], label %[[LOOP_HEADER]]
+; OTHER:       [[EXIT]]:
+; OTHER-NEXT:    [[RES:%.*]] = phi ptr [ [[PTR_IV]], %[[LOOP_HEADER]] ], [ [[END]], %[[LOOP_LATCH]] ]
+; OTHER-NEXT:    [[C_3:%.*]] = icmp eq ptr [[RES]], [[END]]
+; OTHER-NEXT:    ret i1 [[C_3]]
+; OTHER:       [[EXIT_1]]:
+; OTHER-NEXT:    ret i1 false
+; OTHER:       [[EXIT_2]]:
+; OTHER-NEXT:    ret i1 true
+;
+entry:
+  %start = load ptr, ptr %vec, align 8
+  %gep.end = getelementptr inbounds nuw i8, ptr %vec, i64 1
+  %end = load ptr, ptr %gep.end, align 8
+  br label %loop.header
+
+loop.header:
+  %ptr.iv = phi ptr [ %ptr.iv.next, %loop.latch ], [ %start, %entry ]
+  %l = load i8, ptr %ptr.iv, align 8
+  switch i8 %l, label %loop.latch [
+  i8 0, label %exit.1
+  i8 1, label %exit.2
+  i8 2, label %exit ]
+
+loop.latch:
+  %ptr.iv.next = getelementptr inbounds nuw i8, ptr %ptr.iv, i64 1
+  %c.2 = icmp eq ptr %ptr.iv.next, %end
+  br i1 %c.2, label %exit, label %loop.header
+
+exit:
+  %res = phi ptr [ %ptr.iv, %loop.header ], [ %end, %loop.latch ]
+  %c.3 = icmp eq ptr %res, %end
+  ret i1 %c.3
+
+exit.1:
+  ret i1 0
+
+exit.2:
+  ret i1 1
+}
+
+declare void @llvm.assume(i1 noundef)
diff --git a/llvm/test/Transforms/LoopUnroll/unroll-cleanup.ll b/llvm/test/Transforms/LoopUnroll/unroll-cleanup.ll
index 40d6ddc27370e..49d6bc46dab32 100644
--- a/llvm/test/Transforms/LoopUnroll/unroll-cleanup.ll
+++ b/llvm/test/Transforms/LoopUnroll/unroll-cleanup.ll
@@ -21,7 +21,7 @@ target triple = "x86_64-unknown-linux-gnu"
 ; Function Attrs: nounwind uwtable
 define void @_Z3fn1v(ptr %r, ptr %a) #0 {
 ; CHECK-LABEL: define void @_Z3fn1v(
-; CHECK-SAME: ptr nocapture writeonly [[R:%.*]], ptr nocapture readonly [[A:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
+; CHECK-SAME: ptr writeonly captures(none) [[R:%.*]], ptr readonly captures(none) [[A:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
 ; CHECK-NEXT:  [[ENTRY:.*]]:
 ; CHECK-NEXT:    [[TMP:%.*]] = load i32, ptr @b, align 4
 ; CHECK-NEXT:    [[TOBOOL20:%.*]] = icmp eq i32 [[TMP]], 0
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/clamped-trip-count.ll b/llvm/test/Transforms/LoopVectorize/AArch64/clamped-trip-count.ll
index 5b77ced73bce0..1f46073d025a9 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/clamped-trip-count.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/clamped-trip-count.ll
@@ -3,7 +3,7 @@
 
 define void @clamped_tc_8(ptr nocapture %dst, i32 %n, i64 %val) vscale_range(1,16) {
 ; CHECK-LABEL: define void @clamped_tc_8(
-; CHECK-SAME: ptr nocapture [[DST:%.*]], i32 [[N:%.*]], i64 [[VAL:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-SAME: ptr captures(none) [[DST:%.*]], i32 [[N:%.*]], i64 [[VAL:%.*]]) #[[ATTR0:[0-9]+]] {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.ph:
@@ -82,7 +82,7 @@ for.cond.cleanup:                                 ; preds = %for.body
 
 define void @clamped_tc_max_8(ptr nocapture %dst, i32 %n, i64 %val) vscale_range(1,16) {
 ; CHECK-LABEL: define void @clamped_tc_max_8(
-; CHECK-SAME: ptr nocapture [[DST:%.*]], i32 [[N:%.*]], i64 [[VAL:%.*]]) #[[ATTR0]] {
+; CHECK-SAME: ptr captures(none) [[DST:%.*]], i32 [[N:%.*]], i64 [[VAL:%.*]]) #[[ATTR0]] {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[REM:%.*]] = and i32 [[N]], 63
 ; CHECK-NEXT:    [[CMP8_NOT:%.*]] = icmp eq i32 [[REM]], 0
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/conditional-branches-cost.ll b/llvm/test/Transforms/LoopVectorize/AArch64/conditional-branches-cost.ll
index caa98d766a8c3..754b86ab2fb87 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/conditional-branches-cost.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/conditional-branches-cost.ll
@@ -369,7 +369,7 @@ define void @latch_branch_cost(ptr %dst) {
 ; DEFAULT:       vec.epilog.middle.block:
 ; DEFAULT-NEXT:    br i1 true, label [[FOR_END]], label [[SCALAR_PH]]
 ; DEFAULT:       vec.epilog.scalar.ph:
-; DEFAULT-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 100, [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ 0, [[ITER_CHECK:%.*]] ], [ 96, [[VEC_EPILOG_ITER_CHECK]] ]
+; DEFAULT-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 100, [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ 96, [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[ITER_CHECK:%.*]] ]
 ; DEFAULT-NEXT:    br label [[FOR_BODY:%.*]]
 ; DEFAULT:       loop:
 ; DEFAULT-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ]
@@ -588,7 +588,7 @@ define i32 @header_mask_and_invariant_compare(ptr %A, ptr %B, ptr %C, ptr %D, pt
 ; DEFAULT-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]]
 ; DEFAULT-NEXT:    br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
 ; DEFAULT:       scalar.ph:
-; DEFAULT-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[VECTOR_MEMCHECK]] ], [ 0, [[ENTRY:%.*]] ]
+; DEFAULT-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ], [ 0, [[VECTOR_MEMCHECK]] ]
 ; DEFAULT-NEXT:    br label [[LOOP_HEADER:%.*]]
 ; DEFAULT:       loop.header:
 ; DEFAULT-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP_LATCH:%.*]] ]
@@ -715,7 +715,7 @@ define i32 @header_mask_and_invariant_compare(ptr %A, ptr %B, ptr %C, ptr %D, pt
 ; PRED:       middle.block:
 ; PRED-NEXT:    br i1 true, label [[EXIT:%.*]], label [[SCALAR_PH]]
 ; PRED:       scalar.ph:
-; PRED-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[VECTOR_MEMCHECK]] ], [ 0, [[ENTRY:%.*]] ]
+; PRED-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ], [ 0, [[VECTOR_MEMCHECK]] ]
 ; PRED-NEXT:    br label [[LOOP_HEADER:%.*]]
 ; PRED:       loop.header:
 ; PRED-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP_LATCH:%.*]] ]
@@ -1301,7 +1301,7 @@ define void @test_conditional_interleave_group (ptr noalias %src.1, ptr noalias
 ; DEFAULT-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]]
 ; DEFAULT-NEXT:    br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
 ; DEFAULT:       scalar.ph:
-; DEFAULT-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[VECTOR_SCEVCHECK]] ], [ 0, [[ENTRY:%.*]] ]
+; DEFAULT-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ], [ 0, [[VECTOR_SCEVCHECK]] ]
 ; DEFAULT-NEXT:    br label [[LOOP_HEADER:%.*]]
 ; DEFAULT:       loop.header:
 ; DEFAULT-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP_LATCH:%.*]] ]
@@ -1523,7 +1523,7 @@ define void @test_conditional_interleave_group (ptr noalias %src.1, ptr noalias
 ; PRED:       middle.block:
 ; PRED-NEXT:    br i1 true, label [[EXIT:%.*]], label [[SCALAR_PH]]
 ; PRED:       scalar.ph:
-; PRED-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[VECTOR_SCEVCHECK]] ], [ 0, [[ENTRY:%.*]] ]
+; PRED-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ], [ 0, [[VECTOR_SCEVCHECK]] ]
 ; PRED-NEXT:    br label [[LOOP_HEADER:%.*]]
 ; PRED:       loop.header:
 ; PRED-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP_LATCH:%.*]] ]
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/deterministic-type-shrinkage.ll b/llvm/test/Transforms/LoopVectorize/AArch64/deterministic-type-shrinkage.ll
index 2ccc51117bbb0..c2502aac5b61d 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/deterministic-type-shrinkage.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/deterministic-type-shrinkage.ll
@@ -10,7 +10,7 @@ target triple = "aarch64"
 
 define void @test_pr25490(i32 %n, ptr noalias nocapture %a, ptr noalias nocapture %b, ptr noalias nocapture readonly %c) {
 ; CHECK-LABEL: define void @test_pr25490
-; CHECK-SAME: (i32 [[N:%.*]], ptr noalias nocapture [[A:%.*]], ptr noalias nocapture [[B:%.*]], ptr noalias nocapture readonly [[C:%.*]]) {
+; CHECK-SAME: (i32 [[N:%.*]], ptr noalias captures(none) [[A:%.*]], ptr noalias captures(none) [[B:%.*]], ptr noalias readonly captures(none) [[C:%.*]]) {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[CMP_28:%.*]] = icmp eq i32 [[N]], 0
 ; CHECK-NEXT:    br i1 [[CMP_28]], label [[FOR_COND_CLEANUP:%.*]], label [[ITER_CHECK:%.*]]
@@ -83,7 +83,7 @@ define void @test_pr25490(i32 %n, ptr noalias nocapture %a, ptr noalias nocaptur
 ; CHECK-NEXT:    [[CMP_N11:%.*]] = icmp eq i64 [[N_VEC5]], [[TMP0]]
 ; CHECK-NEXT:    br i1 [[CMP_N11]], label [[FOR_COND_CLEANUP_LOOPEXIT]], label [[VEC_EPILOG_SCALAR_PH]]
 ; CHECK:       vec.epilog.scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC5]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ 0, [[ITER_CHECK]] ], [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC5]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[ITER_CHECK]] ]
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK:       for.cond.cleanup.loopexit:
 ; CHECK-NEXT:    br label [[FOR_COND_CLEANUP]]
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/divs-with-scalable-vfs.ll b/llvm/test/Transforms/LoopVectorize/AArch64/divs-with-scalable-vfs.ll
index 2c37593be7861..ecf49a31ba79f 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/divs-with-scalable-vfs.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/divs-with-scalable-vfs.ll
@@ -52,7 +52,7 @@ define void @sdiv_feeding_gep(ptr %dst, i32 %x, i64 %M, i64 %conv6, i64 %N) {
 ; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
 ; CHECK-NEXT:    br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
 ; CHECK:       [[SCALAR_PH]]:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[VECTOR_SCEVCHECK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_SCEVCHECK]] ]
 ; CHECK-NEXT:    br label %[[LOOP:.*]]
 ; CHECK:       [[LOOP]]:
 ; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
@@ -162,7 +162,7 @@ define void @sdiv_feeding_gep_predicated(ptr %dst, i32 %x, i64 %M, i64 %conv6, i
 ; CHECK:       [[MIDDLE_BLOCK]]:
 ; CHECK-NEXT:    br i1 true, label %[[EXIT:.*]], label %[[SCALAR_PH]]
 ; CHECK:       [[SCALAR_PH]]:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[VECTOR_SCEVCHECK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_SCEVCHECK]] ]
 ; CHECK-NEXT:    br label %[[LOOP:.*]]
 ; CHECK:       [[LOOP]]:
 ; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
@@ -289,7 +289,7 @@ define void @udiv_urem_feeding_gep(i64 %x, ptr %dst, i64 %N) {
 ; CHECK:       [[MIDDLE_BLOCK]]:
 ; CHECK-NEXT:    br i1 true, label %[[EXIT:.*]], label %[[SCALAR_PH]]
 ; CHECK:       [[SCALAR_PH]]:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[VECTOR_SCEVCHECK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_SCEVCHECK]] ]
 ; CHECK-NEXT:    br label %[[LOOP:.*]]
 ; CHECK:       [[LOOP]]:
 ; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/drop-poison-generating-flags.ll b/llvm/test/Transforms/LoopVectorize/AArch64/drop-poison-generating-flags.ll
new file mode 100644
index 0000000000000..53bd2d119c1ae
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/drop-poison-generating-flags.ll
@@ -0,0 +1,151 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --version 5
+; RUN: opt -p loop-vectorize -mcpu=neoverse-v2 -force-vector-width=4 -S %s | FileCheck %s
+
+target triple = "aarch64-unknown-linux"
+
+; Test case where we visit a VPWidenIntrinsic (for @llvm.fabs) with nnan flags.
+; For https://github.com/llvm/llvm-project/issues/125301.
+define void @check_widen_intrinsic_with_nnan(ptr noalias %dst.0, ptr noalias %dst.1, ptr noalias %src.1, ptr %src.2) {
+; CHECK-LABEL: define void @check_widen_intrinsic_with_nnan(
+; CHECK-SAME: ptr noalias [[DST_0:%.*]], ptr noalias [[DST_1:%.*]], ptr noalias [[SRC_1:%.*]], ptr [[SRC_2:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK:       [[VECTOR_PH]]:
+; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK:       [[VECTOR_BODY]]:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_LOAD_CONTINUE6:.*]] ]
+; CHECK-NEXT:    [[TMP0:%.*]] = add i64 [[INDEX]], 0
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds double, ptr [[SRC_1]], i64 [[TMP0]]
+; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds double, ptr [[TMP1]], i32 0
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x double>, ptr [[TMP2]], align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = call <4 x double> @llvm.fabs.v4f64(<4 x double> [[WIDE_LOAD]])
+; CHECK-NEXT:    [[TMP4:%.*]] = fcmp olt <4 x double> [[TMP3]], splat (double 1.000000e+00)
+; CHECK-NEXT:    [[TMP5:%.*]] = xor <4 x i1> [[TMP4]], splat (i1 true)
+; CHECK-NEXT:    [[TMP6:%.*]] = add i64 [[TMP0]], -1
+; CHECK-NEXT:    [[TMP7:%.*]] = getelementptr double, ptr [[DST_0]], i64 [[TMP6]]
+; CHECK-NEXT:    [[TMP8:%.*]] = getelementptr double, ptr [[TMP7]], i32 0
+; CHECK-NEXT:    call void @llvm.masked.store.v4f64.p0(<4 x double> zeroinitializer, ptr [[TMP8]], i32 8, <4 x i1> [[TMP5]])
+; CHECK-NEXT:    [[TMP9:%.*]] = extractelement <4 x i1> [[TMP4]], i32 0
+; CHECK-NEXT:    br i1 [[TMP9]], label %[[PRED_LOAD_IF:.*]], label %[[PRED_LOAD_CONTINUE:.*]]
+; CHECK:       [[PRED_LOAD_IF]]:
+; CHECK-NEXT:    [[TMP10:%.*]] = load double, ptr [[SRC_2]], align 8
+; CHECK-NEXT:    [[TMP11:%.*]] = insertelement <4 x double> poison, double [[TMP10]], i32 0
+; CHECK-NEXT:    br label %[[PRED_LOAD_CONTINUE]]
+; CHECK:       [[PRED_LOAD_CONTINUE]]:
+; CHECK-NEXT:    [[TMP12:%.*]] = phi <4 x double> [ poison, %[[VECTOR_BODY]] ], [ [[TMP11]], %[[PRED_LOAD_IF]] ]
+; CHECK-NEXT:    [[TMP13:%.*]] = extractelement <4 x i1> [[TMP4]], i32 1
+; CHECK-NEXT:    br i1 [[TMP13]], label %[[PRED_LOAD_IF1:.*]], label %[[PRED_LOAD_CONTINUE2:.*]]
+; CHECK:       [[PRED_LOAD_IF1]]:
+; CHECK-NEXT:    [[TMP14:%.*]] = load double, ptr [[SRC_2]], align 8
+; CHECK-NEXT:    [[TMP15:%.*]] = insertelement <4 x double> [[TMP12]], double [[TMP14]], i32 1
+; CHECK-NEXT:    br label %[[PRED_LOAD_CONTINUE2]]
+; CHECK:       [[PRED_LOAD_CONTINUE2]]:
+; CHECK-NEXT:    [[TMP16:%.*]] = phi <4 x double> [ [[TMP12]], %[[PRED_LOAD_CONTINUE]] ], [ [[TMP15]], %[[PRED_LOAD_IF1]] ]
+; CHECK-NEXT:    [[TMP17:%.*]] = extractelement <4 x i1> [[TMP4]], i32 2
+; CHECK-NEXT:    br i1 [[TMP17]], label %[[PRED_LOAD_IF3:.*]], label %[[PRED_LOAD_CONTINUE4:.*]]
+; CHECK:       [[PRED_LOAD_IF3]]:
+; CHECK-NEXT:    [[TMP18:%.*]] = load double, ptr [[SRC_2]], align 8
+; CHECK-NEXT:    [[TMP19:%.*]] = insertelement <4 x double> [[TMP16]], double [[TMP18]], i32 2
+; CHECK-NEXT:    br label %[[PRED_LOAD_CONTINUE4]]
+; CHECK:       [[PRED_LOAD_CONTINUE4]]:
+; CHECK-NEXT:    [[TMP20:%.*]] = phi <4 x double> [ [[TMP16]], %[[PRED_LOAD_CONTINUE2]] ], [ [[TMP19]], %[[PRED_LOAD_IF3]] ]
+; CHECK-NEXT:    [[TMP21:%.*]] = extractelement <4 x i1> [[TMP4]], i32 3
+; CHECK-NEXT:    br i1 [[TMP21]], label %[[PRED_LOAD_IF5:.*]], label %[[PRED_LOAD_CONTINUE6]]
+; CHECK:       [[PRED_LOAD_IF5]]:
+; CHECK-NEXT:    [[TMP22:%.*]] = load double, ptr [[SRC_2]], align 8
+; CHECK-NEXT:    [[TMP23:%.*]] = insertelement <4 x double> [[TMP20]], double [[TMP22]], i32 3
+; CHECK-NEXT:    br label %[[PRED_LOAD_CONTINUE6]]
+; CHECK:       [[PRED_LOAD_CONTINUE6]]:
+; CHECK-NEXT:    [[TMP24:%.*]] = phi <4 x double> [ [[TMP20]], %[[PRED_LOAD_CONTINUE4]] ], [ [[TMP23]], %[[PRED_LOAD_IF5]] ]
+; CHECK-NEXT:    [[TMP25:%.*]] = add i64 [[TMP0]], -1
+; CHECK-NEXT:    [[TMP26:%.*]] = getelementptr double, ptr [[DST_0]], i64 [[TMP25]]
+; CHECK-NEXT:    [[TMP27:%.*]] = getelementptr double, ptr [[TMP26]], i32 0
+; CHECK-NEXT:    call void @llvm.masked.store.v4f64.p0(<4 x double> zeroinitializer, ptr [[TMP27]], i32 8, <4 x i1> [[TMP4]])
+; CHECK-NEXT:    [[TMP28:%.*]] = fcmp oeq <4 x double> [[TMP24]], zeroinitializer
+; CHECK-NEXT:    [[TMP29:%.*]] = select <4 x i1> [[TMP4]], <4 x i1> [[TMP28]], <4 x i1> zeroinitializer
+; CHECK-NEXT:    [[TMP30:%.*]] = or <4 x i1> [[TMP5]], [[TMP29]]
+; CHECK-NEXT:    [[TMP31:%.*]] = extractelement <4 x i1> [[TMP29]], i32 0
+; CHECK-NEXT:    [[PREDPHI:%.*]] = select i1 [[TMP31]], i64 [[TMP25]], i64 [[TMP6]]
+; CHECK-NEXT:    [[TMP32:%.*]] = getelementptr i32, ptr [[DST_1]], i64 [[PREDPHI]]
+; CHECK-NEXT:    [[TMP33:%.*]] = getelementptr i32, ptr [[TMP32]], i32 0
+; CHECK-NEXT:    call void @llvm.masked.store.v4i32.p0(<4 x i32> splat (i32 10), ptr [[TMP33]], i32 4, <4 x i1> [[TMP30]])
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT:    [[TMP34:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1000
+; CHECK-NEXT:    br i1 [[TMP34]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK:       [[MIDDLE_BLOCK]]:
+; CHECK-NEXT:    br i1 true, label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK:       [[SCALAR_PH]]:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 1000, %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT:    br label %[[LOOP_HEADER:.*]]
+; CHECK:       [[LOOP_HEADER]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
+; CHECK-NEXT:    [[GEP_SRC_1:%.*]] = getelementptr inbounds double, ptr [[SRC_1]], i64 [[IV]]
+; CHECK-NEXT:    [[L_1:%.*]] = load double, ptr [[GEP_SRC_1]], align 8
+; CHECK-NEXT:    [[ABS:%.*]] = tail call nnan double @llvm.fabs.f64(double [[L_1]])
+; CHECK-NEXT:    [[C_0:%.*]] = fcmp olt double [[ABS]], 1.000000e+00
+; CHECK-NEXT:    br i1 [[C_0]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[L_2:%.*]] = load double, ptr [[SRC_2]], align 8
+; CHECK-NEXT:    [[IV_SUB_1:%.*]] = add nsw i64 [[IV]], -1
+; CHECK-NEXT:    [[GEP_IV_SUB_1:%.*]] = getelementptr double, ptr [[DST_0]], i64 [[IV_SUB_1]]
+; CHECK-NEXT:    store double 0.000000e+00, ptr [[GEP_IV_SUB_1]], align 8
+; CHECK-NEXT:    [[C_1:%.*]] = fcmp oeq double [[L_2]], 0.000000e+00
+; CHECK-NEXT:    br i1 [[C_1]], label %[[MERGE:.*]], label %[[LOOP_LATCH]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[IV_SUB_2:%.*]] = add nsw i64 [[IV]], -1
+; CHECK-NEXT:    [[GEP_IV_SUB_2:%.*]] = getelementptr double, ptr [[DST_0]], i64 [[IV_SUB_2]]
+; CHECK-NEXT:    store double 0.000000e+00, ptr [[GEP_IV_SUB_2]], align 8
+; CHECK-NEXT:    br label %[[MERGE]]
+; CHECK:       [[MERGE]]:
+; CHECK-NEXT:    [[MERGE_IV:%.*]] = phi i64 [ [[IV_SUB_2]], %[[ELSE]] ], [ [[IV_SUB_1]], %[[THEN]] ]
+; CHECK-NEXT:    [[GEP_DST_1:%.*]] = getelementptr inbounds i32, ptr [[DST_1]], i64 [[MERGE_IV]]
+; CHECK-NEXT:    store i32 10, ptr [[GEP_DST_1]], align 4
+; CHECK-NEXT:    br label %[[LOOP_LATCH]]
+; CHECK:       [[LOOP_LATCH]]:
+; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], 1000
+; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label %[[EXIT]], label %[[LOOP_HEADER]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %loop.header
+
+loop.header:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop.latch ]
+  %gep.src.1 = getelementptr inbounds double, ptr %src.1, i64 %iv
+  %l.1 = load double, ptr %gep.src.1, align 8
+  %abs = tail call nnan double @llvm.fabs.f64(double %l.1)
+  %c.0 = fcmp olt double %abs, 1.000000e+00
+  br i1 %c.0, label %then, label %else
+
+then:
+  %l.2 = load double, ptr %src.2, align 8
+  %iv.sub.1 = add nsw i64 %iv, -1
+  %gep.iv.sub.1 = getelementptr double, ptr %dst.0, i64 %iv.sub.1
+  store double 0.000000e+00, ptr %gep.iv.sub.1, align 8
+  %c.1 = fcmp oeq double %l.2, 0.000000e+00
+  br i1 %c.1, label %merge, label %loop.latch
+
+else:
+  %iv.sub.2 = add nsw i64 %iv, -1
+  %gep.iv.sub.2 = getelementptr double, ptr %dst.0, i64 %iv.sub.2
+  store double 0.000000e+00, ptr %gep.iv.sub.2, align 8
+  br label %merge
+
+merge:
+  %merge.iv = phi i64 [ %iv.sub.2, %else ], [ %iv.sub.1, %then ]
+  %gep.dst.1 = getelementptr inbounds i32, ptr %dst.1, i64 %merge.iv
+  store i32 10, ptr %gep.dst.1, align 4
+  br label %loop.latch
+
+loop.latch:
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond.not = icmp eq i64 %iv.next, 1000
+  br i1 %exitcond.not, label %exit, label %loop.header
+
+exit:
+ret void
+}
+
+declare double @llvm.fabs.f64(double)
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/epilog-vectorization-factors.ll b/llvm/test/Transforms/LoopVectorize/AArch64/epilog-vectorization-factors.ll
index 898e515de0fe3..4a2de58938043 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/epilog-vectorization-factors.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/epilog-vectorization-factors.ll
@@ -85,7 +85,7 @@ define void @add_i8(ptr noalias nocapture noundef writeonly %A, ptr nocapture no
 ; CHECK-NEXT:    [[CMP_N15:%.*]] = icmp eq i64 [[ITERATIONS]], [[N_VEC10]]
 ; CHECK-NEXT:    br i1 [[CMP_N15]], label [[EXIT]], label [[VEC_EPILOG_SCALAR_PH]]
 ; CHECK:       vec.epilog.scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC10]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ 0, [[ITER_CHECK:%.*]] ], [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC10]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[ITER_CHECK:%.*]] ]
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK:       for.body:
 ; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[VEC_EPILOG_SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
@@ -203,7 +203,7 @@ define void @add_i16(ptr noalias nocapture noundef writeonly %A, ptr nocapture n
 ; CHECK-NEXT:    [[CMP_N15:%.*]] = icmp eq i64 [[ITERATIONS]], [[N_VEC10]]
 ; CHECK-NEXT:    br i1 [[CMP_N15]], label [[EXIT]], label [[VEC_EPILOG_SCALAR_PH]]
 ; CHECK:       vec.epilog.scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC10]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ 0, [[ITER_CHECK:%.*]] ], [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC10]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[ITER_CHECK:%.*]] ]
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK:       for.body:
 ; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[VEC_EPILOG_SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
@@ -321,7 +321,7 @@ define void @add_i32(ptr noalias nocapture noundef writeonly %A, ptr nocapture n
 ; CHECK-NEXT:    [[CMP_N15:%.*]] = icmp eq i64 [[ITERATIONS]], [[N_VEC10]]
 ; CHECK-NEXT:    br i1 [[CMP_N15]], label [[EXIT]], label [[VEC_EPILOG_SCALAR_PH]]
 ; CHECK:       vec.epilog.scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC10]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ 0, [[ITER_CHECK:%.*]] ], [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC10]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[ITER_CHECK:%.*]] ]
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK:       for.body:
 ; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[VEC_EPILOG_SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/epilog-vectorization-widen-inductions.ll b/llvm/test/Transforms/LoopVectorize/AArch64/epilog-vectorization-widen-inductions.ll
index 88b14b18c1588..6e26cfa17a4da 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/epilog-vectorization-widen-inductions.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/epilog-vectorization-widen-inductions.ll
@@ -73,8 +73,8 @@ define void @test_widen_ptr_induction(ptr %ptr.start.1) {
 ; CHECK:       vec.epilog.middle.block:
 ; CHECK-NEXT:    br i1 false, label [[EXIT]], label [[VEC_EPILOG_SCALAR_PH]]
 ; CHECK:       vec.epilog.scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 10000, [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ 0, [[ITER_CHECK:%.*]] ], [ 10000, [[VEC_EPILOG_ITER_CHECK]] ]
-; CHECK-NEXT:    [[BC_RESUME_VAL5:%.*]] = phi ptr [ [[IND_END]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[PTR_START_1]], [[ITER_CHECK]] ], [ [[IND_END4]], [[VEC_EPILOG_ITER_CHECK]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 10000, [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ 10000, [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[ITER_CHECK:%.*]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL5:%.*]] = phi ptr [ [[IND_END]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[IND_END4]], [[VEC_EPILOG_ITER_CHECK]] ], [ [[PTR_START_1]], [[ITER_CHECK]] ]
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
 ; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[VEC_EPILOG_SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
@@ -165,7 +165,7 @@ define void @test_widen_induction(ptr %A, i64 %N) {
 ; CHECK-NEXT:    [[CMP_N9:%.*]] = icmp eq i64 [[N]], [[N_VEC3]]
 ; CHECK-NEXT:    br i1 [[CMP_N9]], label [[EXIT]], label [[VEC_EPILOG_SCALAR_PH]]
 ; CHECK:       vec.epilog.scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL4:%.*]] = phi i64 [ [[N_VEC3]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ 0, [[ITER_CHECK:%.*]] ], [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL4:%.*]] = phi i64 [ [[N_VEC3]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[ITER_CHECK:%.*]] ]
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
 ; CHECK-NEXT:    [[IV_1:%.*]] = phi i64 [ [[BC_RESUME_VAL4]], [[VEC_EPILOG_SCALAR_PH]] ], [ [[IV_1_NEXT:%.*]], [[LOOP]] ]
@@ -258,7 +258,7 @@ define void @test_widen_induction_variable_start(ptr %A, i64 %N, i64 %start) {
 ; CHECK-NEXT:    [[CMP_N15:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC3]]
 ; CHECK-NEXT:    br i1 [[CMP_N15]], label [[EXIT]], label [[VEC_EPILOG_SCALAR_PH]]
 ; CHECK:       vec.epilog.scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL6:%.*]] = phi i64 [ [[IND_END4]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[START]], [[ITER_CHECK:%.*]] ], [ [[IND_END5]], [[VEC_EPILOG_ITER_CHECK]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL6:%.*]] = phi i64 [ [[IND_END4]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[IND_END5]], [[VEC_EPILOG_ITER_CHECK]] ], [ [[START]], [[ITER_CHECK:%.*]] ]
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
 ; CHECK-NEXT:    [[IV_1:%.*]] = phi i64 [ [[BC_RESUME_VAL6]], [[VEC_EPILOG_SCALAR_PH]] ], [ [[IV_1_NEXT:%.*]], [[LOOP]] ]
@@ -344,8 +344,8 @@ define void @test_widen_induction_step_2(ptr %A, i64 %N, i32 %step) {
 ; CHECK-NEXT:    [[CMP_N11:%.*]] = icmp eq i64 [[N]], [[IND_END]]
 ; CHECK-NEXT:    br i1 [[CMP_N11]], label [[EXIT]], label [[VEC_EPILOG_SCALAR_PH]]
 ; CHECK:       vec.epilog.scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL5:%.*]] = phi i64 [ [[IND_END]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ 0, [[ITER_CHECK:%.*]] ], [ [[IND_END4]], [[VEC_EPILOG_ITER_CHECK]] ]
-; CHECK-NEXT:    [[BC_RESUME_VAL6:%.*]] = phi i64 [ [[IND_END]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ 0, [[ITER_CHECK]] ], [ [[IND_END4]], [[VEC_EPILOG_ITER_CHECK]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL5:%.*]] = phi i64 [ [[IND_END]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[IND_END4]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[ITER_CHECK:%.*]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL6:%.*]] = phi i64 [ [[IND_END]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[IND_END4]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[ITER_CHECK]] ]
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
 ; CHECK-NEXT:    [[IV_1:%.*]] = phi i64 [ [[BC_RESUME_VAL5]], [[VEC_EPILOG_SCALAR_PH]] ], [ [[IV_1_NEXT:%.*]], [[LOOP]] ]
@@ -431,7 +431,7 @@ define void @test_widen_extended_induction(ptr %dst) {
 ; CHECK:       vec.epilog.middle.block:
 ; CHECK-NEXT:    br i1 true, label [[EXIT]], label [[VEC_EPILOG_SCALAR_PH]]
 ; CHECK:       vec.epilog.scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL1:%.*]] = phi i8 [ 16, [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ 0, [[VECTOR_SCEVCHECK]] ], [ 0, [[ITER_CHECK:%.*]] ], [ 16, [[VEC_EPILOG_ITER_CHECK]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL1:%.*]] = phi i8 [ 16, [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ 16, [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[VECTOR_SCEVCHECK]] ], [ 0, [[ITER_CHECK:%.*]] ]
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
 ; CHECK-NEXT:    [[IV:%.*]] = phi i8 [ [[BC_RESUME_VAL1]], [[VEC_EPILOG_SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
@@ -509,7 +509,7 @@ define void @test_widen_truncated_induction(ptr %A) {
 ; CHECK:       vec.epilog.middle.block:
 ; CHECK-NEXT:    br i1 true, label [[EXIT]], label [[VEC_EPILOG_SCALAR_PH]]
 ; CHECK:       vec.epilog.scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL1:%.*]] = phi i64 [ 10000, [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ 0, [[ITER_CHECK:%.*]] ], [ 10000, [[VEC_EPILOG_ITER_CHECK]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL1:%.*]] = phi i64 [ 10000, [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ 10000, [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[ITER_CHECK:%.*]] ]
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
 ; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL1]], [[VEC_EPILOG_SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/force-target-instruction-cost.ll b/llvm/test/Transforms/LoopVectorize/AArch64/force-target-instruction-cost.ll
index 8c5d84e6981bc..987d18c427aef 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/force-target-instruction-cost.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/force-target-instruction-cost.ll
@@ -116,8 +116,8 @@ define void @test_iv_cost(ptr %ptr.start, i8 %a, i64 %b) {
 ; CHECK-NEXT:    [[CMP_N11:%.*]] = icmp eq i64 [[START]], [[N_VEC3]]
 ; CHECK-NEXT:    br i1 [[CMP_N11]], label %[[EXIT_LOOPEXIT]], label %[[VEC_EPILOG_SCALAR_PH]]
 ; CHECK:       [[VEC_EPILOG_SCALAR_PH]]:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[IND_END1]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[START]], %[[ITER_CHECK]] ], [ [[IND_END]], %[[VEC_EPILOG_ITER_CHECK]] ]
-; CHECK-NEXT:    [[BC_RESUME_VAL9:%.*]] = phi ptr [ [[IND_END5]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[PTR_START]], %[[ITER_CHECK]] ], [ [[IND_END2]], %[[VEC_EPILOG_ITER_CHECK]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[IND_END1]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[IND_END]], %[[VEC_EPILOG_ITER_CHECK]] ], [ [[START]], %[[ITER_CHECK]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL9:%.*]] = phi ptr [ [[IND_END5]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[IND_END2]], %[[VEC_EPILOG_ITER_CHECK]] ], [ [[PTR_START]], %[[ITER_CHECK]] ]
 ; CHECK-NEXT:    br label %[[LOOP:.*]]
 ; CHECK:       [[LOOP]]:
 ; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[IV_NEXT:%.*]], %[[LOOP]] ], [ [[BC_RESUME_VAL]], %[[VEC_EPILOG_SCALAR_PH]] ]
@@ -284,7 +284,7 @@ define void @test_exit_branch_cost(ptr %dst, ptr noalias %x.ptr, ptr noalias %y.
 ; CHECK:       [[MIDDLE_BLOCK]]:
 ; CHECK-NEXT:    br i1 false, label %[[EXIT:.*]], label %[[SCALAR_PH]]
 ; CHECK:       [[SCALAR_PH]]:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 64, %[[MIDDLE_BLOCK]] ], [ 0, %[[VECTOR_MEMCHECK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 64, %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_MEMCHECK]] ]
 ; CHECK-NEXT:    br label %[[LOOP_HEADER:.*]]
 ; CHECK:       [[LOOP_HEADER]]:
 ; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/fully-unrolled-cost.ll b/llvm/test/Transforms/LoopVectorize/AArch64/fully-unrolled-cost.ll
index c3e8c895fce24..92857197a7281 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/fully-unrolled-cost.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/fully-unrolled-cost.ll
@@ -82,11 +82,11 @@ define i64 @test_two_ivs(ptr %a, ptr %b, i64 %start) #0 {
 ; CHECK-NEXT: Cost of 0 for VF 8: induction instruction   %j.iv = phi i64 [ %start, %entry ], [ %j.iv.next, %for.body ]
 ; CHECK-NEXT: Cost of 1 for VF 8: exit condition instruction   %exitcond.not = icmp eq i64 %i.iv.next, 16
 ; CHECK-NEXT: Cost of 0 for VF 8: EMIT vp<{{.+}}> = CANONICAL-INDUCTION ir<0>, vp<%index.next>
-; CHECK: Cost for VF 8: 24
+; CHECK: Cost for VF 8: 27
 ; CHECK-NEXT: Cost of 0 for VF 16: induction instruction   %i.iv = phi i64 [ 0, %entry ], [ %i.iv.next, %for.body ]
 ; CHECK-NEXT: Cost of 0 for VF 16: induction instruction   %j.iv = phi i64 [ %start, %entry ], [ %j.iv.next, %for.body ]
 ; CHECK-NEXT: Cost of 0 for VF 16: EMIT vp<{{.+}}> = CANONICAL-INDUCTION ir<0>, vp<%index.next>
-; CHECK: Cost for VF 16: 42
+; CHECK: Cost for VF 16: 48
 ; CHECK: LV: Selecting VF: 16
 entry:
   br label %for.body
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/induction-costs-sve.ll b/llvm/test/Transforms/LoopVectorize/AArch64/induction-costs-sve.ll
index 56a468ed1310b..62186d5a20989 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/induction-costs-sve.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/induction-costs-sve.ll
@@ -70,7 +70,7 @@ define void @iv_casts(ptr %dst, ptr %src, i32 %x, i64 %N) #0 {
 ; DEFAULT-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]]
 ; DEFAULT-NEXT:    br i1 [[CMP_N]], label [[EXIT:%.*]], label [[VEC_EPILOG_SCALAR_PH]]
 ; DEFAULT:       scalar.ph:
-; DEFAULT-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[VECTOR_MEMCHECK]] ], [ 0, [[ENTRY:%.*]] ]
+; DEFAULT-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ], [ 0, [[VECTOR_MEMCHECK]] ]
 ; DEFAULT-NEXT:    br label [[LOOP:%.*]]
 ; DEFAULT:       loop:
 ; DEFAULT-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[VEC_EPILOG_SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
@@ -146,7 +146,7 @@ define void @iv_casts(ptr %dst, ptr %src, i32 %x, i64 %N) #0 {
 ; PRED:       middle.block:
 ; PRED-NEXT:    br i1 true, label [[EXIT:%.*]], label [[SCALAR_PH]]
 ; PRED:       scalar.ph:
-; PRED-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[VECTOR_MEMCHECK]] ], [ 0, [[ENTRY:%.*]] ]
+; PRED-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ], [ 0, [[VECTOR_MEMCHECK]] ]
 ; PRED-NEXT:    br label [[LOOP:%.*]]
 ; PRED:       loop:
 ; PRED-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
@@ -238,7 +238,7 @@ define void @iv_trunc(i32 %x, ptr %dst, i64 %N) #0 {
 ; DEFAULT-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]]
 ; DEFAULT-NEXT:    br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
 ; DEFAULT:       scalar.ph:
-; DEFAULT-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[VECTOR_SCEVCHECK]] ], [ 0, [[ENTRY:%.*]] ]
+; DEFAULT-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ], [ 0, [[VECTOR_SCEVCHECK]] ]
 ; DEFAULT-NEXT:    br label [[FOR_BODY:%.*]]
 ; DEFAULT:       for.body:
 ; DEFAULT-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
@@ -318,7 +318,7 @@ define void @iv_trunc(i32 %x, ptr %dst, i64 %N) #0 {
 ; PRED:       middle.block:
 ; PRED-NEXT:    br i1 true, label [[EXIT:%.*]], label [[SCALAR_PH]]
 ; PRED:       scalar.ph:
-; PRED-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[VECTOR_SCEVCHECK]] ], [ 0, [[ENTRY:%.*]] ]
+; PRED-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ], [ 0, [[VECTOR_SCEVCHECK]] ]
 ; PRED-NEXT:    br label [[FOR_BODY:%.*]]
 ; PRED:       for.body:
 ; PRED-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
@@ -407,8 +407,8 @@ define void @trunc_ivs_and_store(i32 %x, ptr %dst, i64 %N) #0 {
 ; DEFAULT-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]]
 ; DEFAULT-NEXT:    br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
 ; DEFAULT:       scalar.ph:
-; DEFAULT-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[VECTOR_SCEVCHECK]] ], [ 0, [[ENTRY:%.*]] ]
-; DEFAULT-NEXT:    [[BC_RESUME_VAL2:%.*]] = phi i32 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ 0, [[VECTOR_SCEVCHECK]] ], [ 0, [[ENTRY]] ]
+; DEFAULT-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ], [ 0, [[VECTOR_SCEVCHECK]] ]
+; DEFAULT-NEXT:    [[BC_RESUME_VAL2:%.*]] = phi i32 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ], [ 0, [[VECTOR_SCEVCHECK]] ]
 ; DEFAULT-NEXT:    br label [[LOOP:%.*]]
 ; DEFAULT:       loop:
 ; DEFAULT-NEXT:    [[IV_1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_1_NEXT:%.*]], [[LOOP]] ]
@@ -513,8 +513,8 @@ define void @trunc_ivs_and_store(i32 %x, ptr %dst, i64 %N) #0 {
 ; PRED:       middle.block:
 ; PRED-NEXT:    br i1 true, label [[EXIT:%.*]], label [[SCALAR_PH]]
 ; PRED:       scalar.ph:
-; PRED-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[VECTOR_SCEVCHECK]] ], [ 0, [[ENTRY:%.*]] ]
-; PRED-NEXT:    [[BC_RESUME_VAL8:%.*]] = phi i32 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ 0, [[VECTOR_SCEVCHECK]] ], [ 0, [[ENTRY]] ]
+; PRED-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ], [ 0, [[VECTOR_SCEVCHECK]] ]
+; PRED-NEXT:    [[BC_RESUME_VAL8:%.*]] = phi i32 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ], [ 0, [[VECTOR_SCEVCHECK]] ]
 ; PRED-NEXT:    br label [[LOOP:%.*]]
 ; PRED:       loop:
 ; PRED-NEXT:    [[IV_1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_1_NEXT:%.*]], [[LOOP]] ]
@@ -605,8 +605,8 @@ define void @ivs_trunc_and_ext(i32 %x, ptr %dst, i64 %N) #0 {
 ; DEFAULT-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]]
 ; DEFAULT-NEXT:    br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
 ; DEFAULT:       scalar.ph:
-; DEFAULT-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[VECTOR_SCEVCHECK]] ], [ 0, [[ENTRY:%.*]] ]
-; DEFAULT-NEXT:    [[BC_RESUME_VAL1:%.*]] = phi i32 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ 0, [[VECTOR_SCEVCHECK]] ], [ 0, [[ENTRY]] ]
+; DEFAULT-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ], [ 0, [[VECTOR_SCEVCHECK]] ]
+; DEFAULT-NEXT:    [[BC_RESUME_VAL1:%.*]] = phi i32 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ], [ 0, [[VECTOR_SCEVCHECK]] ]
 ; DEFAULT-NEXT:    br label [[LOOP:%.*]]
 ; DEFAULT:       loop:
 ; DEFAULT-NEXT:    [[IV_1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_1_NEXT:%.*]], [[LOOP]] ]
@@ -710,8 +710,8 @@ define void @ivs_trunc_and_ext(i32 %x, ptr %dst, i64 %N) #0 {
 ; PRED:       middle.block:
 ; PRED-NEXT:    br i1 true, label [[EXIT:%.*]], label [[SCALAR_PH]]
 ; PRED:       scalar.ph:
-; PRED-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[VECTOR_SCEVCHECK]] ], [ 0, [[ENTRY:%.*]] ]
-; PRED-NEXT:    [[BC_RESUME_VAL7:%.*]] = phi i32 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ 0, [[VECTOR_SCEVCHECK]] ], [ 0, [[ENTRY]] ]
+; PRED-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ], [ 0, [[VECTOR_SCEVCHECK]] ]
+; PRED-NEXT:    [[BC_RESUME_VAL7:%.*]] = phi i32 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ], [ 0, [[VECTOR_SCEVCHECK]] ]
 ; PRED-NEXT:    br label [[LOOP:%.*]]
 ; PRED:       loop:
 ; PRED-NEXT:    [[IV_1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_1_NEXT:%.*]], [[LOOP]] ]
@@ -786,8 +786,8 @@ define void @exit_cond_zext_iv(ptr %dst, i64 %N) {
 ; DEFAULT-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[UMAX1]], [[N_VEC]]
 ; DEFAULT-NEXT:    br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
 ; DEFAULT:       scalar.ph:
-; DEFAULT-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ 0, [[VECTOR_SCEVCHECK]] ], [ 0, [[ENTRY:%.*]] ]
-; DEFAULT-NEXT:    [[BC_RESUME_VAL2:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[VECTOR_SCEVCHECK]] ], [ 0, [[ENTRY]] ]
+; DEFAULT-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ], [ 0, [[VECTOR_SCEVCHECK]] ]
+; DEFAULT-NEXT:    [[BC_RESUME_VAL2:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ], [ 0, [[VECTOR_SCEVCHECK]] ]
 ; DEFAULT-NEXT:    br label [[LOOP:%.*]]
 ; DEFAULT:       loop:
 ; DEFAULT-NEXT:    [[IV_1:%.*]] = phi i32 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_1_NEXT:%.*]], [[LOOP]] ]
@@ -852,8 +852,8 @@ define void @exit_cond_zext_iv(ptr %dst, i64 %N) {
 ; PRED:       middle.block:
 ; PRED-NEXT:    br i1 true, label [[EXIT:%.*]], label [[SCALAR_PH]]
 ; PRED:       scalar.ph:
-; PRED-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ 0, [[VECTOR_SCEVCHECK]] ], [ 0, [[ENTRY:%.*]] ]
-; PRED-NEXT:    [[BC_RESUME_VAL6:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[VECTOR_SCEVCHECK]] ], [ 0, [[ENTRY]] ]
+; PRED-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ], [ 0, [[VECTOR_SCEVCHECK]] ]
+; PRED-NEXT:    [[BC_RESUME_VAL6:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ], [ 0, [[VECTOR_SCEVCHECK]] ]
 ; PRED-NEXT:    br label [[LOOP:%.*]]
 ; PRED:       loop:
 ; PRED-NEXT:    [[IV_1:%.*]] = phi i32 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_1_NEXT:%.*]], [[LOOP]] ]
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/induction-costs.ll b/llvm/test/Transforms/LoopVectorize/AArch64/induction-costs.ll
index f9cc195e36702..5084f4c48bd0e 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/induction-costs.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/induction-costs.ll
@@ -263,7 +263,7 @@ define void @wide_truncated_iv(ptr %dst) {
 ; CHECK:       vec.epilog.middle.block:
 ; CHECK-NEXT:    br i1 false, label [[EXIT]], label [[SCALAR_PH]]
 ; CHECK:       vec.epilog.scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL2:%.*]] = phi i64 [ 200, [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ 0, [[ITER_CHECK:%.*]] ], [ 192, [[VEC_EPILOG_ITER_CHECK]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL2:%.*]] = phi i64 [ 200, [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ 192, [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[ITER_CHECK:%.*]] ]
 ; CHECK-NEXT:    br label [[LOOP1:%.*]]
 ; CHECK:       loop:
 ; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL2]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP1]] ]
@@ -416,8 +416,8 @@ define void @zext_iv_increment(ptr %dst, i64 %N) {
 ; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[UMAX1]], [[N_VEC]]
 ; CHECK-NEXT:    br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ 0, [[VECTOR_SCEVCHECK]] ], [ 0, [[ENTRY:%.*]] ]
-; CHECK-NEXT:    [[BC_RESUME_VAL2:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[VECTOR_SCEVCHECK]] ], [ 0, [[ENTRY]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ], [ 0, [[VECTOR_SCEVCHECK]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL2:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ], [ 0, [[VECTOR_SCEVCHECK]] ]
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
 ; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/interleave-allocsize-not-equal-typesize.ll b/llvm/test/Transforms/LoopVectorize/AArch64/interleave-allocsize-not-equal-typesize.ll
index bd77f9779b680..79d7ab84b3a0f 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/interleave-allocsize-not-equal-typesize.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/interleave-allocsize-not-equal-typesize.ll
@@ -54,7 +54,7 @@ define void @pr58722_load_interleave_group(ptr %src, ptr %dst) {
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    br label [[SCALAR_PH]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 10000, [[MIDDLE_BLOCK]] ], [ 0, [[VECTOR_MEMCHECK]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 10000, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ], [ 0, [[VECTOR_MEMCHECK]] ]
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
 ; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/interleaving-load-store.ll b/llvm/test/Transforms/LoopVectorize/AArch64/interleaving-load-store.ll
index 24ff9c67f80e3..8320608d67588 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/interleaving-load-store.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/interleaving-load-store.ll
@@ -103,7 +103,7 @@ define void @interleave_single_load_store(ptr %src, ptr %dst, i64 %N, i8 %a, i8
 ; INTERLEAVE-4-NEXT:    [[CMP_N11:%.*]] = icmp eq i64 [[N]], [[N_VEC10]]
 ; INTERLEAVE-4-NEXT:    br i1 [[CMP_N11]], label [[EXIT]], label [[VEC_EPILOG_SCALAR_PH]]
 ; INTERLEAVE-4:       vec.epilog.scalar.ph:
-; INTERLEAVE-4-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC10]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ 0, [[VECTOR_MEMCHECK]] ], [ 0, [[ITER_CHECK:%.*]] ], [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ]
+; INTERLEAVE-4-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC10]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[VECTOR_MEMCHECK]] ], [ 0, [[ITER_CHECK:%.*]] ]
 ; INTERLEAVE-4-NEXT:    br label [[LOOP:%.*]]
 ; INTERLEAVE-4:       loop:
 ; INTERLEAVE-4-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[VEC_EPILOG_SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
@@ -198,7 +198,7 @@ define void @interleave_single_load_store(ptr %src, ptr %dst, i64 %N, i8 %a, i8
 ; INTERLEAVE-2-NEXT:    [[CMP_N9:%.*]] = icmp eq i64 [[N]], [[N_VEC8]]
 ; INTERLEAVE-2-NEXT:    br i1 [[CMP_N9]], label [[EXIT]], label [[VEC_EPILOG_SCALAR_PH]]
 ; INTERLEAVE-2:       vec.epilog.scalar.ph:
-; INTERLEAVE-2-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC8]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ 0, [[VECTOR_MEMCHECK]] ], [ 0, [[ITER_CHECK:%.*]] ], [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ]
+; INTERLEAVE-2-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC8]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[VECTOR_MEMCHECK]] ], [ 0, [[ITER_CHECK:%.*]] ]
 ; INTERLEAVE-2-NEXT:    br label [[LOOP:%.*]]
 ; INTERLEAVE-2:       loop:
 ; INTERLEAVE-2-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[VEC_EPILOG_SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/interleaving-reduction.ll b/llvm/test/Transforms/LoopVectorize/AArch64/interleaving-reduction.ll
index edb58e2d8f12f..07aa57d329c7d 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/interleaving-reduction.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/interleaving-reduction.ll
@@ -79,8 +79,8 @@ define i32 @interleave_integer_reduction(ptr %src, i64 %N) {
 ; INTERLEAVE-4-NEXT:    [[CMP_N16:%.*]] = icmp eq i64 [[N]], [[N_VEC11]]
 ; INTERLEAVE-4-NEXT:    br i1 [[CMP_N16]], label [[EXIT]], label [[VEC_EPILOG_SCALAR_PH]]
 ; INTERLEAVE-4:       vec.epilog.scalar.ph:
-; INTERLEAVE-4-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC11]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ 0, [[ITER_CHECK:%.*]] ], [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ]
-; INTERLEAVE-4-NEXT:    [[BC_MERGE_RDX17:%.*]] = phi i32 [ [[TMP24]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ 0, [[ITER_CHECK]] ], [ [[TMP17]], [[VEC_EPILOG_ITER_CHECK]] ]
+; INTERLEAVE-4-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC11]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[ITER_CHECK:%.*]] ]
+; INTERLEAVE-4-NEXT:    [[BC_MERGE_RDX17:%.*]] = phi i32 [ [[TMP24]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP17]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[ITER_CHECK]] ]
 ; INTERLEAVE-4-NEXT:    br label [[LOOP:%.*]]
 ; INTERLEAVE-4:       loop:
 ; INTERLEAVE-4-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[VEC_EPILOG_SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/intrinsiccost.ll b/llvm/test/Transforms/LoopVectorize/AArch64/intrinsiccost.ll
index 9dceb0167a4ac..1921e5f193aa3 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/intrinsiccost.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/intrinsiccost.ll
@@ -87,9 +87,9 @@ define void @saddsat(ptr nocapture readonly %pSrc, i16 signext %offset, ptr noca
 ; CHECK-NEXT:    [[CMP_N24:%.*]] = icmp eq i64 [[N_VEC6]], [[TMP0]]
 ; CHECK-NEXT:    br i1 [[CMP_N24]], label [[WHILE_END]], label [[VEC_EPILOG_SCALAR_PH]]
 ; CHECK:       vec.epilog.scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ [[IND_END]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[BLOCKSIZE]], [[ITER_CHECK]] ], [ [[IND_END8]], [[VEC_EPILOG_ITER_CHECK]] ]
-; CHECK-NEXT:    [[BC_RESUME_VAL11:%.*]] = phi ptr [ [[IND_END9]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[PSRC]], [[ITER_CHECK]] ], [ [[IND_END10]], [[VEC_EPILOG_ITER_CHECK]] ]
-; CHECK-NEXT:    [[BC_RESUME_VAL14:%.*]] = phi ptr [ [[IND_END12]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[PDST]], [[ITER_CHECK]] ], [ [[IND_END13]], [[VEC_EPILOG_ITER_CHECK]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ [[IND_END]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[IND_END8]], [[VEC_EPILOG_ITER_CHECK]] ], [ [[BLOCKSIZE]], [[ITER_CHECK]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL11:%.*]] = phi ptr [ [[IND_END9]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[IND_END10]], [[VEC_EPILOG_ITER_CHECK]] ], [ [[PSRC]], [[ITER_CHECK]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL14:%.*]] = phi ptr [ [[IND_END12]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[IND_END13]], [[VEC_EPILOG_ITER_CHECK]] ], [ [[PDST]], [[ITER_CHECK]] ]
 ; CHECK-NEXT:    br label [[WHILE_BODY:%.*]]
 ; CHECK:       while.body:
 ; CHECK-NEXT:    [[BLKCNT_09:%.*]] = phi i32 [ [[DEC:%.*]], [[WHILE_BODY]] ], [ [[BC_RESUME_VAL]], [[VEC_EPILOG_SCALAR_PH]] ]
@@ -202,9 +202,9 @@ define void @umin(ptr nocapture readonly %pSrc, i8 signext %offset, ptr nocaptur
 ; CHECK-NEXT:    [[CMP_N21:%.*]] = icmp eq i64 [[N_VEC5]], [[TMP0]]
 ; CHECK-NEXT:    br i1 [[CMP_N21]], label [[WHILE_END]], label [[VEC_EPILOG_SCALAR_PH]]
 ; CHECK:       vec.epilog.scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ [[IND_END]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[BLOCKSIZE]], [[ITER_CHECK]] ], [ [[IND_END7]], [[VEC_EPILOG_ITER_CHECK]] ]
-; CHECK-NEXT:    [[BC_RESUME_VAL10:%.*]] = phi ptr [ [[IND_END8]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[PSRC]], [[ITER_CHECK]] ], [ [[IND_END9]], [[VEC_EPILOG_ITER_CHECK]] ]
-; CHECK-NEXT:    [[BC_RESUME_VAL13:%.*]] = phi ptr [ [[IND_END11]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[PDST]], [[ITER_CHECK]] ], [ [[IND_END12]], [[VEC_EPILOG_ITER_CHECK]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ [[IND_END]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[IND_END7]], [[VEC_EPILOG_ITER_CHECK]] ], [ [[BLOCKSIZE]], [[ITER_CHECK]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL10:%.*]] = phi ptr [ [[IND_END8]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[IND_END9]], [[VEC_EPILOG_ITER_CHECK]] ], [ [[PSRC]], [[ITER_CHECK]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL13:%.*]] = phi ptr [ [[IND_END11]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[IND_END12]], [[VEC_EPILOG_ITER_CHECK]] ], [ [[PDST]], [[ITER_CHECK]] ]
 ; CHECK-NEXT:    br label [[WHILE_BODY:%.*]]
 ; CHECK:       while.body:
 ; CHECK-NEXT:    [[BLKCNT_09:%.*]] = phi i32 [ [[DEC:%.*]], [[WHILE_BODY]] ], [ [[BC_RESUME_VAL]], [[VEC_EPILOG_SCALAR_PH]] ]
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/invariant-replicate-region.ll b/llvm/test/Transforms/LoopVectorize/AArch64/invariant-replicate-region.ll
index e1472b7249ee6..26e96cab3e357 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/invariant-replicate-region.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/invariant-replicate-region.ll
@@ -16,32 +16,28 @@ define i32 @test_invariant_replicate_region(i32 %x, i1 %c) {
 ; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; CHECK:       [[VECTOR_BODY]]:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_UREM_CONTINUE6:.*]] ]
-; CHECK-NEXT:    [[TMP0:%.*]] = extractelement <4 x i1> [[BROADCAST_SPLAT]], i32 0
-; CHECK-NEXT:    br i1 [[TMP0]], label %[[PRED_UREM_IF:.*]], label %[[PRED_UREM_CONTINUE:.*]]
+; CHECK-NEXT:    br i1 [[C]], label %[[PRED_UREM_IF:.*]], label %[[PRED_UREM_CONTINUE:.*]]
 ; CHECK:       [[PRED_UREM_IF]]:
 ; CHECK-NEXT:    [[TMP1:%.*]] = urem i32 10, [[X]]
 ; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <4 x i32> poison, i32 [[TMP1]], i32 0
 ; CHECK-NEXT:    br label %[[PRED_UREM_CONTINUE]]
 ; CHECK:       [[PRED_UREM_CONTINUE]]:
 ; CHECK-NEXT:    [[TMP3:%.*]] = phi <4 x i32> [ poison, %[[VECTOR_BODY]] ], [ [[TMP2]], %[[PRED_UREM_IF]] ]
-; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <4 x i1> [[BROADCAST_SPLAT]], i32 1
-; CHECK-NEXT:    br i1 [[TMP4]], label %[[PRED_UREM_IF1:.*]], label %[[PRED_UREM_CONTINUE2:.*]]
+; CHECK-NEXT:    br i1 [[C]], label %[[PRED_UREM_IF1:.*]], label %[[PRED_UREM_CONTINUE2:.*]]
 ; CHECK:       [[PRED_UREM_IF1]]:
 ; CHECK-NEXT:    [[TMP5:%.*]] = urem i32 10, [[X]]
 ; CHECK-NEXT:    [[TMP6:%.*]] = insertelement <4 x i32> [[TMP3]], i32 [[TMP5]], i32 1
 ; CHECK-NEXT:    br label %[[PRED_UREM_CONTINUE2]]
 ; CHECK:       [[PRED_UREM_CONTINUE2]]:
 ; CHECK-NEXT:    [[TMP7:%.*]] = phi <4 x i32> [ [[TMP3]], %[[PRED_UREM_CONTINUE]] ], [ [[TMP6]], %[[PRED_UREM_IF1]] ]
-; CHECK-NEXT:    [[TMP8:%.*]] = extractelement <4 x i1> [[BROADCAST_SPLAT]], i32 2
-; CHECK-NEXT:    br i1 [[TMP8]], label %[[PRED_UREM_IF3:.*]], label %[[PRED_UREM_CONTINUE4:.*]]
+; CHECK-NEXT:    br i1 [[C]], label %[[PRED_UREM_IF3:.*]], label %[[PRED_UREM_CONTINUE4:.*]]
 ; CHECK:       [[PRED_UREM_IF3]]:
 ; CHECK-NEXT:    [[TMP9:%.*]] = urem i32 10, [[X]]
 ; CHECK-NEXT:    [[TMP10:%.*]] = insertelement <4 x i32> [[TMP7]], i32 [[TMP9]], i32 2
 ; CHECK-NEXT:    br label %[[PRED_UREM_CONTINUE4]]
 ; CHECK:       [[PRED_UREM_CONTINUE4]]:
 ; CHECK-NEXT:    [[TMP11:%.*]] = phi <4 x i32> [ [[TMP7]], %[[PRED_UREM_CONTINUE2]] ], [ [[TMP10]], %[[PRED_UREM_IF3]] ]
-; CHECK-NEXT:    [[TMP12:%.*]] = extractelement <4 x i1> [[BROADCAST_SPLAT]], i32 3
-; CHECK-NEXT:    br i1 [[TMP12]], label %[[PRED_UREM_IF5:.*]], label %[[PRED_UREM_CONTINUE6]]
+; CHECK-NEXT:    br i1 [[C]], label %[[PRED_UREM_IF5:.*]], label %[[PRED_UREM_CONTINUE6]]
 ; CHECK:       [[PRED_UREM_IF5]]:
 ; CHECK-NEXT:    [[TMP13:%.*]] = urem i32 10, [[X]]
 ; CHECK-NEXT:    [[TMP14:%.*]] = insertelement <4 x i32> [[TMP11]], i32 [[TMP13]], i32 3
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/loop-vectorization-factors.ll b/llvm/test/Transforms/LoopVectorize/AArch64/loop-vectorization-factors.ll
index 7f02792d4fe94..4c54796d9d16d 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/loop-vectorization-factors.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/loop-vectorization-factors.ll
@@ -6,7 +6,7 @@ target triple = "aarch64"
 
 define void @add_a(ptr noalias nocapture readonly %p, ptr noalias nocapture %q, i32 %len) #0 {
 ; CHECK-LABEL: define void @add_a
-; CHECK-SAME: (ptr noalias nocapture readonly [[P:%.*]], ptr noalias nocapture [[Q:%.*]], i32 [[LEN:%.*]]) {
+; CHECK-SAME: (ptr noalias readonly captures(none) [[P:%.*]], ptr noalias captures(none) [[Q:%.*]], i32 [[LEN:%.*]]) {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[CMP8:%.*]] = icmp sgt i32 [[LEN]], 0
 ; CHECK-NEXT:    br i1 [[CMP8]], label [[ITER_CHECK:%.*]], label [[FOR_COND_CLEANUP:%.*]]
@@ -63,7 +63,7 @@ define void @add_a(ptr noalias nocapture readonly %p, ptr noalias nocapture %q,
 ; CHECK-NEXT:    [[CMP_N7:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC3]]
 ; CHECK-NEXT:    br i1 [[CMP_N7]], label [[FOR_COND_CLEANUP_LOOPEXIT]], label [[VEC_EPILOG_SCALAR_PH]]
 ; CHECK:       vec.epilog.scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC3]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ 0, [[ITER_CHECK]] ], [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC3]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[ITER_CHECK]] ]
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK:       for.cond.cleanup.loopexit:
 ; CHECK-NEXT:    br label [[FOR_COND_CLEANUP]]
@@ -109,7 +109,7 @@ for.body:                                         ; preds = %entry, %for.body
 ; working with.
 define void @add_a1(ptr noalias nocapture readonly %p, ptr noalias nocapture %q, i32 %len) #0 {
 ; CHECK-LABEL: define void @add_a1
-; CHECK-SAME: (ptr noalias nocapture readonly [[P:%.*]], ptr noalias nocapture [[Q:%.*]], i32 [[LEN:%.*]]) {
+; CHECK-SAME: (ptr noalias readonly captures(none) [[P:%.*]], ptr noalias captures(none) [[Q:%.*]], i32 [[LEN:%.*]]) {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[CMP8:%.*]] = icmp sgt i32 [[LEN]], 0
 ; CHECK-NEXT:    br i1 [[CMP8]], label [[ITER_CHECK:%.*]], label [[FOR_COND_CLEANUP:%.*]]
@@ -166,7 +166,7 @@ define void @add_a1(ptr noalias nocapture readonly %p, ptr noalias nocapture %q,
 ; CHECK-NEXT:    [[CMP_N7:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC3]]
 ; CHECK-NEXT:    br i1 [[CMP_N7]], label [[FOR_COND_CLEANUP_LOOPEXIT]], label [[VEC_EPILOG_SCALAR_PH]]
 ; CHECK:       vec.epilog.scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC3]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ 0, [[ITER_CHECK]] ], [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC3]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[ITER_CHECK]] ]
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK:       for.cond.cleanup.loopexit:
 ; CHECK-NEXT:    br label [[FOR_COND_CLEANUP]]
@@ -206,7 +206,7 @@ for.body:                                         ; preds = %entry, %for.body
 
 define void @add_b(ptr noalias nocapture readonly %p, ptr noalias nocapture %q, i32 %len) #0 {
 ; CHECK-LABEL: define void @add_b
-; CHECK-SAME: (ptr noalias nocapture readonly [[P:%.*]], ptr noalias nocapture [[Q:%.*]], i32 [[LEN:%.*]]) {
+; CHECK-SAME: (ptr noalias readonly captures(none) [[P:%.*]], ptr noalias captures(none) [[Q:%.*]], i32 [[LEN:%.*]]) {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[CMP9:%.*]] = icmp sgt i32 [[LEN]], 0
 ; CHECK-NEXT:    br i1 [[CMP9]], label [[FOR_BODY_PREHEADER:%.*]], label [[FOR_COND_CLEANUP:%.*]]
@@ -279,7 +279,7 @@ for.body:                                         ; preds = %entry, %for.body
 
 define void @add_c(ptr noalias nocapture readonly %p, ptr noalias nocapture %q, i32 %len) #0 {
 ; CHECK-LABEL: define void @add_c
-; CHECK-SAME: (ptr noalias nocapture readonly [[P:%.*]], ptr noalias nocapture [[Q:%.*]], i32 [[LEN:%.*]]) {
+; CHECK-SAME: (ptr noalias readonly captures(none) [[P:%.*]], ptr noalias captures(none) [[Q:%.*]], i32 [[LEN:%.*]]) {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[CMP8:%.*]] = icmp sgt i32 [[LEN]], 0
 ; CHECK-NEXT:    br i1 [[CMP8]], label [[ITER_CHECK:%.*]], label [[FOR_COND_CLEANUP:%.*]]
@@ -338,7 +338,7 @@ define void @add_c(ptr noalias nocapture readonly %p, ptr noalias nocapture %q,
 ; CHECK-NEXT:    [[CMP_N7:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC3]]
 ; CHECK-NEXT:    br i1 [[CMP_N7]], label [[FOR_COND_CLEANUP_LOOPEXIT]], label [[VEC_EPILOG_SCALAR_PH]]
 ; CHECK:       vec.epilog.scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC3]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ 0, [[ITER_CHECK]] ], [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC3]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[ITER_CHECK]] ]
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK:       for.cond.cleanup.loopexit:
 ; CHECK-NEXT:    br label [[FOR_COND_CLEANUP]]
@@ -382,7 +382,7 @@ for.body:                                         ; preds = %entry, %for.body
 
 define void @add_d(ptr noalias nocapture readonly %p, ptr noalias nocapture %q, i32 %len) #0 {
 ; CHECK-LABEL: define void @add_d
-; CHECK-SAME: (ptr noalias nocapture readonly [[P:%.*]], ptr noalias nocapture [[Q:%.*]], i32 [[LEN:%.*]]) {
+; CHECK-SAME: (ptr noalias readonly captures(none) [[P:%.*]], ptr noalias captures(none) [[Q:%.*]], i32 [[LEN:%.*]]) {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[CMP7:%.*]] = icmp sgt i32 [[LEN]], 0
 ; CHECK-NEXT:    br i1 [[CMP7]], label [[FOR_BODY_PREHEADER:%.*]], label [[FOR_COND_CLEANUP:%.*]]
@@ -454,7 +454,7 @@ for.body:                                         ; preds = %entry, %for.body
 
 define void @add_e(ptr noalias nocapture readonly %p, ptr noalias nocapture %q, i8 %arg1, i8 %arg2, i32 %len) #0 {
 ; CHECK-LABEL: define void @add_e
-; CHECK-SAME: (ptr noalias nocapture readonly [[P:%.*]], ptr noalias nocapture [[Q:%.*]], i8 [[ARG1:%.*]], i8 [[ARG2:%.*]], i32 [[LEN:%.*]]) {
+; CHECK-SAME: (ptr noalias readonly captures(none) [[P:%.*]], ptr noalias captures(none) [[Q:%.*]], i8 [[ARG1:%.*]], i8 [[ARG2:%.*]], i32 [[LEN:%.*]]) {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[CMP_32:%.*]] = icmp sgt i32 [[LEN]], 0
 ; CHECK-NEXT:    br i1 [[CMP_32]], label [[ITER_CHECK:%.*]], label [[FOR_COND_CLEANUP:%.*]]
@@ -539,7 +539,7 @@ define void @add_e(ptr noalias nocapture readonly %p, ptr noalias nocapture %q,
 ; CHECK-NEXT:    [[CMP_N13:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC5]]
 ; CHECK-NEXT:    br i1 [[CMP_N13]], label [[FOR_COND_CLEANUP_LOOPEXIT]], label [[VEC_EPILOG_SCALAR_PH]]
 ; CHECK:       vec.epilog.scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC5]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ 0, [[ITER_CHECK]] ], [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC5]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[ITER_CHECK]] ]
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK:       for.cond.cleanup.loopexit:
 ; CHECK-NEXT:    br label [[FOR_COND_CLEANUP]]
@@ -602,7 +602,7 @@ for.body:                                         ; preds = %for.body, %for.body
 
 define void @add_f(ptr noalias nocapture readonly %p, ptr noalias nocapture %q, i8 %arg1, i8 %arg2, i32 %len) #0 {
 ; CHECK-LABEL: define void @add_f
-; CHECK-SAME: (ptr noalias nocapture readonly [[P:%.*]], ptr noalias nocapture [[Q:%.*]], i8 [[ARG1:%.*]], i8 [[ARG2:%.*]], i32 [[LEN:%.*]]) {
+; CHECK-SAME: (ptr noalias readonly captures(none) [[P:%.*]], ptr noalias captures(none) [[Q:%.*]], i8 [[ARG1:%.*]], i8 [[ARG2:%.*]], i32 [[LEN:%.*]]) {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[CMP_32:%.*]] = icmp sgt i32 [[LEN]], 0
 ; CHECK-NEXT:    br i1 [[CMP_32]], label [[ITER_CHECK:%.*]], label [[FOR_COND_CLEANUP:%.*]]
@@ -691,7 +691,7 @@ define void @add_f(ptr noalias nocapture readonly %p, ptr noalias nocapture %q,
 ; CHECK-NEXT:    [[CMP_N13:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC5]]
 ; CHECK-NEXT:    br i1 [[CMP_N13]], label [[FOR_COND_CLEANUP_LOOPEXIT]], label [[VEC_EPILOG_SCALAR_PH]]
 ; CHECK:       vec.epilog.scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC5]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ 0, [[ITER_CHECK]] ], [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC5]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[ITER_CHECK]] ]
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK:       for.cond.cleanup.loopexit:
 ; CHECK-NEXT:    br label [[FOR_COND_CLEANUP]]
@@ -756,7 +756,7 @@ for.body:                                         ; preds = %for.body, %for.body
 
 define void @add_phifail(ptr noalias nocapture readonly %p, ptr noalias nocapture %q, i32 %len) #0 {
 ; CHECK-LABEL: define void @add_phifail
-; CHECK-SAME: (ptr noalias nocapture readonly [[P:%.*]], ptr noalias nocapture [[Q:%.*]], i32 [[LEN:%.*]]) {
+; CHECK-SAME: (ptr noalias readonly captures(none) [[P:%.*]], ptr noalias captures(none) [[Q:%.*]], i32 [[LEN:%.*]]) {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[CMP8:%.*]] = icmp sgt i32 [[LEN]], 0
 ; CHECK-NEXT:    br i1 [[CMP8]], label [[FOR_BODY_PREHEADER:%.*]], label [[FOR_COND_CLEANUP:%.*]]
@@ -840,7 +840,7 @@ for.body:                                         ; preds = %entry, %for.body
 ; the correct value a_phi = p[len -2]
 define i8 @add_phifail2(ptr noalias nocapture readonly %p, ptr noalias nocapture %q, i32 %len) #0 {
 ; CHECK-LABEL: define i8 @add_phifail2
-; CHECK-SAME: (ptr noalias nocapture readonly [[P:%.*]], ptr noalias nocapture [[Q:%.*]], i32 [[LEN:%.*]]) {
+; CHECK-SAME: (ptr noalias readonly captures(none) [[P:%.*]], ptr noalias captures(none) [[Q:%.*]], i32 [[LEN:%.*]]) {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[TMP0:%.*]] = add i32 [[LEN]], -1
 ; CHECK-NEXT:    [[TMP1:%.*]] = zext i32 [[TMP0]] to i64
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/low_trip_count_predicates.ll b/llvm/test/Transforms/LoopVectorize/AArch64/low_trip_count_predicates.ll
index 6d57f212fd88c..9b7e41aa98db6 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/low_trip_count_predicates.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/low_trip_count_predicates.ll
@@ -37,7 +37,7 @@ target triple = "aarch64-unknown-linux-gnu"
 
 define void @low_vf_ic_is_better(ptr nocapture noundef %p, i32 %tc, i16 noundef %val) {
 ; CHECK-VS1-LABEL: define void @low_vf_ic_is_better(
-; CHECK-VS1-SAME: ptr nocapture noundef [[P:%.*]], i32 [[TC:%.*]], i16 noundef [[VAL:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-VS1-SAME: ptr noundef captures(none) [[P:%.*]], i32 [[TC:%.*]], i16 noundef [[VAL:%.*]]) #[[ATTR0:[0-9]+]] {
 ; CHECK-VS1-NEXT:  [[ENTRY:.*:]]
 ; CHECK-VS1-NEXT:    [[CMP7:%.*]] = icmp ult i32 [[TC]], 19
 ; CHECK-VS1-NEXT:    br i1 [[CMP7]], label %[[ITER_CHECK:.*]], label %[[WHILE_END:.*]]
@@ -127,7 +127,7 @@ define void @low_vf_ic_is_better(ptr nocapture noundef %p, i32 %tc, i16 noundef
 ; CHECK-VS1-NEXT:    [[CMP_N10:%.*]] = icmp eq i64 [[TMP3]], [[N_VEC3]]
 ; CHECK-VS1-NEXT:    br i1 [[CMP_N10]], label %[[WHILE_END_LOOPEXIT]], label %[[VEC_EPILOG_SCALAR_PH]]
 ; CHECK-VS1:       [[VEC_EPILOG_SCALAR_PH]]:
-; CHECK-VS1-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP39]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP0]], %[[VECTOR_SCEVCHECK]] ], [ [[TMP0]], %[[ITER_CHECK]] ], [ [[IND_END4]], %[[VEC_EPILOG_ITER_CHECK]] ]
+; CHECK-VS1-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP39]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[IND_END4]], %[[VEC_EPILOG_ITER_CHECK]] ], [ [[TMP0]], %[[VECTOR_SCEVCHECK]] ], [ [[TMP0]], %[[ITER_CHECK]] ]
 ; CHECK-VS1-NEXT:    br label %[[WHILE_BODY:.*]]
 ; CHECK-VS1:       [[WHILE_BODY]]:
 ; CHECK-VS1-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[VEC_EPILOG_SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[WHILE_BODY]] ]
@@ -145,7 +145,7 @@ define void @low_vf_ic_is_better(ptr nocapture noundef %p, i32 %tc, i16 noundef
 ; CHECK-VS1-NEXT:    ret void
 ;
 ; CHECK-VS2-LABEL: define void @low_vf_ic_is_better(
-; CHECK-VS2-SAME: ptr nocapture noundef [[P:%.*]], i32 [[TC:%.*]], i16 noundef [[VAL:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-VS2-SAME: ptr noundef captures(none) [[P:%.*]], i32 [[TC:%.*]], i16 noundef [[VAL:%.*]]) #[[ATTR0:[0-9]+]] {
 ; CHECK-VS2-NEXT:  [[ENTRY:.*:]]
 ; CHECK-VS2-NEXT:    [[CMP7:%.*]] = icmp ult i32 [[TC]], 19
 ; CHECK-VS2-NEXT:    br i1 [[CMP7]], label %[[ITER_CHECK:.*]], label %[[WHILE_END:.*]]
@@ -235,7 +235,7 @@ define void @low_vf_ic_is_better(ptr nocapture noundef %p, i32 %tc, i16 noundef
 ; CHECK-VS2-NEXT:    [[CMP_N10:%.*]] = icmp eq i64 [[TMP3]], [[N_VEC3]]
 ; CHECK-VS2-NEXT:    br i1 [[CMP_N10]], label %[[WHILE_END_LOOPEXIT]], label %[[VEC_EPILOG_SCALAR_PH]]
 ; CHECK-VS2:       [[VEC_EPILOG_SCALAR_PH]]:
-; CHECK-VS2-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP39]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP0]], %[[VECTOR_SCEVCHECK]] ], [ [[TMP0]], %[[ITER_CHECK]] ], [ [[IND_END4]], %[[VEC_EPILOG_ITER_CHECK]] ]
+; CHECK-VS2-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP39]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[IND_END4]], %[[VEC_EPILOG_ITER_CHECK]] ], [ [[TMP0]], %[[VECTOR_SCEVCHECK]] ], [ [[TMP0]], %[[ITER_CHECK]] ]
 ; CHECK-VS2-NEXT:    br label %[[WHILE_BODY:.*]]
 ; CHECK-VS2:       [[WHILE_BODY]]:
 ; CHECK-VS2-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[VEC_EPILOG_SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[WHILE_BODY]] ]
@@ -279,7 +279,7 @@ while.end:
 
 define void @trip_count_too_small(ptr nocapture noundef %p, i32 noundef %tc, i16 noundef %val) {
 ; CHECK-LABEL: define void @trip_count_too_small(
-; CHECK-SAME: ptr nocapture noundef [[P:%.*]], i32 noundef [[TC:%.*]], i16 noundef [[VAL:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-SAME: ptr noundef captures(none) [[P:%.*]], i32 noundef [[TC:%.*]], i16 noundef [[VAL:%.*]]) #[[ATTR0:[0-9]+]] {
 ; CHECK-NEXT:  [[ENTRY:.*:]]
 ; CHECK-NEXT:    [[CMP7:%.*]] = icmp ult i32 [[TC]], 3
 ; CHECK-NEXT:    br i1 [[CMP7]], label %[[WHILE_PREHEADER:.*]], label %[[WHILE_END:.*]]
@@ -330,7 +330,7 @@ while.end:
 
 define void @too_many_runtime_checks(ptr nocapture noundef %p, ptr nocapture noundef %p1, ptr nocapture noundef readonly %p2, ptr nocapture noundef readonly %p3, i32 noundef %tc, i16 noundef %val) {
 ; CHECK-LABEL: define void @too_many_runtime_checks(
-; CHECK-SAME: ptr nocapture noundef [[P:%.*]], ptr nocapture noundef [[P1:%.*]], ptr nocapture noundef readonly [[P2:%.*]], ptr nocapture noundef readonly [[P3:%.*]], i32 noundef [[TC:%.*]], i16 noundef [[VAL:%.*]]) #[[ATTR0]] {
+; CHECK-SAME: ptr noundef captures(none) [[P:%.*]], ptr noundef captures(none) [[P1:%.*]], ptr noundef readonly captures(none) [[P2:%.*]], ptr noundef readonly captures(none) [[P3:%.*]], i32 noundef [[TC:%.*]], i16 noundef [[VAL:%.*]]) #[[ATTR0]] {
 ; CHECK-NEXT:  [[ENTRY:.*:]]
 ; CHECK-NEXT:    [[CMP20:%.*]] = icmp ult i32 [[TC]], 16
 ; CHECK-NEXT:    br i1 [[CMP20]], label %[[WHILE_PREHEADER:.*]], label %[[WHILE_END:.*]]
@@ -399,7 +399,7 @@ while.end:
 
 define void @overflow_indvar_known_false(ptr nocapture noundef %p, i32 noundef %tc, i16 noundef %val) vscale_range(1,16) {
 ; CHECK-LABEL: define void @overflow_indvar_known_false(
-; CHECK-SAME: ptr nocapture noundef [[P:%.*]], i32 noundef [[TC:%.*]], i16 noundef [[VAL:%.*]]) #[[ATTR1:[0-9]+]] {
+; CHECK-SAME: ptr noundef captures(none) [[P:%.*]], i32 noundef [[TC:%.*]], i16 noundef [[VAL:%.*]]) #[[ATTR1:[0-9]+]] {
 ; CHECK-NEXT:  [[ENTRY:.*:]]
 ; CHECK-NEXT:    [[CMP7:%.*]] = icmp ult i32 [[TC]], 1027
 ; CHECK-NEXT:    br i1 [[CMP7]], label %[[WHILE_PREHEADER:.*]], label %[[WHILE_END:.*]]
@@ -453,7 +453,7 @@ define void @overflow_indvar_known_false(ptr nocapture noundef %p, i32 noundef %
 ; CHECK:       [[MIDDLE_BLOCK]]:
 ; CHECK-NEXT:    br i1 true, label %[[WHILE_END_LOOPEXIT:.*]], label %[[SCALAR_PH]]
 ; CHECK:       [[SCALAR_PH]]:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[IND_END]], %[[MIDDLE_BLOCK]] ], [ [[TMP0]], %[[VECTOR_SCEVCHECK]] ], [ [[TMP0]], %[[WHILE_PREHEADER]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[IND_END]], %[[MIDDLE_BLOCK]] ], [ [[TMP0]], %[[WHILE_PREHEADER]] ], [ [[TMP0]], %[[VECTOR_SCEVCHECK]] ]
 ; CHECK-NEXT:    br label %[[WHILE_BODY:.*]]
 ; CHECK:       [[WHILE_BODY]]:
 ; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], %[[WHILE_BODY]] ]
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/masked-call.ll b/llvm/test/Transforms/LoopVectorize/AArch64/masked-call.ll
index 61bbae0b3f16a..5b0f0961a6297 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/masked-call.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/masked-call.ll
@@ -39,7 +39,7 @@ define void @test_widen(ptr noalias %a, ptr readnone %b) #4 {
 ; TFNONE-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ]
 ; TFNONE-NEXT:    [[GEP:%.*]] = getelementptr i64, ptr [[B]], i64 [[INDVARS_IV]]
 ; TFNONE-NEXT:    [[LOAD:%.*]] = load i64, ptr [[GEP]], align 8
-; TFNONE-NEXT:    [[CALL:%.*]] = call i64 @foo(i64 [[LOAD]]) #[[ATTR3:[0-9]+]]
+; TFNONE-NEXT:    [[CALL:%.*]] = call i64 @foo(i64 [[LOAD]]) #[[ATTR4:[0-9]+]]
 ; TFNONE-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[INDVARS_IV]]
 ; TFNONE-NEXT:    store i64 [[CALL]], ptr [[ARRAYIDX]], align 8
 ; TFNONE-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
@@ -177,7 +177,7 @@ define void @test_if_then(ptr noalias %a, ptr readnone %b) #4 {
 ; TFNONE-NEXT:    [[CMP:%.*]] = icmp ugt i64 [[TMP12]], 50
 ; TFNONE-NEXT:    br i1 [[CMP]], label [[IF_THEN:%.*]], label [[IF_END]]
 ; TFNONE:       if.then:
-; TFNONE-NEXT:    [[TMP13:%.*]] = call i64 @foo(i64 [[TMP12]]) #[[ATTR3]]
+; TFNONE-NEXT:    [[TMP13:%.*]] = call i64 @foo(i64 [[TMP12]]) #[[ATTR4]]
 ; TFNONE-NEXT:    br label [[IF_END]]
 ; TFNONE:       if.end:
 ; TFNONE-NEXT:    [[TMP14:%.*]] = phi i64 [ [[TMP13]], [[IF_THEN]] ], [ 0, [[FOR_BODY]] ]
@@ -339,10 +339,10 @@ define void @test_widen_if_then_else(ptr noalias %a, ptr readnone %b) #4 {
 ; TFNONE-NEXT:    [[CMP:%.*]] = icmp ugt i64 [[TMP13]], 50
 ; TFNONE-NEXT:    br i1 [[CMP]], label [[IF_THEN:%.*]], label [[IF_ELSE:%.*]]
 ; TFNONE:       if.then:
-; TFNONE-NEXT:    [[TMP14:%.*]] = call i64 @foo(i64 [[TMP13]]) #[[ATTR4:[0-9]+]]
+; TFNONE-NEXT:    [[TMP14:%.*]] = call i64 @foo(i64 [[TMP13]]) #[[ATTR5:[0-9]+]]
 ; TFNONE-NEXT:    br label [[IF_END]]
 ; TFNONE:       if.else:
-; TFNONE-NEXT:    [[TMP15:%.*]] = call i64 @foo(i64 0) #[[ATTR4]]
+; TFNONE-NEXT:    [[TMP15:%.*]] = call i64 @foo(i64 0) #[[ATTR5]]
 ; TFNONE-NEXT:    br label [[IF_END]]
 ; TFNONE:       if.end:
 ; TFNONE-NEXT:    [[TMP16:%.*]] = phi i64 [ [[TMP14]], [[IF_THEN]] ], [ [[TMP15]], [[IF_ELSE]] ]
@@ -509,7 +509,7 @@ define void @test_widen_nomask(ptr noalias %a, ptr readnone %b) #4 {
 ; TFNONE-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ]
 ; TFNONE-NEXT:    [[GEP:%.*]] = getelementptr i64, ptr [[B]], i64 [[INDVARS_IV]]
 ; TFNONE-NEXT:    [[LOAD:%.*]] = load i64, ptr [[GEP]], align 8
-; TFNONE-NEXT:    [[CALL:%.*]] = call i64 @foo(i64 [[LOAD]]) #[[ATTR5:[0-9]+]]
+; TFNONE-NEXT:    [[CALL:%.*]] = call i64 @foo(i64 [[LOAD]]) #[[ATTR6:[0-9]+]]
 ; TFNONE-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[INDVARS_IV]]
 ; TFNONE-NEXT:    store i64 [[CALL]], ptr [[ARRAYIDX]], align 8
 ; TFNONE-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
@@ -525,7 +525,7 @@ define void @test_widen_nomask(ptr noalias %a, ptr readnone %b) #4 {
 ; TFALWAYS-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ]
 ; TFALWAYS-NEXT:    [[GEP:%.*]] = getelementptr i64, ptr [[B:%.*]], i64 [[INDVARS_IV]]
 ; TFALWAYS-NEXT:    [[LOAD:%.*]] = load i64, ptr [[GEP]], align 8
-; TFALWAYS-NEXT:    [[CALL:%.*]] = call i64 @foo(i64 [[LOAD]]) #[[ATTR5:[0-9]+]]
+; TFALWAYS-NEXT:    [[CALL:%.*]] = call i64 @foo(i64 [[LOAD]]) #[[ATTR6:[0-9]+]]
 ; TFALWAYS-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[INDVARS_IV]]
 ; TFALWAYS-NEXT:    store i64 [[CALL]], ptr [[ARRAYIDX]], align 8
 ; TFALWAYS-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
@@ -557,7 +557,7 @@ define void @test_widen_nomask(ptr noalias %a, ptr readnone %b) #4 {
 ; TFFALLBACK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ], [ [[N_VEC]], [[VECTOR_BODY]] ]
 ; TFFALLBACK-NEXT:    [[GEP:%.*]] = getelementptr i64, ptr [[B]], i64 [[INDVARS_IV]]
 ; TFFALLBACK-NEXT:    [[LOAD:%.*]] = load i64, ptr [[GEP]], align 8
-; TFFALLBACK-NEXT:    [[CALL:%.*]] = call i64 @foo(i64 [[LOAD]]) #[[ATTR5:[0-9]+]]
+; TFFALLBACK-NEXT:    [[CALL:%.*]] = call i64 @foo(i64 [[LOAD]]) #[[ATTR6:[0-9]+]]
 ; TFFALLBACK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[INDVARS_IV]]
 ; TFFALLBACK-NEXT:    store i64 [[CALL]], ptr [[ARRAYIDX]], align 8
 ; TFFALLBACK-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
@@ -573,7 +573,7 @@ define void @test_widen_nomask(ptr noalias %a, ptr readnone %b) #4 {
 ; TFA_INTERLEAVE-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ]
 ; TFA_INTERLEAVE-NEXT:    [[GEP:%.*]] = getelementptr i64, ptr [[B:%.*]], i64 [[INDVARS_IV]]
 ; TFA_INTERLEAVE-NEXT:    [[LOAD:%.*]] = load i64, ptr [[GEP]], align 8
-; TFA_INTERLEAVE-NEXT:    [[CALL:%.*]] = call i64 @foo(i64 [[LOAD]]) #[[ATTR5:[0-9]+]]
+; TFA_INTERLEAVE-NEXT:    [[CALL:%.*]] = call i64 @foo(i64 [[LOAD]]) #[[ATTR6:[0-9]+]]
 ; TFA_INTERLEAVE-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[INDVARS_IV]]
 ; TFA_INTERLEAVE-NEXT:    store i64 [[CALL]], ptr [[ARRAYIDX]], align 8
 ; TFA_INTERLEAVE-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
@@ -634,7 +634,7 @@ define void @test_widen_optmask(ptr noalias %a, ptr readnone %b) #4 {
 ; TFNONE-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ]
 ; TFNONE-NEXT:    [[GEP:%.*]] = getelementptr i64, ptr [[B]], i64 [[INDVARS_IV]]
 ; TFNONE-NEXT:    [[LOAD:%.*]] = load i64, ptr [[GEP]], align 8
-; TFNONE-NEXT:    [[CALL:%.*]] = call i64 @foo(i64 [[LOAD]]) #[[ATTR6:[0-9]+]]
+; TFNONE-NEXT:    [[CALL:%.*]] = call i64 @foo(i64 [[LOAD]]) #[[ATTR7:[0-9]+]]
 ; TFNONE-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[INDVARS_IV]]
 ; TFNONE-NEXT:    store i64 [[CALL]], ptr [[ARRAYIDX]], align 8
 ; TFNONE-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
@@ -806,7 +806,7 @@ define double @test_widen_fmuladd_and_call(ptr noalias %a, ptr readnone %b, doub
 ; TFNONE-NEXT:    [[LOAD:%.*]] = load double, ptr [[GEP]], align 8
 ; TFNONE-NEXT:    [[MULADD]] = tail call double @llvm.fmuladd.f64(double [[LOAD]], double [[M]], double [[FMA_SUM]])
 ; TFNONE-NEXT:    [[TOINT:%.*]] = fptoui double [[LOAD]] to i64
-; TFNONE-NEXT:    [[CALL:%.*]] = call i64 @foo(i64 [[TOINT]]) #[[ATTR3]]
+; TFNONE-NEXT:    [[CALL:%.*]] = call i64 @foo(i64 [[TOINT]]) #[[ATTR4]]
 ; TFNONE-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[INDVARS_IV]]
 ; TFNONE-NEXT:    store i64 [[CALL]], ptr [[ARRAYIDX]], align 8
 ; TFNONE-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
@@ -963,16 +963,266 @@ for.cond.cleanup:
   ret double %muladd
 }
 
+
+define void @test_widen_exp_v2(ptr noalias %p2, ptr noalias %p, i64 %n) #5 {
+; TFNONE-LABEL: @test_widen_exp_v2(
+; TFNONE-NEXT:  entry:
+; TFNONE-NEXT:    [[TMP0:%.*]] = add i64 [[N:%.*]], 1
+; TFNONE-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
+; TFNONE-NEXT:    [[TMP2:%.*]] = mul i64 [[TMP1]], 2
+; TFNONE-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], [[TMP2]]
+; TFNONE-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; TFNONE:       vector.ph:
+; TFNONE-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
+; TFNONE-NEXT:    [[TMP4:%.*]] = mul i64 [[TMP3]], 2
+; TFNONE-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[TMP0]], [[TMP4]]
+; TFNONE-NEXT:    [[N_VEC:%.*]] = sub i64 [[TMP0]], [[N_MOD_VF]]
+; TFNONE-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
+; TFNONE-NEXT:    [[TMP6:%.*]] = mul i64 [[TMP5]], 2
+; TFNONE-NEXT:    br label [[VECTOR_BODY:%.*]]
+; TFNONE:       vector.body:
+; TFNONE-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; TFNONE-NEXT:    [[TMP7:%.*]] = load double, ptr [[P2:%.*]], align 8
+; TFNONE-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 2 x double> poison, double [[TMP7]], i64 0
+; TFNONE-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 2 x double> [[BROADCAST_SPLATINSERT]], <vscale x 2 x double> poison, <vscale x 2 x i32> zeroinitializer
+; TFNONE-NEXT:    [[TMP8:%.*]] = call <vscale x 2 x double> @exp_masked_scalable(<vscale x 2 x double> [[BROADCAST_SPLAT]], <vscale x 2 x i1> splat (i1 true))
+; TFNONE-NEXT:    [[TMP9:%.*]] = fcmp ogt <vscale x 2 x double> [[TMP8]], zeroinitializer
+; TFNONE-NEXT:    [[TMP10:%.*]] = xor <vscale x 2 x i1> [[TMP9]], splat (i1 true)
+; TFNONE-NEXT:    [[PREDPHI:%.*]] = select <vscale x 2 x i1> [[TMP10]], <vscale x 2 x double> splat (double 1.000000e+00), <vscale x 2 x double> zeroinitializer
+; TFNONE-NEXT:    [[TMP11:%.*]] = call i32 @llvm.vscale.i32()
+; TFNONE-NEXT:    [[TMP12:%.*]] = mul i32 [[TMP11]], 2
+; TFNONE-NEXT:    [[TMP13:%.*]] = sub i32 [[TMP12]], 1
+; TFNONE-NEXT:    [[TMP14:%.*]] = extractelement <vscale x 2 x double> [[PREDPHI]], i32 [[TMP13]]
+; TFNONE-NEXT:    store double [[TMP14]], ptr [[P:%.*]], align 8
+; TFNONE-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP6]]
+; TFNONE-NEXT:    [[TMP15:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; TFNONE-NEXT:    br i1 [[TMP15]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
+; TFNONE:       middle.block:
+; TFNONE-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]]
+; TFNONE-NEXT:    br i1 [[CMP_N]], label [[END:%.*]], label [[SCALAR_PH]]
+; TFNONE:       scalar.ph:
+; TFNONE-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
+; TFNONE-NEXT:    br label [[LOOP:%.*]]
+; TFNONE:       loop:
+; TFNONE-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP_END:%.*]] ]
+; TFNONE-NEXT:    [[LD:%.*]] = load double, ptr [[P2]], align 8
+; TFNONE-NEXT:    [[EXP:%.*]] = tail call double @llvm.exp.f64(double [[LD]]) #[[ATTR8:[0-9]+]]
+; TFNONE-NEXT:    [[COND1:%.*]] = fcmp ogt double [[EXP]], 0.000000e+00
+; TFNONE-NEXT:    br i1 [[COND1]], label [[LOOP_MIDDLE:%.*]], label [[LOOP_END]]
+; TFNONE:       loop.middle:
+; TFNONE-NEXT:    br label [[LOOP_END]]
+; TFNONE:       loop.end:
+; TFNONE-NEXT:    [[SINK:%.*]] = phi double [ 0.000000e+00, [[LOOP_MIDDLE]] ], [ 1.000000e+00, [[LOOP]] ]
+; TFNONE-NEXT:    store double [[SINK]], ptr [[P]], align 8
+; TFNONE-NEXT:    [[IV_NEXT]] = add i64 [[IV]], 1
+; TFNONE-NEXT:    [[COND2:%.*]] = icmp eq i64 [[IV]], [[N]]
+; TFNONE-NEXT:    br i1 [[COND2]], label [[END]], label [[LOOP]], !llvm.loop [[LOOP15:![0-9]+]]
+; TFNONE:       end:
+; TFNONE-NEXT:    ret void
+;
+; TFALWAYS-LABEL: @test_widen_exp_v2(
+; TFALWAYS-NEXT:  entry:
+; TFALWAYS-NEXT:    [[TMP0:%.*]] = add i64 [[N:%.*]], 1
+; TFALWAYS-NEXT:    [[N_RND_UP:%.*]] = add i64 [[TMP0]], 1
+; TFALWAYS-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], 2
+; TFALWAYS-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
+; TFALWAYS-NEXT:    [[TMP1:%.*]] = sub i64 [[TMP0]], 2
+; TFALWAYS-NEXT:    [[TMP2:%.*]] = icmp ugt i64 [[TMP0]], 2
+; TFALWAYS-NEXT:    [[TMP3:%.*]] = select i1 [[TMP2]], i64 [[TMP1]], i64 0
+; TFALWAYS-NEXT:    [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <2 x i1> @llvm.get.active.lane.mask.v2i1.i64(i64 0, i64 [[TMP0]])
+; TFALWAYS-NEXT:    br label [[VECTOR_BODY:%.*]]
+; TFALWAYS:       vector.body:
+; TFALWAYS-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[INDEX_NEXT:%.*]], [[PRED_STORE_CONTINUE2:%.*]] ]
+; TFALWAYS-NEXT:    [[ACTIVE_LANE_MASK:%.*]] = phi <2 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], [[ENTRY]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], [[PRED_STORE_CONTINUE2]] ]
+; TFALWAYS-NEXT:    [[TMP4:%.*]] = load double, ptr [[P2:%.*]], align 8
+; TFALWAYS-NEXT:    [[TMP5:%.*]] = tail call double @llvm.exp.f64(double [[TMP4]]) #[[ATTR7:[0-9]+]]
+; TFALWAYS-NEXT:    [[TMP6:%.*]] = tail call double @llvm.exp.f64(double [[TMP4]]) #[[ATTR7]]
+; TFALWAYS-NEXT:    [[TMP7:%.*]] = insertelement <2 x double> poison, double [[TMP5]], i32 0
+; TFALWAYS-NEXT:    [[TMP8:%.*]] = insertelement <2 x double> [[TMP7]], double [[TMP6]], i32 1
+; TFALWAYS-NEXT:    [[TMP9:%.*]] = fcmp ogt <2 x double> [[TMP8]], zeroinitializer
+; TFALWAYS-NEXT:    [[TMP10:%.*]] = xor <2 x i1> [[TMP9]], splat (i1 true)
+; TFALWAYS-NEXT:    [[TMP11:%.*]] = select <2 x i1> [[ACTIVE_LANE_MASK]], <2 x i1> [[TMP10]], <2 x i1> zeroinitializer
+; TFALWAYS-NEXT:    [[PREDPHI:%.*]] = select <2 x i1> [[TMP11]], <2 x double> splat (double 1.000000e+00), <2 x double> zeroinitializer
+; TFALWAYS-NEXT:    [[TMP12:%.*]] = extractelement <2 x i1> [[ACTIVE_LANE_MASK]], i32 0
+; TFALWAYS-NEXT:    br i1 [[TMP12]], label [[PRED_STORE_IF:%.*]], label [[PRED_STORE_CONTINUE:%.*]]
+; TFALWAYS:       pred.store.if:
+; TFALWAYS-NEXT:    [[TMP13:%.*]] = extractelement <2 x double> [[PREDPHI]], i32 0
+; TFALWAYS-NEXT:    store double [[TMP13]], ptr [[P:%.*]], align 8
+; TFALWAYS-NEXT:    br label [[PRED_STORE_CONTINUE]]
+; TFALWAYS:       pred.store.continue:
+; TFALWAYS-NEXT:    [[TMP14:%.*]] = extractelement <2 x i1> [[ACTIVE_LANE_MASK]], i32 1
+; TFALWAYS-NEXT:    br i1 [[TMP14]], label [[PRED_STORE_IF1:%.*]], label [[PRED_STORE_CONTINUE2]]
+; TFALWAYS:       pred.store.if1:
+; TFALWAYS-NEXT:    [[TMP15:%.*]] = extractelement <2 x double> [[PREDPHI]], i32 1
+; TFALWAYS-NEXT:    store double [[TMP15]], ptr [[P]], align 8
+; TFALWAYS-NEXT:    br label [[PRED_STORE_CONTINUE2]]
+; TFALWAYS:       pred.store.continue2:
+; TFALWAYS-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 2
+; TFALWAYS-NEXT:    [[ACTIVE_LANE_MASK_NEXT]] = call <2 x i1> @llvm.get.active.lane.mask.v2i1.i64(i64 [[INDEX]], i64 [[TMP3]])
+; TFALWAYS-NEXT:    [[TMP16:%.*]] = xor <2 x i1> [[ACTIVE_LANE_MASK_NEXT]], splat (i1 true)
+; TFALWAYS-NEXT:    [[TMP17:%.*]] = extractelement <2 x i1> [[TMP16]], i32 0
+; TFALWAYS-NEXT:    br i1 [[TMP17]], label [[END:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
+; TFALWAYS:       end:
+; TFALWAYS-NEXT:    ret void
+;
+; TFFALLBACK-LABEL: @test_widen_exp_v2(
+; TFFALLBACK-NEXT:  entry:
+; TFFALLBACK-NEXT:    [[TMP0:%.*]] = add i64 [[N:%.*]], 1
+; TFFALLBACK-NEXT:    [[N_RND_UP:%.*]] = add i64 [[TMP0]], 1
+; TFFALLBACK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], 2
+; TFFALLBACK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
+; TFFALLBACK-NEXT:    [[TMP1:%.*]] = sub i64 [[TMP0]], 2
+; TFFALLBACK-NEXT:    [[TMP2:%.*]] = icmp ugt i64 [[TMP0]], 2
+; TFFALLBACK-NEXT:    [[TMP3:%.*]] = select i1 [[TMP2]], i64 [[TMP1]], i64 0
+; TFFALLBACK-NEXT:    [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <2 x i1> @llvm.get.active.lane.mask.v2i1.i64(i64 0, i64 [[TMP0]])
+; TFFALLBACK-NEXT:    br label [[VECTOR_BODY:%.*]]
+; TFFALLBACK:       vector.body:
+; TFFALLBACK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[INDEX_NEXT:%.*]], [[PRED_STORE_CONTINUE2:%.*]] ]
+; TFFALLBACK-NEXT:    [[ACTIVE_LANE_MASK:%.*]] = phi <2 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], [[ENTRY]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], [[PRED_STORE_CONTINUE2]] ]
+; TFFALLBACK-NEXT:    [[TMP4:%.*]] = load double, ptr [[P2:%.*]], align 8
+; TFFALLBACK-NEXT:    [[TMP5:%.*]] = tail call double @llvm.exp.f64(double [[TMP4]]) #[[ATTR7:[0-9]+]]
+; TFFALLBACK-NEXT:    [[TMP6:%.*]] = tail call double @llvm.exp.f64(double [[TMP4]]) #[[ATTR7]]
+; TFFALLBACK-NEXT:    [[TMP7:%.*]] = insertelement <2 x double> poison, double [[TMP5]], i32 0
+; TFFALLBACK-NEXT:    [[TMP8:%.*]] = insertelement <2 x double> [[TMP7]], double [[TMP6]], i32 1
+; TFFALLBACK-NEXT:    [[TMP9:%.*]] = fcmp ogt <2 x double> [[TMP8]], zeroinitializer
+; TFFALLBACK-NEXT:    [[TMP10:%.*]] = xor <2 x i1> [[TMP9]], splat (i1 true)
+; TFFALLBACK-NEXT:    [[TMP11:%.*]] = select <2 x i1> [[ACTIVE_LANE_MASK]], <2 x i1> [[TMP10]], <2 x i1> zeroinitializer
+; TFFALLBACK-NEXT:    [[PREDPHI:%.*]] = select <2 x i1> [[TMP11]], <2 x double> splat (double 1.000000e+00), <2 x double> zeroinitializer
+; TFFALLBACK-NEXT:    [[TMP12:%.*]] = extractelement <2 x i1> [[ACTIVE_LANE_MASK]], i32 0
+; TFFALLBACK-NEXT:    br i1 [[TMP12]], label [[PRED_STORE_IF:%.*]], label [[PRED_STORE_CONTINUE:%.*]]
+; TFFALLBACK:       pred.store.if:
+; TFFALLBACK-NEXT:    [[TMP13:%.*]] = extractelement <2 x double> [[PREDPHI]], i32 0
+; TFFALLBACK-NEXT:    store double [[TMP13]], ptr [[P:%.*]], align 8
+; TFFALLBACK-NEXT:    br label [[PRED_STORE_CONTINUE]]
+; TFFALLBACK:       pred.store.continue:
+; TFFALLBACK-NEXT:    [[TMP14:%.*]] = extractelement <2 x i1> [[ACTIVE_LANE_MASK]], i32 1
+; TFFALLBACK-NEXT:    br i1 [[TMP14]], label [[PRED_STORE_IF1:%.*]], label [[PRED_STORE_CONTINUE2]]
+; TFFALLBACK:       pred.store.if1:
+; TFFALLBACK-NEXT:    [[TMP15:%.*]] = extractelement <2 x double> [[PREDPHI]], i32 1
+; TFFALLBACK-NEXT:    store double [[TMP15]], ptr [[P]], align 8
+; TFFALLBACK-NEXT:    br label [[PRED_STORE_CONTINUE2]]
+; TFFALLBACK:       pred.store.continue2:
+; TFFALLBACK-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 2
+; TFFALLBACK-NEXT:    [[ACTIVE_LANE_MASK_NEXT]] = call <2 x i1> @llvm.get.active.lane.mask.v2i1.i64(i64 [[INDEX]], i64 [[TMP3]])
+; TFFALLBACK-NEXT:    [[TMP16:%.*]] = xor <2 x i1> [[ACTIVE_LANE_MASK_NEXT]], splat (i1 true)
+; TFFALLBACK-NEXT:    [[TMP17:%.*]] = extractelement <2 x i1> [[TMP16]], i32 0
+; TFFALLBACK-NEXT:    br i1 [[TMP17]], label [[END:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
+; TFFALLBACK:       end:
+; TFFALLBACK-NEXT:    ret void
+;
+; TFA_INTERLEAVE-LABEL: @test_widen_exp_v2(
+; TFA_INTERLEAVE-NEXT:  entry:
+; TFA_INTERLEAVE-NEXT:    [[TMP0:%.*]] = add i64 [[N:%.*]], 1
+; TFA_INTERLEAVE-NEXT:    [[N_RND_UP:%.*]] = add i64 [[TMP0]], 3
+; TFA_INTERLEAVE-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], 4
+; TFA_INTERLEAVE-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
+; TFA_INTERLEAVE-NEXT:    [[TMP1:%.*]] = sub i64 [[TMP0]], 4
+; TFA_INTERLEAVE-NEXT:    [[TMP2:%.*]] = icmp ugt i64 [[TMP0]], 4
+; TFA_INTERLEAVE-NEXT:    [[TMP3:%.*]] = select i1 [[TMP2]], i64 [[TMP1]], i64 0
+; TFA_INTERLEAVE-NEXT:    [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <2 x i1> @llvm.get.active.lane.mask.v2i1.i64(i64 0, i64 [[TMP0]])
+; TFA_INTERLEAVE-NEXT:    [[ACTIVE_LANE_MASK_ENTRY1:%.*]] = call <2 x i1> @llvm.get.active.lane.mask.v2i1.i64(i64 2, i64 [[TMP0]])
+; TFA_INTERLEAVE-NEXT:    br label [[VECTOR_BODY:%.*]]
+; TFA_INTERLEAVE:       vector.body:
+; TFA_INTERLEAVE-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[INDEX_NEXT:%.*]], [[PRED_STORE_CONTINUE9:%.*]] ]
+; TFA_INTERLEAVE-NEXT:    [[ACTIVE_LANE_MASK:%.*]] = phi <2 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], [[ENTRY]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], [[PRED_STORE_CONTINUE9]] ]
+; TFA_INTERLEAVE-NEXT:    [[ACTIVE_LANE_MASK2:%.*]] = phi <2 x i1> [ [[ACTIVE_LANE_MASK_ENTRY1]], [[ENTRY]] ], [ [[ACTIVE_LANE_MASK_NEXT10:%.*]], [[PRED_STORE_CONTINUE9]] ]
+; TFA_INTERLEAVE-NEXT:    [[TMP4:%.*]] = load double, ptr [[P2:%.*]], align 8
+; TFA_INTERLEAVE-NEXT:    [[TMP5:%.*]] = tail call double @llvm.exp.f64(double [[TMP4]]) #[[ATTR7:[0-9]+]]
+; TFA_INTERLEAVE-NEXT:    [[TMP6:%.*]] = tail call double @llvm.exp.f64(double [[TMP4]]) #[[ATTR7]]
+; TFA_INTERLEAVE-NEXT:    [[TMP7:%.*]] = insertelement <2 x double> poison, double [[TMP5]], i32 0
+; TFA_INTERLEAVE-NEXT:    [[TMP8:%.*]] = insertelement <2 x double> [[TMP7]], double [[TMP6]], i32 1
+; TFA_INTERLEAVE-NEXT:    [[TMP9:%.*]] = tail call double @llvm.exp.f64(double [[TMP4]]) #[[ATTR7]]
+; TFA_INTERLEAVE-NEXT:    [[TMP10:%.*]] = tail call double @llvm.exp.f64(double [[TMP4]]) #[[ATTR7]]
+; TFA_INTERLEAVE-NEXT:    [[TMP11:%.*]] = insertelement <2 x double> poison, double [[TMP9]], i32 0
+; TFA_INTERLEAVE-NEXT:    [[TMP12:%.*]] = insertelement <2 x double> [[TMP11]], double [[TMP10]], i32 1
+; TFA_INTERLEAVE-NEXT:    [[TMP13:%.*]] = fcmp ogt <2 x double> [[TMP8]], zeroinitializer
+; TFA_INTERLEAVE-NEXT:    [[TMP14:%.*]] = fcmp ogt <2 x double> [[TMP12]], zeroinitializer
+; TFA_INTERLEAVE-NEXT:    [[TMP15:%.*]] = xor <2 x i1> [[TMP13]], splat (i1 true)
+; TFA_INTERLEAVE-NEXT:    [[TMP16:%.*]] = xor <2 x i1> [[TMP14]], splat (i1 true)
+; TFA_INTERLEAVE-NEXT:    [[TMP17:%.*]] = select <2 x i1> [[ACTIVE_LANE_MASK]], <2 x i1> [[TMP15]], <2 x i1> zeroinitializer
+; TFA_INTERLEAVE-NEXT:    [[TMP18:%.*]] = select <2 x i1> [[ACTIVE_LANE_MASK2]], <2 x i1> [[TMP16]], <2 x i1> zeroinitializer
+; TFA_INTERLEAVE-NEXT:    [[PREDPHI:%.*]] = select <2 x i1> [[TMP17]], <2 x double> splat (double 1.000000e+00), <2 x double> zeroinitializer
+; TFA_INTERLEAVE-NEXT:    [[PREDPHI3:%.*]] = select <2 x i1> [[TMP18]], <2 x double> splat (double 1.000000e+00), <2 x double> zeroinitializer
+; TFA_INTERLEAVE-NEXT:    [[TMP19:%.*]] = extractelement <2 x i1> [[ACTIVE_LANE_MASK]], i32 0
+; TFA_INTERLEAVE-NEXT:    br i1 [[TMP19]], label [[PRED_STORE_IF:%.*]], label [[PRED_STORE_CONTINUE:%.*]]
+; TFA_INTERLEAVE:       pred.store.if:
+; TFA_INTERLEAVE-NEXT:    [[TMP20:%.*]] = extractelement <2 x double> [[PREDPHI]], i32 0
+; TFA_INTERLEAVE-NEXT:    store double [[TMP20]], ptr [[P:%.*]], align 8
+; TFA_INTERLEAVE-NEXT:    br label [[PRED_STORE_CONTINUE]]
+; TFA_INTERLEAVE:       pred.store.continue:
+; TFA_INTERLEAVE-NEXT:    [[TMP21:%.*]] = extractelement <2 x i1> [[ACTIVE_LANE_MASK]], i32 1
+; TFA_INTERLEAVE-NEXT:    br i1 [[TMP21]], label [[PRED_STORE_IF4:%.*]], label [[PRED_STORE_CONTINUE5:%.*]]
+; TFA_INTERLEAVE:       pred.store.if4:
+; TFA_INTERLEAVE-NEXT:    [[TMP22:%.*]] = extractelement <2 x double> [[PREDPHI]], i32 1
+; TFA_INTERLEAVE-NEXT:    store double [[TMP22]], ptr [[P]], align 8
+; TFA_INTERLEAVE-NEXT:    br label [[PRED_STORE_CONTINUE5]]
+; TFA_INTERLEAVE:       pred.store.continue5:
+; TFA_INTERLEAVE-NEXT:    [[TMP23:%.*]] = extractelement <2 x i1> [[ACTIVE_LANE_MASK2]], i32 0
+; TFA_INTERLEAVE-NEXT:    br i1 [[TMP23]], label [[PRED_STORE_IF6:%.*]], label [[PRED_STORE_CONTINUE7:%.*]]
+; TFA_INTERLEAVE:       pred.store.if6:
+; TFA_INTERLEAVE-NEXT:    [[TMP24:%.*]] = extractelement <2 x double> [[PREDPHI3]], i32 0
+; TFA_INTERLEAVE-NEXT:    store double [[TMP24]], ptr [[P]], align 8
+; TFA_INTERLEAVE-NEXT:    br label [[PRED_STORE_CONTINUE7]]
+; TFA_INTERLEAVE:       pred.store.continue7:
+; TFA_INTERLEAVE-NEXT:    [[TMP25:%.*]] = extractelement <2 x i1> [[ACTIVE_LANE_MASK2]], i32 1
+; TFA_INTERLEAVE-NEXT:    br i1 [[TMP25]], label [[PRED_STORE_IF8:%.*]], label [[PRED_STORE_CONTINUE9]]
+; TFA_INTERLEAVE:       pred.store.if8:
+; TFA_INTERLEAVE-NEXT:    [[TMP26:%.*]] = extractelement <2 x double> [[PREDPHI3]], i32 1
+; TFA_INTERLEAVE-NEXT:    store double [[TMP26]], ptr [[P]], align 8
+; TFA_INTERLEAVE-NEXT:    br label [[PRED_STORE_CONTINUE9]]
+; TFA_INTERLEAVE:       pred.store.continue9:
+; TFA_INTERLEAVE-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 4
+; TFA_INTERLEAVE-NEXT:    [[TMP27:%.*]] = add i64 [[INDEX]], 2
+; TFA_INTERLEAVE-NEXT:    [[ACTIVE_LANE_MASK_NEXT]] = call <2 x i1> @llvm.get.active.lane.mask.v2i1.i64(i64 [[INDEX]], i64 [[TMP3]])
+; TFA_INTERLEAVE-NEXT:    [[ACTIVE_LANE_MASK_NEXT10]] = call <2 x i1> @llvm.get.active.lane.mask.v2i1.i64(i64 [[TMP27]], i64 [[TMP3]])
+; TFA_INTERLEAVE-NEXT:    [[TMP28:%.*]] = xor <2 x i1> [[ACTIVE_LANE_MASK_NEXT]], splat (i1 true)
+; TFA_INTERLEAVE-NEXT:    [[TMP29:%.*]] = extractelement <2 x i1> [[TMP28]], i32 0
+; TFA_INTERLEAVE-NEXT:    br i1 [[TMP29]], label [[END:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
+; TFA_INTERLEAVE:       end:
+; TFA_INTERLEAVE-NEXT:    ret void
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop.end ]
+  %ld = load double, ptr %p2, align 8
+  %exp = tail call double @llvm.exp.f64(double %ld) #6
+  %cond1 = fcmp ogt double %exp, 0.000000e+00
+  br i1 %cond1, label %loop.middle, label %loop.end
+
+loop.middle:
+  br label %loop.end
+
+loop.end:
+  %sink = phi double [ 0.000000e+00, %loop.middle ], [ 1.000000e+00, %loop ]
+  store double %sink, ptr %p, align 8
+  %iv.next = add i64 %iv, 1
+  %cond2 = icmp eq i64 %iv, %n
+  br i1 %cond2, label %end, label %loop
+
+end:
+  ret void
+}
+
+
 declare i64 @foo(i64)
 declare double @llvm.fmuladd.f64(double, double, double)
+declare double @llvm.exp.f64(double)
+
+; fixed-width variants of exp
+declare <2 x double> @exp_fixed(<2 x double>)
 
-;; scalable vector variants of foo
+;; scalable vector variants of foo and exp
 declare <vscale x 2 x i64> @foo_uniform(i64, <vscale x 2 x i1>)
 declare <vscale x 2 x i64> @foo_vector(<vscale x 2 x i64>, <vscale x 2 x i1>)
 declare <vscale x 2 x i64> @foo_vector_nomask(<vscale x 2 x i64>)
+declare <vscale x 2 x double> @exp_masked_scalable(<vscale x 2 x double>, <vscale x 2 x i1>)
 
 attributes #0 = { nounwind "vector-function-abi-variant"="_ZGVsMxv_foo(foo_vector),_ZGVsMxu_foo(foo_uniform)" }
 attributes #1 = { nounwind "vector-function-abi-variant"="_ZGVsMxv_foo(foo_vector)" }
 attributes #2 = { nounwind "vector-function-abi-variant"="_ZGVsNxv_foo(foo_vector_nomask)" }
 attributes #3 = { nounwind "vector-function-abi-variant"="_ZGVsNxv_foo(foo_vector_nomask),_ZGVsMxv_foo(foo_vector)" }
 attributes #4 = { "target-features"="+sve" vscale_range(2,16) "no-trapping-math"="false" }
+attributes #5 = { "target-cpu"="neoverse-v2" vscale_range(1,16) }
+attributes #6 = { "vector-function-abi-variant"="_ZGV_LLVM_N2v_llvm.exp.f64(exp_fixed),_ZGVsMxv_llvm.exp.f64(exp_masked_scalable)" }
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/mul-simplification.ll b/llvm/test/Transforms/LoopVectorize/AArch64/mul-simplification.ll
index 6b55f5291efd8..a8d44421a3c37 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/mul-simplification.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/mul-simplification.ll
@@ -7,24 +7,10 @@ target triple = "arm64-apple-macosx"
 define i64 @mul_select_operand_known_1_via_scev() {
 ; CHECK-LABEL: define i64 @mul_select_operand_known_1_via_scev() {
 ; CHECK-NEXT:  [[ENTRY:.*]]:
-; CHECK-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
-; CHECK:       [[VECTOR_PH]]:
-; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
-; CHECK:       [[VECTOR_BODY]]:
-; CHECK-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <2 x i64> [ <i64 12, i64 1>, %[[VECTOR_PH]] ], [ [[VEC_PHI]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2
-; CHECK-NEXT:    br i1 true, label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
-; CHECK:       [[MIDDLE_BLOCK]]:
-; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vector.reduce.mul.v2i64(<2 x i64> [[VEC_PHI]])
-; CHECK-NEXT:    br i1 true, label %[[EXIT:.*]], label %[[SCALAR_PH]]
-; CHECK:       [[SCALAR_PH]]:
-; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i64 [ [[TMP0]], %[[MIDDLE_BLOCK]] ], [ 12, %[[ENTRY]] ]
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ 2, %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
 ; CHECK-NEXT:    br label %[[LOOP:.*]]
 ; CHECK:       [[LOOP]]:
-; CHECK-NEXT:    [[RED:%.*]] = phi i64 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[RED_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[RED:%.*]] = phi i64 [ 12, %[[ENTRY]] ], [ [[RED_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
 ; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[IV]], 1
 ; CHECK-NEXT:    [[CMP1_I:%.*]] = icmp eq i32 [[TMP1]], 0
 ; CHECK-NEXT:    [[NARROW_I:%.*]] = select i1 [[CMP1_I]], i32 1, i32 [[IV]]
@@ -32,9 +18,9 @@ define i64 @mul_select_operand_known_1_via_scev() {
 ; CHECK-NEXT:    [[RED_NEXT]] = mul nsw i64 [[RED]], [[MUL]]
 ; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i32 [[IV]], 1
 ; CHECK-NEXT:    [[EC:%.*]] = icmp eq i32 [[IV]], 1
-; CHECK-NEXT:    br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK-NEXT:    br i1 [[EC]], label %[[EXIT:.*]], label %[[LOOP]]
 ; CHECK:       [[EXIT]]:
-; CHECK-NEXT:    [[RES:%.*]] = phi i64 [ [[RED_NEXT]], %[[LOOP]] ], [ [[TMP0]], %[[MIDDLE_BLOCK]] ]
+; CHECK-NEXT:    [[RES:%.*]] = phi i64 [ [[RED_NEXT]], %[[LOOP]] ]
 ; CHECK-NEXT:    ret i64 [[RES]]
 ;
 entry:
@@ -65,17 +51,20 @@ define i32 @add_reduction_select_operand_constant_but_non_uniform() {
 ; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; CHECK:       [[VECTOR_BODY]]:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <4 x i32> [ <i32 42, i32 0, i32 0, i32 0>, %[[VECTOR_PH]] ], [ splat (i32 42), %[[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[VEC_PHI1:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ splat (i32 42), %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <4 x i32> [ <i32 42, i32 0, i32 0, i32 0>, %[[VECTOR_PH]] ], [ [[TMP2:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_PHI1:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP1:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP2]] = add <4 x i32> zeroinitializer, [[VEC_PHI]]
+; CHECK-NEXT:    [[TMP1]] = add <4 x i32> zeroinitializer, [[VEC_PHI1]]
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 8
 ; CHECK-NEXT:    [[TMP0:%.*]] = icmp eq i32 [[INDEX_NEXT]], 64
-; CHECK-NEXT:    br i1 [[TMP0]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK-NEXT:    br i1 [[TMP0]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; CHECK:       [[MIDDLE_BLOCK]]:
-; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> splat (i32 84))
+; CHECK-NEXT:    [[BIN_RDX:%.*]] = add <4 x i32> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP3:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[BIN_RDX]])
 ; CHECK-NEXT:    br i1 true, label %[[EXIT:.*]], label %[[SCALAR_PH]]
 ; CHECK:       [[SCALAR_PH]]:
 ; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ 64, %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
-; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP1]], %[[MIDDLE_BLOCK]] ], [ 42, %[[ENTRY]] ]
+; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP3]], %[[MIDDLE_BLOCK]] ], [ 42, %[[ENTRY]] ]
 ; CHECK-NEXT:    br label %[[LOOP:.*]]
 ; CHECK:       [[LOOP]]:
 ; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[ADD2_REASS:%.*]], %[[LOOP]] ]
@@ -83,9 +72,9 @@ define i32 @add_reduction_select_operand_constant_but_non_uniform() {
 ; CHECK-NEXT:    [[ADD2_REASS]] = add i32 [[IV]], 1
 ; CHECK-NEXT:    [[RDX_NEXT]] = add i32 0, [[RDX]]
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i32 [[ADD2_REASS]], 64
-; CHECK-NEXT:    br i1 [[CMP]], label %[[LOOP]], label %[[EXIT]], !llvm.loop [[LOOP5:![0-9]+]]
+; CHECK-NEXT:    br i1 [[CMP]], label %[[LOOP]], label %[[EXIT]], !llvm.loop [[LOOP3:![0-9]+]]
 ; CHECK:       [[EXIT]]:
-; CHECK-NEXT:    [[ADD_LCSSA:%.*]] = phi i32 [ [[RDX_NEXT]], %[[LOOP]] ], [ [[TMP1]], %[[MIDDLE_BLOCK]] ]
+; CHECK-NEXT:    [[ADD_LCSSA:%.*]] = phi i32 [ [[RDX_NEXT]], %[[LOOP]] ], [ [[TMP3]], %[[MIDDLE_BLOCK]] ]
 ; CHECK-NEXT:    ret i32 [[ADD_LCSSA]]
 ;
 entry:
@@ -109,6 +98,4 @@ exit:
 ; CHECK: [[META1]] = !{!"llvm.loop.isvectorized", i32 1}
 ; CHECK: [[META2]] = !{!"llvm.loop.unroll.runtime.disable"}
 ; CHECK: [[LOOP3]] = distinct !{[[LOOP3]], [[META2]], [[META1]]}
-; CHECK: [[LOOP4]] = distinct !{[[LOOP4]], [[META1]], [[META2]]}
-; CHECK: [[LOOP5]] = distinct !{[[LOOP5]], [[META2]], [[META1]]}
 ;.
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/no_vector_instructions.ll b/llvm/test/Transforms/LoopVectorize/AArch64/no_vector_instructions.ll
index 089d279d15245..ebc1f719203de 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/no_vector_instructions.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/no_vector_instructions.ll
@@ -1,5 +1,5 @@
 ; REQUIRES: asserts
-; RUN: opt < %s -passes=loop-vectorize -force-vector-interleave=1 -S -debug-only=loop-vectorize 2>&1 | FileCheck %s
+; RUN: opt < %s -passes=loop-vectorize -force-vector-interleave=1 -S -debug-only=loop-vectorize --disable-output 2>&1 | FileCheck %s
 
 target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
 target triple = "aarch64--linux-gnu"
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-chained.ll b/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-chained.ll
new file mode 100644
index 0000000000000..bedf8b6b3a9b5
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-chained.ll
@@ -0,0 +1,1025 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 4
+; RUN: opt --mattr=+neon,+dotprod -passes=loop-vectorize -force-vector-interleave=1 -enable-epilogue-vectorization=false -S < %s | FileCheck %s --check-prefixes=CHECK-NEON
+; RUN: opt --mattr=+sve -passes=loop-vectorize -force-vector-interleave=1 -enable-epilogue-vectorization=false -S < %s | FileCheck %s --check-prefixes=CHECK-SVE
+; RUN: opt --mattr=+sve -vectorizer-maximize-bandwidth -passes=loop-vectorize -force-vector-interleave=1 -enable-epilogue-vectorization=false -S < %s | FileCheck %s --check-prefixes=CHECK-SVE-MAXBW
+
+target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
+target triple = "aarch64-none-unknown-elf"
+
+; Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: read) vscale_range(1,16)
+define i32 @chained_partial_reduce_add_sub(ptr %a, ptr %b, ptr %c, i32 %N) #0 {
+; CHECK-NEON-LABEL: define i32 @chained_partial_reduce_add_sub(
+; CHECK-NEON-SAME: ptr [[A:%.*]], ptr [[B:%.*]], ptr [[C:%.*]], i32 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEON-NEXT:  entry:
+; CHECK-NEON-NEXT:    [[CMP28_NOT:%.*]] = icmp ult i32 [[N]], 2
+; CHECK-NEON-NEXT:    [[DIV27:%.*]] = lshr i32 [[N]], 1
+; CHECK-NEON-NEXT:    [[WIDE_TRIP_COUNT:%.*]] = zext nneg i32 [[DIV27]] to i64
+; CHECK-NEON-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[WIDE_TRIP_COUNT]], 16
+; CHECK-NEON-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK-NEON:       vector.ph:
+; CHECK-NEON-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[WIDE_TRIP_COUNT]], 16
+; CHECK-NEON-NEXT:    [[N_VEC:%.*]] = sub i64 [[WIDE_TRIP_COUNT]], [[N_MOD_VF]]
+; CHECK-NEON-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK-NEON:       vector.body:
+; CHECK-NEON-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEON-NEXT:    [[VEC_PHI:%.*]] = phi <16 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP13:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEON-NEXT:    [[TMP0:%.*]] = add i64 [[INDEX]], 0
+; CHECK-NEON-NEXT:    [[TMP1:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 [[TMP0]]
+; CHECK-NEON-NEXT:    [[TMP2:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 [[TMP0]]
+; CHECK-NEON-NEXT:    [[TMP3:%.*]] = getelementptr inbounds nuw i8, ptr [[C]], i64 [[TMP0]]
+; CHECK-NEON-NEXT:    [[TMP4:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i32 0
+; CHECK-NEON-NEXT:    [[WIDE_LOAD:%.*]] = load <16 x i8>, ptr [[TMP4]], align 1
+; CHECK-NEON-NEXT:    [[TMP5:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP2]], i32 0
+; CHECK-NEON-NEXT:    [[WIDE_LOAD1:%.*]] = load <16 x i8>, ptr [[TMP5]], align 1
+; CHECK-NEON-NEXT:    [[TMP6:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP3]], i32 0
+; CHECK-NEON-NEXT:    [[WIDE_LOAD2:%.*]] = load <16 x i8>, ptr [[TMP6]], align 1
+; CHECK-NEON-NEXT:    [[TMP7:%.*]] = sext <16 x i8> [[WIDE_LOAD]] to <16 x i32>
+; CHECK-NEON-NEXT:    [[TMP8:%.*]] = sext <16 x i8> [[WIDE_LOAD1]] to <16 x i32>
+; CHECK-NEON-NEXT:    [[TMP9:%.*]] = sext <16 x i8> [[WIDE_LOAD2]] to <16 x i32>
+; CHECK-NEON-NEXT:    [[TMP10:%.*]] = mul nsw <16 x i32> [[TMP7]], [[TMP8]]
+; CHECK-NEON-NEXT:    [[TMP11:%.*]] = add <16 x i32> [[VEC_PHI]], [[TMP10]]
+; CHECK-NEON-NEXT:    [[TMP12:%.*]] = mul nsw <16 x i32> [[TMP7]], [[TMP9]]
+; CHECK-NEON-NEXT:    [[TMP13]] = sub <16 x i32> [[TMP11]], [[TMP12]]
+; CHECK-NEON-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
+; CHECK-NEON-NEXT:    [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEON-NEXT:    br i1 [[TMP14]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK-NEON:       middle.block:
+; CHECK-NEON-NEXT:    [[TMP15:%.*]] = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> [[TMP13]])
+; CHECK-NEON-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[WIDE_TRIP_COUNT]], [[N_VEC]]
+; CHECK-NEON-NEXT:    br i1 [[CMP_N]], label [[FOR_COND_CLEANUP:%.*]], label [[SCALAR_PH]]
+;
+; CHECK-SVE-LABEL: define i32 @chained_partial_reduce_add_sub(
+; CHECK-SVE-SAME: ptr [[A:%.*]], ptr [[B:%.*]], ptr [[C:%.*]], i32 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-SVE-NEXT:  entry:
+; CHECK-SVE-NEXT:    [[CMP28_NOT:%.*]] = icmp ult i32 [[N]], 2
+; CHECK-SVE-NEXT:    [[DIV27:%.*]] = lshr i32 [[N]], 1
+; CHECK-SVE-NEXT:    [[WIDE_TRIP_COUNT:%.*]] = zext nneg i32 [[DIV27]] to i64
+; CHECK-SVE-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-SVE-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 4
+; CHECK-SVE-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[WIDE_TRIP_COUNT]], [[TMP1]]
+; CHECK-SVE-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK-SVE:       vector.ph:
+; CHECK-SVE-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-SVE-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 4
+; CHECK-SVE-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[WIDE_TRIP_COUNT]], [[TMP3]]
+; CHECK-SVE-NEXT:    [[N_VEC:%.*]] = sub i64 [[WIDE_TRIP_COUNT]], [[N_MOD_VF]]
+; CHECK-SVE-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-SVE-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 4
+; CHECK-SVE-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK-SVE:       vector.body:
+; CHECK-SVE-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-SVE-NEXT:    [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP19:%.*]], [[VECTOR_BODY]] ]
+; CHECK-SVE-NEXT:    [[TMP6:%.*]] = add i64 [[INDEX]], 0
+; CHECK-SVE-NEXT:    [[TMP7:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 [[TMP6]]
+; CHECK-SVE-NEXT:    [[TMP8:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 [[TMP6]]
+; CHECK-SVE-NEXT:    [[TMP9:%.*]] = getelementptr inbounds nuw i8, ptr [[C]], i64 [[TMP6]]
+; CHECK-SVE-NEXT:    [[TMP10:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP7]], i32 0
+; CHECK-SVE-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 4 x i8>, ptr [[TMP10]], align 1
+; CHECK-SVE-NEXT:    [[TMP11:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP8]], i32 0
+; CHECK-SVE-NEXT:    [[WIDE_LOAD1:%.*]] = load <vscale x 4 x i8>, ptr [[TMP11]], align 1
+; CHECK-SVE-NEXT:    [[TMP12:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP9]], i32 0
+; CHECK-SVE-NEXT:    [[WIDE_LOAD2:%.*]] = load <vscale x 4 x i8>, ptr [[TMP12]], align 1
+; CHECK-SVE-NEXT:    [[TMP13:%.*]] = sext <vscale x 4 x i8> [[WIDE_LOAD]] to <vscale x 4 x i32>
+; CHECK-SVE-NEXT:    [[TMP14:%.*]] = sext <vscale x 4 x i8> [[WIDE_LOAD1]] to <vscale x 4 x i32>
+; CHECK-SVE-NEXT:    [[TMP15:%.*]] = sext <vscale x 4 x i8> [[WIDE_LOAD2]] to <vscale x 4 x i32>
+; CHECK-SVE-NEXT:    [[TMP16:%.*]] = mul nsw <vscale x 4 x i32> [[TMP13]], [[TMP14]]
+; CHECK-SVE-NEXT:    [[TMP17:%.*]] = add <vscale x 4 x i32> [[VEC_PHI]], [[TMP16]]
+; CHECK-SVE-NEXT:    [[TMP18:%.*]] = mul nsw <vscale x 4 x i32> [[TMP13]], [[TMP15]]
+; CHECK-SVE-NEXT:    [[TMP19]] = sub <vscale x 4 x i32> [[TMP17]], [[TMP18]]
+; CHECK-SVE-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
+; CHECK-SVE-NEXT:    [[TMP20:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-SVE-NEXT:    br i1 [[TMP20]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK-SVE:       middle.block:
+; CHECK-SVE-NEXT:    [[TMP21:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[TMP19]])
+; CHECK-SVE-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[WIDE_TRIP_COUNT]], [[N_VEC]]
+; CHECK-SVE-NEXT:    br i1 [[CMP_N]], label [[FOR_COND_CLEANUP:%.*]], label [[SCALAR_PH]]
+;
+; CHECK-SVE-MAXBW-LABEL: define i32 @chained_partial_reduce_add_sub(
+; CHECK-SVE-MAXBW-SAME: ptr [[A:%.*]], ptr [[B:%.*]], ptr [[C:%.*]], i32 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-SVE-MAXBW-NEXT:  entry:
+; CHECK-SVE-MAXBW-NEXT:    [[CMP28_NOT:%.*]] = icmp ult i32 [[N]], 2
+; CHECK-SVE-MAXBW-NEXT:    [[DIV27:%.*]] = lshr i32 [[N]], 1
+; CHECK-SVE-MAXBW-NEXT:    [[WIDE_TRIP_COUNT:%.*]] = zext nneg i32 [[DIV27]] to i64
+; CHECK-SVE-MAXBW-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-SVE-MAXBW-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 8
+; CHECK-SVE-MAXBW-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[WIDE_TRIP_COUNT]], [[TMP1]]
+; CHECK-SVE-MAXBW-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK-SVE-MAXBW:       vector.ph:
+; CHECK-SVE-MAXBW-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-SVE-MAXBW-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 8
+; CHECK-SVE-MAXBW-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[WIDE_TRIP_COUNT]], [[TMP3]]
+; CHECK-SVE-MAXBW-NEXT:    [[N_VEC:%.*]] = sub i64 [[WIDE_TRIP_COUNT]], [[N_MOD_VF]]
+; CHECK-SVE-MAXBW-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-SVE-MAXBW-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 8
+; CHECK-SVE-MAXBW-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK-SVE-MAXBW:       vector.body:
+; CHECK-SVE-MAXBW-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-SVE-MAXBW-NEXT:    [[VEC_PHI:%.*]] = phi <vscale x 8 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP19:%.*]], [[VECTOR_BODY]] ]
+; CHECK-SVE-MAXBW-NEXT:    [[TMP6:%.*]] = add i64 [[INDEX]], 0
+; CHECK-SVE-MAXBW-NEXT:    [[TMP7:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 [[TMP6]]
+; CHECK-SVE-MAXBW-NEXT:    [[TMP8:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 [[TMP6]]
+; CHECK-SVE-MAXBW-NEXT:    [[TMP9:%.*]] = getelementptr inbounds nuw i8, ptr [[C]], i64 [[TMP6]]
+; CHECK-SVE-MAXBW-NEXT:    [[TMP10:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP7]], i32 0
+; CHECK-SVE-MAXBW-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 8 x i8>, ptr [[TMP10]], align 1
+; CHECK-SVE-MAXBW-NEXT:    [[TMP11:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP8]], i32 0
+; CHECK-SVE-MAXBW-NEXT:    [[WIDE_LOAD1:%.*]] = load <vscale x 8 x i8>, ptr [[TMP11]], align 1
+; CHECK-SVE-MAXBW-NEXT:    [[TMP12:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP9]], i32 0
+; CHECK-SVE-MAXBW-NEXT:    [[WIDE_LOAD2:%.*]] = load <vscale x 8 x i8>, ptr [[TMP12]], align 1
+; CHECK-SVE-MAXBW-NEXT:    [[TMP13:%.*]] = sext <vscale x 8 x i8> [[WIDE_LOAD]] to <vscale x 8 x i32>
+; CHECK-SVE-MAXBW-NEXT:    [[TMP14:%.*]] = sext <vscale x 8 x i8> [[WIDE_LOAD1]] to <vscale x 8 x i32>
+; CHECK-SVE-MAXBW-NEXT:    [[TMP15:%.*]] = sext <vscale x 8 x i8> [[WIDE_LOAD2]] to <vscale x 8 x i32>
+; CHECK-SVE-MAXBW-NEXT:    [[TMP16:%.*]] = mul nsw <vscale x 8 x i32> [[TMP13]], [[TMP14]]
+; CHECK-SVE-MAXBW-NEXT:    [[TMP17:%.*]] = add <vscale x 8 x i32> [[VEC_PHI]], [[TMP16]]
+; CHECK-SVE-MAXBW-NEXT:    [[TMP18:%.*]] = mul nsw <vscale x 8 x i32> [[TMP13]], [[TMP15]]
+; CHECK-SVE-MAXBW-NEXT:    [[TMP19]] = sub <vscale x 8 x i32> [[TMP17]], [[TMP18]]
+; CHECK-SVE-MAXBW-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
+; CHECK-SVE-MAXBW-NEXT:    [[TMP20:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-SVE-MAXBW-NEXT:    br i1 [[TMP20]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK-SVE-MAXBW:       middle.block:
+; CHECK-SVE-MAXBW-NEXT:    [[TMP21:%.*]] = call i32 @llvm.vector.reduce.add.nxv8i32(<vscale x 8 x i32> [[TMP19]])
+; CHECK-SVE-MAXBW-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[WIDE_TRIP_COUNT]], [[N_VEC]]
+; CHECK-SVE-MAXBW-NEXT:    br i1 [[CMP_N]], label [[FOR_COND_CLEANUP:%.*]], label [[SCALAR_PH]]
+;
+entry:
+  %cmp28.not = icmp ult i32 %N, 2
+  %div27 = lshr i32 %N, 1
+  %wide.trip.count = zext nneg i32 %div27 to i64
+  br label %for.body
+
+for.cond.cleanup:                                 ; preds = %for.cond.cleanup.loopexit, %entry
+  %res.0.lcssa = phi i32 [ %sub, %for.body ]
+  ret i32 %res.0.lcssa
+
+for.body:                                         ; preds = %for.body.preheader, %for.body
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+  %res = phi i32 [ 0, %entry ], [ %sub, %for.body ]
+  %a.ptr = getelementptr inbounds nuw i8, ptr %a, i64 %indvars.iv
+  %b.ptr = getelementptr inbounds nuw i8, ptr %b, i64 %indvars.iv
+  %c.ptr = getelementptr inbounds nuw i8, ptr %c, i64 %indvars.iv
+  %a.val = load i8, ptr %a.ptr, align 1
+  %b.val = load i8, ptr %b.ptr, align 1
+  %c.val = load i8, ptr %c.ptr, align 1
+  %a.ext = sext i8 %a.val to i32
+  %b.ext = sext i8 %b.val to i32
+  %c.ext = sext i8 %c.val to i32
+  %mul.ab = mul nsw i32 %a.ext, %b.ext
+  %add = add nsw i32 %res, %mul.ab
+  %mul.ac = mul nsw i32 %a.ext, %c.ext
+  %sub = sub i32 %add, %mul.ac
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %exitcond.not = icmp eq i64 %indvars.iv.next, %wide.trip.count
+  br i1 %exitcond.not, label %for.cond.cleanup, label %for.body, !loop !1
+}
+
+define i32 @chained_partial_reduce_add_add(ptr %a, ptr %b, ptr %c, i32 %N) #0 {
+; CHECK-NEON-LABEL: define i32 @chained_partial_reduce_add_add(
+; CHECK-NEON-SAME: ptr [[A:%.*]], ptr [[B:%.*]], ptr [[C:%.*]], i32 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-NEON-NEXT:  entry:
+; CHECK-NEON-NEXT:    [[CMP28_NOT:%.*]] = icmp ult i32 [[N]], 2
+; CHECK-NEON-NEXT:    [[DIV27:%.*]] = lshr i32 [[N]], 1
+; CHECK-NEON-NEXT:    [[WIDE_TRIP_COUNT:%.*]] = zext nneg i32 [[DIV27]] to i64
+; CHECK-NEON-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[WIDE_TRIP_COUNT]], 16
+; CHECK-NEON-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK-NEON:       vector.ph:
+; CHECK-NEON-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[WIDE_TRIP_COUNT]], 16
+; CHECK-NEON-NEXT:    [[N_VEC:%.*]] = sub i64 [[WIDE_TRIP_COUNT]], [[N_MOD_VF]]
+; CHECK-NEON-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK-NEON:       vector.body:
+; CHECK-NEON-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEON-NEXT:    [[VEC_PHI:%.*]] = phi <4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[PARTIAL_REDUCE3:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEON-NEXT:    [[TMP0:%.*]] = add i64 [[INDEX]], 0
+; CHECK-NEON-NEXT:    [[TMP1:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 [[TMP0]]
+; CHECK-NEON-NEXT:    [[TMP2:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 [[TMP0]]
+; CHECK-NEON-NEXT:    [[TMP3:%.*]] = getelementptr inbounds nuw i8, ptr [[C]], i64 [[TMP0]]
+; CHECK-NEON-NEXT:    [[TMP4:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i32 0
+; CHECK-NEON-NEXT:    [[WIDE_LOAD:%.*]] = load <16 x i8>, ptr [[TMP4]], align 1
+; CHECK-NEON-NEXT:    [[TMP5:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP2]], i32 0
+; CHECK-NEON-NEXT:    [[WIDE_LOAD1:%.*]] = load <16 x i8>, ptr [[TMP5]], align 1
+; CHECK-NEON-NEXT:    [[TMP6:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP3]], i32 0
+; CHECK-NEON-NEXT:    [[WIDE_LOAD2:%.*]] = load <16 x i8>, ptr [[TMP6]], align 1
+; CHECK-NEON-NEXT:    [[TMP7:%.*]] = sext <16 x i8> [[WIDE_LOAD]] to <16 x i32>
+; CHECK-NEON-NEXT:    [[TMP8:%.*]] = sext <16 x i8> [[WIDE_LOAD1]] to <16 x i32>
+; CHECK-NEON-NEXT:    [[TMP9:%.*]] = sext <16 x i8> [[WIDE_LOAD2]] to <16 x i32>
+; CHECK-NEON-NEXT:    [[TMP10:%.*]] = mul nsw <16 x i32> [[TMP7]], [[TMP8]]
+; CHECK-NEON-NEXT:    [[PARTIAL_REDUCE:%.*]] = call <4 x i32> @llvm.experimental.vector.partial.reduce.add.v4i32.v16i32(<4 x i32> [[VEC_PHI]], <16 x i32> [[TMP10]])
+; CHECK-NEON-NEXT:    [[TMP11:%.*]] = mul nsw <16 x i32> [[TMP7]], [[TMP9]]
+; CHECK-NEON-NEXT:    [[PARTIAL_REDUCE3]] = call <4 x i32> @llvm.experimental.vector.partial.reduce.add.v4i32.v16i32(<4 x i32> [[PARTIAL_REDUCE]], <16 x i32> [[TMP11]])
+; CHECK-NEON-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
+; CHECK-NEON-NEXT:    [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEON-NEXT:    br i1 [[TMP12]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK-NEON:       middle.block:
+; CHECK-NEON-NEXT:    [[TMP13:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[PARTIAL_REDUCE3]])
+; CHECK-NEON-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[WIDE_TRIP_COUNT]], [[N_VEC]]
+; CHECK-NEON-NEXT:    br i1 [[CMP_N]], label [[FOR_COND_CLEANUP:%.*]], label [[SCALAR_PH]]
+;
+; CHECK-SVE-LABEL: define i32 @chained_partial_reduce_add_add(
+; CHECK-SVE-SAME: ptr [[A:%.*]], ptr [[B:%.*]], ptr [[C:%.*]], i32 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-SVE-NEXT:  entry:
+; CHECK-SVE-NEXT:    [[CMP28_NOT:%.*]] = icmp ult i32 [[N]], 2
+; CHECK-SVE-NEXT:    [[DIV27:%.*]] = lshr i32 [[N]], 1
+; CHECK-SVE-NEXT:    [[WIDE_TRIP_COUNT:%.*]] = zext nneg i32 [[DIV27]] to i64
+; CHECK-SVE-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-SVE-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 4
+; CHECK-SVE-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[WIDE_TRIP_COUNT]], [[TMP1]]
+; CHECK-SVE-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK-SVE:       vector.ph:
+; CHECK-SVE-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-SVE-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 4
+; CHECK-SVE-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[WIDE_TRIP_COUNT]], [[TMP3]]
+; CHECK-SVE-NEXT:    [[N_VEC:%.*]] = sub i64 [[WIDE_TRIP_COUNT]], [[N_MOD_VF]]
+; CHECK-SVE-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-SVE-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 4
+; CHECK-SVE-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK-SVE:       vector.body:
+; CHECK-SVE-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-SVE-NEXT:    [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP19:%.*]], [[VECTOR_BODY]] ]
+; CHECK-SVE-NEXT:    [[TMP6:%.*]] = add i64 [[INDEX]], 0
+; CHECK-SVE-NEXT:    [[TMP7:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 [[TMP6]]
+; CHECK-SVE-NEXT:    [[TMP8:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 [[TMP6]]
+; CHECK-SVE-NEXT:    [[TMP9:%.*]] = getelementptr inbounds nuw i8, ptr [[C]], i64 [[TMP6]]
+; CHECK-SVE-NEXT:    [[TMP10:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP7]], i32 0
+; CHECK-SVE-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 4 x i8>, ptr [[TMP10]], align 1
+; CHECK-SVE-NEXT:    [[TMP11:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP8]], i32 0
+; CHECK-SVE-NEXT:    [[WIDE_LOAD1:%.*]] = load <vscale x 4 x i8>, ptr [[TMP11]], align 1
+; CHECK-SVE-NEXT:    [[TMP12:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP9]], i32 0
+; CHECK-SVE-NEXT:    [[WIDE_LOAD2:%.*]] = load <vscale x 4 x i8>, ptr [[TMP12]], align 1
+; CHECK-SVE-NEXT:    [[TMP13:%.*]] = sext <vscale x 4 x i8> [[WIDE_LOAD]] to <vscale x 4 x i32>
+; CHECK-SVE-NEXT:    [[TMP14:%.*]] = sext <vscale x 4 x i8> [[WIDE_LOAD1]] to <vscale x 4 x i32>
+; CHECK-SVE-NEXT:    [[TMP15:%.*]] = sext <vscale x 4 x i8> [[WIDE_LOAD2]] to <vscale x 4 x i32>
+; CHECK-SVE-NEXT:    [[TMP16:%.*]] = mul nsw <vscale x 4 x i32> [[TMP13]], [[TMP14]]
+; CHECK-SVE-NEXT:    [[TMP17:%.*]] = add <vscale x 4 x i32> [[VEC_PHI]], [[TMP16]]
+; CHECK-SVE-NEXT:    [[TMP18:%.*]] = mul nsw <vscale x 4 x i32> [[TMP13]], [[TMP15]]
+; CHECK-SVE-NEXT:    [[TMP19]] = add <vscale x 4 x i32> [[TMP17]], [[TMP18]]
+; CHECK-SVE-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
+; CHECK-SVE-NEXT:    [[TMP20:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-SVE-NEXT:    br i1 [[TMP20]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK-SVE:       middle.block:
+; CHECK-SVE-NEXT:    [[TMP21:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[TMP19]])
+; CHECK-SVE-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[WIDE_TRIP_COUNT]], [[N_VEC]]
+; CHECK-SVE-NEXT:    br i1 [[CMP_N]], label [[FOR_COND_CLEANUP:%.*]], label [[SCALAR_PH]]
+;
+; CHECK-SVE-MAXBW-LABEL: define i32 @chained_partial_reduce_add_add(
+; CHECK-SVE-MAXBW-SAME: ptr [[A:%.*]], ptr [[B:%.*]], ptr [[C:%.*]], i32 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-SVE-MAXBW-NEXT:  entry:
+; CHECK-SVE-MAXBW-NEXT:    [[CMP28_NOT:%.*]] = icmp ult i32 [[N]], 2
+; CHECK-SVE-MAXBW-NEXT:    [[DIV27:%.*]] = lshr i32 [[N]], 1
+; CHECK-SVE-MAXBW-NEXT:    [[WIDE_TRIP_COUNT:%.*]] = zext nneg i32 [[DIV27]] to i64
+; CHECK-SVE-MAXBW-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-SVE-MAXBW-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 8
+; CHECK-SVE-MAXBW-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[WIDE_TRIP_COUNT]], [[TMP1]]
+; CHECK-SVE-MAXBW-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK-SVE-MAXBW:       vector.ph:
+; CHECK-SVE-MAXBW-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-SVE-MAXBW-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 8
+; CHECK-SVE-MAXBW-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[WIDE_TRIP_COUNT]], [[TMP3]]
+; CHECK-SVE-MAXBW-NEXT:    [[N_VEC:%.*]] = sub i64 [[WIDE_TRIP_COUNT]], [[N_MOD_VF]]
+; CHECK-SVE-MAXBW-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-SVE-MAXBW-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 8
+; CHECK-SVE-MAXBW-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK-SVE-MAXBW:       vector.body:
+; CHECK-SVE-MAXBW-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-SVE-MAXBW-NEXT:    [[VEC_PHI:%.*]] = phi <vscale x 2 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[PARTIAL_REDUCE3:%.*]], [[VECTOR_BODY]] ]
+; CHECK-SVE-MAXBW-NEXT:    [[TMP6:%.*]] = add i64 [[INDEX]], 0
+; CHECK-SVE-MAXBW-NEXT:    [[TMP7:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 [[TMP6]]
+; CHECK-SVE-MAXBW-NEXT:    [[TMP8:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 [[TMP6]]
+; CHECK-SVE-MAXBW-NEXT:    [[TMP9:%.*]] = getelementptr inbounds nuw i8, ptr [[C]], i64 [[TMP6]]
+; CHECK-SVE-MAXBW-NEXT:    [[TMP10:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP7]], i32 0
+; CHECK-SVE-MAXBW-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 8 x i8>, ptr [[TMP10]], align 1
+; CHECK-SVE-MAXBW-NEXT:    [[TMP11:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP8]], i32 0
+; CHECK-SVE-MAXBW-NEXT:    [[WIDE_LOAD1:%.*]] = load <vscale x 8 x i8>, ptr [[TMP11]], align 1
+; CHECK-SVE-MAXBW-NEXT:    [[TMP12:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP9]], i32 0
+; CHECK-SVE-MAXBW-NEXT:    [[WIDE_LOAD2:%.*]] = load <vscale x 8 x i8>, ptr [[TMP12]], align 1
+; CHECK-SVE-MAXBW-NEXT:    [[TMP13:%.*]] = sext <vscale x 8 x i8> [[WIDE_LOAD]] to <vscale x 8 x i32>
+; CHECK-SVE-MAXBW-NEXT:    [[TMP14:%.*]] = sext <vscale x 8 x i8> [[WIDE_LOAD1]] to <vscale x 8 x i32>
+; CHECK-SVE-MAXBW-NEXT:    [[TMP15:%.*]] = sext <vscale x 8 x i8> [[WIDE_LOAD2]] to <vscale x 8 x i32>
+; CHECK-SVE-MAXBW-NEXT:    [[TMP16:%.*]] = mul nsw <vscale x 8 x i32> [[TMP13]], [[TMP14]]
+; CHECK-SVE-MAXBW-NEXT:    [[PARTIAL_REDUCE:%.*]] = call <vscale x 2 x i32> @llvm.experimental.vector.partial.reduce.add.nxv2i32.nxv8i32(<vscale x 2 x i32> [[VEC_PHI]], <vscale x 8 x i32> [[TMP16]])
+; CHECK-SVE-MAXBW-NEXT:    [[TMP17:%.*]] = mul nsw <vscale x 8 x i32> [[TMP13]], [[TMP15]]
+; CHECK-SVE-MAXBW-NEXT:    [[PARTIAL_REDUCE3]] = call <vscale x 2 x i32> @llvm.experimental.vector.partial.reduce.add.nxv2i32.nxv8i32(<vscale x 2 x i32> [[PARTIAL_REDUCE]], <vscale x 8 x i32> [[TMP17]])
+; CHECK-SVE-MAXBW-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
+; CHECK-SVE-MAXBW-NEXT:    [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-SVE-MAXBW-NEXT:    br i1 [[TMP18]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK-SVE-MAXBW:       middle.block:
+; CHECK-SVE-MAXBW-NEXT:    [[TMP19:%.*]] = call i32 @llvm.vector.reduce.add.nxv2i32(<vscale x 2 x i32> [[PARTIAL_REDUCE3]])
+; CHECK-SVE-MAXBW-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[WIDE_TRIP_COUNT]], [[N_VEC]]
+; CHECK-SVE-MAXBW-NEXT:    br i1 [[CMP_N]], label [[FOR_COND_CLEANUP:%.*]], label [[SCALAR_PH]]
+;
+entry:
+  %cmp28.not = icmp ult i32 %N, 2
+  %div27 = lshr i32 %N, 1
+  %wide.trip.count = zext nneg i32 %div27 to i64
+  br label %for.body
+
+for.cond.cleanup:                                 ; preds = %for.cond.cleanup.loopexit, %entry
+  %res.0.lcssa = phi i32 [ %add.2, %for.body ]
+  ret i32 %res.0.lcssa
+
+for.body:                                         ; preds = %for.body.preheader, %for.body
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+  %res = phi i32 [ 0, %entry ], [ %add.2, %for.body ]
+  %a.ptr = getelementptr inbounds nuw i8, ptr %a, i64 %indvars.iv
+  %b.ptr = getelementptr inbounds nuw i8, ptr %b, i64 %indvars.iv
+  %c.ptr = getelementptr inbounds nuw i8, ptr %c, i64 %indvars.iv
+  %a.val = load i8, ptr %a.ptr, align 1
+  %b.val = load i8, ptr %b.ptr, align 1
+  %c.val = load i8, ptr %c.ptr, align 1
+  %a.ext = sext i8 %a.val to i32
+  %b.ext = sext i8 %b.val to i32
+  %c.ext = sext i8 %c.val to i32
+  %mul.ab = mul nsw i32 %a.ext, %b.ext
+  %add = add nsw i32 %res, %mul.ab
+  %mul.ac = mul nsw i32 %a.ext, %c.ext
+  %add.2 = add i32 %add, %mul.ac
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %exitcond.not = icmp eq i64 %indvars.iv.next, %wide.trip.count
+  br i1 %exitcond.not, label %for.cond.cleanup, label %for.body, !loop !1
+}
+
+define i32 @chained_partial_reduce_sub_add(ptr %a, ptr %b, ptr %c, i32 %N) #0 {
+; CHECK-NEON-LABEL: define i32 @chained_partial_reduce_sub_add(
+; CHECK-NEON-SAME: ptr [[A:%.*]], ptr [[B:%.*]], ptr [[C:%.*]], i32 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-NEON-NEXT:  entry:
+; CHECK-NEON-NEXT:    [[CMP28_NOT:%.*]] = icmp ult i32 [[N]], 2
+; CHECK-NEON-NEXT:    [[DIV27:%.*]] = lshr i32 [[N]], 1
+; CHECK-NEON-NEXT:    [[WIDE_TRIP_COUNT:%.*]] = zext nneg i32 [[DIV27]] to i64
+; CHECK-NEON-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[WIDE_TRIP_COUNT]], 16
+; CHECK-NEON-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK-NEON:       vector.ph:
+; CHECK-NEON-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[WIDE_TRIP_COUNT]], 16
+; CHECK-NEON-NEXT:    [[N_VEC:%.*]] = sub i64 [[WIDE_TRIP_COUNT]], [[N_MOD_VF]]
+; CHECK-NEON-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK-NEON:       vector.body:
+; CHECK-NEON-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEON-NEXT:    [[VEC_PHI:%.*]] = phi <16 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP13:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEON-NEXT:    [[TMP0:%.*]] = add i64 [[INDEX]], 0
+; CHECK-NEON-NEXT:    [[TMP1:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 [[TMP0]]
+; CHECK-NEON-NEXT:    [[TMP2:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 [[TMP0]]
+; CHECK-NEON-NEXT:    [[TMP3:%.*]] = getelementptr inbounds nuw i8, ptr [[C]], i64 [[TMP0]]
+; CHECK-NEON-NEXT:    [[TMP4:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i32 0
+; CHECK-NEON-NEXT:    [[WIDE_LOAD:%.*]] = load <16 x i8>, ptr [[TMP4]], align 1
+; CHECK-NEON-NEXT:    [[TMP5:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP2]], i32 0
+; CHECK-NEON-NEXT:    [[WIDE_LOAD1:%.*]] = load <16 x i8>, ptr [[TMP5]], align 1
+; CHECK-NEON-NEXT:    [[TMP6:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP3]], i32 0
+; CHECK-NEON-NEXT:    [[WIDE_LOAD2:%.*]] = load <16 x i8>, ptr [[TMP6]], align 1
+; CHECK-NEON-NEXT:    [[TMP7:%.*]] = sext <16 x i8> [[WIDE_LOAD]] to <16 x i32>
+; CHECK-NEON-NEXT:    [[TMP8:%.*]] = sext <16 x i8> [[WIDE_LOAD1]] to <16 x i32>
+; CHECK-NEON-NEXT:    [[TMP9:%.*]] = sext <16 x i8> [[WIDE_LOAD2]] to <16 x i32>
+; CHECK-NEON-NEXT:    [[TMP10:%.*]] = mul nsw <16 x i32> [[TMP7]], [[TMP8]]
+; CHECK-NEON-NEXT:    [[TMP11:%.*]] = sub <16 x i32> [[VEC_PHI]], [[TMP10]]
+; CHECK-NEON-NEXT:    [[TMP12:%.*]] = mul nsw <16 x i32> [[TMP7]], [[TMP9]]
+; CHECK-NEON-NEXT:    [[TMP13]] = add <16 x i32> [[TMP11]], [[TMP12]]
+; CHECK-NEON-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
+; CHECK-NEON-NEXT:    [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEON-NEXT:    br i1 [[TMP14]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+; CHECK-NEON:       middle.block:
+; CHECK-NEON-NEXT:    [[TMP15:%.*]] = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> [[TMP13]])
+; CHECK-NEON-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[WIDE_TRIP_COUNT]], [[N_VEC]]
+; CHECK-NEON-NEXT:    br i1 [[CMP_N]], label [[FOR_COND_CLEANUP:%.*]], label [[SCALAR_PH]]
+;
+; CHECK-SVE-LABEL: define i32 @chained_partial_reduce_sub_add(
+; CHECK-SVE-SAME: ptr [[A:%.*]], ptr [[B:%.*]], ptr [[C:%.*]], i32 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-SVE-NEXT:  entry:
+; CHECK-SVE-NEXT:    [[CMP28_NOT:%.*]] = icmp ult i32 [[N]], 2
+; CHECK-SVE-NEXT:    [[DIV27:%.*]] = lshr i32 [[N]], 1
+; CHECK-SVE-NEXT:    [[WIDE_TRIP_COUNT:%.*]] = zext nneg i32 [[DIV27]] to i64
+; CHECK-SVE-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-SVE-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 4
+; CHECK-SVE-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[WIDE_TRIP_COUNT]], [[TMP1]]
+; CHECK-SVE-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK-SVE:       vector.ph:
+; CHECK-SVE-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-SVE-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 4
+; CHECK-SVE-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[WIDE_TRIP_COUNT]], [[TMP3]]
+; CHECK-SVE-NEXT:    [[N_VEC:%.*]] = sub i64 [[WIDE_TRIP_COUNT]], [[N_MOD_VF]]
+; CHECK-SVE-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-SVE-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 4
+; CHECK-SVE-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK-SVE:       vector.body:
+; CHECK-SVE-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-SVE-NEXT:    [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP19:%.*]], [[VECTOR_BODY]] ]
+; CHECK-SVE-NEXT:    [[TMP6:%.*]] = add i64 [[INDEX]], 0
+; CHECK-SVE-NEXT:    [[TMP7:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 [[TMP6]]
+; CHECK-SVE-NEXT:    [[TMP8:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 [[TMP6]]
+; CHECK-SVE-NEXT:    [[TMP9:%.*]] = getelementptr inbounds nuw i8, ptr [[C]], i64 [[TMP6]]
+; CHECK-SVE-NEXT:    [[TMP10:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP7]], i32 0
+; CHECK-SVE-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 4 x i8>, ptr [[TMP10]], align 1
+; CHECK-SVE-NEXT:    [[TMP11:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP8]], i32 0
+; CHECK-SVE-NEXT:    [[WIDE_LOAD1:%.*]] = load <vscale x 4 x i8>, ptr [[TMP11]], align 1
+; CHECK-SVE-NEXT:    [[TMP12:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP9]], i32 0
+; CHECK-SVE-NEXT:    [[WIDE_LOAD2:%.*]] = load <vscale x 4 x i8>, ptr [[TMP12]], align 1
+; CHECK-SVE-NEXT:    [[TMP13:%.*]] = sext <vscale x 4 x i8> [[WIDE_LOAD]] to <vscale x 4 x i32>
+; CHECK-SVE-NEXT:    [[TMP14:%.*]] = sext <vscale x 4 x i8> [[WIDE_LOAD1]] to <vscale x 4 x i32>
+; CHECK-SVE-NEXT:    [[TMP15:%.*]] = sext <vscale x 4 x i8> [[WIDE_LOAD2]] to <vscale x 4 x i32>
+; CHECK-SVE-NEXT:    [[TMP16:%.*]] = mul nsw <vscale x 4 x i32> [[TMP13]], [[TMP14]]
+; CHECK-SVE-NEXT:    [[TMP17:%.*]] = sub <vscale x 4 x i32> [[VEC_PHI]], [[TMP16]]
+; CHECK-SVE-NEXT:    [[TMP18:%.*]] = mul nsw <vscale x 4 x i32> [[TMP13]], [[TMP15]]
+; CHECK-SVE-NEXT:    [[TMP19]] = add <vscale x 4 x i32> [[TMP17]], [[TMP18]]
+; CHECK-SVE-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
+; CHECK-SVE-NEXT:    [[TMP20:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-SVE-NEXT:    br i1 [[TMP20]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+; CHECK-SVE:       middle.block:
+; CHECK-SVE-NEXT:    [[TMP21:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[TMP19]])
+; CHECK-SVE-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[WIDE_TRIP_COUNT]], [[N_VEC]]
+; CHECK-SVE-NEXT:    br i1 [[CMP_N]], label [[FOR_COND_CLEANUP:%.*]], label [[SCALAR_PH]]
+;
+; CHECK-SVE-MAXBW-LABEL: define i32 @chained_partial_reduce_sub_add(
+; CHECK-SVE-MAXBW-SAME: ptr [[A:%.*]], ptr [[B:%.*]], ptr [[C:%.*]], i32 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-SVE-MAXBW-NEXT:  entry:
+; CHECK-SVE-MAXBW-NEXT:    [[CMP28_NOT:%.*]] = icmp ult i32 [[N]], 2
+; CHECK-SVE-MAXBW-NEXT:    [[DIV27:%.*]] = lshr i32 [[N]], 1
+; CHECK-SVE-MAXBW-NEXT:    [[WIDE_TRIP_COUNT:%.*]] = zext nneg i32 [[DIV27]] to i64
+; CHECK-SVE-MAXBW-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-SVE-MAXBW-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 8
+; CHECK-SVE-MAXBW-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[WIDE_TRIP_COUNT]], [[TMP1]]
+; CHECK-SVE-MAXBW-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK-SVE-MAXBW:       vector.ph:
+; CHECK-SVE-MAXBW-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-SVE-MAXBW-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 8
+; CHECK-SVE-MAXBW-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[WIDE_TRIP_COUNT]], [[TMP3]]
+; CHECK-SVE-MAXBW-NEXT:    [[N_VEC:%.*]] = sub i64 [[WIDE_TRIP_COUNT]], [[N_MOD_VF]]
+; CHECK-SVE-MAXBW-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-SVE-MAXBW-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 8
+; CHECK-SVE-MAXBW-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK-SVE-MAXBW:       vector.body:
+; CHECK-SVE-MAXBW-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-SVE-MAXBW-NEXT:    [[VEC_PHI:%.*]] = phi <vscale x 8 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP19:%.*]], [[VECTOR_BODY]] ]
+; CHECK-SVE-MAXBW-NEXT:    [[TMP6:%.*]] = add i64 [[INDEX]], 0
+; CHECK-SVE-MAXBW-NEXT:    [[TMP7:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 [[TMP6]]
+; CHECK-SVE-MAXBW-NEXT:    [[TMP8:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 [[TMP6]]
+; CHECK-SVE-MAXBW-NEXT:    [[TMP9:%.*]] = getelementptr inbounds nuw i8, ptr [[C]], i64 [[TMP6]]
+; CHECK-SVE-MAXBW-NEXT:    [[TMP10:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP7]], i32 0
+; CHECK-SVE-MAXBW-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 8 x i8>, ptr [[TMP10]], align 1
+; CHECK-SVE-MAXBW-NEXT:    [[TMP11:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP8]], i32 0
+; CHECK-SVE-MAXBW-NEXT:    [[WIDE_LOAD1:%.*]] = load <vscale x 8 x i8>, ptr [[TMP11]], align 1
+; CHECK-SVE-MAXBW-NEXT:    [[TMP12:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP9]], i32 0
+; CHECK-SVE-MAXBW-NEXT:    [[WIDE_LOAD2:%.*]] = load <vscale x 8 x i8>, ptr [[TMP12]], align 1
+; CHECK-SVE-MAXBW-NEXT:    [[TMP13:%.*]] = sext <vscale x 8 x i8> [[WIDE_LOAD]] to <vscale x 8 x i32>
+; CHECK-SVE-MAXBW-NEXT:    [[TMP14:%.*]] = sext <vscale x 8 x i8> [[WIDE_LOAD1]] to <vscale x 8 x i32>
+; CHECK-SVE-MAXBW-NEXT:    [[TMP15:%.*]] = sext <vscale x 8 x i8> [[WIDE_LOAD2]] to <vscale x 8 x i32>
+; CHECK-SVE-MAXBW-NEXT:    [[TMP16:%.*]] = mul nsw <vscale x 8 x i32> [[TMP13]], [[TMP14]]
+; CHECK-SVE-MAXBW-NEXT:    [[TMP17:%.*]] = sub <vscale x 8 x i32> [[VEC_PHI]], [[TMP16]]
+; CHECK-SVE-MAXBW-NEXT:    [[TMP18:%.*]] = mul nsw <vscale x 8 x i32> [[TMP13]], [[TMP15]]
+; CHECK-SVE-MAXBW-NEXT:    [[TMP19]] = add <vscale x 8 x i32> [[TMP17]], [[TMP18]]
+; CHECK-SVE-MAXBW-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
+; CHECK-SVE-MAXBW-NEXT:    [[TMP20:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-SVE-MAXBW-NEXT:    br i1 [[TMP20]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+; CHECK-SVE-MAXBW:       middle.block:
+; CHECK-SVE-MAXBW-NEXT:    [[TMP21:%.*]] = call i32 @llvm.vector.reduce.add.nxv8i32(<vscale x 8 x i32> [[TMP19]])
+; CHECK-SVE-MAXBW-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[WIDE_TRIP_COUNT]], [[N_VEC]]
+; CHECK-SVE-MAXBW-NEXT:    br i1 [[CMP_N]], label [[FOR_COND_CLEANUP:%.*]], label [[SCALAR_PH]]
+;
+entry:
+  %cmp28.not = icmp ult i32 %N, 2
+  %div27 = lshr i32 %N, 1
+  %wide.trip.count = zext nneg i32 %div27 to i64
+  br label %for.body
+
+for.cond.cleanup:                                 ; preds = %for.cond.cleanup.loopexit, %entry
+  %res.0.lcssa = phi i32 [ %add, %for.body ]
+  ret i32 %res.0.lcssa
+
+for.body:                                         ; preds = %for.body.preheader, %for.body
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+  %res = phi i32 [ 0, %entry ], [ %add, %for.body ]
+
+  %a.ptr = getelementptr inbounds nuw i8, ptr %a, i64 %indvars.iv
+  %b.ptr = getelementptr inbounds nuw i8, ptr %b, i64 %indvars.iv
+  %c.ptr = getelementptr inbounds nuw i8, ptr %c, i64 %indvars.iv
+  %a.val = load i8, ptr %a.ptr, align 1
+  %b.val = load i8, ptr %b.ptr, align 1
+  %c.val = load i8, ptr %c.ptr, align 1
+
+  %a.ext = sext i8 %a.val to i32
+  %b.ext = sext i8 %b.val to i32
+  %c.ext = sext i8 %c.val to i32
+  %mul.ab = mul nsw i32 %a.ext, %b.ext
+  %sub = sub nsw i32 %res, %mul.ab
+  %mul.ac = mul nsw i32 %a.ext, %c.ext
+  %add = add i32 %sub, %mul.ac
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %exitcond.not = icmp eq i64 %indvars.iv.next, %wide.trip.count
+  br i1 %exitcond.not, label %for.cond.cleanup, label %for.body, !loop !1
+}
+
+define i32 @chained_partial_reduce_sub_sub(ptr %a, ptr %b, ptr %c, i32 %N) #0 {
+; CHECK-NEON-LABEL: define i32 @chained_partial_reduce_sub_sub(
+; CHECK-NEON-SAME: ptr [[A:%.*]], ptr [[B:%.*]], ptr [[C:%.*]], i32 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-NEON-NEXT:  entry:
+; CHECK-NEON-NEXT:    [[CMP28_NOT:%.*]] = icmp ult i32 [[N]], 2
+; CHECK-NEON-NEXT:    [[DIV27:%.*]] = lshr i32 [[N]], 1
+; CHECK-NEON-NEXT:    [[WIDE_TRIP_COUNT:%.*]] = zext nneg i32 [[DIV27]] to i64
+; CHECK-NEON-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[WIDE_TRIP_COUNT]], 16
+; CHECK-NEON-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK-NEON:       vector.ph:
+; CHECK-NEON-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[WIDE_TRIP_COUNT]], 16
+; CHECK-NEON-NEXT:    [[N_VEC:%.*]] = sub i64 [[WIDE_TRIP_COUNT]], [[N_MOD_VF]]
+; CHECK-NEON-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK-NEON:       vector.body:
+; CHECK-NEON-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEON-NEXT:    [[VEC_PHI:%.*]] = phi <16 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP13:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEON-NEXT:    [[TMP0:%.*]] = add i64 [[INDEX]], 0
+; CHECK-NEON-NEXT:    [[TMP1:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 [[TMP0]]
+; CHECK-NEON-NEXT:    [[TMP2:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 [[TMP0]]
+; CHECK-NEON-NEXT:    [[TMP3:%.*]] = getelementptr inbounds nuw i8, ptr [[C]], i64 [[TMP0]]
+; CHECK-NEON-NEXT:    [[TMP4:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i32 0
+; CHECK-NEON-NEXT:    [[WIDE_LOAD:%.*]] = load <16 x i8>, ptr [[TMP4]], align 1
+; CHECK-NEON-NEXT:    [[TMP5:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP2]], i32 0
+; CHECK-NEON-NEXT:    [[WIDE_LOAD1:%.*]] = load <16 x i8>, ptr [[TMP5]], align 1
+; CHECK-NEON-NEXT:    [[TMP6:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP3]], i32 0
+; CHECK-NEON-NEXT:    [[WIDE_LOAD2:%.*]] = load <16 x i8>, ptr [[TMP6]], align 1
+; CHECK-NEON-NEXT:    [[TMP7:%.*]] = sext <16 x i8> [[WIDE_LOAD]] to <16 x i32>
+; CHECK-NEON-NEXT:    [[TMP8:%.*]] = sext <16 x i8> [[WIDE_LOAD1]] to <16 x i32>
+; CHECK-NEON-NEXT:    [[TMP9:%.*]] = sext <16 x i8> [[WIDE_LOAD2]] to <16 x i32>
+; CHECK-NEON-NEXT:    [[TMP10:%.*]] = mul nsw <16 x i32> [[TMP7]], [[TMP8]]
+; CHECK-NEON-NEXT:    [[TMP11:%.*]] = sub <16 x i32> [[VEC_PHI]], [[TMP10]]
+; CHECK-NEON-NEXT:    [[TMP12:%.*]] = mul nsw <16 x i32> [[TMP7]], [[TMP9]]
+; CHECK-NEON-NEXT:    [[TMP13]] = sub <16 x i32> [[TMP11]], [[TMP12]]
+; CHECK-NEON-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
+; CHECK-NEON-NEXT:    [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEON-NEXT:    br i1 [[TMP14]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
+; CHECK-NEON:       middle.block:
+; CHECK-NEON-NEXT:    [[TMP15:%.*]] = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> [[TMP13]])
+; CHECK-NEON-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[WIDE_TRIP_COUNT]], [[N_VEC]]
+; CHECK-NEON-NEXT:    br i1 [[CMP_N]], label [[FOR_COND_CLEANUP:%.*]], label [[SCALAR_PH]]
+;
+; CHECK-SVE-LABEL: define i32 @chained_partial_reduce_sub_sub(
+; CHECK-SVE-SAME: ptr [[A:%.*]], ptr [[B:%.*]], ptr [[C:%.*]], i32 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-SVE-NEXT:  entry:
+; CHECK-SVE-NEXT:    [[CMP28_NOT:%.*]] = icmp ult i32 [[N]], 2
+; CHECK-SVE-NEXT:    [[DIV27:%.*]] = lshr i32 [[N]], 1
+; CHECK-SVE-NEXT:    [[WIDE_TRIP_COUNT:%.*]] = zext nneg i32 [[DIV27]] to i64
+; CHECK-SVE-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-SVE-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 4
+; CHECK-SVE-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[WIDE_TRIP_COUNT]], [[TMP1]]
+; CHECK-SVE-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK-SVE:       vector.ph:
+; CHECK-SVE-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-SVE-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 4
+; CHECK-SVE-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[WIDE_TRIP_COUNT]], [[TMP3]]
+; CHECK-SVE-NEXT:    [[N_VEC:%.*]] = sub i64 [[WIDE_TRIP_COUNT]], [[N_MOD_VF]]
+; CHECK-SVE-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-SVE-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 4
+; CHECK-SVE-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK-SVE:       vector.body:
+; CHECK-SVE-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-SVE-NEXT:    [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP19:%.*]], [[VECTOR_BODY]] ]
+; CHECK-SVE-NEXT:    [[TMP6:%.*]] = add i64 [[INDEX]], 0
+; CHECK-SVE-NEXT:    [[TMP7:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 [[TMP6]]
+; CHECK-SVE-NEXT:    [[TMP8:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 [[TMP6]]
+; CHECK-SVE-NEXT:    [[TMP9:%.*]] = getelementptr inbounds nuw i8, ptr [[C]], i64 [[TMP6]]
+; CHECK-SVE-NEXT:    [[TMP10:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP7]], i32 0
+; CHECK-SVE-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 4 x i8>, ptr [[TMP10]], align 1
+; CHECK-SVE-NEXT:    [[TMP11:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP8]], i32 0
+; CHECK-SVE-NEXT:    [[WIDE_LOAD1:%.*]] = load <vscale x 4 x i8>, ptr [[TMP11]], align 1
+; CHECK-SVE-NEXT:    [[TMP12:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP9]], i32 0
+; CHECK-SVE-NEXT:    [[WIDE_LOAD2:%.*]] = load <vscale x 4 x i8>, ptr [[TMP12]], align 1
+; CHECK-SVE-NEXT:    [[TMP13:%.*]] = sext <vscale x 4 x i8> [[WIDE_LOAD]] to <vscale x 4 x i32>
+; CHECK-SVE-NEXT:    [[TMP14:%.*]] = sext <vscale x 4 x i8> [[WIDE_LOAD1]] to <vscale x 4 x i32>
+; CHECK-SVE-NEXT:    [[TMP15:%.*]] = sext <vscale x 4 x i8> [[WIDE_LOAD2]] to <vscale x 4 x i32>
+; CHECK-SVE-NEXT:    [[TMP16:%.*]] = mul nsw <vscale x 4 x i32> [[TMP13]], [[TMP14]]
+; CHECK-SVE-NEXT:    [[TMP17:%.*]] = sub <vscale x 4 x i32> [[VEC_PHI]], [[TMP16]]
+; CHECK-SVE-NEXT:    [[TMP18:%.*]] = mul nsw <vscale x 4 x i32> [[TMP13]], [[TMP15]]
+; CHECK-SVE-NEXT:    [[TMP19]] = sub <vscale x 4 x i32> [[TMP17]], [[TMP18]]
+; CHECK-SVE-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
+; CHECK-SVE-NEXT:    [[TMP20:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-SVE-NEXT:    br i1 [[TMP20]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
+; CHECK-SVE:       middle.block:
+; CHECK-SVE-NEXT:    [[TMP21:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[TMP19]])
+; CHECK-SVE-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[WIDE_TRIP_COUNT]], [[N_VEC]]
+; CHECK-SVE-NEXT:    br i1 [[CMP_N]], label [[FOR_COND_CLEANUP:%.*]], label [[SCALAR_PH]]
+;
+; CHECK-SVE-MAXBW-LABEL: define i32 @chained_partial_reduce_sub_sub(
+; CHECK-SVE-MAXBW-SAME: ptr [[A:%.*]], ptr [[B:%.*]], ptr [[C:%.*]], i32 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-SVE-MAXBW-NEXT:  entry:
+; CHECK-SVE-MAXBW-NEXT:    [[CMP28_NOT:%.*]] = icmp ult i32 [[N]], 2
+; CHECK-SVE-MAXBW-NEXT:    [[DIV27:%.*]] = lshr i32 [[N]], 1
+; CHECK-SVE-MAXBW-NEXT:    [[WIDE_TRIP_COUNT:%.*]] = zext nneg i32 [[DIV27]] to i64
+; CHECK-SVE-MAXBW-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-SVE-MAXBW-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 8
+; CHECK-SVE-MAXBW-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[WIDE_TRIP_COUNT]], [[TMP1]]
+; CHECK-SVE-MAXBW-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK-SVE-MAXBW:       vector.ph:
+; CHECK-SVE-MAXBW-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-SVE-MAXBW-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 8
+; CHECK-SVE-MAXBW-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[WIDE_TRIP_COUNT]], [[TMP3]]
+; CHECK-SVE-MAXBW-NEXT:    [[N_VEC:%.*]] = sub i64 [[WIDE_TRIP_COUNT]], [[N_MOD_VF]]
+; CHECK-SVE-MAXBW-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-SVE-MAXBW-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 8
+; CHECK-SVE-MAXBW-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK-SVE-MAXBW:       vector.body:
+; CHECK-SVE-MAXBW-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-SVE-MAXBW-NEXT:    [[VEC_PHI:%.*]] = phi <vscale x 8 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP19:%.*]], [[VECTOR_BODY]] ]
+; CHECK-SVE-MAXBW-NEXT:    [[TMP6:%.*]] = add i64 [[INDEX]], 0
+; CHECK-SVE-MAXBW-NEXT:    [[TMP7:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 [[TMP6]]
+; CHECK-SVE-MAXBW-NEXT:    [[TMP8:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 [[TMP6]]
+; CHECK-SVE-MAXBW-NEXT:    [[TMP9:%.*]] = getelementptr inbounds nuw i8, ptr [[C]], i64 [[TMP6]]
+; CHECK-SVE-MAXBW-NEXT:    [[TMP10:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP7]], i32 0
+; CHECK-SVE-MAXBW-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 8 x i8>, ptr [[TMP10]], align 1
+; CHECK-SVE-MAXBW-NEXT:    [[TMP11:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP8]], i32 0
+; CHECK-SVE-MAXBW-NEXT:    [[WIDE_LOAD1:%.*]] = load <vscale x 8 x i8>, ptr [[TMP11]], align 1
+; CHECK-SVE-MAXBW-NEXT:    [[TMP12:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP9]], i32 0
+; CHECK-SVE-MAXBW-NEXT:    [[WIDE_LOAD2:%.*]] = load <vscale x 8 x i8>, ptr [[TMP12]], align 1
+; CHECK-SVE-MAXBW-NEXT:    [[TMP13:%.*]] = sext <vscale x 8 x i8> [[WIDE_LOAD]] to <vscale x 8 x i32>
+; CHECK-SVE-MAXBW-NEXT:    [[TMP14:%.*]] = sext <vscale x 8 x i8> [[WIDE_LOAD1]] to <vscale x 8 x i32>
+; CHECK-SVE-MAXBW-NEXT:    [[TMP15:%.*]] = sext <vscale x 8 x i8> [[WIDE_LOAD2]] to <vscale x 8 x i32>
+; CHECK-SVE-MAXBW-NEXT:    [[TMP16:%.*]] = mul nsw <vscale x 8 x i32> [[TMP13]], [[TMP14]]
+; CHECK-SVE-MAXBW-NEXT:    [[TMP17:%.*]] = sub <vscale x 8 x i32> [[VEC_PHI]], [[TMP16]]
+; CHECK-SVE-MAXBW-NEXT:    [[TMP18:%.*]] = mul nsw <vscale x 8 x i32> [[TMP13]], [[TMP15]]
+; CHECK-SVE-MAXBW-NEXT:    [[TMP19]] = sub <vscale x 8 x i32> [[TMP17]], [[TMP18]]
+; CHECK-SVE-MAXBW-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
+; CHECK-SVE-MAXBW-NEXT:    [[TMP20:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-SVE-MAXBW-NEXT:    br i1 [[TMP20]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
+; CHECK-SVE-MAXBW:       middle.block:
+; CHECK-SVE-MAXBW-NEXT:    [[TMP21:%.*]] = call i32 @llvm.vector.reduce.add.nxv8i32(<vscale x 8 x i32> [[TMP19]])
+; CHECK-SVE-MAXBW-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[WIDE_TRIP_COUNT]], [[N_VEC]]
+; CHECK-SVE-MAXBW-NEXT:    br i1 [[CMP_N]], label [[FOR_COND_CLEANUP:%.*]], label [[SCALAR_PH]]
+;
+entry:
+  %cmp28.not = icmp ult i32 %N, 2
+  %div27 = lshr i32 %N, 1
+  %wide.trip.count = zext nneg i32 %div27 to i64
+  br label %for.body
+
+for.cond.cleanup:                                 ; preds = %for.cond.cleanup.loopexit, %entry
+  %res.0.lcssa = phi i32 [ %sub.2, %for.body ]
+  ret i32 %res.0.lcssa
+
+for.body:                                         ; preds = %for.body.preheader, %for.body
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+  %res = phi i32 [ 0, %entry ], [ %sub.2, %for.body ]
+
+  %a.ptr = getelementptr inbounds nuw i8, ptr %a, i64 %indvars.iv
+  %b.ptr = getelementptr inbounds nuw i8, ptr %b, i64 %indvars.iv
+  %c.ptr = getelementptr inbounds nuw i8, ptr %c, i64 %indvars.iv
+  %a.val = load i8, ptr %a.ptr, align 1
+  %b.val = load i8, ptr %b.ptr, align 1
+  %c.val = load i8, ptr %c.ptr, align 1
+
+  %a.ext = sext i8 %a.val to i32
+  %b.ext = sext i8 %b.val to i32
+  %c.ext = sext i8 %c.val to i32
+
+  %mul.ab = mul nsw i32 %a.ext, %b.ext
+  %sub = sub nsw i32 %res, %mul.ab
+  %mul.ac = mul nsw i32 %a.ext, %c.ext
+  %sub.2 = sub i32 %sub, %mul.ac
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %exitcond.not = icmp eq i64 %indvars.iv.next, %wide.trip.count
+  br i1 %exitcond.not, label %for.cond.cleanup, label %for.body, !loop !1
+}
+
+define i32 @chained_partial_reduce_add_add_add(ptr %a, ptr %b, ptr %c, i32 %N) #0 {
+; CHECK-NEON-LABEL: define i32 @chained_partial_reduce_add_add_add(
+; CHECK-NEON-SAME: ptr [[A:%.*]], ptr [[B:%.*]], ptr [[C:%.*]], i32 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-NEON-NEXT:  entry:
+; CHECK-NEON-NEXT:    [[CMP28_NOT:%.*]] = icmp ult i32 [[N]], 2
+; CHECK-NEON-NEXT:    [[DIV27:%.*]] = lshr i32 [[N]], 1
+; CHECK-NEON-NEXT:    [[WIDE_TRIP_COUNT:%.*]] = zext nneg i32 [[DIV27]] to i64
+; CHECK-NEON-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[WIDE_TRIP_COUNT]], 16
+; CHECK-NEON-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK-NEON:       vector.ph:
+; CHECK-NEON-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[WIDE_TRIP_COUNT]], 16
+; CHECK-NEON-NEXT:    [[N_VEC:%.*]] = sub i64 [[WIDE_TRIP_COUNT]], [[N_MOD_VF]]
+; CHECK-NEON-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK-NEON:       vector.body:
+; CHECK-NEON-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEON-NEXT:    [[VEC_PHI:%.*]] = phi <4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[PARTIAL_REDUCE4:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEON-NEXT:    [[TMP0:%.*]] = add i64 [[INDEX]], 0
+; CHECK-NEON-NEXT:    [[TMP1:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 [[TMP0]]
+; CHECK-NEON-NEXT:    [[TMP2:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 [[TMP0]]
+; CHECK-NEON-NEXT:    [[TMP3:%.*]] = getelementptr inbounds nuw i8, ptr [[C]], i64 [[TMP0]]
+; CHECK-NEON-NEXT:    [[TMP4:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i32 0
+; CHECK-NEON-NEXT:    [[WIDE_LOAD:%.*]] = load <16 x i8>, ptr [[TMP4]], align 1
+; CHECK-NEON-NEXT:    [[TMP5:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP2]], i32 0
+; CHECK-NEON-NEXT:    [[WIDE_LOAD1:%.*]] = load <16 x i8>, ptr [[TMP5]], align 1
+; CHECK-NEON-NEXT:    [[TMP6:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP3]], i32 0
+; CHECK-NEON-NEXT:    [[WIDE_LOAD2:%.*]] = load <16 x i8>, ptr [[TMP6]], align 1
+; CHECK-NEON-NEXT:    [[TMP7:%.*]] = sext <16 x i8> [[WIDE_LOAD]] to <16 x i32>
+; CHECK-NEON-NEXT:    [[TMP8:%.*]] = sext <16 x i8> [[WIDE_LOAD1]] to <16 x i32>
+; CHECK-NEON-NEXT:    [[TMP9:%.*]] = sext <16 x i8> [[WIDE_LOAD2]] to <16 x i32>
+; CHECK-NEON-NEXT:    [[TMP10:%.*]] = mul nsw <16 x i32> [[TMP7]], [[TMP8]]
+; CHECK-NEON-NEXT:    [[PARTIAL_REDUCE:%.*]] = call <4 x i32> @llvm.experimental.vector.partial.reduce.add.v4i32.v16i32(<4 x i32> [[VEC_PHI]], <16 x i32> [[TMP10]])
+; CHECK-NEON-NEXT:    [[TMP11:%.*]] = mul nsw <16 x i32> [[TMP7]], [[TMP9]]
+; CHECK-NEON-NEXT:    [[PARTIAL_REDUCE3:%.*]] = call <4 x i32> @llvm.experimental.vector.partial.reduce.add.v4i32.v16i32(<4 x i32> [[PARTIAL_REDUCE]], <16 x i32> [[TMP11]])
+; CHECK-NEON-NEXT:    [[TMP12:%.*]] = mul nsw <16 x i32> [[TMP8]], [[TMP9]]
+; CHECK-NEON-NEXT:    [[PARTIAL_REDUCE4]] = call <4 x i32> @llvm.experimental.vector.partial.reduce.add.v4i32.v16i32(<4 x i32> [[PARTIAL_REDUCE3]], <16 x i32> [[TMP12]])
+; CHECK-NEON-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
+; CHECK-NEON-NEXT:    [[TMP13:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEON-NEXT:    br i1 [[TMP13]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
+; CHECK-NEON:       middle.block:
+; CHECK-NEON-NEXT:    [[TMP14:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[PARTIAL_REDUCE4]])
+; CHECK-NEON-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[WIDE_TRIP_COUNT]], [[N_VEC]]
+; CHECK-NEON-NEXT:    br i1 [[CMP_N]], label [[FOR_COND_CLEANUP:%.*]], label [[SCALAR_PH]]
+;
+; CHECK-SVE-LABEL: define i32 @chained_partial_reduce_add_add_add(
+; CHECK-SVE-SAME: ptr [[A:%.*]], ptr [[B:%.*]], ptr [[C:%.*]], i32 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-SVE-NEXT:  entry:
+; CHECK-SVE-NEXT:    [[CMP28_NOT:%.*]] = icmp ult i32 [[N]], 2
+; CHECK-SVE-NEXT:    [[DIV27:%.*]] = lshr i32 [[N]], 1
+; CHECK-SVE-NEXT:    [[WIDE_TRIP_COUNT:%.*]] = zext nneg i32 [[DIV27]] to i64
+; CHECK-SVE-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-SVE-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 4
+; CHECK-SVE-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[WIDE_TRIP_COUNT]], [[TMP1]]
+; CHECK-SVE-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK-SVE:       vector.ph:
+; CHECK-SVE-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-SVE-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 4
+; CHECK-SVE-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[WIDE_TRIP_COUNT]], [[TMP3]]
+; CHECK-SVE-NEXT:    [[N_VEC:%.*]] = sub i64 [[WIDE_TRIP_COUNT]], [[N_MOD_VF]]
+; CHECK-SVE-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-SVE-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 4
+; CHECK-SVE-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK-SVE:       vector.body:
+; CHECK-SVE-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-SVE-NEXT:    [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP21:%.*]], [[VECTOR_BODY]] ]
+; CHECK-SVE-NEXT:    [[TMP6:%.*]] = add i64 [[INDEX]], 0
+; CHECK-SVE-NEXT:    [[TMP7:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 [[TMP6]]
+; CHECK-SVE-NEXT:    [[TMP8:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 [[TMP6]]
+; CHECK-SVE-NEXT:    [[TMP9:%.*]] = getelementptr inbounds nuw i8, ptr [[C]], i64 [[TMP6]]
+; CHECK-SVE-NEXT:    [[TMP10:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP7]], i32 0
+; CHECK-SVE-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 4 x i8>, ptr [[TMP10]], align 1
+; CHECK-SVE-NEXT:    [[TMP11:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP8]], i32 0
+; CHECK-SVE-NEXT:    [[WIDE_LOAD1:%.*]] = load <vscale x 4 x i8>, ptr [[TMP11]], align 1
+; CHECK-SVE-NEXT:    [[TMP12:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP9]], i32 0
+; CHECK-SVE-NEXT:    [[WIDE_LOAD2:%.*]] = load <vscale x 4 x i8>, ptr [[TMP12]], align 1
+; CHECK-SVE-NEXT:    [[TMP13:%.*]] = sext <vscale x 4 x i8> [[WIDE_LOAD]] to <vscale x 4 x i32>
+; CHECK-SVE-NEXT:    [[TMP14:%.*]] = sext <vscale x 4 x i8> [[WIDE_LOAD1]] to <vscale x 4 x i32>
+; CHECK-SVE-NEXT:    [[TMP15:%.*]] = sext <vscale x 4 x i8> [[WIDE_LOAD2]] to <vscale x 4 x i32>
+; CHECK-SVE-NEXT:    [[TMP16:%.*]] = mul nsw <vscale x 4 x i32> [[TMP13]], [[TMP14]]
+; CHECK-SVE-NEXT:    [[TMP17:%.*]] = add <vscale x 4 x i32> [[VEC_PHI]], [[TMP16]]
+; CHECK-SVE-NEXT:    [[TMP18:%.*]] = mul nsw <vscale x 4 x i32> [[TMP13]], [[TMP15]]
+; CHECK-SVE-NEXT:    [[TMP19:%.*]] = add <vscale x 4 x i32> [[TMP17]], [[TMP18]]
+; CHECK-SVE-NEXT:    [[TMP20:%.*]] = mul nsw <vscale x 4 x i32> [[TMP14]], [[TMP15]]
+; CHECK-SVE-NEXT:    [[TMP21]] = add <vscale x 4 x i32> [[TMP19]], [[TMP20]]
+; CHECK-SVE-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
+; CHECK-SVE-NEXT:    [[TMP22:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-SVE-NEXT:    br i1 [[TMP22]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
+; CHECK-SVE:       middle.block:
+; CHECK-SVE-NEXT:    [[TMP23:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[TMP21]])
+; CHECK-SVE-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[WIDE_TRIP_COUNT]], [[N_VEC]]
+; CHECK-SVE-NEXT:    br i1 [[CMP_N]], label [[FOR_COND_CLEANUP:%.*]], label [[SCALAR_PH]]
+;
+; CHECK-SVE-MAXBW-LABEL: define i32 @chained_partial_reduce_add_add_add(
+; CHECK-SVE-MAXBW-SAME: ptr [[A:%.*]], ptr [[B:%.*]], ptr [[C:%.*]], i32 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-SVE-MAXBW-NEXT:  entry:
+; CHECK-SVE-MAXBW-NEXT:    [[CMP28_NOT:%.*]] = icmp ult i32 [[N]], 2
+; CHECK-SVE-MAXBW-NEXT:    [[DIV27:%.*]] = lshr i32 [[N]], 1
+; CHECK-SVE-MAXBW-NEXT:    [[WIDE_TRIP_COUNT:%.*]] = zext nneg i32 [[DIV27]] to i64
+; CHECK-SVE-MAXBW-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-SVE-MAXBW-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 16
+; CHECK-SVE-MAXBW-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[WIDE_TRIP_COUNT]], [[TMP1]]
+; CHECK-SVE-MAXBW-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK-SVE-MAXBW:       vector.ph:
+; CHECK-SVE-MAXBW-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-SVE-MAXBW-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 16
+; CHECK-SVE-MAXBW-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[WIDE_TRIP_COUNT]], [[TMP3]]
+; CHECK-SVE-MAXBW-NEXT:    [[N_VEC:%.*]] = sub i64 [[WIDE_TRIP_COUNT]], [[N_MOD_VF]]
+; CHECK-SVE-MAXBW-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-SVE-MAXBW-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 16
+; CHECK-SVE-MAXBW-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK-SVE-MAXBW:       vector.body:
+; CHECK-SVE-MAXBW-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-SVE-MAXBW-NEXT:    [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[PARTIAL_REDUCE4:%.*]], [[VECTOR_BODY]] ]
+; CHECK-SVE-MAXBW-NEXT:    [[TMP6:%.*]] = add i64 [[INDEX]], 0
+; CHECK-SVE-MAXBW-NEXT:    [[TMP7:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 [[TMP6]]
+; CHECK-SVE-MAXBW-NEXT:    [[TMP8:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 [[TMP6]]
+; CHECK-SVE-MAXBW-NEXT:    [[TMP9:%.*]] = getelementptr inbounds nuw i8, ptr [[C]], i64 [[TMP6]]
+; CHECK-SVE-MAXBW-NEXT:    [[TMP10:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP7]], i32 0
+; CHECK-SVE-MAXBW-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 16 x i8>, ptr [[TMP10]], align 1
+; CHECK-SVE-MAXBW-NEXT:    [[TMP11:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP8]], i32 0
+; CHECK-SVE-MAXBW-NEXT:    [[WIDE_LOAD1:%.*]] = load <vscale x 16 x i8>, ptr [[TMP11]], align 1
+; CHECK-SVE-MAXBW-NEXT:    [[TMP12:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP9]], i32 0
+; CHECK-SVE-MAXBW-NEXT:    [[WIDE_LOAD2:%.*]] = load <vscale x 16 x i8>, ptr [[TMP12]], align 1
+; CHECK-SVE-MAXBW-NEXT:    [[TMP13:%.*]] = sext <vscale x 16 x i8> [[WIDE_LOAD]] to <vscale x 16 x i32>
+; CHECK-SVE-MAXBW-NEXT:    [[TMP14:%.*]] = sext <vscale x 16 x i8> [[WIDE_LOAD1]] to <vscale x 16 x i32>
+; CHECK-SVE-MAXBW-NEXT:    [[TMP15:%.*]] = sext <vscale x 16 x i8> [[WIDE_LOAD2]] to <vscale x 16 x i32>
+; CHECK-SVE-MAXBW-NEXT:    [[TMP16:%.*]] = mul nsw <vscale x 16 x i32> [[TMP13]], [[TMP14]]
+; CHECK-SVE-MAXBW-NEXT:    [[PARTIAL_REDUCE:%.*]] = call <vscale x 4 x i32> @llvm.experimental.vector.partial.reduce.add.nxv4i32.nxv16i32(<vscale x 4 x i32> [[VEC_PHI]], <vscale x 16 x i32> [[TMP16]])
+; CHECK-SVE-MAXBW-NEXT:    [[TMP17:%.*]] = mul nsw <vscale x 16 x i32> [[TMP13]], [[TMP15]]
+; CHECK-SVE-MAXBW-NEXT:    [[PARTIAL_REDUCE3:%.*]] = call <vscale x 4 x i32> @llvm.experimental.vector.partial.reduce.add.nxv4i32.nxv16i32(<vscale x 4 x i32> [[PARTIAL_REDUCE]], <vscale x 16 x i32> [[TMP17]])
+; CHECK-SVE-MAXBW-NEXT:    [[TMP18:%.*]] = mul nsw <vscale x 16 x i32> [[TMP14]], [[TMP15]]
+; CHECK-SVE-MAXBW-NEXT:    [[PARTIAL_REDUCE4]] = call <vscale x 4 x i32> @llvm.experimental.vector.partial.reduce.add.nxv4i32.nxv16i32(<vscale x 4 x i32> [[PARTIAL_REDUCE3]], <vscale x 16 x i32> [[TMP18]])
+; CHECK-SVE-MAXBW-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
+; CHECK-SVE-MAXBW-NEXT:    [[TMP19:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-SVE-MAXBW-NEXT:    br i1 [[TMP19]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
+; CHECK-SVE-MAXBW:       middle.block:
+; CHECK-SVE-MAXBW-NEXT:    [[TMP23:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[PARTIAL_REDUCE4]])
+; CHECK-SVE-MAXBW-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[WIDE_TRIP_COUNT]], [[N_VEC]]
+; CHECK-SVE-MAXBW-NEXT:    br i1 [[CMP_N]], label [[FOR_COND_CLEANUP:%.*]], label [[SCALAR_PH]]
+;
+entry:
+  %cmp28.not = icmp ult i32 %N, 2
+  %div27 = lshr i32 %N, 1
+  %wide.trip.count = zext nneg i32 %div27 to i64
+  br label %for.body
+
+for.cond.cleanup:                                 ; preds = %for.cond.cleanup.loopexit, %entry
+  %res.0.lcssa = phi i32 [ %sub.2, %for.body ]
+  ret i32 %res.0.lcssa
+
+for.body:                                         ; preds = %for.body.preheader, %for.body
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+  %res = phi i32 [ 0, %entry ], [ %sub.2, %for.body ]
+
+  %a.ptr = getelementptr inbounds nuw i8, ptr %a, i64 %indvars.iv
+  %b.ptr = getelementptr inbounds nuw i8, ptr %b, i64 %indvars.iv
+  %c.ptr = getelementptr inbounds nuw i8, ptr %c, i64 %indvars.iv
+  %a.val = load i8, ptr %a.ptr, align 1
+  %b.val = load i8, ptr %b.ptr, align 1
+  %c.val = load i8, ptr %c.ptr, align 1
+
+  %a.ext = sext i8 %a.val to i32
+  %b.ext = sext i8 %b.val to i32
+  %c.ext = sext i8 %c.val to i32
+
+  %mul.ab = mul nsw i32 %a.ext, %b.ext
+  %sub = add nsw i32 %res, %mul.ab
+  %mul.ac = mul nsw i32 %a.ext, %c.ext
+  %add = add nsw i32 %sub, %mul.ac
+  %mul.bc = mul nsw i32 %b.ext, %c.ext
+  %sub.2 = add i32 %add, %mul.bc
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %exitcond.not = icmp eq i64 %indvars.iv.next, %wide.trip.count
+  br i1 %exitcond.not, label %for.cond.cleanup, label %for.body, !loop !1
+}
+
+define i32 @chained_partial_reduce_sub_add_sub(ptr %a, ptr %b, ptr %c, i32 %N) #0 {
+; CHECK-NEON-LABEL: define i32 @chained_partial_reduce_sub_add_sub(
+; CHECK-NEON-SAME: ptr [[A:%.*]], ptr [[B:%.*]], ptr [[C:%.*]], i32 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-NEON-NEXT:  entry:
+; CHECK-NEON-NEXT:    [[CMP28_NOT:%.*]] = icmp ult i32 [[N]], 2
+; CHECK-NEON-NEXT:    [[DIV27:%.*]] = lshr i32 [[N]], 1
+; CHECK-NEON-NEXT:    [[WIDE_TRIP_COUNT:%.*]] = zext nneg i32 [[DIV27]] to i64
+; CHECK-NEON-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[WIDE_TRIP_COUNT]], 16
+; CHECK-NEON-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK-NEON:       vector.ph:
+; CHECK-NEON-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[WIDE_TRIP_COUNT]], 16
+; CHECK-NEON-NEXT:    [[N_VEC:%.*]] = sub i64 [[WIDE_TRIP_COUNT]], [[N_MOD_VF]]
+; CHECK-NEON-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK-NEON:       vector.body:
+; CHECK-NEON-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEON-NEXT:    [[VEC_PHI:%.*]] = phi <16 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP15:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEON-NEXT:    [[TMP0:%.*]] = add i64 [[INDEX]], 0
+; CHECK-NEON-NEXT:    [[TMP1:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 [[TMP0]]
+; CHECK-NEON-NEXT:    [[TMP2:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 [[TMP0]]
+; CHECK-NEON-NEXT:    [[TMP3:%.*]] = getelementptr inbounds nuw i8, ptr [[C]], i64 [[TMP0]]
+; CHECK-NEON-NEXT:    [[TMP4:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i32 0
+; CHECK-NEON-NEXT:    [[WIDE_LOAD:%.*]] = load <16 x i8>, ptr [[TMP4]], align 1
+; CHECK-NEON-NEXT:    [[TMP5:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP2]], i32 0
+; CHECK-NEON-NEXT:    [[WIDE_LOAD1:%.*]] = load <16 x i8>, ptr [[TMP5]], align 1
+; CHECK-NEON-NEXT:    [[TMP6:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP3]], i32 0
+; CHECK-NEON-NEXT:    [[WIDE_LOAD2:%.*]] = load <16 x i8>, ptr [[TMP6]], align 1
+; CHECK-NEON-NEXT:    [[TMP7:%.*]] = sext <16 x i8> [[WIDE_LOAD]] to <16 x i32>
+; CHECK-NEON-NEXT:    [[TMP8:%.*]] = sext <16 x i8> [[WIDE_LOAD1]] to <16 x i32>
+; CHECK-NEON-NEXT:    [[TMP9:%.*]] = sext <16 x i8> [[WIDE_LOAD2]] to <16 x i32>
+; CHECK-NEON-NEXT:    [[TMP10:%.*]] = mul nsw <16 x i32> [[TMP7]], [[TMP8]]
+; CHECK-NEON-NEXT:    [[TMP11:%.*]] = sub <16 x i32> [[VEC_PHI]], [[TMP10]]
+; CHECK-NEON-NEXT:    [[TMP12:%.*]] = mul nsw <16 x i32> [[TMP7]], [[TMP9]]
+; CHECK-NEON-NEXT:    [[TMP13:%.*]] = add <16 x i32> [[TMP11]], [[TMP12]]
+; CHECK-NEON-NEXT:    [[TMP14:%.*]] = mul nsw <16 x i32> [[TMP8]], [[TMP9]]
+; CHECK-NEON-NEXT:    [[TMP15]] = sub <16 x i32> [[TMP13]], [[TMP14]]
+; CHECK-NEON-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
+; CHECK-NEON-NEXT:    [[TMP16:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEON-NEXT:    br i1 [[TMP16]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
+; CHECK-NEON:       middle.block:
+; CHECK-NEON-NEXT:    [[TMP17:%.*]] = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> [[TMP15]])
+; CHECK-NEON-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[WIDE_TRIP_COUNT]], [[N_VEC]]
+; CHECK-NEON-NEXT:    br i1 [[CMP_N]], label [[FOR_COND_CLEANUP:%.*]], label [[SCALAR_PH]]
+;
+; CHECK-SVE-LABEL: define i32 @chained_partial_reduce_sub_add_sub(
+; CHECK-SVE-SAME: ptr [[A:%.*]], ptr [[B:%.*]], ptr [[C:%.*]], i32 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-SVE-NEXT:  entry:
+; CHECK-SVE-NEXT:    [[CMP28_NOT:%.*]] = icmp ult i32 [[N]], 2
+; CHECK-SVE-NEXT:    [[DIV27:%.*]] = lshr i32 [[N]], 1
+; CHECK-SVE-NEXT:    [[WIDE_TRIP_COUNT:%.*]] = zext nneg i32 [[DIV27]] to i64
+; CHECK-SVE-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-SVE-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 4
+; CHECK-SVE-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[WIDE_TRIP_COUNT]], [[TMP1]]
+; CHECK-SVE-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK-SVE:       vector.ph:
+; CHECK-SVE-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-SVE-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 4
+; CHECK-SVE-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[WIDE_TRIP_COUNT]], [[TMP3]]
+; CHECK-SVE-NEXT:    [[N_VEC:%.*]] = sub i64 [[WIDE_TRIP_COUNT]], [[N_MOD_VF]]
+; CHECK-SVE-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-SVE-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 4
+; CHECK-SVE-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK-SVE:       vector.body:
+; CHECK-SVE-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-SVE-NEXT:    [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP21:%.*]], [[VECTOR_BODY]] ]
+; CHECK-SVE-NEXT:    [[TMP6:%.*]] = add i64 [[INDEX]], 0
+; CHECK-SVE-NEXT:    [[TMP7:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 [[TMP6]]
+; CHECK-SVE-NEXT:    [[TMP8:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 [[TMP6]]
+; CHECK-SVE-NEXT:    [[TMP9:%.*]] = getelementptr inbounds nuw i8, ptr [[C]], i64 [[TMP6]]
+; CHECK-SVE-NEXT:    [[TMP10:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP7]], i32 0
+; CHECK-SVE-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 4 x i8>, ptr [[TMP10]], align 1
+; CHECK-SVE-NEXT:    [[TMP11:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP8]], i32 0
+; CHECK-SVE-NEXT:    [[WIDE_LOAD1:%.*]] = load <vscale x 4 x i8>, ptr [[TMP11]], align 1
+; CHECK-SVE-NEXT:    [[TMP12:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP9]], i32 0
+; CHECK-SVE-NEXT:    [[WIDE_LOAD2:%.*]] = load <vscale x 4 x i8>, ptr [[TMP12]], align 1
+; CHECK-SVE-NEXT:    [[TMP13:%.*]] = sext <vscale x 4 x i8> [[WIDE_LOAD]] to <vscale x 4 x i32>
+; CHECK-SVE-NEXT:    [[TMP14:%.*]] = sext <vscale x 4 x i8> [[WIDE_LOAD1]] to <vscale x 4 x i32>
+; CHECK-SVE-NEXT:    [[TMP15:%.*]] = sext <vscale x 4 x i8> [[WIDE_LOAD2]] to <vscale x 4 x i32>
+; CHECK-SVE-NEXT:    [[TMP16:%.*]] = mul nsw <vscale x 4 x i32> [[TMP13]], [[TMP14]]
+; CHECK-SVE-NEXT:    [[TMP17:%.*]] = sub <vscale x 4 x i32> [[VEC_PHI]], [[TMP16]]
+; CHECK-SVE-NEXT:    [[TMP18:%.*]] = mul nsw <vscale x 4 x i32> [[TMP13]], [[TMP15]]
+; CHECK-SVE-NEXT:    [[TMP19:%.*]] = add <vscale x 4 x i32> [[TMP17]], [[TMP18]]
+; CHECK-SVE-NEXT:    [[TMP20:%.*]] = mul nsw <vscale x 4 x i32> [[TMP14]], [[TMP15]]
+; CHECK-SVE-NEXT:    [[TMP21]] = sub <vscale x 4 x i32> [[TMP19]], [[TMP20]]
+; CHECK-SVE-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
+; CHECK-SVE-NEXT:    [[TMP22:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-SVE-NEXT:    br i1 [[TMP22]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
+; CHECK-SVE:       middle.block:
+; CHECK-SVE-NEXT:    [[TMP23:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[TMP21]])
+; CHECK-SVE-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[WIDE_TRIP_COUNT]], [[N_VEC]]
+; CHECK-SVE-NEXT:    br i1 [[CMP_N]], label [[FOR_COND_CLEANUP:%.*]], label [[SCALAR_PH]]
+;
+; CHECK-SVE-MAXBW-LABEL: define i32 @chained_partial_reduce_sub_add_sub(
+; CHECK-SVE-MAXBW-SAME: ptr [[A:%.*]], ptr [[B:%.*]], ptr [[C:%.*]], i32 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-SVE-MAXBW-NEXT:  entry:
+; CHECK-SVE-MAXBW-NEXT:    [[CMP28_NOT:%.*]] = icmp ult i32 [[N]], 2
+; CHECK-SVE-MAXBW-NEXT:    [[DIV27:%.*]] = lshr i32 [[N]], 1
+; CHECK-SVE-MAXBW-NEXT:    [[WIDE_TRIP_COUNT:%.*]] = zext nneg i32 [[DIV27]] to i64
+; CHECK-SVE-MAXBW-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-SVE-MAXBW-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 8
+; CHECK-SVE-MAXBW-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[WIDE_TRIP_COUNT]], [[TMP1]]
+; CHECK-SVE-MAXBW-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK-SVE-MAXBW:       vector.ph:
+; CHECK-SVE-MAXBW-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-SVE-MAXBW-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 8
+; CHECK-SVE-MAXBW-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[WIDE_TRIP_COUNT]], [[TMP3]]
+; CHECK-SVE-MAXBW-NEXT:    [[N_VEC:%.*]] = sub i64 [[WIDE_TRIP_COUNT]], [[N_MOD_VF]]
+; CHECK-SVE-MAXBW-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-SVE-MAXBW-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 8
+; CHECK-SVE-MAXBW-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK-SVE-MAXBW:       vector.body:
+; CHECK-SVE-MAXBW-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-SVE-MAXBW-NEXT:    [[VEC_PHI:%.*]] = phi <vscale x 8 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP21:%.*]], [[VECTOR_BODY]] ]
+; CHECK-SVE-MAXBW-NEXT:    [[TMP6:%.*]] = add i64 [[INDEX]], 0
+; CHECK-SVE-MAXBW-NEXT:    [[TMP7:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 [[TMP6]]
+; CHECK-SVE-MAXBW-NEXT:    [[TMP8:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 [[TMP6]]
+; CHECK-SVE-MAXBW-NEXT:    [[TMP9:%.*]] = getelementptr inbounds nuw i8, ptr [[C]], i64 [[TMP6]]
+; CHECK-SVE-MAXBW-NEXT:    [[TMP10:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP7]], i32 0
+; CHECK-SVE-MAXBW-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 8 x i8>, ptr [[TMP10]], align 1
+; CHECK-SVE-MAXBW-NEXT:    [[TMP11:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP8]], i32 0
+; CHECK-SVE-MAXBW-NEXT:    [[WIDE_LOAD1:%.*]] = load <vscale x 8 x i8>, ptr [[TMP11]], align 1
+; CHECK-SVE-MAXBW-NEXT:    [[TMP12:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP9]], i32 0
+; CHECK-SVE-MAXBW-NEXT:    [[WIDE_LOAD2:%.*]] = load <vscale x 8 x i8>, ptr [[TMP12]], align 1
+; CHECK-SVE-MAXBW-NEXT:    [[TMP13:%.*]] = sext <vscale x 8 x i8> [[WIDE_LOAD]] to <vscale x 8 x i32>
+; CHECK-SVE-MAXBW-NEXT:    [[TMP14:%.*]] = sext <vscale x 8 x i8> [[WIDE_LOAD1]] to <vscale x 8 x i32>
+; CHECK-SVE-MAXBW-NEXT:    [[TMP15:%.*]] = sext <vscale x 8 x i8> [[WIDE_LOAD2]] to <vscale x 8 x i32>
+; CHECK-SVE-MAXBW-NEXT:    [[TMP16:%.*]] = mul nsw <vscale x 8 x i32> [[TMP13]], [[TMP14]]
+; CHECK-SVE-MAXBW-NEXT:    [[TMP17:%.*]] = sub <vscale x 8 x i32> [[VEC_PHI]], [[TMP16]]
+; CHECK-SVE-MAXBW-NEXT:    [[TMP18:%.*]] = mul nsw <vscale x 8 x i32> [[TMP13]], [[TMP15]]
+; CHECK-SVE-MAXBW-NEXT:    [[TMP19:%.*]] = add <vscale x 8 x i32> [[TMP17]], [[TMP18]]
+; CHECK-SVE-MAXBW-NEXT:    [[TMP20:%.*]] = mul nsw <vscale x 8 x i32> [[TMP14]], [[TMP15]]
+; CHECK-SVE-MAXBW-NEXT:    [[TMP21]] = sub <vscale x 8 x i32> [[TMP19]], [[TMP20]]
+; CHECK-SVE-MAXBW-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
+; CHECK-SVE-MAXBW-NEXT:    [[TMP22:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-SVE-MAXBW-NEXT:    br i1 [[TMP22]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
+; CHECK-SVE-MAXBW:       middle.block:
+; CHECK-SVE-MAXBW-NEXT:    [[TMP23:%.*]] = call i32 @llvm.vector.reduce.add.nxv8i32(<vscale x 8 x i32> [[TMP21]])
+; CHECK-SVE-MAXBW-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[WIDE_TRIP_COUNT]], [[N_VEC]]
+; CHECK-SVE-MAXBW-NEXT:    br i1 [[CMP_N]], label [[FOR_COND_CLEANUP:%.*]], label [[SCALAR_PH]]
+;
+entry:
+  %cmp28.not = icmp ult i32 %N, 2
+  %div27 = lshr i32 %N, 1
+  %wide.trip.count = zext nneg i32 %div27 to i64
+  br label %for.body
+
+for.cond.cleanup:                                 ; preds = %for.cond.cleanup.loopexit, %entry
+  %res.0.lcssa = phi i32 [ %sub.2, %for.body ]
+  ret i32 %res.0.lcssa
+
+for.body:                                         ; preds = %for.body.preheader, %for.body
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+  %res = phi i32 [ 0, %entry ], [ %sub.2, %for.body ]
+
+  %a.ptr = getelementptr inbounds nuw i8, ptr %a, i64 %indvars.iv
+  %b.ptr = getelementptr inbounds nuw i8, ptr %b, i64 %indvars.iv
+  %c.ptr = getelementptr inbounds nuw i8, ptr %c, i64 %indvars.iv
+  %a.val = load i8, ptr %a.ptr, align 1
+  %b.val = load i8, ptr %b.ptr, align 1
+  %c.val = load i8, ptr %c.ptr, align 1
+
+  %a.ext = sext i8 %a.val to i32
+  %b.ext = sext i8 %b.val to i32
+  %c.ext = sext i8 %c.val to i32
+
+  %mul.ab = mul nsw i32 %a.ext, %b.ext
+  %sub = sub nsw i32 %res, %mul.ab
+  %mul.ac = mul nsw i32 %a.ext, %c.ext
+  %add = add nsw i32 %sub, %mul.ac
+  %mul.bc = mul nsw i32 %b.ext, %c.ext
+  %sub.2 = sub i32 %add, %mul.bc
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %exitcond.not = icmp eq i64 %indvars.iv.next, %wide.trip.count
+  br i1 %exitcond.not, label %for.cond.cleanup, label %for.body, !loop !1
+}
+
+attributes #0 = { vscale_range(1,16) }
+
+
+!0 = !{!"llvm.loop.vectorize.scalable.enable", i1 true}
+!1 = distinct !{!0}
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-dot-product-epilogue.ll b/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-dot-product-epilogue.ll
index 37c489cd0d4cf..ff0f5f3f711f4 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-dot-product-epilogue.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-dot-product-epilogue.ll
@@ -74,8 +74,8 @@ define i32 @dotp(ptr %a, ptr %b) #0 {
 ; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 1024, [[N_VEC]]
 ; CHECK-NEXT:    br i1 [[CMP_N]], label [[FOR_EXIT]], label [[VEC_EPILOG_SCALAR_PH]]
 ; CHECK:       vec.epilog.scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ 0, [[ITER_CHECK:%.*]] ], [ 1024, [[VEC_EPILOG_ITER_CHECK]] ]
-; CHECK-NEXT:    [[BC_MERGE_RDX7:%.*]] = phi i32 [ [[TMP29]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ 0, [[ITER_CHECK]] ], [ [[TMP11]], [[VEC_EPILOG_ITER_CHECK]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ 1024, [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[ITER_CHECK:%.*]] ]
+; CHECK-NEXT:    [[BC_MERGE_RDX7:%.*]] = phi i32 [ [[TMP29]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP11]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[ITER_CHECK]] ]
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK:       for.body:
 ; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[VEC_EPILOG_SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
@@ -122,7 +122,7 @@ define void @dotp_small_epilogue_vf(i64 %idx.neg, i8 %a) #1 {
 ; CHECK-SAME: i64 [[IDX_NEG:%.*]], i8 [[A:%.*]]) #[[ATTR1:[0-9]+]] {
 ; CHECK-NEXT:  iter.check:
 ; CHECK-NEXT:    [[TMP0:%.*]] = sub i64 1, [[IDX_NEG]]
-; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], 8
+; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], 4
 ; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[VEC_EPILOG_SCALAR_PH:%.*]], label [[ENTRY:%.*]]
 ; CHECK:       vector.main.loop.iter.check:
 ; CHECK-NEXT:    [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 [[TMP0]], 16
@@ -132,78 +132,78 @@ define void @dotp_small_epilogue_vf(i64 %idx.neg, i8 %a) #1 {
 ; CHECK-NEXT:    [[IV_NEXT:%.*]] = sub i64 [[TMP0]], [[N_MOD_VF]]
 ; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <16 x i8> poison, i8 [[A]], i64 0
 ; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <16 x i8> [[BROADCAST_SPLATINSERT]], <16 x i8> poison, <16 x i32> zeroinitializer
-; CHECK-NEXT:    [[TMP1:%.*]] = sext <16 x i8> [[BROADCAST_SPLAT]] to <16 x i64>
+; CHECK-NEXT:    [[TMP1:%.*]] = sext <16 x i8> [[BROADCAST_SPLAT]] to <16 x i32>
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <2 x i64> [ zeroinitializer, [[VECTOR_PH]] ], [ [[PARTIAL_REDUCE:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[PARTIAL_REDUCE:%.*]], [[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[TMP2:%.*]] = load i8, ptr null, align 1
 ; CHECK-NEXT:    [[BROADCAST_SPLATINSERT2:%.*]] = insertelement <16 x i8> poison, i8 [[TMP2]], i64 0
 ; CHECK-NEXT:    [[BROADCAST_SPLAT3:%.*]] = shufflevector <16 x i8> [[BROADCAST_SPLATINSERT2]], <16 x i8> poison, <16 x i32> zeroinitializer
-; CHECK-NEXT:    [[TMP3:%.*]] = sext <16 x i8> [[BROADCAST_SPLAT3]] to <16 x i64>
-; CHECK-NEXT:    [[TMP4:%.*]] = mul <16 x i64> [[TMP3]], [[TMP1]]
-; CHECK-NEXT:    [[PARTIAL_REDUCE]] = call <2 x i64> @llvm.experimental.vector.partial.reduce.add.v2i64.v16i64(<2 x i64> [[VEC_PHI]], <16 x i64> [[TMP4]])
+; CHECK-NEXT:    [[TMP3:%.*]] = sext <16 x i8> [[BROADCAST_SPLAT3]] to <16 x i32>
+; CHECK-NEXT:    [[TMP4:%.*]] = mul <16 x i32> [[TMP3]], [[TMP1]]
+; CHECK-NEXT:    [[PARTIAL_REDUCE]] = call <4 x i32> @llvm.experimental.vector.partial.reduce.add.v4i32.v16i32(<4 x i32> [[VEC_PHI]], <16 x i32> [[TMP4]])
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
 ; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[IV_NEXT]]
 ; CHECK-NEXT:    br i1 [[TMP5]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
 ; CHECK:       middle.block:
-; CHECK-NEXT:    [[ADD:%.*]] = call i64 @llvm.vector.reduce.add.v2i64(<2 x i64> [[PARTIAL_REDUCE]])
+; CHECK-NEXT:    [[TMP6:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[PARTIAL_REDUCE]])
 ; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[IV_NEXT]]
 ; CHECK-NEXT:    br i1 [[CMP_N]], label [[WHILE_END_LOOPEXIT:%.*]], label [[WHILE_BODY:%.*]]
 ; CHECK:       vec.epilog.iter.check:
 ; CHECK-NEXT:    [[IND_END6:%.*]] = add i64 [[IDX_NEG]], [[IV_NEXT]]
 ; CHECK-NEXT:    [[N_VEC_REMAINING:%.*]] = sub i64 [[TMP0]], [[IV_NEXT]]
-; CHECK-NEXT:    [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_VEC_REMAINING]], 8
+; CHECK-NEXT:    [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_VEC_REMAINING]], 4
 ; CHECK-NEXT:    br i1 [[MIN_EPILOG_ITERS_CHECK]], label [[VEC_EPILOG_SCALAR_PH]], label [[VEC_EPILOG_PH]]
 ; CHECK:       vec.epilog.ph:
 ; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[IV_NEXT]], [[WHILE_BODY]] ], [ 0, [[ENTRY]] ]
-; CHECK-NEXT:    [[ACCUM:%.*]] = phi i64 [ [[ADD]], [[WHILE_BODY]] ], [ 0, [[ENTRY]] ]
-; CHECK-NEXT:    [[N_MOD_VF4:%.*]] = urem i64 [[TMP0]], 8
+; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP6]], [[WHILE_BODY]] ], [ 0, [[ENTRY]] ]
+; CHECK-NEXT:    [[N_MOD_VF4:%.*]] = urem i64 [[TMP0]], 4
 ; CHECK-NEXT:    [[N_VEC5:%.*]] = sub i64 [[TMP0]], [[N_MOD_VF4]]
 ; CHECK-NEXT:    [[IND_END:%.*]] = add i64 [[IDX_NEG]], [[N_VEC5]]
-; CHECK-NEXT:    [[BROADCAST_SPLATINSERT7:%.*]] = insertelement <8 x i8> poison, i8 [[A]], i64 0
-; CHECK-NEXT:    [[BROADCAST_SPLAT8:%.*]] = shufflevector <8 x i8> [[BROADCAST_SPLATINSERT7]], <8 x i8> poison, <8 x i32> zeroinitializer
-; CHECK-NEXT:    [[TMP7:%.*]] = sext <8 x i8> [[BROADCAST_SPLAT8]] to <8 x i64>
-; CHECK-NEXT:    [[TMP8:%.*]] = insertelement <1 x i64> zeroinitializer, i64 [[ACCUM]], i32 0
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT6:%.*]] = insertelement <4 x i8> poison, i8 [[A]], i64 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT7:%.*]] = shufflevector <4 x i8> [[BROADCAST_SPLATINSERT6]], <4 x i8> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP8:%.*]] = sext <4 x i8> [[BROADCAST_SPLAT7]] to <4 x i32>
+; CHECK-NEXT:    [[TMP10:%.*]] = insertelement <4 x i32> zeroinitializer, i32 [[BC_MERGE_RDX]], i32 0
 ; CHECK-NEXT:    br label [[VEC_EPILOG_VECTOR_BODY:%.*]]
 ; CHECK:       vec.epilog.vector.body:
 ; CHECK-NEXT:    [[INDEX9:%.*]] = phi i64 [ [[IV]], [[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT14:%.*]], [[VEC_EPILOG_VECTOR_BODY]] ]
-; CHECK-NEXT:    [[VEC_PHI10:%.*]] = phi <1 x i64> [ [[TMP8]], [[VEC_EPILOG_PH]] ], [ [[PARTIAL_REDUCE13:%.*]], [[VEC_EPILOG_VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_PHI9:%.*]] = phi <4 x i32> [ [[TMP10]], [[VEC_EPILOG_PH]] ], [ [[TMP13:%.*]], [[VEC_EPILOG_VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[TMP9:%.*]] = load i8, ptr null, align 1
-; CHECK-NEXT:    [[BROADCAST_SPLATINSERT11:%.*]] = insertelement <8 x i8> poison, i8 [[TMP9]], i64 0
-; CHECK-NEXT:    [[BROADCAST_SPLAT12:%.*]] = shufflevector <8 x i8> [[BROADCAST_SPLATINSERT11]], <8 x i8> poison, <8 x i32> zeroinitializer
-; CHECK-NEXT:    [[TMP10:%.*]] = sext <8 x i8> [[BROADCAST_SPLAT12]] to <8 x i64>
-; CHECK-NEXT:    [[TMP11:%.*]] = mul <8 x i64> [[TMP10]], [[TMP7]]
-; CHECK-NEXT:    [[PARTIAL_REDUCE13]] = call <1 x i64> @llvm.experimental.vector.partial.reduce.add.v1i64.v8i64(<1 x i64> [[VEC_PHI10]], <8 x i64> [[TMP11]])
-; CHECK-NEXT:    [[INDEX_NEXT14]] = add nuw i64 [[INDEX9]], 8
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT10:%.*]] = insertelement <4 x i8> poison, i8 [[TMP9]], i64 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT11:%.*]] = shufflevector <4 x i8> [[BROADCAST_SPLATINSERT10]], <4 x i8> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP11:%.*]] = sext <4 x i8> [[BROADCAST_SPLAT11]] to <4 x i32>
+; CHECK-NEXT:    [[TMP14:%.*]] = mul <4 x i32> [[TMP11]], [[TMP8]]
+; CHECK-NEXT:    [[TMP13]] = add <4 x i32> [[TMP14]], [[VEC_PHI9]]
+; CHECK-NEXT:    [[INDEX_NEXT14]] = add nuw i64 [[INDEX9]], 4
 ; CHECK-NEXT:    [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT14]], [[N_VEC5]]
 ; CHECK-NEXT:    br i1 [[TMP12]], label [[VEC_EPILOG_MIDDLE_BLOCK:%.*]], label [[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
 ; CHECK:       vec.epilog.middle.block:
-; CHECK-NEXT:    [[TMP13:%.*]] = call i64 @llvm.vector.reduce.add.v1i64(<1 x i64> [[PARTIAL_REDUCE13]])
+; CHECK-NEXT:    [[TMP15:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP13]])
 ; CHECK-NEXT:    [[CMP_N15:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC5]]
 ; CHECK-NEXT:    br i1 [[CMP_N15]], label [[WHILE_END_LOOPEXIT]], label [[VEC_EPILOG_SCALAR_PH]]
 ; CHECK:       vec.epilog.scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[IND_END]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[IDX_NEG]], [[ITER_CHECK:%.*]] ], [ [[IND_END6]], [[WHILE_BODY]] ]
-; CHECK-NEXT:    [[BC_RESUME_VAL16:%.*]] = phi i64 [ [[N_VEC5]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ 0, [[ITER_CHECK]] ], [ [[IV_NEXT]], [[WHILE_BODY]] ]
-; CHECK-NEXT:    [[BC_MERGE_RDX17:%.*]] = phi i64 [ [[TMP13]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ 0, [[ITER_CHECK]] ], [ [[ADD]], [[WHILE_BODY]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[IND_END]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[IND_END6]], [[WHILE_BODY]] ], [ [[IDX_NEG]], [[ITER_CHECK:%.*]] ]
+; CHECK-NEXT:    [[BC_MERGE_RDX17:%.*]] = phi i64 [ [[N_VEC5]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[IV_NEXT]], [[WHILE_BODY]] ], [ 0, [[ITER_CHECK]] ]
+; CHECK-NEXT:    [[BC_MERGE_RDX15:%.*]] = phi i32 [ [[TMP15]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP6]], [[WHILE_BODY]] ], [ 0, [[ITER_CHECK]] ]
 ; CHECK-NEXT:    br label [[WHILE_BODY1:%.*]]
 ; CHECK:       while.body:
 ; CHECK-NEXT:    [[IV_NEG:%.*]] = phi i64 [ [[IV_NEG_NEXT:%.*]], [[WHILE_BODY1]] ], [ [[BC_RESUME_VAL]], [[VEC_EPILOG_SCALAR_PH]] ]
-; CHECK-NEXT:    [[IV1:%.*]] = phi i64 [ [[IV_NEXT1:%.*]], [[WHILE_BODY1]] ], [ [[BC_RESUME_VAL16]], [[VEC_EPILOG_SCALAR_PH]] ]
 ; CHECK-NEXT:    [[ACCUM1:%.*]] = phi i64 [ [[ADD1:%.*]], [[WHILE_BODY1]] ], [ [[BC_MERGE_RDX17]], [[VEC_EPILOG_SCALAR_PH]] ]
+; CHECK-NEXT:    [[ACCUM:%.*]] = phi i32 [ [[ADD:%.*]], [[WHILE_BODY1]] ], [ [[BC_MERGE_RDX15]], [[VEC_EPILOG_SCALAR_PH]] ]
 ; CHECK-NEXT:    [[IV_NEG_NEXT]] = add i64 [[IV_NEG]], 1
-; CHECK-NEXT:    [[EXT_A:%.*]] = sext i8 [[A]] to i64
-; CHECK-NEXT:    [[IV_NEXT1]] = add i64 [[IV1]], 1
+; CHECK-NEXT:    [[EXT_A:%.*]] = sext i8 [[A]] to i32
+; CHECK-NEXT:    [[ADD1]] = add i64 [[ACCUM1]], 1
 ; CHECK-NEXT:    [[B:%.*]] = load i8, ptr null, align 1
-; CHECK-NEXT:    [[EXT_B:%.*]] = sext i8 [[B]] to i64
-; CHECK-NEXT:    [[MUL:%.*]] = mul i64 [[EXT_B]], [[EXT_A]]
-; CHECK-NEXT:    [[ADD1]] = add i64 [[MUL]], [[ACCUM1]]
+; CHECK-NEXT:    [[EXT_B:%.*]] = sext i8 [[B]] to i32
+; CHECK-NEXT:    [[MUL:%.*]] = mul i32 [[EXT_B]], [[EXT_A]]
+; CHECK-NEXT:    [[ADD]] = add i32 [[MUL]], [[ACCUM]]
 ; CHECK-NEXT:    [[CMP_IV_NEG:%.*]] = icmp ugt i64 [[IV_NEG]], 0
-; CHECK-NEXT:    [[CMP_IV:%.*]] = icmp ne i64 [[IV1]], -1
+; CHECK-NEXT:    [[CMP_IV:%.*]] = icmp ne i64 [[ACCUM1]], -1
 ; CHECK-NEXT:    [[EXITCOND:%.*]] = and i1 [[CMP_IV_NEG]], [[CMP_IV]]
 ; CHECK-NEXT:    br i1 [[EXITCOND]], label [[WHILE_BODY1]], label [[WHILE_END_LOOPEXIT]], !llvm.loop [[LOOP7:![0-9]+]]
 ; CHECK:       while.end.loopexit:
-; CHECK-NEXT:    [[RESULT:%.*]] = phi i64 [ [[ADD1]], [[WHILE_BODY1]] ], [ [[ADD]], [[MIDDLE_BLOCK]] ], [ [[TMP13]], [[VEC_EPILOG_MIDDLE_BLOCK]] ]
+; CHECK-NEXT:    [[RESULT:%.*]] = phi i32 [ [[ADD]], [[WHILE_BODY1]] ], [ [[TMP6]], [[MIDDLE_BLOCK]] ], [ [[TMP15]], [[VEC_EPILOG_MIDDLE_BLOCK]] ]
 ; CHECK-NEXT:    ret void
 ;
 entry:
@@ -212,24 +212,407 @@ entry:
 while.body:                                       ; preds = %while.body, %entry
   %iv.neg = phi i64 [ %iv.neg.next, %while.body ], [ %idx.neg, %entry ]
   %iv = phi i64 [ %iv.next, %while.body ], [ 0, %entry ]
-  %accum = phi i64 [ %add, %while.body ], [ 0, %entry ]
+  %accum = phi i32 [ %add, %while.body ], [ 0, %entry ]
   %iv.neg.next = add i64 %iv.neg, 1
-  %ext.a = sext i8 %a to i64
+  %ext.a = sext i8 %a to i32
   %iv.next = add i64 %iv, 1
   %b = load i8, ptr null, align 1
-  %ext.b = sext i8 %b to i64
-  %mul = mul i64 %ext.b, %ext.a
-  %add = add i64 %mul, %accum
+  %ext.b = sext i8 %b to i32
+  %mul = mul i32 %ext.b, %ext.a
+  %add = add i32 %mul, %accum
   %cmp.iv.neg = icmp ugt i64 %iv.neg, 0
   %cmp.iv = icmp ne i64 %iv, -1
   %exitcond = and i1 %cmp.iv.neg, %cmp.iv
   br i1 %exitcond, label %while.body, label %while.end.loopexit
 
 while.end.loopexit:                               ; preds = %while.body
-  %result = phi i64 [ %add, %while.body ]
+  %result = phi i32 [ %add, %while.body ]
   ret void
 }
 
+define i32 @dotp_predicated(i64 %N, ptr %a, ptr %b) {
+; CHECK-LABEL: define i32 @dotp_predicated(
+; CHECK-SAME: i64 [[N:%.*]], ptr [[A:%.*]], ptr [[B:%.*]]) #[[ATTR2:[0-9]+]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK:       vector.ph:
+; CHECK-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N]], 15
+; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], 16
+; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
+; CHECK-NEXT:    [[TRIP_COUNT_MINUS_1:%.*]] = sub i64 [[N]], 1
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <16 x i64> poison, i64 [[TRIP_COUNT_MINUS_1]], i64 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <16 x i64> [[BROADCAST_SPLATINSERT]], <16 x i64> poison, <16 x i32> zeroinitializer
+; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK:       vector.body:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_LOAD_CONTINUE62:%.*]] ]
+; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <16 x i64> [ <i64 0, i64 1, i64 2, i64 3, i64 4, i64 5, i64 6, i64 7, i64 8, i64 9, i64 10, i64 11, i64 12, i64 13, i64 14, i64 15>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[PRED_LOAD_CONTINUE62]] ]
+; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[PARTIAL_REDUCE:%.*]], [[PRED_LOAD_CONTINUE62]] ]
+; CHECK-NEXT:    [[TMP0:%.*]] = add i64 [[INDEX]], 0
+; CHECK-NEXT:    [[TMP1:%.*]] = add i64 [[INDEX]], 1
+; CHECK-NEXT:    [[TMP2:%.*]] = add i64 [[INDEX]], 2
+; CHECK-NEXT:    [[TMP3:%.*]] = add i64 [[INDEX]], 3
+; CHECK-NEXT:    [[TMP4:%.*]] = add i64 [[INDEX]], 4
+; CHECK-NEXT:    [[TMP5:%.*]] = add i64 [[INDEX]], 5
+; CHECK-NEXT:    [[TMP6:%.*]] = add i64 [[INDEX]], 6
+; CHECK-NEXT:    [[TMP7:%.*]] = add i64 [[INDEX]], 7
+; CHECK-NEXT:    [[TMP8:%.*]] = add i64 [[INDEX]], 8
+; CHECK-NEXT:    [[TMP9:%.*]] = add i64 [[INDEX]], 9
+; CHECK-NEXT:    [[TMP10:%.*]] = add i64 [[INDEX]], 10
+; CHECK-NEXT:    [[TMP11:%.*]] = add i64 [[INDEX]], 11
+; CHECK-NEXT:    [[TMP12:%.*]] = add i64 [[INDEX]], 12
+; CHECK-NEXT:    [[TMP13:%.*]] = add i64 [[INDEX]], 13
+; CHECK-NEXT:    [[TMP14:%.*]] = add i64 [[INDEX]], 14
+; CHECK-NEXT:    [[TMP15:%.*]] = add i64 [[INDEX]], 15
+; CHECK-NEXT:    [[TMP16:%.*]] = icmp ule <16 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT:    [[TMP17:%.*]] = extractelement <16 x i1> [[TMP16]], i32 0
+; CHECK-NEXT:    br i1 [[TMP17]], label [[PRED_LOAD_IF:%.*]], label [[PRED_LOAD_CONTINUE:%.*]]
+; CHECK:       pred.load.if:
+; CHECK-NEXT:    [[TMP18:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[TMP0]]
+; CHECK-NEXT:    [[TMP19:%.*]] = load i8, ptr [[TMP18]], align 1
+; CHECK-NEXT:    [[TMP20:%.*]] = insertelement <16 x i8> poison, i8 [[TMP19]], i32 0
+; CHECK-NEXT:    br label [[PRED_LOAD_CONTINUE]]
+; CHECK:       pred.load.continue:
+; CHECK-NEXT:    [[TMP21:%.*]] = phi <16 x i8> [ poison, [[VECTOR_BODY]] ], [ [[TMP20]], [[PRED_LOAD_IF]] ]
+; CHECK-NEXT:    [[TMP22:%.*]] = extractelement <16 x i1> [[TMP16]], i32 1
+; CHECK-NEXT:    br i1 [[TMP22]], label [[PRED_LOAD_IF1:%.*]], label [[PRED_LOAD_CONTINUE2:%.*]]
+; CHECK:       pred.load.if1:
+; CHECK-NEXT:    [[TMP23:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[TMP1]]
+; CHECK-NEXT:    [[TMP24:%.*]] = load i8, ptr [[TMP23]], align 1
+; CHECK-NEXT:    [[TMP25:%.*]] = insertelement <16 x i8> [[TMP21]], i8 [[TMP24]], i32 1
+; CHECK-NEXT:    br label [[PRED_LOAD_CONTINUE2]]
+; CHECK:       pred.load.continue2:
+; CHECK-NEXT:    [[TMP26:%.*]] = phi <16 x i8> [ [[TMP21]], [[PRED_LOAD_CONTINUE]] ], [ [[TMP25]], [[PRED_LOAD_IF1]] ]
+; CHECK-NEXT:    [[TMP27:%.*]] = extractelement <16 x i1> [[TMP16]], i32 2
+; CHECK-NEXT:    br i1 [[TMP27]], label [[PRED_LOAD_IF3:%.*]], label [[PRED_LOAD_CONTINUE4:%.*]]
+; CHECK:       pred.load.if3:
+; CHECK-NEXT:    [[TMP28:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[TMP2]]
+; CHECK-NEXT:    [[TMP29:%.*]] = load i8, ptr [[TMP28]], align 1
+; CHECK-NEXT:    [[TMP30:%.*]] = insertelement <16 x i8> [[TMP26]], i8 [[TMP29]], i32 2
+; CHECK-NEXT:    br label [[PRED_LOAD_CONTINUE4]]
+; CHECK:       pred.load.continue4:
+; CHECK-NEXT:    [[TMP31:%.*]] = phi <16 x i8> [ [[TMP26]], [[PRED_LOAD_CONTINUE2]] ], [ [[TMP30]], [[PRED_LOAD_IF3]] ]
+; CHECK-NEXT:    [[TMP32:%.*]] = extractelement <16 x i1> [[TMP16]], i32 3
+; CHECK-NEXT:    br i1 [[TMP32]], label [[PRED_LOAD_IF5:%.*]], label [[PRED_LOAD_CONTINUE6:%.*]]
+; CHECK:       pred.load.if5:
+; CHECK-NEXT:    [[TMP33:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[TMP3]]
+; CHECK-NEXT:    [[TMP34:%.*]] = load i8, ptr [[TMP33]], align 1
+; CHECK-NEXT:    [[TMP35:%.*]] = insertelement <16 x i8> [[TMP31]], i8 [[TMP34]], i32 3
+; CHECK-NEXT:    br label [[PRED_LOAD_CONTINUE6]]
+; CHECK:       pred.load.continue6:
+; CHECK-NEXT:    [[TMP36:%.*]] = phi <16 x i8> [ [[TMP31]], [[PRED_LOAD_CONTINUE4]] ], [ [[TMP35]], [[PRED_LOAD_IF5]] ]
+; CHECK-NEXT:    [[TMP37:%.*]] = extractelement <16 x i1> [[TMP16]], i32 4
+; CHECK-NEXT:    br i1 [[TMP37]], label [[PRED_LOAD_IF7:%.*]], label [[PRED_LOAD_CONTINUE8:%.*]]
+; CHECK:       pred.load.if7:
+; CHECK-NEXT:    [[TMP38:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[TMP4]]
+; CHECK-NEXT:    [[TMP39:%.*]] = load i8, ptr [[TMP38]], align 1
+; CHECK-NEXT:    [[TMP40:%.*]] = insertelement <16 x i8> [[TMP36]], i8 [[TMP39]], i32 4
+; CHECK-NEXT:    br label [[PRED_LOAD_CONTINUE8]]
+; CHECK:       pred.load.continue8:
+; CHECK-NEXT:    [[TMP41:%.*]] = phi <16 x i8> [ [[TMP36]], [[PRED_LOAD_CONTINUE6]] ], [ [[TMP40]], [[PRED_LOAD_IF7]] ]
+; CHECK-NEXT:    [[TMP42:%.*]] = extractelement <16 x i1> [[TMP16]], i32 5
+; CHECK-NEXT:    br i1 [[TMP42]], label [[PRED_LOAD_IF9:%.*]], label [[PRED_LOAD_CONTINUE10:%.*]]
+; CHECK:       pred.load.if9:
+; CHECK-NEXT:    [[TMP43:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[TMP5]]
+; CHECK-NEXT:    [[TMP44:%.*]] = load i8, ptr [[TMP43]], align 1
+; CHECK-NEXT:    [[TMP45:%.*]] = insertelement <16 x i8> [[TMP41]], i8 [[TMP44]], i32 5
+; CHECK-NEXT:    br label [[PRED_LOAD_CONTINUE10]]
+; CHECK:       pred.load.continue10:
+; CHECK-NEXT:    [[TMP46:%.*]] = phi <16 x i8> [ [[TMP41]], [[PRED_LOAD_CONTINUE8]] ], [ [[TMP45]], [[PRED_LOAD_IF9]] ]
+; CHECK-NEXT:    [[TMP47:%.*]] = extractelement <16 x i1> [[TMP16]], i32 6
+; CHECK-NEXT:    br i1 [[TMP47]], label [[PRED_LOAD_IF11:%.*]], label [[PRED_LOAD_CONTINUE12:%.*]]
+; CHECK:       pred.load.if11:
+; CHECK-NEXT:    [[TMP48:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[TMP6]]
+; CHECK-NEXT:    [[TMP49:%.*]] = load i8, ptr [[TMP48]], align 1
+; CHECK-NEXT:    [[TMP50:%.*]] = insertelement <16 x i8> [[TMP46]], i8 [[TMP49]], i32 6
+; CHECK-NEXT:    br label [[PRED_LOAD_CONTINUE12]]
+; CHECK:       pred.load.continue12:
+; CHECK-NEXT:    [[TMP51:%.*]] = phi <16 x i8> [ [[TMP46]], [[PRED_LOAD_CONTINUE10]] ], [ [[TMP50]], [[PRED_LOAD_IF11]] ]
+; CHECK-NEXT:    [[TMP52:%.*]] = extractelement <16 x i1> [[TMP16]], i32 7
+; CHECK-NEXT:    br i1 [[TMP52]], label [[PRED_LOAD_IF13:%.*]], label [[PRED_LOAD_CONTINUE14:%.*]]
+; CHECK:       pred.load.if13:
+; CHECK-NEXT:    [[TMP53:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[TMP7]]
+; CHECK-NEXT:    [[TMP54:%.*]] = load i8, ptr [[TMP53]], align 1
+; CHECK-NEXT:    [[TMP55:%.*]] = insertelement <16 x i8> [[TMP51]], i8 [[TMP54]], i32 7
+; CHECK-NEXT:    br label [[PRED_LOAD_CONTINUE14]]
+; CHECK:       pred.load.continue14:
+; CHECK-NEXT:    [[TMP56:%.*]] = phi <16 x i8> [ [[TMP51]], [[PRED_LOAD_CONTINUE12]] ], [ [[TMP55]], [[PRED_LOAD_IF13]] ]
+; CHECK-NEXT:    [[TMP57:%.*]] = extractelement <16 x i1> [[TMP16]], i32 8
+; CHECK-NEXT:    br i1 [[TMP57]], label [[PRED_LOAD_IF15:%.*]], label [[PRED_LOAD_CONTINUE16:%.*]]
+; CHECK:       pred.load.if15:
+; CHECK-NEXT:    [[TMP58:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[TMP8]]
+; CHECK-NEXT:    [[TMP59:%.*]] = load i8, ptr [[TMP58]], align 1
+; CHECK-NEXT:    [[TMP60:%.*]] = insertelement <16 x i8> [[TMP56]], i8 [[TMP59]], i32 8
+; CHECK-NEXT:    br label [[PRED_LOAD_CONTINUE16]]
+; CHECK:       pred.load.continue16:
+; CHECK-NEXT:    [[TMP61:%.*]] = phi <16 x i8> [ [[TMP56]], [[PRED_LOAD_CONTINUE14]] ], [ [[TMP60]], [[PRED_LOAD_IF15]] ]
+; CHECK-NEXT:    [[TMP62:%.*]] = extractelement <16 x i1> [[TMP16]], i32 9
+; CHECK-NEXT:    br i1 [[TMP62]], label [[PRED_LOAD_IF17:%.*]], label [[PRED_LOAD_CONTINUE18:%.*]]
+; CHECK:       pred.load.if17:
+; CHECK-NEXT:    [[TMP63:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[TMP9]]
+; CHECK-NEXT:    [[TMP64:%.*]] = load i8, ptr [[TMP63]], align 1
+; CHECK-NEXT:    [[TMP65:%.*]] = insertelement <16 x i8> [[TMP61]], i8 [[TMP64]], i32 9
+; CHECK-NEXT:    br label [[PRED_LOAD_CONTINUE18]]
+; CHECK:       pred.load.continue18:
+; CHECK-NEXT:    [[TMP66:%.*]] = phi <16 x i8> [ [[TMP61]], [[PRED_LOAD_CONTINUE16]] ], [ [[TMP65]], [[PRED_LOAD_IF17]] ]
+; CHECK-NEXT:    [[TMP67:%.*]] = extractelement <16 x i1> [[TMP16]], i32 10
+; CHECK-NEXT:    br i1 [[TMP67]], label [[PRED_LOAD_IF19:%.*]], label [[PRED_LOAD_CONTINUE20:%.*]]
+; CHECK:       pred.load.if19:
+; CHECK-NEXT:    [[TMP68:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[TMP10]]
+; CHECK-NEXT:    [[TMP69:%.*]] = load i8, ptr [[TMP68]], align 1
+; CHECK-NEXT:    [[TMP70:%.*]] = insertelement <16 x i8> [[TMP66]], i8 [[TMP69]], i32 10
+; CHECK-NEXT:    br label [[PRED_LOAD_CONTINUE20]]
+; CHECK:       pred.load.continue20:
+; CHECK-NEXT:    [[TMP71:%.*]] = phi <16 x i8> [ [[TMP66]], [[PRED_LOAD_CONTINUE18]] ], [ [[TMP70]], [[PRED_LOAD_IF19]] ]
+; CHECK-NEXT:    [[TMP72:%.*]] = extractelement <16 x i1> [[TMP16]], i32 11
+; CHECK-NEXT:    br i1 [[TMP72]], label [[PRED_LOAD_IF21:%.*]], label [[PRED_LOAD_CONTINUE22:%.*]]
+; CHECK:       pred.load.if21:
+; CHECK-NEXT:    [[TMP73:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[TMP11]]
+; CHECK-NEXT:    [[TMP74:%.*]] = load i8, ptr [[TMP73]], align 1
+; CHECK-NEXT:    [[TMP75:%.*]] = insertelement <16 x i8> [[TMP71]], i8 [[TMP74]], i32 11
+; CHECK-NEXT:    br label [[PRED_LOAD_CONTINUE22]]
+; CHECK:       pred.load.continue22:
+; CHECK-NEXT:    [[TMP76:%.*]] = phi <16 x i8> [ [[TMP71]], [[PRED_LOAD_CONTINUE20]] ], [ [[TMP75]], [[PRED_LOAD_IF21]] ]
+; CHECK-NEXT:    [[TMP77:%.*]] = extractelement <16 x i1> [[TMP16]], i32 12
+; CHECK-NEXT:    br i1 [[TMP77]], label [[PRED_LOAD_IF23:%.*]], label [[PRED_LOAD_CONTINUE24:%.*]]
+; CHECK:       pred.load.if23:
+; CHECK-NEXT:    [[TMP78:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[TMP12]]
+; CHECK-NEXT:    [[TMP79:%.*]] = load i8, ptr [[TMP78]], align 1
+; CHECK-NEXT:    [[TMP80:%.*]] = insertelement <16 x i8> [[TMP76]], i8 [[TMP79]], i32 12
+; CHECK-NEXT:    br label [[PRED_LOAD_CONTINUE24]]
+; CHECK:       pred.load.continue24:
+; CHECK-NEXT:    [[TMP81:%.*]] = phi <16 x i8> [ [[TMP76]], [[PRED_LOAD_CONTINUE22]] ], [ [[TMP80]], [[PRED_LOAD_IF23]] ]
+; CHECK-NEXT:    [[TMP82:%.*]] = extractelement <16 x i1> [[TMP16]], i32 13
+; CHECK-NEXT:    br i1 [[TMP82]], label [[PRED_LOAD_IF25:%.*]], label [[PRED_LOAD_CONTINUE26:%.*]]
+; CHECK:       pred.load.if25:
+; CHECK-NEXT:    [[TMP83:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[TMP13]]
+; CHECK-NEXT:    [[TMP84:%.*]] = load i8, ptr [[TMP83]], align 1
+; CHECK-NEXT:    [[TMP85:%.*]] = insertelement <16 x i8> [[TMP81]], i8 [[TMP84]], i32 13
+; CHECK-NEXT:    br label [[PRED_LOAD_CONTINUE26]]
+; CHECK:       pred.load.continue26:
+; CHECK-NEXT:    [[TMP86:%.*]] = phi <16 x i8> [ [[TMP81]], [[PRED_LOAD_CONTINUE24]] ], [ [[TMP85]], [[PRED_LOAD_IF25]] ]
+; CHECK-NEXT:    [[TMP87:%.*]] = extractelement <16 x i1> [[TMP16]], i32 14
+; CHECK-NEXT:    br i1 [[TMP87]], label [[PRED_LOAD_IF27:%.*]], label [[PRED_LOAD_CONTINUE28:%.*]]
+; CHECK:       pred.load.if27:
+; CHECK-NEXT:    [[TMP88:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[TMP14]]
+; CHECK-NEXT:    [[TMP89:%.*]] = load i8, ptr [[TMP88]], align 1
+; CHECK-NEXT:    [[TMP90:%.*]] = insertelement <16 x i8> [[TMP86]], i8 [[TMP89]], i32 14
+; CHECK-NEXT:    br label [[PRED_LOAD_CONTINUE28]]
+; CHECK:       pred.load.continue28:
+; CHECK-NEXT:    [[TMP91:%.*]] = phi <16 x i8> [ [[TMP86]], [[PRED_LOAD_CONTINUE26]] ], [ [[TMP90]], [[PRED_LOAD_IF27]] ]
+; CHECK-NEXT:    [[TMP92:%.*]] = extractelement <16 x i1> [[TMP16]], i32 15
+; CHECK-NEXT:    br i1 [[TMP92]], label [[PRED_LOAD_IF29:%.*]], label [[PRED_LOAD_CONTINUE30:%.*]]
+; CHECK:       pred.load.if29:
+; CHECK-NEXT:    [[TMP93:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[TMP15]]
+; CHECK-NEXT:    [[TMP94:%.*]] = load i8, ptr [[TMP93]], align 1
+; CHECK-NEXT:    [[TMP95:%.*]] = insertelement <16 x i8> [[TMP91]], i8 [[TMP94]], i32 15
+; CHECK-NEXT:    br label [[PRED_LOAD_CONTINUE30]]
+; CHECK:       pred.load.continue30:
+; CHECK-NEXT:    [[TMP96:%.*]] = phi <16 x i8> [ [[TMP91]], [[PRED_LOAD_CONTINUE28]] ], [ [[TMP95]], [[PRED_LOAD_IF29]] ]
+; CHECK-NEXT:    [[TMP97:%.*]] = sext <16 x i8> [[TMP96]] to <16 x i32>
+; CHECK-NEXT:    [[TMP98:%.*]] = extractelement <16 x i1> [[TMP16]], i32 0
+; CHECK-NEXT:    br i1 [[TMP98]], label [[PRED_LOAD_IF31:%.*]], label [[PRED_LOAD_CONTINUE32:%.*]]
+; CHECK:       pred.load.if31:
+; CHECK-NEXT:    [[TMP99:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[TMP0]]
+; CHECK-NEXT:    [[TMP100:%.*]] = load i8, ptr [[TMP99]], align 1
+; CHECK-NEXT:    [[TMP101:%.*]] = insertelement <16 x i8> poison, i8 [[TMP100]], i32 0
+; CHECK-NEXT:    br label [[PRED_LOAD_CONTINUE32]]
+; CHECK:       pred.load.continue32:
+; CHECK-NEXT:    [[TMP102:%.*]] = phi <16 x i8> [ poison, [[PRED_LOAD_CONTINUE30]] ], [ [[TMP101]], [[PRED_LOAD_IF31]] ]
+; CHECK-NEXT:    [[TMP103:%.*]] = extractelement <16 x i1> [[TMP16]], i32 1
+; CHECK-NEXT:    br i1 [[TMP103]], label [[PRED_LOAD_IF33:%.*]], label [[PRED_LOAD_CONTINUE34:%.*]]
+; CHECK:       pred.load.if33:
+; CHECK-NEXT:    [[TMP104:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[TMP1]]
+; CHECK-NEXT:    [[TMP105:%.*]] = load i8, ptr [[TMP104]], align 1
+; CHECK-NEXT:    [[TMP106:%.*]] = insertelement <16 x i8> [[TMP102]], i8 [[TMP105]], i32 1
+; CHECK-NEXT:    br label [[PRED_LOAD_CONTINUE34]]
+; CHECK:       pred.load.continue34:
+; CHECK-NEXT:    [[TMP107:%.*]] = phi <16 x i8> [ [[TMP102]], [[PRED_LOAD_CONTINUE32]] ], [ [[TMP106]], [[PRED_LOAD_IF33]] ]
+; CHECK-NEXT:    [[TMP108:%.*]] = extractelement <16 x i1> [[TMP16]], i32 2
+; CHECK-NEXT:    br i1 [[TMP108]], label [[PRED_LOAD_IF35:%.*]], label [[PRED_LOAD_CONTINUE36:%.*]]
+; CHECK:       pred.load.if35:
+; CHECK-NEXT:    [[TMP109:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[TMP2]]
+; CHECK-NEXT:    [[TMP110:%.*]] = load i8, ptr [[TMP109]], align 1
+; CHECK-NEXT:    [[TMP111:%.*]] = insertelement <16 x i8> [[TMP107]], i8 [[TMP110]], i32 2
+; CHECK-NEXT:    br label [[PRED_LOAD_CONTINUE36]]
+; CHECK:       pred.load.continue36:
+; CHECK-NEXT:    [[TMP112:%.*]] = phi <16 x i8> [ [[TMP107]], [[PRED_LOAD_CONTINUE34]] ], [ [[TMP111]], [[PRED_LOAD_IF35]] ]
+; CHECK-NEXT:    [[TMP113:%.*]] = extractelement <16 x i1> [[TMP16]], i32 3
+; CHECK-NEXT:    br i1 [[TMP113]], label [[PRED_LOAD_IF37:%.*]], label [[PRED_LOAD_CONTINUE38:%.*]]
+; CHECK:       pred.load.if37:
+; CHECK-NEXT:    [[TMP114:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[TMP3]]
+; CHECK-NEXT:    [[TMP115:%.*]] = load i8, ptr [[TMP114]], align 1
+; CHECK-NEXT:    [[TMP116:%.*]] = insertelement <16 x i8> [[TMP112]], i8 [[TMP115]], i32 3
+; CHECK-NEXT:    br label [[PRED_LOAD_CONTINUE38]]
+; CHECK:       pred.load.continue38:
+; CHECK-NEXT:    [[TMP117:%.*]] = phi <16 x i8> [ [[TMP112]], [[PRED_LOAD_CONTINUE36]] ], [ [[TMP116]], [[PRED_LOAD_IF37]] ]
+; CHECK-NEXT:    [[TMP118:%.*]] = extractelement <16 x i1> [[TMP16]], i32 4
+; CHECK-NEXT:    br i1 [[TMP118]], label [[PRED_LOAD_IF39:%.*]], label [[PRED_LOAD_CONTINUE40:%.*]]
+; CHECK:       pred.load.if39:
+; CHECK-NEXT:    [[TMP119:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[TMP4]]
+; CHECK-NEXT:    [[TMP120:%.*]] = load i8, ptr [[TMP119]], align 1
+; CHECK-NEXT:    [[TMP121:%.*]] = insertelement <16 x i8> [[TMP117]], i8 [[TMP120]], i32 4
+; CHECK-NEXT:    br label [[PRED_LOAD_CONTINUE40]]
+; CHECK:       pred.load.continue40:
+; CHECK-NEXT:    [[TMP122:%.*]] = phi <16 x i8> [ [[TMP117]], [[PRED_LOAD_CONTINUE38]] ], [ [[TMP121]], [[PRED_LOAD_IF39]] ]
+; CHECK-NEXT:    [[TMP123:%.*]] = extractelement <16 x i1> [[TMP16]], i32 5
+; CHECK-NEXT:    br i1 [[TMP123]], label [[PRED_LOAD_IF41:%.*]], label [[PRED_LOAD_CONTINUE42:%.*]]
+; CHECK:       pred.load.if41:
+; CHECK-NEXT:    [[TMP124:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[TMP5]]
+; CHECK-NEXT:    [[TMP125:%.*]] = load i8, ptr [[TMP124]], align 1
+; CHECK-NEXT:    [[TMP126:%.*]] = insertelement <16 x i8> [[TMP122]], i8 [[TMP125]], i32 5
+; CHECK-NEXT:    br label [[PRED_LOAD_CONTINUE42]]
+; CHECK:       pred.load.continue42:
+; CHECK-NEXT:    [[TMP127:%.*]] = phi <16 x i8> [ [[TMP122]], [[PRED_LOAD_CONTINUE40]] ], [ [[TMP126]], [[PRED_LOAD_IF41]] ]
+; CHECK-NEXT:    [[TMP128:%.*]] = extractelement <16 x i1> [[TMP16]], i32 6
+; CHECK-NEXT:    br i1 [[TMP128]], label [[PRED_LOAD_IF43:%.*]], label [[PRED_LOAD_CONTINUE44:%.*]]
+; CHECK:       pred.load.if43:
+; CHECK-NEXT:    [[TMP129:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[TMP6]]
+; CHECK-NEXT:    [[TMP130:%.*]] = load i8, ptr [[TMP129]], align 1
+; CHECK-NEXT:    [[TMP131:%.*]] = insertelement <16 x i8> [[TMP127]], i8 [[TMP130]], i32 6
+; CHECK-NEXT:    br label [[PRED_LOAD_CONTINUE44]]
+; CHECK:       pred.load.continue44:
+; CHECK-NEXT:    [[TMP132:%.*]] = phi <16 x i8> [ [[TMP127]], [[PRED_LOAD_CONTINUE42]] ], [ [[TMP131]], [[PRED_LOAD_IF43]] ]
+; CHECK-NEXT:    [[TMP133:%.*]] = extractelement <16 x i1> [[TMP16]], i32 7
+; CHECK-NEXT:    br i1 [[TMP133]], label [[PRED_LOAD_IF45:%.*]], label [[PRED_LOAD_CONTINUE46:%.*]]
+; CHECK:       pred.load.if45:
+; CHECK-NEXT:    [[TMP134:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[TMP7]]
+; CHECK-NEXT:    [[TMP135:%.*]] = load i8, ptr [[TMP134]], align 1
+; CHECK-NEXT:    [[TMP136:%.*]] = insertelement <16 x i8> [[TMP132]], i8 [[TMP135]], i32 7
+; CHECK-NEXT:    br label [[PRED_LOAD_CONTINUE46]]
+; CHECK:       pred.load.continue46:
+; CHECK-NEXT:    [[TMP137:%.*]] = phi <16 x i8> [ [[TMP132]], [[PRED_LOAD_CONTINUE44]] ], [ [[TMP136]], [[PRED_LOAD_IF45]] ]
+; CHECK-NEXT:    [[TMP138:%.*]] = extractelement <16 x i1> [[TMP16]], i32 8
+; CHECK-NEXT:    br i1 [[TMP138]], label [[PRED_LOAD_IF47:%.*]], label [[PRED_LOAD_CONTINUE48:%.*]]
+; CHECK:       pred.load.if47:
+; CHECK-NEXT:    [[TMP139:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[TMP8]]
+; CHECK-NEXT:    [[TMP140:%.*]] = load i8, ptr [[TMP139]], align 1
+; CHECK-NEXT:    [[TMP141:%.*]] = insertelement <16 x i8> [[TMP137]], i8 [[TMP140]], i32 8
+; CHECK-NEXT:    br label [[PRED_LOAD_CONTINUE48]]
+; CHECK:       pred.load.continue48:
+; CHECK-NEXT:    [[TMP142:%.*]] = phi <16 x i8> [ [[TMP137]], [[PRED_LOAD_CONTINUE46]] ], [ [[TMP141]], [[PRED_LOAD_IF47]] ]
+; CHECK-NEXT:    [[TMP143:%.*]] = extractelement <16 x i1> [[TMP16]], i32 9
+; CHECK-NEXT:    br i1 [[TMP143]], label [[PRED_LOAD_IF49:%.*]], label [[PRED_LOAD_CONTINUE50:%.*]]
+; CHECK:       pred.load.if49:
+; CHECK-NEXT:    [[TMP144:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[TMP9]]
+; CHECK-NEXT:    [[TMP145:%.*]] = load i8, ptr [[TMP144]], align 1
+; CHECK-NEXT:    [[TMP146:%.*]] = insertelement <16 x i8> [[TMP142]], i8 [[TMP145]], i32 9
+; CHECK-NEXT:    br label [[PRED_LOAD_CONTINUE50]]
+; CHECK:       pred.load.continue50:
+; CHECK-NEXT:    [[TMP147:%.*]] = phi <16 x i8> [ [[TMP142]], [[PRED_LOAD_CONTINUE48]] ], [ [[TMP146]], [[PRED_LOAD_IF49]] ]
+; CHECK-NEXT:    [[TMP148:%.*]] = extractelement <16 x i1> [[TMP16]], i32 10
+; CHECK-NEXT:    br i1 [[TMP148]], label [[PRED_LOAD_IF51:%.*]], label [[PRED_LOAD_CONTINUE52:%.*]]
+; CHECK:       pred.load.if51:
+; CHECK-NEXT:    [[TMP149:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[TMP10]]
+; CHECK-NEXT:    [[TMP150:%.*]] = load i8, ptr [[TMP149]], align 1
+; CHECK-NEXT:    [[TMP151:%.*]] = insertelement <16 x i8> [[TMP147]], i8 [[TMP150]], i32 10
+; CHECK-NEXT:    br label [[PRED_LOAD_CONTINUE52]]
+; CHECK:       pred.load.continue52:
+; CHECK-NEXT:    [[TMP152:%.*]] = phi <16 x i8> [ [[TMP147]], [[PRED_LOAD_CONTINUE50]] ], [ [[TMP151]], [[PRED_LOAD_IF51]] ]
+; CHECK-NEXT:    [[TMP153:%.*]] = extractelement <16 x i1> [[TMP16]], i32 11
+; CHECK-NEXT:    br i1 [[TMP153]], label [[PRED_LOAD_IF53:%.*]], label [[PRED_LOAD_CONTINUE54:%.*]]
+; CHECK:       pred.load.if53:
+; CHECK-NEXT:    [[TMP154:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[TMP11]]
+; CHECK-NEXT:    [[TMP155:%.*]] = load i8, ptr [[TMP154]], align 1
+; CHECK-NEXT:    [[TMP156:%.*]] = insertelement <16 x i8> [[TMP152]], i8 [[TMP155]], i32 11
+; CHECK-NEXT:    br label [[PRED_LOAD_CONTINUE54]]
+; CHECK:       pred.load.continue54:
+; CHECK-NEXT:    [[TMP157:%.*]] = phi <16 x i8> [ [[TMP152]], [[PRED_LOAD_CONTINUE52]] ], [ [[TMP156]], [[PRED_LOAD_IF53]] ]
+; CHECK-NEXT:    [[TMP158:%.*]] = extractelement <16 x i1> [[TMP16]], i32 12
+; CHECK-NEXT:    br i1 [[TMP158]], label [[PRED_LOAD_IF55:%.*]], label [[PRED_LOAD_CONTINUE56:%.*]]
+; CHECK:       pred.load.if55:
+; CHECK-NEXT:    [[TMP159:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[TMP12]]
+; CHECK-NEXT:    [[TMP160:%.*]] = load i8, ptr [[TMP159]], align 1
+; CHECK-NEXT:    [[TMP161:%.*]] = insertelement <16 x i8> [[TMP157]], i8 [[TMP160]], i32 12
+; CHECK-NEXT:    br label [[PRED_LOAD_CONTINUE56]]
+; CHECK:       pred.load.continue56:
+; CHECK-NEXT:    [[TMP162:%.*]] = phi <16 x i8> [ [[TMP157]], [[PRED_LOAD_CONTINUE54]] ], [ [[TMP161]], [[PRED_LOAD_IF55]] ]
+; CHECK-NEXT:    [[TMP163:%.*]] = extractelement <16 x i1> [[TMP16]], i32 13
+; CHECK-NEXT:    br i1 [[TMP163]], label [[PRED_LOAD_IF57:%.*]], label [[PRED_LOAD_CONTINUE58:%.*]]
+; CHECK:       pred.load.if57:
+; CHECK-NEXT:    [[TMP164:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[TMP13]]
+; CHECK-NEXT:    [[TMP165:%.*]] = load i8, ptr [[TMP164]], align 1
+; CHECK-NEXT:    [[TMP166:%.*]] = insertelement <16 x i8> [[TMP162]], i8 [[TMP165]], i32 13
+; CHECK-NEXT:    br label [[PRED_LOAD_CONTINUE58]]
+; CHECK:       pred.load.continue58:
+; CHECK-NEXT:    [[TMP167:%.*]] = phi <16 x i8> [ [[TMP162]], [[PRED_LOAD_CONTINUE56]] ], [ [[TMP166]], [[PRED_LOAD_IF57]] ]
+; CHECK-NEXT:    [[TMP168:%.*]] = extractelement <16 x i1> [[TMP16]], i32 14
+; CHECK-NEXT:    br i1 [[TMP168]], label [[PRED_LOAD_IF59:%.*]], label [[PRED_LOAD_CONTINUE60:%.*]]
+; CHECK:       pred.load.if59:
+; CHECK-NEXT:    [[TMP169:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[TMP14]]
+; CHECK-NEXT:    [[TMP170:%.*]] = load i8, ptr [[TMP169]], align 1
+; CHECK-NEXT:    [[TMP171:%.*]] = insertelement <16 x i8> [[TMP167]], i8 [[TMP170]], i32 14
+; CHECK-NEXT:    br label [[PRED_LOAD_CONTINUE60]]
+; CHECK:       pred.load.continue60:
+; CHECK-NEXT:    [[TMP172:%.*]] = phi <16 x i8> [ [[TMP167]], [[PRED_LOAD_CONTINUE58]] ], [ [[TMP171]], [[PRED_LOAD_IF59]] ]
+; CHECK-NEXT:    [[TMP173:%.*]] = extractelement <16 x i1> [[TMP16]], i32 15
+; CHECK-NEXT:    br i1 [[TMP173]], label [[PRED_LOAD_IF61:%.*]], label [[PRED_LOAD_CONTINUE62]]
+; CHECK:       pred.load.if61:
+; CHECK-NEXT:    [[TMP174:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[TMP15]]
+; CHECK-NEXT:    [[TMP175:%.*]] = load i8, ptr [[TMP174]], align 1
+; CHECK-NEXT:    [[TMP176:%.*]] = insertelement <16 x i8> [[TMP172]], i8 [[TMP175]], i32 15
+; CHECK-NEXT:    br label [[PRED_LOAD_CONTINUE62]]
+; CHECK:       pred.load.continue62:
+; CHECK-NEXT:    [[TMP177:%.*]] = phi <16 x i8> [ [[TMP172]], [[PRED_LOAD_CONTINUE60]] ], [ [[TMP176]], [[PRED_LOAD_IF61]] ]
+; CHECK-NEXT:    [[TMP178:%.*]] = sext <16 x i8> [[TMP177]] to <16 x i32>
+; CHECK-NEXT:    [[TMP179:%.*]] = mul nsw <16 x i32> [[TMP178]], [[TMP97]]
+; CHECK-NEXT:    [[TMP180:%.*]] = select <16 x i1> [[TMP16]], <16 x i32> [[TMP179]], <16 x i32> zeroinitializer
+; CHECK-NEXT:    [[PARTIAL_REDUCE]] = call <4 x i32> @llvm.experimental.vector.partial.reduce.add.v4i32.v16i32(<4 x i32> [[VEC_PHI]], <16 x i32> [[TMP180]])
+; CHECK-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 16
+; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <16 x i64> [[VEC_IND]], splat (i64 16)
+; CHECK-NEXT:    [[TMP181:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[TMP181]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+; CHECK:       middle.block:
+; CHECK-NEXT:    [[TMP182:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[PARTIAL_REDUCE]])
+; CHECK-NEXT:    br i1 true, label [[EXIT:%.*]], label [[SCALAR_PH]]
+; CHECK:       scalar.ph:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP182]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ]
+; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK:       for.body:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT:    [[ACCUM:%.*]] = phi i32 [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[ADD:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT:    [[GEP_A:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[IV]]
+; CHECK-NEXT:    [[LOAD_A:%.*]] = load i8, ptr [[GEP_A]], align 1
+; CHECK-NEXT:    [[EXT_A:%.*]] = sext i8 [[LOAD_A]] to i32
+; CHECK-NEXT:    [[GEP_B:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[IV]]
+; CHECK-NEXT:    [[LOAD_B:%.*]] = load i8, ptr [[GEP_B]], align 1
+; CHECK-NEXT:    [[EXT_B:%.*]] = sext i8 [[LOAD_B]] to i32
+; CHECK-NEXT:    [[MUL:%.*]] = mul nsw i32 [[EXT_B]], [[EXT_A]]
+; CHECK-NEXT:    [[ADD]] = add nsw i32 [[MUL]], [[ACCUM]]
+; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label [[EXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
+; CHECK:       exit:
+; CHECK-NEXT:    [[ADD_LCSSA:%.*]] = phi i32 [ [[ADD]], [[FOR_BODY]] ], [ [[TMP182]], [[MIDDLE_BLOCK]] ]
+; CHECK-NEXT:    ret i32 [[ADD_LCSSA]]
+;
+entry:
+  br label %for.body
+
+for.body:                                         ; preds = %entry, %for.body
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+  %accum = phi i32 [ 0, %entry ], [ %add, %for.body ]
+  %gep.a = getelementptr inbounds i8, ptr %a, i64 %iv
+  %load.a = load i8, ptr %gep.a, align 1
+  %ext.a = sext i8 %load.a to i32
+  %gep.b = getelementptr inbounds i8, ptr %b, i64 %iv
+  %load.b = load i8, ptr %gep.b, align 1
+  %ext.b = sext i8 %load.b to i32
+  %mul = mul nsw i32 %ext.b, %ext.a
+  %add = add nsw i32 %mul, %accum
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond.not = icmp eq i64 %iv.next, %N
+  br i1 %exitcond.not, label %exit, label %for.body, !llvm.loop !7
+
+exit:                        ; preds = %for.body
+  ret i32 %add
+}
+
+!7 = distinct !{!7, !8, !9, !10}
+!8 = !{!"llvm.loop.mustprogress"}
+!9 = !{!"llvm.loop.vectorize.predicate.enable", i1 true}
+!10 = !{!"llvm.loop.vectorize.enable", i1 true}
+
 attributes #0 = { vscale_range(1,16) "target-features"="+sve" }
 attributes #1 = { "target-cpu"="apple-m1" }
 ;.
@@ -241,4 +624,7 @@ attributes #1 = { "target-cpu"="apple-m1" }
 ; CHECK: [[LOOP5]] = distinct !{[[LOOP5]], [[META1]], [[META2]]}
 ; CHECK: [[LOOP6]] = distinct !{[[LOOP6]], [[META1]], [[META2]]}
 ; CHECK: [[LOOP7]] = distinct !{[[LOOP7]], [[META2]], [[META1]]}
+; CHECK: [[LOOP8]] = distinct !{[[LOOP8]], [[META9:![0-9]+]], [[META1]], [[META2]]}
+; CHECK: [[META9]] = !{!"llvm.loop.mustprogress"}
+; CHECK: [[LOOP10]] = distinct !{[[LOOP10]], [[META9]], [[META2]], [[META1]]}
 ;.
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-dot-product-neon.ll b/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-dot-product-neon.ll
index 97a5801d88108..54dc6ce5d3a92 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-dot-product-neon.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-dot-product-neon.ll
@@ -1391,197 +1391,8 @@ exit:                                        ; preds = %for.body
   ret i32 %result
 }
 
-define i32 @not_dotp_predicated(i64 %N, ptr %a, ptr %b) {
-; CHECK-INTERLEAVE1-LABEL: define i32 @not_dotp_predicated(
-; CHECK-INTERLEAVE1-SAME: i64 [[N:%.*]], ptr [[A:%.*]], ptr [[B:%.*]]) #[[ATTR0]] {
-; CHECK-INTERLEAVE1-NEXT:  entry:
-; CHECK-INTERLEAVE1-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 16
-; CHECK-INTERLEAVE1-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
-; CHECK-INTERLEAVE1:       vector.ph:
-; CHECK-INTERLEAVE1-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], 16
-; CHECK-INTERLEAVE1-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
-; CHECK-INTERLEAVE1-NEXT:    br label [[VECTOR_BODY:%.*]]
-; CHECK-INTERLEAVE1:       vector.body:
-; CHECK-INTERLEAVE1-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-INTERLEAVE1-NEXT:    [[VEC_PHI:%.*]] = phi <4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[PARTIAL_REDUCE:%.*]], [[VECTOR_BODY]] ]
-; CHECK-INTERLEAVE1-NEXT:    [[TMP0:%.*]] = add i64 [[INDEX]], 0
-; CHECK-INTERLEAVE1-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[TMP0]]
-; CHECK-INTERLEAVE1-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i32 0
-; CHECK-INTERLEAVE1-NEXT:    [[WIDE_LOAD:%.*]] = load <16 x i8>, ptr [[TMP2]], align 1
-; CHECK-INTERLEAVE1-NEXT:    [[TMP3:%.*]] = sext <16 x i8> [[WIDE_LOAD]] to <16 x i32>
-; CHECK-INTERLEAVE1-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[TMP0]]
-; CHECK-INTERLEAVE1-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i8, ptr [[TMP4]], i32 0
-; CHECK-INTERLEAVE1-NEXT:    [[WIDE_LOAD1:%.*]] = load <16 x i8>, ptr [[TMP5]], align 1
-; CHECK-INTERLEAVE1-NEXT:    [[TMP6:%.*]] = sext <16 x i8> [[WIDE_LOAD1]] to <16 x i32>
-; CHECK-INTERLEAVE1-NEXT:    [[TMP7:%.*]] = mul nsw <16 x i32> [[TMP6]], [[TMP3]]
-; CHECK-INTERLEAVE1-NEXT:    [[PARTIAL_REDUCE]] = call <4 x i32> @llvm.experimental.vector.partial.reduce.add.v4i32.v16i32(<4 x i32> [[VEC_PHI]], <16 x i32> [[TMP7]])
-; CHECK-INTERLEAVE1-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
-; CHECK-INTERLEAVE1-NEXT:    [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-INTERLEAVE1-NEXT:    br i1 [[TMP8]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
-; CHECK-INTERLEAVE1:       middle.block:
-; CHECK-INTERLEAVE1-NEXT:    [[TMP9:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[PARTIAL_REDUCE]])
-; CHECK-INTERLEAVE1-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
-; CHECK-INTERLEAVE1-NEXT:    br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
-; CHECK-INTERLEAVE1:       scalar.ph:
-; CHECK-INTERLEAVE1-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
-; CHECK-INTERLEAVE1-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP9]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ]
-; CHECK-INTERLEAVE1-NEXT:    br label [[FOR_BODY:%.*]]
-; CHECK-INTERLEAVE1:       for.body:
-; CHECK-INTERLEAVE1-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
-; CHECK-INTERLEAVE1-NEXT:    [[ACCUM:%.*]] = phi i32 [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[ADD:%.*]], [[FOR_BODY]] ]
-; CHECK-INTERLEAVE1-NEXT:    [[GEP_A:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[IV]]
-; CHECK-INTERLEAVE1-NEXT:    [[LOAD_A:%.*]] = load i8, ptr [[GEP_A]], align 1
-; CHECK-INTERLEAVE1-NEXT:    [[EXT_A:%.*]] = sext i8 [[LOAD_A]] to i32
-; CHECK-INTERLEAVE1-NEXT:    [[GEP_B:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[IV]]
-; CHECK-INTERLEAVE1-NEXT:    [[LOAD_B:%.*]] = load i8, ptr [[GEP_B]], align 1
-; CHECK-INTERLEAVE1-NEXT:    [[EXT_B:%.*]] = sext i8 [[LOAD_B]] to i32
-; CHECK-INTERLEAVE1-NEXT:    [[MUL:%.*]] = mul nsw i32 [[EXT_B]], [[EXT_A]]
-; CHECK-INTERLEAVE1-NEXT:    [[ADD]] = add nsw i32 [[MUL]], [[ACCUM]]
-; CHECK-INTERLEAVE1-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
-; CHECK-INTERLEAVE1-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; CHECK-INTERLEAVE1-NEXT:    br i1 [[EXITCOND_NOT]], label [[EXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP13:![0-9]+]]
-; CHECK-INTERLEAVE1:       exit:
-; CHECK-INTERLEAVE1-NEXT:    [[ADD_LCSSA:%.*]] = phi i32 [ [[ADD]], [[FOR_BODY]] ], [ [[TMP9]], [[MIDDLE_BLOCK]] ]
-; CHECK-INTERLEAVE1-NEXT:    ret i32 [[ADD_LCSSA]]
-;
-; CHECK-INTERLEAVED-LABEL: define i32 @not_dotp_predicated(
-; CHECK-INTERLEAVED-SAME: i64 [[N:%.*]], ptr [[A:%.*]], ptr [[B:%.*]]) #[[ATTR0]] {
-; CHECK-INTERLEAVED-NEXT:  entry:
-; CHECK-INTERLEAVED-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 32
-; CHECK-INTERLEAVED-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
-; CHECK-INTERLEAVED:       vector.ph:
-; CHECK-INTERLEAVED-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], 32
-; CHECK-INTERLEAVED-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
-; CHECK-INTERLEAVED-NEXT:    br label [[VECTOR_BODY:%.*]]
-; CHECK-INTERLEAVED:       vector.body:
-; CHECK-INTERLEAVED-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-INTERLEAVED-NEXT:    [[VEC_PHI:%.*]] = phi <4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[PARTIAL_REDUCE:%.*]], [[VECTOR_BODY]] ]
-; CHECK-INTERLEAVED-NEXT:    [[VEC_PHI1:%.*]] = phi <4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[PARTIAL_REDUCE5:%.*]], [[VECTOR_BODY]] ]
-; CHECK-INTERLEAVED-NEXT:    [[TMP0:%.*]] = add i64 [[INDEX]], 0
-; CHECK-INTERLEAVED-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[TMP0]]
-; CHECK-INTERLEAVED-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i32 0
-; CHECK-INTERLEAVED-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i32 16
-; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD:%.*]] = load <16 x i8>, ptr [[TMP2]], align 1
-; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD2:%.*]] = load <16 x i8>, ptr [[TMP3]], align 1
-; CHECK-INTERLEAVED-NEXT:    [[TMP4:%.*]] = sext <16 x i8> [[WIDE_LOAD]] to <16 x i32>
-; CHECK-INTERLEAVED-NEXT:    [[TMP5:%.*]] = sext <16 x i8> [[WIDE_LOAD2]] to <16 x i32>
-; CHECK-INTERLEAVED-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[TMP0]]
-; CHECK-INTERLEAVED-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i8, ptr [[TMP6]], i32 0
-; CHECK-INTERLEAVED-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i8, ptr [[TMP6]], i32 16
-; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD3:%.*]] = load <16 x i8>, ptr [[TMP7]], align 1
-; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD4:%.*]] = load <16 x i8>, ptr [[TMP8]], align 1
-; CHECK-INTERLEAVED-NEXT:    [[TMP9:%.*]] = sext <16 x i8> [[WIDE_LOAD3]] to <16 x i32>
-; CHECK-INTERLEAVED-NEXT:    [[TMP10:%.*]] = sext <16 x i8> [[WIDE_LOAD4]] to <16 x i32>
-; CHECK-INTERLEAVED-NEXT:    [[TMP11:%.*]] = mul nsw <16 x i32> [[TMP9]], [[TMP4]]
-; CHECK-INTERLEAVED-NEXT:    [[TMP12:%.*]] = mul nsw <16 x i32> [[TMP10]], [[TMP5]]
-; CHECK-INTERLEAVED-NEXT:    [[PARTIAL_REDUCE]] = call <4 x i32> @llvm.experimental.vector.partial.reduce.add.v4i32.v16i32(<4 x i32> [[VEC_PHI]], <16 x i32> [[TMP11]])
-; CHECK-INTERLEAVED-NEXT:    [[PARTIAL_REDUCE5]] = call <4 x i32> @llvm.experimental.vector.partial.reduce.add.v4i32.v16i32(<4 x i32> [[VEC_PHI1]], <16 x i32> [[TMP12]])
-; CHECK-INTERLEAVED-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 32
-; CHECK-INTERLEAVED-NEXT:    [[TMP13:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-INTERLEAVED-NEXT:    br i1 [[TMP13]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
-; CHECK-INTERLEAVED:       middle.block:
-; CHECK-INTERLEAVED-NEXT:    [[BIN_RDX:%.*]] = add <4 x i32> [[PARTIAL_REDUCE5]], [[PARTIAL_REDUCE]]
-; CHECK-INTERLEAVED-NEXT:    [[TMP14:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[BIN_RDX]])
-; CHECK-INTERLEAVED-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
-; CHECK-INTERLEAVED-NEXT:    br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
-; CHECK-INTERLEAVED:       scalar.ph:
-; CHECK-INTERLEAVED-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
-; CHECK-INTERLEAVED-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP14]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ]
-; CHECK-INTERLEAVED-NEXT:    br label [[FOR_BODY:%.*]]
-; CHECK-INTERLEAVED:       for.body:
-; CHECK-INTERLEAVED-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
-; CHECK-INTERLEAVED-NEXT:    [[ACCUM:%.*]] = phi i32 [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[ADD:%.*]], [[FOR_BODY]] ]
-; CHECK-INTERLEAVED-NEXT:    [[GEP_A:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[IV]]
-; CHECK-INTERLEAVED-NEXT:    [[LOAD_A:%.*]] = load i8, ptr [[GEP_A]], align 1
-; CHECK-INTERLEAVED-NEXT:    [[EXT_A:%.*]] = sext i8 [[LOAD_A]] to i32
-; CHECK-INTERLEAVED-NEXT:    [[GEP_B:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[IV]]
-; CHECK-INTERLEAVED-NEXT:    [[LOAD_B:%.*]] = load i8, ptr [[GEP_B]], align 1
-; CHECK-INTERLEAVED-NEXT:    [[EXT_B:%.*]] = sext i8 [[LOAD_B]] to i32
-; CHECK-INTERLEAVED-NEXT:    [[MUL:%.*]] = mul nsw i32 [[EXT_B]], [[EXT_A]]
-; CHECK-INTERLEAVED-NEXT:    [[ADD]] = add nsw i32 [[MUL]], [[ACCUM]]
-; CHECK-INTERLEAVED-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
-; CHECK-INTERLEAVED-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; CHECK-INTERLEAVED-NEXT:    br i1 [[EXITCOND_NOT]], label [[EXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP13:![0-9]+]]
-; CHECK-INTERLEAVED:       exit:
-; CHECK-INTERLEAVED-NEXT:    [[ADD_LCSSA:%.*]] = phi i32 [ [[ADD]], [[FOR_BODY]] ], [ [[TMP14]], [[MIDDLE_BLOCK]] ]
-; CHECK-INTERLEAVED-NEXT:    ret i32 [[ADD_LCSSA]]
-;
-; CHECK-MAXBW-LABEL: define i32 @not_dotp_predicated(
-; CHECK-MAXBW-SAME: i64 [[N:%.*]], ptr [[A:%.*]], ptr [[B:%.*]]) #[[ATTR0]] {
-; CHECK-MAXBW-NEXT:  entry:
-; CHECK-MAXBW-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 16
-; CHECK-MAXBW-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
-; CHECK-MAXBW:       vector.ph:
-; CHECK-MAXBW-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], 16
-; CHECK-MAXBW-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
-; CHECK-MAXBW-NEXT:    br label [[VECTOR_BODY:%.*]]
-; CHECK-MAXBW:       vector.body:
-; CHECK-MAXBW-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-MAXBW-NEXT:    [[VEC_PHI:%.*]] = phi <4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[PARTIAL_REDUCE:%.*]], [[VECTOR_BODY]] ]
-; CHECK-MAXBW-NEXT:    [[TMP0:%.*]] = add i64 [[INDEX]], 0
-; CHECK-MAXBW-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[TMP0]]
-; CHECK-MAXBW-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i32 0
-; CHECK-MAXBW-NEXT:    [[WIDE_LOAD:%.*]] = load <16 x i8>, ptr [[TMP2]], align 1
-; CHECK-MAXBW-NEXT:    [[TMP3:%.*]] = sext <16 x i8> [[WIDE_LOAD]] to <16 x i32>
-; CHECK-MAXBW-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[TMP0]]
-; CHECK-MAXBW-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i8, ptr [[TMP4]], i32 0
-; CHECK-MAXBW-NEXT:    [[WIDE_LOAD1:%.*]] = load <16 x i8>, ptr [[TMP5]], align 1
-; CHECK-MAXBW-NEXT:    [[TMP6:%.*]] = sext <16 x i8> [[WIDE_LOAD1]] to <16 x i32>
-; CHECK-MAXBW-NEXT:    [[TMP7:%.*]] = mul nsw <16 x i32> [[TMP6]], [[TMP3]]
-; CHECK-MAXBW-NEXT:    [[PARTIAL_REDUCE]] = call <4 x i32> @llvm.experimental.vector.partial.reduce.add.v4i32.v16i32(<4 x i32> [[VEC_PHI]], <16 x i32> [[TMP7]])
-; CHECK-MAXBW-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
-; CHECK-MAXBW-NEXT:    [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-MAXBW-NEXT:    br i1 [[TMP8]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
-; CHECK-MAXBW:       middle.block:
-; CHECK-MAXBW-NEXT:    [[TMP9:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[PARTIAL_REDUCE]])
-; CHECK-MAXBW-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
-; CHECK-MAXBW-NEXT:    br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
-; CHECK-MAXBW:       scalar.ph:
-; CHECK-MAXBW-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
-; CHECK-MAXBW-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP9]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ]
-; CHECK-MAXBW-NEXT:    br label [[FOR_BODY:%.*]]
-; CHECK-MAXBW:       for.body:
-; CHECK-MAXBW-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
-; CHECK-MAXBW-NEXT:    [[ACCUM:%.*]] = phi i32 [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[ADD:%.*]], [[FOR_BODY]] ]
-; CHECK-MAXBW-NEXT:    [[GEP_A:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[IV]]
-; CHECK-MAXBW-NEXT:    [[LOAD_A:%.*]] = load i8, ptr [[GEP_A]], align 1
-; CHECK-MAXBW-NEXT:    [[EXT_A:%.*]] = sext i8 [[LOAD_A]] to i32
-; CHECK-MAXBW-NEXT:    [[GEP_B:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[IV]]
-; CHECK-MAXBW-NEXT:    [[LOAD_B:%.*]] = load i8, ptr [[GEP_B]], align 1
-; CHECK-MAXBW-NEXT:    [[EXT_B:%.*]] = sext i8 [[LOAD_B]] to i32
-; CHECK-MAXBW-NEXT:    [[MUL:%.*]] = mul nsw i32 [[EXT_B]], [[EXT_A]]
-; CHECK-MAXBW-NEXT:    [[ADD]] = add nsw i32 [[MUL]], [[ACCUM]]
-; CHECK-MAXBW-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
-; CHECK-MAXBW-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; CHECK-MAXBW-NEXT:    br i1 [[EXITCOND_NOT]], label [[EXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP13:![0-9]+]]
-; CHECK-MAXBW:       exit:
-; CHECK-MAXBW-NEXT:    [[ADD_LCSSA:%.*]] = phi i32 [ [[ADD]], [[FOR_BODY]] ], [ [[TMP9]], [[MIDDLE_BLOCK]] ]
-; CHECK-MAXBW-NEXT:    ret i32 [[ADD_LCSSA]]
-;
-entry:
-  br label %for.body
-
-for.body:                                         ; preds = %entry, %for.body
-  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
-  %accum = phi i32 [ 0, %entry ], [ %add, %for.body ]
-  %gep.a = getelementptr inbounds i8, ptr %a, i64 %iv
-  %load.a = load i8, ptr %gep.a, align 1
-  %ext.a = sext i8 %load.a to i32
-  %gep.b = getelementptr inbounds i8, ptr %b, i64 %iv
-  %load.b = load i8, ptr %gep.b, align 1
-  %ext.b = sext i8 %load.b to i32
-  %mul = mul nsw i32 %ext.b, %ext.a
-  %add = add nsw i32 %mul, %accum
-  %iv.next = add nuw nsw i64 %iv, 1
-  %exitcond.not = icmp eq i64 %iv.next, %N
-  br i1 %exitcond.not, label %exit, label %for.body
-
-exit:                        ; preds = %for.body
-  ret i32 %add
-}
-
-define i32 @not_dotp_predicated_pragma(i64 %N, ptr %a, ptr %b) {
-; CHECK-INTERLEAVE1-LABEL: define i32 @not_dotp_predicated_pragma(
+define i32 @dotp_predicated(i64 %N, ptr %a, ptr %b) {
+; CHECK-INTERLEAVE1-LABEL: define i32 @dotp_predicated(
 ; CHECK-INTERLEAVE1-SAME: i64 [[N:%.*]], ptr [[A:%.*]], ptr [[B:%.*]]) #[[ATTR0]] {
 ; CHECK-INTERLEAVE1-NEXT:  entry:
 ; CHECK-INTERLEAVE1-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
@@ -1596,7 +1407,7 @@ define i32 @not_dotp_predicated_pragma(i64 %N, ptr %a, ptr %b) {
 ; CHECK-INTERLEAVE1:       vector.body:
 ; CHECK-INTERLEAVE1-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_LOAD_CONTINUE62:%.*]] ]
 ; CHECK-INTERLEAVE1-NEXT:    [[VEC_IND:%.*]] = phi <16 x i64> [ <i64 0, i64 1, i64 2, i64 3, i64 4, i64 5, i64 6, i64 7, i64 8, i64 9, i64 10, i64 11, i64 12, i64 13, i64 14, i64 15>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[PRED_LOAD_CONTINUE62]] ]
-; CHECK-INTERLEAVE1-NEXT:    [[VEC_PHI:%.*]] = phi <16 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP180:%.*]], [[PRED_LOAD_CONTINUE62]] ]
+; CHECK-INTERLEAVE1-NEXT:    [[VEC_PHI:%.*]] = phi <4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[PARTIAL_REDUCE:%.*]], [[PRED_LOAD_CONTINUE62]] ]
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP0:%.*]] = add i64 [[INDEX]], 0
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP1:%.*]] = add i64 [[INDEX]], 1
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP2:%.*]] = add i64 [[INDEX]], 2
@@ -1617,7 +1428,7 @@ define i32 @not_dotp_predicated_pragma(i64 %N, ptr %a, ptr %b) {
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP17:%.*]] = extractelement <16 x i1> [[TMP16]], i32 0
 ; CHECK-INTERLEAVE1-NEXT:    br i1 [[TMP17]], label [[PRED_LOAD_IF:%.*]], label [[PRED_LOAD_CONTINUE:%.*]]
 ; CHECK-INTERLEAVE1:       pred.load.if:
-; CHECK-INTERLEAVE1-NEXT:    [[TMP18:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[TMP0]]
+; CHECK-INTERLEAVE1-NEXT:    [[TMP18:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[TMP0]]
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP19:%.*]] = load i8, ptr [[TMP18]], align 1
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP20:%.*]] = insertelement <16 x i8> poison, i8 [[TMP19]], i32 0
 ; CHECK-INTERLEAVE1-NEXT:    br label [[PRED_LOAD_CONTINUE]]
@@ -1626,7 +1437,7 @@ define i32 @not_dotp_predicated_pragma(i64 %N, ptr %a, ptr %b) {
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP22:%.*]] = extractelement <16 x i1> [[TMP16]], i32 1
 ; CHECK-INTERLEAVE1-NEXT:    br i1 [[TMP22]], label [[PRED_LOAD_IF1:%.*]], label [[PRED_LOAD_CONTINUE2:%.*]]
 ; CHECK-INTERLEAVE1:       pred.load.if1:
-; CHECK-INTERLEAVE1-NEXT:    [[TMP23:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[TMP1]]
+; CHECK-INTERLEAVE1-NEXT:    [[TMP23:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[TMP1]]
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP24:%.*]] = load i8, ptr [[TMP23]], align 1
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP25:%.*]] = insertelement <16 x i8> [[TMP21]], i8 [[TMP24]], i32 1
 ; CHECK-INTERLEAVE1-NEXT:    br label [[PRED_LOAD_CONTINUE2]]
@@ -1635,7 +1446,7 @@ define i32 @not_dotp_predicated_pragma(i64 %N, ptr %a, ptr %b) {
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP27:%.*]] = extractelement <16 x i1> [[TMP16]], i32 2
 ; CHECK-INTERLEAVE1-NEXT:    br i1 [[TMP27]], label [[PRED_LOAD_IF3:%.*]], label [[PRED_LOAD_CONTINUE4:%.*]]
 ; CHECK-INTERLEAVE1:       pred.load.if3:
-; CHECK-INTERLEAVE1-NEXT:    [[TMP28:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[TMP2]]
+; CHECK-INTERLEAVE1-NEXT:    [[TMP28:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[TMP2]]
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP29:%.*]] = load i8, ptr [[TMP28]], align 1
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP30:%.*]] = insertelement <16 x i8> [[TMP26]], i8 [[TMP29]], i32 2
 ; CHECK-INTERLEAVE1-NEXT:    br label [[PRED_LOAD_CONTINUE4]]
@@ -1644,7 +1455,7 @@ define i32 @not_dotp_predicated_pragma(i64 %N, ptr %a, ptr %b) {
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP32:%.*]] = extractelement <16 x i1> [[TMP16]], i32 3
 ; CHECK-INTERLEAVE1-NEXT:    br i1 [[TMP32]], label [[PRED_LOAD_IF5:%.*]], label [[PRED_LOAD_CONTINUE6:%.*]]
 ; CHECK-INTERLEAVE1:       pred.load.if5:
-; CHECK-INTERLEAVE1-NEXT:    [[TMP33:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[TMP3]]
+; CHECK-INTERLEAVE1-NEXT:    [[TMP33:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[TMP3]]
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP34:%.*]] = load i8, ptr [[TMP33]], align 1
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP35:%.*]] = insertelement <16 x i8> [[TMP31]], i8 [[TMP34]], i32 3
 ; CHECK-INTERLEAVE1-NEXT:    br label [[PRED_LOAD_CONTINUE6]]
@@ -1653,7 +1464,7 @@ define i32 @not_dotp_predicated_pragma(i64 %N, ptr %a, ptr %b) {
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP37:%.*]] = extractelement <16 x i1> [[TMP16]], i32 4
 ; CHECK-INTERLEAVE1-NEXT:    br i1 [[TMP37]], label [[PRED_LOAD_IF7:%.*]], label [[PRED_LOAD_CONTINUE8:%.*]]
 ; CHECK-INTERLEAVE1:       pred.load.if7:
-; CHECK-INTERLEAVE1-NEXT:    [[TMP38:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[TMP4]]
+; CHECK-INTERLEAVE1-NEXT:    [[TMP38:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[TMP4]]
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP39:%.*]] = load i8, ptr [[TMP38]], align 1
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP40:%.*]] = insertelement <16 x i8> [[TMP36]], i8 [[TMP39]], i32 4
 ; CHECK-INTERLEAVE1-NEXT:    br label [[PRED_LOAD_CONTINUE8]]
@@ -1662,7 +1473,7 @@ define i32 @not_dotp_predicated_pragma(i64 %N, ptr %a, ptr %b) {
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP42:%.*]] = extractelement <16 x i1> [[TMP16]], i32 5
 ; CHECK-INTERLEAVE1-NEXT:    br i1 [[TMP42]], label [[PRED_LOAD_IF9:%.*]], label [[PRED_LOAD_CONTINUE10:%.*]]
 ; CHECK-INTERLEAVE1:       pred.load.if9:
-; CHECK-INTERLEAVE1-NEXT:    [[TMP43:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[TMP5]]
+; CHECK-INTERLEAVE1-NEXT:    [[TMP43:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[TMP5]]
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP44:%.*]] = load i8, ptr [[TMP43]], align 1
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP45:%.*]] = insertelement <16 x i8> [[TMP41]], i8 [[TMP44]], i32 5
 ; CHECK-INTERLEAVE1-NEXT:    br label [[PRED_LOAD_CONTINUE10]]
@@ -1671,7 +1482,7 @@ define i32 @not_dotp_predicated_pragma(i64 %N, ptr %a, ptr %b) {
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP47:%.*]] = extractelement <16 x i1> [[TMP16]], i32 6
 ; CHECK-INTERLEAVE1-NEXT:    br i1 [[TMP47]], label [[PRED_LOAD_IF11:%.*]], label [[PRED_LOAD_CONTINUE12:%.*]]
 ; CHECK-INTERLEAVE1:       pred.load.if11:
-; CHECK-INTERLEAVE1-NEXT:    [[TMP48:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[TMP6]]
+; CHECK-INTERLEAVE1-NEXT:    [[TMP48:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[TMP6]]
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP49:%.*]] = load i8, ptr [[TMP48]], align 1
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP50:%.*]] = insertelement <16 x i8> [[TMP46]], i8 [[TMP49]], i32 6
 ; CHECK-INTERLEAVE1-NEXT:    br label [[PRED_LOAD_CONTINUE12]]
@@ -1680,7 +1491,7 @@ define i32 @not_dotp_predicated_pragma(i64 %N, ptr %a, ptr %b) {
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP52:%.*]] = extractelement <16 x i1> [[TMP16]], i32 7
 ; CHECK-INTERLEAVE1-NEXT:    br i1 [[TMP52]], label [[PRED_LOAD_IF13:%.*]], label [[PRED_LOAD_CONTINUE14:%.*]]
 ; CHECK-INTERLEAVE1:       pred.load.if13:
-; CHECK-INTERLEAVE1-NEXT:    [[TMP53:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[TMP7]]
+; CHECK-INTERLEAVE1-NEXT:    [[TMP53:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[TMP7]]
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP54:%.*]] = load i8, ptr [[TMP53]], align 1
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP55:%.*]] = insertelement <16 x i8> [[TMP51]], i8 [[TMP54]], i32 7
 ; CHECK-INTERLEAVE1-NEXT:    br label [[PRED_LOAD_CONTINUE14]]
@@ -1689,7 +1500,7 @@ define i32 @not_dotp_predicated_pragma(i64 %N, ptr %a, ptr %b) {
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP57:%.*]] = extractelement <16 x i1> [[TMP16]], i32 8
 ; CHECK-INTERLEAVE1-NEXT:    br i1 [[TMP57]], label [[PRED_LOAD_IF15:%.*]], label [[PRED_LOAD_CONTINUE16:%.*]]
 ; CHECK-INTERLEAVE1:       pred.load.if15:
-; CHECK-INTERLEAVE1-NEXT:    [[TMP58:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[TMP8]]
+; CHECK-INTERLEAVE1-NEXT:    [[TMP58:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[TMP8]]
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP59:%.*]] = load i8, ptr [[TMP58]], align 1
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP60:%.*]] = insertelement <16 x i8> [[TMP56]], i8 [[TMP59]], i32 8
 ; CHECK-INTERLEAVE1-NEXT:    br label [[PRED_LOAD_CONTINUE16]]
@@ -1698,7 +1509,7 @@ define i32 @not_dotp_predicated_pragma(i64 %N, ptr %a, ptr %b) {
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP62:%.*]] = extractelement <16 x i1> [[TMP16]], i32 9
 ; CHECK-INTERLEAVE1-NEXT:    br i1 [[TMP62]], label [[PRED_LOAD_IF17:%.*]], label [[PRED_LOAD_CONTINUE18:%.*]]
 ; CHECK-INTERLEAVE1:       pred.load.if17:
-; CHECK-INTERLEAVE1-NEXT:    [[TMP63:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[TMP9]]
+; CHECK-INTERLEAVE1-NEXT:    [[TMP63:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[TMP9]]
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP64:%.*]] = load i8, ptr [[TMP63]], align 1
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP65:%.*]] = insertelement <16 x i8> [[TMP61]], i8 [[TMP64]], i32 9
 ; CHECK-INTERLEAVE1-NEXT:    br label [[PRED_LOAD_CONTINUE18]]
@@ -1707,7 +1518,7 @@ define i32 @not_dotp_predicated_pragma(i64 %N, ptr %a, ptr %b) {
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP67:%.*]] = extractelement <16 x i1> [[TMP16]], i32 10
 ; CHECK-INTERLEAVE1-NEXT:    br i1 [[TMP67]], label [[PRED_LOAD_IF19:%.*]], label [[PRED_LOAD_CONTINUE20:%.*]]
 ; CHECK-INTERLEAVE1:       pred.load.if19:
-; CHECK-INTERLEAVE1-NEXT:    [[TMP68:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[TMP10]]
+; CHECK-INTERLEAVE1-NEXT:    [[TMP68:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[TMP10]]
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP69:%.*]] = load i8, ptr [[TMP68]], align 1
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP70:%.*]] = insertelement <16 x i8> [[TMP66]], i8 [[TMP69]], i32 10
 ; CHECK-INTERLEAVE1-NEXT:    br label [[PRED_LOAD_CONTINUE20]]
@@ -1716,7 +1527,7 @@ define i32 @not_dotp_predicated_pragma(i64 %N, ptr %a, ptr %b) {
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP72:%.*]] = extractelement <16 x i1> [[TMP16]], i32 11
 ; CHECK-INTERLEAVE1-NEXT:    br i1 [[TMP72]], label [[PRED_LOAD_IF21:%.*]], label [[PRED_LOAD_CONTINUE22:%.*]]
 ; CHECK-INTERLEAVE1:       pred.load.if21:
-; CHECK-INTERLEAVE1-NEXT:    [[TMP73:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[TMP11]]
+; CHECK-INTERLEAVE1-NEXT:    [[TMP73:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[TMP11]]
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP74:%.*]] = load i8, ptr [[TMP73]], align 1
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP75:%.*]] = insertelement <16 x i8> [[TMP71]], i8 [[TMP74]], i32 11
 ; CHECK-INTERLEAVE1-NEXT:    br label [[PRED_LOAD_CONTINUE22]]
@@ -1725,7 +1536,7 @@ define i32 @not_dotp_predicated_pragma(i64 %N, ptr %a, ptr %b) {
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP77:%.*]] = extractelement <16 x i1> [[TMP16]], i32 12
 ; CHECK-INTERLEAVE1-NEXT:    br i1 [[TMP77]], label [[PRED_LOAD_IF23:%.*]], label [[PRED_LOAD_CONTINUE24:%.*]]
 ; CHECK-INTERLEAVE1:       pred.load.if23:
-; CHECK-INTERLEAVE1-NEXT:    [[TMP78:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[TMP12]]
+; CHECK-INTERLEAVE1-NEXT:    [[TMP78:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[TMP12]]
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP79:%.*]] = load i8, ptr [[TMP78]], align 1
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP80:%.*]] = insertelement <16 x i8> [[TMP76]], i8 [[TMP79]], i32 12
 ; CHECK-INTERLEAVE1-NEXT:    br label [[PRED_LOAD_CONTINUE24]]
@@ -1734,7 +1545,7 @@ define i32 @not_dotp_predicated_pragma(i64 %N, ptr %a, ptr %b) {
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP82:%.*]] = extractelement <16 x i1> [[TMP16]], i32 13
 ; CHECK-INTERLEAVE1-NEXT:    br i1 [[TMP82]], label [[PRED_LOAD_IF25:%.*]], label [[PRED_LOAD_CONTINUE26:%.*]]
 ; CHECK-INTERLEAVE1:       pred.load.if25:
-; CHECK-INTERLEAVE1-NEXT:    [[TMP83:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[TMP13]]
+; CHECK-INTERLEAVE1-NEXT:    [[TMP83:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[TMP13]]
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP84:%.*]] = load i8, ptr [[TMP83]], align 1
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP85:%.*]] = insertelement <16 x i8> [[TMP81]], i8 [[TMP84]], i32 13
 ; CHECK-INTERLEAVE1-NEXT:    br label [[PRED_LOAD_CONTINUE26]]
@@ -1743,7 +1554,7 @@ define i32 @not_dotp_predicated_pragma(i64 %N, ptr %a, ptr %b) {
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP87:%.*]] = extractelement <16 x i1> [[TMP16]], i32 14
 ; CHECK-INTERLEAVE1-NEXT:    br i1 [[TMP87]], label [[PRED_LOAD_IF27:%.*]], label [[PRED_LOAD_CONTINUE28:%.*]]
 ; CHECK-INTERLEAVE1:       pred.load.if27:
-; CHECK-INTERLEAVE1-NEXT:    [[TMP88:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[TMP14]]
+; CHECK-INTERLEAVE1-NEXT:    [[TMP88:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[TMP14]]
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP89:%.*]] = load i8, ptr [[TMP88]], align 1
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP90:%.*]] = insertelement <16 x i8> [[TMP86]], i8 [[TMP89]], i32 14
 ; CHECK-INTERLEAVE1-NEXT:    br label [[PRED_LOAD_CONTINUE28]]
@@ -1752,7 +1563,7 @@ define i32 @not_dotp_predicated_pragma(i64 %N, ptr %a, ptr %b) {
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP92:%.*]] = extractelement <16 x i1> [[TMP16]], i32 15
 ; CHECK-INTERLEAVE1-NEXT:    br i1 [[TMP92]], label [[PRED_LOAD_IF29:%.*]], label [[PRED_LOAD_CONTINUE30:%.*]]
 ; CHECK-INTERLEAVE1:       pred.load.if29:
-; CHECK-INTERLEAVE1-NEXT:    [[TMP93:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[TMP15]]
+; CHECK-INTERLEAVE1-NEXT:    [[TMP93:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[TMP15]]
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP94:%.*]] = load i8, ptr [[TMP93]], align 1
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP95:%.*]] = insertelement <16 x i8> [[TMP91]], i8 [[TMP94]], i32 15
 ; CHECK-INTERLEAVE1-NEXT:    br label [[PRED_LOAD_CONTINUE30]]
@@ -1762,7 +1573,7 @@ define i32 @not_dotp_predicated_pragma(i64 %N, ptr %a, ptr %b) {
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP98:%.*]] = extractelement <16 x i1> [[TMP16]], i32 0
 ; CHECK-INTERLEAVE1-NEXT:    br i1 [[TMP98]], label [[PRED_LOAD_IF31:%.*]], label [[PRED_LOAD_CONTINUE32:%.*]]
 ; CHECK-INTERLEAVE1:       pred.load.if31:
-; CHECK-INTERLEAVE1-NEXT:    [[TMP99:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[TMP0]]
+; CHECK-INTERLEAVE1-NEXT:    [[TMP99:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[TMP0]]
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP100:%.*]] = load i8, ptr [[TMP99]], align 1
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP101:%.*]] = insertelement <16 x i8> poison, i8 [[TMP100]], i32 0
 ; CHECK-INTERLEAVE1-NEXT:    br label [[PRED_LOAD_CONTINUE32]]
@@ -1771,7 +1582,7 @@ define i32 @not_dotp_predicated_pragma(i64 %N, ptr %a, ptr %b) {
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP103:%.*]] = extractelement <16 x i1> [[TMP16]], i32 1
 ; CHECK-INTERLEAVE1-NEXT:    br i1 [[TMP103]], label [[PRED_LOAD_IF33:%.*]], label [[PRED_LOAD_CONTINUE34:%.*]]
 ; CHECK-INTERLEAVE1:       pred.load.if33:
-; CHECK-INTERLEAVE1-NEXT:    [[TMP104:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[TMP1]]
+; CHECK-INTERLEAVE1-NEXT:    [[TMP104:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[TMP1]]
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP105:%.*]] = load i8, ptr [[TMP104]], align 1
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP106:%.*]] = insertelement <16 x i8> [[TMP102]], i8 [[TMP105]], i32 1
 ; CHECK-INTERLEAVE1-NEXT:    br label [[PRED_LOAD_CONTINUE34]]
@@ -1780,7 +1591,7 @@ define i32 @not_dotp_predicated_pragma(i64 %N, ptr %a, ptr %b) {
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP108:%.*]] = extractelement <16 x i1> [[TMP16]], i32 2
 ; CHECK-INTERLEAVE1-NEXT:    br i1 [[TMP108]], label [[PRED_LOAD_IF35:%.*]], label [[PRED_LOAD_CONTINUE36:%.*]]
 ; CHECK-INTERLEAVE1:       pred.load.if35:
-; CHECK-INTERLEAVE1-NEXT:    [[TMP109:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[TMP2]]
+; CHECK-INTERLEAVE1-NEXT:    [[TMP109:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[TMP2]]
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP110:%.*]] = load i8, ptr [[TMP109]], align 1
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP111:%.*]] = insertelement <16 x i8> [[TMP107]], i8 [[TMP110]], i32 2
 ; CHECK-INTERLEAVE1-NEXT:    br label [[PRED_LOAD_CONTINUE36]]
@@ -1789,7 +1600,7 @@ define i32 @not_dotp_predicated_pragma(i64 %N, ptr %a, ptr %b) {
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP113:%.*]] = extractelement <16 x i1> [[TMP16]], i32 3
 ; CHECK-INTERLEAVE1-NEXT:    br i1 [[TMP113]], label [[PRED_LOAD_IF37:%.*]], label [[PRED_LOAD_CONTINUE38:%.*]]
 ; CHECK-INTERLEAVE1:       pred.load.if37:
-; CHECK-INTERLEAVE1-NEXT:    [[TMP114:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[TMP3]]
+; CHECK-INTERLEAVE1-NEXT:    [[TMP114:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[TMP3]]
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP115:%.*]] = load i8, ptr [[TMP114]], align 1
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP116:%.*]] = insertelement <16 x i8> [[TMP112]], i8 [[TMP115]], i32 3
 ; CHECK-INTERLEAVE1-NEXT:    br label [[PRED_LOAD_CONTINUE38]]
@@ -1798,7 +1609,7 @@ define i32 @not_dotp_predicated_pragma(i64 %N, ptr %a, ptr %b) {
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP118:%.*]] = extractelement <16 x i1> [[TMP16]], i32 4
 ; CHECK-INTERLEAVE1-NEXT:    br i1 [[TMP118]], label [[PRED_LOAD_IF39:%.*]], label [[PRED_LOAD_CONTINUE40:%.*]]
 ; CHECK-INTERLEAVE1:       pred.load.if39:
-; CHECK-INTERLEAVE1-NEXT:    [[TMP119:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[TMP4]]
+; CHECK-INTERLEAVE1-NEXT:    [[TMP119:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[TMP4]]
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP120:%.*]] = load i8, ptr [[TMP119]], align 1
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP121:%.*]] = insertelement <16 x i8> [[TMP117]], i8 [[TMP120]], i32 4
 ; CHECK-INTERLEAVE1-NEXT:    br label [[PRED_LOAD_CONTINUE40]]
@@ -1807,7 +1618,7 @@ define i32 @not_dotp_predicated_pragma(i64 %N, ptr %a, ptr %b) {
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP123:%.*]] = extractelement <16 x i1> [[TMP16]], i32 5
 ; CHECK-INTERLEAVE1-NEXT:    br i1 [[TMP123]], label [[PRED_LOAD_IF41:%.*]], label [[PRED_LOAD_CONTINUE42:%.*]]
 ; CHECK-INTERLEAVE1:       pred.load.if41:
-; CHECK-INTERLEAVE1-NEXT:    [[TMP124:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[TMP5]]
+; CHECK-INTERLEAVE1-NEXT:    [[TMP124:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[TMP5]]
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP125:%.*]] = load i8, ptr [[TMP124]], align 1
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP126:%.*]] = insertelement <16 x i8> [[TMP122]], i8 [[TMP125]], i32 5
 ; CHECK-INTERLEAVE1-NEXT:    br label [[PRED_LOAD_CONTINUE42]]
@@ -1816,7 +1627,7 @@ define i32 @not_dotp_predicated_pragma(i64 %N, ptr %a, ptr %b) {
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP128:%.*]] = extractelement <16 x i1> [[TMP16]], i32 6
 ; CHECK-INTERLEAVE1-NEXT:    br i1 [[TMP128]], label [[PRED_LOAD_IF43:%.*]], label [[PRED_LOAD_CONTINUE44:%.*]]
 ; CHECK-INTERLEAVE1:       pred.load.if43:
-; CHECK-INTERLEAVE1-NEXT:    [[TMP129:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[TMP6]]
+; CHECK-INTERLEAVE1-NEXT:    [[TMP129:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[TMP6]]
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP130:%.*]] = load i8, ptr [[TMP129]], align 1
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP131:%.*]] = insertelement <16 x i8> [[TMP127]], i8 [[TMP130]], i32 6
 ; CHECK-INTERLEAVE1-NEXT:    br label [[PRED_LOAD_CONTINUE44]]
@@ -1825,7 +1636,7 @@ define i32 @not_dotp_predicated_pragma(i64 %N, ptr %a, ptr %b) {
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP133:%.*]] = extractelement <16 x i1> [[TMP16]], i32 7
 ; CHECK-INTERLEAVE1-NEXT:    br i1 [[TMP133]], label [[PRED_LOAD_IF45:%.*]], label [[PRED_LOAD_CONTINUE46:%.*]]
 ; CHECK-INTERLEAVE1:       pred.load.if45:
-; CHECK-INTERLEAVE1-NEXT:    [[TMP134:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[TMP7]]
+; CHECK-INTERLEAVE1-NEXT:    [[TMP134:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[TMP7]]
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP135:%.*]] = load i8, ptr [[TMP134]], align 1
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP136:%.*]] = insertelement <16 x i8> [[TMP132]], i8 [[TMP135]], i32 7
 ; CHECK-INTERLEAVE1-NEXT:    br label [[PRED_LOAD_CONTINUE46]]
@@ -1834,7 +1645,7 @@ define i32 @not_dotp_predicated_pragma(i64 %N, ptr %a, ptr %b) {
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP138:%.*]] = extractelement <16 x i1> [[TMP16]], i32 8
 ; CHECK-INTERLEAVE1-NEXT:    br i1 [[TMP138]], label [[PRED_LOAD_IF47:%.*]], label [[PRED_LOAD_CONTINUE48:%.*]]
 ; CHECK-INTERLEAVE1:       pred.load.if47:
-; CHECK-INTERLEAVE1-NEXT:    [[TMP139:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[TMP8]]
+; CHECK-INTERLEAVE1-NEXT:    [[TMP139:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[TMP8]]
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP140:%.*]] = load i8, ptr [[TMP139]], align 1
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP141:%.*]] = insertelement <16 x i8> [[TMP137]], i8 [[TMP140]], i32 8
 ; CHECK-INTERLEAVE1-NEXT:    br label [[PRED_LOAD_CONTINUE48]]
@@ -1843,7 +1654,7 @@ define i32 @not_dotp_predicated_pragma(i64 %N, ptr %a, ptr %b) {
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP143:%.*]] = extractelement <16 x i1> [[TMP16]], i32 9
 ; CHECK-INTERLEAVE1-NEXT:    br i1 [[TMP143]], label [[PRED_LOAD_IF49:%.*]], label [[PRED_LOAD_CONTINUE50:%.*]]
 ; CHECK-INTERLEAVE1:       pred.load.if49:
-; CHECK-INTERLEAVE1-NEXT:    [[TMP144:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[TMP9]]
+; CHECK-INTERLEAVE1-NEXT:    [[TMP144:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[TMP9]]
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP145:%.*]] = load i8, ptr [[TMP144]], align 1
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP146:%.*]] = insertelement <16 x i8> [[TMP142]], i8 [[TMP145]], i32 9
 ; CHECK-INTERLEAVE1-NEXT:    br label [[PRED_LOAD_CONTINUE50]]
@@ -1852,7 +1663,7 @@ define i32 @not_dotp_predicated_pragma(i64 %N, ptr %a, ptr %b) {
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP148:%.*]] = extractelement <16 x i1> [[TMP16]], i32 10
 ; CHECK-INTERLEAVE1-NEXT:    br i1 [[TMP148]], label [[PRED_LOAD_IF51:%.*]], label [[PRED_LOAD_CONTINUE52:%.*]]
 ; CHECK-INTERLEAVE1:       pred.load.if51:
-; CHECK-INTERLEAVE1-NEXT:    [[TMP149:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[TMP10]]
+; CHECK-INTERLEAVE1-NEXT:    [[TMP149:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[TMP10]]
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP150:%.*]] = load i8, ptr [[TMP149]], align 1
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP151:%.*]] = insertelement <16 x i8> [[TMP147]], i8 [[TMP150]], i32 10
 ; CHECK-INTERLEAVE1-NEXT:    br label [[PRED_LOAD_CONTINUE52]]
@@ -1861,7 +1672,7 @@ define i32 @not_dotp_predicated_pragma(i64 %N, ptr %a, ptr %b) {
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP153:%.*]] = extractelement <16 x i1> [[TMP16]], i32 11
 ; CHECK-INTERLEAVE1-NEXT:    br i1 [[TMP153]], label [[PRED_LOAD_IF53:%.*]], label [[PRED_LOAD_CONTINUE54:%.*]]
 ; CHECK-INTERLEAVE1:       pred.load.if53:
-; CHECK-INTERLEAVE1-NEXT:    [[TMP154:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[TMP11]]
+; CHECK-INTERLEAVE1-NEXT:    [[TMP154:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[TMP11]]
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP155:%.*]] = load i8, ptr [[TMP154]], align 1
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP156:%.*]] = insertelement <16 x i8> [[TMP152]], i8 [[TMP155]], i32 11
 ; CHECK-INTERLEAVE1-NEXT:    br label [[PRED_LOAD_CONTINUE54]]
@@ -1870,7 +1681,7 @@ define i32 @not_dotp_predicated_pragma(i64 %N, ptr %a, ptr %b) {
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP158:%.*]] = extractelement <16 x i1> [[TMP16]], i32 12
 ; CHECK-INTERLEAVE1-NEXT:    br i1 [[TMP158]], label [[PRED_LOAD_IF55:%.*]], label [[PRED_LOAD_CONTINUE56:%.*]]
 ; CHECK-INTERLEAVE1:       pred.load.if55:
-; CHECK-INTERLEAVE1-NEXT:    [[TMP159:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[TMP12]]
+; CHECK-INTERLEAVE1-NEXT:    [[TMP159:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[TMP12]]
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP160:%.*]] = load i8, ptr [[TMP159]], align 1
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP161:%.*]] = insertelement <16 x i8> [[TMP157]], i8 [[TMP160]], i32 12
 ; CHECK-INTERLEAVE1-NEXT:    br label [[PRED_LOAD_CONTINUE56]]
@@ -1879,7 +1690,7 @@ define i32 @not_dotp_predicated_pragma(i64 %N, ptr %a, ptr %b) {
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP163:%.*]] = extractelement <16 x i1> [[TMP16]], i32 13
 ; CHECK-INTERLEAVE1-NEXT:    br i1 [[TMP163]], label [[PRED_LOAD_IF57:%.*]], label [[PRED_LOAD_CONTINUE58:%.*]]
 ; CHECK-INTERLEAVE1:       pred.load.if57:
-; CHECK-INTERLEAVE1-NEXT:    [[TMP164:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[TMP13]]
+; CHECK-INTERLEAVE1-NEXT:    [[TMP164:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[TMP13]]
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP165:%.*]] = load i8, ptr [[TMP164]], align 1
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP166:%.*]] = insertelement <16 x i8> [[TMP162]], i8 [[TMP165]], i32 13
 ; CHECK-INTERLEAVE1-NEXT:    br label [[PRED_LOAD_CONTINUE58]]
@@ -1888,7 +1699,7 @@ define i32 @not_dotp_predicated_pragma(i64 %N, ptr %a, ptr %b) {
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP168:%.*]] = extractelement <16 x i1> [[TMP16]], i32 14
 ; CHECK-INTERLEAVE1-NEXT:    br i1 [[TMP168]], label [[PRED_LOAD_IF59:%.*]], label [[PRED_LOAD_CONTINUE60:%.*]]
 ; CHECK-INTERLEAVE1:       pred.load.if59:
-; CHECK-INTERLEAVE1-NEXT:    [[TMP169:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[TMP14]]
+; CHECK-INTERLEAVE1-NEXT:    [[TMP169:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[TMP14]]
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP170:%.*]] = load i8, ptr [[TMP169]], align 1
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP171:%.*]] = insertelement <16 x i8> [[TMP167]], i8 [[TMP170]], i32 14
 ; CHECK-INTERLEAVE1-NEXT:    br label [[PRED_LOAD_CONTINUE60]]
@@ -1897,7 +1708,7 @@ define i32 @not_dotp_predicated_pragma(i64 %N, ptr %a, ptr %b) {
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP173:%.*]] = extractelement <16 x i1> [[TMP16]], i32 15
 ; CHECK-INTERLEAVE1-NEXT:    br i1 [[TMP173]], label [[PRED_LOAD_IF61:%.*]], label [[PRED_LOAD_CONTINUE62]]
 ; CHECK-INTERLEAVE1:       pred.load.if61:
-; CHECK-INTERLEAVE1-NEXT:    [[TMP174:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[TMP15]]
+; CHECK-INTERLEAVE1-NEXT:    [[TMP174:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[TMP15]]
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP175:%.*]] = load i8, ptr [[TMP174]], align 1
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP176:%.*]] = insertelement <16 x i8> [[TMP172]], i8 [[TMP175]], i32 15
 ; CHECK-INTERLEAVE1-NEXT:    br label [[PRED_LOAD_CONTINUE62]]
@@ -1905,38 +1716,38 @@ define i32 @not_dotp_predicated_pragma(i64 %N, ptr %a, ptr %b) {
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP177:%.*]] = phi <16 x i8> [ [[TMP172]], [[PRED_LOAD_CONTINUE60]] ], [ [[TMP176]], [[PRED_LOAD_IF61]] ]
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP178:%.*]] = sext <16 x i8> [[TMP177]] to <16 x i32>
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP179:%.*]] = mul nsw <16 x i32> [[TMP178]], [[TMP97]]
-; CHECK-INTERLEAVE1-NEXT:    [[TMP180]] = add <16 x i32> [[TMP179]], [[VEC_PHI]]
-; CHECK-INTERLEAVE1-NEXT:    [[TMP181:%.*]] = select <16 x i1> [[TMP16]], <16 x i32> [[TMP180]], <16 x i32> [[VEC_PHI]]
+; CHECK-INTERLEAVE1-NEXT:    [[TMP180:%.*]] = select <16 x i1> [[TMP16]], <16 x i32> [[TMP179]], <16 x i32> zeroinitializer
+; CHECK-INTERLEAVE1-NEXT:    [[PARTIAL_REDUCE]] = call <4 x i32> @llvm.experimental.vector.partial.reduce.add.v4i32.v16i32(<4 x i32> [[VEC_PHI]], <16 x i32> [[TMP180]])
 ; CHECK-INTERLEAVE1-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 16
 ; CHECK-INTERLEAVE1-NEXT:    [[VEC_IND_NEXT]] = add <16 x i64> [[VEC_IND]], splat (i64 16)
-; CHECK-INTERLEAVE1-NEXT:    [[TMP182:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-INTERLEAVE1-NEXT:    br i1 [[TMP182]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
+; CHECK-INTERLEAVE1-NEXT:    [[TMP181:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-INTERLEAVE1-NEXT:    br i1 [[TMP181]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
 ; CHECK-INTERLEAVE1:       middle.block:
-; CHECK-INTERLEAVE1-NEXT:    [[TMP183:%.*]] = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> [[TMP181]])
+; CHECK-INTERLEAVE1-NEXT:    [[TMP182:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[PARTIAL_REDUCE]])
 ; CHECK-INTERLEAVE1-NEXT:    br i1 true, label [[EXIT:%.*]], label [[SCALAR_PH]]
 ; CHECK-INTERLEAVE1:       scalar.ph:
 ; CHECK-INTERLEAVE1-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
-; CHECK-INTERLEAVE1-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP183]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ]
+; CHECK-INTERLEAVE1-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP182]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ]
 ; CHECK-INTERLEAVE1-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK-INTERLEAVE1:       for.body:
 ; CHECK-INTERLEAVE1-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
 ; CHECK-INTERLEAVE1-NEXT:    [[ACCUM:%.*]] = phi i32 [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[ADD:%.*]], [[FOR_BODY]] ]
-; CHECK-INTERLEAVE1-NEXT:    [[GEP_A:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[IV]]
+; CHECK-INTERLEAVE1-NEXT:    [[GEP_A:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[IV]]
 ; CHECK-INTERLEAVE1-NEXT:    [[LOAD_A:%.*]] = load i8, ptr [[GEP_A]], align 1
 ; CHECK-INTERLEAVE1-NEXT:    [[EXT_A:%.*]] = sext i8 [[LOAD_A]] to i32
-; CHECK-INTERLEAVE1-NEXT:    [[GEP_A2:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[IV]]
-; CHECK-INTERLEAVE1-NEXT:    [[LOAD_B:%.*]] = load i8, ptr [[GEP_A2]], align 1
+; CHECK-INTERLEAVE1-NEXT:    [[GEP_B:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[IV]]
+; CHECK-INTERLEAVE1-NEXT:    [[LOAD_B:%.*]] = load i8, ptr [[GEP_B]], align 1
 ; CHECK-INTERLEAVE1-NEXT:    [[EXT_B:%.*]] = sext i8 [[LOAD_B]] to i32
 ; CHECK-INTERLEAVE1-NEXT:    [[MUL:%.*]] = mul nsw i32 [[EXT_B]], [[EXT_A]]
 ; CHECK-INTERLEAVE1-NEXT:    [[ADD]] = add nsw i32 [[MUL]], [[ACCUM]]
 ; CHECK-INTERLEAVE1-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
 ; CHECK-INTERLEAVE1-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; CHECK-INTERLEAVE1-NEXT:    br i1 [[EXITCOND_NOT]], label [[EXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]]
+; CHECK-INTERLEAVE1-NEXT:    br i1 [[EXITCOND_NOT]], label [[EXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
 ; CHECK-INTERLEAVE1:       exit:
-; CHECK-INTERLEAVE1-NEXT:    [[ADD_LCSSA:%.*]] = phi i32 [ [[ADD]], [[FOR_BODY]] ], [ [[TMP183]], [[MIDDLE_BLOCK]] ]
+; CHECK-INTERLEAVE1-NEXT:    [[ADD_LCSSA:%.*]] = phi i32 [ [[ADD]], [[FOR_BODY]] ], [ [[TMP182]], [[MIDDLE_BLOCK]] ]
 ; CHECK-INTERLEAVE1-NEXT:    ret i32 [[ADD_LCSSA]]
 ;
-; CHECK-INTERLEAVED-LABEL: define i32 @not_dotp_predicated_pragma(
+; CHECK-INTERLEAVED-LABEL: define i32 @dotp_predicated(
 ; CHECK-INTERLEAVED-SAME: i64 [[N:%.*]], ptr [[A:%.*]], ptr [[B:%.*]]) #[[ATTR0]] {
 ; CHECK-INTERLEAVED-NEXT:  entry:
 ; CHECK-INTERLEAVED-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
@@ -1951,7 +1762,7 @@ define i32 @not_dotp_predicated_pragma(i64 %N, ptr %a, ptr %b) {
 ; CHECK-INTERLEAVED:       vector.body:
 ; CHECK-INTERLEAVED-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_LOAD_CONTINUE62:%.*]] ]
 ; CHECK-INTERLEAVED-NEXT:    [[VEC_IND:%.*]] = phi <16 x i64> [ <i64 0, i64 1, i64 2, i64 3, i64 4, i64 5, i64 6, i64 7, i64 8, i64 9, i64 10, i64 11, i64 12, i64 13, i64 14, i64 15>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[PRED_LOAD_CONTINUE62]] ]
-; CHECK-INTERLEAVED-NEXT:    [[VEC_PHI:%.*]] = phi <16 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP180:%.*]], [[PRED_LOAD_CONTINUE62]] ]
+; CHECK-INTERLEAVED-NEXT:    [[VEC_PHI:%.*]] = phi <4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[PARTIAL_REDUCE:%.*]], [[PRED_LOAD_CONTINUE62]] ]
 ; CHECK-INTERLEAVED-NEXT:    [[TMP0:%.*]] = add i64 [[INDEX]], 0
 ; CHECK-INTERLEAVED-NEXT:    [[TMP1:%.*]] = add i64 [[INDEX]], 1
 ; CHECK-INTERLEAVED-NEXT:    [[TMP2:%.*]] = add i64 [[INDEX]], 2
@@ -1972,7 +1783,7 @@ define i32 @not_dotp_predicated_pragma(i64 %N, ptr %a, ptr %b) {
 ; CHECK-INTERLEAVED-NEXT:    [[TMP17:%.*]] = extractelement <16 x i1> [[TMP16]], i32 0
 ; CHECK-INTERLEAVED-NEXT:    br i1 [[TMP17]], label [[PRED_LOAD_IF:%.*]], label [[PRED_LOAD_CONTINUE:%.*]]
 ; CHECK-INTERLEAVED:       pred.load.if:
-; CHECK-INTERLEAVED-NEXT:    [[TMP18:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[TMP0]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP18:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[TMP0]]
 ; CHECK-INTERLEAVED-NEXT:    [[TMP19:%.*]] = load i8, ptr [[TMP18]], align 1
 ; CHECK-INTERLEAVED-NEXT:    [[TMP20:%.*]] = insertelement <16 x i8> poison, i8 [[TMP19]], i32 0
 ; CHECK-INTERLEAVED-NEXT:    br label [[PRED_LOAD_CONTINUE]]
@@ -1981,7 +1792,7 @@ define i32 @not_dotp_predicated_pragma(i64 %N, ptr %a, ptr %b) {
 ; CHECK-INTERLEAVED-NEXT:    [[TMP22:%.*]] = extractelement <16 x i1> [[TMP16]], i32 1
 ; CHECK-INTERLEAVED-NEXT:    br i1 [[TMP22]], label [[PRED_LOAD_IF1:%.*]], label [[PRED_LOAD_CONTINUE2:%.*]]
 ; CHECK-INTERLEAVED:       pred.load.if1:
-; CHECK-INTERLEAVED-NEXT:    [[TMP23:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[TMP1]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP23:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[TMP1]]
 ; CHECK-INTERLEAVED-NEXT:    [[TMP24:%.*]] = load i8, ptr [[TMP23]], align 1
 ; CHECK-INTERLEAVED-NEXT:    [[TMP25:%.*]] = insertelement <16 x i8> [[TMP21]], i8 [[TMP24]], i32 1
 ; CHECK-INTERLEAVED-NEXT:    br label [[PRED_LOAD_CONTINUE2]]
@@ -1990,7 +1801,7 @@ define i32 @not_dotp_predicated_pragma(i64 %N, ptr %a, ptr %b) {
 ; CHECK-INTERLEAVED-NEXT:    [[TMP27:%.*]] = extractelement <16 x i1> [[TMP16]], i32 2
 ; CHECK-INTERLEAVED-NEXT:    br i1 [[TMP27]], label [[PRED_LOAD_IF3:%.*]], label [[PRED_LOAD_CONTINUE4:%.*]]
 ; CHECK-INTERLEAVED:       pred.load.if3:
-; CHECK-INTERLEAVED-NEXT:    [[TMP28:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[TMP2]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP28:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[TMP2]]
 ; CHECK-INTERLEAVED-NEXT:    [[TMP29:%.*]] = load i8, ptr [[TMP28]], align 1
 ; CHECK-INTERLEAVED-NEXT:    [[TMP30:%.*]] = insertelement <16 x i8> [[TMP26]], i8 [[TMP29]], i32 2
 ; CHECK-INTERLEAVED-NEXT:    br label [[PRED_LOAD_CONTINUE4]]
@@ -1999,7 +1810,7 @@ define i32 @not_dotp_predicated_pragma(i64 %N, ptr %a, ptr %b) {
 ; CHECK-INTERLEAVED-NEXT:    [[TMP32:%.*]] = extractelement <16 x i1> [[TMP16]], i32 3
 ; CHECK-INTERLEAVED-NEXT:    br i1 [[TMP32]], label [[PRED_LOAD_IF5:%.*]], label [[PRED_LOAD_CONTINUE6:%.*]]
 ; CHECK-INTERLEAVED:       pred.load.if5:
-; CHECK-INTERLEAVED-NEXT:    [[TMP33:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[TMP3]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP33:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[TMP3]]
 ; CHECK-INTERLEAVED-NEXT:    [[TMP34:%.*]] = load i8, ptr [[TMP33]], align 1
 ; CHECK-INTERLEAVED-NEXT:    [[TMP35:%.*]] = insertelement <16 x i8> [[TMP31]], i8 [[TMP34]], i32 3
 ; CHECK-INTERLEAVED-NEXT:    br label [[PRED_LOAD_CONTINUE6]]
@@ -2008,7 +1819,7 @@ define i32 @not_dotp_predicated_pragma(i64 %N, ptr %a, ptr %b) {
 ; CHECK-INTERLEAVED-NEXT:    [[TMP37:%.*]] = extractelement <16 x i1> [[TMP16]], i32 4
 ; CHECK-INTERLEAVED-NEXT:    br i1 [[TMP37]], label [[PRED_LOAD_IF7:%.*]], label [[PRED_LOAD_CONTINUE8:%.*]]
 ; CHECK-INTERLEAVED:       pred.load.if7:
-; CHECK-INTERLEAVED-NEXT:    [[TMP38:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[TMP4]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP38:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[TMP4]]
 ; CHECK-INTERLEAVED-NEXT:    [[TMP39:%.*]] = load i8, ptr [[TMP38]], align 1
 ; CHECK-INTERLEAVED-NEXT:    [[TMP40:%.*]] = insertelement <16 x i8> [[TMP36]], i8 [[TMP39]], i32 4
 ; CHECK-INTERLEAVED-NEXT:    br label [[PRED_LOAD_CONTINUE8]]
@@ -2017,7 +1828,7 @@ define i32 @not_dotp_predicated_pragma(i64 %N, ptr %a, ptr %b) {
 ; CHECK-INTERLEAVED-NEXT:    [[TMP42:%.*]] = extractelement <16 x i1> [[TMP16]], i32 5
 ; CHECK-INTERLEAVED-NEXT:    br i1 [[TMP42]], label [[PRED_LOAD_IF9:%.*]], label [[PRED_LOAD_CONTINUE10:%.*]]
 ; CHECK-INTERLEAVED:       pred.load.if9:
-; CHECK-INTERLEAVED-NEXT:    [[TMP43:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[TMP5]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP43:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[TMP5]]
 ; CHECK-INTERLEAVED-NEXT:    [[TMP44:%.*]] = load i8, ptr [[TMP43]], align 1
 ; CHECK-INTERLEAVED-NEXT:    [[TMP45:%.*]] = insertelement <16 x i8> [[TMP41]], i8 [[TMP44]], i32 5
 ; CHECK-INTERLEAVED-NEXT:    br label [[PRED_LOAD_CONTINUE10]]
@@ -2026,7 +1837,7 @@ define i32 @not_dotp_predicated_pragma(i64 %N, ptr %a, ptr %b) {
 ; CHECK-INTERLEAVED-NEXT:    [[TMP47:%.*]] = extractelement <16 x i1> [[TMP16]], i32 6
 ; CHECK-INTERLEAVED-NEXT:    br i1 [[TMP47]], label [[PRED_LOAD_IF11:%.*]], label [[PRED_LOAD_CONTINUE12:%.*]]
 ; CHECK-INTERLEAVED:       pred.load.if11:
-; CHECK-INTERLEAVED-NEXT:    [[TMP48:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[TMP6]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP48:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[TMP6]]
 ; CHECK-INTERLEAVED-NEXT:    [[TMP49:%.*]] = load i8, ptr [[TMP48]], align 1
 ; CHECK-INTERLEAVED-NEXT:    [[TMP50:%.*]] = insertelement <16 x i8> [[TMP46]], i8 [[TMP49]], i32 6
 ; CHECK-INTERLEAVED-NEXT:    br label [[PRED_LOAD_CONTINUE12]]
@@ -2035,7 +1846,7 @@ define i32 @not_dotp_predicated_pragma(i64 %N, ptr %a, ptr %b) {
 ; CHECK-INTERLEAVED-NEXT:    [[TMP52:%.*]] = extractelement <16 x i1> [[TMP16]], i32 7
 ; CHECK-INTERLEAVED-NEXT:    br i1 [[TMP52]], label [[PRED_LOAD_IF13:%.*]], label [[PRED_LOAD_CONTINUE14:%.*]]
 ; CHECK-INTERLEAVED:       pred.load.if13:
-; CHECK-INTERLEAVED-NEXT:    [[TMP53:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[TMP7]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP53:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[TMP7]]
 ; CHECK-INTERLEAVED-NEXT:    [[TMP54:%.*]] = load i8, ptr [[TMP53]], align 1
 ; CHECK-INTERLEAVED-NEXT:    [[TMP55:%.*]] = insertelement <16 x i8> [[TMP51]], i8 [[TMP54]], i32 7
 ; CHECK-INTERLEAVED-NEXT:    br label [[PRED_LOAD_CONTINUE14]]
@@ -2044,7 +1855,7 @@ define i32 @not_dotp_predicated_pragma(i64 %N, ptr %a, ptr %b) {
 ; CHECK-INTERLEAVED-NEXT:    [[TMP57:%.*]] = extractelement <16 x i1> [[TMP16]], i32 8
 ; CHECK-INTERLEAVED-NEXT:    br i1 [[TMP57]], label [[PRED_LOAD_IF15:%.*]], label [[PRED_LOAD_CONTINUE16:%.*]]
 ; CHECK-INTERLEAVED:       pred.load.if15:
-; CHECK-INTERLEAVED-NEXT:    [[TMP58:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[TMP8]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP58:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[TMP8]]
 ; CHECK-INTERLEAVED-NEXT:    [[TMP59:%.*]] = load i8, ptr [[TMP58]], align 1
 ; CHECK-INTERLEAVED-NEXT:    [[TMP60:%.*]] = insertelement <16 x i8> [[TMP56]], i8 [[TMP59]], i32 8
 ; CHECK-INTERLEAVED-NEXT:    br label [[PRED_LOAD_CONTINUE16]]
@@ -2053,7 +1864,7 @@ define i32 @not_dotp_predicated_pragma(i64 %N, ptr %a, ptr %b) {
 ; CHECK-INTERLEAVED-NEXT:    [[TMP62:%.*]] = extractelement <16 x i1> [[TMP16]], i32 9
 ; CHECK-INTERLEAVED-NEXT:    br i1 [[TMP62]], label [[PRED_LOAD_IF17:%.*]], label [[PRED_LOAD_CONTINUE18:%.*]]
 ; CHECK-INTERLEAVED:       pred.load.if17:
-; CHECK-INTERLEAVED-NEXT:    [[TMP63:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[TMP9]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP63:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[TMP9]]
 ; CHECK-INTERLEAVED-NEXT:    [[TMP64:%.*]] = load i8, ptr [[TMP63]], align 1
 ; CHECK-INTERLEAVED-NEXT:    [[TMP65:%.*]] = insertelement <16 x i8> [[TMP61]], i8 [[TMP64]], i32 9
 ; CHECK-INTERLEAVED-NEXT:    br label [[PRED_LOAD_CONTINUE18]]
@@ -2062,7 +1873,7 @@ define i32 @not_dotp_predicated_pragma(i64 %N, ptr %a, ptr %b) {
 ; CHECK-INTERLEAVED-NEXT:    [[TMP67:%.*]] = extractelement <16 x i1> [[TMP16]], i32 10
 ; CHECK-INTERLEAVED-NEXT:    br i1 [[TMP67]], label [[PRED_LOAD_IF19:%.*]], label [[PRED_LOAD_CONTINUE20:%.*]]
 ; CHECK-INTERLEAVED:       pred.load.if19:
-; CHECK-INTERLEAVED-NEXT:    [[TMP68:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[TMP10]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP68:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[TMP10]]
 ; CHECK-INTERLEAVED-NEXT:    [[TMP69:%.*]] = load i8, ptr [[TMP68]], align 1
 ; CHECK-INTERLEAVED-NEXT:    [[TMP70:%.*]] = insertelement <16 x i8> [[TMP66]], i8 [[TMP69]], i32 10
 ; CHECK-INTERLEAVED-NEXT:    br label [[PRED_LOAD_CONTINUE20]]
@@ -2071,7 +1882,7 @@ define i32 @not_dotp_predicated_pragma(i64 %N, ptr %a, ptr %b) {
 ; CHECK-INTERLEAVED-NEXT:    [[TMP72:%.*]] = extractelement <16 x i1> [[TMP16]], i32 11
 ; CHECK-INTERLEAVED-NEXT:    br i1 [[TMP72]], label [[PRED_LOAD_IF21:%.*]], label [[PRED_LOAD_CONTINUE22:%.*]]
 ; CHECK-INTERLEAVED:       pred.load.if21:
-; CHECK-INTERLEAVED-NEXT:    [[TMP73:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[TMP11]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP73:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[TMP11]]
 ; CHECK-INTERLEAVED-NEXT:    [[TMP74:%.*]] = load i8, ptr [[TMP73]], align 1
 ; CHECK-INTERLEAVED-NEXT:    [[TMP75:%.*]] = insertelement <16 x i8> [[TMP71]], i8 [[TMP74]], i32 11
 ; CHECK-INTERLEAVED-NEXT:    br label [[PRED_LOAD_CONTINUE22]]
@@ -2080,7 +1891,7 @@ define i32 @not_dotp_predicated_pragma(i64 %N, ptr %a, ptr %b) {
 ; CHECK-INTERLEAVED-NEXT:    [[TMP77:%.*]] = extractelement <16 x i1> [[TMP16]], i32 12
 ; CHECK-INTERLEAVED-NEXT:    br i1 [[TMP77]], label [[PRED_LOAD_IF23:%.*]], label [[PRED_LOAD_CONTINUE24:%.*]]
 ; CHECK-INTERLEAVED:       pred.load.if23:
-; CHECK-INTERLEAVED-NEXT:    [[TMP78:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[TMP12]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP78:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[TMP12]]
 ; CHECK-INTERLEAVED-NEXT:    [[TMP79:%.*]] = load i8, ptr [[TMP78]], align 1
 ; CHECK-INTERLEAVED-NEXT:    [[TMP80:%.*]] = insertelement <16 x i8> [[TMP76]], i8 [[TMP79]], i32 12
 ; CHECK-INTERLEAVED-NEXT:    br label [[PRED_LOAD_CONTINUE24]]
@@ -2089,7 +1900,7 @@ define i32 @not_dotp_predicated_pragma(i64 %N, ptr %a, ptr %b) {
 ; CHECK-INTERLEAVED-NEXT:    [[TMP82:%.*]] = extractelement <16 x i1> [[TMP16]], i32 13
 ; CHECK-INTERLEAVED-NEXT:    br i1 [[TMP82]], label [[PRED_LOAD_IF25:%.*]], label [[PRED_LOAD_CONTINUE26:%.*]]
 ; CHECK-INTERLEAVED:       pred.load.if25:
-; CHECK-INTERLEAVED-NEXT:    [[TMP83:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[TMP13]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP83:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[TMP13]]
 ; CHECK-INTERLEAVED-NEXT:    [[TMP84:%.*]] = load i8, ptr [[TMP83]], align 1
 ; CHECK-INTERLEAVED-NEXT:    [[TMP85:%.*]] = insertelement <16 x i8> [[TMP81]], i8 [[TMP84]], i32 13
 ; CHECK-INTERLEAVED-NEXT:    br label [[PRED_LOAD_CONTINUE26]]
@@ -2098,7 +1909,7 @@ define i32 @not_dotp_predicated_pragma(i64 %N, ptr %a, ptr %b) {
 ; CHECK-INTERLEAVED-NEXT:    [[TMP87:%.*]] = extractelement <16 x i1> [[TMP16]], i32 14
 ; CHECK-INTERLEAVED-NEXT:    br i1 [[TMP87]], label [[PRED_LOAD_IF27:%.*]], label [[PRED_LOAD_CONTINUE28:%.*]]
 ; CHECK-INTERLEAVED:       pred.load.if27:
-; CHECK-INTERLEAVED-NEXT:    [[TMP88:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[TMP14]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP88:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[TMP14]]
 ; CHECK-INTERLEAVED-NEXT:    [[TMP89:%.*]] = load i8, ptr [[TMP88]], align 1
 ; CHECK-INTERLEAVED-NEXT:    [[TMP90:%.*]] = insertelement <16 x i8> [[TMP86]], i8 [[TMP89]], i32 14
 ; CHECK-INTERLEAVED-NEXT:    br label [[PRED_LOAD_CONTINUE28]]
@@ -2107,7 +1918,7 @@ define i32 @not_dotp_predicated_pragma(i64 %N, ptr %a, ptr %b) {
 ; CHECK-INTERLEAVED-NEXT:    [[TMP92:%.*]] = extractelement <16 x i1> [[TMP16]], i32 15
 ; CHECK-INTERLEAVED-NEXT:    br i1 [[TMP92]], label [[PRED_LOAD_IF29:%.*]], label [[PRED_LOAD_CONTINUE30:%.*]]
 ; CHECK-INTERLEAVED:       pred.load.if29:
-; CHECK-INTERLEAVED-NEXT:    [[TMP93:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[TMP15]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP93:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[TMP15]]
 ; CHECK-INTERLEAVED-NEXT:    [[TMP94:%.*]] = load i8, ptr [[TMP93]], align 1
 ; CHECK-INTERLEAVED-NEXT:    [[TMP95:%.*]] = insertelement <16 x i8> [[TMP91]], i8 [[TMP94]], i32 15
 ; CHECK-INTERLEAVED-NEXT:    br label [[PRED_LOAD_CONTINUE30]]
@@ -2117,7 +1928,7 @@ define i32 @not_dotp_predicated_pragma(i64 %N, ptr %a, ptr %b) {
 ; CHECK-INTERLEAVED-NEXT:    [[TMP98:%.*]] = extractelement <16 x i1> [[TMP16]], i32 0
 ; CHECK-INTERLEAVED-NEXT:    br i1 [[TMP98]], label [[PRED_LOAD_IF31:%.*]], label [[PRED_LOAD_CONTINUE32:%.*]]
 ; CHECK-INTERLEAVED:       pred.load.if31:
-; CHECK-INTERLEAVED-NEXT:    [[TMP99:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[TMP0]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP99:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[TMP0]]
 ; CHECK-INTERLEAVED-NEXT:    [[TMP100:%.*]] = load i8, ptr [[TMP99]], align 1
 ; CHECK-INTERLEAVED-NEXT:    [[TMP101:%.*]] = insertelement <16 x i8> poison, i8 [[TMP100]], i32 0
 ; CHECK-INTERLEAVED-NEXT:    br label [[PRED_LOAD_CONTINUE32]]
@@ -2126,7 +1937,7 @@ define i32 @not_dotp_predicated_pragma(i64 %N, ptr %a, ptr %b) {
 ; CHECK-INTERLEAVED-NEXT:    [[TMP103:%.*]] = extractelement <16 x i1> [[TMP16]], i32 1
 ; CHECK-INTERLEAVED-NEXT:    br i1 [[TMP103]], label [[PRED_LOAD_IF33:%.*]], label [[PRED_LOAD_CONTINUE34:%.*]]
 ; CHECK-INTERLEAVED:       pred.load.if33:
-; CHECK-INTERLEAVED-NEXT:    [[TMP104:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[TMP1]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP104:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[TMP1]]
 ; CHECK-INTERLEAVED-NEXT:    [[TMP105:%.*]] = load i8, ptr [[TMP104]], align 1
 ; CHECK-INTERLEAVED-NEXT:    [[TMP106:%.*]] = insertelement <16 x i8> [[TMP102]], i8 [[TMP105]], i32 1
 ; CHECK-INTERLEAVED-NEXT:    br label [[PRED_LOAD_CONTINUE34]]
@@ -2135,7 +1946,7 @@ define i32 @not_dotp_predicated_pragma(i64 %N, ptr %a, ptr %b) {
 ; CHECK-INTERLEAVED-NEXT:    [[TMP108:%.*]] = extractelement <16 x i1> [[TMP16]], i32 2
 ; CHECK-INTERLEAVED-NEXT:    br i1 [[TMP108]], label [[PRED_LOAD_IF35:%.*]], label [[PRED_LOAD_CONTINUE36:%.*]]
 ; CHECK-INTERLEAVED:       pred.load.if35:
-; CHECK-INTERLEAVED-NEXT:    [[TMP109:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[TMP2]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP109:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[TMP2]]
 ; CHECK-INTERLEAVED-NEXT:    [[TMP110:%.*]] = load i8, ptr [[TMP109]], align 1
 ; CHECK-INTERLEAVED-NEXT:    [[TMP111:%.*]] = insertelement <16 x i8> [[TMP107]], i8 [[TMP110]], i32 2
 ; CHECK-INTERLEAVED-NEXT:    br label [[PRED_LOAD_CONTINUE36]]
@@ -2144,7 +1955,7 @@ define i32 @not_dotp_predicated_pragma(i64 %N, ptr %a, ptr %b) {
 ; CHECK-INTERLEAVED-NEXT:    [[TMP113:%.*]] = extractelement <16 x i1> [[TMP16]], i32 3
 ; CHECK-INTERLEAVED-NEXT:    br i1 [[TMP113]], label [[PRED_LOAD_IF37:%.*]], label [[PRED_LOAD_CONTINUE38:%.*]]
 ; CHECK-INTERLEAVED:       pred.load.if37:
-; CHECK-INTERLEAVED-NEXT:    [[TMP114:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[TMP3]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP114:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[TMP3]]
 ; CHECK-INTERLEAVED-NEXT:    [[TMP115:%.*]] = load i8, ptr [[TMP114]], align 1
 ; CHECK-INTERLEAVED-NEXT:    [[TMP116:%.*]] = insertelement <16 x i8> [[TMP112]], i8 [[TMP115]], i32 3
 ; CHECK-INTERLEAVED-NEXT:    br label [[PRED_LOAD_CONTINUE38]]
@@ -2153,7 +1964,7 @@ define i32 @not_dotp_predicated_pragma(i64 %N, ptr %a, ptr %b) {
 ; CHECK-INTERLEAVED-NEXT:    [[TMP118:%.*]] = extractelement <16 x i1> [[TMP16]], i32 4
 ; CHECK-INTERLEAVED-NEXT:    br i1 [[TMP118]], label [[PRED_LOAD_IF39:%.*]], label [[PRED_LOAD_CONTINUE40:%.*]]
 ; CHECK-INTERLEAVED:       pred.load.if39:
-; CHECK-INTERLEAVED-NEXT:    [[TMP119:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[TMP4]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP119:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[TMP4]]
 ; CHECK-INTERLEAVED-NEXT:    [[TMP120:%.*]] = load i8, ptr [[TMP119]], align 1
 ; CHECK-INTERLEAVED-NEXT:    [[TMP121:%.*]] = insertelement <16 x i8> [[TMP117]], i8 [[TMP120]], i32 4
 ; CHECK-INTERLEAVED-NEXT:    br label [[PRED_LOAD_CONTINUE40]]
@@ -2162,7 +1973,7 @@ define i32 @not_dotp_predicated_pragma(i64 %N, ptr %a, ptr %b) {
 ; CHECK-INTERLEAVED-NEXT:    [[TMP123:%.*]] = extractelement <16 x i1> [[TMP16]], i32 5
 ; CHECK-INTERLEAVED-NEXT:    br i1 [[TMP123]], label [[PRED_LOAD_IF41:%.*]], label [[PRED_LOAD_CONTINUE42:%.*]]
 ; CHECK-INTERLEAVED:       pred.load.if41:
-; CHECK-INTERLEAVED-NEXT:    [[TMP124:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[TMP5]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP124:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[TMP5]]
 ; CHECK-INTERLEAVED-NEXT:    [[TMP125:%.*]] = load i8, ptr [[TMP124]], align 1
 ; CHECK-INTERLEAVED-NEXT:    [[TMP126:%.*]] = insertelement <16 x i8> [[TMP122]], i8 [[TMP125]], i32 5
 ; CHECK-INTERLEAVED-NEXT:    br label [[PRED_LOAD_CONTINUE42]]
@@ -2171,7 +1982,7 @@ define i32 @not_dotp_predicated_pragma(i64 %N, ptr %a, ptr %b) {
 ; CHECK-INTERLEAVED-NEXT:    [[TMP128:%.*]] = extractelement <16 x i1> [[TMP16]], i32 6
 ; CHECK-INTERLEAVED-NEXT:    br i1 [[TMP128]], label [[PRED_LOAD_IF43:%.*]], label [[PRED_LOAD_CONTINUE44:%.*]]
 ; CHECK-INTERLEAVED:       pred.load.if43:
-; CHECK-INTERLEAVED-NEXT:    [[TMP129:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[TMP6]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP129:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[TMP6]]
 ; CHECK-INTERLEAVED-NEXT:    [[TMP130:%.*]] = load i8, ptr [[TMP129]], align 1
 ; CHECK-INTERLEAVED-NEXT:    [[TMP131:%.*]] = insertelement <16 x i8> [[TMP127]], i8 [[TMP130]], i32 6
 ; CHECK-INTERLEAVED-NEXT:    br label [[PRED_LOAD_CONTINUE44]]
@@ -2180,7 +1991,7 @@ define i32 @not_dotp_predicated_pragma(i64 %N, ptr %a, ptr %b) {
 ; CHECK-INTERLEAVED-NEXT:    [[TMP133:%.*]] = extractelement <16 x i1> [[TMP16]], i32 7
 ; CHECK-INTERLEAVED-NEXT:    br i1 [[TMP133]], label [[PRED_LOAD_IF45:%.*]], label [[PRED_LOAD_CONTINUE46:%.*]]
 ; CHECK-INTERLEAVED:       pred.load.if45:
-; CHECK-INTERLEAVED-NEXT:    [[TMP134:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[TMP7]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP134:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[TMP7]]
 ; CHECK-INTERLEAVED-NEXT:    [[TMP135:%.*]] = load i8, ptr [[TMP134]], align 1
 ; CHECK-INTERLEAVED-NEXT:    [[TMP136:%.*]] = insertelement <16 x i8> [[TMP132]], i8 [[TMP135]], i32 7
 ; CHECK-INTERLEAVED-NEXT:    br label [[PRED_LOAD_CONTINUE46]]
@@ -2189,7 +2000,7 @@ define i32 @not_dotp_predicated_pragma(i64 %N, ptr %a, ptr %b) {
 ; CHECK-INTERLEAVED-NEXT:    [[TMP138:%.*]] = extractelement <16 x i1> [[TMP16]], i32 8
 ; CHECK-INTERLEAVED-NEXT:    br i1 [[TMP138]], label [[PRED_LOAD_IF47:%.*]], label [[PRED_LOAD_CONTINUE48:%.*]]
 ; CHECK-INTERLEAVED:       pred.load.if47:
-; CHECK-INTERLEAVED-NEXT:    [[TMP139:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[TMP8]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP139:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[TMP8]]
 ; CHECK-INTERLEAVED-NEXT:    [[TMP140:%.*]] = load i8, ptr [[TMP139]], align 1
 ; CHECK-INTERLEAVED-NEXT:    [[TMP141:%.*]] = insertelement <16 x i8> [[TMP137]], i8 [[TMP140]], i32 8
 ; CHECK-INTERLEAVED-NEXT:    br label [[PRED_LOAD_CONTINUE48]]
@@ -2198,7 +2009,7 @@ define i32 @not_dotp_predicated_pragma(i64 %N, ptr %a, ptr %b) {
 ; CHECK-INTERLEAVED-NEXT:    [[TMP143:%.*]] = extractelement <16 x i1> [[TMP16]], i32 9
 ; CHECK-INTERLEAVED-NEXT:    br i1 [[TMP143]], label [[PRED_LOAD_IF49:%.*]], label [[PRED_LOAD_CONTINUE50:%.*]]
 ; CHECK-INTERLEAVED:       pred.load.if49:
-; CHECK-INTERLEAVED-NEXT:    [[TMP144:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[TMP9]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP144:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[TMP9]]
 ; CHECK-INTERLEAVED-NEXT:    [[TMP145:%.*]] = load i8, ptr [[TMP144]], align 1
 ; CHECK-INTERLEAVED-NEXT:    [[TMP146:%.*]] = insertelement <16 x i8> [[TMP142]], i8 [[TMP145]], i32 9
 ; CHECK-INTERLEAVED-NEXT:    br label [[PRED_LOAD_CONTINUE50]]
@@ -2207,7 +2018,7 @@ define i32 @not_dotp_predicated_pragma(i64 %N, ptr %a, ptr %b) {
 ; CHECK-INTERLEAVED-NEXT:    [[TMP148:%.*]] = extractelement <16 x i1> [[TMP16]], i32 10
 ; CHECK-INTERLEAVED-NEXT:    br i1 [[TMP148]], label [[PRED_LOAD_IF51:%.*]], label [[PRED_LOAD_CONTINUE52:%.*]]
 ; CHECK-INTERLEAVED:       pred.load.if51:
-; CHECK-INTERLEAVED-NEXT:    [[TMP149:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[TMP10]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP149:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[TMP10]]
 ; CHECK-INTERLEAVED-NEXT:    [[TMP150:%.*]] = load i8, ptr [[TMP149]], align 1
 ; CHECK-INTERLEAVED-NEXT:    [[TMP151:%.*]] = insertelement <16 x i8> [[TMP147]], i8 [[TMP150]], i32 10
 ; CHECK-INTERLEAVED-NEXT:    br label [[PRED_LOAD_CONTINUE52]]
@@ -2216,7 +2027,7 @@ define i32 @not_dotp_predicated_pragma(i64 %N, ptr %a, ptr %b) {
 ; CHECK-INTERLEAVED-NEXT:    [[TMP153:%.*]] = extractelement <16 x i1> [[TMP16]], i32 11
 ; CHECK-INTERLEAVED-NEXT:    br i1 [[TMP153]], label [[PRED_LOAD_IF53:%.*]], label [[PRED_LOAD_CONTINUE54:%.*]]
 ; CHECK-INTERLEAVED:       pred.load.if53:
-; CHECK-INTERLEAVED-NEXT:    [[TMP154:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[TMP11]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP154:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[TMP11]]
 ; CHECK-INTERLEAVED-NEXT:    [[TMP155:%.*]] = load i8, ptr [[TMP154]], align 1
 ; CHECK-INTERLEAVED-NEXT:    [[TMP156:%.*]] = insertelement <16 x i8> [[TMP152]], i8 [[TMP155]], i32 11
 ; CHECK-INTERLEAVED-NEXT:    br label [[PRED_LOAD_CONTINUE54]]
@@ -2225,7 +2036,7 @@ define i32 @not_dotp_predicated_pragma(i64 %N, ptr %a, ptr %b) {
 ; CHECK-INTERLEAVED-NEXT:    [[TMP158:%.*]] = extractelement <16 x i1> [[TMP16]], i32 12
 ; CHECK-INTERLEAVED-NEXT:    br i1 [[TMP158]], label [[PRED_LOAD_IF55:%.*]], label [[PRED_LOAD_CONTINUE56:%.*]]
 ; CHECK-INTERLEAVED:       pred.load.if55:
-; CHECK-INTERLEAVED-NEXT:    [[TMP159:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[TMP12]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP159:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[TMP12]]
 ; CHECK-INTERLEAVED-NEXT:    [[TMP160:%.*]] = load i8, ptr [[TMP159]], align 1
 ; CHECK-INTERLEAVED-NEXT:    [[TMP161:%.*]] = insertelement <16 x i8> [[TMP157]], i8 [[TMP160]], i32 12
 ; CHECK-INTERLEAVED-NEXT:    br label [[PRED_LOAD_CONTINUE56]]
@@ -2234,7 +2045,7 @@ define i32 @not_dotp_predicated_pragma(i64 %N, ptr %a, ptr %b) {
 ; CHECK-INTERLEAVED-NEXT:    [[TMP163:%.*]] = extractelement <16 x i1> [[TMP16]], i32 13
 ; CHECK-INTERLEAVED-NEXT:    br i1 [[TMP163]], label [[PRED_LOAD_IF57:%.*]], label [[PRED_LOAD_CONTINUE58:%.*]]
 ; CHECK-INTERLEAVED:       pred.load.if57:
-; CHECK-INTERLEAVED-NEXT:    [[TMP164:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[TMP13]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP164:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[TMP13]]
 ; CHECK-INTERLEAVED-NEXT:    [[TMP165:%.*]] = load i8, ptr [[TMP164]], align 1
 ; CHECK-INTERLEAVED-NEXT:    [[TMP166:%.*]] = insertelement <16 x i8> [[TMP162]], i8 [[TMP165]], i32 13
 ; CHECK-INTERLEAVED-NEXT:    br label [[PRED_LOAD_CONTINUE58]]
@@ -2243,7 +2054,7 @@ define i32 @not_dotp_predicated_pragma(i64 %N, ptr %a, ptr %b) {
 ; CHECK-INTERLEAVED-NEXT:    [[TMP168:%.*]] = extractelement <16 x i1> [[TMP16]], i32 14
 ; CHECK-INTERLEAVED-NEXT:    br i1 [[TMP168]], label [[PRED_LOAD_IF59:%.*]], label [[PRED_LOAD_CONTINUE60:%.*]]
 ; CHECK-INTERLEAVED:       pred.load.if59:
-; CHECK-INTERLEAVED-NEXT:    [[TMP169:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[TMP14]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP169:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[TMP14]]
 ; CHECK-INTERLEAVED-NEXT:    [[TMP170:%.*]] = load i8, ptr [[TMP169]], align 1
 ; CHECK-INTERLEAVED-NEXT:    [[TMP171:%.*]] = insertelement <16 x i8> [[TMP167]], i8 [[TMP170]], i32 14
 ; CHECK-INTERLEAVED-NEXT:    br label [[PRED_LOAD_CONTINUE60]]
@@ -2252,7 +2063,7 @@ define i32 @not_dotp_predicated_pragma(i64 %N, ptr %a, ptr %b) {
 ; CHECK-INTERLEAVED-NEXT:    [[TMP173:%.*]] = extractelement <16 x i1> [[TMP16]], i32 15
 ; CHECK-INTERLEAVED-NEXT:    br i1 [[TMP173]], label [[PRED_LOAD_IF61:%.*]], label [[PRED_LOAD_CONTINUE62]]
 ; CHECK-INTERLEAVED:       pred.load.if61:
-; CHECK-INTERLEAVED-NEXT:    [[TMP174:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[TMP15]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP174:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[TMP15]]
 ; CHECK-INTERLEAVED-NEXT:    [[TMP175:%.*]] = load i8, ptr [[TMP174]], align 1
 ; CHECK-INTERLEAVED-NEXT:    [[TMP176:%.*]] = insertelement <16 x i8> [[TMP172]], i8 [[TMP175]], i32 15
 ; CHECK-INTERLEAVED-NEXT:    br label [[PRED_LOAD_CONTINUE62]]
@@ -2260,38 +2071,38 @@ define i32 @not_dotp_predicated_pragma(i64 %N, ptr %a, ptr %b) {
 ; CHECK-INTERLEAVED-NEXT:    [[TMP177:%.*]] = phi <16 x i8> [ [[TMP172]], [[PRED_LOAD_CONTINUE60]] ], [ [[TMP176]], [[PRED_LOAD_IF61]] ]
 ; CHECK-INTERLEAVED-NEXT:    [[TMP178:%.*]] = sext <16 x i8> [[TMP177]] to <16 x i32>
 ; CHECK-INTERLEAVED-NEXT:    [[TMP179:%.*]] = mul nsw <16 x i32> [[TMP178]], [[TMP97]]
-; CHECK-INTERLEAVED-NEXT:    [[TMP180]] = add <16 x i32> [[TMP179]], [[VEC_PHI]]
-; CHECK-INTERLEAVED-NEXT:    [[TMP181:%.*]] = select <16 x i1> [[TMP16]], <16 x i32> [[TMP180]], <16 x i32> [[VEC_PHI]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP180:%.*]] = select <16 x i1> [[TMP16]], <16 x i32> [[TMP179]], <16 x i32> zeroinitializer
+; CHECK-INTERLEAVED-NEXT:    [[PARTIAL_REDUCE]] = call <4 x i32> @llvm.experimental.vector.partial.reduce.add.v4i32.v16i32(<4 x i32> [[VEC_PHI]], <16 x i32> [[TMP180]])
 ; CHECK-INTERLEAVED-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 16
 ; CHECK-INTERLEAVED-NEXT:    [[VEC_IND_NEXT]] = add <16 x i64> [[VEC_IND]], splat (i64 16)
-; CHECK-INTERLEAVED-NEXT:    [[TMP182:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-INTERLEAVED-NEXT:    br i1 [[TMP182]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP181:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-INTERLEAVED-NEXT:    br i1 [[TMP181]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
 ; CHECK-INTERLEAVED:       middle.block:
-; CHECK-INTERLEAVED-NEXT:    [[TMP183:%.*]] = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> [[TMP181]])
+; CHECK-INTERLEAVED-NEXT:    [[TMP182:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[PARTIAL_REDUCE]])
 ; CHECK-INTERLEAVED-NEXT:    br i1 true, label [[EXIT:%.*]], label [[SCALAR_PH]]
 ; CHECK-INTERLEAVED:       scalar.ph:
 ; CHECK-INTERLEAVED-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
-; CHECK-INTERLEAVED-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP183]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ]
+; CHECK-INTERLEAVED-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP182]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ]
 ; CHECK-INTERLEAVED-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK-INTERLEAVED:       for.body:
 ; CHECK-INTERLEAVED-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
 ; CHECK-INTERLEAVED-NEXT:    [[ACCUM:%.*]] = phi i32 [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[ADD:%.*]], [[FOR_BODY]] ]
-; CHECK-INTERLEAVED-NEXT:    [[GEP_A:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[IV]]
+; CHECK-INTERLEAVED-NEXT:    [[GEP_A:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[IV]]
 ; CHECK-INTERLEAVED-NEXT:    [[LOAD_A:%.*]] = load i8, ptr [[GEP_A]], align 1
 ; CHECK-INTERLEAVED-NEXT:    [[EXT_A:%.*]] = sext i8 [[LOAD_A]] to i32
-; CHECK-INTERLEAVED-NEXT:    [[GEP_A2:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[IV]]
-; CHECK-INTERLEAVED-NEXT:    [[LOAD_B:%.*]] = load i8, ptr [[GEP_A2]], align 1
+; CHECK-INTERLEAVED-NEXT:    [[GEP_B:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[IV]]
+; CHECK-INTERLEAVED-NEXT:    [[LOAD_B:%.*]] = load i8, ptr [[GEP_B]], align 1
 ; CHECK-INTERLEAVED-NEXT:    [[EXT_B:%.*]] = sext i8 [[LOAD_B]] to i32
 ; CHECK-INTERLEAVED-NEXT:    [[MUL:%.*]] = mul nsw i32 [[EXT_B]], [[EXT_A]]
 ; CHECK-INTERLEAVED-NEXT:    [[ADD]] = add nsw i32 [[MUL]], [[ACCUM]]
 ; CHECK-INTERLEAVED-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
 ; CHECK-INTERLEAVED-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; CHECK-INTERLEAVED-NEXT:    br i1 [[EXITCOND_NOT]], label [[EXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]]
+; CHECK-INTERLEAVED-NEXT:    br i1 [[EXITCOND_NOT]], label [[EXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
 ; CHECK-INTERLEAVED:       exit:
-; CHECK-INTERLEAVED-NEXT:    [[ADD_LCSSA:%.*]] = phi i32 [ [[ADD]], [[FOR_BODY]] ], [ [[TMP183]], [[MIDDLE_BLOCK]] ]
+; CHECK-INTERLEAVED-NEXT:    [[ADD_LCSSA:%.*]] = phi i32 [ [[ADD]], [[FOR_BODY]] ], [ [[TMP182]], [[MIDDLE_BLOCK]] ]
 ; CHECK-INTERLEAVED-NEXT:    ret i32 [[ADD_LCSSA]]
 ;
-; CHECK-MAXBW-LABEL: define i32 @not_dotp_predicated_pragma(
+; CHECK-MAXBW-LABEL: define i32 @dotp_predicated(
 ; CHECK-MAXBW-SAME: i64 [[N:%.*]], ptr [[A:%.*]], ptr [[B:%.*]]) #[[ATTR0]] {
 ; CHECK-MAXBW-NEXT:  entry:
 ; CHECK-MAXBW-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
@@ -2306,7 +2117,7 @@ define i32 @not_dotp_predicated_pragma(i64 %N, ptr %a, ptr %b) {
 ; CHECK-MAXBW:       vector.body:
 ; CHECK-MAXBW-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_LOAD_CONTINUE62:%.*]] ]
 ; CHECK-MAXBW-NEXT:    [[VEC_IND:%.*]] = phi <16 x i64> [ <i64 0, i64 1, i64 2, i64 3, i64 4, i64 5, i64 6, i64 7, i64 8, i64 9, i64 10, i64 11, i64 12, i64 13, i64 14, i64 15>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[PRED_LOAD_CONTINUE62]] ]
-; CHECK-MAXBW-NEXT:    [[VEC_PHI:%.*]] = phi <16 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP180:%.*]], [[PRED_LOAD_CONTINUE62]] ]
+; CHECK-MAXBW-NEXT:    [[VEC_PHI:%.*]] = phi <4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[PARTIAL_REDUCE:%.*]], [[PRED_LOAD_CONTINUE62]] ]
 ; CHECK-MAXBW-NEXT:    [[TMP0:%.*]] = add i64 [[INDEX]], 0
 ; CHECK-MAXBW-NEXT:    [[TMP1:%.*]] = add i64 [[INDEX]], 1
 ; CHECK-MAXBW-NEXT:    [[TMP2:%.*]] = add i64 [[INDEX]], 2
@@ -2327,7 +2138,7 @@ define i32 @not_dotp_predicated_pragma(i64 %N, ptr %a, ptr %b) {
 ; CHECK-MAXBW-NEXT:    [[TMP17:%.*]] = extractelement <16 x i1> [[TMP16]], i32 0
 ; CHECK-MAXBW-NEXT:    br i1 [[TMP17]], label [[PRED_LOAD_IF:%.*]], label [[PRED_LOAD_CONTINUE:%.*]]
 ; CHECK-MAXBW:       pred.load.if:
-; CHECK-MAXBW-NEXT:    [[TMP18:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[TMP0]]
+; CHECK-MAXBW-NEXT:    [[TMP18:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[TMP0]]
 ; CHECK-MAXBW-NEXT:    [[TMP19:%.*]] = load i8, ptr [[TMP18]], align 1
 ; CHECK-MAXBW-NEXT:    [[TMP20:%.*]] = insertelement <16 x i8> poison, i8 [[TMP19]], i32 0
 ; CHECK-MAXBW-NEXT:    br label [[PRED_LOAD_CONTINUE]]
@@ -2336,7 +2147,7 @@ define i32 @not_dotp_predicated_pragma(i64 %N, ptr %a, ptr %b) {
 ; CHECK-MAXBW-NEXT:    [[TMP22:%.*]] = extractelement <16 x i1> [[TMP16]], i32 1
 ; CHECK-MAXBW-NEXT:    br i1 [[TMP22]], label [[PRED_LOAD_IF1:%.*]], label [[PRED_LOAD_CONTINUE2:%.*]]
 ; CHECK-MAXBW:       pred.load.if1:
-; CHECK-MAXBW-NEXT:    [[TMP23:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[TMP1]]
+; CHECK-MAXBW-NEXT:    [[TMP23:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[TMP1]]
 ; CHECK-MAXBW-NEXT:    [[TMP24:%.*]] = load i8, ptr [[TMP23]], align 1
 ; CHECK-MAXBW-NEXT:    [[TMP25:%.*]] = insertelement <16 x i8> [[TMP21]], i8 [[TMP24]], i32 1
 ; CHECK-MAXBW-NEXT:    br label [[PRED_LOAD_CONTINUE2]]
@@ -2345,7 +2156,7 @@ define i32 @not_dotp_predicated_pragma(i64 %N, ptr %a, ptr %b) {
 ; CHECK-MAXBW-NEXT:    [[TMP27:%.*]] = extractelement <16 x i1> [[TMP16]], i32 2
 ; CHECK-MAXBW-NEXT:    br i1 [[TMP27]], label [[PRED_LOAD_IF3:%.*]], label [[PRED_LOAD_CONTINUE4:%.*]]
 ; CHECK-MAXBW:       pred.load.if3:
-; CHECK-MAXBW-NEXT:    [[TMP28:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[TMP2]]
+; CHECK-MAXBW-NEXT:    [[TMP28:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[TMP2]]
 ; CHECK-MAXBW-NEXT:    [[TMP29:%.*]] = load i8, ptr [[TMP28]], align 1
 ; CHECK-MAXBW-NEXT:    [[TMP30:%.*]] = insertelement <16 x i8> [[TMP26]], i8 [[TMP29]], i32 2
 ; CHECK-MAXBW-NEXT:    br label [[PRED_LOAD_CONTINUE4]]
@@ -2354,7 +2165,7 @@ define i32 @not_dotp_predicated_pragma(i64 %N, ptr %a, ptr %b) {
 ; CHECK-MAXBW-NEXT:    [[TMP32:%.*]] = extractelement <16 x i1> [[TMP16]], i32 3
 ; CHECK-MAXBW-NEXT:    br i1 [[TMP32]], label [[PRED_LOAD_IF5:%.*]], label [[PRED_LOAD_CONTINUE6:%.*]]
 ; CHECK-MAXBW:       pred.load.if5:
-; CHECK-MAXBW-NEXT:    [[TMP33:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[TMP3]]
+; CHECK-MAXBW-NEXT:    [[TMP33:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[TMP3]]
 ; CHECK-MAXBW-NEXT:    [[TMP34:%.*]] = load i8, ptr [[TMP33]], align 1
 ; CHECK-MAXBW-NEXT:    [[TMP35:%.*]] = insertelement <16 x i8> [[TMP31]], i8 [[TMP34]], i32 3
 ; CHECK-MAXBW-NEXT:    br label [[PRED_LOAD_CONTINUE6]]
@@ -2363,7 +2174,7 @@ define i32 @not_dotp_predicated_pragma(i64 %N, ptr %a, ptr %b) {
 ; CHECK-MAXBW-NEXT:    [[TMP37:%.*]] = extractelement <16 x i1> [[TMP16]], i32 4
 ; CHECK-MAXBW-NEXT:    br i1 [[TMP37]], label [[PRED_LOAD_IF7:%.*]], label [[PRED_LOAD_CONTINUE8:%.*]]
 ; CHECK-MAXBW:       pred.load.if7:
-; CHECK-MAXBW-NEXT:    [[TMP38:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[TMP4]]
+; CHECK-MAXBW-NEXT:    [[TMP38:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[TMP4]]
 ; CHECK-MAXBW-NEXT:    [[TMP39:%.*]] = load i8, ptr [[TMP38]], align 1
 ; CHECK-MAXBW-NEXT:    [[TMP40:%.*]] = insertelement <16 x i8> [[TMP36]], i8 [[TMP39]], i32 4
 ; CHECK-MAXBW-NEXT:    br label [[PRED_LOAD_CONTINUE8]]
@@ -2372,7 +2183,7 @@ define i32 @not_dotp_predicated_pragma(i64 %N, ptr %a, ptr %b) {
 ; CHECK-MAXBW-NEXT:    [[TMP42:%.*]] = extractelement <16 x i1> [[TMP16]], i32 5
 ; CHECK-MAXBW-NEXT:    br i1 [[TMP42]], label [[PRED_LOAD_IF9:%.*]], label [[PRED_LOAD_CONTINUE10:%.*]]
 ; CHECK-MAXBW:       pred.load.if9:
-; CHECK-MAXBW-NEXT:    [[TMP43:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[TMP5]]
+; CHECK-MAXBW-NEXT:    [[TMP43:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[TMP5]]
 ; CHECK-MAXBW-NEXT:    [[TMP44:%.*]] = load i8, ptr [[TMP43]], align 1
 ; CHECK-MAXBW-NEXT:    [[TMP45:%.*]] = insertelement <16 x i8> [[TMP41]], i8 [[TMP44]], i32 5
 ; CHECK-MAXBW-NEXT:    br label [[PRED_LOAD_CONTINUE10]]
@@ -2381,7 +2192,7 @@ define i32 @not_dotp_predicated_pragma(i64 %N, ptr %a, ptr %b) {
 ; CHECK-MAXBW-NEXT:    [[TMP47:%.*]] = extractelement <16 x i1> [[TMP16]], i32 6
 ; CHECK-MAXBW-NEXT:    br i1 [[TMP47]], label [[PRED_LOAD_IF11:%.*]], label [[PRED_LOAD_CONTINUE12:%.*]]
 ; CHECK-MAXBW:       pred.load.if11:
-; CHECK-MAXBW-NEXT:    [[TMP48:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[TMP6]]
+; CHECK-MAXBW-NEXT:    [[TMP48:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[TMP6]]
 ; CHECK-MAXBW-NEXT:    [[TMP49:%.*]] = load i8, ptr [[TMP48]], align 1
 ; CHECK-MAXBW-NEXT:    [[TMP50:%.*]] = insertelement <16 x i8> [[TMP46]], i8 [[TMP49]], i32 6
 ; CHECK-MAXBW-NEXT:    br label [[PRED_LOAD_CONTINUE12]]
@@ -2390,7 +2201,7 @@ define i32 @not_dotp_predicated_pragma(i64 %N, ptr %a, ptr %b) {
 ; CHECK-MAXBW-NEXT:    [[TMP52:%.*]] = extractelement <16 x i1> [[TMP16]], i32 7
 ; CHECK-MAXBW-NEXT:    br i1 [[TMP52]], label [[PRED_LOAD_IF13:%.*]], label [[PRED_LOAD_CONTINUE14:%.*]]
 ; CHECK-MAXBW:       pred.load.if13:
-; CHECK-MAXBW-NEXT:    [[TMP53:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[TMP7]]
+; CHECK-MAXBW-NEXT:    [[TMP53:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[TMP7]]
 ; CHECK-MAXBW-NEXT:    [[TMP54:%.*]] = load i8, ptr [[TMP53]], align 1
 ; CHECK-MAXBW-NEXT:    [[TMP55:%.*]] = insertelement <16 x i8> [[TMP51]], i8 [[TMP54]], i32 7
 ; CHECK-MAXBW-NEXT:    br label [[PRED_LOAD_CONTINUE14]]
@@ -2399,7 +2210,7 @@ define i32 @not_dotp_predicated_pragma(i64 %N, ptr %a, ptr %b) {
 ; CHECK-MAXBW-NEXT:    [[TMP57:%.*]] = extractelement <16 x i1> [[TMP16]], i32 8
 ; CHECK-MAXBW-NEXT:    br i1 [[TMP57]], label [[PRED_LOAD_IF15:%.*]], label [[PRED_LOAD_CONTINUE16:%.*]]
 ; CHECK-MAXBW:       pred.load.if15:
-; CHECK-MAXBW-NEXT:    [[TMP58:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[TMP8]]
+; CHECK-MAXBW-NEXT:    [[TMP58:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[TMP8]]
 ; CHECK-MAXBW-NEXT:    [[TMP59:%.*]] = load i8, ptr [[TMP58]], align 1
 ; CHECK-MAXBW-NEXT:    [[TMP60:%.*]] = insertelement <16 x i8> [[TMP56]], i8 [[TMP59]], i32 8
 ; CHECK-MAXBW-NEXT:    br label [[PRED_LOAD_CONTINUE16]]
@@ -2408,7 +2219,7 @@ define i32 @not_dotp_predicated_pragma(i64 %N, ptr %a, ptr %b) {
 ; CHECK-MAXBW-NEXT:    [[TMP62:%.*]] = extractelement <16 x i1> [[TMP16]], i32 9
 ; CHECK-MAXBW-NEXT:    br i1 [[TMP62]], label [[PRED_LOAD_IF17:%.*]], label [[PRED_LOAD_CONTINUE18:%.*]]
 ; CHECK-MAXBW:       pred.load.if17:
-; CHECK-MAXBW-NEXT:    [[TMP63:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[TMP9]]
+; CHECK-MAXBW-NEXT:    [[TMP63:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[TMP9]]
 ; CHECK-MAXBW-NEXT:    [[TMP64:%.*]] = load i8, ptr [[TMP63]], align 1
 ; CHECK-MAXBW-NEXT:    [[TMP65:%.*]] = insertelement <16 x i8> [[TMP61]], i8 [[TMP64]], i32 9
 ; CHECK-MAXBW-NEXT:    br label [[PRED_LOAD_CONTINUE18]]
@@ -2417,7 +2228,7 @@ define i32 @not_dotp_predicated_pragma(i64 %N, ptr %a, ptr %b) {
 ; CHECK-MAXBW-NEXT:    [[TMP67:%.*]] = extractelement <16 x i1> [[TMP16]], i32 10
 ; CHECK-MAXBW-NEXT:    br i1 [[TMP67]], label [[PRED_LOAD_IF19:%.*]], label [[PRED_LOAD_CONTINUE20:%.*]]
 ; CHECK-MAXBW:       pred.load.if19:
-; CHECK-MAXBW-NEXT:    [[TMP68:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[TMP10]]
+; CHECK-MAXBW-NEXT:    [[TMP68:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[TMP10]]
 ; CHECK-MAXBW-NEXT:    [[TMP69:%.*]] = load i8, ptr [[TMP68]], align 1
 ; CHECK-MAXBW-NEXT:    [[TMP70:%.*]] = insertelement <16 x i8> [[TMP66]], i8 [[TMP69]], i32 10
 ; CHECK-MAXBW-NEXT:    br label [[PRED_LOAD_CONTINUE20]]
@@ -2426,7 +2237,7 @@ define i32 @not_dotp_predicated_pragma(i64 %N, ptr %a, ptr %b) {
 ; CHECK-MAXBW-NEXT:    [[TMP72:%.*]] = extractelement <16 x i1> [[TMP16]], i32 11
 ; CHECK-MAXBW-NEXT:    br i1 [[TMP72]], label [[PRED_LOAD_IF21:%.*]], label [[PRED_LOAD_CONTINUE22:%.*]]
 ; CHECK-MAXBW:       pred.load.if21:
-; CHECK-MAXBW-NEXT:    [[TMP73:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[TMP11]]
+; CHECK-MAXBW-NEXT:    [[TMP73:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[TMP11]]
 ; CHECK-MAXBW-NEXT:    [[TMP74:%.*]] = load i8, ptr [[TMP73]], align 1
 ; CHECK-MAXBW-NEXT:    [[TMP75:%.*]] = insertelement <16 x i8> [[TMP71]], i8 [[TMP74]], i32 11
 ; CHECK-MAXBW-NEXT:    br label [[PRED_LOAD_CONTINUE22]]
@@ -2435,7 +2246,7 @@ define i32 @not_dotp_predicated_pragma(i64 %N, ptr %a, ptr %b) {
 ; CHECK-MAXBW-NEXT:    [[TMP77:%.*]] = extractelement <16 x i1> [[TMP16]], i32 12
 ; CHECK-MAXBW-NEXT:    br i1 [[TMP77]], label [[PRED_LOAD_IF23:%.*]], label [[PRED_LOAD_CONTINUE24:%.*]]
 ; CHECK-MAXBW:       pred.load.if23:
-; CHECK-MAXBW-NEXT:    [[TMP78:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[TMP12]]
+; CHECK-MAXBW-NEXT:    [[TMP78:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[TMP12]]
 ; CHECK-MAXBW-NEXT:    [[TMP79:%.*]] = load i8, ptr [[TMP78]], align 1
 ; CHECK-MAXBW-NEXT:    [[TMP80:%.*]] = insertelement <16 x i8> [[TMP76]], i8 [[TMP79]], i32 12
 ; CHECK-MAXBW-NEXT:    br label [[PRED_LOAD_CONTINUE24]]
@@ -2444,7 +2255,7 @@ define i32 @not_dotp_predicated_pragma(i64 %N, ptr %a, ptr %b) {
 ; CHECK-MAXBW-NEXT:    [[TMP82:%.*]] = extractelement <16 x i1> [[TMP16]], i32 13
 ; CHECK-MAXBW-NEXT:    br i1 [[TMP82]], label [[PRED_LOAD_IF25:%.*]], label [[PRED_LOAD_CONTINUE26:%.*]]
 ; CHECK-MAXBW:       pred.load.if25:
-; CHECK-MAXBW-NEXT:    [[TMP83:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[TMP13]]
+; CHECK-MAXBW-NEXT:    [[TMP83:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[TMP13]]
 ; CHECK-MAXBW-NEXT:    [[TMP84:%.*]] = load i8, ptr [[TMP83]], align 1
 ; CHECK-MAXBW-NEXT:    [[TMP85:%.*]] = insertelement <16 x i8> [[TMP81]], i8 [[TMP84]], i32 13
 ; CHECK-MAXBW-NEXT:    br label [[PRED_LOAD_CONTINUE26]]
@@ -2453,7 +2264,7 @@ define i32 @not_dotp_predicated_pragma(i64 %N, ptr %a, ptr %b) {
 ; CHECK-MAXBW-NEXT:    [[TMP87:%.*]] = extractelement <16 x i1> [[TMP16]], i32 14
 ; CHECK-MAXBW-NEXT:    br i1 [[TMP87]], label [[PRED_LOAD_IF27:%.*]], label [[PRED_LOAD_CONTINUE28:%.*]]
 ; CHECK-MAXBW:       pred.load.if27:
-; CHECK-MAXBW-NEXT:    [[TMP88:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[TMP14]]
+; CHECK-MAXBW-NEXT:    [[TMP88:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[TMP14]]
 ; CHECK-MAXBW-NEXT:    [[TMP89:%.*]] = load i8, ptr [[TMP88]], align 1
 ; CHECK-MAXBW-NEXT:    [[TMP90:%.*]] = insertelement <16 x i8> [[TMP86]], i8 [[TMP89]], i32 14
 ; CHECK-MAXBW-NEXT:    br label [[PRED_LOAD_CONTINUE28]]
@@ -2462,7 +2273,7 @@ define i32 @not_dotp_predicated_pragma(i64 %N, ptr %a, ptr %b) {
 ; CHECK-MAXBW-NEXT:    [[TMP92:%.*]] = extractelement <16 x i1> [[TMP16]], i32 15
 ; CHECK-MAXBW-NEXT:    br i1 [[TMP92]], label [[PRED_LOAD_IF29:%.*]], label [[PRED_LOAD_CONTINUE30:%.*]]
 ; CHECK-MAXBW:       pred.load.if29:
-; CHECK-MAXBW-NEXT:    [[TMP93:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[TMP15]]
+; CHECK-MAXBW-NEXT:    [[TMP93:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[TMP15]]
 ; CHECK-MAXBW-NEXT:    [[TMP94:%.*]] = load i8, ptr [[TMP93]], align 1
 ; CHECK-MAXBW-NEXT:    [[TMP95:%.*]] = insertelement <16 x i8> [[TMP91]], i8 [[TMP94]], i32 15
 ; CHECK-MAXBW-NEXT:    br label [[PRED_LOAD_CONTINUE30]]
@@ -2472,7 +2283,7 @@ define i32 @not_dotp_predicated_pragma(i64 %N, ptr %a, ptr %b) {
 ; CHECK-MAXBW-NEXT:    [[TMP98:%.*]] = extractelement <16 x i1> [[TMP16]], i32 0
 ; CHECK-MAXBW-NEXT:    br i1 [[TMP98]], label [[PRED_LOAD_IF31:%.*]], label [[PRED_LOAD_CONTINUE32:%.*]]
 ; CHECK-MAXBW:       pred.load.if31:
-; CHECK-MAXBW-NEXT:    [[TMP99:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[TMP0]]
+; CHECK-MAXBW-NEXT:    [[TMP99:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[TMP0]]
 ; CHECK-MAXBW-NEXT:    [[TMP100:%.*]] = load i8, ptr [[TMP99]], align 1
 ; CHECK-MAXBW-NEXT:    [[TMP101:%.*]] = insertelement <16 x i8> poison, i8 [[TMP100]], i32 0
 ; CHECK-MAXBW-NEXT:    br label [[PRED_LOAD_CONTINUE32]]
@@ -2481,7 +2292,7 @@ define i32 @not_dotp_predicated_pragma(i64 %N, ptr %a, ptr %b) {
 ; CHECK-MAXBW-NEXT:    [[TMP103:%.*]] = extractelement <16 x i1> [[TMP16]], i32 1
 ; CHECK-MAXBW-NEXT:    br i1 [[TMP103]], label [[PRED_LOAD_IF33:%.*]], label [[PRED_LOAD_CONTINUE34:%.*]]
 ; CHECK-MAXBW:       pred.load.if33:
-; CHECK-MAXBW-NEXT:    [[TMP104:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[TMP1]]
+; CHECK-MAXBW-NEXT:    [[TMP104:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[TMP1]]
 ; CHECK-MAXBW-NEXT:    [[TMP105:%.*]] = load i8, ptr [[TMP104]], align 1
 ; CHECK-MAXBW-NEXT:    [[TMP106:%.*]] = insertelement <16 x i8> [[TMP102]], i8 [[TMP105]], i32 1
 ; CHECK-MAXBW-NEXT:    br label [[PRED_LOAD_CONTINUE34]]
@@ -2490,7 +2301,7 @@ define i32 @not_dotp_predicated_pragma(i64 %N, ptr %a, ptr %b) {
 ; CHECK-MAXBW-NEXT:    [[TMP108:%.*]] = extractelement <16 x i1> [[TMP16]], i32 2
 ; CHECK-MAXBW-NEXT:    br i1 [[TMP108]], label [[PRED_LOAD_IF35:%.*]], label [[PRED_LOAD_CONTINUE36:%.*]]
 ; CHECK-MAXBW:       pred.load.if35:
-; CHECK-MAXBW-NEXT:    [[TMP109:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[TMP2]]
+; CHECK-MAXBW-NEXT:    [[TMP109:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[TMP2]]
 ; CHECK-MAXBW-NEXT:    [[TMP110:%.*]] = load i8, ptr [[TMP109]], align 1
 ; CHECK-MAXBW-NEXT:    [[TMP111:%.*]] = insertelement <16 x i8> [[TMP107]], i8 [[TMP110]], i32 2
 ; CHECK-MAXBW-NEXT:    br label [[PRED_LOAD_CONTINUE36]]
@@ -2499,7 +2310,7 @@ define i32 @not_dotp_predicated_pragma(i64 %N, ptr %a, ptr %b) {
 ; CHECK-MAXBW-NEXT:    [[TMP113:%.*]] = extractelement <16 x i1> [[TMP16]], i32 3
 ; CHECK-MAXBW-NEXT:    br i1 [[TMP113]], label [[PRED_LOAD_IF37:%.*]], label [[PRED_LOAD_CONTINUE38:%.*]]
 ; CHECK-MAXBW:       pred.load.if37:
-; CHECK-MAXBW-NEXT:    [[TMP114:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[TMP3]]
+; CHECK-MAXBW-NEXT:    [[TMP114:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[TMP3]]
 ; CHECK-MAXBW-NEXT:    [[TMP115:%.*]] = load i8, ptr [[TMP114]], align 1
 ; CHECK-MAXBW-NEXT:    [[TMP116:%.*]] = insertelement <16 x i8> [[TMP112]], i8 [[TMP115]], i32 3
 ; CHECK-MAXBW-NEXT:    br label [[PRED_LOAD_CONTINUE38]]
@@ -2508,7 +2319,7 @@ define i32 @not_dotp_predicated_pragma(i64 %N, ptr %a, ptr %b) {
 ; CHECK-MAXBW-NEXT:    [[TMP118:%.*]] = extractelement <16 x i1> [[TMP16]], i32 4
 ; CHECK-MAXBW-NEXT:    br i1 [[TMP118]], label [[PRED_LOAD_IF39:%.*]], label [[PRED_LOAD_CONTINUE40:%.*]]
 ; CHECK-MAXBW:       pred.load.if39:
-; CHECK-MAXBW-NEXT:    [[TMP119:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[TMP4]]
+; CHECK-MAXBW-NEXT:    [[TMP119:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[TMP4]]
 ; CHECK-MAXBW-NEXT:    [[TMP120:%.*]] = load i8, ptr [[TMP119]], align 1
 ; CHECK-MAXBW-NEXT:    [[TMP121:%.*]] = insertelement <16 x i8> [[TMP117]], i8 [[TMP120]], i32 4
 ; CHECK-MAXBW-NEXT:    br label [[PRED_LOAD_CONTINUE40]]
@@ -2517,7 +2328,7 @@ define i32 @not_dotp_predicated_pragma(i64 %N, ptr %a, ptr %b) {
 ; CHECK-MAXBW-NEXT:    [[TMP123:%.*]] = extractelement <16 x i1> [[TMP16]], i32 5
 ; CHECK-MAXBW-NEXT:    br i1 [[TMP123]], label [[PRED_LOAD_IF41:%.*]], label [[PRED_LOAD_CONTINUE42:%.*]]
 ; CHECK-MAXBW:       pred.load.if41:
-; CHECK-MAXBW-NEXT:    [[TMP124:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[TMP5]]
+; CHECK-MAXBW-NEXT:    [[TMP124:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[TMP5]]
 ; CHECK-MAXBW-NEXT:    [[TMP125:%.*]] = load i8, ptr [[TMP124]], align 1
 ; CHECK-MAXBW-NEXT:    [[TMP126:%.*]] = insertelement <16 x i8> [[TMP122]], i8 [[TMP125]], i32 5
 ; CHECK-MAXBW-NEXT:    br label [[PRED_LOAD_CONTINUE42]]
@@ -2526,7 +2337,7 @@ define i32 @not_dotp_predicated_pragma(i64 %N, ptr %a, ptr %b) {
 ; CHECK-MAXBW-NEXT:    [[TMP128:%.*]] = extractelement <16 x i1> [[TMP16]], i32 6
 ; CHECK-MAXBW-NEXT:    br i1 [[TMP128]], label [[PRED_LOAD_IF43:%.*]], label [[PRED_LOAD_CONTINUE44:%.*]]
 ; CHECK-MAXBW:       pred.load.if43:
-; CHECK-MAXBW-NEXT:    [[TMP129:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[TMP6]]
+; CHECK-MAXBW-NEXT:    [[TMP129:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[TMP6]]
 ; CHECK-MAXBW-NEXT:    [[TMP130:%.*]] = load i8, ptr [[TMP129]], align 1
 ; CHECK-MAXBW-NEXT:    [[TMP131:%.*]] = insertelement <16 x i8> [[TMP127]], i8 [[TMP130]], i32 6
 ; CHECK-MAXBW-NEXT:    br label [[PRED_LOAD_CONTINUE44]]
@@ -2535,7 +2346,7 @@ define i32 @not_dotp_predicated_pragma(i64 %N, ptr %a, ptr %b) {
 ; CHECK-MAXBW-NEXT:    [[TMP133:%.*]] = extractelement <16 x i1> [[TMP16]], i32 7
 ; CHECK-MAXBW-NEXT:    br i1 [[TMP133]], label [[PRED_LOAD_IF45:%.*]], label [[PRED_LOAD_CONTINUE46:%.*]]
 ; CHECK-MAXBW:       pred.load.if45:
-; CHECK-MAXBW-NEXT:    [[TMP134:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[TMP7]]
+; CHECK-MAXBW-NEXT:    [[TMP134:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[TMP7]]
 ; CHECK-MAXBW-NEXT:    [[TMP135:%.*]] = load i8, ptr [[TMP134]], align 1
 ; CHECK-MAXBW-NEXT:    [[TMP136:%.*]] = insertelement <16 x i8> [[TMP132]], i8 [[TMP135]], i32 7
 ; CHECK-MAXBW-NEXT:    br label [[PRED_LOAD_CONTINUE46]]
@@ -2544,7 +2355,7 @@ define i32 @not_dotp_predicated_pragma(i64 %N, ptr %a, ptr %b) {
 ; CHECK-MAXBW-NEXT:    [[TMP138:%.*]] = extractelement <16 x i1> [[TMP16]], i32 8
 ; CHECK-MAXBW-NEXT:    br i1 [[TMP138]], label [[PRED_LOAD_IF47:%.*]], label [[PRED_LOAD_CONTINUE48:%.*]]
 ; CHECK-MAXBW:       pred.load.if47:
-; CHECK-MAXBW-NEXT:    [[TMP139:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[TMP8]]
+; CHECK-MAXBW-NEXT:    [[TMP139:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[TMP8]]
 ; CHECK-MAXBW-NEXT:    [[TMP140:%.*]] = load i8, ptr [[TMP139]], align 1
 ; CHECK-MAXBW-NEXT:    [[TMP141:%.*]] = insertelement <16 x i8> [[TMP137]], i8 [[TMP140]], i32 8
 ; CHECK-MAXBW-NEXT:    br label [[PRED_LOAD_CONTINUE48]]
@@ -2553,7 +2364,7 @@ define i32 @not_dotp_predicated_pragma(i64 %N, ptr %a, ptr %b) {
 ; CHECK-MAXBW-NEXT:    [[TMP143:%.*]] = extractelement <16 x i1> [[TMP16]], i32 9
 ; CHECK-MAXBW-NEXT:    br i1 [[TMP143]], label [[PRED_LOAD_IF49:%.*]], label [[PRED_LOAD_CONTINUE50:%.*]]
 ; CHECK-MAXBW:       pred.load.if49:
-; CHECK-MAXBW-NEXT:    [[TMP144:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[TMP9]]
+; CHECK-MAXBW-NEXT:    [[TMP144:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[TMP9]]
 ; CHECK-MAXBW-NEXT:    [[TMP145:%.*]] = load i8, ptr [[TMP144]], align 1
 ; CHECK-MAXBW-NEXT:    [[TMP146:%.*]] = insertelement <16 x i8> [[TMP142]], i8 [[TMP145]], i32 9
 ; CHECK-MAXBW-NEXT:    br label [[PRED_LOAD_CONTINUE50]]
@@ -2562,7 +2373,7 @@ define i32 @not_dotp_predicated_pragma(i64 %N, ptr %a, ptr %b) {
 ; CHECK-MAXBW-NEXT:    [[TMP148:%.*]] = extractelement <16 x i1> [[TMP16]], i32 10
 ; CHECK-MAXBW-NEXT:    br i1 [[TMP148]], label [[PRED_LOAD_IF51:%.*]], label [[PRED_LOAD_CONTINUE52:%.*]]
 ; CHECK-MAXBW:       pred.load.if51:
-; CHECK-MAXBW-NEXT:    [[TMP149:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[TMP10]]
+; CHECK-MAXBW-NEXT:    [[TMP149:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[TMP10]]
 ; CHECK-MAXBW-NEXT:    [[TMP150:%.*]] = load i8, ptr [[TMP149]], align 1
 ; CHECK-MAXBW-NEXT:    [[TMP151:%.*]] = insertelement <16 x i8> [[TMP147]], i8 [[TMP150]], i32 10
 ; CHECK-MAXBW-NEXT:    br label [[PRED_LOAD_CONTINUE52]]
@@ -2571,7 +2382,7 @@ define i32 @not_dotp_predicated_pragma(i64 %N, ptr %a, ptr %b) {
 ; CHECK-MAXBW-NEXT:    [[TMP153:%.*]] = extractelement <16 x i1> [[TMP16]], i32 11
 ; CHECK-MAXBW-NEXT:    br i1 [[TMP153]], label [[PRED_LOAD_IF53:%.*]], label [[PRED_LOAD_CONTINUE54:%.*]]
 ; CHECK-MAXBW:       pred.load.if53:
-; CHECK-MAXBW-NEXT:    [[TMP154:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[TMP11]]
+; CHECK-MAXBW-NEXT:    [[TMP154:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[TMP11]]
 ; CHECK-MAXBW-NEXT:    [[TMP155:%.*]] = load i8, ptr [[TMP154]], align 1
 ; CHECK-MAXBW-NEXT:    [[TMP156:%.*]] = insertelement <16 x i8> [[TMP152]], i8 [[TMP155]], i32 11
 ; CHECK-MAXBW-NEXT:    br label [[PRED_LOAD_CONTINUE54]]
@@ -2580,7 +2391,7 @@ define i32 @not_dotp_predicated_pragma(i64 %N, ptr %a, ptr %b) {
 ; CHECK-MAXBW-NEXT:    [[TMP158:%.*]] = extractelement <16 x i1> [[TMP16]], i32 12
 ; CHECK-MAXBW-NEXT:    br i1 [[TMP158]], label [[PRED_LOAD_IF55:%.*]], label [[PRED_LOAD_CONTINUE56:%.*]]
 ; CHECK-MAXBW:       pred.load.if55:
-; CHECK-MAXBW-NEXT:    [[TMP159:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[TMP12]]
+; CHECK-MAXBW-NEXT:    [[TMP159:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[TMP12]]
 ; CHECK-MAXBW-NEXT:    [[TMP160:%.*]] = load i8, ptr [[TMP159]], align 1
 ; CHECK-MAXBW-NEXT:    [[TMP161:%.*]] = insertelement <16 x i8> [[TMP157]], i8 [[TMP160]], i32 12
 ; CHECK-MAXBW-NEXT:    br label [[PRED_LOAD_CONTINUE56]]
@@ -2589,7 +2400,7 @@ define i32 @not_dotp_predicated_pragma(i64 %N, ptr %a, ptr %b) {
 ; CHECK-MAXBW-NEXT:    [[TMP163:%.*]] = extractelement <16 x i1> [[TMP16]], i32 13
 ; CHECK-MAXBW-NEXT:    br i1 [[TMP163]], label [[PRED_LOAD_IF57:%.*]], label [[PRED_LOAD_CONTINUE58:%.*]]
 ; CHECK-MAXBW:       pred.load.if57:
-; CHECK-MAXBW-NEXT:    [[TMP164:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[TMP13]]
+; CHECK-MAXBW-NEXT:    [[TMP164:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[TMP13]]
 ; CHECK-MAXBW-NEXT:    [[TMP165:%.*]] = load i8, ptr [[TMP164]], align 1
 ; CHECK-MAXBW-NEXT:    [[TMP166:%.*]] = insertelement <16 x i8> [[TMP162]], i8 [[TMP165]], i32 13
 ; CHECK-MAXBW-NEXT:    br label [[PRED_LOAD_CONTINUE58]]
@@ -2598,7 +2409,7 @@ define i32 @not_dotp_predicated_pragma(i64 %N, ptr %a, ptr %b) {
 ; CHECK-MAXBW-NEXT:    [[TMP168:%.*]] = extractelement <16 x i1> [[TMP16]], i32 14
 ; CHECK-MAXBW-NEXT:    br i1 [[TMP168]], label [[PRED_LOAD_IF59:%.*]], label [[PRED_LOAD_CONTINUE60:%.*]]
 ; CHECK-MAXBW:       pred.load.if59:
-; CHECK-MAXBW-NEXT:    [[TMP169:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[TMP14]]
+; CHECK-MAXBW-NEXT:    [[TMP169:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[TMP14]]
 ; CHECK-MAXBW-NEXT:    [[TMP170:%.*]] = load i8, ptr [[TMP169]], align 1
 ; CHECK-MAXBW-NEXT:    [[TMP171:%.*]] = insertelement <16 x i8> [[TMP167]], i8 [[TMP170]], i32 14
 ; CHECK-MAXBW-NEXT:    br label [[PRED_LOAD_CONTINUE60]]
@@ -2607,7 +2418,7 @@ define i32 @not_dotp_predicated_pragma(i64 %N, ptr %a, ptr %b) {
 ; CHECK-MAXBW-NEXT:    [[TMP173:%.*]] = extractelement <16 x i1> [[TMP16]], i32 15
 ; CHECK-MAXBW-NEXT:    br i1 [[TMP173]], label [[PRED_LOAD_IF61:%.*]], label [[PRED_LOAD_CONTINUE62]]
 ; CHECK-MAXBW:       pred.load.if61:
-; CHECK-MAXBW-NEXT:    [[TMP174:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[TMP15]]
+; CHECK-MAXBW-NEXT:    [[TMP174:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[TMP15]]
 ; CHECK-MAXBW-NEXT:    [[TMP175:%.*]] = load i8, ptr [[TMP174]], align 1
 ; CHECK-MAXBW-NEXT:    [[TMP176:%.*]] = insertelement <16 x i8> [[TMP172]], i8 [[TMP175]], i32 15
 ; CHECK-MAXBW-NEXT:    br label [[PRED_LOAD_CONTINUE62]]
@@ -2615,35 +2426,35 @@ define i32 @not_dotp_predicated_pragma(i64 %N, ptr %a, ptr %b) {
 ; CHECK-MAXBW-NEXT:    [[TMP177:%.*]] = phi <16 x i8> [ [[TMP172]], [[PRED_LOAD_CONTINUE60]] ], [ [[TMP176]], [[PRED_LOAD_IF61]] ]
 ; CHECK-MAXBW-NEXT:    [[TMP178:%.*]] = sext <16 x i8> [[TMP177]] to <16 x i32>
 ; CHECK-MAXBW-NEXT:    [[TMP179:%.*]] = mul nsw <16 x i32> [[TMP178]], [[TMP97]]
-; CHECK-MAXBW-NEXT:    [[TMP180]] = add <16 x i32> [[TMP179]], [[VEC_PHI]]
-; CHECK-MAXBW-NEXT:    [[TMP181:%.*]] = select <16 x i1> [[TMP16]], <16 x i32> [[TMP180]], <16 x i32> [[VEC_PHI]]
+; CHECK-MAXBW-NEXT:    [[TMP180:%.*]] = select <16 x i1> [[TMP16]], <16 x i32> [[TMP179]], <16 x i32> zeroinitializer
+; CHECK-MAXBW-NEXT:    [[PARTIAL_REDUCE]] = call <4 x i32> @llvm.experimental.vector.partial.reduce.add.v4i32.v16i32(<4 x i32> [[VEC_PHI]], <16 x i32> [[TMP180]])
 ; CHECK-MAXBW-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 16
 ; CHECK-MAXBW-NEXT:    [[VEC_IND_NEXT]] = add <16 x i64> [[VEC_IND]], splat (i64 16)
-; CHECK-MAXBW-NEXT:    [[TMP182:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-MAXBW-NEXT:    br i1 [[TMP182]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
+; CHECK-MAXBW-NEXT:    [[TMP181:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-MAXBW-NEXT:    br i1 [[TMP181]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
 ; CHECK-MAXBW:       middle.block:
-; CHECK-MAXBW-NEXT:    [[TMP183:%.*]] = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> [[TMP181]])
+; CHECK-MAXBW-NEXT:    [[TMP182:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[PARTIAL_REDUCE]])
 ; CHECK-MAXBW-NEXT:    br i1 true, label [[EXIT:%.*]], label [[SCALAR_PH]]
 ; CHECK-MAXBW:       scalar.ph:
 ; CHECK-MAXBW-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
-; CHECK-MAXBW-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP183]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ]
+; CHECK-MAXBW-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP182]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ]
 ; CHECK-MAXBW-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK-MAXBW:       for.body:
 ; CHECK-MAXBW-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
 ; CHECK-MAXBW-NEXT:    [[ACCUM:%.*]] = phi i32 [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[ADD:%.*]], [[FOR_BODY]] ]
-; CHECK-MAXBW-NEXT:    [[GEP_A:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[IV]]
+; CHECK-MAXBW-NEXT:    [[GEP_A:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[IV]]
 ; CHECK-MAXBW-NEXT:    [[LOAD_A:%.*]] = load i8, ptr [[GEP_A]], align 1
 ; CHECK-MAXBW-NEXT:    [[EXT_A:%.*]] = sext i8 [[LOAD_A]] to i32
-; CHECK-MAXBW-NEXT:    [[GEP_A2:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[IV]]
-; CHECK-MAXBW-NEXT:    [[LOAD_B:%.*]] = load i8, ptr [[GEP_A2]], align 1
+; CHECK-MAXBW-NEXT:    [[GEP_B:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[IV]]
+; CHECK-MAXBW-NEXT:    [[LOAD_B:%.*]] = load i8, ptr [[GEP_B]], align 1
 ; CHECK-MAXBW-NEXT:    [[EXT_B:%.*]] = sext i8 [[LOAD_B]] to i32
 ; CHECK-MAXBW-NEXT:    [[MUL:%.*]] = mul nsw i32 [[EXT_B]], [[EXT_A]]
 ; CHECK-MAXBW-NEXT:    [[ADD]] = add nsw i32 [[MUL]], [[ACCUM]]
 ; CHECK-MAXBW-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
 ; CHECK-MAXBW-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; CHECK-MAXBW-NEXT:    br i1 [[EXITCOND_NOT]], label [[EXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]]
+; CHECK-MAXBW-NEXT:    br i1 [[EXITCOND_NOT]], label [[EXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
 ; CHECK-MAXBW:       exit:
-; CHECK-MAXBW-NEXT:    [[ADD_LCSSA:%.*]] = phi i32 [ [[ADD]], [[FOR_BODY]] ], [ [[TMP183]], [[MIDDLE_BLOCK]] ]
+; CHECK-MAXBW-NEXT:    [[ADD_LCSSA:%.*]] = phi i32 [ [[ADD]], [[FOR_BODY]] ], [ [[TMP182]], [[MIDDLE_BLOCK]] ]
 ; CHECK-MAXBW-NEXT:    ret i32 [[ADD_LCSSA]]
 ;
 entry:
@@ -2652,11 +2463,11 @@ entry:
 for.body:                                         ; preds = %entry, %for.body
   %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
   %accum = phi i32 [ 0, %entry ], [ %add, %for.body ]
-  %gep.a = getelementptr inbounds i8, ptr %b, i64 %iv
+  %gep.a = getelementptr inbounds i8, ptr %a, i64 %iv
   %load.a = load i8, ptr %gep.a, align 1
   %ext.a = sext i8 %load.a to i32
-  %gep.a2 = getelementptr inbounds i8, ptr %a, i64 %iv
-  %load.b = load i8, ptr %gep.a2, align 1
+  %gep.b = getelementptr inbounds i8, ptr %b, i64 %iv
+  %load.b = load i8, ptr %gep.b, align 1
   %ext.b = sext i8 %load.b to i32
   %mul = mul nsw i32 %ext.b, %ext.a
   %add = add nsw i32 %mul, %accum
@@ -2691,7 +2502,7 @@ define i32 @not_dotp_extend_user(ptr %a, ptr %b) {
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP8]] = add <16 x i32> [[TMP7]], [[VEC_PHI]]
 ; CHECK-INTERLEAVE1-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
-; CHECK-INTERLEAVE1-NEXT:    br i1 [[TMP9]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP17:![0-9]+]]
+; CHECK-INTERLEAVE1-NEXT:    br i1 [[TMP9]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP15:![0-9]+]]
 ; CHECK-INTERLEAVE1:       middle.block:
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP10:%.*]] = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> [[TMP8]])
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP11:%.*]] = extractelement <16 x i32> [[TMP6]], i32 15
@@ -2713,7 +2524,7 @@ define i32 @not_dotp_extend_user(ptr %a, ptr %b) {
 ; CHECK-INTERLEAVE1-NEXT:    [[ADD]] = add i32 [[MUL]], [[ACCUM]]
 ; CHECK-INTERLEAVE1-NEXT:    [[IV_NEXT]] = add i64 [[IV]], 1
 ; CHECK-INTERLEAVE1-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], 1024
-; CHECK-INTERLEAVE1-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_EXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP18:![0-9]+]]
+; CHECK-INTERLEAVE1-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_EXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]]
 ; CHECK-INTERLEAVE1:       for.exit:
 ; CHECK-INTERLEAVE1-NEXT:    [[EXT_B_LCSSA:%.*]] = phi i32 [ [[EXT_B]], [[FOR_BODY]] ], [ [[TMP11]], [[MIDDLE_BLOCK]] ]
 ; CHECK-INTERLEAVE1-NEXT:    [[ADD_LCSSA:%.*]] = phi i32 [ [[ADD]], [[FOR_BODY]] ], [ [[TMP10]], [[MIDDLE_BLOCK]] ]
@@ -2751,7 +2562,7 @@ define i32 @not_dotp_extend_user(ptr %a, ptr %b) {
 ; CHECK-INTERLEAVED-NEXT:    [[TMP14]] = add <16 x i32> [[TMP12]], [[VEC_PHI1]]
 ; CHECK-INTERLEAVED-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 32
 ; CHECK-INTERLEAVED-NEXT:    [[TMP15:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
-; CHECK-INTERLEAVED-NEXT:    br i1 [[TMP15]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP17:![0-9]+]]
+; CHECK-INTERLEAVED-NEXT:    br i1 [[TMP15]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP15:![0-9]+]]
 ; CHECK-INTERLEAVED:       middle.block:
 ; CHECK-INTERLEAVED-NEXT:    [[BIN_RDX:%.*]] = add <16 x i32> [[TMP14]], [[TMP13]]
 ; CHECK-INTERLEAVED-NEXT:    [[TMP16:%.*]] = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> [[BIN_RDX]])
@@ -2774,7 +2585,7 @@ define i32 @not_dotp_extend_user(ptr %a, ptr %b) {
 ; CHECK-INTERLEAVED-NEXT:    [[ADD]] = add i32 [[MUL]], [[ACCUM]]
 ; CHECK-INTERLEAVED-NEXT:    [[IV_NEXT]] = add i64 [[IV]], 1
 ; CHECK-INTERLEAVED-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], 1024
-; CHECK-INTERLEAVED-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_EXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP18:![0-9]+]]
+; CHECK-INTERLEAVED-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_EXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]]
 ; CHECK-INTERLEAVED:       for.exit:
 ; CHECK-INTERLEAVED-NEXT:    [[EXT_B_LCSSA:%.*]] = phi i32 [ [[EXT_B]], [[FOR_BODY]] ], [ [[TMP17]], [[MIDDLE_BLOCK]] ]
 ; CHECK-INTERLEAVED-NEXT:    [[ADD_LCSSA:%.*]] = phi i32 [ [[ADD]], [[FOR_BODY]] ], [ [[TMP16]], [[MIDDLE_BLOCK]] ]
@@ -2803,7 +2614,7 @@ define i32 @not_dotp_extend_user(ptr %a, ptr %b) {
 ; CHECK-MAXBW-NEXT:    [[TMP8]] = add <16 x i32> [[TMP7]], [[VEC_PHI]]
 ; CHECK-MAXBW-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
 ; CHECK-MAXBW-NEXT:    [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
-; CHECK-MAXBW-NEXT:    br i1 [[TMP9]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP17:![0-9]+]]
+; CHECK-MAXBW-NEXT:    br i1 [[TMP9]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP15:![0-9]+]]
 ; CHECK-MAXBW:       middle.block:
 ; CHECK-MAXBW-NEXT:    [[TMP10:%.*]] = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> [[TMP8]])
 ; CHECK-MAXBW-NEXT:    [[TMP11:%.*]] = extractelement <16 x i32> [[TMP6]], i32 15
@@ -2825,7 +2636,7 @@ define i32 @not_dotp_extend_user(ptr %a, ptr %b) {
 ; CHECK-MAXBW-NEXT:    [[ADD]] = add i32 [[MUL]], [[ACCUM]]
 ; CHECK-MAXBW-NEXT:    [[IV_NEXT]] = add i64 [[IV]], 1
 ; CHECK-MAXBW-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], 1024
-; CHECK-MAXBW-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_EXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP18:![0-9]+]]
+; CHECK-MAXBW-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_EXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]]
 ; CHECK-MAXBW:       for.exit:
 ; CHECK-MAXBW-NEXT:    [[EXT_B_LCSSA:%.*]] = phi i32 [ [[EXT_B]], [[FOR_BODY]] ], [ [[TMP11]], [[MIDDLE_BLOCK]] ]
 ; CHECK-MAXBW-NEXT:    [[ADD_LCSSA:%.*]] = phi i32 [ [[ADD]], [[FOR_BODY]] ], [ [[TMP10]], [[MIDDLE_BLOCK]] ]
@@ -2872,13 +2683,11 @@ for.exit:                        ; preds = %for.body
 ; CHECK-INTERLEAVE1: [[LOOP9]] = distinct !{[[LOOP9]], [[META2]], [[META1]]}
 ; CHECK-INTERLEAVE1: [[LOOP10]] = distinct !{[[LOOP10]], [[META1]], [[META2]]}
 ; CHECK-INTERLEAVE1: [[LOOP11]] = distinct !{[[LOOP11]], [[META2]], [[META1]]}
-; CHECK-INTERLEAVE1: [[LOOP12]] = distinct !{[[LOOP12]], [[META1]], [[META2]]}
-; CHECK-INTERLEAVE1: [[LOOP13]] = distinct !{[[LOOP13]], [[META2]], [[META1]]}
-; CHECK-INTERLEAVE1: [[LOOP14]] = distinct !{[[LOOP14]], [[META15:![0-9]+]], [[META1]], [[META2]]}
-; CHECK-INTERLEAVE1: [[META15]] = !{!"llvm.loop.mustprogress"}
-; CHECK-INTERLEAVE1: [[LOOP16]] = distinct !{[[LOOP16]], [[META15]], [[META2]], [[META1]]}
-; CHECK-INTERLEAVE1: [[LOOP17]] = distinct !{[[LOOP17]], [[META1]], [[META2]]}
-; CHECK-INTERLEAVE1: [[LOOP18]] = distinct !{[[LOOP18]], [[META2]], [[META1]]}
+; CHECK-INTERLEAVE1: [[LOOP12]] = distinct !{[[LOOP12]], [[META13:![0-9]+]], [[META1]], [[META2]]}
+; CHECK-INTERLEAVE1: [[META13]] = !{!"llvm.loop.mustprogress"}
+; CHECK-INTERLEAVE1: [[LOOP14]] = distinct !{[[LOOP14]], [[META13]], [[META2]], [[META1]]}
+; CHECK-INTERLEAVE1: [[LOOP15]] = distinct !{[[LOOP15]], [[META1]], [[META2]]}
+; CHECK-INTERLEAVE1: [[LOOP16]] = distinct !{[[LOOP16]], [[META2]], [[META1]]}
 ;.
 ; CHECK-INTERLEAVED: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]], [[META2:![0-9]+]]}
 ; CHECK-INTERLEAVED: [[META1]] = !{!"llvm.loop.isvectorized", i32 1}
@@ -2892,13 +2701,11 @@ for.exit:                        ; preds = %for.body
 ; CHECK-INTERLEAVED: [[LOOP9]] = distinct !{[[LOOP9]], [[META2]], [[META1]]}
 ; CHECK-INTERLEAVED: [[LOOP10]] = distinct !{[[LOOP10]], [[META1]], [[META2]]}
 ; CHECK-INTERLEAVED: [[LOOP11]] = distinct !{[[LOOP11]], [[META2]], [[META1]]}
-; CHECK-INTERLEAVED: [[LOOP12]] = distinct !{[[LOOP12]], [[META1]], [[META2]]}
-; CHECK-INTERLEAVED: [[LOOP13]] = distinct !{[[LOOP13]], [[META2]], [[META1]]}
-; CHECK-INTERLEAVED: [[LOOP14]] = distinct !{[[LOOP14]], [[META15:![0-9]+]], [[META1]], [[META2]]}
-; CHECK-INTERLEAVED: [[META15]] = !{!"llvm.loop.mustprogress"}
-; CHECK-INTERLEAVED: [[LOOP16]] = distinct !{[[LOOP16]], [[META15]], [[META2]], [[META1]]}
-; CHECK-INTERLEAVED: [[LOOP17]] = distinct !{[[LOOP17]], [[META1]], [[META2]]}
-; CHECK-INTERLEAVED: [[LOOP18]] = distinct !{[[LOOP18]], [[META2]], [[META1]]}
+; CHECK-INTERLEAVED: [[LOOP12]] = distinct !{[[LOOP12]], [[META13:![0-9]+]], [[META1]], [[META2]]}
+; CHECK-INTERLEAVED: [[META13]] = !{!"llvm.loop.mustprogress"}
+; CHECK-INTERLEAVED: [[LOOP14]] = distinct !{[[LOOP14]], [[META13]], [[META2]], [[META1]]}
+; CHECK-INTERLEAVED: [[LOOP15]] = distinct !{[[LOOP15]], [[META1]], [[META2]]}
+; CHECK-INTERLEAVED: [[LOOP16]] = distinct !{[[LOOP16]], [[META2]], [[META1]]}
 ;.
 ; CHECK-MAXBW: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]], [[META2:![0-9]+]]}
 ; CHECK-MAXBW: [[META1]] = !{!"llvm.loop.isvectorized", i32 1}
@@ -2912,11 +2719,9 @@ for.exit:                        ; preds = %for.body
 ; CHECK-MAXBW: [[LOOP9]] = distinct !{[[LOOP9]], [[META2]], [[META1]]}
 ; CHECK-MAXBW: [[LOOP10]] = distinct !{[[LOOP10]], [[META1]], [[META2]]}
 ; CHECK-MAXBW: [[LOOP11]] = distinct !{[[LOOP11]], [[META2]], [[META1]]}
-; CHECK-MAXBW: [[LOOP12]] = distinct !{[[LOOP12]], [[META1]], [[META2]]}
-; CHECK-MAXBW: [[LOOP13]] = distinct !{[[LOOP13]], [[META2]], [[META1]]}
-; CHECK-MAXBW: [[LOOP14]] = distinct !{[[LOOP14]], [[META15:![0-9]+]], [[META1]], [[META2]]}
-; CHECK-MAXBW: [[META15]] = !{!"llvm.loop.mustprogress"}
-; CHECK-MAXBW: [[LOOP16]] = distinct !{[[LOOP16]], [[META15]], [[META2]], [[META1]]}
-; CHECK-MAXBW: [[LOOP17]] = distinct !{[[LOOP17]], [[META1]], [[META2]]}
-; CHECK-MAXBW: [[LOOP18]] = distinct !{[[LOOP18]], [[META2]], [[META1]]}
+; CHECK-MAXBW: [[LOOP12]] = distinct !{[[LOOP12]], [[META13:![0-9]+]], [[META1]], [[META2]]}
+; CHECK-MAXBW: [[META13]] = !{!"llvm.loop.mustprogress"}
+; CHECK-MAXBW: [[LOOP14]] = distinct !{[[LOOP14]], [[META13]], [[META2]], [[META1]]}
+; CHECK-MAXBW: [[LOOP15]] = distinct !{[[LOOP15]], [[META1]], [[META2]]}
+; CHECK-MAXBW: [[LOOP16]] = distinct !{[[LOOP16]], [[META2]], [[META1]]}
 ;.
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-dot-product.ll b/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-dot-product.ll
index a0214ae88c2d6..a734a0e94198a 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-dot-product.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-dot-product.ll
@@ -1,4 +1,4 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 4
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --version 4
 ; RUN: opt -passes=loop-vectorize -force-vector-interleave=1 -enable-epilogue-vectorization=false -S < %s | FileCheck %s --check-prefixes=CHECK-INTERLEAVE1
 ; RUN: opt -passes=loop-vectorize -enable-epilogue-vectorization=false -S < %s | FileCheck %s --check-prefixes=CHECK-INTERLEAVED
 ; RUN: opt -passes=loop-vectorize -force-vector-interleave=1 -vectorizer-maximize-bandwidth -enable-epilogue-vectorization=false -S < %s | FileCheck %s --check-prefixes=CHECK-MAXBW
@@ -12,7 +12,7 @@ define i32 @dotp(ptr %a, ptr %b) #0 {
 ; CHECK-INTERLEAVE1-NEXT:  entry:
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 4
-; CHECK-INTERLEAVE1-NEXT:    br i1 false, label [[VEC_EPILOG_PH:%.*]], label [[VECTOR_PH1:%.*]]
+; CHECK-INTERLEAVE1-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK-INTERLEAVE1:       vector.ph:
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 4
@@ -20,10 +20,10 @@ define i32 @dotp(ptr %a, ptr %b) #0 {
 ; CHECK-INTERLEAVE1-NEXT:    [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 4
-; CHECK-INTERLEAVE1-NEXT:    br label [[VECTOR_BODY1:%.*]]
+; CHECK-INTERLEAVE1-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK-INTERLEAVE1:       vector.body:
-; CHECK-INTERLEAVE1-NEXT:    [[INDEX1:%.*]] = phi i64 [ 0, [[VECTOR_PH1]] ], [ [[INDEX_NEXT1:%.*]], [[VECTOR_BODY1]] ]
-; CHECK-INTERLEAVE1-NEXT:    [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, [[VECTOR_PH1]] ], [ [[TMP14:%.*]], [[VECTOR_BODY1]] ]
+; CHECK-INTERLEAVE1-NEXT:    [[INDEX1:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT1:%.*]], [[VECTOR_BODY]] ]
+; CHECK-INTERLEAVE1-NEXT:    [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP14:%.*]], [[VECTOR_BODY]] ]
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP12:%.*]] = add i64 [[INDEX1]], 0
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP16:%.*]] = getelementptr i8, ptr [[A]], i64 [[TMP12]]
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP17:%.*]] = getelementptr i8, ptr [[TMP16]], i32 0
@@ -37,18 +37,18 @@ define i32 @dotp(ptr %a, ptr %b) #0 {
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP14]] = add <vscale x 4 x i32> [[TMP13]], [[VEC_PHI]]
 ; CHECK-INTERLEAVE1-NEXT:    [[INDEX_NEXT1]] = add nuw i64 [[INDEX1]], [[TMP5]]
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP15:%.*]] = icmp eq i64 [[INDEX_NEXT1]], [[N_VEC]]
-; CHECK-INTERLEAVE1-NEXT:    br i1 [[TMP15]], label [[VEC_EPILOG_ITER_CHECK:%.*]], label [[VECTOR_BODY1]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK-INTERLEAVE1-NEXT:    br i1 [[TMP15]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; CHECK-INTERLEAVE1:       middle.block:
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP27:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[TMP14]])
 ; CHECK-INTERLEAVE1-NEXT:    [[CMP_N:%.*]] = icmp eq i64 1024, [[N_VEC]]
-; CHECK-INTERLEAVE1-NEXT:    br i1 [[CMP_N]], label [[FOR_EXIT:%.*]], label [[VEC_EPILOG_PH]]
+; CHECK-INTERLEAVE1-NEXT:    br i1 [[CMP_N]], label [[FOR_EXIT:%.*]], label [[SCALAR_PH]]
 ; CHECK-INTERLEAVE1:       scalar.ph:
-; CHECK-INTERLEAVE1-NEXT:    [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[VECTOR_PH:%.*]] ]
-; CHECK-INTERLEAVE1-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP27]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[VECTOR_PH]] ]
-; CHECK-INTERLEAVE1-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK-INTERLEAVE1-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-INTERLEAVE1-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP27]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ]
+; CHECK-INTERLEAVE1-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK-INTERLEAVE1:       for.body:
-; CHECK-INTERLEAVE1-NEXT:    [[INDEX:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], [[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-INTERLEAVE1-NEXT:    [[ACCUM:%.*]] = phi i32 [ [[BC_MERGE_RDX]], [[VEC_EPILOG_PH]] ], [ [[ADD:%.*]], [[VECTOR_BODY]] ]
+; CHECK-INTERLEAVE1-NEXT:    [[INDEX:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INDEX_NEXT:%.*]], [[FOR_BODY]] ]
+; CHECK-INTERLEAVE1-NEXT:    [[ACCUM:%.*]] = phi i32 [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[ADD:%.*]], [[FOR_BODY]] ]
 ; CHECK-INTERLEAVE1-NEXT:    [[GEP_A:%.*]] = getelementptr i8, ptr [[A]], i64 [[INDEX]]
 ; CHECK-INTERLEAVE1-NEXT:    [[LOAD_A:%.*]] = load i8, ptr [[GEP_A]], align 1
 ; CHECK-INTERLEAVE1-NEXT:    [[EXT_A:%.*]] = zext i8 [[LOAD_A]] to i32
@@ -59,9 +59,9 @@ define i32 @dotp(ptr %a, ptr %b) #0 {
 ; CHECK-INTERLEAVE1-NEXT:    [[ADD]] = add i32 [[MUL]], [[ACCUM]]
 ; CHECK-INTERLEAVE1-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 1
 ; CHECK-INTERLEAVE1-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
-; CHECK-INTERLEAVE1-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_EXIT]], label [[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK-INTERLEAVE1-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_EXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
 ; CHECK-INTERLEAVE1:       for.exit:
-; CHECK-INTERLEAVE1-NEXT:    [[ADD_LCSSA:%.*]] = phi i32 [ [[ADD]], [[VECTOR_BODY]] ], [ [[TMP27]], [[VEC_EPILOG_ITER_CHECK]] ]
+; CHECK-INTERLEAVE1-NEXT:    [[ADD_LCSSA:%.*]] = phi i32 [ [[ADD]], [[FOR_BODY]] ], [ [[TMP27]], [[MIDDLE_BLOCK]] ]
 ; CHECK-INTERLEAVE1-NEXT:    ret i32 [[ADD_LCSSA]]
 ;
 ; CHECK-INTERLEAVED-LABEL: define i32 @dotp(
@@ -69,7 +69,7 @@ define i32 @dotp(ptr %a, ptr %b) #0 {
 ; CHECK-INTERLEAVED-NEXT:  entry:
 ; CHECK-INTERLEAVED-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
 ; CHECK-INTERLEAVED-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 8
-; CHECK-INTERLEAVED-NEXT:    br i1 false, label [[VEC_EPILOG_PH:%.*]], label [[VECTOR_PH1:%.*]]
+; CHECK-INTERLEAVED-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK-INTERLEAVED:       vector.ph:
 ; CHECK-INTERLEAVED-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
 ; CHECK-INTERLEAVED-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 8
@@ -77,11 +77,11 @@ define i32 @dotp(ptr %a, ptr %b) #0 {
 ; CHECK-INTERLEAVED-NEXT:    [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
 ; CHECK-INTERLEAVED-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
 ; CHECK-INTERLEAVED-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 8
-; CHECK-INTERLEAVED-NEXT:    br label [[VECTOR_BODY1:%.*]]
+; CHECK-INTERLEAVED-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK-INTERLEAVED:       vector.body:
-; CHECK-INTERLEAVED-NEXT:    [[INDEX1:%.*]] = phi i64 [ 0, [[VECTOR_PH1]] ], [ [[INDEX_NEXT1:%.*]], [[VECTOR_BODY1]] ]
-; CHECK-INTERLEAVED-NEXT:    [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, [[VECTOR_PH1]] ], [ [[TMP23:%.*]], [[VECTOR_BODY1]] ]
-; CHECK-INTERLEAVED-NEXT:    [[VEC_PHI1:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, [[VECTOR_PH1]] ], [ [[TMP24:%.*]], [[VECTOR_BODY1]] ]
+; CHECK-INTERLEAVED-NEXT:    [[INDEX1:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT1:%.*]], [[VECTOR_BODY]] ]
+; CHECK-INTERLEAVED-NEXT:    [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP23:%.*]], [[VECTOR_BODY]] ]
+; CHECK-INTERLEAVED-NEXT:    [[VEC_PHI1:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP24:%.*]], [[VECTOR_BODY]] ]
 ; CHECK-INTERLEAVED-NEXT:    [[TMP17:%.*]] = add i64 [[INDEX1]], 0
 ; CHECK-INTERLEAVED-NEXT:    [[TMP20:%.*]] = getelementptr i8, ptr [[A]], i64 [[TMP17]]
 ; CHECK-INTERLEAVED-NEXT:    [[TMP21:%.*]] = getelementptr i8, ptr [[TMP20]], i32 0
@@ -107,19 +107,19 @@ define i32 @dotp(ptr %a, ptr %b) #0 {
 ; CHECK-INTERLEAVED-NEXT:    [[TMP24]] = add <vscale x 4 x i32> [[TMP22]], [[VEC_PHI1]]
 ; CHECK-INTERLEAVED-NEXT:    [[INDEX_NEXT1]] = add nuw i64 [[INDEX1]], [[TMP5]]
 ; CHECK-INTERLEAVED-NEXT:    [[TMP25:%.*]] = icmp eq i64 [[INDEX_NEXT1]], [[N_VEC]]
-; CHECK-INTERLEAVED-NEXT:    br i1 [[TMP25]], label [[VEC_EPILOG_ITER_CHECK:%.*]], label [[VECTOR_BODY1]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK-INTERLEAVED-NEXT:    br i1 [[TMP25]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; CHECK-INTERLEAVED:       middle.block:
 ; CHECK-INTERLEAVED-NEXT:    [[BIN_RDX:%.*]] = add <vscale x 4 x i32> [[TMP24]], [[TMP23]]
 ; CHECK-INTERLEAVED-NEXT:    [[TMP16:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[BIN_RDX]])
 ; CHECK-INTERLEAVED-NEXT:    [[CMP_N:%.*]] = icmp eq i64 1024, [[N_VEC]]
-; CHECK-INTERLEAVED-NEXT:    br i1 [[CMP_N]], label [[FOR_EXIT:%.*]], label [[VEC_EPILOG_PH]]
+; CHECK-INTERLEAVED-NEXT:    br i1 [[CMP_N]], label [[FOR_EXIT:%.*]], label [[SCALAR_PH]]
 ; CHECK-INTERLEAVED:       scalar.ph:
-; CHECK-INTERLEAVED-NEXT:    [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[VECTOR_PH:%.*]] ]
-; CHECK-INTERLEAVED-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP16]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[VECTOR_PH]] ]
-; CHECK-INTERLEAVED-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK-INTERLEAVED-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-INTERLEAVED-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP16]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ]
+; CHECK-INTERLEAVED-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK-INTERLEAVED:       for.body:
-; CHECK-INTERLEAVED-NEXT:    [[INDEX:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], [[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-INTERLEAVED-NEXT:    [[ACCUM:%.*]] = phi i32 [ [[BC_MERGE_RDX]], [[VEC_EPILOG_PH]] ], [ [[ADD:%.*]], [[VECTOR_BODY]] ]
+; CHECK-INTERLEAVED-NEXT:    [[INDEX:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INDEX_NEXT:%.*]], [[FOR_BODY]] ]
+; CHECK-INTERLEAVED-NEXT:    [[ACCUM:%.*]] = phi i32 [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[ADD:%.*]], [[FOR_BODY]] ]
 ; CHECK-INTERLEAVED-NEXT:    [[GEP_A:%.*]] = getelementptr i8, ptr [[A]], i64 [[INDEX]]
 ; CHECK-INTERLEAVED-NEXT:    [[LOAD_A:%.*]] = load i8, ptr [[GEP_A]], align 1
 ; CHECK-INTERLEAVED-NEXT:    [[EXT_A:%.*]] = zext i8 [[LOAD_A]] to i32
@@ -130,9 +130,9 @@ define i32 @dotp(ptr %a, ptr %b) #0 {
 ; CHECK-INTERLEAVED-NEXT:    [[ADD]] = add i32 [[MUL]], [[ACCUM]]
 ; CHECK-INTERLEAVED-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 1
 ; CHECK-INTERLEAVED-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
-; CHECK-INTERLEAVED-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_EXIT]], label [[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK-INTERLEAVED-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_EXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
 ; CHECK-INTERLEAVED:       for.exit:
-; CHECK-INTERLEAVED-NEXT:    [[ADD_LCSSA:%.*]] = phi i32 [ [[ADD]], [[VECTOR_BODY]] ], [ [[TMP16]], [[VEC_EPILOG_ITER_CHECK]] ]
+; CHECK-INTERLEAVED-NEXT:    [[ADD_LCSSA:%.*]] = phi i32 [ [[ADD]], [[FOR_BODY]] ], [ [[TMP16]], [[MIDDLE_BLOCK]] ]
 ; CHECK-INTERLEAVED-NEXT:    ret i32 [[ADD_LCSSA]]
 ;
 ; CHECK-MAXBW-LABEL: define i32 @dotp(
@@ -214,6 +214,480 @@ for.exit:                        ; preds = %for.body
   ret i32 %add
 }
 
+define i64 @not_dotp_i8_to_i64_has_neon_dotprod(ptr readonly %a, ptr readonly %b) #1 {
+; CHECK-INTERLEAVE1-LABEL: define i64 @not_dotp_i8_to_i64_has_neon_dotprod(
+; CHECK-INTERLEAVE1-SAME: ptr readonly [[A:%.*]], ptr readonly [[B:%.*]]) #[[ATTR1:[0-9]+]] {
+; CHECK-INTERLEAVE1-NEXT:  entry:
+; CHECK-INTERLEAVE1-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-INTERLEAVE1-NEXT:    [[TMP8:%.*]] = mul i64 [[TMP7]], 2
+; CHECK-INTERLEAVE1-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK-INTERLEAVE1:       vector.ph:
+; CHECK-INTERLEAVE1-NEXT:    [[TMP9:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-INTERLEAVE1-NEXT:    [[TMP12:%.*]] = mul i64 [[TMP9]], 2
+; CHECK-INTERLEAVE1-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP12]]
+; CHECK-INTERLEAVE1-NEXT:    [[TMP10:%.*]] = sub i64 1024, [[N_MOD_VF]]
+; CHECK-INTERLEAVE1-NEXT:    [[TMP18:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-INTERLEAVE1-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP18]], 2
+; CHECK-INTERLEAVE1-NEXT:    [[TMP0:%.*]] = getelementptr i8, ptr [[A]], i64 [[TMP10]]
+; CHECK-INTERLEAVE1-NEXT:    [[TMP1:%.*]] = getelementptr i8, ptr [[B]], i64 [[TMP10]]
+; CHECK-INTERLEAVE1-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK-INTERLEAVE1:       vector.body:
+; CHECK-INTERLEAVE1-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-INTERLEAVE1-NEXT:    [[VEC_PHI:%.*]] = phi <vscale x 2 x i64> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP15:%.*]], [[VECTOR_BODY]] ]
+; CHECK-INTERLEAVE1-NEXT:    [[TMP2:%.*]] = add i64 [[INDEX]], 0
+; CHECK-INTERLEAVE1-NEXT:    [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[A]], i64 [[TMP2]]
+; CHECK-INTERLEAVE1-NEXT:    [[TMP3:%.*]] = add i64 [[INDEX]], 0
+; CHECK-INTERLEAVE1-NEXT:    [[NEXT_GEP1:%.*]] = getelementptr i8, ptr [[B]], i64 [[TMP3]]
+; CHECK-INTERLEAVE1-NEXT:    [[TMP4:%.*]] = getelementptr i8, ptr [[NEXT_GEP]], i32 0
+; CHECK-INTERLEAVE1-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 2 x i8>, ptr [[TMP4]], align 1
+; CHECK-INTERLEAVE1-NEXT:    [[TMP11:%.*]] = zext <vscale x 2 x i8> [[WIDE_LOAD]] to <vscale x 2 x i64>
+; CHECK-INTERLEAVE1-NEXT:    [[TMP6:%.*]] = getelementptr i8, ptr [[NEXT_GEP1]], i32 0
+; CHECK-INTERLEAVE1-NEXT:    [[WIDE_LOAD2:%.*]] = load <vscale x 2 x i8>, ptr [[TMP6]], align 1
+; CHECK-INTERLEAVE1-NEXT:    [[TMP13:%.*]] = zext <vscale x 2 x i8> [[WIDE_LOAD2]] to <vscale x 2 x i64>
+; CHECK-INTERLEAVE1-NEXT:    [[TMP14:%.*]] = mul nuw nsw <vscale x 2 x i64> [[TMP13]], [[TMP11]]
+; CHECK-INTERLEAVE1-NEXT:    [[TMP15]] = add <vscale x 2 x i64> [[TMP14]], [[VEC_PHI]]
+; CHECK-INTERLEAVE1-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
+; CHECK-INTERLEAVE1-NEXT:    [[TMP16:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[TMP10]]
+; CHECK-INTERLEAVE1-NEXT:    br i1 [[TMP16]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK-INTERLEAVE1:       middle.block:
+; CHECK-INTERLEAVE1-NEXT:    [[TMP17:%.*]] = call i64 @llvm.vector.reduce.add.nxv2i64(<vscale x 2 x i64> [[TMP15]])
+; CHECK-INTERLEAVE1-NEXT:    [[CMP_N:%.*]] = icmp eq i64 1024, [[TMP10]]
+; CHECK-INTERLEAVE1-NEXT:    br i1 [[CMP_N]], label [[FOR_EXIT:%.*]], label [[SCALAR_PH]]
+; CHECK-INTERLEAVE1:       scalar.ph:
+; CHECK-INTERLEAVE1-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i64 [ [[TMP10]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-INTERLEAVE1-NEXT:    [[BC_MERGE_RDX1:%.*]] = phi i64 [ [[TMP17]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ]
+; CHECK-INTERLEAVE1-NEXT:    [[BC_RESUME_VAL3:%.*]] = phi ptr [ [[TMP0]], [[MIDDLE_BLOCK]] ], [ [[A]], [[ENTRY]] ]
+; CHECK-INTERLEAVE1-NEXT:    [[BC_RESUME_VAL4:%.*]] = phi ptr [ [[TMP1]], [[MIDDLE_BLOCK]] ], [ [[B]], [[ENTRY]] ]
+; CHECK-INTERLEAVE1-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK-INTERLEAVE1:       for.body:
+; CHECK-INTERLEAVE1-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
+; CHECK-INTERLEAVE1-NEXT:    [[ACCUM:%.*]] = phi i64 [ [[BC_MERGE_RDX1]], [[SCALAR_PH]] ], [ [[ADD:%.*]], [[FOR_BODY]] ]
+; CHECK-INTERLEAVE1-NEXT:    [[GEP_A:%.*]] = phi ptr [ [[BC_RESUME_VAL3]], [[SCALAR_PH]] ], [ [[GEP_A_NEXT:%.*]], [[FOR_BODY]] ]
+; CHECK-INTERLEAVE1-NEXT:    [[GEP_B:%.*]] = phi ptr [ [[BC_RESUME_VAL4]], [[SCALAR_PH]] ], [ [[GEP_B_NEXT:%.*]], [[FOR_BODY]] ]
+; CHECK-INTERLEAVE1-NEXT:    [[GEP_A_NEXT]] = getelementptr inbounds nuw i8, ptr [[GEP_A]], i64 1
+; CHECK-INTERLEAVE1-NEXT:    [[LOAD_A:%.*]] = load i8, ptr [[GEP_A]], align 1
+; CHECK-INTERLEAVE1-NEXT:    [[EXT_A:%.*]] = zext i8 [[LOAD_A]] to i64
+; CHECK-INTERLEAVE1-NEXT:    [[GEP_B_NEXT]] = getelementptr inbounds nuw i8, ptr [[GEP_B]], i64 1
+; CHECK-INTERLEAVE1-NEXT:    [[LOAD_B:%.*]] = load i8, ptr [[GEP_B]], align 1
+; CHECK-INTERLEAVE1-NEXT:    [[EXT_B:%.*]] = zext i8 [[LOAD_B]] to i64
+; CHECK-INTERLEAVE1-NEXT:    [[MUL:%.*]] = mul nuw nsw i64 [[EXT_B]], [[EXT_A]]
+; CHECK-INTERLEAVE1-NEXT:    [[ADD]] = add nsw i64 [[MUL]], [[ACCUM]]
+; CHECK-INTERLEAVE1-NEXT:    [[IV_NEXT]] = add i64 [[IV]], 1
+; CHECK-INTERLEAVE1-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], 1024
+; CHECK-INTERLEAVE1-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_EXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
+; CHECK-INTERLEAVE1:       for.exit:
+; CHECK-INTERLEAVE1-NEXT:    [[ADD_LCSSA:%.*]] = phi i64 [ [[ADD]], [[FOR_BODY]] ], [ [[TMP17]], [[MIDDLE_BLOCK]] ]
+; CHECK-INTERLEAVE1-NEXT:    ret i64 [[ADD_LCSSA]]
+;
+; CHECK-INTERLEAVED-LABEL: define i64 @not_dotp_i8_to_i64_has_neon_dotprod(
+; CHECK-INTERLEAVED-SAME: ptr readonly [[A:%.*]], ptr readonly [[B:%.*]]) #[[ATTR1:[0-9]+]] {
+; CHECK-INTERLEAVED-NEXT:  entry:
+; CHECK-INTERLEAVED-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-INTERLEAVED-NEXT:    [[TMP8:%.*]] = mul i64 [[TMP7]], 4
+; CHECK-INTERLEAVED-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK-INTERLEAVED:       vector.ph:
+; CHECK-INTERLEAVED-NEXT:    [[TMP9:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-INTERLEAVED-NEXT:    [[TMP16:%.*]] = mul i64 [[TMP9]], 4
+; CHECK-INTERLEAVED-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP16]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP10:%.*]] = sub i64 1024, [[N_MOD_VF]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP28:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-INTERLEAVED-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP28]], 4
+; CHECK-INTERLEAVED-NEXT:    [[TMP0:%.*]] = getelementptr i8, ptr [[A]], i64 [[TMP10]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP1:%.*]] = getelementptr i8, ptr [[B]], i64 [[TMP10]]
+; CHECK-INTERLEAVED-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK-INTERLEAVED:       vector.body:
+; CHECK-INTERLEAVED-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-INTERLEAVED-NEXT:    [[VEC_PHI:%.*]] = phi <vscale x 2 x i64> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP24:%.*]], [[VECTOR_BODY]] ]
+; CHECK-INTERLEAVED-NEXT:    [[VEC_PHI1:%.*]] = phi <vscale x 2 x i64> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP25:%.*]], [[VECTOR_BODY]] ]
+; CHECK-INTERLEAVED-NEXT:    [[TMP2:%.*]] = add i64 [[INDEX]], 0
+; CHECK-INTERLEAVED-NEXT:    [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[A]], i64 [[TMP2]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP3:%.*]] = add i64 [[INDEX]], 0
+; CHECK-INTERLEAVED-NEXT:    [[NEXT_GEP1:%.*]] = getelementptr i8, ptr [[B]], i64 [[TMP3]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP4:%.*]] = getelementptr i8, ptr [[NEXT_GEP]], i32 0
+; CHECK-INTERLEAVED-NEXT:    [[TMP11:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-INTERLEAVED-NEXT:    [[TMP12:%.*]] = mul i64 [[TMP11]], 2
+; CHECK-INTERLEAVED-NEXT:    [[TMP13:%.*]] = getelementptr i8, ptr [[NEXT_GEP]], i64 [[TMP12]]
+; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 2 x i8>, ptr [[TMP4]], align 1
+; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD3:%.*]] = load <vscale x 2 x i8>, ptr [[TMP13]], align 1
+; CHECK-INTERLEAVED-NEXT:    [[TMP14:%.*]] = zext <vscale x 2 x i8> [[WIDE_LOAD]] to <vscale x 2 x i64>
+; CHECK-INTERLEAVED-NEXT:    [[TMP15:%.*]] = zext <vscale x 2 x i8> [[WIDE_LOAD3]] to <vscale x 2 x i64>
+; CHECK-INTERLEAVED-NEXT:    [[TMP6:%.*]] = getelementptr i8, ptr [[NEXT_GEP1]], i32 0
+; CHECK-INTERLEAVED-NEXT:    [[TMP17:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-INTERLEAVED-NEXT:    [[TMP18:%.*]] = mul i64 [[TMP17]], 2
+; CHECK-INTERLEAVED-NEXT:    [[TMP19:%.*]] = getelementptr i8, ptr [[NEXT_GEP1]], i64 [[TMP18]]
+; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD4:%.*]] = load <vscale x 2 x i8>, ptr [[TMP6]], align 1
+; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD5:%.*]] = load <vscale x 2 x i8>, ptr [[TMP19]], align 1
+; CHECK-INTERLEAVED-NEXT:    [[TMP20:%.*]] = zext <vscale x 2 x i8> [[WIDE_LOAD4]] to <vscale x 2 x i64>
+; CHECK-INTERLEAVED-NEXT:    [[TMP21:%.*]] = zext <vscale x 2 x i8> [[WIDE_LOAD5]] to <vscale x 2 x i64>
+; CHECK-INTERLEAVED-NEXT:    [[TMP22:%.*]] = mul nuw nsw <vscale x 2 x i64> [[TMP20]], [[TMP14]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP23:%.*]] = mul nuw nsw <vscale x 2 x i64> [[TMP21]], [[TMP15]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP24]] = add <vscale x 2 x i64> [[TMP22]], [[VEC_PHI]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP25]] = add <vscale x 2 x i64> [[TMP23]], [[VEC_PHI1]]
+; CHECK-INTERLEAVED-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP26:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[TMP10]]
+; CHECK-INTERLEAVED-NEXT:    br i1 [[TMP26]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK-INTERLEAVED:       middle.block:
+; CHECK-INTERLEAVED-NEXT:    [[BIN_RDX:%.*]] = add <vscale x 2 x i64> [[TMP25]], [[TMP24]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP27:%.*]] = call i64 @llvm.vector.reduce.add.nxv2i64(<vscale x 2 x i64> [[BIN_RDX]])
+; CHECK-INTERLEAVED-NEXT:    [[CMP_N:%.*]] = icmp eq i64 1024, [[TMP10]]
+; CHECK-INTERLEAVED-NEXT:    br i1 [[CMP_N]], label [[FOR_EXIT:%.*]], label [[SCALAR_PH]]
+; CHECK-INTERLEAVED:       scalar.ph:
+; CHECK-INTERLEAVED-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i64 [ [[TMP10]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-INTERLEAVED-NEXT:    [[BC_MERGE_RDX1:%.*]] = phi i64 [ [[TMP27]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ]
+; CHECK-INTERLEAVED-NEXT:    [[BC_RESUME_VAL3:%.*]] = phi ptr [ [[TMP0]], [[MIDDLE_BLOCK]] ], [ [[A]], [[ENTRY]] ]
+; CHECK-INTERLEAVED-NEXT:    [[BC_RESUME_VAL4:%.*]] = phi ptr [ [[TMP1]], [[MIDDLE_BLOCK]] ], [ [[B]], [[ENTRY]] ]
+; CHECK-INTERLEAVED-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK-INTERLEAVED:       for.body:
+; CHECK-INTERLEAVED-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
+; CHECK-INTERLEAVED-NEXT:    [[ACCUM:%.*]] = phi i64 [ [[BC_MERGE_RDX1]], [[SCALAR_PH]] ], [ [[ADD:%.*]], [[FOR_BODY]] ]
+; CHECK-INTERLEAVED-NEXT:    [[GEP_A:%.*]] = phi ptr [ [[BC_RESUME_VAL3]], [[SCALAR_PH]] ], [ [[GEP_A_NEXT:%.*]], [[FOR_BODY]] ]
+; CHECK-INTERLEAVED-NEXT:    [[GEP_B:%.*]] = phi ptr [ [[BC_RESUME_VAL4]], [[SCALAR_PH]] ], [ [[GEP_B_NEXT:%.*]], [[FOR_BODY]] ]
+; CHECK-INTERLEAVED-NEXT:    [[GEP_A_NEXT]] = getelementptr inbounds nuw i8, ptr [[GEP_A]], i64 1
+; CHECK-INTERLEAVED-NEXT:    [[LOAD_A:%.*]] = load i8, ptr [[GEP_A]], align 1
+; CHECK-INTERLEAVED-NEXT:    [[EXT_A:%.*]] = zext i8 [[LOAD_A]] to i64
+; CHECK-INTERLEAVED-NEXT:    [[GEP_B_NEXT]] = getelementptr inbounds nuw i8, ptr [[GEP_B]], i64 1
+; CHECK-INTERLEAVED-NEXT:    [[LOAD_B:%.*]] = load i8, ptr [[GEP_B]], align 1
+; CHECK-INTERLEAVED-NEXT:    [[EXT_B:%.*]] = zext i8 [[LOAD_B]] to i64
+; CHECK-INTERLEAVED-NEXT:    [[MUL:%.*]] = mul nuw nsw i64 [[EXT_B]], [[EXT_A]]
+; CHECK-INTERLEAVED-NEXT:    [[ADD]] = add nsw i64 [[MUL]], [[ACCUM]]
+; CHECK-INTERLEAVED-NEXT:    [[IV_NEXT]] = add i64 [[IV]], 1
+; CHECK-INTERLEAVED-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], 1024
+; CHECK-INTERLEAVED-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_EXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
+; CHECK-INTERLEAVED:       for.exit:
+; CHECK-INTERLEAVED-NEXT:    [[ADD_LCSSA:%.*]] = phi i64 [ [[ADD]], [[FOR_BODY]] ], [ [[TMP27]], [[MIDDLE_BLOCK]] ]
+; CHECK-INTERLEAVED-NEXT:    ret i64 [[ADD_LCSSA]]
+;
+; CHECK-MAXBW-LABEL: define i64 @not_dotp_i8_to_i64_has_neon_dotprod(
+; CHECK-MAXBW-SAME: ptr readonly [[A:%.*]], ptr readonly [[B:%.*]]) #[[ATTR1:[0-9]+]] {
+; CHECK-MAXBW-NEXT:  entry:
+; CHECK-MAXBW-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-MAXBW-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 8
+; CHECK-MAXBW-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK-MAXBW:       vector.ph:
+; CHECK-MAXBW-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-MAXBW-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 8
+; CHECK-MAXBW-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]]
+; CHECK-MAXBW-NEXT:    [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
+; CHECK-MAXBW-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-MAXBW-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 8
+; CHECK-MAXBW-NEXT:    [[TMP6:%.*]] = getelementptr i8, ptr [[A]], i64 [[N_VEC]]
+; CHECK-MAXBW-NEXT:    [[TMP7:%.*]] = getelementptr i8, ptr [[B]], i64 [[N_VEC]]
+; CHECK-MAXBW-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK-MAXBW:       vector.body:
+; CHECK-MAXBW-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-MAXBW-NEXT:    [[VEC_PHI:%.*]] = phi <vscale x 8 x i64> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP15:%.*]], [[VECTOR_BODY]] ]
+; CHECK-MAXBW-NEXT:    [[TMP8:%.*]] = add i64 [[INDEX]], 0
+; CHECK-MAXBW-NEXT:    [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[A]], i64 [[TMP8]]
+; CHECK-MAXBW-NEXT:    [[TMP9:%.*]] = add i64 [[INDEX]], 0
+; CHECK-MAXBW-NEXT:    [[NEXT_GEP1:%.*]] = getelementptr i8, ptr [[B]], i64 [[TMP9]]
+; CHECK-MAXBW-NEXT:    [[TMP10:%.*]] = getelementptr i8, ptr [[NEXT_GEP]], i32 0
+; CHECK-MAXBW-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 8 x i8>, ptr [[TMP10]], align 1
+; CHECK-MAXBW-NEXT:    [[TMP11:%.*]] = zext <vscale x 8 x i8> [[WIDE_LOAD]] to <vscale x 8 x i64>
+; CHECK-MAXBW-NEXT:    [[TMP12:%.*]] = getelementptr i8, ptr [[NEXT_GEP1]], i32 0
+; CHECK-MAXBW-NEXT:    [[WIDE_LOAD2:%.*]] = load <vscale x 8 x i8>, ptr [[TMP12]], align 1
+; CHECK-MAXBW-NEXT:    [[TMP13:%.*]] = zext <vscale x 8 x i8> [[WIDE_LOAD2]] to <vscale x 8 x i64>
+; CHECK-MAXBW-NEXT:    [[TMP14:%.*]] = mul nuw nsw <vscale x 8 x i64> [[TMP13]], [[TMP11]]
+; CHECK-MAXBW-NEXT:    [[TMP15]] = add <vscale x 8 x i64> [[TMP14]], [[VEC_PHI]]
+; CHECK-MAXBW-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
+; CHECK-MAXBW-NEXT:    [[TMP16:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-MAXBW-NEXT:    br i1 [[TMP16]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK-MAXBW:       middle.block:
+; CHECK-MAXBW-NEXT:    [[TMP17:%.*]] = call i64 @llvm.vector.reduce.add.nxv8i64(<vscale x 8 x i64> [[TMP15]])
+; CHECK-MAXBW-NEXT:    [[CMP_N:%.*]] = icmp eq i64 1024, [[N_VEC]]
+; CHECK-MAXBW-NEXT:    br i1 [[CMP_N]], label [[FOR_EXIT:%.*]], label [[SCALAR_PH]]
+; CHECK-MAXBW:       scalar.ph:
+; CHECK-MAXBW-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-MAXBW-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i64 [ [[TMP17]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ]
+; CHECK-MAXBW-NEXT:    [[BC_RESUME_VAL3:%.*]] = phi ptr [ [[TMP6]], [[MIDDLE_BLOCK]] ], [ [[A]], [[ENTRY]] ]
+; CHECK-MAXBW-NEXT:    [[BC_RESUME_VAL4:%.*]] = phi ptr [ [[TMP7]], [[MIDDLE_BLOCK]] ], [ [[B]], [[ENTRY]] ]
+; CHECK-MAXBW-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK-MAXBW:       for.body:
+; CHECK-MAXBW-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
+; CHECK-MAXBW-NEXT:    [[ACCUM:%.*]] = phi i64 [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[ADD:%.*]], [[FOR_BODY]] ]
+; CHECK-MAXBW-NEXT:    [[GEP_A:%.*]] = phi ptr [ [[BC_RESUME_VAL3]], [[SCALAR_PH]] ], [ [[GEP_A_NEXT:%.*]], [[FOR_BODY]] ]
+; CHECK-MAXBW-NEXT:    [[GEP_B:%.*]] = phi ptr [ [[BC_RESUME_VAL4]], [[SCALAR_PH]] ], [ [[GEP_B_NEXT:%.*]], [[FOR_BODY]] ]
+; CHECK-MAXBW-NEXT:    [[GEP_A_NEXT]] = getelementptr inbounds nuw i8, ptr [[GEP_A]], i64 1
+; CHECK-MAXBW-NEXT:    [[LOAD_A:%.*]] = load i8, ptr [[GEP_A]], align 1
+; CHECK-MAXBW-NEXT:    [[EXT_A:%.*]] = zext i8 [[LOAD_A]] to i64
+; CHECK-MAXBW-NEXT:    [[GEP_B_NEXT]] = getelementptr inbounds nuw i8, ptr [[GEP_B]], i64 1
+; CHECK-MAXBW-NEXT:    [[LOAD_B:%.*]] = load i8, ptr [[GEP_B]], align 1
+; CHECK-MAXBW-NEXT:    [[EXT_B:%.*]] = zext i8 [[LOAD_B]] to i64
+; CHECK-MAXBW-NEXT:    [[MUL:%.*]] = mul nuw nsw i64 [[EXT_B]], [[EXT_A]]
+; CHECK-MAXBW-NEXT:    [[ADD]] = add nsw i64 [[MUL]], [[ACCUM]]
+; CHECK-MAXBW-NEXT:    [[IV_NEXT]] = add i64 [[IV]], 1
+; CHECK-MAXBW-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], 1024
+; CHECK-MAXBW-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_EXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
+; CHECK-MAXBW:       for.exit:
+; CHECK-MAXBW-NEXT:    [[ADD_LCSSA:%.*]] = phi i64 [ [[ADD]], [[FOR_BODY]] ], [ [[TMP17]], [[MIDDLE_BLOCK]] ]
+; CHECK-MAXBW-NEXT:    ret i64 [[ADD_LCSSA]]
+;
+entry:
+  br label %for.body
+
+for.body:                                         ; preds = %for.body, %entry
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+  %accum = phi i64 [ 0, %entry ], [ %add, %for.body ]
+  %gep.a = phi ptr [ %a, %entry ], [ %gep.a.next, %for.body ]
+  %gep.b = phi ptr [ %b, %entry ], [ %gep.b.next, %for.body ]
+  %gep.a.next = getelementptr inbounds nuw i8, ptr %gep.a, i64 1
+  %load.a = load i8, ptr %gep.a, align 1
+  %ext.a = zext i8 %load.a to i64
+  %gep.b.next = getelementptr inbounds nuw i8, ptr %gep.b, i64 1
+  %load.b = load i8, ptr %gep.b, align 1
+  %ext.b = zext i8 %load.b to i64
+  %mul = mul nuw nsw i64 %ext.b, %ext.a
+  %add = add nsw i64 %mul, %accum
+  %iv.next = add i64 %iv, 1
+  %exitcond.not = icmp eq i64 %iv.next, 1024
+  br i1 %exitcond.not, label %for.exit, label %for.body
+
+for.exit:                        ; preds = %for.body
+  ret i64 %add
+}
+
+define i64 @not_dotp_i16_to_i64_has_neon_dotprod(ptr readonly %a, ptr readonly %b) #1 {
+; CHECK-INTERLEAVE1-LABEL: define i64 @not_dotp_i16_to_i64_has_neon_dotprod(
+; CHECK-INTERLEAVE1-SAME: ptr readonly [[A:%.*]], ptr readonly [[B:%.*]]) #[[ATTR1]] {
+; CHECK-INTERLEAVE1-NEXT:  entry:
+; CHECK-INTERLEAVE1-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-INTERLEAVE1-NEXT:    [[TMP9:%.*]] = mul i64 [[TMP7]], 2
+; CHECK-INTERLEAVE1-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK-INTERLEAVE1:       vector.ph:
+; CHECK-INTERLEAVE1-NEXT:    [[TMP11:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-INTERLEAVE1-NEXT:    [[TMP12:%.*]] = mul i64 [[TMP11]], 2
+; CHECK-INTERLEAVE1-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP12]]
+; CHECK-INTERLEAVE1-NEXT:    [[TMP10:%.*]] = sub i64 1024, [[N_MOD_VF]]
+; CHECK-INTERLEAVE1-NEXT:    [[TMP14:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-INTERLEAVE1-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP14]], 2
+; CHECK-INTERLEAVE1-NEXT:    [[TMP20:%.*]] = mul i64 [[TMP10]], 2
+; CHECK-INTERLEAVE1-NEXT:    [[TMP0:%.*]] = getelementptr i8, ptr [[A]], i64 [[TMP20]]
+; CHECK-INTERLEAVE1-NEXT:    [[TMP8:%.*]] = mul i64 [[TMP10]], 2
+; CHECK-INTERLEAVE1-NEXT:    [[TMP1:%.*]] = getelementptr i8, ptr [[B]], i64 [[TMP8]]
+; CHECK-INTERLEAVE1-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK-INTERLEAVE1:       vector.body:
+; CHECK-INTERLEAVE1-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-INTERLEAVE1-NEXT:    [[VEC_PHI:%.*]] = phi <vscale x 2 x i64> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP17:%.*]], [[VECTOR_BODY]] ]
+; CHECK-INTERLEAVE1-NEXT:    [[OFFSET_IDX:%.*]] = mul i64 [[INDEX]], 2
+; CHECK-INTERLEAVE1-NEXT:    [[TMP2:%.*]] = add i64 [[OFFSET_IDX]], 0
+; CHECK-INTERLEAVE1-NEXT:    [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[A]], i64 [[TMP2]]
+; CHECK-INTERLEAVE1-NEXT:    [[OFFSET_IDX1:%.*]] = mul i64 [[INDEX]], 2
+; CHECK-INTERLEAVE1-NEXT:    [[TMP3:%.*]] = add i64 [[OFFSET_IDX1]], 0
+; CHECK-INTERLEAVE1-NEXT:    [[NEXT_GEP2:%.*]] = getelementptr i8, ptr [[B]], i64 [[TMP3]]
+; CHECK-INTERLEAVE1-NEXT:    [[TMP4:%.*]] = getelementptr i16, ptr [[NEXT_GEP]], i32 0
+; CHECK-INTERLEAVE1-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 2 x i16>, ptr [[TMP4]], align 2
+; CHECK-INTERLEAVE1-NEXT:    [[TMP13:%.*]] = zext <vscale x 2 x i16> [[WIDE_LOAD]] to <vscale x 2 x i64>
+; CHECK-INTERLEAVE1-NEXT:    [[TMP6:%.*]] = getelementptr i16, ptr [[NEXT_GEP2]], i32 0
+; CHECK-INTERLEAVE1-NEXT:    [[WIDE_LOAD3:%.*]] = load <vscale x 2 x i16>, ptr [[TMP6]], align 2
+; CHECK-INTERLEAVE1-NEXT:    [[TMP15:%.*]] = zext <vscale x 2 x i16> [[WIDE_LOAD3]] to <vscale x 2 x i64>
+; CHECK-INTERLEAVE1-NEXT:    [[TMP16:%.*]] = mul nuw nsw <vscale x 2 x i64> [[TMP15]], [[TMP13]]
+; CHECK-INTERLEAVE1-NEXT:    [[TMP17]] = add <vscale x 2 x i64> [[TMP16]], [[VEC_PHI]]
+; CHECK-INTERLEAVE1-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
+; CHECK-INTERLEAVE1-NEXT:    [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[TMP10]]
+; CHECK-INTERLEAVE1-NEXT:    br i1 [[TMP18]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK-INTERLEAVE1:       middle.block:
+; CHECK-INTERLEAVE1-NEXT:    [[TMP19:%.*]] = call i64 @llvm.vector.reduce.add.nxv2i64(<vscale x 2 x i64> [[TMP17]])
+; CHECK-INTERLEAVE1-NEXT:    [[CMP_N:%.*]] = icmp eq i64 1024, [[TMP10]]
+; CHECK-INTERLEAVE1-NEXT:    br i1 [[CMP_N]], label [[FOR_EXIT:%.*]], label [[SCALAR_PH]]
+; CHECK-INTERLEAVE1:       scalar.ph:
+; CHECK-INTERLEAVE1-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i64 [ [[TMP10]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-INTERLEAVE1-NEXT:    [[BC_MERGE_RDX1:%.*]] = phi i64 [ [[TMP19]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ]
+; CHECK-INTERLEAVE1-NEXT:    [[BC_RESUME_VAL4:%.*]] = phi ptr [ [[TMP0]], [[MIDDLE_BLOCK]] ], [ [[A]], [[ENTRY]] ]
+; CHECK-INTERLEAVE1-NEXT:    [[BC_RESUME_VAL5:%.*]] = phi ptr [ [[TMP1]], [[MIDDLE_BLOCK]] ], [ [[B]], [[ENTRY]] ]
+; CHECK-INTERLEAVE1-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK-INTERLEAVE1:       for.body:
+; CHECK-INTERLEAVE1-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
+; CHECK-INTERLEAVE1-NEXT:    [[ACCUM:%.*]] = phi i64 [ [[BC_MERGE_RDX1]], [[SCALAR_PH]] ], [ [[ADD:%.*]], [[FOR_BODY]] ]
+; CHECK-INTERLEAVE1-NEXT:    [[GEP_A:%.*]] = phi ptr [ [[BC_RESUME_VAL4]], [[SCALAR_PH]] ], [ [[GEP_A_NEXT:%.*]], [[FOR_BODY]] ]
+; CHECK-INTERLEAVE1-NEXT:    [[GEP_B:%.*]] = phi ptr [ [[BC_RESUME_VAL5]], [[SCALAR_PH]] ], [ [[GEP_B_NEXT:%.*]], [[FOR_BODY]] ]
+; CHECK-INTERLEAVE1-NEXT:    [[GEP_A_NEXT]] = getelementptr inbounds nuw i16, ptr [[GEP_A]], i64 1
+; CHECK-INTERLEAVE1-NEXT:    [[LOAD_A:%.*]] = load i16, ptr [[GEP_A]], align 2
+; CHECK-INTERLEAVE1-NEXT:    [[EXT_A:%.*]] = zext i16 [[LOAD_A]] to i64
+; CHECK-INTERLEAVE1-NEXT:    [[GEP_B_NEXT]] = getelementptr inbounds nuw i16, ptr [[GEP_B]], i64 1
+; CHECK-INTERLEAVE1-NEXT:    [[LOAD_B:%.*]] = load i16, ptr [[GEP_B]], align 2
+; CHECK-INTERLEAVE1-NEXT:    [[EXT_B:%.*]] = zext i16 [[LOAD_B]] to i64
+; CHECK-INTERLEAVE1-NEXT:    [[MUL:%.*]] = mul nuw nsw i64 [[EXT_B]], [[EXT_A]]
+; CHECK-INTERLEAVE1-NEXT:    [[ADD]] = add nsw i64 [[MUL]], [[ACCUM]]
+; CHECK-INTERLEAVE1-NEXT:    [[IV_NEXT]] = add i64 [[IV]], 1
+; CHECK-INTERLEAVE1-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], 1024
+; CHECK-INTERLEAVE1-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_EXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
+; CHECK-INTERLEAVE1:       for.exit:
+; CHECK-INTERLEAVE1-NEXT:    [[ADD_LCSSA:%.*]] = phi i64 [ [[ADD]], [[FOR_BODY]] ], [ [[TMP19]], [[MIDDLE_BLOCK]] ]
+; CHECK-INTERLEAVE1-NEXT:    ret i64 [[ADD_LCSSA]]
+;
+; CHECK-INTERLEAVED-LABEL: define i64 @not_dotp_i16_to_i64_has_neon_dotprod(
+; CHECK-INTERLEAVED-SAME: ptr readonly [[A:%.*]], ptr readonly [[B:%.*]]) #[[ATTR1]] {
+; CHECK-INTERLEAVED-NEXT:  entry:
+; CHECK-INTERLEAVED-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-INTERLEAVED-NEXT:    [[TMP9:%.*]] = mul i64 [[TMP7]], 4
+; CHECK-INTERLEAVED-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK-INTERLEAVED:       vector.ph:
+; CHECK-INTERLEAVED-NEXT:    [[TMP10:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-INTERLEAVED-NEXT:    [[TMP11:%.*]] = mul i64 [[TMP10]], 4
+; CHECK-INTERLEAVED-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP11]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP15:%.*]] = sub i64 1024, [[N_MOD_VF]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP12:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-INTERLEAVED-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP12]], 4
+; CHECK-INTERLEAVED-NEXT:    [[TMP6:%.*]] = mul i64 [[TMP15]], 2
+; CHECK-INTERLEAVED-NEXT:    [[TMP0:%.*]] = getelementptr i8, ptr [[A]], i64 [[TMP6]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP18:%.*]] = mul i64 [[TMP15]], 2
+; CHECK-INTERLEAVED-NEXT:    [[TMP1:%.*]] = getelementptr i8, ptr [[B]], i64 [[TMP18]]
+; CHECK-INTERLEAVED-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK-INTERLEAVED:       vector.body:
+; CHECK-INTERLEAVED-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-INTERLEAVED-NEXT:    [[VEC_PHI:%.*]] = phi <vscale x 2 x i64> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP26:%.*]], [[VECTOR_BODY]] ]
+; CHECK-INTERLEAVED-NEXT:    [[VEC_PHI1:%.*]] = phi <vscale x 2 x i64> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP27:%.*]], [[VECTOR_BODY]] ]
+; CHECK-INTERLEAVED-NEXT:    [[OFFSET_IDX:%.*]] = mul i64 [[INDEX]], 2
+; CHECK-INTERLEAVED-NEXT:    [[TMP2:%.*]] = add i64 [[OFFSET_IDX]], 0
+; CHECK-INTERLEAVED-NEXT:    [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[A]], i64 [[TMP2]]
+; CHECK-INTERLEAVED-NEXT:    [[OFFSET_IDX2:%.*]] = mul i64 [[INDEX]], 2
+; CHECK-INTERLEAVED-NEXT:    [[TMP3:%.*]] = add i64 [[OFFSET_IDX2]], 0
+; CHECK-INTERLEAVED-NEXT:    [[NEXT_GEP3:%.*]] = getelementptr i8, ptr [[B]], i64 [[TMP3]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP4:%.*]] = getelementptr i16, ptr [[NEXT_GEP]], i32 0
+; CHECK-INTERLEAVED-NEXT:    [[TMP13:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-INTERLEAVED-NEXT:    [[TMP14:%.*]] = mul i64 [[TMP13]], 2
+; CHECK-INTERLEAVED-NEXT:    [[TMP30:%.*]] = getelementptr i16, ptr [[NEXT_GEP]], i64 [[TMP14]]
+; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 2 x i16>, ptr [[TMP4]], align 2
+; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD4:%.*]] = load <vscale x 2 x i16>, ptr [[TMP30]], align 2
+; CHECK-INTERLEAVED-NEXT:    [[TMP16:%.*]] = zext <vscale x 2 x i16> [[WIDE_LOAD]] to <vscale x 2 x i64>
+; CHECK-INTERLEAVED-NEXT:    [[TMP17:%.*]] = zext <vscale x 2 x i16> [[WIDE_LOAD4]] to <vscale x 2 x i64>
+; CHECK-INTERLEAVED-NEXT:    [[TMP8:%.*]] = getelementptr i16, ptr [[NEXT_GEP3]], i32 0
+; CHECK-INTERLEAVED-NEXT:    [[TMP19:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-INTERLEAVED-NEXT:    [[TMP20:%.*]] = mul i64 [[TMP19]], 2
+; CHECK-INTERLEAVED-NEXT:    [[TMP21:%.*]] = getelementptr i16, ptr [[NEXT_GEP3]], i64 [[TMP20]]
+; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD5:%.*]] = load <vscale x 2 x i16>, ptr [[TMP8]], align 2
+; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD6:%.*]] = load <vscale x 2 x i16>, ptr [[TMP21]], align 2
+; CHECK-INTERLEAVED-NEXT:    [[TMP22:%.*]] = zext <vscale x 2 x i16> [[WIDE_LOAD5]] to <vscale x 2 x i64>
+; CHECK-INTERLEAVED-NEXT:    [[TMP23:%.*]] = zext <vscale x 2 x i16> [[WIDE_LOAD6]] to <vscale x 2 x i64>
+; CHECK-INTERLEAVED-NEXT:    [[TMP24:%.*]] = mul nuw nsw <vscale x 2 x i64> [[TMP22]], [[TMP16]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP25:%.*]] = mul nuw nsw <vscale x 2 x i64> [[TMP23]], [[TMP17]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP26]] = add <vscale x 2 x i64> [[TMP24]], [[VEC_PHI]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP27]] = add <vscale x 2 x i64> [[TMP25]], [[VEC_PHI1]]
+; CHECK-INTERLEAVED-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP28:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[TMP15]]
+; CHECK-INTERLEAVED-NEXT:    br i1 [[TMP28]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK-INTERLEAVED:       middle.block:
+; CHECK-INTERLEAVED-NEXT:    [[BIN_RDX:%.*]] = add <vscale x 2 x i64> [[TMP27]], [[TMP26]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP29:%.*]] = call i64 @llvm.vector.reduce.add.nxv2i64(<vscale x 2 x i64> [[BIN_RDX]])
+; CHECK-INTERLEAVED-NEXT:    [[CMP_N:%.*]] = icmp eq i64 1024, [[TMP15]]
+; CHECK-INTERLEAVED-NEXT:    br i1 [[CMP_N]], label [[FOR_EXIT:%.*]], label [[SCALAR_PH]]
+; CHECK-INTERLEAVED:       scalar.ph:
+; CHECK-INTERLEAVED-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i64 [ [[TMP15]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-INTERLEAVED-NEXT:    [[BC_MERGE_RDX1:%.*]] = phi i64 [ [[TMP29]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ]
+; CHECK-INTERLEAVED-NEXT:    [[BC_RESUME_VAL8:%.*]] = phi ptr [ [[TMP0]], [[MIDDLE_BLOCK]] ], [ [[A]], [[ENTRY]] ]
+; CHECK-INTERLEAVED-NEXT:    [[BC_RESUME_VAL9:%.*]] = phi ptr [ [[TMP1]], [[MIDDLE_BLOCK]] ], [ [[B]], [[ENTRY]] ]
+; CHECK-INTERLEAVED-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK-INTERLEAVED:       for.body:
+; CHECK-INTERLEAVED-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
+; CHECK-INTERLEAVED-NEXT:    [[ACCUM:%.*]] = phi i64 [ [[BC_MERGE_RDX1]], [[SCALAR_PH]] ], [ [[ADD:%.*]], [[FOR_BODY]] ]
+; CHECK-INTERLEAVED-NEXT:    [[GEP_A:%.*]] = phi ptr [ [[BC_RESUME_VAL8]], [[SCALAR_PH]] ], [ [[GEP_A_NEXT:%.*]], [[FOR_BODY]] ]
+; CHECK-INTERLEAVED-NEXT:    [[GEP_B:%.*]] = phi ptr [ [[BC_RESUME_VAL9]], [[SCALAR_PH]] ], [ [[GEP_B_NEXT:%.*]], [[FOR_BODY]] ]
+; CHECK-INTERLEAVED-NEXT:    [[GEP_A_NEXT]] = getelementptr inbounds nuw i16, ptr [[GEP_A]], i64 1
+; CHECK-INTERLEAVED-NEXT:    [[LOAD_A:%.*]] = load i16, ptr [[GEP_A]], align 2
+; CHECK-INTERLEAVED-NEXT:    [[EXT_A:%.*]] = zext i16 [[LOAD_A]] to i64
+; CHECK-INTERLEAVED-NEXT:    [[GEP_B_NEXT]] = getelementptr inbounds nuw i16, ptr [[GEP_B]], i64 1
+; CHECK-INTERLEAVED-NEXT:    [[LOAD_B:%.*]] = load i16, ptr [[GEP_B]], align 2
+; CHECK-INTERLEAVED-NEXT:    [[EXT_B:%.*]] = zext i16 [[LOAD_B]] to i64
+; CHECK-INTERLEAVED-NEXT:    [[MUL:%.*]] = mul nuw nsw i64 [[EXT_B]], [[EXT_A]]
+; CHECK-INTERLEAVED-NEXT:    [[ADD]] = add nsw i64 [[MUL]], [[ACCUM]]
+; CHECK-INTERLEAVED-NEXT:    [[IV_NEXT]] = add i64 [[IV]], 1
+; CHECK-INTERLEAVED-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], 1024
+; CHECK-INTERLEAVED-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_EXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
+; CHECK-INTERLEAVED:       for.exit:
+; CHECK-INTERLEAVED-NEXT:    [[ADD_LCSSA:%.*]] = phi i64 [ [[ADD]], [[FOR_BODY]] ], [ [[TMP29]], [[MIDDLE_BLOCK]] ]
+; CHECK-INTERLEAVED-NEXT:    ret i64 [[ADD_LCSSA]]
+;
+; CHECK-MAXBW-LABEL: define i64 @not_dotp_i16_to_i64_has_neon_dotprod(
+; CHECK-MAXBW-SAME: ptr readonly [[A:%.*]], ptr readonly [[B:%.*]]) #[[ATTR1]] {
+; CHECK-MAXBW-NEXT:  entry:
+; CHECK-MAXBW-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-MAXBW-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 4
+; CHECK-MAXBW-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK-MAXBW:       vector.ph:
+; CHECK-MAXBW-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-MAXBW-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 4
+; CHECK-MAXBW-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]]
+; CHECK-MAXBW-NEXT:    [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
+; CHECK-MAXBW-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-MAXBW-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 4
+; CHECK-MAXBW-NEXT:    [[TMP6:%.*]] = mul i64 [[N_VEC]], 2
+; CHECK-MAXBW-NEXT:    [[TMP7:%.*]] = getelementptr i8, ptr [[A]], i64 [[TMP6]]
+; CHECK-MAXBW-NEXT:    [[TMP8:%.*]] = mul i64 [[N_VEC]], 2
+; CHECK-MAXBW-NEXT:    [[TMP9:%.*]] = getelementptr i8, ptr [[B]], i64 [[TMP8]]
+; CHECK-MAXBW-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK-MAXBW:       vector.body:
+; CHECK-MAXBW-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-MAXBW-NEXT:    [[VEC_PHI:%.*]] = phi <vscale x 4 x i64> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP17:%.*]], [[VECTOR_BODY]] ]
+; CHECK-MAXBW-NEXT:    [[OFFSET_IDX:%.*]] = mul i64 [[INDEX]], 2
+; CHECK-MAXBW-NEXT:    [[TMP10:%.*]] = add i64 [[OFFSET_IDX]], 0
+; CHECK-MAXBW-NEXT:    [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[A]], i64 [[TMP10]]
+; CHECK-MAXBW-NEXT:    [[OFFSET_IDX1:%.*]] = mul i64 [[INDEX]], 2
+; CHECK-MAXBW-NEXT:    [[TMP11:%.*]] = add i64 [[OFFSET_IDX1]], 0
+; CHECK-MAXBW-NEXT:    [[NEXT_GEP2:%.*]] = getelementptr i8, ptr [[B]], i64 [[TMP11]]
+; CHECK-MAXBW-NEXT:    [[TMP12:%.*]] = getelementptr i16, ptr [[NEXT_GEP]], i32 0
+; CHECK-MAXBW-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 4 x i16>, ptr [[TMP12]], align 2
+; CHECK-MAXBW-NEXT:    [[TMP13:%.*]] = zext <vscale x 4 x i16> [[WIDE_LOAD]] to <vscale x 4 x i64>
+; CHECK-MAXBW-NEXT:    [[TMP14:%.*]] = getelementptr i16, ptr [[NEXT_GEP2]], i32 0
+; CHECK-MAXBW-NEXT:    [[WIDE_LOAD3:%.*]] = load <vscale x 4 x i16>, ptr [[TMP14]], align 2
+; CHECK-MAXBW-NEXT:    [[TMP15:%.*]] = zext <vscale x 4 x i16> [[WIDE_LOAD3]] to <vscale x 4 x i64>
+; CHECK-MAXBW-NEXT:    [[TMP16:%.*]] = mul nuw nsw <vscale x 4 x i64> [[TMP15]], [[TMP13]]
+; CHECK-MAXBW-NEXT:    [[TMP17]] = add <vscale x 4 x i64> [[TMP16]], [[VEC_PHI]]
+; CHECK-MAXBW-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
+; CHECK-MAXBW-NEXT:    [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-MAXBW-NEXT:    br i1 [[TMP18]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK-MAXBW:       middle.block:
+; CHECK-MAXBW-NEXT:    [[TMP19:%.*]] = call i64 @llvm.vector.reduce.add.nxv4i64(<vscale x 4 x i64> [[TMP17]])
+; CHECK-MAXBW-NEXT:    [[CMP_N:%.*]] = icmp eq i64 1024, [[N_VEC]]
+; CHECK-MAXBW-NEXT:    br i1 [[CMP_N]], label [[FOR_EXIT:%.*]], label [[SCALAR_PH]]
+; CHECK-MAXBW:       scalar.ph:
+; CHECK-MAXBW-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-MAXBW-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i64 [ [[TMP19]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ]
+; CHECK-MAXBW-NEXT:    [[BC_RESUME_VAL4:%.*]] = phi ptr [ [[TMP7]], [[MIDDLE_BLOCK]] ], [ [[A]], [[ENTRY]] ]
+; CHECK-MAXBW-NEXT:    [[BC_RESUME_VAL5:%.*]] = phi ptr [ [[TMP9]], [[MIDDLE_BLOCK]] ], [ [[B]], [[ENTRY]] ]
+; CHECK-MAXBW-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK-MAXBW:       for.body:
+; CHECK-MAXBW-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
+; CHECK-MAXBW-NEXT:    [[ACCUM:%.*]] = phi i64 [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[ADD:%.*]], [[FOR_BODY]] ]
+; CHECK-MAXBW-NEXT:    [[GEP_A:%.*]] = phi ptr [ [[BC_RESUME_VAL4]], [[SCALAR_PH]] ], [ [[GEP_A_NEXT:%.*]], [[FOR_BODY]] ]
+; CHECK-MAXBW-NEXT:    [[GEP_B:%.*]] = phi ptr [ [[BC_RESUME_VAL5]], [[SCALAR_PH]] ], [ [[GEP_B_NEXT:%.*]], [[FOR_BODY]] ]
+; CHECK-MAXBW-NEXT:    [[GEP_A_NEXT]] = getelementptr inbounds nuw i16, ptr [[GEP_A]], i64 1
+; CHECK-MAXBW-NEXT:    [[LOAD_A:%.*]] = load i16, ptr [[GEP_A]], align 2
+; CHECK-MAXBW-NEXT:    [[EXT_A:%.*]] = zext i16 [[LOAD_A]] to i64
+; CHECK-MAXBW-NEXT:    [[GEP_B_NEXT]] = getelementptr inbounds nuw i16, ptr [[GEP_B]], i64 1
+; CHECK-MAXBW-NEXT:    [[LOAD_B:%.*]] = load i16, ptr [[GEP_B]], align 2
+; CHECK-MAXBW-NEXT:    [[EXT_B:%.*]] = zext i16 [[LOAD_B]] to i64
+; CHECK-MAXBW-NEXT:    [[MUL:%.*]] = mul nuw nsw i64 [[EXT_B]], [[EXT_A]]
+; CHECK-MAXBW-NEXT:    [[ADD]] = add nsw i64 [[MUL]], [[ACCUM]]
+; CHECK-MAXBW-NEXT:    [[IV_NEXT]] = add i64 [[IV]], 1
+; CHECK-MAXBW-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], 1024
+; CHECK-MAXBW-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_EXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
+; CHECK-MAXBW:       for.exit:
+; CHECK-MAXBW-NEXT:    [[ADD_LCSSA:%.*]] = phi i64 [ [[ADD]], [[FOR_BODY]] ], [ [[TMP19]], [[MIDDLE_BLOCK]] ]
+; CHECK-MAXBW-NEXT:    ret i64 [[ADD_LCSSA]]
+;
+entry:
+  br label %for.body
+
+for.body:                                         ; preds = %for.body, %entry
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+  %accum = phi i64 [ 0, %entry ], [ %add, %for.body ]
+  %gep.a = phi ptr [ %a, %entry ], [ %gep.a.next, %for.body ]
+  %gep.b = phi ptr [ %b, %entry ], [ %gep.b.next, %for.body ]
+  %gep.a.next = getelementptr inbounds nuw i16, ptr %gep.a, i64 1
+  %load.a = load i16, ptr %gep.a, align 2
+  %ext.a = zext i16 %load.a to i64
+  %gep.b.next = getelementptr inbounds nuw i16, ptr %gep.b, i64 1
+  %load.b = load i16, ptr %gep.b, align 2
+  %ext.b = zext i16 %load.b to i64
+  %mul = mul nuw nsw i64 %ext.b, %ext.a
+  %add = add nsw i64 %mul, %accum
+  %iv.next = add i64 %iv, 1
+  %exitcond.not = icmp eq i64 %iv.next, 1024
+  br i1 %exitcond.not, label %for.exit, label %for.body
+
+for.exit:                        ; preds = %for.body
+  ret i64 %add
+}
+
 define i32 @not_dotp_different_types(ptr %a, ptr %b) #0 {
 ; CHECK-INTERLEAVE1-LABEL: define i32 @not_dotp_different_types(
 ; CHECK-INTERLEAVE1-SAME: ptr [[A:%.*]], ptr [[B:%.*]]) #[[ATTR0]] {
@@ -297,16 +771,16 @@ define i32 @not_dotp_different_types(ptr %a, ptr %b) #0 {
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP69]] = add <16 x i32> [[TMP68]], [[VEC_PHI]]
 ; CHECK-INTERLEAVE1-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP70:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
-; CHECK-INTERLEAVE1-NEXT:    br i1 [[TMP70]], label [[VEC_EPILOG_ITER_CHECK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK-INTERLEAVE1-NEXT:    br i1 [[TMP70]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
 ; CHECK-INTERLEAVE1:       middle.block:
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP71:%.*]] = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> [[TMP69]])
 ; CHECK-INTERLEAVE1-NEXT:    br i1 true, label [[FOR_EXIT:%.*]], label [[SCALAR_PH]]
 ; CHECK-INTERLEAVE1:       scalar.ph:
-; CHECK-INTERLEAVE1-NEXT:    [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ 1024, [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[VECTOR_MAIN_LOOP_ITER_CHECK:%.*]] ]
-; CHECK-INTERLEAVE1-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP71]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
+; CHECK-INTERLEAVE1-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 1024, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-INTERLEAVE1-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP71]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ]
 ; CHECK-INTERLEAVE1-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK-INTERLEAVE1:       for.body:
-; CHECK-INTERLEAVE1-NEXT:    [[IV:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
+; CHECK-INTERLEAVE1-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
 ; CHECK-INTERLEAVE1-NEXT:    [[ACCUM:%.*]] = phi i32 [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[ADD:%.*]], [[FOR_BODY]] ]
 ; CHECK-INTERLEAVE1-NEXT:    [[GEP_A:%.*]] = getelementptr i8, ptr [[A]], i64 [[IV]]
 ; CHECK-INTERLEAVE1-NEXT:    [[LOAD_A:%.*]] = load i8, ptr [[GEP_A]], align 1
@@ -318,9 +792,9 @@ define i32 @not_dotp_different_types(ptr %a, ptr %b) #0 {
 ; CHECK-INTERLEAVE1-NEXT:    [[ADD]] = add i32 [[MUL]], [[ACCUM]]
 ; CHECK-INTERLEAVE1-NEXT:    [[IV_NEXT]] = add i64 [[IV]], 1
 ; CHECK-INTERLEAVE1-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], 1024
-; CHECK-INTERLEAVE1-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_EXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
+; CHECK-INTERLEAVE1-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_EXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
 ; CHECK-INTERLEAVE1:       for.exit:
-; CHECK-INTERLEAVE1-NEXT:    [[ADD_LCSSA:%.*]] = phi i32 [ [[ADD]], [[FOR_BODY]] ], [ [[TMP71]], [[VEC_EPILOG_ITER_CHECK]] ]
+; CHECK-INTERLEAVE1-NEXT:    [[ADD_LCSSA:%.*]] = phi i32 [ [[ADD]], [[FOR_BODY]] ], [ [[TMP71]], [[MIDDLE_BLOCK]] ]
 ; CHECK-INTERLEAVE1-NEXT:    ret i32 [[ADD_LCSSA]]
 ;
 ; CHECK-INTERLEAVED-LABEL: define i32 @not_dotp_different_types(
@@ -476,7 +950,7 @@ define i32 @not_dotp_different_types(ptr %a, ptr %b) #0 {
 ; CHECK-INTERLEAVED-NEXT:    [[TMP138]] = add <16 x i32> [[TMP136]], [[VEC_PHI1]]
 ; CHECK-INTERLEAVED-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 32
 ; CHECK-INTERLEAVED-NEXT:    [[TMP141:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
-; CHECK-INTERLEAVED-NEXT:    br i1 [[TMP141]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK-INTERLEAVED-NEXT:    br i1 [[TMP141]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
 ; CHECK-INTERLEAVED:       middle.block:
 ; CHECK-INTERLEAVED-NEXT:    [[BIN_RDX:%.*]] = add <16 x i32> [[TMP138]], [[TMP137]]
 ; CHECK-INTERLEAVED-NEXT:    [[TMP142:%.*]] = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> [[BIN_RDX]])
@@ -498,7 +972,7 @@ define i32 @not_dotp_different_types(ptr %a, ptr %b) #0 {
 ; CHECK-INTERLEAVED-NEXT:    [[ADD]] = add i32 [[MUL]], [[ACCUM]]
 ; CHECK-INTERLEAVED-NEXT:    [[IV_NEXT]] = add i64 [[IV]], 1
 ; CHECK-INTERLEAVED-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], 1024
-; CHECK-INTERLEAVED-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_EXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
+; CHECK-INTERLEAVED-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_EXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
 ; CHECK-INTERLEAVED:       for.exit:
 ; CHECK-INTERLEAVED-NEXT:    [[ADD_LCSSA:%.*]] = phi i32 [ [[ADD]], [[FOR_BODY]] ], [ [[TMP142]], [[MIDDLE_BLOCK]] ]
 ; CHECK-INTERLEAVED-NEXT:    ret i32 [[ADD_LCSSA]]
@@ -585,7 +1059,7 @@ define i32 @not_dotp_different_types(ptr %a, ptr %b) #0 {
 ; CHECK-MAXBW-NEXT:    [[TMP138]] = add <16 x i32> [[TMP136]], [[VEC_PHI1]]
 ; CHECK-MAXBW-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
 ; CHECK-MAXBW-NEXT:    [[TMP70:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
-; CHECK-MAXBW-NEXT:    br i1 [[TMP70]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK-MAXBW-NEXT:    br i1 [[TMP70]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
 ; CHECK-MAXBW:       middle.block:
 ; CHECK-MAXBW-NEXT:    [[TMP71:%.*]] = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> [[TMP138]])
 ; CHECK-MAXBW-NEXT:    br i1 true, label [[FOR_EXIT:%.*]], label [[SCALAR_PH]]
@@ -606,7 +1080,7 @@ define i32 @not_dotp_different_types(ptr %a, ptr %b) #0 {
 ; CHECK-MAXBW-NEXT:    [[ADD]] = add i32 [[MUL]], [[ACCUM]]
 ; CHECK-MAXBW-NEXT:    [[IV_NEXT]] = add i64 [[IV]], 1
 ; CHECK-MAXBW-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], 1024
-; CHECK-MAXBW-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_EXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
+; CHECK-MAXBW-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_EXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
 ; CHECK-MAXBW:       for.exit:
 ; CHECK-MAXBW-NEXT:    [[ADD_LCSSA:%.*]] = phi i32 [ [[ADD]], [[FOR_BODY]] ], [ [[TMP71]], [[MIDDLE_BLOCK]] ]
 ; CHECK-MAXBW-NEXT:    ret i32 [[ADD_LCSSA]]
@@ -669,7 +1143,7 @@ define i32 @not_dotp_not_loop_carried(ptr %a, ptr %b) #0 {
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP18:%.*]] = add <vscale x 8 x i32> [[TMP16]], [[TMP17]]
 ; CHECK-INTERLEAVE1-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP19:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-INTERLEAVE1-NEXT:    br i1 [[TMP19]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK-INTERLEAVE1-NEXT:    br i1 [[TMP19]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
 ; CHECK-INTERLEAVE1:       middle.block:
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP20:%.*]] = call i32 @llvm.vscale.i32()
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP21:%.*]] = mul i32 [[TMP20]], 8
@@ -698,7 +1172,7 @@ define i32 @not_dotp_not_loop_carried(ptr %a, ptr %b) #0 {
 ; CHECK-INTERLEAVE1-NEXT:    [[ADD:%.*]] = add i32 [[MUL]], [[ACCUM]]
 ; CHECK-INTERLEAVE1-NEXT:    [[IV_NEXT]] = add i64 [[IV]], 1
 ; CHECK-INTERLEAVE1-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], 1024
-; CHECK-INTERLEAVE1-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_EXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
+; CHECK-INTERLEAVE1-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_EXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]]
 ; CHECK-INTERLEAVE1:       for.exit:
 ; CHECK-INTERLEAVE1-NEXT:    [[ADD_LCSSA:%.*]] = phi i32 [ [[ADD]], [[FOR_BODY]] ], [ [[TMP23]], [[MIDDLE_BLOCK]] ]
 ; CHECK-INTERLEAVE1-NEXT:    ret i32 [[ADD_LCSSA]]
@@ -749,7 +1223,7 @@ define i32 @not_dotp_not_loop_carried(ptr %a, ptr %b) #0 {
 ; CHECK-INTERLEAVED-NEXT:    [[TMP27:%.*]] = add <vscale x 8 x i32> [[TMP25]], [[TMP26]]
 ; CHECK-INTERLEAVED-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
 ; CHECK-INTERLEAVED-NEXT:    [[TMP28:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-INTERLEAVED-NEXT:    br i1 [[TMP28]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK-INTERLEAVED-NEXT:    br i1 [[TMP28]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
 ; CHECK-INTERLEAVED:       middle.block:
 ; CHECK-INTERLEAVED-NEXT:    [[TMP29:%.*]] = call i32 @llvm.vscale.i32()
 ; CHECK-INTERLEAVED-NEXT:    [[TMP30:%.*]] = mul i32 [[TMP29]], 8
@@ -778,7 +1252,7 @@ define i32 @not_dotp_not_loop_carried(ptr %a, ptr %b) #0 {
 ; CHECK-INTERLEAVED-NEXT:    [[ADD:%.*]] = add i32 [[MUL]], [[ACCUM]]
 ; CHECK-INTERLEAVED-NEXT:    [[IV_NEXT]] = add i64 [[IV]], 1
 ; CHECK-INTERLEAVED-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], 1024
-; CHECK-INTERLEAVED-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_EXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
+; CHECK-INTERLEAVED-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_EXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]]
 ; CHECK-INTERLEAVED:       for.exit:
 ; CHECK-INTERLEAVED-NEXT:    [[ADD_LCSSA:%.*]] = phi i32 [ [[ADD]], [[FOR_BODY]] ], [ [[TMP32]], [[MIDDLE_BLOCK]] ]
 ; CHECK-INTERLEAVED-NEXT:    ret i32 [[ADD_LCSSA]]
@@ -818,7 +1292,7 @@ define i32 @not_dotp_not_loop_carried(ptr %a, ptr %b) #0 {
 ; CHECK-MAXBW-NEXT:    [[TMP27:%.*]] = add <vscale x 8 x i32> [[TMP25]], [[TMP26]]
 ; CHECK-MAXBW-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
 ; CHECK-MAXBW-NEXT:    [[TMP28:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-MAXBW-NEXT:    br i1 [[TMP28]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK-MAXBW-NEXT:    br i1 [[TMP28]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
 ; CHECK-MAXBW:       middle.block:
 ; CHECK-MAXBW-NEXT:    [[TMP20:%.*]] = call i32 @llvm.vscale.i32()
 ; CHECK-MAXBW-NEXT:    [[TMP21:%.*]] = mul i32 [[TMP20]], 8
@@ -847,7 +1321,7 @@ define i32 @not_dotp_not_loop_carried(ptr %a, ptr %b) #0 {
 ; CHECK-MAXBW-NEXT:    [[ADD:%.*]] = add i32 [[MUL]], [[ACCUM]]
 ; CHECK-MAXBW-NEXT:    [[IV_NEXT]] = add i64 [[IV]], 1
 ; CHECK-MAXBW-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], 1024
-; CHECK-MAXBW-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_EXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
+; CHECK-MAXBW-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_EXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]]
 ; CHECK-MAXBW:       for.exit:
 ; CHECK-MAXBW-NEXT:    [[ADD_LCSSA:%.*]] = phi i32 [ [[ADD]], [[FOR_BODY]] ], [ [[TMP29]], [[MIDDLE_BLOCK]] ]
 ; CHECK-MAXBW-NEXT:    ret i32 [[ADD_LCSSA]]
@@ -909,7 +1383,7 @@ define i32 @not_dotp_not_phi(ptr %a, ptr %b) #0 {
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP17]] = add <vscale x 8 x i32> [[TMP16]], [[TMP15]]
 ; CHECK-INTERLEAVE1-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-INTERLEAVE1-NEXT:    br i1 [[TMP18]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+; CHECK-INTERLEAVE1-NEXT:    br i1 [[TMP18]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
 ; CHECK-INTERLEAVE1:       middle.block:
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP19:%.*]] = call i32 @llvm.vscale.i32()
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP20:%.*]] = mul i32 [[TMP19]], 8
@@ -938,7 +1412,7 @@ define i32 @not_dotp_not_phi(ptr %a, ptr %b) #0 {
 ; CHECK-INTERLEAVE1-NEXT:    [[ADD]] = add i32 [[MUL]], [[EXT_B]]
 ; CHECK-INTERLEAVE1-NEXT:    [[IV_NEXT]] = add i64 [[IV]], 1
 ; CHECK-INTERLEAVE1-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], 1024
-; CHECK-INTERLEAVE1-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_EXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
+; CHECK-INTERLEAVE1-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_EXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP13:![0-9]+]]
 ; CHECK-INTERLEAVE1:       for.exit:
 ; CHECK-INTERLEAVE1-NEXT:    [[ADD_LCSSA:%.*]] = phi i32 [ [[ADD]], [[FOR_BODY]] ], [ [[TMP22]], [[MIDDLE_BLOCK]] ]
 ; CHECK-INTERLEAVE1-NEXT:    ret i32 [[ADD_LCSSA]]
@@ -981,7 +1455,7 @@ define i32 @not_dotp_not_phi(ptr %a, ptr %b) #0 {
 ; CHECK-INTERLEAVED-NEXT:    [[TMP21]] = add <vscale x 8 x i32> [[TMP30]], [[TMP22]]
 ; CHECK-INTERLEAVED-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
 ; CHECK-INTERLEAVED-NEXT:    [[TMP24:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-INTERLEAVED-NEXT:    br i1 [[TMP24]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+; CHECK-INTERLEAVED-NEXT:    br i1 [[TMP24]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
 ; CHECK-INTERLEAVED:       middle.block:
 ; CHECK-INTERLEAVED-NEXT:    [[TMP23:%.*]] = call i32 @llvm.vscale.i32()
 ; CHECK-INTERLEAVED-NEXT:    [[TMP31:%.*]] = mul i32 [[TMP23]], 8
@@ -1010,7 +1484,7 @@ define i32 @not_dotp_not_phi(ptr %a, ptr %b) #0 {
 ; CHECK-INTERLEAVED-NEXT:    [[ADD]] = add i32 [[MUL]], [[EXT_B]]
 ; CHECK-INTERLEAVED-NEXT:    [[IV_NEXT]] = add i64 [[IV]], 1
 ; CHECK-INTERLEAVED-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], 1024
-; CHECK-INTERLEAVED-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_EXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
+; CHECK-INTERLEAVED-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_EXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP13:![0-9]+]]
 ; CHECK-INTERLEAVED:       for.exit:
 ; CHECK-INTERLEAVED-NEXT:    [[ADD_LCSSA:%.*]] = phi i32 [ [[ADD]], [[FOR_BODY]] ], [ [[TMP26]], [[MIDDLE_BLOCK]] ]
 ; CHECK-INTERLEAVED-NEXT:    ret i32 [[ADD_LCSSA]]
@@ -1049,7 +1523,7 @@ define i32 @not_dotp_not_phi(ptr %a, ptr %b) #0 {
 ; CHECK-MAXBW-NEXT:    [[TMP21]] = add <vscale x 8 x i32> [[TMP20]], [[TMP19]]
 ; CHECK-MAXBW-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
 ; CHECK-MAXBW-NEXT:    [[TMP22:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-MAXBW-NEXT:    br i1 [[TMP22]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+; CHECK-MAXBW-NEXT:    br i1 [[TMP22]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
 ; CHECK-MAXBW:       middle.block:
 ; CHECK-MAXBW-NEXT:    [[TMP26:%.*]] = call i32 @llvm.vscale.i32()
 ; CHECK-MAXBW-NEXT:    [[TMP27:%.*]] = mul i32 [[TMP26]], 8
@@ -1078,7 +1552,7 @@ define i32 @not_dotp_not_phi(ptr %a, ptr %b) #0 {
 ; CHECK-MAXBW-NEXT:    [[ADD]] = add i32 [[MUL]], [[EXT_B]]
 ; CHECK-MAXBW-NEXT:    [[IV_NEXT]] = add i64 [[IV]], 1
 ; CHECK-MAXBW-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], 1024
-; CHECK-MAXBW-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_EXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
+; CHECK-MAXBW-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_EXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP13:![0-9]+]]
 ; CHECK-MAXBW:       for.exit:
 ; CHECK-MAXBW-NEXT:    [[ADD_LCSSA:%.*]] = phi i32 [ [[ADD]], [[FOR_BODY]] ], [ [[TMP29]], [[MIDDLE_BLOCK]] ]
 ; CHECK-MAXBW-NEXT:    ret i32 [[ADD_LCSSA]]
@@ -1173,7 +1647,7 @@ define i32 @dotp_unrolled(i32 %num_out, i64 %num_in, ptr %a, ptr %b) #0 {
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP41]] = add <vscale x 4 x i32> [[TMP40]], [[VEC_PHI]]
 ; CHECK-INTERLEAVE1-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP26]]
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP32:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-INTERLEAVE1-NEXT:    br i1 [[TMP32]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
+; CHECK-INTERLEAVE1-NEXT:    br i1 [[TMP32]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
 ; CHECK-INTERLEAVE1:       middle.block:
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP43:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[TMP41]])
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP44:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[TMP35]])
@@ -1231,7 +1705,7 @@ define i32 @dotp_unrolled(i32 %num_out, i64 %num_in, ptr %a, ptr %b) #0 {
 ; CHECK-INTERLEAVE1-NEXT:    [[ADD_A3]] = add nsw i32 [[MUL_A3]], [[ACCUM3]]
 ; CHECK-INTERLEAVE1-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
 ; CHECK-INTERLEAVE1-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[NUM_IN]]
-; CHECK-INTERLEAVE1-NEXT:    br i1 [[EXITCOND_NOT]], label [[EXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]]
+; CHECK-INTERLEAVE1-NEXT:    br i1 [[EXITCOND_NOT]], label [[EXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP15:![0-9]+]]
 ; CHECK-INTERLEAVE1:       exit:
 ; CHECK-INTERLEAVE1-NEXT:    [[ADD_A0_LCSSA:%.*]] = phi i32 [ [[ADD_A0]], [[FOR_BODY]] ], [ [[TMP46]], [[MIDDLE_BLOCK]] ]
 ; CHECK-INTERLEAVE1-NEXT:    [[ADD_A1_LCSSA:%.*]] = phi i32 [ [[ADD_A1]], [[FOR_BODY]] ], [ [[TMP45]], [[MIDDLE_BLOCK]] ]
@@ -1361,7 +1835,7 @@ define i32 @dotp_unrolled(i32 %num_out, i64 %num_in, ptr %a, ptr %b) #0 {
 ; CHECK-INTERLEAVED-NEXT:    [[TMP81]] = add <vscale x 4 x i32> [[TMP79]], [[VEC_PHI1]]
 ; CHECK-INTERLEAVED-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP40]]
 ; CHECK-INTERLEAVED-NEXT:    [[TMP32:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-INTERLEAVED-NEXT:    br i1 [[TMP32]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
+; CHECK-INTERLEAVED-NEXT:    br i1 [[TMP32]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
 ; CHECK-INTERLEAVED:       middle.block:
 ; CHECK-INTERLEAVED-NEXT:    [[BIN_RDX:%.*]] = add <vscale x 4 x i32> [[TMP81]], [[TMP80]]
 ; CHECK-INTERLEAVED-NEXT:    [[TMP83:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[BIN_RDX]])
@@ -1423,7 +1897,7 @@ define i32 @dotp_unrolled(i32 %num_out, i64 %num_in, ptr %a, ptr %b) #0 {
 ; CHECK-INTERLEAVED-NEXT:    [[ADD_A3]] = add nsw i32 [[MUL_A3]], [[ACCUM3]]
 ; CHECK-INTERLEAVED-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
 ; CHECK-INTERLEAVED-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[NUM_IN]]
-; CHECK-INTERLEAVED-NEXT:    br i1 [[EXITCOND_NOT]], label [[EXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]]
+; CHECK-INTERLEAVED-NEXT:    br i1 [[EXITCOND_NOT]], label [[EXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP15:![0-9]+]]
 ; CHECK-INTERLEAVED:       exit:
 ; CHECK-INTERLEAVED-NEXT:    [[ADD_A0_LCSSA:%.*]] = phi i32 [ [[ADD_A0]], [[FOR_BODY]] ], [ [[TMP86]], [[MIDDLE_BLOCK]] ]
 ; CHECK-INTERLEAVED-NEXT:    [[ADD_A1_LCSSA:%.*]] = phi i32 [ [[ADD_A1]], [[FOR_BODY]] ], [ [[TMP85]], [[MIDDLE_BLOCK]] ]
@@ -1501,7 +1975,7 @@ define i32 @dotp_unrolled(i32 %num_out, i64 %num_in, ptr %a, ptr %b) #0 {
 ; CHECK-MAXBW-NEXT:    [[PARTIAL_REDUCE16]] = call <vscale x 2 x i32> @llvm.experimental.vector.partial.reduce.add.nxv2i32.nxv8i32(<vscale x 2 x i32> [[VEC_PHI4]], <vscale x 8 x i32> [[TMP73]])
 ; CHECK-MAXBW-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
 ; CHECK-MAXBW-NEXT:    [[TMP74:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-MAXBW-NEXT:    br i1 [[TMP74]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
+; CHECK-MAXBW-NEXT:    br i1 [[TMP74]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
 ; CHECK-MAXBW:       middle.block:
 ; CHECK-MAXBW-NEXT:    [[TMP39:%.*]] = call i32 @llvm.vector.reduce.add.nxv2i32(<vscale x 2 x i32> [[PARTIAL_REDUCE16]])
 ; CHECK-MAXBW-NEXT:    [[TMP40:%.*]] = call i32 @llvm.vector.reduce.add.nxv2i32(<vscale x 2 x i32> [[PARTIAL_REDUCE17]])
@@ -1559,7 +2033,7 @@ define i32 @dotp_unrolled(i32 %num_out, i64 %num_in, ptr %a, ptr %b) #0 {
 ; CHECK-MAXBW-NEXT:    [[ADD_A3]] = add nsw i32 [[MUL_A3]], [[ACCUM3]]
 ; CHECK-MAXBW-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
 ; CHECK-MAXBW-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[NUM_IN]]
-; CHECK-MAXBW-NEXT:    br i1 [[EXITCOND_NOT]], label [[EXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]]
+; CHECK-MAXBW-NEXT:    br i1 [[EXITCOND_NOT]], label [[EXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP15:![0-9]+]]
 ; CHECK-MAXBW:       exit:
 ; CHECK-MAXBW-NEXT:    [[ADD_A0_LCSSA:%.*]] = phi i32 [ [[ADD_A0]], [[FOR_BODY]] ], [ [[TMP42]], [[MIDDLE_BLOCK]] ]
 ; CHECK-MAXBW-NEXT:    [[ADD_A1_LCSSA:%.*]] = phi i32 [ [[ADD_A1]], [[FOR_BODY]] ], [ [[TMP41]], [[MIDDLE_BLOCK]] ]
@@ -1625,259 +2099,48 @@ exit:                                        ; preds = %for.body
   ret i32 %result
 }
 
-define i32 @not_dotp_predicated(i64 %N, ptr %a, ptr %b) #0 {
-; CHECK-INTERLEAVE1-LABEL: define i32 @not_dotp_predicated(
+define i32 @dotp_predicated(i64 %N, ptr %a, ptr %b) #0 {
+; CHECK-INTERLEAVE1-LABEL: define i32 @dotp_predicated(
 ; CHECK-INTERLEAVE1-SAME: i64 [[N:%.*]], ptr [[A:%.*]], ptr [[B:%.*]]) #[[ATTR0]] {
 ; CHECK-INTERLEAVE1-NEXT:  entry:
-; CHECK-INTERLEAVE1-NEXT:    [[TMP6:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-INTERLEAVE1-NEXT:    [[TMP7:%.*]] = mul i64 [[TMP6]], 4
-; CHECK-INTERLEAVE1-NEXT:    [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 [[N]], [[TMP7]]
-; CHECK-INTERLEAVE1-NEXT:    br i1 [[MIN_ITERS_CHECK1]], label [[VEC_EPILOG_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK-INTERLEAVE1-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK-INTERLEAVE1:       vector.ph:
-; CHECK-INTERLEAVE1-NEXT:    [[TMP10:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-INTERLEAVE1-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP10]], 4
-; CHECK-INTERLEAVE1-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
-; CHECK-INTERLEAVE1-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
-; CHECK-INTERLEAVE1-NEXT:    [[TMP11:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-INTERLEAVE1-NEXT:    [[TMP15:%.*]] = mul i64 [[TMP11]], 4
-; CHECK-INTERLEAVE1-NEXT:    br label [[VECTOR_BODY:%.*]]
-; CHECK-INTERLEAVE1:       vector.body:
-; CHECK-INTERLEAVE1-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-INTERLEAVE1-NEXT:    [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP14:%.*]], [[VECTOR_BODY]] ]
-; CHECK-INTERLEAVE1-NEXT:    [[TMP0:%.*]] = add i64 [[INDEX]], 0
-; CHECK-INTERLEAVE1-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[TMP0]]
-; CHECK-INTERLEAVE1-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i32 0
-; CHECK-INTERLEAVE1-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 4 x i8>, ptr [[TMP2]], align 1
-; CHECK-INTERLEAVE1-NEXT:    [[TMP9:%.*]] = sext <vscale x 4 x i8> [[WIDE_LOAD]] to <vscale x 4 x i32>
-; CHECK-INTERLEAVE1-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[TMP0]]
-; CHECK-INTERLEAVE1-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i8, ptr [[TMP4]], i32 0
-; CHECK-INTERLEAVE1-NEXT:    [[WIDE_LOAD1:%.*]] = load <vscale x 4 x i8>, ptr [[TMP5]], align 1
-; CHECK-INTERLEAVE1-NEXT:    [[TMP12:%.*]] = sext <vscale x 4 x i8> [[WIDE_LOAD1]] to <vscale x 4 x i32>
-; CHECK-INTERLEAVE1-NEXT:    [[TMP13:%.*]] = mul nsw <vscale x 4 x i32> [[TMP12]], [[TMP9]]
-; CHECK-INTERLEAVE1-NEXT:    [[TMP14]] = add <vscale x 4 x i32> [[TMP13]], [[VEC_PHI]]
-; CHECK-INTERLEAVE1-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP15]]
-; CHECK-INTERLEAVE1-NEXT:    [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-INTERLEAVE1-NEXT:    br i1 [[TMP8]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
-; CHECK-INTERLEAVE1:       middle.block:
-; CHECK-INTERLEAVE1-NEXT:    [[TMP16:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[TMP14]])
-; CHECK-INTERLEAVE1-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
-; CHECK-INTERLEAVE1-NEXT:    br i1 [[CMP_N]], label [[EXIT:%.*]], label [[VEC_EPILOG_PH]]
-; CHECK-INTERLEAVE1:       scalar.ph:
-; CHECK-INTERLEAVE1-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
-; CHECK-INTERLEAVE1-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP16]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ]
-; CHECK-INTERLEAVE1-NEXT:    br label [[FOR_BODY:%.*]]
-; CHECK-INTERLEAVE1:       for.body:
-; CHECK-INTERLEAVE1-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[VEC_EPILOG_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
-; CHECK-INTERLEAVE1-NEXT:    [[ACCUM:%.*]] = phi i32 [ [[BC_MERGE_RDX]], [[VEC_EPILOG_PH]] ], [ [[ADD:%.*]], [[FOR_BODY]] ]
-; CHECK-INTERLEAVE1-NEXT:    [[GEP_A:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[IV]]
-; CHECK-INTERLEAVE1-NEXT:    [[LOAD_A:%.*]] = load i8, ptr [[GEP_A]], align 1
-; CHECK-INTERLEAVE1-NEXT:    [[EXT_A:%.*]] = sext i8 [[LOAD_A]] to i32
-; CHECK-INTERLEAVE1-NEXT:    [[GEP_B:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[IV]]
-; CHECK-INTERLEAVE1-NEXT:    [[LOAD_B:%.*]] = load i8, ptr [[GEP_B]], align 1
-; CHECK-INTERLEAVE1-NEXT:    [[EXT_B:%.*]] = sext i8 [[LOAD_B]] to i32
-; CHECK-INTERLEAVE1-NEXT:    [[MUL:%.*]] = mul nsw i32 [[EXT_B]], [[EXT_A]]
-; CHECK-INTERLEAVE1-NEXT:    [[ADD]] = add nsw i32 [[MUL]], [[ACCUM]]
-; CHECK-INTERLEAVE1-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
-; CHECK-INTERLEAVE1-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; CHECK-INTERLEAVE1-NEXT:    br i1 [[EXITCOND_NOT]], label [[EXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP13:![0-9]+]]
-; CHECK-INTERLEAVE1:       exit:
-; CHECK-INTERLEAVE1-NEXT:    [[ADD_LCSSA:%.*]] = phi i32 [ [[ADD]], [[FOR_BODY]] ], [ [[TMP16]], [[MIDDLE_BLOCK]] ]
-; CHECK-INTERLEAVE1-NEXT:    ret i32 [[ADD_LCSSA]]
-;
-; CHECK-INTERLEAVED-LABEL: define i32 @not_dotp_predicated(
-; CHECK-INTERLEAVED-SAME: i64 [[N:%.*]], ptr [[A:%.*]], ptr [[B:%.*]]) #[[ATTR0]] {
-; CHECK-INTERLEAVED-NEXT:  entry:
-; CHECK-INTERLEAVED-NEXT:    [[TMP8:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-INTERLEAVED-NEXT:    [[TMP14:%.*]] = mul i64 [[TMP8]], 8
-; CHECK-INTERLEAVED-NEXT:    [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 [[N]], [[TMP14]]
-; CHECK-INTERLEAVED-NEXT:    br i1 [[MIN_ITERS_CHECK1]], label [[VEC_EPILOG_PH:%.*]], label [[VECTOR_PH:%.*]]
-; CHECK-INTERLEAVED:       vector.ph:
-; CHECK-INTERLEAVED-NEXT:    [[TMP15:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-INTERLEAVED-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP15]], 8
-; CHECK-INTERLEAVED-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
-; CHECK-INTERLEAVED-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
-; CHECK-INTERLEAVED-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-INTERLEAVED-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 8
-; CHECK-INTERLEAVED-NEXT:    br label [[VECTOR_BODY:%.*]]
-; CHECK-INTERLEAVED:       vector.body:
-; CHECK-INTERLEAVED-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-INTERLEAVED-NEXT:    [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP23:%.*]], [[VECTOR_BODY]] ]
-; CHECK-INTERLEAVED-NEXT:    [[VEC_PHI1:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP24:%.*]], [[VECTOR_BODY]] ]
-; CHECK-INTERLEAVED-NEXT:    [[TMP0:%.*]] = add i64 [[INDEX]], 0
-; CHECK-INTERLEAVED-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[TMP0]]
-; CHECK-INTERLEAVED-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i32 0
-; CHECK-INTERLEAVED-NEXT:    [[TMP9:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-INTERLEAVED-NEXT:    [[TMP10:%.*]] = mul i64 [[TMP9]], 4
-; CHECK-INTERLEAVED-NEXT:    [[TMP11:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i64 [[TMP10]]
-; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 4 x i8>, ptr [[TMP2]], align 1
-; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD2:%.*]] = load <vscale x 4 x i8>, ptr [[TMP11]], align 1
-; CHECK-INTERLEAVED-NEXT:    [[TMP12:%.*]] = sext <vscale x 4 x i8> [[WIDE_LOAD]] to <vscale x 4 x i32>
-; CHECK-INTERLEAVED-NEXT:    [[TMP25:%.*]] = sext <vscale x 4 x i8> [[WIDE_LOAD2]] to <vscale x 4 x i32>
-; CHECK-INTERLEAVED-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[TMP0]]
-; CHECK-INTERLEAVED-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i8, ptr [[TMP6]], i32 0
-; CHECK-INTERLEAVED-NEXT:    [[TMP16:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-INTERLEAVED-NEXT:    [[TMP17:%.*]] = mul i64 [[TMP16]], 4
-; CHECK-INTERLEAVED-NEXT:    [[TMP18:%.*]] = getelementptr inbounds i8, ptr [[TMP6]], i64 [[TMP17]]
-; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD3:%.*]] = load <vscale x 4 x i8>, ptr [[TMP7]], align 1
-; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD4:%.*]] = load <vscale x 4 x i8>, ptr [[TMP18]], align 1
-; CHECK-INTERLEAVED-NEXT:    [[TMP19:%.*]] = sext <vscale x 4 x i8> [[WIDE_LOAD3]] to <vscale x 4 x i32>
-; CHECK-INTERLEAVED-NEXT:    [[TMP20:%.*]] = sext <vscale x 4 x i8> [[WIDE_LOAD4]] to <vscale x 4 x i32>
-; CHECK-INTERLEAVED-NEXT:    [[TMP21:%.*]] = mul nsw <vscale x 4 x i32> [[TMP19]], [[TMP12]]
-; CHECK-INTERLEAVED-NEXT:    [[TMP22:%.*]] = mul nsw <vscale x 4 x i32> [[TMP20]], [[TMP25]]
-; CHECK-INTERLEAVED-NEXT:    [[TMP23]] = add <vscale x 4 x i32> [[TMP21]], [[VEC_PHI]]
-; CHECK-INTERLEAVED-NEXT:    [[TMP24]] = add <vscale x 4 x i32> [[TMP22]], [[VEC_PHI1]]
-; CHECK-INTERLEAVED-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
-; CHECK-INTERLEAVED-NEXT:    [[TMP13:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-INTERLEAVED-NEXT:    br i1 [[TMP13]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
-; CHECK-INTERLEAVED:       middle.block:
-; CHECK-INTERLEAVED-NEXT:    [[BIN_RDX:%.*]] = add <vscale x 4 x i32> [[TMP24]], [[TMP23]]
-; CHECK-INTERLEAVED-NEXT:    [[TMP26:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[BIN_RDX]])
-; CHECK-INTERLEAVED-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
-; CHECK-INTERLEAVED-NEXT:    br i1 [[CMP_N]], label [[EXIT:%.*]], label [[VEC_EPILOG_PH]]
-; CHECK-INTERLEAVED:       scalar.ph:
-; CHECK-INTERLEAVED-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
-; CHECK-INTERLEAVED-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP26]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ]
-; CHECK-INTERLEAVED-NEXT:    br label [[FOR_BODY:%.*]]
-; CHECK-INTERLEAVED:       for.body:
-; CHECK-INTERLEAVED-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[VEC_EPILOG_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
-; CHECK-INTERLEAVED-NEXT:    [[ACCUM:%.*]] = phi i32 [ [[BC_MERGE_RDX]], [[VEC_EPILOG_PH]] ], [ [[ADD:%.*]], [[FOR_BODY]] ]
-; CHECK-INTERLEAVED-NEXT:    [[GEP_A:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[IV]]
-; CHECK-INTERLEAVED-NEXT:    [[LOAD_A:%.*]] = load i8, ptr [[GEP_A]], align 1
-; CHECK-INTERLEAVED-NEXT:    [[EXT_A:%.*]] = sext i8 [[LOAD_A]] to i32
-; CHECK-INTERLEAVED-NEXT:    [[GEP_B:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[IV]]
-; CHECK-INTERLEAVED-NEXT:    [[LOAD_B:%.*]] = load i8, ptr [[GEP_B]], align 1
-; CHECK-INTERLEAVED-NEXT:    [[EXT_B:%.*]] = sext i8 [[LOAD_B]] to i32
-; CHECK-INTERLEAVED-NEXT:    [[MUL:%.*]] = mul nsw i32 [[EXT_B]], [[EXT_A]]
-; CHECK-INTERLEAVED-NEXT:    [[ADD]] = add nsw i32 [[MUL]], [[ACCUM]]
-; CHECK-INTERLEAVED-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
-; CHECK-INTERLEAVED-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; CHECK-INTERLEAVED-NEXT:    br i1 [[EXITCOND_NOT]], label [[EXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP13:![0-9]+]]
-; CHECK-INTERLEAVED:       exit:
-; CHECK-INTERLEAVED-NEXT:    [[ADD_LCSSA:%.*]] = phi i32 [ [[ADD]], [[FOR_BODY]] ], [ [[TMP26]], [[MIDDLE_BLOCK]] ]
-; CHECK-INTERLEAVED-NEXT:    ret i32 [[ADD_LCSSA]]
-;
-; CHECK-MAXBW-LABEL: define i32 @not_dotp_predicated(
-; CHECK-MAXBW-SAME: i64 [[N:%.*]], ptr [[A:%.*]], ptr [[B:%.*]]) #[[ATTR0]] {
-; CHECK-MAXBW-NEXT:  entry:
-; CHECK-MAXBW-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-MAXBW-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 8
-; CHECK-MAXBW-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
-; CHECK-MAXBW-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
-; CHECK-MAXBW:       vector.ph:
-; CHECK-MAXBW-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-MAXBW-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 8
-; CHECK-MAXBW-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
-; CHECK-MAXBW-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
-; CHECK-MAXBW-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-MAXBW-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 8
-; CHECK-MAXBW-NEXT:    br label [[VECTOR_BODY:%.*]]
-; CHECK-MAXBW:       vector.body:
-; CHECK-MAXBW-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-MAXBW-NEXT:    [[VEC_PHI1:%.*]] = phi <vscale x 2 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[PARTIAL_REDUCE5:%.*]], [[VECTOR_BODY]] ]
-; CHECK-MAXBW-NEXT:    [[TMP6:%.*]] = add i64 [[INDEX]], 0
-; CHECK-MAXBW-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[TMP6]]
-; CHECK-MAXBW-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i8, ptr [[TMP7]], i32 0
-; CHECK-MAXBW-NEXT:    [[WIDE_LOAD2:%.*]] = load <vscale x 8 x i8>, ptr [[TMP8]], align 1
-; CHECK-MAXBW-NEXT:    [[TMP13:%.*]] = sext <vscale x 8 x i8> [[WIDE_LOAD2]] to <vscale x 8 x i32>
-; CHECK-MAXBW-NEXT:    [[TMP14:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[TMP6]]
-; CHECK-MAXBW-NEXT:    [[TMP15:%.*]] = getelementptr inbounds i8, ptr [[TMP14]], i32 0
-; CHECK-MAXBW-NEXT:    [[WIDE_LOAD4:%.*]] = load <vscale x 8 x i8>, ptr [[TMP15]], align 1
-; CHECK-MAXBW-NEXT:    [[TMP20:%.*]] = sext <vscale x 8 x i8> [[WIDE_LOAD4]] to <vscale x 8 x i32>
-; CHECK-MAXBW-NEXT:    [[TMP22:%.*]] = mul nsw <vscale x 8 x i32> [[TMP20]], [[TMP13]]
-; CHECK-MAXBW-NEXT:    [[PARTIAL_REDUCE5]] = call <vscale x 2 x i32> @llvm.experimental.vector.partial.reduce.add.nxv2i32.nxv8i32(<vscale x 2 x i32> [[VEC_PHI1]], <vscale x 8 x i32> [[TMP22]])
-; CHECK-MAXBW-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
-; CHECK-MAXBW-NEXT:    [[TMP23:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-MAXBW-NEXT:    br i1 [[TMP23]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
-; CHECK-MAXBW:       middle.block:
-; CHECK-MAXBW-NEXT:    [[TMP16:%.*]] = call i32 @llvm.vector.reduce.add.nxv2i32(<vscale x 2 x i32> [[PARTIAL_REDUCE5]])
-; CHECK-MAXBW-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
-; CHECK-MAXBW-NEXT:    br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
-; CHECK-MAXBW:       scalar.ph:
-; CHECK-MAXBW-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
-; CHECK-MAXBW-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP16]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ]
-; CHECK-MAXBW-NEXT:    br label [[FOR_BODY:%.*]]
-; CHECK-MAXBW:       for.body:
-; CHECK-MAXBW-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
-; CHECK-MAXBW-NEXT:    [[ACCUM:%.*]] = phi i32 [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[ADD:%.*]], [[FOR_BODY]] ]
-; CHECK-MAXBW-NEXT:    [[GEP_A:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[IV]]
-; CHECK-MAXBW-NEXT:    [[LOAD_A:%.*]] = load i8, ptr [[GEP_A]], align 1
-; CHECK-MAXBW-NEXT:    [[EXT_A:%.*]] = sext i8 [[LOAD_A]] to i32
-; CHECK-MAXBW-NEXT:    [[GEP_B:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[IV]]
-; CHECK-MAXBW-NEXT:    [[LOAD_B:%.*]] = load i8, ptr [[GEP_B]], align 1
-; CHECK-MAXBW-NEXT:    [[EXT_B:%.*]] = sext i8 [[LOAD_B]] to i32
-; CHECK-MAXBW-NEXT:    [[MUL:%.*]] = mul nsw i32 [[EXT_B]], [[EXT_A]]
-; CHECK-MAXBW-NEXT:    [[ADD]] = add nsw i32 [[MUL]], [[ACCUM]]
-; CHECK-MAXBW-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
-; CHECK-MAXBW-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; CHECK-MAXBW-NEXT:    br i1 [[EXITCOND_NOT]], label [[EXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP13:![0-9]+]]
-; CHECK-MAXBW:       exit:
-; CHECK-MAXBW-NEXT:    [[ADD_LCSSA:%.*]] = phi i32 [ [[ADD]], [[FOR_BODY]] ], [ [[TMP16]], [[MIDDLE_BLOCK]] ]
-; CHECK-MAXBW-NEXT:    ret i32 [[ADD_LCSSA]]
-;
-entry:
-  br label %for.body
-
-for.body:                                         ; preds = %entry, %for.body
-  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
-  %accum = phi i32 [ 0, %entry ], [ %add, %for.body ]
-  %gep.a = getelementptr inbounds i8, ptr %a, i64 %iv
-  %load.a = load i8, ptr %gep.a, align 1
-  %ext.a = sext i8 %load.a to i32
-  %gep.b = getelementptr inbounds i8, ptr %b, i64 %iv
-  %load.b = load i8, ptr %gep.b, align 1
-  %ext.b = sext i8 %load.b to i32
-  %mul = mul nsw i32 %ext.b, %ext.a
-  %add = add nsw i32 %mul, %accum
-  %iv.next = add nuw nsw i64 %iv, 1
-  %exitcond.not = icmp eq i64 %iv.next, %N
-  br i1 %exitcond.not, label %exit, label %for.body
-
-exit:                        ; preds = %for.body
-  ret i32 %add
-}
-
-define i32 @not_dotp_predicated_pragma(i64 %N, ptr %a, ptr %b) #0 {
-; CHECK-INTERLEAVE1-LABEL: define i32 @not_dotp_predicated_pragma(
-; CHECK-INTERLEAVE1-SAME: i64 [[N:%.*]], ptr [[A:%.*]], ptr [[B:%.*]]) #[[ATTR0]] {
-; CHECK-INTERLEAVE1-NEXT:  entry:
-; CHECK-INTERLEAVE1-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
-; CHECK-INTERLEAVE1:       vector.ph:
-; CHECK-INTERLEAVE1-NEXT:    [[TMP9:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-INTERLEAVE1-NEXT:    [[TMP10:%.*]] = mul i64 [[TMP9]], 4
-; CHECK-INTERLEAVE1-NEXT:    [[TMP11:%.*]] = sub i64 [[TMP10]], 1
-; CHECK-INTERLEAVE1-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N]], [[TMP11]]
-; CHECK-INTERLEAVE1-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP10]]
-; CHECK-INTERLEAVE1-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
-; CHECK-INTERLEAVE1-NEXT:    [[TMP12:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-INTERLEAVE1-NEXT:    [[TMP14:%.*]] = mul i64 [[TMP12]], 4
-; CHECK-INTERLEAVE1-NEXT:    [[TMP15:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-INTERLEAVE1-NEXT:    [[TMP6:%.*]] = mul i64 [[TMP15]], 4
-; CHECK-INTERLEAVE1-NEXT:    [[TMP0:%.*]] = sub i64 [[N]], [[TMP6]]
-; CHECK-INTERLEAVE1-NEXT:    [[TMP1:%.*]] = icmp ugt i64 [[N]], [[TMP6]]
-; CHECK-INTERLEAVE1-NEXT:    [[TMP2:%.*]] = select i1 [[TMP1]], i64 [[TMP0]], i64 0
-; CHECK-INTERLEAVE1-NEXT:    [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 0, i64 [[N]])
+; CHECK-INTERLEAVE1-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-INTERLEAVE1-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 4
+; CHECK-INTERLEAVE1-NEXT:    [[TMP2:%.*]] = sub i64 [[TMP1]], 1
+; CHECK-INTERLEAVE1-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N]], [[TMP2]]
+; CHECK-INTERLEAVE1-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP1]]
+; CHECK-INTERLEAVE1-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
+; CHECK-INTERLEAVE1-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-INTERLEAVE1-NEXT:    [[TMP4:%.*]] = mul i64 [[TMP3]], 4
+; CHECK-INTERLEAVE1-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-INTERLEAVE1-NEXT:    [[TMP6:%.*]] = mul i64 [[TMP5]], 4
+; CHECK-INTERLEAVE1-NEXT:    [[TMP7:%.*]] = sub i64 [[N]], [[TMP6]]
+; CHECK-INTERLEAVE1-NEXT:    [[TMP8:%.*]] = icmp ugt i64 [[N]], [[TMP6]]
+; CHECK-INTERLEAVE1-NEXT:    [[TMP9:%.*]] = select i1 [[TMP8]], i64 [[TMP7]], i64 0
+; CHECK-INTERLEAVE1-NEXT:    [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 0, i64 [[N]])
 ; CHECK-INTERLEAVE1-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK-INTERLEAVE1:       vector.body:
 ; CHECK-INTERLEAVE1-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; CHECK-INTERLEAVE1-NEXT:    [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 4 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], [[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; CHECK-INTERLEAVE1-NEXT:    [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP19:%.*]], [[VECTOR_BODY]] ]
-; CHECK-INTERLEAVE1-NEXT:    [[TMP3:%.*]] = add i64 [[INDEX]], 0
-; CHECK-INTERLEAVE1-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[TMP3]]
-; CHECK-INTERLEAVE1-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i8, ptr [[TMP4]], i32 0
-; CHECK-INTERLEAVE1-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 4 x i8> @llvm.masked.load.nxv4i8.p0(ptr [[TMP5]], i32 1, <vscale x 4 x i1> [[ACTIVE_LANE_MASK]], <vscale x 4 x i8> poison)
+; CHECK-INTERLEAVE1-NEXT:    [[TMP10:%.*]] = add i64 [[INDEX]], 0
+; CHECK-INTERLEAVE1-NEXT:    [[TMP11:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[TMP10]]
+; CHECK-INTERLEAVE1-NEXT:    [[TMP12:%.*]] = getelementptr inbounds i8, ptr [[TMP11]], i32 0
+; CHECK-INTERLEAVE1-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 4 x i8> @llvm.masked.load.nxv4i8.p0(ptr [[TMP12]], i32 1, <vscale x 4 x i1> [[ACTIVE_LANE_MASK]], <vscale x 4 x i8> poison)
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP13:%.*]] = sext <vscale x 4 x i8> [[WIDE_MASKED_LOAD]] to <vscale x 4 x i32>
-; CHECK-INTERLEAVE1-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[TMP3]]
-; CHECK-INTERLEAVE1-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i8, ptr [[TMP7]], i32 0
-; CHECK-INTERLEAVE1-NEXT:    [[WIDE_MASKED_LOAD1:%.*]] = call <vscale x 4 x i8> @llvm.masked.load.nxv4i8.p0(ptr [[TMP8]], i32 1, <vscale x 4 x i1> [[ACTIVE_LANE_MASK]], <vscale x 4 x i8> poison)
+; CHECK-INTERLEAVE1-NEXT:    [[TMP14:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[TMP10]]
+; CHECK-INTERLEAVE1-NEXT:    [[TMP15:%.*]] = getelementptr inbounds i8, ptr [[TMP14]], i32 0
+; CHECK-INTERLEAVE1-NEXT:    [[WIDE_MASKED_LOAD1:%.*]] = call <vscale x 4 x i8> @llvm.masked.load.nxv4i8.p0(ptr [[TMP15]], i32 1, <vscale x 4 x i1> [[ACTIVE_LANE_MASK]], <vscale x 4 x i8> poison)
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP16:%.*]] = sext <vscale x 4 x i8> [[WIDE_MASKED_LOAD1]] to <vscale x 4 x i32>
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP17:%.*]] = mul nsw <vscale x 4 x i32> [[TMP16]], [[TMP13]]
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP18:%.*]] = add <vscale x 4 x i32> [[TMP17]], [[VEC_PHI]]
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP19]] = select <vscale x 4 x i1> [[ACTIVE_LANE_MASK]], <vscale x 4 x i32> [[TMP18]], <vscale x 4 x i32> [[VEC_PHI]]
-; CHECK-INTERLEAVE1-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP14]]
-; CHECK-INTERLEAVE1-NEXT:    [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX]], i64 [[TMP2]])
+; CHECK-INTERLEAVE1-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP4]]
+; CHECK-INTERLEAVE1-NEXT:    [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX]], i64 [[TMP9]])
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP20:%.*]] = xor <vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT]], splat (i1 true)
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP21:%.*]] = extractelement <vscale x 4 x i1> [[TMP20]], i32 0
-; CHECK-INTERLEAVE1-NEXT:    br i1 [[TMP21]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
+; CHECK-INTERLEAVE1-NEXT:    br i1 [[TMP21]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]]
 ; CHECK-INTERLEAVE1:       middle.block:
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP22:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[TMP19]])
 ; CHECK-INTERLEAVE1-NEXT:    br i1 true, label [[EXIT:%.*]], label [[SCALAR_PH]]
@@ -1888,62 +2151,62 @@ define i32 @not_dotp_predicated_pragma(i64 %N, ptr %a, ptr %b) #0 {
 ; CHECK-INTERLEAVE1:       for.body:
 ; CHECK-INTERLEAVE1-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
 ; CHECK-INTERLEAVE1-NEXT:    [[ACCUM:%.*]] = phi i32 [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[ADD:%.*]], [[FOR_BODY]] ]
-; CHECK-INTERLEAVE1-NEXT:    [[GEP_A:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[IV]]
+; CHECK-INTERLEAVE1-NEXT:    [[GEP_A:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[IV]]
 ; CHECK-INTERLEAVE1-NEXT:    [[LOAD_A:%.*]] = load i8, ptr [[GEP_A]], align 1
 ; CHECK-INTERLEAVE1-NEXT:    [[EXT_A:%.*]] = sext i8 [[LOAD_A]] to i32
-; CHECK-INTERLEAVE1-NEXT:    [[GEP_A2:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[IV]]
-; CHECK-INTERLEAVE1-NEXT:    [[LOAD_B:%.*]] = load i8, ptr [[GEP_A2]], align 1
+; CHECK-INTERLEAVE1-NEXT:    [[GEP_B:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[IV]]
+; CHECK-INTERLEAVE1-NEXT:    [[LOAD_B:%.*]] = load i8, ptr [[GEP_B]], align 1
 ; CHECK-INTERLEAVE1-NEXT:    [[EXT_B:%.*]] = sext i8 [[LOAD_B]] to i32
 ; CHECK-INTERLEAVE1-NEXT:    [[MUL:%.*]] = mul nsw i32 [[EXT_B]], [[EXT_A]]
 ; CHECK-INTERLEAVE1-NEXT:    [[ADD]] = add nsw i32 [[MUL]], [[ACCUM]]
 ; CHECK-INTERLEAVE1-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
 ; CHECK-INTERLEAVE1-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; CHECK-INTERLEAVE1-NEXT:    br i1 [[EXITCOND_NOT]], label [[EXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]]
+; CHECK-INTERLEAVE1-NEXT:    br i1 [[EXITCOND_NOT]], label [[EXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP18:![0-9]+]]
 ; CHECK-INTERLEAVE1:       exit:
 ; CHECK-INTERLEAVE1-NEXT:    [[ADD_LCSSA:%.*]] = phi i32 [ [[ADD]], [[FOR_BODY]] ], [ [[TMP22]], [[MIDDLE_BLOCK]] ]
 ; CHECK-INTERLEAVE1-NEXT:    ret i32 [[ADD_LCSSA]]
 ;
-; CHECK-INTERLEAVED-LABEL: define i32 @not_dotp_predicated_pragma(
+; CHECK-INTERLEAVED-LABEL: define i32 @dotp_predicated(
 ; CHECK-INTERLEAVED-SAME: i64 [[N:%.*]], ptr [[A:%.*]], ptr [[B:%.*]]) #[[ATTR0]] {
 ; CHECK-INTERLEAVED-NEXT:  entry:
 ; CHECK-INTERLEAVED-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK-INTERLEAVED:       vector.ph:
-; CHECK-INTERLEAVED-NEXT:    [[TMP9:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-INTERLEAVED-NEXT:    [[TMP10:%.*]] = mul i64 [[TMP9]], 4
-; CHECK-INTERLEAVED-NEXT:    [[TMP11:%.*]] = sub i64 [[TMP10]], 1
-; CHECK-INTERLEAVED-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N]], [[TMP11]]
-; CHECK-INTERLEAVED-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP10]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-INTERLEAVED-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 4
+; CHECK-INTERLEAVED-NEXT:    [[TMP2:%.*]] = sub i64 [[TMP1]], 1
+; CHECK-INTERLEAVED-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N]], [[TMP2]]
+; CHECK-INTERLEAVED-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP1]]
 ; CHECK-INTERLEAVED-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
-; CHECK-INTERLEAVED-NEXT:    [[TMP12:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-INTERLEAVED-NEXT:    [[TMP14:%.*]] = mul i64 [[TMP12]], 4
-; CHECK-INTERLEAVED-NEXT:    [[TMP15:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-INTERLEAVED-NEXT:    [[TMP6:%.*]] = mul i64 [[TMP15]], 4
-; CHECK-INTERLEAVED-NEXT:    [[TMP0:%.*]] = sub i64 [[N]], [[TMP6]]
-; CHECK-INTERLEAVED-NEXT:    [[TMP1:%.*]] = icmp ugt i64 [[N]], [[TMP6]]
-; CHECK-INTERLEAVED-NEXT:    [[TMP2:%.*]] = select i1 [[TMP1]], i64 [[TMP0]], i64 0
+; CHECK-INTERLEAVED-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-INTERLEAVED-NEXT:    [[TMP4:%.*]] = mul i64 [[TMP3]], 4
+; CHECK-INTERLEAVED-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-INTERLEAVED-NEXT:    [[TMP6:%.*]] = mul i64 [[TMP5]], 4
+; CHECK-INTERLEAVED-NEXT:    [[TMP7:%.*]] = sub i64 [[N]], [[TMP6]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP8:%.*]] = icmp ugt i64 [[N]], [[TMP6]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP9:%.*]] = select i1 [[TMP8]], i64 [[TMP7]], i64 0
 ; CHECK-INTERLEAVED-NEXT:    [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 0, i64 [[N]])
 ; CHECK-INTERLEAVED-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK-INTERLEAVED:       vector.body:
 ; CHECK-INTERLEAVED-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; CHECK-INTERLEAVED-NEXT:    [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 4 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], [[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; CHECK-INTERLEAVED-NEXT:    [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP19:%.*]], [[VECTOR_BODY]] ]
-; CHECK-INTERLEAVED-NEXT:    [[TMP3:%.*]] = add i64 [[INDEX]], 0
-; CHECK-INTERLEAVED-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[TMP3]]
-; CHECK-INTERLEAVED-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i8, ptr [[TMP4]], i32 0
-; CHECK-INTERLEAVED-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 4 x i8> @llvm.masked.load.nxv4i8.p0(ptr [[TMP5]], i32 1, <vscale x 4 x i1> [[ACTIVE_LANE_MASK]], <vscale x 4 x i8> poison)
+; CHECK-INTERLEAVED-NEXT:    [[TMP10:%.*]] = add i64 [[INDEX]], 0
+; CHECK-INTERLEAVED-NEXT:    [[TMP11:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[TMP10]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP12:%.*]] = getelementptr inbounds i8, ptr [[TMP11]], i32 0
+; CHECK-INTERLEAVED-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 4 x i8> @llvm.masked.load.nxv4i8.p0(ptr [[TMP12]], i32 1, <vscale x 4 x i1> [[ACTIVE_LANE_MASK]], <vscale x 4 x i8> poison)
 ; CHECK-INTERLEAVED-NEXT:    [[TMP13:%.*]] = sext <vscale x 4 x i8> [[WIDE_MASKED_LOAD]] to <vscale x 4 x i32>
-; CHECK-INTERLEAVED-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[TMP3]]
-; CHECK-INTERLEAVED-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i8, ptr [[TMP7]], i32 0
-; CHECK-INTERLEAVED-NEXT:    [[WIDE_MASKED_LOAD1:%.*]] = call <vscale x 4 x i8> @llvm.masked.load.nxv4i8.p0(ptr [[TMP8]], i32 1, <vscale x 4 x i1> [[ACTIVE_LANE_MASK]], <vscale x 4 x i8> poison)
+; CHECK-INTERLEAVED-NEXT:    [[TMP14:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[TMP10]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP15:%.*]] = getelementptr inbounds i8, ptr [[TMP14]], i32 0
+; CHECK-INTERLEAVED-NEXT:    [[WIDE_MASKED_LOAD1:%.*]] = call <vscale x 4 x i8> @llvm.masked.load.nxv4i8.p0(ptr [[TMP15]], i32 1, <vscale x 4 x i1> [[ACTIVE_LANE_MASK]], <vscale x 4 x i8> poison)
 ; CHECK-INTERLEAVED-NEXT:    [[TMP16:%.*]] = sext <vscale x 4 x i8> [[WIDE_MASKED_LOAD1]] to <vscale x 4 x i32>
 ; CHECK-INTERLEAVED-NEXT:    [[TMP17:%.*]] = mul nsw <vscale x 4 x i32> [[TMP16]], [[TMP13]]
 ; CHECK-INTERLEAVED-NEXT:    [[TMP18:%.*]] = add <vscale x 4 x i32> [[TMP17]], [[VEC_PHI]]
 ; CHECK-INTERLEAVED-NEXT:    [[TMP19]] = select <vscale x 4 x i1> [[ACTIVE_LANE_MASK]], <vscale x 4 x i32> [[TMP18]], <vscale x 4 x i32> [[VEC_PHI]]
-; CHECK-INTERLEAVED-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP14]]
-; CHECK-INTERLEAVED-NEXT:    [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX]], i64 [[TMP2]])
+; CHECK-INTERLEAVED-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP4]]
+; CHECK-INTERLEAVED-NEXT:    [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX]], i64 [[TMP9]])
 ; CHECK-INTERLEAVED-NEXT:    [[TMP20:%.*]] = xor <vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT]], splat (i1 true)
 ; CHECK-INTERLEAVED-NEXT:    [[TMP21:%.*]] = extractelement <vscale x 4 x i1> [[TMP20]], i32 0
-; CHECK-INTERLEAVED-NEXT:    br i1 [[TMP21]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
+; CHECK-INTERLEAVED-NEXT:    br i1 [[TMP21]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]]
 ; CHECK-INTERLEAVED:       middle.block:
 ; CHECK-INTERLEAVED-NEXT:    [[TMP22:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[TMP19]])
 ; CHECK-INTERLEAVED-NEXT:    br i1 true, label [[EXIT:%.*]], label [[SCALAR_PH]]
@@ -1954,85 +2217,85 @@ define i32 @not_dotp_predicated_pragma(i64 %N, ptr %a, ptr %b) #0 {
 ; CHECK-INTERLEAVED:       for.body:
 ; CHECK-INTERLEAVED-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
 ; CHECK-INTERLEAVED-NEXT:    [[ACCUM:%.*]] = phi i32 [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[ADD:%.*]], [[FOR_BODY]] ]
-; CHECK-INTERLEAVED-NEXT:    [[GEP_A:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[IV]]
+; CHECK-INTERLEAVED-NEXT:    [[GEP_A:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[IV]]
 ; CHECK-INTERLEAVED-NEXT:    [[LOAD_A:%.*]] = load i8, ptr [[GEP_A]], align 1
 ; CHECK-INTERLEAVED-NEXT:    [[EXT_A:%.*]] = sext i8 [[LOAD_A]] to i32
-; CHECK-INTERLEAVED-NEXT:    [[GEP_A2:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[IV]]
-; CHECK-INTERLEAVED-NEXT:    [[LOAD_B:%.*]] = load i8, ptr [[GEP_A2]], align 1
+; CHECK-INTERLEAVED-NEXT:    [[GEP_B:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[IV]]
+; CHECK-INTERLEAVED-NEXT:    [[LOAD_B:%.*]] = load i8, ptr [[GEP_B]], align 1
 ; CHECK-INTERLEAVED-NEXT:    [[EXT_B:%.*]] = sext i8 [[LOAD_B]] to i32
 ; CHECK-INTERLEAVED-NEXT:    [[MUL:%.*]] = mul nsw i32 [[EXT_B]], [[EXT_A]]
 ; CHECK-INTERLEAVED-NEXT:    [[ADD]] = add nsw i32 [[MUL]], [[ACCUM]]
 ; CHECK-INTERLEAVED-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
 ; CHECK-INTERLEAVED-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; CHECK-INTERLEAVED-NEXT:    br i1 [[EXITCOND_NOT]], label [[EXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]]
+; CHECK-INTERLEAVED-NEXT:    br i1 [[EXITCOND_NOT]], label [[EXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP18:![0-9]+]]
 ; CHECK-INTERLEAVED:       exit:
 ; CHECK-INTERLEAVED-NEXT:    [[ADD_LCSSA:%.*]] = phi i32 [ [[ADD]], [[FOR_BODY]] ], [ [[TMP22]], [[MIDDLE_BLOCK]] ]
 ; CHECK-INTERLEAVED-NEXT:    ret i32 [[ADD_LCSSA]]
 ;
-; CHECK-MAXBW-LABEL: define i32 @not_dotp_predicated_pragma(
+; CHECK-MAXBW-LABEL: define i32 @dotp_predicated(
 ; CHECK-MAXBW-SAME: i64 [[N:%.*]], ptr [[A:%.*]], ptr [[B:%.*]]) #[[ATTR0]] {
 ; CHECK-MAXBW-NEXT:  entry:
 ; CHECK-MAXBW-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK-MAXBW:       vector.ph:
 ; CHECK-MAXBW-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-MAXBW-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 4
+; CHECK-MAXBW-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 16
 ; CHECK-MAXBW-NEXT:    [[TMP2:%.*]] = sub i64 [[TMP1]], 1
 ; CHECK-MAXBW-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N]], [[TMP2]]
 ; CHECK-MAXBW-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP1]]
 ; CHECK-MAXBW-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
 ; CHECK-MAXBW-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-MAXBW-NEXT:    [[TMP4:%.*]] = mul i64 [[TMP3]], 4
+; CHECK-MAXBW-NEXT:    [[TMP4:%.*]] = mul i64 [[TMP3]], 16
 ; CHECK-MAXBW-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-MAXBW-NEXT:    [[TMP6:%.*]] = mul i64 [[TMP5]], 4
+; CHECK-MAXBW-NEXT:    [[TMP6:%.*]] = mul i64 [[TMP5]], 16
 ; CHECK-MAXBW-NEXT:    [[TMP7:%.*]] = sub i64 [[N]], [[TMP6]]
 ; CHECK-MAXBW-NEXT:    [[TMP8:%.*]] = icmp ugt i64 [[N]], [[TMP6]]
 ; CHECK-MAXBW-NEXT:    [[TMP9:%.*]] = select i1 [[TMP8]], i64 [[TMP7]], i64 0
-; CHECK-MAXBW-NEXT:    [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 0, i64 [[N]])
+; CHECK-MAXBW-NEXT:    [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i64(i64 0, i64 [[N]])
 ; CHECK-MAXBW-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK-MAXBW:       vector.body:
 ; CHECK-MAXBW-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-MAXBW-NEXT:    [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 4 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], [[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-MAXBW-NEXT:    [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP19:%.*]], [[VECTOR_BODY]] ]
+; CHECK-MAXBW-NEXT:    [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 16 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], [[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-MAXBW-NEXT:    [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[PARTIAL_REDUCE:%.*]], [[VECTOR_BODY]] ]
 ; CHECK-MAXBW-NEXT:    [[TMP10:%.*]] = add i64 [[INDEX]], 0
-; CHECK-MAXBW-NEXT:    [[TMP11:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[TMP10]]
+; CHECK-MAXBW-NEXT:    [[TMP11:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[TMP10]]
 ; CHECK-MAXBW-NEXT:    [[TMP12:%.*]] = getelementptr inbounds i8, ptr [[TMP11]], i32 0
-; CHECK-MAXBW-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 4 x i8> @llvm.masked.load.nxv4i8.p0(ptr [[TMP12]], i32 1, <vscale x 4 x i1> [[ACTIVE_LANE_MASK]], <vscale x 4 x i8> poison)
-; CHECK-MAXBW-NEXT:    [[TMP13:%.*]] = sext <vscale x 4 x i8> [[WIDE_MASKED_LOAD]] to <vscale x 4 x i32>
-; CHECK-MAXBW-NEXT:    [[TMP14:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[TMP10]]
+; CHECK-MAXBW-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr [[TMP12]], i32 1, <vscale x 16 x i1> [[ACTIVE_LANE_MASK]], <vscale x 16 x i8> poison)
+; CHECK-MAXBW-NEXT:    [[TMP13:%.*]] = sext <vscale x 16 x i8> [[WIDE_MASKED_LOAD]] to <vscale x 16 x i32>
+; CHECK-MAXBW-NEXT:    [[TMP14:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[TMP10]]
 ; CHECK-MAXBW-NEXT:    [[TMP15:%.*]] = getelementptr inbounds i8, ptr [[TMP14]], i32 0
-; CHECK-MAXBW-NEXT:    [[WIDE_MASKED_LOAD1:%.*]] = call <vscale x 4 x i8> @llvm.masked.load.nxv4i8.p0(ptr [[TMP15]], i32 1, <vscale x 4 x i1> [[ACTIVE_LANE_MASK]], <vscale x 4 x i8> poison)
-; CHECK-MAXBW-NEXT:    [[TMP16:%.*]] = sext <vscale x 4 x i8> [[WIDE_MASKED_LOAD1]] to <vscale x 4 x i32>
-; CHECK-MAXBW-NEXT:    [[TMP17:%.*]] = mul nsw <vscale x 4 x i32> [[TMP16]], [[TMP13]]
-; CHECK-MAXBW-NEXT:    [[TMP18:%.*]] = add <vscale x 4 x i32> [[TMP17]], [[VEC_PHI]]
-; CHECK-MAXBW-NEXT:    [[TMP19]] = select <vscale x 4 x i1> [[ACTIVE_LANE_MASK]], <vscale x 4 x i32> [[TMP18]], <vscale x 4 x i32> [[VEC_PHI]]
+; CHECK-MAXBW-NEXT:    [[WIDE_MASKED_LOAD1:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr [[TMP15]], i32 1, <vscale x 16 x i1> [[ACTIVE_LANE_MASK]], <vscale x 16 x i8> poison)
+; CHECK-MAXBW-NEXT:    [[TMP16:%.*]] = sext <vscale x 16 x i8> [[WIDE_MASKED_LOAD1]] to <vscale x 16 x i32>
+; CHECK-MAXBW-NEXT:    [[TMP17:%.*]] = mul nsw <vscale x 16 x i32> [[TMP16]], [[TMP13]]
+; CHECK-MAXBW-NEXT:    [[TMP18:%.*]] = select <vscale x 16 x i1> [[ACTIVE_LANE_MASK]], <vscale x 16 x i32> [[TMP17]], <vscale x 16 x i32> zeroinitializer
+; CHECK-MAXBW-NEXT:    [[PARTIAL_REDUCE]] = call <vscale x 4 x i32> @llvm.experimental.vector.partial.reduce.add.nxv4i32.nxv16i32(<vscale x 4 x i32> [[VEC_PHI]], <vscale x 16 x i32> [[TMP18]])
 ; CHECK-MAXBW-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP4]]
-; CHECK-MAXBW-NEXT:    [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX]], i64 [[TMP9]])
-; CHECK-MAXBW-NEXT:    [[TMP20:%.*]] = xor <vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT]], splat (i1 true)
-; CHECK-MAXBW-NEXT:    [[TMP21:%.*]] = extractelement <vscale x 4 x i1> [[TMP20]], i32 0
-; CHECK-MAXBW-NEXT:    br i1 [[TMP21]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
+; CHECK-MAXBW-NEXT:    [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i64(i64 [[INDEX]], i64 [[TMP9]])
+; CHECK-MAXBW-NEXT:    [[TMP19:%.*]] = xor <vscale x 16 x i1> [[ACTIVE_LANE_MASK_NEXT]], splat (i1 true)
+; CHECK-MAXBW-NEXT:    [[TMP20:%.*]] = extractelement <vscale x 16 x i1> [[TMP19]], i32 0
+; CHECK-MAXBW-NEXT:    br i1 [[TMP20]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]]
 ; CHECK-MAXBW:       middle.block:
-; CHECK-MAXBW-NEXT:    [[TMP22:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[TMP19]])
+; CHECK-MAXBW-NEXT:    [[TMP21:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[PARTIAL_REDUCE]])
 ; CHECK-MAXBW-NEXT:    br i1 true, label [[EXIT:%.*]], label [[SCALAR_PH]]
 ; CHECK-MAXBW:       scalar.ph:
 ; CHECK-MAXBW-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
-; CHECK-MAXBW-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP22]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ]
+; CHECK-MAXBW-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP21]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ]
 ; CHECK-MAXBW-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK-MAXBW:       for.body:
 ; CHECK-MAXBW-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
 ; CHECK-MAXBW-NEXT:    [[ACCUM:%.*]] = phi i32 [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[ADD:%.*]], [[FOR_BODY]] ]
-; CHECK-MAXBW-NEXT:    [[GEP_A:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[IV]]
+; CHECK-MAXBW-NEXT:    [[GEP_A:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[IV]]
 ; CHECK-MAXBW-NEXT:    [[LOAD_A:%.*]] = load i8, ptr [[GEP_A]], align 1
 ; CHECK-MAXBW-NEXT:    [[EXT_A:%.*]] = sext i8 [[LOAD_A]] to i32
-; CHECK-MAXBW-NEXT:    [[GEP_A2:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[IV]]
-; CHECK-MAXBW-NEXT:    [[LOAD_B:%.*]] = load i8, ptr [[GEP_A2]], align 1
+; CHECK-MAXBW-NEXT:    [[GEP_B:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[IV]]
+; CHECK-MAXBW-NEXT:    [[LOAD_B:%.*]] = load i8, ptr [[GEP_B]], align 1
 ; CHECK-MAXBW-NEXT:    [[EXT_B:%.*]] = sext i8 [[LOAD_B]] to i32
 ; CHECK-MAXBW-NEXT:    [[MUL:%.*]] = mul nsw i32 [[EXT_B]], [[EXT_A]]
 ; CHECK-MAXBW-NEXT:    [[ADD]] = add nsw i32 [[MUL]], [[ACCUM]]
 ; CHECK-MAXBW-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
 ; CHECK-MAXBW-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; CHECK-MAXBW-NEXT:    br i1 [[EXITCOND_NOT]], label [[EXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]]
+; CHECK-MAXBW-NEXT:    br i1 [[EXITCOND_NOT]], label [[EXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP18:![0-9]+]]
 ; CHECK-MAXBW:       exit:
-; CHECK-MAXBW-NEXT:    [[ADD_LCSSA:%.*]] = phi i32 [ [[ADD]], [[FOR_BODY]] ], [ [[TMP22]], [[MIDDLE_BLOCK]] ]
+; CHECK-MAXBW-NEXT:    [[ADD_LCSSA:%.*]] = phi i32 [ [[ADD]], [[FOR_BODY]] ], [ [[TMP21]], [[MIDDLE_BLOCK]] ]
 ; CHECK-MAXBW-NEXT:    ret i32 [[ADD_LCSSA]]
 ;
 entry:
@@ -2041,11 +2304,11 @@ entry:
 for.body:                                         ; preds = %entry, %for.body
   %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
   %accum = phi i32 [ 0, %entry ], [ %add, %for.body ]
-  %gep.a = getelementptr inbounds i8, ptr %b, i64 %iv
+  %gep.a = getelementptr inbounds i8, ptr %a, i64 %iv
   %load.a = load i8, ptr %gep.a, align 1
   %ext.a = sext i8 %load.a to i32
-  %gep.a2 = getelementptr inbounds i8, ptr %a, i64 %iv
-  %load.b = load i8, ptr %gep.a2, align 1
+  %gep.b = getelementptr inbounds i8, ptr %b, i64 %iv
+  %load.b = load i8, ptr %gep.b, align 1
   %ext.b = sext i8 %load.b to i32
   %mul = mul nsw i32 %ext.b, %ext.a
   %add = add nsw i32 %mul, %accum
@@ -2088,7 +2351,7 @@ define i32 @not_dotp_extend_user(ptr %a, ptr %b) #0 {
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP14]] = add <vscale x 4 x i32> [[TMP13]], [[VEC_PHI]]
 ; CHECK-INTERLEAVE1-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP15:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-INTERLEAVE1-NEXT:    br i1 [[TMP15]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP17:![0-9]+]]
+; CHECK-INTERLEAVE1-NEXT:    br i1 [[TMP15]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP19:![0-9]+]]
 ; CHECK-INTERLEAVE1:       middle.block:
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP16:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[TMP14]])
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP17:%.*]] = call i32 @llvm.vscale.i32()
@@ -2114,7 +2377,7 @@ define i32 @not_dotp_extend_user(ptr %a, ptr %b) #0 {
 ; CHECK-INTERLEAVE1-NEXT:    [[ADD]] = add i32 [[MUL]], [[ACCUM]]
 ; CHECK-INTERLEAVE1-NEXT:    [[IV_NEXT]] = add i64 [[IV]], 1
 ; CHECK-INTERLEAVE1-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], 1024
-; CHECK-INTERLEAVE1-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_EXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP18:![0-9]+]]
+; CHECK-INTERLEAVE1-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_EXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP20:![0-9]+]]
 ; CHECK-INTERLEAVE1:       for.exit:
 ; CHECK-INTERLEAVE1-NEXT:    [[EXT_B_LCSSA:%.*]] = phi i32 [ [[EXT_B]], [[FOR_BODY]] ], [ [[TMP20]], [[MIDDLE_BLOCK]] ]
 ; CHECK-INTERLEAVE1-NEXT:    [[ADD_LCSSA:%.*]] = phi i32 [ [[ADD]], [[FOR_BODY]] ], [ [[TMP16]], [[MIDDLE_BLOCK]] ]
@@ -2164,7 +2427,7 @@ define i32 @not_dotp_extend_user(ptr %a, ptr %b) #0 {
 ; CHECK-INTERLEAVED-NEXT:    [[TMP24]] = add <vscale x 4 x i32> [[TMP22]], [[VEC_PHI1]]
 ; CHECK-INTERLEAVED-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
 ; CHECK-INTERLEAVED-NEXT:    [[TMP25:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-INTERLEAVED-NEXT:    br i1 [[TMP25]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP17:![0-9]+]]
+; CHECK-INTERLEAVED-NEXT:    br i1 [[TMP25]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP19:![0-9]+]]
 ; CHECK-INTERLEAVED:       middle.block:
 ; CHECK-INTERLEAVED-NEXT:    [[BIN_RDX:%.*]] = add <vscale x 4 x i32> [[TMP24]], [[TMP23]]
 ; CHECK-INTERLEAVED-NEXT:    [[TMP26:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[BIN_RDX]])
@@ -2191,7 +2454,7 @@ define i32 @not_dotp_extend_user(ptr %a, ptr %b) #0 {
 ; CHECK-INTERLEAVED-NEXT:    [[ADD]] = add i32 [[MUL]], [[ACCUM]]
 ; CHECK-INTERLEAVED-NEXT:    [[IV_NEXT]] = add i64 [[IV]], 1
 ; CHECK-INTERLEAVED-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], 1024
-; CHECK-INTERLEAVED-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_EXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP18:![0-9]+]]
+; CHECK-INTERLEAVED-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_EXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP20:![0-9]+]]
 ; CHECK-INTERLEAVED:       for.exit:
 ; CHECK-INTERLEAVED-NEXT:    [[EXT_B_LCSSA:%.*]] = phi i32 [ [[EXT_B]], [[FOR_BODY]] ], [ [[TMP30]], [[MIDDLE_BLOCK]] ]
 ; CHECK-INTERLEAVED-NEXT:    [[ADD_LCSSA:%.*]] = phi i32 [ [[ADD]], [[FOR_BODY]] ], [ [[TMP26]], [[MIDDLE_BLOCK]] ]
@@ -2228,7 +2491,7 @@ define i32 @not_dotp_extend_user(ptr %a, ptr %b) #0 {
 ; CHECK-MAXBW-NEXT:    [[TMP24]] = add <vscale x 8 x i32> [[TMP22]], [[VEC_PHI1]]
 ; CHECK-MAXBW-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
 ; CHECK-MAXBW-NEXT:    [[TMP25:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-MAXBW-NEXT:    br i1 [[TMP25]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP17:![0-9]+]]
+; CHECK-MAXBW-NEXT:    br i1 [[TMP25]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP19:![0-9]+]]
 ; CHECK-MAXBW:       middle.block:
 ; CHECK-MAXBW-NEXT:    [[TMP16:%.*]] = call i32 @llvm.vector.reduce.add.nxv8i32(<vscale x 8 x i32> [[TMP24]])
 ; CHECK-MAXBW-NEXT:    [[TMP17:%.*]] = call i32 @llvm.vscale.i32()
@@ -2254,7 +2517,7 @@ define i32 @not_dotp_extend_user(ptr %a, ptr %b) #0 {
 ; CHECK-MAXBW-NEXT:    [[ADD]] = add i32 [[MUL]], [[ACCUM]]
 ; CHECK-MAXBW-NEXT:    [[IV_NEXT]] = add i64 [[IV]], 1
 ; CHECK-MAXBW-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], 1024
-; CHECK-MAXBW-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_EXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP18:![0-9]+]]
+; CHECK-MAXBW-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_EXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP20:![0-9]+]]
 ; CHECK-MAXBW:       for.exit:
 ; CHECK-MAXBW-NEXT:    [[EXT_B_LCSSA:%.*]] = phi i32 [ [[EXT_B]], [[FOR_BODY]] ], [ [[TMP21]], [[MIDDLE_BLOCK]] ]
 ; CHECK-MAXBW-NEXT:    [[ADD_LCSSA:%.*]] = phi i32 [ [[ADD]], [[FOR_BODY]] ], [ [[TMP16]], [[MIDDLE_BLOCK]] ]
@@ -2317,7 +2580,7 @@ define i64 @dotp_cost_disagreement(ptr %a, ptr %b) #0 {
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP15]] = add <vscale x 2 x i64> [[VEC_PHI]], [[TMP14]]
 ; CHECK-INTERLEAVE1-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP16:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-INTERLEAVE1-NEXT:    br i1 [[TMP16]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP19:![0-9]+]]
+; CHECK-INTERLEAVE1-NEXT:    br i1 [[TMP16]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP21:![0-9]+]]
 ; CHECK-INTERLEAVE1:       middle.block:
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP17:%.*]] = call i64 @llvm.vector.reduce.add.nxv2i64(<vscale x 2 x i64> [[TMP15]])
 ; CHECK-INTERLEAVE1-NEXT:    [[CMP_N:%.*]] = icmp eq i64 16, [[N_VEC]]
@@ -2339,7 +2602,7 @@ define i64 @dotp_cost_disagreement(ptr %a, ptr %b) #0 {
 ; CHECK-INTERLEAVE1-NEXT:    [[MUL:%.*]] = mul nuw nsw i64 [[CONV3]], [[CONV]]
 ; CHECK-INTERLEAVE1-NEXT:    [[ADD]] = add i64 [[SUM]], [[MUL]]
 ; CHECK-INTERLEAVE1-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[I_IV_NEXT]], 16
-; CHECK-INTERLEAVE1-NEXT:    br i1 [[EXITCOND_NOT]], label [[EXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP20:![0-9]+]]
+; CHECK-INTERLEAVE1-NEXT:    br i1 [[EXITCOND_NOT]], label [[EXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP22:![0-9]+]]
 ; CHECK-INTERLEAVE1:       exit:
 ; CHECK-INTERLEAVE1-NEXT:    [[ADD_LCSSA:%.*]] = phi i64 [ [[ADD]], [[FOR_BODY]] ], [ [[TMP17]], [[MIDDLE_BLOCK]] ]
 ; CHECK-INTERLEAVE1-NEXT:    ret i64 [[ADD_LCSSA]]
@@ -2389,7 +2652,7 @@ define i64 @dotp_cost_disagreement(ptr %a, ptr %b) #0 {
 ; CHECK-INTERLEAVED-NEXT:    [[TMP25]] = add <vscale x 2 x i64> [[VEC_PHI1]], [[TMP23]]
 ; CHECK-INTERLEAVED-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
 ; CHECK-INTERLEAVED-NEXT:    [[TMP26:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-INTERLEAVED-NEXT:    br i1 [[TMP26]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP19:![0-9]+]]
+; CHECK-INTERLEAVED-NEXT:    br i1 [[TMP26]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP21:![0-9]+]]
 ; CHECK-INTERLEAVED:       middle.block:
 ; CHECK-INTERLEAVED-NEXT:    [[BIN_RDX:%.*]] = add <vscale x 2 x i64> [[TMP25]], [[TMP24]]
 ; CHECK-INTERLEAVED-NEXT:    [[TMP27:%.*]] = call i64 @llvm.vector.reduce.add.nxv2i64(<vscale x 2 x i64> [[BIN_RDX]])
@@ -2412,7 +2675,7 @@ define i64 @dotp_cost_disagreement(ptr %a, ptr %b) #0 {
 ; CHECK-INTERLEAVED-NEXT:    [[MUL:%.*]] = mul nuw nsw i64 [[CONV3]], [[CONV]]
 ; CHECK-INTERLEAVED-NEXT:    [[ADD]] = add i64 [[SUM]], [[MUL]]
 ; CHECK-INTERLEAVED-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[I_IV_NEXT]], 16
-; CHECK-INTERLEAVED-NEXT:    br i1 [[EXITCOND_NOT]], label [[EXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP20:![0-9]+]]
+; CHECK-INTERLEAVED-NEXT:    br i1 [[EXITCOND_NOT]], label [[EXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP22:![0-9]+]]
 ; CHECK-INTERLEAVED:       exit:
 ; CHECK-INTERLEAVED-NEXT:    [[ADD_LCSSA:%.*]] = phi i64 [ [[ADD]], [[FOR_BODY]] ], [ [[TMP27]], [[MIDDLE_BLOCK]] ]
 ; CHECK-INTERLEAVED-NEXT:    ret i64 [[ADD_LCSSA]]
@@ -2434,7 +2697,7 @@ define i64 @dotp_cost_disagreement(ptr %a, ptr %b) #0 {
 ; CHECK-MAXBW-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK-MAXBW:       vector.body:
 ; CHECK-MAXBW-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-MAXBW-NEXT:    [[VEC_PHI:%.*]] = phi <vscale x 1 x i64> [ zeroinitializer, [[VECTOR_PH]] ], [ [[PARTIAL_REDUCE:%.*]], [[VECTOR_BODY]] ]
+; CHECK-MAXBW-NEXT:    [[VEC_PHI:%.*]] = phi <vscale x 8 x i64> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP19:%.*]], [[VECTOR_BODY]] ]
 ; CHECK-MAXBW-NEXT:    [[TMP6:%.*]] = add i64 [[INDEX]], 0
 ; CHECK-MAXBW-NEXT:    [[TMP7:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 [[TMP6]]
 ; CHECK-MAXBW-NEXT:    [[TMP8:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP7]], i32 0
@@ -2446,12 +2709,12 @@ define i64 @dotp_cost_disagreement(ptr %a, ptr %b) #0 {
 ; CHECK-MAXBW-NEXT:    [[WIDE_LOAD1:%.*]] = load <vscale x 8 x i8>, ptr [[TMP12]], align 1
 ; CHECK-MAXBW-NEXT:    [[TMP13:%.*]] = zext <vscale x 8 x i8> [[WIDE_LOAD1]] to <vscale x 8 x i64>
 ; CHECK-MAXBW-NEXT:    [[TMP14:%.*]] = mul nuw nsw <vscale x 8 x i64> [[TMP13]], [[TMP9]]
-; CHECK-MAXBW-NEXT:    [[PARTIAL_REDUCE]] = call <vscale x 1 x i64> @llvm.experimental.vector.partial.reduce.add.nxv1i64.nxv8i64(<vscale x 1 x i64> [[VEC_PHI]], <vscale x 8 x i64> [[TMP14]])
+; CHECK-MAXBW-NEXT:    [[TMP19]] = add <vscale x 8 x i64> [[VEC_PHI]], [[TMP14]]
 ; CHECK-MAXBW-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
 ; CHECK-MAXBW-NEXT:    [[TMP15:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-MAXBW-NEXT:    br i1 [[TMP15]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP19:![0-9]+]]
+; CHECK-MAXBW-NEXT:    br i1 [[TMP15]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP21:![0-9]+]]
 ; CHECK-MAXBW:       middle.block:
-; CHECK-MAXBW-NEXT:    [[TMP16:%.*]] = call i64 @llvm.vector.reduce.add.nxv1i64(<vscale x 1 x i64> [[PARTIAL_REDUCE]])
+; CHECK-MAXBW-NEXT:    [[TMP16:%.*]] = call i64 @llvm.vector.reduce.add.nxv8i64(<vscale x 8 x i64> [[TMP19]])
 ; CHECK-MAXBW-NEXT:    [[CMP_N:%.*]] = icmp eq i64 16, [[N_VEC]]
 ; CHECK-MAXBW-NEXT:    br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
 ; CHECK-MAXBW:       scalar.ph:
@@ -2471,7 +2734,7 @@ define i64 @dotp_cost_disagreement(ptr %a, ptr %b) #0 {
 ; CHECK-MAXBW-NEXT:    [[MUL:%.*]] = mul nuw nsw i64 [[CONV3]], [[CONV]]
 ; CHECK-MAXBW-NEXT:    [[ADD]] = add i64 [[SUM]], [[MUL]]
 ; CHECK-MAXBW-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[I_IV_NEXT]], 16
-; CHECK-MAXBW-NEXT:    br i1 [[EXITCOND_NOT]], label [[EXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP20:![0-9]+]]
+; CHECK-MAXBW-NEXT:    br i1 [[EXITCOND_NOT]], label [[EXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP22:![0-9]+]]
 ; CHECK-MAXBW:       exit:
 ; CHECK-MAXBW-NEXT:    [[ADD_LCSSA:%.*]] = phi i64 [ [[ADD]], [[FOR_BODY]] ], [ [[TMP16]], [[MIDDLE_BLOCK]] ]
 ; CHECK-MAXBW-NEXT:    ret i64 [[ADD_LCSSA]]
@@ -2588,7 +2851,7 @@ define void @not_dotp_not_phi2(ptr %matrix, i32 %n) #0 {
 ; CHECK-INTERLEAVED-NEXT:    [[TMP23]] = add i32 [[TMP21]], [[TMP15]]
 ; CHECK-INTERLEAVED-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
 ; CHECK-INTERLEAVED-NEXT:    [[TMP24:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-INTERLEAVED-NEXT:    br i1 [[TMP24]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP21:![0-9]+]]
+; CHECK-INTERLEAVED-NEXT:    br i1 [[TMP24]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP23:![0-9]+]]
 ; CHECK-INTERLEAVED:       middle.block:
 ; CHECK-INTERLEAVED-NEXT:    [[BIN_RDX:%.*]] = add i32 [[TMP23]], [[TMP22]]
 ; CHECK-INTERLEAVED-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]]
@@ -2615,7 +2878,7 @@ define void @not_dotp_not_phi2(ptr %matrix, i32 %n) #0 {
 ; CHECK-INTERLEAVED-NEXT:    [[SCEVGEP]] = getelementptr i8, ptr [[PTR]], i64 16
 ; CHECK-INTERLEAVED-NEXT:    [[IV_NEXT]] = add nuw nsw i32 [[IV]], 1
 ; CHECK-INTERLEAVED-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i32 [[IV_NEXT]], [[N]]
-; CHECK-INTERLEAVED-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_EXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP22:![0-9]+]]
+; CHECK-INTERLEAVED-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_EXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP24:![0-9]+]]
 ; CHECK-INTERLEAVED:       for.exit:
 ; CHECK-INTERLEAVED-NEXT:    [[ADD_1_LCSSA:%.*]] = phi i32 [ [[ADD_1]], [[FOR_BODY]] ], [ [[BIN_RDX]], [[MIDDLE_BLOCK]] ]
 ; CHECK-INTERLEAVED-NEXT:    [[ADD_FLOAT:%.*]] = sitofp i32 [[ADD_1_LCSSA]] to float
@@ -2732,7 +2995,7 @@ define i64 @not_dotp_ext_outside_plan(ptr %a, i16 %b, i64 %n) #0 {
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP5]] = add <8 x i64> [[TMP4]], [[VEC_PHI]]
 ; CHECK-INTERLEAVE1-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-INTERLEAVE1-NEXT:    br i1 [[TMP6]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP21:![0-9]+]]
+; CHECK-INTERLEAVE1-NEXT:    br i1 [[TMP6]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP23:![0-9]+]]
 ; CHECK-INTERLEAVE1:       middle.block:
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vector.reduce.add.v8i64(<8 x i64> [[TMP5]])
 ; CHECK-INTERLEAVE1-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
@@ -2751,7 +3014,7 @@ define i64 @not_dotp_ext_outside_plan(ptr %a, i16 %b, i64 %n) #0 {
 ; CHECK-INTERLEAVE1-NEXT:    [[ADD]] = add i64 [[MUL]], [[ACCUM]]
 ; CHECK-INTERLEAVE1-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
 ; CHECK-INTERLEAVE1-NEXT:    [[CMP_1:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; CHECK-INTERLEAVE1-NEXT:    br i1 [[CMP_1]], label [[EXIT_LOOPEXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP22:![0-9]+]]
+; CHECK-INTERLEAVE1-NEXT:    br i1 [[CMP_1]], label [[EXIT_LOOPEXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP24:![0-9]+]]
 ; CHECK-INTERLEAVE1:       exit.loopexit:
 ; CHECK-INTERLEAVE1-NEXT:    [[ADD_LCSSA:%.*]] = phi i64 [ [[ADD]], [[FOR_BODY]] ], [ [[TMP7]], [[MIDDLE_BLOCK]] ]
 ; CHECK-INTERLEAVE1-NEXT:    br label [[EXIT]]
@@ -2792,7 +3055,7 @@ define i64 @not_dotp_ext_outside_plan(ptr %a, i16 %b, i64 %n) #0 {
 ; CHECK-INTERLEAVED-NEXT:    [[TMP9]] = add <8 x i64> [[TMP7]], [[VEC_PHI1]]
 ; CHECK-INTERLEAVED-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
 ; CHECK-INTERLEAVED-NEXT:    [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-INTERLEAVED-NEXT:    br i1 [[TMP10]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP23:![0-9]+]]
+; CHECK-INTERLEAVED-NEXT:    br i1 [[TMP10]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP25:![0-9]+]]
 ; CHECK-INTERLEAVED:       middle.block:
 ; CHECK-INTERLEAVED-NEXT:    [[BIN_RDX:%.*]] = add <8 x i64> [[TMP9]], [[TMP8]]
 ; CHECK-INTERLEAVED-NEXT:    [[TMP11:%.*]] = call i64 @llvm.vector.reduce.add.v8i64(<8 x i64> [[BIN_RDX]])
@@ -2812,7 +3075,7 @@ define i64 @not_dotp_ext_outside_plan(ptr %a, i16 %b, i64 %n) #0 {
 ; CHECK-INTERLEAVED-NEXT:    [[ADD]] = add i64 [[MUL]], [[ACCUM]]
 ; CHECK-INTERLEAVED-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
 ; CHECK-INTERLEAVED-NEXT:    [[CMP_1:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; CHECK-INTERLEAVED-NEXT:    br i1 [[CMP_1]], label [[EXIT_LOOPEXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP24:![0-9]+]]
+; CHECK-INTERLEAVED-NEXT:    br i1 [[CMP_1]], label [[EXIT_LOOPEXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP26:![0-9]+]]
 ; CHECK-INTERLEAVED:       exit.loopexit:
 ; CHECK-INTERLEAVED-NEXT:    [[ADD_LCSSA:%.*]] = phi i64 [ [[ADD]], [[FOR_BODY]] ], [ [[TMP11]], [[MIDDLE_BLOCK]] ]
 ; CHECK-INTERLEAVED-NEXT:    br label [[EXIT]]
@@ -2853,7 +3116,7 @@ define i64 @not_dotp_ext_outside_plan(ptr %a, i16 %b, i64 %n) #0 {
 ; CHECK-MAXBW-NEXT:    [[TMP11]] = add <vscale x 4 x i64> [[TMP10]], [[VEC_PHI]]
 ; CHECK-MAXBW-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
 ; CHECK-MAXBW-NEXT:    [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-MAXBW-NEXT:    br i1 [[TMP12]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP21:![0-9]+]]
+; CHECK-MAXBW-NEXT:    br i1 [[TMP12]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP23:![0-9]+]]
 ; CHECK-MAXBW:       middle.block:
 ; CHECK-MAXBW-NEXT:    [[TMP13:%.*]] = call i64 @llvm.vector.reduce.add.nxv4i64(<vscale x 4 x i64> [[TMP11]])
 ; CHECK-MAXBW-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
@@ -2872,7 +3135,7 @@ define i64 @not_dotp_ext_outside_plan(ptr %a, i16 %b, i64 %n) #0 {
 ; CHECK-MAXBW-NEXT:    [[ADD]] = add i64 [[MUL]], [[ACCUM]]
 ; CHECK-MAXBW-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
 ; CHECK-MAXBW-NEXT:    [[CMP_1:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; CHECK-MAXBW-NEXT:    br i1 [[CMP_1]], label [[EXIT_LOOPEXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP22:![0-9]+]]
+; CHECK-MAXBW-NEXT:    br i1 [[CMP_1]], label [[EXIT_LOOPEXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP24:![0-9]+]]
 ; CHECK-MAXBW:       exit.loopexit:
 ; CHECK-MAXBW-NEXT:    [[ADD_LCSSA:%.*]] = phi i64 [ [[ADD]], [[FOR_BODY]] ], [ [[TMP13]], [[MIDDLE_BLOCK]] ]
 ; CHECK-MAXBW-NEXT:    br label [[EXIT]]
@@ -2933,7 +3196,7 @@ define i64 @not_dotp_ext_outside_plan2(ptr %a, i16 %b, i64 %n) #0 {
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP5]] = add <8 x i64> [[TMP4]], [[VEC_PHI]]
 ; CHECK-INTERLEAVE1-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-INTERLEAVE1-NEXT:    br i1 [[TMP6]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP23:![0-9]+]]
+; CHECK-INTERLEAVE1-NEXT:    br i1 [[TMP6]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP25:![0-9]+]]
 ; CHECK-INTERLEAVE1:       middle.block:
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vector.reduce.add.v8i64(<8 x i64> [[TMP5]])
 ; CHECK-INTERLEAVE1-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
@@ -2952,7 +3215,7 @@ define i64 @not_dotp_ext_outside_plan2(ptr %a, i16 %b, i64 %n) #0 {
 ; CHECK-INTERLEAVE1-NEXT:    [[ADD]] = add i64 [[MUL]], [[ACCUM]]
 ; CHECK-INTERLEAVE1-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
 ; CHECK-INTERLEAVE1-NEXT:    [[CMP_1:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; CHECK-INTERLEAVE1-NEXT:    br i1 [[CMP_1]], label [[EXIT_LOOPEXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP24:![0-9]+]]
+; CHECK-INTERLEAVE1-NEXT:    br i1 [[CMP_1]], label [[EXIT_LOOPEXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP26:![0-9]+]]
 ; CHECK-INTERLEAVE1:       exit.loopexit:
 ; CHECK-INTERLEAVE1-NEXT:    [[ADD_LCSSA:%.*]] = phi i64 [ [[ADD]], [[FOR_BODY]] ], [ [[TMP7]], [[MIDDLE_BLOCK]] ]
 ; CHECK-INTERLEAVE1-NEXT:    br label [[EXIT]]
@@ -2993,7 +3256,7 @@ define i64 @not_dotp_ext_outside_plan2(ptr %a, i16 %b, i64 %n) #0 {
 ; CHECK-INTERLEAVED-NEXT:    [[TMP9]] = add <8 x i64> [[TMP7]], [[VEC_PHI1]]
 ; CHECK-INTERLEAVED-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
 ; CHECK-INTERLEAVED-NEXT:    [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-INTERLEAVED-NEXT:    br i1 [[TMP10]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP25:![0-9]+]]
+; CHECK-INTERLEAVED-NEXT:    br i1 [[TMP10]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP27:![0-9]+]]
 ; CHECK-INTERLEAVED:       middle.block:
 ; CHECK-INTERLEAVED-NEXT:    [[BIN_RDX:%.*]] = add <8 x i64> [[TMP9]], [[TMP8]]
 ; CHECK-INTERLEAVED-NEXT:    [[TMP11:%.*]] = call i64 @llvm.vector.reduce.add.v8i64(<8 x i64> [[BIN_RDX]])
@@ -3013,7 +3276,7 @@ define i64 @not_dotp_ext_outside_plan2(ptr %a, i16 %b, i64 %n) #0 {
 ; CHECK-INTERLEAVED-NEXT:    [[ADD]] = add i64 [[MUL]], [[ACCUM]]
 ; CHECK-INTERLEAVED-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
 ; CHECK-INTERLEAVED-NEXT:    [[CMP_1:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; CHECK-INTERLEAVED-NEXT:    br i1 [[CMP_1]], label [[EXIT_LOOPEXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP26:![0-9]+]]
+; CHECK-INTERLEAVED-NEXT:    br i1 [[CMP_1]], label [[EXIT_LOOPEXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP28:![0-9]+]]
 ; CHECK-INTERLEAVED:       exit.loopexit:
 ; CHECK-INTERLEAVED-NEXT:    [[ADD_LCSSA:%.*]] = phi i64 [ [[ADD]], [[FOR_BODY]] ], [ [[TMP11]], [[MIDDLE_BLOCK]] ]
 ; CHECK-INTERLEAVED-NEXT:    br label [[EXIT]]
@@ -3054,7 +3317,7 @@ define i64 @not_dotp_ext_outside_plan2(ptr %a, i16 %b, i64 %n) #0 {
 ; CHECK-MAXBW-NEXT:    [[TMP11]] = add <vscale x 4 x i64> [[TMP10]], [[VEC_PHI]]
 ; CHECK-MAXBW-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
 ; CHECK-MAXBW-NEXT:    [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-MAXBW-NEXT:    br i1 [[TMP12]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP23:![0-9]+]]
+; CHECK-MAXBW-NEXT:    br i1 [[TMP12]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP25:![0-9]+]]
 ; CHECK-MAXBW:       middle.block:
 ; CHECK-MAXBW-NEXT:    [[TMP13:%.*]] = call i64 @llvm.vector.reduce.add.nxv4i64(<vscale x 4 x i64> [[TMP11]])
 ; CHECK-MAXBW-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
@@ -3073,7 +3336,7 @@ define i64 @not_dotp_ext_outside_plan2(ptr %a, i16 %b, i64 %n) #0 {
 ; CHECK-MAXBW-NEXT:    [[ADD]] = add i64 [[MUL]], [[ACCUM]]
 ; CHECK-MAXBW-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
 ; CHECK-MAXBW-NEXT:    [[CMP_1:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; CHECK-MAXBW-NEXT:    br i1 [[CMP_1]], label [[EXIT_LOOPEXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP24:![0-9]+]]
+; CHECK-MAXBW-NEXT:    br i1 [[CMP_1]], label [[EXIT_LOOPEXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP26:![0-9]+]]
 ; CHECK-MAXBW:       exit.loopexit:
 ; CHECK-MAXBW-NEXT:    [[ADD_LCSSA:%.*]] = phi i64 [ [[ADD]], [[FOR_BODY]] ], [ [[TMP13]], [[MIDDLE_BLOCK]] ]
 ; CHECK-MAXBW-NEXT:    br label [[EXIT]]
@@ -3106,89 +3369,1039 @@ exit:                                 ; preds = %for.cond.cleanup.loopexit, %ent
   ret i64 %result
 }
 
+define dso_local i32 @not_dotp_vscale1(ptr %a, ptr %b, i32 %n, i64 %cost) #0 {
+; CHECK-INTERLEAVE1-LABEL: define dso_local i32 @not_dotp_vscale1(
+; CHECK-INTERLEAVE1-SAME: ptr [[A:%.*]], ptr [[B:%.*]], i32 [[N:%.*]], i64 [[COST:%.*]]) #[[ATTR0]] {
+; CHECK-INTERLEAVE1-NEXT:  entry:
+; CHECK-INTERLEAVE1-NEXT:    [[CMP:%.*]] = icmp sgt i32 [[N]], 0
+; CHECK-INTERLEAVE1-NEXT:    br i1 [[CMP]], label [[FOR_BODY_PREHEADER:%.*]], label [[EXIT:%.*]]
+; CHECK-INTERLEAVE1:       for.body.preheader:
+; CHECK-INTERLEAVE1-NEXT:    [[TMP0:%.*]] = zext i32 [[N]] to i64
+; CHECK-INTERLEAVE1-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-INTERLEAVE1-NEXT:    [[TMP2:%.*]] = mul i64 [[TMP1]], 2
+; CHECK-INTERLEAVE1-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], [[TMP2]]
+; CHECK-INTERLEAVE1-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK-INTERLEAVE1:       vector.ph:
+; CHECK-INTERLEAVE1-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-INTERLEAVE1-NEXT:    [[TMP4:%.*]] = mul i64 [[TMP3]], 2
+; CHECK-INTERLEAVE1-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[TMP0]], [[TMP4]]
+; CHECK-INTERLEAVE1-NEXT:    [[N_VEC:%.*]] = sub i64 [[TMP0]], [[N_MOD_VF]]
+; CHECK-INTERLEAVE1-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-INTERLEAVE1-NEXT:    [[TMP6:%.*]] = mul i64 [[TMP5]], 2
+; CHECK-INTERLEAVE1-NEXT:    [[TMP7:%.*]] = trunc i64 [[N_VEC]] to i32
+; CHECK-INTERLEAVE1-NEXT:    [[TMP8:%.*]] = getelementptr i8, ptr [[A]], i64 [[N_VEC]]
+; CHECK-INTERLEAVE1-NEXT:    [[TMP9:%.*]] = getelementptr i8, ptr [[B]], i64 [[N_VEC]]
+; CHECK-INTERLEAVE1-NEXT:    [[TMP10:%.*]] = insertelement <vscale x 2 x i64> zeroinitializer, i64 [[COST]], i32 0
+; CHECK-INTERLEAVE1-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK-INTERLEAVE1:       vector.body:
+; CHECK-INTERLEAVE1-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-INTERLEAVE1-NEXT:    [[VEC_PHI:%.*]] = phi <vscale x 2 x i64> [ [[TMP10]], [[VECTOR_PH]] ], [ [[TMP18:%.*]], [[VECTOR_BODY]] ]
+; CHECK-INTERLEAVE1-NEXT:    [[TMP11:%.*]] = add i64 [[INDEX]], 0
+; CHECK-INTERLEAVE1-NEXT:    [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[A]], i64 [[TMP11]]
+; CHECK-INTERLEAVE1-NEXT:    [[TMP12:%.*]] = add i64 [[INDEX]], 0
+; CHECK-INTERLEAVE1-NEXT:    [[NEXT_GEP1:%.*]] = getelementptr i8, ptr [[B]], i64 [[TMP12]]
+; CHECK-INTERLEAVE1-NEXT:    [[TMP13:%.*]] = getelementptr i8, ptr [[NEXT_GEP]], i32 0
+; CHECK-INTERLEAVE1-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 2 x i8>, ptr [[TMP13]], align 1
+; CHECK-INTERLEAVE1-NEXT:    [[TMP14:%.*]] = zext <vscale x 2 x i8> [[WIDE_LOAD]] to <vscale x 2 x i64>
+; CHECK-INTERLEAVE1-NEXT:    [[TMP15:%.*]] = getelementptr i8, ptr [[NEXT_GEP1]], i32 0
+; CHECK-INTERLEAVE1-NEXT:    [[WIDE_LOAD2:%.*]] = load <vscale x 2 x i8>, ptr [[TMP15]], align 1
+; CHECK-INTERLEAVE1-NEXT:    [[TMP16:%.*]] = zext <vscale x 2 x i8> [[WIDE_LOAD2]] to <vscale x 2 x i64>
+; CHECK-INTERLEAVE1-NEXT:    [[TMP17:%.*]] = mul nuw nsw <vscale x 2 x i64> [[TMP16]], [[TMP14]]
+; CHECK-INTERLEAVE1-NEXT:    [[TMP18]] = add <vscale x 2 x i64> [[TMP17]], [[VEC_PHI]]
+; CHECK-INTERLEAVE1-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP6]]
+; CHECK-INTERLEAVE1-NEXT:    [[TMP19:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-INTERLEAVE1-NEXT:    br i1 [[TMP19]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP27:![0-9]+]]
+; CHECK-INTERLEAVE1:       middle.block:
+; CHECK-INTERLEAVE1-NEXT:    [[TMP20:%.*]] = call i64 @llvm.vector.reduce.add.nxv2i64(<vscale x 2 x i64> [[TMP18]])
+; CHECK-INTERLEAVE1-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]]
+; CHECK-INTERLEAVE1-NEXT:    br i1 [[CMP_N]], label [[EXIT_LOOPEXIT:%.*]], label [[SCALAR_PH]]
+; CHECK-INTERLEAVE1:       scalar.ph:
+; CHECK-INTERLEAVE1-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ [[TMP7]], [[MIDDLE_BLOCK]] ], [ 0, [[FOR_BODY_PREHEADER]] ]
+; CHECK-INTERLEAVE1-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i64 [ [[TMP20]], [[MIDDLE_BLOCK]] ], [ [[COST]], [[FOR_BODY_PREHEADER]] ]
+; CHECK-INTERLEAVE1-NEXT:    [[BC_RESUME_VAL3:%.*]] = phi ptr [ [[TMP8]], [[MIDDLE_BLOCK]] ], [ [[A]], [[FOR_BODY_PREHEADER]] ]
+; CHECK-INTERLEAVE1-NEXT:    [[BC_RESUME_VAL4:%.*]] = phi ptr [ [[TMP9]], [[MIDDLE_BLOCK]] ], [ [[B]], [[FOR_BODY_PREHEADER]] ]
+; CHECK-INTERLEAVE1-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK-INTERLEAVE1:       for.body:
+; CHECK-INTERLEAVE1-NEXT:    [[IV:%.*]] = phi i32 [ [[IV_NEXT:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
+; CHECK-INTERLEAVE1-NEXT:    [[ACCUM:%.*]] = phi i64 [ [[ADD:%.*]], [[FOR_BODY]] ], [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ]
+; CHECK-INTERLEAVE1-NEXT:    [[A_PTR:%.*]] = phi ptr [ [[A_GEP:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL3]], [[SCALAR_PH]] ]
+; CHECK-INTERLEAVE1-NEXT:    [[B_PTR:%.*]] = phi ptr [ [[B_GEP:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL4]], [[SCALAR_PH]] ]
+; CHECK-INTERLEAVE1-NEXT:    [[A_LOAD:%.*]] = load i8, ptr [[A_PTR]], align 1
+; CHECK-INTERLEAVE1-NEXT:    [[A_EXT:%.*]] = zext i8 [[A_LOAD]] to i64
+; CHECK-INTERLEAVE1-NEXT:    [[B_LOAD:%.*]] = load i8, ptr [[B_PTR]], align 1
+; CHECK-INTERLEAVE1-NEXT:    [[B_EXT:%.*]] = zext i8 [[B_LOAD]] to i64
+; CHECK-INTERLEAVE1-NEXT:    [[MUL:%.*]] = mul nuw nsw i64 [[B_EXT]], [[A_EXT]]
+; CHECK-INTERLEAVE1-NEXT:    [[ADD]] = add nsw i64 [[MUL]], [[ACCUM]]
+; CHECK-INTERLEAVE1-NEXT:    [[A_GEP]] = getelementptr inbounds nuw i8, ptr [[A_PTR]], i64 1
+; CHECK-INTERLEAVE1-NEXT:    [[B_GEP]] = getelementptr inbounds nuw i8, ptr [[B_PTR]], i64 1
+; CHECK-INTERLEAVE1-NEXT:    [[IV_NEXT]] = add nuw nsw i32 [[IV]], 1
+; CHECK-INTERLEAVE1-NEXT:    [[CMP_2:%.*]] = icmp eq i32 [[IV_NEXT]], [[N]]
+; CHECK-INTERLEAVE1-NEXT:    br i1 [[CMP_2]], label [[EXIT_LOOPEXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP28:![0-9]+]]
+; CHECK-INTERLEAVE1:       exit.loopexit:
+; CHECK-INTERLEAVE1-NEXT:    [[ADD_LCSSA:%.*]] = phi i64 [ [[ADD]], [[FOR_BODY]] ], [ [[TMP20]], [[MIDDLE_BLOCK]] ]
+; CHECK-INTERLEAVE1-NEXT:    br label [[EXIT]]
+; CHECK-INTERLEAVE1:       exit:
+; CHECK-INTERLEAVE1-NEXT:    [[COST_RESULT:%.*]] = phi i64 [ [[COST]], [[ENTRY:%.*]] ], [ [[ADD_LCSSA]], [[EXIT_LOOPEXIT]] ]
+; CHECK-INTERLEAVE1-NEXT:    [[RESULT:%.*]] = trunc i64 [[COST_RESULT]] to i32
+; CHECK-INTERLEAVE1-NEXT:    ret i32 [[RESULT]]
+;
+; CHECK-INTERLEAVED-LABEL: define dso_local i32 @not_dotp_vscale1(
+; CHECK-INTERLEAVED-SAME: ptr [[A:%.*]], ptr [[B:%.*]], i32 [[N:%.*]], i64 [[COST:%.*]]) #[[ATTR0]] {
+; CHECK-INTERLEAVED-NEXT:  entry:
+; CHECK-INTERLEAVED-NEXT:    [[CMP:%.*]] = icmp sgt i32 [[N]], 0
+; CHECK-INTERLEAVED-NEXT:    br i1 [[CMP]], label [[FOR_BODY_PREHEADER:%.*]], label [[EXIT:%.*]]
+; CHECK-INTERLEAVED:       for.body.preheader:
+; CHECK-INTERLEAVED-NEXT:    [[TMP0:%.*]] = zext i32 [[N]] to i64
+; CHECK-INTERLEAVED-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-INTERLEAVED-NEXT:    [[TMP2:%.*]] = mul i64 [[TMP1]], 4
+; CHECK-INTERLEAVED-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], [[TMP2]]
+; CHECK-INTERLEAVED-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK-INTERLEAVED:       vector.ph:
+; CHECK-INTERLEAVED-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-INTERLEAVED-NEXT:    [[TMP4:%.*]] = mul i64 [[TMP3]], 4
+; CHECK-INTERLEAVED-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[TMP0]], [[TMP4]]
+; CHECK-INTERLEAVED-NEXT:    [[N_VEC:%.*]] = sub i64 [[TMP0]], [[N_MOD_VF]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-INTERLEAVED-NEXT:    [[TMP6:%.*]] = mul i64 [[TMP5]], 4
+; CHECK-INTERLEAVED-NEXT:    [[TMP7:%.*]] = trunc i64 [[N_VEC]] to i32
+; CHECK-INTERLEAVED-NEXT:    [[TMP8:%.*]] = getelementptr i8, ptr [[A]], i64 [[N_VEC]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP9:%.*]] = getelementptr i8, ptr [[B]], i64 [[N_VEC]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP10:%.*]] = insertelement <vscale x 2 x i64> zeroinitializer, i64 [[COST]], i32 0
+; CHECK-INTERLEAVED-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK-INTERLEAVED:       vector.body:
+; CHECK-INTERLEAVED-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-INTERLEAVED-NEXT:    [[VEC_PHI:%.*]] = phi <vscale x 2 x i64> [ [[TMP10]], [[VECTOR_PH]] ], [ [[TMP27:%.*]], [[VECTOR_BODY]] ]
+; CHECK-INTERLEAVED-NEXT:    [[VEC_PHI1:%.*]] = phi <vscale x 2 x i64> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP28:%.*]], [[VECTOR_BODY]] ]
+; CHECK-INTERLEAVED-NEXT:    [[TMP11:%.*]] = add i64 [[INDEX]], 0
+; CHECK-INTERLEAVED-NEXT:    [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[A]], i64 [[TMP11]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP12:%.*]] = add i64 [[INDEX]], 0
+; CHECK-INTERLEAVED-NEXT:    [[NEXT_GEP2:%.*]] = getelementptr i8, ptr [[B]], i64 [[TMP12]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP13:%.*]] = getelementptr i8, ptr [[NEXT_GEP]], i32 0
+; CHECK-INTERLEAVED-NEXT:    [[TMP14:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-INTERLEAVED-NEXT:    [[TMP15:%.*]] = mul i64 [[TMP14]], 2
+; CHECK-INTERLEAVED-NEXT:    [[TMP16:%.*]] = getelementptr i8, ptr [[NEXT_GEP]], i64 [[TMP15]]
+; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 2 x i8>, ptr [[TMP13]], align 1
+; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD3:%.*]] = load <vscale x 2 x i8>, ptr [[TMP16]], align 1
+; CHECK-INTERLEAVED-NEXT:    [[TMP17:%.*]] = zext <vscale x 2 x i8> [[WIDE_LOAD]] to <vscale x 2 x i64>
+; CHECK-INTERLEAVED-NEXT:    [[TMP18:%.*]] = zext <vscale x 2 x i8> [[WIDE_LOAD3]] to <vscale x 2 x i64>
+; CHECK-INTERLEAVED-NEXT:    [[TMP19:%.*]] = getelementptr i8, ptr [[NEXT_GEP2]], i32 0
+; CHECK-INTERLEAVED-NEXT:    [[TMP20:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-INTERLEAVED-NEXT:    [[TMP21:%.*]] = mul i64 [[TMP20]], 2
+; CHECK-INTERLEAVED-NEXT:    [[TMP22:%.*]] = getelementptr i8, ptr [[NEXT_GEP2]], i64 [[TMP21]]
+; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD4:%.*]] = load <vscale x 2 x i8>, ptr [[TMP19]], align 1
+; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD5:%.*]] = load <vscale x 2 x i8>, ptr [[TMP22]], align 1
+; CHECK-INTERLEAVED-NEXT:    [[TMP23:%.*]] = zext <vscale x 2 x i8> [[WIDE_LOAD4]] to <vscale x 2 x i64>
+; CHECK-INTERLEAVED-NEXT:    [[TMP24:%.*]] = zext <vscale x 2 x i8> [[WIDE_LOAD5]] to <vscale x 2 x i64>
+; CHECK-INTERLEAVED-NEXT:    [[TMP25:%.*]] = mul nuw nsw <vscale x 2 x i64> [[TMP23]], [[TMP17]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP26:%.*]] = mul nuw nsw <vscale x 2 x i64> [[TMP24]], [[TMP18]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP27]] = add <vscale x 2 x i64> [[TMP25]], [[VEC_PHI]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP28]] = add <vscale x 2 x i64> [[TMP26]], [[VEC_PHI1]]
+; CHECK-INTERLEAVED-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP6]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP29:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-INTERLEAVED-NEXT:    br i1 [[TMP29]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP29:![0-9]+]]
+; CHECK-INTERLEAVED:       middle.block:
+; CHECK-INTERLEAVED-NEXT:    [[BIN_RDX:%.*]] = add <vscale x 2 x i64> [[TMP28]], [[TMP27]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP30:%.*]] = call i64 @llvm.vector.reduce.add.nxv2i64(<vscale x 2 x i64> [[BIN_RDX]])
+; CHECK-INTERLEAVED-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]]
+; CHECK-INTERLEAVED-NEXT:    br i1 [[CMP_N]], label [[EXIT_LOOPEXIT:%.*]], label [[SCALAR_PH]]
+; CHECK-INTERLEAVED:       scalar.ph:
+; CHECK-INTERLEAVED-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ [[TMP7]], [[MIDDLE_BLOCK]] ], [ 0, [[FOR_BODY_PREHEADER]] ]
+; CHECK-INTERLEAVED-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i64 [ [[TMP30]], [[MIDDLE_BLOCK]] ], [ [[COST]], [[FOR_BODY_PREHEADER]] ]
+; CHECK-INTERLEAVED-NEXT:    [[BC_RESUME_VAL6:%.*]] = phi ptr [ [[TMP8]], [[MIDDLE_BLOCK]] ], [ [[A]], [[FOR_BODY_PREHEADER]] ]
+; CHECK-INTERLEAVED-NEXT:    [[BC_RESUME_VAL7:%.*]] = phi ptr [ [[TMP9]], [[MIDDLE_BLOCK]] ], [ [[B]], [[FOR_BODY_PREHEADER]] ]
+; CHECK-INTERLEAVED-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK-INTERLEAVED:       for.body:
+; CHECK-INTERLEAVED-NEXT:    [[IV:%.*]] = phi i32 [ [[IV_NEXT:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
+; CHECK-INTERLEAVED-NEXT:    [[ACCUM:%.*]] = phi i64 [ [[ADD:%.*]], [[FOR_BODY]] ], [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ]
+; CHECK-INTERLEAVED-NEXT:    [[A_PTR:%.*]] = phi ptr [ [[A_GEP:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL6]], [[SCALAR_PH]] ]
+; CHECK-INTERLEAVED-NEXT:    [[B_PTR:%.*]] = phi ptr [ [[B_GEP:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL7]], [[SCALAR_PH]] ]
+; CHECK-INTERLEAVED-NEXT:    [[A_LOAD:%.*]] = load i8, ptr [[A_PTR]], align 1
+; CHECK-INTERLEAVED-NEXT:    [[A_EXT:%.*]] = zext i8 [[A_LOAD]] to i64
+; CHECK-INTERLEAVED-NEXT:    [[B_LOAD:%.*]] = load i8, ptr [[B_PTR]], align 1
+; CHECK-INTERLEAVED-NEXT:    [[B_EXT:%.*]] = zext i8 [[B_LOAD]] to i64
+; CHECK-INTERLEAVED-NEXT:    [[MUL:%.*]] = mul nuw nsw i64 [[B_EXT]], [[A_EXT]]
+; CHECK-INTERLEAVED-NEXT:    [[ADD]] = add nsw i64 [[MUL]], [[ACCUM]]
+; CHECK-INTERLEAVED-NEXT:    [[A_GEP]] = getelementptr inbounds nuw i8, ptr [[A_PTR]], i64 1
+; CHECK-INTERLEAVED-NEXT:    [[B_GEP]] = getelementptr inbounds nuw i8, ptr [[B_PTR]], i64 1
+; CHECK-INTERLEAVED-NEXT:    [[IV_NEXT]] = add nuw nsw i32 [[IV]], 1
+; CHECK-INTERLEAVED-NEXT:    [[CMP_2:%.*]] = icmp eq i32 [[IV_NEXT]], [[N]]
+; CHECK-INTERLEAVED-NEXT:    br i1 [[CMP_2]], label [[EXIT_LOOPEXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP30:![0-9]+]]
+; CHECK-INTERLEAVED:       exit.loopexit:
+; CHECK-INTERLEAVED-NEXT:    [[ADD_LCSSA:%.*]] = phi i64 [ [[ADD]], [[FOR_BODY]] ], [ [[TMP30]], [[MIDDLE_BLOCK]] ]
+; CHECK-INTERLEAVED-NEXT:    br label [[EXIT]]
+; CHECK-INTERLEAVED:       exit:
+; CHECK-INTERLEAVED-NEXT:    [[COST_RESULT:%.*]] = phi i64 [ [[COST]], [[ENTRY:%.*]] ], [ [[ADD_LCSSA]], [[EXIT_LOOPEXIT]] ]
+; CHECK-INTERLEAVED-NEXT:    [[RESULT:%.*]] = trunc i64 [[COST_RESULT]] to i32
+; CHECK-INTERLEAVED-NEXT:    ret i32 [[RESULT]]
+;
+; CHECK-MAXBW-LABEL: define dso_local i32 @not_dotp_vscale1(
+; CHECK-MAXBW-SAME: ptr [[A:%.*]], ptr [[B:%.*]], i32 [[N:%.*]], i64 [[COST:%.*]]) #[[ATTR0]] {
+; CHECK-MAXBW-NEXT:  entry:
+; CHECK-MAXBW-NEXT:    [[CMP:%.*]] = icmp sgt i32 [[N]], 0
+; CHECK-MAXBW-NEXT:    br i1 [[CMP]], label [[FOR_BODY_PREHEADER:%.*]], label [[EXIT:%.*]]
+; CHECK-MAXBW:       for.body.preheader:
+; CHECK-MAXBW-NEXT:    [[TMP0:%.*]] = zext i32 [[N]] to i64
+; CHECK-MAXBW-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-MAXBW-NEXT:    [[TMP2:%.*]] = mul i64 [[TMP1]], 8
+; CHECK-MAXBW-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], [[TMP2]]
+; CHECK-MAXBW-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK-MAXBW:       vector.ph:
+; CHECK-MAXBW-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-MAXBW-NEXT:    [[TMP4:%.*]] = mul i64 [[TMP3]], 8
+; CHECK-MAXBW-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[TMP0]], [[TMP4]]
+; CHECK-MAXBW-NEXT:    [[N_VEC:%.*]] = sub i64 [[TMP0]], [[N_MOD_VF]]
+; CHECK-MAXBW-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-MAXBW-NEXT:    [[TMP6:%.*]] = mul i64 [[TMP5]], 8
+; CHECK-MAXBW-NEXT:    [[TMP7:%.*]] = trunc i64 [[N_VEC]] to i32
+; CHECK-MAXBW-NEXT:    [[TMP8:%.*]] = getelementptr i8, ptr [[A]], i64 [[N_VEC]]
+; CHECK-MAXBW-NEXT:    [[TMP9:%.*]] = getelementptr i8, ptr [[B]], i64 [[N_VEC]]
+; CHECK-MAXBW-NEXT:    [[TMP10:%.*]] = insertelement <vscale x 8 x i64> zeroinitializer, i64 [[COST]], i32 0
+; CHECK-MAXBW-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK-MAXBW:       vector.body:
+; CHECK-MAXBW-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-MAXBW-NEXT:    [[VEC_PHI:%.*]] = phi <vscale x 8 x i64> [ [[TMP10]], [[VECTOR_PH]] ], [ [[TMP20:%.*]], [[VECTOR_BODY]] ]
+; CHECK-MAXBW-NEXT:    [[TMP11:%.*]] = add i64 [[INDEX]], 0
+; CHECK-MAXBW-NEXT:    [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[A]], i64 [[TMP11]]
+; CHECK-MAXBW-NEXT:    [[TMP12:%.*]] = add i64 [[INDEX]], 0
+; CHECK-MAXBW-NEXT:    [[NEXT_GEP1:%.*]] = getelementptr i8, ptr [[B]], i64 [[TMP12]]
+; CHECK-MAXBW-NEXT:    [[TMP13:%.*]] = getelementptr i8, ptr [[NEXT_GEP]], i32 0
+; CHECK-MAXBW-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 8 x i8>, ptr [[TMP13]], align 1
+; CHECK-MAXBW-NEXT:    [[TMP14:%.*]] = zext <vscale x 8 x i8> [[WIDE_LOAD]] to <vscale x 8 x i64>
+; CHECK-MAXBW-NEXT:    [[TMP15:%.*]] = getelementptr i8, ptr [[NEXT_GEP1]], i32 0
+; CHECK-MAXBW-NEXT:    [[WIDE_LOAD2:%.*]] = load <vscale x 8 x i8>, ptr [[TMP15]], align 1
+; CHECK-MAXBW-NEXT:    [[TMP16:%.*]] = zext <vscale x 8 x i8> [[WIDE_LOAD2]] to <vscale x 8 x i64>
+; CHECK-MAXBW-NEXT:    [[TMP17:%.*]] = mul nuw nsw <vscale x 8 x i64> [[TMP16]], [[TMP14]]
+; CHECK-MAXBW-NEXT:    [[TMP20]] = add <vscale x 8 x i64> [[TMP17]], [[VEC_PHI]]
+; CHECK-MAXBW-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP6]]
+; CHECK-MAXBW-NEXT:    [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-MAXBW-NEXT:    br i1 [[TMP18]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP27:![0-9]+]]
+; CHECK-MAXBW:       middle.block:
+; CHECK-MAXBW-NEXT:    [[TMP19:%.*]] = call i64 @llvm.vector.reduce.add.nxv8i64(<vscale x 8 x i64> [[TMP20]])
+; CHECK-MAXBW-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]]
+; CHECK-MAXBW-NEXT:    br i1 [[CMP_N]], label [[EXIT_LOOPEXIT:%.*]], label [[SCALAR_PH]]
+; CHECK-MAXBW:       scalar.ph:
+; CHECK-MAXBW-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ [[TMP7]], [[MIDDLE_BLOCK]] ], [ 0, [[FOR_BODY_PREHEADER]] ]
+; CHECK-MAXBW-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i64 [ [[TMP19]], [[MIDDLE_BLOCK]] ], [ [[COST]], [[FOR_BODY_PREHEADER]] ]
+; CHECK-MAXBW-NEXT:    [[BC_RESUME_VAL3:%.*]] = phi ptr [ [[TMP8]], [[MIDDLE_BLOCK]] ], [ [[A]], [[FOR_BODY_PREHEADER]] ]
+; CHECK-MAXBW-NEXT:    [[BC_RESUME_VAL4:%.*]] = phi ptr [ [[TMP9]], [[MIDDLE_BLOCK]] ], [ [[B]], [[FOR_BODY_PREHEADER]] ]
+; CHECK-MAXBW-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK-MAXBW:       for.body:
+; CHECK-MAXBW-NEXT:    [[IV:%.*]] = phi i32 [ [[IV_NEXT:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
+; CHECK-MAXBW-NEXT:    [[ACCUM:%.*]] = phi i64 [ [[ADD:%.*]], [[FOR_BODY]] ], [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ]
+; CHECK-MAXBW-NEXT:    [[A_PTR:%.*]] = phi ptr [ [[A_GEP:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL3]], [[SCALAR_PH]] ]
+; CHECK-MAXBW-NEXT:    [[B_PTR:%.*]] = phi ptr [ [[B_GEP:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL4]], [[SCALAR_PH]] ]
+; CHECK-MAXBW-NEXT:    [[A_LOAD:%.*]] = load i8, ptr [[A_PTR]], align 1
+; CHECK-MAXBW-NEXT:    [[A_EXT:%.*]] = zext i8 [[A_LOAD]] to i64
+; CHECK-MAXBW-NEXT:    [[B_LOAD:%.*]] = load i8, ptr [[B_PTR]], align 1
+; CHECK-MAXBW-NEXT:    [[B_EXT:%.*]] = zext i8 [[B_LOAD]] to i64
+; CHECK-MAXBW-NEXT:    [[MUL:%.*]] = mul nuw nsw i64 [[B_EXT]], [[A_EXT]]
+; CHECK-MAXBW-NEXT:    [[ADD]] = add nsw i64 [[MUL]], [[ACCUM]]
+; CHECK-MAXBW-NEXT:    [[A_GEP]] = getelementptr inbounds nuw i8, ptr [[A_PTR]], i64 1
+; CHECK-MAXBW-NEXT:    [[B_GEP]] = getelementptr inbounds nuw i8, ptr [[B_PTR]], i64 1
+; CHECK-MAXBW-NEXT:    [[IV_NEXT]] = add nuw nsw i32 [[IV]], 1
+; CHECK-MAXBW-NEXT:    [[CMP_2:%.*]] = icmp eq i32 [[IV_NEXT]], [[N]]
+; CHECK-MAXBW-NEXT:    br i1 [[CMP_2]], label [[EXIT_LOOPEXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP28:![0-9]+]]
+; CHECK-MAXBW:       exit.loopexit:
+; CHECK-MAXBW-NEXT:    [[ADD_LCSSA:%.*]] = phi i64 [ [[ADD]], [[FOR_BODY]] ], [ [[TMP19]], [[MIDDLE_BLOCK]] ]
+; CHECK-MAXBW-NEXT:    br label [[EXIT]]
+; CHECK-MAXBW:       exit:
+; CHECK-MAXBW-NEXT:    [[COST_RESULT:%.*]] = phi i64 [ [[COST]], [[ENTRY:%.*]] ], [ [[ADD_LCSSA]], [[EXIT_LOOPEXIT]] ]
+; CHECK-MAXBW-NEXT:    [[RESULT:%.*]] = trunc i64 [[COST_RESULT]] to i32
+; CHECK-MAXBW-NEXT:    ret i32 [[RESULT]]
+;
+entry:
+  %cmp = icmp sgt i32 %n, 0
+  br i1 %cmp, label %for.body, label %exit
+
+for.body:                                         ; preds = %for.body.preheader, %for.body
+  %iv = phi i32 [ %iv.next, %for.body ], [ 0, %entry ]
+  %accum = phi i64 [ %add, %for.body ], [ %cost, %entry ]
+  %a.ptr = phi ptr [ %a.gep, %for.body ], [ %a, %entry ]
+  %b.ptr = phi ptr [ %b.gep, %for.body ], [ %b, %entry ]
+  %a.load = load i8, ptr %a.ptr, align 1
+  %a.ext = zext i8 %a.load to i64
+  %b.load = load i8, ptr %b.ptr, align 1
+  %b.ext = zext i8 %b.load to i64
+  %mul = mul nuw nsw i64 %b.ext, %a.ext
+  %add = add nsw i64 %mul, %accum
+  %a.gep = getelementptr inbounds nuw i8, ptr %a.ptr, i64 1
+  %b.gep = getelementptr inbounds nuw i8, ptr %b.ptr, i64 1
+  %iv.next = add nuw nsw i32 %iv, 1
+  %cmp.2 = icmp eq i32 %iv.next, %n
+  br i1 %cmp.2, label %exit, label %for.body
+
+exit:                                 ; preds = %for.cond.cleanup.loopexit, %entry
+  %cost.result = phi i64 [ %cost, %entry ], [ %add, %for.body ]
+  %result = trunc i64 %cost.result to i32
+  ret i32 %result
+}
+
+
+; == Partial reductions with add of an extend
+
+define i32 @zext_add_reduc_i8_i32(ptr %a) #0 {
+; CHECK-INTERLEAVE1-LABEL: define i32 @zext_add_reduc_i8_i32(
+; CHECK-INTERLEAVE1-SAME: ptr [[A:%.*]]) #[[ATTR0]] {
+; CHECK-INTERLEAVE1-NEXT:  entry:
+; CHECK-INTERLEAVE1-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK-INTERLEAVE1:       vector.ph:
+; CHECK-INTERLEAVE1-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK-INTERLEAVE1:       vector.body:
+; CHECK-INTERLEAVE1-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-INTERLEAVE1-NEXT:    [[VEC_PHI:%.*]] = phi <16 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP4:%.*]], [[VECTOR_BODY]] ]
+; CHECK-INTERLEAVE1-NEXT:    [[TMP0:%.*]] = add i64 [[INDEX]], 0
+; CHECK-INTERLEAVE1-NEXT:    [[TMP1:%.*]] = getelementptr i8, ptr [[A]], i64 [[TMP0]]
+; CHECK-INTERLEAVE1-NEXT:    [[TMP2:%.*]] = getelementptr i8, ptr [[TMP1]], i32 0
+; CHECK-INTERLEAVE1-NEXT:    [[WIDE_LOAD:%.*]] = load <16 x i8>, ptr [[TMP2]], align 1
+; CHECK-INTERLEAVE1-NEXT:    [[TMP3:%.*]] = zext <16 x i8> [[WIDE_LOAD]] to <16 x i32>
+; CHECK-INTERLEAVE1-NEXT:    [[TMP4]] = add <16 x i32> [[TMP3]], [[VEC_PHI]]
+; CHECK-INTERLEAVE1-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
+; CHECK-INTERLEAVE1-NEXT:    [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; CHECK-INTERLEAVE1-NEXT:    br i1 [[TMP5]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP29:![0-9]+]]
+; CHECK-INTERLEAVE1:       middle.block:
+; CHECK-INTERLEAVE1-NEXT:    [[TMP6:%.*]] = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> [[TMP4]])
+; CHECK-INTERLEAVE1-NEXT:    br i1 false, label [[FOR_EXIT:%.*]], label [[SCALAR_PH]]
+; CHECK-INTERLEAVE1:       scalar.ph:
+; CHECK-INTERLEAVE1-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 1024, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-INTERLEAVE1-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP6]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ]
+; CHECK-INTERLEAVE1-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK-INTERLEAVE1:       for.body:
+; CHECK-INTERLEAVE1-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
+; CHECK-INTERLEAVE1-NEXT:    [[ACCUM:%.*]] = phi i32 [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[ADD:%.*]], [[FOR_BODY]] ]
+; CHECK-INTERLEAVE1-NEXT:    [[GEP_A:%.*]] = getelementptr i8, ptr [[A]], i64 [[IV]]
+; CHECK-INTERLEAVE1-NEXT:    [[LOAD_A:%.*]] = load i8, ptr [[GEP_A]], align 1
+; CHECK-INTERLEAVE1-NEXT:    [[EXT_A:%.*]] = zext i8 [[LOAD_A]] to i32
+; CHECK-INTERLEAVE1-NEXT:    [[ADD]] = add i32 [[EXT_A]], [[ACCUM]]
+; CHECK-INTERLEAVE1-NEXT:    [[IV_NEXT]] = add i64 [[IV]], 1
+; CHECK-INTERLEAVE1-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], 1025
+; CHECK-INTERLEAVE1-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_EXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP30:![0-9]+]]
+; CHECK-INTERLEAVE1:       for.exit:
+; CHECK-INTERLEAVE1-NEXT:    [[ADD_LCSSA:%.*]] = phi i32 [ [[ADD]], [[FOR_BODY]] ], [ [[TMP6]], [[MIDDLE_BLOCK]] ]
+; CHECK-INTERLEAVE1-NEXT:    ret i32 [[ADD_LCSSA]]
+;
+; CHECK-INTERLEAVED-LABEL: define i32 @zext_add_reduc_i8_i32(
+; CHECK-INTERLEAVED-SAME: ptr [[A:%.*]]) #[[ATTR0]] {
+; CHECK-INTERLEAVED-NEXT:  entry:
+; CHECK-INTERLEAVED-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK-INTERLEAVED:       vector.ph:
+; CHECK-INTERLEAVED-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK-INTERLEAVED:       vector.body:
+; CHECK-INTERLEAVED-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-INTERLEAVED-NEXT:    [[VEC_PHI:%.*]] = phi <16 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP6:%.*]], [[VECTOR_BODY]] ]
+; CHECK-INTERLEAVED-NEXT:    [[VEC_PHI1:%.*]] = phi <16 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP7:%.*]], [[VECTOR_BODY]] ]
+; CHECK-INTERLEAVED-NEXT:    [[TMP0:%.*]] = add i64 [[INDEX]], 0
+; CHECK-INTERLEAVED-NEXT:    [[TMP1:%.*]] = getelementptr i8, ptr [[A]], i64 [[TMP0]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP2:%.*]] = getelementptr i8, ptr [[TMP1]], i32 0
+; CHECK-INTERLEAVED-NEXT:    [[TMP3:%.*]] = getelementptr i8, ptr [[TMP1]], i32 16
+; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD:%.*]] = load <16 x i8>, ptr [[TMP2]], align 1
+; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD2:%.*]] = load <16 x i8>, ptr [[TMP3]], align 1
+; CHECK-INTERLEAVED-NEXT:    [[TMP4:%.*]] = zext <16 x i8> [[WIDE_LOAD]] to <16 x i32>
+; CHECK-INTERLEAVED-NEXT:    [[TMP5:%.*]] = zext <16 x i8> [[WIDE_LOAD2]] to <16 x i32>
+; CHECK-INTERLEAVED-NEXT:    [[TMP6]] = add <16 x i32> [[TMP4]], [[VEC_PHI]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP7]] = add <16 x i32> [[TMP5]], [[VEC_PHI1]]
+; CHECK-INTERLEAVED-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 32
+; CHECK-INTERLEAVED-NEXT:    [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; CHECK-INTERLEAVED-NEXT:    br i1 [[TMP8]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP31:![0-9]+]]
+; CHECK-INTERLEAVED:       middle.block:
+; CHECK-INTERLEAVED-NEXT:    [[BIN_RDX:%.*]] = add <16 x i32> [[TMP7]], [[TMP6]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP9:%.*]] = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> [[BIN_RDX]])
+; CHECK-INTERLEAVED-NEXT:    br i1 false, label [[FOR_EXIT:%.*]], label [[SCALAR_PH]]
+; CHECK-INTERLEAVED:       scalar.ph:
+; CHECK-INTERLEAVED-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 1024, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-INTERLEAVED-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP9]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ]
+; CHECK-INTERLEAVED-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK-INTERLEAVED:       for.body:
+; CHECK-INTERLEAVED-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
+; CHECK-INTERLEAVED-NEXT:    [[ACCUM:%.*]] = phi i32 [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[ADD:%.*]], [[FOR_BODY]] ]
+; CHECK-INTERLEAVED-NEXT:    [[GEP_A:%.*]] = getelementptr i8, ptr [[A]], i64 [[IV]]
+; CHECK-INTERLEAVED-NEXT:    [[LOAD_A:%.*]] = load i8, ptr [[GEP_A]], align 1
+; CHECK-INTERLEAVED-NEXT:    [[EXT_A:%.*]] = zext i8 [[LOAD_A]] to i32
+; CHECK-INTERLEAVED-NEXT:    [[ADD]] = add i32 [[EXT_A]], [[ACCUM]]
+; CHECK-INTERLEAVED-NEXT:    [[IV_NEXT]] = add i64 [[IV]], 1
+; CHECK-INTERLEAVED-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], 1025
+; CHECK-INTERLEAVED-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_EXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP32:![0-9]+]]
+; CHECK-INTERLEAVED:       for.exit:
+; CHECK-INTERLEAVED-NEXT:    [[ADD_LCSSA:%.*]] = phi i32 [ [[ADD]], [[FOR_BODY]] ], [ [[TMP9]], [[MIDDLE_BLOCK]] ]
+; CHECK-INTERLEAVED-NEXT:    ret i32 [[ADD_LCSSA]]
+;
+; CHECK-MAXBW-LABEL: define i32 @zext_add_reduc_i8_i32(
+; CHECK-MAXBW-SAME: ptr [[A:%.*]]) #[[ATTR0]] {
+; CHECK-MAXBW-NEXT:  entry:
+; CHECK-MAXBW-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-MAXBW-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 8
+; CHECK-MAXBW-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK-MAXBW:       vector.ph:
+; CHECK-MAXBW-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-MAXBW-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 8
+; CHECK-MAXBW-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1025, [[TMP3]]
+; CHECK-MAXBW-NEXT:    [[N_VEC:%.*]] = sub i64 1025, [[N_MOD_VF]]
+; CHECK-MAXBW-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-MAXBW-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 8
+; CHECK-MAXBW-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK-MAXBW:       vector.body:
+; CHECK-MAXBW-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-MAXBW-NEXT:    [[VEC_PHI:%.*]] = phi <vscale x 8 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP12:%.*]], [[VECTOR_BODY]] ]
+; CHECK-MAXBW-NEXT:    [[TMP6:%.*]] = add i64 [[INDEX]], 0
+; CHECK-MAXBW-NEXT:    [[TMP7:%.*]] = getelementptr i8, ptr [[A]], i64 [[TMP6]]
+; CHECK-MAXBW-NEXT:    [[TMP8:%.*]] = getelementptr i8, ptr [[TMP7]], i32 0
+; CHECK-MAXBW-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 8 x i8>, ptr [[TMP8]], align 1
+; CHECK-MAXBW-NEXT:    [[TMP9:%.*]] = zext <vscale x 8 x i8> [[WIDE_LOAD]] to <vscale x 8 x i32>
+; CHECK-MAXBW-NEXT:    [[TMP12]] = add <vscale x 8 x i32> [[TMP9]], [[VEC_PHI]]
+; CHECK-MAXBW-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
+; CHECK-MAXBW-NEXT:    [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-MAXBW-NEXT:    br i1 [[TMP10]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP29:![0-9]+]]
+; CHECK-MAXBW:       middle.block:
+; CHECK-MAXBW-NEXT:    [[TMP11:%.*]] = call i32 @llvm.vector.reduce.add.nxv8i32(<vscale x 8 x i32> [[TMP12]])
+; CHECK-MAXBW-NEXT:    [[CMP_N:%.*]] = icmp eq i64 1025, [[N_VEC]]
+; CHECK-MAXBW-NEXT:    br i1 [[CMP_N]], label [[FOR_EXIT:%.*]], label [[SCALAR_PH]]
+; CHECK-MAXBW:       scalar.ph:
+; CHECK-MAXBW-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-MAXBW-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP11]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ]
+; CHECK-MAXBW-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK-MAXBW:       for.body:
+; CHECK-MAXBW-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
+; CHECK-MAXBW-NEXT:    [[ACCUM:%.*]] = phi i32 [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[ADD:%.*]], [[FOR_BODY]] ]
+; CHECK-MAXBW-NEXT:    [[GEP_A:%.*]] = getelementptr i8, ptr [[A]], i64 [[IV]]
+; CHECK-MAXBW-NEXT:    [[LOAD_A:%.*]] = load i8, ptr [[GEP_A]], align 1
+; CHECK-MAXBW-NEXT:    [[EXT_A:%.*]] = zext i8 [[LOAD_A]] to i32
+; CHECK-MAXBW-NEXT:    [[ADD]] = add i32 [[EXT_A]], [[ACCUM]]
+; CHECK-MAXBW-NEXT:    [[IV_NEXT]] = add i64 [[IV]], 1
+; CHECK-MAXBW-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], 1025
+; CHECK-MAXBW-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_EXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP30:![0-9]+]]
+; CHECK-MAXBW:       for.exit:
+; CHECK-MAXBW-NEXT:    [[ADD_LCSSA:%.*]] = phi i32 [ [[ADD]], [[FOR_BODY]] ], [ [[TMP11]], [[MIDDLE_BLOCK]] ]
+; CHECK-MAXBW-NEXT:    ret i32 [[ADD_LCSSA]]
+;
+entry:
+  br label %for.body
+
+for.body:                                         ; preds = %for.body, %entry
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+  %accum = phi i32 [ 0, %entry ], [ %add, %for.body ]
+  %gep.a = getelementptr i8, ptr %a, i64 %iv
+  %load.a = load i8, ptr %gep.a, align 1
+  %ext.a = zext i8 %load.a to i32
+  %add = add i32 %ext.a, %accum
+  %iv.next = add i64 %iv, 1
+  %exitcond.not = icmp eq i64 %iv.next, 1025
+  br i1 %exitcond.not, label %for.exit, label %for.body
+
+for.exit:                        ; preds = %for.body
+  ret i32 %add
+}
+
+
+define i64 @zext_add_reduc_i8_i64(ptr %a) #0 {
+; CHECK-INTERLEAVE1-LABEL: define i64 @zext_add_reduc_i8_i64(
+; CHECK-INTERLEAVE1-SAME: ptr [[A:%.*]]) #[[ATTR0]] {
+; CHECK-INTERLEAVE1-NEXT:  entry:
+; CHECK-INTERLEAVE1-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK-INTERLEAVE1:       vector.ph:
+; CHECK-INTERLEAVE1-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK-INTERLEAVE1:       vector.body:
+; CHECK-INTERLEAVE1-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-INTERLEAVE1-NEXT:    [[VEC_PHI:%.*]] = phi <16 x i64> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP4:%.*]], [[VECTOR_BODY]] ]
+; CHECK-INTERLEAVE1-NEXT:    [[TMP0:%.*]] = add i64 [[INDEX]], 0
+; CHECK-INTERLEAVE1-NEXT:    [[TMP1:%.*]] = getelementptr i8, ptr [[A]], i64 [[TMP0]]
+; CHECK-INTERLEAVE1-NEXT:    [[TMP2:%.*]] = getelementptr i8, ptr [[TMP1]], i32 0
+; CHECK-INTERLEAVE1-NEXT:    [[WIDE_LOAD:%.*]] = load <16 x i8>, ptr [[TMP2]], align 1
+; CHECK-INTERLEAVE1-NEXT:    [[TMP3:%.*]] = zext <16 x i8> [[WIDE_LOAD]] to <16 x i64>
+; CHECK-INTERLEAVE1-NEXT:    [[TMP4]] = add <16 x i64> [[TMP3]], [[VEC_PHI]]
+; CHECK-INTERLEAVE1-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
+; CHECK-INTERLEAVE1-NEXT:    [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; CHECK-INTERLEAVE1-NEXT:    br i1 [[TMP5]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP31:![0-9]+]]
+; CHECK-INTERLEAVE1:       middle.block:
+; CHECK-INTERLEAVE1-NEXT:    [[TMP6:%.*]] = call i64 @llvm.vector.reduce.add.v16i64(<16 x i64> [[TMP4]])
+; CHECK-INTERLEAVE1-NEXT:    br i1 false, label [[FOR_EXIT:%.*]], label [[SCALAR_PH]]
+; CHECK-INTERLEAVE1:       scalar.ph:
+; CHECK-INTERLEAVE1-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 1024, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-INTERLEAVE1-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i64 [ [[TMP6]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ]
+; CHECK-INTERLEAVE1-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK-INTERLEAVE1:       for.body:
+; CHECK-INTERLEAVE1-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
+; CHECK-INTERLEAVE1-NEXT:    [[ACCUM:%.*]] = phi i64 [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[ADD:%.*]], [[FOR_BODY]] ]
+; CHECK-INTERLEAVE1-NEXT:    [[GEP_A:%.*]] = getelementptr i8, ptr [[A]], i64 [[IV]]
+; CHECK-INTERLEAVE1-NEXT:    [[LOAD_A:%.*]] = load i8, ptr [[GEP_A]], align 1
+; CHECK-INTERLEAVE1-NEXT:    [[EXT_A:%.*]] = zext i8 [[LOAD_A]] to i64
+; CHECK-INTERLEAVE1-NEXT:    [[ADD]] = add i64 [[EXT_A]], [[ACCUM]]
+; CHECK-INTERLEAVE1-NEXT:    [[IV_NEXT]] = add i64 [[IV]], 1
+; CHECK-INTERLEAVE1-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], 1025
+; CHECK-INTERLEAVE1-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_EXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP32:![0-9]+]]
+; CHECK-INTERLEAVE1:       for.exit:
+; CHECK-INTERLEAVE1-NEXT:    [[ADD_LCSSA:%.*]] = phi i64 [ [[ADD]], [[FOR_BODY]] ], [ [[TMP6]], [[MIDDLE_BLOCK]] ]
+; CHECK-INTERLEAVE1-NEXT:    ret i64 [[ADD_LCSSA]]
+;
+; CHECK-INTERLEAVED-LABEL: define i64 @zext_add_reduc_i8_i64(
+; CHECK-INTERLEAVED-SAME: ptr [[A:%.*]]) #[[ATTR0]] {
+; CHECK-INTERLEAVED-NEXT:  entry:
+; CHECK-INTERLEAVED-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK-INTERLEAVED:       vector.ph:
+; CHECK-INTERLEAVED-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK-INTERLEAVED:       vector.body:
+; CHECK-INTERLEAVED-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-INTERLEAVED-NEXT:    [[VEC_PHI:%.*]] = phi <16 x i64> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP6:%.*]], [[VECTOR_BODY]] ]
+; CHECK-INTERLEAVED-NEXT:    [[VEC_PHI1:%.*]] = phi <16 x i64> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP7:%.*]], [[VECTOR_BODY]] ]
+; CHECK-INTERLEAVED-NEXT:    [[TMP0:%.*]] = add i64 [[INDEX]], 0
+; CHECK-INTERLEAVED-NEXT:    [[TMP1:%.*]] = getelementptr i8, ptr [[A]], i64 [[TMP0]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP2:%.*]] = getelementptr i8, ptr [[TMP1]], i32 0
+; CHECK-INTERLEAVED-NEXT:    [[TMP3:%.*]] = getelementptr i8, ptr [[TMP1]], i32 16
+; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD:%.*]] = load <16 x i8>, ptr [[TMP2]], align 1
+; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD2:%.*]] = load <16 x i8>, ptr [[TMP3]], align 1
+; CHECK-INTERLEAVED-NEXT:    [[TMP4:%.*]] = zext <16 x i8> [[WIDE_LOAD]] to <16 x i64>
+; CHECK-INTERLEAVED-NEXT:    [[TMP5:%.*]] = zext <16 x i8> [[WIDE_LOAD2]] to <16 x i64>
+; CHECK-INTERLEAVED-NEXT:    [[TMP6]] = add <16 x i64> [[TMP4]], [[VEC_PHI]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP7]] = add <16 x i64> [[TMP5]], [[VEC_PHI1]]
+; CHECK-INTERLEAVED-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 32
+; CHECK-INTERLEAVED-NEXT:    [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; CHECK-INTERLEAVED-NEXT:    br i1 [[TMP8]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP33:![0-9]+]]
+; CHECK-INTERLEAVED:       middle.block:
+; CHECK-INTERLEAVED-NEXT:    [[BIN_RDX:%.*]] = add <16 x i64> [[TMP7]], [[TMP6]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP9:%.*]] = call i64 @llvm.vector.reduce.add.v16i64(<16 x i64> [[BIN_RDX]])
+; CHECK-INTERLEAVED-NEXT:    br i1 false, label [[FOR_EXIT:%.*]], label [[SCALAR_PH]]
+; CHECK-INTERLEAVED:       scalar.ph:
+; CHECK-INTERLEAVED-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 1024, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-INTERLEAVED-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i64 [ [[TMP9]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ]
+; CHECK-INTERLEAVED-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK-INTERLEAVED:       for.body:
+; CHECK-INTERLEAVED-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
+; CHECK-INTERLEAVED-NEXT:    [[ACCUM:%.*]] = phi i64 [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[ADD:%.*]], [[FOR_BODY]] ]
+; CHECK-INTERLEAVED-NEXT:    [[GEP_A:%.*]] = getelementptr i8, ptr [[A]], i64 [[IV]]
+; CHECK-INTERLEAVED-NEXT:    [[LOAD_A:%.*]] = load i8, ptr [[GEP_A]], align 1
+; CHECK-INTERLEAVED-NEXT:    [[EXT_A:%.*]] = zext i8 [[LOAD_A]] to i64
+; CHECK-INTERLEAVED-NEXT:    [[ADD]] = add i64 [[EXT_A]], [[ACCUM]]
+; CHECK-INTERLEAVED-NEXT:    [[IV_NEXT]] = add i64 [[IV]], 1
+; CHECK-INTERLEAVED-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], 1025
+; CHECK-INTERLEAVED-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_EXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP34:![0-9]+]]
+; CHECK-INTERLEAVED:       for.exit:
+; CHECK-INTERLEAVED-NEXT:    [[ADD_LCSSA:%.*]] = phi i64 [ [[ADD]], [[FOR_BODY]] ], [ [[TMP9]], [[MIDDLE_BLOCK]] ]
+; CHECK-INTERLEAVED-NEXT:    ret i64 [[ADD_LCSSA]]
+;
+; CHECK-MAXBW-LABEL: define i64 @zext_add_reduc_i8_i64(
+; CHECK-MAXBW-SAME: ptr [[A:%.*]]) #[[ATTR0]] {
+; CHECK-MAXBW-NEXT:  entry:
+; CHECK-MAXBW-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-MAXBW-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 8
+; CHECK-MAXBW-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK-MAXBW:       vector.ph:
+; CHECK-MAXBW-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-MAXBW-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 8
+; CHECK-MAXBW-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1025, [[TMP3]]
+; CHECK-MAXBW-NEXT:    [[N_VEC:%.*]] = sub i64 1025, [[N_MOD_VF]]
+; CHECK-MAXBW-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-MAXBW-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 8
+; CHECK-MAXBW-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK-MAXBW:       vector.body:
+; CHECK-MAXBW-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-MAXBW-NEXT:    [[VEC_PHI:%.*]] = phi <vscale x 8 x i64> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP12:%.*]], [[VECTOR_BODY]] ]
+; CHECK-MAXBW-NEXT:    [[TMP6:%.*]] = add i64 [[INDEX]], 0
+; CHECK-MAXBW-NEXT:    [[TMP7:%.*]] = getelementptr i8, ptr [[A]], i64 [[TMP6]]
+; CHECK-MAXBW-NEXT:    [[TMP8:%.*]] = getelementptr i8, ptr [[TMP7]], i32 0
+; CHECK-MAXBW-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 8 x i8>, ptr [[TMP8]], align 1
+; CHECK-MAXBW-NEXT:    [[TMP9:%.*]] = zext <vscale x 8 x i8> [[WIDE_LOAD]] to <vscale x 8 x i64>
+; CHECK-MAXBW-NEXT:    [[TMP12]] = add <vscale x 8 x i64> [[TMP9]], [[VEC_PHI]]
+; CHECK-MAXBW-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
+; CHECK-MAXBW-NEXT:    [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-MAXBW-NEXT:    br i1 [[TMP10]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP31:![0-9]+]]
+; CHECK-MAXBW:       middle.block:
+; CHECK-MAXBW-NEXT:    [[TMP11:%.*]] = call i64 @llvm.vector.reduce.add.nxv8i64(<vscale x 8 x i64> [[TMP12]])
+; CHECK-MAXBW-NEXT:    [[CMP_N:%.*]] = icmp eq i64 1025, [[N_VEC]]
+; CHECK-MAXBW-NEXT:    br i1 [[CMP_N]], label [[FOR_EXIT:%.*]], label [[SCALAR_PH]]
+; CHECK-MAXBW:       scalar.ph:
+; CHECK-MAXBW-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-MAXBW-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i64 [ [[TMP11]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ]
+; CHECK-MAXBW-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK-MAXBW:       for.body:
+; CHECK-MAXBW-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
+; CHECK-MAXBW-NEXT:    [[ACCUM:%.*]] = phi i64 [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[ADD:%.*]], [[FOR_BODY]] ]
+; CHECK-MAXBW-NEXT:    [[GEP_A:%.*]] = getelementptr i8, ptr [[A]], i64 [[IV]]
+; CHECK-MAXBW-NEXT:    [[LOAD_A:%.*]] = load i8, ptr [[GEP_A]], align 1
+; CHECK-MAXBW-NEXT:    [[EXT_A:%.*]] = zext i8 [[LOAD_A]] to i64
+; CHECK-MAXBW-NEXT:    [[ADD]] = add i64 [[EXT_A]], [[ACCUM]]
+; CHECK-MAXBW-NEXT:    [[IV_NEXT]] = add i64 [[IV]], 1
+; CHECK-MAXBW-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], 1025
+; CHECK-MAXBW-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_EXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP32:![0-9]+]]
+; CHECK-MAXBW:       for.exit:
+; CHECK-MAXBW-NEXT:    [[ADD_LCSSA:%.*]] = phi i64 [ [[ADD]], [[FOR_BODY]] ], [ [[TMP11]], [[MIDDLE_BLOCK]] ]
+; CHECK-MAXBW-NEXT:    ret i64 [[ADD_LCSSA]]
+;
+entry:
+  br label %for.body
+
+for.body:                                         ; preds = %for.body, %entry
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+  %accum = phi i64 [ 0, %entry ], [ %add, %for.body ]
+  %gep.a = getelementptr i8, ptr %a, i64 %iv
+  %load.a = load i8, ptr %gep.a, align 1
+  %ext.a = zext i8 %load.a to i64
+  %add = add i64 %ext.a, %accum
+  %iv.next = add i64 %iv, 1
+  %exitcond.not = icmp eq i64 %iv.next, 1025
+  br i1 %exitcond.not, label %for.exit, label %for.body
+
+for.exit:                        ; preds = %for.body
+  ret i64 %add
+}
+
+
+define i64 @zext_add_reduc_i16_i64(ptr %a) #0 {
+; CHECK-INTERLEAVE1-LABEL: define i64 @zext_add_reduc_i16_i64(
+; CHECK-INTERLEAVE1-SAME: ptr [[A:%.*]]) #[[ATTR0]] {
+; CHECK-INTERLEAVE1-NEXT:  entry:
+; CHECK-INTERLEAVE1-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK-INTERLEAVE1:       vector.ph:
+; CHECK-INTERLEAVE1-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK-INTERLEAVE1:       vector.body:
+; CHECK-INTERLEAVE1-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-INTERLEAVE1-NEXT:    [[VEC_PHI:%.*]] = phi <8 x i64> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP4:%.*]], [[VECTOR_BODY]] ]
+; CHECK-INTERLEAVE1-NEXT:    [[TMP0:%.*]] = add i64 [[INDEX]], 0
+; CHECK-INTERLEAVE1-NEXT:    [[TMP1:%.*]] = getelementptr i16, ptr [[A]], i64 [[TMP0]]
+; CHECK-INTERLEAVE1-NEXT:    [[TMP2:%.*]] = getelementptr i16, ptr [[TMP1]], i32 0
+; CHECK-INTERLEAVE1-NEXT:    [[WIDE_LOAD:%.*]] = load <8 x i16>, ptr [[TMP2]], align 2
+; CHECK-INTERLEAVE1-NEXT:    [[TMP3:%.*]] = zext <8 x i16> [[WIDE_LOAD]] to <8 x i64>
+; CHECK-INTERLEAVE1-NEXT:    [[TMP4]] = add <8 x i64> [[TMP3]], [[VEC_PHI]]
+; CHECK-INTERLEAVE1-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
+; CHECK-INTERLEAVE1-NEXT:    [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; CHECK-INTERLEAVE1-NEXT:    br i1 [[TMP5]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP33:![0-9]+]]
+; CHECK-INTERLEAVE1:       middle.block:
+; CHECK-INTERLEAVE1-NEXT:    [[TMP6:%.*]] = call i64 @llvm.vector.reduce.add.v8i64(<8 x i64> [[TMP4]])
+; CHECK-INTERLEAVE1-NEXT:    br i1 false, label [[FOR_EXIT:%.*]], label [[SCALAR_PH]]
+; CHECK-INTERLEAVE1:       scalar.ph:
+; CHECK-INTERLEAVE1-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 1024, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-INTERLEAVE1-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i64 [ [[TMP6]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ]
+; CHECK-INTERLEAVE1-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK-INTERLEAVE1:       for.body:
+; CHECK-INTERLEAVE1-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
+; CHECK-INTERLEAVE1-NEXT:    [[ACCUM:%.*]] = phi i64 [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[ADD:%.*]], [[FOR_BODY]] ]
+; CHECK-INTERLEAVE1-NEXT:    [[GEP_A:%.*]] = getelementptr i16, ptr [[A]], i64 [[IV]]
+; CHECK-INTERLEAVE1-NEXT:    [[LOAD_A:%.*]] = load i16, ptr [[GEP_A]], align 2
+; CHECK-INTERLEAVE1-NEXT:    [[EXT_A:%.*]] = zext i16 [[LOAD_A]] to i64
+; CHECK-INTERLEAVE1-NEXT:    [[ADD]] = add i64 [[EXT_A]], [[ACCUM]]
+; CHECK-INTERLEAVE1-NEXT:    [[IV_NEXT]] = add i64 [[IV]], 1
+; CHECK-INTERLEAVE1-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], 1025
+; CHECK-INTERLEAVE1-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_EXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP34:![0-9]+]]
+; CHECK-INTERLEAVE1:       for.exit:
+; CHECK-INTERLEAVE1-NEXT:    [[ADD_LCSSA:%.*]] = phi i64 [ [[ADD]], [[FOR_BODY]] ], [ [[TMP6]], [[MIDDLE_BLOCK]] ]
+; CHECK-INTERLEAVE1-NEXT:    ret i64 [[ADD_LCSSA]]
+;
+; CHECK-INTERLEAVED-LABEL: define i64 @zext_add_reduc_i16_i64(
+; CHECK-INTERLEAVED-SAME: ptr [[A:%.*]]) #[[ATTR0]] {
+; CHECK-INTERLEAVED-NEXT:  entry:
+; CHECK-INTERLEAVED-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK-INTERLEAVED:       vector.ph:
+; CHECK-INTERLEAVED-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK-INTERLEAVED:       vector.body:
+; CHECK-INTERLEAVED-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-INTERLEAVED-NEXT:    [[VEC_PHI:%.*]] = phi <8 x i64> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP6:%.*]], [[VECTOR_BODY]] ]
+; CHECK-INTERLEAVED-NEXT:    [[VEC_PHI1:%.*]] = phi <8 x i64> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP7:%.*]], [[VECTOR_BODY]] ]
+; CHECK-INTERLEAVED-NEXT:    [[TMP0:%.*]] = add i64 [[INDEX]], 0
+; CHECK-INTERLEAVED-NEXT:    [[TMP1:%.*]] = getelementptr i16, ptr [[A]], i64 [[TMP0]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP2:%.*]] = getelementptr i16, ptr [[TMP1]], i32 0
+; CHECK-INTERLEAVED-NEXT:    [[TMP3:%.*]] = getelementptr i16, ptr [[TMP1]], i32 8
+; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD:%.*]] = load <8 x i16>, ptr [[TMP2]], align 2
+; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD2:%.*]] = load <8 x i16>, ptr [[TMP3]], align 2
+; CHECK-INTERLEAVED-NEXT:    [[TMP4:%.*]] = zext <8 x i16> [[WIDE_LOAD]] to <8 x i64>
+; CHECK-INTERLEAVED-NEXT:    [[TMP5:%.*]] = zext <8 x i16> [[WIDE_LOAD2]] to <8 x i64>
+; CHECK-INTERLEAVED-NEXT:    [[TMP6]] = add <8 x i64> [[TMP4]], [[VEC_PHI]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP7]] = add <8 x i64> [[TMP5]], [[VEC_PHI1]]
+; CHECK-INTERLEAVED-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
+; CHECK-INTERLEAVED-NEXT:    [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; CHECK-INTERLEAVED-NEXT:    br i1 [[TMP8]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP35:![0-9]+]]
+; CHECK-INTERLEAVED:       middle.block:
+; CHECK-INTERLEAVED-NEXT:    [[BIN_RDX:%.*]] = add <8 x i64> [[TMP7]], [[TMP6]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP9:%.*]] = call i64 @llvm.vector.reduce.add.v8i64(<8 x i64> [[BIN_RDX]])
+; CHECK-INTERLEAVED-NEXT:    br i1 false, label [[FOR_EXIT:%.*]], label [[SCALAR_PH]]
+; CHECK-INTERLEAVED:       scalar.ph:
+; CHECK-INTERLEAVED-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 1024, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-INTERLEAVED-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i64 [ [[TMP9]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ]
+; CHECK-INTERLEAVED-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK-INTERLEAVED:       for.body:
+; CHECK-INTERLEAVED-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
+; CHECK-INTERLEAVED-NEXT:    [[ACCUM:%.*]] = phi i64 [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[ADD:%.*]], [[FOR_BODY]] ]
+; CHECK-INTERLEAVED-NEXT:    [[GEP_A:%.*]] = getelementptr i16, ptr [[A]], i64 [[IV]]
+; CHECK-INTERLEAVED-NEXT:    [[LOAD_A:%.*]] = load i16, ptr [[GEP_A]], align 2
+; CHECK-INTERLEAVED-NEXT:    [[EXT_A:%.*]] = zext i16 [[LOAD_A]] to i64
+; CHECK-INTERLEAVED-NEXT:    [[ADD]] = add i64 [[EXT_A]], [[ACCUM]]
+; CHECK-INTERLEAVED-NEXT:    [[IV_NEXT]] = add i64 [[IV]], 1
+; CHECK-INTERLEAVED-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], 1025
+; CHECK-INTERLEAVED-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_EXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP36:![0-9]+]]
+; CHECK-INTERLEAVED:       for.exit:
+; CHECK-INTERLEAVED-NEXT:    [[ADD_LCSSA:%.*]] = phi i64 [ [[ADD]], [[FOR_BODY]] ], [ [[TMP9]], [[MIDDLE_BLOCK]] ]
+; CHECK-INTERLEAVED-NEXT:    ret i64 [[ADD_LCSSA]]
+;
+; CHECK-MAXBW-LABEL: define i64 @zext_add_reduc_i16_i64(
+; CHECK-MAXBW-SAME: ptr [[A:%.*]]) #[[ATTR0]] {
+; CHECK-MAXBW-NEXT:  entry:
+; CHECK-MAXBW-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-MAXBW-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 4
+; CHECK-MAXBW-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK-MAXBW:       vector.ph:
+; CHECK-MAXBW-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-MAXBW-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 4
+; CHECK-MAXBW-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1025, [[TMP3]]
+; CHECK-MAXBW-NEXT:    [[N_VEC:%.*]] = sub i64 1025, [[N_MOD_VF]]
+; CHECK-MAXBW-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-MAXBW-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 4
+; CHECK-MAXBW-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK-MAXBW:       vector.body:
+; CHECK-MAXBW-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-MAXBW-NEXT:    [[VEC_PHI:%.*]] = phi <vscale x 4 x i64> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP12:%.*]], [[VECTOR_BODY]] ]
+; CHECK-MAXBW-NEXT:    [[TMP6:%.*]] = add i64 [[INDEX]], 0
+; CHECK-MAXBW-NEXT:    [[TMP7:%.*]] = getelementptr i16, ptr [[A]], i64 [[TMP6]]
+; CHECK-MAXBW-NEXT:    [[TMP8:%.*]] = getelementptr i16, ptr [[TMP7]], i32 0
+; CHECK-MAXBW-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 4 x i16>, ptr [[TMP8]], align 2
+; CHECK-MAXBW-NEXT:    [[TMP9:%.*]] = zext <vscale x 4 x i16> [[WIDE_LOAD]] to <vscale x 4 x i64>
+; CHECK-MAXBW-NEXT:    [[TMP12]] = add <vscale x 4 x i64> [[TMP9]], [[VEC_PHI]]
+; CHECK-MAXBW-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
+; CHECK-MAXBW-NEXT:    [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-MAXBW-NEXT:    br i1 [[TMP10]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP33:![0-9]+]]
+; CHECK-MAXBW:       middle.block:
+; CHECK-MAXBW-NEXT:    [[TMP11:%.*]] = call i64 @llvm.vector.reduce.add.nxv4i64(<vscale x 4 x i64> [[TMP12]])
+; CHECK-MAXBW-NEXT:    [[CMP_N:%.*]] = icmp eq i64 1025, [[N_VEC]]
+; CHECK-MAXBW-NEXT:    br i1 [[CMP_N]], label [[FOR_EXIT:%.*]], label [[SCALAR_PH]]
+; CHECK-MAXBW:       scalar.ph:
+; CHECK-MAXBW-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-MAXBW-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i64 [ [[TMP11]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ]
+; CHECK-MAXBW-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK-MAXBW:       for.body:
+; CHECK-MAXBW-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
+; CHECK-MAXBW-NEXT:    [[ACCUM:%.*]] = phi i64 [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[ADD:%.*]], [[FOR_BODY]] ]
+; CHECK-MAXBW-NEXT:    [[GEP_A:%.*]] = getelementptr i16, ptr [[A]], i64 [[IV]]
+; CHECK-MAXBW-NEXT:    [[LOAD_A:%.*]] = load i16, ptr [[GEP_A]], align 2
+; CHECK-MAXBW-NEXT:    [[EXT_A:%.*]] = zext i16 [[LOAD_A]] to i64
+; CHECK-MAXBW-NEXT:    [[ADD]] = add i64 [[EXT_A]], [[ACCUM]]
+; CHECK-MAXBW-NEXT:    [[IV_NEXT]] = add i64 [[IV]], 1
+; CHECK-MAXBW-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], 1025
+; CHECK-MAXBW-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_EXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP34:![0-9]+]]
+; CHECK-MAXBW:       for.exit:
+; CHECK-MAXBW-NEXT:    [[ADD_LCSSA:%.*]] = phi i64 [ [[ADD]], [[FOR_BODY]] ], [ [[TMP11]], [[MIDDLE_BLOCK]] ]
+; CHECK-MAXBW-NEXT:    ret i64 [[ADD_LCSSA]]
+;
+entry:
+  br label %for.body
+
+for.body:                                         ; preds = %for.body, %entry
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+  %accum = phi i64 [ 0, %entry ], [ %add, %for.body ]
+  %gep.a = getelementptr i16, ptr %a, i64 %iv
+  %load.a = load i16, ptr %gep.a, align 2
+  %ext.a = zext i16 %load.a to i64
+  %add = add i64 %ext.a, %accum
+  %iv.next = add i64 %iv, 1
+  %exitcond.not = icmp eq i64 %iv.next, 1025
+  br i1 %exitcond.not, label %for.exit, label %for.body
+
+for.exit:                        ; preds = %for.body
+  ret i64 %add
+}
+
+
+define i64 @zext_add_reduc_i8_i64_has_neon_dotprod(ptr %a) #1 {
+; CHECK-INTERLEAVE1-LABEL: define i64 @zext_add_reduc_i8_i64_has_neon_dotprod(
+; CHECK-INTERLEAVE1-SAME: ptr [[A:%.*]]) #[[ATTR1]] {
+; CHECK-INTERLEAVE1-NEXT:  entry:
+; CHECK-INTERLEAVE1-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK-INTERLEAVE1:       vector.ph:
+; CHECK-INTERLEAVE1-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK-INTERLEAVE1:       vector.body:
+; CHECK-INTERLEAVE1-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-INTERLEAVE1-NEXT:    [[VEC_PHI:%.*]] = phi <16 x i64> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP4:%.*]], [[VECTOR_BODY]] ]
+; CHECK-INTERLEAVE1-NEXT:    [[TMP0:%.*]] = add i64 [[INDEX]], 0
+; CHECK-INTERLEAVE1-NEXT:    [[TMP1:%.*]] = getelementptr i8, ptr [[A]], i64 [[TMP0]]
+; CHECK-INTERLEAVE1-NEXT:    [[TMP2:%.*]] = getelementptr i8, ptr [[TMP1]], i32 0
+; CHECK-INTERLEAVE1-NEXT:    [[WIDE_LOAD:%.*]] = load <16 x i8>, ptr [[TMP2]], align 1
+; CHECK-INTERLEAVE1-NEXT:    [[TMP3:%.*]] = zext <16 x i8> [[WIDE_LOAD]] to <16 x i64>
+; CHECK-INTERLEAVE1-NEXT:    [[TMP4]] = add <16 x i64> [[TMP3]], [[VEC_PHI]]
+; CHECK-INTERLEAVE1-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
+; CHECK-INTERLEAVE1-NEXT:    [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; CHECK-INTERLEAVE1-NEXT:    br i1 [[TMP5]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP35:![0-9]+]]
+; CHECK-INTERLEAVE1:       middle.block:
+; CHECK-INTERLEAVE1-NEXT:    [[TMP6:%.*]] = call i64 @llvm.vector.reduce.add.v16i64(<16 x i64> [[TMP4]])
+; CHECK-INTERLEAVE1-NEXT:    br i1 false, label [[FOR_EXIT:%.*]], label [[SCALAR_PH]]
+; CHECK-INTERLEAVE1:       scalar.ph:
+; CHECK-INTERLEAVE1-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 1024, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-INTERLEAVE1-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i64 [ [[TMP6]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ]
+; CHECK-INTERLEAVE1-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK-INTERLEAVE1:       for.body:
+; CHECK-INTERLEAVE1-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
+; CHECK-INTERLEAVE1-NEXT:    [[ACCUM:%.*]] = phi i64 [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[ADD:%.*]], [[FOR_BODY]] ]
+; CHECK-INTERLEAVE1-NEXT:    [[GEP_A:%.*]] = getelementptr i8, ptr [[A]], i64 [[IV]]
+; CHECK-INTERLEAVE1-NEXT:    [[LOAD_A:%.*]] = load i8, ptr [[GEP_A]], align 1
+; CHECK-INTERLEAVE1-NEXT:    [[EXT_A:%.*]] = zext i8 [[LOAD_A]] to i64
+; CHECK-INTERLEAVE1-NEXT:    [[ADD]] = add i64 [[EXT_A]], [[ACCUM]]
+; CHECK-INTERLEAVE1-NEXT:    [[IV_NEXT]] = add i64 [[IV]], 1
+; CHECK-INTERLEAVE1-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], 1025
+; CHECK-INTERLEAVE1-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_EXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP36:![0-9]+]]
+; CHECK-INTERLEAVE1:       for.exit:
+; CHECK-INTERLEAVE1-NEXT:    [[ADD_LCSSA:%.*]] = phi i64 [ [[ADD]], [[FOR_BODY]] ], [ [[TMP6]], [[MIDDLE_BLOCK]] ]
+; CHECK-INTERLEAVE1-NEXT:    ret i64 [[ADD_LCSSA]]
+;
+; CHECK-INTERLEAVED-LABEL: define i64 @zext_add_reduc_i8_i64_has_neon_dotprod(
+; CHECK-INTERLEAVED-SAME: ptr [[A:%.*]]) #[[ATTR1]] {
+; CHECK-INTERLEAVED-NEXT:  entry:
+; CHECK-INTERLEAVED-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK-INTERLEAVED:       vector.ph:
+; CHECK-INTERLEAVED-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK-INTERLEAVED:       vector.body:
+; CHECK-INTERLEAVED-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-INTERLEAVED-NEXT:    [[VEC_PHI:%.*]] = phi <16 x i64> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP6:%.*]], [[VECTOR_BODY]] ]
+; CHECK-INTERLEAVED-NEXT:    [[VEC_PHI1:%.*]] = phi <16 x i64> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP7:%.*]], [[VECTOR_BODY]] ]
+; CHECK-INTERLEAVED-NEXT:    [[TMP0:%.*]] = add i64 [[INDEX]], 0
+; CHECK-INTERLEAVED-NEXT:    [[TMP1:%.*]] = getelementptr i8, ptr [[A]], i64 [[TMP0]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP2:%.*]] = getelementptr i8, ptr [[TMP1]], i32 0
+; CHECK-INTERLEAVED-NEXT:    [[TMP3:%.*]] = getelementptr i8, ptr [[TMP1]], i32 16
+; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD:%.*]] = load <16 x i8>, ptr [[TMP2]], align 1
+; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD2:%.*]] = load <16 x i8>, ptr [[TMP3]], align 1
+; CHECK-INTERLEAVED-NEXT:    [[TMP4:%.*]] = zext <16 x i8> [[WIDE_LOAD]] to <16 x i64>
+; CHECK-INTERLEAVED-NEXT:    [[TMP5:%.*]] = zext <16 x i8> [[WIDE_LOAD2]] to <16 x i64>
+; CHECK-INTERLEAVED-NEXT:    [[TMP6]] = add <16 x i64> [[TMP4]], [[VEC_PHI]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP7]] = add <16 x i64> [[TMP5]], [[VEC_PHI1]]
+; CHECK-INTERLEAVED-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 32
+; CHECK-INTERLEAVED-NEXT:    [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; CHECK-INTERLEAVED-NEXT:    br i1 [[TMP8]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP37:![0-9]+]]
+; CHECK-INTERLEAVED:       middle.block:
+; CHECK-INTERLEAVED-NEXT:    [[BIN_RDX:%.*]] = add <16 x i64> [[TMP7]], [[TMP6]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP9:%.*]] = call i64 @llvm.vector.reduce.add.v16i64(<16 x i64> [[BIN_RDX]])
+; CHECK-INTERLEAVED-NEXT:    br i1 false, label [[FOR_EXIT:%.*]], label [[SCALAR_PH]]
+; CHECK-INTERLEAVED:       scalar.ph:
+; CHECK-INTERLEAVED-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 1024, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-INTERLEAVED-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i64 [ [[TMP9]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ]
+; CHECK-INTERLEAVED-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK-INTERLEAVED:       for.body:
+; CHECK-INTERLEAVED-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
+; CHECK-INTERLEAVED-NEXT:    [[ACCUM:%.*]] = phi i64 [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[ADD:%.*]], [[FOR_BODY]] ]
+; CHECK-INTERLEAVED-NEXT:    [[GEP_A:%.*]] = getelementptr i8, ptr [[A]], i64 [[IV]]
+; CHECK-INTERLEAVED-NEXT:    [[LOAD_A:%.*]] = load i8, ptr [[GEP_A]], align 1
+; CHECK-INTERLEAVED-NEXT:    [[EXT_A:%.*]] = zext i8 [[LOAD_A]] to i64
+; CHECK-INTERLEAVED-NEXT:    [[ADD]] = add i64 [[EXT_A]], [[ACCUM]]
+; CHECK-INTERLEAVED-NEXT:    [[IV_NEXT]] = add i64 [[IV]], 1
+; CHECK-INTERLEAVED-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], 1025
+; CHECK-INTERLEAVED-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_EXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP38:![0-9]+]]
+; CHECK-INTERLEAVED:       for.exit:
+; CHECK-INTERLEAVED-NEXT:    [[ADD_LCSSA:%.*]] = phi i64 [ [[ADD]], [[FOR_BODY]] ], [ [[TMP9]], [[MIDDLE_BLOCK]] ]
+; CHECK-INTERLEAVED-NEXT:    ret i64 [[ADD_LCSSA]]
+;
+; CHECK-MAXBW-LABEL: define i64 @zext_add_reduc_i8_i64_has_neon_dotprod(
+; CHECK-MAXBW-SAME: ptr [[A:%.*]]) #[[ATTR1]] {
+; CHECK-MAXBW-NEXT:  entry:
+; CHECK-MAXBW-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-MAXBW-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 8
+; CHECK-MAXBW-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK-MAXBW:       vector.ph:
+; CHECK-MAXBW-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-MAXBW-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 8
+; CHECK-MAXBW-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1025, [[TMP3]]
+; CHECK-MAXBW-NEXT:    [[N_VEC:%.*]] = sub i64 1025, [[N_MOD_VF]]
+; CHECK-MAXBW-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-MAXBW-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 8
+; CHECK-MAXBW-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK-MAXBW:       vector.body:
+; CHECK-MAXBW-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-MAXBW-NEXT:    [[VEC_PHI:%.*]] = phi <vscale x 8 x i64> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP10:%.*]], [[VECTOR_BODY]] ]
+; CHECK-MAXBW-NEXT:    [[TMP6:%.*]] = add i64 [[INDEX]], 0
+; CHECK-MAXBW-NEXT:    [[TMP7:%.*]] = getelementptr i8, ptr [[A]], i64 [[TMP6]]
+; CHECK-MAXBW-NEXT:    [[TMP8:%.*]] = getelementptr i8, ptr [[TMP7]], i32 0
+; CHECK-MAXBW-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 8 x i8>, ptr [[TMP8]], align 1
+; CHECK-MAXBW-NEXT:    [[TMP9:%.*]] = zext <vscale x 8 x i8> [[WIDE_LOAD]] to <vscale x 8 x i64>
+; CHECK-MAXBW-NEXT:    [[TMP10]] = add <vscale x 8 x i64> [[TMP9]], [[VEC_PHI]]
+; CHECK-MAXBW-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
+; CHECK-MAXBW-NEXT:    [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-MAXBW-NEXT:    br i1 [[TMP11]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP35:![0-9]+]]
+; CHECK-MAXBW:       middle.block:
+; CHECK-MAXBW-NEXT:    [[TMP12:%.*]] = call i64 @llvm.vector.reduce.add.nxv8i64(<vscale x 8 x i64> [[TMP10]])
+; CHECK-MAXBW-NEXT:    [[CMP_N:%.*]] = icmp eq i64 1025, [[N_VEC]]
+; CHECK-MAXBW-NEXT:    br i1 [[CMP_N]], label [[FOR_EXIT:%.*]], label [[SCALAR_PH]]
+; CHECK-MAXBW:       scalar.ph:
+; CHECK-MAXBW-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-MAXBW-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i64 [ [[TMP12]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ]
+; CHECK-MAXBW-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK-MAXBW:       for.body:
+; CHECK-MAXBW-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
+; CHECK-MAXBW-NEXT:    [[ACCUM:%.*]] = phi i64 [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[ADD:%.*]], [[FOR_BODY]] ]
+; CHECK-MAXBW-NEXT:    [[GEP_A:%.*]] = getelementptr i8, ptr [[A]], i64 [[IV]]
+; CHECK-MAXBW-NEXT:    [[LOAD_A:%.*]] = load i8, ptr [[GEP_A]], align 1
+; CHECK-MAXBW-NEXT:    [[EXT_A:%.*]] = zext i8 [[LOAD_A]] to i64
+; CHECK-MAXBW-NEXT:    [[ADD]] = add i64 [[EXT_A]], [[ACCUM]]
+; CHECK-MAXBW-NEXT:    [[IV_NEXT]] = add i64 [[IV]], 1
+; CHECK-MAXBW-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], 1025
+; CHECK-MAXBW-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_EXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP36:![0-9]+]]
+; CHECK-MAXBW:       for.exit:
+; CHECK-MAXBW-NEXT:    [[ADD_LCSSA:%.*]] = phi i64 [ [[ADD]], [[FOR_BODY]] ], [ [[TMP12]], [[MIDDLE_BLOCK]] ]
+; CHECK-MAXBW-NEXT:    ret i64 [[ADD_LCSSA]]
+;
+entry:
+  br label %for.body
+
+for.body:                                         ; preds = %for.body, %entry
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+  %accum = phi i64 [ 0, %entry ], [ %add, %for.body ]
+  %gep.a = getelementptr i8, ptr %a, i64 %iv
+  %load.a = load i8, ptr %gep.a, align 1
+  %ext.a = zext i8 %load.a to i64
+  %add = add i64 %ext.a, %accum
+  %iv.next = add i64 %iv, 1
+  %exitcond.not = icmp eq i64 %iv.next, 1025
+  br i1 %exitcond.not, label %for.exit, label %for.body
+
+for.exit:                        ; preds = %for.body
+  ret i64 %add
+}
+
+
+define i32 @sext_add_reduc_i8_i32(ptr %a) #0 {
+; CHECK-INTERLEAVE1-LABEL: define i32 @sext_add_reduc_i8_i32(
+; CHECK-INTERLEAVE1-SAME: ptr [[A:%.*]]) #[[ATTR0]] {
+; CHECK-INTERLEAVE1-NEXT:  entry:
+; CHECK-INTERLEAVE1-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK-INTERLEAVE1:       vector.ph:
+; CHECK-INTERLEAVE1-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK-INTERLEAVE1:       vector.body:
+; CHECK-INTERLEAVE1-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-INTERLEAVE1-NEXT:    [[VEC_PHI:%.*]] = phi <16 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP4:%.*]], [[VECTOR_BODY]] ]
+; CHECK-INTERLEAVE1-NEXT:    [[TMP0:%.*]] = add i64 [[INDEX]], 0
+; CHECK-INTERLEAVE1-NEXT:    [[TMP1:%.*]] = getelementptr i8, ptr [[A]], i64 [[TMP0]]
+; CHECK-INTERLEAVE1-NEXT:    [[TMP2:%.*]] = getelementptr i8, ptr [[TMP1]], i32 0
+; CHECK-INTERLEAVE1-NEXT:    [[WIDE_LOAD:%.*]] = load <16 x i8>, ptr [[TMP2]], align 1
+; CHECK-INTERLEAVE1-NEXT:    [[TMP3:%.*]] = sext <16 x i8> [[WIDE_LOAD]] to <16 x i32>
+; CHECK-INTERLEAVE1-NEXT:    [[TMP4]] = add <16 x i32> [[TMP3]], [[VEC_PHI]]
+; CHECK-INTERLEAVE1-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
+; CHECK-INTERLEAVE1-NEXT:    [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; CHECK-INTERLEAVE1-NEXT:    br i1 [[TMP5]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP37:![0-9]+]]
+; CHECK-INTERLEAVE1:       middle.block:
+; CHECK-INTERLEAVE1-NEXT:    [[TMP6:%.*]] = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> [[TMP4]])
+; CHECK-INTERLEAVE1-NEXT:    br i1 false, label [[FOR_EXIT:%.*]], label [[SCALAR_PH]]
+; CHECK-INTERLEAVE1:       scalar.ph:
+; CHECK-INTERLEAVE1-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 1024, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-INTERLEAVE1-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP6]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ]
+; CHECK-INTERLEAVE1-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK-INTERLEAVE1:       for.body:
+; CHECK-INTERLEAVE1-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
+; CHECK-INTERLEAVE1-NEXT:    [[ACCUM:%.*]] = phi i32 [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[ADD:%.*]], [[FOR_BODY]] ]
+; CHECK-INTERLEAVE1-NEXT:    [[GEP_A:%.*]] = getelementptr i8, ptr [[A]], i64 [[IV]]
+; CHECK-INTERLEAVE1-NEXT:    [[LOAD_A:%.*]] = load i8, ptr [[GEP_A]], align 1
+; CHECK-INTERLEAVE1-NEXT:    [[EXT_A:%.*]] = sext i8 [[LOAD_A]] to i32
+; CHECK-INTERLEAVE1-NEXT:    [[ADD]] = add i32 [[EXT_A]], [[ACCUM]]
+; CHECK-INTERLEAVE1-NEXT:    [[IV_NEXT]] = add i64 [[IV]], 1
+; CHECK-INTERLEAVE1-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], 1025
+; CHECK-INTERLEAVE1-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_EXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP38:![0-9]+]]
+; CHECK-INTERLEAVE1:       for.exit:
+; CHECK-INTERLEAVE1-NEXT:    [[ADD_LCSSA:%.*]] = phi i32 [ [[ADD]], [[FOR_BODY]] ], [ [[TMP6]], [[MIDDLE_BLOCK]] ]
+; CHECK-INTERLEAVE1-NEXT:    ret i32 [[ADD_LCSSA]]
+;
+; CHECK-INTERLEAVED-LABEL: define i32 @sext_add_reduc_i8_i32(
+; CHECK-INTERLEAVED-SAME: ptr [[A:%.*]]) #[[ATTR0]] {
+; CHECK-INTERLEAVED-NEXT:  entry:
+; CHECK-INTERLEAVED-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK-INTERLEAVED:       vector.ph:
+; CHECK-INTERLEAVED-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK-INTERLEAVED:       vector.body:
+; CHECK-INTERLEAVED-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-INTERLEAVED-NEXT:    [[VEC_PHI:%.*]] = phi <16 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP6:%.*]], [[VECTOR_BODY]] ]
+; CHECK-INTERLEAVED-NEXT:    [[VEC_PHI1:%.*]] = phi <16 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP7:%.*]], [[VECTOR_BODY]] ]
+; CHECK-INTERLEAVED-NEXT:    [[TMP0:%.*]] = add i64 [[INDEX]], 0
+; CHECK-INTERLEAVED-NEXT:    [[TMP1:%.*]] = getelementptr i8, ptr [[A]], i64 [[TMP0]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP2:%.*]] = getelementptr i8, ptr [[TMP1]], i32 0
+; CHECK-INTERLEAVED-NEXT:    [[TMP3:%.*]] = getelementptr i8, ptr [[TMP1]], i32 16
+; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD:%.*]] = load <16 x i8>, ptr [[TMP2]], align 1
+; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD2:%.*]] = load <16 x i8>, ptr [[TMP3]], align 1
+; CHECK-INTERLEAVED-NEXT:    [[TMP4:%.*]] = sext <16 x i8> [[WIDE_LOAD]] to <16 x i32>
+; CHECK-INTERLEAVED-NEXT:    [[TMP5:%.*]] = sext <16 x i8> [[WIDE_LOAD2]] to <16 x i32>
+; CHECK-INTERLEAVED-NEXT:    [[TMP6]] = add <16 x i32> [[TMP4]], [[VEC_PHI]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP7]] = add <16 x i32> [[TMP5]], [[VEC_PHI1]]
+; CHECK-INTERLEAVED-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 32
+; CHECK-INTERLEAVED-NEXT:    [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; CHECK-INTERLEAVED-NEXT:    br i1 [[TMP8]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP39:![0-9]+]]
+; CHECK-INTERLEAVED:       middle.block:
+; CHECK-INTERLEAVED-NEXT:    [[BIN_RDX:%.*]] = add <16 x i32> [[TMP7]], [[TMP6]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP9:%.*]] = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> [[BIN_RDX]])
+; CHECK-INTERLEAVED-NEXT:    br i1 false, label [[FOR_EXIT:%.*]], label [[SCALAR_PH]]
+; CHECK-INTERLEAVED:       scalar.ph:
+; CHECK-INTERLEAVED-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 1024, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-INTERLEAVED-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP9]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ]
+; CHECK-INTERLEAVED-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK-INTERLEAVED:       for.body:
+; CHECK-INTERLEAVED-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
+; CHECK-INTERLEAVED-NEXT:    [[ACCUM:%.*]] = phi i32 [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[ADD:%.*]], [[FOR_BODY]] ]
+; CHECK-INTERLEAVED-NEXT:    [[GEP_A:%.*]] = getelementptr i8, ptr [[A]], i64 [[IV]]
+; CHECK-INTERLEAVED-NEXT:    [[LOAD_A:%.*]] = load i8, ptr [[GEP_A]], align 1
+; CHECK-INTERLEAVED-NEXT:    [[EXT_A:%.*]] = sext i8 [[LOAD_A]] to i32
+; CHECK-INTERLEAVED-NEXT:    [[ADD]] = add i32 [[EXT_A]], [[ACCUM]]
+; CHECK-INTERLEAVED-NEXT:    [[IV_NEXT]] = add i64 [[IV]], 1
+; CHECK-INTERLEAVED-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], 1025
+; CHECK-INTERLEAVED-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_EXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP40:![0-9]+]]
+; CHECK-INTERLEAVED:       for.exit:
+; CHECK-INTERLEAVED-NEXT:    [[ADD_LCSSA:%.*]] = phi i32 [ [[ADD]], [[FOR_BODY]] ], [ [[TMP9]], [[MIDDLE_BLOCK]] ]
+; CHECK-INTERLEAVED-NEXT:    ret i32 [[ADD_LCSSA]]
+;
+; CHECK-MAXBW-LABEL: define i32 @sext_add_reduc_i8_i32(
+; CHECK-MAXBW-SAME: ptr [[A:%.*]]) #[[ATTR0]] {
+; CHECK-MAXBW-NEXT:  entry:
+; CHECK-MAXBW-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-MAXBW-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 8
+; CHECK-MAXBW-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK-MAXBW:       vector.ph:
+; CHECK-MAXBW-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-MAXBW-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 8
+; CHECK-MAXBW-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1025, [[TMP3]]
+; CHECK-MAXBW-NEXT:    [[N_VEC:%.*]] = sub i64 1025, [[N_MOD_VF]]
+; CHECK-MAXBW-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-MAXBW-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 8
+; CHECK-MAXBW-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK-MAXBW:       vector.body:
+; CHECK-MAXBW-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-MAXBW-NEXT:    [[VEC_PHI:%.*]] = phi <vscale x 8 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP12:%.*]], [[VECTOR_BODY]] ]
+; CHECK-MAXBW-NEXT:    [[TMP6:%.*]] = add i64 [[INDEX]], 0
+; CHECK-MAXBW-NEXT:    [[TMP7:%.*]] = getelementptr i8, ptr [[A]], i64 [[TMP6]]
+; CHECK-MAXBW-NEXT:    [[TMP8:%.*]] = getelementptr i8, ptr [[TMP7]], i32 0
+; CHECK-MAXBW-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 8 x i8>, ptr [[TMP8]], align 1
+; CHECK-MAXBW-NEXT:    [[TMP9:%.*]] = sext <vscale x 8 x i8> [[WIDE_LOAD]] to <vscale x 8 x i32>
+; CHECK-MAXBW-NEXT:    [[TMP12]] = add <vscale x 8 x i32> [[TMP9]], [[VEC_PHI]]
+; CHECK-MAXBW-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
+; CHECK-MAXBW-NEXT:    [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-MAXBW-NEXT:    br i1 [[TMP10]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP37:![0-9]+]]
+; CHECK-MAXBW:       middle.block:
+; CHECK-MAXBW-NEXT:    [[TMP11:%.*]] = call i32 @llvm.vector.reduce.add.nxv8i32(<vscale x 8 x i32> [[TMP12]])
+; CHECK-MAXBW-NEXT:    [[CMP_N:%.*]] = icmp eq i64 1025, [[N_VEC]]
+; CHECK-MAXBW-NEXT:    br i1 [[CMP_N]], label [[FOR_EXIT:%.*]], label [[SCALAR_PH]]
+; CHECK-MAXBW:       scalar.ph:
+; CHECK-MAXBW-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-MAXBW-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP11]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ]
+; CHECK-MAXBW-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK-MAXBW:       for.body:
+; CHECK-MAXBW-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
+; CHECK-MAXBW-NEXT:    [[ACCUM:%.*]] = phi i32 [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[ADD:%.*]], [[FOR_BODY]] ]
+; CHECK-MAXBW-NEXT:    [[GEP_A:%.*]] = getelementptr i8, ptr [[A]], i64 [[IV]]
+; CHECK-MAXBW-NEXT:    [[LOAD_A:%.*]] = load i8, ptr [[GEP_A]], align 1
+; CHECK-MAXBW-NEXT:    [[EXT_A:%.*]] = sext i8 [[LOAD_A]] to i32
+; CHECK-MAXBW-NEXT:    [[ADD]] = add i32 [[EXT_A]], [[ACCUM]]
+; CHECK-MAXBW-NEXT:    [[IV_NEXT]] = add i64 [[IV]], 1
+; CHECK-MAXBW-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], 1025
+; CHECK-MAXBW-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_EXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP38:![0-9]+]]
+; CHECK-MAXBW:       for.exit:
+; CHECK-MAXBW-NEXT:    [[ADD_LCSSA:%.*]] = phi i32 [ [[ADD]], [[FOR_BODY]] ], [ [[TMP11]], [[MIDDLE_BLOCK]] ]
+; CHECK-MAXBW-NEXT:    ret i32 [[ADD_LCSSA]]
+;
+entry:
+  br label %for.body
+
+for.body:                                         ; preds = %for.body, %entry
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+  %accum = phi i32 [ 0, %entry ], [ %add, %for.body ]
+  %gep.a = getelementptr i8, ptr %a, i64 %iv
+  %load.a = load i8, ptr %gep.a, align 1
+  %ext.a = sext i8 %load.a to i32
+  %add = add i32 %ext.a, %accum
+  %iv.next = add i64 %iv, 1
+  %exitcond.not = icmp eq i64 %iv.next, 1025
+  br i1 %exitcond.not, label %for.exit, label %for.body
+
+for.exit:                        ; preds = %for.body
+  ret i32 %add
+}
+
+
 !7 = distinct !{!7, !8, !9, !10}
 !8 = !{!"llvm.loop.mustprogress"}
 !9 = !{!"llvm.loop.vectorize.predicate.enable", i1 true}
 !10 = !{!"llvm.loop.vectorize.enable", i1 true}
 attributes #0 = { vscale_range(1,16) "target-features"="+sve" }
-;.
-; CHECK-INTERLEAVE1: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]], [[META2:![0-9]+]]}
-; CHECK-INTERLEAVE1: [[META1]] = !{!"llvm.loop.isvectorized", i32 1}
-; CHECK-INTERLEAVE1: [[META2]] = !{!"llvm.loop.unroll.runtime.disable"}
-; CHECK-INTERLEAVE1: [[LOOP3]] = distinct !{[[LOOP3]], [[META2]], [[META1]]}
-; CHECK-INTERLEAVE1: [[LOOP4]] = distinct !{[[LOOP4]], [[META1]], [[META2]]}
-; CHECK-INTERLEAVE1: [[LOOP5]] = distinct !{[[LOOP5]], [[META2]], [[META1]]}
-; CHECK-INTERLEAVE1: [[LOOP6]] = distinct !{[[LOOP6]], [[META1]], [[META2]]}
-; CHECK-INTERLEAVE1: [[LOOP7]] = distinct !{[[LOOP7]], [[META2]], [[META1]]}
-; CHECK-INTERLEAVE1: [[LOOP8]] = distinct !{[[LOOP8]], [[META1]], [[META2]]}
-; CHECK-INTERLEAVE1: [[LOOP9]] = distinct !{[[LOOP9]], [[META2]], [[META1]]}
-; CHECK-INTERLEAVE1: [[LOOP10]] = distinct !{[[LOOP10]], [[META1]], [[META2]]}
-; CHECK-INTERLEAVE1: [[LOOP11]] = distinct !{[[LOOP11]], [[META2]], [[META1]]}
-; CHECK-INTERLEAVE1: [[LOOP12]] = distinct !{[[LOOP12]], [[META1]], [[META2]]}
-; CHECK-INTERLEAVE1: [[LOOP13]] = distinct !{[[LOOP13]], [[META2]], [[META1]]}
-; CHECK-INTERLEAVE1: [[LOOP14]] = distinct !{[[LOOP14]], [[META15:![0-9]+]], [[META1]], [[META2]]}
-; CHECK-INTERLEAVE1: [[META15]] = !{!"llvm.loop.mustprogress"}
-; CHECK-INTERLEAVE1: [[LOOP16]] = distinct !{[[LOOP16]], [[META15]], [[META2]], [[META1]]}
-; CHECK-INTERLEAVE1: [[LOOP17]] = distinct !{[[LOOP17]], [[META1]], [[META2]]}
-; CHECK-INTERLEAVE1: [[LOOP18]] = distinct !{[[LOOP18]], [[META2]], [[META1]]}
-; CHECK-INTERLEAVE1: [[LOOP19]] = distinct !{[[LOOP19]], [[META1]], [[META2]]}
-; CHECK-INTERLEAVE1: [[LOOP20]] = distinct !{[[LOOP20]], [[META2]], [[META1]]}
-; CHECK-INTERLEAVE1: [[LOOP21]] = distinct !{[[LOOP21]], [[META1]], [[META2]]}
-; CHECK-INTERLEAVE1: [[LOOP22]] = distinct !{[[LOOP22]], [[META2]], [[META1]]}
-; CHECK-INTERLEAVE1: [[LOOP23]] = distinct !{[[LOOP23]], [[META1]], [[META2]]}
-; CHECK-INTERLEAVE1: [[LOOP24]] = distinct !{[[LOOP24]], [[META2]], [[META1]]}
-;.
-; CHECK-INTERLEAVED: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]], [[META2:![0-9]+]]}
-; CHECK-INTERLEAVED: [[META1]] = !{!"llvm.loop.isvectorized", i32 1}
-; CHECK-INTERLEAVED: [[META2]] = !{!"llvm.loop.unroll.runtime.disable"}
-; CHECK-INTERLEAVED: [[LOOP3]] = distinct !{[[LOOP3]], [[META2]], [[META1]]}
-; CHECK-INTERLEAVED: [[LOOP4]] = distinct !{[[LOOP4]], [[META1]], [[META2]]}
-; CHECK-INTERLEAVED: [[LOOP5]] = distinct !{[[LOOP5]], [[META2]], [[META1]]}
-; CHECK-INTERLEAVED: [[LOOP6]] = distinct !{[[LOOP6]], [[META1]], [[META2]]}
-; CHECK-INTERLEAVED: [[LOOP7]] = distinct !{[[LOOP7]], [[META2]], [[META1]]}
-; CHECK-INTERLEAVED: [[LOOP8]] = distinct !{[[LOOP8]], [[META1]], [[META2]]}
-; CHECK-INTERLEAVED: [[LOOP9]] = distinct !{[[LOOP9]], [[META2]], [[META1]]}
-; CHECK-INTERLEAVED: [[LOOP10]] = distinct !{[[LOOP10]], [[META1]], [[META2]]}
-; CHECK-INTERLEAVED: [[LOOP11]] = distinct !{[[LOOP11]], [[META2]], [[META1]]}
-; CHECK-INTERLEAVED: [[LOOP12]] = distinct !{[[LOOP12]], [[META1]], [[META2]]}
-; CHECK-INTERLEAVED: [[LOOP13]] = distinct !{[[LOOP13]], [[META2]], [[META1]]}
-; CHECK-INTERLEAVED: [[LOOP14]] = distinct !{[[LOOP14]], [[META15:![0-9]+]], [[META1]], [[META2]]}
-; CHECK-INTERLEAVED: [[META15]] = !{!"llvm.loop.mustprogress"}
-; CHECK-INTERLEAVED: [[LOOP16]] = distinct !{[[LOOP16]], [[META15]], [[META2]], [[META1]]}
-; CHECK-INTERLEAVED: [[LOOP17]] = distinct !{[[LOOP17]], [[META1]], [[META2]]}
-; CHECK-INTERLEAVED: [[LOOP18]] = distinct !{[[LOOP18]], [[META2]], [[META1]]}
-; CHECK-INTERLEAVED: [[LOOP19]] = distinct !{[[LOOP19]], [[META1]], [[META2]]}
-; CHECK-INTERLEAVED: [[LOOP20]] = distinct !{[[LOOP20]], [[META2]], [[META1]]}
-; CHECK-INTERLEAVED: [[LOOP21]] = distinct !{[[LOOP21]], [[META1]], [[META2]]}
-; CHECK-INTERLEAVED: [[LOOP22]] = distinct !{[[LOOP22]], [[META1]]}
-; CHECK-INTERLEAVED: [[LOOP23]] = distinct !{[[LOOP23]], [[META1]], [[META2]]}
-; CHECK-INTERLEAVED: [[LOOP24]] = distinct !{[[LOOP24]], [[META2]], [[META1]]}
-; CHECK-INTERLEAVED: [[LOOP25]] = distinct !{[[LOOP25]], [[META1]], [[META2]]}
-; CHECK-INTERLEAVED: [[LOOP26]] = distinct !{[[LOOP26]], [[META2]], [[META1]]}
-;.
-; CHECK-MAXBW: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]], [[META2:![0-9]+]]}
-; CHECK-MAXBW: [[META1]] = !{!"llvm.loop.isvectorized", i32 1}
-; CHECK-MAXBW: [[META2]] = !{!"llvm.loop.unroll.runtime.disable"}
-; CHECK-MAXBW: [[LOOP3]] = distinct !{[[LOOP3]], [[META2]], [[META1]]}
-; CHECK-MAXBW: [[LOOP4]] = distinct !{[[LOOP4]], [[META1]], [[META2]]}
-; CHECK-MAXBW: [[LOOP5]] = distinct !{[[LOOP5]], [[META2]], [[META1]]}
-; CHECK-MAXBW: [[LOOP6]] = distinct !{[[LOOP6]], [[META1]], [[META2]]}
-; CHECK-MAXBW: [[LOOP7]] = distinct !{[[LOOP7]], [[META2]], [[META1]]}
-; CHECK-MAXBW: [[LOOP8]] = distinct !{[[LOOP8]], [[META1]], [[META2]]}
-; CHECK-MAXBW: [[LOOP9]] = distinct !{[[LOOP9]], [[META2]], [[META1]]}
-; CHECK-MAXBW: [[LOOP10]] = distinct !{[[LOOP10]], [[META1]], [[META2]]}
-; CHECK-MAXBW: [[LOOP11]] = distinct !{[[LOOP11]], [[META2]], [[META1]]}
-; CHECK-MAXBW: [[LOOP12]] = distinct !{[[LOOP12]], [[META1]], [[META2]]}
-; CHECK-MAXBW: [[LOOP13]] = distinct !{[[LOOP13]], [[META2]], [[META1]]}
-; CHECK-MAXBW: [[LOOP14]] = distinct !{[[LOOP14]], [[META15:![0-9]+]], [[META1]], [[META2]]}
-; CHECK-MAXBW: [[META15]] = !{!"llvm.loop.mustprogress"}
-; CHECK-MAXBW: [[LOOP16]] = distinct !{[[LOOP16]], [[META15]], [[META2]], [[META1]]}
-; CHECK-MAXBW: [[LOOP17]] = distinct !{[[LOOP17]], [[META1]], [[META2]]}
-; CHECK-MAXBW: [[LOOP18]] = distinct !{[[LOOP18]], [[META2]], [[META1]]}
-; CHECK-MAXBW: [[LOOP19]] = distinct !{[[LOOP19]], [[META1]], [[META2]]}
-; CHECK-MAXBW: [[LOOP20]] = distinct !{[[LOOP20]], [[META2]], [[META1]]}
-; CHECK-MAXBW: [[LOOP21]] = distinct !{[[LOOP21]], [[META1]], [[META2]]}
-; CHECK-MAXBW: [[LOOP22]] = distinct !{[[LOOP22]], [[META2]], [[META1]]}
-; CHECK-MAXBW: [[LOOP23]] = distinct !{[[LOOP23]], [[META1]], [[META2]]}
-; CHECK-MAXBW: [[LOOP24]] = distinct !{[[LOOP24]], [[META2]], [[META1]]}
-;.
+attributes #1 = { vscale_range(1,16) "target-features"="+neon,+dotprod,+sve" "cpu"="neoverse-v2" }
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/pr60831-sve-inv-store-crash.ll b/llvm/test/Transforms/LoopVectorize/AArch64/pr60831-sve-inv-store-crash.ll
index d95cb6be0e858..72a1bb2c00e54 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/pr60831-sve-inv-store-crash.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/pr60831-sve-inv-store-crash.ll
@@ -181,7 +181,7 @@ define void @test_loop2(i64 %n, ptr %dst) {
 ; CHECK:       vec.epilog.middle.block:
 ; CHECK-NEXT:    br i1 false, label [[EXIT]], label [[VEC_EPILOG_SCALAR_PH]]
 ; CHECK:       vec.epilog.scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 1000, [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ 0, [[ITER_CHECK:%.*]] ], [ 992, [[VEC_EPILOG_ITER_CHECK]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 1000, [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ 992, [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[ITER_CHECK:%.*]] ]
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
 ; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[VEC_EPILOG_SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/scalable-strict-fadd.ll b/llvm/test/Transforms/LoopVectorize/AArch64/scalable-strict-fadd.ll
index cb4fd04d1bc4f..d612e1f3dab5b 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/scalable-strict-fadd.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/scalable-strict-fadd.ll
@@ -14,7 +14,7 @@
 
 define float @fadd_strict(ptr noalias nocapture readonly %a, i64 %n) #0 {
 ; CHECK-NOT-VECTORIZED-LABEL: define float @fadd_strict
-; CHECK-NOT-VECTORIZED-SAME: (ptr noalias nocapture readonly [[A:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NOT-VECTORIZED-SAME: (ptr noalias readonly captures(none) [[A:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
 ; CHECK-NOT-VECTORIZED-NEXT:  entry:
 ; CHECK-NOT-VECTORIZED-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK-NOT-VECTORIZED:       for.body:
@@ -31,7 +31,7 @@ define float @fadd_strict(ptr noalias nocapture readonly %a, i64 %n) #0 {
 ; CHECK-NOT-VECTORIZED-NEXT:    ret float [[ADD_LCSSA]]
 ;
 ; CHECK-UNORDERED-LABEL: define float @fadd_strict
-; CHECK-UNORDERED-SAME: (ptr noalias nocapture readonly [[A:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-UNORDERED-SAME: (ptr noalias readonly captures(none) [[A:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
 ; CHECK-UNORDERED-NEXT:  entry:
 ; CHECK-UNORDERED-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
 ; CHECK-UNORDERED-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 8
@@ -78,7 +78,7 @@ define float @fadd_strict(ptr noalias nocapture readonly %a, i64 %n) #0 {
 ; CHECK-UNORDERED-NEXT:    ret float [[ADD_LCSSA]]
 ;
 ; CHECK-ORDERED-LABEL: define float @fadd_strict
-; CHECK-ORDERED-SAME: (ptr noalias nocapture readonly [[A:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-ORDERED-SAME: (ptr noalias readonly captures(none) [[A:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
 ; CHECK-ORDERED-NEXT:  entry:
 ; CHECK-ORDERED-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
 ; CHECK-ORDERED-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 8
@@ -124,7 +124,7 @@ define float @fadd_strict(ptr noalias nocapture readonly %a, i64 %n) #0 {
 ; CHECK-ORDERED-NEXT:    ret float [[ADD_LCSSA]]
 ;
 ; CHECK-ORDERED-TF-LABEL: define float @fadd_strict
-; CHECK-ORDERED-TF-SAME: (ptr noalias nocapture readonly [[A:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-ORDERED-TF-SAME: (ptr noalias readonly captures(none) [[A:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
 ; CHECK-ORDERED-TF-NEXT:  entry:
 ; CHECK-ORDERED-TF-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK-ORDERED-TF:       vector.ph:
@@ -200,7 +200,7 @@ for.end:
 
 define float @fadd_strict_unroll(ptr noalias nocapture readonly %a, i64 %n) #0 {
 ; CHECK-NOT-VECTORIZED-LABEL: define float @fadd_strict_unroll
-; CHECK-NOT-VECTORIZED-SAME: (ptr noalias nocapture readonly [[A:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-NOT-VECTORIZED-SAME: (ptr noalias readonly captures(none) [[A:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
 ; CHECK-NOT-VECTORIZED-NEXT:  entry:
 ; CHECK-NOT-VECTORIZED-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK-NOT-VECTORIZED:       for.body:
@@ -217,7 +217,7 @@ define float @fadd_strict_unroll(ptr noalias nocapture readonly %a, i64 %n) #0 {
 ; CHECK-NOT-VECTORIZED-NEXT:    ret float [[ADD_LCSSA]]
 ;
 ; CHECK-UNORDERED-LABEL: define float @fadd_strict_unroll
-; CHECK-UNORDERED-SAME: (ptr noalias nocapture readonly [[A:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-UNORDERED-SAME: (ptr noalias readonly captures(none) [[A:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
 ; CHECK-UNORDERED-NEXT:  entry:
 ; CHECK-UNORDERED-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
 ; CHECK-UNORDERED-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 32
@@ -285,7 +285,7 @@ define float @fadd_strict_unroll(ptr noalias nocapture readonly %a, i64 %n) #0 {
 ; CHECK-UNORDERED-NEXT:    ret float [[ADD_LCSSA]]
 ;
 ; CHECK-ORDERED-LABEL: define float @fadd_strict_unroll
-; CHECK-ORDERED-SAME: (ptr noalias nocapture readonly [[A:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-ORDERED-SAME: (ptr noalias readonly captures(none) [[A:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
 ; CHECK-ORDERED-NEXT:  entry:
 ; CHECK-ORDERED-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
 ; CHECK-ORDERED-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 32
@@ -346,7 +346,7 @@ define float @fadd_strict_unroll(ptr noalias nocapture readonly %a, i64 %n) #0 {
 ; CHECK-ORDERED-NEXT:    ret float [[ADD_LCSSA]]
 ;
 ; CHECK-ORDERED-TF-LABEL: define float @fadd_strict_unroll
-; CHECK-ORDERED-TF-SAME: (ptr noalias nocapture readonly [[A:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-ORDERED-TF-SAME: (ptr noalias readonly captures(none) [[A:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
 ; CHECK-ORDERED-TF-NEXT:  entry:
 ; CHECK-ORDERED-TF-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK-ORDERED-TF:       vector.ph:
@@ -467,7 +467,7 @@ for.end:
 
 define void @fadd_strict_interleave(ptr noalias nocapture readonly %a, ptr noalias nocapture readonly %b, i64 %n) #0 {
 ; CHECK-NOT-VECTORIZED-LABEL: define void @fadd_strict_interleave
-; CHECK-NOT-VECTORIZED-SAME: (ptr noalias nocapture readonly [[A:%.*]], ptr noalias nocapture readonly [[B:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-NOT-VECTORIZED-SAME: (ptr noalias readonly captures(none) [[A:%.*]], ptr noalias readonly captures(none) [[B:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
 ; CHECK-NOT-VECTORIZED-NEXT:  entry:
 ; CHECK-NOT-VECTORIZED-NEXT:    [[ARRAYIDXA:%.*]] = getelementptr inbounds float, ptr [[A]], i64 1
 ; CHECK-NOT-VECTORIZED-NEXT:    [[A1:%.*]] = load float, ptr [[A]], align 4
@@ -495,7 +495,7 @@ define void @fadd_strict_interleave(ptr noalias nocapture readonly %a, ptr noali
 ; CHECK-NOT-VECTORIZED-NEXT:    ret void
 ;
 ; CHECK-UNORDERED-LABEL: define void @fadd_strict_interleave
-; CHECK-UNORDERED-SAME: (ptr noalias nocapture readonly [[A:%.*]], ptr noalias nocapture readonly [[B:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-UNORDERED-SAME: (ptr noalias readonly captures(none) [[A:%.*]], ptr noalias readonly captures(none) [[B:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
 ; CHECK-UNORDERED-NEXT:  entry:
 ; CHECK-UNORDERED-NEXT:    [[ARRAYIDXA:%.*]] = getelementptr inbounds float, ptr [[A]], i64 1
 ; CHECK-UNORDERED-NEXT:    [[A1:%.*]] = load float, ptr [[A]], align 4
@@ -514,59 +514,59 @@ define void @fadd_strict_interleave(ptr noalias nocapture readonly %a, ptr noali
 ; CHECK-UNORDERED-NEXT:    [[N_VEC:%.*]] = sub i64 [[TMP2]], [[N_MOD_VF]]
 ; CHECK-UNORDERED-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
 ; CHECK-UNORDERED-NEXT:    [[TMP8:%.*]] = mul i64 [[TMP7]], 4
-; CHECK-UNORDERED-NEXT:    [[IND_END:%.*]] = mul i64 [[N_VEC]], 2
-; CHECK-UNORDERED-NEXT:    [[TMP9:%.*]] = insertelement <vscale x 4 x float> splat (float -0.000000e+00), float [[A2]], i32 0
-; CHECK-UNORDERED-NEXT:    [[TMP10:%.*]] = insertelement <vscale x 4 x float> splat (float -0.000000e+00), float [[A1]], i32 0
+; CHECK-UNORDERED-NEXT:    [[TMP9:%.*]] = mul i64 [[N_VEC]], 2
+; CHECK-UNORDERED-NEXT:    [[TMP10:%.*]] = insertelement <vscale x 4 x float> splat (float -0.000000e+00), float [[A2]], i32 0
+; CHECK-UNORDERED-NEXT:    [[TMP11:%.*]] = insertelement <vscale x 4 x float> splat (float -0.000000e+00), float [[A1]], i32 0
 ; CHECK-UNORDERED-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK-UNORDERED:       vector.body:
 ; CHECK-UNORDERED-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-UNORDERED-NEXT:    [[VEC_PHI:%.*]] = phi <vscale x 4 x float> [ [[TMP9]], [[VECTOR_PH]] ], [ [[TMP16:%.*]], [[VECTOR_BODY]] ]
-; CHECK-UNORDERED-NEXT:    [[VEC_PHI1:%.*]] = phi <vscale x 4 x float> [ [[TMP10]], [[VECTOR_PH]] ], [ [[TMP15:%.*]], [[VECTOR_BODY]] ]
+; CHECK-UNORDERED-NEXT:    [[VEC_PHI:%.*]] = phi <vscale x 4 x float> [ [[TMP10]], [[VECTOR_PH]] ], [ [[TMP17:%.*]], [[VECTOR_BODY]] ]
+; CHECK-UNORDERED-NEXT:    [[VEC_PHI1:%.*]] = phi <vscale x 4 x float> [ [[TMP11]], [[VECTOR_PH]] ], [ [[TMP16:%.*]], [[VECTOR_BODY]] ]
 ; CHECK-UNORDERED-NEXT:    [[OFFSET_IDX:%.*]] = mul i64 [[INDEX]], 2
-; CHECK-UNORDERED-NEXT:    [[TMP11:%.*]] = add i64 [[OFFSET_IDX]], 0
-; CHECK-UNORDERED-NEXT:    [[TMP12:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[TMP11]]
-; CHECK-UNORDERED-NEXT:    [[WIDE_VEC:%.*]] = load <vscale x 8 x float>, ptr [[TMP12]], align 4
+; CHECK-UNORDERED-NEXT:    [[TMP12:%.*]] = add i64 [[OFFSET_IDX]], 0
+; CHECK-UNORDERED-NEXT:    [[TMP13:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[TMP12]]
+; CHECK-UNORDERED-NEXT:    [[WIDE_VEC:%.*]] = load <vscale x 8 x float>, ptr [[TMP13]], align 4
 ; CHECK-UNORDERED-NEXT:    [[STRIDED_VEC:%.*]] = call { <vscale x 4 x float>, <vscale x 4 x float> } @llvm.vector.deinterleave2.nxv8f32(<vscale x 8 x float> [[WIDE_VEC]])
-; CHECK-UNORDERED-NEXT:    [[TMP13:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } [[STRIDED_VEC]], 0
-; CHECK-UNORDERED-NEXT:    [[TMP14:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } [[STRIDED_VEC]], 1
-; CHECK-UNORDERED-NEXT:    [[TMP15]] = fadd <vscale x 4 x float> [[TMP13]], [[VEC_PHI1]]
-; CHECK-UNORDERED-NEXT:    [[TMP16]] = fadd <vscale x 4 x float> [[TMP14]], [[VEC_PHI]]
+; CHECK-UNORDERED-NEXT:    [[TMP14:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } [[STRIDED_VEC]], 0
+; CHECK-UNORDERED-NEXT:    [[TMP15:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } [[STRIDED_VEC]], 1
+; CHECK-UNORDERED-NEXT:    [[TMP16]] = fadd <vscale x 4 x float> [[TMP14]], [[VEC_PHI1]]
+; CHECK-UNORDERED-NEXT:    [[TMP17]] = fadd <vscale x 4 x float> [[TMP15]], [[VEC_PHI]]
 ; CHECK-UNORDERED-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP8]]
-; CHECK-UNORDERED-NEXT:    [[TMP17:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-UNORDERED-NEXT:    br i1 [[TMP17]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK-UNORDERED-NEXT:    [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-UNORDERED-NEXT:    br i1 [[TMP18]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
 ; CHECK-UNORDERED:       middle.block:
-; CHECK-UNORDERED-NEXT:    [[TMP18:%.*]] = call float @llvm.vector.reduce.fadd.nxv4f32(float -0.000000e+00, <vscale x 4 x float> [[TMP16]])
-; CHECK-UNORDERED-NEXT:    [[TMP19:%.*]] = call float @llvm.vector.reduce.fadd.nxv4f32(float -0.000000e+00, <vscale x 4 x float> [[TMP15]])
+; CHECK-UNORDERED-NEXT:    [[TMP19:%.*]] = call float @llvm.vector.reduce.fadd.nxv4f32(float -0.000000e+00, <vscale x 4 x float> [[TMP17]])
+; CHECK-UNORDERED-NEXT:    [[TMP20:%.*]] = call float @llvm.vector.reduce.fadd.nxv4f32(float -0.000000e+00, <vscale x 4 x float> [[TMP16]])
 ; CHECK-UNORDERED-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[TMP2]], [[N_VEC]]
 ; CHECK-UNORDERED-NEXT:    br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]]
 ; CHECK-UNORDERED:       scalar.ph:
-; CHECK-UNORDERED-NEXT:    [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP18]], [[MIDDLE_BLOCK]] ], [ [[A2]], [[ENTRY]] ]
-; CHECK-UNORDERED-NEXT:    [[BC_MERGE_RDX2:%.*]] = phi float [ [[TMP19]], [[MIDDLE_BLOCK]] ], [ [[A1]], [[ENTRY]] ]
-; CHECK-UNORDERED-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-UNORDERED-NEXT:    [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP19]], [[MIDDLE_BLOCK]] ], [ [[A2]], [[ENTRY:%.*]] ]
+; CHECK-UNORDERED-NEXT:    [[BC_MERGE_RDX2:%.*]] = phi float [ [[TMP20]], [[MIDDLE_BLOCK]] ], [ [[A1]], [[ENTRY]] ]
+; CHECK-UNORDERED-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP9]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ]
 ; CHECK-UNORDERED-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK-UNORDERED:       for.body:
 ; CHECK-UNORDERED-NEXT:    [[ADD_PHI1:%.*]] = phi float [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[ADD2:%.*]], [[FOR_BODY]] ]
 ; CHECK-UNORDERED-NEXT:    [[ADD_PHI2:%.*]] = phi float [ [[BC_MERGE_RDX2]], [[SCALAR_PH]] ], [ [[ADD1:%.*]], [[FOR_BODY]] ]
 ; CHECK-UNORDERED-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
 ; CHECK-UNORDERED-NEXT:    [[ARRAYIDXB1:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[IV]]
-; CHECK-UNORDERED-NEXT:    [[TMP20:%.*]] = load float, ptr [[ARRAYIDXB1]], align 4
-; CHECK-UNORDERED-NEXT:    [[ADD1]] = fadd float [[TMP20]], [[ADD_PHI2]]
+; CHECK-UNORDERED-NEXT:    [[TMP21:%.*]] = load float, ptr [[ARRAYIDXB1]], align 4
+; CHECK-UNORDERED-NEXT:    [[ADD1]] = fadd float [[TMP21]], [[ADD_PHI2]]
 ; CHECK-UNORDERED-NEXT:    [[OR:%.*]] = or disjoint i64 [[IV]], 1
 ; CHECK-UNORDERED-NEXT:    [[ARRAYIDXB2:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[OR]]
-; CHECK-UNORDERED-NEXT:    [[TMP21:%.*]] = load float, ptr [[ARRAYIDXB2]], align 4
-; CHECK-UNORDERED-NEXT:    [[ADD2]] = fadd float [[TMP21]], [[ADD_PHI1]]
+; CHECK-UNORDERED-NEXT:    [[TMP22:%.*]] = load float, ptr [[ARRAYIDXB2]], align 4
+; CHECK-UNORDERED-NEXT:    [[ADD2]] = fadd float [[TMP22]], [[ADD_PHI1]]
 ; CHECK-UNORDERED-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 2
 ; CHECK-UNORDERED-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
 ; CHECK-UNORDERED-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
 ; CHECK-UNORDERED:       for.end:
-; CHECK-UNORDERED-NEXT:    [[ADD1_LCSSA:%.*]] = phi float [ [[ADD1]], [[FOR_BODY]] ], [ [[TMP19]], [[MIDDLE_BLOCK]] ]
-; CHECK-UNORDERED-NEXT:    [[ADD2_LCSSA:%.*]] = phi float [ [[ADD2]], [[FOR_BODY]] ], [ [[TMP18]], [[MIDDLE_BLOCK]] ]
+; CHECK-UNORDERED-NEXT:    [[ADD1_LCSSA:%.*]] = phi float [ [[ADD1]], [[FOR_BODY]] ], [ [[TMP20]], [[MIDDLE_BLOCK]] ]
+; CHECK-UNORDERED-NEXT:    [[ADD2_LCSSA:%.*]] = phi float [ [[ADD2]], [[FOR_BODY]] ], [ [[TMP19]], [[MIDDLE_BLOCK]] ]
 ; CHECK-UNORDERED-NEXT:    store float [[ADD1_LCSSA]], ptr [[A]], align 4
 ; CHECK-UNORDERED-NEXT:    store float [[ADD2_LCSSA]], ptr [[ARRAYIDXA]], align 4
 ; CHECK-UNORDERED-NEXT:    ret void
 ;
 ; CHECK-ORDERED-LABEL: define void @fadd_strict_interleave
-; CHECK-ORDERED-SAME: (ptr noalias nocapture readonly [[A:%.*]], ptr noalias nocapture readonly [[B:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-ORDERED-SAME: (ptr noalias readonly captures(none) [[A:%.*]], ptr noalias readonly captures(none) [[B:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
 ; CHECK-ORDERED-NEXT:  entry:
 ; CHECK-ORDERED-NEXT:    [[ARRAYIDXA:%.*]] = getelementptr inbounds float, ptr [[A]], i64 1
 ; CHECK-ORDERED-NEXT:    [[A1:%.*]] = load float, ptr [[A]], align 4
@@ -585,55 +585,55 @@ define void @fadd_strict_interleave(ptr noalias nocapture readonly %a, ptr noali
 ; CHECK-ORDERED-NEXT:    [[N_VEC:%.*]] = sub i64 [[TMP2]], [[N_MOD_VF]]
 ; CHECK-ORDERED-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
 ; CHECK-ORDERED-NEXT:    [[TMP8:%.*]] = mul i64 [[TMP7]], 4
-; CHECK-ORDERED-NEXT:    [[IND_END:%.*]] = mul i64 [[N_VEC]], 2
+; CHECK-ORDERED-NEXT:    [[TMP9:%.*]] = mul i64 [[N_VEC]], 2
 ; CHECK-ORDERED-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK-ORDERED:       vector.body:
 ; CHECK-ORDERED-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-ORDERED-NEXT:    [[VEC_PHI:%.*]] = phi float [ [[A2]], [[VECTOR_PH]] ], [ [[TMP13:%.*]], [[VECTOR_BODY]] ]
-; CHECK-ORDERED-NEXT:    [[VEC_PHI1:%.*]] = phi float [ [[A1]], [[VECTOR_PH]] ], [ [[TMP14:%.*]], [[VECTOR_BODY]] ]
+; CHECK-ORDERED-NEXT:    [[VEC_PHI:%.*]] = phi float [ [[A2]], [[VECTOR_PH]] ], [ [[TMP14:%.*]], [[VECTOR_BODY]] ]
+; CHECK-ORDERED-NEXT:    [[VEC_PHI1:%.*]] = phi float [ [[A1]], [[VECTOR_PH]] ], [ [[TMP15:%.*]], [[VECTOR_BODY]] ]
 ; CHECK-ORDERED-NEXT:    [[OFFSET_IDX:%.*]] = mul i64 [[INDEX]], 2
-; CHECK-ORDERED-NEXT:    [[TMP9:%.*]] = add i64 [[OFFSET_IDX]], 0
-; CHECK-ORDERED-NEXT:    [[TMP10:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[TMP9]]
-; CHECK-ORDERED-NEXT:    [[WIDE_VEC:%.*]] = load <vscale x 8 x float>, ptr [[TMP10]], align 4
+; CHECK-ORDERED-NEXT:    [[TMP10:%.*]] = add i64 [[OFFSET_IDX]], 0
+; CHECK-ORDERED-NEXT:    [[TMP11:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[TMP10]]
+; CHECK-ORDERED-NEXT:    [[WIDE_VEC:%.*]] = load <vscale x 8 x float>, ptr [[TMP11]], align 4
 ; CHECK-ORDERED-NEXT:    [[STRIDED_VEC:%.*]] = call { <vscale x 4 x float>, <vscale x 4 x float> } @llvm.vector.deinterleave2.nxv8f32(<vscale x 8 x float> [[WIDE_VEC]])
-; CHECK-ORDERED-NEXT:    [[TMP11:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } [[STRIDED_VEC]], 0
-; CHECK-ORDERED-NEXT:    [[TMP12:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } [[STRIDED_VEC]], 1
-; CHECK-ORDERED-NEXT:    [[TMP13]] = call float @llvm.vector.reduce.fadd.nxv4f32(float [[VEC_PHI]], <vscale x 4 x float> [[TMP12]])
-; CHECK-ORDERED-NEXT:    [[TMP14]] = call float @llvm.vector.reduce.fadd.nxv4f32(float [[VEC_PHI1]], <vscale x 4 x float> [[TMP11]])
+; CHECK-ORDERED-NEXT:    [[TMP12:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } [[STRIDED_VEC]], 0
+; CHECK-ORDERED-NEXT:    [[TMP13:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } [[STRIDED_VEC]], 1
+; CHECK-ORDERED-NEXT:    [[TMP14]] = call float @llvm.vector.reduce.fadd.nxv4f32(float [[VEC_PHI]], <vscale x 4 x float> [[TMP13]])
+; CHECK-ORDERED-NEXT:    [[TMP15]] = call float @llvm.vector.reduce.fadd.nxv4f32(float [[VEC_PHI1]], <vscale x 4 x float> [[TMP12]])
 ; CHECK-ORDERED-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP8]]
-; CHECK-ORDERED-NEXT:    [[TMP15:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-ORDERED-NEXT:    br i1 [[TMP15]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK-ORDERED-NEXT:    [[TMP16:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-ORDERED-NEXT:    br i1 [[TMP16]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
 ; CHECK-ORDERED:       middle.block:
 ; CHECK-ORDERED-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[TMP2]], [[N_VEC]]
 ; CHECK-ORDERED-NEXT:    br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]]
 ; CHECK-ORDERED:       scalar.ph:
-; CHECK-ORDERED-NEXT:    [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP13]], [[MIDDLE_BLOCK]] ], [ [[A2]], [[ENTRY]] ]
-; CHECK-ORDERED-NEXT:    [[BC_MERGE_RDX2:%.*]] = phi float [ [[TMP14]], [[MIDDLE_BLOCK]] ], [ [[A1]], [[ENTRY]] ]
-; CHECK-ORDERED-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-ORDERED-NEXT:    [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP14]], [[MIDDLE_BLOCK]] ], [ [[A2]], [[ENTRY:%.*]] ]
+; CHECK-ORDERED-NEXT:    [[BC_MERGE_RDX2:%.*]] = phi float [ [[TMP15]], [[MIDDLE_BLOCK]] ], [ [[A1]], [[ENTRY]] ]
+; CHECK-ORDERED-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP9]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ]
 ; CHECK-ORDERED-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK-ORDERED:       for.body:
 ; CHECK-ORDERED-NEXT:    [[ADD_PHI1:%.*]] = phi float [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[ADD2:%.*]], [[FOR_BODY]] ]
 ; CHECK-ORDERED-NEXT:    [[ADD_PHI2:%.*]] = phi float [ [[BC_MERGE_RDX2]], [[SCALAR_PH]] ], [ [[ADD1:%.*]], [[FOR_BODY]] ]
 ; CHECK-ORDERED-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
 ; CHECK-ORDERED-NEXT:    [[ARRAYIDXB1:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[IV]]
-; CHECK-ORDERED-NEXT:    [[TMP16:%.*]] = load float, ptr [[ARRAYIDXB1]], align 4
-; CHECK-ORDERED-NEXT:    [[ADD1]] = fadd float [[TMP16]], [[ADD_PHI2]]
+; CHECK-ORDERED-NEXT:    [[TMP17:%.*]] = load float, ptr [[ARRAYIDXB1]], align 4
+; CHECK-ORDERED-NEXT:    [[ADD1]] = fadd float [[TMP17]], [[ADD_PHI2]]
 ; CHECK-ORDERED-NEXT:    [[OR:%.*]] = or disjoint i64 [[IV]], 1
 ; CHECK-ORDERED-NEXT:    [[ARRAYIDXB2:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[OR]]
-; CHECK-ORDERED-NEXT:    [[TMP17:%.*]] = load float, ptr [[ARRAYIDXB2]], align 4
-; CHECK-ORDERED-NEXT:    [[ADD2]] = fadd float [[TMP17]], [[ADD_PHI1]]
+; CHECK-ORDERED-NEXT:    [[TMP18:%.*]] = load float, ptr [[ARRAYIDXB2]], align 4
+; CHECK-ORDERED-NEXT:    [[ADD2]] = fadd float [[TMP18]], [[ADD_PHI1]]
 ; CHECK-ORDERED-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 2
 ; CHECK-ORDERED-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
 ; CHECK-ORDERED-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
 ; CHECK-ORDERED:       for.end:
-; CHECK-ORDERED-NEXT:    [[ADD1_LCSSA:%.*]] = phi float [ [[ADD1]], [[FOR_BODY]] ], [ [[TMP14]], [[MIDDLE_BLOCK]] ]
-; CHECK-ORDERED-NEXT:    [[ADD2_LCSSA:%.*]] = phi float [ [[ADD2]], [[FOR_BODY]] ], [ [[TMP13]], [[MIDDLE_BLOCK]] ]
+; CHECK-ORDERED-NEXT:    [[ADD1_LCSSA:%.*]] = phi float [ [[ADD1]], [[FOR_BODY]] ], [ [[TMP15]], [[MIDDLE_BLOCK]] ]
+; CHECK-ORDERED-NEXT:    [[ADD2_LCSSA:%.*]] = phi float [ [[ADD2]], [[FOR_BODY]] ], [ [[TMP14]], [[MIDDLE_BLOCK]] ]
 ; CHECK-ORDERED-NEXT:    store float [[ADD1_LCSSA]], ptr [[A]], align 4
 ; CHECK-ORDERED-NEXT:    store float [[ADD2_LCSSA]], ptr [[ARRAYIDXA]], align 4
 ; CHECK-ORDERED-NEXT:    ret void
 ;
 ; CHECK-ORDERED-TF-LABEL: define void @fadd_strict_interleave
-; CHECK-ORDERED-TF-SAME: (ptr noalias nocapture readonly [[A:%.*]], ptr noalias nocapture readonly [[B:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-ORDERED-TF-SAME: (ptr noalias readonly captures(none) [[A:%.*]], ptr noalias readonly captures(none) [[B:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
 ; CHECK-ORDERED-TF-NEXT:  entry:
 ; CHECK-ORDERED-TF-NEXT:    [[ARRAYIDXA:%.*]] = getelementptr inbounds float, ptr [[A]], i64 1
 ; CHECK-ORDERED-TF-NEXT:    [[A1:%.*]] = load float, ptr [[A]], align 4
@@ -651,60 +651,60 @@ define void @fadd_strict_interleave(ptr noalias nocapture readonly %a, ptr noali
 ; CHECK-ORDERED-TF-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
 ; CHECK-ORDERED-TF-NEXT:    [[TMP6:%.*]] = call i64 @llvm.vscale.i64()
 ; CHECK-ORDERED-TF-NEXT:    [[TMP7:%.*]] = mul i64 [[TMP6]], 4
-; CHECK-ORDERED-TF-NEXT:    [[IND_END:%.*]] = mul i64 [[N_VEC]], 2
-; CHECK-ORDERED-TF-NEXT:    [[TMP8:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-ORDERED-TF-NEXT:    [[TMP9:%.*]] = mul i64 [[TMP8]], 4
-; CHECK-ORDERED-TF-NEXT:    [[TMP10:%.*]] = sub i64 [[TMP2]], [[TMP9]]
-; CHECK-ORDERED-TF-NEXT:    [[TMP11:%.*]] = icmp ugt i64 [[TMP2]], [[TMP9]]
-; CHECK-ORDERED-TF-NEXT:    [[TMP12:%.*]] = select i1 [[TMP11]], i64 [[TMP10]], i64 0
+; CHECK-ORDERED-TF-NEXT:    [[TMP8:%.*]] = mul i64 [[N_VEC]], 2
+; CHECK-ORDERED-TF-NEXT:    [[TMP9:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-ORDERED-TF-NEXT:    [[TMP10:%.*]] = mul i64 [[TMP9]], 4
+; CHECK-ORDERED-TF-NEXT:    [[TMP11:%.*]] = sub i64 [[TMP2]], [[TMP10]]
+; CHECK-ORDERED-TF-NEXT:    [[TMP12:%.*]] = icmp ugt i64 [[TMP2]], [[TMP10]]
+; CHECK-ORDERED-TF-NEXT:    [[TMP13:%.*]] = select i1 [[TMP12]], i64 [[TMP11]], i64 0
 ; CHECK-ORDERED-TF-NEXT:    [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 0, i64 [[TMP2]])
 ; CHECK-ORDERED-TF-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK-ORDERED-TF:       vector.body:
 ; CHECK-ORDERED-TF-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; CHECK-ORDERED-TF-NEXT:    [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 4 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], [[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-ORDERED-TF-NEXT:    [[VEC_PHI:%.*]] = phi float [ [[A2]], [[VECTOR_PH]] ], [ [[TMP18:%.*]], [[VECTOR_BODY]] ]
-; CHECK-ORDERED-TF-NEXT:    [[VEC_PHI1:%.*]] = phi float [ [[A1]], [[VECTOR_PH]] ], [ [[TMP20:%.*]], [[VECTOR_BODY]] ]
+; CHECK-ORDERED-TF-NEXT:    [[VEC_PHI:%.*]] = phi float [ [[A2]], [[VECTOR_PH]] ], [ [[TMP19:%.*]], [[VECTOR_BODY]] ]
+; CHECK-ORDERED-TF-NEXT:    [[VEC_PHI1:%.*]] = phi float [ [[A1]], [[VECTOR_PH]] ], [ [[TMP21:%.*]], [[VECTOR_BODY]] ]
 ; CHECK-ORDERED-TF-NEXT:    [[OFFSET_IDX:%.*]] = mul i64 [[INDEX]], 2
-; CHECK-ORDERED-TF-NEXT:    [[TMP13:%.*]] = add i64 [[OFFSET_IDX]], 0
-; CHECK-ORDERED-TF-NEXT:    [[TMP14:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[TMP13]]
+; CHECK-ORDERED-TF-NEXT:    [[TMP14:%.*]] = add i64 [[OFFSET_IDX]], 0
+; CHECK-ORDERED-TF-NEXT:    [[TMP15:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[TMP14]]
 ; CHECK-ORDERED-TF-NEXT:    [[INTERLEAVED_MASK:%.*]] = call <vscale x 8 x i1> @llvm.vector.interleave2.nxv8i1(<vscale x 4 x i1> [[ACTIVE_LANE_MASK]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK]])
-; CHECK-ORDERED-TF-NEXT:    [[WIDE_MASKED_VEC:%.*]] = call <vscale x 8 x float> @llvm.masked.load.nxv8f32.p0(ptr [[TMP14]], i32 4, <vscale x 8 x i1> [[INTERLEAVED_MASK]], <vscale x 8 x float> poison)
+; CHECK-ORDERED-TF-NEXT:    [[WIDE_MASKED_VEC:%.*]] = call <vscale x 8 x float> @llvm.masked.load.nxv8f32.p0(ptr [[TMP15]], i32 4, <vscale x 8 x i1> [[INTERLEAVED_MASK]], <vscale x 8 x float> poison)
 ; CHECK-ORDERED-TF-NEXT:    [[STRIDED_VEC:%.*]] = call { <vscale x 4 x float>, <vscale x 4 x float> } @llvm.vector.deinterleave2.nxv8f32(<vscale x 8 x float> [[WIDE_MASKED_VEC]])
-; CHECK-ORDERED-TF-NEXT:    [[TMP15:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } [[STRIDED_VEC]], 0
-; CHECK-ORDERED-TF-NEXT:    [[TMP16:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } [[STRIDED_VEC]], 1
-; CHECK-ORDERED-TF-NEXT:    [[TMP17:%.*]] = select <vscale x 4 x i1> [[ACTIVE_LANE_MASK]], <vscale x 4 x float> [[TMP16]], <vscale x 4 x float> splat (float -0.000000e+00)
-; CHECK-ORDERED-TF-NEXT:    [[TMP18]] = call float @llvm.vector.reduce.fadd.nxv4f32(float [[VEC_PHI]], <vscale x 4 x float> [[TMP17]])
-; CHECK-ORDERED-TF-NEXT:    [[TMP19:%.*]] = select <vscale x 4 x i1> [[ACTIVE_LANE_MASK]], <vscale x 4 x float> [[TMP15]], <vscale x 4 x float> splat (float -0.000000e+00)
-; CHECK-ORDERED-TF-NEXT:    [[TMP20]] = call float @llvm.vector.reduce.fadd.nxv4f32(float [[VEC_PHI1]], <vscale x 4 x float> [[TMP19]])
+; CHECK-ORDERED-TF-NEXT:    [[TMP16:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } [[STRIDED_VEC]], 0
+; CHECK-ORDERED-TF-NEXT:    [[TMP17:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } [[STRIDED_VEC]], 1
+; CHECK-ORDERED-TF-NEXT:    [[TMP18:%.*]] = select <vscale x 4 x i1> [[ACTIVE_LANE_MASK]], <vscale x 4 x float> [[TMP17]], <vscale x 4 x float> splat (float -0.000000e+00)
+; CHECK-ORDERED-TF-NEXT:    [[TMP19]] = call float @llvm.vector.reduce.fadd.nxv4f32(float [[VEC_PHI]], <vscale x 4 x float> [[TMP18]])
+; CHECK-ORDERED-TF-NEXT:    [[TMP20:%.*]] = select <vscale x 4 x i1> [[ACTIVE_LANE_MASK]], <vscale x 4 x float> [[TMP16]], <vscale x 4 x float> splat (float -0.000000e+00)
+; CHECK-ORDERED-TF-NEXT:    [[TMP21]] = call float @llvm.vector.reduce.fadd.nxv4f32(float [[VEC_PHI1]], <vscale x 4 x float> [[TMP20]])
 ; CHECK-ORDERED-TF-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP7]]
-; CHECK-ORDERED-TF-NEXT:    [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX]], i64 [[TMP12]])
-; CHECK-ORDERED-TF-NEXT:    [[TMP21:%.*]] = xor <vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT]], splat (i1 true)
-; CHECK-ORDERED-TF-NEXT:    [[TMP22:%.*]] = extractelement <vscale x 4 x i1> [[TMP21]], i32 0
-; CHECK-ORDERED-TF-NEXT:    br i1 [[TMP22]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK-ORDERED-TF-NEXT:    [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX]], i64 [[TMP13]])
+; CHECK-ORDERED-TF-NEXT:    [[TMP22:%.*]] = xor <vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT]], splat (i1 true)
+; CHECK-ORDERED-TF-NEXT:    [[TMP23:%.*]] = extractelement <vscale x 4 x i1> [[TMP22]], i32 0
+; CHECK-ORDERED-TF-NEXT:    br i1 [[TMP23]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
 ; CHECK-ORDERED-TF:       middle.block:
 ; CHECK-ORDERED-TF-NEXT:    br i1 true, label [[FOR_END:%.*]], label [[SCALAR_PH]]
 ; CHECK-ORDERED-TF:       scalar.ph:
-; CHECK-ORDERED-TF-NEXT:    [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP18]], [[MIDDLE_BLOCK]] ], [ [[A2]], [[ENTRY]] ]
-; CHECK-ORDERED-TF-NEXT:    [[BC_MERGE_RDX2:%.*]] = phi float [ [[TMP20]], [[MIDDLE_BLOCK]] ], [ [[A1]], [[ENTRY]] ]
-; CHECK-ORDERED-TF-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-ORDERED-TF-NEXT:    [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP19]], [[MIDDLE_BLOCK]] ], [ [[A2]], [[ENTRY:%.*]] ]
+; CHECK-ORDERED-TF-NEXT:    [[BC_MERGE_RDX2:%.*]] = phi float [ [[TMP21]], [[MIDDLE_BLOCK]] ], [ [[A1]], [[ENTRY]] ]
+; CHECK-ORDERED-TF-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP8]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ]
 ; CHECK-ORDERED-TF-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK-ORDERED-TF:       for.body:
 ; CHECK-ORDERED-TF-NEXT:    [[ADD_PHI1:%.*]] = phi float [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[ADD2:%.*]], [[FOR_BODY]] ]
 ; CHECK-ORDERED-TF-NEXT:    [[ADD_PHI2:%.*]] = phi float [ [[BC_MERGE_RDX2]], [[SCALAR_PH]] ], [ [[ADD1:%.*]], [[FOR_BODY]] ]
 ; CHECK-ORDERED-TF-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
 ; CHECK-ORDERED-TF-NEXT:    [[ARRAYIDXB1:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[IV]]
-; CHECK-ORDERED-TF-NEXT:    [[TMP23:%.*]] = load float, ptr [[ARRAYIDXB1]], align 4
-; CHECK-ORDERED-TF-NEXT:    [[ADD1]] = fadd float [[TMP23]], [[ADD_PHI2]]
+; CHECK-ORDERED-TF-NEXT:    [[TMP24:%.*]] = load float, ptr [[ARRAYIDXB1]], align 4
+; CHECK-ORDERED-TF-NEXT:    [[ADD1]] = fadd float [[TMP24]], [[ADD_PHI2]]
 ; CHECK-ORDERED-TF-NEXT:    [[OR:%.*]] = or disjoint i64 [[IV]], 1
 ; CHECK-ORDERED-TF-NEXT:    [[ARRAYIDXB2:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[OR]]
-; CHECK-ORDERED-TF-NEXT:    [[TMP24:%.*]] = load float, ptr [[ARRAYIDXB2]], align 4
-; CHECK-ORDERED-TF-NEXT:    [[ADD2]] = fadd float [[TMP24]], [[ADD_PHI1]]
+; CHECK-ORDERED-TF-NEXT:    [[TMP25:%.*]] = load float, ptr [[ARRAYIDXB2]], align 4
+; CHECK-ORDERED-TF-NEXT:    [[ADD2]] = fadd float [[TMP25]], [[ADD_PHI1]]
 ; CHECK-ORDERED-TF-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 2
 ; CHECK-ORDERED-TF-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
 ; CHECK-ORDERED-TF-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
 ; CHECK-ORDERED-TF:       for.end:
-; CHECK-ORDERED-TF-NEXT:    [[ADD1_LCSSA:%.*]] = phi float [ [[ADD1]], [[FOR_BODY]] ], [ [[TMP20]], [[MIDDLE_BLOCK]] ]
-; CHECK-ORDERED-TF-NEXT:    [[ADD2_LCSSA:%.*]] = phi float [ [[ADD2]], [[FOR_BODY]] ], [ [[TMP18]], [[MIDDLE_BLOCK]] ]
+; CHECK-ORDERED-TF-NEXT:    [[ADD1_LCSSA:%.*]] = phi float [ [[ADD1]], [[FOR_BODY]] ], [ [[TMP21]], [[MIDDLE_BLOCK]] ]
+; CHECK-ORDERED-TF-NEXT:    [[ADD2_LCSSA:%.*]] = phi float [ [[ADD2]], [[FOR_BODY]] ], [ [[TMP19]], [[MIDDLE_BLOCK]] ]
 ; CHECK-ORDERED-TF-NEXT:    store float [[ADD1_LCSSA]], ptr [[A]], align 4
 ; CHECK-ORDERED-TF-NEXT:    store float [[ADD2_LCSSA]], ptr [[ARRAYIDXA]], align 4
 ; CHECK-ORDERED-TF-NEXT:    ret void
@@ -742,7 +742,7 @@ for.end:
 
 define float @fadd_of_sum(ptr noalias nocapture readonly %a, ptr noalias nocapture readonly %b, i64 %n) #0 {
 ; CHECK-NOT-VECTORIZED-LABEL: define float @fadd_of_sum
-; CHECK-NOT-VECTORIZED-SAME: (ptr noalias nocapture readonly [[A:%.*]], ptr noalias nocapture readonly [[B:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-NOT-VECTORIZED-SAME: (ptr noalias readonly captures(none) [[A:%.*]], ptr noalias readonly captures(none) [[B:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
 ; CHECK-NOT-VECTORIZED-NEXT:  entry:
 ; CHECK-NOT-VECTORIZED-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds float, ptr [[A]], i64 1
 ; CHECK-NOT-VECTORIZED-NEXT:    [[TMP0:%.*]] = load float, ptr [[ARRAYIDX]], align 4
@@ -770,7 +770,7 @@ define float @fadd_of_sum(ptr noalias nocapture readonly %a, ptr noalias nocaptu
 ; CHECK-NOT-VECTORIZED-NEXT:    ret float [[RES]]
 ;
 ; CHECK-UNORDERED-LABEL: define float @fadd_of_sum
-; CHECK-UNORDERED-SAME: (ptr noalias nocapture readonly [[A:%.*]], ptr noalias nocapture readonly [[B:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-UNORDERED-SAME: (ptr noalias readonly captures(none) [[A:%.*]], ptr noalias readonly captures(none) [[B:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
 ; CHECK-UNORDERED-NEXT:  entry:
 ; CHECK-UNORDERED-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds float, ptr [[A]], i64 1
 ; CHECK-UNORDERED-NEXT:    [[TMP0:%.*]] = load float, ptr [[ARRAYIDX]], align 4
@@ -832,7 +832,7 @@ define float @fadd_of_sum(ptr noalias nocapture readonly %a, ptr noalias nocaptu
 ; CHECK-UNORDERED-NEXT:    ret float [[RES]]
 ;
 ; CHECK-ORDERED-LABEL: define float @fadd_of_sum
-; CHECK-ORDERED-SAME: (ptr noalias nocapture readonly [[A:%.*]], ptr noalias nocapture readonly [[B:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-ORDERED-SAME: (ptr noalias readonly captures(none) [[A:%.*]], ptr noalias readonly captures(none) [[B:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
 ; CHECK-ORDERED-NEXT:  entry:
 ; CHECK-ORDERED-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds float, ptr [[A]], i64 1
 ; CHECK-ORDERED-NEXT:    [[TMP0:%.*]] = load float, ptr [[ARRAYIDX]], align 4
@@ -893,7 +893,7 @@ define float @fadd_of_sum(ptr noalias nocapture readonly %a, ptr noalias nocaptu
 ; CHECK-ORDERED-NEXT:    ret float [[RES]]
 ;
 ; CHECK-ORDERED-TF-LABEL: define float @fadd_of_sum
-; CHECK-ORDERED-TF-SAME: (ptr noalias nocapture readonly [[A:%.*]], ptr noalias nocapture readonly [[B:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-ORDERED-TF-SAME: (ptr noalias readonly captures(none) [[A:%.*]], ptr noalias readonly captures(none) [[B:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
 ; CHECK-ORDERED-TF-NEXT:  entry:
 ; CHECK-ORDERED-TF-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds float, ptr [[A]], i64 1
 ; CHECK-ORDERED-TF-NEXT:    [[TMP0:%.*]] = load float, ptr [[ARRAYIDX]], align 4
@@ -991,7 +991,7 @@ for.end:                                 ; preds = %for.body, %entry
 
 define float @fadd_conditional(ptr noalias nocapture readonly %a, ptr noalias nocapture readonly %b, i64 %n) #0 {
 ; CHECK-NOT-VECTORIZED-LABEL: define float @fadd_conditional
-; CHECK-NOT-VECTORIZED-SAME: (ptr noalias nocapture readonly [[A:%.*]], ptr noalias nocapture readonly [[B:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-NOT-VECTORIZED-SAME: (ptr noalias readonly captures(none) [[A:%.*]], ptr noalias readonly captures(none) [[B:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
 ; CHECK-NOT-VECTORIZED-NEXT:  entry:
 ; CHECK-NOT-VECTORIZED-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK-NOT-VECTORIZED:       for.body:
@@ -1016,7 +1016,7 @@ define float @fadd_conditional(ptr noalias nocapture readonly %a, ptr noalias no
 ; CHECK-NOT-VECTORIZED-NEXT:    ret float [[RDX]]
 ;
 ; CHECK-UNORDERED-LABEL: define float @fadd_conditional
-; CHECK-UNORDERED-SAME: (ptr noalias nocapture readonly [[A:%.*]], ptr noalias nocapture readonly [[B:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-UNORDERED-SAME: (ptr noalias readonly captures(none) [[A:%.*]], ptr noalias readonly captures(none) [[B:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
 ; CHECK-UNORDERED-NEXT:  entry:
 ; CHECK-UNORDERED-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
 ; CHECK-UNORDERED-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 4
@@ -1077,7 +1077,7 @@ define float @fadd_conditional(ptr noalias nocapture readonly %a, ptr noalias no
 ; CHECK-UNORDERED-NEXT:    ret float [[RDX]]
 ;
 ; CHECK-ORDERED-LABEL: define float @fadd_conditional
-; CHECK-ORDERED-SAME: (ptr noalias nocapture readonly [[A:%.*]], ptr noalias nocapture readonly [[B:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-ORDERED-SAME: (ptr noalias readonly captures(none) [[A:%.*]], ptr noalias readonly captures(none) [[B:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
 ; CHECK-ORDERED-NEXT:  entry:
 ; CHECK-ORDERED-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
 ; CHECK-ORDERED-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 4
@@ -1137,7 +1137,7 @@ define float @fadd_conditional(ptr noalias nocapture readonly %a, ptr noalias no
 ; CHECK-ORDERED-NEXT:    ret float [[RDX]]
 ;
 ; CHECK-ORDERED-TF-LABEL: define float @fadd_conditional
-; CHECK-ORDERED-TF-SAME: (ptr noalias nocapture readonly [[A:%.*]], ptr noalias nocapture readonly [[B:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-ORDERED-TF-SAME: (ptr noalias readonly captures(none) [[A:%.*]], ptr noalias readonly captures(none) [[B:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
 ; CHECK-ORDERED-TF-NEXT:  entry:
 ; CHECK-ORDERED-TF-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK-ORDERED-TF:       vector.ph:
@@ -1239,7 +1239,7 @@ for.end:
 ; Negative test - loop contains multiple fadds which we cannot safely reorder
 define float @fadd_multiple(ptr noalias nocapture %a, ptr noalias nocapture %b, i64 %n) #0 {
 ; CHECK-NOT-VECTORIZED-LABEL: define float @fadd_multiple
-; CHECK-NOT-VECTORIZED-SAME: (ptr noalias nocapture [[A:%.*]], ptr noalias nocapture [[B:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-NOT-VECTORIZED-SAME: (ptr noalias captures(none) [[A:%.*]], ptr noalias captures(none) [[B:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
 ; CHECK-NOT-VECTORIZED-NEXT:  entry:
 ; CHECK-NOT-VECTORIZED-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK-NOT-VECTORIZED:       for.body:
@@ -1259,7 +1259,7 @@ define float @fadd_multiple(ptr noalias nocapture %a, ptr noalias nocapture %b,
 ; CHECK-NOT-VECTORIZED-NEXT:    ret float [[RDX]]
 ;
 ; CHECK-UNORDERED-LABEL: define float @fadd_multiple
-; CHECK-UNORDERED-SAME: (ptr noalias nocapture [[A:%.*]], ptr noalias nocapture [[B:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-UNORDERED-SAME: (ptr noalias captures(none) [[A:%.*]], ptr noalias captures(none) [[B:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
 ; CHECK-UNORDERED-NEXT:  entry:
 ; CHECK-UNORDERED-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
 ; CHECK-UNORDERED-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 8
@@ -1313,7 +1313,7 @@ define float @fadd_multiple(ptr noalias nocapture %a, ptr noalias nocapture %b,
 ; CHECK-UNORDERED-NEXT:    ret float [[RDX]]
 ;
 ; CHECK-ORDERED-LABEL: define float @fadd_multiple
-; CHECK-ORDERED-SAME: (ptr noalias nocapture [[A:%.*]], ptr noalias nocapture [[B:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-ORDERED-SAME: (ptr noalias captures(none) [[A:%.*]], ptr noalias captures(none) [[B:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
 ; CHECK-ORDERED-NEXT:  entry:
 ; CHECK-ORDERED-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK-ORDERED:       for.body:
@@ -1333,7 +1333,7 @@ define float @fadd_multiple(ptr noalias nocapture %a, ptr noalias nocapture %b,
 ; CHECK-ORDERED-NEXT:    ret float [[RDX]]
 ;
 ; CHECK-ORDERED-TF-LABEL: define float @fadd_multiple
-; CHECK-ORDERED-TF-SAME: (ptr noalias nocapture [[A:%.*]], ptr noalias nocapture [[B:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-ORDERED-TF-SAME: (ptr noalias captures(none) [[A:%.*]], ptr noalias captures(none) [[B:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
 ; CHECK-ORDERED-TF-NEXT:  entry:
 ; CHECK-ORDERED-TF-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK-ORDERED-TF:       for.body:
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/scalable-vectorization-cost-tuning.ll b/llvm/test/Transforms/LoopVectorize/AArch64/scalable-vectorization-cost-tuning.ll
index 1565d1ce982e3..c4aee69db70b3 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/scalable-vectorization-cost-tuning.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/scalable-vectorization-cost-tuning.ll
@@ -1,22 +1,22 @@
 ; REQUIRES: asserts
 ; RUN: opt -mtriple=aarch64 -mattr=+sve \
-; RUN:     -force-target-instruction-cost=1 -passes=loop-vectorize -S -debug-only=loop-vectorize < %s 2>&1 \
+; RUN:     -force-target-instruction-cost=1 -passes=loop-vectorize -S -debug-only=loop-vectorize --disable-output < %s 2>&1 \
 ; RUN:     | FileCheck %s --check-prefixes=VSCALEFORTUNING1
 
 ; RUN: opt -mtriple=aarch64 -mattr=+sve -mcpu=generic \
-; RUN:     -force-target-instruction-cost=1 -passes=loop-vectorize -S -debug-only=loop-vectorize < %s 2>&1 \
+; RUN:     -force-target-instruction-cost=1 -passes=loop-vectorize -S -debug-only=loop-vectorize --disable-output < %s 2>&1 \
 ; RUN:     | FileCheck %s --check-prefixes=VSCALEFORTUNING1
 
 ; RUN: opt -mtriple=aarch64 -mcpu=neoverse-v1 \
-; RUN:     -force-target-instruction-cost=1 -passes=loop-vectorize -S -debug-only=loop-vectorize < %s 2>&1 \
+; RUN:     -force-target-instruction-cost=1 -passes=loop-vectorize -S -debug-only=loop-vectorize --disable-output < %s 2>&1 \
 ; RUN:     | FileCheck %s --check-prefixes=VSCALEFORTUNING2
 
 ; RUN: opt -mtriple=aarch64 -mcpu=neoverse-n2 \
-; RUN:     -force-target-instruction-cost=1 -passes=loop-vectorize -S -debug-only=loop-vectorize < %s 2>&1 \
+; RUN:     -force-target-instruction-cost=1 -passes=loop-vectorize -S -debug-only=loop-vectorize --disable-output < %s 2>&1 \
 ; RUN:     | FileCheck %s --check-prefixes=VSCALEFORTUNING1
 
 ; RUN: opt -mtriple=aarch64 -mcpu=neoverse-v2 \
-; RUN:     -force-target-instruction-cost=1 -passes=loop-vectorize -S -debug-only=loop-vectorize < %s 2>&1 \
+; RUN:     -force-target-instruction-cost=1 -passes=loop-vectorize -S -debug-only=loop-vectorize --disable-output < %s 2>&1 \
 ; RUN:     | FileCheck %s --check-prefixes=NEOVERSEV2
 
 ; VSCALEFORTUNING1: Cost for VF vscale x 2: 11 (Estimated cost per lane: 5.
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/scalable-vectorization.ll b/llvm/test/Transforms/LoopVectorize/AArch64/scalable-vectorization.ll
index e83eb729b521c..dcb2b9b08d1e9 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/scalable-vectorization.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/scalable-vectorization.ll
@@ -1,7 +1,7 @@
 ; REQUIRES: asserts
-; RUN: opt -mtriple=aarch64-none-linux-gnu -mattr=+sve -force-target-instruction-cost=1 -passes=loop-vectorize -S -debug-only=loop-vectorize -scalable-vectorization=off < %s 2>&1 | FileCheck %s --check-prefixes=CHECK,CHECK_SCALABLE_DISABLED
-; RUN: opt -mtriple=aarch64-none-linux-gnu -mattr=+sve -force-target-instruction-cost=1 -passes=loop-vectorize -S -debug-only=loop-vectorize -scalable-vectorization=on < %s 2>&1 | FileCheck %s --check-prefixes=CHECK,CHECK_SCALABLE_ON
-; RUN: opt -mtriple=aarch64-none-linux-gnu -mattr=+sve -force-target-instruction-cost=1 -passes=loop-vectorize -S -debug-only=loop-vectorize -vectorizer-maximize-bandwidth -scalable-vectorization=on < %s 2>&1 | FileCheck %s --check-prefixes=CHECK,CHECK_SCALABLE_ON_MAXBW
+; RUN: opt -mtriple=aarch64-none-linux-gnu -mattr=+sve -force-target-instruction-cost=1 -passes=loop-vectorize -S -debug-only=loop-vectorize --disable-output -scalable-vectorization=off < %s 2>&1 | FileCheck %s --check-prefixes=CHECK,CHECK_SCALABLE_DISABLED
+; RUN: opt -mtriple=aarch64-none-linux-gnu -mattr=+sve -force-target-instruction-cost=1 -passes=loop-vectorize -S -debug-only=loop-vectorize --disable-output -scalable-vectorization=on < %s 2>&1 | FileCheck %s --check-prefixes=CHECK,CHECK_SCALABLE_ON
+; RUN: opt -mtriple=aarch64-none-linux-gnu -mattr=+sve -force-target-instruction-cost=1 -passes=loop-vectorize -S -debug-only=loop-vectorize --disable-output -vectorizer-maximize-bandwidth -scalable-vectorization=on < %s 2>&1 | FileCheck %s --check-prefixes=CHECK,CHECK_SCALABLE_ON_MAXBW
 
 ; Test that the MaxVF for the following loop, that has no dependence distances,
 ; is calculated as vscale x 4 (max legal SVE vector size) or vscale x 16
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/simple_early_exit.ll b/llvm/test/Transforms/LoopVectorize/AArch64/simple_early_exit.ll
index cb33dfebddc23..3982ed6dd26ab 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/simple_early_exit.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/simple_early_exit.ll
@@ -13,22 +13,71 @@ define i64 @same_exit_block_pre_inc_use1() #1 {
 ; CHECK-NEXT:    [[P2:%.*]] = alloca [1024 x i8], align 4
 ; CHECK-NEXT:    call void @init_mem(ptr [[P1]], i64 1024)
 ; CHECK-NEXT:    call void @init_mem(ptr [[P2]], i64 1024)
+; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 16
+; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 64, [[TMP1]]
+; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK:       vector.ph:
+; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 16
+; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 64, [[TMP3]]
+; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 64, [[N_MOD_VF]]
+; CHECK-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 16
+; CHECK-NEXT:    [[TMP6:%.*]] = add i64 3, [[N_VEC]]
+; CHECK-NEXT:    [[TMP7:%.*]] = call <vscale x 16 x i64> @llvm.stepvector.nxv16i64()
+; CHECK-NEXT:    [[TMP8:%.*]] = mul <vscale x 16 x i64> [[TMP7]], splat (i64 1)
+; CHECK-NEXT:    [[INDUCTION:%.*]] = add <vscale x 16 x i64> splat (i64 3), [[TMP8]]
+; CHECK-NEXT:    [[TMP9:%.*]] = mul i64 1, [[TMP5]]
+; CHECK-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 16 x i64> poison, i64 [[TMP9]], i64 0
+; CHECK-NEXT:    [[DOTSPLAT:%.*]] = shufflevector <vscale x 16 x i64> [[DOTSPLATINSERT]], <vscale x 16 x i64> poison, <vscale x 16 x i32> zeroinitializer
+; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK:       vector.body:
+; CHECK-NEXT:    [[INDEX1:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT3:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <vscale x 16 x i64> [ [[INDUCTION]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[OFFSET_IDX:%.*]] = add i64 3, [[INDEX1]]
+; CHECK-NEXT:    [[TMP10:%.*]] = add i64 [[OFFSET_IDX]], 0
+; CHECK-NEXT:    [[TMP11:%.*]] = getelementptr inbounds i8, ptr [[P1]], i64 [[TMP10]]
+; CHECK-NEXT:    [[TMP12:%.*]] = getelementptr inbounds i8, ptr [[TMP11]], i32 0
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 16 x i8>, ptr [[TMP12]], align 1
+; CHECK-NEXT:    [[TMP13:%.*]] = getelementptr inbounds i8, ptr [[P2]], i64 [[TMP10]]
+; CHECK-NEXT:    [[TMP14:%.*]] = getelementptr inbounds i8, ptr [[TMP13]], i32 0
+; CHECK-NEXT:    [[WIDE_LOAD2:%.*]] = load <vscale x 16 x i8>, ptr [[TMP14]], align 1
+; CHECK-NEXT:    [[TMP15:%.*]] = icmp eq <vscale x 16 x i8> [[WIDE_LOAD]], [[WIDE_LOAD2]]
+; CHECK-NEXT:    [[INDEX_NEXT3]] = add nuw i64 [[INDEX1]], [[TMP5]]
+; CHECK-NEXT:    [[TMP16:%.*]] = xor <vscale x 16 x i1> [[TMP15]], splat (i1 true)
+; CHECK-NEXT:    [[TMP17:%.*]] = call i1 @llvm.vector.reduce.or.nxv16i1(<vscale x 16 x i1> [[TMP16]])
+; CHECK-NEXT:    [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT3]], [[N_VEC]]
+; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <vscale x 16 x i64> [[VEC_IND]], [[DOTSPLAT]]
+; CHECK-NEXT:    [[TMP19:%.*]] = or i1 [[TMP17]], [[TMP18]]
+; CHECK-NEXT:    br i1 [[TMP19]], label [[MIDDLE_SPLIT:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK:       middle.split:
+; CHECK-NEXT:    br i1 [[TMP17]], label [[VECTOR_EARLY_EXIT:%.*]], label [[MIDDLE_BLOCK:%.*]]
+; CHECK:       vector.early.exit:
+; CHECK-NEXT:    [[FIRST_ACTIVE_LANE:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.nxv16i1(<vscale x 16 x i1> [[TMP16]], i1 true)
+; CHECK-NEXT:    [[EARLY_EXIT_VALUE:%.*]] = extractelement <vscale x 16 x i64> [[VEC_IND]], i64 [[FIRST_ACTIVE_LANE]]
+; CHECK-NEXT:    br label [[LOOP_END:%.*]]
+; CHECK:       middle.block:
+; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 64, [[N_VEC]]
+; CHECK-NEXT:    br i1 [[CMP_N]], label [[LOOP_END]], label [[SCALAR_PH]]
+; CHECK:       scalar.ph:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP6]], [[MIDDLE_BLOCK]] ], [ 3, [[ENTRY:%.*]] ]
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
-; CHECK-NEXT:    [[INDEX2:%.*]] = phi i64 [ [[INDEX_NEXT1:%.*]], [[LOOP_INC:%.*]] ], [ 3, [[ENTRY:%.*]] ]
-; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i8, ptr [[P1]], i64 [[INDEX2]]
-; CHECK-NEXT:    [[TMP38:%.*]] = load i8, ptr [[ARRAYIDX]], align 1
-; CHECK-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds i8, ptr [[P2]], i64 [[INDEX2]]
-; CHECK-NEXT:    [[TMP39:%.*]] = load i8, ptr [[ARRAYIDX1]], align 1
-; CHECK-NEXT:    [[CMP3:%.*]] = icmp eq i8 [[TMP38]], [[TMP39]]
-; CHECK-NEXT:    br i1 [[CMP3]], label [[LOOP_INC]], label [[LOOP_END:%.*]]
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ [[INDEX_NEXT:%.*]], [[LOOP_INC:%.*]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i8, ptr [[P1]], i64 [[INDEX]]
+; CHECK-NEXT:    [[LD1:%.*]] = load i8, ptr [[ARRAYIDX]], align 1
+; CHECK-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds i8, ptr [[P2]], i64 [[INDEX]]
+; CHECK-NEXT:    [[LD2:%.*]] = load i8, ptr [[ARRAYIDX1]], align 1
+; CHECK-NEXT:    [[CMP3:%.*]] = icmp eq i8 [[LD1]], [[LD2]]
+; CHECK-NEXT:    br i1 [[CMP3]], label [[LOOP_INC]], label [[LOOP_END]]
 ; CHECK:       loop.inc:
-; CHECK-NEXT:    [[INDEX_NEXT1]] = add i64 [[INDEX2]], 1
-; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp ne i64 [[INDEX_NEXT1]], 67
-; CHECK-NEXT:    br i1 [[EXITCOND]], label [[LOOP]], label [[LOOP_END]]
+; CHECK-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 1
+; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp ne i64 [[INDEX_NEXT]], 67
+; CHECK-NEXT:    br i1 [[EXITCOND]], label [[LOOP]], label [[LOOP_END]], !llvm.loop [[LOOP3:![0-9]+]]
 ; CHECK:       loop.end:
-; CHECK-NEXT:    [[START_0_LCSSA:%.*]] = phi i64 [ [[INDEX2]], [[LOOP]] ], [ 67, [[LOOP_INC]] ]
-; CHECK-NEXT:    ret i64 [[START_0_LCSSA]]
+; CHECK-NEXT:    [[RETVAL:%.*]] = phi i64 [ [[INDEX]], [[LOOP]] ], [ 67, [[LOOP_INC]] ], [ 67, [[MIDDLE_BLOCK]] ], [ [[EARLY_EXIT_VALUE]], [[VECTOR_EARLY_EXIT]] ]
+; CHECK-NEXT:    ret i64 [[RETVAL]]
 ;
 entry:
   %p1 = alloca [1024 x i8]
@@ -66,19 +115,48 @@ define i64 @same_exit_block_pre_inc_use4() {
 ; CHECK-NEXT:    [[P2:%.*]] = alloca [1024 x i64], align 8
 ; CHECK-NEXT:    call void @init_mem(ptr [[P1]], i64 1024)
 ; CHECK-NEXT:    call void @init_mem(ptr [[P2]], i64 1024)
-; CHECK-NEXT:    br label [[LOOP1:%.*]]
+; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK:       vector.ph:
+; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK:       vector.body:
+; CHECK-NEXT:    [[INDEX1:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT2:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <2 x i64> [ <i64 3, i64 4>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[OFFSET_IDX:%.*]] = add i64 3, [[INDEX1]]
+; CHECK-NEXT:    [[TMP0:%.*]] = add i64 [[OFFSET_IDX]], 0
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i64, ptr [[P1]], i64 [[TMP0]]
+; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i64, ptr [[TMP1]], i32 0
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <2 x i64>, ptr [[TMP2]], align 1
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp ult <2 x i64> [[VEC_IND]], [[WIDE_LOAD]]
+; CHECK-NEXT:    [[INDEX_NEXT2]] = add nuw i64 [[INDEX1]], 2
+; CHECK-NEXT:    [[TMP4:%.*]] = xor <2 x i1> [[TMP3]], splat (i1 true)
+; CHECK-NEXT:    [[TMP5:%.*]] = call i1 @llvm.vector.reduce.or.v2i1(<2 x i1> [[TMP4]])
+; CHECK-NEXT:    [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT2]], 64
+; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <2 x i64> [[VEC_IND]], splat (i64 2)
+; CHECK-NEXT:    [[TMP7:%.*]] = or i1 [[TMP5]], [[TMP6]]
+; CHECK-NEXT:    br i1 [[TMP7]], label [[MIDDLE_SPLIT:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK:       middle.split:
+; CHECK-NEXT:    br i1 [[TMP5]], label [[VECTOR_EARLY_EXIT:%.*]], label [[MIDDLE_BLOCK:%.*]]
+; CHECK:       vector.early.exit:
+; CHECK-NEXT:    [[FIRST_ACTIVE_LANE:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v2i1(<2 x i1> [[TMP4]], i1 true)
+; CHECK-NEXT:    [[EARLY_EXIT_VALUE:%.*]] = extractelement <2 x i64> [[VEC_IND]], i64 [[FIRST_ACTIVE_LANE]]
+; CHECK-NEXT:    br label [[LOOP_END:%.*]]
+; CHECK:       middle.block:
+; CHECK-NEXT:    br i1 true, label [[LOOP_END]], label [[SCALAR_PH]]
+; CHECK:       scalar.ph:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 67, [[MIDDLE_BLOCK]] ], [ 3, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
-; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ [[INDEX_NEXT:%.*]], [[LOOP_INC:%.*]] ], [ 3, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ [[INDEX_NEXT:%.*]], [[LOOP_INC:%.*]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
 ; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i64, ptr [[P1]], i64 [[INDEX]]
 ; CHECK-NEXT:    [[LD1:%.*]] = load i64, ptr [[ARRAYIDX]], align 1
 ; CHECK-NEXT:    [[CMP3:%.*]] = icmp ult i64 [[INDEX]], [[LD1]]
-; CHECK-NEXT:    br i1 [[CMP3]], label [[LOOP_INC]], label [[LOOP_END:%.*]]
+; CHECK-NEXT:    br i1 [[CMP3]], label [[LOOP_INC]], label [[LOOP_END]]
 ; CHECK:       loop.inc:
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 1
 ; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp ne i64 [[INDEX_NEXT]], 67
-; CHECK-NEXT:    br i1 [[EXITCOND]], label [[LOOP1]], label [[LOOP_END]]
+; CHECK-NEXT:    br i1 [[EXITCOND]], label [[LOOP]], label [[LOOP_END]], !llvm.loop [[LOOP5:![0-9]+]]
 ; CHECK:       loop.end:
-; CHECK-NEXT:    [[RETVAL:%.*]] = phi i64 [ [[INDEX]], [[LOOP1]] ], [ 67, [[LOOP_INC]] ]
+; CHECK-NEXT:    [[RETVAL:%.*]] = phi i64 [ [[INDEX]], [[LOOP]] ], [ 67, [[LOOP_INC]] ], [ 67, [[MIDDLE_BLOCK]] ], [ [[EARLY_EXIT_VALUE]], [[VECTOR_EARLY_EXIT]] ]
 ; CHECK-NEXT:    ret i64 [[RETVAL]]
 ;
 entry:
@@ -114,20 +192,50 @@ define i64 @loop_contains_safe_call() #1 {
 ; CHECK-NEXT:    [[P2:%.*]] = alloca [1024 x i8], align 4
 ; CHECK-NEXT:    call void @init_mem(ptr [[P1]], i64 1024)
 ; CHECK-NEXT:    call void @init_mem(ptr [[P2]], i64 1024)
-; CHECK-NEXT:    br label [[LOOP1:%.*]]
+; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK:       vector.ph:
+; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK:       vector.body:
+; CHECK-NEXT:    [[INDEX1:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT2:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 3, i64 4, i64 5, i64 6>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[OFFSET_IDX:%.*]] = add i64 3, [[INDEX1]]
+; CHECK-NEXT:    [[TMP0:%.*]] = add i64 [[OFFSET_IDX]], 0
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds float, ptr [[P1]], i64 [[TMP0]]
+; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds float, ptr [[TMP1]], i32 0
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x float>, ptr [[TMP2]], align 1
+; CHECK-NEXT:    [[TMP3:%.*]] = call fast <4 x float> @llvm.sqrt.v4f32(<4 x float> [[WIDE_LOAD]])
+; CHECK-NEXT:    [[TMP4:%.*]] = fcmp fast ult <4 x float> [[TMP3]], splat (float 3.000000e+00)
+; CHECK-NEXT:    [[INDEX_NEXT2]] = add nuw i64 [[INDEX1]], 4
+; CHECK-NEXT:    [[TMP5:%.*]] = xor <4 x i1> [[TMP4]], splat (i1 true)
+; CHECK-NEXT:    [[TMP6:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP5]])
+; CHECK-NEXT:    [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT2]], 64
+; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <4 x i64> [[VEC_IND]], splat (i64 4)
+; CHECK-NEXT:    [[TMP8:%.*]] = or i1 [[TMP6]], [[TMP7]]
+; CHECK-NEXT:    br i1 [[TMP8]], label [[MIDDLE_SPLIT:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK:       middle.split:
+; CHECK-NEXT:    br i1 [[TMP6]], label [[VECTOR_EARLY_EXIT:%.*]], label [[MIDDLE_BLOCK:%.*]]
+; CHECK:       vector.early.exit:
+; CHECK-NEXT:    [[FIRST_ACTIVE_LANE:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP5]], i1 true)
+; CHECK-NEXT:    [[EARLY_EXIT_VALUE:%.*]] = extractelement <4 x i64> [[VEC_IND]], i64 [[FIRST_ACTIVE_LANE]]
+; CHECK-NEXT:    br label [[LOOP_END:%.*]]
+; CHECK:       middle.block:
+; CHECK-NEXT:    br i1 true, label [[LOOP_END]], label [[SCALAR_PH]]
+; CHECK:       scalar.ph:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 67, [[MIDDLE_BLOCK]] ], [ 3, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
-; CHECK-NEXT:    [[INDEX2:%.*]] = phi i64 [ [[INDEX_NEXT1:%.*]], [[LOOP_INC1:%.*]] ], [ 3, [[ENTRY:%.*]] ]
-; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds float, ptr [[P1]], i64 [[INDEX2]]
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ [[INDEX_NEXT:%.*]], [[LOOP_INC:%.*]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds float, ptr [[P1]], i64 [[INDEX]]
 ; CHECK-NEXT:    [[LD1:%.*]] = load float, ptr [[ARRAYIDX]], align 1
 ; CHECK-NEXT:    [[SQRT:%.*]] = tail call fast float @llvm.sqrt.f32(float [[LD1]])
 ; CHECK-NEXT:    [[CMP:%.*]] = fcmp fast ult float [[SQRT]], 3.000000e+00
-; CHECK-NEXT:    br i1 [[CMP]], label [[LOOP_INC1]], label [[LOOP_END:%.*]]
+; CHECK-NEXT:    br i1 [[CMP]], label [[LOOP_INC]], label [[LOOP_END]]
 ; CHECK:       loop.inc:
-; CHECK-NEXT:    [[INDEX_NEXT1]] = add i64 [[INDEX2]], 1
-; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp ne i64 [[INDEX_NEXT1]], 67
-; CHECK-NEXT:    br i1 [[EXITCOND]], label [[LOOP1]], label [[LOOP_END]]
+; CHECK-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 1
+; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp ne i64 [[INDEX_NEXT]], 67
+; CHECK-NEXT:    br i1 [[EXITCOND]], label [[LOOP]], label [[LOOP_END]], !llvm.loop [[LOOP7:![0-9]+]]
 ; CHECK:       loop.end:
-; CHECK-NEXT:    [[RETVAL:%.*]] = phi i64 [ [[INDEX2]], [[LOOP1]] ], [ 67, [[LOOP_INC1]] ]
+; CHECK-NEXT:    [[RETVAL:%.*]] = phi i64 [ [[INDEX]], [[LOOP]] ], [ 67, [[LOOP_INC]] ], [ 67, [[MIDDLE_BLOCK]] ], [ [[EARLY_EXIT_VALUE]], [[VECTOR_EARLY_EXIT]] ]
 ; CHECK-NEXT:    ret i64 [[RETVAL]]
 ;
 entry:
@@ -164,20 +272,66 @@ define i64 @loop_contains_safe_div() #1 {
 ; CHECK-NEXT:    [[P2:%.*]] = alloca [1024 x i8], align 4
 ; CHECK-NEXT:    call void @init_mem(ptr [[P1]], i64 1024)
 ; CHECK-NEXT:    call void @init_mem(ptr [[P2]], i64 1024)
+; CHECK-NEXT:    [[TMP11:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    [[TMP12:%.*]] = mul i64 [[TMP11]], 4
+; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK:       vector.ph:
+; CHECK-NEXT:    [[TMP10:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP10]], 4
+; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 64, [[TMP3]]
+; CHECK-NEXT:    [[INDEX1:%.*]] = sub i64 64, [[N_MOD_VF]]
+; CHECK-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 4
+; CHECK-NEXT:    [[OFFSET_IDX:%.*]] = add i64 3, [[INDEX1]]
+; CHECK-NEXT:    [[TMP16:%.*]] = call <vscale x 4 x i64> @llvm.stepvector.nxv4i64()
+; CHECK-NEXT:    [[TMP17:%.*]] = mul <vscale x 4 x i64> [[TMP16]], splat (i64 1)
+; CHECK-NEXT:    [[INDUCTION:%.*]] = add <vscale x 4 x i64> splat (i64 3), [[TMP17]]
+; CHECK-NEXT:    [[TMP9:%.*]] = mul i64 1, [[TMP5]]
+; CHECK-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[TMP9]], i64 0
+; CHECK-NEXT:    [[DOTSPLAT:%.*]] = shufflevector <vscale x 4 x i64> [[DOTSPLATINSERT]], <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer
+; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK:       vector.body:
+; CHECK-NEXT:    [[INDEX2:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT2:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <vscale x 4 x i64> [ [[INDUCTION]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[OFFSET_IDX1:%.*]] = add i64 3, [[INDEX2]]
+; CHECK-NEXT:    [[TMP0:%.*]] = add i64 [[OFFSET_IDX1]], 0
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[P1]], i64 [[TMP0]]
+; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i32 0
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 4 x i32>, ptr [[TMP2]], align 1
+; CHECK-NEXT:    [[TMP13:%.*]] = udiv <vscale x 4 x i32> [[WIDE_LOAD]], splat (i32 20000)
+; CHECK-NEXT:    [[TMP14:%.*]] = icmp eq <vscale x 4 x i32> [[TMP13]], splat (i32 1)
+; CHECK-NEXT:    [[INDEX_NEXT2]] = add nuw i64 [[INDEX2]], [[TMP5]]
+; CHECK-NEXT:    [[TMP15:%.*]] = xor <vscale x 4 x i1> [[TMP14]], splat (i1 true)
+; CHECK-NEXT:    [[TMP6:%.*]] = call i1 @llvm.vector.reduce.or.nxv4i1(<vscale x 4 x i1> [[TMP15]])
+; CHECK-NEXT:    [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT2]], [[INDEX1]]
+; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <vscale x 4 x i64> [[VEC_IND]], [[DOTSPLAT]]
+; CHECK-NEXT:    [[TMP8:%.*]] = or i1 [[TMP6]], [[TMP7]]
+; CHECK-NEXT:    br i1 [[TMP8]], label [[MIDDLE_SPLIT:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+; CHECK:       middle.split:
+; CHECK-NEXT:    br i1 [[TMP6]], label [[VECTOR_EARLY_EXIT:%.*]], label [[MIDDLE_BLOCK:%.*]]
+; CHECK:       vector.early.exit:
+; CHECK-NEXT:    [[FIRST_ACTIVE_LANE:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.nxv4i1(<vscale x 4 x i1> [[TMP15]], i1 true)
+; CHECK-NEXT:    [[EARLY_EXIT_VALUE:%.*]] = extractelement <vscale x 4 x i64> [[VEC_IND]], i64 [[FIRST_ACTIVE_LANE]]
+; CHECK-NEXT:    br label [[LOOP_END:%.*]]
+; CHECK:       middle.block:
+; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 64, [[INDEX1]]
+; CHECK-NEXT:    br i1 [[CMP_N]], label [[LOOP_END]], label [[SCALAR_PH]]
+; CHECK:       scalar.ph:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[OFFSET_IDX]], [[MIDDLE_BLOCK]] ], [ 3, [[ENTRY:%.*]] ]
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
-; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ [[INDEX_NEXT:%.*]], [[LOOP_INC:%.*]] ], [ 3, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ [[INDEX_NEXT:%.*]], [[LOOP_INC:%.*]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
 ; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[P1]], i64 [[INDEX]]
 ; CHECK-NEXT:    [[LD1:%.*]] = load i32, ptr [[ARRAYIDX]], align 1
 ; CHECK-NEXT:    [[DIV:%.*]] = udiv i32 [[LD1]], 20000
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[DIV]], 1
-; CHECK-NEXT:    br i1 [[CMP]], label [[LOOP_INC]], label [[LOOP_END:%.*]]
+; CHECK-NEXT:    br i1 [[CMP]], label [[LOOP_INC]], label [[LOOP_END]]
 ; CHECK:       loop.inc:
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 1
 ; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp ne i64 [[INDEX_NEXT]], 67
-; CHECK-NEXT:    br i1 [[EXITCOND]], label [[LOOP]], label [[LOOP_END]]
+; CHECK-NEXT:    br i1 [[EXITCOND]], label [[LOOP]], label [[LOOP_END]], !llvm.loop [[LOOP9:![0-9]+]]
 ; CHECK:       loop.end:
-; CHECK-NEXT:    [[RETVAL:%.*]] = phi i64 [ [[INDEX]], [[LOOP]] ], [ 67, [[LOOP_INC]] ]
+; CHECK-NEXT:    [[RETVAL:%.*]] = phi i64 [ [[INDEX]], [[LOOP]] ], [ 67, [[LOOP_INC]] ], [ 67, [[MIDDLE_BLOCK]] ], [ [[EARLY_EXIT_VALUE]], [[VECTOR_EARLY_EXIT]] ]
 ; CHECK-NEXT:    ret i64 [[RETVAL]]
 ;
 entry:
@@ -216,21 +370,54 @@ define i64 @loop_contains_load_after_early_exit(ptr dereferenceable(1024) align(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[P1:%.*]] = alloca [1024 x i8], align 4
 ; CHECK-NEXT:    call void @init_mem(ptr [[P1]], i64 1024)
-; CHECK-NEXT:    br label [[LOOP1:%.*]]
+; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK:       vector.ph:
+; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK:       vector.body:
+; CHECK-NEXT:    [[INDEX1:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT3:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 3, i64 4, i64 5, i64 6>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[OFFSET_IDX:%.*]] = add i64 3, [[INDEX1]]
+; CHECK-NEXT:    [[TMP0:%.*]] = add i64 [[OFFSET_IDX]], 0
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[P1]], i64 [[TMP0]]
+; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i32 0
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP2]], align 1
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp eq <4 x i32> [[WIDE_LOAD]], splat (i32 1)
+; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr i64, ptr [[P2]], i64 [[TMP0]]
+; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr i64, ptr [[TMP4]], i32 0
+; CHECK-NEXT:    [[WIDE_LOAD2:%.*]] = load <4 x i64>, ptr [[TMP5]], align 8
+; CHECK-NEXT:    [[INDEX_NEXT3]] = add nuw i64 [[INDEX1]], 4
+; CHECK-NEXT:    [[TMP6:%.*]] = xor <4 x i1> [[TMP3]], splat (i1 true)
+; CHECK-NEXT:    [[TMP7:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP6]])
+; CHECK-NEXT:    [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT3]], 64
+; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <4 x i64> [[VEC_IND]], splat (i64 4)
+; CHECK-NEXT:    [[TMP9:%.*]] = or i1 [[TMP7]], [[TMP8]]
+; CHECK-NEXT:    br i1 [[TMP9]], label [[MIDDLE_SPLIT:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
+; CHECK:       middle.split:
+; CHECK-NEXT:    [[TMP10:%.*]] = extractelement <4 x i64> [[WIDE_LOAD2]], i32 3
+; CHECK-NEXT:    br i1 [[TMP7]], label [[VECTOR_EARLY_EXIT:%.*]], label [[MIDDLE_BLOCK:%.*]]
+; CHECK:       vector.early.exit:
+; CHECK-NEXT:    [[FIRST_ACTIVE_LANE:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP6]], i1 true)
+; CHECK-NEXT:    [[EARLY_EXIT_VALUE:%.*]] = extractelement <4 x i64> [[VEC_IND]], i64 [[FIRST_ACTIVE_LANE]]
+; CHECK-NEXT:    br label [[LOOP_END:%.*]]
+; CHECK:       middle.block:
+; CHECK-NEXT:    br i1 true, label [[LOOP_END]], label [[SCALAR_PH]]
+; CHECK:       scalar.ph:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 67, [[MIDDLE_BLOCK]] ], [ 3, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
-; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ [[INDEX_NEXT:%.*]], [[LOOP_INC:%.*]] ], [ 3, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ [[INDEX_NEXT:%.*]], [[LOOP_INC:%.*]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
 ; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[P1]], i64 [[INDEX]]
 ; CHECK-NEXT:    [[LD1:%.*]] = load i32, ptr [[ARRAYIDX]], align 1
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[LD1]], 1
-; CHECK-NEXT:    br i1 [[CMP]], label [[LOOP_INC]], label [[LOOP_END:%.*]]
+; CHECK-NEXT:    br i1 [[CMP]], label [[LOOP_INC]], label [[LOOP_END]]
 ; CHECK:       loop.inc:
 ; CHECK-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds i64, ptr [[P2]], i64 [[INDEX]]
 ; CHECK-NEXT:    [[LD2:%.*]] = load i64, ptr [[ARRAYIDX2]], align 8
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 1
 ; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp ne i64 [[INDEX_NEXT]], 67
-; CHECK-NEXT:    br i1 [[EXITCOND]], label [[LOOP1]], label [[LOOP_END]]
+; CHECK-NEXT:    br i1 [[EXITCOND]], label [[LOOP]], label [[LOOP_END]], !llvm.loop [[LOOP11:![0-9]+]]
 ; CHECK:       loop.end:
-; CHECK-NEXT:    [[RETVAL:%.*]] = phi i64 [ [[INDEX]], [[LOOP1]] ], [ [[LD2]], [[LOOP_INC]] ]
+; CHECK-NEXT:    [[RETVAL:%.*]] = phi i64 [ [[INDEX]], [[LOOP]] ], [ [[LD2]], [[LOOP_INC]] ], [ [[TMP10]], [[MIDDLE_BLOCK]] ], [ [[EARLY_EXIT_VALUE]], [[VECTOR_EARLY_EXIT]] ]
 ; CHECK-NEXT:    ret i64 [[RETVAL]]
 ;
 entry:
@@ -272,9 +459,9 @@ define i32 @diff_exit_block_needs_scev_check(i32 %end) {
 ; CHECK-NEXT:    call void @init_mem(ptr [[P1]], i64 1024)
 ; CHECK-NEXT:    call void @init_mem(ptr [[P2]], i64 1024)
 ; CHECK-NEXT:    [[END_CLAMPED:%.*]] = and i32 [[END]], 1023
-; CHECK-NEXT:    [[TMP19:%.*]] = trunc i32 [[END]] to i10
-; CHECK-NEXT:    [[TMP20:%.*]] = zext i10 [[TMP19]] to i64
-; CHECK-NEXT:    [[UMAX1:%.*]] = call i64 @llvm.umax.i64(i64 [[TMP20]], i64 1)
+; CHECK-NEXT:    [[TMP0:%.*]] = trunc i32 [[END]] to i10
+; CHECK-NEXT:    [[TMP1:%.*]] = zext i10 [[TMP0]] to i64
+; CHECK-NEXT:    [[UMAX1:%.*]] = call i64 @llvm.umax.i64(i64 [[TMP1]], i64 1)
 ; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[UMAX1]], 12
 ; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_SCEVCHECK:%.*]]
 ; CHECK:       vector.scevcheck:
@@ -289,47 +476,49 @@ define i32 @diff_exit_block_needs_scev_check(i32 %end) {
 ; CHECK:       vector.ph:
 ; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[UMAX1]], 4
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[UMAX1]], [[N_MOD_VF]]
-; CHECK-NEXT:    [[IND_END:%.*]] = trunc i64 [[N_VEC]] to i8
-; CHECK-NEXT:    br label [[FOR_BODY1:%.*]]
+; CHECK-NEXT:    [[TMP8:%.*]] = trunc i64 [[N_VEC]] to i8
+; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
-; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[FOR_BODY1]] ]
-; CHECK-NEXT:    [[TMP8:%.*]] = add i64 [[INDEX]], 0
-; CHECK-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i32, ptr [[P1]], i64 [[TMP8]]
-; CHECK-NEXT:    [[TMP10:%.*]] = getelementptr inbounds i32, ptr [[TMP9]], i32 0
-; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP10]], align 4
-; CHECK-NEXT:    [[TMP11:%.*]] = getelementptr inbounds i32, ptr [[P2]], i64 [[TMP8]]
-; CHECK-NEXT:    [[TMP12:%.*]] = getelementptr inbounds i32, ptr [[TMP11]], i32 0
-; CHECK-NEXT:    [[WIDE_LOAD3:%.*]] = load <4 x i32>, ptr [[TMP12]], align 4
-; CHECK-NEXT:    [[TMP13:%.*]] = icmp eq <4 x i32> [[WIDE_LOAD]], [[WIDE_LOAD3]]
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP9:%.*]] = add i64 [[INDEX]], 0
+; CHECK-NEXT:    [[TMP10:%.*]] = getelementptr inbounds i32, ptr [[P1]], i64 [[TMP9]]
+; CHECK-NEXT:    [[TMP11:%.*]] = getelementptr inbounds i32, ptr [[TMP10]], i32 0
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP11]], align 4
+; CHECK-NEXT:    [[TMP12:%.*]] = getelementptr inbounds i32, ptr [[P2]], i64 [[TMP9]]
+; CHECK-NEXT:    [[TMP13:%.*]] = getelementptr inbounds i32, ptr [[TMP12]], i32 0
+; CHECK-NEXT:    [[WIDE_LOAD2:%.*]] = load <4 x i32>, ptr [[TMP13]], align 4
+; CHECK-NEXT:    [[TMP14:%.*]] = icmp eq <4 x i32> [[WIDE_LOAD]], [[WIDE_LOAD2]]
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
-; CHECK-NEXT:    [[TMP16:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP13]])
-; CHECK-NEXT:    [[TMP17:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT:    [[TMP18:%.*]] = or i1 [[TMP16]], [[TMP17]]
-; CHECK-NEXT:    br i1 [[TMP18]], label [[MIDDLE_SPLIT:%.*]], label [[FOR_BODY1]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK-NEXT:    [[TMP15:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP14]])
+; CHECK-NEXT:    [[TMP16:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT:    [[TMP17:%.*]] = or i1 [[TMP15]], [[TMP16]]
+; CHECK-NEXT:    br i1 [[TMP17]], label [[MIDDLE_SPLIT:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
 ; CHECK:       middle.split:
-; CHECK-NEXT:    br i1 [[TMP16]], label [[FOUND:%.*]], label [[MIDDLE_BLOCK:%.*]]
+; CHECK-NEXT:    br i1 [[TMP15]], label [[VECTOR_EARLY_EXIT:%.*]], label [[MIDDLE_BLOCK:%.*]]
+; CHECK:       vector.early.exit:
+; CHECK-NEXT:    br label [[FOUND:%.*]]
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[UMAX1]], [[N_VEC]]
 ; CHECK-NEXT:    br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i8 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ 0, [[VECTOR_SCEVCHECK]] ], [ 0, [[ENTRY:%.*]] ]
-; CHECK-NEXT:    [[BC_RESUME_VAL2:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[VECTOR_SCEVCHECK]] ], [ 0, [[ENTRY]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i8 [ [[TMP8]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ], [ 0, [[VECTOR_SCEVCHECK]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL3:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ], [ 0, [[VECTOR_SCEVCHECK]] ]
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK:       for.body:
 ; CHECK-NEXT:    [[IND:%.*]] = phi i8 [ [[IND_NEXT:%.*]], [[FOR_INC:%.*]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
-; CHECK-NEXT:    [[GEP_IND:%.*]] = phi i64 [ [[GEP_IND_NEXT:%.*]], [[FOR_INC]] ], [ [[BC_RESUME_VAL2]], [[SCALAR_PH]] ]
+; CHECK-NEXT:    [[GEP_IND:%.*]] = phi i64 [ [[GEP_IND_NEXT:%.*]], [[FOR_INC]] ], [ [[BC_RESUME_VAL3]], [[SCALAR_PH]] ]
 ; CHECK-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds i32, ptr [[P1]], i64 [[GEP_IND]]
-; CHECK-NEXT:    [[TMP0:%.*]] = load i32, ptr [[ARRAYIDX1]], align 4
+; CHECK-NEXT:    [[TMP18:%.*]] = load i32, ptr [[ARRAYIDX1]], align 4
 ; CHECK-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds i32, ptr [[P2]], i64 [[GEP_IND]]
-; CHECK-NEXT:    [[TMP1:%.*]] = load i32, ptr [[ARRAYIDX2]], align 4
-; CHECK-NEXT:    [[CMP_EARLY:%.*]] = icmp eq i32 [[TMP0]], [[TMP1]]
+; CHECK-NEXT:    [[TMP19:%.*]] = load i32, ptr [[ARRAYIDX2]], align 4
+; CHECK-NEXT:    [[CMP_EARLY:%.*]] = icmp eq i32 [[TMP18]], [[TMP19]]
 ; CHECK-NEXT:    br i1 [[CMP_EARLY]], label [[FOUND]], label [[FOR_INC]]
 ; CHECK:       for.inc:
 ; CHECK-NEXT:    [[IND_NEXT]] = add i8 [[IND]], 1
 ; CHECK-NEXT:    [[CONV:%.*]] = zext i8 [[IND_NEXT]] to i32
 ; CHECK-NEXT:    [[GEP_IND_NEXT]] = add i64 [[GEP_IND]], 1
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i32 [[CONV]], [[END_CLAMPED]]
-; CHECK-NEXT:    br i1 [[CMP]], label [[FOR_BODY]], label [[EXIT]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK-NEXT:    br i1 [[CMP]], label [[FOR_BODY]], label [[EXIT]], !llvm.loop [[LOOP13:![0-9]+]]
 ; CHECK:       found:
 ; CHECK-NEXT:    ret i32 1
 ; CHECK:       exit:
@@ -420,5 +609,15 @@ attributes #1 = { "target-features"="+sve" vscale_range(1,16) }
 ; CHECK: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]], [[META2:![0-9]+]]}
 ; CHECK: [[META1]] = !{!"llvm.loop.isvectorized", i32 1}
 ; CHECK: [[META2]] = !{!"llvm.loop.unroll.runtime.disable"}
-; CHECK: [[LOOP3]] = distinct !{[[LOOP3]], [[META1]]}
+; CHECK: [[LOOP3]] = distinct !{[[LOOP3]], [[META2]], [[META1]]}
+; CHECK: [[LOOP4]] = distinct !{[[LOOP4]], [[META1]], [[META2]]}
+; CHECK: [[LOOP5]] = distinct !{[[LOOP5]], [[META2]], [[META1]]}
+; CHECK: [[LOOP6]] = distinct !{[[LOOP6]], [[META1]], [[META2]]}
+; CHECK: [[LOOP7]] = distinct !{[[LOOP7]], [[META2]], [[META1]]}
+; CHECK: [[LOOP8]] = distinct !{[[LOOP8]], [[META1]], [[META2]]}
+; CHECK: [[LOOP9]] = distinct !{[[LOOP9]], [[META2]], [[META1]]}
+; CHECK: [[LOOP10]] = distinct !{[[LOOP10]], [[META1]], [[META2]]}
+; CHECK: [[LOOP11]] = distinct !{[[LOOP11]], [[META2]], [[META1]]}
+; CHECK: [[LOOP12]] = distinct !{[[LOOP12]], [[META1]], [[META2]]}
+; CHECK: [[LOOP13]] = distinct !{[[LOOP13]], [[META1]]}
 ;.
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/store-costs-sve.ll b/llvm/test/Transforms/LoopVectorize/AArch64/store-costs-sve.ll
index 1d4b808a612a0..cd71a1c2c04d2 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/store-costs-sve.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/store-costs-sve.ll
@@ -70,7 +70,7 @@ define void @cost_store_i8(ptr %dst) #0 {
 ; DEFAULT-NEXT:    [[CMP_N4:%.*]] = icmp eq i64 101, [[N_VEC3]]
 ; DEFAULT-NEXT:    br i1 [[CMP_N4]], label [[EXIT]], label [[VEC_EPILOG_SCALAR_PH]]
 ; DEFAULT:       vec.epilog.scalar.ph:
-; DEFAULT-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC3]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ 0, [[ITER_CHECK:%.*]] ], [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ]
+; DEFAULT-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC3]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[ITER_CHECK:%.*]] ]
 ; DEFAULT-NEXT:    br label [[LOOP:%.*]]
 ; DEFAULT:       loop:
 ; DEFAULT-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[VEC_EPILOG_SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
@@ -207,7 +207,7 @@ define void @trunc_store(ptr %dst, ptr %src, i16 %x) #1 {
 ; DEFAULT:       vec.epilog.middle.block:
 ; DEFAULT-NEXT:    br i1 true, label [[EXIT]], label [[VEC_EPILOG_SCALAR_PH]]
 ; DEFAULT:       vec.epilog.scalar.ph:
-; DEFAULT-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 1000, [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ 0, [[VECTOR_MEMCHECK]] ], [ 0, [[ITER_CHECK:%.*]] ], [ 992, [[VEC_EPILOG_ITER_CHECK]] ]
+; DEFAULT-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 1000, [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ 992, [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[VECTOR_MEMCHECK]] ], [ 0, [[ITER_CHECK:%.*]] ]
 ; DEFAULT-NEXT:    br label [[LOOP:%.*]]
 ; DEFAULT:       loop:
 ; DEFAULT-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[VEC_EPILOG_SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
@@ -268,7 +268,7 @@ define void @trunc_store(ptr %dst, ptr %src, i16 %x) #1 {
 ; PRED:       middle.block:
 ; PRED-NEXT:    br i1 true, label [[EXIT:%.*]], label [[SCALAR_PH]]
 ; PRED:       scalar.ph:
-; PRED-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[VECTOR_MEMCHECK]] ], [ 0, [[ENTRY:%.*]] ]
+; PRED-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ], [ 0, [[VECTOR_MEMCHECK]] ]
 ; PRED-NEXT:    br label [[LOOP:%.*]]
 ; PRED:       loop:
 ; PRED-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/streaming-vectorization.ll b/llvm/test/Transforms/LoopVectorize/AArch64/streaming-vectorization.ll
index 924d4bfb7836a..a26404eabedef 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/streaming-vectorization.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/streaming-vectorization.ll
@@ -1,6 +1,6 @@
 ; REQUIRES: asserts
-; RUN: opt -S -passes=loop-vectorize -debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefixes=CHECK,NOVEC
-; RUN: opt -S -passes=loop-vectorize -debug-only=loop-vectorize -enable-scalable-autovec-in-streaming-mode < %s 2>&1 | FileCheck %s --check-prefixes=CHECK,VEC
+; RUN: opt -S -passes=loop-vectorize -debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefixes=CHECK,NOVEC
+; RUN: opt -S -passes=loop-vectorize -debug-only=loop-vectorize --disable-output -enable-scalable-autovec-in-streaming-mode < %s 2>&1 | FileCheck %s --check-prefixes=CHECK,VEC
 
 target triple = "aarch64-unknown-linux-gnu"
 
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-epilog-vect-inloop-reductions.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-epilog-vect-inloop-reductions.ll
index 6dda3de0d7b0b..754f97c21608a 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-epilog-vect-inloop-reductions.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-epilog-vect-inloop-reductions.ll
@@ -71,8 +71,8 @@ define i64 @int_reduction_and(ptr noalias nocapture %a, i64 %N) {
 ; CHECK-NEXT:    [[CMP_N6:%.*]] = icmp eq i64 [[N]], [[N_VEC5]]
 ; CHECK-NEXT:    br i1 [[CMP_N6]], label [[FOR_END]], label [[VEC_EPILOG_SCALAR_PH]]
 ; CHECK:       vec.epilog.scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC5]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ 0, [[ITER_CHECK:%.*]] ], [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ]
-; CHECK-NEXT:    [[BC_MERGE_RDX11:%.*]] = phi i64 [ [[TMP27]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ 1, [[ITER_CHECK]] ], [ [[BIN_RDX]], [[VEC_EPILOG_ITER_CHECK]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC5]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[ITER_CHECK:%.*]] ]
+; CHECK-NEXT:    [[BC_MERGE_RDX11:%.*]] = phi i64 [ [[TMP27]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[BIN_RDX]], [[VEC_EPILOG_ITER_CHECK]] ], [ 1, [[ITER_CHECK]] ]
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK:       for.body:
 ; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[IV_NEXT:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL]], [[VEC_EPILOG_SCALAR_PH]] ]
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-epilog-vect-reductions.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-epilog-vect-reductions.ll
index c2b32d87c7a16..9d06363846254 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-epilog-vect-reductions.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-epilog-vect-reductions.ll
@@ -71,8 +71,8 @@ define i64 @int_reduction_add(ptr %a, i64 %N) {
 ; CHECK-NEXT:    [[CMP_N6:%.*]] = icmp eq i64 [[N]], [[N_VEC5]]
 ; CHECK-NEXT:    br i1 [[CMP_N6]], label [[FOR_END]], label [[VEC_EPILOG_SCALAR_PH]]
 ; CHECK:       vec.epilog.scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC5]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ 0, [[ITER_CHECK:%.*]] ], [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ]
-; CHECK-NEXT:    [[BC_MERGE_RDX11:%.*]] = phi i64 [ [[TMP28]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ 5, [[ITER_CHECK]] ], [ [[TMP21]], [[VEC_EPILOG_ITER_CHECK]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC5]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[ITER_CHECK:%.*]] ]
+; CHECK-NEXT:    [[BC_MERGE_RDX11:%.*]] = phi i64 [ [[TMP28]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP21]], [[VEC_EPILOG_ITER_CHECK]] ], [ 5, [[ITER_CHECK]] ]
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK:       for.body:
 ; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[VEC_EPILOG_SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-epilog-vect-strict-reductions.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-epilog-vect-strict-reductions.ll
index 5c90ee3a1bc88..af678d7a20f42 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-epilog-vect-strict-reductions.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-epilog-vect-strict-reductions.ll
@@ -66,8 +66,8 @@ define float @fadd_strict(ptr noalias nocapture readonly %a, i64 %n) {
 ; CHECK-NEXT:    [[CMP_N5:%.*]] = icmp eq i64 [[N]], [[N_VEC4]]
 ; CHECK-NEXT:    br i1 [[CMP_N5]], label [[FOR_END]], label [[VEC_EPILOG_SCALAR_PH]]
 ; CHECK:       vec.epilog.scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC4]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ 0, [[ITER_CHECK:%.*]] ], [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ]
-; CHECK-NEXT:    [[BC_MERGE_RDX10:%.*]] = phi float [ [[TMP24]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ 0xFFFFFFFFE0000000, [[ITER_CHECK]] ], [ [[TMP19]], [[VEC_EPILOG_ITER_CHECK]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC4]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[ITER_CHECK:%.*]] ]
+; CHECK-NEXT:    [[BC_MERGE_RDX10:%.*]] = phi float [ [[TMP24]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP19]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0xFFFFFFFFE0000000, [[ITER_CHECK]] ]
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK:       for.body:
 ; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[VEC_EPILOG_SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-epilog-vect.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-epilog-vect.ll
index 295c0655a4b4d..b7eddd7fdbccf 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-epilog-vect.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-epilog-vect.ll
@@ -82,7 +82,7 @@ define void @main_vf_vscale_x_16(ptr %A) #0 {
 ; CHECK-NEXT:    [[CMP_N4:%.*]] = icmp eq i64 1024, [[N_VEC3]]
 ; CHECK-NEXT:    br i1 [[CMP_N4]], label [[EXIT]], label [[VEC_EPILOG_SCALAR_PH]]
 ; CHECK:       vec.epilog.scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC3]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ 0, [[ITER_CHECK:%.*]] ], [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC3]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[ITER_CHECK:%.*]] ]
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK:       for.body:
 ; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[VEC_EPILOG_SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
@@ -145,7 +145,7 @@ define void @main_vf_vscale_x_16(ptr %A) #0 {
 ; CHECK-VF8:       vec.epilog.middle.block:
 ; CHECK-VF8-NEXT:    br i1 true, label [[EXIT]], label [[VEC_EPILOG_SCALAR_PH]]
 ; CHECK-VF8:       vec.epilog.scalar.ph:
-; CHECK-VF8-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 1024, [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ 0, [[ITER_CHECK:%.*]] ], [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ]
+; CHECK-VF8-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 1024, [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[ITER_CHECK:%.*]] ]
 ; CHECK-VF8-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK-VF8:       for.body:
 ; CHECK-VF8-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[VEC_EPILOG_SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
@@ -239,7 +239,7 @@ define void @main_vf_vscale_x_2(ptr %A) #0 vscale_range(8, 8) {
 ; CHECK:       vec.epilog.middle.block:
 ; CHECK-NEXT:    br i1 true, label [[EXIT]], label [[VEC_EPILOG_SCALAR_PH]]
 ; CHECK:       vec.epilog.scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 1024, [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ 0, [[ITER_CHECK:%.*]] ], [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 1024, [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[ITER_CHECK:%.*]] ]
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK:       for.body:
 ; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[VEC_EPILOG_SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
@@ -302,7 +302,7 @@ define void @main_vf_vscale_x_2(ptr %A) #0 vscale_range(8, 8) {
 ; CHECK-VF8:       vec.epilog.middle.block:
 ; CHECK-VF8-NEXT:    br i1 true, label [[EXIT]], label [[VEC_EPILOG_SCALAR_PH]]
 ; CHECK-VF8:       vec.epilog.scalar.ph:
-; CHECK-VF8-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 1024, [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ 0, [[ITER_CHECK:%.*]] ], [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ]
+; CHECK-VF8-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 1024, [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[ITER_CHECK:%.*]] ]
 ; CHECK-VF8-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK-VF8:       for.body:
 ; CHECK-VF8-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[VEC_EPILOG_SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
@@ -397,8 +397,8 @@ define void @test_pr57912_pointer_induction(ptr %start) #0 {
 ; CHECK-NEXT:    [[CMP_N6:%.*]] = icmp eq i64 10000, [[N_VEC3]]
 ; CHECK-NEXT:    br i1 [[CMP_N6]], label [[EXIT]], label [[VEC_EPILOG_SCALAR_PH]]
 ; CHECK:       vec.epilog.scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC3]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ 0, [[ITER_CHECK:%.*]] ], [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ]
-; CHECK-NEXT:    [[BC_RESUME_VAL5:%.*]] = phi ptr [ [[IND_END]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[START]], [[ITER_CHECK]] ], [ [[IND_END4]], [[VEC_EPILOG_ITER_CHECK]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC3]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[ITER_CHECK:%.*]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL5:%.*]] = phi ptr [ [[IND_END]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[IND_END4]], [[VEC_EPILOG_ITER_CHECK]] ], [ [[START]], [[ITER_CHECK]] ]
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
 ; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[VEC_EPILOG_SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
@@ -464,8 +464,8 @@ define void @test_pr57912_pointer_induction(ptr %start) #0 {
 ; CHECK-VF8:       vec.epilog.middle.block:
 ; CHECK-VF8-NEXT:    br i1 true, label [[EXIT]], label [[VEC_EPILOG_SCALAR_PH]]
 ; CHECK-VF8:       vec.epilog.scalar.ph:
-; CHECK-VF8-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 10000, [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ 0, [[ITER_CHECK:%.*]] ], [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ]
-; CHECK-VF8-NEXT:    [[BC_RESUME_VAL2:%.*]] = phi ptr [ [[IND_END]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[START]], [[ITER_CHECK]] ], [ [[IND_END1]], [[VEC_EPILOG_ITER_CHECK]] ]
+; CHECK-VF8-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 10000, [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[ITER_CHECK:%.*]] ]
+; CHECK-VF8-NEXT:    [[BC_RESUME_VAL2:%.*]] = phi ptr [ [[IND_END]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[IND_END1]], [[VEC_EPILOG_ITER_CHECK]] ], [ [[START]], [[ITER_CHECK]] ]
 ; CHECK-VF8-NEXT:    br label [[LOOP:%.*]]
 ; CHECK-VF8:       loop:
 ; CHECK-VF8-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[VEC_EPILOG_SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-fneg.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-fneg.ll
index a8122849c203c..08d0fb77e456d 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-fneg.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-fneg.ll
@@ -60,7 +60,7 @@ define void @fneg(ptr nocapture noundef writeonly %d, ptr nocapture noundef read
 ; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[WIDE_TRIP_COUNT]], [[N_VEC]]
 ; CHECK-NEXT:    br i1 [[CMP_N]], label [[FOR_COND_CLEANUP_LOOPEXIT:%.*]], label [[SCALAR_PH]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[VECTOR_MEMCHECK]] ], [ 0, [[FOR_BODY_PREHEADER]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[FOR_BODY_PREHEADER]] ], [ 0, [[VECTOR_MEMCHECK]] ]
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK:       for.cond.cleanup.loopexit:
 ; CHECK-NEXT:    br label [[FOR_COND_CLEANUP]]
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-interleaved-accesses.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-interleaved-accesses.ll
index bf95622733461..eb805b1f8c571 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-interleaved-accesses.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-interleaved-accesses.ll
@@ -396,8 +396,8 @@ define void @test_reversed_load2_store2(ptr noalias nocapture readonly %A, ptr n
 ; CHECK-NEXT:    [[WIDE_VEC:%.*]] = load <vscale x 8 x i32>, ptr [[TMP9]], align 4
 ; CHECK-NEXT:    [[STRIDED_VEC:%.*]] = call { <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.vector.deinterleave2.nxv8i32(<vscale x 8 x i32> [[WIDE_VEC]])
 ; CHECK-NEXT:    [[TMP10:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } [[STRIDED_VEC]], 0
-; CHECK-NEXT:    [[REVERSE:%.*]] = call <vscale x 4 x i32> @llvm.vector.reverse.nxv4i32(<vscale x 4 x i32> [[TMP10]])
 ; CHECK-NEXT:    [[TMP11:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } [[STRIDED_VEC]], 1
+; CHECK-NEXT:    [[REVERSE:%.*]] = call <vscale x 4 x i32> @llvm.vector.reverse.nxv4i32(<vscale x 4 x i32> [[TMP10]])
 ; CHECK-NEXT:    [[REVERSE1:%.*]] = call <vscale x 4 x i32> @llvm.vector.reverse.nxv4i32(<vscale x 4 x i32> [[TMP11]])
 ; CHECK-NEXT:    [[TMP12:%.*]] = add nsw <vscale x 4 x i32> [[REVERSE]], [[VEC_IND]]
 ; CHECK-NEXT:    [[TMP13:%.*]] = sub nsw <vscale x 4 x i32> [[REVERSE1]], [[VEC_IND]]
@@ -1492,9 +1492,9 @@ define void @PR34743(ptr %a, ptr %b, i64 %n) #1 {
 ; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[TMP1]], [[N_VEC]]
 ; CHECK-NEXT:    br i1 [[CMP_N]], label [[END:%.*]], label [[SCALAR_PH]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[SCALAR_RECUR_INIT:%.*]] = phi i16 [ [[VECTOR_RECUR_EXTRACT]], [[MIDDLE_BLOCK]] ], [ [[DOTPRE]], [[VECTOR_MEMCHECK]] ], [ [[DOTPRE]], [[ENTRY:%.*]] ]
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ 0, [[VECTOR_MEMCHECK]] ], [ 0, [[ENTRY]] ]
-; CHECK-NEXT:    [[BC_RESUME_VAL3:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[VECTOR_MEMCHECK]] ], [ 0, [[ENTRY]] ]
+; CHECK-NEXT:    [[SCALAR_RECUR_INIT:%.*]] = phi i16 [ [[VECTOR_RECUR_EXTRACT]], [[MIDDLE_BLOCK]] ], [ [[DOTPRE]], [[ENTRY:%.*]] ], [ [[DOTPRE]], [[VECTOR_MEMCHECK]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ], [ 0, [[VECTOR_MEMCHECK]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL3:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ], [ 0, [[VECTOR_MEMCHECK]] ]
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
 ; CHECK-NEXT:    [[TMP33:%.*]] = phi i16 [ [[SCALAR_RECUR_INIT]], [[SCALAR_PH]] ], [ [[LOAD2:%.*]], [[LOOP]] ]
@@ -1548,5 +1548,263 @@ end:
   ret void
 }
 
+; Check vectorization on an interleaved load/store groups of factor 4
+
+; for (int i = 0; i < 1024; ++i) {
+;   dst[i].x = a[i].x + b[i].x;
+;   dst[i].y = a[i].y - b[i].y;
+;   dst[i].z = a[i].z << b[i].z;
+;   dst[i].t = a[i].t >> b[i].t;
+; }
+%struct.xyzt = type { i32, i32, i32, i32 }
+
+define void @interleave_deinterleave(ptr writeonly noalias %dst, ptr readonly %a, ptr readonly %b) {
+; CHECK-LABEL: @interleave_deinterleave(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    [[TMP1:%.*]] = shl i64 [[TMP0]], 2
+; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ugt i64 [[TMP1]], 1024
+; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK:       vector.ph:
+; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    [[TMP3:%.*]] = shl i64 [[TMP2]], 2
+; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]]
+; CHECK-NEXT:    [[N_VEC:%.*]] = sub nuw nsw i64 1024, [[N_MOD_VF]]
+; CHECK-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    [[TMP5:%.*]] = shl i64 [[TMP4]], 2
+; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK:       vector.body:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr inbounds [[STRUCT_XYZT:%.*]], ptr [[A:%.*]], i64 [[INDEX]]
+; CHECK-NEXT:    [[WIDE_VEC:%.*]] = load <vscale x 16 x i32>, ptr [[TMP6]], align 4
+; CHECK-NEXT:    [[STRIDED_VEC:%.*]] = call { <vscale x 8 x i32>, <vscale x 8 x i32> } @llvm.vector.deinterleave2.nxv16i32(<vscale x 16 x i32> [[WIDE_VEC]])
+; CHECK-NEXT:    [[TMP7:%.*]] = extractvalue { <vscale x 8 x i32>, <vscale x 8 x i32> } [[STRIDED_VEC]], 0
+; CHECK-NEXT:    [[TMP8:%.*]] = extractvalue { <vscale x 8 x i32>, <vscale x 8 x i32> } [[STRIDED_VEC]], 1
+; CHECK-NEXT:    [[STRIDED_VEC6:%.*]] = call { <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.vector.deinterleave2.nxv8i32(<vscale x 8 x i32> [[TMP7]])
+; CHECK-NEXT:    [[STRIDED_VEC7:%.*]] = call { <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.vector.deinterleave2.nxv8i32(<vscale x 8 x i32> [[TMP8]])
+; CHECK-NEXT:    [[TMP9:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } [[STRIDED_VEC6]], 0
+; CHECK-NEXT:    [[TMP10:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } [[STRIDED_VEC7]], 0
+; CHECK-NEXT:    [[TMP11:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } [[STRIDED_VEC6]], 1
+; CHECK-NEXT:    [[TMP12:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } [[STRIDED_VEC7]], 1
+; CHECK-NEXT:    [[TMP13:%.*]] = getelementptr inbounds [[STRUCT_XYZT]], ptr [[B:%.*]], i64 [[INDEX]]
+; CHECK-NEXT:    [[WIDE_VEC8:%.*]] = load <vscale x 16 x i32>, ptr [[TMP13]], align 4
+; CHECK-NEXT:    [[STRIDED_VEC9:%.*]] = call { <vscale x 8 x i32>, <vscale x 8 x i32> } @llvm.vector.deinterleave2.nxv16i32(<vscale x 16 x i32> [[WIDE_VEC8]])
+; CHECK-NEXT:    [[TMP14:%.*]] = extractvalue { <vscale x 8 x i32>, <vscale x 8 x i32> } [[STRIDED_VEC9]], 0
+; CHECK-NEXT:    [[TMP15:%.*]] = extractvalue { <vscale x 8 x i32>, <vscale x 8 x i32> } [[STRIDED_VEC9]], 1
+; CHECK-NEXT:    [[STRIDED_VEC10:%.*]] = call { <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.vector.deinterleave2.nxv8i32(<vscale x 8 x i32> [[TMP14]])
+; CHECK-NEXT:    [[STRIDED_VEC11:%.*]] = call { <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.vector.deinterleave2.nxv8i32(<vscale x 8 x i32> [[TMP15]])
+; CHECK-NEXT:    [[TMP16:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } [[STRIDED_VEC10]], 0
+; CHECK-NEXT:    [[TMP17:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } [[STRIDED_VEC11]], 0
+; CHECK-NEXT:    [[TMP18:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } [[STRIDED_VEC10]], 1
+; CHECK-NEXT:    [[TMP19:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } [[STRIDED_VEC11]], 1
+; CHECK-NEXT:    [[TMP20:%.*]] = add nsw <vscale x 4 x i32> [[TMP16]], [[TMP9]]
+; CHECK-NEXT:    [[TMP21:%.*]] = getelementptr inbounds [[STRUCT_XYZT]], ptr [[DST:%.*]], i64 [[INDEX]]
+; CHECK-NEXT:    [[TMP22:%.*]] = sub nsw <vscale x 4 x i32> [[TMP10]], [[TMP17]]
+; CHECK-NEXT:    [[TMP23:%.*]] = shl <vscale x 4 x i32> [[TMP11]], [[TMP18]]
+; CHECK-NEXT:    [[TMP24:%.*]] = ashr <vscale x 4 x i32> [[TMP12]], [[TMP19]]
+; CHECK-NEXT:    [[INTERLEAVED_VEC:%.*]] = call <vscale x 8 x i32> @llvm.vector.interleave2.nxv8i32(<vscale x 4 x i32> [[TMP20]], <vscale x 4 x i32> [[TMP23]])
+; CHECK-NEXT:    [[INTERLEAVED_VEC12:%.*]] = call <vscale x 8 x i32> @llvm.vector.interleave2.nxv8i32(<vscale x 4 x i32> [[TMP22]], <vscale x 4 x i32> [[TMP24]])
+; CHECK-NEXT:    [[INTERLEAVED_VEC13:%.*]] = call <vscale x 16 x i32> @llvm.vector.interleave2.nxv16i32(<vscale x 8 x i32> [[INTERLEAVED_VEC]], <vscale x 8 x i32> [[INTERLEAVED_VEC12]])
+; CHECK-NEXT:    store <vscale x 16 x i32> [[INTERLEAVED_VEC13]], ptr [[TMP21]], align 4
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
+; CHECK-NEXT:    [[TMP25:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[TMP25]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP41:![0-9]+]]
+; CHECK:       middle.block:
+; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N_MOD_VF]], 0
+; CHECK-NEXT:    br i1 [[CMP_N]], label [[FOR_COND_CLEANUP:%.*]], label [[SCALAR_PH]]
+; CHECK:       scalar.ph:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK:       for.body:
+; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds nuw [[STRUCT_XYZT]], ptr [[A]], i64 [[INDVARS_IV]]
+; CHECK-NEXT:    [[TMP26:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
+; CHECK-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds nuw [[STRUCT_XYZT]], ptr [[B]], i64 [[INDVARS_IV]]
+; CHECK-NEXT:    [[TMP27:%.*]] = load i32, ptr [[ARRAYIDX2]], align 4
+; CHECK-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP27]], [[TMP26]]
+; CHECK-NEXT:    [[ARRAYIDX5:%.*]] = getelementptr inbounds nuw [[STRUCT_XYZT]], ptr [[DST]], i64 [[INDVARS_IV]]
+; CHECK-NEXT:    store i32 [[ADD]], ptr [[ARRAYIDX5]], align 4
+; CHECK-NEXT:    [[Y:%.*]] = getelementptr inbounds nuw i8, ptr [[ARRAYIDX]], i64 4
+; CHECK-NEXT:    [[TMP28:%.*]] = load i32, ptr [[Y]], align 4
+; CHECK-NEXT:    [[Y11:%.*]] = getelementptr inbounds nuw i8, ptr [[ARRAYIDX2]], i64 4
+; CHECK-NEXT:    [[TMP29:%.*]] = load i32, ptr [[Y11]], align 4
+; CHECK-NEXT:    [[SUB:%.*]] = sub nsw i32 [[TMP28]], [[TMP29]]
+; CHECK-NEXT:    [[Y14:%.*]] = getelementptr inbounds nuw i8, ptr [[ARRAYIDX5]], i64 4
+; CHECK-NEXT:    store i32 [[SUB]], ptr [[Y14]], align 4
+; CHECK-NEXT:    [[Z:%.*]] = getelementptr inbounds nuw i8, ptr [[ARRAYIDX]], i64 8
+; CHECK-NEXT:    [[TMP30:%.*]] = load i32, ptr [[Z]], align 4
+; CHECK-NEXT:    [[Z19:%.*]] = getelementptr inbounds nuw i8, ptr [[ARRAYIDX2]], i64 8
+; CHECK-NEXT:    [[TMP31:%.*]] = load i32, ptr [[Z19]], align 4
+; CHECK-NEXT:    [[SHL:%.*]] = shl i32 [[TMP30]], [[TMP31]]
+; CHECK-NEXT:    [[Z22:%.*]] = getelementptr inbounds nuw i8, ptr [[ARRAYIDX5]], i64 8
+; CHECK-NEXT:    store i32 [[SHL]], ptr [[Z22]], align 4
+; CHECK-NEXT:    [[T:%.*]] = getelementptr inbounds nuw i8, ptr [[ARRAYIDX]], i64 12
+; CHECK-NEXT:    [[TMP32:%.*]] = load i32, ptr [[T]], align 4
+; CHECK-NEXT:    [[T27:%.*]] = getelementptr inbounds nuw i8, ptr [[ARRAYIDX2]], i64 12
+; CHECK-NEXT:    [[TMP33:%.*]] = load i32, ptr [[T27]], align 4
+; CHECK-NEXT:    [[SHR:%.*]] = ashr i32 [[TMP32]], [[TMP33]]
+; CHECK-NEXT:    [[T30:%.*]] = getelementptr inbounds nuw i8, ptr [[ARRAYIDX5]], i64 12
+; CHECK-NEXT:    store i32 [[SHR]], ptr [[T30]], align 4
+; CHECK-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
+; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], 1024
+; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_COND_CLEANUP]], label [[FOR_BODY]], !llvm.loop [[LOOP42:![0-9]+]]
+; CHECK:       for.end:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %for.body
+
+for.body:
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+  %arrayidx = getelementptr inbounds %struct.xyzt, ptr %a, i64 %indvars.iv
+  %0 = load i32, ptr %arrayidx, align 4
+  %arrayidx2 = getelementptr inbounds %struct.xyzt, ptr %b, i64 %indvars.iv
+  %1 = load i32, ptr %arrayidx2, align 4
+  %add = add nsw i32 %1, %0
+  %arrayidx5 = getelementptr inbounds %struct.xyzt, ptr %dst, i64 %indvars.iv
+  store i32 %add, ptr %arrayidx5, align 4
+  %y = getelementptr inbounds nuw i8, ptr %arrayidx, i64 4
+  %2 = load i32, ptr %y, align 4
+  %y11 = getelementptr inbounds nuw i8, ptr %arrayidx2, i64 4
+  %3 = load i32, ptr %y11, align 4
+  %sub = sub nsw i32 %2, %3
+  %y14 = getelementptr inbounds nuw i8, ptr %arrayidx5, i64 4
+  store i32 %sub, ptr %y14, align 4
+  %z = getelementptr inbounds nuw i8, ptr %arrayidx, i64 8
+  %4 = load i32, ptr %z, align 4
+  %z19 = getelementptr inbounds nuw i8, ptr %arrayidx2, i64 8
+  %5 = load i32, ptr %z19, align 4
+  %shl = shl i32 %4, %5
+  %z22 = getelementptr inbounds nuw i8, ptr %arrayidx5, i64 8
+  store i32 %shl, ptr %z22, align 4
+  %t = getelementptr inbounds nuw i8, ptr %arrayidx, i64 12
+  %6 = load i32, ptr %t, align 4
+  %t27 = getelementptr inbounds nuw i8, ptr %arrayidx2, i64 12
+  %7 = load i32, ptr %t27, align 4
+  %shr = ashr i32 %6, %7
+  %t30 = getelementptr inbounds nuw i8, ptr %arrayidx5, i64 12
+  store i32 %shr, ptr %t30, align 4
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %exitcond.not = icmp eq i64 %indvars.iv.next, 1024
+  br i1 %exitcond.not, label %for.end, label %for.body
+
+for.end:
+  ret void
+}
+
+; Check vectorization on a reverse interleaved load/store groups of factor 4
+
+; for (int i = 1023; i >= 0; i--) {
+;   int a = A[i].x + i;
+;   int b = A[i].y - i;
+;   int c = A[i].z * i;
+;   int d = A[i].t << i;
+;   B[i].x = a;
+;   B[i].y = b;
+;   B[i].z = c;
+;   B[i].t = d;
+; }
+
+define void @interleave_deinterleave_reverse(ptr noalias nocapture readonly %A, ptr noalias nocapture %B) #1{
+; CHECK-LABEL: @interleave_deinterleave_reverse(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK:       vector.ph:
+; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    [[TMP1:%.*]] = shl nuw nsw i64 [[TMP0]], 2
+; CHECK-NEXT:    [[TMP2:%.*]] = call <vscale x 4 x i32> @llvm.stepvector.nxv4i32()
+; CHECK-NEXT:    [[INDUCTION:%.*]] = sub <vscale x 4 x i32> splat (i32 1023), [[TMP2]]
+; CHECK-NEXT:    [[TMP3:%.*]] = trunc nuw nsw i64 [[TMP1]] to i32
+; CHECK-NEXT:    [[TMP4:%.*]] = sub nsw i32 0, [[TMP3]]
+; CHECK-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[TMP4]], i64 0
+; CHECK-NEXT:    [[DOTSPLAT:%.*]] = shufflevector <vscale x 4 x i32> [[DOTSPLATINSERT]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
+; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK:       vector.body:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <vscale x 4 x i32> [ [[INDUCTION]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[OFFSET_IDX:%.*]] = sub i64 1023, [[INDEX]]
+; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr inbounds [[STRUCT_XYZT:%.*]], ptr [[A:%.*]], i64 [[OFFSET_IDX]], i32 0
+; CHECK-NEXT:    [[TMP6:%.*]] = call i32 @llvm.vscale.i32()
+; CHECK-NEXT:    [[TMP7:%.*]] = shl nuw nsw i32 [[TMP6]], 4
+; CHECK-NEXT:    [[TMP8:%.*]] = sub nsw i32 4, [[TMP7]]
+; CHECK-NEXT:    [[TMP9:%.*]] = sext i32 [[TMP8]] to i64
+; CHECK-NEXT:    [[TMP10:%.*]] = getelementptr inbounds i32, ptr [[TMP5]], i64 [[TMP9]]
+; CHECK-NEXT:    [[WIDE_VEC:%.*]] = load <vscale x 16 x i32>, ptr [[TMP10]], align 4
+; CHECK-NEXT:    [[STRIDED_VEC:%.*]] = call { <vscale x 8 x i32>, <vscale x 8 x i32> } @llvm.vector.deinterleave2.nxv16i32(<vscale x 16 x i32> [[WIDE_VEC]])
+; CHECK-NEXT:    [[TMP11:%.*]] = extractvalue { <vscale x 8 x i32>, <vscale x 8 x i32> } [[STRIDED_VEC]], 0
+; CHECK-NEXT:    [[TMP12:%.*]] = extractvalue { <vscale x 8 x i32>, <vscale x 8 x i32> } [[STRIDED_VEC]], 1
+; CHECK-NEXT:    [[STRIDED_VEC1:%.*]] = call { <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.vector.deinterleave2.nxv8i32(<vscale x 8 x i32> [[TMP11]])
+; CHECK-NEXT:    [[STRIDED_VEC2:%.*]] = call { <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.vector.deinterleave2.nxv8i32(<vscale x 8 x i32> [[TMP12]])
+; CHECK-NEXT:    [[TMP13:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } [[STRIDED_VEC1]], 0
+; CHECK-NEXT:    [[TMP14:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } [[STRIDED_VEC2]], 0
+; CHECK-NEXT:    [[TMP15:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } [[STRIDED_VEC1]], 1
+; CHECK-NEXT:    [[TMP16:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } [[STRIDED_VEC2]], 1
+; CHECK-NEXT:    [[REVERSE:%.*]] = call <vscale x 4 x i32> @llvm.vector.reverse.nxv4i32(<vscale x 4 x i32> [[TMP13]])
+; CHECK-NEXT:    [[REVERSE3:%.*]] = call <vscale x 4 x i32> @llvm.vector.reverse.nxv4i32(<vscale x 4 x i32> [[TMP14]])
+; CHECK-NEXT:    [[REVERSE4:%.*]] = call <vscale x 4 x i32> @llvm.vector.reverse.nxv4i32(<vscale x 4 x i32> [[TMP15]])
+; CHECK-NEXT:    [[REVERSE5:%.*]] = call <vscale x 4 x i32> @llvm.vector.reverse.nxv4i32(<vscale x 4 x i32> [[TMP16]])
+; CHECK-NEXT:    [[TMP17:%.*]] = add nsw <vscale x 4 x i32> [[REVERSE]], [[VEC_IND]]
+; CHECK-NEXT:    [[TMP18:%.*]] = sub nsw <vscale x 4 x i32> [[REVERSE3]], [[VEC_IND]]
+; CHECK-NEXT:    [[TMP19:%.*]] = mul nsw <vscale x 4 x i32> [[REVERSE4]], [[VEC_IND]]
+; CHECK-NEXT:    [[TMP20:%.*]] = shl nuw nsw <vscale x 4 x i32> [[REVERSE5]], [[VEC_IND]]
+; CHECK-NEXT:    [[TMP21:%.*]] = getelementptr inbounds [[STRUCT_XYZT]], ptr [[B:%.*]], i64 [[OFFSET_IDX]], i32 0
+; CHECK-NEXT:    [[TMP22:%.*]] = call i32 @llvm.vscale.i32()
+; CHECK-NEXT:    [[TMP23:%.*]] = shl nuw nsw i32 [[TMP22]], 4
+; CHECK-NEXT:    [[TMP24:%.*]] = sub nsw i32 4, [[TMP23]]
+; CHECK-NEXT:    [[TMP25:%.*]] = sext i32 [[TMP24]] to i64
+; CHECK-NEXT:    [[TMP26:%.*]] = getelementptr inbounds i32, ptr [[TMP21]], i64 [[TMP25]]
+; CHECK-NEXT:    [[REVERSE6:%.*]] = call <vscale x 4 x i32> @llvm.vector.reverse.nxv4i32(<vscale x 4 x i32> [[TMP17]])
+; CHECK-NEXT:    [[REVERSE7:%.*]] = call <vscale x 4 x i32> @llvm.vector.reverse.nxv4i32(<vscale x 4 x i32> [[TMP18]])
+; CHECK-NEXT:    [[REVERSE8:%.*]] = call <vscale x 4 x i32> @llvm.vector.reverse.nxv4i32(<vscale x 4 x i32> [[TMP19]])
+; CHECK-NEXT:    [[REVERSE9:%.*]] = call <vscale x 4 x i32> @llvm.vector.reverse.nxv4i32(<vscale x 4 x i32> [[TMP20]])
+; CHECK-NEXT:    [[INTERLEAVED_VEC:%.*]] = call <vscale x 8 x i32> @llvm.vector.interleave2.nxv8i32(<vscale x 4 x i32> [[REVERSE6]], <vscale x 4 x i32> [[REVERSE8]])
+; CHECK-NEXT:    [[INTERLEAVED_VEC10:%.*]] = call <vscale x 8 x i32> @llvm.vector.interleave2.nxv8i32(<vscale x 4 x i32> [[REVERSE7]], <vscale x 4 x i32> [[REVERSE9]])
+; CHECK-NEXT:    [[INTERLEAVED_VEC11:%.*]] = call <vscale x 16 x i32> @llvm.vector.interleave2.nxv16i32(<vscale x 8 x i32> [[INTERLEAVED_VEC]], <vscale x 8 x i32> [[INTERLEAVED_VEC10]])
+; CHECK-NEXT:    store <vscale x 16 x i32> [[INTERLEAVED_VEC11]], ptr [[TMP26]], align 4
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP1]]
+; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <vscale x 4 x i32> [[VEC_IND]], [[DOTSPLAT]]
+; CHECK-NEXT:    [[TMP27:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; CHECK-NEXT:    br i1 [[TMP27]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP43:![0-9]+]]
+; CHECK:       middle.block:
+; CHECK-NEXT:    br i1 true, label [[FOR_COND_CLEANUP:%.*]], label [[SCALAR_PH]]
+; CHECK:       scalar.ph:
+; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK:       for.cond.cleanup:
+; CHECK-NEXT:    ret void
+; CHECK:       for.body:
+; CHECK-NEXT:    br i1 poison, label [[FOR_BODY]], label [[FOR_COND_CLEANUP]], !llvm.loop [[LOOP44:![0-9]+]]
+;
+entry:
+  br label %for.body
+for.cond.cleanup:                                 ; preds = %for.body
+  ret void
+for.body:                                         ; preds = %for.body, %entry
+  %indvars.iv = phi i64 [ 1023, %entry ], [ %indvars.iv.next, %for.body ]
+  %x = getelementptr inbounds %struct.xyzt, ptr %A, i64 %indvars.iv, i32 0
+  %load1 = load i32, ptr %x, align 4
+  %trunc = trunc i64 %indvars.iv to i32
+  %add = add nsw i32 %load1, %trunc
+  %y = getelementptr inbounds %struct.xyzt, ptr %A, i64 %indvars.iv, i32 1
+  %load2 = load i32, ptr %y, align 4
+  %sub = sub nsw i32 %load2, %trunc
+  %z = getelementptr inbounds %struct.xyzt, ptr %A, i64 %indvars.iv, i32 2
+  %load3 = load i32, ptr %z, align 4
+  %mul = mul nsw i32 %load3, %trunc
+  %t = getelementptr inbounds %struct.xyzt, ptr %A, i64 %indvars.iv, i32 3
+  %load4 = load i32, ptr %t, align 4
+  %shl = shl nuw nsw i32 %load4, %trunc
+  %x5 = getelementptr inbounds %struct.xyzt, ptr %B, i64 %indvars.iv, i32 0
+  store i32 %add, ptr %x5, align 4
+  %y8 = getelementptr inbounds %struct.xyzt, ptr %B, i64 %indvars.iv, i32 1
+  store i32 %sub, ptr %y8, align 4
+  %z5 = getelementptr inbounds %struct.xyzt, ptr %B, i64 %indvars.iv, i32 2
+  store i32 %mul, ptr %z5, align 4
+  %t8 = getelementptr inbounds %struct.xyzt, ptr %B, i64 %indvars.iv, i32 3
+  store i32 %shl, ptr %t8, align 4
+  %indvars.iv.next = add nsw i64 %indvars.iv, -1
+  %cmp = icmp sgt i64 %indvars.iv, 0
+  br i1 %cmp, label %for.body, label %for.cond.cleanup
+
+}
 attributes #1 = { "target-features"="+sve" vscale_range(1, 16) }
 attributes #0 = { "unsafe-fp-math"="true" "target-features"="+sve" vscale_range(1, 16) }
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-interleaved-masked-accesses.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-interleaved-masked-accesses.ll
index 1a281fe7c6f7f..a2da67c2cbb81 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-interleaved-masked-accesses.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-interleaved-masked-accesses.ll
@@ -23,7 +23,7 @@ target datalayout = "e-m:e-i64:64-i128:128-n32:64-S128"
 ;}
 define dso_local void @masked_strided1(ptr noalias nocapture readonly %p, ptr noalias nocapture %q, i8 zeroext %guard) local_unnamed_addr #0 {
 ; SCALAR_TAIL_FOLDING-LABEL: define dso_local void @masked_strided1
-; SCALAR_TAIL_FOLDING-SAME: (ptr noalias nocapture readonly [[P:%.*]], ptr noalias nocapture [[Q:%.*]], i8 zeroext [[GUARD:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
+; SCALAR_TAIL_FOLDING-SAME: (ptr noalias readonly captures(none) [[P:%.*]], ptr noalias captures(none) [[Q:%.*]], i8 zeroext [[GUARD:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
 ; SCALAR_TAIL_FOLDING-NEXT:  entry:
 ; SCALAR_TAIL_FOLDING-NEXT:    [[CONV:%.*]] = zext i8 [[GUARD]] to i32
 ; SCALAR_TAIL_FOLDING-NEXT:    [[TMP0:%.*]] = call i32 @llvm.vscale.i32()
@@ -102,7 +102,7 @@ define dso_local void @masked_strided1(ptr noalias nocapture readonly %p, ptr no
 ; SCALAR_TAIL_FOLDING-NEXT:    ret void
 ;
 ; PREDICATED_TAIL_FOLDING-LABEL: define dso_local void @masked_strided1
-; PREDICATED_TAIL_FOLDING-SAME: (ptr noalias nocapture readonly [[P:%.*]], ptr noalias nocapture [[Q:%.*]], i8 zeroext [[GUARD:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
+; PREDICATED_TAIL_FOLDING-SAME: (ptr noalias readonly captures(none) [[P:%.*]], ptr noalias captures(none) [[Q:%.*]], i8 zeroext [[GUARD:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
 ; PREDICATED_TAIL_FOLDING-NEXT:  entry:
 ; PREDICATED_TAIL_FOLDING-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; PREDICATED_TAIL_FOLDING:       vector.ph:
@@ -207,7 +207,7 @@ for.end:
 ;}
 define dso_local void @masked_strided2(ptr noalias nocapture readnone %p, ptr noalias nocapture %q, i8 zeroext %guard) local_unnamed_addr #0 {
 ; SCALAR_TAIL_FOLDING-LABEL: define dso_local void @masked_strided2
-; SCALAR_TAIL_FOLDING-SAME: (ptr noalias nocapture readnone [[P:%.*]], ptr noalias nocapture [[Q:%.*]], i8 zeroext [[GUARD:%.*]]) local_unnamed_addr #[[ATTR0]] {
+; SCALAR_TAIL_FOLDING-SAME: (ptr noalias readnone captures(none) [[P:%.*]], ptr noalias captures(none) [[Q:%.*]], i8 zeroext [[GUARD:%.*]]) local_unnamed_addr #[[ATTR0]] {
 ; SCALAR_TAIL_FOLDING-NEXT:  entry:
 ; SCALAR_TAIL_FOLDING-NEXT:    [[CONV:%.*]] = zext i8 [[GUARD]] to i32
 ; SCALAR_TAIL_FOLDING-NEXT:    [[TMP0:%.*]] = call i32 @llvm.vscale.i32()
@@ -271,7 +271,7 @@ define dso_local void @masked_strided2(ptr noalias nocapture readnone %p, ptr no
 ; SCALAR_TAIL_FOLDING-NEXT:    ret void
 ;
 ; PREDICATED_TAIL_FOLDING-LABEL: define dso_local void @masked_strided2
-; PREDICATED_TAIL_FOLDING-SAME: (ptr noalias nocapture readnone [[P:%.*]], ptr noalias nocapture [[Q:%.*]], i8 zeroext [[GUARD:%.*]]) local_unnamed_addr #[[ATTR0]] {
+; PREDICATED_TAIL_FOLDING-SAME: (ptr noalias readnone captures(none) [[P:%.*]], ptr noalias captures(none) [[Q:%.*]], i8 zeroext [[GUARD:%.*]]) local_unnamed_addr #[[ATTR0]] {
 ; PREDICATED_TAIL_FOLDING-NEXT:  entry:
 ; PREDICATED_TAIL_FOLDING-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; PREDICATED_TAIL_FOLDING:       vector.ph:
@@ -364,7 +364,7 @@ for.end:
 ;}
 define dso_local void @masked_strided3(ptr noalias nocapture readnone %p, ptr noalias nocapture %q, i8 zeroext %guard1, i8 zeroext %guard2) local_unnamed_addr #0 {
 ; SCALAR_TAIL_FOLDING-LABEL: define dso_local void @masked_strided3
-; SCALAR_TAIL_FOLDING-SAME: (ptr noalias nocapture readnone [[P:%.*]], ptr noalias nocapture [[Q:%.*]], i8 zeroext [[GUARD1:%.*]], i8 zeroext [[GUARD2:%.*]]) local_unnamed_addr #[[ATTR0]] {
+; SCALAR_TAIL_FOLDING-SAME: (ptr noalias readnone captures(none) [[P:%.*]], ptr noalias captures(none) [[Q:%.*]], i8 zeroext [[GUARD1:%.*]], i8 zeroext [[GUARD2:%.*]]) local_unnamed_addr #[[ATTR0]] {
 ; SCALAR_TAIL_FOLDING-NEXT:  entry:
 ; SCALAR_TAIL_FOLDING-NEXT:    [[CONV:%.*]] = zext i8 [[GUARD1]] to i32
 ; SCALAR_TAIL_FOLDING-NEXT:    [[CONV3:%.*]] = zext i8 [[GUARD2]] to i32
@@ -437,7 +437,7 @@ define dso_local void @masked_strided3(ptr noalias nocapture readnone %p, ptr no
 ; SCALAR_TAIL_FOLDING-NEXT:    ret void
 ;
 ; PREDICATED_TAIL_FOLDING-LABEL: define dso_local void @masked_strided3
-; PREDICATED_TAIL_FOLDING-SAME: (ptr noalias nocapture readnone [[P:%.*]], ptr noalias nocapture [[Q:%.*]], i8 zeroext [[GUARD1:%.*]], i8 zeroext [[GUARD2:%.*]]) local_unnamed_addr #[[ATTR0]] {
+; PREDICATED_TAIL_FOLDING-SAME: (ptr noalias readnone captures(none) [[P:%.*]], ptr noalias captures(none) [[Q:%.*]], i8 zeroext [[GUARD1:%.*]], i8 zeroext [[GUARD2:%.*]]) local_unnamed_addr #[[ATTR0]] {
 ; PREDICATED_TAIL_FOLDING-NEXT:  entry:
 ; PREDICATED_TAIL_FOLDING-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; PREDICATED_TAIL_FOLDING:       vector.ph:
@@ -529,3 +529,255 @@ for.inc:
 for.end:
   ret void
 }
+
+; Expected to contain interleave2/deinterleave2 instructions
+;
+; void masked_strided_factor4(const unsigned char* restrict p,
+;                            unsigned char* restrict q,
+;                            unsigned char guard) {
+; for(ix=0; ix < 1024; ++ix) {
+;     if (ix > guard) {
+;         char left1 = p[4*ix];
+;         char right1 = p[4*ix + 1];
+;         char left2 = p[4*ix + 2];
+;         char right2 = p[4*ix + 3];
+;         char max1 = max(left1, right1);
+;         char max2 = max(left2, right2);
+;         q[4*ix] = max1;
+;         q[4*ix + 1] = 0 - max1;
+;         q[4*ix + 2] = max2;
+;         q[4*ix + 3] = 0 - max2;
+;     }
+; }
+;}
+define dso_local void @masked_strided_factor4(ptr noalias nocapture readonly %p, ptr noalias nocapture %q, i8 zeroext %guard) local_unnamed_addr #0 {
+; SCALAR_TAIL_FOLDING-LABEL: define dso_local void @masked_strided_factor4
+; SCALAR_TAIL_FOLDING-SAME: (ptr noalias readonly captures(none) [[P:%.*]], ptr noalias captures(none) [[Q:%.*]], i8 zeroext [[GUARD:%.*]]) local_unnamed_addr #[[ATTR0]] {
+; SCALAR_TAIL_FOLDING-NEXT:  entry:
+; SCALAR_TAIL_FOLDING-NEXT:    [[CONV:%.*]] = zext i8 [[GUARD]] to i32
+; SCALAR_TAIL_FOLDING-NEXT:    [[TMP0:%.*]] = call i32 @llvm.vscale.i32()
+; SCALAR_TAIL_FOLDING-NEXT:    [[TMP1:%.*]] = shl i32 [[TMP0]], 4
+; SCALAR_TAIL_FOLDING-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ugt i32 [[TMP1]], 1024
+; SCALAR_TAIL_FOLDING-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; SCALAR_TAIL_FOLDING:       vector.ph:
+; SCALAR_TAIL_FOLDING-NEXT:    [[TMP2:%.*]] = call i32 @llvm.vscale.i32()
+; SCALAR_TAIL_FOLDING-NEXT:    [[TMP3:%.*]] = shl i32 [[TMP2]], 4
+; SCALAR_TAIL_FOLDING-NEXT:    [[N_MOD_VF:%.*]] = urem i32 1024, [[TMP3]]
+; SCALAR_TAIL_FOLDING-NEXT:    [[N_VEC:%.*]] = sub nuw nsw i32 1024, [[N_MOD_VF]]
+; SCALAR_TAIL_FOLDING-NEXT:    [[TMP4:%.*]] = call i32 @llvm.vscale.i32()
+; SCALAR_TAIL_FOLDING-NEXT:    [[TMP5:%.*]] = shl i32 [[TMP4]], 4
+; SCALAR_TAIL_FOLDING-NEXT:    [[TMP6:%.*]] = call <vscale x 16 x i32> @llvm.stepvector.nxv16i32()
+; SCALAR_TAIL_FOLDING-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 16 x i32> poison, i32 [[TMP5]], i64 0
+; SCALAR_TAIL_FOLDING-NEXT:    [[DOTSPLAT:%.*]] = shufflevector <vscale x 16 x i32> [[DOTSPLATINSERT]], <vscale x 16 x i32> poison, <vscale x 16 x i32> zeroinitializer
+; SCALAR_TAIL_FOLDING-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 16 x i32> poison, i32 [[CONV]], i64 0
+; SCALAR_TAIL_FOLDING-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 16 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 16 x i32> poison, <vscale x 16 x i32> zeroinitializer
+; SCALAR_TAIL_FOLDING-NEXT:    br label [[VECTOR_BODY:%.*]]
+; SCALAR_TAIL_FOLDING:       vector.body:
+; SCALAR_TAIL_FOLDING-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; SCALAR_TAIL_FOLDING-NEXT:    [[VEC_IND:%.*]] = phi <vscale x 16 x i32> [ [[TMP6]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
+; SCALAR_TAIL_FOLDING-NEXT:    [[TMP7:%.*]] = icmp ugt <vscale x 16 x i32> [[VEC_IND]], [[BROADCAST_SPLAT]]
+; SCALAR_TAIL_FOLDING-NEXT:    [[TMP8:%.*]] = shl i32 [[INDEX]], 2
+; SCALAR_TAIL_FOLDING-NEXT:    [[TMP9:%.*]] = sext i32 [[TMP8]] to i64
+; SCALAR_TAIL_FOLDING-NEXT:    [[TMP10:%.*]] = getelementptr i8, ptr [[P]], i64 [[TMP9]]
+; SCALAR_TAIL_FOLDING-NEXT:    [[INTERLEAVED_MASK:%.*]] = call <vscale x 32 x i1> @llvm.vector.interleave2.nxv32i1(<vscale x 16 x i1> [[TMP7]], <vscale x 16 x i1> [[TMP7]])
+; SCALAR_TAIL_FOLDING-NEXT:    [[INTERLEAVED_MASK1:%.*]] = call <vscale x 32 x i1> @llvm.vector.interleave2.nxv32i1(<vscale x 16 x i1> [[TMP7]], <vscale x 16 x i1> [[TMP7]])
+; SCALAR_TAIL_FOLDING-NEXT:    [[INTERLEAVED_MASK2:%.*]] = call <vscale x 64 x i1> @llvm.vector.interleave2.nxv64i1(<vscale x 32 x i1> [[INTERLEAVED_MASK]], <vscale x 32 x i1> [[INTERLEAVED_MASK1]])
+; SCALAR_TAIL_FOLDING-NEXT:    [[WIDE_MASKED_VEC:%.*]] = call <vscale x 64 x i8> @llvm.masked.load.nxv64i8.p0(ptr [[TMP10]], i32 1, <vscale x 64 x i1> [[INTERLEAVED_MASK2]], <vscale x 64 x i8> poison)
+; SCALAR_TAIL_FOLDING-NEXT:    [[STRIDED_VEC:%.*]] = call { <vscale x 32 x i8>, <vscale x 32 x i8> } @llvm.vector.deinterleave2.nxv64i8(<vscale x 64 x i8> [[WIDE_MASKED_VEC]])
+; SCALAR_TAIL_FOLDING-NEXT:    [[TMP11:%.*]] = extractvalue { <vscale x 32 x i8>, <vscale x 32 x i8> } [[STRIDED_VEC]], 0
+; SCALAR_TAIL_FOLDING-NEXT:    [[TMP12:%.*]] = extractvalue { <vscale x 32 x i8>, <vscale x 32 x i8> } [[STRIDED_VEC]], 1
+; SCALAR_TAIL_FOLDING-NEXT:    [[STRIDED_VEC3:%.*]] = call { <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.vector.deinterleave2.nxv32i8(<vscale x 32 x i8> [[TMP11]])
+; SCALAR_TAIL_FOLDING-NEXT:    [[STRIDED_VEC4:%.*]] = call { <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.vector.deinterleave2.nxv32i8(<vscale x 32 x i8> [[TMP12]])
+; SCALAR_TAIL_FOLDING-NEXT:    [[TMP13:%.*]] = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8> } [[STRIDED_VEC3]], 0
+; SCALAR_TAIL_FOLDING-NEXT:    [[TMP14:%.*]] = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8> } [[STRIDED_VEC4]], 0
+; SCALAR_TAIL_FOLDING-NEXT:    [[TMP15:%.*]] = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8> } [[STRIDED_VEC3]], 1
+; SCALAR_TAIL_FOLDING-NEXT:    [[TMP16:%.*]] = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8> } [[STRIDED_VEC4]], 1
+; SCALAR_TAIL_FOLDING-NEXT:    [[TMP17:%.*]] = call <vscale x 16 x i8> @llvm.smax.nxv16i8(<vscale x 16 x i8> [[TMP13]], <vscale x 16 x i8> [[TMP14]])
+; SCALAR_TAIL_FOLDING-NEXT:    [[TMP18:%.*]] = sub <vscale x 16 x i8> zeroinitializer, [[TMP17]]
+; SCALAR_TAIL_FOLDING-NEXT:    [[TMP19:%.*]] = call <vscale x 16 x i8> @llvm.smax.nxv16i8(<vscale x 16 x i8> [[TMP15]], <vscale x 16 x i8> [[TMP16]])
+; SCALAR_TAIL_FOLDING-NEXT:    [[TMP20:%.*]] = sub <vscale x 16 x i8> zeroinitializer, [[TMP19]]
+; SCALAR_TAIL_FOLDING-NEXT:    [[TMP21:%.*]] = sext i32 [[TMP8]] to i64
+; SCALAR_TAIL_FOLDING-NEXT:    [[TMP22:%.*]] = getelementptr i8, ptr [[Q]], i64 [[TMP21]]
+; SCALAR_TAIL_FOLDING-NEXT:    [[INTERLEAVED_VEC:%.*]] = call <vscale x 32 x i8> @llvm.vector.interleave2.nxv32i8(<vscale x 16 x i8> [[TMP17]], <vscale x 16 x i8> [[TMP19]])
+; SCALAR_TAIL_FOLDING-NEXT:    [[INTERLEAVED_VEC5:%.*]] = call <vscale x 32 x i8> @llvm.vector.interleave2.nxv32i8(<vscale x 16 x i8> [[TMP18]], <vscale x 16 x i8> [[TMP20]])
+; SCALAR_TAIL_FOLDING-NEXT:    [[INTERLEAVED_VEC6:%.*]] = call <vscale x 64 x i8> @llvm.vector.interleave2.nxv64i8(<vscale x 32 x i8> [[INTERLEAVED_VEC]], <vscale x 32 x i8> [[INTERLEAVED_VEC5]])
+; SCALAR_TAIL_FOLDING-NEXT:    [[INTERLEAVED_MASK7:%.*]] = call <vscale x 32 x i1> @llvm.vector.interleave2.nxv32i1(<vscale x 16 x i1> [[TMP7]], <vscale x 16 x i1> [[TMP7]])
+; SCALAR_TAIL_FOLDING-NEXT:    [[INTERLEAVED_MASK8:%.*]] = call <vscale x 32 x i1> @llvm.vector.interleave2.nxv32i1(<vscale x 16 x i1> [[TMP7]], <vscale x 16 x i1> [[TMP7]])
+; SCALAR_TAIL_FOLDING-NEXT:    [[INTERLEAVED_MASK9:%.*]] = call <vscale x 64 x i1> @llvm.vector.interleave2.nxv64i1(<vscale x 32 x i1> [[INTERLEAVED_MASK7]], <vscale x 32 x i1> [[INTERLEAVED_MASK8]])
+; SCALAR_TAIL_FOLDING-NEXT:    call void @llvm.masked.store.nxv64i8.p0(<vscale x 64 x i8> [[INTERLEAVED_VEC6]], ptr [[TMP22]], i32 1, <vscale x 64 x i1> [[INTERLEAVED_MASK9]])
+; SCALAR_TAIL_FOLDING-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], [[TMP5]]
+; SCALAR_TAIL_FOLDING-NEXT:    [[VEC_IND_NEXT]] = add <vscale x 16 x i32> [[VEC_IND]], [[DOTSPLAT]]
+; SCALAR_TAIL_FOLDING-NEXT:    [[TMP23:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
+; SCALAR_TAIL_FOLDING-NEXT:    br i1 [[TMP23]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+; SCALAR_TAIL_FOLDING:       middle.block:
+; SCALAR_TAIL_FOLDING-NEXT:    [[CMP_N:%.*]] = icmp eq i32 [[N_MOD_VF]], 0
+; SCALAR_TAIL_FOLDING-NEXT:    br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]]
+; SCALAR_TAIL_FOLDING:       scalar.ph:
+; SCALAR_TAIL_FOLDING-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
+; SCALAR_TAIL_FOLDING-NEXT:    br label [[FOR_BODY:%.*]]
+; SCALAR_TAIL_FOLDING:       for.body:
+; SCALAR_TAIL_FOLDING-NEXT:    [[IX_024:%.*]] = phi i32 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INC:%.*]], [[FOR_INC:%.*]] ]
+; SCALAR_TAIL_FOLDING-NEXT:    [[CMP1:%.*]] = icmp samesign ugt i32 [[IX_024]], [[CONV]]
+; SCALAR_TAIL_FOLDING-NEXT:    br i1 [[CMP1]], label [[IF_THEN:%.*]], label [[FOR_INC]]
+; SCALAR_TAIL_FOLDING:       if.then:
+; SCALAR_TAIL_FOLDING-NEXT:    [[IDX0:%.*]] = shl nuw nsw i32 [[IX_024]], 2
+; SCALAR_TAIL_FOLDING-NEXT:    [[IDX1:%.*]] = or disjoint i32 [[IDX0]], 1
+; SCALAR_TAIL_FOLDING-NEXT:    [[IDX2:%.*]] = or disjoint i32 [[IDX0]], 2
+; SCALAR_TAIL_FOLDING-NEXT:    [[IDX3:%.*]] = or disjoint i32 [[IDX0]], 3
+; SCALAR_TAIL_FOLDING-NEXT:    [[TMP24:%.*]] = zext nneg i32 [[IDX0]] to i64
+; SCALAR_TAIL_FOLDING-NEXT:    [[ARRAY1IDX0:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i64 [[TMP24]]
+; SCALAR_TAIL_FOLDING-NEXT:    [[TMP25:%.*]] = load i8, ptr [[ARRAY1IDX0]], align 1
+; SCALAR_TAIL_FOLDING-NEXT:    [[TMP26:%.*]] = zext nneg i32 [[IDX1]] to i64
+; SCALAR_TAIL_FOLDING-NEXT:    [[ARRAY1IDX1:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i64 [[TMP26]]
+; SCALAR_TAIL_FOLDING-NEXT:    [[TMP27:%.*]] = load i8, ptr [[ARRAY1IDX1]], align 1
+; SCALAR_TAIL_FOLDING-NEXT:    [[TMP28:%.*]] = zext nneg i32 [[IDX2]] to i64
+; SCALAR_TAIL_FOLDING-NEXT:    [[ARRAY1IDX2:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i64 [[TMP28]]
+; SCALAR_TAIL_FOLDING-NEXT:    [[TMP29:%.*]] = load i8, ptr [[ARRAY1IDX2]], align 1
+; SCALAR_TAIL_FOLDING-NEXT:    [[TMP30:%.*]] = zext nneg i32 [[IDX3]] to i64
+; SCALAR_TAIL_FOLDING-NEXT:    [[ARRAY1IDX3:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i64 [[TMP30]]
+; SCALAR_TAIL_FOLDING-NEXT:    [[TMP31:%.*]] = load i8, ptr [[ARRAY1IDX3]], align 1
+; SCALAR_TAIL_FOLDING-NEXT:    [[SPEC_SELECT_I1:%.*]] = call i8 @llvm.smax.i8(i8 [[TMP25]], i8 [[TMP27]])
+; SCALAR_TAIL_FOLDING-NEXT:    [[SUB1:%.*]] = sub i8 0, [[SPEC_SELECT_I1]]
+; SCALAR_TAIL_FOLDING-NEXT:    [[SPEC_SELECT_I2:%.*]] = call i8 @llvm.smax.i8(i8 [[TMP29]], i8 [[TMP31]])
+; SCALAR_TAIL_FOLDING-NEXT:    [[SUB2:%.*]] = sub i8 0, [[SPEC_SELECT_I2]]
+; SCALAR_TAIL_FOLDING-NEXT:    [[TMP32:%.*]] = zext nneg i32 [[IDX0]] to i64
+; SCALAR_TAIL_FOLDING-NEXT:    [[ARRAY3IDX0:%.*]] = getelementptr inbounds nuw i8, ptr [[Q]], i64 [[TMP32]]
+; SCALAR_TAIL_FOLDING-NEXT:    store i8 [[SPEC_SELECT_I1]], ptr [[ARRAY3IDX0]], align 1
+; SCALAR_TAIL_FOLDING-NEXT:    [[TMP33:%.*]] = zext nneg i32 [[IDX1]] to i64
+; SCALAR_TAIL_FOLDING-NEXT:    [[ARRAY3IDX1:%.*]] = getelementptr inbounds nuw i8, ptr [[Q]], i64 [[TMP33]]
+; SCALAR_TAIL_FOLDING-NEXT:    store i8 [[SUB1]], ptr [[ARRAY3IDX1]], align 1
+; SCALAR_TAIL_FOLDING-NEXT:    [[TMP34:%.*]] = zext nneg i32 [[IDX2]] to i64
+; SCALAR_TAIL_FOLDING-NEXT:    [[ARRAY3IDX2:%.*]] = getelementptr inbounds nuw i8, ptr [[Q]], i64 [[TMP34]]
+; SCALAR_TAIL_FOLDING-NEXT:    store i8 [[SPEC_SELECT_I2]], ptr [[ARRAY3IDX2]], align 1
+; SCALAR_TAIL_FOLDING-NEXT:    [[TMP35:%.*]] = zext nneg i32 [[IDX3]] to i64
+; SCALAR_TAIL_FOLDING-NEXT:    [[ARRAY3IDX3:%.*]] = getelementptr inbounds nuw i8, ptr [[Q]], i64 [[TMP35]]
+; SCALAR_TAIL_FOLDING-NEXT:    store i8 [[SUB2]], ptr [[ARRAY3IDX3]], align 1
+; SCALAR_TAIL_FOLDING-NEXT:    br label [[FOR_INC]]
+; SCALAR_TAIL_FOLDING:       for.inc:
+; SCALAR_TAIL_FOLDING-NEXT:    [[INC]] = add nuw nsw i32 [[IX_024]], 1
+; SCALAR_TAIL_FOLDING-NEXT:    [[EXITCOND:%.*]] = icmp eq i32 [[INC]], 1024
+; SCALAR_TAIL_FOLDING-NEXT:    br i1 [[EXITCOND]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
+; SCALAR_TAIL_FOLDING:       for.end:
+; SCALAR_TAIL_FOLDING-NEXT:    ret void
+;
+; PREDICATED_TAIL_FOLDING-LABEL: define dso_local void @masked_strided_factor4
+; PREDICATED_TAIL_FOLDING-SAME: (ptr noalias readonly captures(none) [[P:%.*]], ptr noalias captures(none) [[Q:%.*]], i8 zeroext [[GUARD:%.*]]) local_unnamed_addr #[[ATTR0]] {
+; PREDICATED_TAIL_FOLDING-NEXT:  entry:
+; PREDICATED_TAIL_FOLDING-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; PREDICATED_TAIL_FOLDING:       vector.ph:
+; PREDICATED_TAIL_FOLDING-NEXT:    [[CONV:%.*]] = zext i8 [[GUARD]] to i32
+; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP0:%.*]] = call i32 @llvm.vscale.i32()
+; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP1:%.*]] = shl i32 [[TMP0]], 4
+; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP2:%.*]] = call i32 @llvm.vscale.i32()
+; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP3:%.*]] = shl i32 [[TMP2]], 4
+; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP4:%.*]] = call i32 @llvm.usub.sat.i32(i32 1024, i32 [[TMP3]])
+; PREDICATED_TAIL_FOLDING-NEXT:    [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i32(i32 0, i32 1024)
+; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP5:%.*]] = call <vscale x 16 x i32> @llvm.stepvector.nxv16i32()
+; PREDICATED_TAIL_FOLDING-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 16 x i32> poison, i32 [[TMP1]], i64 0
+; PREDICATED_TAIL_FOLDING-NEXT:    [[DOTSPLAT:%.*]] = shufflevector <vscale x 16 x i32> [[DOTSPLATINSERT]], <vscale x 16 x i32> poison, <vscale x 16 x i32> zeroinitializer
+; PREDICATED_TAIL_FOLDING-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 16 x i32> poison, i32 [[CONV]], i64 0
+; PREDICATED_TAIL_FOLDING-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 16 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 16 x i32> poison, <vscale x 16 x i32> zeroinitializer
+; PREDICATED_TAIL_FOLDING-NEXT:    br label [[VECTOR_BODY:%.*]]
+; PREDICATED_TAIL_FOLDING:       vector.body:
+; PREDICATED_TAIL_FOLDING-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; PREDICATED_TAIL_FOLDING-NEXT:    [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 16 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], [[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], [[VECTOR_BODY]] ]
+; PREDICATED_TAIL_FOLDING-NEXT:    [[VEC_IND:%.*]] = phi <vscale x 16 x i32> [ [[TMP5]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
+; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP6:%.*]] = icmp ugt <vscale x 16 x i32> [[VEC_IND]], [[BROADCAST_SPLAT]]
+; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP7:%.*]] = select <vscale x 16 x i1> [[ACTIVE_LANE_MASK]], <vscale x 16 x i1> [[TMP6]], <vscale x 16 x i1> zeroinitializer
+; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP8:%.*]] = shl i32 [[INDEX]], 2
+; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP9:%.*]] = sext i32 [[TMP8]] to i64
+; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP10:%.*]] = getelementptr i8, ptr [[P]], i64 [[TMP9]]
+; PREDICATED_TAIL_FOLDING-NEXT:    [[INTERLEAVED_MASK:%.*]] = call <vscale x 32 x i1> @llvm.vector.interleave2.nxv32i1(<vscale x 16 x i1> [[TMP7]], <vscale x 16 x i1> [[TMP7]])
+; PREDICATED_TAIL_FOLDING-NEXT:    [[INTERLEAVED_MASK1:%.*]] = call <vscale x 32 x i1> @llvm.vector.interleave2.nxv32i1(<vscale x 16 x i1> [[TMP7]], <vscale x 16 x i1> [[TMP7]])
+; PREDICATED_TAIL_FOLDING-NEXT:    [[INTERLEAVED_MASK2:%.*]] = call <vscale x 64 x i1> @llvm.vector.interleave2.nxv64i1(<vscale x 32 x i1> [[INTERLEAVED_MASK]], <vscale x 32 x i1> [[INTERLEAVED_MASK1]])
+; PREDICATED_TAIL_FOLDING-NEXT:    [[WIDE_MASKED_VEC:%.*]] = call <vscale x 64 x i8> @llvm.masked.load.nxv64i8.p0(ptr [[TMP10]], i32 1, <vscale x 64 x i1> [[INTERLEAVED_MASK2]], <vscale x 64 x i8> poison)
+; PREDICATED_TAIL_FOLDING-NEXT:    [[STRIDED_VEC:%.*]] = call { <vscale x 32 x i8>, <vscale x 32 x i8> } @llvm.vector.deinterleave2.nxv64i8(<vscale x 64 x i8> [[WIDE_MASKED_VEC]])
+; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP11:%.*]] = extractvalue { <vscale x 32 x i8>, <vscale x 32 x i8> } [[STRIDED_VEC]], 0
+; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP12:%.*]] = extractvalue { <vscale x 32 x i8>, <vscale x 32 x i8> } [[STRIDED_VEC]], 1
+; PREDICATED_TAIL_FOLDING-NEXT:    [[STRIDED_VEC3:%.*]] = call { <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.vector.deinterleave2.nxv32i8(<vscale x 32 x i8> [[TMP11]])
+; PREDICATED_TAIL_FOLDING-NEXT:    [[STRIDED_VEC4:%.*]] = call { <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.vector.deinterleave2.nxv32i8(<vscale x 32 x i8> [[TMP12]])
+; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP13:%.*]] = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8> } [[STRIDED_VEC3]], 0
+; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP14:%.*]] = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8> } [[STRIDED_VEC4]], 0
+; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP15:%.*]] = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8> } [[STRIDED_VEC3]], 1
+; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP16:%.*]] = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8> } [[STRIDED_VEC4]], 1
+; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP17:%.*]] = call <vscale x 16 x i8> @llvm.smax.nxv16i8(<vscale x 16 x i8> [[TMP13]], <vscale x 16 x i8> [[TMP14]])
+; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP18:%.*]] = sub <vscale x 16 x i8> zeroinitializer, [[TMP17]]
+; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP19:%.*]] = call <vscale x 16 x i8> @llvm.smax.nxv16i8(<vscale x 16 x i8> [[TMP15]], <vscale x 16 x i8> [[TMP16]])
+; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP20:%.*]] = sub <vscale x 16 x i8> zeroinitializer, [[TMP19]]
+; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP21:%.*]] = sext i32 [[TMP8]] to i64
+; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP22:%.*]] = getelementptr i8, ptr [[Q]], i64 [[TMP21]]
+; PREDICATED_TAIL_FOLDING-NEXT:    [[INTERLEAVED_VEC:%.*]] = call <vscale x 32 x i8> @llvm.vector.interleave2.nxv32i8(<vscale x 16 x i8> [[TMP17]], <vscale x 16 x i8> [[TMP19]])
+; PREDICATED_TAIL_FOLDING-NEXT:    [[INTERLEAVED_VEC5:%.*]] = call <vscale x 32 x i8> @llvm.vector.interleave2.nxv32i8(<vscale x 16 x i8> [[TMP18]], <vscale x 16 x i8> [[TMP20]])
+; PREDICATED_TAIL_FOLDING-NEXT:    [[INTERLEAVED_VEC6:%.*]] = call <vscale x 64 x i8> @llvm.vector.interleave2.nxv64i8(<vscale x 32 x i8> [[INTERLEAVED_VEC]], <vscale x 32 x i8> [[INTERLEAVED_VEC5]])
+; PREDICATED_TAIL_FOLDING-NEXT:    [[INTERLEAVED_MASK7:%.*]] = call <vscale x 32 x i1> @llvm.vector.interleave2.nxv32i1(<vscale x 16 x i1> [[TMP7]], <vscale x 16 x i1> [[TMP7]])
+; PREDICATED_TAIL_FOLDING-NEXT:    [[INTERLEAVED_MASK8:%.*]] = call <vscale x 32 x i1> @llvm.vector.interleave2.nxv32i1(<vscale x 16 x i1> [[TMP7]], <vscale x 16 x i1> [[TMP7]])
+; PREDICATED_TAIL_FOLDING-NEXT:    [[INTERLEAVED_MASK9:%.*]] = call <vscale x 64 x i1> @llvm.vector.interleave2.nxv64i1(<vscale x 32 x i1> [[INTERLEAVED_MASK7]], <vscale x 32 x i1> [[INTERLEAVED_MASK8]])
+; PREDICATED_TAIL_FOLDING-NEXT:    call void @llvm.masked.store.nxv64i8.p0(<vscale x 64 x i8> [[INTERLEAVED_VEC6]], ptr [[TMP22]], i32 1, <vscale x 64 x i1> [[INTERLEAVED_MASK9]])
+; PREDICATED_TAIL_FOLDING-NEXT:    [[INDEX_NEXT]] = add i32 [[INDEX]], [[TMP1]]
+; PREDICATED_TAIL_FOLDING-NEXT:    [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i32(i32 [[INDEX]], i32 [[TMP4]])
+; PREDICATED_TAIL_FOLDING-NEXT:    [[VEC_IND_NEXT]] = add <vscale x 16 x i32> [[VEC_IND]], [[DOTSPLAT]]
+; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP23:%.*]] = extractelement <vscale x 16 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
+; PREDICATED_TAIL_FOLDING-NEXT:    br i1 [[TMP23]], label [[VECTOR_BODY]], label [[MIDDLE_BLOCK:%.*]], !llvm.loop [[LOOP8:![0-9]+]]
+; PREDICATED_TAIL_FOLDING:       middle.block:
+; PREDICATED_TAIL_FOLDING-NEXT:    br i1 true, label [[FOR_END:%.*]], label [[SCALAR_PH]]
+; PREDICATED_TAIL_FOLDING:       scalar.ph:
+; PREDICATED_TAIL_FOLDING-NEXT:    br label [[FOR_BODY:%.*]]
+; PREDICATED_TAIL_FOLDING:       for.body:
+; PREDICATED_TAIL_FOLDING-NEXT:    br i1 poison, label [[IF_THEN:%.*]], label [[FOR_INC:%.*]]
+; PREDICATED_TAIL_FOLDING:       if.then:
+; PREDICATED_TAIL_FOLDING-NEXT:    br label [[FOR_INC]]
+; PREDICATED_TAIL_FOLDING:       for.inc:
+; PREDICATED_TAIL_FOLDING-NEXT:    br i1 poison, label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
+; PREDICATED_TAIL_FOLDING:       for.end:
+; PREDICATED_TAIL_FOLDING-NEXT:    ret void
+;
+entry:
+  %conv = zext i8 %guard to i32
+  br label %for.body
+
+for.body:
+  %ix.024 = phi i32 [ 0, %entry ], [ %inc, %for.inc ]
+  %cmp1 = icmp ugt i32 %ix.024, %conv
+  br i1 %cmp1, label %if.then, label %for.inc
+
+if.then:
+  %idx0 = shl nuw nsw i32 %ix.024, 2
+  %idx1 = add i32 %idx0, 1
+  %idx2 = add i32 %idx0, 2
+  %idx3 = add i32 %idx0, 3
+
+  %array1idx0 = getelementptr inbounds i8, ptr %p, i32 %idx0
+  %0 = load i8, ptr %array1idx0, align 1
+  %array1idx1 = getelementptr inbounds i8, ptr %p, i32 %idx1
+  %1 = load i8, ptr %array1idx1, align 1
+  %array1idx2 = getelementptr inbounds i8, ptr %p, i32 %idx2
+  %2 = load i8, ptr %array1idx2, align 1
+  %array1idx3 = getelementptr inbounds i8, ptr %p, i32 %idx3
+  %3 = load i8, ptr %array1idx3, align 1
+
+  %cmp.i1 = icmp slt i8 %0, %1
+  %spec.select.i1 = select i1 %cmp.i1, i8 %1, i8 %0
+  %sub1 = sub i8 0, %spec.select.i1
+  %cmp.i2 = icmp slt i8 %2, %3
+  %spec.select.i2 = select i1 %cmp.i2, i8 %3, i8 %2
+  %sub2 = sub i8 0, %spec.select.i2
+
+  %array3idx0 = getelementptr inbounds i8, ptr %q, i32 %idx0
+  store i8 %spec.select.i1, ptr %array3idx0, align 1
+  %array3idx1 = getelementptr inbounds i8, ptr %q, i32 %idx1
+  store i8 %sub1, ptr %array3idx1, align 1
+  %array3idx2 = getelementptr inbounds i8, ptr %q, i32 %idx2
+  store i8 %spec.select.i2, ptr %array3idx2, align 1
+  %array3idx3 = getelementptr inbounds i8, ptr %q, i32 %idx3
+  store i8 %sub2, ptr %array3idx3, align 1
+
+  br label %for.inc
+
+for.inc:
+  %inc = add nuw nsw i32 %ix.024, 1
+  %exitcond = icmp eq i32 %inc, 1024
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-multiexit.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-multiexit.ll
index c20be943a2ccf..6e01a5232d1a1 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-multiexit.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-multiexit.ll
@@ -59,7 +59,7 @@ define void @multiple_exits_unique_exit_block(ptr %A, ptr %B, i32 %N) #0 {
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    br label [[SCALAR_PH]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[VECTOR_MEMCHECK]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ], [ 0, [[VECTOR_MEMCHECK]] ]
 ; CHECK-NEXT:    br label [[LOOP_HEADER:%.*]]
 ; CHECK:       loop.header:
 ; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY:%.*]] ]
@@ -150,7 +150,7 @@ define i32 @multiple_exits_multiple_exit_blocks(ptr %A, ptr %B, i32 %N) #0 {
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    br label [[SCALAR_PH]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[VECTOR_MEMCHECK]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ], [ 0, [[VECTOR_MEMCHECK]] ]
 ; CHECK-NEXT:    br label [[LOOP_HEADER:%.*]]
 ; CHECK:       loop.header:
 ; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY:%.*]] ]
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-runtime-check-size-based-threshold.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-runtime-check-size-based-threshold.ll
index 1373266497c61..feb27caf305a2 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-runtime-check-size-based-threshold.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-runtime-check-size-based-threshold.ll
@@ -90,7 +90,7 @@ define void @min_trip_count_due_to_runtime_checks_1(ptr %dst.1, ptr %dst.2, ptr
 ; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[UMAX]], [[N_VEC]]
 ; CHECK-NEXT:    br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[VECTOR_MEMCHECK]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ], [ 0, [[VECTOR_MEMCHECK]] ]
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
 ; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-vector-reverse.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-vector-reverse.ll
index d55ef855604bd..9567123ab8ae3 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-vector-reverse.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-vector-reverse.ll
@@ -153,7 +153,7 @@ define void @vector_reverse_i64(i64 %N, ptr %a, ptr %b) #0 {
 ; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N_MOD_VF]], 0
 ; CHECK-NEXT:    br i1 [[CMP_N]], label [[FOR_COND_CLEANUP_LOOPEXIT:%.*]], label [[SCALAR_PH]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_MOD_VF]], [[MIDDLE_BLOCK]] ], [ [[N]], [[VECTOR_MEMCHECK]] ], [ [[N]], [[FOR_BODY_PREHEADER]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_MOD_VF]], [[MIDDLE_BLOCK]] ], [ [[N]], [[FOR_BODY_PREHEADER]] ], [ [[N]], [[VECTOR_MEMCHECK]] ]
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK:       for.cond.cleanup.loopexit:
 ; CHECK-NEXT:    br label [[FOR_COND_CLEANUP]]
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt-epilogue.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt-epilogue.ll
index c74ceecf7cfe6..84fc963833cf2 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt-epilogue.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt-epilogue.ll
@@ -66,7 +66,7 @@ define void @simple_histogram(ptr noalias %buckets, ptr readonly %indices, i64 %
 ; CHECK-NEXT:    [[CMP_N7:%.*]] = icmp eq i64 [[N_MOD_VF2]], 0
 ; CHECK-NEXT:    br i1 [[CMP_N7]], label [[FOR_EXIT]], label [[SCALAR_PH]]
 ; CHECK:       vec.epilog.scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC3]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ 0, [[ITER_CHECK:%.*]] ], [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC3]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[ITER_CHECK:%.*]] ]
 ; CHECK-NEXT:    br label [[FOR_BODY1:%.*]]
 ; CHECK:       for.body:
 ; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY1]] ]
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt-outerloop-scevaddrec.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt-outerloop-scevaddrec.ll
index 02bbfa839e690..f118de1128a11 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt-outerloop-scevaddrec.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt-outerloop-scevaddrec.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -mattr=+sve2 -passes=loop-vectorize,instcombine -enable-histogram-loop-vectorization -sve-gather-overhead=2 -sve-scatter-overhead=2 -debug-only=loop-vectorize -S 2>&1 | FileCheck %s
+; RUN: opt < %s -mattr=+sve2 -passes=loop-vectorize,instcombine -enable-histogram-loop-vectorization -sve-gather-overhead=2 -sve-scatter-overhead=2 -debug-only=loop-vectorize --disable-output -S 2>&1 | FileCheck %s
 ; REQUIRES: asserts
 
 target triple = "aarch64-unknown-linux-gnu"
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt-too-many-deps.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt-too-many-deps.ll
index d4c144ebe5dfb..c430e72cea703 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt-too-many-deps.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt-too-many-deps.ll
@@ -99,7 +99,7 @@ define void @many_deps(ptr noalias %buckets, ptr %array, ptr %indices, ptr %othe
 ; NORMAL_DEP_LIMIT-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N_MOD_VF]], 0
 ; NORMAL_DEP_LIMIT-NEXT:    br i1 [[CMP_N]], label [[FOR_EXIT:%.*]], label [[SCALAR_PH]]
 ; NORMAL_DEP_LIMIT:       scalar.ph:
-; NORMAL_DEP_LIMIT-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[VECTOR_MEMCHECK]] ], [ 0, [[ENTRY1:%.*]] ]
+; NORMAL_DEP_LIMIT-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY1:%.*]] ], [ 0, [[VECTOR_MEMCHECK]] ]
 ; NORMAL_DEP_LIMIT-NEXT:    br label [[FOR_BODY1:%.*]]
 ; NORMAL_DEP_LIMIT:       for.body:
 ; NORMAL_DEP_LIMIT-NEXT:    [[IV1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT1:%.*]], [[FOR_BODY1]] ]
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt-vplan.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt-vplan.ll
index c119248c0be43..55f82fd55daf4 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt-vplan.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt-vplan.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -mattr=+sve2 -passes=loop-vectorize,instcombine -enable-histogram-loop-vectorization -sve-gather-overhead=2 -sve-scatter-overhead=2 -force-vector-interleave=1 -debug-only=loop-vectorize -S 2>&1 | FileCheck %s
+; RUN: opt < %s -mattr=+sve2 -passes=loop-vectorize,instcombine -enable-histogram-loop-vectorization -sve-gather-overhead=2 -sve-scatter-overhead=2 -force-vector-interleave=1 -debug-only=loop-vectorize --disable-output -S 2>&1 | FileCheck %s
 ; REQUIRES: asserts
 
 target triple = "aarch64-unknown-linux-gnu"
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt.ll
index 30ee30b595fcb..3b00312959d8a 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt.ll
@@ -779,7 +779,7 @@ define void @simple_histogram_rtdepcheck(ptr noalias %buckets, ptr %array, ptr %
 ; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
 ; CHECK-NEXT:    br i1 [[CMP_N]], label [[FOR_EXIT:%.*]], label [[SCALAR_PH]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[VECTOR_MEMCHECK]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ], [ 0, [[VECTOR_MEMCHECK]] ]
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK:       for.body:
 ; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory.ll b/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory.ll
new file mode 100644
index 0000000000000..3fca274a3bb12
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory.ll
@@ -0,0 +1,205 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --version 5
+; RUN: opt -p loop-vectorize -S %s | FileCheck --check-prefixes=CHECK %s
+
+target datalayout = "e-m:o-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-n32:64-S128-Fn32"
+target triple = "arm64-apple-macosx15.0.0"
+
+define void @test_complex_add_float(ptr %res, ptr noalias %A, ptr noalias %B, i64 %N) {
+; CHECK-LABEL: define void @test_complex_add_float(
+; CHECK-SAME: ptr [[RES:%.*]], ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 8
+; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK:       [[VECTOR_PH]]:
+; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], 8
+; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK:       [[VECTOR_BODY]]:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[IV:%.*]] = add i64 [[INDEX]], 0
+; CHECK-NEXT:    [[TMP1:%.*]] = add i64 [[INDEX]], 4
+; CHECK-NEXT:    [[GEP_A_0:%.*]] = getelementptr inbounds nuw { float, float }, ptr [[A]], i64 [[IV]]
+; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds nuw { float, float }, ptr [[A]], i64 [[TMP1]]
+; CHECK-NEXT:    [[GEP_B_0:%.*]] = getelementptr inbounds nuw { float, float }, ptr [[B]], i64 [[IV]]
+; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr inbounds nuw { float, float }, ptr [[B]], i64 [[TMP1]]
+; CHECK-NEXT:    [[WIDE_VEC:%.*]] = load <8 x float>, ptr [[GEP_A_0]], align 4
+; CHECK-NEXT:    [[STRIDED_VEC:%.*]] = shufflevector <8 x float> [[WIDE_VEC]], <8 x float> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+; CHECK-NEXT:    [[STRIDED_VEC1:%.*]] = shufflevector <8 x float> [[WIDE_VEC]], <8 x float> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+; CHECK-NEXT:    [[WIDE_VEC2:%.*]] = load <8 x float>, ptr [[TMP3]], align 4
+; CHECK-NEXT:    [[STRIDED_VEC3:%.*]] = shufflevector <8 x float> [[WIDE_VEC2]], <8 x float> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+; CHECK-NEXT:    [[STRIDED_VEC4:%.*]] = shufflevector <8 x float> [[WIDE_VEC2]], <8 x float> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+; CHECK-NEXT:    [[WIDE_VEC5:%.*]] = load <8 x float>, ptr [[GEP_B_0]], align 4
+; CHECK-NEXT:    [[STRIDED_VEC6:%.*]] = shufflevector <8 x float> [[WIDE_VEC5]], <8 x float> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+; CHECK-NEXT:    [[STRIDED_VEC7:%.*]] = shufflevector <8 x float> [[WIDE_VEC5]], <8 x float> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+; CHECK-NEXT:    [[WIDE_VEC8:%.*]] = load <8 x float>, ptr [[TMP5]], align 4
+; CHECK-NEXT:    [[STRIDED_VEC9:%.*]] = shufflevector <8 x float> [[WIDE_VEC8]], <8 x float> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+; CHECK-NEXT:    [[STRIDED_VEC10:%.*]] = shufflevector <8 x float> [[WIDE_VEC8]], <8 x float> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+; CHECK-NEXT:    [[TMP6:%.*]] = fadd <4 x float> [[STRIDED_VEC]], [[STRIDED_VEC6]]
+; CHECK-NEXT:    [[TMP7:%.*]] = fadd <4 x float> [[STRIDED_VEC3]], [[STRIDED_VEC9]]
+; CHECK-NEXT:    [[TMP8:%.*]] = fadd <4 x float> [[STRIDED_VEC1]], [[STRIDED_VEC7]]
+; CHECK-NEXT:    [[TMP9:%.*]] = fadd <4 x float> [[STRIDED_VEC4]], [[STRIDED_VEC10]]
+; CHECK-NEXT:    [[TMP10:%.*]] = getelementptr inbounds nuw { float, float }, ptr [[RES]], i64 [[IV]]
+; CHECK-NEXT:    [[TMP11:%.*]] = getelementptr inbounds nuw { float, float }, ptr [[RES]], i64 [[TMP1]]
+; CHECK-NEXT:    [[TMP12:%.*]] = shufflevector <4 x float> [[TMP6]], <4 x float> [[TMP8]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <8 x float> [[TMP12]], <8 x float> poison, <8 x i32> <i32 0, i32 4, i32 1, i32 5, i32 2, i32 6, i32 3, i32 7>
+; CHECK-NEXT:    store <8 x float> [[INTERLEAVED_VEC]], ptr [[TMP10]], align 4
+; CHECK-NEXT:    [[TMP13:%.*]] = shufflevector <4 x float> [[TMP7]], <4 x float> [[TMP9]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:    [[INTERLEAVED_VEC11:%.*]] = shufflevector <8 x float> [[TMP13]], <8 x float> poison, <8 x i32> <i32 0, i32 4, i32 1, i32 5, i32 2, i32 6, i32 3, i32 7>
+; CHECK-NEXT:    store <8 x float> [[INTERLEAVED_VEC11]], ptr [[TMP11]], align 4
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
+; CHECK-NEXT:    [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[TMP14]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK:       [[MIDDLE_BLOCK]]:
+; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK:       [[SCALAR_PH]]:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT:    br label %[[LOOP:.*]]
+; CHECK:       [[LOOP]]:
+; CHECK-NEXT:    [[IV1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[GEP_A_2:%.*]] = getelementptr inbounds nuw { float, float }, ptr [[A]], i64 [[IV1]]
+; CHECK-NEXT:    [[GEP_B_2:%.*]] = getelementptr inbounds nuw { float, float }, ptr [[B]], i64 [[IV1]]
+; CHECK-NEXT:    [[L_A_0:%.*]] = load float, ptr [[GEP_A_2]], align 4
+; CHECK-NEXT:    [[GEP_A_1:%.*]] = getelementptr inbounds nuw i8, ptr [[GEP_A_2]], i64 4
+; CHECK-NEXT:    [[L_A_1:%.*]] = load float, ptr [[GEP_A_1]], align 4
+; CHECK-NEXT:    [[L_B_0:%.*]] = load float, ptr [[GEP_B_2]], align 4
+; CHECK-NEXT:    [[ADD_0:%.*]] = fadd float [[L_A_0]], [[L_B_0]]
+; CHECK-NEXT:    [[GEP_B_1:%.*]] = getelementptr inbounds nuw i8, ptr [[GEP_B_2]], i64 4
+; CHECK-NEXT:    [[L_B_1:%.*]] = load float, ptr [[GEP_B_1]], align 4
+; CHECK-NEXT:    [[ADD_1:%.*]] = fadd float [[L_A_1]], [[L_B_1]]
+; CHECK-NEXT:    [[GEP_RES_0:%.*]] = getelementptr inbounds nuw { float, float }, ptr [[RES]], i64 [[IV1]]
+; CHECK-NEXT:    store float [[ADD_0]], ptr [[GEP_RES_0]], align 4
+; CHECK-NEXT:    [[GEP_RES_1:%.*]] = getelementptr inbounds nuw i8, ptr [[GEP_RES_0]], i64 4
+; CHECK-NEXT:    store float [[ADD_1]], ptr [[GEP_RES_1]], align 4
+; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV1]], 1
+; CHECK-NEXT:    [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+  %gep.A.0 = getelementptr inbounds nuw { float, float }, ptr %A, i64 %iv
+  %gep.B.0 = getelementptr inbounds nuw { float, float }, ptr %B, i64 %iv
+  %l.A.0 = load float, ptr %gep.A.0, align 4
+  %gep.A.1 = getelementptr inbounds nuw i8, ptr %gep.A.0, i64 4
+  %l.A.1 = load float, ptr %gep.A.1, align 4
+  %l.B.0 = load float, ptr %gep.B.0, align 4
+  %add.0 = fadd float %l.A.0, %l.B.0
+  %gep.B.1 = getelementptr inbounds nuw i8, ptr %gep.B.0, i64 4
+  %l.B.1 = load float, ptr %gep.B.1, align 4
+  %add.1 = fadd float %l.A.1, %l.B.1
+  %gep.res.0 = getelementptr inbounds nuw { float, float }, ptr %res, i64 %iv
+  store float %add.0, ptr %gep.res.0, align 4
+  %gep.res.1 = getelementptr inbounds nuw i8, ptr %gep.res.0, i64 4
+  store float %add.1, ptr %gep.res.1, align 4
+  %iv.next = add nuw nsw i64 %iv, 1
+  %ec = icmp eq i64 %iv.next, %N
+  br i1 %ec, label %exit, label %loop
+
+exit:
+  ret void
+}
+
+define void @test_complex_add_double(ptr %res, ptr noalias %A, ptr noalias %B, i64 %N) {
+; CHECK-LABEL: define void @test_complex_add_double(
+; CHECK-SAME: ptr [[RES:%.*]], ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
+; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK:       [[VECTOR_PH]]:
+; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], 4
+; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK:       [[VECTOR_BODY]]:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP0:%.*]] = add i64 [[INDEX]], 0
+; CHECK-NEXT:    [[TMP1:%.*]] = add i64 [[INDEX]], 2
+; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds nuw { double, double }, ptr [[A]], i64 [[TMP0]]
+; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds nuw { double, double }, ptr [[A]], i64 [[TMP1]]
+; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr inbounds nuw { double, double }, ptr [[B]], i64 [[TMP0]]
+; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr inbounds nuw { double, double }, ptr [[B]], i64 [[TMP1]]
+; CHECK-NEXT:    [[WIDE_VEC:%.*]] = load <4 x double>, ptr [[TMP2]], align 4
+; CHECK-NEXT:    [[STRIDED_VEC:%.*]] = shufflevector <4 x double> [[WIDE_VEC]], <4 x double> poison, <2 x i32> <i32 0, i32 2>
+; CHECK-NEXT:    [[STRIDED_VEC1:%.*]] = shufflevector <4 x double> [[WIDE_VEC]], <4 x double> poison, <2 x i32> <i32 1, i32 3>
+; CHECK-NEXT:    [[WIDE_VEC2:%.*]] = load <4 x double>, ptr [[TMP3]], align 4
+; CHECK-NEXT:    [[STRIDED_VEC3:%.*]] = shufflevector <4 x double> [[WIDE_VEC2]], <4 x double> poison, <2 x i32> <i32 0, i32 2>
+; CHECK-NEXT:    [[STRIDED_VEC4:%.*]] = shufflevector <4 x double> [[WIDE_VEC2]], <4 x double> poison, <2 x i32> <i32 1, i32 3>
+; CHECK-NEXT:    [[WIDE_VEC5:%.*]] = load <4 x double>, ptr [[TMP4]], align 4
+; CHECK-NEXT:    [[STRIDED_VEC6:%.*]] = shufflevector <4 x double> [[WIDE_VEC5]], <4 x double> poison, <2 x i32> <i32 0, i32 2>
+; CHECK-NEXT:    [[STRIDED_VEC7:%.*]] = shufflevector <4 x double> [[WIDE_VEC5]], <4 x double> poison, <2 x i32> <i32 1, i32 3>
+; CHECK-NEXT:    [[WIDE_VEC8:%.*]] = load <4 x double>, ptr [[TMP5]], align 4
+; CHECK-NEXT:    [[STRIDED_VEC9:%.*]] = shufflevector <4 x double> [[WIDE_VEC8]], <4 x double> poison, <2 x i32> <i32 0, i32 2>
+; CHECK-NEXT:    [[STRIDED_VEC10:%.*]] = shufflevector <4 x double> [[WIDE_VEC8]], <4 x double> poison, <2 x i32> <i32 1, i32 3>
+; CHECK-NEXT:    [[TMP6:%.*]] = fadd <2 x double> [[STRIDED_VEC]], [[STRIDED_VEC6]]
+; CHECK-NEXT:    [[TMP7:%.*]] = fadd <2 x double> [[STRIDED_VEC3]], [[STRIDED_VEC9]]
+; CHECK-NEXT:    [[TMP8:%.*]] = fadd <2 x double> [[STRIDED_VEC1]], [[STRIDED_VEC7]]
+; CHECK-NEXT:    [[TMP9:%.*]] = fadd <2 x double> [[STRIDED_VEC4]], [[STRIDED_VEC10]]
+; CHECK-NEXT:    [[TMP10:%.*]] = getelementptr inbounds nuw { double, double }, ptr [[RES]], i64 [[TMP0]]
+; CHECK-NEXT:    [[TMP11:%.*]] = getelementptr inbounds nuw { double, double }, ptr [[RES]], i64 [[TMP1]]
+; CHECK-NEXT:    [[TMP12:%.*]] = shufflevector <2 x double> [[TMP6]], <2 x double> [[TMP8]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <4 x double> [[TMP12]], <4 x double> poison, <4 x i32> <i32 0, i32 2, i32 1, i32 3>
+; CHECK-NEXT:    store <4 x double> [[INTERLEAVED_VEC]], ptr [[TMP10]], align 4
+; CHECK-NEXT:    [[TMP13:%.*]] = shufflevector <2 x double> [[TMP7]], <2 x double> [[TMP9]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[INTERLEAVED_VEC11:%.*]] = shufflevector <4 x double> [[TMP13]], <4 x double> poison, <4 x i32> <i32 0, i32 2, i32 1, i32 3>
+; CHECK-NEXT:    store <4 x double> [[INTERLEAVED_VEC11]], ptr [[TMP11]], align 4
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT:    [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[TMP14]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK:       [[MIDDLE_BLOCK]]:
+; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK:       [[SCALAR_PH]]:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT:    br label %[[LOOP:.*]]
+; CHECK:       [[LOOP]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[GEP_A_0:%.*]] = getelementptr inbounds nuw { double, double }, ptr [[A]], i64 [[IV]]
+; CHECK-NEXT:    [[GEP_B_0:%.*]] = getelementptr inbounds nuw { double, double }, ptr [[B]], i64 [[IV]]
+; CHECK-NEXT:    [[L_A_0:%.*]] = load double, ptr [[GEP_A_0]], align 4
+; CHECK-NEXT:    [[GEP_A_1:%.*]] = getelementptr inbounds nuw i8, ptr [[GEP_A_0]], i64 8
+; CHECK-NEXT:    [[L_A_1:%.*]] = load double, ptr [[GEP_A_1]], align 4
+; CHECK-NEXT:    [[L_B_0:%.*]] = load double, ptr [[GEP_B_0]], align 4
+; CHECK-NEXT:    [[ADD_0:%.*]] = fadd double [[L_A_0]], [[L_B_0]]
+; CHECK-NEXT:    [[GEP_B_1:%.*]] = getelementptr inbounds nuw i8, ptr [[GEP_B_0]], i64 8
+; CHECK-NEXT:    [[L_B_1:%.*]] = load double, ptr [[GEP_B_1]], align 4
+; CHECK-NEXT:    [[ADD_1:%.*]] = fadd double [[L_A_1]], [[L_B_1]]
+; CHECK-NEXT:    [[GEP_RES_0:%.*]] = getelementptr inbounds nuw { double, double }, ptr [[RES]], i64 [[IV]]
+; CHECK-NEXT:    store double [[ADD_0]], ptr [[GEP_RES_0]], align 4
+; CHECK-NEXT:    [[GEP_RES_1:%.*]] = getelementptr inbounds nuw i8, ptr [[GEP_RES_0]], i64 8
+; CHECK-NEXT:    store double [[ADD_1]], ptr [[GEP_RES_1]], align 4
+; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT:    [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP5:![0-9]+]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+  %gep.A.0 = getelementptr inbounds nuw { double, double }, ptr %A, i64 %iv
+  %gep.B.0 = getelementptr inbounds nuw { double, double }, ptr %B, i64 %iv
+  %l.A.0 = load double, ptr %gep.A.0, align 4
+  %gep.A.1 = getelementptr inbounds nuw i8, ptr %gep.A.0, i64 8
+  %l.A.1 = load double, ptr %gep.A.1, align 4
+  %l.B.0 = load double, ptr %gep.B.0, align 4
+  %add.0 = fadd double %l.A.0, %l.B.0
+  %gep.B.1 = getelementptr inbounds nuw i8, ptr %gep.B.0, i64 8
+  %l.B.1 = load double, ptr %gep.B.1, align 4
+  %add.1 = fadd double %l.A.1, %l.B.1
+  %gep.res.0 = getelementptr inbounds nuw { double, double }, ptr %res, i64 %iv
+  store double %add.0, ptr %gep.res.0, align 4
+  %gep.res.1 = getelementptr inbounds nuw i8, ptr %gep.res.0, i64 8
+  store double %add.1, ptr %gep.res.1, align 4
+  %iv.next = add nuw nsw i64 %iv, 1
+  %ec = icmp eq i64 %iv.next, %N
+  br i1 %ec, label %exit, label %loop
+
+exit:
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/type-shrinkage-zext-costs.ll b/llvm/test/Transforms/LoopVectorize/AArch64/type-shrinkage-zext-costs.ll
index 0bc3ea94a37e3..7bc606f5c61b3 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/type-shrinkage-zext-costs.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/type-shrinkage-zext-costs.ll
@@ -18,7 +18,7 @@ define void @zext_i8_i16(ptr noalias nocapture readonly %p, ptr noalias nocaptur
 ; CHECK-COST: Cost of 0 for VF vscale x 8: WIDEN-CAST ir<%conv> = zext ir<%0> to i16
 ; CHECK-COST: LV: Found an estimated cost of 0 for VF 1 For instruction:   %conv = zext i8 %0 to i32
 ; CHECK-LABEL: define void @zext_i8_i16
-; CHECK-SAME: (ptr noalias nocapture readonly [[P:%.*]], ptr noalias nocapture [[Q:%.*]], i32 [[LEN:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-SAME: (ptr noalias readonly captures(none) [[P:%.*]], ptr noalias captures(none) [[Q:%.*]], i32 [[LEN:%.*]]) #[[ATTR0:[0-9]+]] {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[TMP0:%.*]] = add i32 [[LEN]], -1
 ; CHECK-NEXT:    [[TMP1:%.*]] = zext i32 [[TMP0]] to i64
@@ -94,7 +94,7 @@ define void @sext_i8_i16(ptr noalias nocapture readonly %p, ptr noalias nocaptur
 ; CHECK-COST: Cost of 1 for VF vscale x 4: WIDEN-CAST ir<%conv> = sext ir<%0> to i16
 ; CHECK-COST: Cost of 0 for VF vscale x 8: WIDEN-CAST ir<%conv> = sext ir<%0> to i16
 ; CHECK-LABEL: define void @sext_i8_i16
-; CHECK-SAME: (ptr noalias nocapture readonly [[P:%.*]], ptr noalias nocapture [[Q:%.*]], i32 [[LEN:%.*]]) #[[ATTR0]] {
+; CHECK-SAME: (ptr noalias readonly captures(none) [[P:%.*]], ptr noalias captures(none) [[Q:%.*]], i32 [[LEN:%.*]]) #[[ATTR0]] {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[TMP0:%.*]] = add i32 [[LEN]], -1
 ; CHECK-NEXT:    [[TMP1:%.*]] = zext i32 [[TMP0]] to i64
diff --git a/llvm/test/Transforms/LoopVectorize/ARM/arm-ieee-vectorize.ll b/llvm/test/Transforms/LoopVectorize/ARM/arm-ieee-vectorize.ll
index aee0aa4b271f0..44a48a9c262f5 100644
--- a/llvm/test/Transforms/LoopVectorize/ARM/arm-ieee-vectorize.ll
+++ b/llvm/test/Transforms/LoopVectorize/ARM/arm-ieee-vectorize.ll
@@ -1,7 +1,7 @@
-; RUN: opt -mtriple armv7-linux-gnueabihf -passes=loop-vectorize -S %s -debug-only=loop-vectorize -o /dev/null 2>&1 | FileCheck %s --check-prefix=CHECK --check-prefix=LINUX
-; RUN: opt -mtriple armv8-linux-gnu -passes=loop-vectorize -S %s -debug-only=loop-vectorize -o /dev/null 2>&1 | FileCheck %s --check-prefix=CHECK --check-prefix=LINUX
-; RUN: opt -mtriple armv8.1.m-none-eabi -mattr=+mve.fp -passes=loop-vectorize -S %s -debug-only=loop-vectorize -o /dev/null 2>&1 | FileCheck %s --check-prefix=CHECK --check-prefix=MVE
-; RUN: opt -mtriple armv7-unknwon-darwin -passes=loop-vectorize -S %s -debug-only=loop-vectorize -o /dev/null 2>&1 | FileCheck %s --check-prefix=CHECK --check-prefix=DARWIN
+; RUN: opt -mtriple armv7-linux-gnueabihf -passes=loop-vectorize -S %s -debug-only=loop-vectorize --disable-output -o /dev/null 2>&1 | FileCheck %s --check-prefix=CHECK --check-prefix=LINUX
+; RUN: opt -mtriple armv8-linux-gnu -passes=loop-vectorize -S %s -debug-only=loop-vectorize --disable-output -o /dev/null 2>&1 | FileCheck %s --check-prefix=CHECK --check-prefix=LINUX
+; RUN: opt -mtriple armv8.1.m-none-eabi -mattr=+mve.fp -passes=loop-vectorize -S %s -debug-only=loop-vectorize --disable-output -o /dev/null 2>&1 | FileCheck %s --check-prefix=CHECK --check-prefix=MVE
+; RUN: opt -mtriple armv7-unknwon-darwin -passes=loop-vectorize -S %s -debug-only=loop-vectorize --disable-output -o /dev/null 2>&1 | FileCheck %s --check-prefix=CHECK --check-prefix=DARWIN
 ; REQUIRES: asserts
 
 target datalayout = "e-m:e-p:32:32-Fi8-i64:64-v128:64:128-a:0:32-n32-S64"
diff --git a/llvm/test/Transforms/LoopVectorize/ARM/mve-gather-scatter-tailpred.ll b/llvm/test/Transforms/LoopVectorize/ARM/mve-gather-scatter-tailpred.ll
index b629dfd4fce9f..fda9ef2cf6c2f 100644
--- a/llvm/test/Transforms/LoopVectorize/ARM/mve-gather-scatter-tailpred.ll
+++ b/llvm/test/Transforms/LoopVectorize/ARM/mve-gather-scatter-tailpred.ll
@@ -353,7 +353,7 @@ define void @test_stride_loopinvar_4i32(ptr readonly %data, ptr noalias nocaptur
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    br i1 true, label [[END:%.*]], label [[SCALAR_PH]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[VECTOR_SCEVCHECK]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ], [ 0, [[VECTOR_SCEVCHECK]] ]
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK:       for.body:
 ; CHECK-NEXT:    [[I_023:%.*]] = phi i32 [ [[INC:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
diff --git a/llvm/test/Transforms/LoopVectorize/ARM/mve-hoist-runtime-checks.ll b/llvm/test/Transforms/LoopVectorize/ARM/mve-hoist-runtime-checks.ll
index 845cd08cbae19..6e8bef26d3e83 100644
--- a/llvm/test/Transforms/LoopVectorize/ARM/mve-hoist-runtime-checks.ll
+++ b/llvm/test/Transforms/LoopVectorize/ARM/mve-hoist-runtime-checks.ll
@@ -32,7 +32,7 @@ target triple = "thumbv8.1m.main-none-unknown-eabi"
 
 define void @diff_checks(ptr nocapture noundef writeonly %dst, ptr nocapture noundef readonly %src, i32 noundef %m, i32 noundef %n) #0 {
 ; CHECK-LABEL: define void @diff_checks
-; CHECK-SAME: (ptr nocapture noundef writeonly [[DST:%.*]], ptr nocapture noundef readonly [[SRC:%.*]], i32 noundef [[M:%.*]], i32 noundef [[N:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-SAME: (ptr noundef writeonly captures(none) [[DST:%.*]], ptr noundef readonly captures(none) [[SRC:%.*]], i32 noundef [[M:%.*]], i32 noundef [[N:%.*]]) #[[ATTR0:[0-9]+]] {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[ADD5:%.*]] = add nsw i32 [[N]], 1
 ; CHECK-NEXT:    [[TMP0:%.*]] = add i32 [[M]], -1
@@ -72,17 +72,17 @@ define void @diff_checks(ptr nocapture noundef writeonly %dst, ptr nocapture nou
 ; CHECK-NEXT:    [[ACTIVE_LANE_MASK:%.*]] = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i32(i32 [[TMP11]], i32 [[N]])
 ; CHECK-NEXT:    [[TMP12:%.*]] = getelementptr i32, ptr [[TMP7]], i32 [[TMP11]]
 ; CHECK-NEXT:    [[TMP13:%.*]] = getelementptr i32, ptr [[TMP12]], i32 0
-; CHECK-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr [[TMP13]], i32 4, <4 x i1> [[ACTIVE_LANE_MASK]], <4 x i32> poison), !alias.scope !0
+; CHECK-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr [[TMP13]], i32 4, <4 x i1> [[ACTIVE_LANE_MASK]], <4 x i32> poison), !alias.scope [[META0:![0-9]+]]
 ; CHECK-NEXT:    [[TMP14:%.*]] = getelementptr i32, ptr [[TMP8]], i32 [[TMP11]]
 ; CHECK-NEXT:    [[TMP15:%.*]] = getelementptr i32, ptr [[TMP14]], i32 0
-; CHECK-NEXT:    call void @llvm.masked.store.v4i32.p0(<4 x i32> [[WIDE_MASKED_LOAD]], ptr [[TMP15]], i32 4, <4 x i1> [[ACTIVE_LANE_MASK]]), !alias.scope !3, !noalias !0
+; CHECK-NEXT:    call void @llvm.masked.store.v4i32.p0(<4 x i32> [[WIDE_MASKED_LOAD]], ptr [[TMP15]], i32 4, <4 x i1> [[ACTIVE_LANE_MASK]]), !alias.scope [[META3:![0-9]+]], !noalias [[META0]]
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add i32 [[INDEX]], 4
 ; CHECK-NEXT:    [[TMP16:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-NEXT:    br i1 [[TMP16]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    br i1 true, label [[INNER_LOOP_EXIT]], label [[SCALAR_PH]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[VECTOR_MEMCHECK]] ], [ 0, [[OUTER_LOOP]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[OUTER_LOOP]] ], [ 0, [[VECTOR_MEMCHECK]] ]
 ; CHECK-NEXT:    br label [[INNER_LOOP:%.*]]
 ; CHECK:       inner.loop:
 ; CHECK-NEXT:    [[J_021_US:%.*]] = phi i32 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INC_US:%.*]], [[INNER_LOOP]] ]
diff --git a/llvm/test/Transforms/LoopVectorize/ARM/mve-multiexit.ll b/llvm/test/Transforms/LoopVectorize/ARM/mve-multiexit.ll
index cc2fbb1b0df79..789a97c052a96 100644
--- a/llvm/test/Transforms/LoopVectorize/ARM/mve-multiexit.ll
+++ b/llvm/test/Transforms/LoopVectorize/ARM/mve-multiexit.ll
@@ -42,7 +42,7 @@ define void @multiple_exits_unique_exit_block(ptr %A, ptr %B, i32 %N) #0 {
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    br label [[SCALAR_PH]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[VECTOR_MEMCHECK]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ], [ 0, [[VECTOR_MEMCHECK]] ]
 ; CHECK-NEXT:    br label [[LOOP_HEADER:%.*]]
 ; CHECK:       loop.header:
 ; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY:%.*]] ]
@@ -116,7 +116,7 @@ define i32 @multiple_exits_multiple_exit_blocks(ptr %A, ptr %B, i32 %N) #0 {
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    br label [[SCALAR_PH]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[VECTOR_MEMCHECK]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ], [ 0, [[VECTOR_MEMCHECK]] ]
 ; CHECK-NEXT:    br label [[LOOP_HEADER:%.*]]
 ; CHECK:       loop.header:
 ; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY:%.*]] ]
diff --git a/llvm/test/Transforms/LoopVectorize/LoongArch/defaults.ll b/llvm/test/Transforms/LoopVectorize/LoongArch/defaults.ll
index 236a156fb58a1..feb2b4f2c64a7 100644
--- a/llvm/test/Transforms/LoopVectorize/LoongArch/defaults.ll
+++ b/llvm/test/Transforms/LoopVectorize/LoongArch/defaults.ll
@@ -12,7 +12,7 @@ target triple = "loongarch64"
 
 define void @vector_add(ptr noalias nocapture %a, i64 %v) {
 ; CHECK-LABEL: define void @vector_add(
-; CHECK-SAME: ptr noalias nocapture [[A:%.*]], i64 [[V:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-SAME: ptr noalias captures(none) [[A:%.*]], i64 [[V:%.*]]) #[[ATTR0:[0-9]+]] {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.ph:
@@ -67,3 +67,9 @@ for.body:
 for.end:
   ret void
 }
+;.
+; CHECK: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]], [[META2:![0-9]+]]}
+; CHECK: [[META1]] = !{!"llvm.loop.isvectorized", i32 1}
+; CHECK: [[META2]] = !{!"llvm.loop.unroll.runtime.disable"}
+; CHECK: [[LOOP3]] = distinct !{[[LOOP3]], [[META2]], [[META1]]}
+;.
diff --git a/llvm/test/Transforms/LoopVectorize/PowerPC/exit-branch-cost.ll b/llvm/test/Transforms/LoopVectorize/PowerPC/exit-branch-cost.ll
index 2f1af7951dbc2..e5717c4f1d91a 100644
--- a/llvm/test/Transforms/LoopVectorize/PowerPC/exit-branch-cost.ll
+++ b/llvm/test/Transforms/LoopVectorize/PowerPC/exit-branch-cost.ll
@@ -144,9 +144,9 @@ define i1 @select_exit_cond(ptr %start, ptr %end, i64 %N) {
 ; CHECK-NEXT:    [[CMP_N33:%.*]] = icmp eq i64 [[TMP2]], [[N_VEC25]]
 ; CHECK-NEXT:    br i1 [[CMP_N33]], label %[[EXIT]], label %[[VEC_EPILOG_SCALAR_PH]]
 ; CHECK:       [[VEC_EPILOG_SCALAR_PH]]:
-; CHECK-NEXT:    [[BC_RESUME_VAL34:%.*]] = phi i64 [ [[N_VEC25]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ 0, %[[ITER_CHECK]] ], [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ]
-; CHECK-NEXT:    [[BC_MERGE_RDX35:%.*]] = phi i64 [ [[TMP55]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ 0, %[[ITER_CHECK]] ], [ [[TMP52]], %[[VEC_EPILOG_ITER_CHECK]] ]
-; CHECK-NEXT:    [[BC_RESUME_VAL36:%.*]] = phi ptr [ [[TMP56]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[START]], %[[ITER_CHECK]] ], [ [[IND_END]], %[[VEC_EPILOG_ITER_CHECK]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL34:%.*]] = phi i64 [ [[N_VEC25]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ]
+; CHECK-NEXT:    [[BC_MERGE_RDX35:%.*]] = phi i64 [ [[TMP55]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP52]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL36:%.*]] = phi ptr [ [[TMP56]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[IND_END]], %[[VEC_EPILOG_ITER_CHECK]] ], [ [[START]], %[[ITER_CHECK]] ]
 ; CHECK-NEXT:    br label %[[LOOP:.*]]
 ; CHECK:       [[LOOP]]:
 ; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL34]], %[[VEC_EPILOG_SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
diff --git a/llvm/test/Transforms/LoopVectorize/PowerPC/optimal-epilog-vectorization.ll b/llvm/test/Transforms/LoopVectorize/PowerPC/optimal-epilog-vectorization.ll
index 32d62befe9fc5..ba9d49fc682c4 100644
--- a/llvm/test/Transforms/LoopVectorize/PowerPC/optimal-epilog-vectorization.ll
+++ b/llvm/test/Transforms/LoopVectorize/PowerPC/optimal-epilog-vectorization.ll
@@ -120,7 +120,7 @@ define dso_local void @f1(ptr noalias %aa, ptr noalias %bb, ptr noalias %cc, i32
 ; VF-TWO-CHECK-NEXT:    [[CMP_N19:%.*]] = icmp eq i64 [[WIDE_TRIP_COUNT]], [[N_VEC18]]
 ; VF-TWO-CHECK-NEXT:    br i1 [[CMP_N19]], label [[FOR_END_LOOPEXIT]], label [[VEC_EPILOG_SCALAR_PH]]
 ; VF-TWO-CHECK:       vec.epilog.scalar.ph:
-; VF-TWO-CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC18]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ 0, [[ITER_CHECK]] ], [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ]
+; VF-TWO-CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC18]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[ITER_CHECK]] ]
 ; VF-TWO-CHECK-NEXT:    br label [[FOR_BODY:%.*]]
 ; VF-TWO-CHECK:       for.body:
 ; VF-TWO-CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[VEC_EPILOG_SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ]
@@ -252,7 +252,7 @@ define dso_local void @f1(ptr noalias %aa, ptr noalias %bb, ptr noalias %cc, i32
 ; VF-FOUR-CHECK-NEXT:    [[CMP_N19:%.*]] = icmp eq i64 [[WIDE_TRIP_COUNT]], [[N_VEC18]]
 ; VF-FOUR-CHECK-NEXT:    br i1 [[CMP_N19]], label [[FOR_END_LOOPEXIT]], label [[VEC_EPILOG_SCALAR_PH]]
 ; VF-FOUR-CHECK:       vec.epilog.scalar.ph:
-; VF-FOUR-CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC18]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ 0, [[ITER_CHECK]] ], [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ]
+; VF-FOUR-CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC18]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[ITER_CHECK]] ]
 ; VF-FOUR-CHECK-NEXT:    br label [[FOR_BODY:%.*]]
 ; VF-FOUR-CHECK:       for.body:
 ; VF-FOUR-CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[VEC_EPILOG_SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ]
@@ -438,8 +438,8 @@ define dso_local signext i32 @f2(ptr noalias %A, ptr noalias %B, i32 signext %n)
 ; VF-TWO-CHECK-NEXT:    [[CMP_N20:%.*]] = icmp eq i64 [[WIDE_TRIP_COUNT]], [[N_VEC17]]
 ; VF-TWO-CHECK-NEXT:    br i1 [[CMP_N20]], label [[FOR_END_LOOPEXIT]], label [[VEC_EPILOG_SCALAR_PH]]
 ; VF-TWO-CHECK:       vec.epilog.scalar.ph:
-; VF-TWO-CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC17]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ 0, [[VECTOR_SCEVCHECK]] ], [ 0, [[ITER_CHECK]] ], [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ]
-; VF-TWO-CHECK-NEXT:    [[BC_RESUME_VAL19:%.*]] = phi i32 [ [[IND_END]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ 0, [[VECTOR_SCEVCHECK]] ], [ 0, [[ITER_CHECK]] ], [ [[IND_END18]], [[VEC_EPILOG_ITER_CHECK]] ]
+; VF-TWO-CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC17]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[VECTOR_SCEVCHECK]] ], [ 0, [[ITER_CHECK]] ]
+; VF-TWO-CHECK-NEXT:    [[BC_RESUME_VAL19:%.*]] = phi i32 [ [[IND_END]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[IND_END18]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[VECTOR_SCEVCHECK]] ], [ 0, [[ITER_CHECK]] ]
 ; VF-TWO-CHECK-NEXT:    br label [[FOR_BODY:%.*]]
 ; VF-TWO-CHECK:       for.body:
 ; VF-TWO-CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[VEC_EPILOG_SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ]
@@ -597,8 +597,8 @@ define dso_local signext i32 @f2(ptr noalias %A, ptr noalias %B, i32 signext %n)
 ; VF-FOUR-CHECK-NEXT:    [[CMP_N20:%.*]] = icmp eq i64 [[WIDE_TRIP_COUNT]], [[N_VEC17]]
 ; VF-FOUR-CHECK-NEXT:    br i1 [[CMP_N20]], label [[FOR_END_LOOPEXIT]], label [[VEC_EPILOG_SCALAR_PH]]
 ; VF-FOUR-CHECK:       vec.epilog.scalar.ph:
-; VF-FOUR-CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC17]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ 0, [[VECTOR_SCEVCHECK]] ], [ 0, [[ITER_CHECK]] ], [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ]
-; VF-FOUR-CHECK-NEXT:    [[BC_RESUME_VAL19:%.*]] = phi i32 [ [[IND_END]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ 0, [[VECTOR_SCEVCHECK]] ], [ 0, [[ITER_CHECK]] ], [ [[IND_END18]], [[VEC_EPILOG_ITER_CHECK]] ]
+; VF-FOUR-CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC17]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[VECTOR_SCEVCHECK]] ], [ 0, [[ITER_CHECK]] ]
+; VF-FOUR-CHECK-NEXT:    [[BC_RESUME_VAL19:%.*]] = phi i32 [ [[IND_END]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[IND_END18]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[VECTOR_SCEVCHECK]] ], [ 0, [[ITER_CHECK]] ]
 ; VF-FOUR-CHECK-NEXT:    br label [[FOR_BODY:%.*]]
 ; VF-FOUR-CHECK:       for.body:
 ; VF-FOUR-CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[VEC_EPILOG_SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ]
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/bf16.ll b/llvm/test/Transforms/LoopVectorize/RISCV/bf16.ll
index 27923f82411d0..fb278c7c17c85 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/bf16.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/bf16.ll
@@ -91,10 +91,10 @@ define void @vfwmaccbf16.vv(ptr noalias %a, ptr noalias %b, ptr noalias %c, i64
 ; NO-ZVFBFMIN-LABEL: define void @vfwmaccbf16.vv(
 ; NO-ZVFBFMIN-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], ptr noalias [[C:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
 ; NO-ZVFBFMIN-NEXT:  [[ENTRY:.*]]:
-; NO-ZVFBFMIN-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 8
+; NO-ZVFBFMIN-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
 ; NO-ZVFBFMIN-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; NO-ZVFBFMIN:       [[VECTOR_PH]]:
-; NO-ZVFBFMIN-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], 8
+; NO-ZVFBFMIN-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], 4
 ; NO-ZVFBFMIN-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
 ; NO-ZVFBFMIN-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; NO-ZVFBFMIN:       [[VECTOR_BODY]]:
@@ -104,16 +104,16 @@ define void @vfwmaccbf16.vv(ptr noalias %a, ptr noalias %b, ptr noalias %c, i64
 ; NO-ZVFBFMIN-NEXT:    [[TMP2:%.*]] = getelementptr bfloat, ptr [[B]], i64 [[TMP0]]
 ; NO-ZVFBFMIN-NEXT:    [[TMP3:%.*]] = getelementptr float, ptr [[C]], i64 [[TMP0]]
 ; NO-ZVFBFMIN-NEXT:    [[TMP4:%.*]] = getelementptr bfloat, ptr [[TMP1]], i32 0
-; NO-ZVFBFMIN-NEXT:    [[WIDE_LOAD:%.*]] = load <8 x bfloat>, ptr [[TMP4]], align 2
+; NO-ZVFBFMIN-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x bfloat>, ptr [[TMP4]], align 2
 ; NO-ZVFBFMIN-NEXT:    [[TMP5:%.*]] = getelementptr bfloat, ptr [[TMP2]], i32 0
-; NO-ZVFBFMIN-NEXT:    [[WIDE_LOAD1:%.*]] = load <8 x bfloat>, ptr [[TMP5]], align 2
+; NO-ZVFBFMIN-NEXT:    [[WIDE_LOAD1:%.*]] = load <4 x bfloat>, ptr [[TMP5]], align 2
 ; NO-ZVFBFMIN-NEXT:    [[TMP6:%.*]] = getelementptr float, ptr [[TMP3]], i32 0
-; NO-ZVFBFMIN-NEXT:    [[WIDE_LOAD2:%.*]] = load <8 x float>, ptr [[TMP6]], align 4
-; NO-ZVFBFMIN-NEXT:    [[TMP7:%.*]] = fpext <8 x bfloat> [[WIDE_LOAD]] to <8 x float>
-; NO-ZVFBFMIN-NEXT:    [[TMP8:%.*]] = fpext <8 x bfloat> [[WIDE_LOAD1]] to <8 x float>
-; NO-ZVFBFMIN-NEXT:    [[TMP9:%.*]] = call <8 x float> @llvm.fmuladd.v8f32(<8 x float> [[TMP7]], <8 x float> [[TMP8]], <8 x float> [[WIDE_LOAD2]])
-; NO-ZVFBFMIN-NEXT:    store <8 x float> [[TMP9]], ptr [[TMP6]], align 4
-; NO-ZVFBFMIN-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
+; NO-ZVFBFMIN-NEXT:    [[WIDE_LOAD2:%.*]] = load <4 x float>, ptr [[TMP6]], align 4
+; NO-ZVFBFMIN-NEXT:    [[TMP7:%.*]] = fpext <4 x bfloat> [[WIDE_LOAD]] to <4 x float>
+; NO-ZVFBFMIN-NEXT:    [[TMP8:%.*]] = fpext <4 x bfloat> [[WIDE_LOAD1]] to <4 x float>
+; NO-ZVFBFMIN-NEXT:    [[TMP9:%.*]] = call <4 x float> @llvm.fmuladd.v4f32(<4 x float> [[TMP7]], <4 x float> [[TMP8]], <4 x float> [[WIDE_LOAD2]])
+; NO-ZVFBFMIN-NEXT:    store <4 x float> [[TMP9]], ptr [[TMP6]], align 4
+; NO-ZVFBFMIN-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
 ; NO-ZVFBFMIN-NEXT:    [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; NO-ZVFBFMIN-NEXT:    br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; NO-ZVFBFMIN:       [[MIDDLE_BLOCK]]:
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/blocks-with-dead-instructions.ll b/llvm/test/Transforms/LoopVectorize/RISCV/blocks-with-dead-instructions.ll
index f2318d6057eec..50dfb689c2060 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/blocks-with-dead-instructions.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/blocks-with-dead-instructions.ll
@@ -874,7 +874,7 @@ define void @dead_load_in_block(ptr %dst, ptr %src, i8 %N, i64 %x) #0 {
 ; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[TMP3]], [[N_VEC]]
 ; CHECK-NEXT:    br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
 ; CHECK:       [[SCALAR_PH]]:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[IND_END]], %[[MIDDLE_BLOCK]] ], [ 0, %[[VECTOR_MEMCHECK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[IND_END]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_MEMCHECK]] ]
 ; CHECK-NEXT:    br label %[[LOOP_HEADER:.*]]
 ; CHECK:       [[LOOP_HEADER]]:
 ; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/dead-ops-cost.ll b/llvm/test/Transforms/LoopVectorize/RISCV/dead-ops-cost.ll
index 450405f193465..3839b367ae08c 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/dead-ops-cost.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/dead-ops-cost.ll
@@ -134,7 +134,7 @@ define i8 @dead_live_out_due_to_scalar_epilogue_required(ptr %src, ptr %dst) {
 ; CHECK:       [[MIDDLE_BLOCK]]:
 ; CHECK-NEXT:    br label %[[SCALAR_PH]]
 ; CHECK:       [[SCALAR_PH]]:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ [[IND_END]], %[[MIDDLE_BLOCK]] ], [ 0, %[[VECTOR_MEMCHECK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ [[IND_END]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_MEMCHECK]] ]
 ; CHECK-NEXT:    br label %[[LOOP:.*]]
 ; CHECK:       [[LOOP]]:
 ; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
@@ -209,50 +209,42 @@ define i32 @cost_of_exit_branch_and_cond_insts(ptr %a, ptr %b, i1 %c, i16 %x) #0
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE18:.*]] ]
 ; CHECK-NEXT:    [[TMP10:%.*]] = add i32 [[INDEX]], 0
 ; CHECK-NEXT:    [[TMP11:%.*]] = getelementptr i32, ptr [[B]], i32 [[TMP10]]
-; CHECK-NEXT:    [[TMP12:%.*]] = extractelement <8 x i1> [[BROADCAST_SPLAT]], i32 0
-; CHECK-NEXT:    br i1 [[TMP12]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
+; CHECK-NEXT:    br i1 [[C]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
 ; CHECK:       [[PRED_STORE_IF]]:
 ; CHECK-NEXT:    store i1 false, ptr [[A]], align 1, !alias.scope [[META11:![0-9]+]], !noalias [[META14:![0-9]+]]
 ; CHECK-NEXT:    br label %[[PRED_STORE_CONTINUE]]
 ; CHECK:       [[PRED_STORE_CONTINUE]]:
-; CHECK-NEXT:    [[TMP13:%.*]] = extractelement <8 x i1> [[BROADCAST_SPLAT]], i32 1
-; CHECK-NEXT:    br i1 [[TMP13]], label %[[PRED_STORE_IF5:.*]], label %[[PRED_STORE_CONTINUE6:.*]]
+; CHECK-NEXT:    br i1 [[C]], label %[[PRED_STORE_IF5:.*]], label %[[PRED_STORE_CONTINUE6:.*]]
 ; CHECK:       [[PRED_STORE_IF5]]:
 ; CHECK-NEXT:    store i1 false, ptr [[A]], align 1, !alias.scope [[META11]], !noalias [[META14]]
 ; CHECK-NEXT:    br label %[[PRED_STORE_CONTINUE6]]
 ; CHECK:       [[PRED_STORE_CONTINUE6]]:
-; CHECK-NEXT:    [[TMP14:%.*]] = extractelement <8 x i1> [[BROADCAST_SPLAT]], i32 2
-; CHECK-NEXT:    br i1 [[TMP14]], label %[[PRED_STORE_IF7:.*]], label %[[PRED_STORE_CONTINUE8:.*]]
+; CHECK-NEXT:    br i1 [[C]], label %[[PRED_STORE_IF7:.*]], label %[[PRED_STORE_CONTINUE8:.*]]
 ; CHECK:       [[PRED_STORE_IF7]]:
 ; CHECK-NEXT:    store i1 false, ptr [[A]], align 1, !alias.scope [[META11]], !noalias [[META14]]
 ; CHECK-NEXT:    br label %[[PRED_STORE_CONTINUE8]]
 ; CHECK:       [[PRED_STORE_CONTINUE8]]:
-; CHECK-NEXT:    [[TMP15:%.*]] = extractelement <8 x i1> [[BROADCAST_SPLAT]], i32 3
-; CHECK-NEXT:    br i1 [[TMP15]], label %[[PRED_STORE_IF9:.*]], label %[[PRED_STORE_CONTINUE10:.*]]
+; CHECK-NEXT:    br i1 [[C]], label %[[PRED_STORE_IF9:.*]], label %[[PRED_STORE_CONTINUE10:.*]]
 ; CHECK:       [[PRED_STORE_IF9]]:
 ; CHECK-NEXT:    store i1 false, ptr [[A]], align 1, !alias.scope [[META11]], !noalias [[META14]]
 ; CHECK-NEXT:    br label %[[PRED_STORE_CONTINUE10]]
 ; CHECK:       [[PRED_STORE_CONTINUE10]]:
-; CHECK-NEXT:    [[TMP16:%.*]] = extractelement <8 x i1> [[BROADCAST_SPLAT]], i32 4
-; CHECK-NEXT:    br i1 [[TMP16]], label %[[PRED_STORE_IF11:.*]], label %[[PRED_STORE_CONTINUE12:.*]]
+; CHECK-NEXT:    br i1 [[C]], label %[[PRED_STORE_IF11:.*]], label %[[PRED_STORE_CONTINUE12:.*]]
 ; CHECK:       [[PRED_STORE_IF11]]:
 ; CHECK-NEXT:    store i1 false, ptr [[A]], align 1, !alias.scope [[META11]], !noalias [[META14]]
 ; CHECK-NEXT:    br label %[[PRED_STORE_CONTINUE12]]
 ; CHECK:       [[PRED_STORE_CONTINUE12]]:
-; CHECK-NEXT:    [[TMP17:%.*]] = extractelement <8 x i1> [[BROADCAST_SPLAT]], i32 5
-; CHECK-NEXT:    br i1 [[TMP17]], label %[[PRED_STORE_IF13:.*]], label %[[PRED_STORE_CONTINUE14:.*]]
+; CHECK-NEXT:    br i1 [[C]], label %[[PRED_STORE_IF13:.*]], label %[[PRED_STORE_CONTINUE14:.*]]
 ; CHECK:       [[PRED_STORE_IF13]]:
 ; CHECK-NEXT:    store i1 false, ptr [[A]], align 1, !alias.scope [[META11]], !noalias [[META14]]
 ; CHECK-NEXT:    br label %[[PRED_STORE_CONTINUE14]]
 ; CHECK:       [[PRED_STORE_CONTINUE14]]:
-; CHECK-NEXT:    [[TMP18:%.*]] = extractelement <8 x i1> [[BROADCAST_SPLAT]], i32 6
-; CHECK-NEXT:    br i1 [[TMP18]], label %[[PRED_STORE_IF15:.*]], label %[[PRED_STORE_CONTINUE16:.*]]
+; CHECK-NEXT:    br i1 [[C]], label %[[PRED_STORE_IF15:.*]], label %[[PRED_STORE_CONTINUE16:.*]]
 ; CHECK:       [[PRED_STORE_IF15]]:
 ; CHECK-NEXT:    store i1 false, ptr [[A]], align 1, !alias.scope [[META11]], !noalias [[META14]]
 ; CHECK-NEXT:    br label %[[PRED_STORE_CONTINUE16]]
 ; CHECK:       [[PRED_STORE_CONTINUE16]]:
-; CHECK-NEXT:    [[TMP19:%.*]] = extractelement <8 x i1> [[BROADCAST_SPLAT]], i32 7
-; CHECK-NEXT:    br i1 [[TMP19]], label %[[PRED_STORE_IF17:.*]], label %[[PRED_STORE_CONTINUE18]]
+; CHECK-NEXT:    br i1 [[C]], label %[[PRED_STORE_IF17:.*]], label %[[PRED_STORE_CONTINUE18]]
 ; CHECK:       [[PRED_STORE_IF17]]:
 ; CHECK-NEXT:    store i1 false, ptr [[A]], align 1, !alias.scope [[META11]], !noalias [[META14]]
 ; CHECK-NEXT:    br label %[[PRED_STORE_CONTINUE18]]
@@ -265,7 +257,7 @@ define i32 @cost_of_exit_branch_and_cond_insts(ptr %a, ptr %b, i1 %c, i16 %x) #0
 ; CHECK:       [[MIDDLE_BLOCK]]:
 ; CHECK-NEXT:    br label %[[SCALAR_PH]]
 ; CHECK:       [[SCALAR_PH]]:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[VECTOR_MEMCHECK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_MEMCHECK]] ]
 ; CHECK-NEXT:    br label %[[LOOP_HEADER:.*]]
 ; CHECK:       [[LOOP_HEADER]]:
 ; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/force-vect-msg.ll b/llvm/test/Transforms/LoopVectorize/RISCV/force-vect-msg.ll
index 09a1aaab6cc2d..f68ffd1aea3cd 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/force-vect-msg.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/force-vect-msg.ll
@@ -1,5 +1,5 @@
 ; REQUIRES: asserts
-; RUN: opt < %s -passes=loop-vectorize -mtriple riscv64 -riscv-v-vector-bits-min=128 -mattr="+v" -debug-only=loop-vectorize -S 2>&1 | FileCheck %s
+; RUN: opt < %s -passes=loop-vectorize -mtriple riscv64 -riscv-v-vector-bits-min=128 -mattr="+v" -debug-only=loop-vectorize --disable-output -S 2>&1 | FileCheck %s
 
 ; CHECK: LV: Loop hints: force=enabled
 ; CHECK: LV: Scalar loop costs: 4.
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/induction-costs.ll b/llvm/test/Transforms/LoopVectorize/RISCV/induction-costs.ll
index e4425a9327385..2582882baba00 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/induction-costs.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/induction-costs.ll
@@ -91,8 +91,8 @@ define void @skip_free_iv_truncate(i16 %x, ptr %A) #0 {
 ; CHECK:       [[MIDDLE_BLOCK]]:
 ; CHECK-NEXT:    br label %[[SCALAR_PH]]
 ; CHECK:       [[SCALAR_PH]]:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[IND_END]], %[[MIDDLE_BLOCK]] ], [ [[X_I64]], %[[VECTOR_MEMCHECK]] ], [ [[X_I64]], %[[ENTRY]] ]
-; CHECK-NEXT:    [[BC_RESUME_VAL13:%.*]] = phi i32 [ [[IND_END22]], %[[MIDDLE_BLOCK]] ], [ [[X_I32]], %[[VECTOR_MEMCHECK]] ], [ [[X_I32]], %[[ENTRY]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[IND_END]], %[[MIDDLE_BLOCK]] ], [ [[X_I64]], %[[ENTRY]] ], [ [[X_I64]], %[[VECTOR_MEMCHECK]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL13:%.*]] = phi i32 [ [[IND_END22]], %[[MIDDLE_BLOCK]] ], [ [[X_I32]], %[[ENTRY]] ], [ [[X_I32]], %[[VECTOR_MEMCHECK]] ]
 ; CHECK-NEXT:    br label %[[LOOP:.*]]
 ; CHECK:       [[LOOP]]:
 ; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/interleaved-accesses.ll b/llvm/test/Transforms/LoopVectorize/RISCV/interleaved-accesses.ll
index bda4839dead51..b1ff589fe51bf 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/interleaved-accesses.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/interleaved-accesses.ll
@@ -9,7 +9,7 @@ define void @load_store_factor2_i32(ptr %p) {
 ; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
 ; CHECK-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 4
 ; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP1]]
-; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[ENTRY:%.*]]
 ; CHECK:       vector.ph:
 ; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
 ; CHECK-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 4
@@ -17,88 +17,88 @@ define void @load_store_factor2_i32(ptr %p) {
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
 ; CHECK-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
 ; CHECK-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 4
-; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       vector.body:
-; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[TMP6:%.*]] = add i64 [[INDEX]], 0
-; CHECK-NEXT:    [[TMP7:%.*]] = shl i64 [[TMP6]], 1
-; CHECK-NEXT:    [[TMP8:%.*]] = getelementptr i32, ptr [[P:%.*]], i64 [[TMP7]]
-; CHECK-NEXT:    [[WIDE_VEC:%.*]] = load <vscale x 8 x i32>, ptr [[TMP8]], align 4
+; CHECK-NEXT:    [[I:%.*]] = phi i64 [ 0, [[ENTRY]] ], [ [[NEXTI:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[TMP6:%.*]] = add i64 [[I]], 0
+; CHECK-NEXT:    [[OFFSET0:%.*]] = shl i64 [[TMP6]], 1
+; CHECK-NEXT:    [[Q0:%.*]] = getelementptr i32, ptr [[P:%.*]], i64 [[OFFSET0]]
+; CHECK-NEXT:    [[WIDE_VEC:%.*]] = load <vscale x 8 x i32>, ptr [[Q0]], align 4
 ; CHECK-NEXT:    [[STRIDED_VEC:%.*]] = call { <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.vector.deinterleave2.nxv8i32(<vscale x 8 x i32> [[WIDE_VEC]])
-; CHECK-NEXT:    [[TMP10:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } [[STRIDED_VEC]], 0
-; CHECK-NEXT:    [[TMP11:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } [[STRIDED_VEC]], 1
-; CHECK-NEXT:    [[TMP12:%.*]] = add <vscale x 4 x i32> [[TMP10]], splat (i32 1)
-; CHECK-NEXT:    [[TMP15:%.*]] = add <vscale x 4 x i32> [[TMP11]], splat (i32 2)
-; CHECK-NEXT:    [[INTERLEAVED_VEC:%.*]] = call <vscale x 8 x i32> @llvm.vector.interleave2.nxv8i32(<vscale x 4 x i32> [[TMP12]], <vscale x 4 x i32> [[TMP15]])
-; CHECK-NEXT:    store <vscale x 8 x i32> [[INTERLEAVED_VEC]], ptr [[TMP8]], align 4
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
-; CHECK-NEXT:    [[TMP17:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT:    br i1 [[TMP17]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK-NEXT:    [[TMP9:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } [[STRIDED_VEC]], 0
+; CHECK-NEXT:    [[TMP10:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } [[STRIDED_VEC]], 1
+; CHECK-NEXT:    [[TMP11:%.*]] = add <vscale x 4 x i32> [[TMP9]], splat (i32 1)
+; CHECK-NEXT:    [[TMP12:%.*]] = add <vscale x 4 x i32> [[TMP10]], splat (i32 2)
+; CHECK-NEXT:    [[INTERLEAVED_VEC:%.*]] = call <vscale x 8 x i32> @llvm.vector.interleave2.nxv8i32(<vscale x 4 x i32> [[TMP11]], <vscale x 4 x i32> [[TMP12]])
+; CHECK-NEXT:    store <vscale x 8 x i32> [[INTERLEAVED_VEC]], ptr [[Q0]], align 4
+; CHECK-NEXT:    [[NEXTI]] = add nuw i64 [[I]], [[TMP5]]
+; CHECK-NEXT:    [[TMP13:%.*]] = icmp eq i64 [[NEXTI]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[TMP13]], label [[MIDDLE_BLOCK:%.*]], label [[LOOP]], !llvm.loop [[LOOP0:![0-9]+]]
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 1024, [[N_VEC]]
 ; CHECK-NEXT:    br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
-; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY1:%.*]] ]
+; CHECK-NEXT:    br label [[LOOP1:%.*]]
 ; CHECK:       loop:
-; CHECK-NEXT:    [[I:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[NEXTI:%.*]], [[LOOP]] ]
-; CHECK-NEXT:    [[OFFSET0:%.*]] = shl i64 [[I]], 1
-; CHECK-NEXT:    [[Q0:%.*]] = getelementptr i32, ptr [[P]], i64 [[OFFSET0]]
-; CHECK-NEXT:    [[X0:%.*]] = load i32, ptr [[Q0]], align 4
+; CHECK-NEXT:    [[I1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[NEXTI1:%.*]], [[LOOP1]] ]
+; CHECK-NEXT:    [[OFFSET2:%.*]] = shl i64 [[I1]], 1
+; CHECK-NEXT:    [[Q2:%.*]] = getelementptr i32, ptr [[P]], i64 [[OFFSET2]]
+; CHECK-NEXT:    [[X0:%.*]] = load i32, ptr [[Q2]], align 4
 ; CHECK-NEXT:    [[Y0:%.*]] = add i32 [[X0]], 1
-; CHECK-NEXT:    store i32 [[Y0]], ptr [[Q0]], align 4
-; CHECK-NEXT:    [[OFFSET1:%.*]] = add i64 [[OFFSET0]], 1
+; CHECK-NEXT:    store i32 [[Y0]], ptr [[Q2]], align 4
+; CHECK-NEXT:    [[OFFSET1:%.*]] = add i64 [[OFFSET2]], 1
 ; CHECK-NEXT:    [[Q1:%.*]] = getelementptr i32, ptr [[P]], i64 [[OFFSET1]]
 ; CHECK-NEXT:    [[X1:%.*]] = load i32, ptr [[Q1]], align 4
 ; CHECK-NEXT:    [[Y1:%.*]] = add i32 [[X1]], 2
 ; CHECK-NEXT:    store i32 [[Y1]], ptr [[Q1]], align 4
-; CHECK-NEXT:    [[NEXTI]] = add i64 [[I]], 1
-; CHECK-NEXT:    [[DONE:%.*]] = icmp eq i64 [[NEXTI]], 1024
-; CHECK-NEXT:    br i1 [[DONE]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK-NEXT:    [[NEXTI1]] = add i64 [[I1]], 1
+; CHECK-NEXT:    [[DONE:%.*]] = icmp eq i64 [[NEXTI1]], 1024
+; CHECK-NEXT:    br i1 [[DONE]], label [[EXIT]], label [[LOOP1]], !llvm.loop [[LOOP3:![0-9]+]]
 ; CHECK:       exit:
 ; CHECK-NEXT:    ret void
 ;
 ; FIXED-LABEL: @load_store_factor2_i32(
 ; FIXED-NEXT:  entry:
-; FIXED-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; FIXED-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[ENTRY:%.*]]
 ; FIXED:       vector.ph:
-; FIXED-NEXT:    br label [[VECTOR_BODY:%.*]]
+; FIXED-NEXT:    br label [[LOOP:%.*]]
 ; FIXED:       vector.body:
-; FIXED-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; FIXED-NEXT:    [[TMP0:%.*]] = add i64 [[INDEX]], 0
-; FIXED-NEXT:    [[TMP1:%.*]] = shl i64 [[TMP0]], 1
-; FIXED-NEXT:    [[TMP2:%.*]] = getelementptr i32, ptr [[P:%.*]], i64 [[TMP1]]
-; FIXED-NEXT:    [[WIDE_VEC:%.*]] = load <16 x i32>, ptr [[TMP2]], align 4
+; FIXED-NEXT:    [[I:%.*]] = phi i64 [ 0, [[ENTRY]] ], [ [[NEXTI:%.*]], [[LOOP]] ]
+; FIXED-NEXT:    [[TMP0:%.*]] = add i64 [[I]], 0
+; FIXED-NEXT:    [[OFFSET0:%.*]] = shl i64 [[TMP0]], 1
+; FIXED-NEXT:    [[Q0:%.*]] = getelementptr i32, ptr [[P:%.*]], i64 [[OFFSET0]]
+; FIXED-NEXT:    [[WIDE_VEC:%.*]] = load <16 x i32>, ptr [[Q0]], align 4
 ; FIXED-NEXT:    [[STRIDED_VEC:%.*]] = shufflevector <16 x i32> [[WIDE_VEC]], <16 x i32> poison, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>
 ; FIXED-NEXT:    [[STRIDED_VEC1:%.*]] = shufflevector <16 x i32> [[WIDE_VEC]], <16 x i32> poison, <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15>
-; FIXED-NEXT:    [[TMP4:%.*]] = add <8 x i32> [[STRIDED_VEC]], splat (i32 1)
-; FIXED-NEXT:    [[TMP7:%.*]] = add <8 x i32> [[STRIDED_VEC1]], splat (i32 2)
-; FIXED-NEXT:    [[TMP9:%.*]] = shufflevector <8 x i32> [[TMP4]], <8 x i32> [[TMP7]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; FIXED-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <16 x i32> [[TMP9]], <16 x i32> poison, <16 x i32> <i32 0, i32 8, i32 1, i32 9, i32 2, i32 10, i32 3, i32 11, i32 4, i32 12, i32 5, i32 13, i32 6, i32 14, i32 7, i32 15>
-; FIXED-NEXT:    store <16 x i32> [[INTERLEAVED_VEC]], ptr [[TMP2]], align 4
-; FIXED-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
-; FIXED-NEXT:    [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
-; FIXED-NEXT:    br i1 [[TMP10]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; FIXED-NEXT:    [[TMP3:%.*]] = add <8 x i32> [[STRIDED_VEC]], splat (i32 1)
+; FIXED-NEXT:    [[TMP4:%.*]] = add <8 x i32> [[STRIDED_VEC1]], splat (i32 2)
+; FIXED-NEXT:    [[TMP5:%.*]] = shufflevector <8 x i32> [[TMP3]], <8 x i32> [[TMP4]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; FIXED-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <16 x i32> [[TMP5]], <16 x i32> poison, <16 x i32> <i32 0, i32 8, i32 1, i32 9, i32 2, i32 10, i32 3, i32 11, i32 4, i32 12, i32 5, i32 13, i32 6, i32 14, i32 7, i32 15>
+; FIXED-NEXT:    store <16 x i32> [[INTERLEAVED_VEC]], ptr [[Q0]], align 4
+; FIXED-NEXT:    [[NEXTI]] = add nuw i64 [[I]], 8
+; FIXED-NEXT:    [[TMP6:%.*]] = icmp eq i64 [[NEXTI]], 1024
+; FIXED-NEXT:    br i1 [[TMP6]], label [[MIDDLE_BLOCK:%.*]], label [[LOOP]], !llvm.loop [[LOOP0:![0-9]+]]
 ; FIXED:       middle.block:
 ; FIXED-NEXT:    br i1 true, label [[EXIT:%.*]], label [[SCALAR_PH]]
 ; FIXED:       scalar.ph:
-; FIXED-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 1024, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
-; FIXED-NEXT:    br label [[LOOP:%.*]]
+; FIXED-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 1024, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY1:%.*]] ]
+; FIXED-NEXT:    br label [[LOOP1:%.*]]
 ; FIXED:       loop:
-; FIXED-NEXT:    [[I:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[NEXTI:%.*]], [[LOOP]] ]
-; FIXED-NEXT:    [[OFFSET0:%.*]] = shl i64 [[I]], 1
-; FIXED-NEXT:    [[Q0:%.*]] = getelementptr i32, ptr [[P]], i64 [[OFFSET0]]
-; FIXED-NEXT:    [[X0:%.*]] = load i32, ptr [[Q0]], align 4
+; FIXED-NEXT:    [[I1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[NEXTI1:%.*]], [[LOOP1]] ]
+; FIXED-NEXT:    [[OFFSET2:%.*]] = shl i64 [[I1]], 1
+; FIXED-NEXT:    [[Q2:%.*]] = getelementptr i32, ptr [[P]], i64 [[OFFSET2]]
+; FIXED-NEXT:    [[X0:%.*]] = load i32, ptr [[Q2]], align 4
 ; FIXED-NEXT:    [[Y0:%.*]] = add i32 [[X0]], 1
-; FIXED-NEXT:    store i32 [[Y0]], ptr [[Q0]], align 4
-; FIXED-NEXT:    [[OFFSET1:%.*]] = add i64 [[OFFSET0]], 1
+; FIXED-NEXT:    store i32 [[Y0]], ptr [[Q2]], align 4
+; FIXED-NEXT:    [[OFFSET1:%.*]] = add i64 [[OFFSET2]], 1
 ; FIXED-NEXT:    [[Q1:%.*]] = getelementptr i32, ptr [[P]], i64 [[OFFSET1]]
 ; FIXED-NEXT:    [[X1:%.*]] = load i32, ptr [[Q1]], align 4
 ; FIXED-NEXT:    [[Y1:%.*]] = add i32 [[X1]], 2
 ; FIXED-NEXT:    store i32 [[Y1]], ptr [[Q1]], align 4
-; FIXED-NEXT:    [[NEXTI]] = add i64 [[I]], 1
-; FIXED-NEXT:    [[DONE:%.*]] = icmp eq i64 [[NEXTI]], 1024
-; FIXED-NEXT:    br i1 [[DONE]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP3:![0-9]+]]
+; FIXED-NEXT:    [[NEXTI1]] = add i64 [[I1]], 1
+; FIXED-NEXT:    [[DONE:%.*]] = icmp eq i64 [[NEXTI1]], 1024
+; FIXED-NEXT:    br i1 [[DONE]], label [[EXIT]], label [[LOOP1]], !llvm.loop [[LOOP3:![0-9]+]]
 ; FIXED:       exit:
 ; FIXED-NEXT:    ret void
 ;
@@ -107,7 +107,7 @@ define void @load_store_factor2_i32(ptr %p) {
 ; SCALABLE-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
 ; SCALABLE-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 4
 ; SCALABLE-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP1]]
-; SCALABLE-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; SCALABLE-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[ENTRY:%.*]]
 ; SCALABLE:       vector.ph:
 ; SCALABLE-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
 ; SCALABLE-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 4
@@ -115,44 +115,44 @@ define void @load_store_factor2_i32(ptr %p) {
 ; SCALABLE-NEXT:    [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
 ; SCALABLE-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
 ; SCALABLE-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 4
-; SCALABLE-NEXT:    br label [[VECTOR_BODY:%.*]]
+; SCALABLE-NEXT:    br label [[LOOP:%.*]]
 ; SCALABLE:       vector.body:
-; SCALABLE-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; SCALABLE-NEXT:    [[TMP6:%.*]] = add i64 [[INDEX]], 0
-; SCALABLE-NEXT:    [[TMP7:%.*]] = shl i64 [[TMP6]], 1
-; SCALABLE-NEXT:    [[TMP8:%.*]] = getelementptr i32, ptr [[P:%.*]], i64 [[TMP7]]
-; SCALABLE-NEXT:    [[WIDE_VEC:%.*]] = load <vscale x 8 x i32>, ptr [[TMP8]], align 4
+; SCALABLE-NEXT:    [[I:%.*]] = phi i64 [ 0, [[ENTRY]] ], [ [[NEXTI:%.*]], [[LOOP]] ]
+; SCALABLE-NEXT:    [[TMP6:%.*]] = add i64 [[I]], 0
+; SCALABLE-NEXT:    [[OFFSET0:%.*]] = shl i64 [[TMP6]], 1
+; SCALABLE-NEXT:    [[Q0:%.*]] = getelementptr i32, ptr [[P:%.*]], i64 [[OFFSET0]]
+; SCALABLE-NEXT:    [[WIDE_VEC:%.*]] = load <vscale x 8 x i32>, ptr [[Q0]], align 4
 ; SCALABLE-NEXT:    [[STRIDED_VEC:%.*]] = call { <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.vector.deinterleave2.nxv8i32(<vscale x 8 x i32> [[WIDE_VEC]])
-; SCALABLE-NEXT:    [[TMP10:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } [[STRIDED_VEC]], 0
-; SCALABLE-NEXT:    [[TMP11:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } [[STRIDED_VEC]], 1
-; SCALABLE-NEXT:    [[TMP12:%.*]] = add <vscale x 4 x i32> [[TMP10]], splat (i32 1)
-; SCALABLE-NEXT:    [[TMP15:%.*]] = add <vscale x 4 x i32> [[TMP11]], splat (i32 2)
-; SCALABLE-NEXT:    [[INTERLEAVED_VEC:%.*]] = call <vscale x 8 x i32> @llvm.vector.interleave2.nxv8i32(<vscale x 4 x i32> [[TMP12]], <vscale x 4 x i32> [[TMP15]])
-; SCALABLE-NEXT:    store <vscale x 8 x i32> [[INTERLEAVED_VEC]], ptr [[TMP8]], align 4
-; SCALABLE-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
-; SCALABLE-NEXT:    [[TMP17:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; SCALABLE-NEXT:    br i1 [[TMP17]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; SCALABLE-NEXT:    [[TMP9:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } [[STRIDED_VEC]], 0
+; SCALABLE-NEXT:    [[TMP10:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } [[STRIDED_VEC]], 1
+; SCALABLE-NEXT:    [[TMP11:%.*]] = add <vscale x 4 x i32> [[TMP9]], splat (i32 1)
+; SCALABLE-NEXT:    [[TMP12:%.*]] = add <vscale x 4 x i32> [[TMP10]], splat (i32 2)
+; SCALABLE-NEXT:    [[INTERLEAVED_VEC:%.*]] = call <vscale x 8 x i32> @llvm.vector.interleave2.nxv8i32(<vscale x 4 x i32> [[TMP11]], <vscale x 4 x i32> [[TMP12]])
+; SCALABLE-NEXT:    store <vscale x 8 x i32> [[INTERLEAVED_VEC]], ptr [[Q0]], align 4
+; SCALABLE-NEXT:    [[NEXTI]] = add nuw i64 [[I]], [[TMP5]]
+; SCALABLE-NEXT:    [[TMP13:%.*]] = icmp eq i64 [[NEXTI]], [[N_VEC]]
+; SCALABLE-NEXT:    br i1 [[TMP13]], label [[MIDDLE_BLOCK:%.*]], label [[LOOP]], !llvm.loop [[LOOP0:![0-9]+]]
 ; SCALABLE:       middle.block:
 ; SCALABLE-NEXT:    [[CMP_N:%.*]] = icmp eq i64 1024, [[N_VEC]]
 ; SCALABLE-NEXT:    br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
 ; SCALABLE:       scalar.ph:
-; SCALABLE-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
-; SCALABLE-NEXT:    br label [[LOOP:%.*]]
+; SCALABLE-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY1:%.*]] ]
+; SCALABLE-NEXT:    br label [[LOOP1:%.*]]
 ; SCALABLE:       loop:
-; SCALABLE-NEXT:    [[I:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[NEXTI:%.*]], [[LOOP]] ]
-; SCALABLE-NEXT:    [[OFFSET0:%.*]] = shl i64 [[I]], 1
-; SCALABLE-NEXT:    [[Q0:%.*]] = getelementptr i32, ptr [[P]], i64 [[OFFSET0]]
-; SCALABLE-NEXT:    [[X0:%.*]] = load i32, ptr [[Q0]], align 4
+; SCALABLE-NEXT:    [[I1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[NEXTI1:%.*]], [[LOOP1]] ]
+; SCALABLE-NEXT:    [[OFFSET2:%.*]] = shl i64 [[I1]], 1
+; SCALABLE-NEXT:    [[Q2:%.*]] = getelementptr i32, ptr [[P]], i64 [[OFFSET2]]
+; SCALABLE-NEXT:    [[X0:%.*]] = load i32, ptr [[Q2]], align 4
 ; SCALABLE-NEXT:    [[Y0:%.*]] = add i32 [[X0]], 1
-; SCALABLE-NEXT:    store i32 [[Y0]], ptr [[Q0]], align 4
-; SCALABLE-NEXT:    [[OFFSET1:%.*]] = add i64 [[OFFSET0]], 1
+; SCALABLE-NEXT:    store i32 [[Y0]], ptr [[Q2]], align 4
+; SCALABLE-NEXT:    [[OFFSET1:%.*]] = add i64 [[OFFSET2]], 1
 ; SCALABLE-NEXT:    [[Q1:%.*]] = getelementptr i32, ptr [[P]], i64 [[OFFSET1]]
 ; SCALABLE-NEXT:    [[X1:%.*]] = load i32, ptr [[Q1]], align 4
 ; SCALABLE-NEXT:    [[Y1:%.*]] = add i32 [[X1]], 2
 ; SCALABLE-NEXT:    store i32 [[Y1]], ptr [[Q1]], align 4
-; SCALABLE-NEXT:    [[NEXTI]] = add i64 [[I]], 1
-; SCALABLE-NEXT:    [[DONE:%.*]] = icmp eq i64 [[NEXTI]], 1024
-; SCALABLE-NEXT:    br i1 [[DONE]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP3:![0-9]+]]
+; SCALABLE-NEXT:    [[NEXTI1]] = add i64 [[I1]], 1
+; SCALABLE-NEXT:    [[DONE:%.*]] = icmp eq i64 [[NEXTI1]], 1024
+; SCALABLE-NEXT:    br i1 [[DONE]], label [[EXIT]], label [[LOOP1]], !llvm.loop [[LOOP3:![0-9]+]]
 ; SCALABLE:       exit:
 ; SCALABLE-NEXT:    ret void
 ;
@@ -186,7 +186,7 @@ define void @load_store_factor2_i64(ptr %p) {
 ; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
 ; CHECK-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 2
 ; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP1]]
-; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[ENTRY:%.*]]
 ; CHECK:       vector.ph:
 ; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
 ; CHECK-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 2
@@ -194,88 +194,88 @@ define void @load_store_factor2_i64(ptr %p) {
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
 ; CHECK-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
 ; CHECK-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 2
-; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       vector.body:
-; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[TMP6:%.*]] = add i64 [[INDEX]], 0
-; CHECK-NEXT:    [[TMP7:%.*]] = shl i64 [[TMP6]], 1
-; CHECK-NEXT:    [[TMP8:%.*]] = getelementptr i64, ptr [[P:%.*]], i64 [[TMP7]]
-; CHECK-NEXT:    [[WIDE_VEC:%.*]] = load <vscale x 4 x i64>, ptr [[TMP8]], align 8
+; CHECK-NEXT:    [[I:%.*]] = phi i64 [ 0, [[ENTRY]] ], [ [[NEXTI:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[TMP6:%.*]] = add i64 [[I]], 0
+; CHECK-NEXT:    [[OFFSET0:%.*]] = shl i64 [[TMP6]], 1
+; CHECK-NEXT:    [[Q0:%.*]] = getelementptr i64, ptr [[P:%.*]], i64 [[OFFSET0]]
+; CHECK-NEXT:    [[WIDE_VEC:%.*]] = load <vscale x 4 x i64>, ptr [[Q0]], align 8
 ; CHECK-NEXT:    [[STRIDED_VEC:%.*]] = call { <vscale x 2 x i64>, <vscale x 2 x i64> } @llvm.vector.deinterleave2.nxv4i64(<vscale x 4 x i64> [[WIDE_VEC]])
-; CHECK-NEXT:    [[TMP10:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64> } [[STRIDED_VEC]], 0
-; CHECK-NEXT:    [[TMP11:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64> } [[STRIDED_VEC]], 1
-; CHECK-NEXT:    [[TMP12:%.*]] = add <vscale x 2 x i64> [[TMP10]], splat (i64 1)
-; CHECK-NEXT:    [[TMP15:%.*]] = add <vscale x 2 x i64> [[TMP11]], splat (i64 2)
-; CHECK-NEXT:    [[INTERLEAVED_VEC:%.*]] = call <vscale x 4 x i64> @llvm.vector.interleave2.nxv4i64(<vscale x 2 x i64> [[TMP12]], <vscale x 2 x i64> [[TMP15]])
-; CHECK-NEXT:    store <vscale x 4 x i64> [[INTERLEAVED_VEC]], ptr [[TMP8]], align 8
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
-; CHECK-NEXT:    [[TMP17:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT:    br i1 [[TMP17]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK-NEXT:    [[TMP9:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64> } [[STRIDED_VEC]], 0
+; CHECK-NEXT:    [[TMP10:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64> } [[STRIDED_VEC]], 1
+; CHECK-NEXT:    [[TMP11:%.*]] = add <vscale x 2 x i64> [[TMP9]], splat (i64 1)
+; CHECK-NEXT:    [[TMP12:%.*]] = add <vscale x 2 x i64> [[TMP10]], splat (i64 2)
+; CHECK-NEXT:    [[INTERLEAVED_VEC:%.*]] = call <vscale x 4 x i64> @llvm.vector.interleave2.nxv4i64(<vscale x 2 x i64> [[TMP11]], <vscale x 2 x i64> [[TMP12]])
+; CHECK-NEXT:    store <vscale x 4 x i64> [[INTERLEAVED_VEC]], ptr [[Q0]], align 8
+; CHECK-NEXT:    [[NEXTI]] = add nuw i64 [[I]], [[TMP5]]
+; CHECK-NEXT:    [[TMP13:%.*]] = icmp eq i64 [[NEXTI]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[TMP13]], label [[MIDDLE_BLOCK:%.*]], label [[LOOP]], !llvm.loop [[LOOP4:![0-9]+]]
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 1024, [[N_VEC]]
 ; CHECK-NEXT:    br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
-; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY1:%.*]] ]
+; CHECK-NEXT:    br label [[LOOP1:%.*]]
 ; CHECK:       loop:
-; CHECK-NEXT:    [[I:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[NEXTI:%.*]], [[LOOP]] ]
-; CHECK-NEXT:    [[OFFSET0:%.*]] = shl i64 [[I]], 1
-; CHECK-NEXT:    [[Q0:%.*]] = getelementptr i64, ptr [[P]], i64 [[OFFSET0]]
-; CHECK-NEXT:    [[X0:%.*]] = load i64, ptr [[Q0]], align 8
+; CHECK-NEXT:    [[I1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[NEXTI1:%.*]], [[LOOP1]] ]
+; CHECK-NEXT:    [[OFFSET2:%.*]] = shl i64 [[I1]], 1
+; CHECK-NEXT:    [[Q2:%.*]] = getelementptr i64, ptr [[P]], i64 [[OFFSET2]]
+; CHECK-NEXT:    [[X0:%.*]] = load i64, ptr [[Q2]], align 8
 ; CHECK-NEXT:    [[Y0:%.*]] = add i64 [[X0]], 1
-; CHECK-NEXT:    store i64 [[Y0]], ptr [[Q0]], align 8
-; CHECK-NEXT:    [[OFFSET1:%.*]] = add i64 [[OFFSET0]], 1
+; CHECK-NEXT:    store i64 [[Y0]], ptr [[Q2]], align 8
+; CHECK-NEXT:    [[OFFSET1:%.*]] = add i64 [[OFFSET2]], 1
 ; CHECK-NEXT:    [[Q1:%.*]] = getelementptr i64, ptr [[P]], i64 [[OFFSET1]]
 ; CHECK-NEXT:    [[X1:%.*]] = load i64, ptr [[Q1]], align 8
 ; CHECK-NEXT:    [[Y1:%.*]] = add i64 [[X1]], 2
 ; CHECK-NEXT:    store i64 [[Y1]], ptr [[Q1]], align 8
-; CHECK-NEXT:    [[NEXTI]] = add i64 [[I]], 1
-; CHECK-NEXT:    [[DONE:%.*]] = icmp eq i64 [[NEXTI]], 1024
-; CHECK-NEXT:    br i1 [[DONE]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP5:![0-9]+]]
+; CHECK-NEXT:    [[NEXTI1]] = add i64 [[I1]], 1
+; CHECK-NEXT:    [[DONE:%.*]] = icmp eq i64 [[NEXTI1]], 1024
+; CHECK-NEXT:    br i1 [[DONE]], label [[EXIT]], label [[LOOP1]], !llvm.loop [[LOOP5:![0-9]+]]
 ; CHECK:       exit:
 ; CHECK-NEXT:    ret void
 ;
 ; FIXED-LABEL: @load_store_factor2_i64(
 ; FIXED-NEXT:  entry:
-; FIXED-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; FIXED-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[ENTRY:%.*]]
 ; FIXED:       vector.ph:
-; FIXED-NEXT:    br label [[VECTOR_BODY:%.*]]
+; FIXED-NEXT:    br label [[LOOP:%.*]]
 ; FIXED:       vector.body:
-; FIXED-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; FIXED-NEXT:    [[TMP0:%.*]] = add i64 [[INDEX]], 0
-; FIXED-NEXT:    [[TMP1:%.*]] = shl i64 [[TMP0]], 1
-; FIXED-NEXT:    [[TMP2:%.*]] = getelementptr i64, ptr [[P:%.*]], i64 [[TMP1]]
-; FIXED-NEXT:    [[WIDE_VEC:%.*]] = load <8 x i64>, ptr [[TMP2]], align 8
+; FIXED-NEXT:    [[I:%.*]] = phi i64 [ 0, [[ENTRY]] ], [ [[NEXTI:%.*]], [[LOOP]] ]
+; FIXED-NEXT:    [[TMP0:%.*]] = add i64 [[I]], 0
+; FIXED-NEXT:    [[OFFSET0:%.*]] = shl i64 [[TMP0]], 1
+; FIXED-NEXT:    [[Q0:%.*]] = getelementptr i64, ptr [[P:%.*]], i64 [[OFFSET0]]
+; FIXED-NEXT:    [[WIDE_VEC:%.*]] = load <8 x i64>, ptr [[Q0]], align 8
 ; FIXED-NEXT:    [[STRIDED_VEC:%.*]] = shufflevector <8 x i64> [[WIDE_VEC]], <8 x i64> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
 ; FIXED-NEXT:    [[STRIDED_VEC1:%.*]] = shufflevector <8 x i64> [[WIDE_VEC]], <8 x i64> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
-; FIXED-NEXT:    [[TMP4:%.*]] = add <4 x i64> [[STRIDED_VEC]], splat (i64 1)
-; FIXED-NEXT:    [[TMP7:%.*]] = add <4 x i64> [[STRIDED_VEC1]], splat (i64 2)
-; FIXED-NEXT:    [[TMP9:%.*]] = shufflevector <4 x i64> [[TMP4]], <4 x i64> [[TMP7]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; FIXED-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <8 x i64> [[TMP9]], <8 x i64> poison, <8 x i32> <i32 0, i32 4, i32 1, i32 5, i32 2, i32 6, i32 3, i32 7>
-; FIXED-NEXT:    store <8 x i64> [[INTERLEAVED_VEC]], ptr [[TMP2]], align 8
-; FIXED-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
-; FIXED-NEXT:    [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
-; FIXED-NEXT:    br i1 [[TMP10]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; FIXED-NEXT:    [[TMP3:%.*]] = add <4 x i64> [[STRIDED_VEC]], splat (i64 1)
+; FIXED-NEXT:    [[TMP4:%.*]] = add <4 x i64> [[STRIDED_VEC1]], splat (i64 2)
+; FIXED-NEXT:    [[TMP5:%.*]] = shufflevector <4 x i64> [[TMP3]], <4 x i64> [[TMP4]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; FIXED-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <8 x i64> [[TMP5]], <8 x i64> poison, <8 x i32> <i32 0, i32 4, i32 1, i32 5, i32 2, i32 6, i32 3, i32 7>
+; FIXED-NEXT:    store <8 x i64> [[INTERLEAVED_VEC]], ptr [[Q0]], align 8
+; FIXED-NEXT:    [[NEXTI]] = add nuw i64 [[I]], 4
+; FIXED-NEXT:    [[TMP6:%.*]] = icmp eq i64 [[NEXTI]], 1024
+; FIXED-NEXT:    br i1 [[TMP6]], label [[MIDDLE_BLOCK:%.*]], label [[LOOP]], !llvm.loop [[LOOP4:![0-9]+]]
 ; FIXED:       middle.block:
 ; FIXED-NEXT:    br i1 true, label [[EXIT:%.*]], label [[SCALAR_PH]]
 ; FIXED:       scalar.ph:
-; FIXED-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 1024, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
-; FIXED-NEXT:    br label [[LOOP:%.*]]
+; FIXED-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 1024, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY1:%.*]] ]
+; FIXED-NEXT:    br label [[LOOP1:%.*]]
 ; FIXED:       loop:
-; FIXED-NEXT:    [[I:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[NEXTI:%.*]], [[LOOP]] ]
-; FIXED-NEXT:    [[OFFSET0:%.*]] = shl i64 [[I]], 1
-; FIXED-NEXT:    [[Q0:%.*]] = getelementptr i64, ptr [[P]], i64 [[OFFSET0]]
-; FIXED-NEXT:    [[X0:%.*]] = load i64, ptr [[Q0]], align 8
+; FIXED-NEXT:    [[I1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[NEXTI1:%.*]], [[LOOP1]] ]
+; FIXED-NEXT:    [[OFFSET2:%.*]] = shl i64 [[I1]], 1
+; FIXED-NEXT:    [[Q2:%.*]] = getelementptr i64, ptr [[P]], i64 [[OFFSET2]]
+; FIXED-NEXT:    [[X0:%.*]] = load i64, ptr [[Q2]], align 8
 ; FIXED-NEXT:    [[Y0:%.*]] = add i64 [[X0]], 1
-; FIXED-NEXT:    store i64 [[Y0]], ptr [[Q0]], align 8
-; FIXED-NEXT:    [[OFFSET1:%.*]] = add i64 [[OFFSET0]], 1
+; FIXED-NEXT:    store i64 [[Y0]], ptr [[Q2]], align 8
+; FIXED-NEXT:    [[OFFSET1:%.*]] = add i64 [[OFFSET2]], 1
 ; FIXED-NEXT:    [[Q1:%.*]] = getelementptr i64, ptr [[P]], i64 [[OFFSET1]]
 ; FIXED-NEXT:    [[X1:%.*]] = load i64, ptr [[Q1]], align 8
 ; FIXED-NEXT:    [[Y1:%.*]] = add i64 [[X1]], 2
 ; FIXED-NEXT:    store i64 [[Y1]], ptr [[Q1]], align 8
-; FIXED-NEXT:    [[NEXTI]] = add i64 [[I]], 1
-; FIXED-NEXT:    [[DONE:%.*]] = icmp eq i64 [[NEXTI]], 1024
-; FIXED-NEXT:    br i1 [[DONE]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP5:![0-9]+]]
+; FIXED-NEXT:    [[NEXTI1]] = add i64 [[I1]], 1
+; FIXED-NEXT:    [[DONE:%.*]] = icmp eq i64 [[NEXTI1]], 1024
+; FIXED-NEXT:    br i1 [[DONE]], label [[EXIT]], label [[LOOP1]], !llvm.loop [[LOOP5:![0-9]+]]
 ; FIXED:       exit:
 ; FIXED-NEXT:    ret void
 ;
@@ -284,7 +284,7 @@ define void @load_store_factor2_i64(ptr %p) {
 ; SCALABLE-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
 ; SCALABLE-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 2
 ; SCALABLE-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP1]]
-; SCALABLE-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; SCALABLE-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[ENTRY:%.*]]
 ; SCALABLE:       vector.ph:
 ; SCALABLE-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
 ; SCALABLE-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 2
@@ -292,44 +292,44 @@ define void @load_store_factor2_i64(ptr %p) {
 ; SCALABLE-NEXT:    [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
 ; SCALABLE-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
 ; SCALABLE-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 2
-; SCALABLE-NEXT:    br label [[VECTOR_BODY:%.*]]
+; SCALABLE-NEXT:    br label [[LOOP:%.*]]
 ; SCALABLE:       vector.body:
-; SCALABLE-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; SCALABLE-NEXT:    [[TMP6:%.*]] = add i64 [[INDEX]], 0
-; SCALABLE-NEXT:    [[TMP7:%.*]] = shl i64 [[TMP6]], 1
-; SCALABLE-NEXT:    [[TMP8:%.*]] = getelementptr i64, ptr [[P:%.*]], i64 [[TMP7]]
-; SCALABLE-NEXT:    [[WIDE_VEC:%.*]] = load <vscale x 4 x i64>, ptr [[TMP8]], align 8
+; SCALABLE-NEXT:    [[I:%.*]] = phi i64 [ 0, [[ENTRY]] ], [ [[NEXTI:%.*]], [[LOOP]] ]
+; SCALABLE-NEXT:    [[TMP6:%.*]] = add i64 [[I]], 0
+; SCALABLE-NEXT:    [[OFFSET0:%.*]] = shl i64 [[TMP6]], 1
+; SCALABLE-NEXT:    [[Q0:%.*]] = getelementptr i64, ptr [[P:%.*]], i64 [[OFFSET0]]
+; SCALABLE-NEXT:    [[WIDE_VEC:%.*]] = load <vscale x 4 x i64>, ptr [[Q0]], align 8
 ; SCALABLE-NEXT:    [[STRIDED_VEC:%.*]] = call { <vscale x 2 x i64>, <vscale x 2 x i64> } @llvm.vector.deinterleave2.nxv4i64(<vscale x 4 x i64> [[WIDE_VEC]])
-; SCALABLE-NEXT:    [[TMP10:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64> } [[STRIDED_VEC]], 0
-; SCALABLE-NEXT:    [[TMP11:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64> } [[STRIDED_VEC]], 1
-; SCALABLE-NEXT:    [[TMP12:%.*]] = add <vscale x 2 x i64> [[TMP10]], splat (i64 1)
-; SCALABLE-NEXT:    [[TMP15:%.*]] = add <vscale x 2 x i64> [[TMP11]], splat (i64 2)
-; SCALABLE-NEXT:    [[INTERLEAVED_VEC:%.*]] = call <vscale x 4 x i64> @llvm.vector.interleave2.nxv4i64(<vscale x 2 x i64> [[TMP12]], <vscale x 2 x i64> [[TMP15]])
-; SCALABLE-NEXT:    store <vscale x 4 x i64> [[INTERLEAVED_VEC]], ptr [[TMP8]], align 8
-; SCALABLE-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
-; SCALABLE-NEXT:    [[TMP17:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; SCALABLE-NEXT:    br i1 [[TMP17]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; SCALABLE-NEXT:    [[TMP9:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64> } [[STRIDED_VEC]], 0
+; SCALABLE-NEXT:    [[TMP10:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64> } [[STRIDED_VEC]], 1
+; SCALABLE-NEXT:    [[TMP11:%.*]] = add <vscale x 2 x i64> [[TMP9]], splat (i64 1)
+; SCALABLE-NEXT:    [[TMP12:%.*]] = add <vscale x 2 x i64> [[TMP10]], splat (i64 2)
+; SCALABLE-NEXT:    [[INTERLEAVED_VEC:%.*]] = call <vscale x 4 x i64> @llvm.vector.interleave2.nxv4i64(<vscale x 2 x i64> [[TMP11]], <vscale x 2 x i64> [[TMP12]])
+; SCALABLE-NEXT:    store <vscale x 4 x i64> [[INTERLEAVED_VEC]], ptr [[Q0]], align 8
+; SCALABLE-NEXT:    [[NEXTI]] = add nuw i64 [[I]], [[TMP5]]
+; SCALABLE-NEXT:    [[TMP13:%.*]] = icmp eq i64 [[NEXTI]], [[N_VEC]]
+; SCALABLE-NEXT:    br i1 [[TMP13]], label [[MIDDLE_BLOCK:%.*]], label [[LOOP]], !llvm.loop [[LOOP4:![0-9]+]]
 ; SCALABLE:       middle.block:
 ; SCALABLE-NEXT:    [[CMP_N:%.*]] = icmp eq i64 1024, [[N_VEC]]
 ; SCALABLE-NEXT:    br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
 ; SCALABLE:       scalar.ph:
-; SCALABLE-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
-; SCALABLE-NEXT:    br label [[LOOP:%.*]]
+; SCALABLE-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY1:%.*]] ]
+; SCALABLE-NEXT:    br label [[LOOP1:%.*]]
 ; SCALABLE:       loop:
-; SCALABLE-NEXT:    [[I:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[NEXTI:%.*]], [[LOOP]] ]
-; SCALABLE-NEXT:    [[OFFSET0:%.*]] = shl i64 [[I]], 1
-; SCALABLE-NEXT:    [[Q0:%.*]] = getelementptr i64, ptr [[P]], i64 [[OFFSET0]]
-; SCALABLE-NEXT:    [[X0:%.*]] = load i64, ptr [[Q0]], align 8
+; SCALABLE-NEXT:    [[I1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[NEXTI1:%.*]], [[LOOP1]] ]
+; SCALABLE-NEXT:    [[OFFSET2:%.*]] = shl i64 [[I1]], 1
+; SCALABLE-NEXT:    [[Q2:%.*]] = getelementptr i64, ptr [[P]], i64 [[OFFSET2]]
+; SCALABLE-NEXT:    [[X0:%.*]] = load i64, ptr [[Q2]], align 8
 ; SCALABLE-NEXT:    [[Y0:%.*]] = add i64 [[X0]], 1
-; SCALABLE-NEXT:    store i64 [[Y0]], ptr [[Q0]], align 8
-; SCALABLE-NEXT:    [[OFFSET1:%.*]] = add i64 [[OFFSET0]], 1
+; SCALABLE-NEXT:    store i64 [[Y0]], ptr [[Q2]], align 8
+; SCALABLE-NEXT:    [[OFFSET1:%.*]] = add i64 [[OFFSET2]], 1
 ; SCALABLE-NEXT:    [[Q1:%.*]] = getelementptr i64, ptr [[P]], i64 [[OFFSET1]]
 ; SCALABLE-NEXT:    [[X1:%.*]] = load i64, ptr [[Q1]], align 8
 ; SCALABLE-NEXT:    [[Y1:%.*]] = add i64 [[X1]], 2
 ; SCALABLE-NEXT:    store i64 [[Y1]], ptr [[Q1]], align 8
-; SCALABLE-NEXT:    [[NEXTI]] = add i64 [[I]], 1
-; SCALABLE-NEXT:    [[DONE:%.*]] = icmp eq i64 [[NEXTI]], 1024
-; SCALABLE-NEXT:    br i1 [[DONE]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP5:![0-9]+]]
+; SCALABLE-NEXT:    [[NEXTI1]] = add i64 [[I1]], 1
+; SCALABLE-NEXT:    [[DONE:%.*]] = icmp eq i64 [[NEXTI1]], 1024
+; SCALABLE-NEXT:    br i1 [[DONE]], label [[EXIT]], label [[LOOP1]], !llvm.loop [[LOOP5:![0-9]+]]
 ; SCALABLE:       exit:
 ; SCALABLE-NEXT:    ret void
 ;
@@ -360,42 +360,42 @@ exit:
 define void @load_store_factor3_i32(ptr %p) {
 ; CHECK-LABEL: @load_store_factor3_i32(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[ENTRY:%.*]]
 ; CHECK:       vector.ph:
-; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       vector.body:
-; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[TMP0:%.*]] = add i64 [[INDEX]], 0
-; CHECK-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 3
-; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr i32, ptr [[P:%.*]], i64 [[TMP1]]
-; CHECK-NEXT:    [[WIDE_VEC:%.*]] = load <24 x i32>, ptr [[TMP2]], align 4
+; CHECK-NEXT:    [[I:%.*]] = phi i64 [ 0, [[ENTRY]] ], [ [[NEXTI:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[TMP0:%.*]] = add i64 [[I]], 0
+; CHECK-NEXT:    [[OFFSET0:%.*]] = mul i64 [[TMP0]], 3
+; CHECK-NEXT:    [[Q0:%.*]] = getelementptr i32, ptr [[P:%.*]], i64 [[OFFSET0]]
+; CHECK-NEXT:    [[WIDE_VEC:%.*]] = load <24 x i32>, ptr [[Q0]], align 4
 ; CHECK-NEXT:    [[STRIDED_VEC:%.*]] = shufflevector <24 x i32> [[WIDE_VEC]], <24 x i32> poison, <8 x i32> <i32 0, i32 3, i32 6, i32 9, i32 12, i32 15, i32 18, i32 21>
 ; CHECK-NEXT:    [[STRIDED_VEC1:%.*]] = shufflevector <24 x i32> [[WIDE_VEC]], <24 x i32> poison, <8 x i32> <i32 1, i32 4, i32 7, i32 10, i32 13, i32 16, i32 19, i32 22>
 ; CHECK-NEXT:    [[STRIDED_VEC2:%.*]] = shufflevector <24 x i32> [[WIDE_VEC]], <24 x i32> poison, <8 x i32> <i32 2, i32 5, i32 8, i32 11, i32 14, i32 17, i32 20, i32 23>
-; CHECK-NEXT:    [[TMP4:%.*]] = add <8 x i32> [[STRIDED_VEC]], splat (i32 1)
-; CHECK-NEXT:    [[TMP6:%.*]] = add <8 x i32> [[STRIDED_VEC1]], splat (i32 2)
-; CHECK-NEXT:    [[TMP9:%.*]] = add <8 x i32> [[STRIDED_VEC2]], splat (i32 3)
-; CHECK-NEXT:    [[TMP11:%.*]] = shufflevector <8 x i32> [[TMP4]], <8 x i32> [[TMP6]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; CHECK-NEXT:    [[TMP12:%.*]] = shufflevector <8 x i32> [[TMP9]], <8 x i32> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT:    [[TMP13:%.*]] = shufflevector <16 x i32> [[TMP11]], <16 x i32> [[TMP12]], <24 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23>
-; CHECK-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <24 x i32> [[TMP13]], <24 x i32> poison, <24 x i32> <i32 0, i32 8, i32 16, i32 1, i32 9, i32 17, i32 2, i32 10, i32 18, i32 3, i32 11, i32 19, i32 4, i32 12, i32 20, i32 5, i32 13, i32 21, i32 6, i32 14, i32 22, i32 7, i32 15, i32 23>
-; CHECK-NEXT:    store <24 x i32> [[INTERLEAVED_VEC]], ptr [[TMP2]], align 4
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
-; CHECK-NEXT:    [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
-; CHECK-NEXT:    br i1 [[TMP14]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK-NEXT:    [[TMP3:%.*]] = add <8 x i32> [[STRIDED_VEC]], splat (i32 1)
+; CHECK-NEXT:    [[TMP4:%.*]] = add <8 x i32> [[STRIDED_VEC1]], splat (i32 2)
+; CHECK-NEXT:    [[TMP5:%.*]] = add <8 x i32> [[STRIDED_VEC2]], splat (i32 3)
+; CHECK-NEXT:    [[TMP6:%.*]] = shufflevector <8 x i32> [[TMP3]], <8 x i32> [[TMP4]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; CHECK-NEXT:    [[TMP7:%.*]] = shufflevector <8 x i32> [[TMP5]], <8 x i32> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP8:%.*]] = shufflevector <16 x i32> [[TMP6]], <16 x i32> [[TMP7]], <24 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23>
+; CHECK-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <24 x i32> [[TMP8]], <24 x i32> poison, <24 x i32> <i32 0, i32 8, i32 16, i32 1, i32 9, i32 17, i32 2, i32 10, i32 18, i32 3, i32 11, i32 19, i32 4, i32 12, i32 20, i32 5, i32 13, i32 21, i32 6, i32 14, i32 22, i32 7, i32 15, i32 23>
+; CHECK-NEXT:    store <24 x i32> [[INTERLEAVED_VEC]], ptr [[Q0]], align 4
+; CHECK-NEXT:    [[NEXTI]] = add nuw i64 [[I]], 8
+; CHECK-NEXT:    [[TMP9:%.*]] = icmp eq i64 [[NEXTI]], 1024
+; CHECK-NEXT:    br i1 [[TMP9]], label [[MIDDLE_BLOCK:%.*]], label [[LOOP]], !llvm.loop [[LOOP6:![0-9]+]]
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    br i1 true, label [[EXIT:%.*]], label [[SCALAR_PH]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 1024, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
-; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 1024, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY1:%.*]] ]
+; CHECK-NEXT:    br label [[LOOP1:%.*]]
 ; CHECK:       loop:
-; CHECK-NEXT:    [[I:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[NEXTI:%.*]], [[LOOP]] ]
-; CHECK-NEXT:    [[OFFSET0:%.*]] = mul i64 [[I]], 3
-; CHECK-NEXT:    [[Q0:%.*]] = getelementptr i32, ptr [[P]], i64 [[OFFSET0]]
-; CHECK-NEXT:    [[X0:%.*]] = load i32, ptr [[Q0]], align 4
+; CHECK-NEXT:    [[I1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[NEXTI1:%.*]], [[LOOP1]] ]
+; CHECK-NEXT:    [[OFFSET3:%.*]] = mul i64 [[I1]], 3
+; CHECK-NEXT:    [[Q3:%.*]] = getelementptr i32, ptr [[P]], i64 [[OFFSET3]]
+; CHECK-NEXT:    [[X0:%.*]] = load i32, ptr [[Q3]], align 4
 ; CHECK-NEXT:    [[Y0:%.*]] = add i32 [[X0]], 1
-; CHECK-NEXT:    store i32 [[Y0]], ptr [[Q0]], align 4
-; CHECK-NEXT:    [[OFFSET1:%.*]] = add i64 [[OFFSET0]], 1
+; CHECK-NEXT:    store i32 [[Y0]], ptr [[Q3]], align 4
+; CHECK-NEXT:    [[OFFSET1:%.*]] = add i64 [[OFFSET3]], 1
 ; CHECK-NEXT:    [[Q1:%.*]] = getelementptr i32, ptr [[P]], i64 [[OFFSET1]]
 ; CHECK-NEXT:    [[X1:%.*]] = load i32, ptr [[Q1]], align 4
 ; CHECK-NEXT:    [[Y1:%.*]] = add i32 [[X1]], 2
@@ -405,50 +405,50 @@ define void @load_store_factor3_i32(ptr %p) {
 ; CHECK-NEXT:    [[X2:%.*]] = load i32, ptr [[Q2]], align 4
 ; CHECK-NEXT:    [[Y2:%.*]] = add i32 [[X2]], 3
 ; CHECK-NEXT:    store i32 [[Y2]], ptr [[Q2]], align 4
-; CHECK-NEXT:    [[NEXTI]] = add i64 [[I]], 1
-; CHECK-NEXT:    [[DONE:%.*]] = icmp eq i64 [[NEXTI]], 1024
-; CHECK-NEXT:    br i1 [[DONE]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP7:![0-9]+]]
+; CHECK-NEXT:    [[NEXTI1]] = add i64 [[I1]], 1
+; CHECK-NEXT:    [[DONE:%.*]] = icmp eq i64 [[NEXTI1]], 1024
+; CHECK-NEXT:    br i1 [[DONE]], label [[EXIT]], label [[LOOP1]], !llvm.loop [[LOOP7:![0-9]+]]
 ; CHECK:       exit:
 ; CHECK-NEXT:    ret void
 ;
 ; FIXED-LABEL: @load_store_factor3_i32(
 ; FIXED-NEXT:  entry:
-; FIXED-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; FIXED-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[ENTRY:%.*]]
 ; FIXED:       vector.ph:
-; FIXED-NEXT:    br label [[VECTOR_BODY:%.*]]
+; FIXED-NEXT:    br label [[LOOP:%.*]]
 ; FIXED:       vector.body:
-; FIXED-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; FIXED-NEXT:    [[TMP0:%.*]] = add i64 [[INDEX]], 0
-; FIXED-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 3
-; FIXED-NEXT:    [[TMP2:%.*]] = getelementptr i32, ptr [[P:%.*]], i64 [[TMP1]]
-; FIXED-NEXT:    [[WIDE_VEC:%.*]] = load <24 x i32>, ptr [[TMP2]], align 4
+; FIXED-NEXT:    [[I:%.*]] = phi i64 [ 0, [[ENTRY]] ], [ [[NEXTI:%.*]], [[LOOP]] ]
+; FIXED-NEXT:    [[TMP0:%.*]] = add i64 [[I]], 0
+; FIXED-NEXT:    [[OFFSET0:%.*]] = mul i64 [[TMP0]], 3
+; FIXED-NEXT:    [[Q0:%.*]] = getelementptr i32, ptr [[P:%.*]], i64 [[OFFSET0]]
+; FIXED-NEXT:    [[WIDE_VEC:%.*]] = load <24 x i32>, ptr [[Q0]], align 4
 ; FIXED-NEXT:    [[STRIDED_VEC:%.*]] = shufflevector <24 x i32> [[WIDE_VEC]], <24 x i32> poison, <8 x i32> <i32 0, i32 3, i32 6, i32 9, i32 12, i32 15, i32 18, i32 21>
 ; FIXED-NEXT:    [[STRIDED_VEC1:%.*]] = shufflevector <24 x i32> [[WIDE_VEC]], <24 x i32> poison, <8 x i32> <i32 1, i32 4, i32 7, i32 10, i32 13, i32 16, i32 19, i32 22>
 ; FIXED-NEXT:    [[STRIDED_VEC2:%.*]] = shufflevector <24 x i32> [[WIDE_VEC]], <24 x i32> poison, <8 x i32> <i32 2, i32 5, i32 8, i32 11, i32 14, i32 17, i32 20, i32 23>
-; FIXED-NEXT:    [[TMP4:%.*]] = add <8 x i32> [[STRIDED_VEC]], splat (i32 1)
-; FIXED-NEXT:    [[TMP6:%.*]] = add <8 x i32> [[STRIDED_VEC1]], splat (i32 2)
-; FIXED-NEXT:    [[TMP9:%.*]] = add <8 x i32> [[STRIDED_VEC2]], splat (i32 3)
-; FIXED-NEXT:    [[TMP11:%.*]] = shufflevector <8 x i32> [[TMP4]], <8 x i32> [[TMP6]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; FIXED-NEXT:    [[TMP12:%.*]] = shufflevector <8 x i32> [[TMP9]], <8 x i32> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; FIXED-NEXT:    [[TMP13:%.*]] = shufflevector <16 x i32> [[TMP11]], <16 x i32> [[TMP12]], <24 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23>
-; FIXED-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <24 x i32> [[TMP13]], <24 x i32> poison, <24 x i32> <i32 0, i32 8, i32 16, i32 1, i32 9, i32 17, i32 2, i32 10, i32 18, i32 3, i32 11, i32 19, i32 4, i32 12, i32 20, i32 5, i32 13, i32 21, i32 6, i32 14, i32 22, i32 7, i32 15, i32 23>
-; FIXED-NEXT:    store <24 x i32> [[INTERLEAVED_VEC]], ptr [[TMP2]], align 4
-; FIXED-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
-; FIXED-NEXT:    [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
-; FIXED-NEXT:    br i1 [[TMP14]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; FIXED-NEXT:    [[TMP3:%.*]] = add <8 x i32> [[STRIDED_VEC]], splat (i32 1)
+; FIXED-NEXT:    [[TMP4:%.*]] = add <8 x i32> [[STRIDED_VEC1]], splat (i32 2)
+; FIXED-NEXT:    [[TMP5:%.*]] = add <8 x i32> [[STRIDED_VEC2]], splat (i32 3)
+; FIXED-NEXT:    [[TMP6:%.*]] = shufflevector <8 x i32> [[TMP3]], <8 x i32> [[TMP4]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; FIXED-NEXT:    [[TMP7:%.*]] = shufflevector <8 x i32> [[TMP5]], <8 x i32> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; FIXED-NEXT:    [[TMP8:%.*]] = shufflevector <16 x i32> [[TMP6]], <16 x i32> [[TMP7]], <24 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23>
+; FIXED-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <24 x i32> [[TMP8]], <24 x i32> poison, <24 x i32> <i32 0, i32 8, i32 16, i32 1, i32 9, i32 17, i32 2, i32 10, i32 18, i32 3, i32 11, i32 19, i32 4, i32 12, i32 20, i32 5, i32 13, i32 21, i32 6, i32 14, i32 22, i32 7, i32 15, i32 23>
+; FIXED-NEXT:    store <24 x i32> [[INTERLEAVED_VEC]], ptr [[Q0]], align 4
+; FIXED-NEXT:    [[NEXTI]] = add nuw i64 [[I]], 8
+; FIXED-NEXT:    [[TMP9:%.*]] = icmp eq i64 [[NEXTI]], 1024
+; FIXED-NEXT:    br i1 [[TMP9]], label [[MIDDLE_BLOCK:%.*]], label [[LOOP]], !llvm.loop [[LOOP6:![0-9]+]]
 ; FIXED:       middle.block:
 ; FIXED-NEXT:    br i1 true, label [[EXIT:%.*]], label [[SCALAR_PH]]
 ; FIXED:       scalar.ph:
-; FIXED-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 1024, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
-; FIXED-NEXT:    br label [[LOOP:%.*]]
+; FIXED-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 1024, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY1:%.*]] ]
+; FIXED-NEXT:    br label [[LOOP1:%.*]]
 ; FIXED:       loop:
-; FIXED-NEXT:    [[I:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[NEXTI:%.*]], [[LOOP]] ]
-; FIXED-NEXT:    [[OFFSET0:%.*]] = mul i64 [[I]], 3
-; FIXED-NEXT:    [[Q0:%.*]] = getelementptr i32, ptr [[P]], i64 [[OFFSET0]]
-; FIXED-NEXT:    [[X0:%.*]] = load i32, ptr [[Q0]], align 4
+; FIXED-NEXT:    [[I1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[NEXTI1:%.*]], [[LOOP1]] ]
+; FIXED-NEXT:    [[OFFSET3:%.*]] = mul i64 [[I1]], 3
+; FIXED-NEXT:    [[Q3:%.*]] = getelementptr i32, ptr [[P]], i64 [[OFFSET3]]
+; FIXED-NEXT:    [[X0:%.*]] = load i32, ptr [[Q3]], align 4
 ; FIXED-NEXT:    [[Y0:%.*]] = add i32 [[X0]], 1
-; FIXED-NEXT:    store i32 [[Y0]], ptr [[Q0]], align 4
-; FIXED-NEXT:    [[OFFSET1:%.*]] = add i64 [[OFFSET0]], 1
+; FIXED-NEXT:    store i32 [[Y0]], ptr [[Q3]], align 4
+; FIXED-NEXT:    [[OFFSET1:%.*]] = add i64 [[OFFSET3]], 1
 ; FIXED-NEXT:    [[Q1:%.*]] = getelementptr i32, ptr [[P]], i64 [[OFFSET1]]
 ; FIXED-NEXT:    [[X1:%.*]] = load i32, ptr [[Q1]], align 4
 ; FIXED-NEXT:    [[Y1:%.*]] = add i32 [[X1]], 2
@@ -458,50 +458,50 @@ define void @load_store_factor3_i32(ptr %p) {
 ; FIXED-NEXT:    [[X2:%.*]] = load i32, ptr [[Q2]], align 4
 ; FIXED-NEXT:    [[Y2:%.*]] = add i32 [[X2]], 3
 ; FIXED-NEXT:    store i32 [[Y2]], ptr [[Q2]], align 4
-; FIXED-NEXT:    [[NEXTI]] = add i64 [[I]], 1
-; FIXED-NEXT:    [[DONE:%.*]] = icmp eq i64 [[NEXTI]], 1024
-; FIXED-NEXT:    br i1 [[DONE]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP7:![0-9]+]]
+; FIXED-NEXT:    [[NEXTI1]] = add i64 [[I1]], 1
+; FIXED-NEXT:    [[DONE:%.*]] = icmp eq i64 [[NEXTI1]], 1024
+; FIXED-NEXT:    br i1 [[DONE]], label [[EXIT]], label [[LOOP1]], !llvm.loop [[LOOP7:![0-9]+]]
 ; FIXED:       exit:
 ; FIXED-NEXT:    ret void
 ;
 ; SCALABLE-LABEL: @load_store_factor3_i32(
 ; SCALABLE-NEXT:  entry:
-; SCALABLE-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; SCALABLE-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[ENTRY:%.*]]
 ; SCALABLE:       vector.ph:
-; SCALABLE-NEXT:    br label [[VECTOR_BODY:%.*]]
+; SCALABLE-NEXT:    br label [[LOOP:%.*]]
 ; SCALABLE:       vector.body:
-; SCALABLE-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; SCALABLE-NEXT:    [[TMP0:%.*]] = add i64 [[INDEX]], 0
-; SCALABLE-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 3
-; SCALABLE-NEXT:    [[TMP2:%.*]] = getelementptr i32, ptr [[P:%.*]], i64 [[TMP1]]
-; SCALABLE-NEXT:    [[WIDE_VEC:%.*]] = load <24 x i32>, ptr [[TMP2]], align 4
+; SCALABLE-NEXT:    [[I:%.*]] = phi i64 [ 0, [[ENTRY]] ], [ [[NEXTI:%.*]], [[LOOP]] ]
+; SCALABLE-NEXT:    [[TMP0:%.*]] = add i64 [[I]], 0
+; SCALABLE-NEXT:    [[OFFSET0:%.*]] = mul i64 [[TMP0]], 3
+; SCALABLE-NEXT:    [[Q0:%.*]] = getelementptr i32, ptr [[P:%.*]], i64 [[OFFSET0]]
+; SCALABLE-NEXT:    [[WIDE_VEC:%.*]] = load <24 x i32>, ptr [[Q0]], align 4
 ; SCALABLE-NEXT:    [[STRIDED_VEC:%.*]] = shufflevector <24 x i32> [[WIDE_VEC]], <24 x i32> poison, <8 x i32> <i32 0, i32 3, i32 6, i32 9, i32 12, i32 15, i32 18, i32 21>
 ; SCALABLE-NEXT:    [[STRIDED_VEC1:%.*]] = shufflevector <24 x i32> [[WIDE_VEC]], <24 x i32> poison, <8 x i32> <i32 1, i32 4, i32 7, i32 10, i32 13, i32 16, i32 19, i32 22>
 ; SCALABLE-NEXT:    [[STRIDED_VEC2:%.*]] = shufflevector <24 x i32> [[WIDE_VEC]], <24 x i32> poison, <8 x i32> <i32 2, i32 5, i32 8, i32 11, i32 14, i32 17, i32 20, i32 23>
-; SCALABLE-NEXT:    [[TMP4:%.*]] = add <8 x i32> [[STRIDED_VEC]], splat (i32 1)
-; SCALABLE-NEXT:    [[TMP6:%.*]] = add <8 x i32> [[STRIDED_VEC1]], splat (i32 2)
-; SCALABLE-NEXT:    [[TMP9:%.*]] = add <8 x i32> [[STRIDED_VEC2]], splat (i32 3)
-; SCALABLE-NEXT:    [[TMP11:%.*]] = shufflevector <8 x i32> [[TMP4]], <8 x i32> [[TMP6]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; SCALABLE-NEXT:    [[TMP12:%.*]] = shufflevector <8 x i32> [[TMP9]], <8 x i32> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; SCALABLE-NEXT:    [[TMP13:%.*]] = shufflevector <16 x i32> [[TMP11]], <16 x i32> [[TMP12]], <24 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23>
-; SCALABLE-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <24 x i32> [[TMP13]], <24 x i32> poison, <24 x i32> <i32 0, i32 8, i32 16, i32 1, i32 9, i32 17, i32 2, i32 10, i32 18, i32 3, i32 11, i32 19, i32 4, i32 12, i32 20, i32 5, i32 13, i32 21, i32 6, i32 14, i32 22, i32 7, i32 15, i32 23>
-; SCALABLE-NEXT:    store <24 x i32> [[INTERLEAVED_VEC]], ptr [[TMP2]], align 4
-; SCALABLE-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
-; SCALABLE-NEXT:    [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
-; SCALABLE-NEXT:    br i1 [[TMP14]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; SCALABLE-NEXT:    [[TMP3:%.*]] = add <8 x i32> [[STRIDED_VEC]], splat (i32 1)
+; SCALABLE-NEXT:    [[TMP4:%.*]] = add <8 x i32> [[STRIDED_VEC1]], splat (i32 2)
+; SCALABLE-NEXT:    [[TMP5:%.*]] = add <8 x i32> [[STRIDED_VEC2]], splat (i32 3)
+; SCALABLE-NEXT:    [[TMP6:%.*]] = shufflevector <8 x i32> [[TMP3]], <8 x i32> [[TMP4]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; SCALABLE-NEXT:    [[TMP7:%.*]] = shufflevector <8 x i32> [[TMP5]], <8 x i32> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SCALABLE-NEXT:    [[TMP8:%.*]] = shufflevector <16 x i32> [[TMP6]], <16 x i32> [[TMP7]], <24 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23>
+; SCALABLE-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <24 x i32> [[TMP8]], <24 x i32> poison, <24 x i32> <i32 0, i32 8, i32 16, i32 1, i32 9, i32 17, i32 2, i32 10, i32 18, i32 3, i32 11, i32 19, i32 4, i32 12, i32 20, i32 5, i32 13, i32 21, i32 6, i32 14, i32 22, i32 7, i32 15, i32 23>
+; SCALABLE-NEXT:    store <24 x i32> [[INTERLEAVED_VEC]], ptr [[Q0]], align 4
+; SCALABLE-NEXT:    [[NEXTI]] = add nuw i64 [[I]], 8
+; SCALABLE-NEXT:    [[TMP9:%.*]] = icmp eq i64 [[NEXTI]], 1024
+; SCALABLE-NEXT:    br i1 [[TMP9]], label [[MIDDLE_BLOCK:%.*]], label [[LOOP]], !llvm.loop [[LOOP6:![0-9]+]]
 ; SCALABLE:       middle.block:
 ; SCALABLE-NEXT:    br i1 true, label [[EXIT:%.*]], label [[SCALAR_PH]]
 ; SCALABLE:       scalar.ph:
-; SCALABLE-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 1024, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
-; SCALABLE-NEXT:    br label [[LOOP:%.*]]
+; SCALABLE-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 1024, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY1:%.*]] ]
+; SCALABLE-NEXT:    br label [[LOOP1:%.*]]
 ; SCALABLE:       loop:
-; SCALABLE-NEXT:    [[I:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[NEXTI:%.*]], [[LOOP]] ]
-; SCALABLE-NEXT:    [[OFFSET0:%.*]] = mul i64 [[I]], 3
-; SCALABLE-NEXT:    [[Q0:%.*]] = getelementptr i32, ptr [[P]], i64 [[OFFSET0]]
-; SCALABLE-NEXT:    [[X0:%.*]] = load i32, ptr [[Q0]], align 4
+; SCALABLE-NEXT:    [[I1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[NEXTI1:%.*]], [[LOOP1]] ]
+; SCALABLE-NEXT:    [[OFFSET3:%.*]] = mul i64 [[I1]], 3
+; SCALABLE-NEXT:    [[Q3:%.*]] = getelementptr i32, ptr [[P]], i64 [[OFFSET3]]
+; SCALABLE-NEXT:    [[X0:%.*]] = load i32, ptr [[Q3]], align 4
 ; SCALABLE-NEXT:    [[Y0:%.*]] = add i32 [[X0]], 1
-; SCALABLE-NEXT:    store i32 [[Y0]], ptr [[Q0]], align 4
-; SCALABLE-NEXT:    [[OFFSET1:%.*]] = add i64 [[OFFSET0]], 1
+; SCALABLE-NEXT:    store i32 [[Y0]], ptr [[Q3]], align 4
+; SCALABLE-NEXT:    [[OFFSET1:%.*]] = add i64 [[OFFSET3]], 1
 ; SCALABLE-NEXT:    [[Q1:%.*]] = getelementptr i32, ptr [[P]], i64 [[OFFSET1]]
 ; SCALABLE-NEXT:    [[X1:%.*]] = load i32, ptr [[Q1]], align 4
 ; SCALABLE-NEXT:    [[Y1:%.*]] = add i32 [[X1]], 2
@@ -511,9 +511,9 @@ define void @load_store_factor3_i32(ptr %p) {
 ; SCALABLE-NEXT:    [[X2:%.*]] = load i32, ptr [[Q2]], align 4
 ; SCALABLE-NEXT:    [[Y2:%.*]] = add i32 [[X2]], 3
 ; SCALABLE-NEXT:    store i32 [[Y2]], ptr [[Q2]], align 4
-; SCALABLE-NEXT:    [[NEXTI]] = add i64 [[I]], 1
-; SCALABLE-NEXT:    [[DONE:%.*]] = icmp eq i64 [[NEXTI]], 1024
-; SCALABLE-NEXT:    br i1 [[DONE]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP7:![0-9]+]]
+; SCALABLE-NEXT:    [[NEXTI1]] = add i64 [[I1]], 1
+; SCALABLE-NEXT:    [[DONE:%.*]] = icmp eq i64 [[NEXTI1]], 1024
+; SCALABLE-NEXT:    br i1 [[DONE]], label [[EXIT]], label [[LOOP1]], !llvm.loop [[LOOP7:![0-9]+]]
 ; SCALABLE:       exit:
 ; SCALABLE-NEXT:    ret void
 ;
@@ -550,42 +550,42 @@ exit:
 define void @load_store_factor3_i64(ptr %p) {
 ; CHECK-LABEL: @load_store_factor3_i64(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[ENTRY:%.*]]
 ; CHECK:       vector.ph:
-; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       vector.body:
-; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[TMP0:%.*]] = add i64 [[INDEX]], 0
-; CHECK-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 3
-; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr i64, ptr [[P:%.*]], i64 [[TMP1]]
-; CHECK-NEXT:    [[WIDE_VEC:%.*]] = load <12 x i64>, ptr [[TMP2]], align 8
+; CHECK-NEXT:    [[I:%.*]] = phi i64 [ 0, [[ENTRY]] ], [ [[NEXTI:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[TMP0:%.*]] = add i64 [[I]], 0
+; CHECK-NEXT:    [[OFFSET0:%.*]] = mul i64 [[TMP0]], 3
+; CHECK-NEXT:    [[Q0:%.*]] = getelementptr i64, ptr [[P:%.*]], i64 [[OFFSET0]]
+; CHECK-NEXT:    [[WIDE_VEC:%.*]] = load <12 x i64>, ptr [[Q0]], align 8
 ; CHECK-NEXT:    [[STRIDED_VEC:%.*]] = shufflevector <12 x i64> [[WIDE_VEC]], <12 x i64> poison, <4 x i32> <i32 0, i32 3, i32 6, i32 9>
 ; CHECK-NEXT:    [[STRIDED_VEC1:%.*]] = shufflevector <12 x i64> [[WIDE_VEC]], <12 x i64> poison, <4 x i32> <i32 1, i32 4, i32 7, i32 10>
 ; CHECK-NEXT:    [[STRIDED_VEC2:%.*]] = shufflevector <12 x i64> [[WIDE_VEC]], <12 x i64> poison, <4 x i32> <i32 2, i32 5, i32 8, i32 11>
-; CHECK-NEXT:    [[TMP4:%.*]] = add <4 x i64> [[STRIDED_VEC]], splat (i64 1)
-; CHECK-NEXT:    [[TMP6:%.*]] = add <4 x i64> [[STRIDED_VEC1]], splat (i64 2)
-; CHECK-NEXT:    [[TMP9:%.*]] = add <4 x i64> [[STRIDED_VEC2]], splat (i64 3)
-; CHECK-NEXT:    [[TMP11:%.*]] = shufflevector <4 x i64> [[TMP4]], <4 x i64> [[TMP6]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; CHECK-NEXT:    [[TMP12:%.*]] = shufflevector <4 x i64> [[TMP9]], <4 x i64> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT:    [[TMP13:%.*]] = shufflevector <8 x i64> [[TMP11]], <8 x i64> [[TMP12]], <12 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11>
-; CHECK-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <12 x i64> [[TMP13]], <12 x i64> poison, <12 x i32> <i32 0, i32 4, i32 8, i32 1, i32 5, i32 9, i32 2, i32 6, i32 10, i32 3, i32 7, i32 11>
-; CHECK-NEXT:    store <12 x i64> [[INTERLEAVED_VEC]], ptr [[TMP2]], align 8
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
-; CHECK-NEXT:    [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
-; CHECK-NEXT:    br i1 [[TMP14]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+; CHECK-NEXT:    [[TMP3:%.*]] = add <4 x i64> [[STRIDED_VEC]], splat (i64 1)
+; CHECK-NEXT:    [[TMP4:%.*]] = add <4 x i64> [[STRIDED_VEC1]], splat (i64 2)
+; CHECK-NEXT:    [[TMP5:%.*]] = add <4 x i64> [[STRIDED_VEC2]], splat (i64 3)
+; CHECK-NEXT:    [[TMP6:%.*]] = shufflevector <4 x i64> [[TMP3]], <4 x i64> [[TMP4]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:    [[TMP7:%.*]] = shufflevector <4 x i64> [[TMP5]], <4 x i64> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP8:%.*]] = shufflevector <8 x i64> [[TMP6]], <8 x i64> [[TMP7]], <12 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11>
+; CHECK-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <12 x i64> [[TMP8]], <12 x i64> poison, <12 x i32> <i32 0, i32 4, i32 8, i32 1, i32 5, i32 9, i32 2, i32 6, i32 10, i32 3, i32 7, i32 11>
+; CHECK-NEXT:    store <12 x i64> [[INTERLEAVED_VEC]], ptr [[Q0]], align 8
+; CHECK-NEXT:    [[NEXTI]] = add nuw i64 [[I]], 4
+; CHECK-NEXT:    [[TMP9:%.*]] = icmp eq i64 [[NEXTI]], 1024
+; CHECK-NEXT:    br i1 [[TMP9]], label [[MIDDLE_BLOCK:%.*]], label [[LOOP]], !llvm.loop [[LOOP8:![0-9]+]]
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    br i1 true, label [[EXIT:%.*]], label [[SCALAR_PH]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 1024, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
-; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 1024, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY1:%.*]] ]
+; CHECK-NEXT:    br label [[LOOP1:%.*]]
 ; CHECK:       loop:
-; CHECK-NEXT:    [[I:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[NEXTI:%.*]], [[LOOP]] ]
-; CHECK-NEXT:    [[OFFSET0:%.*]] = mul i64 [[I]], 3
-; CHECK-NEXT:    [[Q0:%.*]] = getelementptr i64, ptr [[P]], i64 [[OFFSET0]]
-; CHECK-NEXT:    [[X0:%.*]] = load i64, ptr [[Q0]], align 8
+; CHECK-NEXT:    [[I1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[NEXTI1:%.*]], [[LOOP1]] ]
+; CHECK-NEXT:    [[OFFSET3:%.*]] = mul i64 [[I1]], 3
+; CHECK-NEXT:    [[Q3:%.*]] = getelementptr i64, ptr [[P]], i64 [[OFFSET3]]
+; CHECK-NEXT:    [[X0:%.*]] = load i64, ptr [[Q3]], align 8
 ; CHECK-NEXT:    [[Y0:%.*]] = add i64 [[X0]], 1
-; CHECK-NEXT:    store i64 [[Y0]], ptr [[Q0]], align 8
-; CHECK-NEXT:    [[OFFSET1:%.*]] = add i64 [[OFFSET0]], 1
+; CHECK-NEXT:    store i64 [[Y0]], ptr [[Q3]], align 8
+; CHECK-NEXT:    [[OFFSET1:%.*]] = add i64 [[OFFSET3]], 1
 ; CHECK-NEXT:    [[Q1:%.*]] = getelementptr i64, ptr [[P]], i64 [[OFFSET1]]
 ; CHECK-NEXT:    [[X1:%.*]] = load i64, ptr [[Q1]], align 8
 ; CHECK-NEXT:    [[Y1:%.*]] = add i64 [[X1]], 2
@@ -595,50 +595,50 @@ define void @load_store_factor3_i64(ptr %p) {
 ; CHECK-NEXT:    [[X2:%.*]] = load i64, ptr [[Q2]], align 8
 ; CHECK-NEXT:    [[Y2:%.*]] = add i64 [[X2]], 3
 ; CHECK-NEXT:    store i64 [[Y2]], ptr [[Q2]], align 8
-; CHECK-NEXT:    [[NEXTI]] = add i64 [[I]], 1
-; CHECK-NEXT:    [[DONE:%.*]] = icmp eq i64 [[NEXTI]], 1024
-; CHECK-NEXT:    br i1 [[DONE]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP9:![0-9]+]]
+; CHECK-NEXT:    [[NEXTI1]] = add i64 [[I1]], 1
+; CHECK-NEXT:    [[DONE:%.*]] = icmp eq i64 [[NEXTI1]], 1024
+; CHECK-NEXT:    br i1 [[DONE]], label [[EXIT]], label [[LOOP1]], !llvm.loop [[LOOP9:![0-9]+]]
 ; CHECK:       exit:
 ; CHECK-NEXT:    ret void
 ;
 ; FIXED-LABEL: @load_store_factor3_i64(
 ; FIXED-NEXT:  entry:
-; FIXED-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; FIXED-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[ENTRY:%.*]]
 ; FIXED:       vector.ph:
-; FIXED-NEXT:    br label [[VECTOR_BODY:%.*]]
+; FIXED-NEXT:    br label [[LOOP:%.*]]
 ; FIXED:       vector.body:
-; FIXED-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; FIXED-NEXT:    [[TMP0:%.*]] = add i64 [[INDEX]], 0
-; FIXED-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 3
-; FIXED-NEXT:    [[TMP2:%.*]] = getelementptr i64, ptr [[P:%.*]], i64 [[TMP1]]
-; FIXED-NEXT:    [[WIDE_VEC:%.*]] = load <12 x i64>, ptr [[TMP2]], align 8
+; FIXED-NEXT:    [[I:%.*]] = phi i64 [ 0, [[ENTRY]] ], [ [[NEXTI:%.*]], [[LOOP]] ]
+; FIXED-NEXT:    [[TMP0:%.*]] = add i64 [[I]], 0
+; FIXED-NEXT:    [[OFFSET0:%.*]] = mul i64 [[TMP0]], 3
+; FIXED-NEXT:    [[Q0:%.*]] = getelementptr i64, ptr [[P:%.*]], i64 [[OFFSET0]]
+; FIXED-NEXT:    [[WIDE_VEC:%.*]] = load <12 x i64>, ptr [[Q0]], align 8
 ; FIXED-NEXT:    [[STRIDED_VEC:%.*]] = shufflevector <12 x i64> [[WIDE_VEC]], <12 x i64> poison, <4 x i32> <i32 0, i32 3, i32 6, i32 9>
 ; FIXED-NEXT:    [[STRIDED_VEC1:%.*]] = shufflevector <12 x i64> [[WIDE_VEC]], <12 x i64> poison, <4 x i32> <i32 1, i32 4, i32 7, i32 10>
 ; FIXED-NEXT:    [[STRIDED_VEC2:%.*]] = shufflevector <12 x i64> [[WIDE_VEC]], <12 x i64> poison, <4 x i32> <i32 2, i32 5, i32 8, i32 11>
-; FIXED-NEXT:    [[TMP4:%.*]] = add <4 x i64> [[STRIDED_VEC]], splat (i64 1)
-; FIXED-NEXT:    [[TMP6:%.*]] = add <4 x i64> [[STRIDED_VEC1]], splat (i64 2)
-; FIXED-NEXT:    [[TMP9:%.*]] = add <4 x i64> [[STRIDED_VEC2]], splat (i64 3)
-; FIXED-NEXT:    [[TMP11:%.*]] = shufflevector <4 x i64> [[TMP4]], <4 x i64> [[TMP6]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; FIXED-NEXT:    [[TMP12:%.*]] = shufflevector <4 x i64> [[TMP9]], <4 x i64> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
-; FIXED-NEXT:    [[TMP13:%.*]] = shufflevector <8 x i64> [[TMP11]], <8 x i64> [[TMP12]], <12 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11>
-; FIXED-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <12 x i64> [[TMP13]], <12 x i64> poison, <12 x i32> <i32 0, i32 4, i32 8, i32 1, i32 5, i32 9, i32 2, i32 6, i32 10, i32 3, i32 7, i32 11>
-; FIXED-NEXT:    store <12 x i64> [[INTERLEAVED_VEC]], ptr [[TMP2]], align 8
-; FIXED-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
-; FIXED-NEXT:    [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
-; FIXED-NEXT:    br i1 [[TMP14]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+; FIXED-NEXT:    [[TMP3:%.*]] = add <4 x i64> [[STRIDED_VEC]], splat (i64 1)
+; FIXED-NEXT:    [[TMP4:%.*]] = add <4 x i64> [[STRIDED_VEC1]], splat (i64 2)
+; FIXED-NEXT:    [[TMP5:%.*]] = add <4 x i64> [[STRIDED_VEC2]], splat (i64 3)
+; FIXED-NEXT:    [[TMP6:%.*]] = shufflevector <4 x i64> [[TMP3]], <4 x i64> [[TMP4]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; FIXED-NEXT:    [[TMP7:%.*]] = shufflevector <4 x i64> [[TMP5]], <4 x i64> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; FIXED-NEXT:    [[TMP8:%.*]] = shufflevector <8 x i64> [[TMP6]], <8 x i64> [[TMP7]], <12 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11>
+; FIXED-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <12 x i64> [[TMP8]], <12 x i64> poison, <12 x i32> <i32 0, i32 4, i32 8, i32 1, i32 5, i32 9, i32 2, i32 6, i32 10, i32 3, i32 7, i32 11>
+; FIXED-NEXT:    store <12 x i64> [[INTERLEAVED_VEC]], ptr [[Q0]], align 8
+; FIXED-NEXT:    [[NEXTI]] = add nuw i64 [[I]], 4
+; FIXED-NEXT:    [[TMP9:%.*]] = icmp eq i64 [[NEXTI]], 1024
+; FIXED-NEXT:    br i1 [[TMP9]], label [[MIDDLE_BLOCK:%.*]], label [[LOOP]], !llvm.loop [[LOOP8:![0-9]+]]
 ; FIXED:       middle.block:
 ; FIXED-NEXT:    br i1 true, label [[EXIT:%.*]], label [[SCALAR_PH]]
 ; FIXED:       scalar.ph:
-; FIXED-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 1024, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
-; FIXED-NEXT:    br label [[LOOP:%.*]]
+; FIXED-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 1024, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY1:%.*]] ]
+; FIXED-NEXT:    br label [[LOOP1:%.*]]
 ; FIXED:       loop:
-; FIXED-NEXT:    [[I:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[NEXTI:%.*]], [[LOOP]] ]
-; FIXED-NEXT:    [[OFFSET0:%.*]] = mul i64 [[I]], 3
-; FIXED-NEXT:    [[Q0:%.*]] = getelementptr i64, ptr [[P]], i64 [[OFFSET0]]
-; FIXED-NEXT:    [[X0:%.*]] = load i64, ptr [[Q0]], align 8
+; FIXED-NEXT:    [[I1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[NEXTI1:%.*]], [[LOOP1]] ]
+; FIXED-NEXT:    [[OFFSET3:%.*]] = mul i64 [[I1]], 3
+; FIXED-NEXT:    [[Q3:%.*]] = getelementptr i64, ptr [[P]], i64 [[OFFSET3]]
+; FIXED-NEXT:    [[X0:%.*]] = load i64, ptr [[Q3]], align 8
 ; FIXED-NEXT:    [[Y0:%.*]] = add i64 [[X0]], 1
-; FIXED-NEXT:    store i64 [[Y0]], ptr [[Q0]], align 8
-; FIXED-NEXT:    [[OFFSET1:%.*]] = add i64 [[OFFSET0]], 1
+; FIXED-NEXT:    store i64 [[Y0]], ptr [[Q3]], align 8
+; FIXED-NEXT:    [[OFFSET1:%.*]] = add i64 [[OFFSET3]], 1
 ; FIXED-NEXT:    [[Q1:%.*]] = getelementptr i64, ptr [[P]], i64 [[OFFSET1]]
 ; FIXED-NEXT:    [[X1:%.*]] = load i64, ptr [[Q1]], align 8
 ; FIXED-NEXT:    [[Y1:%.*]] = add i64 [[X1]], 2
@@ -648,50 +648,50 @@ define void @load_store_factor3_i64(ptr %p) {
 ; FIXED-NEXT:    [[X2:%.*]] = load i64, ptr [[Q2]], align 8
 ; FIXED-NEXT:    [[Y2:%.*]] = add i64 [[X2]], 3
 ; FIXED-NEXT:    store i64 [[Y2]], ptr [[Q2]], align 8
-; FIXED-NEXT:    [[NEXTI]] = add i64 [[I]], 1
-; FIXED-NEXT:    [[DONE:%.*]] = icmp eq i64 [[NEXTI]], 1024
-; FIXED-NEXT:    br i1 [[DONE]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP9:![0-9]+]]
+; FIXED-NEXT:    [[NEXTI1]] = add i64 [[I1]], 1
+; FIXED-NEXT:    [[DONE:%.*]] = icmp eq i64 [[NEXTI1]], 1024
+; FIXED-NEXT:    br i1 [[DONE]], label [[EXIT]], label [[LOOP1]], !llvm.loop [[LOOP9:![0-9]+]]
 ; FIXED:       exit:
 ; FIXED-NEXT:    ret void
 ;
 ; SCALABLE-LABEL: @load_store_factor3_i64(
 ; SCALABLE-NEXT:  entry:
-; SCALABLE-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; SCALABLE-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[ENTRY:%.*]]
 ; SCALABLE:       vector.ph:
-; SCALABLE-NEXT:    br label [[VECTOR_BODY:%.*]]
+; SCALABLE-NEXT:    br label [[LOOP:%.*]]
 ; SCALABLE:       vector.body:
-; SCALABLE-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; SCALABLE-NEXT:    [[TMP0:%.*]] = add i64 [[INDEX]], 0
-; SCALABLE-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 3
-; SCALABLE-NEXT:    [[TMP2:%.*]] = getelementptr i64, ptr [[P:%.*]], i64 [[TMP1]]
-; SCALABLE-NEXT:    [[WIDE_VEC:%.*]] = load <12 x i64>, ptr [[TMP2]], align 8
+; SCALABLE-NEXT:    [[I:%.*]] = phi i64 [ 0, [[ENTRY]] ], [ [[NEXTI:%.*]], [[LOOP]] ]
+; SCALABLE-NEXT:    [[TMP0:%.*]] = add i64 [[I]], 0
+; SCALABLE-NEXT:    [[OFFSET0:%.*]] = mul i64 [[TMP0]], 3
+; SCALABLE-NEXT:    [[Q0:%.*]] = getelementptr i64, ptr [[P:%.*]], i64 [[OFFSET0]]
+; SCALABLE-NEXT:    [[WIDE_VEC:%.*]] = load <12 x i64>, ptr [[Q0]], align 8
 ; SCALABLE-NEXT:    [[STRIDED_VEC:%.*]] = shufflevector <12 x i64> [[WIDE_VEC]], <12 x i64> poison, <4 x i32> <i32 0, i32 3, i32 6, i32 9>
 ; SCALABLE-NEXT:    [[STRIDED_VEC1:%.*]] = shufflevector <12 x i64> [[WIDE_VEC]], <12 x i64> poison, <4 x i32> <i32 1, i32 4, i32 7, i32 10>
 ; SCALABLE-NEXT:    [[STRIDED_VEC2:%.*]] = shufflevector <12 x i64> [[WIDE_VEC]], <12 x i64> poison, <4 x i32> <i32 2, i32 5, i32 8, i32 11>
-; SCALABLE-NEXT:    [[TMP4:%.*]] = add <4 x i64> [[STRIDED_VEC]], splat (i64 1)
-; SCALABLE-NEXT:    [[TMP6:%.*]] = add <4 x i64> [[STRIDED_VEC1]], splat (i64 2)
-; SCALABLE-NEXT:    [[TMP9:%.*]] = add <4 x i64> [[STRIDED_VEC2]], splat (i64 3)
-; SCALABLE-NEXT:    [[TMP11:%.*]] = shufflevector <4 x i64> [[TMP4]], <4 x i64> [[TMP6]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; SCALABLE-NEXT:    [[TMP12:%.*]] = shufflevector <4 x i64> [[TMP9]], <4 x i64> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
-; SCALABLE-NEXT:    [[TMP13:%.*]] = shufflevector <8 x i64> [[TMP11]], <8 x i64> [[TMP12]], <12 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11>
-; SCALABLE-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <12 x i64> [[TMP13]], <12 x i64> poison, <12 x i32> <i32 0, i32 4, i32 8, i32 1, i32 5, i32 9, i32 2, i32 6, i32 10, i32 3, i32 7, i32 11>
-; SCALABLE-NEXT:    store <12 x i64> [[INTERLEAVED_VEC]], ptr [[TMP2]], align 8
-; SCALABLE-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
-; SCALABLE-NEXT:    [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
-; SCALABLE-NEXT:    br i1 [[TMP14]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+; SCALABLE-NEXT:    [[TMP3:%.*]] = add <4 x i64> [[STRIDED_VEC]], splat (i64 1)
+; SCALABLE-NEXT:    [[TMP4:%.*]] = add <4 x i64> [[STRIDED_VEC1]], splat (i64 2)
+; SCALABLE-NEXT:    [[TMP5:%.*]] = add <4 x i64> [[STRIDED_VEC2]], splat (i64 3)
+; SCALABLE-NEXT:    [[TMP6:%.*]] = shufflevector <4 x i64> [[TMP3]], <4 x i64> [[TMP4]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; SCALABLE-NEXT:    [[TMP7:%.*]] = shufflevector <4 x i64> [[TMP5]], <4 x i64> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; SCALABLE-NEXT:    [[TMP8:%.*]] = shufflevector <8 x i64> [[TMP6]], <8 x i64> [[TMP7]], <12 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11>
+; SCALABLE-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <12 x i64> [[TMP8]], <12 x i64> poison, <12 x i32> <i32 0, i32 4, i32 8, i32 1, i32 5, i32 9, i32 2, i32 6, i32 10, i32 3, i32 7, i32 11>
+; SCALABLE-NEXT:    store <12 x i64> [[INTERLEAVED_VEC]], ptr [[Q0]], align 8
+; SCALABLE-NEXT:    [[NEXTI]] = add nuw i64 [[I]], 4
+; SCALABLE-NEXT:    [[TMP9:%.*]] = icmp eq i64 [[NEXTI]], 1024
+; SCALABLE-NEXT:    br i1 [[TMP9]], label [[MIDDLE_BLOCK:%.*]], label [[LOOP]], !llvm.loop [[LOOP8:![0-9]+]]
 ; SCALABLE:       middle.block:
 ; SCALABLE-NEXT:    br i1 true, label [[EXIT:%.*]], label [[SCALAR_PH]]
 ; SCALABLE:       scalar.ph:
-; SCALABLE-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 1024, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
-; SCALABLE-NEXT:    br label [[LOOP:%.*]]
+; SCALABLE-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 1024, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY1:%.*]] ]
+; SCALABLE-NEXT:    br label [[LOOP1:%.*]]
 ; SCALABLE:       loop:
-; SCALABLE-NEXT:    [[I:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[NEXTI:%.*]], [[LOOP]] ]
-; SCALABLE-NEXT:    [[OFFSET0:%.*]] = mul i64 [[I]], 3
-; SCALABLE-NEXT:    [[Q0:%.*]] = getelementptr i64, ptr [[P]], i64 [[OFFSET0]]
-; SCALABLE-NEXT:    [[X0:%.*]] = load i64, ptr [[Q0]], align 8
+; SCALABLE-NEXT:    [[I1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[NEXTI1:%.*]], [[LOOP1]] ]
+; SCALABLE-NEXT:    [[OFFSET3:%.*]] = mul i64 [[I1]], 3
+; SCALABLE-NEXT:    [[Q3:%.*]] = getelementptr i64, ptr [[P]], i64 [[OFFSET3]]
+; SCALABLE-NEXT:    [[X0:%.*]] = load i64, ptr [[Q3]], align 8
 ; SCALABLE-NEXT:    [[Y0:%.*]] = add i64 [[X0]], 1
-; SCALABLE-NEXT:    store i64 [[Y0]], ptr [[Q0]], align 8
-; SCALABLE-NEXT:    [[OFFSET1:%.*]] = add i64 [[OFFSET0]], 1
+; SCALABLE-NEXT:    store i64 [[Y0]], ptr [[Q3]], align 8
+; SCALABLE-NEXT:    [[OFFSET1:%.*]] = add i64 [[OFFSET3]], 1
 ; SCALABLE-NEXT:    [[Q1:%.*]] = getelementptr i64, ptr [[P]], i64 [[OFFSET1]]
 ; SCALABLE-NEXT:    [[X1:%.*]] = load i64, ptr [[Q1]], align 8
 ; SCALABLE-NEXT:    [[Y1:%.*]] = add i64 [[X1]], 2
@@ -701,9 +701,9 @@ define void @load_store_factor3_i64(ptr %p) {
 ; SCALABLE-NEXT:    [[X2:%.*]] = load i64, ptr [[Q2]], align 8
 ; SCALABLE-NEXT:    [[Y2:%.*]] = add i64 [[X2]], 3
 ; SCALABLE-NEXT:    store i64 [[Y2]], ptr [[Q2]], align 8
-; SCALABLE-NEXT:    [[NEXTI]] = add i64 [[I]], 1
-; SCALABLE-NEXT:    [[DONE:%.*]] = icmp eq i64 [[NEXTI]], 1024
-; SCALABLE-NEXT:    br i1 [[DONE]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP9:![0-9]+]]
+; SCALABLE-NEXT:    [[NEXTI1]] = add i64 [[I1]], 1
+; SCALABLE-NEXT:    [[DONE:%.*]] = icmp eq i64 [[NEXTI1]], 1024
+; SCALABLE-NEXT:    br i1 [[DONE]], label [[EXIT]], label [[LOOP1]], !llvm.loop [[LOOP9:![0-9]+]]
 ; SCALABLE:       exit:
 ; SCALABLE-NEXT:    ret void
 ;
@@ -740,56 +740,75 @@ exit:
 define void @load_store_factor8(ptr %p) {
 ; CHECK-LABEL: @load_store_factor8(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP0]]
+; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[ENTRY:%.*]]
 ; CHECK:       vector.ph:
-; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP1]]
+; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
+; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       vector.body:
-; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[TMP0:%.*]] = add i64 [[INDEX]], 0
-; CHECK-NEXT:    [[TMP1:%.*]] = shl i64 [[TMP0]], 3
-; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr i64, ptr [[P:%.*]], i64 [[TMP1]]
-; CHECK-NEXT:    [[WIDE_VEC:%.*]] = load <16 x i64>, ptr [[TMP2]], align 8
-; CHECK-NEXT:    [[STRIDED_VEC:%.*]] = shufflevector <16 x i64> [[WIDE_VEC]], <16 x i64> poison, <2 x i32> <i32 0, i32 8>
-; CHECK-NEXT:    [[STRIDED_VEC1:%.*]] = shufflevector <16 x i64> [[WIDE_VEC]], <16 x i64> poison, <2 x i32> <i32 1, i32 9>
-; CHECK-NEXT:    [[STRIDED_VEC2:%.*]] = shufflevector <16 x i64> [[WIDE_VEC]], <16 x i64> poison, <2 x i32> <i32 2, i32 10>
-; CHECK-NEXT:    [[STRIDED_VEC3:%.*]] = shufflevector <16 x i64> [[WIDE_VEC]], <16 x i64> poison, <2 x i32> <i32 3, i32 11>
-; CHECK-NEXT:    [[STRIDED_VEC4:%.*]] = shufflevector <16 x i64> [[WIDE_VEC]], <16 x i64> poison, <2 x i32> <i32 4, i32 12>
-; CHECK-NEXT:    [[STRIDED_VEC5:%.*]] = shufflevector <16 x i64> [[WIDE_VEC]], <16 x i64> poison, <2 x i32> <i32 5, i32 13>
-; CHECK-NEXT:    [[STRIDED_VEC6:%.*]] = shufflevector <16 x i64> [[WIDE_VEC]], <16 x i64> poison, <2 x i32> <i32 6, i32 14>
-; CHECK-NEXT:    [[STRIDED_VEC7:%.*]] = shufflevector <16 x i64> [[WIDE_VEC]], <16 x i64> poison, <2 x i32> <i32 7, i32 15>
-; CHECK-NEXT:    [[TMP4:%.*]] = add <2 x i64> [[STRIDED_VEC]], splat (i64 1)
-; CHECK-NEXT:    [[TMP6:%.*]] = add <2 x i64> [[STRIDED_VEC1]], splat (i64 2)
-; CHECK-NEXT:    [[TMP8:%.*]] = add <2 x i64> [[STRIDED_VEC2]], splat (i64 3)
-; CHECK-NEXT:    [[TMP10:%.*]] = add <2 x i64> [[STRIDED_VEC3]], splat (i64 4)
-; CHECK-NEXT:    [[TMP12:%.*]] = add <2 x i64> [[STRIDED_VEC4]], splat (i64 5)
-; CHECK-NEXT:    [[TMP14:%.*]] = add <2 x i64> [[STRIDED_VEC5]], splat (i64 6)
-; CHECK-NEXT:    [[TMP16:%.*]] = add <2 x i64> [[STRIDED_VEC6]], splat (i64 7)
-; CHECK-NEXT:    [[TMP19:%.*]] = add <2 x i64> [[STRIDED_VEC7]], splat (i64 8)
-; CHECK-NEXT:    [[TMP21:%.*]] = shufflevector <2 x i64> [[TMP4]], <2 x i64> [[TMP6]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; CHECK-NEXT:    [[TMP22:%.*]] = shufflevector <2 x i64> [[TMP8]], <2 x i64> [[TMP10]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; CHECK-NEXT:    [[TMP23:%.*]] = shufflevector <2 x i64> [[TMP12]], <2 x i64> [[TMP14]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; CHECK-NEXT:    [[TMP24:%.*]] = shufflevector <2 x i64> [[TMP16]], <2 x i64> [[TMP19]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; CHECK-NEXT:    [[TMP25:%.*]] = shufflevector <4 x i64> [[TMP21]], <4 x i64> [[TMP22]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; CHECK-NEXT:    [[TMP26:%.*]] = shufflevector <4 x i64> [[TMP23]], <4 x i64> [[TMP24]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; CHECK-NEXT:    [[TMP27:%.*]] = shufflevector <8 x i64> [[TMP25]], <8 x i64> [[TMP26]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; CHECK-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <16 x i64> [[TMP27]], <16 x i64> poison, <16 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14, i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15>
-; CHECK-NEXT:    store <16 x i64> [[INTERLEAVED_VEC]], ptr [[TMP2]], align 8
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
-; CHECK-NEXT:    [[TMP28:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
-; CHECK-NEXT:    br i1 [[TMP28]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
+; CHECK-NEXT:    [[I:%.*]] = phi i64 [ 0, [[ENTRY]] ], [ [[NEXTI:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[TMP3:%.*]] = add i64 [[I]], 0
+; CHECK-NEXT:    [[OFFSET0:%.*]] = shl i64 [[TMP3]], 3
+; CHECK-NEXT:    [[Q0:%.*]] = getelementptr i64, ptr [[P:%.*]], i64 [[OFFSET0]]
+; CHECK-NEXT:    [[WIDE_VEC:%.*]] = load <vscale x 8 x i64>, ptr [[Q0]], align 8
+; CHECK-NEXT:    [[STRIDED_VEC:%.*]] = call { <vscale x 4 x i64>, <vscale x 4 x i64> } @llvm.vector.deinterleave2.nxv8i64(<vscale x 8 x i64> [[WIDE_VEC]])
+; CHECK-NEXT:    [[TMP6:%.*]] = extractvalue { <vscale x 4 x i64>, <vscale x 4 x i64> } [[STRIDED_VEC]], 0
+; CHECK-NEXT:    [[TMP7:%.*]] = extractvalue { <vscale x 4 x i64>, <vscale x 4 x i64> } [[STRIDED_VEC]], 1
+; CHECK-NEXT:    [[STRIDED_VEC1:%.*]] = call { <vscale x 2 x i64>, <vscale x 2 x i64> } @llvm.vector.deinterleave2.nxv4i64(<vscale x 4 x i64> [[TMP6]])
+; CHECK-NEXT:    [[STRIDED_VEC2:%.*]] = call { <vscale x 2 x i64>, <vscale x 2 x i64> } @llvm.vector.deinterleave2.nxv4i64(<vscale x 4 x i64> [[TMP7]])
+; CHECK-NEXT:    [[TMP8:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64> } [[STRIDED_VEC1]], 0
+; CHECK-NEXT:    [[TMP9:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64> } [[STRIDED_VEC2]], 0
+; CHECK-NEXT:    [[TMP10:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64> } [[STRIDED_VEC1]], 1
+; CHECK-NEXT:    [[TMP11:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64> } [[STRIDED_VEC2]], 1
+; CHECK-NEXT:    [[STRIDED_VEC3:%.*]] = call { <vscale x 1 x i64>, <vscale x 1 x i64> } @llvm.vector.deinterleave2.nxv2i64(<vscale x 2 x i64> [[TMP8]])
+; CHECK-NEXT:    [[STRIDED_VEC4:%.*]] = call { <vscale x 1 x i64>, <vscale x 1 x i64> } @llvm.vector.deinterleave2.nxv2i64(<vscale x 2 x i64> [[TMP9]])
+; CHECK-NEXT:    [[STRIDED_VEC5:%.*]] = call { <vscale x 1 x i64>, <vscale x 1 x i64> } @llvm.vector.deinterleave2.nxv2i64(<vscale x 2 x i64> [[TMP10]])
+; CHECK-NEXT:    [[STRIDED_VEC6:%.*]] = call { <vscale x 1 x i64>, <vscale x 1 x i64> } @llvm.vector.deinterleave2.nxv2i64(<vscale x 2 x i64> [[TMP11]])
+; CHECK-NEXT:    [[TMP12:%.*]] = extractvalue { <vscale x 1 x i64>, <vscale x 1 x i64> } [[STRIDED_VEC3]], 0
+; CHECK-NEXT:    [[TMP13:%.*]] = extractvalue { <vscale x 1 x i64>, <vscale x 1 x i64> } [[STRIDED_VEC4]], 0
+; CHECK-NEXT:    [[TMP14:%.*]] = extractvalue { <vscale x 1 x i64>, <vscale x 1 x i64> } [[STRIDED_VEC5]], 0
+; CHECK-NEXT:    [[TMP15:%.*]] = extractvalue { <vscale x 1 x i64>, <vscale x 1 x i64> } [[STRIDED_VEC6]], 0
+; CHECK-NEXT:    [[TMP16:%.*]] = extractvalue { <vscale x 1 x i64>, <vscale x 1 x i64> } [[STRIDED_VEC3]], 1
+; CHECK-NEXT:    [[TMP17:%.*]] = extractvalue { <vscale x 1 x i64>, <vscale x 1 x i64> } [[STRIDED_VEC4]], 1
+; CHECK-NEXT:    [[TMP18:%.*]] = extractvalue { <vscale x 1 x i64>, <vscale x 1 x i64> } [[STRIDED_VEC5]], 1
+; CHECK-NEXT:    [[TMP19:%.*]] = extractvalue { <vscale x 1 x i64>, <vscale x 1 x i64> } [[STRIDED_VEC6]], 1
+; CHECK-NEXT:    [[TMP20:%.*]] = add <vscale x 1 x i64> [[TMP12]], splat (i64 1)
+; CHECK-NEXT:    [[TMP21:%.*]] = add <vscale x 1 x i64> [[TMP13]], splat (i64 2)
+; CHECK-NEXT:    [[TMP22:%.*]] = add <vscale x 1 x i64> [[TMP14]], splat (i64 3)
+; CHECK-NEXT:    [[TMP23:%.*]] = add <vscale x 1 x i64> [[TMP15]], splat (i64 4)
+; CHECK-NEXT:    [[TMP24:%.*]] = add <vscale x 1 x i64> [[TMP16]], splat (i64 5)
+; CHECK-NEXT:    [[TMP25:%.*]] = add <vscale x 1 x i64> [[TMP17]], splat (i64 6)
+; CHECK-NEXT:    [[TMP26:%.*]] = add <vscale x 1 x i64> [[TMP18]], splat (i64 7)
+; CHECK-NEXT:    [[TMP27:%.*]] = add <vscale x 1 x i64> [[TMP19]], splat (i64 8)
+; CHECK-NEXT:    [[INTERLEAVED_VEC:%.*]] = call <vscale x 2 x i64> @llvm.vector.interleave2.nxv2i64(<vscale x 1 x i64> [[TMP20]], <vscale x 1 x i64> [[TMP24]])
+; CHECK-NEXT:    [[INTERLEAVED_VEC7:%.*]] = call <vscale x 2 x i64> @llvm.vector.interleave2.nxv2i64(<vscale x 1 x i64> [[TMP21]], <vscale x 1 x i64> [[TMP25]])
+; CHECK-NEXT:    [[INTERLEAVED_VEC8:%.*]] = call <vscale x 2 x i64> @llvm.vector.interleave2.nxv2i64(<vscale x 1 x i64> [[TMP22]], <vscale x 1 x i64> [[TMP26]])
+; CHECK-NEXT:    [[INTERLEAVED_VEC9:%.*]] = call <vscale x 2 x i64> @llvm.vector.interleave2.nxv2i64(<vscale x 1 x i64> [[TMP23]], <vscale x 1 x i64> [[TMP27]])
+; CHECK-NEXT:    [[INTERLEAVED_VEC10:%.*]] = call <vscale x 4 x i64> @llvm.vector.interleave2.nxv4i64(<vscale x 2 x i64> [[INTERLEAVED_VEC]], <vscale x 2 x i64> [[INTERLEAVED_VEC8]])
+; CHECK-NEXT:    [[INTERLEAVED_VEC11:%.*]] = call <vscale x 4 x i64> @llvm.vector.interleave2.nxv4i64(<vscale x 2 x i64> [[INTERLEAVED_VEC7]], <vscale x 2 x i64> [[INTERLEAVED_VEC9]])
+; CHECK-NEXT:    [[INTERLEAVED_VEC12:%.*]] = call <vscale x 8 x i64> @llvm.vector.interleave2.nxv8i64(<vscale x 4 x i64> [[INTERLEAVED_VEC10]], <vscale x 4 x i64> [[INTERLEAVED_VEC11]])
+; CHECK-NEXT:    store <vscale x 8 x i64> [[INTERLEAVED_VEC12]], ptr [[Q0]], align 8
+; CHECK-NEXT:    [[NEXTI]] = add nuw i64 [[I]], [[TMP2]]
+; CHECK-NEXT:    [[TMP28:%.*]] = icmp eq i64 [[NEXTI]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[TMP28]], label [[MIDDLE_BLOCK:%.*]], label [[LOOP]], !llvm.loop [[LOOP10:![0-9]+]]
 ; CHECK:       middle.block:
-; CHECK-NEXT:    br i1 true, label [[EXIT:%.*]], label [[SCALAR_PH]]
+; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 1024, [[N_VEC]]
+; CHECK-NEXT:    br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 1024, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
-; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY1:%.*]] ]
+; CHECK-NEXT:    br label [[LOOP1:%.*]]
 ; CHECK:       loop:
-; CHECK-NEXT:    [[I:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[NEXTI:%.*]], [[LOOP]] ]
-; CHECK-NEXT:    [[OFFSET0:%.*]] = shl i64 [[I]], 3
-; CHECK-NEXT:    [[Q0:%.*]] = getelementptr i64, ptr [[P]], i64 [[OFFSET0]]
-; CHECK-NEXT:    [[X0:%.*]] = load i64, ptr [[Q0]], align 8
+; CHECK-NEXT:    [[I1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[NEXTI1:%.*]], [[LOOP1]] ]
+; CHECK-NEXT:    [[OFFSET8:%.*]] = shl i64 [[I1]], 3
+; CHECK-NEXT:    [[Q8:%.*]] = getelementptr i64, ptr [[P]], i64 [[OFFSET8]]
+; CHECK-NEXT:    [[X0:%.*]] = load i64, ptr [[Q8]], align 8
 ; CHECK-NEXT:    [[Y0:%.*]] = add i64 [[X0]], 1
-; CHECK-NEXT:    store i64 [[Y0]], ptr [[Q0]], align 8
-; CHECK-NEXT:    [[OFFSET1:%.*]] = add i64 [[OFFSET0]], 1
+; CHECK-NEXT:    store i64 [[Y0]], ptr [[Q8]], align 8
+; CHECK-NEXT:    [[OFFSET1:%.*]] = add i64 [[OFFSET8]], 1
 ; CHECK-NEXT:    [[Q1:%.*]] = getelementptr i64, ptr [[P]], i64 [[OFFSET1]]
 ; CHECK-NEXT:    [[X1:%.*]] = load i64, ptr [[Q1]], align 8
 ; CHECK-NEXT:    [[Y1:%.*]] = add i64 [[X1]], 2
@@ -824,23 +843,23 @@ define void @load_store_factor8(ptr %p) {
 ; CHECK-NEXT:    [[X7:%.*]] = load i64, ptr [[Q7]], align 8
 ; CHECK-NEXT:    [[Y7:%.*]] = add i64 [[X7]], 8
 ; CHECK-NEXT:    store i64 [[Y7]], ptr [[Q7]], align 8
-; CHECK-NEXT:    [[NEXTI]] = add i64 [[I]], 1
-; CHECK-NEXT:    [[DONE:%.*]] = icmp eq i64 [[NEXTI]], 1024
-; CHECK-NEXT:    br i1 [[DONE]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP11:![0-9]+]]
+; CHECK-NEXT:    [[NEXTI1]] = add i64 [[I1]], 1
+; CHECK-NEXT:    [[DONE:%.*]] = icmp eq i64 [[NEXTI1]], 1024
+; CHECK-NEXT:    br i1 [[DONE]], label [[EXIT]], label [[LOOP1]], !llvm.loop [[LOOP11:![0-9]+]]
 ; CHECK:       exit:
 ; CHECK-NEXT:    ret void
 ;
 ; FIXED-LABEL: @load_store_factor8(
 ; FIXED-NEXT:  entry:
-; FIXED-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; FIXED-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[ENTRY:%.*]]
 ; FIXED:       vector.ph:
-; FIXED-NEXT:    br label [[VECTOR_BODY:%.*]]
+; FIXED-NEXT:    br label [[LOOP:%.*]]
 ; FIXED:       vector.body:
-; FIXED-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; FIXED-NEXT:    [[TMP0:%.*]] = add i64 [[INDEX]], 0
-; FIXED-NEXT:    [[TMP1:%.*]] = shl i64 [[TMP0]], 3
-; FIXED-NEXT:    [[TMP2:%.*]] = getelementptr i64, ptr [[P:%.*]], i64 [[TMP1]]
-; FIXED-NEXT:    [[WIDE_VEC:%.*]] = load <16 x i64>, ptr [[TMP2]], align 8
+; FIXED-NEXT:    [[I:%.*]] = phi i64 [ 0, [[ENTRY]] ], [ [[NEXTI:%.*]], [[LOOP]] ]
+; FIXED-NEXT:    [[TMP0:%.*]] = add i64 [[I]], 0
+; FIXED-NEXT:    [[OFFSET0:%.*]] = shl i64 [[TMP0]], 3
+; FIXED-NEXT:    [[Q0:%.*]] = getelementptr i64, ptr [[P:%.*]], i64 [[OFFSET0]]
+; FIXED-NEXT:    [[WIDE_VEC:%.*]] = load <16 x i64>, ptr [[Q0]], align 8
 ; FIXED-NEXT:    [[STRIDED_VEC:%.*]] = shufflevector <16 x i64> [[WIDE_VEC]], <16 x i64> poison, <2 x i32> <i32 0, i32 8>
 ; FIXED-NEXT:    [[STRIDED_VEC1:%.*]] = shufflevector <16 x i64> [[WIDE_VEC]], <16 x i64> poison, <2 x i32> <i32 1, i32 9>
 ; FIXED-NEXT:    [[STRIDED_VEC2:%.*]] = shufflevector <16 x i64> [[WIDE_VEC]], <16 x i64> poison, <2 x i32> <i32 2, i32 10>
@@ -849,39 +868,39 @@ define void @load_store_factor8(ptr %p) {
 ; FIXED-NEXT:    [[STRIDED_VEC5:%.*]] = shufflevector <16 x i64> [[WIDE_VEC]], <16 x i64> poison, <2 x i32> <i32 5, i32 13>
 ; FIXED-NEXT:    [[STRIDED_VEC6:%.*]] = shufflevector <16 x i64> [[WIDE_VEC]], <16 x i64> poison, <2 x i32> <i32 6, i32 14>
 ; FIXED-NEXT:    [[STRIDED_VEC7:%.*]] = shufflevector <16 x i64> [[WIDE_VEC]], <16 x i64> poison, <2 x i32> <i32 7, i32 15>
-; FIXED-NEXT:    [[TMP4:%.*]] = add <2 x i64> [[STRIDED_VEC]], splat (i64 1)
-; FIXED-NEXT:    [[TMP6:%.*]] = add <2 x i64> [[STRIDED_VEC1]], splat (i64 2)
-; FIXED-NEXT:    [[TMP8:%.*]] = add <2 x i64> [[STRIDED_VEC2]], splat (i64 3)
-; FIXED-NEXT:    [[TMP10:%.*]] = add <2 x i64> [[STRIDED_VEC3]], splat (i64 4)
-; FIXED-NEXT:    [[TMP12:%.*]] = add <2 x i64> [[STRIDED_VEC4]], splat (i64 5)
-; FIXED-NEXT:    [[TMP14:%.*]] = add <2 x i64> [[STRIDED_VEC5]], splat (i64 6)
-; FIXED-NEXT:    [[TMP16:%.*]] = add <2 x i64> [[STRIDED_VEC6]], splat (i64 7)
-; FIXED-NEXT:    [[TMP19:%.*]] = add <2 x i64> [[STRIDED_VEC7]], splat (i64 8)
-; FIXED-NEXT:    [[TMP21:%.*]] = shufflevector <2 x i64> [[TMP4]], <2 x i64> [[TMP6]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; FIXED-NEXT:    [[TMP22:%.*]] = shufflevector <2 x i64> [[TMP8]], <2 x i64> [[TMP10]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; FIXED-NEXT:    [[TMP23:%.*]] = shufflevector <2 x i64> [[TMP12]], <2 x i64> [[TMP14]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; FIXED-NEXT:    [[TMP24:%.*]] = shufflevector <2 x i64> [[TMP16]], <2 x i64> [[TMP19]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; FIXED-NEXT:    [[TMP25:%.*]] = shufflevector <4 x i64> [[TMP21]], <4 x i64> [[TMP22]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; FIXED-NEXT:    [[TMP26:%.*]] = shufflevector <4 x i64> [[TMP23]], <4 x i64> [[TMP24]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; FIXED-NEXT:    [[TMP27:%.*]] = shufflevector <8 x i64> [[TMP25]], <8 x i64> [[TMP26]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; FIXED-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <16 x i64> [[TMP27]], <16 x i64> poison, <16 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14, i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15>
-; FIXED-NEXT:    store <16 x i64> [[INTERLEAVED_VEC]], ptr [[TMP2]], align 8
-; FIXED-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
-; FIXED-NEXT:    [[TMP28:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
-; FIXED-NEXT:    br i1 [[TMP28]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
+; FIXED-NEXT:    [[TMP3:%.*]] = add <2 x i64> [[STRIDED_VEC]], splat (i64 1)
+; FIXED-NEXT:    [[TMP4:%.*]] = add <2 x i64> [[STRIDED_VEC1]], splat (i64 2)
+; FIXED-NEXT:    [[TMP5:%.*]] = add <2 x i64> [[STRIDED_VEC2]], splat (i64 3)
+; FIXED-NEXT:    [[TMP6:%.*]] = add <2 x i64> [[STRIDED_VEC3]], splat (i64 4)
+; FIXED-NEXT:    [[TMP7:%.*]] = add <2 x i64> [[STRIDED_VEC4]], splat (i64 5)
+; FIXED-NEXT:    [[TMP8:%.*]] = add <2 x i64> [[STRIDED_VEC5]], splat (i64 6)
+; FIXED-NEXT:    [[TMP9:%.*]] = add <2 x i64> [[STRIDED_VEC6]], splat (i64 7)
+; FIXED-NEXT:    [[TMP10:%.*]] = add <2 x i64> [[STRIDED_VEC7]], splat (i64 8)
+; FIXED-NEXT:    [[TMP11:%.*]] = shufflevector <2 x i64> [[TMP3]], <2 x i64> [[TMP4]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; FIXED-NEXT:    [[TMP12:%.*]] = shufflevector <2 x i64> [[TMP5]], <2 x i64> [[TMP6]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; FIXED-NEXT:    [[TMP13:%.*]] = shufflevector <2 x i64> [[TMP7]], <2 x i64> [[TMP8]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; FIXED-NEXT:    [[TMP14:%.*]] = shufflevector <2 x i64> [[TMP9]], <2 x i64> [[TMP10]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; FIXED-NEXT:    [[TMP15:%.*]] = shufflevector <4 x i64> [[TMP11]], <4 x i64> [[TMP12]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; FIXED-NEXT:    [[TMP16:%.*]] = shufflevector <4 x i64> [[TMP13]], <4 x i64> [[TMP14]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; FIXED-NEXT:    [[TMP17:%.*]] = shufflevector <8 x i64> [[TMP15]], <8 x i64> [[TMP16]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; FIXED-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <16 x i64> [[TMP17]], <16 x i64> poison, <16 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14, i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15>
+; FIXED-NEXT:    store <16 x i64> [[INTERLEAVED_VEC]], ptr [[Q0]], align 8
+; FIXED-NEXT:    [[NEXTI]] = add nuw i64 [[I]], 2
+; FIXED-NEXT:    [[TMP18:%.*]] = icmp eq i64 [[NEXTI]], 1024
+; FIXED-NEXT:    br i1 [[TMP18]], label [[MIDDLE_BLOCK:%.*]], label [[LOOP]], !llvm.loop [[LOOP10:![0-9]+]]
 ; FIXED:       middle.block:
 ; FIXED-NEXT:    br i1 true, label [[EXIT:%.*]], label [[SCALAR_PH]]
 ; FIXED:       scalar.ph:
-; FIXED-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 1024, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
-; FIXED-NEXT:    br label [[LOOP:%.*]]
+; FIXED-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 1024, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY1:%.*]] ]
+; FIXED-NEXT:    br label [[LOOP1:%.*]]
 ; FIXED:       loop:
-; FIXED-NEXT:    [[I:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[NEXTI:%.*]], [[LOOP]] ]
-; FIXED-NEXT:    [[OFFSET0:%.*]] = shl i64 [[I]], 3
-; FIXED-NEXT:    [[Q0:%.*]] = getelementptr i64, ptr [[P]], i64 [[OFFSET0]]
-; FIXED-NEXT:    [[X0:%.*]] = load i64, ptr [[Q0]], align 8
+; FIXED-NEXT:    [[I1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[NEXTI1:%.*]], [[LOOP1]] ]
+; FIXED-NEXT:    [[OFFSET8:%.*]] = shl i64 [[I1]], 3
+; FIXED-NEXT:    [[Q8:%.*]] = getelementptr i64, ptr [[P]], i64 [[OFFSET8]]
+; FIXED-NEXT:    [[X0:%.*]] = load i64, ptr [[Q8]], align 8
 ; FIXED-NEXT:    [[Y0:%.*]] = add i64 [[X0]], 1
-; FIXED-NEXT:    store i64 [[Y0]], ptr [[Q0]], align 8
-; FIXED-NEXT:    [[OFFSET1:%.*]] = add i64 [[OFFSET0]], 1
+; FIXED-NEXT:    store i64 [[Y0]], ptr [[Q8]], align 8
+; FIXED-NEXT:    [[OFFSET1:%.*]] = add i64 [[OFFSET8]], 1
 ; FIXED-NEXT:    [[Q1:%.*]] = getelementptr i64, ptr [[P]], i64 [[OFFSET1]]
 ; FIXED-NEXT:    [[X1:%.*]] = load i64, ptr [[Q1]], align 8
 ; FIXED-NEXT:    [[Y1:%.*]] = add i64 [[X1]], 2
@@ -916,64 +935,83 @@ define void @load_store_factor8(ptr %p) {
 ; FIXED-NEXT:    [[X7:%.*]] = load i64, ptr [[Q7]], align 8
 ; FIXED-NEXT:    [[Y7:%.*]] = add i64 [[X7]], 8
 ; FIXED-NEXT:    store i64 [[Y7]], ptr [[Q7]], align 8
-; FIXED-NEXT:    [[NEXTI]] = add i64 [[I]], 1
-; FIXED-NEXT:    [[DONE:%.*]] = icmp eq i64 [[NEXTI]], 1024
-; FIXED-NEXT:    br i1 [[DONE]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP11:![0-9]+]]
+; FIXED-NEXT:    [[NEXTI1]] = add i64 [[I1]], 1
+; FIXED-NEXT:    [[DONE:%.*]] = icmp eq i64 [[NEXTI1]], 1024
+; FIXED-NEXT:    br i1 [[DONE]], label [[EXIT]], label [[LOOP1]], !llvm.loop [[LOOP11:![0-9]+]]
 ; FIXED:       exit:
 ; FIXED-NEXT:    ret void
 ;
 ; SCALABLE-LABEL: @load_store_factor8(
 ; SCALABLE-NEXT:  entry:
-; SCALABLE-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; SCALABLE-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; SCALABLE-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP0]]
+; SCALABLE-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[ENTRY:%.*]]
 ; SCALABLE:       vector.ph:
-; SCALABLE-NEXT:    br label [[VECTOR_BODY:%.*]]
+; SCALABLE-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
+; SCALABLE-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP1]]
+; SCALABLE-NEXT:    [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
+; SCALABLE-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
+; SCALABLE-NEXT:    br label [[LOOP:%.*]]
 ; SCALABLE:       vector.body:
-; SCALABLE-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; SCALABLE-NEXT:    [[TMP0:%.*]] = add i64 [[INDEX]], 0
-; SCALABLE-NEXT:    [[TMP1:%.*]] = shl i64 [[TMP0]], 3
-; SCALABLE-NEXT:    [[TMP2:%.*]] = getelementptr i64, ptr [[P:%.*]], i64 [[TMP1]]
-; SCALABLE-NEXT:    [[WIDE_VEC:%.*]] = load <16 x i64>, ptr [[TMP2]], align 8
-; SCALABLE-NEXT:    [[STRIDED_VEC:%.*]] = shufflevector <16 x i64> [[WIDE_VEC]], <16 x i64> poison, <2 x i32> <i32 0, i32 8>
-; SCALABLE-NEXT:    [[STRIDED_VEC1:%.*]] = shufflevector <16 x i64> [[WIDE_VEC]], <16 x i64> poison, <2 x i32> <i32 1, i32 9>
-; SCALABLE-NEXT:    [[STRIDED_VEC2:%.*]] = shufflevector <16 x i64> [[WIDE_VEC]], <16 x i64> poison, <2 x i32> <i32 2, i32 10>
-; SCALABLE-NEXT:    [[STRIDED_VEC3:%.*]] = shufflevector <16 x i64> [[WIDE_VEC]], <16 x i64> poison, <2 x i32> <i32 3, i32 11>
-; SCALABLE-NEXT:    [[STRIDED_VEC4:%.*]] = shufflevector <16 x i64> [[WIDE_VEC]], <16 x i64> poison, <2 x i32> <i32 4, i32 12>
-; SCALABLE-NEXT:    [[STRIDED_VEC5:%.*]] = shufflevector <16 x i64> [[WIDE_VEC]], <16 x i64> poison, <2 x i32> <i32 5, i32 13>
-; SCALABLE-NEXT:    [[STRIDED_VEC6:%.*]] = shufflevector <16 x i64> [[WIDE_VEC]], <16 x i64> poison, <2 x i32> <i32 6, i32 14>
-; SCALABLE-NEXT:    [[STRIDED_VEC7:%.*]] = shufflevector <16 x i64> [[WIDE_VEC]], <16 x i64> poison, <2 x i32> <i32 7, i32 15>
-; SCALABLE-NEXT:    [[TMP4:%.*]] = add <2 x i64> [[STRIDED_VEC]], splat (i64 1)
-; SCALABLE-NEXT:    [[TMP6:%.*]] = add <2 x i64> [[STRIDED_VEC1]], splat (i64 2)
-; SCALABLE-NEXT:    [[TMP8:%.*]] = add <2 x i64> [[STRIDED_VEC2]], splat (i64 3)
-; SCALABLE-NEXT:    [[TMP10:%.*]] = add <2 x i64> [[STRIDED_VEC3]], splat (i64 4)
-; SCALABLE-NEXT:    [[TMP12:%.*]] = add <2 x i64> [[STRIDED_VEC4]], splat (i64 5)
-; SCALABLE-NEXT:    [[TMP14:%.*]] = add <2 x i64> [[STRIDED_VEC5]], splat (i64 6)
-; SCALABLE-NEXT:    [[TMP16:%.*]] = add <2 x i64> [[STRIDED_VEC6]], splat (i64 7)
-; SCALABLE-NEXT:    [[TMP19:%.*]] = add <2 x i64> [[STRIDED_VEC7]], splat (i64 8)
-; SCALABLE-NEXT:    [[TMP21:%.*]] = shufflevector <2 x i64> [[TMP4]], <2 x i64> [[TMP6]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; SCALABLE-NEXT:    [[TMP22:%.*]] = shufflevector <2 x i64> [[TMP8]], <2 x i64> [[TMP10]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; SCALABLE-NEXT:    [[TMP23:%.*]] = shufflevector <2 x i64> [[TMP12]], <2 x i64> [[TMP14]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; SCALABLE-NEXT:    [[TMP24:%.*]] = shufflevector <2 x i64> [[TMP16]], <2 x i64> [[TMP19]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; SCALABLE-NEXT:    [[TMP25:%.*]] = shufflevector <4 x i64> [[TMP21]], <4 x i64> [[TMP22]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; SCALABLE-NEXT:    [[TMP26:%.*]] = shufflevector <4 x i64> [[TMP23]], <4 x i64> [[TMP24]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; SCALABLE-NEXT:    [[TMP27:%.*]] = shufflevector <8 x i64> [[TMP25]], <8 x i64> [[TMP26]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; SCALABLE-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <16 x i64> [[TMP27]], <16 x i64> poison, <16 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14, i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15>
-; SCALABLE-NEXT:    store <16 x i64> [[INTERLEAVED_VEC]], ptr [[TMP2]], align 8
-; SCALABLE-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
-; SCALABLE-NEXT:    [[TMP28:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
-; SCALABLE-NEXT:    br i1 [[TMP28]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
+; SCALABLE-NEXT:    [[I:%.*]] = phi i64 [ 0, [[ENTRY]] ], [ [[NEXTI:%.*]], [[LOOP]] ]
+; SCALABLE-NEXT:    [[TMP3:%.*]] = add i64 [[I]], 0
+; SCALABLE-NEXT:    [[OFFSET0:%.*]] = shl i64 [[TMP3]], 3
+; SCALABLE-NEXT:    [[Q0:%.*]] = getelementptr i64, ptr [[P:%.*]], i64 [[OFFSET0]]
+; SCALABLE-NEXT:    [[WIDE_VEC:%.*]] = load <vscale x 8 x i64>, ptr [[Q0]], align 8
+; SCALABLE-NEXT:    [[STRIDED_VEC:%.*]] = call { <vscale x 4 x i64>, <vscale x 4 x i64> } @llvm.vector.deinterleave2.nxv8i64(<vscale x 8 x i64> [[WIDE_VEC]])
+; SCALABLE-NEXT:    [[TMP6:%.*]] = extractvalue { <vscale x 4 x i64>, <vscale x 4 x i64> } [[STRIDED_VEC]], 0
+; SCALABLE-NEXT:    [[TMP7:%.*]] = extractvalue { <vscale x 4 x i64>, <vscale x 4 x i64> } [[STRIDED_VEC]], 1
+; SCALABLE-NEXT:    [[STRIDED_VEC1:%.*]] = call { <vscale x 2 x i64>, <vscale x 2 x i64> } @llvm.vector.deinterleave2.nxv4i64(<vscale x 4 x i64> [[TMP6]])
+; SCALABLE-NEXT:    [[STRIDED_VEC2:%.*]] = call { <vscale x 2 x i64>, <vscale x 2 x i64> } @llvm.vector.deinterleave2.nxv4i64(<vscale x 4 x i64> [[TMP7]])
+; SCALABLE-NEXT:    [[TMP8:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64> } [[STRIDED_VEC1]], 0
+; SCALABLE-NEXT:    [[TMP9:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64> } [[STRIDED_VEC2]], 0
+; SCALABLE-NEXT:    [[TMP10:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64> } [[STRIDED_VEC1]], 1
+; SCALABLE-NEXT:    [[TMP11:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64> } [[STRIDED_VEC2]], 1
+; SCALABLE-NEXT:    [[STRIDED_VEC3:%.*]] = call { <vscale x 1 x i64>, <vscale x 1 x i64> } @llvm.vector.deinterleave2.nxv2i64(<vscale x 2 x i64> [[TMP8]])
+; SCALABLE-NEXT:    [[STRIDED_VEC4:%.*]] = call { <vscale x 1 x i64>, <vscale x 1 x i64> } @llvm.vector.deinterleave2.nxv2i64(<vscale x 2 x i64> [[TMP9]])
+; SCALABLE-NEXT:    [[STRIDED_VEC5:%.*]] = call { <vscale x 1 x i64>, <vscale x 1 x i64> } @llvm.vector.deinterleave2.nxv2i64(<vscale x 2 x i64> [[TMP10]])
+; SCALABLE-NEXT:    [[STRIDED_VEC6:%.*]] = call { <vscale x 1 x i64>, <vscale x 1 x i64> } @llvm.vector.deinterleave2.nxv2i64(<vscale x 2 x i64> [[TMP11]])
+; SCALABLE-NEXT:    [[TMP12:%.*]] = extractvalue { <vscale x 1 x i64>, <vscale x 1 x i64> } [[STRIDED_VEC3]], 0
+; SCALABLE-NEXT:    [[TMP13:%.*]] = extractvalue { <vscale x 1 x i64>, <vscale x 1 x i64> } [[STRIDED_VEC4]], 0
+; SCALABLE-NEXT:    [[TMP14:%.*]] = extractvalue { <vscale x 1 x i64>, <vscale x 1 x i64> } [[STRIDED_VEC5]], 0
+; SCALABLE-NEXT:    [[TMP15:%.*]] = extractvalue { <vscale x 1 x i64>, <vscale x 1 x i64> } [[STRIDED_VEC6]], 0
+; SCALABLE-NEXT:    [[TMP16:%.*]] = extractvalue { <vscale x 1 x i64>, <vscale x 1 x i64> } [[STRIDED_VEC3]], 1
+; SCALABLE-NEXT:    [[TMP17:%.*]] = extractvalue { <vscale x 1 x i64>, <vscale x 1 x i64> } [[STRIDED_VEC4]], 1
+; SCALABLE-NEXT:    [[TMP18:%.*]] = extractvalue { <vscale x 1 x i64>, <vscale x 1 x i64> } [[STRIDED_VEC5]], 1
+; SCALABLE-NEXT:    [[TMP19:%.*]] = extractvalue { <vscale x 1 x i64>, <vscale x 1 x i64> } [[STRIDED_VEC6]], 1
+; SCALABLE-NEXT:    [[TMP20:%.*]] = add <vscale x 1 x i64> [[TMP12]], splat (i64 1)
+; SCALABLE-NEXT:    [[TMP21:%.*]] = add <vscale x 1 x i64> [[TMP13]], splat (i64 2)
+; SCALABLE-NEXT:    [[TMP22:%.*]] = add <vscale x 1 x i64> [[TMP14]], splat (i64 3)
+; SCALABLE-NEXT:    [[TMP23:%.*]] = add <vscale x 1 x i64> [[TMP15]], splat (i64 4)
+; SCALABLE-NEXT:    [[TMP24:%.*]] = add <vscale x 1 x i64> [[TMP16]], splat (i64 5)
+; SCALABLE-NEXT:    [[TMP25:%.*]] = add <vscale x 1 x i64> [[TMP17]], splat (i64 6)
+; SCALABLE-NEXT:    [[TMP26:%.*]] = add <vscale x 1 x i64> [[TMP18]], splat (i64 7)
+; SCALABLE-NEXT:    [[TMP27:%.*]] = add <vscale x 1 x i64> [[TMP19]], splat (i64 8)
+; SCALABLE-NEXT:    [[INTERLEAVED_VEC:%.*]] = call <vscale x 2 x i64> @llvm.vector.interleave2.nxv2i64(<vscale x 1 x i64> [[TMP20]], <vscale x 1 x i64> [[TMP24]])
+; SCALABLE-NEXT:    [[INTERLEAVED_VEC7:%.*]] = call <vscale x 2 x i64> @llvm.vector.interleave2.nxv2i64(<vscale x 1 x i64> [[TMP21]], <vscale x 1 x i64> [[TMP25]])
+; SCALABLE-NEXT:    [[INTERLEAVED_VEC8:%.*]] = call <vscale x 2 x i64> @llvm.vector.interleave2.nxv2i64(<vscale x 1 x i64> [[TMP22]], <vscale x 1 x i64> [[TMP26]])
+; SCALABLE-NEXT:    [[INTERLEAVED_VEC9:%.*]] = call <vscale x 2 x i64> @llvm.vector.interleave2.nxv2i64(<vscale x 1 x i64> [[TMP23]], <vscale x 1 x i64> [[TMP27]])
+; SCALABLE-NEXT:    [[INTERLEAVED_VEC10:%.*]] = call <vscale x 4 x i64> @llvm.vector.interleave2.nxv4i64(<vscale x 2 x i64> [[INTERLEAVED_VEC]], <vscale x 2 x i64> [[INTERLEAVED_VEC8]])
+; SCALABLE-NEXT:    [[INTERLEAVED_VEC11:%.*]] = call <vscale x 4 x i64> @llvm.vector.interleave2.nxv4i64(<vscale x 2 x i64> [[INTERLEAVED_VEC7]], <vscale x 2 x i64> [[INTERLEAVED_VEC9]])
+; SCALABLE-NEXT:    [[INTERLEAVED_VEC12:%.*]] = call <vscale x 8 x i64> @llvm.vector.interleave2.nxv8i64(<vscale x 4 x i64> [[INTERLEAVED_VEC10]], <vscale x 4 x i64> [[INTERLEAVED_VEC11]])
+; SCALABLE-NEXT:    store <vscale x 8 x i64> [[INTERLEAVED_VEC12]], ptr [[Q0]], align 8
+; SCALABLE-NEXT:    [[NEXTI]] = add nuw i64 [[I]], [[TMP2]]
+; SCALABLE-NEXT:    [[TMP28:%.*]] = icmp eq i64 [[NEXTI]], [[N_VEC]]
+; SCALABLE-NEXT:    br i1 [[TMP28]], label [[MIDDLE_BLOCK:%.*]], label [[LOOP]], !llvm.loop [[LOOP10:![0-9]+]]
 ; SCALABLE:       middle.block:
-; SCALABLE-NEXT:    br i1 true, label [[EXIT:%.*]], label [[SCALAR_PH]]
+; SCALABLE-NEXT:    [[CMP_N:%.*]] = icmp eq i64 1024, [[N_VEC]]
+; SCALABLE-NEXT:    br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
 ; SCALABLE:       scalar.ph:
-; SCALABLE-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 1024, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
-; SCALABLE-NEXT:    br label [[LOOP:%.*]]
+; SCALABLE-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY1:%.*]] ]
+; SCALABLE-NEXT:    br label [[LOOP1:%.*]]
 ; SCALABLE:       loop:
-; SCALABLE-NEXT:    [[I:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[NEXTI:%.*]], [[LOOP]] ]
-; SCALABLE-NEXT:    [[OFFSET0:%.*]] = shl i64 [[I]], 3
-; SCALABLE-NEXT:    [[Q0:%.*]] = getelementptr i64, ptr [[P]], i64 [[OFFSET0]]
-; SCALABLE-NEXT:    [[X0:%.*]] = load i64, ptr [[Q0]], align 8
+; SCALABLE-NEXT:    [[I1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[NEXTI1:%.*]], [[LOOP1]] ]
+; SCALABLE-NEXT:    [[OFFSET8:%.*]] = shl i64 [[I1]], 3
+; SCALABLE-NEXT:    [[Q8:%.*]] = getelementptr i64, ptr [[P]], i64 [[OFFSET8]]
+; SCALABLE-NEXT:    [[X0:%.*]] = load i64, ptr [[Q8]], align 8
 ; SCALABLE-NEXT:    [[Y0:%.*]] = add i64 [[X0]], 1
-; SCALABLE-NEXT:    store i64 [[Y0]], ptr [[Q0]], align 8
-; SCALABLE-NEXT:    [[OFFSET1:%.*]] = add i64 [[OFFSET0]], 1
+; SCALABLE-NEXT:    store i64 [[Y0]], ptr [[Q8]], align 8
+; SCALABLE-NEXT:    [[OFFSET1:%.*]] = add i64 [[OFFSET8]], 1
 ; SCALABLE-NEXT:    [[Q1:%.*]] = getelementptr i64, ptr [[P]], i64 [[OFFSET1]]
 ; SCALABLE-NEXT:    [[X1:%.*]] = load i64, ptr [[Q1]], align 8
 ; SCALABLE-NEXT:    [[Y1:%.*]] = add i64 [[X1]], 2
@@ -1008,9 +1046,9 @@ define void @load_store_factor8(ptr %p) {
 ; SCALABLE-NEXT:    [[X7:%.*]] = load i64, ptr [[Q7]], align 8
 ; SCALABLE-NEXT:    [[Y7:%.*]] = add i64 [[X7]], 8
 ; SCALABLE-NEXT:    store i64 [[Y7]], ptr [[Q7]], align 8
-; SCALABLE-NEXT:    [[NEXTI]] = add i64 [[I]], 1
-; SCALABLE-NEXT:    [[DONE:%.*]] = icmp eq i64 [[NEXTI]], 1024
-; SCALABLE-NEXT:    br i1 [[DONE]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP11:![0-9]+]]
+; SCALABLE-NEXT:    [[NEXTI1]] = add i64 [[I1]], 1
+; SCALABLE-NEXT:    [[DONE:%.*]] = icmp eq i64 [[NEXTI1]], 1024
+; SCALABLE-NEXT:    br i1 [[DONE]], label [[EXIT]], label [[LOOP1]], !llvm.loop [[LOOP11:![0-9]+]]
 ; SCALABLE:       exit:
 ; SCALABLE-NEXT:    ret void
 ;
@@ -1080,7 +1118,7 @@ define void @combine_load_factor2_i32(ptr noalias %p, ptr noalias %q) {
 ; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
 ; CHECK-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 4
 ; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP1]]
-; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[ENTRY:%.*]]
 ; CHECK:       vector.ph:
 ; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
 ; CHECK-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 4
@@ -1088,94 +1126,94 @@ define void @combine_load_factor2_i32(ptr noalias %p, ptr noalias %q) {
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
 ; CHECK-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
 ; CHECK-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 4
-; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       vector.body:
-; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[TMP6:%.*]] = add i64 [[INDEX]], 0
-; CHECK-NEXT:    [[TMP7:%.*]] = shl i64 [[TMP6]], 1
-; CHECK-NEXT:    [[TMP8:%.*]] = getelementptr i32, ptr [[P:%.*]], i64 [[TMP7]]
-; CHECK-NEXT:    [[WIDE_VEC:%.*]] = load <vscale x 8 x i32>, ptr [[TMP8]], align 4
+; CHECK-NEXT:    [[I:%.*]] = phi i64 [ 0, [[ENTRY]] ], [ [[NEXTI:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[TMP6:%.*]] = add i64 [[I]], 0
+; CHECK-NEXT:    [[OFFSET0:%.*]] = shl i64 [[TMP6]], 1
+; CHECK-NEXT:    [[Q0:%.*]] = getelementptr i32, ptr [[P:%.*]], i64 [[OFFSET0]]
+; CHECK-NEXT:    [[WIDE_VEC:%.*]] = load <vscale x 8 x i32>, ptr [[Q0]], align 4
 ; CHECK-NEXT:    [[STRIDED_VEC:%.*]] = call { <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.vector.deinterleave2.nxv8i32(<vscale x 8 x i32> [[WIDE_VEC]])
-; CHECK-NEXT:    [[TMP10:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } [[STRIDED_VEC]], 0
-; CHECK-NEXT:    [[TMP11:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } [[STRIDED_VEC]], 1
-; CHECK-NEXT:    [[TMP12:%.*]] = add <vscale x 4 x i32> [[TMP10]], [[TMP11]]
-; CHECK-NEXT:    [[TMP13:%.*]] = getelementptr i32, ptr [[Q:%.*]], i64 [[TMP6]]
-; CHECK-NEXT:    [[TMP14:%.*]] = getelementptr i32, ptr [[TMP13]], i32 0
-; CHECK-NEXT:    store <vscale x 4 x i32> [[TMP12]], ptr [[TMP14]], align 4
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
-; CHECK-NEXT:    [[TMP15:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT:    br i1 [[TMP15]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
+; CHECK-NEXT:    [[TMP9:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } [[STRIDED_VEC]], 0
+; CHECK-NEXT:    [[TMP10:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } [[STRIDED_VEC]], 1
+; CHECK-NEXT:    [[TMP11:%.*]] = add <vscale x 4 x i32> [[TMP9]], [[TMP10]]
+; CHECK-NEXT:    [[TMP12:%.*]] = getelementptr i32, ptr [[Q:%.*]], i64 [[TMP6]]
+; CHECK-NEXT:    [[TMP13:%.*]] = getelementptr i32, ptr [[TMP12]], i32 0
+; CHECK-NEXT:    store <vscale x 4 x i32> [[TMP11]], ptr [[TMP13]], align 4
+; CHECK-NEXT:    [[NEXTI]] = add nuw i64 [[I]], [[TMP5]]
+; CHECK-NEXT:    [[TMP14:%.*]] = icmp eq i64 [[NEXTI]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[TMP14]], label [[MIDDLE_BLOCK:%.*]], label [[LOOP]], !llvm.loop [[LOOP12:![0-9]+]]
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 1024, [[N_VEC]]
 ; CHECK-NEXT:    br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
-; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY1:%.*]] ]
+; CHECK-NEXT:    br label [[LOOP1:%.*]]
 ; CHECK:       loop:
-; CHECK-NEXT:    [[I:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[NEXTI:%.*]], [[LOOP]] ]
-; CHECK-NEXT:    [[OFFSET0:%.*]] = shl i64 [[I]], 1
-; CHECK-NEXT:    [[Q0:%.*]] = getelementptr i32, ptr [[P]], i64 [[OFFSET0]]
-; CHECK-NEXT:    [[X0:%.*]] = load i32, ptr [[Q0]], align 4
-; CHECK-NEXT:    [[OFFSET1:%.*]] = add i64 [[OFFSET0]], 1
+; CHECK-NEXT:    [[I1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[NEXTI1:%.*]], [[LOOP1]] ]
+; CHECK-NEXT:    [[OFFSET2:%.*]] = shl i64 [[I1]], 1
+; CHECK-NEXT:    [[Q2:%.*]] = getelementptr i32, ptr [[P]], i64 [[OFFSET2]]
+; CHECK-NEXT:    [[X0:%.*]] = load i32, ptr [[Q2]], align 4
+; CHECK-NEXT:    [[OFFSET1:%.*]] = add i64 [[OFFSET2]], 1
 ; CHECK-NEXT:    [[Q1:%.*]] = getelementptr i32, ptr [[P]], i64 [[OFFSET1]]
 ; CHECK-NEXT:    [[X1:%.*]] = load i32, ptr [[Q1]], align 4
 ; CHECK-NEXT:    [[RES:%.*]] = add i32 [[X0]], [[X1]]
-; CHECK-NEXT:    [[DST:%.*]] = getelementptr i32, ptr [[Q]], i64 [[I]]
+; CHECK-NEXT:    [[DST:%.*]] = getelementptr i32, ptr [[Q]], i64 [[I1]]
 ; CHECK-NEXT:    store i32 [[RES]], ptr [[DST]], align 4
-; CHECK-NEXT:    [[NEXTI]] = add i64 [[I]], 1
-; CHECK-NEXT:    [[DONE:%.*]] = icmp eq i64 [[NEXTI]], 1024
-; CHECK-NEXT:    br i1 [[DONE]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP13:![0-9]+]]
+; CHECK-NEXT:    [[NEXTI1]] = add i64 [[I1]], 1
+; CHECK-NEXT:    [[DONE:%.*]] = icmp eq i64 [[NEXTI1]], 1024
+; CHECK-NEXT:    br i1 [[DONE]], label [[EXIT]], label [[LOOP1]], !llvm.loop [[LOOP13:![0-9]+]]
 ; CHECK:       exit:
 ; CHECK-NEXT:    ret void
 ;
 ; FIXED-LABEL: @combine_load_factor2_i32(
 ; FIXED-NEXT:  entry:
-; FIXED-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; FIXED-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[ENTRY:%.*]]
 ; FIXED:       vector.ph:
-; FIXED-NEXT:    br label [[VECTOR_BODY:%.*]]
+; FIXED-NEXT:    br label [[LOOP:%.*]]
 ; FIXED:       vector.body:
-; FIXED-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; FIXED-NEXT:    [[TMP0:%.*]] = add i64 [[INDEX]], 0
-; FIXED-NEXT:    [[TMP1:%.*]] = add i64 [[INDEX]], 8
-; FIXED-NEXT:    [[TMP2:%.*]] = shl i64 [[TMP0]], 1
+; FIXED-NEXT:    [[I:%.*]] = phi i64 [ 0, [[ENTRY]] ], [ [[NEXTI:%.*]], [[LOOP]] ]
+; FIXED-NEXT:    [[TMP0:%.*]] = add i64 [[I]], 0
+; FIXED-NEXT:    [[TMP1:%.*]] = add i64 [[I]], 8
+; FIXED-NEXT:    [[OFFSET0:%.*]] = shl i64 [[TMP0]], 1
 ; FIXED-NEXT:    [[TMP3:%.*]] = shl i64 [[TMP1]], 1
-; FIXED-NEXT:    [[TMP4:%.*]] = getelementptr i32, ptr [[P:%.*]], i64 [[TMP2]]
+; FIXED-NEXT:    [[Q0:%.*]] = getelementptr i32, ptr [[P:%.*]], i64 [[OFFSET0]]
 ; FIXED-NEXT:    [[TMP5:%.*]] = getelementptr i32, ptr [[P]], i64 [[TMP3]]
-; FIXED-NEXT:    [[WIDE_VEC:%.*]] = load <16 x i32>, ptr [[TMP4]], align 4
+; FIXED-NEXT:    [[WIDE_VEC:%.*]] = load <16 x i32>, ptr [[Q0]], align 4
 ; FIXED-NEXT:    [[STRIDED_VEC:%.*]] = shufflevector <16 x i32> [[WIDE_VEC]], <16 x i32> poison, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>
-; FIXED-NEXT:    [[STRIDED_VEC3:%.*]] = shufflevector <16 x i32> [[WIDE_VEC]], <16 x i32> poison, <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15>
-; FIXED-NEXT:    [[WIDE_VEC1:%.*]] = load <16 x i32>, ptr [[TMP5]], align 4
-; FIXED-NEXT:    [[STRIDED_VEC2:%.*]] = shufflevector <16 x i32> [[WIDE_VEC1]], <16 x i32> poison, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>
-; FIXED-NEXT:    [[STRIDED_VEC4:%.*]] = shufflevector <16 x i32> [[WIDE_VEC1]], <16 x i32> poison, <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15>
-; FIXED-NEXT:    [[TMP8:%.*]] = add <8 x i32> [[STRIDED_VEC]], [[STRIDED_VEC3]]
-; FIXED-NEXT:    [[TMP9:%.*]] = add <8 x i32> [[STRIDED_VEC2]], [[STRIDED_VEC4]]
-; FIXED-NEXT:    [[TMP10:%.*]] = getelementptr i32, ptr [[Q:%.*]], i64 [[TMP0]]
-; FIXED-NEXT:    [[TMP12:%.*]] = getelementptr i32, ptr [[TMP10]], i32 0
-; FIXED-NEXT:    [[TMP13:%.*]] = getelementptr i32, ptr [[TMP10]], i32 8
-; FIXED-NEXT:    store <8 x i32> [[TMP8]], ptr [[TMP12]], align 4
-; FIXED-NEXT:    store <8 x i32> [[TMP9]], ptr [[TMP13]], align 4
-; FIXED-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
-; FIXED-NEXT:    [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
-; FIXED-NEXT:    br i1 [[TMP14]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
+; FIXED-NEXT:    [[STRIDED_VEC1:%.*]] = shufflevector <16 x i32> [[WIDE_VEC]], <16 x i32> poison, <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15>
+; FIXED-NEXT:    [[WIDE_VEC2:%.*]] = load <16 x i32>, ptr [[TMP5]], align 4
+; FIXED-NEXT:    [[STRIDED_VEC3:%.*]] = shufflevector <16 x i32> [[WIDE_VEC2]], <16 x i32> poison, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>
+; FIXED-NEXT:    [[STRIDED_VEC4:%.*]] = shufflevector <16 x i32> [[WIDE_VEC2]], <16 x i32> poison, <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15>
+; FIXED-NEXT:    [[TMP6:%.*]] = add <8 x i32> [[STRIDED_VEC]], [[STRIDED_VEC1]]
+; FIXED-NEXT:    [[TMP7:%.*]] = add <8 x i32> [[STRIDED_VEC3]], [[STRIDED_VEC4]]
+; FIXED-NEXT:    [[TMP8:%.*]] = getelementptr i32, ptr [[Q:%.*]], i64 [[TMP0]]
+; FIXED-NEXT:    [[TMP9:%.*]] = getelementptr i32, ptr [[TMP8]], i32 0
+; FIXED-NEXT:    [[TMP10:%.*]] = getelementptr i32, ptr [[TMP8]], i32 8
+; FIXED-NEXT:    store <8 x i32> [[TMP6]], ptr [[TMP9]], align 4
+; FIXED-NEXT:    store <8 x i32> [[TMP7]], ptr [[TMP10]], align 4
+; FIXED-NEXT:    [[NEXTI]] = add nuw i64 [[I]], 16
+; FIXED-NEXT:    [[TMP11:%.*]] = icmp eq i64 [[NEXTI]], 1024
+; FIXED-NEXT:    br i1 [[TMP11]], label [[MIDDLE_BLOCK:%.*]], label [[LOOP]], !llvm.loop [[LOOP12:![0-9]+]]
 ; FIXED:       middle.block:
 ; FIXED-NEXT:    br i1 true, label [[EXIT:%.*]], label [[SCALAR_PH]]
 ; FIXED:       scalar.ph:
-; FIXED-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 1024, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
-; FIXED-NEXT:    br label [[LOOP:%.*]]
+; FIXED-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 1024, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY1:%.*]] ]
+; FIXED-NEXT:    br label [[LOOP1:%.*]]
 ; FIXED:       loop:
-; FIXED-NEXT:    [[I:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[NEXTI:%.*]], [[LOOP]] ]
-; FIXED-NEXT:    [[OFFSET0:%.*]] = shl i64 [[I]], 1
-; FIXED-NEXT:    [[Q0:%.*]] = getelementptr i32, ptr [[P]], i64 [[OFFSET0]]
-; FIXED-NEXT:    [[X0:%.*]] = load i32, ptr [[Q0]], align 4
-; FIXED-NEXT:    [[OFFSET1:%.*]] = add i64 [[OFFSET0]], 1
+; FIXED-NEXT:    [[I1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[NEXTI1:%.*]], [[LOOP1]] ]
+; FIXED-NEXT:    [[OFFSET2:%.*]] = shl i64 [[I1]], 1
+; FIXED-NEXT:    [[Q2:%.*]] = getelementptr i32, ptr [[P]], i64 [[OFFSET2]]
+; FIXED-NEXT:    [[X0:%.*]] = load i32, ptr [[Q2]], align 4
+; FIXED-NEXT:    [[OFFSET1:%.*]] = add i64 [[OFFSET2]], 1
 ; FIXED-NEXT:    [[Q1:%.*]] = getelementptr i32, ptr [[P]], i64 [[OFFSET1]]
 ; FIXED-NEXT:    [[X1:%.*]] = load i32, ptr [[Q1]], align 4
 ; FIXED-NEXT:    [[RES:%.*]] = add i32 [[X0]], [[X1]]
-; FIXED-NEXT:    [[DST:%.*]] = getelementptr i32, ptr [[Q]], i64 [[I]]
+; FIXED-NEXT:    [[DST:%.*]] = getelementptr i32, ptr [[Q]], i64 [[I1]]
 ; FIXED-NEXT:    store i32 [[RES]], ptr [[DST]], align 4
-; FIXED-NEXT:    [[NEXTI]] = add i64 [[I]], 1
-; FIXED-NEXT:    [[DONE:%.*]] = icmp eq i64 [[NEXTI]], 1024
-; FIXED-NEXT:    br i1 [[DONE]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP13:![0-9]+]]
+; FIXED-NEXT:    [[NEXTI1]] = add i64 [[I1]], 1
+; FIXED-NEXT:    [[DONE:%.*]] = icmp eq i64 [[NEXTI1]], 1024
+; FIXED-NEXT:    br i1 [[DONE]], label [[EXIT]], label [[LOOP1]], !llvm.loop [[LOOP13:![0-9]+]]
 ; FIXED:       exit:
 ; FIXED-NEXT:    ret void
 ;
@@ -1184,7 +1222,7 @@ define void @combine_load_factor2_i32(ptr noalias %p, ptr noalias %q) {
 ; SCALABLE-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
 ; SCALABLE-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 4
 ; SCALABLE-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP1]]
-; SCALABLE-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; SCALABLE-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[ENTRY:%.*]]
 ; SCALABLE:       vector.ph:
 ; SCALABLE-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
 ; SCALABLE-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 4
@@ -1192,43 +1230,43 @@ define void @combine_load_factor2_i32(ptr noalias %p, ptr noalias %q) {
 ; SCALABLE-NEXT:    [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
 ; SCALABLE-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
 ; SCALABLE-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 4
-; SCALABLE-NEXT:    br label [[VECTOR_BODY:%.*]]
+; SCALABLE-NEXT:    br label [[LOOP:%.*]]
 ; SCALABLE:       vector.body:
-; SCALABLE-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; SCALABLE-NEXT:    [[TMP6:%.*]] = add i64 [[INDEX]], 0
-; SCALABLE-NEXT:    [[TMP7:%.*]] = shl i64 [[TMP6]], 1
-; SCALABLE-NEXT:    [[TMP8:%.*]] = getelementptr i32, ptr [[P:%.*]], i64 [[TMP7]]
-; SCALABLE-NEXT:    [[WIDE_VEC:%.*]] = load <vscale x 8 x i32>, ptr [[TMP8]], align 4
+; SCALABLE-NEXT:    [[I:%.*]] = phi i64 [ 0, [[ENTRY]] ], [ [[NEXTI:%.*]], [[LOOP]] ]
+; SCALABLE-NEXT:    [[TMP6:%.*]] = add i64 [[I]], 0
+; SCALABLE-NEXT:    [[OFFSET0:%.*]] = shl i64 [[TMP6]], 1
+; SCALABLE-NEXT:    [[Q0:%.*]] = getelementptr i32, ptr [[P:%.*]], i64 [[OFFSET0]]
+; SCALABLE-NEXT:    [[WIDE_VEC:%.*]] = load <vscale x 8 x i32>, ptr [[Q0]], align 4
 ; SCALABLE-NEXT:    [[STRIDED_VEC:%.*]] = call { <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.vector.deinterleave2.nxv8i32(<vscale x 8 x i32> [[WIDE_VEC]])
-; SCALABLE-NEXT:    [[TMP10:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } [[STRIDED_VEC]], 0
-; SCALABLE-NEXT:    [[TMP11:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } [[STRIDED_VEC]], 1
-; SCALABLE-NEXT:    [[TMP12:%.*]] = add <vscale x 4 x i32> [[TMP10]], [[TMP11]]
-; SCALABLE-NEXT:    [[TMP13:%.*]] = getelementptr i32, ptr [[Q:%.*]], i64 [[TMP6]]
-; SCALABLE-NEXT:    [[TMP14:%.*]] = getelementptr i32, ptr [[TMP13]], i32 0
-; SCALABLE-NEXT:    store <vscale x 4 x i32> [[TMP12]], ptr [[TMP14]], align 4
-; SCALABLE-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
-; SCALABLE-NEXT:    [[TMP15:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; SCALABLE-NEXT:    br i1 [[TMP15]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
+; SCALABLE-NEXT:    [[TMP9:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } [[STRIDED_VEC]], 0
+; SCALABLE-NEXT:    [[TMP10:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } [[STRIDED_VEC]], 1
+; SCALABLE-NEXT:    [[TMP11:%.*]] = add <vscale x 4 x i32> [[TMP9]], [[TMP10]]
+; SCALABLE-NEXT:    [[TMP12:%.*]] = getelementptr i32, ptr [[Q:%.*]], i64 [[TMP6]]
+; SCALABLE-NEXT:    [[TMP13:%.*]] = getelementptr i32, ptr [[TMP12]], i32 0
+; SCALABLE-NEXT:    store <vscale x 4 x i32> [[TMP11]], ptr [[TMP13]], align 4
+; SCALABLE-NEXT:    [[NEXTI]] = add nuw i64 [[I]], [[TMP5]]
+; SCALABLE-NEXT:    [[TMP14:%.*]] = icmp eq i64 [[NEXTI]], [[N_VEC]]
+; SCALABLE-NEXT:    br i1 [[TMP14]], label [[MIDDLE_BLOCK:%.*]], label [[LOOP]], !llvm.loop [[LOOP12:![0-9]+]]
 ; SCALABLE:       middle.block:
 ; SCALABLE-NEXT:    [[CMP_N:%.*]] = icmp eq i64 1024, [[N_VEC]]
 ; SCALABLE-NEXT:    br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
 ; SCALABLE:       scalar.ph:
-; SCALABLE-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
-; SCALABLE-NEXT:    br label [[LOOP:%.*]]
+; SCALABLE-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY1:%.*]] ]
+; SCALABLE-NEXT:    br label [[LOOP1:%.*]]
 ; SCALABLE:       loop:
-; SCALABLE-NEXT:    [[I:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[NEXTI:%.*]], [[LOOP]] ]
-; SCALABLE-NEXT:    [[OFFSET0:%.*]] = shl i64 [[I]], 1
-; SCALABLE-NEXT:    [[Q0:%.*]] = getelementptr i32, ptr [[P]], i64 [[OFFSET0]]
-; SCALABLE-NEXT:    [[X0:%.*]] = load i32, ptr [[Q0]], align 4
-; SCALABLE-NEXT:    [[OFFSET1:%.*]] = add i64 [[OFFSET0]], 1
+; SCALABLE-NEXT:    [[I1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[NEXTI1:%.*]], [[LOOP1]] ]
+; SCALABLE-NEXT:    [[OFFSET2:%.*]] = shl i64 [[I1]], 1
+; SCALABLE-NEXT:    [[Q2:%.*]] = getelementptr i32, ptr [[P]], i64 [[OFFSET2]]
+; SCALABLE-NEXT:    [[X0:%.*]] = load i32, ptr [[Q2]], align 4
+; SCALABLE-NEXT:    [[OFFSET1:%.*]] = add i64 [[OFFSET2]], 1
 ; SCALABLE-NEXT:    [[Q1:%.*]] = getelementptr i32, ptr [[P]], i64 [[OFFSET1]]
 ; SCALABLE-NEXT:    [[X1:%.*]] = load i32, ptr [[Q1]], align 4
 ; SCALABLE-NEXT:    [[RES:%.*]] = add i32 [[X0]], [[X1]]
-; SCALABLE-NEXT:    [[DST:%.*]] = getelementptr i32, ptr [[Q]], i64 [[I]]
+; SCALABLE-NEXT:    [[DST:%.*]] = getelementptr i32, ptr [[Q]], i64 [[I1]]
 ; SCALABLE-NEXT:    store i32 [[RES]], ptr [[DST]], align 4
-; SCALABLE-NEXT:    [[NEXTI]] = add i64 [[I]], 1
-; SCALABLE-NEXT:    [[DONE:%.*]] = icmp eq i64 [[NEXTI]], 1024
-; SCALABLE-NEXT:    br i1 [[DONE]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP13:![0-9]+]]
+; SCALABLE-NEXT:    [[NEXTI1]] = add i64 [[I1]], 1
+; SCALABLE-NEXT:    [[DONE:%.*]] = icmp eq i64 [[NEXTI1]], 1024
+; SCALABLE-NEXT:    br i1 [[DONE]], label [[EXIT]], label [[LOOP1]], !llvm.loop [[LOOP13:![0-9]+]]
 ; SCALABLE:       exit:
 ; SCALABLE-NEXT:    ret void
 ;
@@ -1263,7 +1301,7 @@ define void @combine_load_factor2_i64(ptr noalias %p, ptr noalias %q) {
 ; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
 ; CHECK-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 2
 ; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP1]]
-; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[ENTRY:%.*]]
 ; CHECK:       vector.ph:
 ; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
 ; CHECK-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 2
@@ -1271,94 +1309,94 @@ define void @combine_load_factor2_i64(ptr noalias %p, ptr noalias %q) {
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
 ; CHECK-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
 ; CHECK-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 2
-; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       vector.body:
-; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[TMP6:%.*]] = add i64 [[INDEX]], 0
-; CHECK-NEXT:    [[TMP7:%.*]] = shl i64 [[TMP6]], 1
-; CHECK-NEXT:    [[TMP8:%.*]] = getelementptr i64, ptr [[P:%.*]], i64 [[TMP7]]
-; CHECK-NEXT:    [[WIDE_VEC:%.*]] = load <vscale x 4 x i64>, ptr [[TMP8]], align 8
+; CHECK-NEXT:    [[I:%.*]] = phi i64 [ 0, [[ENTRY]] ], [ [[NEXTI:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[TMP6:%.*]] = add i64 [[I]], 0
+; CHECK-NEXT:    [[OFFSET0:%.*]] = shl i64 [[TMP6]], 1
+; CHECK-NEXT:    [[Q0:%.*]] = getelementptr i64, ptr [[P:%.*]], i64 [[OFFSET0]]
+; CHECK-NEXT:    [[WIDE_VEC:%.*]] = load <vscale x 4 x i64>, ptr [[Q0]], align 8
 ; CHECK-NEXT:    [[STRIDED_VEC:%.*]] = call { <vscale x 2 x i64>, <vscale x 2 x i64> } @llvm.vector.deinterleave2.nxv4i64(<vscale x 4 x i64> [[WIDE_VEC]])
-; CHECK-NEXT:    [[TMP10:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64> } [[STRIDED_VEC]], 0
-; CHECK-NEXT:    [[TMP11:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64> } [[STRIDED_VEC]], 1
-; CHECK-NEXT:    [[TMP12:%.*]] = add <vscale x 2 x i64> [[TMP10]], [[TMP11]]
-; CHECK-NEXT:    [[TMP13:%.*]] = getelementptr i64, ptr [[Q:%.*]], i64 [[TMP6]]
-; CHECK-NEXT:    [[TMP14:%.*]] = getelementptr i64, ptr [[TMP13]], i32 0
-; CHECK-NEXT:    store <vscale x 2 x i64> [[TMP12]], ptr [[TMP14]], align 8
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
-; CHECK-NEXT:    [[TMP15:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT:    br i1 [[TMP15]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
+; CHECK-NEXT:    [[TMP9:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64> } [[STRIDED_VEC]], 0
+; CHECK-NEXT:    [[TMP10:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64> } [[STRIDED_VEC]], 1
+; CHECK-NEXT:    [[TMP11:%.*]] = add <vscale x 2 x i64> [[TMP9]], [[TMP10]]
+; CHECK-NEXT:    [[TMP12:%.*]] = getelementptr i64, ptr [[Q:%.*]], i64 [[TMP6]]
+; CHECK-NEXT:    [[TMP13:%.*]] = getelementptr i64, ptr [[TMP12]], i32 0
+; CHECK-NEXT:    store <vscale x 2 x i64> [[TMP11]], ptr [[TMP13]], align 8
+; CHECK-NEXT:    [[NEXTI]] = add nuw i64 [[I]], [[TMP5]]
+; CHECK-NEXT:    [[TMP14:%.*]] = icmp eq i64 [[NEXTI]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[TMP14]], label [[MIDDLE_BLOCK:%.*]], label [[LOOP]], !llvm.loop [[LOOP14:![0-9]+]]
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 1024, [[N_VEC]]
 ; CHECK-NEXT:    br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
-; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY1:%.*]] ]
+; CHECK-NEXT:    br label [[LOOP1:%.*]]
 ; CHECK:       loop:
-; CHECK-NEXT:    [[I:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[NEXTI:%.*]], [[LOOP]] ]
-; CHECK-NEXT:    [[OFFSET0:%.*]] = shl i64 [[I]], 1
-; CHECK-NEXT:    [[Q0:%.*]] = getelementptr i64, ptr [[P]], i64 [[OFFSET0]]
-; CHECK-NEXT:    [[X0:%.*]] = load i64, ptr [[Q0]], align 8
-; CHECK-NEXT:    [[OFFSET1:%.*]] = add i64 [[OFFSET0]], 1
+; CHECK-NEXT:    [[I1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[NEXTI1:%.*]], [[LOOP1]] ]
+; CHECK-NEXT:    [[OFFSET2:%.*]] = shl i64 [[I1]], 1
+; CHECK-NEXT:    [[Q2:%.*]] = getelementptr i64, ptr [[P]], i64 [[OFFSET2]]
+; CHECK-NEXT:    [[X0:%.*]] = load i64, ptr [[Q2]], align 8
+; CHECK-NEXT:    [[OFFSET1:%.*]] = add i64 [[OFFSET2]], 1
 ; CHECK-NEXT:    [[Q1:%.*]] = getelementptr i64, ptr [[P]], i64 [[OFFSET1]]
 ; CHECK-NEXT:    [[X1:%.*]] = load i64, ptr [[Q1]], align 8
 ; CHECK-NEXT:    [[RES:%.*]] = add i64 [[X0]], [[X1]]
-; CHECK-NEXT:    [[DST:%.*]] = getelementptr i64, ptr [[Q]], i64 [[I]]
+; CHECK-NEXT:    [[DST:%.*]] = getelementptr i64, ptr [[Q]], i64 [[I1]]
 ; CHECK-NEXT:    store i64 [[RES]], ptr [[DST]], align 8
-; CHECK-NEXT:    [[NEXTI]] = add i64 [[I]], 1
-; CHECK-NEXT:    [[DONE:%.*]] = icmp eq i64 [[NEXTI]], 1024
-; CHECK-NEXT:    br i1 [[DONE]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP15:![0-9]+]]
+; CHECK-NEXT:    [[NEXTI1]] = add i64 [[I1]], 1
+; CHECK-NEXT:    [[DONE:%.*]] = icmp eq i64 [[NEXTI1]], 1024
+; CHECK-NEXT:    br i1 [[DONE]], label [[EXIT]], label [[LOOP1]], !llvm.loop [[LOOP15:![0-9]+]]
 ; CHECK:       exit:
 ; CHECK-NEXT:    ret void
 ;
 ; FIXED-LABEL: @combine_load_factor2_i64(
 ; FIXED-NEXT:  entry:
-; FIXED-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; FIXED-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[ENTRY:%.*]]
 ; FIXED:       vector.ph:
-; FIXED-NEXT:    br label [[VECTOR_BODY:%.*]]
+; FIXED-NEXT:    br label [[LOOP:%.*]]
 ; FIXED:       vector.body:
-; FIXED-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; FIXED-NEXT:    [[TMP0:%.*]] = add i64 [[INDEX]], 0
-; FIXED-NEXT:    [[TMP1:%.*]] = add i64 [[INDEX]], 4
-; FIXED-NEXT:    [[TMP2:%.*]] = shl i64 [[TMP0]], 1
+; FIXED-NEXT:    [[I:%.*]] = phi i64 [ 0, [[ENTRY]] ], [ [[NEXTI:%.*]], [[LOOP]] ]
+; FIXED-NEXT:    [[TMP0:%.*]] = add i64 [[I]], 0
+; FIXED-NEXT:    [[TMP1:%.*]] = add i64 [[I]], 4
+; FIXED-NEXT:    [[OFFSET0:%.*]] = shl i64 [[TMP0]], 1
 ; FIXED-NEXT:    [[TMP3:%.*]] = shl i64 [[TMP1]], 1
-; FIXED-NEXT:    [[TMP4:%.*]] = getelementptr i64, ptr [[P:%.*]], i64 [[TMP2]]
+; FIXED-NEXT:    [[Q0:%.*]] = getelementptr i64, ptr [[P:%.*]], i64 [[OFFSET0]]
 ; FIXED-NEXT:    [[TMP5:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP3]]
-; FIXED-NEXT:    [[WIDE_VEC:%.*]] = load <8 x i64>, ptr [[TMP4]], align 8
+; FIXED-NEXT:    [[WIDE_VEC:%.*]] = load <8 x i64>, ptr [[Q0]], align 8
 ; FIXED-NEXT:    [[STRIDED_VEC:%.*]] = shufflevector <8 x i64> [[WIDE_VEC]], <8 x i64> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
-; FIXED-NEXT:    [[STRIDED_VEC3:%.*]] = shufflevector <8 x i64> [[WIDE_VEC]], <8 x i64> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
-; FIXED-NEXT:    [[WIDE_VEC1:%.*]] = load <8 x i64>, ptr [[TMP5]], align 8
-; FIXED-NEXT:    [[STRIDED_VEC2:%.*]] = shufflevector <8 x i64> [[WIDE_VEC1]], <8 x i64> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
-; FIXED-NEXT:    [[STRIDED_VEC4:%.*]] = shufflevector <8 x i64> [[WIDE_VEC1]], <8 x i64> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
-; FIXED-NEXT:    [[TMP8:%.*]] = add <4 x i64> [[STRIDED_VEC]], [[STRIDED_VEC3]]
-; FIXED-NEXT:    [[TMP9:%.*]] = add <4 x i64> [[STRIDED_VEC2]], [[STRIDED_VEC4]]
-; FIXED-NEXT:    [[TMP10:%.*]] = getelementptr i64, ptr [[Q:%.*]], i64 [[TMP0]]
-; FIXED-NEXT:    [[TMP12:%.*]] = getelementptr i64, ptr [[TMP10]], i32 0
-; FIXED-NEXT:    [[TMP13:%.*]] = getelementptr i64, ptr [[TMP10]], i32 4
-; FIXED-NEXT:    store <4 x i64> [[TMP8]], ptr [[TMP12]], align 8
-; FIXED-NEXT:    store <4 x i64> [[TMP9]], ptr [[TMP13]], align 8
-; FIXED-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
-; FIXED-NEXT:    [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
-; FIXED-NEXT:    br i1 [[TMP14]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
+; FIXED-NEXT:    [[STRIDED_VEC1:%.*]] = shufflevector <8 x i64> [[WIDE_VEC]], <8 x i64> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+; FIXED-NEXT:    [[WIDE_VEC2:%.*]] = load <8 x i64>, ptr [[TMP5]], align 8
+; FIXED-NEXT:    [[STRIDED_VEC3:%.*]] = shufflevector <8 x i64> [[WIDE_VEC2]], <8 x i64> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+; FIXED-NEXT:    [[STRIDED_VEC4:%.*]] = shufflevector <8 x i64> [[WIDE_VEC2]], <8 x i64> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+; FIXED-NEXT:    [[TMP6:%.*]] = add <4 x i64> [[STRIDED_VEC]], [[STRIDED_VEC1]]
+; FIXED-NEXT:    [[TMP7:%.*]] = add <4 x i64> [[STRIDED_VEC3]], [[STRIDED_VEC4]]
+; FIXED-NEXT:    [[TMP8:%.*]] = getelementptr i64, ptr [[Q:%.*]], i64 [[TMP0]]
+; FIXED-NEXT:    [[TMP9:%.*]] = getelementptr i64, ptr [[TMP8]], i32 0
+; FIXED-NEXT:    [[TMP10:%.*]] = getelementptr i64, ptr [[TMP8]], i32 4
+; FIXED-NEXT:    store <4 x i64> [[TMP6]], ptr [[TMP9]], align 8
+; FIXED-NEXT:    store <4 x i64> [[TMP7]], ptr [[TMP10]], align 8
+; FIXED-NEXT:    [[NEXTI]] = add nuw i64 [[I]], 8
+; FIXED-NEXT:    [[TMP11:%.*]] = icmp eq i64 [[NEXTI]], 1024
+; FIXED-NEXT:    br i1 [[TMP11]], label [[MIDDLE_BLOCK:%.*]], label [[LOOP]], !llvm.loop [[LOOP14:![0-9]+]]
 ; FIXED:       middle.block:
 ; FIXED-NEXT:    br i1 true, label [[EXIT:%.*]], label [[SCALAR_PH]]
 ; FIXED:       scalar.ph:
-; FIXED-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 1024, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
-; FIXED-NEXT:    br label [[LOOP:%.*]]
+; FIXED-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 1024, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY1:%.*]] ]
+; FIXED-NEXT:    br label [[LOOP1:%.*]]
 ; FIXED:       loop:
-; FIXED-NEXT:    [[I:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[NEXTI:%.*]], [[LOOP]] ]
-; FIXED-NEXT:    [[OFFSET0:%.*]] = shl i64 [[I]], 1
-; FIXED-NEXT:    [[Q0:%.*]] = getelementptr i64, ptr [[P]], i64 [[OFFSET0]]
-; FIXED-NEXT:    [[X0:%.*]] = load i64, ptr [[Q0]], align 8
-; FIXED-NEXT:    [[OFFSET1:%.*]] = add i64 [[OFFSET0]], 1
+; FIXED-NEXT:    [[I1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[NEXTI1:%.*]], [[LOOP1]] ]
+; FIXED-NEXT:    [[OFFSET2:%.*]] = shl i64 [[I1]], 1
+; FIXED-NEXT:    [[Q2:%.*]] = getelementptr i64, ptr [[P]], i64 [[OFFSET2]]
+; FIXED-NEXT:    [[X0:%.*]] = load i64, ptr [[Q2]], align 8
+; FIXED-NEXT:    [[OFFSET1:%.*]] = add i64 [[OFFSET2]], 1
 ; FIXED-NEXT:    [[Q1:%.*]] = getelementptr i64, ptr [[P]], i64 [[OFFSET1]]
 ; FIXED-NEXT:    [[X1:%.*]] = load i64, ptr [[Q1]], align 8
 ; FIXED-NEXT:    [[RES:%.*]] = add i64 [[X0]], [[X1]]
-; FIXED-NEXT:    [[DST:%.*]] = getelementptr i64, ptr [[Q]], i64 [[I]]
+; FIXED-NEXT:    [[DST:%.*]] = getelementptr i64, ptr [[Q]], i64 [[I1]]
 ; FIXED-NEXT:    store i64 [[RES]], ptr [[DST]], align 8
-; FIXED-NEXT:    [[NEXTI]] = add i64 [[I]], 1
-; FIXED-NEXT:    [[DONE:%.*]] = icmp eq i64 [[NEXTI]], 1024
-; FIXED-NEXT:    br i1 [[DONE]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP15:![0-9]+]]
+; FIXED-NEXT:    [[NEXTI1]] = add i64 [[I1]], 1
+; FIXED-NEXT:    [[DONE:%.*]] = icmp eq i64 [[NEXTI1]], 1024
+; FIXED-NEXT:    br i1 [[DONE]], label [[EXIT]], label [[LOOP1]], !llvm.loop [[LOOP15:![0-9]+]]
 ; FIXED:       exit:
 ; FIXED-NEXT:    ret void
 ;
@@ -1367,7 +1405,7 @@ define void @combine_load_factor2_i64(ptr noalias %p, ptr noalias %q) {
 ; SCALABLE-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
 ; SCALABLE-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 2
 ; SCALABLE-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP1]]
-; SCALABLE-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; SCALABLE-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[ENTRY:%.*]]
 ; SCALABLE:       vector.ph:
 ; SCALABLE-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
 ; SCALABLE-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 2
@@ -1375,43 +1413,43 @@ define void @combine_load_factor2_i64(ptr noalias %p, ptr noalias %q) {
 ; SCALABLE-NEXT:    [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
 ; SCALABLE-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
 ; SCALABLE-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 2
-; SCALABLE-NEXT:    br label [[VECTOR_BODY:%.*]]
+; SCALABLE-NEXT:    br label [[LOOP:%.*]]
 ; SCALABLE:       vector.body:
-; SCALABLE-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; SCALABLE-NEXT:    [[TMP6:%.*]] = add i64 [[INDEX]], 0
-; SCALABLE-NEXT:    [[TMP7:%.*]] = shl i64 [[TMP6]], 1
-; SCALABLE-NEXT:    [[TMP8:%.*]] = getelementptr i64, ptr [[P:%.*]], i64 [[TMP7]]
-; SCALABLE-NEXT:    [[WIDE_VEC:%.*]] = load <vscale x 4 x i64>, ptr [[TMP8]], align 8
+; SCALABLE-NEXT:    [[I:%.*]] = phi i64 [ 0, [[ENTRY]] ], [ [[NEXTI:%.*]], [[LOOP]] ]
+; SCALABLE-NEXT:    [[TMP6:%.*]] = add i64 [[I]], 0
+; SCALABLE-NEXT:    [[OFFSET0:%.*]] = shl i64 [[TMP6]], 1
+; SCALABLE-NEXT:    [[Q0:%.*]] = getelementptr i64, ptr [[P:%.*]], i64 [[OFFSET0]]
+; SCALABLE-NEXT:    [[WIDE_VEC:%.*]] = load <vscale x 4 x i64>, ptr [[Q0]], align 8
 ; SCALABLE-NEXT:    [[STRIDED_VEC:%.*]] = call { <vscale x 2 x i64>, <vscale x 2 x i64> } @llvm.vector.deinterleave2.nxv4i64(<vscale x 4 x i64> [[WIDE_VEC]])
-; SCALABLE-NEXT:    [[TMP10:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64> } [[STRIDED_VEC]], 0
-; SCALABLE-NEXT:    [[TMP11:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64> } [[STRIDED_VEC]], 1
-; SCALABLE-NEXT:    [[TMP12:%.*]] = add <vscale x 2 x i64> [[TMP10]], [[TMP11]]
-; SCALABLE-NEXT:    [[TMP13:%.*]] = getelementptr i64, ptr [[Q:%.*]], i64 [[TMP6]]
-; SCALABLE-NEXT:    [[TMP14:%.*]] = getelementptr i64, ptr [[TMP13]], i32 0
-; SCALABLE-NEXT:    store <vscale x 2 x i64> [[TMP12]], ptr [[TMP14]], align 8
-; SCALABLE-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
-; SCALABLE-NEXT:    [[TMP15:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; SCALABLE-NEXT:    br i1 [[TMP15]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
+; SCALABLE-NEXT:    [[TMP9:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64> } [[STRIDED_VEC]], 0
+; SCALABLE-NEXT:    [[TMP10:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64> } [[STRIDED_VEC]], 1
+; SCALABLE-NEXT:    [[TMP11:%.*]] = add <vscale x 2 x i64> [[TMP9]], [[TMP10]]
+; SCALABLE-NEXT:    [[TMP12:%.*]] = getelementptr i64, ptr [[Q:%.*]], i64 [[TMP6]]
+; SCALABLE-NEXT:    [[TMP13:%.*]] = getelementptr i64, ptr [[TMP12]], i32 0
+; SCALABLE-NEXT:    store <vscale x 2 x i64> [[TMP11]], ptr [[TMP13]], align 8
+; SCALABLE-NEXT:    [[NEXTI]] = add nuw i64 [[I]], [[TMP5]]
+; SCALABLE-NEXT:    [[TMP14:%.*]] = icmp eq i64 [[NEXTI]], [[N_VEC]]
+; SCALABLE-NEXT:    br i1 [[TMP14]], label [[MIDDLE_BLOCK:%.*]], label [[LOOP]], !llvm.loop [[LOOP14:![0-9]+]]
 ; SCALABLE:       middle.block:
 ; SCALABLE-NEXT:    [[CMP_N:%.*]] = icmp eq i64 1024, [[N_VEC]]
 ; SCALABLE-NEXT:    br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
 ; SCALABLE:       scalar.ph:
-; SCALABLE-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
-; SCALABLE-NEXT:    br label [[LOOP:%.*]]
+; SCALABLE-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY1:%.*]] ]
+; SCALABLE-NEXT:    br label [[LOOP1:%.*]]
 ; SCALABLE:       loop:
-; SCALABLE-NEXT:    [[I:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[NEXTI:%.*]], [[LOOP]] ]
-; SCALABLE-NEXT:    [[OFFSET0:%.*]] = shl i64 [[I]], 1
-; SCALABLE-NEXT:    [[Q0:%.*]] = getelementptr i64, ptr [[P]], i64 [[OFFSET0]]
-; SCALABLE-NEXT:    [[X0:%.*]] = load i64, ptr [[Q0]], align 8
-; SCALABLE-NEXT:    [[OFFSET1:%.*]] = add i64 [[OFFSET0]], 1
+; SCALABLE-NEXT:    [[I1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[NEXTI1:%.*]], [[LOOP1]] ]
+; SCALABLE-NEXT:    [[OFFSET2:%.*]] = shl i64 [[I1]], 1
+; SCALABLE-NEXT:    [[Q2:%.*]] = getelementptr i64, ptr [[P]], i64 [[OFFSET2]]
+; SCALABLE-NEXT:    [[X0:%.*]] = load i64, ptr [[Q2]], align 8
+; SCALABLE-NEXT:    [[OFFSET1:%.*]] = add i64 [[OFFSET2]], 1
 ; SCALABLE-NEXT:    [[Q1:%.*]] = getelementptr i64, ptr [[P]], i64 [[OFFSET1]]
 ; SCALABLE-NEXT:    [[X1:%.*]] = load i64, ptr [[Q1]], align 8
 ; SCALABLE-NEXT:    [[RES:%.*]] = add i64 [[X0]], [[X1]]
-; SCALABLE-NEXT:    [[DST:%.*]] = getelementptr i64, ptr [[Q]], i64 [[I]]
+; SCALABLE-NEXT:    [[DST:%.*]] = getelementptr i64, ptr [[Q]], i64 [[I1]]
 ; SCALABLE-NEXT:    store i64 [[RES]], ptr [[DST]], align 8
-; SCALABLE-NEXT:    [[NEXTI]] = add i64 [[I]], 1
-; SCALABLE-NEXT:    [[DONE:%.*]] = icmp eq i64 [[NEXTI]], 1024
-; SCALABLE-NEXT:    br i1 [[DONE]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP15:![0-9]+]]
+; SCALABLE-NEXT:    [[NEXTI1]] = add i64 [[I1]], 1
+; SCALABLE-NEXT:    [[DONE:%.*]] = icmp eq i64 [[NEXTI1]], 1024
+; SCALABLE-NEXT:    br i1 [[DONE]], label [[EXIT]], label [[LOOP1]], !llvm.loop [[LOOP15:![0-9]+]]
 ; SCALABLE:       exit:
 ; SCALABLE-NEXT:    ret void
 ;
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/masked_gather_scatter.ll b/llvm/test/Transforms/LoopVectorize/RISCV/masked_gather_scatter.ll
index dc63072aa795f..2276b592aac8a 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/masked_gather_scatter.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/masked_gather_scatter.ll
@@ -69,7 +69,7 @@ define void @foo4(ptr nocapture %A, ptr nocapture readonly %B, ptr nocapture rea
 ; RV32-NEXT:    [[CMP_N:%.*]] = icmp eq i64 625, [[N_VEC]]
 ; RV32-NEXT:    br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]]
 ; RV32:       scalar.ph:
-; RV32-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ 0, [[VECTOR_MEMCHECK]] ], [ 0, [[ENTRY:%.*]] ]
+; RV32-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ], [ 0, [[VECTOR_MEMCHECK]] ]
 ; RV32-NEXT:    br label [[FOR_BODY:%.*]]
 ; RV32:       for.body:
 ; RV32-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_INC:%.*]] ]
@@ -148,7 +148,7 @@ define void @foo4(ptr nocapture %A, ptr nocapture readonly %B, ptr nocapture rea
 ; RV64-NEXT:    [[CMP_N:%.*]] = icmp eq i64 625, [[N_VEC]]
 ; RV64-NEXT:    br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]]
 ; RV64:       scalar.ph:
-; RV64-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ 0, [[VECTOR_MEMCHECK]] ], [ 0, [[ENTRY:%.*]] ]
+; RV64-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ], [ 0, [[VECTOR_MEMCHECK]] ]
 ; RV64-NEXT:    br label [[FOR_BODY:%.*]]
 ; RV64:       for.body:
 ; RV64-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_INC:%.*]] ]
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/reg-usage-bf16.ll b/llvm/test/Transforms/LoopVectorize/RISCV/reg-usage-bf16.ll
index 71ee00fbf3ec5..40d6e8bc33471 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/reg-usage-bf16.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/reg-usage-bf16.ll
@@ -1,5 +1,5 @@
 ; REQUIRES: asserts
-; RUN: opt -passes=loop-vectorize -mtriple riscv64 -mattr=+v,+zvfbfmin -debug-only=loop-vectorize -riscv-v-register-bit-width-lmul=1 -S < %s 2>&1 | FileCheck %s
+; RUN: opt -passes=loop-vectorize -mtriple riscv64 -mattr=+v,+zvfbfmin -debug-only=loop-vectorize --disable-output -riscv-v-register-bit-width-lmul=1 -S < %s 2>&1 | FileCheck %s
 
 define void @add(ptr noalias nocapture readonly %src1, ptr noalias nocapture readonly %src2, i32 signext %size, ptr noalias nocapture writeonly %result) {
 ; CHECK-LABEL: add
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/reg-usage-f16.ll b/llvm/test/Transforms/LoopVectorize/RISCV/reg-usage-f16.ll
index c58307d960786..e07c7b6b40729 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/reg-usage-f16.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/reg-usage-f16.ll
@@ -1,6 +1,6 @@
 ; REQUIRES: asserts
-; RUN: opt -passes=loop-vectorize -mtriple riscv64 -mattr=+v,+zvfh -debug-only=loop-vectorize -riscv-v-register-bit-width-lmul=1 -S < %s 2>&1 | FileCheck %s --check-prefix=ZVFH
-; RUN: opt -passes=loop-vectorize -mtriple riscv64 -mattr=+v,+zvfhmin -debug-only=loop-vectorize -riscv-v-register-bit-width-lmul=1 -S < %s 2>&1 | FileCheck %s --check-prefix=ZVFHMIN
+; RUN: opt -passes=loop-vectorize -mtriple riscv64 -mattr=+v,+zvfh -debug-only=loop-vectorize --disable-output -riscv-v-register-bit-width-lmul=1 -S < %s 2>&1 | FileCheck %s --check-prefix=ZVFH
+; RUN: opt -passes=loop-vectorize -mtriple riscv64 -mattr=+v,+zvfhmin -debug-only=loop-vectorize --disable-output -riscv-v-register-bit-width-lmul=1 -S < %s 2>&1 | FileCheck %s --check-prefix=ZVFHMIN
 
 define void @add(ptr noalias nocapture readonly %src1, ptr noalias nocapture readonly %src2, i32 signext %size, ptr noalias nocapture writeonly %result) {
 ; CHECK-LABEL: add
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/reg-usage.ll b/llvm/test/Transforms/LoopVectorize/RISCV/reg-usage.ll
index 5e3de92bca885..a2f55e49c9e0e 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/reg-usage.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/reg-usage.ll
@@ -1,22 +1,22 @@
 ; REQUIRES: asserts
 ; RUN: opt -passes=loop-vectorize -mtriple riscv64-linux-gnu \
-; RUN:   -mattr=+v,+d -debug-only=loop-vectorize \
+; RUN:   -mattr=+v,+d -debug-only=loop-vectorize --disable-output \
 ; RUN:   -riscv-v-vector-bits-min=128 -force-vector-width=1 \
 ; RUN:   -S < %s 2>&1 | FileCheck %s --check-prefix=CHECK-SCALAR
 ; RUN: opt -passes=loop-vectorize -mtriple riscv64-linux-gnu \
-; RUN:   -mattr=+v,+d -debug-only=loop-vectorize \
+; RUN:   -mattr=+v,+d -debug-only=loop-vectorize --disable-output \
 ; RUN:   -riscv-v-vector-bits-min=128 -riscv-v-register-bit-width-lmul=1 \
 ; RUN:   -S < %s 2>&1 | FileCheck %s --check-prefix=CHECK-LMUL1
 ; RUN: opt -passes=loop-vectorize -mtriple riscv64-linux-gnu \
-; RUN:   -mattr=+v,+d -debug-only=loop-vectorize \
+; RUN:   -mattr=+v,+d -debug-only=loop-vectorize --disable-output \
 ; RUN:   -riscv-v-vector-bits-min=128 -riscv-v-register-bit-width-lmul=2 \
 ; RUN:   -S < %s 2>&1 | FileCheck %s --check-prefix=CHECK-LMUL2
 ; RUN: opt -passes=loop-vectorize -mtriple riscv64-linux-gnu \
-; RUN:   -mattr=+v,+d -debug-only=loop-vectorize \
+; RUN:   -mattr=+v,+d -debug-only=loop-vectorize --disable-output \
 ; RUN:   -riscv-v-vector-bits-min=128 -riscv-v-register-bit-width-lmul=4 \
 ; RUN:   -S < %s 2>&1 | FileCheck %s --check-prefix=CHECK-LMUL4
 ; RUN: opt -passes=loop-vectorize -mtriple riscv64-linux-gnu \
-; RUN:   -mattr=+v,+d -debug-only=loop-vectorize \
+; RUN:   -mattr=+v,+d -debug-only=loop-vectorize --disable-output \
 ; RUN:   -riscv-v-vector-bits-min=128 -riscv-v-register-bit-width-lmul=8 \
 ; RUN:   -S < %s 2>&1 | FileCheck %s --check-prefix=CHECK-LMUL8
 
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/remark-reductions.ll b/llvm/test/Transforms/LoopVectorize/RISCV/remark-reductions.ll
new file mode 100644
index 0000000000000..e3e727b41c02d
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/remark-reductions.ll
@@ -0,0 +1,67 @@
+; RUN: opt < %s -mtriple=riscv64 -mattr=+v -p loop-vectorize -pass-remarks-analysis=loop-vectorize -S 2>&1 | FileCheck %s
+
+; CHECK: remark: <unknown>:0:0: the cost-model indicates that interleaving is not beneficial
+define float @s311(float %a_0, float %s311_sum) {
+; CHECK-LABEL: define float @s311(
+; CHECK-SAME: float [[A_0:%.*]], float [[S311_SUM:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    [[TMP0:%.*]] = call i32 @llvm.vscale.i32()
+; CHECK-NEXT:    [[TMP1:%.*]] = mul i32 [[TMP0]], 4
+; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 1200, [[TMP1]]
+; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK:       [[VECTOR_PH]]:
+; CHECK-NEXT:    [[TMP2:%.*]] = call i32 @llvm.vscale.i32()
+; CHECK-NEXT:    [[TMP3:%.*]] = mul i32 [[TMP2]], 4
+; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i32 1200, [[TMP3]]
+; CHECK-NEXT:    [[N_VEC:%.*]] = sub i32 1200, [[N_MOD_VF]]
+; CHECK-NEXT:    [[TMP4:%.*]] = call i32 @llvm.vscale.i32()
+; CHECK-NEXT:    [[TMP5:%.*]] = mul i32 [[TMP4]], 4
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x float> poison, float [[A_0]], i64 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x float> [[BROADCAST_SPLATINSERT]], <vscale x 4 x float> poison, <vscale x 4 x i32> zeroinitializer
+; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK:       [[VECTOR_BODY]]:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi float [ [[S311_SUM]], %[[VECTOR_PH]] ], [ [[TMP6:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP6]] = call float @llvm.vector.reduce.fadd.nxv4f32(float [[VEC_PHI]], <vscale x 4 x float> [[BROADCAST_SPLAT]])
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], [[TMP5]]
+; CHECK-NEXT:    [[TMP7:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK:       [[MIDDLE_BLOCK]]:
+; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i32 1200, [[N_VEC]]
+; CHECK-NEXT:    br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK:       [[SCALAR_PH]]:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP6]], %[[MIDDLE_BLOCK]] ], [ [[S311_SUM]], %[[ENTRY]] ]
+; CHECK-NEXT:    br label %[[LOOP:.*]]
+; CHECK:       [[LOOP]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[RED:%.*]] = phi float [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[RED_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[RED_NEXT]] = fadd float [[A_0]], [[RED]]
+; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i32 [[IV]], 1
+; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i32 [[IV_NEXT]], 1200
+; CHECK-NEXT:    br i1 [[EXITCOND]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    [[RED_LCSSA:%.*]] = phi float [ [[RED_NEXT]], %[[LOOP]] ], [ [[TMP6]], %[[MIDDLE_BLOCK]] ]
+; CHECK-NEXT:    ret float [[RED_LCSSA]]
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop ]
+  %red = phi float [ %s311_sum, %entry ], [ %red.next, %loop ]
+  %red.next = fadd float %a_0, %red
+  %iv.next = add nuw nsw i32 %iv, 1
+  %exitcond = icmp eq i32 %iv.next, 1200
+  br i1 %exitcond, label %exit, label %loop
+
+exit:
+  %red.lcssa = phi float [ %red.next, %loop ]
+  ret float %red.lcssa
+}
+;.
+; CHECK: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]], [[META2:![0-9]+]]}
+; CHECK: [[META1]] = !{!"llvm.loop.isvectorized", i32 1}
+; CHECK: [[META2]] = !{!"llvm.loop.unroll.runtime.disable"}
+; CHECK: [[LOOP3]] = distinct !{[[LOOP3]], [[META2]], [[META1]]}
+;.
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/riscv-interleaved.ll b/llvm/test/Transforms/LoopVectorize/RISCV/riscv-interleaved.ll
index 95eb67b3823ee..c332e2172b077 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/riscv-interleaved.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/riscv-interleaved.ll
@@ -1,6 +1,6 @@
 ; REQUIRES: asserts
 ; RUN: opt -passes=loop-vectorize,dce,instcombine -mtriple riscv64-linux-gnu \
-; RUN:   -mattr=+v -debug-only=loop-vectorize \
+; RUN:   -mattr=+v -debug-only=loop-vectorize --disable-output \
 ; RUN:   -riscv-v-vector-bits-min=128 -scalable-vectorization=off -S < %s 2>&1 | FileCheck %s
 
 ; CHECK-LABEL: foo
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/strided-accesses.ll b/llvm/test/Transforms/LoopVectorize/RISCV/strided-accesses.ll
index 30cb33e64eccf..4ea248254f2c6 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/strided-accesses.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/strided-accesses.ll
@@ -263,7 +263,7 @@ define void @single_stride_int_scaled(ptr %p, i64 %stride) {
 ; NOSTRIDED-NEXT:    [[CMP_N:%.*]] = icmp eq i64 1024, [[N_VEC]]
 ; NOSTRIDED-NEXT:    br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
 ; NOSTRIDED:       scalar.ph:
-; NOSTRIDED-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[VECTOR_SCEVCHECK]] ], [ 0, [[ENTRY:%.*]] ]
+; NOSTRIDED-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ], [ 0, [[VECTOR_SCEVCHECK]] ]
 ; NOSTRIDED-NEXT:    br label [[LOOP:%.*]]
 ; NOSTRIDED:       loop:
 ; NOSTRIDED-NEXT:    [[I:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[NEXTI:%.*]], [[LOOP]] ]
@@ -346,8 +346,8 @@ define void @single_stride_int_iv(ptr %p, i64 %stride) {
 ; NOSTRIDED-NEXT:    [[CMP_N:%.*]] = icmp eq i64 1024, [[N_VEC]]
 ; NOSTRIDED-NEXT:    br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
 ; NOSTRIDED:       scalar.ph:
-; NOSTRIDED-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[VECTOR_SCEVCHECK]] ], [ 0, [[ENTRY:%.*]] ]
-; NOSTRIDED-NEXT:    [[BC_RESUME_VAL1:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[VECTOR_SCEVCHECK]] ], [ 0, [[ENTRY]] ]
+; NOSTRIDED-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ], [ 0, [[VECTOR_SCEVCHECK]] ]
+; NOSTRIDED-NEXT:    [[BC_RESUME_VAL1:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ], [ 0, [[VECTOR_SCEVCHECK]] ]
 ; NOSTRIDED-NEXT:    br label [[LOOP:%.*]]
 ; NOSTRIDED:       loop:
 ; NOSTRIDED-NEXT:    [[I:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[NEXTI:%.*]], [[LOOP]] ]
@@ -480,7 +480,7 @@ define void @double_stride_int_scaled(ptr %p, ptr %p2, i64 %stride) {
 ; NOSTRIDED-NEXT:    [[CMP_N:%.*]] = icmp eq i64 1024, [[N_VEC]]
 ; NOSTRIDED-NEXT:    br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
 ; NOSTRIDED:       scalar.ph:
-; NOSTRIDED-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[VECTOR_MEMCHECK]] ], [ 0, [[VECTOR_SCEVCHECK]] ], [ 0, [[ENTRY:%.*]] ]
+; NOSTRIDED-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ], [ 0, [[VECTOR_SCEVCHECK]] ], [ 0, [[VECTOR_MEMCHECK]] ]
 ; NOSTRIDED-NEXT:    br label [[LOOP:%.*]]
 ; NOSTRIDED:       loop:
 ; NOSTRIDED-NEXT:    [[I:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[NEXTI:%.*]], [[LOOP]] ]
@@ -554,7 +554,7 @@ define void @double_stride_int_scaled(ptr %p, ptr %p2, i64 %stride) {
 ; STRIDED-NEXT:    [[CMP_N:%.*]] = icmp eq i64 1024, [[N_VEC]]
 ; STRIDED-NEXT:    br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
 ; STRIDED:       scalar.ph:
-; STRIDED-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[VECTOR_MEMCHECK]] ], [ 0, [[ENTRY:%.*]] ]
+; STRIDED-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ], [ 0, [[VECTOR_MEMCHECK]] ]
 ; STRIDED-NEXT:    br label [[LOOP:%.*]]
 ; STRIDED:       loop:
 ; STRIDED-NEXT:    [[I:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[NEXTI:%.*]], [[LOOP]] ]
@@ -623,8 +623,8 @@ define void @double_stride_int_iv(ptr %p, ptr %p2, i64 %stride) {
 ; NOSTRIDED-NEXT:    [[CMP_N:%.*]] = icmp eq i64 1024, [[N_VEC]]
 ; NOSTRIDED-NEXT:    br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
 ; NOSTRIDED:       scalar.ph:
-; NOSTRIDED-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[VECTOR_SCEVCHECK]] ], [ 0, [[ENTRY:%.*]] ]
-; NOSTRIDED-NEXT:    [[BC_RESUME_VAL1:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[VECTOR_SCEVCHECK]] ], [ 0, [[ENTRY]] ]
+; NOSTRIDED-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ], [ 0, [[VECTOR_SCEVCHECK]] ]
+; NOSTRIDED-NEXT:    [[BC_RESUME_VAL1:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ], [ 0, [[VECTOR_SCEVCHECK]] ]
 ; NOSTRIDED-NEXT:    br label [[LOOP:%.*]]
 ; NOSTRIDED:       loop:
 ; NOSTRIDED-NEXT:    [[I:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[NEXTI:%.*]], [[LOOP]] ]
@@ -776,9 +776,9 @@ define void @double_stride_ptr_iv(ptr %p, ptr %p2, i64 %stride) {
 ; STRIDED-NEXT:    [[CMP_N:%.*]] = icmp eq i64 1024, [[N_VEC]]
 ; STRIDED-NEXT:    br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
 ; STRIDED:       scalar.ph:
-; STRIDED-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[VECTOR_MEMCHECK]] ], [ 0, [[ENTRY:%.*]] ]
-; STRIDED-NEXT:    [[BC_RESUME_VAL6:%.*]] = phi ptr [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ [[P]], [[VECTOR_MEMCHECK]] ], [ [[P]], [[ENTRY]] ]
-; STRIDED-NEXT:    [[BC_RESUME_VAL8:%.*]] = phi ptr [ [[IND_END7]], [[MIDDLE_BLOCK]] ], [ [[P2]], [[VECTOR_MEMCHECK]] ], [ [[P2]], [[ENTRY]] ]
+; STRIDED-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ], [ 0, [[VECTOR_MEMCHECK]] ]
+; STRIDED-NEXT:    [[BC_RESUME_VAL6:%.*]] = phi ptr [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ [[P]], [[ENTRY]] ], [ [[P]], [[VECTOR_MEMCHECK]] ]
+; STRIDED-NEXT:    [[BC_RESUME_VAL8:%.*]] = phi ptr [ [[IND_END7]], [[MIDDLE_BLOCK]] ], [ [[P2]], [[ENTRY]] ], [ [[P2]], [[VECTOR_MEMCHECK]] ]
 ; STRIDED-NEXT:    br label [[LOOP:%.*]]
 ; STRIDED:       loop:
 ; STRIDED-NEXT:    [[I:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[NEXTI:%.*]], [[LOOP]] ]
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/type-info-cache-evl-crash.ll b/llvm/test/Transforms/LoopVectorize/RISCV/type-info-cache-evl-crash.ll
index 48b73c7f1a4de..7de51bc3a8a68 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/type-info-cache-evl-crash.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/type-info-cache-evl-crash.ll
@@ -62,7 +62,7 @@ define void @type_info_cache_clobber(ptr %dstv, ptr %src, i64 %wide.trip.count)
 ; CHECK:       [[MIDDLE_BLOCK]]:
 ; CHECK-NEXT:    br i1 true, label %[[EXIT:.*]], label %[[SCALAR_PH]]
 ; CHECK:       [[SCALAR_PH]]:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[VECTOR_MEMCHECK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_MEMCHECK]] ]
 ; CHECK-NEXT:    br label %[[LOOP:.*]]
 ; CHECK:       [[LOOP]]:
 ; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-bin-unary-ops-args.ll b/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-bin-unary-ops-args.ll
index 38fa8d3d9d9c5..13286a3394126 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-bin-unary-ops-args.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-bin-unary-ops-args.ll
@@ -12,7 +12,7 @@
 
 define void @test_and(ptr nocapture %a, ptr nocapture readonly %b) {
 ; IF-EVL-LABEL: define void @test_and(
-; IF-EVL-SAME: ptr nocapture [[A:%.*]], ptr nocapture readonly [[B:%.*]]) #[[ATTR0:[0-9]+]] {
+; IF-EVL-SAME: ptr captures(none) [[A:%.*]], ptr readonly captures(none) [[B:%.*]]) #[[ATTR0:[0-9]+]] {
 ; IF-EVL-NEXT:  [[LOOP_PREHEADER:.*]]:
 ; IF-EVL-NEXT:    [[A2:%.*]] = ptrtoint ptr [[A]] to i64
 ; IF-EVL-NEXT:    [[B1:%.*]] = ptrtoint ptr [[B]] to i64
@@ -54,7 +54,7 @@ define void @test_and(ptr nocapture %a, ptr nocapture readonly %b) {
 ; IF-EVL:       [[MIDDLE_BLOCK]]:
 ; IF-EVL-NEXT:    br i1 true, label %[[FINISH_LOOPEXIT:.*]], label %[[SCALAR_PH]]
 ; IF-EVL:       [[SCALAR_PH]]:
-; IF-EVL-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[VECTOR_MEMCHECK]] ], [ 0, %[[LOOP_PREHEADER]] ]
+; IF-EVL-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[LOOP_PREHEADER]] ], [ 0, %[[VECTOR_MEMCHECK]] ]
 ; IF-EVL-NEXT:    br label %[[LOOP:.*]]
 ; IF-EVL:       [[LOOP]]:
 ; IF-EVL-NEXT:    [[LEN:%.*]] = phi i64 [ [[DEC:%.*]], %[[LOOP]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
@@ -70,7 +70,7 @@ define void @test_and(ptr nocapture %a, ptr nocapture readonly %b) {
 ; IF-EVL-NEXT:    ret void
 ;
 ; NO-VP-LABEL: define void @test_and(
-; NO-VP-SAME: ptr nocapture [[A:%.*]], ptr nocapture readonly [[B:%.*]]) #[[ATTR0:[0-9]+]] {
+; NO-VP-SAME: ptr captures(none) [[A:%.*]], ptr readonly captures(none) [[B:%.*]]) #[[ATTR0:[0-9]+]] {
 ; NO-VP-NEXT:  [[LOOP_PREHEADER:.*]]:
 ; NO-VP-NEXT:    br label %[[LOOP:.*]]
 ; NO-VP:       [[LOOP]]:
@@ -106,7 +106,7 @@ finish.loopexit:
 
 define void @test_or(ptr nocapture %a, ptr nocapture readonly %b) {
 ; IF-EVL-LABEL: define void @test_or(
-; IF-EVL-SAME: ptr nocapture [[A:%.*]], ptr nocapture readonly [[B:%.*]]) #[[ATTR0]] {
+; IF-EVL-SAME: ptr captures(none) [[A:%.*]], ptr readonly captures(none) [[B:%.*]]) #[[ATTR0]] {
 ; IF-EVL-NEXT:  [[LOOP_PREHEADER:.*]]:
 ; IF-EVL-NEXT:    [[A2:%.*]] = ptrtoint ptr [[A]] to i64
 ; IF-EVL-NEXT:    [[B1:%.*]] = ptrtoint ptr [[B]] to i64
@@ -148,7 +148,7 @@ define void @test_or(ptr nocapture %a, ptr nocapture readonly %b) {
 ; IF-EVL:       [[MIDDLE_BLOCK]]:
 ; IF-EVL-NEXT:    br i1 true, label %[[FINISH_LOOPEXIT:.*]], label %[[SCALAR_PH]]
 ; IF-EVL:       [[SCALAR_PH]]:
-; IF-EVL-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[VECTOR_MEMCHECK]] ], [ 0, %[[LOOP_PREHEADER]] ]
+; IF-EVL-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[LOOP_PREHEADER]] ], [ 0, %[[VECTOR_MEMCHECK]] ]
 ; IF-EVL-NEXT:    br label %[[LOOP:.*]]
 ; IF-EVL:       [[LOOP]]:
 ; IF-EVL-NEXT:    [[LEN:%.*]] = phi i64 [ [[DEC:%.*]], %[[LOOP]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
@@ -164,7 +164,7 @@ define void @test_or(ptr nocapture %a, ptr nocapture readonly %b) {
 ; IF-EVL-NEXT:    ret void
 ;
 ; NO-VP-LABEL: define void @test_or(
-; NO-VP-SAME: ptr nocapture [[A:%.*]], ptr nocapture readonly [[B:%.*]]) #[[ATTR0]] {
+; NO-VP-SAME: ptr captures(none) [[A:%.*]], ptr readonly captures(none) [[B:%.*]]) #[[ATTR0]] {
 ; NO-VP-NEXT:  [[LOOP_PREHEADER:.*]]:
 ; NO-VP-NEXT:    br label %[[LOOP:.*]]
 ; NO-VP:       [[LOOP]]:
@@ -200,7 +200,7 @@ finish.loopexit:
 
 define void @test_xor(ptr nocapture %a, ptr nocapture readonly %b) {
 ; IF-EVL-LABEL: define void @test_xor(
-; IF-EVL-SAME: ptr nocapture [[A:%.*]], ptr nocapture readonly [[B:%.*]]) #[[ATTR0]] {
+; IF-EVL-SAME: ptr captures(none) [[A:%.*]], ptr readonly captures(none) [[B:%.*]]) #[[ATTR0]] {
 ; IF-EVL-NEXT:  [[LOOP_PREHEADER:.*]]:
 ; IF-EVL-NEXT:    [[A2:%.*]] = ptrtoint ptr [[A]] to i64
 ; IF-EVL-NEXT:    [[B1:%.*]] = ptrtoint ptr [[B]] to i64
@@ -242,7 +242,7 @@ define void @test_xor(ptr nocapture %a, ptr nocapture readonly %b) {
 ; IF-EVL:       [[MIDDLE_BLOCK]]:
 ; IF-EVL-NEXT:    br i1 true, label %[[FINISH_LOOPEXIT:.*]], label %[[SCALAR_PH]]
 ; IF-EVL:       [[SCALAR_PH]]:
-; IF-EVL-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[VECTOR_MEMCHECK]] ], [ 0, %[[LOOP_PREHEADER]] ]
+; IF-EVL-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[LOOP_PREHEADER]] ], [ 0, %[[VECTOR_MEMCHECK]] ]
 ; IF-EVL-NEXT:    br label %[[LOOP:.*]]
 ; IF-EVL:       [[LOOP]]:
 ; IF-EVL-NEXT:    [[LEN:%.*]] = phi i64 [ [[DEC:%.*]], %[[LOOP]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
@@ -258,7 +258,7 @@ define void @test_xor(ptr nocapture %a, ptr nocapture readonly %b) {
 ; IF-EVL-NEXT:    ret void
 ;
 ; NO-VP-LABEL: define void @test_xor(
-; NO-VP-SAME: ptr nocapture [[A:%.*]], ptr nocapture readonly [[B:%.*]]) #[[ATTR0]] {
+; NO-VP-SAME: ptr captures(none) [[A:%.*]], ptr readonly captures(none) [[B:%.*]]) #[[ATTR0]] {
 ; NO-VP-NEXT:  [[LOOP_PREHEADER:.*]]:
 ; NO-VP-NEXT:    br label %[[LOOP:.*]]
 ; NO-VP:       [[LOOP]]:
@@ -294,7 +294,7 @@ finish.loopexit:
 
 define void @test_shl(ptr nocapture %a, ptr nocapture readonly %b) {
 ; IF-EVL-LABEL: define void @test_shl(
-; IF-EVL-SAME: ptr nocapture [[A:%.*]], ptr nocapture readonly [[B:%.*]]) #[[ATTR0]] {
+; IF-EVL-SAME: ptr captures(none) [[A:%.*]], ptr readonly captures(none) [[B:%.*]]) #[[ATTR0]] {
 ; IF-EVL-NEXT:  [[LOOP_PREHEADER:.*]]:
 ; IF-EVL-NEXT:    [[A2:%.*]] = ptrtoint ptr [[A]] to i64
 ; IF-EVL-NEXT:    [[B1:%.*]] = ptrtoint ptr [[B]] to i64
@@ -336,7 +336,7 @@ define void @test_shl(ptr nocapture %a, ptr nocapture readonly %b) {
 ; IF-EVL:       [[MIDDLE_BLOCK]]:
 ; IF-EVL-NEXT:    br i1 true, label %[[FINISH_LOOPEXIT:.*]], label %[[SCALAR_PH]]
 ; IF-EVL:       [[SCALAR_PH]]:
-; IF-EVL-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[VECTOR_MEMCHECK]] ], [ 0, %[[LOOP_PREHEADER]] ]
+; IF-EVL-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[LOOP_PREHEADER]] ], [ 0, %[[VECTOR_MEMCHECK]] ]
 ; IF-EVL-NEXT:    br label %[[LOOP:.*]]
 ; IF-EVL:       [[LOOP]]:
 ; IF-EVL-NEXT:    [[LEN:%.*]] = phi i64 [ [[DEC:%.*]], %[[LOOP]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
@@ -352,7 +352,7 @@ define void @test_shl(ptr nocapture %a, ptr nocapture readonly %b) {
 ; IF-EVL-NEXT:    ret void
 ;
 ; NO-VP-LABEL: define void @test_shl(
-; NO-VP-SAME: ptr nocapture [[A:%.*]], ptr nocapture readonly [[B:%.*]]) #[[ATTR0]] {
+; NO-VP-SAME: ptr captures(none) [[A:%.*]], ptr readonly captures(none) [[B:%.*]]) #[[ATTR0]] {
 ; NO-VP-NEXT:  [[LOOP_PREHEADER:.*]]:
 ; NO-VP-NEXT:    br label %[[LOOP:.*]]
 ; NO-VP:       [[LOOP]]:
@@ -388,7 +388,7 @@ finish.loopexit:
 
 define void @test_lshr(ptr nocapture %a, ptr nocapture readonly %b) {
 ; IF-EVL-LABEL: define void @test_lshr(
-; IF-EVL-SAME: ptr nocapture [[A:%.*]], ptr nocapture readonly [[B:%.*]]) #[[ATTR0]] {
+; IF-EVL-SAME: ptr captures(none) [[A:%.*]], ptr readonly captures(none) [[B:%.*]]) #[[ATTR0]] {
 ; IF-EVL-NEXT:  [[LOOP_PREHEADER:.*]]:
 ; IF-EVL-NEXT:    [[A2:%.*]] = ptrtoint ptr [[A]] to i64
 ; IF-EVL-NEXT:    [[B1:%.*]] = ptrtoint ptr [[B]] to i64
@@ -430,7 +430,7 @@ define void @test_lshr(ptr nocapture %a, ptr nocapture readonly %b) {
 ; IF-EVL:       [[MIDDLE_BLOCK]]:
 ; IF-EVL-NEXT:    br i1 true, label %[[FINISH_LOOPEXIT:.*]], label %[[SCALAR_PH]]
 ; IF-EVL:       [[SCALAR_PH]]:
-; IF-EVL-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[VECTOR_MEMCHECK]] ], [ 0, %[[LOOP_PREHEADER]] ]
+; IF-EVL-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[LOOP_PREHEADER]] ], [ 0, %[[VECTOR_MEMCHECK]] ]
 ; IF-EVL-NEXT:    br label %[[LOOP:.*]]
 ; IF-EVL:       [[LOOP]]:
 ; IF-EVL-NEXT:    [[LEN:%.*]] = phi i64 [ [[DEC:%.*]], %[[LOOP]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
@@ -446,7 +446,7 @@ define void @test_lshr(ptr nocapture %a, ptr nocapture readonly %b) {
 ; IF-EVL-NEXT:    ret void
 ;
 ; NO-VP-LABEL: define void @test_lshr(
-; NO-VP-SAME: ptr nocapture [[A:%.*]], ptr nocapture readonly [[B:%.*]]) #[[ATTR0]] {
+; NO-VP-SAME: ptr captures(none) [[A:%.*]], ptr readonly captures(none) [[B:%.*]]) #[[ATTR0]] {
 ; NO-VP-NEXT:  [[LOOP_PREHEADER:.*]]:
 ; NO-VP-NEXT:    br label %[[LOOP:.*]]
 ; NO-VP:       [[LOOP]]:
@@ -482,7 +482,7 @@ finish.loopexit:
 
 define void @test_ashr(ptr nocapture %a, ptr nocapture readonly %b) {
 ; IF-EVL-LABEL: define void @test_ashr(
-; IF-EVL-SAME: ptr nocapture [[A:%.*]], ptr nocapture readonly [[B:%.*]]) #[[ATTR0]] {
+; IF-EVL-SAME: ptr captures(none) [[A:%.*]], ptr readonly captures(none) [[B:%.*]]) #[[ATTR0]] {
 ; IF-EVL-NEXT:  [[LOOP_PREHEADER:.*]]:
 ; IF-EVL-NEXT:    [[A2:%.*]] = ptrtoint ptr [[A]] to i64
 ; IF-EVL-NEXT:    [[B1:%.*]] = ptrtoint ptr [[B]] to i64
@@ -524,7 +524,7 @@ define void @test_ashr(ptr nocapture %a, ptr nocapture readonly %b) {
 ; IF-EVL:       [[MIDDLE_BLOCK]]:
 ; IF-EVL-NEXT:    br i1 true, label %[[FINISH_LOOPEXIT:.*]], label %[[SCALAR_PH]]
 ; IF-EVL:       [[SCALAR_PH]]:
-; IF-EVL-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[VECTOR_MEMCHECK]] ], [ 0, %[[LOOP_PREHEADER]] ]
+; IF-EVL-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[LOOP_PREHEADER]] ], [ 0, %[[VECTOR_MEMCHECK]] ]
 ; IF-EVL-NEXT:    br label %[[LOOP:.*]]
 ; IF-EVL:       [[LOOP]]:
 ; IF-EVL-NEXT:    [[LEN:%.*]] = phi i64 [ [[DEC:%.*]], %[[LOOP]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
@@ -540,7 +540,7 @@ define void @test_ashr(ptr nocapture %a, ptr nocapture readonly %b) {
 ; IF-EVL-NEXT:    ret void
 ;
 ; NO-VP-LABEL: define void @test_ashr(
-; NO-VP-SAME: ptr nocapture [[A:%.*]], ptr nocapture readonly [[B:%.*]]) #[[ATTR0]] {
+; NO-VP-SAME: ptr captures(none) [[A:%.*]], ptr readonly captures(none) [[B:%.*]]) #[[ATTR0]] {
 ; NO-VP-NEXT:  [[LOOP_PREHEADER:.*]]:
 ; NO-VP-NEXT:    br label %[[LOOP:.*]]
 ; NO-VP:       [[LOOP]]:
@@ -576,7 +576,7 @@ finish.loopexit:
 
 define void @test_add(ptr nocapture %a, ptr nocapture readonly %b) {
 ; IF-EVL-LABEL: define void @test_add(
-; IF-EVL-SAME: ptr nocapture [[A:%.*]], ptr nocapture readonly [[B:%.*]]) #[[ATTR0]] {
+; IF-EVL-SAME: ptr captures(none) [[A:%.*]], ptr readonly captures(none) [[B:%.*]]) #[[ATTR0]] {
 ; IF-EVL-NEXT:  [[LOOP_PREHEADER:.*]]:
 ; IF-EVL-NEXT:    [[A2:%.*]] = ptrtoint ptr [[A]] to i64
 ; IF-EVL-NEXT:    [[B1:%.*]] = ptrtoint ptr [[B]] to i64
@@ -618,7 +618,7 @@ define void @test_add(ptr nocapture %a, ptr nocapture readonly %b) {
 ; IF-EVL:       [[MIDDLE_BLOCK]]:
 ; IF-EVL-NEXT:    br i1 true, label %[[FINISH_LOOPEXIT:.*]], label %[[SCALAR_PH]]
 ; IF-EVL:       [[SCALAR_PH]]:
-; IF-EVL-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[VECTOR_MEMCHECK]] ], [ 0, %[[LOOP_PREHEADER]] ]
+; IF-EVL-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[LOOP_PREHEADER]] ], [ 0, %[[VECTOR_MEMCHECK]] ]
 ; IF-EVL-NEXT:    br label %[[LOOP:.*]]
 ; IF-EVL:       [[LOOP]]:
 ; IF-EVL-NEXT:    [[LEN:%.*]] = phi i64 [ [[DEC:%.*]], %[[LOOP]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
@@ -634,7 +634,7 @@ define void @test_add(ptr nocapture %a, ptr nocapture readonly %b) {
 ; IF-EVL-NEXT:    ret void
 ;
 ; NO-VP-LABEL: define void @test_add(
-; NO-VP-SAME: ptr nocapture [[A:%.*]], ptr nocapture readonly [[B:%.*]]) #[[ATTR0]] {
+; NO-VP-SAME: ptr captures(none) [[A:%.*]], ptr readonly captures(none) [[B:%.*]]) #[[ATTR0]] {
 ; NO-VP-NEXT:  [[LOOP_PREHEADER:.*]]:
 ; NO-VP-NEXT:    br label %[[LOOP:.*]]
 ; NO-VP:       [[LOOP]]:
@@ -670,7 +670,7 @@ finish.loopexit:
 
 define void @test_sub(ptr nocapture %a, ptr nocapture readonly %b) {
 ; IF-EVL-LABEL: define void @test_sub(
-; IF-EVL-SAME: ptr nocapture [[A:%.*]], ptr nocapture readonly [[B:%.*]]) #[[ATTR0]] {
+; IF-EVL-SAME: ptr captures(none) [[A:%.*]], ptr readonly captures(none) [[B:%.*]]) #[[ATTR0]] {
 ; IF-EVL-NEXT:  [[LOOP_PREHEADER:.*]]:
 ; IF-EVL-NEXT:    [[A2:%.*]] = ptrtoint ptr [[A]] to i64
 ; IF-EVL-NEXT:    [[B1:%.*]] = ptrtoint ptr [[B]] to i64
@@ -712,7 +712,7 @@ define void @test_sub(ptr nocapture %a, ptr nocapture readonly %b) {
 ; IF-EVL:       [[MIDDLE_BLOCK]]:
 ; IF-EVL-NEXT:    br i1 true, label %[[FINISH_LOOPEXIT:.*]], label %[[SCALAR_PH]]
 ; IF-EVL:       [[SCALAR_PH]]:
-; IF-EVL-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[VECTOR_MEMCHECK]] ], [ 0, %[[LOOP_PREHEADER]] ]
+; IF-EVL-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[LOOP_PREHEADER]] ], [ 0, %[[VECTOR_MEMCHECK]] ]
 ; IF-EVL-NEXT:    br label %[[LOOP:.*]]
 ; IF-EVL:       [[LOOP]]:
 ; IF-EVL-NEXT:    [[LEN:%.*]] = phi i64 [ [[DEC:%.*]], %[[LOOP]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
@@ -728,7 +728,7 @@ define void @test_sub(ptr nocapture %a, ptr nocapture readonly %b) {
 ; IF-EVL-NEXT:    ret void
 ;
 ; NO-VP-LABEL: define void @test_sub(
-; NO-VP-SAME: ptr nocapture [[A:%.*]], ptr nocapture readonly [[B:%.*]]) #[[ATTR0]] {
+; NO-VP-SAME: ptr captures(none) [[A:%.*]], ptr readonly captures(none) [[B:%.*]]) #[[ATTR0]] {
 ; NO-VP-NEXT:  [[LOOP_PREHEADER:.*]]:
 ; NO-VP-NEXT:    br label %[[LOOP:.*]]
 ; NO-VP:       [[LOOP]]:
@@ -764,7 +764,7 @@ finish.loopexit:
 
 define void @test_mul(ptr nocapture %a, ptr nocapture readonly %b) {
 ; IF-EVL-LABEL: define void @test_mul(
-; IF-EVL-SAME: ptr nocapture [[A:%.*]], ptr nocapture readonly [[B:%.*]]) #[[ATTR0]] {
+; IF-EVL-SAME: ptr captures(none) [[A:%.*]], ptr readonly captures(none) [[B:%.*]]) #[[ATTR0]] {
 ; IF-EVL-NEXT:  [[LOOP_PREHEADER:.*]]:
 ; IF-EVL-NEXT:    [[A2:%.*]] = ptrtoint ptr [[A]] to i64
 ; IF-EVL-NEXT:    [[B1:%.*]] = ptrtoint ptr [[B]] to i64
@@ -806,7 +806,7 @@ define void @test_mul(ptr nocapture %a, ptr nocapture readonly %b) {
 ; IF-EVL:       [[MIDDLE_BLOCK]]:
 ; IF-EVL-NEXT:    br i1 true, label %[[FINISH_LOOPEXIT:.*]], label %[[SCALAR_PH]]
 ; IF-EVL:       [[SCALAR_PH]]:
-; IF-EVL-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[VECTOR_MEMCHECK]] ], [ 0, %[[LOOP_PREHEADER]] ]
+; IF-EVL-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[LOOP_PREHEADER]] ], [ 0, %[[VECTOR_MEMCHECK]] ]
 ; IF-EVL-NEXT:    br label %[[LOOP:.*]]
 ; IF-EVL:       [[LOOP]]:
 ; IF-EVL-NEXT:    [[LEN:%.*]] = phi i64 [ [[DEC:%.*]], %[[LOOP]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
@@ -822,7 +822,7 @@ define void @test_mul(ptr nocapture %a, ptr nocapture readonly %b) {
 ; IF-EVL-NEXT:    ret void
 ;
 ; NO-VP-LABEL: define void @test_mul(
-; NO-VP-SAME: ptr nocapture [[A:%.*]], ptr nocapture readonly [[B:%.*]]) #[[ATTR0]] {
+; NO-VP-SAME: ptr captures(none) [[A:%.*]], ptr readonly captures(none) [[B:%.*]]) #[[ATTR0]] {
 ; NO-VP-NEXT:  [[LOOP_PREHEADER:.*]]:
 ; NO-VP-NEXT:    br label %[[LOOP:.*]]
 ; NO-VP:       [[LOOP]]:
@@ -858,7 +858,7 @@ finish.loopexit:
 
 define void @test_sdiv(ptr nocapture %a, ptr nocapture readonly %b) {
 ; IF-EVL-LABEL: define void @test_sdiv(
-; IF-EVL-SAME: ptr nocapture [[A:%.*]], ptr nocapture readonly [[B:%.*]]) #[[ATTR0]] {
+; IF-EVL-SAME: ptr captures(none) [[A:%.*]], ptr readonly captures(none) [[B:%.*]]) #[[ATTR0]] {
 ; IF-EVL-NEXT:  [[LOOP_PREHEADER:.*]]:
 ; IF-EVL-NEXT:    [[A2:%.*]] = ptrtoint ptr [[A]] to i64
 ; IF-EVL-NEXT:    [[B1:%.*]] = ptrtoint ptr [[B]] to i64
@@ -900,7 +900,7 @@ define void @test_sdiv(ptr nocapture %a, ptr nocapture readonly %b) {
 ; IF-EVL:       [[MIDDLE_BLOCK]]:
 ; IF-EVL-NEXT:    br i1 true, label %[[FINISH_LOOPEXIT:.*]], label %[[SCALAR_PH]]
 ; IF-EVL:       [[SCALAR_PH]]:
-; IF-EVL-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[VECTOR_MEMCHECK]] ], [ 0, %[[LOOP_PREHEADER]] ]
+; IF-EVL-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[LOOP_PREHEADER]] ], [ 0, %[[VECTOR_MEMCHECK]] ]
 ; IF-EVL-NEXT:    br label %[[LOOP:.*]]
 ; IF-EVL:       [[LOOP]]:
 ; IF-EVL-NEXT:    [[LEN:%.*]] = phi i64 [ [[DEC:%.*]], %[[LOOP]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
@@ -916,7 +916,7 @@ define void @test_sdiv(ptr nocapture %a, ptr nocapture readonly %b) {
 ; IF-EVL-NEXT:    ret void
 ;
 ; NO-VP-LABEL: define void @test_sdiv(
-; NO-VP-SAME: ptr nocapture [[A:%.*]], ptr nocapture readonly [[B:%.*]]) #[[ATTR0]] {
+; NO-VP-SAME: ptr captures(none) [[A:%.*]], ptr readonly captures(none) [[B:%.*]]) #[[ATTR0]] {
 ; NO-VP-NEXT:  [[LOOP_PREHEADER:.*]]:
 ; NO-VP-NEXT:    br label %[[LOOP:.*]]
 ; NO-VP:       [[LOOP]]:
@@ -952,7 +952,7 @@ finish.loopexit:
 
 define void @test_udiv(ptr nocapture %a, ptr nocapture readonly %b) {
 ; IF-EVL-LABEL: define void @test_udiv(
-; IF-EVL-SAME: ptr nocapture [[A:%.*]], ptr nocapture readonly [[B:%.*]]) #[[ATTR0]] {
+; IF-EVL-SAME: ptr captures(none) [[A:%.*]], ptr readonly captures(none) [[B:%.*]]) #[[ATTR0]] {
 ; IF-EVL-NEXT:  [[LOOP_PREHEADER:.*]]:
 ; IF-EVL-NEXT:    [[A2:%.*]] = ptrtoint ptr [[A]] to i64
 ; IF-EVL-NEXT:    [[B1:%.*]] = ptrtoint ptr [[B]] to i64
@@ -994,7 +994,7 @@ define void @test_udiv(ptr nocapture %a, ptr nocapture readonly %b) {
 ; IF-EVL:       [[MIDDLE_BLOCK]]:
 ; IF-EVL-NEXT:    br i1 true, label %[[FINISH_LOOPEXIT:.*]], label %[[SCALAR_PH]]
 ; IF-EVL:       [[SCALAR_PH]]:
-; IF-EVL-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[VECTOR_MEMCHECK]] ], [ 0, %[[LOOP_PREHEADER]] ]
+; IF-EVL-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[LOOP_PREHEADER]] ], [ 0, %[[VECTOR_MEMCHECK]] ]
 ; IF-EVL-NEXT:    br label %[[LOOP:.*]]
 ; IF-EVL:       [[LOOP]]:
 ; IF-EVL-NEXT:    [[LEN:%.*]] = phi i64 [ [[DEC:%.*]], %[[LOOP]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
@@ -1010,7 +1010,7 @@ define void @test_udiv(ptr nocapture %a, ptr nocapture readonly %b) {
 ; IF-EVL-NEXT:    ret void
 ;
 ; NO-VP-LABEL: define void @test_udiv(
-; NO-VP-SAME: ptr nocapture [[A:%.*]], ptr nocapture readonly [[B:%.*]]) #[[ATTR0]] {
+; NO-VP-SAME: ptr captures(none) [[A:%.*]], ptr readonly captures(none) [[B:%.*]]) #[[ATTR0]] {
 ; NO-VP-NEXT:  [[LOOP_PREHEADER:.*]]:
 ; NO-VP-NEXT:    br label %[[LOOP:.*]]
 ; NO-VP:       [[LOOP]]:
@@ -1046,7 +1046,7 @@ finish.loopexit:
 
 define void @test_srem(ptr nocapture %a, ptr nocapture readonly %b) {
 ; IF-EVL-LABEL: define void @test_srem(
-; IF-EVL-SAME: ptr nocapture [[A:%.*]], ptr nocapture readonly [[B:%.*]]) #[[ATTR0]] {
+; IF-EVL-SAME: ptr captures(none) [[A:%.*]], ptr readonly captures(none) [[B:%.*]]) #[[ATTR0]] {
 ; IF-EVL-NEXT:  [[LOOP_PREHEADER:.*]]:
 ; IF-EVL-NEXT:    [[A2:%.*]] = ptrtoint ptr [[A]] to i64
 ; IF-EVL-NEXT:    [[B1:%.*]] = ptrtoint ptr [[B]] to i64
@@ -1088,7 +1088,7 @@ define void @test_srem(ptr nocapture %a, ptr nocapture readonly %b) {
 ; IF-EVL:       [[MIDDLE_BLOCK]]:
 ; IF-EVL-NEXT:    br i1 true, label %[[FINISH_LOOPEXIT:.*]], label %[[SCALAR_PH]]
 ; IF-EVL:       [[SCALAR_PH]]:
-; IF-EVL-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[VECTOR_MEMCHECK]] ], [ 0, %[[LOOP_PREHEADER]] ]
+; IF-EVL-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[LOOP_PREHEADER]] ], [ 0, %[[VECTOR_MEMCHECK]] ]
 ; IF-EVL-NEXT:    br label %[[LOOP:.*]]
 ; IF-EVL:       [[LOOP]]:
 ; IF-EVL-NEXT:    [[LEN:%.*]] = phi i64 [ [[DEC:%.*]], %[[LOOP]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
@@ -1104,7 +1104,7 @@ define void @test_srem(ptr nocapture %a, ptr nocapture readonly %b) {
 ; IF-EVL-NEXT:    ret void
 ;
 ; NO-VP-LABEL: define void @test_srem(
-; NO-VP-SAME: ptr nocapture [[A:%.*]], ptr nocapture readonly [[B:%.*]]) #[[ATTR0]] {
+; NO-VP-SAME: ptr captures(none) [[A:%.*]], ptr readonly captures(none) [[B:%.*]]) #[[ATTR0]] {
 ; NO-VP-NEXT:  [[LOOP_PREHEADER:.*]]:
 ; NO-VP-NEXT:    br label %[[LOOP:.*]]
 ; NO-VP:       [[LOOP]]:
@@ -1140,7 +1140,7 @@ finish.loopexit:
 
 define void @test_urem(ptr nocapture %a, ptr nocapture readonly %b) {
 ; IF-EVL-LABEL: define void @test_urem(
-; IF-EVL-SAME: ptr nocapture [[A:%.*]], ptr nocapture readonly [[B:%.*]]) #[[ATTR0]] {
+; IF-EVL-SAME: ptr captures(none) [[A:%.*]], ptr readonly captures(none) [[B:%.*]]) #[[ATTR0]] {
 ; IF-EVL-NEXT:  [[LOOP_PREHEADER:.*]]:
 ; IF-EVL-NEXT:    [[A2:%.*]] = ptrtoint ptr [[A]] to i64
 ; IF-EVL-NEXT:    [[B1:%.*]] = ptrtoint ptr [[B]] to i64
@@ -1182,7 +1182,7 @@ define void @test_urem(ptr nocapture %a, ptr nocapture readonly %b) {
 ; IF-EVL:       [[MIDDLE_BLOCK]]:
 ; IF-EVL-NEXT:    br i1 true, label %[[FINISH_LOOPEXIT:.*]], label %[[SCALAR_PH]]
 ; IF-EVL:       [[SCALAR_PH]]:
-; IF-EVL-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[VECTOR_MEMCHECK]] ], [ 0, %[[LOOP_PREHEADER]] ]
+; IF-EVL-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[LOOP_PREHEADER]] ], [ 0, %[[VECTOR_MEMCHECK]] ]
 ; IF-EVL-NEXT:    br label %[[LOOP:.*]]
 ; IF-EVL:       [[LOOP]]:
 ; IF-EVL-NEXT:    [[LEN:%.*]] = phi i64 [ [[DEC:%.*]], %[[LOOP]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
@@ -1198,7 +1198,7 @@ define void @test_urem(ptr nocapture %a, ptr nocapture readonly %b) {
 ; IF-EVL-NEXT:    ret void
 ;
 ; NO-VP-LABEL: define void @test_urem(
-; NO-VP-SAME: ptr nocapture [[A:%.*]], ptr nocapture readonly [[B:%.*]]) #[[ATTR0]] {
+; NO-VP-SAME: ptr captures(none) [[A:%.*]], ptr readonly captures(none) [[B:%.*]]) #[[ATTR0]] {
 ; NO-VP-NEXT:  [[LOOP_PREHEADER:.*]]:
 ; NO-VP-NEXT:    br label %[[LOOP:.*]]
 ; NO-VP:       [[LOOP]]:
@@ -1236,7 +1236,7 @@ finish.loopexit:
 
 define void @test_fadd(ptr nocapture %a, ptr nocapture readonly %b) {
 ; IF-EVL-LABEL: define void @test_fadd(
-; IF-EVL-SAME: ptr nocapture [[A:%.*]], ptr nocapture readonly [[B:%.*]]) #[[ATTR0]] {
+; IF-EVL-SAME: ptr captures(none) [[A:%.*]], ptr readonly captures(none) [[B:%.*]]) #[[ATTR0]] {
 ; IF-EVL-NEXT:  [[LOOP_PREHEADER:.*]]:
 ; IF-EVL-NEXT:    [[A2:%.*]] = ptrtoint ptr [[A]] to i64
 ; IF-EVL-NEXT:    [[B1:%.*]] = ptrtoint ptr [[B]] to i64
@@ -1279,7 +1279,7 @@ define void @test_fadd(ptr nocapture %a, ptr nocapture readonly %b) {
 ; IF-EVL:       [[MIDDLE_BLOCK]]:
 ; IF-EVL-NEXT:    br i1 true, label %[[FINISH_LOOPEXIT:.*]], label %[[SCALAR_PH]]
 ; IF-EVL:       [[SCALAR_PH]]:
-; IF-EVL-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[VECTOR_MEMCHECK]] ], [ 0, %[[LOOP_PREHEADER]] ]
+; IF-EVL-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[LOOP_PREHEADER]] ], [ 0, %[[VECTOR_MEMCHECK]] ]
 ; IF-EVL-NEXT:    br label %[[LOOP:.*]]
 ; IF-EVL:       [[LOOP]]:
 ; IF-EVL-NEXT:    [[LEN:%.*]] = phi i64 [ [[DEC:%.*]], %[[LOOP]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
@@ -1295,7 +1295,7 @@ define void @test_fadd(ptr nocapture %a, ptr nocapture readonly %b) {
 ; IF-EVL-NEXT:    ret void
 ;
 ; NO-VP-LABEL: define void @test_fadd(
-; NO-VP-SAME: ptr nocapture [[A:%.*]], ptr nocapture readonly [[B:%.*]]) #[[ATTR0]] {
+; NO-VP-SAME: ptr captures(none) [[A:%.*]], ptr readonly captures(none) [[B:%.*]]) #[[ATTR0]] {
 ; NO-VP-NEXT:  [[LOOP_PREHEADER:.*]]:
 ; NO-VP-NEXT:    br label %[[LOOP:.*]]
 ; NO-VP:       [[LOOP]]:
@@ -1331,7 +1331,7 @@ finish.loopexit:
 
 define void @test_fsub(ptr nocapture %a, ptr nocapture readonly %b) {
 ; IF-EVL-LABEL: define void @test_fsub(
-; IF-EVL-SAME: ptr nocapture [[A:%.*]], ptr nocapture readonly [[B:%.*]]) #[[ATTR0]] {
+; IF-EVL-SAME: ptr captures(none) [[A:%.*]], ptr readonly captures(none) [[B:%.*]]) #[[ATTR0]] {
 ; IF-EVL-NEXT:  [[LOOP_PREHEADER:.*]]:
 ; IF-EVL-NEXT:    [[A2:%.*]] = ptrtoint ptr [[A]] to i64
 ; IF-EVL-NEXT:    [[B1:%.*]] = ptrtoint ptr [[B]] to i64
@@ -1374,7 +1374,7 @@ define void @test_fsub(ptr nocapture %a, ptr nocapture readonly %b) {
 ; IF-EVL:       [[MIDDLE_BLOCK]]:
 ; IF-EVL-NEXT:    br i1 true, label %[[FINISH_LOOPEXIT:.*]], label %[[SCALAR_PH]]
 ; IF-EVL:       [[SCALAR_PH]]:
-; IF-EVL-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[VECTOR_MEMCHECK]] ], [ 0, %[[LOOP_PREHEADER]] ]
+; IF-EVL-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[LOOP_PREHEADER]] ], [ 0, %[[VECTOR_MEMCHECK]] ]
 ; IF-EVL-NEXT:    br label %[[LOOP:.*]]
 ; IF-EVL:       [[LOOP]]:
 ; IF-EVL-NEXT:    [[LEN:%.*]] = phi i64 [ [[DEC:%.*]], %[[LOOP]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
@@ -1390,7 +1390,7 @@ define void @test_fsub(ptr nocapture %a, ptr nocapture readonly %b) {
 ; IF-EVL-NEXT:    ret void
 ;
 ; NO-VP-LABEL: define void @test_fsub(
-; NO-VP-SAME: ptr nocapture [[A:%.*]], ptr nocapture readonly [[B:%.*]]) #[[ATTR0]] {
+; NO-VP-SAME: ptr captures(none) [[A:%.*]], ptr readonly captures(none) [[B:%.*]]) #[[ATTR0]] {
 ; NO-VP-NEXT:  [[LOOP_PREHEADER:.*]]:
 ; NO-VP-NEXT:    br label %[[LOOP:.*]]
 ; NO-VP:       [[LOOP]]:
@@ -1426,7 +1426,7 @@ finish.loopexit:
 
 define void @test_fmul(ptr nocapture %a, ptr nocapture readonly %b) {
 ; IF-EVL-LABEL: define void @test_fmul(
-; IF-EVL-SAME: ptr nocapture [[A:%.*]], ptr nocapture readonly [[B:%.*]]) #[[ATTR0]] {
+; IF-EVL-SAME: ptr captures(none) [[A:%.*]], ptr readonly captures(none) [[B:%.*]]) #[[ATTR0]] {
 ; IF-EVL-NEXT:  [[LOOP_PREHEADER:.*]]:
 ; IF-EVL-NEXT:    [[A2:%.*]] = ptrtoint ptr [[A]] to i64
 ; IF-EVL-NEXT:    [[B1:%.*]] = ptrtoint ptr [[B]] to i64
@@ -1469,7 +1469,7 @@ define void @test_fmul(ptr nocapture %a, ptr nocapture readonly %b) {
 ; IF-EVL:       [[MIDDLE_BLOCK]]:
 ; IF-EVL-NEXT:    br i1 true, label %[[FINISH_LOOPEXIT:.*]], label %[[SCALAR_PH]]
 ; IF-EVL:       [[SCALAR_PH]]:
-; IF-EVL-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[VECTOR_MEMCHECK]] ], [ 0, %[[LOOP_PREHEADER]] ]
+; IF-EVL-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[LOOP_PREHEADER]] ], [ 0, %[[VECTOR_MEMCHECK]] ]
 ; IF-EVL-NEXT:    br label %[[LOOP:.*]]
 ; IF-EVL:       [[LOOP]]:
 ; IF-EVL-NEXT:    [[LEN:%.*]] = phi i64 [ [[DEC:%.*]], %[[LOOP]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
@@ -1485,7 +1485,7 @@ define void @test_fmul(ptr nocapture %a, ptr nocapture readonly %b) {
 ; IF-EVL-NEXT:    ret void
 ;
 ; NO-VP-LABEL: define void @test_fmul(
-; NO-VP-SAME: ptr nocapture [[A:%.*]], ptr nocapture readonly [[B:%.*]]) #[[ATTR0]] {
+; NO-VP-SAME: ptr captures(none) [[A:%.*]], ptr readonly captures(none) [[B:%.*]]) #[[ATTR0]] {
 ; NO-VP-NEXT:  [[LOOP_PREHEADER:.*]]:
 ; NO-VP-NEXT:    br label %[[LOOP:.*]]
 ; NO-VP:       [[LOOP]]:
@@ -1521,7 +1521,7 @@ finish.loopexit:
 
 define void @test_fdiv(ptr nocapture %a, ptr nocapture readonly %b) {
 ; IF-EVL-LABEL: define void @test_fdiv(
-; IF-EVL-SAME: ptr nocapture [[A:%.*]], ptr nocapture readonly [[B:%.*]]) #[[ATTR0]] {
+; IF-EVL-SAME: ptr captures(none) [[A:%.*]], ptr readonly captures(none) [[B:%.*]]) #[[ATTR0]] {
 ; IF-EVL-NEXT:  [[LOOP_PREHEADER:.*]]:
 ; IF-EVL-NEXT:    [[A2:%.*]] = ptrtoint ptr [[A]] to i64
 ; IF-EVL-NEXT:    [[B1:%.*]] = ptrtoint ptr [[B]] to i64
@@ -1564,7 +1564,7 @@ define void @test_fdiv(ptr nocapture %a, ptr nocapture readonly %b) {
 ; IF-EVL:       [[MIDDLE_BLOCK]]:
 ; IF-EVL-NEXT:    br i1 true, label %[[FINISH_LOOPEXIT:.*]], label %[[SCALAR_PH]]
 ; IF-EVL:       [[SCALAR_PH]]:
-; IF-EVL-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[VECTOR_MEMCHECK]] ], [ 0, %[[LOOP_PREHEADER]] ]
+; IF-EVL-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[LOOP_PREHEADER]] ], [ 0, %[[VECTOR_MEMCHECK]] ]
 ; IF-EVL-NEXT:    br label %[[LOOP:.*]]
 ; IF-EVL:       [[LOOP]]:
 ; IF-EVL-NEXT:    [[LEN:%.*]] = phi i64 [ [[DEC:%.*]], %[[LOOP]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
@@ -1580,7 +1580,7 @@ define void @test_fdiv(ptr nocapture %a, ptr nocapture readonly %b) {
 ; IF-EVL-NEXT:    ret void
 ;
 ; NO-VP-LABEL: define void @test_fdiv(
-; NO-VP-SAME: ptr nocapture [[A:%.*]], ptr nocapture readonly [[B:%.*]]) #[[ATTR0]] {
+; NO-VP-SAME: ptr captures(none) [[A:%.*]], ptr readonly captures(none) [[B:%.*]]) #[[ATTR0]] {
 ; NO-VP-NEXT:  [[LOOP_PREHEADER:.*]]:
 ; NO-VP-NEXT:    br label %[[LOOP:.*]]
 ; NO-VP:       [[LOOP]]:
@@ -1616,7 +1616,7 @@ finish.loopexit:
 
 define void @test_frem(ptr nocapture %a, ptr nocapture readonly %b) {
 ; IF-EVL-LABEL: define void @test_frem(
-; IF-EVL-SAME: ptr nocapture [[A:%.*]], ptr nocapture readonly [[B:%.*]]) #[[ATTR0]] {
+; IF-EVL-SAME: ptr captures(none) [[A:%.*]], ptr readonly captures(none) [[B:%.*]]) #[[ATTR0]] {
 ; IF-EVL-NEXT:  [[LOOP_PREHEADER:.*]]:
 ; IF-EVL-NEXT:    br label %[[LOOP:.*]]
 ; IF-EVL:       [[LOOP]]:
@@ -1633,7 +1633,7 @@ define void @test_frem(ptr nocapture %a, ptr nocapture readonly %b) {
 ; IF-EVL-NEXT:    ret void
 ;
 ; NO-VP-LABEL: define void @test_frem(
-; NO-VP-SAME: ptr nocapture [[A:%.*]], ptr nocapture readonly [[B:%.*]]) #[[ATTR0]] {
+; NO-VP-SAME: ptr captures(none) [[A:%.*]], ptr readonly captures(none) [[B:%.*]]) #[[ATTR0]] {
 ; NO-VP-NEXT:  [[LOOP_PREHEADER:.*]]:
 ; NO-VP-NEXT:    br label %[[LOOP:.*]]
 ; NO-VP:       [[LOOP]]:
@@ -1669,7 +1669,7 @@ finish.loopexit:
 
 define void @test_fneg(ptr nocapture %a, ptr nocapture readonly %b) {
 ; IF-EVL-LABEL: define void @test_fneg(
-; IF-EVL-SAME: ptr nocapture [[A:%.*]], ptr nocapture readonly [[B:%.*]]) #[[ATTR0]] {
+; IF-EVL-SAME: ptr captures(none) [[A:%.*]], ptr readonly captures(none) [[B:%.*]]) #[[ATTR0]] {
 ; IF-EVL-NEXT:  [[LOOP_PREHEADER:.*]]:
 ; IF-EVL-NEXT:    [[A2:%.*]] = ptrtoint ptr [[A]] to i64
 ; IF-EVL-NEXT:    [[B1:%.*]] = ptrtoint ptr [[B]] to i64
@@ -1712,7 +1712,7 @@ define void @test_fneg(ptr nocapture %a, ptr nocapture readonly %b) {
 ; IF-EVL:       [[MIDDLE_BLOCK]]:
 ; IF-EVL-NEXT:    br i1 true, label %[[FINISH_LOOPEXIT:.*]], label %[[SCALAR_PH]]
 ; IF-EVL:       [[SCALAR_PH]]:
-; IF-EVL-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[VECTOR_MEMCHECK]] ], [ 0, %[[LOOP_PREHEADER]] ]
+; IF-EVL-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[LOOP_PREHEADER]] ], [ 0, %[[VECTOR_MEMCHECK]] ]
 ; IF-EVL-NEXT:    br label %[[LOOP:.*]]
 ; IF-EVL:       [[LOOP]]:
 ; IF-EVL-NEXT:    [[LEN:%.*]] = phi i64 [ [[DEC:%.*]], %[[LOOP]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
@@ -1728,7 +1728,7 @@ define void @test_fneg(ptr nocapture %a, ptr nocapture readonly %b) {
 ; IF-EVL-NEXT:    ret void
 ;
 ; NO-VP-LABEL: define void @test_fneg(
-; NO-VP-SAME: ptr nocapture [[A:%.*]], ptr nocapture readonly [[B:%.*]]) #[[ATTR0]] {
+; NO-VP-SAME: ptr captures(none) [[A:%.*]], ptr readonly captures(none) [[B:%.*]]) #[[ATTR0]] {
 ; NO-VP-NEXT:  [[LOOP_PREHEADER:.*]]:
 ; NO-VP-NEXT:    br label %[[LOOP:.*]]
 ; NO-VP:       [[LOOP]]:
@@ -1761,3 +1761,43 @@ loop:
 finish.loopexit:
   ret void
 }
+;.
+; IF-EVL: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]], [[META2:![0-9]+]]}
+; IF-EVL: [[META1]] = !{!"llvm.loop.isvectorized", i32 1}
+; IF-EVL: [[META2]] = !{!"llvm.loop.unroll.runtime.disable"}
+; IF-EVL: [[LOOP3]] = distinct !{[[LOOP3]], [[META1]]}
+; IF-EVL: [[LOOP4]] = distinct !{[[LOOP4]], [[META1]], [[META2]]}
+; IF-EVL: [[LOOP5]] = distinct !{[[LOOP5]], [[META1]]}
+; IF-EVL: [[LOOP6]] = distinct !{[[LOOP6]], [[META1]], [[META2]]}
+; IF-EVL: [[LOOP7]] = distinct !{[[LOOP7]], [[META1]]}
+; IF-EVL: [[LOOP8]] = distinct !{[[LOOP8]], [[META1]], [[META2]]}
+; IF-EVL: [[LOOP9]] = distinct !{[[LOOP9]], [[META1]]}
+; IF-EVL: [[LOOP10]] = distinct !{[[LOOP10]], [[META1]], [[META2]]}
+; IF-EVL: [[LOOP11]] = distinct !{[[LOOP11]], [[META1]]}
+; IF-EVL: [[LOOP12]] = distinct !{[[LOOP12]], [[META1]], [[META2]]}
+; IF-EVL: [[LOOP13]] = distinct !{[[LOOP13]], [[META1]]}
+; IF-EVL: [[LOOP14]] = distinct !{[[LOOP14]], [[META1]], [[META2]]}
+; IF-EVL: [[LOOP15]] = distinct !{[[LOOP15]], [[META1]]}
+; IF-EVL: [[LOOP16]] = distinct !{[[LOOP16]], [[META1]], [[META2]]}
+; IF-EVL: [[LOOP17]] = distinct !{[[LOOP17]], [[META1]]}
+; IF-EVL: [[LOOP18]] = distinct !{[[LOOP18]], [[META1]], [[META2]]}
+; IF-EVL: [[LOOP19]] = distinct !{[[LOOP19]], [[META1]]}
+; IF-EVL: [[LOOP20]] = distinct !{[[LOOP20]], [[META1]], [[META2]]}
+; IF-EVL: [[LOOP21]] = distinct !{[[LOOP21]], [[META1]]}
+; IF-EVL: [[LOOP22]] = distinct !{[[LOOP22]], [[META1]], [[META2]]}
+; IF-EVL: [[LOOP23]] = distinct !{[[LOOP23]], [[META1]]}
+; IF-EVL: [[LOOP24]] = distinct !{[[LOOP24]], [[META1]], [[META2]]}
+; IF-EVL: [[LOOP25]] = distinct !{[[LOOP25]], [[META1]]}
+; IF-EVL: [[LOOP26]] = distinct !{[[LOOP26]], [[META1]], [[META2]]}
+; IF-EVL: [[LOOP27]] = distinct !{[[LOOP27]], [[META1]]}
+; IF-EVL: [[LOOP28]] = distinct !{[[LOOP28]], [[META1]], [[META2]]}
+; IF-EVL: [[LOOP29]] = distinct !{[[LOOP29]], [[META1]]}
+; IF-EVL: [[LOOP30]] = distinct !{[[LOOP30]], [[META1]], [[META2]]}
+; IF-EVL: [[LOOP31]] = distinct !{[[LOOP31]], [[META1]]}
+; IF-EVL: [[LOOP32]] = distinct !{[[LOOP32]], [[META1]], [[META2]]}
+; IF-EVL: [[LOOP33]] = distinct !{[[LOOP33]], [[META1]]}
+; IF-EVL: [[LOOP34]] = distinct !{[[LOOP34]], [[META1]], [[META2]]}
+; IF-EVL: [[LOOP35]] = distinct !{[[LOOP35]], [[META1]]}
+; IF-EVL: [[LOOP36]] = distinct !{[[LOOP36]], [[META1]], [[META2]]}
+; IF-EVL: [[LOOP37]] = distinct !{[[LOOP37]], [[META1]]}
+;.
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-call-intrinsics.ll b/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-call-intrinsics.ll
index f07aaecfa8467..28ba5efb53c2a 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-call-intrinsics.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-call-intrinsics.ll
@@ -67,7 +67,7 @@ define void @vp_smax(ptr %a, ptr %b, ptr %c, i64 %N) {
 ; IF-EVL:       [[MIDDLE_BLOCK]]:
 ; IF-EVL-NEXT:    br i1 true, label %[[EXIT:.*]], label %[[SCALAR_PH]]
 ; IF-EVL:       [[SCALAR_PH]]:
-; IF-EVL-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[VECTOR_MEMCHECK]] ], [ 0, %[[ENTRY]] ]
+; IF-EVL-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_MEMCHECK]] ]
 ; IF-EVL-NEXT:    br label %[[LOOP:.*]]
 ; IF-EVL:       [[LOOP]]:
 ; IF-EVL-NEXT:    [[IV:%.*]] = phi i64 [ [[IV_NEXT:%.*]], %[[LOOP]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
@@ -182,7 +182,7 @@ define void @vp_smin(ptr %a, ptr %b, ptr %c, i64 %N) {
 ; IF-EVL:       [[MIDDLE_BLOCK]]:
 ; IF-EVL-NEXT:    br i1 true, label %[[EXIT:.*]], label %[[SCALAR_PH]]
 ; IF-EVL:       [[SCALAR_PH]]:
-; IF-EVL-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[VECTOR_MEMCHECK]] ], [ 0, %[[ENTRY]] ]
+; IF-EVL-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_MEMCHECK]] ]
 ; IF-EVL-NEXT:    br label %[[LOOP:.*]]
 ; IF-EVL:       [[LOOP]]:
 ; IF-EVL-NEXT:    [[IV:%.*]] = phi i64 [ [[IV_NEXT:%.*]], %[[LOOP]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
@@ -297,7 +297,7 @@ define void @vp_umax(ptr %a, ptr %b, ptr %c, i64 %N) {
 ; IF-EVL:       [[MIDDLE_BLOCK]]:
 ; IF-EVL-NEXT:    br i1 true, label %[[EXIT:.*]], label %[[SCALAR_PH]]
 ; IF-EVL:       [[SCALAR_PH]]:
-; IF-EVL-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[VECTOR_MEMCHECK]] ], [ 0, %[[ENTRY]] ]
+; IF-EVL-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_MEMCHECK]] ]
 ; IF-EVL-NEXT:    br label %[[LOOP:.*]]
 ; IF-EVL:       [[LOOP]]:
 ; IF-EVL-NEXT:    [[IV:%.*]] = phi i64 [ [[IV_NEXT:%.*]], %[[LOOP]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
@@ -412,7 +412,7 @@ define void @vp_umin(ptr %a, ptr %b, ptr %c, i64 %N) {
 ; IF-EVL:       [[MIDDLE_BLOCK]]:
 ; IF-EVL-NEXT:    br i1 true, label %[[EXIT:.*]], label %[[SCALAR_PH]]
 ; IF-EVL:       [[SCALAR_PH]]:
-; IF-EVL-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[VECTOR_MEMCHECK]] ], [ 0, %[[ENTRY]] ]
+; IF-EVL-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_MEMCHECK]] ]
 ; IF-EVL-NEXT:    br label %[[LOOP:.*]]
 ; IF-EVL:       [[LOOP]]:
 ; IF-EVL-NEXT:    [[IV:%.*]] = phi i64 [ [[IV_NEXT:%.*]], %[[LOOP]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
@@ -519,7 +519,7 @@ define void @vp_ctlz(ptr %a, ptr %b, i64 %N) {
 ; IF-EVL:       [[MIDDLE_BLOCK]]:
 ; IF-EVL-NEXT:    br i1 true, label %[[EXIT:.*]], label %[[SCALAR_PH]]
 ; IF-EVL:       [[SCALAR_PH]]:
-; IF-EVL-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[VECTOR_MEMCHECK]] ], [ 0, %[[ENTRY]] ]
+; IF-EVL-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_MEMCHECK]] ]
 ; IF-EVL-NEXT:    br label %[[LOOP:.*]]
 ; IF-EVL:       [[LOOP]]:
 ; IF-EVL-NEXT:    [[IV:%.*]] = phi i64 [ [[IV_NEXT:%.*]], %[[LOOP]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
@@ -619,7 +619,7 @@ define void @vp_cttz(ptr %a, ptr %b, i64 %N) {
 ; IF-EVL:       [[MIDDLE_BLOCK]]:
 ; IF-EVL-NEXT:    br i1 true, label %[[EXIT:.*]], label %[[SCALAR_PH]]
 ; IF-EVL:       [[SCALAR_PH]]:
-; IF-EVL-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[VECTOR_MEMCHECK]] ], [ 0, %[[ENTRY]] ]
+; IF-EVL-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_MEMCHECK]] ]
 ; IF-EVL-NEXT:    br label %[[LOOP:.*]]
 ; IF-EVL:       [[LOOP]]:
 ; IF-EVL-NEXT:    [[IV:%.*]] = phi i64 [ [[IV_NEXT:%.*]], %[[LOOP]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
@@ -722,7 +722,7 @@ define void @vp_lrint(ptr %a, ptr %b, i64 %N) {
 ; IF-EVL:       [[MIDDLE_BLOCK]]:
 ; IF-EVL-NEXT:    br i1 true, label %[[EXIT:.*]], label %[[SCALAR_PH]]
 ; IF-EVL:       [[SCALAR_PH]]:
-; IF-EVL-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[VECTOR_MEMCHECK]] ], [ 0, %[[ENTRY]] ]
+; IF-EVL-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_MEMCHECK]] ]
 ; IF-EVL-NEXT:    br label %[[LOOP:.*]]
 ; IF-EVL:       [[LOOP]]:
 ; IF-EVL-NEXT:    [[IV:%.*]] = phi i64 [ [[IV_NEXT:%.*]], %[[LOOP]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
@@ -831,7 +831,7 @@ define void @vp_llrint(ptr %a, ptr %b, i64 %N) {
 ; IF-EVL:       [[MIDDLE_BLOCK]]:
 ; IF-EVL-NEXT:    br i1 true, label %[[EXIT:.*]], label %[[SCALAR_PH]]
 ; IF-EVL:       [[SCALAR_PH]]:
-; IF-EVL-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[VECTOR_MEMCHECK]] ], [ 0, %[[ENTRY]] ]
+; IF-EVL-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_MEMCHECK]] ]
 ; IF-EVL-NEXT:    br label %[[LOOP:.*]]
 ; IF-EVL:       [[LOOP]]:
 ; IF-EVL-NEXT:    [[IV:%.*]] = phi i64 [ [[IV_NEXT:%.*]], %[[LOOP]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
@@ -938,7 +938,7 @@ define void @vp_abs(ptr %a, ptr %b, i64 %N) {
 ; IF-EVL:       [[MIDDLE_BLOCK]]:
 ; IF-EVL-NEXT:    br i1 true, label %[[EXIT:.*]], label %[[SCALAR_PH]]
 ; IF-EVL:       [[SCALAR_PH]]:
-; IF-EVL-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[VECTOR_MEMCHECK]] ], [ 0, %[[ENTRY]] ]
+; IF-EVL-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_MEMCHECK]] ]
 ; IF-EVL-NEXT:    br label %[[LOOP:.*]]
 ; IF-EVL:       [[LOOP]]:
 ; IF-EVL-NEXT:    [[IV:%.*]] = phi i64 [ [[IV_NEXT:%.*]], %[[LOOP]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-cast-intrinsics.ll b/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-cast-intrinsics.ll
index 78b9e19fb3966..4557e95f1e1b6 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-cast-intrinsics.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-cast-intrinsics.ll
@@ -59,7 +59,7 @@ define void @vp_sext(ptr %a, ptr %b, i64 %N) {
 ; IF-EVL:       [[MIDDLE_BLOCK]]:
 ; IF-EVL-NEXT:    br i1 true, label %[[EXIT:.*]], label %[[SCALAR_PH]]
 ; IF-EVL:       [[SCALAR_PH]]:
-; IF-EVL-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[VECTOR_MEMCHECK]] ], [ 0, %[[ENTRY]] ]
+; IF-EVL-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_MEMCHECK]] ]
 ; IF-EVL-NEXT:    br label %[[LOOP:.*]]
 ; IF-EVL:       [[LOOP]]:
 ; IF-EVL-NEXT:    [[IV:%.*]] = phi i64 [ [[IV_NEXT:%.*]], %[[LOOP]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
@@ -159,7 +159,7 @@ define void @vp_zext(ptr %a, ptr %b, i64 %N) {
 ; IF-EVL:       [[MIDDLE_BLOCK]]:
 ; IF-EVL-NEXT:    br i1 true, label %[[EXIT:.*]], label %[[SCALAR_PH]]
 ; IF-EVL:       [[SCALAR_PH]]:
-; IF-EVL-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[VECTOR_MEMCHECK]] ], [ 0, %[[ENTRY]] ]
+; IF-EVL-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_MEMCHECK]] ]
 ; IF-EVL-NEXT:    br label %[[LOOP:.*]]
 ; IF-EVL:       [[LOOP]]:
 ; IF-EVL-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
@@ -259,7 +259,7 @@ define void @vp_trunc(ptr %a, ptr %b, i64 %N) {
 ; IF-EVL:       [[MIDDLE_BLOCK]]:
 ; IF-EVL-NEXT:    br i1 true, label %[[EXIT:.*]], label %[[SCALAR_PH]]
 ; IF-EVL:       [[SCALAR_PH]]:
-; IF-EVL-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[VECTOR_MEMCHECK]] ], [ 0, %[[ENTRY]] ]
+; IF-EVL-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_MEMCHECK]] ]
 ; IF-EVL-NEXT:    br label %[[LOOP:.*]]
 ; IF-EVL:       [[LOOP]]:
 ; IF-EVL-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
@@ -359,7 +359,7 @@ define void @vp_fpext(ptr %a, ptr %b, i64 %N) {
 ; IF-EVL:       [[MIDDLE_BLOCK]]:
 ; IF-EVL-NEXT:    br i1 true, label %[[EXIT:.*]], label %[[SCALAR_PH]]
 ; IF-EVL:       [[SCALAR_PH]]:
-; IF-EVL-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[VECTOR_MEMCHECK]] ], [ 0, %[[ENTRY]] ]
+; IF-EVL-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_MEMCHECK]] ]
 ; IF-EVL-NEXT:    br label %[[LOOP:.*]]
 ; IF-EVL:       [[LOOP]]:
 ; IF-EVL-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
@@ -459,7 +459,7 @@ define void @vp_fptrunc(ptr %a, ptr %b, i64 %N) {
 ; IF-EVL:       [[MIDDLE_BLOCK]]:
 ; IF-EVL-NEXT:    br i1 true, label %[[EXIT:.*]], label %[[SCALAR_PH]]
 ; IF-EVL:       [[SCALAR_PH]]:
-; IF-EVL-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[VECTOR_MEMCHECK]] ], [ 0, %[[ENTRY]] ]
+; IF-EVL-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_MEMCHECK]] ]
 ; IF-EVL-NEXT:    br label %[[LOOP:.*]]
 ; IF-EVL:       [[LOOP]]:
 ; IF-EVL-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
@@ -559,7 +559,7 @@ define void @vp_sitofp(ptr %a, ptr %b, i64 %N) {
 ; IF-EVL:       [[MIDDLE_BLOCK]]:
 ; IF-EVL-NEXT:    br i1 true, label %[[EXIT:.*]], label %[[SCALAR_PH]]
 ; IF-EVL:       [[SCALAR_PH]]:
-; IF-EVL-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[VECTOR_MEMCHECK]] ], [ 0, %[[ENTRY]] ]
+; IF-EVL-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_MEMCHECK]] ]
 ; IF-EVL-NEXT:    br label %[[LOOP:.*]]
 ; IF-EVL:       [[LOOP]]:
 ; IF-EVL-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
@@ -659,7 +659,7 @@ define void @vp_uitofp(ptr %a, ptr %b, i64 %N) {
 ; IF-EVL:       [[MIDDLE_BLOCK]]:
 ; IF-EVL-NEXT:    br i1 true, label %[[EXIT:.*]], label %[[SCALAR_PH]]
 ; IF-EVL:       [[SCALAR_PH]]:
-; IF-EVL-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[VECTOR_MEMCHECK]] ], [ 0, %[[ENTRY]] ]
+; IF-EVL-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_MEMCHECK]] ]
 ; IF-EVL-NEXT:    br label %[[LOOP:.*]]
 ; IF-EVL:       [[LOOP]]:
 ; IF-EVL-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
@@ -759,7 +759,7 @@ define void @vp_fptosi(ptr %a, ptr %b, i64 %N) {
 ; IF-EVL:       [[MIDDLE_BLOCK]]:
 ; IF-EVL-NEXT:    br i1 true, label %[[EXIT:.*]], label %[[SCALAR_PH]]
 ; IF-EVL:       [[SCALAR_PH]]:
-; IF-EVL-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[VECTOR_MEMCHECK]] ], [ 0, %[[ENTRY]] ]
+; IF-EVL-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_MEMCHECK]] ]
 ; IF-EVL-NEXT:    br label %[[LOOP:.*]]
 ; IF-EVL:       [[LOOP]]:
 ; IF-EVL-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
@@ -859,7 +859,7 @@ define void @vp_fptoui(ptr %a, ptr %b, i64 %N) {
 ; IF-EVL:       [[MIDDLE_BLOCK]]:
 ; IF-EVL-NEXT:    br i1 true, label %[[EXIT:.*]], label %[[SCALAR_PH]]
 ; IF-EVL:       [[SCALAR_PH]]:
-; IF-EVL-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[VECTOR_MEMCHECK]] ], [ 0, %[[ENTRY]] ]
+; IF-EVL-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_MEMCHECK]] ]
 ; IF-EVL-NEXT:    br label %[[LOOP:.*]]
 ; IF-EVL:       [[LOOP]]:
 ; IF-EVL-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
@@ -959,7 +959,7 @@ define void @vp_inttoptr(ptr %a, ptr %b, i64 %N) {
 ; IF-EVL:       [[MIDDLE_BLOCK]]:
 ; IF-EVL-NEXT:    br i1 true, label %[[EXIT:.*]], label %[[SCALAR_PH]]
 ; IF-EVL:       [[SCALAR_PH]]:
-; IF-EVL-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[VECTOR_MEMCHECK]] ], [ 0, %[[ENTRY]] ]
+; IF-EVL-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_MEMCHECK]] ]
 ; IF-EVL-NEXT:    br label %[[LOOP:.*]]
 ; IF-EVL:       [[LOOP]]:
 ; IF-EVL-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-inloop-reduction.ll b/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-inloop-reduction.ll
index 46340dea4bf8c..6e7f634ce9459 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-inloop-reduction.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-inloop-reduction.ll
@@ -132,18 +132,49 @@ for.end:
 define i32 @mul(ptr %a, i64 %n, i32 %start) {
 ; IF-EVL-LABEL: @mul(
 ; IF-EVL-NEXT:  entry:
+; IF-EVL-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[ENTRY:%.*]]
+; IF-EVL:       vector.ph:
+; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N:%.*]], 3
+; IF-EVL-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], 4
+; IF-EVL-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
+; IF-EVL-NEXT:    [[TRIP_COUNT_MINUS_1:%.*]] = sub i64 [[N]], 1
+; IF-EVL-NEXT:    [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <4 x i64> poison, i64 [[TRIP_COUNT_MINUS_1]], i64 0
+; IF-EVL-NEXT:    [[BROADCAST_SPLAT2:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT1]], <4 x i64> poison, <4 x i32> zeroinitializer
 ; IF-EVL-NEXT:    br label [[FOR_BODY:%.*]]
-; IF-EVL:       for.body:
-; IF-EVL-NEXT:    [[IV:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
+; IF-EVL:       vector.body:
+; IF-EVL-NEXT:    [[IV:%.*]] = phi i64 [ 0, [[ENTRY]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
 ; IF-EVL-NEXT:    [[RDX:%.*]] = phi i32 [ [[START:%.*]], [[ENTRY]] ], [ [[MUL:%.*]], [[FOR_BODY]] ]
-; IF-EVL-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i64 [[IV]]
+; IF-EVL-NEXT:    [[TMP6:%.*]] = add i64 [[IV]], 0
+; IF-EVL-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[IV]], i64 0
+; IF-EVL-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
+; IF-EVL-NEXT:    [[VEC_IV:%.*]] = add <4 x i64> [[BROADCAST_SPLAT]], <i64 0, i64 1, i64 2, i64 3>
+; IF-EVL-NEXT:    [[TMP1:%.*]] = icmp ule <4 x i64> [[VEC_IV]], [[BROADCAST_SPLAT2]]
+; IF-EVL-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i64 [[TMP6]]
+; IF-EVL-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[TMP2]], i32 0
+; IF-EVL-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr [[TMP3]], i32 4, <4 x i1> [[TMP1]], <4 x i32> poison)
+; IF-EVL-NEXT:    [[TMP4:%.*]] = select <4 x i1> [[TMP1]], <4 x i32> [[WIDE_MASKED_LOAD]], <4 x i32> splat (i32 1)
+; IF-EVL-NEXT:    [[TMP5:%.*]] = call i32 @llvm.vector.reduce.mul.v4i32(<4 x i32> [[TMP4]])
+; IF-EVL-NEXT:    [[MUL]] = mul i32 [[TMP5]], [[RDX]]
+; IF-EVL-NEXT:    [[IV_NEXT]] = add i64 [[IV]], 4
+; IF-EVL-NEXT:    [[TMP7:%.*]] = icmp eq i64 [[IV_NEXT]], [[N_VEC]]
+; IF-EVL-NEXT:    br i1 [[TMP7]], label [[MIDDLE_BLOCK:%.*]], label [[FOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; IF-EVL:       middle.block:
+; IF-EVL-NEXT:    br i1 true, label [[FOR_END:%.*]], label [[SCALAR_PH]]
+; IF-EVL:       scalar.ph:
+; IF-EVL-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY1:%.*]] ]
+; IF-EVL-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ [[MUL]], [[MIDDLE_BLOCK]] ], [ [[START]], [[ENTRY1]] ]
+; IF-EVL-NEXT:    br label [[FOR_BODY1:%.*]]
+; IF-EVL:       for.body:
+; IF-EVL-NEXT:    [[IV1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT1:%.*]], [[FOR_BODY1]] ]
+; IF-EVL-NEXT:    [[RDX1:%.*]] = phi i32 [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[MUL1:%.*]], [[FOR_BODY1]] ]
+; IF-EVL-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[IV1]]
 ; IF-EVL-NEXT:    [[TMP0:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
-; IF-EVL-NEXT:    [[MUL]] = mul nsw i32 [[TMP0]], [[RDX]]
-; IF-EVL-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
-; IF-EVL-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N:%.*]]
-; IF-EVL-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_END:%.*]], label [[FOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; IF-EVL-NEXT:    [[MUL1]] = mul nsw i32 [[TMP0]], [[RDX1]]
+; IF-EVL-NEXT:    [[IV_NEXT1]] = add nuw nsw i64 [[IV1]], 1
+; IF-EVL-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT1]], [[N]]
+; IF-EVL-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_END]], label [[FOR_BODY1]], !llvm.loop [[LOOP5:![0-9]+]]
 ; IF-EVL:       for.end:
-; IF-EVL-NEXT:    [[MUL_LCSSA:%.*]] = phi i32 [ [[MUL]], [[FOR_BODY]] ]
+; IF-EVL-NEXT:    [[MUL_LCSSA:%.*]] = phi i32 [ [[MUL1]], [[FOR_BODY1]] ], [ [[MUL]], [[MIDDLE_BLOCK]] ]
 ; IF-EVL-NEXT:    ret i32 [[MUL_LCSSA]]
 ;
 ; NO-VP-LABEL: @mul(
@@ -1161,18 +1192,49 @@ for.end:
 define float @fmul(ptr %a, i64 %n, float %start) {
 ; IF-EVL-LABEL: @fmul(
 ; IF-EVL-NEXT:  entry:
+; IF-EVL-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[ENTRY:%.*]]
+; IF-EVL:       vector.ph:
+; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N:%.*]], 3
+; IF-EVL-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], 4
+; IF-EVL-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
+; IF-EVL-NEXT:    [[TRIP_COUNT_MINUS_1:%.*]] = sub i64 [[N]], 1
+; IF-EVL-NEXT:    [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <4 x i64> poison, i64 [[TRIP_COUNT_MINUS_1]], i64 0
+; IF-EVL-NEXT:    [[BROADCAST_SPLAT2:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT1]], <4 x i64> poison, <4 x i32> zeroinitializer
 ; IF-EVL-NEXT:    br label [[FOR_BODY:%.*]]
-; IF-EVL:       for.body:
-; IF-EVL-NEXT:    [[IV:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
+; IF-EVL:       vector.body:
+; IF-EVL-NEXT:    [[IV:%.*]] = phi i64 [ 0, [[ENTRY]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
 ; IF-EVL-NEXT:    [[RDX:%.*]] = phi float [ [[START:%.*]], [[ENTRY]] ], [ [[MUL:%.*]], [[FOR_BODY]] ]
-; IF-EVL-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds float, ptr [[A:%.*]], i64 [[IV]]
+; IF-EVL-NEXT:    [[TMP6:%.*]] = add i64 [[IV]], 0
+; IF-EVL-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[IV]], i64 0
+; IF-EVL-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
+; IF-EVL-NEXT:    [[VEC_IV:%.*]] = add <4 x i64> [[BROADCAST_SPLAT]], <i64 0, i64 1, i64 2, i64 3>
+; IF-EVL-NEXT:    [[TMP1:%.*]] = icmp ule <4 x i64> [[VEC_IV]], [[BROADCAST_SPLAT2]]
+; IF-EVL-NEXT:    [[TMP2:%.*]] = getelementptr inbounds float, ptr [[A:%.*]], i64 [[TMP6]]
+; IF-EVL-NEXT:    [[TMP3:%.*]] = getelementptr inbounds float, ptr [[TMP2]], i32 0
+; IF-EVL-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <4 x float> @llvm.masked.load.v4f32.p0(ptr [[TMP3]], i32 4, <4 x i1> [[TMP1]], <4 x float> poison)
+; IF-EVL-NEXT:    [[TMP4:%.*]] = select reassoc <4 x i1> [[TMP1]], <4 x float> [[WIDE_MASKED_LOAD]], <4 x float> splat (float 1.000000e+00)
+; IF-EVL-NEXT:    [[TMP5:%.*]] = call reassoc float @llvm.vector.reduce.fmul.v4f32(float 1.000000e+00, <4 x float> [[TMP4]])
+; IF-EVL-NEXT:    [[MUL]] = fmul reassoc float [[TMP5]], [[RDX]]
+; IF-EVL-NEXT:    [[IV_NEXT]] = add i64 [[IV]], 4
+; IF-EVL-NEXT:    [[TMP7:%.*]] = icmp eq i64 [[IV_NEXT]], [[N_VEC]]
+; IF-EVL-NEXT:    br i1 [[TMP7]], label [[MIDDLE_BLOCK:%.*]], label [[FOR_BODY]], !llvm.loop [[LOOP22:![0-9]+]]
+; IF-EVL:       middle.block:
+; IF-EVL-NEXT:    br i1 true, label [[FOR_END:%.*]], label [[SCALAR_PH]]
+; IF-EVL:       scalar.ph:
+; IF-EVL-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY1:%.*]] ]
+; IF-EVL-NEXT:    [[BC_MERGE_RDX:%.*]] = phi float [ [[MUL]], [[MIDDLE_BLOCK]] ], [ [[START]], [[ENTRY1]] ]
+; IF-EVL-NEXT:    br label [[FOR_BODY1:%.*]]
+; IF-EVL:       for.body:
+; IF-EVL-NEXT:    [[IV1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT1:%.*]], [[FOR_BODY1]] ]
+; IF-EVL-NEXT:    [[RDX1:%.*]] = phi float [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[MUL1:%.*]], [[FOR_BODY1]] ]
+; IF-EVL-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[IV1]]
 ; IF-EVL-NEXT:    [[TMP0:%.*]] = load float, ptr [[ARRAYIDX]], align 4
-; IF-EVL-NEXT:    [[MUL]] = fmul reassoc float [[TMP0]], [[RDX]]
-; IF-EVL-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
-; IF-EVL-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N:%.*]]
-; IF-EVL-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_END:%.*]], label [[FOR_BODY]], !llvm.loop [[LOOP4]]
+; IF-EVL-NEXT:    [[MUL1]] = fmul reassoc float [[TMP0]], [[RDX1]]
+; IF-EVL-NEXT:    [[IV_NEXT1]] = add nuw nsw i64 [[IV1]], 1
+; IF-EVL-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT1]], [[N]]
+; IF-EVL-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_END]], label [[FOR_BODY1]], !llvm.loop [[LOOP23:![0-9]+]]
 ; IF-EVL:       for.end:
-; IF-EVL-NEXT:    [[MUL_LCSSA:%.*]] = phi float [ [[MUL]], [[FOR_BODY]] ]
+; IF-EVL-NEXT:    [[MUL_LCSSA:%.*]] = phi float [ [[MUL1]], [[FOR_BODY1]] ], [ [[MUL]], [[MIDDLE_BLOCK]] ]
 ; IF-EVL-NEXT:    ret float [[MUL_LCSSA]]
 ;
 ; NO-VP-LABEL: @fmul(
@@ -1273,7 +1335,7 @@ define float @fmin(ptr %a, i64 %n, float %start) #0 {
 ; IF-EVL-NEXT:    [[INDEX_EVL_NEXT]] = add i64 [[TMP15]], [[EVL_BASED_IV]]
 ; IF-EVL-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP8]]
 ; IF-EVL-NEXT:    [[TMP16:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; IF-EVL-NEXT:    br i1 [[TMP16]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP22:![0-9]+]]
+; IF-EVL-NEXT:    br i1 [[TMP16]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP24:![0-9]+]]
 ; IF-EVL:       middle.block:
 ; IF-EVL-NEXT:    br i1 true, label [[FOR_END:%.*]], label [[SCALAR_PH]]
 ; IF-EVL:       scalar.ph:
@@ -1289,7 +1351,7 @@ define float @fmin(ptr %a, i64 %n, float %start) #0 {
 ; IF-EVL-NEXT:    [[MIN]] = select i1 [[CMP]], float [[TMP17]], float [[RDX]]
 ; IF-EVL-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
 ; IF-EVL-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; IF-EVL-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP23:![0-9]+]]
+; IF-EVL-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP25:![0-9]+]]
 ; IF-EVL:       for.end:
 ; IF-EVL-NEXT:    [[MIN_LCSSA:%.*]] = phi float [ [[MIN]], [[FOR_BODY]] ], [ [[RDX_MINMAX_SELECT]], [[MIDDLE_BLOCK]] ]
 ; IF-EVL-NEXT:    ret float [[MIN_LCSSA]]
@@ -1395,7 +1457,7 @@ define float @fmax(ptr %a, i64 %n, float %start) #0 {
 ; IF-EVL-NEXT:    [[INDEX_EVL_NEXT]] = add i64 [[TMP15]], [[EVL_BASED_IV]]
 ; IF-EVL-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP8]]
 ; IF-EVL-NEXT:    [[TMP16:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; IF-EVL-NEXT:    br i1 [[TMP16]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP24:![0-9]+]]
+; IF-EVL-NEXT:    br i1 [[TMP16]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP26:![0-9]+]]
 ; IF-EVL:       middle.block:
 ; IF-EVL-NEXT:    br i1 true, label [[FOR_END:%.*]], label [[SCALAR_PH]]
 ; IF-EVL:       scalar.ph:
@@ -1411,7 +1473,7 @@ define float @fmax(ptr %a, i64 %n, float %start) #0 {
 ; IF-EVL-NEXT:    [[MAX]] = select i1 [[CMP]], float [[TMP17]], float [[RDX]]
 ; IF-EVL-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
 ; IF-EVL-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; IF-EVL-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP25:![0-9]+]]
+; IF-EVL-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP27:![0-9]+]]
 ; IF-EVL:       for.end:
 ; IF-EVL-NEXT:    [[MAX_LCSSA:%.*]] = phi float [ [[MAX]], [[FOR_BODY]] ], [ [[RDX_MINMAX_SELECT]], [[MIDDLE_BLOCK]] ]
 ; IF-EVL-NEXT:    ret float [[MAX_LCSSA]]
@@ -1485,18 +1547,51 @@ for.end:
 define float @fminimum(ptr %a, i64 %n, float %start) {
 ; IF-EVL-LABEL: @fminimum(
 ; IF-EVL-NEXT:  entry:
+; IF-EVL-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[ENTRY:%.*]]
+; IF-EVL:       vector.ph:
+; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N:%.*]], 7
+; IF-EVL-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], 8
+; IF-EVL-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
+; IF-EVL-NEXT:    [[TRIP_COUNT_MINUS_1:%.*]] = sub i64 [[N]], 1
+; IF-EVL-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <8 x float> poison, float [[START:%.*]], i64 0
+; IF-EVL-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <8 x float> [[BROADCAST_SPLATINSERT]], <8 x float> poison, <8 x i32> zeroinitializer
+; IF-EVL-NEXT:    [[BROADCAST_SPLATINSERT3:%.*]] = insertelement <8 x i64> poison, i64 [[TRIP_COUNT_MINUS_1]], i64 0
+; IF-EVL-NEXT:    [[BROADCAST_SPLAT4:%.*]] = shufflevector <8 x i64> [[BROADCAST_SPLATINSERT3]], <8 x i64> poison, <8 x i32> zeroinitializer
 ; IF-EVL-NEXT:    br label [[FOR_BODY:%.*]]
+; IF-EVL:       vector.body:
+; IF-EVL-NEXT:    [[IV:%.*]] = phi i64 [ 0, [[ENTRY]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
+; IF-EVL-NEXT:    [[VEC_PHI:%.*]] = phi <8 x float> [ [[BROADCAST_SPLAT]], [[ENTRY]] ], [ [[TMP4:%.*]], [[FOR_BODY]] ]
+; IF-EVL-NEXT:    [[TMP8:%.*]] = add i64 [[IV]], 0
+; IF-EVL-NEXT:    [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <8 x i64> poison, i64 [[IV]], i64 0
+; IF-EVL-NEXT:    [[BROADCAST_SPLAT2:%.*]] = shufflevector <8 x i64> [[BROADCAST_SPLATINSERT1]], <8 x i64> poison, <8 x i32> zeroinitializer
+; IF-EVL-NEXT:    [[VEC_IV:%.*]] = add <8 x i64> [[BROADCAST_SPLAT2]], <i64 0, i64 1, i64 2, i64 3, i64 4, i64 5, i64 6, i64 7>
+; IF-EVL-NEXT:    [[TMP1:%.*]] = icmp ule <8 x i64> [[VEC_IV]], [[BROADCAST_SPLAT4]]
+; IF-EVL-NEXT:    [[TMP2:%.*]] = getelementptr inbounds float, ptr [[A:%.*]], i64 [[TMP8]]
+; IF-EVL-NEXT:    [[TMP3:%.*]] = getelementptr inbounds float, ptr [[TMP2]], i32 0
+; IF-EVL-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <8 x float> @llvm.masked.load.v8f32.p0(ptr [[TMP3]], i32 4, <8 x i1> [[TMP1]], <8 x float> poison)
+; IF-EVL-NEXT:    [[TMP4]] = call <8 x float> @llvm.minimum.v8f32(<8 x float> [[VEC_PHI]], <8 x float> [[WIDE_MASKED_LOAD]])
+; IF-EVL-NEXT:    [[TMP5:%.*]] = select <8 x i1> [[TMP1]], <8 x float> [[TMP4]], <8 x float> [[VEC_PHI]]
+; IF-EVL-NEXT:    [[IV_NEXT]] = add i64 [[IV]], 8
+; IF-EVL-NEXT:    [[TMP6:%.*]] = icmp eq i64 [[IV_NEXT]], [[N_VEC]]
+; IF-EVL-NEXT:    br i1 [[TMP6]], label [[MIDDLE_BLOCK:%.*]], label [[FOR_BODY]], !llvm.loop [[LOOP28:![0-9]+]]
+; IF-EVL:       middle.block:
+; IF-EVL-NEXT:    [[TMP7:%.*]] = call float @llvm.vector.reduce.fminimum.v8f32(<8 x float> [[TMP5]])
+; IF-EVL-NEXT:    br i1 true, label [[FOR_END:%.*]], label [[SCALAR_PH]]
+; IF-EVL:       scalar.ph:
+; IF-EVL-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY1:%.*]] ]
+; IF-EVL-NEXT:    [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP7]], [[MIDDLE_BLOCK]] ], [ [[START]], [[ENTRY1]] ]
+; IF-EVL-NEXT:    br label [[FOR_BODY1:%.*]]
 ; IF-EVL:       for.body:
-; IF-EVL-NEXT:    [[IV:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
-; IF-EVL-NEXT:    [[RDX:%.*]] = phi float [ [[START:%.*]], [[ENTRY]] ], [ [[MIN:%.*]], [[FOR_BODY]] ]
-; IF-EVL-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds float, ptr [[A:%.*]], i64 [[IV]]
+; IF-EVL-NEXT:    [[IV1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT1:%.*]], [[FOR_BODY1]] ]
+; IF-EVL-NEXT:    [[RDX:%.*]] = phi float [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[MIN:%.*]], [[FOR_BODY1]] ]
+; IF-EVL-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[IV1]]
 ; IF-EVL-NEXT:    [[TMP0:%.*]] = load float, ptr [[ARRAYIDX]], align 4
 ; IF-EVL-NEXT:    [[MIN]] = tail call float @llvm.minimum.f32(float [[RDX]], float [[TMP0]])
-; IF-EVL-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
-; IF-EVL-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N:%.*]]
-; IF-EVL-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_END:%.*]], label [[FOR_BODY]], !llvm.loop [[LOOP4]]
+; IF-EVL-NEXT:    [[IV_NEXT1]] = add nuw nsw i64 [[IV1]], 1
+; IF-EVL-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT1]], [[N]]
+; IF-EVL-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_END]], label [[FOR_BODY1]], !llvm.loop [[LOOP29:![0-9]+]]
 ; IF-EVL:       for.end:
-; IF-EVL-NEXT:    [[MIN_LCSSA:%.*]] = phi float [ [[MIN]], [[FOR_BODY]] ]
+; IF-EVL-NEXT:    [[MIN_LCSSA:%.*]] = phi float [ [[MIN]], [[FOR_BODY1]] ], [ [[TMP7]], [[MIDDLE_BLOCK]] ]
 ; IF-EVL-NEXT:    ret float [[MIN_LCSSA]]
 ;
 ; NO-VP-LABEL: @fminimum(
@@ -1566,18 +1661,51 @@ for.end:
 define float @fmaximum(ptr %a, i64 %n, float %start) {
 ; IF-EVL-LABEL: @fmaximum(
 ; IF-EVL-NEXT:  entry:
+; IF-EVL-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[ENTRY:%.*]]
+; IF-EVL:       vector.ph:
+; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N:%.*]], 7
+; IF-EVL-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], 8
+; IF-EVL-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
+; IF-EVL-NEXT:    [[TRIP_COUNT_MINUS_1:%.*]] = sub i64 [[N]], 1
+; IF-EVL-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <8 x float> poison, float [[START:%.*]], i64 0
+; IF-EVL-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <8 x float> [[BROADCAST_SPLATINSERT]], <8 x float> poison, <8 x i32> zeroinitializer
+; IF-EVL-NEXT:    [[BROADCAST_SPLATINSERT3:%.*]] = insertelement <8 x i64> poison, i64 [[TRIP_COUNT_MINUS_1]], i64 0
+; IF-EVL-NEXT:    [[BROADCAST_SPLAT4:%.*]] = shufflevector <8 x i64> [[BROADCAST_SPLATINSERT3]], <8 x i64> poison, <8 x i32> zeroinitializer
 ; IF-EVL-NEXT:    br label [[FOR_BODY:%.*]]
+; IF-EVL:       vector.body:
+; IF-EVL-NEXT:    [[IV:%.*]] = phi i64 [ 0, [[ENTRY]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
+; IF-EVL-NEXT:    [[VEC_PHI:%.*]] = phi <8 x float> [ [[BROADCAST_SPLAT]], [[ENTRY]] ], [ [[TMP4:%.*]], [[FOR_BODY]] ]
+; IF-EVL-NEXT:    [[TMP8:%.*]] = add i64 [[IV]], 0
+; IF-EVL-NEXT:    [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <8 x i64> poison, i64 [[IV]], i64 0
+; IF-EVL-NEXT:    [[BROADCAST_SPLAT2:%.*]] = shufflevector <8 x i64> [[BROADCAST_SPLATINSERT1]], <8 x i64> poison, <8 x i32> zeroinitializer
+; IF-EVL-NEXT:    [[VEC_IV:%.*]] = add <8 x i64> [[BROADCAST_SPLAT2]], <i64 0, i64 1, i64 2, i64 3, i64 4, i64 5, i64 6, i64 7>
+; IF-EVL-NEXT:    [[TMP1:%.*]] = icmp ule <8 x i64> [[VEC_IV]], [[BROADCAST_SPLAT4]]
+; IF-EVL-NEXT:    [[TMP2:%.*]] = getelementptr inbounds float, ptr [[A:%.*]], i64 [[TMP8]]
+; IF-EVL-NEXT:    [[TMP3:%.*]] = getelementptr inbounds float, ptr [[TMP2]], i32 0
+; IF-EVL-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <8 x float> @llvm.masked.load.v8f32.p0(ptr [[TMP3]], i32 4, <8 x i1> [[TMP1]], <8 x float> poison)
+; IF-EVL-NEXT:    [[TMP4]] = call <8 x float> @llvm.maximum.v8f32(<8 x float> [[VEC_PHI]], <8 x float> [[WIDE_MASKED_LOAD]])
+; IF-EVL-NEXT:    [[TMP5:%.*]] = select <8 x i1> [[TMP1]], <8 x float> [[TMP4]], <8 x float> [[VEC_PHI]]
+; IF-EVL-NEXT:    [[IV_NEXT]] = add i64 [[IV]], 8
+; IF-EVL-NEXT:    [[TMP6:%.*]] = icmp eq i64 [[IV_NEXT]], [[N_VEC]]
+; IF-EVL-NEXT:    br i1 [[TMP6]], label [[MIDDLE_BLOCK:%.*]], label [[FOR_BODY]], !llvm.loop [[LOOP30:![0-9]+]]
+; IF-EVL:       middle.block:
+; IF-EVL-NEXT:    [[TMP7:%.*]] = call float @llvm.vector.reduce.fmaximum.v8f32(<8 x float> [[TMP5]])
+; IF-EVL-NEXT:    br i1 true, label [[FOR_END:%.*]], label [[SCALAR_PH]]
+; IF-EVL:       scalar.ph:
+; IF-EVL-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY1:%.*]] ]
+; IF-EVL-NEXT:    [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP7]], [[MIDDLE_BLOCK]] ], [ [[START]], [[ENTRY1]] ]
+; IF-EVL-NEXT:    br label [[FOR_BODY1:%.*]]
 ; IF-EVL:       for.body:
-; IF-EVL-NEXT:    [[IV:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
-; IF-EVL-NEXT:    [[RDX:%.*]] = phi float [ [[START:%.*]], [[ENTRY]] ], [ [[MAX:%.*]], [[FOR_BODY]] ]
-; IF-EVL-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds float, ptr [[A:%.*]], i64 [[IV]]
+; IF-EVL-NEXT:    [[IV1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT1:%.*]], [[FOR_BODY1]] ]
+; IF-EVL-NEXT:    [[RDX:%.*]] = phi float [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[MAX:%.*]], [[FOR_BODY1]] ]
+; IF-EVL-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[IV1]]
 ; IF-EVL-NEXT:    [[TMP0:%.*]] = load float, ptr [[ARRAYIDX]], align 4
 ; IF-EVL-NEXT:    [[MAX]] = tail call float @llvm.maximum.f32(float [[RDX]], float [[TMP0]])
-; IF-EVL-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
-; IF-EVL-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N:%.*]]
-; IF-EVL-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_END:%.*]], label [[FOR_BODY]], !llvm.loop [[LOOP4]]
+; IF-EVL-NEXT:    [[IV_NEXT1]] = add nuw nsw i64 [[IV1]], 1
+; IF-EVL-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT1]], [[N]]
+; IF-EVL-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_END]], label [[FOR_BODY1]], !llvm.loop [[LOOP31:![0-9]+]]
 ; IF-EVL:       for.end:
-; IF-EVL-NEXT:    [[MAX_LCSSA:%.*]] = phi float [ [[MAX]], [[FOR_BODY]] ]
+; IF-EVL-NEXT:    [[MAX_LCSSA:%.*]] = phi float [ [[MAX]], [[FOR_BODY1]] ], [ [[TMP7]], [[MIDDLE_BLOCK]] ]
 ; IF-EVL-NEXT:    ret float [[MAX_LCSSA]]
 ;
 ; NO-VP-LABEL: @fmaximum(
@@ -1682,7 +1810,7 @@ define float @fmuladd(ptr %a, ptr %b, i64 %n, float %start) {
 ; IF-EVL-NEXT:    [[INDEX_EVL_NEXT]] = add i64 [[TMP19]], [[EVL_BASED_IV]]
 ; IF-EVL-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP8]]
 ; IF-EVL-NEXT:    [[TMP20:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; IF-EVL-NEXT:    br i1 [[TMP20]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP26:![0-9]+]]
+; IF-EVL-NEXT:    br i1 [[TMP20]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP32:![0-9]+]]
 ; IF-EVL:       middle.block:
 ; IF-EVL-NEXT:    br i1 true, label [[FOR_END:%.*]], label [[SCALAR_PH]]
 ; IF-EVL:       scalar.ph:
@@ -1699,7 +1827,7 @@ define float @fmuladd(ptr %a, ptr %b, i64 %n, float %start) {
 ; IF-EVL-NEXT:    [[MULADD]] = tail call reassoc float @llvm.fmuladd.f32(float [[TMP21]], float [[TMP22]], float [[RDX]])
 ; IF-EVL-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
 ; IF-EVL-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; IF-EVL-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP27:![0-9]+]]
+; IF-EVL-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP33:![0-9]+]]
 ; IF-EVL:       for.end:
 ; IF-EVL-NEXT:    [[MULADD_LCSSA:%.*]] = phi float [ [[MULADD]], [[FOR_BODY]] ], [ [[TMP18]], [[MIDDLE_BLOCK]] ]
 ; IF-EVL-NEXT:    ret float [[MULADD_LCSSA]]
@@ -1810,7 +1938,7 @@ define i32 @anyof_icmp(ptr %a, i64 %n, i32 %start, i32 %inv) {
 ; IF-EVL-NEXT:    [[INDEX_EVL_NEXT]] = add i64 [[TMP17]], [[EVL_BASED_IV]]
 ; IF-EVL-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP8]]
 ; IF-EVL-NEXT:    [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; IF-EVL-NEXT:    br i1 [[TMP18]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP28:![0-9]+]]
+; IF-EVL-NEXT:    br i1 [[TMP18]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP34:![0-9]+]]
 ; IF-EVL:       middle.block:
 ; IF-EVL-NEXT:    [[TMP19:%.*]] = call i1 @llvm.vector.reduce.or.nxv4i1(<vscale x 4 x i1> [[TMP16]])
 ; IF-EVL-NEXT:    [[TMP20:%.*]] = freeze i1 [[TMP19]]
@@ -1829,7 +1957,7 @@ define i32 @anyof_icmp(ptr %a, i64 %n, i32 %start, i32 %inv) {
 ; IF-EVL-NEXT:    [[ANYOF]] = select i1 [[CMP_I]], i32 [[INV]], i32 [[RDX]]
 ; IF-EVL-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
 ; IF-EVL-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; IF-EVL-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP29:![0-9]+]]
+; IF-EVL-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP35:![0-9]+]]
 ; IF-EVL:       for.end:
 ; IF-EVL-NEXT:    [[ANYOF_LCSSA:%.*]] = phi i32 [ [[ANYOF]], [[FOR_BODY]] ], [ [[RDX_SELECT]], [[MIDDLE_BLOCK]] ]
 ; IF-EVL-NEXT:    ret i32 [[ANYOF_LCSSA]]
@@ -1937,7 +2065,7 @@ define i32 @anyof_fcmp(ptr %a, i64 %n, i32 %start, i32 %inv) {
 ; IF-EVL-NEXT:    [[INDEX_EVL_NEXT]] = add i64 [[TMP17]], [[EVL_BASED_IV]]
 ; IF-EVL-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP8]]
 ; IF-EVL-NEXT:    [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; IF-EVL-NEXT:    br i1 [[TMP18]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP30:![0-9]+]]
+; IF-EVL-NEXT:    br i1 [[TMP18]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP36:![0-9]+]]
 ; IF-EVL:       middle.block:
 ; IF-EVL-NEXT:    [[TMP19:%.*]] = call i1 @llvm.vector.reduce.or.nxv4i1(<vscale x 4 x i1> [[TMP16]])
 ; IF-EVL-NEXT:    [[TMP20:%.*]] = freeze i1 [[TMP19]]
@@ -1956,7 +2084,7 @@ define i32 @anyof_fcmp(ptr %a, i64 %n, i32 %start, i32 %inv) {
 ; IF-EVL-NEXT:    [[ANYOF]] = select i1 [[CMP_I]], i32 [[INV]], i32 [[RDX]]
 ; IF-EVL-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
 ; IF-EVL-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; IF-EVL-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP31:![0-9]+]]
+; IF-EVL-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP37:![0-9]+]]
 ; IF-EVL:       for.end:
 ; IF-EVL-NEXT:    [[ANYOF_LCSSA:%.*]] = phi i32 [ [[ANYOF]], [[FOR_BODY]] ], [ [[RDX_SELECT]], [[MIDDLE_BLOCK]] ]
 ; IF-EVL-NEXT:    ret i32 [[ANYOF_LCSSA]]
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-intermediate-store.ll b/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-intermediate-store.ll
index d4881bc50229c..733c05fd9259c 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-intermediate-store.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-intermediate-store.ll
@@ -71,8 +71,8 @@ define void @reduction_intermediate_store(ptr %a, i64 %n, i32 %start, ptr %addr)
 ; IF-EVL-OUTLOOP-NEXT:    store i32 [[TMP23]], ptr [[ADDR]], align 4, !alias.scope [[META6:![0-9]+]], !noalias [[META0]]
 ; IF-EVL-OUTLOOP-NEXT:    br i1 true, label [[FOR_END:%.*]], label [[SCALAR_PH]]
 ; IF-EVL-OUTLOOP:       scalar.ph:
-; IF-EVL-OUTLOOP-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[VECTOR_MEMCHECK]] ], [ 0, [[ENTRY1:%.*]] ]
-; IF-EVL-OUTLOOP-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP23]], [[MIDDLE_BLOCK]] ], [ [[START]], [[VECTOR_MEMCHECK]] ], [ [[START]], [[ENTRY1]] ]
+; IF-EVL-OUTLOOP-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY1:%.*]] ], [ 0, [[VECTOR_MEMCHECK]] ]
+; IF-EVL-OUTLOOP-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP23]], [[MIDDLE_BLOCK]] ], [ [[START]], [[ENTRY1]] ], [ [[START]], [[VECTOR_MEMCHECK]] ]
 ; IF-EVL-OUTLOOP-NEXT:    br label [[FOR_BODY1:%.*]]
 ; IF-EVL-OUTLOOP:       for.body:
 ; IF-EVL-OUTLOOP-NEXT:    [[IV1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT1:%.*]], [[FOR_BODY1]] ]
@@ -134,8 +134,8 @@ define void @reduction_intermediate_store(ptr %a, i64 %n, i32 %start, ptr %addr)
 ; IF-EVL-INLOOP-NEXT:    store i32 [[TMP22]], ptr [[ADDR]], align 4, !alias.scope [[META6:![0-9]+]], !noalias [[META0]]
 ; IF-EVL-INLOOP-NEXT:    br i1 true, label [[FOR_END:%.*]], label [[SCALAR_PH]]
 ; IF-EVL-INLOOP:       scalar.ph:
-; IF-EVL-INLOOP-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[VECTOR_MEMCHECK]] ], [ 0, [[ENTRY:%.*]] ]
-; IF-EVL-INLOOP-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP22]], [[MIDDLE_BLOCK]] ], [ [[START]], [[VECTOR_MEMCHECK]] ], [ [[START]], [[ENTRY]] ]
+; IF-EVL-INLOOP-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ], [ 0, [[VECTOR_MEMCHECK]] ]
+; IF-EVL-INLOOP-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP22]], [[MIDDLE_BLOCK]] ], [ [[START]], [[ENTRY]] ], [ [[START]], [[VECTOR_MEMCHECK]] ]
 ; IF-EVL-INLOOP-NEXT:    br label [[FOR_BODY:%.*]]
 ; IF-EVL-INLOOP:       for.body:
 ; IF-EVL-INLOOP-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
@@ -191,8 +191,8 @@ define void @reduction_intermediate_store(ptr %a, i64 %n, i32 %start, ptr %addr)
 ; NO-VP-OUTLOOP-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
 ; NO-VP-OUTLOOP-NEXT:    br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]]
 ; NO-VP-OUTLOOP:       scalar.ph:
-; NO-VP-OUTLOOP-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[VECTOR_MEMCHECK]] ], [ 0, [[ENTRY:%.*]] ]
-; NO-VP-OUTLOOP-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP14]], [[MIDDLE_BLOCK]] ], [ [[START]], [[VECTOR_MEMCHECK]] ], [ [[START]], [[ENTRY]] ]
+; NO-VP-OUTLOOP-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ], [ 0, [[VECTOR_MEMCHECK]] ]
+; NO-VP-OUTLOOP-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP14]], [[MIDDLE_BLOCK]] ], [ [[START]], [[ENTRY]] ], [ [[START]], [[VECTOR_MEMCHECK]] ]
 ; NO-VP-OUTLOOP-NEXT:    br label [[FOR_BODY:%.*]]
 ; NO-VP-OUTLOOP:       for.body:
 ; NO-VP-OUTLOOP-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
@@ -247,8 +247,8 @@ define void @reduction_intermediate_store(ptr %a, i64 %n, i32 %start, ptr %addr)
 ; NO-VP-INLOOP-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
 ; NO-VP-INLOOP-NEXT:    br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]]
 ; NO-VP-INLOOP:       scalar.ph:
-; NO-VP-INLOOP-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[VECTOR_MEMCHECK]] ], [ 0, [[ENTRY:%.*]] ]
-; NO-VP-INLOOP-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP12]], [[MIDDLE_BLOCK]] ], [ [[START]], [[VECTOR_MEMCHECK]] ], [ [[START]], [[ENTRY]] ]
+; NO-VP-INLOOP-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ], [ 0, [[VECTOR_MEMCHECK]] ]
+; NO-VP-INLOOP-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP12]], [[MIDDLE_BLOCK]] ], [ [[START]], [[ENTRY]] ], [ [[START]], [[VECTOR_MEMCHECK]] ]
 ; NO-VP-INLOOP-NEXT:    br label [[FOR_BODY:%.*]]
 ; NO-VP-INLOOP:       for.body:
 ; NO-VP-INLOOP-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-reduction-cost.ll b/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-reduction-cost.ll
index 69418d9fea00e..15f99931d165f 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-reduction-cost.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-reduction-cost.ll
@@ -1,5 +1,5 @@
 ; REQUIRES: asserts
-; RUN: opt -passes=loop-vectorize -debug-only=loop-vectorize \
+; RUN: opt -passes=loop-vectorize -debug-only=loop-vectorize --disable-output \
 ; RUN: -force-tail-folding-style=data-with-evl \
 ; RUN: -prefer-predicate-over-epilogue=predicate-dont-vectorize \
 ; RUN: -mtriple=riscv64 -mattr=+v -S < %s 2>&1 | FileCheck %s
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-reduction.ll b/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-reduction.ll
index 7557c10892d6d..b81805d473002 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-reduction.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-reduction.ll
@@ -133,18 +133,50 @@ for.end:
 define i32 @mul(ptr %a, i64 %n, i32 %start) {
 ; IF-EVL-LABEL: @mul(
 ; IF-EVL-NEXT:  entry:
+; IF-EVL-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[ENTRY:%.*]]
+; IF-EVL:       vector.ph:
+; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N:%.*]], 7
+; IF-EVL-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], 8
+; IF-EVL-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
+; IF-EVL-NEXT:    [[TRIP_COUNT_MINUS_1:%.*]] = sub i64 [[N]], 1
+; IF-EVL-NEXT:    [[TMP9:%.*]] = insertelement <8 x i32> splat (i32 1), i32 [[START:%.*]], i32 0
+; IF-EVL-NEXT:    [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <8 x i64> poison, i64 [[TRIP_COUNT_MINUS_1]], i64 0
+; IF-EVL-NEXT:    [[BROADCAST_SPLAT2:%.*]] = shufflevector <8 x i64> [[BROADCAST_SPLATINSERT1]], <8 x i64> poison, <8 x i32> zeroinitializer
 ; IF-EVL-NEXT:    br label [[FOR_BODY:%.*]]
+; IF-EVL:       vector.body:
+; IF-EVL-NEXT:    [[IV:%.*]] = phi i64 [ 0, [[ENTRY]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
+; IF-EVL-NEXT:    [[VEC_PHI:%.*]] = phi <8 x i32> [ [[TMP9]], [[ENTRY]] ], [ [[TMP5:%.*]], [[FOR_BODY]] ]
+; IF-EVL-NEXT:    [[TMP1:%.*]] = add i64 [[IV]], 0
+; IF-EVL-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <8 x i64> poison, i64 [[IV]], i64 0
+; IF-EVL-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <8 x i64> [[BROADCAST_SPLATINSERT]], <8 x i64> poison, <8 x i32> zeroinitializer
+; IF-EVL-NEXT:    [[VEC_IV:%.*]] = add <8 x i64> [[BROADCAST_SPLAT]], <i64 0, i64 1, i64 2, i64 3, i64 4, i64 5, i64 6, i64 7>
+; IF-EVL-NEXT:    [[TMP2:%.*]] = icmp ule <8 x i64> [[VEC_IV]], [[BROADCAST_SPLAT2]]
+; IF-EVL-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i64 [[TMP1]]
+; IF-EVL-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i32, ptr [[TMP3]], i32 0
+; IF-EVL-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <8 x i32> @llvm.masked.load.v8i32.p0(ptr [[TMP4]], i32 4, <8 x i1> [[TMP2]], <8 x i32> poison)
+; IF-EVL-NEXT:    [[TMP5]] = mul <8 x i32> [[WIDE_MASKED_LOAD]], [[VEC_PHI]]
+; IF-EVL-NEXT:    [[TMP6:%.*]] = select <8 x i1> [[TMP2]], <8 x i32> [[TMP5]], <8 x i32> [[VEC_PHI]]
+; IF-EVL-NEXT:    [[IV_NEXT]] = add i64 [[IV]], 8
+; IF-EVL-NEXT:    [[TMP7:%.*]] = icmp eq i64 [[IV_NEXT]], [[N_VEC]]
+; IF-EVL-NEXT:    br i1 [[TMP7]], label [[MIDDLE_BLOCK:%.*]], label [[FOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; IF-EVL:       middle.block:
+; IF-EVL-NEXT:    [[TMP8:%.*]] = call i32 @llvm.vector.reduce.mul.v8i32(<8 x i32> [[TMP6]])
+; IF-EVL-NEXT:    br i1 true, label [[FOR_END:%.*]], label [[SCALAR_PH]]
+; IF-EVL:       scalar.ph:
+; IF-EVL-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY1:%.*]] ]
+; IF-EVL-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP8]], [[MIDDLE_BLOCK]] ], [ [[START]], [[ENTRY1]] ]
+; IF-EVL-NEXT:    br label [[FOR_BODY1:%.*]]
 ; IF-EVL:       for.body:
-; IF-EVL-NEXT:    [[IV:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
-; IF-EVL-NEXT:    [[RDX:%.*]] = phi i32 [ [[START:%.*]], [[ENTRY]] ], [ [[MUL:%.*]], [[FOR_BODY]] ]
-; IF-EVL-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i64 [[IV]]
+; IF-EVL-NEXT:    [[IV1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT1:%.*]], [[FOR_BODY1]] ]
+; IF-EVL-NEXT:    [[RDX:%.*]] = phi i32 [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[MUL:%.*]], [[FOR_BODY1]] ]
+; IF-EVL-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[IV1]]
 ; IF-EVL-NEXT:    [[TMP0:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
 ; IF-EVL-NEXT:    [[MUL]] = mul nsw i32 [[TMP0]], [[RDX]]
-; IF-EVL-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
-; IF-EVL-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N:%.*]]
-; IF-EVL-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_END:%.*]], label [[FOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; IF-EVL-NEXT:    [[IV_NEXT1]] = add nuw nsw i64 [[IV1]], 1
+; IF-EVL-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT1]], [[N]]
+; IF-EVL-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_END]], label [[FOR_BODY1]], !llvm.loop [[LOOP5:![0-9]+]]
 ; IF-EVL:       for.end:
-; IF-EVL-NEXT:    [[MUL_LCSSA:%.*]] = phi i32 [ [[MUL]], [[FOR_BODY]] ]
+; IF-EVL-NEXT:    [[MUL_LCSSA:%.*]] = phi i32 [ [[MUL]], [[FOR_BODY1]] ], [ [[TMP8]], [[MIDDLE_BLOCK]] ]
 ; IF-EVL-NEXT:    ret i32 [[MUL_LCSSA]]
 ;
 ; NO-VP-LABEL: @mul(
@@ -1202,18 +1234,50 @@ for.end:
 define float @fmul(ptr %a, i64 %n, float %start) {
 ; IF-EVL-LABEL: @fmul(
 ; IF-EVL-NEXT:  entry:
+; IF-EVL-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[ENTRY:%.*]]
+; IF-EVL:       vector.ph:
+; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N:%.*]], 7
+; IF-EVL-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], 8
+; IF-EVL-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
+; IF-EVL-NEXT:    [[TRIP_COUNT_MINUS_1:%.*]] = sub i64 [[N]], 1
+; IF-EVL-NEXT:    [[TMP9:%.*]] = insertelement <8 x float> splat (float 1.000000e+00), float [[START:%.*]], i32 0
+; IF-EVL-NEXT:    [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <8 x i64> poison, i64 [[TRIP_COUNT_MINUS_1]], i64 0
+; IF-EVL-NEXT:    [[BROADCAST_SPLAT2:%.*]] = shufflevector <8 x i64> [[BROADCAST_SPLATINSERT1]], <8 x i64> poison, <8 x i32> zeroinitializer
 ; IF-EVL-NEXT:    br label [[FOR_BODY:%.*]]
+; IF-EVL:       vector.body:
+; IF-EVL-NEXT:    [[IV:%.*]] = phi i64 [ 0, [[ENTRY]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
+; IF-EVL-NEXT:    [[VEC_PHI:%.*]] = phi <8 x float> [ [[TMP9]], [[ENTRY]] ], [ [[TMP5:%.*]], [[FOR_BODY]] ]
+; IF-EVL-NEXT:    [[TMP1:%.*]] = add i64 [[IV]], 0
+; IF-EVL-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <8 x i64> poison, i64 [[IV]], i64 0
+; IF-EVL-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <8 x i64> [[BROADCAST_SPLATINSERT]], <8 x i64> poison, <8 x i32> zeroinitializer
+; IF-EVL-NEXT:    [[VEC_IV:%.*]] = add <8 x i64> [[BROADCAST_SPLAT]], <i64 0, i64 1, i64 2, i64 3, i64 4, i64 5, i64 6, i64 7>
+; IF-EVL-NEXT:    [[TMP2:%.*]] = icmp ule <8 x i64> [[VEC_IV]], [[BROADCAST_SPLAT2]]
+; IF-EVL-NEXT:    [[TMP3:%.*]] = getelementptr inbounds float, ptr [[A:%.*]], i64 [[TMP1]]
+; IF-EVL-NEXT:    [[TMP4:%.*]] = getelementptr inbounds float, ptr [[TMP3]], i32 0
+; IF-EVL-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <8 x float> @llvm.masked.load.v8f32.p0(ptr [[TMP4]], i32 4, <8 x i1> [[TMP2]], <8 x float> poison)
+; IF-EVL-NEXT:    [[TMP5]] = fmul reassoc <8 x float> [[WIDE_MASKED_LOAD]], [[VEC_PHI]]
+; IF-EVL-NEXT:    [[TMP6:%.*]] = select reassoc <8 x i1> [[TMP2]], <8 x float> [[TMP5]], <8 x float> [[VEC_PHI]]
+; IF-EVL-NEXT:    [[IV_NEXT]] = add i64 [[IV]], 8
+; IF-EVL-NEXT:    [[TMP7:%.*]] = icmp eq i64 [[IV_NEXT]], [[N_VEC]]
+; IF-EVL-NEXT:    br i1 [[TMP7]], label [[MIDDLE_BLOCK:%.*]], label [[FOR_BODY]], !llvm.loop [[LOOP22:![0-9]+]]
+; IF-EVL:       middle.block:
+; IF-EVL-NEXT:    [[TMP8:%.*]] = call reassoc float @llvm.vector.reduce.fmul.v8f32(float 1.000000e+00, <8 x float> [[TMP6]])
+; IF-EVL-NEXT:    br i1 true, label [[FOR_END:%.*]], label [[SCALAR_PH]]
+; IF-EVL:       scalar.ph:
+; IF-EVL-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY1:%.*]] ]
+; IF-EVL-NEXT:    [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP8]], [[MIDDLE_BLOCK]] ], [ [[START]], [[ENTRY1]] ]
+; IF-EVL-NEXT:    br label [[FOR_BODY1:%.*]]
 ; IF-EVL:       for.body:
-; IF-EVL-NEXT:    [[IV:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
-; IF-EVL-NEXT:    [[RDX:%.*]] = phi float [ [[START:%.*]], [[ENTRY]] ], [ [[MUL:%.*]], [[FOR_BODY]] ]
-; IF-EVL-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds float, ptr [[A:%.*]], i64 [[IV]]
+; IF-EVL-NEXT:    [[IV1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT1:%.*]], [[FOR_BODY1]] ]
+; IF-EVL-NEXT:    [[RDX:%.*]] = phi float [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[MUL:%.*]], [[FOR_BODY1]] ]
+; IF-EVL-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[IV1]]
 ; IF-EVL-NEXT:    [[TMP0:%.*]] = load float, ptr [[ARRAYIDX]], align 4
 ; IF-EVL-NEXT:    [[MUL]] = fmul reassoc float [[TMP0]], [[RDX]]
-; IF-EVL-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
-; IF-EVL-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N:%.*]]
-; IF-EVL-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_END:%.*]], label [[FOR_BODY]], !llvm.loop [[LOOP4]]
+; IF-EVL-NEXT:    [[IV_NEXT1]] = add nuw nsw i64 [[IV1]], 1
+; IF-EVL-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT1]], [[N]]
+; IF-EVL-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_END]], label [[FOR_BODY1]], !llvm.loop [[LOOP23:![0-9]+]]
 ; IF-EVL:       for.end:
-; IF-EVL-NEXT:    [[MUL_LCSSA:%.*]] = phi float [ [[MUL]], [[FOR_BODY]] ]
+; IF-EVL-NEXT:    [[MUL_LCSSA:%.*]] = phi float [ [[MUL]], [[FOR_BODY1]] ], [ [[TMP8]], [[MIDDLE_BLOCK]] ]
 ; IF-EVL-NEXT:    ret float [[MUL_LCSSA]]
 ;
 ; NO-VP-LABEL: @fmul(
@@ -1316,7 +1380,7 @@ define float @fmin(ptr %a, i64 %n, float %start) #0 {
 ; IF-EVL-NEXT:    [[INDEX_EVL_NEXT]] = add i64 [[TMP16]], [[EVL_BASED_IV]]
 ; IF-EVL-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP8]]
 ; IF-EVL-NEXT:    [[TMP17:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; IF-EVL-NEXT:    br i1 [[TMP17]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP22:![0-9]+]]
+; IF-EVL-NEXT:    br i1 [[TMP17]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP24:![0-9]+]]
 ; IF-EVL:       middle.block:
 ; IF-EVL-NEXT:    [[TMP18:%.*]] = call fast float @llvm.vector.reduce.fmin.nxv4f32(<vscale x 4 x float> [[TMP15]])
 ; IF-EVL-NEXT:    br i1 true, label [[FOR_END:%.*]], label [[SCALAR_PH]]
@@ -1333,7 +1397,7 @@ define float @fmin(ptr %a, i64 %n, float %start) #0 {
 ; IF-EVL-NEXT:    [[MIN]] = select i1 [[CMP]], float [[TMP19]], float [[RDX]]
 ; IF-EVL-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
 ; IF-EVL-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; IF-EVL-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP23:![0-9]+]]
+; IF-EVL-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP25:![0-9]+]]
 ; IF-EVL:       for.end:
 ; IF-EVL-NEXT:    [[MIN_LCSSA:%.*]] = phi float [ [[MIN]], [[FOR_BODY]] ], [ [[TMP18]], [[MIDDLE_BLOCK]] ]
 ; IF-EVL-NEXT:    ret float [[MIN_LCSSA]]
@@ -1443,7 +1507,7 @@ define float @fmax(ptr %a, i64 %n, float %start) #0 {
 ; IF-EVL-NEXT:    [[INDEX_EVL_NEXT]] = add i64 [[TMP16]], [[EVL_BASED_IV]]
 ; IF-EVL-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP8]]
 ; IF-EVL-NEXT:    [[TMP17:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; IF-EVL-NEXT:    br i1 [[TMP17]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP24:![0-9]+]]
+; IF-EVL-NEXT:    br i1 [[TMP17]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP26:![0-9]+]]
 ; IF-EVL:       middle.block:
 ; IF-EVL-NEXT:    [[TMP18:%.*]] = call fast float @llvm.vector.reduce.fmax.nxv4f32(<vscale x 4 x float> [[TMP15]])
 ; IF-EVL-NEXT:    br i1 true, label [[FOR_END:%.*]], label [[SCALAR_PH]]
@@ -1460,7 +1524,7 @@ define float @fmax(ptr %a, i64 %n, float %start) #0 {
 ; IF-EVL-NEXT:    [[MAX]] = select i1 [[CMP]], float [[TMP19]], float [[RDX]]
 ; IF-EVL-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
 ; IF-EVL-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; IF-EVL-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP25:![0-9]+]]
+; IF-EVL-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP27:![0-9]+]]
 ; IF-EVL:       for.end:
 ; IF-EVL-NEXT:    [[MAX_LCSSA:%.*]] = phi float [ [[MAX]], [[FOR_BODY]] ], [ [[TMP18]], [[MIDDLE_BLOCK]] ]
 ; IF-EVL-NEXT:    ret float [[MAX_LCSSA]]
@@ -1536,18 +1600,51 @@ for.end:
 define float @fminimum(ptr %a, i64 %n, float %start) {
 ; IF-EVL-LABEL: @fminimum(
 ; IF-EVL-NEXT:  entry:
+; IF-EVL-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[ENTRY:%.*]]
+; IF-EVL:       vector.ph:
+; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N:%.*]], 7
+; IF-EVL-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], 8
+; IF-EVL-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
+; IF-EVL-NEXT:    [[TRIP_COUNT_MINUS_1:%.*]] = sub i64 [[N]], 1
+; IF-EVL-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <8 x float> poison, float [[START:%.*]], i64 0
+; IF-EVL-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <8 x float> [[BROADCAST_SPLATINSERT]], <8 x float> poison, <8 x i32> zeroinitializer
+; IF-EVL-NEXT:    [[BROADCAST_SPLATINSERT3:%.*]] = insertelement <8 x i64> poison, i64 [[TRIP_COUNT_MINUS_1]], i64 0
+; IF-EVL-NEXT:    [[BROADCAST_SPLAT4:%.*]] = shufflevector <8 x i64> [[BROADCAST_SPLATINSERT3]], <8 x i64> poison, <8 x i32> zeroinitializer
 ; IF-EVL-NEXT:    br label [[FOR_BODY:%.*]]
+; IF-EVL:       vector.body:
+; IF-EVL-NEXT:    [[IV:%.*]] = phi i64 [ 0, [[ENTRY]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
+; IF-EVL-NEXT:    [[VEC_PHI:%.*]] = phi <8 x float> [ [[BROADCAST_SPLAT]], [[ENTRY]] ], [ [[TMP4:%.*]], [[FOR_BODY]] ]
+; IF-EVL-NEXT:    [[TMP8:%.*]] = add i64 [[IV]], 0
+; IF-EVL-NEXT:    [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <8 x i64> poison, i64 [[IV]], i64 0
+; IF-EVL-NEXT:    [[BROADCAST_SPLAT2:%.*]] = shufflevector <8 x i64> [[BROADCAST_SPLATINSERT1]], <8 x i64> poison, <8 x i32> zeroinitializer
+; IF-EVL-NEXT:    [[VEC_IV:%.*]] = add <8 x i64> [[BROADCAST_SPLAT2]], <i64 0, i64 1, i64 2, i64 3, i64 4, i64 5, i64 6, i64 7>
+; IF-EVL-NEXT:    [[TMP1:%.*]] = icmp ule <8 x i64> [[VEC_IV]], [[BROADCAST_SPLAT4]]
+; IF-EVL-NEXT:    [[TMP2:%.*]] = getelementptr inbounds float, ptr [[A:%.*]], i64 [[TMP8]]
+; IF-EVL-NEXT:    [[TMP3:%.*]] = getelementptr inbounds float, ptr [[TMP2]], i32 0
+; IF-EVL-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <8 x float> @llvm.masked.load.v8f32.p0(ptr [[TMP3]], i32 4, <8 x i1> [[TMP1]], <8 x float> poison)
+; IF-EVL-NEXT:    [[TMP4]] = call <8 x float> @llvm.minimum.v8f32(<8 x float> [[VEC_PHI]], <8 x float> [[WIDE_MASKED_LOAD]])
+; IF-EVL-NEXT:    [[TMP5:%.*]] = select <8 x i1> [[TMP1]], <8 x float> [[TMP4]], <8 x float> [[VEC_PHI]]
+; IF-EVL-NEXT:    [[IV_NEXT]] = add i64 [[IV]], 8
+; IF-EVL-NEXT:    [[TMP6:%.*]] = icmp eq i64 [[IV_NEXT]], [[N_VEC]]
+; IF-EVL-NEXT:    br i1 [[TMP6]], label [[MIDDLE_BLOCK:%.*]], label [[FOR_BODY]], !llvm.loop [[LOOP28:![0-9]+]]
+; IF-EVL:       middle.block:
+; IF-EVL-NEXT:    [[TMP7:%.*]] = call float @llvm.vector.reduce.fminimum.v8f32(<8 x float> [[TMP5]])
+; IF-EVL-NEXT:    br i1 true, label [[FOR_END:%.*]], label [[SCALAR_PH]]
+; IF-EVL:       scalar.ph:
+; IF-EVL-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY1:%.*]] ]
+; IF-EVL-NEXT:    [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP7]], [[MIDDLE_BLOCK]] ], [ [[START]], [[ENTRY1]] ]
+; IF-EVL-NEXT:    br label [[FOR_BODY1:%.*]]
 ; IF-EVL:       for.body:
-; IF-EVL-NEXT:    [[IV:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
-; IF-EVL-NEXT:    [[RDX:%.*]] = phi float [ [[START:%.*]], [[ENTRY]] ], [ [[MIN:%.*]], [[FOR_BODY]] ]
-; IF-EVL-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds float, ptr [[A:%.*]], i64 [[IV]]
+; IF-EVL-NEXT:    [[IV1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT1:%.*]], [[FOR_BODY1]] ]
+; IF-EVL-NEXT:    [[RDX:%.*]] = phi float [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[MIN:%.*]], [[FOR_BODY1]] ]
+; IF-EVL-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[IV1]]
 ; IF-EVL-NEXT:    [[TMP0:%.*]] = load float, ptr [[ARRAYIDX]], align 4
 ; IF-EVL-NEXT:    [[MIN]] = tail call float @llvm.minimum.f32(float [[RDX]], float [[TMP0]])
-; IF-EVL-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
-; IF-EVL-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N:%.*]]
-; IF-EVL-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_END:%.*]], label [[FOR_BODY]], !llvm.loop [[LOOP4]]
+; IF-EVL-NEXT:    [[IV_NEXT1]] = add nuw nsw i64 [[IV1]], 1
+; IF-EVL-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT1]], [[N]]
+; IF-EVL-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_END]], label [[FOR_BODY1]], !llvm.loop [[LOOP29:![0-9]+]]
 ; IF-EVL:       for.end:
-; IF-EVL-NEXT:    [[MIN_LCSSA:%.*]] = phi float [ [[MIN]], [[FOR_BODY]] ]
+; IF-EVL-NEXT:    [[MIN_LCSSA:%.*]] = phi float [ [[MIN]], [[FOR_BODY1]] ], [ [[TMP7]], [[MIDDLE_BLOCK]] ]
 ; IF-EVL-NEXT:    ret float [[MIN_LCSSA]]
 ;
 ; NO-VP-LABEL: @fminimum(
@@ -1617,18 +1714,51 @@ for.end:
 define float @fmaximum(ptr %a, i64 %n, float %start) {
 ; IF-EVL-LABEL: @fmaximum(
 ; IF-EVL-NEXT:  entry:
+; IF-EVL-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[ENTRY:%.*]]
+; IF-EVL:       vector.ph:
+; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N:%.*]], 7
+; IF-EVL-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], 8
+; IF-EVL-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
+; IF-EVL-NEXT:    [[TRIP_COUNT_MINUS_1:%.*]] = sub i64 [[N]], 1
+; IF-EVL-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <8 x float> poison, float [[START:%.*]], i64 0
+; IF-EVL-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <8 x float> [[BROADCAST_SPLATINSERT]], <8 x float> poison, <8 x i32> zeroinitializer
+; IF-EVL-NEXT:    [[BROADCAST_SPLATINSERT3:%.*]] = insertelement <8 x i64> poison, i64 [[TRIP_COUNT_MINUS_1]], i64 0
+; IF-EVL-NEXT:    [[BROADCAST_SPLAT4:%.*]] = shufflevector <8 x i64> [[BROADCAST_SPLATINSERT3]], <8 x i64> poison, <8 x i32> zeroinitializer
 ; IF-EVL-NEXT:    br label [[FOR_BODY:%.*]]
+; IF-EVL:       vector.body:
+; IF-EVL-NEXT:    [[IV:%.*]] = phi i64 [ 0, [[ENTRY]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
+; IF-EVL-NEXT:    [[VEC_PHI:%.*]] = phi <8 x float> [ [[BROADCAST_SPLAT]], [[ENTRY]] ], [ [[TMP4:%.*]], [[FOR_BODY]] ]
+; IF-EVL-NEXT:    [[TMP8:%.*]] = add i64 [[IV]], 0
+; IF-EVL-NEXT:    [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <8 x i64> poison, i64 [[IV]], i64 0
+; IF-EVL-NEXT:    [[BROADCAST_SPLAT2:%.*]] = shufflevector <8 x i64> [[BROADCAST_SPLATINSERT1]], <8 x i64> poison, <8 x i32> zeroinitializer
+; IF-EVL-NEXT:    [[VEC_IV:%.*]] = add <8 x i64> [[BROADCAST_SPLAT2]], <i64 0, i64 1, i64 2, i64 3, i64 4, i64 5, i64 6, i64 7>
+; IF-EVL-NEXT:    [[TMP1:%.*]] = icmp ule <8 x i64> [[VEC_IV]], [[BROADCAST_SPLAT4]]
+; IF-EVL-NEXT:    [[TMP2:%.*]] = getelementptr inbounds float, ptr [[A:%.*]], i64 [[TMP8]]
+; IF-EVL-NEXT:    [[TMP3:%.*]] = getelementptr inbounds float, ptr [[TMP2]], i32 0
+; IF-EVL-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <8 x float> @llvm.masked.load.v8f32.p0(ptr [[TMP3]], i32 4, <8 x i1> [[TMP1]], <8 x float> poison)
+; IF-EVL-NEXT:    [[TMP4]] = call <8 x float> @llvm.maximum.v8f32(<8 x float> [[VEC_PHI]], <8 x float> [[WIDE_MASKED_LOAD]])
+; IF-EVL-NEXT:    [[TMP5:%.*]] = select <8 x i1> [[TMP1]], <8 x float> [[TMP4]], <8 x float> [[VEC_PHI]]
+; IF-EVL-NEXT:    [[IV_NEXT]] = add i64 [[IV]], 8
+; IF-EVL-NEXT:    [[TMP6:%.*]] = icmp eq i64 [[IV_NEXT]], [[N_VEC]]
+; IF-EVL-NEXT:    br i1 [[TMP6]], label [[MIDDLE_BLOCK:%.*]], label [[FOR_BODY]], !llvm.loop [[LOOP30:![0-9]+]]
+; IF-EVL:       middle.block:
+; IF-EVL-NEXT:    [[TMP7:%.*]] = call float @llvm.vector.reduce.fmaximum.v8f32(<8 x float> [[TMP5]])
+; IF-EVL-NEXT:    br i1 true, label [[FOR_END:%.*]], label [[SCALAR_PH]]
+; IF-EVL:       scalar.ph:
+; IF-EVL-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY1:%.*]] ]
+; IF-EVL-NEXT:    [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP7]], [[MIDDLE_BLOCK]] ], [ [[START]], [[ENTRY1]] ]
+; IF-EVL-NEXT:    br label [[FOR_BODY1:%.*]]
 ; IF-EVL:       for.body:
-; IF-EVL-NEXT:    [[IV:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
-; IF-EVL-NEXT:    [[RDX:%.*]] = phi float [ [[START:%.*]], [[ENTRY]] ], [ [[MAX:%.*]], [[FOR_BODY]] ]
-; IF-EVL-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds float, ptr [[A:%.*]], i64 [[IV]]
+; IF-EVL-NEXT:    [[IV1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT1:%.*]], [[FOR_BODY1]] ]
+; IF-EVL-NEXT:    [[RDX:%.*]] = phi float [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[MAX:%.*]], [[FOR_BODY1]] ]
+; IF-EVL-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[IV1]]
 ; IF-EVL-NEXT:    [[TMP0:%.*]] = load float, ptr [[ARRAYIDX]], align 4
 ; IF-EVL-NEXT:    [[MAX]] = tail call float @llvm.maximum.f32(float [[RDX]], float [[TMP0]])
-; IF-EVL-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
-; IF-EVL-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N:%.*]]
-; IF-EVL-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_END:%.*]], label [[FOR_BODY]], !llvm.loop [[LOOP4]]
+; IF-EVL-NEXT:    [[IV_NEXT1]] = add nuw nsw i64 [[IV1]], 1
+; IF-EVL-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT1]], [[N]]
+; IF-EVL-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_END]], label [[FOR_BODY1]], !llvm.loop [[LOOP31:![0-9]+]]
 ; IF-EVL:       for.end:
-; IF-EVL-NEXT:    [[MAX_LCSSA:%.*]] = phi float [ [[MAX]], [[FOR_BODY]] ]
+; IF-EVL-NEXT:    [[MAX_LCSSA:%.*]] = phi float [ [[MAX]], [[FOR_BODY1]] ], [ [[TMP7]], [[MIDDLE_BLOCK]] ]
 ; IF-EVL-NEXT:    ret float [[MAX_LCSSA]]
 ;
 ; NO-VP-LABEL: @fmaximum(
@@ -1733,7 +1863,7 @@ define float @fmuladd(ptr %a, ptr %b, i64 %n, float %start) {
 ; IF-EVL-NEXT:    [[INDEX_EVL_NEXT]] = add i64 [[TMP18]], [[EVL_BASED_IV]]
 ; IF-EVL-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP8]]
 ; IF-EVL-NEXT:    [[TMP19:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; IF-EVL-NEXT:    br i1 [[TMP19]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP26:![0-9]+]]
+; IF-EVL-NEXT:    br i1 [[TMP19]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP32:![0-9]+]]
 ; IF-EVL:       middle.block:
 ; IF-EVL-NEXT:    [[TMP20:%.*]] = call reassoc float @llvm.vector.reduce.fadd.nxv4f32(float -0.000000e+00, <vscale x 4 x float> [[TMP17]])
 ; IF-EVL-NEXT:    br i1 true, label [[FOR_END:%.*]], label [[SCALAR_PH]]
@@ -1751,7 +1881,7 @@ define float @fmuladd(ptr %a, ptr %b, i64 %n, float %start) {
 ; IF-EVL-NEXT:    [[MULADD]] = tail call reassoc float @llvm.fmuladd.f32(float [[TMP21]], float [[TMP22]], float [[RDX]])
 ; IF-EVL-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
 ; IF-EVL-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; IF-EVL-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP27:![0-9]+]]
+; IF-EVL-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP33:![0-9]+]]
 ; IF-EVL:       for.end:
 ; IF-EVL-NEXT:    [[MULADD_LCSSA:%.*]] = phi float [ [[MULADD]], [[FOR_BODY]] ], [ [[TMP20]], [[MIDDLE_BLOCK]] ]
 ; IF-EVL-NEXT:    ret float [[MULADD_LCSSA]]
@@ -1862,7 +1992,7 @@ define i32 @anyof_icmp(ptr %a, i64 %n, i32 %start, i32 %inv) {
 ; IF-EVL-NEXT:    [[INDEX_EVL_NEXT]] = add i64 [[TMP16]], [[EVL_BASED_IV]]
 ; IF-EVL-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP8]]
 ; IF-EVL-NEXT:    [[TMP17:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; IF-EVL-NEXT:    br i1 [[TMP17]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP28:![0-9]+]]
+; IF-EVL-NEXT:    br i1 [[TMP17]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP34:![0-9]+]]
 ; IF-EVL:       middle.block:
 ; IF-EVL-NEXT:    [[TMP18:%.*]] = call i1 @llvm.vector.reduce.or.nxv4i1(<vscale x 4 x i1> [[TMP15]])
 ; IF-EVL-NEXT:    [[TMP19:%.*]] = freeze i1 [[TMP18]]
@@ -1881,7 +2011,7 @@ define i32 @anyof_icmp(ptr %a, i64 %n, i32 %start, i32 %inv) {
 ; IF-EVL-NEXT:    [[ANYOF]] = select i1 [[CMP_I]], i32 [[INV]], i32 [[RDX]]
 ; IF-EVL-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
 ; IF-EVL-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; IF-EVL-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP29:![0-9]+]]
+; IF-EVL-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP35:![0-9]+]]
 ; IF-EVL:       for.end:
 ; IF-EVL-NEXT:    [[ANYOF_LCSSA:%.*]] = phi i32 [ [[ANYOF]], [[FOR_BODY]] ], [ [[RDX_SELECT]], [[MIDDLE_BLOCK]] ]
 ; IF-EVL-NEXT:    ret i32 [[ANYOF_LCSSA]]
@@ -1989,7 +2119,7 @@ define i32 @anyof_fcmp(ptr %a, i64 %n, i32 %start, i32 %inv) {
 ; IF-EVL-NEXT:    [[INDEX_EVL_NEXT]] = add i64 [[TMP16]], [[EVL_BASED_IV]]
 ; IF-EVL-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP8]]
 ; IF-EVL-NEXT:    [[TMP17:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; IF-EVL-NEXT:    br i1 [[TMP17]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP30:![0-9]+]]
+; IF-EVL-NEXT:    br i1 [[TMP17]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP36:![0-9]+]]
 ; IF-EVL:       middle.block:
 ; IF-EVL-NEXT:    [[TMP18:%.*]] = call i1 @llvm.vector.reduce.or.nxv4i1(<vscale x 4 x i1> [[TMP15]])
 ; IF-EVL-NEXT:    [[TMP19:%.*]] = freeze i1 [[TMP18]]
@@ -2008,7 +2138,7 @@ define i32 @anyof_fcmp(ptr %a, i64 %n, i32 %start, i32 %inv) {
 ; IF-EVL-NEXT:    [[ANYOF]] = select i1 [[CMP_I]], i32 [[INV]], i32 [[RDX]]
 ; IF-EVL-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
 ; IF-EVL-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; IF-EVL-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP31:![0-9]+]]
+; IF-EVL-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP37:![0-9]+]]
 ; IF-EVL:       for.end:
 ; IF-EVL-NEXT:    [[ANYOF_LCSSA:%.*]] = phi i32 [ [[ANYOF]], [[FOR_BODY]] ], [ [[RDX_SELECT]], [[MIDDLE_BLOCK]] ]
 ; IF-EVL-NEXT:    ret i32 [[ANYOF_LCSSA]]
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-uniform-store.ll b/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-uniform-store.ll
index a2f85b9ed4ffe..0829bab26f062 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-uniform-store.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-uniform-store.ll
@@ -8,7 +8,7 @@ target triple = "riscv64-unknown-linux-gnu"
 
 define void @lshift_significand(i32 %n, ptr nocapture writeonly %dst) {
 ; CHECK-LABEL: define void @lshift_significand(
-; CHECK-SAME: i32 [[N:%.*]], ptr nocapture writeonly [[DST:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-SAME: i32 [[N:%.*]], ptr writeonly captures(none) [[DST:%.*]]) #[[ATTR0:[0-9]+]] {
 ; CHECK-NEXT:  [[ENTRY:.*]]:
 ; CHECK-NEXT:    [[CMP1_PEEL:%.*]] = icmp eq i32 [[N]], 0
 ; CHECK-NEXT:    [[SPEC_SELECT:%.*]] = select i1 [[CMP1_PEEL]], i64 2, i64 0
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/vf-will-not-generate-any-vector-insts.ll b/llvm/test/Transforms/LoopVectorize/RISCV/vf-will-not-generate-any-vector-insts.ll
index 4efc231c92a4d..e7fdfbcf76caa 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/vf-will-not-generate-any-vector-insts.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/vf-will-not-generate-any-vector-insts.ll
@@ -33,7 +33,7 @@ define void @vf_will_not_generate_any_vector_insts(ptr %src, ptr %dst) {
 ; CHECK:       [[MIDDLE_BLOCK]]:
 ; CHECK-NEXT:    br i1 true, label %[[EXIT:.*]], label %[[SCALAR_PH]]
 ; CHECK:       [[SCALAR_PH]]:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 100, %[[MIDDLE_BLOCK]] ], [ 0, %[[VECTOR_MEMCHECK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 100, %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_MEMCHECK]] ]
 ; CHECK-NEXT:    br label %[[LOOP:.*]]
 ; CHECK:       [[LOOP]]:
 ; CHECK-NEXT:    [[TMP2:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[TMP3:%.*]], %[[LOOP]] ]
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/vplan-vp-call-intrinsics.ll b/llvm/test/Transforms/LoopVectorize/RISCV/vplan-vp-call-intrinsics.ll
index 05c687a205353..81a2675dc6583 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/vplan-vp-call-intrinsics.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/vplan-vp-call-intrinsics.ll
@@ -6,6 +6,9 @@
 ; RUN: -mtriple=riscv64 -mattr=+v -riscv-v-vector-bits-max=128 -disable-output < %s 2>&1 | FileCheck --check-prefix=IF-EVL %s
 
 define void @vp_smax(ptr %a, ptr %b, ptr %c, i64 %N) {
+; IF-EVL: VPlan 'Initial VPlan for VF={1},UF>=1'
+; IF-EVL-NOT: EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI
+;
 ; IF-EVL: VPlan 'Initial VPlan for VF={vscale x 1,vscale x 2,vscale x 4},UF={1}' {
 ; IF-EVL-NEXT: Live-in vp<[[VFUF:%[0-9]+]]> = VF * UF
 ; IF-EVL-NEXT: Live-in vp<[[VTC:%[0-9]+]]> = vector-trip-count
@@ -59,6 +62,9 @@ exit:
 }
 
 define void @vp_smin(ptr %a, ptr %b, ptr %c, i64 %N) {
+; IF-EVL: VPlan 'Initial VPlan for VF={1},UF>=1'
+; IF-EVL-NOT: EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI
+;
 ; IF-EVL: VPlan 'Initial VPlan for VF={vscale x 1,vscale x 2,vscale x 4},UF={1}' {
 ; IF-EVL-NEXT: Live-in vp<[[VFUF:%[0-9]+]]> = VF * UF
 ; IF-EVL-NEXT: Live-in vp<[[VTC:%[0-9]+]]> = vector-trip-count
@@ -112,6 +118,9 @@ exit:
 }
 
 define void @vp_umax(ptr %a, ptr %b, ptr %c, i64 %N) {
+; IF-EVL: VPlan 'Initial VPlan for VF={1},UF>=1'
+; IF-EVL-NOT: EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI
+;
 ; IF-EVL: VPlan 'Initial VPlan for VF={vscale x 1,vscale x 2,vscale x 4},UF={1}' {
 ; IF-EVL-NEXT: Live-in vp<[[VFUF:%[0-9]+]]> = VF * UF
 ; IF-EVL-NEXT: Live-in vp<[[VTC:%[0-9]+]]> = vector-trip-count
@@ -165,6 +174,9 @@ exit:
 }
 
 define void @vp_umin(ptr %a, ptr %b, ptr %c, i64 %N) {
+; IF-EVL: VPlan 'Initial VPlan for VF={1},UF>=1'
+; IF-EVL-NOT: EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI
+;
 ; IF-EVL: VPlan 'Initial VPlan for VF={vscale x 1,vscale x 2,vscale x 4},UF={1}' {
 ; IF-EVL-NEXT: Live-in vp<[[VFUF:%[0-9]+]]> = VF * UF
 ; IF-EVL-NEXT: Live-in vp<[[VTC:%[0-9]+]]> = vector-trip-count
@@ -218,6 +230,9 @@ exit:
 }
 
 define void @vp_ctlz(ptr %a, ptr %b, i64 %N) {
+; IF-EVL: VPlan 'Initial VPlan for VF={1},UF>=1'
+; IF-EVL-NOT: EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI
+;
 ; IF-EVL: VPlan 'Initial VPlan for VF={vscale x 1,vscale x 2,vscale x 4},UF={1}' {
 ; IF-EVL-NEXT: Live-in vp<[[VFUF:%[0-9]+]]> = VF * UF
 ; IF-EVL-NEXT: Live-in vp<[[VTC:%[0-9]+]]> = vector-trip-count
@@ -266,6 +281,9 @@ exit:
 }
 
 define void @vp_cttz(ptr %a, ptr %b, i64 %N) {
+; IF-EVL: VPlan 'Initial VPlan for VF={1},UF>=1'
+; IF-EVL-NOT: EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI
+;
 ; IF-EVL: VPlan 'Initial VPlan for VF={vscale x 1,vscale x 2,vscale x 4},UF={1}' {
 ; IF-EVL-NEXT: Live-in vp<[[VFUF:%[0-9]+]]> = VF * UF
 ; IF-EVL-NEXT: Live-in vp<[[VTC:%[0-9]+]]> = vector-trip-count
@@ -314,6 +332,9 @@ exit:
 }
 
 define void @vp_lrint(ptr %a, ptr %b, i64 %N) {
+; IF-EVL: VPlan 'Initial VPlan for VF={1},UF>=1'
+; IF-EVL-NOT: EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI
+;
 ; IF-EVL: VPlan 'Initial VPlan for VF={vscale x 1,vscale x 2,vscale x 4},UF={1}' {
 ; IF-EVL-NEXT: Live-in vp<[[VFUF:%[0-9]+]]> = VF * UF
 ; IF-EVL-NEXT: Live-in vp<[[VTC:%[0-9]+]]> = vector-trip-count
@@ -366,6 +387,9 @@ exit:
 }
 
 define void @vp_llrint(ptr %a, ptr %b, i64 %N) {
+; IF-EVL: VPlan 'Initial VPlan for VF={1},UF>=1'
+; IF-EVL-NOT: EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI
+;
 ; IF-EVL: VPlan 'Initial VPlan for VF={vscale x 1,vscale x 2,vscale x 4},UF={1}' {
 ; IF-EVL-NEXT: Live-in vp<[[VFUF:%[0-9]+]]> = VF * UF
 ; IF-EVL-NEXT: Live-in vp<[[VTC:%[0-9]+]]> = vector-trip-count
@@ -418,6 +442,9 @@ exit:
 }
 
 define void @vp_abs(ptr %a, ptr %b, i64 %N) {
+; IF-EVL: VPlan 'Initial VPlan for VF={1},UF>=1'
+; IF-EVL-NOT: EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI
+;
 ; IF-EVL: VPlan 'Initial VPlan for VF={vscale x 1,vscale x 2,vscale x 4},UF={1}' {
 ; IF-EVL-NEXT: Live-in vp<[[VFUF:%[0-9]+]]> = VF * UF
 ; IF-EVL-NEXT: Live-in vp<[[VTC:%[0-9]+]]> = vector-trip-count
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/vplan-vp-cast-intrinsics.ll b/llvm/test/Transforms/LoopVectorize/RISCV/vplan-vp-cast-intrinsics.ll
index f4acfd6185644..40cf02c4663fe 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/vplan-vp-cast-intrinsics.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/vplan-vp-cast-intrinsics.ll
@@ -5,6 +5,9 @@
 ; RUN: -mtriple=riscv64 -mattr=+v -riscv-v-vector-bits-max=128 -disable-output < %s 2>&1 | FileCheck --check-prefix=IF-EVL %s
 
 define void @vp_sext(ptr %a, ptr %b, i64 %N) {
+; IF-EVL: VPlan 'Initial VPlan for VF={1},UF>=1'
+; IF-EVL-NOT: EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI
+;
 ; IF-EVL: VPlan 'Initial VPlan for VF={vscale x 1,vscale x 2},UF={1}' {
 ; IF-EVL-NEXT: Live-in vp<[[VFUF:%[0-9]+]]> = VF * UF
 ; IF-EVL-NEXT: Live-in vp<[[VTC:%[0-9]+]]> = vector-trip-count
@@ -55,6 +58,9 @@ exit:
 }
 
 define void @vp_zext(ptr %a, ptr %b, i64 %N) {
+; IF-EVL: VPlan 'Initial VPlan for VF={1},UF>=1'
+; IF-EVL-NOT: EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI
+;
 ; IF-EVL: VPlan 'Initial VPlan for VF={vscale x 1,vscale x 2},UF={1}' {
 ; IF-EVL-NEXT: Live-in vp<[[VFUF:%[0-9]+]]> = VF * UF
 ; IF-EVL-NEXT: Live-in vp<[[VTC:%[0-9]+]]> = vector-trip-count
@@ -103,6 +109,9 @@ exit:
 }
 
 define void @vp_trunc(ptr %a, ptr %b, i64 %N) {
+; IF-EVL: VPlan 'Initial VPlan for VF={1},UF>=1'
+; IF-EVL-NOT: EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI
+;
 ; IF-EVL: VPlan 'Initial VPlan for VF={vscale x 1,vscale x 2,vscale x 4},UF={1}' {
 ; IF-EVL-NEXT: Live-in vp<[[VFUF:%[0-9]+]]> = VF * UF
 ; IF-EVL-NEXT: Live-in vp<[[VTC:%[0-9]+]]> = vector-trip-count
@@ -151,6 +160,9 @@ exit:
 }
 
 define void @vp_fpext(ptr %a, ptr %b, i64 %N) {
+; IF-EVL: VPlan 'Initial VPlan for VF={1},UF>=1'
+; IF-EVL-NOT: EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI
+;
 ; IF-EVL: VPlan 'Initial VPlan for VF={vscale x 1,vscale x 2},UF={1}' {
 ; IF-EVL-NEXT: Live-in vp<[[VFUF:%[0-9]+]]> = VF * UF
 ; IF-EVL-NEXT: Live-in vp<[[VTC:%[0-9]+]]> = vector-trip-count
@@ -199,6 +211,9 @@ exit:
 }
 
 define void @vp_fptrunc(ptr %a, ptr %b, i64 %N) {
+; IF-EVL: VPlan 'Initial VPlan for VF={1},UF>=1'
+; IF-EVL-NOT: EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI
+;
 ; IF-EVL: VPlan 'Initial VPlan for VF={vscale x 1,vscale x 2},UF={1}' {
 ; IF-EVL-NEXT: Live-in vp<[[VFUF:%[0-9]+]]> = VF * UF
 ; IF-EVL-NEXT: Live-in vp<[[VTC:%[0-9]+]]> = vector-trip-count
@@ -247,6 +262,9 @@ exit:
 }
 
 define void @vp_sitofp(ptr %a, ptr %b, i64 %N) {
+; IF-EVL: VPlan 'Initial VPlan for VF={1},UF>=1'
+; IF-EVL-NOT: EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI
+;
 ; IF-EVL: VPlan 'Initial VPlan for VF={vscale x 1,vscale x 2,vscale x 4},UF={1}' {
 ; IF-EVL-NEXT: Live-in vp<[[VFUF:%[0-9]+]]> = VF * UF
 ; IF-EVL-NEXT: Live-in vp<[[VTC:%[0-9]+]]> = vector-trip-count
@@ -295,6 +313,9 @@ exit:
 }
 
 define void @vp_uitofp(ptr %a, ptr %b, i64 %N) {
+; IF-EVL: VPlan 'Initial VPlan for VF={1},UF>=1'
+; IF-EVL-NOT: EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI
+;
 ; IF-EVL: VPlan 'Initial VPlan for VF={vscale x 1,vscale x 2,vscale x 4},UF={1}' {
 ; IF-EVL-NEXT: Live-in vp<[[VFUF:%[0-9]+]]> = VF * UF
 ; IF-EVL-NEXT: Live-in vp<[[VTC:%[0-9]+]]> = vector-trip-count
@@ -343,6 +364,9 @@ exit:
 }
 
 define void @vp_fptosi(ptr %a, ptr %b, i64 %N) {
+; IF-EVL: VPlan 'Initial VPlan for VF={1},UF>=1'
+; IF-EVL-NOT: EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI
+;
 ; IF-EVL: VPlan 'Initial VPlan for VF={vscale x 1,vscale x 2,vscale x 4},UF={1}' {
 ; IF-EVL-NEXT: Live-in vp<[[VFUF:%[0-9]+]]> = VF * UF
 ; IF-EVL-NEXT: Live-in vp<[[VTC:%[0-9]+]]> = vector-trip-count
@@ -391,6 +415,9 @@ exit:
 }
 
 define void @vp_fptoui(ptr %a, ptr %b, i64 %N) {
+; IF-EVL: VPlan 'Initial VPlan for VF={1},UF>=1'
+; IF-EVL-NOT: EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI
+;
 ; IF-EVL: VPlan 'Initial VPlan for VF={vscale x 1,vscale x 2,vscale x 4},UF={1}' {
 ; IF-EVL-NEXT: Live-in vp<[[VFUF:%[0-9]+]]> = VF * UF
 ; IF-EVL-NEXT: Live-in vp<[[VTC:%[0-9]+]]> = vector-trip-count
@@ -439,6 +466,9 @@ exit:
 }
 
 define void @vp_inttoptr(ptr %a, ptr %b, i64 %N) {
+; IF-EVL: VPlan 'Initial VPlan for VF={1},UF>=1'
+; IF-EVL-NOT: EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI
+;
 ; IF-EVL: VPlan 'Initial VPlan for VF={vscale x 1,vscale x 2},UF={1}' {
 ; IF-EVL-NEXT: Live-in vp<[[VFUF:%[0-9]+]]> = VF * UF
 ; IF-EVL-NEXT: Live-in vp<[[VTC:%[0-9]+]]> = vector-trip-count
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/vplan-vp-intrinsics-reduction.ll b/llvm/test/Transforms/LoopVectorize/RISCV/vplan-vp-intrinsics-reduction.ll
index cd1d734f00eaa..0daf15d04af35 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/vplan-vp-intrinsics-reduction.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/vplan-vp-intrinsics-reduction.ll
@@ -3,13 +3,13 @@
 ; RUN: opt -passes=loop-vectorize -debug-only=loop-vectorize \
 ; RUN: -force-tail-folding-style=data-with-evl \
 ; RUN: -prefer-predicate-over-epilogue=predicate-dont-vectorize \
-; RUN: -mtriple=riscv64 -mattr=+v -riscv-v-vector-bits-max=128 -disable-output < %s 2>&1 | FileCheck --check-prefixes=IF-EVL-OUTLOOP %s
+; RUN: -mtriple=riscv64 -mattr=+v -riscv-v-vector-bits-max=128 -disable-output < %s 2>&1 | FileCheck --check-prefixes=IF-EVL-OUTLOOP,IF-EVL %s
 
 ; RUN: opt -passes=loop-vectorize -debug-only=loop-vectorize \
 ; RUN: -prefer-inloop-reductions \
 ; RUN: -force-tail-folding-style=data-with-evl \
 ; RUN: -prefer-predicate-over-epilogue=predicate-dont-vectorize \
-; RUN: -mtriple=riscv64 -mattr=+v -riscv-v-vector-bits-max=128 -disable-output < %s 2>&1 | FileCheck --check-prefixes=IF-EVL-INLOOP %s
+; RUN: -mtriple=riscv64 -mattr=+v -riscv-v-vector-bits-max=128 -disable-output < %s 2>&1 | FileCheck --check-prefixes=IF-EVL-INLOOP,IF-EVL %s
 
 ; RUN: opt -passes=loop-vectorize -debug-only=loop-vectorize \
 ; RUN: -force-tail-folding-style=none \
@@ -24,6 +24,9 @@
 
 
 define i32 @reduction(ptr %a, i64 %n, i32 %start) {
+; IF-EVL: VPlan 'Initial VPlan for VF={1},UF>=1'
+; IF-EVL-NOT: EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI
+
 ; IF-EVL-OUTLOOP: VPlan 'Initial VPlan for VF={vscale x 1,vscale x 2,vscale x 4},UF={1}' {
 ; IF-EVL-OUTLOOP-NEXT: Live-in vp<[[VFUF:%[0-9]+]]> = VF * UF
 ; IF-EVL-OUTLOOP-NEXT: Live-in vp<[[VTC:%[0-9]+]]> = vector-trip-count
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/vplan-vp-intrinsics.ll b/llvm/test/Transforms/LoopVectorize/RISCV/vplan-vp-intrinsics.ll
index 706b6f8882984..e64ea387d4582 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/vplan-vp-intrinsics.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/vplan-vp-intrinsics.ll
@@ -11,6 +11,9 @@
 ; RUN: -mtriple=riscv64 -mattr=+v -riscv-v-vector-bits-max=128 -disable-output < %s 2>&1 | FileCheck --check-prefixes=NO-VP,CHECK %s
 
 define void @foo(ptr noalias %a, ptr noalias %b, ptr noalias %c, i64 %N) {
+; IF-EVL: VPlan 'Initial VPlan for VF={1},UF>=1'
+; IF-EVL-NOT: EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI
+;
 ; IF-EVL: VPlan 'Initial VPlan for VF={vscale x 1,vscale x 2,vscale x 4},UF={1}' {
 ; IF-EVL-NEXT: Live-in vp<[[VFUF:%[0-9]+]]> = VF * UF
 ; IF-EVL-NEXT: Live-in vp<[[VTC:%[0-9]+]]> = vector-trip-count
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/vplan-vp-select-intrinsics.ll b/llvm/test/Transforms/LoopVectorize/RISCV/vplan-vp-select-intrinsics.ll
index 53c9fb0c604da..90de5b6bfd1d3 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/vplan-vp-select-intrinsics.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/vplan-vp-select-intrinsics.ll
@@ -6,6 +6,11 @@
  ; RUN: -mtriple=riscv64 -mattr=+v -riscv-v-vector-bits-max=128 -disable-output < %s 2>&1 | FileCheck --check-prefix=IF-EVL %s
 
  define void @vp_select(ptr noalias %a, ptr noalias %b, ptr noalias %c, i64 %N) {
+ ; IF-EVL: VPlan 'Initial VPlan for VF={1},UF>=1'
+ ; IF-EVL-NOT: EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI
+ ; IF-EVL: VPlan 'Initial VPlan for VF={vscale x 1,vscale x 2,vscale x 4},UF={1}'
+ ; IF-EVL: EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI
+ ;
  ; IF-EVL: VPlan 'Final VPlan for VF={vscale x 1,vscale x 2,vscale x 4},UF={1}' {
  ; IF-EVL-NEXT: Live-in ir<[[VFUF:%.+]]> = VF * UF
  ; IF-EVL-NEXT: Live-in ir<[[VTC:%.+]]> = vector-trip-count
diff --git a/llvm/test/Transforms/LoopVectorize/SystemZ/zero_unroll.ll b/llvm/test/Transforms/LoopVectorize/SystemZ/zero_unroll.ll
index bd0423b6f8414..eba5a89939f78 100644
--- a/llvm/test/Transforms/LoopVectorize/SystemZ/zero_unroll.ll
+++ b/llvm/test/Transforms/LoopVectorize/SystemZ/zero_unroll.ll
@@ -3,7 +3,7 @@
 
 define i32 @main(i32 %arg, ptr nocapture readnone %arg1) #0 {
 ; CHECK-LABEL: define i32 @main(
-; CHECK-SAME: i32 [[ARG:%.*]], ptr nocapture readnone [[ARG1:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-SAME: i32 [[ARG:%.*]], ptr readnone captures(none) [[ARG1:%.*]]) #[[ATTR0:[0-9]+]] {
 ; CHECK-NEXT:  [[ENTRY:.*]]:
 ; CHECK-NEXT:    [[TMP0:%.*]] = alloca i8, align 1
 ; CHECK-NEXT:    br label %[[LOOP:.*]]
diff --git a/llvm/test/Transforms/LoopVectorize/WebAssembly/int-mac-reduction-costs.ll b/llvm/test/Transforms/LoopVectorize/WebAssembly/int-mac-reduction-costs.ll
new file mode 100644
index 0000000000000..d23c2272d9c0d
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/WebAssembly/int-mac-reduction-costs.ll
@@ -0,0 +1,326 @@
+; REQUIRES: asserts
+; RUN: opt -mattr=+simd128 -passes=loop-vectorize -debug-only=loop-vectorize -disable-output -S < %s 2>&1 | FileCheck %s
+
+target triple = "wasm32"
+
+define hidden i32 @i32_mac_s8(ptr nocapture noundef readonly %a, ptr nocapture noundef readonly %b, i32 noundef %N) {
+; CHECK-LABEL: 'i32_mac_s8'
+; CHECK: LV: Found an estimated cost of 1 for VF 1 For instruction:   %0 = load i8, ptr %arrayidx, align 1
+; CHECK: LV: Found an estimated cost of 0 for VF 1 For instruction:   %conv = sext i8 %0 to i32
+; CHECK: LV: Found an estimated cost of 1 for VF 1 For instruction:   %1 = load i8, ptr %arrayidx1, align 1
+; CHECK: LV: Found an estimated cost of 0 for VF 1 For instruction:   %conv2 = sext i8 %1 to i32
+; CHECK: LV: Found an estimated cost of 1 for VF 1 For instruction:   %mul = mul nsw i32 %conv2, %conv
+
+; CHECK: LV: Found an estimated cost of 3 for VF 2 For instruction:   %0 = load i8, ptr %arrayidx, align 1
+; CHECK: LV: Found an estimated cost of 0 for VF 2 For instruction:   %conv = sext i8 %0 to i32
+; CHECK: LV: Found an estimated cost of 3 for VF 2 For instruction:   %1 = load i8, ptr %arrayidx1, align 1
+; CHECK: LV: Found an estimated cost of 0 for VF 2 For instruction:   %conv2 = sext i8 %1 to i32
+; CHECK: LV: Found an estimated cost of 1 for VF 2 For instruction:   %mul = mul nsw i32 %conv2, %conv
+
+; CHECK: LV: Found an estimated cost of 2 for VF 4 For instruction:   %0 = load i8, ptr %arrayidx, align 1
+; CHECK: LV: Found an estimated cost of 1 for VF 4 For instruction:   %conv = sext i8 %0 to i32
+; CHECK: LV: Found an estimated cost of 2 for VF 4 For instruction:   %1 = load i8, ptr %arrayidx1, align 1
+; CHECK: LV: Found an estimated cost of 1 for VF 4 For instruction:   %conv2 = sext i8 %1 to i32
+; CHECK: LV: Found an estimated cost of 1 for VF 4 For instruction:   %mul = mul nsw i32 %conv2, %conv
+; CHECK: LV: Selecting VF: 4.
+entry:
+  %cmp7.not = icmp eq i32 %N, 0
+  br i1 %cmp7.not, label %for.cond.cleanup, label %for.body
+
+for.cond.cleanup:                                 ; preds = %for.body, %entry
+  %res.0.lcssa = phi i32 [ 0, %entry ], [ %add, %for.body ]
+  ret i32 %res.0.lcssa
+
+for.body:                                         ; preds = %entry, %for.body
+  %i.09 = phi i32 [ %inc, %for.body ], [ 0, %entry ]
+  %res.08 = phi i32 [ %add, %for.body ], [ 0, %entry ]
+  %arrayidx = getelementptr inbounds nuw i8, ptr %a, i32 %i.09
+  %0 = load i8, ptr %arrayidx, align 1
+  %conv = sext i8 %0 to i32
+  %arrayidx1 = getelementptr inbounds nuw i8, ptr %b, i32 %i.09
+  %1 = load i8, ptr %arrayidx1, align 1
+  %conv2 = sext i8 %1 to i32
+  %mul = mul nsw i32 %conv2, %conv
+  %add = add nsw i32 %mul, %res.08
+  %inc = add nuw i32 %i.09, 1
+  %exitcond.not = icmp eq i32 %inc, %N
+  br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
+
+define hidden i32 @i32_mac_s16(ptr nocapture noundef readonly %a, ptr nocapture noundef readonly %b, i32 noundef %N) {
+; CHECK-LABEL: 'i32_mac_s16'
+; CHECK: LV: Found an estimated cost of 1 for VF 1 For instruction:   %0 = load i16, ptr %arrayidx, align 2
+; CHECK: LV: Found an estimated cost of 0 for VF 1 For instruction:   %conv = sext i16 %0 to i32
+; CHECK: LV: Found an estimated cost of 1 for VF 1 For instruction:   %1 = load i16, ptr %arrayidx1, align 2
+; CHECK: LV: Found an estimated cost of 0 for VF 1 For instruction:   %conv2 = sext i16 %1 to i32
+; CHECK: LV: Found an estimated cost of 1 for VF 1 For instruction:   %mul = mul nsw i32 %conv2, %conv
+
+; CHECK: LV: Found an estimated cost of 2 for VF 2 For instruction:   %0 = load i16, ptr %arrayidx, align 2
+; CHECK: LV: Found an estimated cost of 0 for VF 2 For instruction:   %conv = sext i16 %0 to i32
+; CHECK: LV: Found an estimated cost of 2 for VF 2 For instruction:   %1 = load i16, ptr %arrayidx1, align 2
+; CHECK: LV: Found an estimated cost of 0 for VF 2 For instruction:   %conv2 = sext i16 %1 to i32
+; CHECK: LV: Found an estimated cost of 1 for VF 2 For instruction:   %mul = mul nsw i32 %conv2, %conv
+
+; CHECK: LV: Found an estimated cost of 2 for VF 4 For instruction:   %0 = load i16, ptr %arrayidx, align 2
+; CHECK: LV: Found an estimated cost of 0 for VF 4 For instruction:   %conv = sext i16 %0 to i32
+; CHECK: LV: Found an estimated cost of 2 for VF 4 For instruction:   %1 = load i16, ptr %arrayidx1, align 2
+; CHECK: LV: Found an estimated cost of 0 for VF 4 For instruction:   %conv2 = sext i16 %1 to i32
+; CHECK: LV: Found an estimated cost of 1 for VF 4 For instruction:   %mul = mul nsw i32 %conv2, %conv
+; CHECK: LV: Selecting VF: 4.
+entry:
+  %cmp7.not = icmp eq i32 %N, 0
+  br i1 %cmp7.not, label %for.cond.cleanup, label %for.body
+
+for.cond.cleanup:                                 ; preds = %for.body, %entry
+  %res.0.lcssa = phi i32 [ 0, %entry ], [ %add, %for.body ]
+  ret i32 %res.0.lcssa
+
+for.body:                                         ; preds = %entry, %for.body
+  %i.09 = phi i32 [ %inc, %for.body ], [ 0, %entry ]
+  %res.08 = phi i32 [ %add, %for.body ], [ 0, %entry ]
+  %arrayidx = getelementptr inbounds nuw i16, ptr %a, i32 %i.09
+  %0 = load i16, ptr %arrayidx, align 2
+  %conv = sext i16 %0 to i32
+  %arrayidx1 = getelementptr inbounds nuw i16, ptr %b, i32 %i.09
+  %1 = load i16, ptr %arrayidx1, align 2
+  %conv2 = sext i16 %1 to i32
+  %mul = mul nsw i32 %conv2, %conv
+  %add = add nsw i32 %mul, %res.08
+  %inc = add nuw i32 %i.09, 1
+  %exitcond.not = icmp eq i32 %inc, %N
+  br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
+
+define hidden i64 @i64_mac_s16(ptr nocapture noundef readonly %a, ptr nocapture noundef readonly %b, i32 noundef %N) {
+; CHECK-LABEL: 'i64_mac_s16'
+; CHECK: LV: Found an estimated cost of 1 for VF 1 For instruction:   %0 = load i16, ptr %arrayidx, align 2
+; CHECK: LV: Found an estimated cost of 0 for VF 1 For instruction:   %conv = sext i16 %0 to i64
+; CHECK: LV: Found an estimated cost of 1 for VF 1 For instruction:   %1 = load i16, ptr %arrayidx1, align 2
+; CHECK: LV: Found an estimated cost of 0 for VF 1 For instruction:   %conv2 = sext i16 %1 to i64
+; CHECK: LV: Found an estimated cost of 1 for VF 1 For instruction:   %mul = mul nsw i64 %conv2, %conv
+
+; CHECK: LV: Found an estimated cost of 2 for VF 2 For instruction:   %0 = load i16, ptr %arrayidx, align 2
+; CHECK: LV: Found an estimated cost of 1 for VF 2 For instruction:   %conv = sext i16 %0 to i64
+; CHECK: LV: Found an estimated cost of 2 for VF 2 For instruction:   %1 = load i16, ptr %arrayidx1, align 2
+; CHECK: LV: Found an estimated cost of 1 for VF 2 For instruction:   %conv2 = sext i16 %1 to i64
+; CHECK: LV: Found an estimated cost of 1 for VF 2 For instruction:   %mul = mul nsw i64 %conv2, %conv
+; CHECK: LV: Selecting VF: 2.
+entry:
+  %cmp7.not = icmp eq i32 %N, 0
+  br i1 %cmp7.not, label %for.cond.cleanup, label %for.body
+
+for.cond.cleanup:                                 ; preds = %for.body, %entry
+  %res.0.lcssa = phi i64 [ 0, %entry ], [ %add, %for.body ]
+  ret i64 %res.0.lcssa
+
+for.body:                                         ; preds = %entry, %for.body
+  %i.09 = phi i32 [ %inc, %for.body ], [ 0, %entry ]
+  %res.08 = phi i64 [ %add, %for.body ], [ 0, %entry ]
+  %arrayidx = getelementptr inbounds nuw i16, ptr %a, i32 %i.09
+  %0 = load i16, ptr %arrayidx, align 2
+  %conv = sext i16 %0 to i64
+  %arrayidx1 = getelementptr inbounds nuw i16, ptr %b, i32 %i.09
+  %1 = load i16, ptr %arrayidx1, align 2
+  %conv2 = sext i16 %1 to i64
+  %mul = mul nsw i64 %conv2, %conv
+  %add = add nsw i64 %mul, %res.08
+  %inc = add nuw i32 %i.09, 1
+  %exitcond.not = icmp eq i32 %inc, %N
+  br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
+
+define hidden i64 @i64_mac_s32(ptr nocapture noundef readonly %a, ptr nocapture noundef readonly %b, i32 noundef %N) {
+; CHECK-LABEL: 'i64_mac_s32'
+; CHECK: LV: Found an estimated cost of 1 for VF 1 For instruction:   %0 = load i32, ptr %arrayidx, align 4
+; CHECK: LV: Found an estimated cost of 1 for VF 1 For instruction:   %1 = load i32, ptr %arrayidx1, align 4
+; CHECK: LV: Found an estimated cost of 1 for VF 1 For instruction:   %mul = mul i32 %1, %0
+; CHECK: LV: Found an estimated cost of 1 for VF 1 For instruction:   %conv = sext i32 %mul to i64
+
+; CHECK: LV: Found an estimated cost of 2 for VF 2 For instruction:   %0 = load i32, ptr %arrayidx, align 4
+; CHECK: LV: Found an estimated cost of 2 for VF 2 For instruction:   %1 = load i32, ptr %arrayidx1, align 4
+; CHECK: LV: Found an estimated cost of 1 for VF 2 For instruction:   %mul = mul i32 %1, %0
+; CHECK: LV: Found an estimated cost of 1 for VF 2 For instruction:   %conv = sext i32 %mul to i64
+; CHECK: LV: Selecting VF: 2.
+entry:
+  %cmp6.not = icmp eq i32 %N, 0
+  br i1 %cmp6.not, label %for.cond.cleanup, label %for.body
+
+for.cond.cleanup:                                 ; preds = %for.body, %entry
+  %res.0.lcssa = phi i64 [ 0, %entry ], [ %add, %for.body ]
+  ret i64 %res.0.lcssa
+
+for.body:                                         ; preds = %entry, %for.body
+  %i.08 = phi i32 [ %inc, %for.body ], [ 0, %entry ]
+  %res.07 = phi i64 [ %add, %for.body ], [ 0, %entry ]
+  %arrayidx = getelementptr inbounds nuw i32, ptr %a, i32 %i.08
+  %0 = load i32, ptr %arrayidx, align 4
+  %arrayidx1 = getelementptr inbounds nuw i32, ptr %b, i32 %i.08
+  %1 = load i32, ptr %arrayidx1, align 4
+  %mul = mul i32 %1, %0
+  %conv = sext i32 %mul to i64
+  %add = add i64 %res.07, %conv
+  %inc = add nuw i32 %i.08, 1
+  %exitcond.not = icmp eq i32 %inc, %N
+  br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
+
+define hidden i32 @i32_mac_u8(ptr nocapture noundef readonly %a, ptr nocapture noundef readonly %b, i32 noundef %N) {
+; CHECK-LABEL: 'i32_mac_u8'
+; CHECK: LV: Found an estimated cost of 1 for VF 1 For instruction:   %0 = load i8, ptr %arrayidx, align 1
+; CHECK: LV: Found an estimated cost of 0 for VF 1 For instruction:   %conv = zext i8 %0 to i32
+; CHECK: LV: Found an estimated cost of 1 for VF 1 For instruction:   %1 = load i8, ptr %arrayidx1, align 1
+; CHECK: LV: Found an estimated cost of 0 for VF 1 For instruction:   %conv2 = zext i8 %1 to i32
+; CHECK: LV: Found an estimated cost of 1 for VF 1 For instruction:   %mul = mul nuw nsw i32 %conv2, %conv
+
+; CHECK: LV: Found an estimated cost of 3 for VF 2 For instruction:   %0 = load i8, ptr %arrayidx, align 1
+; CHECK: LV: Found an estimated cost of 0 for VF 2 For instruction:   %conv = zext i8 %0 to i32
+; CHECK: LV: Found an estimated cost of 3 for VF 2 For instruction:   %1 = load i8, ptr %arrayidx1, align 1
+; CHECK: LV: Found an estimated cost of 0 for VF 2 For instruction:   %conv2 = zext i8 %1 to i32
+; CHECK: LV: Found an estimated cost of 1 for VF 2 For instruction:   %mul = mul nuw nsw i32 %conv2, %conv
+
+; CHECK: LV: Found an estimated cost of 2 for VF 4 For instruction:   %0 = load i8, ptr %arrayidx, align 1
+; CHECK: LV: Found an estimated cost of 1 for VF 4 For instruction:   %conv = zext i8 %0 to i32
+; CHECK: LV: Found an estimated cost of 2 for VF 4 For instruction:   %1 = load i8, ptr %arrayidx1, align 1
+; CHECK: LV: Found an estimated cost of 1 for VF 4 For instruction:   %conv2 = zext i8 %1 to i32
+; CHECK: LV: Found an estimated cost of 1 for VF 4 For instruction:   %mul = mul nuw nsw i32 %conv2, %conv
+; CHECK: LV: Selecting VF: 4.
+entry:
+  %cmp7.not = icmp eq i32 %N, 0
+  br i1 %cmp7.not, label %for.cond.cleanup, label %for.body
+
+for.cond.cleanup:                                 ; preds = %for.body, %entry
+  %res.0.lcssa = phi i32 [ 0, %entry ], [ %add, %for.body ]
+  ret i32 %res.0.lcssa
+
+for.body:                                         ; preds = %entry, %for.body
+  %i.09 = phi i32 [ %inc, %for.body ], [ 0, %entry ]
+  %res.08 = phi i32 [ %add, %for.body ], [ 0, %entry ]
+  %arrayidx = getelementptr inbounds nuw i8, ptr %a, i32 %i.09
+  %0 = load i8, ptr %arrayidx, align 1
+  %conv = zext i8 %0 to i32
+  %arrayidx1 = getelementptr inbounds nuw i8, ptr %b, i32 %i.09
+  %1 = load i8, ptr %arrayidx1, align 1
+  %conv2 = zext i8 %1 to i32
+  %mul = mul nuw nsw i32 %conv2, %conv
+  %add = add i32 %mul, %res.08
+  %inc = add nuw i32 %i.09, 1
+  %exitcond.not = icmp eq i32 %inc, %N
+  br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
+
+define hidden i32 @i32_mac_u16(ptr nocapture noundef readonly %a, ptr nocapture noundef readonly %b, i32 noundef %N) {
+; CHECK-LABEL: 'i32_mac_u16'
+; CHECK: LV: Found an estimated cost of 1 for VF 1 For instruction:   %0 = load i16, ptr %arrayidx, align 2
+; CHECK: LV: Found an estimated cost of 0 for VF 1 For instruction:   %conv = zext i16 %0 to i32
+; CHECK: LV: Found an estimated cost of 1 for VF 1 For instruction:   %1 = load i16, ptr %arrayidx1, align 2
+; CHECK: LV: Found an estimated cost of 0 for VF 1 For instruction:   %conv2 = zext i16 %1 to i32
+; CHECK: LV: Found an estimated cost of 1 for VF 1 For instruction:   %mul = mul nuw nsw i32 %conv2, %conv
+
+; CHECK: LV: Found an estimated cost of 2 for VF 2 For instruction:   %0 = load i16, ptr %arrayidx, align 2
+; CHECK: LV: Found an estimated cost of 0 for VF 2 For instruction:   %conv = zext i16 %0 to i32
+; CHECK: LV: Found an estimated cost of 2 for VF 2 For instruction:   %1 = load i16, ptr %arrayidx1, align 2
+; CHECK: LV: Found an estimated cost of 0 for VF 2 For instruction:   %conv2 = zext i16 %1 to i32
+; CHECK: LV: Found an estimated cost of 1 for VF 2 For instruction:   %mul = mul nuw nsw i32 %conv2, %conv
+
+; CHECK: LV: Found an estimated cost of 2 for VF 4 For instruction:   %0 = load i16, ptr %arrayidx, align 2
+; CHECK: LV: Found an estimated cost of 0 for VF 4 For instruction:   %conv = zext i16 %0 to i32
+; CHECK: LV: Found an estimated cost of 2 for VF 4 For instruction:   %1 = load i16, ptr %arrayidx1, align 2
+; CHECK: LV: Found an estimated cost of 0 for VF 4 For instruction:   %conv2 = zext i16 %1 to i32
+; CHECK: LV: Found an estimated cost of 1 for VF 4 For instruction:   %mul = mul nuw nsw i32 %conv2, %conv
+; CHECK: LV: Selecting VF: 4.
+entry:
+  %cmp7.not = icmp eq i32 %N, 0
+  br i1 %cmp7.not, label %for.cond.cleanup, label %for.body
+
+for.cond.cleanup:                                 ; preds = %for.body, %entry
+  %res.0.lcssa = phi i32 [ 0, %entry ], [ %add, %for.body ]
+  ret i32 %res.0.lcssa
+
+for.body:                                         ; preds = %entry, %for.body
+  %i.09 = phi i32 [ %inc, %for.body ], [ 0, %entry ]
+  %res.08 = phi i32 [ %add, %for.body ], [ 0, %entry ]
+  %arrayidx = getelementptr inbounds nuw i16, ptr %a, i32 %i.09
+  %0 = load i16, ptr %arrayidx, align 2
+  %conv = zext i16 %0 to i32
+  %arrayidx1 = getelementptr inbounds nuw i16, ptr %b, i32 %i.09
+  %1 = load i16, ptr %arrayidx1, align 2
+  %conv2 = zext i16 %1 to i32
+  %mul = mul nuw nsw i32 %conv2, %conv
+  %add = add i32 %mul, %res.08
+  %inc = add nuw i32 %i.09, 1
+  %exitcond.not = icmp eq i32 %inc, %N
+  br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
+
+define hidden i64 @i64_mac_u16(ptr nocapture noundef readonly %a, ptr nocapture noundef readonly %b, i32 noundef %N) {
+; CHECK-LABEL: 'i64_mac_u16'
+; CHECK: LV: Found an estimated cost of 1 for VF 1 For instruction:   %0 = load i16, ptr %arrayidx, align 2
+; CHECK: LV: Found an estimated cost of 0 for VF 1 For instruction:   %conv = zext i16 %0 to i64
+; CHECK: LV: Found an estimated cost of 1 for VF 1 For instruction:   %1 = load i16, ptr %arrayidx1, align 2
+; CHECK: LV: Found an estimated cost of 0 for VF 1 For instruction:   %conv2 = zext i16 %1 to i64
+; CHECK: LV: Found an estimated cost of 1 for VF 1 For instruction:   %mul = mul nuw nsw i64 %conv2, %conv
+
+; CHECK: LV: Found an estimated cost of 2 for VF 2 For instruction:   %0 = load i16, ptr %arrayidx, align 2
+; CHECK: LV: Found an estimated cost of 1 for VF 2 For instruction:   %conv = zext i16 %0 to i64
+; CHECK: LV: Found an estimated cost of 2 for VF 2 For instruction:   %1 = load i16, ptr %arrayidx1, align 2
+; CHECK: LV: Found an estimated cost of 1 for VF 2 For instruction:   %conv2 = zext i16 %1 to i64
+; CHECK: LV: Found an estimated cost of 1 for VF 2 For instruction:   %mul = mul nuw nsw i64 %conv2, %conv
+; CHECK: LV: Selecting VF: 2.
+entry:
+  %cmp8.not = icmp eq i32 %N, 0
+  br i1 %cmp8.not, label %for.cond.cleanup, label %for.body
+
+for.cond.cleanup:                                 ; preds = %for.body, %entry
+  %res.0.lcssa = phi i64 [ 0, %entry ], [ %add, %for.body ]
+  ret i64 %res.0.lcssa
+
+for.body:                                         ; preds = %entry, %for.body
+  %i.010 = phi i32 [ %inc, %for.body ], [ 0, %entry ]
+  %res.09 = phi i64 [ %add, %for.body ], [ 0, %entry ]
+  %arrayidx = getelementptr inbounds nuw i16, ptr %a, i32 %i.010
+  %0 = load i16, ptr %arrayidx, align 2
+  %conv = zext i16 %0 to i64
+  %arrayidx1 = getelementptr inbounds nuw i16, ptr %b, i32 %i.010
+  %1 = load i16, ptr %arrayidx1, align 2
+  %conv2 = zext i16 %1 to i64
+  %mul = mul nuw nsw i64 %conv2, %conv
+  %add = add i64 %mul, %res.09
+  %inc = add nuw i32 %i.010, 1
+  %exitcond.not = icmp eq i32 %inc, %N
+  br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
+
+define hidden i64 @i64_mac_u32(ptr nocapture noundef readonly %a, ptr nocapture noundef readonly %b, i32 noundef %N) {
+; CHECK-LABEL: 'i64_mac_u32'
+; CHECK: LV: Found an estimated cost of 1 for VF 1 For instruction:   %0 = load i32, ptr %arrayidx, align 4
+; CHECK: LV: Found an estimated cost of 1 for VF 1 For instruction:   %1 = load i32, ptr %arrayidx1, align 4
+; CHECK: LV: Found an estimated cost of 1 for VF 1 For instruction:   %mul = mul i32 %1, %0
+; CHECK: LV: Found an estimated cost of 1 for VF 1 For instruction:   %conv = zext i32 %mul to i64
+
+; CHECK: LV: Found an estimated cost of 2 for VF 2 For instruction:   %0 = load i32, ptr %arrayidx, align 4
+; CHECK: LV: Found an estimated cost of 2 for VF 2 For instruction:   %1 = load i32, ptr %arrayidx1, align 4
+; CHECK: LV: Found an estimated cost of 1 for VF 2 For instruction:   %mul = mul i32 %1, %0
+; CHECK: LV: Found an estimated cost of 1 for VF 2 For instruction:   %conv = zext i32 %mul to i64
+; CHECK: LV: Selecting VF: 2.
+entry:
+  %cmp6.not = icmp eq i32 %N, 0
+  br i1 %cmp6.not, label %for.cond.cleanup, label %for.body
+
+for.cond.cleanup:                                 ; preds = %for.body, %entry
+  %res.0.lcssa = phi i64 [ 0, %entry ], [ %add, %for.body ]
+  ret i64 %res.0.lcssa
+
+for.body:                                         ; preds = %entry, %for.body
+  %i.08 = phi i32 [ %inc, %for.body ], [ 0, %entry ]
+  %res.07 = phi i64 [ %add, %for.body ], [ 0, %entry ]
+  %arrayidx = getelementptr inbounds nuw i32, ptr %a, i32 %i.08
+  %0 = load i32, ptr %arrayidx, align 4
+  %arrayidx1 = getelementptr inbounds nuw i32, ptr %b, i32 %i.08
+  %1 = load i32, ptr %arrayidx1, align 4
+  %mul = mul i32 %1, %0
+  %conv = zext i32 %mul to i64
+  %add = add i64 %res.07, %conv
+  %inc = add nuw i32 %i.08, 1
+  %exitcond.not = icmp eq i32 %inc, %N
+  br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
diff --git a/llvm/test/Transforms/LoopVectorize/X86/CostModel/gather-i16-with-i8-index.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/gather-i16-with-i8-index.ll
index 1c427a0107df1..93d25a7ae9469 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/CostModel/gather-i16-with-i8-index.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/CostModel/gather-i16-with-i8-index.ll
@@ -1,10 +1,10 @@
 ; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --filter "LV: Found an estimated cost of [0-9]+ for VF [0-9]+ For instruction:\s*%valB = load i16, ptr %inB, align 2"
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse2 --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefixes=SSE
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse4.2 --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefixes=SSE
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx  --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefixes=AVX1
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2,-fast-gather --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefixes=AVX2-SLOWGATHER
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2,+fast-gather --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefixes=AVX2-FASTGATHER
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512bw --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefixes=AVX512
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse2 --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefixes=SSE
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse4.2 --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefixes=SSE
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx  --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefixes=AVX1
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2,-fast-gather --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefixes=AVX2-SLOWGATHER
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2,+fast-gather --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefixes=AVX2-FASTGATHER
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512bw --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefixes=AVX512
 
 ; REQUIRES: asserts
 
diff --git a/llvm/test/Transforms/LoopVectorize/X86/CostModel/gather-i32-with-i8-index.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/gather-i32-with-i8-index.ll
index 8cde13761bc54..a23d57cfd4448 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/CostModel/gather-i32-with-i8-index.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/CostModel/gather-i32-with-i8-index.ll
@@ -1,10 +1,10 @@
 ; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --filter "LV: Found an estimated cost of [0-9]+ for VF [0-9]+ For instruction:\s*%valB = load i32, ptr %inB, align 4"
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse2 --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefixes=SSE2
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse4.2 --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefixes=SSE42
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx  --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefixes=AVX1
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2,-fast-gather --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefixes=AVX2-SLOWGATHER
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2,+fast-gather --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefixes=AVX2-FASTGATHER
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512bw --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefixes=AVX512
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse2 --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefixes=SSE2
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse4.2 --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefixes=SSE42
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx  --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefixes=AVX1
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2,-fast-gather --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefixes=AVX2-SLOWGATHER
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2,+fast-gather --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefixes=AVX2-FASTGATHER
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512bw --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefixes=AVX512
 
 ; REQUIRES: asserts
 
diff --git a/llvm/test/Transforms/LoopVectorize/X86/CostModel/gather-i64-with-i8-index.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/gather-i64-with-i8-index.ll
index 3435df1b494c6..5e52483aa93b3 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/CostModel/gather-i64-with-i8-index.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/CostModel/gather-i64-with-i8-index.ll
@@ -1,10 +1,10 @@
 ; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --filter "LV: Found an estimated cost of [0-9]+ for VF [0-9]+ For instruction:\s*%valB = load i64, ptr %inB, align 8"
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse2 --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefixes=SSE2
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse4.2 --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefixes=SSE42
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx  --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefixes=AVX1
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2,-fast-gather --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefixes=AVX2-SLOWGATHER
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2,+fast-gather --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefixes=AVX2-FASTGATHER
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512bw --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefixes=AVX512
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse2 --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefixes=SSE2
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse4.2 --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefixes=SSE42
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx  --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefixes=AVX1
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2,-fast-gather --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefixes=AVX2-SLOWGATHER
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2,+fast-gather --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefixes=AVX2-FASTGATHER
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512bw --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefixes=AVX512
 
 ; REQUIRES: asserts
 
diff --git a/llvm/test/Transforms/LoopVectorize/X86/CostModel/gather-i8-with-i8-index.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/gather-i8-with-i8-index.ll
index 7aeeff6518416..81d646715160a 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/CostModel/gather-i8-with-i8-index.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/CostModel/gather-i8-with-i8-index.ll
@@ -1,10 +1,10 @@
 ; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --filter "LV: Found an estimated cost of [0-9]+ for VF [0-9]+ For instruction:\s*%valB = load i8, ptr %inB, align 1"
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse2 --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefixes=SSE2
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse4.2 --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefixes=SSE42
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx  --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefixes=AVX1
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2,-fast-gather --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefixes=AVX2-SLOWGATHER
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2,+fast-gather --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefixes=AVX2-FASTGATHER
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512bw --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefixes=AVX512
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse2 --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefixes=SSE2
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse4.2 --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefixes=SSE42
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx  --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefixes=AVX1
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2,-fast-gather --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefixes=AVX2-SLOWGATHER
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2,+fast-gather --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefixes=AVX2-FASTGATHER
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512bw --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefixes=AVX512
 
 ; REQUIRES: asserts
 
diff --git a/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-f32-stride-2.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-f32-stride-2.ll
index 41ad1ebcf6ffb..b14e2a2a597da 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-f32-stride-2.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-f32-stride-2.ll
@@ -1,8 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --filter "LV: Found an estimated cost of [0-9]+ for VF [0-9]+ For instruction:\s*%v0 = load float, ptr %in0"
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse2 --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=SSE2
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx  --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX1
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2 --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX2
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512vl --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX512
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse2 --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=SSE2
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx  --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX1
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2 --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX2
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512vl --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX512
 ; REQUIRES: asserts
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
@@ -14,35 +14,39 @@ target triple = "x86_64-unknown-linux-gnu"
 define void @test() {
 ; SSE2-LABEL: 'test'
 ; SSE2:  LV: Found an estimated cost of 1 for VF 1 For instruction: %v0 = load float, ptr %in0, align 4
-; SSE2:  Cost of 3 for VF 2: INTERLEAVE-GROUP with factor 2 at %v0
-; SSE2:  Cost of 4 for VF 4: INTERLEAVE-GROUP with factor 2 at %v0
-; SSE2:  Cost of 28 for VF 8: INTERLEAVE-GROUP with factor 2 at %v0
-; SSE2:  Cost of 56 for VF 16: INTERLEAVE-GROUP with factor 2 at %v0
+; SSE2:  LV: Found an estimated cost of 1 for VF 1 For instruction: %v0 = load float, ptr %in0, align 4
+; SSE2:  LV: Found an estimated cost of 3 for VF 2 For instruction: %v0 = load float, ptr %in0, align 4
+; SSE2:  LV: Found an estimated cost of 4 for VF 4 For instruction: %v0 = load float, ptr %in0, align 4
+; SSE2:  LV: Found an estimated cost of 28 for VF 8 For instruction: %v0 = load float, ptr %in0, align 4
+; SSE2:  LV: Found an estimated cost of 56 for VF 16 For instruction: %v0 = load float, ptr %in0, align 4
 ;
 ; AVX1-LABEL: 'test'
 ; AVX1:  LV: Found an estimated cost of 1 for VF 1 For instruction: %v0 = load float, ptr %in0, align 4
-; AVX1:  Cost of 3 for VF 2: INTERLEAVE-GROUP with factor 2 at %v0
-; AVX1:  Cost of 3 for VF 4: INTERLEAVE-GROUP with factor 2 at %v0
-; AVX1:  Cost of 30 for VF 8: INTERLEAVE-GROUP with factor 2 at %v0
-; AVX1:  Cost of 60 for VF 16: INTERLEAVE-GROUP with factor 2 at %v0
-; AVX1:  Cost of 120 for VF 32: INTERLEAVE-GROUP with factor 2 at %v0
+; AVX1:  LV: Found an estimated cost of 1 for VF 1 For instruction: %v0 = load float, ptr %in0, align 4
+; AVX1:  LV: Found an estimated cost of 3 for VF 2 For instruction: %v0 = load float, ptr %in0, align 4
+; AVX1:  LV: Found an estimated cost of 3 for VF 4 For instruction: %v0 = load float, ptr %in0, align 4
+; AVX1:  LV: Found an estimated cost of 30 for VF 8 For instruction: %v0 = load float, ptr %in0, align 4
+; AVX1:  LV: Found an estimated cost of 60 for VF 16 For instruction: %v0 = load float, ptr %in0, align 4
+; AVX1:  LV: Found an estimated cost of 120 for VF 32 For instruction: %v0 = load float, ptr %in0, align 4
 ;
 ; AVX2-LABEL: 'test'
 ; AVX2:  LV: Found an estimated cost of 1 for VF 1 For instruction: %v0 = load float, ptr %in0, align 4
-; AVX2:  Cost of 3 for VF 2: INTERLEAVE-GROUP with factor 2 at %v0
-; AVX2:  Cost of 3 for VF 4: INTERLEAVE-GROUP with factor 2 at %v0
-; AVX2:  Cost of 6 for VF 8: INTERLEAVE-GROUP with factor 2 at %v0
-; AVX2:  Cost of 12 for VF 16: INTERLEAVE-GROUP with factor 2 at %v0
-; AVX2:  Cost of 24 for VF 32: INTERLEAVE-GROUP with factor 2 at %v0
+; AVX2:  LV: Found an estimated cost of 1 for VF 1 For instruction: %v0 = load float, ptr %in0, align 4
+; AVX2:  LV: Found an estimated cost of 3 for VF 2 For instruction: %v0 = load float, ptr %in0, align 4
+; AVX2:  LV: Found an estimated cost of 3 for VF 4 For instruction: %v0 = load float, ptr %in0, align 4
+; AVX2:  LV: Found an estimated cost of 6 for VF 8 For instruction: %v0 = load float, ptr %in0, align 4
+; AVX2:  LV: Found an estimated cost of 12 for VF 16 For instruction: %v0 = load float, ptr %in0, align 4
+; AVX2:  LV: Found an estimated cost of 24 for VF 32 For instruction: %v0 = load float, ptr %in0, align 4
 ;
 ; AVX512-LABEL: 'test'
 ; AVX512:  LV: Found an estimated cost of 1 for VF 1 For instruction: %v0 = load float, ptr %in0, align 4
-; AVX512:  Cost of 3 for VF 2: INTERLEAVE-GROUP with factor 2 at %v0
-; AVX512:  Cost of 3 for VF 4: INTERLEAVE-GROUP with factor 2 at %v0
-; AVX512:  Cost of 3 for VF 8: INTERLEAVE-GROUP with factor 2 at %v0
-; AVX512:  Cost of 5 for VF 16: INTERLEAVE-GROUP with factor 2 at %v0
-; AVX512:  Cost of 22 for VF 32: INTERLEAVE-GROUP with factor 2 at %v0
-; AVX512:  Cost of 92 for VF 64: INTERLEAVE-GROUP with factor 2 at %v0
+; AVX512:  LV: Found an estimated cost of 1 for VF 1 For instruction: %v0 = load float, ptr %in0, align 4
+; AVX512:  LV: Found an estimated cost of 3 for VF 2 For instruction: %v0 = load float, ptr %in0, align 4
+; AVX512:  LV: Found an estimated cost of 3 for VF 4 For instruction: %v0 = load float, ptr %in0, align 4
+; AVX512:  LV: Found an estimated cost of 3 for VF 8 For instruction: %v0 = load float, ptr %in0, align 4
+; AVX512:  LV: Found an estimated cost of 5 for VF 16 For instruction: %v0 = load float, ptr %in0, align 4
+; AVX512:  LV: Found an estimated cost of 22 for VF 32 For instruction: %v0 = load float, ptr %in0, align 4
+; AVX512:  LV: Found an estimated cost of 92 for VF 64 For instruction: %v0 = load float, ptr %in0, align 4
 ;
 entry:
   br label %for.body
diff --git a/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-f32-stride-3.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-f32-stride-3.ll
index 2f6d958e87530..4986d48b5eff9 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-f32-stride-3.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-f32-stride-3.ll
@@ -1,8 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --filter "LV: Found an estimated cost of [0-9]+ for VF [0-9]+ For instruction:\s*%v0 = load float, ptr %in0"
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse2 --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=SSE2
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx  --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX1
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2 --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX2
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512vl --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX512
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse2 --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=SSE2
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx  --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX1
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2 --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX2
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512vl --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX512
 ; REQUIRES: asserts
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
@@ -14,35 +14,39 @@ target triple = "x86_64-unknown-linux-gnu"
 define void @test() {
 ; SSE2-LABEL: 'test'
 ; SSE2:  LV: Found an estimated cost of 1 for VF 1 For instruction: %v0 = load float, ptr %in0, align 4
-; SSE2:  Cost of 9 for VF 2: INTERLEAVE-GROUP with factor 3 at %v0
-; SSE2:  Cost of 21 for VF 4: INTERLEAVE-GROUP with factor 3 at %v0
-; SSE2:  Cost of 42 for VF 8: INTERLEAVE-GROUP with factor 3 at %v0
-; SSE2:  Cost of 84 for VF 16: INTERLEAVE-GROUP with factor 3 at %v0
+; SSE2:  LV: Found an estimated cost of 1 for VF 1 For instruction: %v0 = load float, ptr %in0, align 4
+; SSE2:  LV: Found an estimated cost of 9 for VF 2 For instruction: %v0 = load float, ptr %in0, align 4
+; SSE2:  LV: Found an estimated cost of 21 for VF 4 For instruction: %v0 = load float, ptr %in0, align 4
+; SSE2:  LV: Found an estimated cost of 42 for VF 8 For instruction: %v0 = load float, ptr %in0, align 4
+; SSE2:  LV: Found an estimated cost of 84 for VF 16 For instruction: %v0 = load float, ptr %in0, align 4
 ;
 ; AVX1-LABEL: 'test'
 ; AVX1:  LV: Found an estimated cost of 1 for VF 1 For instruction: %v0 = load float, ptr %in0, align 4
-; AVX1:  Cost of 11 for VF 2: INTERLEAVE-GROUP with factor 3 at %v0
-; AVX1:  Cost of 21 for VF 4: INTERLEAVE-GROUP with factor 3 at %v0
-; AVX1:  Cost of 45 for VF 8: INTERLEAVE-GROUP with factor 3 at %v0
-; AVX1:  Cost of 90 for VF 16: INTERLEAVE-GROUP with factor 3 at %v0
-; AVX1:  Cost of 180 for VF 32: INTERLEAVE-GROUP with factor 3 at %v0
+; AVX1:  LV: Found an estimated cost of 1 for VF 1 For instruction: %v0 = load float, ptr %in0, align 4
+; AVX1:  LV: Found an estimated cost of 11 for VF 2 For instruction: %v0 = load float, ptr %in0, align 4
+; AVX1:  LV: Found an estimated cost of 21 for VF 4 For instruction: %v0 = load float, ptr %in0, align 4
+; AVX1:  LV: Found an estimated cost of 45 for VF 8 For instruction: %v0 = load float, ptr %in0, align 4
+; AVX1:  LV: Found an estimated cost of 90 for VF 16 For instruction: %v0 = load float, ptr %in0, align 4
+; AVX1:  LV: Found an estimated cost of 180 for VF 32 For instruction: %v0 = load float, ptr %in0, align 4
 ;
 ; AVX2-LABEL: 'test'
 ; AVX2:  LV: Found an estimated cost of 1 for VF 1 For instruction: %v0 = load float, ptr %in0, align 4
-; AVX2:  Cost of 6 for VF 2: INTERLEAVE-GROUP with factor 3 at %v0
-; AVX2:  Cost of 5 for VF 4: INTERLEAVE-GROUP with factor 3 at %v0
-; AVX2:  Cost of 10 for VF 8: INTERLEAVE-GROUP with factor 3 at %v0
-; AVX2:  Cost of 20 for VF 16: INTERLEAVE-GROUP with factor 3 at %v0
-; AVX2:  Cost of 44 for VF 32: INTERLEAVE-GROUP with factor 3 at %v0
+; AVX2:  LV: Found an estimated cost of 1 for VF 1 For instruction: %v0 = load float, ptr %in0, align 4
+; AVX2:  LV: Found an estimated cost of 6 for VF 2 For instruction: %v0 = load float, ptr %in0, align 4
+; AVX2:  LV: Found an estimated cost of 5 for VF 4 For instruction: %v0 = load float, ptr %in0, align 4
+; AVX2:  LV: Found an estimated cost of 10 for VF 8 For instruction: %v0 = load float, ptr %in0, align 4
+; AVX2:  LV: Found an estimated cost of 20 for VF 16 For instruction: %v0 = load float, ptr %in0, align 4
+; AVX2:  LV: Found an estimated cost of 44 for VF 32 For instruction: %v0 = load float, ptr %in0, align 4
 ;
 ; AVX512-LABEL: 'test'
 ; AVX512:  LV: Found an estimated cost of 1 for VF 1 For instruction: %v0 = load float, ptr %in0, align 4
-; AVX512:  Cost of 4 for VF 2: INTERLEAVE-GROUP with factor 3 at %v0
-; AVX512:  Cost of 4 for VF 4: INTERLEAVE-GROUP with factor 3 at %v0
-; AVX512:  Cost of 6 for VF 8: INTERLEAVE-GROUP with factor 3 at %v0
-; AVX512:  Cost of 12 for VF 16: INTERLEAVE-GROUP with factor 3 at %v0
-; AVX512:  Cost of 51 for VF 32: INTERLEAVE-GROUP with factor 3 at %v0
-; AVX512:  Cost of 210 for VF 64: INTERLEAVE-GROUP with factor 3 at %v0
+; AVX512:  LV: Found an estimated cost of 1 for VF 1 For instruction: %v0 = load float, ptr %in0, align 4
+; AVX512:  LV: Found an estimated cost of 4 for VF 2 For instruction: %v0 = load float, ptr %in0, align 4
+; AVX512:  LV: Found an estimated cost of 4 for VF 4 For instruction: %v0 = load float, ptr %in0, align 4
+; AVX512:  LV: Found an estimated cost of 6 for VF 8 For instruction: %v0 = load float, ptr %in0, align 4
+; AVX512:  LV: Found an estimated cost of 12 for VF 16 For instruction: %v0 = load float, ptr %in0, align 4
+; AVX512:  LV: Found an estimated cost of 51 for VF 32 For instruction: %v0 = load float, ptr %in0, align 4
+; AVX512:  LV: Found an estimated cost of 210 for VF 64 For instruction: %v0 = load float, ptr %in0, align 4
 ;
 entry:
   br label %for.body
diff --git a/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-f32-stride-4.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-f32-stride-4.ll
index 4ae991958deea..6a3e2471f393a 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-f32-stride-4.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-f32-stride-4.ll
@@ -1,8 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --filter "LV: Found an estimated cost of [0-9]+ for VF [0-9]+ For instruction:\s*%v0 = load float, ptr %in0"
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse2 --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=SSE2
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx  --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX1
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2 --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX2
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512vl --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX512
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse2 --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=SSE2
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx  --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX1
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2 --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX2
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512vl --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX512
 ; REQUIRES: asserts
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
@@ -14,34 +14,39 @@ target triple = "x86_64-unknown-linux-gnu"
 define void @test() {
 ; SSE2-LABEL: 'test'
 ; SSE2:  LV: Found an estimated cost of 1 for VF 1 For instruction: %v0 = load float, ptr %in0, align 4
-; SSE2:  Cost of 12 for VF 2: INTERLEAVE-GROUP with factor 4 at %v0
-; SSE2:  Cost of 28 for VF 4: INTERLEAVE-GROUP with factor 4 at %v0
-; SSE2:  Cost of 56 for VF 8: INTERLEAVE-GROUP with factor 4 at %v0
-; SSE2:  Cost of 112 for VF 16: INTERLEAVE-GROUP with factor 4 at %v0
+; SSE2:  LV: Found an estimated cost of 1 for VF 1 For instruction: %v0 = load float, ptr %in0, align 4
+; SSE2:  LV: Found an estimated cost of 12 for VF 2 For instruction: %v0 = load float, ptr %in0, align 4
+; SSE2:  LV: Found an estimated cost of 28 for VF 4 For instruction: %v0 = load float, ptr %in0, align 4
+; SSE2:  LV: Found an estimated cost of 56 for VF 8 For instruction: %v0 = load float, ptr %in0, align 4
+; SSE2:  LV: Found an estimated cost of 112 for VF 16 For instruction: %v0 = load float, ptr %in0, align 4
 ;
 ; AVX1-LABEL: 'test'
 ; AVX1:  LV: Found an estimated cost of 1 for VF 1 For instruction: %v0 = load float, ptr %in0, align 4
-; AVX1:  Cost of 12 for VF 2: INTERLEAVE-GROUP with factor 4 at %v0
-; AVX1:  Cost of 28 for VF 4: INTERLEAVE-GROUP with factor 4 at %v0
-; AVX1:  Cost of 60 for VF 8: INTERLEAVE-GROUP with factor 4 at %v0
-; AVX1:  Cost of 120 for VF 16: INTERLEAVE-GROUP with factor 4 at %v0
-; AVX1:  Cost of 240 for VF 32: INTERLEAVE-GROUP with factor 4 at %v0
+; AVX1:  LV: Found an estimated cost of 1 for VF 1 For instruction: %v0 = load float, ptr %in0, align 4
+; AVX1:  LV: Found an estimated cost of 12 for VF 2 For instruction: %v0 = load float, ptr %in0, align 4
+; AVX1:  LV: Found an estimated cost of 28 for VF 4 For instruction: %v0 = load float, ptr %in0, align 4
+; AVX1:  LV: Found an estimated cost of 60 for VF 8 For instruction: %v0 = load float, ptr %in0, align 4
+; AVX1:  LV: Found an estimated cost of 120 for VF 16 For instruction: %v0 = load float, ptr %in0, align 4
+; AVX1:  LV: Found an estimated cost of 240 for VF 32 For instruction: %v0 = load float, ptr %in0, align 4
 ;
 ; AVX2-LABEL: 'test'
 ; AVX2:  LV: Found an estimated cost of 1 for VF 1 For instruction: %v0 = load float, ptr %in0, align 4
-; AVX2:  Cost of 5 for VF 2: INTERLEAVE-GROUP with factor 4 at %v0
-; AVX2:  Cost of 10 for VF 4: INTERLEAVE-GROUP with factor 4 at %v0
-; AVX2:  Cost of 20 for VF 8: INTERLEAVE-GROUP with factor 4 at %v0
-; AVX2:  Cost of 40 for VF 16: INTERLEAVE-GROUP with factor 4 at %v0
-; AVX2:  Cost of 84 for VF 32: INTERLEAVE-GROUP with factor 4 at %v0
+; AVX2:  LV: Found an estimated cost of 1 for VF 1 For instruction: %v0 = load float, ptr %in0, align 4
+; AVX2:  LV: Found an estimated cost of 5 for VF 2 For instruction: %v0 = load float, ptr %in0, align 4
+; AVX2:  LV: Found an estimated cost of 10 for VF 4 For instruction: %v0 = load float, ptr %in0, align 4
+; AVX2:  LV: Found an estimated cost of 20 for VF 8 For instruction: %v0 = load float, ptr %in0, align 4
+; AVX2:  LV: Found an estimated cost of 40 for VF 16 For instruction: %v0 = load float, ptr %in0, align 4
+; AVX2:  LV: Found an estimated cost of 84 for VF 32 For instruction: %v0 = load float, ptr %in0, align 4
 ;
 ; AVX512-LABEL: 'test'
 ; AVX512:  LV: Found an estimated cost of 1 for VF 1 For instruction: %v0 = load float, ptr %in0, align 4
-; AVX512:  Cost of 5 for VF 2: INTERLEAVE-GROUP with factor 4 at %v0
-; AVX512:  Cost of 5 for VF 4: INTERLEAVE-GROUP with factor 4 at %v0
-; AVX512:  Cost of 8 for VF 8: INTERLEAVE-GROUP with factor 4 at %v0
-; AVX512:  Cost of 22 for VF 16: INTERLEAVE-GROUP with factor 4 at %v0
-; AVX512:  Cost of 92 for VF 32: INTERLEAVE-GROUP with factor 4 at %v0
+; AVX512:  LV: Found an estimated cost of 1 for VF 1 For instruction: %v0 = load float, ptr %in0, align 4
+; AVX512:  LV: Found an estimated cost of 5 for VF 2 For instruction: %v0 = load float, ptr %in0, align 4
+; AVX512:  LV: Found an estimated cost of 5 for VF 4 For instruction: %v0 = load float, ptr %in0, align 4
+; AVX512:  LV: Found an estimated cost of 8 for VF 8 For instruction: %v0 = load float, ptr %in0, align 4
+; AVX512:  LV: Found an estimated cost of 22 for VF 16 For instruction: %v0 = load float, ptr %in0, align 4
+; AVX512:  LV: Found an estimated cost of 92 for VF 32 For instruction: %v0 = load float, ptr %in0, align 4
+; AVX512:  LV: Found an estimated cost of 80 for VF 64 For instruction: %v0 = load float, ptr %in0, align 4
 ;
 entry:
   br label %for.body
diff --git a/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-f32-stride-5.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-f32-stride-5.ll
index 57ae02abc1199..5a17a00d16d96 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-f32-stride-5.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-f32-stride-5.ll
@@ -1,8 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --filter "LV: Found an estimated cost of [0-9]+ for VF [0-9]+ For instruction:\s*%v. = load float, ptr %in."
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse2 --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=SSE2
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx  --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX1
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2 --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX2
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512vl --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX512
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse2 --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=SSE2
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx  --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX1
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2 --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX2
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512vl --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX512
 ; REQUIRES: asserts
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
diff --git a/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-f32-stride-6.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-f32-stride-6.ll
index 155f42f8f23c8..600381d8f8c02 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-f32-stride-6.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-f32-stride-6.ll
@@ -1,8 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --filter "LV: Found an estimated cost of [0-9]+ for VF [0-9]+ For instruction:\s*%v0 = load float, ptr %in0"
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse2 --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=SSE2
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx  --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX1
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2 --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX2
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512vl --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX512
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse2 --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=SSE2
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx  --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX1
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2 --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX2
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512vl --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX512
 ; REQUIRES: asserts
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
@@ -14,12 +14,14 @@ target triple = "x86_64-unknown-linux-gnu"
 define void @test() {
 ; SSE2-LABEL: 'test'
 ; SSE2:  LV: Found an estimated cost of 1 for VF 1 For instruction: %v0 = load float, ptr %in0, align 4
+; SSE2:  LV: Found an estimated cost of 1 for VF 1 For instruction: %v0 = load float, ptr %in0, align 4
 ; SSE2:  LV: Found an estimated cost of 18 for VF 2 For instruction: %v0 = load float, ptr %in0, align 4
 ; SSE2:  LV: Found an estimated cost of 42 for VF 4 For instruction: %v0 = load float, ptr %in0, align 4
 ; SSE2:  LV: Found an estimated cost of 84 for VF 8 For instruction: %v0 = load float, ptr %in0, align 4
 ;
 ; AVX1-LABEL: 'test'
 ; AVX1:  LV: Found an estimated cost of 1 for VF 1 For instruction: %v0 = load float, ptr %in0, align 4
+; AVX1:  LV: Found an estimated cost of 1 for VF 1 For instruction: %v0 = load float, ptr %in0, align 4
 ; AVX1:  LV: Found an estimated cost of 18 for VF 2 For instruction: %v0 = load float, ptr %in0, align 4
 ; AVX1:  LV: Found an estimated cost of 42 for VF 4 For instruction: %v0 = load float, ptr %in0, align 4
 ; AVX1:  LV: Found an estimated cost of 90 for VF 8 For instruction: %v0 = load float, ptr %in0, align 4
@@ -27,6 +29,7 @@ define void @test() {
 ;
 ; AVX2-LABEL: 'test'
 ; AVX2:  LV: Found an estimated cost of 1 for VF 1 For instruction: %v0 = load float, ptr %in0, align 4
+; AVX2:  LV: Found an estimated cost of 1 for VF 1 For instruction: %v0 = load float, ptr %in0, align 4
 ; AVX2:  LV: Found an estimated cost of 8 for VF 2 For instruction: %v0 = load float, ptr %in0, align 4
 ; AVX2:  LV: Found an estimated cost of 18 for VF 4 For instruction: %v0 = load float, ptr %in0, align 4
 ; AVX2:  LV: Found an estimated cost of 37 for VF 8 For instruction: %v0 = load float, ptr %in0, align 4
@@ -34,11 +37,13 @@ define void @test() {
 ;
 ; AVX512-LABEL: 'test'
 ; AVX512:  LV: Found an estimated cost of 1 for VF 1 For instruction: %v0 = load float, ptr %in0, align 4
+; AVX512:  LV: Found an estimated cost of 1 for VF 1 For instruction: %v0 = load float, ptr %in0, align 4
 ; AVX512:  LV: Found an estimated cost of 7 for VF 2 For instruction: %v0 = load float, ptr %in0, align 4
 ; AVX512:  LV: Found an estimated cost of 11 for VF 4 For instruction: %v0 = load float, ptr %in0, align 4
 ; AVX512:  LV: Found an estimated cost of 21 for VF 8 For instruction: %v0 = load float, ptr %in0, align 4
 ; AVX512:  LV: Found an estimated cost of 51 for VF 16 For instruction: %v0 = load float, ptr %in0, align 4
 ; AVX512:  LV: Found an estimated cost of 210 for VF 32 For instruction: %v0 = load float, ptr %in0, align 4
+; AVX512:  LV: Found an estimated cost of 80 for VF 64 For instruction: %v0 = load float, ptr %in0, align 4
 ;
 entry:
   br label %for.body
diff --git a/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-f32-stride-7.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-f32-stride-7.ll
index 2cc5150f3c887..5a145dafc831d 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-f32-stride-7.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-f32-stride-7.ll
@@ -1,8 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --filter "LV: Found an estimated cost of [0-9]+ for VF [0-9]+ For instruction:\s*%v. = load float, ptr %in."
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse2 --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=SSE2
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx  --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX1
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2 --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX2
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512vl --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX512
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse2 --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=SSE2
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx  --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX1
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2 --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX2
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512vl --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX512
 ; REQUIRES: asserts
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
diff --git a/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-f32-stride-8.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-f32-stride-8.ll
index 1899741b8a3b4..1e7dde431db25 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-f32-stride-8.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-f32-stride-8.ll
@@ -1,8 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --filter "LV: Found an estimated cost of [0-9]+ for VF [0-9]+ For instruction:\s*%v. = load float, ptr %in."
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse2 --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=SSE2
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx  --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX1
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2 --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX2
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512vl --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX512
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse2 --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=SSE2
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx  --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX1
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2 --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX2
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512vl --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX512
 ; REQUIRES: asserts
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
diff --git a/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-f64-stride-2.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-f64-stride-2.ll
index 2d4b300a8100a..d76e101033f6a 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-f64-stride-2.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-f64-stride-2.ll
@@ -1,8 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --filter "LV: Found an estimated cost of [0-9]+ for VF [0-9]+ For instruction:\s*%v. = load double, ptr %in."
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse2 --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=SSE2
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx  --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX1
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2 --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX2
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512vl --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX512
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse2 --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=SSE2
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx  --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX1
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2 --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX2
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512vl --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX512
 ; REQUIRES: asserts
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
diff --git a/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-f64-stride-3.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-f64-stride-3.ll
index 5dfb25e25d6e6..a7d5f0ca881ce 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-f64-stride-3.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-f64-stride-3.ll
@@ -1,8 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --filter "LV: Found an estimated cost of [0-9]+ for VF [0-9]+ For instruction:\s*%v. = load double, ptr %in."
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse2 --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=SSE2
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx  --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX1
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2 --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX2
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512vl --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX512
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse2 --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=SSE2
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx  --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX1
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2 --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX2
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512vl --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX512
 ; REQUIRES: asserts
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
diff --git a/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-f64-stride-4.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-f64-stride-4.ll
index bd88ca810728b..e7619f14f63f1 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-f64-stride-4.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-f64-stride-4.ll
@@ -1,8 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --filter "LV: Found an estimated cost of [0-9]+ for VF [0-9]+ For instruction:\s*%v. = load double, ptr %in."
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse2 --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=SSE2
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx  --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX1
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2 --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX2
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512vl --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX512
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse2 --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=SSE2
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx  --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX1
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2 --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX2
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512vl --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX512
 ; REQUIRES: asserts
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
diff --git a/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-f64-stride-5.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-f64-stride-5.ll
index 9c0798631fdba..243a476f174da 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-f64-stride-5.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-f64-stride-5.ll
@@ -1,8 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --filter "LV: Found an estimated cost of [0-9]+ for VF [0-9]+ For instruction:\s*%v. = load double, ptr %in."
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse2 --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=SSE2
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx  --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX1
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2 --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX2
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512vl --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX512
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse2 --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=SSE2
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx  --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX1
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2 --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX2
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512vl --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX512
 ; REQUIRES: asserts
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
diff --git a/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-f64-stride-6.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-f64-stride-6.ll
index 99a735d3f552c..9849062fcb48e 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-f64-stride-6.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-f64-stride-6.ll
@@ -1,8 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --filter "LV: Found an estimated cost of [0-9]+ for VF [0-9]+ For instruction:\s*%v. = load double, ptr %in."
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse2 --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=SSE2
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx  --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX1
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2 --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX2
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512vl --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX512
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse2 --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=SSE2
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx  --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX1
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2 --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX2
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512vl --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX512
 ; REQUIRES: asserts
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
diff --git a/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-f64-stride-7.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-f64-stride-7.ll
index 168e9166ea1dd..8b4ac35bed12e 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-f64-stride-7.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-f64-stride-7.ll
@@ -1,8 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --filter "LV: Found an estimated cost of [0-9]+ for VF [0-9]+ For instruction:\s*%v. = load double, ptr %in."
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse2 --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=SSE2
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx  --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX1
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2 --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX2
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512vl --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX512
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse2 --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=SSE2
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx  --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX1
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2 --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX2
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512vl --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX512
 ; REQUIRES: asserts
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
diff --git a/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-f64-stride-8.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-f64-stride-8.ll
index 919a17e8729e0..6050edec2900c 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-f64-stride-8.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-f64-stride-8.ll
@@ -1,8 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --filter "LV: Found an estimated cost of [0-9]+ for VF [0-9]+ For instruction:\s*%v. = load double, ptr %in."
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse2 --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=SSE2
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx  --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX1
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2 --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX2
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512vl --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX512
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse2 --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=SSE2
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx  --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX1
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2 --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX2
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512vl --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX512
 ; REQUIRES: asserts
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
diff --git a/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-i16-stride-2.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-i16-stride-2.ll
index 9fcd04bd52def..39f9bec780425 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-i16-stride-2.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-i16-stride-2.ll
@@ -1,9 +1,9 @@
 ; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --filter "LV: Found an estimated cost of [0-9]+ for VF [0-9]+ For instruction:\s*%v0 = load i16, ptr %in0"
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse2 --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=SSE2
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx  --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX1
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2 --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX2
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512vl --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX512DQ
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512vl,+avx512bw --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX512BW
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse2 --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=SSE2
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx  --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX1
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2 --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX2
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512vl --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX512DQ
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512vl,+avx512bw --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX512BW
 ; REQUIRES: asserts
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
@@ -15,44 +15,49 @@ target triple = "x86_64-unknown-linux-gnu"
 define void @test() {
 ; SSE2-LABEL: 'test'
 ; SSE2:  LV: Found an estimated cost of 1 for VF 1 For instruction: %v0 = load i16, ptr %in0, align 2
-; SSE2:  Cost of 3 for VF 2: INTERLEAVE-GROUP with factor 2 at %v0
-; SSE2:  Cost of 8 for VF 4: INTERLEAVE-GROUP with factor 2 at %v0
-; SSE2:  Cost of 34 for VF 8: INTERLEAVE-GROUP with factor 2 at %v0
-; SSE2:  Cost of 68 for VF 16: INTERLEAVE-GROUP with factor 2 at %v0
+; SSE2:  LV: Found an estimated cost of 1 for VF 1 For instruction: %v0 = load i16, ptr %in0, align 2
+; SSE2:  LV: Found an estimated cost of 3 for VF 2 For instruction: %v0 = load i16, ptr %in0, align 2
+; SSE2:  LV: Found an estimated cost of 8 for VF 4 For instruction: %v0 = load i16, ptr %in0, align 2
+; SSE2:  LV: Found an estimated cost of 34 for VF 8 For instruction: %v0 = load i16, ptr %in0, align 2
+; SSE2:  LV: Found an estimated cost of 68 for VF 16 For instruction: %v0 = load i16, ptr %in0, align 2
 ;
 ; AVX1-LABEL: 'test'
 ; AVX1:  LV: Found an estimated cost of 1 for VF 1 For instruction: %v0 = load i16, ptr %in0, align 2
-; AVX1:  Cost of 3 for VF 2: INTERLEAVE-GROUP with factor 2 at %v0
-; AVX1:  Cost of 3 for VF 4: INTERLEAVE-GROUP with factor 2 at %v0
-; AVX1:  Cost of 34 for VF 8: INTERLEAVE-GROUP with factor 2 at %v0
-; AVX1:  Cost of 70 for VF 16: INTERLEAVE-GROUP with factor 2 at %v0
-; AVX1:  Cost of 140 for VF 32: INTERLEAVE-GROUP with factor 2 at %v0
+; AVX1:  LV: Found an estimated cost of 1 for VF 1 For instruction: %v0 = load i16, ptr %in0, align 2
+; AVX1:  LV: Found an estimated cost of 3 for VF 2 For instruction: %v0 = load i16, ptr %in0, align 2
+; AVX1:  LV: Found an estimated cost of 3 for VF 4 For instruction: %v0 = load i16, ptr %in0, align 2
+; AVX1:  LV: Found an estimated cost of 34 for VF 8 For instruction: %v0 = load i16, ptr %in0, align 2
+; AVX1:  LV: Found an estimated cost of 70 for VF 16 For instruction: %v0 = load i16, ptr %in0, align 2
+; AVX1:  LV: Found an estimated cost of 140 for VF 32 For instruction: %v0 = load i16, ptr %in0, align 2
 ;
 ; AVX2-LABEL: 'test'
 ; AVX2:  LV: Found an estimated cost of 1 for VF 1 For instruction: %v0 = load i16, ptr %in0, align 2
-; AVX2:  Cost of 3 for VF 2: INTERLEAVE-GROUP with factor 2 at %v0
-; AVX2:  Cost of 3 for VF 4: INTERLEAVE-GROUP with factor 2 at %v0
-; AVX2:  Cost of 7 for VF 8: INTERLEAVE-GROUP with factor 2 at %v0
-; AVX2:  Cost of 11 for VF 16: INTERLEAVE-GROUP with factor 2 at %v0
-; AVX2:  Cost of 22 for VF 32: INTERLEAVE-GROUP with factor 2 at %v0
+; AVX2:  LV: Found an estimated cost of 1 for VF 1 For instruction: %v0 = load i16, ptr %in0, align 2
+; AVX2:  LV: Found an estimated cost of 3 for VF 2 For instruction: %v0 = load i16, ptr %in0, align 2
+; AVX2:  LV: Found an estimated cost of 3 for VF 4 For instruction: %v0 = load i16, ptr %in0, align 2
+; AVX2:  LV: Found an estimated cost of 7 for VF 8 For instruction: %v0 = load i16, ptr %in0, align 2
+; AVX2:  LV: Found an estimated cost of 11 for VF 16 For instruction: %v0 = load i16, ptr %in0, align 2
+; AVX2:  LV: Found an estimated cost of 22 for VF 32 For instruction: %v0 = load i16, ptr %in0, align 2
 ;
 ; AVX512DQ-LABEL: 'test'
 ; AVX512DQ:  LV: Found an estimated cost of 1 for VF 1 For instruction: %v0 = load i16, ptr %in0, align 2
-; AVX512DQ:  Cost of 3 for VF 2: INTERLEAVE-GROUP with factor 2 at %v0
-; AVX512DQ:  Cost of 3 for VF 4: INTERLEAVE-GROUP with factor 2 at %v0
-; AVX512DQ:  Cost of 7 for VF 8: INTERLEAVE-GROUP with factor 2 at %v0
-; AVX512DQ:  Cost of 10 for VF 16: INTERLEAVE-GROUP with factor 2 at %v0
-; AVX512DQ:  Cost of 20 for VF 32: INTERLEAVE-GROUP with factor 2 at %v0
-; AVX512DQ:  Cost of 284 for VF 64: INTERLEAVE-GROUP with factor 2 at %v0
+; AVX512DQ:  LV: Found an estimated cost of 1 for VF 1 For instruction: %v0 = load i16, ptr %in0, align 2
+; AVX512DQ:  LV: Found an estimated cost of 3 for VF 2 For instruction: %v0 = load i16, ptr %in0, align 2
+; AVX512DQ:  LV: Found an estimated cost of 3 for VF 4 For instruction: %v0 = load i16, ptr %in0, align 2
+; AVX512DQ:  LV: Found an estimated cost of 7 for VF 8 For instruction: %v0 = load i16, ptr %in0, align 2
+; AVX512DQ:  LV: Found an estimated cost of 10 for VF 16 For instruction: %v0 = load i16, ptr %in0, align 2
+; AVX512DQ:  LV: Found an estimated cost of 20 for VF 32 For instruction: %v0 = load i16, ptr %in0, align 2
+; AVX512DQ:  LV: Found an estimated cost of 284 for VF 64 For instruction: %v0 = load i16, ptr %in0, align 2
 ;
 ; AVX512BW-LABEL: 'test'
 ; AVX512BW:  LV: Found an estimated cost of 1 for VF 1 For instruction: %v0 = load i16, ptr %in0, align 2
-; AVX512BW:  Cost of 3 for VF 2: INTERLEAVE-GROUP with factor 2 at %v0
-; AVX512BW:  Cost of 3 for VF 4: INTERLEAVE-GROUP with factor 2 at %v0
-; AVX512BW:  Cost of 5 for VF 8: INTERLEAVE-GROUP with factor 2 at %v0
-; AVX512BW:  Cost of 5 for VF 16: INTERLEAVE-GROUP with factor 2 at %v0
-; AVX512BW:  Cost of 7 for VF 32: INTERLEAVE-GROUP with factor 2 at %v0
-; AVX512BW:  Cost of 34 for VF 64: INTERLEAVE-GROUP with factor 2 at %v0
+; AVX512BW:  LV: Found an estimated cost of 1 for VF 1 For instruction: %v0 = load i16, ptr %in0, align 2
+; AVX512BW:  LV: Found an estimated cost of 3 for VF 2 For instruction: %v0 = load i16, ptr %in0, align 2
+; AVX512BW:  LV: Found an estimated cost of 3 for VF 4 For instruction: %v0 = load i16, ptr %in0, align 2
+; AVX512BW:  LV: Found an estimated cost of 5 for VF 8 For instruction: %v0 = load i16, ptr %in0, align 2
+; AVX512BW:  LV: Found an estimated cost of 5 for VF 16 For instruction: %v0 = load i16, ptr %in0, align 2
+; AVX512BW:  LV: Found an estimated cost of 7 for VF 32 For instruction: %v0 = load i16, ptr %in0, align 2
+; AVX512BW:  LV: Found an estimated cost of 34 for VF 64 For instruction: %v0 = load i16, ptr %in0, align 2
 ;
 entry:
   br label %for.body
diff --git a/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-i16-stride-3.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-i16-stride-3.ll
index 9a4f8417921e7..7371ee8b62df1 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-i16-stride-3.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-i16-stride-3.ll
@@ -1,9 +1,9 @@
 ; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --filter "LV: Found an estimated cost of [0-9]+ for VF [0-9]+ For instruction:\s*%v0 = load i16, ptr %in0"
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse2 --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=SSE2
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx  --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX1
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2 --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX2
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512vl --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX512DQ
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512vl,+avx512bw --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX512BW
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse2 --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=SSE2
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx  --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX1
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2 --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX2
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512vl --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX512DQ
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512vl,+avx512bw --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX512BW
 ; REQUIRES: asserts
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
@@ -15,44 +15,49 @@ target triple = "x86_64-unknown-linux-gnu"
 define void @test() {
 ; SSE2-LABEL: 'test'
 ; SSE2:  LV: Found an estimated cost of 1 for VF 1 For instruction: %v0 = load i16, ptr %in0, align 2
-; SSE2:  Cost of 18 for VF 2: INTERLEAVE-GROUP with factor 3 at %v0
-; SSE2:  Cost of 26 for VF 4: INTERLEAVE-GROUP with factor 3 at %v0
-; SSE2:  Cost of 51 for VF 8: INTERLEAVE-GROUP with factor 3 at %v0
-; SSE2:  Cost of 102 for VF 16: INTERLEAVE-GROUP with factor 3 at %v0
+; SSE2:  LV: Found an estimated cost of 1 for VF 1 For instruction: %v0 = load i16, ptr %in0, align 2
+; SSE2:  LV: Found an estimated cost of 18 for VF 2 For instruction: %v0 = load i16, ptr %in0, align 2
+; SSE2:  LV: Found an estimated cost of 26 for VF 4 For instruction: %v0 = load i16, ptr %in0, align 2
+; SSE2:  LV: Found an estimated cost of 51 for VF 8 For instruction: %v0 = load i16, ptr %in0, align 2
+; SSE2:  LV: Found an estimated cost of 102 for VF 16 For instruction: %v0 = load i16, ptr %in0, align 2
 ;
 ; AVX1-LABEL: 'test'
 ; AVX1:  LV: Found an estimated cost of 1 for VF 1 For instruction: %v0 = load i16, ptr %in0, align 2
-; AVX1:  Cost of 15 for VF 2: INTERLEAVE-GROUP with factor 3 at %v0
-; AVX1:  Cost of 28 for VF 4: INTERLEAVE-GROUP with factor 3 at %v0
-; AVX1:  Cost of 51 for VF 8: INTERLEAVE-GROUP with factor 3 at %v0
-; AVX1:  Cost of 105 for VF 16: INTERLEAVE-GROUP with factor 3 at %v0
-; AVX1:  Cost of 210 for VF 32: INTERLEAVE-GROUP with factor 3 at %v0
+; AVX1:  LV: Found an estimated cost of 1 for VF 1 For instruction: %v0 = load i16, ptr %in0, align 2
+; AVX1:  LV: Found an estimated cost of 15 for VF 2 For instruction: %v0 = load i16, ptr %in0, align 2
+; AVX1:  LV: Found an estimated cost of 28 for VF 4 For instruction: %v0 = load i16, ptr %in0, align 2
+; AVX1:  LV: Found an estimated cost of 51 for VF 8 For instruction: %v0 = load i16, ptr %in0, align 2
+; AVX1:  LV: Found an estimated cost of 105 for VF 16 For instruction: %v0 = load i16, ptr %in0, align 2
+; AVX1:  LV: Found an estimated cost of 210 for VF 32 For instruction: %v0 = load i16, ptr %in0, align 2
 ;
 ; AVX2-LABEL: 'test'
 ; AVX2:  LV: Found an estimated cost of 1 for VF 1 For instruction: %v0 = load i16, ptr %in0, align 2
-; AVX2:  Cost of 8 for VF 2: INTERLEAVE-GROUP with factor 3 at %v0
-; AVX2:  Cost of 10 for VF 4: INTERLEAVE-GROUP with factor 3 at %v0
-; AVX2:  Cost of 11 for VF 8: INTERLEAVE-GROUP with factor 3 at %v0
-; AVX2:  Cost of 31 for VF 16: INTERLEAVE-GROUP with factor 3 at %v0
-; AVX2:  Cost of 62 for VF 32: INTERLEAVE-GROUP with factor 3 at %v0
+; AVX2:  LV: Found an estimated cost of 1 for VF 1 For instruction: %v0 = load i16, ptr %in0, align 2
+; AVX2:  LV: Found an estimated cost of 8 for VF 2 For instruction: %v0 = load i16, ptr %in0, align 2
+; AVX2:  LV: Found an estimated cost of 10 for VF 4 For instruction: %v0 = load i16, ptr %in0, align 2
+; AVX2:  LV: Found an estimated cost of 11 for VF 8 For instruction: %v0 = load i16, ptr %in0, align 2
+; AVX2:  LV: Found an estimated cost of 31 for VF 16 For instruction: %v0 = load i16, ptr %in0, align 2
+; AVX2:  LV: Found an estimated cost of 62 for VF 32 For instruction: %v0 = load i16, ptr %in0, align 2
 ;
 ; AVX512DQ-LABEL: 'test'
 ; AVX512DQ:  LV: Found an estimated cost of 1 for VF 1 For instruction: %v0 = load i16, ptr %in0, align 2
-; AVX512DQ:  Cost of 8 for VF 2: INTERLEAVE-GROUP with factor 3 at %v0
-; AVX512DQ:  Cost of 10 for VF 4: INTERLEAVE-GROUP with factor 3 at %v0
-; AVX512DQ:  Cost of 12 for VF 8: INTERLEAVE-GROUP with factor 3 at %v0
-; AVX512DQ:  Cost of 30 for VF 16: INTERLEAVE-GROUP with factor 3 at %v0
-; AVX512DQ:  Cost of 59 for VF 32: INTERLEAVE-GROUP with factor 3 at %v0
-; AVX512DQ:  Cost of 426 for VF 64: INTERLEAVE-GROUP with factor 3 at %v0
+; AVX512DQ:  LV: Found an estimated cost of 1 for VF 1 For instruction: %v0 = load i16, ptr %in0, align 2
+; AVX512DQ:  LV: Found an estimated cost of 8 for VF 2 For instruction: %v0 = load i16, ptr %in0, align 2
+; AVX512DQ:  LV: Found an estimated cost of 10 for VF 4 For instruction: %v0 = load i16, ptr %in0, align 2
+; AVX512DQ:  LV: Found an estimated cost of 12 for VF 8 For instruction: %v0 = load i16, ptr %in0, align 2
+; AVX512DQ:  LV: Found an estimated cost of 30 for VF 16 For instruction: %v0 = load i16, ptr %in0, align 2
+; AVX512DQ:  LV: Found an estimated cost of 59 for VF 32 For instruction: %v0 = load i16, ptr %in0, align 2
+; AVX512DQ:  LV: Found an estimated cost of 426 for VF 64 For instruction: %v0 = load i16, ptr %in0, align 2
 ;
 ; AVX512BW-LABEL: 'test'
 ; AVX512BW:  LV: Found an estimated cost of 1 for VF 1 For instruction: %v0 = load i16, ptr %in0, align 2
-; AVX512BW:  Cost of 4 for VF 2: INTERLEAVE-GROUP with factor 3 at %v0
-; AVX512BW:  Cost of 7 for VF 4: INTERLEAVE-GROUP with factor 3 at %v0
-; AVX512BW:  Cost of 7 for VF 8: INTERLEAVE-GROUP with factor 3 at %v0
-; AVX512BW:  Cost of 9 for VF 16: INTERLEAVE-GROUP with factor 3 at %v0
-; AVX512BW:  Cost of 18 for VF 32: INTERLEAVE-GROUP with factor 3 at %v0
-; AVX512BW:  Cost of 81 for VF 64: INTERLEAVE-GROUP with factor 3 at %v0
+; AVX512BW:  LV: Found an estimated cost of 1 for VF 1 For instruction: %v0 = load i16, ptr %in0, align 2
+; AVX512BW:  LV: Found an estimated cost of 4 for VF 2 For instruction: %v0 = load i16, ptr %in0, align 2
+; AVX512BW:  LV: Found an estimated cost of 7 for VF 4 For instruction: %v0 = load i16, ptr %in0, align 2
+; AVX512BW:  LV: Found an estimated cost of 7 for VF 8 For instruction: %v0 = load i16, ptr %in0, align 2
+; AVX512BW:  LV: Found an estimated cost of 9 for VF 16 For instruction: %v0 = load i16, ptr %in0, align 2
+; AVX512BW:  LV: Found an estimated cost of 18 for VF 32 For instruction: %v0 = load i16, ptr %in0, align 2
+; AVX512BW:  LV: Found an estimated cost of 81 for VF 64 For instruction: %v0 = load i16, ptr %in0, align 2
 ;
 entry:
   br label %for.body
diff --git a/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-i16-stride-4.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-i16-stride-4.ll
index 4a7a5a2dbc9b9..85254340fecff 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-i16-stride-4.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-i16-stride-4.ll
@@ -1,9 +1,9 @@
 ; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --filter "LV: Found an estimated cost of [0-9]+ for VF [0-9]+ For instruction:\s*%v0 = load i16, ptr %in0"
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse2 --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=SSE2
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx  --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX1
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2 --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX2
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512vl --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX512DQ
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512vl,+avx512bw --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX512BW
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse2 --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=SSE2
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx  --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX1
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2 --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX2
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512vl --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX512DQ
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512vl,+avx512bw --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX512BW
 ; REQUIRES: asserts
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
@@ -15,44 +15,49 @@ target triple = "x86_64-unknown-linux-gnu"
 define void @test() {
 ; SSE2-LABEL: 'test'
 ; SSE2:  LV: Found an estimated cost of 1 for VF 1 For instruction: %v0 = load i16, ptr %in0, align 2
-; SSE2:  Cost of 17 for VF 2: INTERLEAVE-GROUP with factor 4 at %v0
-; SSE2:  Cost of 34 for VF 4: INTERLEAVE-GROUP with factor 4 at %v0
-; SSE2:  Cost of 68 for VF 8: INTERLEAVE-GROUP with factor 4 at %v0
-; SSE2:  Cost of 136 for VF 16: INTERLEAVE-GROUP with factor 4 at %v0
+; SSE2:  LV: Found an estimated cost of 1 for VF 1 For instruction: %v0 = load i16, ptr %in0, align 2
+; SSE2:  LV: Found an estimated cost of 17 for VF 2 For instruction: %v0 = load i16, ptr %in0, align 2
+; SSE2:  LV: Found an estimated cost of 34 for VF 4 For instruction: %v0 = load i16, ptr %in0, align 2
+; SSE2:  LV: Found an estimated cost of 68 for VF 8 For instruction: %v0 = load i16, ptr %in0, align 2
+; SSE2:  LV: Found an estimated cost of 136 for VF 16 For instruction: %v0 = load i16, ptr %in0, align 2
 ;
 ; AVX1-LABEL: 'test'
 ; AVX1:  LV: Found an estimated cost of 1 for VF 1 For instruction: %v0 = load i16, ptr %in0, align 2
-; AVX1:  Cost of 17 for VF 2: INTERLEAVE-GROUP with factor 4 at %v0
-; AVX1:  Cost of 34 for VF 4: INTERLEAVE-GROUP with factor 4 at %v0
-; AVX1:  Cost of 68 for VF 8: INTERLEAVE-GROUP with factor 4 at %v0
-; AVX1:  Cost of 140 for VF 16: INTERLEAVE-GROUP with factor 4 at %v0
-; AVX1:  Cost of 280 for VF 32: INTERLEAVE-GROUP with factor 4 at %v0
+; AVX1:  LV: Found an estimated cost of 1 for VF 1 For instruction: %v0 = load i16, ptr %in0, align 2
+; AVX1:  LV: Found an estimated cost of 17 for VF 2 For instruction: %v0 = load i16, ptr %in0, align 2
+; AVX1:  LV: Found an estimated cost of 34 for VF 4 For instruction: %v0 = load i16, ptr %in0, align 2
+; AVX1:  LV: Found an estimated cost of 68 for VF 8 For instruction: %v0 = load i16, ptr %in0, align 2
+; AVX1:  LV: Found an estimated cost of 140 for VF 16 For instruction: %v0 = load i16, ptr %in0, align 2
+; AVX1:  LV: Found an estimated cost of 280 for VF 32 For instruction: %v0 = load i16, ptr %in0, align 2
 ;
 ; AVX2-LABEL: 'test'
 ; AVX2:  LV: Found an estimated cost of 1 for VF 1 For instruction: %v0 = load i16, ptr %in0, align 2
-; AVX2:  Cost of 7 for VF 2: INTERLEAVE-GROUP with factor 4 at %v0
-; AVX2:  Cost of 18 for VF 4: INTERLEAVE-GROUP with factor 4 at %v0
-; AVX2:  Cost of 35 for VF 8: INTERLEAVE-GROUP with factor 4 at %v0
-; AVX2:  Cost of 79 for VF 16: INTERLEAVE-GROUP with factor 4 at %v0
-; AVX2:  Cost of 158 for VF 32: INTERLEAVE-GROUP with factor 4 at %v0
+; AVX2:  LV: Found an estimated cost of 1 for VF 1 For instruction: %v0 = load i16, ptr %in0, align 2
+; AVX2:  LV: Found an estimated cost of 7 for VF 2 For instruction: %v0 = load i16, ptr %in0, align 2
+; AVX2:  LV: Found an estimated cost of 18 for VF 4 For instruction: %v0 = load i16, ptr %in0, align 2
+; AVX2:  LV: Found an estimated cost of 35 for VF 8 For instruction: %v0 = load i16, ptr %in0, align 2
+; AVX2:  LV: Found an estimated cost of 79 for VF 16 For instruction: %v0 = load i16, ptr %in0, align 2
+; AVX2:  LV: Found an estimated cost of 158 for VF 32 For instruction: %v0 = load i16, ptr %in0, align 2
 ;
 ; AVX512DQ-LABEL: 'test'
 ; AVX512DQ:  LV: Found an estimated cost of 1 for VF 1 For instruction: %v0 = load i16, ptr %in0, align 2
-; AVX512DQ:  Cost of 7 for VF 2: INTERLEAVE-GROUP with factor 4 at %v0
-; AVX512DQ:  Cost of 18 for VF 4: INTERLEAVE-GROUP with factor 4 at %v0
-; AVX512DQ:  Cost of 34 for VF 8: INTERLEAVE-GROUP with factor 4 at %v0
-; AVX512DQ:  Cost of 77 for VF 16: INTERLEAVE-GROUP with factor 4 at %v0
-; AVX512DQ:  Cost of 154 for VF 32: INTERLEAVE-GROUP with factor 4 at %v0
-; AVX512DQ:  Cost of 568 for VF 64: INTERLEAVE-GROUP with factor 4 at %v0
+; AVX512DQ:  LV: Found an estimated cost of 1 for VF 1 For instruction: %v0 = load i16, ptr %in0, align 2
+; AVX512DQ:  LV: Found an estimated cost of 7 for VF 2 For instruction: %v0 = load i16, ptr %in0, align 2
+; AVX512DQ:  LV: Found an estimated cost of 18 for VF 4 For instruction: %v0 = load i16, ptr %in0, align 2
+; AVX512DQ:  LV: Found an estimated cost of 34 for VF 8 For instruction: %v0 = load i16, ptr %in0, align 2
+; AVX512DQ:  LV: Found an estimated cost of 77 for VF 16 For instruction: %v0 = load i16, ptr %in0, align 2
+; AVX512DQ:  LV: Found an estimated cost of 154 for VF 32 For instruction: %v0 = load i16, ptr %in0, align 2
+; AVX512DQ:  LV: Found an estimated cost of 568 for VF 64 For instruction: %v0 = load i16, ptr %in0, align 2
 ;
 ; AVX512BW-LABEL: 'test'
 ; AVX512BW:  LV: Found an estimated cost of 1 for VF 1 For instruction: %v0 = load i16, ptr %in0, align 2
-; AVX512BW:  Cost of 5 for VF 2: INTERLEAVE-GROUP with factor 4 at %v0
-; AVX512BW:  Cost of 9 for VF 4: INTERLEAVE-GROUP with factor 4 at %v0
-; AVX512BW:  Cost of 9 for VF 8: INTERLEAVE-GROUP with factor 4 at %v0
-; AVX512BW:  Cost of 12 for VF 16: INTERLEAVE-GROUP with factor 4 at %v0
-; AVX512BW:  Cost of 34 for VF 32: INTERLEAVE-GROUP with factor 4 at %v0
-; AVX512BW:  Cost of 148 for VF 64: INTERLEAVE-GROUP with factor 4 at %v0
+; AVX512BW:  LV: Found an estimated cost of 1 for VF 1 For instruction: %v0 = load i16, ptr %in0, align 2
+; AVX512BW:  LV: Found an estimated cost of 5 for VF 2 For instruction: %v0 = load i16, ptr %in0, align 2
+; AVX512BW:  LV: Found an estimated cost of 9 for VF 4 For instruction: %v0 = load i16, ptr %in0, align 2
+; AVX512BW:  LV: Found an estimated cost of 9 for VF 8 For instruction: %v0 = load i16, ptr %in0, align 2
+; AVX512BW:  LV: Found an estimated cost of 12 for VF 16 For instruction: %v0 = load i16, ptr %in0, align 2
+; AVX512BW:  LV: Found an estimated cost of 34 for VF 32 For instruction: %v0 = load i16, ptr %in0, align 2
+; AVX512BW:  LV: Found an estimated cost of 148 for VF 64 For instruction: %v0 = load i16, ptr %in0, align 2
 ;
 entry:
   br label %for.body
diff --git a/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-i16-stride-5.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-i16-stride-5.ll
index 6737c722b46ff..60db7a55eedd5 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-i16-stride-5.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-i16-stride-5.ll
@@ -1,9 +1,9 @@
 ; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --filter "LV: Found an estimated cost of [0-9]+ for VF [0-9]+ For instruction:\s*%v. = load i16, ptr %in."
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse2 --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=SSE2
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx  --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX1
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2 --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX2
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512vl --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX512DQ
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512vl,+avx512bw --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX512BW
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse2 --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=SSE2
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx  --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX1
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2 --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX2
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512vl --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX512DQ
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512vl,+avx512bw --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX512BW
 ; REQUIRES: asserts
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
diff --git a/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-i16-stride-6.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-i16-stride-6.ll
index 1e5b242433b3e..b46969f1c5ea8 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-i16-stride-6.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-i16-stride-6.ll
@@ -1,9 +1,9 @@
 ; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --filter "LV: Found an estimated cost of [0-9]+ for VF [0-9]+ For instruction:\s*%v0 = load i16, ptr %in0"
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse2 --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=SSE2
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx  --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX1
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2 --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX2
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512vl --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX512DQ
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512vl,+avx512bw --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX512BW
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse2 --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=SSE2
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx  --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX1
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2 --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX2
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512vl --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX512DQ
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512vl,+avx512bw --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX512BW
 ; REQUIRES: asserts
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
@@ -15,44 +15,49 @@ target triple = "x86_64-unknown-linux-gnu"
 define void @test() {
 ; SSE2-LABEL: 'test'
 ; SSE2:  LV: Found an estimated cost of 1 for VF 1 For instruction: %v0 = load i16, ptr %in0, align 2
-; SSE2:  Cost of 26 for VF 2: INTERLEAVE-GROUP with factor 6 at %v0
-; SSE2:  Cost of 51 for VF 4: INTERLEAVE-GROUP with factor 6 at %v0
-; SSE2:  Cost of 102 for VF 8: INTERLEAVE-GROUP with factor 6 at %v0
-; SSE2:  Cost of 204 for VF 16: INTERLEAVE-GROUP with factor 6 at %v0
+; SSE2:  LV: Found an estimated cost of 1 for VF 1 For instruction: %v0 = load i16, ptr %in0, align 2
+; SSE2:  LV: Found an estimated cost of 26 for VF 2 For instruction: %v0 = load i16, ptr %in0, align 2
+; SSE2:  LV: Found an estimated cost of 51 for VF 4 For instruction: %v0 = load i16, ptr %in0, align 2
+; SSE2:  LV: Found an estimated cost of 102 for VF 8 For instruction: %v0 = load i16, ptr %in0, align 2
+; SSE2:  LV: Found an estimated cost of 204 for VF 16 For instruction: %v0 = load i16, ptr %in0, align 2
 ;
 ; AVX1-LABEL: 'test'
 ; AVX1:  LV: Found an estimated cost of 1 for VF 1 For instruction: %v0 = load i16, ptr %in0, align 2
-; AVX1:  Cost of 28 for VF 2: INTERLEAVE-GROUP with factor 6 at %v0
-; AVX1:  Cost of 51 for VF 4: INTERLEAVE-GROUP with factor 6 at %v0
-; AVX1:  Cost of 102 for VF 8: INTERLEAVE-GROUP with factor 6 at %v0
-; AVX1:  Cost of 210 for VF 16: INTERLEAVE-GROUP with factor 6 at %v0
-; AVX1:  Cost of 420 for VF 32: INTERLEAVE-GROUP with factor 6 at %v0
+; AVX1:  LV: Found an estimated cost of 1 for VF 1 For instruction: %v0 = load i16, ptr %in0, align 2
+; AVX1:  LV: Found an estimated cost of 28 for VF 2 For instruction: %v0 = load i16, ptr %in0, align 2
+; AVX1:  LV: Found an estimated cost of 51 for VF 4 For instruction: %v0 = load i16, ptr %in0, align 2
+; AVX1:  LV: Found an estimated cost of 102 for VF 8 For instruction: %v0 = load i16, ptr %in0, align 2
+; AVX1:  LV: Found an estimated cost of 210 for VF 16 For instruction: %v0 = load i16, ptr %in0, align 2
+; AVX1:  LV: Found an estimated cost of 420 for VF 32 For instruction: %v0 = load i16, ptr %in0, align 2
 ;
 ; AVX2-LABEL: 'test'
 ; AVX2:  LV: Found an estimated cost of 1 for VF 1 For instruction: %v0 = load i16, ptr %in0, align 2
-; AVX2:  Cost of 16 for VF 2: INTERLEAVE-GROUP with factor 6 at %v0
-; AVX2:  Cost of 11 for VF 4: INTERLEAVE-GROUP with factor 6 at %v0
-; AVX2:  Cost of 42 for VF 8: INTERLEAVE-GROUP with factor 6 at %v0
-; AVX2:  Cost of 112 for VF 16: INTERLEAVE-GROUP with factor 6 at %v0
-; AVX2:  Cost of 224 for VF 32: INTERLEAVE-GROUP with factor 6 at %v0
+; AVX2:  LV: Found an estimated cost of 1 for VF 1 For instruction: %v0 = load i16, ptr %in0, align 2
+; AVX2:  LV: Found an estimated cost of 16 for VF 2 For instruction: %v0 = load i16, ptr %in0, align 2
+; AVX2:  LV: Found an estimated cost of 11 for VF 4 For instruction: %v0 = load i16, ptr %in0, align 2
+; AVX2:  LV: Found an estimated cost of 42 for VF 8 For instruction: %v0 = load i16, ptr %in0, align 2
+; AVX2:  LV: Found an estimated cost of 112 for VF 16 For instruction: %v0 = load i16, ptr %in0, align 2
+; AVX2:  LV: Found an estimated cost of 224 for VF 32 For instruction: %v0 = load i16, ptr %in0, align 2
 ;
 ; AVX512DQ-LABEL: 'test'
 ; AVX512DQ:  LV: Found an estimated cost of 1 for VF 1 For instruction: %v0 = load i16, ptr %in0, align 2
-; AVX512DQ:  Cost of 16 for VF 2: INTERLEAVE-GROUP with factor 6 at %v0
-; AVX512DQ:  Cost of 12 for VF 4: INTERLEAVE-GROUP with factor 6 at %v0
-; AVX512DQ:  Cost of 41 for VF 8: INTERLEAVE-GROUP with factor 6 at %v0
-; AVX512DQ:  Cost of 109 for VF 16: INTERLEAVE-GROUP with factor 6 at %v0
-; AVX512DQ:  Cost of 218 for VF 32: INTERLEAVE-GROUP with factor 6 at %v0
-; AVX512DQ:  Cost of 852 for VF 64: INTERLEAVE-GROUP with factor 6 at %v0
+; AVX512DQ:  LV: Found an estimated cost of 1 for VF 1 For instruction: %v0 = load i16, ptr %in0, align 2
+; AVX512DQ:  LV: Found an estimated cost of 16 for VF 2 For instruction: %v0 = load i16, ptr %in0, align 2
+; AVX512DQ:  LV: Found an estimated cost of 12 for VF 4 For instruction: %v0 = load i16, ptr %in0, align 2
+; AVX512DQ:  LV: Found an estimated cost of 41 for VF 8 For instruction: %v0 = load i16, ptr %in0, align 2
+; AVX512DQ:  LV: Found an estimated cost of 109 for VF 16 For instruction: %v0 = load i16, ptr %in0, align 2
+; AVX512DQ:  LV: Found an estimated cost of 218 for VF 32 For instruction: %v0 = load i16, ptr %in0, align 2
+; AVX512DQ:  LV: Found an estimated cost of 852 for VF 64 For instruction: %v0 = load i16, ptr %in0, align 2
 ;
 ; AVX512BW-LABEL: 'test'
 ; AVX512BW:  LV: Found an estimated cost of 1 for VF 1 For instruction: %v0 = load i16, ptr %in0, align 2
-; AVX512BW:  Cost of 13 for VF 2: INTERLEAVE-GROUP with factor 6 at %v0
-; AVX512BW:  Cost of 13 for VF 4: INTERLEAVE-GROUP with factor 6 at %v0
-; AVX512BW:  Cost of 17 for VF 8: INTERLEAVE-GROUP with factor 6 at %v0
-; AVX512BW:  Cost of 33 for VF 16: INTERLEAVE-GROUP with factor 6 at %v0
-; AVX512BW:  Cost of 81 for VF 32: INTERLEAVE-GROUP with factor 6 at %v0
-; AVX512BW:  Cost of 342 for VF 64: INTERLEAVE-GROUP with factor 6 at %v0
+; AVX512BW:  LV: Found an estimated cost of 1 for VF 1 For instruction: %v0 = load i16, ptr %in0, align 2
+; AVX512BW:  LV: Found an estimated cost of 13 for VF 2 For instruction: %v0 = load i16, ptr %in0, align 2
+; AVX512BW:  LV: Found an estimated cost of 13 for VF 4 For instruction: %v0 = load i16, ptr %in0, align 2
+; AVX512BW:  LV: Found an estimated cost of 17 for VF 8 For instruction: %v0 = load i16, ptr %in0, align 2
+; AVX512BW:  LV: Found an estimated cost of 33 for VF 16 For instruction: %v0 = load i16, ptr %in0, align 2
+; AVX512BW:  LV: Found an estimated cost of 81 for VF 32 For instruction: %v0 = load i16, ptr %in0, align 2
+; AVX512BW:  LV: Found an estimated cost of 342 for VF 64 For instruction: %v0 = load i16, ptr %in0, align 2
 ;
 entry:
   br label %for.body
diff --git a/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-i16-stride-7.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-i16-stride-7.ll
index 46d56a75f1c4d..b7f49ef1710f3 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-i16-stride-7.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-i16-stride-7.ll
@@ -1,9 +1,9 @@
 ; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --filter "LV: Found an estimated cost of [0-9]+ for VF [0-9]+ For instruction:\s*%v. = load i16, ptr %in."
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse2 --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=SSE2
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx  --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX1
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2 --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX2
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512vl --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX512DQ
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512vl,+avx512bw --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX512BW
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse2 --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=SSE2
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx  --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX1
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2 --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX2
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512vl --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX512DQ
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512vl,+avx512bw --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX512BW
 ; REQUIRES: asserts
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
diff --git a/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-i16-stride-8.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-i16-stride-8.ll
index 4d65abdaf688c..1d99bcd5bec02 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-i16-stride-8.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-i16-stride-8.ll
@@ -1,9 +1,9 @@
 ; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --filter "LV: Found an estimated cost of [0-9]+ for VF [0-9]+ For instruction:\s*%v. = load i16, ptr %in."
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse2 --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=SSE2
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx  --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX1
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2 --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX2
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512vl --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX512DQ
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512vl,+avx512bw --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX512BW
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse2 --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=SSE2
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx  --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX1
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2 --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX2
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512vl --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX512DQ
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512vl,+avx512bw --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX512BW
 ; REQUIRES: asserts
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
diff --git a/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-i32-stride-2-indices-0u.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-i32-stride-2-indices-0u.ll
index 12ffe4b1632ed..3704264f4e221 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-i32-stride-2-indices-0u.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-i32-stride-2-indices-0u.ll
@@ -1,8 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --filter "LV: Found an estimated cost of [0-9]+ for VF [0-9]+ For instruction:\s*%v0 = load i32, ptr %in0"
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse2 --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=SSE2
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx  --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX1
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2 --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX2
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512vl --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX512
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse2 --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=SSE2
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx  --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX1
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2 --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX2
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512vl --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX512
 ; REQUIRES: asserts
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
@@ -14,35 +14,39 @@ target triple = "x86_64-unknown-linux-gnu"
 define void @test() {
 ; SSE2-LABEL: 'test'
 ; SSE2:  LV: Found an estimated cost of 1 for VF 1 For instruction: %v0 = load i32, ptr %in0, align 4
-; SSE2:  Cost of 2 for VF 2: INTERLEAVE-GROUP with factor 2 at %v0
-; SSE2:  Cost of 3 for VF 4: INTERLEAVE-GROUP with factor 2 at %v0
-; SSE2:  Cost of 30 for VF 8: INTERLEAVE-GROUP with factor 2 at %v0
-; SSE2:  Cost of 60 for VF 16: INTERLEAVE-GROUP with factor 2 at %v0
+; SSE2:  LV: Found an estimated cost of 1 for VF 1 For instruction: %v0 = load i32, ptr %in0, align 4
+; SSE2:  LV: Found an estimated cost of 2 for VF 2 For instruction: %v0 = load i32, ptr %in0, align 4
+; SSE2:  LV: Found an estimated cost of 3 for VF 4 For instruction: %v0 = load i32, ptr %in0, align 4
+; SSE2:  LV: Found an estimated cost of 30 for VF 8 For instruction: %v0 = load i32, ptr %in0, align 4
+; SSE2:  LV: Found an estimated cost of 60 for VF 16 For instruction: %v0 = load i32, ptr %in0, align 4
 ;
 ; AVX1-LABEL: 'test'
 ; AVX1:  LV: Found an estimated cost of 1 for VF 1 For instruction: %v0 = load i32, ptr %in0, align 4
-; AVX1:  Cost of 2 for VF 2: INTERLEAVE-GROUP with factor 2 at %v0
-; AVX1:  Cost of 2 for VF 4: INTERLEAVE-GROUP with factor 2 at %v0
-; AVX1:  Cost of 21 for VF 8: INTERLEAVE-GROUP with factor 2 at %v0
-; AVX1:  Cost of 42 for VF 16: INTERLEAVE-GROUP with factor 2 at %v0
-; AVX1:  Cost of 84 for VF 32: INTERLEAVE-GROUP with factor 2 at %v0
+; AVX1:  LV: Found an estimated cost of 1 for VF 1 For instruction: %v0 = load i32, ptr %in0, align 4
+; AVX1:  LV: Found an estimated cost of 2 for VF 2 For instruction: %v0 = load i32, ptr %in0, align 4
+; AVX1:  LV: Found an estimated cost of 2 for VF 4 For instruction: %v0 = load i32, ptr %in0, align 4
+; AVX1:  LV: Found an estimated cost of 21 for VF 8 For instruction: %v0 = load i32, ptr %in0, align 4
+; AVX1:  LV: Found an estimated cost of 42 for VF 16 For instruction: %v0 = load i32, ptr %in0, align 4
+; AVX1:  LV: Found an estimated cost of 84 for VF 32 For instruction: %v0 = load i32, ptr %in0, align 4
 ;
 ; AVX2-LABEL: 'test'
 ; AVX2:  LV: Found an estimated cost of 1 for VF 1 For instruction: %v0 = load i32, ptr %in0, align 4
-; AVX2:  Cost of 2 for VF 2: INTERLEAVE-GROUP with factor 2 at %v0
-; AVX2:  Cost of 2 for VF 4: INTERLEAVE-GROUP with factor 2 at %v0
-; AVX2:  Cost of 4 for VF 8: INTERLEAVE-GROUP with factor 2 at %v0
-; AVX2:  Cost of 8 for VF 16: INTERLEAVE-GROUP with factor 2 at %v0
-; AVX2:  Cost of 16 for VF 32: INTERLEAVE-GROUP with factor 2 at %v0
+; AVX2:  LV: Found an estimated cost of 1 for VF 1 For instruction: %v0 = load i32, ptr %in0, align 4
+; AVX2:  LV: Found an estimated cost of 2 for VF 2 For instruction: %v0 = load i32, ptr %in0, align 4
+; AVX2:  LV: Found an estimated cost of 2 for VF 4 For instruction: %v0 = load i32, ptr %in0, align 4
+; AVX2:  LV: Found an estimated cost of 4 for VF 8 For instruction: %v0 = load i32, ptr %in0, align 4
+; AVX2:  LV: Found an estimated cost of 8 for VF 16 For instruction: %v0 = load i32, ptr %in0, align 4
+; AVX2:  LV: Found an estimated cost of 16 for VF 32 For instruction: %v0 = load i32, ptr %in0, align 4
 ;
 ; AVX512-LABEL: 'test'
 ; AVX512:  LV: Found an estimated cost of 1 for VF 1 For instruction: %v0 = load i32, ptr %in0, align 4
-; AVX512:  Cost of 1 for VF 2: INTERLEAVE-GROUP with factor 2 at %v0
-; AVX512:  Cost of 1 for VF 4: INTERLEAVE-GROUP with factor 2 at %v0
-; AVX512:  Cost of 1 for VF 8: INTERLEAVE-GROUP with factor 2 at %v0
-; AVX512:  Cost of 2 for VF 16: INTERLEAVE-GROUP with factor 2 at %v0
-; AVX512:  Cost of 13 for VF 32: INTERLEAVE-GROUP with factor 2 at %v0
-; AVX512:  Cost of 50 for VF 64: INTERLEAVE-GROUP with factor 2 at %v0
+; AVX512:  LV: Found an estimated cost of 1 for VF 1 For instruction: %v0 = load i32, ptr %in0, align 4
+; AVX512:  LV: Found an estimated cost of 1 for VF 2 For instruction: %v0 = load i32, ptr %in0, align 4
+; AVX512:  LV: Found an estimated cost of 1 for VF 4 For instruction: %v0 = load i32, ptr %in0, align 4
+; AVX512:  LV: Found an estimated cost of 1 for VF 8 For instruction: %v0 = load i32, ptr %in0, align 4
+; AVX512:  LV: Found an estimated cost of 2 for VF 16 For instruction: %v0 = load i32, ptr %in0, align 4
+; AVX512:  LV: Found an estimated cost of 13 for VF 32 For instruction: %v0 = load i32, ptr %in0, align 4
+; AVX512:  LV: Found an estimated cost of 50 for VF 64 For instruction: %v0 = load i32, ptr %in0, align 4
 ;
 entry:
   br label %for.body
diff --git a/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-i32-stride-2.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-i32-stride-2.ll
index 9c055eabe0816..196f4aae3a485 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-i32-stride-2.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-i32-stride-2.ll
@@ -1,8 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --filter "LV: Found an estimated cost of [0-9]+ for VF [0-9]+ For instruction:\s*%v0 = load i32, ptr %in0"
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse2 --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=SSE2
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx  --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX1
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2 --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX2
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512vl --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX512
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse2 --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=SSE2
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx  --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX1
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2 --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX2
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512vl --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX512
 ; REQUIRES: asserts
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
@@ -14,35 +14,39 @@ target triple = "x86_64-unknown-linux-gnu"
 define void @test() {
 ; SSE2-LABEL: 'test'
 ; SSE2:  LV: Found an estimated cost of 1 for VF 1 For instruction: %v0 = load i32, ptr %in0, align 4
-; SSE2:  Cost of 3 for VF 2: INTERLEAVE-GROUP with factor 2 at %v0
-; SSE2:  Cost of 4 for VF 4: INTERLEAVE-GROUP with factor 2 at %v0
-; SSE2:  Cost of 60 for VF 8: INTERLEAVE-GROUP with factor 2 at %v0
-; SSE2:  Cost of 120 for VF 16: INTERLEAVE-GROUP with factor 2 at %v0
+; SSE2:  LV: Found an estimated cost of 1 for VF 1 For instruction: %v0 = load i32, ptr %in0, align 4
+; SSE2:  LV: Found an estimated cost of 3 for VF 2 For instruction: %v0 = load i32, ptr %in0, align 4
+; SSE2:  LV: Found an estimated cost of 4 for VF 4 For instruction: %v0 = load i32, ptr %in0, align 4
+; SSE2:  LV: Found an estimated cost of 60 for VF 8 For instruction: %v0 = load i32, ptr %in0, align 4
+; SSE2:  LV: Found an estimated cost of 120 for VF 16 For instruction: %v0 = load i32, ptr %in0, align 4
 ;
 ; AVX1-LABEL: 'test'
 ; AVX1:  LV: Found an estimated cost of 1 for VF 1 For instruction: %v0 = load i32, ptr %in0, align 4
-; AVX1:  Cost of 3 for VF 2: INTERLEAVE-GROUP with factor 2 at %v0
-; AVX1:  Cost of 3 for VF 4: INTERLEAVE-GROUP with factor 2 at %v0
-; AVX1:  Cost of 38 for VF 8: INTERLEAVE-GROUP with factor 2 at %v0
-; AVX1:  Cost of 76 for VF 16: INTERLEAVE-GROUP with factor 2 at %v0
-; AVX1:  Cost of 152 for VF 32: INTERLEAVE-GROUP with factor 2 at %v0
+; AVX1:  LV: Found an estimated cost of 1 for VF 1 For instruction: %v0 = load i32, ptr %in0, align 4
+; AVX1:  LV: Found an estimated cost of 3 for VF 2 For instruction: %v0 = load i32, ptr %in0, align 4
+; AVX1:  LV: Found an estimated cost of 3 for VF 4 For instruction: %v0 = load i32, ptr %in0, align 4
+; AVX1:  LV: Found an estimated cost of 38 for VF 8 For instruction: %v0 = load i32, ptr %in0, align 4
+; AVX1:  LV: Found an estimated cost of 76 for VF 16 For instruction: %v0 = load i32, ptr %in0, align 4
+; AVX1:  LV: Found an estimated cost of 152 for VF 32 For instruction: %v0 = load i32, ptr %in0, align 4
 ;
 ; AVX2-LABEL: 'test'
 ; AVX2:  LV: Found an estimated cost of 1 for VF 1 For instruction: %v0 = load i32, ptr %in0, align 4
-; AVX2:  Cost of 3 for VF 2: INTERLEAVE-GROUP with factor 2 at %v0
-; AVX2:  Cost of 3 for VF 4: INTERLEAVE-GROUP with factor 2 at %v0
-; AVX2:  Cost of 6 for VF 8: INTERLEAVE-GROUP with factor 2 at %v0
-; AVX2:  Cost of 12 for VF 16: INTERLEAVE-GROUP with factor 2 at %v0
-; AVX2:  Cost of 24 for VF 32: INTERLEAVE-GROUP with factor 2 at %v0
+; AVX2:  LV: Found an estimated cost of 1 for VF 1 For instruction: %v0 = load i32, ptr %in0, align 4
+; AVX2:  LV: Found an estimated cost of 3 for VF 2 For instruction: %v0 = load i32, ptr %in0, align 4
+; AVX2:  LV: Found an estimated cost of 3 for VF 4 For instruction: %v0 = load i32, ptr %in0, align 4
+; AVX2:  LV: Found an estimated cost of 6 for VF 8 For instruction: %v0 = load i32, ptr %in0, align 4
+; AVX2:  LV: Found an estimated cost of 12 for VF 16 For instruction: %v0 = load i32, ptr %in0, align 4
+; AVX2:  LV: Found an estimated cost of 24 for VF 32 For instruction: %v0 = load i32, ptr %in0, align 4
 ;
 ; AVX512-LABEL: 'test'
 ; AVX512:  LV: Found an estimated cost of 1 for VF 1 For instruction: %v0 = load i32, ptr %in0, align 4
-; AVX512:  Cost of 3 for VF 2: INTERLEAVE-GROUP with factor 2 at %v0
-; AVX512:  Cost of 3 for VF 4: INTERLEAVE-GROUP with factor 2 at %v0
-; AVX512:  Cost of 3 for VF 8: INTERLEAVE-GROUP with factor 2 at %v0
-; AVX512:  Cost of 5 for VF 16: INTERLEAVE-GROUP with factor 2 at %v0
-; AVX512:  Cost of 22 for VF 32: INTERLEAVE-GROUP with factor 2 at %v0
-; AVX512:  Cost of 92 for VF 64: INTERLEAVE-GROUP with factor 2 at %v0
+; AVX512:  LV: Found an estimated cost of 1 for VF 1 For instruction: %v0 = load i32, ptr %in0, align 4
+; AVX512:  LV: Found an estimated cost of 3 for VF 2 For instruction: %v0 = load i32, ptr %in0, align 4
+; AVX512:  LV: Found an estimated cost of 3 for VF 4 For instruction: %v0 = load i32, ptr %in0, align 4
+; AVX512:  LV: Found an estimated cost of 3 for VF 8 For instruction: %v0 = load i32, ptr %in0, align 4
+; AVX512:  LV: Found an estimated cost of 5 for VF 16 For instruction: %v0 = load i32, ptr %in0, align 4
+; AVX512:  LV: Found an estimated cost of 22 for VF 32 For instruction: %v0 = load i32, ptr %in0, align 4
+; AVX512:  LV: Found an estimated cost of 92 for VF 64 For instruction: %v0 = load i32, ptr %in0, align 4
 ;
 entry:
   br label %for.body
diff --git a/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-i32-stride-3-indices-01u.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-i32-stride-3-indices-01u.ll
index 382e5bfcde341..679d74c4e6ed3 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-i32-stride-3-indices-01u.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-i32-stride-3-indices-01u.ll
@@ -1,8 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --filter "LV: Found an estimated cost of [0-9]+ for VF [0-9]+ For instruction:\s*%v0 = load i32, ptr %in0"
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse2 --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=SSE2
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx  --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX1
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2 --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX2
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512vl --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX512
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse2 --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=SSE2
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx  --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX1
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2 --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX2
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512vl --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX512
 ; REQUIRES: asserts
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
@@ -14,35 +14,39 @@ target triple = "x86_64-unknown-linux-gnu"
 define void @test() {
 ; SSE2-LABEL: 'test'
 ; SSE2:  LV: Found an estimated cost of 1 for VF 1 For instruction: %v0 = load i32, ptr %in0, align 4
-; SSE2:  Cost of 14 for VF 2: INTERLEAVE-GROUP with factor 3 at %v0
-; SSE2:  Cost of 31 for VF 4: INTERLEAVE-GROUP with factor 3 at %v0
-; SSE2:  Cost of 62 for VF 8: INTERLEAVE-GROUP with factor 3 at %v0
-; SSE2:  Cost of 124 for VF 16: INTERLEAVE-GROUP with factor 3 at %v0
+; SSE2:  LV: Found an estimated cost of 1 for VF 1 For instruction: %v0 = load i32, ptr %in0, align 4
+; SSE2:  LV: Found an estimated cost of 14 for VF 2 For instruction: %v0 = load i32, ptr %in0, align 4
+; SSE2:  LV: Found an estimated cost of 31 for VF 4 For instruction: %v0 = load i32, ptr %in0, align 4
+; SSE2:  LV: Found an estimated cost of 62 for VF 8 For instruction: %v0 = load i32, ptr %in0, align 4
+; SSE2:  LV: Found an estimated cost of 124 for VF 16 For instruction: %v0 = load i32, ptr %in0, align 4
 ;
 ; AVX1-LABEL: 'test'
 ; AVX1:  LV: Found an estimated cost of 1 for VF 1 For instruction: %v0 = load i32, ptr %in0, align 4
-; AVX1:  Cost of 12 for VF 2: INTERLEAVE-GROUP with factor 3 at %v0
-; AVX1:  Cost of 19 for VF 4: INTERLEAVE-GROUP with factor 3 at %v0
-; AVX1:  Cost of 40 for VF 8: INTERLEAVE-GROUP with factor 3 at %v0
-; AVX1:  Cost of 80 for VF 16: INTERLEAVE-GROUP with factor 3 at %v0
-; AVX1:  Cost of 160 for VF 32: INTERLEAVE-GROUP with factor 3 at %v0
+; AVX1:  LV: Found an estimated cost of 1 for VF 1 For instruction: %v0 = load i32, ptr %in0, align 4
+; AVX1:  LV: Found an estimated cost of 12 for VF 2 For instruction: %v0 = load i32, ptr %in0, align 4
+; AVX1:  LV: Found an estimated cost of 19 for VF 4 For instruction: %v0 = load i32, ptr %in0, align 4
+; AVX1:  LV: Found an estimated cost of 40 for VF 8 For instruction: %v0 = load i32, ptr %in0, align 4
+; AVX1:  LV: Found an estimated cost of 80 for VF 16 For instruction: %v0 = load i32, ptr %in0, align 4
+; AVX1:  LV: Found an estimated cost of 160 for VF 32 For instruction: %v0 = load i32, ptr %in0, align 4
 ;
 ; AVX2-LABEL: 'test'
 ; AVX2:  LV: Found an estimated cost of 1 for VF 1 For instruction: %v0 = load i32, ptr %in0, align 4
-; AVX2:  Cost of 5 for VF 2: INTERLEAVE-GROUP with factor 3 at %v0
-; AVX2:  Cost of 4 for VF 4: INTERLEAVE-GROUP with factor 3 at %v0
-; AVX2:  Cost of 8 for VF 8: INTERLEAVE-GROUP with factor 3 at %v0
-; AVX2:  Cost of 16 for VF 16: INTERLEAVE-GROUP with factor 3 at %v0
-; AVX2:  Cost of 34 for VF 32: INTERLEAVE-GROUP with factor 3 at %v0
+; AVX2:  LV: Found an estimated cost of 1 for VF 1 For instruction: %v0 = load i32, ptr %in0, align 4
+; AVX2:  LV: Found an estimated cost of 5 for VF 2 For instruction: %v0 = load i32, ptr %in0, align 4
+; AVX2:  LV: Found an estimated cost of 4 for VF 4 For instruction: %v0 = load i32, ptr %in0, align 4
+; AVX2:  LV: Found an estimated cost of 8 for VF 8 For instruction: %v0 = load i32, ptr %in0, align 4
+; AVX2:  LV: Found an estimated cost of 16 for VF 16 For instruction: %v0 = load i32, ptr %in0, align 4
+; AVX2:  LV: Found an estimated cost of 34 for VF 32 For instruction: %v0 = load i32, ptr %in0, align 4
 ;
 ; AVX512-LABEL: 'test'
 ; AVX512:  LV: Found an estimated cost of 1 for VF 1 For instruction: %v0 = load i32, ptr %in0, align 4
-; AVX512:  Cost of 3 for VF 2: INTERLEAVE-GROUP with factor 3 at %v0
-; AVX512:  Cost of 3 for VF 4: INTERLEAVE-GROUP with factor 3 at %v0
-; AVX512:  Cost of 5 for VF 8: INTERLEAVE-GROUP with factor 3 at %v0
-; AVX512:  Cost of 9 for VF 16: INTERLEAVE-GROUP with factor 3 at %v0
-; AVX512:  Cost of 36 for VF 32: INTERLEAVE-GROUP with factor 3 at %v0
-; AVX512:  Cost of 144 for VF 64: INTERLEAVE-GROUP with factor 3 at %v0
+; AVX512:  LV: Found an estimated cost of 1 for VF 1 For instruction: %v0 = load i32, ptr %in0, align 4
+; AVX512:  LV: Found an estimated cost of 3 for VF 2 For instruction: %v0 = load i32, ptr %in0, align 4
+; AVX512:  LV: Found an estimated cost of 3 for VF 4 For instruction: %v0 = load i32, ptr %in0, align 4
+; AVX512:  LV: Found an estimated cost of 5 for VF 8 For instruction: %v0 = load i32, ptr %in0, align 4
+; AVX512:  LV: Found an estimated cost of 9 for VF 16 For instruction: %v0 = load i32, ptr %in0, align 4
+; AVX512:  LV: Found an estimated cost of 36 for VF 32 For instruction: %v0 = load i32, ptr %in0, align 4
+; AVX512:  LV: Found an estimated cost of 144 for VF 64 For instruction: %v0 = load i32, ptr %in0, align 4
 ;
 entry:
   br label %for.body
diff --git a/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-i32-stride-3-indices-0uu.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-i32-stride-3-indices-0uu.ll
index 3c6401715311e..5472915442b83 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-i32-stride-3-indices-0uu.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-i32-stride-3-indices-0uu.ll
@@ -1,8 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --filter "LV: Found an estimated cost of [0-9]+ for VF [0-9]+ For instruction:\s*%v0 = load i32, ptr %in0"
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse2 --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=SSE2
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx  --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX1
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2 --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX2
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512vl --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX512
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse2 --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=SSE2
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx  --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX1
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2 --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX2
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512vl --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX512
 ; REQUIRES: asserts
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
@@ -14,35 +14,39 @@ target triple = "x86_64-unknown-linux-gnu"
 define void @test() {
 ; SSE2-LABEL: 'test'
 ; SSE2:  LV: Found an estimated cost of 1 for VF 1 For instruction: %v0 = load i32, ptr %in0, align 4
-; SSE2:  Cost of 8 for VF 2: INTERLEAVE-GROUP with factor 3 at %v0
-; SSE2:  Cost of 17 for VF 4: INTERLEAVE-GROUP with factor 3 at %v0
-; SSE2:  Cost of 34 for VF 8: INTERLEAVE-GROUP with factor 3 at %v0
-; SSE2:  Cost of 68 for VF 16: INTERLEAVE-GROUP with factor 3 at %v0
+; SSE2:  LV: Found an estimated cost of 1 for VF 1 For instruction: %v0 = load i32, ptr %in0, align 4
+; SSE2:  LV: Found an estimated cost of 8 for VF 2 For instruction: %v0 = load i32, ptr %in0, align 4
+; SSE2:  LV: Found an estimated cost of 17 for VF 4 For instruction: %v0 = load i32, ptr %in0, align 4
+; SSE2:  LV: Found an estimated cost of 34 for VF 8 For instruction: %v0 = load i32, ptr %in0, align 4
+; SSE2:  LV: Found an estimated cost of 68 for VF 16 For instruction: %v0 = load i32, ptr %in0, align 4
 ;
 ; AVX1-LABEL: 'test'
 ; AVX1:  LV: Found an estimated cost of 1 for VF 1 For instruction: %v0 = load i32, ptr %in0, align 4
-; AVX1:  Cost of 7 for VF 2: INTERLEAVE-GROUP with factor 3 at %v0
-; AVX1:  Cost of 11 for VF 4: INTERLEAVE-GROUP with factor 3 at %v0
-; AVX1:  Cost of 23 for VF 8: INTERLEAVE-GROUP with factor 3 at %v0
-; AVX1:  Cost of 46 for VF 16: INTERLEAVE-GROUP with factor 3 at %v0
-; AVX1:  Cost of 92 for VF 32: INTERLEAVE-GROUP with factor 3 at %v0
+; AVX1:  LV: Found an estimated cost of 1 for VF 1 For instruction: %v0 = load i32, ptr %in0, align 4
+; AVX1:  LV: Found an estimated cost of 7 for VF 2 For instruction: %v0 = load i32, ptr %in0, align 4
+; AVX1:  LV: Found an estimated cost of 11 for VF 4 For instruction: %v0 = load i32, ptr %in0, align 4
+; AVX1:  LV: Found an estimated cost of 23 for VF 8 For instruction: %v0 = load i32, ptr %in0, align 4
+; AVX1:  LV: Found an estimated cost of 46 for VF 16 For instruction: %v0 = load i32, ptr %in0, align 4
+; AVX1:  LV: Found an estimated cost of 92 for VF 32 For instruction: %v0 = load i32, ptr %in0, align 4
 ;
 ; AVX2-LABEL: 'test'
 ; AVX2:  LV: Found an estimated cost of 1 for VF 1 For instruction: %v0 = load i32, ptr %in0, align 4
-; AVX2:  Cost of 4 for VF 2: {{.*}}ir<%v0> = load
-; AVX2:  Cost of 3 for VF 4: INTERLEAVE-GROUP with factor 3 at %v0
-; AVX2:  Cost of 6 for VF 8: INTERLEAVE-GROUP with factor 3 at %v0
-; AVX2:  Cost of 11 for VF 16: INTERLEAVE-GROUP with factor 3 at %v0
-; AVX2:  Cost of 23 for VF 32: INTERLEAVE-GROUP with factor 3 at %v0
+; AVX2:  LV: Found an estimated cost of 1 for VF 1 For instruction: %v0 = load i32, ptr %in0, align 4
+; AVX2:  LV: Found an estimated cost of 4 for VF 2 For instruction: %v0 = load i32, ptr %in0, align 4
+; AVX2:  LV: Found an estimated cost of 3 for VF 4 For instruction: %v0 = load i32, ptr %in0, align 4
+; AVX2:  LV: Found an estimated cost of 6 for VF 8 For instruction: %v0 = load i32, ptr %in0, align 4
+; AVX2:  LV: Found an estimated cost of 11 for VF 16 For instruction: %v0 = load i32, ptr %in0, align 4
+; AVX2:  LV: Found an estimated cost of 23 for VF 32 For instruction: %v0 = load i32, ptr %in0, align 4
 ;
 ; AVX512-LABEL: 'test'
 ; AVX512:  LV: Found an estimated cost of 1 for VF 1 For instruction: %v0 = load i32, ptr %in0, align 4
-; AVX512:  Cost of 1 for VF 2: INTERLEAVE-GROUP with factor 3 at %v0
-; AVX512:  Cost of 1 for VF 4: INTERLEAVE-GROUP with factor 3 at %v0
-; AVX512:  Cost of 2 for VF 8: INTERLEAVE-GROUP with factor 3 at %v0
-; AVX512:  Cost of 3 for VF 16: INTERLEAVE-GROUP with factor 3 at %v0
-; AVX512:  Cost of 21 for VF 32: INTERLEAVE-GROUP with factor 3 at %v0
-; AVX512:  Cost of 78 for VF 64: INTERLEAVE-GROUP with factor 3 at %v0
+; AVX512:  LV: Found an estimated cost of 1 for VF 1 For instruction: %v0 = load i32, ptr %in0, align 4
+; AVX512:  LV: Found an estimated cost of 1 for VF 2 For instruction: %v0 = load i32, ptr %in0, align 4
+; AVX512:  LV: Found an estimated cost of 1 for VF 4 For instruction: %v0 = load i32, ptr %in0, align 4
+; AVX512:  LV: Found an estimated cost of 2 for VF 8 For instruction: %v0 = load i32, ptr %in0, align 4
+; AVX512:  LV: Found an estimated cost of 3 for VF 16 For instruction: %v0 = load i32, ptr %in0, align 4
+; AVX512:  LV: Found an estimated cost of 21 for VF 32 For instruction: %v0 = load i32, ptr %in0, align 4
+; AVX512:  LV: Found an estimated cost of 78 for VF 64 For instruction: %v0 = load i32, ptr %in0, align 4
 ;
 entry:
   br label %for.body
diff --git a/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-i32-stride-3.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-i32-stride-3.ll
index ee3c510991503..09e5816b4841a 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-i32-stride-3.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-i32-stride-3.ll
@@ -1,8 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --filter "LV: Found an estimated cost of [0-9]+ for VF [0-9]+ For instruction:\s*%v0 = load i32, ptr %in0"
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse2 --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=SSE2
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx  --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX1
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2 --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX2
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512vl --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX512
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse2 --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=SSE2
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx  --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX1
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2 --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX2
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512vl --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX512
 ; REQUIRES: asserts
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
@@ -14,35 +14,39 @@ target triple = "x86_64-unknown-linux-gnu"
 define void @test() {
 ; SSE2-LABEL: 'test'
 ; SSE2:  LV: Found an estimated cost of 1 for VF 1 For instruction: %v0 = load i32, ptr %in0, align 4
-; SSE2:  Cost of 21 for VF 2: INTERLEAVE-GROUP with factor 3 at %v0
-; SSE2:  Cost of 45 for VF 4: INTERLEAVE-GROUP with factor 3 at %v0
-; SSE2:  Cost of 90 for VF 8: INTERLEAVE-GROUP with factor 3 at %v0
-; SSE2:  Cost of 180 for VF 16: INTERLEAVE-GROUP with factor 3 at %v0
+; SSE2:  LV: Found an estimated cost of 1 for VF 1 For instruction: %v0 = load i32, ptr %in0, align 4
+; SSE2:  LV: Found an estimated cost of 21 for VF 2 For instruction: %v0 = load i32, ptr %in0, align 4
+; SSE2:  LV: Found an estimated cost of 45 for VF 4 For instruction: %v0 = load i32, ptr %in0, align 4
+; SSE2:  LV: Found an estimated cost of 90 for VF 8 For instruction: %v0 = load i32, ptr %in0, align 4
+; SSE2:  LV: Found an estimated cost of 180 for VF 16 For instruction: %v0 = load i32, ptr %in0, align 4
 ;
 ; AVX1-LABEL: 'test'
 ; AVX1:  LV: Found an estimated cost of 1 for VF 1 For instruction: %v0 = load i32, ptr %in0, align 4
-; AVX1:  Cost of 16 for VF 2: INTERLEAVE-GROUP with factor 3 at %v0
-; AVX1:  Cost of 27 for VF 4: INTERLEAVE-GROUP with factor 3 at %v0
-; AVX1:  Cost of 57 for VF 8: INTERLEAVE-GROUP with factor 3 at %v0
-; AVX1:  Cost of 114 for VF 16: INTERLEAVE-GROUP with factor 3 at %v0
-; AVX1:  Cost of 228 for VF 32: INTERLEAVE-GROUP with factor 3 at %v0
+; AVX1:  LV: Found an estimated cost of 1 for VF 1 For instruction: %v0 = load i32, ptr %in0, align 4
+; AVX1:  LV: Found an estimated cost of 16 for VF 2 For instruction: %v0 = load i32, ptr %in0, align 4
+; AVX1:  LV: Found an estimated cost of 27 for VF 4 For instruction: %v0 = load i32, ptr %in0, align 4
+; AVX1:  LV: Found an estimated cost of 57 for VF 8 For instruction: %v0 = load i32, ptr %in0, align 4
+; AVX1:  LV: Found an estimated cost of 114 for VF 16 For instruction: %v0 = load i32, ptr %in0, align 4
+; AVX1:  LV: Found an estimated cost of 228 for VF 32 For instruction: %v0 = load i32, ptr %in0, align 4
 ;
 ; AVX2-LABEL: 'test'
 ; AVX2:  LV: Found an estimated cost of 1 for VF 1 For instruction: %v0 = load i32, ptr %in0, align 4
-; AVX2:  Cost of 6 for VF 2: INTERLEAVE-GROUP with factor 3 at %v0
-; AVX2:  Cost of 5 for VF 4: INTERLEAVE-GROUP with factor 3 at %v0
-; AVX2:  Cost of 10 for VF 8: INTERLEAVE-GROUP with factor 3 at %v0
-; AVX2:  Cost of 20 for VF 16: INTERLEAVE-GROUP with factor 3 at %v0
-; AVX2:  Cost of 44 for VF 32: INTERLEAVE-GROUP with factor 3 at %v0
+; AVX2:  LV: Found an estimated cost of 1 for VF 1 For instruction: %v0 = load i32, ptr %in0, align 4
+; AVX2:  LV: Found an estimated cost of 6 for VF 2 For instruction: %v0 = load i32, ptr %in0, align 4
+; AVX2:  LV: Found an estimated cost of 5 for VF 4 For instruction: %v0 = load i32, ptr %in0, align 4
+; AVX2:  LV: Found an estimated cost of 10 for VF 8 For instruction: %v0 = load i32, ptr %in0, align 4
+; AVX2:  LV: Found an estimated cost of 20 for VF 16 For instruction: %v0 = load i32, ptr %in0, align 4
+; AVX2:  LV: Found an estimated cost of 44 for VF 32 For instruction: %v0 = load i32, ptr %in0, align 4
 ;
 ; AVX512-LABEL: 'test'
 ; AVX512:  LV: Found an estimated cost of 1 for VF 1 For instruction: %v0 = load i32, ptr %in0, align 4
-; AVX512:  Cost of 4 for VF 2: INTERLEAVE-GROUP with factor 3 at %v0
-; AVX512:  Cost of 4 for VF 4: INTERLEAVE-GROUP with factor 3 at %v0
-; AVX512:  Cost of 6 for VF 8: INTERLEAVE-GROUP with factor 3 at %v0
-; AVX512:  Cost of 12 for VF 16: INTERLEAVE-GROUP with factor 3 at %v0
-; AVX512:  Cost of 51 for VF 32: INTERLEAVE-GROUP with factor 3 at %v0
-; AVX512:  Cost of 210 for VF 64: INTERLEAVE-GROUP with factor 3 at %v0
+; AVX512:  LV: Found an estimated cost of 1 for VF 1 For instruction: %v0 = load i32, ptr %in0, align 4
+; AVX512:  LV: Found an estimated cost of 4 for VF 2 For instruction: %v0 = load i32, ptr %in0, align 4
+; AVX512:  LV: Found an estimated cost of 4 for VF 4 For instruction: %v0 = load i32, ptr %in0, align 4
+; AVX512:  LV: Found an estimated cost of 6 for VF 8 For instruction: %v0 = load i32, ptr %in0, align 4
+; AVX512:  LV: Found an estimated cost of 12 for VF 16 For instruction: %v0 = load i32, ptr %in0, align 4
+; AVX512:  LV: Found an estimated cost of 51 for VF 32 For instruction: %v0 = load i32, ptr %in0, align 4
+; AVX512:  LV: Found an estimated cost of 210 for VF 64 For instruction: %v0 = load i32, ptr %in0, align 4
 ;
 entry:
   br label %for.body
diff --git a/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-i32-stride-4-indices-012u.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-i32-stride-4-indices-012u.ll
index 17dad2598dc82..880fb82ebacd7 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-i32-stride-4-indices-012u.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-i32-stride-4-indices-012u.ll
@@ -1,8 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --filter "LV: Found an estimated cost of [0-9]+ for VF [0-9]+ For instruction:\s*%v0 = load i32, ptr %in0"
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse2 --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=SSE2
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx  --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX1
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2 --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX2
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512vl --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX512
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse2 --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=SSE2
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx  --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX1
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2 --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX2
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512vl --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX512
 ; REQUIRES: asserts
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
@@ -14,34 +14,39 @@ target triple = "x86_64-unknown-linux-gnu"
 define void @test() {
 ; SSE2-LABEL: 'test'
 ; SSE2:  LV: Found an estimated cost of 1 for VF 1 For instruction: %v0 = load i32, ptr %in0, align 4
-; SSE2:  Cost of 21 for VF 2: INTERLEAVE-GROUP with factor 4 at %v0
-; SSE2:  Cost of 45 for VF 4: INTERLEAVE-GROUP with factor 4 at %v0
-; SSE2:  Cost of 90 for VF 8: INTERLEAVE-GROUP with factor 4 at %v0
-; SSE2:  Cost of 180 for VF 16: INTERLEAVE-GROUP with factor 4 at %v0
+; SSE2:  LV: Found an estimated cost of 1 for VF 1 For instruction: %v0 = load i32, ptr %in0, align 4
+; SSE2:  LV: Found an estimated cost of 21 for VF 2 For instruction: %v0 = load i32, ptr %in0, align 4
+; SSE2:  LV: Found an estimated cost of 45 for VF 4 For instruction: %v0 = load i32, ptr %in0, align 4
+; SSE2:  LV: Found an estimated cost of 90 for VF 8 For instruction: %v0 = load i32, ptr %in0, align 4
+; SSE2:  LV: Found an estimated cost of 180 for VF 16 For instruction: %v0 = load i32, ptr %in0, align 4
 ;
 ; AVX1-LABEL: 'test'
 ; AVX1:  LV: Found an estimated cost of 1 for VF 1 For instruction: %v0 = load i32, ptr %in0, align 4
-; AVX1:  Cost of 14 for VF 2: INTERLEAVE-GROUP with factor 4 at %v0
-; AVX1:  Cost of 28 for VF 4: INTERLEAVE-GROUP with factor 4 at %v0
-; AVX1:  Cost of 59 for VF 8: INTERLEAVE-GROUP with factor 4 at %v0
-; AVX1:  Cost of 118 for VF 16: INTERLEAVE-GROUP with factor 4 at %v0
-; AVX1:  Cost of 236 for VF 32: INTERLEAVE-GROUP with factor 4 at %v0
+; AVX1:  LV: Found an estimated cost of 1 for VF 1 For instruction: %v0 = load i32, ptr %in0, align 4
+; AVX1:  LV: Found an estimated cost of 14 for VF 2 For instruction: %v0 = load i32, ptr %in0, align 4
+; AVX1:  LV: Found an estimated cost of 28 for VF 4 For instruction: %v0 = load i32, ptr %in0, align 4
+; AVX1:  LV: Found an estimated cost of 59 for VF 8 For instruction: %v0 = load i32, ptr %in0, align 4
+; AVX1:  LV: Found an estimated cost of 118 for VF 16 For instruction: %v0 = load i32, ptr %in0, align 4
+; AVX1:  LV: Found an estimated cost of 236 for VF 32 For instruction: %v0 = load i32, ptr %in0, align 4
 ;
 ; AVX2-LABEL: 'test'
 ; AVX2:  LV: Found an estimated cost of 1 for VF 1 For instruction: %v0 = load i32, ptr %in0, align 4
-; AVX2:  Cost of 4 for VF 2: INTERLEAVE-GROUP with factor 4 at %v0
-; AVX2:  Cost of 8 for VF 4: INTERLEAVE-GROUP with factor 4 at %v0
-; AVX2:  Cost of 16 for VF 8: INTERLEAVE-GROUP with factor 4 at %v0
-; AVX2:  Cost of 32 for VF 16: INTERLEAVE-GROUP with factor 4 at %v0
-; AVX2:  Cost of 67 for VF 32: INTERLEAVE-GROUP with factor 4 at %v0
+; AVX2:  LV: Found an estimated cost of 1 for VF 1 For instruction: %v0 = load i32, ptr %in0, align 4
+; AVX2:  LV: Found an estimated cost of 4 for VF 2 For instruction: %v0 = load i32, ptr %in0, align 4
+; AVX2:  LV: Found an estimated cost of 8 for VF 4 For instruction: %v0 = load i32, ptr %in0, align 4
+; AVX2:  LV: Found an estimated cost of 16 for VF 8 For instruction: %v0 = load i32, ptr %in0, align 4
+; AVX2:  LV: Found an estimated cost of 32 for VF 16 For instruction: %v0 = load i32, ptr %in0, align 4
+; AVX2:  LV: Found an estimated cost of 67 for VF 32 For instruction: %v0 = load i32, ptr %in0, align 4
 ;
 ; AVX512-LABEL: 'test'
 ; AVX512:  LV: Found an estimated cost of 1 for VF 1 For instruction: %v0 = load i32, ptr %in0, align 4
-; AVX512:  Cost of 4 for VF 2: INTERLEAVE-GROUP with factor 4 at %v0
-; AVX512:  Cost of 4 for VF 4: INTERLEAVE-GROUP with factor 4 at %v0
-; AVX512:  Cost of 6 for VF 8: INTERLEAVE-GROUP with factor 4 at %v0
-; AVX512:  Cost of 17 for VF 16: INTERLEAVE-GROUP with factor 4 at %v0
-; AVX512:  Cost of 71 for VF 32: INTERLEAVE-GROUP with factor 4 at %v0
+; AVX512:  LV: Found an estimated cost of 1 for VF 1 For instruction: %v0 = load i32, ptr %in0, align 4
+; AVX512:  LV: Found an estimated cost of 4 for VF 2 For instruction: %v0 = load i32, ptr %in0, align 4
+; AVX512:  LV: Found an estimated cost of 4 for VF 4 For instruction: %v0 = load i32, ptr %in0, align 4
+; AVX512:  LV: Found an estimated cost of 6 for VF 8 For instruction: %v0 = load i32, ptr %in0, align 4
+; AVX512:  LV: Found an estimated cost of 17 for VF 16 For instruction: %v0 = load i32, ptr %in0, align 4
+; AVX512:  LV: Found an estimated cost of 71 for VF 32 For instruction: %v0 = load i32, ptr %in0, align 4
+; AVX512:  LV: Found an estimated cost of 80 for VF 64 For instruction: %v0 = load i32, ptr %in0, align 4
 ;
 entry:
   br label %for.body
diff --git a/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-i32-stride-4-indices-01uu.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-i32-stride-4-indices-01uu.ll
index 28a6443efcfb9..7fc088bebf9c3 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-i32-stride-4-indices-01uu.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-i32-stride-4-indices-01uu.ll
@@ -1,8 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --filter "LV: Found an estimated cost of [0-9]+ for VF [0-9]+ For instruction:\s*%v. = load i32, ptr %in."
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse2 --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=SSE2
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx  --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX1
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2 --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX2
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512vl --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX512
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse2 --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=SSE2
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx  --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX1
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2 --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX2
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512vl --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX512
 ; REQUIRES: asserts
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
diff --git a/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-i32-stride-4-indices-0uuu.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-i32-stride-4-indices-0uuu.ll
index 55339f958bc2a..c76199d0f9765 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-i32-stride-4-indices-0uuu.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-i32-stride-4-indices-0uuu.ll
@@ -1,8 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --filter "LV: Found an estimated cost of [0-9]+ for VF [0-9]+ For instruction:\s*%v0 = load i32, ptr %in0"
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse2 --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=SSE2
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx  --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX1
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2 --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX2
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512vl --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX512
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse2 --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=SSE2
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx  --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX1
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2 --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX2
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512vl --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX512
 ; REQUIRES: asserts
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
@@ -14,35 +14,39 @@ target triple = "x86_64-unknown-linux-gnu"
 define void @test() {
 ; SSE2-LABEL: 'test'
 ; SSE2:  LV: Found an estimated cost of 1 for VF 1 For instruction: %v0 = load i32, ptr %in0, align 4
-; SSE2:  Cost of 7 for VF 2: INTERLEAVE-GROUP with factor 4 at %v0
-; SSE2:  Cost of 15 for VF 4: INTERLEAVE-GROUP with factor 4 at %v0
-; SSE2:  Cost of 30 for VF 8: INTERLEAVE-GROUP with factor 4 at %v0
-; SSE2:  Cost of 60 for VF 16: INTERLEAVE-GROUP with factor 4 at %v0
+; SSE2:  LV: Found an estimated cost of 1 for VF 1 For instruction: %v0 = load i32, ptr %in0, align 4
+; SSE2:  LV: Found an estimated cost of 7 for VF 2 For instruction: %v0 = load i32, ptr %in0, align 4
+; SSE2:  LV: Found an estimated cost of 15 for VF 4 For instruction: %v0 = load i32, ptr %in0, align 4
+; SSE2:  LV: Found an estimated cost of 30 for VF 8 For instruction: %v0 = load i32, ptr %in0, align 4
+; SSE2:  LV: Found an estimated cost of 60 for VF 16 For instruction: %v0 = load i32, ptr %in0, align 4
 ;
 ; AVX1-LABEL: 'test'
 ; AVX1:  LV: Found an estimated cost of 1 for VF 1 For instruction: %v0 = load i32, ptr %in0, align 4
-; AVX1:  Cost of 6 for VF 2: INTERLEAVE-GROUP with factor 4 at %v0
-; AVX1:  Cost of 12 for VF 4: INTERLEAVE-GROUP with factor 4 at %v0
-; AVX1:  Cost of 25 for VF 8: INTERLEAVE-GROUP with factor 4 at %v0
-; AVX1:  Cost of 50 for VF 16: INTERLEAVE-GROUP with factor 4 at %v0
-; AVX1:  Cost of 100 for VF 32: INTERLEAVE-GROUP with factor 4 at %v0
+; AVX1:  LV: Found an estimated cost of 1 for VF 1 For instruction: %v0 = load i32, ptr %in0, align 4
+; AVX1:  LV: Found an estimated cost of 6 for VF 2 For instruction: %v0 = load i32, ptr %in0, align 4
+; AVX1:  LV: Found an estimated cost of 12 for VF 4 For instruction: %v0 = load i32, ptr %in0, align 4
+; AVX1:  LV: Found an estimated cost of 25 for VF 8 For instruction: %v0 = load i32, ptr %in0, align 4
+; AVX1:  LV: Found an estimated cost of 50 for VF 16 For instruction: %v0 = load i32, ptr %in0, align 4
+; AVX1:  LV: Found an estimated cost of 100 for VF 32 For instruction: %v0 = load i32, ptr %in0, align 4
 ;
 ; AVX2-LABEL: 'test'
 ; AVX2:  LV: Found an estimated cost of 1 for VF 1 For instruction: %v0 = load i32, ptr %in0, align 4
-; AVX2:  Cost of 2 for VF 2: INTERLEAVE-GROUP with factor 4 at %v0
-; AVX2:  Cost of 4 for VF 4: INTERLEAVE-GROUP with factor 4 at %v0
-; AVX2:  Cost of 8 for VF 8: INTERLEAVE-GROUP with factor 4 at %v0
-; AVX2:  Cost of 16 for VF 16: INTERLEAVE-GROUP with factor 4 at %v0
-; AVX2:  Cost of 33 for VF 32: INTERLEAVE-GROUP with factor 4 at %v0
+; AVX2:  LV: Found an estimated cost of 1 for VF 1 For instruction: %v0 = load i32, ptr %in0, align 4
+; AVX2:  LV: Found an estimated cost of 2 for VF 2 For instruction: %v0 = load i32, ptr %in0, align 4
+; AVX2:  LV: Found an estimated cost of 4 for VF 4 For instruction: %v0 = load i32, ptr %in0, align 4
+; AVX2:  LV: Found an estimated cost of 8 for VF 8 For instruction: %v0 = load i32, ptr %in0, align 4
+; AVX2:  LV: Found an estimated cost of 16 for VF 16 For instruction: %v0 = load i32, ptr %in0, align 4
+; AVX2:  LV: Found an estimated cost of 33 for VF 32 For instruction: %v0 = load i32, ptr %in0, align 4
 ;
 ; AVX512-LABEL: 'test'
 ; AVX512:  LV: Found an estimated cost of 1 for VF 1 For instruction: %v0 = load i32, ptr %in0, align 4
-; AVX512:  Cost of 1 for VF 2: INTERLEAVE-GROUP with factor 4 at %v0
-; AVX512:  Cost of 1 for VF 4: INTERLEAVE-GROUP with factor 4 at %v0
-; AVX512:  Cost of 2 for VF 8: INTERLEAVE-GROUP with factor 4 at %v0
-; AVX512:  Cost of 5 for VF 16: INTERLEAVE-GROUP with factor 4 at %v0
-; AVX512:  Cost of 29 for VF 32: INTERLEAVE-GROUP with factor 4 at %v0
-; AVX512:  Cost of 80 for VF 64: {{.*}}ir<%v0> = load
+; AVX512:  LV: Found an estimated cost of 1 for VF 1 For instruction: %v0 = load i32, ptr %in0, align 4
+; AVX512:  LV: Found an estimated cost of 1 for VF 2 For instruction: %v0 = load i32, ptr %in0, align 4
+; AVX512:  LV: Found an estimated cost of 1 for VF 4 For instruction: %v0 = load i32, ptr %in0, align 4
+; AVX512:  LV: Found an estimated cost of 2 for VF 8 For instruction: %v0 = load i32, ptr %in0, align 4
+; AVX512:  LV: Found an estimated cost of 5 for VF 16 For instruction: %v0 = load i32, ptr %in0, align 4
+; AVX512:  LV: Found an estimated cost of 29 for VF 32 For instruction: %v0 = load i32, ptr %in0, align 4
+; AVX512:  LV: Found an estimated cost of 80 for VF 64 For instruction: %v0 = load i32, ptr %in0, align 4
 ;
 entry:
   br label %for.body
diff --git a/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-i32-stride-4.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-i32-stride-4.ll
index d95ab28c1ff74..4b35a71b2b40c 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-i32-stride-4.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-i32-stride-4.ll
@@ -1,8 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --filter "LV: Found an estimated cost of [0-9]+ for VF [0-9]+ For instruction:\s*%v0 = load i32, ptr %in0"
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse2 --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=SSE2
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx  --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX1
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2 --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX2
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512vl --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX512
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse2 --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=SSE2
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx  --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX1
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2 --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX2
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512vl --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX512
 ; REQUIRES: asserts
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
@@ -14,34 +14,39 @@ target triple = "x86_64-unknown-linux-gnu"
 define void @test() {
 ; SSE2-LABEL: 'test'
 ; SSE2:  LV: Found an estimated cost of 1 for VF 1 For instruction: %v0 = load i32, ptr %in0, align 4
-; SSE2:  Cost of 28 for VF 2: INTERLEAVE-GROUP with factor 4 at %v0
-; SSE2:  Cost of 60 for VF 4: INTERLEAVE-GROUP with factor 4 at %v0
-; SSE2:  Cost of 120 for VF 8: INTERLEAVE-GROUP with factor 4 at %v0
-; SSE2:  Cost of 240 for VF 16: INTERLEAVE-GROUP with factor 4 at %v0
+; SSE2:  LV: Found an estimated cost of 1 for VF 1 For instruction: %v0 = load i32, ptr %in0, align 4
+; SSE2:  LV: Found an estimated cost of 28 for VF 2 For instruction: %v0 = load i32, ptr %in0, align 4
+; SSE2:  LV: Found an estimated cost of 60 for VF 4 For instruction: %v0 = load i32, ptr %in0, align 4
+; SSE2:  LV: Found an estimated cost of 120 for VF 8 For instruction: %v0 = load i32, ptr %in0, align 4
+; SSE2:  LV: Found an estimated cost of 240 for VF 16 For instruction: %v0 = load i32, ptr %in0, align 4
 ;
 ; AVX1-LABEL: 'test'
 ; AVX1:  LV: Found an estimated cost of 1 for VF 1 For instruction: %v0 = load i32, ptr %in0, align 4
-; AVX1:  Cost of 18 for VF 2: INTERLEAVE-GROUP with factor 4 at %v0
-; AVX1:  Cost of 36 for VF 4: INTERLEAVE-GROUP with factor 4 at %v0
-; AVX1:  Cost of 76 for VF 8: INTERLEAVE-GROUP with factor 4 at %v0
-; AVX1:  Cost of 152 for VF 16: INTERLEAVE-GROUP with factor 4 at %v0
-; AVX1:  Cost of 304 for VF 32: INTERLEAVE-GROUP with factor 4 at %v0
+; AVX1:  LV: Found an estimated cost of 1 for VF 1 For instruction: %v0 = load i32, ptr %in0, align 4
+; AVX1:  LV: Found an estimated cost of 18 for VF 2 For instruction: %v0 = load i32, ptr %in0, align 4
+; AVX1:  LV: Found an estimated cost of 36 for VF 4 For instruction: %v0 = load i32, ptr %in0, align 4
+; AVX1:  LV: Found an estimated cost of 76 for VF 8 For instruction: %v0 = load i32, ptr %in0, align 4
+; AVX1:  LV: Found an estimated cost of 152 for VF 16 For instruction: %v0 = load i32, ptr %in0, align 4
+; AVX1:  LV: Found an estimated cost of 304 for VF 32 For instruction: %v0 = load i32, ptr %in0, align 4
 ;
 ; AVX2-LABEL: 'test'
 ; AVX2:  LV: Found an estimated cost of 1 for VF 1 For instruction: %v0 = load i32, ptr %in0, align 4
-; AVX2:  Cost of 5 for VF 2: INTERLEAVE-GROUP with factor 4 at %v0
-; AVX2:  Cost of 10 for VF 4: INTERLEAVE-GROUP with factor 4 at %v0
-; AVX2:  Cost of 20 for VF 8: INTERLEAVE-GROUP with factor 4 at %v0
-; AVX2:  Cost of 40 for VF 16: INTERLEAVE-GROUP with factor 4 at %v0
-; AVX2:  Cost of 84 for VF 32: INTERLEAVE-GROUP with factor 4 at %v0
+; AVX2:  LV: Found an estimated cost of 1 for VF 1 For instruction: %v0 = load i32, ptr %in0, align 4
+; AVX2:  LV: Found an estimated cost of 5 for VF 2 For instruction: %v0 = load i32, ptr %in0, align 4
+; AVX2:  LV: Found an estimated cost of 10 for VF 4 For instruction: %v0 = load i32, ptr %in0, align 4
+; AVX2:  LV: Found an estimated cost of 20 for VF 8 For instruction: %v0 = load i32, ptr %in0, align 4
+; AVX2:  LV: Found an estimated cost of 40 for VF 16 For instruction: %v0 = load i32, ptr %in0, align 4
+; AVX2:  LV: Found an estimated cost of 84 for VF 32 For instruction: %v0 = load i32, ptr %in0, align 4
 ;
 ; AVX512-LABEL: 'test'
 ; AVX512:  LV: Found an estimated cost of 1 for VF 1 For instruction: %v0 = load i32, ptr %in0, align 4
-; AVX512:  Cost of 5 for VF 2: INTERLEAVE-GROUP with factor 4 at %v0
-; AVX512:  Cost of 5 for VF 4: INTERLEAVE-GROUP with factor 4 at %v0
-; AVX512:  Cost of 8 for VF 8: INTERLEAVE-GROUP with factor 4 at %v0
-; AVX512:  Cost of 22 for VF 16: INTERLEAVE-GROUP with factor 4 at %v0
-; AVX512:  Cost of 92 for VF 32: INTERLEAVE-GROUP with factor 4 at %v0
+; AVX512:  LV: Found an estimated cost of 1 for VF 1 For instruction: %v0 = load i32, ptr %in0, align 4
+; AVX512:  LV: Found an estimated cost of 5 for VF 2 For instruction: %v0 = load i32, ptr %in0, align 4
+; AVX512:  LV: Found an estimated cost of 5 for VF 4 For instruction: %v0 = load i32, ptr %in0, align 4
+; AVX512:  LV: Found an estimated cost of 8 for VF 8 For instruction: %v0 = load i32, ptr %in0, align 4
+; AVX512:  LV: Found an estimated cost of 22 for VF 16 For instruction: %v0 = load i32, ptr %in0, align 4
+; AVX512:  LV: Found an estimated cost of 92 for VF 32 For instruction: %v0 = load i32, ptr %in0, align 4
+; AVX512:  LV: Found an estimated cost of 80 for VF 64 For instruction: %v0 = load i32, ptr %in0, align 4
 ;
 entry:
   br label %for.body
diff --git a/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-i32-stride-5.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-i32-stride-5.ll
index 5cad7bf662c5b..cd451d0cb70bc 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-i32-stride-5.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-i32-stride-5.ll
@@ -1,8 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --filter "LV: Found an estimated cost of [0-9]+ for VF [0-9]+ For instruction:\s*%v. = load i32, ptr %in."
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse2 --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=SSE2
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx  --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX1
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2 --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX2
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512vl --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX512
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse2 --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=SSE2
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx  --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX1
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2 --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX2
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512vl --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX512
 ; REQUIRES: asserts
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
diff --git a/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-i32-stride-6.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-i32-stride-6.ll
index 0f1fc532b6df4..0bfb4df3ddfe7 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-i32-stride-6.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-i32-stride-6.ll
@@ -1,8 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --filter "LV: Found an estimated cost of [0-9]+ for VF [0-9]+ For instruction:\s*%v0 = load i32, ptr %in0"
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse2 --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=SSE2
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx  --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX1
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2 --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX2
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512vl --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX512
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse2 --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=SSE2
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx  --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX1
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2 --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX2
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512vl --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX512
 ; REQUIRES: asserts
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
@@ -14,31 +14,36 @@ target triple = "x86_64-unknown-linux-gnu"
 define void @test() {
 ; SSE2-LABEL: 'test'
 ; SSE2:  LV: Found an estimated cost of 1 for VF 1 For instruction: %v0 = load i32, ptr %in0, align 4
-; SSE2:  Cost of 42 for VF 2: INTERLEAVE-GROUP with factor 6 at %v0
-; SSE2:  Cost of 90 for VF 4: INTERLEAVE-GROUP with factor 6 at %v0
-; SSE2:  Cost of 180 for VF 8: INTERLEAVE-GROUP with factor 6 at %v0
+; SSE2:  LV: Found an estimated cost of 1 for VF 1 For instruction: %v0 = load i32, ptr %in0, align 4
+; SSE2:  LV: Found an estimated cost of 42 for VF 2 For instruction: %v0 = load i32, ptr %in0, align 4
+; SSE2:  LV: Found an estimated cost of 90 for VF 4 For instruction: %v0 = load i32, ptr %in0, align 4
+; SSE2:  LV: Found an estimated cost of 180 for VF 8 For instruction: %v0 = load i32, ptr %in0, align 4
 ;
 ; AVX1-LABEL: 'test'
 ; AVX1:  LV: Found an estimated cost of 1 for VF 1 For instruction: %v0 = load i32, ptr %in0, align 4
-; AVX1:  Cost of 27 for VF 2: INTERLEAVE-GROUP with factor 6 at %v0
-; AVX1:  Cost of 54 for VF 4: INTERLEAVE-GROUP with factor 6 at %v0
-; AVX1:  Cost of 114 for VF 8: INTERLEAVE-GROUP with factor 6 at %v0
-; AVX1:  Cost of 228 for VF 16: INTERLEAVE-GROUP with factor 6 at %v0
+; AVX1:  LV: Found an estimated cost of 1 for VF 1 For instruction: %v0 = load i32, ptr %in0, align 4
+; AVX1:  LV: Found an estimated cost of 27 for VF 2 For instruction: %v0 = load i32, ptr %in0, align 4
+; AVX1:  LV: Found an estimated cost of 54 for VF 4 For instruction: %v0 = load i32, ptr %in0, align 4
+; AVX1:  LV: Found an estimated cost of 114 for VF 8 For instruction: %v0 = load i32, ptr %in0, align 4
+; AVX1:  LV: Found an estimated cost of 228 for VF 16 For instruction: %v0 = load i32, ptr %in0, align 4
 ;
 ; AVX2-LABEL: 'test'
 ; AVX2:  LV: Found an estimated cost of 1 for VF 1 For instruction: %v0 = load i32, ptr %in0, align 4
-; AVX2:  Cost of 8 for VF 2: INTERLEAVE-GROUP with factor 6 at %v0
-; AVX2:  Cost of 18 for VF 4: INTERLEAVE-GROUP with factor 6 at %v0
-; AVX2:  Cost of 37 for VF 8: INTERLEAVE-GROUP with factor 6 at %v0
-; AVX2:  Cost of 76 for VF 16: INTERLEAVE-GROUP with factor 6 at %v0
+; AVX2:  LV: Found an estimated cost of 1 for VF 1 For instruction: %v0 = load i32, ptr %in0, align 4
+; AVX2:  LV: Found an estimated cost of 8 for VF 2 For instruction: %v0 = load i32, ptr %in0, align 4
+; AVX2:  LV: Found an estimated cost of 18 for VF 4 For instruction: %v0 = load i32, ptr %in0, align 4
+; AVX2:  LV: Found an estimated cost of 37 for VF 8 For instruction: %v0 = load i32, ptr %in0, align 4
+; AVX2:  LV: Found an estimated cost of 76 for VF 16 For instruction: %v0 = load i32, ptr %in0, align 4
 ;
 ; AVX512-LABEL: 'test'
 ; AVX512:  LV: Found an estimated cost of 1 for VF 1 For instruction: %v0 = load i32, ptr %in0, align 4
-; AVX512:  Cost of 7 for VF 2: INTERLEAVE-GROUP with factor 6 at %v0
-; AVX512:  Cost of 11 for VF 4: INTERLEAVE-GROUP with factor 6 at %v0
-; AVX512:  Cost of 21 for VF 8: INTERLEAVE-GROUP with factor 6 at %v0
-; AVX512:  Cost of 51 for VF 16: INTERLEAVE-GROUP with factor 6 at %v0
-; AVX512:  Cost of 210 for VF 32: INTERLEAVE-GROUP with factor 6 at %v0
+; AVX512:  LV: Found an estimated cost of 1 for VF 1 For instruction: %v0 = load i32, ptr %in0, align 4
+; AVX512:  LV: Found an estimated cost of 7 for VF 2 For instruction: %v0 = load i32, ptr %in0, align 4
+; AVX512:  LV: Found an estimated cost of 11 for VF 4 For instruction: %v0 = load i32, ptr %in0, align 4
+; AVX512:  LV: Found an estimated cost of 21 for VF 8 For instruction: %v0 = load i32, ptr %in0, align 4
+; AVX512:  LV: Found an estimated cost of 51 for VF 16 For instruction: %v0 = load i32, ptr %in0, align 4
+; AVX512:  LV: Found an estimated cost of 210 for VF 32 For instruction: %v0 = load i32, ptr %in0, align 4
+; AVX512:  LV: Found an estimated cost of 80 for VF 64 For instruction: %v0 = load i32, ptr %in0, align 4
 ;
 entry:
   br label %for.body
diff --git a/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-i32-stride-7.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-i32-stride-7.ll
index cfb83d4a02365..26613caf76faa 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-i32-stride-7.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-i32-stride-7.ll
@@ -1,8 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --filter "LV: Found an estimated cost of [0-9]+ for VF [0-9]+ For instruction:\s*%v. = load i32, ptr %in."
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse2 --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=SSE2
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx  --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX1
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2 --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX2
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512vl --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX512
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse2 --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=SSE2
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx  --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX1
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2 --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX2
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512vl --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX512
 ; REQUIRES: asserts
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
diff --git a/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-i32-stride-8.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-i32-stride-8.ll
index 775785462de47..46f4351dff0bf 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-i32-stride-8.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-i32-stride-8.ll
@@ -1,8 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --filter "LV: Found an estimated cost of [0-9]+ for VF [0-9]+ For instruction:\s*%v. = load i32, ptr %in."
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse2 --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=SSE2
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx  --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX1
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2 --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX2
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512vl --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX512
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse2 --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=SSE2
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx  --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX1
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2 --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX2
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512vl --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX512
 ; REQUIRES: asserts
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
diff --git a/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-i64-stride-2.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-i64-stride-2.ll
index cf350cc9f8307..866db993c2ec0 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-i64-stride-2.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-i64-stride-2.ll
@@ -1,8 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --filter "LV: Found an estimated cost of [0-9]+ for VF [0-9]+ For instruction:\s*%v. = load i64, ptr %in."
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse2 --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=SSE2
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx  --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX1
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2 --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX2
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512vl --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX512
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse2 --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=SSE2
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx  --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX1
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2 --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX2
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512vl --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX512
 ; REQUIRES: asserts
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
diff --git a/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-i64-stride-3.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-i64-stride-3.ll
index 9ca0d8c9d7e33..7c575b6dc8f37 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-i64-stride-3.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-i64-stride-3.ll
@@ -1,8 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --filter "LV: Found an estimated cost of [0-9]+ for VF [0-9]+ For instruction:\s*%v. = load i64, ptr %in."
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse2 --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=SSE2
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx  --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX1
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2 --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX2
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512vl --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX512
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse2 --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=SSE2
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx  --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX1
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2 --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX2
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512vl --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX512
 ; REQUIRES: asserts
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
diff --git a/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-i64-stride-4.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-i64-stride-4.ll
index 86ee6c8b30bda..4089026b957ad 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-i64-stride-4.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-i64-stride-4.ll
@@ -1,8 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --filter "LV: Found an estimated cost of [0-9]+ for VF [0-9]+ For instruction:\s*%v. = load i64, ptr %in."
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse2 --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=SSE2
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx  --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX1
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2 --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX2
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512vl --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX512
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse2 --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=SSE2
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx  --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX1
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2 --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX2
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512vl --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX512
 ; REQUIRES: asserts
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
diff --git a/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-i64-stride-5.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-i64-stride-5.ll
index f6143d4ae9f3b..60fb3bf6ea959 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-i64-stride-5.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-i64-stride-5.ll
@@ -1,8 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --filter "LV: Found an estimated cost of [0-9]+ for VF [0-9]+ For instruction:\s*%v. = load i64, ptr %in."
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse2 --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=SSE2
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx  --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX1
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2 --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX2
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512vl --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX512
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse2 --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=SSE2
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx  --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX1
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2 --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX2
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512vl --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX512
 ; REQUIRES: asserts
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
diff --git a/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-i64-stride-6.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-i64-stride-6.ll
index 43dc53d683de3..f2d5bc8eafe85 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-i64-stride-6.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-i64-stride-6.ll
@@ -1,8 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --filter "LV: Found an estimated cost of [0-9]+ for VF [0-9]+ For instruction:\s*%v. = load i64, ptr %in."
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse2 --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=SSE2
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx  --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX1
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2 --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX2
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512vl --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX512
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse2 --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=SSE2
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx  --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX1
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2 --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX2
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512vl --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX512
 ; REQUIRES: asserts
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
diff --git a/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-i64-stride-7.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-i64-stride-7.ll
index 70ed74dcc26da..c741e473e7739 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-i64-stride-7.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-i64-stride-7.ll
@@ -1,8 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --filter "LV: Found an estimated cost of [0-9]+ for VF [0-9]+ For instruction:\s*%v. = load i64, ptr %in."
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse2 --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=SSE2
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx  --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX1
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2 --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX2
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512vl --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX512
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse2 --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=SSE2
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx  --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX1
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2 --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX2
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512vl --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX512
 ; REQUIRES: asserts
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
diff --git a/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-i64-stride-8.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-i64-stride-8.ll
index 401e4de111d73..68370852ce85a 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-i64-stride-8.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-i64-stride-8.ll
@@ -1,8 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --filter "LV: Found an estimated cost of [0-9]+ for VF [0-9]+ For instruction:\s*%v. = load i64, ptr %in."
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse2 --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=SSE2
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx  --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX1
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2 --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX2
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512vl --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX512
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse2 --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=SSE2
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx  --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX1
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2 --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX2
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512vl --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX512
 ; REQUIRES: asserts
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
diff --git a/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-i8-stride-2.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-i8-stride-2.ll
index 66430532d863f..f5185befc688b 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-i8-stride-2.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-i8-stride-2.ll
@@ -1,9 +1,9 @@
 ; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --filter "LV: Found an estimated cost of [0-9]+ for VF [0-9]+ For instruction:\s*%v0 = load i8, ptr %in0"
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse2 --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=SSE2
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx  --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX1
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2 --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX2
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512vl --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX512DQ
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512vl,+avx512bw --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX512BW
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse2 --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=SSE2
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx  --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX1
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2 --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX2
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512vl --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX512DQ
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512vl,+avx512bw --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX512BW
 ; REQUIRES: asserts
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
@@ -15,44 +15,49 @@ target triple = "x86_64-unknown-linux-gnu"
 define void @test() {
 ; SSE2-LABEL: 'test'
 ; SSE2:  LV: Found an estimated cost of 1 for VF 1 For instruction: %v0 = load i8, ptr %in0, align 1
-; SSE2:  Cost of 14 for VF 2: INTERLEAVE-GROUP with factor 2 at %v0
-; SSE2:  Cost of 30 for VF 4: INTERLEAVE-GROUP with factor 2 at %v0
-; SSE2:  Cost of 62 for VF 8: INTERLEAVE-GROUP with factor 2 at %v0
-; SSE2:  Cost of 126 for VF 16: INTERLEAVE-GROUP with factor 2 at %v0
+; SSE2:  LV: Found an estimated cost of 1 for VF 1 For instruction: %v0 = load i8, ptr %in0, align 1
+; SSE2:  LV: Found an estimated cost of 14 for VF 2 For instruction: %v0 = load i8, ptr %in0, align 1
+; SSE2:  LV: Found an estimated cost of 30 for VF 4 For instruction: %v0 = load i8, ptr %in0, align 1
+; SSE2:  LV: Found an estimated cost of 62 for VF 8 For instruction: %v0 = load i8, ptr %in0, align 1
+; SSE2:  LV: Found an estimated cost of 126 for VF 16 For instruction: %v0 = load i8, ptr %in0, align 1
 ;
 ; AVX1-LABEL: 'test'
 ; AVX1:  LV: Found an estimated cost of 1 for VF 1 For instruction: %v0 = load i8, ptr %in0, align 1
-; AVX1:  Cost of 9 for VF 2: INTERLEAVE-GROUP with factor 2 at %v0
-; AVX1:  Cost of 17 for VF 4: INTERLEAVE-GROUP with factor 2 at %v0
-; AVX1:  Cost of 33 for VF 8: INTERLEAVE-GROUP with factor 2 at %v0
-; AVX1:  Cost of 66 for VF 16: INTERLEAVE-GROUP with factor 2 at %v0
-; AVX1:  Cost of 134 for VF 32: INTERLEAVE-GROUP with factor 2 at %v0
+; AVX1:  LV: Found an estimated cost of 1 for VF 1 For instruction: %v0 = load i8, ptr %in0, align 1
+; AVX1:  LV: Found an estimated cost of 9 for VF 2 For instruction: %v0 = load i8, ptr %in0, align 1
+; AVX1:  LV: Found an estimated cost of 17 for VF 4 For instruction: %v0 = load i8, ptr %in0, align 1
+; AVX1:  LV: Found an estimated cost of 33 for VF 8 For instruction: %v0 = load i8, ptr %in0, align 1
+; AVX1:  LV: Found an estimated cost of 66 for VF 16 For instruction: %v0 = load i8, ptr %in0, align 1
+; AVX1:  LV: Found an estimated cost of 134 for VF 32 For instruction: %v0 = load i8, ptr %in0, align 1
 ;
 ; AVX2-LABEL: 'test'
 ; AVX2:  LV: Found an estimated cost of 1 for VF 1 For instruction: %v0 = load i8, ptr %in0, align 1
-; AVX2:  Cost of 3 for VF 2: INTERLEAVE-GROUP with factor 2 at %v0
-; AVX2:  Cost of 3 for VF 4: INTERLEAVE-GROUP with factor 2 at %v0
-; AVX2:  Cost of 3 for VF 8: INTERLEAVE-GROUP with factor 2 at %v0
-; AVX2:  Cost of 5 for VF 16: INTERLEAVE-GROUP with factor 2 at %v0
-; AVX2:  Cost of 8 for VF 32: INTERLEAVE-GROUP with factor 2 at %v0
+; AVX2:  LV: Found an estimated cost of 1 for VF 1 For instruction: %v0 = load i8, ptr %in0, align 1
+; AVX2:  LV: Found an estimated cost of 3 for VF 2 For instruction: %v0 = load i8, ptr %in0, align 1
+; AVX2:  LV: Found an estimated cost of 3 for VF 4 For instruction: %v0 = load i8, ptr %in0, align 1
+; AVX2:  LV: Found an estimated cost of 3 for VF 8 For instruction: %v0 = load i8, ptr %in0, align 1
+; AVX2:  LV: Found an estimated cost of 5 for VF 16 For instruction: %v0 = load i8, ptr %in0, align 1
+; AVX2:  LV: Found an estimated cost of 8 for VF 32 For instruction: %v0 = load i8, ptr %in0, align 1
 ;
 ; AVX512DQ-LABEL: 'test'
 ; AVX512DQ:  LV: Found an estimated cost of 1 for VF 1 For instruction: %v0 = load i8, ptr %in0, align 1
-; AVX512DQ:  Cost of 3 for VF 2: INTERLEAVE-GROUP with factor 2 at %v0
-; AVX512DQ:  Cost of 3 for VF 4: INTERLEAVE-GROUP with factor 2 at %v0
-; AVX512DQ:  Cost of 3 for VF 8: INTERLEAVE-GROUP with factor 2 at %v0
-; AVX512DQ:  Cost of 5 for VF 16: INTERLEAVE-GROUP with factor 2 at %v0
-; AVX512DQ:  Cost of 7 for VF 32: INTERLEAVE-GROUP with factor 2 at %v0
-; AVX512DQ:  Cost of 270 for VF 64: INTERLEAVE-GROUP with factor 2 at %v0
+; AVX512DQ:  LV: Found an estimated cost of 1 for VF 1 For instruction: %v0 = load i8, ptr %in0, align 1
+; AVX512DQ:  LV: Found an estimated cost of 3 for VF 2 For instruction: %v0 = load i8, ptr %in0, align 1
+; AVX512DQ:  LV: Found an estimated cost of 3 for VF 4 For instruction: %v0 = load i8, ptr %in0, align 1
+; AVX512DQ:  LV: Found an estimated cost of 3 for VF 8 For instruction: %v0 = load i8, ptr %in0, align 1
+; AVX512DQ:  LV: Found an estimated cost of 5 for VF 16 For instruction: %v0 = load i8, ptr %in0, align 1
+; AVX512DQ:  LV: Found an estimated cost of 7 for VF 32 For instruction: %v0 = load i8, ptr %in0, align 1
+; AVX512DQ:  LV: Found an estimated cost of 270 for VF 64 For instruction: %v0 = load i8, ptr %in0, align 1
 ;
 ; AVX512BW-LABEL: 'test'
 ; AVX512BW:  LV: Found an estimated cost of 1 for VF 1 For instruction: %v0 = load i8, ptr %in0, align 1
-; AVX512BW:  Cost of 3 for VF 2: INTERLEAVE-GROUP with factor 2 at %v0
-; AVX512BW:  Cost of 3 for VF 4: INTERLEAVE-GROUP with factor 2 at %v0
-; AVX512BW:  Cost of 3 for VF 8: INTERLEAVE-GROUP with factor 2 at %v0
-; AVX512BW:  Cost of 9 for VF 16: INTERLEAVE-GROUP with factor 2 at %v0
-; AVX512BW:  Cost of 17 for VF 32: INTERLEAVE-GROUP with factor 2 at %v0
-; AVX512BW:  Cost of 41 for VF 64: INTERLEAVE-GROUP with factor 2 at %v0
+; AVX512BW:  LV: Found an estimated cost of 1 for VF 1 For instruction: %v0 = load i8, ptr %in0, align 1
+; AVX512BW:  LV: Found an estimated cost of 3 for VF 2 For instruction: %v0 = load i8, ptr %in0, align 1
+; AVX512BW:  LV: Found an estimated cost of 3 for VF 4 For instruction: %v0 = load i8, ptr %in0, align 1
+; AVX512BW:  LV: Found an estimated cost of 3 for VF 8 For instruction: %v0 = load i8, ptr %in0, align 1
+; AVX512BW:  LV: Found an estimated cost of 9 for VF 16 For instruction: %v0 = load i8, ptr %in0, align 1
+; AVX512BW:  LV: Found an estimated cost of 17 for VF 32 For instruction: %v0 = load i8, ptr %in0, align 1
+; AVX512BW:  LV: Found an estimated cost of 41 for VF 64 For instruction: %v0 = load i8, ptr %in0, align 1
 ;
 entry:
   br label %for.body
diff --git a/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-i8-stride-3.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-i8-stride-3.ll
index 508a8a6acecec..f6b775457123b 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-i8-stride-3.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-i8-stride-3.ll
@@ -1,9 +1,9 @@
 ; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --filter "LV: Found an estimated cost of [0-9]+ for VF [0-9]+ For instruction:\s*%v0 = load i8, ptr %in0"
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse2 --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=SSE2
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx  --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX1
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2 --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX2
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512vl --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX512DQ
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512vl,+avx512bw --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX512BW
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse2 --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=SSE2
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx  --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX1
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2 --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX2
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512vl --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX512DQ
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512vl,+avx512bw --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX512BW
 ; REQUIRES: asserts
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
@@ -15,44 +15,49 @@ target triple = "x86_64-unknown-linux-gnu"
 define void @test() {
 ; SSE2-LABEL: 'test'
 ; SSE2:  LV: Found an estimated cost of 1 for VF 1 For instruction: %v0 = load i8, ptr %in0, align 1
-; SSE2:  Cost of 24 for VF 2: INTERLEAVE-GROUP with factor 3 at %v0
-; SSE2:  Cost of 50 for VF 4: INTERLEAVE-GROUP with factor 3 at %v0
-; SSE2:  Cost of 93 for VF 8: INTERLEAVE-GROUP with factor 3 at %v0
-; SSE2:  Cost of 189 for VF 16: INTERLEAVE-GROUP with factor 3 at %v0
+; SSE2:  LV: Found an estimated cost of 1 for VF 1 For instruction: %v0 = load i8, ptr %in0, align 1
+; SSE2:  LV: Found an estimated cost of 24 for VF 2 For instruction: %v0 = load i8, ptr %in0, align 1
+; SSE2:  LV: Found an estimated cost of 50 for VF 4 For instruction: %v0 = load i8, ptr %in0, align 1
+; SSE2:  LV: Found an estimated cost of 93 for VF 8 For instruction: %v0 = load i8, ptr %in0, align 1
+; SSE2:  LV: Found an estimated cost of 189 for VF 16 For instruction: %v0 = load i8, ptr %in0, align 1
 ;
 ; AVX1-LABEL: 'test'
 ; AVX1:  LV: Found an estimated cost of 1 for VF 1 For instruction: %v0 = load i8, ptr %in0, align 1
-; AVX1:  Cost of 16 for VF 2: INTERLEAVE-GROUP with factor 3 at %v0
-; AVX1:  Cost of 27 for VF 4: INTERLEAVE-GROUP with factor 3 at %v0
-; AVX1:  Cost of 52 for VF 8: INTERLEAVE-GROUP with factor 3 at %v0
-; AVX1:  Cost of 99 for VF 16: INTERLEAVE-GROUP with factor 3 at %v0
-; AVX1:  Cost of 201 for VF 32: INTERLEAVE-GROUP with factor 3 at %v0
+; AVX1:  LV: Found an estimated cost of 1 for VF 1 For instruction: %v0 = load i8, ptr %in0, align 1
+; AVX1:  LV: Found an estimated cost of 16 for VF 2 For instruction: %v0 = load i8, ptr %in0, align 1
+; AVX1:  LV: Found an estimated cost of 27 for VF 4 For instruction: %v0 = load i8, ptr %in0, align 1
+; AVX1:  LV: Found an estimated cost of 52 for VF 8 For instruction: %v0 = load i8, ptr %in0, align 1
+; AVX1:  LV: Found an estimated cost of 99 for VF 16 For instruction: %v0 = load i8, ptr %in0, align 1
+; AVX1:  LV: Found an estimated cost of 201 for VF 32 For instruction: %v0 = load i8, ptr %in0, align 1
 ;
 ; AVX2-LABEL: 'test'
 ; AVX2:  LV: Found an estimated cost of 1 for VF 1 For instruction: %v0 = load i8, ptr %in0, align 1
-; AVX2:  Cost of 7 for VF 2: INTERLEAVE-GROUP with factor 3 at %v0
-; AVX2:  Cost of 6 for VF 4: INTERLEAVE-GROUP with factor 3 at %v0
-; AVX2:  Cost of 9 for VF 8: INTERLEAVE-GROUP with factor 3 at %v0
-; AVX2:  Cost of 13 for VF 16: INTERLEAVE-GROUP with factor 3 at %v0
-; AVX2:  Cost of 17 for VF 32: INTERLEAVE-GROUP with factor 3 at %v0
+; AVX2:  LV: Found an estimated cost of 1 for VF 1 For instruction: %v0 = load i8, ptr %in0, align 1
+; AVX2:  LV: Found an estimated cost of 7 for VF 2 For instruction: %v0 = load i8, ptr %in0, align 1
+; AVX2:  LV: Found an estimated cost of 6 for VF 4 For instruction: %v0 = load i8, ptr %in0, align 1
+; AVX2:  LV: Found an estimated cost of 9 for VF 8 For instruction: %v0 = load i8, ptr %in0, align 1
+; AVX2:  LV: Found an estimated cost of 13 for VF 16 For instruction: %v0 = load i8, ptr %in0, align 1
+; AVX2:  LV: Found an estimated cost of 17 for VF 32 For instruction: %v0 = load i8, ptr %in0, align 1
 ;
 ; AVX512DQ-LABEL: 'test'
 ; AVX512DQ:  LV: Found an estimated cost of 1 for VF 1 For instruction: %v0 = load i8, ptr %in0, align 1
-; AVX512DQ:  Cost of 7 for VF 2: INTERLEAVE-GROUP with factor 3 at %v0
-; AVX512DQ:  Cost of 6 for VF 4: INTERLEAVE-GROUP with factor 3 at %v0
-; AVX512DQ:  Cost of 9 for VF 8: INTERLEAVE-GROUP with factor 3 at %v0
-; AVX512DQ:  Cost of 14 for VF 16: INTERLEAVE-GROUP with factor 3 at %v0
-; AVX512DQ:  Cost of 16 for VF 32: INTERLEAVE-GROUP with factor 3 at %v0
-; AVX512DQ:  Cost of 405 for VF 64: INTERLEAVE-GROUP with factor 3 at %v0
+; AVX512DQ:  LV: Found an estimated cost of 1 for VF 1 For instruction: %v0 = load i8, ptr %in0, align 1
+; AVX512DQ:  LV: Found an estimated cost of 7 for VF 2 For instruction: %v0 = load i8, ptr %in0, align 1
+; AVX512DQ:  LV: Found an estimated cost of 6 for VF 4 For instruction: %v0 = load i8, ptr %in0, align 1
+; AVX512DQ:  LV: Found an estimated cost of 9 for VF 8 For instruction: %v0 = load i8, ptr %in0, align 1
+; AVX512DQ:  LV: Found an estimated cost of 14 for VF 16 For instruction: %v0 = load i8, ptr %in0, align 1
+; AVX512DQ:  LV: Found an estimated cost of 16 for VF 32 For instruction: %v0 = load i8, ptr %in0, align 1
+; AVX512DQ:  LV: Found an estimated cost of 405 for VF 64 For instruction: %v0 = load i8, ptr %in0, align 1
 ;
 ; AVX512BW-LABEL: 'test'
 ; AVX512BW:  LV: Found an estimated cost of 1 for VF 1 For instruction: %v0 = load i8, ptr %in0, align 1
-; AVX512BW:  Cost of 4 for VF 2: INTERLEAVE-GROUP with factor 3 at %v0
-; AVX512BW:  Cost of 4 for VF 4: INTERLEAVE-GROUP with factor 3 at %v0
-; AVX512BW:  Cost of 13 for VF 8: INTERLEAVE-GROUP with factor 3 at %v0
-; AVX512BW:  Cost of 13 for VF 16: INTERLEAVE-GROUP with factor 3 at %v0
-; AVX512BW:  Cost of 16 for VF 32: INTERLEAVE-GROUP with factor 3 at %v0
-; AVX512BW:  Cost of 25 for VF 64: INTERLEAVE-GROUP with factor 3 at %v0
+; AVX512BW:  LV: Found an estimated cost of 1 for VF 1 For instruction: %v0 = load i8, ptr %in0, align 1
+; AVX512BW:  LV: Found an estimated cost of 4 for VF 2 For instruction: %v0 = load i8, ptr %in0, align 1
+; AVX512BW:  LV: Found an estimated cost of 4 for VF 4 For instruction: %v0 = load i8, ptr %in0, align 1
+; AVX512BW:  LV: Found an estimated cost of 13 for VF 8 For instruction: %v0 = load i8, ptr %in0, align 1
+; AVX512BW:  LV: Found an estimated cost of 13 for VF 16 For instruction: %v0 = load i8, ptr %in0, align 1
+; AVX512BW:  LV: Found an estimated cost of 16 for VF 32 For instruction: %v0 = load i8, ptr %in0, align 1
+; AVX512BW:  LV: Found an estimated cost of 25 for VF 64 For instruction: %v0 = load i8, ptr %in0, align 1
 ;
 entry:
   br label %for.body
diff --git a/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-i8-stride-4.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-i8-stride-4.ll
index 509562c19114d..13d9884727392 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-i8-stride-4.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-i8-stride-4.ll
@@ -1,9 +1,9 @@
 ; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --filter "LV: Found an estimated cost of [0-9]+ for VF [0-9]+ For instruction:\s*%v0 = load i8, ptr %in0"
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse2 --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=SSE2
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx  --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX1
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2 --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX2
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512vl --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX512DQ
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512vl,+avx512bw --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX512BW
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse2 --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=SSE2
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx  --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX1
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2 --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX2
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512vl --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX512DQ
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512vl,+avx512bw --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX512BW
 ; REQUIRES: asserts
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
@@ -15,44 +15,49 @@ target triple = "x86_64-unknown-linux-gnu"
 define void @test() {
 ; SSE2-LABEL: 'test'
 ; SSE2:  LV: Found an estimated cost of 1 for VF 1 For instruction: %v0 = load i8, ptr %in0, align 1
-; SSE2:  Cost of 28 for VF 2: INTERLEAVE-GROUP with factor 4 at %v0
-; SSE2:  Cost of 60 for VF 4: INTERLEAVE-GROUP with factor 4 at %v0
-; SSE2:  Cost of 124 for VF 8: INTERLEAVE-GROUP with factor 4 at %v0
-; SSE2:  Cost of 252 for VF 16: INTERLEAVE-GROUP with factor 4 at %v0
+; SSE2:  LV: Found an estimated cost of 1 for VF 1 For instruction: %v0 = load i8, ptr %in0, align 1
+; SSE2:  LV: Found an estimated cost of 28 for VF 2 For instruction: %v0 = load i8, ptr %in0, align 1
+; SSE2:  LV: Found an estimated cost of 60 for VF 4 For instruction: %v0 = load i8, ptr %in0, align 1
+; SSE2:  LV: Found an estimated cost of 124 for VF 8 For instruction: %v0 = load i8, ptr %in0, align 1
+; SSE2:  LV: Found an estimated cost of 252 for VF 16 For instruction: %v0 = load i8, ptr %in0, align 1
 ;
 ; AVX1-LABEL: 'test'
 ; AVX1:  LV: Found an estimated cost of 1 for VF 1 For instruction: %v0 = load i8, ptr %in0, align 1
-; AVX1:  Cost of 17 for VF 2: INTERLEAVE-GROUP with factor 4 at %v0
-; AVX1:  Cost of 33 for VF 4: INTERLEAVE-GROUP with factor 4 at %v0
-; AVX1:  Cost of 66 for VF 8: INTERLEAVE-GROUP with factor 4 at %v0
-; AVX1:  Cost of 132 for VF 16: INTERLEAVE-GROUP with factor 4 at %v0
-; AVX1:  Cost of 268 for VF 32: INTERLEAVE-GROUP with factor 4 at %v0
+; AVX1:  LV: Found an estimated cost of 1 for VF 1 For instruction: %v0 = load i8, ptr %in0, align 1
+; AVX1:  LV: Found an estimated cost of 17 for VF 2 For instruction: %v0 = load i8, ptr %in0, align 1
+; AVX1:  LV: Found an estimated cost of 33 for VF 4 For instruction: %v0 = load i8, ptr %in0, align 1
+; AVX1:  LV: Found an estimated cost of 66 for VF 8 For instruction: %v0 = load i8, ptr %in0, align 1
+; AVX1:  LV: Found an estimated cost of 132 for VF 16 For instruction: %v0 = load i8, ptr %in0, align 1
+; AVX1:  LV: Found an estimated cost of 268 for VF 32 For instruction: %v0 = load i8, ptr %in0, align 1
 ;
 ; AVX2-LABEL: 'test'
 ; AVX2:  LV: Found an estimated cost of 1 for VF 1 For instruction: %v0 = load i8, ptr %in0, align 1
-; AVX2:  Cost of 5 for VF 2: INTERLEAVE-GROUP with factor 4 at %v0
-; AVX2:  Cost of 5 for VF 4: INTERLEAVE-GROUP with factor 4 at %v0
-; AVX2:  Cost of 13 for VF 8: INTERLEAVE-GROUP with factor 4 at %v0
-; AVX2:  Cost of 26 for VF 16: INTERLEAVE-GROUP with factor 4 at %v0
-; AVX2:  Cost of 60 for VF 32: INTERLEAVE-GROUP with factor 4 at %v0
+; AVX2:  LV: Found an estimated cost of 1 for VF 1 For instruction: %v0 = load i8, ptr %in0, align 1
+; AVX2:  LV: Found an estimated cost of 5 for VF 2 For instruction: %v0 = load i8, ptr %in0, align 1
+; AVX2:  LV: Found an estimated cost of 5 for VF 4 For instruction: %v0 = load i8, ptr %in0, align 1
+; AVX2:  LV: Found an estimated cost of 13 for VF 8 For instruction: %v0 = load i8, ptr %in0, align 1
+; AVX2:  LV: Found an estimated cost of 26 for VF 16 For instruction: %v0 = load i8, ptr %in0, align 1
+; AVX2:  LV: Found an estimated cost of 60 for VF 32 For instruction: %v0 = load i8, ptr %in0, align 1
 ;
 ; AVX512DQ-LABEL: 'test'
 ; AVX512DQ:  LV: Found an estimated cost of 1 for VF 1 For instruction: %v0 = load i8, ptr %in0, align 1
-; AVX512DQ:  Cost of 5 for VF 2: INTERLEAVE-GROUP with factor 4 at %v0
-; AVX512DQ:  Cost of 5 for VF 4: INTERLEAVE-GROUP with factor 4 at %v0
-; AVX512DQ:  Cost of 13 for VF 8: INTERLEAVE-GROUP with factor 4 at %v0
-; AVX512DQ:  Cost of 25 for VF 16: INTERLEAVE-GROUP with factor 4 at %v0
-; AVX512DQ:  Cost of 58 for VF 32: INTERLEAVE-GROUP with factor 4 at %v0
-; AVX512DQ:  Cost of 540 for VF 64: INTERLEAVE-GROUP with factor 4 at %v0
+; AVX512DQ:  LV: Found an estimated cost of 1 for VF 1 For instruction: %v0 = load i8, ptr %in0, align 1
+; AVX512DQ:  LV: Found an estimated cost of 5 for VF 2 For instruction: %v0 = load i8, ptr %in0, align 1
+; AVX512DQ:  LV: Found an estimated cost of 5 for VF 4 For instruction: %v0 = load i8, ptr %in0, align 1
+; AVX512DQ:  LV: Found an estimated cost of 13 for VF 8 For instruction: %v0 = load i8, ptr %in0, align 1
+; AVX512DQ:  LV: Found an estimated cost of 25 for VF 16 For instruction: %v0 = load i8, ptr %in0, align 1
+; AVX512DQ:  LV: Found an estimated cost of 58 for VF 32 For instruction: %v0 = load i8, ptr %in0, align 1
+; AVX512DQ:  LV: Found an estimated cost of 540 for VF 64 For instruction: %v0 = load i8, ptr %in0, align 1
 ;
 ; AVX512BW-LABEL: 'test'
 ; AVX512BW:  LV: Found an estimated cost of 1 for VF 1 For instruction: %v0 = load i8, ptr %in0, align 1
-; AVX512BW:  Cost of 5 for VF 2: INTERLEAVE-GROUP with factor 4 at %v0
-; AVX512BW:  Cost of 5 for VF 4: INTERLEAVE-GROUP with factor 4 at %v0
-; AVX512BW:  Cost of 17 for VF 8: INTERLEAVE-GROUP with factor 4 at %v0
-; AVX512BW:  Cost of 33 for VF 16: INTERLEAVE-GROUP with factor 4 at %v0
-; AVX512BW:  Cost of 80 for VF 32: INTERLEAVE-GROUP with factor 4 at %v0
-; AVX512BW:  Cost of 238 for VF 64: INTERLEAVE-GROUP with factor 4 at %v0
+; AVX512BW:  LV: Found an estimated cost of 1 for VF 1 For instruction: %v0 = load i8, ptr %in0, align 1
+; AVX512BW:  LV: Found an estimated cost of 5 for VF 2 For instruction: %v0 = load i8, ptr %in0, align 1
+; AVX512BW:  LV: Found an estimated cost of 5 for VF 4 For instruction: %v0 = load i8, ptr %in0, align 1
+; AVX512BW:  LV: Found an estimated cost of 17 for VF 8 For instruction: %v0 = load i8, ptr %in0, align 1
+; AVX512BW:  LV: Found an estimated cost of 33 for VF 16 For instruction: %v0 = load i8, ptr %in0, align 1
+; AVX512BW:  LV: Found an estimated cost of 80 for VF 32 For instruction: %v0 = load i8, ptr %in0, align 1
+; AVX512BW:  LV: Found an estimated cost of 238 for VF 64 For instruction: %v0 = load i8, ptr %in0, align 1
 ;
 entry:
   br label %for.body
diff --git a/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-i8-stride-5.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-i8-stride-5.ll
index ef3c80c27550a..5207c5b2019fd 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-i8-stride-5.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-i8-stride-5.ll
@@ -1,9 +1,9 @@
 ; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --filter "LV: Found an estimated cost of [0-9]+ for VF [0-9]+ For instruction:\s*%v. = load i8, ptr %in."
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse2 --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=SSE2
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx  --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX1
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2 --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX2
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512vl --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX512DQ
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512vl,+avx512bw --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX512BW
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse2 --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=SSE2
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx  --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX1
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2 --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX2
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512vl --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX512DQ
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512vl,+avx512bw --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX512BW
 ; REQUIRES: asserts
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
diff --git a/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-i8-stride-6.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-i8-stride-6.ll
index ed8bc84e771f8..fd267ddd19229 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-i8-stride-6.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-i8-stride-6.ll
@@ -1,9 +1,9 @@
 ; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --filter "LV: Found an estimated cost of [0-9]+ for VF [0-9]+ For instruction:\s*%v. = load i8, ptr %in."
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse2 --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=SSE2
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx  --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX1
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2 --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX2
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512vl --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX512DQ
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512vl,+avx512bw --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX512BW
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse2 --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=SSE2
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx  --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX1
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2 --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX2
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512vl --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX512DQ
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512vl,+avx512bw --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX512BW
 ; REQUIRES: asserts
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
diff --git a/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-i8-stride-7.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-i8-stride-7.ll
index 8e7c316b69b3a..bcf93dc2d005b 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-i8-stride-7.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-i8-stride-7.ll
@@ -1,9 +1,9 @@
 ; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --filter "LV: Found an estimated cost of [0-9]+ for VF [0-9]+ For instruction:\s*%v. = load i8, ptr %in."
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse2 --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=SSE2
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx  --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX1
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2 --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX2
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512vl --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX512DQ
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512vl,+avx512bw --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX512BW
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse2 --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=SSE2
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx  --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX1
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2 --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX2
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512vl --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX512DQ
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512vl,+avx512bw --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX512BW
 ; REQUIRES: asserts
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
diff --git a/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-i8-stride-8.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-i8-stride-8.ll
index 752cc229922be..c03f09e6a003e 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-i8-stride-8.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-i8-stride-8.ll
@@ -1,9 +1,9 @@
 ; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --filter "LV: Found an estimated cost of [0-9]+ for VF [0-9]+ For instruction:\s*%v. = load i8, ptr %in."
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse2 --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=SSE2
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx  --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX1
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2 --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX2
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512vl --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX512DQ
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512vl,+avx512bw --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX512BW
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse2 --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=SSE2
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx  --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX1
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2 --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX2
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512vl --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX512DQ
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512vl,+avx512bw --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX512BW
 ; REQUIRES: asserts
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
diff --git a/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-store-f32-stride-2.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-store-f32-stride-2.ll
index 6076aa274623a..85ab5b85c9958 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-store-f32-stride-2.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-store-f32-stride-2.ll
@@ -1,8 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --filter "LV: Found an estimated cost of [0-9]+ for VF [0-9]+ For instruction:\s*store float %v1, ptr %out1"
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse2 --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=SSE2
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx  --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX1
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2 --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX2
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512vl --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX512
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse2 --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=SSE2
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx  --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX1
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2 --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX2
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512vl --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX512
 ; REQUIRES: asserts
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
@@ -14,35 +14,39 @@ target triple = "x86_64-unknown-linux-gnu"
 define void @test() {
 ; SSE2-LABEL: 'test'
 ; SSE2:  LV: Found an estimated cost of 1 for VF 1 For instruction: store float %v1, ptr %out1, align 4
-; SSE2:  Cost of 2 for VF 2: INTERLEAVE-GROUP with factor 2 at <badref>, ir<%out0>
-; SSE2:  Cost of 14 for VF 4: INTERLEAVE-GROUP with factor 2 at <badref>, ir<%out0>
-; SSE2:  Cost of 28 for VF 8: INTERLEAVE-GROUP with factor 2 at <badref>, ir<%out0>
-; SSE2:  Cost of 56 for VF 16: INTERLEAVE-GROUP with factor 2 at <badref>, ir<%out0>
+; SSE2:  LV: Found an estimated cost of 1 for VF 1 For instruction: store float %v1, ptr %out1, align 4
+; SSE2:  LV: Found an estimated cost of 2 for VF 2 For instruction: store float %v1, ptr %out1, align 4
+; SSE2:  LV: Found an estimated cost of 14 for VF 4 For instruction: store float %v1, ptr %out1, align 4
+; SSE2:  LV: Found an estimated cost of 28 for VF 8 For instruction: store float %v1, ptr %out1, align 4
+; SSE2:  LV: Found an estimated cost of 56 for VF 16 For instruction: store float %v1, ptr %out1, align 4
 ;
 ; AVX1-LABEL: 'test'
 ; AVX1:  LV: Found an estimated cost of 1 for VF 1 For instruction: store float %v1, ptr %out1, align 4
-; AVX1:  Cost of 2 for VF 2: INTERLEAVE-GROUP with factor 2 at <badref>, ir<%out0>
-; AVX1:  Cost of 14 for VF 4: INTERLEAVE-GROUP with factor 2 at <badref>, ir<%out0>
-; AVX1:  Cost of 30 for VF 8: INTERLEAVE-GROUP with factor 2 at <badref>, ir<%out0>
-; AVX1:  Cost of 60 for VF 16: INTERLEAVE-GROUP with factor 2 at <badref>, ir<%out0>
-; AVX1:  Cost of 120 for VF 32: INTERLEAVE-GROUP with factor 2 at <badref>, ir<%out0>
+; AVX1:  LV: Found an estimated cost of 1 for VF 1 For instruction: store float %v1, ptr %out1, align 4
+; AVX1:  LV: Found an estimated cost of 2 for VF 2 For instruction: store float %v1, ptr %out1, align 4
+; AVX1:  LV: Found an estimated cost of 14 for VF 4 For instruction: store float %v1, ptr %out1, align 4
+; AVX1:  LV: Found an estimated cost of 30 for VF 8 For instruction: store float %v1, ptr %out1, align 4
+; AVX1:  LV: Found an estimated cost of 60 for VF 16 For instruction: store float %v1, ptr %out1, align 4
+; AVX1:  LV: Found an estimated cost of 120 for VF 32 For instruction: store float %v1, ptr %out1, align 4
 ;
 ; AVX2-LABEL: 'test'
 ; AVX2:  LV: Found an estimated cost of 1 for VF 1 For instruction: store float %v1, ptr %out1, align 4
-; AVX2:  Cost of 2 for VF 2: INTERLEAVE-GROUP with factor 2 at <badref>, ir<%out0>
-; AVX2:  Cost of 3 for VF 4: INTERLEAVE-GROUP with factor 2 at <badref>, ir<%out0>
-; AVX2:  Cost of 6 for VF 8: INTERLEAVE-GROUP with factor 2 at <badref>, ir<%out0>
-; AVX2:  Cost of 12 for VF 16: INTERLEAVE-GROUP with factor 2 at <badref>, ir<%out0>
-; AVX2:  Cost of 24 for VF 32: INTERLEAVE-GROUP with factor 2 at <badref>, ir<%out0>
+; AVX2:  LV: Found an estimated cost of 1 for VF 1 For instruction: store float %v1, ptr %out1, align 4
+; AVX2:  LV: Found an estimated cost of 2 for VF 2 For instruction: store float %v1, ptr %out1, align 4
+; AVX2:  LV: Found an estimated cost of 3 for VF 4 For instruction: store float %v1, ptr %out1, align 4
+; AVX2:  LV: Found an estimated cost of 6 for VF 8 For instruction: store float %v1, ptr %out1, align 4
+; AVX2:  LV: Found an estimated cost of 12 for VF 16 For instruction: store float %v1, ptr %out1, align 4
+; AVX2:  LV: Found an estimated cost of 24 for VF 32 For instruction: store float %v1, ptr %out1, align 4
 ;
 ; AVX512-LABEL: 'test'
 ; AVX512:  LV: Found an estimated cost of 1 for VF 1 For instruction: store float %v1, ptr %out1, align 4
-; AVX512:  Cost of 2 for VF 2: INTERLEAVE-GROUP with factor 2 at <badref>, ir<%out0>
-; AVX512:  Cost of 2 for VF 4: INTERLEAVE-GROUP with factor 2 at <badref>, ir<%out0>
-; AVX512:  Cost of 2 for VF 8: INTERLEAVE-GROUP with factor 2 at <badref>, ir<%out0>
-; AVX512:  Cost of 5 for VF 16: INTERLEAVE-GROUP with factor 2 at <badref>, ir<%out0>
-; AVX512:  Cost of 10 for VF 32: INTERLEAVE-GROUP with factor 2 at <badref>, ir<%out0>
-; AVX512:  Cost of 20 for VF 64: INTERLEAVE-GROUP with factor 2 at <badref>, ir<%out0>
+; AVX512:  LV: Found an estimated cost of 1 for VF 1 For instruction: store float %v1, ptr %out1, align 4
+; AVX512:  LV: Found an estimated cost of 2 for VF 2 For instruction: store float %v1, ptr %out1, align 4
+; AVX512:  LV: Found an estimated cost of 2 for VF 4 For instruction: store float %v1, ptr %out1, align 4
+; AVX512:  LV: Found an estimated cost of 2 for VF 8 For instruction: store float %v1, ptr %out1, align 4
+; AVX512:  LV: Found an estimated cost of 5 for VF 16 For instruction: store float %v1, ptr %out1, align 4
+; AVX512:  LV: Found an estimated cost of 10 for VF 32 For instruction: store float %v1, ptr %out1, align 4
+; AVX512:  LV: Found an estimated cost of 20 for VF 64 For instruction: store float %v1, ptr %out1, align 4
 ;
 entry:
   br label %for.body
diff --git a/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-store-f32-stride-3.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-store-f32-stride-3.ll
index ce0289998100b..77abfc65a3e59 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-store-f32-stride-3.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-store-f32-stride-3.ll
@@ -1,8 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --filter "LV: Found an estimated cost of [0-9]+ for VF [0-9]+ For instruction:\s*store float %v2, ptr %out2"
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse2 --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=SSE2
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx  --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX1
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2 --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX2
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512vl --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX512
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse2 --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=SSE2
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx  --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX1
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2 --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX2
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512vl --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX512
 ; REQUIRES: asserts
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
@@ -14,35 +14,39 @@ target triple = "x86_64-unknown-linux-gnu"
 define void @test() {
 ; SSE2-LABEL: 'test'
 ; SSE2:  LV: Found an estimated cost of 1 for VF 1 For instruction: store float %v2, ptr %out2, align 4
-; SSE2:  Cost of 11 for VF 2: INTERLEAVE-GROUP with factor 3 at <badref>, ir<%out0>
-; SSE2:  Cost of 24 for VF 4: INTERLEAVE-GROUP with factor 3 at <badref>, ir<%out0>
-; SSE2:  Cost of 48 for VF 8: INTERLEAVE-GROUP with factor 3 at <badref>, ir<%out0>
-; SSE2:  Cost of 96 for VF 16: INTERLEAVE-GROUP with factor 3 at <badref>, ir<%out0>
+; SSE2:  LV: Found an estimated cost of 1 for VF 1 For instruction: store float %v2, ptr %out2, align 4
+; SSE2:  LV: Found an estimated cost of 11 for VF 2 For instruction: store float %v2, ptr %out2, align 4
+; SSE2:  LV: Found an estimated cost of 24 for VF 4 For instruction: store float %v2, ptr %out2, align 4
+; SSE2:  LV: Found an estimated cost of 48 for VF 8 For instruction: store float %v2, ptr %out2, align 4
+; SSE2:  LV: Found an estimated cost of 96 for VF 16 For instruction: store float %v2, ptr %out2, align 4
 ;
 ; AVX1-LABEL: 'test'
 ; AVX1:  LV: Found an estimated cost of 1 for VF 1 For instruction: store float %v2, ptr %out2, align 4
-; AVX1:  Cost of 12 for VF 2: INTERLEAVE-GROUP with factor 3 at <badref>, ir<%out0>
-; AVX1:  Cost of 22 for VF 4: INTERLEAVE-GROUP with factor 3 at <badref>, ir<%out0>
-; AVX1:  Cost of 45 for VF 8: INTERLEAVE-GROUP with factor 3 at <badref>, ir<%out0>
-; AVX1:  Cost of 90 for VF 16: INTERLEAVE-GROUP with factor 3 at <badref>, ir<%out0>
-; AVX1:  Cost of 180 for VF 32: INTERLEAVE-GROUP with factor 3 at <badref>, ir<%out0>
+; AVX1:  LV: Found an estimated cost of 1 for VF 1 For instruction: store float %v2, ptr %out2, align 4
+; AVX1:  LV: Found an estimated cost of 12 for VF 2 For instruction: store float %v2, ptr %out2, align 4
+; AVX1:  LV: Found an estimated cost of 22 for VF 4 For instruction: store float %v2, ptr %out2, align 4
+; AVX1:  LV: Found an estimated cost of 45 for VF 8 For instruction: store float %v2, ptr %out2, align 4
+; AVX1:  LV: Found an estimated cost of 90 for VF 16 For instruction: store float %v2, ptr %out2, align 4
+; AVX1:  LV: Found an estimated cost of 180 for VF 32 For instruction: store float %v2, ptr %out2, align 4
 ;
 ; AVX2-LABEL: 'test'
 ; AVX2:  LV: Found an estimated cost of 1 for VF 1 For instruction: store float %v2, ptr %out2, align 4
-; AVX2:  Cost of 7 for VF 2: INTERLEAVE-GROUP with factor 3 at <badref>, ir<%out0>
-; AVX2:  Cost of 7 for VF 4: INTERLEAVE-GROUP with factor 3 at <badref>, ir<%out0>
-; AVX2:  Cost of 14 for VF 8: INTERLEAVE-GROUP with factor 3 at <badref>, ir<%out0>
-; AVX2:  Cost of 28 for VF 16: INTERLEAVE-GROUP with factor 3 at <badref>, ir<%out0>
-; AVX2:  Cost of 60 for VF 32: INTERLEAVE-GROUP with factor 3 at <badref>, ir<%out0>
+; AVX2:  LV: Found an estimated cost of 1 for VF 1 For instruction: store float %v2, ptr %out2, align 4
+; AVX2:  LV: Found an estimated cost of 7 for VF 2 For instruction: store float %v2, ptr %out2, align 4
+; AVX2:  LV: Found an estimated cost of 7 for VF 4 For instruction: store float %v2, ptr %out2, align 4
+; AVX2:  LV: Found an estimated cost of 14 for VF 8 For instruction: store float %v2, ptr %out2, align 4
+; AVX2:  LV: Found an estimated cost of 28 for VF 16 For instruction: store float %v2, ptr %out2, align 4
+; AVX2:  LV: Found an estimated cost of 60 for VF 32 For instruction: store float %v2, ptr %out2, align 4
 ;
 ; AVX512-LABEL: 'test'
 ; AVX512:  LV: Found an estimated cost of 1 for VF 1 For instruction: store float %v2, ptr %out2, align 4
-; AVX512:  Cost of 4 for VF 2: INTERLEAVE-GROUP with factor 3 at <badref>, ir<%out0>
-; AVX512:  Cost of 4 for VF 4: INTERLEAVE-GROUP with factor 3 at <badref>, ir<%out0>
-; AVX512:  Cost of 8 for VF 8: INTERLEAVE-GROUP with factor 3 at <badref>, ir<%out0>
-; AVX512:  Cost of 12 for VF 16: INTERLEAVE-GROUP with factor 3 at <badref>, ir<%out0>
-; AVX512:  Cost of 24 for VF 32: INTERLEAVE-GROUP with factor 3 at <badref>, ir<%out0>
-; AVX512:  Cost of 48 for VF 64: INTERLEAVE-GROUP with factor 3 at <badref>, ir<%out0>
+; AVX512:  LV: Found an estimated cost of 1 for VF 1 For instruction: store float %v2, ptr %out2, align 4
+; AVX512:  LV: Found an estimated cost of 4 for VF 2 For instruction: store float %v2, ptr %out2, align 4
+; AVX512:  LV: Found an estimated cost of 4 for VF 4 For instruction: store float %v2, ptr %out2, align 4
+; AVX512:  LV: Found an estimated cost of 8 for VF 8 For instruction: store float %v2, ptr %out2, align 4
+; AVX512:  LV: Found an estimated cost of 12 for VF 16 For instruction: store float %v2, ptr %out2, align 4
+; AVX512:  LV: Found an estimated cost of 24 for VF 32 For instruction: store float %v2, ptr %out2, align 4
+; AVX512:  LV: Found an estimated cost of 48 for VF 64 For instruction: store float %v2, ptr %out2, align 4
 ;
 entry:
   br label %for.body
diff --git a/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-store-f32-stride-4.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-store-f32-stride-4.ll
index e0fa19298c577..699c2eb63cb31 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-store-f32-stride-4.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-store-f32-stride-4.ll
@@ -1,8 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --filter "LV: Found an estimated cost of [0-9]+ for VF [0-9]+ For instruction:\s*store float %v3, ptr %out3"
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse2 --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=SSE2
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx  --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX1
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2 --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX2
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512vl --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX512
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse2 --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=SSE2
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx  --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX1
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2 --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX2
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512vl --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX512
 ; REQUIRES: asserts
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
@@ -14,35 +14,39 @@ target triple = "x86_64-unknown-linux-gnu"
 define void @test() {
 ; SSE2-LABEL: 'test'
 ; SSE2:  LV: Found an estimated cost of 1 for VF 1 For instruction: store float %v3, ptr %out3, align 4
-; SSE2:  Cost of 12 for VF 2: INTERLEAVE-GROUP with factor 4 at <badref>, ir<%out0>
-; SSE2:  Cost of 28 for VF 4: INTERLEAVE-GROUP with factor 4 at <badref>, ir<%out0>
-; SSE2:  Cost of 56 for VF 8: INTERLEAVE-GROUP with factor 4 at <badref>, ir<%out0>
-; SSE2:  Cost of 112 for VF 16: INTERLEAVE-GROUP with factor 4 at <badref>, ir<%out0>
+; SSE2:  LV: Found an estimated cost of 1 for VF 1 For instruction: store float %v3, ptr %out3, align 4
+; SSE2:  LV: Found an estimated cost of 12 for VF 2 For instruction: store float %v3, ptr %out3, align 4
+; SSE2:  LV: Found an estimated cost of 28 for VF 4 For instruction: store float %v3, ptr %out3, align 4
+; SSE2:  LV: Found an estimated cost of 56 for VF 8 For instruction: store float %v3, ptr %out3, align 4
+; SSE2:  LV: Found an estimated cost of 112 for VF 16 For instruction: store float %v3, ptr %out3, align 4
 ;
 ; AVX1-LABEL: 'test'
 ; AVX1:  LV: Found an estimated cost of 1 for VF 1 For instruction: store float %v3, ptr %out3, align 4
-; AVX1:  Cost of 12 for VF 2: INTERLEAVE-GROUP with factor 4 at <badref>, ir<%out0>
-; AVX1:  Cost of 28 for VF 4: INTERLEAVE-GROUP with factor 4 at <badref>, ir<%out0>
-; AVX1:  Cost of 60 for VF 8: INTERLEAVE-GROUP with factor 4 at <badref>, ir<%out0>
-; AVX1:  Cost of 120 for VF 16: INTERLEAVE-GROUP with factor 4 at <badref>, ir<%out0>
-; AVX1:  Cost of 240 for VF 32: INTERLEAVE-GROUP with factor 4 at <badref>, ir<%out0>
+; AVX1:  LV: Found an estimated cost of 1 for VF 1 For instruction: store float %v3, ptr %out3, align 4
+; AVX1:  LV: Found an estimated cost of 12 for VF 2 For instruction: store float %v3, ptr %out3, align 4
+; AVX1:  LV: Found an estimated cost of 28 for VF 4 For instruction: store float %v3, ptr %out3, align 4
+; AVX1:  LV: Found an estimated cost of 60 for VF 8 For instruction: store float %v3, ptr %out3, align 4
+; AVX1:  LV: Found an estimated cost of 120 for VF 16 For instruction: store float %v3, ptr %out3, align 4
+; AVX1:  LV: Found an estimated cost of 240 for VF 32 For instruction: store float %v3, ptr %out3, align 4
 ;
 ; AVX2-LABEL: 'test'
 ; AVX2:  LV: Found an estimated cost of 1 for VF 1 For instruction: store float %v3, ptr %out3, align 4
-; AVX2:  Cost of 6 for VF 2: INTERLEAVE-GROUP with factor 4 at <badref>, ir<%out0>
-; AVX2:  Cost of 8 for VF 4: INTERLEAVE-GROUP with factor 4 at <badref>, ir<%out0>
-; AVX2:  Cost of 20 for VF 8: INTERLEAVE-GROUP with factor 4 at <badref>, ir<%out0>
-; AVX2:  Cost of 40 for VF 16: INTERLEAVE-GROUP with factor 4 at <badref>, ir<%out0>
-; AVX2:  Cost of 80 for VF 32: INTERLEAVE-GROUP with factor 4 at <badref>, ir<%out0>
+; AVX2:  LV: Found an estimated cost of 1 for VF 1 For instruction: store float %v3, ptr %out3, align 4
+; AVX2:  LV: Found an estimated cost of 6 for VF 2 For instruction: store float %v3, ptr %out3, align 4
+; AVX2:  LV: Found an estimated cost of 8 for VF 4 For instruction: store float %v3, ptr %out3, align 4
+; AVX2:  LV: Found an estimated cost of 20 for VF 8 For instruction: store float %v3, ptr %out3, align 4
+; AVX2:  LV: Found an estimated cost of 40 for VF 16 For instruction: store float %v3, ptr %out3, align 4
+; AVX2:  LV: Found an estimated cost of 80 for VF 32 For instruction: store float %v3, ptr %out3, align 4
 ;
 ; AVX512-LABEL: 'test'
 ; AVX512:  LV: Found an estimated cost of 1 for VF 1 For instruction: store float %v3, ptr %out3, align 4
-; AVX512:  Cost of 5 for VF 2: INTERLEAVE-GROUP with factor 4 at <badref>, ir<%out0>
-; AVX512:  Cost of 5 for VF 4: INTERLEAVE-GROUP with factor 4 at <badref>, ir<%out0>
-; AVX512:  Cost of 11 for VF 8: INTERLEAVE-GROUP with factor 4 at <badref>, ir<%out0>
-; AVX512:  Cost of 22 for VF 16: INTERLEAVE-GROUP with factor 4 at <badref>, ir<%out0>
-; AVX512:  Cost of 44 for VF 32: INTERLEAVE-GROUP with factor 4 at <badref>, ir<%out0>
-; AVX512:  Cost of 88 for VF 64: INTERLEAVE-GROUP with factor 4 at <badref>, ir<%out0>
+; AVX512:  LV: Found an estimated cost of 1 for VF 1 For instruction: store float %v3, ptr %out3, align 4
+; AVX512:  LV: Found an estimated cost of 5 for VF 2 For instruction: store float %v3, ptr %out3, align 4
+; AVX512:  LV: Found an estimated cost of 5 for VF 4 For instruction: store float %v3, ptr %out3, align 4
+; AVX512:  LV: Found an estimated cost of 11 for VF 8 For instruction: store float %v3, ptr %out3, align 4
+; AVX512:  LV: Found an estimated cost of 22 for VF 16 For instruction: store float %v3, ptr %out3, align 4
+; AVX512:  LV: Found an estimated cost of 44 for VF 32 For instruction: store float %v3, ptr %out3, align 4
+; AVX512:  LV: Found an estimated cost of 88 for VF 64 For instruction: store float %v3, ptr %out3, align 4
 ;
 entry:
   br label %for.body
diff --git a/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-store-f32-stride-5.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-store-f32-stride-5.ll
index 940864a59a93a..86758b5a24fe9 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-store-f32-stride-5.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-store-f32-stride-5.ll
@@ -1,8 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --filter "LV: Found an estimated cost of [0-9]+ for VF [0-9]+ For instruction:\s*store float %v4, ptr %out4"
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse2 --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=SSE2
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx  --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX1
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2 --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX2
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512vl --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX512
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse2 --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=SSE2
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx  --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX1
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2 --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX2
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512vl --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX512
 ; REQUIRES: asserts
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
@@ -14,32 +14,36 @@ target triple = "x86_64-unknown-linux-gnu"
 define void @test() {
 ; SSE2-LABEL: 'test'
 ; SSE2:  LV: Found an estimated cost of 1 for VF 1 For instruction: store float %v4, ptr %out4, align 4
-; SSE2:  Cost of 20 for VF 2: INTERLEAVE-GROUP with factor 5 at <badref>, ir<%out0>
-; SSE2:  Cost of 44 for VF 4: INTERLEAVE-GROUP with factor 5 at <badref>, ir<%out0>
-; SSE2:  Cost of 88 for VF 8: INTERLEAVE-GROUP with factor 5 at <badref>, ir<%out0>
+; SSE2:  LV: Found an estimated cost of 1 for VF 1 For instruction: store float %v4, ptr %out4, align 4
+; SSE2:  LV: Found an estimated cost of 20 for VF 2 For instruction: store float %v4, ptr %out4, align 4
+; SSE2:  LV: Found an estimated cost of 44 for VF 4 For instruction: store float %v4, ptr %out4, align 4
+; SSE2:  LV: Found an estimated cost of 88 for VF 8 For instruction: store float %v4, ptr %out4, align 4
 ;
 ; AVX1-LABEL: 'test'
 ; AVX1:  LV: Found an estimated cost of 1 for VF 1 For instruction: store float %v4, ptr %out4, align 4
-; AVX1:  Cost of 16 for VF 2: INTERLEAVE-GROUP with factor 5 at <badref>, ir<%out0>
-; AVX1:  Cost of 36 for VF 4: INTERLEAVE-GROUP with factor 5 at <badref>, ir<%out0>
-; AVX1:  Cost of 75 for VF 8: INTERLEAVE-GROUP with factor 5 at <badref>, ir<%out0>
-; AVX1:  Cost of 150 for VF 16: INTERLEAVE-GROUP with factor 5 at <badref>, ir<%out0>
+; AVX1:  LV: Found an estimated cost of 1 for VF 1 For instruction: store float %v4, ptr %out4, align 4
+; AVX1:  LV: Found an estimated cost of 16 for VF 2 For instruction: store float %v4, ptr %out4, align 4
+; AVX1:  LV: Found an estimated cost of 36 for VF 4 For instruction: store float %v4, ptr %out4, align 4
+; AVX1:  LV: Found an estimated cost of 75 for VF 8 For instruction: store float %v4, ptr %out4, align 4
+; AVX1:  LV: Found an estimated cost of 150 for VF 16 For instruction: store float %v4, ptr %out4, align 4
 ;
 ; AVX2-LABEL: 'test'
 ; AVX2:  LV: Found an estimated cost of 1 for VF 1 For instruction: store float %v4, ptr %out4, align 4
-; AVX2:  Cost of 16 for VF 2: INTERLEAVE-GROUP with factor 5 at <badref>, ir<%out0>
-; AVX2:  Cost of 36 for VF 4: INTERLEAVE-GROUP with factor 5 at <badref>, ir<%out0>
-; AVX2:  Cost of 75 for VF 8: INTERLEAVE-GROUP with factor 5 at <badref>, ir<%out0>
-; AVX2:  Cost of 150 for VF 16: INTERLEAVE-GROUP with factor 5 at <badref>, ir<%out0>
+; AVX2:  LV: Found an estimated cost of 1 for VF 1 For instruction: store float %v4, ptr %out4, align 4
+; AVX2:  LV: Found an estimated cost of 16 for VF 2 For instruction: store float %v4, ptr %out4, align 4
+; AVX2:  LV: Found an estimated cost of 36 for VF 4 For instruction: store float %v4, ptr %out4, align 4
+; AVX2:  LV: Found an estimated cost of 75 for VF 8 For instruction: store float %v4, ptr %out4, align 4
+; AVX2:  LV: Found an estimated cost of 150 for VF 16 For instruction: store float %v4, ptr %out4, align 4
 ;
 ; AVX512-LABEL: 'test'
 ; AVX512:  LV: Found an estimated cost of 1 for VF 1 For instruction: store float %v4, ptr %out4, align 4
-; AVX512:  Cost of 7 for VF 2: INTERLEAVE-GROUP with factor 5 at <badref>, ir<%out0>
-; AVX512:  Cost of 14 for VF 4: INTERLEAVE-GROUP with factor 5 at <badref>, ir<%out0>
-; AVX512:  Cost of 21 for VF 8: INTERLEAVE-GROUP with factor 5 at <badref>, ir<%out0>
-; AVX512:  Cost of 35 for VF 16: INTERLEAVE-GROUP with factor 5 at <badref>, ir<%out0>
-; AVX512:  Cost of 70 for VF 32: INTERLEAVE-GROUP with factor 5 at <badref>, ir<%out0>
-; AVX512:  Cost of 140 for VF 64: INTERLEAVE-GROUP with factor 5 at <badref>, ir<%out0>
+; AVX512:  LV: Found an estimated cost of 1 for VF 1 For instruction: store float %v4, ptr %out4, align 4
+; AVX512:  LV: Found an estimated cost of 7 for VF 2 For instruction: store float %v4, ptr %out4, align 4
+; AVX512:  LV: Found an estimated cost of 14 for VF 4 For instruction: store float %v4, ptr %out4, align 4
+; AVX512:  LV: Found an estimated cost of 21 for VF 8 For instruction: store float %v4, ptr %out4, align 4
+; AVX512:  LV: Found an estimated cost of 35 for VF 16 For instruction: store float %v4, ptr %out4, align 4
+; AVX512:  LV: Found an estimated cost of 70 for VF 32 For instruction: store float %v4, ptr %out4, align 4
+; AVX512:  LV: Found an estimated cost of 140 for VF 64 For instruction: store float %v4, ptr %out4, align 4
 ;
 entry:
   br label %for.body
diff --git a/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-store-f32-stride-6.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-store-f32-stride-6.ll
index 3d947e13b3732..53c8f59491e76 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-store-f32-stride-6.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-store-f32-stride-6.ll
@@ -1,8 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --filter "LV: Found an estimated cost of [0-9]+ for VF [0-9]+ For instruction:\s*store float %v5, ptr %out5"
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse2 --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=SSE2
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx  --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX1
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2 --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX2
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512vl --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX512
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse2 --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=SSE2
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx  --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX1
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2 --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX2
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512vl --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX512
 ; REQUIRES: asserts
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
@@ -14,32 +14,36 @@ target triple = "x86_64-unknown-linux-gnu"
 define void @test() {
 ; SSE2-LABEL: 'test'
 ; SSE2:  LV: Found an estimated cost of 1 for VF 1 For instruction: store float %v5, ptr %out5, align 4
-; SSE2:  Cost of 21 for VF 2: INTERLEAVE-GROUP with factor 6 at <badref>, ir<%out0>
-; SSE2:  Cost of 48 for VF 4: INTERLEAVE-GROUP with factor 6 at <badref>, ir<%out0>
-; SSE2:  Cost of 96 for VF 8: INTERLEAVE-GROUP with factor 6 at <badref>, ir<%out0>
+; SSE2:  LV: Found an estimated cost of 1 for VF 1 For instruction: store float %v5, ptr %out5, align 4
+; SSE2:  LV: Found an estimated cost of 21 for VF 2 For instruction: store float %v5, ptr %out5, align 4
+; SSE2:  LV: Found an estimated cost of 48 for VF 4 For instruction: store float %v5, ptr %out5, align 4
+; SSE2:  LV: Found an estimated cost of 96 for VF 8 For instruction: store float %v5, ptr %out5, align 4
 ;
 ; AVX1-LABEL: 'test'
 ; AVX1:  LV: Found an estimated cost of 1 for VF 1 For instruction: store float %v5, ptr %out5, align 4
-; AVX1:  Cost of 19 for VF 2: INTERLEAVE-GROUP with factor 6 at <badref>, ir<%out0>
-; AVX1:  Cost of 42 for VF 4: INTERLEAVE-GROUP with factor 6 at <badref>, ir<%out0>
-; AVX1:  Cost of 90 for VF 8: INTERLEAVE-GROUP with factor 6 at <badref>, ir<%out0>
-; AVX1:  Cost of 180 for VF 16: INTERLEAVE-GROUP with factor 6 at <badref>, ir<%out0>
+; AVX1:  LV: Found an estimated cost of 1 for VF 1 For instruction: store float %v5, ptr %out5, align 4
+; AVX1:  LV: Found an estimated cost of 19 for VF 2 For instruction: store float %v5, ptr %out5, align 4
+; AVX1:  LV: Found an estimated cost of 42 for VF 4 For instruction: store float %v5, ptr %out5, align 4
+; AVX1:  LV: Found an estimated cost of 90 for VF 8 For instruction: store float %v5, ptr %out5, align 4
+; AVX1:  LV: Found an estimated cost of 180 for VF 16 For instruction: store float %v5, ptr %out5, align 4
 ;
 ; AVX2-LABEL: 'test'
 ; AVX2:  LV: Found an estimated cost of 1 for VF 1 For instruction: store float %v5, ptr %out5, align 4
-; AVX2:  Cost of 11 for VF 2: INTERLEAVE-GROUP with factor 6 at <badref>, ir<%out0>
-; AVX2:  Cost of 15 for VF 4: INTERLEAVE-GROUP with factor 6 at <badref>, ir<%out0>
-; AVX2:  Cost of 39 for VF 8: INTERLEAVE-GROUP with factor 6 at <badref>, ir<%out0>
-; AVX2:  Cost of 78 for VF 16: INTERLEAVE-GROUP with factor 6 at <badref>, ir<%out0>
+; AVX2:  LV: Found an estimated cost of 1 for VF 1 For instruction: store float %v5, ptr %out5, align 4
+; AVX2:  LV: Found an estimated cost of 11 for VF 2 For instruction: store float %v5, ptr %out5, align 4
+; AVX2:  LV: Found an estimated cost of 15 for VF 4 For instruction: store float %v5, ptr %out5, align 4
+; AVX2:  LV: Found an estimated cost of 39 for VF 8 For instruction: store float %v5, ptr %out5, align 4
+; AVX2:  LV: Found an estimated cost of 78 for VF 16 For instruction: store float %v5, ptr %out5, align 4
 ;
 ; AVX512-LABEL: 'test'
 ; AVX512:  LV: Found an estimated cost of 1 for VF 1 For instruction: store float %v5, ptr %out5, align 4
-; AVX512:  Cost of 8 for VF 2: INTERLEAVE-GROUP with factor 6 at <badref>, ir<%out0>
-; AVX512:  Cost of 17 for VF 4: INTERLEAVE-GROUP with factor 6 at <badref>, ir<%out0>
-; AVX512:  Cost of 25 for VF 8: INTERLEAVE-GROUP with factor 6 at <badref>, ir<%out0>
-; AVX512:  Cost of 51 for VF 16: INTERLEAVE-GROUP with factor 6 at <badref>, ir<%out0>
-; AVX512:  Cost of 102 for VF 32: INTERLEAVE-GROUP with factor 6 at <badref>, ir<%out0>
-; AVX512:  Cost of 204 for VF 64: INTERLEAVE-GROUP with factor 6 at <badref>, ir<%out0>
+; AVX512:  LV: Found an estimated cost of 1 for VF 1 For instruction: store float %v5, ptr %out5, align 4
+; AVX512:  LV: Found an estimated cost of 8 for VF 2 For instruction: store float %v5, ptr %out5, align 4
+; AVX512:  LV: Found an estimated cost of 17 for VF 4 For instruction: store float %v5, ptr %out5, align 4
+; AVX512:  LV: Found an estimated cost of 25 for VF 8 For instruction: store float %v5, ptr %out5, align 4
+; AVX512:  LV: Found an estimated cost of 51 for VF 16 For instruction: store float %v5, ptr %out5, align 4
+; AVX512:  LV: Found an estimated cost of 102 for VF 32 For instruction: store float %v5, ptr %out5, align 4
+; AVX512:  LV: Found an estimated cost of 204 for VF 64 For instruction: store float %v5, ptr %out5, align 4
 ;
 entry:
   br label %for.body
diff --git a/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-store-f32-stride-7.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-store-f32-stride-7.ll
index ffa4cab905cea..244bceb69f97d 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-store-f32-stride-7.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-store-f32-stride-7.ll
@@ -1,8 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --filter "LV: Found an estimated cost of [0-9]+ for VF [0-9]+ For instruction:\s*store float %v6, ptr %out6"
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse2 --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=SSE2
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx  --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX1
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2 --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX2
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512vl --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX512
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse2 --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=SSE2
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx  --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX1
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2 --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX2
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512vl --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX512
 ; REQUIRES: asserts
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
@@ -14,31 +14,35 @@ target triple = "x86_64-unknown-linux-gnu"
 define void @test() {
 ; SSE2-LABEL: 'test'
 ; SSE2:  LV: Found an estimated cost of 1 for VF 1 For instruction: store float %v6, ptr %out6, align 4
-; SSE2:  Cost of 23 for VF 2: INTERLEAVE-GROUP with factor 7 at <badref>, ir<%out0>
-; SSE2:  Cost of 52 for VF 4: INTERLEAVE-GROUP with factor 7 at <badref>, ir<%out0>
-; SSE2:  Cost of 104 for VF 8: INTERLEAVE-GROUP with factor 7 at <badref>, ir<%out0>
+; SSE2:  LV: Found an estimated cost of 1 for VF 1 For instruction: store float %v6, ptr %out6, align 4
+; SSE2:  LV: Found an estimated cost of 23 for VF 2 For instruction: store float %v6, ptr %out6, align 4
+; SSE2:  LV: Found an estimated cost of 52 for VF 4 For instruction: store float %v6, ptr %out6, align 4
+; SSE2:  LV: Found an estimated cost of 104 for VF 8 For instruction: store float %v6, ptr %out6, align 4
 ;
 ; AVX1-LABEL: 'test'
 ; AVX1:  LV: Found an estimated cost of 1 for VF 1 For instruction: store float %v6, ptr %out6, align 4
-; AVX1:  Cost of 24 for VF 2: INTERLEAVE-GROUP with factor 7 at <badref>, ir<%out0>
-; AVX1:  Cost of 50 for VF 4: INTERLEAVE-GROUP with factor 7 at <badref>, ir<%out0>
-; AVX1:  Cost of 105 for VF 8: INTERLEAVE-GROUP with factor 7 at <badref>, ir<%out0>
-; AVX1:  Cost of 210 for VF 16: INTERLEAVE-GROUP with factor 7 at <badref>, ir<%out0>
+; AVX1:  LV: Found an estimated cost of 1 for VF 1 For instruction: store float %v6, ptr %out6, align 4
+; AVX1:  LV: Found an estimated cost of 24 for VF 2 For instruction: store float %v6, ptr %out6, align 4
+; AVX1:  LV: Found an estimated cost of 50 for VF 4 For instruction: store float %v6, ptr %out6, align 4
+; AVX1:  LV: Found an estimated cost of 105 for VF 8 For instruction: store float %v6, ptr %out6, align 4
+; AVX1:  LV: Found an estimated cost of 210 for VF 16 For instruction: store float %v6, ptr %out6, align 4
 ;
 ; AVX2-LABEL: 'test'
 ; AVX2:  LV: Found an estimated cost of 1 for VF 1 For instruction: store float %v6, ptr %out6, align 4
-; AVX2:  Cost of 24 for VF 2: INTERLEAVE-GROUP with factor 7 at <badref>, ir<%out0>
-; AVX2:  Cost of 50 for VF 4: INTERLEAVE-GROUP with factor 7 at <badref>, ir<%out0>
-; AVX2:  Cost of 105 for VF 8: INTERLEAVE-GROUP with factor 7 at <badref>, ir<%out0>
-; AVX2:  Cost of 210 for VF 16: INTERLEAVE-GROUP with factor 7 at <badref>, ir<%out0>
+; AVX2:  LV: Found an estimated cost of 1 for VF 1 For instruction: store float %v6, ptr %out6, align 4
+; AVX2:  LV: Found an estimated cost of 24 for VF 2 For instruction: store float %v6, ptr %out6, align 4
+; AVX2:  LV: Found an estimated cost of 50 for VF 4 For instruction: store float %v6, ptr %out6, align 4
+; AVX2:  LV: Found an estimated cost of 105 for VF 8 For instruction: store float %v6, ptr %out6, align 4
+; AVX2:  LV: Found an estimated cost of 210 for VF 16 For instruction: store float %v6, ptr %out6, align 4
 ;
 ; AVX512-LABEL: 'test'
 ; AVX512:  LV: Found an estimated cost of 1 for VF 1 For instruction: store float %v6, ptr %out6, align 4
-; AVX512:  Cost of 10 for VF 2: INTERLEAVE-GROUP with factor 7 at <badref>, ir<%out0>
-; AVX512:  Cost of 20 for VF 4: INTERLEAVE-GROUP with factor 7 at <badref>, ir<%out0>
-; AVX512:  Cost of 40 for VF 8: INTERLEAVE-GROUP with factor 7 at <badref>, ir<%out0>
-; AVX512:  Cost of 70 for VF 16: INTERLEAVE-GROUP with factor 7 at <badref>, ir<%out0>
-; AVX512:  Cost of 140 for VF 32: INTERLEAVE-GROUP with factor 7 at <badref>, ir<%out0>
+; AVX512:  LV: Found an estimated cost of 1 for VF 1 For instruction: store float %v6, ptr %out6, align 4
+; AVX512:  LV: Found an estimated cost of 10 for VF 2 For instruction: store float %v6, ptr %out6, align 4
+; AVX512:  LV: Found an estimated cost of 20 for VF 4 For instruction: store float %v6, ptr %out6, align 4
+; AVX512:  LV: Found an estimated cost of 40 for VF 8 For instruction: store float %v6, ptr %out6, align 4
+; AVX512:  LV: Found an estimated cost of 70 for VF 16 For instruction: store float %v6, ptr %out6, align 4
+; AVX512:  LV: Found an estimated cost of 140 for VF 32 For instruction: store float %v6, ptr %out6, align 4
 ;
 entry:
   br label %for.body
diff --git a/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-store-f32-stride-8.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-store-f32-stride-8.ll
index 2ad37bee35bed..c1fe64e324378 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-store-f32-stride-8.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-store-f32-stride-8.ll
@@ -1,8 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --filter "LV: Found an estimated cost of [0-9]+ for VF [0-9]+ For instruction:\s*store float %v7, ptr %out7"
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse2 --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=SSE2
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx  --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX1
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2 --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX2
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512vl --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX512
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse2 --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=SSE2
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx  --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX1
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2 --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX2
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512vl --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX512
 ; REQUIRES: asserts
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
diff --git a/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-store-f64-stride-2.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-store-f64-stride-2.ll
index 30faeb74644be..dd6094e4a7d5c 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-store-f64-stride-2.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-store-f64-stride-2.ll
@@ -1,8 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --filter "LV: Found an estimated cost of [0-9]+ for VF [0-9]+ For instruction:\s*store double %v1, ptr %out1"
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse2 --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=SSE2
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx  --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX1
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2 --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX2
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512vl --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX512
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse2 --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=SSE2
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx  --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX1
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2 --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX2
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512vl --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX512
 ; REQUIRES: asserts
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
@@ -14,35 +14,39 @@ target triple = "x86_64-unknown-linux-gnu"
 define void @test() {
 ; SSE2-LABEL: 'test'
 ; SSE2:  LV: Found an estimated cost of 1 for VF 1 For instruction: store double %v1, ptr %out1, align 8
-; SSE2:  Cost of 6 for VF 2: INTERLEAVE-GROUP with factor 2 at <badref>, ir<%out0>
-; SSE2:  Cost of 12 for VF 4: INTERLEAVE-GROUP with factor 2 at <badref>, ir<%out0>
-; SSE2:  Cost of 24 for VF 8: INTERLEAVE-GROUP with factor 2 at <badref>, ir<%out0>
-; SSE2:  Cost of 48 for VF 16: INTERLEAVE-GROUP with factor 2 at <badref>, ir<%out0>
+; SSE2:  LV: Found an estimated cost of 1 for VF 1 For instruction: store double %v1, ptr %out1, align 8
+; SSE2:  LV: Found an estimated cost of 6 for VF 2 For instruction: store double %v1, ptr %out1, align 8
+; SSE2:  LV: Found an estimated cost of 12 for VF 4 For instruction: store double %v1, ptr %out1, align 8
+; SSE2:  LV: Found an estimated cost of 24 for VF 8 For instruction: store double %v1, ptr %out1, align 8
+; SSE2:  LV: Found an estimated cost of 48 for VF 16 For instruction: store double %v1, ptr %out1, align 8
 ;
 ; AVX1-LABEL: 'test'
 ; AVX1:  LV: Found an estimated cost of 1 for VF 1 For instruction: store double %v1, ptr %out1, align 8
-; AVX1:  Cost of 6 for VF 2: INTERLEAVE-GROUP with factor 2 at <badref>, ir<%out0>
-; AVX1:  Cost of 14 for VF 4: INTERLEAVE-GROUP with factor 2 at <badref>, ir<%out0>
-; AVX1:  Cost of 28 for VF 8: INTERLEAVE-GROUP with factor 2 at <badref>, ir<%out0>
-; AVX1:  Cost of 56 for VF 16: INTERLEAVE-GROUP with factor 2 at <badref>, ir<%out0>
-; AVX1:  Cost of 112 for VF 32: INTERLEAVE-GROUP with factor 2 at <badref>, ir<%out0>
+; AVX1:  LV: Found an estimated cost of 1 for VF 1 For instruction: store double %v1, ptr %out1, align 8
+; AVX1:  LV: Found an estimated cost of 6 for VF 2 For instruction: store double %v1, ptr %out1, align 8
+; AVX1:  LV: Found an estimated cost of 14 for VF 4 For instruction: store double %v1, ptr %out1, align 8
+; AVX1:  LV: Found an estimated cost of 28 for VF 8 For instruction: store double %v1, ptr %out1, align 8
+; AVX1:  LV: Found an estimated cost of 56 for VF 16 For instruction: store double %v1, ptr %out1, align 8
+; AVX1:  LV: Found an estimated cost of 112 for VF 32 For instruction: store double %v1, ptr %out1, align 8
 ;
 ; AVX2-LABEL: 'test'
 ; AVX2:  LV: Found an estimated cost of 1 for VF 1 For instruction: store double %v1, ptr %out1, align 8
-; AVX2:  Cost of 3 for VF 2: INTERLEAVE-GROUP with factor 2 at <badref>, ir<%out0>
-; AVX2:  Cost of 6 for VF 4: INTERLEAVE-GROUP with factor 2 at <badref>, ir<%out0>
-; AVX2:  Cost of 12 for VF 8: INTERLEAVE-GROUP with factor 2 at <badref>, ir<%out0>
-; AVX2:  Cost of 24 for VF 16: INTERLEAVE-GROUP with factor 2 at <badref>, ir<%out0>
-; AVX2:  Cost of 48 for VF 32: INTERLEAVE-GROUP with factor 2 at <badref>, ir<%out0>
+; AVX2:  LV: Found an estimated cost of 1 for VF 1 For instruction: store double %v1, ptr %out1, align 8
+; AVX2:  LV: Found an estimated cost of 3 for VF 2 For instruction: store double %v1, ptr %out1, align 8
+; AVX2:  LV: Found an estimated cost of 6 for VF 4 For instruction: store double %v1, ptr %out1, align 8
+; AVX2:  LV: Found an estimated cost of 12 for VF 8 For instruction: store double %v1, ptr %out1, align 8
+; AVX2:  LV: Found an estimated cost of 24 for VF 16 For instruction: store double %v1, ptr %out1, align 8
+; AVX2:  LV: Found an estimated cost of 48 for VF 32 For instruction: store double %v1, ptr %out1, align 8
 ;
 ; AVX512-LABEL: 'test'
 ; AVX512:  LV: Found an estimated cost of 1 for VF 1 For instruction: store double %v1, ptr %out1, align 8
-; AVX512:  Cost of 2 for VF 2: INTERLEAVE-GROUP with factor 2 at <badref>, ir<%out0>
-; AVX512:  Cost of 2 for VF 4: INTERLEAVE-GROUP with factor 2 at <badref>, ir<%out0>
-; AVX512:  Cost of 5 for VF 8: INTERLEAVE-GROUP with factor 2 at <badref>, ir<%out0>
-; AVX512:  Cost of 10 for VF 16: INTERLEAVE-GROUP with factor 2 at <badref>, ir<%out0>
-; AVX512:  Cost of 20 for VF 32: INTERLEAVE-GROUP with factor 2 at <badref>, ir<%out0>
-; AVX512:  Cost of 40 for VF 64: INTERLEAVE-GROUP with factor 2 at <badref>, ir<%out0>
+; AVX512:  LV: Found an estimated cost of 1 for VF 1 For instruction: store double %v1, ptr %out1, align 8
+; AVX512:  LV: Found an estimated cost of 2 for VF 2 For instruction: store double %v1, ptr %out1, align 8
+; AVX512:  LV: Found an estimated cost of 2 for VF 4 For instruction: store double %v1, ptr %out1, align 8
+; AVX512:  LV: Found an estimated cost of 5 for VF 8 For instruction: store double %v1, ptr %out1, align 8
+; AVX512:  LV: Found an estimated cost of 10 for VF 16 For instruction: store double %v1, ptr %out1, align 8
+; AVX512:  LV: Found an estimated cost of 20 for VF 32 For instruction: store double %v1, ptr %out1, align 8
+; AVX512:  LV: Found an estimated cost of 40 for VF 64 For instruction: store double %v1, ptr %out1, align 8
 ;
 entry:
   br label %for.body
diff --git a/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-store-f64-stride-3.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-store-f64-stride-3.ll
index a4405db7e8099..3f4216bb3a1ef 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-store-f64-stride-3.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-store-f64-stride-3.ll
@@ -1,8 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --filter "LV: Found an estimated cost of [0-9]+ for VF [0-9]+ For instruction:\s*store double %v2, ptr %out2"
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse2 --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=SSE2
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx  --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX1
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2 --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX2
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512vl --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX512
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse2 --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=SSE2
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx  --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX1
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2 --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX2
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512vl --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX512
 ; REQUIRES: asserts
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
@@ -14,32 +14,36 @@ target triple = "x86_64-unknown-linux-gnu"
 define void @test() {
 ; SSE2-LABEL: 'test'
 ; SSE2:  LV: Found an estimated cost of 1 for VF 1 For instruction: store double %v2, ptr %out2, align 8
-; SSE2:  Cost of 10 for VF 2: INTERLEAVE-GROUP with factor 3 at <badref>, ir<%out0>
-; SSE2:  Cost of 20 for VF 4: INTERLEAVE-GROUP with factor 3 at <badref>, ir<%out0>
-; SSE2:  Cost of 40 for VF 8: INTERLEAVE-GROUP with factor 3 at <badref>, ir<%out0>
+; SSE2:  LV: Found an estimated cost of 1 for VF 1 For instruction: store double %v2, ptr %out2, align 8
+; SSE2:  LV: Found an estimated cost of 10 for VF 2 For instruction: store double %v2, ptr %out2, align 8
+; SSE2:  LV: Found an estimated cost of 20 for VF 4 For instruction: store double %v2, ptr %out2, align 8
+; SSE2:  LV: Found an estimated cost of 40 for VF 8 For instruction: store double %v2, ptr %out2, align 8
 ;
 ; AVX1-LABEL: 'test'
 ; AVX1:  LV: Found an estimated cost of 1 for VF 1 For instruction: store double %v2, ptr %out2, align 8
-; AVX1:  Cost of 11 for VF 2: INTERLEAVE-GROUP with factor 3 at <badref>, ir<%out0>
-; AVX1:  Cost of 24 for VF 4: INTERLEAVE-GROUP with factor 3 at <badref>, ir<%out0>
-; AVX1:  Cost of 48 for VF 8: INTERLEAVE-GROUP with factor 3 at <badref>, ir<%out0>
-; AVX1:  Cost of 96 for VF 16: INTERLEAVE-GROUP with factor 3 at <badref>, ir<%out0>
+; AVX1:  LV: Found an estimated cost of 1 for VF 1 For instruction: store double %v2, ptr %out2, align 8
+; AVX1:  LV: Found an estimated cost of 11 for VF 2 For instruction: store double %v2, ptr %out2, align 8
+; AVX1:  LV: Found an estimated cost of 24 for VF 4 For instruction: store double %v2, ptr %out2, align 8
+; AVX1:  LV: Found an estimated cost of 48 for VF 8 For instruction: store double %v2, ptr %out2, align 8
+; AVX1:  LV: Found an estimated cost of 96 for VF 16 For instruction: store double %v2, ptr %out2, align 8
 ;
 ; AVX2-LABEL: 'test'
 ; AVX2:  LV: Found an estimated cost of 1 for VF 1 For instruction: store double %v2, ptr %out2, align 8
-; AVX2:  Cost of 6 for VF 2: INTERLEAVE-GROUP with factor 3 at <badref>, ir<%out0>
-; AVX2:  Cost of 9 for VF 4: INTERLEAVE-GROUP with factor 3 at <badref>, ir<%out0>
-; AVX2:  Cost of 18 for VF 8: INTERLEAVE-GROUP with factor 3 at <badref>, ir<%out0>
-; AVX2:  Cost of 36 for VF 16: INTERLEAVE-GROUP with factor 3 at <badref>, ir<%out0>
+; AVX2:  LV: Found an estimated cost of 1 for VF 1 For instruction: store double %v2, ptr %out2, align 8
+; AVX2:  LV: Found an estimated cost of 6 for VF 2 For instruction: store double %v2, ptr %out2, align 8
+; AVX2:  LV: Found an estimated cost of 9 for VF 4 For instruction: store double %v2, ptr %out2, align 8
+; AVX2:  LV: Found an estimated cost of 18 for VF 8 For instruction: store double %v2, ptr %out2, align 8
+; AVX2:  LV: Found an estimated cost of 36 for VF 16 For instruction: store double %v2, ptr %out2, align 8
 ;
 ; AVX512-LABEL: 'test'
 ; AVX512:  LV: Found an estimated cost of 1 for VF 1 For instruction: store double %v2, ptr %out2, align 8
-; AVX512:  Cost of 4 for VF 2: INTERLEAVE-GROUP with factor 3 at <badref>, ir<%out0>
-; AVX512:  Cost of 8 for VF 4: INTERLEAVE-GROUP with factor 3 at <badref>, ir<%out0>
-; AVX512:  Cost of 12 for VF 8: INTERLEAVE-GROUP with factor 3 at <badref>, ir<%out0>
-; AVX512:  Cost of 24 for VF 16: INTERLEAVE-GROUP with factor 3 at <badref>, ir<%out0>
-; AVX512:  Cost of 48 for VF 32: INTERLEAVE-GROUP with factor 3 at <badref>, ir<%out0>
-; AVX512:  Cost of 96 for VF 64: INTERLEAVE-GROUP with factor 3 at <badref>, ir<%out0>
+; AVX512:  LV: Found an estimated cost of 1 for VF 1 For instruction: store double %v2, ptr %out2, align 8
+; AVX512:  LV: Found an estimated cost of 4 for VF 2 For instruction: store double %v2, ptr %out2, align 8
+; AVX512:  LV: Found an estimated cost of 8 for VF 4 For instruction: store double %v2, ptr %out2, align 8
+; AVX512:  LV: Found an estimated cost of 12 for VF 8 For instruction: store double %v2, ptr %out2, align 8
+; AVX512:  LV: Found an estimated cost of 24 for VF 16 For instruction: store double %v2, ptr %out2, align 8
+; AVX512:  LV: Found an estimated cost of 48 for VF 32 For instruction: store double %v2, ptr %out2, align 8
+; AVX512:  LV: Found an estimated cost of 96 for VF 64 For instruction: store double %v2, ptr %out2, align 8
 ;
 entry:
   br label %for.body
diff --git a/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-store-f64-stride-4.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-store-f64-stride-4.ll
index aea5871ec8b87..44f9a14424cda 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-store-f64-stride-4.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-store-f64-stride-4.ll
@@ -1,8 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --filter "LV: Found an estimated cost of [0-9]+ for VF [0-9]+ For instruction:\s*store double %v3, ptr %out3"
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse2 --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=SSE2
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx  --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX1
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2 --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX2
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512vl --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX512
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse2 --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=SSE2
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx  --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX1
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2 --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX2
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512vl --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX512
 ; REQUIRES: asserts
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
@@ -14,31 +14,35 @@ target triple = "x86_64-unknown-linux-gnu"
 define void @test() {
 ; SSE2-LABEL: 'test'
 ; SSE2:  LV: Found an estimated cost of 1 for VF 1 For instruction: store double %v3, ptr %out3, align 8
-; SSE2:  Cost of 12 for VF 2: INTERLEAVE-GROUP with factor 4 at <badref>, ir<%out0>
-; SSE2:  Cost of 24 for VF 4: INTERLEAVE-GROUP with factor 4 at <badref>, ir<%out0>
-; SSE2:  Cost of 48 for VF 8: INTERLEAVE-GROUP with factor 4 at <badref>, ir<%out0>
+; SSE2:  LV: Found an estimated cost of 1 for VF 1 For instruction: store double %v3, ptr %out3, align 8
+; SSE2:  LV: Found an estimated cost of 12 for VF 2 For instruction: store double %v3, ptr %out3, align 8
+; SSE2:  LV: Found an estimated cost of 24 for VF 4 For instruction: store double %v3, ptr %out3, align 8
+; SSE2:  LV: Found an estimated cost of 48 for VF 8 For instruction: store double %v3, ptr %out3, align 8
 ;
 ; AVX1-LABEL: 'test'
 ; AVX1:  LV: Found an estimated cost of 1 for VF 1 For instruction: store double %v3, ptr %out3, align 8
-; AVX1:  Cost of 12 for VF 2: INTERLEAVE-GROUP with factor 4 at <badref>, ir<%out0>
-; AVX1:  Cost of 28 for VF 4: INTERLEAVE-GROUP with factor 4 at <badref>, ir<%out0>
-; AVX1:  Cost of 56 for VF 8: INTERLEAVE-GROUP with factor 4 at <badref>, ir<%out0>
-; AVX1:  Cost of 112 for VF 16: INTERLEAVE-GROUP with factor 4 at <badref>, ir<%out0>
+; AVX1:  LV: Found an estimated cost of 1 for VF 1 For instruction: store double %v3, ptr %out3, align 8
+; AVX1:  LV: Found an estimated cost of 12 for VF 2 For instruction: store double %v3, ptr %out3, align 8
+; AVX1:  LV: Found an estimated cost of 28 for VF 4 For instruction: store double %v3, ptr %out3, align 8
+; AVX1:  LV: Found an estimated cost of 56 for VF 8 For instruction: store double %v3, ptr %out3, align 8
+; AVX1:  LV: Found an estimated cost of 112 for VF 16 For instruction: store double %v3, ptr %out3, align 8
 ;
 ; AVX2-LABEL: 'test'
 ; AVX2:  LV: Found an estimated cost of 1 for VF 1 For instruction: store double %v3, ptr %out3, align 8
-; AVX2:  Cost of 8 for VF 2: INTERLEAVE-GROUP with factor 4 at <badref>, ir<%out0>
-; AVX2:  Cost of 12 for VF 4: INTERLEAVE-GROUP with factor 4 at <badref>, ir<%out0>
-; AVX2:  Cost of 28 for VF 8: INTERLEAVE-GROUP with factor 4 at <badref>, ir<%out0>
-; AVX2:  Cost of 56 for VF 16: INTERLEAVE-GROUP with factor 4 at <badref>, ir<%out0>
+; AVX2:  LV: Found an estimated cost of 1 for VF 1 For instruction: store double %v3, ptr %out3, align 8
+; AVX2:  LV: Found an estimated cost of 8 for VF 2 For instruction: store double %v3, ptr %out3, align 8
+; AVX2:  LV: Found an estimated cost of 12 for VF 4 For instruction: store double %v3, ptr %out3, align 8
+; AVX2:  LV: Found an estimated cost of 28 for VF 8 For instruction: store double %v3, ptr %out3, align 8
+; AVX2:  LV: Found an estimated cost of 56 for VF 16 For instruction: store double %v3, ptr %out3, align 8
 ;
 ; AVX512-LABEL: 'test'
 ; AVX512:  LV: Found an estimated cost of 1 for VF 1 For instruction: store double %v3, ptr %out3, align 8
-; AVX512:  Cost of 5 for VF 2: INTERLEAVE-GROUP with factor 4 at <badref>, ir<%out0>
-; AVX512:  Cost of 11 for VF 4: INTERLEAVE-GROUP with factor 4 at <badref>, ir<%out0>
-; AVX512:  Cost of 22 for VF 8: INTERLEAVE-GROUP with factor 4 at <badref>, ir<%out0>
-; AVX512:  Cost of 44 for VF 16: INTERLEAVE-GROUP with factor 4 at <badref>, ir<%out0>
-; AVX512:  Cost of 88 for VF 32: INTERLEAVE-GROUP with factor 4 at <badref>, ir<%out0>
+; AVX512:  LV: Found an estimated cost of 1 for VF 1 For instruction: store double %v3, ptr %out3, align 8
+; AVX512:  LV: Found an estimated cost of 5 for VF 2 For instruction: store double %v3, ptr %out3, align 8
+; AVX512:  LV: Found an estimated cost of 11 for VF 4 For instruction: store double %v3, ptr %out3, align 8
+; AVX512:  LV: Found an estimated cost of 22 for VF 8 For instruction: store double %v3, ptr %out3, align 8
+; AVX512:  LV: Found an estimated cost of 44 for VF 16 For instruction: store double %v3, ptr %out3, align 8
+; AVX512:  LV: Found an estimated cost of 88 for VF 32 For instruction: store double %v3, ptr %out3, align 8
 ;
 entry:
   br label %for.body
diff --git a/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-store-f64-stride-5.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-store-f64-stride-5.ll
index 888ba09ea8452..5946c4d7df295 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-store-f64-stride-5.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-store-f64-stride-5.ll
@@ -1,8 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --filter "LV: Found an estimated cost of [0-9]+ for VF [0-9]+ For instruction:\s*store double %v4, ptr %out4"
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse2 --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=SSE2
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx  --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX1
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2 --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX2
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512vl --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX512
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse2 --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=SSE2
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx  --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX1
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2 --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX2
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512vl --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX512
 ; REQUIRES: asserts
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
@@ -14,28 +14,32 @@ target triple = "x86_64-unknown-linux-gnu"
 define void @test() {
 ; SSE2-LABEL: 'test'
 ; SSE2:  LV: Found an estimated cost of 1 for VF 1 For instruction: store double %v4, ptr %out4, align 8
-; SSE2:  Cost of 18 for VF 2: INTERLEAVE-GROUP with factor 5 at <badref>, ir<%out0>
-; SSE2:  Cost of 36 for VF 4: INTERLEAVE-GROUP with factor 5 at <badref>, ir<%out0>
+; SSE2:  LV: Found an estimated cost of 1 for VF 1 For instruction: store double %v4, ptr %out4, align 8
+; SSE2:  LV: Found an estimated cost of 18 for VF 2 For instruction: store double %v4, ptr %out4, align 8
+; SSE2:  LV: Found an estimated cost of 36 for VF 4 For instruction: store double %v4, ptr %out4, align 8
 ;
 ; AVX1-LABEL: 'test'
 ; AVX1:  LV: Found an estimated cost of 1 for VF 1 For instruction: store double %v4, ptr %out4, align 8
-; AVX1:  Cost of 20 for VF 2: INTERLEAVE-GROUP with factor 5 at <badref>, ir<%out0>
-; AVX1:  Cost of 44 for VF 4: INTERLEAVE-GROUP with factor 5 at <badref>, ir<%out0>
-; AVX1:  Cost of 88 for VF 8: INTERLEAVE-GROUP with factor 5 at <badref>, ir<%out0>
+; AVX1:  LV: Found an estimated cost of 1 for VF 1 For instruction: store double %v4, ptr %out4, align 8
+; AVX1:  LV: Found an estimated cost of 20 for VF 2 For instruction: store double %v4, ptr %out4, align 8
+; AVX1:  LV: Found an estimated cost of 44 for VF 4 For instruction: store double %v4, ptr %out4, align 8
+; AVX1:  LV: Found an estimated cost of 88 for VF 8 For instruction: store double %v4, ptr %out4, align 8
 ;
 ; AVX2-LABEL: 'test'
 ; AVX2:  LV: Found an estimated cost of 1 for VF 1 For instruction: store double %v4, ptr %out4, align 8
-; AVX2:  Cost of 20 for VF 2: INTERLEAVE-GROUP with factor 5 at <badref>, ir<%out0>
-; AVX2:  Cost of 44 for VF 4: INTERLEAVE-GROUP with factor 5 at <badref>, ir<%out0>
-; AVX2:  Cost of 88 for VF 8: INTERLEAVE-GROUP with factor 5 at <badref>, ir<%out0>
+; AVX2:  LV: Found an estimated cost of 1 for VF 1 For instruction: store double %v4, ptr %out4, align 8
+; AVX2:  LV: Found an estimated cost of 20 for VF 2 For instruction: store double %v4, ptr %out4, align 8
+; AVX2:  LV: Found an estimated cost of 44 for VF 4 For instruction: store double %v4, ptr %out4, align 8
+; AVX2:  LV: Found an estimated cost of 88 for VF 8 For instruction: store double %v4, ptr %out4, align 8
 ;
 ; AVX512-LABEL: 'test'
 ; AVX512:  LV: Found an estimated cost of 1 for VF 1 For instruction: store double %v4, ptr %out4, align 8
-; AVX512:  Cost of 14 for VF 2: INTERLEAVE-GROUP with factor 5 at <badref>, ir<%out0>
-; AVX512:  Cost of 21 for VF 4: INTERLEAVE-GROUP with factor 5 at <badref>, ir<%out0>
-; AVX512:  Cost of 35 for VF 8: INTERLEAVE-GROUP with factor 5 at <badref>, ir<%out0>
-; AVX512:  Cost of 70 for VF 16: INTERLEAVE-GROUP with factor 5 at <badref>, ir<%out0>
-; AVX512:  Cost of 140 for VF 32: INTERLEAVE-GROUP with factor 5 at <badref>, ir<%out0>
+; AVX512:  LV: Found an estimated cost of 1 for VF 1 For instruction: store double %v4, ptr %out4, align 8
+; AVX512:  LV: Found an estimated cost of 14 for VF 2 For instruction: store double %v4, ptr %out4, align 8
+; AVX512:  LV: Found an estimated cost of 21 for VF 4 For instruction: store double %v4, ptr %out4, align 8
+; AVX512:  LV: Found an estimated cost of 35 for VF 8 For instruction: store double %v4, ptr %out4, align 8
+; AVX512:  LV: Found an estimated cost of 70 for VF 16 For instruction: store double %v4, ptr %out4, align 8
+; AVX512:  LV: Found an estimated cost of 140 for VF 32 For instruction: store double %v4, ptr %out4, align 8
 ;
 entry:
   br label %for.body
diff --git a/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-store-f64-stride-6.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-store-f64-stride-6.ll
index 6200f8fec3201..e9c97283fab18 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-store-f64-stride-6.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-store-f64-stride-6.ll
@@ -1,8 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --filter "LV: Found an estimated cost of [0-9]+ for VF [0-9]+ For instruction:\s*store double %v5, ptr %out5"
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse2 --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=SSE2
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx  --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX1
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2 --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX2
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512vl --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX512
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse2 --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=SSE2
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx  --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX1
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2 --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX2
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512vl --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX512
 ; REQUIRES: asserts
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
@@ -14,28 +14,32 @@ target triple = "x86_64-unknown-linux-gnu"
 define void @test() {
 ; SSE2-LABEL: 'test'
 ; SSE2:  LV: Found an estimated cost of 1 for VF 1 For instruction: store double %v5, ptr %out5, align 8
-; SSE2:  Cost of 20 for VF 2: INTERLEAVE-GROUP with factor 6 at <badref>, ir<%out0>
-; SSE2:  Cost of 40 for VF 4: INTERLEAVE-GROUP with factor 6 at <badref>, ir<%out0>
+; SSE2:  LV: Found an estimated cost of 1 for VF 1 For instruction: store double %v5, ptr %out5, align 8
+; SSE2:  LV: Found an estimated cost of 20 for VF 2 For instruction: store double %v5, ptr %out5, align 8
+; SSE2:  LV: Found an estimated cost of 40 for VF 4 For instruction: store double %v5, ptr %out5, align 8
 ;
 ; AVX1-LABEL: 'test'
 ; AVX1:  LV: Found an estimated cost of 1 for VF 1 For instruction: store double %v5, ptr %out5, align 8
-; AVX1:  Cost of 21 for VF 2: INTERLEAVE-GROUP with factor 6 at <badref>, ir<%out0>
-; AVX1:  Cost of 48 for VF 4: INTERLEAVE-GROUP with factor 6 at <badref>, ir<%out0>
-; AVX1:  Cost of 96 for VF 8: INTERLEAVE-GROUP with factor 6 at <badref>, ir<%out0>
+; AVX1:  LV: Found an estimated cost of 1 for VF 1 For instruction: store double %v5, ptr %out5, align 8
+; AVX1:  LV: Found an estimated cost of 21 for VF 2 For instruction: store double %v5, ptr %out5, align 8
+; AVX1:  LV: Found an estimated cost of 48 for VF 4 For instruction: store double %v5, ptr %out5, align 8
+; AVX1:  LV: Found an estimated cost of 96 for VF 8 For instruction: store double %v5, ptr %out5, align 8
 ;
 ; AVX2-LABEL: 'test'
 ; AVX2:  LV: Found an estimated cost of 1 for VF 1 For instruction: store double %v5, ptr %out5, align 8
-; AVX2:  Cost of 11 for VF 2: INTERLEAVE-GROUP with factor 6 at <badref>, ir<%out0>
-; AVX2:  Cost of 21 for VF 4: INTERLEAVE-GROUP with factor 6 at <badref>, ir<%out0>
-; AVX2:  Cost of 42 for VF 8: INTERLEAVE-GROUP with factor 6 at <badref>, ir<%out0>
+; AVX2:  LV: Found an estimated cost of 1 for VF 1 For instruction: store double %v5, ptr %out5, align 8
+; AVX2:  LV: Found an estimated cost of 11 for VF 2 For instruction: store double %v5, ptr %out5, align 8
+; AVX2:  LV: Found an estimated cost of 21 for VF 4 For instruction: store double %v5, ptr %out5, align 8
+; AVX2:  LV: Found an estimated cost of 42 for VF 8 For instruction: store double %v5, ptr %out5, align 8
 ;
 ; AVX512-LABEL: 'test'
 ; AVX512:  LV: Found an estimated cost of 1 for VF 1 For instruction: store double %v5, ptr %out5, align 8
-; AVX512:  Cost of 17 for VF 2: INTERLEAVE-GROUP with factor 6 at <badref>, ir<%out0>
-; AVX512:  Cost of 25 for VF 4: INTERLEAVE-GROUP with factor 6 at <badref>, ir<%out0>
-; AVX512:  Cost of 51 for VF 8: INTERLEAVE-GROUP with factor 6 at <badref>, ir<%out0>
-; AVX512:  Cost of 102 for VF 16: INTERLEAVE-GROUP with factor 6 at <badref>, ir<%out0>
-; AVX512:  Cost of 204 for VF 32: INTERLEAVE-GROUP with factor 6 at <badref>, ir<%out0>
+; AVX512:  LV: Found an estimated cost of 1 for VF 1 For instruction: store double %v5, ptr %out5, align 8
+; AVX512:  LV: Found an estimated cost of 17 for VF 2 For instruction: store double %v5, ptr %out5, align 8
+; AVX512:  LV: Found an estimated cost of 25 for VF 4 For instruction: store double %v5, ptr %out5, align 8
+; AVX512:  LV: Found an estimated cost of 51 for VF 8 For instruction: store double %v5, ptr %out5, align 8
+; AVX512:  LV: Found an estimated cost of 102 for VF 16 For instruction: store double %v5, ptr %out5, align 8
+; AVX512:  LV: Found an estimated cost of 204 for VF 32 For instruction: store double %v5, ptr %out5, align 8
 ;
 entry:
   br label %for.body
diff --git a/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-store-f64-stride-7.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-store-f64-stride-7.ll
index b8258ae845954..555bbe8e44269 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-store-f64-stride-7.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-store-f64-stride-7.ll
@@ -1,8 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --filter "LV: Found an estimated cost of [0-9]+ for VF [0-9]+ For instruction:\s*store double %v6, ptr %out6"
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse2 --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=SSE2
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx  --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX1
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2 --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX2
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512vl --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX512
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse2 --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=SSE2
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx  --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX1
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2 --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX2
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512vl --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX512
 ; REQUIRES: asserts
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
@@ -14,28 +14,32 @@ target triple = "x86_64-unknown-linux-gnu"
 define void @test() {
 ; SSE2-LABEL: 'test'
 ; SSE2:  LV: Found an estimated cost of 1 for VF 1 For instruction: store double %v6, ptr %out6, align 8
-; SSE2:  Cost of 22 for VF 2: INTERLEAVE-GROUP with factor 7 at <badref>, ir<%out0>
-; SSE2:  Cost of 44 for VF 4: INTERLEAVE-GROUP with factor 7 at <badref>, ir<%out0>
+; SSE2:  LV: Found an estimated cost of 1 for VF 1 For instruction: store double %v6, ptr %out6, align 8
+; SSE2:  LV: Found an estimated cost of 22 for VF 2 For instruction: store double %v6, ptr %out6, align 8
+; SSE2:  LV: Found an estimated cost of 44 for VF 4 For instruction: store double %v6, ptr %out6, align 8
 ;
 ; AVX1-LABEL: 'test'
 ; AVX1:  LV: Found an estimated cost of 1 for VF 1 For instruction: store double %v6, ptr %out6, align 8
-; AVX1:  Cost of 23 for VF 2: INTERLEAVE-GROUP with factor 7 at <badref>, ir<%out0>
-; AVX1:  Cost of 52 for VF 4: INTERLEAVE-GROUP with factor 7 at <badref>, ir<%out0>
-; AVX1:  Cost of 104 for VF 8: INTERLEAVE-GROUP with factor 7 at <badref>, ir<%out0>
+; AVX1:  LV: Found an estimated cost of 1 for VF 1 For instruction: store double %v6, ptr %out6, align 8
+; AVX1:  LV: Found an estimated cost of 23 for VF 2 For instruction: store double %v6, ptr %out6, align 8
+; AVX1:  LV: Found an estimated cost of 52 for VF 4 For instruction: store double %v6, ptr %out6, align 8
+; AVX1:  LV: Found an estimated cost of 104 for VF 8 For instruction: store double %v6, ptr %out6, align 8
 ;
 ; AVX2-LABEL: 'test'
 ; AVX2:  LV: Found an estimated cost of 1 for VF 1 For instruction: store double %v6, ptr %out6, align 8
-; AVX2:  Cost of 23 for VF 2: INTERLEAVE-GROUP with factor 7 at <badref>, ir<%out0>
-; AVX2:  Cost of 52 for VF 4: INTERLEAVE-GROUP with factor 7 at <badref>, ir<%out0>
-; AVX2:  Cost of 104 for VF 8: INTERLEAVE-GROUP with factor 7 at <badref>, ir<%out0>
+; AVX2:  LV: Found an estimated cost of 1 for VF 1 For instruction: store double %v6, ptr %out6, align 8
+; AVX2:  LV: Found an estimated cost of 23 for VF 2 For instruction: store double %v6, ptr %out6, align 8
+; AVX2:  LV: Found an estimated cost of 52 for VF 4 For instruction: store double %v6, ptr %out6, align 8
+; AVX2:  LV: Found an estimated cost of 104 for VF 8 For instruction: store double %v6, ptr %out6, align 8
 ;
 ; AVX512-LABEL: 'test'
 ; AVX512:  LV: Found an estimated cost of 1 for VF 1 For instruction: store double %v6, ptr %out6, align 8
-; AVX512:  Cost of 20 for VF 2: INTERLEAVE-GROUP with factor 7 at <badref>, ir<%out0>
-; AVX512:  Cost of 40 for VF 4: INTERLEAVE-GROUP with factor 7 at <badref>, ir<%out0>
-; AVX512:  Cost of 70 for VF 8: INTERLEAVE-GROUP with factor 7 at <badref>, ir<%out0>
-; AVX512:  Cost of 140 for VF 16: INTERLEAVE-GROUP with factor 7 at <badref>, ir<%out0>
-; AVX512:  Cost of 280 for VF 32: INTERLEAVE-GROUP with factor 7 at <badref>, ir<%out0>
+; AVX512:  LV: Found an estimated cost of 1 for VF 1 For instruction: store double %v6, ptr %out6, align 8
+; AVX512:  LV: Found an estimated cost of 20 for VF 2 For instruction: store double %v6, ptr %out6, align 8
+; AVX512:  LV: Found an estimated cost of 40 for VF 4 For instruction: store double %v6, ptr %out6, align 8
+; AVX512:  LV: Found an estimated cost of 70 for VF 8 For instruction: store double %v6, ptr %out6, align 8
+; AVX512:  LV: Found an estimated cost of 140 for VF 16 For instruction: store double %v6, ptr %out6, align 8
+; AVX512:  LV: Found an estimated cost of 280 for VF 32 For instruction: store double %v6, ptr %out6, align 8
 ;
 entry:
   br label %for.body
diff --git a/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-store-f64-stride-8.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-store-f64-stride-8.ll
index ed2bb3f750b01..fb0f86e1db7a6 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-store-f64-stride-8.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-store-f64-stride-8.ll
@@ -1,8 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --filter "LV: Found an estimated cost of [0-9]+ for VF [0-9]+ For instruction:\s*store double %v., ptr %out."
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse2 --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=SSE2
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx  --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX1
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2 --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX2
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512vl --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX512
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse2 --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=SSE2
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx  --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX1
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2 --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX2
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512vl --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX512
 ; REQUIRES: asserts
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
diff --git a/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-store-i16-stride-2.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-store-i16-stride-2.ll
index 70aa1a017605e..5a48776cdad2f 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-store-i16-stride-2.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-store-i16-stride-2.ll
@@ -1,9 +1,9 @@
 ; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --filter "LV: Found an estimated cost of [0-9]+ for VF [0-9]+ For instruction:\s*store i16 %v1, ptr %out1"
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse2 --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=SSE2
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx  --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX1
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2 --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX2
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512vl --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX512DQ
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512vl,+avx512bw --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX512BW
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse2 --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=SSE2
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx  --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX1
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2 --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX2
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512vl --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX512DQ
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512vl,+avx512bw --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX512BW
 ; REQUIRES: asserts
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
@@ -15,44 +15,49 @@ target triple = "x86_64-unknown-linux-gnu"
 define void @test() {
 ; SSE2-LABEL: 'test'
 ; SSE2:  LV: Found an estimated cost of 1 for VF 1 For instruction: store i16 %v1, ptr %out1, align 2
-; SSE2:  Cost of 2 for VF 2: INTERLEAVE-GROUP with factor 2 at <badref>, ir<%out0>
-; SSE2:  Cost of 2 for VF 4: INTERLEAVE-GROUP with factor 2 at <badref>, ir<%out0>
-; SSE2:  Cost of 34 for VF 8: INTERLEAVE-GROUP with factor 2 at <badref>, ir<%out0>
-; SSE2:  Cost of 68 for VF 16: INTERLEAVE-GROUP with factor 2 at <badref>, ir<%out0>
+; SSE2:  LV: Found an estimated cost of 1 for VF 1 For instruction: store i16 %v1, ptr %out1, align 2
+; SSE2:  LV: Found an estimated cost of 2 for VF 2 For instruction: store i16 %v1, ptr %out1, align 2
+; SSE2:  LV: Found an estimated cost of 2 for VF 4 For instruction: store i16 %v1, ptr %out1, align 2
+; SSE2:  LV: Found an estimated cost of 34 for VF 8 For instruction: store i16 %v1, ptr %out1, align 2
+; SSE2:  LV: Found an estimated cost of 68 for VF 16 For instruction: store i16 %v1, ptr %out1, align 2
 ;
 ; AVX1-LABEL: 'test'
 ; AVX1:  LV: Found an estimated cost of 1 for VF 1 For instruction: store i16 %v1, ptr %out1, align 2
-; AVX1:  Cost of 2 for VF 2: INTERLEAVE-GROUP with factor 2 at <badref>, ir<%out0>
-; AVX1:  Cost of 2 for VF 4: INTERLEAVE-GROUP with factor 2 at <badref>, ir<%out0>
-; AVX1:  Cost of 34 for VF 8: INTERLEAVE-GROUP with factor 2 at <badref>, ir<%out0>
-; AVX1:  Cost of 70 for VF 16: INTERLEAVE-GROUP with factor 2 at <badref>, ir<%out0>
-; AVX1:  Cost of 140 for VF 32: INTERLEAVE-GROUP with factor 2 at <badref>, ir<%out0>
+; AVX1:  LV: Found an estimated cost of 1 for VF 1 For instruction: store i16 %v1, ptr %out1, align 2
+; AVX1:  LV: Found an estimated cost of 2 for VF 2 For instruction: store i16 %v1, ptr %out1, align 2
+; AVX1:  LV: Found an estimated cost of 2 for VF 4 For instruction: store i16 %v1, ptr %out1, align 2
+; AVX1:  LV: Found an estimated cost of 34 for VF 8 For instruction: store i16 %v1, ptr %out1, align 2
+; AVX1:  LV: Found an estimated cost of 70 for VF 16 For instruction: store i16 %v1, ptr %out1, align 2
+; AVX1:  LV: Found an estimated cost of 140 for VF 32 For instruction: store i16 %v1, ptr %out1, align 2
 ;
 ; AVX2-LABEL: 'test'
 ; AVX2:  LV: Found an estimated cost of 1 for VF 1 For instruction: store i16 %v1, ptr %out1, align 2
-; AVX2:  Cost of 2 for VF 2: INTERLEAVE-GROUP with factor 2 at <badref>, ir<%out0>
-; AVX2:  Cost of 2 for VF 4: INTERLEAVE-GROUP with factor 2 at <badref>, ir<%out0>
-; AVX2:  Cost of 4 for VF 8: INTERLEAVE-GROUP with factor 2 at <badref>, ir<%out0>
-; AVX2:  Cost of 6 for VF 16: INTERLEAVE-GROUP with factor 2 at <badref>, ir<%out0>
-; AVX2:  Cost of 12 for VF 32: INTERLEAVE-GROUP with factor 2 at <badref>, ir<%out0>
+; AVX2:  LV: Found an estimated cost of 1 for VF 1 For instruction: store i16 %v1, ptr %out1, align 2
+; AVX2:  LV: Found an estimated cost of 2 for VF 2 For instruction: store i16 %v1, ptr %out1, align 2
+; AVX2:  LV: Found an estimated cost of 2 for VF 4 For instruction: store i16 %v1, ptr %out1, align 2
+; AVX2:  LV: Found an estimated cost of 4 for VF 8 For instruction: store i16 %v1, ptr %out1, align 2
+; AVX2:  LV: Found an estimated cost of 6 for VF 16 For instruction: store i16 %v1, ptr %out1, align 2
+; AVX2:  LV: Found an estimated cost of 12 for VF 32 For instruction: store i16 %v1, ptr %out1, align 2
 ;
 ; AVX512DQ-LABEL: 'test'
 ; AVX512DQ:  LV: Found an estimated cost of 1 for VF 1 For instruction: store i16 %v1, ptr %out1, align 2
-; AVX512DQ:  Cost of 2 for VF 2: INTERLEAVE-GROUP with factor 2 at <badref>, ir<%out0>
-; AVX512DQ:  Cost of 2 for VF 4: INTERLEAVE-GROUP with factor 2 at <badref>, ir<%out0>
-; AVX512DQ:  Cost of 4 for VF 8: INTERLEAVE-GROUP with factor 2 at <badref>, ir<%out0>
-; AVX512DQ:  Cost of 5 for VF 16: INTERLEAVE-GROUP with factor 2 at <badref>, ir<%out0>
-; AVX512DQ:  Cost of 10 for VF 32: INTERLEAVE-GROUP with factor 2 at <badref>, ir<%out0>
-; AVX512DQ:  Cost of 284 for VF 64: INTERLEAVE-GROUP with factor 2 at <badref>, ir<%out0>
+; AVX512DQ:  LV: Found an estimated cost of 1 for VF 1 For instruction: store i16 %v1, ptr %out1, align 2
+; AVX512DQ:  LV: Found an estimated cost of 2 for VF 2 For instruction: store i16 %v1, ptr %out1, align 2
+; AVX512DQ:  LV: Found an estimated cost of 2 for VF 4 For instruction: store i16 %v1, ptr %out1, align 2
+; AVX512DQ:  LV: Found an estimated cost of 4 for VF 8 For instruction: store i16 %v1, ptr %out1, align 2
+; AVX512DQ:  LV: Found an estimated cost of 5 for VF 16 For instruction: store i16 %v1, ptr %out1, align 2
+; AVX512DQ:  LV: Found an estimated cost of 10 for VF 32 For instruction: store i16 %v1, ptr %out1, align 2
+; AVX512DQ:  LV: Found an estimated cost of 284 for VF 64 For instruction: store i16 %v1, ptr %out1, align 2
 ;
 ; AVX512BW-LABEL: 'test'
 ; AVX512BW:  LV: Found an estimated cost of 1 for VF 1 For instruction: store i16 %v1, ptr %out1, align 2
-; AVX512BW:  Cost of 3 for VF 2: INTERLEAVE-GROUP with factor 2 at <badref>, ir<%out0>
-; AVX512BW:  Cost of 3 for VF 4: INTERLEAVE-GROUP with factor 2 at <badref>, ir<%out0>
-; AVX512BW:  Cost of 3 for VF 8: INTERLEAVE-GROUP with factor 2 at <badref>, ir<%out0>
-; AVX512BW:  Cost of 3 for VF 16: INTERLEAVE-GROUP with factor 2 at <badref>, ir<%out0>
-; AVX512BW:  Cost of 7 for VF 32: INTERLEAVE-GROUP with factor 2 at <badref>, ir<%out0>
-; AVX512BW:  Cost of 14 for VF 64: INTERLEAVE-GROUP with factor 2 at <badref>, ir<%out0>
+; AVX512BW:  LV: Found an estimated cost of 1 for VF 1 For instruction: store i16 %v1, ptr %out1, align 2
+; AVX512BW:  LV: Found an estimated cost of 3 for VF 2 For instruction: store i16 %v1, ptr %out1, align 2
+; AVX512BW:  LV: Found an estimated cost of 3 for VF 4 For instruction: store i16 %v1, ptr %out1, align 2
+; AVX512BW:  LV: Found an estimated cost of 3 for VF 8 For instruction: store i16 %v1, ptr %out1, align 2
+; AVX512BW:  LV: Found an estimated cost of 3 for VF 16 For instruction: store i16 %v1, ptr %out1, align 2
+; AVX512BW:  LV: Found an estimated cost of 7 for VF 32 For instruction: store i16 %v1, ptr %out1, align 2
+; AVX512BW:  LV: Found an estimated cost of 14 for VF 64 For instruction: store i16 %v1, ptr %out1, align 2
 ;
 entry:
   br label %for.body
diff --git a/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-store-i16-stride-3.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-store-i16-stride-3.ll
index 03bffeb36eedc..2cfb488d5007f 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-store-i16-stride-3.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-store-i16-stride-3.ll
@@ -1,9 +1,9 @@
 ; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --filter "LV: Found an estimated cost of [0-9]+ for VF [0-9]+ For instruction:\s*store i16 %v2, ptr %out2"
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse2 --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=SSE2
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx  --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX1
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2 --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX2
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512vl --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX512DQ
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512vl,+avx512bw --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX512BW
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse2 --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=SSE2
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx  --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX1
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2 --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX2
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512vl --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX512DQ
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512vl,+avx512bw --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX512BW
 ; REQUIRES: asserts
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
@@ -15,44 +15,49 @@ target triple = "x86_64-unknown-linux-gnu"
 define void @test() {
 ; SSE2-LABEL: 'test'
 ; SSE2:  LV: Found an estimated cost of 1 for VF 1 For instruction: store i16 %v2, ptr %out2, align 2
-; SSE2:  Cost of 16 for VF 2: INTERLEAVE-GROUP with factor 3 at <badref>, ir<%out0>
-; SSE2:  Cost of 26 for VF 4: INTERLEAVE-GROUP with factor 3 at <badref>, ir<%out0>
-; SSE2:  Cost of 51 for VF 8: INTERLEAVE-GROUP with factor 3 at <badref>, ir<%out0>
-; SSE2:  Cost of 102 for VF 16: INTERLEAVE-GROUP with factor 3 at <badref>, ir<%out0>
+; SSE2:  LV: Found an estimated cost of 1 for VF 1 For instruction: store i16 %v2, ptr %out2, align 2
+; SSE2:  LV: Found an estimated cost of 16 for VF 2 For instruction: store i16 %v2, ptr %out2, align 2
+; SSE2:  LV: Found an estimated cost of 26 for VF 4 For instruction: store i16 %v2, ptr %out2, align 2
+; SSE2:  LV: Found an estimated cost of 51 for VF 8 For instruction: store i16 %v2, ptr %out2, align 2
+; SSE2:  LV: Found an estimated cost of 102 for VF 16 For instruction: store i16 %v2, ptr %out2, align 2
 ;
 ; AVX1-LABEL: 'test'
 ; AVX1:  LV: Found an estimated cost of 1 for VF 1 For instruction: store i16 %v2, ptr %out2, align 2
-; AVX1:  Cost of 15 for VF 2: INTERLEAVE-GROUP with factor 3 at <badref>, ir<%out0>
-; AVX1:  Cost of 29 for VF 4: INTERLEAVE-GROUP with factor 3 at <badref>, ir<%out0>
-; AVX1:  Cost of 52 for VF 8: INTERLEAVE-GROUP with factor 3 at <badref>, ir<%out0>
-; AVX1:  Cost of 105 for VF 16: INTERLEAVE-GROUP with factor 3 at <badref>, ir<%out0>
-; AVX1:  Cost of 210 for VF 32: INTERLEAVE-GROUP with factor 3 at <badref>, ir<%out0>
+; AVX1:  LV: Found an estimated cost of 1 for VF 1 For instruction: store i16 %v2, ptr %out2, align 2
+; AVX1:  LV: Found an estimated cost of 15 for VF 2 For instruction: store i16 %v2, ptr %out2, align 2
+; AVX1:  LV: Found an estimated cost of 29 for VF 4 For instruction: store i16 %v2, ptr %out2, align 2
+; AVX1:  LV: Found an estimated cost of 52 for VF 8 For instruction: store i16 %v2, ptr %out2, align 2
+; AVX1:  LV: Found an estimated cost of 105 for VF 16 For instruction: store i16 %v2, ptr %out2, align 2
+; AVX1:  LV: Found an estimated cost of 210 for VF 32 For instruction: store i16 %v2, ptr %out2, align 2
 ;
 ; AVX2-LABEL: 'test'
 ; AVX2:  LV: Found an estimated cost of 1 for VF 1 For instruction: store i16 %v2, ptr %out2, align 2
-; AVX2:  Cost of 7 for VF 2: INTERLEAVE-GROUP with factor 3 at <badref>, ir<%out0>
-; AVX2:  Cost of 9 for VF 4: INTERLEAVE-GROUP with factor 3 at <badref>, ir<%out0>
-; AVX2:  Cost of 14 for VF 8: INTERLEAVE-GROUP with factor 3 at <badref>, ir<%out0>
-; AVX2:  Cost of 30 for VF 16: INTERLEAVE-GROUP with factor 3 at <badref>, ir<%out0>
-; AVX2:  Cost of 60 for VF 32: INTERLEAVE-GROUP with factor 3 at <badref>, ir<%out0>
+; AVX2:  LV: Found an estimated cost of 1 for VF 1 For instruction: store i16 %v2, ptr %out2, align 2
+; AVX2:  LV: Found an estimated cost of 7 for VF 2 For instruction: store i16 %v2, ptr %out2, align 2
+; AVX2:  LV: Found an estimated cost of 9 for VF 4 For instruction: store i16 %v2, ptr %out2, align 2
+; AVX2:  LV: Found an estimated cost of 14 for VF 8 For instruction: store i16 %v2, ptr %out2, align 2
+; AVX2:  LV: Found an estimated cost of 30 for VF 16 For instruction: store i16 %v2, ptr %out2, align 2
+; AVX2:  LV: Found an estimated cost of 60 for VF 32 For instruction: store i16 %v2, ptr %out2, align 2
 ;
 ; AVX512DQ-LABEL: 'test'
 ; AVX512DQ:  LV: Found an estimated cost of 1 for VF 1 For instruction: store i16 %v2, ptr %out2, align 2
-; AVX512DQ:  Cost of 7 for VF 2: INTERLEAVE-GROUP with factor 3 at <badref>, ir<%out0>
-; AVX512DQ:  Cost of 9 for VF 4: INTERLEAVE-GROUP with factor 3 at <badref>, ir<%out0>
-; AVX512DQ:  Cost of 15 for VF 8: INTERLEAVE-GROUP with factor 3 at <badref>, ir<%out0>
-; AVX512DQ:  Cost of 29 for VF 16: INTERLEAVE-GROUP with factor 3 at <badref>, ir<%out0>
-; AVX512DQ:  Cost of 57 for VF 32: INTERLEAVE-GROUP with factor 3 at <badref>, ir<%out0>
-; AVX512DQ:  Cost of 426 for VF 64: INTERLEAVE-GROUP with factor 3 at <badref>, ir<%out0>
+; AVX512DQ:  LV: Found an estimated cost of 1 for VF 1 For instruction: store i16 %v2, ptr %out2, align 2
+; AVX512DQ:  LV: Found an estimated cost of 7 for VF 2 For instruction: store i16 %v2, ptr %out2, align 2
+; AVX512DQ:  LV: Found an estimated cost of 9 for VF 4 For instruction: store i16 %v2, ptr %out2, align 2
+; AVX512DQ:  LV: Found an estimated cost of 15 for VF 8 For instruction: store i16 %v2, ptr %out2, align 2
+; AVX512DQ:  LV: Found an estimated cost of 29 for VF 16 For instruction: store i16 %v2, ptr %out2, align 2
+; AVX512DQ:  LV: Found an estimated cost of 57 for VF 32 For instruction: store i16 %v2, ptr %out2, align 2
+; AVX512DQ:  LV: Found an estimated cost of 426 for VF 64 For instruction: store i16 %v2, ptr %out2, align 2
 ;
 ; AVX512BW-LABEL: 'test'
 ; AVX512BW:  LV: Found an estimated cost of 1 for VF 1 For instruction: store i16 %v2, ptr %out2, align 2
-; AVX512BW:  Cost of 6 for VF 2: INTERLEAVE-GROUP with factor 3 at <badref>, ir<%out0>
-; AVX512BW:  Cost of 6 for VF 4: INTERLEAVE-GROUP with factor 3 at <badref>, ir<%out0>
-; AVX512BW:  Cost of 6 for VF 8: INTERLEAVE-GROUP with factor 3 at <badref>, ir<%out0>
-; AVX512BW:  Cost of 12 for VF 16: INTERLEAVE-GROUP with factor 3 at <badref>, ir<%out0>
-; AVX512BW:  Cost of 18 for VF 32: INTERLEAVE-GROUP with factor 3 at <badref>, ir<%out0>
-; AVX512BW:  Cost of 36 for VF 64: INTERLEAVE-GROUP with factor 3 at <badref>, ir<%out0>
+; AVX512BW:  LV: Found an estimated cost of 1 for VF 1 For instruction: store i16 %v2, ptr %out2, align 2
+; AVX512BW:  LV: Found an estimated cost of 6 for VF 2 For instruction: store i16 %v2, ptr %out2, align 2
+; AVX512BW:  LV: Found an estimated cost of 6 for VF 4 For instruction: store i16 %v2, ptr %out2, align 2
+; AVX512BW:  LV: Found an estimated cost of 6 for VF 8 For instruction: store i16 %v2, ptr %out2, align 2
+; AVX512BW:  LV: Found an estimated cost of 12 for VF 16 For instruction: store i16 %v2, ptr %out2, align 2
+; AVX512BW:  LV: Found an estimated cost of 18 for VF 32 For instruction: store i16 %v2, ptr %out2, align 2
+; AVX512BW:  LV: Found an estimated cost of 36 for VF 64 For instruction: store i16 %v2, ptr %out2, align 2
 ;
 entry:
   br label %for.body
diff --git a/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-store-i16-stride-4.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-store-i16-stride-4.ll
index 58b742521cfaa..2e4594fac2361 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-store-i16-stride-4.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-store-i16-stride-4.ll
@@ -1,9 +1,9 @@
 ; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --filter "LV: Found an estimated cost of [0-9]+ for VF [0-9]+ For instruction:\s*store i16 %v3, ptr %out3"
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse2 --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=SSE2
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx  --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX1
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2 --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX2
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512vl --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX512DQ
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512vl,+avx512bw --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX512BW
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse2 --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=SSE2
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx  --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX1
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2 --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX2
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512vl --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX512DQ
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512vl,+avx512bw --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX512BW
 ; REQUIRES: asserts
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
@@ -15,44 +15,49 @@ target triple = "x86_64-unknown-linux-gnu"
 define void @test() {
 ; SSE2-LABEL: 'test'
 ; SSE2:  LV: Found an estimated cost of 1 for VF 1 For instruction: store i16 %v3, ptr %out3, align 2
-; SSE2:  Cost of 17 for VF 2: INTERLEAVE-GROUP with factor 4 at <badref>, ir<%out0>
-; SSE2:  Cost of 34 for VF 4: INTERLEAVE-GROUP with factor 4 at <badref>, ir<%out0>
-; SSE2:  Cost of 68 for VF 8: INTERLEAVE-GROUP with factor 4 at <badref>, ir<%out0>
-; SSE2:  Cost of 136 for VF 16: INTERLEAVE-GROUP with factor 4 at <badref>, ir<%out0>
+; SSE2:  LV: Found an estimated cost of 1 for VF 1 For instruction: store i16 %v3, ptr %out3, align 2
+; SSE2:  LV: Found an estimated cost of 17 for VF 2 For instruction: store i16 %v3, ptr %out3, align 2
+; SSE2:  LV: Found an estimated cost of 34 for VF 4 For instruction: store i16 %v3, ptr %out3, align 2
+; SSE2:  LV: Found an estimated cost of 68 for VF 8 For instruction: store i16 %v3, ptr %out3, align 2
+; SSE2:  LV: Found an estimated cost of 136 for VF 16 For instruction: store i16 %v3, ptr %out3, align 2
 ;
 ; AVX1-LABEL: 'test'
 ; AVX1:  LV: Found an estimated cost of 1 for VF 1 For instruction: store i16 %v3, ptr %out3, align 2
-; AVX1:  Cost of 17 for VF 2: INTERLEAVE-GROUP with factor 4 at <badref>, ir<%out0>
-; AVX1:  Cost of 34 for VF 4: INTERLEAVE-GROUP with factor 4 at <badref>, ir<%out0>
-; AVX1:  Cost of 68 for VF 8: INTERLEAVE-GROUP with factor 4 at <badref>, ir<%out0>
-; AVX1:  Cost of 140 for VF 16: INTERLEAVE-GROUP with factor 4 at <badref>, ir<%out0>
-; AVX1:  Cost of 280 for VF 32: INTERLEAVE-GROUP with factor 4 at <badref>, ir<%out0>
+; AVX1:  LV: Found an estimated cost of 1 for VF 1 For instruction: store i16 %v3, ptr %out3, align 2
+; AVX1:  LV: Found an estimated cost of 17 for VF 2 For instruction: store i16 %v3, ptr %out3, align 2
+; AVX1:  LV: Found an estimated cost of 34 for VF 4 For instruction: store i16 %v3, ptr %out3, align 2
+; AVX1:  LV: Found an estimated cost of 68 for VF 8 For instruction: store i16 %v3, ptr %out3, align 2
+; AVX1:  LV: Found an estimated cost of 140 for VF 16 For instruction: store i16 %v3, ptr %out3, align 2
+; AVX1:  LV: Found an estimated cost of 280 for VF 32 For instruction: store i16 %v3, ptr %out3, align 2
 ;
 ; AVX2-LABEL: 'test'
 ; AVX2:  LV: Found an estimated cost of 1 for VF 1 For instruction: store i16 %v3, ptr %out3, align 2
-; AVX2:  Cost of 3 for VF 2: INTERLEAVE-GROUP with factor 4 at <badref>, ir<%out0>
-; AVX2:  Cost of 7 for VF 4: INTERLEAVE-GROUP with factor 4 at <badref>, ir<%out0>
-; AVX2:  Cost of 12 for VF 8: INTERLEAVE-GROUP with factor 4 at <badref>, ir<%out0>
-; AVX2:  Cost of 36 for VF 16: INTERLEAVE-GROUP with factor 4 at <badref>, ir<%out0>
-; AVX2:  Cost of 72 for VF 32: INTERLEAVE-GROUP with factor 4 at <badref>, ir<%out0>
+; AVX2:  LV: Found an estimated cost of 1 for VF 1 For instruction: store i16 %v3, ptr %out3, align 2
+; AVX2:  LV: Found an estimated cost of 3 for VF 2 For instruction: store i16 %v3, ptr %out3, align 2
+; AVX2:  LV: Found an estimated cost of 7 for VF 4 For instruction: store i16 %v3, ptr %out3, align 2
+; AVX2:  LV: Found an estimated cost of 12 for VF 8 For instruction: store i16 %v3, ptr %out3, align 2
+; AVX2:  LV: Found an estimated cost of 36 for VF 16 For instruction: store i16 %v3, ptr %out3, align 2
+; AVX2:  LV: Found an estimated cost of 72 for VF 32 For instruction: store i16 %v3, ptr %out3, align 2
 ;
 ; AVX512DQ-LABEL: 'test'
 ; AVX512DQ:  LV: Found an estimated cost of 1 for VF 1 For instruction: store i16 %v3, ptr %out3, align 2
-; AVX512DQ:  Cost of 3 for VF 2: INTERLEAVE-GROUP with factor 4 at <badref>, ir<%out0>
-; AVX512DQ:  Cost of 7 for VF 4: INTERLEAVE-GROUP with factor 4 at <badref>, ir<%out0>
-; AVX512DQ:  Cost of 11 for VF 8: INTERLEAVE-GROUP with factor 4 at <badref>, ir<%out0>
-; AVX512DQ:  Cost of 34 for VF 16: INTERLEAVE-GROUP with factor 4 at <badref>, ir<%out0>
-; AVX512DQ:  Cost of 68 for VF 32: INTERLEAVE-GROUP with factor 4 at <badref>, ir<%out0>
-; AVX512DQ:  Cost of 568 for VF 64: INTERLEAVE-GROUP with factor 4 at <badref>, ir<%out0>
+; AVX512DQ:  LV: Found an estimated cost of 1 for VF 1 For instruction: store i16 %v3, ptr %out3, align 2
+; AVX512DQ:  LV: Found an estimated cost of 3 for VF 2 For instruction: store i16 %v3, ptr %out3, align 2
+; AVX512DQ:  LV: Found an estimated cost of 7 for VF 4 For instruction: store i16 %v3, ptr %out3, align 2
+; AVX512DQ:  LV: Found an estimated cost of 11 for VF 8 For instruction: store i16 %v3, ptr %out3, align 2
+; AVX512DQ:  LV: Found an estimated cost of 34 for VF 16 For instruction: store i16 %v3, ptr %out3, align 2
+; AVX512DQ:  LV: Found an estimated cost of 68 for VF 32 For instruction: store i16 %v3, ptr %out3, align 2
+; AVX512DQ:  LV: Found an estimated cost of 568 for VF 64 For instruction: store i16 %v3, ptr %out3, align 2
 ;
 ; AVX512BW-LABEL: 'test'
 ; AVX512BW:  LV: Found an estimated cost of 1 for VF 1 For instruction: store i16 %v3, ptr %out3, align 2
-; AVX512BW:  Cost of 8 for VF 2: INTERLEAVE-GROUP with factor 4 at <badref>, ir<%out0>
-; AVX512BW:  Cost of 8 for VF 4: INTERLEAVE-GROUP with factor 4 at <badref>, ir<%out0>
-; AVX512BW:  Cost of 8 for VF 8: INTERLEAVE-GROUP with factor 4 at <badref>, ir<%out0>
-; AVX512BW:  Cost of 17 for VF 16: INTERLEAVE-GROUP with factor 4 at <badref>, ir<%out0>
-; AVX512BW:  Cost of 34 for VF 32: INTERLEAVE-GROUP with factor 4 at <badref>, ir<%out0>
-; AVX512BW:  Cost of 68 for VF 64: INTERLEAVE-GROUP with factor 4 at <badref>, ir<%out0>
+; AVX512BW:  LV: Found an estimated cost of 1 for VF 1 For instruction: store i16 %v3, ptr %out3, align 2
+; AVX512BW:  LV: Found an estimated cost of 8 for VF 2 For instruction: store i16 %v3, ptr %out3, align 2
+; AVX512BW:  LV: Found an estimated cost of 8 for VF 4 For instruction: store i16 %v3, ptr %out3, align 2
+; AVX512BW:  LV: Found an estimated cost of 8 for VF 8 For instruction: store i16 %v3, ptr %out3, align 2
+; AVX512BW:  LV: Found an estimated cost of 17 for VF 16 For instruction: store i16 %v3, ptr %out3, align 2
+; AVX512BW:  LV: Found an estimated cost of 34 for VF 32 For instruction: store i16 %v3, ptr %out3, align 2
+; AVX512BW:  LV: Found an estimated cost of 68 for VF 64 For instruction: store i16 %v3, ptr %out3, align 2
 ;
 entry:
   br label %for.body
diff --git a/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-store-i16-stride-5.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-store-i16-stride-5.ll
index b12b9ddf09529..f536f4438649d 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-store-i16-stride-5.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-store-i16-stride-5.ll
@@ -1,9 +1,9 @@
 ; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --filter "LV: Found an estimated cost of [0-9]+ for VF [0-9]+ For instruction:\s*store i16 %v4, ptr %out4"
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse2 --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=SSE2
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx  --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX1
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2 --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX2
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512vl --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX512DQ
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512vl,+avx512bw --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX512BW
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse2 --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=SSE2
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx  --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX1
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2 --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX2
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512vl --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX512DQ
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512vl,+avx512bw --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX512BW
 ; REQUIRES: asserts
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
@@ -15,44 +15,49 @@ target triple = "x86_64-unknown-linux-gnu"
 define void @test() {
 ; SSE2-LABEL: 'test'
 ; SSE2:  LV: Found an estimated cost of 1 for VF 1 For instruction: store i16 %v4, ptr %out4, align 2
-; SSE2:  Cost of 22 for VF 2: INTERLEAVE-GROUP with factor 5 at <badref>, ir<%out0>
-; SSE2:  Cost of 43 for VF 4: INTERLEAVE-GROUP with factor 5 at <badref>, ir<%out0>
-; SSE2:  Cost of 85 for VF 8: INTERLEAVE-GROUP with factor 5 at <badref>, ir<%out0>
-; SSE2:  Cost of 170 for VF 16: INTERLEAVE-GROUP with factor 5 at <badref>, ir<%out0>
+; SSE2:  LV: Found an estimated cost of 1 for VF 1 For instruction: store i16 %v4, ptr %out4, align 2
+; SSE2:  LV: Found an estimated cost of 22 for VF 2 For instruction: store i16 %v4, ptr %out4, align 2
+; SSE2:  LV: Found an estimated cost of 43 for VF 4 For instruction: store i16 %v4, ptr %out4, align 2
+; SSE2:  LV: Found an estimated cost of 85 for VF 8 For instruction: store i16 %v4, ptr %out4, align 2
+; SSE2:  LV: Found an estimated cost of 170 for VF 16 For instruction: store i16 %v4, ptr %out4, align 2
 ;
 ; AVX1-LABEL: 'test'
 ; AVX1:  LV: Found an estimated cost of 1 for VF 1 For instruction: store i16 %v4, ptr %out4, align 2
-; AVX1:  Cost of 26 for VF 2: INTERLEAVE-GROUP with factor 5 at <badref>, ir<%out0>
-; AVX1:  Cost of 44 for VF 4: INTERLEAVE-GROUP with factor 5 at <badref>, ir<%out0>
-; AVX1:  Cost of 86 for VF 8: INTERLEAVE-GROUP with factor 5 at <badref>, ir<%out0>
-; AVX1:  Cost of 175 for VF 16: INTERLEAVE-GROUP with factor 5 at <badref>, ir<%out0>
-; AVX1:  Cost of 350 for VF 32: INTERLEAVE-GROUP with factor 5 at <badref>, ir<%out0>
+; AVX1:  LV: Found an estimated cost of 1 for VF 1 For instruction: store i16 %v4, ptr %out4, align 2
+; AVX1:  LV: Found an estimated cost of 26 for VF 2 For instruction: store i16 %v4, ptr %out4, align 2
+; AVX1:  LV: Found an estimated cost of 44 for VF 4 For instruction: store i16 %v4, ptr %out4, align 2
+; AVX1:  LV: Found an estimated cost of 86 for VF 8 For instruction: store i16 %v4, ptr %out4, align 2
+; AVX1:  LV: Found an estimated cost of 175 for VF 16 For instruction: store i16 %v4, ptr %out4, align 2
+; AVX1:  LV: Found an estimated cost of 350 for VF 32 For instruction: store i16 %v4, ptr %out4, align 2
 ;
 ; AVX2-LABEL: 'test'
 ; AVX2:  LV: Found an estimated cost of 1 for VF 1 For instruction: store i16 %v4, ptr %out4, align 2
-; AVX2:  Cost of 26 for VF 2: INTERLEAVE-GROUP with factor 5 at <badref>, ir<%out0>
-; AVX2:  Cost of 44 for VF 4: INTERLEAVE-GROUP with factor 5 at <badref>, ir<%out0>
-; AVX2:  Cost of 86 for VF 8: INTERLEAVE-GROUP with factor 5 at <badref>, ir<%out0>
-; AVX2:  Cost of 175 for VF 16: INTERLEAVE-GROUP with factor 5 at <badref>, ir<%out0>
-; AVX2:  Cost of 350 for VF 32: INTERLEAVE-GROUP with factor 5 at <badref>, ir<%out0>
+; AVX2:  LV: Found an estimated cost of 1 for VF 1 For instruction: store i16 %v4, ptr %out4, align 2
+; AVX2:  LV: Found an estimated cost of 26 for VF 2 For instruction: store i16 %v4, ptr %out4, align 2
+; AVX2:  LV: Found an estimated cost of 44 for VF 4 For instruction: store i16 %v4, ptr %out4, align 2
+; AVX2:  LV: Found an estimated cost of 86 for VF 8 For instruction: store i16 %v4, ptr %out4, align 2
+; AVX2:  LV: Found an estimated cost of 175 for VF 16 For instruction: store i16 %v4, ptr %out4, align 2
+; AVX2:  LV: Found an estimated cost of 350 for VF 32 For instruction: store i16 %v4, ptr %out4, align 2
 ;
 ; AVX512DQ-LABEL: 'test'
 ; AVX512DQ:  LV: Found an estimated cost of 1 for VF 1 For instruction: store i16 %v4, ptr %out4, align 2
-; AVX512DQ:  Cost of 26 for VF 2: INTERLEAVE-GROUP with factor 5 at <badref>, ir<%out0>
-; AVX512DQ:  Cost of 47 for VF 4: INTERLEAVE-GROUP with factor 5 at <badref>, ir<%out0>
-; AVX512DQ:  Cost of 86 for VF 8: INTERLEAVE-GROUP with factor 5 at <badref>, ir<%out0>
-; AVX512DQ:  Cost of 176 for VF 16: INTERLEAVE-GROUP with factor 5 at <badref>, ir<%out0>
-; AVX512DQ:  Cost of 355 for VF 32: INTERLEAVE-GROUP with factor 5 at <badref>, ir<%out0>
-; AVX512DQ:  Cost of 710 for VF 64: INTERLEAVE-GROUP with factor 5 at <badref>, ir<%out0>
+; AVX512DQ:  LV: Found an estimated cost of 1 for VF 1 For instruction: store i16 %v4, ptr %out4, align 2
+; AVX512DQ:  LV: Found an estimated cost of 26 for VF 2 For instruction: store i16 %v4, ptr %out4, align 2
+; AVX512DQ:  LV: Found an estimated cost of 47 for VF 4 For instruction: store i16 %v4, ptr %out4, align 2
+; AVX512DQ:  LV: Found an estimated cost of 86 for VF 8 For instruction: store i16 %v4, ptr %out4, align 2
+; AVX512DQ:  LV: Found an estimated cost of 176 for VF 16 For instruction: store i16 %v4, ptr %out4, align 2
+; AVX512DQ:  LV: Found an estimated cost of 355 for VF 32 For instruction: store i16 %v4, ptr %out4, align 2
+; AVX512DQ:  LV: Found an estimated cost of 710 for VF 64 For instruction: store i16 %v4, ptr %out4, align 2
 ;
 ; AVX512BW-LABEL: 'test'
 ; AVX512BW:  LV: Found an estimated cost of 1 for VF 1 For instruction: store i16 %v4, ptr %out4, align 2
-; AVX512BW:  Cost of 11 for VF 2: INTERLEAVE-GROUP with factor 5 at <badref>, ir<%out0>
-; AVX512BW:  Cost of 11 for VF 4: INTERLEAVE-GROUP with factor 5 at <badref>, ir<%out0>
-; AVX512BW:  Cost of 22 for VF 8: INTERLEAVE-GROUP with factor 5 at <badref>, ir<%out0>
-; AVX512BW:  Cost of 33 for VF 16: INTERLEAVE-GROUP with factor 5 at <badref>, ir<%out0>
-; AVX512BW:  Cost of 55 for VF 32: INTERLEAVE-GROUP with factor 5 at <badref>, ir<%out0>
-; AVX512BW:  Cost of 110 for VF 64: INTERLEAVE-GROUP with factor 5 at <badref>, ir<%out0>
+; AVX512BW:  LV: Found an estimated cost of 1 for VF 1 For instruction: store i16 %v4, ptr %out4, align 2
+; AVX512BW:  LV: Found an estimated cost of 11 for VF 2 For instruction: store i16 %v4, ptr %out4, align 2
+; AVX512BW:  LV: Found an estimated cost of 11 for VF 4 For instruction: store i16 %v4, ptr %out4, align 2
+; AVX512BW:  LV: Found an estimated cost of 22 for VF 8 For instruction: store i16 %v4, ptr %out4, align 2
+; AVX512BW:  LV: Found an estimated cost of 33 for VF 16 For instruction: store i16 %v4, ptr %out4, align 2
+; AVX512BW:  LV: Found an estimated cost of 55 for VF 32 For instruction: store i16 %v4, ptr %out4, align 2
+; AVX512BW:  LV: Found an estimated cost of 110 for VF 64 For instruction: store i16 %v4, ptr %out4, align 2
 ;
 entry:
   br label %for.body
diff --git a/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-store-i16-stride-6.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-store-i16-stride-6.ll
index c41493d916fb9..1b7522d01ae2e 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-store-i16-stride-6.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-store-i16-stride-6.ll
@@ -1,9 +1,9 @@
 ; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --filter "LV: Found an estimated cost of [0-9]+ for VF [0-9]+ For instruction:\s*store i16 %v5, ptr %out5"
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse2 --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=SSE2
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx  --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX1
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2 --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX2
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512vl --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX512DQ
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512vl,+avx512bw --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX512BW
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse2 --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=SSE2
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx  --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX1
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2 --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX2
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512vl --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX512DQ
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512vl,+avx512bw --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX512BW
 ; REQUIRES: asserts
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
@@ -15,44 +15,49 @@ target triple = "x86_64-unknown-linux-gnu"
 define void @test() {
 ; SSE2-LABEL: 'test'
 ; SSE2:  LV: Found an estimated cost of 1 for VF 1 For instruction: store i16 %v5, ptr %out5, align 2
-; SSE2:  Cost of 26 for VF 2: INTERLEAVE-GROUP with factor 6 at <badref>, ir<%out0>
-; SSE2:  Cost of 51 for VF 4: INTERLEAVE-GROUP with factor 6 at <badref>, ir<%out0>
-; SSE2:  Cost of 102 for VF 8: INTERLEAVE-GROUP with factor 6 at <badref>, ir<%out0>
-; SSE2:  Cost of 204 for VF 16: INTERLEAVE-GROUP with factor 6 at <badref>, ir<%out0>
+; SSE2:  LV: Found an estimated cost of 1 for VF 1 For instruction: store i16 %v5, ptr %out5, align 2
+; SSE2:  LV: Found an estimated cost of 26 for VF 2 For instruction: store i16 %v5, ptr %out5, align 2
+; SSE2:  LV: Found an estimated cost of 51 for VF 4 For instruction: store i16 %v5, ptr %out5, align 2
+; SSE2:  LV: Found an estimated cost of 102 for VF 8 For instruction: store i16 %v5, ptr %out5, align 2
+; SSE2:  LV: Found an estimated cost of 204 for VF 16 For instruction: store i16 %v5, ptr %out5, align 2
 ;
 ; AVX1-LABEL: 'test'
 ; AVX1:  LV: Found an estimated cost of 1 for VF 1 For instruction: store i16 %v5, ptr %out5, align 2
-; AVX1:  Cost of 29 for VF 2: INTERLEAVE-GROUP with factor 6 at <badref>, ir<%out0>
-; AVX1:  Cost of 52 for VF 4: INTERLEAVE-GROUP with factor 6 at <badref>, ir<%out0>
-; AVX1:  Cost of 102 for VF 8: INTERLEAVE-GROUP with factor 6 at <badref>, ir<%out0>
-; AVX1:  Cost of 210 for VF 16: INTERLEAVE-GROUP with factor 6 at <badref>, ir<%out0>
-; AVX1:  Cost of 420 for VF 32: INTERLEAVE-GROUP with factor 6 at <badref>, ir<%out0>
+; AVX1:  LV: Found an estimated cost of 1 for VF 1 For instruction: store i16 %v5, ptr %out5, align 2
+; AVX1:  LV: Found an estimated cost of 29 for VF 2 For instruction: store i16 %v5, ptr %out5, align 2
+; AVX1:  LV: Found an estimated cost of 52 for VF 4 For instruction: store i16 %v5, ptr %out5, align 2
+; AVX1:  LV: Found an estimated cost of 102 for VF 8 For instruction: store i16 %v5, ptr %out5, align 2
+; AVX1:  LV: Found an estimated cost of 210 for VF 16 For instruction: store i16 %v5, ptr %out5, align 2
+; AVX1:  LV: Found an estimated cost of 420 for VF 32 For instruction: store i16 %v5, ptr %out5, align 2
 ;
 ; AVX2-LABEL: 'test'
 ; AVX2:  LV: Found an estimated cost of 1 for VF 1 For instruction: store i16 %v5, ptr %out5, align 2
-; AVX2:  Cost of 13 for VF 2: INTERLEAVE-GROUP with factor 6 at <badref>, ir<%out0>
-; AVX2:  Cost of 17 for VF 4: INTERLEAVE-GROUP with factor 6 at <badref>, ir<%out0>
-; AVX2:  Cost of 24 for VF 8: INTERLEAVE-GROUP with factor 6 at <badref>, ir<%out0>
-; AVX2:  Cost of 64 for VF 16: INTERLEAVE-GROUP with factor 6 at <badref>, ir<%out0>
-; AVX2:  Cost of 102 for VF 32: INTERLEAVE-GROUP with factor 6 at <badref>, ir<%out0>
+; AVX2:  LV: Found an estimated cost of 1 for VF 1 For instruction: store i16 %v5, ptr %out5, align 2
+; AVX2:  LV: Found an estimated cost of 13 for VF 2 For instruction: store i16 %v5, ptr %out5, align 2
+; AVX2:  LV: Found an estimated cost of 17 for VF 4 For instruction: store i16 %v5, ptr %out5, align 2
+; AVX2:  LV: Found an estimated cost of 24 for VF 8 For instruction: store i16 %v5, ptr %out5, align 2
+; AVX2:  LV: Found an estimated cost of 64 for VF 16 For instruction: store i16 %v5, ptr %out5, align 2
+; AVX2:  LV: Found an estimated cost of 102 for VF 32 For instruction: store i16 %v5, ptr %out5, align 2
 ;
 ; AVX512DQ-LABEL: 'test'
 ; AVX512DQ:  LV: Found an estimated cost of 1 for VF 1 For instruction: store i16 %v5, ptr %out5, align 2
-; AVX512DQ:  Cost of 13 for VF 2: INTERLEAVE-GROUP with factor 6 at <badref>, ir<%out0>
-; AVX512DQ:  Cost of 18 for VF 4: INTERLEAVE-GROUP with factor 6 at <badref>, ir<%out0>
-; AVX512DQ:  Cost of 23 for VF 8: INTERLEAVE-GROUP with factor 6 at <badref>, ir<%out0>
-; AVX512DQ:  Cost of 61 for VF 16: INTERLEAVE-GROUP with factor 6 at <badref>, ir<%out0>
-; AVX512DQ:  Cost of 96 for VF 32: INTERLEAVE-GROUP with factor 6 at <badref>, ir<%out0>
-; AVX512DQ:  Cost of 852 for VF 64: INTERLEAVE-GROUP with factor 6 at <badref>, ir<%out0>
+; AVX512DQ:  LV: Found an estimated cost of 1 for VF 1 For instruction: store i16 %v5, ptr %out5, align 2
+; AVX512DQ:  LV: Found an estimated cost of 13 for VF 2 For instruction: store i16 %v5, ptr %out5, align 2
+; AVX512DQ:  LV: Found an estimated cost of 18 for VF 4 For instruction: store i16 %v5, ptr %out5, align 2
+; AVX512DQ:  LV: Found an estimated cost of 23 for VF 8 For instruction: store i16 %v5, ptr %out5, align 2
+; AVX512DQ:  LV: Found an estimated cost of 61 for VF 16 For instruction: store i16 %v5, ptr %out5, align 2
+; AVX512DQ:  LV: Found an estimated cost of 96 for VF 32 For instruction: store i16 %v5, ptr %out5, align 2
+; AVX512DQ:  LV: Found an estimated cost of 852 for VF 64 For instruction: store i16 %v5, ptr %out5, align 2
 ;
 ; AVX512BW-LABEL: 'test'
 ; AVX512BW:  LV: Found an estimated cost of 1 for VF 1 For instruction: store i16 %v5, ptr %out5, align 2
-; AVX512BW:  Cost of 13 for VF 2: INTERLEAVE-GROUP with factor 6 at <badref>, ir<%out0>
-; AVX512BW:  Cost of 13 for VF 4: INTERLEAVE-GROUP with factor 6 at <badref>, ir<%out0>
-; AVX512BW:  Cost of 27 for VF 8: INTERLEAVE-GROUP with factor 6 at <badref>, ir<%out0>
-; AVX512BW:  Cost of 40 for VF 16: INTERLEAVE-GROUP with factor 6 at <badref>, ir<%out0>
-; AVX512BW:  Cost of 81 for VF 32: INTERLEAVE-GROUP with factor 6 at <badref>, ir<%out0>
-; AVX512BW:  Cost of 162 for VF 64: INTERLEAVE-GROUP with factor 6 at <badref>, ir<%out0>
+; AVX512BW:  LV: Found an estimated cost of 1 for VF 1 For instruction: store i16 %v5, ptr %out5, align 2
+; AVX512BW:  LV: Found an estimated cost of 13 for VF 2 For instruction: store i16 %v5, ptr %out5, align 2
+; AVX512BW:  LV: Found an estimated cost of 13 for VF 4 For instruction: store i16 %v5, ptr %out5, align 2
+; AVX512BW:  LV: Found an estimated cost of 27 for VF 8 For instruction: store i16 %v5, ptr %out5, align 2
+; AVX512BW:  LV: Found an estimated cost of 40 for VF 16 For instruction: store i16 %v5, ptr %out5, align 2
+; AVX512BW:  LV: Found an estimated cost of 81 for VF 32 For instruction: store i16 %v5, ptr %out5, align 2
+; AVX512BW:  LV: Found an estimated cost of 162 for VF 64 For instruction: store i16 %v5, ptr %out5, align 2
 ;
 entry:
   br label %for.body
diff --git a/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-store-i16-stride-7.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-store-i16-stride-7.ll
index 2a4d1bbf77cd5..f6ed77887c802 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-store-i16-stride-7.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-store-i16-stride-7.ll
@@ -1,9 +1,9 @@
 ; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --filter "LV: Found an estimated cost of [0-9]+ for VF [0-9]+ For instruction:\s*store i16 %v6, ptr %out6"
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse2 --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=SSE2
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx  --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX1
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2 --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX2
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512vl --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX512DQ
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512vl,+avx512bw --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX512BW
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse2 --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=SSE2
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx  --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX1
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2 --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX2
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512vl --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX512DQ
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512vl,+avx512bw --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX512BW
 ; REQUIRES: asserts
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
@@ -15,44 +15,49 @@ target triple = "x86_64-unknown-linux-gnu"
 define void @test() {
 ; SSE2-LABEL: 'test'
 ; SSE2:  LV: Found an estimated cost of 1 for VF 1 For instruction: store i16 %v6, ptr %out6, align 2
-; SSE2:  Cost of 33 for VF 2: INTERLEAVE-GROUP with factor 7 at <badref>, ir<%out0>
-; SSE2:  Cost of 60 for VF 4: INTERLEAVE-GROUP with factor 7 at <badref>, ir<%out0>
-; SSE2:  Cost of 119 for VF 8: INTERLEAVE-GROUP with factor 7 at <badref>, ir<%out0>
-; SSE2:  Cost of 238 for VF 16: INTERLEAVE-GROUP with factor 7 at <badref>, ir<%out0>
+; SSE2:  LV: Found an estimated cost of 1 for VF 1 For instruction: store i16 %v6, ptr %out6, align 2
+; SSE2:  LV: Found an estimated cost of 33 for VF 2 For instruction: store i16 %v6, ptr %out6, align 2
+; SSE2:  LV: Found an estimated cost of 60 for VF 4 For instruction: store i16 %v6, ptr %out6, align 2
+; SSE2:  LV: Found an estimated cost of 119 for VF 8 For instruction: store i16 %v6, ptr %out6, align 2
+; SSE2:  LV: Found an estimated cost of 238 for VF 16 For instruction: store i16 %v6, ptr %out6, align 2
 ;
 ; AVX1-LABEL: 'test'
 ; AVX1:  LV: Found an estimated cost of 1 for VF 1 For instruction: store i16 %v6, ptr %out6, align 2
-; AVX1:  Cost of 35 for VF 2: INTERLEAVE-GROUP with factor 7 at <badref>, ir<%out0>
-; AVX1:  Cost of 63 for VF 4: INTERLEAVE-GROUP with factor 7 at <badref>, ir<%out0>
-; AVX1:  Cost of 120 for VF 8: INTERLEAVE-GROUP with factor 7 at <badref>, ir<%out0>
-; AVX1:  Cost of 245 for VF 16: INTERLEAVE-GROUP with factor 7 at <badref>, ir<%out0>
-; AVX1:  Cost of 490 for VF 32: INTERLEAVE-GROUP with factor 7 at <badref>, ir<%out0>
+; AVX1:  LV: Found an estimated cost of 1 for VF 1 For instruction: store i16 %v6, ptr %out6, align 2
+; AVX1:  LV: Found an estimated cost of 35 for VF 2 For instruction: store i16 %v6, ptr %out6, align 2
+; AVX1:  LV: Found an estimated cost of 63 for VF 4 For instruction: store i16 %v6, ptr %out6, align 2
+; AVX1:  LV: Found an estimated cost of 120 for VF 8 For instruction: store i16 %v6, ptr %out6, align 2
+; AVX1:  LV: Found an estimated cost of 245 for VF 16 For instruction: store i16 %v6, ptr %out6, align 2
+; AVX1:  LV: Found an estimated cost of 490 for VF 32 For instruction: store i16 %v6, ptr %out6, align 2
 ;
 ; AVX2-LABEL: 'test'
 ; AVX2:  LV: Found an estimated cost of 1 for VF 1 For instruction: store i16 %v6, ptr %out6, align 2
-; AVX2:  Cost of 35 for VF 2: INTERLEAVE-GROUP with factor 7 at <badref>, ir<%out0>
-; AVX2:  Cost of 63 for VF 4: INTERLEAVE-GROUP with factor 7 at <badref>, ir<%out0>
-; AVX2:  Cost of 120 for VF 8: INTERLEAVE-GROUP with factor 7 at <badref>, ir<%out0>
-; AVX2:  Cost of 245 for VF 16: INTERLEAVE-GROUP with factor 7 at <badref>, ir<%out0>
-; AVX2:  Cost of 490 for VF 32: INTERLEAVE-GROUP with factor 7 at <badref>, ir<%out0>
+; AVX2:  LV: Found an estimated cost of 1 for VF 1 For instruction: store i16 %v6, ptr %out6, align 2
+; AVX2:  LV: Found an estimated cost of 35 for VF 2 For instruction: store i16 %v6, ptr %out6, align 2
+; AVX2:  LV: Found an estimated cost of 63 for VF 4 For instruction: store i16 %v6, ptr %out6, align 2
+; AVX2:  LV: Found an estimated cost of 120 for VF 8 For instruction: store i16 %v6, ptr %out6, align 2
+; AVX2:  LV: Found an estimated cost of 245 for VF 16 For instruction: store i16 %v6, ptr %out6, align 2
+; AVX2:  LV: Found an estimated cost of 490 for VF 32 For instruction: store i16 %v6, ptr %out6, align 2
 ;
 ; AVX512DQ-LABEL: 'test'
 ; AVX512DQ:  LV: Found an estimated cost of 1 for VF 1 For instruction: store i16 %v6, ptr %out6, align 2
-; AVX512DQ:  Cost of 35 for VF 2: INTERLEAVE-GROUP with factor 7 at <badref>, ir<%out0>
-; AVX512DQ:  Cost of 65 for VF 4: INTERLEAVE-GROUP with factor 7 at <badref>, ir<%out0>
-; AVX512DQ:  Cost of 122 for VF 8: INTERLEAVE-GROUP with factor 7 at <badref>, ir<%out0>
-; AVX512DQ:  Cost of 246 for VF 16: INTERLEAVE-GROUP with factor 7 at <badref>, ir<%out0>
-; AVX512DQ:  Cost of 497 for VF 32: INTERLEAVE-GROUP with factor 7 at <badref>, ir<%out0>
-; AVX512DQ:  Cost of 994 for VF 64: INTERLEAVE-GROUP with factor 7 at <badref>, ir<%out0>
+; AVX512DQ:  LV: Found an estimated cost of 1 for VF 1 For instruction: store i16 %v6, ptr %out6, align 2
+; AVX512DQ:  LV: Found an estimated cost of 35 for VF 2 For instruction: store i16 %v6, ptr %out6, align 2
+; AVX512DQ:  LV: Found an estimated cost of 65 for VF 4 For instruction: store i16 %v6, ptr %out6, align 2
+; AVX512DQ:  LV: Found an estimated cost of 122 for VF 8 For instruction: store i16 %v6, ptr %out6, align 2
+; AVX512DQ:  LV: Found an estimated cost of 246 for VF 16 For instruction: store i16 %v6, ptr %out6, align 2
+; AVX512DQ:  LV: Found an estimated cost of 497 for VF 32 For instruction: store i16 %v6, ptr %out6, align 2
+; AVX512DQ:  LV: Found an estimated cost of 994 for VF 64 For instruction: store i16 %v6, ptr %out6, align 2
 ;
 ; AVX512BW-LABEL: 'test'
 ; AVX512BW:  LV: Found an estimated cost of 1 for VF 1 For instruction: store i16 %v6, ptr %out6, align 2
-; AVX512BW:  Cost of 16 for VF 2: INTERLEAVE-GROUP with factor 7 at <badref>, ir<%out0>
-; AVX512BW:  Cost of 16 for VF 4: INTERLEAVE-GROUP with factor 7 at <badref>, ir<%out0>
-; AVX512BW:  Cost of 32 for VF 8: INTERLEAVE-GROUP with factor 7 at <badref>, ir<%out0>
-; AVX512BW:  Cost of 64 for VF 16: INTERLEAVE-GROUP with factor 7 at <badref>, ir<%out0>
-; AVX512BW:  Cost of 112 for VF 32: INTERLEAVE-GROUP with factor 7 at <badref>, ir<%out0>
-; AVX512BW:  Cost of 224 for VF 64: INTERLEAVE-GROUP with factor 7 at <badref>, ir<%out0>
+; AVX512BW:  LV: Found an estimated cost of 1 for VF 1 For instruction: store i16 %v6, ptr %out6, align 2
+; AVX512BW:  LV: Found an estimated cost of 16 for VF 2 For instruction: store i16 %v6, ptr %out6, align 2
+; AVX512BW:  LV: Found an estimated cost of 16 for VF 4 For instruction: store i16 %v6, ptr %out6, align 2
+; AVX512BW:  LV: Found an estimated cost of 32 for VF 8 For instruction: store i16 %v6, ptr %out6, align 2
+; AVX512BW:  LV: Found an estimated cost of 64 for VF 16 For instruction: store i16 %v6, ptr %out6, align 2
+; AVX512BW:  LV: Found an estimated cost of 112 for VF 32 For instruction: store i16 %v6, ptr %out6, align 2
+; AVX512BW:  LV: Found an estimated cost of 224 for VF 64 For instruction: store i16 %v6, ptr %out6, align 2
 ;
 entry:
   br label %for.body
diff --git a/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-store-i16-stride-8.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-store-i16-stride-8.ll
index ada56b16827b1..cc82d48fadb2c 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-store-i16-stride-8.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-store-i16-stride-8.ll
@@ -1,9 +1,9 @@
 ; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --filter "LV: Found an estimated cost of [0-9]+ for VF [0-9]+ For instruction:\s*store i16 %v7, ptr %out7"
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse2 --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=SSE2
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx  --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX1
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2 --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX2
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512vl --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX512DQ
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512vl,+avx512bw --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX512BW
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse2 --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=SSE2
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx  --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX1
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2 --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX2
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512vl --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX512DQ
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512vl,+avx512bw --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX512BW
 ; REQUIRES: asserts
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
@@ -15,6 +15,7 @@ target triple = "x86_64-unknown-linux-gnu"
 define void @test() {
 ; SSE2-LABEL: 'test'
 ; SSE2:  LV: Found an estimated cost of 1 for VF 1 For instruction: store i16 %v7, ptr %out7, align 2
+; SSE2:  LV: Found an estimated cost of 1 for VF 1 For instruction: store i16 %v7, ptr %out7, align 2
 ; SSE2:  LV: Found an estimated cost of 34 for VF 2 For instruction: store i16 %v7, ptr %out7, align 2
 ; SSE2:  LV: Found an estimated cost of 68 for VF 4 For instruction: store i16 %v7, ptr %out7, align 2
 ; SSE2:  LV: Found an estimated cost of 136 for VF 8 For instruction: store i16 %v7, ptr %out7, align 2
@@ -22,6 +23,7 @@ define void @test() {
 ;
 ; AVX1-LABEL: 'test'
 ; AVX1:  LV: Found an estimated cost of 1 for VF 1 For instruction: store i16 %v7, ptr %out7, align 2
+; AVX1:  LV: Found an estimated cost of 1 for VF 1 For instruction: store i16 %v7, ptr %out7, align 2
 ; AVX1:  LV: Found an estimated cost of 34 for VF 2 For instruction: store i16 %v7, ptr %out7, align 2
 ; AVX1:  LV: Found an estimated cost of 68 for VF 4 For instruction: store i16 %v7, ptr %out7, align 2
 ; AVX1:  LV: Found an estimated cost of 136 for VF 8 For instruction: store i16 %v7, ptr %out7, align 2
@@ -30,6 +32,7 @@ define void @test() {
 ;
 ; AVX2-LABEL: 'test'
 ; AVX2:  LV: Found an estimated cost of 1 for VF 1 For instruction: store i16 %v7, ptr %out7, align 2
+; AVX2:  LV: Found an estimated cost of 1 for VF 1 For instruction: store i16 %v7, ptr %out7, align 2
 ; AVX2:  LV: Found an estimated cost of 34 for VF 2 For instruction: store i16 %v7, ptr %out7, align 2
 ; AVX2:  LV: Found an estimated cost of 68 for VF 4 For instruction: store i16 %v7, ptr %out7, align 2
 ; AVX2:  LV: Found an estimated cost of 136 for VF 8 For instruction: store i16 %v7, ptr %out7, align 2
@@ -38,6 +41,7 @@ define void @test() {
 ;
 ; AVX512DQ-LABEL: 'test'
 ; AVX512DQ:  LV: Found an estimated cost of 1 for VF 1 For instruction: store i16 %v7, ptr %out7, align 2
+; AVX512DQ:  LV: Found an estimated cost of 1 for VF 1 For instruction: store i16 %v7, ptr %out7, align 2
 ; AVX512DQ:  LV: Found an estimated cost of 34 for VF 2 For instruction: store i16 %v7, ptr %out7, align 2
 ; AVX512DQ:  LV: Found an estimated cost of 68 for VF 4 For instruction: store i16 %v7, ptr %out7, align 2
 ; AVX512DQ:  LV: Found an estimated cost of 136 for VF 8 For instruction: store i16 %v7, ptr %out7, align 2
@@ -47,6 +51,7 @@ define void @test() {
 ;
 ; AVX512BW-LABEL: 'test'
 ; AVX512BW:  LV: Found an estimated cost of 1 for VF 1 For instruction: store i16 %v7, ptr %out7, align 2
+; AVX512BW:  LV: Found an estimated cost of 1 for VF 1 For instruction: store i16 %v7, ptr %out7, align 2
 ; AVX512BW:  LV: Found an estimated cost of 18 for VF 2 For instruction: store i16 %v7, ptr %out7, align 2
 ; AVX512BW:  LV: Found an estimated cost of 18 for VF 4 For instruction: store i16 %v7, ptr %out7, align 2
 ; AVX512BW:  LV: Found an estimated cost of 37 for VF 8 For instruction: store i16 %v7, ptr %out7, align 2
diff --git a/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-store-i32-stride-2.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-store-i32-stride-2.ll
index cc697305e3b0e..0a33a96bb6ba1 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-store-i32-stride-2.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-store-i32-stride-2.ll
@@ -1,8 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --filter "LV: Found an estimated cost of [0-9]+ for VF [0-9]+ For instruction:\s*store i32 %v1, ptr %out1"
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse2 --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=SSE2
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx  --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX1
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2 --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX2
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512vl --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX512
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse2 --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=SSE2
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx  --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX1
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2 --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX2
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512vl --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX512
 ; REQUIRES: asserts
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
@@ -14,35 +14,39 @@ target triple = "x86_64-unknown-linux-gnu"
 define void @test() {
 ; SSE2-LABEL: 'test'
 ; SSE2:  LV: Found an estimated cost of 1 for VF 1 For instruction: store i32 %v1, ptr %out1, align 4
-; SSE2:  Cost of 2 for VF 2: INTERLEAVE-GROUP with factor 2 at <badref>, ir<%out0>
-; SSE2:  Cost of 30 for VF 4: INTERLEAVE-GROUP with factor 2 at <badref>, ir<%out0>
-; SSE2:  Cost of 60 for VF 8: INTERLEAVE-GROUP with factor 2 at <badref>, ir<%out0>
-; SSE2:  Cost of 120 for VF 16: INTERLEAVE-GROUP with factor 2 at <badref>, ir<%out0>
+; SSE2:  LV: Found an estimated cost of 1 for VF 1 For instruction: store i32 %v1, ptr %out1, align 4
+; SSE2:  LV: Found an estimated cost of 2 for VF 2 For instruction: store i32 %v1, ptr %out1, align 4
+; SSE2:  LV: Found an estimated cost of 30 for VF 4 For instruction: store i32 %v1, ptr %out1, align 4
+; SSE2:  LV: Found an estimated cost of 60 for VF 8 For instruction: store i32 %v1, ptr %out1, align 4
+; SSE2:  LV: Found an estimated cost of 120 for VF 16 For instruction: store i32 %v1, ptr %out1, align 4
 ;
 ; AVX1-LABEL: 'test'
 ; AVX1:  LV: Found an estimated cost of 1 for VF 1 For instruction: store i32 %v1, ptr %out1, align 4
-; AVX1:  Cost of 2 for VF 2: INTERLEAVE-GROUP with factor 2 at <badref>, ir<%out0>
-; AVX1:  Cost of 18 for VF 4: INTERLEAVE-GROUP with factor 2 at <badref>, ir<%out0>
-; AVX1:  Cost of 38 for VF 8: INTERLEAVE-GROUP with factor 2 at <badref>, ir<%out0>
-; AVX1:  Cost of 76 for VF 16: INTERLEAVE-GROUP with factor 2 at <badref>, ir<%out0>
-; AVX1:  Cost of 152 for VF 32: INTERLEAVE-GROUP with factor 2 at <badref>, ir<%out0>
+; AVX1:  LV: Found an estimated cost of 1 for VF 1 For instruction: store i32 %v1, ptr %out1, align 4
+; AVX1:  LV: Found an estimated cost of 2 for VF 2 For instruction: store i32 %v1, ptr %out1, align 4
+; AVX1:  LV: Found an estimated cost of 18 for VF 4 For instruction: store i32 %v1, ptr %out1, align 4
+; AVX1:  LV: Found an estimated cost of 38 for VF 8 For instruction: store i32 %v1, ptr %out1, align 4
+; AVX1:  LV: Found an estimated cost of 76 for VF 16 For instruction: store i32 %v1, ptr %out1, align 4
+; AVX1:  LV: Found an estimated cost of 152 for VF 32 For instruction: store i32 %v1, ptr %out1, align 4
 ;
 ; AVX2-LABEL: 'test'
 ; AVX2:  LV: Found an estimated cost of 1 for VF 1 For instruction: store i32 %v1, ptr %out1, align 4
-; AVX2:  Cost of 2 for VF 2: INTERLEAVE-GROUP with factor 2 at <badref>, ir<%out0>
-; AVX2:  Cost of 3 for VF 4: INTERLEAVE-GROUP with factor 2 at <badref>, ir<%out0>
-; AVX2:  Cost of 6 for VF 8: INTERLEAVE-GROUP with factor 2 at <badref>, ir<%out0>
-; AVX2:  Cost of 12 for VF 16: INTERLEAVE-GROUP with factor 2 at <badref>, ir<%out0>
-; AVX2:  Cost of 24 for VF 32: INTERLEAVE-GROUP with factor 2 at <badref>, ir<%out0>
+; AVX2:  LV: Found an estimated cost of 1 for VF 1 For instruction: store i32 %v1, ptr %out1, align 4
+; AVX2:  LV: Found an estimated cost of 2 for VF 2 For instruction: store i32 %v1, ptr %out1, align 4
+; AVX2:  LV: Found an estimated cost of 3 for VF 4 For instruction: store i32 %v1, ptr %out1, align 4
+; AVX2:  LV: Found an estimated cost of 6 for VF 8 For instruction: store i32 %v1, ptr %out1, align 4
+; AVX2:  LV: Found an estimated cost of 12 for VF 16 For instruction: store i32 %v1, ptr %out1, align 4
+; AVX2:  LV: Found an estimated cost of 24 for VF 32 For instruction: store i32 %v1, ptr %out1, align 4
 ;
 ; AVX512-LABEL: 'test'
 ; AVX512:  LV: Found an estimated cost of 1 for VF 1 For instruction: store i32 %v1, ptr %out1, align 4
-; AVX512:  Cost of 2 for VF 2: INTERLEAVE-GROUP with factor 2 at <badref>, ir<%out0>
-; AVX512:  Cost of 2 for VF 4: INTERLEAVE-GROUP with factor 2 at <badref>, ir<%out0>
-; AVX512:  Cost of 2 for VF 8: INTERLEAVE-GROUP with factor 2 at <badref>, ir<%out0>
-; AVX512:  Cost of 5 for VF 16: INTERLEAVE-GROUP with factor 2 at <badref>, ir<%out0>
-; AVX512:  Cost of 10 for VF 32: INTERLEAVE-GROUP with factor 2 at <badref>, ir<%out0>
-; AVX512:  Cost of 20 for VF 64: INTERLEAVE-GROUP with factor 2 at <badref>, ir<%out0>
+; AVX512:  LV: Found an estimated cost of 1 for VF 1 For instruction: store i32 %v1, ptr %out1, align 4
+; AVX512:  LV: Found an estimated cost of 2 for VF 2 For instruction: store i32 %v1, ptr %out1, align 4
+; AVX512:  LV: Found an estimated cost of 2 for VF 4 For instruction: store i32 %v1, ptr %out1, align 4
+; AVX512:  LV: Found an estimated cost of 2 for VF 8 For instruction: store i32 %v1, ptr %out1, align 4
+; AVX512:  LV: Found an estimated cost of 5 for VF 16 For instruction: store i32 %v1, ptr %out1, align 4
+; AVX512:  LV: Found an estimated cost of 10 for VF 32 For instruction: store i32 %v1, ptr %out1, align 4
+; AVX512:  LV: Found an estimated cost of 20 for VF 64 For instruction: store i32 %v1, ptr %out1, align 4
 ;
 entry:
   br label %for.body
diff --git a/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-store-i32-stride-3.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-store-i32-stride-3.ll
index 47bed35fd42bc..ad8eca60b43bd 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-store-i32-stride-3.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-store-i32-stride-3.ll
@@ -1,8 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --filter "LV: Found an estimated cost of [0-9]+ for VF [0-9]+ For instruction:\s*store i32 %v2, ptr %out2"
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse2 --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=SSE2
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx  --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX1
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2 --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX2
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512vl --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX512
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse2 --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=SSE2
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx  --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX1
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2 --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX2
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512vl --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX512
 ; REQUIRES: asserts
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
@@ -14,35 +14,39 @@ target triple = "x86_64-unknown-linux-gnu"
 define void @test() {
 ; SSE2-LABEL: 'test'
 ; SSE2:  LV: Found an estimated cost of 1 for VF 1 For instruction: store i32 %v2, ptr %out2, align 4
-; SSE2:  Cost of 23 for VF 2: INTERLEAVE-GROUP with factor 3 at <badref>, ir<%out0>
-; SSE2:  Cost of 48 for VF 4: INTERLEAVE-GROUP with factor 3 at <badref>, ir<%out0>
-; SSE2:  Cost of 96 for VF 8: INTERLEAVE-GROUP with factor 3 at <badref>, ir<%out0>
-; SSE2:  Cost of 192 for VF 16: INTERLEAVE-GROUP with factor 3 at <badref>, ir<%out0>
+; SSE2:  LV: Found an estimated cost of 1 for VF 1 For instruction: store i32 %v2, ptr %out2, align 4
+; SSE2:  LV: Found an estimated cost of 23 for VF 2 For instruction: store i32 %v2, ptr %out2, align 4
+; SSE2:  LV: Found an estimated cost of 48 for VF 4 For instruction: store i32 %v2, ptr %out2, align 4
+; SSE2:  LV: Found an estimated cost of 96 for VF 8 For instruction: store i32 %v2, ptr %out2, align 4
+; SSE2:  LV: Found an estimated cost of 192 for VF 16 For instruction: store i32 %v2, ptr %out2, align 4
 ;
 ; AVX1-LABEL: 'test'
 ; AVX1:  LV: Found an estimated cost of 1 for VF 1 For instruction: store i32 %v2, ptr %out2, align 4
-; AVX1:  Cost of 17 for VF 2: INTERLEAVE-GROUP with factor 3 at <badref>, ir<%out0>
-; AVX1:  Cost of 28 for VF 4: INTERLEAVE-GROUP with factor 3 at <badref>, ir<%out0>
-; AVX1:  Cost of 57 for VF 8: INTERLEAVE-GROUP with factor 3 at <badref>, ir<%out0>
-; AVX1:  Cost of 114 for VF 16: INTERLEAVE-GROUP with factor 3 at <badref>, ir<%out0>
-; AVX1:  Cost of 228 for VF 32: INTERLEAVE-GROUP with factor 3 at <badref>, ir<%out0>
+; AVX1:  LV: Found an estimated cost of 1 for VF 1 For instruction: store i32 %v2, ptr %out2, align 4
+; AVX1:  LV: Found an estimated cost of 17 for VF 2 For instruction: store i32 %v2, ptr %out2, align 4
+; AVX1:  LV: Found an estimated cost of 28 for VF 4 For instruction: store i32 %v2, ptr %out2, align 4
+; AVX1:  LV: Found an estimated cost of 57 for VF 8 For instruction: store i32 %v2, ptr %out2, align 4
+; AVX1:  LV: Found an estimated cost of 114 for VF 16 For instruction: store i32 %v2, ptr %out2, align 4
+; AVX1:  LV: Found an estimated cost of 228 for VF 32 For instruction: store i32 %v2, ptr %out2, align 4
 ;
 ; AVX2-LABEL: 'test'
 ; AVX2:  LV: Found an estimated cost of 1 for VF 1 For instruction: store i32 %v2, ptr %out2, align 4
-; AVX2:  Cost of 7 for VF 2: INTERLEAVE-GROUP with factor 3 at <badref>, ir<%out0>
-; AVX2:  Cost of 7 for VF 4: INTERLEAVE-GROUP with factor 3 at <badref>, ir<%out0>
-; AVX2:  Cost of 14 for VF 8: INTERLEAVE-GROUP with factor 3 at <badref>, ir<%out0>
-; AVX2:  Cost of 28 for VF 16: INTERLEAVE-GROUP with factor 3 at <badref>, ir<%out0>
-; AVX2:  Cost of 60 for VF 32: INTERLEAVE-GROUP with factor 3 at <badref>, ir<%out0>
+; AVX2:  LV: Found an estimated cost of 1 for VF 1 For instruction: store i32 %v2, ptr %out2, align 4
+; AVX2:  LV: Found an estimated cost of 7 for VF 2 For instruction: store i32 %v2, ptr %out2, align 4
+; AVX2:  LV: Found an estimated cost of 7 for VF 4 For instruction: store i32 %v2, ptr %out2, align 4
+; AVX2:  LV: Found an estimated cost of 14 for VF 8 For instruction: store i32 %v2, ptr %out2, align 4
+; AVX2:  LV: Found an estimated cost of 28 for VF 16 For instruction: store i32 %v2, ptr %out2, align 4
+; AVX2:  LV: Found an estimated cost of 60 for VF 32 For instruction: store i32 %v2, ptr %out2, align 4
 ;
 ; AVX512-LABEL: 'test'
 ; AVX512:  LV: Found an estimated cost of 1 for VF 1 For instruction: store i32 %v2, ptr %out2, align 4
-; AVX512:  Cost of 4 for VF 2: INTERLEAVE-GROUP with factor 3 at <badref>, ir<%out0>
-; AVX512:  Cost of 4 for VF 4: INTERLEAVE-GROUP with factor 3 at <badref>, ir<%out0>
-; AVX512:  Cost of 8 for VF 8: INTERLEAVE-GROUP with factor 3 at <badref>, ir<%out0>
-; AVX512:  Cost of 12 for VF 16: INTERLEAVE-GROUP with factor 3 at <badref>, ir<%out0>
-; AVX512:  Cost of 24 for VF 32: INTERLEAVE-GROUP with factor 3 at <badref>, ir<%out0>
-; AVX512:  Cost of 48 for VF 64: INTERLEAVE-GROUP with factor 3 at <badref>, ir<%out0>
+; AVX512:  LV: Found an estimated cost of 1 for VF 1 For instruction: store i32 %v2, ptr %out2, align 4
+; AVX512:  LV: Found an estimated cost of 4 for VF 2 For instruction: store i32 %v2, ptr %out2, align 4
+; AVX512:  LV: Found an estimated cost of 4 for VF 4 For instruction: store i32 %v2, ptr %out2, align 4
+; AVX512:  LV: Found an estimated cost of 8 for VF 8 For instruction: store i32 %v2, ptr %out2, align 4
+; AVX512:  LV: Found an estimated cost of 12 for VF 16 For instruction: store i32 %v2, ptr %out2, align 4
+; AVX512:  LV: Found an estimated cost of 24 for VF 32 For instruction: store i32 %v2, ptr %out2, align 4
+; AVX512:  LV: Found an estimated cost of 48 for VF 64 For instruction: store i32 %v2, ptr %out2, align 4
 ;
 entry:
   br label %for.body
diff --git a/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-store-i32-stride-4.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-store-i32-stride-4.ll
index 6e19bc197e240..6c86f56a2da3c 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-store-i32-stride-4.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-store-i32-stride-4.ll
@@ -1,8 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --filter "LV: Found an estimated cost of [0-9]+ for VF [0-9]+ For instruction:\s*store i32 %v3, ptr %out3"
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse2 --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=SSE2
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx  --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX1
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2 --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX2
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512vl --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX512
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse2 --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=SSE2
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx  --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX1
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2 --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX2
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512vl --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX512
 ; REQUIRES: asserts
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
@@ -14,35 +14,39 @@ target triple = "x86_64-unknown-linux-gnu"
 define void @test() {
 ; SSE2-LABEL: 'test'
 ; SSE2:  LV: Found an estimated cost of 1 for VF 1 For instruction: store i32 %v3, ptr %out3, align 4
-; SSE2:  Cost of 28 for VF 2: INTERLEAVE-GROUP with factor 4 at <badref>, ir<%out0>
-; SSE2:  Cost of 60 for VF 4: INTERLEAVE-GROUP with factor 4 at <badref>, ir<%out0>
-; SSE2:  Cost of 120 for VF 8: INTERLEAVE-GROUP with factor 4 at <badref>, ir<%out0>
-; SSE2:  Cost of 240 for VF 16: INTERLEAVE-GROUP with factor 4 at <badref>, ir<%out0>
+; SSE2:  LV: Found an estimated cost of 1 for VF 1 For instruction: store i32 %v3, ptr %out3, align 4
+; SSE2:  LV: Found an estimated cost of 28 for VF 2 For instruction: store i32 %v3, ptr %out3, align 4
+; SSE2:  LV: Found an estimated cost of 60 for VF 4 For instruction: store i32 %v3, ptr %out3, align 4
+; SSE2:  LV: Found an estimated cost of 120 for VF 8 For instruction: store i32 %v3, ptr %out3, align 4
+; SSE2:  LV: Found an estimated cost of 240 for VF 16 For instruction: store i32 %v3, ptr %out3, align 4
 ;
 ; AVX1-LABEL: 'test'
 ; AVX1:  LV: Found an estimated cost of 1 for VF 1 For instruction: store i32 %v3, ptr %out3, align 4
-; AVX1:  Cost of 18 for VF 2: INTERLEAVE-GROUP with factor 4 at <badref>, ir<%out0>
-; AVX1:  Cost of 36 for VF 4: INTERLEAVE-GROUP with factor 4 at <badref>, ir<%out0>
-; AVX1:  Cost of 76 for VF 8: INTERLEAVE-GROUP with factor 4 at <badref>, ir<%out0>
-; AVX1:  Cost of 152 for VF 16: INTERLEAVE-GROUP with factor 4 at <badref>, ir<%out0>
-; AVX1:  Cost of 304 for VF 32: INTERLEAVE-GROUP with factor 4 at <badref>, ir<%out0>
+; AVX1:  LV: Found an estimated cost of 1 for VF 1 For instruction: store i32 %v3, ptr %out3, align 4
+; AVX1:  LV: Found an estimated cost of 18 for VF 2 For instruction: store i32 %v3, ptr %out3, align 4
+; AVX1:  LV: Found an estimated cost of 36 for VF 4 For instruction: store i32 %v3, ptr %out3, align 4
+; AVX1:  LV: Found an estimated cost of 76 for VF 8 For instruction: store i32 %v3, ptr %out3, align 4
+; AVX1:  LV: Found an estimated cost of 152 for VF 16 For instruction: store i32 %v3, ptr %out3, align 4
+; AVX1:  LV: Found an estimated cost of 304 for VF 32 For instruction: store i32 %v3, ptr %out3, align 4
 ;
 ; AVX2-LABEL: 'test'
 ; AVX2:  LV: Found an estimated cost of 1 for VF 1 For instruction: store i32 %v3, ptr %out3, align 4
-; AVX2:  Cost of 6 for VF 2: INTERLEAVE-GROUP with factor 4 at <badref>, ir<%out0>
-; AVX2:  Cost of 8 for VF 4: INTERLEAVE-GROUP with factor 4 at <badref>, ir<%out0>
-; AVX2:  Cost of 20 for VF 8: INTERLEAVE-GROUP with factor 4 at <badref>, ir<%out0>
-; AVX2:  Cost of 40 for VF 16: INTERLEAVE-GROUP with factor 4 at <badref>, ir<%out0>
-; AVX2:  Cost of 80 for VF 32: INTERLEAVE-GROUP with factor 4 at <badref>, ir<%out0>
+; AVX2:  LV: Found an estimated cost of 1 for VF 1 For instruction: store i32 %v3, ptr %out3, align 4
+; AVX2:  LV: Found an estimated cost of 6 for VF 2 For instruction: store i32 %v3, ptr %out3, align 4
+; AVX2:  LV: Found an estimated cost of 8 for VF 4 For instruction: store i32 %v3, ptr %out3, align 4
+; AVX2:  LV: Found an estimated cost of 20 for VF 8 For instruction: store i32 %v3, ptr %out3, align 4
+; AVX2:  LV: Found an estimated cost of 40 for VF 16 For instruction: store i32 %v3, ptr %out3, align 4
+; AVX2:  LV: Found an estimated cost of 80 for VF 32 For instruction: store i32 %v3, ptr %out3, align 4
 ;
 ; AVX512-LABEL: 'test'
 ; AVX512:  LV: Found an estimated cost of 1 for VF 1 For instruction: store i32 %v3, ptr %out3, align 4
-; AVX512:  Cost of 5 for VF 2: INTERLEAVE-GROUP with factor 4 at <badref>, ir<%out0>
-; AVX512:  Cost of 5 for VF 4: INTERLEAVE-GROUP with factor 4 at <badref>, ir<%out0>
-; AVX512:  Cost of 11 for VF 8: INTERLEAVE-GROUP with factor 4 at <badref>, ir<%out0>
-; AVX512:  Cost of 22 for VF 16: INTERLEAVE-GROUP with factor 4 at <badref>, ir<%out0>
-; AVX512:  Cost of 44 for VF 32: INTERLEAVE-GROUP with factor 4 at <badref>, ir<%out0>
-; AVX512:  Cost of 88 for VF 64: INTERLEAVE-GROUP with factor 4 at <badref>, ir<%out0>
+; AVX512:  LV: Found an estimated cost of 1 for VF 1 For instruction: store i32 %v3, ptr %out3, align 4
+; AVX512:  LV: Found an estimated cost of 5 for VF 2 For instruction: store i32 %v3, ptr %out3, align 4
+; AVX512:  LV: Found an estimated cost of 5 for VF 4 For instruction: store i32 %v3, ptr %out3, align 4
+; AVX512:  LV: Found an estimated cost of 11 for VF 8 For instruction: store i32 %v3, ptr %out3, align 4
+; AVX512:  LV: Found an estimated cost of 22 for VF 16 For instruction: store i32 %v3, ptr %out3, align 4
+; AVX512:  LV: Found an estimated cost of 44 for VF 32 For instruction: store i32 %v3, ptr %out3, align 4
+; AVX512:  LV: Found an estimated cost of 88 for VF 64 For instruction: store i32 %v3, ptr %out3, align 4
 ;
 entry:
   br label %for.body
diff --git a/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-store-i32-stride-5.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-store-i32-stride-5.ll
index d109b746efc42..f4fbbec3a46f5 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-store-i32-stride-5.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-store-i32-stride-5.ll
@@ -1,8 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --filter "LV: Found an estimated cost of [0-9]+ for VF [0-9]+ For instruction:\s*store i32 %v4, ptr %out4"
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse2 --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=SSE2
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx  --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX1
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2 --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX2
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512vl --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX512
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse2 --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=SSE2
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx  --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX1
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2 --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX2
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512vl --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX512
 ; REQUIRES: asserts
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
@@ -14,32 +14,36 @@ target triple = "x86_64-unknown-linux-gnu"
 define void @test() {
 ; SSE2-LABEL: 'test'
 ; SSE2:  LV: Found an estimated cost of 1 for VF 1 For instruction: store i32 %v4, ptr %out4, align 4
-; SSE2:  Cost of 40 for VF 2: INTERLEAVE-GROUP with factor 5 at <badref>, ir<%out0>
-; SSE2:  Cost of 84 for VF 4: INTERLEAVE-GROUP with factor 5 at <badref>, ir<%out0>
-; SSE2:  Cost of 168 for VF 8: INTERLEAVE-GROUP with factor 5 at <badref>, ir<%out0>
+; SSE2:  LV: Found an estimated cost of 1 for VF 1 For instruction: store i32 %v4, ptr %out4, align 4
+; SSE2:  LV: Found an estimated cost of 40 for VF 2 For instruction: store i32 %v4, ptr %out4, align 4
+; SSE2:  LV: Found an estimated cost of 84 for VF 4 For instruction: store i32 %v4, ptr %out4, align 4
+; SSE2:  LV: Found an estimated cost of 168 for VF 8 For instruction: store i32 %v4, ptr %out4, align 4
 ;
 ; AVX1-LABEL: 'test'
 ; AVX1:  LV: Found an estimated cost of 1 for VF 1 For instruction: store i32 %v4, ptr %out4, align 4
-; AVX1:  Cost of 24 for VF 2: INTERLEAVE-GROUP with factor 5 at <badref>, ir<%out0>
-; AVX1:  Cost of 46 for VF 4: INTERLEAVE-GROUP with factor 5 at <badref>, ir<%out0>
-; AVX1:  Cost of 95 for VF 8: INTERLEAVE-GROUP with factor 5 at <badref>, ir<%out0>
-; AVX1:  Cost of 190 for VF 16: INTERLEAVE-GROUP with factor 5 at <badref>, ir<%out0>
+; AVX1:  LV: Found an estimated cost of 1 for VF 1 For instruction: store i32 %v4, ptr %out4, align 4
+; AVX1:  LV: Found an estimated cost of 24 for VF 2 For instruction: store i32 %v4, ptr %out4, align 4
+; AVX1:  LV: Found an estimated cost of 46 for VF 4 For instruction: store i32 %v4, ptr %out4, align 4
+; AVX1:  LV: Found an estimated cost of 95 for VF 8 For instruction: store i32 %v4, ptr %out4, align 4
+; AVX1:  LV: Found an estimated cost of 190 for VF 16 For instruction: store i32 %v4, ptr %out4, align 4
 ;
 ; AVX2-LABEL: 'test'
 ; AVX2:  LV: Found an estimated cost of 1 for VF 1 For instruction: store i32 %v4, ptr %out4, align 4
-; AVX2:  Cost of 24 for VF 2: INTERLEAVE-GROUP with factor 5 at <badref>, ir<%out0>
-; AVX2:  Cost of 46 for VF 4: INTERLEAVE-GROUP with factor 5 at <badref>, ir<%out0>
-; AVX2:  Cost of 95 for VF 8: INTERLEAVE-GROUP with factor 5 at <badref>, ir<%out0>
-; AVX2:  Cost of 190 for VF 16: INTERLEAVE-GROUP with factor 5 at <badref>, ir<%out0>
+; AVX2:  LV: Found an estimated cost of 1 for VF 1 For instruction: store i32 %v4, ptr %out4, align 4
+; AVX2:  LV: Found an estimated cost of 24 for VF 2 For instruction: store i32 %v4, ptr %out4, align 4
+; AVX2:  LV: Found an estimated cost of 46 for VF 4 For instruction: store i32 %v4, ptr %out4, align 4
+; AVX2:  LV: Found an estimated cost of 95 for VF 8 For instruction: store i32 %v4, ptr %out4, align 4
+; AVX2:  LV: Found an estimated cost of 190 for VF 16 For instruction: store i32 %v4, ptr %out4, align 4
 ;
 ; AVX512-LABEL: 'test'
 ; AVX512:  LV: Found an estimated cost of 1 for VF 1 For instruction: store i32 %v4, ptr %out4, align 4
-; AVX512:  Cost of 7 for VF 2: INTERLEAVE-GROUP with factor 5 at <badref>, ir<%out0>
-; AVX512:  Cost of 14 for VF 4: INTERLEAVE-GROUP with factor 5 at <badref>, ir<%out0>
-; AVX512:  Cost of 21 for VF 8: INTERLEAVE-GROUP with factor 5 at <badref>, ir<%out0>
-; AVX512:  Cost of 35 for VF 16: INTERLEAVE-GROUP with factor 5 at <badref>, ir<%out0>
-; AVX512:  Cost of 70 for VF 32: INTERLEAVE-GROUP with factor 5 at <badref>, ir<%out0>
-; AVX512:  Cost of 140 for VF 64: INTERLEAVE-GROUP with factor 5 at <badref>, ir<%out0>
+; AVX512:  LV: Found an estimated cost of 1 for VF 1 For instruction: store i32 %v4, ptr %out4, align 4
+; AVX512:  LV: Found an estimated cost of 7 for VF 2 For instruction: store i32 %v4, ptr %out4, align 4
+; AVX512:  LV: Found an estimated cost of 14 for VF 4 For instruction: store i32 %v4, ptr %out4, align 4
+; AVX512:  LV: Found an estimated cost of 21 for VF 8 For instruction: store i32 %v4, ptr %out4, align 4
+; AVX512:  LV: Found an estimated cost of 35 for VF 16 For instruction: store i32 %v4, ptr %out4, align 4
+; AVX512:  LV: Found an estimated cost of 70 for VF 32 For instruction: store i32 %v4, ptr %out4, align 4
+; AVX512:  LV: Found an estimated cost of 140 for VF 64 For instruction: store i32 %v4, ptr %out4, align 4
 ;
 entry:
   br label %for.body
diff --git a/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-store-i32-stride-6.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-store-i32-stride-6.ll
index 93ab0b7e10369..4f35f667276d8 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-store-i32-stride-6.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-store-i32-stride-6.ll
@@ -1,8 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --filter "LV: Found an estimated cost of [0-9]+ for VF [0-9]+ For instruction:\s*store i32 %v5, ptr %out5"
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse2 --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=SSE2
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx  --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX1
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2 --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX2
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512vl --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX512
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse2 --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=SSE2
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx  --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX1
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2 --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX2
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512vl --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX512
 ; REQUIRES: asserts
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
@@ -14,32 +14,36 @@ target triple = "x86_64-unknown-linux-gnu"
 define void @test() {
 ; SSE2-LABEL: 'test'
 ; SSE2:  LV: Found an estimated cost of 1 for VF 1 For instruction: store i32 %v5, ptr %out5, align 4
-; SSE2:  Cost of 45 for VF 2: INTERLEAVE-GROUP with factor 6 at <badref>, ir<%out0>
-; SSE2:  Cost of 96 for VF 4: INTERLEAVE-GROUP with factor 6 at <badref>, ir<%out0>
-; SSE2:  Cost of 192 for VF 8: INTERLEAVE-GROUP with factor 6 at <badref>, ir<%out0>
+; SSE2:  LV: Found an estimated cost of 1 for VF 1 For instruction: store i32 %v5, ptr %out5, align 4
+; SSE2:  LV: Found an estimated cost of 45 for VF 2 For instruction: store i32 %v5, ptr %out5, align 4
+; SSE2:  LV: Found an estimated cost of 96 for VF 4 For instruction: store i32 %v5, ptr %out5, align 4
+; SSE2:  LV: Found an estimated cost of 192 for VF 8 For instruction: store i32 %v5, ptr %out5, align 4
 ;
 ; AVX1-LABEL: 'test'
 ; AVX1:  LV: Found an estimated cost of 1 for VF 1 For instruction: store i32 %v5, ptr %out5, align 4
-; AVX1:  Cost of 28 for VF 2: INTERLEAVE-GROUP with factor 6 at <badref>, ir<%out0>
-; AVX1:  Cost of 54 for VF 4: INTERLEAVE-GROUP with factor 6 at <badref>, ir<%out0>
-; AVX1:  Cost of 114 for VF 8: INTERLEAVE-GROUP with factor 6 at <badref>, ir<%out0>
-; AVX1:  Cost of 228 for VF 16: INTERLEAVE-GROUP with factor 6 at <badref>, ir<%out0>
+; AVX1:  LV: Found an estimated cost of 1 for VF 1 For instruction: store i32 %v5, ptr %out5, align 4
+; AVX1:  LV: Found an estimated cost of 28 for VF 2 For instruction: store i32 %v5, ptr %out5, align 4
+; AVX1:  LV: Found an estimated cost of 54 for VF 4 For instruction: store i32 %v5, ptr %out5, align 4
+; AVX1:  LV: Found an estimated cost of 114 for VF 8 For instruction: store i32 %v5, ptr %out5, align 4
+; AVX1:  LV: Found an estimated cost of 228 for VF 16 For instruction: store i32 %v5, ptr %out5, align 4
 ;
 ; AVX2-LABEL: 'test'
 ; AVX2:  LV: Found an estimated cost of 1 for VF 1 For instruction: store i32 %v5, ptr %out5, align 4
-; AVX2:  Cost of 11 for VF 2: INTERLEAVE-GROUP with factor 6 at <badref>, ir<%out0>
-; AVX2:  Cost of 15 for VF 4: INTERLEAVE-GROUP with factor 6 at <badref>, ir<%out0>
-; AVX2:  Cost of 39 for VF 8: INTERLEAVE-GROUP with factor 6 at <badref>, ir<%out0>
-; AVX2:  Cost of 78 for VF 16: INTERLEAVE-GROUP with factor 6 at <badref>, ir<%out0>
+; AVX2:  LV: Found an estimated cost of 1 for VF 1 For instruction: store i32 %v5, ptr %out5, align 4
+; AVX2:  LV: Found an estimated cost of 11 for VF 2 For instruction: store i32 %v5, ptr %out5, align 4
+; AVX2:  LV: Found an estimated cost of 15 for VF 4 For instruction: store i32 %v5, ptr %out5, align 4
+; AVX2:  LV: Found an estimated cost of 39 for VF 8 For instruction: store i32 %v5, ptr %out5, align 4
+; AVX2:  LV: Found an estimated cost of 78 for VF 16 For instruction: store i32 %v5, ptr %out5, align 4
 ;
 ; AVX512-LABEL: 'test'
 ; AVX512:  LV: Found an estimated cost of 1 for VF 1 For instruction: store i32 %v5, ptr %out5, align 4
-; AVX512:  Cost of 8 for VF 2: INTERLEAVE-GROUP with factor 6 at <badref>, ir<%out0>
-; AVX512:  Cost of 17 for VF 4: INTERLEAVE-GROUP with factor 6 at <badref>, ir<%out0>
-; AVX512:  Cost of 25 for VF 8: INTERLEAVE-GROUP with factor 6 at <badref>, ir<%out0>
-; AVX512:  Cost of 51 for VF 16: INTERLEAVE-GROUP with factor 6 at <badref>, ir<%out0>
-; AVX512:  Cost of 102 for VF 32: INTERLEAVE-GROUP with factor 6 at <badref>, ir<%out0>
-; AVX512:  Cost of 204 for VF 64: INTERLEAVE-GROUP with factor 6 at <badref>, ir<%out0>
+; AVX512:  LV: Found an estimated cost of 1 for VF 1 For instruction: store i32 %v5, ptr %out5, align 4
+; AVX512:  LV: Found an estimated cost of 8 for VF 2 For instruction: store i32 %v5, ptr %out5, align 4
+; AVX512:  LV: Found an estimated cost of 17 for VF 4 For instruction: store i32 %v5, ptr %out5, align 4
+; AVX512:  LV: Found an estimated cost of 25 for VF 8 For instruction: store i32 %v5, ptr %out5, align 4
+; AVX512:  LV: Found an estimated cost of 51 for VF 16 For instruction: store i32 %v5, ptr %out5, align 4
+; AVX512:  LV: Found an estimated cost of 102 for VF 32 For instruction: store i32 %v5, ptr %out5, align 4
+; AVX512:  LV: Found an estimated cost of 204 for VF 64 For instruction: store i32 %v5, ptr %out5, align 4
 ;
 entry:
   br label %for.body
diff --git a/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-store-i32-stride-7.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-store-i32-stride-7.ll
index 8a01e2fbceaa2..9a4f580a7a2ca 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-store-i32-stride-7.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-store-i32-stride-7.ll
@@ -1,8 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --filter "LV: Found an estimated cost of [0-9]+ for VF [0-9]+ For instruction:\s*store i32 %v6, ptr %out6"
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse2 --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=SSE2
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx  --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX1
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2 --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX2
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512vl --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX512
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse2 --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=SSE2
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx  --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX1
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2 --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX2
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512vl --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX512
 ; REQUIRES: asserts
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
@@ -14,31 +14,35 @@ target triple = "x86_64-unknown-linux-gnu"
 define void @test() {
 ; SSE2-LABEL: 'test'
 ; SSE2:  LV: Found an estimated cost of 1 for VF 1 For instruction: store i32 %v6, ptr %out6, align 4
-; SSE2:  Cost of 51 for VF 2: INTERLEAVE-GROUP with factor 7 at <badref>, ir<%out0>
-; SSE2:  Cost of 108 for VF 4: INTERLEAVE-GROUP with factor 7 at <badref>, ir<%out0>
-; SSE2:  Cost of 216 for VF 8: INTERLEAVE-GROUP with factor 7 at <badref>, ir<%out0>
+; SSE2:  LV: Found an estimated cost of 1 for VF 1 For instruction: store i32 %v6, ptr %out6, align 4
+; SSE2:  LV: Found an estimated cost of 51 for VF 2 For instruction: store i32 %v6, ptr %out6, align 4
+; SSE2:  LV: Found an estimated cost of 108 for VF 4 For instruction: store i32 %v6, ptr %out6, align 4
+; SSE2:  LV: Found an estimated cost of 216 for VF 8 For instruction: store i32 %v6, ptr %out6, align 4
 ;
 ; AVX1-LABEL: 'test'
 ; AVX1:  LV: Found an estimated cost of 1 for VF 1 For instruction: store i32 %v6, ptr %out6, align 4
-; AVX1:  Cost of 35 for VF 2: INTERLEAVE-GROUP with factor 7 at <badref>, ir<%out0>
-; AVX1:  Cost of 64 for VF 4: INTERLEAVE-GROUP with factor 7 at <badref>, ir<%out0>
-; AVX1:  Cost of 133 for VF 8: INTERLEAVE-GROUP with factor 7 at <badref>, ir<%out0>
-; AVX1:  Cost of 266 for VF 16: INTERLEAVE-GROUP with factor 7 at <badref>, ir<%out0>
+; AVX1:  LV: Found an estimated cost of 1 for VF 1 For instruction: store i32 %v6, ptr %out6, align 4
+; AVX1:  LV: Found an estimated cost of 35 for VF 2 For instruction: store i32 %v6, ptr %out6, align 4
+; AVX1:  LV: Found an estimated cost of 64 for VF 4 For instruction: store i32 %v6, ptr %out6, align 4
+; AVX1:  LV: Found an estimated cost of 133 for VF 8 For instruction: store i32 %v6, ptr %out6, align 4
+; AVX1:  LV: Found an estimated cost of 266 for VF 16 For instruction: store i32 %v6, ptr %out6, align 4
 ;
 ; AVX2-LABEL: 'test'
 ; AVX2:  LV: Found an estimated cost of 1 for VF 1 For instruction: store i32 %v6, ptr %out6, align 4
-; AVX2:  Cost of 35 for VF 2: INTERLEAVE-GROUP with factor 7 at <badref>, ir<%out0>
-; AVX2:  Cost of 64 for VF 4: INTERLEAVE-GROUP with factor 7 at <badref>, ir<%out0>
-; AVX2:  Cost of 133 for VF 8: INTERLEAVE-GROUP with factor 7 at <badref>, ir<%out0>
-; AVX2:  Cost of 266 for VF 16: INTERLEAVE-GROUP with factor 7 at <badref>, ir<%out0>
+; AVX2:  LV: Found an estimated cost of 1 for VF 1 For instruction: store i32 %v6, ptr %out6, align 4
+; AVX2:  LV: Found an estimated cost of 35 for VF 2 For instruction: store i32 %v6, ptr %out6, align 4
+; AVX2:  LV: Found an estimated cost of 64 for VF 4 For instruction: store i32 %v6, ptr %out6, align 4
+; AVX2:  LV: Found an estimated cost of 133 for VF 8 For instruction: store i32 %v6, ptr %out6, align 4
+; AVX2:  LV: Found an estimated cost of 266 for VF 16 For instruction: store i32 %v6, ptr %out6, align 4
 ;
 ; AVX512-LABEL: 'test'
 ; AVX512:  LV: Found an estimated cost of 1 for VF 1 For instruction: store i32 %v6, ptr %out6, align 4
-; AVX512:  Cost of 10 for VF 2: INTERLEAVE-GROUP with factor 7 at <badref>, ir<%out0>
-; AVX512:  Cost of 20 for VF 4: INTERLEAVE-GROUP with factor 7 at <badref>, ir<%out0>
-; AVX512:  Cost of 40 for VF 8: INTERLEAVE-GROUP with factor 7 at <badref>, ir<%out0>
-; AVX512:  Cost of 70 for VF 16: INTERLEAVE-GROUP with factor 7 at <badref>, ir<%out0>
-; AVX512:  Cost of 140 for VF 32: INTERLEAVE-GROUP with factor 7 at <badref>, ir<%out0>
+; AVX512:  LV: Found an estimated cost of 1 for VF 1 For instruction: store i32 %v6, ptr %out6, align 4
+; AVX512:  LV: Found an estimated cost of 10 for VF 2 For instruction: store i32 %v6, ptr %out6, align 4
+; AVX512:  LV: Found an estimated cost of 20 for VF 4 For instruction: store i32 %v6, ptr %out6, align 4
+; AVX512:  LV: Found an estimated cost of 40 for VF 8 For instruction: store i32 %v6, ptr %out6, align 4
+; AVX512:  LV: Found an estimated cost of 70 for VF 16 For instruction: store i32 %v6, ptr %out6, align 4
+; AVX512:  LV: Found an estimated cost of 140 for VF 32 For instruction: store i32 %v6, ptr %out6, align 4
 ;
 entry:
   br label %for.body
diff --git a/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-store-i32-stride-8.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-store-i32-stride-8.ll
index cf0efab954813..2c4ca9993f53d 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-store-i32-stride-8.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-store-i32-stride-8.ll
@@ -1,8 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --filter "LV: Found an estimated cost of [0-9]+ for VF [0-9]+ For instruction:\s*store i32 %v7, ptr %out7"
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse2 --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=SSE2
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx  --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX1
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2 --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX2
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512vl --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX512
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse2 --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=SSE2
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx  --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX1
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2 --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX2
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512vl --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX512
 ; REQUIRES: asserts
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
@@ -14,12 +14,14 @@ target triple = "x86_64-unknown-linux-gnu"
 define void @test() {
 ; SSE2-LABEL: 'test'
 ; SSE2:  LV: Found an estimated cost of 1 for VF 1 For instruction: store i32 %v7, ptr %out7, align 4
+; SSE2:  LV: Found an estimated cost of 1 for VF 1 For instruction: store i32 %v7, ptr %out7, align 4
 ; SSE2:  LV: Found an estimated cost of 56 for VF 2 For instruction: store i32 %v7, ptr %out7, align 4
 ; SSE2:  LV: Found an estimated cost of 120 for VF 4 For instruction: store i32 %v7, ptr %out7, align 4
 ; SSE2:  LV: Found an estimated cost of 240 for VF 8 For instruction: store i32 %v7, ptr %out7, align 4
 ;
 ; AVX1-LABEL: 'test'
 ; AVX1:  LV: Found an estimated cost of 1 for VF 1 For instruction: store i32 %v7, ptr %out7, align 4
+; AVX1:  LV: Found an estimated cost of 1 for VF 1 For instruction: store i32 %v7, ptr %out7, align 4
 ; AVX1:  LV: Found an estimated cost of 36 for VF 2 For instruction: store i32 %v7, ptr %out7, align 4
 ; AVX1:  LV: Found an estimated cost of 72 for VF 4 For instruction: store i32 %v7, ptr %out7, align 4
 ; AVX1:  LV: Found an estimated cost of 152 for VF 8 For instruction: store i32 %v7, ptr %out7, align 4
@@ -27,6 +29,7 @@ define void @test() {
 ;
 ; AVX2-LABEL: 'test'
 ; AVX2:  LV: Found an estimated cost of 1 for VF 1 For instruction: store i32 %v7, ptr %out7, align 4
+; AVX2:  LV: Found an estimated cost of 1 for VF 1 For instruction: store i32 %v7, ptr %out7, align 4
 ; AVX2:  LV: Found an estimated cost of 36 for VF 2 For instruction: store i32 %v7, ptr %out7, align 4
 ; AVX2:  LV: Found an estimated cost of 72 for VF 4 For instruction: store i32 %v7, ptr %out7, align 4
 ; AVX2:  LV: Found an estimated cost of 152 for VF 8 For instruction: store i32 %v7, ptr %out7, align 4
@@ -34,6 +37,7 @@ define void @test() {
 ;
 ; AVX512-LABEL: 'test'
 ; AVX512:  LV: Found an estimated cost of 1 for VF 1 For instruction: store i32 %v7, ptr %out7, align 4
+; AVX512:  LV: Found an estimated cost of 1 for VF 1 For instruction: store i32 %v7, ptr %out7, align 4
 ; AVX512:  LV: Found an estimated cost of 11 for VF 2 For instruction: store i32 %v7, ptr %out7, align 4
 ; AVX512:  LV: Found an estimated cost of 23 for VF 4 For instruction: store i32 %v7, ptr %out7, align 4
 ; AVX512:  LV: Found an estimated cost of 46 for VF 8 For instruction: store i32 %v7, ptr %out7, align 4
diff --git a/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-store-i64-stride-2.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-store-i64-stride-2.ll
index 895d7a7623dba..6fbc678408f6b 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-store-i64-stride-2.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-store-i64-stride-2.ll
@@ -1,8 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --filter "LV: Found an estimated cost of [0-9]+ for VF [0-9]+ For instruction:\s*store i64 %v1, ptr %out1"
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse2 --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=SSE2
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx  --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX1
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2 --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX2
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512vl --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX512
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse2 --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=SSE2
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx  --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX1
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2 --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX2
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512vl --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX512
 ; REQUIRES: asserts
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
@@ -14,35 +14,39 @@ target triple = "x86_64-unknown-linux-gnu"
 define void @test() {
 ; SSE2-LABEL: 'test'
 ; SSE2:  LV: Found an estimated cost of 1 for VF 1 For instruction: store i64 %v1, ptr %out1, align 8
-; SSE2:  Cost of 14 for VF 2: INTERLEAVE-GROUP with factor 2 at <badref>, ir<%out0>
-; SSE2:  Cost of 28 for VF 4: INTERLEAVE-GROUP with factor 2 at <badref>, ir<%out0>
-; SSE2:  Cost of 56 for VF 8: INTERLEAVE-GROUP with factor 2 at <badref>, ir<%out0>
-; SSE2:  Cost of 112 for VF 16: INTERLEAVE-GROUP with factor 2 at <badref>, ir<%out0>
+; SSE2:  LV: Found an estimated cost of 1 for VF 1 For instruction: store i64 %v1, ptr %out1, align 8
+; SSE2:  LV: Found an estimated cost of 14 for VF 2 For instruction: store i64 %v1, ptr %out1, align 8
+; SSE2:  LV: Found an estimated cost of 28 for VF 4 For instruction: store i64 %v1, ptr %out1, align 8
+; SSE2:  LV: Found an estimated cost of 56 for VF 8 For instruction: store i64 %v1, ptr %out1, align 8
+; SSE2:  LV: Found an estimated cost of 112 for VF 16 For instruction: store i64 %v1, ptr %out1, align 8
 ;
 ; AVX1-LABEL: 'test'
 ; AVX1:  LV: Found an estimated cost of 1 for VF 1 For instruction: store i64 %v1, ptr %out1, align 8
-; AVX1:  Cost of 10 for VF 2: INTERLEAVE-GROUP with factor 2 at <badref>, ir<%out0>
-; AVX1:  Cost of 22 for VF 4: INTERLEAVE-GROUP with factor 2 at <badref>, ir<%out0>
-; AVX1:  Cost of 44 for VF 8: INTERLEAVE-GROUP with factor 2 at <badref>, ir<%out0>
-; AVX1:  Cost of 88 for VF 16: INTERLEAVE-GROUP with factor 2 at <badref>, ir<%out0>
-; AVX1:  Cost of 176 for VF 32: INTERLEAVE-GROUP with factor 2 at <badref>, ir<%out0>
+; AVX1:  LV: Found an estimated cost of 1 for VF 1 For instruction: store i64 %v1, ptr %out1, align 8
+; AVX1:  LV: Found an estimated cost of 10 for VF 2 For instruction: store i64 %v1, ptr %out1, align 8
+; AVX1:  LV: Found an estimated cost of 22 for VF 4 For instruction: store i64 %v1, ptr %out1, align 8
+; AVX1:  LV: Found an estimated cost of 44 for VF 8 For instruction: store i64 %v1, ptr %out1, align 8
+; AVX1:  LV: Found an estimated cost of 88 for VF 16 For instruction: store i64 %v1, ptr %out1, align 8
+; AVX1:  LV: Found an estimated cost of 176 for VF 32 For instruction: store i64 %v1, ptr %out1, align 8
 ;
 ; AVX2-LABEL: 'test'
 ; AVX2:  LV: Found an estimated cost of 1 for VF 1 For instruction: store i64 %v1, ptr %out1, align 8
-; AVX2:  Cost of 3 for VF 2: INTERLEAVE-GROUP with factor 2 at <badref>, ir<%out0>
-; AVX2:  Cost of 6 for VF 4: INTERLEAVE-GROUP with factor 2 at <badref>, ir<%out0>
-; AVX2:  Cost of 12 for VF 8: INTERLEAVE-GROUP with factor 2 at <badref>, ir<%out0>
-; AVX2:  Cost of 24 for VF 16: INTERLEAVE-GROUP with factor 2 at <badref>, ir<%out0>
-; AVX2:  Cost of 48 for VF 32: INTERLEAVE-GROUP with factor 2 at <badref>, ir<%out0>
+; AVX2:  LV: Found an estimated cost of 1 for VF 1 For instruction: store i64 %v1, ptr %out1, align 8
+; AVX2:  LV: Found an estimated cost of 3 for VF 2 For instruction: store i64 %v1, ptr %out1, align 8
+; AVX2:  LV: Found an estimated cost of 6 for VF 4 For instruction: store i64 %v1, ptr %out1, align 8
+; AVX2:  LV: Found an estimated cost of 12 for VF 8 For instruction: store i64 %v1, ptr %out1, align 8
+; AVX2:  LV: Found an estimated cost of 24 for VF 16 For instruction: store i64 %v1, ptr %out1, align 8
+; AVX2:  LV: Found an estimated cost of 48 for VF 32 For instruction: store i64 %v1, ptr %out1, align 8
 ;
 ; AVX512-LABEL: 'test'
 ; AVX512:  LV: Found an estimated cost of 1 for VF 1 For instruction: store i64 %v1, ptr %out1, align 8
-; AVX512:  Cost of 2 for VF 2: INTERLEAVE-GROUP with factor 2 at <badref>, ir<%out0>
-; AVX512:  Cost of 2 for VF 4: INTERLEAVE-GROUP with factor 2 at <badref>, ir<%out0>
-; AVX512:  Cost of 5 for VF 8: INTERLEAVE-GROUP with factor 2 at <badref>, ir<%out0>
-; AVX512:  Cost of 10 for VF 16: INTERLEAVE-GROUP with factor 2 at <badref>, ir<%out0>
-; AVX512:  Cost of 20 for VF 32: INTERLEAVE-GROUP with factor 2 at <badref>, ir<%out0>
-; AVX512:  Cost of 40 for VF 64: INTERLEAVE-GROUP with factor 2 at <badref>, ir<%out0>
+; AVX512:  LV: Found an estimated cost of 1 for VF 1 For instruction: store i64 %v1, ptr %out1, align 8
+; AVX512:  LV: Found an estimated cost of 2 for VF 2 For instruction: store i64 %v1, ptr %out1, align 8
+; AVX512:  LV: Found an estimated cost of 2 for VF 4 For instruction: store i64 %v1, ptr %out1, align 8
+; AVX512:  LV: Found an estimated cost of 5 for VF 8 For instruction: store i64 %v1, ptr %out1, align 8
+; AVX512:  LV: Found an estimated cost of 10 for VF 16 For instruction: store i64 %v1, ptr %out1, align 8
+; AVX512:  LV: Found an estimated cost of 20 for VF 32 For instruction: store i64 %v1, ptr %out1, align 8
+; AVX512:  LV: Found an estimated cost of 40 for VF 64 For instruction: store i64 %v1, ptr %out1, align 8
 ;
 entry:
   br label %for.body
diff --git a/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-store-i64-stride-3.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-store-i64-stride-3.ll
index c4cc65212ffa9..fe1dad3c3effc 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-store-i64-stride-3.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-store-i64-stride-3.ll
@@ -1,8 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --filter "LV: Found an estimated cost of [0-9]+ for VF [0-9]+ For instruction:\s*store i64 %v2, ptr %out2"
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse2 --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=SSE2
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx  --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX1
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2 --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX2
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512vl --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX512
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse2 --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=SSE2
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx  --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX1
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2 --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX2
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512vl --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX512
 ; REQUIRES: asserts
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
@@ -14,32 +14,36 @@ target triple = "x86_64-unknown-linux-gnu"
 define void @test() {
 ; SSE2-LABEL: 'test'
 ; SSE2:  LV: Found an estimated cost of 1 for VF 1 For instruction: store i64 %v2, ptr %out2, align 8
-; SSE2:  Cost of 22 for VF 2: INTERLEAVE-GROUP with factor 3 at <badref>, ir<%out0>
-; SSE2:  Cost of 44 for VF 4: INTERLEAVE-GROUP with factor 3 at <badref>, ir<%out0>
-; SSE2:  Cost of 88 for VF 8: INTERLEAVE-GROUP with factor 3 at <badref>, ir<%out0>
+; SSE2:  LV: Found an estimated cost of 1 for VF 1 For instruction: store i64 %v2, ptr %out2, align 8
+; SSE2:  LV: Found an estimated cost of 22 for VF 2 For instruction: store i64 %v2, ptr %out2, align 8
+; SSE2:  LV: Found an estimated cost of 44 for VF 4 For instruction: store i64 %v2, ptr %out2, align 8
+; SSE2:  LV: Found an estimated cost of 88 for VF 8 For instruction: store i64 %v2, ptr %out2, align 8
 ;
 ; AVX1-LABEL: 'test'
 ; AVX1:  LV: Found an estimated cost of 1 for VF 1 For instruction: store i64 %v2, ptr %out2, align 8
-; AVX1:  Cost of 16 for VF 2: INTERLEAVE-GROUP with factor 3 at <badref>, ir<%out0>
-; AVX1:  Cost of 33 for VF 4: INTERLEAVE-GROUP with factor 3 at <badref>, ir<%out0>
-; AVX1:  Cost of 66 for VF 8: INTERLEAVE-GROUP with factor 3 at <badref>, ir<%out0>
-; AVX1:  Cost of 132 for VF 16: INTERLEAVE-GROUP with factor 3 at <badref>, ir<%out0>
+; AVX1:  LV: Found an estimated cost of 1 for VF 1 For instruction: store i64 %v2, ptr %out2, align 8
+; AVX1:  LV: Found an estimated cost of 16 for VF 2 For instruction: store i64 %v2, ptr %out2, align 8
+; AVX1:  LV: Found an estimated cost of 33 for VF 4 For instruction: store i64 %v2, ptr %out2, align 8
+; AVX1:  LV: Found an estimated cost of 66 for VF 8 For instruction: store i64 %v2, ptr %out2, align 8
+; AVX1:  LV: Found an estimated cost of 132 for VF 16 For instruction: store i64 %v2, ptr %out2, align 8
 ;
 ; AVX2-LABEL: 'test'
 ; AVX2:  LV: Found an estimated cost of 1 for VF 1 For instruction: store i64 %v2, ptr %out2, align 8
-; AVX2:  Cost of 6 for VF 2: INTERLEAVE-GROUP with factor 3 at <badref>, ir<%out0>
-; AVX2:  Cost of 9 for VF 4: INTERLEAVE-GROUP with factor 3 at <badref>, ir<%out0>
-; AVX2:  Cost of 18 for VF 8: INTERLEAVE-GROUP with factor 3 at <badref>, ir<%out0>
-; AVX2:  Cost of 36 for VF 16: INTERLEAVE-GROUP with factor 3 at <badref>, ir<%out0>
+; AVX2:  LV: Found an estimated cost of 1 for VF 1 For instruction: store i64 %v2, ptr %out2, align 8
+; AVX2:  LV: Found an estimated cost of 6 for VF 2 For instruction: store i64 %v2, ptr %out2, align 8
+; AVX2:  LV: Found an estimated cost of 9 for VF 4 For instruction: store i64 %v2, ptr %out2, align 8
+; AVX2:  LV: Found an estimated cost of 18 for VF 8 For instruction: store i64 %v2, ptr %out2, align 8
+; AVX2:  LV: Found an estimated cost of 36 for VF 16 For instruction: store i64 %v2, ptr %out2, align 8
 ;
 ; AVX512-LABEL: 'test'
 ; AVX512:  LV: Found an estimated cost of 1 for VF 1 For instruction: store i64 %v2, ptr %out2, align 8
-; AVX512:  Cost of 4 for VF 2: INTERLEAVE-GROUP with factor 3 at <badref>, ir<%out0>
-; AVX512:  Cost of 8 for VF 4: INTERLEAVE-GROUP with factor 3 at <badref>, ir<%out0>
-; AVX512:  Cost of 12 for VF 8: INTERLEAVE-GROUP with factor 3 at <badref>, ir<%out0>
-; AVX512:  Cost of 24 for VF 16: INTERLEAVE-GROUP with factor 3 at <badref>, ir<%out0>
-; AVX512:  Cost of 48 for VF 32: INTERLEAVE-GROUP with factor 3 at <badref>, ir<%out0>
-; AVX512:  Cost of 96 for VF 64: INTERLEAVE-GROUP with factor 3 at <badref>, ir<%out0>
+; AVX512:  LV: Found an estimated cost of 1 for VF 1 For instruction: store i64 %v2, ptr %out2, align 8
+; AVX512:  LV: Found an estimated cost of 4 for VF 2 For instruction: store i64 %v2, ptr %out2, align 8
+; AVX512:  LV: Found an estimated cost of 8 for VF 4 For instruction: store i64 %v2, ptr %out2, align 8
+; AVX512:  LV: Found an estimated cost of 12 for VF 8 For instruction: store i64 %v2, ptr %out2, align 8
+; AVX512:  LV: Found an estimated cost of 24 for VF 16 For instruction: store i64 %v2, ptr %out2, align 8
+; AVX512:  LV: Found an estimated cost of 48 for VF 32 For instruction: store i64 %v2, ptr %out2, align 8
+; AVX512:  LV: Found an estimated cost of 96 for VF 64 For instruction: store i64 %v2, ptr %out2, align 8
 ;
 entry:
   br label %for.body
diff --git a/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-store-i64-stride-4.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-store-i64-stride-4.ll
index 6b7be24b54f6b..4ebb2283f9b7f 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-store-i64-stride-4.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-store-i64-stride-4.ll
@@ -1,8 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --filter "LV: Found an estimated cost of [0-9]+ for VF [0-9]+ For instruction:\s*store i64 %v3, ptr %out3"
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse2 --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=SSE2
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx  --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX1
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2 --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX2
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512vl --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX512
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse2 --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=SSE2
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx  --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX1
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2 --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX2
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512vl --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX512
 ; REQUIRES: asserts
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
@@ -14,31 +14,35 @@ target triple = "x86_64-unknown-linux-gnu"
 define void @test() {
 ; SSE2-LABEL: 'test'
 ; SSE2:  LV: Found an estimated cost of 1 for VF 1 For instruction: store i64 %v3, ptr %out3, align 8
-; SSE2:  Cost of 28 for VF 2: INTERLEAVE-GROUP with factor 4 at <badref>, ir<%out0>
-; SSE2:  Cost of 56 for VF 4: INTERLEAVE-GROUP with factor 4 at <badref>, ir<%out0>
-; SSE2:  Cost of 112 for VF 8: INTERLEAVE-GROUP with factor 4 at <badref>, ir<%out0>
+; SSE2:  LV: Found an estimated cost of 1 for VF 1 For instruction: store i64 %v3, ptr %out3, align 8
+; SSE2:  LV: Found an estimated cost of 28 for VF 2 For instruction: store i64 %v3, ptr %out3, align 8
+; SSE2:  LV: Found an estimated cost of 56 for VF 4 For instruction: store i64 %v3, ptr %out3, align 8
+; SSE2:  LV: Found an estimated cost of 112 for VF 8 For instruction: store i64 %v3, ptr %out3, align 8
 ;
 ; AVX1-LABEL: 'test'
 ; AVX1:  LV: Found an estimated cost of 1 for VF 1 For instruction: store i64 %v3, ptr %out3, align 8
-; AVX1:  Cost of 20 for VF 2: INTERLEAVE-GROUP with factor 4 at <badref>, ir<%out0>
-; AVX1:  Cost of 44 for VF 4: INTERLEAVE-GROUP with factor 4 at <badref>, ir<%out0>
-; AVX1:  Cost of 88 for VF 8: INTERLEAVE-GROUP with factor 4 at <badref>, ir<%out0>
-; AVX1:  Cost of 176 for VF 16: INTERLEAVE-GROUP with factor 4 at <badref>, ir<%out0>
+; AVX1:  LV: Found an estimated cost of 1 for VF 1 For instruction: store i64 %v3, ptr %out3, align 8
+; AVX1:  LV: Found an estimated cost of 20 for VF 2 For instruction: store i64 %v3, ptr %out3, align 8
+; AVX1:  LV: Found an estimated cost of 44 for VF 4 For instruction: store i64 %v3, ptr %out3, align 8
+; AVX1:  LV: Found an estimated cost of 88 for VF 8 For instruction: store i64 %v3, ptr %out3, align 8
+; AVX1:  LV: Found an estimated cost of 176 for VF 16 For instruction: store i64 %v3, ptr %out3, align 8
 ;
 ; AVX2-LABEL: 'test'
 ; AVX2:  LV: Found an estimated cost of 1 for VF 1 For instruction: store i64 %v3, ptr %out3, align 8
-; AVX2:  Cost of 8 for VF 2: INTERLEAVE-GROUP with factor 4 at <badref>, ir<%out0>
-; AVX2:  Cost of 12 for VF 4: INTERLEAVE-GROUP with factor 4 at <badref>, ir<%out0>
-; AVX2:  Cost of 28 for VF 8: INTERLEAVE-GROUP with factor 4 at <badref>, ir<%out0>
-; AVX2:  Cost of 56 for VF 16: INTERLEAVE-GROUP with factor 4 at <badref>, ir<%out0>
+; AVX2:  LV: Found an estimated cost of 1 for VF 1 For instruction: store i64 %v3, ptr %out3, align 8
+; AVX2:  LV: Found an estimated cost of 8 for VF 2 For instruction: store i64 %v3, ptr %out3, align 8
+; AVX2:  LV: Found an estimated cost of 12 for VF 4 For instruction: store i64 %v3, ptr %out3, align 8
+; AVX2:  LV: Found an estimated cost of 28 for VF 8 For instruction: store i64 %v3, ptr %out3, align 8
+; AVX2:  LV: Found an estimated cost of 56 for VF 16 For instruction: store i64 %v3, ptr %out3, align 8
 ;
 ; AVX512-LABEL: 'test'
 ; AVX512:  LV: Found an estimated cost of 1 for VF 1 For instruction: store i64 %v3, ptr %out3, align 8
-; AVX512:  Cost of 5 for VF 2: INTERLEAVE-GROUP with factor 4 at <badref>, ir<%out0>
-; AVX512:  Cost of 11 for VF 4: INTERLEAVE-GROUP with factor 4 at <badref>, ir<%out0>
-; AVX512:  Cost of 22 for VF 8: INTERLEAVE-GROUP with factor 4 at <badref>, ir<%out0>
-; AVX512:  Cost of 44 for VF 16: INTERLEAVE-GROUP with factor 4 at <badref>, ir<%out0>
-; AVX512:  Cost of 88 for VF 32: INTERLEAVE-GROUP with factor 4 at <badref>, ir<%out0>
+; AVX512:  LV: Found an estimated cost of 1 for VF 1 For instruction: store i64 %v3, ptr %out3, align 8
+; AVX512:  LV: Found an estimated cost of 5 for VF 2 For instruction: store i64 %v3, ptr %out3, align 8
+; AVX512:  LV: Found an estimated cost of 11 for VF 4 For instruction: store i64 %v3, ptr %out3, align 8
+; AVX512:  LV: Found an estimated cost of 22 for VF 8 For instruction: store i64 %v3, ptr %out3, align 8
+; AVX512:  LV: Found an estimated cost of 44 for VF 16 For instruction: store i64 %v3, ptr %out3, align 8
+; AVX512:  LV: Found an estimated cost of 88 for VF 32 For instruction: store i64 %v3, ptr %out3, align 8
 ;
 entry:
   br label %for.body
diff --git a/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-store-i64-stride-5.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-store-i64-stride-5.ll
index 52a4d4f9eb658..79c64ecfe457d 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-store-i64-stride-5.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-store-i64-stride-5.ll
@@ -1,8 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --filter "LV: Found an estimated cost of [0-9]+ for VF [0-9]+ For instruction:\s*store i64 %v4, ptr %out4"
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse2 --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=SSE2
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx  --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX1
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2 --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX2
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512vl --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX512
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse2 --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=SSE2
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx  --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX1
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2 --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX2
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512vl --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX512
 ; REQUIRES: asserts
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
@@ -14,28 +14,32 @@ target triple = "x86_64-unknown-linux-gnu"
 define void @test() {
 ; SSE2-LABEL: 'test'
 ; SSE2:  LV: Found an estimated cost of 1 for VF 1 For instruction: store i64 %v4, ptr %out4, align 8
-; SSE2:  Cost of 38 for VF 2: INTERLEAVE-GROUP with factor 5 at <badref>, ir<%out0>
-; SSE2:  Cost of 76 for VF 4: INTERLEAVE-GROUP with factor 5 at <badref>, ir<%out0>
+; SSE2:  LV: Found an estimated cost of 1 for VF 1 For instruction: store i64 %v4, ptr %out4, align 8
+; SSE2:  LV: Found an estimated cost of 38 for VF 2 For instruction: store i64 %v4, ptr %out4, align 8
+; SSE2:  LV: Found an estimated cost of 76 for VF 4 For instruction: store i64 %v4, ptr %out4, align 8
 ;
 ; AVX1-LABEL: 'test'
 ; AVX1:  LV: Found an estimated cost of 1 for VF 1 For instruction: store i64 %v4, ptr %out4, align 8
-; AVX1:  Cost of 26 for VF 2: INTERLEAVE-GROUP with factor 5 at <badref>, ir<%out0>
-; AVX1:  Cost of 55 for VF 4: INTERLEAVE-GROUP with factor 5 at <badref>, ir<%out0>
-; AVX1:  Cost of 110 for VF 8: INTERLEAVE-GROUP with factor 5 at <badref>, ir<%out0>
+; AVX1:  LV: Found an estimated cost of 1 for VF 1 For instruction: store i64 %v4, ptr %out4, align 8
+; AVX1:  LV: Found an estimated cost of 26 for VF 2 For instruction: store i64 %v4, ptr %out4, align 8
+; AVX1:  LV: Found an estimated cost of 55 for VF 4 For instruction: store i64 %v4, ptr %out4, align 8
+; AVX1:  LV: Found an estimated cost of 110 for VF 8 For instruction: store i64 %v4, ptr %out4, align 8
 ;
 ; AVX2-LABEL: 'test'
 ; AVX2:  LV: Found an estimated cost of 1 for VF 1 For instruction: store i64 %v4, ptr %out4, align 8
-; AVX2:  Cost of 26 for VF 2: INTERLEAVE-GROUP with factor 5 at <badref>, ir<%out0>
-; AVX2:  Cost of 55 for VF 4: INTERLEAVE-GROUP with factor 5 at <badref>, ir<%out0>
-; AVX2:  Cost of 110 for VF 8: INTERLEAVE-GROUP with factor 5 at <badref>, ir<%out0>
+; AVX2:  LV: Found an estimated cost of 1 for VF 1 For instruction: store i64 %v4, ptr %out4, align 8
+; AVX2:  LV: Found an estimated cost of 26 for VF 2 For instruction: store i64 %v4, ptr %out4, align 8
+; AVX2:  LV: Found an estimated cost of 55 for VF 4 For instruction: store i64 %v4, ptr %out4, align 8
+; AVX2:  LV: Found an estimated cost of 110 for VF 8 For instruction: store i64 %v4, ptr %out4, align 8
 ;
 ; AVX512-LABEL: 'test'
 ; AVX512:  LV: Found an estimated cost of 1 for VF 1 For instruction: store i64 %v4, ptr %out4, align 8
-; AVX512:  Cost of 14 for VF 2: INTERLEAVE-GROUP with factor 5 at <badref>, ir<%out0>
-; AVX512:  Cost of 21 for VF 4: INTERLEAVE-GROUP with factor 5 at <badref>, ir<%out0>
-; AVX512:  Cost of 35 for VF 8: INTERLEAVE-GROUP with factor 5 at <badref>, ir<%out0>
-; AVX512:  Cost of 70 for VF 16: INTERLEAVE-GROUP with factor 5 at <badref>, ir<%out0>
-; AVX512:  Cost of 140 for VF 32: INTERLEAVE-GROUP with factor 5 at <badref>, ir<%out0>
+; AVX512:  LV: Found an estimated cost of 1 for VF 1 For instruction: store i64 %v4, ptr %out4, align 8
+; AVX512:  LV: Found an estimated cost of 14 for VF 2 For instruction: store i64 %v4, ptr %out4, align 8
+; AVX512:  LV: Found an estimated cost of 21 for VF 4 For instruction: store i64 %v4, ptr %out4, align 8
+; AVX512:  LV: Found an estimated cost of 35 for VF 8 For instruction: store i64 %v4, ptr %out4, align 8
+; AVX512:  LV: Found an estimated cost of 70 for VF 16 For instruction: store i64 %v4, ptr %out4, align 8
+; AVX512:  LV: Found an estimated cost of 140 for VF 32 For instruction: store i64 %v4, ptr %out4, align 8
 ;
 entry:
   br label %for.body
diff --git a/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-store-i64-stride-6.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-store-i64-stride-6.ll
index 14f36529a13b3..05909fa7a1fb9 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-store-i64-stride-6.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-store-i64-stride-6.ll
@@ -1,8 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --filter "LV: Found an estimated cost of [0-9]+ for VF [0-9]+ For instruction:\s*store i64 %v5, ptr %out5"
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse2 --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=SSE2
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx  --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX1
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2 --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX2
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512vl --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX512
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse2 --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=SSE2
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx  --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX1
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2 --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX2
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512vl --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX512
 ; REQUIRES: asserts
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
@@ -14,28 +14,32 @@ target triple = "x86_64-unknown-linux-gnu"
 define void @test() {
 ; SSE2-LABEL: 'test'
 ; SSE2:  LV: Found an estimated cost of 1 for VF 1 For instruction: store i64 %v5, ptr %out5, align 8
-; SSE2:  Cost of 44 for VF 2: INTERLEAVE-GROUP with factor 6 at <badref>, ir<%out0>
-; SSE2:  Cost of 88 for VF 4: INTERLEAVE-GROUP with factor 6 at <badref>, ir<%out0>
+; SSE2:  LV: Found an estimated cost of 1 for VF 1 For instruction: store i64 %v5, ptr %out5, align 8
+; SSE2:  LV: Found an estimated cost of 44 for VF 2 For instruction: store i64 %v5, ptr %out5, align 8
+; SSE2:  LV: Found an estimated cost of 88 for VF 4 For instruction: store i64 %v5, ptr %out5, align 8
 ;
 ; AVX1-LABEL: 'test'
 ; AVX1:  LV: Found an estimated cost of 1 for VF 1 For instruction: store i64 %v5, ptr %out5, align 8
-; AVX1:  Cost of 30 for VF 2: INTERLEAVE-GROUP with factor 6 at <badref>, ir<%out0>
-; AVX1:  Cost of 66 for VF 4: INTERLEAVE-GROUP with factor 6 at <badref>, ir<%out0>
-; AVX1:  Cost of 132 for VF 8: INTERLEAVE-GROUP with factor 6 at <badref>, ir<%out0>
+; AVX1:  LV: Found an estimated cost of 1 for VF 1 For instruction: store i64 %v5, ptr %out5, align 8
+; AVX1:  LV: Found an estimated cost of 30 for VF 2 For instruction: store i64 %v5, ptr %out5, align 8
+; AVX1:  LV: Found an estimated cost of 66 for VF 4 For instruction: store i64 %v5, ptr %out5, align 8
+; AVX1:  LV: Found an estimated cost of 132 for VF 8 For instruction: store i64 %v5, ptr %out5, align 8
 ;
 ; AVX2-LABEL: 'test'
 ; AVX2:  LV: Found an estimated cost of 1 for VF 1 For instruction: store i64 %v5, ptr %out5, align 8
-; AVX2:  Cost of 11 for VF 2: INTERLEAVE-GROUP with factor 6 at <badref>, ir<%out0>
-; AVX2:  Cost of 21 for VF 4: INTERLEAVE-GROUP with factor 6 at <badref>, ir<%out0>
-; AVX2:  Cost of 42 for VF 8: INTERLEAVE-GROUP with factor 6 at <badref>, ir<%out0>
+; AVX2:  LV: Found an estimated cost of 1 for VF 1 For instruction: store i64 %v5, ptr %out5, align 8
+; AVX2:  LV: Found an estimated cost of 11 for VF 2 For instruction: store i64 %v5, ptr %out5, align 8
+; AVX2:  LV: Found an estimated cost of 21 for VF 4 For instruction: store i64 %v5, ptr %out5, align 8
+; AVX2:  LV: Found an estimated cost of 42 for VF 8 For instruction: store i64 %v5, ptr %out5, align 8
 ;
 ; AVX512-LABEL: 'test'
 ; AVX512:  LV: Found an estimated cost of 1 for VF 1 For instruction: store i64 %v5, ptr %out5, align 8
-; AVX512:  Cost of 17 for VF 2: INTERLEAVE-GROUP with factor 6 at <badref>, ir<%out0>
-; AVX512:  Cost of 25 for VF 4: INTERLEAVE-GROUP with factor 6 at <badref>, ir<%out0>
-; AVX512:  Cost of 51 for VF 8: INTERLEAVE-GROUP with factor 6 at <badref>, ir<%out0>
-; AVX512:  Cost of 102 for VF 16: INTERLEAVE-GROUP with factor 6 at <badref>, ir<%out0>
-; AVX512:  Cost of 204 for VF 32: INTERLEAVE-GROUP with factor 6 at <badref>, ir<%out0>
+; AVX512:  LV: Found an estimated cost of 1 for VF 1 For instruction: store i64 %v5, ptr %out5, align 8
+; AVX512:  LV: Found an estimated cost of 17 for VF 2 For instruction: store i64 %v5, ptr %out5, align 8
+; AVX512:  LV: Found an estimated cost of 25 for VF 4 For instruction: store i64 %v5, ptr %out5, align 8
+; AVX512:  LV: Found an estimated cost of 51 for VF 8 For instruction: store i64 %v5, ptr %out5, align 8
+; AVX512:  LV: Found an estimated cost of 102 for VF 16 For instruction: store i64 %v5, ptr %out5, align 8
+; AVX512:  LV: Found an estimated cost of 204 for VF 32 For instruction: store i64 %v5, ptr %out5, align 8
 ;
 entry:
   br label %for.body
diff --git a/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-store-i64-stride-7.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-store-i64-stride-7.ll
index 41a45a3072c95..881c7867614b7 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-store-i64-stride-7.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-store-i64-stride-7.ll
@@ -1,8 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --filter "LV: Found an estimated cost of [0-9]+ for VF [0-9]+ For instruction:\s*store i64 %v6, ptr %out6"
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse2 --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=SSE2
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx  --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX1
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2 --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX2
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512vl --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX512
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse2 --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=SSE2
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx  --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX1
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2 --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX2
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512vl --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX512
 ; REQUIRES: asserts
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
@@ -14,28 +14,32 @@ target triple = "x86_64-unknown-linux-gnu"
 define void @test() {
 ; SSE2-LABEL: 'test'
 ; SSE2:  LV: Found an estimated cost of 1 for VF 1 For instruction: store i64 %v6, ptr %out6, align 8
-; SSE2:  Cost of 50 for VF 2: INTERLEAVE-GROUP with factor 7 at <badref>, ir<%out0>
-; SSE2:  Cost of 100 for VF 4: INTERLEAVE-GROUP with factor 7 at <badref>, ir<%out0>
+; SSE2:  LV: Found an estimated cost of 1 for VF 1 For instruction: store i64 %v6, ptr %out6, align 8
+; SSE2:  LV: Found an estimated cost of 50 for VF 2 For instruction: store i64 %v6, ptr %out6, align 8
+; SSE2:  LV: Found an estimated cost of 100 for VF 4 For instruction: store i64 %v6, ptr %out6, align 8
 ;
 ; AVX1-LABEL: 'test'
 ; AVX1:  LV: Found an estimated cost of 1 for VF 1 For instruction: store i64 %v6, ptr %out6, align 8
-; AVX1:  Cost of 36 for VF 2: INTERLEAVE-GROUP with factor 7 at <badref>, ir<%out0>
-; AVX1:  Cost of 77 for VF 4: INTERLEAVE-GROUP with factor 7 at <badref>, ir<%out0>
-; AVX1:  Cost of 154 for VF 8: INTERLEAVE-GROUP with factor 7 at <badref>, ir<%out0>
+; AVX1:  LV: Found an estimated cost of 1 for VF 1 For instruction: store i64 %v6, ptr %out6, align 8
+; AVX1:  LV: Found an estimated cost of 36 for VF 2 For instruction: store i64 %v6, ptr %out6, align 8
+; AVX1:  LV: Found an estimated cost of 77 for VF 4 For instruction: store i64 %v6, ptr %out6, align 8
+; AVX1:  LV: Found an estimated cost of 154 for VF 8 For instruction: store i64 %v6, ptr %out6, align 8
 ;
 ; AVX2-LABEL: 'test'
 ; AVX2:  LV: Found an estimated cost of 1 for VF 1 For instruction: store i64 %v6, ptr %out6, align 8
-; AVX2:  Cost of 36 for VF 2: INTERLEAVE-GROUP with factor 7 at <badref>, ir<%out0>
-; AVX2:  Cost of 77 for VF 4: INTERLEAVE-GROUP with factor 7 at <badref>, ir<%out0>
-; AVX2:  Cost of 154 for VF 8: INTERLEAVE-GROUP with factor 7 at <badref>, ir<%out0>
+; AVX2:  LV: Found an estimated cost of 1 for VF 1 For instruction: store i64 %v6, ptr %out6, align 8
+; AVX2:  LV: Found an estimated cost of 36 for VF 2 For instruction: store i64 %v6, ptr %out6, align 8
+; AVX2:  LV: Found an estimated cost of 77 for VF 4 For instruction: store i64 %v6, ptr %out6, align 8
+; AVX2:  LV: Found an estimated cost of 154 for VF 8 For instruction: store i64 %v6, ptr %out6, align 8
 ;
 ; AVX512-LABEL: 'test'
 ; AVX512:  LV: Found an estimated cost of 1 for VF 1 For instruction: store i64 %v6, ptr %out6, align 8
-; AVX512:  Cost of 20 for VF 2: INTERLEAVE-GROUP with factor 7 at <badref>, ir<%out0>
-; AVX512:  Cost of 40 for VF 4: INTERLEAVE-GROUP with factor 7 at <badref>, ir<%out0>
-; AVX512:  Cost of 70 for VF 8: INTERLEAVE-GROUP with factor 7 at <badref>, ir<%out0>
-; AVX512:  Cost of 140 for VF 16: INTERLEAVE-GROUP with factor 7 at <badref>, ir<%out0>
-; AVX512:  Cost of 280 for VF 32: INTERLEAVE-GROUP with factor 7 at <badref>, ir<%out0>
+; AVX512:  LV: Found an estimated cost of 1 for VF 1 For instruction: store i64 %v6, ptr %out6, align 8
+; AVX512:  LV: Found an estimated cost of 20 for VF 2 For instruction: store i64 %v6, ptr %out6, align 8
+; AVX512:  LV: Found an estimated cost of 40 for VF 4 For instruction: store i64 %v6, ptr %out6, align 8
+; AVX512:  LV: Found an estimated cost of 70 for VF 8 For instruction: store i64 %v6, ptr %out6, align 8
+; AVX512:  LV: Found an estimated cost of 140 for VF 16 For instruction: store i64 %v6, ptr %out6, align 8
+; AVX512:  LV: Found an estimated cost of 280 for VF 32 For instruction: store i64 %v6, ptr %out6, align 8
 ;
 entry:
   br label %for.body
diff --git a/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-store-i64-stride-8.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-store-i64-stride-8.ll
index a539877104129..6bf88fce8df80 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-store-i64-stride-8.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-store-i64-stride-8.ll
@@ -1,8 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --filter "LV: Found an estimated cost of [0-9]+ for VF [0-9]+ For instruction:\s*store i64 %v., ptr %out."
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse2 --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=SSE2
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx  --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX1
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2 --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX2
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512vl --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX512
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse2 --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=SSE2
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx  --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX1
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2 --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX2
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512vl --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX512
 ; REQUIRES: asserts
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
diff --git a/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-store-i8-stride-2.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-store-i8-stride-2.ll
index 78105db5e5ad3..87b5f4d3858dc 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-store-i8-stride-2.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-store-i8-stride-2.ll
@@ -1,9 +1,9 @@
 ; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --filter "LV: Found an estimated cost of [0-9]+ for VF [0-9]+ For instruction:\s*store i8 %v1, ptr %out1"
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse2 --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=SSE2
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx  --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX1
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2 --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX2
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512vl --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX512DQ
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512vl,+avx512bw --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX512BW
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse2 --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=SSE2
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx  --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX1
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2 --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX2
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512vl --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX512DQ
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512vl,+avx512bw --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX512BW
 ; REQUIRES: asserts
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
@@ -15,44 +15,49 @@ target triple = "x86_64-unknown-linux-gnu"
 define void @test() {
 ; SSE2-LABEL: 'test'
 ; SSE2:  LV: Found an estimated cost of 1 for VF 1 For instruction: store i8 %v1, ptr %out1, align 1
-; SSE2:  Cost of 2 for VF 2: INTERLEAVE-GROUP with factor 2 at <badref>, ir<%out0>
-; SSE2:  Cost of 2 for VF 4: INTERLEAVE-GROUP with factor 2 at <badref>, ir<%out0>
-; SSE2:  Cost of 2 for VF 8: INTERLEAVE-GROUP with factor 2 at <badref>, ir<%out0>
-; SSE2:  Cost of 126 for VF 16: INTERLEAVE-GROUP with factor 2 at <badref>, ir<%out0>
+; SSE2:  LV: Found an estimated cost of 1 for VF 1 For instruction: store i8 %v1, ptr %out1, align 1
+; SSE2:  LV: Found an estimated cost of 2 for VF 2 For instruction: store i8 %v1, ptr %out1, align 1
+; SSE2:  LV: Found an estimated cost of 2 for VF 4 For instruction: store i8 %v1, ptr %out1, align 1
+; SSE2:  LV: Found an estimated cost of 2 for VF 8 For instruction: store i8 %v1, ptr %out1, align 1
+; SSE2:  LV: Found an estimated cost of 126 for VF 16 For instruction: store i8 %v1, ptr %out1, align 1
 ;
 ; AVX1-LABEL: 'test'
 ; AVX1:  LV: Found an estimated cost of 1 for VF 1 For instruction: store i8 %v1, ptr %out1, align 1
-; AVX1:  Cost of 2 for VF 2: INTERLEAVE-GROUP with factor 2 at <badref>, ir<%out0>
-; AVX1:  Cost of 2 for VF 4: INTERLEAVE-GROUP with factor 2 at <badref>, ir<%out0>
-; AVX1:  Cost of 2 for VF 8: INTERLEAVE-GROUP with factor 2 at <badref>, ir<%out0>
-; AVX1:  Cost of 66 for VF 16: INTERLEAVE-GROUP with factor 2 at <badref>, ir<%out0>
-; AVX1:  Cost of 134 for VF 32: INTERLEAVE-GROUP with factor 2 at <badref>, ir<%out0>
+; AVX1:  LV: Found an estimated cost of 1 for VF 1 For instruction: store i8 %v1, ptr %out1, align 1
+; AVX1:  LV: Found an estimated cost of 2 for VF 2 For instruction: store i8 %v1, ptr %out1, align 1
+; AVX1:  LV: Found an estimated cost of 2 for VF 4 For instruction: store i8 %v1, ptr %out1, align 1
+; AVX1:  LV: Found an estimated cost of 2 for VF 8 For instruction: store i8 %v1, ptr %out1, align 1
+; AVX1:  LV: Found an estimated cost of 66 for VF 16 For instruction: store i8 %v1, ptr %out1, align 1
+; AVX1:  LV: Found an estimated cost of 134 for VF 32 For instruction: store i8 %v1, ptr %out1, align 1
 ;
 ; AVX2-LABEL: 'test'
 ; AVX2:  LV: Found an estimated cost of 1 for VF 1 For instruction: store i8 %v1, ptr %out1, align 1
-; AVX2:  Cost of 2 for VF 2: INTERLEAVE-GROUP with factor 2 at <badref>, ir<%out0>
-; AVX2:  Cost of 2 for VF 4: INTERLEAVE-GROUP with factor 2 at <badref>, ir<%out0>
-; AVX2:  Cost of 2 for VF 8: INTERLEAVE-GROUP with factor 2 at <badref>, ir<%out0>
-; AVX2:  Cost of 4 for VF 16: INTERLEAVE-GROUP with factor 2 at <badref>, ir<%out0>
-; AVX2:  Cost of 6 for VF 32: INTERLEAVE-GROUP with factor 2 at <badref>, ir<%out0>
+; AVX2:  LV: Found an estimated cost of 1 for VF 1 For instruction: store i8 %v1, ptr %out1, align 1
+; AVX2:  LV: Found an estimated cost of 2 for VF 2 For instruction: store i8 %v1, ptr %out1, align 1
+; AVX2:  LV: Found an estimated cost of 2 for VF 4 For instruction: store i8 %v1, ptr %out1, align 1
+; AVX2:  LV: Found an estimated cost of 2 for VF 8 For instruction: store i8 %v1, ptr %out1, align 1
+; AVX2:  LV: Found an estimated cost of 4 for VF 16 For instruction: store i8 %v1, ptr %out1, align 1
+; AVX2:  LV: Found an estimated cost of 6 for VF 32 For instruction: store i8 %v1, ptr %out1, align 1
 ;
 ; AVX512DQ-LABEL: 'test'
 ; AVX512DQ:  LV: Found an estimated cost of 1 for VF 1 For instruction: store i8 %v1, ptr %out1, align 1
-; AVX512DQ:  Cost of 2 for VF 2: INTERLEAVE-GROUP with factor 2 at <badref>, ir<%out0>
-; AVX512DQ:  Cost of 2 for VF 4: INTERLEAVE-GROUP with factor 2 at <badref>, ir<%out0>
-; AVX512DQ:  Cost of 2 for VF 8: INTERLEAVE-GROUP with factor 2 at <badref>, ir<%out0>
-; AVX512DQ:  Cost of 4 for VF 16: INTERLEAVE-GROUP with factor 2 at <badref>, ir<%out0>
-; AVX512DQ:  Cost of 5 for VF 32: INTERLEAVE-GROUP with factor 2 at <badref>, ir<%out0>
-; AVX512DQ:  Cost of 270 for VF 64: INTERLEAVE-GROUP with factor 2 at <badref>, ir<%out0>
+; AVX512DQ:  LV: Found an estimated cost of 1 for VF 1 For instruction: store i8 %v1, ptr %out1, align 1
+; AVX512DQ:  LV: Found an estimated cost of 2 for VF 2 For instruction: store i8 %v1, ptr %out1, align 1
+; AVX512DQ:  LV: Found an estimated cost of 2 for VF 4 For instruction: store i8 %v1, ptr %out1, align 1
+; AVX512DQ:  LV: Found an estimated cost of 2 for VF 8 For instruction: store i8 %v1, ptr %out1, align 1
+; AVX512DQ:  LV: Found an estimated cost of 4 for VF 16 For instruction: store i8 %v1, ptr %out1, align 1
+; AVX512DQ:  LV: Found an estimated cost of 5 for VF 32 For instruction: store i8 %v1, ptr %out1, align 1
+; AVX512DQ:  LV: Found an estimated cost of 270 for VF 64 For instruction: store i8 %v1, ptr %out1, align 1
 ;
 ; AVX512BW-LABEL: 'test'
 ; AVX512BW:  LV: Found an estimated cost of 1 for VF 1 For instruction: store i8 %v1, ptr %out1, align 1
-; AVX512BW:  Cost of 4 for VF 2: INTERLEAVE-GROUP with factor 2 at <badref>, ir<%out0>
-; AVX512BW:  Cost of 4 for VF 4: INTERLEAVE-GROUP with factor 2 at <badref>, ir<%out0>
-; AVX512BW:  Cost of 4 for VF 8: INTERLEAVE-GROUP with factor 2 at <badref>, ir<%out0>
-; AVX512BW:  Cost of 8 for VF 16: INTERLEAVE-GROUP with factor 2 at <badref>, ir<%out0>
-; AVX512BW:  Cost of 20 for VF 32: INTERLEAVE-GROUP with factor 2 at <badref>, ir<%out0>
-; AVX512BW:  Cost of 41 for VF 64: INTERLEAVE-GROUP with factor 2 at <badref>, ir<%out0>
+; AVX512BW:  LV: Found an estimated cost of 1 for VF 1 For instruction: store i8 %v1, ptr %out1, align 1
+; AVX512BW:  LV: Found an estimated cost of 4 for VF 2 For instruction: store i8 %v1, ptr %out1, align 1
+; AVX512BW:  LV: Found an estimated cost of 4 for VF 4 For instruction: store i8 %v1, ptr %out1, align 1
+; AVX512BW:  LV: Found an estimated cost of 4 for VF 8 For instruction: store i8 %v1, ptr %out1, align 1
+; AVX512BW:  LV: Found an estimated cost of 8 for VF 16 For instruction: store i8 %v1, ptr %out1, align 1
+; AVX512BW:  LV: Found an estimated cost of 20 for VF 32 For instruction: store i8 %v1, ptr %out1, align 1
+; AVX512BW:  LV: Found an estimated cost of 41 for VF 64 For instruction: store i8 %v1, ptr %out1, align 1
 ;
 entry:
   br label %for.body
diff --git a/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-store-i8-stride-3.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-store-i8-stride-3.ll
index 8e6892d0158e7..9a7503b63f9ff 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-store-i8-stride-3.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-store-i8-stride-3.ll
@@ -1,9 +1,9 @@
 ; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --filter "LV: Found an estimated cost of [0-9]+ for VF [0-9]+ For instruction:\s*store i8 %v2, ptr %out2"
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse2 --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=SSE2
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx  --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX1
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2 --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX2
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512vl --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX512DQ
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512vl,+avx512bw --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX512BW
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse2 --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=SSE2
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx  --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX1
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2 --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX2
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512vl --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX512DQ
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512vl,+avx512bw --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX512BW
 ; REQUIRES: asserts
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
@@ -15,44 +15,49 @@ target triple = "x86_64-unknown-linux-gnu"
 define void @test() {
 ; SSE2-LABEL: 'test'
 ; SSE2:  LV: Found an estimated cost of 1 for VF 1 For instruction: store i8 %v2, ptr %out2, align 1
-; SSE2:  Cost of 26 for VF 2: INTERLEAVE-GROUP with factor 3 at <badref>, ir<%out0>
-; SSE2:  Cost of 52 for VF 4: INTERLEAVE-GROUP with factor 3 at <badref>, ir<%out0>
-; SSE2:  Cost of 101 for VF 8: INTERLEAVE-GROUP with factor 3 at <badref>, ir<%out0>
-; SSE2:  Cost of 204 for VF 16: INTERLEAVE-GROUP with factor 3 at <badref>, ir<%out0>
+; SSE2:  LV: Found an estimated cost of 1 for VF 1 For instruction: store i8 %v2, ptr %out2, align 1
+; SSE2:  LV: Found an estimated cost of 26 for VF 2 For instruction: store i8 %v2, ptr %out2, align 1
+; SSE2:  LV: Found an estimated cost of 52 for VF 4 For instruction: store i8 %v2, ptr %out2, align 1
+; SSE2:  LV: Found an estimated cost of 101 for VF 8 For instruction: store i8 %v2, ptr %out2, align 1
+; SSE2:  LV: Found an estimated cost of 204 for VF 16 For instruction: store i8 %v2, ptr %out2, align 1
 ;
 ; AVX1-LABEL: 'test'
 ; AVX1:  LV: Found an estimated cost of 1 for VF 1 For instruction: store i8 %v2, ptr %out2, align 1
-; AVX1:  Cost of 16 for VF 2: INTERLEAVE-GROUP with factor 3 at <badref>, ir<%out0>
-; AVX1:  Cost of 27 for VF 4: INTERLEAVE-GROUP with factor 3 at <badref>, ir<%out0>
-; AVX1:  Cost of 53 for VF 8: INTERLEAVE-GROUP with factor 3 at <badref>, ir<%out0>
-; AVX1:  Cost of 100 for VF 16: INTERLEAVE-GROUP with factor 3 at <badref>, ir<%out0>
-; AVX1:  Cost of 201 for VF 32: INTERLEAVE-GROUP with factor 3 at <badref>, ir<%out0>
+; AVX1:  LV: Found an estimated cost of 1 for VF 1 For instruction: store i8 %v2, ptr %out2, align 1
+; AVX1:  LV: Found an estimated cost of 16 for VF 2 For instruction: store i8 %v2, ptr %out2, align 1
+; AVX1:  LV: Found an estimated cost of 27 for VF 4 For instruction: store i8 %v2, ptr %out2, align 1
+; AVX1:  LV: Found an estimated cost of 53 for VF 8 For instruction: store i8 %v2, ptr %out2, align 1
+; AVX1:  LV: Found an estimated cost of 100 for VF 16 For instruction: store i8 %v2, ptr %out2, align 1
+; AVX1:  LV: Found an estimated cost of 201 for VF 32 For instruction: store i8 %v2, ptr %out2, align 1
 ;
 ; AVX2-LABEL: 'test'
 ; AVX2:  LV: Found an estimated cost of 1 for VF 1 For instruction: store i8 %v2, ptr %out2, align 1
-; AVX2:  Cost of 8 for VF 2: INTERLEAVE-GROUP with factor 3 at <badref>, ir<%out0>
-; AVX2:  Cost of 7 for VF 4: INTERLEAVE-GROUP with factor 3 at <badref>, ir<%out0>
-; AVX2:  Cost of 9 for VF 8: INTERLEAVE-GROUP with factor 3 at <badref>, ir<%out0>
-; AVX2:  Cost of 13 for VF 16: INTERLEAVE-GROUP with factor 3 at <badref>, ir<%out0>
-; AVX2:  Cost of 16 for VF 32: INTERLEAVE-GROUP with factor 3 at <badref>, ir<%out0>
+; AVX2:  LV: Found an estimated cost of 1 for VF 1 For instruction: store i8 %v2, ptr %out2, align 1
+; AVX2:  LV: Found an estimated cost of 8 for VF 2 For instruction: store i8 %v2, ptr %out2, align 1
+; AVX2:  LV: Found an estimated cost of 7 for VF 4 For instruction: store i8 %v2, ptr %out2, align 1
+; AVX2:  LV: Found an estimated cost of 9 for VF 8 For instruction: store i8 %v2, ptr %out2, align 1
+; AVX2:  LV: Found an estimated cost of 13 for VF 16 For instruction: store i8 %v2, ptr %out2, align 1
+; AVX2:  LV: Found an estimated cost of 16 for VF 32 For instruction: store i8 %v2, ptr %out2, align 1
 ;
 ; AVX512DQ-LABEL: 'test'
 ; AVX512DQ:  LV: Found an estimated cost of 1 for VF 1 For instruction: store i8 %v2, ptr %out2, align 1
-; AVX512DQ:  Cost of 8 for VF 2: INTERLEAVE-GROUP with factor 3 at <badref>, ir<%out0>
-; AVX512DQ:  Cost of 7 for VF 4: INTERLEAVE-GROUP with factor 3 at <badref>, ir<%out0>
-; AVX512DQ:  Cost of 9 for VF 8: INTERLEAVE-GROUP with factor 3 at <badref>, ir<%out0>
-; AVX512DQ:  Cost of 14 for VF 16: INTERLEAVE-GROUP with factor 3 at <badref>, ir<%out0>
-; AVX512DQ:  Cost of 15 for VF 32: INTERLEAVE-GROUP with factor 3 at <badref>, ir<%out0>
-; AVX512DQ:  Cost of 405 for VF 64: INTERLEAVE-GROUP with factor 3 at <badref>, ir<%out0>
+; AVX512DQ:  LV: Found an estimated cost of 1 for VF 1 For instruction: store i8 %v2, ptr %out2, align 1
+; AVX512DQ:  LV: Found an estimated cost of 8 for VF 2 For instruction: store i8 %v2, ptr %out2, align 1
+; AVX512DQ:  LV: Found an estimated cost of 7 for VF 4 For instruction: store i8 %v2, ptr %out2, align 1
+; AVX512DQ:  LV: Found an estimated cost of 9 for VF 8 For instruction: store i8 %v2, ptr %out2, align 1
+; AVX512DQ:  LV: Found an estimated cost of 14 for VF 16 For instruction: store i8 %v2, ptr %out2, align 1
+; AVX512DQ:  LV: Found an estimated cost of 15 for VF 32 For instruction: store i8 %v2, ptr %out2, align 1
+; AVX512DQ:  LV: Found an estimated cost of 405 for VF 64 For instruction: store i8 %v2, ptr %out2, align 1
 ;
 ; AVX512BW-LABEL: 'test'
 ; AVX512BW:  LV: Found an estimated cost of 1 for VF 1 For instruction: store i8 %v2, ptr %out2, align 1
-; AVX512BW:  Cost of 8 for VF 2: INTERLEAVE-GROUP with factor 3 at <badref>, ir<%out0>
-; AVX512BW:  Cost of 8 for VF 4: INTERLEAVE-GROUP with factor 3 at <badref>, ir<%out0>
-; AVX512BW:  Cost of 16 for VF 8: INTERLEAVE-GROUP with factor 3 at <badref>, ir<%out0>
-; AVX512BW:  Cost of 13 for VF 16: INTERLEAVE-GROUP with factor 3 at <badref>, ir<%out0>
-; AVX512BW:  Cost of 16 for VF 32: INTERLEAVE-GROUP with factor 3 at <badref>, ir<%out0>
-; AVX512BW:  Cost of 29 for VF 64: INTERLEAVE-GROUP with factor 3 at <badref>, ir<%out0>
+; AVX512BW:  LV: Found an estimated cost of 1 for VF 1 For instruction: store i8 %v2, ptr %out2, align 1
+; AVX512BW:  LV: Found an estimated cost of 8 for VF 2 For instruction: store i8 %v2, ptr %out2, align 1
+; AVX512BW:  LV: Found an estimated cost of 8 for VF 4 For instruction: store i8 %v2, ptr %out2, align 1
+; AVX512BW:  LV: Found an estimated cost of 16 for VF 8 For instruction: store i8 %v2, ptr %out2, align 1
+; AVX512BW:  LV: Found an estimated cost of 13 for VF 16 For instruction: store i8 %v2, ptr %out2, align 1
+; AVX512BW:  LV: Found an estimated cost of 16 for VF 32 For instruction: store i8 %v2, ptr %out2, align 1
+; AVX512BW:  LV: Found an estimated cost of 29 for VF 64 For instruction: store i8 %v2, ptr %out2, align 1
 ;
 entry:
   br label %for.body
diff --git a/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-store-i8-stride-4.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-store-i8-stride-4.ll
index 3083dde7ac66f..e072d707e23e5 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-store-i8-stride-4.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-store-i8-stride-4.ll
@@ -1,9 +1,9 @@
 ; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --filter "LV: Found an estimated cost of [0-9]+ for VF [0-9]+ For instruction:\s*store i8 %v3, ptr %out3"
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse2 --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=SSE2
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx  --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX1
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2 --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX2
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512vl --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX512DQ
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512vl,+avx512bw --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX512BW
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse2 --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=SSE2
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx  --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX1
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2 --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX2
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512vl --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX512DQ
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512vl,+avx512bw --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX512BW
 ; REQUIRES: asserts
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
@@ -15,44 +15,49 @@ target triple = "x86_64-unknown-linux-gnu"
 define void @test() {
 ; SSE2-LABEL: 'test'
 ; SSE2:  LV: Found an estimated cost of 1 for VF 1 For instruction: store i8 %v3, ptr %out3, align 1
-; SSE2:  Cost of 28 for VF 2: INTERLEAVE-GROUP with factor 4 at <badref>, ir<%out0>
-; SSE2:  Cost of 60 for VF 4: INTERLEAVE-GROUP with factor 4 at <badref>, ir<%out0>
-; SSE2:  Cost of 124 for VF 8: INTERLEAVE-GROUP with factor 4 at <badref>, ir<%out0>
-; SSE2:  Cost of 252 for VF 16: INTERLEAVE-GROUP with factor 4 at <badref>, ir<%out0>
+; SSE2:  LV: Found an estimated cost of 1 for VF 1 For instruction: store i8 %v3, ptr %out3, align 1
+; SSE2:  LV: Found an estimated cost of 28 for VF 2 For instruction: store i8 %v3, ptr %out3, align 1
+; SSE2:  LV: Found an estimated cost of 60 for VF 4 For instruction: store i8 %v3, ptr %out3, align 1
+; SSE2:  LV: Found an estimated cost of 124 for VF 8 For instruction: store i8 %v3, ptr %out3, align 1
+; SSE2:  LV: Found an estimated cost of 252 for VF 16 For instruction: store i8 %v3, ptr %out3, align 1
 ;
 ; AVX1-LABEL: 'test'
 ; AVX1:  LV: Found an estimated cost of 1 for VF 1 For instruction: store i8 %v3, ptr %out3, align 1
-; AVX1:  Cost of 17 for VF 2: INTERLEAVE-GROUP with factor 4 at <badref>, ir<%out0>
-; AVX1:  Cost of 33 for VF 4: INTERLEAVE-GROUP with factor 4 at <badref>, ir<%out0>
-; AVX1:  Cost of 66 for VF 8: INTERLEAVE-GROUP with factor 4 at <badref>, ir<%out0>
-; AVX1:  Cost of 132 for VF 16: INTERLEAVE-GROUP with factor 4 at <badref>, ir<%out0>
-; AVX1:  Cost of 268 for VF 32: INTERLEAVE-GROUP with factor 4 at <badref>, ir<%out0>
+; AVX1:  LV: Found an estimated cost of 1 for VF 1 For instruction: store i8 %v3, ptr %out3, align 1
+; AVX1:  LV: Found an estimated cost of 17 for VF 2 For instruction: store i8 %v3, ptr %out3, align 1
+; AVX1:  LV: Found an estimated cost of 33 for VF 4 For instruction: store i8 %v3, ptr %out3, align 1
+; AVX1:  LV: Found an estimated cost of 66 for VF 8 For instruction: store i8 %v3, ptr %out3, align 1
+; AVX1:  LV: Found an estimated cost of 132 for VF 16 For instruction: store i8 %v3, ptr %out3, align 1
+; AVX1:  LV: Found an estimated cost of 268 for VF 32 For instruction: store i8 %v3, ptr %out3, align 1
 ;
 ; AVX2-LABEL: 'test'
 ; AVX2:  LV: Found an estimated cost of 1 for VF 1 For instruction: store i8 %v3, ptr %out3, align 1
-; AVX2:  Cost of 5 for VF 2: INTERLEAVE-GROUP with factor 4 at <badref>, ir<%out0>
-; AVX2:  Cost of 5 for VF 4: INTERLEAVE-GROUP with factor 4 at <badref>, ir<%out0>
-; AVX2:  Cost of 5 for VF 8: INTERLEAVE-GROUP with factor 4 at <badref>, ir<%out0>
-; AVX2:  Cost of 10 for VF 16: INTERLEAVE-GROUP with factor 4 at <badref>, ir<%out0>
-; AVX2:  Cost of 16 for VF 32: INTERLEAVE-GROUP with factor 4 at <badref>, ir<%out0>
+; AVX2:  LV: Found an estimated cost of 1 for VF 1 For instruction: store i8 %v3, ptr %out3, align 1
+; AVX2:  LV: Found an estimated cost of 5 for VF 2 For instruction: store i8 %v3, ptr %out3, align 1
+; AVX2:  LV: Found an estimated cost of 5 for VF 4 For instruction: store i8 %v3, ptr %out3, align 1
+; AVX2:  LV: Found an estimated cost of 5 for VF 8 For instruction: store i8 %v3, ptr %out3, align 1
+; AVX2:  LV: Found an estimated cost of 10 for VF 16 For instruction: store i8 %v3, ptr %out3, align 1
+; AVX2:  LV: Found an estimated cost of 16 for VF 32 For instruction: store i8 %v3, ptr %out3, align 1
 ;
 ; AVX512DQ-LABEL: 'test'
 ; AVX512DQ:  LV: Found an estimated cost of 1 for VF 1 For instruction: store i8 %v3, ptr %out3, align 1
-; AVX512DQ:  Cost of 5 for VF 2: INTERLEAVE-GROUP with factor 4 at <badref>, ir<%out0>
-; AVX512DQ:  Cost of 5 for VF 4: INTERLEAVE-GROUP with factor 4 at <badref>, ir<%out0>
-; AVX512DQ:  Cost of 5 for VF 8: INTERLEAVE-GROUP with factor 4 at <badref>, ir<%out0>
-; AVX512DQ:  Cost of 9 for VF 16: INTERLEAVE-GROUP with factor 4 at <badref>, ir<%out0>
-; AVX512DQ:  Cost of 14 for VF 32: INTERLEAVE-GROUP with factor 4 at <badref>, ir<%out0>
-; AVX512DQ:  Cost of 540 for VF 64: INTERLEAVE-GROUP with factor 4 at <badref>, ir<%out0>
+; AVX512DQ:  LV: Found an estimated cost of 1 for VF 1 For instruction: store i8 %v3, ptr %out3, align 1
+; AVX512DQ:  LV: Found an estimated cost of 5 for VF 2 For instruction: store i8 %v3, ptr %out3, align 1
+; AVX512DQ:  LV: Found an estimated cost of 5 for VF 4 For instruction: store i8 %v3, ptr %out3, align 1
+; AVX512DQ:  LV: Found an estimated cost of 5 for VF 8 For instruction: store i8 %v3, ptr %out3, align 1
+; AVX512DQ:  LV: Found an estimated cost of 9 for VF 16 For instruction: store i8 %v3, ptr %out3, align 1
+; AVX512DQ:  LV: Found an estimated cost of 14 for VF 32 For instruction: store i8 %v3, ptr %out3, align 1
+; AVX512DQ:  LV: Found an estimated cost of 540 for VF 64 For instruction: store i8 %v3, ptr %out3, align 1
 ;
 ; AVX512BW-LABEL: 'test'
 ; AVX512BW:  LV: Found an estimated cost of 1 for VF 1 For instruction: store i8 %v3, ptr %out3, align 1
-; AVX512BW:  Cost of 11 for VF 2: INTERLEAVE-GROUP with factor 4 at <badref>, ir<%out0>
-; AVX512BW:  Cost of 11 for VF 4: INTERLEAVE-GROUP with factor 4 at <badref>, ir<%out0>
-; AVX512BW:  Cost of 11 for VF 8: INTERLEAVE-GROUP with factor 4 at <badref>, ir<%out0>
-; AVX512BW:  Cost of 12 for VF 16: INTERLEAVE-GROUP with factor 4 at <badref>, ir<%out0>
-; AVX512BW:  Cost of 16 for VF 32: INTERLEAVE-GROUP with factor 4 at <badref>, ir<%out0>
-; AVX512BW:  Cost of 28 for VF 64: INTERLEAVE-GROUP with factor 4 at <badref>, ir<%out0>
+; AVX512BW:  LV: Found an estimated cost of 1 for VF 1 For instruction: store i8 %v3, ptr %out3, align 1
+; AVX512BW:  LV: Found an estimated cost of 11 for VF 2 For instruction: store i8 %v3, ptr %out3, align 1
+; AVX512BW:  LV: Found an estimated cost of 11 for VF 4 For instruction: store i8 %v3, ptr %out3, align 1
+; AVX512BW:  LV: Found an estimated cost of 11 for VF 8 For instruction: store i8 %v3, ptr %out3, align 1
+; AVX512BW:  LV: Found an estimated cost of 12 for VF 16 For instruction: store i8 %v3, ptr %out3, align 1
+; AVX512BW:  LV: Found an estimated cost of 16 for VF 32 For instruction: store i8 %v3, ptr %out3, align 1
+; AVX512BW:  LV: Found an estimated cost of 28 for VF 64 For instruction: store i8 %v3, ptr %out3, align 1
 ;
 entry:
   br label %for.body
diff --git a/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-store-i8-stride-5.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-store-i8-stride-5.ll
index d788a7b8d469e..1c079204cc3bb 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-store-i8-stride-5.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-store-i8-stride-5.ll
@@ -1,9 +1,9 @@
 ; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --filter "LV: Found an estimated cost of [0-9]+ for VF [0-9]+ For instruction:\s*store i8 %v4, ptr %out4"
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse2 --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=SSE2
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx  --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX1
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2 --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX2
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512vl --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX512DQ
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512vl,+avx512bw --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX512BW
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse2 --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=SSE2
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx  --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX1
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2 --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX2
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512vl --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX512DQ
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512vl,+avx512bw --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX512BW
 ; REQUIRES: asserts
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
@@ -15,44 +15,49 @@ target triple = "x86_64-unknown-linux-gnu"
 define void @test() {
 ; SSE2-LABEL: 'test'
 ; SSE2:  LV: Found an estimated cost of 1 for VF 1 For instruction: store i8 %v4, ptr %out4, align 1
-; SSE2:  Cost of 44 for VF 2: INTERLEAVE-GROUP with factor 5 at <badref>, ir<%out0>
-; SSE2:  Cost of 87 for VF 4: INTERLEAVE-GROUP with factor 5 at <badref>, ir<%out0>
-; SSE2:  Cost of 178 for VF 8: INTERLEAVE-GROUP with factor 5 at <badref>, ir<%out0>
-; SSE2:  Cost of 360 for VF 16: INTERLEAVE-GROUP with factor 5 at <badref>, ir<%out0>
+; SSE2:  LV: Found an estimated cost of 1 for VF 1 For instruction: store i8 %v4, ptr %out4, align 1
+; SSE2:  LV: Found an estimated cost of 44 for VF 2 For instruction: store i8 %v4, ptr %out4, align 1
+; SSE2:  LV: Found an estimated cost of 87 for VF 4 For instruction: store i8 %v4, ptr %out4, align 1
+; SSE2:  LV: Found an estimated cost of 178 for VF 8 For instruction: store i8 %v4, ptr %out4, align 1
+; SSE2:  LV: Found an estimated cost of 360 for VF 16 For instruction: store i8 %v4, ptr %out4, align 1
 ;
 ; AVX1-LABEL: 'test'
 ; AVX1:  LV: Found an estimated cost of 1 for VF 1 For instruction: store i8 %v4, ptr %out4, align 1
-; AVX1:  Cost of 24 for VF 2: INTERLEAVE-GROUP with factor 5 at <badref>, ir<%out0>
-; AVX1:  Cost of 46 for VF 4: INTERLEAVE-GROUP with factor 5 at <badref>, ir<%out0>
-; AVX1:  Cost of 84 for VF 8: INTERLEAVE-GROUP with factor 5 at <badref>, ir<%out0>
-; AVX1:  Cost of 166 for VF 16: INTERLEAVE-GROUP with factor 5 at <badref>, ir<%out0>
-; AVX1:  Cost of 335 for VF 32: INTERLEAVE-GROUP with factor 5 at <badref>, ir<%out0>
+; AVX1:  LV: Found an estimated cost of 1 for VF 1 For instruction: store i8 %v4, ptr %out4, align 1
+; AVX1:  LV: Found an estimated cost of 24 for VF 2 For instruction: store i8 %v4, ptr %out4, align 1
+; AVX1:  LV: Found an estimated cost of 46 for VF 4 For instruction: store i8 %v4, ptr %out4, align 1
+; AVX1:  LV: Found an estimated cost of 84 for VF 8 For instruction: store i8 %v4, ptr %out4, align 1
+; AVX1:  LV: Found an estimated cost of 166 for VF 16 For instruction: store i8 %v4, ptr %out4, align 1
+; AVX1:  LV: Found an estimated cost of 335 for VF 32 For instruction: store i8 %v4, ptr %out4, align 1
 ;
 ; AVX2-LABEL: 'test'
 ; AVX2:  LV: Found an estimated cost of 1 for VF 1 For instruction: store i8 %v4, ptr %out4, align 1
-; AVX2:  Cost of 24 for VF 2: INTERLEAVE-GROUP with factor 5 at <badref>, ir<%out0>
-; AVX2:  Cost of 46 for VF 4: INTERLEAVE-GROUP with factor 5 at <badref>, ir<%out0>
-; AVX2:  Cost of 84 for VF 8: INTERLEAVE-GROUP with factor 5 at <badref>, ir<%out0>
-; AVX2:  Cost of 166 for VF 16: INTERLEAVE-GROUP with factor 5 at <badref>, ir<%out0>
-; AVX2:  Cost of 335 for VF 32: INTERLEAVE-GROUP with factor 5 at <badref>, ir<%out0>
+; AVX2:  LV: Found an estimated cost of 1 for VF 1 For instruction: store i8 %v4, ptr %out4, align 1
+; AVX2:  LV: Found an estimated cost of 24 for VF 2 For instruction: store i8 %v4, ptr %out4, align 1
+; AVX2:  LV: Found an estimated cost of 46 for VF 4 For instruction: store i8 %v4, ptr %out4, align 1
+; AVX2:  LV: Found an estimated cost of 84 for VF 8 For instruction: store i8 %v4, ptr %out4, align 1
+; AVX2:  LV: Found an estimated cost of 166 for VF 16 For instruction: store i8 %v4, ptr %out4, align 1
+; AVX2:  LV: Found an estimated cost of 335 for VF 32 For instruction: store i8 %v4, ptr %out4, align 1
 ;
 ; AVX512DQ-LABEL: 'test'
 ; AVX512DQ:  LV: Found an estimated cost of 1 for VF 1 For instruction: store i8 %v4, ptr %out4, align 1
-; AVX512DQ:  Cost of 24 for VF 2: INTERLEAVE-GROUP with factor 5 at <badref>, ir<%out0>
-; AVX512DQ:  Cost of 46 for VF 4: INTERLEAVE-GROUP with factor 5 at <badref>, ir<%out0>
-; AVX512DQ:  Cost of 87 for VF 8: INTERLEAVE-GROUP with factor 5 at <badref>, ir<%out0>
-; AVX512DQ:  Cost of 166 for VF 16: INTERLEAVE-GROUP with factor 5 at <badref>, ir<%out0>
-; AVX512DQ:  Cost of 336 for VF 32: INTERLEAVE-GROUP with factor 5 at <badref>, ir<%out0>
-; AVX512DQ:  Cost of 675 for VF 64: INTERLEAVE-GROUP with factor 5 at <badref>, ir<%out0>
+; AVX512DQ:  LV: Found an estimated cost of 1 for VF 1 For instruction: store i8 %v4, ptr %out4, align 1
+; AVX512DQ:  LV: Found an estimated cost of 24 for VF 2 For instruction: store i8 %v4, ptr %out4, align 1
+; AVX512DQ:  LV: Found an estimated cost of 46 for VF 4 For instruction: store i8 %v4, ptr %out4, align 1
+; AVX512DQ:  LV: Found an estimated cost of 87 for VF 8 For instruction: store i8 %v4, ptr %out4, align 1
+; AVX512DQ:  LV: Found an estimated cost of 166 for VF 16 For instruction: store i8 %v4, ptr %out4, align 1
+; AVX512DQ:  LV: Found an estimated cost of 336 for VF 32 For instruction: store i8 %v4, ptr %out4, align 1
+; AVX512DQ:  LV: Found an estimated cost of 675 for VF 64 For instruction: store i8 %v4, ptr %out4, align 1
 ;
 ; AVX512BW-LABEL: 'test'
 ; AVX512BW:  LV: Found an estimated cost of 1 for VF 1 For instruction: store i8 %v4, ptr %out4, align 1
-; AVX512BW:  Cost of 15 for VF 2: INTERLEAVE-GROUP with factor 5 at <badref>, ir<%out0>
-; AVX512BW:  Cost of 31 for VF 4: INTERLEAVE-GROUP with factor 5 at <badref>, ir<%out0>
-; AVX512BW:  Cost of 79 for VF 8: INTERLEAVE-GROUP with factor 5 at <badref>, ir<%out0>
-; AVX512BW:  Cost of 158 for VF 16: INTERLEAVE-GROUP with factor 5 at <badref>, ir<%out0>
-; AVX512BW:  Cost of 237 for VF 32: INTERLEAVE-GROUP with factor 5 at <badref>, ir<%out0>
-; AVX512BW:  Cost of 395 for VF 64: INTERLEAVE-GROUP with factor 5 at <badref>, ir<%out0>
+; AVX512BW:  LV: Found an estimated cost of 1 for VF 1 For instruction: store i8 %v4, ptr %out4, align 1
+; AVX512BW:  LV: Found an estimated cost of 15 for VF 2 For instruction: store i8 %v4, ptr %out4, align 1
+; AVX512BW:  LV: Found an estimated cost of 31 for VF 4 For instruction: store i8 %v4, ptr %out4, align 1
+; AVX512BW:  LV: Found an estimated cost of 79 for VF 8 For instruction: store i8 %v4, ptr %out4, align 1
+; AVX512BW:  LV: Found an estimated cost of 158 for VF 16 For instruction: store i8 %v4, ptr %out4, align 1
+; AVX512BW:  LV: Found an estimated cost of 237 for VF 32 For instruction: store i8 %v4, ptr %out4, align 1
+; AVX512BW:  LV: Found an estimated cost of 395 for VF 64 For instruction: store i8 %v4, ptr %out4, align 1
 ;
 entry:
   br label %for.body
diff --git a/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-store-i8-stride-6.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-store-i8-stride-6.ll
index f3ec189573a77..0b260d46b9173 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-store-i8-stride-6.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-store-i8-stride-6.ll
@@ -1,9 +1,9 @@
 ; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --filter "LV: Found an estimated cost of [0-9]+ for VF [0-9]+ For instruction:\s*store i8 %v5, ptr %out5"
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse2 --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=SSE2
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx  --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX1
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2 --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX2
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512vl --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX512DQ
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512vl,+avx512bw --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX512BW
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse2 --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=SSE2
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx  --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX1
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2 --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX2
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512vl --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX512DQ
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512vl,+avx512bw --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX512BW
 ; REQUIRES: asserts
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
@@ -15,44 +15,49 @@ target triple = "x86_64-unknown-linux-gnu"
 define void @test() {
 ; SSE2-LABEL: 'test'
 ; SSE2:  LV: Found an estimated cost of 1 for VF 1 For instruction: store i8 %v5, ptr %out5, align 1
-; SSE2:  Cost of 49 for VF 2: INTERLEAVE-GROUP with factor 6 at <badref>, ir<%out0>
-; SSE2:  Cost of 98 for VF 4: INTERLEAVE-GROUP with factor 6 at <badref>, ir<%out0>
-; SSE2:  Cost of 201 for VF 8: INTERLEAVE-GROUP with factor 6 at <badref>, ir<%out0>
-; SSE2:  Cost of 408 for VF 16: INTERLEAVE-GROUP with factor 6 at <badref>, ir<%out0>
+; SSE2:  LV: Found an estimated cost of 1 for VF 1 For instruction: store i8 %v5, ptr %out5, align 1
+; SSE2:  LV: Found an estimated cost of 49 for VF 2 For instruction: store i8 %v5, ptr %out5, align 1
+; SSE2:  LV: Found an estimated cost of 98 for VF 4 For instruction: store i8 %v5, ptr %out5, align 1
+; SSE2:  LV: Found an estimated cost of 201 for VF 8 For instruction: store i8 %v5, ptr %out5, align 1
+; SSE2:  LV: Found an estimated cost of 408 for VF 16 For instruction: store i8 %v5, ptr %out5, align 1
 ;
 ; AVX1-LABEL: 'test'
 ; AVX1:  LV: Found an estimated cost of 1 for VF 1 For instruction: store i8 %v5, ptr %out5, align 1
-; AVX1:  Cost of 27 for VF 2: INTERLEAVE-GROUP with factor 6 at <badref>, ir<%out0>
-; AVX1:  Cost of 53 for VF 4: INTERLEAVE-GROUP with factor 6 at <badref>, ir<%out0>
-; AVX1:  Cost of 100 for VF 8: INTERLEAVE-GROUP with factor 6 at <badref>, ir<%out0>
-; AVX1:  Cost of 198 for VF 16: INTERLEAVE-GROUP with factor 6 at <badref>, ir<%out0>
-; AVX1:  Cost of 402 for VF 32: INTERLEAVE-GROUP with factor 6 at <badref>, ir<%out0>
+; AVX1:  LV: Found an estimated cost of 1 for VF 1 For instruction: store i8 %v5, ptr %out5, align 1
+; AVX1:  LV: Found an estimated cost of 27 for VF 2 For instruction: store i8 %v5, ptr %out5, align 1
+; AVX1:  LV: Found an estimated cost of 53 for VF 4 For instruction: store i8 %v5, ptr %out5, align 1
+; AVX1:  LV: Found an estimated cost of 100 for VF 8 For instruction: store i8 %v5, ptr %out5, align 1
+; AVX1:  LV: Found an estimated cost of 198 for VF 16 For instruction: store i8 %v5, ptr %out5, align 1
+; AVX1:  LV: Found an estimated cost of 402 for VF 32 For instruction: store i8 %v5, ptr %out5, align 1
 ;
 ; AVX2-LABEL: 'test'
 ; AVX2:  LV: Found an estimated cost of 1 for VF 1 For instruction: store i8 %v5, ptr %out5, align 1
-; AVX2:  Cost of 10 for VF 2: INTERLEAVE-GROUP with factor 6 at <badref>, ir<%out0>
-; AVX2:  Cost of 12 for VF 4: INTERLEAVE-GROUP with factor 6 at <badref>, ir<%out0>
-; AVX2:  Cost of 18 for VF 8: INTERLEAVE-GROUP with factor 6 at <badref>, ir<%out0>
-; AVX2:  Cost of 30 for VF 16: INTERLEAVE-GROUP with factor 6 at <badref>, ir<%out0>
-; AVX2:  Cost of 96 for VF 32: INTERLEAVE-GROUP with factor 6 at <badref>, ir<%out0>
+; AVX2:  LV: Found an estimated cost of 1 for VF 1 For instruction: store i8 %v5, ptr %out5, align 1
+; AVX2:  LV: Found an estimated cost of 10 for VF 2 For instruction: store i8 %v5, ptr %out5, align 1
+; AVX2:  LV: Found an estimated cost of 12 for VF 4 For instruction: store i8 %v5, ptr %out5, align 1
+; AVX2:  LV: Found an estimated cost of 18 for VF 8 For instruction: store i8 %v5, ptr %out5, align 1
+; AVX2:  LV: Found an estimated cost of 30 for VF 16 For instruction: store i8 %v5, ptr %out5, align 1
+; AVX2:  LV: Found an estimated cost of 96 for VF 32 For instruction: store i8 %v5, ptr %out5, align 1
 ;
 ; AVX512DQ-LABEL: 'test'
 ; AVX512DQ:  LV: Found an estimated cost of 1 for VF 1 For instruction: store i8 %v5, ptr %out5, align 1
-; AVX512DQ:  Cost of 10 for VF 2: INTERLEAVE-GROUP with factor 6 at <badref>, ir<%out0>
-; AVX512DQ:  Cost of 12 for VF 4: INTERLEAVE-GROUP with factor 6 at <badref>, ir<%out0>
-; AVX512DQ:  Cost of 19 for VF 8: INTERLEAVE-GROUP with factor 6 at <badref>, ir<%out0>
-; AVX512DQ:  Cost of 29 for VF 16: INTERLEAVE-GROUP with factor 6 at <badref>, ir<%out0>
-; AVX512DQ:  Cost of 93 for VF 32: INTERLEAVE-GROUP with factor 6 at <badref>, ir<%out0>
-; AVX512DQ:  Cost of 810 for VF 64: INTERLEAVE-GROUP with factor 6 at <badref>, ir<%out0>
+; AVX512DQ:  LV: Found an estimated cost of 1 for VF 1 For instruction: store i8 %v5, ptr %out5, align 1
+; AVX512DQ:  LV: Found an estimated cost of 10 for VF 2 For instruction: store i8 %v5, ptr %out5, align 1
+; AVX512DQ:  LV: Found an estimated cost of 12 for VF 4 For instruction: store i8 %v5, ptr %out5, align 1
+; AVX512DQ:  LV: Found an estimated cost of 19 for VF 8 For instruction: store i8 %v5, ptr %out5, align 1
+; AVX512DQ:  LV: Found an estimated cost of 29 for VF 16 For instruction: store i8 %v5, ptr %out5, align 1
+; AVX512DQ:  LV: Found an estimated cost of 93 for VF 32 For instruction: store i8 %v5, ptr %out5, align 1
+; AVX512DQ:  LV: Found an estimated cost of 810 for VF 64 For instruction: store i8 %v5, ptr %out5, align 1
 ;
 ; AVX512BW-LABEL: 'test'
 ; AVX512BW:  LV: Found an estimated cost of 1 for VF 1 For instruction: store i8 %v5, ptr %out5, align 1
-; AVX512BW:  Cost of 18 for VF 2: INTERLEAVE-GROUP with factor 6 at <badref>, ir<%out0>
-; AVX512BW:  Cost of 38 for VF 4: INTERLEAVE-GROUP with factor 6 at <badref>, ir<%out0>
-; AVX512BW:  Cost of 98 for VF 8: INTERLEAVE-GROUP with factor 6 at <badref>, ir<%out0>
-; AVX512BW:  Cost of 197 for VF 16: INTERLEAVE-GROUP with factor 6 at <badref>, ir<%out0>
-; AVX512BW:  Cost of 295 for VF 32: INTERLEAVE-GROUP with factor 6 at <badref>, ir<%out0>
-; AVX512BW:  Cost of 591 for VF 64: INTERLEAVE-GROUP with factor 6 at <badref>, ir<%out0>
+; AVX512BW:  LV: Found an estimated cost of 1 for VF 1 For instruction: store i8 %v5, ptr %out5, align 1
+; AVX512BW:  LV: Found an estimated cost of 18 for VF 2 For instruction: store i8 %v5, ptr %out5, align 1
+; AVX512BW:  LV: Found an estimated cost of 38 for VF 4 For instruction: store i8 %v5, ptr %out5, align 1
+; AVX512BW:  LV: Found an estimated cost of 98 for VF 8 For instruction: store i8 %v5, ptr %out5, align 1
+; AVX512BW:  LV: Found an estimated cost of 197 for VF 16 For instruction: store i8 %v5, ptr %out5, align 1
+; AVX512BW:  LV: Found an estimated cost of 295 for VF 32 For instruction: store i8 %v5, ptr %out5, align 1
+; AVX512BW:  LV: Found an estimated cost of 591 for VF 64 For instruction: store i8 %v5, ptr %out5, align 1
 ;
 entry:
   br label %for.body
diff --git a/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-store-i8-stride-7.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-store-i8-stride-7.ll
index ed04d806bf415..b69559c6dae62 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-store-i8-stride-7.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-store-i8-stride-7.ll
@@ -1,9 +1,9 @@
 ; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --filter "LV: Found an estimated cost of [0-9]+ for VF [0-9]+ For instruction:\s*store i8 %v6, ptr %out6"
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse2 --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=SSE2
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx  --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX1
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2 --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX2
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512vl --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX512DQ
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512vl,+avx512bw --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX512BW
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse2 --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=SSE2
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx  --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX1
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2 --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX2
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512vl --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX512DQ
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512vl,+avx512bw --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX512BW
 ; REQUIRES: asserts
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
@@ -15,44 +15,49 @@ target triple = "x86_64-unknown-linux-gnu"
 define void @test() {
 ; SSE2-LABEL: 'test'
 ; SSE2:  LV: Found an estimated cost of 1 for VF 1 For instruction: store i8 %v6, ptr %out6, align 1
-; SSE2:  Cost of 57 for VF 2: INTERLEAVE-GROUP with factor 7 at <badref>, ir<%out0>
-; SSE2:  Cost of 112 for VF 4: INTERLEAVE-GROUP with factor 7 at <badref>, ir<%out0>
-; SSE2:  Cost of 225 for VF 8: INTERLEAVE-GROUP with factor 7 at <badref>, ir<%out0>
-; SSE2:  Cost of 456 for VF 16: INTERLEAVE-GROUP with factor 7 at <badref>, ir<%out0>
+; SSE2:  LV: Found an estimated cost of 1 for VF 1 For instruction: store i8 %v6, ptr %out6, align 1
+; SSE2:  LV: Found an estimated cost of 57 for VF 2 For instruction: store i8 %v6, ptr %out6, align 1
+; SSE2:  LV: Found an estimated cost of 112 for VF 4 For instruction: store i8 %v6, ptr %out6, align 1
+; SSE2:  LV: Found an estimated cost of 225 for VF 8 For instruction: store i8 %v6, ptr %out6, align 1
+; SSE2:  LV: Found an estimated cost of 456 for VF 16 For instruction: store i8 %v6, ptr %out6, align 1
 ;
 ; AVX1-LABEL: 'test'
 ; AVX1:  LV: Found an estimated cost of 1 for VF 1 For instruction: store i8 %v6, ptr %out6, align 1
-; AVX1:  Cost of 34 for VF 2: INTERLEAVE-GROUP with factor 7 at <badref>, ir<%out0>
-; AVX1:  Cost of 63 for VF 4: INTERLEAVE-GROUP with factor 7 at <badref>, ir<%out0>
-; AVX1:  Cost of 119 for VF 8: INTERLEAVE-GROUP with factor 7 at <badref>, ir<%out0>
-; AVX1:  Cost of 232 for VF 16: INTERLEAVE-GROUP with factor 7 at <badref>, ir<%out0>
-; AVX1:  Cost of 469 for VF 32: INTERLEAVE-GROUP with factor 7 at <badref>, ir<%out0>
+; AVX1:  LV: Found an estimated cost of 1 for VF 1 For instruction: store i8 %v6, ptr %out6, align 1
+; AVX1:  LV: Found an estimated cost of 34 for VF 2 For instruction: store i8 %v6, ptr %out6, align 1
+; AVX1:  LV: Found an estimated cost of 63 for VF 4 For instruction: store i8 %v6, ptr %out6, align 1
+; AVX1:  LV: Found an estimated cost of 119 for VF 8 For instruction: store i8 %v6, ptr %out6, align 1
+; AVX1:  LV: Found an estimated cost of 232 for VF 16 For instruction: store i8 %v6, ptr %out6, align 1
+; AVX1:  LV: Found an estimated cost of 469 for VF 32 For instruction: store i8 %v6, ptr %out6, align 1
 ;
 ; AVX2-LABEL: 'test'
 ; AVX2:  LV: Found an estimated cost of 1 for VF 1 For instruction: store i8 %v6, ptr %out6, align 1
-; AVX2:  Cost of 34 for VF 2: INTERLEAVE-GROUP with factor 7 at <badref>, ir<%out0>
-; AVX2:  Cost of 63 for VF 4: INTERLEAVE-GROUP with factor 7 at <badref>, ir<%out0>
-; AVX2:  Cost of 119 for VF 8: INTERLEAVE-GROUP with factor 7 at <badref>, ir<%out0>
-; AVX2:  Cost of 232 for VF 16: INTERLEAVE-GROUP with factor 7 at <badref>, ir<%out0>
-; AVX2:  Cost of 469 for VF 32: INTERLEAVE-GROUP with factor 7 at <badref>, ir<%out0>
+; AVX2:  LV: Found an estimated cost of 1 for VF 1 For instruction: store i8 %v6, ptr %out6, align 1
+; AVX2:  LV: Found an estimated cost of 34 for VF 2 For instruction: store i8 %v6, ptr %out6, align 1
+; AVX2:  LV: Found an estimated cost of 63 for VF 4 For instruction: store i8 %v6, ptr %out6, align 1
+; AVX2:  LV: Found an estimated cost of 119 for VF 8 For instruction: store i8 %v6, ptr %out6, align 1
+; AVX2:  LV: Found an estimated cost of 232 for VF 16 For instruction: store i8 %v6, ptr %out6, align 1
+; AVX2:  LV: Found an estimated cost of 469 for VF 32 For instruction: store i8 %v6, ptr %out6, align 1
 ;
 ; AVX512DQ-LABEL: 'test'
 ; AVX512DQ:  LV: Found an estimated cost of 1 for VF 1 For instruction: store i8 %v6, ptr %out6, align 1
-; AVX512DQ:  Cost of 34 for VF 2: INTERLEAVE-GROUP with factor 7 at <badref>, ir<%out0>
-; AVX512DQ:  Cost of 63 for VF 4: INTERLEAVE-GROUP with factor 7 at <badref>, ir<%out0>
-; AVX512DQ:  Cost of 121 for VF 8: INTERLEAVE-GROUP with factor 7 at <badref>, ir<%out0>
-; AVX512DQ:  Cost of 234 for VF 16: INTERLEAVE-GROUP with factor 7 at <badref>, ir<%out0>
-; AVX512DQ:  Cost of 470 for VF 32: INTERLEAVE-GROUP with factor 7 at <badref>, ir<%out0>
-; AVX512DQ:  Cost of 945 for VF 64: INTERLEAVE-GROUP with factor 7 at <badref>, ir<%out0>
+; AVX512DQ:  LV: Found an estimated cost of 1 for VF 1 For instruction: store i8 %v6, ptr %out6, align 1
+; AVX512DQ:  LV: Found an estimated cost of 34 for VF 2 For instruction: store i8 %v6, ptr %out6, align 1
+; AVX512DQ:  LV: Found an estimated cost of 63 for VF 4 For instruction: store i8 %v6, ptr %out6, align 1
+; AVX512DQ:  LV: Found an estimated cost of 121 for VF 8 For instruction: store i8 %v6, ptr %out6, align 1
+; AVX512DQ:  LV: Found an estimated cost of 234 for VF 16 For instruction: store i8 %v6, ptr %out6, align 1
+; AVX512DQ:  LV: Found an estimated cost of 470 for VF 32 For instruction: store i8 %v6, ptr %out6, align 1
+; AVX512DQ:  LV: Found an estimated cost of 945 for VF 64 For instruction: store i8 %v6, ptr %out6, align 1
 ;
 ; AVX512BW-LABEL: 'test'
 ; AVX512BW:  LV: Found an estimated cost of 1 for VF 1 For instruction: store i8 %v6, ptr %out6, align 1
-; AVX512BW:  Cost of 22 for VF 2: INTERLEAVE-GROUP with factor 7 at <badref>, ir<%out0>
-; AVX512BW:  Cost of 46 for VF 4: INTERLEAVE-GROUP with factor 7 at <badref>, ir<%out0>
-; AVX512BW:  Cost of 118 for VF 8: INTERLEAVE-GROUP with factor 7 at <badref>, ir<%out0>
-; AVX512BW:  Cost of 236 for VF 16: INTERLEAVE-GROUP with factor 7 at <badref>, ir<%out0>
-; AVX512BW:  Cost of 472 for VF 32: INTERLEAVE-GROUP with factor 7 at <badref>, ir<%out0>
-; AVX512BW:  Cost of 826 for VF 64: INTERLEAVE-GROUP with factor 7 at <badref>, ir<%out0>
+; AVX512BW:  LV: Found an estimated cost of 1 for VF 1 For instruction: store i8 %v6, ptr %out6, align 1
+; AVX512BW:  LV: Found an estimated cost of 22 for VF 2 For instruction: store i8 %v6, ptr %out6, align 1
+; AVX512BW:  LV: Found an estimated cost of 46 for VF 4 For instruction: store i8 %v6, ptr %out6, align 1
+; AVX512BW:  LV: Found an estimated cost of 118 for VF 8 For instruction: store i8 %v6, ptr %out6, align 1
+; AVX512BW:  LV: Found an estimated cost of 236 for VF 16 For instruction: store i8 %v6, ptr %out6, align 1
+; AVX512BW:  LV: Found an estimated cost of 472 for VF 32 For instruction: store i8 %v6, ptr %out6, align 1
+; AVX512BW:  LV: Found an estimated cost of 826 for VF 64 For instruction: store i8 %v6, ptr %out6, align 1
 ;
 entry:
   br label %for.body
diff --git a/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-store-i8-stride-8.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-store-i8-stride-8.ll
index 62f8734c487a0..078528b58f6f7 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-store-i8-stride-8.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-store-i8-stride-8.ll
@@ -1,9 +1,9 @@
 ; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --filter "LV: Found an estimated cost of [0-9]+ for VF [0-9]+ For instruction:\s*store i8 %v7, ptr %out7"
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse2 --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=SSE2
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx  --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX1
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2 --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX2
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512vl --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX512DQ
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512vl,+avx512bw --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=AVX512BW
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse2 --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=SSE2
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx  --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX1
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2 --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX2
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512vl --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX512DQ
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512vl,+avx512bw --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=AVX512BW
 ; REQUIRES: asserts
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
@@ -15,6 +15,7 @@ target triple = "x86_64-unknown-linux-gnu"
 define void @test() {
 ; SSE2-LABEL: 'test'
 ; SSE2:  LV: Found an estimated cost of 1 for VF 1 For instruction: store i8 %v7, ptr %out7, align 1
+; SSE2:  LV: Found an estimated cost of 1 for VF 1 For instruction: store i8 %v7, ptr %out7, align 1
 ; SSE2:  LV: Found an estimated cost of 56 for VF 2 For instruction: store i8 %v7, ptr %out7, align 1
 ; SSE2:  LV: Found an estimated cost of 120 for VF 4 For instruction: store i8 %v7, ptr %out7, align 1
 ; SSE2:  LV: Found an estimated cost of 248 for VF 8 For instruction: store i8 %v7, ptr %out7, align 1
@@ -22,6 +23,7 @@ define void @test() {
 ;
 ; AVX1-LABEL: 'test'
 ; AVX1:  LV: Found an estimated cost of 1 for VF 1 For instruction: store i8 %v7, ptr %out7, align 1
+; AVX1:  LV: Found an estimated cost of 1 for VF 1 For instruction: store i8 %v7, ptr %out7, align 1
 ; AVX1:  LV: Found an estimated cost of 33 for VF 2 For instruction: store i8 %v7, ptr %out7, align 1
 ; AVX1:  LV: Found an estimated cost of 66 for VF 4 For instruction: store i8 %v7, ptr %out7, align 1
 ; AVX1:  LV: Found an estimated cost of 132 for VF 8 For instruction: store i8 %v7, ptr %out7, align 1
@@ -30,6 +32,7 @@ define void @test() {
 ;
 ; AVX2-LABEL: 'test'
 ; AVX2:  LV: Found an estimated cost of 1 for VF 1 For instruction: store i8 %v7, ptr %out7, align 1
+; AVX2:  LV: Found an estimated cost of 1 for VF 1 For instruction: store i8 %v7, ptr %out7, align 1
 ; AVX2:  LV: Found an estimated cost of 33 for VF 2 For instruction: store i8 %v7, ptr %out7, align 1
 ; AVX2:  LV: Found an estimated cost of 66 for VF 4 For instruction: store i8 %v7, ptr %out7, align 1
 ; AVX2:  LV: Found an estimated cost of 132 for VF 8 For instruction: store i8 %v7, ptr %out7, align 1
@@ -38,6 +41,7 @@ define void @test() {
 ;
 ; AVX512DQ-LABEL: 'test'
 ; AVX512DQ:  LV: Found an estimated cost of 1 for VF 1 For instruction: store i8 %v7, ptr %out7, align 1
+; AVX512DQ:  LV: Found an estimated cost of 1 for VF 1 For instruction: store i8 %v7, ptr %out7, align 1
 ; AVX512DQ:  LV: Found an estimated cost of 33 for VF 2 For instruction: store i8 %v7, ptr %out7, align 1
 ; AVX512DQ:  LV: Found an estimated cost of 66 for VF 4 For instruction: store i8 %v7, ptr %out7, align 1
 ; AVX512DQ:  LV: Found an estimated cost of 132 for VF 8 For instruction: store i8 %v7, ptr %out7, align 1
@@ -47,6 +51,7 @@ define void @test() {
 ;
 ; AVX512BW-LABEL: 'test'
 ; AVX512BW:  LV: Found an estimated cost of 1 for VF 1 For instruction: store i8 %v7, ptr %out7, align 1
+; AVX512BW:  LV: Found an estimated cost of 1 for VF 1 For instruction: store i8 %v7, ptr %out7, align 1
 ; AVX512BW:  LV: Found an estimated cost of 25 for VF 2 For instruction: store i8 %v7, ptr %out7, align 1
 ; AVX512BW:  LV: Found an estimated cost of 53 for VF 4 For instruction: store i8 %v7, ptr %out7, align 1
 ; AVX512BW:  LV: Found an estimated cost of 137 for VF 8 For instruction: store i8 %v7, ptr %out7, align 1
diff --git a/llvm/test/Transforms/LoopVectorize/X86/CostModel/masked-gather-i32-with-i8-index.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/masked-gather-i32-with-i8-index.ll
index d3d439f37c440..caf8f10d1169c 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/CostModel/masked-gather-i32-with-i8-index.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/CostModel/masked-gather-i32-with-i8-index.ll
@@ -1,10 +1,10 @@
 ; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --filter "LV: Found an estimated cost of [0-9]+ for VF [0-9]+ For instruction:\s*%valB.loaded = load i32, ptr %inB, align 4"
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse2 --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefixes=SSE
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse4.2 --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefixes=SSE
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx  --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefixes=AVX1
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2,-fast-gather --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefixes=AVX2-SLOWGATHER
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2,+fast-gather --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefixes=AVX2-FASTGATHER
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512bw --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefixes=AVX512
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse2 --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefixes=SSE
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse4.2 --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefixes=SSE
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx  --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefixes=AVX1
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2,-fast-gather --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefixes=AVX2-SLOWGATHER
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2,+fast-gather --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefixes=AVX2-FASTGATHER
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512bw --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefixes=AVX512
 
 ; REQUIRES: asserts
 
diff --git a/llvm/test/Transforms/LoopVectorize/X86/CostModel/masked-gather-i64-with-i8-index.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/masked-gather-i64-with-i8-index.ll
index f1dec928391eb..c70ab9962785b 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/CostModel/masked-gather-i64-with-i8-index.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/CostModel/masked-gather-i64-with-i8-index.ll
@@ -1,10 +1,10 @@
 ; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --filter "LV: Found an estimated cost of [0-9]+ for VF [0-9]+ For instruction:\s*%valB.loaded = load i64, ptr %inB, align 8"
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse2 --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefixes=SSE
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse4.2 --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefixes=SSE
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx  --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefixes=AVX1
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2,-fast-gather --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefixes=AVX2-SLOWGATHER
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2,+fast-gather --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefixes=AVX2-FASTGATHER
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512bw --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefixes=AVX512
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse2 --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefixes=SSE
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse4.2 --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefixes=SSE
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx  --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefixes=AVX1
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2,-fast-gather --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefixes=AVX2-SLOWGATHER
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2,+fast-gather --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefixes=AVX2-FASTGATHER
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512bw --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefixes=AVX512
 
 ; REQUIRES: asserts
 
diff --git a/llvm/test/Transforms/LoopVectorize/X86/CostModel/masked-interleaved-load-i16.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/masked-interleaved-load-i16.ll
index e867e5f5bcfc7..665b0c3bac040 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/CostModel/masked-interleaved-load-i16.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/CostModel/masked-interleaved-load-i16.ll
@@ -1,6 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --filter "LV: Found an estimated cost of [0-9]+ for VF [0-9]+ For instruction:\s*%i[2,4] = load i16, ptr %[a-zA-Z0-7]+, align 2"
-; RUN: opt -passes=loop-vectorize -enable-interleaved-mem-accesses -prefer-predicate-over-epilogue=predicate-dont-vectorize -S -mcpu=skx --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=DISABLED_MASKED_STRIDED
-; RUN: opt -passes=loop-vectorize -enable-interleaved-mem-accesses -enable-masked-interleaved-mem-accesses -prefer-predicate-over-epilogue=predicate-dont-vectorize -S -mcpu=skx --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=ENABLED_MASKED_STRIDED
+; RUN: opt -passes=loop-vectorize -enable-interleaved-mem-accesses -prefer-predicate-over-epilogue=predicate-dont-vectorize -S -mcpu=skx --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=DISABLED_MASKED_STRIDED
+; RUN: opt -passes=loop-vectorize -enable-interleaved-mem-accesses -enable-masked-interleaved-mem-accesses -prefer-predicate-over-epilogue=predicate-dont-vectorize -S -mcpu=skx --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=ENABLED_MASKED_STRIDED
 ; REQUIRES: asserts
 
 target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
diff --git a/llvm/test/Transforms/LoopVectorize/X86/CostModel/masked-interleaved-store-i16.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/masked-interleaved-store-i16.ll
index 41dd266d0a87a..a286df9bc2fc7 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/CostModel/masked-interleaved-store-i16.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/CostModel/masked-interleaved-store-i16.ll
@@ -1,6 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --filter "LV: Found an estimated cost of [0-9]+ for VF [0-9]+ For instruction:\s*store i16 %[0,2], ptr %[a-zA-Z0-7]+, align 2"
-; RUN: opt -passes=loop-vectorize -enable-interleaved-mem-accesses -prefer-predicate-over-epilogue=predicate-dont-vectorize -S -mcpu=skx --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=DISABLED_MASKED_STRIDED
-; RUN: opt -passes=loop-vectorize -enable-interleaved-mem-accesses -enable-masked-interleaved-mem-accesses -prefer-predicate-over-epilogue=predicate-dont-vectorize -S -mcpu=skx --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=ENABLED_MASKED_STRIDED
+; RUN: opt -passes=loop-vectorize -enable-interleaved-mem-accesses -prefer-predicate-over-epilogue=predicate-dont-vectorize -S -mcpu=skx --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=DISABLED_MASKED_STRIDED
+; RUN: opt -passes=loop-vectorize -enable-interleaved-mem-accesses -enable-masked-interleaved-mem-accesses -prefer-predicate-over-epilogue=predicate-dont-vectorize -S -mcpu=skx --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=ENABLED_MASKED_STRIDED
 ; REQUIRES: asserts
 
 target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
diff --git a/llvm/test/Transforms/LoopVectorize/X86/CostModel/masked-load-i16.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/masked-load-i16.ll
index 5f910a6dfaa61..4f3f141de3adb 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/CostModel/masked-load-i16.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/CostModel/masked-load-i16.ll
@@ -1,10 +1,10 @@
 ; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --filter "LV: Found an estimated cost of [0-9]+ for VF [0-9]+ For instruction:\s*%valB.loaded = load i16, ptr %inB, align 2"
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse2 --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefixes=SSE
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse4.2 --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefixes=SSE
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx  --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefixes=AVX1
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2,-fast-gather --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefixes=AVX2
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2,+fast-gather --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefixes=AVX2
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512bw --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefixes=AVX512
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse2 --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefixes=SSE
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse4.2 --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefixes=SSE
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx  --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefixes=AVX1
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2,-fast-gather --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefixes=AVX2
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2,+fast-gather --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefixes=AVX2
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512bw --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefixes=AVX512
 
 ; REQUIRES: asserts
 
diff --git a/llvm/test/Transforms/LoopVectorize/X86/CostModel/masked-load-i32.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/masked-load-i32.ll
index 095444a727fe8..6d613871639bb 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/CostModel/masked-load-i32.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/CostModel/masked-load-i32.ll
@@ -1,10 +1,10 @@
 ; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --filter "LV: Found an estimated cost of [0-9]+ for VF [0-9]+ For instruction:\s*%valB.loaded = load i32, ptr %inB, align 4"
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse2 --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefixes=SSE
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse4.2 --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefixes=SSE
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx  --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefixes=AVX1
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2,-fast-gather --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefixes=AVX2
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2,+fast-gather --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefixes=AVX2
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512bw --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefixes=AVX512
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse2 --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefixes=SSE
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse4.2 --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefixes=SSE
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx  --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefixes=AVX1
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2,-fast-gather --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefixes=AVX2
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2,+fast-gather --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefixes=AVX2
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512bw --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefixes=AVX512
 
 ; REQUIRES: asserts
 
diff --git a/llvm/test/Transforms/LoopVectorize/X86/CostModel/masked-load-i64.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/masked-load-i64.ll
index cd161fcdeaeae..c0d1144fab800 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/CostModel/masked-load-i64.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/CostModel/masked-load-i64.ll
@@ -1,10 +1,10 @@
 ; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --filter "LV: Found an estimated cost of [0-9]+ for VF [0-9]+ For instruction:\s*%valB.loaded = load i64, ptr %inB, align 8"
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse2 --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefixes=SSE
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse4.2 --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefixes=SSE
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx  --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefixes=AVX1
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2,-fast-gather --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefixes=AVX2
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2,+fast-gather --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefixes=AVX2
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512bw --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefixes=AVX512
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse2 --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefixes=SSE
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse4.2 --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefixes=SSE
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx  --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefixes=AVX1
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2,-fast-gather --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefixes=AVX2
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2,+fast-gather --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefixes=AVX2
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512bw --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefixes=AVX512
 
 ; REQUIRES: asserts
 
diff --git a/llvm/test/Transforms/LoopVectorize/X86/CostModel/masked-load-i8.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/masked-load-i8.ll
index 4c2cbc41e53ff..65075cd094527 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/CostModel/masked-load-i8.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/CostModel/masked-load-i8.ll
@@ -1,10 +1,10 @@
 ; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --filter "LV: Found an estimated cost of [0-9]+ for VF [0-9]+ For instruction:\s*%valB.loaded = load i8, ptr %inB, align 1"
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse2 --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefixes=SSE
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse4.2 --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefixes=SSE
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx  --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefixes=AVX1
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2,-fast-gather --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefixes=AVX2
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2,+fast-gather --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefixes=AVX2
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512bw --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefixes=AVX512
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse2 --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefixes=SSE
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse4.2 --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefixes=SSE
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx  --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefixes=AVX1
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2,-fast-gather --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefixes=AVX2
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2,+fast-gather --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefixes=AVX2
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512bw --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefixes=AVX512
 
 ; REQUIRES: asserts
 
diff --git a/llvm/test/Transforms/LoopVectorize/X86/CostModel/masked-scatter-i32-with-i8-index.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/masked-scatter-i32-with-i8-index.ll
index 95d6d457ee4d8..5e67bd57754e4 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/CostModel/masked-scatter-i32-with-i8-index.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/CostModel/masked-scatter-i32-with-i8-index.ll
@@ -1,10 +1,10 @@
 ; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --filter "LV: Found an estimated cost of [0-9]+ for VF [0-9]+ For instruction:\s*store i32 %valB, ptr %out, align 4"
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse2 --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefixes=SSE2
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse4.2 --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefixes=SSE42
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx  --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefixes=AVX1
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2,-fast-gather --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefixes=AVX2
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2,+fast-gather --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefixes=AVX2
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512bw --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefixes=AVX512
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse2 --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefixes=SSE2
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse4.2 --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefixes=SSE42
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx  --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefixes=AVX1
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2,-fast-gather --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefixes=AVX2
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2,+fast-gather --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefixes=AVX2
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512bw --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefixes=AVX512
 
 ; REQUIRES: asserts
 
diff --git a/llvm/test/Transforms/LoopVectorize/X86/CostModel/masked-scatter-i64-with-i8-index.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/masked-scatter-i64-with-i8-index.ll
index 3ae93b14f0d62..629ee1d2711ae 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/CostModel/masked-scatter-i64-with-i8-index.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/CostModel/masked-scatter-i64-with-i8-index.ll
@@ -1,10 +1,10 @@
 ; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --filter "LV: Found an estimated cost of [0-9]+ for VF [0-9]+ For instruction:\s*store i64 %valB, ptr %out, align 8"
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse2 --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefixes=SSE2
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse4.2 --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefixes=SSE42
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx  --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefixes=AVX1
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2,-fast-gather --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefixes=AVX2
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2,+fast-gather --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefixes=AVX2
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512bw --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefixes=AVX512
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse2 --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefixes=SSE2
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse4.2 --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefixes=SSE42
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx  --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefixes=AVX1
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2,-fast-gather --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefixes=AVX2
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2,+fast-gather --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefixes=AVX2
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512bw --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefixes=AVX512
 
 ; REQUIRES: asserts
 
diff --git a/llvm/test/Transforms/LoopVectorize/X86/CostModel/masked-store-i16.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/masked-store-i16.ll
index fa7c3d9fcdcab..1d51a32a520a9 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/CostModel/masked-store-i16.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/CostModel/masked-store-i16.ll
@@ -1,10 +1,10 @@
 ; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --filter "LV: Found an estimated cost of [0-9]+ for VF [0-9]+ For instruction:\s*store i16 %valB, ptr %out, align 2"
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse2 --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefixes=SSE
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse4.2 --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefixes=SSE
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx  --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefixes=AVX1
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2,-fast-gather --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefixes=AVX2
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2,+fast-gather --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefixes=AVX2
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512bw --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefixes=AVX512
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse2 --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefixes=SSE
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse4.2 --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefixes=SSE
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx  --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefixes=AVX1
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2,-fast-gather --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefixes=AVX2
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2,+fast-gather --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefixes=AVX2
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512bw --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefixes=AVX512
 
 ; REQUIRES: asserts
 
diff --git a/llvm/test/Transforms/LoopVectorize/X86/CostModel/masked-store-i32.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/masked-store-i32.ll
index 0efaefb1db5ce..f011d06d319bb 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/CostModel/masked-store-i32.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/CostModel/masked-store-i32.ll
@@ -1,10 +1,10 @@
 ; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --filter "LV: Found an estimated cost of [0-9]+ for VF [0-9]+ For instruction:\s*store i32 %valB, ptr %out, align 4"
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse2 --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefixes=SSE2
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse4.2 --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefixes=SSE42
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx  --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefixes=AVX1
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2,-fast-gather --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefixes=AVX2
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2,+fast-gather --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefixes=AVX2
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512bw --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefixes=AVX512
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse2 --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefixes=SSE2
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse4.2 --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefixes=SSE42
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx  --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefixes=AVX1
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2,-fast-gather --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefixes=AVX2
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2,+fast-gather --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefixes=AVX2
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512bw --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefixes=AVX512
 
 ; REQUIRES: asserts
 
diff --git a/llvm/test/Transforms/LoopVectorize/X86/CostModel/masked-store-i64.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/masked-store-i64.ll
index 9bd452fffcd3d..c004b16ae207d 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/CostModel/masked-store-i64.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/CostModel/masked-store-i64.ll
@@ -1,10 +1,10 @@
 ; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --filter "LV: Found an estimated cost of [0-9]+ for VF [0-9]+ For instruction:\s*store i64 %valB, ptr %out, align 8"
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse2 --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefixes=SSE2
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse4.2 --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefixes=SSE42
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx  --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefixes=AVX1
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2,-fast-gather --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefixes=AVX2
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2,+fast-gather --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefixes=AVX2
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512bw --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefixes=AVX512
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse2 --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefixes=SSE2
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse4.2 --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefixes=SSE42
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx  --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefixes=AVX1
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2,-fast-gather --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefixes=AVX2
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2,+fast-gather --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefixes=AVX2
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512bw --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefixes=AVX512
 
 ; REQUIRES: asserts
 
diff --git a/llvm/test/Transforms/LoopVectorize/X86/CostModel/masked-store-i8.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/masked-store-i8.ll
index 48130f0b57bec..8bbe624849783 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/CostModel/masked-store-i8.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/CostModel/masked-store-i8.ll
@@ -1,10 +1,10 @@
 ; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --filter "LV: Found an estimated cost of [0-9]+ for VF [0-9]+ For instruction:\s*store i8 %valB, ptr %out, align 1"
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse2 --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefixes=SSE2
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse4.2 --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefixes=SSE42
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx  --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefixes=AVX1
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2,-fast-gather --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefixes=AVX2
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2,+fast-gather --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefixes=AVX2
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512bw --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefixes=AVX512
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse2 --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefixes=SSE2
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse4.2 --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefixes=SSE42
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx  --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefixes=AVX1
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2,-fast-gather --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefixes=AVX2
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2,+fast-gather --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefixes=AVX2
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512bw --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefixes=AVX512
 
 ; REQUIRES: asserts
 
diff --git a/llvm/test/Transforms/LoopVectorize/X86/CostModel/scatter-i16-with-i8-index.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/scatter-i16-with-i8-index.ll
index 3fd22a89d3cd0..bd6b16831d09c 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/CostModel/scatter-i16-with-i8-index.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/CostModel/scatter-i16-with-i8-index.ll
@@ -1,10 +1,10 @@
 ; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --filter "LV: Found an estimated cost of [0-9]+ for VF [0-9]+ For instruction:\s*store i16 %valB, ptr %out, align 2"
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse2 --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefixes=SSE2
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse4.2 --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefixes=SSE42
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx  --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefixes=AVX1
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2,-fast-gather --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefixes=AVX2
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2,+fast-gather --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefixes=AVX2
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512bw --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefixes=AVX512
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse2 --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefixes=SSE2
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse4.2 --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefixes=SSE42
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx  --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefixes=AVX1
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2,-fast-gather --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefixes=AVX2
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2,+fast-gather --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefixes=AVX2
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512bw --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefixes=AVX512
 
 ; REQUIRES: asserts
 
diff --git a/llvm/test/Transforms/LoopVectorize/X86/CostModel/scatter-i32-with-i8-index.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/scatter-i32-with-i8-index.ll
index 45732f3d76745..de76eb0782c0d 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/CostModel/scatter-i32-with-i8-index.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/CostModel/scatter-i32-with-i8-index.ll
@@ -1,10 +1,10 @@
 ; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --filter "LV: Found an estimated cost of [0-9]+ for VF [0-9]+ For instruction:\s*store i32 %valB, ptr %out, align 4"
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse2 --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefixes=SSE2
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse4.2 --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefixes=SSE42
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx  --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefixes=AVX1
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2,-fast-gather --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefixes=AVX2
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2,+fast-gather --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefixes=AVX2
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512bw --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefixes=AVX512
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse2 --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefixes=SSE2
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse4.2 --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefixes=SSE42
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx  --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefixes=AVX1
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2,-fast-gather --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefixes=AVX2
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2,+fast-gather --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefixes=AVX2
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512bw --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefixes=AVX512
 
 ; REQUIRES: asserts
 
diff --git a/llvm/test/Transforms/LoopVectorize/X86/CostModel/scatter-i64-with-i8-index.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/scatter-i64-with-i8-index.ll
index 08033ccd200e2..c69711d1b71d5 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/CostModel/scatter-i64-with-i8-index.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/CostModel/scatter-i64-with-i8-index.ll
@@ -1,10 +1,10 @@
 ; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --filter "LV: Found an estimated cost of [0-9]+ for VF [0-9]+ For instruction:\s*store i64 %valB, ptr %out, align 8"
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse2 --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefixes=SSE2
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse4.2 --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefixes=SSE42
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx  --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefixes=AVX1
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2,-fast-gather --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefixes=AVX2
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2,+fast-gather --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefixes=AVX2
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512bw --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefixes=AVX512
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse2 --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefixes=SSE2
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse4.2 --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefixes=SSE42
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx  --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefixes=AVX1
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2,-fast-gather --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefixes=AVX2
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2,+fast-gather --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefixes=AVX2
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512bw --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefixes=AVX512
 
 ; REQUIRES: asserts
 
diff --git a/llvm/test/Transforms/LoopVectorize/X86/CostModel/scatter-i8-with-i8-index.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/scatter-i8-with-i8-index.ll
index 344d39631ff70..4f62383d67927 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/CostModel/scatter-i8-with-i8-index.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/CostModel/scatter-i8-with-i8-index.ll
@@ -1,10 +1,10 @@
 ; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --filter "LV: Found an estimated cost of [0-9]+ for VF [0-9]+ For instruction:\s*store i8 %valB, ptr %out, align 1"
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse2 --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefixes=SSE2
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse4.2 --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefixes=SSE42
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx  --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefixes=AVX1
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2,-fast-gather --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefixes=AVX2
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2,+fast-gather --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefixes=AVX2
-; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512bw --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefixes=AVX512
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse2 --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefixes=SSE2
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse4.2 --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefixes=SSE42
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx  --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefixes=AVX1
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2,-fast-gather --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefixes=AVX2
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2,+fast-gather --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefixes=AVX2
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512bw --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefixes=AVX512
 
 ; REQUIRES: asserts
 
diff --git a/llvm/test/Transforms/LoopVectorize/X86/CostModel/strided-load-i16.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/strided-load-i16.ll
index fee6e279877ed..a7ddbe2ed2c6b 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/CostModel/strided-load-i16.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/CostModel/strided-load-i16.ll
@@ -1,5 +1,5 @@
 ; REQUIRES: asserts
-; RUN: opt -passes=loop-vectorize -S -mattr=avx512bw --debug-only=loop-vectorize < %s 2>&1| FileCheck %s
+; RUN: opt -passes=loop-vectorize -S -mattr=avx512bw --debug-only=loop-vectorize --disable-output < %s 2>&1| FileCheck %s
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
diff --git a/llvm/test/Transforms/LoopVectorize/X86/CostModel/strided-load-i32.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/strided-load-i32.ll
index 1ef918c83c359..89920bb83eca9 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/CostModel/strided-load-i32.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/CostModel/strided-load-i32.ll
@@ -1,5 +1,5 @@
 ; REQUIRES: asserts
-; RUN: opt -passes=loop-vectorize -S -mattr=avx512f --debug-only=loop-vectorize < %s 2>&1| FileCheck %s
+; RUN: opt -passes=loop-vectorize -S -mattr=avx512f --debug-only=loop-vectorize --disable-output < %s 2>&1| FileCheck %s
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
diff --git a/llvm/test/Transforms/LoopVectorize/X86/CostModel/strided-load-i64.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/strided-load-i64.ll
index 9efc0737d5592..8428eadf0d280 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/CostModel/strided-load-i64.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/CostModel/strided-load-i64.ll
@@ -1,5 +1,5 @@
 ; REQUIRES: asserts
-; RUN: opt -passes=loop-vectorize -S -mattr=avx512f --debug-only=loop-vectorize < %s 2>&1| FileCheck %s
+; RUN: opt -passes=loop-vectorize -S -mattr=avx512f --debug-only=loop-vectorize --disable-output < %s 2>&1| FileCheck %s
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
diff --git a/llvm/test/Transforms/LoopVectorize/X86/CostModel/strided-load-i8.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/strided-load-i8.ll
index 018f6e94e02d9..85e189c88aca2 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/CostModel/strided-load-i8.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/CostModel/strided-load-i8.ll
@@ -1,5 +1,5 @@
 ; REQUIRES: asserts
-; RUN: opt -passes=loop-vectorize -S -mattr=avx512bw --debug-only=loop-vectorize < %s 2>&1| FileCheck %s
+; RUN: opt -passes=loop-vectorize -S -mattr=avx512bw --debug-only=loop-vectorize --disable-output < %s 2>&1| FileCheck %s
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
diff --git a/llvm/test/Transforms/LoopVectorize/X86/CostModel/vpinstruction-cost.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/vpinstruction-cost.ll
new file mode 100644
index 0000000000000..bb85b88f181f7
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/X86/CostModel/vpinstruction-cost.ll
@@ -0,0 +1,74 @@
+; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --filter "Cost of"
+; RUN: opt -S -passes=loop-vectorize -mcpu=skylake-avx512 -mtriple=x86_64-apple-macosx -debug -disable-output -S %s 2>&1 | FileCheck %s
+
+; REQUIRES: asserts
+
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128"
+
+define void @wide_or_replaced_with_add_vpinstruction(ptr %src, ptr noalias %dst) {
+; CHECK-LABEL: 'wide_or_replaced_with_add_vpinstruction'
+; CHECK:  Cost of 1 for VF 2: induction instruction %iv.next = add nuw nsw i64 %iv, 1
+; CHECK:  Cost of 0 for VF 2: induction instruction %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop.latch ]
+; CHECK:  Cost of 1 for VF 2: exit condition instruction %exitcond = icmp eq i64 %iv.next, 32
+; CHECK:  Cost of 0 for VF 2: EMIT vp<%3> = CANONICAL-INDUCTION ir<0>, vp<%index.next>
+; CHECK:  Cost of 0 for VF 2: ir<%iv> = WIDEN-INDUCTION ir<0>, ir<1>, vp<%0>
+; CHECK:  Cost of 0 for VF 2: vp<%4> = SCALAR-STEPS vp<%3>, ir<1>
+; CHECK:  Cost of 0 for VF 2: CLONE ir<%g.src> = getelementptr inbounds ir<%src>, vp<%4>
+; CHECK:  Cost of 0 for VF 2: vp<%5> = vector-pointer ir<%g.src>
+; CHECK:  Cost of 1 for VF 2: WIDEN ir<%l> = load vp<%5>
+; CHECK:  Cost of 1 for VF 2: WIDEN ir<%iv.4> = add ir<%iv>, ir<4>
+; CHECK:  Cost of 1 for VF 2: WIDEN ir<%c> = icmp ule ir<%l>, ir<128>
+; CHECK:  Cost of 1 for VF 2: EMIT ir<%or> = add ir<%iv.4>, ir<1>
+; CHECK:  Cost of 0 for VF 2: CLONE ir<%g.dst> = getelementptr ir<%dst>, ir<%or>
+; CHECK:  Cost of 0 for VF 2: vp<%6> = vector-pointer ir<%g.dst>
+; CHECK:  Cost of 1 for VF 2: WIDEN store vp<%6>, ir<%iv.4>, ir<%c>
+; CHECK:  Cost of 0 for VF 2: EMIT vp<%index.next> = add nuw vp<%3>, vp<%1>
+; CHECK:  Cost of 0 for VF 2: EMIT branch-on-count vp<%index.next>, vp<%2>
+; CHECK:  Cost of 0 for VF 2: vector loop backedge
+; CHECK:  Cost of 1 for VF 4: induction instruction %iv.next = add nuw nsw i64 %iv, 1
+; CHECK:  Cost of 0 for VF 4: induction instruction %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop.latch ]
+; CHECK:  Cost of 1 for VF 4: exit condition instruction %exitcond = icmp eq i64 %iv.next, 32
+; CHECK:  Cost of 0 for VF 4: EMIT vp<%3> = CANONICAL-INDUCTION ir<0>, vp<%index.next>
+; CHECK:  Cost of 0 for VF 4: ir<%iv> = WIDEN-INDUCTION ir<0>, ir<1>, vp<%0>
+; CHECK:  Cost of 0 for VF 4: vp<%4> = SCALAR-STEPS vp<%3>, ir<1>
+; CHECK:  Cost of 0 for VF 4: CLONE ir<%g.src> = getelementptr inbounds ir<%src>, vp<%4>
+; CHECK:  Cost of 0 for VF 4: vp<%5> = vector-pointer ir<%g.src>
+; CHECK:  Cost of 1 for VF 4: WIDEN ir<%l> = load vp<%5>
+; CHECK:  Cost of 1 for VF 4: WIDEN ir<%iv.4> = add ir<%iv>, ir<4>
+; CHECK:  Cost of 1 for VF 4: WIDEN ir<%c> = icmp ule ir<%l>, ir<128>
+; CHECK:  Cost of 1 for VF 4: EMIT ir<%or> = add ir<%iv.4>, ir<1>
+; CHECK:  Cost of 0 for VF 4: CLONE ir<%g.dst> = getelementptr ir<%dst>, ir<%or>
+; CHECK:  Cost of 0 for VF 4: vp<%6> = vector-pointer ir<%g.dst>
+; CHECK:  Cost of 1 for VF 4: WIDEN store vp<%6>, ir<%iv.4>, ir<%c>
+; CHECK:  Cost of 0 for VF 4: EMIT vp<%index.next> = add nuw vp<%3>, vp<%1>
+; CHECK:  Cost of 0 for VF 4: EMIT branch-on-count vp<%index.next>, vp<%2>
+; CHECK:  Cost of 0 for VF 4: vector loop backedge
+; CHECK:  Cost of 1 for VF 4: induction instruction %iv.next = add nuw nsw i64 %iv, 1
+; CHECK:  Cost of 0 for VF 4: induction instruction %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop.latch ]
+; CHECK:  Cost of 1 for VF 4: exit condition instruction %exitcond = icmp eq i64 %iv.next, 32
+;
+entry:
+  br label %loop.header
+
+loop.header:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop.latch ]
+  %g.src = getelementptr inbounds i64, ptr %src, i64 %iv
+  %l = load i64, ptr %g.src
+  %iv.4 = add nuw nsw i64 %iv, 4
+  %c = icmp ule i64 %l, 128
+  br i1 %c, label %loop.then, label %loop.latch
+
+loop.then:
+  %or = or disjoint i64 %iv.4, 1
+  %g.dst = getelementptr inbounds i64, ptr %dst, i64 %or
+  store i64 %iv.4, ptr %g.dst, align 4
+  br label %loop.latch
+
+loop.latch:
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond = icmp eq i64 %iv.next, 32
+  br i1 %exitcond, label %exit, label %loop.header
+
+exit:
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopVectorize/X86/conversion-cost.ll b/llvm/test/Transforms/LoopVectorize/X86/conversion-cost.ll
index 15bdbea612a70..f50dffb9ddf15 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/conversion-cost.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/conversion-cost.ll
@@ -68,7 +68,7 @@ define i32 @conversion_cost1(i32 %n, ptr nocapture %A, ptr nocapture %B) nounwin
 ; CHECK-NEXT:    [[CMP_N12:%.*]] = icmp eq i64 [[TMP3]], [[N_VEC3]]
 ; CHECK-NEXT:    br i1 [[CMP_N12]], label [[DOT_CRIT_EDGE_LOOPEXIT]], label [[VEC_EPILOG_SCALAR_PH]]
 ; CHECK:       vec.epilog.scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL6:%.*]] = phi i64 [ [[IND_END4]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ 3, [[ITER_CHECK]] ], [ [[IND_END5]], [[VEC_EPILOG_ITER_CHECK]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL6:%.*]] = phi i64 [ [[IND_END4]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[IND_END5]], [[VEC_EPILOG_ITER_CHECK]] ], [ 3, [[ITER_CHECK]] ]
 ; CHECK-NEXT:    br label [[DOTLR_PH:%.*]]
 ; CHECK:       .lr.ph:
 ; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], [[DOTLR_PH]] ], [ [[BC_RESUME_VAL6]], [[VEC_EPILOG_SCALAR_PH]] ]
diff --git a/llvm/test/Transforms/LoopVectorize/X86/cost-model.ll b/llvm/test/Transforms/LoopVectorize/X86/cost-model.ll
index bd28e28ddff95..50539668b6f70 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/cost-model.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/cost-model.ll
@@ -275,8 +275,8 @@ define float @PR27826(ptr nocapture readonly %a, ptr nocapture readonly %b, i32
 ; CHECK-NEXT:    [[CMP_N14:%.*]] = icmp eq i64 [[TMP2]], [[N_VEC8]]
 ; CHECK-NEXT:    br i1 [[CMP_N14]], label [[LOOPEXIT]], label [[VEC_EPILOG_SCALAR_PH]]
 ; CHECK:       vec.epilog.scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[IND_END]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ 0, [[ITER_CHECK]] ], [ [[IND_END9]], [[VEC_EPILOG_ITER_CHECK]] ]
-; CHECK-NEXT:    [[BC_MERGE_RDX15:%.*]] = phi float [ [[TMP157]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ 0.000000e+00, [[ITER_CHECK]] ], [ [[TMP124]], [[VEC_EPILOG_ITER_CHECK]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[IND_END]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[IND_END9]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[ITER_CHECK]] ]
+; CHECK-NEXT:    [[BC_MERGE_RDX15:%.*]] = phi float [ [[TMP157]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP124]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0.000000e+00, [[ITER_CHECK]] ]
 ; CHECK-NEXT:    br label [[FOR:%.*]]
 ; CHECK:       for:
 ; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[VEC_EPILOG_SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR]] ]
@@ -387,8 +387,8 @@ define void @multi_exit(ptr %dst, ptr %src.1, ptr %src.2, i64 %A, i64 %B) #0 {
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    br label [[SCALAR_PH]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[VECTOR_MEMCHECK]] ], [ 0, [[VECTOR_SCEVCHECK]] ], [ 0, [[ENTRY:%.*]] ]
-; CHECK-NEXT:    [[BC_RESUME_VAL8:%.*]] = phi i32 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ 0, [[VECTOR_MEMCHECK]] ], [ 0, [[VECTOR_SCEVCHECK]] ], [ 0, [[ENTRY]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ], [ 0, [[VECTOR_SCEVCHECK]] ], [ 0, [[VECTOR_MEMCHECK]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL8:%.*]] = phi i32 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ], [ 0, [[VECTOR_SCEVCHECK]] ], [ 0, [[VECTOR_MEMCHECK]] ]
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
 ; CHECK-NEXT:    [[IV_1_WIDE:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_1_NEXT_WIDE:%.*]], [[LOOP_LATCH:%.*]] ]
@@ -832,7 +832,7 @@ define void @cost_duplicate_recipe_for_sinking(ptr %A, i64 %N) #2 {
 ; CHECK:       vec.epilog.middle.block:
 ; CHECK-NEXT:    br label [[VEC_EPILOG_SCALAR_PH]]
 ; CHECK:       vec.epilog.scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC39]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ 0, [[ITER_CHECK:%.*]] ], [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC39]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[ITER_CHECK:%.*]] ]
 ; CHECK-NEXT:    br label [[LOOP_HEADER:%.*]]
 ; CHECK:       loop.header:
 ; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[VEC_EPILOG_SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP_LATCH:%.*]] ]
diff --git a/llvm/test/Transforms/LoopVectorize/X86/divs-with-tail-folding.ll b/llvm/test/Transforms/LoopVectorize/X86/divs-with-tail-folding.ll
index f14422e0a6069..c861aa8172b9b 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/divs-with-tail-folding.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/divs-with-tail-folding.ll
@@ -46,7 +46,7 @@ define void @sdiv_feeding_gep(ptr %dst, i32 %x, i64 %M, i64 %conv6, i64 %N) {
 ; CHECK:       [[MIDDLE_BLOCK]]:
 ; CHECK-NEXT:    br i1 true, label %[[EXIT:.*]], label %[[SCALAR_PH]]
 ; CHECK:       [[SCALAR_PH]]:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[VECTOR_SCEVCHECK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_SCEVCHECK]] ]
 ; CHECK-NEXT:    br label %[[LOOP:.*]]
 ; CHECK:       [[LOOP]]:
 ; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
@@ -162,7 +162,7 @@ define void @sdiv_feeding_gep_predicated(ptr %dst, i32 %x, i64 %M, i64 %conv6, i
 ; CHECK:       [[MIDDLE_BLOCK]]:
 ; CHECK-NEXT:    br i1 true, label %[[EXIT:.*]], label %[[SCALAR_PH]]
 ; CHECK:       [[SCALAR_PH]]:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[VECTOR_SCEVCHECK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_SCEVCHECK]] ]
 ; CHECK-NEXT:    br label %[[LOOP:.*]]
 ; CHECK:       [[LOOP]]:
 ; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
diff --git a/llvm/test/Transforms/LoopVectorize/X86/epilog-vectorization-inductions.ll b/llvm/test/Transforms/LoopVectorize/X86/epilog-vectorization-inductions.ll
index c14ddca6c913d..7614be0bd3fce 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/epilog-vectorization-inductions.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/epilog-vectorization-inductions.ll
@@ -92,7 +92,7 @@ define void @test_pr59459(i64 %iv.start, ptr %arr) {
 ; CHECK-NEXT:    [[CMP_N16:%.*]] = icmp eq i64 [[TMP3]], [[N_VEC4]]
 ; CHECK-NEXT:    br i1 [[CMP_N16]], label [[EXIT]], label [[VEC_EPILOG_SCALAR_PH]]
 ; CHECK:       vec.epilog.scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL7:%.*]] = phi i64 [ [[IND_END5]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[IV_START]], [[VECTOR_SCEVCHECK]] ], [ [[IV_START]], [[ITER_CHECK:%.*]] ], [ [[IND_END6]], [[VEC_EPILOG_ITER_CHECK]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL7:%.*]] = phi i64 [ [[IND_END5]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[IND_END6]], [[VEC_EPILOG_ITER_CHECK]] ], [ [[IV_START]], [[VECTOR_SCEVCHECK]] ], [ [[IV_START]], [[ITER_CHECK:%.*]] ]
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
 ; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL7]], [[VEC_EPILOG_SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
@@ -225,8 +225,8 @@ define void @test_induction_step_needs_expansion(ptr noalias %j, ptr %k, i64 %l,
 ; CHECK-NEXT:    [[CMP_N25:%.*]] = icmp eq i64 [[L]], [[N_VEC5]]
 ; CHECK-NEXT:    br i1 [[CMP_N25]], label [[EXIT]], label [[VEC_EPILOG_SCALAR_PH]]
 ; CHECK:       vec.epilog.scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL6:%.*]] = phi i64 [ [[N_VEC5]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ 0, [[ITER_CHECK:%.*]] ], [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ]
-; CHECK-NEXT:    [[BC_RESUME_VAL11:%.*]] = phi i16 [ [[IND_END8]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ 0, [[ITER_CHECK]] ], [ [[IND_END10]], [[VEC_EPILOG_ITER_CHECK]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL6:%.*]] = phi i64 [ [[N_VEC5]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[ITER_CHECK:%.*]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL11:%.*]] = phi i16 [ [[IND_END8]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[IND_END10]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[ITER_CHECK]] ]
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
 ; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL6]], [[VEC_EPILOG_SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
diff --git a/llvm/test/Transforms/LoopVectorize/X86/float-induction-x86.ll b/llvm/test/Transforms/LoopVectorize/X86/float-induction-x86.ll
index fc6059d036cd0..537fda42d3a1e 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/float-induction-x86.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/float-induction-x86.ll
@@ -84,8 +84,8 @@ define void @fp_iv_loop1(ptr noalias nocapture %A, i32 %N) #0 {
 ; AUTO_VEC-NEXT:    [[CMP_N14:%.*]] = icmp eq i64 [[N_VEC3]], [[ZEXT]]
 ; AUTO_VEC-NEXT:    br i1 [[CMP_N14]], label [[FOR_END]], label [[FOR_BODY]]
 ; AUTO_VEC:       for.body:
-; AUTO_VEC-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ], [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[ITER_CHECK]] ], [ [[N_VEC3]], [[VEC_EPILOG_MIDDLE_BLOCK]] ]
-; AUTO_VEC-NEXT:    [[X_06:%.*]] = phi float [ [[CONV1:%.*]], [[FOR_BODY]] ], [ [[IND_END8]], [[VEC_EPILOG_ITER_CHECK]] ], [ 1.000000e+00, [[ITER_CHECK]] ], [ [[IND_END6]], [[VEC_EPILOG_MIDDLE_BLOCK]] ]
+; AUTO_VEC-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ], [ 0, [[ITER_CHECK]] ], [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ [[N_VEC3]], [[VEC_EPILOG_MIDDLE_BLOCK]] ]
+; AUTO_VEC-NEXT:    [[X_06:%.*]] = phi float [ [[CONV1:%.*]], [[FOR_BODY]] ], [ 1.000000e+00, [[ITER_CHECK]] ], [ [[IND_END8]], [[VEC_EPILOG_ITER_CHECK]] ], [ [[IND_END6]], [[VEC_EPILOG_MIDDLE_BLOCK]] ]
 ; AUTO_VEC-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds nuw float, ptr [[A]], i64 [[INDVARS_IV]]
 ; AUTO_VEC-NEXT:    store float [[X_06]], ptr [[ARRAYIDX]], align 4
 ; AUTO_VEC-NEXT:    [[CONV1]] = fadd fast float [[X_06]], 5.000000e-01
@@ -468,8 +468,8 @@ define void @fadd_reassoc_FMF(ptr nocapture %p, i32 %N) {
 ; AUTO_VEC:       for.cond.cleanup:
 ; AUTO_VEC-NEXT:    ret void
 ; AUTO_VEC:       for.body:
-; AUTO_VEC-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ], [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[ITER_CHECK]] ], [ [[N_VEC6]], [[VEC_EPILOG_MIDDLE_BLOCK]] ]
-; AUTO_VEC-NEXT:    [[X_012:%.*]] = phi float [ [[ADD3:%.*]], [[FOR_BODY]] ], [ [[IND_END11]], [[VEC_EPILOG_ITER_CHECK]] ], [ 1.000000e+00, [[ITER_CHECK]] ], [ [[IND_END9]], [[VEC_EPILOG_MIDDLE_BLOCK]] ]
+; AUTO_VEC-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ], [ 0, [[ITER_CHECK]] ], [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ [[N_VEC6]], [[VEC_EPILOG_MIDDLE_BLOCK]] ]
+; AUTO_VEC-NEXT:    [[X_012:%.*]] = phi float [ [[ADD3:%.*]], [[FOR_BODY]] ], [ 1.000000e+00, [[ITER_CHECK]] ], [ [[IND_END11]], [[VEC_EPILOG_ITER_CHECK]] ], [ [[IND_END9]], [[VEC_EPILOG_MIDDLE_BLOCK]] ]
 ; AUTO_VEC-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds nuw float, ptr [[P]], i64 [[INDVARS_IV]]
 ; AUTO_VEC-NEXT:    [[TMP16:%.*]] = load float, ptr [[ARRAYIDX]], align 4
 ; AUTO_VEC-NEXT:    [[ADD:%.*]] = fadd reassoc float [[X_012]], [[TMP16]]
diff --git a/llvm/test/Transforms/LoopVectorize/X86/gather_scatter.ll b/llvm/test/Transforms/LoopVectorize/X86/gather_scatter.ll
index 8c338d6a746c4..175f4f379146b 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/gather_scatter.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/gather_scatter.ll
@@ -705,8 +705,8 @@ define void @test_gather_not_profitable_pr48429(i32 %d, ptr readonly noalias %pt
 ; AVX512-NEXT:    [[CMP_N17:%.*]] = icmp eq i64 [[TMP3]], [[N_VEC10]]
 ; AVX512-NEXT:    br i1 [[CMP_N17]], label [[FOR_END]], label [[VEC_EPILOG_SCALAR_PH]]
 ; AVX512:       vec.epilog.scalar.ph:
-; AVX512-NEXT:    [[BC_RESUME_VAL13:%.*]] = phi ptr [ [[IND_END11]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[PTR]], [[VECTOR_MEMCHECK]] ], [ [[PTR]], [[ITER_CHECK]] ], [ [[IND_END12]], [[VEC_EPILOG_ITER_CHECK]] ]
-; AVX512-NEXT:    [[BC_RESUME_VAL16:%.*]] = phi ptr [ [[IND_END14]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[DEST]], [[VECTOR_MEMCHECK]] ], [ [[DEST]], [[ITER_CHECK]] ], [ [[IND_END15]], [[VEC_EPILOG_ITER_CHECK]] ]
+; AVX512-NEXT:    [[BC_RESUME_VAL13:%.*]] = phi ptr [ [[IND_END11]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[IND_END12]], [[VEC_EPILOG_ITER_CHECK]] ], [ [[PTR]], [[VECTOR_MEMCHECK]] ], [ [[PTR]], [[ITER_CHECK]] ]
+; AVX512-NEXT:    [[BC_RESUME_VAL16:%.*]] = phi ptr [ [[IND_END14]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[IND_END15]], [[VEC_EPILOG_ITER_CHECK]] ], [ [[DEST]], [[VECTOR_MEMCHECK]] ], [ [[DEST]], [[ITER_CHECK]] ]
 ; AVX512-NEXT:    br label [[FOR_BODY:%.*]]
 ; AVX512:       for.body:
 ; AVX512-NEXT:    [[PTR_ADDR_012:%.*]] = phi ptr [ [[BC_RESUME_VAL13]], [[VEC_EPILOG_SCALAR_PH]] ], [ [[INCDEC_PTR:%.*]], [[FOR_BODY]] ]
@@ -801,8 +801,8 @@ define void @test_gather_not_profitable_pr48429(i32 %d, ptr readonly noalias %pt
 ; FVW2-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[TMP3]], [[N_VEC]]
 ; FVW2-NEXT:    br i1 [[CMP_N]], label [[FOR_END]], label [[SCALAR_PH]]
 ; FVW2:       scalar.ph:
-; FVW2-NEXT:    [[BC_RESUME_VAL:%.*]] = phi ptr [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ [[PTR]], [[VECTOR_MEMCHECK]] ], [ [[PTR]], [[FOR_BODY_LR_PH]] ]
-; FVW2-NEXT:    [[BC_RESUME_VAL8:%.*]] = phi ptr [ [[IND_END7]], [[MIDDLE_BLOCK]] ], [ [[DEST]], [[VECTOR_MEMCHECK]] ], [ [[DEST]], [[FOR_BODY_LR_PH]] ]
+; FVW2-NEXT:    [[BC_RESUME_VAL:%.*]] = phi ptr [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ [[PTR]], [[FOR_BODY_LR_PH]] ], [ [[PTR]], [[VECTOR_MEMCHECK]] ]
+; FVW2-NEXT:    [[BC_RESUME_VAL8:%.*]] = phi ptr [ [[IND_END7]], [[MIDDLE_BLOCK]] ], [ [[DEST]], [[FOR_BODY_LR_PH]] ], [ [[DEST]], [[VECTOR_MEMCHECK]] ]
 ; FVW2-NEXT:    br label [[FOR_BODY:%.*]]
 ; FVW2:       for.body:
 ; FVW2-NEXT:    [[PTR_ADDR_012:%.*]] = phi ptr [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INCDEC_PTR:%.*]], [[FOR_BODY]] ]
diff --git a/llvm/test/Transforms/LoopVectorize/X86/illegal-parallel-loop-uniform-write.ll b/llvm/test/Transforms/LoopVectorize/X86/illegal-parallel-loop-uniform-write.ll
index 304105fd9925a..5ba559af077ca 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/illegal-parallel-loop-uniform-write.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/illegal-parallel-loop-uniform-write.ll
@@ -87,7 +87,7 @@ define void @foo(ptr nocapture %a, ptr nocapture %b, i32 %k, i32 %m) #0 {
 ; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[TMP2]], [[N_VEC]]
 ; CHECK-NEXT:    br i1 [[CMP_N]], label [[FOR_END_US]], label [[SCALAR_PH]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[VECTOR_SCEVCHECK]] ], [ 0, [[FOR_BODY3_LR_PH_US]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[FOR_BODY3_LR_PH_US]] ], [ 0, [[VECTOR_SCEVCHECK]] ]
 ; CHECK-NEXT:    br label [[FOR_BODY3_US]]
 ; CHECK:       for.end15.loopexit:
 ; CHECK-NEXT:    br label [[FOR_END15]]
diff --git a/llvm/test/Transforms/LoopVectorize/X86/induction-costs.ll b/llvm/test/Transforms/LoopVectorize/X86/induction-costs.ll
index 68cbfad91c541..b6e9dfc2322ff 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/induction-costs.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/induction-costs.ll
@@ -74,7 +74,7 @@ define i32 @iv_used_widened_and_truncated(ptr %dst, i64 %N) #0 {
 ; CHECK-NEXT:    [[CMP_N20:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC9]]
 ; CHECK-NEXT:    br i1 [[CMP_N20]], label [[EXIT]], label [[VEC_EPILOG_SCALAR_PH]]
 ; CHECK:       vec.epilog.scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL10:%.*]] = phi i64 [ [[N_VEC9]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ 0, [[ITER_CHECK:%.*]] ], [ [[N_VEC]], [[MIDDLE_BLOCK]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL10:%.*]] = phi i64 [ [[N_VEC9]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ITER_CHECK:%.*]] ]
 ; CHECK-NEXT:    br label [[LOOP1:%.*]]
 ; CHECK:       loop:
 ; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL10]], [[VEC_EPILOG_SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP1]] ]
@@ -144,7 +144,7 @@ define void @multiple_truncated_ivs_with_wide_uses(i1 %c, ptr %A, ptr %B) {
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    br i1 false, label [[EXIT:%.*]], label [[SCALAR_PH]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 64, [[MIDDLE_BLOCK]] ], [ 0, [[VECTOR_MEMCHECK]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 64, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ], [ 0, [[VECTOR_MEMCHECK]] ]
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
 ; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
@@ -347,11 +347,11 @@ define void @multiple_pointer_ivs_with_scalar_uses_only(ptr %A, ptr %B) #0 {
 ; CHECK-NEXT:    [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <16 x i32> [[TMP22]], i32 15
 ; CHECK-NEXT:    br i1 false, label [[EXIT:%.*]], label [[SCALAR_PH]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ -12, [[MIDDLE_BLOCK]] ], [ 100, [[VECTOR_MEMCHECK]] ], [ 100, [[ENTRY:%.*]] ]
-; CHECK-NEXT:    [[SCALAR_RECUR_INIT:%.*]] = phi i32 [ [[VECTOR_RECUR_EXTRACT]], [[MIDDLE_BLOCK]] ], [ 2048, [[VECTOR_MEMCHECK]] ], [ 2048, [[ENTRY]] ]
-; CHECK-NEXT:    [[BC_RESUME_VAL2:%.*]] = phi ptr [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ [[A]], [[VECTOR_MEMCHECK]] ], [ [[A]], [[ENTRY]] ]
-; CHECK-NEXT:    [[BC_RESUME_VAL4:%.*]] = phi ptr [ [[IND_END3]], [[MIDDLE_BLOCK]] ], [ [[B]], [[VECTOR_MEMCHECK]] ], [ [[B]], [[ENTRY]] ]
-; CHECK-NEXT:    [[BC_RESUME_VAL6:%.*]] = phi ptr [ [[IND_END5]], [[MIDDLE_BLOCK]] ], [ [[B]], [[VECTOR_MEMCHECK]] ], [ [[B]], [[ENTRY]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ -12, [[MIDDLE_BLOCK]] ], [ 100, [[ENTRY:%.*]] ], [ 100, [[VECTOR_MEMCHECK]] ]
+; CHECK-NEXT:    [[SCALAR_RECUR_INIT:%.*]] = phi i32 [ [[VECTOR_RECUR_EXTRACT]], [[MIDDLE_BLOCK]] ], [ 2048, [[ENTRY]] ], [ 2048, [[VECTOR_MEMCHECK]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL2:%.*]] = phi ptr [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ [[A]], [[ENTRY]] ], [ [[A]], [[VECTOR_MEMCHECK]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL4:%.*]] = phi ptr [ [[IND_END3]], [[MIDDLE_BLOCK]] ], [ [[B]], [[ENTRY]] ], [ [[B]], [[VECTOR_MEMCHECK]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL6:%.*]] = phi ptr [ [[IND_END5]], [[MIDDLE_BLOCK]] ], [ [[B]], [[ENTRY]] ], [ [[B]], [[VECTOR_MEMCHECK]] ]
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
 ; CHECK-NEXT:    [[IV_1:%.*]] = phi i32 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[DEC:%.*]], [[LOOP]] ]
@@ -541,7 +541,7 @@ define i32 @test_scalar_predicated_cost(i64 %x, i64 %y, ptr %A) #0 {
 ; CHECK:       vec.epilog.middle.block:
 ; CHECK-NEXT:    br i1 false, label [[EXIT]], label [[SCALAR_PH]]
 ; CHECK:       vec.epilog.scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 100, [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ 0, [[ITER_CHECK:%.*]] ], [ 96, [[VEC_EPILOG_ITER_CHECK]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 100, [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ 96, [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[ITER_CHECK:%.*]] ]
 ; CHECK-NEXT:    br label [[LOOP_HEADER1:%.*]]
 ; CHECK:       loop.header:
 ; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP_LATCH:%.*]] ]
diff --git a/llvm/test/Transforms/LoopVectorize/X86/interleave-cost.ll b/llvm/test/Transforms/LoopVectorize/X86/interleave-cost.ll
index d18d618c6a447..17809b3caf210 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/interleave-cost.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/interleave-cost.ll
@@ -203,7 +203,7 @@ define void @geps_feeding_interleave_groups_with_reuse(ptr %arg, i64 %arg1, ptr
 ; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]]
 ; CHECK-NEXT:    br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
 ; CHECK:       [[SCALAR_PH]]:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[VECTOR_MEMCHECK]] ], [ 0, %[[VECTOR_SCEVCHECK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_SCEVCHECK]] ], [ 0, %[[VECTOR_MEMCHECK]] ]
 ; CHECK-NEXT:    br label %[[LOOP:.*]]
 ; CHECK:       [[LOOP]]:
 ; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
@@ -430,7 +430,7 @@ define void @geps_feeding_interleave_groups_with_reuse2(ptr %A, ptr %B, i64 %N)
 ; CHECK:       [[MIDDLE_BLOCK]]:
 ; CHECK-NEXT:    br label %[[SCALAR_PH]]
 ; CHECK:       [[SCALAR_PH]]:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[IND_END]], %[[MIDDLE_BLOCK]] ], [ 0, %[[VECTOR_MEMCHECK]] ], [ 0, %[[VECTOR_SCEVCHECK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[IND_END]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_SCEVCHECK]] ], [ 0, %[[VECTOR_MEMCHECK]] ]
 ; CHECK-NEXT:    br label %[[LOOP:.*]]
 ; CHECK:       [[LOOP]]:
 ; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT_7:%.*]], %[[LOOP]] ]
diff --git a/llvm/test/Transforms/LoopVectorize/X86/intrinsiccost.ll b/llvm/test/Transforms/LoopVectorize/X86/intrinsiccost.ll
index f50177e61ef08..8f9fb3dd63b8b 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/intrinsiccost.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/intrinsiccost.ll
@@ -93,9 +93,9 @@ define void @uaddsat(ptr nocapture readonly %pSrc, i16 signext %offset, ptr noca
 ; CHECK-NEXT:    [[CMP_N26:%.*]] = icmp eq i64 [[N_VEC8]], [[TMP0]]
 ; CHECK-NEXT:    br i1 [[CMP_N26]], label [[WHILE_END]], label [[VEC_EPILOG_SCALAR_PH]]
 ; CHECK:       vec.epilog.scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ [[IND_END]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[BLOCKSIZE]], [[ITER_CHECK]] ], [ [[IND_END10]], [[VEC_EPILOG_ITER_CHECK]] ]
-; CHECK-NEXT:    [[BC_RESUME_VAL13:%.*]] = phi ptr [ [[IND_END11]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[PSRC]], [[ITER_CHECK]] ], [ [[IND_END12]], [[VEC_EPILOG_ITER_CHECK]] ]
-; CHECK-NEXT:    [[BC_RESUME_VAL16:%.*]] = phi ptr [ [[IND_END14]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[PDST]], [[ITER_CHECK]] ], [ [[IND_END15]], [[VEC_EPILOG_ITER_CHECK]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ [[IND_END]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[IND_END10]], [[VEC_EPILOG_ITER_CHECK]] ], [ [[BLOCKSIZE]], [[ITER_CHECK]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL13:%.*]] = phi ptr [ [[IND_END11]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[IND_END12]], [[VEC_EPILOG_ITER_CHECK]] ], [ [[PSRC]], [[ITER_CHECK]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL16:%.*]] = phi ptr [ [[IND_END14]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[IND_END15]], [[VEC_EPILOG_ITER_CHECK]] ], [ [[PDST]], [[ITER_CHECK]] ]
 ; CHECK-NEXT:    br label [[WHILE_BODY:%.*]]
 ; CHECK:       while.body:
 ; CHECK-NEXT:    [[BLKCNT_09:%.*]] = phi i32 [ [[DEC:%.*]], [[WHILE_BODY]] ], [ [[BC_RESUME_VAL]], [[VEC_EPILOG_SCALAR_PH]] ]
@@ -212,9 +212,9 @@ define void @fshl(ptr nocapture readonly %pSrc, i8 signext %offset, ptr nocaptur
 ; CHECK-NEXT:    [[CMP_N23:%.*]] = icmp eq i64 [[N_VEC7]], [[TMP0]]
 ; CHECK-NEXT:    br i1 [[CMP_N23]], label [[WHILE_END]], label [[VEC_EPILOG_SCALAR_PH]]
 ; CHECK:       vec.epilog.scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ [[IND_END]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[BLOCKSIZE]], [[ITER_CHECK]] ], [ [[IND_END9]], [[VEC_EPILOG_ITER_CHECK]] ]
-; CHECK-NEXT:    [[BC_RESUME_VAL12:%.*]] = phi ptr [ [[IND_END10]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[PSRC]], [[ITER_CHECK]] ], [ [[IND_END11]], [[VEC_EPILOG_ITER_CHECK]] ]
-; CHECK-NEXT:    [[BC_RESUME_VAL15:%.*]] = phi ptr [ [[IND_END13]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[PDST]], [[ITER_CHECK]] ], [ [[IND_END14]], [[VEC_EPILOG_ITER_CHECK]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ [[IND_END]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[IND_END9]], [[VEC_EPILOG_ITER_CHECK]] ], [ [[BLOCKSIZE]], [[ITER_CHECK]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL12:%.*]] = phi ptr [ [[IND_END10]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[IND_END11]], [[VEC_EPILOG_ITER_CHECK]] ], [ [[PSRC]], [[ITER_CHECK]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL15:%.*]] = phi ptr [ [[IND_END13]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[IND_END14]], [[VEC_EPILOG_ITER_CHECK]] ], [ [[PDST]], [[ITER_CHECK]] ]
 ; CHECK-NEXT:    br label [[WHILE_BODY:%.*]]
 ; CHECK:       while.body:
 ; CHECK-NEXT:    [[BLKCNT_09:%.*]] = phi i32 [ [[DEC:%.*]], [[WHILE_BODY]] ], [ [[BC_RESUME_VAL]], [[VEC_EPILOG_SCALAR_PH]] ]
diff --git a/llvm/test/Transforms/LoopVectorize/X86/invariant-load-gather.ll b/llvm/test/Transforms/LoopVectorize/X86/invariant-load-gather.ll
index e8e0608329d89..07f17ec8c2cfa 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/invariant-load-gather.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/invariant-load-gather.ll
@@ -70,7 +70,7 @@ define i32 @inv_load_conditional(ptr %a, i64 %n, ptr %b, i32 %k) {
 ; CHECK-NEXT:    [[CMP_N16:%.*]] = icmp eq i64 [[SMAX2]], [[N_VEC7]]
 ; CHECK-NEXT:    br i1 [[CMP_N16]], label [[FOR_END]], label [[VEC_EPILOG_SCALAR_PH]]
 ; CHECK:       vec.epilog.scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC7]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ 0, [[VECTOR_MEMCHECK]] ], [ 0, [[ITER_CHECK:%.*]] ], [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC7]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[VECTOR_MEMCHECK]] ], [ 0, [[ITER_CHECK:%.*]] ]
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK:       for.body:
 ; CHECK-NEXT:    [[I:%.*]] = phi i64 [ [[I_NEXT:%.*]], [[LATCH:%.*]] ], [ [[BC_RESUME_VAL]], [[VEC_EPILOG_SCALAR_PH]] ]
diff --git a/llvm/test/Transforms/LoopVectorize/X86/invariant-store-vectorization.ll b/llvm/test/Transforms/LoopVectorize/X86/invariant-store-vectorization.ll
index adfffccb6bcac..64d3d4121d660 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/invariant-store-vectorization.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/invariant-store-vectorization.ll
@@ -82,8 +82,8 @@ define i32 @inv_val_store_to_inv_address_with_reduction(ptr %a, i64 %n, ptr %b)
 ; CHECK-NEXT:    [[CMP_N18:%.*]] = icmp eq i64 [[SMAX2]], [[N_VEC13]]
 ; CHECK-NEXT:    br i1 [[CMP_N18]], label [[FOR_END]], label [[VEC_EPILOG_SCALAR_PH]]
 ; CHECK:       vec.epilog.scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC13]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ 0, [[VECTOR_MEMCHECK]] ], [ 0, [[ITER_CHECK:%.*]] ], [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ]
-; CHECK-NEXT:    [[BC_MERGE_RDX19:%.*]] = phi i32 [ [[TMP15]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ 0, [[VECTOR_MEMCHECK]] ], [ 0, [[ITER_CHECK]] ], [ [[TMP10]], [[VEC_EPILOG_ITER_CHECK]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC13]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[VECTOR_MEMCHECK]] ], [ 0, [[ITER_CHECK:%.*]] ]
+; CHECK-NEXT:    [[BC_MERGE_RDX19:%.*]] = phi i32 [ [[TMP15]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP10]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[VECTOR_MEMCHECK]] ], [ 0, [[ITER_CHECK]] ]
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK:       for.body:
 ; CHECK-NEXT:    [[I:%.*]] = phi i64 [ [[I_NEXT:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL]], [[VEC_EPILOG_SCALAR_PH]] ]
@@ -210,7 +210,7 @@ define void @inv_val_store_to_inv_address_conditional(ptr %a, i64 %n, ptr %b, i3
 ; CHECK-NEXT:    [[CMP_N23:%.*]] = icmp eq i64 [[SMAX2]], [[N_VEC9]]
 ; CHECK-NEXT:    br i1 [[CMP_N23]], label [[FOR_END]], label [[VEC_EPILOG_SCALAR_PH]]
 ; CHECK:       vec.epilog.scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC9]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ 0, [[VECTOR_MEMCHECK]] ], [ 0, [[ITER_CHECK:%.*]] ], [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC9]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[VECTOR_MEMCHECK]] ], [ 0, [[ITER_CHECK:%.*]] ]
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK:       for.body:
 ; CHECK-NEXT:    [[I:%.*]] = phi i64 [ [[I_NEXT:%.*]], [[LATCH:%.*]] ], [ [[BC_RESUME_VAL]], [[VEC_EPILOG_SCALAR_PH]] ]
@@ -335,7 +335,7 @@ define void @variant_val_store_to_inv_address_conditional(ptr %a, i64 %n, ptr %b
 ; CHECK-NEXT:    [[CMP_N28:%.*]] = icmp eq i64 [[SMAX10]], [[N_VEC17]]
 ; CHECK-NEXT:    br i1 [[CMP_N28]], label [[FOR_END]], label [[VEC_EPILOG_SCALAR_PH]]
 ; CHECK:       vec.epilog.scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC17]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ 0, [[VECTOR_MEMCHECK]] ], [ 0, [[ITER_CHECK:%.*]] ], [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC17]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[VECTOR_MEMCHECK]] ], [ 0, [[ITER_CHECK:%.*]] ]
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK:       for.body:
 ; CHECK-NEXT:    [[I:%.*]] = phi i64 [ [[I_NEXT:%.*]], [[LATCH:%.*]] ], [ [[BC_RESUME_VAL]], [[VEC_EPILOG_SCALAR_PH]] ]
diff --git a/llvm/test/Transforms/LoopVectorize/X86/limit-vf-by-tripcount.ll b/llvm/test/Transforms/LoopVectorize/X86/limit-vf-by-tripcount.ll
index 50414cc29312c..8688b246c60f4 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/limit-vf-by-tripcount.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/limit-vf-by-tripcount.ll
@@ -99,7 +99,7 @@ define void @test_tc_18(ptr noalias %src, ptr noalias %dst) {
 ; CHECK:       vec.epilog.middle.block:
 ; CHECK-NEXT:    br i1 true, label [[EXIT]], label [[VEC_EPILOG_SCALAR_PH]]
 ; CHECK:       vec.epilog.scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 18, [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ 0, [[ITER_CHECK:%.*]] ], [ 16, [[VEC_EPILOG_ITER_CHECK]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 18, [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ 16, [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[ITER_CHECK:%.*]] ]
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
 ; CHECK-NEXT:    [[I:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[VEC_EPILOG_SCALAR_PH]] ], [ [[I_NEXT:%.*]], [[LOOP]] ]
@@ -172,7 +172,7 @@ define void @test_tc_19(ptr noalias %src, ptr noalias %dst) {
 ; CHECK:       vec.epilog.middle.block:
 ; CHECK-NEXT:    br i1 false, label [[EXIT]], label [[VEC_EPILOG_SCALAR_PH]]
 ; CHECK:       vec.epilog.scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 18, [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ 0, [[ITER_CHECK:%.*]] ], [ 16, [[VEC_EPILOG_ITER_CHECK]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 18, [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ 16, [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[ITER_CHECK:%.*]] ]
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
 ; CHECK-NEXT:    [[I:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[VEC_EPILOG_SCALAR_PH]] ], [ [[I_NEXT:%.*]], [[LOOP]] ]
@@ -257,7 +257,7 @@ define void @test_tc_20(ptr noalias %src, ptr noalias %dst) {
 ; CHECK:       vec.epilog.middle.block:
 ; CHECK-NEXT:    br i1 true, label [[EXIT]], label [[VEC_EPILOG_SCALAR_PH]]
 ; CHECK:       vec.epilog.scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 20, [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ 0, [[ITER_CHECK:%.*]] ], [ 16, [[VEC_EPILOG_ITER_CHECK]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 20, [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ 16, [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[ITER_CHECK:%.*]] ]
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
 ; CHECK-NEXT:    [[I:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[VEC_EPILOG_SCALAR_PH]] ], [ [[I_NEXT:%.*]], [[LOOP]] ]
diff --git a/llvm/test/Transforms/LoopVectorize/X86/load-deref-pred.ll b/llvm/test/Transforms/LoopVectorize/X86/load-deref-pred.ll
index 1433e48690bc6..b40037734d22e 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/load-deref-pred.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/load-deref-pred.ll
@@ -524,8 +524,7 @@ define i32 @test_step_narrower_than_access(i64 %len, ptr %test_base) {
 ; CHECK-NEXT:    [[TMP61:%.*]] = insertelement <4 x i1> [[TMP60]], i1 [[TMP57]], i32 1
 ; CHECK-NEXT:    [[TMP62:%.*]] = insertelement <4 x i1> [[TMP61]], i1 [[TMP58]], i32 2
 ; CHECK-NEXT:    [[TMP63:%.*]] = insertelement <4 x i1> [[TMP62]], i1 [[TMP59]], i32 3
-; CHECK-NEXT:    [[TMP64:%.*]] = extractelement <4 x i1> [[TMP39]], i32 0
-; CHECK-NEXT:    br i1 [[TMP64]], label [[PRED_LOAD_IF:%.*]], label [[PRED_LOAD_CONTINUE:%.*]]
+; CHECK-NEXT:    br i1 [[TMP32]], label [[PRED_LOAD_IF:%.*]], label [[PRED_LOAD_CONTINUE:%.*]]
 ; CHECK:       pred.load.if:
 ; CHECK-NEXT:    [[TMP65:%.*]] = getelementptr inbounds i16, ptr [[ALLOCA]], i64 [[TMP0]]
 ; CHECK-NEXT:    [[TMP66:%.*]] = load i32, ptr [[TMP65]], align 4
@@ -533,8 +532,7 @@ define i32 @test_step_narrower_than_access(i64 %len, ptr %test_base) {
 ; CHECK-NEXT:    br label [[PRED_LOAD_CONTINUE]]
 ; CHECK:       pred.load.continue:
 ; CHECK-NEXT:    [[TMP68:%.*]] = phi <4 x i32> [ poison, [[VECTOR_BODY]] ], [ [[TMP67]], [[PRED_LOAD_IF]] ]
-; CHECK-NEXT:    [[TMP69:%.*]] = extractelement <4 x i1> [[TMP39]], i32 1
-; CHECK-NEXT:    br i1 [[TMP69]], label [[PRED_LOAD_IF4:%.*]], label [[PRED_LOAD_CONTINUE5:%.*]]
+; CHECK-NEXT:    br i1 [[TMP33]], label [[PRED_LOAD_IF4:%.*]], label [[PRED_LOAD_CONTINUE5:%.*]]
 ; CHECK:       pred.load.if4:
 ; CHECK-NEXT:    [[TMP70:%.*]] = getelementptr inbounds i16, ptr [[ALLOCA]], i64 [[TMP1]]
 ; CHECK-NEXT:    [[TMP71:%.*]] = load i32, ptr [[TMP70]], align 4
@@ -542,8 +540,7 @@ define i32 @test_step_narrower_than_access(i64 %len, ptr %test_base) {
 ; CHECK-NEXT:    br label [[PRED_LOAD_CONTINUE5]]
 ; CHECK:       pred.load.continue5:
 ; CHECK-NEXT:    [[TMP73:%.*]] = phi <4 x i32> [ [[TMP68]], [[PRED_LOAD_CONTINUE]] ], [ [[TMP72]], [[PRED_LOAD_IF4]] ]
-; CHECK-NEXT:    [[TMP74:%.*]] = extractelement <4 x i1> [[TMP39]], i32 2
-; CHECK-NEXT:    br i1 [[TMP74]], label [[PRED_LOAD_IF6:%.*]], label [[PRED_LOAD_CONTINUE7:%.*]]
+; CHECK-NEXT:    br i1 [[TMP34]], label [[PRED_LOAD_IF6:%.*]], label [[PRED_LOAD_CONTINUE7:%.*]]
 ; CHECK:       pred.load.if6:
 ; CHECK-NEXT:    [[TMP75:%.*]] = getelementptr inbounds i16, ptr [[ALLOCA]], i64 [[TMP2]]
 ; CHECK-NEXT:    [[TMP76:%.*]] = load i32, ptr [[TMP75]], align 4
@@ -551,8 +548,7 @@ define i32 @test_step_narrower_than_access(i64 %len, ptr %test_base) {
 ; CHECK-NEXT:    br label [[PRED_LOAD_CONTINUE7]]
 ; CHECK:       pred.load.continue7:
 ; CHECK-NEXT:    [[TMP78:%.*]] = phi <4 x i32> [ [[TMP73]], [[PRED_LOAD_CONTINUE5]] ], [ [[TMP77]], [[PRED_LOAD_IF6]] ]
-; CHECK-NEXT:    [[TMP79:%.*]] = extractelement <4 x i1> [[TMP39]], i32 3
-; CHECK-NEXT:    br i1 [[TMP79]], label [[PRED_LOAD_IF8:%.*]], label [[PRED_LOAD_CONTINUE9:%.*]]
+; CHECK-NEXT:    br i1 [[TMP35]], label [[PRED_LOAD_IF8:%.*]], label [[PRED_LOAD_CONTINUE9:%.*]]
 ; CHECK:       pred.load.if8:
 ; CHECK-NEXT:    [[TMP80:%.*]] = getelementptr inbounds i16, ptr [[ALLOCA]], i64 [[TMP3]]
 ; CHECK-NEXT:    [[TMP81:%.*]] = load i32, ptr [[TMP80]], align 4
@@ -560,8 +556,7 @@ define i32 @test_step_narrower_than_access(i64 %len, ptr %test_base) {
 ; CHECK-NEXT:    br label [[PRED_LOAD_CONTINUE9]]
 ; CHECK:       pred.load.continue9:
 ; CHECK-NEXT:    [[TMP83:%.*]] = phi <4 x i32> [ [[TMP78]], [[PRED_LOAD_CONTINUE7]] ], [ [[TMP82]], [[PRED_LOAD_IF8]] ]
-; CHECK-NEXT:    [[TMP84:%.*]] = extractelement <4 x i1> [[TMP47]], i32 0
-; CHECK-NEXT:    br i1 [[TMP84]], label [[PRED_LOAD_IF10:%.*]], label [[PRED_LOAD_CONTINUE11:%.*]]
+; CHECK-NEXT:    br i1 [[TMP40]], label [[PRED_LOAD_IF10:%.*]], label [[PRED_LOAD_CONTINUE11:%.*]]
 ; CHECK:       pred.load.if10:
 ; CHECK-NEXT:    [[TMP85:%.*]] = getelementptr inbounds i16, ptr [[ALLOCA]], i64 [[TMP4]]
 ; CHECK-NEXT:    [[TMP86:%.*]] = load i32, ptr [[TMP85]], align 4
@@ -569,8 +564,7 @@ define i32 @test_step_narrower_than_access(i64 %len, ptr %test_base) {
 ; CHECK-NEXT:    br label [[PRED_LOAD_CONTINUE11]]
 ; CHECK:       pred.load.continue11:
 ; CHECK-NEXT:    [[TMP88:%.*]] = phi <4 x i32> [ poison, [[PRED_LOAD_CONTINUE9]] ], [ [[TMP87]], [[PRED_LOAD_IF10]] ]
-; CHECK-NEXT:    [[TMP89:%.*]] = extractelement <4 x i1> [[TMP47]], i32 1
-; CHECK-NEXT:    br i1 [[TMP89]], label [[PRED_LOAD_IF12:%.*]], label [[PRED_LOAD_CONTINUE13:%.*]]
+; CHECK-NEXT:    br i1 [[TMP41]], label [[PRED_LOAD_IF12:%.*]], label [[PRED_LOAD_CONTINUE13:%.*]]
 ; CHECK:       pred.load.if12:
 ; CHECK-NEXT:    [[TMP90:%.*]] = getelementptr inbounds i16, ptr [[ALLOCA]], i64 [[TMP5]]
 ; CHECK-NEXT:    [[TMP91:%.*]] = load i32, ptr [[TMP90]], align 4
@@ -578,8 +572,7 @@ define i32 @test_step_narrower_than_access(i64 %len, ptr %test_base) {
 ; CHECK-NEXT:    br label [[PRED_LOAD_CONTINUE13]]
 ; CHECK:       pred.load.continue13:
 ; CHECK-NEXT:    [[TMP93:%.*]] = phi <4 x i32> [ [[TMP88]], [[PRED_LOAD_CONTINUE11]] ], [ [[TMP92]], [[PRED_LOAD_IF12]] ]
-; CHECK-NEXT:    [[TMP94:%.*]] = extractelement <4 x i1> [[TMP47]], i32 2
-; CHECK-NEXT:    br i1 [[TMP94]], label [[PRED_LOAD_IF14:%.*]], label [[PRED_LOAD_CONTINUE15:%.*]]
+; CHECK-NEXT:    br i1 [[TMP42]], label [[PRED_LOAD_IF14:%.*]], label [[PRED_LOAD_CONTINUE15:%.*]]
 ; CHECK:       pred.load.if14:
 ; CHECK-NEXT:    [[TMP95:%.*]] = getelementptr inbounds i16, ptr [[ALLOCA]], i64 [[TMP6]]
 ; CHECK-NEXT:    [[TMP96:%.*]] = load i32, ptr [[TMP95]], align 4
@@ -587,8 +580,7 @@ define i32 @test_step_narrower_than_access(i64 %len, ptr %test_base) {
 ; CHECK-NEXT:    br label [[PRED_LOAD_CONTINUE15]]
 ; CHECK:       pred.load.continue15:
 ; CHECK-NEXT:    [[TMP98:%.*]] = phi <4 x i32> [ [[TMP93]], [[PRED_LOAD_CONTINUE13]] ], [ [[TMP97]], [[PRED_LOAD_IF14]] ]
-; CHECK-NEXT:    [[TMP99:%.*]] = extractelement <4 x i1> [[TMP47]], i32 3
-; CHECK-NEXT:    br i1 [[TMP99]], label [[PRED_LOAD_IF16:%.*]], label [[PRED_LOAD_CONTINUE17:%.*]]
+; CHECK-NEXT:    br i1 [[TMP43]], label [[PRED_LOAD_IF16:%.*]], label [[PRED_LOAD_CONTINUE17:%.*]]
 ; CHECK:       pred.load.if16:
 ; CHECK-NEXT:    [[TMP100:%.*]] = getelementptr inbounds i16, ptr [[ALLOCA]], i64 [[TMP7]]
 ; CHECK-NEXT:    [[TMP101:%.*]] = load i32, ptr [[TMP100]], align 4
@@ -596,8 +588,7 @@ define i32 @test_step_narrower_than_access(i64 %len, ptr %test_base) {
 ; CHECK-NEXT:    br label [[PRED_LOAD_CONTINUE17]]
 ; CHECK:       pred.load.continue17:
 ; CHECK-NEXT:    [[TMP103:%.*]] = phi <4 x i32> [ [[TMP98]], [[PRED_LOAD_CONTINUE15]] ], [ [[TMP102]], [[PRED_LOAD_IF16]] ]
-; CHECK-NEXT:    [[TMP104:%.*]] = extractelement <4 x i1> [[TMP55]], i32 0
-; CHECK-NEXT:    br i1 [[TMP104]], label [[PRED_LOAD_IF18:%.*]], label [[PRED_LOAD_CONTINUE19:%.*]]
+; CHECK-NEXT:    br i1 [[TMP48]], label [[PRED_LOAD_IF18:%.*]], label [[PRED_LOAD_CONTINUE19:%.*]]
 ; CHECK:       pred.load.if18:
 ; CHECK-NEXT:    [[TMP105:%.*]] = getelementptr inbounds i16, ptr [[ALLOCA]], i64 [[TMP8]]
 ; CHECK-NEXT:    [[TMP106:%.*]] = load i32, ptr [[TMP105]], align 4
@@ -605,8 +596,7 @@ define i32 @test_step_narrower_than_access(i64 %len, ptr %test_base) {
 ; CHECK-NEXT:    br label [[PRED_LOAD_CONTINUE19]]
 ; CHECK:       pred.load.continue19:
 ; CHECK-NEXT:    [[TMP108:%.*]] = phi <4 x i32> [ poison, [[PRED_LOAD_CONTINUE17]] ], [ [[TMP107]], [[PRED_LOAD_IF18]] ]
-; CHECK-NEXT:    [[TMP109:%.*]] = extractelement <4 x i1> [[TMP55]], i32 1
-; CHECK-NEXT:    br i1 [[TMP109]], label [[PRED_LOAD_IF20:%.*]], label [[PRED_LOAD_CONTINUE21:%.*]]
+; CHECK-NEXT:    br i1 [[TMP49]], label [[PRED_LOAD_IF20:%.*]], label [[PRED_LOAD_CONTINUE21:%.*]]
 ; CHECK:       pred.load.if20:
 ; CHECK-NEXT:    [[TMP110:%.*]] = getelementptr inbounds i16, ptr [[ALLOCA]], i64 [[TMP9]]
 ; CHECK-NEXT:    [[TMP111:%.*]] = load i32, ptr [[TMP110]], align 4
@@ -614,8 +604,7 @@ define i32 @test_step_narrower_than_access(i64 %len, ptr %test_base) {
 ; CHECK-NEXT:    br label [[PRED_LOAD_CONTINUE21]]
 ; CHECK:       pred.load.continue21:
 ; CHECK-NEXT:    [[TMP113:%.*]] = phi <4 x i32> [ [[TMP108]], [[PRED_LOAD_CONTINUE19]] ], [ [[TMP112]], [[PRED_LOAD_IF20]] ]
-; CHECK-NEXT:    [[TMP114:%.*]] = extractelement <4 x i1> [[TMP55]], i32 2
-; CHECK-NEXT:    br i1 [[TMP114]], label [[PRED_LOAD_IF22:%.*]], label [[PRED_LOAD_CONTINUE23:%.*]]
+; CHECK-NEXT:    br i1 [[TMP50]], label [[PRED_LOAD_IF22:%.*]], label [[PRED_LOAD_CONTINUE23:%.*]]
 ; CHECK:       pred.load.if22:
 ; CHECK-NEXT:    [[TMP115:%.*]] = getelementptr inbounds i16, ptr [[ALLOCA]], i64 [[TMP10]]
 ; CHECK-NEXT:    [[TMP116:%.*]] = load i32, ptr [[TMP115]], align 4
@@ -623,8 +612,7 @@ define i32 @test_step_narrower_than_access(i64 %len, ptr %test_base) {
 ; CHECK-NEXT:    br label [[PRED_LOAD_CONTINUE23]]
 ; CHECK:       pred.load.continue23:
 ; CHECK-NEXT:    [[TMP118:%.*]] = phi <4 x i32> [ [[TMP113]], [[PRED_LOAD_CONTINUE21]] ], [ [[TMP117]], [[PRED_LOAD_IF22]] ]
-; CHECK-NEXT:    [[TMP119:%.*]] = extractelement <4 x i1> [[TMP55]], i32 3
-; CHECK-NEXT:    br i1 [[TMP119]], label [[PRED_LOAD_IF24:%.*]], label [[PRED_LOAD_CONTINUE25:%.*]]
+; CHECK-NEXT:    br i1 [[TMP51]], label [[PRED_LOAD_IF24:%.*]], label [[PRED_LOAD_CONTINUE25:%.*]]
 ; CHECK:       pred.load.if24:
 ; CHECK-NEXT:    [[TMP120:%.*]] = getelementptr inbounds i16, ptr [[ALLOCA]], i64 [[TMP11]]
 ; CHECK-NEXT:    [[TMP121:%.*]] = load i32, ptr [[TMP120]], align 4
@@ -632,8 +620,7 @@ define i32 @test_step_narrower_than_access(i64 %len, ptr %test_base) {
 ; CHECK-NEXT:    br label [[PRED_LOAD_CONTINUE25]]
 ; CHECK:       pred.load.continue25:
 ; CHECK-NEXT:    [[TMP123:%.*]] = phi <4 x i32> [ [[TMP118]], [[PRED_LOAD_CONTINUE23]] ], [ [[TMP122]], [[PRED_LOAD_IF24]] ]
-; CHECK-NEXT:    [[TMP124:%.*]] = extractelement <4 x i1> [[TMP63]], i32 0
-; CHECK-NEXT:    br i1 [[TMP124]], label [[PRED_LOAD_IF26:%.*]], label [[PRED_LOAD_CONTINUE27:%.*]]
+; CHECK-NEXT:    br i1 [[TMP56]], label [[PRED_LOAD_IF26:%.*]], label [[PRED_LOAD_CONTINUE27:%.*]]
 ; CHECK:       pred.load.if26:
 ; CHECK-NEXT:    [[TMP125:%.*]] = getelementptr inbounds i16, ptr [[ALLOCA]], i64 [[TMP12]]
 ; CHECK-NEXT:    [[TMP126:%.*]] = load i32, ptr [[TMP125]], align 4
@@ -641,8 +628,7 @@ define i32 @test_step_narrower_than_access(i64 %len, ptr %test_base) {
 ; CHECK-NEXT:    br label [[PRED_LOAD_CONTINUE27]]
 ; CHECK:       pred.load.continue27:
 ; CHECK-NEXT:    [[TMP128:%.*]] = phi <4 x i32> [ poison, [[PRED_LOAD_CONTINUE25]] ], [ [[TMP127]], [[PRED_LOAD_IF26]] ]
-; CHECK-NEXT:    [[TMP129:%.*]] = extractelement <4 x i1> [[TMP63]], i32 1
-; CHECK-NEXT:    br i1 [[TMP129]], label [[PRED_LOAD_IF28:%.*]], label [[PRED_LOAD_CONTINUE29:%.*]]
+; CHECK-NEXT:    br i1 [[TMP57]], label [[PRED_LOAD_IF28:%.*]], label [[PRED_LOAD_CONTINUE29:%.*]]
 ; CHECK:       pred.load.if28:
 ; CHECK-NEXT:    [[TMP130:%.*]] = getelementptr inbounds i16, ptr [[ALLOCA]], i64 [[TMP13]]
 ; CHECK-NEXT:    [[TMP131:%.*]] = load i32, ptr [[TMP130]], align 4
@@ -650,8 +636,7 @@ define i32 @test_step_narrower_than_access(i64 %len, ptr %test_base) {
 ; CHECK-NEXT:    br label [[PRED_LOAD_CONTINUE29]]
 ; CHECK:       pred.load.continue29:
 ; CHECK-NEXT:    [[TMP133:%.*]] = phi <4 x i32> [ [[TMP128]], [[PRED_LOAD_CONTINUE27]] ], [ [[TMP132]], [[PRED_LOAD_IF28]] ]
-; CHECK-NEXT:    [[TMP134:%.*]] = extractelement <4 x i1> [[TMP63]], i32 2
-; CHECK-NEXT:    br i1 [[TMP134]], label [[PRED_LOAD_IF30:%.*]], label [[PRED_LOAD_CONTINUE31:%.*]]
+; CHECK-NEXT:    br i1 [[TMP58]], label [[PRED_LOAD_IF30:%.*]], label [[PRED_LOAD_CONTINUE31:%.*]]
 ; CHECK:       pred.load.if30:
 ; CHECK-NEXT:    [[TMP135:%.*]] = getelementptr inbounds i16, ptr [[ALLOCA]], i64 [[TMP14]]
 ; CHECK-NEXT:    [[TMP136:%.*]] = load i32, ptr [[TMP135]], align 4
@@ -659,8 +644,7 @@ define i32 @test_step_narrower_than_access(i64 %len, ptr %test_base) {
 ; CHECK-NEXT:    br label [[PRED_LOAD_CONTINUE31]]
 ; CHECK:       pred.load.continue31:
 ; CHECK-NEXT:    [[TMP138:%.*]] = phi <4 x i32> [ [[TMP133]], [[PRED_LOAD_CONTINUE29]] ], [ [[TMP137]], [[PRED_LOAD_IF30]] ]
-; CHECK-NEXT:    [[TMP139:%.*]] = extractelement <4 x i1> [[TMP63]], i32 3
-; CHECK-NEXT:    br i1 [[TMP139]], label [[PRED_LOAD_IF32:%.*]], label [[PRED_LOAD_CONTINUE33]]
+; CHECK-NEXT:    br i1 [[TMP59]], label [[PRED_LOAD_IF32:%.*]], label [[PRED_LOAD_CONTINUE33]]
 ; CHECK:       pred.load.if32:
 ; CHECK-NEXT:    [[TMP140:%.*]] = getelementptr inbounds i16, ptr [[ALLOCA]], i64 [[TMP15]]
 ; CHECK-NEXT:    [[TMP141:%.*]] = load i32, ptr [[TMP140]], align 4
@@ -2920,8 +2904,8 @@ loop_exit:
   ret i32 %accum.next
 }
 
-define i32 @neg_test_non_unit_stride_off_by_four_bytes(i64 %len, ptr %test_base) {
-; CHECK-LABEL: @neg_test_non_unit_stride_off_by_four_bytes(
+define i32 @test_non_unit_stride_off_by_four_bytes(i64 %len, ptr %test_base) {
+; CHECK-LABEL: @test_non_unit_stride_off_by_four_bytes(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[ALLOCA:%.*]] = alloca [103 x i32], align 4
 ; CHECK-NEXT:    call void @init(ptr [[ALLOCA]])
@@ -2929,11 +2913,11 @@ define i32 @neg_test_non_unit_stride_off_by_four_bytes(i64 %len, ptr %test_base)
 ; CHECK:       vector.ph:
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
-; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_LOAD_CONTINUE33:%.*]] ]
-; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP144:%.*]], [[PRED_LOAD_CONTINUE33]] ]
-; CHECK-NEXT:    [[VEC_PHI1:%.*]] = phi <4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP145:%.*]], [[PRED_LOAD_CONTINUE33]] ]
-; CHECK-NEXT:    [[VEC_PHI2:%.*]] = phi <4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP146:%.*]], [[PRED_LOAD_CONTINUE33]] ]
-; CHECK-NEXT:    [[VEC_PHI3:%.*]] = phi <4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP147:%.*]], [[PRED_LOAD_CONTINUE33]] ]
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP112:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_PHI1:%.*]] = phi <4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP113:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_PHI2:%.*]] = phi <4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP114:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_PHI3:%.*]] = phi <4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP115:%.*]], [[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[OFFSET_IDX:%.*]] = mul i64 [[INDEX]], 2
 ; CHECK-NEXT:    [[TMP0:%.*]] = add i64 [[OFFSET_IDX]], 0
 ; CHECK-NEXT:    [[TMP1:%.*]] = add i64 [[OFFSET_IDX]], 2
@@ -2999,170 +2983,74 @@ define i32 @neg_test_non_unit_stride_off_by_four_bytes(i64 %len, ptr %test_base)
 ; CHECK-NEXT:    [[TMP61:%.*]] = insertelement <4 x i1> [[TMP60]], i1 [[TMP57]], i32 1
 ; CHECK-NEXT:    [[TMP62:%.*]] = insertelement <4 x i1> [[TMP61]], i1 [[TMP58]], i32 2
 ; CHECK-NEXT:    [[TMP63:%.*]] = insertelement <4 x i1> [[TMP62]], i1 [[TMP59]], i32 3
-; CHECK-NEXT:    [[TMP64:%.*]] = extractelement <4 x i1> [[TMP39]], i32 0
-; CHECK-NEXT:    br i1 [[TMP64]], label [[PRED_LOAD_IF:%.*]], label [[PRED_LOAD_CONTINUE:%.*]]
-; CHECK:       pred.load.if:
-; CHECK-NEXT:    [[TMP65:%.*]] = getelementptr inbounds i32, ptr [[ALLOCA]], i64 [[TMP0]]
-; CHECK-NEXT:    [[TMP66:%.*]] = load i32, ptr [[TMP65]], align 4
-; CHECK-NEXT:    [[TMP67:%.*]] = insertelement <4 x i32> poison, i32 [[TMP66]], i32 0
-; CHECK-NEXT:    br label [[PRED_LOAD_CONTINUE]]
-; CHECK:       pred.load.continue:
-; CHECK-NEXT:    [[TMP68:%.*]] = phi <4 x i32> [ poison, [[VECTOR_BODY]] ], [ [[TMP67]], [[PRED_LOAD_IF]] ]
-; CHECK-NEXT:    [[TMP69:%.*]] = extractelement <4 x i1> [[TMP39]], i32 1
-; CHECK-NEXT:    br i1 [[TMP69]], label [[PRED_LOAD_IF4:%.*]], label [[PRED_LOAD_CONTINUE5:%.*]]
-; CHECK:       pred.load.if4:
-; CHECK-NEXT:    [[TMP70:%.*]] = getelementptr inbounds i32, ptr [[ALLOCA]], i64 [[TMP1]]
-; CHECK-NEXT:    [[TMP71:%.*]] = load i32, ptr [[TMP70]], align 4
-; CHECK-NEXT:    [[TMP72:%.*]] = insertelement <4 x i32> [[TMP68]], i32 [[TMP71]], i32 1
-; CHECK-NEXT:    br label [[PRED_LOAD_CONTINUE5]]
-; CHECK:       pred.load.continue5:
-; CHECK-NEXT:    [[TMP73:%.*]] = phi <4 x i32> [ [[TMP68]], [[PRED_LOAD_CONTINUE]] ], [ [[TMP72]], [[PRED_LOAD_IF4]] ]
-; CHECK-NEXT:    [[TMP74:%.*]] = extractelement <4 x i1> [[TMP39]], i32 2
-; CHECK-NEXT:    br i1 [[TMP74]], label [[PRED_LOAD_IF6:%.*]], label [[PRED_LOAD_CONTINUE7:%.*]]
-; CHECK:       pred.load.if6:
-; CHECK-NEXT:    [[TMP75:%.*]] = getelementptr inbounds i32, ptr [[ALLOCA]], i64 [[TMP2]]
-; CHECK-NEXT:    [[TMP76:%.*]] = load i32, ptr [[TMP75]], align 4
-; CHECK-NEXT:    [[TMP77:%.*]] = insertelement <4 x i32> [[TMP73]], i32 [[TMP76]], i32 2
-; CHECK-NEXT:    br label [[PRED_LOAD_CONTINUE7]]
-; CHECK:       pred.load.continue7:
-; CHECK-NEXT:    [[TMP78:%.*]] = phi <4 x i32> [ [[TMP73]], [[PRED_LOAD_CONTINUE5]] ], [ [[TMP77]], [[PRED_LOAD_IF6]] ]
-; CHECK-NEXT:    [[TMP79:%.*]] = extractelement <4 x i1> [[TMP39]], i32 3
-; CHECK-NEXT:    br i1 [[TMP79]], label [[PRED_LOAD_IF8:%.*]], label [[PRED_LOAD_CONTINUE9:%.*]]
-; CHECK:       pred.load.if8:
-; CHECK-NEXT:    [[TMP80:%.*]] = getelementptr inbounds i32, ptr [[ALLOCA]], i64 [[TMP3]]
-; CHECK-NEXT:    [[TMP81:%.*]] = load i32, ptr [[TMP80]], align 4
-; CHECK-NEXT:    [[TMP82:%.*]] = insertelement <4 x i32> [[TMP78]], i32 [[TMP81]], i32 3
-; CHECK-NEXT:    br label [[PRED_LOAD_CONTINUE9]]
-; CHECK:       pred.load.continue9:
-; CHECK-NEXT:    [[TMP83:%.*]] = phi <4 x i32> [ [[TMP78]], [[PRED_LOAD_CONTINUE7]] ], [ [[TMP82]], [[PRED_LOAD_IF8]] ]
-; CHECK-NEXT:    [[TMP84:%.*]] = extractelement <4 x i1> [[TMP47]], i32 0
-; CHECK-NEXT:    br i1 [[TMP84]], label [[PRED_LOAD_IF10:%.*]], label [[PRED_LOAD_CONTINUE11:%.*]]
-; CHECK:       pred.load.if10:
-; CHECK-NEXT:    [[TMP85:%.*]] = getelementptr inbounds i32, ptr [[ALLOCA]], i64 [[TMP4]]
-; CHECK-NEXT:    [[TMP86:%.*]] = load i32, ptr [[TMP85]], align 4
-; CHECK-NEXT:    [[TMP87:%.*]] = insertelement <4 x i32> poison, i32 [[TMP86]], i32 0
-; CHECK-NEXT:    br label [[PRED_LOAD_CONTINUE11]]
-; CHECK:       pred.load.continue11:
-; CHECK-NEXT:    [[TMP88:%.*]] = phi <4 x i32> [ poison, [[PRED_LOAD_CONTINUE9]] ], [ [[TMP87]], [[PRED_LOAD_IF10]] ]
-; CHECK-NEXT:    [[TMP89:%.*]] = extractelement <4 x i1> [[TMP47]], i32 1
-; CHECK-NEXT:    br i1 [[TMP89]], label [[PRED_LOAD_IF12:%.*]], label [[PRED_LOAD_CONTINUE13:%.*]]
-; CHECK:       pred.load.if12:
-; CHECK-NEXT:    [[TMP90:%.*]] = getelementptr inbounds i32, ptr [[ALLOCA]], i64 [[TMP5]]
-; CHECK-NEXT:    [[TMP91:%.*]] = load i32, ptr [[TMP90]], align 4
-; CHECK-NEXT:    [[TMP92:%.*]] = insertelement <4 x i32> [[TMP88]], i32 [[TMP91]], i32 1
-; CHECK-NEXT:    br label [[PRED_LOAD_CONTINUE13]]
-; CHECK:       pred.load.continue13:
-; CHECK-NEXT:    [[TMP93:%.*]] = phi <4 x i32> [ [[TMP88]], [[PRED_LOAD_CONTINUE11]] ], [ [[TMP92]], [[PRED_LOAD_IF12]] ]
-; CHECK-NEXT:    [[TMP94:%.*]] = extractelement <4 x i1> [[TMP47]], i32 2
-; CHECK-NEXT:    br i1 [[TMP94]], label [[PRED_LOAD_IF14:%.*]], label [[PRED_LOAD_CONTINUE15:%.*]]
-; CHECK:       pred.load.if14:
-; CHECK-NEXT:    [[TMP95:%.*]] = getelementptr inbounds i32, ptr [[ALLOCA]], i64 [[TMP6]]
-; CHECK-NEXT:    [[TMP96:%.*]] = load i32, ptr [[TMP95]], align 4
-; CHECK-NEXT:    [[TMP97:%.*]] = insertelement <4 x i32> [[TMP93]], i32 [[TMP96]], i32 2
-; CHECK-NEXT:    br label [[PRED_LOAD_CONTINUE15]]
-; CHECK:       pred.load.continue15:
-; CHECK-NEXT:    [[TMP98:%.*]] = phi <4 x i32> [ [[TMP93]], [[PRED_LOAD_CONTINUE13]] ], [ [[TMP97]], [[PRED_LOAD_IF14]] ]
-; CHECK-NEXT:    [[TMP99:%.*]] = extractelement <4 x i1> [[TMP47]], i32 3
-; CHECK-NEXT:    br i1 [[TMP99]], label [[PRED_LOAD_IF16:%.*]], label [[PRED_LOAD_CONTINUE17:%.*]]
-; CHECK:       pred.load.if16:
-; CHECK-NEXT:    [[TMP100:%.*]] = getelementptr inbounds i32, ptr [[ALLOCA]], i64 [[TMP7]]
-; CHECK-NEXT:    [[TMP101:%.*]] = load i32, ptr [[TMP100]], align 4
-; CHECK-NEXT:    [[TMP102:%.*]] = insertelement <4 x i32> [[TMP98]], i32 [[TMP101]], i32 3
-; CHECK-NEXT:    br label [[PRED_LOAD_CONTINUE17]]
-; CHECK:       pred.load.continue17:
-; CHECK-NEXT:    [[TMP103:%.*]] = phi <4 x i32> [ [[TMP98]], [[PRED_LOAD_CONTINUE15]] ], [ [[TMP102]], [[PRED_LOAD_IF16]] ]
-; CHECK-NEXT:    [[TMP104:%.*]] = extractelement <4 x i1> [[TMP55]], i32 0
-; CHECK-NEXT:    br i1 [[TMP104]], label [[PRED_LOAD_IF18:%.*]], label [[PRED_LOAD_CONTINUE19:%.*]]
-; CHECK:       pred.load.if18:
-; CHECK-NEXT:    [[TMP105:%.*]] = getelementptr inbounds i32, ptr [[ALLOCA]], i64 [[TMP8]]
-; CHECK-NEXT:    [[TMP106:%.*]] = load i32, ptr [[TMP105]], align 4
-; CHECK-NEXT:    [[TMP107:%.*]] = insertelement <4 x i32> poison, i32 [[TMP106]], i32 0
-; CHECK-NEXT:    br label [[PRED_LOAD_CONTINUE19]]
-; CHECK:       pred.load.continue19:
-; CHECK-NEXT:    [[TMP108:%.*]] = phi <4 x i32> [ poison, [[PRED_LOAD_CONTINUE17]] ], [ [[TMP107]], [[PRED_LOAD_IF18]] ]
-; CHECK-NEXT:    [[TMP109:%.*]] = extractelement <4 x i1> [[TMP55]], i32 1
-; CHECK-NEXT:    br i1 [[TMP109]], label [[PRED_LOAD_IF20:%.*]], label [[PRED_LOAD_CONTINUE21:%.*]]
-; CHECK:       pred.load.if20:
-; CHECK-NEXT:    [[TMP110:%.*]] = getelementptr inbounds i32, ptr [[ALLOCA]], i64 [[TMP9]]
-; CHECK-NEXT:    [[TMP111:%.*]] = load i32, ptr [[TMP110]], align 4
-; CHECK-NEXT:    [[TMP112:%.*]] = insertelement <4 x i32> [[TMP108]], i32 [[TMP111]], i32 1
-; CHECK-NEXT:    br label [[PRED_LOAD_CONTINUE21]]
-; CHECK:       pred.load.continue21:
-; CHECK-NEXT:    [[TMP113:%.*]] = phi <4 x i32> [ [[TMP108]], [[PRED_LOAD_CONTINUE19]] ], [ [[TMP112]], [[PRED_LOAD_IF20]] ]
-; CHECK-NEXT:    [[TMP114:%.*]] = extractelement <4 x i1> [[TMP55]], i32 2
-; CHECK-NEXT:    br i1 [[TMP114]], label [[PRED_LOAD_IF22:%.*]], label [[PRED_LOAD_CONTINUE23:%.*]]
-; CHECK:       pred.load.if22:
-; CHECK-NEXT:    [[TMP115:%.*]] = getelementptr inbounds i32, ptr [[ALLOCA]], i64 [[TMP10]]
-; CHECK-NEXT:    [[TMP116:%.*]] = load i32, ptr [[TMP115]], align 4
-; CHECK-NEXT:    [[TMP117:%.*]] = insertelement <4 x i32> [[TMP113]], i32 [[TMP116]], i32 2
-; CHECK-NEXT:    br label [[PRED_LOAD_CONTINUE23]]
-; CHECK:       pred.load.continue23:
-; CHECK-NEXT:    [[TMP118:%.*]] = phi <4 x i32> [ [[TMP113]], [[PRED_LOAD_CONTINUE21]] ], [ [[TMP117]], [[PRED_LOAD_IF22]] ]
-; CHECK-NEXT:    [[TMP119:%.*]] = extractelement <4 x i1> [[TMP55]], i32 3
-; CHECK-NEXT:    br i1 [[TMP119]], label [[PRED_LOAD_IF24:%.*]], label [[PRED_LOAD_CONTINUE25:%.*]]
-; CHECK:       pred.load.if24:
-; CHECK-NEXT:    [[TMP120:%.*]] = getelementptr inbounds i32, ptr [[ALLOCA]], i64 [[TMP11]]
-; CHECK-NEXT:    [[TMP121:%.*]] = load i32, ptr [[TMP120]], align 4
-; CHECK-NEXT:    [[TMP122:%.*]] = insertelement <4 x i32> [[TMP118]], i32 [[TMP121]], i32 3
-; CHECK-NEXT:    br label [[PRED_LOAD_CONTINUE25]]
-; CHECK:       pred.load.continue25:
-; CHECK-NEXT:    [[TMP123:%.*]] = phi <4 x i32> [ [[TMP118]], [[PRED_LOAD_CONTINUE23]] ], [ [[TMP122]], [[PRED_LOAD_IF24]] ]
-; CHECK-NEXT:    [[TMP124:%.*]] = extractelement <4 x i1> [[TMP63]], i32 0
-; CHECK-NEXT:    br i1 [[TMP124]], label [[PRED_LOAD_IF26:%.*]], label [[PRED_LOAD_CONTINUE27:%.*]]
-; CHECK:       pred.load.if26:
-; CHECK-NEXT:    [[TMP125:%.*]] = getelementptr inbounds i32, ptr [[ALLOCA]], i64 [[TMP12]]
-; CHECK-NEXT:    [[TMP126:%.*]] = load i32, ptr [[TMP125]], align 4
-; CHECK-NEXT:    [[TMP127:%.*]] = insertelement <4 x i32> poison, i32 [[TMP126]], i32 0
-; CHECK-NEXT:    br label [[PRED_LOAD_CONTINUE27]]
-; CHECK:       pred.load.continue27:
-; CHECK-NEXT:    [[TMP128:%.*]] = phi <4 x i32> [ poison, [[PRED_LOAD_CONTINUE25]] ], [ [[TMP127]], [[PRED_LOAD_IF26]] ]
-; CHECK-NEXT:    [[TMP129:%.*]] = extractelement <4 x i1> [[TMP63]], i32 1
-; CHECK-NEXT:    br i1 [[TMP129]], label [[PRED_LOAD_IF28:%.*]], label [[PRED_LOAD_CONTINUE29:%.*]]
-; CHECK:       pred.load.if28:
-; CHECK-NEXT:    [[TMP130:%.*]] = getelementptr inbounds i32, ptr [[ALLOCA]], i64 [[TMP13]]
-; CHECK-NEXT:    [[TMP131:%.*]] = load i32, ptr [[TMP130]], align 4
-; CHECK-NEXT:    [[TMP132:%.*]] = insertelement <4 x i32> [[TMP128]], i32 [[TMP131]], i32 1
-; CHECK-NEXT:    br label [[PRED_LOAD_CONTINUE29]]
-; CHECK:       pred.load.continue29:
-; CHECK-NEXT:    [[TMP133:%.*]] = phi <4 x i32> [ [[TMP128]], [[PRED_LOAD_CONTINUE27]] ], [ [[TMP132]], [[PRED_LOAD_IF28]] ]
-; CHECK-NEXT:    [[TMP134:%.*]] = extractelement <4 x i1> [[TMP63]], i32 2
-; CHECK-NEXT:    br i1 [[TMP134]], label [[PRED_LOAD_IF30:%.*]], label [[PRED_LOAD_CONTINUE31:%.*]]
-; CHECK:       pred.load.if30:
-; CHECK-NEXT:    [[TMP135:%.*]] = getelementptr inbounds i32, ptr [[ALLOCA]], i64 [[TMP14]]
-; CHECK-NEXT:    [[TMP136:%.*]] = load i32, ptr [[TMP135]], align 4
-; CHECK-NEXT:    [[TMP137:%.*]] = insertelement <4 x i32> [[TMP133]], i32 [[TMP136]], i32 2
-; CHECK-NEXT:    br label [[PRED_LOAD_CONTINUE31]]
-; CHECK:       pred.load.continue31:
-; CHECK-NEXT:    [[TMP138:%.*]] = phi <4 x i32> [ [[TMP133]], [[PRED_LOAD_CONTINUE29]] ], [ [[TMP137]], [[PRED_LOAD_IF30]] ]
-; CHECK-NEXT:    [[TMP139:%.*]] = extractelement <4 x i1> [[TMP63]], i32 3
-; CHECK-NEXT:    br i1 [[TMP139]], label [[PRED_LOAD_IF32:%.*]], label [[PRED_LOAD_CONTINUE33]]
-; CHECK:       pred.load.if32:
-; CHECK-NEXT:    [[TMP140:%.*]] = getelementptr inbounds i32, ptr [[ALLOCA]], i64 [[TMP15]]
-; CHECK-NEXT:    [[TMP141:%.*]] = load i32, ptr [[TMP140]], align 4
-; CHECK-NEXT:    [[TMP142:%.*]] = insertelement <4 x i32> [[TMP138]], i32 [[TMP141]], i32 3
-; CHECK-NEXT:    br label [[PRED_LOAD_CONTINUE33]]
-; CHECK:       pred.load.continue33:
-; CHECK-NEXT:    [[TMP143:%.*]] = phi <4 x i32> [ [[TMP138]], [[PRED_LOAD_CONTINUE31]] ], [ [[TMP142]], [[PRED_LOAD_IF32]] ]
-; CHECK-NEXT:    [[PREDPHI:%.*]] = select <4 x i1> [[TMP39]], <4 x i32> [[TMP83]], <4 x i32> zeroinitializer
-; CHECK-NEXT:    [[PREDPHI34:%.*]] = select <4 x i1> [[TMP47]], <4 x i32> [[TMP103]], <4 x i32> zeroinitializer
-; CHECK-NEXT:    [[PREDPHI35:%.*]] = select <4 x i1> [[TMP55]], <4 x i32> [[TMP123]], <4 x i32> zeroinitializer
-; CHECK-NEXT:    [[PREDPHI36:%.*]] = select <4 x i1> [[TMP63]], <4 x i32> [[TMP143]], <4 x i32> zeroinitializer
-; CHECK-NEXT:    [[TMP144]] = add <4 x i32> [[VEC_PHI]], [[PREDPHI]]
-; CHECK-NEXT:    [[TMP145]] = add <4 x i32> [[VEC_PHI1]], [[PREDPHI34]]
-; CHECK-NEXT:    [[TMP146]] = add <4 x i32> [[VEC_PHI2]], [[PREDPHI35]]
-; CHECK-NEXT:    [[TMP147]] = add <4 x i32> [[VEC_PHI3]], [[PREDPHI36]]
+; CHECK-NEXT:    [[TMP64:%.*]] = getelementptr inbounds i32, ptr [[ALLOCA]], i64 [[TMP0]]
+; CHECK-NEXT:    [[TMP65:%.*]] = getelementptr inbounds i32, ptr [[ALLOCA]], i64 [[TMP1]]
+; CHECK-NEXT:    [[TMP66:%.*]] = getelementptr inbounds i32, ptr [[ALLOCA]], i64 [[TMP2]]
+; CHECK-NEXT:    [[TMP67:%.*]] = getelementptr inbounds i32, ptr [[ALLOCA]], i64 [[TMP3]]
+; CHECK-NEXT:    [[TMP68:%.*]] = getelementptr inbounds i32, ptr [[ALLOCA]], i64 [[TMP4]]
+; CHECK-NEXT:    [[TMP69:%.*]] = getelementptr inbounds i32, ptr [[ALLOCA]], i64 [[TMP5]]
+; CHECK-NEXT:    [[TMP70:%.*]] = getelementptr inbounds i32, ptr [[ALLOCA]], i64 [[TMP6]]
+; CHECK-NEXT:    [[TMP71:%.*]] = getelementptr inbounds i32, ptr [[ALLOCA]], i64 [[TMP7]]
+; CHECK-NEXT:    [[TMP72:%.*]] = getelementptr inbounds i32, ptr [[ALLOCA]], i64 [[TMP8]]
+; CHECK-NEXT:    [[TMP73:%.*]] = getelementptr inbounds i32, ptr [[ALLOCA]], i64 [[TMP9]]
+; CHECK-NEXT:    [[TMP74:%.*]] = getelementptr inbounds i32, ptr [[ALLOCA]], i64 [[TMP10]]
+; CHECK-NEXT:    [[TMP75:%.*]] = getelementptr inbounds i32, ptr [[ALLOCA]], i64 [[TMP11]]
+; CHECK-NEXT:    [[TMP76:%.*]] = getelementptr inbounds i32, ptr [[ALLOCA]], i64 [[TMP12]]
+; CHECK-NEXT:    [[TMP77:%.*]] = getelementptr inbounds i32, ptr [[ALLOCA]], i64 [[TMP13]]
+; CHECK-NEXT:    [[TMP78:%.*]] = getelementptr inbounds i32, ptr [[ALLOCA]], i64 [[TMP14]]
+; CHECK-NEXT:    [[TMP79:%.*]] = getelementptr inbounds i32, ptr [[ALLOCA]], i64 [[TMP15]]
+; CHECK-NEXT:    [[TMP80:%.*]] = load i32, ptr [[TMP64]], align 4
+; CHECK-NEXT:    [[TMP81:%.*]] = load i32, ptr [[TMP65]], align 4
+; CHECK-NEXT:    [[TMP82:%.*]] = load i32, ptr [[TMP66]], align 4
+; CHECK-NEXT:    [[TMP83:%.*]] = load i32, ptr [[TMP67]], align 4
+; CHECK-NEXT:    [[TMP84:%.*]] = insertelement <4 x i32> poison, i32 [[TMP80]], i32 0
+; CHECK-NEXT:    [[TMP85:%.*]] = insertelement <4 x i32> [[TMP84]], i32 [[TMP81]], i32 1
+; CHECK-NEXT:    [[TMP86:%.*]] = insertelement <4 x i32> [[TMP85]], i32 [[TMP82]], i32 2
+; CHECK-NEXT:    [[TMP87:%.*]] = insertelement <4 x i32> [[TMP86]], i32 [[TMP83]], i32 3
+; CHECK-NEXT:    [[TMP88:%.*]] = load i32, ptr [[TMP68]], align 4
+; CHECK-NEXT:    [[TMP89:%.*]] = load i32, ptr [[TMP69]], align 4
+; CHECK-NEXT:    [[TMP90:%.*]] = load i32, ptr [[TMP70]], align 4
+; CHECK-NEXT:    [[TMP91:%.*]] = load i32, ptr [[TMP71]], align 4
+; CHECK-NEXT:    [[TMP92:%.*]] = insertelement <4 x i32> poison, i32 [[TMP88]], i32 0
+; CHECK-NEXT:    [[TMP93:%.*]] = insertelement <4 x i32> [[TMP92]], i32 [[TMP89]], i32 1
+; CHECK-NEXT:    [[TMP94:%.*]] = insertelement <4 x i32> [[TMP93]], i32 [[TMP90]], i32 2
+; CHECK-NEXT:    [[TMP95:%.*]] = insertelement <4 x i32> [[TMP94]], i32 [[TMP91]], i32 3
+; CHECK-NEXT:    [[TMP96:%.*]] = load i32, ptr [[TMP72]], align 4
+; CHECK-NEXT:    [[TMP97:%.*]] = load i32, ptr [[TMP73]], align 4
+; CHECK-NEXT:    [[TMP98:%.*]] = load i32, ptr [[TMP74]], align 4
+; CHECK-NEXT:    [[TMP99:%.*]] = load i32, ptr [[TMP75]], align 4
+; CHECK-NEXT:    [[TMP100:%.*]] = insertelement <4 x i32> poison, i32 [[TMP96]], i32 0
+; CHECK-NEXT:    [[TMP101:%.*]] = insertelement <4 x i32> [[TMP100]], i32 [[TMP97]], i32 1
+; CHECK-NEXT:    [[TMP102:%.*]] = insertelement <4 x i32> [[TMP101]], i32 [[TMP98]], i32 2
+; CHECK-NEXT:    [[TMP103:%.*]] = insertelement <4 x i32> [[TMP102]], i32 [[TMP99]], i32 3
+; CHECK-NEXT:    [[TMP104:%.*]] = load i32, ptr [[TMP76]], align 4
+; CHECK-NEXT:    [[TMP105:%.*]] = load i32, ptr [[TMP77]], align 4
+; CHECK-NEXT:    [[TMP106:%.*]] = load i32, ptr [[TMP78]], align 4
+; CHECK-NEXT:    [[TMP107:%.*]] = load i32, ptr [[TMP79]], align 4
+; CHECK-NEXT:    [[TMP108:%.*]] = insertelement <4 x i32> poison, i32 [[TMP104]], i32 0
+; CHECK-NEXT:    [[TMP109:%.*]] = insertelement <4 x i32> [[TMP108]], i32 [[TMP105]], i32 1
+; CHECK-NEXT:    [[TMP110:%.*]] = insertelement <4 x i32> [[TMP109]], i32 [[TMP106]], i32 2
+; CHECK-NEXT:    [[TMP111:%.*]] = insertelement <4 x i32> [[TMP110]], i32 [[TMP107]], i32 3
+; CHECK-NEXT:    [[PREDPHI:%.*]] = select <4 x i1> [[TMP39]], <4 x i32> [[TMP87]], <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[PREDPHI4:%.*]] = select <4 x i1> [[TMP47]], <4 x i32> [[TMP95]], <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[PREDPHI5:%.*]] = select <4 x i1> [[TMP55]], <4 x i32> [[TMP103]], <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[PREDPHI6:%.*]] = select <4 x i1> [[TMP63]], <4 x i32> [[TMP111]], <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP112]] = add <4 x i32> [[VEC_PHI]], [[PREDPHI]]
+; CHECK-NEXT:    [[TMP113]] = add <4 x i32> [[VEC_PHI1]], [[PREDPHI4]]
+; CHECK-NEXT:    [[TMP114]] = add <4 x i32> [[VEC_PHI2]], [[PREDPHI5]]
+; CHECK-NEXT:    [[TMP115]] = add <4 x i32> [[VEC_PHI3]], [[PREDPHI6]]
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
-; CHECK-NEXT:    [[TMP148:%.*]] = icmp eq i64 [[INDEX_NEXT]], 48
-; CHECK-NEXT:    br i1 [[TMP148]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP36:![0-9]+]]
+; CHECK-NEXT:    [[TMP116:%.*]] = icmp eq i64 [[INDEX_NEXT]], 48
+; CHECK-NEXT:    br i1 [[TMP116]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP36:![0-9]+]]
 ; CHECK:       middle.block:
-; CHECK-NEXT:    [[BIN_RDX:%.*]] = add <4 x i32> [[TMP145]], [[TMP144]]
-; CHECK-NEXT:    [[BIN_RDX37:%.*]] = add <4 x i32> [[TMP146]], [[BIN_RDX]]
-; CHECK-NEXT:    [[BIN_RDX38:%.*]] = add <4 x i32> [[TMP147]], [[BIN_RDX37]]
-; CHECK-NEXT:    [[TMP149:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[BIN_RDX38]])
+; CHECK-NEXT:    [[BIN_RDX:%.*]] = add <4 x i32> [[TMP113]], [[TMP112]]
+; CHECK-NEXT:    [[BIN_RDX7:%.*]] = add <4 x i32> [[TMP114]], [[BIN_RDX]]
+; CHECK-NEXT:    [[BIN_RDX8:%.*]] = add <4 x i32> [[TMP115]], [[BIN_RDX7]]
+; CHECK-NEXT:    [[TMP117:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[BIN_RDX8]])
 ; CHECK-NEXT:    br i1 false, label [[LOOP_EXIT:%.*]], label [[SCALAR_PH]]
 ; CHECK:       scalar.ph:
 ; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 96, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
-; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP149]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ]
+; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP117]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ]
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
 ; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LATCH:%.*]] ]
@@ -3181,7 +3069,7 @@ define i32 @neg_test_non_unit_stride_off_by_four_bytes(i64 %len, ptr %test_base)
 ; CHECK-NEXT:    [[EXIT:%.*]] = icmp ugt i64 [[IV]], 100
 ; CHECK-NEXT:    br i1 [[EXIT]], label [[LOOP_EXIT]], label [[LOOP]], !llvm.loop [[LOOP37:![0-9]+]]
 ; CHECK:       loop_exit:
-; CHECK-NEXT:    [[ACCUM_NEXT_LCSSA:%.*]] = phi i32 [ [[ACCUM_NEXT]], [[LATCH]] ], [ [[TMP149]], [[MIDDLE_BLOCK]] ]
+; CHECK-NEXT:    [[ACCUM_NEXT_LCSSA:%.*]] = phi i32 [ [[ACCUM_NEXT]], [[LATCH]] ], [ [[TMP117]], [[MIDDLE_BLOCK]] ]
 ; CHECK-NEXT:    ret i32 [[ACCUM_NEXT_LCSSA]]
 ;
 entry:
diff --git a/llvm/test/Transforms/LoopVectorize/X86/masked-store-cost.ll b/llvm/test/Transforms/LoopVectorize/X86/masked-store-cost.ll
index f0a1e5c47d06a..7f0b6b2f9b4d7 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/masked-store-cost.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/masked-store-cost.ll
@@ -84,7 +84,7 @@ define i32 @test_scalar_predicated_cost(i64 %x, i64 %y, ptr %A) #0 {
 ; CHECK:       vec.epilog.middle.block:
 ; CHECK-NEXT:    br i1 false, label [[EXIT]], label [[SCALAR_PH]]
 ; CHECK:       vec.epilog.scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 100, [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ 0, [[ITER_CHECK:%.*]] ], [ 96, [[VEC_EPILOG_ITER_CHECK]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 100, [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ 96, [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[ITER_CHECK:%.*]] ]
 ; CHECK-NEXT:    br label [[LOOP_HEADER1:%.*]]
 ; CHECK:       loop.header:
 ; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP_LATCH:%.*]] ]
diff --git a/llvm/test/Transforms/LoopVectorize/X86/masked_load_store.ll b/llvm/test/Transforms/LoopVectorize/X86/masked_load_store.ll
index 7e42ffd0f44c3..206bbdf262b72 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/masked_load_store.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/masked_load_store.ll
@@ -53,7 +53,7 @@ define void @foo1(ptr nocapture %A, ptr nocapture readonly %B, ptr nocapture rea
 ; AVX1:       middle.block:
 ; AVX1-NEXT:    br i1 true, label [[FOR_END:%.*]], label [[SCALAR_PH]]
 ; AVX1:       scalar.ph:
-; AVX1-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 10000, [[MIDDLE_BLOCK]] ], [ 0, [[VECTOR_MEMCHECK]] ], [ 0, [[ENTRY:%.*]] ]
+; AVX1-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 10000, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ], [ 0, [[VECTOR_MEMCHECK]] ]
 ; AVX1-NEXT:    br label [[FOR_BODY:%.*]]
 ; AVX1:       for.body:
 ; AVX1-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_INC:%.*]] ]
@@ -160,7 +160,7 @@ define void @foo1(ptr nocapture %A, ptr nocapture readonly %B, ptr nocapture rea
 ; AVX2:       vec.epilog.middle.block:
 ; AVX2-NEXT:    br i1 true, label [[FOR_END]], label [[SCALAR_PH]]
 ; AVX2:       vec.epilog.scalar.ph:
-; AVX2-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 10000, [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ 0, [[VECTOR_MEMCHECK]] ], [ 0, [[ITER_CHECK:%.*]] ], [ 9984, [[VEC_EPILOG_ITER_CHECK]] ]
+; AVX2-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 10000, [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ 9984, [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[VECTOR_MEMCHECK]] ], [ 0, [[ITER_CHECK:%.*]] ]
 ; AVX2-NEXT:    br label [[FOR_BODY1:%.*]]
 ; AVX2:       for.body:
 ; AVX2-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_INC:%.*]] ]
@@ -267,7 +267,7 @@ define void @foo1(ptr nocapture %A, ptr nocapture readonly %B, ptr nocapture rea
 ; AVX512:       vec.epilog.middle.block:
 ; AVX512-NEXT:    br i1 true, label [[FOR_END]], label [[VEC_EPILOG_SCALAR_PH]]
 ; AVX512:       vec.epilog.scalar.ph:
-; AVX512-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 10000, [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ 0, [[VECTOR_MEMCHECK]] ], [ 0, [[ITER_CHECK:%.*]] ], [ 9984, [[VEC_EPILOG_ITER_CHECK]] ]
+; AVX512-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 10000, [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ 9984, [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[VECTOR_MEMCHECK]] ], [ 0, [[ITER_CHECK:%.*]] ]
 ; AVX512-NEXT:    br label [[FOR_BODY:%.*]]
 ; AVX512:       for.body:
 ; AVX512-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[VEC_EPILOG_SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_INC:%.*]] ]
@@ -354,7 +354,7 @@ define void @foo1_addrspace1(ptr addrspace(1) nocapture %A, ptr addrspace(1) noc
 ; AVX1:       middle.block:
 ; AVX1-NEXT:    br i1 true, label [[FOR_END:%.*]], label [[SCALAR_PH]]
 ; AVX1:       scalar.ph:
-; AVX1-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 10000, [[MIDDLE_BLOCK]] ], [ 0, [[VECTOR_MEMCHECK]] ], [ 0, [[ENTRY:%.*]] ]
+; AVX1-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 10000, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ], [ 0, [[VECTOR_MEMCHECK]] ]
 ; AVX1-NEXT:    br label [[FOR_BODY:%.*]]
 ; AVX1:       for.body:
 ; AVX1-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_INC:%.*]] ]
@@ -461,7 +461,7 @@ define void @foo1_addrspace1(ptr addrspace(1) nocapture %A, ptr addrspace(1) noc
 ; AVX2:       vec.epilog.middle.block:
 ; AVX2-NEXT:    br i1 true, label [[FOR_END]], label [[SCALAR_PH]]
 ; AVX2:       vec.epilog.scalar.ph:
-; AVX2-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 10000, [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ 0, [[VECTOR_MEMCHECK]] ], [ 0, [[ITER_CHECK:%.*]] ], [ 9984, [[VEC_EPILOG_ITER_CHECK]] ]
+; AVX2-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 10000, [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ 9984, [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[VECTOR_MEMCHECK]] ], [ 0, [[ITER_CHECK:%.*]] ]
 ; AVX2-NEXT:    br label [[FOR_BODY1:%.*]]
 ; AVX2:       for.body:
 ; AVX2-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_INC:%.*]] ]
@@ -568,7 +568,7 @@ define void @foo1_addrspace1(ptr addrspace(1) nocapture %A, ptr addrspace(1) noc
 ; AVX512:       vec.epilog.middle.block:
 ; AVX512-NEXT:    br i1 true, label [[FOR_END]], label [[VEC_EPILOG_SCALAR_PH]]
 ; AVX512:       vec.epilog.scalar.ph:
-; AVX512-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 10000, [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ 0, [[VECTOR_MEMCHECK]] ], [ 0, [[ITER_CHECK:%.*]] ], [ 9984, [[VEC_EPILOG_ITER_CHECK]] ]
+; AVX512-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 10000, [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ 9984, [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[VECTOR_MEMCHECK]] ], [ 0, [[ITER_CHECK:%.*]] ]
 ; AVX512-NEXT:    br label [[FOR_BODY:%.*]]
 ; AVX512:       for.body:
 ; AVX512-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[VEC_EPILOG_SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_INC:%.*]] ]
@@ -665,7 +665,7 @@ define void @foo2(ptr nocapture %A, ptr nocapture readonly %B, ptr nocapture rea
 ; AVX1:       middle.block:
 ; AVX1-NEXT:    br i1 true, label [[FOR_END:%.*]], label [[SCALAR_PH]]
 ; AVX1:       scalar.ph:
-; AVX1-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 10000, [[MIDDLE_BLOCK]] ], [ 0, [[VECTOR_MEMCHECK]] ], [ 0, [[ENTRY:%.*]] ]
+; AVX1-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 10000, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ], [ 0, [[VECTOR_MEMCHECK]] ]
 ; AVX1-NEXT:    br label [[FOR_BODY:%.*]]
 ; AVX1:       for.body:
 ; AVX1-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_INC:%.*]] ]
@@ -778,7 +778,7 @@ define void @foo2(ptr nocapture %A, ptr nocapture readonly %B, ptr nocapture rea
 ; AVX2:       vec.epilog.middle.block:
 ; AVX2-NEXT:    br i1 true, label [[FOR_END]], label [[SCALAR_PH]]
 ; AVX2:       vec.epilog.scalar.ph:
-; AVX2-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 10000, [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ 0, [[VECTOR_MEMCHECK]] ], [ 0, [[ITER_CHECK:%.*]] ], [ 9984, [[VEC_EPILOG_ITER_CHECK]] ]
+; AVX2-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 10000, [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ 9984, [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[VECTOR_MEMCHECK]] ], [ 0, [[ITER_CHECK:%.*]] ]
 ; AVX2-NEXT:    br label [[FOR_BODY1:%.*]]
 ; AVX2:       for.body:
 ; AVX2-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_INC:%.*]] ]
@@ -891,7 +891,7 @@ define void @foo2(ptr nocapture %A, ptr nocapture readonly %B, ptr nocapture rea
 ; AVX512:       vec.epilog.middle.block:
 ; AVX512-NEXT:    br i1 true, label [[FOR_END]], label [[VEC_EPILOG_SCALAR_PH]]
 ; AVX512:       vec.epilog.scalar.ph:
-; AVX512-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 10000, [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ 0, [[VECTOR_MEMCHECK]] ], [ 0, [[ITER_CHECK:%.*]] ], [ 9984, [[VEC_EPILOG_ITER_CHECK]] ]
+; AVX512-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 10000, [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ 9984, [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[VECTOR_MEMCHECK]] ], [ 0, [[ITER_CHECK:%.*]] ]
 ; AVX512-NEXT:    br label [[FOR_BODY:%.*]]
 ; AVX512:       for.body:
 ; AVX512-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[VEC_EPILOG_SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_INC:%.*]] ]
@@ -1019,7 +1019,7 @@ define void @foo3(ptr nocapture %A, ptr nocapture readonly %B, ptr nocapture rea
 ; AVX1:       middle.block:
 ; AVX1-NEXT:    br i1 true, label [[FOR_END:%.*]], label [[SCALAR_PH]]
 ; AVX1:       scalar.ph:
-; AVX1-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 10000, [[MIDDLE_BLOCK]] ], [ 0, [[VECTOR_MEMCHECK]] ], [ 0, [[ENTRY:%.*]] ]
+; AVX1-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 10000, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ], [ 0, [[VECTOR_MEMCHECK]] ]
 ; AVX1-NEXT:    br label [[FOR_BODY:%.*]]
 ; AVX1:       for.body:
 ; AVX1-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_INC:%.*]] ]
@@ -1107,7 +1107,7 @@ define void @foo3(ptr nocapture %A, ptr nocapture readonly %B, ptr nocapture rea
 ; AVX2:       middle.block:
 ; AVX2-NEXT:    br i1 true, label [[FOR_END:%.*]], label [[SCALAR_PH]]
 ; AVX2:       scalar.ph:
-; AVX2-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 10000, [[MIDDLE_BLOCK]] ], [ 0, [[VECTOR_MEMCHECK]] ], [ 0, [[ENTRY:%.*]] ]
+; AVX2-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 10000, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ], [ 0, [[VECTOR_MEMCHECK]] ]
 ; AVX2-NEXT:    br label [[FOR_BODY:%.*]]
 ; AVX2:       for.body:
 ; AVX2-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_INC:%.*]] ]
@@ -1222,7 +1222,7 @@ define void @foo3(ptr nocapture %A, ptr nocapture readonly %B, ptr nocapture rea
 ; AVX512:       vec.epilog.middle.block:
 ; AVX512-NEXT:    br i1 true, label [[FOR_END]], label [[SCALAR_PH]]
 ; AVX512:       vec.epilog.scalar.ph:
-; AVX512-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 10000, [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ 0, [[VECTOR_MEMCHECK]] ], [ 0, [[ITER_CHECK:%.*]] ], [ 9984, [[VEC_EPILOG_ITER_CHECK]] ]
+; AVX512-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 10000, [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ 9984, [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[VECTOR_MEMCHECK]] ], [ 0, [[ITER_CHECK:%.*]] ]
 ; AVX512-NEXT:    br label [[FOR_BODY1:%.*]]
 ; AVX512:       for.body:
 ; AVX512-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_INC:%.*]] ]
@@ -1347,7 +1347,7 @@ define void @foo4(ptr nocapture %A, ptr nocapture readonly %B, ptr nocapture rea
 ; AVX512:       middle.block:
 ; AVX512-NEXT:    br i1 false, label [[FOR_END:%.*]], label [[SCALAR_PH]]
 ; AVX512:       scalar.ph:
-; AVX512-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 9984, [[MIDDLE_BLOCK]] ], [ 0, [[VECTOR_MEMCHECK]] ], [ 0, [[ENTRY:%.*]] ]
+; AVX512-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 9984, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ], [ 0, [[VECTOR_MEMCHECK]] ]
 ; AVX512-NEXT:    br label [[FOR_BODY:%.*]]
 ; AVX512:       for.body:
 ; AVX512-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_INC:%.*]] ]
@@ -1527,7 +1527,7 @@ define void @foo6(ptr nocapture readonly %in, ptr nocapture %out, i32 %size, ptr
 ; AVX2:       middle.block:
 ; AVX2-NEXT:    br i1 true, label [[FOR_END:%.*]], label [[SCALAR_PH]]
 ; AVX2:       scalar.ph:
-; AVX2-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ -1, [[MIDDLE_BLOCK]] ], [ 4095, [[VECTOR_MEMCHECK]] ], [ 4095, [[ENTRY:%.*]] ]
+; AVX2-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ -1, [[MIDDLE_BLOCK]] ], [ 4095, [[ENTRY:%.*]] ], [ 4095, [[VECTOR_MEMCHECK]] ]
 ; AVX2-NEXT:    br label [[FOR_BODY:%.*]]
 ; AVX2:       for.body:
 ; AVX2-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_INC:%.*]] ]
@@ -1639,7 +1639,7 @@ define void @foo6(ptr nocapture readonly %in, ptr nocapture %out, i32 %size, ptr
 ; AVX512:       middle.block:
 ; AVX512-NEXT:    br i1 true, label [[FOR_END:%.*]], label [[SCALAR_PH]]
 ; AVX512:       scalar.ph:
-; AVX512-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ -1, [[MIDDLE_BLOCK]] ], [ 4095, [[VECTOR_MEMCHECK]] ], [ 4095, [[ENTRY:%.*]] ]
+; AVX512-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ -1, [[MIDDLE_BLOCK]] ], [ 4095, [[ENTRY:%.*]] ], [ 4095, [[VECTOR_MEMCHECK]] ]
 ; AVX512-NEXT:    br label [[FOR_BODY:%.*]]
 ; AVX512:       for.body:
 ; AVX512-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_INC:%.*]] ]
@@ -1806,7 +1806,7 @@ define void @foo7(ptr noalias nocapture %out, ptr noalias nocapture readonly %in
 ; AVX1-NEXT:    [[CMP_N14:%.*]] = icmp eq i64 [[WIDE_TRIP_COUNT]], [[N_VEC9]]
 ; AVX1-NEXT:    br i1 [[CMP_N14]], label [[FOR_END_LOOPEXIT]], label [[VEC_EPILOG_SCALAR_PH]]
 ; AVX1:       vec.epilog.scalar.ph:
-; AVX1-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC9]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ 0, [[FOR_BODY_PREHEADER]] ], [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ]
+; AVX1-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC9]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[FOR_BODY_PREHEADER]] ]
 ; AVX1-NEXT:    br label [[FOR_BODY1:%.*]]
 ; AVX1:       for.body:
 ; AVX1-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[VEC_EPILOG_SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_INC:%.*]] ]
@@ -1942,7 +1942,7 @@ define void @foo7(ptr noalias nocapture %out, ptr noalias nocapture readonly %in
 ; AVX2-NEXT:    [[CMP_N14:%.*]] = icmp eq i64 [[WIDE_TRIP_COUNT]], [[N_VEC9]]
 ; AVX2-NEXT:    br i1 [[CMP_N14]], label [[FOR_END_LOOPEXIT]], label [[VEC_EPILOG_SCALAR_PH]]
 ; AVX2:       vec.epilog.scalar.ph:
-; AVX2-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC9]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ 0, [[FOR_BODY_PREHEADER]] ], [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ]
+; AVX2-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC9]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[FOR_BODY_PREHEADER]] ]
 ; AVX2-NEXT:    br label [[FOR_BODY1:%.*]]
 ; AVX2:       for.body:
 ; AVX2-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[VEC_EPILOG_SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_INC:%.*]] ]
@@ -2078,7 +2078,7 @@ define void @foo7(ptr noalias nocapture %out, ptr noalias nocapture readonly %in
 ; AVX512-NEXT:    [[CMP_N14:%.*]] = icmp eq i64 [[WIDE_TRIP_COUNT]], [[N_VEC9]]
 ; AVX512-NEXT:    br i1 [[CMP_N14]], label [[FOR_END_LOOPEXIT]], label [[VEC_EPILOG_SCALAR_PH]]
 ; AVX512:       vec.epilog.scalar.ph:
-; AVX512-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC9]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ 0, [[FOR_BODY_PREHEADER]] ], [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ]
+; AVX512-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC9]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[FOR_BODY_PREHEADER]] ]
 ; AVX512-NEXT:    br label [[FOR_BODY1:%.*]]
 ; AVX512:       for.body:
 ; AVX512-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[VEC_EPILOG_SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_INC:%.*]] ]
@@ -2259,7 +2259,7 @@ define void @foo8(ptr noalias nocapture %out, ptr noalias nocapture readonly %in
 ; AVX1-NEXT:    [[CMP_N14:%.*]] = icmp eq i64 [[WIDE_TRIP_COUNT]], [[N_VEC9]]
 ; AVX1-NEXT:    br i1 [[CMP_N14]], label [[FOR_END_LOOPEXIT]], label [[VEC_EPILOG_SCALAR_PH]]
 ; AVX1:       vec.epilog.scalar.ph:
-; AVX1-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC9]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ 0, [[FOR_BODY_PREHEADER]] ], [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ]
+; AVX1-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC9]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[FOR_BODY_PREHEADER]] ]
 ; AVX1-NEXT:    br label [[FOR_BODY1:%.*]]
 ; AVX1:       for.body:
 ; AVX1-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[VEC_EPILOG_SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_INC:%.*]] ]
@@ -2395,7 +2395,7 @@ define void @foo8(ptr noalias nocapture %out, ptr noalias nocapture readonly %in
 ; AVX2-NEXT:    [[CMP_N14:%.*]] = icmp eq i64 [[WIDE_TRIP_COUNT]], [[N_VEC9]]
 ; AVX2-NEXT:    br i1 [[CMP_N14]], label [[FOR_END_LOOPEXIT]], label [[VEC_EPILOG_SCALAR_PH]]
 ; AVX2:       vec.epilog.scalar.ph:
-; AVX2-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC9]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ 0, [[FOR_BODY_PREHEADER]] ], [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ]
+; AVX2-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC9]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[FOR_BODY_PREHEADER]] ]
 ; AVX2-NEXT:    br label [[FOR_BODY1:%.*]]
 ; AVX2:       for.body:
 ; AVX2-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[VEC_EPILOG_SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_INC:%.*]] ]
@@ -2531,7 +2531,7 @@ define void @foo8(ptr noalias nocapture %out, ptr noalias nocapture readonly %in
 ; AVX512-NEXT:    [[CMP_N14:%.*]] = icmp eq i64 [[WIDE_TRIP_COUNT]], [[N_VEC9]]
 ; AVX512-NEXT:    br i1 [[CMP_N14]], label [[FOR_END_LOOPEXIT]], label [[VEC_EPILOG_SCALAR_PH]]
 ; AVX512:       vec.epilog.scalar.ph:
-; AVX512-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC9]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ 0, [[FOR_BODY_PREHEADER]] ], [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ]
+; AVX512-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC9]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[FOR_BODY_PREHEADER]] ]
 ; AVX512-NEXT:    br label [[FOR_BODY1:%.*]]
 ; AVX512:       for.body:
 ; AVX512-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[VEC_EPILOG_SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_INC:%.*]] ]
diff --git a/llvm/test/Transforms/LoopVectorize/X86/multi-exit-cost.ll b/llvm/test/Transforms/LoopVectorize/X86/multi-exit-cost.ll
index 6c97ab362fc86..99d39f3e88983 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/multi-exit-cost.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/multi-exit-cost.ll
@@ -43,8 +43,8 @@ define i64 @test_value_in_exit_compare_chain_used_outside(ptr %src, i64 %x, i64
 ; CHECK-NEXT:    [[TMP31:%.*]] = call i8 @llvm.vector.reduce.xor.v8i8(<8 x i8> [[TMP29]])
 ; CHECK-NEXT:    br label %[[SCALAR_PH]]
 ; CHECK:       [[SCALAR_PH]]:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[VECTOR_SCEVCHECK]] ], [ 0, %[[ENTRY]] ]
-; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i8 [ [[TMP31]], %[[MIDDLE_BLOCK]] ], [ 0, %[[VECTOR_SCEVCHECK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_SCEVCHECK]] ]
+; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i8 [ [[TMP31]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_SCEVCHECK]] ]
 ; CHECK-NEXT:    br label %[[LOOP_HEADER:.*]]
 ; CHECK:       [[LOOP_HEADER]]:
 ; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
diff --git a/llvm/test/Transforms/LoopVectorize/X86/pr23997.ll b/llvm/test/Transforms/LoopVectorize/X86/pr23997.ll
index f7a3a1245c286..d00edeb1c9e6d 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/pr23997.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/pr23997.ll
@@ -75,7 +75,7 @@ define void @foo(ptr addrspace(1) align 8 dereferenceable_or_null(16), ptr addrs
 ; CHECK-NEXT:    [[CMP_N12:%.*]] = icmp eq i64 [[TMP2]], [[N_VEC8]]
 ; CHECK-NEXT:    br i1 [[CMP_N12]], label [[LOOPEXIT]], label [[VEC_EPILOG_SCALAR_PH]]
 ; CHECK:       vec.epilog.scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC8]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ 0, [[VECTOR_MEMCHECK]] ], [ 0, [[ITER_CHECK]] ], [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC8]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[VECTOR_MEMCHECK]] ], [ 0, [[ITER_CHECK]] ]
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
 ; CHECK-NEXT:    [[INDVARS_IV3:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[VEC_EPILOG_SCALAR_PH]] ], [ [[INDVARS_IV_NEXT4:%.*]], [[LOOP]] ]
diff --git a/llvm/test/Transforms/LoopVectorize/X86/pr35432.ll b/llvm/test/Transforms/LoopVectorize/X86/pr35432.ll
index 1194e82325ef7..1845243d1278e 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/pr35432.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/pr35432.ll
@@ -83,8 +83,8 @@ define i32 @main(ptr %ptr) {
 ; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i32 [[TMP4]], [[N_VEC]]
 ; CHECK-NEXT:    br i1 [[CMP_N]], label [[FOR_COND4_FOR_INC9_CRIT_EDGE:%.*]], label [[SCALAR_PH]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ [[DOTPROMOTED]], [[VECTOR_SCEVCHECK]] ], [ [[DOTPROMOTED]], [[FOR_BODY8_LR_PH]] ]
-; CHECK-NEXT:    [[BC_RESUME_VAL3:%.*]] = phi i8 [ [[IND_END2]], [[MIDDLE_BLOCK]] ], [ [[CONV3]], [[VECTOR_SCEVCHECK]] ], [ [[CONV3]], [[FOR_BODY8_LR_PH]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ [[DOTPROMOTED]], [[FOR_BODY8_LR_PH]] ], [ [[DOTPROMOTED]], [[VECTOR_SCEVCHECK]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL3:%.*]] = phi i8 [ [[IND_END2]], [[MIDDLE_BLOCK]] ], [ [[CONV3]], [[FOR_BODY8_LR_PH]] ], [ [[CONV3]], [[VECTOR_SCEVCHECK]] ]
 ; CHECK-NEXT:    br label [[FOR_BODY8:%.*]]
 ; CHECK:       for.body8:
 ; CHECK-NEXT:    [[INC5:%.*]] = phi i32 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INC:%.*]], [[FOR_BODY8]] ]
diff --git a/llvm/test/Transforms/LoopVectorize/X86/pr36524.ll b/llvm/test/Transforms/LoopVectorize/X86/pr36524.ll
index 90b86bb3ffb07..ee8374f952c7a 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/pr36524.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/pr36524.ll
@@ -37,8 +37,8 @@ define void @foo(ptr %ptr, ptr %ptr.2) {
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    br i1 true, label [[EXIT:%.*]], label [[SCALAR_PH]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 80, [[MIDDLE_BLOCK]] ], [ 0, [[VECTOR_MEMCHECK]] ], [ 0, [[ENTRY:%.*]] ]
-; CHECK-NEXT:    [[BC_RESUME_VAL2:%.*]] = phi i64 [ 82, [[MIDDLE_BLOCK]] ], [ 2, [[VECTOR_MEMCHECK]] ], [ 2, [[ENTRY]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 80, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ], [ 0, [[VECTOR_MEMCHECK]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL2:%.*]] = phi i64 [ 82, [[MIDDLE_BLOCK]] ], [ 2, [[ENTRY]] ], [ 2, [[VECTOR_MEMCHECK]] ]
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       vector.scevcheck:
 ; CHECK-NEXT:    unreachable
diff --git a/llvm/test/Transforms/LoopVectorize/X86/pr47437.ll b/llvm/test/Transforms/LoopVectorize/X86/pr47437.ll
index d8ec92124682a..7b1c7ae94ff41 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/pr47437.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/pr47437.ll
@@ -307,7 +307,7 @@ define void @test_muladd(ptr noalias nocapture %d1, ptr noalias nocapture readon
 ; AVX1-NEXT:    [[CMP_N34:%.*]] = icmp eq i64 [[WIDE_TRIP_COUNT]], [[N_VEC25]]
 ; AVX1-NEXT:    br i1 [[CMP_N34]], label [[FOR_END_LOOPEXIT]], label [[VEC_EPILOG_SCALAR_PH]]
 ; AVX1:       vec.epilog.scalar.ph:
-; AVX1-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC25]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ 0, [[FOR_BODY_PREHEADER]] ], [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ]
+; AVX1-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC25]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[FOR_BODY_PREHEADER]] ]
 ; AVX1-NEXT:    br label [[FOR_BODY1:%.*]]
 ; AVX1:       for.body:
 ; AVX1-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[VEC_EPILOG_SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY1]] ]
diff --git a/llvm/test/Transforms/LoopVectorize/X86/pr54634.ll b/llvm/test/Transforms/LoopVectorize/X86/pr54634.ll
index 1cee80f88ec62..23a6a1286a0f0 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/pr54634.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/pr54634.ll
@@ -92,7 +92,7 @@ define ptr addrspace(10) @japi1_vect_42283(ptr nocapture readonly %0, i32 %1) lo
 ; CHECK-NEXT:    [[CMP_N15:%.*]] = icmp eq i64 [[TMP8]], [[N_VEC5]]
 ; CHECK-NEXT:    br i1 [[CMP_N15]], label [[L44]], label [[VEC_EPILOG_SCALAR_PH]]
 ; CHECK:       vec.epilog.scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL6:%.*]] = phi i64 [ [[N_VEC5]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ 0, [[ITER_CHECK:%.*]] ], [ [[N_VEC]], [[MIDDLE_BLOCK]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL6:%.*]] = phi i64 [ [[N_VEC5]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ITER_CHECK:%.*]] ]
 ; CHECK-NEXT:    br label [[L27:%.*]]
 ; CHECK:       L26:
 ; CHECK-NEXT:    [[VALUE_PHI5:%.*]] = phi i64 [ [[BC_RESUME_VAL6]], [[VEC_EPILOG_SCALAR_PH]] ], [ [[TMP27:%.*]], [[L27]] ]
diff --git a/llvm/test/Transforms/LoopVectorize/X86/pr56319-vector-exit-cond-optimization-epilogue-vectorization.ll b/llvm/test/Transforms/LoopVectorize/X86/pr56319-vector-exit-cond-optimization-epilogue-vectorization.ll
index c317e877ee8a5..5476ff504edb3 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/pr56319-vector-exit-cond-optimization-epilogue-vectorization.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/pr56319-vector-exit-cond-optimization-epilogue-vectorization.ll
@@ -49,7 +49,7 @@ define void @pr56319(ptr noalias %src, ptr noalias %dst) {
 ; CHECK:       vec.epilog.middle.block:
 ; CHECK-NEXT:    br label [[VEC_EPILOG_SCALAR_PH]]
 ; CHECK:       vec.epilog.scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 36, [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ 0, [[ITER_CHECK:%.*]] ], [ 32, [[VEC_EPILOG_ITER_CHECK]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 36, [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ 32, [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[ITER_CHECK:%.*]] ]
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
 ; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[VEC_EPILOG_SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
diff --git a/llvm/test/Transforms/LoopVectorize/X86/pr72969.ll b/llvm/test/Transforms/LoopVectorize/X86/pr72969.ll
index d1c0201ccb9a4..0cd746590e0f1 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/pr72969.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/pr72969.ll
@@ -85,9 +85,9 @@ define void @test(ptr %p) {
 ; VEC-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[TMP4]], [[N_VEC]]
 ; VEC-NEXT:    br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
 ; VEC:       scalar.ph:
-; VEC-NEXT:    [[SCALAR_RECUR_INIT:%.*]] = phi i64 [ [[VECTOR_RECUR_EXTRACT]], [[MIDDLE_BLOCK]] ], [ 1, [[VECTOR_SCEVCHECK]] ], [ 1, [[ENTRY:%.*]] ]
-; VEC-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i16 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ 1, [[VECTOR_SCEVCHECK]] ], [ 1, [[ENTRY]] ]
-; VEC-NEXT:    [[BC_RESUME_VAL3:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[VECTOR_SCEVCHECK]] ], [ 0, [[ENTRY]] ]
+; VEC-NEXT:    [[SCALAR_RECUR_INIT:%.*]] = phi i64 [ [[VECTOR_RECUR_EXTRACT]], [[MIDDLE_BLOCK]] ], [ 1, [[ENTRY:%.*]] ], [ 1, [[VECTOR_SCEVCHECK]] ]
+; VEC-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i16 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ 1, [[ENTRY]] ], [ 1, [[VECTOR_SCEVCHECK]] ]
+; VEC-NEXT:    [[BC_RESUME_VAL3:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ], [ 0, [[VECTOR_SCEVCHECK]] ]
 ; VEC-NEXT:    br label [[FOR_BODY:%.*]]
 ; VEC:       for.body:
 ; VEC-NEXT:    [[SCALAR_RECUR:%.*]] = phi i64 [ [[SCALAR_RECUR_INIT]], [[SCALAR_PH]] ], [ [[IDX:%.*]], [[FOR_BODY]] ]
diff --git a/llvm/test/Transforms/LoopVectorize/X86/scatter_crash.ll b/llvm/test/Transforms/LoopVectorize/X86/scatter_crash.ll
index a0294f7ac7992..58d3ead2d2919 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/scatter_crash.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/scatter_crash.ll
@@ -109,8 +109,8 @@ define void @_Z3fn1v() #0 {
 ; CHECK-NEXT:    [[CMP_N23:%.*]] = icmp eq i64 [[TMP6]], [[N_VEC7]]
 ; CHECK-NEXT:    br i1 [[CMP_N23]], label [[FOR_COND_CLEANUP_LOOPEXIT99]], label [[VEC_EPILOG_SCALAR_PH]]
 ; CHECK:       vec.epilog.scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL10:%.*]] = phi i64 [ [[IND_END8]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ 8, [[ITER_CHECK]] ], [ [[IND_END9]], [[VEC_EPILOG_ITER_CHECK]] ]
-; CHECK-NEXT:    [[BC_RESUME_VAL13:%.*]] = phi i64 [ [[IND_END11]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ 0, [[ITER_CHECK]] ], [ [[IND_END12]], [[VEC_EPILOG_ITER_CHECK]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL10:%.*]] = phi i64 [ [[IND_END8]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[IND_END9]], [[VEC_EPILOG_ITER_CHECK]] ], [ 8, [[ITER_CHECK]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL13:%.*]] = phi i64 [ [[IND_END11]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[IND_END12]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[ITER_CHECK]] ]
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK:       iter.check23:
 ; CHECK-NEXT:    [[TMP26:%.*]] = add nsw i64 [[TMP3]], -9
@@ -210,8 +210,8 @@ define void @_Z3fn1v() #0 {
 ; CHECK-NEXT:    [[CMP_N65:%.*]] = icmp eq i64 [[TMP28]], [[N_VEC53]]
 ; CHECK-NEXT:    br i1 [[CMP_N65]], label [[FOR_COND_CLEANUP_LOOPEXIT]], label [[VEC_EPILOG_SCALAR_PH41]]
 ; CHECK:       vec.epilog.scalar.ph41:
-; CHECK-NEXT:    [[BC_RESUME_VAL56:%.*]] = phi i64 [ [[IND_END54]], [[VEC_EPILOG_MIDDLE_BLOCK40]] ], [ 8, [[ITER_CHECK27]] ], [ [[IND_END55]], [[VEC_EPILOG_ITER_CHECK43]] ]
-; CHECK-NEXT:    [[BC_RESUME_VAL67:%.*]] = phi i64 [ [[IND_END57]], [[VEC_EPILOG_MIDDLE_BLOCK40]] ], [ 0, [[ITER_CHECK27]] ], [ [[IND_END58]], [[VEC_EPILOG_ITER_CHECK43]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL56:%.*]] = phi i64 [ [[IND_END54]], [[VEC_EPILOG_MIDDLE_BLOCK40]] ], [ [[IND_END55]], [[VEC_EPILOG_ITER_CHECK43]] ], [ 8, [[ITER_CHECK27]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL67:%.*]] = phi i64 [ [[IND_END57]], [[VEC_EPILOG_MIDDLE_BLOCK40]] ], [ [[IND_END58]], [[VEC_EPILOG_ITER_CHECK43]] ], [ 0, [[ITER_CHECK27]] ]
 ; CHECK-NEXT:    br label [[FOR_BODY_US:%.*]]
 ; CHECK:       for.body.us:
 ; CHECK-NEXT:    [[INDVARS_IV78:%.*]] = phi i64 [ [[INDVARS_IV_NEXT79:%.*]], [[FOR_COND_CLEANUP4_US_LCSSA_US_US:%.*]] ], [ [[BC_RESUME_VAL56]], [[VEC_EPILOG_SCALAR_PH41]] ]
diff --git a/llvm/test/Transforms/LoopVectorize/X86/scev-checks-unprofitable.ll b/llvm/test/Transforms/LoopVectorize/X86/scev-checks-unprofitable.ll
new file mode 100644
index 0000000000000..868e07aa0490e
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/X86/scev-checks-unprofitable.ll
@@ -0,0 +1,110 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -p loop-vectorize -vectorizer-min-trip-count=8  -mcpu=skylake-avx512 -S %s | FileCheck %s
+
+target triple = "x86_64-unknown-linux-gnu"
+
+; Test case for https://github.com/llvm/llvm-project/issues/121518. Make sure
+; that we preserve LCSSA form when using %iv.1 from loop.1 in the trip count
+; expression when vectorizing loop.2
+define void @value_defined_in_loop1_used_for_trip_counts(i32 %start, i1 %c, ptr %dst) {
+; CHECK-LABEL: define void @value_defined_in_loop1_used_for_trip_counts(
+; CHECK-SAME: i32 [[START:%.*]], i1 [[C:%.*]], ptr [[DST:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    [[SELECT:%.*]] = select i1 [[C]], i32 0, i32 7
+; CHECK-NEXT:    [[ZEXT:%.*]] = zext i32 [[SELECT]] to i64
+; CHECK-NEXT:    br label %[[LOOP_1:.*]]
+; CHECK:       [[LOOP_1]]:
+; CHECK-NEXT:    [[IV_1:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[ZEXT]], %[[LOOP_1]] ]
+; CHECK-NEXT:    br i1 false, label %[[LOOP_1_EXIT:.*]], label %[[LOOP_1]]
+; CHECK:       [[LOOP_1_EXIT]]:
+; CHECK-NEXT:    [[IV_1_LCSSA2:%.*]] = phi i64 [ [[IV_1]], %[[LOOP_1]] ]
+; CHECK-NEXT:    [[IV_1_LCSSA:%.*]] = phi i64 [ [[IV_1]], %[[LOOP_1]] ]
+; CHECK-NEXT:    br i1 [[C]], label %[[LOOP_2_PREHEADER:.*]], label %[[LOOP_3_PREHEADER:.*]]
+; CHECK:       [[LOOP_3_PREHEADER]]:
+; CHECK-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK:       [[VECTOR_PH]]:
+; CHECK-NEXT:    [[N_RND_UP:%.*]] = add i64 [[IV_1_LCSSA2]], 15
+; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], 16
+; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
+; CHECK-NEXT:    [[TRIP_COUNT_MINUS_1:%.*]] = sub i64 [[IV_1_LCSSA2]], 1
+; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK:       [[VECTOR_BODY]]:
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <16 x i64> poison, i64 [[TRIP_COUNT_MINUS_1]], i64 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <16 x i64> [[BROADCAST_SPLATINSERT]], <16 x i64> poison, <16 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP0:%.*]] = icmp ule <16 x i64> <i64 0, i64 1, i64 2, i64 3, i64 4, i64 5, i64 6, i64 7, i64 8, i64 9, i64 10, i64 11, i64 12, i64 13, i64 14, i64 15>, [[BROADCAST_SPLAT]]
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr i8, ptr [[DST]], i64 0
+; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr i8, ptr [[TMP1]], i32 0
+; CHECK-NEXT:    call void @llvm.masked.store.v16i8.p0(<16 x i8> zeroinitializer, ptr [[TMP2]], i32 1, <16 x i1> [[TMP0]])
+; CHECK-NEXT:    br label %[[MIDDLE_BLOCK:.*]]
+; CHECK:       [[MIDDLE_BLOCK]]:
+; CHECK-NEXT:    br i1 true, label %[[EXIT_1_LOOPEXIT1:.*]], label %[[SCALAR_PH]]
+; CHECK:       [[SCALAR_PH]]:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[LOOP_3_PREHEADER]] ]
+; CHECK-NEXT:    br label %[[LOOP_3:.*]]
+; CHECK:       [[LOOP_2_PREHEADER]]:
+; CHECK-NEXT:    br label %[[LOOP_2:.*]]
+; CHECK:       [[LOOP_2]]:
+; CHECK-NEXT:    [[IV_2:%.*]] = phi i64 [ [[IV_2_NEXT:%.*]], %[[LOOP_2]] ], [ 0, %[[LOOP_2_PREHEADER]] ]
+; CHECK-NEXT:    [[IV_3:%.*]] = phi i32 [ [[IV_3_NEXT:%.*]], %[[LOOP_2]] ], [ [[START]], %[[LOOP_2_PREHEADER]] ]
+; CHECK-NEXT:    [[IV_3_NEXT]] = add i32 [[IV_3]], 1
+; CHECK-NEXT:    [[IV_2_NEXT]] = add i64 [[IV_2]], 1
+; CHECK-NEXT:    [[SHL:%.*]] = shl i32 [[IV_3]], 1
+; CHECK-NEXT:    [[ZEXT8:%.*]] = zext i32 [[SHL]] to i64
+; CHECK-NEXT:    [[GEP_DST:%.*]] = getelementptr i8, ptr [[DST]], i64 [[ZEXT8]]
+; CHECK-NEXT:    store i16 0, ptr [[GEP_DST]], align 2
+; CHECK-NEXT:    [[EC_2:%.*]] = icmp ult i64 [[IV_2]], [[IV_1_LCSSA]]
+; CHECK-NEXT:    br i1 [[EC_2]], label %[[LOOP_2]], label %[[EXIT_1_LOOPEXIT:.*]]
+; CHECK:       [[LOOP_3]]:
+; CHECK-NEXT:    [[IV_4:%.*]] = phi i64 [ [[IV_4_NEXT:%.*]], %[[LOOP_3]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
+; CHECK-NEXT:    [[GEP_DST_2:%.*]] = getelementptr i8, ptr [[DST]], i64 [[IV_4]]
+; CHECK-NEXT:    store i8 0, ptr [[GEP_DST_2]], align 1
+; CHECK-NEXT:    [[IV_4_NEXT]] = add i64 [[IV_4]], 1
+; CHECK-NEXT:    [[EC_3:%.*]] = icmp ult i64 [[IV_4_NEXT]], [[IV_1_LCSSA]]
+; CHECK-NEXT:    br i1 [[EC_3]], label %[[LOOP_3]], label %[[EXIT_1_LOOPEXIT1]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK:       [[EXIT_1_LOOPEXIT]]:
+; CHECK-NEXT:    br label %[[EXIT_1:.*]]
+; CHECK:       [[EXIT_1_LOOPEXIT1]]:
+; CHECK-NEXT:    br label %[[EXIT_1]]
+; CHECK:       [[EXIT_1]]:
+; CHECK-NEXT:    ret void
+;
+entry:
+  %select = select i1 %c, i32 0, i32 7
+  %zext = zext i32 %select to i64
+  br label %loop.1
+
+loop.1:
+  %iv.1 = phi i64 [ 0, %entry ], [ %zext, %loop.1 ]
+  br i1 false, label %loop.1.exit, label %loop.1
+
+loop.1.exit:
+  br i1 %c, label %loop.2, label %loop.3
+
+loop.2:
+  %iv.2 = phi i64 [ 0, %loop.1.exit ], [ %iv.2.next, %loop.2 ]
+  %iv.3 = phi i32 [ %start, %loop.1.exit ], [ %iv.3.next, %loop.2 ]
+  %iv.3.next = add i32 %iv.3, 1
+  %iv.2.next = add i64 %iv.2, 1
+  %shl = shl i32 %iv.3, 1
+  %zext8 = zext i32 %shl to i64
+  %gep.dst = getelementptr i8, ptr %dst, i64 %zext8
+  store i16 0, ptr %gep.dst, align 2
+  %ec.2 = icmp ult i64 %iv.2, %iv.1
+  br i1 %ec.2, label %loop.2, label %exit.1
+
+loop.3:
+  %iv.4 = phi i64 [ 0, %loop.1.exit ], [ %iv.4.next, %loop.3 ]
+  %gep.dst.2 = getelementptr i8, ptr %dst, i64 %iv.4
+  store i8 0, ptr %gep.dst.2, align 1
+  %iv.4.next = add i64 %iv.4, 1
+  %ec.3 = icmp ult i64 %iv.4.next, %iv.1
+  br i1 %ec.3, label %loop.3, label %exit.1
+
+exit.1:
+  ret void
+}
+;.
+; CHECK: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]], [[META2:![0-9]+]]}
+; CHECK: [[META1]] = !{!"llvm.loop.unroll.runtime.disable"}
+; CHECK: [[META2]] = !{!"llvm.loop.isvectorized", i32 1}
+;.
diff --git a/llvm/test/Transforms/LoopVectorize/X86/slm-no-vectorize.ll b/llvm/test/Transforms/LoopVectorize/X86/slm-no-vectorize.ll
index 71697ba727dd5..180a82badcaa1 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/slm-no-vectorize.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/slm-no-vectorize.ll
@@ -13,7 +13,7 @@ target triple = "x86_64-unknown-linux-gnu"
 
 define i32 @no_vec(i32 %LastIndex, ptr nocapture readonly %InputData, i16 signext %lag, i16 signext %Scale) {
 ; CHECK-LABEL: define i32 @no_vec
-; CHECK-SAME: (i32 [[LASTINDEX:%.*]], ptr nocapture readonly [[INPUTDATA:%.*]], i16 signext [[LAG:%.*]], i16 signext [[SCALE:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-SAME: (i32 [[LASTINDEX:%.*]], ptr readonly captures(none) [[INPUTDATA:%.*]], i16 signext [[LAG:%.*]], i16 signext [[SCALE:%.*]]) #[[ATTR0:[0-9]+]] {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[CMP17:%.*]] = icmp sgt i32 [[LASTINDEX]], 0
 ; CHECK-NEXT:    br i1 [[CMP17]], label [[FOR_BODY_LR_PH:%.*]], label [[FOR_COND_CLEANUP:%.*]]
diff --git a/llvm/test/Transforms/LoopVectorize/X86/strided_load_cost.ll b/llvm/test/Transforms/LoopVectorize/X86/strided_load_cost.ll
index d316befb9548d..ad8f1fb3ccd21 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/strided_load_cost.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/strided_load_cost.ll
@@ -222,8 +222,8 @@ define i32 @matrix_row_col(ptr nocapture readonly %data, i32 %i, i32 %j) local_u
 ; CHECK-NEXT:    [[TMP170:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP168]])
 ; CHECK-NEXT:    br i1 true, label [[FOR_COND_CLEANUP]], label [[SCALAR_PH]]
 ; CHECK:       vec.epilog.scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 100, [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ 0, [[ITER_CHECK:%.*]] ], [ 96, [[VEC_EPILOG_ITER_CHECK]] ]
-; CHECK-NEXT:    [[BC_MERGE_RDX13:%.*]] = phi i32 [ [[TMP170]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ 0, [[ITER_CHECK]] ], [ [[TMP149]], [[VEC_EPILOG_ITER_CHECK]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 100, [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ 96, [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[ITER_CHECK:%.*]] ]
+; CHECK-NEXT:    [[BC_MERGE_RDX13:%.*]] = phi i32 [ [[TMP170]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP149]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[ITER_CHECK]] ]
 ; CHECK-NEXT:    br label [[FOR_BODY1:%.*]]
 ; CHECK:       for.cond.cleanup:
 ; CHECK-NEXT:    [[ADD7_LCSSA:%.*]] = phi i32 [ [[ADD7:%.*]], [[FOR_BODY1]] ], [ [[TMP149]], [[MIDDLE_BLOCK]] ], [ [[TMP170]], [[VEC_EPILOG_MIDDLE_BLOCK]] ]
@@ -454,8 +454,8 @@ define i32 @matrix_row_col(ptr nocapture readonly %data, i32 %i, i32 %j) local_u
 ; MAX-BW-NEXT:    [[TMP170:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP168]])
 ; MAX-BW-NEXT:    br i1 true, label [[FOR_COND_CLEANUP]], label [[SCALAR_PH]]
 ; MAX-BW:       vec.epilog.scalar.ph:
-; MAX-BW-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 100, [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ 0, [[ITER_CHECK:%.*]] ], [ 96, [[VEC_EPILOG_ITER_CHECK]] ]
-; MAX-BW-NEXT:    [[BC_MERGE_RDX13:%.*]] = phi i32 [ [[TMP170]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ 0, [[ITER_CHECK]] ], [ [[TMP149]], [[VEC_EPILOG_ITER_CHECK]] ]
+; MAX-BW-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 100, [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ 96, [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[ITER_CHECK:%.*]] ]
+; MAX-BW-NEXT:    [[BC_MERGE_RDX13:%.*]] = phi i32 [ [[TMP170]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP149]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[ITER_CHECK]] ]
 ; MAX-BW-NEXT:    br label [[FOR_BODY1:%.*]]
 ; MAX-BW:       for.cond.cleanup:
 ; MAX-BW-NEXT:    [[ADD7_LCSSA:%.*]] = phi i32 [ [[ADD7:%.*]], [[FOR_BODY1]] ], [ [[TMP149]], [[MIDDLE_BLOCK]] ], [ [[TMP170]], [[VEC_EPILOG_MIDDLE_BLOCK]] ]
diff --git a/llvm/test/Transforms/LoopVectorize/X86/uniform_mem_op.ll b/llvm/test/Transforms/LoopVectorize/X86/uniform_mem_op.ll
index 26a9e179e3c53..45594b0335336 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/uniform_mem_op.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/uniform_mem_op.ll
@@ -299,7 +299,7 @@ define void @uniform_copy(ptr %A, ptr %B) {
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    br i1 false, label [[LOOPEXIT:%.*]], label [[SCALAR_PH]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 4096, [[MIDDLE_BLOCK]] ], [ 0, [[VECTOR_MEMCHECK]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 4096, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ], [ 0, [[VECTOR_MEMCHECK]] ]
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK:       for.body:
 ; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[IV_NEXT:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
diff --git a/llvm/test/Transforms/LoopVectorize/X86/vect.omp.force.small-tc.ll b/llvm/test/Transforms/LoopVectorize/X86/vect.omp.force.small-tc.ll
index eb52420949f8d..90ba702ed232e 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/vect.omp.force.small-tc.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/vect.omp.force.small-tc.ll
@@ -82,7 +82,7 @@ define void @vectorized(ptr noalias nocapture %A, ptr noalias nocapture readonly
 ; CHECK:       vec.epilog.middle.block:
 ; CHECK-NEXT:    br i1 true, label [[FOR_END]], label [[VEC_EPILOG_SCALAR_PH]]
 ; CHECK:       vec.epilog.scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 20, [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ 0, [[ITER_CHECK:%.*]] ], [ 16, [[VEC_EPILOG_ITER_CHECK]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 20, [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ 16, [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[ITER_CHECK:%.*]] ]
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK:       for.body:
 ; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[VEC_EPILOG_SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ]
diff --git a/llvm/test/Transforms/LoopVectorize/X86/vectorize-force-tail-with-evl.ll b/llvm/test/Transforms/LoopVectorize/X86/vectorize-force-tail-with-evl.ll
index a3b066ed82216..e37eae4c1f390 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/vectorize-force-tail-with-evl.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/vectorize-force-tail-with-evl.ll
@@ -141,7 +141,7 @@ define void @foo(ptr noalias %a, ptr noalias %b, ptr noalias %c, i64 %N) {
 ; NO-VP-NEXT:    [[CMP_N11:%.*]] = icmp eq i64 [[N]], [[N_VEC10]]
 ; NO-VP-NEXT:    br i1 [[CMP_N11]], label [[FOR_COND_CLEANUP]], label [[VEC_EPILOG_SCALAR_PH]]
 ; NO-VP:       vec.epilog.scalar.ph:
-; NO-VP-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC10]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ 0, [[ITER_CHECK:%.*]] ], [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ]
+; NO-VP-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC10]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[ITER_CHECK:%.*]] ]
 ; NO-VP-NEXT:    br label [[FOR_BODY:%.*]]
 ; NO-VP:       for.body:
 ; NO-VP-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[VEC_EPILOG_SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
diff --git a/llvm/test/Transforms/LoopVectorize/X86/x86-predication.ll b/llvm/test/Transforms/LoopVectorize/X86/x86-predication.ll
index 126bbf9afc34e..2ac30337a4fb9 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/x86-predication.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/x86-predication.ll
@@ -25,8 +25,7 @@ define i32 @predicated_sdiv_masked_load(ptr %a, ptr %b, i32 %x, i1 %c) {
 ; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr i32, ptr [[B:%.*]], i64 [[TMP0]]
 ; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr i32, ptr [[TMP3]], i32 0
 ; CHECK-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <2 x i32> @llvm.masked.load.v2i32.p0(ptr [[TMP4]], i32 4, <2 x i1> [[BROADCAST_SPLAT]], <2 x i32> poison)
-; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <2 x i1> [[BROADCAST_SPLAT]], i32 0
-; CHECK-NEXT:    br i1 [[TMP5]], label [[PRED_SDIV_IF:%.*]], label [[PRED_SDIV_CONTINUE:%.*]]
+; CHECK-NEXT:    br i1 [[C]], label [[PRED_SDIV_IF:%.*]], label [[PRED_SDIV_CONTINUE:%.*]]
 ; CHECK:       pred.sdiv.if:
 ; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <2 x i32> [[WIDE_MASKED_LOAD]], i32 0
 ; CHECK-NEXT:    [[TMP7:%.*]] = sdiv i32 [[TMP6]], [[X:%.*]]
@@ -34,8 +33,7 @@ define i32 @predicated_sdiv_masked_load(ptr %a, ptr %b, i32 %x, i1 %c) {
 ; CHECK-NEXT:    br label [[PRED_SDIV_CONTINUE]]
 ; CHECK:       pred.sdiv.continue:
 ; CHECK-NEXT:    [[TMP9:%.*]] = phi <2 x i32> [ poison, [[VECTOR_BODY]] ], [ [[TMP8]], [[PRED_SDIV_IF]] ]
-; CHECK-NEXT:    [[TMP10:%.*]] = extractelement <2 x i1> [[BROADCAST_SPLAT]], i32 1
-; CHECK-NEXT:    br i1 [[TMP10]], label [[PRED_SDIV_IF1:%.*]], label [[PRED_SDIV_CONTINUE2]]
+; CHECK-NEXT:    br i1 [[C]], label [[PRED_SDIV_IF1:%.*]], label [[PRED_SDIV_CONTINUE2]]
 ; CHECK:       pred.sdiv.if1:
 ; CHECK-NEXT:    [[TMP11:%.*]] = extractelement <2 x i32> [[WIDE_MASKED_LOAD]], i32 1
 ; CHECK-NEXT:    [[TMP12:%.*]] = sdiv i32 [[TMP11]], [[X]]
@@ -70,8 +68,7 @@ define i32 @predicated_sdiv_masked_load(ptr %a, ptr %b, i32 %x, i1 %c) {
 ; SINK-GATHER-NEXT:    [[TMP3:%.*]] = getelementptr i32, ptr [[B:%.*]], i64 [[TMP0]]
 ; SINK-GATHER-NEXT:    [[TMP4:%.*]] = getelementptr i32, ptr [[TMP3]], i32 0
 ; SINK-GATHER-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <8 x i32> @llvm.masked.load.v8i32.p0(ptr [[TMP4]], i32 4, <8 x i1> [[BROADCAST_SPLAT]], <8 x i32> poison)
-; SINK-GATHER-NEXT:    [[TMP5:%.*]] = extractelement <8 x i1> [[BROADCAST_SPLAT]], i32 0
-; SINK-GATHER-NEXT:    br i1 [[TMP5]], label [[PRED_SDIV_IF:%.*]], label [[PRED_SDIV_CONTINUE:%.*]]
+; SINK-GATHER-NEXT:    br i1 [[C]], label [[PRED_SDIV_IF:%.*]], label [[PRED_SDIV_CONTINUE:%.*]]
 ; SINK-GATHER:       pred.sdiv.if:
 ; SINK-GATHER-NEXT:    [[TMP6:%.*]] = extractelement <8 x i32> [[WIDE_MASKED_LOAD]], i32 0
 ; SINK-GATHER-NEXT:    [[TMP7:%.*]] = sdiv i32 [[TMP6]], [[X:%.*]]
@@ -79,8 +76,7 @@ define i32 @predicated_sdiv_masked_load(ptr %a, ptr %b, i32 %x, i1 %c) {
 ; SINK-GATHER-NEXT:    br label [[PRED_SDIV_CONTINUE]]
 ; SINK-GATHER:       pred.sdiv.continue:
 ; SINK-GATHER-NEXT:    [[TMP9:%.*]] = phi <8 x i32> [ poison, [[VECTOR_BODY]] ], [ [[TMP8]], [[PRED_SDIV_IF]] ]
-; SINK-GATHER-NEXT:    [[TMP10:%.*]] = extractelement <8 x i1> [[BROADCAST_SPLAT]], i32 1
-; SINK-GATHER-NEXT:    br i1 [[TMP10]], label [[PRED_SDIV_IF1:%.*]], label [[PRED_SDIV_CONTINUE2:%.*]]
+; SINK-GATHER-NEXT:    br i1 [[C]], label [[PRED_SDIV_IF1:%.*]], label [[PRED_SDIV_CONTINUE2:%.*]]
 ; SINK-GATHER:       pred.sdiv.if1:
 ; SINK-GATHER-NEXT:    [[TMP11:%.*]] = extractelement <8 x i32> [[WIDE_MASKED_LOAD]], i32 1
 ; SINK-GATHER-NEXT:    [[TMP12:%.*]] = sdiv i32 [[TMP11]], [[X]]
@@ -88,8 +84,7 @@ define i32 @predicated_sdiv_masked_load(ptr %a, ptr %b, i32 %x, i1 %c) {
 ; SINK-GATHER-NEXT:    br label [[PRED_SDIV_CONTINUE2]]
 ; SINK-GATHER:       pred.sdiv.continue2:
 ; SINK-GATHER-NEXT:    [[TMP14:%.*]] = phi <8 x i32> [ [[TMP9]], [[PRED_SDIV_CONTINUE]] ], [ [[TMP13]], [[PRED_SDIV_IF1]] ]
-; SINK-GATHER-NEXT:    [[TMP15:%.*]] = extractelement <8 x i1> [[BROADCAST_SPLAT]], i32 2
-; SINK-GATHER-NEXT:    br i1 [[TMP15]], label [[PRED_SDIV_IF3:%.*]], label [[PRED_SDIV_CONTINUE4:%.*]]
+; SINK-GATHER-NEXT:    br i1 [[C]], label [[PRED_SDIV_IF3:%.*]], label [[PRED_SDIV_CONTINUE4:%.*]]
 ; SINK-GATHER:       pred.sdiv.if3:
 ; SINK-GATHER-NEXT:    [[TMP16:%.*]] = extractelement <8 x i32> [[WIDE_MASKED_LOAD]], i32 2
 ; SINK-GATHER-NEXT:    [[TMP17:%.*]] = sdiv i32 [[TMP16]], [[X]]
@@ -97,8 +92,7 @@ define i32 @predicated_sdiv_masked_load(ptr %a, ptr %b, i32 %x, i1 %c) {
 ; SINK-GATHER-NEXT:    br label [[PRED_SDIV_CONTINUE4]]
 ; SINK-GATHER:       pred.sdiv.continue4:
 ; SINK-GATHER-NEXT:    [[TMP19:%.*]] = phi <8 x i32> [ [[TMP14]], [[PRED_SDIV_CONTINUE2]] ], [ [[TMP18]], [[PRED_SDIV_IF3]] ]
-; SINK-GATHER-NEXT:    [[TMP20:%.*]] = extractelement <8 x i1> [[BROADCAST_SPLAT]], i32 3
-; SINK-GATHER-NEXT:    br i1 [[TMP20]], label [[PRED_SDIV_IF5:%.*]], label [[PRED_SDIV_CONTINUE6:%.*]]
+; SINK-GATHER-NEXT:    br i1 [[C]], label [[PRED_SDIV_IF5:%.*]], label [[PRED_SDIV_CONTINUE6:%.*]]
 ; SINK-GATHER:       pred.sdiv.if5:
 ; SINK-GATHER-NEXT:    [[TMP21:%.*]] = extractelement <8 x i32> [[WIDE_MASKED_LOAD]], i32 3
 ; SINK-GATHER-NEXT:    [[TMP22:%.*]] = sdiv i32 [[TMP21]], [[X]]
@@ -106,8 +100,7 @@ define i32 @predicated_sdiv_masked_load(ptr %a, ptr %b, i32 %x, i1 %c) {
 ; SINK-GATHER-NEXT:    br label [[PRED_SDIV_CONTINUE6]]
 ; SINK-GATHER:       pred.sdiv.continue6:
 ; SINK-GATHER-NEXT:    [[TMP24:%.*]] = phi <8 x i32> [ [[TMP19]], [[PRED_SDIV_CONTINUE4]] ], [ [[TMP23]], [[PRED_SDIV_IF5]] ]
-; SINK-GATHER-NEXT:    [[TMP25:%.*]] = extractelement <8 x i1> [[BROADCAST_SPLAT]], i32 4
-; SINK-GATHER-NEXT:    br i1 [[TMP25]], label [[PRED_SDIV_IF7:%.*]], label [[PRED_SDIV_CONTINUE8:%.*]]
+; SINK-GATHER-NEXT:    br i1 [[C]], label [[PRED_SDIV_IF7:%.*]], label [[PRED_SDIV_CONTINUE8:%.*]]
 ; SINK-GATHER:       pred.sdiv.if7:
 ; SINK-GATHER-NEXT:    [[TMP26:%.*]] = extractelement <8 x i32> [[WIDE_MASKED_LOAD]], i32 4
 ; SINK-GATHER-NEXT:    [[TMP27:%.*]] = sdiv i32 [[TMP26]], [[X]]
@@ -115,8 +108,7 @@ define i32 @predicated_sdiv_masked_load(ptr %a, ptr %b, i32 %x, i1 %c) {
 ; SINK-GATHER-NEXT:    br label [[PRED_SDIV_CONTINUE8]]
 ; SINK-GATHER:       pred.sdiv.continue8:
 ; SINK-GATHER-NEXT:    [[TMP29:%.*]] = phi <8 x i32> [ [[TMP24]], [[PRED_SDIV_CONTINUE6]] ], [ [[TMP28]], [[PRED_SDIV_IF7]] ]
-; SINK-GATHER-NEXT:    [[TMP30:%.*]] = extractelement <8 x i1> [[BROADCAST_SPLAT]], i32 5
-; SINK-GATHER-NEXT:    br i1 [[TMP30]], label [[PRED_SDIV_IF9:%.*]], label [[PRED_SDIV_CONTINUE10:%.*]]
+; SINK-GATHER-NEXT:    br i1 [[C]], label [[PRED_SDIV_IF9:%.*]], label [[PRED_SDIV_CONTINUE10:%.*]]
 ; SINK-GATHER:       pred.sdiv.if9:
 ; SINK-GATHER-NEXT:    [[TMP31:%.*]] = extractelement <8 x i32> [[WIDE_MASKED_LOAD]], i32 5
 ; SINK-GATHER-NEXT:    [[TMP32:%.*]] = sdiv i32 [[TMP31]], [[X]]
@@ -124,8 +116,7 @@ define i32 @predicated_sdiv_masked_load(ptr %a, ptr %b, i32 %x, i1 %c) {
 ; SINK-GATHER-NEXT:    br label [[PRED_SDIV_CONTINUE10]]
 ; SINK-GATHER:       pred.sdiv.continue10:
 ; SINK-GATHER-NEXT:    [[TMP34:%.*]] = phi <8 x i32> [ [[TMP29]], [[PRED_SDIV_CONTINUE8]] ], [ [[TMP33]], [[PRED_SDIV_IF9]] ]
-; SINK-GATHER-NEXT:    [[TMP35:%.*]] = extractelement <8 x i1> [[BROADCAST_SPLAT]], i32 6
-; SINK-GATHER-NEXT:    br i1 [[TMP35]], label [[PRED_SDIV_IF11:%.*]], label [[PRED_SDIV_CONTINUE12:%.*]]
+; SINK-GATHER-NEXT:    br i1 [[C]], label [[PRED_SDIV_IF11:%.*]], label [[PRED_SDIV_CONTINUE12:%.*]]
 ; SINK-GATHER:       pred.sdiv.if11:
 ; SINK-GATHER-NEXT:    [[TMP36:%.*]] = extractelement <8 x i32> [[WIDE_MASKED_LOAD]], i32 6
 ; SINK-GATHER-NEXT:    [[TMP37:%.*]] = sdiv i32 [[TMP36]], [[X]]
@@ -133,8 +124,7 @@ define i32 @predicated_sdiv_masked_load(ptr %a, ptr %b, i32 %x, i1 %c) {
 ; SINK-GATHER-NEXT:    br label [[PRED_SDIV_CONTINUE12]]
 ; SINK-GATHER:       pred.sdiv.continue12:
 ; SINK-GATHER-NEXT:    [[TMP39:%.*]] = phi <8 x i32> [ [[TMP34]], [[PRED_SDIV_CONTINUE10]] ], [ [[TMP38]], [[PRED_SDIV_IF11]] ]
-; SINK-GATHER-NEXT:    [[TMP40:%.*]] = extractelement <8 x i1> [[BROADCAST_SPLAT]], i32 7
-; SINK-GATHER-NEXT:    br i1 [[TMP40]], label [[PRED_SDIV_IF13:%.*]], label [[PRED_SDIV_CONTINUE14]]
+; SINK-GATHER-NEXT:    br i1 [[C]], label [[PRED_SDIV_IF13:%.*]], label [[PRED_SDIV_CONTINUE14]]
 ; SINK-GATHER:       pred.sdiv.if13:
 ; SINK-GATHER-NEXT:    [[TMP41:%.*]] = extractelement <8 x i32> [[WIDE_MASKED_LOAD]], i32 7
 ; SINK-GATHER-NEXT:    [[TMP42:%.*]] = sdiv i32 [[TMP41]], [[X]]
@@ -218,17 +208,16 @@ define i32 @scalarize_and_sink_gather(ptr %a, i1 %c, i32 %x, i64 %n) {
 ; CHECK:       vector.ph:
 ; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[SMAX]], 2
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[SMAX]], [[N_MOD_VF]]
-; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x i1> poison, i1 [[C:%.*]], i64 0
-; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <2 x i1> [[BROADCAST_SPLATINSERT]], <2 x i1> poison, <2 x i32> zeroinitializer
 ; CHECK-NEXT:    [[BROADCAST_SPLATINSERT3:%.*]] = insertelement <2 x i32> poison, i32 [[X:%.*]], i64 0
 ; CHECK-NEXT:    [[BROADCAST_SPLAT4:%.*]] = shufflevector <2 x i32> [[BROADCAST_SPLATINSERT3]], <2 x i32> poison, <2 x i32> zeroinitializer
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT4:%.*]] = insertelement <2 x i1> poison, i1 [[TMP1:%.*]], i64 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <2 x i1> [[BROADCAST_SPLATINSERT4]], <2 x i1> poison, <2 x i32> zeroinitializer
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_UDIV_CONTINUE2:%.*]] ]
 ; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <2 x i64> [ <i64 0, i64 1>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[PRED_UDIV_CONTINUE2]] ]
 ; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <2 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP18:%.*]], [[PRED_UDIV_CONTINUE2]] ]
 ; CHECK-NEXT:    [[TMP0:%.*]] = mul <2 x i64> [[VEC_IND]], splat (i64 777)
-; CHECK-NEXT:    [[TMP1:%.*]] = extractelement <2 x i1> [[BROADCAST_SPLAT]], i32 0
 ; CHECK-NEXT:    br i1 [[TMP1]], label [[PRED_UDIV_IF:%.*]], label [[PRED_UDIV_CONTINUE:%.*]]
 ; CHECK:       pred.udiv.if:
 ; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <2 x i64> [[TMP0]], i32 0
@@ -239,8 +228,7 @@ define i32 @scalarize_and_sink_gather(ptr %a, i1 %c, i32 %x, i64 %n) {
 ; CHECK-NEXT:    br label [[PRED_UDIV_CONTINUE]]
 ; CHECK:       pred.udiv.continue:
 ; CHECK-NEXT:    [[TMP8:%.*]] = phi <2 x i32> [ poison, [[VECTOR_BODY]] ], [ [[TMP6]], [[PRED_UDIV_IF]] ]
-; CHECK-NEXT:    [[TMP9:%.*]] = extractelement <2 x i1> [[BROADCAST_SPLAT]], i32 1
-; CHECK-NEXT:    br i1 [[TMP9]], label [[PRED_UDIV_IF1:%.*]], label [[PRED_UDIV_CONTINUE2]]
+; CHECK-NEXT:    br i1 [[TMP1]], label [[PRED_UDIV_IF1:%.*]], label [[PRED_UDIV_CONTINUE2]]
 ; CHECK:       pred.udiv.if1:
 ; CHECK-NEXT:    [[TMP10:%.*]] = extractelement <2 x i64> [[TMP0]], i32 1
 ; CHECK-NEXT:    [[TMP11:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP10]]
@@ -268,7 +256,7 @@ define i32 @scalarize_and_sink_gather(ptr %a, i1 %c, i32 %x, i64 %n) {
 ; CHECK-NEXT:    [[I:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[I_NEXT:%.*]], [[FOR_INC:%.*]] ]
 ; CHECK-NEXT:    [[R:%.*]] = phi i32 [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[T6:%.*]], [[FOR_INC]] ]
 ; CHECK-NEXT:    [[I7:%.*]] = mul i64 [[I]], 777
-; CHECK-NEXT:    br i1 [[C]], label [[IF_THEN:%.*]], label [[FOR_INC]]
+; CHECK-NEXT:    br i1 [[TMP1]], label [[IF_THEN:%.*]], label [[FOR_INC]]
 ; CHECK:       if.then:
 ; CHECK-NEXT:    [[T0:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[I7]]
 ; CHECK-NEXT:    [[T2:%.*]] = load i32, ptr [[T0]], align 4
@@ -292,17 +280,16 @@ define i32 @scalarize_and_sink_gather(ptr %a, i1 %c, i32 %x, i64 %n) {
 ; SINK-GATHER:       vector.ph:
 ; SINK-GATHER-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[SMAX]], 8
 ; SINK-GATHER-NEXT:    [[N_VEC:%.*]] = sub i64 [[SMAX]], [[N_MOD_VF]]
-; SINK-GATHER-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <8 x i1> poison, i1 [[C:%.*]], i64 0
-; SINK-GATHER-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <8 x i1> [[BROADCAST_SPLATINSERT]], <8 x i1> poison, <8 x i32> zeroinitializer
 ; SINK-GATHER-NEXT:    [[BROADCAST_SPLATINSERT15:%.*]] = insertelement <8 x i32> poison, i32 [[X:%.*]], i64 0
 ; SINK-GATHER-NEXT:    [[BROADCAST_SPLAT16:%.*]] = shufflevector <8 x i32> [[BROADCAST_SPLATINSERT15]], <8 x i32> poison, <8 x i32> zeroinitializer
+; SINK-GATHER-NEXT:    [[BROADCAST_SPLATINSERT16:%.*]] = insertelement <8 x i1> poison, i1 [[TMP1:%.*]], i64 0
+; SINK-GATHER-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <8 x i1> [[BROADCAST_SPLATINSERT16]], <8 x i1> poison, <8 x i32> zeroinitializer
 ; SINK-GATHER-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; SINK-GATHER:       vector.body:
 ; SINK-GATHER-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_UDIV_CONTINUE14:%.*]] ]
 ; SINK-GATHER-NEXT:    [[VEC_IND:%.*]] = phi <8 x i64> [ <i64 0, i64 1, i64 2, i64 3, i64 4, i64 5, i64 6, i64 7>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[PRED_UDIV_CONTINUE14]] ]
 ; SINK-GATHER-NEXT:    [[VEC_PHI:%.*]] = phi <8 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP66:%.*]], [[PRED_UDIV_CONTINUE14]] ]
 ; SINK-GATHER-NEXT:    [[TMP0:%.*]] = mul <8 x i64> [[VEC_IND]], splat (i64 777)
-; SINK-GATHER-NEXT:    [[TMP1:%.*]] = extractelement <8 x i1> [[BROADCAST_SPLAT]], i32 0
 ; SINK-GATHER-NEXT:    br i1 [[TMP1]], label [[PRED_UDIV_IF:%.*]], label [[PRED_UDIV_CONTINUE:%.*]]
 ; SINK-GATHER:       pred.udiv.if:
 ; SINK-GATHER-NEXT:    [[TMP2:%.*]] = extractelement <8 x i64> [[TMP0]], i32 0
@@ -313,8 +300,7 @@ define i32 @scalarize_and_sink_gather(ptr %a, i1 %c, i32 %x, i64 %n) {
 ; SINK-GATHER-NEXT:    br label [[PRED_UDIV_CONTINUE]]
 ; SINK-GATHER:       pred.udiv.continue:
 ; SINK-GATHER-NEXT:    [[TMP8:%.*]] = phi <8 x i32> [ poison, [[VECTOR_BODY]] ], [ [[TMP6]], [[PRED_UDIV_IF]] ]
-; SINK-GATHER-NEXT:    [[TMP9:%.*]] = extractelement <8 x i1> [[BROADCAST_SPLAT]], i32 1
-; SINK-GATHER-NEXT:    br i1 [[TMP9]], label [[PRED_UDIV_IF1:%.*]], label [[PRED_UDIV_CONTINUE2:%.*]]
+; SINK-GATHER-NEXT:    br i1 [[TMP1]], label [[PRED_UDIV_IF1:%.*]], label [[PRED_UDIV_CONTINUE2:%.*]]
 ; SINK-GATHER:       pred.udiv.if1:
 ; SINK-GATHER-NEXT:    [[TMP10:%.*]] = extractelement <8 x i64> [[TMP0]], i32 1
 ; SINK-GATHER-NEXT:    [[TMP11:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP10]]
@@ -324,8 +310,7 @@ define i32 @scalarize_and_sink_gather(ptr %a, i1 %c, i32 %x, i64 %n) {
 ; SINK-GATHER-NEXT:    br label [[PRED_UDIV_CONTINUE2]]
 ; SINK-GATHER:       pred.udiv.continue2:
 ; SINK-GATHER-NEXT:    [[TMP16:%.*]] = phi <8 x i32> [ [[TMP8]], [[PRED_UDIV_CONTINUE]] ], [ [[TMP14]], [[PRED_UDIV_IF1]] ]
-; SINK-GATHER-NEXT:    [[TMP17:%.*]] = extractelement <8 x i1> [[BROADCAST_SPLAT]], i32 2
-; SINK-GATHER-NEXT:    br i1 [[TMP17]], label [[PRED_UDIV_IF3:%.*]], label [[PRED_UDIV_CONTINUE4:%.*]]
+; SINK-GATHER-NEXT:    br i1 [[TMP1]], label [[PRED_UDIV_IF3:%.*]], label [[PRED_UDIV_CONTINUE4:%.*]]
 ; SINK-GATHER:       pred.udiv.if3:
 ; SINK-GATHER-NEXT:    [[TMP18:%.*]] = extractelement <8 x i64> [[TMP0]], i32 2
 ; SINK-GATHER-NEXT:    [[TMP19:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP18]]
@@ -335,8 +320,7 @@ define i32 @scalarize_and_sink_gather(ptr %a, i1 %c, i32 %x, i64 %n) {
 ; SINK-GATHER-NEXT:    br label [[PRED_UDIV_CONTINUE4]]
 ; SINK-GATHER:       pred.udiv.continue4:
 ; SINK-GATHER-NEXT:    [[TMP24:%.*]] = phi <8 x i32> [ [[TMP16]], [[PRED_UDIV_CONTINUE2]] ], [ [[TMP22]], [[PRED_UDIV_IF3]] ]
-; SINK-GATHER-NEXT:    [[TMP25:%.*]] = extractelement <8 x i1> [[BROADCAST_SPLAT]], i32 3
-; SINK-GATHER-NEXT:    br i1 [[TMP25]], label [[PRED_UDIV_IF5:%.*]], label [[PRED_UDIV_CONTINUE6:%.*]]
+; SINK-GATHER-NEXT:    br i1 [[TMP1]], label [[PRED_UDIV_IF5:%.*]], label [[PRED_UDIV_CONTINUE6:%.*]]
 ; SINK-GATHER:       pred.udiv.if5:
 ; SINK-GATHER-NEXT:    [[TMP26:%.*]] = extractelement <8 x i64> [[TMP0]], i32 3
 ; SINK-GATHER-NEXT:    [[TMP27:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP26]]
@@ -346,8 +330,7 @@ define i32 @scalarize_and_sink_gather(ptr %a, i1 %c, i32 %x, i64 %n) {
 ; SINK-GATHER-NEXT:    br label [[PRED_UDIV_CONTINUE6]]
 ; SINK-GATHER:       pred.udiv.continue6:
 ; SINK-GATHER-NEXT:    [[TMP32:%.*]] = phi <8 x i32> [ [[TMP24]], [[PRED_UDIV_CONTINUE4]] ], [ [[TMP30]], [[PRED_UDIV_IF5]] ]
-; SINK-GATHER-NEXT:    [[TMP33:%.*]] = extractelement <8 x i1> [[BROADCAST_SPLAT]], i32 4
-; SINK-GATHER-NEXT:    br i1 [[TMP33]], label [[PRED_UDIV_IF7:%.*]], label [[PRED_UDIV_CONTINUE8:%.*]]
+; SINK-GATHER-NEXT:    br i1 [[TMP1]], label [[PRED_UDIV_IF7:%.*]], label [[PRED_UDIV_CONTINUE8:%.*]]
 ; SINK-GATHER:       pred.udiv.if7:
 ; SINK-GATHER-NEXT:    [[TMP34:%.*]] = extractelement <8 x i64> [[TMP0]], i32 4
 ; SINK-GATHER-NEXT:    [[TMP35:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP34]]
@@ -357,8 +340,7 @@ define i32 @scalarize_and_sink_gather(ptr %a, i1 %c, i32 %x, i64 %n) {
 ; SINK-GATHER-NEXT:    br label [[PRED_UDIV_CONTINUE8]]
 ; SINK-GATHER:       pred.udiv.continue8:
 ; SINK-GATHER-NEXT:    [[TMP40:%.*]] = phi <8 x i32> [ [[TMP32]], [[PRED_UDIV_CONTINUE6]] ], [ [[TMP38]], [[PRED_UDIV_IF7]] ]
-; SINK-GATHER-NEXT:    [[TMP41:%.*]] = extractelement <8 x i1> [[BROADCAST_SPLAT]], i32 5
-; SINK-GATHER-NEXT:    br i1 [[TMP41]], label [[PRED_UDIV_IF9:%.*]], label [[PRED_UDIV_CONTINUE10:%.*]]
+; SINK-GATHER-NEXT:    br i1 [[TMP1]], label [[PRED_UDIV_IF9:%.*]], label [[PRED_UDIV_CONTINUE10:%.*]]
 ; SINK-GATHER:       pred.udiv.if9:
 ; SINK-GATHER-NEXT:    [[TMP42:%.*]] = extractelement <8 x i64> [[TMP0]], i32 5
 ; SINK-GATHER-NEXT:    [[TMP43:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP42]]
@@ -368,8 +350,7 @@ define i32 @scalarize_and_sink_gather(ptr %a, i1 %c, i32 %x, i64 %n) {
 ; SINK-GATHER-NEXT:    br label [[PRED_UDIV_CONTINUE10]]
 ; SINK-GATHER:       pred.udiv.continue10:
 ; SINK-GATHER-NEXT:    [[TMP48:%.*]] = phi <8 x i32> [ [[TMP40]], [[PRED_UDIV_CONTINUE8]] ], [ [[TMP46]], [[PRED_UDIV_IF9]] ]
-; SINK-GATHER-NEXT:    [[TMP49:%.*]] = extractelement <8 x i1> [[BROADCAST_SPLAT]], i32 6
-; SINK-GATHER-NEXT:    br i1 [[TMP49]], label [[PRED_UDIV_IF11:%.*]], label [[PRED_UDIV_CONTINUE12:%.*]]
+; SINK-GATHER-NEXT:    br i1 [[TMP1]], label [[PRED_UDIV_IF11:%.*]], label [[PRED_UDIV_CONTINUE12:%.*]]
 ; SINK-GATHER:       pred.udiv.if11:
 ; SINK-GATHER-NEXT:    [[TMP50:%.*]] = extractelement <8 x i64> [[TMP0]], i32 6
 ; SINK-GATHER-NEXT:    [[TMP51:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP50]]
@@ -379,8 +360,7 @@ define i32 @scalarize_and_sink_gather(ptr %a, i1 %c, i32 %x, i64 %n) {
 ; SINK-GATHER-NEXT:    br label [[PRED_UDIV_CONTINUE12]]
 ; SINK-GATHER:       pred.udiv.continue12:
 ; SINK-GATHER-NEXT:    [[TMP56:%.*]] = phi <8 x i32> [ [[TMP48]], [[PRED_UDIV_CONTINUE10]] ], [ [[TMP54]], [[PRED_UDIV_IF11]] ]
-; SINK-GATHER-NEXT:    [[TMP57:%.*]] = extractelement <8 x i1> [[BROADCAST_SPLAT]], i32 7
-; SINK-GATHER-NEXT:    br i1 [[TMP57]], label [[PRED_UDIV_IF13:%.*]], label [[PRED_UDIV_CONTINUE14]]
+; SINK-GATHER-NEXT:    br i1 [[TMP1]], label [[PRED_UDIV_IF13:%.*]], label [[PRED_UDIV_CONTINUE14]]
 ; SINK-GATHER:       pred.udiv.if13:
 ; SINK-GATHER-NEXT:    [[TMP58:%.*]] = extractelement <8 x i64> [[TMP0]], i32 7
 ; SINK-GATHER-NEXT:    [[TMP59:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP58]]
@@ -408,7 +388,7 @@ define i32 @scalarize_and_sink_gather(ptr %a, i1 %c, i32 %x, i64 %n) {
 ; SINK-GATHER-NEXT:    [[I:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[I_NEXT:%.*]], [[FOR_INC:%.*]] ]
 ; SINK-GATHER-NEXT:    [[R:%.*]] = phi i32 [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[T6:%.*]], [[FOR_INC]] ]
 ; SINK-GATHER-NEXT:    [[I7:%.*]] = mul i64 [[I]], 777
-; SINK-GATHER-NEXT:    br i1 [[C]], label [[IF_THEN:%.*]], label [[FOR_INC]]
+; SINK-GATHER-NEXT:    br i1 [[TMP1]], label [[IF_THEN:%.*]], label [[FOR_INC]]
 ; SINK-GATHER:       if.then:
 ; SINK-GATHER-NEXT:    [[T0:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[I7]]
 ; SINK-GATHER-NEXT:    [[T2:%.*]] = load i32, ptr [[T0]], align 4
diff --git a/llvm/test/Transforms/LoopVectorize/dead_instructions.ll b/llvm/test/Transforms/LoopVectorize/dead_instructions.ll
index d19fe8416200e..e1fd07bf590c4 100644
--- a/llvm/test/Transforms/LoopVectorize/dead_instructions.ll
+++ b/llvm/test/Transforms/LoopVectorize/dead_instructions.ll
@@ -165,7 +165,7 @@ define void @dead_load_and_vector_pointer(ptr %a, ptr %b) {
 ; CHECK:       [[MIDDLE_BLOCK]]:
 ; CHECK-NEXT:    br i1 false, label %[[EXIT:.*]], label %[[SCALAR_PH]]
 ; CHECK:       [[SCALAR_PH]]:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ 128, %[[MIDDLE_BLOCK]] ], [ 0, %[[VECTOR_MEMCHECK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ 128, %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_MEMCHECK]] ]
 ; CHECK-NEXT:    br label %[[LOOP:.*]]
 ; CHECK:       [[EXIT]]:
 ; CHECK-NEXT:    ret void
diff --git a/llvm/test/Transforms/LoopVectorize/early_exit_legality.ll b/llvm/test/Transforms/LoopVectorize/early_exit_legality.ll
index 6d365b9d77e80..de455c81d363e 100644
--- a/llvm/test/Transforms/LoopVectorize/early_exit_legality.ll
+++ b/llvm/test/Transforms/LoopVectorize/early_exit_legality.ll
@@ -49,7 +49,7 @@ define i64 @same_exit_block_pre_inc_use1() {
 ; CHECK-LABEL: LV: Checking a loop in 'same_exit_block_pre_inc_use1'
 ; CHECK:       LV: Found an early exit loop with symbolic max backedge taken count: 63
 ; CHECK-NEXT:  LV: We can vectorize this loop!
-; CHECK:  LV: Not vectorizing: Some exit values in loop with uncountable exit not supported yet.
+; CHECK-NOT:   LV: Not vectorizing
 entry:
   %p1 = alloca [1024 x i8]
   %p2 = alloca [1024 x i8]
@@ -141,7 +141,7 @@ define i64 @loop_contains_load_after_early_exit(ptr dereferenceable(1024) align(
 ; CHECK-LABEL: LV: Checking a loop in 'loop_contains_load_after_early_exit'
 ; CHECK:       LV: Found an early exit loop with symbolic max backedge taken count: 63
 ; CHECK-NEXT:  LV: We can vectorize this loop!
-; CHECK:       LV: Not vectorizing: Some exit values in loop with uncountable exit not supported yet.
+; CHECK-NOT:   LV: Not vectorizing
 entry:
   %p1 = alloca [1024 x i8]
   call void @init_mem(ptr %p1, i64 1024)
diff --git a/llvm/test/Transforms/LoopVectorize/epilog-iv-select-cmp.ll b/llvm/test/Transforms/LoopVectorize/epilog-iv-select-cmp.ll
index 06f0f05889116..5437c54409cd5 100644
--- a/llvm/test/Transforms/LoopVectorize/epilog-iv-select-cmp.ll
+++ b/llvm/test/Transforms/LoopVectorize/epilog-iv-select-cmp.ll
@@ -72,8 +72,8 @@ define i64 @select_icmp_const(ptr %a, i64 %n) {
 ; CHECK-NEXT:    [[CMP_N12:%.*]] = icmp eq i64 [[N]], [[N_VEC3]]
 ; CHECK-NEXT:    br i1 [[CMP_N12]], label %[[EXIT]], label %[[VEC_EPILOG_SCALAR_PH]]
 ; CHECK:       [[VEC_EPILOG_SCALAR_PH]]:
-; CHECK-NEXT:    [[BC_RESUME_VAL15:%.*]] = phi i64 [ [[N_VEC3]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ 0, %[[ITER_CHECK]] ], [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ]
-; CHECK-NEXT:    [[BC_MERGE_RDX16:%.*]] = phi i64 [ [[RDX_SELECT11]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ 3, %[[ITER_CHECK]] ], [ [[RDX_SELECT]], %[[VEC_EPILOG_ITER_CHECK]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL15:%.*]] = phi i64 [ [[N_VEC3]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ]
+; CHECK-NEXT:    [[BC_MERGE_RDX16:%.*]] = phi i64 [ [[RDX_SELECT11]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[RDX_SELECT]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 3, %[[ITER_CHECK]] ]
 ; CHECK-NEXT:    br label %[[LOOP:.*]]
 ; CHECK:       [[LOOP]]:
 ; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL15]], %[[VEC_EPILOG_SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
@@ -178,8 +178,8 @@ define i64 @select_fcmp_const_fast(ptr %a, i64 %n) {
 ; CHECK-NEXT:    [[CMP_N12:%.*]] = icmp eq i64 [[N]], [[N_VEC3]]
 ; CHECK-NEXT:    br i1 [[CMP_N12]], label %[[EXIT]], label %[[VEC_EPILOG_SCALAR_PH]]
 ; CHECK:       [[VEC_EPILOG_SCALAR_PH]]:
-; CHECK-NEXT:    [[BC_RESUME_VAL15:%.*]] = phi i64 [ [[N_VEC3]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ 0, %[[ITER_CHECK]] ], [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ]
-; CHECK-NEXT:    [[BC_MERGE_RDX16:%.*]] = phi i64 [ [[RDX_SELECT11]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ 2, %[[ITER_CHECK]] ], [ [[RDX_SELECT]], %[[VEC_EPILOG_ITER_CHECK]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL15:%.*]] = phi i64 [ [[N_VEC3]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ]
+; CHECK-NEXT:    [[BC_MERGE_RDX16:%.*]] = phi i64 [ [[RDX_SELECT11]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[RDX_SELECT]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 2, %[[ITER_CHECK]] ]
 ; CHECK-NEXT:    br label %[[LOOP:.*]]
 ; CHECK:       [[LOOP]]:
 ; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL15]], %[[VEC_EPILOG_SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
diff --git a/llvm/test/Transforms/LoopVectorize/epilog-vectorization-any-of-reductions.ll b/llvm/test/Transforms/LoopVectorize/epilog-vectorization-any-of-reductions.ll
index 94593a7d9a81d..a829decb5231e 100644
--- a/llvm/test/Transforms/LoopVectorize/epilog-vectorization-any-of-reductions.ll
+++ b/llvm/test/Transforms/LoopVectorize/epilog-vectorization-any-of-reductions.ll
@@ -67,8 +67,8 @@ define i32 @any_of_reduction_epilog(ptr %src, i64 %N) {
 ; CHECK-NEXT:    [[CMP_N4:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC3]]
 ; CHECK-NEXT:    br i1 [[CMP_N4]], label [[EXIT]], label [[VEC_EPILOG_SCALAR_PH]]
 ; CHECK:       vec.epilog.scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC3]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ 0, [[ITER_CHECK:%.*]] ], [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ]
-; CHECK-NEXT:    [[BC_MERGE_RDX10:%.*]] = phi i32 [ [[RDX_SELECT9]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ 0, [[ITER_CHECK]] ], [ [[RDX_SELECT]], [[VEC_EPILOG_ITER_CHECK]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC3]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[ITER_CHECK:%.*]] ]
+; CHECK-NEXT:    [[BC_MERGE_RDX10:%.*]] = phi i32 [ [[RDX_SELECT9]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[RDX_SELECT]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[ITER_CHECK]] ]
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
 ; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[VEC_EPILOG_SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
@@ -166,8 +166,8 @@ define i32 @any_of_reduction_epilog_arg_as_start_value(ptr %src, i64 %N, i32 %st
 ; CHECK-NEXT:    [[CMP_N4:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC3]]
 ; CHECK-NEXT:    br i1 [[CMP_N4]], label [[EXIT]], label [[VEC_EPILOG_SCALAR_PH]]
 ; CHECK:       vec.epilog.scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC3]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ 0, [[ITER_CHECK:%.*]] ], [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ]
-; CHECK-NEXT:    [[BC_MERGE_RDX10:%.*]] = phi i32 [ [[RDX_SELECT9]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[START]], [[ITER_CHECK]] ], [ [[RDX_SELECT]], [[VEC_EPILOG_ITER_CHECK]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC3]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[ITER_CHECK:%.*]] ]
+; CHECK-NEXT:    [[BC_MERGE_RDX10:%.*]] = phi i32 [ [[RDX_SELECT9]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[RDX_SELECT]], [[VEC_EPILOG_ITER_CHECK]] ], [ [[START]], [[ITER_CHECK]] ]
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
 ; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[VEC_EPILOG_SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
@@ -274,9 +274,9 @@ define i1 @any_of_reduction_i1_epilog(i64 %N, i32 %a) {
 ; CHECK-NEXT:    [[CMP_N8:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC3]]
 ; CHECK-NEXT:    br i1 [[CMP_N8]], label [[EXIT]], label [[VEC_EPILOG_SCALAR_PH]]
 ; CHECK:       vec.epilog.scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL4:%.*]] = phi i64 [ [[N_VEC3]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ 0, [[ITER_CHECK:%.*]] ], [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ]
-; CHECK-NEXT:    [[BC_MERGE_RDX17:%.*]] = phi i1 [ [[RDX_SELECT16]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ false, [[ITER_CHECK]] ], [ [[RDX_SELECT]], [[VEC_EPILOG_ITER_CHECK]] ]
-; CHECK-NEXT:    [[BC_RESUME_VAL7:%.*]] = phi i32 [ [[IND_END5]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ 0, [[ITER_CHECK]] ], [ [[IND_END6]], [[VEC_EPILOG_ITER_CHECK]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL4:%.*]] = phi i64 [ [[N_VEC3]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[ITER_CHECK:%.*]] ]
+; CHECK-NEXT:    [[BC_MERGE_RDX17:%.*]] = phi i1 [ [[RDX_SELECT16]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[RDX_SELECT]], [[VEC_EPILOG_ITER_CHECK]] ], [ false, [[ITER_CHECK]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL7:%.*]] = phi i32 [ [[IND_END5]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[IND_END6]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[ITER_CHECK]] ]
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
 ; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL4]], [[VEC_EPILOG_SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
@@ -426,8 +426,8 @@ define i1 @any_of_reduction_i1_epilog2(ptr %start, ptr %end, i64 %x) {
 ; CHECK-NEXT:    [[CMP_N10:%.*]] = icmp eq i64 [[TMP3]], [[N_VEC8]]
 ; CHECK-NEXT:    br i1 [[CMP_N10]], label [[EXIT]], label [[VEC_EPILOG_SCALAR_PH]]
 ; CHECK:       vec.epilog.scalar.ph:
-; CHECK-NEXT:    [[BC_MERGE_RDX23:%.*]] = phi i1 [ [[RDX_SELECT22]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ true, [[ITER_CHECK:%.*]] ], [ [[RDX_SELECT]], [[VEC_EPILOG_ITER_CHECK]] ]
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi ptr [ [[IND_END]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[START]], [[ITER_CHECK]] ], [ [[IND_END9]], [[VEC_EPILOG_ITER_CHECK]] ]
+; CHECK-NEXT:    [[BC_MERGE_RDX23:%.*]] = phi i1 [ [[RDX_SELECT22]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[RDX_SELECT]], [[VEC_EPILOG_ITER_CHECK]] ], [ true, [[ITER_CHECK:%.*]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi ptr [ [[IND_END]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[IND_END9]], [[VEC_EPILOG_ITER_CHECK]] ], [ [[START]], [[ITER_CHECK]] ]
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
 ; CHECK-NEXT:    [[RED:%.*]] = phi i1 [ [[BC_MERGE_RDX23]], [[VEC_EPILOG_SCALAR_PH]] ], [ [[SELECT:%.*]], [[LOOP]] ]
diff --git a/llvm/test/Transforms/LoopVectorize/epilog-vectorization-reductions.ll b/llvm/test/Transforms/LoopVectorize/epilog-vectorization-reductions.ll
index 54489af8c9f12..4d260984bfe9d 100644
--- a/llvm/test/Transforms/LoopVectorize/epilog-vectorization-reductions.ll
+++ b/llvm/test/Transforms/LoopVectorize/epilog-vectorization-reductions.ll
@@ -58,8 +58,8 @@ define i64 @int_reduction_add(ptr %a, i64 %N) {
 ; CHECK-NEXT:    [[CMP_N8:%.*]] = icmp eq i64 [[N]], [[N_VEC3]]
 ; CHECK-NEXT:    br i1 [[CMP_N8]], label [[FOR_END]], label [[VEC_EPILOG_SCALAR_PH]]
 ; CHECK:       vec.epilog.scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC3]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ 0, [[ITER_CHECK:%.*]] ], [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ]
-; CHECK-NEXT:    [[BC_MERGE_RDX9:%.*]] = phi i64 [ [[TMP12]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ 5, [[ITER_CHECK]] ], [ [[TMP5]], [[VEC_EPILOG_ITER_CHECK]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC3]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[ITER_CHECK:%.*]] ]
+; CHECK-NEXT:    [[BC_MERGE_RDX9:%.*]] = phi i64 [ [[TMP12]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP5]], [[VEC_EPILOG_ITER_CHECK]] ], [ 5, [[ITER_CHECK]] ]
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK:       for.body:
 ; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[VEC_EPILOG_SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
@@ -151,8 +151,8 @@ define float @fp_reduction_max(ptr noalias %a, i64 %N) {
 ; CHECK-NEXT:    [[CMP_N8:%.*]] = icmp eq i64 [[N]], [[N_VEC3]]
 ; CHECK-NEXT:    br i1 [[CMP_N8]], label [[FOR_END]], label [[VEC_EPILOG_SCALAR_PH]]
 ; CHECK:       vec.epilog.scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC3]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ 0, [[ITER_CHECK:%.*]] ], [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ]
-; CHECK-NEXT:    [[BC_MERGE_RDX9:%.*]] = phi float [ [[TMP13]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ 0.000000e+00, [[ITER_CHECK]] ], [ [[TMP6]], [[VEC_EPILOG_ITER_CHECK]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC3]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[ITER_CHECK:%.*]] ]
+; CHECK-NEXT:    [[BC_MERGE_RDX9:%.*]] = phi float [ [[TMP13]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP6]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0.000000e+00, [[ITER_CHECK]] ]
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK:       for.body:
 ; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[IV_NEXT:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL]], [[VEC_EPILOG_SCALAR_PH]] ]
@@ -245,8 +245,8 @@ define i16 @reduction_or_trunc(ptr noalias nocapture %ptr) {
 ; CHECK-NEXT:    [[TMP24:%.*]] = zext i16 [[TMP23]] to i32
 ; CHECK-NEXT:    br i1 true, label [[FOR_END]], label [[VEC_EPILOG_SCALAR_PH]]
 ; CHECK:       vec.epilog.scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ 256, [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ 0, [[ITER_CHECK:%.*]] ], [ 256, [[VEC_EPILOG_ITER_CHECK]] ]
-; CHECK-NEXT:    [[BC_MERGE_RDX5:%.*]] = phi i32 [ [[TMP24]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ 0, [[ITER_CHECK]] ], [ [[TMP11]], [[VEC_EPILOG_ITER_CHECK]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ 256, [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ 256, [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[ITER_CHECK:%.*]] ]
+; CHECK-NEXT:    [[BC_MERGE_RDX5:%.*]] = phi i32 [ [[TMP24]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP11]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[ITER_CHECK]] ]
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK:       for.body:
 ; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ [[IV_NEXT:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL]], [[VEC_EPILOG_SCALAR_PH]] ]
@@ -349,9 +349,9 @@ define float @multiple_fp_rdx(ptr %A, i64 %N) {
 ; CHECK-NEXT:    [[CMP_N11:%.*]] = icmp eq i64 [[N]], [[N_VEC5]]
 ; CHECK-NEXT:    br i1 [[CMP_N11]], label [[FOR_END]], label [[VEC_EPILOG_SCALAR_PH]]
 ; CHECK:       vec.epilog.scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC5]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ 0, [[ITER_CHECK:%.*]] ], [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ]
-; CHECK-NEXT:    [[BC_MERGE_RDX12:%.*]] = phi float [ [[TMP16]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ 1.500000e+01, [[ITER_CHECK]] ], [ [[TMP6]], [[VEC_EPILOG_ITER_CHECK]] ]
-; CHECK-NEXT:    [[BC_MERGE_RDX13:%.*]] = phi float [ [[TMP17]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ 1.000000e+01, [[ITER_CHECK]] ], [ [[TMP7]], [[VEC_EPILOG_ITER_CHECK]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC5]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[ITER_CHECK:%.*]] ]
+; CHECK-NEXT:    [[BC_MERGE_RDX12:%.*]] = phi float [ [[TMP16]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP6]], [[VEC_EPILOG_ITER_CHECK]] ], [ 1.500000e+01, [[ITER_CHECK]] ]
+; CHECK-NEXT:    [[BC_MERGE_RDX13:%.*]] = phi float [ [[TMP17]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP7]], [[VEC_EPILOG_ITER_CHECK]] ], [ 1.000000e+01, [[ITER_CHECK]] ]
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK:       for.body:
 ; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[VEC_EPILOG_SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
@@ -452,8 +452,8 @@ define i32 @reduction_phi_start_val(ptr %A, i64 %N) {
 ; CHECK-NEXT:    [[CMP_N8:%.*]] = icmp eq i64 [[N]], [[N_VEC3]]
 ; CHECK-NEXT:    br i1 [[CMP_N8]], label [[FOR_COND]], label [[VEC_EPILOG_SCALAR_PH]]
 ; CHECK:       vec.epilog.scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC3]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ 0, [[ITER_CHECK]] ], [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ]
-; CHECK-NEXT:    [[BC_MERGE_RDX9:%.*]] = phi i32 [ [[TMP13]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[START_SUM]], [[ITER_CHECK]] ], [ [[TMP6]], [[VEC_EPILOG_ITER_CHECK]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC3]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[ITER_CHECK]] ]
+; CHECK-NEXT:    [[BC_MERGE_RDX9:%.*]] = phi i32 [ [[TMP13]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP6]], [[VEC_EPILOG_ITER_CHECK]] ], [ [[START_SUM]], [[ITER_CHECK]] ]
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK:       for.body:
 ; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[VEC_EPILOG_SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
diff --git a/llvm/test/Transforms/LoopVectorize/epilog-vectorization-trunc-induction-steps.ll b/llvm/test/Transforms/LoopVectorize/epilog-vectorization-trunc-induction-steps.ll
index 06ca7b197279c..7a92d1a1c9ea5 100644
--- a/llvm/test/Transforms/LoopVectorize/epilog-vectorization-trunc-induction-steps.ll
+++ b/llvm/test/Transforms/LoopVectorize/epilog-vectorization-trunc-induction-steps.ll
@@ -63,7 +63,7 @@ define void @trunc_iv_steps_with_epilogue(ptr %A, i64 %N) {
 ; CHECK-NEXT:    [[CMP_N4:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC3]]
 ; CHECK-NEXT:    br i1 [[CMP_N4]], label [[EXIT]], label [[VEC_EPILOG_SCALAR_PH]]
 ; CHECK:       vec.epilog.scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC3]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ 0, [[VECTOR_SCEVCHECK]] ], [ 0, [[ITER_CHECK:%.*]] ], [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC3]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[VECTOR_SCEVCHECK]] ], [ 0, [[ITER_CHECK:%.*]] ]
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
 ; CHECK-NEXT:    [[IV_I:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[VEC_EPILOG_SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
diff --git a/llvm/test/Transforms/LoopVectorize/explicit_outer_detection.ll b/llvm/test/Transforms/LoopVectorize/explicit_outer_detection.ll
index e01115888446a..35bd58ccbe00e 100644
--- a/llvm/test/Transforms/LoopVectorize/explicit_outer_detection.ll
+++ b/llvm/test/Transforms/LoopVectorize/explicit_outer_detection.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -passes=loop-vectorize -enable-vplan-native-path -debug-only=loop-vectorize -S 2>&1 | FileCheck %s
+; RUN: opt < %s -passes=loop-vectorize -enable-vplan-native-path -debug-only=loop-vectorize --disable-output -S 2>&1 | FileCheck %s
 ; REQUIRES: asserts
 
 ; Verify that outer loops annotated only with the expected explicit
diff --git a/llvm/test/Transforms/LoopVectorize/explicit_outer_nonuniform_inner.ll b/llvm/test/Transforms/LoopVectorize/explicit_outer_nonuniform_inner.ll
index 1253bbff85fd4..3fddc2fc3f108 100644
--- a/llvm/test/Transforms/LoopVectorize/explicit_outer_nonuniform_inner.ll
+++ b/llvm/test/Transforms/LoopVectorize/explicit_outer_nonuniform_inner.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -passes=loop-vectorize -enable-vplan-native-path -pass-remarks-analysis=loop-vectorize -debug-only=loop-vectorize -S 2>&1 | FileCheck %s
+; RUN: opt < %s -passes=loop-vectorize -enable-vplan-native-path -pass-remarks-analysis=loop-vectorize -debug-only=loop-vectorize --disable-output -S 2>&1 | FileCheck %s
 ; REQUIRES: asserts
 
 ; Verify that LV bails out on explicit vectorization outer loops that contain
diff --git a/llvm/test/Transforms/LoopVectorize/explicit_outer_uniform_diverg_branch.ll b/llvm/test/Transforms/LoopVectorize/explicit_outer_uniform_diverg_branch.ll
index ac87c1500114b..2131beb2932c9 100644
--- a/llvm/test/Transforms/LoopVectorize/explicit_outer_uniform_diverg_branch.ll
+++ b/llvm/test/Transforms/LoopVectorize/explicit_outer_uniform_diverg_branch.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -passes=loop-vectorize -enable-vplan-native-path -debug-only=loop-vectorize -S 2>&1 | FileCheck %s
+; RUN: opt < %s -passes=loop-vectorize -enable-vplan-native-path -debug-only=loop-vectorize --disable-output -S 2>&1 | FileCheck %s
 ; REQUIRES: asserts
 
 ; Verify that LV can handle explicit vectorization outer loops with uniform branches
diff --git a/llvm/test/Transforms/LoopVectorize/first-order-recurrence-complex.ll b/llvm/test/Transforms/LoopVectorize/first-order-recurrence-complex.ll
index 253ecaca75be8..a97fcc9f437c4 100644
--- a/llvm/test/Transforms/LoopVectorize/first-order-recurrence-complex.ll
+++ b/llvm/test/Transforms/LoopVectorize/first-order-recurrence-complex.ll
@@ -852,8 +852,8 @@ define void @sink_dominance(ptr %ptr, i32 %N) {
 ; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i32 [[UMAX1]], [[N_VEC]]
 ; CHECK-NEXT:    br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[SCALAR_RECUR_INIT:%.*]] = phi i64 [ [[VECTOR_RECUR_EXTRACT]], [[MIDDLE_BLOCK]] ], [ 0, [[VECTOR_SCEVCHECK]] ], [ 0, [[ENTRY:%.*]] ]
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[VECTOR_SCEVCHECK]] ], [ 0, [[ENTRY]] ]
+; CHECK-NEXT:    [[SCALAR_RECUR_INIT:%.*]] = phi i64 [ [[VECTOR_RECUR_EXTRACT]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ], [ 0, [[VECTOR_SCEVCHECK]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ], [ 0, [[VECTOR_SCEVCHECK]] ]
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
 ; CHECK-NEXT:    [[SCALAR_RECUR:%.*]] = phi i64 [ [[SCALAR_RECUR_INIT]], [[SCALAR_PH]] ], [ [[FOR_NEXT:%.*]], [[LOOP]] ]
@@ -935,8 +935,8 @@ define void @sink_dominance_2(ptr %ptr, i32 %N) {
 ; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i32 [[UMAX1]], [[N_VEC]]
 ; CHECK-NEXT:    br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[SCALAR_RECUR_INIT:%.*]] = phi i64 [ [[VECTOR_RECUR_EXTRACT]], [[MIDDLE_BLOCK]] ], [ 0, [[VECTOR_SCEVCHECK]] ], [ 0, [[ENTRY:%.*]] ]
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[VECTOR_SCEVCHECK]] ], [ 0, [[ENTRY]] ]
+; CHECK-NEXT:    [[SCALAR_RECUR_INIT:%.*]] = phi i64 [ [[VECTOR_RECUR_EXTRACT]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ], [ 0, [[VECTOR_SCEVCHECK]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ], [ 0, [[VECTOR_SCEVCHECK]] ]
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
 ; CHECK-NEXT:    [[SCALAR_RECUR:%.*]] = phi i64 [ [[SCALAR_RECUR_INIT]], [[SCALAR_PH]] ], [ [[FOR_NEXT:%.*]], [[LOOP]] ]
diff --git a/llvm/test/Transforms/LoopVectorize/float-induction.ll b/llvm/test/Transforms/LoopVectorize/float-induction.ll
index 601c475df56cd..29b29c500c46e 100644
--- a/llvm/test/Transforms/LoopVectorize/float-induction.ll
+++ b/llvm/test/Transforms/LoopVectorize/float-induction.ll
@@ -842,14 +842,16 @@ define void @fp_iv_loop3(float %init, ptr noalias nocapture %A, ptr noalias noca
 ; VEC4_INTERL2-NEXT:    [[TMP3:%.*]] = fmul fast float [[TMP0]], [[DOTCAST2]]
 ; VEC4_INTERL2-NEXT:    [[IND_END3:%.*]] = fadd fast float [[INIT:%.*]], [[TMP3]]
 ; VEC4_INTERL2-NEXT:    [[DOTSPLATINSERT2:%.*]] = insertelement <4 x float> poison, float [[TMP0]], i64 0
-; VEC4_INTERL2-NEXT:    [[BROADCAST:%.*]] = shufflevector <4 x float> [[DOTSPLATINSERT2]], <4 x float> poison, <4 x i32> zeroinitializer
-; VEC4_INTERL2-NEXT:    [[DOTSPLAT5:%.*]] = fmul fast <4 x float> [[BROADCAST]], splat (float 4.000000e+00)
+; VEC4_INTERL2-NEXT:    [[TMP19:%.*]] = fmul fast <4 x float> [[DOTSPLATINSERT2]], <float 4.000000e+00, float poison, float poison, float poison>
+; VEC4_INTERL2-NEXT:    [[DOTSPLAT5:%.*]] = shufflevector <4 x float> [[TMP19]], <4 x float> poison, <4 x i32> zeroinitializer
 ; VEC4_INTERL2-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <4 x float> poison, float [[INIT]], i64 0
 ; VEC4_INTERL2-NEXT:    [[DOTSPLAT:%.*]] = shufflevector <4 x float> [[DOTSPLATINSERT]], <4 x float> poison, <4 x i32> zeroinitializer
 ; VEC4_INTERL2-NEXT:    [[DOTSPLATINSERT6:%.*]] = insertelement <4 x float> poison, float [[TMP0]], i64 0
 ; VEC4_INTERL2-NEXT:    [[DOTSPLAT7:%.*]] = shufflevector <4 x float> [[DOTSPLATINSERT6]], <4 x float> poison, <4 x i32> zeroinitializer
 ; VEC4_INTERL2-NEXT:    [[TMP4:%.*]] = fmul fast <4 x float> [[DOTSPLAT7]], <float 0.000000e+00, float 1.000000e+00, float 2.000000e+00, float 3.000000e+00>
 ; VEC4_INTERL2-NEXT:    [[INDUCTION:%.*]] = fadd fast <4 x float> [[DOTSPLAT]], [[TMP4]]
+; VEC4_INTERL2-NEXT:    [[BROADCAST_SPLATINSERT7:%.*]] = insertelement <4 x float> poison, float [[TMP0]], i64 0
+; VEC4_INTERL2-NEXT:    [[BROADCAST:%.*]] = shufflevector <4 x float> [[BROADCAST_SPLATINSERT7]], <4 x float> poison, <4 x i32> zeroinitializer
 ; VEC4_INTERL2-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; VEC4_INTERL2:       vector.body:
 ; VEC4_INTERL2-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
diff --git a/llvm/test/Transforms/LoopVectorize/fpsat.ll b/llvm/test/Transforms/LoopVectorize/fpsat.ll
index 7df0a34c35b85..77c4e8d7c68bf 100644
--- a/llvm/test/Transforms/LoopVectorize/fpsat.ll
+++ b/llvm/test/Transforms/LoopVectorize/fpsat.ll
@@ -37,7 +37,7 @@ define void @signed(ptr %x, ptr %y, i32 %n) {
 ; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[WIDE_TRIP_COUNT]], [[N_VEC]]
 ; CHECK-NEXT:    br i1 [[CMP_N]], label [[FOR_COND_CLEANUP_LOOPEXIT:%.*]], label [[SCALAR_PH]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[VECTOR_MEMCHECK]] ], [ 0, [[FOR_BODY_PREHEADER]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[FOR_BODY_PREHEADER]] ], [ 0, [[VECTOR_MEMCHECK]] ]
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK:       for.cond.cleanup.loopexit:
 ; CHECK-NEXT:    br label [[FOR_COND_CLEANUP]]
@@ -113,7 +113,7 @@ define void @unsigned(ptr %x, ptr %y, i32 %n) {
 ; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[WIDE_TRIP_COUNT]], [[N_VEC]]
 ; CHECK-NEXT:    br i1 [[CMP_N]], label [[FOR_COND_CLEANUP_LOOPEXIT:%.*]], label [[SCALAR_PH]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[VECTOR_MEMCHECK]] ], [ 0, [[FOR_BODY_PREHEADER]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[FOR_BODY_PREHEADER]] ], [ 0, [[VECTOR_MEMCHECK]] ]
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK:       for.cond.cleanup.loopexit:
 ; CHECK-NEXT:    br label [[FOR_COND_CLEANUP]]
diff --git a/llvm/test/Transforms/LoopVectorize/if-conversion-nest.ll b/llvm/test/Transforms/LoopVectorize/if-conversion-nest.ll
index ff5d45d3f7f0f..492eb091175e2 100644
--- a/llvm/test/Transforms/LoopVectorize/if-conversion-nest.ll
+++ b/llvm/test/Transforms/LoopVectorize/if-conversion-nest.ll
@@ -46,7 +46,7 @@ define i32 @foo(ptr nocapture %A, ptr nocapture %B, i32 %n) {
 ; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N_VEC]], [[TMP0]]
 ; CHECK-NEXT:    br i1 [[CMP_N]], label [[FOR_END_LOOPEXIT:%.*]], label [[SCALAR_PH]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[VECTOR_MEMCHECK]] ], [ 0, [[FOR_BODY_PREHEADER]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[FOR_BODY_PREHEADER]] ], [ 0, [[VECTOR_MEMCHECK]] ]
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK:       for.body:
 ; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], [[IF_END14:%.*]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
@@ -160,7 +160,7 @@ define i32 @multi_variable_if_nest(ptr nocapture %A, ptr nocapture %B, i32 %n) {
 ; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N_VEC]], [[TMP0]]
 ; CHECK-NEXT:    br i1 [[CMP_N]], label [[FOR_END_LOOPEXIT:%.*]], label [[SCALAR_PH]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[VECTOR_MEMCHECK]] ], [ 0, [[FOR_BODY_PREHEADER]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[FOR_BODY_PREHEADER]] ], [ 0, [[VECTOR_MEMCHECK]] ]
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK:       for.body:
 ; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], [[IF_END14:%.*]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
diff --git a/llvm/test/Transforms/LoopVectorize/if-pred-non-void.ll b/llvm/test/Transforms/LoopVectorize/if-pred-non-void.ll
index e733a4a381e50..19660063f509b 100644
--- a/llvm/test/Transforms/LoopVectorize/if-pred-non-void.ll
+++ b/llvm/test/Transforms/LoopVectorize/if-pred-non-void.ll
@@ -273,7 +273,7 @@ define void @test(ptr nocapture %asd, ptr nocapture %aud,
 ; UNROLL-NO-VF:       middle.block:
 ; UNROLL-NO-VF-NEXT:    br i1 true, label [[FOR_COND_CLEANUP:%.*]], label [[SCALAR_PH]]
 ; UNROLL-NO-VF:       scalar.ph:
-; UNROLL-NO-VF-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 128, [[MIDDLE_BLOCK]] ], [ 0, [[VECTOR_MEMCHECK]] ], [ 0, [[ENTRY:%.*]] ]
+; UNROLL-NO-VF-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 128, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ], [ 0, [[VECTOR_MEMCHECK]] ]
 ; UNROLL-NO-VF-NEXT:    br label [[FOR_BODY:%.*]]
 ; UNROLL-NO-VF:       for.cond.cleanup:
 ; UNROLL-NO-VF-NEXT:    ret void
@@ -484,7 +484,7 @@ define void @test_scalar2scalar(ptr nocapture %asd, ptr nocapture %bsd) {
 ; UNROLL-NO-VF:       middle.block:
 ; UNROLL-NO-VF-NEXT:    br i1 true, label [[FOR_COND_CLEANUP:%.*]], label [[SCALAR_PH]]
 ; UNROLL-NO-VF:       scalar.ph:
-; UNROLL-NO-VF-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 128, [[MIDDLE_BLOCK]] ], [ 0, [[VECTOR_MEMCHECK]] ], [ 0, [[ENTRY:%.*]] ]
+; UNROLL-NO-VF-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 128, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ], [ 0, [[VECTOR_MEMCHECK]] ]
 ; UNROLL-NO-VF-NEXT:    br label [[FOR_BODY:%.*]]
 ; UNROLL-NO-VF:       for.cond.cleanup:
 ; UNROLL-NO-VF-NEXT:    ret void
@@ -682,7 +682,7 @@ define void @pr30172(ptr nocapture %asd, ptr nocapture %bsd) !dbg !5 {;
 ; UNROLL-NO-VF:       middle.block:
 ; UNROLL-NO-VF-NEXT:    br i1 true, label [[FOR_COND_CLEANUP:%.*]], label [[SCALAR_PH]]
 ; UNROLL-NO-VF:       scalar.ph:
-; UNROLL-NO-VF-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 128, [[MIDDLE_BLOCK]] ], [ 0, [[VECTOR_MEMCHECK]] ], [ 0, [[ENTRY:%.*]] ]
+; UNROLL-NO-VF-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 128, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ], [ 0, [[VECTOR_MEMCHECK]] ]
 ; UNROLL-NO-VF-NEXT:    br label [[FOR_BODY:%.*]]
 ; UNROLL-NO-VF:       for.cond.cleanup:
 ; UNROLL-NO-VF-NEXT:    ret void
@@ -761,8 +761,7 @@ define i32 @predicated_udiv_scalarized_operand(ptr %a, i1 %c, i32 %x, i64 %n) {
 ; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i64 [[TMP0]]
 ; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i32 0
 ; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <2 x i32>, ptr [[TMP2]], align 4
-; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <2 x i1> [[BROADCAST_SPLAT]], i32 0
-; CHECK-NEXT:    br i1 [[TMP3]], label [[PRED_UDIV_IF:%.*]], label [[PRED_UDIV_CONTINUE:%.*]]
+; CHECK-NEXT:    br i1 [[C]], label [[PRED_UDIV_IF:%.*]], label [[PRED_UDIV_CONTINUE:%.*]]
 ; CHECK:       pred.udiv.if:
 ; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <2 x i32> [[WIDE_LOAD]], i32 0
 ; CHECK-NEXT:    [[TMP5:%.*]] = add nsw i32 [[TMP4]], [[X:%.*]]
@@ -772,8 +771,7 @@ define i32 @predicated_udiv_scalarized_operand(ptr %a, i1 %c, i32 %x, i64 %n) {
 ; CHECK-NEXT:    br label [[PRED_UDIV_CONTINUE]]
 ; CHECK:       pred.udiv.continue:
 ; CHECK-NEXT:    [[TMP9:%.*]] = phi <2 x i32> [ poison, [[VECTOR_BODY]] ], [ [[TMP8]], [[PRED_UDIV_IF]] ]
-; CHECK-NEXT:    [[TMP10:%.*]] = extractelement <2 x i1> [[BROADCAST_SPLAT]], i32 1
-; CHECK-NEXT:    br i1 [[TMP10]], label [[PRED_UDIV_IF1:%.*]], label [[PRED_UDIV_CONTINUE2]]
+; CHECK-NEXT:    br i1 [[C]], label [[PRED_UDIV_IF1:%.*]], label [[PRED_UDIV_CONTINUE2]]
 ; CHECK:       pred.udiv.if1:
 ; CHECK-NEXT:    [[TMP11:%.*]] = extractelement <2 x i32> [[WIDE_LOAD]], i32 1
 ; CHECK-NEXT:    [[TMP12:%.*]] = add nsw i32 [[TMP11]], [[X]]
diff --git a/llvm/test/Transforms/LoopVectorize/if-pred-stores.ll b/llvm/test/Transforms/LoopVectorize/if-pred-stores.ll
index 2a85761da1e52..c060ccda6b011 100644
--- a/llvm/test/Transforms/LoopVectorize/if-pred-stores.ll
+++ b/llvm/test/Transforms/LoopVectorize/if-pred-stores.ll
@@ -332,17 +332,11 @@ define void @bug18724(i1 %cond, ptr %ptr, i1 %cond.2, i64 %v.1, i32 %v.2) {
 ; VEC-NEXT:    [[TMP7:%.*]] = getelementptr inbounds [768 x i32], ptr [[PTR:%.*]], i64 0, i64 [[TMP6]]
 ; VEC-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i32, ptr [[TMP7]], i32 0
 ; VEC-NEXT:    [[WIDE_LOAD:%.*]] = load <2 x i32>, ptr [[TMP8]], align 4
-; VEC-NEXT:    [[TMP9:%.*]] = extractelement <2 x i1> [[BROADCAST_SPLAT]], i32 0
-; VEC-NEXT:    br i1 [[TMP9]], label [[PRED_STORE_IF:%.*]], label [[PRED_STORE_CONTINUE:%.*]]
+; VEC-NEXT:    br i1 [[COND_2]], label [[PRED_STORE_IF:%.*]], label [[PRED_STORE_CONTINUE2]]
 ; VEC:       pred.store.if:
 ; VEC-NEXT:    [[TMP10:%.*]] = getelementptr inbounds [768 x i32], ptr [[PTR]], i64 0, i64 [[TMP6]]
 ; VEC-NEXT:    [[TMP11:%.*]] = extractelement <2 x i32> [[WIDE_LOAD]], i32 0
 ; VEC-NEXT:    store i32 [[TMP11]], ptr [[TMP10]], align 4
-; VEC-NEXT:    br label [[PRED_STORE_CONTINUE]]
-; VEC:       pred.store.continue:
-; VEC-NEXT:    [[TMP12:%.*]] = extractelement <2 x i1> [[BROADCAST_SPLAT]], i32 1
-; VEC-NEXT:    br i1 [[TMP12]], label [[PRED_STORE_IF1:%.*]], label [[PRED_STORE_CONTINUE2]]
-; VEC:       pred.store.if1:
 ; VEC-NEXT:    [[TMP13:%.*]] = add i64 [[OFFSET_IDX]], 1
 ; VEC-NEXT:    [[TMP14:%.*]] = getelementptr inbounds [768 x i32], ptr [[PTR]], i64 0, i64 [[TMP13]]
 ; VEC-NEXT:    [[TMP15:%.*]] = extractelement <2 x i32> [[WIDE_LOAD]], i32 1
@@ -493,8 +487,6 @@ define void @minimal_bit_widths(i1 %c) {
 ;
 ; VEC-LABEL: @minimal_bit_widths(
 ; VEC-NEXT:  entry:
-; VEC-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x i1> poison, i1 [[C:%.*]], i64 0
-; VEC-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <2 x i1> [[BROADCAST_SPLATINSERT]], <2 x i1> poison, <2 x i32> zeroinitializer
 ; VEC-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; VEC:       vector.body:
 ; VEC-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[INDEX_NEXT:%.*]], [[PRED_STORE_CONTINUE2:%.*]] ]
@@ -502,17 +494,11 @@ define void @minimal_bit_widths(i1 %c) {
 ; VEC-NEXT:    [[TMP1:%.*]] = getelementptr i8, ptr undef, i64 [[TMP0]]
 ; VEC-NEXT:    [[TMP2:%.*]] = getelementptr i8, ptr [[TMP1]], i32 0
 ; VEC-NEXT:    [[WIDE_LOAD:%.*]] = load <2 x i8>, ptr [[TMP2]], align 1
-; VEC-NEXT:    [[TMP3:%.*]] = extractelement <2 x i1> [[BROADCAST_SPLAT]], i32 0
-; VEC-NEXT:    br i1 [[TMP3]], label [[PRED_STORE_IF:%.*]], label [[PRED_STORE_CONTINUE:%.*]]
+; VEC-NEXT:    br i1 [[C:%.*]], label [[PRED_STORE_IF:%.*]], label [[PRED_STORE_CONTINUE2]]
 ; VEC:       pred.store.if:
 ; VEC-NEXT:    [[TMP4:%.*]] = getelementptr i8, ptr undef, i64 [[TMP0]]
 ; VEC-NEXT:    [[TMP5:%.*]] = extractelement <2 x i8> [[WIDE_LOAD]], i32 0
 ; VEC-NEXT:    store i8 [[TMP5]], ptr [[TMP4]], align 1
-; VEC-NEXT:    br label [[PRED_STORE_CONTINUE]]
-; VEC:       pred.store.continue:
-; VEC-NEXT:    [[TMP6:%.*]] = extractelement <2 x i1> [[BROADCAST_SPLAT]], i32 1
-; VEC-NEXT:    br i1 [[TMP6]], label [[PRED_STORE_IF1:%.*]], label [[PRED_STORE_CONTINUE2]]
-; VEC:       pred.store.if1:
 ; VEC-NEXT:    [[TMP7:%.*]] = add i64 [[INDEX]], 1
 ; VEC-NEXT:    [[TMP8:%.*]] = getelementptr i8, ptr undef, i64 [[TMP7]]
 ; VEC-NEXT:    [[TMP9:%.*]] = extractelement <2 x i8> [[WIDE_LOAD]], i32 1
@@ -633,8 +619,6 @@ define void @minimal_bit_widths_with_aliasing_store(i1 %c, ptr %ptr) {
 ;
 ; VEC-LABEL: @minimal_bit_widths_with_aliasing_store(
 ; VEC-NEXT:  entry:
-; VEC-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x i1> poison, i1 [[C1:%.*]], i64 0
-; VEC-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <2 x i1> [[BROADCAST_SPLATINSERT]], <2 x i1> poison, <2 x i32> zeroinitializer
 ; VEC-NEXT:    br label [[FOR_BODY:%.*]]
 ; VEC:       vector.body:
 ; VEC-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[INDEX_NEXT:%.*]], [[PRED_STORE_CONTINUE2:%.*]] ]
@@ -643,17 +627,11 @@ define void @minimal_bit_widths_with_aliasing_store(i1 %c, ptr %ptr) {
 ; VEC-NEXT:    [[TMP3:%.*]] = getelementptr i8, ptr [[TMP2]], i32 0
 ; VEC-NEXT:    [[WIDE_LOAD:%.*]] = load <2 x i8>, ptr [[TMP3]], align 1
 ; VEC-NEXT:    store <2 x i8> zeroinitializer, ptr [[TMP3]], align 1
-; VEC-NEXT:    [[C:%.*]] = extractelement <2 x i1> [[BROADCAST_SPLAT]], i32 0
-; VEC-NEXT:    br i1 [[C]], label [[IF_THEN:%.*]], label [[FOR_INC:%.*]]
+; VEC-NEXT:    br i1 [[C:%.*]], label [[PRED_STORE_IF:%.*]], label [[PRED_STORE_CONTINUE2]]
 ; VEC:       pred.store.if:
 ; VEC-NEXT:    [[TMP4:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[TMP0]]
 ; VEC-NEXT:    [[TMP5:%.*]] = extractelement <2 x i8> [[WIDE_LOAD]], i32 0
 ; VEC-NEXT:    store i8 [[TMP5]], ptr [[TMP4]], align 1
-; VEC-NEXT:    br label [[FOR_INC]]
-; VEC:       pred.store.continue:
-; VEC-NEXT:    [[TMP6:%.*]] = extractelement <2 x i1> [[BROADCAST_SPLAT]], i32 1
-; VEC-NEXT:    br i1 [[TMP6]], label [[PRED_STORE_IF1:%.*]], label [[PRED_STORE_CONTINUE2]]
-; VEC:       pred.store.if1:
 ; VEC-NEXT:    [[TMP7:%.*]] = add i64 [[INDEX]], 1
 ; VEC-NEXT:    [[TMP8:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[TMP7]]
 ; VEC-NEXT:    [[TMP9:%.*]] = extractelement <2 x i8> [[WIDE_LOAD]], i32 1
diff --git a/llvm/test/Transforms/LoopVectorize/if-reduction.ll b/llvm/test/Transforms/LoopVectorize/if-reduction.ll
index 220e62d9e3730..f266a13924f06 100644
--- a/llvm/test/Transforms/LoopVectorize/if-reduction.ll
+++ b/llvm/test/Transforms/LoopVectorize/if-reduction.ll
@@ -1197,7 +1197,7 @@ for.end:                                          ; preds = %for.body, %entry
 
 define float @fcmp_multi(ptr nocapture readonly %a, i32 %n) nounwind readonly {
 ; CHECK-LABEL: define float @fcmp_multi(
-; CHECK-SAME: ptr nocapture readonly [[A:%.*]], i32 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-SAME: ptr readonly captures(none) [[A:%.*]], i32 [[N:%.*]]) #[[ATTR0]] {
 ; CHECK-NEXT:  [[ENTRY:.*]]:
 ; CHECK-NEXT:    [[CMP10:%.*]] = icmp sgt i32 [[N]], 0
 ; CHECK-NEXT:    br i1 [[CMP10]], label %[[FOR_BODY_PREHEADER:.*]], label %[[FOR_END:.*]]
@@ -1323,7 +1323,7 @@ for.end:                                          ; preds = %for.inc, %entry
 
 define float @fcmp_fadd_fsub(ptr nocapture readonly %a, i32 %n) nounwind readonly {
 ; CHECK-LABEL: define float @fcmp_fadd_fsub(
-; CHECK-SAME: ptr nocapture readonly [[A:%.*]], i32 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-SAME: ptr readonly captures(none) [[A:%.*]], i32 [[N:%.*]]) #[[ATTR0]] {
 ; CHECK-NEXT:  [[ENTRY:.*]]:
 ; CHECK-NEXT:    [[CMP9:%.*]] = icmp sgt i32 [[N]], 0
 ; CHECK-NEXT:    br i1 [[CMP9]], label %[[FOR_BODY_PREHEADER:.*]], label %[[FOR_END:.*]]
@@ -1446,7 +1446,7 @@ for.end:                                          ; preds = %for.inc, %entry
 
 define float @fcmp_fadd_fmul(ptr nocapture readonly %a, i32 %n) nounwind readonly {
 ; CHECK-LABEL: define float @fcmp_fadd_fmul(
-; CHECK-SAME: ptr nocapture readonly [[A:%.*]], i32 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-SAME: ptr readonly captures(none) [[A:%.*]], i32 [[N:%.*]]) #[[ATTR0]] {
 ; CHECK-NEXT:  [[ENTRY:.*]]:
 ; CHECK-NEXT:    [[CMP9:%.*]] = icmp sgt i32 [[N]], 0
 ; CHECK-NEXT:    br i1 [[CMP9]], label %[[FOR_BODY_PREHEADER:.*]], label %[[FOR_END:.*]]
@@ -1535,7 +1535,7 @@ for.end:                                          ; preds = %for.inc, %entry
 
 define float @fcmp_store_back(ptr nocapture %a, i32 %LEN) nounwind readonly {
 ; CHECK-LABEL: define float @fcmp_store_back(
-; CHECK-SAME: ptr nocapture [[A:%.*]], i32 [[LEN:%.*]]) #[[ATTR0]] {
+; CHECK-SAME: ptr captures(none) [[A:%.*]], i32 [[LEN:%.*]]) #[[ATTR0]] {
 ; CHECK-NEXT:  [[ENTRY:.*]]:
 ; CHECK-NEXT:    [[CMP7:%.*]] = icmp sgt i32 [[LEN]], 0
 ; CHECK-NEXT:    br i1 [[CMP7]], label %[[FOR_BODY_PREHEADER:.*]], label %[[FOR_END:.*]]
diff --git a/llvm/test/Transforms/LoopVectorize/induction-step.ll b/llvm/test/Transforms/LoopVectorize/induction-step.ll
index f553864b5fc38..29c8d1f36ea59 100644
--- a/llvm/test/Transforms/LoopVectorize/induction-step.ll
+++ b/llvm/test/Transforms/LoopVectorize/induction-step.ll
@@ -19,7 +19,7 @@ target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 
 define void @induction_with_global(i32 %init, ptr noalias nocapture %A, i32 %N) {
 ; CHECK-LABEL: define void @induction_with_global(
-; CHECK-SAME: i32 [[INIT:%.*]], ptr noalias nocapture [[A:%.*]], i32 [[N:%.*]]) {
+; CHECK-SAME: i32 [[INIT:%.*]], ptr noalias captures(none) [[A:%.*]], i32 [[N:%.*]]) {
 ; CHECK-NEXT:  [[ENTRY:.*]]:
 ; CHECK-NEXT:    [[TMP0:%.*]] = load i32, ptr @int_inc, align 4
 ; CHECK-NEXT:    [[TMP1:%.*]] = add i32 [[N]], -1
@@ -106,7 +106,7 @@ exit:
 
 define i32 @induction_with_loop_inv(i32 %init, ptr noalias nocapture %A, i32 %N, i32 %M) {
 ; CHECK-LABEL: define i32 @induction_with_loop_inv(
-; CHECK-SAME: i32 [[INIT:%.*]], ptr noalias nocapture [[A:%.*]], i32 [[N:%.*]], i32 [[M:%.*]]) {
+; CHECK-SAME: i32 [[INIT:%.*]], ptr noalias captures(none) [[A:%.*]], i32 [[N:%.*]], i32 [[M:%.*]]) {
 ; CHECK-NEXT:  [[ENTRY:.*]]:
 ; CHECK-NEXT:    [[TMP3:%.*]] = add i32 [[N]], -1
 ; CHECK-NEXT:    [[TMP11:%.*]] = zext i32 [[TMP3]] to i64
diff --git a/llvm/test/Transforms/LoopVectorize/induction.ll b/llvm/test/Transforms/LoopVectorize/induction.ll
index 939709b91062e..f6ca9ff2e7b2e 100644
--- a/llvm/test/Transforms/LoopVectorize/induction.ll
+++ b/llvm/test/Transforms/LoopVectorize/induction.ll
@@ -302,7 +302,7 @@ define void @scalar_use(ptr %a, float %b, i64 %offset, i64 %offset2, i64 %n) {
 ; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
 ; CHECK-NEXT:    br i1 [[CMP_N]], label [[LOOPEXIT:%.*]], label [[SCALAR_PH]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[VECTOR_MEMCHECK]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ], [ 0, [[VECTOR_MEMCHECK]] ]
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK:       for.body:
 ; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
@@ -362,7 +362,7 @@ define void @scalar_use(ptr %a, float %b, i64 %offset, i64 %offset2, i64 %n) {
 ; IND-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
 ; IND-NEXT:    br i1 [[CMP_N]], label [[LOOPEXIT:%.*]], label [[SCALAR_PH]]
 ; IND:       scalar.ph:
-; IND-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[VECTOR_MEMCHECK]] ], [ 0, [[ENTRY:%.*]] ]
+; IND-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ], [ 0, [[VECTOR_MEMCHECK]] ]
 ; IND-NEXT:    br label [[FOR_BODY:%.*]]
 ; IND:       for.body:
 ; IND-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
@@ -429,7 +429,7 @@ define void @scalar_use(ptr %a, float %b, i64 %offset, i64 %offset2, i64 %n) {
 ; UNROLL-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
 ; UNROLL-NEXT:    br i1 [[CMP_N]], label [[LOOPEXIT:%.*]], label [[SCALAR_PH]]
 ; UNROLL:       scalar.ph:
-; UNROLL-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[VECTOR_MEMCHECK]] ], [ 0, [[ENTRY:%.*]] ]
+; UNROLL-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ], [ 0, [[VECTOR_MEMCHECK]] ]
 ; UNROLL-NEXT:    br label [[FOR_BODY:%.*]]
 ; UNROLL:       for.body:
 ; UNROLL-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
@@ -500,7 +500,7 @@ define void @scalar_use(ptr %a, float %b, i64 %offset, i64 %offset2, i64 %n) {
 ; UNROLL-NO-IC-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
 ; UNROLL-NO-IC-NEXT:    br i1 [[CMP_N]], label [[LOOPEXIT:%.*]], label [[SCALAR_PH]]
 ; UNROLL-NO-IC:       scalar.ph:
-; UNROLL-NO-IC-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[VECTOR_MEMCHECK]] ], [ 0, [[ENTRY:%.*]] ]
+; UNROLL-NO-IC-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ], [ 0, [[VECTOR_MEMCHECK]] ]
 ; UNROLL-NO-IC-NEXT:    br label [[FOR_BODY:%.*]]
 ; UNROLL-NO-IC:       for.body:
 ; UNROLL-NO-IC-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
@@ -567,7 +567,7 @@ define void @scalar_use(ptr %a, float %b, i64 %offset, i64 %offset2, i64 %n) {
 ; INTERLEAVE-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
 ; INTERLEAVE-NEXT:    br i1 [[CMP_N]], label [[LOOPEXIT:%.*]], label [[SCALAR_PH]]
 ; INTERLEAVE:       scalar.ph:
-; INTERLEAVE-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[VECTOR_MEMCHECK]] ], [ 0, [[ENTRY:%.*]] ]
+; INTERLEAVE-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ], [ 0, [[VECTOR_MEMCHECK]] ]
 ; INTERLEAVE-NEXT:    br label [[FOR_BODY:%.*]]
 ; INTERLEAVE:       for.body:
 ; INTERLEAVE-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
@@ -1593,7 +1593,7 @@ define void @scalarize_induction_variable_04(ptr %a, ptr %p, i32 %n) {
 ; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[TMP2]], [[N_VEC]]
 ; CHECK-NEXT:    br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[VECTOR_MEMCHECK]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ], [ 0, [[VECTOR_MEMCHECK]] ]
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK:       for.body:
 ; CHECK-NEXT:    [[I:%.*]] = phi i64 [ [[I_NEXT:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
@@ -1656,7 +1656,7 @@ define void @scalarize_induction_variable_04(ptr %a, ptr %p, i32 %n) {
 ; IND-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[TMP2]], [[N_VEC]]
 ; IND-NEXT:    br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]]
 ; IND:       scalar.ph:
-; IND-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[VECTOR_MEMCHECK]] ], [ 0, [[ENTRY:%.*]] ]
+; IND-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ], [ 0, [[VECTOR_MEMCHECK]] ]
 ; IND-NEXT:    br label [[FOR_BODY:%.*]]
 ; IND:       for.body:
 ; IND-NEXT:    [[I:%.*]] = phi i64 [ [[I_NEXT:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
@@ -1733,7 +1733,7 @@ define void @scalarize_induction_variable_04(ptr %a, ptr %p, i32 %n) {
 ; UNROLL-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[TMP2]], [[N_VEC]]
 ; UNROLL-NEXT:    br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]]
 ; UNROLL:       scalar.ph:
-; UNROLL-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[VECTOR_MEMCHECK]] ], [ 0, [[ENTRY:%.*]] ]
+; UNROLL-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ], [ 0, [[VECTOR_MEMCHECK]] ]
 ; UNROLL-NEXT:    br label [[FOR_BODY:%.*]]
 ; UNROLL:       for.body:
 ; UNROLL-NEXT:    [[I:%.*]] = phi i64 [ [[I_NEXT:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
@@ -1812,7 +1812,7 @@ define void @scalarize_induction_variable_04(ptr %a, ptr %p, i32 %n) {
 ; UNROLL-NO-IC-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[TMP2]], [[N_VEC]]
 ; UNROLL-NO-IC-NEXT:    br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]]
 ; UNROLL-NO-IC:       scalar.ph:
-; UNROLL-NO-IC-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[VECTOR_MEMCHECK]] ], [ 0, [[ENTRY:%.*]] ]
+; UNROLL-NO-IC-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ], [ 0, [[VECTOR_MEMCHECK]] ]
 ; UNROLL-NO-IC-NEXT:    br label [[FOR_BODY:%.*]]
 ; UNROLL-NO-IC:       for.body:
 ; UNROLL-NO-IC-NEXT:    [[I:%.*]] = phi i64 [ [[I_NEXT:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
@@ -1900,7 +1900,7 @@ define void @scalarize_induction_variable_04(ptr %a, ptr %p, i32 %n) {
 ; INTERLEAVE:       middle.block:
 ; INTERLEAVE-NEXT:    br label [[SCALAR_PH]]
 ; INTERLEAVE:       scalar.ph:
-; INTERLEAVE-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[VECTOR_MEMCHECK]] ], [ 0, [[ENTRY:%.*]] ]
+; INTERLEAVE-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ], [ 0, [[VECTOR_MEMCHECK]] ]
 ; INTERLEAVE-NEXT:    br label [[FOR_BODY:%.*]]
 ; INTERLEAVE:       for.body:
 ; INTERLEAVE-NEXT:    [[I:%.*]] = phi i64 [ [[I_NEXT:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
@@ -1971,8 +1971,7 @@ define i32 @scalarize_induction_variable_05(ptr %a, i32 %x, i1 %c, i32 %n) {
 ; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i32 [[TMP0]]
 ; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i32 0
 ; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <2 x i32>, ptr [[TMP2]], align 4
-; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <2 x i1> [[BROADCAST_SPLAT]], i32 0
-; CHECK-NEXT:    br i1 [[TMP3]], label [[PRED_UDIV_IF:%.*]], label [[PRED_UDIV_CONTINUE:%.*]]
+; CHECK-NEXT:    br i1 [[C]], label [[PRED_UDIV_IF:%.*]], label [[PRED_UDIV_CONTINUE:%.*]]
 ; CHECK:       pred.udiv.if:
 ; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <2 x i32> [[WIDE_LOAD]], i32 0
 ; CHECK-NEXT:    [[TMP5:%.*]] = udiv i32 [[TMP4]], [[TMP0]]
@@ -1980,8 +1979,7 @@ define i32 @scalarize_induction_variable_05(ptr %a, i32 %x, i1 %c, i32 %n) {
 ; CHECK-NEXT:    br label [[PRED_UDIV_CONTINUE]]
 ; CHECK:       pred.udiv.continue:
 ; CHECK-NEXT:    [[TMP7:%.*]] = phi <2 x i32> [ poison, [[VECTOR_BODY]] ], [ [[TMP6]], [[PRED_UDIV_IF]] ]
-; CHECK-NEXT:    [[TMP8:%.*]] = extractelement <2 x i1> [[BROADCAST_SPLAT]], i32 1
-; CHECK-NEXT:    br i1 [[TMP8]], label [[PRED_UDIV_IF1:%.*]], label [[PRED_UDIV_CONTINUE2]]
+; CHECK-NEXT:    br i1 [[C]], label [[PRED_UDIV_IF1:%.*]], label [[PRED_UDIV_CONTINUE2]]
 ; CHECK:       pred.udiv.if1:
 ; CHECK-NEXT:    [[TMP9:%.*]] = add i32 [[INDEX]], 1
 ; CHECK-NEXT:    [[TMP10:%.*]] = extractelement <2 x i32> [[WIDE_LOAD]], i32 1
@@ -2202,8 +2200,7 @@ define i32 @scalarize_induction_variable_05(ptr %a, i32 %x, i1 %c, i32 %n) {
 ; UNROLL-NO-IC-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i32 2
 ; UNROLL-NO-IC-NEXT:    [[WIDE_LOAD:%.*]] = load <2 x i32>, ptr [[TMP2]], align 4
 ; UNROLL-NO-IC-NEXT:    [[WIDE_LOAD2:%.*]] = load <2 x i32>, ptr [[TMP3]], align 4
-; UNROLL-NO-IC-NEXT:    [[TMP4:%.*]] = extractelement <2 x i1> [[BROADCAST_SPLAT]], i32 0
-; UNROLL-NO-IC-NEXT:    br i1 [[TMP4]], label [[PRED_UDIV_IF:%.*]], label [[PRED_UDIV_CONTINUE:%.*]]
+; UNROLL-NO-IC-NEXT:    br i1 [[C]], label [[PRED_UDIV_IF:%.*]], label [[PRED_UDIV_CONTINUE:%.*]]
 ; UNROLL-NO-IC:       pred.udiv.if:
 ; UNROLL-NO-IC-NEXT:    [[TMP5:%.*]] = extractelement <2 x i32> [[WIDE_LOAD]], i32 0
 ; UNROLL-NO-IC-NEXT:    [[TMP6:%.*]] = udiv i32 [[TMP5]], [[TMP0]]
@@ -2211,8 +2208,7 @@ define i32 @scalarize_induction_variable_05(ptr %a, i32 %x, i1 %c, i32 %n) {
 ; UNROLL-NO-IC-NEXT:    br label [[PRED_UDIV_CONTINUE]]
 ; UNROLL-NO-IC:       pred.udiv.continue:
 ; UNROLL-NO-IC-NEXT:    [[TMP8:%.*]] = phi <2 x i32> [ poison, [[VECTOR_BODY]] ], [ [[TMP7]], [[PRED_UDIV_IF]] ]
-; UNROLL-NO-IC-NEXT:    [[TMP9:%.*]] = extractelement <2 x i1> [[BROADCAST_SPLAT]], i32 1
-; UNROLL-NO-IC-NEXT:    br i1 [[TMP9]], label [[PRED_UDIV_IF3:%.*]], label [[PRED_UDIV_CONTINUE4:%.*]]
+; UNROLL-NO-IC-NEXT:    br i1 [[C]], label [[PRED_UDIV_IF3:%.*]], label [[PRED_UDIV_CONTINUE4:%.*]]
 ; UNROLL-NO-IC:       pred.udiv.if3:
 ; UNROLL-NO-IC-NEXT:    [[TMP10:%.*]] = add i32 [[INDEX]], 1
 ; UNROLL-NO-IC-NEXT:    [[TMP11:%.*]] = extractelement <2 x i32> [[WIDE_LOAD]], i32 1
@@ -2221,8 +2217,7 @@ define i32 @scalarize_induction_variable_05(ptr %a, i32 %x, i1 %c, i32 %n) {
 ; UNROLL-NO-IC-NEXT:    br label [[PRED_UDIV_CONTINUE4]]
 ; UNROLL-NO-IC:       pred.udiv.continue4:
 ; UNROLL-NO-IC-NEXT:    [[TMP14:%.*]] = phi <2 x i32> [ [[TMP8]], [[PRED_UDIV_CONTINUE]] ], [ [[TMP13]], [[PRED_UDIV_IF3]] ]
-; UNROLL-NO-IC-NEXT:    [[TMP15:%.*]] = extractelement <2 x i1> [[BROADCAST_SPLAT]], i32 0
-; UNROLL-NO-IC-NEXT:    br i1 [[TMP15]], label [[PRED_UDIV_IF5:%.*]], label [[PRED_UDIV_CONTINUE6:%.*]]
+; UNROLL-NO-IC-NEXT:    br i1 [[C]], label [[PRED_UDIV_IF5:%.*]], label [[PRED_UDIV_CONTINUE6:%.*]]
 ; UNROLL-NO-IC:       pred.udiv.if5:
 ; UNROLL-NO-IC-NEXT:    [[TMP16:%.*]] = add i32 [[INDEX]], 2
 ; UNROLL-NO-IC-NEXT:    [[TMP17:%.*]] = extractelement <2 x i32> [[WIDE_LOAD2]], i32 0
@@ -2231,8 +2226,7 @@ define i32 @scalarize_induction_variable_05(ptr %a, i32 %x, i1 %c, i32 %n) {
 ; UNROLL-NO-IC-NEXT:    br label [[PRED_UDIV_CONTINUE6]]
 ; UNROLL-NO-IC:       pred.udiv.continue6:
 ; UNROLL-NO-IC-NEXT:    [[TMP20:%.*]] = phi <2 x i32> [ poison, [[PRED_UDIV_CONTINUE4]] ], [ [[TMP19]], [[PRED_UDIV_IF5]] ]
-; UNROLL-NO-IC-NEXT:    [[TMP21:%.*]] = extractelement <2 x i1> [[BROADCAST_SPLAT]], i32 1
-; UNROLL-NO-IC-NEXT:    br i1 [[TMP21]], label [[PRED_UDIV_IF7:%.*]], label [[PRED_UDIV_CONTINUE8]]
+; UNROLL-NO-IC-NEXT:    br i1 [[C]], label [[PRED_UDIV_IF7:%.*]], label [[PRED_UDIV_CONTINUE8]]
 ; UNROLL-NO-IC:       pred.udiv.if7:
 ; UNROLL-NO-IC-NEXT:    [[TMP22:%.*]] = add i32 [[INDEX]], 3
 ; UNROLL-NO-IC-NEXT:    [[TMP23:%.*]] = extractelement <2 x i32> [[WIDE_LOAD2]], i32 1
@@ -3412,9 +3406,9 @@ define void @wrappingindvars1(i8 %t, i32 %len, ptr %A) {
 ; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i32 [[TMP0]], [[N_VEC]]
 ; CHECK-NEXT:    br i1 [[CMP_N]], label [[EXIT_LOOPEXIT:%.*]], label [[SCALAR_PH]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i8 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ [[T]], [[VECTOR_SCEVCHECK]] ], [ [[T]], [[LOOP_PREHEADER]] ]
-; CHECK-NEXT:    [[BC_RESUME_VAL1:%.*]] = phi i32 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[VECTOR_SCEVCHECK]] ], [ 0, [[LOOP_PREHEADER]] ]
-; CHECK-NEXT:    [[BC_RESUME_VAL3:%.*]] = phi i32 [ [[IND_END2]], [[MIDDLE_BLOCK]] ], [ [[EXT]], [[VECTOR_SCEVCHECK]] ], [ [[EXT]], [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i8 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ [[T]], [[LOOP_PREHEADER]] ], [ [[T]], [[VECTOR_SCEVCHECK]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL1:%.*]] = phi i32 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[LOOP_PREHEADER]] ], [ 0, [[VECTOR_SCEVCHECK]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL3:%.*]] = phi i32 [ [[IND_END2]], [[MIDDLE_BLOCK]] ], [ [[EXT]], [[LOOP_PREHEADER]] ], [ [[EXT]], [[VECTOR_SCEVCHECK]] ]
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
 ; CHECK-NEXT:    [[IDX:%.*]] = phi i8 [ [[IDX_INC:%.*]], [[LOOP]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
@@ -3472,14 +3466,14 @@ define void @wrappingindvars1(i8 %t, i32 %len, ptr %A) {
 ; IND-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2
 ; IND-NEXT:    [[VEC_IND_NEXT]] = add <2 x i32> [[VEC_IND]], splat (i32 2)
 ; IND-NEXT:    [[TMP12:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
-; IND-NEXT:    br i1 [[TMP12]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP36:![0-9]+]]
+; IND-NEXT:    br i1 [[TMP12]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP34:![0-9]+]]
 ; IND:       middle.block:
 ; IND-NEXT:    [[CMP_N:%.*]] = icmp eq i32 [[TMP0]], [[N_VEC]]
 ; IND-NEXT:    br i1 [[CMP_N]], label [[EXIT_LOOPEXIT:%.*]], label [[SCALAR_PH]]
 ; IND:       scalar.ph:
-; IND-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i8 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ [[T]], [[VECTOR_SCEVCHECK]] ], [ [[T]], [[LOOP_PREHEADER]] ]
-; IND-NEXT:    [[BC_RESUME_VAL1:%.*]] = phi i32 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[VECTOR_SCEVCHECK]] ], [ 0, [[LOOP_PREHEADER]] ]
-; IND-NEXT:    [[BC_RESUME_VAL3:%.*]] = phi i32 [ [[IND_END2]], [[MIDDLE_BLOCK]] ], [ [[EXT]], [[VECTOR_SCEVCHECK]] ], [ [[EXT]], [[LOOP_PREHEADER]] ]
+; IND-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i8 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ [[T]], [[LOOP_PREHEADER]] ], [ [[T]], [[VECTOR_SCEVCHECK]] ]
+; IND-NEXT:    [[BC_RESUME_VAL1:%.*]] = phi i32 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[LOOP_PREHEADER]] ], [ 0, [[VECTOR_SCEVCHECK]] ]
+; IND-NEXT:    [[BC_RESUME_VAL3:%.*]] = phi i32 [ [[IND_END2]], [[MIDDLE_BLOCK]] ], [ [[EXT]], [[LOOP_PREHEADER]] ], [ [[EXT]], [[VECTOR_SCEVCHECK]] ]
 ; IND-NEXT:    br label [[LOOP:%.*]]
 ; IND:       loop:
 ; IND-NEXT:    [[IDX:%.*]] = phi i8 [ [[IDX_INC:%.*]], [[LOOP]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
@@ -3546,9 +3540,9 @@ define void @wrappingindvars1(i8 %t, i32 %len, ptr %A) {
 ; UNROLL-NEXT:    [[CMP_N:%.*]] = icmp eq i32 [[TMP0]], [[N_VEC]]
 ; UNROLL-NEXT:    br i1 [[CMP_N]], label [[EXIT_LOOPEXIT:%.*]], label [[SCALAR_PH]]
 ; UNROLL:       scalar.ph:
-; UNROLL-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i8 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ [[T]], [[VECTOR_SCEVCHECK]] ], [ [[T]], [[LOOP_PREHEADER]] ]
-; UNROLL-NEXT:    [[BC_RESUME_VAL1:%.*]] = phi i32 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[VECTOR_SCEVCHECK]] ], [ 0, [[LOOP_PREHEADER]] ]
-; UNROLL-NEXT:    [[BC_RESUME_VAL3:%.*]] = phi i32 [ [[IND_END2]], [[MIDDLE_BLOCK]] ], [ [[EXT]], [[VECTOR_SCEVCHECK]] ], [ [[EXT]], [[LOOP_PREHEADER]] ]
+; UNROLL-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i8 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ [[T]], [[LOOP_PREHEADER]] ], [ [[T]], [[VECTOR_SCEVCHECK]] ]
+; UNROLL-NEXT:    [[BC_RESUME_VAL1:%.*]] = phi i32 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[LOOP_PREHEADER]] ], [ 0, [[VECTOR_SCEVCHECK]] ]
+; UNROLL-NEXT:    [[BC_RESUME_VAL3:%.*]] = phi i32 [ [[IND_END2]], [[MIDDLE_BLOCK]] ], [ [[EXT]], [[LOOP_PREHEADER]] ], [ [[EXT]], [[VECTOR_SCEVCHECK]] ]
 ; UNROLL-NEXT:    br label [[LOOP:%.*]]
 ; UNROLL:       loop:
 ; UNROLL-NEXT:    [[IDX:%.*]] = phi i8 [ [[IDX_INC:%.*]], [[LOOP]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
@@ -3620,9 +3614,9 @@ define void @wrappingindvars1(i8 %t, i32 %len, ptr %A) {
 ; UNROLL-NO-IC-NEXT:    [[CMP_N:%.*]] = icmp eq i32 [[TMP0]], [[N_VEC]]
 ; UNROLL-NO-IC-NEXT:    br i1 [[CMP_N]], label [[EXIT_LOOPEXIT:%.*]], label [[SCALAR_PH]]
 ; UNROLL-NO-IC:       scalar.ph:
-; UNROLL-NO-IC-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i8 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ [[T]], [[VECTOR_SCEVCHECK]] ], [ [[T]], [[LOOP_PREHEADER]] ]
-; UNROLL-NO-IC-NEXT:    [[BC_RESUME_VAL1:%.*]] = phi i32 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[VECTOR_SCEVCHECK]] ], [ 0, [[LOOP_PREHEADER]] ]
-; UNROLL-NO-IC-NEXT:    [[BC_RESUME_VAL3:%.*]] = phi i32 [ [[IND_END2]], [[MIDDLE_BLOCK]] ], [ [[EXT]], [[VECTOR_SCEVCHECK]] ], [ [[EXT]], [[LOOP_PREHEADER]] ]
+; UNROLL-NO-IC-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i8 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ [[T]], [[LOOP_PREHEADER]] ], [ [[T]], [[VECTOR_SCEVCHECK]] ]
+; UNROLL-NO-IC-NEXT:    [[BC_RESUME_VAL1:%.*]] = phi i32 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[LOOP_PREHEADER]] ], [ 0, [[VECTOR_SCEVCHECK]] ]
+; UNROLL-NO-IC-NEXT:    [[BC_RESUME_VAL3:%.*]] = phi i32 [ [[IND_END2]], [[MIDDLE_BLOCK]] ], [ [[EXT]], [[LOOP_PREHEADER]] ], [ [[EXT]], [[VECTOR_SCEVCHECK]] ]
 ; UNROLL-NO-IC-NEXT:    br label [[LOOP:%.*]]
 ; UNROLL-NO-IC:       loop:
 ; UNROLL-NO-IC-NEXT:    [[IDX:%.*]] = phi i8 [ [[IDX_INC:%.*]], [[LOOP]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
@@ -3688,9 +3682,9 @@ define void @wrappingindvars1(i8 %t, i32 %len, ptr %A) {
 ; INTERLEAVE-NEXT:    [[CMP_N:%.*]] = icmp eq i32 [[TMP0]], [[N_VEC]]
 ; INTERLEAVE-NEXT:    br i1 [[CMP_N]], label [[EXIT_LOOPEXIT:%.*]], label [[SCALAR_PH]]
 ; INTERLEAVE:       scalar.ph:
-; INTERLEAVE-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i8 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ [[T]], [[VECTOR_SCEVCHECK]] ], [ [[T]], [[LOOP_PREHEADER]] ]
-; INTERLEAVE-NEXT:    [[BC_RESUME_VAL1:%.*]] = phi i32 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[VECTOR_SCEVCHECK]] ], [ 0, [[LOOP_PREHEADER]] ]
-; INTERLEAVE-NEXT:    [[BC_RESUME_VAL3:%.*]] = phi i32 [ [[IND_END2]], [[MIDDLE_BLOCK]] ], [ [[EXT]], [[VECTOR_SCEVCHECK]] ], [ [[EXT]], [[LOOP_PREHEADER]] ]
+; INTERLEAVE-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i8 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ [[T]], [[LOOP_PREHEADER]] ], [ [[T]], [[VECTOR_SCEVCHECK]] ]
+; INTERLEAVE-NEXT:    [[BC_RESUME_VAL1:%.*]] = phi i32 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[LOOP_PREHEADER]] ], [ 0, [[VECTOR_SCEVCHECK]] ]
+; INTERLEAVE-NEXT:    [[BC_RESUME_VAL3:%.*]] = phi i32 [ [[IND_END2]], [[MIDDLE_BLOCK]] ], [ [[EXT]], [[LOOP_PREHEADER]] ], [ [[EXT]], [[VECTOR_SCEVCHECK]] ]
 ; INTERLEAVE-NEXT:    br label [[LOOP:%.*]]
 ; INTERLEAVE:       loop:
 ; INTERLEAVE-NEXT:    [[IDX:%.*]] = phi i8 [ [[IDX_INC:%.*]], [[LOOP]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
@@ -3792,9 +3786,9 @@ define void @wrappingindvars2(i8 %t, i32 %len, ptr %A) {
 ; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i32 [[TMP0]], [[N_VEC]]
 ; CHECK-NEXT:    br i1 [[CMP_N]], label [[EXIT_LOOPEXIT:%.*]], label [[SCALAR_PH]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i8 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ [[T]], [[VECTOR_SCEVCHECK]] ], [ [[T]], [[LOOP_PREHEADER]] ]
-; CHECK-NEXT:    [[BC_RESUME_VAL2:%.*]] = phi i32 [ [[IND_END1]], [[MIDDLE_BLOCK]] ], [ [[EXT_MUL]], [[VECTOR_SCEVCHECK]] ], [ [[EXT_MUL]], [[LOOP_PREHEADER]] ]
-; CHECK-NEXT:    [[BC_RESUME_VAL3:%.*]] = phi i32 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[VECTOR_SCEVCHECK]] ], [ 0, [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i8 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ [[T]], [[LOOP_PREHEADER]] ], [ [[T]], [[VECTOR_SCEVCHECK]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL2:%.*]] = phi i32 [ [[IND_END1]], [[MIDDLE_BLOCK]] ], [ [[EXT_MUL]], [[LOOP_PREHEADER]] ], [ [[EXT_MUL]], [[VECTOR_SCEVCHECK]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL3:%.*]] = phi i32 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[LOOP_PREHEADER]] ], [ 0, [[VECTOR_SCEVCHECK]] ]
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
 ; CHECK-NEXT:    [[IDX:%.*]] = phi i8 [ [[IDX_INC:%.*]], [[LOOP]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
@@ -3860,9 +3854,9 @@ define void @wrappingindvars2(i8 %t, i32 %len, ptr %A) {
 ; IND-NEXT:    [[CMP_N:%.*]] = icmp eq i32 [[TMP0]], [[N_VEC]]
 ; IND-NEXT:    br i1 [[CMP_N]], label [[EXIT_LOOPEXIT:%.*]], label [[SCALAR_PH]]
 ; IND:       scalar.ph:
-; IND-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i8 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ [[T]], [[VECTOR_SCEVCHECK]] ], [ [[T]], [[LOOP_PREHEADER]] ]
-; IND-NEXT:    [[BC_RESUME_VAL2:%.*]] = phi i32 [ [[IND_END1]], [[MIDDLE_BLOCK]] ], [ [[EXT_MUL]], [[VECTOR_SCEVCHECK]] ], [ [[EXT_MUL]], [[LOOP_PREHEADER]] ]
-; IND-NEXT:    [[BC_RESUME_VAL3:%.*]] = phi i32 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[VECTOR_SCEVCHECK]] ], [ 0, [[LOOP_PREHEADER]] ]
+; IND-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i8 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ [[T]], [[LOOP_PREHEADER]] ], [ [[T]], [[VECTOR_SCEVCHECK]] ]
+; IND-NEXT:    [[BC_RESUME_VAL2:%.*]] = phi i32 [ [[IND_END1]], [[MIDDLE_BLOCK]] ], [ [[EXT_MUL]], [[LOOP_PREHEADER]] ], [ [[EXT_MUL]], [[VECTOR_SCEVCHECK]] ]
+; IND-NEXT:    [[BC_RESUME_VAL3:%.*]] = phi i32 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[LOOP_PREHEADER]] ], [ 0, [[VECTOR_SCEVCHECK]] ]
 ; IND-NEXT:    br label [[LOOP:%.*]]
 ; IND:       loop:
 ; IND-NEXT:    [[IDX:%.*]] = phi i8 [ [[IDX_INC:%.*]], [[LOOP]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
@@ -3932,9 +3926,9 @@ define void @wrappingindvars2(i8 %t, i32 %len, ptr %A) {
 ; UNROLL-NEXT:    [[CMP_N:%.*]] = icmp eq i32 [[TMP0]], [[N_VEC]]
 ; UNROLL-NEXT:    br i1 [[CMP_N]], label [[EXIT_LOOPEXIT:%.*]], label [[SCALAR_PH]]
 ; UNROLL:       scalar.ph:
-; UNROLL-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i8 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ [[T]], [[VECTOR_SCEVCHECK]] ], [ [[T]], [[LOOP_PREHEADER]] ]
-; UNROLL-NEXT:    [[BC_RESUME_VAL2:%.*]] = phi i32 [ [[IND_END1]], [[MIDDLE_BLOCK]] ], [ [[EXT_MUL]], [[VECTOR_SCEVCHECK]] ], [ [[EXT_MUL]], [[LOOP_PREHEADER]] ]
-; UNROLL-NEXT:    [[BC_RESUME_VAL3:%.*]] = phi i32 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[VECTOR_SCEVCHECK]] ], [ 0, [[LOOP_PREHEADER]] ]
+; UNROLL-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i8 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ [[T]], [[LOOP_PREHEADER]] ], [ [[T]], [[VECTOR_SCEVCHECK]] ]
+; UNROLL-NEXT:    [[BC_RESUME_VAL2:%.*]] = phi i32 [ [[IND_END1]], [[MIDDLE_BLOCK]] ], [ [[EXT_MUL]], [[LOOP_PREHEADER]] ], [ [[EXT_MUL]], [[VECTOR_SCEVCHECK]] ]
+; UNROLL-NEXT:    [[BC_RESUME_VAL3:%.*]] = phi i32 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[LOOP_PREHEADER]] ], [ 0, [[VECTOR_SCEVCHECK]] ]
 ; UNROLL-NEXT:    br label [[LOOP:%.*]]
 ; UNROLL:       loop:
 ; UNROLL-NEXT:    [[IDX:%.*]] = phi i8 [ [[IDX_INC:%.*]], [[LOOP]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
@@ -4009,9 +4003,9 @@ define void @wrappingindvars2(i8 %t, i32 %len, ptr %A) {
 ; UNROLL-NO-IC-NEXT:    [[CMP_N:%.*]] = icmp eq i32 [[TMP0]], [[N_VEC]]
 ; UNROLL-NO-IC-NEXT:    br i1 [[CMP_N]], label [[EXIT_LOOPEXIT:%.*]], label [[SCALAR_PH]]
 ; UNROLL-NO-IC:       scalar.ph:
-; UNROLL-NO-IC-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i8 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ [[T]], [[VECTOR_SCEVCHECK]] ], [ [[T]], [[LOOP_PREHEADER]] ]
-; UNROLL-NO-IC-NEXT:    [[BC_RESUME_VAL2:%.*]] = phi i32 [ [[IND_END1]], [[MIDDLE_BLOCK]] ], [ [[EXT_MUL]], [[VECTOR_SCEVCHECK]] ], [ [[EXT_MUL]], [[LOOP_PREHEADER]] ]
-; UNROLL-NO-IC-NEXT:    [[BC_RESUME_VAL3:%.*]] = phi i32 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[VECTOR_SCEVCHECK]] ], [ 0, [[LOOP_PREHEADER]] ]
+; UNROLL-NO-IC-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i8 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ [[T]], [[LOOP_PREHEADER]] ], [ [[T]], [[VECTOR_SCEVCHECK]] ]
+; UNROLL-NO-IC-NEXT:    [[BC_RESUME_VAL2:%.*]] = phi i32 [ [[IND_END1]], [[MIDDLE_BLOCK]] ], [ [[EXT_MUL]], [[LOOP_PREHEADER]] ], [ [[EXT_MUL]], [[VECTOR_SCEVCHECK]] ]
+; UNROLL-NO-IC-NEXT:    [[BC_RESUME_VAL3:%.*]] = phi i32 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[LOOP_PREHEADER]] ], [ 0, [[VECTOR_SCEVCHECK]] ]
 ; UNROLL-NO-IC-NEXT:    br label [[LOOP:%.*]]
 ; UNROLL-NO-IC:       loop:
 ; UNROLL-NO-IC-NEXT:    [[IDX:%.*]] = phi i8 [ [[IDX_INC:%.*]], [[LOOP]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
@@ -4080,9 +4074,9 @@ define void @wrappingindvars2(i8 %t, i32 %len, ptr %A) {
 ; INTERLEAVE-NEXT:    [[CMP_N:%.*]] = icmp eq i32 [[TMP0]], [[N_VEC]]
 ; INTERLEAVE-NEXT:    br i1 [[CMP_N]], label [[EXIT_LOOPEXIT:%.*]], label [[SCALAR_PH]]
 ; INTERLEAVE:       scalar.ph:
-; INTERLEAVE-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i8 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ [[T]], [[VECTOR_SCEVCHECK]] ], [ [[T]], [[LOOP_PREHEADER]] ]
-; INTERLEAVE-NEXT:    [[BC_RESUME_VAL2:%.*]] = phi i32 [ [[IND_END1]], [[MIDDLE_BLOCK]] ], [ [[EXT_MUL]], [[VECTOR_SCEVCHECK]] ], [ [[EXT_MUL]], [[LOOP_PREHEADER]] ]
-; INTERLEAVE-NEXT:    [[BC_RESUME_VAL3:%.*]] = phi i32 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[VECTOR_SCEVCHECK]] ], [ 0, [[LOOP_PREHEADER]] ]
+; INTERLEAVE-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i8 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ [[T]], [[LOOP_PREHEADER]] ], [ [[T]], [[VECTOR_SCEVCHECK]] ]
+; INTERLEAVE-NEXT:    [[BC_RESUME_VAL2:%.*]] = phi i32 [ [[IND_END1]], [[MIDDLE_BLOCK]] ], [ [[EXT_MUL]], [[LOOP_PREHEADER]] ], [ [[EXT_MUL]], [[VECTOR_SCEVCHECK]] ]
+; INTERLEAVE-NEXT:    [[BC_RESUME_VAL3:%.*]] = phi i32 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[LOOP_PREHEADER]] ], [ 0, [[VECTOR_SCEVCHECK]] ]
 ; INTERLEAVE-NEXT:    br label [[LOOP:%.*]]
 ; INTERLEAVE:       loop:
 ; INTERLEAVE-NEXT:    [[IDX:%.*]] = phi i8 [ [[IDX_INC:%.*]], [[LOOP]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
@@ -4363,7 +4357,7 @@ define void @trunciv(ptr nocapture %a, i32 %start, i64 %k) {
 ; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[K]], [[N_VEC]]
 ; CHECK-NEXT:    br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[VECTOR_SCEVCHECK]] ], [ 0, [[FOR_BODY_PREHEADER:%.*]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[FOR_BODY_PREHEADER:%.*]] ], [ 0, [[VECTOR_SCEVCHECK]] ]
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK:       for.body:
 ; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
@@ -4401,7 +4395,7 @@ define void @trunciv(ptr nocapture %a, i32 %start, i64 %k) {
 ; IND-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[K]], [[N_VEC]]
 ; IND-NEXT:    br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
 ; IND:       scalar.ph:
-; IND-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[VECTOR_SCEVCHECK]] ], [ 0, [[FOR_BODY_PREHEADER:%.*]] ]
+; IND-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[FOR_BODY_PREHEADER:%.*]] ], [ 0, [[VECTOR_SCEVCHECK]] ]
 ; IND-NEXT:    br label [[FOR_BODY:%.*]]
 ; IND:       for.body:
 ; IND-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
@@ -4444,7 +4438,7 @@ define void @trunciv(ptr nocapture %a, i32 %start, i64 %k) {
 ; UNROLL-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[K]], [[N_VEC]]
 ; UNROLL-NEXT:    br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
 ; UNROLL:       scalar.ph:
-; UNROLL-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[VECTOR_SCEVCHECK]] ], [ 0, [[FOR_BODY_PREHEADER:%.*]] ]
+; UNROLL-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[FOR_BODY_PREHEADER:%.*]] ], [ 0, [[VECTOR_SCEVCHECK]] ]
 ; UNROLL-NEXT:    br label [[FOR_BODY:%.*]]
 ; UNROLL:       for.body:
 ; UNROLL-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
@@ -4493,7 +4487,7 @@ define void @trunciv(ptr nocapture %a, i32 %start, i64 %k) {
 ; UNROLL-NO-IC-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[K]], [[N_VEC]]
 ; UNROLL-NO-IC-NEXT:    br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
 ; UNROLL-NO-IC:       scalar.ph:
-; UNROLL-NO-IC-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[VECTOR_SCEVCHECK]] ], [ 0, [[FOR_BODY_PREHEADER:%.*]] ]
+; UNROLL-NO-IC-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[FOR_BODY_PREHEADER:%.*]] ], [ 0, [[VECTOR_SCEVCHECK]] ]
 ; UNROLL-NO-IC-NEXT:    br label [[FOR_BODY:%.*]]
 ; UNROLL-NO-IC:       for.body:
 ; UNROLL-NO-IC-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
@@ -4534,7 +4528,7 @@ define void @trunciv(ptr nocapture %a, i32 %start, i64 %k) {
 ; INTERLEAVE-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[K]], [[N_VEC]]
 ; INTERLEAVE-NEXT:    br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
 ; INTERLEAVE:       scalar.ph:
-; INTERLEAVE-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[VECTOR_SCEVCHECK]] ], [ 0, [[FOR_BODY_PREHEADER:%.*]] ]
+; INTERLEAVE-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[FOR_BODY_PREHEADER:%.*]] ], [ 0, [[VECTOR_SCEVCHECK]] ]
 ; INTERLEAVE-NEXT:    br label [[FOR_BODY:%.*]]
 ; INTERLEAVE:       for.body:
 ; INTERLEAVE-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
@@ -6176,9 +6170,9 @@ define void @test_optimized_cast_induction_feeding_first_order_recurrence(i64 %n
 ; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
 ; CHECK-NEXT:    br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[SCALAR_RECUR_INIT:%.*]] = phi i32 [ [[VECTOR_RECUR_EXTRACT]], [[MIDDLE_BLOCK]] ], [ 0, [[VECTOR_SCEVCHECK]] ], [ 0, [[ENTRY:%.*]] ]
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[VECTOR_SCEVCHECK]] ], [ 0, [[ENTRY]] ]
-; CHECK-NEXT:    [[BC_RESUME_VAL1:%.*]] = phi i32 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ 0, [[VECTOR_SCEVCHECK]] ], [ 0, [[ENTRY]] ]
+; CHECK-NEXT:    [[SCALAR_RECUR_INIT:%.*]] = phi i32 [ [[VECTOR_RECUR_EXTRACT]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ], [ 0, [[VECTOR_SCEVCHECK]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ], [ 0, [[VECTOR_SCEVCHECK]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL1:%.*]] = phi i32 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ], [ 0, [[VECTOR_SCEVCHECK]] ]
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
 ; CHECK-NEXT:    [[FOR:%.*]] = phi i32 [ [[SCALAR_RECUR_INIT]], [[SCALAR_PH]] ], [ [[IV_2_CONV:%.*]], [[LOOP]] ]
@@ -6247,9 +6241,9 @@ define void @test_optimized_cast_induction_feeding_first_order_recurrence(i64 %n
 ; IND-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
 ; IND-NEXT:    br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
 ; IND:       scalar.ph:
-; IND-NEXT:    [[SCALAR_RECUR_INIT:%.*]] = phi i32 [ [[VECTOR_RECUR_EXTRACT]], [[MIDDLE_BLOCK]] ], [ 0, [[VECTOR_SCEVCHECK]] ], [ 0, [[ENTRY:%.*]] ]
-; IND-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[VECTOR_SCEVCHECK]] ], [ 0, [[ENTRY]] ]
-; IND-NEXT:    [[BC_RESUME_VAL1:%.*]] = phi i32 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ 0, [[VECTOR_SCEVCHECK]] ], [ 0, [[ENTRY]] ]
+; IND-NEXT:    [[SCALAR_RECUR_INIT:%.*]] = phi i32 [ [[VECTOR_RECUR_EXTRACT]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ], [ 0, [[VECTOR_SCEVCHECK]] ]
+; IND-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ], [ 0, [[VECTOR_SCEVCHECK]] ]
+; IND-NEXT:    [[BC_RESUME_VAL1:%.*]] = phi i32 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ], [ 0, [[VECTOR_SCEVCHECK]] ]
 ; IND-NEXT:    br label [[LOOP:%.*]]
 ; IND:       loop:
 ; IND-NEXT:    [[FOR:%.*]] = phi i32 [ [[SCALAR_RECUR_INIT]], [[SCALAR_PH]] ], [ [[IV_2_CONV:%.*]], [[LOOP]] ]
@@ -6322,9 +6316,9 @@ define void @test_optimized_cast_induction_feeding_first_order_recurrence(i64 %n
 ; UNROLL-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
 ; UNROLL-NEXT:    br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
 ; UNROLL:       scalar.ph:
-; UNROLL-NEXT:    [[SCALAR_RECUR_INIT:%.*]] = phi i32 [ [[VECTOR_RECUR_EXTRACT]], [[MIDDLE_BLOCK]] ], [ 0, [[VECTOR_SCEVCHECK]] ], [ 0, [[ENTRY:%.*]] ]
-; UNROLL-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[VECTOR_SCEVCHECK]] ], [ 0, [[ENTRY]] ]
-; UNROLL-NEXT:    [[BC_RESUME_VAL1:%.*]] = phi i32 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ 0, [[VECTOR_SCEVCHECK]] ], [ 0, [[ENTRY]] ]
+; UNROLL-NEXT:    [[SCALAR_RECUR_INIT:%.*]] = phi i32 [ [[VECTOR_RECUR_EXTRACT]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ], [ 0, [[VECTOR_SCEVCHECK]] ]
+; UNROLL-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ], [ 0, [[VECTOR_SCEVCHECK]] ]
+; UNROLL-NEXT:    [[BC_RESUME_VAL1:%.*]] = phi i32 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ], [ 0, [[VECTOR_SCEVCHECK]] ]
 ; UNROLL-NEXT:    br label [[LOOP:%.*]]
 ; UNROLL:       loop:
 ; UNROLL-NEXT:    [[FOR:%.*]] = phi i32 [ [[SCALAR_RECUR_INIT]], [[SCALAR_PH]] ], [ [[IV_2_CONV:%.*]], [[LOOP]] ]
@@ -6403,9 +6397,9 @@ define void @test_optimized_cast_induction_feeding_first_order_recurrence(i64 %n
 ; UNROLL-NO-IC-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
 ; UNROLL-NO-IC-NEXT:    br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
 ; UNROLL-NO-IC:       scalar.ph:
-; UNROLL-NO-IC-NEXT:    [[SCALAR_RECUR_INIT:%.*]] = phi i32 [ [[VECTOR_RECUR_EXTRACT]], [[MIDDLE_BLOCK]] ], [ 0, [[VECTOR_SCEVCHECK]] ], [ 0, [[ENTRY:%.*]] ]
-; UNROLL-NO-IC-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[VECTOR_SCEVCHECK]] ], [ 0, [[ENTRY]] ]
-; UNROLL-NO-IC-NEXT:    [[BC_RESUME_VAL1:%.*]] = phi i32 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ 0, [[VECTOR_SCEVCHECK]] ], [ 0, [[ENTRY]] ]
+; UNROLL-NO-IC-NEXT:    [[SCALAR_RECUR_INIT:%.*]] = phi i32 [ [[VECTOR_RECUR_EXTRACT]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ], [ 0, [[VECTOR_SCEVCHECK]] ]
+; UNROLL-NO-IC-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ], [ 0, [[VECTOR_SCEVCHECK]] ]
+; UNROLL-NO-IC-NEXT:    [[BC_RESUME_VAL1:%.*]] = phi i32 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ], [ 0, [[VECTOR_SCEVCHECK]] ]
 ; UNROLL-NO-IC-NEXT:    br label [[LOOP:%.*]]
 ; UNROLL-NO-IC:       loop:
 ; UNROLL-NO-IC-NEXT:    [[FOR:%.*]] = phi i32 [ [[SCALAR_RECUR_INIT]], [[SCALAR_PH]] ], [ [[IV_2_CONV:%.*]], [[LOOP]] ]
@@ -6478,9 +6472,9 @@ define void @test_optimized_cast_induction_feeding_first_order_recurrence(i64 %n
 ; INTERLEAVE-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
 ; INTERLEAVE-NEXT:    br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
 ; INTERLEAVE:       scalar.ph:
-; INTERLEAVE-NEXT:    [[SCALAR_RECUR_INIT:%.*]] = phi i32 [ [[VECTOR_RECUR_EXTRACT]], [[MIDDLE_BLOCK]] ], [ 0, [[VECTOR_SCEVCHECK]] ], [ 0, [[ENTRY:%.*]] ]
-; INTERLEAVE-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[VECTOR_SCEVCHECK]] ], [ 0, [[ENTRY]] ]
-; INTERLEAVE-NEXT:    [[BC_RESUME_VAL1:%.*]] = phi i32 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ 0, [[VECTOR_SCEVCHECK]] ], [ 0, [[ENTRY]] ]
+; INTERLEAVE-NEXT:    [[SCALAR_RECUR_INIT:%.*]] = phi i32 [ [[VECTOR_RECUR_EXTRACT]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ], [ 0, [[VECTOR_SCEVCHECK]] ]
+; INTERLEAVE-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ], [ 0, [[VECTOR_SCEVCHECK]] ]
+; INTERLEAVE-NEXT:    [[BC_RESUME_VAL1:%.*]] = phi i32 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ], [ 0, [[VECTOR_SCEVCHECK]] ]
 ; INTERLEAVE-NEXT:    br label [[LOOP:%.*]]
 ; INTERLEAVE:       loop:
 ; INTERLEAVE-NEXT:    [[FOR:%.*]] = phi i32 [ [[SCALAR_RECUR_INIT]], [[SCALAR_PH]] ], [ [[IV_2_CONV:%.*]], [[LOOP]] ]
diff --git a/llvm/test/Transforms/LoopVectorize/interleaved-accesses-2.ll b/llvm/test/Transforms/LoopVectorize/interleaved-accesses-2.ll
index 435da2c17d152..b8eb5311b4bb2 100644
--- a/llvm/test/Transforms/LoopVectorize/interleaved-accesses-2.ll
+++ b/llvm/test/Transforms/LoopVectorize/interleaved-accesses-2.ll
@@ -24,7 +24,7 @@ target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 
 define void @_Z4testPfS_m(ptr noalias nocapture %out, ptr noalias nocapture readonly %in, i64 %size) local_unnamed_addr {
 ; CHECK-LABEL: define void @_Z4testPfS_m(
-; CHECK-SAME: ptr noalias nocapture [[OUT:%.*]], ptr noalias nocapture readonly [[IN:%.*]], i64 [[SIZE:%.*]]) local_unnamed_addr {
+; CHECK-SAME: ptr noalias captures(none) [[OUT:%.*]], ptr noalias readonly captures(none) [[IN:%.*]], i64 [[SIZE:%.*]]) local_unnamed_addr {
 ; CHECK-NEXT:  [[ENTRY:.*:]]
 ; CHECK-NEXT:    [[CMP7:%.*]] = icmp eq i64 [[SIZE]], 0
 ; CHECK-NEXT:    br i1 [[CMP7]], label %[[FOR_COND_CLEANUP:.*]], label %[[FOR_BODY_PREHEADER:.*]]
diff --git a/llvm/test/Transforms/LoopVectorize/interleaved-accesses-3.ll b/llvm/test/Transforms/LoopVectorize/interleaved-accesses-3.ll
index e101a61ba4148..661e8eb666d54 100644
--- a/llvm/test/Transforms/LoopVectorize/interleaved-accesses-3.ll
+++ b/llvm/test/Transforms/LoopVectorize/interleaved-accesses-3.ll
@@ -23,7 +23,7 @@ target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 
 define void @_Z4funcPjS_hh(ptr noalias nocapture readonly %a, ptr noalias nocapture %b, i8 zeroext %x, i8 zeroext %y) local_unnamed_addr {
 ; CHECK-LABEL: define void @_Z4funcPjS_hh(
-; CHECK-SAME: ptr noalias nocapture readonly [[A:%.*]], ptr noalias nocapture [[B:%.*]], i8 zeroext [[X:%.*]], i8 zeroext [[Y:%.*]]) local_unnamed_addr {
+; CHECK-SAME: ptr noalias readonly captures(none) [[A:%.*]], ptr noalias captures(none) [[B:%.*]], i8 zeroext [[X:%.*]], i8 zeroext [[Y:%.*]]) local_unnamed_addr {
 ; CHECK-NEXT:  [[ENTRY:.*:]]
 ; CHECK-NEXT:    [[CMP9:%.*]] = icmp eq i8 [[Y]], 0
 ; CHECK-NEXT:    br i1 [[CMP9]], label %[[FOR_COND_CLEANUP:.*]], label %[[FOR_BODY_PREHEADER:.*]]
@@ -67,8 +67,8 @@ define void @_Z4funcPjS_hh(ptr noalias nocapture readonly %a, ptr noalias nocapt
 ; CHECK:       [[MIDDLE_BLOCK]]:
 ; CHECK-NEXT:    br label %[[SCALAR_PH]]
 ; CHECK:       [[SCALAR_PH]]:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[VECTOR_SCEVCHECK]] ], [ 0, %[[FOR_BODY_PREHEADER]] ]
-; CHECK-NEXT:    [[BC_RESUME_VAL3:%.*]] = phi i8 [ [[IND_END]], %[[MIDDLE_BLOCK]] ], [ [[X]], %[[VECTOR_SCEVCHECK]] ], [ [[X]], %[[FOR_BODY_PREHEADER]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[FOR_BODY_PREHEADER]] ], [ 0, %[[VECTOR_SCEVCHECK]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL3:%.*]] = phi i8 [ [[IND_END]], %[[MIDDLE_BLOCK]] ], [ [[X]], %[[FOR_BODY_PREHEADER]] ], [ [[X]], %[[VECTOR_SCEVCHECK]] ]
 ; CHECK-NEXT:    br label %[[FOR_BODY:.*]]
 ; CHECK:       [[FOR_COND_CLEANUP_LOOPEXIT:.*]]:
 ; CHECK-NEXT:    br label %[[FOR_COND_CLEANUP]]
diff --git a/llvm/test/Transforms/LoopVectorize/interleaved-accesses.ll b/llvm/test/Transforms/LoopVectorize/interleaved-accesses.ll
index abd91d33157e6..e8ae770ff4c0f 100644
--- a/llvm/test/Transforms/LoopVectorize/interleaved-accesses.ll
+++ b/llvm/test/Transforms/LoopVectorize/interleaved-accesses.ll
@@ -1508,9 +1508,9 @@ define void @PR34743(ptr %a, ptr %b, i64 %n) {
 ; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[TMP1]], [[N_VEC]]
 ; CHECK-NEXT:    br i1 [[CMP_N]], label [[END:%.*]], label [[SCALAR_PH]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[SCALAR_RECUR_INIT:%.*]] = phi i16 [ [[VECTOR_RECUR_EXTRACT]], [[MIDDLE_BLOCK]] ], [ [[DOTPRE]], [[VECTOR_MEMCHECK]] ], [ [[DOTPRE]], [[ENTRY:%.*]] ]
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ 0, [[VECTOR_MEMCHECK]] ], [ 0, [[ENTRY]] ]
-; CHECK-NEXT:    [[BC_RESUME_VAL3:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[VECTOR_MEMCHECK]] ], [ 0, [[ENTRY]] ]
+; CHECK-NEXT:    [[SCALAR_RECUR_INIT:%.*]] = phi i16 [ [[VECTOR_RECUR_EXTRACT]], [[MIDDLE_BLOCK]] ], [ [[DOTPRE]], [[ENTRY:%.*]] ], [ [[DOTPRE]], [[VECTOR_MEMCHECK]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ], [ 0, [[VECTOR_MEMCHECK]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL3:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ], [ 0, [[VECTOR_MEMCHECK]] ]
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
 ; CHECK-NEXT:    [[TMP16:%.*]] = phi i16 [ [[SCALAR_RECUR_INIT]], [[SCALAR_PH]] ], [ [[LOAD2:%.*]], [[LOOP]] ]
diff --git a/llvm/test/Transforms/LoopVectorize/invalidate-scev-at-scope-after-vectorization.ll b/llvm/test/Transforms/LoopVectorize/invalidate-scev-at-scope-after-vectorization.ll
new file mode 100644
index 0000000000000..235a8f0fa34a8
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/invalidate-scev-at-scope-after-vectorization.ll
@@ -0,0 +1,118 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -passes='print<scalar-evolution>,loop-vectorize' -force-vector-width=4 -scalar-evolution-classify-expressions=false -S %s | FileCheck %s
+
+; Test case for https://github.com/llvm/llvm-project/issues/119665.
+
+; %loop.2's backedge-taken-count depends on %add.1 from %loop.1 via its
+; corresponding SCEV at the scope of %loop.2. After vectorizing %loop.1, %add.1
+; isn't available at the entry of %loop.2 anymore and %add.1 at %loop.2's scope
+; must be invalidated, as well as %loop.2's backedge-taken count.
+define void @test_invalidate_scevs_at_scope(ptr %p) {
+; CHECK-LABEL: define void @test_invalidate_scevs_at_scope(
+; CHECK-SAME: ptr [[P:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK:       [[VECTOR_PH]]:
+; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK:       [[VECTOR_BODY]]:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <4 x i32> [ <i32 0, i32 1, i32 2, i32 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP0:%.*]] = load i32, ptr [[P]], align 4
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[TMP0]], i64 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP1:%.*]] = add <4 x i32> [[BROADCAST_SPLAT]], [[VEC_IND]]
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
+; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <4 x i32> [[VEC_IND]], splat (i32 4)
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp eq i32 [[INDEX_NEXT]], 100
+; CHECK-NEXT:    br i1 [[TMP2]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK:       [[MIDDLE_BLOCK]]:
+; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <4 x i32> [[TMP1]], i32 3
+; CHECK-NEXT:    br i1 false, label %[[EXIT_1:.*]], label %[[SCALAR_PH]]
+; CHECK:       [[SCALAR_PH]]:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ 100, %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT:    br label %[[LOOP_1:.*]]
+; CHECK:       [[LOOP_1]]:
+; CHECK-NEXT:    [[IV_1:%.*]] = phi i32 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_1_NEXT:%.*]], %[[LOOP_1]] ]
+; CHECK-NEXT:    [[TMP4:%.*]] = load i32, ptr [[P]], align 4
+; CHECK-NEXT:    [[ADD_1:%.*]] = add i32 [[TMP4]], [[IV_1]]
+; CHECK-NEXT:    [[IV_1_NEXT]] = add i32 [[IV_1]], 1
+; CHECK-NEXT:    [[C_1:%.*]] = icmp eq i32 [[IV_1]], 100
+; CHECK-NEXT:    br i1 [[C_1]], label %[[EXIT_1]], label %[[LOOP_1]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK:       [[EXIT_1]]:
+; CHECK-NEXT:    [[ADD_LCSSA:%.*]] = phi i32 [ [[ADD_1]], %[[LOOP_1]] ], [ [[TMP3]], %[[MIDDLE_BLOCK]] ]
+; CHECK-NEXT:    [[SMAX:%.*]] = call i32 @llvm.smax.i32(i32 [[ADD_LCSSA]], i32 100)
+; CHECK-NEXT:    [[TMP5:%.*]] = sub i32 [[SMAX]], [[ADD_LCSSA]]
+; CHECK-NEXT:    [[TMP6:%.*]] = zext i32 [[TMP5]] to i64
+; CHECK-NEXT:    [[TMP7:%.*]] = add nuw nsw i64 [[TMP6]], 1
+; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP7]], 4
+; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH2:.*]], label %[[VECTOR_PH3:.*]]
+; CHECK:       [[VECTOR_PH3]]:
+; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[TMP7]], 4
+; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[TMP7]], [[N_MOD_VF]]
+; CHECK-NEXT:    br label %[[VECTOR_BODY4:.*]]
+; CHECK:       [[VECTOR_BODY4]]:
+; CHECK-NEXT:    [[INDEX5:%.*]] = phi i64 [ 0, %[[VECTOR_PH3]] ], [ [[INDEX_NEXT8:%.*]], %[[VECTOR_BODY4]] ]
+; CHECK-NEXT:    [[VEC_IND6:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH3]] ], [ [[VEC_IND_NEXT7:%.*]], %[[VECTOR_BODY4]] ]
+; CHECK-NEXT:    [[TMP8:%.*]] = add i64 [[INDEX5]], 0
+; CHECK-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i64, ptr [[P]], i64 [[TMP8]]
+; CHECK-NEXT:    [[TMP10:%.*]] = getelementptr inbounds i64, ptr [[TMP9]], i32 0
+; CHECK-NEXT:    store <4 x i64> [[VEC_IND6]], ptr [[TMP10]], align 4
+; CHECK-NEXT:    [[INDEX_NEXT8]] = add nuw i64 [[INDEX5]], 4
+; CHECK-NEXT:    [[VEC_IND_NEXT7]] = add <4 x i64> [[VEC_IND6]], splat (i64 4)
+; CHECK-NEXT:    [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT8]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[TMP11]], label %[[MIDDLE_BLOCK1:.*]], label %[[VECTOR_BODY4]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK:       [[MIDDLE_BLOCK1]]:
+; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[TMP7]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[CMP_N]], label %[[EXIT_2:.*]], label %[[SCALAR_PH2]]
+; CHECK:       [[SCALAR_PH2]]:
+; CHECK-NEXT:    [[BC_RESUME_VAL9:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK1]] ], [ 0, %[[EXIT_1]] ]
+; CHECK-NEXT:    br label %[[LOOP_2:.*]]
+; CHECK:       [[LOOP_2]]:
+; CHECK-NEXT:    [[IV_2:%.*]] = phi i64 [ [[BC_RESUME_VAL9]], %[[SCALAR_PH2]] ], [ [[IV_2_NEXT:%.*]], %[[LOOP_2]] ]
+; CHECK-NEXT:    [[IV_2_TRUNC:%.*]] = trunc i64 [[IV_2]] to i32
+; CHECK-NEXT:    [[IV_2_NEXT]] = add i64 [[IV_2]], 1
+; CHECK-NEXT:    [[GEP:%.*]] = getelementptr inbounds i64, ptr [[P]], i64 [[IV_2]]
+; CHECK-NEXT:    store i64 [[IV_2]], ptr [[GEP]], align 4
+; CHECK-NEXT:    [[ADD_2:%.*]] = add i32 [[ADD_LCSSA]], [[IV_2_TRUNC]]
+; CHECK-NEXT:    [[C_2:%.*]] = icmp slt i32 [[ADD_2]], 100
+; CHECK-NEXT:    br i1 [[C_2]], label %[[LOOP_2]], label %[[EXIT_2]], !llvm.loop [[LOOP5:![0-9]+]]
+; CHECK:       [[EXIT_2]]:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %loop.1
+
+loop.1:
+  %iv.1 = phi i32 [ 0, %entry ], [ %iv.1.next, %loop.1 ]
+  %1 = load i32, ptr %p, align 4
+  %add.1 = add i32 %1, %iv.1
+  %iv.1.next = add i32 %iv.1, 1
+  %c.1 = icmp eq i32 %iv.1, 100
+  br i1 %c.1, label %exit.1, label %loop.1
+
+exit.1:
+  %add.lcssa = phi i32 [ %add.1, %loop.1 ]
+  br label %loop.2
+
+loop.2:
+  %iv.2 = phi i64 [ 0, %exit.1 ], [ %iv.2.next, %loop.2 ]
+  %iv.2.trunc = trunc i64 %iv.2 to i32
+  %iv.2.next = add i64 %iv.2, 1
+  %gep = getelementptr inbounds i64, ptr %p, i64 %iv.2
+  store i64 %iv.2, ptr %gep
+  %add.2 = add i32 %add.lcssa, %iv.2.trunc
+  %c.2 = icmp slt i32 %add.2, 100
+  br i1 %c.2, label %loop.2, label %exit.2
+
+exit.2:
+  ret void
+}
+
+;.
+; CHECK: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]], [[META2:![0-9]+]]}
+; CHECK: [[META1]] = !{!"llvm.loop.isvectorized", i32 1}
+; CHECK: [[META2]] = !{!"llvm.loop.unroll.runtime.disable"}
+; CHECK: [[LOOP3]] = distinct !{[[LOOP3]], [[META2]], [[META1]]}
+; CHECK: [[LOOP4]] = distinct !{[[LOOP4]], [[META1]], [[META2]]}
+; CHECK: [[LOOP5]] = distinct !{[[LOOP5]], [[META2]], [[META1]]}
+;.
diff --git a/llvm/test/Transforms/LoopVectorize/invariant-store-vectorization-2.ll b/llvm/test/Transforms/LoopVectorize/invariant-store-vectorization-2.ll
index e8ad6a38d742c..14ed4c91c8757 100644
--- a/llvm/test/Transforms/LoopVectorize/invariant-store-vectorization-2.ll
+++ b/llvm/test/Transforms/LoopVectorize/invariant-store-vectorization-2.ll
@@ -55,7 +55,7 @@ define void @inv_val_store_to_inv_address_conditional_diff_values_ic(ptr %a, i64
 ; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[SMAX2]], [[N_VEC]]
 ; CHECK-NEXT:    br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[VECTOR_MEMCHECK]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ], [ 0, [[VECTOR_MEMCHECK]] ]
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK:       for.body:
 ; CHECK-NEXT:    [[I:%.*]] = phi i64 [ [[I_NEXT:%.*]], [[LATCH:%.*]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
@@ -153,7 +153,7 @@ define void @inv_val_store_to_inv_address_conditional_inv(ptr %a, i64 %n, ptr %b
 ; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[SMAX2]], [[N_VEC]]
 ; CHECK-NEXT:    br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[VECTOR_MEMCHECK]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ], [ 0, [[VECTOR_MEMCHECK]] ]
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK:       for.body:
 ; CHECK-NEXT:    [[I:%.*]] = phi i64 [ [[I_NEXT:%.*]], [[LATCH:%.*]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
@@ -241,8 +241,8 @@ define i32 @variant_val_store_to_inv_address(ptr %a, i64 %n, ptr %b, i32 %k) {
 ; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[SMAX2]], [[N_VEC]]
 ; CHECK-NEXT:    br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[VECTOR_MEMCHECK]] ], [ 0, [[ENTRY:%.*]] ]
-; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP5]], [[MIDDLE_BLOCK]] ], [ 0, [[VECTOR_MEMCHECK]] ], [ 0, [[ENTRY]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ], [ 0, [[VECTOR_MEMCHECK]] ]
+; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP5]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ], [ 0, [[VECTOR_MEMCHECK]] ]
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK:       for.body:
 ; CHECK-NEXT:    [[I:%.*]] = phi i64 [ [[I_NEXT:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
diff --git a/llvm/test/Transforms/LoopVectorize/invariant-store-vectorization.ll b/llvm/test/Transforms/LoopVectorize/invariant-store-vectorization.ll
index aef25a05ea124..d26362f0c162e 100644
--- a/llvm/test/Transforms/LoopVectorize/invariant-store-vectorization.ll
+++ b/llvm/test/Transforms/LoopVectorize/invariant-store-vectorization.ll
@@ -50,8 +50,8 @@ define i32 @inv_val_store_to_inv_address_with_reduction(ptr %a, i64 %n, ptr %b)
 ; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[SMAX2]], [[N_VEC]]
 ; CHECK-NEXT:    br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[VECTOR_MEMCHECK]] ], [ 0, [[ENTRY:%.*]] ]
-; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP4]], [[MIDDLE_BLOCK]] ], [ 0, [[VECTOR_MEMCHECK]] ], [ 0, [[ENTRY]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ], [ 0, [[VECTOR_MEMCHECK]] ]
+; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP4]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ], [ 0, [[VECTOR_MEMCHECK]] ]
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK:       for.body:
 ; CHECK-NEXT:    [[I:%.*]] = phi i64 [ [[I_NEXT:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
@@ -122,7 +122,7 @@ define void @inv_val_store_to_inv_address(ptr %a, i64 %n, ptr %b) {
 ; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[SMAX2]], [[N_VEC]]
 ; CHECK-NEXT:    br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[VECTOR_MEMCHECK]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ], [ 0, [[VECTOR_MEMCHECK]] ]
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK:       for.body:
 ; CHECK-NEXT:    [[I:%.*]] = phi i64 [ [[I_NEXT:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
@@ -224,7 +224,7 @@ define void @inv_val_store_to_inv_address_conditional(ptr %a, i64 %n, ptr %b, i3
 ; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[SMAX2]], [[N_VEC]]
 ; CHECK-NEXT:    br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[VECTOR_MEMCHECK]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ], [ 0, [[VECTOR_MEMCHECK]] ]
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK:       for.body:
 ; CHECK-NEXT:    [[I:%.*]] = phi i64 [ [[I_NEXT:%.*]], [[LATCH:%.*]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
@@ -401,8 +401,8 @@ define i32 @multiple_uniform_stores(ptr nocapture %var1, ptr nocapture readonly
 ; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[TMP8]], [[N_VEC]]
 ; CHECK-NEXT:    br i1 [[CMP_N]], label [[FOR_INC8_LOOPEXIT:%.*]], label [[SCALAR_PH]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP19]], [[MIDDLE_BLOCK]] ], [ [[ARRAYIDX5_PROMOTED]], [[VECTOR_MEMCHECK]] ], [ [[ARRAYIDX5_PROMOTED]], [[FOR_BODY3_LR_PH]] ]
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ [[TMP4]], [[VECTOR_MEMCHECK]] ], [ [[TMP4]], [[FOR_BODY3_LR_PH]] ]
+; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP19]], [[MIDDLE_BLOCK]] ], [ [[ARRAYIDX5_PROMOTED]], [[FOR_BODY3_LR_PH]] ], [ [[ARRAYIDX5_PROMOTED]], [[VECTOR_MEMCHECK]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ [[TMP4]], [[FOR_BODY3_LR_PH]] ], [ [[TMP4]], [[VECTOR_MEMCHECK]] ]
 ; CHECK-NEXT:    br label [[FOR_BODY3:%.*]]
 ; CHECK:       for.body3:
 ; CHECK-NEXT:    [[TMP20:%.*]] = phi i32 [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[TMP22:%.*]], [[FOR_BODY3]] ]
diff --git a/llvm/test/Transforms/LoopVectorize/iv_outside_user.ll b/llvm/test/Transforms/LoopVectorize/iv_outside_user.ll
index eb1dc9debc6b5..848831eab8165 100644
--- a/llvm/test/Transforms/LoopVectorize/iv_outside_user.ll
+++ b/llvm/test/Transforms/LoopVectorize/iv_outside_user.ll
@@ -203,13 +203,13 @@ define ptr @both(i32 %k)  {
 ; CHECK-NEXT:    br i1 [[CMP_N]], label %[[FOR_END:.*]], label %[[SCALAR_PH]]
 ; CHECK:       [[SCALAR_PH]]:
 ; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ [[IND_END]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
-; CHECK-NEXT:    [[BC_RESUME_VAL2:%.*]] = phi ptr [ [[IND_END1]], %[[MIDDLE_BLOCK]] ], [ [[BASE]], %[[ENTRY]] ]
-; CHECK-NEXT:    [[BC_RESUME_VAL4:%.*]] = phi ptr [ [[IND_END2]], %[[MIDDLE_BLOCK]] ], [ undef, %[[ENTRY]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL1:%.*]] = phi ptr [ [[IND_END1]], %[[MIDDLE_BLOCK]] ], [ [[BASE]], %[[ENTRY]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL2:%.*]] = phi ptr [ [[IND_END2]], %[[MIDDLE_BLOCK]] ], [ undef, %[[ENTRY]] ]
 ; CHECK-NEXT:    br label %[[FOR_BODY:.*]]
 ; CHECK:       [[FOR_BODY]]:
 ; CHECK-NEXT:    [[INC_PHI:%.*]] = phi i32 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[INC:%.*]], %[[FOR_BODY]] ]
-; CHECK-NEXT:    [[INC_LAG1:%.*]] = phi ptr [ [[BC_RESUME_VAL2]], %[[SCALAR_PH]] ], [ [[TMP:%.*]], %[[FOR_BODY]] ]
-; CHECK-NEXT:    [[INC_LAG2:%.*]] = phi ptr [ [[BC_RESUME_VAL4]], %[[SCALAR_PH]] ], [ [[INC_LAG1]], %[[FOR_BODY]] ]
+; CHECK-NEXT:    [[INC_LAG1:%.*]] = phi ptr [ [[BC_RESUME_VAL1]], %[[SCALAR_PH]] ], [ [[TMP:%.*]], %[[FOR_BODY]] ]
+; CHECK-NEXT:    [[INC_LAG2:%.*]] = phi ptr [ [[BC_RESUME_VAL2]], %[[SCALAR_PH]] ], [ [[INC_LAG1]], %[[FOR_BODY]] ]
 ; CHECK-NEXT:    [[TMP]] = getelementptr inbounds i32, ptr [[INC_LAG1]], i64 1
 ; CHECK-NEXT:    [[INC]] = add nsw i32 [[INC_PHI]], 1
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[INC]], [[K]]
@@ -308,21 +308,21 @@ define void @PR30742() {
 ; CHECK-NEXT:    [[N_VEC7:%.*]] = sub i32 [[TMP4]], [[N_MOD_VF6]]
 ; CHECK-NEXT:    [[TMP5:%.*]] = mul i32 [[N_VEC7]], -8
 ; CHECK-NEXT:    [[IND_END8:%.*]] = add i32 [[TMP04]], [[TMP5]]
-; CHECK-NEXT:    br label %[[VECTOR_BODY10:.*]]
-; CHECK:       [[VECTOR_BODY10]]:
-; CHECK-NEXT:    [[INDEX10:%.*]] = phi i32 [ 0, %[[VECTOR_PH5]] ], [ [[INDEX_NEXT11:%.*]], %[[VECTOR_BODY10]] ]
+; CHECK-NEXT:    br label %[[VECTOR_BODY8:.*]]
+; CHECK:       [[VECTOR_BODY8]]:
+; CHECK-NEXT:    [[INDEX10:%.*]] = phi i32 [ 0, %[[VECTOR_PH5]] ], [ [[INDEX_NEXT11:%.*]], %[[VECTOR_BODY8]] ]
 ; CHECK-NEXT:    [[INDEX_NEXT11]] = add nuw i32 [[INDEX10]], 2
 ; CHECK-NEXT:    [[TMP6:%.*]] = icmp eq i32 [[INDEX_NEXT11]], [[N_VEC7]]
-; CHECK-NEXT:    br i1 [[TMP6]], label %[[MIDDLE_BLOCK2:.*]], label %[[VECTOR_BODY10]], {{!llvm.loop ![0-9]+}}
+; CHECK-NEXT:    br i1 [[TMP6]], label %[[MIDDLE_BLOCK2:.*]], label %[[VECTOR_BODY8]], {{!llvm.loop ![0-9]+}}
 ; CHECK:       [[MIDDLE_BLOCK2]]:
 ; CHECK-NEXT:    [[CMP_N12:%.*]] = icmp eq i32 [[TMP4]], [[N_VEC7]]
 ; CHECK-NEXT:    [[IND_ESCAPE:%.*]] = sub i32 [[IND_END8]], -8
 ; CHECK-NEXT:    br i1 [[CMP_N12]], label %[[BB3:.*]], label %[[SCALAR_PH3]]
 ; CHECK:       [[SCALAR_PH3]]:
-; CHECK-NEXT:    [[BC_RESUME_VAL9:%.*]] = phi i32 [ [[IND_END8]], %[[MIDDLE_BLOCK2]] ], [ [[TMP04]], %[[BB1]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL12:%.*]] = phi i32 [ [[IND_END8]], %[[MIDDLE_BLOCK2]] ], [ [[TMP04]], %[[BB1]] ]
 ; CHECK-NEXT:    br label %[[BB2:.*]]
 ; CHECK:       [[BB2]]:
-; CHECK-NEXT:    [[TMP05:%.*]] = phi i32 [ [[BC_RESUME_VAL9]], %[[SCALAR_PH3]] ], [ [[TMP06:%.*]], %[[BB2]] ]
+; CHECK-NEXT:    [[TMP05:%.*]] = phi i32 [ [[BC_RESUME_VAL12]], %[[SCALAR_PH3]] ], [ [[TMP06:%.*]], %[[BB2]] ]
 ; CHECK-NEXT:    [[TMP06]] = add i32 [[TMP05]], -8
 ; CHECK-NEXT:    [[TMP07:%.*]] = icmp sgt i32 [[TMP06]], 0
 ; CHECK-NEXT:    br i1 [[TMP07]], label %[[BB2]], label %[[BB3]], {{!llvm.loop ![0-9]+}}
@@ -335,14 +335,14 @@ define void @PR30742() {
 ; CHECK-NEXT:    [[TMP7:%.*]] = add i32 [[TMP11]], -8
 ; CHECK-NEXT:    [[TMP8:%.*]] = add nsw i32 [[TMP11]], -15
 ; CHECK-NEXT:    [[SMIN:%.*]] = call i32 @llvm.smin.i32(i32 [[TMP8]], i32 0)
-; CHECK-NEXT:    [[TMP9:%.*]] = sub i32 [[TMP7]], [[SMIN]]
-; CHECK-NEXT:    [[TMP10:%.*]] = lshr i32 [[TMP9]], 3
-; CHECK-NEXT:    [[TMP11:%.*]] = add nuw nsw i32 [[TMP10]], 1
-; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[TMP11]], 2
+; CHECK-NEXT:    [[TMP10:%.*]] = sub i32 [[TMP7]], [[SMIN]]
+; CHECK-NEXT:    [[TMP11:%.*]] = lshr i32 [[TMP10]], 3
+; CHECK-NEXT:    [[TMP14:%.*]] = add nuw nsw i32 [[TMP11]], 1
+; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[TMP14]], 2
 ; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; CHECK:       [[VECTOR_PH]]:
-; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i32 [[TMP11]], 2
-; CHECK-NEXT:    [[N_VEC:%.*]] = sub i32 [[TMP11]], [[N_MOD_VF]]
+; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i32 [[TMP14]], 2
+; CHECK-NEXT:    [[N_VEC:%.*]] = sub i32 [[TMP14]], [[N_MOD_VF]]
 ; CHECK-NEXT:    [[TMP16:%.*]] = mul i32 [[N_VEC]], -8
 ; CHECK-NEXT:    [[IND_END:%.*]] = add i32 [[TMP12]], [[TMP16]]
 ; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
@@ -352,16 +352,16 @@ define void @PR30742() {
 ; CHECK-NEXT:    [[TMP13:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-NEXT:    br i1 [[TMP13]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], {{!llvm.loop ![0-9]+}}
 ; CHECK:       [[MIDDLE_BLOCK]]:
-; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i32 [[TMP11]], [[N_VEC]]
+; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i32 [[TMP14]], [[N_VEC]]
 ; CHECK-NEXT:    br i1 [[CMP_N]], label %[[BB1_LOOPEXIT]], label %[[SCALAR_PH]]
 ; CHECK:       [[SCALAR_PH]]:
 ; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ [[IND_END]], %[[MIDDLE_BLOCK]] ], [ [[TMP12]], %[[BB3]] ]
 ; CHECK-NEXT:    br label %[[BB4:.*]]
 ; CHECK:       [[BB4]]:
-; CHECK-NEXT:    [[TMP13:%.*]] = phi i32 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[TMP14:%.*]], %[[BB4]] ]
-; CHECK-NEXT:    [[TMP14]] = add i32 [[TMP13]], -8
-; CHECK-NEXT:    [[TMP15:%.*]] = icmp sgt i32 [[TMP14]], 0
-; CHECK-NEXT:    br i1 [[TMP15]], label %[[BB4]], label %[[BB1_LOOPEXIT]], {{!llvm.loop ![0-9]+}}
+; CHECK-NEXT:    [[TMP16:%.*]] = phi i32 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IND_END:%.*]], %[[BB4]] ]
+; CHECK-NEXT:    [[IND_END]] = add i32 [[TMP16]], -8
+; CHECK-NEXT:    [[TMP13:%.*]] = icmp sgt i32 [[IND_END]], 0
+; CHECK-NEXT:    br i1 [[TMP13]], label %[[BB4]], label %[[BB1_LOOPEXIT]], {{!llvm.loop ![0-9]+}}
 ;
 BB0:
   br label %BB1
@@ -742,7 +742,7 @@ for.end:
 
 define float @fp_postinc_use_fadd(float %init, ptr noalias nocapture %A, i64 %N, float %fpinc) {
 ; VEC-LABEL: define float @fp_postinc_use_fadd(
-; VEC-SAME: float [[INIT:%.*]], ptr noalias nocapture [[A:%.*]], i64 [[N:%.*]], float [[FPINC:%.*]]) {
+; VEC-SAME: float [[INIT:%.*]], ptr noalias captures(none) [[A:%.*]], i64 [[N:%.*]], float [[FPINC:%.*]]) {
 ; VEC-NEXT:  [[ENTRY:.*]]:
 ; VEC-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 2
 ; VEC-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
@@ -794,7 +794,7 @@ define float @fp_postinc_use_fadd(float %init, ptr noalias nocapture %A, i64 %N,
 ; VEC-NEXT:    ret float [[ADD_LCSSA]]
 ;
 ; INTERLEAVE-LABEL: define float @fp_postinc_use_fadd(
-; INTERLEAVE-SAME: float [[INIT:%.*]], ptr noalias nocapture [[A:%.*]], i64 [[N:%.*]], float [[FPINC:%.*]]) {
+; INTERLEAVE-SAME: float [[INIT:%.*]], ptr noalias captures(none) [[A:%.*]], i64 [[N:%.*]], float [[FPINC:%.*]]) {
 ; INTERLEAVE-NEXT:  [[ENTRY:.*]]:
 ; INTERLEAVE-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 2
 ; INTERLEAVE-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
@@ -862,7 +862,7 @@ exit:
 
 define float @fp_postinc_use_fadd_ops_swapped(float %init, ptr noalias nocapture %A, i64 %N, float %fpinc) {
 ; VEC-LABEL: define float @fp_postinc_use_fadd_ops_swapped(
-; VEC-SAME: float [[INIT:%.*]], ptr noalias nocapture [[A:%.*]], i64 [[N:%.*]], float [[FPINC:%.*]]) {
+; VEC-SAME: float [[INIT:%.*]], ptr noalias captures(none) [[A:%.*]], i64 [[N:%.*]], float [[FPINC:%.*]]) {
 ; VEC-NEXT:  [[ENTRY:.*]]:
 ; VEC-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 2
 ; VEC-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
@@ -914,7 +914,7 @@ define float @fp_postinc_use_fadd_ops_swapped(float %init, ptr noalias nocapture
 ; VEC-NEXT:    ret float [[ADD_LCSSA]]
 ;
 ; INTERLEAVE-LABEL: define float @fp_postinc_use_fadd_ops_swapped(
-; INTERLEAVE-SAME: float [[INIT:%.*]], ptr noalias nocapture [[A:%.*]], i64 [[N:%.*]], float [[FPINC:%.*]]) {
+; INTERLEAVE-SAME: float [[INIT:%.*]], ptr noalias captures(none) [[A:%.*]], i64 [[N:%.*]], float [[FPINC:%.*]]) {
 ; INTERLEAVE-NEXT:  [[ENTRY:.*]]:
 ; INTERLEAVE-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 2
 ; INTERLEAVE-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
@@ -982,7 +982,7 @@ exit:
 
 define float @fp_postinc_use_fsub(float %init, ptr noalias nocapture %A, i64 %N, float %fpinc) {
 ; VEC-LABEL: define float @fp_postinc_use_fsub(
-; VEC-SAME: float [[INIT:%.*]], ptr noalias nocapture [[A:%.*]], i64 [[N:%.*]], float [[FPINC:%.*]]) {
+; VEC-SAME: float [[INIT:%.*]], ptr noalias captures(none) [[A:%.*]], i64 [[N:%.*]], float [[FPINC:%.*]]) {
 ; VEC-NEXT:  [[ENTRY:.*]]:
 ; VEC-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 2
 ; VEC-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
@@ -1034,7 +1034,7 @@ define float @fp_postinc_use_fsub(float %init, ptr noalias nocapture %A, i64 %N,
 ; VEC-NEXT:    ret float [[ADD_LCSSA]]
 ;
 ; INTERLEAVE-LABEL: define float @fp_postinc_use_fsub(
-; INTERLEAVE-SAME: float [[INIT:%.*]], ptr noalias nocapture [[A:%.*]], i64 [[N:%.*]], float [[FPINC:%.*]]) {
+; INTERLEAVE-SAME: float [[INIT:%.*]], ptr noalias captures(none) [[A:%.*]], i64 [[N:%.*]], float [[FPINC:%.*]]) {
 ; INTERLEAVE-NEXT:  [[ENTRY:.*]]:
 ; INTERLEAVE-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 2
 ; INTERLEAVE-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
diff --git a/llvm/test/Transforms/LoopVectorize/load-deref-pred-align.ll b/llvm/test/Transforms/LoopVectorize/load-deref-pred-align.ll
index 0f4e327891899..5fcd0de911f85 100644
--- a/llvm/test/Transforms/LoopVectorize/load-deref-pred-align.ll
+++ b/llvm/test/Transforms/LoopVectorize/load-deref-pred-align.ll
@@ -265,8 +265,8 @@ define i32 @loop_requires_scev_predicate(ptr %dest, i32 %end) {
 ; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[UMAX1]], [[N_VEC]]
 ; CHECK-NEXT:    br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i8 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ 0, [[VECTOR_SCEVCHECK]] ], [ 0, [[ENTRY:%.*]] ]
-; CHECK-NEXT:    [[BC_RESUME_VAL2:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[VECTOR_SCEVCHECK]] ], [ 0, [[ENTRY]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i8 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ], [ 0, [[VECTOR_SCEVCHECK]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL2:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ], [ 0, [[VECTOR_SCEVCHECK]] ]
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK:       for.body:
 ; CHECK-NEXT:    [[IND:%.*]] = phi i8 [ [[IND_NEXT:%.*]], [[FOR_INC:%.*]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
@@ -351,27 +351,30 @@ define void @test_rev_loops_deref_loads(ptr nocapture noundef writeonly %dest) {
 ; CHECK-NEXT:    [[REVERSE:%.*]] = shufflevector <2 x i32> [[WIDE_LOAD]], <2 x i32> poison, <2 x i32> <i32 1, i32 0>
 ; CHECK-NEXT:    [[TMP4:%.*]] = icmp eq <2 x i32> [[REVERSE]], splat (i32 3)
 ; CHECK-NEXT:    [[TMP5:%.*]] = xor <2 x i1> [[TMP4]], splat (i1 true)
+; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr [1024 x i32], ptr [[LOCAL_SRC]], i64 0, i64 [[TMP0]]
+; CHECK-NEXT:    [[TMP7:%.*]] = getelementptr i32, ptr [[TMP6]], i32 0
+; CHECK-NEXT:    [[TMP8:%.*]] = getelementptr i32, ptr [[TMP7]], i32 -1
+; CHECK-NEXT:    [[WIDE_LOAD1:%.*]] = load <2 x i32>, ptr [[TMP8]], align 4
+; CHECK-NEXT:    [[REVERSE2:%.*]] = shufflevector <2 x i32> [[WIDE_LOAD1]], <2 x i32> poison, <2 x i32> <i32 1, i32 0>
 ; CHECK-NEXT:    [[TMP9:%.*]] = extractelement <2 x i1> [[TMP5]], i32 0
 ; CHECK-NEXT:    br i1 [[TMP9]], label [[PRED_STORE_IF:%.*]], label [[PRED_STORE_CONTINUE:%.*]]
 ; CHECK:       pred.store.if:
-; CHECK-NEXT:    [[TMP7:%.*]] = getelementptr inbounds [1024 x i32], ptr [[LOCAL_SRC]], i64 0, i64 [[TMP0]]
-; CHECK-NEXT:    [[TMP11:%.*]] = load i32, ptr [[TMP7]], align 4
 ; CHECK-NEXT:    [[TMP10:%.*]] = getelementptr inbounds [1024 x i32], ptr [[LOCAL_DEST]], i64 0, i64 [[TMP0]]
+; CHECK-NEXT:    [[TMP11:%.*]] = extractelement <2 x i32> [[REVERSE2]], i32 0
 ; CHECK-NEXT:    [[TMP12:%.*]] = shl nsw i32 [[TMP11]], 2
 ; CHECK-NEXT:    store i32 [[TMP12]], ptr [[TMP10]], align 4
 ; CHECK-NEXT:    br label [[PRED_STORE_CONTINUE]]
 ; CHECK:       pred.store.continue:
 ; CHECK-NEXT:    [[TMP13:%.*]] = extractelement <2 x i1> [[TMP5]], i32 1
 ; CHECK-NEXT:    br i1 [[TMP13]], label [[PRED_STORE_IF3:%.*]], label [[PRED_STORE_CONTINUE4]]
-; CHECK:       pred.store.if1:
+; CHECK:       pred.store.if3:
 ; CHECK-NEXT:    [[TMP14:%.*]] = add i64 [[OFFSET_IDX]], -1
-; CHECK-NEXT:    [[TMP21:%.*]] = getelementptr inbounds [1024 x i32], ptr [[LOCAL_SRC]], i64 0, i64 [[TMP14]]
-; CHECK-NEXT:    [[TMP16:%.*]] = load i32, ptr [[TMP21]], align 4
 ; CHECK-NEXT:    [[TMP15:%.*]] = getelementptr inbounds [1024 x i32], ptr [[LOCAL_DEST]], i64 0, i64 [[TMP14]]
+; CHECK-NEXT:    [[TMP16:%.*]] = extractelement <2 x i32> [[REVERSE2]], i32 1
 ; CHECK-NEXT:    [[TMP17:%.*]] = shl nsw i32 [[TMP16]], 2
 ; CHECK-NEXT:    store i32 [[TMP17]], ptr [[TMP15]], align 4
 ; CHECK-NEXT:    br label [[PRED_STORE_CONTINUE4]]
-; CHECK:       pred.store.continue2:
+; CHECK:       pred.store.continue4:
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
 ; CHECK-NEXT:    [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
 ; CHECK-NEXT:    br i1 [[TMP18]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
@@ -665,12 +668,15 @@ define void @test_rev_loops_strided_deref_loads(ptr nocapture noundef writeonly
 ; CHECK-NEXT:    [[TMP4:%.*]] = icmp eq <2 x i32> [[REVERSE]], splat (i32 3)
 ; CHECK-NEXT:    [[TMP5:%.*]] = xor <2 x i1> [[TMP4]], splat (i1 true)
 ; CHECK-NEXT:    [[TMP6:%.*]] = mul <2 x i64> [[VEC_IND]], splat (i64 2)
+; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <2 x i64> [[TMP6]], i32 0
+; CHECK-NEXT:    [[TMP8:%.*]] = getelementptr inbounds [1024 x i32], ptr [[LOCAL_SRC]], i64 0, i64 [[TMP7]]
+; CHECK-NEXT:    [[TMP9:%.*]] = extractelement <2 x i64> [[TMP6]], i32 1
+; CHECK-NEXT:    [[TMP10:%.*]] = getelementptr inbounds [1024 x i32], ptr [[LOCAL_SRC]], i64 0, i64 [[TMP9]]
+; CHECK-NEXT:    [[TMP11:%.*]] = load i32, ptr [[TMP8]], align 4
+; CHECK-NEXT:    [[TMP12:%.*]] = load i32, ptr [[TMP10]], align 4
 ; CHECK-NEXT:    [[TMP13:%.*]] = extractelement <2 x i1> [[TMP5]], i32 0
 ; CHECK-NEXT:    br i1 [[TMP13]], label [[PRED_STORE_IF:%.*]], label [[PRED_STORE_CONTINUE:%.*]]
 ; CHECK:       pred.store.if:
-; CHECK-NEXT:    [[TMP8:%.*]] = extractelement <2 x i64> [[TMP6]], i32 0
-; CHECK-NEXT:    [[TMP9:%.*]] = getelementptr inbounds [1024 x i32], ptr [[LOCAL_SRC]], i64 0, i64 [[TMP8]]
-; CHECK-NEXT:    [[TMP11:%.*]] = load i32, ptr [[TMP9]], align 4
 ; CHECK-NEXT:    [[TMP14:%.*]] = getelementptr inbounds [1024 x i32], ptr [[LOCAL_DEST]], i64 0, i64 [[TMP0]]
 ; CHECK-NEXT:    [[TMP15:%.*]] = shl nsw i32 [[TMP11]], 2
 ; CHECK-NEXT:    store i32 [[TMP15]], ptr [[TMP14]], align 4
@@ -680,9 +686,6 @@ define void @test_rev_loops_strided_deref_loads(ptr nocapture noundef writeonly
 ; CHECK-NEXT:    br i1 [[TMP16]], label [[PRED_STORE_IF1:%.*]], label [[PRED_STORE_CONTINUE2]]
 ; CHECK:       pred.store.if1:
 ; CHECK-NEXT:    [[TMP17:%.*]] = add i64 [[OFFSET_IDX]], -1
-; CHECK-NEXT:    [[TMP23:%.*]] = extractelement <2 x i64> [[TMP6]], i32 1
-; CHECK-NEXT:    [[TMP24:%.*]] = getelementptr inbounds [1024 x i32], ptr [[LOCAL_SRC]], i64 0, i64 [[TMP23]]
-; CHECK-NEXT:    [[TMP12:%.*]] = load i32, ptr [[TMP24]], align 4
 ; CHECK-NEXT:    [[TMP18:%.*]] = getelementptr inbounds [1024 x i32], ptr [[LOCAL_DEST]], i64 0, i64 [[TMP17]]
 ; CHECK-NEXT:    [[TMP19:%.*]] = shl nsw i32 [[TMP12]], 2
 ; CHECK-NEXT:    store i32 [[TMP19]], ptr [[TMP18]], align 4
diff --git a/llvm/test/Transforms/LoopVectorize/load-deref-pred-neg-off.ll b/llvm/test/Transforms/LoopVectorize/load-deref-pred-neg-off.ll
index 1dd526df503bd..c6103f5200053 100644
--- a/llvm/test/Transforms/LoopVectorize/load-deref-pred-neg-off.ll
+++ b/llvm/test/Transforms/LoopVectorize/load-deref-pred-neg-off.ll
@@ -29,8 +29,7 @@ define i8 @test_negative_off(i16 %len, ptr %test_base) {
 ; CHECK-NEXT:    [[TMP5:%.*]] = load i1, ptr [[TMP3]], align 1
 ; CHECK-NEXT:    [[TMP6:%.*]] = insertelement <2 x i1> poison, i1 [[TMP4]], i32 0
 ; CHECK-NEXT:    [[TMP7:%.*]] = insertelement <2 x i1> [[TMP6]], i1 [[TMP5]], i32 1
-; CHECK-NEXT:    [[TMP8:%.*]] = extractelement <2 x i1> [[TMP7]], i32 0
-; CHECK-NEXT:    br i1 [[TMP8]], label [[PRED_LOAD_IF:%.*]], label [[PRED_LOAD_CONTINUE:%.*]]
+; CHECK-NEXT:    br i1 [[TMP4]], label [[PRED_LOAD_IF:%.*]], label [[PRED_LOAD_CONTINUE:%.*]]
 ; CHECK:       pred.load.if:
 ; CHECK-NEXT:    [[TMP9:%.*]] = getelementptr i8, ptr [[ALLOCA]], i16 [[TMP0]]
 ; CHECK-NEXT:    [[TMP10:%.*]] = load i8, ptr [[TMP9]], align 1
@@ -38,8 +37,7 @@ define i8 @test_negative_off(i16 %len, ptr %test_base) {
 ; CHECK-NEXT:    br label [[PRED_LOAD_CONTINUE]]
 ; CHECK:       pred.load.continue:
 ; CHECK-NEXT:    [[TMP12:%.*]] = phi <2 x i8> [ poison, [[VECTOR_BODY]] ], [ [[TMP11]], [[PRED_LOAD_IF]] ]
-; CHECK-NEXT:    [[TMP13:%.*]] = extractelement <2 x i1> [[TMP7]], i32 1
-; CHECK-NEXT:    br i1 [[TMP13]], label [[PRED_LOAD_IF1:%.*]], label [[PRED_LOAD_CONTINUE2]]
+; CHECK-NEXT:    br i1 [[TMP5]], label [[PRED_LOAD_IF1:%.*]], label [[PRED_LOAD_CONTINUE2]]
 ; CHECK:       pred.load.if1:
 ; CHECK-NEXT:    [[TMP14:%.*]] = getelementptr i8, ptr [[ALLOCA]], i16 [[TMP1]]
 ; CHECK-NEXT:    [[TMP15:%.*]] = load i8, ptr [[TMP14]], align 1
diff --git a/llvm/test/Transforms/LoopVectorize/loop-vect-memdep.ll b/llvm/test/Transforms/LoopVectorize/loop-vect-memdep.ll
index cd4d63be4d1a8..789e9c0adb63f 100644
--- a/llvm/test/Transforms/LoopVectorize/loop-vect-memdep.ll
+++ b/llvm/test/Transforms/LoopVectorize/loop-vect-memdep.ll
@@ -1,6 +1,6 @@
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 
-; RUN: opt < %s -S -passes=loop-vectorize -debug-only=loop-vectorize 2>&1 | FileCheck %s
+; RUN: opt < %s -S -passes=loop-vectorize -debug-only=loop-vectorize --disable-output 2>&1 | FileCheck %s
 ; REQUIRES: asserts
 ; CHECK: LV: Can't vectorize due to memory conflicts
 
diff --git a/llvm/test/Transforms/LoopVectorize/multiple-strides-vectorization.ll b/llvm/test/Transforms/LoopVectorize/multiple-strides-vectorization.ll
index 851db7c44d51e..a0cd3c64f2d77 100644
--- a/llvm/test/Transforms/LoopVectorize/multiple-strides-vectorization.ll
+++ b/llvm/test/Transforms/LoopVectorize/multiple-strides-vectorization.ll
@@ -84,7 +84,7 @@ define void @Test(ptr nocapture %obj, i64 %z) #0 {
 ; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[Z]], [[N_VEC]]
 ; CHECK-NEXT:    br i1 [[CMP_N]], label [[DOTOUTER]], label [[SCALAR_PH]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[VECTOR_MEMCHECK]] ], [ 0, [[DOTOUTER_PREHEADER]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[DOTOUTER_PREHEADER]] ], [ 0, [[VECTOR_MEMCHECK]] ]
 ; CHECK-NEXT:    br label [[DOTINNER:%.*]]
 ; CHECK:       .exit:
 ; CHECK-NEXT:    ret void
@@ -151,7 +151,7 @@ define void @Test(ptr nocapture %obj, i64 %z) #0 {
 ; CHECK-HOIST-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[Z]], [[N_VEC]]
 ; CHECK-HOIST-NEXT:    br i1 [[CMP_N]], label [[DOTOUTER]], label [[SCALAR_PH]]
 ; CHECK-HOIST:       scalar.ph:
-; CHECK-HOIST-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[VECTOR_MEMCHECK]] ], [ 0, [[DOTOUTER_PREHEADER]] ]
+; CHECK-HOIST-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[DOTOUTER_PREHEADER]] ], [ 0, [[VECTOR_MEMCHECK]] ]
 ; CHECK-HOIST-NEXT:    br label [[DOTINNER:%.*]]
 ; CHECK-HOIST:       .exit:
 ; CHECK-HOIST-NEXT:    ret void
diff --git a/llvm/test/Transforms/LoopVectorize/no-fold-tail-by-masking-iv-external-uses.ll b/llvm/test/Transforms/LoopVectorize/no-fold-tail-by-masking-iv-external-uses.ll
index 66996316b47b7..5f3abd2d8dbd3 100644
--- a/llvm/test/Transforms/LoopVectorize/no-fold-tail-by-masking-iv-external-uses.ll
+++ b/llvm/test/Transforms/LoopVectorize/no-fold-tail-by-masking-iv-external-uses.ll
@@ -48,8 +48,8 @@ define i32 @test(ptr %arr, i64 %n) {
 ; CHECK-NEXT:    [[IND_ESCAPE:%.*]] = sub i64 [[IND_END]], 1
 ; CHECK-NEXT:    br i1 [[CMP_N]], label [[LOAD_VAL:%.*]], label [[SCALAR_PH]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ 1, [[VECTOR_SCEVCHECK]] ], [ 1, [[PREHEADER]] ]
-; CHECK-NEXT:    [[BC_RESUME_VAL2:%.*]] = phi i8 [ [[IND_END1]], [[MIDDLE_BLOCK]] ], [ 1, [[VECTOR_SCEVCHECK]] ], [ 1, [[PREHEADER]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ 1, [[PREHEADER]] ], [ 1, [[VECTOR_SCEVCHECK]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL2:%.*]] = phi i8 [ [[IND_END1]], [[MIDDLE_BLOCK]] ], [ 1, [[PREHEADER]] ], [ 1, [[VECTOR_SCEVCHECK]] ]
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
 ; CHECK-NEXT:    [[CONV:%.*]] = phi i64 [ [[CONV2:%.*]], [[LOOP]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
diff --git a/llvm/test/Transforms/LoopVectorize/no_outside_user.ll b/llvm/test/Transforms/LoopVectorize/no_outside_user.ll
index 81912b078b3b7..b3198587e5ae1 100644
--- a/llvm/test/Transforms/LoopVectorize/no_outside_user.ll
+++ b/llvm/test/Transforms/LoopVectorize/no_outside_user.ll
@@ -330,7 +330,7 @@ f1.exit.loopexit:
 ; not vectorize this.
 define i32 @reduction_sum(i32 %n, ptr noalias nocapture %A, ptr noalias nocapture %B) nounwind uwtable readonly noinline ssp {
 ; CHECK-LABEL: define i32 @reduction_sum(
-; CHECK-SAME: i32 [[N:%.*]], ptr noalias nocapture [[A:%.*]], ptr noalias nocapture [[B:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-SAME: i32 [[N:%.*]], ptr noalias captures(none) [[A:%.*]], ptr noalias captures(none) [[B:%.*]]) #[[ATTR0:[0-9]+]] {
 ; CHECK-NEXT:  [[ENTRY:.*]]:
 ; CHECK-NEXT:    [[C1:%.*]] = icmp sgt i32 [[N]], 0
 ; CHECK-NEXT:    br i1 [[C1]], label %[[HEADER_PREHEADER:.*]], [[DOT_CRIT_EDGE:label %.*]]
@@ -450,7 +450,7 @@ f1.exit.loopexit:
 ; cannot vectorize.
 define i32 @not_valid_reduction(i32 %n, ptr noalias nocapture %A) nounwind uwtable readonly {
 ; CHECK-LABEL: define i32 @not_valid_reduction(
-; CHECK-SAME: i32 [[N:%.*]], ptr noalias nocapture [[A:%.*]]) #[[ATTR1:[0-9]+]] {
+; CHECK-SAME: i32 [[N:%.*]], ptr noalias captures(none) [[A:%.*]]) #[[ATTR1:[0-9]+]] {
 ; CHECK-NEXT:  [[ENTRY:.*]]:
 ; CHECK-NEXT:    [[CMP4:%.*]] = icmp sgt i32 [[N]], 0
 ; CHECK-NEXT:    br i1 [[CMP4]], label %[[FOR_BODY_PREHEADER:.*]], label %[[FOR_END:.*]]
@@ -588,7 +588,7 @@ f1.exit.loopexit:
 
 define i32 @no_vectorize_reduction_with_outside_use(i32 %n, ptr nocapture %A, ptr nocapture %B) nounwind uwtable readonly {
 ; CHECK-LABEL: define i32 @no_vectorize_reduction_with_outside_use(
-; CHECK-SAME: i32 [[N:%.*]], ptr nocapture [[A:%.*]], ptr nocapture [[B:%.*]]) #[[ATTR1]] {
+; CHECK-SAME: i32 [[N:%.*]], ptr captures(none) [[A:%.*]], ptr captures(none) [[B:%.*]]) #[[ATTR1]] {
 ; CHECK-NEXT:  [[ENTRY:.*]]:
 ; CHECK-NEXT:    [[CMP7:%.*]] = icmp sgt i32 [[N]], 0
 ; CHECK-NEXT:    br i1 [[CMP7]], label %[[FOR_BODY_PREHEADER:.*]], label %[[FOR_END:.*]]
@@ -687,7 +687,7 @@ define i32 @sum_arrays_outside_use(ptr %B, ptr %A, ptr %C, i32 %N)  {
 ; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i32 [[TMP1]], [[N_VEC]]
 ; CHECK-NEXT:    br i1 [[CMP_N]], label %[[F1_EXIT_LOOPEXIT:.*]], label %[[SCALAR_PH]]
 ; CHECK:       [[SCALAR_PH]]:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ [[TMP4]], %[[MIDDLE_BLOCK]] ], [ [[B_PROMOTED]], %[[VECTOR_MEMCHECK]] ], [ [[B_PROMOTED]], %[[BB]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ [[TMP4]], %[[MIDDLE_BLOCK]] ], [ [[B_PROMOTED]], %[[BB]] ], [ [[B_PROMOTED]], %[[VECTOR_MEMCHECK]] ]
 ; CHECK-NEXT:    br label %[[DOTLR_PH_I:.*]]
 ; CHECK:       [[_LR_PH_I:.*:]]
 ; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ [[IVNEXT:%.*]], %[[DOTLR_PH_I]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
diff --git a/llvm/test/Transforms/LoopVectorize/nounroll.ll b/llvm/test/Transforms/LoopVectorize/nounroll.ll
index 7e4decb4dedb0..f6349404536b7 100644
--- a/llvm/test/Transforms/LoopVectorize/nounroll.ll
+++ b/llvm/test/Transforms/LoopVectorize/nounroll.ll
@@ -1,5 +1,5 @@
 ; REQUIRES: asserts
-; RUN: opt < %s -passes='loop-vectorize' -debug-only=loop-vectorize -S 2>&1 | FileCheck %s
+; RUN: opt < %s -passes='loop-vectorize' -debug-only=loop-vectorize --disable-output -S 2>&1 | FileCheck %s
 
 target datalayout = "e-m:e-i64:64-n32:64-S128-v256:256:256-v512:512:512"
 
diff --git a/llvm/test/Transforms/LoopVectorize/opaque-ptr.ll b/llvm/test/Transforms/LoopVectorize/opaque-ptr.ll
index b001b6fa56208..f62c3c7f42ec4 100644
--- a/llvm/test/Transforms/LoopVectorize/opaque-ptr.ll
+++ b/llvm/test/Transforms/LoopVectorize/opaque-ptr.ll
@@ -64,8 +64,8 @@ define void @test_ptr_iv_no_inbounds(ptr %p1.start, ptr %p2.start, ptr %p1.end)
 ; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[TMP3]], [[N_VEC]]
 ; CHECK-NEXT:    br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi ptr [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ [[P1_START]], [[VECTOR_MEMCHECK]] ], [ [[P1_START]], [[VECTOR_SCEVCHECK]] ], [ [[P1_START]], [[ENTRY:%.*]] ]
-; CHECK-NEXT:    [[BC_RESUME_VAL9:%.*]] = phi ptr [ [[IND_END8]], [[MIDDLE_BLOCK]] ], [ [[P2_START]], [[VECTOR_MEMCHECK]] ], [ [[P2_START]], [[VECTOR_SCEVCHECK]] ], [ [[P2_START]], [[ENTRY]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi ptr [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ [[P1_START]], [[ENTRY:%.*]] ], [ [[P1_START]], [[VECTOR_SCEVCHECK]] ], [ [[P1_START]], [[VECTOR_MEMCHECK]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL9:%.*]] = phi ptr [ [[IND_END8]], [[MIDDLE_BLOCK]] ], [ [[P2_START]], [[ENTRY]] ], [ [[P2_START]], [[VECTOR_SCEVCHECK]] ], [ [[P2_START]], [[VECTOR_MEMCHECK]] ]
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
 ; CHECK-NEXT:    [[P1:%.*]] = phi ptr [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[P1_NEXT:%.*]], [[LOOP]] ]
@@ -154,8 +154,8 @@ define void @test_ptr_iv_with_inbounds(ptr %p1.start, ptr %p2.start, ptr %p1.end
 ; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[TMP3]], [[N_VEC]]
 ; CHECK-NEXT:    br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi ptr [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ [[P1_START]], [[VECTOR_MEMCHECK]] ], [ [[P1_START]], [[ENTRY:%.*]] ]
-; CHECK-NEXT:    [[BC_RESUME_VAL7:%.*]] = phi ptr [ [[IND_END6]], [[MIDDLE_BLOCK]] ], [ [[P2_START]], [[VECTOR_MEMCHECK]] ], [ [[P2_START]], [[ENTRY]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi ptr [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ [[P1_START]], [[ENTRY:%.*]] ], [ [[P1_START]], [[VECTOR_MEMCHECK]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL7:%.*]] = phi ptr [ [[IND_END6]], [[MIDDLE_BLOCK]] ], [ [[P2_START]], [[ENTRY]] ], [ [[P2_START]], [[VECTOR_MEMCHECK]] ]
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
 ; CHECK-NEXT:    [[P1:%.*]] = phi ptr [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[P1_NEXT:%.*]], [[LOOP]] ]
diff --git a/llvm/test/Transforms/LoopVectorize/optimal-epilog-vectorization-limitations.ll b/llvm/test/Transforms/LoopVectorize/optimal-epilog-vectorization-limitations.ll
index 331aaed602aa2..a4b96c96c62f3 100644
--- a/llvm/test/Transforms/LoopVectorize/optimal-epilog-vectorization-limitations.ll
+++ b/llvm/test/Transforms/LoopVectorize/optimal-epilog-vectorization-limitations.ll
@@ -1,5 +1,5 @@
 ; REQUIRES: asserts
-; RUN: opt < %s -passes='loop-vectorize' -force-vector-width=2 -enable-epilogue-vectorization -epilogue-vectorization-force-VF=2 --debug-only=loop-vectorize -S 2>&1 | FileCheck %s
+; RUN: opt < %s -passes='loop-vectorize' -force-vector-width=2 -enable-epilogue-vectorization -epilogue-vectorization-force-VF=2 --debug-only=loop-vectorize --disable-output -S 2>&1 | FileCheck %s
 
 target datalayout = "e-m:e-i64:64-n32:64-v256:256:256-v512:512:512"
 
diff --git a/llvm/test/Transforms/LoopVectorize/optimal-epilog-vectorization-liveout.ll b/llvm/test/Transforms/LoopVectorize/optimal-epilog-vectorization-liveout.ll
index 06ac6e75cd74b..d328b565b83fe 100644
--- a/llvm/test/Transforms/LoopVectorize/optimal-epilog-vectorization-liveout.ll
+++ b/llvm/test/Transforms/LoopVectorize/optimal-epilog-vectorization-liveout.ll
@@ -70,7 +70,7 @@ define signext i32 @f1(ptr noalias %A, ptr noalias %B, i32 signext %n) {
 ; VF-TWO-CHECK-NEXT:    [[CMP_N5:%.*]] = icmp eq i64 [[WIDE_TRIP_COUNT]], [[N_VEC4]]
 ; VF-TWO-CHECK-NEXT:    br i1 [[CMP_N5]], label [[FOR_END_LOOPEXIT]], label [[VEC_EPILOG_SCALAR_PH]]
 ; VF-TWO-CHECK:       vec.epilog.scalar.ph:
-; VF-TWO-CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC4]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ 0, [[ITER_CHECK]] ], [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ]
+; VF-TWO-CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC4]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[ITER_CHECK]] ]
 ; VF-TWO-CHECK-NEXT:    br label [[FOR_BODY:%.*]]
 ; VF-TWO-CHECK:       for.body:
 ; VF-TWO-CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[VEC_EPILOG_SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ]
diff --git a/llvm/test/Transforms/LoopVectorize/optimal-epilog-vectorization-scalable.ll b/llvm/test/Transforms/LoopVectorize/optimal-epilog-vectorization-scalable.ll
index 336bba76000c3..05bc0f4addec5 100644
--- a/llvm/test/Transforms/LoopVectorize/optimal-epilog-vectorization-scalable.ll
+++ b/llvm/test/Transforms/LoopVectorize/optimal-epilog-vectorization-scalable.ll
@@ -1,5 +1,5 @@
 ; REQUIRES: asserts
-; RUN: opt < %s -passes='loop-vectorize' -force-target-supports-scalable-vectors=true -enable-epilogue-vectorization -epilogue-vectorization-force-VF=2 --debug-only=loop-vectorize -S -scalable-vectorization=on 2>&1 | FileCheck %s
+; RUN: opt < %s -passes='loop-vectorize' -force-target-supports-scalable-vectors=true -enable-epilogue-vectorization -epilogue-vectorization-force-VF=2 --debug-only=loop-vectorize --disable-output -S -scalable-vectorization=on 2>&1 | FileCheck %s
 
 target datalayout = "e-m:e-i64:64-n32:64-v256:256:256-v512:512:512"
 
diff --git a/llvm/test/Transforms/LoopVectorize/optimal-epilog-vectorization.ll b/llvm/test/Transforms/LoopVectorize/optimal-epilog-vectorization.ll
index 065e38e9fa5cf..4db0153fe8491 100644
--- a/llvm/test/Transforms/LoopVectorize/optimal-epilog-vectorization.ll
+++ b/llvm/test/Transforms/LoopVectorize/optimal-epilog-vectorization.ll
@@ -76,7 +76,7 @@ define dso_local void @f1(ptr noalias %aa, ptr noalias %bb, ptr noalias %cc, i32
 ; CHECK-NEXT:    [[CMP_N5:%.*]] = icmp eq i64 [[WIDE_TRIP_COUNT]], [[N_VEC4]]
 ; CHECK-NEXT:    br i1 [[CMP_N5]], label [[FOR_END_LOOPEXIT]], label [[VEC_EPILOG_SCALAR_PH]]
 ; CHECK:       vec.epilog.scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC4]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ 0, [[ITER_CHECK]] ], [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC4]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[ITER_CHECK]] ]
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK:       for.body:
 ; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[VEC_EPILOG_SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ]
@@ -234,8 +234,8 @@ define dso_local signext i32 @f2(ptr noalias %A, ptr noalias %B, i32 signext %n)
 ; CHECK-NEXT:    [[CMP_N6:%.*]] = icmp eq i64 [[WIDE_TRIP_COUNT]], [[N_VEC3]]
 ; CHECK-NEXT:    br i1 [[CMP_N6]], label [[FOR_END_LOOPEXIT]], label [[VEC_EPILOG_SCALAR_PH]]
 ; CHECK:       vec.epilog.scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC3]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ 0, [[VECTOR_SCEVCHECK]] ], [ 0, [[ITER_CHECK]] ], [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ]
-; CHECK-NEXT:    [[BC_RESUME_VAL5:%.*]] = phi i32 [ [[IND_END]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ 0, [[VECTOR_SCEVCHECK]] ], [ 0, [[ITER_CHECK]] ], [ [[IND_END4]], [[VEC_EPILOG_ITER_CHECK]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC3]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[VECTOR_SCEVCHECK]] ], [ 0, [[ITER_CHECK]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL5:%.*]] = phi i32 [ [[IND_END]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[IND_END4]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[VECTOR_SCEVCHECK]] ], [ 0, [[ITER_CHECK]] ]
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK:       for.body:
 ; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[VEC_EPILOG_SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ]
@@ -363,7 +363,7 @@ define void @f3(ptr noalias %A, i64 %n) {
 ; CHECK-NEXT:    [[CMP_N4:%.*]] = icmp eq i64 [[N]], [[N_VEC3]]
 ; CHECK-NEXT:    br i1 [[CMP_N4]], label [[FOR_END_LOOPEXIT]], label [[VEC_EPILOG_SCALAR_PH]]
 ; CHECK:       vec.epilog.scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC3]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ 0, [[ITER_CHECK:%.*]] ], [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC3]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[ITER_CHECK:%.*]] ]
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK:       for.body:
 ; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[VEC_EPILOG_SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
@@ -422,7 +422,7 @@ define void @f3(ptr noalias %A, i64 %n) {
 ; CHECK-PROFITABLE-BY-DEFAULT-NEXT:    [[CMP_N4:%.*]] = icmp eq i64 [[N]], [[N_VEC3]]
 ; CHECK-PROFITABLE-BY-DEFAULT-NEXT:    br i1 [[CMP_N4]], label [[FOR_END_LOOPEXIT]], label [[VEC_EPILOG_SCALAR_PH]]
 ; CHECK-PROFITABLE-BY-DEFAULT:       vec.epilog.scalar.ph:
-; CHECK-PROFITABLE-BY-DEFAULT-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC3]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ 0, [[ITER_CHECK:%.*]] ], [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ]
+; CHECK-PROFITABLE-BY-DEFAULT-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC3]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[ITER_CHECK:%.*]] ]
 ; CHECK-PROFITABLE-BY-DEFAULT-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK-PROFITABLE-BY-DEFAULT:       for.body:
 ; CHECK-PROFITABLE-BY-DEFAULT-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[VEC_EPILOG_SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
@@ -528,8 +528,8 @@ define void @induction_resume_value_requires_non_trivial_scev_expansion(ptr %dst
 ; CHECK:       vec.epilog.middle.block:
 ; CHECK-NEXT:    br i1 true, label [[OUTER_LATCH]], label [[VEC_EPILOG_SCALAR_PH]]
 ; CHECK:       vec.epilog.scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL3:%.*]] = phi i64 [ 85, [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ 1, [[ITER_CHECK]] ], [ 85, [[VEC_EPILOG_ITER_CHECK]] ]
-; CHECK-NEXT:    [[BC_RESUME_VAL6:%.*]] = phi i8 [ [[IND_END4]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ 0, [[ITER_CHECK]] ], [ [[IND_END5]], [[VEC_EPILOG_ITER_CHECK]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL3:%.*]] = phi i64 [ 85, [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ 85, [[VEC_EPILOG_ITER_CHECK]] ], [ 1, [[ITER_CHECK]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL6:%.*]] = phi i8 [ [[IND_END4]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[IND_END5]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[ITER_CHECK]] ]
 ; CHECK-NEXT:    br label [[INNER:%.*]]
 ; CHECK:       inner:
 ; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL3]], [[VEC_EPILOG_SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[INNER]] ]
@@ -618,8 +618,8 @@ define void @induction_resume_value_requires_non_trivial_scev_expansion(ptr %dst
 ; CHECK-PROFITABLE-BY-DEFAULT:       vec.epilog.middle.block:
 ; CHECK-PROFITABLE-BY-DEFAULT-NEXT:    br i1 true, label [[OUTER_LATCH]], label [[VEC_EPILOG_SCALAR_PH]]
 ; CHECK-PROFITABLE-BY-DEFAULT:       vec.epilog.scalar.ph:
-; CHECK-PROFITABLE-BY-DEFAULT-NEXT:    [[BC_RESUME_VAL3:%.*]] = phi i64 [ 85, [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ 1, [[ITER_CHECK]] ], [ 85, [[VEC_EPILOG_ITER_CHECK]] ]
-; CHECK-PROFITABLE-BY-DEFAULT-NEXT:    [[BC_RESUME_VAL6:%.*]] = phi i8 [ [[IND_END4]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ 0, [[ITER_CHECK]] ], [ [[IND_END5]], [[VEC_EPILOG_ITER_CHECK]] ]
+; CHECK-PROFITABLE-BY-DEFAULT-NEXT:    [[BC_RESUME_VAL3:%.*]] = phi i64 [ 85, [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ 85, [[VEC_EPILOG_ITER_CHECK]] ], [ 1, [[ITER_CHECK]] ]
+; CHECK-PROFITABLE-BY-DEFAULT-NEXT:    [[BC_RESUME_VAL6:%.*]] = phi i8 [ [[IND_END4]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[IND_END5]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[ITER_CHECK]] ]
 ; CHECK-PROFITABLE-BY-DEFAULT-NEXT:    br label [[INNER:%.*]]
 ; CHECK-PROFITABLE-BY-DEFAULT:       inner:
 ; CHECK-PROFITABLE-BY-DEFAULT-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL3]], [[VEC_EPILOG_SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[INNER]] ]
@@ -720,7 +720,7 @@ define void @f4(ptr noalias %A, i32 signext %n) {
 ; CHECK-NEXT:    [[CMP_N5:%.*]] = icmp eq i64 [[WIDE_TRIP_COUNT]], [[N_VEC3]]
 ; CHECK-NEXT:    br i1 [[CMP_N5]], label [[EXIT]], label [[VEC_EPILOG_SCALAR_PH]]
 ; CHECK:       vec.epilog.scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL4:%.*]] = phi i64 [ [[N_VEC3]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ 0, [[ITER_CHECK:%.*]] ], [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL4:%.*]] = phi i64 [ [[N_VEC3]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[ITER_CHECK:%.*]] ]
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
 ; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL4]], [[VEC_EPILOG_SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
@@ -790,7 +790,7 @@ define void @f4(ptr noalias %A, i32 signext %n) {
 ; CHECK-PROFITABLE-BY-DEFAULT-NEXT:    [[CMP_N5:%.*]] = icmp eq i64 [[WIDE_TRIP_COUNT]], [[N_VEC3]]
 ; CHECK-PROFITABLE-BY-DEFAULT-NEXT:    br i1 [[CMP_N5]], label [[EXIT]], label [[VEC_EPILOG_SCALAR_PH]]
 ; CHECK-PROFITABLE-BY-DEFAULT:       vec.epilog.scalar.ph:
-; CHECK-PROFITABLE-BY-DEFAULT-NEXT:    [[BC_RESUME_VAL4:%.*]] = phi i64 [ [[N_VEC3]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ 0, [[ITER_CHECK:%.*]] ], [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ]
+; CHECK-PROFITABLE-BY-DEFAULT-NEXT:    [[BC_RESUME_VAL4:%.*]] = phi i64 [ [[N_VEC3]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[ITER_CHECK:%.*]] ]
 ; CHECK-PROFITABLE-BY-DEFAULT-NEXT:    br label [[LOOP:%.*]]
 ; CHECK-PROFITABLE-BY-DEFAULT:       loop:
 ; CHECK-PROFITABLE-BY-DEFAULT-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL4]], [[VEC_EPILOG_SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
@@ -894,8 +894,8 @@ define void @multiple_ivs_wide(ptr %dst) {
 ; CHECK:       vec.epilog.middle.block:
 ; CHECK-NEXT:    br i1 true, label [[EXIT]], label [[VEC_EPILOG_SCALAR_PH]]
 ; CHECK:       vec.epilog.scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL5:%.*]] = phi i32 [ 128, [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ 0, [[ITER_CHECK:%.*]] ], [ 128, [[VEC_EPILOG_ITER_CHECK]] ]
-; CHECK-NEXT:    [[BC_RESUME_VAL6:%.*]] = phi i32 [ 64, [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ -64, [[ITER_CHECK]] ], [ 64, [[VEC_EPILOG_ITER_CHECK]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL5:%.*]] = phi i32 [ 128, [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ 128, [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[ITER_CHECK:%.*]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL6:%.*]] = phi i32 [ 64, [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ 64, [[VEC_EPILOG_ITER_CHECK]] ], [ -64, [[ITER_CHECK]] ]
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
 ; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ [[BC_RESUME_VAL5]], [[VEC_EPILOG_SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
diff --git a/llvm/test/Transforms/LoopVectorize/optsize.ll b/llvm/test/Transforms/LoopVectorize/optsize.ll
index 03812f4acb1eb..738f265b89d65 100644
--- a/llvm/test/Transforms/LoopVectorize/optsize.ll
+++ b/llvm/test/Transforms/LoopVectorize/optsize.ll
@@ -489,7 +489,7 @@ define void @pr43371_pgso() !prof !14 {
 ; NPGSO:       [[MIDDLE_BLOCK]]:
 ; NPGSO-NEXT:    br i1 true, label %[[FOR_COND_CLEANUP28:.*]], label %[[SCALAR_PH]]
 ; NPGSO:       [[SCALAR_PH]]:
-; NPGSO-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i16 [ 756, %[[MIDDLE_BLOCK]] ], [ 0, %[[VECTOR_SCEVCHECK]] ], [ 0, %[[ENTRY]] ]
+; NPGSO-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i16 [ 756, %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_SCEVCHECK]] ]
 ; NPGSO-NEXT:    br label %[[FOR_BODY29:.*]]
 ; NPGSO:       [[FOR_COND_CLEANUP28]]:
 ; NPGSO-NEXT:    unreachable
@@ -854,7 +854,7 @@ define void @stride1_pgso(ptr noalias %B, i32 %BStride) !prof !14 {
 ; CHECK:       [[MIDDLE_BLOCK]]:
 ; CHECK-NEXT:    br i1 false, label %[[FOR_END:.*]], label %[[SCALAR_PH]]
 ; CHECK:       [[SCALAR_PH]]:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ 1024, %[[MIDDLE_BLOCK]] ], [ 0, %[[VECTOR_SCEVCHECK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ 1024, %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_SCEVCHECK]] ]
 ; CHECK-NEXT:    br label %[[FOR_BODY:.*]]
 ; CHECK:       [[FOR_BODY]]:
 ; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ [[IV_NEXT:%.*]], %[[FOR_BODY]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
@@ -888,7 +888,7 @@ define void @stride1_pgso(ptr noalias %B, i32 %BStride) !prof !14 {
 ; PGSO:       [[MIDDLE_BLOCK]]:
 ; PGSO-NEXT:    br i1 false, label %[[FOR_END:.*]], label %[[SCALAR_PH]]
 ; PGSO:       [[SCALAR_PH]]:
-; PGSO-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ 1024, %[[MIDDLE_BLOCK]] ], [ 0, %[[VECTOR_SCEVCHECK]] ], [ 0, %[[ENTRY]] ]
+; PGSO-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ 1024, %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_SCEVCHECK]] ]
 ; PGSO-NEXT:    br label %[[FOR_BODY:.*]]
 ; PGSO:       [[FOR_BODY]]:
 ; PGSO-NEXT:    [[IV:%.*]] = phi i32 [ [[IV_NEXT:%.*]], %[[FOR_BODY]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
@@ -922,7 +922,7 @@ define void @stride1_pgso(ptr noalias %B, i32 %BStride) !prof !14 {
 ; NPGSO:       [[MIDDLE_BLOCK]]:
 ; NPGSO-NEXT:    br i1 false, label %[[FOR_END:.*]], label %[[SCALAR_PH]]
 ; NPGSO:       [[SCALAR_PH]]:
-; NPGSO-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ 1024, %[[MIDDLE_BLOCK]] ], [ 0, %[[VECTOR_SCEVCHECK]] ], [ 0, %[[ENTRY]] ]
+; NPGSO-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ 1024, %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_SCEVCHECK]] ]
 ; NPGSO-NEXT:    br label %[[FOR_BODY:.*]]
 ; NPGSO:       [[FOR_BODY]]:
 ; NPGSO-NEXT:    [[IV:%.*]] = phi i32 [ [[IV_NEXT:%.*]], %[[FOR_BODY]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
diff --git a/llvm/test/Transforms/LoopVectorize/outer-loop-wide-phis.ll b/llvm/test/Transforms/LoopVectorize/outer-loop-wide-phis.ll
new file mode 100644
index 0000000000000..3f81c0f5c822a
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/outer-loop-wide-phis.ll
@@ -0,0 +1,224 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --version 5
+; RUN: opt -passes=loop-vectorize -enable-vplan-native-path -S %s | FileCheck %s
+
+define void @wide_phi_2_predecessors(ptr noalias %A, ptr noalias %B, i32 %c, i1 %cond) {
+; CHECK-LABEL: define void @wide_phi_2_predecessors(
+; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], i32 [[C:%.*]], i1 [[COND:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK:       [[VECTOR_PH]]:
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[C]], i64 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK:       [[VECTOR_BODY]]:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_LATCH:.*]] ]
+; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_LATCH]] ]
+; CHECK-NEXT:    [[TMP0:%.*]] = getelementptr inbounds i64, ptr [[A]], <4 x i64> [[VEC_IND]]
+; CHECK-NEXT:    call void @llvm.masked.scatter.v4i32.v4p0(<4 x i32> [[BROADCAST_SPLAT]], <4 x ptr> [[TMP0]], i32 4, <4 x i1> splat (i1 true))
+; CHECK-NEXT:    br label %[[INNER_HEADER1:.*]]
+; CHECK:       [[INNER_HEADER1]]:
+; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <4 x i64> [ zeroinitializer, %[[VECTOR_BODY]] ], [ [[TMP4:%.*]], %[[INNER_LATCH4:.*]] ]
+; CHECK-NEXT:    [[VEC_PHI2:%.*]] = phi <4 x i64> [ zeroinitializer, %[[VECTOR_BODY]] ], [ [[TMP3:%.*]], %[[INNER_LATCH4]] ]
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i64, ptr [[B]], <4 x i64> [[VEC_PHI]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[THEN3:.*]], label %[[INNER_LATCH4]]
+; CHECK:       [[THEN3]]:
+; CHECK-NEXT:    [[WIDE_MASKED_GATHER:%.*]] = call <4 x i64> @llvm.masked.gather.v4i64.v4p0(<4 x ptr> [[TMP1]], i32 8, <4 x i1> splat (i1 true), <4 x i64> poison)
+; CHECK-NEXT:    br label %[[INNER_LATCH4]]
+; CHECK:       [[INNER_LATCH4]]:
+; CHECK-NEXT:    [[VEC_PHI5:%.*]] = phi <4 x i64> [ [[WIDE_MASKED_GATHER]], %[[THEN3]] ], [ zeroinitializer, %[[INNER_HEADER1]] ]
+; CHECK-NEXT:    [[TMP2:%.*]] = add nsw <4 x i64> [[VEC_PHI5]], [[VEC_IND]]
+; CHECK-NEXT:    [[TMP3]] = add nsw <4 x i64> [[TMP2]], [[VEC_PHI2]]
+; CHECK-NEXT:    [[TMP4]] = add nuw nsw <4 x i64> [[VEC_PHI]], splat (i64 1)
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq <4 x i64> [[TMP4]], splat (i64 1000)
+; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <4 x i1> [[TMP5]], i32 0
+; CHECK-NEXT:    br i1 [[TMP6]], label %[[VECTOR_LATCH]], label %[[INNER_HEADER1]]
+; CHECK:       [[VECTOR_LATCH]]:
+; CHECK-NEXT:    [[VEC_PHI6:%.*]] = phi <4 x i64> [ [[TMP3]], %[[INNER_LATCH4]] ]
+; CHECK-NEXT:    call void @llvm.masked.scatter.v4i64.v4p0(<4 x i64> [[VEC_PHI6]], <4 x ptr> [[TMP0]], i32 8, <4 x i1> splat (i1 true))
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <4 x i64> [[VEC_IND]], splat (i64 4)
+; CHECK-NEXT:    [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1000
+; CHECK-NEXT:    br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK:       [[MIDDLE_BLOCK]]:
+; CHECK-NEXT:    br i1 true, label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK:       [[SCALAR_PH]]:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 1000, %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT:    br label %[[OUTER_HEADER:.*]]
+; CHECK:       [[OUTER_HEADER]]:
+; CHECK-NEXT:    [[OUTER_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[OUTER_IV_NEXT:%.*]], %[[OUTER_LATCH:.*]] ]
+; CHECK-NEXT:    [[GEP_A:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[OUTER_IV]]
+; CHECK-NEXT:    store i32 [[C]], ptr [[GEP_A]], align 4
+; CHECK-NEXT:    br label %[[INNER_HEADER:.*]]
+; CHECK:       [[INNER_HEADER]]:
+; CHECK-NEXT:    [[INNER_IV:%.*]] = phi i64 [ 0, %[[OUTER_HEADER]] ], [ [[INNER_IV_NEXT:%.*]], %[[INNER_LATCH:.*]] ]
+; CHECK-NEXT:    [[RED:%.*]] = phi i64 [ 0, %[[OUTER_HEADER]] ], [ [[RED_NEXT:%.*]], %[[INNER_LATCH]] ]
+; CHECK-NEXT:    [[GEP_B:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[INNER_IV]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[INNER_LATCH]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[L_B:%.*]] = load i64, ptr [[GEP_B]], align 8
+; CHECK-NEXT:    br label %[[INNER_LATCH]]
+; CHECK:       [[INNER_LATCH]]:
+; CHECK-NEXT:    [[P:%.*]] = phi i64 [ [[L_B]], %[[THEN]] ], [ 0, %[[INNER_HEADER]] ]
+; CHECK-NEXT:    [[ADD_1:%.*]] = add nsw i64 [[P]], [[OUTER_IV]]
+; CHECK-NEXT:    [[RED_NEXT]] = add nsw i64 [[ADD_1]], [[RED]]
+; CHECK-NEXT:    [[INNER_IV_NEXT]] = add nuw nsw i64 [[INNER_IV]], 1
+; CHECK-NEXT:    [[INNER_EC:%.*]] = icmp eq i64 [[INNER_IV_NEXT]], 1000
+; CHECK-NEXT:    br i1 [[INNER_EC]], label %[[OUTER_LATCH]], label %[[INNER_HEADER]]
+; CHECK:       [[OUTER_LATCH]]:
+; CHECK-NEXT:    [[RED_NEXT_LCSSA:%.*]] = phi i64 [ [[RED_NEXT]], %[[INNER_LATCH]] ]
+; CHECK-NEXT:    store i64 [[RED_NEXT_LCSSA]], ptr [[GEP_A]], align 8
+; CHECK-NEXT:    [[OUTER_IV_NEXT]] = add nuw nsw i64 [[OUTER_IV]], 1
+; CHECK-NEXT:    [[OUTER_EC:%.*]] = icmp eq i64 [[OUTER_IV_NEXT]], 1000
+; CHECK-NEXT:    br i1 [[OUTER_EC]], label %[[EXIT]], label %[[OUTER_HEADER]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %outer.header
+
+outer.header:                                         ; preds = %outer.latch, %outer.header.lr.ph
+  %outer.iv = phi i64 [ 0, %entry ], [ %outer.iv.next, %outer.latch ]
+  %gep.A = getelementptr inbounds i64, ptr %A, i64 %outer.iv
+  store i32 %c, ptr %gep.A, align 4
+  br label %inner.header
+
+inner.header:
+  %inner.iv = phi i64 [ 0, %outer.header ], [ %inner.iv.next, %inner.latch ]
+  %red = phi i64 [ 0, %outer.header ], [ %red.next, %inner.latch ]
+  %gep.B = getelementptr inbounds i64, ptr %B, i64 %inner.iv
+  br i1 %cond, label %then, label %inner.latch
+
+then:
+  %l.b = load i64, ptr %gep.B, align 8
+  br label %inner.latch
+
+inner.latch:
+  %p = phi i64 [ %l.b, %then ], [ 0, %inner.header ]
+  %add.1 = add nsw i64 %p, %outer.iv
+  %red.next = add nsw i64 %add.1, %red
+  %inner.iv.next = add nuw nsw i64 %inner.iv, 1
+  %inner.ec = icmp eq i64 %inner.iv.next, 1000
+  br i1 %inner.ec, label %outer.latch, label %inner.header
+
+outer.latch:
+  store i64 %red.next, ptr %gep.A, align 8
+  %outer.iv.next = add nuw nsw i64 %outer.iv, 1
+  %outer.ec = icmp eq i64 %outer.iv.next, 1000
+  br i1 %outer.ec, label %exit, label %outer.header, !llvm.loop !1
+
+exit:
+  ret void
+}
+
+define void @wide_phi_2_predecessors_phi_ops_swapped(ptr noalias %A, ptr noalias %B, i32 %c, i1 %cond) {
+; CHECK-LABEL: define void @wide_phi_2_predecessors_phi_ops_swapped(
+; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], i32 [[C:%.*]], i1 [[COND:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK:       [[VECTOR_PH]]:
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[C]], i64 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK:       [[VECTOR_BODY]]:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_LATCH:.*]] ]
+; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_LATCH]] ]
+; CHECK-NEXT:    [[TMP0:%.*]] = getelementptr inbounds i64, ptr [[A]], <4 x i64> [[VEC_IND]]
+; CHECK-NEXT:    call void @llvm.masked.scatter.v4i32.v4p0(<4 x i32> [[BROADCAST_SPLAT]], <4 x ptr> [[TMP0]], i32 4, <4 x i1> splat (i1 true))
+; CHECK-NEXT:    br label %[[INNER_HEADER1:.*]]
+; CHECK:       [[INNER_HEADER1]]:
+; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <4 x i64> [ zeroinitializer, %[[VECTOR_BODY]] ], [ [[TMP4:%.*]], %[[INNER_LATCH4:.*]] ]
+; CHECK-NEXT:    [[VEC_PHI2:%.*]] = phi <4 x i64> [ zeroinitializer, %[[VECTOR_BODY]] ], [ [[TMP3:%.*]], %[[INNER_LATCH4]] ]
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i64, ptr [[B]], <4 x i64> [[VEC_PHI]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[THEN3:.*]], label %[[INNER_LATCH4]]
+; CHECK:       [[THEN3]]:
+; CHECK-NEXT:    [[WIDE_MASKED_GATHER:%.*]] = call <4 x i64> @llvm.masked.gather.v4i64.v4p0(<4 x ptr> [[TMP1]], i32 8, <4 x i1> splat (i1 true), <4 x i64> poison)
+; CHECK-NEXT:    br label %[[INNER_LATCH4]]
+; CHECK:       [[INNER_LATCH4]]:
+; CHECK-NEXT:    [[VEC_PHI5:%.*]] = phi <4 x i64> [ zeroinitializer, %[[INNER_HEADER1]] ], [ [[WIDE_MASKED_GATHER]], %[[THEN3]] ]
+; CHECK-NEXT:    [[TMP2:%.*]] = add nsw <4 x i64> [[VEC_PHI5]], [[VEC_IND]]
+; CHECK-NEXT:    [[TMP3]] = add nsw <4 x i64> [[TMP2]], [[VEC_PHI2]]
+; CHECK-NEXT:    [[TMP4]] = add nuw nsw <4 x i64> [[VEC_PHI]], splat (i64 1)
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq <4 x i64> [[TMP4]], splat (i64 1000)
+; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <4 x i1> [[TMP5]], i32 0
+; CHECK-NEXT:    br i1 [[TMP6]], label %[[VECTOR_LATCH]], label %[[INNER_HEADER1]]
+; CHECK:       [[VECTOR_LATCH]]:
+; CHECK-NEXT:    [[VEC_PHI6:%.*]] = phi <4 x i64> [ [[TMP3]], %[[INNER_LATCH4]] ]
+; CHECK-NEXT:    call void @llvm.masked.scatter.v4i64.v4p0(<4 x i64> [[VEC_PHI6]], <4 x ptr> [[TMP0]], i32 8, <4 x i1> splat (i1 true))
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <4 x i64> [[VEC_IND]], splat (i64 4)
+; CHECK-NEXT:    [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1000
+; CHECK-NEXT:    br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK:       [[MIDDLE_BLOCK]]:
+; CHECK-NEXT:    br i1 true, label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK:       [[SCALAR_PH]]:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 1000, %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT:    br label %[[OUTER_HEADER:.*]]
+; CHECK:       [[OUTER_HEADER]]:
+; CHECK-NEXT:    [[OUTER_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[OUTER_IV_NEXT:%.*]], %[[OUTER_LATCH:.*]] ]
+; CHECK-NEXT:    [[GEP_A:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[OUTER_IV]]
+; CHECK-NEXT:    store i32 [[C]], ptr [[GEP_A]], align 4
+; CHECK-NEXT:    br label %[[INNER_HEADER:.*]]
+; CHECK:       [[INNER_HEADER]]:
+; CHECK-NEXT:    [[INNER_IV:%.*]] = phi i64 [ 0, %[[OUTER_HEADER]] ], [ [[INNER_IV_NEXT:%.*]], %[[INNER_LATCH:.*]] ]
+; CHECK-NEXT:    [[RED:%.*]] = phi i64 [ 0, %[[OUTER_HEADER]] ], [ [[RED_NEXT:%.*]], %[[INNER_LATCH]] ]
+; CHECK-NEXT:    [[GEP_B:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[INNER_IV]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[INNER_LATCH]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[L_B:%.*]] = load i64, ptr [[GEP_B]], align 8
+; CHECK-NEXT:    br label %[[INNER_LATCH]]
+; CHECK:       [[INNER_LATCH]]:
+; CHECK-NEXT:    [[P:%.*]] = phi i64 [ 0, %[[INNER_HEADER]] ], [ [[L_B]], %[[THEN]] ]
+; CHECK-NEXT:    [[ADD_1:%.*]] = add nsw i64 [[P]], [[OUTER_IV]]
+; CHECK-NEXT:    [[RED_NEXT]] = add nsw i64 [[ADD_1]], [[RED]]
+; CHECK-NEXT:    [[INNER_IV_NEXT]] = add nuw nsw i64 [[INNER_IV]], 1
+; CHECK-NEXT:    [[INNER_EC:%.*]] = icmp eq i64 [[INNER_IV_NEXT]], 1000
+; CHECK-NEXT:    br i1 [[INNER_EC]], label %[[OUTER_LATCH]], label %[[INNER_HEADER]]
+; CHECK:       [[OUTER_LATCH]]:
+; CHECK-NEXT:    [[RED_NEXT_LCSSA:%.*]] = phi i64 [ [[RED_NEXT]], %[[INNER_LATCH]] ]
+; CHECK-NEXT:    store i64 [[RED_NEXT_LCSSA]], ptr [[GEP_A]], align 8
+; CHECK-NEXT:    [[OUTER_IV_NEXT]] = add nuw nsw i64 [[OUTER_IV]], 1
+; CHECK-NEXT:    [[OUTER_EC:%.*]] = icmp eq i64 [[OUTER_IV_NEXT]], 1000
+; CHECK-NEXT:    br i1 [[OUTER_EC]], label %[[EXIT]], label %[[OUTER_HEADER]], !llvm.loop [[LOOP5:![0-9]+]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %outer.header
+
+outer.header:                                         ; preds = %outer.latch, %outer.header.lr.ph
+  %outer.iv = phi i64 [ 0, %entry ], [ %outer.iv.next, %outer.latch ]
+  %gep.A = getelementptr inbounds i64, ptr %A, i64 %outer.iv
+  store i32 %c, ptr %gep.A, align 4
+  br label %inner.header
+
+inner.header:
+  %inner.iv = phi i64 [ 0, %outer.header ], [ %inner.iv.next, %inner.latch ]
+  %red = phi i64 [ 0, %outer.header ], [ %red.next, %inner.latch ]
+  %gep.B = getelementptr inbounds i64, ptr %B, i64 %inner.iv
+  br i1 %cond, label %then, label %inner.latch
+
+then:
+  %l.b = load i64, ptr %gep.B, align 8
+  br label %inner.latch
+
+inner.latch:
+  %p = phi i64 [ 0, %inner.header ], [ %l.b, %then ]
+  %add.1 = add nsw i64 %p, %outer.iv
+  %red.next = add nsw i64 %add.1, %red
+  %inner.iv.next = add nuw nsw i64 %inner.iv, 1
+  %inner.ec = icmp eq i64 %inner.iv.next, 1000
+  br i1 %inner.ec, label %outer.latch, label %inner.header
+
+outer.latch:
+  store i64 %red.next, ptr %gep.A, align 8
+  %outer.iv.next = add nuw nsw i64 %outer.iv, 1
+  %outer.ec = icmp eq i64 %outer.iv.next, 1000
+  br i1 %outer.ec, label %exit, label %outer.header, !llvm.loop !1
+
+exit:
+  ret void
+}
+
+!1 = distinct !{!1, !2, !3}
+!2 = !{!"llvm.loop.vectorize.width", i32 4}
+!3 = !{!"llvm.loop.vectorize.enable", i1 true}
diff --git a/llvm/test/Transforms/LoopVectorize/pointer-select-runtime-checks.ll b/llvm/test/Transforms/LoopVectorize/pointer-select-runtime-checks.ll
index 00daf7d34483c..e652d86944c4d 100644
--- a/llvm/test/Transforms/LoopVectorize/pointer-select-runtime-checks.ll
+++ b/llvm/test/Transforms/LoopVectorize/pointer-select-runtime-checks.ll
@@ -42,7 +42,7 @@ define void @test1_select_invariant(ptr %src.1, ptr %src.2, ptr %dst, i1 %c, i8
 ; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i32 [[TMP2]], [[N_VEC]]
 ; CHECK-NEXT:    br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i8 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ 0, [[VECTOR_MEMCHECK]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i8 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ], [ 0, [[VECTOR_MEMCHECK]] ]
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
 ; CHECK-NEXT:    [[IV:%.*]] = phi i8 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
@@ -121,7 +121,7 @@ define void @test_loop_dependent_select1(ptr %src.1, ptr %src.2, ptr %dst, i1 %c
 ; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i32 [[TMP2]], [[N_VEC]]
 ; CHECK-NEXT:    br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i8 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ 0, [[VECTOR_MEMCHECK]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i8 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ], [ 0, [[VECTOR_MEMCHECK]] ]
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
 ; CHECK-NEXT:    [[IV:%.*]] = phi i8 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
@@ -211,7 +211,7 @@ define void @test_loop_dependent_select2(ptr %src.1, ptr %src.2, ptr %dst, i8 %n
 ; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i32 [[TMP2]], [[N_VEC]]
 ; CHECK-NEXT:    br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i8 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ 0, [[VECTOR_MEMCHECK]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i8 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ], [ 0, [[VECTOR_MEMCHECK]] ]
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
 ; CHECK-NEXT:    [[IV:%.*]] = phi i8 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
@@ -296,7 +296,7 @@ define void @test_loop_dependent_select_first_ptr_noundef(ptr noundef %src.1, pt
 ; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i32 [[TMP2]], [[N_VEC]]
 ; CHECK-NEXT:    br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i8 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ 0, [[VECTOR_MEMCHECK]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i8 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ], [ 0, [[VECTOR_MEMCHECK]] ]
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
 ; CHECK-NEXT:    [[IV:%.*]] = phi i8 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
@@ -381,7 +381,7 @@ define void @test_loop_dependent_select_second_ptr_noundef(ptr %src.1, ptr nound
 ; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i32 [[TMP2]], [[N_VEC]]
 ; CHECK-NEXT:    br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i8 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ 0, [[VECTOR_MEMCHECK]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i8 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ], [ 0, [[VECTOR_MEMCHECK]] ]
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
 ; CHECK-NEXT:    [[IV:%.*]] = phi i8 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
diff --git a/llvm/test/Transforms/LoopVectorize/pr30654-phiscev-sext-trunc.ll b/llvm/test/Transforms/LoopVectorize/pr30654-phiscev-sext-trunc.ll
index 2741b39693579..0cda697e0337a 100644
--- a/llvm/test/Transforms/LoopVectorize/pr30654-phiscev-sext-trunc.ll
+++ b/llvm/test/Transforms/LoopVectorize/pr30654-phiscev-sext-trunc.ll
@@ -92,8 +92,8 @@ define void @doit1(i32 %n, i32 %step) local_unnamed_addr {
 ; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[WIDE_TRIP_COUNT]], [[N_VEC]]
 ; CHECK-NEXT:    br i1 [[CMP_N]], label [[FOR_END_LOOPEXIT:%.*]], label [[SCALAR_PH]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[VECTOR_SCEVCHECK]] ], [ 0, [[FOR_BODY_PREHEADER]] ]
-; CHECK-NEXT:    [[BC_RESUME_VAL1:%.*]] = phi i32 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ 0, [[VECTOR_SCEVCHECK]] ], [ 0, [[FOR_BODY_PREHEADER]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[FOR_BODY_PREHEADER]] ], [ 0, [[VECTOR_SCEVCHECK]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL1:%.*]] = phi i32 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ 0, [[FOR_BODY_PREHEADER]] ], [ 0, [[VECTOR_SCEVCHECK]] ]
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK:       for.body:
 ; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
@@ -215,8 +215,8 @@ define void @doit2(i32 %n, i32 %step) local_unnamed_addr  {
 ; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[WIDE_TRIP_COUNT]], [[N_VEC]]
 ; CHECK-NEXT:    br i1 [[CMP_N]], label [[FOR_END_LOOPEXIT:%.*]], label [[SCALAR_PH]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[VECTOR_SCEVCHECK]] ], [ 0, [[FOR_BODY_PREHEADER]] ]
-; CHECK-NEXT:    [[BC_RESUME_VAL1:%.*]] = phi i32 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ 0, [[VECTOR_SCEVCHECK]] ], [ 0, [[FOR_BODY_PREHEADER]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[FOR_BODY_PREHEADER]] ], [ 0, [[VECTOR_SCEVCHECK]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL1:%.*]] = phi i32 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ 0, [[FOR_BODY_PREHEADER]] ], [ 0, [[VECTOR_SCEVCHECK]] ]
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK:       for.body:
 ; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
@@ -410,8 +410,8 @@ define void @doit4(i32 %n, i8 signext %cstep) local_unnamed_addr {
 ; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[WIDE_TRIP_COUNT]], [[N_VEC]]
 ; CHECK-NEXT:    br i1 [[CMP_N]], label [[FOR_END_LOOPEXIT:%.*]], label [[SCALAR_PH]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[VECTOR_SCEVCHECK]] ], [ 0, [[FOR_BODY_PREHEADER]] ]
-; CHECK-NEXT:    [[BC_RESUME_VAL1:%.*]] = phi i32 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ 0, [[VECTOR_SCEVCHECK]] ], [ 0, [[FOR_BODY_PREHEADER]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[FOR_BODY_PREHEADER]] ], [ 0, [[VECTOR_SCEVCHECK]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL1:%.*]] = phi i32 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ 0, [[FOR_BODY_PREHEADER]] ], [ 0, [[VECTOR_SCEVCHECK]] ]
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK:       for.body:
 ; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
diff --git a/llvm/test/Transforms/LoopVectorize/pr33706.ll b/llvm/test/Transforms/LoopVectorize/pr33706.ll
index e37f69cbd214f..e45546d3825af 100644
--- a/llvm/test/Transforms/LoopVectorize/pr33706.ll
+++ b/llvm/test/Transforms/LoopVectorize/pr33706.ll
@@ -7,7 +7,7 @@
 
 define void @PR33706(ptr nocapture readonly %arg, ptr nocapture %arg1, i32 %arg2) local_unnamed_addr {
 ; CHECK-LABEL: define void @PR33706
-; CHECK-SAME: (ptr nocapture readonly [[ARG:%.*]], ptr nocapture [[ARG1:%.*]], i32 [[ARG2:%.*]]) local_unnamed_addr {
+; CHECK-SAME: (ptr readonly captures(none) [[ARG:%.*]], ptr captures(none) [[ARG1:%.*]], i32 [[ARG2:%.*]]) local_unnamed_addr {
 ; CHECK-NEXT:  bb:
 ; CHECK-NEXT:    [[TMP:%.*]] = load i32, ptr @global.1, align 4
 ; CHECK-NEXT:    [[VAR_TMP3:%.*]] = getelementptr inbounds float, ptr [[ARG]], i64 190
diff --git a/llvm/test/Transforms/LoopVectorize/pr37248.ll b/llvm/test/Transforms/LoopVectorize/pr37248.ll
index fe660a8269672..767fd2e188aae 100644
--- a/llvm/test/Transforms/LoopVectorize/pr37248.ll
+++ b/llvm/test/Transforms/LoopVectorize/pr37248.ll
@@ -71,7 +71,7 @@ define void @f1(ptr noalias %b, i1 %c, i32 %start) {
 ; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i32 [[TMP1]], [[N_VEC]]
 ; CHECK-NEXT:    br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ [[START]], [[VECTOR_SCEVCHECK]] ], [ [[START]], [[ENTRY:%.*]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ [[START]], [[ENTRY:%.*]] ], [ [[START]], [[VECTOR_SCEVCHECK]] ]
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK:       for.body:
 ; CHECK-NEXT:    [[TMP19:%.*]] = phi i32 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[DEC:%.*]], [[LAND_END:%.*]] ]
@@ -156,7 +156,7 @@ define void @f2(ptr noalias %b, i1 %c, i32 %start) {
 ; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i32 [[TMP1]], [[N_VEC]]
 ; CHECK-NEXT:    br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ [[START]], [[VECTOR_SCEVCHECK]] ], [ [[START]], [[ENTRY:%.*]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ [[START]], [[ENTRY:%.*]] ], [ [[START]], [[VECTOR_SCEVCHECK]] ]
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK:       for.body:
 ; CHECK-NEXT:    [[TMP16:%.*]] = phi i32 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[DEC:%.*]], [[LAND_END:%.*]] ]
diff --git a/llvm/test/Transforms/LoopVectorize/pr45259.ll b/llvm/test/Transforms/LoopVectorize/pr45259.ll
index 9ae8f69b50a90..6baed089fb6b6 100644
--- a/llvm/test/Transforms/LoopVectorize/pr45259.ll
+++ b/llvm/test/Transforms/LoopVectorize/pr45259.ll
@@ -58,7 +58,7 @@ define i8 @widget(ptr %arr, i8 %t9) {
 ; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i32 [[TMP3]], [[N_VEC]]
 ; CHECK-NEXT:    br i1 [[CMP_N]], label [[FOR_EXIT:%.*]], label [[SCALAR_PH]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i8 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ 0, [[VECTOR_SCEVCHECK]] ], [ 0, [[FOR_PREHEADER]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i8 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ 0, [[FOR_PREHEADER]] ], [ 0, [[VECTOR_SCEVCHECK]] ]
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK:       for.body:
 ; CHECK-NEXT:    [[IV:%.*]] = phi i8 [ [[IV_NEXT:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
diff --git a/llvm/test/Transforms/LoopVectorize/pr47343-expander-lcssa-after-cfg-update.ll b/llvm/test/Transforms/LoopVectorize/pr47343-expander-lcssa-after-cfg-update.ll
index 3d85a7289637f..6aaa44344ae46 100644
--- a/llvm/test/Transforms/LoopVectorize/pr47343-expander-lcssa-after-cfg-update.ll
+++ b/llvm/test/Transforms/LoopVectorize/pr47343-expander-lcssa-after-cfg-update.ll
@@ -54,7 +54,7 @@ define void @f() {
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    br i1 true, label [[EXIT:%.*]], label [[SCALAR_PH]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ 500, [[MIDDLE_BLOCK]] ], [ 0, [[VECTOR_MEMCHECK]] ], [ 0, [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ 500, [[MIDDLE_BLOCK]] ], [ 0, [[LOOP_PREHEADER]] ], [ 0, [[VECTOR_MEMCHECK]] ]
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
 ; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ [[IV_NEXT:%.*]], [[LOOP]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
diff --git a/llvm/test/Transforms/LoopVectorize/pr50686.ll b/llvm/test/Transforms/LoopVectorize/pr50686.ll
index 4d8562247871a..6ce491e53c256 100644
--- a/llvm/test/Transforms/LoopVectorize/pr50686.ll
+++ b/llvm/test/Transforms/LoopVectorize/pr50686.ll
@@ -40,7 +40,7 @@ define void @m(ptr nocapture %p, ptr nocapture %p2, i32 %q) {
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    br i1 false, label [[FOR_END17:%.*]], label [[SCALAR_PH]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 60, [[MIDDLE_BLOCK]] ], [ 0, [[VECTOR_MEMCHECK]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 60, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ], [ 0, [[VECTOR_MEMCHECK]] ]
 ; CHECK-NEXT:    br label [[FOR_COND5:%.*]]
 ; CHECK:       for.cond5:
 ; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_COND5]] ]
diff --git a/llvm/test/Transforms/LoopVectorize/pr59319-loop-access-info-invalidation.ll b/llvm/test/Transforms/LoopVectorize/pr59319-loop-access-info-invalidation.ll
index a4b229d0a96b2..95ae2de117666 100644
--- a/llvm/test/Transforms/LoopVectorize/pr59319-loop-access-info-invalidation.ll
+++ b/llvm/test/Transforms/LoopVectorize/pr59319-loop-access-info-invalidation.ll
@@ -63,7 +63,7 @@ define void @reduced(ptr %0, ptr %1, i64 %iv, ptr %2, i64 %iv76, i64 %iv93) {
 ; CHECK-NEXT:    [[CMP_N10:%.*]] = icmp eq i64 [[TMP3]], [[N_VEC]]
 ; CHECK-NEXT:    br i1 [[CMP_N10]], label [[LOOP_3_LR_PH:%.*]], label [[SCALAR_PH5]]
 ; CHECK:       scalar.ph5:
-; CHECK-NEXT:    [[BC_RESUME_VAL13:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK4]] ], [ 0, [[VECTOR_MEMCHECK]] ], [ 0, [[LOOP_2_PREHEADER]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL13:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK4]] ], [ 0, [[LOOP_2_PREHEADER]] ], [ 0, [[VECTOR_MEMCHECK]] ]
 ; CHECK-NEXT:    br label [[LOOP_2:%.*]]
 ; CHECK:       loop.3.lr.ph:
 ; CHECK-NEXT:    [[IDXPROM_I_I61:%.*]] = and i64 [[IV761_LCSSA]], 1
@@ -93,7 +93,7 @@ define void @reduced(ptr %0, ptr %1, i64 %iv, ptr %2, i64 %iv76, i64 %iv93) {
 ; CHECK-NEXT:    [[CMP_N27:%.*]] = icmp eq i64 [[TMP3]], [[N_VEC25]]
 ; CHECK-NEXT:    br i1 [[CMP_N27]], label [[LOOP_CLEANUP:%.*]], label [[SCALAR_PH22]]
 ; CHECK:       scalar.ph22:
-; CHECK-NEXT:    [[BC_RESUME_VAL26:%.*]] = phi i64 [ [[N_VEC25]], [[MIDDLE_BLOCK21]] ], [ 0, [[VECTOR_MEMCHECK15]] ], [ 0, [[LOOP_3_LR_PH]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL26:%.*]] = phi i64 [ [[N_VEC25]], [[MIDDLE_BLOCK21]] ], [ 0, [[LOOP_3_LR_PH]] ], [ 0, [[VECTOR_MEMCHECK15]] ]
 ; CHECK-NEXT:    br label [[LOOP_3:%.*]]
 ; CHECK:       loop.2:
 ; CHECK-NEXT:    [[IV846:%.*]] = phi i64 [ [[IV_NEXT85:%.*]], [[LOOP_2]] ], [ [[BC_RESUME_VAL13]], [[SCALAR_PH5]] ]
diff --git a/llvm/test/Transforms/LoopVectorize/reduction-align.ll b/llvm/test/Transforms/LoopVectorize/reduction-align.ll
index 3216c9233ea21..028eb3b05957d 100644
--- a/llvm/test/Transforms/LoopVectorize/reduction-align.ll
+++ b/llvm/test/Transforms/LoopVectorize/reduction-align.ll
@@ -40,8 +40,8 @@ define void @fn(ptr %hbuf, ptr %ref, i32 %height) {
 ; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i32 [[HEIGHT]], [[N_VEC]]
 ; CHECK-NEXT:    br i1 [[CMP_N]], label [[FOR_END_LOOPEXIT:%.*]], label [[SCALAR_PH]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i16 [ [[TMP3]], [[MIDDLE_BLOCK]] ], [ 0, [[VECTOR_MEMCHECK]] ], [ 0, [[FOR_BODY_PREHEADER]] ]
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[VECTOR_MEMCHECK]] ], [ 0, [[FOR_BODY_PREHEADER]] ]
+; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i16 [ [[TMP3]], [[MIDDLE_BLOCK]] ], [ 0, [[FOR_BODY_PREHEADER]] ], [ 0, [[VECTOR_MEMCHECK]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[FOR_BODY_PREHEADER]] ], [ 0, [[VECTOR_MEMCHECK]] ]
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK:       for.body:
 ; CHECK-NEXT:    [[TMP4:%.*]] = phi i16 [ [[ADD:%.*]], [[FOR_BODY]] ], [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ]
diff --git a/llvm/test/Transforms/LoopVectorize/reduction-inloop-cond.ll b/llvm/test/Transforms/LoopVectorize/reduction-inloop-cond.ll
index ad2f9c608f204..073c2d91bcc1a 100644
--- a/llvm/test/Transforms/LoopVectorize/reduction-inloop-cond.ll
+++ b/llvm/test/Transforms/LoopVectorize/reduction-inloop-cond.ll
@@ -3,7 +3,7 @@
 
 define float @cond_fadd(ptr noalias nocapture readonly %a, ptr noalias nocapture readonly %cond, i64 %N){
 ; CHECK-LABEL: define float @cond_fadd(
-; CHECK-SAME: ptr noalias nocapture readonly [[A:%.*]], ptr noalias nocapture readonly [[COND:%.*]], i64 [[N:%.*]]) {
+; CHECK-SAME: ptr noalias readonly captures(none) [[A:%.*]], ptr noalias readonly captures(none) [[COND:%.*]], i64 [[N:%.*]]) {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
 ; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
@@ -497,7 +497,7 @@ for.end:
 ;
 define i64 @nested_cond_and(ptr noalias nocapture readonly %a, ptr noalias nocapture readonly %b, ptr noalias nocapture readonly %cond, i64 %N){
 ; CHECK-LABEL: define i64 @nested_cond_and(
-; CHECK-SAME: ptr noalias nocapture readonly [[A:%.*]], ptr noalias nocapture readonly [[B:%.*]], ptr noalias nocapture readonly [[COND:%.*]], i64 [[N:%.*]]) {
+; CHECK-SAME: ptr noalias readonly captures(none) [[A:%.*]], ptr noalias readonly captures(none) [[B:%.*]], ptr noalias readonly captures(none) [[COND:%.*]], i64 [[N:%.*]]) {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
 ; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
@@ -733,8 +733,8 @@ define i32 @cond-uncond(ptr noalias %src1, ptr noalias %src2, ptr noalias %cond,
 ; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
 ; CHECK-NEXT:    br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP29]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ]
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP29]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ]
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK:       for.body:
 ; CHECK-NEXT:    [[RDX1:%.*]] = phi i32 [ [[ADD2:%.*]], [[IF_END:%.*]] ], [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ]
@@ -897,8 +897,8 @@ define float @cond_cond(ptr noalias %src1, ptr noalias %src2, ptr noalias %cond,
 ; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
 ; CHECK-NEXT:    br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP49]], [[MIDDLE_BLOCK]] ], [ 2.000000e+00, [[ENTRY]] ]
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP49]], [[MIDDLE_BLOCK]] ], [ 2.000000e+00, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ]
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK:       for.body:
 ; CHECK-NEXT:    [[RDX1:%.*]] = phi float [ [[RES:%.*]], [[FOR_INC:%.*]] ], [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ]
@@ -1039,8 +1039,8 @@ define i32 @uncond_cond(ptr noalias %src1, ptr noalias %src2, ptr noalias %cond,
 ; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
 ; CHECK-NEXT:    br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP29]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ]
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP29]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ]
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK:       for.body:
 ; CHECK-NEXT:    [[RDX:%.*]] = phi i32 [ [[RES:%.*]], [[FOR_INC:%.*]] ], [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ]
@@ -1168,8 +1168,8 @@ define i32 @uncond_cond_uncond(ptr noalias %src1, ptr noalias %src2, ptr noalias
 ; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
 ; CHECK-NEXT:    br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP30]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ]
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP30]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ]
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK:       for.body:
 ; CHECK-NEXT:    [[RDX:%.*]] = phi i32 [ [[ADD3:%.*]], [[IF_END:%.*]] ], [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ]
diff --git a/llvm/test/Transforms/LoopVectorize/reduction-inloop-min-max.ll b/llvm/test/Transforms/LoopVectorize/reduction-inloop-min-max.ll
index 3b07a76440e85..3488f52174e66 100644
--- a/llvm/test/Transforms/LoopVectorize/reduction-inloop-min-max.ll
+++ b/llvm/test/Transforms/LoopVectorize/reduction-inloop-min-max.ll
@@ -3,7 +3,7 @@
 
 define i32 @reduction_smin(ptr nocapture %A, ptr nocapture %B) {
 ; CHECK-LABEL: define i32 @reduction_smin
-; CHECK-SAME: (ptr nocapture [[A:%.*]], ptr nocapture [[B:%.*]]) {
+; CHECK-SAME: (ptr captures(none) [[A:%.*]], ptr captures(none) [[B:%.*]]) {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.ph:
@@ -50,7 +50,7 @@ for.end:                                          ; preds = %for.body, %entry
 
 define i32 @reduction_smin_select_ops_flipped(ptr nocapture %A, ptr nocapture %B) {
 ; CHECK-LABEL: define i32 @reduction_smin_select_ops_flipped
-; CHECK-SAME: (ptr nocapture [[A:%.*]], ptr nocapture [[B:%.*]]) {
+; CHECK-SAME: (ptr captures(none) [[A:%.*]], ptr captures(none) [[B:%.*]]) {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.ph:
@@ -97,7 +97,7 @@ for.end:                                          ; preds = %for.body, %entry
 
 define i32 @reduction_smin_intrinsic(ptr nocapture %A, ptr nocapture %B) {
 ; CHECK-LABEL: define i32 @reduction_smin_intrinsic
-; CHECK-SAME: (ptr nocapture [[A:%.*]], ptr nocapture [[B:%.*]]) {
+; CHECK-SAME: (ptr captures(none) [[A:%.*]], ptr captures(none) [[B:%.*]]) {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.ph:
@@ -145,7 +145,7 @@ declare i32 @llvm.smin.i32(i32, i32)
 
 define i32 @reduction_umax(ptr nocapture %A, ptr nocapture %B) {
 ; CHECK-LABEL: define i32 @reduction_umax
-; CHECK-SAME: (ptr nocapture [[A:%.*]], ptr nocapture [[B:%.*]]) {
+; CHECK-SAME: (ptr captures(none) [[A:%.*]], ptr captures(none) [[B:%.*]]) {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.ph:
@@ -192,7 +192,7 @@ for.end:                                          ; preds = %for.body, %entry
 
 define i32 @reduction_umax_select_ops_flipped(ptr nocapture %A, ptr nocapture %B) {
 ; CHECK-LABEL: define i32 @reduction_umax_select_ops_flipped
-; CHECK-SAME: (ptr nocapture [[A:%.*]], ptr nocapture [[B:%.*]]) {
+; CHECK-SAME: (ptr captures(none) [[A:%.*]], ptr captures(none) [[B:%.*]]) {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.ph:
@@ -239,7 +239,7 @@ for.end:                                          ; preds = %for.body, %entry
 
 define i32 @reduction_umax_intrinsic(ptr nocapture %A, ptr nocapture %B) {
 ; CHECK-LABEL: define i32 @reduction_umax_intrinsic
-; CHECK-SAME: (ptr nocapture [[A:%.*]], ptr nocapture [[B:%.*]]) {
+; CHECK-SAME: (ptr captures(none) [[A:%.*]], ptr captures(none) [[B:%.*]]) {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.ph:
diff --git a/llvm/test/Transforms/LoopVectorize/remarks-reduction-inloop.ll b/llvm/test/Transforms/LoopVectorize/remarks-reduction-inloop.ll
index f87c97299205b..094793023de60 100644
--- a/llvm/test/Transforms/LoopVectorize/remarks-reduction-inloop.ll
+++ b/llvm/test/Transforms/LoopVectorize/remarks-reduction-inloop.ll
@@ -7,7 +7,7 @@ target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f3
 
 define i32 @reduction_sum(ptr noalias nocapture %A, ptr noalias nocapture %B) {
 ; CHECK-LABEL: define i32 @reduction_sum(
-; CHECK-SAME: ptr noalias nocapture [[A:%.*]], ptr noalias nocapture [[B:%.*]]) {
+; CHECK-SAME: ptr noalias captures(none) [[A:%.*]], ptr noalias captures(none) [[B:%.*]]) {
 ; CHECK-NEXT:  [[ENTRY:.*]]:
 ; CHECK-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; CHECK:       [[VECTOR_PH]]:
diff --git a/llvm/test/Transforms/LoopVectorize/reverse_induction.ll b/llvm/test/Transforms/LoopVectorize/reverse_induction.ll
index 5028dc5355c50..aff61aea3f9e9 100644
--- a/llvm/test/Transforms/LoopVectorize/reverse_induction.ll
+++ b/llvm/test/Transforms/LoopVectorize/reverse_induction.ll
@@ -196,9 +196,9 @@ define i32 @reverse_induction_i16(i16 %startval, ptr %ptr) {
 ; CHECK-NEXT:    [[TMP17:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[BIN_RDX]])
 ; CHECK-NEXT:    br i1 true, label %[[LOOPEND:.*]], label %[[SCALAR_PH]]
 ; CHECK:       [[SCALAR_PH]]:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i16 [ [[IND_END]], %[[MIDDLE_BLOCK]] ], [ [[STARTVAL]], %[[VECTOR_SCEVCHECK]] ], [ [[STARTVAL]], %[[ENTRY]] ]
-; CHECK-NEXT:    [[BC_RESUME_VAL4:%.*]] = phi i32 [ 1024, %[[MIDDLE_BLOCK]] ], [ 0, %[[VECTOR_SCEVCHECK]] ], [ 0, %[[ENTRY]] ]
-; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP17]], %[[MIDDLE_BLOCK]] ], [ 0, %[[VECTOR_SCEVCHECK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i16 [ [[IND_END]], %[[MIDDLE_BLOCK]] ], [ [[STARTVAL]], %[[ENTRY]] ], [ [[STARTVAL]], %[[VECTOR_SCEVCHECK]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL4:%.*]] = phi i32 [ 1024, %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_SCEVCHECK]] ]
+; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP17]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_SCEVCHECK]] ]
 ; CHECK-NEXT:    br label %[[FOR_BODY:.*]]
 ; CHECK:       [[FOR_BODY]]:
 ; CHECK-NEXT:    [[ADD_I7:%.*]] = phi i16 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[ADD_I:%.*]], %[[FOR_BODY]] ]
diff --git a/llvm/test/Transforms/LoopVectorize/runtime-check-needed-but-empty.ll b/llvm/test/Transforms/LoopVectorize/runtime-check-needed-but-empty.ll
index e14c547d96ad3..7b2af60fcfd23 100644
--- a/llvm/test/Transforms/LoopVectorize/runtime-check-needed-but-empty.ll
+++ b/llvm/test/Transforms/LoopVectorize/runtime-check-needed-but-empty.ll
@@ -31,7 +31,7 @@ define void @test(ptr %A, i32 %x) {
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    br i1 undef, label [[EXIT:%.*]], label [[SCALAR_PH]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ undef, [[MIDDLE_BLOCK]] ], [ 0, [[VECTOR_SCEVCHECK]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ undef, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ], [ 0, [[VECTOR_SCEVCHECK]] ]
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
 ; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[IV_NEXT:%.*]], [[LOOP]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
diff --git a/llvm/test/Transforms/LoopVectorize/runtime-check-small-clamped-bounds.ll b/llvm/test/Transforms/LoopVectorize/runtime-check-small-clamped-bounds.ll
index a3bf5c76a20a6..bb515cd583e5b 100644
--- a/llvm/test/Transforms/LoopVectorize/runtime-check-small-clamped-bounds.ll
+++ b/llvm/test/Transforms/LoopVectorize/runtime-check-small-clamped-bounds.ll
@@ -47,7 +47,7 @@ define void @load_clamped_index(ptr %A, ptr %B, i32 %N) {
 ; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i32 [[N]], [[N_VEC]]
 ; CHECK-NEXT:    br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[VECTOR_MEMCHECK]] ], [ 0, [[VECTOR_SCEVCHECK]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ], [ 0, [[VECTOR_SCEVCHECK]] ], [ 0, [[VECTOR_MEMCHECK]] ]
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
 ; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
@@ -120,7 +120,7 @@ define void @store_clamped_index(ptr %A, ptr %B, i32 %N) {
 ; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i32 [[N]], [[N_VEC]]
 ; CHECK-NEXT:    br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[VECTOR_MEMCHECK]] ], [ 0, [[VECTOR_SCEVCHECK]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ], [ 0, [[VECTOR_SCEVCHECK]] ], [ 0, [[VECTOR_MEMCHECK]] ]
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
 ; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
@@ -201,7 +201,7 @@ define void @load_clamped_index_offset_1(ptr %A, ptr %B, i32 %N) {
 ; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i32 [[TMP0]], [[N_VEC]]
 ; CHECK-NEXT:    br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ 1, [[VECTOR_MEMCHECK]] ], [ 1, [[VECTOR_SCEVCHECK]] ], [ 1, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ 1, [[ENTRY:%.*]] ], [ 1, [[VECTOR_SCEVCHECK]] ], [ 1, [[VECTOR_MEMCHECK]] ]
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
 ; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
@@ -384,7 +384,7 @@ define void @clamped_index_equal_dependence(ptr %A, ptr %B, i32 %N) {
 ; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i32 [[N]], [[N_VEC]]
 ; CHECK-NEXT:    br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[VECTOR_SCEVCHECK]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ], [ 0, [[VECTOR_SCEVCHECK]] ]
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
 ; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
diff --git a/llvm/test/Transforms/LoopVectorize/runtime-check.ll b/llvm/test/Transforms/LoopVectorize/runtime-check.ll
index e0e80271c0d26..5c817ea313183 100644
--- a/llvm/test/Transforms/LoopVectorize/runtime-check.ll
+++ b/llvm/test/Transforms/LoopVectorize/runtime-check.ll
@@ -42,7 +42,7 @@ define i32 @foo(ptr nocapture %a, ptr nocapture %b, i32 %n) nounwind uwtable ssp
 ; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N_VEC]], [[TMP0]], !dbg [[DBG9]]
 ; CHECK-NEXT:    br i1 [[CMP_N]], label [[FOR_END_LOOPEXIT:%.*]], label [[SCALAR_PH]], !dbg [[DBG9]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[VECTOR_MEMCHECK]] ], [ 0, [[FOR_BODY_PREHEADER]] ], !dbg [[DBG9]]
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[FOR_BODY_PREHEADER]] ], [ 0, [[VECTOR_MEMCHECK]] ], !dbg [[DBG9]]
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]], !dbg [[DBG9]]
 ; CHECK:       for.body:
 ; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], !dbg [[DBG9]]
@@ -147,7 +147,7 @@ define void @test_runtime_check(ptr %a, float %b, i64 %offset, i64 %offset2, i64
 ; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
 ; CHECK-NEXT:    br i1 [[CMP_N]], label [[LOOPEXIT:%.*]], label [[SCALAR_PH]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[VECTOR_MEMCHECK]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ], [ 0, [[VECTOR_MEMCHECK]] ]
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK:       for.body:
 ; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
diff --git a/llvm/test/Transforms/LoopVectorize/runtime-checks-difference-simplifications.ll b/llvm/test/Transforms/LoopVectorize/runtime-checks-difference-simplifications.ll
index 038b482209592..d1324314eb953 100644
--- a/llvm/test/Transforms/LoopVectorize/runtime-checks-difference-simplifications.ll
+++ b/llvm/test/Transforms/LoopVectorize/runtime-checks-difference-simplifications.ll
@@ -115,7 +115,7 @@ define void @test_large_number_of_group(ptr %dst, i64 %off, i64 %N) {
 ; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
 ; CHECK-NEXT:    br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[VECTOR_MEMCHECK]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ], [ 0, [[VECTOR_MEMCHECK]] ]
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
 ; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
@@ -259,7 +259,7 @@ define void @check_creation_order(ptr %a, ptr %b, i32 %m) {
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    br i1 false, label [[EXIT:%.*]], label [[SCALAR_PH]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 31996, [[MIDDLE_BLOCK]] ], [ 0, [[VECTOR_MEMCHECK]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 31996, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ], [ 0, [[VECTOR_MEMCHECK]] ]
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
 ; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
diff --git a/llvm/test/Transforms/LoopVectorize/runtime-checks-hoist.ll b/llvm/test/Transforms/LoopVectorize/runtime-checks-hoist.ll
index 5f0a0c5d69a42..a5501a0037b2c 100644
--- a/llvm/test/Transforms/LoopVectorize/runtime-checks-hoist.ll
+++ b/llvm/test/Transforms/LoopVectorize/runtime-checks-hoist.ll
@@ -30,7 +30,7 @@ target datalayout = "e-m:e-i64:64-i128:128-n32:64-S128"
 
 define void @diff_checks(ptr nocapture noundef writeonly %dst, ptr nocapture noundef readonly %src, i32 noundef %m, i32 noundef %n) {
 ; CHECK-LABEL: define void @diff_checks
-; CHECK-SAME: (ptr nocapture noundef writeonly [[DST:%.*]], ptr nocapture noundef readonly [[SRC:%.*]], i32 noundef [[M:%.*]], i32 noundef [[N:%.*]]) {
+; CHECK-SAME: (ptr noundef writeonly captures(none) [[DST:%.*]], ptr noundef readonly captures(none) [[SRC:%.*]], i32 noundef [[M:%.*]], i32 noundef [[N:%.*]]) {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[ADD5:%.*]] = add nuw i32 [[N]], 1
 ; CHECK-NEXT:    [[TMP0:%.*]] = zext i32 [[N]] to i64
@@ -83,7 +83,7 @@ define void @diff_checks(ptr nocapture noundef writeonly %dst, ptr nocapture nou
 ; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[WIDE_N]], [[N_VEC]]
 ; CHECK-NEXT:    br i1 [[CMP_N]], label [[INNER_EXIT]], label [[SCALAR_PH]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[VECTOR_MEMCHECK]] ], [ 0, [[OUTER_LOOP]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[OUTER_LOOP]] ], [ 0, [[VECTOR_MEMCHECK]] ]
 ; CHECK-NEXT:    br label [[INNER_LOOP:%.*]]
 ; CHECK:       inner.loop:
 ; CHECK-NEXT:    [[IV_INNER:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_INNER_NEXT:%.*]], [[INNER_LOOP]] ]
@@ -162,7 +162,7 @@ outer.exit:
 
 define void @full_checks(ptr nocapture noundef %dst, ptr nocapture noundef readonly %src, i32 noundef %m, i32 noundef %n) {
 ; CHECK-LABEL: define void @full_checks
-; CHECK-SAME: (ptr nocapture noundef [[DST:%.*]], ptr nocapture noundef readonly [[SRC:%.*]], i32 noundef [[M:%.*]], i32 noundef [[N:%.*]]) {
+; CHECK-SAME: (ptr noundef captures(none) [[DST:%.*]], ptr noundef readonly captures(none) [[SRC:%.*]], i32 noundef [[M:%.*]], i32 noundef [[N:%.*]]) {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[TMP0:%.*]] = zext i32 [[N]] to i64
 ; CHECK-NEXT:    [[WIDE_M:%.*]] = zext i32 [[M]] to i64
@@ -205,7 +205,7 @@ define void @full_checks(ptr nocapture noundef %dst, ptr nocapture noundef reado
 ; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[WIDE_N]], [[N_VEC]]
 ; CHECK-NEXT:    br i1 [[CMP_N]], label [[INNER_EXIT]], label [[SCALAR_PH]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[VECTOR_MEMCHECK]] ], [ 0, [[OUTER_LOOP]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[OUTER_LOOP]] ], [ 0, [[VECTOR_MEMCHECK]] ]
 ; CHECK-NEXT:    br label [[INNER_LOOP:%.*]]
 ; CHECK:       inner.loop:
 ; CHECK-NEXT:    [[IV_INNER:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_INNER_NEXT:%.*]], [[INNER_LOOP]] ]
@@ -286,7 +286,7 @@ outer.exit:
 
 define void @full_checks_diff_strides(ptr nocapture noundef %dst, ptr nocapture noundef readonly %src, i32 noundef %m, i32 noundef %n) {
 ; CHECK-LABEL: define void @full_checks_diff_strides
-; CHECK-SAME: (ptr nocapture noundef [[DST:%.*]], ptr nocapture noundef readonly [[SRC:%.*]], i32 noundef [[M:%.*]], i32 noundef [[N:%.*]]) {
+; CHECK-SAME: (ptr noundef captures(none) [[DST:%.*]], ptr noundef readonly captures(none) [[SRC:%.*]], i32 noundef [[M:%.*]], i32 noundef [[N:%.*]]) {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[WIDE_M:%.*]] = zext i32 [[M]] to i64
 ; CHECK-NEXT:    [[WIDE_N:%.*]] = zext i32 [[N]] to i64
@@ -337,7 +337,7 @@ define void @full_checks_diff_strides(ptr nocapture noundef %dst, ptr nocapture
 ; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[WIDE_N]], [[N_VEC]]
 ; CHECK-NEXT:    br i1 [[CMP_N]], label [[INNER_EXIT]], label [[SCALAR_PH]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[VECTOR_MEMCHECK]] ], [ 0, [[OUTER_LOOP]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[OUTER_LOOP]] ], [ 0, [[VECTOR_MEMCHECK]] ]
 ; CHECK-NEXT:    br label [[INNER_LOOP:%.*]]
 ; CHECK:       inner.loop:
 ; CHECK-NEXT:    [[IV_INNER:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_INNER_NEXT:%.*]], [[INNER_LOOP]] ]
@@ -411,7 +411,7 @@ outer.exit:
 
 define void @diff_checks_src_start_invariant(ptr nocapture noundef writeonly %dst, ptr nocapture noundef readonly %src, i32 noundef %m, i32 noundef %n) {
 ; CHECK-LABEL: define void @diff_checks_src_start_invariant
-; CHECK-SAME: (ptr nocapture noundef writeonly [[DST:%.*]], ptr nocapture noundef readonly [[SRC:%.*]], i32 noundef [[M:%.*]], i32 noundef [[N:%.*]]) {
+; CHECK-SAME: (ptr noundef writeonly captures(none) [[DST:%.*]], ptr noundef readonly captures(none) [[SRC:%.*]], i32 noundef [[M:%.*]], i32 noundef [[N:%.*]]) {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[SRC2:%.*]] = ptrtoint ptr [[SRC]] to i64
 ; CHECK-NEXT:    [[DST1:%.*]] = ptrtoint ptr [[DST]] to i64
@@ -452,7 +452,7 @@ define void @diff_checks_src_start_invariant(ptr nocapture noundef writeonly %ds
 ; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[WIDE_N]], [[N_VEC]]
 ; CHECK-NEXT:    br i1 [[CMP_N]], label [[INNER_LOOP_EXIT]], label [[SCALAR_PH]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[VECTOR_MEMCHECK]] ], [ 0, [[OUTER_LOOP]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[OUTER_LOOP]] ], [ 0, [[VECTOR_MEMCHECK]] ]
 ; CHECK-NEXT:    br label [[INNER_LOOP:%.*]]
 ; CHECK:       inner.loop:
 ; CHECK-NEXT:    [[IV_INNER:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_INNER_NEXT:%.*]], [[INNER_LOOP]] ]
@@ -522,7 +522,7 @@ outer.loop.exit:
 
 define void @full_checks_src_start_invariant(ptr nocapture noundef %dst, ptr nocapture noundef readonly %src, i32 noundef %m, i32 noundef %n) {
 ; CHECK-LABEL: define void @full_checks_src_start_invariant
-; CHECK-SAME: (ptr nocapture noundef [[DST:%.*]], ptr nocapture noundef readonly [[SRC:%.*]], i32 noundef [[M:%.*]], i32 noundef [[N:%.*]]) {
+; CHECK-SAME: (ptr noundef captures(none) [[DST:%.*]], ptr noundef readonly captures(none) [[SRC:%.*]], i32 noundef [[M:%.*]], i32 noundef [[N:%.*]]) {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[TMP0:%.*]] = zext i32 [[N]] to i64
 ; CHECK-NEXT:    [[WIDE_M:%.*]] = zext i32 [[M]] to i64
@@ -566,7 +566,7 @@ define void @full_checks_src_start_invariant(ptr nocapture noundef %dst, ptr noc
 ; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[WIDE_N]], [[N_VEC]]
 ; CHECK-NEXT:    br i1 [[CMP_N]], label [[INNER_LOOP_EXIT]], label [[SCALAR_PH]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[VECTOR_MEMCHECK]] ], [ 0, [[OUTER_LOOP]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[OUTER_LOOP]] ], [ 0, [[VECTOR_MEMCHECK]] ]
 ; CHECK-NEXT:    br label [[INNER_LOOP:%.*]]
 ; CHECK:       inner.loop:
 ; CHECK-NEXT:    [[IV_INNER:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_INNER_NEXT:%.*]], [[INNER_LOOP]] ]
@@ -645,7 +645,7 @@ outer.loop.exit:
 
 define void @triple_nested_loop_mixed_access(ptr nocapture noundef %dst, ptr nocapture noundef readonly %src, i32 noundef %m, i32 noundef %n, i32 noundef %o) {
 ; CHECK-LABEL: define void @triple_nested_loop_mixed_access
-; CHECK-SAME: (ptr nocapture noundef [[DST:%.*]], ptr nocapture noundef readonly [[SRC:%.*]], i32 noundef [[M:%.*]], i32 noundef [[N:%.*]], i32 noundef [[O:%.*]]) {
+; CHECK-SAME: (ptr noundef captures(none) [[DST:%.*]], ptr noundef readonly captures(none) [[SRC:%.*]], i32 noundef [[M:%.*]], i32 noundef [[N:%.*]], i32 noundef [[O:%.*]]) {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[ADD11:%.*]] = add nsw i32 [[O]], 1
 ; CHECK-NEXT:    [[TMP0:%.*]] = zext i32 [[O]] to i64
@@ -711,7 +711,7 @@ define void @triple_nested_loop_mixed_access(ptr nocapture noundef %dst, ptr noc
 ; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[WIDE_TRIP_COUNT]], [[N_VEC]]
 ; CHECK-NEXT:    br i1 [[CMP_N]], label [[INNER_LOOP_END]], label [[SCALAR_PH]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[VECTOR_MEMCHECK]] ], [ 0, [[OUTER_LOOP]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[OUTER_LOOP]] ], [ 0, [[VECTOR_MEMCHECK]] ]
 ; CHECK-NEXT:    br label [[INNER_LOOP:%.*]]
 ; CHECK:       inner.loop:
 ; CHECK-NEXT:    [[INNER_IV:%.*]] = phi i64 [ [[INNER_IV_NEXT:%.*]], [[INNER_LOOP]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
@@ -810,7 +810,7 @@ exit:
 
 define void @uncomputable_outer_tc(ptr nocapture noundef %dst, ptr nocapture noundef readonly %src, ptr nocapture noundef readonly %str, i32 noundef %n) {
 ; CHECK-LABEL: define void @uncomputable_outer_tc
-; CHECK-SAME: (ptr nocapture noundef [[DST:%.*]], ptr nocapture noundef readonly [[SRC:%.*]], ptr nocapture noundef readonly [[STR:%.*]], i32 noundef [[N:%.*]]) {
+; CHECK-SAME: (ptr noundef captures(none) [[DST:%.*]], ptr noundef readonly captures(none) [[SRC:%.*]], ptr noundef readonly captures(none) [[STR:%.*]], i32 noundef [[N:%.*]]) {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[TMP0:%.*]] = load i8, ptr [[STR]], align 1
 ; CHECK-NEXT:    [[CMP_NOT23:%.*]] = icmp ne i8 [[TMP0]], 0
@@ -869,7 +869,7 @@ define void @uncomputable_outer_tc(ptr nocapture noundef %dst, ptr nocapture nou
 ; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[WIDE_TRIP_COUNT]], [[N_VEC]]
 ; CHECK-NEXT:    br i1 [[CMP_N]], label [[INNER_LOOP_EXIT]], label [[SCALAR_PH]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[VECTOR_MEMCHECK]] ], [ 0, [[OUTER_LOOP]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[OUTER_LOOP]] ], [ 0, [[VECTOR_MEMCHECK]] ]
 ; CHECK-NEXT:    br label [[INNER_LOOP:%.*]]
 ; CHECK:       inner.loop:
 ; CHECK-NEXT:    [[INNER_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INNER_IV_NEXT:%.*]], [[INNER_LOOP]] ]
@@ -965,7 +965,7 @@ while.end:
 
 define void @decreasing_inner_iv(ptr nocapture noundef %dst, ptr nocapture noundef readonly %src, i32 noundef %stride1, i32 noundef %stride2, i32 noundef %m, i32 noundef %n) {
 ; CHECK-LABEL: define void @decreasing_inner_iv
-; CHECK-SAME: (ptr nocapture noundef [[DST:%.*]], ptr nocapture noundef readonly [[SRC:%.*]], i32 noundef [[STRIDE1:%.*]], i32 noundef [[STRIDE2:%.*]], i32 noundef [[M:%.*]], i32 noundef [[N:%.*]]) {
+; CHECK-SAME: (ptr noundef captures(none) [[DST:%.*]], ptr noundef readonly captures(none) [[SRC:%.*]], i32 noundef [[STRIDE1:%.*]], i32 noundef [[STRIDE2:%.*]], i32 noundef [[M:%.*]], i32 noundef [[N:%.*]]) {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[CMP20:%.*]] = icmp sgt i32 [[M]], 0
 ; CHECK-NEXT:    [[CMP218:%.*]] = icmp sgt i32 [[N]], -1
@@ -1010,45 +1010,45 @@ define void @decreasing_inner_iv(ptr nocapture noundef %dst, ptr nocapture nound
 ; CHECK:       vector.ph:
 ; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[TMP15]], 4
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[TMP15]], [[N_MOD_VF]]
-; CHECK-NEXT:    [[IND_END:%.*]] = sub i64 [[TMP0]], [[N_VEC]]
+; CHECK-NEXT:    [[TMP20:%.*]] = sub i64 [[TMP0]], [[N_VEC]]
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[OFFSET_IDX:%.*]] = sub i64 [[TMP0]], [[INDEX]]
-; CHECK-NEXT:    [[TMP20:%.*]] = add i64 [[OFFSET_IDX]], 0
-; CHECK-NEXT:    [[TMP21:%.*]] = add nsw i64 [[TMP20]], [[TMP16]]
-; CHECK-NEXT:    [[TMP22:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[TMP21]]
-; CHECK-NEXT:    [[TMP23:%.*]] = getelementptr inbounds i32, ptr [[TMP22]], i32 0
-; CHECK-NEXT:    [[TMP24:%.*]] = getelementptr inbounds i32, ptr [[TMP23]], i32 -3
-; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP24]], align 4, !alias.scope [[META46:![0-9]+]]
+; CHECK-NEXT:    [[TMP21:%.*]] = add i64 [[OFFSET_IDX]], 0
+; CHECK-NEXT:    [[TMP22:%.*]] = add nsw i64 [[TMP21]], [[TMP16]]
+; CHECK-NEXT:    [[TMP23:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[TMP22]]
+; CHECK-NEXT:    [[TMP24:%.*]] = getelementptr inbounds i32, ptr [[TMP23]], i32 0
+; CHECK-NEXT:    [[TMP25:%.*]] = getelementptr inbounds i32, ptr [[TMP24]], i32 -3
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP25]], align 4, !alias.scope [[META46:![0-9]+]]
 ; CHECK-NEXT:    [[REVERSE:%.*]] = shufflevector <4 x i32> [[WIDE_LOAD]], <4 x i32> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
-; CHECK-NEXT:    [[TMP25:%.*]] = add nsw i64 [[TMP20]], [[TMP17]]
-; CHECK-NEXT:    [[TMP26:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 [[TMP25]]
-; CHECK-NEXT:    [[TMP27:%.*]] = getelementptr inbounds i32, ptr [[TMP26]], i32 0
-; CHECK-NEXT:    [[TMP28:%.*]] = getelementptr inbounds i32, ptr [[TMP27]], i32 -3
-; CHECK-NEXT:    [[WIDE_LOAD3:%.*]] = load <4 x i32>, ptr [[TMP28]], align 4, !alias.scope [[META49:![0-9]+]], !noalias [[META46]]
+; CHECK-NEXT:    [[TMP26:%.*]] = add nsw i64 [[TMP21]], [[TMP17]]
+; CHECK-NEXT:    [[TMP27:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 [[TMP26]]
+; CHECK-NEXT:    [[TMP28:%.*]] = getelementptr inbounds i32, ptr [[TMP27]], i32 0
+; CHECK-NEXT:    [[TMP29:%.*]] = getelementptr inbounds i32, ptr [[TMP28]], i32 -3
+; CHECK-NEXT:    [[WIDE_LOAD3:%.*]] = load <4 x i32>, ptr [[TMP29]], align 4, !alias.scope [[META49:![0-9]+]], !noalias [[META46]]
 ; CHECK-NEXT:    [[REVERSE4:%.*]] = shufflevector <4 x i32> [[WIDE_LOAD3]], <4 x i32> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
-; CHECK-NEXT:    [[TMP29:%.*]] = add nsw <4 x i32> [[REVERSE4]], [[REVERSE]]
-; CHECK-NEXT:    [[REVERSE5:%.*]] = shufflevector <4 x i32> [[TMP29]], <4 x i32> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
-; CHECK-NEXT:    store <4 x i32> [[REVERSE5]], ptr [[TMP28]], align 4, !alias.scope [[META49]], !noalias [[META46]]
+; CHECK-NEXT:    [[TMP30:%.*]] = add nsw <4 x i32> [[REVERSE4]], [[REVERSE]]
+; CHECK-NEXT:    [[REVERSE5:%.*]] = shufflevector <4 x i32> [[TMP30]], <4 x i32> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    store <4 x i32> [[REVERSE5]], ptr [[TMP29]], align 4, !alias.scope [[META49]], !noalias [[META46]]
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
-; CHECK-NEXT:    [[TMP30:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT:    br i1 [[TMP30]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP51:![0-9]+]]
+; CHECK-NEXT:    [[TMP31:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[TMP31]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP51:![0-9]+]]
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[TMP15]], [[N_VEC]]
 ; CHECK-NEXT:    br i1 [[CMP_N]], label [[INNER_LOOP_EXIT]], label [[SCALAR_PH]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ [[TMP0]], [[VECTOR_MEMCHECK]] ], [ [[TMP0]], [[OUTER_LOOP]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP20]], [[MIDDLE_BLOCK]] ], [ [[TMP0]], [[OUTER_LOOP]] ], [ [[TMP0]], [[VECTOR_MEMCHECK]] ]
 ; CHECK-NEXT:    br label [[INNER_LOOP:%.*]]
 ; CHECK:       inner.loop:
 ; CHECK-NEXT:    [[INNER_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INNER_IV_NEXT:%.*]], [[INNER_LOOP]] ]
-; CHECK-NEXT:    [[TMP31:%.*]] = add nsw i64 [[INNER_IV]], [[TMP16]]
-; CHECK-NEXT:    [[ARRAYIDX_US:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[TMP31]]
-; CHECK-NEXT:    [[TMP32:%.*]] = load i32, ptr [[ARRAYIDX_US]], align 4
-; CHECK-NEXT:    [[TMP33:%.*]] = add nsw i64 [[INNER_IV]], [[TMP17]]
-; CHECK-NEXT:    [[ARRAYIDX8_US:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 [[TMP33]]
-; CHECK-NEXT:    [[TMP34:%.*]] = load i32, ptr [[ARRAYIDX8_US]], align 4
-; CHECK-NEXT:    [[ADD9_US:%.*]] = add nsw i32 [[TMP34]], [[TMP32]]
+; CHECK-NEXT:    [[TMP32:%.*]] = add nsw i64 [[INNER_IV]], [[TMP16]]
+; CHECK-NEXT:    [[ARRAYIDX_US:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[TMP32]]
+; CHECK-NEXT:    [[TMP33:%.*]] = load i32, ptr [[ARRAYIDX_US]], align 4
+; CHECK-NEXT:    [[TMP34:%.*]] = add nsw i64 [[INNER_IV]], [[TMP17]]
+; CHECK-NEXT:    [[ARRAYIDX8_US:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 [[TMP34]]
+; CHECK-NEXT:    [[TMP35:%.*]] = load i32, ptr [[ARRAYIDX8_US]], align 4
+; CHECK-NEXT:    [[ADD9_US:%.*]] = add nsw i32 [[TMP35]], [[TMP33]]
 ; CHECK-NEXT:    store i32 [[ADD9_US]], ptr [[ARRAYIDX8_US]], align 4
 ; CHECK-NEXT:    [[INNER_IV_NEXT]] = add nsw i64 [[INNER_IV]], -1
 ; CHECK-NEXT:    [[CMP2_US:%.*]] = icmp sgt i64 [[INNER_IV]], 0
@@ -1132,7 +1132,7 @@ exit:
 
 define void @decreasing_outer_iv(ptr nocapture noundef %dst, ptr nocapture noundef readonly %src, i32 noundef %stride1, i32 noundef %stride2, i32 noundef %m, i32 noundef %n) {
 ; CHECK-LABEL: define void @decreasing_outer_iv
-; CHECK-SAME: (ptr nocapture noundef [[DST:%.*]], ptr nocapture noundef readonly [[SRC:%.*]], i32 noundef [[STRIDE1:%.*]], i32 noundef [[STRIDE2:%.*]], i32 noundef [[M:%.*]], i32 noundef [[N:%.*]]) {
+; CHECK-SAME: (ptr noundef captures(none) [[DST:%.*]], ptr noundef readonly captures(none) [[SRC:%.*]], i32 noundef [[STRIDE1:%.*]], i32 noundef [[STRIDE2:%.*]], i32 noundef [[M:%.*]], i32 noundef [[N:%.*]]) {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[CMP21:%.*]] = icmp slt i32 [[M]], 1
 ; CHECK-NEXT:    [[CMP2_NOT18:%.*]] = icmp slt i32 [[N]], 0
@@ -1200,7 +1200,7 @@ define void @decreasing_outer_iv(ptr nocapture noundef %dst, ptr nocapture nound
 ; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[WIDE_TRIP_COUNT]], [[N_VEC]]
 ; CHECK-NEXT:    br i1 [[CMP_N]], label [[INNER_LOOP_EXIT]], label [[SCALAR_PH]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[VECTOR_MEMCHECK]] ], [ 0, [[OUTER_LOOP]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[OUTER_LOOP]] ], [ 0, [[VECTOR_MEMCHECK]] ]
 ; CHECK-NEXT:    br label [[INNER_LOOP:%.*]]
 ; CHECK:       inner.loop:
 ; CHECK-NEXT:    [[INNER_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INNER_IV_NEXT:%.*]], [[INNER_LOOP]] ]
@@ -1291,7 +1291,7 @@ exit:
 
 define void @unknown_inner_stride(ptr nocapture noundef %dst, ptr nocapture noundef readonly %src, i32 noundef %stride1, i32 noundef %stride2, i32 noundef %m, i32 noundef %n) {
 ; CHECK-LABEL: define void @unknown_inner_stride
-; CHECK-SAME: (ptr nocapture noundef [[DST:%.*]], ptr nocapture noundef readonly [[SRC:%.*]], i32 noundef [[STRIDE1:%.*]], i32 noundef [[STRIDE2:%.*]], i32 noundef [[M:%.*]], i32 noundef [[N:%.*]]) {
+; CHECK-SAME: (ptr noundef captures(none) [[DST:%.*]], ptr noundef readonly captures(none) [[SRC:%.*]], i32 noundef [[STRIDE1:%.*]], i32 noundef [[STRIDE2:%.*]], i32 noundef [[M:%.*]], i32 noundef [[N:%.*]]) {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[CMP26:%.*]] = icmp sgt i32 [[M]], 0
 ; CHECK-NEXT:    [[CMP224:%.*]] = icmp sgt i32 [[N]], 0
@@ -1355,7 +1355,7 @@ define void @unknown_inner_stride(ptr nocapture noundef %dst, ptr nocapture noun
 ; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[WIDE_TRIP_COUNT]], [[N_VEC]]
 ; CHECK-NEXT:    br i1 [[CMP_N]], label [[INNER_LOOP_EXIT]], label [[SCALAR_PH]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[VECTOR_MEMCHECK]] ], [ 0, [[VECTOR_SCEVCHECK]] ], [ 0, [[OUTER_LOOP]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[OUTER_LOOP]] ], [ 0, [[VECTOR_SCEVCHECK]] ], [ 0, [[VECTOR_MEMCHECK]] ]
 ; CHECK-NEXT:    br label [[INNER_LOOP:%.*]]
 ; CHECK:       inner.loop:
 ; CHECK-NEXT:    [[INNER_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INNER_IV_NEXT:%.*]], [[INNER_LOOP]] ]
@@ -1434,7 +1434,7 @@ exit:
 ; values (%dst, %src) of the outer AddRecs.
 define void @nested_loop_start_of_inner_ptr_addrec_is_same_outer_addrec(ptr nocapture noundef %dst, ptr nocapture noundef readonly %src, i64 noundef %m, i64 noundef %n) {
 ; CHECK-LABEL: define void @nested_loop_start_of_inner_ptr_addrec_is_same_outer_addrec
-; CHECK-SAME: (ptr nocapture noundef [[DST:%.*]], ptr nocapture noundef readonly [[SRC:%.*]], i64 noundef [[M:%.*]], i64 noundef [[N:%.*]]) {
+; CHECK-SAME: (ptr noundef captures(none) [[DST:%.*]], ptr noundef readonly captures(none) [[SRC:%.*]], i64 noundef [[M:%.*]], i64 noundef [[N:%.*]]) {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[SRC2:%.*]] = ptrtoint ptr [[SRC]] to i64
 ; CHECK-NEXT:    [[DST1:%.*]] = ptrtoint ptr [[DST]] to i64
@@ -1470,7 +1470,7 @@ define void @nested_loop_start_of_inner_ptr_addrec_is_same_outer_addrec(ptr noca
 ; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
 ; CHECK-NEXT:    br i1 [[CMP_N]], label [[INNER_EXIT]], label [[SCALAR_PH]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[VECTOR_MEMCHECK]] ], [ 0, [[OUTER_LOOP]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[OUTER_LOOP]] ], [ 0, [[VECTOR_MEMCHECK]] ]
 ; CHECK-NEXT:    br label [[INNER_LOOP:%.*]]
 ; CHECK:       inner.loop:
 ; CHECK-NEXT:    [[IV_INNER:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_INNER_NEXT:%.*]], [[INNER_LOOP]] ]
@@ -1558,7 +1558,7 @@ define void @stride_check_known_via_loop_guard(ptr %C, ptr %A, i32 %Acols) {
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    br i1 true, label [[OUTER_LATCH]], label [[SCALAR_PH]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ 1000, [[MIDDLE_BLOCK]] ], [ 0, [[VECTOR_MEMCHECK]] ], [ 0, [[OUTER_HEADER]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ 1000, [[MIDDLE_BLOCK]] ], [ 0, [[OUTER_HEADER]] ], [ 0, [[VECTOR_MEMCHECK]] ]
 ; CHECK-NEXT:    br label [[INNER:%.*]]
 ; CHECK:       inner:
 ; CHECK-NEXT:    [[INNER_IV:%.*]] = phi i32 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INNER_IV_NEXT:%.*]], [[INNER]] ]
diff --git a/llvm/test/Transforms/LoopVectorize/scalable-vf-hint.ll b/llvm/test/Transforms/LoopVectorize/scalable-vf-hint.ll
index e51e49e1cde7c..d956a24e0e64a 100644
--- a/llvm/test/Transforms/LoopVectorize/scalable-vf-hint.ll
+++ b/llvm/test/Transforms/LoopVectorize/scalable-vf-hint.ll
@@ -1,5 +1,5 @@
 ; REQUIRES: asserts
-; RUN: opt -passes=loop-vectorize -pass-remarks-analysis=loop-vectorize -debug-only=loop-vectorize -S -scalable-vectorization=on < %s 2>&1 | FileCheck %s
+; RUN: opt -passes=loop-vectorize -pass-remarks-analysis=loop-vectorize -debug-only=loop-vectorize --disable-output -S -scalable-vectorization=on < %s 2>&1 | FileCheck %s
 
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
 
diff --git a/llvm/test/Transforms/LoopVectorize/scalarized-bitcast.ll b/llvm/test/Transforms/LoopVectorize/scalarized-bitcast.ll
index 0a82c0a3e4cc1..1a36807c1c1ac 100644
--- a/llvm/test/Transforms/LoopVectorize/scalarized-bitcast.ll
+++ b/llvm/test/Transforms/LoopVectorize/scalarized-bitcast.ll
@@ -1,5 +1,5 @@
 ; REQUIRES: asserts
-; RUN: opt -passes=loop-vectorize -force-vector-width=2 -debug-only=loop-vectorize -S -o - < %s 2>&1 | FileCheck %s
+; RUN: opt -passes=loop-vectorize -force-vector-width=2 -debug-only=loop-vectorize --disable-output -S -o - < %s 2>&1 | FileCheck %s
 
 %struct.foo = type { i32, i64 }
 
diff --git a/llvm/test/Transforms/LoopVectorize/scev-exit-phi-invalidation.ll b/llvm/test/Transforms/LoopVectorize/scev-exit-phi-invalidation.ll
index 63bf01fe604e8..a848f98e0949e 100644
--- a/llvm/test/Transforms/LoopVectorize/scev-exit-phi-invalidation.ll
+++ b/llvm/test/Transforms/LoopVectorize/scev-exit-phi-invalidation.ll
@@ -50,24 +50,24 @@ define void @test_pr63368(i1 %c, ptr %A) {
 ; CHECK:       vector.ph4:
 ; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i32 [[TMP2]], 4
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub i32 [[TMP2]], [[N_MOD_VF]]
-; CHECK-NEXT:    [[IND_END:%.*]] = trunc i32 [[N_VEC]] to i8
+; CHECK-NEXT:    [[TMP9:%.*]] = trunc i32 [[N_VEC]] to i8
 ; CHECK-NEXT:    br label [[VECTOR_BODY5:%.*]]
 ; CHECK:       vector.body5:
-; CHECK-NEXT:    [[INDEX8:%.*]] = phi i32 [ 0, [[VECTOR_PH4]] ], [ [[INDEX_NEXT9:%.*]], [[VECTOR_BODY5]] ]
-; CHECK-NEXT:    [[OFFSET_IDX:%.*]] = trunc i32 [[INDEX8]] to i8
-; CHECK-NEXT:    [[TMP14:%.*]] = add i8 [[OFFSET_IDX]], 0
-; CHECK-NEXT:    [[TMP15:%.*]] = add i8 [[TMP14]], 1
-; CHECK-NEXT:    [[TMP16:%.*]] = getelementptr i8, ptr [[A]], i8 [[TMP15]]
-; CHECK-NEXT:    [[TMP17:%.*]] = getelementptr i8, ptr [[TMP16]], i32 0
-; CHECK-NEXT:    store <4 x i8> zeroinitializer, ptr [[TMP17]], align 1
-; CHECK-NEXT:    [[INDEX_NEXT9]] = add nuw i32 [[INDEX8]], 4
-; CHECK-NEXT:    [[TMP18:%.*]] = icmp eq i32 [[INDEX_NEXT9]], [[N_VEC]]
-; CHECK-NEXT:    br i1 [[TMP18]], label [[MIDDLE_BLOCK2:%.*]], label [[VECTOR_BODY5]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK-NEXT:    [[INDEX6:%.*]] = phi i32 [ 0, [[VECTOR_PH4]] ], [ [[INDEX_NEXT7:%.*]], [[VECTOR_BODY5]] ]
+; CHECK-NEXT:    [[OFFSET_IDX:%.*]] = trunc i32 [[INDEX6]] to i8
+; CHECK-NEXT:    [[TMP10:%.*]] = add i8 [[OFFSET_IDX]], 0
+; CHECK-NEXT:    [[TMP11:%.*]] = add i8 [[TMP10]], 1
+; CHECK-NEXT:    [[TMP12:%.*]] = getelementptr i8, ptr [[A]], i8 [[TMP11]]
+; CHECK-NEXT:    [[TMP13:%.*]] = getelementptr i8, ptr [[TMP12]], i32 0
+; CHECK-NEXT:    store <4 x i8> zeroinitializer, ptr [[TMP13]], align 1
+; CHECK-NEXT:    [[INDEX_NEXT7]] = add nuw i32 [[INDEX6]], 4
+; CHECK-NEXT:    [[TMP14:%.*]] = icmp eq i32 [[INDEX_NEXT7]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[TMP14]], label [[MIDDLE_BLOCK2:%.*]], label [[VECTOR_BODY5]], !llvm.loop [[LOOP4:![0-9]+]]
 ; CHECK:       middle.block2:
 ; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i32 [[TMP2]], [[N_VEC]]
 ; CHECK-NEXT:    br i1 [[CMP_N]], label [[EXIT_2:%.*]], label [[SCALAR_PH3]]
 ; CHECK:       scalar.ph3:
-; CHECK-NEXT:    [[BC_RESUME_VAL8:%.*]] = phi i8 [ [[IND_END]], [[MIDDLE_BLOCK2]] ], [ 0, [[VECTOR_SCEVCHECK]] ], [ 0, [[EXIT_1]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL8:%.*]] = phi i8 [ [[TMP9]], [[MIDDLE_BLOCK2]] ], [ 0, [[EXIT_1]] ], [ 0, [[VECTOR_SCEVCHECK]] ]
 ; CHECK-NEXT:    br label [[LOOP_2:%.*]]
 ; CHECK:       loop.2:
 ; CHECK-NEXT:    [[IV_2:%.*]] = phi i8 [ [[BC_RESUME_VAL8]], [[SCALAR_PH3]] ], [ [[IV_2_NEXT:%.*]], [[LOOP_2]] ]
diff --git a/llvm/test/Transforms/LoopVectorize/scev-predicate-reasoning.ll b/llvm/test/Transforms/LoopVectorize/scev-predicate-reasoning.ll
index 20053cd8661d1..122469b8c95e6 100644
--- a/llvm/test/Transforms/LoopVectorize/scev-predicate-reasoning.ll
+++ b/llvm/test/Transforms/LoopVectorize/scev-predicate-reasoning.ll
@@ -48,7 +48,7 @@ define void @step_direction_unknown(i32 %arg, ptr %dst) {
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    br i1 true, label [[EXIT:%.*]], label [[SCALAR_PH]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 1024, [[MIDDLE_BLOCK]] ], [ 0, [[VECTOR_SCEVCHECK]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 1024, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ], [ 0, [[VECTOR_SCEVCHECK]] ]
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
 ; CHECK-NEXT:    [[PHI:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[ADD2:%.*]], [[LOOP]] ]
@@ -204,8 +204,8 @@ define void @implied_wrap_predicate(ptr %A, ptr %B, ptr %C) {
 ; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[TMP4]], [[N_VEC]]
 ; CHECK-NEXT:    br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i16 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ 1, [[VECTOR_MEMCHECK]] ], [ 1, [[VECTOR_SCEVCHECK]] ], [ 1, [[ENTRY:%.*]] ]
-; CHECK-NEXT:    [[BC_RESUME_VAL6:%.*]] = phi i64 [ [[IND_END5]], [[MIDDLE_BLOCK]] ], [ 1, [[VECTOR_MEMCHECK]] ], [ 1, [[VECTOR_SCEVCHECK]] ], [ 1, [[ENTRY]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i16 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ 1, [[ENTRY:%.*]] ], [ 1, [[VECTOR_SCEVCHECK]] ], [ 1, [[VECTOR_MEMCHECK]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL6:%.*]] = phi i64 [ [[IND_END5]], [[MIDDLE_BLOCK]] ], [ 1, [[ENTRY]] ], [ 1, [[VECTOR_SCEVCHECK]] ], [ 1, [[VECTOR_MEMCHECK]] ]
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
 ; CHECK-NEXT:    [[IV:%.*]] = phi i16 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
diff --git a/llvm/test/Transforms/LoopVectorize/select-cmp-multiuse.ll b/llvm/test/Transforms/LoopVectorize/select-cmp-multiuse.ll
index 60f85e7d5936a..576a971c5eaa8 100644
--- a/llvm/test/Transforms/LoopVectorize/select-cmp-multiuse.ll
+++ b/llvm/test/Transforms/LoopVectorize/select-cmp-multiuse.ll
@@ -549,9 +549,9 @@ define i32 @multi_user_cmp_branch_use(ptr readonly %a, ptr %b, i64 noundef %n) {
 ; CHECK-VF4-IC1-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
 ; CHECK-VF4-IC1-NEXT:    br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
 ; CHECK-VF4-IC1:       scalar.ph:
-; CHECK-VF4-IC1-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[VECTOR_MEMCHECK]] ], [ 0, [[ENTRY:%.*]] ]
-; CHECK-VF4-IC1-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i1 [ [[RDX_SELECT]], [[MIDDLE_BLOCK]] ], [ true, [[VECTOR_MEMCHECK]] ], [ true, [[ENTRY]] ]
-; CHECK-VF4-IC1-NEXT:    [[BC_MERGE_RDX10:%.*]] = phi i1 [ [[RDX_SELECT9]], [[MIDDLE_BLOCK]] ], [ false, [[VECTOR_MEMCHECK]] ], [ false, [[ENTRY]] ]
+; CHECK-VF4-IC1-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ], [ 0, [[VECTOR_MEMCHECK]] ]
+; CHECK-VF4-IC1-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i1 [ [[RDX_SELECT]], [[MIDDLE_BLOCK]] ], [ true, [[ENTRY]] ], [ true, [[VECTOR_MEMCHECK]] ]
+; CHECK-VF4-IC1-NEXT:    [[BC_MERGE_RDX10:%.*]] = phi i1 [ [[RDX_SELECT9]], [[MIDDLE_BLOCK]] ], [ false, [[ENTRY]] ], [ false, [[VECTOR_MEMCHECK]] ]
 ; CHECK-VF4-IC1-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK-VF4-IC1:       for.body:
 ; CHECK-VF4-IC1-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[IF_END6:%.*]] ]
@@ -711,9 +711,9 @@ define i32 @multi_user_cmp_branch_use(ptr readonly %a, ptr %b, i64 noundef %n) {
 ; CHECK-VF4-IC2-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
 ; CHECK-VF4-IC2-NEXT:    br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
 ; CHECK-VF4-IC2:       scalar.ph:
-; CHECK-VF4-IC2-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[VECTOR_MEMCHECK]] ], [ 0, [[ENTRY:%.*]] ]
-; CHECK-VF4-IC2-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i1 [ [[RDX_SELECT]], [[MIDDLE_BLOCK]] ], [ true, [[VECTOR_MEMCHECK]] ], [ true, [[ENTRY]] ]
-; CHECK-VF4-IC2-NEXT:    [[BC_MERGE_RDX22:%.*]] = phi i1 [ [[RDX_SELECT21]], [[MIDDLE_BLOCK]] ], [ false, [[VECTOR_MEMCHECK]] ], [ false, [[ENTRY]] ]
+; CHECK-VF4-IC2-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ], [ 0, [[VECTOR_MEMCHECK]] ]
+; CHECK-VF4-IC2-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i1 [ [[RDX_SELECT]], [[MIDDLE_BLOCK]] ], [ true, [[ENTRY]] ], [ true, [[VECTOR_MEMCHECK]] ]
+; CHECK-VF4-IC2-NEXT:    [[BC_MERGE_RDX22:%.*]] = phi i1 [ [[RDX_SELECT21]], [[MIDDLE_BLOCK]] ], [ false, [[ENTRY]] ], [ false, [[VECTOR_MEMCHECK]] ]
 ; CHECK-VF4-IC2-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK-VF4-IC2:       for.body:
 ; CHECK-VF4-IC2-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[IF_END6:%.*]] ]
@@ -808,9 +808,9 @@ define i32 @multi_user_cmp_branch_use(ptr readonly %a, ptr %b, i64 noundef %n) {
 ; CHECK-VF1-IC2-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
 ; CHECK-VF1-IC2-NEXT:    br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
 ; CHECK-VF1-IC2:       scalar.ph:
-; CHECK-VF1-IC2-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[VECTOR_MEMCHECK]] ], [ 0, [[ENTRY:%.*]] ]
-; CHECK-VF1-IC2-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i1 [ [[RDX_SELECT]], [[MIDDLE_BLOCK]] ], [ true, [[VECTOR_MEMCHECK]] ], [ true, [[ENTRY]] ]
-; CHECK-VF1-IC2-NEXT:    [[BC_MERGE_RDX9:%.*]] = phi i1 [ [[RDX_SELECT8]], [[MIDDLE_BLOCK]] ], [ false, [[VECTOR_MEMCHECK]] ], [ false, [[ENTRY]] ]
+; CHECK-VF1-IC2-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ], [ 0, [[VECTOR_MEMCHECK]] ]
+; CHECK-VF1-IC2-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i1 [ [[RDX_SELECT]], [[MIDDLE_BLOCK]] ], [ true, [[ENTRY]] ], [ true, [[VECTOR_MEMCHECK]] ]
+; CHECK-VF1-IC2-NEXT:    [[BC_MERGE_RDX9:%.*]] = phi i1 [ [[RDX_SELECT8]], [[MIDDLE_BLOCK]] ], [ false, [[ENTRY]] ], [ false, [[VECTOR_MEMCHECK]] ]
 ; CHECK-VF1-IC2-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK-VF1-IC2:       for.body:
 ; CHECK-VF1-IC2-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[IF_END6:%.*]] ]
diff --git a/llvm/test/Transforms/LoopVectorize/single_early_exit.ll b/llvm/test/Transforms/LoopVectorize/single_early_exit.ll
index 4ba9cc661313b..b414f53291df4 100644
--- a/llvm/test/Transforms/LoopVectorize/single_early_exit.ll
+++ b/llvm/test/Transforms/LoopVectorize/single_early_exit.ll
@@ -13,9 +13,9 @@ define i64 @same_exit_block_phi_of_consts() {
 ; CHECK-NEXT:    call void @init_mem(ptr [[P2]], i64 1024)
 ; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.ph:
-; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
-; CHECK-NEXT:    [[INDEX1:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT3:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[INDEX1:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT3:%.*]], [[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[OFFSET_IDX:%.*]] = add i64 3, [[INDEX1]]
 ; CHECK-NEXT:    [[TMP0:%.*]] = add i64 [[OFFSET_IDX]], 0
 ; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i8, ptr [[P1]], i64 [[TMP0]]
@@ -30,14 +30,16 @@ define i64 @same_exit_block_phi_of_consts() {
 ; CHECK-NEXT:    [[TMP7:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP6]])
 ; CHECK-NEXT:    [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT3]], 64
 ; CHECK-NEXT:    [[TMP9:%.*]] = or i1 [[TMP7]], [[TMP8]]
-; CHECK-NEXT:    br i1 [[TMP9]], label [[MIDDLE_SPLIT:%.*]], label [[LOOP]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK-NEXT:    br i1 [[TMP9]], label [[MIDDLE_SPLIT:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; CHECK:       middle.split:
-; CHECK-NEXT:    br i1 [[TMP7]], label [[LOOP_END:%.*]], label [[MIDDLE_BLOCK:%.*]]
+; CHECK-NEXT:    br i1 [[TMP7]], label [[VECTOR_EARLY_EXIT:%.*]], label [[MIDDLE_BLOCK:%.*]]
+; CHECK:       vector.early.exit:
+; CHECK-NEXT:    br label [[LOOP_END:%.*]]
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    br i1 true, label [[LOOP_END]], label [[SCALAR_PH]]
 ; CHECK:       scalar.ph:
 ; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 67, [[MIDDLE_BLOCK]] ], [ 3, [[ENTRY:%.*]] ]
-; CHECK-NEXT:    br label [[LOOP1:%.*]]
+; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ [[INDEX_NEXT:%.*]], [[LOOP_INC:%.*]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
 ; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i8, ptr [[P1]], i64 [[INDEX]]
@@ -49,9 +51,9 @@ define i64 @same_exit_block_phi_of_consts() {
 ; CHECK:       loop.inc:
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 1
 ; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp ne i64 [[INDEX_NEXT]], 67
-; CHECK-NEXT:    br i1 [[EXITCOND]], label [[LOOP1]], label [[LOOP_END]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK-NEXT:    br i1 [[EXITCOND]], label [[LOOP]], label [[LOOP_END]], !llvm.loop [[LOOP3:![0-9]+]]
 ; CHECK:       loop.end:
-; CHECK-NEXT:    [[RETVAL:%.*]] = phi i64 [ 0, [[LOOP1]] ], [ 1, [[LOOP_INC]] ], [ 1, [[MIDDLE_BLOCK]] ], [ 0, [[MIDDLE_SPLIT]] ]
+; CHECK-NEXT:    [[RETVAL:%.*]] = phi i64 [ 0, [[LOOP]] ], [ 1, [[LOOP_INC]] ], [ 1, [[MIDDLE_BLOCK]] ], [ 0, [[VECTOR_EARLY_EXIT]] ]
 ; CHECK-NEXT:    ret i64 [[RETVAL]]
 ;
 entry:
@@ -90,9 +92,9 @@ define i64 @diff_exit_block_phi_of_consts() {
 ; CHECK-NEXT:    call void @init_mem(ptr [[P2]], i64 1024)
 ; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.ph:
-; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
-; CHECK-NEXT:    [[INDEX1:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT3:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[INDEX1:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT3:%.*]], [[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[OFFSET_IDX:%.*]] = add i64 3, [[INDEX1]]
 ; CHECK-NEXT:    [[TMP0:%.*]] = add i64 [[OFFSET_IDX]], 0
 ; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i8, ptr [[P1]], i64 [[TMP0]]
@@ -107,14 +109,16 @@ define i64 @diff_exit_block_phi_of_consts() {
 ; CHECK-NEXT:    [[TMP7:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP6]])
 ; CHECK-NEXT:    [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT3]], 64
 ; CHECK-NEXT:    [[TMP9:%.*]] = or i1 [[TMP7]], [[TMP8]]
-; CHECK-NEXT:    br i1 [[TMP9]], label [[MIDDLE_SPLIT:%.*]], label [[LOOP]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK-NEXT:    br i1 [[TMP9]], label [[MIDDLE_SPLIT:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
 ; CHECK:       middle.split:
-; CHECK-NEXT:    br i1 [[TMP7]], label [[LOOP_EARLY_EXIT:%.*]], label [[MIDDLE_BLOCK:%.*]]
+; CHECK-NEXT:    br i1 [[TMP7]], label [[VECTOR_EARLY_EXIT:%.*]], label [[MIDDLE_BLOCK:%.*]]
+; CHECK:       vector.early.exit:
+; CHECK-NEXT:    br label [[LOOP_EARLY_EXIT:%.*]]
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    br i1 true, label [[LOOP_END:%.*]], label [[SCALAR_PH]]
 ; CHECK:       scalar.ph:
 ; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 67, [[MIDDLE_BLOCK]] ], [ 3, [[ENTRY:%.*]] ]
-; CHECK-NEXT:    br label [[LOOP1:%.*]]
+; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ [[INDEX_NEXT:%.*]], [[LOOP_INC:%.*]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
 ; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i8, ptr [[P1]], i64 [[INDEX]]
@@ -126,7 +130,7 @@ define i64 @diff_exit_block_phi_of_consts() {
 ; CHECK:       loop.inc:
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 1
 ; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp ne i64 [[INDEX_NEXT]], 67
-; CHECK-NEXT:    br i1 [[EXITCOND]], label [[LOOP1]], label [[LOOP_END]], !llvm.loop [[LOOP5:![0-9]+]]
+; CHECK-NEXT:    br i1 [[EXITCOND]], label [[LOOP]], label [[LOOP_END]], !llvm.loop [[LOOP5:![0-9]+]]
 ; CHECK:       loop.early.exit:
 ; CHECK-NEXT:    ret i64 0
 ; CHECK:       loop.end:
@@ -171,9 +175,9 @@ define i32 @diff_exit_block_needs_scev_check(i32 %end) {
 ; CHECK-NEXT:    call void @init_mem(ptr [[P1]], i64 1024)
 ; CHECK-NEXT:    call void @init_mem(ptr [[P2]], i64 1024)
 ; CHECK-NEXT:    [[END_CLAMPED:%.*]] = and i32 [[END]], 1023
-; CHECK-NEXT:    [[TMP19:%.*]] = trunc i32 [[END]] to i10
-; CHECK-NEXT:    [[TMP20:%.*]] = zext i10 [[TMP19]] to i64
-; CHECK-NEXT:    [[UMAX1:%.*]] = call i64 @llvm.umax.i64(i64 [[TMP20]], i64 1)
+; CHECK-NEXT:    [[TMP0:%.*]] = trunc i32 [[END]] to i10
+; CHECK-NEXT:    [[TMP1:%.*]] = zext i10 [[TMP0]] to i64
+; CHECK-NEXT:    [[UMAX1:%.*]] = call i64 @llvm.umax.i64(i64 [[TMP1]], i64 1)
 ; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[UMAX1]], 4
 ; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_SCEVCHECK:%.*]]
 ; CHECK:       vector.scevcheck:
@@ -188,47 +192,49 @@ define i32 @diff_exit_block_needs_scev_check(i32 %end) {
 ; CHECK:       vector.ph:
 ; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[UMAX1]], 4
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[UMAX1]], [[N_MOD_VF]]
-; CHECK-NEXT:    [[IND_END:%.*]] = trunc i64 [[N_VEC]] to i8
-; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK-NEXT:    [[TMP8:%.*]] = trunc i64 [[N_VEC]] to i8
+; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
-; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[FOR_BODY]] ]
-; CHECK-NEXT:    [[TMP8:%.*]] = add i64 [[INDEX]], 0
-; CHECK-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i32, ptr [[P1]], i64 [[TMP8]]
-; CHECK-NEXT:    [[TMP10:%.*]] = getelementptr inbounds i32, ptr [[TMP9]], i32 0
-; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP10]], align 4
-; CHECK-NEXT:    [[TMP11:%.*]] = getelementptr inbounds i32, ptr [[P2]], i64 [[TMP8]]
-; CHECK-NEXT:    [[TMP12:%.*]] = getelementptr inbounds i32, ptr [[TMP11]], i32 0
-; CHECK-NEXT:    [[WIDE_LOAD3:%.*]] = load <4 x i32>, ptr [[TMP12]], align 4
-; CHECK-NEXT:    [[TMP13:%.*]] = icmp eq <4 x i32> [[WIDE_LOAD]], [[WIDE_LOAD3]]
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP9:%.*]] = add i64 [[INDEX]], 0
+; CHECK-NEXT:    [[TMP10:%.*]] = getelementptr inbounds i32, ptr [[P1]], i64 [[TMP9]]
+; CHECK-NEXT:    [[TMP11:%.*]] = getelementptr inbounds i32, ptr [[TMP10]], i32 0
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP11]], align 4
+; CHECK-NEXT:    [[TMP12:%.*]] = getelementptr inbounds i32, ptr [[P2]], i64 [[TMP9]]
+; CHECK-NEXT:    [[TMP13:%.*]] = getelementptr inbounds i32, ptr [[TMP12]], i32 0
+; CHECK-NEXT:    [[WIDE_LOAD2:%.*]] = load <4 x i32>, ptr [[TMP13]], align 4
+; CHECK-NEXT:    [[TMP14:%.*]] = icmp eq <4 x i32> [[WIDE_LOAD]], [[WIDE_LOAD2]]
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
-; CHECK-NEXT:    [[TMP16:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP13]])
-; CHECK-NEXT:    [[TMP17:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT:    [[TMP18:%.*]] = or i1 [[TMP16]], [[TMP17]]
-; CHECK-NEXT:    br i1 [[TMP18]], label [[MIDDLE_SPLIT:%.*]], label [[FOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK-NEXT:    [[TMP15:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP14]])
+; CHECK-NEXT:    [[TMP16:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT:    [[TMP17:%.*]] = or i1 [[TMP15]], [[TMP16]]
+; CHECK-NEXT:    br i1 [[TMP17]], label [[MIDDLE_SPLIT:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
 ; CHECK:       middle.split:
-; CHECK-NEXT:    br i1 [[TMP16]], label [[FOUND:%.*]], label [[MIDDLE_BLOCK:%.*]]
+; CHECK-NEXT:    br i1 [[TMP15]], label [[VECTOR_EARLY_EXIT:%.*]], label [[MIDDLE_BLOCK:%.*]]
+; CHECK:       vector.early.exit:
+; CHECK-NEXT:    br label [[FOUND:%.*]]
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[UMAX1]], [[N_VEC]]
 ; CHECK-NEXT:    br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i8 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ 0, [[VECTOR_SCEVCHECK]] ], [ 0, [[ENTRY:%.*]] ]
-; CHECK-NEXT:    [[BC_RESUME_VAL2:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[VECTOR_SCEVCHECK]] ], [ 0, [[ENTRY]] ]
-; CHECK-NEXT:    br label [[FOR_BODY1:%.*]]
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i8 [ [[TMP8]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ], [ 0, [[VECTOR_SCEVCHECK]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL3:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ], [ 0, [[VECTOR_SCEVCHECK]] ]
+; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK:       for.body:
 ; CHECK-NEXT:    [[IND:%.*]] = phi i8 [ [[IND_NEXT:%.*]], [[FOR_INC:%.*]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
-; CHECK-NEXT:    [[GEP_IND:%.*]] = phi i64 [ [[GEP_IND_NEXT:%.*]], [[FOR_INC]] ], [ [[BC_RESUME_VAL2]], [[SCALAR_PH]] ]
+; CHECK-NEXT:    [[GEP_IND:%.*]] = phi i64 [ [[GEP_IND_NEXT:%.*]], [[FOR_INC]] ], [ [[BC_RESUME_VAL3]], [[SCALAR_PH]] ]
 ; CHECK-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds i32, ptr [[P1]], i64 [[GEP_IND]]
-; CHECK-NEXT:    [[TMP0:%.*]] = load i32, ptr [[ARRAYIDX1]], align 4
+; CHECK-NEXT:    [[TMP18:%.*]] = load i32, ptr [[ARRAYIDX1]], align 4
 ; CHECK-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds i32, ptr [[P2]], i64 [[GEP_IND]]
-; CHECK-NEXT:    [[TMP1:%.*]] = load i32, ptr [[ARRAYIDX2]], align 4
-; CHECK-NEXT:    [[CMP_EARLY:%.*]] = icmp eq i32 [[TMP0]], [[TMP1]]
+; CHECK-NEXT:    [[TMP19:%.*]] = load i32, ptr [[ARRAYIDX2]], align 4
+; CHECK-NEXT:    [[CMP_EARLY:%.*]] = icmp eq i32 [[TMP18]], [[TMP19]]
 ; CHECK-NEXT:    br i1 [[CMP_EARLY]], label [[FOUND]], label [[FOR_INC]]
 ; CHECK:       for.inc:
 ; CHECK-NEXT:    [[IND_NEXT]] = add i8 [[IND]], 1
 ; CHECK-NEXT:    [[CONV:%.*]] = zext i8 [[IND_NEXT]] to i32
 ; CHECK-NEXT:    [[GEP_IND_NEXT]] = add i64 [[GEP_IND]], 1
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i32 [[CONV]], [[END_CLAMPED]]
-; CHECK-NEXT:    br i1 [[CMP]], label [[FOR_BODY1]], label [[EXIT]], !llvm.loop [[LOOP7:![0-9]+]]
+; CHECK-NEXT:    br i1 [[CMP]], label [[FOR_BODY]], label [[EXIT]], !llvm.loop [[LOOP7:![0-9]+]]
 ; CHECK:       found:
 ; CHECK-NEXT:    ret i32 1
 ; CHECK:       exit:
@@ -283,27 +289,29 @@ define i32 @diff_blocks_invariant_early_exit_cond(ptr %s) {
 ; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT]], <4 x i1> poison, <4 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP0:%.*]] = xor <4 x i1> [[BROADCAST_SPLAT]], splat (i1 true)
 ; CHECK-NEXT:    [[TMP1:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP0]])
-; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
-; CHECK-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
 ; CHECK-NEXT:    [[TMP2:%.*]] = icmp eq i32 [[INDEX_NEXT]], 276
 ; CHECK-NEXT:    [[TMP3:%.*]] = or i1 [[TMP1]], [[TMP2]]
-; CHECK-NEXT:    br i1 [[TMP3]], label [[MIDDLE_SPLIT:%.*]], label [[FOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+; CHECK-NEXT:    br i1 [[TMP3]], label [[MIDDLE_SPLIT:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
 ; CHECK:       middle.split:
-; CHECK-NEXT:    br i1 [[TMP1]], label [[EARLY_EXIT:%.*]], label [[MIDDLE_BLOCK:%.*]]
+; CHECK-NEXT:    br i1 [[TMP1]], label [[VECTOR_EARLY_EXIT:%.*]], label [[MIDDLE_BLOCK:%.*]]
+; CHECK:       vector.early.exit:
+; CHECK-NEXT:    br label [[EARLY_EXIT:%.*]]
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    br i1 true, label [[FOR_END:%.*]], label [[SCALAR_PH]]
 ; CHECK:       scalar.ph:
 ; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ 266, [[MIDDLE_BLOCK]] ], [ -10, [[ENTRY:%.*]] ]
-; CHECK-NEXT:    br label [[FOR_BODY1:%.*]]
+; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK:       for.body:
 ; CHECK-NEXT:    [[IND:%.*]] = phi i32 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IND_NEXT:%.*]], [[FOR_INC:%.*]] ]
 ; CHECK-NEXT:    br i1 [[COND]], label [[FOR_INC]], label [[EARLY_EXIT]]
 ; CHECK:       for.inc:
 ; CHECK-NEXT:    [[IND_NEXT]] = add nsw i32 [[IND]], 1
 ; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i32 [[IND_NEXT]], 266
-; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_END]], label [[FOR_BODY1]], !llvm.loop [[LOOP9:![0-9]+]]
+; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
 ; CHECK:       early.exit:
 ; CHECK-NEXT:    tail call void @abort()
 ; CHECK-NEXT:    unreachable
diff --git a/llvm/test/Transforms/LoopVectorize/single_early_exit_live_outs.ll b/llvm/test/Transforms/LoopVectorize/single_early_exit_live_outs.ll
index 6e542bd873b8c..1bfe054057089 100644
--- a/llvm/test/Transforms/LoopVectorize/single_early_exit_live_outs.ll
+++ b/llvm/test/Transforms/LoopVectorize/single_early_exit_live_outs.ll
@@ -10,22 +10,54 @@ define i64 @same_exit_block_pre_inc_use1() {
 ; CHECK-NEXT:    [[P2:%.*]] = alloca [1024 x i8], align 1
 ; CHECK-NEXT:    call void @init_mem(ptr [[P1]], i64 1024)
 ; CHECK-NEXT:    call void @init_mem(ptr [[P2]], i64 1024)
-; CHECK-NEXT:    br label [[LAND_RHS:%.*]]
+; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK:       vector.ph:
+; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK:       vector.body:
+; CHECK-NEXT:    [[INDEX1:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT3:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 3, i64 4, i64 5, i64 6>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[OFFSET_IDX:%.*]] = add i64 3, [[INDEX1]]
+; CHECK-NEXT:    [[TMP0:%.*]] = add i64 [[OFFSET_IDX]], 0
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i8, ptr [[P1]], i64 [[TMP0]]
+; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i32 0
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i8>, ptr [[TMP2]], align 1
+; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i8, ptr [[P2]], i64 [[TMP0]]
+; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i8, ptr [[TMP3]], i32 0
+; CHECK-NEXT:    [[WIDE_LOAD2:%.*]] = load <4 x i8>, ptr [[TMP4]], align 1
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq <4 x i8> [[WIDE_LOAD]], [[WIDE_LOAD2]]
+; CHECK-NEXT:    [[INDEX_NEXT3]] = add nuw i64 [[INDEX1]], 4
+; CHECK-NEXT:    [[TMP6:%.*]] = xor <4 x i1> [[TMP5]], splat (i1 true)
+; CHECK-NEXT:    [[TMP7:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP6]])
+; CHECK-NEXT:    [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT3]], 64
+; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <4 x i64> [[VEC_IND]], splat (i64 4)
+; CHECK-NEXT:    [[TMP9:%.*]] = or i1 [[TMP7]], [[TMP8]]
+; CHECK-NEXT:    br i1 [[TMP9]], label [[MIDDLE_SPLIT:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK:       middle.split:
+; CHECK-NEXT:    br i1 [[TMP7]], label [[VECTOR_EARLY_EXIT:%.*]], label [[MIDDLE_BLOCK:%.*]]
+; CHECK:       vector.early.exit:
+; CHECK-NEXT:    [[FIRST_ACTIVE_LANE:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP6]], i1 true)
+; CHECK-NEXT:    [[EARLY_EXIT_VALUE:%.*]] = extractelement <4 x i64> [[VEC_IND]], i64 [[FIRST_ACTIVE_LANE]]
+; CHECK-NEXT:    br label [[LOOP_END:%.*]]
+; CHECK:       middle.block:
+; CHECK-NEXT:    br i1 true, label [[LOOP_END]], label [[SCALAR_PH]]
+; CHECK:       scalar.ph:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 67, [[MIDDLE_BLOCK]] ], [ 3, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
-; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ [[INDEX_NEXT:%.*]], [[FOR_INC:%.*]] ], [ 3, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ [[INDEX_NEXT:%.*]], [[LOOP_INC:%.*]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
 ; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i8, ptr [[P1]], i64 [[INDEX]]
-; CHECK-NEXT:    [[TMP38:%.*]] = load i8, ptr [[ARRAYIDX]], align 1
+; CHECK-NEXT:    [[LD1:%.*]] = load i8, ptr [[ARRAYIDX]], align 1
 ; CHECK-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds i8, ptr [[P2]], i64 [[INDEX]]
-; CHECK-NEXT:    [[TMP39:%.*]] = load i8, ptr [[ARRAYIDX1]], align 1
-; CHECK-NEXT:    [[CMP3:%.*]] = icmp eq i8 [[TMP38]], [[TMP39]]
-; CHECK-NEXT:    br i1 [[CMP3]], label [[FOR_INC]], label [[FOR_END_LOOPEXIT:%.*]]
+; CHECK-NEXT:    [[LD2:%.*]] = load i8, ptr [[ARRAYIDX1]], align 1
+; CHECK-NEXT:    [[CMP3:%.*]] = icmp eq i8 [[LD1]], [[LD2]]
+; CHECK-NEXT:    br i1 [[CMP3]], label [[LOOP_INC]], label [[LOOP_END]]
 ; CHECK:       loop.inc:
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 1
 ; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp ne i64 [[INDEX_NEXT]], 67
-; CHECK-NEXT:    br i1 [[EXITCOND]], label [[LAND_RHS]], label [[FOR_END_LOOPEXIT]]
+; CHECK-NEXT:    br i1 [[EXITCOND]], label [[LOOP]], label [[LOOP_END]], !llvm.loop [[LOOP3:![0-9]+]]
 ; CHECK:       loop.end:
-; CHECK-NEXT:    [[START_0_LCSSA:%.*]] = phi i64 [ [[INDEX]], [[LAND_RHS]] ], [ 67, [[FOR_INC]] ]
-; CHECK-NEXT:    ret i64 [[START_0_LCSSA]]
+; CHECK-NEXT:    [[RETVAL:%.*]] = phi i64 [ [[INDEX]], [[LOOP]] ], [ 67, [[LOOP_INC]] ], [ 67, [[MIDDLE_BLOCK]] ], [ [[EARLY_EXIT_VALUE]], [[VECTOR_EARLY_EXIT]] ]
+; CHECK-NEXT:    ret i64 [[RETVAL]]
 ;
 entry:
   %p1 = alloca [1024 x i8]
@@ -62,22 +94,55 @@ define i64 @same_exit_block_pre_inc1_use_inv_cond(i1 %cond) {
 ; CHECK-NEXT:    [[P2:%.*]] = alloca [1024 x i8], align 1
 ; CHECK-NEXT:    call void @init_mem(ptr [[P1]], i64 1024)
 ; CHECK-NEXT:    call void @init_mem(ptr [[P2]], i64 1024)
+; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK:       vector.ph:
+; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK:       vector.body:
+; CHECK-NEXT:    [[INDEX1:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT3:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 3, i64 4, i64 5, i64 6>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[OFFSET_IDX:%.*]] = add i64 3, [[INDEX1]]
+; CHECK-NEXT:    [[TMP0:%.*]] = add i64 [[OFFSET_IDX]], 0
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i8, ptr [[P1]], i64 [[TMP0]]
+; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i32 0
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i8>, ptr [[TMP2]], align 1
+; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i8, ptr [[P2]], i64 [[TMP0]]
+; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i8, ptr [[TMP3]], i32 0
+; CHECK-NEXT:    [[WIDE_LOAD2:%.*]] = load <4 x i8>, ptr [[TMP4]], align 1
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq <4 x i8> [[WIDE_LOAD]], [[WIDE_LOAD2]]
+; CHECK-NEXT:    [[TMP6:%.*]] = select i1 [[COND]], <4 x i1> [[TMP5]], <4 x i1> zeroinitializer
+; CHECK-NEXT:    [[INDEX_NEXT3]] = add nuw i64 [[INDEX1]], 4
+; CHECK-NEXT:    [[TMP7:%.*]] = xor <4 x i1> [[TMP6]], splat (i1 true)
+; CHECK-NEXT:    [[TMP8:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP7]])
+; CHECK-NEXT:    [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT3]], 64
+; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <4 x i64> [[VEC_IND]], splat (i64 4)
+; CHECK-NEXT:    [[TMP10:%.*]] = or i1 [[TMP8]], [[TMP9]]
+; CHECK-NEXT:    br i1 [[TMP10]], label [[MIDDLE_SPLIT:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK:       middle.split:
+; CHECK-NEXT:    br i1 [[TMP8]], label [[VECTOR_EARLY_EXIT:%.*]], label [[MIDDLE_BLOCK:%.*]]
+; CHECK:       vector.early.exit:
+; CHECK-NEXT:    [[FIRST_ACTIVE_LANE:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP7]], i1 true)
+; CHECK-NEXT:    [[EARLY_EXIT_VALUE:%.*]] = extractelement <4 x i64> [[VEC_IND]], i64 [[FIRST_ACTIVE_LANE]]
+; CHECK-NEXT:    br label [[LOOP_END:%.*]]
+; CHECK:       middle.block:
+; CHECK-NEXT:    br i1 true, label [[LOOP_END]], label [[SCALAR_PH]]
+; CHECK:       scalar.ph:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 67, [[MIDDLE_BLOCK]] ], [ 3, [[ENTRY:%.*]] ]
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
-; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ [[INDEX_NEXT:%.*]], [[LOOP_INC:%.*]] ], [ 3, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ [[INDEX_NEXT:%.*]], [[LOOP_INC:%.*]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
 ; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i8, ptr [[P1]], i64 [[INDEX]]
 ; CHECK-NEXT:    [[LD1:%.*]] = load i8, ptr [[ARRAYIDX]], align 1
 ; CHECK-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds i8, ptr [[P2]], i64 [[INDEX]]
 ; CHECK-NEXT:    [[LD2:%.*]] = load i8, ptr [[ARRAYIDX1]], align 1
 ; CHECK-NEXT:    [[CMP3:%.*]] = icmp eq i8 [[LD1]], [[LD2]]
 ; CHECK-NEXT:    [[CMP4:%.*]] = select i1 [[COND]], i1 [[CMP3]], i1 false
-; CHECK-NEXT:    br i1 [[CMP4]], label [[LOOP_INC]], label [[LOOP_END:%.*]]
+; CHECK-NEXT:    br i1 [[CMP4]], label [[LOOP_INC]], label [[LOOP_END]]
 ; CHECK:       loop.inc:
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 1
 ; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp ne i64 [[INDEX_NEXT]], 67
-; CHECK-NEXT:    br i1 [[EXITCOND]], label [[LOOP]], label [[LOOP_END]]
+; CHECK-NEXT:    br i1 [[EXITCOND]], label [[LOOP]], label [[LOOP_END]], !llvm.loop [[LOOP5:![0-9]+]]
 ; CHECK:       loop.end:
-; CHECK-NEXT:    [[RETVAL:%.*]] = phi i64 [ [[INDEX]], [[LOOP]] ], [ 67, [[LOOP_INC]] ]
+; CHECK-NEXT:    [[RETVAL:%.*]] = phi i64 [ [[INDEX]], [[LOOP]] ], [ 67, [[LOOP_INC]] ], [ 67, [[MIDDLE_BLOCK]] ], [ [[EARLY_EXIT_VALUE]], [[VECTOR_EARLY_EXIT]] ]
 ; CHECK-NEXT:    ret i64 [[RETVAL]]
 ;
 entry:
@@ -115,21 +180,53 @@ define i64 @same_exit_block_pre_inc_use1_gep_two_indices() {
 ; CHECK-NEXT:    [[P2:%.*]] = alloca [1024 x i8], align 1
 ; CHECK-NEXT:    call void @init_mem(ptr [[P1]], i64 1024)
 ; CHECK-NEXT:    call void @init_mem(ptr [[P2]], i64 1024)
+; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK:       vector.ph:
+; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK:       vector.body:
+; CHECK-NEXT:    [[INDEX1:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT3:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 3, i64 4, i64 5, i64 6>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[OFFSET_IDX:%.*]] = add i64 3, [[INDEX1]]
+; CHECK-NEXT:    [[TMP0:%.*]] = add i64 [[OFFSET_IDX]], 0
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds [1024 x i8], ptr [[P1]], i64 0, i64 [[TMP0]]
+; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i32 0
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i8>, ptr [[TMP2]], align 1
+; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds [1024 x i8], ptr [[P2]], i64 0, i64 [[TMP0]]
+; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i8, ptr [[TMP3]], i32 0
+; CHECK-NEXT:    [[WIDE_LOAD2:%.*]] = load <4 x i8>, ptr [[TMP4]], align 1
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq <4 x i8> [[WIDE_LOAD]], [[WIDE_LOAD2]]
+; CHECK-NEXT:    [[INDEX_NEXT3]] = add nuw i64 [[INDEX1]], 4
+; CHECK-NEXT:    [[TMP6:%.*]] = xor <4 x i1> [[TMP5]], splat (i1 true)
+; CHECK-NEXT:    [[TMP7:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP6]])
+; CHECK-NEXT:    [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT3]], 64
+; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <4 x i64> [[VEC_IND]], splat (i64 4)
+; CHECK-NEXT:    [[TMP9:%.*]] = or i1 [[TMP7]], [[TMP8]]
+; CHECK-NEXT:    br i1 [[TMP9]], label [[MIDDLE_SPLIT:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK:       middle.split:
+; CHECK-NEXT:    br i1 [[TMP7]], label [[VECTOR_EARLY_EXIT:%.*]], label [[MIDDLE_BLOCK:%.*]]
+; CHECK:       vector.early.exit:
+; CHECK-NEXT:    [[FIRST_ACTIVE_LANE:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP6]], i1 true)
+; CHECK-NEXT:    [[EARLY_EXIT_VALUE:%.*]] = extractelement <4 x i64> [[VEC_IND]], i64 [[FIRST_ACTIVE_LANE]]
+; CHECK-NEXT:    br label [[LOOP_END:%.*]]
+; CHECK:       middle.block:
+; CHECK-NEXT:    br i1 true, label [[LOOP_END]], label [[SCALAR_PH]]
+; CHECK:       scalar.ph:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 67, [[MIDDLE_BLOCK]] ], [ 3, [[ENTRY:%.*]] ]
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
-; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ [[INDEX_NEXT:%.*]], [[LOOP_INC:%.*]] ], [ 3, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ [[INDEX_NEXT:%.*]], [[LOOP_INC:%.*]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
 ; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [1024 x i8], ptr [[P1]], i64 0, i64 [[INDEX]]
 ; CHECK-NEXT:    [[LD1:%.*]] = load i8, ptr [[ARRAYIDX]], align 1
 ; CHECK-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds [1024 x i8], ptr [[P2]], i64 0, i64 [[INDEX]]
 ; CHECK-NEXT:    [[LD2:%.*]] = load i8, ptr [[ARRAYIDX1]], align 1
 ; CHECK-NEXT:    [[CMP3:%.*]] = icmp eq i8 [[LD1]], [[LD2]]
-; CHECK-NEXT:    br i1 [[CMP3]], label [[LOOP_INC]], label [[LOOP_END:%.*]]
+; CHECK-NEXT:    br i1 [[CMP3]], label [[LOOP_INC]], label [[LOOP_END]]
 ; CHECK:       loop.inc:
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 1
 ; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp ne i64 [[INDEX_NEXT]], 67
-; CHECK-NEXT:    br i1 [[EXITCOND]], label [[LOOP]], label [[LOOP_END]]
+; CHECK-NEXT:    br i1 [[EXITCOND]], label [[LOOP]], label [[LOOP_END]], !llvm.loop [[LOOP7:![0-9]+]]
 ; CHECK:       loop.end:
-; CHECK-NEXT:    [[RETVAL:%.*]] = phi i64 [ [[INDEX]], [[LOOP]] ], [ 67, [[LOOP_INC]] ]
+; CHECK-NEXT:    [[RETVAL:%.*]] = phi i64 [ [[INDEX]], [[LOOP]] ], [ 67, [[LOOP_INC]] ], [ 67, [[MIDDLE_BLOCK]] ], [ [[EARLY_EXIT_VALUE]], [[VECTOR_EARLY_EXIT]] ]
 ; CHECK-NEXT:    ret i64 [[RETVAL]]
 ;
 entry:
@@ -166,22 +263,54 @@ define i64 @same_exit_block_pre_inc_use1_alloca_diff_type() {
 ; CHECK-NEXT:    [[P2:%.*]] = alloca [40 x i32], align 4
 ; CHECK-NEXT:    call void @init_mem(ptr [[P1]], i64 1024)
 ; CHECK-NEXT:    call void @init_mem(ptr [[P2]], i64 1024)
-; CHECK-NEXT:    br label [[LAND_RHS:%.*]]
+; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK:       vector.ph:
+; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK:       vector.body:
+; CHECK-NEXT:    [[INDEX1:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT3:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 3, i64 4, i64 5, i64 6>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[OFFSET_IDX:%.*]] = add i64 3, [[INDEX1]]
+; CHECK-NEXT:    [[TMP0:%.*]] = add i64 [[OFFSET_IDX]], 0
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i8, ptr [[P1]], i64 [[TMP0]]
+; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i32 0
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i8>, ptr [[TMP2]], align 1
+; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i8, ptr [[P2]], i64 [[TMP0]]
+; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i8, ptr [[TMP3]], i32 0
+; CHECK-NEXT:    [[WIDE_LOAD2:%.*]] = load <4 x i8>, ptr [[TMP4]], align 1
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq <4 x i8> [[WIDE_LOAD]], [[WIDE_LOAD2]]
+; CHECK-NEXT:    [[INDEX_NEXT3]] = add nuw i64 [[INDEX1]], 4
+; CHECK-NEXT:    [[TMP6:%.*]] = xor <4 x i1> [[TMP5]], splat (i1 true)
+; CHECK-NEXT:    [[TMP7:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP6]])
+; CHECK-NEXT:    [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT3]], 64
+; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <4 x i64> [[VEC_IND]], splat (i64 4)
+; CHECK-NEXT:    [[TMP9:%.*]] = or i1 [[TMP7]], [[TMP8]]
+; CHECK-NEXT:    br i1 [[TMP9]], label [[MIDDLE_SPLIT:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+; CHECK:       middle.split:
+; CHECK-NEXT:    br i1 [[TMP7]], label [[VECTOR_EARLY_EXIT:%.*]], label [[MIDDLE_BLOCK:%.*]]
+; CHECK:       vector.early.exit:
+; CHECK-NEXT:    [[FIRST_ACTIVE_LANE:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP6]], i1 true)
+; CHECK-NEXT:    [[EARLY_EXIT_VALUE:%.*]] = extractelement <4 x i64> [[VEC_IND]], i64 [[FIRST_ACTIVE_LANE]]
+; CHECK-NEXT:    br label [[LOOP_END:%.*]]
+; CHECK:       middle.block:
+; CHECK-NEXT:    br i1 true, label [[LOOP_END]], label [[SCALAR_PH]]
+; CHECK:       scalar.ph:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 67, [[MIDDLE_BLOCK]] ], [ 3, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
-; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ [[INDEX_NEXT:%.*]], [[FOR_INC:%.*]] ], [ 3, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ [[INDEX_NEXT:%.*]], [[LOOP_INC:%.*]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
 ; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i8, ptr [[P1]], i64 [[INDEX]]
-; CHECK-NEXT:    [[TMP38:%.*]] = load i8, ptr [[ARRAYIDX]], align 1
+; CHECK-NEXT:    [[LD1:%.*]] = load i8, ptr [[ARRAYIDX]], align 1
 ; CHECK-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds i8, ptr [[P2]], i64 [[INDEX]]
-; CHECK-NEXT:    [[TMP39:%.*]] = load i8, ptr [[ARRAYIDX1]], align 1
-; CHECK-NEXT:    [[CMP3:%.*]] = icmp eq i8 [[TMP38]], [[TMP39]]
-; CHECK-NEXT:    br i1 [[CMP3]], label [[FOR_INC]], label [[FOR_END_LOOPEXIT:%.*]]
+; CHECK-NEXT:    [[LD2:%.*]] = load i8, ptr [[ARRAYIDX1]], align 1
+; CHECK-NEXT:    [[CMP3:%.*]] = icmp eq i8 [[LD1]], [[LD2]]
+; CHECK-NEXT:    br i1 [[CMP3]], label [[LOOP_INC]], label [[LOOP_END]]
 ; CHECK:       loop.inc:
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 1
 ; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp ne i64 [[INDEX_NEXT]], 67
-; CHECK-NEXT:    br i1 [[EXITCOND]], label [[LAND_RHS]], label [[FOR_END_LOOPEXIT]]
+; CHECK-NEXT:    br i1 [[EXITCOND]], label [[LOOP]], label [[LOOP_END]], !llvm.loop [[LOOP9:![0-9]+]]
 ; CHECK:       loop.end:
-; CHECK-NEXT:    [[START_0_LCSSA:%.*]] = phi i64 [ [[INDEX]], [[LAND_RHS]] ], [ 67, [[FOR_INC]] ]
-; CHECK-NEXT:    ret i64 [[START_0_LCSSA]]
+; CHECK-NEXT:    [[RETVAL:%.*]] = phi i64 [ [[INDEX]], [[LOOP]] ], [ 67, [[LOOP_INC]] ], [ 67, [[MIDDLE_BLOCK]] ], [ [[EARLY_EXIT_VALUE]], [[VECTOR_EARLY_EXIT]] ]
+; CHECK-NEXT:    ret i64 [[RETVAL]]
 ;
 entry:
   %p1 = alloca [40 x i32]
@@ -217,21 +346,52 @@ define i64 @same_exit_block_pre_inc_use2() {
 ; CHECK-NEXT:    [[P2:%.*]] = alloca [1024 x i8], align 1
 ; CHECK-NEXT:    call void @init_mem(ptr [[P1]], i64 1024)
 ; CHECK-NEXT:    call void @init_mem(ptr [[P2]], i64 1024)
+; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK:       vector.ph:
+; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK:       vector.body:
+; CHECK-NEXT:    [[INDEX1:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT3:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 3, i64 4, i64 5, i64 6>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[OFFSET_IDX:%.*]] = add i64 3, [[INDEX1]]
+; CHECK-NEXT:    [[TMP0:%.*]] = add i64 [[OFFSET_IDX]], 0
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i8, ptr [[P1]], i64 [[TMP0]]
+; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i32 0
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i8>, ptr [[TMP2]], align 1
+; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i8, ptr [[P2]], i64 [[TMP0]]
+; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i8, ptr [[TMP3]], i32 0
+; CHECK-NEXT:    [[WIDE_LOAD2:%.*]] = load <4 x i8>, ptr [[TMP4]], align 1
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq <4 x i8> [[WIDE_LOAD]], [[WIDE_LOAD2]]
+; CHECK-NEXT:    [[INDEX_NEXT3]] = add nuw i64 [[INDEX1]], 4
+; CHECK-NEXT:    [[TMP6:%.*]] = xor <4 x i1> [[TMP5]], splat (i1 true)
+; CHECK-NEXT:    [[TMP7:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP6]])
+; CHECK-NEXT:    [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT3]], 64
+; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <4 x i64> [[VEC_IND]], splat (i64 4)
+; CHECK-NEXT:    [[TMP9:%.*]] = or i1 [[TMP7]], [[TMP8]]
+; CHECK-NEXT:    br i1 [[TMP9]], label [[MIDDLE_SPLIT:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
+; CHECK:       middle.split:
+; CHECK-NEXT:    [[TMP10:%.*]] = extractelement <4 x i64> [[VEC_IND]], i32 3
+; CHECK-NEXT:    br i1 [[TMP7]], label [[VECTOR_EARLY_EXIT:%.*]], label [[MIDDLE_BLOCK:%.*]]
+; CHECK:       vector.early.exit:
+; CHECK-NEXT:    br label [[LOOP_END:%.*]]
+; CHECK:       middle.block:
+; CHECK-NEXT:    br i1 true, label [[LOOP_END]], label [[SCALAR_PH]]
+; CHECK:       scalar.ph:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 67, [[MIDDLE_BLOCK]] ], [ 3, [[ENTRY:%.*]] ]
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
-; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ [[INDEX_NEXT:%.*]], [[LOOP_INC:%.*]] ], [ 3, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ [[INDEX_NEXT:%.*]], [[LOOP_INC:%.*]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
 ; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i8, ptr [[P1]], i64 [[INDEX]]
 ; CHECK-NEXT:    [[LD1:%.*]] = load i8, ptr [[ARRAYIDX]], align 1
 ; CHECK-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds i8, ptr [[P2]], i64 [[INDEX]]
 ; CHECK-NEXT:    [[LD2:%.*]] = load i8, ptr [[ARRAYIDX1]], align 1
 ; CHECK-NEXT:    [[CMP3:%.*]] = icmp eq i8 [[LD1]], [[LD2]]
-; CHECK-NEXT:    br i1 [[CMP3]], label [[LOOP_INC]], label [[LOOP_END:%.*]]
+; CHECK-NEXT:    br i1 [[CMP3]], label [[LOOP_INC]], label [[LOOP_END]]
 ; CHECK:       loop.inc:
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 1
 ; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp ne i64 [[INDEX_NEXT]], 67
-; CHECK-NEXT:    br i1 [[EXITCOND]], label [[LOOP]], label [[LOOP_END]]
+; CHECK-NEXT:    br i1 [[EXITCOND]], label [[LOOP]], label [[LOOP_END]], !llvm.loop [[LOOP11:![0-9]+]]
 ; CHECK:       loop.end:
-; CHECK-NEXT:    [[RETVAL:%.*]] = phi i64 [ 67, [[LOOP]] ], [ [[INDEX]], [[LOOP_INC]] ]
+; CHECK-NEXT:    [[RETVAL:%.*]] = phi i64 [ 67, [[LOOP]] ], [ [[INDEX]], [[LOOP_INC]] ], [ [[TMP10]], [[MIDDLE_BLOCK]] ], [ 67, [[VECTOR_EARLY_EXIT]] ]
 ; CHECK-NEXT:    ret i64 [[RETVAL]]
 ;
 entry:
@@ -268,21 +428,54 @@ define i64 @same_exit_block_pre_inc_use3() {
 ; CHECK-NEXT:    [[P2:%.*]] = alloca [1024 x i8], align 1
 ; CHECK-NEXT:    call void @init_mem(ptr [[P1]], i64 1024)
 ; CHECK-NEXT:    call void @init_mem(ptr [[P2]], i64 1024)
+; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK:       vector.ph:
+; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK:       vector.body:
+; CHECK-NEXT:    [[INDEX1:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT3:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 3, i64 4, i64 5, i64 6>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[OFFSET_IDX:%.*]] = add i64 3, [[INDEX1]]
+; CHECK-NEXT:    [[TMP0:%.*]] = add i64 [[OFFSET_IDX]], 0
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i8, ptr [[P1]], i64 [[TMP0]]
+; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i32 0
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i8>, ptr [[TMP2]], align 1
+; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i8, ptr [[P2]], i64 [[TMP0]]
+; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i8, ptr [[TMP3]], i32 0
+; CHECK-NEXT:    [[WIDE_LOAD2:%.*]] = load <4 x i8>, ptr [[TMP4]], align 1
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq <4 x i8> [[WIDE_LOAD]], [[WIDE_LOAD2]]
+; CHECK-NEXT:    [[INDEX_NEXT3]] = add nuw i64 [[INDEX1]], 4
+; CHECK-NEXT:    [[TMP6:%.*]] = xor <4 x i1> [[TMP5]], splat (i1 true)
+; CHECK-NEXT:    [[TMP7:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP6]])
+; CHECK-NEXT:    [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT3]], 64
+; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <4 x i64> [[VEC_IND]], splat (i64 4)
+; CHECK-NEXT:    [[TMP9:%.*]] = or i1 [[TMP7]], [[TMP8]]
+; CHECK-NEXT:    br i1 [[TMP9]], label [[MIDDLE_SPLIT:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
+; CHECK:       middle.split:
+; CHECK-NEXT:    [[TMP10:%.*]] = extractelement <4 x i64> [[VEC_IND]], i32 3
+; CHECK-NEXT:    br i1 [[TMP7]], label [[VECTOR_EARLY_EXIT:%.*]], label [[MIDDLE_BLOCK:%.*]]
+; CHECK:       vector.early.exit:
+; CHECK-NEXT:    [[FIRST_ACTIVE_LANE:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP6]], i1 true)
+; CHECK-NEXT:    [[EARLY_EXIT_VALUE:%.*]] = extractelement <4 x i64> [[VEC_IND]], i64 [[FIRST_ACTIVE_LANE]]
+; CHECK-NEXT:    br label [[LOOP_END:%.*]]
+; CHECK:       middle.block:
+; CHECK-NEXT:    br i1 true, label [[LOOP_END]], label [[SCALAR_PH]]
+; CHECK:       scalar.ph:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 67, [[MIDDLE_BLOCK]] ], [ 3, [[ENTRY:%.*]] ]
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
-; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ [[INDEX_NEXT:%.*]], [[LOOP_INC:%.*]] ], [ 3, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ [[INDEX_NEXT:%.*]], [[LOOP_INC:%.*]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
 ; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i8, ptr [[P1]], i64 [[INDEX]]
 ; CHECK-NEXT:    [[LD1:%.*]] = load i8, ptr [[ARRAYIDX]], align 1
 ; CHECK-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds i8, ptr [[P2]], i64 [[INDEX]]
 ; CHECK-NEXT:    [[LD2:%.*]] = load i8, ptr [[ARRAYIDX1]], align 1
 ; CHECK-NEXT:    [[CMP3:%.*]] = icmp eq i8 [[LD1]], [[LD2]]
-; CHECK-NEXT:    br i1 [[CMP3]], label [[LOOP_INC]], label [[LOOP_END:%.*]]
+; CHECK-NEXT:    br i1 [[CMP3]], label [[LOOP_INC]], label [[LOOP_END]]
 ; CHECK:       loop.inc:
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 1
 ; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp ne i64 [[INDEX_NEXT]], 67
-; CHECK-NEXT:    br i1 [[EXITCOND]], label [[LOOP]], label [[LOOP_END]]
+; CHECK-NEXT:    br i1 [[EXITCOND]], label [[LOOP]], label [[LOOP_END]], !llvm.loop [[LOOP13:![0-9]+]]
 ; CHECK:       loop.end:
-; CHECK-NEXT:    [[INDEX_LCSSA:%.*]] = phi i64 [ [[INDEX]], [[LOOP_INC]] ], [ [[INDEX]], [[LOOP]] ]
+; CHECK-NEXT:    [[INDEX_LCSSA:%.*]] = phi i64 [ [[INDEX]], [[LOOP_INC]] ], [ [[INDEX]], [[LOOP]] ], [ [[TMP10]], [[MIDDLE_BLOCK]] ], [ [[EARLY_EXIT_VALUE]], [[VECTOR_EARLY_EXIT]] ]
 ; CHECK-NEXT:    ret i64 [[INDEX_LCSSA]]
 ;
 entry:
@@ -320,19 +513,48 @@ define i64 @same_exit_block_pre_inc_use4() {
 ; CHECK-NEXT:    [[P2:%.*]] = alloca [1024 x i64], align 8
 ; CHECK-NEXT:    call void @init_mem(ptr [[P1]], i64 1024)
 ; CHECK-NEXT:    call void @init_mem(ptr [[P2]], i64 1024)
+; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK:       vector.ph:
+; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK:       vector.body:
+; CHECK-NEXT:    [[INDEX1:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT2:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 3, i64 4, i64 5, i64 6>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[OFFSET_IDX:%.*]] = add i64 3, [[INDEX1]]
+; CHECK-NEXT:    [[TMP0:%.*]] = add i64 [[OFFSET_IDX]], 0
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i64, ptr [[P1]], i64 [[TMP0]]
+; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i64, ptr [[TMP1]], i32 0
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i64>, ptr [[TMP2]], align 1
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp ult <4 x i64> [[VEC_IND]], [[WIDE_LOAD]]
+; CHECK-NEXT:    [[INDEX_NEXT2]] = add nuw i64 [[INDEX1]], 4
+; CHECK-NEXT:    [[TMP4:%.*]] = xor <4 x i1> [[TMP3]], splat (i1 true)
+; CHECK-NEXT:    [[TMP5:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP4]])
+; CHECK-NEXT:    [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT2]], 64
+; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <4 x i64> [[VEC_IND]], splat (i64 4)
+; CHECK-NEXT:    [[TMP7:%.*]] = or i1 [[TMP5]], [[TMP6]]
+; CHECK-NEXT:    br i1 [[TMP7]], label [[MIDDLE_SPLIT:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
+; CHECK:       middle.split:
+; CHECK-NEXT:    br i1 [[TMP5]], label [[VECTOR_EARLY_EXIT:%.*]], label [[MIDDLE_BLOCK:%.*]]
+; CHECK:       vector.early.exit:
+; CHECK-NEXT:    [[FIRST_ACTIVE_LANE:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP4]], i1 true)
+; CHECK-NEXT:    [[EARLY_EXIT_VALUE:%.*]] = extractelement <4 x i64> [[VEC_IND]], i64 [[FIRST_ACTIVE_LANE]]
+; CHECK-NEXT:    br label [[LOOP_END:%.*]]
+; CHECK:       middle.block:
+; CHECK-NEXT:    br i1 true, label [[LOOP_END]], label [[SCALAR_PH]]
+; CHECK:       scalar.ph:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 67, [[MIDDLE_BLOCK]] ], [ 3, [[ENTRY:%.*]] ]
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
-; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ [[INDEX_NEXT:%.*]], [[LOOP_INC:%.*]] ], [ 3, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ [[INDEX_NEXT:%.*]], [[LOOP_INC:%.*]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
 ; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i64, ptr [[P1]], i64 [[INDEX]]
 ; CHECK-NEXT:    [[LD1:%.*]] = load i64, ptr [[ARRAYIDX]], align 1
 ; CHECK-NEXT:    [[CMP3:%.*]] = icmp ult i64 [[INDEX]], [[LD1]]
-; CHECK-NEXT:    br i1 [[CMP3]], label [[LOOP_INC]], label [[LOOP_END:%.*]]
+; CHECK-NEXT:    br i1 [[CMP3]], label [[LOOP_INC]], label [[LOOP_END]]
 ; CHECK:       loop.inc:
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 1
 ; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp ne i64 [[INDEX_NEXT]], 67
-; CHECK-NEXT:    br i1 [[EXITCOND]], label [[LOOP]], label [[LOOP_END]]
+; CHECK-NEXT:    br i1 [[EXITCOND]], label [[LOOP]], label [[LOOP_END]], !llvm.loop [[LOOP15:![0-9]+]]
 ; CHECK:       loop.end:
-; CHECK-NEXT:    [[RETVAL:%.*]] = phi i64 [ [[INDEX]], [[LOOP]] ], [ 67, [[LOOP_INC]] ]
+; CHECK-NEXT:    [[RETVAL:%.*]] = phi i64 [ [[INDEX]], [[LOOP]] ], [ 67, [[LOOP_INC]] ], [ 67, [[MIDDLE_BLOCK]] ], [ [[EARLY_EXIT_VALUE]], [[VECTOR_EARLY_EXIT]] ]
 ; CHECK-NEXT:    ret i64 [[RETVAL]]
 ;
 entry:
@@ -367,21 +589,60 @@ define i64 @same_exit_block_post_inc_use() {
 ; CHECK-NEXT:    [[P2:%.*]] = alloca [1024 x i8], align 1
 ; CHECK-NEXT:    call void @init_mem(ptr [[P1]], i64 1024)
 ; CHECK-NEXT:    call void @init_mem(ptr [[P2]], i64 1024)
+; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK:       vector.ph:
+; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK:       vector.body:
+; CHECK-NEXT:    [[INDEX1:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT3:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 3, i64 4, i64 5, i64 6>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[OFFSET_IDX:%.*]] = add i64 3, [[INDEX1]]
+; CHECK-NEXT:    [[TMP0:%.*]] = add i64 [[OFFSET_IDX]], 0
+; CHECK-NEXT:    [[TMP1:%.*]] = add i64 [[OFFSET_IDX]], 1
+; CHECK-NEXT:    [[TMP2:%.*]] = add i64 [[OFFSET_IDX]], 2
+; CHECK-NEXT:    [[TMP3:%.*]] = add i64 [[OFFSET_IDX]], 3
+; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i8, ptr [[P1]], i64 [[TMP0]]
+; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i8, ptr [[TMP4]], i32 0
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i8>, ptr [[TMP5]], align 1
+; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i8, ptr [[P2]], i64 [[TMP0]]
+; CHECK-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i8, ptr [[TMP6]], i32 0
+; CHECK-NEXT:    [[WIDE_LOAD2:%.*]] = load <4 x i8>, ptr [[TMP7]], align 1
+; CHECK-NEXT:    [[TMP8:%.*]] = icmp eq <4 x i8> [[WIDE_LOAD]], [[WIDE_LOAD2]]
+; CHECK-NEXT:    [[TMP9:%.*]] = add i64 [[TMP0]], 1
+; CHECK-NEXT:    [[TMP10:%.*]] = add i64 [[TMP1]], 1
+; CHECK-NEXT:    [[TMP11:%.*]] = add i64 [[TMP2]], 1
+; CHECK-NEXT:    [[TMP12:%.*]] = add i64 [[TMP3]], 1
+; CHECK-NEXT:    [[INDEX_NEXT3]] = add nuw i64 [[INDEX1]], 4
+; CHECK-NEXT:    [[TMP13:%.*]] = xor <4 x i1> [[TMP8]], splat (i1 true)
+; CHECK-NEXT:    [[TMP14:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP13]])
+; CHECK-NEXT:    [[TMP15:%.*]] = icmp eq i64 [[INDEX_NEXT3]], 64
+; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <4 x i64> [[VEC_IND]], splat (i64 4)
+; CHECK-NEXT:    [[TMP16:%.*]] = or i1 [[TMP14]], [[TMP15]]
+; CHECK-NEXT:    br i1 [[TMP16]], label [[MIDDLE_SPLIT:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]]
+; CHECK:       middle.split:
+; CHECK-NEXT:    br i1 [[TMP14]], label [[VECTOR_EARLY_EXIT:%.*]], label [[MIDDLE_BLOCK:%.*]]
+; CHECK:       vector.early.exit:
+; CHECK-NEXT:    [[FIRST_ACTIVE_LANE:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP13]], i1 true)
+; CHECK-NEXT:    [[EARLY_EXIT_VALUE:%.*]] = extractelement <4 x i64> [[VEC_IND]], i64 [[FIRST_ACTIVE_LANE]]
+; CHECK-NEXT:    br label [[LOOP_END:%.*]]
+; CHECK:       middle.block:
+; CHECK-NEXT:    br i1 true, label [[LOOP_END]], label [[SCALAR_PH]]
+; CHECK:       scalar.ph:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 67, [[MIDDLE_BLOCK]] ], [ 3, [[ENTRY:%.*]] ]
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
-; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ [[INDEX_NEXT:%.*]], [[LOOP_INC:%.*]] ], [ 3, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ [[INDEX_NEXT:%.*]], [[LOOP_INC:%.*]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
 ; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i8, ptr [[P1]], i64 [[INDEX]]
 ; CHECK-NEXT:    [[LD1:%.*]] = load i8, ptr [[ARRAYIDX]], align 1
 ; CHECK-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds i8, ptr [[P2]], i64 [[INDEX]]
 ; CHECK-NEXT:    [[LD2:%.*]] = load i8, ptr [[ARRAYIDX1]], align 1
 ; CHECK-NEXT:    [[CMP3:%.*]] = icmp eq i8 [[LD1]], [[LD2]]
-; CHECK-NEXT:    br i1 [[CMP3]], label [[LOOP_INC]], label [[LOOP_END:%.*]]
+; CHECK-NEXT:    br i1 [[CMP3]], label [[LOOP_INC]], label [[LOOP_END]]
 ; CHECK:       loop.inc:
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 1
 ; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp ne i64 [[INDEX_NEXT]], 67
-; CHECK-NEXT:    br i1 [[EXITCOND]], label [[LOOP]], label [[LOOP_END]]
+; CHECK-NEXT:    br i1 [[EXITCOND]], label [[LOOP]], label [[LOOP_END]], !llvm.loop [[LOOP17:![0-9]+]]
 ; CHECK:       loop.end:
-; CHECK-NEXT:    [[RETVAL:%.*]] = phi i64 [ [[INDEX]], [[LOOP]] ], [ [[INDEX_NEXT]], [[LOOP_INC]] ]
+; CHECK-NEXT:    [[RETVAL:%.*]] = phi i64 [ [[INDEX]], [[LOOP]] ], [ [[INDEX_NEXT]], [[LOOP_INC]] ], [ [[TMP12]], [[MIDDLE_BLOCK]] ], [ [[EARLY_EXIT_VALUE]], [[VECTOR_EARLY_EXIT]] ]
 ; CHECK-NEXT:    ret i64 [[RETVAL]]
 ;
 entry:
@@ -418,21 +679,65 @@ define i64 @same_exit_block_post_inc_use2() {
 ; CHECK-NEXT:    [[P2:%.*]] = alloca [1024 x i8], align 1
 ; CHECK-NEXT:    call void @init_mem(ptr [[P1]], i64 1024)
 ; CHECK-NEXT:    call void @init_mem(ptr [[P2]], i64 1024)
+; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK:       vector.ph:
+; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK:       vector.body:
+; CHECK-NEXT:    [[INDEX1:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT3:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 3, i64 4, i64 5, i64 6>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[OFFSET_IDX:%.*]] = add i64 3, [[INDEX1]]
+; CHECK-NEXT:    [[TMP0:%.*]] = add i64 [[OFFSET_IDX]], 0
+; CHECK-NEXT:    [[TMP1:%.*]] = add i64 [[OFFSET_IDX]], 1
+; CHECK-NEXT:    [[TMP2:%.*]] = add i64 [[OFFSET_IDX]], 2
+; CHECK-NEXT:    [[TMP3:%.*]] = add i64 [[OFFSET_IDX]], 3
+; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i8, ptr [[P1]], i64 [[TMP0]]
+; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i8, ptr [[TMP4]], i32 0
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i8>, ptr [[TMP5]], align 1
+; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i8, ptr [[P2]], i64 [[TMP0]]
+; CHECK-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i8, ptr [[TMP6]], i32 0
+; CHECK-NEXT:    [[WIDE_LOAD2:%.*]] = load <4 x i8>, ptr [[TMP7]], align 1
+; CHECK-NEXT:    [[TMP8:%.*]] = add i64 [[TMP0]], 1
+; CHECK-NEXT:    [[TMP9:%.*]] = add i64 [[TMP1]], 1
+; CHECK-NEXT:    [[TMP10:%.*]] = add i64 [[TMP2]], 1
+; CHECK-NEXT:    [[TMP11:%.*]] = add i64 [[TMP3]], 1
+; CHECK-NEXT:    [[TMP12:%.*]] = insertelement <4 x i64> poison, i64 [[TMP8]], i32 0
+; CHECK-NEXT:    [[TMP13:%.*]] = insertelement <4 x i64> [[TMP12]], i64 [[TMP9]], i32 1
+; CHECK-NEXT:    [[TMP14:%.*]] = insertelement <4 x i64> [[TMP13]], i64 [[TMP10]], i32 2
+; CHECK-NEXT:    [[TMP15:%.*]] = insertelement <4 x i64> [[TMP14]], i64 [[TMP11]], i32 3
+; CHECK-NEXT:    [[TMP16:%.*]] = icmp eq <4 x i8> [[WIDE_LOAD]], [[WIDE_LOAD2]]
+; CHECK-NEXT:    [[INDEX_NEXT3]] = add nuw i64 [[INDEX1]], 4
+; CHECK-NEXT:    [[TMP17:%.*]] = xor <4 x i1> [[TMP16]], splat (i1 true)
+; CHECK-NEXT:    [[TMP18:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP17]])
+; CHECK-NEXT:    [[TMP19:%.*]] = icmp eq i64 [[INDEX_NEXT3]], 64
+; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <4 x i64> [[VEC_IND]], splat (i64 4)
+; CHECK-NEXT:    [[TMP20:%.*]] = or i1 [[TMP18]], [[TMP19]]
+; CHECK-NEXT:    br i1 [[TMP20]], label [[MIDDLE_SPLIT:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP18:![0-9]+]]
+; CHECK:       middle.split:
+; CHECK-NEXT:    [[TMP21:%.*]] = extractelement <4 x i64> [[VEC_IND]], i32 3
+; CHECK-NEXT:    br i1 [[TMP18]], label [[VECTOR_EARLY_EXIT:%.*]], label [[MIDDLE_BLOCK:%.*]]
+; CHECK:       vector.early.exit:
+; CHECK-NEXT:    [[FIRST_ACTIVE_LANE:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP17]], i1 true)
+; CHECK-NEXT:    [[EARLY_EXIT_VALUE:%.*]] = extractelement <4 x i64> [[TMP15]], i64 [[FIRST_ACTIVE_LANE]]
+; CHECK-NEXT:    br label [[LOOP_END:%.*]]
+; CHECK:       middle.block:
+; CHECK-NEXT:    br i1 true, label [[LOOP_END]], label [[SCALAR_PH]]
+; CHECK:       scalar.ph:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 67, [[MIDDLE_BLOCK]] ], [ 3, [[ENTRY:%.*]] ]
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
-; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ [[INDEX_NEXT:%.*]], [[LOOP_INC:%.*]] ], [ 3, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ [[INDEX_NEXT:%.*]], [[LOOP_INC:%.*]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
 ; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i8, ptr [[P1]], i64 [[INDEX]]
 ; CHECK-NEXT:    [[LD1:%.*]] = load i8, ptr [[ARRAYIDX]], align 1
 ; CHECK-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds i8, ptr [[P2]], i64 [[INDEX]]
 ; CHECK-NEXT:    [[LD2:%.*]] = load i8, ptr [[ARRAYIDX1]], align 1
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 1
 ; CHECK-NEXT:    [[CMP3:%.*]] = icmp eq i8 [[LD1]], [[LD2]]
-; CHECK-NEXT:    br i1 [[CMP3]], label [[LOOP_INC]], label [[LOOP_END:%.*]]
+; CHECK-NEXT:    br i1 [[CMP3]], label [[LOOP_INC]], label [[LOOP_END]]
 ; CHECK:       loop.inc:
 ; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp ne i64 [[INDEX_NEXT]], 67
-; CHECK-NEXT:    br i1 [[EXITCOND]], label [[LOOP]], label [[LOOP_END]]
+; CHECK-NEXT:    br i1 [[EXITCOND]], label [[LOOP]], label [[LOOP_END]], !llvm.loop [[LOOP19:![0-9]+]]
 ; CHECK:       loop.end:
-; CHECK-NEXT:    [[RETVAL:%.*]] = phi i64 [ [[INDEX_NEXT]], [[LOOP]] ], [ [[INDEX]], [[LOOP_INC]] ]
+; CHECK-NEXT:    [[RETVAL:%.*]] = phi i64 [ [[INDEX_NEXT]], [[LOOP]] ], [ [[INDEX]], [[LOOP_INC]] ], [ [[TMP21]], [[MIDDLE_BLOCK]] ], [ [[EARLY_EXIT_VALUE]], [[VECTOR_EARLY_EXIT]] ]
 ; CHECK-NEXT:    ret i64 [[RETVAL]]
 ;
 entry:
@@ -469,24 +774,56 @@ define i64 @diff_exit_block_pre_inc_use1() {
 ; CHECK-NEXT:    [[P2:%.*]] = alloca [1024 x i8], align 1
 ; CHECK-NEXT:    call void @init_mem(ptr [[P1]], i64 1024)
 ; CHECK-NEXT:    call void @init_mem(ptr [[P2]], i64 1024)
+; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK:       vector.ph:
+; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK:       vector.body:
+; CHECK-NEXT:    [[INDEX1:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT3:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 3, i64 4, i64 5, i64 6>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[OFFSET_IDX:%.*]] = add i64 3, [[INDEX1]]
+; CHECK-NEXT:    [[TMP0:%.*]] = add i64 [[OFFSET_IDX]], 0
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i8, ptr [[P1]], i64 [[TMP0]]
+; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i32 0
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i8>, ptr [[TMP2]], align 1
+; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i8, ptr [[P2]], i64 [[TMP0]]
+; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i8, ptr [[TMP3]], i32 0
+; CHECK-NEXT:    [[WIDE_LOAD2:%.*]] = load <4 x i8>, ptr [[TMP4]], align 1
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq <4 x i8> [[WIDE_LOAD]], [[WIDE_LOAD2]]
+; CHECK-NEXT:    [[INDEX_NEXT3]] = add nuw i64 [[INDEX1]], 4
+; CHECK-NEXT:    [[TMP6:%.*]] = xor <4 x i1> [[TMP5]], splat (i1 true)
+; CHECK-NEXT:    [[TMP7:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP6]])
+; CHECK-NEXT:    [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT3]], 64
+; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <4 x i64> [[VEC_IND]], splat (i64 4)
+; CHECK-NEXT:    [[TMP9:%.*]] = or i1 [[TMP7]], [[TMP8]]
+; CHECK-NEXT:    br i1 [[TMP9]], label [[MIDDLE_SPLIT:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP20:![0-9]+]]
+; CHECK:       middle.split:
+; CHECK-NEXT:    br i1 [[TMP7]], label [[VECTOR_EARLY_EXIT:%.*]], label [[MIDDLE_BLOCK:%.*]]
+; CHECK:       vector.early.exit:
+; CHECK-NEXT:    [[FIRST_ACTIVE_LANE:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP6]], i1 true)
+; CHECK-NEXT:    [[EARLY_EXIT_VALUE:%.*]] = extractelement <4 x i64> [[VEC_IND]], i64 [[FIRST_ACTIVE_LANE]]
+; CHECK-NEXT:    br label [[LOOP_EARLY_EXIT:%.*]]
+; CHECK:       middle.block:
+; CHECK-NEXT:    br i1 true, label [[LOOP_END:%.*]], label [[SCALAR_PH]]
+; CHECK:       scalar.ph:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 67, [[MIDDLE_BLOCK]] ], [ 3, [[ENTRY:%.*]] ]
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
-; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ [[INDEX_NEXT:%.*]], [[LOOP_INC:%.*]] ], [ 3, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ [[INDEX_NEXT:%.*]], [[LOOP_INC:%.*]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
 ; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i8, ptr [[P1]], i64 [[INDEX]]
 ; CHECK-NEXT:    [[LD1:%.*]] = load i8, ptr [[ARRAYIDX]], align 1
 ; CHECK-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds i8, ptr [[P2]], i64 [[INDEX]]
 ; CHECK-NEXT:    [[LD2:%.*]] = load i8, ptr [[ARRAYIDX1]], align 1
 ; CHECK-NEXT:    [[CMP3:%.*]] = icmp eq i8 [[LD1]], [[LD2]]
-; CHECK-NEXT:    br i1 [[CMP3]], label [[LOOP_INC]], label [[LOOP_EARLY_EXIT:%.*]]
+; CHECK-NEXT:    br i1 [[CMP3]], label [[LOOP_INC]], label [[LOOP_EARLY_EXIT]]
 ; CHECK:       loop.inc:
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 1
 ; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp ne i64 [[INDEX_NEXT]], 67
-; CHECK-NEXT:    br i1 [[EXITCOND]], label [[LOOP]], label [[LOOP_END:%.*]]
+; CHECK-NEXT:    br i1 [[EXITCOND]], label [[LOOP]], label [[LOOP_END]], !llvm.loop [[LOOP21:![0-9]+]]
 ; CHECK:       loop.early.exit:
-; CHECK-NEXT:    [[RETVAL1:%.*]] = phi i64 [ [[INDEX]], [[LOOP]] ]
+; CHECK-NEXT:    [[RETVAL1:%.*]] = phi i64 [ [[INDEX]], [[LOOP]] ], [ [[EARLY_EXIT_VALUE]], [[VECTOR_EARLY_EXIT]] ]
 ; CHECK-NEXT:    ret i64 [[RETVAL1]]
 ; CHECK:       loop.end:
-; CHECK-NEXT:    [[RETVAL2:%.*]] = phi i64 [ 67, [[LOOP_INC]] ]
+; CHECK-NEXT:    [[RETVAL2:%.*]] = phi i64 [ 67, [[LOOP_INC]] ], [ 67, [[MIDDLE_BLOCK]] ]
 ; CHECK-NEXT:    ret i64 [[RETVAL2]]
 ;
 entry:
@@ -529,10 +866,10 @@ define i64 @diff_exit_block_pre_inc_use2() {
 ; CHECK-NEXT:    call void @init_mem(ptr [[P2]], i64 1024)
 ; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.ph:
-; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
-; CHECK-NEXT:    [[INDEX1:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT3:%.*]], [[LOOP]] ]
-; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 3, i64 4, i64 5, i64 6>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[INDEX1:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT3:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 3, i64 4, i64 5, i64 6>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[OFFSET_IDX:%.*]] = add i64 3, [[INDEX1]]
 ; CHECK-NEXT:    [[TMP0:%.*]] = add i64 [[OFFSET_IDX]], 0
 ; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i8, ptr [[P1]], i64 [[TMP0]]
@@ -548,15 +885,17 @@ define i64 @diff_exit_block_pre_inc_use2() {
 ; CHECK-NEXT:    [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT3]], 64
 ; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <4 x i64> [[VEC_IND]], splat (i64 4)
 ; CHECK-NEXT:    [[TMP9:%.*]] = or i1 [[TMP7]], [[TMP8]]
-; CHECK-NEXT:    br i1 [[TMP9]], label [[MIDDLE_SPLIT:%.*]], label [[LOOP]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK-NEXT:    br i1 [[TMP9]], label [[MIDDLE_SPLIT:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP22:![0-9]+]]
 ; CHECK:       middle.split:
-; CHECK-NEXT:    br i1 [[TMP7]], label [[LOOP_EARLY_EXIT:%.*]], label [[MIDDLE_BLOCK:%.*]]
+; CHECK-NEXT:    br i1 [[TMP7]], label [[VECTOR_EARLY_EXIT:%.*]], label [[MIDDLE_BLOCK:%.*]]
+; CHECK:       vector.early.exit:
+; CHECK-NEXT:    br label [[LOOP_EARLY_EXIT:%.*]]
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    [[TMP10:%.*]] = extractelement <4 x i64> [[VEC_IND]], i32 3
 ; CHECK-NEXT:    br i1 true, label [[LOOP_END:%.*]], label [[SCALAR_PH]]
 ; CHECK:       scalar.ph:
 ; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 67, [[MIDDLE_BLOCK]] ], [ 3, [[ENTRY:%.*]] ]
-; CHECK-NEXT:    br label [[LOOP1:%.*]]
+; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ [[INDEX_NEXT:%.*]], [[LOOP_INC:%.*]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
 ; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i8, ptr [[P1]], i64 [[INDEX]]
@@ -568,9 +907,9 @@ define i64 @diff_exit_block_pre_inc_use2() {
 ; CHECK:       loop.inc:
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 1
 ; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp ne i64 [[INDEX_NEXT]], 67
-; CHECK-NEXT:    br i1 [[EXITCOND]], label [[LOOP1]], label [[LOOP_END]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK-NEXT:    br i1 [[EXITCOND]], label [[LOOP]], label [[LOOP_END]], !llvm.loop [[LOOP23:![0-9]+]]
 ; CHECK:       loop.early.exit:
-; CHECK-NEXT:    [[RETVAL1:%.*]] = phi i64 [ 67, [[LOOP1]] ], [ 67, [[MIDDLE_SPLIT]] ]
+; CHECK-NEXT:    [[RETVAL1:%.*]] = phi i64 [ 67, [[LOOP]] ], [ 67, [[VECTOR_EARLY_EXIT]] ]
 ; CHECK-NEXT:    ret i64 [[RETVAL1]]
 ; CHECK:       loop.end:
 ; CHECK-NEXT:    [[RETVAL2:%.*]] = phi i64 [ [[INDEX]], [[LOOP_INC]] ], [ [[TMP10]], [[MIDDLE_BLOCK]] ]
@@ -614,24 +953,57 @@ define i64 @diff_exit_block_pre_inc_use3() {
 ; CHECK-NEXT:    [[P2:%.*]] = alloca [1024 x i8], align 1
 ; CHECK-NEXT:    call void @init_mem(ptr [[P1]], i64 1024)
 ; CHECK-NEXT:    call void @init_mem(ptr [[P2]], i64 1024)
+; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK:       vector.ph:
+; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK:       vector.body:
+; CHECK-NEXT:    [[INDEX2:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT4:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 3, i64 4, i64 5, i64 6>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[OFFSET_IDX:%.*]] = add i64 3, [[INDEX2]]
+; CHECK-NEXT:    [[TMP0:%.*]] = add i64 [[OFFSET_IDX]], 0
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i8, ptr [[P1]], i64 [[TMP0]]
+; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i32 0
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i8>, ptr [[TMP2]], align 1
+; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i8, ptr [[P2]], i64 [[TMP0]]
+; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i8, ptr [[TMP3]], i32 0
+; CHECK-NEXT:    [[WIDE_LOAD3:%.*]] = load <4 x i8>, ptr [[TMP4]], align 1
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq <4 x i8> [[WIDE_LOAD]], [[WIDE_LOAD3]]
+; CHECK-NEXT:    [[INDEX_NEXT4]] = add nuw i64 [[INDEX2]], 4
+; CHECK-NEXT:    [[TMP6:%.*]] = xor <4 x i1> [[TMP5]], splat (i1 true)
+; CHECK-NEXT:    [[TMP7:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP6]])
+; CHECK-NEXT:    [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT4]], 64
+; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <4 x i64> [[VEC_IND]], splat (i64 4)
+; CHECK-NEXT:    [[TMP9:%.*]] = or i1 [[TMP7]], [[TMP8]]
+; CHECK-NEXT:    br i1 [[TMP9]], label [[MIDDLE_SPLIT:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP24:![0-9]+]]
+; CHECK:       middle.split:
+; CHECK-NEXT:    br i1 [[TMP7]], label [[VECTOR_EARLY_EXIT:%.*]], label [[MIDDLE_BLOCK:%.*]]
+; CHECK:       vector.early.exit:
+; CHECK-NEXT:    [[FIRST_ACTIVE_LANE:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP6]], i1 true)
+; CHECK-NEXT:    [[EARLY_EXIT_VALUE:%.*]] = extractelement <4 x i64> [[VEC_IND]], i64 [[FIRST_ACTIVE_LANE]]
+; CHECK-NEXT:    br label [[LOOP_EARLY_EXIT:%.*]]
+; CHECK:       middle.block:
+; CHECK-NEXT:    [[TMP10:%.*]] = extractelement <4 x i64> [[VEC_IND]], i32 3
+; CHECK-NEXT:    br i1 true, label [[LOOP_END:%.*]], label [[SCALAR_PH]]
+; CHECK:       scalar.ph:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 67, [[MIDDLE_BLOCK]] ], [ 3, [[ENTRY:%.*]] ]
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
-; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ [[INDEX_NEXT:%.*]], [[LOOP_INC:%.*]] ], [ 3, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ [[INDEX_NEXT:%.*]], [[LOOP_INC:%.*]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
 ; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i8, ptr [[P1]], i64 [[INDEX]]
 ; CHECK-NEXT:    [[LD1:%.*]] = load i8, ptr [[ARRAYIDX]], align 1
 ; CHECK-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds i8, ptr [[P2]], i64 [[INDEX]]
 ; CHECK-NEXT:    [[LD2:%.*]] = load i8, ptr [[ARRAYIDX1]], align 1
 ; CHECK-NEXT:    [[CMP3:%.*]] = icmp eq i8 [[LD1]], [[LD2]]
-; CHECK-NEXT:    br i1 [[CMP3]], label [[LOOP_INC]], label [[LOOP_EARLY_EXIT:%.*]]
+; CHECK-NEXT:    br i1 [[CMP3]], label [[LOOP_INC]], label [[LOOP_EARLY_EXIT]]
 ; CHECK:       loop.inc:
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 1
 ; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp ne i64 [[INDEX_NEXT]], 67
-; CHECK-NEXT:    br i1 [[EXITCOND]], label [[LOOP]], label [[LOOP_END:%.*]]
+; CHECK-NEXT:    br i1 [[EXITCOND]], label [[LOOP]], label [[LOOP_END]], !llvm.loop [[LOOP25:![0-9]+]]
 ; CHECK:       loop.early.exit:
-; CHECK-NEXT:    [[INDEX_LCSSA:%.*]] = phi i64 [ [[INDEX]], [[LOOP]] ]
+; CHECK-NEXT:    [[INDEX_LCSSA:%.*]] = phi i64 [ [[INDEX]], [[LOOP]] ], [ [[EARLY_EXIT_VALUE]], [[VECTOR_EARLY_EXIT]] ]
 ; CHECK-NEXT:    ret i64 [[INDEX_LCSSA]]
 ; CHECK:       loop.end:
-; CHECK-NEXT:    [[INDEX_LCSSA1:%.*]] = phi i64 [ [[INDEX]], [[LOOP_INC]] ]
+; CHECK-NEXT:    [[INDEX_LCSSA1:%.*]] = phi i64 [ [[INDEX]], [[LOOP_INC]] ], [ [[TMP10]], [[MIDDLE_BLOCK]] ]
 ; CHECK-NEXT:    ret i64 [[INDEX_LCSSA1]]
 ;
 entry:
@@ -670,24 +1042,63 @@ define i64 @diff_exit_block_post_inc_use1() {
 ; CHECK-NEXT:    [[P2:%.*]] = alloca [1024 x i8], align 1
 ; CHECK-NEXT:    call void @init_mem(ptr [[P1]], i64 1024)
 ; CHECK-NEXT:    call void @init_mem(ptr [[P2]], i64 1024)
+; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK:       vector.ph:
+; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK:       vector.body:
+; CHECK-NEXT:    [[INDEX1:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT3:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 3, i64 4, i64 5, i64 6>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[OFFSET_IDX:%.*]] = add i64 3, [[INDEX1]]
+; CHECK-NEXT:    [[TMP0:%.*]] = add i64 [[OFFSET_IDX]], 0
+; CHECK-NEXT:    [[TMP1:%.*]] = add i64 [[OFFSET_IDX]], 1
+; CHECK-NEXT:    [[TMP2:%.*]] = add i64 [[OFFSET_IDX]], 2
+; CHECK-NEXT:    [[TMP3:%.*]] = add i64 [[OFFSET_IDX]], 3
+; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i8, ptr [[P1]], i64 [[TMP0]]
+; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i8, ptr [[TMP4]], i32 0
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i8>, ptr [[TMP5]], align 1
+; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i8, ptr [[P2]], i64 [[TMP0]]
+; CHECK-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i8, ptr [[TMP6]], i32 0
+; CHECK-NEXT:    [[WIDE_LOAD2:%.*]] = load <4 x i8>, ptr [[TMP7]], align 1
+; CHECK-NEXT:    [[TMP8:%.*]] = icmp eq <4 x i8> [[WIDE_LOAD]], [[WIDE_LOAD2]]
+; CHECK-NEXT:    [[TMP9:%.*]] = add i64 [[TMP0]], 1
+; CHECK-NEXT:    [[TMP10:%.*]] = add i64 [[TMP1]], 1
+; CHECK-NEXT:    [[TMP11:%.*]] = add i64 [[TMP2]], 1
+; CHECK-NEXT:    [[TMP12:%.*]] = add i64 [[TMP3]], 1
+; CHECK-NEXT:    [[INDEX_NEXT3]] = add nuw i64 [[INDEX1]], 4
+; CHECK-NEXT:    [[TMP13:%.*]] = xor <4 x i1> [[TMP8]], splat (i1 true)
+; CHECK-NEXT:    [[TMP14:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP13]])
+; CHECK-NEXT:    [[TMP15:%.*]] = icmp eq i64 [[INDEX_NEXT3]], 64
+; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <4 x i64> [[VEC_IND]], splat (i64 4)
+; CHECK-NEXT:    [[TMP16:%.*]] = or i1 [[TMP14]], [[TMP15]]
+; CHECK-NEXT:    br i1 [[TMP16]], label [[MIDDLE_SPLIT:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP26:![0-9]+]]
+; CHECK:       middle.split:
+; CHECK-NEXT:    br i1 [[TMP14]], label [[VECTOR_EARLY_EXIT:%.*]], label [[MIDDLE_BLOCK:%.*]]
+; CHECK:       vector.early.exit:
+; CHECK-NEXT:    [[FIRST_ACTIVE_LANE:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP13]], i1 true)
+; CHECK-NEXT:    [[EARLY_EXIT_VALUE:%.*]] = extractelement <4 x i64> [[VEC_IND]], i64 [[FIRST_ACTIVE_LANE]]
+; CHECK-NEXT:    br label [[LOOP_EARLY_EXIT:%.*]]
+; CHECK:       middle.block:
+; CHECK-NEXT:    br i1 true, label [[LOOP_END:%.*]], label [[SCALAR_PH]]
+; CHECK:       scalar.ph:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 67, [[MIDDLE_BLOCK]] ], [ 3, [[ENTRY:%.*]] ]
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
-; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ [[INDEX_NEXT:%.*]], [[LOOP_INC:%.*]] ], [ 3, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ [[INDEX_NEXT:%.*]], [[LOOP_INC:%.*]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
 ; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i8, ptr [[P1]], i64 [[INDEX]]
 ; CHECK-NEXT:    [[LD1:%.*]] = load i8, ptr [[ARRAYIDX]], align 1
 ; CHECK-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds i8, ptr [[P2]], i64 [[INDEX]]
 ; CHECK-NEXT:    [[LD2:%.*]] = load i8, ptr [[ARRAYIDX1]], align 1
 ; CHECK-NEXT:    [[CMP3:%.*]] = icmp eq i8 [[LD1]], [[LD2]]
-; CHECK-NEXT:    br i1 [[CMP3]], label [[LOOP_INC]], label [[LOOP_EARLY_EXIT:%.*]]
+; CHECK-NEXT:    br i1 [[CMP3]], label [[LOOP_INC]], label [[LOOP_EARLY_EXIT]]
 ; CHECK:       loop.inc:
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 1
 ; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp ne i64 [[INDEX_NEXT]], 67
-; CHECK-NEXT:    br i1 [[EXITCOND]], label [[LOOP]], label [[LOOP_END:%.*]]
+; CHECK-NEXT:    br i1 [[EXITCOND]], label [[LOOP]], label [[LOOP_END]], !llvm.loop [[LOOP27:![0-9]+]]
 ; CHECK:       loop.early.exit:
-; CHECK-NEXT:    [[RETVAL1:%.*]] = phi i64 [ [[INDEX]], [[LOOP]] ]
+; CHECK-NEXT:    [[RETVAL1:%.*]] = phi i64 [ [[INDEX]], [[LOOP]] ], [ [[EARLY_EXIT_VALUE]], [[VECTOR_EARLY_EXIT]] ]
 ; CHECK-NEXT:    ret i64 [[RETVAL1]]
 ; CHECK:       loop.end:
-; CHECK-NEXT:    [[RETVAL2:%.*]] = phi i64 [ [[INDEX_NEXT]], [[LOOP_INC]] ]
+; CHECK-NEXT:    [[RETVAL2:%.*]] = phi i64 [ [[INDEX_NEXT]], [[LOOP_INC]] ], [ [[TMP12]], [[MIDDLE_BLOCK]] ]
 ; CHECK-NEXT:    ret i64 [[RETVAL2]]
 ;
 entry:
@@ -728,24 +1139,68 @@ define i64 @diff_exit_block_post_inc_use2() {
 ; CHECK-NEXT:    [[P2:%.*]] = alloca [1024 x i8], align 1
 ; CHECK-NEXT:    call void @init_mem(ptr [[P1]], i64 1024)
 ; CHECK-NEXT:    call void @init_mem(ptr [[P2]], i64 1024)
+; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK:       vector.ph:
+; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK:       vector.body:
+; CHECK-NEXT:    [[INDEX1:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT3:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 3, i64 4, i64 5, i64 6>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[OFFSET_IDX:%.*]] = add i64 3, [[INDEX1]]
+; CHECK-NEXT:    [[TMP0:%.*]] = add i64 [[OFFSET_IDX]], 0
+; CHECK-NEXT:    [[TMP1:%.*]] = add i64 [[OFFSET_IDX]], 1
+; CHECK-NEXT:    [[TMP2:%.*]] = add i64 [[OFFSET_IDX]], 2
+; CHECK-NEXT:    [[TMP3:%.*]] = add i64 [[OFFSET_IDX]], 3
+; CHECK-NEXT:    [[TMP4:%.*]] = add i64 [[TMP0]], 1
+; CHECK-NEXT:    [[TMP5:%.*]] = add i64 [[TMP1]], 1
+; CHECK-NEXT:    [[TMP6:%.*]] = add i64 [[TMP2]], 1
+; CHECK-NEXT:    [[TMP7:%.*]] = add i64 [[TMP3]], 1
+; CHECK-NEXT:    [[TMP8:%.*]] = insertelement <4 x i64> poison, i64 [[TMP4]], i32 0
+; CHECK-NEXT:    [[TMP9:%.*]] = insertelement <4 x i64> [[TMP8]], i64 [[TMP5]], i32 1
+; CHECK-NEXT:    [[TMP10:%.*]] = insertelement <4 x i64> [[TMP9]], i64 [[TMP6]], i32 2
+; CHECK-NEXT:    [[TMP11:%.*]] = insertelement <4 x i64> [[TMP10]], i64 [[TMP7]], i32 3
+; CHECK-NEXT:    [[TMP12:%.*]] = getelementptr inbounds i8, ptr [[P1]], i64 [[TMP0]]
+; CHECK-NEXT:    [[TMP13:%.*]] = getelementptr inbounds i8, ptr [[TMP12]], i32 0
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i8>, ptr [[TMP13]], align 1
+; CHECK-NEXT:    [[TMP14:%.*]] = getelementptr inbounds i8, ptr [[P2]], i64 [[TMP0]]
+; CHECK-NEXT:    [[TMP15:%.*]] = getelementptr inbounds i8, ptr [[TMP14]], i32 0
+; CHECK-NEXT:    [[WIDE_LOAD2:%.*]] = load <4 x i8>, ptr [[TMP15]], align 1
+; CHECK-NEXT:    [[TMP16:%.*]] = icmp eq <4 x i8> [[WIDE_LOAD]], [[WIDE_LOAD2]]
+; CHECK-NEXT:    [[INDEX_NEXT3]] = add nuw i64 [[INDEX1]], 4
+; CHECK-NEXT:    [[TMP17:%.*]] = xor <4 x i1> [[TMP16]], splat (i1 true)
+; CHECK-NEXT:    [[TMP18:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP17]])
+; CHECK-NEXT:    [[TMP19:%.*]] = icmp eq i64 [[INDEX_NEXT3]], 64
+; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <4 x i64> [[VEC_IND]], splat (i64 4)
+; CHECK-NEXT:    [[TMP20:%.*]] = or i1 [[TMP18]], [[TMP19]]
+; CHECK-NEXT:    br i1 [[TMP20]], label [[MIDDLE_SPLIT:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP28:![0-9]+]]
+; CHECK:       middle.split:
+; CHECK-NEXT:    br i1 [[TMP18]], label [[VECTOR_EARLY_EXIT:%.*]], label [[MIDDLE_BLOCK:%.*]]
+; CHECK:       vector.early.exit:
+; CHECK-NEXT:    [[FIRST_ACTIVE_LANE:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP17]], i1 true)
+; CHECK-NEXT:    [[EARLY_EXIT_VALUE:%.*]] = extractelement <4 x i64> [[TMP11]], i64 [[FIRST_ACTIVE_LANE]]
+; CHECK-NEXT:    br label [[LOOP_EARLY_EXIT:%.*]]
+; CHECK:       middle.block:
+; CHECK-NEXT:    [[TMP21:%.*]] = extractelement <4 x i64> [[VEC_IND]], i32 3
+; CHECK-NEXT:    br i1 true, label [[LOOP_END:%.*]], label [[SCALAR_PH]]
+; CHECK:       scalar.ph:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 67, [[MIDDLE_BLOCK]] ], [ 3, [[ENTRY:%.*]] ]
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
-; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ [[INDEX_NEXT:%.*]], [[LOOP_INC:%.*]] ], [ 3, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ [[INDEX_NEXT:%.*]], [[LOOP_INC:%.*]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 1
 ; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i8, ptr [[P1]], i64 [[INDEX]]
 ; CHECK-NEXT:    [[LD1:%.*]] = load i8, ptr [[ARRAYIDX]], align 1
 ; CHECK-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds i8, ptr [[P2]], i64 [[INDEX]]
 ; CHECK-NEXT:    [[LD2:%.*]] = load i8, ptr [[ARRAYIDX1]], align 1
 ; CHECK-NEXT:    [[CMP3:%.*]] = icmp eq i8 [[LD1]], [[LD2]]
-; CHECK-NEXT:    br i1 [[CMP3]], label [[LOOP_INC]], label [[LOOP_EARLY_EXIT:%.*]]
+; CHECK-NEXT:    br i1 [[CMP3]], label [[LOOP_INC]], label [[LOOP_EARLY_EXIT]]
 ; CHECK:       loop.inc:
 ; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp ne i64 [[INDEX_NEXT]], 67
-; CHECK-NEXT:    br i1 [[EXITCOND]], label [[LOOP]], label [[LOOP_END:%.*]]
+; CHECK-NEXT:    br i1 [[EXITCOND]], label [[LOOP]], label [[LOOP_END]], !llvm.loop [[LOOP29:![0-9]+]]
 ; CHECK:       loop.early.exit:
-; CHECK-NEXT:    [[RETVAL1:%.*]] = phi i64 [ [[INDEX_NEXT]], [[LOOP]] ]
+; CHECK-NEXT:    [[RETVAL1:%.*]] = phi i64 [ [[INDEX_NEXT]], [[LOOP]] ], [ [[EARLY_EXIT_VALUE]], [[VECTOR_EARLY_EXIT]] ]
 ; CHECK-NEXT:    ret i64 [[RETVAL1]]
 ; CHECK:       loop.end:
-; CHECK-NEXT:    [[RETVAL2:%.*]] = phi i64 [ [[INDEX]], [[LOOP_INC]] ]
+; CHECK-NEXT:    [[RETVAL2:%.*]] = phi i64 [ [[INDEX]], [[LOOP_INC]] ], [ [[TMP21]], [[MIDDLE_BLOCK]] ]
 ; CHECK-NEXT:    ret i64 [[RETVAL2]]
 ;
 entry:
@@ -786,20 +1241,50 @@ define i64 @loop_contains_safe_call() {
 ; CHECK-NEXT:    [[P2:%.*]] = alloca [1024 x i8], align 1
 ; CHECK-NEXT:    call void @init_mem(ptr [[P1]], i64 1024)
 ; CHECK-NEXT:    call void @init_mem(ptr [[P2]], i64 1024)
+; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK:       vector.ph:
+; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK:       vector.body:
+; CHECK-NEXT:    [[INDEX1:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT2:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 3, i64 4, i64 5, i64 6>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[OFFSET_IDX:%.*]] = add i64 3, [[INDEX1]]
+; CHECK-NEXT:    [[TMP0:%.*]] = add i64 [[OFFSET_IDX]], 0
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds float, ptr [[P1]], i64 [[TMP0]]
+; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds float, ptr [[TMP1]], i32 0
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x float>, ptr [[TMP2]], align 1
+; CHECK-NEXT:    [[TMP3:%.*]] = call fast <4 x float> @llvm.sqrt.v4f32(<4 x float> [[WIDE_LOAD]])
+; CHECK-NEXT:    [[TMP4:%.*]] = fcmp fast ult <4 x float> [[TMP3]], splat (float 3.000000e+00)
+; CHECK-NEXT:    [[INDEX_NEXT2]] = add nuw i64 [[INDEX1]], 4
+; CHECK-NEXT:    [[TMP5:%.*]] = xor <4 x i1> [[TMP4]], splat (i1 true)
+; CHECK-NEXT:    [[TMP6:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP5]])
+; CHECK-NEXT:    [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT2]], 64
+; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <4 x i64> [[VEC_IND]], splat (i64 4)
+; CHECK-NEXT:    [[TMP8:%.*]] = or i1 [[TMP6]], [[TMP7]]
+; CHECK-NEXT:    br i1 [[TMP8]], label [[MIDDLE_SPLIT:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP30:![0-9]+]]
+; CHECK:       middle.split:
+; CHECK-NEXT:    br i1 [[TMP6]], label [[VECTOR_EARLY_EXIT:%.*]], label [[MIDDLE_BLOCK:%.*]]
+; CHECK:       vector.early.exit:
+; CHECK-NEXT:    [[FIRST_ACTIVE_LANE:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP5]], i1 true)
+; CHECK-NEXT:    [[EARLY_EXIT_VALUE:%.*]] = extractelement <4 x i64> [[VEC_IND]], i64 [[FIRST_ACTIVE_LANE]]
+; CHECK-NEXT:    br label [[LOOP_END:%.*]]
+; CHECK:       middle.block:
+; CHECK-NEXT:    br i1 true, label [[LOOP_END]], label [[SCALAR_PH]]
+; CHECK:       scalar.ph:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 67, [[MIDDLE_BLOCK]] ], [ 3, [[ENTRY:%.*]] ]
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
-; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ [[INDEX_NEXT:%.*]], [[LOOP_INC:%.*]] ], [ 3, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ [[INDEX_NEXT:%.*]], [[LOOP_INC:%.*]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
 ; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds float, ptr [[P1]], i64 [[INDEX]]
 ; CHECK-NEXT:    [[LD1:%.*]] = load float, ptr [[ARRAYIDX]], align 1
 ; CHECK-NEXT:    [[SQRT:%.*]] = tail call fast float @llvm.sqrt.f32(float [[LD1]])
 ; CHECK-NEXT:    [[CMP:%.*]] = fcmp fast ult float [[SQRT]], 3.000000e+00
-; CHECK-NEXT:    br i1 [[CMP]], label [[LOOP_INC]], label [[LOOP_END:%.*]]
+; CHECK-NEXT:    br i1 [[CMP]], label [[LOOP_INC]], label [[LOOP_END]]
 ; CHECK:       loop.inc:
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 1
 ; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp ne i64 [[INDEX_NEXT]], 67
-; CHECK-NEXT:    br i1 [[EXITCOND]], label [[LOOP]], label [[LOOP_END]]
+; CHECK-NEXT:    br i1 [[EXITCOND]], label [[LOOP]], label [[LOOP_END]], !llvm.loop [[LOOP31:![0-9]+]]
 ; CHECK:       loop.end:
-; CHECK-NEXT:    [[RETVAL:%.*]] = phi i64 [ [[INDEX]], [[LOOP]] ], [ 67, [[LOOP_INC]] ]
+; CHECK-NEXT:    [[RETVAL:%.*]] = phi i64 [ [[INDEX]], [[LOOP]] ], [ 67, [[LOOP_INC]] ], [ 67, [[MIDDLE_BLOCK]] ], [ [[EARLY_EXIT_VALUE]], [[VECTOR_EARLY_EXIT]] ]
 ; CHECK-NEXT:    ret i64 [[RETVAL]]
 ;
 entry:
@@ -835,20 +1320,50 @@ define i64 @loop_contains_safe_div() {
 ; CHECK-NEXT:    [[P2:%.*]] = alloca [1024 x i8], align 1
 ; CHECK-NEXT:    call void @init_mem(ptr [[P1]], i64 1024)
 ; CHECK-NEXT:    call void @init_mem(ptr [[P2]], i64 1024)
-; CHECK-NEXT:    br label [[LOOP1:%.*]]
+; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK:       vector.ph:
+; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK:       vector.body:
+; CHECK-NEXT:    [[INDEX1:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT2:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 3, i64 4, i64 5, i64 6>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[OFFSET_IDX:%.*]] = add i64 3, [[INDEX1]]
+; CHECK-NEXT:    [[TMP0:%.*]] = add i64 [[OFFSET_IDX]], 0
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[P1]], i64 [[TMP0]]
+; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i32 0
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP2]], align 1
+; CHECK-NEXT:    [[TMP3:%.*]] = udiv <4 x i32> [[WIDE_LOAD]], splat (i32 20000)
+; CHECK-NEXT:    [[TMP4:%.*]] = icmp eq <4 x i32> [[TMP3]], splat (i32 1)
+; CHECK-NEXT:    [[INDEX_NEXT2]] = add nuw i64 [[INDEX1]], 4
+; CHECK-NEXT:    [[TMP5:%.*]] = xor <4 x i1> [[TMP4]], splat (i1 true)
+; CHECK-NEXT:    [[TMP6:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP5]])
+; CHECK-NEXT:    [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT2]], 64
+; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <4 x i64> [[VEC_IND]], splat (i64 4)
+; CHECK-NEXT:    [[TMP8:%.*]] = or i1 [[TMP6]], [[TMP7]]
+; CHECK-NEXT:    br i1 [[TMP8]], label [[MIDDLE_SPLIT:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP32:![0-9]+]]
+; CHECK:       middle.split:
+; CHECK-NEXT:    br i1 [[TMP6]], label [[VECTOR_EARLY_EXIT:%.*]], label [[MIDDLE_BLOCK:%.*]]
+; CHECK:       vector.early.exit:
+; CHECK-NEXT:    [[FIRST_ACTIVE_LANE:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP5]], i1 true)
+; CHECK-NEXT:    [[EARLY_EXIT_VALUE:%.*]] = extractelement <4 x i64> [[VEC_IND]], i64 [[FIRST_ACTIVE_LANE]]
+; CHECK-NEXT:    br label [[LOOP_END:%.*]]
+; CHECK:       middle.block:
+; CHECK-NEXT:    br i1 true, label [[LOOP_END]], label [[SCALAR_PH]]
+; CHECK:       scalar.ph:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 67, [[MIDDLE_BLOCK]] ], [ 3, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
-; CHECK-NEXT:    [[INDEX2:%.*]] = phi i64 [ [[INDEX_NEXT1:%.*]], [[LOOP_INC1:%.*]] ], [ 3, [[ENTRY:%.*]] ]
-; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[P1]], i64 [[INDEX2]]
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ [[INDEX_NEXT:%.*]], [[LOOP_INC:%.*]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[P1]], i64 [[INDEX]]
 ; CHECK-NEXT:    [[LD1:%.*]] = load i32, ptr [[ARRAYIDX]], align 1
 ; CHECK-NEXT:    [[DIV:%.*]] = udiv i32 [[LD1]], 20000
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[DIV]], 1
-; CHECK-NEXT:    br i1 [[CMP]], label [[LOOP_INC1]], label [[LOOP_END:%.*]]
+; CHECK-NEXT:    br i1 [[CMP]], label [[LOOP_INC]], label [[LOOP_END]]
 ; CHECK:       loop.inc:
-; CHECK-NEXT:    [[INDEX_NEXT1]] = add i64 [[INDEX2]], 1
-; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp ne i64 [[INDEX_NEXT1]], 67
-; CHECK-NEXT:    br i1 [[EXITCOND]], label [[LOOP1]], label [[LOOP_END]]
+; CHECK-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 1
+; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp ne i64 [[INDEX_NEXT]], 67
+; CHECK-NEXT:    br i1 [[EXITCOND]], label [[LOOP]], label [[LOOP_END]], !llvm.loop [[LOOP33:![0-9]+]]
 ; CHECK:       loop.end:
-; CHECK-NEXT:    [[RETVAL:%.*]] = phi i64 [ [[INDEX2]], [[LOOP1]] ], [ 67, [[LOOP_INC1]] ]
+; CHECK-NEXT:    [[RETVAL:%.*]] = phi i64 [ [[INDEX]], [[LOOP]] ], [ 67, [[LOOP_INC]] ], [ 67, [[MIDDLE_BLOCK]] ], [ [[EARLY_EXIT_VALUE]], [[VECTOR_EARLY_EXIT]] ]
 ; CHECK-NEXT:    ret i64 [[RETVAL]]
 ;
 entry:
@@ -883,21 +1398,54 @@ define i64 @loop_contains_load_after_early_exit(ptr dereferenceable(1024) align(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[P1:%.*]] = alloca [1024 x i8], align 1
 ; CHECK-NEXT:    call void @init_mem(ptr [[P1]], i64 1024)
+; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK:       vector.ph:
+; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK:       vector.body:
+; CHECK-NEXT:    [[INDEX1:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT3:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 3, i64 4, i64 5, i64 6>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[OFFSET_IDX:%.*]] = add i64 3, [[INDEX1]]
+; CHECK-NEXT:    [[TMP0:%.*]] = add i64 [[OFFSET_IDX]], 0
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[P1]], i64 [[TMP0]]
+; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i32 0
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP2]], align 1
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp eq <4 x i32> [[WIDE_LOAD]], splat (i32 1)
+; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr i64, ptr [[P2]], i64 [[TMP0]]
+; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr i64, ptr [[TMP4]], i32 0
+; CHECK-NEXT:    [[WIDE_LOAD2:%.*]] = load <4 x i64>, ptr [[TMP5]], align 8
+; CHECK-NEXT:    [[INDEX_NEXT3]] = add nuw i64 [[INDEX1]], 4
+; CHECK-NEXT:    [[TMP6:%.*]] = xor <4 x i1> [[TMP3]], splat (i1 true)
+; CHECK-NEXT:    [[TMP7:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP6]])
+; CHECK-NEXT:    [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT3]], 64
+; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <4 x i64> [[VEC_IND]], splat (i64 4)
+; CHECK-NEXT:    [[TMP9:%.*]] = or i1 [[TMP7]], [[TMP8]]
+; CHECK-NEXT:    br i1 [[TMP9]], label [[MIDDLE_SPLIT:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP34:![0-9]+]]
+; CHECK:       middle.split:
+; CHECK-NEXT:    [[TMP10:%.*]] = extractelement <4 x i64> [[WIDE_LOAD2]], i32 3
+; CHECK-NEXT:    br i1 [[TMP7]], label [[VECTOR_EARLY_EXIT:%.*]], label [[MIDDLE_BLOCK:%.*]]
+; CHECK:       vector.early.exit:
+; CHECK-NEXT:    [[FIRST_ACTIVE_LANE:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP6]], i1 true)
+; CHECK-NEXT:    [[EARLY_EXIT_VALUE:%.*]] = extractelement <4 x i64> [[VEC_IND]], i64 [[FIRST_ACTIVE_LANE]]
+; CHECK-NEXT:    br label [[LOOP_END:%.*]]
+; CHECK:       middle.block:
+; CHECK-NEXT:    br i1 true, label [[LOOP_END]], label [[SCALAR_PH]]
+; CHECK:       scalar.ph:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 67, [[MIDDLE_BLOCK]] ], [ 3, [[ENTRY:%.*]] ]
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
-; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ [[INDEX_NEXT:%.*]], [[LOOP_INC:%.*]] ], [ 3, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ [[INDEX_NEXT:%.*]], [[LOOP_INC:%.*]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
 ; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[P1]], i64 [[INDEX]]
 ; CHECK-NEXT:    [[LD1:%.*]] = load i32, ptr [[ARRAYIDX]], align 1
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[LD1]], 1
-; CHECK-NEXT:    br i1 [[CMP]], label [[LOOP_INC]], label [[LOOP_END:%.*]]
+; CHECK-NEXT:    br i1 [[CMP]], label [[LOOP_INC]], label [[LOOP_END]]
 ; CHECK:       loop.inc:
 ; CHECK-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds i64, ptr [[P2]], i64 [[INDEX]]
 ; CHECK-NEXT:    [[LD2:%.*]] = load i64, ptr [[ARRAYIDX2]], align 8
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 1
 ; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp ne i64 [[INDEX_NEXT]], 67
-; CHECK-NEXT:    br i1 [[EXITCOND]], label [[LOOP]], label [[LOOP_END]]
+; CHECK-NEXT:    br i1 [[EXITCOND]], label [[LOOP]], label [[LOOP_END]], !llvm.loop [[LOOP35:![0-9]+]]
 ; CHECK:       loop.end:
-; CHECK-NEXT:    [[RETVAL:%.*]] = phi i64 [ [[INDEX]], [[LOOP]] ], [ [[LD2]], [[LOOP_INC]] ]
+; CHECK-NEXT:    [[RETVAL:%.*]] = phi i64 [ [[INDEX]], [[LOOP]] ], [ [[LD2]], [[LOOP_INC]] ], [ [[TMP10]], [[MIDDLE_BLOCK]] ], [ [[EARLY_EXIT_VALUE]], [[VECTOR_EARLY_EXIT]] ]
 ; CHECK-NEXT:    ret i64 [[RETVAL]]
 ;
 entry:
@@ -932,21 +1480,57 @@ define i64 @same_exit_block_pre_inc_use1_reverse() {
 ; CHECK-NEXT:    [[P2:%.*]] = alloca [1024 x i8], align 1
 ; CHECK-NEXT:    call void @init_mem(ptr [[P1]], i64 1024)
 ; CHECK-NEXT:    call void @init_mem(ptr [[P2]], i64 1024)
+; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK:       vector.ph:
+; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK:       vector.body:
+; CHECK-NEXT:    [[INDEX1:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT4:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 1023, i64 1022, i64 1021, i64 1020>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[OFFSET_IDX:%.*]] = sub i64 1023, [[INDEX1]]
+; CHECK-NEXT:    [[TMP0:%.*]] = add i64 [[OFFSET_IDX]], 0
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i8, ptr [[P1]], i64 [[TMP0]]
+; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i32 0
+; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i8, ptr [[TMP2]], i32 -3
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i8>, ptr [[TMP3]], align 1
+; CHECK-NEXT:    [[REVERSE:%.*]] = shufflevector <4 x i8> [[WIDE_LOAD]], <4 x i8> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i8, ptr [[P2]], i64 [[TMP0]]
+; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i8, ptr [[TMP4]], i32 0
+; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i8, ptr [[TMP5]], i32 -3
+; CHECK-NEXT:    [[WIDE_LOAD2:%.*]] = load <4 x i8>, ptr [[TMP6]], align 1
+; CHECK-NEXT:    [[REVERSE3:%.*]] = shufflevector <4 x i8> [[WIDE_LOAD2]], <4 x i8> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    [[TMP7:%.*]] = icmp eq <4 x i8> [[REVERSE]], [[REVERSE3]]
+; CHECK-NEXT:    [[INDEX_NEXT4]] = add nuw i64 [[INDEX1]], 4
+; CHECK-NEXT:    [[TMP8:%.*]] = xor <4 x i1> [[TMP7]], splat (i1 true)
+; CHECK-NEXT:    [[TMP9:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP8]])
+; CHECK-NEXT:    [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT4]], 1020
+; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <4 x i64> [[VEC_IND]], splat (i64 -4)
+; CHECK-NEXT:    [[TMP11:%.*]] = or i1 [[TMP9]], [[TMP10]]
+; CHECK-NEXT:    br i1 [[TMP11]], label [[MIDDLE_SPLIT:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP36:![0-9]+]]
+; CHECK:       middle.split:
+; CHECK-NEXT:    br i1 [[TMP9]], label [[VECTOR_EARLY_EXIT:%.*]], label [[MIDDLE_BLOCK:%.*]]
+; CHECK:       vector.early.exit:
+; CHECK-NEXT:    [[FIRST_ACTIVE_LANE:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP8]], i1 true)
+; CHECK-NEXT:    [[EARLY_EXIT_VALUE:%.*]] = extractelement <4 x i64> [[VEC_IND]], i64 [[FIRST_ACTIVE_LANE]]
+; CHECK-NEXT:    br label [[LOOP_END:%.*]]
+; CHECK:       middle.block:
+; CHECK-NEXT:    br i1 false, label [[LOOP_END]], label [[SCALAR_PH]]
+; CHECK:       scalar.ph:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 3, [[MIDDLE_BLOCK]] ], [ 1023, [[ENTRY:%.*]] ]
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
-; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ [[INDEX_NEXT:%.*]], [[LOOP_INC:%.*]] ], [ 1023, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ [[INDEX_NEXT:%.*]], [[LOOP_INC:%.*]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
 ; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i8, ptr [[P1]], i64 [[INDEX]]
 ; CHECK-NEXT:    [[LD1:%.*]] = load i8, ptr [[ARRAYIDX]], align 1
 ; CHECK-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds i8, ptr [[P2]], i64 [[INDEX]]
 ; CHECK-NEXT:    [[LD2:%.*]] = load i8, ptr [[ARRAYIDX1]], align 1
 ; CHECK-NEXT:    [[CMP3:%.*]] = icmp eq i8 [[LD1]], [[LD2]]
-; CHECK-NEXT:    br i1 [[CMP3]], label [[LOOP_INC]], label [[LOOP_END:%.*]]
+; CHECK-NEXT:    br i1 [[CMP3]], label [[LOOP_INC]], label [[LOOP_END]]
 ; CHECK:       loop.inc:
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], -1
 ; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i64 [[INDEX_NEXT]], 0
-; CHECK-NEXT:    br i1 [[EXITCOND]], label [[LOOP_END]], label [[LOOP]]
+; CHECK-NEXT:    br i1 [[EXITCOND]], label [[LOOP_END]], label [[LOOP]], !llvm.loop [[LOOP37:![0-9]+]]
 ; CHECK:       loop.end:
-; CHECK-NEXT:    [[RETVAL:%.*]] = phi i64 [ [[INDEX]], [[LOOP]] ], [ 1024, [[LOOP_INC]] ]
+; CHECK-NEXT:    [[RETVAL:%.*]] = phi i64 [ [[INDEX]], [[LOOP]] ], [ 1024, [[LOOP_INC]] ], [ 1024, [[MIDDLE_BLOCK]] ], [ [[EARLY_EXIT_VALUE]], [[VECTOR_EARLY_EXIT]] ]
 ; CHECK-NEXT:    ret i64 [[RETVAL]]
 ;
 entry:
@@ -980,21 +1564,53 @@ define i64 @same_exit_block_pre_inc_use1_deref_ptrs(ptr dereferenceable(1024) %p
 ; CHECK-LABEL: define i64 @same_exit_block_pre_inc_use1_deref_ptrs(
 ; CHECK-SAME: ptr dereferenceable(1024) [[P1:%.*]], ptr dereferenceable(1024) [[P2:%.*]]) {
 ; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK:       vector.ph:
+; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK:       vector.body:
+; CHECK-NEXT:    [[INDEX1:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT3:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 3, i64 4, i64 5, i64 6>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[OFFSET_IDX:%.*]] = add i64 3, [[INDEX1]]
+; CHECK-NEXT:    [[TMP0:%.*]] = add i64 [[OFFSET_IDX]], 0
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i8, ptr [[P1]], i64 [[TMP0]]
+; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i32 0
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i8>, ptr [[TMP2]], align 1
+; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i8, ptr [[P2]], i64 [[TMP0]]
+; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i8, ptr [[TMP3]], i32 0
+; CHECK-NEXT:    [[WIDE_LOAD2:%.*]] = load <4 x i8>, ptr [[TMP4]], align 1
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq <4 x i8> [[WIDE_LOAD]], [[WIDE_LOAD2]]
+; CHECK-NEXT:    [[INDEX_NEXT3]] = add nuw i64 [[INDEX1]], 4
+; CHECK-NEXT:    [[TMP6:%.*]] = xor <4 x i1> [[TMP5]], splat (i1 true)
+; CHECK-NEXT:    [[TMP7:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP6]])
+; CHECK-NEXT:    [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT3]], 64
+; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <4 x i64> [[VEC_IND]], splat (i64 4)
+; CHECK-NEXT:    [[TMP9:%.*]] = or i1 [[TMP7]], [[TMP8]]
+; CHECK-NEXT:    br i1 [[TMP9]], label [[MIDDLE_SPLIT:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP38:![0-9]+]]
+; CHECK:       middle.split:
+; CHECK-NEXT:    br i1 [[TMP7]], label [[VECTOR_EARLY_EXIT:%.*]], label [[MIDDLE_BLOCK:%.*]]
+; CHECK:       vector.early.exit:
+; CHECK-NEXT:    [[FIRST_ACTIVE_LANE:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP6]], i1 true)
+; CHECK-NEXT:    [[EARLY_EXIT_VALUE:%.*]] = extractelement <4 x i64> [[VEC_IND]], i64 [[FIRST_ACTIVE_LANE]]
+; CHECK-NEXT:    br label [[LOOP_END:%.*]]
+; CHECK:       middle.block:
+; CHECK-NEXT:    br i1 true, label [[LOOP_END]], label [[SCALAR_PH]]
+; CHECK:       scalar.ph:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 67, [[MIDDLE_BLOCK]] ], [ 3, [[ENTRY:%.*]] ]
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
-; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ [[INDEX_NEXT:%.*]], [[LOOP_INC:%.*]] ], [ 3, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ [[INDEX_NEXT:%.*]], [[LOOP_INC:%.*]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
 ; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i8, ptr [[P1]], i64 [[INDEX]]
 ; CHECK-NEXT:    [[LD1:%.*]] = load i8, ptr [[ARRAYIDX]], align 1
 ; CHECK-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds i8, ptr [[P2]], i64 [[INDEX]]
 ; CHECK-NEXT:    [[LD2:%.*]] = load i8, ptr [[ARRAYIDX1]], align 1
 ; CHECK-NEXT:    [[CMP3:%.*]] = icmp eq i8 [[LD1]], [[LD2]]
-; CHECK-NEXT:    br i1 [[CMP3]], label [[LOOP_INC]], label [[LOOP_END:%.*]]
+; CHECK-NEXT:    br i1 [[CMP3]], label [[LOOP_INC]], label [[LOOP_END]]
 ; CHECK:       loop.inc:
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 1
 ; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp ne i64 [[INDEX_NEXT]], 67
-; CHECK-NEXT:    br i1 [[EXITCOND]], label [[LOOP]], label [[LOOP_END]]
+; CHECK-NEXT:    br i1 [[EXITCOND]], label [[LOOP]], label [[LOOP_END]], !llvm.loop [[LOOP39:![0-9]+]]
 ; CHECK:       loop.end:
-; CHECK-NEXT:    [[RETVAL:%.*]] = phi i64 [ [[INDEX]], [[LOOP]] ], [ 67, [[LOOP_INC]] ]
+; CHECK-NEXT:    [[RETVAL:%.*]] = phi i64 [ [[INDEX]], [[LOOP]] ], [ 67, [[LOOP_INC]] ], [ 67, [[MIDDLE_BLOCK]] ], [ [[EARLY_EXIT_VALUE]], [[VECTOR_EARLY_EXIT]] ]
 ; CHECK-NEXT:    ret i64 [[RETVAL]]
 ;
 entry:
@@ -1029,4 +1645,40 @@ attributes #0 = { "vector-function-abi-variant"="_ZGVsNxv_foo(foo_vec)" }
 ; CHECK: [[META1]] = !{!"llvm.loop.isvectorized", i32 1}
 ; CHECK: [[META2]] = !{!"llvm.loop.unroll.runtime.disable"}
 ; CHECK: [[LOOP3]] = distinct !{[[LOOP3]], [[META2]], [[META1]]}
+; CHECK: [[LOOP4]] = distinct !{[[LOOP4]], [[META1]], [[META2]]}
+; CHECK: [[LOOP5]] = distinct !{[[LOOP5]], [[META2]], [[META1]]}
+; CHECK: [[LOOP6]] = distinct !{[[LOOP6]], [[META1]], [[META2]]}
+; CHECK: [[LOOP7]] = distinct !{[[LOOP7]], [[META2]], [[META1]]}
+; CHECK: [[LOOP8]] = distinct !{[[LOOP8]], [[META1]], [[META2]]}
+; CHECK: [[LOOP9]] = distinct !{[[LOOP9]], [[META2]], [[META1]]}
+; CHECK: [[LOOP10]] = distinct !{[[LOOP10]], [[META1]], [[META2]]}
+; CHECK: [[LOOP11]] = distinct !{[[LOOP11]], [[META2]], [[META1]]}
+; CHECK: [[LOOP12]] = distinct !{[[LOOP12]], [[META1]], [[META2]]}
+; CHECK: [[LOOP13]] = distinct !{[[LOOP13]], [[META2]], [[META1]]}
+; CHECK: [[LOOP14]] = distinct !{[[LOOP14]], [[META1]], [[META2]]}
+; CHECK: [[LOOP15]] = distinct !{[[LOOP15]], [[META2]], [[META1]]}
+; CHECK: [[LOOP16]] = distinct !{[[LOOP16]], [[META1]], [[META2]]}
+; CHECK: [[LOOP17]] = distinct !{[[LOOP17]], [[META2]], [[META1]]}
+; CHECK: [[LOOP18]] = distinct !{[[LOOP18]], [[META1]], [[META2]]}
+; CHECK: [[LOOP19]] = distinct !{[[LOOP19]], [[META2]], [[META1]]}
+; CHECK: [[LOOP20]] = distinct !{[[LOOP20]], [[META1]], [[META2]]}
+; CHECK: [[LOOP21]] = distinct !{[[LOOP21]], [[META2]], [[META1]]}
+; CHECK: [[LOOP22]] = distinct !{[[LOOP22]], [[META1]], [[META2]]}
+; CHECK: [[LOOP23]] = distinct !{[[LOOP23]], [[META2]], [[META1]]}
+; CHECK: [[LOOP24]] = distinct !{[[LOOP24]], [[META1]], [[META2]]}
+; CHECK: [[LOOP25]] = distinct !{[[LOOP25]], [[META2]], [[META1]]}
+; CHECK: [[LOOP26]] = distinct !{[[LOOP26]], [[META1]], [[META2]]}
+; CHECK: [[LOOP27]] = distinct !{[[LOOP27]], [[META2]], [[META1]]}
+; CHECK: [[LOOP28]] = distinct !{[[LOOP28]], [[META1]], [[META2]]}
+; CHECK: [[LOOP29]] = distinct !{[[LOOP29]], [[META2]], [[META1]]}
+; CHECK: [[LOOP30]] = distinct !{[[LOOP30]], [[META1]], [[META2]]}
+; CHECK: [[LOOP31]] = distinct !{[[LOOP31]], [[META2]], [[META1]]}
+; CHECK: [[LOOP32]] = distinct !{[[LOOP32]], [[META1]], [[META2]]}
+; CHECK: [[LOOP33]] = distinct !{[[LOOP33]], [[META2]], [[META1]]}
+; CHECK: [[LOOP34]] = distinct !{[[LOOP34]], [[META1]], [[META2]]}
+; CHECK: [[LOOP35]] = distinct !{[[LOOP35]], [[META2]], [[META1]]}
+; CHECK: [[LOOP36]] = distinct !{[[LOOP36]], [[META1]], [[META2]]}
+; CHECK: [[LOOP37]] = distinct !{[[LOOP37]], [[META2]], [[META1]]}
+; CHECK: [[LOOP38]] = distinct !{[[LOOP38]], [[META1]], [[META2]]}
+; CHECK: [[LOOP39]] = distinct !{[[LOOP39]], [[META2]], [[META1]]}
 ;.
diff --git a/llvm/test/Transforms/LoopVectorize/skeleton-lcssa-crash.ll b/llvm/test/Transforms/LoopVectorize/skeleton-lcssa-crash.ll
index 75472d29d6ed0..d9827a8c71ee6 100644
--- a/llvm/test/Transforms/LoopVectorize/skeleton-lcssa-crash.ll
+++ b/llvm/test/Transforms/LoopVectorize/skeleton-lcssa-crash.ll
@@ -59,7 +59,7 @@ define i16 @test(ptr %arg, i64 %N) {
 ; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]]
 ; CHECK-NEXT:    br i1 [[CMP_N]], label [[EXIT_LOOPEXIT:%.*]], label [[SCALAR_PH]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[VECTOR_MEMCHECK]] ], [ 0, [[LOOP_3_PREHEADER]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[LOOP_3_PREHEADER]] ], [ 0, [[VECTOR_MEMCHECK]] ]
 ; CHECK-NEXT:    br label [[LOOP_3:%.*]]
 ; CHECK:       inner.latch:
 ; CHECK-NEXT:    [[C_4:%.*]] = call i1 @cond()
diff --git a/llvm/test/Transforms/LoopVectorize/strict-fadd-interleave-only.ll b/llvm/test/Transforms/LoopVectorize/strict-fadd-interleave-only.ll
index 9e7c667f9c8ad..760bdbf227fff 100644
--- a/llvm/test/Transforms/LoopVectorize/strict-fadd-interleave-only.ll
+++ b/llvm/test/Transforms/LoopVectorize/strict-fadd-interleave-only.ll
@@ -217,3 +217,94 @@ exit:
   %.lcssa = phi float [ %rdx.next, %loop ]
   ret float %.lcssa
 }
+
+define float @fadd_reduction_with_live_in(float %inc) {
+; CHECK-LABEL: define float @fadd_reduction_with_live_in(
+; CHECK-SAME: float [[INC:%.*]]) {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK:       vector.ph:
+; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK:       vector.body:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi float [ 0.000000e+00, [[VECTOR_PH]] ], [ [[TMP5:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_IV:%.*]] = add i32 [[INDEX]], 0
+; CHECK-NEXT:    [[VEC_IV1:%.*]] = add i32 [[INDEX]], 1
+; CHECK-NEXT:    [[TMP0:%.*]] = icmp ule i32 [[VEC_IV]], 1000
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ule i32 [[VEC_IV1]], 1000
+; CHECK-NEXT:    [[TMP2:%.*]] = select i1 [[TMP0]], float [[INC]], float -0.000000e+00
+; CHECK-NEXT:    [[TMP3:%.*]] = fadd float [[VEC_PHI]], [[TMP2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = select i1 [[TMP1]], float [[INC]], float -0.000000e+00
+; CHECK-NEXT:    [[TMP5]] = fadd float [[TMP3]], [[TMP4]]
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2
+; CHECK-NEXT:    [[TMP6:%.*]] = icmp eq i32 [[INDEX_NEXT]], 1002
+; CHECK-NEXT:    br i1 [[TMP6]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK:       middle.block:
+; CHECK-NEXT:    br i1 true, label [[EXIT:%.*]], label [[SCALAR_PH]]
+; CHECK:       scalar.ph:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ 1002, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP5]], [[MIDDLE_BLOCK]] ], [ 0.000000e+00, [[ENTRY]] ]
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[SUM:%.*]] = phi float [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[SUM_NEXT:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[SUM_NEXT]] = fadd float [[SUM]], [[INC]]
+; CHECK-NEXT:    [[IV_NEXT]] = add i32 [[IV]], 1
+; CHECK-NEXT:    [[EC:%.*]] = icmp eq i32 [[IV]], 1000
+; CHECK-NEXT:    br i1 [[EC]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP7:![0-9]+]]
+; CHECK:       exit:
+; CHECK-NEXT:    [[LCSSA:%.*]] = phi float [ [[SUM_NEXT]], [[LOOP]] ], [ [[TMP5]], [[MIDDLE_BLOCK]] ]
+; CHECK-NEXT:    ret float [[LCSSA]]
+;
+; CHECK-ALM-LABEL: define float @fadd_reduction_with_live_in(
+; CHECK-ALM-SAME: float [[INC:%.*]]) {
+; CHECK-ALM-NEXT:  entry:
+; CHECK-ALM-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK-ALM:       vector.ph:
+; CHECK-ALM-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK-ALM:       vector.body:
+; CHECK-ALM-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-ALM-NEXT:    [[VEC_PHI:%.*]] = phi float [ 0.000000e+00, [[VECTOR_PH]] ], [ [[TMP5:%.*]], [[VECTOR_BODY]] ]
+; CHECK-ALM-NEXT:    [[TMP0:%.*]] = add i32 [[INDEX]], 0
+; CHECK-ALM-NEXT:    [[TMP1:%.*]] = add i32 [[INDEX]], 1
+; CHECK-ALM-NEXT:    [[ACTIVE_LANE_MASK:%.*]] = icmp ult i32 [[TMP0]], 1001
+; CHECK-ALM-NEXT:    [[ACTIVE_LANE_MASK1:%.*]] = icmp ult i32 [[TMP1]], 1001
+; CHECK-ALM-NEXT:    [[TMP2:%.*]] = select i1 [[ACTIVE_LANE_MASK]], float [[INC]], float -0.000000e+00
+; CHECK-ALM-NEXT:    [[TMP3:%.*]] = fadd float [[VEC_PHI]], [[TMP2]]
+; CHECK-ALM-NEXT:    [[TMP4:%.*]] = select i1 [[ACTIVE_LANE_MASK1]], float [[INC]], float -0.000000e+00
+; CHECK-ALM-NEXT:    [[TMP5]] = fadd float [[TMP3]], [[TMP4]]
+; CHECK-ALM-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2
+; CHECK-ALM-NEXT:    [[TMP6:%.*]] = icmp eq i32 [[INDEX_NEXT]], 1002
+; CHECK-ALM-NEXT:    br i1 [[TMP6]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK-ALM:       middle.block:
+; CHECK-ALM-NEXT:    br i1 true, label [[EXIT:%.*]], label [[SCALAR_PH]]
+; CHECK-ALM:       scalar.ph:
+; CHECK-ALM-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ 1002, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-ALM-NEXT:    [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP5]], [[MIDDLE_BLOCK]] ], [ 0.000000e+00, [[ENTRY]] ]
+; CHECK-ALM-NEXT:    br label [[LOOP:%.*]]
+; CHECK-ALM:       loop:
+; CHECK-ALM-NEXT:    [[IV:%.*]] = phi i32 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
+; CHECK-ALM-NEXT:    [[SUM:%.*]] = phi float [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[SUM_NEXT:%.*]], [[LOOP]] ]
+; CHECK-ALM-NEXT:    [[SUM_NEXT]] = fadd float [[SUM]], [[INC]]
+; CHECK-ALM-NEXT:    [[IV_NEXT]] = add i32 [[IV]], 1
+; CHECK-ALM-NEXT:    [[EC:%.*]] = icmp eq i32 [[IV]], 1000
+; CHECK-ALM-NEXT:    br i1 [[EC]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP7:![0-9]+]]
+; CHECK-ALM:       exit:
+; CHECK-ALM-NEXT:    [[LCSSA:%.*]] = phi float [ [[SUM_NEXT]], [[LOOP]] ], [ [[TMP5]], [[MIDDLE_BLOCK]] ]
+; CHECK-ALM-NEXT:    ret float [[LCSSA]]
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop ]
+  %sum = phi float [ 0.000000e+00, %entry ], [ %sum.next, %loop ]
+  %sum.next = fadd float %sum, %inc
+  %iv.next = add i32 %iv, 1
+  %ec = icmp eq i32 %iv, 1000
+  br i1 %ec, label %exit, label %loop
+
+exit:
+  %lcssa = phi float [ %sum.next, %loop ]
+  ret float %lcssa
+}
diff --git a/llvm/test/Transforms/LoopVectorize/uncountable-early-exit-vplan.ll b/llvm/test/Transforms/LoopVectorize/uncountable-early-exit-vplan.ll
index b427b43cdb133..0520b3c77b5b6 100644
--- a/llvm/test/Transforms/LoopVectorize/uncountable-early-exit-vplan.ll
+++ b/llvm/test/Transforms/LoopVectorize/uncountable-early-exit-vplan.ll
@@ -39,7 +39,7 @@ define i64 @multi_exiting_to_different_exits_live_in_exit_values() {
 ; CHECK-EMPTY:
 ; CHECK-NEXT: middle.split:
 ; CHECK-NEXT:   EMIT branch-on-cond vp<[[EA_TAKEN]]>
-; CHECK-NEXT: Successor(s): ir-bb<e1>, middle.block
+; CHECK-NEXT: Successor(s): vector.early.exit, middle.block
 ; CHECK-EMPTY:
 ; CHECK-NEXT: middle.block:
 ; CHECK-NEXT:   EMIT vp<[[MIDDLE_CMP:%.+]]> = icmp eq ir<128>, vp<[[VTC]]>
@@ -58,8 +58,11 @@ define i64 @multi_exiting_to_different_exits_live_in_exit_values() {
 ; CHECK-NEXT:  IR %p2 = phi i64 [ 1, %loop.latch ] (extra operand: ir<1> from middle.block)
 ; CHECK-NEXT: No successors
 ; CHECK-EMPTY:
+; CHECK-NEXT: vector.early.exit:
+; CHECK-NEXT: Successor(s): ir-bb<e1>
+; CHECK-EMPTY:
 ; CHECK-NEXT: ir-bb<e1>:
-; CHECK-NEXT:   IR %p1 = phi i64 [ 0, %loop.header ] (extra operand: ir<0> from middle.split)
+; CHECK-NEXT:   IR %p1 = phi i64 [ 0, %loop.header ] (extra operand: ir<0> from vector.early.exit)
 ; CHECK-NEXT: No successors
 ; CHECK-NEXT: }
 entry:
@@ -122,7 +125,7 @@ define i64 @multi_exiting_to_same_exit_live_in_exit_values() {
 ; CHECK-EMPTY:
 ; CHECK-NEXT: middle.split:
 ; CHECK-NEXT:   EMIT branch-on-cond vp<[[EA_TAKEN]]>
-; CHECK-NEXT: Successor(s): ir-bb<exit>, middle.block
+; CHECK-NEXT: Successor(s): vector.early.exit, middle.block
 ; CHECK-EMPTY:
 ; CHECK-NEXT: middle.block:
 ; CHECK-NEXT:   EMIT vp<[[MIDDLE_CMP:%.+]]> = icmp eq ir<128>, vp<[[VTC]]>
@@ -137,8 +140,11 @@ define i64 @multi_exiting_to_same_exit_live_in_exit_values() {
 ; CHECK-NEXT:   IR   %iv = phi i64 [ %inc, %loop.latch ], [ 0, %entry ] (extra operand: vp<[[RESUME]]> from scalar.ph)
 ; CHECK:      No successors
 ; CHECK-EMPTY:
+; CHECK-NEXT: vector.early.exit:
+; CHECK-NEXT: Successor(s): ir-bb<exit>
+; CHECK-EMPTY:
 ; CHECK-NEXT: ir-bb<exit>:
-; CHECK-NEXT:   IR %p = phi i64 [ 0, %loop.header ], [ 1, %loop.latch ] (extra operands: ir<1> from middle.block, ir<0> from middle.split)
+; CHECK-NEXT:   IR %p = phi i64 [ 0, %loop.header ], [ 1, %loop.latch ] (extra operands: ir<1> from middle.block, ir<0> from vector.early.exit)
 ; CHECK-NEXT: No successors
 ; CHECK-NEXT: }
 
@@ -198,7 +204,7 @@ define i64 @multi_exiting_to_same_exit_live_in_exit_values_2() {
 ; CHECK-EMPTY:
 ; CHECK-NEXT: middle.split:
 ; CHECK-NEXT:   EMIT branch-on-cond vp<[[EA_TAKEN]]>
-; CHECK-NEXT: Successor(s): ir-bb<exit>, middle.block
+; CHECK-NEXT: Successor(s): vector.early.exit, middle.block
 ; CHECK-EMPTY:
 ; CHECK-NEXT: middle.block:
 ; CHECK-NEXT:   EMIT vp<[[MIDDLE_CMP:%.+]]> = icmp eq ir<128>, vp<[[VTC]]>
@@ -213,8 +219,11 @@ define i64 @multi_exiting_to_same_exit_live_in_exit_values_2() {
 ; CHECK-NEXT:   IR   %iv = phi i64 [ %inc, %loop.latch ], [ 0, %entry ] (extra operand: vp<[[RESUME]]> from scalar.ph)
 ; CHECK:      No successors
 ; CHECK-EMPTY:
+; CHECK-NEXT: vector.early.exit:
+; CHECK-NEXT: Successor(s): ir-bb<exit>
+; CHECK-EMPTY:
 ; CHECK-NEXT: ir-bb<exit>:
-; CHECK-NEXT:   IR %p = phi i64 [ 0, %loop.header ], [ 1, %loop.latch ] (extra operands: ir<1> from middle.block, ir<0> from middle.split)
+; CHECK-NEXT:   IR %p = phi i64 [ 0, %loop.header ], [ 1, %loop.latch ] (extra operands: ir<1> from middle.block, ir<0> from vector.early.exit)
 ; CHECK-NEXT: No successors
 ; CHECK-NEXT: }
 
diff --git a/llvm/test/Transforms/LoopVectorize/uncountable-single-exit-loops.ll b/llvm/test/Transforms/LoopVectorize/uncountable-single-exit-loops.ll
index c3958262f0af8..de08e333192df 100644
--- a/llvm/test/Transforms/LoopVectorize/uncountable-single-exit-loops.ll
+++ b/llvm/test/Transforms/LoopVectorize/uncountable-single-exit-loops.ll
@@ -1,5 +1,5 @@
 ; REQUIRES: asserts
-; RUN: opt -p loop-vectorize -debug-only=loop-vectorize -S %s 2>&1 | FileCheck %s
+; RUN: opt -p loop-vectorize -debug-only=loop-vectorize --disable-output -S %s 2>&1 | FileCheck %s
 
 
 ; CHECK-LABEL: LV: Checking a loop in 'latch_exit_cannot_compute_btc_due_to_step'
diff --git a/llvm/test/Transforms/LoopVectorize/vect.stats.ll b/llvm/test/Transforms/LoopVectorize/vect.stats.ll
index 05aa13b22b6f4..9a55dc99c316b 100644
--- a/llvm/test/Transforms/LoopVectorize/vect.stats.ll
+++ b/llvm/test/Transforms/LoopVectorize/vect.stats.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -passes=loop-vectorize -force-vector-interleave=4 -force-vector-width=4 -debug-only=loop-vectorize -stats -S 2>&1 | FileCheck %s
+; RUN: opt < %s -passes=loop-vectorize -force-vector-interleave=4 -force-vector-width=4 -debug-only=loop-vectorize --disable-output -stats -S 2>&1 | FileCheck %s
 ; REQUIRES: asserts
 
 ;
diff --git a/llvm/test/Transforms/LoopVectorize/version-stride-with-integer-casts.ll b/llvm/test/Transforms/LoopVectorize/version-stride-with-integer-casts.ll
index 791c995d88c14..16e071dec9604 100644
--- a/llvm/test/Transforms/LoopVectorize/version-stride-with-integer-casts.ll
+++ b/llvm/test/Transforms/LoopVectorize/version-stride-with-integer-casts.ll
@@ -40,8 +40,8 @@ define void @test_versioned_with_sext_use(i32 %offset, ptr %dst) {
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    br i1 false, label [[OUTER_HEADER_LOOPEXIT]], label [[SCALAR_PH]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ [[IV_1]], [[VECTOR_SCEVCHECK]] ], [ [[IV_1]], [[INNER_LOOP_PREHEADER]] ]
-; CHECK-NEXT:    [[BC_RESUME_VAL1:%.*]] = phi i32 [ 200, [[MIDDLE_BLOCK]] ], [ 0, [[VECTOR_SCEVCHECK]] ], [ 0, [[INNER_LOOP_PREHEADER]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ [[IV_1]], [[INNER_LOOP_PREHEADER]] ], [ [[IV_1]], [[VECTOR_SCEVCHECK]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL1:%.*]] = phi i32 [ 200, [[MIDDLE_BLOCK]] ], [ 0, [[INNER_LOOP_PREHEADER]] ], [ 0, [[VECTOR_SCEVCHECK]] ]
 ; CHECK-NEXT:    br label [[INNER_LOOP]]
 ; CHECK:       inner.loop:
 ; CHECK-NEXT:    [[IV_2:%.*]] = phi i64 [ [[IV_2_NEXT]], [[INNER_LOOP]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
@@ -115,8 +115,8 @@ define void @test_versioned_with_zext_use(i32 %offset, ptr %dst) {
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    br i1 false, label [[OUTER_HEADER_LOOPEXIT]], label [[SCALAR_PH]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ [[IV_1]], [[VECTOR_SCEVCHECK]] ], [ [[IV_1]], [[INNER_LOOP_PREHEADER]] ]
-; CHECK-NEXT:    [[BC_RESUME_VAL1:%.*]] = phi i32 [ 200, [[MIDDLE_BLOCK]] ], [ 0, [[VECTOR_SCEVCHECK]] ], [ 0, [[INNER_LOOP_PREHEADER]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ [[IV_1]], [[INNER_LOOP_PREHEADER]] ], [ [[IV_1]], [[VECTOR_SCEVCHECK]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL1:%.*]] = phi i32 [ 200, [[MIDDLE_BLOCK]] ], [ 0, [[INNER_LOOP_PREHEADER]] ], [ 0, [[VECTOR_SCEVCHECK]] ]
 ; CHECK-NEXT:    br label [[INNER_LOOP]]
 ; CHECK:       inner.loop:
 ; CHECK-NEXT:    [[IV_2:%.*]] = phi i64 [ [[IV_2_NEXT]], [[INNER_LOOP]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
@@ -188,7 +188,7 @@ define void @versioned_sext_use_in_gep(i32 %scale, ptr %dst, i64 %scale.2) {
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    br i1 true, label [[EXIT:%.*]], label [[SCALAR_PH]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 256, [[MIDDLE_BLOCK]] ], [ 0, [[VECTOR_SCEVCHECK]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 256, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ], [ 0, [[VECTOR_SCEVCHECK]] ]
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
 ; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
@@ -274,8 +274,8 @@ define void @test_versioned_with_different_uses(i32 %offset, ptr noalias %dst.1,
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    br i1 false, label [[OUTER_HEADER_LOOPEXIT]], label [[SCALAR_PH]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ [[IV_1]], [[VECTOR_SCEVCHECK]] ], [ [[IV_1]], [[INNER_LOOP_PREHEADER]] ]
-; CHECK-NEXT:    [[BC_RESUME_VAL1:%.*]] = phi i32 [ 200, [[MIDDLE_BLOCK]] ], [ 0, [[VECTOR_SCEVCHECK]] ], [ 0, [[INNER_LOOP_PREHEADER]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ [[IV_1]], [[INNER_LOOP_PREHEADER]] ], [ [[IV_1]], [[VECTOR_SCEVCHECK]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL1:%.*]] = phi i32 [ 200, [[MIDDLE_BLOCK]] ], [ 0, [[INNER_LOOP_PREHEADER]] ], [ 0, [[VECTOR_SCEVCHECK]] ]
 ; CHECK-NEXT:    br label [[INNER_LOOP]]
 ; CHECK:       inner.loop:
 ; CHECK-NEXT:    [[IV_2:%.*]] = phi i64 [ [[IV_2_NEXT]], [[INNER_LOOP]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
@@ -371,8 +371,8 @@ define void @test_versioned_with_non_ex_use(i32 %offset, ptr noalias %dst.1, ptr
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    br i1 false, label [[EXIT:%.*]], label [[SCALAR_PH]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 200, [[MIDDLE_BLOCK]] ], [ 0, [[VECTOR_SCEVCHECK]] ], [ 0, [[ENTRY:%.*]] ]
-; CHECK-NEXT:    [[BC_RESUME_VAL1:%.*]] = phi i32 [ 200, [[MIDDLE_BLOCK]] ], [ 0, [[VECTOR_SCEVCHECK]] ], [ 0, [[ENTRY]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 200, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ], [ 0, [[VECTOR_SCEVCHECK]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL1:%.*]] = phi i32 [ 200, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ], [ 0, [[VECTOR_SCEVCHECK]] ]
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
 ; CHECK-NEXT:    [[IV_2:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_2_NEXT:%.*]], [[LOOP]] ]
@@ -448,7 +448,7 @@ define void @zext_of_i1_stride(i1 %g, ptr %dst) mustprogress {
 ; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[TMP1]], [[N_VEC]]
 ; CHECK-NEXT:    br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ 0, [[VECTOR_SCEVCHECK]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ], [ 0, [[VECTOR_SCEVCHECK]] ]
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
 ; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
@@ -510,7 +510,7 @@ define void @sext_of_i1_stride(i1 %g, ptr %dst) mustprogress {
 ; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[TMP2]], [[N_VEC]]
 ; CHECK-NEXT:    br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ 0, [[VECTOR_SCEVCHECK]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ], [ 0, [[VECTOR_SCEVCHECK]] ]
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
 ; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
diff --git a/llvm/test/Transforms/LoopVectorize/vplan-printing-outer-loop.ll b/llvm/test/Transforms/LoopVectorize/vplan-printing-outer-loop.ll
index 2adeb5920cb5b..52b2bcd9aac11 100644
--- a/llvm/test/Transforms/LoopVectorize/vplan-printing-outer-loop.ll
+++ b/llvm/test/Transforms/LoopVectorize/vplan-printing-outer-loop.ll
@@ -35,12 +35,14 @@ define void @foo(i64 %n) {
 ; CHECK-NEXT:       EMIT branch-on-cond ir<%inner.ec>
 ; CHECK-NEXT:   No successors
 ; CHECK-NEXT:  }
-; CHECK-NEXT:  Successor(s): vector.latch
+; CHECK-NEXT:  Successor(s): outer.latch
 ; CHECK-EMPTY:
-; CHECK-NEXT:   vector.latch:
+; CHECK-NEXT:  outer.latch:
 ; CHECK-NEXT:     EMIT ir<%outer.iv.next> = add ir<%outer.iv>, ir<1>
 ; CHECK-NEXT:     EMIT ir<%outer.ec> = icmp ir<%outer.iv.next>, ir<8>
-; CHECK-NEXT:     EMIT branch-on-cond ir<%outer.ec>
+; CHECK-NEXT:  Successor(s): vector.latch
+; CHECK-EMPTY:
+; CHECK-NEXT:   vector.latch:
 ; CHECK-NEXT:   No successors
 ; CHECK-NEXT:  }
 ; CHECK-NEXT: Successor(s): middle.block
diff --git a/llvm/test/Transforms/LoopVectorize/vplan-unused-interleave-group.ll b/llvm/test/Transforms/LoopVectorize/vplan-unused-interleave-group.ll
index 9778ef6853a70..cd63ca3368e30 100644
--- a/llvm/test/Transforms/LoopVectorize/vplan-unused-interleave-group.ll
+++ b/llvm/test/Transforms/LoopVectorize/vplan-unused-interleave-group.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 3
-; RUN: opt -passes=loop-vectorize -S -force-vector-width=4 -enable-interleaved-mem-accesses=true -debug-only=loop-vectorize < %s 2>&1 | FileCheck %s
+; RUN: opt -passes=loop-vectorize -S -force-vector-width=4 -enable-interleaved-mem-accesses=true -debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s
 ; REQUIRES: asserts
 
 ; This test checks if an unused interleave group is removed by removeDeadRecipes.
diff --git a/llvm/test/Transforms/LowerMatrixIntrinsics/multiply-minimal.ll b/llvm/test/Transforms/LowerMatrixIntrinsics/multiply-minimal.ll
index fa52ce39aad9b..279950fbaeff5 100644
--- a/llvm/test/Transforms/LowerMatrixIntrinsics/multiply-minimal.ll
+++ b/llvm/test/Transforms/LowerMatrixIntrinsics/multiply-minimal.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt -passes='lower-matrix-intrinsics<minimal>,instcombine,verify<domtree>' -fuse-matrix-tile-size=2 -matrix-allow-contract -force-fuse-matrix %s -S | FileCheck %s
+; RUN: opt -passes="lower-matrix-intrinsics<minimal>,instcombine,verify<domtree>" -fuse-matrix-tile-size=2 -matrix-allow-contract -force-fuse-matrix %s -S | FileCheck %s
 
 ; Test for the minimal version of the matrix lowering pass, which does not
 ; require DT or AA. Make sure no tiling is happening, even though it was
diff --git a/llvm/test/Transforms/LowerMatrixIntrinsics/strided-store-double.ll b/llvm/test/Transforms/LowerMatrixIntrinsics/strided-store-double.ll
index 80e6df557b81b..28e9cdba1afc4 100644
--- a/llvm/test/Transforms/LowerMatrixIntrinsics/strided-store-double.ll
+++ b/llvm/test/Transforms/LowerMatrixIntrinsics/strided-store-double.ll
@@ -72,6 +72,3 @@ declare void @llvm.matrix.column.major.store.v6f64.i64(<6 x double>, ptr, i64, i
 declare void @llvm.matrix.column.major.store.v6f64.i32(<6 x double>, ptr, i32, i1, i32, i32)
 declare void @llvm.matrix.column.major.store.v10f64.i64(<10 x double>, ptr, i64, i1, i32, i32)
 
-; CHECK: declare void @llvm.matrix.column.major.store.v6f64.i64(<6 x double>, ptr nocapture writeonly, i64, i1 immarg, i32 immarg, i32 immarg) #0
-; CHECK: declare void @llvm.matrix.column.major.store.v10f64.i64(<10 x double>, ptr nocapture writeonly, i64, i1 immarg, i32 immarg, i32 immarg) #0
-; CHECK: attributes #0 = { nocallback nofree nosync nounwind willreturn memory(argmem: write) }
diff --git a/llvm/test/Transforms/MemCpyOpt/atomic.ll b/llvm/test/Transforms/MemCpyOpt/atomic.ll
index 1fb9bdf82b5b0..631a6242df1b3 100644
--- a/llvm/test/Transforms/MemCpyOpt/atomic.ll
+++ b/llvm/test/Transforms/MemCpyOpt/atomic.ll
@@ -33,9 +33,9 @@ define void @test2() nounwind uwtable ssp {
 ; CHECK-LABEL: @test2(
 ; CHECK-NEXT:    [[OLD:%.*]] = alloca i32, align 4
 ; CHECK-NEXT:    [[NEW:%.*]] = alloca i32, align 4
-; CHECK-NEXT:    call void @otherf(ptr nocapture [[NEW]])
+; CHECK-NEXT:    call void @otherf(ptr captures(none) [[NEW]])
 ; CHECK-NEXT:    store atomic i32 0, ptr @x unordered, align 4
-; CHECK-NEXT:    call void @otherf(ptr nocapture [[NEW]])
+; CHECK-NEXT:    call void @otherf(ptr captures(none) [[NEW]])
 ; CHECK-NEXT:    ret void
 ;
   %old = alloca i32
diff --git a/llvm/test/Transforms/MemCpyOpt/callslot.ll b/llvm/test/Transforms/MemCpyOpt/callslot.ll
index 877b73316f901..6c999c57f9187 100644
--- a/llvm/test/Transforms/MemCpyOpt/callslot.ll
+++ b/llvm/test/Transforms/MemCpyOpt/callslot.ll
@@ -137,7 +137,7 @@ define void @capture_before_call_argmemonly() {
 ; CHECK-NEXT:    [[DEST:%.*]] = alloca [16 x i8], align 1
 ; CHECK-NEXT:    [[SRC:%.*]] = alloca [16 x i8], align 1
 ; CHECK-NEXT:    call void @accept_ptr(ptr [[DEST]])
-; CHECK-NEXT:    call void @accept_ptr(ptr nocapture [[DEST]]) #[[ATTR4:[0-9]+]]
+; CHECK-NEXT:    call void @accept_ptr(ptr captures(none) [[DEST]]) #[[ATTR4:[0-9]+]]
 ; CHECK-NEXT:    ret void
 ;
   %dest = alloca [16 x i8]
@@ -153,7 +153,7 @@ define void @capture_before_call_argmemonly_nounwind() {
 ; CHECK-NEXT:    [[DEST:%.*]] = alloca [16 x i8], align 1
 ; CHECK-NEXT:    [[SRC:%.*]] = alloca [16 x i8], align 1
 ; CHECK-NEXT:    call void @accept_ptr(ptr [[DEST]])
-; CHECK-NEXT:    call void @accept_ptr(ptr nocapture [[DEST]]) #[[ATTR5:[0-9]+]]
+; CHECK-NEXT:    call void @accept_ptr(ptr captures(none) [[DEST]]) #[[ATTR5:[0-9]+]]
 ; CHECK-NEXT:    ret void
 ;
   %dest = alloca [16 x i8]
@@ -170,7 +170,7 @@ define void @capture_before_call_argmemonly_nounwind_willreturn() {
 ; CHECK-NEXT:    [[DEST:%.*]] = alloca [16 x i8], align 1
 ; CHECK-NEXT:    [[SRC:%.*]] = alloca [16 x i8], align 1
 ; CHECK-NEXT:    call void @accept_ptr(ptr [[DEST]])
-; CHECK-NEXT:    call void @accept_ptr(ptr nocapture [[DEST]]) #[[ATTR6:[0-9]+]]
+; CHECK-NEXT:    call void @accept_ptr(ptr captures(none) [[DEST]]) #[[ATTR6:[0-9]+]]
 ; CHECK-NEXT:    ret void
 ;
   %dest = alloca [16 x i8]
@@ -212,7 +212,7 @@ nocaptures:
 define void @source_alignment(ptr noalias writable dereferenceable(128) %dst) {
 ; CHECK-LABEL: @source_alignment(
 ; CHECK-NEXT:    [[SRC:%.*]] = alloca [128 x i8], align 4
-; CHECK-NEXT:    call void @accept_ptr(ptr nocapture [[DST:%.*]]) #[[ATTR3]]
+; CHECK-NEXT:    call void @accept_ptr(ptr captures(none) [[DST:%.*]]) #[[ATTR3]]
 ; CHECK-NEXT:    ret void
 ;
   %src = alloca [128 x i8], align 4
@@ -224,7 +224,7 @@ define void @source_alignment(ptr noalias writable dereferenceable(128) %dst) {
 define void @dest_not_writable(ptr noalias dereferenceable(128) %dst) {
 ; CHECK-LABEL: @dest_not_writable(
 ; CHECK-NEXT:    [[SRC:%.*]] = alloca [128 x i8], align 4
-; CHECK-NEXT:    call void @accept_ptr(ptr nocapture [[SRC]]) #[[ATTR3]]
+; CHECK-NEXT:    call void @accept_ptr(ptr captures(none) [[SRC]]) #[[ATTR3]]
 ; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 4 [[DST:%.*]], ptr [[SRC]], i64 128, i1 false)
 ; CHECK-NEXT:    ret void
 ;
diff --git a/llvm/test/Transforms/MemCpyOpt/callslot_noalias.ll b/llvm/test/Transforms/MemCpyOpt/callslot_noalias.ll
index 9b51b6cec3b45..9948d3dacad5f 100644
--- a/llvm/test/Transforms/MemCpyOpt/callslot_noalias.ll
+++ b/llvm/test/Transforms/MemCpyOpt/callslot_noalias.ll
@@ -8,8 +8,8 @@ declare void @func(ptr %dst)
 define i8 @test(ptr writable dereferenceable(1) noalias %dst) {
 ; CHECK-LABEL: @test(
 ; CHECK-NEXT:    [[TMP:%.*]] = alloca i8, align 1
-; CHECK-NEXT:    call void @func(ptr nocapture [[DST:%.*]]) #[[ATTR0:[0-9]+]]
-; CHECK-NEXT:    [[V2:%.*]] = load i8, ptr [[DST]], align 1, !alias.scope !0
+; CHECK-NEXT:    call void @func(ptr captures(none) [[DST:%.*]]) #[[ATTR0:[0-9]+]]
+; CHECK-NEXT:    [[V2:%.*]] = load i8, ptr [[DST]], align 1, !alias.scope [[META0:![0-9]+]]
 ; CHECK-NEXT:    ret i8 [[V2]]
 ;
   %tmp = alloca i8
diff --git a/llvm/test/Transforms/MemCpyOpt/memcpy.ll b/llvm/test/Transforms/MemCpyOpt/memcpy.ll
index 65d78f4199aa0..061b85357b771 100644
--- a/llvm/test/Transforms/MemCpyOpt/memcpy.ll
+++ b/llvm/test/Transforms/MemCpyOpt/memcpy.ll
@@ -395,7 +395,7 @@ define void @test10(ptr noalias nocapture sret(%opaque) %x, i32 %y) {
 ; CHECK-LABEL: @test10(
 ; CHECK-NEXT:    [[A:%.*]] = alloca i32, align 4
 ; CHECK-NEXT:    store i32 [[Y:%.*]], ptr [[A]], align 4
-; CHECK-NEXT:    call void @foo(ptr noalias nocapture [[A]])
+; CHECK-NEXT:    call void @foo(ptr noalias captures(none) [[A]])
 ; CHECK-NEXT:    [[C:%.*]] = load i32, ptr [[A]], align 4
 ; CHECK-NEXT:    store i32 [[C]], ptr [[X:%.*]], align 4
 ; CHECK-NEXT:    ret void
@@ -429,7 +429,7 @@ declare void @f_full_readonly(ptr nocapture noalias readonly)
 
 define void @immut_param(ptr align 4 noalias %val) {
 ; CHECK-LABEL: @immut_param(
-; CHECK-NEXT:    call void @f(ptr noalias nocapture readonly align 4 [[VAL:%.*]])
+; CHECK-NEXT:    call void @f(ptr noalias readonly align 4 captures(none) [[VAL:%.*]])
 ; CHECK-NEXT:    ret void
 ;
   %val1 = alloca i8, align 4
@@ -458,7 +458,7 @@ define void @immut_param_mayalias(ptr align 4 noalias %val) {
 ; CHECK-NEXT:    [[VAL1:%.*]] = alloca i8, align 4
 ; CHECK-NEXT:    call void @f(ptr [[VAL1]])
 ; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 4 [[VAL1]], ptr align 4 [[VAL:%.*]], i64 1, i1 false)
-; CHECK-NEXT:    call void @f(ptr nocapture readonly align 4 [[VAL1]])
+; CHECK-NEXT:    call void @f(ptr readonly align 4 captures(none) [[VAL1]])
 ; CHECK-NEXT:    ret void
 ;
   %val1 = alloca i8, align 4
@@ -472,7 +472,7 @@ define void @immut_param_mayalias(ptr align 4 noalias %val) {
 ; argument doesn't matter.
 define void @immut_param_unescaped_alloca(ptr align 4 noalias %val) {
 ; CHECK-LABEL: @immut_param_unescaped_alloca(
-; CHECK-NEXT:    call void @f(ptr nocapture readonly align 4 [[VAL:%.*]])
+; CHECK-NEXT:    call void @f(ptr readonly align 4 captures(none) [[VAL:%.*]])
 ; CHECK-NEXT:    ret void
 ;
   %val1 = alloca i8, align 4
@@ -487,7 +487,7 @@ define void @immut_param_memory_argmem_read(ptr align 4 noalias %val) {
 ; CHECK-LABEL: @immut_param_memory_argmem_read(
 ; CHECK-NEXT:    [[VAL1:%.*]] = alloca i8, align 4
 ; CHECK-NEXT:    call void @f(ptr [[VAL1]])
-; CHECK-NEXT:    call void @f(ptr nocapture readonly align 4 [[VAL:%.*]]) #[[ATTR6:[0-9]+]]
+; CHECK-NEXT:    call void @f(ptr readonly align 4 captures(none) [[VAL:%.*]]) #[[ATTR6:[0-9]+]]
 ; CHECK-NEXT:    ret void
 ;
   %val1 = alloca i8, align 4
@@ -505,7 +505,7 @@ define void @immut_param_memory_argmem_read_no_readonly(ptr align 4 noalias %val
 ; CHECK-NEXT:    [[VAL1:%.*]] = alloca i8, align 4
 ; CHECK-NEXT:    call void @f(ptr [[VAL1]])
 ; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 4 [[VAL1]], ptr align 4 [[VAL:%.*]], i64 1, i1 false)
-; CHECK-NEXT:    call void @f(ptr nocapture align 4 [[VAL1]]) #[[ATTR6]]
+; CHECK-NEXT:    call void @f(ptr align 4 captures(none) [[VAL1]]) #[[ATTR6]]
 ; CHECK-NEXT:    ret void
 ;
   %val1 = alloca i8, align 4
@@ -520,7 +520,7 @@ define void @immut_param_maywrite(ptr align 4 noalias %val) {
 ; CHECK-LABEL: @immut_param_maywrite(
 ; CHECK-NEXT:    [[VAL1:%.*]] = alloca i8, align 4
 ; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 4 [[VAL1]], ptr align 4 [[VAL:%.*]], i64 1, i1 false)
-; CHECK-NEXT:    call void @f(ptr noalias nocapture align 4 [[VAL1]])
+; CHECK-NEXT:    call void @f(ptr noalias align 4 captures(none) [[VAL1]])
 ; CHECK-NEXT:    ret void
 ;
   %val1 = alloca i8, align 4
@@ -542,7 +542,7 @@ define void @immut_param_readonly(ptr align 4 noalias %val) {
 
 define void @immut_param_no_align(ptr align 4 noalias %val) {
 ; CHECK-LABEL: @immut_param_no_align(
-; CHECK-NEXT:    call void @f(ptr noalias nocapture readonly [[VAL:%.*]])
+; CHECK-NEXT:    call void @f(ptr noalias readonly captures(none) [[VAL:%.*]])
 ; CHECK-NEXT:    ret void
 ;
   %val1 = alloca i8, align 4
@@ -556,7 +556,7 @@ define void @immut_param_no_align(ptr align 4 noalias %val) {
 define void @immut_param_global(ptr align 4 noalias %val) {
 ; CHECK-LABEL: @immut_param_global(
 ; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 4 @gp, ptr align 4 [[VAL:%.*]], i64 1, i1 false)
-; CHECK-NEXT:    call void @f(ptr noalias nocapture readonly align 4 @gp)
+; CHECK-NEXT:    call void @f(ptr noalias readonly align 4 captures(none) @gp)
 ; CHECK-NEXT:    ret void
 ;
   call void @llvm.memcpy.p0.p0.i64(ptr align 4 @gp, ptr align 4 %val, i64 1, i1 false)
@@ -569,7 +569,7 @@ define void @immut_param_vla(ptr align 4 noalias %val, i64 %n) {
 ; CHECK-LABEL: @immut_param_vla(
 ; CHECK-NEXT:    [[VAL1:%.*]] = alloca ptr, i64 [[N:%.*]], align 4
 ; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 4 [[VAL1]], ptr align 4 [[VAL:%.*]], i64 1, i1 false)
-; CHECK-NEXT:    call void @f(ptr noalias nocapture readonly align 4 [[VAL1]])
+; CHECK-NEXT:    call void @f(ptr noalias readonly align 4 captures(none) [[VAL1]])
 ; CHECK-NEXT:    ret void
 ;
   %val1 = alloca ptr, i64 %n
@@ -583,7 +583,7 @@ define void @immut_param_scalable_vector(ptr align 4 noalias %val) {
 ; CHECK-LABEL: @immut_param_scalable_vector(
 ; CHECK-NEXT:    [[VAL1:%.*]] = alloca <vscale x 2 x i32>, align 8
 ; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 4 [[VAL1]], ptr align 4 [[VAL:%.*]], i64 2, i1 false)
-; CHECK-NEXT:    call void @f(ptr noalias nocapture readonly align 4 [[VAL1]])
+; CHECK-NEXT:    call void @f(ptr noalias readonly align 4 captures(none) [[VAL1]])
 ; CHECK-NEXT:    ret void
 ;
   %val1 = alloca <vscale x 2 x i32>
@@ -597,7 +597,7 @@ define void @immut_param_modified_dst(ptr align 4 noalias %val) {
 ; CHECK-LABEL: @immut_param_modified_dst(
 ; CHECK-NEXT:    [[VAL1:%.*]] = alloca i8, align 4
 ; CHECK-NEXT:    store i32 13, ptr [[VAL1]], align 4
-; CHECK-NEXT:    call void @f(ptr noalias nocapture readonly align 4 [[VAL1]])
+; CHECK-NEXT:    call void @f(ptr noalias readonly align 4 captures(none) [[VAL1]])
 ; CHECK-NEXT:    ret void
 ;
   %val1 = alloca i8, align 4
@@ -613,7 +613,7 @@ define void @immut_param_modified_src(ptr align 4 noalias %val) {
 ; CHECK-NEXT:    [[VAL1:%.*]] = alloca i8, align 4
 ; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 4 [[VAL1]], ptr align 4 [[VAL:%.*]], i64 1, i1 false)
 ; CHECK-NEXT:    store i32 13, ptr [[VAL]], align 4
-; CHECK-NEXT:    call void @f(ptr noalias nocapture readonly align 4 [[VAL1]])
+; CHECK-NEXT:    call void @f(ptr noalias readonly align 4 captures(none) [[VAL1]])
 ; CHECK-NEXT:    ret void
 ;
   %val1 = alloca i8, align 4
@@ -628,7 +628,7 @@ define void @immut_param_volatile(ptr align 4 noalias %val) {
 ; CHECK-LABEL: @immut_param_volatile(
 ; CHECK-NEXT:    [[VAL1:%.*]] = alloca i8, align 4
 ; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 4 [[VAL1]], ptr align 4 [[VAL:%.*]], i64 1, i1 true)
-; CHECK-NEXT:    call void @f(ptr noalias nocapture readonly align 4 [[VAL1]])
+; CHECK-NEXT:    call void @f(ptr noalias readonly align 4 captures(none) [[VAL1]])
 ; CHECK-NEXT:    ret void
 ;
   %val1 = alloca i8, align 4
@@ -642,7 +642,7 @@ define void @immut_param_different_addrespace(ptr addrspace(1) align 4 noalias %
 ; CHECK-LABEL: @immut_param_different_addrespace(
 ; CHECK-NEXT:    [[VAL1:%.*]] = alloca i8, align 4
 ; CHECK-NEXT:    call void @llvm.memcpy.p0.p1.i64(ptr align 4 [[VAL1]], ptr addrspace(1) align 4 [[VAL:%.*]], i64 1, i1 false)
-; CHECK-NEXT:    call void @f(ptr noalias nocapture readonly align 4 [[VAL1]])
+; CHECK-NEXT:    call void @f(ptr noalias readonly align 4 captures(none) [[VAL1]])
 ; CHECK-NEXT:    ret void
 ;
   %val1 = alloca i8, align 4
@@ -653,7 +653,7 @@ define void @immut_param_different_addrespace(ptr addrspace(1) align 4 noalias %
 
 define void @immut_param_bigger_align(ptr align 16 noalias %val) {
 ; CHECK-LABEL: @immut_param_bigger_align(
-; CHECK-NEXT:    call void @f(ptr noalias nocapture readonly [[VAL:%.*]])
+; CHECK-NEXT:    call void @f(ptr noalias readonly captures(none) [[VAL:%.*]])
 ; CHECK-NEXT:    ret void
 ;
   %val1 = alloca i8, align 4
@@ -667,7 +667,7 @@ define void @immut_param_smaller_align(ptr align 4 noalias %val) {
 ; CHECK-LABEL: @immut_param_smaller_align(
 ; CHECK-NEXT:    [[VAL1:%.*]] = alloca i8, align 16
 ; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 16 [[VAL1]], ptr [[VAL:%.*]], i64 1, i1 false)
-; CHECK-NEXT:    call void @f(ptr noalias nocapture readonly [[VAL1]])
+; CHECK-NEXT:    call void @f(ptr noalias readonly captures(none) [[VAL1]])
 ; CHECK-NEXT:    ret void
 ;
   %val1 = alloca i8, align 16
@@ -680,7 +680,7 @@ define void @immut_param_enforced_alignment() {
 ; CHECK-LABEL: @immut_param_enforced_alignment(
 ; CHECK-NEXT:    [[VAL:%.*]] = alloca i8, align 4
 ; CHECK-NEXT:    store i32 42, ptr [[VAL]], align 4
-; CHECK-NEXT:    call void @f(ptr noalias nocapture readonly [[VAL]])
+; CHECK-NEXT:    call void @f(ptr noalias readonly captures(none) [[VAL]])
 ; CHECK-NEXT:    ret void
 ;
   %val = alloca i8, align 1
@@ -699,7 +699,7 @@ define void @immut_invalid_align_branched(i1 %c, ptr noalias %val) {
 ; CHECK-NEXT:    [[VAL2:%.*]] = alloca [16 x i8], align 16
 ; CHECK-NEXT:    [[VAL3:%.*]] = select i1 [[C:%.*]], ptr [[VAL1]], ptr [[VAL2]]
 ; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 4 [[VAL3]], ptr align 4 [[VAL:%.*]], i64 4, i1 false)
-; CHECK-NEXT:    call void @f(ptr noalias nocapture readonly [[VAL3]])
+; CHECK-NEXT:    call void @f(ptr noalias readonly captures(none) [[VAL3]])
 ; CHECK-NEXT:    ret void
 ;
   %val1 = alloca [4 x i8], align 4
@@ -715,7 +715,7 @@ define void @immut_but_alias_src(ptr %val) {
 ; CHECK-LABEL: @immut_but_alias_src(
 ; CHECK-NEXT:    [[VAL1:%.*]] = alloca i8, align 4
 ; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 4 [[VAL1]], ptr align 4 [[VAL:%.*]], i64 1, i1 false)
-; CHECK-NEXT:    call void @f(ptr noalias nocapture readonly [[VAL1]])
+; CHECK-NEXT:    call void @f(ptr noalias readonly captures(none) [[VAL1]])
 ; CHECK-NEXT:    ret void
 ;
   %val1 = alloca i8, align 4
@@ -765,7 +765,7 @@ define void @immut_valid_align_branched(i1 %c, ptr noalias align 4 %val) {
 ; CHECK-NEXT:    [[VAL2:%.*]] = alloca [16 x i8], align 4
 ; CHECK-NEXT:    [[VAL3:%.*]] = select i1 [[C:%.*]], ptr [[VAL1]], ptr [[VAL2]]
 ; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 4 [[VAL3]], ptr align 4 [[VAL:%.*]], i64 4, i1 false)
-; CHECK-NEXT:    call void @f(ptr noalias nocapture readonly [[VAL3]])
+; CHECK-NEXT:    call void @f(ptr noalias readonly captures(none) [[VAL3]])
 ; CHECK-NEXT:    ret void
 ;
   %val1 = alloca [4 x i8], align 4
@@ -780,7 +780,7 @@ define void @immut_valid_align_branched(i1 %c, ptr noalias align 4 %val) {
 define void @immut_param_noalias_metadata(ptr align 4 byval(i32) %ptr) {
 ; CHECK-LABEL: @immut_param_noalias_metadata(
 ; CHECK-NEXT:    store i32 1, ptr [[PTR:%.*]], align 4, !noalias [[META0:![0-9]+]]
-; CHECK-NEXT:    call void @f(ptr noalias nocapture readonly [[PTR]])
+; CHECK-NEXT:    call void @f(ptr noalias readonly captures(none) [[PTR]])
 ; CHECK-NEXT:    ret void
 ;
   %tmp = alloca i32, align 4
@@ -895,7 +895,7 @@ define void @memcpy_byval_escape_after(ptr noalias %P) {
 define void @memcpy_immut_escape_after(ptr align 4 noalias %val) {
 ; CHECK-LABEL: @memcpy_immut_escape_after(
 ; CHECK-NEXT:    call void @do_something()
-; CHECK-NEXT:    call void @f(ptr noalias nocapture readonly align 4 [[VAL:%.*]])
+; CHECK-NEXT:    call void @f(ptr noalias readonly align 4 captures(none) [[VAL:%.*]])
 ; CHECK-NEXT:    call void @capture(ptr [[VAL]])
 ; CHECK-NEXT:    ret void
 ;
diff --git a/llvm/test/Transforms/MemCpyOpt/pr29105.ll b/llvm/test/Transforms/MemCpyOpt/pr29105.ll
index 86fc93f1aa6da..d47bddd4597fb 100644
--- a/llvm/test/Transforms/MemCpyOpt/pr29105.ll
+++ b/llvm/test/Transforms/MemCpyOpt/pr29105.ll
@@ -9,7 +9,7 @@ define void @baz() unnamed_addr #0 {
 ; CHECK-NEXT:    [[TMP2:%.*]] = alloca [[FOO:%.*]], align 8
 ; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 16384, ptr nonnull [[TMP2]])
 ; CHECK-NEXT:    call void @llvm.memset.p0.i64(ptr noundef nonnull align 8 dereferenceable(16384) [[TMP2]], i8 0, i64 16384, i1 false)
-; CHECK-NEXT:    call void @bar(ptr noalias nocapture nonnull dereferenceable(16384) [[TMP2]])
+; CHECK-NEXT:    call void @bar(ptr noalias nonnull captures(none) dereferenceable(16384) [[TMP2]])
 ; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 16384, ptr nonnull [[TMP2]])
 ; CHECK-NEXT:    ret void
 ;
diff --git a/llvm/test/Transforms/MemCpyOpt/stack-move.ll b/llvm/test/Transforms/MemCpyOpt/stack-move.ll
index 6089c0a4d7cf5..28412d875be11 100644
--- a/llvm/test/Transforms/MemCpyOpt/stack-move.ll
+++ b/llvm/test/Transforms/MemCpyOpt/stack-move.ll
@@ -25,8 +25,8 @@ define void @basic_memcpy() {
 ; CHECK-LABEL: define void @basic_memcpy() {
 ; CHECK-NEXT:    [[SRC:%.*]] = alloca [[STRUCT_FOO:%.*]], align 4
 ; CHECK-NEXT:    store [[STRUCT_FOO]] { i32 10, i32 20, i32 30 }, ptr [[SRC]], align 4
-; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @use_nocapture(ptr nocapture [[SRC]])
-; CHECK-NEXT:    [[TMP2:%.*]] = call i32 @use_nocapture(ptr nocapture [[SRC]])
+; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @use_nocapture(ptr captures(none) [[SRC]])
+; CHECK-NEXT:    [[TMP2:%.*]] = call i32 @use_nocapture(ptr captures(none) [[SRC]])
 ; CHECK-NEXT:    ret void
 ;
   %src = alloca %struct.Foo, align 4
@@ -68,8 +68,8 @@ define void @basic_memmove() {
 ; CHECK-LABEL: define void @basic_memmove() {
 ; CHECK-NEXT:    [[SRC:%.*]] = alloca [[STRUCT_FOO:%.*]], align 4
 ; CHECK-NEXT:    store [[STRUCT_FOO]] { i32 10, i32 20, i32 30 }, ptr [[SRC]], align 4
-; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @use_nocapture(ptr nocapture [[SRC]])
-; CHECK-NEXT:    [[TMP2:%.*]] = call i32 @use_nocapture(ptr nocapture [[SRC]])
+; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @use_nocapture(ptr captures(none) [[SRC]])
+; CHECK-NEXT:    [[TMP2:%.*]] = call i32 @use_nocapture(ptr captures(none) [[SRC]])
 ; CHECK-NEXT:    ret void
 ;
   %src = alloca %struct.Foo, align 4
@@ -93,8 +93,8 @@ define void @load_store() {
 ; CHECK-LABEL: define void @load_store() {
 ; CHECK-NEXT:    [[SRC:%.*]] = alloca i32, align 4
 ; CHECK-NEXT:    store i32 42, ptr [[SRC]], align 4
-; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @use_nocapture(ptr nocapture [[SRC]])
-; CHECK-NEXT:    [[TMP2:%.*]] = call i32 @use_nocapture(ptr nocapture [[SRC]])
+; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @use_nocapture(ptr captures(none) [[SRC]])
+; CHECK-NEXT:    [[TMP2:%.*]] = call i32 @use_nocapture(ptr captures(none) [[SRC]])
 ; CHECK-NEXT:    ret void
 ;
   %src = alloca i32, align 4
@@ -119,8 +119,8 @@ define void @load_store_scalable(<vscale x 4 x i32> %x) {
 ; CHECK-SAME: (<vscale x 4 x i32> [[X:%.*]]) {
 ; CHECK-NEXT:    [[SRC:%.*]] = alloca <vscale x 4 x i32>, align 16
 ; CHECK-NEXT:    store <vscale x 4 x i32> [[X]], ptr [[SRC]], align 16
-; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @use_nocapture(ptr nocapture [[SRC]])
-; CHECK-NEXT:    [[TMP2:%.*]] = call i32 @use_nocapture(ptr nocapture [[SRC]])
+; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @use_nocapture(ptr captures(none) [[SRC]])
+; CHECK-NEXT:    [[TMP2:%.*]] = call i32 @use_nocapture(ptr captures(none) [[SRC]])
 ; CHECK-NEXT:    ret void
 ;
   %src = alloca <vscale x 4 x i32>
@@ -145,8 +145,8 @@ define void @align_up() {
 ; CHECK-LABEL: define void @align_up() {
 ; CHECK-NEXT:    [[SRC:%.*]] = alloca [[STRUCT_FOO:%.*]], align 8
 ; CHECK-NEXT:    store [[STRUCT_FOO]] { i32 10, i32 20, i32 30 }, ptr [[SRC]], align 4
-; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @use_nocapture(ptr nocapture [[SRC]])
-; CHECK-NEXT:    [[TMP2:%.*]] = call i32 @use_nocapture(ptr nocapture [[SRC]])
+; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @use_nocapture(ptr captures(none) [[SRC]])
+; CHECK-NEXT:    [[TMP2:%.*]] = call i32 @use_nocapture(ptr captures(none) [[SRC]])
 ; CHECK-NEXT:    ret void
 ;
   %src = alloca %struct.Foo, align 4
@@ -170,9 +170,9 @@ define void @remove_extra_lifetime_intrinsics() {
 ; CHECK-LABEL: define void @remove_extra_lifetime_intrinsics() {
 ; CHECK-NEXT:    [[SRC:%.*]] = alloca [[STRUCT_FOO:%.*]], align 4
 ; CHECK-NEXT:    store [[STRUCT_FOO]] { i32 10, i32 20, i32 30 }, ptr [[SRC]], align 4
-; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @use_nocapture(ptr nocapture [[SRC]])
-; CHECK-NEXT:    [[TMP2:%.*]] = call i32 @use_nocapture(ptr nocapture [[SRC]])
-; CHECK-NEXT:    [[TMP3:%.*]] = call i32 @use_nocapture(ptr nocapture [[SRC]])
+; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @use_nocapture(ptr captures(none) [[SRC]])
+; CHECK-NEXT:    [[TMP2:%.*]] = call i32 @use_nocapture(ptr captures(none) [[SRC]])
+; CHECK-NEXT:    [[TMP3:%.*]] = call i32 @use_nocapture(ptr captures(none) [[SRC]])
 ; CHECK-NEXT:    ret void
 ;
   %src = alloca %struct.Foo, align 4
@@ -200,9 +200,9 @@ define void @no_lifetime() {
 ; CHECK-LABEL: define void @no_lifetime() {
 ; CHECK-NEXT:    [[SRC:%.*]] = alloca [[STRUCT_FOO:%.*]], align 4
 ; CHECK-NEXT:    store [[STRUCT_FOO]] { i32 10, i32 20, i32 30 }, ptr [[SRC]], align 4
-; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @use_nocapture(ptr nocapture [[SRC]])
-; CHECK-NEXT:    [[TMP2:%.*]] = call i32 @use_nocapture(ptr nocapture [[SRC]])
-; CHECK-NEXT:    [[TMP3:%.*]] = call i32 @use_nocapture(ptr nocapture [[SRC]])
+; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @use_nocapture(ptr captures(none) [[SRC]])
+; CHECK-NEXT:    [[TMP2:%.*]] = call i32 @use_nocapture(ptr captures(none) [[SRC]])
+; CHECK-NEXT:    [[TMP3:%.*]] = call i32 @use_nocapture(ptr captures(none) [[SRC]])
 ; CHECK-NEXT:    ret void
 ;
   %src = alloca %struct.Foo, align 4
@@ -223,9 +223,9 @@ define void @alias_no_mod() {
 ; CHECK-NEXT:    [[SRC:%.*]] = alloca [[STRUCT_FOO:%.*]], align 4
 ; CHECK-NEXT:    [[DEST_ALIAS:%.*]] = getelementptr [[STRUCT_FOO]], ptr [[SRC]], i32 0, i32 0
 ; CHECK-NEXT:    store [[STRUCT_FOO]] { i32 10, i32 20, i32 30 }, ptr [[SRC]], align 4
-; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @use_nocapture(ptr nocapture [[SRC]])
+; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @use_nocapture(ptr captures(none) [[SRC]])
 ; CHECK-NEXT:    [[SRC_ALIAS:%.*]] = getelementptr [[STRUCT_FOO]], ptr [[SRC]], i32 0, i32 0
-; CHECK-NEXT:    [[TMP2:%.*]] = call i32 @use_nocapture(ptr nocapture [[SRC]])
+; CHECK-NEXT:    [[TMP2:%.*]] = call i32 @use_nocapture(ptr captures(none) [[SRC]])
 ; CHECK-NEXT:    ret void
 ;
   %src = alloca %struct.Foo, align 4
@@ -259,8 +259,8 @@ define void @remove_scoped_noalias() {
 ; CHECK-LABEL: define void @remove_scoped_noalias() {
 ; CHECK-NEXT:    [[SRC:%.*]] = alloca [[STRUCT_FOO:%.*]], align 4
 ; CHECK-NEXT:    store [[STRUCT_FOO]] { i32 10, i32 20, i32 30 }, ptr [[SRC]], align 4
-; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @use_nocapture(ptr nocapture [[SRC]]), !alias.scope !0
-; CHECK-NEXT:    [[TMP2:%.*]] = call i32 @use_nocapture(ptr nocapture [[SRC]])
+; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @use_nocapture(ptr captures(none) [[SRC]]), !alias.scope [[META0:![0-9]+]]
+; CHECK-NEXT:    [[TMP2:%.*]] = call i32 @use_nocapture(ptr captures(none) [[SRC]])
 ; CHECK-NEXT:    ret void
 ;
   %src = alloca %struct.Foo, align 4
@@ -283,8 +283,8 @@ define void @remove_alloca_metadata() {
 ; CHECK-LABEL: define void @remove_alloca_metadata() {
 ; CHECK-NEXT:    [[SRC:%.*]] = alloca [[STRUCT_FOO:%.*]], align 4
 ; CHECK-NEXT:    store [[STRUCT_FOO]] { i32 10, i32 20, i32 30 }, ptr [[SRC]], align 4
-; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @use_nocapture(ptr nocapture [[SRC]]), !alias.scope !0
-; CHECK-NEXT:    [[TMP2:%.*]] = call i32 @use_nocapture(ptr nocapture [[SRC]])
+; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @use_nocapture(ptr captures(none) [[SRC]]), !alias.scope [[META0]]
+; CHECK-NEXT:    [[TMP2:%.*]] = call i32 @use_nocapture(ptr captures(none) [[SRC]])
 ; CHECK-NEXT:    ret void
 ;
   %src = alloca %struct.Foo, align 4, !annotation !3
@@ -308,8 +308,8 @@ define void @noalias_on_lifetime() {
 ; CHECK-LABEL: define void @noalias_on_lifetime() {
 ; CHECK-NEXT:    [[SRC:%.*]] = alloca [[STRUCT_FOO:%.*]], align 4
 ; CHECK-NEXT:    store [[STRUCT_FOO]] { i32 10, i32 20, i32 30 }, ptr [[SRC]], align 4
-; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @use_nocapture(ptr nocapture [[SRC]]), !alias.scope !0
-; CHECK-NEXT:    [[TMP2:%.*]] = call i32 @use_nocapture(ptr nocapture [[SRC]])
+; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @use_nocapture(ptr captures(none) [[SRC]]), !alias.scope [[META0]]
+; CHECK-NEXT:    [[TMP2:%.*]] = call i32 @use_nocapture(ptr captures(none) [[SRC]])
 ; CHECK-NEXT:    ret void
 ;
   %src = alloca %struct.Foo, align 4
@@ -332,8 +332,8 @@ define void @src_ref_dest_ref_after_copy() {
 ; CHECK-LABEL: define void @src_ref_dest_ref_after_copy() {
 ; CHECK-NEXT:    [[SRC:%.*]] = alloca [[STRUCT_FOO:%.*]], align 4
 ; CHECK-NEXT:    store [[STRUCT_FOO]] { i32 10, i32 20, i32 30 }, ptr [[SRC]], align 4
-; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @use_readonly(ptr nocapture [[SRC]])
-; CHECK-NEXT:    [[TMP2:%.*]] = call i32 @use_readonly(ptr nocapture [[SRC]])
+; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @use_readonly(ptr captures(none) [[SRC]])
+; CHECK-NEXT:    [[TMP2:%.*]] = call i32 @use_readonly(ptr captures(none) [[SRC]])
 ; CHECK-NEXT:    ret void
 ;
   %src = alloca %struct.Foo, align 4
@@ -356,8 +356,8 @@ define void @src_mod_dest_mod_after_copy() {
 ; CHECK-LABEL: define void @src_mod_dest_mod_after_copy() {
 ; CHECK-NEXT:    [[SRC:%.*]] = alloca [[STRUCT_FOO:%.*]], align 4
 ; CHECK-NEXT:    store [[STRUCT_FOO]] { i32 10, i32 20, i32 30 }, ptr [[SRC]], align 4
-; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @use_writeonly(ptr nocapture [[SRC]])
-; CHECK-NEXT:    [[TMP2:%.*]] = call i32 @use_writeonly(ptr nocapture [[SRC]])
+; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @use_writeonly(ptr captures(none) [[SRC]])
+; CHECK-NEXT:    [[TMP2:%.*]] = call i32 @use_writeonly(ptr captures(none) [[SRC]])
 ; CHECK-NEXT:    ret void
 ;
   %src = alloca %struct.Foo, align 4
@@ -397,12 +397,12 @@ define void @terminator_lastuse() personality i32 0 {
 ; CHECK-LABEL: define void @terminator_lastuse() personality i32 0 {
 ; CHECK-NEXT:    [[SRC:%.*]] = alloca [[STRUCT_FOO:%.*]], align 4
 ; CHECK-NEXT:    store [[STRUCT_FOO]] { i32 10, i32 20, i32 30 }, ptr [[SRC]], align 4
-; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @use_nocapture(ptr nocapture [[SRC]])
+; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @use_nocapture(ptr captures(none) [[SRC]])
 ; CHECK-NEXT:    [[RV:%.*]] = invoke i32 @use_nocapture(ptr [[SRC]])
-; CHECK-NEXT:    to label [[SUC:%.*]] unwind label [[UNW:%.*]]
+; CHECK-NEXT:            to label [[SUC:%.*]] unwind label [[UNW:%.*]]
 ; CHECK:       unw:
 ; CHECK-NEXT:    [[LP:%.*]] = landingpad i32
-; CHECK-NEXT:    cleanup
+; CHECK-NEXT:            cleanup
 ; CHECK-NEXT:    resume i32 0
 ; CHECK:       suc:
 ; CHECK-NEXT:    ret void
@@ -431,12 +431,12 @@ define void @multi_bb_memcpy(i1 %b) {
 ; CHECK-SAME: (i1 [[B:%.*]]) {
 ; CHECK-NEXT:    [[SRC:%.*]] = alloca i32, align 4
 ; CHECK-NEXT:    store i32 42, ptr [[SRC]], align 4
-; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @use_nocapture(ptr nocapture [[SRC]])
+; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @use_nocapture(ptr captures(none) [[SRC]])
 ; CHECK-NEXT:    br label [[BB0:%.*]]
 ; CHECK:       bb0:
 ; CHECK-NEXT:    br label [[BB1:%.*]]
 ; CHECK:       bb1:
-; CHECK-NEXT:    [[TMP2:%.*]] = call i32 @use_nocapture(ptr nocapture [[SRC]])
+; CHECK-NEXT:    [[TMP2:%.*]] = call i32 @use_nocapture(ptr captures(none) [[SRC]])
 ; CHECK-NEXT:    ret void
 ;
   %src = alloca i32, align 4
@@ -463,10 +463,10 @@ define void @multi_bb_load_store(i1 %b) {
 ; CHECK-SAME: (i1 [[B:%.*]]) {
 ; CHECK-NEXT:    [[SRC:%.*]] = alloca i32, align 4
 ; CHECK-NEXT:    store i32 42, ptr [[SRC]], align 4
-; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @use_nocapture(ptr nocapture [[SRC]])
+; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @use_nocapture(ptr captures(none) [[SRC]])
 ; CHECK-NEXT:    br label [[BB0:%.*]]
 ; CHECK:       bb0:
-; CHECK-NEXT:    [[TMP2:%.*]] = call i32 @use_nocapture(ptr nocapture [[SRC]])
+; CHECK-NEXT:    [[TMP2:%.*]] = call i32 @use_nocapture(ptr captures(none) [[SRC]])
 ; CHECK-NEXT:    ret void
 ;
   %src = alloca i32, align 4
@@ -494,19 +494,19 @@ define void @multi_bb_separated_load_store(i1 %b) {
 ; CHECK-SAME: (i1 [[B:%.*]]) {
 ; CHECK-NEXT:    [[SRC:%.*]] = alloca i32, align 4
 ; CHECK-NEXT:    [[DEST:%.*]] = alloca i32, align 4
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr nocapture [[SRC]])
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr nocapture [[DEST]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr captures(none) [[SRC]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr captures(none) [[DEST]])
 ; CHECK-NEXT:    store i32 42, ptr [[SRC]], align 4
-; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @use_nocapture(ptr nocapture [[SRC]])
+; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @use_nocapture(ptr captures(none) [[SRC]])
 ; CHECK-NEXT:    [[SRC_VAL:%.*]] = load i32, ptr [[SRC]], align 4
 ; CHECK-NEXT:    br label [[BB0:%.*]]
 ; CHECK:       bb0:
 ; CHECK-NEXT:    store i32 [[SRC_VAL]], ptr [[DEST]], align 4
 ; CHECK-NEXT:    br label [[BB1:%.*]]
 ; CHECK:       bb1:
-; CHECK-NEXT:    [[TMP2:%.*]] = call i32 @use_nocapture(ptr nocapture [[DEST]])
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 4, ptr nocapture [[SRC]])
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 4, ptr nocapture [[DEST]])
+; CHECK-NEXT:    [[TMP2:%.*]] = call i32 @use_nocapture(ptr captures(none) [[DEST]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 4, ptr captures(none) [[SRC]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 4, ptr captures(none) [[DEST]])
 ; CHECK-NEXT:    ret void
 ;
   %src = alloca i32, align 4
@@ -535,13 +535,13 @@ define void @multi_bb_simple_br(i1 %b) {
 ; CHECK-SAME: (i1 [[B:%.*]]) {
 ; CHECK-NEXT:    [[SRC:%.*]] = alloca [[STRUCT_FOO:%.*]], align 4
 ; CHECK-NEXT:    store [[STRUCT_FOO]] { i32 10, i32 20, i32 30 }, ptr [[SRC]], align 4
-; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @use_nocapture(ptr nocapture noundef [[SRC]])
+; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @use_nocapture(ptr noundef captures(none) [[SRC]])
 ; CHECK-NEXT:    br i1 [[B]], label [[BB0:%.*]], label [[BB1:%.*]]
 ; CHECK:       bb0:
-; CHECK-NEXT:    [[TMP2:%.*]] = call i32 @use_nocapture(ptr nocapture noundef [[SRC]])
+; CHECK-NEXT:    [[TMP2:%.*]] = call i32 @use_nocapture(ptr noundef captures(none) [[SRC]])
 ; CHECK-NEXT:    br label [[BB2:%.*]]
 ; CHECK:       bb1:
-; CHECK-NEXT:    [[TMP3:%.*]] = call i32 @use_nocapture(ptr nocapture noundef [[SRC]])
+; CHECK-NEXT:    [[TMP3:%.*]] = call i32 @use_nocapture(ptr noundef captures(none) [[SRC]])
 ; CHECK-NEXT:    br label [[BB2]]
 ; CHECK:       bb2:
 ; CHECK-NEXT:    ret void
@@ -582,7 +582,7 @@ define void @multi_bb_dom_test0(i1 %b) {
 ; CHECK-NEXT:    store [[STRUCT_FOO]] { i32 40, i32 50, i32 60 }, ptr [[SRC]], align 4
 ; CHECK-NEXT:    br label [[BB2]]
 ; CHECK:       bb2:
-; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @use_nocapture(ptr nocapture noundef [[SRC]])
+; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @use_nocapture(ptr noundef captures(none) [[SRC]])
 ; CHECK-NEXT:    ret void
 ;
   %src = alloca %struct.Foo, align 4
@@ -620,12 +620,12 @@ define void @multi_bb_dom_test1(i1 %b) {
 ; CHECK-NEXT:    store [[STRUCT_FOO]] { i32 40, i32 50, i32 60 }, ptr [[SRC]], align 4
 ; CHECK-NEXT:    br label [[BB2]]
 ; CHECK:       bb2:
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 12, ptr nocapture [[DEST]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 12, ptr captures(none) [[DEST]])
 ; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 4 [[DEST]], ptr align 4 [[SRC]], i64 12, i1 false)
-; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @use_nocapture(ptr nocapture noundef [[DEST]])
+; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @use_nocapture(ptr noundef captures(none) [[DEST]])
 ; CHECK-NEXT:    ret void
 ; CHECK:       unr:
-; CHECK-NEXT:    [[TMP2:%.*]] = call i32 @use_nocapture(ptr nocapture noundef [[DEST]])
+; CHECK-NEXT:    [[TMP2:%.*]] = call i32 @use_nocapture(ptr noundef captures(none) [[DEST]])
 ; CHECK-NEXT:    br label [[BB2]]
 ;
   %src = alloca %struct.Foo, align 4
@@ -660,13 +660,13 @@ define void @multi_bb_pdom_test0(i1 %b) {
 ; CHECK-NEXT:    store [[STRUCT_FOO]] { i32 10, i32 20, i32 30 }, ptr [[SRC]], align 4
 ; CHECK-NEXT:    br i1 [[B]], label [[BB0:%.*]], label [[BB1:%.*]]
 ; CHECK:       bb0:
-; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @use_nocapture(ptr nocapture noundef [[SRC]])
+; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @use_nocapture(ptr noundef captures(none) [[SRC]])
 ; CHECK-NEXT:    br label [[BB2:%.*]]
 ; CHECK:       bb1:
-; CHECK-NEXT:    [[TMP2:%.*]] = call i32 @use_nocapture(ptr nocapture noundef [[SRC]])
+; CHECK-NEXT:    [[TMP2:%.*]] = call i32 @use_nocapture(ptr noundef captures(none) [[SRC]])
 ; CHECK-NEXT:    br label [[BB2]]
 ; CHECK:       bb2:
-; CHECK-NEXT:    [[TMP3:%.*]] = call i32 @use_nocapture(ptr nocapture noundef [[SRC]])
+; CHECK-NEXT:    [[TMP3:%.*]] = call i32 @use_nocapture(ptr noundef captures(none) [[SRC]])
 ; CHECK-NEXT:    ret void
 ;
   %src = alloca %struct.Foo, align 4
@@ -700,10 +700,10 @@ define void @multi_bb_pdom_test1(i1 %b) {
 ; CHECK-NEXT:    store [[STRUCT_FOO]] { i32 10, i32 20, i32 30 }, ptr [[SRC]], align 4
 ; CHECK-NEXT:    br i1 [[B]], label [[BB0:%.*]], label [[BB1:%.*]]
 ; CHECK:       bb0:
-; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @use_nocapture(ptr nocapture noundef [[SRC]])
+; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @use_nocapture(ptr noundef captures(none) [[SRC]])
 ; CHECK-NEXT:    br label [[BB2:%.*]]
 ; CHECK:       bb1:
-; CHECK-NEXT:    [[TMP2:%.*]] = call i32 @use_nocapture(ptr nocapture noundef [[SRC]])
+; CHECK-NEXT:    [[TMP2:%.*]] = call i32 @use_nocapture(ptr noundef captures(none) [[SRC]])
 ; CHECK-NEXT:    br label [[BB2]]
 ; CHECK:       bb2:
 ; CHECK-NEXT:    [[I:%.*]] = phi i32 [ 42, [[BB0]] ], [ 41, [[BB1]] ]
@@ -736,13 +736,13 @@ define void @multi_bb_pdom_test2(i1 %b) {
 ; CHECK-SAME: (i1 [[B:%.*]]) {
 ; CHECK-NEXT:    [[SRC:%.*]] = alloca [[STRUCT_FOO:%.*]], align 4
 ; CHECK-NEXT:    store [[STRUCT_FOO]] { i32 10, i32 20, i32 30 }, ptr [[SRC]], align 4
-; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @use_nocapture(ptr nocapture noundef [[SRC]])
+; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @use_nocapture(ptr noundef captures(none) [[SRC]])
 ; CHECK-NEXT:    ret void
 ; CHECK:       unr1:
-; CHECK-NEXT:    [[TMP2:%.*]] = call i32 @use_nocapture(ptr nocapture noundef [[SRC]])
+; CHECK-NEXT:    [[TMP2:%.*]] = call i32 @use_nocapture(ptr noundef captures(none) [[SRC]])
 ; CHECK-NEXT:    br label [[UNR2:%.*]]
 ; CHECK:       unr2:
-; CHECK-NEXT:    [[TMP3:%.*]] = call i32 @use_nocapture(ptr nocapture noundef [[SRC]])
+; CHECK-NEXT:    [[TMP3:%.*]] = call i32 @use_nocapture(ptr noundef captures(none) [[SRC]])
 ; CHECK-NEXT:    br label [[UNR1:%.*]]
 ;
   %src = alloca %struct.Foo, align 4
@@ -806,10 +806,10 @@ define void @multi_bb_unreachable_modref(i1 %b0) {
 ; CHECK-SAME: (i1 [[B0:%.*]]) {
 ; CHECK-NEXT:    [[SRC:%.*]] = alloca [[STRUCT_FOO:%.*]], align 4
 ; CHECK-NEXT:    store [[STRUCT_FOO]] { i32 10, i32 20, i32 30 }, ptr [[SRC]], align 4
-; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @use_nocapture(ptr nocapture noundef [[SRC]])
+; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @use_nocapture(ptr noundef captures(none) [[SRC]])
 ; CHECK-NEXT:    br i1 [[B0]], label [[BB0:%.*]], label [[EXIT:%.*]]
 ; CHECK:       exit:
-; CHECK-NEXT:    [[TMP2:%.*]] = call i32 @use_nocapture(ptr nocapture noundef [[SRC]])
+; CHECK-NEXT:    [[TMP2:%.*]] = call i32 @use_nocapture(ptr noundef captures(none) [[SRC]])
 ; CHECK-NEXT:    ret void
 ; CHECK:       bb0:
 ; CHECK-NEXT:    ret void
@@ -838,12 +838,12 @@ define void @multi_bb_non_dominated(i1 %b0, i1 %b1) {
 ; CHECK-SAME: (i1 [[B0:%.*]], i1 [[B1:%.*]]) {
 ; CHECK-NEXT:    [[SRC:%.*]] = alloca [[STRUCT_FOO:%.*]], align 4
 ; CHECK-NEXT:    store [[STRUCT_FOO]] { i32 10, i32 20, i32 30 }, ptr [[SRC]], align 4
-; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @use_nocapture(ptr nocapture noundef [[SRC]])
+; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @use_nocapture(ptr noundef captures(none) [[SRC]])
 ; CHECK-NEXT:    br i1 [[B0]], label [[BB0:%.*]], label [[BB1:%.*]]
 ; CHECK:       bb0:
 ; CHECK-NEXT:    br label [[BB2:%.*]]
 ; CHECK:       bb1:
-; CHECK-NEXT:    [[TMP2:%.*]] = call i32 @use_nocapture(ptr nocapture noundef [[SRC]])
+; CHECK-NEXT:    [[TMP2:%.*]] = call i32 @use_nocapture(ptr noundef captures(none) [[SRC]])
 ; CHECK-NEXT:    br label [[BB2]]
 ; CHECK:       bb2:
 ; CHECK-NEXT:    ret void
@@ -878,16 +878,16 @@ define void @memcpy_is_def() {
 ; CHECK-LABEL: define void @memcpy_is_def() {
 ; CHECK-NEXT:    [[SRC:%.*]] = alloca [[STRUCT_FOO:%.*]], align 4
 ; CHECK-NEXT:    [[DEST:%.*]] = alloca [[STRUCT_FOO]], align 4
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 12, ptr nocapture [[SRC]])
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 12, ptr nocapture [[DEST]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 12, ptr captures(none) [[SRC]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 12, ptr captures(none) [[DEST]])
 ; CHECK-NEXT:    store [[STRUCT_FOO]] { i32 10, i32 20, i32 30 }, ptr [[SRC]], align 4
-; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @use_nocapture(ptr nocapture noundef [[SRC]])
+; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @use_nocapture(ptr noundef captures(none) [[SRC]])
 ; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 4 [[DEST]], ptr align 4 [[SRC]], i64 12, i1 false)
-; CHECK-NEXT:    [[TMP2:%.*]] = call i32 @use_nocapture(ptr nocapture noundef [[DEST]])
+; CHECK-NEXT:    [[TMP2:%.*]] = call i32 @use_nocapture(ptr noundef captures(none) [[DEST]])
 ; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 4 [[SRC]], ptr align 4 [[DEST]], i64 12, i1 false)
-; CHECK-NEXT:    [[TMP3:%.*]] = call i32 @use_nocapture(ptr nocapture noundef [[SRC]])
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 12, ptr nocapture [[SRC]])
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 12, ptr nocapture [[DEST]])
+; CHECK-NEXT:    [[TMP3:%.*]] = call i32 @use_nocapture(ptr noundef captures(none) [[SRC]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 12, ptr captures(none) [[SRC]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 12, ptr captures(none) [[DEST]])
 ; CHECK-NEXT:    ret void
 ;
   %src = alloca %struct.Foo, align 4
@@ -912,16 +912,16 @@ define void @memset_is_def() {
 ; CHECK-LABEL: define void @memset_is_def() {
 ; CHECK-NEXT:    [[SRC:%.*]] = alloca [[STRUCT_FOO:%.*]], align 4
 ; CHECK-NEXT:    [[DEST:%.*]] = alloca [[STRUCT_FOO]], align 4
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 12, ptr nocapture [[SRC]])
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 12, ptr nocapture [[DEST]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 12, ptr captures(none) [[SRC]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 12, ptr captures(none) [[DEST]])
 ; CHECK-NEXT:    store [[STRUCT_FOO]] { i32 10, i32 20, i32 30 }, ptr [[SRC]], align 4
-; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @use_nocapture(ptr nocapture noundef [[SRC]])
+; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @use_nocapture(ptr noundef captures(none) [[SRC]])
 ; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 4 [[DEST]], ptr align 4 [[SRC]], i64 12, i1 false)
-; CHECK-NEXT:    [[TMP2:%.*]] = call i32 @use_nocapture(ptr nocapture noundef [[DEST]])
+; CHECK-NEXT:    [[TMP2:%.*]] = call i32 @use_nocapture(ptr noundef captures(none) [[DEST]])
 ; CHECK-NEXT:    call void @llvm.memset.p0.i64(ptr align 4 [[SRC]], i8 42, i64 12, i1 false)
-; CHECK-NEXT:    [[TMP3:%.*]] = call i32 @use_nocapture(ptr nocapture noundef [[SRC]])
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 12, ptr nocapture [[SRC]])
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 12, ptr nocapture [[DEST]])
+; CHECK-NEXT:    [[TMP3:%.*]] = call i32 @use_nocapture(ptr noundef captures(none) [[SRC]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 12, ptr captures(none) [[SRC]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 12, ptr captures(none) [[DEST]])
 ; CHECK-NEXT:    ret void
 ;
   %src = alloca %struct.Foo, align 4
@@ -946,17 +946,17 @@ define void @store_is_def() {
 ; CHECK-LABEL: define void @store_is_def() {
 ; CHECK-NEXT:    [[SRC:%.*]] = alloca i32, align 4
 ; CHECK-NEXT:    [[DEST:%.*]] = alloca i32, align 4
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr nocapture [[SRC]])
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr nocapture [[DEST]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr captures(none) [[SRC]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr captures(none) [[DEST]])
 ; CHECK-NEXT:    store i32 42, ptr [[SRC]], align 4
-; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @use_nocapture(ptr nocapture noundef [[SRC]])
+; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @use_nocapture(ptr noundef captures(none) [[SRC]])
 ; CHECK-NEXT:    [[TMP2:%.*]] = load i32, ptr [[SRC]], align 4
 ; CHECK-NEXT:    store i32 [[TMP2]], ptr [[DEST]], align 4
-; CHECK-NEXT:    [[TMP3:%.*]] = call i32 @use_nocapture(ptr nocapture noundef [[DEST]])
+; CHECK-NEXT:    [[TMP3:%.*]] = call i32 @use_nocapture(ptr noundef captures(none) [[DEST]])
 ; CHECK-NEXT:    store i32 64, ptr [[SRC]], align 4
-; CHECK-NEXT:    [[TMP4:%.*]] = call i32 @use_nocapture(ptr nocapture noundef [[SRC]])
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 4, ptr nocapture [[SRC]])
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 4, ptr nocapture [[DEST]])
+; CHECK-NEXT:    [[TMP4:%.*]] = call i32 @use_nocapture(ptr noundef captures(none) [[SRC]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 4, ptr captures(none) [[SRC]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 4, ptr captures(none) [[DEST]])
 ; CHECK-NEXT:    ret void
 ;
   %src = alloca i32, align 4
@@ -982,21 +982,21 @@ define void @multi_bb_dataflow(i1 %b) {
 ; CHECK-SAME: (i1 [[B:%.*]]) {
 ; CHECK-NEXT:    [[SRC:%.*]] = alloca [[STRUCT_FOO:%.*]], align 4
 ; CHECK-NEXT:    [[DEST:%.*]] = alloca [[STRUCT_FOO]], align 4
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 12, ptr nocapture [[SRC]])
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 12, ptr nocapture [[DEST]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 12, ptr captures(none) [[SRC]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 12, ptr captures(none) [[DEST]])
 ; CHECK-NEXT:    store [[STRUCT_FOO]] { i32 10, i32 20, i32 30 }, ptr [[SRC]], align 4
-; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @use_nocapture(ptr nocapture noundef [[SRC]])
+; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @use_nocapture(ptr noundef captures(none) [[SRC]])
 ; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 4 [[DEST]], ptr align 4 [[SRC]], i64 12, i1 false)
 ; CHECK-NEXT:    br i1 [[B]], label [[BB0:%.*]], label [[BB1:%.*]]
 ; CHECK:       bb0:
-; CHECK-NEXT:    [[TMP2:%.*]] = call i32 @use_nocapture(ptr nocapture noundef [[SRC]])
+; CHECK-NEXT:    [[TMP2:%.*]] = call i32 @use_nocapture(ptr noundef captures(none) [[SRC]])
 ; CHECK-NEXT:    br label [[BB2:%.*]]
 ; CHECK:       bb1:
-; CHECK-NEXT:    [[TMP3:%.*]] = call i32 @use_nocapture(ptr nocapture noundef [[DEST]])
+; CHECK-NEXT:    [[TMP3:%.*]] = call i32 @use_nocapture(ptr noundef captures(none) [[DEST]])
 ; CHECK-NEXT:    br label [[BB2]]
 ; CHECK:       bb2:
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 12, ptr nocapture [[SRC]])
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 12, ptr nocapture [[DEST]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 12, ptr captures(none) [[SRC]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 12, ptr captures(none) [[DEST]])
 ; CHECK-NEXT:    ret void
 ;
   %src = alloca %struct.Foo, align 4
@@ -1031,14 +1031,14 @@ define void @incomplete_memcpy() {
 ; CHECK-LABEL: define void @incomplete_memcpy() {
 ; CHECK-NEXT:    [[SRC:%.*]] = alloca [[STRUCT_FOO:%.*]], align 4
 ; CHECK-NEXT:    [[DEST:%.*]] = alloca [[STRUCT_FOO]], align 4
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 12, ptr nocapture [[SRC]])
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 12, ptr nocapture [[DEST]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 12, ptr captures(none) [[SRC]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 12, ptr captures(none) [[DEST]])
 ; CHECK-NEXT:    store [[STRUCT_FOO]] { i32 10, i32 20, i32 30 }, ptr [[SRC]], align 4
-; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @use_nocapture(ptr nocapture noundef [[SRC]])
+; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @use_nocapture(ptr noundef captures(none) [[SRC]])
 ; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 4 [[DEST]], ptr align 4 [[SRC]], i64 11, i1 false)
-; CHECK-NEXT:    [[TMP2:%.*]] = call i32 @use_nocapture(ptr nocapture noundef [[DEST]])
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 12, ptr nocapture [[SRC]])
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 12, ptr nocapture [[DEST]])
+; CHECK-NEXT:    [[TMP2:%.*]] = call i32 @use_nocapture(ptr noundef captures(none) [[DEST]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 12, ptr captures(none) [[SRC]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 12, ptr captures(none) [[DEST]])
 ; CHECK-NEXT:    ret void
 ;
   %src = alloca %struct.Foo, align 4
@@ -1060,15 +1060,15 @@ define void @incomplete_store() {
 ; CHECK-LABEL: define void @incomplete_store() {
 ; CHECK-NEXT:    [[SRC:%.*]] = alloca [[STRUCT_FOO:%.*]], align 4
 ; CHECK-NEXT:    [[DEST:%.*]] = alloca [[STRUCT_FOO]], align 4
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 12, ptr nocapture [[SRC]])
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 12, ptr nocapture [[DEST]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 12, ptr captures(none) [[SRC]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 12, ptr captures(none) [[DEST]])
 ; CHECK-NEXT:    store [[STRUCT_FOO]] { i32 10, i32 20, i32 30 }, ptr [[SRC]], align 4
-; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @use_nocapture(ptr nocapture noundef [[SRC]])
+; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @use_nocapture(ptr noundef captures(none) [[SRC]])
 ; CHECK-NEXT:    [[TMP2:%.*]] = load i32, ptr [[SRC]], align 4
 ; CHECK-NEXT:    store i32 [[TMP2]], ptr [[DEST]], align 4
-; CHECK-NEXT:    [[TMP3:%.*]] = call i32 @use_nocapture(ptr nocapture noundef [[DEST]])
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 12, ptr nocapture [[SRC]])
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 12, ptr nocapture [[DEST]])
+; CHECK-NEXT:    [[TMP3:%.*]] = call i32 @use_nocapture(ptr noundef captures(none) [[DEST]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 12, ptr captures(none) [[SRC]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 12, ptr captures(none) [[DEST]])
 ; CHECK-NEXT:    ret void
 ;
   %src = alloca %struct.Foo, align 4
@@ -1091,14 +1091,14 @@ define void @dynamically_sized_alloca(i64 %i) {
 ; CHECK-SAME: (i64 [[I:%.*]]) {
 ; CHECK-NEXT:    [[SRC:%.*]] = alloca i8, i64 [[I]], align 4
 ; CHECK-NEXT:    [[DEST:%.*]] = alloca i8, i64 [[I]], align 4
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 -1, ptr nocapture [[SRC]])
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 -1, ptr nocapture [[DEST]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 -1, ptr captures(none) [[SRC]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 -1, ptr captures(none) [[DEST]])
 ; CHECK-NEXT:    store [[STRUCT_FOO:%.*]] { i32 10, i32 20, i32 30 }, ptr [[SRC]], align 4
-; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @use_nocapture(ptr nocapture [[SRC]])
+; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @use_nocapture(ptr captures(none) [[SRC]])
 ; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 4 [[DEST]], ptr align 4 [[SRC]], i64 12, i1 false)
-; CHECK-NEXT:    [[TMP2:%.*]] = call i32 @use_nocapture(ptr nocapture [[DEST]])
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 -1, ptr nocapture [[SRC]])
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 -1, ptr nocapture [[DEST]])
+; CHECK-NEXT:    [[TMP2:%.*]] = call i32 @use_nocapture(ptr captures(none) [[DEST]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 -1, ptr captures(none) [[SRC]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 -1, ptr captures(none) [[DEST]])
 ; CHECK-NEXT:    ret void
 ;
   %src = alloca i8, i64 %i, align 4
@@ -1122,14 +1122,14 @@ define void @inalloca() {
 ; CHECK-LABEL: define void @inalloca() {
 ; CHECK-NEXT:    [[SRC:%.*]] = alloca [[STRUCT_FOO:%.*]], align 4
 ; CHECK-NEXT:    [[DEST:%.*]] = alloca inalloca [[STRUCT_FOO]], align 4
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 12, ptr nocapture [[SRC]])
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 12, ptr nocapture [[DEST]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 12, ptr captures(none) [[SRC]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 12, ptr captures(none) [[DEST]])
 ; CHECK-NEXT:    store [[STRUCT_FOO]] { i32 10, i32 20, i32 30 }, ptr [[SRC]], align 4
-; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @use_nocapture(ptr nocapture [[SRC]])
+; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @use_nocapture(ptr captures(none) [[SRC]])
 ; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 4 [[DEST]], ptr align 4 [[SRC]], i64 12, i1 false)
-; CHECK-NEXT:    [[TMP2:%.*]] = call i32 @use_nocapture(ptr nocapture [[DEST]])
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 12, ptr nocapture [[SRC]])
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 12, ptr nocapture [[DEST]])
+; CHECK-NEXT:    [[TMP2:%.*]] = call i32 @use_nocapture(ptr captures(none) [[DEST]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 12, ptr captures(none) [[SRC]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 12, ptr captures(none) [[DEST]])
 ; CHECK-NEXT:    ret void
 ;
   %src = alloca %struct.Foo, align 4
@@ -1153,14 +1153,14 @@ define void @dynamically_sized_memcpy(i64 %size) {
 ; CHECK-SAME: (i64 [[SIZE:%.*]]) {
 ; CHECK-NEXT:    [[SRC:%.*]] = alloca [[STRUCT_FOO:%.*]], align 4
 ; CHECK-NEXT:    [[DEST:%.*]] = alloca [[STRUCT_FOO]], align 4
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 12, ptr nocapture [[SRC]])
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 12, ptr nocapture [[DEST]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 12, ptr captures(none) [[SRC]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 12, ptr captures(none) [[DEST]])
 ; CHECK-NEXT:    store [[STRUCT_FOO]] { i32 10, i32 20, i32 30 }, ptr [[SRC]], align 4
-; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @use_nocapture(ptr nocapture [[SRC]])
+; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @use_nocapture(ptr captures(none) [[SRC]])
 ; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 4 [[DEST]], ptr align 4 [[SRC]], i64 [[SIZE]], i1 false)
-; CHECK-NEXT:    [[TMP2:%.*]] = call i32 @use_nocapture(ptr nocapture [[DEST]])
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 12, ptr nocapture [[DEST]])
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 12, ptr nocapture [[SRC]])
+; CHECK-NEXT:    [[TMP2:%.*]] = call i32 @use_nocapture(ptr captures(none) [[DEST]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 12, ptr captures(none) [[DEST]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 12, ptr captures(none) [[SRC]])
 ; CHECK-NEXT:    ret void
 ;
   %src = alloca %struct.Foo, align 4
@@ -1183,14 +1183,14 @@ define void @mismatched_alloca_size() {
 ; CHECK-LABEL: define void @mismatched_alloca_size() {
 ; CHECK-NEXT:    [[SRC:%.*]] = alloca i8, i64 24, align 4
 ; CHECK-NEXT:    [[DEST:%.*]] = alloca i8, i64 12, align 4
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 24, ptr nocapture [[SRC]])
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 12, ptr nocapture [[DEST]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 24, ptr captures(none) [[SRC]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 12, ptr captures(none) [[DEST]])
 ; CHECK-NEXT:    store [[STRUCT_FOO:%.*]] { i32 10, i32 20, i32 30 }, ptr [[SRC]], align 4
-; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @use_nocapture(ptr nocapture [[SRC]])
+; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @use_nocapture(ptr captures(none) [[SRC]])
 ; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 4 [[DEST]], ptr align 4 [[SRC]], i64 12, i1 false)
-; CHECK-NEXT:    [[TMP2:%.*]] = call i32 @use_nocapture(ptr nocapture [[DEST]])
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 24, ptr nocapture [[SRC]])
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 12, ptr nocapture [[DEST]])
+; CHECK-NEXT:    [[TMP2:%.*]] = call i32 @use_nocapture(ptr captures(none) [[DEST]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 24, ptr captures(none) [[SRC]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 12, ptr captures(none) [[DEST]])
 ; CHECK-NEXT:    ret void
 ;
   %src = alloca i8, i64 24, align 4
@@ -1213,14 +1213,14 @@ define void @mismatched_alloca_addrspace() {
 ; CHECK-LABEL: define void @mismatched_alloca_addrspace() {
 ; CHECK-NEXT:    [[SRC:%.*]] = alloca i8, i64 24, align 4, addrspace(1)
 ; CHECK-NEXT:    [[DEST:%.*]] = alloca i8, i64 12, align 4, addrspace(2)
-; CHECK-NEXT:    call void @llvm.lifetime.start.p1(i64 24, ptr addrspace(1) nocapture [[SRC]])
-; CHECK-NEXT:    call void @llvm.lifetime.start.p2(i64 12, ptr addrspace(2) nocapture [[DEST]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p1(i64 24, ptr addrspace(1) captures(none) [[SRC]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p2(i64 12, ptr addrspace(2) captures(none) [[DEST]])
 ; CHECK-NEXT:    store [[STRUCT_FOO:%.*]] { i32 10, i32 20, i32 30 }, ptr addrspace(1) [[SRC]], align 4
-; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @use_nocapture(ptr addrspace(1) nocapture [[SRC]])
+; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @use_nocapture(ptr addrspace(1) captures(none) [[SRC]])
 ; CHECK-NEXT:    call void @llvm.memcpy.p2.p1.i64(ptr addrspace(2) align 4 [[DEST]], ptr addrspace(1) align 4 [[SRC]], i64 12, i1 false)
-; CHECK-NEXT:    call void @llvm.lifetime.end.p1(i64 24, ptr addrspace(1) nocapture [[SRC]])
-; CHECK-NEXT:    [[TMP2:%.*]] = call i32 @use_nocapture(ptr addrspace(2) nocapture [[DEST]])
-; CHECK-NEXT:    call void @llvm.lifetime.end.p2(i64 12, ptr addrspace(2) nocapture [[DEST]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p1(i64 24, ptr addrspace(1) captures(none) [[SRC]])
+; CHECK-NEXT:    [[TMP2:%.*]] = call i32 @use_nocapture(ptr addrspace(2) captures(none) [[DEST]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p2(i64 12, ptr addrspace(2) captures(none) [[DEST]])
 ; CHECK-NEXT:    ret void
 ;
   %src = alloca i8, i64 24, align 4, addrspace(1)
@@ -1243,14 +1243,14 @@ define void @volatile_memcpy() {
 ; CHECK-LABEL: define void @volatile_memcpy() {
 ; CHECK-NEXT:    [[SRC:%.*]] = alloca [[STRUCT_FOO:%.*]], align 4
 ; CHECK-NEXT:    [[DEST:%.*]] = alloca [[STRUCT_FOO]], align 4
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 12, ptr nocapture [[SRC]])
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 12, ptr nocapture [[DEST]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 12, ptr captures(none) [[SRC]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 12, ptr captures(none) [[DEST]])
 ; CHECK-NEXT:    store [[STRUCT_FOO]] { i32 10, i32 20, i32 30 }, ptr [[SRC]], align 4
-; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @use_nocapture(ptr nocapture [[SRC]])
+; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @use_nocapture(ptr captures(none) [[SRC]])
 ; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 4 [[DEST]], ptr align 4 [[SRC]], i64 12, i1 true)
-; CHECK-NEXT:    [[TMP2:%.*]] = call i32 @use_nocapture(ptr nocapture [[DEST]])
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 12, ptr nocapture [[SRC]])
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 12, ptr nocapture [[DEST]])
+; CHECK-NEXT:    [[TMP2:%.*]] = call i32 @use_nocapture(ptr captures(none) [[DEST]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 12, ptr captures(none) [[SRC]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 12, ptr captures(none) [[DEST]])
 ; CHECK-NEXT:    ret void
 ;
   %src = alloca %struct.Foo, align 4
@@ -1273,14 +1273,14 @@ define void @dest_captured() {
 ; CHECK-LABEL: define void @dest_captured() {
 ; CHECK-NEXT:    [[SRC:%.*]] = alloca [[STRUCT_FOO:%.*]], align 4
 ; CHECK-NEXT:    [[DEST:%.*]] = alloca [[STRUCT_FOO]], align 4
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 12, ptr nocapture [[SRC]])
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 12, ptr nocapture [[DEST]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 12, ptr captures(none) [[SRC]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 12, ptr captures(none) [[DEST]])
 ; CHECK-NEXT:    store [[STRUCT_FOO]] { i32 10, i32 20, i32 30 }, ptr [[SRC]], align 4
-; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @use_nocapture(ptr nocapture [[SRC]])
+; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @use_nocapture(ptr captures(none) [[SRC]])
 ; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 4 [[DEST]], ptr align 4 [[SRC]], i64 12, i1 false)
 ; CHECK-NEXT:    [[TMP2:%.*]] = call i32 @use_maycapture(ptr [[DEST]])
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 12, ptr nocapture [[SRC]])
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 12, ptr nocapture [[DEST]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 12, ptr captures(none) [[SRC]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 12, ptr captures(none) [[DEST]])
 ; CHECK-NEXT:    ret void
 ;
   %src = alloca %struct.Foo, align 4
@@ -1303,14 +1303,14 @@ define void @src_captured() {
 ; CHECK-LABEL: define void @src_captured() {
 ; CHECK-NEXT:    [[SRC:%.*]] = alloca [[STRUCT_FOO:%.*]], align 4
 ; CHECK-NEXT:    [[DEST:%.*]] = alloca [[STRUCT_FOO]], align 4
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 12, ptr nocapture [[SRC]])
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 12, ptr nocapture [[DEST]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 12, ptr captures(none) [[SRC]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 12, ptr captures(none) [[DEST]])
 ; CHECK-NEXT:    store [[STRUCT_FOO]] { i32 10, i32 20, i32 30 }, ptr [[SRC]], align 4
 ; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @use_maycapture(ptr [[SRC]])
 ; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 4 [[DEST]], ptr align 4 [[SRC]], i64 12, i1 false)
-; CHECK-NEXT:    [[TMP2:%.*]] = call i32 @use_nocapture(ptr nocapture [[DEST]])
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 12, ptr nocapture [[SRC]])
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 12, ptr nocapture [[DEST]])
+; CHECK-NEXT:    [[TMP2:%.*]] = call i32 @use_nocapture(ptr captures(none) [[DEST]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 12, ptr captures(none) [[SRC]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 12, ptr captures(none) [[DEST]])
 ; CHECK-NEXT:    ret void
 ;
   %src = alloca %struct.Foo, align 4
@@ -1334,15 +1334,15 @@ define void @mod_ref_before_copy() {
 ; CHECK-LABEL: define void @mod_ref_before_copy() {
 ; CHECK-NEXT:    [[SRC:%.*]] = alloca [[STRUCT_FOO:%.*]], align 4
 ; CHECK-NEXT:    [[DEST:%.*]] = alloca [[STRUCT_FOO]], align 4
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 12, ptr nocapture [[SRC]])
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 12, ptr nocapture [[DEST]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 12, ptr captures(none) [[SRC]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 12, ptr captures(none) [[DEST]])
 ; CHECK-NEXT:    store [[STRUCT_FOO]] { i32 10, i32 20, i32 30 }, ptr [[SRC]], align 4
-; CHECK-NEXT:    [[R:%.*]] = call i32 @use_readonly(ptr nocapture [[DEST]])
-; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @use_nocapture(ptr nocapture [[SRC]])
+; CHECK-NEXT:    [[R:%.*]] = call i32 @use_readonly(ptr captures(none) [[DEST]])
+; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @use_nocapture(ptr captures(none) [[SRC]])
 ; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 4 [[DEST]], ptr align 4 [[SRC]], i64 12, i1 false)
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 12, ptr nocapture [[SRC]])
-; CHECK-NEXT:    [[TMP2:%.*]] = call i32 @use_nocapture(ptr nocapture [[DEST]])
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 12, ptr nocapture [[DEST]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 12, ptr captures(none) [[SRC]])
+; CHECK-NEXT:    [[TMP2:%.*]] = call i32 @use_nocapture(ptr captures(none) [[DEST]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 12, ptr captures(none) [[DEST]])
 ; CHECK-NEXT:    ret void
 ;
   %src = alloca %struct.Foo, align 4
@@ -1366,15 +1366,15 @@ define void @mod_dest_before_copy() {
 ; CHECK-LABEL: define void @mod_dest_before_copy() {
 ; CHECK-NEXT:    [[SRC:%.*]] = alloca [[STRUCT_FOO:%.*]], align 4
 ; CHECK-NEXT:    [[DEST:%.*]] = alloca [[STRUCT_FOO]], align 4
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 12, ptr nocapture [[SRC]])
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 12, ptr nocapture [[DEST]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 12, ptr captures(none) [[SRC]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 12, ptr captures(none) [[DEST]])
 ; CHECK-NEXT:    store [[STRUCT_FOO]] { i32 10, i32 20, i32 30 }, ptr [[SRC]], align 4
 ; CHECK-NEXT:    store i32 13, ptr [[DEST]], align 4
-; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @use_nocapture(ptr nocapture [[SRC]])
+; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @use_nocapture(ptr captures(none) [[SRC]])
 ; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 4 [[DEST]], ptr align 4 [[SRC]], i64 12, i1 false)
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 12, ptr nocapture [[SRC]])
-; CHECK-NEXT:    [[TMP2:%.*]] = call i32 @use_nocapture(ptr nocapture [[DEST]])
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 12, ptr nocapture [[DEST]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 12, ptr captures(none) [[SRC]])
+; CHECK-NEXT:    [[TMP2:%.*]] = call i32 @use_nocapture(ptr captures(none) [[DEST]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 12, ptr captures(none) [[DEST]])
 ; CHECK-NEXT:    ret void
 ;
   %src = alloca %struct.Foo, align 4
@@ -1397,16 +1397,16 @@ define void @mod_src_before_store_after_load() {
 ; CHECK-LABEL: define void @mod_src_before_store_after_load() {
 ; CHECK-NEXT:    [[SRC:%.*]] = alloca [[STRUCT_FOO:%.*]], align 4
 ; CHECK-NEXT:    [[DEST:%.*]] = alloca [[STRUCT_FOO]], align 4
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 12, ptr nocapture [[SRC]])
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 12, ptr nocapture [[DEST]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 12, ptr captures(none) [[SRC]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 12, ptr captures(none) [[DEST]])
 ; CHECK-NEXT:    store [[STRUCT_FOO]] { i32 10, i32 20, i32 30 }, ptr [[SRC]], align 4
 ; CHECK-NEXT:    store i32 13, ptr [[DEST]], align 4
-; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @use_nocapture(ptr nocapture [[SRC]])
+; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @use_nocapture(ptr captures(none) [[SRC]])
 ; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 4 [[DEST]], ptr align 4 [[SRC]], i64 12, i1 false)
 ; CHECK-NEXT:    store [[STRUCT_FOO]] { i32 13, i32 13, i32 13 }, ptr [[SRC]], align 4
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 12, ptr nocapture [[SRC]])
-; CHECK-NEXT:    [[TMP2:%.*]] = call i32 @use_nocapture(ptr nocapture [[DEST]])
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 12, ptr nocapture [[DEST]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 12, ptr captures(none) [[SRC]])
+; CHECK-NEXT:    [[TMP2:%.*]] = call i32 @use_nocapture(ptr captures(none) [[DEST]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 12, ptr captures(none) [[DEST]])
 ; CHECK-NEXT:    ret void
 ;
   %src = alloca %struct.Foo, align 4
@@ -1433,14 +1433,14 @@ define void @src_mod_dest_ref_after_copy() {
 ; CHECK-LABEL: define void @src_mod_dest_ref_after_copy() {
 ; CHECK-NEXT:    [[SRC:%.*]] = alloca [[STRUCT_FOO:%.*]], align 4
 ; CHECK-NEXT:    [[DEST:%.*]] = alloca [[STRUCT_FOO]], align 4
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 12, ptr nocapture [[SRC]])
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 12, ptr nocapture [[DEST]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 12, ptr captures(none) [[SRC]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 12, ptr captures(none) [[DEST]])
 ; CHECK-NEXT:    store [[STRUCT_FOO]] { i32 10, i32 20, i32 30 }, ptr [[SRC]], align 4
 ; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 4 [[DEST]], ptr align 4 [[SRC]], i64 12, i1 false)
 ; CHECK-NEXT:    store [[STRUCT_FOO]] { i32 13, i32 13, i32 13 }, ptr [[SRC]], align 4
-; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @use_nocapture(ptr nocapture [[DEST]])
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 12, ptr nocapture [[SRC]])
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 12, ptr nocapture [[DEST]])
+; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @use_nocapture(ptr captures(none) [[DEST]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 12, ptr captures(none) [[SRC]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 12, ptr captures(none) [[DEST]])
 ; CHECK-NEXT:    ret void
 ;
   %src = alloca %struct.Foo, align 4
@@ -1464,14 +1464,14 @@ define void @src_ref_dest_mod_after_copy() {
 ; CHECK-LABEL: define void @src_ref_dest_mod_after_copy() {
 ; CHECK-NEXT:    [[SRC:%.*]] = alloca [[STRUCT_FOO:%.*]], align 4
 ; CHECK-NEXT:    [[DEST:%.*]] = alloca [[STRUCT_FOO]], align 4
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 12, ptr nocapture [[SRC]])
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 12, ptr nocapture [[DEST]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 12, ptr captures(none) [[SRC]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 12, ptr captures(none) [[DEST]])
 ; CHECK-NEXT:    store [[STRUCT_FOO]] { i32 10, i32 20, i32 30 }, ptr [[SRC]], align 4
 ; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 4 [[DEST]], ptr align 4 [[SRC]], i64 12, i1 false)
 ; CHECK-NEXT:    store [[STRUCT_FOO]] { i32 13, i32 13, i32 13 }, ptr [[DEST]], align 4
-; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @use_nocapture(ptr nocapture [[SRC]])
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 12, ptr nocapture [[SRC]])
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 12, ptr nocapture [[DEST]])
+; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @use_nocapture(ptr captures(none) [[SRC]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 12, ptr captures(none) [[SRC]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 12, ptr captures(none) [[DEST]])
 ; CHECK-NEXT:    ret void
 ;
   %src = alloca %struct.Foo, align 4
@@ -1494,16 +1494,16 @@ define void @dest_alias_mod_before_copy() {
 ; CHECK-LABEL: define void @dest_alias_mod_before_copy() {
 ; CHECK-NEXT:    [[SRC:%.*]] = alloca [[STRUCT_FOO:%.*]], align 4
 ; CHECK-NEXT:    [[DEST:%.*]] = alloca [[STRUCT_FOO]], align 4
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 12, ptr nocapture [[SRC]])
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 12, ptr nocapture [[DEST]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 12, ptr captures(none) [[SRC]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 12, ptr captures(none) [[DEST]])
 ; CHECK-NEXT:    store [[STRUCT_FOO]] { i32 10, i32 20, i32 30 }, ptr [[SRC]], align 4
 ; CHECK-NEXT:    [[DEST_ALIAS:%.*]] = getelementptr inbounds [[STRUCT_FOO]], ptr [[DEST]], i64 0, i32 1
 ; CHECK-NEXT:    store i32 13, ptr [[DEST_ALIAS]], align 4
-; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @use_nocapture(ptr nocapture [[SRC]])
+; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @use_nocapture(ptr captures(none) [[SRC]])
 ; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 4 [[DEST]], ptr align 4 [[SRC]], i64 12, i1 false)
-; CHECK-NEXT:    [[TMP2:%.*]] = call i32 @use_nocapture(ptr nocapture [[DEST]])
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 12, ptr nocapture [[SRC]])
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 12, ptr nocapture [[DEST]])
+; CHECK-NEXT:    [[TMP2:%.*]] = call i32 @use_nocapture(ptr captures(none) [[DEST]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 12, ptr captures(none) [[SRC]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 12, ptr captures(none) [[DEST]])
 ; CHECK-NEXT:    ret void
 ;
   %src = alloca %struct.Foo, align 4
@@ -1528,16 +1528,16 @@ define void @alias_src_ref_dest_mod_after_copy() {
 ; CHECK-LABEL: define void @alias_src_ref_dest_mod_after_copy() {
 ; CHECK-NEXT:    [[SRC:%.*]] = alloca [[STRUCT_FOO:%.*]], align 4
 ; CHECK-NEXT:    [[DEST:%.*]] = alloca [[STRUCT_FOO]], align 4
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 12, ptr nocapture [[SRC]])
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 12, ptr nocapture [[DEST]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 12, ptr captures(none) [[SRC]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 12, ptr captures(none) [[DEST]])
 ; CHECK-NEXT:    store [[STRUCT_FOO]] { i32 10, i32 20, i32 30 }, ptr [[SRC]], align 4
-; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @use_nocapture(ptr nocapture [[SRC]])
+; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @use_nocapture(ptr captures(none) [[SRC]])
 ; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 4 [[DEST]], ptr align 4 [[SRC]], i64 12, i1 false)
 ; CHECK-NEXT:    [[DEST_ALIAS:%.*]] = getelementptr inbounds [[STRUCT_FOO]], ptr [[DEST]], i64 0, i32 1
 ; CHECK-NEXT:    store i32 13, ptr [[DEST_ALIAS]], align 4
-; CHECK-NEXT:    [[TMP2:%.*]] = call i32 @use_nocapture(ptr nocapture [[SRC]])
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 12, ptr nocapture [[SRC]])
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 12, ptr nocapture [[DEST]])
+; CHECK-NEXT:    [[TMP2:%.*]] = call i32 @use_nocapture(ptr captures(none) [[SRC]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 12, ptr captures(none) [[SRC]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 12, ptr captures(none) [[DEST]])
 ; CHECK-NEXT:    ret void
 ;
   %src = alloca %struct.Foo, align 4
@@ -1564,22 +1564,22 @@ define void @multi_bb_dataflow_conflict(i1 %b) {
 ; CHECK-SAME: (i1 [[B:%.*]]) {
 ; CHECK-NEXT:    [[SRC:%.*]] = alloca [[STRUCT_FOO:%.*]], align 4
 ; CHECK-NEXT:    [[DEST:%.*]] = alloca [[STRUCT_FOO]], align 4
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 12, ptr nocapture [[SRC]])
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 12, ptr nocapture [[DEST]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 12, ptr captures(none) [[SRC]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 12, ptr captures(none) [[DEST]])
 ; CHECK-NEXT:    store [[STRUCT_FOO]] { i32 10, i32 20, i32 30 }, ptr [[SRC]], align 4
-; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @use_nocapture(ptr nocapture noundef [[SRC]])
+; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @use_nocapture(ptr noundef captures(none) [[SRC]])
 ; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 4 [[DEST]], ptr align 4 [[SRC]], i64 12, i1 false)
 ; CHECK-NEXT:    br i1 [[B]], label [[BB0:%.*]], label [[BB1:%.*]]
 ; CHECK:       bb0:
-; CHECK-NEXT:    [[TMP2:%.*]] = call i32 @use_nocapture(ptr nocapture noundef [[SRC]])
+; CHECK-NEXT:    [[TMP2:%.*]] = call i32 @use_nocapture(ptr noundef captures(none) [[SRC]])
 ; CHECK-NEXT:    br label [[BB2:%.*]]
 ; CHECK:       bb1:
-; CHECK-NEXT:    [[TMP3:%.*]] = call i32 @use_nocapture(ptr nocapture noundef [[DEST]])
+; CHECK-NEXT:    [[TMP3:%.*]] = call i32 @use_nocapture(ptr noundef captures(none) [[DEST]])
 ; CHECK-NEXT:    br label [[BB2]]
 ; CHECK:       bb2:
-; CHECK-NEXT:    [[TMP4:%.*]] = call i32 @use_nocapture(ptr nocapture noundef [[DEST]])
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 12, ptr nocapture [[SRC]])
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 12, ptr nocapture [[DEST]])
+; CHECK-NEXT:    [[TMP4:%.*]] = call i32 @use_nocapture(ptr noundef captures(none) [[DEST]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 12, ptr captures(none) [[SRC]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 12, ptr captures(none) [[DEST]])
 ; CHECK-NEXT:    ret void
 ;
   %src = alloca %struct.Foo, align 4
@@ -1614,8 +1614,8 @@ define void @multi_bb_loop_dest_mod_before_copy(i32 %n) {
 ; CHECK-NEXT:    [[NLT1:%.*]] = icmp slt i32 [[N]], 1
 ; CHECK-NEXT:    [[SRC:%.*]] = alloca [[STRUCT_FOO:%.*]], align 8
 ; CHECK-NEXT:    [[DEST:%.*]] = alloca [[STRUCT_FOO]], align 8
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 12, ptr nocapture [[SRC]])
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 12, ptr nocapture [[DEST]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 12, ptr captures(none) [[SRC]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 12, ptr captures(none) [[DEST]])
 ; CHECK-NEXT:    store [[STRUCT_FOO]] { i32 0, i32 1, i32 42 }, ptr [[SRC]], align 4
 ; CHECK-NEXT:    br i1 [[NLT1]], label [[LOOP_EXIT:%.*]], label [[LOOP_BODY:%.*]]
 ; CHECK:       loop_body:
@@ -1654,15 +1654,15 @@ define void @partial_lifetime() {
 ; CHECK-LABEL: define void @partial_lifetime() {
 ; CHECK-NEXT:    [[SRC:%.*]] = alloca [[STRUCT_FOO:%.*]], align 4
 ; CHECK-NEXT:    [[DEST:%.*]] = alloca [[STRUCT_FOO]], align 4
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 12, ptr nocapture [[SRC]])
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 3, ptr nocapture [[DEST]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 12, ptr captures(none) [[SRC]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 3, ptr captures(none) [[DEST]])
 ; CHECK-NEXT:    store [[STRUCT_FOO]] { i32 10, i32 20, i32 30 }, ptr [[SRC]], align 4
-; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @use_nocapture(ptr nocapture [[SRC]])
+; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @use_nocapture(ptr captures(none) [[SRC]])
 ; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 4 [[DEST]], ptr align 4 [[SRC]], i64 12, i1 false)
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 3, ptr nocapture [[SRC]])
-; CHECK-NEXT:    [[TMP2:%.*]] = call i32 @use_nocapture(ptr nocapture [[DEST]])
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 12, ptr nocapture [[SRC]])
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 12, ptr nocapture [[DEST]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 3, ptr captures(none) [[SRC]])
+; CHECK-NEXT:    [[TMP2:%.*]] = call i32 @use_nocapture(ptr captures(none) [[DEST]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 12, ptr captures(none) [[SRC]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 12, ptr captures(none) [[DEST]])
 ; CHECK-NEXT:    ret void
 ;
   %src = alloca %struct.Foo, align 4
@@ -1690,10 +1690,10 @@ define void @crash_store63851(i1 %b) {
 ; CHECK-NEXT:    store i32 0, ptr [[DEST]], align 4
 ; CHECK-NEXT:    br i1 [[B]], label [[THEN:%.*]], label [[ELSE:%.*]]
 ; CHECK:       then:
-; CHECK-NEXT:    [[T:%.*]] = call i32 @use_nocapture(ptr nocapture noundef [[SRC]])
+; CHECK-NEXT:    [[T:%.*]] = call i32 @use_nocapture(ptr noundef captures(none) [[SRC]])
 ; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr [[DEST]], ptr [[SRC]], i64 12, i1 false)
-; CHECK-NEXT:    [[T3:%.*]] = call i32 @use_nocapture(ptr nocapture noundef [[SRC]])
-; CHECK-NEXT:    [[T4:%.*]] = call i32 @use_nocapture(ptr nocapture noundef [[DEST]])
+; CHECK-NEXT:    [[T3:%.*]] = call i32 @use_nocapture(ptr noundef captures(none) [[SRC]])
+; CHECK-NEXT:    [[T4:%.*]] = call i32 @use_nocapture(ptr noundef captures(none) [[DEST]])
 ; CHECK-NEXT:    br label [[ELSE]]
 ; CHECK:       else:
 ; CHECK-NEXT:    ret void
diff --git a/llvm/test/Transforms/MemProfContextDisambiguation/recursive.ll b/llvm/test/Transforms/MemProfContextDisambiguation/recursive.ll
index fd87cb535c42e..1a8020540f3a8 100644
--- a/llvm/test/Transforms/MemProfContextDisambiguation/recursive.ll
+++ b/llvm/test/Transforms/MemProfContextDisambiguation/recursive.ll
@@ -57,15 +57,13 @@
 ; RUN:	--implicit-check-not="marked with memprof allocation attribute cold" \
 ; RUN:  --check-prefix=ALL
 
-;; Check the default behavior (disabled recursive callsites).
+;; Check the default behavior (enabled recursive callsites).
 ; RUN: opt -passes=memprof-context-disambiguation -supports-hot-cold-new \
 ; RUN:  -memprof-verify-ccg -memprof-verify-nodes \
 ; RUN:  -pass-remarks=memprof-context-disambiguation \
 ; RUN:  %s -S 2>&1 | FileCheck %s \
 ; RUN:  --implicit-check-not "memprof_recursive3.cc:12:10: call in clone _Z1Ci.memprof.1 assigned" \
-; RUN:  --implicit-check-not="created clone" \
-; RUN:	--implicit-check-not="marked with memprof allocation attribute cold" \
-; RUN:  --check-prefix=ALL
+; RUN:  --check-prefix=ALL --check-prefix=ALLOW-RECUR-CALLSITES --check-prefix=ALLOW-RECUR-CONTEXTS
 
 ;; Skipping recursive contexts should prevent spurious call to cloned version of
 ;; B from the context starting at memprof_recursive.cc:19:13, which is actually
diff --git a/llvm/test/Transforms/MergeFunc/inttoptr-address-space.ll b/llvm/test/Transforms/MergeFunc/inttoptr-address-space.ll
index 49ba7b7feb6af..ff91f56919c7c 100644
--- a/llvm/test/Transforms/MergeFunc/inttoptr-address-space.ll
+++ b/llvm/test/Transforms/MergeFunc/inttoptr-address-space.ll
@@ -20,7 +20,7 @@ bb:
 define internal ptr @func35(ptr addrspace(1) nocapture %this) align 2 {
 bb:
 ; CHECK-LABEL: @func35(
-; CHECK: %[[V3:.+]] = tail call i32 @func10(ptr addrspace(1) nocapture %{{.*}})
+; CHECK: %[[V3:.+]] = tail call i32 @func10(ptr addrspace(1) captures(none) %{{.*}})
 ; CHECK: %{{.*}} = inttoptr i32 %[[V3]] to ptr
   %tmp = getelementptr inbounds %.qux.2585, ptr addrspace(1) %this, i32 0, i32 2
   %tmp1 = load ptr, ptr addrspace(1) %tmp, align 4
diff --git a/llvm/test/Transforms/MergeFunc/inttoptr.ll b/llvm/test/Transforms/MergeFunc/inttoptr.ll
index 6d6168b6c6988..56a070a30b9db 100644
--- a/llvm/test/Transforms/MergeFunc/inttoptr.ll
+++ b/llvm/test/Transforms/MergeFunc/inttoptr.ll
@@ -46,7 +46,7 @@ bb:
 define internal ptr @func35(ptr nocapture %this) align 2 {
 bb:
 ; CHECK-LABEL: @func35(
-; CHECK: %[[V3:.+]] = tail call i32 @func10(ptr nocapture %{{.*}})
+; CHECK: %[[V3:.+]] = tail call i32 @func10(ptr captures(none) %{{.*}})
 ; CHECK: %{{.*}} = inttoptr i32 %[[V3]] to ptr
   %tmp = getelementptr inbounds %.qux.2585, ptr %this, i32 0, i32 2
   %tmp1 = load ptr, ptr %tmp, align 4
diff --git a/llvm/test/Transforms/MergedLoadStoreMotion/st_sink_barrier_call.ll b/llvm/test/Transforms/MergedLoadStoreMotion/st_sink_barrier_call.ll
index c0723c4e82a3b..c08c6f7b9d07f 100644
--- a/llvm/test/Transforms/MergedLoadStoreMotion/st_sink_barrier_call.ll
+++ b/llvm/test/Transforms/MergedLoadStoreMotion/st_sink_barrier_call.ll
@@ -11,7 +11,7 @@ declare i32 @foo(i32 %x)
 
 define void @sink_store(ptr nocapture %r, i32 %index) {
 ; CHECK-LABEL: define void @sink_store
-; CHECK-SAME: (ptr nocapture [[R:%.*]], i32 [[INDEX:%.*]]) {
+; CHECK-SAME: (ptr captures(none) [[R:%.*]], i32 [[INDEX:%.*]]) {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[NODE_0_IN16:%.*]] = getelementptr inbounds [[STRUCT_NODE:%.*]], ptr [[R]], i64 0, i32 2
 ; CHECK-NEXT:    [[NODE_017:%.*]] = load ptr, ptr [[NODE_0_IN16]], align 8
diff --git a/llvm/test/Transforms/MergedLoadStoreMotion/st_sink_no_barrier_store.ll b/llvm/test/Transforms/MergedLoadStoreMotion/st_sink_no_barrier_store.ll
index 9a20011af0dcb..ff13255ec1589 100644
--- a/llvm/test/Transforms/MergedLoadStoreMotion/st_sink_no_barrier_store.ll
+++ b/llvm/test/Transforms/MergedLoadStoreMotion/st_sink_no_barrier_store.ll
@@ -9,7 +9,7 @@ target datalayout = "e-m:o-i64:64-i128:128-n32:64-S128"
 
 define void @sink_store(ptr nocapture %r, i32 %index) {
 ; CHECK-LABEL: define void @sink_store
-; CHECK-SAME: (ptr nocapture [[R:%.*]], i32 [[INDEX:%.*]]) {
+; CHECK-SAME: (ptr captures(none) [[R:%.*]], i32 [[INDEX:%.*]]) {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[NODE_0_IN16:%.*]] = getelementptr inbounds [[STRUCT_NODE:%.*]], ptr [[R]], i64 0, i32 2
 ; CHECK-NEXT:    [[NODE_017:%.*]] = load ptr, ptr [[NODE_0_IN16]], align 8
diff --git a/llvm/test/Transforms/MergedLoadStoreMotion/st_sink_with_barrier.ll b/llvm/test/Transforms/MergedLoadStoreMotion/st_sink_with_barrier.ll
index a741e5cd5aa93..e229c5243a2b2 100644
--- a/llvm/test/Transforms/MergedLoadStoreMotion/st_sink_with_barrier.ll
+++ b/llvm/test/Transforms/MergedLoadStoreMotion/st_sink_with_barrier.ll
@@ -8,7 +8,7 @@ target datalayout = "e-m:o-i64:64-i128:128-n32:64-S128"
 
 define void @sink_store(ptr nocapture %r, i32 %index) {
 ; CHECK-LABEL: define void @sink_store
-; CHECK-SAME: (ptr nocapture [[R:%.*]], i32 [[INDEX:%.*]]) {
+; CHECK-SAME: (ptr captures(none) [[R:%.*]], i32 [[INDEX:%.*]]) {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[NODE_0_IN16:%.*]] = getelementptr inbounds [[STRUCT_NODE:%.*]], ptr [[R]], i64 0, i32 2
 ; CHECK-NEXT:    [[NODE_017:%.*]] = load ptr, ptr [[NODE_0_IN16]], align 8
diff --git a/llvm/test/Transforms/NaryReassociate/nary-gep.ll b/llvm/test/Transforms/NaryReassociate/nary-gep.ll
index d0ece1e11de5a..b56fdbe4a0669 100644
--- a/llvm/test/Transforms/NaryReassociate/nary-gep.ll
+++ b/llvm/test/Transforms/NaryReassociate/nary-gep.ll
@@ -21,6 +21,17 @@ define void @no_sext_fat_pointer(ptr addrspace(2) %a, i32 %i, i32 %j) {
   ret void
 }
 
+define ptr addrspace(2) @zext_fat_pointer_crash(ptr addrspace(2) %p, i32 %a) {
+; CHECK-LABEL: @zext_fat_pointer_crash(
+; CHECK-NEXT:    [[C:%.*]] = add i32 [[A:%.*]], 1
+; CHECK-NEXT:    [[Q:%.*]] = getelementptr double, ptr addrspace(2) [[P:%.*]], i32 [[C]]
+; CHECK-NEXT:    ret ptr addrspace(2) [[Q]]
+;
+  %c = add i32 %a, 1
+  %q = getelementptr double, ptr addrspace(2) %p, i32 %c
+  ret ptr addrspace(2) %q
+}
+
 define void @or_disjoint(ptr addrspace(2) %a, i32 %i, i32 %j, i32 %k) {
 ; CHECK-LABEL: @or_disjoint(
 ; CHECK-NEXT:    [[OR:%.*]] = or disjoint i32 [[I:%.*]], [[J:%.*]]
diff --git a/llvm/test/Transforms/NewGVN/memory-handling.ll b/llvm/test/Transforms/NewGVN/memory-handling.ll
index c72ff749ba410..bf07edf91f2ba 100644
--- a/llvm/test/Transforms/NewGVN/memory-handling.ll
+++ b/llvm/test/Transforms/NewGVN/memory-handling.ll
@@ -22,7 +22,7 @@ declare ptr @__ctype_b_loc() local_unnamed_addr #1
 ; Function Attrs: nounwind uwtable
 define void @BuildMask(ptr nocapture readonly) local_unnamed_addr #0 {
 ; CHECK-LABEL: define void @BuildMask(
-; CHECK-SAME: ptr nocapture readonly [[TMP0:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
+; CHECK-SAME: ptr readonly captures(none) [[TMP0:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
 ; CHECK-NEXT:    tail call void @llvm.memset.p0.i64(ptr align 16 @alPhrase, i8 0, i64 416, i1 false)
 ; CHECK-NEXT:    tail call void @llvm.memset.p0.i64(ptr align 16 @aqMainMask, i8 0, i64 16, i1 false)
 ; CHECK-NEXT:    tail call void @llvm.memset.p0.i64(ptr align 16 @aqMainSign, i8 0, i64 16, i1 false)
@@ -35,17 +35,17 @@ define void @BuildMask(ptr nocapture readonly) local_unnamed_addr #0 {
 ; CHECK:       2:
 ; CHECK-NEXT:    [[DOT1:%.*]] = phi ptr [ [[DOT0]], [[DOTSINK_SPLIT]] ], [ [[TMP3]], [[TMP6:%.*]] ]
 ; CHECK-NEXT:    [[TMP3]] = getelementptr inbounds i8, ptr [[DOT1]], i64 1
-; CHECK-NEXT:    [[TMP4:%.*]] = load i8, ptr [[DOT1]], align 1
+; CHECK-NEXT:    [[TMP4:%.*]] = load i8, ptr [[DOT1]], align 1, !tbaa [[TBAA5:![0-9]+]]
 ; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq i8 [[TMP4]], 0
 ; CHECK-NEXT:    br i1 [[TMP5]], label [[DOTPREHEADER_PREHEADER:%.*]], label [[TMP6]]
 ; CHECK:       .preheader.preheader:
 ; CHECK-NEXT:    br label [[DOTPREHEADER:%.*]]
 ; CHECK:       6:
 ; CHECK-NEXT:    [[TMP7:%.*]] = tail call ptr @__ctype_b_loc() #[[ATTR4:[0-9]+]]
-; CHECK-NEXT:    [[TMP8:%.*]] = load ptr, ptr [[TMP7]], align 8, !tbaa [[TBAA5:![0-9]+]]
+; CHECK-NEXT:    [[TMP8:%.*]] = load ptr, ptr [[TMP7]], align 8, !tbaa [[TBAA6:![0-9]+]]
 ; CHECK-NEXT:    [[TMP9:%.*]] = sext i8 [[TMP4]] to i64
 ; CHECK-NEXT:    [[TMP10:%.*]] = getelementptr inbounds i16, ptr [[TMP8]], i64 [[TMP9]]
-; CHECK-NEXT:    [[TMP11:%.*]] = load i16, ptr [[TMP10]], align 2, !tbaa [[TBAA7:![0-9]+]]
+; CHECK-NEXT:    [[TMP11:%.*]] = load i16, ptr [[TMP10]], align 2, !tbaa [[TBAA8:![0-9]+]]
 ; CHECK-NEXT:    [[TMP12:%.*]] = and i16 [[TMP11]], 1024
 ; CHECK-NEXT:    [[TMP13:%.*]] = icmp eq i16 [[TMP12]], 0
 ; CHECK-NEXT:    br i1 [[TMP13]], label [[TMP2]], label [[TMP14]]
@@ -55,9 +55,9 @@ define void @BuildMask(ptr nocapture readonly) local_unnamed_addr #0 {
 ; CHECK-NEXT:    [[TMP17:%.*]] = add nsw i32 [[TMP16]], -97
 ; CHECK-NEXT:    [[TMP18:%.*]] = sext i32 [[TMP17]] to i64
 ; CHECK-NEXT:    [[TMP19:%.*]] = getelementptr inbounds [26 x %struct.Letter], ptr @alPhrase, i64 0, i64 [[TMP18]], i32 0
-; CHECK-NEXT:    [[TMP20:%.*]] = load i32, ptr [[TMP19]], align 16, !tbaa [[TBAA9:![0-9]+]]
+; CHECK-NEXT:    [[TMP20:%.*]] = load i32, ptr [[TMP19]], align 16, !tbaa [[TBAA10:![0-9]+]]
 ; CHECK-NEXT:    [[TMP21:%.*]] = add i32 [[TMP20]], 1
-; CHECK-NEXT:    store i32 [[TMP21]], ptr [[TMP19]], align 16, !tbaa [[TBAA9]]
+; CHECK-NEXT:    store i32 [[TMP21]], ptr [[TMP19]], align 16, !tbaa [[TBAA10]]
 ; CHECK-NEXT:    [[TMP22]] = add nsw i32 [[DOTSINK]], 1
 ; CHECK-NEXT:    br label [[DOTSINK_SPLIT]]
 ; CHECK:       .preheader:
@@ -65,7 +65,7 @@ define void @BuildMask(ptr nocapture readonly) local_unnamed_addr #0 {
 ; CHECK-NEXT:    [[DOT04961:%.*]] = phi i32 [ [[DOT2:%.*]], [[TMP57]] ], [ 0, [[DOTPREHEADER_PREHEADER]] ]
 ; CHECK-NEXT:    [[DOT05160:%.*]] = phi i32 [ [[DOT253:%.*]], [[TMP57]] ], [ 0, [[DOTPREHEADER_PREHEADER]] ]
 ; CHECK-NEXT:    [[TMP23:%.*]] = getelementptr inbounds [26 x %struct.Letter], ptr @alPhrase, i64 0, i64 [[INDVARS_IV]], i32 0
-; CHECK-NEXT:    [[TMP24:%.*]] = load i32, ptr [[TMP23]], align 16, !tbaa [[TBAA9]]
+; CHECK-NEXT:    [[TMP24:%.*]] = load i32, ptr [[TMP23]], align 16, !tbaa [[TBAA10]]
 ; CHECK-NEXT:    [[TMP25:%.*]] = icmp eq i32 [[TMP24]], 0
 ; CHECK-NEXT:    [[TMP26:%.*]] = getelementptr inbounds [26 x i32], ptr @auGlobalFrequency, i64 0, i64 [[INDVARS_IV]]
 ; CHECK-NEXT:    br i1 [[TMP25]], label [[TMP27:%.*]], label [[TMP28:%.*]]
@@ -106,25 +106,25 @@ define void @BuildMask(ptr nocapture readonly) local_unnamed_addr #0 {
 ; CHECK-NEXT:    [[TMP40:%.*]] = add i64 [[DOT046_LCSSA]], 4294967295
 ; CHECK-NEXT:    [[TMP41:%.*]] = trunc i64 [[TMP40]] to i32
 ; CHECK-NEXT:    [[TMP42:%.*]] = getelementptr inbounds [26 x %struct.Letter], ptr @alPhrase, i64 0, i64 [[INDVARS_IV]], i32 2
-; CHECK-NEXT:    store i32 [[TMP41]], ptr [[TMP42]], align 8, !tbaa [[TBAA11:![0-9]+]]
+; CHECK-NEXT:    store i32 [[TMP41]], ptr [[TMP42]], align 8, !tbaa [[TBAA12:![0-9]+]]
 ; CHECK-NEXT:    [[TMP43:%.*]] = zext i32 [[DOT150]] to i64
 ; CHECK-NEXT:    [[DOT046_:%.*]] = shl i64 [[DOT046_LCSSA]], [[TMP43]]
 ; CHECK-NEXT:    [[TMP44:%.*]] = zext i32 [[DOT152]] to i64
 ; CHECK-NEXT:    [[TMP45:%.*]] = getelementptr inbounds [2 x i64], ptr @aqMainSign, i64 0, i64 [[TMP44]]
-; CHECK-NEXT:    [[TMP46:%.*]] = load i64, ptr [[TMP45]], align 8, !tbaa [[TBAA12:![0-9]+]]
+; CHECK-NEXT:    [[TMP46:%.*]] = load i64, ptr [[TMP45]], align 8, !tbaa [[TBAA13:![0-9]+]]
 ; CHECK-NEXT:    [[TMP47:%.*]] = or i64 [[TMP46]], [[DOT046_]]
-; CHECK-NEXT:    store i64 [[TMP47]], ptr [[TMP45]], align 8, !tbaa [[TBAA12]]
-; CHECK-NEXT:    [[TMP48:%.*]] = load i32, ptr [[TMP23]], align 16, !tbaa [[TBAA9]]
+; CHECK-NEXT:    store i64 [[TMP47]], ptr [[TMP45]], align 8, !tbaa [[TBAA13]]
+; CHECK-NEXT:    [[TMP48:%.*]] = load i32, ptr [[TMP23]], align 16, !tbaa [[TBAA10]]
 ; CHECK-NEXT:    [[TMP49:%.*]] = zext i32 [[TMP48]] to i64
 ; CHECK-NEXT:    [[TMP50:%.*]] = shl i64 [[TMP49]], [[TMP43]]
 ; CHECK-NEXT:    [[TMP51:%.*]] = getelementptr inbounds [2 x i64], ptr @aqMainMask, i64 0, i64 [[TMP44]]
-; CHECK-NEXT:    [[TMP52:%.*]] = load i64, ptr [[TMP51]], align 8, !tbaa [[TBAA12]]
+; CHECK-NEXT:    [[TMP52:%.*]] = load i64, ptr [[TMP51]], align 8, !tbaa [[TBAA13]]
 ; CHECK-NEXT:    [[TMP53:%.*]] = or i64 [[TMP50]], [[TMP52]]
-; CHECK-NEXT:    store i64 [[TMP53]], ptr [[TMP51]], align 8, !tbaa [[TBAA12]]
+; CHECK-NEXT:    store i64 [[TMP53]], ptr [[TMP51]], align 8, !tbaa [[TBAA13]]
 ; CHECK-NEXT:    [[TMP54:%.*]] = getelementptr inbounds [26 x %struct.Letter], ptr @alPhrase, i64 0, i64 [[INDVARS_IV]], i32 1
-; CHECK-NEXT:    store i32 [[DOT150]], ptr [[TMP54]], align 4, !tbaa [[TBAA14:![0-9]+]]
+; CHECK-NEXT:    store i32 [[DOT150]], ptr [[TMP54]], align 4, !tbaa [[TBAA15:![0-9]+]]
 ; CHECK-NEXT:    [[TMP55:%.*]] = getelementptr inbounds [26 x %struct.Letter], ptr @alPhrase, i64 0, i64 [[INDVARS_IV]], i32 3
-; CHECK-NEXT:    store i32 [[DOT152]], ptr [[TMP55]], align 4, !tbaa [[TBAA15:![0-9]+]]
+; CHECK-NEXT:    store i32 [[DOT152]], ptr [[TMP55]], align 4, !tbaa [[TBAA16:![0-9]+]]
 ; CHECK-NEXT:    [[TMP56:%.*]] = add nsw i32 [[DOT150]], [[DOT048_LCSSA]]
 ; CHECK-NEXT:    br label [[TMP57]]
 ; CHECK:       57:
@@ -313,15 +313,16 @@ attributes #5 = { nounwind readonly }
 ; CHECK: [[META2]] = !{!"int", [[META3:![0-9]+]], i64 0}
 ; CHECK: [[META3]] = !{!"omnipotent char", [[META4:![0-9]+]], i64 0}
 ; CHECK: [[META4]] = !{!"Simple C/C++ TBAA"}
-; CHECK: [[TBAA5]] = !{[[META6:![0-9]+]], [[META6]], i64 0}
-; CHECK: [[META6]] = !{!"any pointer", [[META3]], i64 0}
-; CHECK: [[TBAA7]] = !{[[META8:![0-9]+]], [[META8]], i64 0}
-; CHECK: [[META8]] = !{!"short", [[META3]], i64 0}
-; CHECK: [[TBAA9]] = !{[[META10:![0-9]+]], [[META2]], i64 0}
-; CHECK: [[META10]] = !{!"", [[META2]], i64 0, [[META2]], i64 4, [[META2]], i64 8, [[META2]], i64 12}
-; CHECK: [[TBAA11]] = !{[[META10]], [[META2]], i64 8}
-; CHECK: [[TBAA12]] = !{[[META13:![0-9]+]], [[META13]], i64 0}
-; CHECK: [[META13]] = !{!"long", [[META3]], i64 0}
-; CHECK: [[TBAA14]] = !{[[META10]], [[META2]], i64 4}
-; CHECK: [[TBAA15]] = !{[[META10]], [[META2]], i64 12}
+; CHECK: [[TBAA5]] = !{[[META3]], [[META3]], i64 0}
+; CHECK: [[TBAA6]] = !{[[META7:![0-9]+]], [[META7]], i64 0}
+; CHECK: [[META7]] = !{!"any pointer", [[META3]], i64 0}
+; CHECK: [[TBAA8]] = !{[[META9:![0-9]+]], [[META9]], i64 0}
+; CHECK: [[META9]] = !{!"short", [[META3]], i64 0}
+; CHECK: [[TBAA10]] = !{[[META11:![0-9]+]], [[META2]], i64 0}
+; CHECK: [[META11]] = !{!"", [[META2]], i64 0, [[META2]], i64 4, [[META2]], i64 8, [[META2]], i64 12}
+; CHECK: [[TBAA12]] = !{[[META11]], [[META2]], i64 8}
+; CHECK: [[TBAA13]] = !{[[META14:![0-9]+]], [[META14]], i64 0}
+; CHECK: [[META14]] = !{!"long", [[META3]], i64 0}
+; CHECK: [[TBAA15]] = !{[[META11]], [[META2]], i64 4}
+; CHECK: [[TBAA16]] = !{[[META11]], [[META2]], i64 12}
 ;.
diff --git a/llvm/test/Transforms/NewGVN/pr25440.ll b/llvm/test/Transforms/NewGVN/pr25440.ll
index 2d74b2e06640e..2331469e50ad5 100644
--- a/llvm/test/Transforms/NewGVN/pr25440.ll
+++ b/llvm/test/Transforms/NewGVN/pr25440.ll
@@ -12,7 +12,7 @@ target triple = "thumbv7--linux-gnueabi"
 ; Function Attrs: nounwind
 define fastcc void @foo(ptr nocapture readonly %x, i1 %arg, i1 %arg2, i1 %arg3, i1 %arg4) {
 ; CHECK-LABEL: define fastcc void @foo(
-; CHECK-SAME: ptr nocapture readonly [[X:%.*]], i1 [[ARG:%.*]], i1 [[ARG2:%.*]], i1 [[ARG3:%.*]], i1 [[ARG4:%.*]]) {
+; CHECK-SAME: ptr readonly captures(none) [[X:%.*]], i1 [[ARG:%.*]], i1 [[ARG2:%.*]], i1 [[ARG3:%.*]], i1 [[ARG4:%.*]]) {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    br label [[BB0:%.*]]
 ; CHECK:       bb0:
diff --git a/llvm/test/Transforms/NewGVN/volatile-nonvolatile.ll b/llvm/test/Transforms/NewGVN/volatile-nonvolatile.ll
index d6daff99591f9..d8b28d73f24ee 100644
--- a/llvm/test/Transforms/NewGVN/volatile-nonvolatile.ll
+++ b/llvm/test/Transforms/NewGVN/volatile-nonvolatile.ll
@@ -7,7 +7,7 @@
 ; so the second reload is not necessary. Check that it can be eliminated.
 define void @test1(ptr nocapture readonly %p, i32 %v) #0 {
 ; CHECK-LABEL: define void @test1(
-; CHECK-SAME: ptr nocapture readonly [[P:%.*]], i32 [[V:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-SAME: ptr readonly captures(none) [[P:%.*]], i32 [[V:%.*]]) #[[ATTR0:[0-9]+]] {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[P]], align 4, !tbaa [[TBAA0:![0-9]+]]
 ; CHECK-NEXT:    store volatile i32 [[V]], ptr [[TMP0]], align 4, !tbaa [[TBAA5:![0-9]+]]
@@ -26,7 +26,7 @@ entry:
 ; Make sure that both loads remain.
 define void @test2(ptr nocapture readonly %p, i32 %v) #0 {
 ; CHECK-LABEL: define void @test2(
-; CHECK-SAME: ptr nocapture readonly [[P:%.*]], i32 [[V:%.*]]) #[[ATTR0]] {
+; CHECK-SAME: ptr readonly captures(none) [[P:%.*]], i32 [[V:%.*]]) #[[ATTR0]] {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[P]], align 4, !tbaa [[TBAA0]]
 ; CHECK-NEXT:    store volatile i32 [[V]], ptr [[TMP0]], align 4, !tbaa [[TBAA0]]
@@ -46,7 +46,7 @@ entry:
 ; the stores, make sure both are preserved.
 define void @test3(ptr nocapture readonly %p, i32 %v) #0 {
 ; CHECK-LABEL: define void @test3(
-; CHECK-SAME: ptr nocapture readonly [[P:%.*]], i32 [[V:%.*]]) #[[ATTR0]] {
+; CHECK-SAME: ptr readonly captures(none) [[P:%.*]], i32 [[V:%.*]]) #[[ATTR0]] {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[TMP0:%.*]] = load atomic ptr, ptr [[P]] acquire, align 4, !tbaa [[TBAA0]]
 ; CHECK-NEXT:    store volatile i32 [[V]], ptr [[TMP0]], align 4, !tbaa [[TBAA5]]
diff --git a/llvm/test/Transforms/NewGVN/vscale.ll b/llvm/test/Transforms/NewGVN/vscale.ll
index 500d58baed1a2..702117213f6eb 100644
--- a/llvm/test/Transforms/NewGVN/vscale.ll
+++ b/llvm/test/Transforms/NewGVN/vscale.ll
@@ -646,3 +646,131 @@ entry:
   call void @llvm.lifetime.end.p0(i64 -1, ptr nonnull %ref.tmp)
   ret { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %15
 }
+
+define <vscale x 4 x float> @scalable_store_to_fixed_load(<vscale x 4 x float> %.coerce) vscale_range(4,4) {
+; CHECK-LABEL: @scalable_store_to_fixed_load(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[RETVAL:%.*]] = alloca { <16 x float> }, align 64
+; CHECK-NEXT:    [[TMP0:%.*]] = fadd <vscale x 4 x float> [[DOTCOERCE:%.*]], [[DOTCOERCE]]
+; CHECK-NEXT:    store <vscale x 4 x float> [[TMP0]], ptr [[RETVAL]], align 16
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x float>, ptr [[RETVAL]], align 64
+; CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = tail call <vscale x 4 x float> @llvm.vector.insert.nxv4f32.v16f32(<vscale x 4 x float> poison, <16 x float> [[TMP1]], i64 0)
+; CHECK-NEXT:    ret <vscale x 4 x float> [[CAST_SCALABLE]]
+;
+entry:
+  %retval = alloca { <16 x float> }
+  %0 = fadd <vscale x 4 x float> %.coerce, %.coerce
+  store <vscale x 4 x float> %0, ptr %retval
+  %1 = load <16 x float>, ptr %retval
+  %cast.scalable = tail call <vscale x 4 x float> @llvm.vector.insert.nxv4f32.v16f32(<vscale x 4 x float> poison, <16 x float> %1, i64 0)
+  ret <vscale x 4 x float> %cast.scalable
+}
+
+; Here, only the lower bound for the vscale is known, but this is enough to allow a forward to a load to 16 elements.
+define <vscale x 4 x float> @scalable_store_to_fixed_load_only_lower_bound(<vscale x 4 x float> %a) vscale_range(4) {
+; CHECK-LABEL: @scalable_store_to_fixed_load_only_lower_bound(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[RETVAL:%.*]] = alloca { <vscale x 4 x float> }, align 16
+; CHECK-NEXT:    store <vscale x 4 x float> [[A:%.*]], ptr [[RETVAL]], align 16
+; CHECK-NEXT:    [[TMP0:%.*]] = load <16 x float>, ptr [[RETVAL]], align 64
+; CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = tail call <vscale x 4 x float> @llvm.vector.insert.nxv4f32.v16f32(<vscale x 4 x float> poison, <16 x float> [[TMP0]], i64 0)
+; CHECK-NEXT:    ret <vscale x 4 x float> [[CAST_SCALABLE]]
+;
+entry:
+  %retval = alloca { <vscale x 4 x float> }
+  store <vscale x 4 x float> %a, ptr %retval
+  %1 = load <16 x float>, ptr %retval
+  %cast.scalable = tail call <vscale x 4 x float> @llvm.vector.insert.nxv4f32.v16f32(<vscale x 4 x float> poison, <16 x float> %1, i64 0)
+  ret <vscale x 4 x float> %cast.scalable
+}
+
+define <vscale x 4 x float> @scalable_store_to_fixed_load_with_offset(<vscale x 4 x float> %a) vscale_range(4,4) {
+; CHECK-LABEL: @scalable_store_to_fixed_load_with_offset(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[PTR:%.*]] = alloca { <32 x float> }, align 128
+; CHECK-NEXT:    store <vscale x 4 x float> [[A:%.*]], ptr [[PTR]], align 16
+; CHECK-NEXT:    [[GEP:%.*]] = getelementptr inbounds i8, ptr [[PTR]], i64 8
+; CHECK-NEXT:    [[TMP0:%.*]] = load <16 x float>, ptr [[GEP]], align 64
+; CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = tail call <vscale x 4 x float> @llvm.vector.insert.nxv4f32.v16f32(<vscale x 4 x float> poison, <16 x float> [[TMP0]], i64 0)
+; CHECK-NEXT:    ret <vscale x 4 x float> [[CAST_SCALABLE]]
+;
+entry:
+  %ptr = alloca { <32 x float> }
+  store <vscale x 4 x float> %a, ptr %ptr
+  %gep = getelementptr inbounds i8, ptr %ptr, i64 8
+  %1 = load <16 x float>, ptr %gep
+  %cast.scalable = tail call <vscale x 4 x float> @llvm.vector.insert.nxv4f32.v16f32(<vscale x 4 x float> poison, <16 x float> %1, i64 0)
+  ret <vscale x 4 x float> %cast.scalable
+}
+
+define <vscale x 4 x float> @scalable_store_to_fixed_load_unknown_vscale(<vscale x 4 x float> %.coerce) {
+; CHECK-LABEL: @scalable_store_to_fixed_load_unknown_vscale(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[RETVAL:%.*]] = alloca { <16 x float> }, align 64
+; CHECK-NEXT:    [[TMP0:%.*]] = fadd <vscale x 4 x float> [[DOTCOERCE:%.*]], [[DOTCOERCE]]
+; CHECK-NEXT:    store <vscale x 4 x float> [[TMP0]], ptr [[RETVAL]], align 16
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x float>, ptr [[RETVAL]], align 64
+; CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = tail call <vscale x 4 x float> @llvm.vector.insert.nxv4f32.v16f32(<vscale x 4 x float> poison, <16 x float> [[TMP1]], i64 0)
+; CHECK-NEXT:    ret <vscale x 4 x float> [[CAST_SCALABLE]]
+;
+entry:
+  %retval = alloca { <16 x float> }
+  %0 = fadd <vscale x 4 x float> %.coerce, %.coerce
+  store <vscale x 4 x float> %0, ptr %retval
+  %1 = load <16 x float>, ptr %retval
+  %cast.scalable = tail call <vscale x 4 x float> @llvm.vector.insert.nxv4f32.v16f32(<vscale x 4 x float> poison, <16 x float> %1, i64 0)
+  ret <vscale x 4 x float> %cast.scalable
+}
+
+define <vscale x 4 x float> @scalable_store_to_fixed_load_size_missmatch(<vscale x 4 x float> %.coerce) vscale_range(4,4) {
+; CHECK-LABEL: @scalable_store_to_fixed_load_size_missmatch(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[RETVAL:%.*]] = alloca { <32 x float> }, align 128
+; CHECK-NEXT:    [[TMP0:%.*]] = fadd <vscale x 4 x float> [[DOTCOERCE:%.*]], [[DOTCOERCE]]
+; CHECK-NEXT:    store <vscale x 4 x float> [[TMP0]], ptr [[RETVAL]], align 16
+; CHECK-NEXT:    [[TMP1:%.*]] = load <32 x float>, ptr [[RETVAL]], align 128
+; CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = tail call <vscale x 4 x float> @llvm.vector.insert.nxv4f32.v32f32(<vscale x 4 x float> poison, <32 x float> [[TMP1]], i64 0)
+; CHECK-NEXT:    ret <vscale x 4 x float> [[CAST_SCALABLE]]
+;
+entry:
+  %retval = alloca { <32 x float> }
+  %0 = fadd <vscale x 4 x float> %.coerce, %.coerce
+  store <vscale x 4 x float> %0, ptr %retval
+  %1 = load <32 x float>, ptr %retval
+  %cast.scalable = tail call <vscale x 4 x float> @llvm.vector.insert.nxv4f32.v32f32(<vscale x 4 x float> poison, <32 x float> %1, i64 0)
+  ret <vscale x 4 x float> %cast.scalable
+}
+
+define <vscale x 4 x i32> @scalable_store_to_fixed_load_different_types(<vscale x 4 x float> %a) vscale_range(4,4) {
+; CHECK-LABEL: @scalable_store_to_fixed_load_different_types(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[PTR:%.*]] = alloca { <16 x float> }, align 64
+; CHECK-NEXT:    store <vscale x 4 x float> [[A:%.*]], ptr [[PTR]], align 16
+; CHECK-NEXT:    [[TMP0:%.*]] = load <16 x i32>, ptr [[PTR]], align 64
+; CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = tail call <vscale x 4 x i32> @llvm.vector.insert.nxv4i32.v16i32(<vscale x 4 x i32> poison, <16 x i32> [[TMP0]], i64 0)
+; CHECK-NEXT:    ret <vscale x 4 x i32> [[CAST_SCALABLE]]
+;
+entry:
+  %ptr = alloca { <16 x float> }
+  store <vscale x 4 x float> %a, ptr %ptr
+  %1 = load <16 x i32>, ptr %ptr
+  %cast.scalable = tail call <vscale x 4 x i32> @llvm.vector.insert.nxv4i32.v16i32(<vscale x 4 x i32> poison, <16 x i32> %1, i64 0)
+  ret <vscale x 4 x i32> %cast.scalable
+}
+
+; This function does not have a fixed vscale, but the loaded vector is still known
+; to be smaller or equal in size compared to the stored vector.
+define <4 x float> @scalable_store_to_small_fixed_load(<vscale x 4 x float> %a) {
+; CHECK-LABEL: @scalable_store_to_small_fixed_load(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[PTR:%.*]] = alloca <vscale x 4 x float>, align 16
+; CHECK-NEXT:    store <vscale x 4 x float> [[A:%.*]], ptr [[PTR]], align 16
+; CHECK-NEXT:    [[TMP0:%.*]] = load <4 x float>, ptr [[PTR]], align 16
+; CHECK-NEXT:    ret <4 x float> [[TMP0]]
+;
+entry:
+  %ptr = alloca <vscale x 4 x float>
+  store <vscale x 4 x float> %a, ptr %ptr
+  %1 = load <4 x float>, ptr %ptr
+  ret <4 x float> %1
+}
diff --git a/llvm/test/Transforms/ObjCARC/basic.ll b/llvm/test/Transforms/ObjCARC/basic.ll
index d461bc0af680d..aa0c53b4f4851 100644
--- a/llvm/test/Transforms/ObjCARC/basic.ll
+++ b/llvm/test/Transforms/ObjCARC/basic.ll
@@ -2607,17 +2607,17 @@ cond.end:                                         ; preds = %cond.true, %entry
 
 @.str4 = private unnamed_addr constant [33 x i8] c"-[A z] = { %f, %f, { %f, %f } }\0A\00"
 @"OBJC_IVAR_$_A.myZ" = global i64 20, section "__DATA, __objc_const", align 8
-declare i32 @printf(ptr nocapture, ...) nounwind
-declare i32 @puts(ptr nocapture) nounwind
+declare i32 @printf(ptr captures(none), ...) nounwind
+declare i32 @puts(ptr captures(none)) nounwind
 @str = internal constant [16 x i8] c"-[ Top0 _getX ]\00"
 
 ; FIXME: Should be able to eliminate the retain and release
-; CHECK-LABEL: define { <2 x float>, <2 x float> } @"\01-[A z]"(ptr %self, ptr nocapture %_cmd)
+; CHECK-LABEL: define { <2 x float>, <2 x float> } @"\01-[A z]"(ptr %self, ptr captures(none) %_cmd)
 ; CHECK: tail call ptr @llvm.objc.retain(ptr %self)
 ; CHECK-NEXT: %call = tail call i32 (ptr, ...) @printf(
 ; CHECK: tail call void @llvm.objc.release(ptr %self)
 ; CHECK: {{^}}}
-define { <2 x float>, <2 x float> } @"\01-[A z]"(ptr %self, ptr nocapture %_cmd) nounwind {
+define { <2 x float>, <2 x float> } @"\01-[A z]"(ptr %self, ptr captures(none) %_cmd) nounwind {
 invoke.cont:
   %i1 = tail call ptr @llvm.objc.retain(ptr %self) nounwind
   tail call void @llvm.dbg.value(metadata ptr %self, metadata !DILocalVariable(scope: !2), metadata !DIExpression()), !dbg !DILocation(scope: !2)
@@ -2654,11 +2654,11 @@ invoke.cont:
 }
 
 ; FIXME: Should be able to eliminate the retain and release
-; CHECK-LABEL: @"\01-[Top0 _getX]"(ptr %self, ptr nocapture %_cmd)
+; CHECK-LABEL: @"\01-[Top0 _getX]"(ptr %self, ptr captures(none) %_cmd)
 ; CHECK: tail call ptr @llvm.objc.retain(ptr %self)
 ; CHECK: %puts = tail call i32 @puts
 ; CHECK: tail call void @llvm.objc.release(ptr %self)
-define i32 @"\01-[Top0 _getX]"(ptr %self, ptr nocapture %_cmd) nounwind {
+define i32 @"\01-[Top0 _getX]"(ptr %self, ptr captures(none) %_cmd) nounwind {
 invoke.cont:
   %i1 = tail call ptr @llvm.objc.retain(ptr %self) nounwind
   %puts = tail call i32 @puts(ptr @str)
diff --git a/llvm/test/Transforms/OpenMP/add_attributes.ll b/llvm/test/Transforms/OpenMP/add_attributes.ll
index ce28643267a6a..b19e2c2a3ef02 100644
--- a/llvm/test/Transforms/OpenMP/add_attributes.ll
+++ b/llvm/test/Transforms/OpenMP/add_attributes.ll
@@ -670,7 +670,7 @@ declare ptr @__kmpc_task_reduction_modifier_init(ptr, i32, i32, i32, ptr)
 declare void @__kmpc_proxy_task_completed_ooo(ptr)
 
 ; Function Attrs: noinline cold
-declare void @__kmpc_barrier_simple_spmd(ptr nocapture nofree readonly, i32) #0
+declare void @__kmpc_barrier_simple_spmd(ptr nofree readonly captures(none), i32) #0
 
 attributes #0 = { noinline cold }
 
@@ -814,7 +814,7 @@ declare i32 @__tgt_target_kernel_nowait(ptr, i64, i32, i32, ptr, ptr, i32, ptr,
 ; CHECK-NEXT: declare dso_local i32 @omp_get_max_active_levels()
 
 ; CHECK: ; Function Attrs: nounwind
-; CHECK-NEXT: declare dso_local void @omp_get_schedule(ptr nocapture writeonly, ptr nocapture writeonly)
+; CHECK-NEXT: declare dso_local void @omp_get_schedule(ptr writeonly captures(none), ptr writeonly captures(none))
 
 ; CHECK-NOT: Function Attrs
 ; CHECK: declare dso_local i32 @omp_get_max_task_priority()
@@ -922,7 +922,7 @@ declare i32 @__tgt_target_kernel_nowait(ptr, i64, i32, i32, ptr, ptr, i32, ptr,
 ; CHECK: declare dso_local i32 @omp_get_place_num_procs(i32)
 
 ; CHECK: ; Function Attrs: nounwind
-; CHECK-NEXT: declare dso_local void @omp_get_place_proc_ids(i32, ptr nocapture writeonly)
+; CHECK-NEXT: declare dso_local void @omp_get_place_proc_ids(i32, ptr writeonly captures(none))
 
 ; CHECK: ; Function Attrs: nounwind
 ; CHECK-NEXT: declare dso_local i32 @omp_get_place_num()
@@ -1288,7 +1288,7 @@ declare i32 @__tgt_target_kernel_nowait(ptr, i64, i32, i32, ptr, ptr, i32, ptr,
 ; CHECK-NEXT: declare void @__kmpc_proxy_task_completed_ooo(ptr)
 
 ; CHECK: ; Function Attrs: cold convergent noinline nounwind
-; CHECK-NEXT: declare void @__kmpc_barrier_simple_spmd(ptr nocapture nofree readonly, i32)
+; CHECK-NEXT: declare void @__kmpc_barrier_simple_spmd(ptr nofree readonly captures(none), i32)
 
 ; CHECK: ; Function Attrs: nounwind
 ; CHECK-NEXT: declare noalias ptr @__kmpc_aligned_alloc(i32, i64, i64, ptr)
@@ -1330,7 +1330,7 @@ declare i32 @__tgt_target_kernel_nowait(ptr, i64, i32, i32, ptr, ptr, i32, ptr,
 ; CHECK: declare void @__kmpc_fork_call_if(ptr, i32, ptr, i32, ptr)
 
 ; CHECK: ; Function Attrs: nosync nounwind
-; CHECK-NEXT: declare void @__kmpc_free_shared(ptr allocptr nocapture, i64)
+; CHECK-NEXT: declare void @__kmpc_free_shared(ptr allocptr captures(none), i64)
 
 ; CHECK: ; Function Attrs: nounwind
 ; CHECK-NEXT: declare i32 @__kmpc_get_hardware_num_blocks()
@@ -1468,7 +1468,7 @@ declare i32 @__tgt_target_kernel_nowait(ptr, i64, i32, i32, ptr, ptr, i32, ptr,
 ; OPTIMISTIC-NEXT: declare dso_local i32 @omp_get_max_active_levels()
 
 ; OPTIMISTIC: ; Function Attrs: nofree nosync nounwind willreturn memory(argmem: readwrite, inaccessiblemem: read)
-; OPTIMISTIC-NEXT: declare dso_local void @omp_get_schedule(ptr nocapture writeonly, ptr nocapture writeonly)
+; OPTIMISTIC-NEXT: declare dso_local void @omp_get_schedule(ptr writeonly captures(none), ptr writeonly captures(none))
 
 ; OPTIMISTIC-NOT: Function Attrs
 ; OPTIMISTIC: declare dso_local i32 @omp_get_max_task_priority()
@@ -1576,7 +1576,7 @@ declare i32 @__tgt_target_kernel_nowait(ptr, i64, i32, i32, ptr, ptr, i32, ptr,
 ; OPTIMISTIC: declare dso_local i32 @omp_get_place_num_procs(i32)
 
 ; OPTIMISTIC: ; Function Attrs: nofree nosync nounwind willreturn memory(argmem: readwrite, inaccessiblemem: read)
-; OPTIMISTIC-NEXT: declare dso_local void @omp_get_place_proc_ids(i32, ptr nocapture writeonly)
+; OPTIMISTIC-NEXT: declare dso_local void @omp_get_place_proc_ids(i32, ptr writeonly captures(none))
 
 ; OPTIMISTIC: ; Function Attrs: nofree nosync nounwind willreturn memory(inaccessiblemem: read)
 ; OPTIMISTIC-NEXT: declare dso_local i32 @omp_get_place_num()
@@ -1633,229 +1633,229 @@ declare i32 @__tgt_target_kernel_nowait(ptr, i64, i32, i32, ptr, ptr, i32, ptr,
 ; OPTIMISTIC-NEXT: declare dso_local i32 @omp_get_supported_active_levels()
 
 ; OPTIMISTIC: ; Function Attrs: nofree nosync nounwind willreturn memory(argmem: read, inaccessiblemem: read)
-; OPTIMISTIC-NEXT: declare i32 @__kmpc_global_thread_num(ptr nocapture nofree readonly)
+; OPTIMISTIC-NEXT: declare i32 @__kmpc_global_thread_num(ptr nofree readonly captures(none))
 
 ; OPTIMISTIC: ; Function Attrs: nounwind
-; OPTIMISTIC-NEXT: declare void @__kmpc_fork_call(ptr nocapture nofree readonly, i32, ptr nocapture nofree readonly, ...)
+; OPTIMISTIC-NEXT: declare void @__kmpc_fork_call(ptr nofree readonly captures(none), i32, ptr nofree readonly captures(none), ...)
 
 ; OPTIMISTIC: ; Function Attrs: convergent nounwind
-; OPTIMISTIC-NEXT: declare i32 @__kmpc_omp_taskwait(ptr nocapture nofree readonly, i32)
+; OPTIMISTIC-NEXT: declare i32 @__kmpc_omp_taskwait(ptr nofree readonly captures(none), i32)
 
 ; OPTIMISTIC: ; Function Attrs: nofree nosync nounwind willreturn memory(argmem: readwrite, inaccessiblemem: readwrite)
-; OPTIMISTIC-NEXT: declare i32 @__kmpc_omp_taskyield(ptr nocapture nofree readonly, i32, i32)
+; OPTIMISTIC-NEXT: declare i32 @__kmpc_omp_taskyield(ptr nofree readonly captures(none), i32, i32)
 
 ; OPTIMISTIC: ; Function Attrs: nofree nosync nounwind willreturn memory(argmem: readwrite, inaccessiblemem: readwrite)
-; OPTIMISTIC-NEXT: declare void @__kmpc_push_num_threads(ptr nocapture nofree readonly, i32, i32)
+; OPTIMISTIC-NEXT: declare void @__kmpc_push_num_threads(ptr nofree readonly captures(none), i32, i32)
 
 ; OPTIMISTIC: ; Function Attrs: nofree nosync nounwind willreturn memory(argmem: readwrite, inaccessiblemem: readwrite)
-; OPTIMISTIC-NEXT: declare void @__kmpc_push_proc_bind(ptr nocapture nofree readonly, i32, i32)
+; OPTIMISTIC-NEXT: declare void @__kmpc_push_proc_bind(ptr nofree readonly captures(none), i32, i32)
 
 ; OPTIMISTIC: ; Function Attrs: nofree nosync nounwind willreturn memory(argmem: readwrite, inaccessiblemem: readwrite)
-; OPTIMISTIC-NEXT: declare void @__kmpc_serialized_parallel(ptr nocapture nofree readonly, i32)
+; OPTIMISTIC-NEXT: declare void @__kmpc_serialized_parallel(ptr nofree readonly captures(none), i32)
 
 ; OPTIMISTIC: ; Function Attrs: nofree nosync nounwind willreturn memory(argmem: readwrite, inaccessiblemem: readwrite)
-; OPTIMISTIC-NEXT: declare void @__kmpc_end_serialized_parallel(ptr nocapture nofree readonly, i32)
+; OPTIMISTIC-NEXT: declare void @__kmpc_end_serialized_parallel(ptr nofree readonly captures(none), i32)
 
 ; OPTIMISTIC: ; Function Attrs: nofree nosync nounwind willreturn memory(argmem: readwrite, inaccessiblemem: readwrite)
-; OPTIMISTIC-NEXT: declare i32 @__kmpc_master(ptr nocapture nofree readonly, i32)
+; OPTIMISTIC-NEXT: declare i32 @__kmpc_master(ptr nofree readonly captures(none), i32)
 
 ; OPTIMISTIC: ; Function Attrs: nofree nosync nounwind willreturn memory(argmem: readwrite, inaccessiblemem: readwrite)
-; OPTIMISTIC-NEXT: declare void @__kmpc_end_master(ptr nocapture nofree readonly, i32)
+; OPTIMISTIC-NEXT: declare void @__kmpc_end_master(ptr nofree readonly captures(none), i32)
 
 ; OPTIMISTIC: ; Function Attrs: convergent nounwind
-; OPTIMISTIC-NEXT: declare void @__kmpc_critical(ptr nocapture nofree readonly, i32, ptr)
+; OPTIMISTIC-NEXT: declare void @__kmpc_critical(ptr nofree readonly captures(none), i32, ptr)
 
 ; OPTIMISTIC: ; Function Attrs: convergent nounwind
-; OPTIMISTIC-NEXT: declare void @__kmpc_critical_with_hint(ptr nocapture nofree readonly, i32, ptr, i32)
+; OPTIMISTIC-NEXT: declare void @__kmpc_critical_with_hint(ptr nofree readonly captures(none), i32, ptr, i32)
 
 ; OPTIMISTIC: ; Function Attrs: convergent nounwind
-; OPTIMISTIC-NEXT: declare void @__kmpc_end_critical(ptr nocapture nofree readonly, i32, ptr)
+; OPTIMISTIC-NEXT: declare void @__kmpc_end_critical(ptr nofree readonly captures(none), i32, ptr)
 
 ; OPTIMISTIC: ; Function Attrs: nofree nosync nounwind willreturn
-; OPTIMISTIC-NEXT: declare void @__kmpc_begin(ptr nocapture nofree readonly, i32)
+; OPTIMISTIC-NEXT: declare void @__kmpc_begin(ptr nofree readonly captures(none), i32)
 
 ; OPTIMISTIC: ; Function Attrs: nofree nosync nounwind willreturn
-; OPTIMISTIC-NEXT: declare void @__kmpc_end(ptr nocapture nofree readonly)
+; OPTIMISTIC-NEXT: declare void @__kmpc_end(ptr nofree readonly captures(none))
 
 ; OPTIMISTIC: ; Function Attrs: convergent nounwind
-; OPTIMISTIC-NEXT: declare i32 @__kmpc_reduce(ptr nocapture nofree readonly, i32, i32, i64, ptr nocapture nofree readonly, ptr, ptr)
+; OPTIMISTIC-NEXT: declare i32 @__kmpc_reduce(ptr nofree readonly captures(none), i32, i32, i64, ptr nofree readonly captures(none), ptr, ptr)
 
 ; OPTIMISTIC: ; Function Attrs: convergent nounwind
-; OPTIMISTIC-NEXT: declare i32 @__kmpc_reduce_nowait(ptr nocapture nofree readonly, i32, i32, i64, ptr nocapture nofree readonly, ptr, ptr)
+; OPTIMISTIC-NEXT: declare i32 @__kmpc_reduce_nowait(ptr nofree readonly captures(none), i32, i32, i64, ptr nofree readonly captures(none), ptr, ptr)
 
 ; OPTIMISTIC: ; Function Attrs: convergent nounwind
-; OPTIMISTIC-NEXT: declare void @__kmpc_end_reduce(ptr nocapture nofree readonly, i32, ptr)
+; OPTIMISTIC-NEXT: declare void @__kmpc_end_reduce(ptr nofree readonly captures(none), i32, ptr)
 
 ; OPTIMISTIC: ; Function Attrs: convergent nounwind
-; OPTIMISTIC-NEXT: declare void @__kmpc_end_reduce_nowait(ptr nocapture nofree readonly, i32, ptr)
+; OPTIMISTIC-NEXT: declare void @__kmpc_end_reduce_nowait(ptr nofree readonly captures(none), i32, ptr)
 
 ; OPTIMISTIC: ; Function Attrs: convergent nounwind
-; OPTIMISTIC-NEXT: declare void @__kmpc_ordered(ptr nocapture nofree readonly, i32)
+; OPTIMISTIC-NEXT: declare void @__kmpc_ordered(ptr nofree readonly captures(none), i32)
 
 ; OPTIMISTIC: ; Function Attrs: convergent nounwind
-; OPTIMISTIC-NEXT: declare void @__kmpc_end_ordered(ptr nocapture nofree readonly, i32)
+; OPTIMISTIC-NEXT: declare void @__kmpc_end_ordered(ptr nofree readonly captures(none), i32)
 
 ; OPTIMISTIC: ; Function Attrs: nofree nosync nounwind willreturn memory(argmem: readwrite, inaccessiblemem: read)
-; OPTIMISTIC-NEXT: declare void @__kmpc_for_static_init_4(ptr nocapture nofree readonly, i32, i32, ptr nocapture nofree, ptr nocapture nofree, ptr nocapture nofree, ptr nocapture nofree, i32, i32)
+; OPTIMISTIC-NEXT: declare void @__kmpc_for_static_init_4(ptr nofree readonly captures(none), i32, i32, ptr nofree captures(none), ptr nofree captures(none), ptr nofree captures(none), ptr nofree captures(none), i32, i32)
 
 ; OPTIMISTIC: ; Function Attrs: nofree nosync nounwind willreturn memory(argmem: readwrite, inaccessiblemem: read)
-; OPTIMISTIC-NEXT: declare void @__kmpc_for_static_init_4u(ptr nocapture nofree readonly, i32, i32, ptr nocapture nofree, ptr nocapture nofree, ptr nocapture nofree, ptr nocapture nofree, i32, i32)
+; OPTIMISTIC-NEXT: declare void @__kmpc_for_static_init_4u(ptr nofree readonly captures(none), i32, i32, ptr nofree captures(none), ptr nofree captures(none), ptr nofree captures(none), ptr nofree captures(none), i32, i32)
 
 ; OPTIMISTIC: ; Function Attrs: nofree nosync nounwind willreturn memory(argmem: readwrite, inaccessiblemem: read)
-; OPTIMISTIC-NEXT: declare void @__kmpc_for_static_init_8(ptr nocapture nofree readonly, i32, i32, ptr nocapture nofree, ptr nocapture nofree, ptr nocapture nofree, ptr nocapture nofree, i64, i64)
+; OPTIMISTIC-NEXT: declare void @__kmpc_for_static_init_8(ptr nofree readonly captures(none), i32, i32, ptr nofree captures(none), ptr nofree captures(none), ptr nofree captures(none), ptr nofree captures(none), i64, i64)
 
 ; OPTIMISTIC: ; Function Attrs: nofree nosync nounwind willreturn memory(argmem: readwrite, inaccessiblemem: read)
-; OPTIMISTIC-NEXT: declare void @__kmpc_for_static_init_8u(ptr nocapture nofree readonly, i32, i32, ptr nocapture nofree, ptr nocapture nofree, ptr nocapture nofree, ptr nocapture nofree, i64, i64)
+; OPTIMISTIC-NEXT: declare void @__kmpc_for_static_init_8u(ptr nofree readonly captures(none), i32, i32, ptr nofree captures(none), ptr nofree captures(none), ptr nofree captures(none), ptr nofree captures(none), i64, i64)
 
 ; OPTIMISTIC: ; Function Attrs: nofree nosync nounwind willreturn memory(argmem: readwrite, inaccessiblemem: readwrite)
-; OPTIMISTIC-NEXT: declare void @__kmpc_for_static_fini(ptr nocapture nofree readonly, i32)
+; OPTIMISTIC-NEXT: declare void @__kmpc_for_static_fini(ptr nofree readonly captures(none), i32)
 
 ; OPTIMISTIC: ; Function Attrs: nofree nosync nounwind willreturn memory(argmem: readwrite, inaccessiblemem: read)
-; OPTIMISTIC-NEXT: declare void @__kmpc_team_static_init_4(ptr nocapture nofree readonly, i32, ptr nocapture nofree, ptr nocapture nofree, ptr nocapture nofree, ptr nocapture nofree, i32, i32)
+; OPTIMISTIC-NEXT: declare void @__kmpc_team_static_init_4(ptr nofree readonly captures(none), i32, ptr nofree captures(none), ptr nofree captures(none), ptr nofree captures(none), ptr nofree captures(none), i32, i32)
 
 ; OPTIMISTIC: ; Function Attrs: nofree nosync nounwind willreturn memory(argmem: readwrite, inaccessiblemem: read)
-; OPTIMISTIC-NEXT: declare void @__kmpc_team_static_init_4u(ptr nocapture nofree readonly, i32, ptr nocapture nofree, ptr nocapture nofree, ptr nocapture nofree, ptr nocapture nofree, i32, i32)
+; OPTIMISTIC-NEXT: declare void @__kmpc_team_static_init_4u(ptr nofree readonly captures(none), i32, ptr nofree captures(none), ptr nofree captures(none), ptr nofree captures(none), ptr nofree captures(none), i32, i32)
 
 ; OPTIMISTIC: ; Function Attrs: nofree nosync nounwind willreturn memory(argmem: readwrite, inaccessiblemem: read)
-; OPTIMISTIC-NEXT: declare void @__kmpc_team_static_init_8(ptr nocapture nofree readonly, i32, ptr nocapture nofree, ptr nocapture nofree, ptr nocapture nofree, ptr nocapture nofree, i64, i64)
+; OPTIMISTIC-NEXT: declare void @__kmpc_team_static_init_8(ptr nofree readonly captures(none), i32, ptr nofree captures(none), ptr nofree captures(none), ptr nofree captures(none), ptr nofree captures(none), i64, i64)
 
 ; OPTIMISTIC: ; Function Attrs: nofree nosync nounwind willreturn memory(argmem: readwrite, inaccessiblemem: read)
-; OPTIMISTIC-NEXT: declare void @__kmpc_team_static_init_8u(ptr nocapture nofree readonly, i32, ptr nocapture nofree, ptr nocapture nofree, ptr nocapture nofree, ptr nocapture nofree, i64, i64)
+; OPTIMISTIC-NEXT: declare void @__kmpc_team_static_init_8u(ptr nofree readonly captures(none), i32, ptr nofree captures(none), ptr nofree captures(none), ptr nofree captures(none), ptr nofree captures(none), i64, i64)
 
 ; OPTIMISTIC: ; Function Attrs: nofree nosync nounwind willreturn memory(argmem: readwrite, inaccessiblemem: read)
-; OPTIMISTIC-NEXT: declare void @__kmpc_dist_for_static_init_4(ptr nocapture nofree readonly, i32, i32, ptr nocapture nofree, ptr nocapture nofree, ptr nocapture nofree, ptr nocapture nofree, ptr nocapture nofree, i32, i32)
+; OPTIMISTIC-NEXT: declare void @__kmpc_dist_for_static_init_4(ptr nofree readonly captures(none), i32, i32, ptr nofree captures(none), ptr nofree captures(none), ptr nofree captures(none), ptr nofree captures(none), ptr nofree captures(none), i32, i32)
 
 ; OPTIMISTIC: ; Function Attrs: nofree nosync nounwind willreturn memory(argmem: readwrite, inaccessiblemem: read)
-; OPTIMISTIC-NEXT: declare void @__kmpc_dist_for_static_init_4u(ptr nocapture nofree readonly, i32, i32, ptr nocapture nofree, ptr nocapture nofree, ptr nocapture nofree, ptr nocapture nofree, ptr nocapture nofree, i32, i32)
+; OPTIMISTIC-NEXT: declare void @__kmpc_dist_for_static_init_4u(ptr nofree readonly captures(none), i32, i32, ptr nofree captures(none), ptr nofree captures(none), ptr nofree captures(none), ptr nofree captures(none), ptr nofree captures(none), i32, i32)
 
 ; OPTIMISTIC: ; Function Attrs: nofree nosync nounwind willreturn memory(argmem: readwrite, inaccessiblemem: read)
-; OPTIMISTIC-NEXT: declare void @__kmpc_dist_for_static_init_8(ptr nocapture nofree readonly, i32, i32, ptr nocapture nofree, ptr nocapture nofree, ptr nocapture nofree, ptr nocapture nofree, ptr nocapture nofree, i64, i64)
+; OPTIMISTIC-NEXT: declare void @__kmpc_dist_for_static_init_8(ptr nofree readonly captures(none), i32, i32, ptr nofree captures(none), ptr nofree captures(none), ptr nofree captures(none), ptr nofree captures(none), ptr nofree captures(none), i64, i64)
 
 ; OPTIMISTIC: ; Function Attrs: nofree nosync nounwind willreturn memory(argmem: readwrite, inaccessiblemem: read)
-; OPTIMISTIC-NEXT: declare void @__kmpc_dist_for_static_init_8u(ptr nocapture nofree readonly, i32, i32, ptr nocapture nofree, ptr nocapture nofree, ptr nocapture nofree, ptr nocapture nofree, ptr nocapture nofree, i64, i64)
+; OPTIMISTIC-NEXT: declare void @__kmpc_dist_for_static_init_8u(ptr nofree readonly captures(none), i32, i32, ptr nofree captures(none), ptr nofree captures(none), ptr nofree captures(none), ptr nofree captures(none), ptr nofree captures(none), i64, i64)
 
 ; OPTIMISTIC: ; Function Attrs: convergent nounwind
-; OPTIMISTIC-NEXT: declare i32 @__kmpc_single(ptr nocapture nofree readonly, i32)
+; OPTIMISTIC-NEXT: declare i32 @__kmpc_single(ptr nofree readonly captures(none), i32)
 
 ; OPTIMISTIC: ; Function Attrs: convergent nounwind
-; OPTIMISTIC-NEXT: declare void @__kmpc_end_single(ptr nocapture nofree readonly, i32)
+; OPTIMISTIC-NEXT: declare void @__kmpc_end_single(ptr nofree readonly captures(none), i32)
 
 ; OPTIMISTIC: ; Function Attrs: nofree nosync nounwind willreturn
-; OPTIMISTIC-NEXT: declare noalias ptr @__kmpc_omp_task_alloc(ptr nocapture nofree readonly, i32, i32, i64, i64, ptr nocapture nofree readonly)
+; OPTIMISTIC-NEXT: declare noalias ptr @__kmpc_omp_task_alloc(ptr nofree readonly captures(none), i32, i32, i64, i64, ptr nofree readonly captures(none))
 
 ; OPTIMISTIC: ; Function Attrs: nofree nosync nounwind willreturn
-; OPTIMISTIC-NEXT: declare i32 @__kmpc_omp_task(ptr nocapture nofree readonly, i32, ptr)
+; OPTIMISTIC-NEXT: declare i32 @__kmpc_omp_task(ptr nofree readonly captures(none), i32, ptr)
 
 ; OPTIMISTIC: ; Function Attrs: convergent nounwind
-; OPTIMISTIC-NEXT: declare void @__kmpc_end_taskgroup(ptr nocapture nofree readonly, i32)
+; OPTIMISTIC-NEXT: declare void @__kmpc_end_taskgroup(ptr nofree readonly captures(none), i32)
 
 ; OPTIMISTIC: ; Function Attrs: convergent nounwind
-; OPTIMISTIC-NEXT: declare void @__kmpc_taskgroup(ptr nocapture nofree readonly, i32)
+; OPTIMISTIC-NEXT: declare void @__kmpc_taskgroup(ptr nofree readonly captures(none), i32)
 
 ; OPTIMISTIC: ; Function Attrs: nofree nosync nounwind willreturn memory(argmem: readwrite, inaccessiblemem: read)
-; OPTIMISTIC-NEXT: declare void @__kmpc_dist_dispatch_init_4(ptr nocapture nofree readonly, i32, i32, ptr nocapture nofree, i32, i32, i32, i32)
+; OPTIMISTIC-NEXT: declare void @__kmpc_dist_dispatch_init_4(ptr nofree readonly captures(none), i32, i32, ptr nofree captures(none), i32, i32, i32, i32)
 
 ; OPTIMISTIC: ; Function Attrs: nofree nosync nounwind willreturn memory(argmem: readwrite, inaccessiblemem: read)
-; OPTIMISTIC-NEXT: declare void @__kmpc_dist_dispatch_init_4u(ptr nocapture nofree readonly, i32, i32, ptr nocapture nofree, i32, i32, i32, i32)
+; OPTIMISTIC-NEXT: declare void @__kmpc_dist_dispatch_init_4u(ptr nofree readonly captures(none), i32, i32, ptr nofree captures(none), i32, i32, i32, i32)
 
 ; OPTIMISTIC: ; Function Attrs: nofree nosync nounwind willreturn memory(argmem: readwrite, inaccessiblemem: read)
-; OPTIMISTIC-NEXT: declare void @__kmpc_dist_dispatch_init_8(ptr nocapture nofree readonly, i32, i32, ptr nocapture nofree, i64, i64, i64, i64)
+; OPTIMISTIC-NEXT: declare void @__kmpc_dist_dispatch_init_8(ptr nofree readonly captures(none), i32, i32, ptr nofree captures(none), i64, i64, i64, i64)
 
 ; OPTIMISTIC: ; Function Attrs: nofree nosync nounwind willreturn memory(argmem: readwrite, inaccessiblemem: read)
-; OPTIMISTIC-NEXT: declare void @__kmpc_dist_dispatch_init_8u(ptr nocapture nofree readonly, i32, i32, ptr nocapture nofree, i64, i64, i64, i64)
+; OPTIMISTIC-NEXT: declare void @__kmpc_dist_dispatch_init_8u(ptr nofree readonly captures(none), i32, i32, ptr nofree captures(none), i64, i64, i64, i64)
 
 ; OPTIMISTIC: ; Function Attrs: nofree nosync nounwind willreturn memory(argmem: readwrite, inaccessiblemem: read)
-; OPTIMISTIC-NEXT: declare void @__kmpc_dispatch_init_4(ptr nocapture nofree readonly, i32, i32, i32, i32, i32, i32)
+; OPTIMISTIC-NEXT: declare void @__kmpc_dispatch_init_4(ptr nofree readonly captures(none), i32, i32, i32, i32, i32, i32)
 
 ; OPTIMISTIC: ; Function Attrs: nofree nosync nounwind willreturn memory(argmem: readwrite, inaccessiblemem: read)
-; OPTIMISTIC-NEXT: declare void @__kmpc_dispatch_init_4u(ptr nocapture nofree readonly, i32, i32, i32, i32, i32, i32)
+; OPTIMISTIC-NEXT: declare void @__kmpc_dispatch_init_4u(ptr nofree readonly captures(none), i32, i32, i32, i32, i32, i32)
 
 ; OPTIMISTIC: ; Function Attrs: nofree nosync nounwind willreturn memory(argmem: readwrite, inaccessiblemem: read)
-; OPTIMISTIC-NEXT: declare void @__kmpc_dispatch_init_8(ptr nocapture nofree readonly, i32, i32, i64, i64, i64, i64)
+; OPTIMISTIC-NEXT: declare void @__kmpc_dispatch_init_8(ptr nofree readonly captures(none), i32, i32, i64, i64, i64, i64)
 
 ; OPTIMISTIC: ; Function Attrs: nofree nosync nounwind willreturn memory(argmem: readwrite, inaccessiblemem: read)
-; OPTIMISTIC-NEXT: declare void @__kmpc_dispatch_init_8u(ptr nocapture nofree readonly, i32, i32, i64, i64, i64, i64)
+; OPTIMISTIC-NEXT: declare void @__kmpc_dispatch_init_8u(ptr nofree readonly captures(none), i32, i32, i64, i64, i64, i64)
 
 ; OPTIMISTIC: ; Function Attrs: nofree nosync nounwind willreturn memory(argmem: readwrite, inaccessiblemem: read)
-; OPTIMISTIC-NEXT: declare i32 @__kmpc_dispatch_next_4(ptr nocapture nofree readonly, i32, ptr nocapture nofree, ptr nocapture nofree, ptr nocapture nofree, ptr nocapture nofree)
+; OPTIMISTIC-NEXT: declare i32 @__kmpc_dispatch_next_4(ptr nofree readonly captures(none), i32, ptr nofree captures(none), ptr nofree captures(none), ptr nofree captures(none), ptr nofree captures(none))
 
 ; OPTIMISTIC: ; Function Attrs: nofree nosync nounwind willreturn memory(argmem: readwrite, inaccessiblemem: read)
-; OPTIMISTIC-NEXT: declare i32 @__kmpc_dispatch_next_4u(ptr nocapture nofree readonly, i32, ptr nocapture nofree, ptr nocapture nofree, ptr nocapture nofree, ptr nocapture nofree)
+; OPTIMISTIC-NEXT: declare i32 @__kmpc_dispatch_next_4u(ptr nofree readonly captures(none), i32, ptr nofree captures(none), ptr nofree captures(none), ptr nofree captures(none), ptr nofree captures(none))
 
 ; OPTIMISTIC: ; Function Attrs: nofree nosync nounwind willreturn memory(argmem: readwrite, inaccessiblemem: read)
-; OPTIMISTIC-NEXT: declare i32 @__kmpc_dispatch_next_8(ptr nocapture nofree readonly, i32, ptr nocapture nofree, ptr nocapture nofree, ptr nocapture nofree, ptr nocapture nofree)
+; OPTIMISTIC-NEXT: declare i32 @__kmpc_dispatch_next_8(ptr nofree readonly captures(none), i32, ptr nofree captures(none), ptr nofree captures(none), ptr nofree captures(none), ptr nofree captures(none))
 
 ; OPTIMISTIC: ; Function Attrs: nofree nosync nounwind willreturn memory(argmem: readwrite, inaccessiblemem: read)
-; OPTIMISTIC-NEXT: declare i32 @__kmpc_dispatch_next_8u(ptr nocapture nofree readonly, i32, ptr nocapture nofree, ptr nocapture nofree, ptr nocapture nofree, ptr nocapture nofree)
+; OPTIMISTIC-NEXT: declare i32 @__kmpc_dispatch_next_8u(ptr nofree readonly captures(none), i32, ptr nofree captures(none), ptr nofree captures(none), ptr nofree captures(none), ptr nofree captures(none))
 
 ; OPTIMISTIC: ; Function Attrs: nofree nosync nounwind willreturn memory(argmem: readwrite, inaccessiblemem: readwrite)
-; OPTIMISTIC-NEXT: declare void @__kmpc_dispatch_fini_4(ptr nocapture nofree readonly, i32)
+; OPTIMISTIC-NEXT: declare void @__kmpc_dispatch_fini_4(ptr nofree readonly captures(none), i32)
 
 ; OPTIMISTIC: ; Function Attrs: nofree nosync nounwind willreturn memory(argmem: readwrite, inaccessiblemem: readwrite)
-; OPTIMISTIC-NEXT: declare void @__kmpc_dispatch_fini_4u(ptr nocapture nofree readonly, i32)
+; OPTIMISTIC-NEXT: declare void @__kmpc_dispatch_fini_4u(ptr nofree readonly captures(none), i32)
 
 ; OPTIMISTIC: ; Function Attrs: nofree nosync nounwind willreturn memory(argmem: readwrite, inaccessiblemem: readwrite)
-; OPTIMISTIC-NEXT: declare void @__kmpc_dispatch_fini_8(ptr nocapture nofree readonly, i32)
+; OPTIMISTIC-NEXT: declare void @__kmpc_dispatch_fini_8(ptr nofree readonly captures(none), i32)
 
 ; OPTIMISTIC: ; Function Attrs: nofree nosync nounwind willreturn memory(argmem: readwrite, inaccessiblemem: readwrite)
-; OPTIMISTIC-NEXT: declare void @__kmpc_dispatch_fini_8u(ptr nocapture nofree readonly, i32)
+; OPTIMISTIC-NEXT: declare void @__kmpc_dispatch_fini_8u(ptr nofree readonly captures(none), i32)
 
 ; OPTIMISTIC: ; Function Attrs: nofree nosync nounwind willreturn
-; OPTIMISTIC-NEXT: declare void @__kmpc_omp_task_begin_if0(ptr nocapture nofree readonly, i32, ptr)
+; OPTIMISTIC-NEXT: declare void @__kmpc_omp_task_begin_if0(ptr nofree readonly captures(none), i32, ptr)
 
 ; OPTIMISTIC: ; Function Attrs: nofree nosync nounwind willreturn
-; OPTIMISTIC-NEXT: declare void @__kmpc_omp_task_complete_if0(ptr nocapture nofree readonly, i32, ptr)
+; OPTIMISTIC-NEXT: declare void @__kmpc_omp_task_complete_if0(ptr nofree readonly captures(none), i32, ptr)
 
 ; OPTIMISTIC: ; Function Attrs: nofree nosync nounwind willreturn
-; OPTIMISTIC-NEXT: declare i32 @__kmpc_omp_task_with_deps(ptr nocapture nofree readonly, i32, ptr, i32, ptr nocapture nofree readonly, i32, ptr nocapture nofree readonly)
+; OPTIMISTIC-NEXT: declare i32 @__kmpc_omp_task_with_deps(ptr nofree readonly captures(none), i32, ptr, i32, ptr nofree readonly captures(none), i32, ptr nofree readonly captures(none))
 
 ; OPTIMISTIC: ; Function Attrs: convergent nounwind
-; OPTIMISTIC-NEXT: declare void @__kmpc_omp_wait_deps(ptr nocapture nofree readonly, i32, i32, ptr nocapture nofree readonly, i32, ptr)
+; OPTIMISTIC-NEXT: declare void @__kmpc_omp_wait_deps(ptr nofree readonly captures(none), i32, i32, ptr nofree readonly captures(none), i32, ptr)
 
 ; OPTIMISTIC: ; Function Attrs: nofree nosync nounwind willreturn
-; OPTIMISTIC-NEXT: declare i32 @__kmpc_cancellationpoint(ptr nocapture nofree readonly, i32, i32)
+; OPTIMISTIC-NEXT: declare i32 @__kmpc_cancellationpoint(ptr nofree readonly captures(none), i32, i32)
 
 ; OPTIMISTIC: ; Function Attrs: nofree nosync nounwind willreturn memory(argmem: readwrite, inaccessiblemem: readwrite)
-; OPTIMISTIC-NEXT: declare void @__kmpc_push_num_teams(ptr nocapture nofree readonly, i32, i32, i32)
+; OPTIMISTIC-NEXT: declare void @__kmpc_push_num_teams(ptr nofree readonly captures(none), i32, i32, i32)
 
 ; OPTIMISTIC: ; Function Attrs: nounwind
-; OPTIMISTIC-NEXT: declare void @__kmpc_fork_teams(ptr nocapture nofree readonly, i32, ptr nocapture nofree readonly, ...)
+; OPTIMISTIC-NEXT: declare void @__kmpc_fork_teams(ptr nofree readonly captures(none), i32, ptr nofree readonly captures(none), ...)
 
 ; OPTIMISTIC: ; Function Attrs: nofree nosync nounwind willreturn
-; OPTIMISTIC-NEXT: declare void @__kmpc_taskloop(ptr nocapture nofree readonly, i32, ptr, i32, ptr nocapture nofree, ptr nocapture nofree, i64, i32, i32, i64, ptr)
+; OPTIMISTIC-NEXT: declare void @__kmpc_taskloop(ptr nofree readonly captures(none), i32, ptr, i32, ptr nofree captures(none), ptr nofree captures(none), i64, i32, i32, i64, ptr)
 
 ; OPTIMISTIC: ; Function Attrs: nofree nosync nounwind willreturn
-; OPTIMISTIC-NEXT: declare noalias ptr @__kmpc_omp_target_task_alloc(ptr nocapture nofree readonly, i32, i32, i64, i64, ptr nocapture nofree readonly, i64)
+; OPTIMISTIC-NEXT: declare noalias ptr @__kmpc_omp_target_task_alloc(ptr nofree readonly captures(none), i32, i32, i64, i64, ptr nofree readonly captures(none), i64)
 
 ; OPTIMISTIC: ; Function Attrs: nofree nosync nounwind willreturn
-; OPTIMISTIC-NEXT: declare ptr @__kmpc_taskred_modifier_init(ptr nocapture nofree readonly, i32, i32, i32, ptr)
+; OPTIMISTIC-NEXT: declare ptr @__kmpc_taskred_modifier_init(ptr nofree readonly captures(none), i32, i32, i32, ptr)
 
 ; OPTIMISTIC: ; Function Attrs: nofree nosync nounwind willreturn
 ; OPTIMISTIC-NEXT: declare ptr @__kmpc_taskred_init(i32, i32, ptr)
 
 ; OPTIMISTIC: ; Function Attrs: convergent nounwind
-; OPTIMISTIC-NEXT: declare void @__kmpc_task_reduction_modifier_fini(ptr nocapture nofree readonly, i32, i32)
+; OPTIMISTIC-NEXT: declare void @__kmpc_task_reduction_modifier_fini(ptr nofree readonly captures(none), i32, i32)
 
 ; OPTIMISTIC: ; Function Attrs: nofree nosync nounwind willreturn
-; OPTIMISTIC-NEXT: declare void @__kmpc_copyprivate(ptr nocapture nofree readonly, i32, i64, ptr nocapture nofree readonly, ptr, i32)
+; OPTIMISTIC-NEXT: declare void @__kmpc_copyprivate(ptr nofree readonly captures(none), i32, i64, ptr nofree readonly captures(none), ptr, i32)
 
 ; OPTIMISTIC: ; Function Attrs: nofree nosync nounwind willreturn
-; OPTIMISTIC-NEXT: declare ptr @__kmpc_threadprivate_cached(ptr nocapture nofree readonly, i32, ptr, i64, ptr)
+; OPTIMISTIC-NEXT: declare ptr @__kmpc_threadprivate_cached(ptr nofree readonly captures(none), i32, ptr, i64, ptr)
 
 ; OPTIMISTIC: ; Function Attrs: nofree nosync nounwind willreturn
-; OPTIMISTIC-NEXT: declare void @__kmpc_threadprivate_register(ptr nocapture nofree readonly, ptr, ptr nocapture nofree readonly, ptr nocapture nofree readonly, ptr nocapture nofree readonly)
+; OPTIMISTIC-NEXT: declare void @__kmpc_threadprivate_register(ptr nofree readonly captures(none), ptr, ptr nofree readonly captures(none), ptr nofree readonly captures(none), ptr nofree readonly captures(none))
 
 ; OPTIMISTIC: ; Function Attrs: convergent nounwind
-; OPTIMISTIC-NEXT: declare void @__kmpc_doacross_init(ptr nocapture nofree readonly, i32, i32, ptr)
+; OPTIMISTIC-NEXT: declare void @__kmpc_doacross_init(ptr nofree readonly captures(none), i32, i32, ptr)
 
 ; OPTIMISTIC: ; Function Attrs: convergent nounwind
-; OPTIMISTIC-NEXT: declare void @__kmpc_doacross_wait(ptr nocapture nofree readonly, i32, ptr nocapture nofree readonly)
+; OPTIMISTIC-NEXT: declare void @__kmpc_doacross_wait(ptr nofree readonly captures(none), i32, ptr nofree readonly captures(none))
 
 ; OPTIMISTIC: ; Function Attrs: convergent nounwind
-; OPTIMISTIC-NEXT: declare void @__kmpc_doacross_post(ptr nocapture nofree readonly, i32, ptr nocapture nofree readonly)
+; OPTIMISTIC-NEXT: declare void @__kmpc_doacross_post(ptr nofree readonly captures(none), i32, ptr nofree readonly captures(none))
 
 ; OPTIMISTIC: ; Function Attrs: convergent nounwind
-; OPTIMISTIC-NEXT: declare void @__kmpc_doacross_fini(ptr nocapture nofree readonly, i32)
+; OPTIMISTIC-NEXT: declare void @__kmpc_doacross_fini(ptr nofree readonly captures(none), i32)
 
 ; OPTIMISTIC: ; Function Attrs: nofree nosync nounwind willreturn
 ; OPTIMISTIC-NEXT: declare noalias ptr @__kmpc_alloc(i32, i64, ptr)
@@ -1915,7 +1915,7 @@ declare i32 @__tgt_target_kernel_nowait(ptr, i64, i32, i32, ptr, ptr, i32, ptr,
 ; OPTIMISTIC-NEXT: declare void @__tgt_push_mapper_component(ptr, ptr, ptr, i64, i64, ptr)
 
 ; OPTIMISTIC: ; Function Attrs: nofree nosync nounwind willreturn
-; OPTIMISTIC-NEXT: declare ptr @__kmpc_task_allow_completion_event(ptr nocapture nofree readonly, i32, ptr)
+; OPTIMISTIC-NEXT: declare ptr @__kmpc_task_allow_completion_event(ptr nofree readonly captures(none), i32, ptr)
 
 ; OPTIMISTIC: ; Function Attrs: nofree nosync nounwind willreturn
 ; OPTIMISTIC-NEXT: declare ptr @__kmpc_task_reduction_get_th_data(i32, ptr, ptr)
@@ -1930,7 +1930,7 @@ declare i32 @__tgt_target_kernel_nowait(ptr, i64, i32, i32, ptr, ptr, i32, ptr,
 ; OPTIMISTIC-NEXT: declare void @__kmpc_proxy_task_completed_ooo(ptr)
 
 ; OPTIMISTIC: ; Function Attrs: cold convergent noinline nounwind
-; OPTIMISTIC-NEXT: declare void @__kmpc_barrier_simple_spmd(ptr nocapture nofree readonly, i32)
+; OPTIMISTIC-NEXT: declare void @__kmpc_barrier_simple_spmd(ptr nofree readonly captures(none), i32)
 
 ; OPTIMISTIC: ; Function Attrs: nofree nosync nounwind willreturn
 ; OPTIMISTIC-NEXT: declare noalias ptr @__kmpc_aligned_alloc(i32, i64, i64, ptr)
@@ -1939,28 +1939,28 @@ declare i32 @__tgt_target_kernel_nowait(ptr, i64, i32, i32, ptr, ptr, i32, ptr,
 ; OPTIMISTIC-NEXT: declare noalias ptr @__kmpc_alloc_shared(i64)
 
 ; OPTIMISTIC: ; Function Attrs: convergent nounwind
-; OPTIMISTIC: declare void @__kmpc_barrier_simple_generic(ptr nocapture nofree readonly, i32)
+; OPTIMISTIC: declare void @__kmpc_barrier_simple_generic(ptr nofree readonly captures(none), i32)
 
 ; OPTIMISTIC-NOT: Function Attrs
 ; OPTIMISTIC: declare void @__kmpc_begin_sharing_variables(ptr, i64)
 
 ; OPTIMISTIC: ; Function Attrs: nofree nosync nounwind willreturn memory(argmem: readwrite, inaccessiblemem: readwrite)
-; OPTIMISTIC: declare void @__kmpc_distribute_static_fini(ptr nocapture nofree readonly, i32)
+; OPTIMISTIC: declare void @__kmpc_distribute_static_fini(ptr nofree readonly captures(none), i32)
 
 ; OPTIMISTIC: ; Function Attrs: nofree nosync nounwind willreturn memory(argmem: readwrite, inaccessiblemem: read)
-; OPTIMISTIC: declare void @__kmpc_distribute_static_init_4(ptr nocapture nofree readonly, i32, i32, ptr nocapture nofree, ptr nocapture nofree, ptr nocapture nofree, ptr nocapture nofree, i32, i32)
+; OPTIMISTIC: declare void @__kmpc_distribute_static_init_4(ptr nofree readonly captures(none), i32, i32, ptr nofree captures(none), ptr nofree captures(none), ptr nofree captures(none), ptr nofree captures(none), i32, i32)
 
 ; OPTIMISTIC: ; Function Attrs: nofree nosync nounwind willreturn memory(argmem: readwrite, inaccessiblemem: read)
-; OPTIMISTIC: declare void @__kmpc_distribute_static_init_4u(ptr nocapture nofree readonly, i32, i32, ptr nocapture nofree, ptr nocapture nofree, ptr nocapture nofree, ptr nocapture nofree, i32, i32)
+; OPTIMISTIC: declare void @__kmpc_distribute_static_init_4u(ptr nofree readonly captures(none), i32, i32, ptr nofree captures(none), ptr nofree captures(none), ptr nofree captures(none), ptr nofree captures(none), i32, i32)
 
 ; OPTIMISTIC: ; Function Attrs: nofree nosync nounwind willreturn memory(argmem: readwrite, inaccessiblemem: read)
-; OPTIMISTIC: declare void @__kmpc_distribute_static_init_8(ptr nocapture nofree readonly, i32, i32, ptr nocapture nofree, ptr nocapture nofree, ptr nocapture nofree, ptr nocapture nofree, i64, i64)
+; OPTIMISTIC: declare void @__kmpc_distribute_static_init_8(ptr nofree readonly captures(none), i32, i32, ptr nofree captures(none), ptr nofree captures(none), ptr nofree captures(none), ptr nofree captures(none), i64, i64)
 
 ; OPTIMISTIC: ; Function Attrs: nofree nosync nounwind willreturn memory(argmem: readwrite, inaccessiblemem: read)
-; OPTIMISTIC: declare void @__kmpc_distribute_static_init_8u(ptr nocapture nofree readonly, i32, i32, ptr nocapture nofree, ptr nocapture nofree, ptr nocapture nofree, ptr nocapture nofree, i64, i64)
+; OPTIMISTIC: declare void @__kmpc_distribute_static_init_8u(ptr nofree readonly captures(none), i32, i32, ptr nofree captures(none), ptr nofree captures(none), ptr nofree captures(none), ptr nofree captures(none), i64, i64)
 
 ; OPTIMISTIC: ; Function Attrs: nofree nosync nounwind willreturn memory(argmem: readwrite, inaccessiblemem: readwrite)
-; OPTIMISTIC: declare void @__kmpc_end_masked(ptr nocapture nofree readonly, i32)
+; OPTIMISTIC: declare void @__kmpc_end_masked(ptr nofree readonly captures(none), i32)
 
 ; OPTIMISTIC-NOT: Function Attrs
 ; OPTIMISTIC: declare void @__kmpc_end_sharing_variables()
@@ -1969,10 +1969,10 @@ declare i32 @__tgt_target_kernel_nowait(ptr, i64, i32, i32, ptr, ptr, i32, ptr,
 ; OPTIMISTIC: declare void @__kmpc_error(ptr, i32, ptr)
 
 ; OPTIMISTIC-NOT: Function Attrs
-; OPTIMISTIC: declare void @__kmpc_fork_call_if(ptr nocapture nofree readonly, i32, ptr nocapture nofree readonly, i32, ptr)
+; OPTIMISTIC: declare void @__kmpc_fork_call_if(ptr nofree readonly captures(none), i32, ptr nofree readonly captures(none), i32, ptr)
 
 ; OPTIMISTIC: ; Function Attrs: nosync nounwind
-; OPTIMISTIC-NEXT: declare void @__kmpc_free_shared(ptr allocptr nocapture, i64)
+; OPTIMISTIC-NEXT: declare void @__kmpc_free_shared(ptr allocptr captures(none), i64)
 
 ; OPTIMISTIC: ; Function Attrs: nofree nosync nounwind willreturn memory(inaccessiblemem: read)
 ; OPTIMISTIC-NEXT: declare i32 @__kmpc_get_hardware_num_blocks()
@@ -2002,7 +2002,7 @@ declare i32 @__tgt_target_kernel_nowait(ptr, i64, i32, i32, ptr, ptr, i32, ptr,
 ; OPTIMISTIC: declare void @__kmpc_kernel_prepare_parallel(ptr)
 
 ; OPTIMISTIC: ; Function Attrs: nofree nosync nounwind willreturn memory(argmem: readwrite, inaccessiblemem: readwrite)
-; OPTIMISTIC: declare i32 @__kmpc_masked(ptr nocapture nofree readonly, i32, i32)
+; OPTIMISTIC: declare i32 @__kmpc_masked(ptr nofree readonly captures(none), i32, i32)
 
 ; OPTIMISTIC-NOT: Function Attrs
 ; OPTIMISTIC: declare i32 @__kmpc_nvptx_parallel_reduce_nowait_v2(ptr, i64, ptr, ptr, ptr)
@@ -2011,7 +2011,7 @@ declare i32 @__tgt_target_kernel_nowait(ptr, i64, i32, i32, ptr, ptr, i32, ptr,
 ; OPTIMISTIC: declare i32 @__kmpc_nvptx_teams_reduce_nowait_v2(ptr, ptr, i32, i64, ptr, ptr, ptr, ptr, ptr, ptr, ptr)
 
 ; OPTIMISTIC: nofree nosync nounwind willreturn
-; OPTIMISTIC: declare i32 @__kmpc_omp_reg_task_with_affinity(ptr nocapture nofree readonly, i32, ptr nocapture nofree readonly, i32, ptr nocapture nofree readonly)
+; OPTIMISTIC: declare i32 @__kmpc_omp_reg_task_with_affinity(ptr nofree readonly captures(none), i32, ptr nofree readonly captures(none), i32, ptr nofree readonly captures(none))
 
 ; OPTIMISTIC: alwaysinline
 ; OPTIMISTIC: declare void @__kmpc_parallel_51(ptr, i32, i32, i32, i32, ptr, ptr, ptr, i64)
@@ -2111,7 +2111,7 @@ declare i32 @__tgt_target_kernel_nowait(ptr, i64, i32, i32, ptr, ptr, i32, ptr,
 ; EXT-NEXT: declare dso_local signext i32 @omp_get_max_active_levels()
 
 ; EXT: ; Function Attrs: nounwind
-; EXT-NEXT: declare dso_local void @omp_get_schedule(ptr nocapture writeonly, ptr nocapture writeonly)
+; EXT-NEXT: declare dso_local void @omp_get_schedule(ptr writeonly captures(none), ptr writeonly captures(none))
 
 ; EXT-NOT: Function Attrs
 ; EXT: declare dso_local i32 @omp_get_max_task_priority()
@@ -2219,7 +2219,7 @@ declare i32 @__tgt_target_kernel_nowait(ptr, i64, i32, i32, ptr, ptr, i32, ptr,
 ; EXT: declare dso_local i32 @omp_get_place_num_procs(i32)
 
 ; EXT: ; Function Attrs: nounwind
-; EXT-NEXT: declare dso_local void @omp_get_place_proc_ids(i32 signext, ptr nocapture writeonly)
+; EXT-NEXT: declare dso_local void @omp_get_place_proc_ids(i32 signext, ptr writeonly captures(none))
 
 ; EXT: ; Function Attrs: nounwind
 ; EXT-NEXT: declare dso_local signext i32 @omp_get_place_num()
@@ -2588,7 +2588,7 @@ declare i32 @__tgt_target_kernel_nowait(ptr, i64, i32, i32, ptr, ptr, i32, ptr,
 ; EXT-NEXT: declare void @__kmpc_proxy_task_completed_ooo(ptr)
 
 ; EXT: ; Function Attrs: cold convergent noinline nounwind
-; EXT-NEXT: declare void @__kmpc_barrier_simple_spmd(ptr nocapture nofree readonly, i32 signext)
+; EXT-NEXT: declare void @__kmpc_barrier_simple_spmd(ptr nofree readonly captures(none), i32 signext)
 
 ; EXT: ; Function Attrs: nounwind
 ; EXT-NEXT: declare noalias ptr @__kmpc_aligned_alloc(i32 signext, i64, i64, ptr)
@@ -2630,7 +2630,7 @@ declare i32 @__tgt_target_kernel_nowait(ptr, i64, i32, i32, ptr, ptr, i32, ptr,
 ; EXT: declare void @__kmpc_fork_call_if(ptr, i32 signext, ptr, i32 signext, ptr)
 
 ; EXT: ; Function Attrs: nosync nounwind
-; EXT-NEXT: declare void @__kmpc_free_shared(ptr allocptr nocapture, i64)
+; EXT-NEXT: declare void @__kmpc_free_shared(ptr allocptr captures(none), i64)
 
 ; EXT: ; Function Attrs: nounwind
 ; EXT-NEXT: declare zeroext i32 @__kmpc_get_hardware_num_blocks()
diff --git a/llvm/test/Transforms/OpenMP/always_inline_device.ll b/llvm/test/Transforms/OpenMP/always_inline_device.ll
index 6028ff5278037..9c5b19f7a6c88 100644
--- a/llvm/test/Transforms/OpenMP/always_inline_device.ll
+++ b/llvm/test/Transforms/OpenMP/always_inline_device.ll
@@ -17,7 +17,7 @@
 ; CHECK: @G = external global i8
 ; CHECK: @kernel_environment = local_unnamed_addr constant %struct.KernelEnvironmentTy { %struct.ConfigurationEnvironmentTy { i8 0, i8 0, i8 3, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0 }, ptr @[[GLOB1]], ptr null }
 ;.
-define weak void @__omp_offloading_fd02_c0934fc2_foo_l4(ptr %dyn) #0 {
+define weak ptx_kernel void @__omp_offloading_fd02_c0934fc2_foo_l4(ptr %dyn) #0 {
 ; CHECK: Function Attrs: norecurse nounwind
 ; CHECK-LABEL: @__omp_offloading_fd02_c0934fc2_foo_l4(
 ; CHECK-NEXT:  entry:
@@ -79,12 +79,10 @@ attributes #1 = { convergent nounwind "frame-pointer"="all" "min-legal-vector-wi
 attributes #2 = { convergent }
 
 !omp_offload.info = !{!0}
-!nvvm.annotations = !{!1}
 !llvm.module.flags = !{!2, !3, !4, !5, !6}
 !llvm.ident = !{!7}
 
 !0 = !{i32 0, i32 64770, i32 -1064087614, !"foo", i32 4, i32 0}
-!1 = !{ptr @__omp_offloading_fd02_c0934fc2_foo_l4, !"kernel", i32 1}
 !2 = !{i32 1, !"wchar_size", i32 4}
 !3 = !{i32 7, !"openmp", i32 50}
 !4 = !{i32 7, !"openmp-device", i32 50}
@@ -97,11 +95,10 @@ attributes #2 = { convergent }
 ; CHECK: attributes #[[ATTR2:[0-9]+]] = { nounwind }
 ;.
 ; CHECK: [[META0:![0-9]+]] = !{i32 0, i32 64770, i32 -1064087614, !"foo", i32 4, i32 0}
-; CHECK: [[META1:![0-9]+]] = !{ptr @__omp_offloading_fd02_c0934fc2_foo_l4, !"kernel", i32 1}
-; CHECK: [[META2:![0-9]+]] = !{i32 1, !"wchar_size", i32 4}
-; CHECK: [[META3:![0-9]+]] = !{i32 7, !"openmp", i32 50}
-; CHECK: [[META4:![0-9]+]] = !{i32 7, !"openmp-device", i32 50}
-; CHECK: [[META5:![0-9]+]] = !{i32 8, !"PIC Level", i32 2}
-; CHECK: [[META6:![0-9]+]] = !{i32 7, !"frame-pointer", i32 2}
-; CHECK: [[META7:![0-9]+]] = !{!"{{.*}}clang version {{.*}}"}
+; CHECK: [[META1:![0-9]+]] = !{i32 1, !"wchar_size", i32 4}
+; CHECK: [[META2:![0-9]+]] = !{i32 7, !"openmp", i32 50}
+; CHECK: [[META3:![0-9]+]] = !{i32 7, !"openmp-device", i32 50}
+; CHECK: [[META4:![0-9]+]] = !{i32 8, !"PIC Level", i32 2}
+; CHECK: [[META5:![0-9]+]] = !{i32 7, !"frame-pointer", i32 2}
+; CHECK: [[META6:![0-9]+]] = !{!"{{.*}}clang version {{.*}}"}
 ;.
diff --git a/llvm/test/Transforms/OpenMP/attributor_module_slice_reproducer.ll b/llvm/test/Transforms/OpenMP/attributor_module_slice_reproducer.ll
index 9c0416af359d4..3f4790ee15ac8 100644
--- a/llvm/test/Transforms/OpenMP/attributor_module_slice_reproducer.ll
+++ b/llvm/test/Transforms/OpenMP/attributor_module_slice_reproducer.ll
@@ -13,10 +13,6 @@ define linkonce_odr hidden i8 @_ZStplIdESt7complexIT_ERKS2_S4_() local_unnamed_a
   ret i8 undef
 }
 
-declare void @__omp_offloading_2b_4010cad__ZN11qmcplusplus7ompBLAS17gemv_batched_implIfEEiRiciiPKT_PKS5_iS7_iS5_PKPS3_ii_l148(i64, i64, i64, ptr, ptr, i64, ptr, ptr, ptr, i64) local_unnamed_addr
+declare ptx_kernel void @__omp_offloading_2b_4010cad__ZN11qmcplusplus7ompBLAS17gemv_batched_implIfEEiRiciiPKT_PKS5_iS7_iS5_PKPS3_ii_l148(i64, i64, i64, ptr, ptr, i64, ptr, ptr, ptr, i64) local_unnamed_addr
 
 declare dso_local fastcc void @__kmpc_for_static_init_8u() unnamed_addr
-
-!nvvm.annotations = !{!0}
-
-!0 = !{ptr @__omp_offloading_2b_4010cad__ZN11qmcplusplus7ompBLAS17gemv_batched_implIfEEiRiciiPKT_PKS5_iS7_iS5_PKPS3_ii_l148, !"kernel", i32 1}
diff --git a/llvm/test/Transforms/OpenMP/barrier_removal.ll b/llvm/test/Transforms/OpenMP/barrier_removal.ll
index 47a5d5104aa8b..5b7544b1a7961 100644
--- a/llvm/test/Transforms/OpenMP/barrier_removal.ll
+++ b/llvm/test/Transforms/OpenMP/barrier_removal.ll
@@ -28,7 +28,7 @@ declare void @llvm.assume(i1)
 ; CHECK: @G1 = global i32 42
 ; CHECK: @G2 = addrspace(1) global i32 0
 ;.
-define void @pos_empty_1(i1 %c) "kernel" {
+define amdgpu_kernel void @pos_empty_1(i1 %c) "kernel" {
 ; MODULE-LABEL: define {{[^@]+}}@pos_empty_1
 ; MODULE-SAME: (i1 [[C:%.*]]) #[[ATTR4:[0-9]+]] {
 ; MODULE-NEXT:    ret void
@@ -45,7 +45,7 @@ define void @pos_empty_1(i1 %c) "kernel" {
   call void @llvm.assume(i1 %c)
   ret void
 }
-define void @pos_empty_2() "kernel" {
+define amdgpu_kernel void @pos_empty_2() "kernel" {
 ; CHECK-LABEL: define {{[^@]+}}@pos_empty_2
 ; CHECK-SAME: () #[[ATTR4:[0-9]+]] {
 ; CHECK-NEXT:    ret void
@@ -53,7 +53,7 @@ define void @pos_empty_2() "kernel" {
   call void @aligned_barrier()
   ret void
 }
-define void @pos_empty_3() "kernel" {
+define amdgpu_kernel void @pos_empty_3() "kernel" {
 ; CHECK-LABEL: define {{[^@]+}}@pos_empty_3
 ; CHECK-SAME: () #[[ATTR4]] {
 ; CHECK-NEXT:    ret void
@@ -61,7 +61,7 @@ define void @pos_empty_3() "kernel" {
   call void @llvm.nvvm.barrier0()
   ret void
 }
-define void @pos_empty_4() "kernel" {
+define amdgpu_kernel void @pos_empty_4() "kernel" {
 ; CHECK-LABEL: define {{[^@]+}}@pos_empty_4
 ; CHECK-SAME: () #[[ATTR4]] {
 ; CHECK-NEXT:    ret void
@@ -69,7 +69,7 @@ define void @pos_empty_4() "kernel" {
   call i32 @llvm.nvvm.barrier0.and(i32 0)
   ret void
 }
-define void @pos_empty_5() "kernel" {
+define amdgpu_kernel void @pos_empty_5() "kernel" {
 ; CHECK-LABEL: define {{[^@]+}}@pos_empty_5
 ; CHECK-SAME: () #[[ATTR4]] {
 ; CHECK-NEXT:    ret void
@@ -77,7 +77,7 @@ define void @pos_empty_5() "kernel" {
   call i32 @llvm.nvvm.barrier0.or(i32 0)
   ret void
 }
-define void @pos_empty_6() "kernel" {
+define amdgpu_kernel void @pos_empty_6() "kernel" {
 ; CHECK-LABEL: define {{[^@]+}}@pos_empty_6
 ; CHECK-SAME: () #[[ATTR4]] {
 ; CHECK-NEXT:    ret void
@@ -85,7 +85,7 @@ define void @pos_empty_6() "kernel" {
   call i32 @llvm.nvvm.barrier0.popc(i32 0)
   ret void
 }
-define void @pos_empty_7a() "kernel" {
+define amdgpu_kernel void @pos_empty_7a() "kernel" {
 ; CHECK-LABEL: define {{[^@]+}}@pos_empty_7a
 ; CHECK-SAME: () #[[ATTR4]] {
 ; CHECK-NEXT:    call void @unknown()
@@ -96,7 +96,7 @@ define void @pos_empty_7a() "kernel" {
   ret void
 }
 ; FIXME: We should remove the barrier.
-define void @pos_empty_7b() "kernel" {
+define amdgpu_kernel void @pos_empty_7b() "kernel" {
 ; CHECK-LABEL: define {{[^@]+}}@pos_empty_7b
 ; CHECK-SAME: () #[[ATTR4]] {
 ; CHECK-NEXT:    call void @unknown() #[[ATTR5:[0-9]+]]
@@ -109,7 +109,7 @@ define void @pos_empty_7b() "kernel" {
   call void @unknown()
   ret void
 }
-define void @pos_empty_8(i1 %c) "kernel" {
+define amdgpu_kernel void @pos_empty_8(i1 %c) "kernel" {
 ; CHECK-LABEL: define {{[^@]+}}@pos_empty_8
 ; CHECK-SAME: (i1 [[C:%.*]]) #[[ATTR4]] {
 ; CHECK-NEXT:    br i1 [[C]], label [[T:%.*]], label [[F:%.*]]
@@ -126,7 +126,7 @@ t:
 f:
   ret void
 }
-define void @neg_empty_8() "kernel" {
+define amdgpu_kernel void @neg_empty_8() "kernel" {
 ; CHECK-LABEL: define {{[^@]+}}@neg_empty_8
 ; CHECK-SAME: () #[[ATTR4]] {
 ; CHECK-NEXT:    call void @unknown()
@@ -137,7 +137,7 @@ define void @neg_empty_8() "kernel" {
   call void @llvm.amdgcn.s.barrier()
   ret void
 }
-define void @neg_empty_9(i1 %c) "kernel" {
+define amdgpu_kernel void @neg_empty_9(i1 %c) "kernel" {
 ; CHECK-LABEL: define {{[^@]+}}@neg_empty_9
 ; CHECK-SAME: (i1 [[C:%.*]]) #[[ATTR4]] {
 ; CHECK-NEXT:    br i1 [[C]], label [[T:%.*]], label [[F:%.*]]
@@ -173,7 +173,7 @@ m:
   ret void
 }
 ; FIXME: We should remove the barrier
-define void @pos_empty_10() "kernel" {
+define amdgpu_kernel void @pos_empty_10() "kernel" {
 ; CHECK-LABEL: define {{[^@]+}}@pos_empty_10
 ; CHECK-SAME: () #[[ATTR4]] {
 ; CHECK-NEXT:    br label [[M:%.*]]
@@ -186,7 +186,7 @@ m:
   call void @llvm.amdgcn.s.barrier()
   ret void
 }
-define void @pos_empty_11() "kernel" {
+define amdgpu_kernel void @pos_empty_11() "kernel" {
 ; CHECK-LABEL: define {{[^@]+}}@pos_empty_11
 ; CHECK-SAME: () #[[ATTR4]] {
 ; CHECK-NEXT:    br label [[M:%.*]]
@@ -206,7 +206,7 @@ define void @empty() {
   ret void
 }
 ; FIXME: We should remove the barrier in the end but not the first one.
-define void @neg_empty_12(i1 %c) "kernel" {
+define amdgpu_kernel void @neg_empty_12(i1 %c) "kernel" {
 ; MODULE-LABEL: define {{[^@]+}}@neg_empty_12
 ; MODULE-SAME: (i1 [[C:%.*]]) #[[ATTR4]] {
 ; MODULE-NEXT:    br i1 [[C]], label [[T:%.*]], label [[F:%.*]]
@@ -266,7 +266,7 @@ define void @neg_empty_2() "kernel" {
 @GC1 = constant i32 42
 @GC2 = addrspace(4) global i32 0
 @GPtr4 = addrspace(4) global ptr addrspace(4) null
-define void @pos_constant_loads() "kernel" {
+define amdgpu_kernel void @pos_constant_loads() "kernel" {
 ; CHECK-LABEL: define {{[^@]+}}@pos_constant_loads
 ; CHECK-SAME: () #[[ATTR4]] {
 ; CHECK-NEXT:    [[ARG:%.*]] = load ptr addrspace(4), ptr addrspace(4) @GPtr4, align 8
@@ -296,7 +296,7 @@ define void @pos_constant_loads() "kernel" {
 @GS = addrspace(3) global i32 0
 @GPtr = global ptr null
 ; TODO: We could remove some of the barriers due to the lack of write effects.
-define void @neg_loads() "kernel" {
+define amdgpu_kernel void @neg_loads() "kernel" {
 ; CHECK-LABEL: define {{[^@]+}}@neg_loads
 ; CHECK-SAME: () #[[ATTR4]] {
 ; CHECK-NEXT:    [[ARG:%.*]] = load ptr, ptr @GPtr, align 8
@@ -327,7 +327,7 @@ define void @neg_loads() "kernel" {
 @PG1 = thread_local global i32 42
 @PG2 = addrspace(5) global i32 0
 @GPtr5 = global ptr addrspace(5) null
-define void @pos_priv_mem() "kernel" {
+define amdgpu_kernel void @pos_priv_mem() "kernel" {
 ; CHECK-LABEL: define {{[^@]+}}@pos_priv_mem
 ; CHECK-SAME: () #[[ATTR4]] {
 ; CHECK-NEXT:    [[ARG:%.*]] = load ptr addrspace(5), ptr @GPtr5, align 4
@@ -358,7 +358,7 @@ define void @pos_priv_mem() "kernel" {
 }
 @G1 = global i32 42
 @G2 = addrspace(1) global i32 0
-define void @neg_mem() "kernel" {
+define amdgpu_kernel void @neg_mem() "kernel" {
 ; CHECK-LABEL: define {{[^@]+}}@neg_mem
 ; CHECK-SAME: () #[[ATTR4]] {
 ; CHECK-NEXT:    [[ARG:%.*]] = load ptr, ptr @GPtr, align 8
@@ -388,7 +388,7 @@ define void @neg_mem() "kernel" {
   ret void
 }
 
-define void @pos_multiple() "kernel" {
+define amdgpu_kernel void @pos_multiple() "kernel" {
 ; CHECK-LABEL: define {{[^@]+}}@pos_multiple
 ; CHECK-SAME: () #[[ATTR4]] {
 ; CHECK-NEXT:    ret void
@@ -404,7 +404,7 @@ define void @pos_multiple() "kernel" {
   ret void
 }
 
-define void @multiple_blocks_kernel_1(i1 %c0, i1 %c1) "kernel" {
+define amdgpu_kernel void @multiple_blocks_kernel_1(i1 %c0, i1 %c1) "kernel" {
 ; CHECK-LABEL: define {{[^@]+}}@multiple_blocks_kernel_1
 ; CHECK-SAME: (i1 [[C0:%.*]], i1 [[C1:%.*]]) #[[ATTR4]] {
 ; CHECK-NEXT:    br i1 [[C0]], label [[T0:%.*]], label [[F0:%.*]]
@@ -461,7 +461,7 @@ m:
   ret void
 }
 
-define void @multiple_blocks_kernel_2(i1 %c0, i1 %c1, ptr %p) "kernel" {
+define amdgpu_kernel void @multiple_blocks_kernel_2(i1 %c0, i1 %c1, ptr %p) "kernel" {
 ; CHECK-LABEL: define {{[^@]+}}@multiple_blocks_kernel_2
 ; CHECK-SAME: (i1 [[C0:%.*]], i1 [[C1:%.*]], ptr [[P:%.*]]) #[[ATTR4]] {
 ; CHECK-NEXT:    store i32 4, ptr [[P]], align 4
@@ -727,7 +727,7 @@ define internal void @barrier_then_write_then_barrier0(ptr %p) {
   call void @aligned_barrier()
   ret void
 }
-define void @multiple_blocks_functions_kernel_effects_0(i1 %c0, i1 %c1, ptr %p) "kernel" {
+define amdgpu_kernel void @multiple_blocks_functions_kernel_effects_0(i1 %c0, i1 %c1, ptr %p) "kernel" {
 ; MODULE-LABEL: define {{[^@]+}}@multiple_blocks_functions_kernel_effects_0
 ; MODULE-SAME: (i1 [[C0:%.*]], i1 [[C1:%.*]], ptr [[P:%.*]]) #[[ATTR4]] {
 ; MODULE-NEXT:    call void @barrier_then_write_then_barrier0(ptr [[P]])
@@ -1040,7 +1040,7 @@ define internal void @callee_barrier() {
   call void @aligned_barrier()
   ret void
 }
-define void @caller_barrier1() "kernel" {
+define amdgpu_kernel void @caller_barrier1() "kernel" {
 ; CHECK-LABEL: define {{[^@]+}}@caller_barrier1
 ; CHECK-SAME: () #[[ATTR4]] {
 ; CHECK-NEXT:    call void @callee_barrier()
@@ -1051,7 +1051,7 @@ define void @caller_barrier1() "kernel" {
   call void @aligned_barrier()
   ret void
 }
-define void @caller_barrier2() "kernel" {
+define amdgpu_kernel void @caller_barrier2() "kernel" {
 ; CHECK-LABEL: define {{[^@]+}}@caller_barrier2
 ; CHECK-SAME: () #[[ATTR4]] {
 ; CHECK-NEXT:    call void @unknown()
@@ -1065,7 +1065,7 @@ define void @caller_barrier2() "kernel" {
   ret void
 }
 
-define void @loop_barrier() "kernel" {
+define amdgpu_kernel void @loop_barrier() "kernel" {
 ; CHECK-LABEL: define {{[^@]+}}@loop_barrier
 ; CHECK-SAME: () #[[ATTR4]] {
 ; CHECK-NEXT:  entry:
@@ -1095,7 +1095,7 @@ exit:
   ret void
 }
 
-define void @loop_barrier_end_barriers() "kernel" {
+define amdgpu_kernel void @loop_barrier_end_barriers() "kernel" {
 ; CHECK-LABEL: define {{[^@]+}}@loop_barrier_end_barriers
 ; CHECK-SAME: () #[[ATTR4]] {
 ; CHECK-NEXT:  entry:
@@ -1129,7 +1129,7 @@ exit:
   ret void
 }
 
-define void @loop_barrier_end_barriers_unknown() "kernel" {
+define amdgpu_kernel void @loop_barrier_end_barriers_unknown() "kernel" {
 ; CHECK-LABEL: define {{[^@]+}}@loop_barrier_end_barriers_unknown
 ; CHECK-SAME: () #[[ATTR4]] {
 ; CHECK-NEXT:  entry:
@@ -1165,7 +1165,7 @@ exit:
   ret void
 }
 
-define void @loop_barrier_store() "kernel" {
+define amdgpu_kernel void @loop_barrier_store() "kernel" {
 ; CHECK-LABEL: define {{[^@]+}}@loop_barrier_store
 ; CHECK-SAME: () #[[ATTR4]] {
 ; CHECK-NEXT:  entry:
@@ -1195,7 +1195,7 @@ exit:
   ret void
 }
 
-define void @loop_barrier_end_barriers_store() "kernel" {
+define amdgpu_kernel void @loop_barrier_end_barriers_store() "kernel" {
 ; CHECK-LABEL: define {{[^@]+}}@loop_barrier_end_barriers_store
 ; CHECK-SAME: () #[[ATTR4]] {
 ; CHECK-NEXT:  entry:
@@ -1232,37 +1232,7 @@ exit:
 }
 
 !llvm.module.flags = !{!16,!15}
-!nvvm.annotations = !{!0,!1,!2,!3,!4,!5,!6,!7,!8,!9,!10,!11,!12,!13,!14,!17,!18,!19,!20,!21,!22,!23,!24,!25,!26,!27,!28,!29,!30}
 
-!0 = !{ptr @pos_empty_1, !"kernel", i32 1}
-!1 = !{ptr @pos_empty_2, !"kernel", i32 1}
-!2 = !{ptr @pos_empty_3, !"kernel", i32 1}
-!3 = !{ptr @pos_empty_4, !"kernel", i32 1}
-!4 = !{ptr @pos_empty_5, !"kernel", i32 1}
-!5 = !{ptr @pos_empty_6, !"kernel", i32 1}
-!17 = !{ptr @pos_empty_7a, !"kernel", i32 1}
-!18 = !{ptr @pos_empty_7b, !"kernel", i32 1}
-!23 = !{ptr @pos_empty_8, !"kernel", i32 1}
-!24 = !{ptr @caller_barrier1, !"kernel", i32 1}
-!25 = !{ptr @caller_barrier2, !"kernel", i32 1}
-!26 = !{ptr @loop_barrier, !"kernel", i32 1}
-!27 = !{ptr @loop_barrier_end_barriers, !"kernel", i32 1}
-!28 = !{ptr @loop_barrier_end_barriers_unknown, !"kernel", i32 1}
-!29 = !{ptr @loop_barrier_store, !"kernel", i32 1}
-!30 = !{ptr @loop_barrier_end_barriers_store, !"kernel", i32 1}
-!6 = !{ptr @neg_empty_8, !"kernel", i32 1}
-!19 = !{ptr @neg_empty_9, !"kernel", i32 1}
-!20 = !{ptr @pos_empty_10, !"kernel", i32 1}
-!21 = !{ptr @pos_empty_11, !"kernel", i32 1}
-!22 = !{ptr @neg_empty_12, !"kernel", i32 1}
-!7 = !{ptr @pos_constant_loads, !"kernel", i32 1}
-!8 = !{ptr @neg_loads, !"kernel", i32 1}
-!9 = !{ptr @pos_priv_mem, !"kernel", i32 1}
-!10 = !{ptr @neg_mem, !"kernel", i32 1}
-!11 = !{ptr @pos_multiple, !"kernel", i32 1}
-!12 = !{ptr @multiple_blocks_kernel_1, !"kernel", i32 1}
-!13 = !{ptr @multiple_blocks_kernel_2, !"kernel", i32 1}
-!14 = !{ptr @multiple_blocks_functions_kernel_effects_0, !"kernel", i32 1}
 !15 = !{i32 7, !"openmp", i32 50}
 !16 = !{i32 7, !"openmp-device", i32 50}
 ;.
@@ -1282,65 +1252,7 @@ exit:
 ;.
 ; MODULE: [[META0:![0-9]+]] = !{i32 7, !"openmp-device", i32 50}
 ; MODULE: [[META1:![0-9]+]] = !{i32 7, !"openmp", i32 50}
-; MODULE: [[META2:![0-9]+]] = !{ptr @pos_empty_1, !"kernel", i32 1}
-; MODULE: [[META3:![0-9]+]] = !{ptr @pos_empty_2, !"kernel", i32 1}
-; MODULE: [[META4:![0-9]+]] = !{ptr @pos_empty_3, !"kernel", i32 1}
-; MODULE: [[META5:![0-9]+]] = !{ptr @pos_empty_4, !"kernel", i32 1}
-; MODULE: [[META6:![0-9]+]] = !{ptr @pos_empty_5, !"kernel", i32 1}
-; MODULE: [[META7:![0-9]+]] = !{ptr @pos_empty_6, !"kernel", i32 1}
-; MODULE: [[META8:![0-9]+]] = !{ptr @neg_empty_8, !"kernel", i32 1}
-; MODULE: [[META9:![0-9]+]] = !{ptr @pos_constant_loads, !"kernel", i32 1}
-; MODULE: [[META10:![0-9]+]] = !{ptr @neg_loads, !"kernel", i32 1}
-; MODULE: [[META11:![0-9]+]] = !{ptr @pos_priv_mem, !"kernel", i32 1}
-; MODULE: [[META12:![0-9]+]] = !{ptr @neg_mem, !"kernel", i32 1}
-; MODULE: [[META13:![0-9]+]] = !{ptr @pos_multiple, !"kernel", i32 1}
-; MODULE: [[META14:![0-9]+]] = !{ptr @multiple_blocks_kernel_1, !"kernel", i32 1}
-; MODULE: [[META15:![0-9]+]] = !{ptr @multiple_blocks_kernel_2, !"kernel", i32 1}
-; MODULE: [[META16:![0-9]+]] = !{ptr @multiple_blocks_functions_kernel_effects_0, !"kernel", i32 1}
-; MODULE: [[META17:![0-9]+]] = !{ptr @pos_empty_7a, !"kernel", i32 1}
-; MODULE: [[META18:![0-9]+]] = !{ptr @pos_empty_7b, !"kernel", i32 1}
-; MODULE: [[META19:![0-9]+]] = !{ptr @neg_empty_9, !"kernel", i32 1}
-; MODULE: [[META20:![0-9]+]] = !{ptr @pos_empty_10, !"kernel", i32 1}
-; MODULE: [[META21:![0-9]+]] = !{ptr @pos_empty_11, !"kernel", i32 1}
-; MODULE: [[META22:![0-9]+]] = !{ptr @neg_empty_12, !"kernel", i32 1}
-; MODULE: [[META23:![0-9]+]] = !{ptr @pos_empty_8, !"kernel", i32 1}
-; MODULE: [[META24:![0-9]+]] = !{ptr @caller_barrier1, !"kernel", i32 1}
-; MODULE: [[META25:![0-9]+]] = !{ptr @caller_barrier2, !"kernel", i32 1}
-; MODULE: [[META26:![0-9]+]] = !{ptr @loop_barrier, !"kernel", i32 1}
-; MODULE: [[META27:![0-9]+]] = !{ptr @loop_barrier_end_barriers, !"kernel", i32 1}
-; MODULE: [[META28:![0-9]+]] = !{ptr @loop_barrier_end_barriers_unknown, !"kernel", i32 1}
-; MODULE: [[META29:![0-9]+]] = !{ptr @loop_barrier_store, !"kernel", i32 1}
-; MODULE: [[META30:![0-9]+]] = !{ptr @loop_barrier_end_barriers_store, !"kernel", i32 1}
 ;.
 ; CGSCC: [[META0:![0-9]+]] = !{i32 7, !"openmp-device", i32 50}
 ; CGSCC: [[META1:![0-9]+]] = !{i32 7, !"openmp", i32 50}
-; CGSCC: [[META2:![0-9]+]] = !{ptr @pos_empty_1, !"kernel", i32 1}
-; CGSCC: [[META3:![0-9]+]] = !{ptr @pos_empty_2, !"kernel", i32 1}
-; CGSCC: [[META4:![0-9]+]] = !{ptr @pos_empty_3, !"kernel", i32 1}
-; CGSCC: [[META5:![0-9]+]] = !{ptr @pos_empty_4, !"kernel", i32 1}
-; CGSCC: [[META6:![0-9]+]] = !{ptr @pos_empty_5, !"kernel", i32 1}
-; CGSCC: [[META7:![0-9]+]] = !{ptr @pos_empty_6, !"kernel", i32 1}
-; CGSCC: [[META8:![0-9]+]] = !{ptr @neg_empty_8, !"kernel", i32 1}
-; CGSCC: [[META9:![0-9]+]] = !{ptr @pos_constant_loads, !"kernel", i32 1}
-; CGSCC: [[META10:![0-9]+]] = !{ptr @neg_loads, !"kernel", i32 1}
-; CGSCC: [[META11:![0-9]+]] = !{ptr @pos_priv_mem, !"kernel", i32 1}
-; CGSCC: [[META12:![0-9]+]] = !{ptr @neg_mem, !"kernel", i32 1}
-; CGSCC: [[META13:![0-9]+]] = !{ptr @pos_multiple, !"kernel", i32 1}
-; CGSCC: [[META14:![0-9]+]] = !{ptr @multiple_blocks_kernel_1, !"kernel", i32 1}
-; CGSCC: [[META15:![0-9]+]] = !{ptr @multiple_blocks_kernel_2, !"kernel", i32 1}
-; CGSCC: [[META16:![0-9]+]] = !{ptr @multiple_blocks_functions_kernel_effects_0, !"kernel", i32 1}
-; CGSCC: [[META17:![0-9]+]] = !{ptr @pos_empty_7a, !"kernel", i32 1}
-; CGSCC: [[META18:![0-9]+]] = !{ptr @pos_empty_7b, !"kernel", i32 1}
-; CGSCC: [[META19:![0-9]+]] = !{ptr @neg_empty_9, !"kernel", i32 1}
-; CGSCC: [[META20:![0-9]+]] = !{ptr @pos_empty_10, !"kernel", i32 1}
-; CGSCC: [[META21:![0-9]+]] = !{ptr @pos_empty_11, !"kernel", i32 1}
-; CGSCC: [[META22:![0-9]+]] = !{ptr @neg_empty_12, !"kernel", i32 1}
-; CGSCC: [[META23:![0-9]+]] = !{ptr @pos_empty_8, !"kernel", i32 1}
-; CGSCC: [[META24:![0-9]+]] = !{ptr @caller_barrier1, !"kernel", i32 1}
-; CGSCC: [[META25:![0-9]+]] = !{ptr @caller_barrier2, !"kernel", i32 1}
-; CGSCC: [[META26:![0-9]+]] = !{ptr @loop_barrier, !"kernel", i32 1}
-; CGSCC: [[META27:![0-9]+]] = !{ptr @loop_barrier_end_barriers, !"kernel", i32 1}
-; CGSCC: [[META28:![0-9]+]] = !{ptr @loop_barrier_end_barriers_unknown, !"kernel", i32 1}
-; CGSCC: [[META29:![0-9]+]] = !{ptr @loop_barrier_store, !"kernel", i32 1}
-; CGSCC: [[META30:![0-9]+]] = !{ptr @loop_barrier_end_barriers_store, !"kernel", i32 1}
 ;.
diff --git a/llvm/test/Transforms/OpenMP/bug66687.ll b/llvm/test/Transforms/OpenMP/bug66687.ll
index e0a9b825a8804..9bb069b1735be 100644
--- a/llvm/test/Transforms/OpenMP/bug66687.ll
+++ b/llvm/test/Transforms/OpenMP/bug66687.ll
@@ -5,25 +5,22 @@ source_filename = "bug66687.ll"
 target datalayout = "e-i64:64-i128:128-v16:16-v32:32-n16:32:64"
 target triple = "nvptx64-nvidia-cuda"
 
-define weak void @openmp_kernel() "kernel" {
-; CHECK-LABEL: define weak void @openmp_kernel(
+define weak ptx_kernel void @openmp_kernel() "kernel" {
+; CHECK-LABEL: define weak ptx_kernel void @openmp_kernel(
 ; CHECK-SAME: ) #[[ATTR0:[0-9]+]] {
 ; CHECK-NEXT:    ret void
 ;
   ret void
 }
 
-define weak_odr void @non_openmp_kernel() {
-; CHECK-LABEL: define weak_odr void @non_openmp_kernel() {
+define weak_odr ptx_kernel void @non_openmp_kernel() {
+; CHECK-LABEL: define weak_odr ptx_kernel void @non_openmp_kernel() {
 ; CHECK-NEXT:    ret void
 ;
   ret void
 }
 
 !llvm.module.flags = !{!0, !1}
-!nvvm.annotations = !{!2, !3}
 
 !0 = !{i32 7, !"openmp", i32 51}
 !1 = !{i32 7, !"openmp-device", i32 51}
-!2 = !{ptr @openmp_kernel, !"kernel", i32 1}
-!3 = !{ptr @non_openmp_kernel, !"kernel", i32 1}
diff --git a/llvm/test/Transforms/OpenMP/custom_state_machines.ll b/llvm/test/Transforms/OpenMP/custom_state_machines.ll
index e6ddf16f06763..10e521bbfcc10 100644
--- a/llvm/test/Transforms/OpenMP/custom_state_machines.ll
+++ b/llvm/test/Transforms/OpenMP/custom_state_machines.ll
@@ -138,7 +138,7 @@
 @__omp_offloading_14_a36502b_simple_state_machine_interprocedural_nested_recursive_l92_kernel_environment = local_unnamed_addr constant %struct.KernelEnvironmentTy { %struct.ConfigurationEnvironmentTy { i8 1, i8 0, i8 1, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0 }, ptr @1, ptr null }
 @__omp_offloading_14_a36502b_no_state_machine_weak_callee_l112_kernel_environment = local_unnamed_addr constant %struct.KernelEnvironmentTy { %struct.ConfigurationEnvironmentTy { i8 1, i8 0, i8 1, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0 }, ptr @1, ptr null }
 
-define weak void @__omp_offloading_14_a36502b_no_state_machine_needed_l14(ptr %dyn) #0 {
+define weak ptx_kernel void @__omp_offloading_14_a36502b_no_state_machine_needed_l14(ptr %dyn) #0 {
 entry:
   %.zero.addr = alloca i32, align 4
   %.threadid_temp. = alloca i32, align 4
@@ -197,7 +197,7 @@ declare i32 @__kmpc_global_thread_num(ptr) #3
 
 declare void @__kmpc_target_deinit()
 
-define weak void @__omp_offloading_14_a36502b_simple_state_machine_l22(ptr %dyn) #0 {
+define weak ptx_kernel void @__omp_offloading_14_a36502b_simple_state_machine_l22(ptr %dyn) #0 {
 entry:
   %.zero.addr = alloca i32, align 4
   %.threadid_temp. = alloca i32, align 4
@@ -290,7 +290,7 @@ entry:
   ret void
 }
 
-define weak void @__omp_offloading_14_a36502b_simple_state_machine_interprocedural_l39(ptr %dyn) #0 {
+define weak ptx_kernel void @__omp_offloading_14_a36502b_simple_state_machine_interprocedural_l39(ptr %dyn) #0 {
 entry:
   %.zero.addr = alloca i32, align 4
   %.threadid_temp. = alloca i32, align 4
@@ -367,7 +367,7 @@ entry:
   ret void
 }
 
-define weak void @__omp_offloading_14_a36502b_simple_state_machine_with_fallback_l55(ptr %dyn) #0 {
+define weak ptx_kernel void @__omp_offloading_14_a36502b_simple_state_machine_with_fallback_l55(ptr %dyn) #0 {
 entry:
   %.zero.addr = alloca i32, align 4
   %.threadid_temp. = alloca i32, align 4
@@ -453,7 +453,7 @@ entry:
   ret void
 }
 
-define weak void @__omp_offloading_14_a36502b_simple_state_machine_no_openmp_attr_l66(ptr %dyn) #0 {
+define weak ptx_kernel void @__omp_offloading_14_a36502b_simple_state_machine_no_openmp_attr_l66(ptr %dyn) #0 {
 entry:
   %.zero.addr = alloca i32, align 4
   %.threadid_temp. = alloca i32, align 4
@@ -537,7 +537,7 @@ entry:
   ret void
 }
 
-define weak void @__omp_offloading_14_a36502b_simple_state_machine_pure_l77(ptr %dyn) #0 {
+define weak ptx_kernel void @__omp_offloading_14_a36502b_simple_state_machine_pure_l77(ptr %dyn) #0 {
 entry:
   %.zero.addr = alloca i32, align 4
   %.threadid_temp. = alloca i32, align 4
@@ -624,7 +624,7 @@ entry:
   ret void
 }
 
-define weak void @__omp_offloading_14_a36502b_simple_state_machine_interprocedural_nested_recursive_l92(ptr %dyn) #0 {
+define weak ptx_kernel void @__omp_offloading_14_a36502b_simple_state_machine_interprocedural_nested_recursive_l92(ptr %dyn) #0 {
 entry:
   %.zero.addr = alloca i32, align 4
   %.threadid_temp. = alloca i32, align 4
@@ -679,7 +679,7 @@ return:                                           ; preds = %if.end, %if.then
 
 declare i32 @omp_get_thread_num(...) #4
 
-define weak void @__omp_offloading_14_a36502b_no_state_machine_weak_callee_l112(ptr %dyn) #0 {
+define weak ptx_kernel void @__omp_offloading_14_a36502b_no_state_machine_weak_callee_l112(ptr %dyn) #0 {
 entry:
   %.zero.addr = alloca i32, align 4
   %.threadid_temp. = alloca i32, align 4
@@ -812,7 +812,6 @@ attributes #8 = { convergent "llvm.assume"="omp_no_openmp" }
 attributes #9 = { convergent nounwind readonly willreturn }
 
 !omp_offload.info = !{!0, !1, !2, !3, !4, !5, !6, !7}
-!nvvm.annotations = !{!8, !9, !10, !11, !12, !13, !14, !15}
 !llvm.module.flags = !{!16, !17, !18}
 
 !0 = !{i32 0, i32 20, i32 171331627, !"simple_state_machine_interprocedural", i32 39, i32 2}
@@ -823,14 +822,6 @@ attributes #9 = { convergent nounwind readonly willreturn }
 !5 = !{i32 0, i32 20, i32 171331627, !"simple_state_machine_interprocedural_nested_recursive", i32 92, i32 6}
 !6 = !{i32 0, i32 20, i32 171331627, !"no_state_machine_weak_callee", i32 112, i32 7}
 !7 = !{i32 0, i32 20, i32 171331627, !"simple_state_machine", i32 22, i32 1}
-!8 = !{ptr @__omp_offloading_14_a36502b_no_state_machine_needed_l14, !"kernel", i32 1}
-!9 = !{ptr @__omp_offloading_14_a36502b_simple_state_machine_l22, !"kernel", i32 1}
-!10 = !{ptr @__omp_offloading_14_a36502b_simple_state_machine_interprocedural_l39, !"kernel", i32 1}
-!11 = !{ptr @__omp_offloading_14_a36502b_simple_state_machine_with_fallback_l55, !"kernel", i32 1}
-!12 = !{ptr @__omp_offloading_14_a36502b_simple_state_machine_no_openmp_attr_l66, !"kernel", i32 1}
-!13 = !{ptr @__omp_offloading_14_a36502b_simple_state_machine_pure_l77, !"kernel", i32 1}
-!14 = !{ptr @__omp_offloading_14_a36502b_simple_state_machine_interprocedural_nested_recursive_l92, !"kernel", i32 1}
-!15 = !{ptr @__omp_offloading_14_a36502b_no_state_machine_weak_callee_l112, !"kernel", i32 1}
 !16 = !{i32 1, !"wchar_size", i32 4}
 !17 = !{i32 7, !"openmp", i32 50}
 !18 = !{i32 7, !"openmp-device", i32 50}
@@ -4107,17 +4098,9 @@ attributes #9 = { convergent nounwind readonly willreturn }
 ; AMDGPU: [[META5:![0-9]+]] = !{i32 0, i32 20, i32 171331627, !"simple_state_machine_interprocedural_nested_recursive", i32 92, i32 6}
 ; AMDGPU: [[META6:![0-9]+]] = !{i32 0, i32 20, i32 171331627, !"no_state_machine_weak_callee", i32 112, i32 7}
 ; AMDGPU: [[META7:![0-9]+]] = !{i32 0, i32 20, i32 171331627, !"simple_state_machine", i32 22, i32 1}
-; AMDGPU: [[META8:![0-9]+]] = !{ptr @__omp_offloading_14_a36502b_no_state_machine_needed_l14, !"kernel", i32 1}
-; AMDGPU: [[META9:![0-9]+]] = !{ptr @__omp_offloading_14_a36502b_simple_state_machine_l22, !"kernel", i32 1}
-; AMDGPU: [[META10:![0-9]+]] = !{ptr @__omp_offloading_14_a36502b_simple_state_machine_interprocedural_l39, !"kernel", i32 1}
-; AMDGPU: [[META11:![0-9]+]] = !{ptr @__omp_offloading_14_a36502b_simple_state_machine_with_fallback_l55, !"kernel", i32 1}
-; AMDGPU: [[META12:![0-9]+]] = !{ptr @__omp_offloading_14_a36502b_simple_state_machine_no_openmp_attr_l66, !"kernel", i32 1}
-; AMDGPU: [[META13:![0-9]+]] = !{ptr @__omp_offloading_14_a36502b_simple_state_machine_pure_l77, !"kernel", i32 1}
-; AMDGPU: [[META14:![0-9]+]] = !{ptr @__omp_offloading_14_a36502b_simple_state_machine_interprocedural_nested_recursive_l92, !"kernel", i32 1}
-; AMDGPU: [[META15:![0-9]+]] = !{ptr @__omp_offloading_14_a36502b_no_state_machine_weak_callee_l112, !"kernel", i32 1}
-; AMDGPU: [[META16:![0-9]+]] = !{i32 1, !"wchar_size", i32 4}
-; AMDGPU: [[META17:![0-9]+]] = !{i32 7, !"openmp", i32 50}
-; AMDGPU: [[META18:![0-9]+]] = !{i32 7, !"openmp-device", i32 50}
+; AMDGPU: [[META8:![0-9]+]] = !{i32 1, !"wchar_size", i32 4}
+; AMDGPU: [[META9:![0-9]+]] = !{i32 7, !"openmp", i32 50}
+; AMDGPU: [[META10:![0-9]+]] = !{i32 7, !"openmp-device", i32 50}
 ;.
 ; NVPTX: [[META0:![0-9]+]] = !{i32 0, i32 20, i32 171331627, !"simple_state_machine_interprocedural", i32 39, i32 2}
 ; NVPTX: [[META1:![0-9]+]] = !{i32 0, i32 20, i32 171331627, !"simple_state_machine_no_openmp_attr", i32 66, i32 4}
@@ -4127,17 +4110,9 @@ attributes #9 = { convergent nounwind readonly willreturn }
 ; NVPTX: [[META5:![0-9]+]] = !{i32 0, i32 20, i32 171331627, !"simple_state_machine_interprocedural_nested_recursive", i32 92, i32 6}
 ; NVPTX: [[META6:![0-9]+]] = !{i32 0, i32 20, i32 171331627, !"no_state_machine_weak_callee", i32 112, i32 7}
 ; NVPTX: [[META7:![0-9]+]] = !{i32 0, i32 20, i32 171331627, !"simple_state_machine", i32 22, i32 1}
-; NVPTX: [[META8:![0-9]+]] = !{ptr @__omp_offloading_14_a36502b_no_state_machine_needed_l14, !"kernel", i32 1}
-; NVPTX: [[META9:![0-9]+]] = !{ptr @__omp_offloading_14_a36502b_simple_state_machine_l22, !"kernel", i32 1}
-; NVPTX: [[META10:![0-9]+]] = !{ptr @__omp_offloading_14_a36502b_simple_state_machine_interprocedural_l39, !"kernel", i32 1}
-; NVPTX: [[META11:![0-9]+]] = !{ptr @__omp_offloading_14_a36502b_simple_state_machine_with_fallback_l55, !"kernel", i32 1}
-; NVPTX: [[META12:![0-9]+]] = !{ptr @__omp_offloading_14_a36502b_simple_state_machine_no_openmp_attr_l66, !"kernel", i32 1}
-; NVPTX: [[META13:![0-9]+]] = !{ptr @__omp_offloading_14_a36502b_simple_state_machine_pure_l77, !"kernel", i32 1}
-; NVPTX: [[META14:![0-9]+]] = !{ptr @__omp_offloading_14_a36502b_simple_state_machine_interprocedural_nested_recursive_l92, !"kernel", i32 1}
-; NVPTX: [[META15:![0-9]+]] = !{ptr @__omp_offloading_14_a36502b_no_state_machine_weak_callee_l112, !"kernel", i32 1}
-; NVPTX: [[META16:![0-9]+]] = !{i32 1, !"wchar_size", i32 4}
-; NVPTX: [[META17:![0-9]+]] = !{i32 7, !"openmp", i32 50}
-; NVPTX: [[META18:![0-9]+]] = !{i32 7, !"openmp-device", i32 50}
+; NVPTX: [[META8:![0-9]+]] = !{i32 1, !"wchar_size", i32 4}
+; NVPTX: [[META9:![0-9]+]] = !{i32 7, !"openmp", i32 50}
+; NVPTX: [[META10:![0-9]+]] = !{i32 7, !"openmp-device", i32 50}
 ;.
 ; AMDGPU-DISABLED: [[META0:![0-9]+]] = !{i32 0, i32 20, i32 171331627, !"simple_state_machine_interprocedural", i32 39, i32 2}
 ; AMDGPU-DISABLED: [[META1:![0-9]+]] = !{i32 0, i32 20, i32 171331627, !"simple_state_machine_no_openmp_attr", i32 66, i32 4}
@@ -4147,17 +4122,9 @@ attributes #9 = { convergent nounwind readonly willreturn }
 ; AMDGPU-DISABLED: [[META5:![0-9]+]] = !{i32 0, i32 20, i32 171331627, !"simple_state_machine_interprocedural_nested_recursive", i32 92, i32 6}
 ; AMDGPU-DISABLED: [[META6:![0-9]+]] = !{i32 0, i32 20, i32 171331627, !"no_state_machine_weak_callee", i32 112, i32 7}
 ; AMDGPU-DISABLED: [[META7:![0-9]+]] = !{i32 0, i32 20, i32 171331627, !"simple_state_machine", i32 22, i32 1}
-; AMDGPU-DISABLED: [[META8:![0-9]+]] = !{ptr @__omp_offloading_14_a36502b_no_state_machine_needed_l14, !"kernel", i32 1}
-; AMDGPU-DISABLED: [[META9:![0-9]+]] = !{ptr @__omp_offloading_14_a36502b_simple_state_machine_l22, !"kernel", i32 1}
-; AMDGPU-DISABLED: [[META10:![0-9]+]] = !{ptr @__omp_offloading_14_a36502b_simple_state_machine_interprocedural_l39, !"kernel", i32 1}
-; AMDGPU-DISABLED: [[META11:![0-9]+]] = !{ptr @__omp_offloading_14_a36502b_simple_state_machine_with_fallback_l55, !"kernel", i32 1}
-; AMDGPU-DISABLED: [[META12:![0-9]+]] = !{ptr @__omp_offloading_14_a36502b_simple_state_machine_no_openmp_attr_l66, !"kernel", i32 1}
-; AMDGPU-DISABLED: [[META13:![0-9]+]] = !{ptr @__omp_offloading_14_a36502b_simple_state_machine_pure_l77, !"kernel", i32 1}
-; AMDGPU-DISABLED: [[META14:![0-9]+]] = !{ptr @__omp_offloading_14_a36502b_simple_state_machine_interprocedural_nested_recursive_l92, !"kernel", i32 1}
-; AMDGPU-DISABLED: [[META15:![0-9]+]] = !{ptr @__omp_offloading_14_a36502b_no_state_machine_weak_callee_l112, !"kernel", i32 1}
-; AMDGPU-DISABLED: [[META16:![0-9]+]] = !{i32 1, !"wchar_size", i32 4}
-; AMDGPU-DISABLED: [[META17:![0-9]+]] = !{i32 7, !"openmp", i32 50}
-; AMDGPU-DISABLED: [[META18:![0-9]+]] = !{i32 7, !"openmp-device", i32 50}
+; AMDGPU-DISABLED: [[META8:![0-9]+]] = !{i32 1, !"wchar_size", i32 4}
+; AMDGPU-DISABLED: [[META9:![0-9]+]] = !{i32 7, !"openmp", i32 50}
+; AMDGPU-DISABLED: [[META10:![0-9]+]] = !{i32 7, !"openmp-device", i32 50}
 ;.
 ; NVPTX-DISABLED: [[META0:![0-9]+]] = !{i32 0, i32 20, i32 171331627, !"simple_state_machine_interprocedural", i32 39, i32 2}
 ; NVPTX-DISABLED: [[META1:![0-9]+]] = !{i32 0, i32 20, i32 171331627, !"simple_state_machine_no_openmp_attr", i32 66, i32 4}
@@ -4167,15 +4134,7 @@ attributes #9 = { convergent nounwind readonly willreturn }
 ; NVPTX-DISABLED: [[META5:![0-9]+]] = !{i32 0, i32 20, i32 171331627, !"simple_state_machine_interprocedural_nested_recursive", i32 92, i32 6}
 ; NVPTX-DISABLED: [[META6:![0-9]+]] = !{i32 0, i32 20, i32 171331627, !"no_state_machine_weak_callee", i32 112, i32 7}
 ; NVPTX-DISABLED: [[META7:![0-9]+]] = !{i32 0, i32 20, i32 171331627, !"simple_state_machine", i32 22, i32 1}
-; NVPTX-DISABLED: [[META8:![0-9]+]] = !{ptr @__omp_offloading_14_a36502b_no_state_machine_needed_l14, !"kernel", i32 1}
-; NVPTX-DISABLED: [[META9:![0-9]+]] = !{ptr @__omp_offloading_14_a36502b_simple_state_machine_l22, !"kernel", i32 1}
-; NVPTX-DISABLED: [[META10:![0-9]+]] = !{ptr @__omp_offloading_14_a36502b_simple_state_machine_interprocedural_l39, !"kernel", i32 1}
-; NVPTX-DISABLED: [[META11:![0-9]+]] = !{ptr @__omp_offloading_14_a36502b_simple_state_machine_with_fallback_l55, !"kernel", i32 1}
-; NVPTX-DISABLED: [[META12:![0-9]+]] = !{ptr @__omp_offloading_14_a36502b_simple_state_machine_no_openmp_attr_l66, !"kernel", i32 1}
-; NVPTX-DISABLED: [[META13:![0-9]+]] = !{ptr @__omp_offloading_14_a36502b_simple_state_machine_pure_l77, !"kernel", i32 1}
-; NVPTX-DISABLED: [[META14:![0-9]+]] = !{ptr @__omp_offloading_14_a36502b_simple_state_machine_interprocedural_nested_recursive_l92, !"kernel", i32 1}
-; NVPTX-DISABLED: [[META15:![0-9]+]] = !{ptr @__omp_offloading_14_a36502b_no_state_machine_weak_callee_l112, !"kernel", i32 1}
-; NVPTX-DISABLED: [[META16:![0-9]+]] = !{i32 1, !"wchar_size", i32 4}
-; NVPTX-DISABLED: [[META17:![0-9]+]] = !{i32 7, !"openmp", i32 50}
-; NVPTX-DISABLED: [[META18:![0-9]+]] = !{i32 7, !"openmp-device", i32 50}
+; NVPTX-DISABLED: [[META8:![0-9]+]] = !{i32 1, !"wchar_size", i32 4}
+; NVPTX-DISABLED: [[META9:![0-9]+]] = !{i32 7, !"openmp", i32 50}
+; NVPTX-DISABLED: [[META10:![0-9]+]] = !{i32 7, !"openmp-device", i32 50}
 ;.
diff --git a/llvm/test/Transforms/OpenMP/custom_state_machines_pre_lto.ll b/llvm/test/Transforms/OpenMP/custom_state_machines_pre_lto.ll
index d20821d450365..9576ff6ca6aee 100644
--- a/llvm/test/Transforms/OpenMP/custom_state_machines_pre_lto.ll
+++ b/llvm/test/Transforms/OpenMP/custom_state_machines_pre_lto.ll
@@ -139,7 +139,7 @@
 @__omp_offloading_14_a36502b_simple_state_machine_interprocedural_nested_recursive_l92_kernel_environment = local_unnamed_addr constant %struct.KernelEnvironmentTy { %struct.ConfigurationEnvironmentTy { i8 1, i8 0, i8 1, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0 }, ptr @1, ptr null }
 @__omp_offloading_14_a36502b_no_state_machine_weak_callee_l112_kernel_environment = local_unnamed_addr constant %struct.KernelEnvironmentTy { %struct.ConfigurationEnvironmentTy { i8 1, i8 0, i8 1, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0 }, ptr @1, ptr null }
 
-define weak void @__omp_offloading_14_a36502b_no_state_machine_needed_l14(ptr %dyn) #0 {
+define weak ptx_kernel void @__omp_offloading_14_a36502b_no_state_machine_needed_l14(ptr %dyn) #0 {
 entry:
   %.zero.addr = alloca i32, align 4
   %.threadid_temp. = alloca i32, align 4
@@ -196,7 +196,7 @@ declare i32 @__kmpc_global_thread_num(ptr) #3
 
 declare void @__kmpc_target_deinit()
 
-define weak void @__omp_offloading_14_a36502b_simple_state_machine_l22(ptr %dyn) #0 {
+define weak ptx_kernel void @__omp_offloading_14_a36502b_simple_state_machine_l22(ptr %dyn) #0 {
 entry:
   %.zero.addr = alloca i32, align 4
   %.threadid_temp. = alloca i32, align 4
@@ -289,7 +289,7 @@ entry:
   ret void
 }
 
-define weak void @__omp_offloading_14_a36502b_simple_state_machine_interprocedural_l39(ptr %dyn) #0 {
+define weak ptx_kernel void @__omp_offloading_14_a36502b_simple_state_machine_interprocedural_l39(ptr %dyn) #0 {
 entry:
   %.zero.addr = alloca i32, align 4
   %.threadid_temp. = alloca i32, align 4
@@ -366,7 +366,7 @@ entry:
   ret void
 }
 
-define weak void @__omp_offloading_14_a36502b_simple_state_machine_with_fallback_l55(ptr %dyn) #0 {
+define weak ptx_kernel void @__omp_offloading_14_a36502b_simple_state_machine_with_fallback_l55(ptr %dyn) #0 {
 entry:
   %.zero.addr = alloca i32, align 4
   %.threadid_temp. = alloca i32, align 4
@@ -452,7 +452,7 @@ entry:
   ret void
 }
 
-define weak void @__omp_offloading_14_a36502b_simple_state_machine_no_openmp_attr_l66(ptr %dyn) #0 {
+define weak ptx_kernel void @__omp_offloading_14_a36502b_simple_state_machine_no_openmp_attr_l66(ptr %dyn) #0 {
 entry:
   %.zero.addr = alloca i32, align 4
   %.threadid_temp. = alloca i32, align 4
@@ -536,7 +536,7 @@ entry:
   ret void
 }
 
-define weak void @__omp_offloading_14_a36502b_simple_state_machine_pure_l77(ptr %dyn) #0 {
+define weak ptx_kernel void @__omp_offloading_14_a36502b_simple_state_machine_pure_l77(ptr %dyn) #0 {
 entry:
   %.zero.addr = alloca i32, align 4
   %.threadid_temp. = alloca i32, align 4
@@ -623,7 +623,7 @@ entry:
   ret void
 }
 
-define weak void @__omp_offloading_14_a36502b_simple_state_machine_interprocedural_nested_recursive_l92(ptr %dyn) #0 {
+define weak ptx_kernel void @__omp_offloading_14_a36502b_simple_state_machine_interprocedural_nested_recursive_l92(ptr %dyn) #0 {
 entry:
   %.zero.addr = alloca i32, align 4
   %.threadid_temp. = alloca i32, align 4
@@ -678,7 +678,7 @@ return:                                           ; preds = %if.end, %if.then
 
 declare i32 @omp_get_thread_num(...) #4
 
-define weak void @__omp_offloading_14_a36502b_no_state_machine_weak_callee_l112(ptr %dyn) #0 {
+define weak ptx_kernel void @__omp_offloading_14_a36502b_no_state_machine_weak_callee_l112(ptr %dyn) #0 {
 entry:
   %.zero.addr = alloca i32, align 4
   %.threadid_temp. = alloca i32, align 4
@@ -811,7 +811,6 @@ attributes #8 = { convergent "llvm.assume"="omp_no_openmp" }
 attributes #9 = { convergent nounwind readonly willreturn }
 
 !omp_offload.info = !{!0, !1, !2, !3, !4, !5, !6, !7}
-!nvvm.annotations = !{!8, !9, !10, !11, !12, !13, !14, !15}
 !llvm.module.flags = !{!16, !17, !18}
 
 !0 = !{i32 0, i32 20, i32 171331627, !"simple_state_machine_interprocedural", i32 39, i32 2}
@@ -822,14 +821,6 @@ attributes #9 = { convergent nounwind readonly willreturn }
 !5 = !{i32 0, i32 20, i32 171331627, !"simple_state_machine_interprocedural_nested_recursive", i32 92, i32 6}
 !6 = !{i32 0, i32 20, i32 171331627, !"no_state_machine_weak_callee", i32 112, i32 7}
 !7 = !{i32 0, i32 20, i32 171331627, !"simple_state_machine", i32 22, i32 1}
-!8 = !{ptr @__omp_offloading_14_a36502b_no_state_machine_needed_l14, !"kernel", i32 1}
-!9 = !{ptr @__omp_offloading_14_a36502b_simple_state_machine_l22, !"kernel", i32 1}
-!10 = !{ptr @__omp_offloading_14_a36502b_simple_state_machine_interprocedural_l39, !"kernel", i32 1}
-!11 = !{ptr @__omp_offloading_14_a36502b_simple_state_machine_with_fallback_l55, !"kernel", i32 1}
-!12 = !{ptr @__omp_offloading_14_a36502b_simple_state_machine_no_openmp_attr_l66, !"kernel", i32 1}
-!13 = !{ptr @__omp_offloading_14_a36502b_simple_state_machine_pure_l77, !"kernel", i32 1}
-!14 = !{ptr @__omp_offloading_14_a36502b_simple_state_machine_interprocedural_nested_recursive_l92, !"kernel", i32 1}
-!15 = !{ptr @__omp_offloading_14_a36502b_no_state_machine_weak_callee_l112, !"kernel", i32 1}
 !16 = !{i32 1, !"wchar_size", i32 4}
 !17 = !{i32 7, !"openmp", i32 50}
 !18 = !{i32 7, !"openmp-device", i32 50}
@@ -4976,17 +4967,9 @@ attributes #9 = { convergent nounwind readonly willreturn }
 ; AMDGPU1: [[META5:![0-9]+]] = !{i32 0, i32 20, i32 171331627, !"simple_state_machine_interprocedural_nested_recursive", i32 92, i32 6}
 ; AMDGPU1: [[META6:![0-9]+]] = !{i32 0, i32 20, i32 171331627, !"no_state_machine_weak_callee", i32 112, i32 7}
 ; AMDGPU1: [[META7:![0-9]+]] = !{i32 0, i32 20, i32 171331627, !"simple_state_machine", i32 22, i32 1}
-; AMDGPU1: [[META8:![0-9]+]] = !{ptr @__omp_offloading_14_a36502b_no_state_machine_needed_l14, !"kernel", i32 1}
-; AMDGPU1: [[META9:![0-9]+]] = !{ptr @__omp_offloading_14_a36502b_simple_state_machine_l22, !"kernel", i32 1}
-; AMDGPU1: [[META10:![0-9]+]] = !{ptr @__omp_offloading_14_a36502b_simple_state_machine_interprocedural_l39, !"kernel", i32 1}
-; AMDGPU1: [[META11:![0-9]+]] = !{ptr @__omp_offloading_14_a36502b_simple_state_machine_with_fallback_l55, !"kernel", i32 1}
-; AMDGPU1: [[META12:![0-9]+]] = !{ptr @__omp_offloading_14_a36502b_simple_state_machine_no_openmp_attr_l66, !"kernel", i32 1}
-; AMDGPU1: [[META13:![0-9]+]] = !{ptr @__omp_offloading_14_a36502b_simple_state_machine_pure_l77, !"kernel", i32 1}
-; AMDGPU1: [[META14:![0-9]+]] = !{ptr @__omp_offloading_14_a36502b_simple_state_machine_interprocedural_nested_recursive_l92, !"kernel", i32 1}
-; AMDGPU1: [[META15:![0-9]+]] = !{ptr @__omp_offloading_14_a36502b_no_state_machine_weak_callee_l112, !"kernel", i32 1}
-; AMDGPU1: [[META16:![0-9]+]] = !{i32 1, !"wchar_size", i32 4}
-; AMDGPU1: [[META17:![0-9]+]] = !{i32 7, !"openmp", i32 50}
-; AMDGPU1: [[META18:![0-9]+]] = !{i32 7, !"openmp-device", i32 50}
+; AMDGPU1: [[META8:![0-9]+]] = !{i32 1, !"wchar_size", i32 4}
+; AMDGPU1: [[META9:![0-9]+]] = !{i32 7, !"openmp", i32 50}
+; AMDGPU1: [[META10:![0-9]+]] = !{i32 7, !"openmp-device", i32 50}
 ;.
 ; NVPTX1: [[META0:![0-9]+]] = !{i32 0, i32 20, i32 171331627, !"simple_state_machine_interprocedural", i32 39, i32 2}
 ; NVPTX1: [[META1:![0-9]+]] = !{i32 0, i32 20, i32 171331627, !"simple_state_machine_no_openmp_attr", i32 66, i32 4}
@@ -4996,17 +4979,9 @@ attributes #9 = { convergent nounwind readonly willreturn }
 ; NVPTX1: [[META5:![0-9]+]] = !{i32 0, i32 20, i32 171331627, !"simple_state_machine_interprocedural_nested_recursive", i32 92, i32 6}
 ; NVPTX1: [[META6:![0-9]+]] = !{i32 0, i32 20, i32 171331627, !"no_state_machine_weak_callee", i32 112, i32 7}
 ; NVPTX1: [[META7:![0-9]+]] = !{i32 0, i32 20, i32 171331627, !"simple_state_machine", i32 22, i32 1}
-; NVPTX1: [[META8:![0-9]+]] = !{ptr @__omp_offloading_14_a36502b_no_state_machine_needed_l14, !"kernel", i32 1}
-; NVPTX1: [[META9:![0-9]+]] = !{ptr @__omp_offloading_14_a36502b_simple_state_machine_l22, !"kernel", i32 1}
-; NVPTX1: [[META10:![0-9]+]] = !{ptr @__omp_offloading_14_a36502b_simple_state_machine_interprocedural_l39, !"kernel", i32 1}
-; NVPTX1: [[META11:![0-9]+]] = !{ptr @__omp_offloading_14_a36502b_simple_state_machine_with_fallback_l55, !"kernel", i32 1}
-; NVPTX1: [[META12:![0-9]+]] = !{ptr @__omp_offloading_14_a36502b_simple_state_machine_no_openmp_attr_l66, !"kernel", i32 1}
-; NVPTX1: [[META13:![0-9]+]] = !{ptr @__omp_offloading_14_a36502b_simple_state_machine_pure_l77, !"kernel", i32 1}
-; NVPTX1: [[META14:![0-9]+]] = !{ptr @__omp_offloading_14_a36502b_simple_state_machine_interprocedural_nested_recursive_l92, !"kernel", i32 1}
-; NVPTX1: [[META15:![0-9]+]] = !{ptr @__omp_offloading_14_a36502b_no_state_machine_weak_callee_l112, !"kernel", i32 1}
-; NVPTX1: [[META16:![0-9]+]] = !{i32 1, !"wchar_size", i32 4}
-; NVPTX1: [[META17:![0-9]+]] = !{i32 7, !"openmp", i32 50}
-; NVPTX1: [[META18:![0-9]+]] = !{i32 7, !"openmp-device", i32 50}
+; NVPTX1: [[META8:![0-9]+]] = !{i32 1, !"wchar_size", i32 4}
+; NVPTX1: [[META9:![0-9]+]] = !{i32 7, !"openmp", i32 50}
+; NVPTX1: [[META10:![0-9]+]] = !{i32 7, !"openmp-device", i32 50}
 ;.
 ; AMDGPU2: [[META0:![0-9]+]] = !{i32 0, i32 20, i32 171331627, !"simple_state_machine_interprocedural", i32 39, i32 2}
 ; AMDGPU2: [[META1:![0-9]+]] = !{i32 0, i32 20, i32 171331627, !"simple_state_machine_no_openmp_attr", i32 66, i32 4}
@@ -5016,17 +4991,9 @@ attributes #9 = { convergent nounwind readonly willreturn }
 ; AMDGPU2: [[META5:![0-9]+]] = !{i32 0, i32 20, i32 171331627, !"simple_state_machine_interprocedural_nested_recursive", i32 92, i32 6}
 ; AMDGPU2: [[META6:![0-9]+]] = !{i32 0, i32 20, i32 171331627, !"no_state_machine_weak_callee", i32 112, i32 7}
 ; AMDGPU2: [[META7:![0-9]+]] = !{i32 0, i32 20, i32 171331627, !"simple_state_machine", i32 22, i32 1}
-; AMDGPU2: [[META8:![0-9]+]] = !{ptr @__omp_offloading_14_a36502b_no_state_machine_needed_l14, !"kernel", i32 1}
-; AMDGPU2: [[META9:![0-9]+]] = !{ptr @__omp_offloading_14_a36502b_simple_state_machine_l22, !"kernel", i32 1}
-; AMDGPU2: [[META10:![0-9]+]] = !{ptr @__omp_offloading_14_a36502b_simple_state_machine_interprocedural_l39, !"kernel", i32 1}
-; AMDGPU2: [[META11:![0-9]+]] = !{ptr @__omp_offloading_14_a36502b_simple_state_machine_with_fallback_l55, !"kernel", i32 1}
-; AMDGPU2: [[META12:![0-9]+]] = !{ptr @__omp_offloading_14_a36502b_simple_state_machine_no_openmp_attr_l66, !"kernel", i32 1}
-; AMDGPU2: [[META13:![0-9]+]] = !{ptr @__omp_offloading_14_a36502b_simple_state_machine_pure_l77, !"kernel", i32 1}
-; AMDGPU2: [[META14:![0-9]+]] = !{ptr @__omp_offloading_14_a36502b_simple_state_machine_interprocedural_nested_recursive_l92, !"kernel", i32 1}
-; AMDGPU2: [[META15:![0-9]+]] = !{ptr @__omp_offloading_14_a36502b_no_state_machine_weak_callee_l112, !"kernel", i32 1}
-; AMDGPU2: [[META16:![0-9]+]] = !{i32 1, !"wchar_size", i32 4}
-; AMDGPU2: [[META17:![0-9]+]] = !{i32 7, !"openmp", i32 50}
-; AMDGPU2: [[META18:![0-9]+]] = !{i32 7, !"openmp-device", i32 50}
+; AMDGPU2: [[META8:![0-9]+]] = !{i32 1, !"wchar_size", i32 4}
+; AMDGPU2: [[META9:![0-9]+]] = !{i32 7, !"openmp", i32 50}
+; AMDGPU2: [[META10:![0-9]+]] = !{i32 7, !"openmp-device", i32 50}
 ;.
 ; AMDGPU3: [[META0:![0-9]+]] = !{i32 0, i32 20, i32 171331627, !"simple_state_machine_interprocedural", i32 39, i32 2}
 ; AMDGPU3: [[META1:![0-9]+]] = !{i32 0, i32 20, i32 171331627, !"simple_state_machine_no_openmp_attr", i32 66, i32 4}
@@ -5036,17 +5003,9 @@ attributes #9 = { convergent nounwind readonly willreturn }
 ; AMDGPU3: [[META5:![0-9]+]] = !{i32 0, i32 20, i32 171331627, !"simple_state_machine_interprocedural_nested_recursive", i32 92, i32 6}
 ; AMDGPU3: [[META6:![0-9]+]] = !{i32 0, i32 20, i32 171331627, !"no_state_machine_weak_callee", i32 112, i32 7}
 ; AMDGPU3: [[META7:![0-9]+]] = !{i32 0, i32 20, i32 171331627, !"simple_state_machine", i32 22, i32 1}
-; AMDGPU3: [[META8:![0-9]+]] = !{ptr @__omp_offloading_14_a36502b_no_state_machine_needed_l14, !"kernel", i32 1}
-; AMDGPU3: [[META9:![0-9]+]] = !{ptr @__omp_offloading_14_a36502b_simple_state_machine_l22, !"kernel", i32 1}
-; AMDGPU3: [[META10:![0-9]+]] = !{ptr @__omp_offloading_14_a36502b_simple_state_machine_interprocedural_l39, !"kernel", i32 1}
-; AMDGPU3: [[META11:![0-9]+]] = !{ptr @__omp_offloading_14_a36502b_simple_state_machine_with_fallback_l55, !"kernel", i32 1}
-; AMDGPU3: [[META12:![0-9]+]] = !{ptr @__omp_offloading_14_a36502b_simple_state_machine_no_openmp_attr_l66, !"kernel", i32 1}
-; AMDGPU3: [[META13:![0-9]+]] = !{ptr @__omp_offloading_14_a36502b_simple_state_machine_pure_l77, !"kernel", i32 1}
-; AMDGPU3: [[META14:![0-9]+]] = !{ptr @__omp_offloading_14_a36502b_simple_state_machine_interprocedural_nested_recursive_l92, !"kernel", i32 1}
-; AMDGPU3: [[META15:![0-9]+]] = !{ptr @__omp_offloading_14_a36502b_no_state_machine_weak_callee_l112, !"kernel", i32 1}
-; AMDGPU3: [[META16:![0-9]+]] = !{i32 1, !"wchar_size", i32 4}
-; AMDGPU3: [[META17:![0-9]+]] = !{i32 7, !"openmp", i32 50}
-; AMDGPU3: [[META18:![0-9]+]] = !{i32 7, !"openmp-device", i32 50}
+; AMDGPU3: [[META8:![0-9]+]] = !{i32 1, !"wchar_size", i32 4}
+; AMDGPU3: [[META9:![0-9]+]] = !{i32 7, !"openmp", i32 50}
+; AMDGPU3: [[META10:![0-9]+]] = !{i32 7, !"openmp-device", i32 50}
 ;.
 ; NVPTX2: [[META0:![0-9]+]] = !{i32 0, i32 20, i32 171331627, !"simple_state_machine_interprocedural", i32 39, i32 2}
 ; NVPTX2: [[META1:![0-9]+]] = !{i32 0, i32 20, i32 171331627, !"simple_state_machine_no_openmp_attr", i32 66, i32 4}
@@ -5056,17 +5015,9 @@ attributes #9 = { convergent nounwind readonly willreturn }
 ; NVPTX2: [[META5:![0-9]+]] = !{i32 0, i32 20, i32 171331627, !"simple_state_machine_interprocedural_nested_recursive", i32 92, i32 6}
 ; NVPTX2: [[META6:![0-9]+]] = !{i32 0, i32 20, i32 171331627, !"no_state_machine_weak_callee", i32 112, i32 7}
 ; NVPTX2: [[META7:![0-9]+]] = !{i32 0, i32 20, i32 171331627, !"simple_state_machine", i32 22, i32 1}
-; NVPTX2: [[META8:![0-9]+]] = !{ptr @__omp_offloading_14_a36502b_no_state_machine_needed_l14, !"kernel", i32 1}
-; NVPTX2: [[META9:![0-9]+]] = !{ptr @__omp_offloading_14_a36502b_simple_state_machine_l22, !"kernel", i32 1}
-; NVPTX2: [[META10:![0-9]+]] = !{ptr @__omp_offloading_14_a36502b_simple_state_machine_interprocedural_l39, !"kernel", i32 1}
-; NVPTX2: [[META11:![0-9]+]] = !{ptr @__omp_offloading_14_a36502b_simple_state_machine_with_fallback_l55, !"kernel", i32 1}
-; NVPTX2: [[META12:![0-9]+]] = !{ptr @__omp_offloading_14_a36502b_simple_state_machine_no_openmp_attr_l66, !"kernel", i32 1}
-; NVPTX2: [[META13:![0-9]+]] = !{ptr @__omp_offloading_14_a36502b_simple_state_machine_pure_l77, !"kernel", i32 1}
-; NVPTX2: [[META14:![0-9]+]] = !{ptr @__omp_offloading_14_a36502b_simple_state_machine_interprocedural_nested_recursive_l92, !"kernel", i32 1}
-; NVPTX2: [[META15:![0-9]+]] = !{ptr @__omp_offloading_14_a36502b_no_state_machine_weak_callee_l112, !"kernel", i32 1}
-; NVPTX2: [[META16:![0-9]+]] = !{i32 1, !"wchar_size", i32 4}
-; NVPTX2: [[META17:![0-9]+]] = !{i32 7, !"openmp", i32 50}
-; NVPTX2: [[META18:![0-9]+]] = !{i32 7, !"openmp-device", i32 50}
+; NVPTX2: [[META8:![0-9]+]] = !{i32 1, !"wchar_size", i32 4}
+; NVPTX2: [[META9:![0-9]+]] = !{i32 7, !"openmp", i32 50}
+; NVPTX2: [[META10:![0-9]+]] = !{i32 7, !"openmp-device", i32 50}
 ;.
 ; NVPTX3: [[META0:![0-9]+]] = !{i32 0, i32 20, i32 171331627, !"simple_state_machine_interprocedural", i32 39, i32 2}
 ; NVPTX3: [[META1:![0-9]+]] = !{i32 0, i32 20, i32 171331627, !"simple_state_machine_no_openmp_attr", i32 66, i32 4}
@@ -5076,15 +5027,7 @@ attributes #9 = { convergent nounwind readonly willreturn }
 ; NVPTX3: [[META5:![0-9]+]] = !{i32 0, i32 20, i32 171331627, !"simple_state_machine_interprocedural_nested_recursive", i32 92, i32 6}
 ; NVPTX3: [[META6:![0-9]+]] = !{i32 0, i32 20, i32 171331627, !"no_state_machine_weak_callee", i32 112, i32 7}
 ; NVPTX3: [[META7:![0-9]+]] = !{i32 0, i32 20, i32 171331627, !"simple_state_machine", i32 22, i32 1}
-; NVPTX3: [[META8:![0-9]+]] = !{ptr @__omp_offloading_14_a36502b_no_state_machine_needed_l14, !"kernel", i32 1}
-; NVPTX3: [[META9:![0-9]+]] = !{ptr @__omp_offloading_14_a36502b_simple_state_machine_l22, !"kernel", i32 1}
-; NVPTX3: [[META10:![0-9]+]] = !{ptr @__omp_offloading_14_a36502b_simple_state_machine_interprocedural_l39, !"kernel", i32 1}
-; NVPTX3: [[META11:![0-9]+]] = !{ptr @__omp_offloading_14_a36502b_simple_state_machine_with_fallback_l55, !"kernel", i32 1}
-; NVPTX3: [[META12:![0-9]+]] = !{ptr @__omp_offloading_14_a36502b_simple_state_machine_no_openmp_attr_l66, !"kernel", i32 1}
-; NVPTX3: [[META13:![0-9]+]] = !{ptr @__omp_offloading_14_a36502b_simple_state_machine_pure_l77, !"kernel", i32 1}
-; NVPTX3: [[META14:![0-9]+]] = !{ptr @__omp_offloading_14_a36502b_simple_state_machine_interprocedural_nested_recursive_l92, !"kernel", i32 1}
-; NVPTX3: [[META15:![0-9]+]] = !{ptr @__omp_offloading_14_a36502b_no_state_machine_weak_callee_l112, !"kernel", i32 1}
-; NVPTX3: [[META16:![0-9]+]] = !{i32 1, !"wchar_size", i32 4}
-; NVPTX3: [[META17:![0-9]+]] = !{i32 7, !"openmp", i32 50}
-; NVPTX3: [[META18:![0-9]+]] = !{i32 7, !"openmp-device", i32 50}
+; NVPTX3: [[META8:![0-9]+]] = !{i32 1, !"wchar_size", i32 4}
+; NVPTX3: [[META9:![0-9]+]] = !{i32 7, !"openmp", i32 50}
+; NVPTX3: [[META10:![0-9]+]] = !{i32 7, !"openmp-device", i32 50}
 ;.
diff --git a/llvm/test/Transforms/OpenMP/custom_state_machines_remarks.ll b/llvm/test/Transforms/OpenMP/custom_state_machines_remarks.ll
index f7bfd30650694..ad41639511e99 100644
--- a/llvm/test/Transforms/OpenMP/custom_state_machines_remarks.ll
+++ b/llvm/test/Transforms/OpenMP/custom_state_machines_remarks.ll
@@ -59,7 +59,7 @@ target triple = "nvptx64"
 
 
 ; Function Attrs: convergent norecurse nounwind
-define weak void @__omp_offloading_2a_d80d3d_test_fallback_l11(ptr %dyn) local_unnamed_addr #0 !dbg !15 {
+define weak ptx_kernel void @__omp_offloading_2a_d80d3d_test_fallback_l11(ptr %dyn) local_unnamed_addr #0 !dbg !15 {
 entry:
   %captured_vars_addrs.i.i = alloca [0 x ptr], align 8
   %0 = call i32 @__kmpc_target_init(ptr @__omp_offloading_2a_d80d3d_test_fallback_l11_kernel_environment, ptr %dyn) #3, !dbg !18
@@ -104,7 +104,7 @@ declare i32 @__kmpc_global_thread_num(ptr) local_unnamed_addr #3
 declare void @__kmpc_target_deinit() local_unnamed_addr
 
 ; Function Attrs: norecurse nounwind
-define weak void @__omp_offloading_2a_d80d3d_test_no_fallback_l20(ptr %dyn) local_unnamed_addr #4 !dbg !32 {
+define weak ptx_kernel void @__omp_offloading_2a_d80d3d_test_no_fallback_l20(ptr %dyn) local_unnamed_addr #4 !dbg !32 {
 entry:
   %captured_vars_addrs.i2.i = alloca [0 x ptr], align 8
   %0 = call i32 @__kmpc_target_init(ptr @__omp_offloading_2a_d80d3d_test_no_fallback_l20_kernel_environment, ptr %dyn) #3, !dbg !33
@@ -175,7 +175,6 @@ attributes #8 = { "llvm.assume"="omp_no_parallelism" }
 
 !llvm.dbg.cu = !{!0}
 !omp_offload.info = !{!3, !4}
-!nvvm.annotations = !{!5, !6}
 !llvm.module.flags = !{!7, !8, !9, !10, !11, !12, !13}
 !llvm.ident = !{!14}
 
@@ -184,8 +183,6 @@ attributes #8 = { "llvm.assume"="omp_no_parallelism" }
 !2 = !{}
 !3 = !{i32 0, i32 42, i32 14159165, !"test_no_fallback", i32 20, i32 1}
 !4 = !{i32 0, i32 42, i32 14159165, !"test_fallback", i32 11, i32 0}
-!5 = !{ptr @__omp_offloading_2a_d80d3d_test_fallback_l11, !"kernel", i32 1}
-!6 = !{ptr @__omp_offloading_2a_d80d3d_test_no_fallback_l20, !"kernel", i32 1}
 !7 = !{i32 7, !"Dwarf Version", i32 2}
 !8 = !{i32 2, !"Debug Info Version", i32 3}
 !9 = !{i32 1, !"wchar_size", i32 4}
diff --git a/llvm/test/Transforms/OpenMP/deduplication_target.ll b/llvm/test/Transforms/OpenMP/deduplication_target.ll
index 6b0563365c648..7027c3275b932 100644
--- a/llvm/test/Transforms/OpenMP/deduplication_target.ll
+++ b/llvm/test/Transforms/OpenMP/deduplication_target.ll
@@ -15,7 +15,7 @@ target triple = "nvptx64"
 
 declare void @use(i32)
 
-define weak void @__omp_offloading_50_a3e09bf8_foo_l2(ptr %dyn) #0 {
+define weak ptx_kernel void @__omp_offloading_50_a3e09bf8_foo_l2(ptr %dyn) #0 {
 ; CHECK-LABEL: define {{[^@]+}}@__omp_offloading_50_a3e09bf8_foo_l2
 ; CHECK-SAME: (ptr [[DYN:%.*]]) #[[ATTR0:[0-9]+]] {
 ; CHECK-NEXT:  entry:
@@ -56,11 +56,9 @@ attributes #0 = { convergent noinline norecurse nounwind "kernel" "frame-pointer
 attributes #1 = { nounwind }
 
 !omp_offload.info = !{!0}
-!nvvm.annotations = !{!1}
 !llvm.module.flags = !{!2, !3, !4}
 
 !0 = !{i32 0, i32 80, i32 -1545561096, !"foo", i32 2, i32 0}
-!1 = !{ptr @__omp_offloading_50_a3e09bf8_foo_l2, !"kernel", i32 1}
 !2 = !{i32 1, !"wchar_size", i32 4}
 !3 = !{i32 7, !"openmp", i32 50}
 !4 = !{i32 7, !"openmp-device", i32 50}
diff --git a/llvm/test/Transforms/OpenMP/get_hardware_num_threads_in_block_fold.ll b/llvm/test/Transforms/OpenMP/get_hardware_num_threads_in_block_fold.ll
index 6102201ad4bac..6a4519a161fd6 100644
--- a/llvm/test/Transforms/OpenMP/get_hardware_num_threads_in_block_fold.ll
+++ b/llvm/test/Transforms/OpenMP/get_hardware_num_threads_in_block_fold.ll
@@ -19,7 +19,7 @@ target triple = "nvptx64"
 ; CHECK: @[[GLOB0:[0-9]+]] = private unnamed_addr constant [23 x i8] c"
 ; CHECK: @[[GLOB1:[0-9]+]] = private unnamed_addr constant %struct.ident_t { i32 0, i32 2, i32 0, i32 22, ptr @[[GLOB0]] }, align 8
 ;.
-define weak void @kernel0(ptr %dyn) "kernel" #0 {
+define weak ptx_kernel void @kernel0(ptr %dyn) "kernel" #0 {
 ; CHECK-LABEL: define {{[^@]+}}@kernel0
 ; CHECK-SAME: (ptr [[DYN:%.*]]) #[[ATTR0:[0-9]+]] {
 ; CHECK-NEXT:    [[I:%.*]] = call i32 @__kmpc_target_init(ptr @kernel0_kernel_environment, ptr [[DYN]])
@@ -43,7 +43,7 @@ define weak void @kernel0(ptr %dyn) "kernel" #0 {
   ret void
 }
 
-define weak void @kernel1(ptr %dyn) "kernel" #0 {
+define weak ptx_kernel void @kernel1(ptr %dyn) "kernel" #0 {
 ; CHECK-LABEL: define {{[^@]+}}@kernel1
 ; CHECK-SAME: (ptr [[DYN:%.*]]) #[[ATTR0]] {
 ; CHECK-NEXT:    [[I:%.*]] = call i32 @__kmpc_target_init(ptr @kernel1_kernel_environment, ptr [[DYN]])
@@ -63,7 +63,7 @@ define weak void @kernel1(ptr %dyn) "kernel" #0 {
   ret void
 }
 
-define weak void @kernel2(ptr %dyn) "kernel" #0 {
+define weak ptx_kernel void @kernel2(ptr %dyn) "kernel" #0 {
 ; CHECK-LABEL: define {{[^@]+}}@kernel2
 ; CHECK-SAME: (ptr [[DYN:%.*]]) #[[ATTR0]] {
 ; CHECK-NEXT:  entry:
@@ -200,15 +200,11 @@ declare i32 @__kmpc_global_thread_num(ptr)
 
 
 !llvm.module.flags = !{!0, !1}
-!nvvm.annotations = !{!2, !3, !4}
 
 attributes #0 = { "omp_target_thread_limit"="666" "omp_target_num_teams"="777"}
 
 !0 = !{i32 7, !"openmp", i32 50}
 !1 = !{i32 7, !"openmp-device", i32 50}
-!2 = !{ptr @kernel0, !"kernel", i32 1}
-!3 = !{ptr @kernel1, !"kernel", i32 1}
-!4 = !{ptr @kernel2, !"kernel", i32 1}
 ;.
 ; CHECK: attributes #[[ATTR0]] = { "kernel" "omp_target_num_teams"="777" "omp_target_thread_limit"="666" }
 ; CHECK: attributes #[[ATTR1]] = { nounwind }
@@ -217,7 +213,4 @@ attributes #0 = { "omp_target_thread_limit"="666" "omp_target_num_teams"="777"}
 ;.
 ; CHECK: [[META0:![0-9]+]] = !{i32 7, !"openmp", i32 50}
 ; CHECK: [[META1:![0-9]+]] = !{i32 7, !"openmp-device", i32 50}
-; CHECK: [[META2:![0-9]+]] = !{ptr @kernel0, !"kernel", i32 1}
-; CHECK: [[META3:![0-9]+]] = !{ptr @kernel1, !"kernel", i32 1}
-; CHECK: [[META4:![0-9]+]] = !{ptr @kernel2, !"kernel", i32 1}
 ;.
diff --git a/llvm/test/Transforms/OpenMP/get_hardware_num_threads_in_block_fold_optnone.ll b/llvm/test/Transforms/OpenMP/get_hardware_num_threads_in_block_fold_optnone.ll
index 0cf6e7488b4dd..3037d24b8c448 100644
--- a/llvm/test/Transforms/OpenMP/get_hardware_num_threads_in_block_fold_optnone.ll
+++ b/llvm/test/Transforms/OpenMP/get_hardware_num_threads_in_block_fold_optnone.ll
@@ -7,7 +7,7 @@ target triple = "nvptx64"
 ;.
 ; CHECK: @G = external global i32
 ;.
-define weak void @kernel0() #0 {
+define weak ptx_kernel void @kernel0() #0 {
 ; CHECK-LABEL: define {{[^@]+}}@kernel0
 ; CHECK-SAME: () #[[ATTR0:[0-9]+]] {
 ; CHECK-NEXT:    [[I:%.*]] = call i32 @__kmpc_target_init(ptr null, ptr null)
@@ -25,7 +25,7 @@ define weak void @kernel0() #0 {
   ret void
 }
 
-define weak void @kernel1() #0 {
+define weak ptx_kernel void @kernel1() #0 {
 ; CHECK-LABEL: define {{[^@]+}}@kernel1
 ; CHECK-SAME: () #[[ATTR0]] {
 ; CHECK-NEXT:    [[I:%.*]] = call i32 @__kmpc_target_init(ptr null, ptr null)
@@ -39,7 +39,7 @@ define weak void @kernel1() #0 {
   ret void
 }
 
-define weak void @kernel2() #0 {
+define weak ptx_kernel void @kernel2() #0 {
 ; CHECK-LABEL: define {{[^@]+}}@kernel2
 ; CHECK-SAME: () #[[ATTR0]] {
 ; CHECK-NEXT:    [[I:%.*]] = call i32 @__kmpc_target_init(ptr null, ptr null)
@@ -107,15 +107,11 @@ declare void @__kmpc_target_deinit() #1
 
 
 !llvm.module.flags = !{!0, !1}
-!nvvm.annotations = !{!2, !3, !4}
 
 attributes #0 = { optnone noinline "kernel" "omp_target_thread_limit"="666" "omp_target_num_teams"="777"}
 
 !0 = !{i32 7, !"openmp", i32 50}
 !1 = !{i32 7, !"openmp-device", i32 50}
-!2 = !{ptr @kernel0, !"kernel", i32 1}
-!3 = !{ptr @kernel1, !"kernel", i32 1}
-!4 = !{ptr @kernel2, !"kernel", i32 1}
 ;
 ;.
 ; CHECK: attributes #[[ATTR0]] = { noinline optnone "kernel" "omp_target_num_teams"="777" "omp_target_thread_limit"="666" }
@@ -123,7 +119,4 @@ attributes #0 = { optnone noinline "kernel" "omp_target_thread_limit"="666" "omp
 ;.
 ; CHECK: [[META0:![0-9]+]] = !{i32 7, !"openmp", i32 50}
 ; CHECK: [[META1:![0-9]+]] = !{i32 7, !"openmp-device", i32 50}
-; CHECK: [[META2:![0-9]+]] = !{ptr @kernel0, !"kernel", i32 1}
-; CHECK: [[META3:![0-9]+]] = !{ptr @kernel1, !"kernel", i32 1}
-; CHECK: [[META4:![0-9]+]] = !{ptr @kernel2, !"kernel", i32 1}
 ;.
diff --git a/llvm/test/Transforms/OpenMP/global_constructor.ll b/llvm/test/Transforms/OpenMP/global_constructor.ll
index 804b910dcd308..1d18e527e1466 100644
--- a/llvm/test/Transforms/OpenMP/global_constructor.ll
+++ b/llvm/test/Transforms/OpenMP/global_constructor.ll
@@ -10,7 +10,7 @@
 @_ZL6Device = internal global double 0.000000e+00, align 8
 @__omp_offloading_fd02_85283c04_main_l11_kernel_environment = local_unnamed_addr constant %struct.KernelEnvironmentTy { %struct.ConfigurationEnvironmentTy { i8 0, i8 0, i8 1, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0 }, ptr @1, ptr null }
 
-define weak void @__omp_offloading_fd02_85283c04_main_l11(ptr %dyn, ptr nonnull align 8 dereferenceable(8) %X) local_unnamed_addr "kernel" {
+define weak ptx_kernel void @__omp_offloading_fd02_85283c04_main_l11(ptr %dyn, ptr nonnull align 8 dereferenceable(8) %X) local_unnamed_addr "kernel" {
 entry:
   %0 = tail call i32 @__kmpc_target_init(ptr @__omp_offloading_fd02_85283c04_main_l11_kernel_environment, ptr %dyn) #0
   %exec_user_code = icmp eq i32 %0, -1
@@ -39,7 +39,7 @@ declare i32 @__kmpc_target_init(ptr, ptr) local_unnamed_addr
 
 declare void @__kmpc_target_deinit() local_unnamed_addr
 
-define weak void @__omp_offloading__fd02_85283c04_Device_l6_ctor() "kernel" {
+define weak ptx_kernel void @__omp_offloading__fd02_85283c04_Device_l6_ctor() "kernel" {
 entry:
   %call.i = tail call double @__nv_log(double noundef 2.000000e+00) #1
   %call.i2 = tail call double @__nv_log(double noundef 2.000000e+00) #1
@@ -58,15 +58,12 @@ attributes #0 = { nounwind }
 attributes #1 = { convergent nounwind }
 
 !omp_offload.info = !{!0, !1, !2}
-!nvvm.annotations = !{!3, !4}
 !llvm.module.flags = !{!5, !6, !7, !8, !9}
 !llvm.ident = !{!10}
 
 !0 = !{i32 0, i32 64770, i32 -2060960764, !"__omp_offloading__fd02_85283c04_Device_l6_ctor", i32 6, i32 1}
 !1 = !{i32 0, i32 64770, i32 -2060960764, !"main", i32 11, i32 2}
 !2 = !{i32 1, !"_ZL6Device", i32 0, i32 0}
-!3 = !{ptr @__omp_offloading__fd02_85283c04_Device_l6_ctor, !"kernel", i32 1}
-!4 = !{ptr @__omp_offloading_fd02_85283c04_main_l11, !"kernel", i32 1}
 !5 = !{i32 1, !"wchar_size", i32 4}
 !6 = !{i32 7, !"openmp", i32 50}
 !7 = !{i32 7, !"openmp-device", i32 50}
@@ -86,12 +83,12 @@ attributes #1 = { convergent nounwind }
 ; CHECK:       common.ret:
 ; CHECK-NEXT:    ret void
 ; CHECK:       user_code.entry:
-; CHECK-NEXT:    [[TMP1:%.*]] = load double, ptr @_ZL6Device, align 8, !tbaa [[TBAA11:![0-9]+]]
+; CHECK-NEXT:    [[TMP1:%.*]] = load double, ptr @_ZL6Device, align 8, !tbaa [[TBAA9:![0-9]+]]
 ; CHECK-NEXT:    [[TMP2:%.*]] = tail call i32 @__kmpc_get_hardware_thread_id_in_block() #[[ATTR1]]
 ; CHECK-NEXT:    [[TMP3:%.*]] = icmp eq i32 [[TMP2]], 0
 ; CHECK-NEXT:    br i1 [[TMP3]], label [[REGION_GUARDED:%.*]], label [[REGION_BARRIER:%.*]]
 ; CHECK:       region.guarded:
-; CHECK-NEXT:    store double [[TMP1]], ptr [[X]], align 8, !tbaa [[TBAA11]]
+; CHECK-NEXT:    store double [[TMP1]], ptr [[X]], align 8, !tbaa [[TBAA9]]
 ; CHECK-NEXT:    br label [[REGION_BARRIER]]
 ; CHECK:       region.barrier:
 ; CHECK-NEXT:    tail call void @__kmpc_barrier_simple_spmd(ptr nonnull @[[GLOB1:[0-9]+]], i32 [[TMP2]]) #[[ATTR1]]
@@ -105,6 +102,6 @@ attributes #1 = { convergent nounwind }
 ; CHECK-NEXT:    [[CALL_I:%.*]] = tail call double @__nv_log(double noundef 2.000000e+00) #[[ATTR2:[0-9]+]]
 ; CHECK-NEXT:    [[CALL_I2:%.*]] = tail call double @__nv_log(double noundef 2.000000e+00) #[[ATTR2]]
 ; CHECK-NEXT:    [[DIV:%.*]] = fdiv double [[CALL_I]], [[CALL_I2]]
-; CHECK-NEXT:    store double [[DIV]], ptr @_ZL6Device, align 8, !tbaa [[TBAA11]]
+; CHECK-NEXT:    store double [[DIV]], ptr @_ZL6Device, align 8, !tbaa [[TBAA9]]
 ; CHECK-NEXT:    ret void
 ;
diff --git a/llvm/test/Transforms/OpenMP/globalization_remarks.ll b/llvm/test/Transforms/OpenMP/globalization_remarks.ll
index 878ac9010a7dc..0f37b3e070acd 100644
--- a/llvm/test/Transforms/OpenMP/globalization_remarks.ll
+++ b/llvm/test/Transforms/OpenMP/globalization_remarks.ll
@@ -13,7 +13,7 @@ target triple = "nvptx64"
 @S = external local_unnamed_addr global ptr
 @foo_kernel_environment = local_unnamed_addr constant %struct.KernelEnvironmentTy { %struct.ConfigurationEnvironmentTy { i8 0, i8 0, i8 1, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0 }, ptr null, ptr null }
 
-define void @foo() "kernel" {
+define ptx_kernel void @foo() "kernel" {
 entry:
   %c = call i32 @__kmpc_target_init(ptr @foo_kernel_environment, ptr null)
   %0 = call ptr @__kmpc_alloc_shared(i64 4), !dbg !10
@@ -39,7 +39,6 @@ declare void @__kmpc_target_deinit()
 
 !llvm.dbg.cu = !{!0}
 !llvm.module.flags = !{!3, !4, !5, !6}
-!nvvm.annotations = !{!7, !8}
 
 !0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 12.0.0", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, enums: !2, splitDebugInlining: false, nameTableKind: None)
 !1 = !DIFile(filename: "globalization_remarks.c", directory: "/tmp/globalization_remarks.c")
@@ -48,7 +47,6 @@ declare void @__kmpc_target_deinit()
 !4 = !{i32 1, !"wchar_size", i32 4}
 !5 = !{i32 7, !"openmp", i32 50}
 !6 = !{i32 7, !"openmp-device", i32 50}
-!7 = !{ptr @foo, !"kernel", i32 1}
 !8 = distinct !DISubprogram(name: "foo", scope: !1, file: !1, line: 1, type: !9, scopeLine: 1, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, retainedNodes: !2)
 !9 = !DISubroutineType(types: !2)
 !10 = !DILocation(line: 5, column: 7, scope: !8)
diff --git a/llvm/test/Transforms/OpenMP/gpu_kernel_detection_remarks.ll b/llvm/test/Transforms/OpenMP/gpu_kernel_detection_remarks.ll
index b029efbbe3c68..ce17ffcbb2084 100644
--- a/llvm/test/Transforms/OpenMP/gpu_kernel_detection_remarks.ll
+++ b/llvm/test/Transforms/OpenMP/gpu_kernel_detection_remarks.ll
@@ -3,11 +3,11 @@
 ; CHECK-DAG: remark: <unknown>:0:0: OpenMP GPU kernel kernel1
 ; CHECK-DAG: remark: <unknown>:0:0: OpenMP GPU kernel kernel2
 
-define void @kernel1() "kernel" {
+define ptx_kernel void @kernel1() "kernel" {
   ret void
 }
 
-define void @kernel2() "kernel" {
+define ptx_kernel void @kernel2() "kernel" {
   ret void
 }
 
@@ -19,10 +19,5 @@ define void @non_kernel() {
 declare dso_local void @__kmpc_kernel_prepare_parallel(ptr)
 
 !llvm.module.flags = !{!4}
-!nvvm.annotations = !{!2, !0, !1, !3, !1, !2}
 
-!0 = !{ptr @kernel1, !"kernel", i32 1}
-!1 = !{ptr @non_kernel, !"non_kernel", i32 1}
-!2 = !{null, !"align", i32 1}
-!3 = !{ptr @kernel2, !"kernel", i32 1}
 !4 = !{i32 7, !"openmp", i32 50}
diff --git a/llvm/test/Transforms/OpenMP/gpu_state_machine_function_ptr_replacement.ll b/llvm/test/Transforms/OpenMP/gpu_state_machine_function_ptr_replacement.ll
index 936f7d1c46781..760c5a354a37c 100644
--- a/llvm/test/Transforms/OpenMP/gpu_state_machine_function_ptr_replacement.ll
+++ b/llvm/test/Transforms/OpenMP/gpu_state_machine_function_ptr_replacement.ll
@@ -44,7 +44,7 @@
 @2 = private unnamed_addr constant %struct.ident_t { i32 0, i32 2, i32 2, i32 0, ptr @0 }, align 8
 @__omp_offloading_10301_87b2c_foo_l7_kernel_environment = local_unnamed_addr constant %struct.KernelEnvironmentTy { %struct.ConfigurationEnvironmentTy { i8 1, i8 0, i8 1, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0 }, ptr @1, ptr null }
 
-define weak void @__omp_offloading_10301_87b2c_foo_l7() "kernel" {
+define weak ptx_kernel void @__omp_offloading_10301_87b2c_foo_l7() "kernel" {
 entry:
   %.zero.addr = alloca i32, align 4
   %.threadid_temp. = alloca i32, align 4
@@ -173,10 +173,8 @@ entry:
 }
 
 !omp_offload.info = !{!0}
-!nvvm.annotations = !{!1}
 !llvm.module.flags = !{!2, !3}
 
 !0 = !{i32 0, i32 66305, i32 555956, !"foo", i32 7, i32 0}
-!1 = !{ptr @__omp_offloading_10301_87b2c_foo_l7, !"kernel", i32 1}
 !2 = !{i32 7, !"openmp", i32 50}
 !3 = !{i32 7, !"openmp-device", i32 50}
diff --git a/llvm/test/Transforms/OpenMP/icv_tracking.ll b/llvm/test/Transforms/OpenMP/icv_tracking.ll
index 7adab7a191220..47256ea599fdd 100644
--- a/llvm/test/Transforms/OpenMP/icv_tracking.ll
+++ b/llvm/test/Transforms/OpenMP/icv_tracking.ll
@@ -34,12 +34,14 @@ define i32 @ok_use_assume(i32 %0) {
 ; CHECK-SAME: (i32 [[TMP0:%.*]]) {
 ; CHECK-NEXT:    call void @use(i32 [[TMP0]]) #[[ATTR1:[0-9]+]]
 ; CHECK-NEXT:    call void @use(i32 [[TMP0]]) #[[ATTR2:[0-9]+]]
+; CHECK-NEXT:    call void @use(i32 [[TMP0]]) #[[ATTR2:[0-9]+]]
 ; CHECK-NEXT:    call void @no_openmp_use(i32 [[TMP0]])
 ; CHECK-NEXT:    [[TMP2:%.*]] = add nsw i32 [[TMP0]], 1
 ; CHECK-NEXT:    ret i32 [[TMP2]]
 ;
   call void @use(i32 %0) "no_openmp"
   call void @use(i32 %0) "no_openmp_routines"
+  call void @use(i32 %0) "no_openmp_constructs"
   call void @no_openmp_use(i32 %0)
   %2 = add nsw i32 %0, 1
   ret i32 %2
diff --git a/llvm/test/Transforms/OpenMP/is_spmd_exec_mode_fold.ll b/llvm/test/Transforms/OpenMP/is_spmd_exec_mode_fold.ll
index 310ac0a8296c3..2b3a7fabfb459 100644
--- a/llvm/test/Transforms/OpenMP/is_spmd_exec_mode_fold.ll
+++ b/llvm/test/Transforms/OpenMP/is_spmd_exec_mode_fold.ll
@@ -18,7 +18,7 @@ target triple = "nvptx64"
 ; CHECK: @none_spmd_kernel_environment = local_unnamed_addr constant %struct.KernelEnvironmentTy { %struct.ConfigurationEnvironmentTy { i8 0, i8 0, i8 1, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0 }, ptr null, ptr null }
 ; CHECK: @will_not_be_spmd_kernel_environment = local_unnamed_addr constant %struct.KernelEnvironmentTy { %struct.ConfigurationEnvironmentTy { i8 0, i8 0, i8 1, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0 }, ptr null, ptr null }
 ;.
-define weak void @is_spmd() "kernel" {
+define weak ptx_kernel void @is_spmd() "kernel" {
 ; CHECK-LABEL: define {{[^@]+}}@is_spmd
 ; CHECK-SAME: () #[[ATTR0:[0-9]+]] {
 ; CHECK-NEXT:    [[I:%.*]] = call i32 @__kmpc_target_init(ptr @is_spmd_kernel_environment, ptr null)
@@ -36,7 +36,7 @@ define weak void @is_spmd() "kernel" {
   ret void
 }
 
-define weak void @will_be_spmd() "kernel" {
+define weak ptx_kernel void @will_be_spmd() "kernel" {
 ; CHECK-LABEL: define {{[^@]+}}@will_be_spmd
 ; CHECK-SAME: () #[[ATTR0]] {
 ; CHECK-NEXT:  entry:
@@ -70,7 +70,7 @@ user_code.entry:
   ret void
 }
 
-define weak void @non_spmd() "kernel" {
+define weak ptx_kernel void @non_spmd() "kernel" {
 ; CHECK-LABEL: define {{[^@]+}}@non_spmd
 ; CHECK-SAME: () #[[ATTR0]] {
 ; CHECK-NEXT:    [[I:%.*]] = call i32 @__kmpc_target_init(ptr @none_spmd_kernel_environment, ptr null)
@@ -88,7 +88,7 @@ define weak void @non_spmd() "kernel" {
   ret void
 }
 
-define weak void @will_not_be_spmd() "kernel" {
+define weak ptx_kernel void @will_not_be_spmd() "kernel" {
 ; CHECK-LABEL: define {{[^@]+}}@will_not_be_spmd
 ; CHECK-SAME: () #[[ATTR0]] {
 ; CHECK-NEXT:    [[I:%.*]] = call i32 @__kmpc_target_init(ptr @will_not_be_spmd_kernel_environment, ptr null)
@@ -207,14 +207,9 @@ declare void @foo()
 declare void @bar()
 
 !llvm.module.flags = !{!0, !1}
-!nvvm.annotations = !{!2, !3, !4, !5}
 
 !0 = !{i32 7, !"openmp", i32 50}
 !1 = !{i32 7, !"openmp-device", i32 50}
-!2 = !{ptr @is_spmd, !"kernel", i32 1}
-!3 = !{ptr @will_be_spmd, !"kernel", i32 1}
-!4 = !{ptr @non_spmd, !"kernel", i32 1}
-!5 = !{ptr @will_not_be_spmd, !"kernel", i32 1}
 ;.
 ; CHECK: attributes #[[ATTR0]] = { "kernel" }
 ; CHECK: attributes #[[ATTR1:[0-9]+]] = { "llvm.assume"="ompx_spmd_amenable" }
@@ -223,8 +218,4 @@ declare void @bar()
 ;.
 ; CHECK: [[META0:![0-9]+]] = !{i32 7, !"openmp", i32 50}
 ; CHECK: [[META1:![0-9]+]] = !{i32 7, !"openmp-device", i32 50}
-; CHECK: [[META2:![0-9]+]] = !{ptr @is_spmd, !"kernel", i32 1}
-; CHECK: [[META3:![0-9]+]] = !{ptr @will_be_spmd, !"kernel", i32 1}
-; CHECK: [[META4:![0-9]+]] = !{ptr @non_spmd, !"kernel", i32 1}
-; CHECK: [[META5:![0-9]+]] = !{ptr @will_not_be_spmd, !"kernel", i32 1}
 ;.
diff --git a/llvm/test/Transforms/OpenMP/nested_parallelism.ll b/llvm/test/Transforms/OpenMP/nested_parallelism.ll
index 5c4386b24a3d5..1679a27fdae8b 100644
--- a/llvm/test/Transforms/OpenMP/nested_parallelism.ll
+++ b/llvm/test/Transforms/OpenMP/nested_parallelism.ll
@@ -43,7 +43,7 @@ target triple = "nvptx64"
 ; CHECK: @__omp_offloading_10302_bd7e0_main_l13_kernel_environment = local_unnamed_addr constant %struct.KernelEnvironmentTy { %struct.ConfigurationEnvironmentTy { i8 0, i8 0, i8 2, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0 }, ptr @[[GLOB1]], ptr null }
 ; CHECK: @__omp_offloading_10302_bd7e0_main_l16_kernel_environment = local_unnamed_addr constant %struct.KernelEnvironmentTy { %struct.ConfigurationEnvironmentTy { i8 1, i8 0, i8 1, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0 }, ptr @[[GLOB1]], ptr null }
 ;.
-define weak_odr protected void @__omp_offloading_10302_bd7e0_main_l13(ptr %dyn, i64 noundef %i) local_unnamed_addr "kernel" {
+define weak_odr protected ptx_kernel void @__omp_offloading_10302_bd7e0_main_l13(ptr %dyn, i64 noundef %i) local_unnamed_addr "kernel" {
 ; CHECK-LABEL: @__omp_offloading_10302_bd7e0_main_l13(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[CAPTURED_VARS_ADDRS_I:%.*]] = alloca [1 x ptr], align 8
@@ -127,7 +127,7 @@ entry:
 
 declare void @__kmpc_target_deinit(ptr, i8) local_unnamed_addr
 
-define weak_odr protected void @__omp_offloading_10302_bd7e0_main_l16(ptr %dyn, i64 noundef %i) local_unnamed_addr "kernel" {
+define weak_odr protected ptx_kernel void @__omp_offloading_10302_bd7e0_main_l16(ptr %dyn, i64 noundef %i) local_unnamed_addr "kernel" {
 ; CHECK-LABEL: @__omp_offloading_10302_bd7e0_main_l16(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[CAPTURED_VARS_ADDRS_I:%.*]] = alloca [1 x ptr], align 8
@@ -315,13 +315,10 @@ declare void @llvm.lifetime.end.p0(i64 immarg, ptr nocapture) #11
 
 
 !omp_offload.info = !{!0, !1}
-!nvvm.annotations = !{!2, !3}
 !llvm.module.flags = !{!4, !5}
 
 !0 = !{i32 0, i32 66306, i32 776160, !"main", i32 13, i32 0, i32 0}
 !1 = !{i32 0, i32 66306, i32 776160, !"main", i32 16, i32 0, i32 1}
-!2 = !{ptr @__omp_offloading_10302_bd7e0_main_l13, !"kernel", i32 1}
-!3 = !{ptr @__omp_offloading_10302_bd7e0_main_l16, !"kernel", i32 1}
 
 !4 = !{i32 7, !"openmp", i32 50}
 !5 = !{i32 7, !"openmp-device", i32 50}
@@ -336,8 +333,6 @@ declare void @llvm.lifetime.end.p0(i64 immarg, ptr nocapture) #11
 ;.
 ; CHECK: [[META0:![0-9]+]] = !{i32 0, i32 66306, i32 776160, !"main", i32 13, i32 0, i32 0}
 ; CHECK: [[META1:![0-9]+]] = !{i32 0, i32 66306, i32 776160, !"main", i32 16, i32 0, i32 1}
-; CHECK: [[META2:![0-9]+]] = !{ptr @__omp_offloading_10302_bd7e0_main_l13, !"kernel", i32 1}
-; CHECK: [[META3:![0-9]+]] = !{ptr @__omp_offloading_10302_bd7e0_main_l16, !"kernel", i32 1}
-; CHECK: [[META4:![0-9]+]] = !{i32 7, !"openmp", i32 50}
-; CHECK: [[META5:![0-9]+]] = !{i32 7, !"openmp-device", i32 50}
+; CHECK: [[META2:![0-9]+]] = !{i32 7, !"openmp", i32 50}
+; CHECK: [[META3:![0-9]+]] = !{i32 7, !"openmp-device", i32 50}
 ;.
diff --git a/llvm/test/Transforms/OpenMP/parallel_deletion.ll b/llvm/test/Transforms/OpenMP/parallel_deletion.ll
index 4619da1206092..3e16d96eb482b 100644
--- a/llvm/test/Transforms/OpenMP/parallel_deletion.ll
+++ b/llvm/test/Transforms/OpenMP/parallel_deletion.ll
@@ -47,7 +47,7 @@ entry:
 
 define internal void @.omp_outlined.willreturn(ptr noalias %.global_tid., ptr noalias %.bound_tid.) {
 ; CHECK-LABEL: define {{[^@]+}}@.omp_outlined.willreturn
-; CHECK-SAME: (ptr noalias nocapture nofree readnone [[DOTGLOBAL_TID_:%.*]], ptr noalias nocapture nofree readnone [[DOTBOUND_TID_:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-SAME: (ptr noalias nofree readnone captures(none) [[DOTGLOBAL_TID_:%.*]], ptr noalias nofree readnone captures(none) [[DOTBOUND_TID_:%.*]]) #[[ATTR0:[0-9]+]] {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    call void @unknown() #[[ATTR15:[0-9]+]]
 ; CHECK-NEXT:    ret void
@@ -69,7 +69,7 @@ entry:
 
 define internal void @.omp_outlined.willreturn.0(ptr noalias %.global_tid., ptr noalias %.bound_tid.) willreturn {
 ; CHECK-LABEL: define {{[^@]+}}@.omp_outlined.willreturn.0
-; CHECK-SAME: (ptr noalias nocapture nofree readnone [[DOTGLOBAL_TID_:%.*]], ptr noalias nocapture nofree readnone [[DOTBOUND_TID_:%.*]]) #[[ATTR1:[0-9]+]] {
+; CHECK-SAME: (ptr noalias nofree readnone captures(none) [[DOTGLOBAL_TID_:%.*]], ptr noalias nofree readnone captures(none) [[DOTBOUND_TID_:%.*]]) #[[ATTR1:[0-9]+]] {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    call void @readonly() #[[ATTR4:[0-9]+]]
 ; CHECK-NEXT:    ret void
@@ -91,7 +91,7 @@ entry:
 
 define internal void @.omp_outlined.willreturn.1(ptr noalias %.global_tid., ptr noalias %.bound_tid.) {
 ; CHECK-LABEL: define {{[^@]+}}@.omp_outlined.willreturn.1
-; CHECK-SAME: (ptr noalias nocapture nofree readnone [[DOTGLOBAL_TID_:%.*]], ptr noalias nocapture nofree readnone [[DOTBOUND_TID_:%.*]]) #[[ATTR2:[0-9]+]] {
+; CHECK-SAME: (ptr noalias nofree readnone captures(none) [[DOTGLOBAL_TID_:%.*]], ptr noalias nofree readnone captures(none) [[DOTBOUND_TID_:%.*]]) #[[ATTR2:[0-9]+]] {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    call void @readnone() #[[ATTR16:[0-9]+]]
 ; CHECK-NEXT:    ret void
@@ -113,7 +113,7 @@ entry:
 
 define internal void @.omp_outlined.willreturn.2(ptr noalias %.global_tid., ptr noalias %.bound_tid.) {
 ; CHECK-LABEL: define {{[^@]+}}@.omp_outlined.willreturn.2
-; CHECK-SAME: (ptr noalias nocapture nofree readnone [[DOTGLOBAL_TID_:%.*]], ptr noalias nocapture nofree readnone [[DOTBOUND_TID_:%.*]]) #[[ATTR3:[0-9]+]] {
+; CHECK-SAME: (ptr noalias nofree readnone captures(none) [[DOTGLOBAL_TID_:%.*]], ptr noalias nofree readnone captures(none) [[DOTBOUND_TID_:%.*]]) #[[ATTR3:[0-9]+]] {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    ret void
 ;
@@ -171,7 +171,7 @@ entry:
 
 define internal void @.omp_outlined.(ptr noalias %.global_tid., ptr noalias %.bound_tid.) {
 ; CHECK-LABEL: define {{[^@]+}}@.omp_outlined.
-; CHECK-SAME: (ptr noalias nocapture nofree readnone [[DOTGLOBAL_TID_:%.*]], ptr noalias nocapture nofree readnone [[DOTBOUND_TID_:%.*]]) {
+; CHECK-SAME: (ptr noalias nofree readnone captures(none) [[DOTGLOBAL_TID_:%.*]], ptr noalias nofree readnone captures(none) [[DOTBOUND_TID_:%.*]]) {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    call void @unknown()
 ; CHECK-NEXT:    ret void
@@ -193,7 +193,7 @@ entry:
 
 define internal void @.omp_outlined..0(ptr noalias %.global_tid., ptr noalias %.bound_tid.) {
 ; CHECK-LABEL: define {{[^@]+}}@.omp_outlined..0
-; CHECK-SAME: (ptr noalias nocapture nofree readnone [[DOTGLOBAL_TID_:%.*]], ptr noalias nocapture nofree readnone [[DOTBOUND_TID_:%.*]]) #[[ATTR4]] {
+; CHECK-SAME: (ptr noalias nofree readnone captures(none) [[DOTGLOBAL_TID_:%.*]], ptr noalias nofree readnone captures(none) [[DOTBOUND_TID_:%.*]]) #[[ATTR4]] {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    call void @readonly() #[[ATTR4]]
 ; CHECK-NEXT:    ret void
@@ -215,7 +215,7 @@ entry:
 
 define internal void @.omp_outlined..1(ptr noalias %.global_tid., ptr noalias %.bound_tid.) {
 ; CHECK-LABEL: define {{[^@]+}}@.omp_outlined..1
-; CHECK-SAME: (ptr noalias nocapture nofree readnone [[DOTGLOBAL_TID_:%.*]], ptr noalias nocapture nofree readnone [[DOTBOUND_TID_:%.*]]) #[[ATTR5:[0-9]+]] {
+; CHECK-SAME: (ptr noalias nofree readnone captures(none) [[DOTGLOBAL_TID_:%.*]], ptr noalias nofree readnone captures(none) [[DOTBOUND_TID_:%.*]]) #[[ATTR5:[0-9]+]] {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    call void @readnone() #[[ATTR17:[0-9]+]]
 ; CHECK-NEXT:    ret void
@@ -237,7 +237,7 @@ entry:
 
 define internal void @.omp_outlined..2(ptr noalias %.global_tid., ptr noalias %.bound_tid.) {
 ; CHECK-LABEL: define {{[^@]+}}@.omp_outlined..2
-; CHECK-SAME: (ptr noalias nocapture nofree readnone [[DOTGLOBAL_TID_:%.*]], ptr noalias nocapture nofree readnone [[DOTBOUND_TID_:%.*]]) #[[ATTR3]] {
+; CHECK-SAME: (ptr noalias nofree readnone captures(none) [[DOTGLOBAL_TID_:%.*]], ptr noalias nofree readnone captures(none) [[DOTBOUND_TID_:%.*]]) #[[ATTR3]] {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    ret void
 ;
@@ -284,10 +284,10 @@ define void @delete_parallel_2() {
 ; CHECK-NEXT:    [[A:%.*]] = alloca i32, align 4
 ; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 noundef 4, ptr noundef nonnull align 4 dereferenceable(4) [[A]]) #[[ATTR18:[0-9]+]]
 ; CHECK-NEXT:    store i32 0, ptr [[A]], align 4
-; CHECK-NEXT:    call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr noundef nonnull align 8 dereferenceable(24) @[[GLOB0]], i32 noundef 1, ptr noundef nonnull @.omp_outlined..3, ptr nocapture nofree noundef nonnull align 4 dereferenceable(4) [[A]])
-; CHECK-NEXT:    call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr noundef nonnull align 8 dereferenceable(24) @[[GLOB0]], i32 noundef 1, ptr noundef nonnull @.omp_outlined..4, ptr nocapture nofree noundef nonnull align 4 dereferenceable(4) [[A]])
-; CHECK-NEXT:    call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr noundef nonnull align 8 dereferenceable(24) @[[GLOB0]], i32 noundef 1, ptr noundef nonnull @.omp_outlined..5, ptr nocapture nofree noundef nonnull align 4 dereferenceable(4) [[A]])
-; CHECK-NEXT:    call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr noundef nonnull align 8 dereferenceable(24) @[[GLOB0]], i32 noundef 1, ptr noundef nonnull @.omp_outlined..6, ptr nocapture noundef nonnull align 4 dereferenceable(4) [[A]])
+; CHECK-NEXT:    call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr noundef nonnull align 8 dereferenceable(24) @[[GLOB0]], i32 noundef 1, ptr noundef nonnull @.omp_outlined..3, ptr nofree noundef nonnull align 4 captures(none) dereferenceable(4) [[A]])
+; CHECK-NEXT:    call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr noundef nonnull align 8 dereferenceable(24) @[[GLOB0]], i32 noundef 1, ptr noundef nonnull @.omp_outlined..4, ptr nofree noundef nonnull align 4 captures(none) dereferenceable(4) [[A]])
+; CHECK-NEXT:    call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr noundef nonnull align 8 dereferenceable(24) @[[GLOB0]], i32 noundef 1, ptr noundef nonnull @.omp_outlined..5, ptr nofree noundef nonnull align 4 captures(none) dereferenceable(4) [[A]])
+; CHECK-NEXT:    call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr noundef nonnull align 8 dereferenceable(24) @[[GLOB0]], i32 noundef 1, ptr noundef nonnull @.omp_outlined..6, ptr noundef nonnull align 4 captures(none) dereferenceable(4) [[A]])
 ; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 noundef 4, ptr noundef nonnull [[A]])
 ; CHECK-NEXT:    ret void
 ;
@@ -327,7 +327,7 @@ entry:
 
 define internal void @.omp_outlined..3(ptr noalias %.global_tid., ptr noalias %.bound_tid., ptr dereferenceable(4) %a) {
 ; CHECK-LABEL: define {{[^@]+}}@.omp_outlined..3
-; CHECK-SAME: (ptr noalias nocapture nofree readnone [[DOTGLOBAL_TID_:%.*]], ptr noalias nocapture nofree readnone [[DOTBOUND_TID_:%.*]], ptr nocapture nofree noundef nonnull align 4 dereferenceable(4) [[A:%.*]]) #[[ATTR6:[0-9]+]] {
+; CHECK-SAME: (ptr noalias nofree readnone captures(none) [[DOTGLOBAL_TID_:%.*]], ptr noalias nofree readnone captures(none) [[DOTBOUND_TID_:%.*]], ptr nofree noundef nonnull align 4 captures(none) dereferenceable(4) [[A:%.*]]) #[[ATTR6:[0-9]+]] {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[CALL:%.*]] = call i32 @omp_get_thread_num() #[[ATTR19:[0-9]+]]
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[CALL]], 0
@@ -383,7 +383,7 @@ if.end:                                           ; preds = %if.then, %entry
 
 define internal void @.omp_outlined..4(ptr noalias %.global_tid., ptr noalias %.bound_tid., ptr dereferenceable(4) %a) {
 ; CHECK-LABEL: define {{[^@]+}}@.omp_outlined..4
-; CHECK-SAME: (ptr noalias nocapture nofree noundef nonnull readonly align 4 dereferenceable(4) [[DOTGLOBAL_TID_:%.*]], ptr noalias nocapture nofree readnone [[DOTBOUND_TID_:%.*]], ptr nocapture nofree noundef nonnull align 4 dereferenceable(4) [[A:%.*]]) {
+; CHECK-SAME: (ptr noalias nofree noundef nonnull readonly align 4 captures(none) dereferenceable(4) [[DOTGLOBAL_TID_:%.*]], ptr noalias nofree readnone captures(none) [[DOTBOUND_TID_:%.*]], ptr nofree noundef nonnull align 4 captures(none) dereferenceable(4) [[A:%.*]]) {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[TMP:%.*]] = load i32, ptr [[DOTGLOBAL_TID_]], align 4
 ; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @__kmpc_master(ptr noundef nonnull @[[GLOB0]], i32 [[TMP]])
@@ -455,7 +455,7 @@ declare void @__kmpc_end_master(ptr, i32)
 
 define internal void @.omp_outlined..5(ptr noalias %.global_tid., ptr noalias %.bound_tid., ptr dereferenceable(4) %a) {
 ; CHECK-LABEL: define {{[^@]+}}@.omp_outlined..5
-; CHECK-SAME: (ptr noalias nocapture nofree readonly [[DOTGLOBAL_TID_:%.*]], ptr noalias nocapture nofree readnone [[DOTBOUND_TID_:%.*]], ptr nocapture nofree noundef nonnull align 4 dereferenceable(4) [[A:%.*]]) {
+; CHECK-SAME: (ptr noalias nofree readonly captures(none) [[DOTGLOBAL_TID_:%.*]], ptr noalias nofree readnone captures(none) [[DOTBOUND_TID_:%.*]], ptr nofree noundef nonnull align 4 captures(none) dereferenceable(4) [[A:%.*]]) {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[OMP_GLOBAL_THREAD_NUM:%.*]] = call i32 @__kmpc_global_thread_num(ptr noundef nonnull @[[GLOB0]]) #[[ATTR19]]
 ; CHECK-NEXT:    [[TMP:%.*]] = load i32, ptr [[DOTGLOBAL_TID_]], align 4
@@ -527,7 +527,7 @@ omp_if.end:                                       ; preds = %entry, %omp_if.then
 
 define internal void @.omp_outlined..6(ptr noalias %.global_tid., ptr noalias %.bound_tid., ptr dereferenceable(4) %a) {
 ; CHECK-LABEL: define {{[^@]+}}@.omp_outlined..6
-; CHECK-SAME: (ptr noalias nocapture nofree noundef nonnull readonly align 4 dereferenceable(4) [[DOTGLOBAL_TID_:%.*]], ptr noalias nocapture nofree readnone [[DOTBOUND_TID_:%.*]], ptr nocapture noundef nonnull align 4 dereferenceable(4) [[A:%.*]]) {
+; CHECK-SAME: (ptr noalias nofree noundef nonnull readonly align 4 captures(none) dereferenceable(4) [[DOTGLOBAL_TID_:%.*]], ptr noalias nofree readnone captures(none) [[DOTBOUND_TID_:%.*]], ptr noundef nonnull align 4 captures(none) dereferenceable(4) [[A:%.*]]) {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[A1:%.*]] = alloca i32, align 4
 ; CHECK-NEXT:    [[DOTOMP_REDUCTION_RED_LIST:%.*]] = alloca [1 x ptr], align 8
@@ -644,7 +644,7 @@ entry:
 
 define internal void @.omp.reduction.reduction_func(ptr %arg, ptr %arg1) {
 ; CHECK-LABEL: define {{[^@]+}}@.omp.reduction.reduction_func
-; CHECK-SAME: (ptr nocapture nofree noundef nonnull readonly align 8 dereferenceable(8) [[ARG:%.*]], ptr nocapture nofree noundef nonnull readonly align 8 dereferenceable(8) [[ARG1:%.*]]) #[[ATTR10:[0-9]+]] {
+; CHECK-SAME: (ptr nofree noundef nonnull readonly align 8 captures(none) dereferenceable(8) [[ARG:%.*]], ptr nofree noundef nonnull readonly align 8 captures(none) dereferenceable(8) [[ARG1:%.*]]) #[[ATTR10:[0-9]+]] {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[ARG1]], align 8
 ; CHECK-NEXT:    [[TMP4:%.*]] = load ptr, ptr [[ARG]], align 8
diff --git a/llvm/test/Transforms/OpenMP/parallel_level_fold.ll b/llvm/test/Transforms/OpenMP/parallel_level_fold.ll
index fd6e7683af8e3..e5f65b26ed223 100644
--- a/llvm/test/Transforms/OpenMP/parallel_level_fold.ll
+++ b/llvm/test/Transforms/OpenMP/parallel_level_fold.ll
@@ -16,7 +16,7 @@ target triple = "nvptx64"
 ; CHECK: @spmd_kernel_environment = local_unnamed_addr constant %struct.KernelEnvironmentTy { %struct.ConfigurationEnvironmentTy { i8 0, i8 0, i8 2, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0 }, ptr null, ptr null }
 ; CHECK: @parallel_kernel_environment = local_unnamed_addr constant %struct.KernelEnvironmentTy { %struct.ConfigurationEnvironmentTy { i8 0, i8 1, i8 2, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0 }, ptr null, ptr null }
 ;.
-define weak void @none_spmd() "kernel" {
+define weak ptx_kernel void @none_spmd() "kernel" {
 ; CHECK-LABEL: define {{[^@]+}}@none_spmd
 ; CHECK-SAME: () #[[ATTR0:[0-9]+]] {
 ; CHECK-NEXT:    [[I:%.*]] = call i32 @__kmpc_target_init(ptr @none_spmd_kernel_environment, ptr null)
@@ -32,7 +32,7 @@ define weak void @none_spmd() "kernel" {
   ret void
 }
 
-define weak void @spmd() "kernel" {
+define weak ptx_kernel void @spmd() "kernel" {
 ; CHECK-LABEL: define {{[^@]+}}@spmd
 ; CHECK-SAME: () #[[ATTR0]] {
 ; CHECK-NEXT:    [[I:%.*]] = call i32 @__kmpc_target_init(ptr @spmd_kernel_environment, ptr null)
@@ -48,7 +48,7 @@ define weak void @spmd() "kernel" {
   ret void
 }
 
-define weak void @parallel() "kernel" {
+define weak ptx_kernel void @parallel() "kernel" {
 ; CHECK-LABEL: define {{[^@]+}}@parallel
 ; CHECK-SAME: () #[[ATTR0]] {
 ; CHECK-NEXT:    [[I:%.*]] = call i32 @__kmpc_target_init(ptr @parallel_kernel_environment, ptr null)
@@ -136,20 +136,13 @@ declare i32 @__kmpc_target_init(ptr, ptr) #1
 declare void @__kmpc_target_deinit() #1
 
 !llvm.module.flags = !{!0, !1}
-!nvvm.annotations = !{!2, !3, !4}
 
 !0 = !{i32 7, !"openmp", i32 50}
 !1 = !{i32 7, !"openmp-device", i32 50}
-!2 = !{ptr @none_spmd, !"kernel", i32 1}
-!3 = !{ptr @spmd, !"kernel", i32 1}
-!4 = !{ptr @parallel, !"kernel", i32 1}
 ;.
 ; CHECK: attributes #[[ATTR0]] = { "kernel" }
 ; CHECK: attributes #[[ATTR1]] = { alwaysinline }
 ;.
 ; CHECK: [[META0:![0-9]+]] = !{i32 7, !"openmp", i32 50}
 ; CHECK: [[META1:![0-9]+]] = !{i32 7, !"openmp-device", i32 50}
-; CHECK: [[META2:![0-9]+]] = !{ptr @none_spmd, !"kernel", i32 1}
-; CHECK: [[META3:![0-9]+]] = !{ptr @spmd, !"kernel", i32 1}
-; CHECK: [[META4:![0-9]+]] = !{ptr @parallel, !"kernel", i32 1}
 ;.
diff --git a/llvm/test/Transforms/OpenMP/parallel_region_merging.ll b/llvm/test/Transforms/OpenMP/parallel_region_merging.ll
index f169fea09d0ea..bddab532b0d76 100644
--- a/llvm/test/Transforms/OpenMP/parallel_region_merging.ll
+++ b/llvm/test/Transforms/OpenMP/parallel_region_merging.ll
@@ -4704,10 +4704,10 @@ entry:
 ; CHECK2:       omp.par.region:
 ; CHECK2-NEXT:    br label [[OMP_PAR_MERGED:%.*]]
 ; CHECK2:       omp.par.merged:
-; CHECK2-NEXT:    call void (ptr, ptr, ...) @.omp_outlined.(ptr [[TID_ADDR]], ptr [[ZERO_ADDR]], ptr nocapture nofree noundef nonnull readonly align 4 dereferenceable(4) [[LOADGEP_A_ADDR]])
+; CHECK2-NEXT:    call void (ptr, ptr, ...) @.omp_outlined.(ptr [[TID_ADDR]], ptr [[ZERO_ADDR]], ptr nofree noundef nonnull readonly align 4 captures(none) dereferenceable(4) [[LOADGEP_A_ADDR]])
 ; CHECK2-NEXT:    [[OMP_GLOBAL_THREAD_NUM:%.*]] = call i32 @__kmpc_global_thread_num(ptr @[[GLOB2]])
 ; CHECK2-NEXT:    call void @__kmpc_barrier(ptr @[[GLOB3:[0-9]+]], i32 [[OMP_GLOBAL_THREAD_NUM]])
-; CHECK2-NEXT:    call void (ptr, ptr, ...) @.omp_outlined..1(ptr [[TID_ADDR]], ptr [[ZERO_ADDR]], ptr nocapture nofree noundef nonnull readonly align 4 dereferenceable(4) [[LOADGEP_A_ADDR]])
+; CHECK2-NEXT:    call void (ptr, ptr, ...) @.omp_outlined..1(ptr [[TID_ADDR]], ptr [[ZERO_ADDR]], ptr nofree noundef nonnull readonly align 4 captures(none) dereferenceable(4) [[LOADGEP_A_ADDR]])
 ; CHECK2-NEXT:    br label [[ENTRY_SPLIT:%.*]]
 ; CHECK2:       entry.split:
 ; CHECK2-NEXT:    br label [[OMP_PAR_REGION_SPLIT:%.*]]
@@ -4720,7 +4720,7 @@ entry:
 ;
 ;
 ; CHECK2-LABEL: define {{[^@]+}}@.omp_outlined.
-; CHECK2-SAME: (ptr noalias nocapture nofree readnone [[DOTGLOBAL_TID_:%.*]], ptr noalias nocapture nofree readnone [[DOTBOUND_TID_:%.*]], ptr nocapture nofree noundef nonnull readonly align 4 dereferenceable(4) [[A:%.*]]) {
+; CHECK2-SAME: (ptr noalias nofree readnone captures(none) [[DOTGLOBAL_TID_:%.*]], ptr noalias nofree readnone captures(none) [[DOTBOUND_TID_:%.*]], ptr nofree noundef nonnull readonly align 4 captures(none) dereferenceable(4) [[A:%.*]]) {
 ; CHECK2-NEXT:  entry:
 ; CHECK2-NEXT:    [[TMP0:%.*]] = load i32, ptr [[A]], align 4
 ; CHECK2-NEXT:    call void @use(i32 [[TMP0]])
@@ -4728,7 +4728,7 @@ entry:
 ;
 ;
 ; CHECK2-LABEL: define {{[^@]+}}@.omp_outlined..1
-; CHECK2-SAME: (ptr noalias nocapture nofree readnone [[DOTGLOBAL_TID_:%.*]], ptr noalias nocapture nofree readnone [[DOTBOUND_TID_:%.*]], ptr nocapture nofree noundef nonnull readonly align 4 dereferenceable(4) [[A:%.*]]) {
+; CHECK2-SAME: (ptr noalias nofree readnone captures(none) [[DOTGLOBAL_TID_:%.*]], ptr noalias nofree readnone captures(none) [[DOTBOUND_TID_:%.*]], ptr nofree noundef nonnull readonly align 4 captures(none) dereferenceable(4) [[A:%.*]]) {
 ; CHECK2-NEXT:  entry:
 ; CHECK2-NEXT:    [[TMP0:%.*]] = load i32, ptr [[A]], align 4
 ; CHECK2-NEXT:    call void @use(i32 [[TMP0]])
@@ -4742,13 +4742,13 @@ entry:
 ; CHECK2-NEXT:    [[TMP0:%.*]] = call i32 @__kmpc_global_thread_num(ptr noundef nonnull align 8 dereferenceable(24) @[[GLOB1:[0-9]+]])
 ; CHECK2-NEXT:    store i32 [[A]], ptr [[A_ADDR]], align 4
 ; CHECK2-NEXT:    call void @__kmpc_push_proc_bind(ptr noundef nonnull align 8 dereferenceable(24) @[[GLOB1]], i32 [[TMP0]], i32 noundef 3)
-; CHECK2-NEXT:    call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr noundef nonnull align 8 dereferenceable(24) @[[GLOB1]], i32 noundef 1, ptr noundef nonnull @.omp_outlined..2, ptr nocapture nofree noundef nonnull readonly align 4 dereferenceable(4) [[A_ADDR]])
-; CHECK2-NEXT:    call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr noundef nonnull align 8 dereferenceable(24) @[[GLOB1]], i32 noundef 1, ptr noundef nonnull @.omp_outlined..3, ptr nocapture nofree noundef nonnull readonly align 4 dereferenceable(4) [[A_ADDR]])
+; CHECK2-NEXT:    call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr noundef nonnull align 8 dereferenceable(24) @[[GLOB1]], i32 noundef 1, ptr noundef nonnull @.omp_outlined..2, ptr nofree noundef nonnull readonly align 4 captures(none) dereferenceable(4) [[A_ADDR]])
+; CHECK2-NEXT:    call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr noundef nonnull align 8 dereferenceable(24) @[[GLOB1]], i32 noundef 1, ptr noundef nonnull @.omp_outlined..3, ptr nofree noundef nonnull readonly align 4 captures(none) dereferenceable(4) [[A_ADDR]])
 ; CHECK2-NEXT:    ret void
 ;
 ;
 ; CHECK2-LABEL: define {{[^@]+}}@.omp_outlined..2
-; CHECK2-SAME: (ptr noalias nocapture nofree readnone [[DOTGLOBAL_TID_:%.*]], ptr noalias nocapture nofree readnone [[DOTBOUND_TID_:%.*]], ptr nocapture nofree noundef nonnull readonly align 4 dereferenceable(4) [[A:%.*]]) {
+; CHECK2-SAME: (ptr noalias nofree readnone captures(none) [[DOTGLOBAL_TID_:%.*]], ptr noalias nofree readnone captures(none) [[DOTBOUND_TID_:%.*]], ptr nofree noundef nonnull readonly align 4 captures(none) dereferenceable(4) [[A:%.*]]) {
 ; CHECK2-NEXT:  entry:
 ; CHECK2-NEXT:    [[TMP0:%.*]] = load i32, ptr [[A]], align 4
 ; CHECK2-NEXT:    call void @use(i32 [[TMP0]])
@@ -4756,7 +4756,7 @@ entry:
 ;
 ;
 ; CHECK2-LABEL: define {{[^@]+}}@.omp_outlined..3
-; CHECK2-SAME: (ptr noalias nocapture nofree readnone [[DOTGLOBAL_TID_:%.*]], ptr noalias nocapture nofree readnone [[DOTBOUND_TID_:%.*]], ptr nocapture nofree noundef nonnull readonly align 4 dereferenceable(4) [[A:%.*]]) {
+; CHECK2-SAME: (ptr noalias nofree readnone captures(none) [[DOTGLOBAL_TID_:%.*]], ptr noalias nofree readnone captures(none) [[DOTBOUND_TID_:%.*]], ptr nofree noundef nonnull readonly align 4 captures(none) dereferenceable(4) [[A:%.*]]) {
 ; CHECK2-NEXT:  entry:
 ; CHECK2-NEXT:    [[TMP0:%.*]] = load i32, ptr [[A]], align 4
 ; CHECK2-NEXT:    call void @use(i32 [[TMP0]])
@@ -4770,13 +4770,13 @@ entry:
 ; CHECK2-NEXT:    [[TMP0:%.*]] = call i32 @__kmpc_global_thread_num(ptr noundef nonnull align 8 dereferenceable(24) @[[GLOB1]])
 ; CHECK2-NEXT:    store i32 [[A]], ptr [[A_ADDR]], align 4
 ; CHECK2-NEXT:    call void @__kmpc_push_num_threads(ptr noundef nonnull align 8 dereferenceable(24) @[[GLOB1]], i32 [[TMP0]], i32 [[A]])
-; CHECK2-NEXT:    call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr noundef nonnull align 8 dereferenceable(24) @[[GLOB1]], i32 noundef 1, ptr noundef nonnull @.omp_outlined..4, ptr nocapture nofree noundef nonnull readonly align 4 dereferenceable(4) [[A_ADDR]])
-; CHECK2-NEXT:    call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr noundef nonnull align 8 dereferenceable(24) @[[GLOB1]], i32 noundef 1, ptr noundef nonnull @.omp_outlined..5, ptr nocapture nofree noundef nonnull readonly align 4 dereferenceable(4) [[A_ADDR]])
+; CHECK2-NEXT:    call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr noundef nonnull align 8 dereferenceable(24) @[[GLOB1]], i32 noundef 1, ptr noundef nonnull @.omp_outlined..4, ptr nofree noundef nonnull readonly align 4 captures(none) dereferenceable(4) [[A_ADDR]])
+; CHECK2-NEXT:    call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr noundef nonnull align 8 dereferenceable(24) @[[GLOB1]], i32 noundef 1, ptr noundef nonnull @.omp_outlined..5, ptr nofree noundef nonnull readonly align 4 captures(none) dereferenceable(4) [[A_ADDR]])
 ; CHECK2-NEXT:    ret void
 ;
 ;
 ; CHECK2-LABEL: define {{[^@]+}}@.omp_outlined..4
-; CHECK2-SAME: (ptr noalias nocapture nofree readnone [[DOTGLOBAL_TID_:%.*]], ptr noalias nocapture nofree readnone [[DOTBOUND_TID_:%.*]], ptr nocapture nofree noundef nonnull readonly align 4 dereferenceable(4) [[A:%.*]]) {
+; CHECK2-SAME: (ptr noalias nofree readnone captures(none) [[DOTGLOBAL_TID_:%.*]], ptr noalias nofree readnone captures(none) [[DOTBOUND_TID_:%.*]], ptr nofree noundef nonnull readonly align 4 captures(none) dereferenceable(4) [[A:%.*]]) {
 ; CHECK2-NEXT:  entry:
 ; CHECK2-NEXT:    [[TMP0:%.*]] = load i32, ptr [[A]], align 4
 ; CHECK2-NEXT:    call void @use(i32 [[TMP0]])
@@ -4784,7 +4784,7 @@ entry:
 ;
 ;
 ; CHECK2-LABEL: define {{[^@]+}}@.omp_outlined..5
-; CHECK2-SAME: (ptr noalias nocapture nofree readnone [[DOTGLOBAL_TID_:%.*]], ptr noalias nocapture nofree readnone [[DOTBOUND_TID_:%.*]], ptr nocapture nofree noundef nonnull readonly align 4 dereferenceable(4) [[A:%.*]]) {
+; CHECK2-SAME: (ptr noalias nofree readnone captures(none) [[DOTGLOBAL_TID_:%.*]], ptr noalias nofree readnone captures(none) [[DOTBOUND_TID_:%.*]], ptr nofree noundef nonnull readonly align 4 captures(none) dereferenceable(4) [[A:%.*]]) {
 ; CHECK2-NEXT:  entry:
 ; CHECK2-NEXT:    [[TMP0:%.*]] = load i32, ptr [[A]], align 4
 ; CHECK2-NEXT:    call void @use(i32 [[TMP0]])
@@ -4796,14 +4796,14 @@ entry:
 ; CHECK2-NEXT:  entry:
 ; CHECK2-NEXT:    [[A_ADDR:%.*]] = alloca i32, align 4
 ; CHECK2-NEXT:    store i32 [[A]], ptr [[A_ADDR]], align 4
-; CHECK2-NEXT:    call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr noundef nonnull align 8 dereferenceable(24) @[[GLOB1]], i32 noundef 1, ptr noundef nonnull @.omp_outlined..6, ptr nocapture nofree noundef nonnull readonly align 4 dereferenceable(4) [[A_ADDR]])
+; CHECK2-NEXT:    call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr noundef nonnull align 8 dereferenceable(24) @[[GLOB1]], i32 noundef 1, ptr noundef nonnull @.omp_outlined..6, ptr nofree noundef nonnull readonly align 4 captures(none) dereferenceable(4) [[A_ADDR]])
 ; CHECK2-NEXT:    call void (...) @foo()
-; CHECK2-NEXT:    call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr noundef nonnull align 8 dereferenceable(24) @[[GLOB1]], i32 noundef 1, ptr noundef nonnull @.omp_outlined..7, ptr nocapture nofree noundef nonnull readonly align 4 dereferenceable(4) [[A_ADDR]])
+; CHECK2-NEXT:    call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr noundef nonnull align 8 dereferenceable(24) @[[GLOB1]], i32 noundef 1, ptr noundef nonnull @.omp_outlined..7, ptr nofree noundef nonnull readonly align 4 captures(none) dereferenceable(4) [[A_ADDR]])
 ; CHECK2-NEXT:    ret void
 ;
 ;
 ; CHECK2-LABEL: define {{[^@]+}}@.omp_outlined..6
-; CHECK2-SAME: (ptr noalias nocapture nofree readnone [[DOTGLOBAL_TID_:%.*]], ptr noalias nocapture nofree readnone [[DOTBOUND_TID_:%.*]], ptr nocapture nofree noundef nonnull readonly align 4 dereferenceable(4) [[A:%.*]]) {
+; CHECK2-SAME: (ptr noalias nofree readnone captures(none) [[DOTGLOBAL_TID_:%.*]], ptr noalias nofree readnone captures(none) [[DOTBOUND_TID_:%.*]], ptr nofree noundef nonnull readonly align 4 captures(none) dereferenceable(4) [[A:%.*]]) {
 ; CHECK2-NEXT:  entry:
 ; CHECK2-NEXT:    [[TMP0:%.*]] = load i32, ptr [[A]], align 4
 ; CHECK2-NEXT:    call void @use(i32 [[TMP0]])
@@ -4811,7 +4811,7 @@ entry:
 ;
 ;
 ; CHECK2-LABEL: define {{[^@]+}}@.omp_outlined..7
-; CHECK2-SAME: (ptr noalias nocapture nofree readnone [[DOTGLOBAL_TID_:%.*]], ptr noalias nocapture nofree readnone [[DOTBOUND_TID_:%.*]], ptr nocapture nofree noundef nonnull readonly align 4 dereferenceable(4) [[A:%.*]]) {
+; CHECK2-SAME: (ptr noalias nofree readnone captures(none) [[DOTGLOBAL_TID_:%.*]], ptr noalias nofree readnone captures(none) [[DOTBOUND_TID_:%.*]], ptr nofree noundef nonnull readonly align 4 captures(none) dereferenceable(4) [[A:%.*]]) {
 ; CHECK2-NEXT:  entry:
 ; CHECK2-NEXT:    [[TMP0:%.*]] = load i32, ptr [[A]], align 4
 ; CHECK2-NEXT:    call void @use(i32 [[TMP0]])
@@ -4854,7 +4854,7 @@ entry:
 ; CHECK2:       omp.par.region:
 ; CHECK2-NEXT:    br label [[OMP_PAR_MERGED:%.*]]
 ; CHECK2:       omp.par.merged:
-; CHECK2-NEXT:    call void (ptr, ptr, ...) @.omp_outlined..8(ptr [[TID_ADDR]], ptr [[ZERO_ADDR]], ptr nocapture nofree noundef nonnull readonly align 4 dereferenceable(4) [[LOADGEP_A_ADDR]])
+; CHECK2-NEXT:    call void (ptr, ptr, ...) @.omp_outlined..8(ptr [[TID_ADDR]], ptr [[ZERO_ADDR]], ptr nofree noundef nonnull readonly align 4 captures(none) dereferenceable(4) [[LOADGEP_A_ADDR]])
 ; CHECK2-NEXT:    [[OMP_GLOBAL_THREAD_NUM2:%.*]] = call i32 @__kmpc_global_thread_num(ptr @[[GLOB2]])
 ; CHECK2-NEXT:    call void @__kmpc_barrier(ptr @[[GLOB3]], i32 [[OMP_GLOBAL_THREAD_NUM2]])
 ; CHECK2-NEXT:    [[OMP_GLOBAL_THREAD_NUM:%.*]] = call i32 @__kmpc_global_thread_num(ptr @[[GLOB2]])
@@ -4866,7 +4866,7 @@ entry:
 ; CHECK2-NEXT:    call void @__kmpc_barrier(ptr @[[GLOB3]], i32 [[OMP_GLOBAL_THREAD_NUM1]])
 ; CHECK2-NEXT:    br label [[OMP_PAR_MERGED_SPLIT_SPLIT:%.*]]
 ; CHECK2:       omp.par.merged.split.split:
-; CHECK2-NEXT:    call void (ptr, ptr, ...) @.omp_outlined..9(ptr [[TID_ADDR]], ptr [[ZERO_ADDR]], ptr nocapture nofree noundef nonnull readonly align 4 dereferenceable(4) [[LOADGEP_A_ADDR]])
+; CHECK2-NEXT:    call void (ptr, ptr, ...) @.omp_outlined..9(ptr [[TID_ADDR]], ptr [[ZERO_ADDR]], ptr nofree noundef nonnull readonly align 4 captures(none) dereferenceable(4) [[LOADGEP_A_ADDR]])
 ; CHECK2-NEXT:    br label [[ENTRY_SPLIT:%.*]]
 ; CHECK2:       entry.split:
 ; CHECK2-NEXT:    br label [[OMP_PAR_REGION_SPLIT:%.*]]
@@ -4891,7 +4891,7 @@ entry:
 ;
 ;
 ; CHECK2-LABEL: define {{[^@]+}}@.omp_outlined..8
-; CHECK2-SAME: (ptr noalias nocapture nofree readnone [[DOTGLOBAL_TID_:%.*]], ptr noalias nocapture nofree readnone [[DOTBOUND_TID_:%.*]], ptr nocapture nofree noundef nonnull readonly align 4 dereferenceable(4) [[A:%.*]]) {
+; CHECK2-SAME: (ptr noalias nofree readnone captures(none) [[DOTGLOBAL_TID_:%.*]], ptr noalias nofree readnone captures(none) [[DOTBOUND_TID_:%.*]], ptr nofree noundef nonnull readonly align 4 captures(none) dereferenceable(4) [[A:%.*]]) {
 ; CHECK2-NEXT:  entry:
 ; CHECK2-NEXT:    [[TMP0:%.*]] = load i32, ptr [[A]], align 4
 ; CHECK2-NEXT:    call void @use(i32 [[TMP0]])
@@ -4899,7 +4899,7 @@ entry:
 ;
 ;
 ; CHECK2-LABEL: define {{[^@]+}}@.omp_outlined..9
-; CHECK2-SAME: (ptr noalias nocapture nofree readnone [[DOTGLOBAL_TID_:%.*]], ptr noalias nocapture nofree readnone [[DOTBOUND_TID_:%.*]], ptr nocapture nofree noundef nonnull readonly align 4 dereferenceable(4) [[A:%.*]]) {
+; CHECK2-SAME: (ptr noalias nofree readnone captures(none) [[DOTGLOBAL_TID_:%.*]], ptr noalias nofree readnone captures(none) [[DOTBOUND_TID_:%.*]], ptr nofree noundef nonnull readonly align 4 captures(none) dereferenceable(4) [[A:%.*]]) {
 ; CHECK2-NEXT:  entry:
 ; CHECK2-NEXT:    [[TMP0:%.*]] = load i32, ptr [[A]], align 4
 ; CHECK2-NEXT:    call void @use(i32 [[TMP0]])
@@ -4907,7 +4907,7 @@ entry:
 ;
 ;
 ; CHECK2-LABEL: define {{[^@]+}}@merge_seq_float
-; CHECK2-SAME: (float [[F:%.*]], ptr nocapture nofree writeonly [[P:%.*]]) local_unnamed_addr {
+; CHECK2-SAME: (float [[F:%.*]], ptr nofree writeonly captures(none) [[P:%.*]]) local_unnamed_addr {
 ; CHECK2-NEXT:  entry:
 ; CHECK2-NEXT:    [[STRUCTARG:%.*]] = alloca { ptr, ptr, ptr }, align 8
 ; CHECK2-NEXT:    [[F_RELOADED:%.*]] = alloca float, align 4
@@ -4951,7 +4951,7 @@ entry:
 ; CHECK2:       omp.par.region:
 ; CHECK2-NEXT:    br label [[OMP_PAR_MERGED:%.*]]
 ; CHECK2:       omp.par.merged:
-; CHECK2-NEXT:    call void (ptr, ptr, ...) @.omp_outlined..10(ptr [[TID_ADDR]], ptr [[ZERO_ADDR]], ptr nocapture nofree noundef nonnull readonly align 4 dereferenceable(4) [[LOADGEP_F_ADDR]])
+; CHECK2-NEXT:    call void (ptr, ptr, ...) @.omp_outlined..10(ptr [[TID_ADDR]], ptr [[ZERO_ADDR]], ptr nofree noundef nonnull readonly align 4 captures(none) dereferenceable(4) [[LOADGEP_F_ADDR]])
 ; CHECK2-NEXT:    [[OMP_GLOBAL_THREAD_NUM2:%.*]] = call i32 @__kmpc_global_thread_num(ptr @[[GLOB2]])
 ; CHECK2-NEXT:    call void @__kmpc_barrier(ptr @[[GLOB3]], i32 [[OMP_GLOBAL_THREAD_NUM2]])
 ; CHECK2-NEXT:    [[OMP_GLOBAL_THREAD_NUM:%.*]] = call i32 @__kmpc_global_thread_num(ptr @[[GLOB2]])
@@ -4963,7 +4963,7 @@ entry:
 ; CHECK2-NEXT:    call void @__kmpc_barrier(ptr @[[GLOB3]], i32 [[OMP_GLOBAL_THREAD_NUM1]])
 ; CHECK2-NEXT:    br label [[OMP_PAR_MERGED_SPLIT_SPLIT:%.*]]
 ; CHECK2:       omp.par.merged.split.split:
-; CHECK2-NEXT:    call void (ptr, ptr, ...) @.omp_outlined..11(ptr [[TID_ADDR]], ptr [[ZERO_ADDR]], ptr nocapture nofree noundef nonnull readonly align 4 dereferenceable(4) [[LOADGEP_F_ADDR]])
+; CHECK2-NEXT:    call void (ptr, ptr, ...) @.omp_outlined..11(ptr [[TID_ADDR]], ptr [[ZERO_ADDR]], ptr nofree noundef nonnull readonly align 4 captures(none) dereferenceable(4) [[LOADGEP_F_ADDR]])
 ; CHECK2-NEXT:    br label [[ENTRY_SPLIT:%.*]]
 ; CHECK2:       entry.split:
 ; CHECK2-NEXT:    br label [[OMP_PAR_REGION_SPLIT:%.*]]
@@ -4987,7 +4987,7 @@ entry:
 ;
 ;
 ; CHECK2-LABEL: define {{[^@]+}}@.omp_outlined..10
-; CHECK2-SAME: (ptr noalias nocapture nofree readnone [[DOTGLOBAL_TID_:%.*]], ptr noalias nocapture nofree readnone [[DOTBOUND_TID_:%.*]], ptr nocapture nofree noundef nonnull readonly align 4 dereferenceable(4) [[F:%.*]]) {
+; CHECK2-SAME: (ptr noalias nofree readnone captures(none) [[DOTGLOBAL_TID_:%.*]], ptr noalias nofree readnone captures(none) [[DOTBOUND_TID_:%.*]], ptr nofree noundef nonnull readonly align 4 captures(none) dereferenceable(4) [[F:%.*]]) {
 ; CHECK2-NEXT:  entry:
 ; CHECK2-NEXT:    [[TMP0:%.*]] = load float, ptr [[F]], align 4
 ; CHECK2-NEXT:    [[CONV:%.*]] = fptosi float [[TMP0]] to i32
@@ -4996,7 +4996,7 @@ entry:
 ;
 ;
 ; CHECK2-LABEL: define {{[^@]+}}@.omp_outlined..11
-; CHECK2-SAME: (ptr noalias nocapture nofree readnone [[DOTGLOBAL_TID_:%.*]], ptr noalias nocapture nofree readnone [[DOTBOUND_TID_:%.*]], ptr nocapture nofree noundef nonnull readonly align 4 dereferenceable(4) [[F:%.*]]) {
+; CHECK2-SAME: (ptr noalias nofree readnone captures(none) [[DOTGLOBAL_TID_:%.*]], ptr noalias nofree readnone captures(none) [[DOTBOUND_TID_:%.*]], ptr nofree noundef nonnull readonly align 4 captures(none) dereferenceable(4) [[F:%.*]]) {
 ; CHECK2-NEXT:  entry:
 ; CHECK2-NEXT:    [[TMP0:%.*]] = load float, ptr [[F]], align 4
 ; CHECK2-NEXT:    [[CONV:%.*]] = fptosi float [[TMP0]] to i32
@@ -5045,7 +5045,7 @@ entry:
 ; CHECK2:       omp.par.region:
 ; CHECK2-NEXT:    br label [[OMP_PAR_MERGED:%.*]]
 ; CHECK2:       omp.par.merged:
-; CHECK2-NEXT:    call void (ptr, ptr, ...) @.omp_outlined..12(ptr [[TID_ADDR]], ptr [[ZERO_ADDR]], ptr nocapture nofree noundef nonnull readonly align 4 dereferenceable(4) [[LOADGEP_A_ADDR]])
+; CHECK2-NEXT:    call void (ptr, ptr, ...) @.omp_outlined..12(ptr [[TID_ADDR]], ptr [[ZERO_ADDR]], ptr nofree noundef nonnull readonly align 4 captures(none) dereferenceable(4) [[LOADGEP_A_ADDR]])
 ; CHECK2-NEXT:    [[OMP_GLOBAL_THREAD_NUM2:%.*]] = call i32 @__kmpc_global_thread_num(ptr @[[GLOB2]])
 ; CHECK2-NEXT:    call void @__kmpc_barrier(ptr @[[GLOB3]], i32 [[OMP_GLOBAL_THREAD_NUM2]])
 ; CHECK2-NEXT:    [[OMP_GLOBAL_THREAD_NUM:%.*]] = call i32 @__kmpc_global_thread_num(ptr @[[GLOB2]])
@@ -5085,7 +5085,7 @@ entry:
 ;
 ;
 ; CHECK2-LABEL: define {{[^@]+}}@.omp_outlined..12
-; CHECK2-SAME: (ptr noalias nocapture nofree readnone [[DOTGLOBAL_TID_:%.*]], ptr noalias nocapture nofree readnone [[DOTBOUND_TID_:%.*]], ptr nocapture nofree noundef nonnull readonly align 4 dereferenceable(4) [[A:%.*]]) {
+; CHECK2-SAME: (ptr noalias nofree readnone captures(none) [[DOTGLOBAL_TID_:%.*]], ptr noalias nofree readnone captures(none) [[DOTBOUND_TID_:%.*]], ptr nofree noundef nonnull readonly align 4 captures(none) dereferenceable(4) [[A:%.*]]) {
 ; CHECK2-NEXT:  entry:
 ; CHECK2-NEXT:    [[TMP0:%.*]] = load i32, ptr [[A]], align 4
 ; CHECK2-NEXT:    call void @use(i32 [[TMP0]])
@@ -5093,7 +5093,7 @@ entry:
 ;
 ;
 ; CHECK2-LABEL: define {{[^@]+}}@.omp_outlined..13
-; CHECK2-SAME: (ptr noalias nocapture nofree readnone [[DOTGLOBAL_TID_:%.*]], ptr noalias nocapture nofree readnone [[DOTBOUND_TID_:%.*]], i64 [[A:%.*]]) {
+; CHECK2-SAME: (ptr noalias nofree readnone captures(none) [[DOTGLOBAL_TID_:%.*]], ptr noalias nofree readnone captures(none) [[DOTBOUND_TID_:%.*]], i64 [[A:%.*]]) {
 ; CHECK2-NEXT:  entry:
 ; CHECK2-NEXT:    [[A_ADDR_SROA_0_0_EXTRACT_TRUNC:%.*]] = trunc i64 [[A]] to i32
 ; CHECK2-NEXT:    call void @use(i32 [[A_ADDR_SROA_0_0_EXTRACT_TRUNC]])
@@ -5135,7 +5135,7 @@ entry:
 ; CHECK2:       omp.par.region:
 ; CHECK2-NEXT:    br label [[OMP_PAR_MERGED:%.*]]
 ; CHECK2:       omp.par.merged:
-; CHECK2-NEXT:    call void (ptr, ptr, ...) @.omp_outlined..14(ptr [[TID_ADDR]], ptr [[ZERO_ADDR]], ptr nocapture nofree noundef nonnull readonly align 4 dereferenceable(4) [[LOADGEP_A_ADDR]])
+; CHECK2-NEXT:    call void (ptr, ptr, ...) @.omp_outlined..14(ptr [[TID_ADDR]], ptr [[ZERO_ADDR]], ptr nofree noundef nonnull readonly align 4 captures(none) dereferenceable(4) [[LOADGEP_A_ADDR]])
 ; CHECK2-NEXT:    [[OMP_GLOBAL_THREAD_NUM2:%.*]] = call i32 @__kmpc_global_thread_num(ptr @[[GLOB2]])
 ; CHECK2-NEXT:    call void @__kmpc_barrier(ptr @[[GLOB3]], i32 [[OMP_GLOBAL_THREAD_NUM2]])
 ; CHECK2-NEXT:    [[OMP_GLOBAL_THREAD_NUM:%.*]] = call i32 @__kmpc_global_thread_num(ptr @[[GLOB2]])
@@ -5147,7 +5147,7 @@ entry:
 ; CHECK2-NEXT:    call void @__kmpc_barrier(ptr @[[GLOB3]], i32 [[OMP_GLOBAL_THREAD_NUM1]])
 ; CHECK2-NEXT:    br label [[OMP_PAR_MERGED_SPLIT_SPLIT:%.*]]
 ; CHECK2:       omp.par.merged.split.split:
-; CHECK2-NEXT:    call void (ptr, ptr, ...) @.omp_outlined..15(ptr [[TID_ADDR]], ptr [[ZERO_ADDR]], ptr nocapture nofree noundef nonnull readonly align 4 dereferenceable(4) [[LOADGEP_A_ADDR]])
+; CHECK2-NEXT:    call void (ptr, ptr, ...) @.omp_outlined..15(ptr [[TID_ADDR]], ptr [[ZERO_ADDR]], ptr nofree noundef nonnull readonly align 4 captures(none) dereferenceable(4) [[LOADGEP_A_ADDR]])
 ; CHECK2-NEXT:    br label [[ENTRY_SPLIT:%.*]]
 ; CHECK2:       entry.split:
 ; CHECK2-NEXT:    br label [[OMP_PAR_REGION_SPLIT:%.*]]
@@ -5174,7 +5174,7 @@ entry:
 ;
 ;
 ; CHECK2-LABEL: define {{[^@]+}}@.omp_outlined..14
-; CHECK2-SAME: (ptr noalias nocapture nofree readnone [[DOTGLOBAL_TID_:%.*]], ptr noalias nocapture nofree readnone [[DOTBOUND_TID_:%.*]], ptr nocapture nofree noundef nonnull readonly align 4 dereferenceable(4) [[A:%.*]]) {
+; CHECK2-SAME: (ptr noalias nofree readnone captures(none) [[DOTGLOBAL_TID_:%.*]], ptr noalias nofree readnone captures(none) [[DOTBOUND_TID_:%.*]], ptr nofree noundef nonnull readonly align 4 captures(none) dereferenceable(4) [[A:%.*]]) {
 ; CHECK2-NEXT:  entry:
 ; CHECK2-NEXT:    [[TMP0:%.*]] = load i32, ptr [[A]], align 4
 ; CHECK2-NEXT:    call void @use(i32 [[TMP0]])
@@ -5182,7 +5182,7 @@ entry:
 ;
 ;
 ; CHECK2-LABEL: define {{[^@]+}}@.omp_outlined..15
-; CHECK2-SAME: (ptr noalias nocapture nofree readnone [[DOTGLOBAL_TID_:%.*]], ptr noalias nocapture nofree readnone [[DOTBOUND_TID_:%.*]], ptr nocapture nofree noundef nonnull readonly align 4 dereferenceable(4) [[A:%.*]]) {
+; CHECK2-SAME: (ptr noalias nofree readnone captures(none) [[DOTGLOBAL_TID_:%.*]], ptr noalias nofree readnone captures(none) [[DOTBOUND_TID_:%.*]], ptr nofree noundef nonnull readonly align 4 captures(none) dereferenceable(4) [[A:%.*]]) {
 ; CHECK2-NEXT:  entry:
 ; CHECK2-NEXT:    [[TMP0:%.*]] = load i32, ptr [[A]], align 4
 ; CHECK2-NEXT:    call void @use(i32 [[TMP0]])
@@ -5237,7 +5237,7 @@ entry:
 ; CHECK2:       omp.par.region:
 ; CHECK2-NEXT:    br label [[OMP_PAR_MERGED:%.*]]
 ; CHECK2:       omp.par.merged:
-; CHECK2-NEXT:    call void (ptr, ptr, ...) @.omp_outlined..16(ptr [[TID_ADDR]], ptr [[ZERO_ADDR]], ptr nocapture nofree noundef nonnull readonly align 4 dereferenceable(4) [[LOADGEP_A_ADDR]])
+; CHECK2-NEXT:    call void (ptr, ptr, ...) @.omp_outlined..16(ptr [[TID_ADDR]], ptr [[ZERO_ADDR]], ptr nofree noundef nonnull readonly align 4 captures(none) dereferenceable(4) [[LOADGEP_A_ADDR]])
 ; CHECK2-NEXT:    [[OMP_GLOBAL_THREAD_NUM2:%.*]] = call i32 @__kmpc_global_thread_num(ptr @[[GLOB2]])
 ; CHECK2-NEXT:    call void @__kmpc_barrier(ptr @[[GLOB3]], i32 [[OMP_GLOBAL_THREAD_NUM2]])
 ; CHECK2-NEXT:    [[OMP_GLOBAL_THREAD_NUM:%.*]] = call i32 @__kmpc_global_thread_num(ptr @[[GLOB2]])
@@ -5249,7 +5249,7 @@ entry:
 ; CHECK2-NEXT:    call void @__kmpc_barrier(ptr @[[GLOB3]], i32 [[OMP_GLOBAL_THREAD_NUM1]])
 ; CHECK2-NEXT:    br label [[OMP_PAR_MERGED_SPLIT_SPLIT:%.*]]
 ; CHECK2:       omp.par.merged.split.split:
-; CHECK2-NEXT:    call void (ptr, ptr, ...) @.omp_outlined..17(ptr [[TID_ADDR]], ptr [[ZERO_ADDR]], ptr nocapture nofree noundef nonnull readonly align 4 dereferenceable(4) [[LOADGEP_A_ADDR]], ptr nocapture nofree noundef nonnull readonly align 4 dereferenceable(4) [[LOADGEP_B]])
+; CHECK2-NEXT:    call void (ptr, ptr, ...) @.omp_outlined..17(ptr [[TID_ADDR]], ptr [[ZERO_ADDR]], ptr nofree noundef nonnull readonly align 4 captures(none) dereferenceable(4) [[LOADGEP_A_ADDR]], ptr nofree noundef nonnull readonly align 4 captures(none) dereferenceable(4) [[LOADGEP_B]])
 ; CHECK2-NEXT:    br label [[ENTRY_SPLIT:%.*]]
 ; CHECK2:       entry.split:
 ; CHECK2-NEXT:    br label [[OMP_PAR_REGION_SPLIT:%.*]]
@@ -5273,7 +5273,7 @@ entry:
 ;
 ;
 ; CHECK2-LABEL: define {{[^@]+}}@.omp_outlined..16
-; CHECK2-SAME: (ptr noalias nocapture nofree readnone [[DOTGLOBAL_TID_:%.*]], ptr noalias nocapture nofree readnone [[DOTBOUND_TID_:%.*]], ptr nocapture nofree noundef nonnull readonly align 4 dereferenceable(4) [[A:%.*]]) {
+; CHECK2-SAME: (ptr noalias nofree readnone captures(none) [[DOTGLOBAL_TID_:%.*]], ptr noalias nofree readnone captures(none) [[DOTBOUND_TID_:%.*]], ptr nofree noundef nonnull readonly align 4 captures(none) dereferenceable(4) [[A:%.*]]) {
 ; CHECK2-NEXT:  entry:
 ; CHECK2-NEXT:    [[TMP0:%.*]] = load i32, ptr [[A]], align 4
 ; CHECK2-NEXT:    call void @use(i32 [[TMP0]])
@@ -5281,7 +5281,7 @@ entry:
 ;
 ;
 ; CHECK2-LABEL: define {{[^@]+}}@.omp_outlined..17
-; CHECK2-SAME: (ptr noalias nocapture nofree readnone [[DOTGLOBAL_TID_:%.*]], ptr noalias nocapture nofree readnone [[DOTBOUND_TID_:%.*]], ptr nocapture nofree noundef nonnull readonly align 4 dereferenceable(4) [[A:%.*]], ptr nocapture nofree noundef nonnull readonly align 4 dereferenceable(4) [[B:%.*]]) {
+; CHECK2-SAME: (ptr noalias nofree readnone captures(none) [[DOTGLOBAL_TID_:%.*]], ptr noalias nofree readnone captures(none) [[DOTBOUND_TID_:%.*]], ptr nofree noundef nonnull readonly align 4 captures(none) dereferenceable(4) [[A:%.*]], ptr nofree noundef nonnull readonly align 4 captures(none) dereferenceable(4) [[B:%.*]]) {
 ; CHECK2-NEXT:  entry:
 ; CHECK2-NEXT:    [[TMP0:%.*]] = load i32, ptr [[A]], align 4
 ; CHECK2-NEXT:    call void @use(i32 [[TMP0]])
@@ -5330,10 +5330,10 @@ entry:
 ; CHECK2:       omp.par.region:
 ; CHECK2-NEXT:    br label [[OMP_PAR_MERGED:%.*]]
 ; CHECK2:       omp.par.merged:
-; CHECK2-NEXT:    call void (ptr, ptr, ...) @.omp_outlined..18(ptr [[TID_ADDR]], ptr [[ZERO_ADDR]], ptr nocapture nofree noundef nonnull readonly align 4 dereferenceable(4) [[LOADGEP_CANCEL1_ADDR]])
+; CHECK2-NEXT:    call void (ptr, ptr, ...) @.omp_outlined..18(ptr [[TID_ADDR]], ptr [[ZERO_ADDR]], ptr nofree noundef nonnull readonly align 4 captures(none) dereferenceable(4) [[LOADGEP_CANCEL1_ADDR]])
 ; CHECK2-NEXT:    [[OMP_GLOBAL_THREAD_NUM:%.*]] = call i32 @__kmpc_global_thread_num(ptr @[[GLOB2]])
 ; CHECK2-NEXT:    call void @__kmpc_barrier(ptr @[[GLOB3]], i32 [[OMP_GLOBAL_THREAD_NUM]])
-; CHECK2-NEXT:    call void (ptr, ptr, ...) @.omp_outlined..19(ptr [[TID_ADDR]], ptr [[ZERO_ADDR]], ptr nocapture nofree noundef nonnull readonly align 4 dereferenceable(4) [[LOADGEP_CANCEL2_ADDR]])
+; CHECK2-NEXT:    call void (ptr, ptr, ...) @.omp_outlined..19(ptr [[TID_ADDR]], ptr [[ZERO_ADDR]], ptr nofree noundef nonnull readonly align 4 captures(none) dereferenceable(4) [[LOADGEP_CANCEL2_ADDR]])
 ; CHECK2-NEXT:    br label [[ENTRY_SPLIT:%.*]]
 ; CHECK2:       entry.split:
 ; CHECK2-NEXT:    br label [[OMP_PAR_REGION_SPLIT:%.*]]
@@ -5346,7 +5346,7 @@ entry:
 ;
 ;
 ; CHECK2-LABEL: define {{[^@]+}}@.omp_outlined..18
-; CHECK2-SAME: (ptr noalias nocapture nofree readonly [[DOTGLOBAL_TID_:%.*]], ptr noalias nocapture nofree readnone [[DOTBOUND_TID_:%.*]], ptr nocapture nofree noundef nonnull readonly align 4 dereferenceable(4) [[CANCEL1:%.*]]) {
+; CHECK2-SAME: (ptr noalias nofree readonly captures(none) [[DOTGLOBAL_TID_:%.*]], ptr noalias nofree readnone captures(none) [[DOTBOUND_TID_:%.*]], ptr nofree noundef nonnull readonly align 4 captures(none) dereferenceable(4) [[CANCEL1:%.*]]) {
 ; CHECK2-NEXT:  entry:
 ; CHECK2-NEXT:    [[TMP0:%.*]] = load i32, ptr [[CANCEL1]], align 4
 ; CHECK2-NEXT:    [[TOBOOL_NOT:%.*]] = icmp eq i32 [[TMP0]], 0
@@ -5360,7 +5360,7 @@ entry:
 ;
 ;
 ; CHECK2-LABEL: define {{[^@]+}}@.omp_outlined..19
-; CHECK2-SAME: (ptr noalias nocapture nofree readonly [[DOTGLOBAL_TID_:%.*]], ptr noalias nocapture nofree readnone [[DOTBOUND_TID_:%.*]], ptr nocapture nofree noundef nonnull readonly align 4 dereferenceable(4) [[CANCEL2:%.*]]) {
+; CHECK2-SAME: (ptr noalias nofree readonly captures(none) [[DOTGLOBAL_TID_:%.*]], ptr noalias nofree readnone captures(none) [[DOTBOUND_TID_:%.*]], ptr nofree noundef nonnull readonly align 4 captures(none) dereferenceable(4) [[CANCEL2:%.*]]) {
 ; CHECK2-NEXT:  entry:
 ; CHECK2-NEXT:    [[TMP0:%.*]] = load i32, ptr [[CANCEL2]], align 4
 ; CHECK2-NEXT:    [[TOBOOL_NOT:%.*]] = icmp eq i32 [[TMP0]], 0
@@ -5420,7 +5420,7 @@ entry:
 ; CHECK2:       omp.par.region:
 ; CHECK2-NEXT:    br label [[OMP_PAR_MERGED:%.*]]
 ; CHECK2:       omp.par.merged:
-; CHECK2-NEXT:    call void (ptr, ptr, ...) @.omp_outlined..20(ptr [[TID_ADDR]], ptr [[ZERO_ADDR]], ptr nocapture nofree noundef nonnull readonly align 4 dereferenceable(4) [[LOADGEP_CANCEL1_ADDR]])
+; CHECK2-NEXT:    call void (ptr, ptr, ...) @.omp_outlined..20(ptr [[TID_ADDR]], ptr [[ZERO_ADDR]], ptr nofree noundef nonnull readonly align 4 captures(none) dereferenceable(4) [[LOADGEP_CANCEL1_ADDR]])
 ; CHECK2-NEXT:    [[OMP_GLOBAL_THREAD_NUM2:%.*]] = call i32 @__kmpc_global_thread_num(ptr @[[GLOB2]])
 ; CHECK2-NEXT:    call void @__kmpc_barrier(ptr @[[GLOB3]], i32 [[OMP_GLOBAL_THREAD_NUM2]])
 ; CHECK2-NEXT:    [[OMP_GLOBAL_THREAD_NUM:%.*]] = call i32 @__kmpc_global_thread_num(ptr @[[GLOB2]])
@@ -5432,7 +5432,7 @@ entry:
 ; CHECK2-NEXT:    call void @__kmpc_barrier(ptr @[[GLOB3]], i32 [[OMP_GLOBAL_THREAD_NUM1]])
 ; CHECK2-NEXT:    br label [[OMP_PAR_MERGED_SPLIT_SPLIT:%.*]]
 ; CHECK2:       omp.par.merged.split.split:
-; CHECK2-NEXT:    call void (ptr, ptr, ...) @.omp_outlined..21(ptr [[TID_ADDR]], ptr [[ZERO_ADDR]], ptr nocapture nofree noundef nonnull readonly align 4 dereferenceable(4) [[LOADGEP_CANCEL2_ADDR]])
+; CHECK2-NEXT:    call void (ptr, ptr, ...) @.omp_outlined..21(ptr [[TID_ADDR]], ptr [[ZERO_ADDR]], ptr nofree noundef nonnull readonly align 4 captures(none) dereferenceable(4) [[LOADGEP_CANCEL2_ADDR]])
 ; CHECK2-NEXT:    br label [[ENTRY_SPLIT:%.*]]
 ; CHECK2:       entry.split:
 ; CHECK2-NEXT:    br label [[OMP_PAR_REGION_SPLIT:%.*]]
@@ -5457,7 +5457,7 @@ entry:
 ;
 ;
 ; CHECK2-LABEL: define {{[^@]+}}@.omp_outlined..20
-; CHECK2-SAME: (ptr noalias nocapture nofree readonly [[DOTGLOBAL_TID_:%.*]], ptr noalias nocapture nofree readnone [[DOTBOUND_TID_:%.*]], ptr nocapture nofree noundef nonnull readonly align 4 dereferenceable(4) [[CANCEL1:%.*]]) {
+; CHECK2-SAME: (ptr noalias nofree readonly captures(none) [[DOTGLOBAL_TID_:%.*]], ptr noalias nofree readnone captures(none) [[DOTBOUND_TID_:%.*]], ptr nofree noundef nonnull readonly align 4 captures(none) dereferenceable(4) [[CANCEL1:%.*]]) {
 ; CHECK2-NEXT:  entry:
 ; CHECK2-NEXT:    [[TMP0:%.*]] = load i32, ptr [[CANCEL1]], align 4
 ; CHECK2-NEXT:    [[TOBOOL_NOT:%.*]] = icmp eq i32 [[TMP0]], 0
@@ -5471,7 +5471,7 @@ entry:
 ;
 ;
 ; CHECK2-LABEL: define {{[^@]+}}@.omp_outlined..21
-; CHECK2-SAME: (ptr noalias nocapture nofree readonly [[DOTGLOBAL_TID_:%.*]], ptr noalias nocapture nofree readnone [[DOTBOUND_TID_:%.*]], ptr nocapture nofree noundef nonnull readonly align 4 dereferenceable(4) [[CANCEL2:%.*]]) {
+; CHECK2-SAME: (ptr noalias nofree readonly captures(none) [[DOTGLOBAL_TID_:%.*]], ptr noalias nofree readnone captures(none) [[DOTBOUND_TID_:%.*]], ptr nofree noundef nonnull readonly align 4 captures(none) dereferenceable(4) [[CANCEL2:%.*]]) {
 ; CHECK2-NEXT:  entry:
 ; CHECK2-NEXT:    [[TMP0:%.*]] = load i32, ptr [[CANCEL2]], align 4
 ; CHECK2-NEXT:    [[TOBOOL_NOT:%.*]] = icmp eq i32 [[TMP0]], 0
@@ -5518,13 +5518,13 @@ entry:
 ; CHECK2:       omp.par.region:
 ; CHECK2-NEXT:    br label [[OMP_PAR_MERGED:%.*]]
 ; CHECK2:       omp.par.merged:
-; CHECK2-NEXT:    call void (ptr, ptr, ...) @.omp_outlined..22(ptr [[TID_ADDR]], ptr [[ZERO_ADDR]], ptr nocapture nofree noundef nonnull readonly align 4 dereferenceable(4) [[LOADGEP_A_ADDR]])
+; CHECK2-NEXT:    call void (ptr, ptr, ...) @.omp_outlined..22(ptr [[TID_ADDR]], ptr [[ZERO_ADDR]], ptr nofree noundef nonnull readonly align 4 captures(none) dereferenceable(4) [[LOADGEP_A_ADDR]])
 ; CHECK2-NEXT:    [[OMP_GLOBAL_THREAD_NUM:%.*]] = call i32 @__kmpc_global_thread_num(ptr @[[GLOB2]])
 ; CHECK2-NEXT:    call void @__kmpc_barrier(ptr @[[GLOB3]], i32 [[OMP_GLOBAL_THREAD_NUM]])
-; CHECK2-NEXT:    call void (ptr, ptr, ...) @.omp_outlined..23(ptr [[TID_ADDR]], ptr [[ZERO_ADDR]], ptr nocapture nofree noundef nonnull readonly align 4 dereferenceable(4) [[LOADGEP_A_ADDR]])
+; CHECK2-NEXT:    call void (ptr, ptr, ...) @.omp_outlined..23(ptr [[TID_ADDR]], ptr [[ZERO_ADDR]], ptr nofree noundef nonnull readonly align 4 captures(none) dereferenceable(4) [[LOADGEP_A_ADDR]])
 ; CHECK2-NEXT:    [[OMP_GLOBAL_THREAD_NUM1:%.*]] = call i32 @__kmpc_global_thread_num(ptr @[[GLOB2]])
 ; CHECK2-NEXT:    call void @__kmpc_barrier(ptr @[[GLOB3]], i32 [[OMP_GLOBAL_THREAD_NUM1]])
-; CHECK2-NEXT:    call void (ptr, ptr, ...) @.omp_outlined..24(ptr [[TID_ADDR]], ptr [[ZERO_ADDR]], ptr nocapture nofree noundef nonnull readonly align 4 dereferenceable(4) [[LOADGEP_A_ADDR]])
+; CHECK2-NEXT:    call void (ptr, ptr, ...) @.omp_outlined..24(ptr [[TID_ADDR]], ptr [[ZERO_ADDR]], ptr nofree noundef nonnull readonly align 4 captures(none) dereferenceable(4) [[LOADGEP_A_ADDR]])
 ; CHECK2-NEXT:    br label [[ENTRY_SPLIT:%.*]]
 ; CHECK2:       entry.split:
 ; CHECK2-NEXT:    br label [[OMP_PAR_REGION_SPLIT:%.*]]
@@ -5537,7 +5537,7 @@ entry:
 ;
 ;
 ; CHECK2-LABEL: define {{[^@]+}}@.omp_outlined..22
-; CHECK2-SAME: (ptr noalias nocapture nofree readnone [[DOTGLOBAL_TID_:%.*]], ptr noalias nocapture nofree readnone [[DOTBOUND_TID_:%.*]], ptr nocapture nofree noundef nonnull readonly align 4 dereferenceable(4) [[A:%.*]]) {
+; CHECK2-SAME: (ptr noalias nofree readnone captures(none) [[DOTGLOBAL_TID_:%.*]], ptr noalias nofree readnone captures(none) [[DOTBOUND_TID_:%.*]], ptr nofree noundef nonnull readonly align 4 captures(none) dereferenceable(4) [[A:%.*]]) {
 ; CHECK2-NEXT:  entry:
 ; CHECK2-NEXT:    [[TMP0:%.*]] = load i32, ptr [[A]], align 4
 ; CHECK2-NEXT:    call void @use(i32 [[TMP0]])
@@ -5545,7 +5545,7 @@ entry:
 ;
 ;
 ; CHECK2-LABEL: define {{[^@]+}}@.omp_outlined..23
-; CHECK2-SAME: (ptr noalias nocapture nofree readnone [[DOTGLOBAL_TID_:%.*]], ptr noalias nocapture nofree readnone [[DOTBOUND_TID_:%.*]], ptr nocapture nofree noundef nonnull readonly align 4 dereferenceable(4) [[A:%.*]]) {
+; CHECK2-SAME: (ptr noalias nofree readnone captures(none) [[DOTGLOBAL_TID_:%.*]], ptr noalias nofree readnone captures(none) [[DOTBOUND_TID_:%.*]], ptr nofree noundef nonnull readonly align 4 captures(none) dereferenceable(4) [[A:%.*]]) {
 ; CHECK2-NEXT:  entry:
 ; CHECK2-NEXT:    [[TMP0:%.*]] = load i32, ptr [[A]], align 4
 ; CHECK2-NEXT:    call void @use(i32 [[TMP0]])
@@ -5553,7 +5553,7 @@ entry:
 ;
 ;
 ; CHECK2-LABEL: define {{[^@]+}}@.omp_outlined..24
-; CHECK2-SAME: (ptr noalias nocapture nofree readnone [[DOTGLOBAL_TID_:%.*]], ptr noalias nocapture nofree readnone [[DOTBOUND_TID_:%.*]], ptr nocapture nofree noundef nonnull readonly align 4 dereferenceable(4) [[A:%.*]]) {
+; CHECK2-SAME: (ptr noalias nofree readnone captures(none) [[DOTGLOBAL_TID_:%.*]], ptr noalias nofree readnone captures(none) [[DOTBOUND_TID_:%.*]], ptr nofree noundef nonnull readonly align 4 captures(none) dereferenceable(4) [[A:%.*]]) {
 ; CHECK2-NEXT:  entry:
 ; CHECK2-NEXT:    [[TMP0:%.*]] = load i32, ptr [[A]], align 4
 ; CHECK2-NEXT:    call void @use(i32 [[TMP0]])
@@ -5613,7 +5613,7 @@ entry:
 ; CHECK2:       omp.par.region:
 ; CHECK2-NEXT:    br label [[OMP_PAR_MERGED:%.*]]
 ; CHECK2:       omp.par.merged:
-; CHECK2-NEXT:    call void (ptr, ptr, ...) @.omp_outlined..25(ptr [[TID_ADDR]], ptr [[ZERO_ADDR]], ptr nocapture nofree noundef nonnull readonly align 4 dereferenceable(4) [[LOADGEP_A_ADDR]])
+; CHECK2-NEXT:    call void (ptr, ptr, ...) @.omp_outlined..25(ptr [[TID_ADDR]], ptr [[ZERO_ADDR]], ptr nofree noundef nonnull readonly align 4 captures(none) dereferenceable(4) [[LOADGEP_A_ADDR]])
 ; CHECK2-NEXT:    [[OMP_GLOBAL_THREAD_NUM2:%.*]] = call i32 @__kmpc_global_thread_num(ptr @[[GLOB2]])
 ; CHECK2-NEXT:    call void @__kmpc_barrier(ptr @[[GLOB3]], i32 [[OMP_GLOBAL_THREAD_NUM2]])
 ; CHECK2-NEXT:    [[OMP_GLOBAL_THREAD_NUM:%.*]] = call i32 @__kmpc_global_thread_num(ptr @[[GLOB2]])
@@ -5625,7 +5625,7 @@ entry:
 ; CHECK2-NEXT:    call void @__kmpc_barrier(ptr @[[GLOB3]], i32 [[OMP_GLOBAL_THREAD_NUM1]])
 ; CHECK2-NEXT:    br label [[OMP_PAR_MERGED_SPLIT_SPLIT:%.*]]
 ; CHECK2:       omp.par.merged.split.split:
-; CHECK2-NEXT:    call void (ptr, ptr, ...) @.omp_outlined..26(ptr [[TID_ADDR]], ptr [[ZERO_ADDR]], ptr nocapture nofree noundef nonnull readonly align 4 dereferenceable(4) [[LOADGEP_A_ADDR]])
+; CHECK2-NEXT:    call void (ptr, ptr, ...) @.omp_outlined..26(ptr [[TID_ADDR]], ptr [[ZERO_ADDR]], ptr nofree noundef nonnull readonly align 4 captures(none) dereferenceable(4) [[LOADGEP_A_ADDR]])
 ; CHECK2-NEXT:    [[OMP_GLOBAL_THREAD_NUM4:%.*]] = call i32 @__kmpc_global_thread_num(ptr @[[GLOB2]])
 ; CHECK2-NEXT:    call void @__kmpc_barrier(ptr @[[GLOB3]], i32 [[OMP_GLOBAL_THREAD_NUM4]])
 ; CHECK2-NEXT:    [[OMP_GLOBAL_THREAD_NUM3:%.*]] = call i32 @__kmpc_global_thread_num(ptr @[[GLOB2]])
@@ -5637,7 +5637,7 @@ entry:
 ; CHECK2-NEXT:    call void @__kmpc_barrier(ptr @[[GLOB3]], i32 [[OMP_GLOBAL_THREAD_NUM6]])
 ; CHECK2-NEXT:    br label [[OMP_PAR_MERGED_SPLIT_SPLIT_SPLIT_SPLIT:%.*]]
 ; CHECK2:       omp.par.merged.split.split.split.split:
-; CHECK2-NEXT:    call void (ptr, ptr, ...) @.omp_outlined..27(ptr [[TID_ADDR]], ptr [[ZERO_ADDR]], ptr nocapture nofree noundef nonnull readonly align 4 dereferenceable(4) [[LOADGEP_A_ADDR]])
+; CHECK2-NEXT:    call void (ptr, ptr, ...) @.omp_outlined..27(ptr [[TID_ADDR]], ptr [[ZERO_ADDR]], ptr nofree noundef nonnull readonly align 4 captures(none) dereferenceable(4) [[LOADGEP_A_ADDR]])
 ; CHECK2-NEXT:    br label [[ENTRY_SPLIT:%.*]]
 ; CHECK2:       entry.split:
 ; CHECK2-NEXT:    br label [[OMP_PAR_REGION_SPLIT:%.*]]
@@ -5673,7 +5673,7 @@ entry:
 ;
 ;
 ; CHECK2-LABEL: define {{[^@]+}}@.omp_outlined..25
-; CHECK2-SAME: (ptr noalias nocapture nofree readnone [[DOTGLOBAL_TID_:%.*]], ptr noalias nocapture nofree readnone [[DOTBOUND_TID_:%.*]], ptr nocapture nofree noundef nonnull readonly align 4 dereferenceable(4) [[A:%.*]]) {
+; CHECK2-SAME: (ptr noalias nofree readnone captures(none) [[DOTGLOBAL_TID_:%.*]], ptr noalias nofree readnone captures(none) [[DOTBOUND_TID_:%.*]], ptr nofree noundef nonnull readonly align 4 captures(none) dereferenceable(4) [[A:%.*]]) {
 ; CHECK2-NEXT:  entry:
 ; CHECK2-NEXT:    [[TMP0:%.*]] = load i32, ptr [[A]], align 4
 ; CHECK2-NEXT:    call void @use(i32 [[TMP0]])
@@ -5681,7 +5681,7 @@ entry:
 ;
 ;
 ; CHECK2-LABEL: define {{[^@]+}}@.omp_outlined..26
-; CHECK2-SAME: (ptr noalias nocapture nofree readnone [[DOTGLOBAL_TID_:%.*]], ptr noalias nocapture nofree readnone [[DOTBOUND_TID_:%.*]], ptr nocapture nofree noundef nonnull readonly align 4 dereferenceable(4) [[A:%.*]]) {
+; CHECK2-SAME: (ptr noalias nofree readnone captures(none) [[DOTGLOBAL_TID_:%.*]], ptr noalias nofree readnone captures(none) [[DOTBOUND_TID_:%.*]], ptr nofree noundef nonnull readonly align 4 captures(none) dereferenceable(4) [[A:%.*]]) {
 ; CHECK2-NEXT:  entry:
 ; CHECK2-NEXT:    [[TMP0:%.*]] = load i32, ptr [[A]], align 4
 ; CHECK2-NEXT:    call void @use(i32 [[TMP0]])
@@ -5689,7 +5689,7 @@ entry:
 ;
 ;
 ; CHECK2-LABEL: define {{[^@]+}}@.omp_outlined..27
-; CHECK2-SAME: (ptr noalias nocapture nofree readnone [[DOTGLOBAL_TID_:%.*]], ptr noalias nocapture nofree readnone [[DOTBOUND_TID_:%.*]], ptr nocapture nofree noundef nonnull readonly align 4 dereferenceable(4) [[A:%.*]]) {
+; CHECK2-SAME: (ptr noalias nofree readnone captures(none) [[DOTGLOBAL_TID_:%.*]], ptr noalias nofree readnone captures(none) [[DOTBOUND_TID_:%.*]], ptr nofree noundef nonnull readonly align 4 captures(none) dereferenceable(4) [[A:%.*]]) {
 ; CHECK2-NEXT:  entry:
 ; CHECK2-NEXT:    [[TMP0:%.*]] = load i32, ptr [[A]], align 4
 ; CHECK2-NEXT:    call void @use(i32 [[TMP0]])
@@ -5701,16 +5701,16 @@ entry:
 ; CHECK2-NEXT:  entry:
 ; CHECK2-NEXT:    [[A_ADDR:%.*]] = alloca i32, align 4
 ; CHECK2-NEXT:    store i32 [[A]], ptr [[A_ADDR]], align 4
-; CHECK2-NEXT:    call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr noundef nonnull align 8 dereferenceable(24) @[[GLOB1]], i32 noundef 1, ptr noundef nonnull @.omp_outlined..28, ptr nocapture nofree noundef nonnull readonly align 4 dereferenceable(4) [[A_ADDR]])
+; CHECK2-NEXT:    call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr noundef nonnull align 8 dereferenceable(24) @[[GLOB1]], i32 noundef 1, ptr noundef nonnull @.omp_outlined..28, ptr nofree noundef nonnull readonly align 4 captures(none) dereferenceable(4) [[A_ADDR]])
 ; CHECK2-NEXT:    call void (...) @foo()
-; CHECK2-NEXT:    call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr noundef nonnull align 8 dereferenceable(24) @[[GLOB1]], i32 noundef 1, ptr noundef nonnull @.omp_outlined..29, ptr nocapture nofree noundef nonnull readonly align 4 dereferenceable(4) [[A_ADDR]])
+; CHECK2-NEXT:    call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr noundef nonnull align 8 dereferenceable(24) @[[GLOB1]], i32 noundef 1, ptr noundef nonnull @.omp_outlined..29, ptr nofree noundef nonnull readonly align 4 captures(none) dereferenceable(4) [[A_ADDR]])
 ; CHECK2-NEXT:    call void (...) @foo()
-; CHECK2-NEXT:    call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr noundef nonnull align 8 dereferenceable(24) @[[GLOB1]], i32 noundef 1, ptr noundef nonnull @.omp_outlined..30, ptr nocapture nofree noundef nonnull readonly align 4 dereferenceable(4) [[A_ADDR]])
+; CHECK2-NEXT:    call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr noundef nonnull align 8 dereferenceable(24) @[[GLOB1]], i32 noundef 1, ptr noundef nonnull @.omp_outlined..30, ptr nofree noundef nonnull readonly align 4 captures(none) dereferenceable(4) [[A_ADDR]])
 ; CHECK2-NEXT:    ret void
 ;
 ;
 ; CHECK2-LABEL: define {{[^@]+}}@.omp_outlined..28
-; CHECK2-SAME: (ptr noalias nocapture nofree readnone [[DOTGLOBAL_TID_:%.*]], ptr noalias nocapture nofree readnone [[DOTBOUND_TID_:%.*]], ptr nocapture nofree noundef nonnull readonly align 4 dereferenceable(4) [[A:%.*]]) {
+; CHECK2-SAME: (ptr noalias nofree readnone captures(none) [[DOTGLOBAL_TID_:%.*]], ptr noalias nofree readnone captures(none) [[DOTBOUND_TID_:%.*]], ptr nofree noundef nonnull readonly align 4 captures(none) dereferenceable(4) [[A:%.*]]) {
 ; CHECK2-NEXT:  entry:
 ; CHECK2-NEXT:    [[TMP0:%.*]] = load i32, ptr [[A]], align 4
 ; CHECK2-NEXT:    call void @use(i32 [[TMP0]])
@@ -5718,7 +5718,7 @@ entry:
 ;
 ;
 ; CHECK2-LABEL: define {{[^@]+}}@.omp_outlined..29
-; CHECK2-SAME: (ptr noalias nocapture nofree readnone [[DOTGLOBAL_TID_:%.*]], ptr noalias nocapture nofree readnone [[DOTBOUND_TID_:%.*]], ptr nocapture nofree noundef nonnull readonly align 4 dereferenceable(4) [[A:%.*]]) {
+; CHECK2-SAME: (ptr noalias nofree readnone captures(none) [[DOTGLOBAL_TID_:%.*]], ptr noalias nofree readnone captures(none) [[DOTBOUND_TID_:%.*]], ptr nofree noundef nonnull readonly align 4 captures(none) dereferenceable(4) [[A:%.*]]) {
 ; CHECK2-NEXT:  entry:
 ; CHECK2-NEXT:    [[TMP0:%.*]] = load i32, ptr [[A]], align 4
 ; CHECK2-NEXT:    call void @use(i32 [[TMP0]])
@@ -5726,7 +5726,7 @@ entry:
 ;
 ;
 ; CHECK2-LABEL: define {{[^@]+}}@.omp_outlined..30
-; CHECK2-SAME: (ptr noalias nocapture nofree readnone [[DOTGLOBAL_TID_:%.*]], ptr noalias nocapture nofree readnone [[DOTBOUND_TID_:%.*]], ptr nocapture nofree noundef nonnull readonly align 4 dereferenceable(4) [[A:%.*]]) {
+; CHECK2-SAME: (ptr noalias nofree readnone captures(none) [[DOTGLOBAL_TID_:%.*]], ptr noalias nofree readnone captures(none) [[DOTBOUND_TID_:%.*]], ptr nofree noundef nonnull readonly align 4 captures(none) dereferenceable(4) [[A:%.*]]) {
 ; CHECK2-NEXT:  entry:
 ; CHECK2-NEXT:    [[TMP0:%.*]] = load i32, ptr [[A]], align 4
 ; CHECK2-NEXT:    call void @use(i32 [[TMP0]])
@@ -5739,15 +5739,15 @@ entry:
 ; CHECK2-NEXT:    [[A_ADDR:%.*]] = alloca i32, align 4
 ; CHECK2-NEXT:    [[TMP0:%.*]] = call i32 @__kmpc_global_thread_num(ptr noundef nonnull align 8 dereferenceable(24) @[[GLOB1]])
 ; CHECK2-NEXT:    store i32 [[A]], ptr [[A_ADDR]], align 4
-; CHECK2-NEXT:    call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr noundef nonnull align 8 dereferenceable(24) @[[GLOB1]], i32 noundef 1, ptr noundef nonnull @.omp_outlined..31, ptr nocapture nofree noundef nonnull readonly align 4 dereferenceable(4) [[A_ADDR]])
+; CHECK2-NEXT:    call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr noundef nonnull align 8 dereferenceable(24) @[[GLOB1]], i32 noundef 1, ptr noundef nonnull @.omp_outlined..31, ptr nofree noundef nonnull readonly align 4 captures(none) dereferenceable(4) [[A_ADDR]])
 ; CHECK2-NEXT:    call void @__kmpc_push_proc_bind(ptr noundef nonnull align 8 dereferenceable(24) @[[GLOB1]], i32 [[TMP0]], i32 noundef 3)
-; CHECK2-NEXT:    call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr noundef nonnull align 8 dereferenceable(24) @[[GLOB1]], i32 noundef 1, ptr noundef nonnull @.omp_outlined..32, ptr nocapture nofree noundef nonnull readonly align 4 dereferenceable(4) [[A_ADDR]])
-; CHECK2-NEXT:    call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr noundef nonnull align 8 dereferenceable(24) @[[GLOB1]], i32 noundef 1, ptr noundef nonnull @.omp_outlined..33, ptr nocapture nofree noundef nonnull readonly align 4 dereferenceable(4) [[A_ADDR]])
+; CHECK2-NEXT:    call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr noundef nonnull align 8 dereferenceable(24) @[[GLOB1]], i32 noundef 1, ptr noundef nonnull @.omp_outlined..32, ptr nofree noundef nonnull readonly align 4 captures(none) dereferenceable(4) [[A_ADDR]])
+; CHECK2-NEXT:    call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr noundef nonnull align 8 dereferenceable(24) @[[GLOB1]], i32 noundef 1, ptr noundef nonnull @.omp_outlined..33, ptr nofree noundef nonnull readonly align 4 captures(none) dereferenceable(4) [[A_ADDR]])
 ; CHECK2-NEXT:    ret void
 ;
 ;
 ; CHECK2-LABEL: define {{[^@]+}}@.omp_outlined..31
-; CHECK2-SAME: (ptr noalias nocapture nofree readnone [[DOTGLOBAL_TID_:%.*]], ptr noalias nocapture nofree readnone [[DOTBOUND_TID_:%.*]], ptr nocapture nofree noundef nonnull readonly align 4 dereferenceable(4) [[A:%.*]]) {
+; CHECK2-SAME: (ptr noalias nofree readnone captures(none) [[DOTGLOBAL_TID_:%.*]], ptr noalias nofree readnone captures(none) [[DOTBOUND_TID_:%.*]], ptr nofree noundef nonnull readonly align 4 captures(none) dereferenceable(4) [[A:%.*]]) {
 ; CHECK2-NEXT:  entry:
 ; CHECK2-NEXT:    [[TMP0:%.*]] = load i32, ptr [[A]], align 4
 ; CHECK2-NEXT:    call void @use(i32 [[TMP0]])
@@ -5755,7 +5755,7 @@ entry:
 ;
 ;
 ; CHECK2-LABEL: define {{[^@]+}}@.omp_outlined..32
-; CHECK2-SAME: (ptr noalias nocapture nofree readnone [[DOTGLOBAL_TID_:%.*]], ptr noalias nocapture nofree readnone [[DOTBOUND_TID_:%.*]], ptr nocapture nofree noundef nonnull readonly align 4 dereferenceable(4) [[A:%.*]]) {
+; CHECK2-SAME: (ptr noalias nofree readnone captures(none) [[DOTGLOBAL_TID_:%.*]], ptr noalias nofree readnone captures(none) [[DOTBOUND_TID_:%.*]], ptr nofree noundef nonnull readonly align 4 captures(none) dereferenceable(4) [[A:%.*]]) {
 ; CHECK2-NEXT:  entry:
 ; CHECK2-NEXT:    [[TMP0:%.*]] = load i32, ptr [[A]], align 4
 ; CHECK2-NEXT:    call void @use(i32 [[TMP0]])
@@ -5763,7 +5763,7 @@ entry:
 ;
 ;
 ; CHECK2-LABEL: define {{[^@]+}}@.omp_outlined..33
-; CHECK2-SAME: (ptr noalias nocapture nofree readnone [[DOTGLOBAL_TID_:%.*]], ptr noalias nocapture nofree readnone [[DOTBOUND_TID_:%.*]], ptr nocapture nofree noundef nonnull readonly align 4 dereferenceable(4) [[A:%.*]]) {
+; CHECK2-SAME: (ptr noalias nofree readnone captures(none) [[DOTGLOBAL_TID_:%.*]], ptr noalias nofree readnone captures(none) [[DOTBOUND_TID_:%.*]], ptr nofree noundef nonnull readonly align 4 captures(none) dereferenceable(4) [[A:%.*]]) {
 ; CHECK2-NEXT:  entry:
 ; CHECK2-NEXT:    [[TMP0:%.*]] = load i32, ptr [[A]], align 4
 ; CHECK2-NEXT:    call void @use(i32 [[TMP0]])
@@ -5776,15 +5776,15 @@ entry:
 ; CHECK2-NEXT:    [[A_ADDR:%.*]] = alloca i32, align 4
 ; CHECK2-NEXT:    [[TMP0:%.*]] = call i32 @__kmpc_global_thread_num(ptr noundef nonnull align 8 dereferenceable(24) @[[GLOB1]])
 ; CHECK2-NEXT:    store i32 [[A]], ptr [[A_ADDR]], align 4
-; CHECK2-NEXT:    call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr noundef nonnull align 8 dereferenceable(24) @[[GLOB1]], i32 noundef 1, ptr noundef nonnull @.omp_outlined..34, ptr nocapture nofree noundef nonnull readonly align 4 dereferenceable(4) [[A_ADDR]])
+; CHECK2-NEXT:    call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr noundef nonnull align 8 dereferenceable(24) @[[GLOB1]], i32 noundef 1, ptr noundef nonnull @.omp_outlined..34, ptr nofree noundef nonnull readonly align 4 captures(none) dereferenceable(4) [[A_ADDR]])
 ; CHECK2-NEXT:    call void @__kmpc_push_num_threads(ptr noundef nonnull align 8 dereferenceable(24) @[[GLOB1]], i32 [[TMP0]], i32 [[A]])
-; CHECK2-NEXT:    call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr noundef nonnull align 8 dereferenceable(24) @[[GLOB1]], i32 noundef 1, ptr noundef nonnull @.omp_outlined..35, ptr nocapture nofree noundef nonnull readonly align 4 dereferenceable(4) [[A_ADDR]])
-; CHECK2-NEXT:    call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr noundef nonnull align 8 dereferenceable(24) @[[GLOB1]], i32 noundef 1, ptr noundef nonnull @.omp_outlined..36, ptr nocapture nofree noundef nonnull readonly align 4 dereferenceable(4) [[A_ADDR]])
+; CHECK2-NEXT:    call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr noundef nonnull align 8 dereferenceable(24) @[[GLOB1]], i32 noundef 1, ptr noundef nonnull @.omp_outlined..35, ptr nofree noundef nonnull readonly align 4 captures(none) dereferenceable(4) [[A_ADDR]])
+; CHECK2-NEXT:    call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr noundef nonnull align 8 dereferenceable(24) @[[GLOB1]], i32 noundef 1, ptr noundef nonnull @.omp_outlined..36, ptr nofree noundef nonnull readonly align 4 captures(none) dereferenceable(4) [[A_ADDR]])
 ; CHECK2-NEXT:    ret void
 ;
 ;
 ; CHECK2-LABEL: define {{[^@]+}}@.omp_outlined..34
-; CHECK2-SAME: (ptr noalias nocapture nofree readnone [[DOTGLOBAL_TID_:%.*]], ptr noalias nocapture nofree readnone [[DOTBOUND_TID_:%.*]], ptr nocapture nofree noundef nonnull readonly align 4 dereferenceable(4) [[A:%.*]]) {
+; CHECK2-SAME: (ptr noalias nofree readnone captures(none) [[DOTGLOBAL_TID_:%.*]], ptr noalias nofree readnone captures(none) [[DOTBOUND_TID_:%.*]], ptr nofree noundef nonnull readonly align 4 captures(none) dereferenceable(4) [[A:%.*]]) {
 ; CHECK2-NEXT:  entry:
 ; CHECK2-NEXT:    [[TMP0:%.*]] = load i32, ptr [[A]], align 4
 ; CHECK2-NEXT:    call void @use(i32 [[TMP0]])
@@ -5792,7 +5792,7 @@ entry:
 ;
 ;
 ; CHECK2-LABEL: define {{[^@]+}}@.omp_outlined..35
-; CHECK2-SAME: (ptr noalias nocapture nofree readnone [[DOTGLOBAL_TID_:%.*]], ptr noalias nocapture nofree readnone [[DOTBOUND_TID_:%.*]], ptr nocapture nofree noundef nonnull readonly align 4 dereferenceable(4) [[A:%.*]]) {
+; CHECK2-SAME: (ptr noalias nofree readnone captures(none) [[DOTGLOBAL_TID_:%.*]], ptr noalias nofree readnone captures(none) [[DOTBOUND_TID_:%.*]], ptr nofree noundef nonnull readonly align 4 captures(none) dereferenceable(4) [[A:%.*]]) {
 ; CHECK2-NEXT:  entry:
 ; CHECK2-NEXT:    [[TMP0:%.*]] = load i32, ptr [[A]], align 4
 ; CHECK2-NEXT:    call void @use(i32 [[TMP0]])
@@ -5800,7 +5800,7 @@ entry:
 ;
 ;
 ; CHECK2-LABEL: define {{[^@]+}}@.omp_outlined..36
-; CHECK2-SAME: (ptr noalias nocapture nofree readnone [[DOTGLOBAL_TID_:%.*]], ptr noalias nocapture nofree readnone [[DOTBOUND_TID_:%.*]], ptr nocapture nofree noundef nonnull readonly align 4 dereferenceable(4) [[A:%.*]]) {
+; CHECK2-SAME: (ptr noalias nofree readnone captures(none) [[DOTGLOBAL_TID_:%.*]], ptr noalias nofree readnone captures(none) [[DOTBOUND_TID_:%.*]], ptr nofree noundef nonnull readonly align 4 captures(none) dereferenceable(4) [[A:%.*]]) {
 ; CHECK2-NEXT:  entry:
 ; CHECK2-NEXT:    [[TMP0:%.*]] = load i32, ptr [[A]], align 4
 ; CHECK2-NEXT:    call void @use(i32 [[TMP0]])
@@ -5826,7 +5826,7 @@ entry:
 ; CHECK2-NEXT:    br label [[ENTRY_SPLIT_SPLIT:%.*]]
 ; CHECK2:       entry.split.split:
 ; CHECK2-NEXT:    call void (...) @foo()
-; CHECK2-NEXT:    call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr noundef nonnull align 8 dereferenceable(24) @[[GLOB1]], i32 noundef 1, ptr noundef nonnull @.omp_outlined..39, ptr nocapture nofree noundef nonnull readonly align 4 dereferenceable(4) [[A_ADDR]])
+; CHECK2-NEXT:    call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr noundef nonnull align 8 dereferenceable(24) @[[GLOB1]], i32 noundef 1, ptr noundef nonnull @.omp_outlined..39, ptr nofree noundef nonnull readonly align 4 captures(none) dereferenceable(4) [[A_ADDR]])
 ; CHECK2-NEXT:    ret void
 ;
 ;
@@ -5843,10 +5843,10 @@ entry:
 ; CHECK2:       omp.par.region:
 ; CHECK2-NEXT:    br label [[OMP_PAR_MERGED:%.*]]
 ; CHECK2:       omp.par.merged:
-; CHECK2-NEXT:    call void (ptr, ptr, ...) @.omp_outlined..37(ptr [[TID_ADDR]], ptr [[ZERO_ADDR]], ptr nocapture nofree noundef nonnull readonly align 4 dereferenceable(4) [[LOADGEP_A_ADDR]])
+; CHECK2-NEXT:    call void (ptr, ptr, ...) @.omp_outlined..37(ptr [[TID_ADDR]], ptr [[ZERO_ADDR]], ptr nofree noundef nonnull readonly align 4 captures(none) dereferenceable(4) [[LOADGEP_A_ADDR]])
 ; CHECK2-NEXT:    [[OMP_GLOBAL_THREAD_NUM:%.*]] = call i32 @__kmpc_global_thread_num(ptr @[[GLOB2]])
 ; CHECK2-NEXT:    call void @__kmpc_barrier(ptr @[[GLOB3]], i32 [[OMP_GLOBAL_THREAD_NUM]])
-; CHECK2-NEXT:    call void (ptr, ptr, ...) @.omp_outlined..38(ptr [[TID_ADDR]], ptr [[ZERO_ADDR]], ptr nocapture nofree noundef nonnull readonly align 4 dereferenceable(4) [[LOADGEP_A_ADDR]])
+; CHECK2-NEXT:    call void (ptr, ptr, ...) @.omp_outlined..38(ptr [[TID_ADDR]], ptr [[ZERO_ADDR]], ptr nofree noundef nonnull readonly align 4 captures(none) dereferenceable(4) [[LOADGEP_A_ADDR]])
 ; CHECK2-NEXT:    br label [[ENTRY_SPLIT:%.*]]
 ; CHECK2:       entry.split:
 ; CHECK2-NEXT:    br label [[OMP_PAR_REGION_SPLIT:%.*]]
@@ -5859,7 +5859,7 @@ entry:
 ;
 ;
 ; CHECK2-LABEL: define {{[^@]+}}@.omp_outlined..37
-; CHECK2-SAME: (ptr noalias nocapture nofree readnone [[DOTGLOBAL_TID_:%.*]], ptr noalias nocapture nofree readnone [[DOTBOUND_TID_:%.*]], ptr nocapture nofree noundef nonnull readonly align 4 dereferenceable(4) [[A:%.*]]) {
+; CHECK2-SAME: (ptr noalias nofree readnone captures(none) [[DOTGLOBAL_TID_:%.*]], ptr noalias nofree readnone captures(none) [[DOTBOUND_TID_:%.*]], ptr nofree noundef nonnull readonly align 4 captures(none) dereferenceable(4) [[A:%.*]]) {
 ; CHECK2-NEXT:  entry:
 ; CHECK2-NEXT:    [[TMP0:%.*]] = load i32, ptr [[A]], align 4
 ; CHECK2-NEXT:    call void @use(i32 [[TMP0]])
@@ -5867,7 +5867,7 @@ entry:
 ;
 ;
 ; CHECK2-LABEL: define {{[^@]+}}@.omp_outlined..38
-; CHECK2-SAME: (ptr noalias nocapture nofree readnone [[DOTGLOBAL_TID_:%.*]], ptr noalias nocapture nofree readnone [[DOTBOUND_TID_:%.*]], ptr nocapture nofree noundef nonnull readonly align 4 dereferenceable(4) [[A:%.*]]) {
+; CHECK2-SAME: (ptr noalias nofree readnone captures(none) [[DOTGLOBAL_TID_:%.*]], ptr noalias nofree readnone captures(none) [[DOTBOUND_TID_:%.*]], ptr nofree noundef nonnull readonly align 4 captures(none) dereferenceable(4) [[A:%.*]]) {
 ; CHECK2-NEXT:  entry:
 ; CHECK2-NEXT:    [[TMP0:%.*]] = load i32, ptr [[A]], align 4
 ; CHECK2-NEXT:    call void @use(i32 [[TMP0]])
@@ -5875,7 +5875,7 @@ entry:
 ;
 ;
 ; CHECK2-LABEL: define {{[^@]+}}@.omp_outlined..39
-; CHECK2-SAME: (ptr noalias nocapture nofree readnone [[DOTGLOBAL_TID_:%.*]], ptr noalias nocapture nofree readnone [[DOTBOUND_TID_:%.*]], ptr nocapture nofree noundef nonnull readonly align 4 dereferenceable(4) [[A:%.*]]) {
+; CHECK2-SAME: (ptr noalias nofree readnone captures(none) [[DOTGLOBAL_TID_:%.*]], ptr noalias nofree readnone captures(none) [[DOTBOUND_TID_:%.*]], ptr nofree noundef nonnull readonly align 4 captures(none) dereferenceable(4) [[A:%.*]]) {
 ; CHECK2-NEXT:  entry:
 ; CHECK2-NEXT:    [[TMP0:%.*]] = load i32, ptr [[A]], align 4
 ; CHECK2-NEXT:    call void @use(i32 [[TMP0]])
diff --git a/llvm/test/Transforms/OpenMP/reduced_pointer_info_assertion.ll b/llvm/test/Transforms/OpenMP/reduced_pointer_info_assertion.ll
index eb1ae5f0bc2c1..134a538f02b4b 100644
--- a/llvm/test/Transforms/OpenMP/reduced_pointer_info_assertion.ll
+++ b/llvm/test/Transforms/OpenMP/reduced_pointer_info_assertion.ll
@@ -38,9 +38,9 @@ define fastcc void @rec(ptr %0, i64 %1) {
 ;
 ;
 ; CGSCC-LABEL: define {{[^@]+}}@rec
-; CGSCC-SAME: (ptr nocapture writeonly [[TMP0:%.*]], i64 [[TMP1:%.*]]) #[[ATTR0:[0-9]+]] {
+; CGSCC-SAME: (ptr writeonly captures(none) [[TMP0:%.*]], i64 [[TMP1:%.*]]) #[[ATTR0:[0-9]+]] {
 ; CGSCC-NEXT:    [[TMP3:%.*]] = getelementptr i32, ptr [[TMP0]], i64 [[TMP1]]
 ; CGSCC-NEXT:    store i32 0, ptr [[TMP3]], align 4
-; CGSCC-NEXT:    call fastcc void @rec(ptr nocapture writeonly [[TMP0]], i64 0) #[[ATTR1:[0-9]+]]
+; CGSCC-NEXT:    call fastcc void @rec(ptr writeonly captures(none) [[TMP0]], i64 0) #[[ATTR1:[0-9]+]]
 ; CGSCC-NEXT:    ret void
 ;
diff --git a/llvm/test/Transforms/OpenMP/remove_globalization.ll b/llvm/test/Transforms/OpenMP/remove_globalization.ll
index 31e3ef2b9079f..29f2030c4d42b 100644
--- a/llvm/test/Transforms/OpenMP/remove_globalization.ll
+++ b/llvm/test/Transforms/OpenMP/remove_globalization.ll
@@ -40,7 +40,7 @@ define weak i32 @__kmpc_target_init(ptr %0, ptr) {
 }
 declare void @__kmpc_target_deinit()
 
-define void @kernel(ptr %dyn) "kernel" {
+define ptx_kernel void @kernel(ptr %dyn) "kernel" {
 ; CHECK-LABEL: define {{[^@]+}}@kernel
 ; CHECK-SAME: (ptr [[DYN:%.*]]) #[[ATTR0:[0-9]+]] {
 ; CHECK-NEXT:  entry:
@@ -98,14 +98,14 @@ define internal void @bar() {
 ; CHECK-SAME: () #[[ATTR1]] {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[DOTH2S:%.*]] = alloca i8, i64 4, align 4
-; CHECK-NEXT:    call void @share(ptr nofree [[DOTH2S]]) #[[ATTR5:[0-9]+]], !dbg [[DBG8:![0-9]+]]
+; CHECK-NEXT:    call void @share(ptr nofree [[DOTH2S]]) #[[ATTR5:[0-9]+]], !dbg [[DBG7:![0-9]+]]
 ; CHECK-NEXT:    ret void
 ;
 ; CHECK-DISABLED-LABEL: define {{[^@]+}}@bar
 ; CHECK-DISABLED-SAME: () #[[ATTR1]] {
 ; CHECK-DISABLED-NEXT:  entry:
 ; CHECK-DISABLED-NEXT:    [[DOTH2S:%.*]] = alloca i8, i64 4, align 4
-; CHECK-DISABLED-NEXT:    call void @share(ptr nofree [[DOTH2S]]) #[[ATTR5:[0-9]+]], !dbg [[DBG8:![0-9]+]]
+; CHECK-DISABLED-NEXT:    call void @share(ptr nofree [[DOTH2S]]) #[[ATTR5:[0-9]+]], !dbg [[DBG7:![0-9]+]]
 ; CHECK-DISABLED-NEXT:    ret void
 ;
 entry:
@@ -146,7 +146,7 @@ define void @unused() {
 ;
 ; CHECK-DISABLED-LABEL: define {{[^@]+}}@unused() {
 ; CHECK-DISABLED-NEXT:  entry:
-; CHECK-DISABLED-NEXT:    [[TMP0:%.*]] = call align 4 ptr @__kmpc_alloc_shared(i64 4) #[[ATTR6:[0-9]+]], !dbg [[DBG11:![0-9]+]]
+; CHECK-DISABLED-NEXT:    [[TMP0:%.*]] = call align 4 ptr @__kmpc_alloc_shared(i64 4) #[[ATTR6:[0-9]+]], !dbg [[DBG10:![0-9]+]]
 ; CHECK-DISABLED-NEXT:    call void @__kmpc_free_shared(ptr [[TMP0]], i64 4) #[[ATTR6]]
 ; CHECK-DISABLED-NEXT:    ret void
 ;
@@ -234,14 +234,12 @@ declare void @unknown_no_openmp() "llvm.assume"="omp_no_openmp"
 
 !llvm.dbg.cu = !{!0}
 !llvm.module.flags = !{!3, !4, !6, !7}
-!nvvm.annotations = !{!5}
 
 !0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 13.0.0", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, enums: !2, splitDebugInlining: false, nameTableKind: None)
 !1 = !DIFile(filename: "remove_globalization.c", directory: "/tmp/remove_globalization.c")
 !2 = !{}
 !3 = !{i32 2, !"Debug Info Version", i32 3}
 !4 = !{i32 1, !"wchar_size", i32 4}
-!5 = !{ptr @kernel, !"kernel", i32 1}
 !6 = !{i32 7, !"openmp", i32 50}
 !7 = !{i32 7, !"openmp-device", i32 50}
 !8 = distinct !DISubprogram(name: "foo", scope: !1, file: !1, line: 1, type: !11, scopeLine: 1, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, retainedNodes: !2)
@@ -276,10 +274,9 @@ declare void @unknown_no_openmp() "llvm.assume"="omp_no_openmp"
 ; CHECK: [[META4:![0-9]+]] = !{i32 1, !"wchar_size", i32 4}
 ; CHECK: [[META5:![0-9]+]] = !{i32 7, !"openmp", i32 50}
 ; CHECK: [[META6:![0-9]+]] = !{i32 7, !"openmp-device", i32 50}
-; CHECK: [[META7:![0-9]+]] = !{ptr @kernel, !"kernel", i32 1}
-; CHECK: [[DBG8]] = !DILocation(line: 4, column: 2, scope: [[META9:![0-9]+]])
-; CHECK: [[META9]] = distinct !DISubprogram(name: "bar", scope: [[META1]], file: [[META1]], line: 1, type: [[META10:![0-9]+]], scopeLine: 1, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: [[META0]], retainedNodes: [[META2]])
-; CHECK: [[META10]] = !DISubroutineType(types: [[META2]])
+; CHECK: [[DBG7]] = !DILocation(line: 4, column: 2, scope: [[META8:![0-9]+]])
+; CHECK: [[META8]] = distinct !DISubprogram(name: "bar", scope: [[META1]], file: [[META1]], line: 1, type: [[META9:![0-9]+]], scopeLine: 1, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: [[META0]], retainedNodes: [[META2]])
+; CHECK: [[META9]] = !DISubroutineType(types: [[META2]])
 ;.
 ; CHECK-DISABLED: [[META0:![0-9]+]] = distinct !DICompileUnit(language: DW_LANG_C99, file: [[META1:![0-9]+]], producer: "{{.*}}clang version {{.*}}", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, enums: [[META2:![0-9]+]], splitDebugInlining: false, nameTableKind: None)
 ; CHECK-DISABLED: [[META1]] = !DIFile(filename: "remove_globalization.c", directory: {{.*}})
@@ -288,11 +285,10 @@ declare void @unknown_no_openmp() "llvm.assume"="omp_no_openmp"
 ; CHECK-DISABLED: [[META4:![0-9]+]] = !{i32 1, !"wchar_size", i32 4}
 ; CHECK-DISABLED: [[META5:![0-9]+]] = !{i32 7, !"openmp", i32 50}
 ; CHECK-DISABLED: [[META6:![0-9]+]] = !{i32 7, !"openmp-device", i32 50}
-; CHECK-DISABLED: [[META7:![0-9]+]] = !{ptr @kernel, !"kernel", i32 1}
-; CHECK-DISABLED: [[DBG8]] = !DILocation(line: 4, column: 2, scope: [[META9:![0-9]+]])
-; CHECK-DISABLED: [[META9]] = distinct !DISubprogram(name: "bar", scope: [[META1]], file: [[META1]], line: 1, type: [[META10:![0-9]+]], scopeLine: 1, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: [[META0]], retainedNodes: [[META2]])
-; CHECK-DISABLED: [[META10]] = !DISubroutineType(types: [[META2]])
-; CHECK-DISABLED: [[DBG11]] = !DILocation(line: 6, column: 2, scope: [[META9]])
+; CHECK-DISABLED: [[DBG7]] = !DILocation(line: 4, column: 2, scope: [[META8:![0-9]+]])
+; CHECK-DISABLED: [[META8]] = distinct !DISubprogram(name: "bar", scope: [[META1]], file: [[META1]], line: 1, type: [[META9:![0-9]+]], scopeLine: 1, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: [[META0]], retainedNodes: [[META2]])
+; CHECK-DISABLED: [[META9]] = !DISubroutineType(types: [[META2]])
+; CHECK-DISABLED: [[DBG10]] = !DILocation(line: 6, column: 2, scope: [[META8]])
 ;.
 ;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
 ; CHECK-REMARKS: {{.*}}
diff --git a/llvm/test/Transforms/OpenMP/replace_globalization.ll b/llvm/test/Transforms/OpenMP/replace_globalization.ll
index 6e4fb9e57388b..92cfd75049226 100644
--- a/llvm/test/Transforms/OpenMP/replace_globalization.ll
+++ b/llvm/test/Transforms/OpenMP/replace_globalization.ll
@@ -25,7 +25,7 @@ target triple = "nvptx64"
 @baz_kernel_environment = local_unnamed_addr constant %struct.KernelEnvironmentTy { %struct.ConfigurationEnvironmentTy { i8 0, i8 0, i8 2, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0 }, ptr @1, ptr null }
 
 
-define dso_local void @foo(ptr %dyn) "kernel" {
+define dso_local ptx_kernel void @foo(ptr %dyn) "kernel" {
 entry:
   %c = call i32 @__kmpc_target_init(ptr @foo_kernel_environment, ptr %dyn)
   %x = call align 4 ptr @__kmpc_alloc_shared(i64 4)
@@ -36,7 +36,7 @@ entry:
   ret void
 }
 
-define void @bar(ptr %dyn) "kernel" {
+define ptx_kernel void @bar(ptr %dyn) "kernel" {
   %c = call i32 @__kmpc_target_init(ptr @bar_kernel_environment, ptr %dyn)
   call void @unknown_no_openmp()
   %cmp = icmp eq i32 %c, -1
@@ -60,7 +60,7 @@ exit:
   ret void
 }
 
-define void @baz_spmd(ptr %dyn) "kernel" {
+define ptx_kernel void @baz_spmd(ptr %dyn) "kernel" {
   %c = call i32 @__kmpc_target_init(ptr @baz_kernel_environment, ptr %dyn)
   call void @unknown_no_openmp()
   %c0 = icmp eq i32 %c, -1
@@ -109,7 +109,6 @@ declare void @unknown_no_openmp() "llvm.assume"="omp_no_openmp"
 
 !llvm.dbg.cu = !{!0}
 !llvm.module.flags = !{!3, !4, !5, !6}
-!nvvm.annotations = !{!7, !8, !13}
 
 !0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 12.0.0", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, enums: !2, splitDebugInlining: false, nameTableKind: None)
 !1 = !DIFile(filename: "replace_globalization.c", directory: "/tmp/replace_globalization.c")
@@ -118,9 +117,6 @@ declare void @unknown_no_openmp() "llvm.assume"="omp_no_openmp"
 !4 = !{i32 1, !"wchar_size", i32 4}
 !5 = !{i32 7, !"openmp", i32 50}
 !6 = !{i32 7, !"openmp-device", i32 50}
-!7 = !{ptr @foo, !"kernel", i32 1}
-!8 = !{ptr @bar, !"kernel", i32 1}
-!13 = !{ptr @baz_spmd, !"kernel", i32 1}
 !9 = distinct !DISubprogram(name: "bar", scope: !1, file: !1, line: 1, type: !10, scopeLine: 1, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, retainedNodes: !2)
 !10 = !DISubroutineType(types: !2)
 !11 = !DILocation(line: 5, column: 7, scope: !9)
@@ -177,7 +173,7 @@ declare void @unknown_no_openmp() "llvm.assume"="omp_no_openmp"
 ; CHECK-NEXT:    [[C0:%.*]] = icmp eq i32 [[C]], -1
 ; CHECK-NEXT:    br i1 [[C0]], label [[MASTER3:%.*]], label [[EXIT:%.*]]
 ; CHECK:       master3:
-; CHECK-NEXT:    [[Z:%.*]] = call align 4 ptr @__kmpc_alloc_shared(i64 24) #[[ATTR6]], !dbg [[DBG10:![0-9]+]]
+; CHECK-NEXT:    [[Z:%.*]] = call align 4 ptr @__kmpc_alloc_shared(i64 24) #[[ATTR6]], !dbg [[DBG7:![0-9]+]]
 ; CHECK-NEXT:    call void @use.internalized(ptr nofree [[Z]]) #[[ATTR7]]
 ; CHECK-NEXT:    call void @__kmpc_free_shared(ptr [[Z]], i64 24) #[[ATTR8]]
 ; CHECK-NEXT:    br label [[EXIT]]
@@ -231,12 +227,9 @@ declare void @unknown_no_openmp() "llvm.assume"="omp_no_openmp"
 ; CHECK: [[META4:![0-9]+]] = !{i32 1, !"wchar_size", i32 4}
 ; CHECK: [[META5:![0-9]+]] = !{i32 7, !"openmp", i32 50}
 ; CHECK: [[META6:![0-9]+]] = !{i32 7, !"openmp-device", i32 50}
-; CHECK: [[META7:![0-9]+]] = !{ptr @foo, !"kernel", i32 1}
-; CHECK: [[META8:![0-9]+]] = !{ptr @bar, !"kernel", i32 1}
-; CHECK: [[META9:![0-9]+]] = !{ptr @baz_spmd, !"kernel", i32 1}
-; CHECK: [[DBG10]] = !DILocation(line: 5, column: 14, scope: [[META11:![0-9]+]])
-; CHECK: [[META11]] = distinct !DISubprogram(name: "bar", scope: [[META1]], file: [[META1]], line: 1, type: [[META12:![0-9]+]], scopeLine: 1, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: [[META0]], retainedNodes: [[META2]])
-; CHECK: [[META12]] = !DISubroutineType(types: [[META2]])
+; CHECK: [[DBG7]] = !DILocation(line: 5, column: 14, scope: [[META8:![0-9]+]])
+; CHECK: [[META8]] = distinct !DISubprogram(name: "bar", scope: [[META1]], file: [[META1]], line: 1, type: [[META9:![0-9]+]], scopeLine: 1, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: [[META0]], retainedNodes: [[META2]])
+; CHECK: [[META9]] = !DISubroutineType(types: [[META2]])
 ;.
 ;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
 ; CHECK-LIMIT: {{.*}}
diff --git a/llvm/test/Transforms/OpenMP/single_threaded_execution.ll b/llvm/test/Transforms/OpenMP/single_threaded_execution.ll
index c186e5f04f092..70b9ce41c1a43 100644
--- a/llvm/test/Transforms/OpenMP/single_threaded_execution.ll
+++ b/llvm/test/Transforms/OpenMP/single_threaded_execution.ll
@@ -16,7 +16,7 @@
 ; CHECK: [openmp-opt] Basic block @kernel if.then is executed by a single thread.
 ; CHECK-NOT: [openmp-opt] Basic block @kernel if.else is executed by a single thread.
 ; CHECK-NOT: [openmp-opt] Basic block @kernel if.end is executed by a single thread.
-define void @kernel(ptr %dyn) "kernel" {
+define ptx_kernel void @kernel(ptr %dyn) "kernel" {
   %call = call i32 @__kmpc_target_init(ptr @kernel_kernel_environment, ptr %dyn)
   %cmp = icmp eq i32 %call, -1
   br i1 %cmp, label %if.then, label %if.else
@@ -116,7 +116,6 @@ attributes #0 = { cold noinline }
 
 !llvm.dbg.cu = !{!0}
 !llvm.module.flags = !{!3, !4, !5, !6}
-!nvvm.annotations = !{!7}
 
 !0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 12.0.0", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, enums: !2, splitDebugInlining: false, nameTableKind: None)
 !1 = !DIFile(filename: "single_threaded_execution.c", directory: "/tmp/single_threaded_execution.c")
@@ -125,7 +124,6 @@ attributes #0 = { cold noinline }
 !4 = !{i32 1, !"wchar_size", i32 4}
 !5 = !{i32 7, !"openmp", i32 50}
 !6 = !{i32 7, !"openmp-device", i32 50}
-!7 = !{ptr @kernel, !"kernel", i32 1}
 !8 = distinct !DISubprogram(name: "bar", scope: !1, file: !1, line: 8, type: !10, scopeLine: 1, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, retainedNodes: !2)
 !9 = distinct !DISubprogram(name: "cold", scope: !1, file: !1, line: 8, type: !10, scopeLine: 2, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, retainedNodes: !2)
 !10 = !DISubroutineType(types: !2)
diff --git a/llvm/test/Transforms/OpenMP/spmdization.ll b/llvm/test/Transforms/OpenMP/spmdization.ll
index 6ff4b96b57556..1a629ecfee06d 100644
--- a/llvm/test/Transforms/OpenMP/spmdization.ll
+++ b/llvm/test/Transforms/OpenMP/spmdization.ll
@@ -105,36 +105,6 @@
 @__omp_offloading_fd02_2044372e_do_not_spmdize_target_l65_kernel_environment = local_unnamed_addr constant %struct.KernelEnvironmentTy { %struct.ConfigurationEnvironmentTy { i8 1, i8 0, i8 1, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0 }, ptr @1, ptr null }
 @__omp_offloading_fd02_2044372e_do_not_spmdize_task_l74_kernel_environment = local_unnamed_addr constant %struct.KernelEnvironmentTy { %struct.ConfigurationEnvironmentTy { i8 1, i8 0, i8 1, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0 }, ptr @1, ptr null }
 
-; AMDGPU-DISABLED: @[[GLOB0:[0-9]+]] = private unnamed_addr constant [23 x i8] c"
-; AMDGPU-DISABLED: @[[GLOB1:[0-9]+]] = private unnamed_addr constant [[STRUCT_IDENT_T:%.*]] { i32 0, i32 2, i32 0, i32 0, ptr @[[GLOB0]] }, align 8
-; AMDGPU-DISABLED: @[[__OMP_OFFLOADING_FD02_2044372E_SEQUENTIAL_LOOP_L5_KERNEL_ENVIRONMENT:[a-zA-Z0-9_$"\\.-]+]] = local_unnamed_addr constant [[STRUCT_KERNELENVIRONMENTTY:%.*]] { [[STRUCT_CONFIGURATIONENVIRONMENTTY:%.*]] { i8 0, i8 1, i8 3 }, ptr @[[GLOB1]], ptr null }
-; AMDGPU-DISABLED: @[[__OMP_OFFLOADING_FD02_2044372E_SEQUENTIAL_LOOP_TO_STACK_VAR_L20_KERNEL_ENVIRONMENT:[a-zA-Z0-9_$"\\.-]+]] = local_unnamed_addr constant [[STRUCT_KERNELENVIRONMENTTY:%.*]] { [[STRUCT_CONFIGURATIONENVIRONMENTTY:%.*]] { i8 0, i8 1, i8 3 }, ptr @[[GLOB1]], ptr null }
-; AMDGPU-DISABLED: @[[__OMP_OFFLOADING_FD02_2044372E_SEQUENTIAL_LOOP_TO_SHARED_VAR_L35_KERNEL_ENVIRONMENT:[a-zA-Z0-9_$"\\.-]+]] = local_unnamed_addr constant [[STRUCT_KERNELENVIRONMENTTY:%.*]] { [[STRUCT_CONFIGURATIONENVIRONMENTTY:%.*]] { i8 0, i8 1, i8 3 }, ptr @[[GLOB1]], ptr null }
-; AMDGPU-DISABLED: @[[__OMP_OFFLOADING_FD02_2044372E_SEQUENTIAL_LOOP_TO_SHARED_VAR_GUARDED_L50_KERNEL_ENVIRONMENT:[a-zA-Z0-9_$"\\.-]+]] = local_unnamed_addr constant [[STRUCT_KERNELENVIRONMENTTY:%.*]] { [[STRUCT_CONFIGURATIONENVIRONMENTTY:%.*]] { i8 0, i8 1, i8 3 }, ptr @[[GLOB1]], ptr null }
-; AMDGPU-DISABLED: @[[__OMP_OFFLOADING_FD02_2044372E_DO_NOT_SPMDIZE_TARGET_L65_KERNEL_ENVIRONMENT:[a-zA-Z0-9_$"\\.-]+]] = local_unnamed_addr constant [[STRUCT_KERNELENVIRONMENTTY:%.*]] { [[STRUCT_CONFIGURATIONENVIRONMENTTY:%.*]] { i8 0, i8 0, i8 1 }, ptr @[[GLOB1]], ptr null }
-; AMDGPU-DISABLED: @[[__OMP_OFFLOADING_FD02_2044372E_DO_NOT_SPMDIZE_TASK_L74_KERNEL_ENVIRONMENT:[a-zA-Z0-9_$"\\.-]+]] = local_unnamed_addr constant [[STRUCT_KERNELENVIRONMENTTY:%.*]] { [[STRUCT_CONFIGURATIONENVIRONMENTTY:%.*]] { i8 0, i8 1, i8 1 }, ptr @[[GLOB1]], ptr null }
-; AMDGPU-DISABLED: @[[X_SHARED:[a-zA-Z0-9_$"\\.-]+]] = internal addrspace(3) global [4 x i8] poison, align 4
-; AMDGPU-DISABLED: @[[X_SHARED_1:[a-zA-Z0-9_$"\\.-]+]] = internal addrspace(3) global [4 x i8] poison, align 4
-; AMDGPU-DISABLED: @[[__OMP_OUTLINED__1_WRAPPER_ID:[a-zA-Z0-9_$"\\.-]+]] = private constant i8 undef
-; AMDGPU-DISABLED: @[[__OMP_OUTLINED__3_WRAPPER_ID:[a-zA-Z0-9_$"\\.-]+]] = private constant i8 undef
-; AMDGPU-DISABLED: @[[__OMP_OUTLINED__5_WRAPPER_ID:[a-zA-Z0-9_$"\\.-]+]] = private constant i8 undef
-; AMDGPU-DISABLED: @[[__OMP_OUTLINED__7_WRAPPER_ID:[a-zA-Z0-9_$"\\.-]+]] = private constant i8 undef
-; AMDGPU-DISABLED: @[[__OMP_OUTLINED__9_WRAPPER_ID:[a-zA-Z0-9_$"\\.-]+]] = private constant i8 undef
-; NVPTX-DISABLED: @[[GLOB0:[0-9]+]] = private unnamed_addr constant [23 x i8] c"
-; NVPTX-DISABLED: @[[GLOB1:[0-9]+]] = private unnamed_addr constant [[STRUCT_IDENT_T:%.*]] { i32 0, i32 2, i32 0, i32 0, ptr @[[GLOB0]] }, align 8
-; NVPTX-DISABLED: @[[__OMP_OFFLOADING_FD02_2044372E_SEQUENTIAL_LOOP_L5_KERNEL_ENVIRONMENT:[a-zA-Z0-9_$"\\.-]+]] = local_unnamed_addr constant [[STRUCT_KERNELENVIRONMENTTY:%.*]] { [[STRUCT_CONFIGURATIONENVIRONMENTTY:%.*]] { i8 0, i8 1, i8 3 }, ptr @[[GLOB1]], ptr null }
-; NVPTX-DISABLED: @[[__OMP_OFFLOADING_FD02_2044372E_SEQUENTIAL_LOOP_TO_STACK_VAR_L20_KERNEL_ENVIRONMENT:[a-zA-Z0-9_$"\\.-]+]] = local_unnamed_addr constant [[STRUCT_KERNELENVIRONMENTTY:%.*]] { [[STRUCT_CONFIGURATIONENVIRONMENTTY:%.*]] { i8 0, i8 1, i8 3 }, ptr @[[GLOB1]], ptr null }
-; NVPTX-DISABLED: @[[__OMP_OFFLOADING_FD02_2044372E_SEQUENTIAL_LOOP_TO_SHARED_VAR_L35_KERNEL_ENVIRONMENT:[a-zA-Z0-9_$"\\.-]+]] = local_unnamed_addr constant [[STRUCT_KERNELENVIRONMENTTY:%.*]] { [[STRUCT_CONFIGURATIONENVIRONMENTTY:%.*]] { i8 0, i8 1, i8 3 }, ptr @[[GLOB1]], ptr null }
-; NVPTX-DISABLED: @[[__OMP_OFFLOADING_FD02_2044372E_SEQUENTIAL_LOOP_TO_SHARED_VAR_GUARDED_L50_KERNEL_ENVIRONMENT:[a-zA-Z0-9_$"\\.-]+]] = local_unnamed_addr constant [[STRUCT_KERNELENVIRONMENTTY:%.*]] { [[STRUCT_CONFIGURATIONENVIRONMENTTY:%.*]] { i8 0, i8 1, i8 3 }, ptr @[[GLOB1]], ptr null }
-; NVPTX-DISABLED: @[[__OMP_OFFLOADING_FD02_2044372E_DO_NOT_SPMDIZE_TARGET_L65_KERNEL_ENVIRONMENT:[a-zA-Z0-9_$"\\.-]+]] = local_unnamed_addr constant [[STRUCT_KERNELENVIRONMENTTY:%.*]] { [[STRUCT_CONFIGURATIONENVIRONMENTTY:%.*]] { i8 0, i8 0, i8 1 }, ptr @[[GLOB1]], ptr null }
-; NVPTX-DISABLED: @[[__OMP_OFFLOADING_FD02_2044372E_DO_NOT_SPMDIZE_TASK_L74_KERNEL_ENVIRONMENT:[a-zA-Z0-9_$"\\.-]+]] = local_unnamed_addr constant [[STRUCT_KERNELENVIRONMENTTY:%.*]] { [[STRUCT_CONFIGURATIONENVIRONMENTTY:%.*]] { i8 0, i8 1, i8 1 }, ptr @[[GLOB1]], ptr null }
-; NVPTX-DISABLED: @[[X_SHARED:[a-zA-Z0-9_$"\\.-]+]] = internal addrspace(3) global [4 x i8] poison, align 4
-; NVPTX-DISABLED: @[[X_SHARED1:[a-zA-Z0-9_$"\\.-]+]] = internal addrspace(3) global [4 x i8] poison, align 4
-; NVPTX-DISABLED: @[[__OMP_OUTLINED__1_WRAPPER_ID:[a-zA-Z0-9_$"\\.-]+]] = private constant i8 undef
-; NVPTX-DISABLED: @[[__OMP_OUTLINED__3_WRAPPER_ID:[a-zA-Z0-9_$"\\.-]+]] = private constant i8 undef
-; NVPTX-DISABLED: @[[__OMP_OUTLINED__5_WRAPPER_ID:[a-zA-Z0-9_$"\\.-]+]] = private constant i8 undef
-; NVPTX-DISABLED: @[[__OMP_OUTLINED__7_WRAPPER_ID:[a-zA-Z0-9_$"\\.-]+]] = private constant i8 undef
-; NVPTX-DISABLED: @[[__OMP_OUTLINED__9_WRAPPER_ID:[a-zA-Z0-9_$"\\.-]+]] = private constant i8 undef
 ;.
 ; AMDGPU: @[[GLOB0:[0-9]+]] = private unnamed_addr constant [23 x i8] c"
 ; AMDGPU: @[[GLOB1:[0-9]+]] = private unnamed_addr constant %struct.ident_t { i32 0, i32 2, i32 0, i32 0, ptr @[[GLOB0]] }, align 8
@@ -180,19 +150,14 @@
 ;.
 ; AMDGPU-DISABLED2: @[[GLOB0:[0-9]+]] = private unnamed_addr constant [23 x i8] c"
 ; AMDGPU-DISABLED2: @[[GLOB1:[0-9]+]] = private unnamed_addr constant %struct.ident_t { i32 0, i32 2, i32 0, i32 0, ptr @[[GLOB0]] }, align 8
-; AMDGPU-DISABLED2: @__omp_offloading_fd02_2044372e_sequential_loop_l5_kernel_environment = local_unnamed_addr constant %struct.KernelEnvironmentTy { %struct.ConfigurationEnvironmentTy { i8 0, i8 1, i8 1, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0 }, ptr @[[GLOB1]], ptr null }
-; AMDGPU-DISABLED2: @__omp_offloading_fd02_2044372e_sequential_loop_to_stack_var_l20_kernel_environment = local_unnamed_addr constant %struct.KernelEnvironmentTy { %struct.ConfigurationEnvironmentTy { i8 0, i8 1, i8 1, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0 }, ptr @[[GLOB1]], ptr null }
-; AMDGPU-DISABLED2: @__omp_offloading_fd02_2044372e_sequential_loop_to_shared_var_l35_kernel_environment = local_unnamed_addr constant %struct.KernelEnvironmentTy { %struct.ConfigurationEnvironmentTy { i8 0, i8 1, i8 1, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0 }, ptr @[[GLOB1]], ptr null }
-; AMDGPU-DISABLED2: @__omp_offloading_fd02_2044372e_sequential_loop_to_shared_var_guarded_l50_kernel_environment = local_unnamed_addr constant %struct.KernelEnvironmentTy { %struct.ConfigurationEnvironmentTy { i8 0, i8 1, i8 1, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0 }, ptr @[[GLOB1]], ptr null }
-; AMDGPU-DISABLED2: @__omp_offloading_fd02_2044372e_do_not_spmdize_target_l65_kernel_environment = local_unnamed_addr constant %struct.KernelEnvironmentTy { %struct.ConfigurationEnvironmentTy { i8 0, i8 0, i8 1, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0 }, ptr @[[GLOB1]], ptr null }
-; AMDGPU-DISABLED2: @__omp_offloading_fd02_2044372e_do_not_spmdize_task_l74_kernel_environment = local_unnamed_addr constant %struct.KernelEnvironmentTy { %struct.ConfigurationEnvironmentTy { i8 0, i8 1, i8 1, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0 }, ptr @[[GLOB1]], ptr null }
+; AMDGPU-DISABLED2: @__omp_offloading_fd02_2044372e_sequential_loop_l5_kernel_environment = local_unnamed_addr constant %struct.KernelEnvironmentTy { %struct.ConfigurationEnvironmentTy { i8 1, i8 1, i8 1, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0 }, ptr @[[GLOB1]], ptr null }
+; AMDGPU-DISABLED2: @__omp_offloading_fd02_2044372e_sequential_loop_to_stack_var_l20_kernel_environment = local_unnamed_addr constant %struct.KernelEnvironmentTy { %struct.ConfigurationEnvironmentTy { i8 1, i8 1, i8 1, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0 }, ptr @[[GLOB1]], ptr null }
+; AMDGPU-DISABLED2: @__omp_offloading_fd02_2044372e_sequential_loop_to_shared_var_l35_kernel_environment = local_unnamed_addr constant %struct.KernelEnvironmentTy { %struct.ConfigurationEnvironmentTy { i8 1, i8 1, i8 1, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0 }, ptr @[[GLOB1]], ptr null }
+; AMDGPU-DISABLED2: @__omp_offloading_fd02_2044372e_sequential_loop_to_shared_var_guarded_l50_kernel_environment = local_unnamed_addr constant %struct.KernelEnvironmentTy { %struct.ConfigurationEnvironmentTy { i8 1, i8 1, i8 1, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0 }, ptr @[[GLOB1]], ptr null }
+; AMDGPU-DISABLED2: @__omp_offloading_fd02_2044372e_do_not_spmdize_target_l65_kernel_environment = local_unnamed_addr constant %struct.KernelEnvironmentTy { %struct.ConfigurationEnvironmentTy { i8 1, i8 0, i8 1, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0 }, ptr @[[GLOB1]], ptr null }
+; AMDGPU-DISABLED2: @__omp_offloading_fd02_2044372e_do_not_spmdize_task_l74_kernel_environment = local_unnamed_addr constant %struct.KernelEnvironmentTy { %struct.ConfigurationEnvironmentTy { i8 1, i8 1, i8 1, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0 }, ptr @[[GLOB1]], ptr null }
 ; AMDGPU-DISABLED2: @x_shared = internal addrspace(3) global [4 x i8] poison, align 4
 ; AMDGPU-DISABLED2: @x_shared.1 = internal addrspace(3) global [4 x i8] poison, align 4
-; AMDGPU-DISABLED2: @__omp_outlined__1_wrapper.ID = private constant i8 undef
-; AMDGPU-DISABLED2: @__omp_outlined__3_wrapper.ID = private constant i8 undef
-; AMDGPU-DISABLED2: @__omp_outlined__5_wrapper.ID = private constant i8 undef
-; AMDGPU-DISABLED2: @__omp_outlined__7_wrapper.ID = private constant i8 undef
-; AMDGPU-DISABLED2: @__omp_outlined__9_wrapper.ID = private constant i8 undef
 ;.
 ; NVPTX-DISABLED1: @[[GLOB0:[0-9]+]] = private unnamed_addr constant [23 x i8] c"
 ; NVPTX-DISABLED1: @[[GLOB1:[0-9]+]] = private unnamed_addr constant %struct.ident_t { i32 0, i32 2, i32 0, i32 0, ptr @[[GLOB0]] }, align 8
@@ -212,21 +177,16 @@
 ;.
 ; NVPTX-DISABLED2: @[[GLOB0:[0-9]+]] = private unnamed_addr constant [23 x i8] c"
 ; NVPTX-DISABLED2: @[[GLOB1:[0-9]+]] = private unnamed_addr constant %struct.ident_t { i32 0, i32 2, i32 0, i32 0, ptr @[[GLOB0]] }, align 8
-; NVPTX-DISABLED2: @__omp_offloading_fd02_2044372e_sequential_loop_l5_kernel_environment = local_unnamed_addr constant %struct.KernelEnvironmentTy { %struct.ConfigurationEnvironmentTy { i8 0, i8 1, i8 1, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0 }, ptr @[[GLOB1]], ptr null }
-; NVPTX-DISABLED2: @__omp_offloading_fd02_2044372e_sequential_loop_to_stack_var_l20_kernel_environment = local_unnamed_addr constant %struct.KernelEnvironmentTy { %struct.ConfigurationEnvironmentTy { i8 0, i8 1, i8 1, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0 }, ptr @[[GLOB1]], ptr null }
-; NVPTX-DISABLED2: @__omp_offloading_fd02_2044372e_sequential_loop_to_shared_var_l35_kernel_environment = local_unnamed_addr constant %struct.KernelEnvironmentTy { %struct.ConfigurationEnvironmentTy { i8 0, i8 1, i8 1, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0 }, ptr @[[GLOB1]], ptr null }
-; NVPTX-DISABLED2: @__omp_offloading_fd02_2044372e_sequential_loop_to_shared_var_guarded_l50_kernel_environment = local_unnamed_addr constant %struct.KernelEnvironmentTy { %struct.ConfigurationEnvironmentTy { i8 0, i8 1, i8 1, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0 }, ptr @[[GLOB1]], ptr null }
-; NVPTX-DISABLED2: @__omp_offloading_fd02_2044372e_do_not_spmdize_target_l65_kernel_environment = local_unnamed_addr constant %struct.KernelEnvironmentTy { %struct.ConfigurationEnvironmentTy { i8 0, i8 0, i8 1, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0 }, ptr @[[GLOB1]], ptr null }
-; NVPTX-DISABLED2: @__omp_offloading_fd02_2044372e_do_not_spmdize_task_l74_kernel_environment = local_unnamed_addr constant %struct.KernelEnvironmentTy { %struct.ConfigurationEnvironmentTy { i8 0, i8 1, i8 1, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0 }, ptr @[[GLOB1]], ptr null }
+; NVPTX-DISABLED2: @__omp_offloading_fd02_2044372e_sequential_loop_l5_kernel_environment = local_unnamed_addr constant %struct.KernelEnvironmentTy { %struct.ConfigurationEnvironmentTy { i8 1, i8 1, i8 1, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0 }, ptr @[[GLOB1]], ptr null }
+; NVPTX-DISABLED2: @__omp_offloading_fd02_2044372e_sequential_loop_to_stack_var_l20_kernel_environment = local_unnamed_addr constant %struct.KernelEnvironmentTy { %struct.ConfigurationEnvironmentTy { i8 1, i8 1, i8 1, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0 }, ptr @[[GLOB1]], ptr null }
+; NVPTX-DISABLED2: @__omp_offloading_fd02_2044372e_sequential_loop_to_shared_var_l35_kernel_environment = local_unnamed_addr constant %struct.KernelEnvironmentTy { %struct.ConfigurationEnvironmentTy { i8 1, i8 1, i8 1, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0 }, ptr @[[GLOB1]], ptr null }
+; NVPTX-DISABLED2: @__omp_offloading_fd02_2044372e_sequential_loop_to_shared_var_guarded_l50_kernel_environment = local_unnamed_addr constant %struct.KernelEnvironmentTy { %struct.ConfigurationEnvironmentTy { i8 1, i8 1, i8 1, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0 }, ptr @[[GLOB1]], ptr null }
+; NVPTX-DISABLED2: @__omp_offloading_fd02_2044372e_do_not_spmdize_target_l65_kernel_environment = local_unnamed_addr constant %struct.KernelEnvironmentTy { %struct.ConfigurationEnvironmentTy { i8 1, i8 0, i8 1, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0 }, ptr @[[GLOB1]], ptr null }
+; NVPTX-DISABLED2: @__omp_offloading_fd02_2044372e_do_not_spmdize_task_l74_kernel_environment = local_unnamed_addr constant %struct.KernelEnvironmentTy { %struct.ConfigurationEnvironmentTy { i8 1, i8 1, i8 1, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0 }, ptr @[[GLOB1]], ptr null }
 ; NVPTX-DISABLED2: @x_shared = internal addrspace(3) global [4 x i8] poison, align 4
 ; NVPTX-DISABLED2: @x_shared1 = internal addrspace(3) global [4 x i8] poison, align 4
-; NVPTX-DISABLED2: @__omp_outlined__1_wrapper.ID = private constant i8 undef
-; NVPTX-DISABLED2: @__omp_outlined__3_wrapper.ID = private constant i8 undef
-; NVPTX-DISABLED2: @__omp_outlined__5_wrapper.ID = private constant i8 undef
-; NVPTX-DISABLED2: @__omp_outlined__7_wrapper.ID = private constant i8 undef
-; NVPTX-DISABLED2: @__omp_outlined__9_wrapper.ID = private constant i8 undef
 ;.
-define weak void @__omp_offloading_fd02_2044372e_sequential_loop_l5() #0 {
+define weak ptx_kernel void @__omp_offloading_fd02_2044372e_sequential_loop_l5() #0 {
 ; AMDGPU-LABEL: define {{[^@]+}}@__omp_offloading_fd02_2044372e_sequential_loop_l5
 ; AMDGPU-SAME: () #[[ATTR0:[0-9]+]] {
 ; AMDGPU-NEXT:    call void @__omp_offloading_fd02_2044372e_sequential_loop_l5__debug()
@@ -257,14 +217,6 @@ define weak void @__omp_offloading_fd02_2044372e_sequential_loop_l5() #0 {
 ; NVPTX-DISABLED2-NEXT:    call void @__omp_offloading_fd02_2044372e_sequential_loop_l5__debug()
 ; NVPTX-DISABLED2-NEXT:    ret void
 ;
-; AMDGPU-DISABLED-LABEL: define {{[^@]+}}@__omp_offloading_fd02_2044372e_sequential_loop_l5
-; AMDGPU-DISABLED-SAME: () #[[ATTR0:[0-9]+]] {
-; AMDGPU-DISABLED-NEXT:    call void @__omp_offloading_fd02_2044372e_sequential_loop_l5__debug()
-; AMDGPU-DISABLED-NEXT:    ret void
-; NVPTX-DISABLED-LABEL: define {{[^@]+}}@__omp_offloading_fd02_2044372e_sequential_loop_l5
-; NVPTX-DISABLED-SAME: () #[[ATTR0:[0-9]+]] {
-; NVPTX-DISABLED-NEXT:    call void @__omp_offloading_fd02_2044372e_sequential_loop_l5__debug()
-; NVPTX-DISABLED-NEXT:    ret void
   call void @__omp_offloading_fd02_2044372e_sequential_loop_l5__debug()
   ret void
 }
@@ -282,7 +234,7 @@ define internal void @__omp_offloading_fd02_2044372e_sequential_loop_l5__debug()
 ; AMDGPU-NEXT:    ret void
 ; AMDGPU:       user_code.entry:
 ; AMDGPU-NEXT:    [[TMP1:%.*]] = call i32 @__kmpc_global_thread_num(ptr @[[GLOB1]]) #[[ATTR4:[0-9]+]]
-; AMDGPU-NEXT:    store i32 [[TMP1]], ptr [[DOTTHREADID_TEMP_]], align 4, !tbaa [[TBAA18:![0-9]+]]
+; AMDGPU-NEXT:    store i32 [[TMP1]], ptr [[DOTTHREADID_TEMP_]], align 4, !tbaa [[TBAA12:![0-9]+]]
 ; AMDGPU-NEXT:    call void @__omp_outlined__(ptr [[DOTTHREADID_TEMP_]], ptr [[DOTZERO_ADDR]]) #[[ATTR4]]
 ; AMDGPU-NEXT:    call void @__kmpc_target_deinit()
 ; AMDGPU-NEXT:    br label [[COMMON_RET]]
@@ -299,7 +251,7 @@ define internal void @__omp_offloading_fd02_2044372e_sequential_loop_l5__debug()
 ; NVPTX-NEXT:    ret void
 ; NVPTX:       user_code.entry:
 ; NVPTX-NEXT:    [[TMP1:%.*]] = call i32 @__kmpc_global_thread_num(ptr @[[GLOB1]]) #[[ATTR4:[0-9]+]]
-; NVPTX-NEXT:    store i32 [[TMP1]], ptr [[DOTTHREADID_TEMP_]], align 4, !tbaa [[TBAA18:![0-9]+]]
+; NVPTX-NEXT:    store i32 [[TMP1]], ptr [[DOTTHREADID_TEMP_]], align 4, !tbaa [[TBAA12:![0-9]+]]
 ; NVPTX-NEXT:    call void @__omp_outlined__(ptr [[DOTTHREADID_TEMP_]], ptr [[DOTZERO_ADDR]]) #[[ATTR4]]
 ; NVPTX-NEXT:    call void @__kmpc_target_deinit()
 ; NVPTX-NEXT:    br label [[COMMON_RET]]
@@ -350,7 +302,7 @@ define internal void @__omp_offloading_fd02_2044372e_sequential_loop_l5__debug()
 ; AMDGPU-DISABLED1-NEXT:    ret void
 ; AMDGPU-DISABLED1:       user_code.entry:
 ; AMDGPU-DISABLED1-NEXT:    [[TMP1:%.*]] = call i32 @__kmpc_global_thread_num(ptr @[[GLOB1]]) #[[ATTR4:[0-9]+]]
-; AMDGPU-DISABLED1-NEXT:    store i32 [[TMP1]], ptr [[DOTTHREADID_TEMP_]], align 4, !tbaa [[TBAA18:![0-9]+]]
+; AMDGPU-DISABLED1-NEXT:    store i32 [[TMP1]], ptr [[DOTTHREADID_TEMP_]], align 4, !tbaa [[TBAA12:![0-9]+]]
 ; AMDGPU-DISABLED1-NEXT:    call void @__omp_outlined__(ptr [[DOTTHREADID_TEMP_]], ptr [[DOTZERO_ADDR]]) #[[ATTR4]]
 ; AMDGPU-DISABLED1-NEXT:    call void @__kmpc_target_deinit()
 ; AMDGPU-DISABLED1-NEXT:    br label [[COMMON_RET]]
@@ -358,50 +310,16 @@ define internal void @__omp_offloading_fd02_2044372e_sequential_loop_l5__debug()
 ; AMDGPU-DISABLED2-LABEL: define {{[^@]+}}@__omp_offloading_fd02_2044372e_sequential_loop_l5__debug
 ; AMDGPU-DISABLED2-SAME: () #[[ATTR1:[0-9]+]] {
 ; AMDGPU-DISABLED2-NEXT:  entry:
-; AMDGPU-DISABLED2-NEXT:    [[WORKER_WORK_FN_ADDR:%.*]] = alloca ptr, align 8, addrspace(5)
 ; AMDGPU-DISABLED2-NEXT:    [[DOTZERO_ADDR:%.*]] = alloca i32, align 4
 ; AMDGPU-DISABLED2-NEXT:    [[DOTTHREADID_TEMP_:%.*]] = alloca i32, align 4
 ; AMDGPU-DISABLED2-NEXT:    [[TMP0:%.*]] = call i32 @__kmpc_target_init(ptr @__omp_offloading_fd02_2044372e_sequential_loop_l5_kernel_environment, ptr null)
-; AMDGPU-DISABLED2-NEXT:    [[THREAD_IS_WORKER:%.*]] = icmp ne i32 [[TMP0]], -1
-; AMDGPU-DISABLED2-NEXT:    br i1 [[THREAD_IS_WORKER]], label [[IS_WORKER_CHECK:%.*]], label [[THREAD_USER_CODE_CHECK:%.*]]
-; AMDGPU-DISABLED2:       is_worker_check:
-; AMDGPU-DISABLED2-NEXT:    [[BLOCK_HW_SIZE:%.*]] = call i32 @__kmpc_get_hardware_num_threads_in_block()
-; AMDGPU-DISABLED2-NEXT:    [[WARP_SIZE:%.*]] = call i32 @__kmpc_get_warp_size()
-; AMDGPU-DISABLED2-NEXT:    [[BLOCK_SIZE:%.*]] = sub i32 [[BLOCK_HW_SIZE]], [[WARP_SIZE]]
-; AMDGPU-DISABLED2-NEXT:    [[THREAD_IS_MAIN_OR_WORKER:%.*]] = icmp slt i32 [[TMP0]], [[BLOCK_SIZE]]
-; AMDGPU-DISABLED2-NEXT:    br i1 [[THREAD_IS_MAIN_OR_WORKER]], label [[WORKER_STATE_MACHINE_BEGIN:%.*]], label [[WORKER_STATE_MACHINE_FINISHED:%.*]]
-; AMDGPU-DISABLED2:       worker_state_machine.begin:
-; AMDGPU-DISABLED2-NEXT:    call void @__kmpc_barrier_simple_generic(ptr @[[GLOB1]], i32 [[TMP0]])
-; AMDGPU-DISABLED2-NEXT:    [[WORKER_WORK_FN_ADDR_GENERIC:%.*]] = addrspacecast ptr addrspace(5) [[WORKER_WORK_FN_ADDR]] to ptr
-; AMDGPU-DISABLED2-NEXT:    [[WORKER_IS_ACTIVE:%.*]] = call i1 @__kmpc_kernel_parallel(ptr [[WORKER_WORK_FN_ADDR_GENERIC]])
-; AMDGPU-DISABLED2-NEXT:    [[WORKER_WORK_FN:%.*]] = load ptr, ptr [[WORKER_WORK_FN_ADDR_GENERIC]], align 8
-; AMDGPU-DISABLED2-NEXT:    [[WORKER_IS_DONE:%.*]] = icmp eq ptr [[WORKER_WORK_FN]], null
-; AMDGPU-DISABLED2-NEXT:    br i1 [[WORKER_IS_DONE]], label [[WORKER_STATE_MACHINE_FINISHED]], label [[WORKER_STATE_MACHINE_IS_ACTIVE_CHECK:%.*]]
-; AMDGPU-DISABLED2:       worker_state_machine.finished:
-; AMDGPU-DISABLED2-NEXT:    ret void
-; AMDGPU-DISABLED2:       worker_state_machine.is_active.check:
-; AMDGPU-DISABLED2-NEXT:    br i1 [[WORKER_IS_ACTIVE]], label [[WORKER_STATE_MACHINE_PARALLEL_REGION_CHECK:%.*]], label [[WORKER_STATE_MACHINE_DONE_BARRIER:%.*]]
-; AMDGPU-DISABLED2:       worker_state_machine.parallel_region.check:
-; AMDGPU-DISABLED2-NEXT:    br i1 true, label [[WORKER_STATE_MACHINE_PARALLEL_REGION_EXECUTE:%.*]], label [[WORKER_STATE_MACHINE_PARALLEL_REGION_CHECK1:%.*]]
-; AMDGPU-DISABLED2:       worker_state_machine.parallel_region.execute:
-; AMDGPU-DISABLED2-NEXT:    call void @__omp_outlined__1_wrapper(i16 0, i32 [[TMP0]])
-; AMDGPU-DISABLED2-NEXT:    br label [[WORKER_STATE_MACHINE_PARALLEL_REGION_END:%.*]]
-; AMDGPU-DISABLED2:       worker_state_machine.parallel_region.check1:
-; AMDGPU-DISABLED2-NEXT:    br label [[WORKER_STATE_MACHINE_PARALLEL_REGION_END]]
-; AMDGPU-DISABLED2:       worker_state_machine.parallel_region.end:
-; AMDGPU-DISABLED2-NEXT:    call void @__kmpc_kernel_end_parallel()
-; AMDGPU-DISABLED2-NEXT:    br label [[WORKER_STATE_MACHINE_DONE_BARRIER]]
-; AMDGPU-DISABLED2:       worker_state_machine.done.barrier:
-; AMDGPU-DISABLED2-NEXT:    call void @__kmpc_barrier_simple_generic(ptr @[[GLOB1]], i32 [[TMP0]])
-; AMDGPU-DISABLED2-NEXT:    br label [[WORKER_STATE_MACHINE_BEGIN]]
-; AMDGPU-DISABLED2:       thread.user_code.check:
 ; AMDGPU-DISABLED2-NEXT:    [[EXEC_USER_CODE:%.*]] = icmp eq i32 [[TMP0]], -1
 ; AMDGPU-DISABLED2-NEXT:    br i1 [[EXEC_USER_CODE]], label [[USER_CODE_ENTRY:%.*]], label [[COMMON_RET:%.*]]
 ; AMDGPU-DISABLED2:       common.ret:
 ; AMDGPU-DISABLED2-NEXT:    ret void
 ; AMDGPU-DISABLED2:       user_code.entry:
 ; AMDGPU-DISABLED2-NEXT:    [[TMP1:%.*]] = call i32 @__kmpc_global_thread_num(ptr @[[GLOB1]]) #[[ATTR4:[0-9]+]]
-; AMDGPU-DISABLED2-NEXT:    store i32 [[TMP1]], ptr [[DOTTHREADID_TEMP_]], align 4, !tbaa [[TBAA18:![0-9]+]]
+; AMDGPU-DISABLED2-NEXT:    store i32 [[TMP1]], ptr [[DOTTHREADID_TEMP_]], align 4, !tbaa [[TBAA12:![0-9]+]]
 ; AMDGPU-DISABLED2-NEXT:    call void @__omp_outlined__(ptr [[DOTTHREADID_TEMP_]], ptr [[DOTZERO_ADDR]]) #[[ATTR4]]
 ; AMDGPU-DISABLED2-NEXT:    call void @__kmpc_target_deinit()
 ; AMDGPU-DISABLED2-NEXT:    br label [[COMMON_RET]]
@@ -451,7 +369,7 @@ define internal void @__omp_offloading_fd02_2044372e_sequential_loop_l5__debug()
 ; NVPTX-DISABLED1-NEXT:    ret void
 ; NVPTX-DISABLED1:       user_code.entry:
 ; NVPTX-DISABLED1-NEXT:    [[TMP1:%.*]] = call i32 @__kmpc_global_thread_num(ptr @[[GLOB1]]) #[[ATTR4:[0-9]+]]
-; NVPTX-DISABLED1-NEXT:    store i32 [[TMP1]], ptr [[DOTTHREADID_TEMP_]], align 4, !tbaa [[TBAA18:![0-9]+]]
+; NVPTX-DISABLED1-NEXT:    store i32 [[TMP1]], ptr [[DOTTHREADID_TEMP_]], align 4, !tbaa [[TBAA12:![0-9]+]]
 ; NVPTX-DISABLED1-NEXT:    call void @__omp_outlined__(ptr [[DOTTHREADID_TEMP_]], ptr [[DOTZERO_ADDR]]) #[[ATTR4]]
 ; NVPTX-DISABLED1-NEXT:    call void @__kmpc_target_deinit()
 ; NVPTX-DISABLED1-NEXT:    br label [[COMMON_RET]]
@@ -459,156 +377,20 @@ define internal void @__omp_offloading_fd02_2044372e_sequential_loop_l5__debug()
 ; NVPTX-DISABLED2-LABEL: define {{[^@]+}}@__omp_offloading_fd02_2044372e_sequential_loop_l5__debug
 ; NVPTX-DISABLED2-SAME: () #[[ATTR1:[0-9]+]] {
 ; NVPTX-DISABLED2-NEXT:  entry:
-; NVPTX-DISABLED2-NEXT:    [[WORKER_WORK_FN_ADDR:%.*]] = alloca ptr, align 8
 ; NVPTX-DISABLED2-NEXT:    [[DOTZERO_ADDR:%.*]] = alloca i32, align 4
 ; NVPTX-DISABLED2-NEXT:    [[DOTTHREADID_TEMP_:%.*]] = alloca i32, align 4
 ; NVPTX-DISABLED2-NEXT:    [[TMP0:%.*]] = call i32 @__kmpc_target_init(ptr @__omp_offloading_fd02_2044372e_sequential_loop_l5_kernel_environment, ptr null)
-; NVPTX-DISABLED2-NEXT:    [[THREAD_IS_WORKER:%.*]] = icmp ne i32 [[TMP0]], -1
-; NVPTX-DISABLED2-NEXT:    br i1 [[THREAD_IS_WORKER]], label [[IS_WORKER_CHECK:%.*]], label [[THREAD_USER_CODE_CHECK:%.*]]
-; NVPTX-DISABLED2:       is_worker_check:
-; NVPTX-DISABLED2-NEXT:    [[BLOCK_HW_SIZE:%.*]] = call i32 @__kmpc_get_hardware_num_threads_in_block()
-; NVPTX-DISABLED2-NEXT:    [[WARP_SIZE:%.*]] = call i32 @__kmpc_get_warp_size()
-; NVPTX-DISABLED2-NEXT:    [[BLOCK_SIZE:%.*]] = sub i32 [[BLOCK_HW_SIZE]], [[WARP_SIZE]]
-; NVPTX-DISABLED2-NEXT:    [[THREAD_IS_MAIN_OR_WORKER:%.*]] = icmp slt i32 [[TMP0]], [[BLOCK_SIZE]]
-; NVPTX-DISABLED2-NEXT:    br i1 [[THREAD_IS_MAIN_OR_WORKER]], label [[WORKER_STATE_MACHINE_BEGIN:%.*]], label [[WORKER_STATE_MACHINE_FINISHED:%.*]]
-; NVPTX-DISABLED2:       worker_state_machine.begin:
-; NVPTX-DISABLED2-NEXT:    call void @__kmpc_barrier_simple_generic(ptr @[[GLOB1]], i32 [[TMP0]])
-; NVPTX-DISABLED2-NEXT:    [[WORKER_IS_ACTIVE:%.*]] = call i1 @__kmpc_kernel_parallel(ptr [[WORKER_WORK_FN_ADDR]])
-; NVPTX-DISABLED2-NEXT:    [[WORKER_WORK_FN:%.*]] = load ptr, ptr [[WORKER_WORK_FN_ADDR]], align 8
-; NVPTX-DISABLED2-NEXT:    [[WORKER_IS_DONE:%.*]] = icmp eq ptr [[WORKER_WORK_FN]], null
-; NVPTX-DISABLED2-NEXT:    br i1 [[WORKER_IS_DONE]], label [[WORKER_STATE_MACHINE_FINISHED]], label [[WORKER_STATE_MACHINE_IS_ACTIVE_CHECK:%.*]]
-; NVPTX-DISABLED2:       worker_state_machine.finished:
-; NVPTX-DISABLED2-NEXT:    ret void
-; NVPTX-DISABLED2:       worker_state_machine.is_active.check:
-; NVPTX-DISABLED2-NEXT:    br i1 [[WORKER_IS_ACTIVE]], label [[WORKER_STATE_MACHINE_PARALLEL_REGION_CHECK:%.*]], label [[WORKER_STATE_MACHINE_DONE_BARRIER:%.*]]
-; NVPTX-DISABLED2:       worker_state_machine.parallel_region.check:
-; NVPTX-DISABLED2-NEXT:    br i1 true, label [[WORKER_STATE_MACHINE_PARALLEL_REGION_EXECUTE:%.*]], label [[WORKER_STATE_MACHINE_PARALLEL_REGION_CHECK1:%.*]]
-; NVPTX-DISABLED2:       worker_state_machine.parallel_region.execute:
-; NVPTX-DISABLED2-NEXT:    call void @__omp_outlined__1_wrapper(i16 0, i32 [[TMP0]])
-; NVPTX-DISABLED2-NEXT:    br label [[WORKER_STATE_MACHINE_PARALLEL_REGION_END:%.*]]
-; NVPTX-DISABLED2:       worker_state_machine.parallel_region.check1:
-; NVPTX-DISABLED2-NEXT:    br label [[WORKER_STATE_MACHINE_PARALLEL_REGION_END]]
-; NVPTX-DISABLED2:       worker_state_machine.parallel_region.end:
-; NVPTX-DISABLED2-NEXT:    call void @__kmpc_kernel_end_parallel()
-; NVPTX-DISABLED2-NEXT:    br label [[WORKER_STATE_MACHINE_DONE_BARRIER]]
-; NVPTX-DISABLED2:       worker_state_machine.done.barrier:
-; NVPTX-DISABLED2-NEXT:    call void @__kmpc_barrier_simple_generic(ptr @[[GLOB1]], i32 [[TMP0]])
-; NVPTX-DISABLED2-NEXT:    br label [[WORKER_STATE_MACHINE_BEGIN]]
-; NVPTX-DISABLED2:       thread.user_code.check:
 ; NVPTX-DISABLED2-NEXT:    [[EXEC_USER_CODE:%.*]] = icmp eq i32 [[TMP0]], -1
 ; NVPTX-DISABLED2-NEXT:    br i1 [[EXEC_USER_CODE]], label [[USER_CODE_ENTRY:%.*]], label [[COMMON_RET:%.*]]
 ; NVPTX-DISABLED2:       common.ret:
 ; NVPTX-DISABLED2-NEXT:    ret void
 ; NVPTX-DISABLED2:       user_code.entry:
 ; NVPTX-DISABLED2-NEXT:    [[TMP1:%.*]] = call i32 @__kmpc_global_thread_num(ptr @[[GLOB1]]) #[[ATTR4:[0-9]+]]
-; NVPTX-DISABLED2-NEXT:    store i32 [[TMP1]], ptr [[DOTTHREADID_TEMP_]], align 4, !tbaa [[TBAA18:![0-9]+]]
+; NVPTX-DISABLED2-NEXT:    store i32 [[TMP1]], ptr [[DOTTHREADID_TEMP_]], align 4, !tbaa [[TBAA12:![0-9]+]]
 ; NVPTX-DISABLED2-NEXT:    call void @__omp_outlined__(ptr [[DOTTHREADID_TEMP_]], ptr [[DOTZERO_ADDR]]) #[[ATTR4]]
 ; NVPTX-DISABLED2-NEXT:    call void @__kmpc_target_deinit()
 ; NVPTX-DISABLED2-NEXT:    br label [[COMMON_RET]]
 ;
-; AMDGPU-DISABLED-LABEL: define {{[^@]+}}@__omp_offloading_fd02_2044372e_sequential_loop_l5__debug
-; AMDGPU-DISABLED-SAME: () #[[ATTR1:[0-9]+]] {
-; AMDGPU-DISABLED-NEXT:  entry:
-; AMDGPU-DISABLED-NEXT:    [[WORKER_WORK_FN_ADDR:%.*]] = alloca ptr, align 8, addrspace(5)
-; AMDGPU-DISABLED-NEXT:    [[DOTZERO_ADDR:%.*]] = alloca i32, align 4
-; AMDGPU-DISABLED-NEXT:    [[DOTTHREADID_TEMP_:%.*]] = alloca i32, align 4
-; AMDGPU-DISABLED-NEXT:    [[TMP0:%.*]] = call i32 @__kmpc_target_init(ptr @__omp_offloading_fd02_2044372e_sequential_loop_l5_kernel_environment)
-; AMDGPU-DISABLED-NEXT:    [[THREAD_IS_WORKER:%.*]] = icmp ne i32 [[TMP0]], -1
-; AMDGPU-DISABLED-NEXT:    br i1 [[THREAD_IS_WORKER]], label [[IS_WORKER_CHECK:%.*]], label [[THREAD_USER_CODE_CHECK:%.*]]
-; AMDGPU-DISABLED:       is_worker_check:
-; AMDGPU-DISABLED-NEXT:    [[BLOCK_HW_SIZE:%.*]] = call i32 @__kmpc_get_hardware_num_threads_in_block()
-; AMDGPU-DISABLED-NEXT:    [[WARP_SIZE:%.*]] = call i32 @__kmpc_get_warp_size()
-; AMDGPU-DISABLED-NEXT:    [[BLOCK_SIZE:%.*]] = sub i32 [[BLOCK_HW_SIZE]], [[WARP_SIZE]]
-; AMDGPU-DISABLED-NEXT:    [[THREAD_IS_MAIN_OR_WORKER:%.*]] = icmp slt i32 [[TMP0]], [[BLOCK_SIZE]]
-; AMDGPU-DISABLED-NEXT:    br i1 [[THREAD_IS_MAIN_OR_WORKER]], label [[WORKER_STATE_MACHINE_BEGIN:%.*]], label [[WORKER_STATE_MACHINE_FINISHED:%.*]]
-; AMDGPU-DISABLED:       worker_state_machine.begin:
-; AMDGPU-DISABLED-NEXT:    call void @__kmpc_barrier_simple_generic(ptr @[[GLOB1]], i32 [[TMP0]])
-; AMDGPU-DISABLED-NEXT:    [[WORKER_WORK_FN_ADDR_GENERIC:%.*]] = addrspacecast ptr addrspace(5) [[WORKER_WORK_FN_ADDR]] to ptr
-; AMDGPU-DISABLED-NEXT:    [[WORKER_IS_ACTIVE:%.*]] = call i1 @__kmpc_kernel_parallel(ptr [[WORKER_WORK_FN_ADDR_GENERIC]])
-; AMDGPU-DISABLED-NEXT:    [[WORKER_WORK_FN:%.*]] = load ptr, ptr [[WORKER_WORK_FN_ADDR_GENERIC]], align 8
-; AMDGPU-DISABLED-NEXT:    [[WORKER_IS_DONE:%.*]] = icmp eq ptr [[WORKER_WORK_FN]], null
-; AMDGPU-DISABLED-NEXT:    br i1 [[WORKER_IS_DONE]], label [[WORKER_STATE_MACHINE_FINISHED]], label [[WORKER_STATE_MACHINE_IS_ACTIVE_CHECK:%.*]]
-; AMDGPU-DISABLED:       worker_state_machine.finished:
-; AMDGPU-DISABLED-NEXT:    ret void
-; AMDGPU-DISABLED:       worker_state_machine.is_active.check:
-; AMDGPU-DISABLED-NEXT:    br i1 [[WORKER_IS_ACTIVE]], label [[WORKER_STATE_MACHINE_PARALLEL_REGION_CHECK:%.*]], label [[WORKER_STATE_MACHINE_DONE_BARRIER:%.*]]
-; AMDGPU-DISABLED:       worker_state_machine.parallel_region.check:
-; AMDGPU-DISABLED-NEXT:    [[WORKER_CHECK_PARALLEL_REGION:%.*]] = icmp eq ptr [[WORKER_WORK_FN]], @__omp_outlined__1_wrapper.ID
-; AMDGPU-DISABLED-NEXT:    br i1 [[WORKER_CHECK_PARALLEL_REGION]], label [[WORKER_STATE_MACHINE_PARALLEL_REGION_EXECUTE:%.*]], label [[WORKER_STATE_MACHINE_PARALLEL_REGION_FALLBACK_EXECUTE:%.*]]
-; AMDGPU-DISABLED:       worker_state_machine.parallel_region.execute:
-; AMDGPU-DISABLED-NEXT:    call void @__omp_outlined__1_wrapper(i16 0, i32 [[TMP0]])
-; AMDGPU-DISABLED-NEXT:    br label [[WORKER_STATE_MACHINE_PARALLEL_REGION_END:%.*]]
-; AMDGPU-DISABLED:       worker_state_machine.parallel_region.fallback.execute:
-; AMDGPU-DISABLED-NEXT:    call void [[WORKER_WORK_FN]](i16 0, i32 [[TMP0]])
-; AMDGPU-DISABLED-NEXT:    br label [[WORKER_STATE_MACHINE_PARALLEL_REGION_END]]
-; AMDGPU-DISABLED:       worker_state_machine.parallel_region.end:
-; AMDGPU-DISABLED-NEXT:    call void @__kmpc_kernel_end_parallel()
-; AMDGPU-DISABLED-NEXT:    br label [[WORKER_STATE_MACHINE_DONE_BARRIER]]
-; AMDGPU-DISABLED:       worker_state_machine.done.barrier:
-; AMDGPU-DISABLED-NEXT:    call void @__kmpc_barrier_simple_generic(ptr @[[GLOB1]], i32 [[TMP0]])
-; AMDGPU-DISABLED-NEXT:    br label [[WORKER_STATE_MACHINE_BEGIN]]
-; AMDGPU-DISABLED:       thread.user_code.check:
-; AMDGPU-DISABLED-NEXT:    [[EXEC_USER_CODE:%.*]] = icmp eq i32 [[TMP0]], -1
-; AMDGPU-DISABLED-NEXT:    br i1 [[EXEC_USER_CODE]], label [[USER_CODE_ENTRY:%.*]], label [[COMMON_RET:%.*]]
-; AMDGPU-DISABLED:       common.ret:
-; AMDGPU-DISABLED-NEXT:    ret void
-; AMDGPU-DISABLED:       user_code.entry:
-; AMDGPU-DISABLED-NEXT:    [[TMP1:%.*]] = call i32 @__kmpc_global_thread_num(ptr @[[GLOB1]]) #[[ATTR4:[0-9]+]]
-; AMDGPU-DISABLED-NEXT:    store i32 [[TMP1]], ptr [[DOTTHREADID_TEMP_]], align 4, !tbaa [[TBAA18:![0-9]+]]
-; AMDGPU-DISABLED-NEXT:    call void @__omp_outlined__(ptr [[DOTTHREADID_TEMP_]], ptr [[DOTZERO_ADDR]]) #[[ATTR5]]
-; AMDGPU-DISABLED-NEXT:    call void @__kmpc_target_deinit()
-; AMDGPU-DISABLED-NEXT:    br label [[COMMON_RET]]
-; NVPTX-DISABLED-LABEL: define {{[^@]+}}@__omp_offloading_fd02_2044372e_sequential_loop_l5__debug
-; NVPTX-DISABLED-SAME: () #[[ATTR1:[0-9]+]] {
-; NVPTX-DISABLED-NEXT:  entry:
-; NVPTX-DISABLED-NEXT:    [[WORKER_WORK_FN_ADDR:%.*]] = alloca ptr, align 8
-; NVPTX-DISABLED-NEXT:    [[DOTZERO_ADDR:%.*]] = alloca i32, align 4
-; NVPTX-DISABLED-NEXT:    [[DOTTHREADID_TEMP_:%.*]] = alloca i32, align 4
-; NVPTX-DISABLED-NEXT:    [[TMP0:%.*]] = call i32 @__kmpc_target_init(ptr @__omp_offloading_fd02_2044372e_sequential_loop_l5_kernel_environment)
-; NVPTX-DISABLED-NEXT:    [[THREAD_IS_WORKER:%.*]] = icmp ne i32 [[TMP0]], -1
-; NVPTX-DISABLED-NEXT:    br i1 [[THREAD_IS_WORKER]], label [[IS_WORKER_CHECK:%.*]], label [[THREAD_USER_CODE_CHECK:%.*]]
-; NVPTX-DISABLED:       is_worker_check:
-; NVPTX-DISABLED-NEXT:    [[BLOCK_HW_SIZE:%.*]] = call i32 @__kmpc_get_hardware_num_threads_in_block()
-; NVPTX-DISABLED-NEXT:    [[WARP_SIZE:%.*]] = call i32 @__kmpc_get_warp_size()
-; NVPTX-DISABLED-NEXT:    [[BLOCK_SIZE:%.*]] = sub i32 [[BLOCK_HW_SIZE]], [[WARP_SIZE]]
-; NVPTX-DISABLED-NEXT:    [[THREAD_IS_MAIN_OR_WORKER:%.*]] = icmp slt i32 [[TMP0]], [[BLOCK_SIZE]]
-; NVPTX-DISABLED-NEXT:    br i1 [[THREAD_IS_MAIN_OR_WORKER]], label [[WORKER_STATE_MACHINE_BEGIN:%.*]], label [[WORKER_STATE_MACHINE_FINISHED:%.*]]
-; NVPTX-DISABLED:       worker_state_machine.begin:
-; NVPTX-DISABLED-NEXT:    call void @__kmpc_barrier_simple_generic(ptr @[[GLOB1]], i32 [[TMP0]])
-; NVPTX-DISABLED-NEXT:    [[WORKER_IS_ACTIVE:%.*]] = call i1 @__kmpc_kernel_parallel(ptr [[WORKER_WORK_FN_ADDR]])
-; NVPTX-DISABLED-NEXT:    [[WORKER_WORK_FN:%.*]] = load ptr, ptr [[WORKER_WORK_FN_ADDR]], align 8
-; NVPTX-DISABLED-NEXT:    [[WORKER_IS_DONE:%.*]] = icmp eq ptr [[WORKER_WORK_FN]], null
-; NVPTX-DISABLED-NEXT:    br i1 [[WORKER_IS_DONE]], label [[WORKER_STATE_MACHINE_FINISHED]], label [[WORKER_STATE_MACHINE_IS_ACTIVE_CHECK:%.*]]
-; NVPTX-DISABLED:       worker_state_machine.finished:
-; NVPTX-DISABLED-NEXT:    ret void
-; NVPTX-DISABLED:       worker_state_machine.is_active.check:
-; NVPTX-DISABLED-NEXT:    br i1 [[WORKER_IS_ACTIVE]], label [[WORKER_STATE_MACHINE_PARALLEL_REGION_CHECK:%.*]], label [[WORKER_STATE_MACHINE_DONE_BARRIER:%.*]]
-; NVPTX-DISABLED:       worker_state_machine.parallel_region.check:
-; NVPTX-DISABLED-NEXT:    [[WORKER_CHECK_PARALLEL_REGION:%.*]] = icmp eq ptr [[WORKER_WORK_FN]], @__omp_outlined__1_wrapper.ID
-; NVPTX-DISABLED-NEXT:    br i1 [[WORKER_CHECK_PARALLEL_REGION]], label [[WORKER_STATE_MACHINE_PARALLEL_REGION_EXECUTE:%.*]], label [[WORKER_STATE_MACHINE_PARALLEL_REGION_FALLBACK_EXECUTE:%.*]]
-; NVPTX-DISABLED:       worker_state_machine.parallel_region.execute:
-; NVPTX-DISABLED-NEXT:    call void @__omp_outlined__1_wrapper(i16 0, i32 [[TMP0]])
-; NVPTX-DISABLED-NEXT:    br label [[WORKER_STATE_MACHINE_PARALLEL_REGION_END:%.*]]
-; NVPTX-DISABLED:       worker_state_machine.parallel_region.fallback.execute:
-; NVPTX-DISABLED-NEXT:    call void [[WORKER_WORK_FN]](i16 0, i32 [[TMP0]])
-; NVPTX-DISABLED-NEXT:    br label [[WORKER_STATE_MACHINE_PARALLEL_REGION_END]]
-; NVPTX-DISABLED:       worker_state_machine.parallel_region.end:
-; NVPTX-DISABLED-NEXT:    call void @__kmpc_kernel_end_parallel()
-; NVPTX-DISABLED-NEXT:    br label [[WORKER_STATE_MACHINE_DONE_BARRIER]]
-; NVPTX-DISABLED:       worker_state_machine.done.barrier:
-; NVPTX-DISABLED-NEXT:    call void @__kmpc_barrier_simple_generic(ptr @[[GLOB1]], i32 [[TMP0]])
-; NVPTX-DISABLED-NEXT:    br label [[WORKER_STATE_MACHINE_BEGIN]]
-; NVPTX-DISABLED:       thread.user_code.check:
-; NVPTX-DISABLED-NEXT:    [[EXEC_USER_CODE:%.*]] = icmp eq i32 [[TMP0]], -1
-; NVPTX-DISABLED-NEXT:    br i1 [[EXEC_USER_CODE]], label [[USER_CODE_ENTRY:%.*]], label [[COMMON_RET:%.*]]
-; NVPTX-DISABLED:       common.ret:
-; NVPTX-DISABLED-NEXT:    ret void
-; NVPTX-DISABLED:       user_code.entry:
-; NVPTX-DISABLED-NEXT:    [[TMP1:%.*]] = call i32 @__kmpc_global_thread_num(ptr @[[GLOB1]]) #[[ATTR4:[0-9]+]]
-; NVPTX-DISABLED-NEXT:    store i32 [[TMP1]], ptr [[DOTTHREADID_TEMP_]], align 4, !tbaa [[TBAA18:![0-9]+]]
-; NVPTX-DISABLED-NEXT:    call void @__omp_outlined__(ptr [[DOTTHREADID_TEMP_]], ptr [[DOTZERO_ADDR]]) #[[ATTR5]]
-; NVPTX-DISABLED-NEXT:    call void @__kmpc_target_deinit()
-; NVPTX-DISABLED-NEXT:    br label [[COMMON_RET]]
 entry:
   %.zero.addr = alloca i32, align 4
   %.threadid_temp. = alloca i32, align 4
@@ -643,10 +425,10 @@ define internal void @__omp_outlined__(ptr noalias %.global_tid., ptr noalias %.
 ; AMDGPU-NEXT:    call void @spmd_amenable() #[[ATTR7:[0-9]+]]
 ; AMDGPU-NEXT:    ret void
 ; AMDGPU:       for.body:
-; AMDGPU-NEXT:    [[TMP0:%.*]] = load i32, ptr [[DOTGLOBAL_TID_]], align 4, !tbaa [[TBAA18]]
+; AMDGPU-NEXT:    [[TMP0:%.*]] = load i32, ptr [[DOTGLOBAL_TID_]], align 4, !tbaa [[TBAA12]]
 ; AMDGPU-NEXT:    call void @__kmpc_parallel_51(ptr @[[GLOB1]], i32 [[TMP0]], i32 1, i32 -1, i32 -1, ptr @__omp_outlined__1, ptr @__omp_outlined__1_wrapper, ptr [[CAPTURED_VARS_ADDRS]], i64 0)
 ; AMDGPU-NEXT:    [[INC]] = add nsw i32 [[I_0]], 1
-; AMDGPU-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP22:![0-9]+]]
+; AMDGPU-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP16:![0-9]+]]
 ;
 ; NVPTX-LABEL: define {{[^@]+}}@__omp_outlined__
 ; NVPTX-SAME: (ptr noalias [[DOTGLOBAL_TID_:%.*]], ptr noalias [[DOTBOUND_TID_:%.*]]) #[[ATTR1]] {
@@ -661,10 +443,10 @@ define internal void @__omp_outlined__(ptr noalias %.global_tid., ptr noalias %.
 ; NVPTX-NEXT:    call void @spmd_amenable() #[[ATTR7:[0-9]+]]
 ; NVPTX-NEXT:    ret void
 ; NVPTX:       for.body:
-; NVPTX-NEXT:    [[TMP0:%.*]] = load i32, ptr [[DOTGLOBAL_TID_]], align 4, !tbaa [[TBAA18]]
+; NVPTX-NEXT:    [[TMP0:%.*]] = load i32, ptr [[DOTGLOBAL_TID_]], align 4, !tbaa [[TBAA12]]
 ; NVPTX-NEXT:    call void @__kmpc_parallel_51(ptr @[[GLOB1]], i32 [[TMP0]], i32 1, i32 -1, i32 -1, ptr @__omp_outlined__1, ptr @__omp_outlined__1_wrapper, ptr [[CAPTURED_VARS_ADDRS]], i64 0)
 ; NVPTX-NEXT:    [[INC]] = add nsw i32 [[I_0]], 1
-; NVPTX-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP22:![0-9]+]]
+; NVPTX-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP16:![0-9]+]]
 ;
 ; AMDGPU-DISABLED1-LABEL: define {{[^@]+}}@__omp_outlined__
 ; AMDGPU-DISABLED1-SAME: (ptr noalias [[DOTGLOBAL_TID_:%.*]], ptr noalias [[DOTBOUND_TID_:%.*]]) #[[ATTR1]] {
@@ -679,10 +461,10 @@ define internal void @__omp_outlined__(ptr noalias %.global_tid., ptr noalias %.
 ; AMDGPU-DISABLED1-NEXT:    call void @spmd_amenable() #[[ATTR7:[0-9]+]]
 ; AMDGPU-DISABLED1-NEXT:    ret void
 ; AMDGPU-DISABLED1:       for.body:
-; AMDGPU-DISABLED1-NEXT:    [[TMP0:%.*]] = load i32, ptr [[DOTGLOBAL_TID_]], align 4, !tbaa [[TBAA18]]
+; AMDGPU-DISABLED1-NEXT:    [[TMP0:%.*]] = load i32, ptr [[DOTGLOBAL_TID_]], align 4, !tbaa [[TBAA12]]
 ; AMDGPU-DISABLED1-NEXT:    call void @__kmpc_parallel_51(ptr @[[GLOB1]], i32 [[TMP0]], i32 1, i32 -1, i32 -1, ptr @__omp_outlined__1, ptr @__omp_outlined__1_wrapper.ID, ptr [[CAPTURED_VARS_ADDRS]], i64 0)
 ; AMDGPU-DISABLED1-NEXT:    [[INC]] = add nsw i32 [[I_0]], 1
-; AMDGPU-DISABLED1-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP22:![0-9]+]]
+; AMDGPU-DISABLED1-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP16:![0-9]+]]
 ;
 ; AMDGPU-DISABLED2-LABEL: define {{[^@]+}}@__omp_outlined__
 ; AMDGPU-DISABLED2-SAME: (ptr noalias [[DOTGLOBAL_TID_:%.*]], ptr noalias [[DOTBOUND_TID_:%.*]]) #[[ATTR1]] {
@@ -697,10 +479,10 @@ define internal void @__omp_outlined__(ptr noalias %.global_tid., ptr noalias %.
 ; AMDGPU-DISABLED2-NEXT:    call void @spmd_amenable() #[[ATTR7:[0-9]+]]
 ; AMDGPU-DISABLED2-NEXT:    ret void
 ; AMDGPU-DISABLED2:       for.body:
-; AMDGPU-DISABLED2-NEXT:    [[TMP0:%.*]] = load i32, ptr [[DOTGLOBAL_TID_]], align 4, !tbaa [[TBAA18]]
-; AMDGPU-DISABLED2-NEXT:    call void @__kmpc_parallel_51(ptr @[[GLOB1]], i32 [[TMP0]], i32 1, i32 -1, i32 -1, ptr @__omp_outlined__1, ptr @__omp_outlined__1_wrapper.ID, ptr [[CAPTURED_VARS_ADDRS]], i64 0)
+; AMDGPU-DISABLED2-NEXT:    [[TMP0:%.*]] = load i32, ptr [[DOTGLOBAL_TID_]], align 4, !tbaa [[TBAA12]]
+; AMDGPU-DISABLED2-NEXT:    call void @__kmpc_parallel_51(ptr @[[GLOB1]], i32 [[TMP0]], i32 1, i32 -1, i32 -1, ptr @__omp_outlined__1, ptr @__omp_outlined__1_wrapper, ptr [[CAPTURED_VARS_ADDRS]], i64 0)
 ; AMDGPU-DISABLED2-NEXT:    [[INC]] = add nsw i32 [[I_0]], 1
-; AMDGPU-DISABLED2-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP22:![0-9]+]]
+; AMDGPU-DISABLED2-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP16:![0-9]+]]
 ;
 ; NVPTX-DISABLED1-LABEL: define {{[^@]+}}@__omp_outlined__
 ; NVPTX-DISABLED1-SAME: (ptr noalias [[DOTGLOBAL_TID_:%.*]], ptr noalias [[DOTBOUND_TID_:%.*]]) #[[ATTR1]] {
@@ -715,10 +497,10 @@ define internal void @__omp_outlined__(ptr noalias %.global_tid., ptr noalias %.
 ; NVPTX-DISABLED1-NEXT:    call void @spmd_amenable() #[[ATTR7:[0-9]+]]
 ; NVPTX-DISABLED1-NEXT:    ret void
 ; NVPTX-DISABLED1:       for.body:
-; NVPTX-DISABLED1-NEXT:    [[TMP0:%.*]] = load i32, ptr [[DOTGLOBAL_TID_]], align 4, !tbaa [[TBAA18]]
+; NVPTX-DISABLED1-NEXT:    [[TMP0:%.*]] = load i32, ptr [[DOTGLOBAL_TID_]], align 4, !tbaa [[TBAA12]]
 ; NVPTX-DISABLED1-NEXT:    call void @__kmpc_parallel_51(ptr @[[GLOB1]], i32 [[TMP0]], i32 1, i32 -1, i32 -1, ptr @__omp_outlined__1, ptr @__omp_outlined__1_wrapper.ID, ptr [[CAPTURED_VARS_ADDRS]], i64 0)
 ; NVPTX-DISABLED1-NEXT:    [[INC]] = add nsw i32 [[I_0]], 1
-; NVPTX-DISABLED1-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP22:![0-9]+]]
+; NVPTX-DISABLED1-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP16:![0-9]+]]
 ;
 ; NVPTX-DISABLED2-LABEL: define {{[^@]+}}@__omp_outlined__
 ; NVPTX-DISABLED2-SAME: (ptr noalias [[DOTGLOBAL_TID_:%.*]], ptr noalias [[DOTBOUND_TID_:%.*]]) #[[ATTR1]] {
@@ -733,45 +515,11 @@ define internal void @__omp_outlined__(ptr noalias %.global_tid., ptr noalias %.
 ; NVPTX-DISABLED2-NEXT:    call void @spmd_amenable() #[[ATTR7:[0-9]+]]
 ; NVPTX-DISABLED2-NEXT:    ret void
 ; NVPTX-DISABLED2:       for.body:
-; NVPTX-DISABLED2-NEXT:    [[TMP0:%.*]] = load i32, ptr [[DOTGLOBAL_TID_]], align 4, !tbaa [[TBAA18]]
-; NVPTX-DISABLED2-NEXT:    call void @__kmpc_parallel_51(ptr @[[GLOB1]], i32 [[TMP0]], i32 1, i32 -1, i32 -1, ptr @__omp_outlined__1, ptr @__omp_outlined__1_wrapper.ID, ptr [[CAPTURED_VARS_ADDRS]], i64 0)
+; NVPTX-DISABLED2-NEXT:    [[TMP0:%.*]] = load i32, ptr [[DOTGLOBAL_TID_]], align 4, !tbaa [[TBAA12]]
+; NVPTX-DISABLED2-NEXT:    call void @__kmpc_parallel_51(ptr @[[GLOB1]], i32 [[TMP0]], i32 1, i32 -1, i32 -1, ptr @__omp_outlined__1, ptr @__omp_outlined__1_wrapper, ptr [[CAPTURED_VARS_ADDRS]], i64 0)
 ; NVPTX-DISABLED2-NEXT:    [[INC]] = add nsw i32 [[I_0]], 1
-; NVPTX-DISABLED2-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP22:![0-9]+]]
+; NVPTX-DISABLED2-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP16:![0-9]+]]
 ;
-; AMDGPU-DISABLED-LABEL: define {{[^@]+}}@__omp_outlined__
-; AMDGPU-DISABLED-SAME: (ptr noalias [[DOTGLOBAL_TID_:%.*]], ptr noalias [[DOTBOUND_TID_:%.*]]) #[[ATTR1]] {
-; AMDGPU-DISABLED-NEXT:  entry:
-; AMDGPU-DISABLED-NEXT:    [[CAPTURED_VARS_ADDRS:%.*]] = alloca [0 x ptr], align 8
-; AMDGPU-DISABLED-NEXT:    br label [[FOR_COND:%.*]]
-; AMDGPU-DISABLED:       for.cond:
-; AMDGPU-DISABLED-NEXT:    [[I_0:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[INC:%.*]], [[FOR_BODY:%.*]] ]
-; AMDGPU-DISABLED-NEXT:    [[CMP:%.*]] = icmp slt i32 [[I_0]], 100
-; AMDGPU-DISABLED-NEXT:    br i1 [[CMP]], label [[FOR_BODY]], label [[FOR_COND_CLEANUP:%.*]]
-; AMDGPU-DISABLED:       for.cond.cleanup:
-; AMDGPU-DISABLED-NEXT:    call void @spmd_amenable() #[[ATTR7:[0-9]+]]
-; AMDGPU-DISABLED-NEXT:    ret void
-; AMDGPU-DISABLED:       for.body:
-; AMDGPU-DISABLED-NEXT:    [[TMP0:%.*]] = load i32, ptr [[DOTGLOBAL_TID_]], align 4, !tbaa [[TBAA18]]
-; AMDGPU-DISABLED-NEXT:    call void @__kmpc_parallel_51(ptr @[[GLOB1]], i32 [[TMP0]], i32 1, i32 -1, i32 -1, ptr @__omp_outlined__1, ptr @__omp_outlined__1_wrapper.ID, ptr [[CAPTURED_VARS_ADDRS]], i64 0)
-; AMDGPU-DISABLED-NEXT:    [[INC]] = add nsw i32 [[I_0]], 1
-; AMDGPU-DISABLED-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP22:![0-9]+]]
-; NVPTX-DISABLED-LABEL: define {{[^@]+}}@__omp_outlined__
-; NVPTX-DISABLED-SAME: (ptr noalias [[DOTGLOBAL_TID_:%.*]], ptr noalias [[DOTBOUND_TID_:%.*]]) #[[ATTR1]] {
-; NVPTX-DISABLED-NEXT:  entry:
-; NVPTX-DISABLED-NEXT:    [[CAPTURED_VARS_ADDRS:%.*]] = alloca [0 x ptr], align 8
-; NVPTX-DISABLED-NEXT:    br label [[FOR_COND:%.*]]
-; NVPTX-DISABLED:       for.cond:
-; NVPTX-DISABLED-NEXT:    [[I_0:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[INC:%.*]], [[FOR_BODY:%.*]] ]
-; NVPTX-DISABLED-NEXT:    [[CMP:%.*]] = icmp slt i32 [[I_0]], 100
-; NVPTX-DISABLED-NEXT:    br i1 [[CMP]], label [[FOR_BODY]], label [[FOR_COND_CLEANUP:%.*]]
-; NVPTX-DISABLED:       for.cond.cleanup:
-; NVPTX-DISABLED-NEXT:    call void @spmd_amenable() #[[ATTR7:[0-9]+]]
-; NVPTX-DISABLED-NEXT:    ret void
-; NVPTX-DISABLED:       for.body:
-; NVPTX-DISABLED-NEXT:    [[TMP0:%.*]] = load i32, ptr [[DOTGLOBAL_TID_]], align 4, !tbaa [[TBAA18]]
-; NVPTX-DISABLED-NEXT:    call void @__kmpc_parallel_51(ptr @[[GLOB1]], i32 [[TMP0]], i32 1, i32 -1, i32 -1, ptr @__omp_outlined__1, ptr @__omp_outlined__1_wrapper.ID, ptr [[CAPTURED_VARS_ADDRS]], i64 0)
-; NVPTX-DISABLED-NEXT:    [[INC]] = add nsw i32 [[I_0]], 1
-; NVPTX-DISABLED-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP22:![0-9]+]]
 entry:
   %captured_vars_addrs = alloca [0 x ptr], align 8
   br label %for.cond
@@ -830,16 +578,6 @@ define internal void @__omp_outlined__1(ptr noalias %.global_tid., ptr noalias %
 ; NVPTX-DISABLED2-NEXT:    call void @unknown() #[[ATTR8:[0-9]+]]
 ; NVPTX-DISABLED2-NEXT:    ret void
 ;
-; AMDGPU-DISABLED-LABEL: define {{[^@]+}}@__omp_outlined__1
-; AMDGPU-DISABLED-SAME: (ptr noalias [[DOTGLOBAL_TID_:%.*]], ptr noalias [[DOTBOUND_TID_:%.*]]) {
-; AMDGPU-DISABLED-NEXT:  entry:
-; AMDGPU-DISABLED-NEXT:    call void @unknown() #[[ATTR8:[0-9]+]]
-; AMDGPU-DISABLED-NEXT:    ret void
-; NVPTX-DISABLED-LABEL: define {{[^@]+}}@__omp_outlined__1
-; NVPTX-DISABLED-SAME: (ptr noalias [[DOTGLOBAL_TID_:%.*]], ptr noalias [[DOTBOUND_TID_:%.*]]) {
-; NVPTX-DISABLED-NEXT:  entry:
-; NVPTX-DISABLED-NEXT:    call void @unknown() #[[ATTR8:[0-9]+]]
-; NVPTX-DISABLED-NEXT:    ret void
 entry:
   call void @unknown() #11
   ret void
@@ -907,24 +645,6 @@ define internal void @__omp_outlined__1_wrapper(i16 zeroext %0, i32 %1) #3 {
 ; NVPTX-DISABLED2-NEXT:    call void @__omp_outlined__1(ptr [[DOTADDR1]], ptr [[DOTZERO_ADDR]]) #[[ATTR4]]
 ; NVPTX-DISABLED2-NEXT:    ret void
 ;
-; AMDGPU-DISABLED-LABEL: define {{[^@]+}}@__omp_outlined__1_wrapper
-; AMDGPU-DISABLED-SAME: (i16 zeroext [[TMP0:%.*]], i32 [[TMP1:%.*]]) #[[ATTR2:[0-9]+]] {
-; AMDGPU-DISABLED-NEXT:  entry:
-; AMDGPU-DISABLED-NEXT:    [[DOTADDR1:%.*]] = alloca i32, align 4
-; AMDGPU-DISABLED-NEXT:    [[DOTZERO_ADDR:%.*]] = alloca i32, align 4
-; AMDGPU-DISABLED-NEXT:    [[GLOBAL_ARGS:%.*]] = alloca ptr, align 8
-; AMDGPU-DISABLED-NEXT:    call void @__kmpc_get_shared_variables(ptr [[GLOBAL_ARGS]])
-; AMDGPU-DISABLED-NEXT:    call void @__omp_outlined__1(ptr [[DOTADDR1]], ptr [[DOTZERO_ADDR]]) #[[ATTR4]]
-; AMDGPU-DISABLED-NEXT:    ret void
-; NVPTX-DISABLED-LABEL: define {{[^@]+}}@__omp_outlined__1_wrapper
-; NVPTX-DISABLED-SAME: (i16 zeroext [[TMP0:%.*]], i32 [[TMP1:%.*]]) #[[ATTR2:[0-9]+]] {
-; NVPTX-DISABLED-NEXT:  entry:
-; NVPTX-DISABLED-NEXT:    [[DOTADDR1:%.*]] = alloca i32, align 4
-; NVPTX-DISABLED-NEXT:    [[DOTZERO_ADDR:%.*]] = alloca i32, align 4
-; NVPTX-DISABLED-NEXT:    [[GLOBAL_ARGS:%.*]] = alloca ptr, align 8
-; NVPTX-DISABLED-NEXT:    call void @__kmpc_get_shared_variables(ptr [[GLOBAL_ARGS]])
-; NVPTX-DISABLED-NEXT:    call void @__omp_outlined__1(ptr [[DOTADDR1]], ptr [[DOTZERO_ADDR]]) #[[ATTR4]]
-; NVPTX-DISABLED-NEXT:    ret void
 entry:
   %.addr1 = alloca i32, align 4
   %.zero.addr = alloca i32, align 4
@@ -937,7 +657,7 @@ entry:
 }
 
 ; Function Attrs: alwaysinline convergent norecurse nounwind
-define weak void @__omp_offloading_fd02_2044372e_sequential_loop_to_stack_var_l20() #0 {
+define weak ptx_kernel void @__omp_offloading_fd02_2044372e_sequential_loop_to_stack_var_l20() #0 {
 ; AMDGPU-LABEL: define {{[^@]+}}@__omp_offloading_fd02_2044372e_sequential_loop_to_stack_var_l20
 ; AMDGPU-SAME: () #[[ATTR0]] {
 ; AMDGPU-NEXT:  entry:
@@ -950,7 +670,7 @@ define weak void @__omp_offloading_fd02_2044372e_sequential_loop_to_stack_var_l2
 ; AMDGPU-NEXT:    ret void
 ; AMDGPU:       user_code.entry:
 ; AMDGPU-NEXT:    [[TMP1:%.*]] = call i32 @__kmpc_global_thread_num(ptr @[[GLOB1]]) #[[ATTR4]]
-; AMDGPU-NEXT:    store i32 [[TMP1]], ptr [[DOTTHREADID_TEMP_]], align 4, !tbaa [[TBAA18]]
+; AMDGPU-NEXT:    store i32 [[TMP1]], ptr [[DOTTHREADID_TEMP_]], align 4, !tbaa [[TBAA12]]
 ; AMDGPU-NEXT:    call void @__omp_outlined__2(ptr [[DOTTHREADID_TEMP_]], ptr [[DOTZERO_ADDR]]) #[[ATTR4]]
 ; AMDGPU-NEXT:    call void @__kmpc_target_deinit()
 ; AMDGPU-NEXT:    br label [[COMMON_RET]]
@@ -967,7 +687,7 @@ define weak void @__omp_offloading_fd02_2044372e_sequential_loop_to_stack_var_l2
 ; NVPTX-NEXT:    ret void
 ; NVPTX:       user_code.entry:
 ; NVPTX-NEXT:    [[TMP1:%.*]] = call i32 @__kmpc_global_thread_num(ptr @[[GLOB1]]) #[[ATTR4]]
-; NVPTX-NEXT:    store i32 [[TMP1]], ptr [[DOTTHREADID_TEMP_]], align 4, !tbaa [[TBAA18]]
+; NVPTX-NEXT:    store i32 [[TMP1]], ptr [[DOTTHREADID_TEMP_]], align 4, !tbaa [[TBAA12]]
 ; NVPTX-NEXT:    call void @__omp_outlined__2(ptr [[DOTTHREADID_TEMP_]], ptr [[DOTZERO_ADDR]]) #[[ATTR4]]
 ; NVPTX-NEXT:    call void @__kmpc_target_deinit()
 ; NVPTX-NEXT:    br label [[COMMON_RET]]
@@ -1018,7 +738,7 @@ define weak void @__omp_offloading_fd02_2044372e_sequential_loop_to_stack_var_l2
 ; AMDGPU-DISABLED1-NEXT:    ret void
 ; AMDGPU-DISABLED1:       user_code.entry:
 ; AMDGPU-DISABLED1-NEXT:    [[TMP1:%.*]] = call i32 @__kmpc_global_thread_num(ptr @[[GLOB1]]) #[[ATTR4]]
-; AMDGPU-DISABLED1-NEXT:    store i32 [[TMP1]], ptr [[DOTTHREADID_TEMP_]], align 4, !tbaa [[TBAA18]]
+; AMDGPU-DISABLED1-NEXT:    store i32 [[TMP1]], ptr [[DOTTHREADID_TEMP_]], align 4, !tbaa [[TBAA12]]
 ; AMDGPU-DISABLED1-NEXT:    call void @__omp_outlined__2(ptr [[DOTTHREADID_TEMP_]], ptr [[DOTZERO_ADDR]]) #[[ATTR4]]
 ; AMDGPU-DISABLED1-NEXT:    call void @__kmpc_target_deinit()
 ; AMDGPU-DISABLED1-NEXT:    br label [[COMMON_RET]]
@@ -1026,50 +746,16 @@ define weak void @__omp_offloading_fd02_2044372e_sequential_loop_to_stack_var_l2
 ; AMDGPU-DISABLED2-LABEL: define {{[^@]+}}@__omp_offloading_fd02_2044372e_sequential_loop_to_stack_var_l20
 ; AMDGPU-DISABLED2-SAME: () #[[ATTR0]] {
 ; AMDGPU-DISABLED2-NEXT:  entry:
-; AMDGPU-DISABLED2-NEXT:    [[WORKER_WORK_FN_ADDR:%.*]] = alloca ptr, align 8, addrspace(5)
 ; AMDGPU-DISABLED2-NEXT:    [[DOTZERO_ADDR:%.*]] = alloca i32, align 4
 ; AMDGPU-DISABLED2-NEXT:    [[DOTTHREADID_TEMP_:%.*]] = alloca i32, align 4
 ; AMDGPU-DISABLED2-NEXT:    [[TMP0:%.*]] = call i32 @__kmpc_target_init(ptr @__omp_offloading_fd02_2044372e_sequential_loop_to_stack_var_l20_kernel_environment, ptr null)
-; AMDGPU-DISABLED2-NEXT:    [[THREAD_IS_WORKER:%.*]] = icmp ne i32 [[TMP0]], -1
-; AMDGPU-DISABLED2-NEXT:    br i1 [[THREAD_IS_WORKER]], label [[IS_WORKER_CHECK:%.*]], label [[THREAD_USER_CODE_CHECK:%.*]]
-; AMDGPU-DISABLED2:       is_worker_check:
-; AMDGPU-DISABLED2-NEXT:    [[BLOCK_HW_SIZE:%.*]] = call i32 @__kmpc_get_hardware_num_threads_in_block()
-; AMDGPU-DISABLED2-NEXT:    [[WARP_SIZE:%.*]] = call i32 @__kmpc_get_warp_size()
-; AMDGPU-DISABLED2-NEXT:    [[BLOCK_SIZE:%.*]] = sub i32 [[BLOCK_HW_SIZE]], [[WARP_SIZE]]
-; AMDGPU-DISABLED2-NEXT:    [[THREAD_IS_MAIN_OR_WORKER:%.*]] = icmp slt i32 [[TMP0]], [[BLOCK_SIZE]]
-; AMDGPU-DISABLED2-NEXT:    br i1 [[THREAD_IS_MAIN_OR_WORKER]], label [[WORKER_STATE_MACHINE_BEGIN:%.*]], label [[WORKER_STATE_MACHINE_FINISHED:%.*]]
-; AMDGPU-DISABLED2:       worker_state_machine.begin:
-; AMDGPU-DISABLED2-NEXT:    call void @__kmpc_barrier_simple_generic(ptr @[[GLOB1]], i32 [[TMP0]])
-; AMDGPU-DISABLED2-NEXT:    [[WORKER_WORK_FN_ADDR_GENERIC:%.*]] = addrspacecast ptr addrspace(5) [[WORKER_WORK_FN_ADDR]] to ptr
-; AMDGPU-DISABLED2-NEXT:    [[WORKER_IS_ACTIVE:%.*]] = call i1 @__kmpc_kernel_parallel(ptr [[WORKER_WORK_FN_ADDR_GENERIC]])
-; AMDGPU-DISABLED2-NEXT:    [[WORKER_WORK_FN:%.*]] = load ptr, ptr [[WORKER_WORK_FN_ADDR_GENERIC]], align 8
-; AMDGPU-DISABLED2-NEXT:    [[WORKER_IS_DONE:%.*]] = icmp eq ptr [[WORKER_WORK_FN]], null
-; AMDGPU-DISABLED2-NEXT:    br i1 [[WORKER_IS_DONE]], label [[WORKER_STATE_MACHINE_FINISHED]], label [[WORKER_STATE_MACHINE_IS_ACTIVE_CHECK:%.*]]
-; AMDGPU-DISABLED2:       worker_state_machine.finished:
-; AMDGPU-DISABLED2-NEXT:    ret void
-; AMDGPU-DISABLED2:       worker_state_machine.is_active.check:
-; AMDGPU-DISABLED2-NEXT:    br i1 [[WORKER_IS_ACTIVE]], label [[WORKER_STATE_MACHINE_PARALLEL_REGION_CHECK:%.*]], label [[WORKER_STATE_MACHINE_DONE_BARRIER:%.*]]
-; AMDGPU-DISABLED2:       worker_state_machine.parallel_region.check:
-; AMDGPU-DISABLED2-NEXT:    br i1 true, label [[WORKER_STATE_MACHINE_PARALLEL_REGION_EXECUTE:%.*]], label [[WORKER_STATE_MACHINE_PARALLEL_REGION_CHECK1:%.*]]
-; AMDGPU-DISABLED2:       worker_state_machine.parallel_region.execute:
-; AMDGPU-DISABLED2-NEXT:    call void @__omp_outlined__3_wrapper(i16 0, i32 [[TMP0]])
-; AMDGPU-DISABLED2-NEXT:    br label [[WORKER_STATE_MACHINE_PARALLEL_REGION_END:%.*]]
-; AMDGPU-DISABLED2:       worker_state_machine.parallel_region.check1:
-; AMDGPU-DISABLED2-NEXT:    br label [[WORKER_STATE_MACHINE_PARALLEL_REGION_END]]
-; AMDGPU-DISABLED2:       worker_state_machine.parallel_region.end:
-; AMDGPU-DISABLED2-NEXT:    call void @__kmpc_kernel_end_parallel()
-; AMDGPU-DISABLED2-NEXT:    br label [[WORKER_STATE_MACHINE_DONE_BARRIER]]
-; AMDGPU-DISABLED2:       worker_state_machine.done.barrier:
-; AMDGPU-DISABLED2-NEXT:    call void @__kmpc_barrier_simple_generic(ptr @[[GLOB1]], i32 [[TMP0]])
-; AMDGPU-DISABLED2-NEXT:    br label [[WORKER_STATE_MACHINE_BEGIN]]
-; AMDGPU-DISABLED2:       thread.user_code.check:
 ; AMDGPU-DISABLED2-NEXT:    [[EXEC_USER_CODE:%.*]] = icmp eq i32 [[TMP0]], -1
 ; AMDGPU-DISABLED2-NEXT:    br i1 [[EXEC_USER_CODE]], label [[USER_CODE_ENTRY:%.*]], label [[COMMON_RET:%.*]]
 ; AMDGPU-DISABLED2:       common.ret:
 ; AMDGPU-DISABLED2-NEXT:    ret void
 ; AMDGPU-DISABLED2:       user_code.entry:
 ; AMDGPU-DISABLED2-NEXT:    [[TMP1:%.*]] = call i32 @__kmpc_global_thread_num(ptr @[[GLOB1]]) #[[ATTR4]]
-; AMDGPU-DISABLED2-NEXT:    store i32 [[TMP1]], ptr [[DOTTHREADID_TEMP_]], align 4, !tbaa [[TBAA18]]
+; AMDGPU-DISABLED2-NEXT:    store i32 [[TMP1]], ptr [[DOTTHREADID_TEMP_]], align 4, !tbaa [[TBAA12]]
 ; AMDGPU-DISABLED2-NEXT:    call void @__omp_outlined__2(ptr [[DOTTHREADID_TEMP_]], ptr [[DOTZERO_ADDR]]) #[[ATTR4]]
 ; AMDGPU-DISABLED2-NEXT:    call void @__kmpc_target_deinit()
 ; AMDGPU-DISABLED2-NEXT:    br label [[COMMON_RET]]
@@ -1119,7 +805,7 @@ define weak void @__omp_offloading_fd02_2044372e_sequential_loop_to_stack_var_l2
 ; NVPTX-DISABLED1-NEXT:    ret void
 ; NVPTX-DISABLED1:       user_code.entry:
 ; NVPTX-DISABLED1-NEXT:    [[TMP1:%.*]] = call i32 @__kmpc_global_thread_num(ptr @[[GLOB1]]) #[[ATTR4]]
-; NVPTX-DISABLED1-NEXT:    store i32 [[TMP1]], ptr [[DOTTHREADID_TEMP_]], align 4, !tbaa [[TBAA18]]
+; NVPTX-DISABLED1-NEXT:    store i32 [[TMP1]], ptr [[DOTTHREADID_TEMP_]], align 4, !tbaa [[TBAA12]]
 ; NVPTX-DISABLED1-NEXT:    call void @__omp_outlined__2(ptr [[DOTTHREADID_TEMP_]], ptr [[DOTZERO_ADDR]]) #[[ATTR4]]
 ; NVPTX-DISABLED1-NEXT:    call void @__kmpc_target_deinit()
 ; NVPTX-DISABLED1-NEXT:    br label [[COMMON_RET]]
@@ -1127,156 +813,20 @@ define weak void @__omp_offloading_fd02_2044372e_sequential_loop_to_stack_var_l2
 ; NVPTX-DISABLED2-LABEL: define {{[^@]+}}@__omp_offloading_fd02_2044372e_sequential_loop_to_stack_var_l20
 ; NVPTX-DISABLED2-SAME: () #[[ATTR0]] {
 ; NVPTX-DISABLED2-NEXT:  entry:
-; NVPTX-DISABLED2-NEXT:    [[WORKER_WORK_FN_ADDR:%.*]] = alloca ptr, align 8
 ; NVPTX-DISABLED2-NEXT:    [[DOTZERO_ADDR:%.*]] = alloca i32, align 4
 ; NVPTX-DISABLED2-NEXT:    [[DOTTHREADID_TEMP_:%.*]] = alloca i32, align 4
 ; NVPTX-DISABLED2-NEXT:    [[TMP0:%.*]] = call i32 @__kmpc_target_init(ptr @__omp_offloading_fd02_2044372e_sequential_loop_to_stack_var_l20_kernel_environment, ptr null)
-; NVPTX-DISABLED2-NEXT:    [[THREAD_IS_WORKER:%.*]] = icmp ne i32 [[TMP0]], -1
-; NVPTX-DISABLED2-NEXT:    br i1 [[THREAD_IS_WORKER]], label [[IS_WORKER_CHECK:%.*]], label [[THREAD_USER_CODE_CHECK:%.*]]
-; NVPTX-DISABLED2:       is_worker_check:
-; NVPTX-DISABLED2-NEXT:    [[BLOCK_HW_SIZE:%.*]] = call i32 @__kmpc_get_hardware_num_threads_in_block()
-; NVPTX-DISABLED2-NEXT:    [[WARP_SIZE:%.*]] = call i32 @__kmpc_get_warp_size()
-; NVPTX-DISABLED2-NEXT:    [[BLOCK_SIZE:%.*]] = sub i32 [[BLOCK_HW_SIZE]], [[WARP_SIZE]]
-; NVPTX-DISABLED2-NEXT:    [[THREAD_IS_MAIN_OR_WORKER:%.*]] = icmp slt i32 [[TMP0]], [[BLOCK_SIZE]]
-; NVPTX-DISABLED2-NEXT:    br i1 [[THREAD_IS_MAIN_OR_WORKER]], label [[WORKER_STATE_MACHINE_BEGIN:%.*]], label [[WORKER_STATE_MACHINE_FINISHED:%.*]]
-; NVPTX-DISABLED2:       worker_state_machine.begin:
-; NVPTX-DISABLED2-NEXT:    call void @__kmpc_barrier_simple_generic(ptr @[[GLOB1]], i32 [[TMP0]])
-; NVPTX-DISABLED2-NEXT:    [[WORKER_IS_ACTIVE:%.*]] = call i1 @__kmpc_kernel_parallel(ptr [[WORKER_WORK_FN_ADDR]])
-; NVPTX-DISABLED2-NEXT:    [[WORKER_WORK_FN:%.*]] = load ptr, ptr [[WORKER_WORK_FN_ADDR]], align 8
-; NVPTX-DISABLED2-NEXT:    [[WORKER_IS_DONE:%.*]] = icmp eq ptr [[WORKER_WORK_FN]], null
-; NVPTX-DISABLED2-NEXT:    br i1 [[WORKER_IS_DONE]], label [[WORKER_STATE_MACHINE_FINISHED]], label [[WORKER_STATE_MACHINE_IS_ACTIVE_CHECK:%.*]]
-; NVPTX-DISABLED2:       worker_state_machine.finished:
-; NVPTX-DISABLED2-NEXT:    ret void
-; NVPTX-DISABLED2:       worker_state_machine.is_active.check:
-; NVPTX-DISABLED2-NEXT:    br i1 [[WORKER_IS_ACTIVE]], label [[WORKER_STATE_MACHINE_PARALLEL_REGION_CHECK:%.*]], label [[WORKER_STATE_MACHINE_DONE_BARRIER:%.*]]
-; NVPTX-DISABLED2:       worker_state_machine.parallel_region.check:
-; NVPTX-DISABLED2-NEXT:    br i1 true, label [[WORKER_STATE_MACHINE_PARALLEL_REGION_EXECUTE:%.*]], label [[WORKER_STATE_MACHINE_PARALLEL_REGION_CHECK1:%.*]]
-; NVPTX-DISABLED2:       worker_state_machine.parallel_region.execute:
-; NVPTX-DISABLED2-NEXT:    call void @__omp_outlined__3_wrapper(i16 0, i32 [[TMP0]])
-; NVPTX-DISABLED2-NEXT:    br label [[WORKER_STATE_MACHINE_PARALLEL_REGION_END:%.*]]
-; NVPTX-DISABLED2:       worker_state_machine.parallel_region.check1:
-; NVPTX-DISABLED2-NEXT:    br label [[WORKER_STATE_MACHINE_PARALLEL_REGION_END]]
-; NVPTX-DISABLED2:       worker_state_machine.parallel_region.end:
-; NVPTX-DISABLED2-NEXT:    call void @__kmpc_kernel_end_parallel()
-; NVPTX-DISABLED2-NEXT:    br label [[WORKER_STATE_MACHINE_DONE_BARRIER]]
-; NVPTX-DISABLED2:       worker_state_machine.done.barrier:
-; NVPTX-DISABLED2-NEXT:    call void @__kmpc_barrier_simple_generic(ptr @[[GLOB1]], i32 [[TMP0]])
-; NVPTX-DISABLED2-NEXT:    br label [[WORKER_STATE_MACHINE_BEGIN]]
-; NVPTX-DISABLED2:       thread.user_code.check:
 ; NVPTX-DISABLED2-NEXT:    [[EXEC_USER_CODE:%.*]] = icmp eq i32 [[TMP0]], -1
 ; NVPTX-DISABLED2-NEXT:    br i1 [[EXEC_USER_CODE]], label [[USER_CODE_ENTRY:%.*]], label [[COMMON_RET:%.*]]
 ; NVPTX-DISABLED2:       common.ret:
 ; NVPTX-DISABLED2-NEXT:    ret void
 ; NVPTX-DISABLED2:       user_code.entry:
 ; NVPTX-DISABLED2-NEXT:    [[TMP1:%.*]] = call i32 @__kmpc_global_thread_num(ptr @[[GLOB1]]) #[[ATTR4]]
-; NVPTX-DISABLED2-NEXT:    store i32 [[TMP1]], ptr [[DOTTHREADID_TEMP_]], align 4, !tbaa [[TBAA18]]
+; NVPTX-DISABLED2-NEXT:    store i32 [[TMP1]], ptr [[DOTTHREADID_TEMP_]], align 4, !tbaa [[TBAA12]]
 ; NVPTX-DISABLED2-NEXT:    call void @__omp_outlined__2(ptr [[DOTTHREADID_TEMP_]], ptr [[DOTZERO_ADDR]]) #[[ATTR4]]
 ; NVPTX-DISABLED2-NEXT:    call void @__kmpc_target_deinit()
 ; NVPTX-DISABLED2-NEXT:    br label [[COMMON_RET]]
 ;
-; AMDGPU-DISABLED-LABEL: define {{[^@]+}}@__omp_offloading_fd02_2044372e_sequential_loop_to_stack_var_l20
-; AMDGPU-DISABLED-SAME: () #[[ATTR0]] {
-; AMDGPU-DISABLED-NEXT:  entry:
-; AMDGPU-DISABLED-NEXT:    [[WORKER_WORK_FN_ADDR:%.*]] = alloca ptr, align 8, addrspace(5)
-; AMDGPU-DISABLED-NEXT:    [[DOTZERO_ADDR:%.*]] = alloca i32, align 4
-; AMDGPU-DISABLED-NEXT:    [[DOTTHREADID_TEMP_:%.*]] = alloca i32, align 4
-; AMDGPU-DISABLED-NEXT:    [[TMP0:%.*]] = call i32 @__kmpc_target_init(ptr @__omp_offloading_fd02_2044372e_sequential_loop_to_stack_var_l20_kernel_environment)
-; AMDGPU-DISABLED-NEXT:    [[THREAD_IS_WORKER:%.*]] = icmp ne i32 [[TMP0]], -1
-; AMDGPU-DISABLED-NEXT:    br i1 [[THREAD_IS_WORKER]], label [[IS_WORKER_CHECK:%.*]], label [[THREAD_USER_CODE_CHECK:%.*]]
-; AMDGPU-DISABLED:       is_worker_check:
-; AMDGPU-DISABLED-NEXT:    [[BLOCK_HW_SIZE:%.*]] = call i32 @__kmpc_get_hardware_num_threads_in_block()
-; AMDGPU-DISABLED-NEXT:    [[WARP_SIZE:%.*]] = call i32 @__kmpc_get_warp_size()
-; AMDGPU-DISABLED-NEXT:    [[BLOCK_SIZE:%.*]] = sub i32 [[BLOCK_HW_SIZE]], [[WARP_SIZE]]
-; AMDGPU-DISABLED-NEXT:    [[THREAD_IS_MAIN_OR_WORKER:%.*]] = icmp slt i32 [[TMP0]], [[BLOCK_SIZE]]
-; AMDGPU-DISABLED-NEXT:    br i1 [[THREAD_IS_MAIN_OR_WORKER]], label [[WORKER_STATE_MACHINE_BEGIN:%.*]], label [[WORKER_STATE_MACHINE_FINISHED:%.*]]
-; AMDGPU-DISABLED:       worker_state_machine.begin:
-; AMDGPU-DISABLED-NEXT:    call void @__kmpc_barrier_simple_generic(ptr @[[GLOB1]], i32 [[TMP0]])
-; AMDGPU-DISABLED-NEXT:    [[WORKER_WORK_FN_ADDR_GENERIC:%.*]] = addrspacecast ptr addrspace(5) [[WORKER_WORK_FN_ADDR]] to ptr
-; AMDGPU-DISABLED-NEXT:    [[WORKER_IS_ACTIVE:%.*]] = call i1 @__kmpc_kernel_parallel(ptr [[WORKER_WORK_FN_ADDR_GENERIC]])
-; AMDGPU-DISABLED-NEXT:    [[WORKER_WORK_FN:%.*]] = load ptr, ptr [[WORKER_WORK_FN_ADDR_GENERIC]], align 8
-; AMDGPU-DISABLED-NEXT:    [[WORKER_IS_DONE:%.*]] = icmp eq ptr [[WORKER_WORK_FN]], null
-; AMDGPU-DISABLED-NEXT:    br i1 [[WORKER_IS_DONE]], label [[WORKER_STATE_MACHINE_FINISHED]], label [[WORKER_STATE_MACHINE_IS_ACTIVE_CHECK:%.*]]
-; AMDGPU-DISABLED:       worker_state_machine.finished:
-; AMDGPU-DISABLED-NEXT:    ret void
-; AMDGPU-DISABLED:       worker_state_machine.is_active.check:
-; AMDGPU-DISABLED-NEXT:    br i1 [[WORKER_IS_ACTIVE]], label [[WORKER_STATE_MACHINE_PARALLEL_REGION_CHECK:%.*]], label [[WORKER_STATE_MACHINE_DONE_BARRIER:%.*]]
-; AMDGPU-DISABLED:       worker_state_machine.parallel_region.check:
-; AMDGPU-DISABLED-NEXT:    [[WORKER_CHECK_PARALLEL_REGION:%.*]] = icmp eq ptr [[WORKER_WORK_FN]], @__omp_outlined__3_wrapper.ID
-; AMDGPU-DISABLED-NEXT:    br i1 [[WORKER_CHECK_PARALLEL_REGION]], label [[WORKER_STATE_MACHINE_PARALLEL_REGION_EXECUTE:%.*]], label [[WORKER_STATE_MACHINE_PARALLEL_REGION_FALLBACK_EXECUTE:%.*]]
-; AMDGPU-DISABLED:       worker_state_machine.parallel_region.execute:
-; AMDGPU-DISABLED-NEXT:    call void @__omp_outlined__3_wrapper(i16 0, i32 [[TMP0]])
-; AMDGPU-DISABLED-NEXT:    br label [[WORKER_STATE_MACHINE_PARALLEL_REGION_END:%.*]]
-; AMDGPU-DISABLED:       worker_state_machine.parallel_region.fallback.execute:
-; AMDGPU-DISABLED-NEXT:    call void [[WORKER_WORK_FN]](i16 0, i32 [[TMP0]])
-; AMDGPU-DISABLED-NEXT:    br label [[WORKER_STATE_MACHINE_PARALLEL_REGION_END]]
-; AMDGPU-DISABLED:       worker_state_machine.parallel_region.end:
-; AMDGPU-DISABLED-NEXT:    call void @__kmpc_kernel_end_parallel()
-; AMDGPU-DISABLED-NEXT:    br label [[WORKER_STATE_MACHINE_DONE_BARRIER]]
-; AMDGPU-DISABLED:       worker_state_machine.done.barrier:
-; AMDGPU-DISABLED-NEXT:    call void @__kmpc_barrier_simple_generic(ptr @[[GLOB1]], i32 [[TMP0]])
-; AMDGPU-DISABLED-NEXT:    br label [[WORKER_STATE_MACHINE_BEGIN]]
-; AMDGPU-DISABLED:       thread.user_code.check:
-; AMDGPU-DISABLED-NEXT:    [[EXEC_USER_CODE:%.*]] = icmp eq i32 [[TMP0]], -1
-; AMDGPU-DISABLED-NEXT:    br i1 [[EXEC_USER_CODE]], label [[USER_CODE_ENTRY:%.*]], label [[COMMON_RET:%.*]]
-; AMDGPU-DISABLED:       common.ret:
-; AMDGPU-DISABLED-NEXT:    ret void
-; AMDGPU-DISABLED:       user_code.entry:
-; AMDGPU-DISABLED-NEXT:    [[TMP1:%.*]] = call i32 @__kmpc_global_thread_num(ptr @[[GLOB1]]) #[[ATTR4]]
-; AMDGPU-DISABLED-NEXT:    store i32 [[TMP1]], ptr [[DOTTHREADID_TEMP_]], align 4, !tbaa [[TBAA18]]
-; AMDGPU-DISABLED-NEXT:    call void @__omp_outlined__2(ptr [[DOTTHREADID_TEMP_]], ptr [[DOTZERO_ADDR]]) #[[ATTR5]]
-; AMDGPU-DISABLED-NEXT:    call void @__kmpc_target_deinit()
-; AMDGPU-DISABLED-NEXT:    br label [[COMMON_RET]]
-; NVPTX-DISABLED-LABEL: define {{[^@]+}}@__omp_offloading_fd02_2044372e_sequential_loop_to_stack_var_l20
-; NVPTX-DISABLED-SAME: () #[[ATTR0]] {
-; NVPTX-DISABLED-NEXT:  entry:
-; NVPTX-DISABLED-NEXT:    [[WORKER_WORK_FN_ADDR:%.*]] = alloca ptr, align 8
-; NVPTX-DISABLED-NEXT:    [[DOTZERO_ADDR:%.*]] = alloca i32, align 4
-; NVPTX-DISABLED-NEXT:    [[DOTTHREADID_TEMP_:%.*]] = alloca i32, align 4
-; NVPTX-DISABLED-NEXT:    [[TMP0:%.*]] = call i32 @__kmpc_target_init(ptr @__omp_offloading_fd02_2044372e_sequential_loop_to_stack_var_l20_kernel_environment)
-; NVPTX-DISABLED-NEXT:    [[THREAD_IS_WORKER:%.*]] = icmp ne i32 [[TMP0]], -1
-; NVPTX-DISABLED-NEXT:    br i1 [[THREAD_IS_WORKER]], label [[IS_WORKER_CHECK:%.*]], label [[THREAD_USER_CODE_CHECK:%.*]]
-; NVPTX-DISABLED:       is_worker_check:
-; NVPTX-DISABLED-NEXT:    [[BLOCK_HW_SIZE:%.*]] = call i32 @__kmpc_get_hardware_num_threads_in_block()
-; NVPTX-DISABLED-NEXT:    [[WARP_SIZE:%.*]] = call i32 @__kmpc_get_warp_size()
-; NVPTX-DISABLED-NEXT:    [[BLOCK_SIZE:%.*]] = sub i32 [[BLOCK_HW_SIZE]], [[WARP_SIZE]]
-; NVPTX-DISABLED-NEXT:    [[THREAD_IS_MAIN_OR_WORKER:%.*]] = icmp slt i32 [[TMP0]], [[BLOCK_SIZE]]
-; NVPTX-DISABLED-NEXT:    br i1 [[THREAD_IS_MAIN_OR_WORKER]], label [[WORKER_STATE_MACHINE_BEGIN:%.*]], label [[WORKER_STATE_MACHINE_FINISHED:%.*]]
-; NVPTX-DISABLED:       worker_state_machine.begin:
-; NVPTX-DISABLED-NEXT:    call void @__kmpc_barrier_simple_generic(ptr @[[GLOB1]], i32 [[TMP0]])
-; NVPTX-DISABLED-NEXT:    [[WORKER_IS_ACTIVE:%.*]] = call i1 @__kmpc_kernel_parallel(ptr [[WORKER_WORK_FN_ADDR]])
-; NVPTX-DISABLED-NEXT:    [[WORKER_WORK_FN:%.*]] = load ptr, ptr [[WORKER_WORK_FN_ADDR]], align 8
-; NVPTX-DISABLED-NEXT:    [[WORKER_IS_DONE:%.*]] = icmp eq ptr [[WORKER_WORK_FN]], null
-; NVPTX-DISABLED-NEXT:    br i1 [[WORKER_IS_DONE]], label [[WORKER_STATE_MACHINE_FINISHED]], label [[WORKER_STATE_MACHINE_IS_ACTIVE_CHECK:%.*]]
-; NVPTX-DISABLED:       worker_state_machine.finished:
-; NVPTX-DISABLED-NEXT:    ret void
-; NVPTX-DISABLED:       worker_state_machine.is_active.check:
-; NVPTX-DISABLED-NEXT:    br i1 [[WORKER_IS_ACTIVE]], label [[WORKER_STATE_MACHINE_PARALLEL_REGION_CHECK:%.*]], label [[WORKER_STATE_MACHINE_DONE_BARRIER:%.*]]
-; NVPTX-DISABLED:       worker_state_machine.parallel_region.check:
-; NVPTX-DISABLED-NEXT:    [[WORKER_CHECK_PARALLEL_REGION:%.*]] = icmp eq ptr [[WORKER_WORK_FN]], @__omp_outlined__3_wrapper.ID
-; NVPTX-DISABLED-NEXT:    br i1 [[WORKER_CHECK_PARALLEL_REGION]], label [[WORKER_STATE_MACHINE_PARALLEL_REGION_EXECUTE:%.*]], label [[WORKER_STATE_MACHINE_PARALLEL_REGION_FALLBACK_EXECUTE:%.*]]
-; NVPTX-DISABLED:       worker_state_machine.parallel_region.execute:
-; NVPTX-DISABLED-NEXT:    call void @__omp_outlined__3_wrapper(i16 0, i32 [[TMP0]])
-; NVPTX-DISABLED-NEXT:    br label [[WORKER_STATE_MACHINE_PARALLEL_REGION_END:%.*]]
-; NVPTX-DISABLED:       worker_state_machine.parallel_region.fallback.execute:
-; NVPTX-DISABLED-NEXT:    call void [[WORKER_WORK_FN]](i16 0, i32 [[TMP0]])
-; NVPTX-DISABLED-NEXT:    br label [[WORKER_STATE_MACHINE_PARALLEL_REGION_END]]
-; NVPTX-DISABLED:       worker_state_machine.parallel_region.end:
-; NVPTX-DISABLED-NEXT:    call void @__kmpc_kernel_end_parallel()
-; NVPTX-DISABLED-NEXT:    br label [[WORKER_STATE_MACHINE_DONE_BARRIER]]
-; NVPTX-DISABLED:       worker_state_machine.done.barrier:
-; NVPTX-DISABLED-NEXT:    call void @__kmpc_barrier_simple_generic(ptr @[[GLOB1]], i32 [[TMP0]])
-; NVPTX-DISABLED-NEXT:    br label [[WORKER_STATE_MACHINE_BEGIN]]
-; NVPTX-DISABLED:       thread.user_code.check:
-; NVPTX-DISABLED-NEXT:    [[EXEC_USER_CODE:%.*]] = icmp eq i32 [[TMP0]], -1
-; NVPTX-DISABLED-NEXT:    br i1 [[EXEC_USER_CODE]], label [[USER_CODE_ENTRY:%.*]], label [[COMMON_RET:%.*]]
-; NVPTX-DISABLED:       common.ret:
-; NVPTX-DISABLED-NEXT:    ret void
-; NVPTX-DISABLED:       user_code.entry:
-; NVPTX-DISABLED-NEXT:    [[TMP1:%.*]] = call i32 @__kmpc_global_thread_num(ptr @[[GLOB1]]) #[[ATTR4]]
-; NVPTX-DISABLED-NEXT:    store i32 [[TMP1]], ptr [[DOTTHREADID_TEMP_]], align 4, !tbaa [[TBAA18]]
-; NVPTX-DISABLED-NEXT:    call void @__omp_outlined__2(ptr [[DOTTHREADID_TEMP_]], ptr [[DOTZERO_ADDR]]) #[[ATTR5]]
-; NVPTX-DISABLED-NEXT:    call void @__kmpc_target_deinit()
-; NVPTX-DISABLED-NEXT:    br label [[COMMON_RET]]
 entry:
   %.zero.addr = alloca i32, align 4
   %.threadid_temp. = alloca i32, align 4
@@ -1304,7 +854,7 @@ define internal void @__omp_outlined__2(ptr noalias %.global_tid., ptr noalias %
 ; AMDGPU-NEXT:    [[X_H2S:%.*]] = alloca i8, i64 4, align 4, addrspace(5)
 ; AMDGPU-NEXT:    [[CAPTURED_VARS_ADDRS:%.*]] = alloca [0 x ptr], align 8
 ; AMDGPU-NEXT:    [[MALLOC_CAST:%.*]] = addrspacecast ptr addrspace(5) [[X_H2S]] to ptr
-; AMDGPU-NEXT:    call void @use(ptr nocapture [[MALLOC_CAST]]) #[[ATTR7]]
+; AMDGPU-NEXT:    call void @use(ptr captures(none) [[MALLOC_CAST]]) #[[ATTR7]]
 ; AMDGPU-NEXT:    br label [[FOR_COND:%.*]]
 ; AMDGPU:       for.cond:
 ; AMDGPU-NEXT:    [[I_0:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[INC:%.*]], [[FOR_BODY:%.*]] ]
@@ -1314,17 +864,17 @@ define internal void @__omp_outlined__2(ptr noalias %.global_tid., ptr noalias %
 ; AMDGPU-NEXT:    call void @spmd_amenable() #[[ATTR7]]
 ; AMDGPU-NEXT:    ret void
 ; AMDGPU:       for.body:
-; AMDGPU-NEXT:    [[TMP0:%.*]] = load i32, ptr [[DOTGLOBAL_TID_]], align 4, !tbaa [[TBAA18]]
+; AMDGPU-NEXT:    [[TMP0:%.*]] = load i32, ptr [[DOTGLOBAL_TID_]], align 4, !tbaa [[TBAA12]]
 ; AMDGPU-NEXT:    call void @__kmpc_parallel_51(ptr @[[GLOB1]], i32 [[TMP0]], i32 1, i32 -1, i32 -1, ptr @__omp_outlined__3, ptr @__omp_outlined__3_wrapper, ptr [[CAPTURED_VARS_ADDRS]], i64 0)
 ; AMDGPU-NEXT:    [[INC]] = add nsw i32 [[I_0]], 1
-; AMDGPU-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP25:![0-9]+]]
+; AMDGPU-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP19:![0-9]+]]
 ;
 ; NVPTX-LABEL: define {{[^@]+}}@__omp_outlined__2
 ; NVPTX-SAME: (ptr noalias [[DOTGLOBAL_TID_:%.*]], ptr noalias [[DOTBOUND_TID_:%.*]]) #[[ATTR1]] {
 ; NVPTX-NEXT:  entry:
 ; NVPTX-NEXT:    [[X_H2S:%.*]] = alloca i8, i64 4, align 4
 ; NVPTX-NEXT:    [[CAPTURED_VARS_ADDRS:%.*]] = alloca [0 x ptr], align 8
-; NVPTX-NEXT:    call void @use(ptr nocapture [[X_H2S]]) #[[ATTR7]]
+; NVPTX-NEXT:    call void @use(ptr captures(none) [[X_H2S]]) #[[ATTR7]]
 ; NVPTX-NEXT:    br label [[FOR_COND:%.*]]
 ; NVPTX:       for.cond:
 ; NVPTX-NEXT:    [[I_0:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[INC:%.*]], [[FOR_BODY:%.*]] ]
@@ -1334,10 +884,10 @@ define internal void @__omp_outlined__2(ptr noalias %.global_tid., ptr noalias %
 ; NVPTX-NEXT:    call void @spmd_amenable() #[[ATTR7]]
 ; NVPTX-NEXT:    ret void
 ; NVPTX:       for.body:
-; NVPTX-NEXT:    [[TMP0:%.*]] = load i32, ptr [[DOTGLOBAL_TID_]], align 4, !tbaa [[TBAA18]]
+; NVPTX-NEXT:    [[TMP0:%.*]] = load i32, ptr [[DOTGLOBAL_TID_]], align 4, !tbaa [[TBAA12]]
 ; NVPTX-NEXT:    call void @__kmpc_parallel_51(ptr @[[GLOB1]], i32 [[TMP0]], i32 1, i32 -1, i32 -1, ptr @__omp_outlined__3, ptr @__omp_outlined__3_wrapper, ptr [[CAPTURED_VARS_ADDRS]], i64 0)
 ; NVPTX-NEXT:    [[INC]] = add nsw i32 [[I_0]], 1
-; NVPTX-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP25:![0-9]+]]
+; NVPTX-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP19:![0-9]+]]
 ;
 ; AMDGPU-DISABLED1-LABEL: define {{[^@]+}}@__omp_outlined__2
 ; AMDGPU-DISABLED1-SAME: (ptr noalias [[DOTGLOBAL_TID_:%.*]], ptr noalias [[DOTBOUND_TID_:%.*]]) #[[ATTR1]] {
@@ -1345,7 +895,7 @@ define internal void @__omp_outlined__2(ptr noalias %.global_tid., ptr noalias %
 ; AMDGPU-DISABLED1-NEXT:    [[X_H2S:%.*]] = alloca i8, i64 4, align 4, addrspace(5)
 ; AMDGPU-DISABLED1-NEXT:    [[CAPTURED_VARS_ADDRS:%.*]] = alloca [0 x ptr], align 8
 ; AMDGPU-DISABLED1-NEXT:    [[MALLOC_CAST:%.*]] = addrspacecast ptr addrspace(5) [[X_H2S]] to ptr
-; AMDGPU-DISABLED1-NEXT:    call void @use(ptr nocapture [[MALLOC_CAST]]) #[[ATTR7]]
+; AMDGPU-DISABLED1-NEXT:    call void @use(ptr captures(none) [[MALLOC_CAST]]) #[[ATTR7]]
 ; AMDGPU-DISABLED1-NEXT:    br label [[FOR_COND:%.*]]
 ; AMDGPU-DISABLED1:       for.cond:
 ; AMDGPU-DISABLED1-NEXT:    [[I_0:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[INC:%.*]], [[FOR_BODY:%.*]] ]
@@ -1355,10 +905,10 @@ define internal void @__omp_outlined__2(ptr noalias %.global_tid., ptr noalias %
 ; AMDGPU-DISABLED1-NEXT:    call void @spmd_amenable() #[[ATTR7]]
 ; AMDGPU-DISABLED1-NEXT:    ret void
 ; AMDGPU-DISABLED1:       for.body:
-; AMDGPU-DISABLED1-NEXT:    [[TMP0:%.*]] = load i32, ptr [[DOTGLOBAL_TID_]], align 4, !tbaa [[TBAA18]]
+; AMDGPU-DISABLED1-NEXT:    [[TMP0:%.*]] = load i32, ptr [[DOTGLOBAL_TID_]], align 4, !tbaa [[TBAA12]]
 ; AMDGPU-DISABLED1-NEXT:    call void @__kmpc_parallel_51(ptr @[[GLOB1]], i32 [[TMP0]], i32 1, i32 -1, i32 -1, ptr @__omp_outlined__3, ptr @__omp_outlined__3_wrapper.ID, ptr [[CAPTURED_VARS_ADDRS]], i64 0)
 ; AMDGPU-DISABLED1-NEXT:    [[INC]] = add nsw i32 [[I_0]], 1
-; AMDGPU-DISABLED1-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP25:![0-9]+]]
+; AMDGPU-DISABLED1-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP19:![0-9]+]]
 ;
 ; AMDGPU-DISABLED2-LABEL: define {{[^@]+}}@__omp_outlined__2
 ; AMDGPU-DISABLED2-SAME: (ptr noalias [[DOTGLOBAL_TID_:%.*]], ptr noalias [[DOTBOUND_TID_:%.*]]) #[[ATTR1]] {
@@ -1366,7 +916,7 @@ define internal void @__omp_outlined__2(ptr noalias %.global_tid., ptr noalias %
 ; AMDGPU-DISABLED2-NEXT:    [[X_H2S:%.*]] = alloca i8, i64 4, align 4, addrspace(5)
 ; AMDGPU-DISABLED2-NEXT:    [[CAPTURED_VARS_ADDRS:%.*]] = alloca [0 x ptr], align 8
 ; AMDGPU-DISABLED2-NEXT:    [[MALLOC_CAST:%.*]] = addrspacecast ptr addrspace(5) [[X_H2S]] to ptr
-; AMDGPU-DISABLED2-NEXT:    call void @use(ptr nocapture [[MALLOC_CAST]]) #[[ATTR7]]
+; AMDGPU-DISABLED2-NEXT:    call void @use(ptr captures(none) [[MALLOC_CAST]]) #[[ATTR7]]
 ; AMDGPU-DISABLED2-NEXT:    br label [[FOR_COND:%.*]]
 ; AMDGPU-DISABLED2:       for.cond:
 ; AMDGPU-DISABLED2-NEXT:    [[I_0:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[INC:%.*]], [[FOR_BODY:%.*]] ]
@@ -1376,17 +926,17 @@ define internal void @__omp_outlined__2(ptr noalias %.global_tid., ptr noalias %
 ; AMDGPU-DISABLED2-NEXT:    call void @spmd_amenable() #[[ATTR7]]
 ; AMDGPU-DISABLED2-NEXT:    ret void
 ; AMDGPU-DISABLED2:       for.body:
-; AMDGPU-DISABLED2-NEXT:    [[TMP0:%.*]] = load i32, ptr [[DOTGLOBAL_TID_]], align 4, !tbaa [[TBAA18]]
-; AMDGPU-DISABLED2-NEXT:    call void @__kmpc_parallel_51(ptr @[[GLOB1]], i32 [[TMP0]], i32 1, i32 -1, i32 -1, ptr @__omp_outlined__3, ptr @__omp_outlined__3_wrapper.ID, ptr [[CAPTURED_VARS_ADDRS]], i64 0)
+; AMDGPU-DISABLED2-NEXT:    [[TMP0:%.*]] = load i32, ptr [[DOTGLOBAL_TID_]], align 4, !tbaa [[TBAA12]]
+; AMDGPU-DISABLED2-NEXT:    call void @__kmpc_parallel_51(ptr @[[GLOB1]], i32 [[TMP0]], i32 1, i32 -1, i32 -1, ptr @__omp_outlined__3, ptr @__omp_outlined__3_wrapper, ptr [[CAPTURED_VARS_ADDRS]], i64 0)
 ; AMDGPU-DISABLED2-NEXT:    [[INC]] = add nsw i32 [[I_0]], 1
-; AMDGPU-DISABLED2-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP25:![0-9]+]]
+; AMDGPU-DISABLED2-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP19:![0-9]+]]
 ;
 ; NVPTX-DISABLED1-LABEL: define {{[^@]+}}@__omp_outlined__2
 ; NVPTX-DISABLED1-SAME: (ptr noalias [[DOTGLOBAL_TID_:%.*]], ptr noalias [[DOTBOUND_TID_:%.*]]) #[[ATTR1]] {
 ; NVPTX-DISABLED1-NEXT:  entry:
 ; NVPTX-DISABLED1-NEXT:    [[X_H2S:%.*]] = alloca i8, i64 4, align 4
 ; NVPTX-DISABLED1-NEXT:    [[CAPTURED_VARS_ADDRS:%.*]] = alloca [0 x ptr], align 8
-; NVPTX-DISABLED1-NEXT:    call void @use(ptr nocapture [[X_H2S]]) #[[ATTR7]]
+; NVPTX-DISABLED1-NEXT:    call void @use(ptr captures(none) [[X_H2S]]) #[[ATTR7]]
 ; NVPTX-DISABLED1-NEXT:    br label [[FOR_COND:%.*]]
 ; NVPTX-DISABLED1:       for.cond:
 ; NVPTX-DISABLED1-NEXT:    [[I_0:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[INC:%.*]], [[FOR_BODY:%.*]] ]
@@ -1396,17 +946,17 @@ define internal void @__omp_outlined__2(ptr noalias %.global_tid., ptr noalias %
 ; NVPTX-DISABLED1-NEXT:    call void @spmd_amenable() #[[ATTR7]]
 ; NVPTX-DISABLED1-NEXT:    ret void
 ; NVPTX-DISABLED1:       for.body:
-; NVPTX-DISABLED1-NEXT:    [[TMP0:%.*]] = load i32, ptr [[DOTGLOBAL_TID_]], align 4, !tbaa [[TBAA18]]
+; NVPTX-DISABLED1-NEXT:    [[TMP0:%.*]] = load i32, ptr [[DOTGLOBAL_TID_]], align 4, !tbaa [[TBAA12]]
 ; NVPTX-DISABLED1-NEXT:    call void @__kmpc_parallel_51(ptr @[[GLOB1]], i32 [[TMP0]], i32 1, i32 -1, i32 -1, ptr @__omp_outlined__3, ptr @__omp_outlined__3_wrapper.ID, ptr [[CAPTURED_VARS_ADDRS]], i64 0)
 ; NVPTX-DISABLED1-NEXT:    [[INC]] = add nsw i32 [[I_0]], 1
-; NVPTX-DISABLED1-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP25:![0-9]+]]
+; NVPTX-DISABLED1-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP19:![0-9]+]]
 ;
 ; NVPTX-DISABLED2-LABEL: define {{[^@]+}}@__omp_outlined__2
 ; NVPTX-DISABLED2-SAME: (ptr noalias [[DOTGLOBAL_TID_:%.*]], ptr noalias [[DOTBOUND_TID_:%.*]]) #[[ATTR1]] {
 ; NVPTX-DISABLED2-NEXT:  entry:
 ; NVPTX-DISABLED2-NEXT:    [[X_H2S:%.*]] = alloca i8, i64 4, align 4
 ; NVPTX-DISABLED2-NEXT:    [[CAPTURED_VARS_ADDRS:%.*]] = alloca [0 x ptr], align 8
-; NVPTX-DISABLED2-NEXT:    call void @use(ptr nocapture [[X_H2S]]) #[[ATTR7]]
+; NVPTX-DISABLED2-NEXT:    call void @use(ptr captures(none) [[X_H2S]]) #[[ATTR7]]
 ; NVPTX-DISABLED2-NEXT:    br label [[FOR_COND:%.*]]
 ; NVPTX-DISABLED2:       for.cond:
 ; NVPTX-DISABLED2-NEXT:    [[I_0:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[INC:%.*]], [[FOR_BODY:%.*]] ]
@@ -1416,50 +966,11 @@ define internal void @__omp_outlined__2(ptr noalias %.global_tid., ptr noalias %
 ; NVPTX-DISABLED2-NEXT:    call void @spmd_amenable() #[[ATTR7]]
 ; NVPTX-DISABLED2-NEXT:    ret void
 ; NVPTX-DISABLED2:       for.body:
-; NVPTX-DISABLED2-NEXT:    [[TMP0:%.*]] = load i32, ptr [[DOTGLOBAL_TID_]], align 4, !tbaa [[TBAA18]]
-; NVPTX-DISABLED2-NEXT:    call void @__kmpc_parallel_51(ptr @[[GLOB1]], i32 [[TMP0]], i32 1, i32 -1, i32 -1, ptr @__omp_outlined__3, ptr @__omp_outlined__3_wrapper.ID, ptr [[CAPTURED_VARS_ADDRS]], i64 0)
+; NVPTX-DISABLED2-NEXT:    [[TMP0:%.*]] = load i32, ptr [[DOTGLOBAL_TID_]], align 4, !tbaa [[TBAA12]]
+; NVPTX-DISABLED2-NEXT:    call void @__kmpc_parallel_51(ptr @[[GLOB1]], i32 [[TMP0]], i32 1, i32 -1, i32 -1, ptr @__omp_outlined__3, ptr @__omp_outlined__3_wrapper, ptr [[CAPTURED_VARS_ADDRS]], i64 0)
 ; NVPTX-DISABLED2-NEXT:    [[INC]] = add nsw i32 [[I_0]], 1
-; NVPTX-DISABLED2-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP25:![0-9]+]]
-;
-; AMDGPU-DISABLED-LABEL: define {{[^@]+}}@__omp_outlined__2
-; AMDGPU-DISABLED-SAME: (ptr noalias [[DOTGLOBAL_TID_:%.*]], ptr noalias [[DOTBOUND_TID_:%.*]]) #[[ATTR1]] {
-; AMDGPU-DISABLED-NEXT:  entry:
-; AMDGPU-DISABLED-NEXT:    [[X_H2S:%.*]] = alloca i8, i64 4, align 4, addrspace(5)
-; AMDGPU-DISABLED-NEXT:    [[CAPTURED_VARS_ADDRS:%.*]] = alloca [0 x ptr], align 8
-; AMDGPU-DISABLED-NEXT:    [[MALLOC_CAST:%.*]] = addrspacecast ptr addrspace(5) [[X_H2S]] to ptr
-; AMDGPU-DISABLED-NEXT:    call void @use(ptr nocapture [[MALLOC_CAST]]) #[[ATTR7]]
-; AMDGPU-DISABLED-NEXT:    br label [[FOR_COND:%.*]]
-; AMDGPU-DISABLED:       for.cond:
-; AMDGPU-DISABLED-NEXT:    [[I_0:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[INC:%.*]], [[FOR_BODY:%.*]] ]
-; AMDGPU-DISABLED-NEXT:    [[CMP:%.*]] = icmp slt i32 [[I_0]], 100
-; AMDGPU-DISABLED-NEXT:    br i1 [[CMP]], label [[FOR_BODY]], label [[FOR_COND_CLEANUP:%.*]]
-; AMDGPU-DISABLED:       for.cond.cleanup:
-; AMDGPU-DISABLED-NEXT:    call void @spmd_amenable() #[[ATTR7]]
-; AMDGPU-DISABLED-NEXT:    ret void
-; AMDGPU-DISABLED:       for.body:
-; AMDGPU-DISABLED-NEXT:    [[TMP0:%.*]] = load i32, ptr [[DOTGLOBAL_TID_]], align 4, !tbaa [[TBAA18]]
-; AMDGPU-DISABLED-NEXT:    call void @__kmpc_parallel_51(ptr @[[GLOB1]], i32 [[TMP0]], i32 1, i32 -1, i32 -1, ptr @__omp_outlined__3, ptr @__omp_outlined__3_wrapper.ID, ptr [[CAPTURED_VARS_ADDRS]], i64 0)
-; AMDGPU-DISABLED-NEXT:    [[INC]] = add nsw i32 [[I_0]], 1
-; AMDGPU-DISABLED-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP25:![0-9]+]]
-; NVPTX-DISABLED-LABEL: define {{[^@]+}}@__omp_outlined__2
-; NVPTX-DISABLED-SAME: (ptr noalias [[DOTGLOBAL_TID_:%.*]], ptr noalias [[DOTBOUND_TID_:%.*]]) #[[ATTR1]] {
-; NVPTX-DISABLED-NEXT:  entry:
-; NVPTX-DISABLED-NEXT:    [[X_H2S:%.*]] = alloca i8, i64 4, align 4
-; NVPTX-DISABLED-NEXT:    [[CAPTURED_VARS_ADDRS:%.*]] = alloca [0 x ptr], align 8
-; NVPTX-DISABLED-NEXT:    call void @use(ptr nocapture [[X_H2S]]) #[[ATTR7]]
-; NVPTX-DISABLED-NEXT:    br label [[FOR_COND:%.*]]
-; NVPTX-DISABLED:       for.cond:
-; NVPTX-DISABLED-NEXT:    [[I_0:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[INC:%.*]], [[FOR_BODY:%.*]] ]
-; NVPTX-DISABLED-NEXT:    [[CMP:%.*]] = icmp slt i32 [[I_0]], 100
-; NVPTX-DISABLED-NEXT:    br i1 [[CMP]], label [[FOR_BODY]], label [[FOR_COND_CLEANUP:%.*]]
-; NVPTX-DISABLED:       for.cond.cleanup:
-; NVPTX-DISABLED-NEXT:    call void @spmd_amenable() #[[ATTR7]]
-; NVPTX-DISABLED-NEXT:    ret void
-; NVPTX-DISABLED:       for.body:
-; NVPTX-DISABLED-NEXT:    [[TMP0:%.*]] = load i32, ptr [[DOTGLOBAL_TID_]], align 4, !tbaa [[TBAA18]]
-; NVPTX-DISABLED-NEXT:    call void @__kmpc_parallel_51(ptr @[[GLOB1]], i32 [[TMP0]], i32 1, i32 -1, i32 -1, ptr @__omp_outlined__3, ptr @__omp_outlined__3_wrapper.ID, ptr [[CAPTURED_VARS_ADDRS]], i64 0)
-; NVPTX-DISABLED-NEXT:    [[INC]] = add nsw i32 [[I_0]], 1
-; NVPTX-DISABLED-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP25:![0-9]+]]
+; NVPTX-DISABLED2-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP19:![0-9]+]]
+;
 entry:
   %captured_vars_addrs = alloca [0 x ptr], align 8
   %x = call align 4 ptr @__kmpc_alloc_shared(i64 4)
@@ -1520,16 +1031,6 @@ define internal void @__omp_outlined__3(ptr noalias %.global_tid., ptr noalias %
 ; NVPTX-DISABLED2-NEXT:    call void @unknown() #[[ATTR8]]
 ; NVPTX-DISABLED2-NEXT:    ret void
 ;
-; AMDGPU-DISABLED-LABEL: define {{[^@]+}}@__omp_outlined__3
-; AMDGPU-DISABLED-SAME: (ptr noalias [[DOTGLOBAL_TID_:%.*]], ptr noalias [[DOTBOUND_TID_:%.*]]) {
-; AMDGPU-DISABLED-NEXT:  entry:
-; AMDGPU-DISABLED-NEXT:    call void @unknown() #[[ATTR8]]
-; AMDGPU-DISABLED-NEXT:    ret void
-; NVPTX-DISABLED-LABEL: define {{[^@]+}}@__omp_outlined__3
-; NVPTX-DISABLED-SAME: (ptr noalias [[DOTGLOBAL_TID_:%.*]], ptr noalias [[DOTBOUND_TID_:%.*]]) {
-; NVPTX-DISABLED-NEXT:  entry:
-; NVPTX-DISABLED-NEXT:    call void @unknown() #[[ATTR8]]
-; NVPTX-DISABLED-NEXT:    ret void
 entry:
   call void @unknown() #11
   ret void
@@ -1597,24 +1098,6 @@ define internal void @__omp_outlined__3_wrapper(i16 zeroext %0, i32 %1) #3 {
 ; NVPTX-DISABLED2-NEXT:    call void @__omp_outlined__3(ptr [[DOTADDR1]], ptr [[DOTZERO_ADDR]]) #[[ATTR4]]
 ; NVPTX-DISABLED2-NEXT:    ret void
 ;
-; AMDGPU-DISABLED-LABEL: define {{[^@]+}}@__omp_outlined__3_wrapper
-; AMDGPU-DISABLED-SAME: (i16 zeroext [[TMP0:%.*]], i32 [[TMP1:%.*]]) #[[ATTR2]] {
-; AMDGPU-DISABLED-NEXT:  entry:
-; AMDGPU-DISABLED-NEXT:    [[DOTADDR1:%.*]] = alloca i32, align 4
-; AMDGPU-DISABLED-NEXT:    [[DOTZERO_ADDR:%.*]] = alloca i32, align 4
-; AMDGPU-DISABLED-NEXT:    [[GLOBAL_ARGS:%.*]] = alloca ptr, align 8
-; AMDGPU-DISABLED-NEXT:    call void @__kmpc_get_shared_variables(ptr [[GLOBAL_ARGS]])
-; AMDGPU-DISABLED-NEXT:    call void @__omp_outlined__3(ptr [[DOTADDR1]], ptr [[DOTZERO_ADDR]]) #[[ATTR4]]
-; AMDGPU-DISABLED-NEXT:    ret void
-; NVPTX-DISABLED-LABEL: define {{[^@]+}}@__omp_outlined__3_wrapper
-; NVPTX-DISABLED-SAME: (i16 zeroext [[TMP0:%.*]], i32 [[TMP1:%.*]]) #[[ATTR2]] {
-; NVPTX-DISABLED-NEXT:  entry:
-; NVPTX-DISABLED-NEXT:    [[DOTADDR1:%.*]] = alloca i32, align 4
-; NVPTX-DISABLED-NEXT:    [[DOTZERO_ADDR:%.*]] = alloca i32, align 4
-; NVPTX-DISABLED-NEXT:    [[GLOBAL_ARGS:%.*]] = alloca ptr, align 8
-; NVPTX-DISABLED-NEXT:    call void @__kmpc_get_shared_variables(ptr [[GLOBAL_ARGS]])
-; NVPTX-DISABLED-NEXT:    call void @__omp_outlined__3(ptr [[DOTADDR1]], ptr [[DOTZERO_ADDR]]) #[[ATTR4]]
-; NVPTX-DISABLED-NEXT:    ret void
 entry:
   %.addr1 = alloca i32, align 4
   %.zero.addr = alloca i32, align 4
@@ -1628,7 +1111,7 @@ entry:
 
 
 ; Function Attrs: alwaysinline convergent norecurse nounwind
-define weak void @__omp_offloading_fd02_2044372e_sequential_loop_to_shared_var_l35() #0 {
+define weak ptx_kernel void @__omp_offloading_fd02_2044372e_sequential_loop_to_shared_var_l35() #0 {
 ; AMDGPU-LABEL: define {{[^@]+}}@__omp_offloading_fd02_2044372e_sequential_loop_to_shared_var_l35
 ; AMDGPU-SAME: () #[[ATTR0]] {
 ; AMDGPU-NEXT:  entry:
@@ -1641,7 +1124,7 @@ define weak void @__omp_offloading_fd02_2044372e_sequential_loop_to_shared_var_l
 ; AMDGPU-NEXT:    ret void
 ; AMDGPU:       user_code.entry:
 ; AMDGPU-NEXT:    [[TMP1:%.*]] = call i32 @__kmpc_global_thread_num(ptr @[[GLOB1]]) #[[ATTR4]]
-; AMDGPU-NEXT:    store i32 [[TMP1]], ptr [[DOTTHREADID_TEMP_]], align 4, !tbaa [[TBAA18]]
+; AMDGPU-NEXT:    store i32 [[TMP1]], ptr [[DOTTHREADID_TEMP_]], align 4, !tbaa [[TBAA12]]
 ; AMDGPU-NEXT:    call void @__omp_outlined__4(ptr [[DOTTHREADID_TEMP_]], ptr [[DOTZERO_ADDR]]) #[[ATTR4]]
 ; AMDGPU-NEXT:    call void @__kmpc_target_deinit()
 ; AMDGPU-NEXT:    br label [[COMMON_RET]]
@@ -1658,7 +1141,7 @@ define weak void @__omp_offloading_fd02_2044372e_sequential_loop_to_shared_var_l
 ; NVPTX-NEXT:    ret void
 ; NVPTX:       user_code.entry:
 ; NVPTX-NEXT:    [[TMP1:%.*]] = call i32 @__kmpc_global_thread_num(ptr @[[GLOB1]]) #[[ATTR4]]
-; NVPTX-NEXT:    store i32 [[TMP1]], ptr [[DOTTHREADID_TEMP_]], align 4, !tbaa [[TBAA18]]
+; NVPTX-NEXT:    store i32 [[TMP1]], ptr [[DOTTHREADID_TEMP_]], align 4, !tbaa [[TBAA12]]
 ; NVPTX-NEXT:    call void @__omp_outlined__4(ptr [[DOTTHREADID_TEMP_]], ptr [[DOTZERO_ADDR]]) #[[ATTR4]]
 ; NVPTX-NEXT:    call void @__kmpc_target_deinit()
 ; NVPTX-NEXT:    br label [[COMMON_RET]]
@@ -1709,7 +1192,7 @@ define weak void @__omp_offloading_fd02_2044372e_sequential_loop_to_shared_var_l
 ; AMDGPU-DISABLED1-NEXT:    ret void
 ; AMDGPU-DISABLED1:       user_code.entry:
 ; AMDGPU-DISABLED1-NEXT:    [[TMP1:%.*]] = call i32 @__kmpc_global_thread_num(ptr @[[GLOB1]]) #[[ATTR4]]
-; AMDGPU-DISABLED1-NEXT:    store i32 [[TMP1]], ptr [[DOTTHREADID_TEMP_]], align 4, !tbaa [[TBAA18]]
+; AMDGPU-DISABLED1-NEXT:    store i32 [[TMP1]], ptr [[DOTTHREADID_TEMP_]], align 4, !tbaa [[TBAA12]]
 ; AMDGPU-DISABLED1-NEXT:    call void @__omp_outlined__4(ptr [[DOTTHREADID_TEMP_]], ptr [[DOTZERO_ADDR]]) #[[ATTR4]]
 ; AMDGPU-DISABLED1-NEXT:    call void @__kmpc_target_deinit()
 ; AMDGPU-DISABLED1-NEXT:    br label [[COMMON_RET]]
@@ -1717,50 +1200,16 @@ define weak void @__omp_offloading_fd02_2044372e_sequential_loop_to_shared_var_l
 ; AMDGPU-DISABLED2-LABEL: define {{[^@]+}}@__omp_offloading_fd02_2044372e_sequential_loop_to_shared_var_l35
 ; AMDGPU-DISABLED2-SAME: () #[[ATTR0]] {
 ; AMDGPU-DISABLED2-NEXT:  entry:
-; AMDGPU-DISABLED2-NEXT:    [[WORKER_WORK_FN_ADDR:%.*]] = alloca ptr, align 8, addrspace(5)
 ; AMDGPU-DISABLED2-NEXT:    [[DOTZERO_ADDR:%.*]] = alloca i32, align 4
 ; AMDGPU-DISABLED2-NEXT:    [[DOTTHREADID_TEMP_:%.*]] = alloca i32, align 4
 ; AMDGPU-DISABLED2-NEXT:    [[TMP0:%.*]] = call i32 @__kmpc_target_init(ptr @__omp_offloading_fd02_2044372e_sequential_loop_to_shared_var_l35_kernel_environment, ptr null)
-; AMDGPU-DISABLED2-NEXT:    [[THREAD_IS_WORKER:%.*]] = icmp ne i32 [[TMP0]], -1
-; AMDGPU-DISABLED2-NEXT:    br i1 [[THREAD_IS_WORKER]], label [[IS_WORKER_CHECK:%.*]], label [[THREAD_USER_CODE_CHECK:%.*]]
-; AMDGPU-DISABLED2:       is_worker_check:
-; AMDGPU-DISABLED2-NEXT:    [[BLOCK_HW_SIZE:%.*]] = call i32 @__kmpc_get_hardware_num_threads_in_block()
-; AMDGPU-DISABLED2-NEXT:    [[WARP_SIZE:%.*]] = call i32 @__kmpc_get_warp_size()
-; AMDGPU-DISABLED2-NEXT:    [[BLOCK_SIZE:%.*]] = sub i32 [[BLOCK_HW_SIZE]], [[WARP_SIZE]]
-; AMDGPU-DISABLED2-NEXT:    [[THREAD_IS_MAIN_OR_WORKER:%.*]] = icmp slt i32 [[TMP0]], [[BLOCK_SIZE]]
-; AMDGPU-DISABLED2-NEXT:    br i1 [[THREAD_IS_MAIN_OR_WORKER]], label [[WORKER_STATE_MACHINE_BEGIN:%.*]], label [[WORKER_STATE_MACHINE_FINISHED:%.*]]
-; AMDGPU-DISABLED2:       worker_state_machine.begin:
-; AMDGPU-DISABLED2-NEXT:    call void @__kmpc_barrier_simple_generic(ptr @[[GLOB1]], i32 [[TMP0]])
-; AMDGPU-DISABLED2-NEXT:    [[WORKER_WORK_FN_ADDR_GENERIC:%.*]] = addrspacecast ptr addrspace(5) [[WORKER_WORK_FN_ADDR]] to ptr
-; AMDGPU-DISABLED2-NEXT:    [[WORKER_IS_ACTIVE:%.*]] = call i1 @__kmpc_kernel_parallel(ptr [[WORKER_WORK_FN_ADDR_GENERIC]])
-; AMDGPU-DISABLED2-NEXT:    [[WORKER_WORK_FN:%.*]] = load ptr, ptr [[WORKER_WORK_FN_ADDR_GENERIC]], align 8
-; AMDGPU-DISABLED2-NEXT:    [[WORKER_IS_DONE:%.*]] = icmp eq ptr [[WORKER_WORK_FN]], null
-; AMDGPU-DISABLED2-NEXT:    br i1 [[WORKER_IS_DONE]], label [[WORKER_STATE_MACHINE_FINISHED]], label [[WORKER_STATE_MACHINE_IS_ACTIVE_CHECK:%.*]]
-; AMDGPU-DISABLED2:       worker_state_machine.finished:
-; AMDGPU-DISABLED2-NEXT:    ret void
-; AMDGPU-DISABLED2:       worker_state_machine.is_active.check:
-; AMDGPU-DISABLED2-NEXT:    br i1 [[WORKER_IS_ACTIVE]], label [[WORKER_STATE_MACHINE_PARALLEL_REGION_CHECK:%.*]], label [[WORKER_STATE_MACHINE_DONE_BARRIER:%.*]]
-; AMDGPU-DISABLED2:       worker_state_machine.parallel_region.check:
-; AMDGPU-DISABLED2-NEXT:    br i1 true, label [[WORKER_STATE_MACHINE_PARALLEL_REGION_EXECUTE:%.*]], label [[WORKER_STATE_MACHINE_PARALLEL_REGION_CHECK1:%.*]]
-; AMDGPU-DISABLED2:       worker_state_machine.parallel_region.execute:
-; AMDGPU-DISABLED2-NEXT:    call void @__omp_outlined__5_wrapper(i16 0, i32 [[TMP0]])
-; AMDGPU-DISABLED2-NEXT:    br label [[WORKER_STATE_MACHINE_PARALLEL_REGION_END:%.*]]
-; AMDGPU-DISABLED2:       worker_state_machine.parallel_region.check1:
-; AMDGPU-DISABLED2-NEXT:    br label [[WORKER_STATE_MACHINE_PARALLEL_REGION_END]]
-; AMDGPU-DISABLED2:       worker_state_machine.parallel_region.end:
-; AMDGPU-DISABLED2-NEXT:    call void @__kmpc_kernel_end_parallel()
-; AMDGPU-DISABLED2-NEXT:    br label [[WORKER_STATE_MACHINE_DONE_BARRIER]]
-; AMDGPU-DISABLED2:       worker_state_machine.done.barrier:
-; AMDGPU-DISABLED2-NEXT:    call void @__kmpc_barrier_simple_generic(ptr @[[GLOB1]], i32 [[TMP0]])
-; AMDGPU-DISABLED2-NEXT:    br label [[WORKER_STATE_MACHINE_BEGIN]]
-; AMDGPU-DISABLED2:       thread.user_code.check:
 ; AMDGPU-DISABLED2-NEXT:    [[EXEC_USER_CODE:%.*]] = icmp eq i32 [[TMP0]], -1
 ; AMDGPU-DISABLED2-NEXT:    br i1 [[EXEC_USER_CODE]], label [[USER_CODE_ENTRY:%.*]], label [[COMMON_RET:%.*]]
 ; AMDGPU-DISABLED2:       common.ret:
 ; AMDGPU-DISABLED2-NEXT:    ret void
 ; AMDGPU-DISABLED2:       user_code.entry:
 ; AMDGPU-DISABLED2-NEXT:    [[TMP1:%.*]] = call i32 @__kmpc_global_thread_num(ptr @[[GLOB1]]) #[[ATTR4]]
-; AMDGPU-DISABLED2-NEXT:    store i32 [[TMP1]], ptr [[DOTTHREADID_TEMP_]], align 4, !tbaa [[TBAA18]]
+; AMDGPU-DISABLED2-NEXT:    store i32 [[TMP1]], ptr [[DOTTHREADID_TEMP_]], align 4, !tbaa [[TBAA12]]
 ; AMDGPU-DISABLED2-NEXT:    call void @__omp_outlined__4(ptr [[DOTTHREADID_TEMP_]], ptr [[DOTZERO_ADDR]]) #[[ATTR4]]
 ; AMDGPU-DISABLED2-NEXT:    call void @__kmpc_target_deinit()
 ; AMDGPU-DISABLED2-NEXT:    br label [[COMMON_RET]]
@@ -1810,7 +1259,7 @@ define weak void @__omp_offloading_fd02_2044372e_sequential_loop_to_shared_var_l
 ; NVPTX-DISABLED1-NEXT:    ret void
 ; NVPTX-DISABLED1:       user_code.entry:
 ; NVPTX-DISABLED1-NEXT:    [[TMP1:%.*]] = call i32 @__kmpc_global_thread_num(ptr @[[GLOB1]]) #[[ATTR4]]
-; NVPTX-DISABLED1-NEXT:    store i32 [[TMP1]], ptr [[DOTTHREADID_TEMP_]], align 4, !tbaa [[TBAA18]]
+; NVPTX-DISABLED1-NEXT:    store i32 [[TMP1]], ptr [[DOTTHREADID_TEMP_]], align 4, !tbaa [[TBAA12]]
 ; NVPTX-DISABLED1-NEXT:    call void @__omp_outlined__4(ptr [[DOTTHREADID_TEMP_]], ptr [[DOTZERO_ADDR]]) #[[ATTR4]]
 ; NVPTX-DISABLED1-NEXT:    call void @__kmpc_target_deinit()
 ; NVPTX-DISABLED1-NEXT:    br label [[COMMON_RET]]
@@ -1818,156 +1267,20 @@ define weak void @__omp_offloading_fd02_2044372e_sequential_loop_to_shared_var_l
 ; NVPTX-DISABLED2-LABEL: define {{[^@]+}}@__omp_offloading_fd02_2044372e_sequential_loop_to_shared_var_l35
 ; NVPTX-DISABLED2-SAME: () #[[ATTR0]] {
 ; NVPTX-DISABLED2-NEXT:  entry:
-; NVPTX-DISABLED2-NEXT:    [[WORKER_WORK_FN_ADDR:%.*]] = alloca ptr, align 8
 ; NVPTX-DISABLED2-NEXT:    [[DOTZERO_ADDR:%.*]] = alloca i32, align 4
 ; NVPTX-DISABLED2-NEXT:    [[DOTTHREADID_TEMP_:%.*]] = alloca i32, align 4
 ; NVPTX-DISABLED2-NEXT:    [[TMP0:%.*]] = call i32 @__kmpc_target_init(ptr @__omp_offloading_fd02_2044372e_sequential_loop_to_shared_var_l35_kernel_environment, ptr null)
-; NVPTX-DISABLED2-NEXT:    [[THREAD_IS_WORKER:%.*]] = icmp ne i32 [[TMP0]], -1
-; NVPTX-DISABLED2-NEXT:    br i1 [[THREAD_IS_WORKER]], label [[IS_WORKER_CHECK:%.*]], label [[THREAD_USER_CODE_CHECK:%.*]]
-; NVPTX-DISABLED2:       is_worker_check:
-; NVPTX-DISABLED2-NEXT:    [[BLOCK_HW_SIZE:%.*]] = call i32 @__kmpc_get_hardware_num_threads_in_block()
-; NVPTX-DISABLED2-NEXT:    [[WARP_SIZE:%.*]] = call i32 @__kmpc_get_warp_size()
-; NVPTX-DISABLED2-NEXT:    [[BLOCK_SIZE:%.*]] = sub i32 [[BLOCK_HW_SIZE]], [[WARP_SIZE]]
-; NVPTX-DISABLED2-NEXT:    [[THREAD_IS_MAIN_OR_WORKER:%.*]] = icmp slt i32 [[TMP0]], [[BLOCK_SIZE]]
-; NVPTX-DISABLED2-NEXT:    br i1 [[THREAD_IS_MAIN_OR_WORKER]], label [[WORKER_STATE_MACHINE_BEGIN:%.*]], label [[WORKER_STATE_MACHINE_FINISHED:%.*]]
-; NVPTX-DISABLED2:       worker_state_machine.begin:
-; NVPTX-DISABLED2-NEXT:    call void @__kmpc_barrier_simple_generic(ptr @[[GLOB1]], i32 [[TMP0]])
-; NVPTX-DISABLED2-NEXT:    [[WORKER_IS_ACTIVE:%.*]] = call i1 @__kmpc_kernel_parallel(ptr [[WORKER_WORK_FN_ADDR]])
-; NVPTX-DISABLED2-NEXT:    [[WORKER_WORK_FN:%.*]] = load ptr, ptr [[WORKER_WORK_FN_ADDR]], align 8
-; NVPTX-DISABLED2-NEXT:    [[WORKER_IS_DONE:%.*]] = icmp eq ptr [[WORKER_WORK_FN]], null
-; NVPTX-DISABLED2-NEXT:    br i1 [[WORKER_IS_DONE]], label [[WORKER_STATE_MACHINE_FINISHED]], label [[WORKER_STATE_MACHINE_IS_ACTIVE_CHECK:%.*]]
-; NVPTX-DISABLED2:       worker_state_machine.finished:
-; NVPTX-DISABLED2-NEXT:    ret void
-; NVPTX-DISABLED2:       worker_state_machine.is_active.check:
-; NVPTX-DISABLED2-NEXT:    br i1 [[WORKER_IS_ACTIVE]], label [[WORKER_STATE_MACHINE_PARALLEL_REGION_CHECK:%.*]], label [[WORKER_STATE_MACHINE_DONE_BARRIER:%.*]]
-; NVPTX-DISABLED2:       worker_state_machine.parallel_region.check:
-; NVPTX-DISABLED2-NEXT:    br i1 true, label [[WORKER_STATE_MACHINE_PARALLEL_REGION_EXECUTE:%.*]], label [[WORKER_STATE_MACHINE_PARALLEL_REGION_CHECK1:%.*]]
-; NVPTX-DISABLED2:       worker_state_machine.parallel_region.execute:
-; NVPTX-DISABLED2-NEXT:    call void @__omp_outlined__5_wrapper(i16 0, i32 [[TMP0]])
-; NVPTX-DISABLED2-NEXT:    br label [[WORKER_STATE_MACHINE_PARALLEL_REGION_END:%.*]]
-; NVPTX-DISABLED2:       worker_state_machine.parallel_region.check1:
-; NVPTX-DISABLED2-NEXT:    br label [[WORKER_STATE_MACHINE_PARALLEL_REGION_END]]
-; NVPTX-DISABLED2:       worker_state_machine.parallel_region.end:
-; NVPTX-DISABLED2-NEXT:    call void @__kmpc_kernel_end_parallel()
-; NVPTX-DISABLED2-NEXT:    br label [[WORKER_STATE_MACHINE_DONE_BARRIER]]
-; NVPTX-DISABLED2:       worker_state_machine.done.barrier:
-; NVPTX-DISABLED2-NEXT:    call void @__kmpc_barrier_simple_generic(ptr @[[GLOB1]], i32 [[TMP0]])
-; NVPTX-DISABLED2-NEXT:    br label [[WORKER_STATE_MACHINE_BEGIN]]
-; NVPTX-DISABLED2:       thread.user_code.check:
 ; NVPTX-DISABLED2-NEXT:    [[EXEC_USER_CODE:%.*]] = icmp eq i32 [[TMP0]], -1
 ; NVPTX-DISABLED2-NEXT:    br i1 [[EXEC_USER_CODE]], label [[USER_CODE_ENTRY:%.*]], label [[COMMON_RET:%.*]]
 ; NVPTX-DISABLED2:       common.ret:
 ; NVPTX-DISABLED2-NEXT:    ret void
 ; NVPTX-DISABLED2:       user_code.entry:
 ; NVPTX-DISABLED2-NEXT:    [[TMP1:%.*]] = call i32 @__kmpc_global_thread_num(ptr @[[GLOB1]]) #[[ATTR4]]
-; NVPTX-DISABLED2-NEXT:    store i32 [[TMP1]], ptr [[DOTTHREADID_TEMP_]], align 4, !tbaa [[TBAA18]]
+; NVPTX-DISABLED2-NEXT:    store i32 [[TMP1]], ptr [[DOTTHREADID_TEMP_]], align 4, !tbaa [[TBAA12]]
 ; NVPTX-DISABLED2-NEXT:    call void @__omp_outlined__4(ptr [[DOTTHREADID_TEMP_]], ptr [[DOTZERO_ADDR]]) #[[ATTR4]]
 ; NVPTX-DISABLED2-NEXT:    call void @__kmpc_target_deinit()
 ; NVPTX-DISABLED2-NEXT:    br label [[COMMON_RET]]
 ;
-; AMDGPU-DISABLED-LABEL: define {{[^@]+}}@__omp_offloading_fd02_2044372e_sequential_loop_to_shared_var_l35
-; AMDGPU-DISABLED-SAME: () #[[ATTR0]] {
-; AMDGPU-DISABLED-NEXT:  entry:
-; AMDGPU-DISABLED-NEXT:    [[WORKER_WORK_FN_ADDR:%.*]] = alloca ptr, align 8, addrspace(5)
-; AMDGPU-DISABLED-NEXT:    [[DOTZERO_ADDR:%.*]] = alloca i32, align 4
-; AMDGPU-DISABLED-NEXT:    [[DOTTHREADID_TEMP_:%.*]] = alloca i32, align 4
-; AMDGPU-DISABLED-NEXT:    [[TMP0:%.*]] = call i32 @__kmpc_target_init(ptr @__omp_offloading_fd02_2044372e_sequential_loop_to_shared_var_l35_kernel_environment)
-; AMDGPU-DISABLED-NEXT:    [[THREAD_IS_WORKER:%.*]] = icmp ne i32 [[TMP0]], -1
-; AMDGPU-DISABLED-NEXT:    br i1 [[THREAD_IS_WORKER]], label [[IS_WORKER_CHECK:%.*]], label [[THREAD_USER_CODE_CHECK:%.*]]
-; AMDGPU-DISABLED:       is_worker_check:
-; AMDGPU-DISABLED-NEXT:    [[BLOCK_HW_SIZE:%.*]] = call i32 @__kmpc_get_hardware_num_threads_in_block()
-; AMDGPU-DISABLED-NEXT:    [[WARP_SIZE:%.*]] = call i32 @__kmpc_get_warp_size()
-; AMDGPU-DISABLED-NEXT:    [[BLOCK_SIZE:%.*]] = sub i32 [[BLOCK_HW_SIZE]], [[WARP_SIZE]]
-; AMDGPU-DISABLED-NEXT:    [[THREAD_IS_MAIN_OR_WORKER:%.*]] = icmp slt i32 [[TMP0]], [[BLOCK_SIZE]]
-; AMDGPU-DISABLED-NEXT:    br i1 [[THREAD_IS_MAIN_OR_WORKER]], label [[WORKER_STATE_MACHINE_BEGIN:%.*]], label [[WORKER_STATE_MACHINE_FINISHED:%.*]]
-; AMDGPU-DISABLED:       worker_state_machine.begin:
-; AMDGPU-DISABLED-NEXT:    call void @__kmpc_barrier_simple_generic(ptr @[[GLOB1]], i32 [[TMP0]])
-; AMDGPU-DISABLED-NEXT:    [[WORKER_WORK_FN_ADDR_GENERIC:%.*]] = addrspacecast ptr addrspace(5) [[WORKER_WORK_FN_ADDR]] to ptr
-; AMDGPU-DISABLED-NEXT:    [[WORKER_IS_ACTIVE:%.*]] = call i1 @__kmpc_kernel_parallel(ptr [[WORKER_WORK_FN_ADDR_GENERIC]])
-; AMDGPU-DISABLED-NEXT:    [[WORKER_WORK_FN:%.*]] = load ptr, ptr [[WORKER_WORK_FN_ADDR_GENERIC]], align 8
-; AMDGPU-DISABLED-NEXT:    [[WORKER_IS_DONE:%.*]] = icmp eq ptr [[WORKER_WORK_FN]], null
-; AMDGPU-DISABLED-NEXT:    br i1 [[WORKER_IS_DONE]], label [[WORKER_STATE_MACHINE_FINISHED]], label [[WORKER_STATE_MACHINE_IS_ACTIVE_CHECK:%.*]]
-; AMDGPU-DISABLED:       worker_state_machine.finished:
-; AMDGPU-DISABLED-NEXT:    ret void
-; AMDGPU-DISABLED:       worker_state_machine.is_active.check:
-; AMDGPU-DISABLED-NEXT:    br i1 [[WORKER_IS_ACTIVE]], label [[WORKER_STATE_MACHINE_PARALLEL_REGION_CHECK:%.*]], label [[WORKER_STATE_MACHINE_DONE_BARRIER:%.*]]
-; AMDGPU-DISABLED:       worker_state_machine.parallel_region.check:
-; AMDGPU-DISABLED-NEXT:    [[WORKER_CHECK_PARALLEL_REGION:%.*]] = icmp eq ptr [[WORKER_WORK_FN]], @__omp_outlined__5_wrapper.ID
-; AMDGPU-DISABLED-NEXT:    br i1 [[WORKER_CHECK_PARALLEL_REGION]], label [[WORKER_STATE_MACHINE_PARALLEL_REGION_EXECUTE:%.*]], label [[WORKER_STATE_MACHINE_PARALLEL_REGION_FALLBACK_EXECUTE:%.*]]
-; AMDGPU-DISABLED:       worker_state_machine.parallel_region.execute:
-; AMDGPU-DISABLED-NEXT:    call void @__omp_outlined__5_wrapper(i16 0, i32 [[TMP0]])
-; AMDGPU-DISABLED-NEXT:    br label [[WORKER_STATE_MACHINE_PARALLEL_REGION_END:%.*]]
-; AMDGPU-DISABLED:       worker_state_machine.parallel_region.fallback.execute:
-; AMDGPU-DISABLED-NEXT:    call void [[WORKER_WORK_FN]](i16 0, i32 [[TMP0]])
-; AMDGPU-DISABLED-NEXT:    br label [[WORKER_STATE_MACHINE_PARALLEL_REGION_END]]
-; AMDGPU-DISABLED:       worker_state_machine.parallel_region.end:
-; AMDGPU-DISABLED-NEXT:    call void @__kmpc_kernel_end_parallel()
-; AMDGPU-DISABLED-NEXT:    br label [[WORKER_STATE_MACHINE_DONE_BARRIER]]
-; AMDGPU-DISABLED:       worker_state_machine.done.barrier:
-; AMDGPU-DISABLED-NEXT:    call void @__kmpc_barrier_simple_generic(ptr @[[GLOB1]], i32 [[TMP0]])
-; AMDGPU-DISABLED-NEXT:    br label [[WORKER_STATE_MACHINE_BEGIN]]
-; AMDGPU-DISABLED:       thread.user_code.check:
-; AMDGPU-DISABLED-NEXT:    [[EXEC_USER_CODE:%.*]] = icmp eq i32 [[TMP0]], -1
-; AMDGPU-DISABLED-NEXT:    br i1 [[EXEC_USER_CODE]], label [[USER_CODE_ENTRY:%.*]], label [[COMMON_RET:%.*]]
-; AMDGPU-DISABLED:       common.ret:
-; AMDGPU-DISABLED-NEXT:    ret void
-; AMDGPU-DISABLED:       user_code.entry:
-; AMDGPU-DISABLED-NEXT:    [[TMP1:%.*]] = call i32 @__kmpc_global_thread_num(ptr @[[GLOB1]]) #[[ATTR4]]
-; AMDGPU-DISABLED-NEXT:    store i32 [[TMP1]], ptr [[DOTTHREADID_TEMP_]], align 4, !tbaa [[TBAA18]]
-; AMDGPU-DISABLED-NEXT:    call void @__omp_outlined__4(ptr [[DOTTHREADID_TEMP_]], ptr [[DOTZERO_ADDR]]) #[[ATTR5]]
-; AMDGPU-DISABLED-NEXT:    call void @__kmpc_target_deinit()
-; AMDGPU-DISABLED-NEXT:    br label [[COMMON_RET]]
-; NVPTX-DISABLED-LABEL: define {{[^@]+}}@__omp_offloading_fd02_2044372e_sequential_loop_to_shared_var_l35
-; NVPTX-DISABLED-SAME: () #[[ATTR0]] {
-; NVPTX-DISABLED-NEXT:  entry:
-; NVPTX-DISABLED-NEXT:    [[WORKER_WORK_FN_ADDR:%.*]] = alloca ptr, align 8
-; NVPTX-DISABLED-NEXT:    [[DOTZERO_ADDR:%.*]] = alloca i32, align 4
-; NVPTX-DISABLED-NEXT:    [[DOTTHREADID_TEMP_:%.*]] = alloca i32, align 4
-; NVPTX-DISABLED-NEXT:    [[TMP0:%.*]] = call i32 @__kmpc_target_init(ptr @__omp_offloading_fd02_2044372e_sequential_loop_to_shared_var_l35_kernel_environment)
-; NVPTX-DISABLED-NEXT:    [[THREAD_IS_WORKER:%.*]] = icmp ne i32 [[TMP0]], -1
-; NVPTX-DISABLED-NEXT:    br i1 [[THREAD_IS_WORKER]], label [[IS_WORKER_CHECK:%.*]], label [[THREAD_USER_CODE_CHECK:%.*]]
-; NVPTX-DISABLED:       is_worker_check:
-; NVPTX-DISABLED-NEXT:    [[BLOCK_HW_SIZE:%.*]] = call i32 @__kmpc_get_hardware_num_threads_in_block()
-; NVPTX-DISABLED-NEXT:    [[WARP_SIZE:%.*]] = call i32 @__kmpc_get_warp_size()
-; NVPTX-DISABLED-NEXT:    [[BLOCK_SIZE:%.*]] = sub i32 [[BLOCK_HW_SIZE]], [[WARP_SIZE]]
-; NVPTX-DISABLED-NEXT:    [[THREAD_IS_MAIN_OR_WORKER:%.*]] = icmp slt i32 [[TMP0]], [[BLOCK_SIZE]]
-; NVPTX-DISABLED-NEXT:    br i1 [[THREAD_IS_MAIN_OR_WORKER]], label [[WORKER_STATE_MACHINE_BEGIN:%.*]], label [[WORKER_STATE_MACHINE_FINISHED:%.*]]
-; NVPTX-DISABLED:       worker_state_machine.begin:
-; NVPTX-DISABLED-NEXT:    call void @__kmpc_barrier_simple_generic(ptr @[[GLOB1]], i32 [[TMP0]])
-; NVPTX-DISABLED-NEXT:    [[WORKER_IS_ACTIVE:%.*]] = call i1 @__kmpc_kernel_parallel(ptr [[WORKER_WORK_FN_ADDR]])
-; NVPTX-DISABLED-NEXT:    [[WORKER_WORK_FN:%.*]] = load ptr, ptr [[WORKER_WORK_FN_ADDR]], align 8
-; NVPTX-DISABLED-NEXT:    [[WORKER_IS_DONE:%.*]] = icmp eq ptr [[WORKER_WORK_FN]], null
-; NVPTX-DISABLED-NEXT:    br i1 [[WORKER_IS_DONE]], label [[WORKER_STATE_MACHINE_FINISHED]], label [[WORKER_STATE_MACHINE_IS_ACTIVE_CHECK:%.*]]
-; NVPTX-DISABLED:       worker_state_machine.finished:
-; NVPTX-DISABLED-NEXT:    ret void
-; NVPTX-DISABLED:       worker_state_machine.is_active.check:
-; NVPTX-DISABLED-NEXT:    br i1 [[WORKER_IS_ACTIVE]], label [[WORKER_STATE_MACHINE_PARALLEL_REGION_CHECK:%.*]], label [[WORKER_STATE_MACHINE_DONE_BARRIER:%.*]]
-; NVPTX-DISABLED:       worker_state_machine.parallel_region.check:
-; NVPTX-DISABLED-NEXT:    [[WORKER_CHECK_PARALLEL_REGION:%.*]] = icmp eq ptr [[WORKER_WORK_FN]], @__omp_outlined__5_wrapper.ID
-; NVPTX-DISABLED-NEXT:    br i1 [[WORKER_CHECK_PARALLEL_REGION]], label [[WORKER_STATE_MACHINE_PARALLEL_REGION_EXECUTE:%.*]], label [[WORKER_STATE_MACHINE_PARALLEL_REGION_FALLBACK_EXECUTE:%.*]]
-; NVPTX-DISABLED:       worker_state_machine.parallel_region.execute:
-; NVPTX-DISABLED-NEXT:    call void @__omp_outlined__5_wrapper(i16 0, i32 [[TMP0]])
-; NVPTX-DISABLED-NEXT:    br label [[WORKER_STATE_MACHINE_PARALLEL_REGION_END:%.*]]
-; NVPTX-DISABLED:       worker_state_machine.parallel_region.fallback.execute:
-; NVPTX-DISABLED-NEXT:    call void [[WORKER_WORK_FN]](i16 0, i32 [[TMP0]])
-; NVPTX-DISABLED-NEXT:    br label [[WORKER_STATE_MACHINE_PARALLEL_REGION_END]]
-; NVPTX-DISABLED:       worker_state_machine.parallel_region.end:
-; NVPTX-DISABLED-NEXT:    call void @__kmpc_kernel_end_parallel()
-; NVPTX-DISABLED-NEXT:    br label [[WORKER_STATE_MACHINE_DONE_BARRIER]]
-; NVPTX-DISABLED:       worker_state_machine.done.barrier:
-; NVPTX-DISABLED-NEXT:    call void @__kmpc_barrier_simple_generic(ptr @[[GLOB1]], i32 [[TMP0]])
-; NVPTX-DISABLED-NEXT:    br label [[WORKER_STATE_MACHINE_BEGIN]]
-; NVPTX-DISABLED:       thread.user_code.check:
-; NVPTX-DISABLED-NEXT:    [[EXEC_USER_CODE:%.*]] = icmp eq i32 [[TMP0]], -1
-; NVPTX-DISABLED-NEXT:    br i1 [[EXEC_USER_CODE]], label [[USER_CODE_ENTRY:%.*]], label [[COMMON_RET:%.*]]
-; NVPTX-DISABLED:       common.ret:
-; NVPTX-DISABLED-NEXT:    ret void
-; NVPTX-DISABLED:       user_code.entry:
-; NVPTX-DISABLED-NEXT:    [[TMP1:%.*]] = call i32 @__kmpc_global_thread_num(ptr @[[GLOB1]]) #[[ATTR4]]
-; NVPTX-DISABLED-NEXT:    store i32 [[TMP1]], ptr [[DOTTHREADID_TEMP_]], align 4, !tbaa [[TBAA18]]
-; NVPTX-DISABLED-NEXT:    call void @__omp_outlined__4(ptr [[DOTTHREADID_TEMP_]], ptr [[DOTZERO_ADDR]]) #[[ATTR5]]
-; NVPTX-DISABLED-NEXT:    call void @__kmpc_target_deinit()
-; NVPTX-DISABLED-NEXT:    br label [[COMMON_RET]]
 entry:
   %.zero.addr = alloca i32, align 4
   %.threadid_temp. = alloca i32, align 4
@@ -2002,11 +1315,11 @@ define internal void @__omp_outlined__4(ptr noalias %.global_tid., ptr noalias %
 ; AMDGPU-NEXT:    call void @spmd_amenable() #[[ATTR7]]
 ; AMDGPU-NEXT:    ret void
 ; AMDGPU:       for.body:
-; AMDGPU-NEXT:    store ptr addrspacecast (ptr addrspace(3) @x_shared to ptr), ptr [[CAPTURED_VARS_ADDRS]], align 8, !tbaa [[TBAA26:![0-9]+]]
-; AMDGPU-NEXT:    [[TMP0:%.*]] = load i32, ptr [[DOTGLOBAL_TID_]], align 4, !tbaa [[TBAA18]]
+; AMDGPU-NEXT:    store ptr addrspacecast (ptr addrspace(3) @x_shared to ptr), ptr [[CAPTURED_VARS_ADDRS]], align 8, !tbaa [[TBAA20:![0-9]+]]
+; AMDGPU-NEXT:    [[TMP0:%.*]] = load i32, ptr [[DOTGLOBAL_TID_]], align 4, !tbaa [[TBAA12]]
 ; AMDGPU-NEXT:    call void @__kmpc_parallel_51(ptr @[[GLOB1]], i32 [[TMP0]], i32 1, i32 -1, i32 -1, ptr @__omp_outlined__5, ptr @__omp_outlined__5_wrapper, ptr [[CAPTURED_VARS_ADDRS]], i64 1)
 ; AMDGPU-NEXT:    [[INC]] = add nsw i32 [[I_0]], 1
-; AMDGPU-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP28:![0-9]+]]
+; AMDGPU-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP22:![0-9]+]]
 ;
 ; NVPTX-LABEL: define {{[^@]+}}@__omp_outlined__4
 ; NVPTX-SAME: (ptr noalias [[DOTGLOBAL_TID_:%.*]], ptr noalias [[DOTBOUND_TID_:%.*]]) #[[ATTR1]] {
@@ -2021,11 +1334,11 @@ define internal void @__omp_outlined__4(ptr noalias %.global_tid., ptr noalias %
 ; NVPTX-NEXT:    call void @spmd_amenable() #[[ATTR7]]
 ; NVPTX-NEXT:    ret void
 ; NVPTX:       for.body:
-; NVPTX-NEXT:    store ptr addrspacecast (ptr addrspace(3) @x_shared to ptr), ptr [[CAPTURED_VARS_ADDRS]], align 8, !tbaa [[TBAA26:![0-9]+]]
-; NVPTX-NEXT:    [[TMP0:%.*]] = load i32, ptr [[DOTGLOBAL_TID_]], align 4, !tbaa [[TBAA18]]
+; NVPTX-NEXT:    store ptr addrspacecast (ptr addrspace(3) @x_shared to ptr), ptr [[CAPTURED_VARS_ADDRS]], align 8, !tbaa [[TBAA20:![0-9]+]]
+; NVPTX-NEXT:    [[TMP0:%.*]] = load i32, ptr [[DOTGLOBAL_TID_]], align 4, !tbaa [[TBAA12]]
 ; NVPTX-NEXT:    call void @__kmpc_parallel_51(ptr @[[GLOB1]], i32 [[TMP0]], i32 1, i32 -1, i32 -1, ptr @__omp_outlined__5, ptr @__omp_outlined__5_wrapper, ptr [[CAPTURED_VARS_ADDRS]], i64 1)
 ; NVPTX-NEXT:    [[INC]] = add nsw i32 [[I_0]], 1
-; NVPTX-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP28:![0-9]+]]
+; NVPTX-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP22:![0-9]+]]
 ;
 ; AMDGPU-DISABLED1-LABEL: define {{[^@]+}}@__omp_outlined__4
 ; AMDGPU-DISABLED1-SAME: (ptr noalias [[DOTGLOBAL_TID_:%.*]], ptr noalias [[DOTBOUND_TID_:%.*]]) #[[ATTR1]] {
@@ -2040,11 +1353,11 @@ define internal void @__omp_outlined__4(ptr noalias %.global_tid., ptr noalias %
 ; AMDGPU-DISABLED1-NEXT:    call void @spmd_amenable() #[[ATTR7]]
 ; AMDGPU-DISABLED1-NEXT:    ret void
 ; AMDGPU-DISABLED1:       for.body:
-; AMDGPU-DISABLED1-NEXT:    store ptr addrspacecast (ptr addrspace(3) @x_shared to ptr), ptr [[CAPTURED_VARS_ADDRS]], align 8, !tbaa [[TBAA26:![0-9]+]]
-; AMDGPU-DISABLED1-NEXT:    [[TMP0:%.*]] = load i32, ptr [[DOTGLOBAL_TID_]], align 4, !tbaa [[TBAA18]]
+; AMDGPU-DISABLED1-NEXT:    store ptr addrspacecast (ptr addrspace(3) @x_shared to ptr), ptr [[CAPTURED_VARS_ADDRS]], align 8, !tbaa [[TBAA20:![0-9]+]]
+; AMDGPU-DISABLED1-NEXT:    [[TMP0:%.*]] = load i32, ptr [[DOTGLOBAL_TID_]], align 4, !tbaa [[TBAA12]]
 ; AMDGPU-DISABLED1-NEXT:    call void @__kmpc_parallel_51(ptr @[[GLOB1]], i32 [[TMP0]], i32 1, i32 -1, i32 -1, ptr @__omp_outlined__5, ptr @__omp_outlined__5_wrapper.ID, ptr [[CAPTURED_VARS_ADDRS]], i64 1)
 ; AMDGPU-DISABLED1-NEXT:    [[INC]] = add nsw i32 [[I_0]], 1
-; AMDGPU-DISABLED1-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP28:![0-9]+]]
+; AMDGPU-DISABLED1-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP22:![0-9]+]]
 ;
 ; AMDGPU-DISABLED2-LABEL: define {{[^@]+}}@__omp_outlined__4
 ; AMDGPU-DISABLED2-SAME: (ptr noalias [[DOTGLOBAL_TID_:%.*]], ptr noalias [[DOTBOUND_TID_:%.*]]) #[[ATTR1]] {
@@ -2059,11 +1372,11 @@ define internal void @__omp_outlined__4(ptr noalias %.global_tid., ptr noalias %
 ; AMDGPU-DISABLED2-NEXT:    call void @spmd_amenable() #[[ATTR7]]
 ; AMDGPU-DISABLED2-NEXT:    ret void
 ; AMDGPU-DISABLED2:       for.body:
-; AMDGPU-DISABLED2-NEXT:    store ptr addrspacecast (ptr addrspace(3) @x_shared to ptr), ptr [[CAPTURED_VARS_ADDRS]], align 8, !tbaa [[TBAA26:![0-9]+]]
-; AMDGPU-DISABLED2-NEXT:    [[TMP0:%.*]] = load i32, ptr [[DOTGLOBAL_TID_]], align 4, !tbaa [[TBAA18]]
-; AMDGPU-DISABLED2-NEXT:    call void @__kmpc_parallel_51(ptr @[[GLOB1]], i32 [[TMP0]], i32 1, i32 -1, i32 -1, ptr @__omp_outlined__5, ptr @__omp_outlined__5_wrapper.ID, ptr [[CAPTURED_VARS_ADDRS]], i64 1)
+; AMDGPU-DISABLED2-NEXT:    store ptr addrspacecast (ptr addrspace(3) @x_shared to ptr), ptr [[CAPTURED_VARS_ADDRS]], align 8, !tbaa [[TBAA20:![0-9]+]]
+; AMDGPU-DISABLED2-NEXT:    [[TMP0:%.*]] = load i32, ptr [[DOTGLOBAL_TID_]], align 4, !tbaa [[TBAA12]]
+; AMDGPU-DISABLED2-NEXT:    call void @__kmpc_parallel_51(ptr @[[GLOB1]], i32 [[TMP0]], i32 1, i32 -1, i32 -1, ptr @__omp_outlined__5, ptr @__omp_outlined__5_wrapper, ptr [[CAPTURED_VARS_ADDRS]], i64 1)
 ; AMDGPU-DISABLED2-NEXT:    [[INC]] = add nsw i32 [[I_0]], 1
-; AMDGPU-DISABLED2-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP28:![0-9]+]]
+; AMDGPU-DISABLED2-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP22:![0-9]+]]
 ;
 ; NVPTX-DISABLED1-LABEL: define {{[^@]+}}@__omp_outlined__4
 ; NVPTX-DISABLED1-SAME: (ptr noalias [[DOTGLOBAL_TID_:%.*]], ptr noalias [[DOTBOUND_TID_:%.*]]) #[[ATTR1]] {
@@ -2078,11 +1391,11 @@ define internal void @__omp_outlined__4(ptr noalias %.global_tid., ptr noalias %
 ; NVPTX-DISABLED1-NEXT:    call void @spmd_amenable() #[[ATTR7]]
 ; NVPTX-DISABLED1-NEXT:    ret void
 ; NVPTX-DISABLED1:       for.body:
-; NVPTX-DISABLED1-NEXT:    store ptr addrspacecast (ptr addrspace(3) @x_shared to ptr), ptr [[CAPTURED_VARS_ADDRS]], align 8, !tbaa [[TBAA26:![0-9]+]]
-; NVPTX-DISABLED1-NEXT:    [[TMP0:%.*]] = load i32, ptr [[DOTGLOBAL_TID_]], align 4, !tbaa [[TBAA18]]
+; NVPTX-DISABLED1-NEXT:    store ptr addrspacecast (ptr addrspace(3) @x_shared to ptr), ptr [[CAPTURED_VARS_ADDRS]], align 8, !tbaa [[TBAA20:![0-9]+]]
+; NVPTX-DISABLED1-NEXT:    [[TMP0:%.*]] = load i32, ptr [[DOTGLOBAL_TID_]], align 4, !tbaa [[TBAA12]]
 ; NVPTX-DISABLED1-NEXT:    call void @__kmpc_parallel_51(ptr @[[GLOB1]], i32 [[TMP0]], i32 1, i32 -1, i32 -1, ptr @__omp_outlined__5, ptr @__omp_outlined__5_wrapper.ID, ptr [[CAPTURED_VARS_ADDRS]], i64 1)
 ; NVPTX-DISABLED1-NEXT:    [[INC]] = add nsw i32 [[I_0]], 1
-; NVPTX-DISABLED1-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP28:![0-9]+]]
+; NVPTX-DISABLED1-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP22:![0-9]+]]
 ;
 ; NVPTX-DISABLED2-LABEL: define {{[^@]+}}@__omp_outlined__4
 ; NVPTX-DISABLED2-SAME: (ptr noalias [[DOTGLOBAL_TID_:%.*]], ptr noalias [[DOTBOUND_TID_:%.*]]) #[[ATTR1]] {
@@ -2097,48 +1410,12 @@ define internal void @__omp_outlined__4(ptr noalias %.global_tid., ptr noalias %
 ; NVPTX-DISABLED2-NEXT:    call void @spmd_amenable() #[[ATTR7]]
 ; NVPTX-DISABLED2-NEXT:    ret void
 ; NVPTX-DISABLED2:       for.body:
-; NVPTX-DISABLED2-NEXT:    store ptr addrspacecast (ptr addrspace(3) @x_shared to ptr), ptr [[CAPTURED_VARS_ADDRS]], align 8, !tbaa [[TBAA26:![0-9]+]]
-; NVPTX-DISABLED2-NEXT:    [[TMP0:%.*]] = load i32, ptr [[DOTGLOBAL_TID_]], align 4, !tbaa [[TBAA18]]
-; NVPTX-DISABLED2-NEXT:    call void @__kmpc_parallel_51(ptr @[[GLOB1]], i32 [[TMP0]], i32 1, i32 -1, i32 -1, ptr @__omp_outlined__5, ptr @__omp_outlined__5_wrapper.ID, ptr [[CAPTURED_VARS_ADDRS]], i64 1)
+; NVPTX-DISABLED2-NEXT:    store ptr addrspacecast (ptr addrspace(3) @x_shared to ptr), ptr [[CAPTURED_VARS_ADDRS]], align 8, !tbaa [[TBAA20:![0-9]+]]
+; NVPTX-DISABLED2-NEXT:    [[TMP0:%.*]] = load i32, ptr [[DOTGLOBAL_TID_]], align 4, !tbaa [[TBAA12]]
+; NVPTX-DISABLED2-NEXT:    call void @__kmpc_parallel_51(ptr @[[GLOB1]], i32 [[TMP0]], i32 1, i32 -1, i32 -1, ptr @__omp_outlined__5, ptr @__omp_outlined__5_wrapper, ptr [[CAPTURED_VARS_ADDRS]], i64 1)
 ; NVPTX-DISABLED2-NEXT:    [[INC]] = add nsw i32 [[I_0]], 1
-; NVPTX-DISABLED2-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP28:![0-9]+]]
-;
-; AMDGPU-DISABLED-LABEL: define {{[^@]+}}@__omp_outlined__4
-; AMDGPU-DISABLED-SAME: (ptr noalias [[DOTGLOBAL_TID_:%.*]], ptr noalias [[DOTBOUND_TID_:%.*]]) #[[ATTR1]] {
-; AMDGPU-DISABLED-NEXT:  entry:
-; AMDGPU-DISABLED-NEXT:    [[CAPTURED_VARS_ADDRS:%.*]] = alloca [1 x ptr], align 8
-; AMDGPU-DISABLED-NEXT:    br label [[FOR_COND:%.*]]
-; AMDGPU-DISABLED:       for.cond:
-; AMDGPU-DISABLED-NEXT:    [[I_0:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[INC:%.*]], [[FOR_BODY:%.*]] ]
-; AMDGPU-DISABLED-NEXT:    [[CMP:%.*]] = icmp slt i32 [[I_0]], 100
-; AMDGPU-DISABLED-NEXT:    br i1 [[CMP]], label [[FOR_BODY]], label [[FOR_COND_CLEANUP:%.*]]
-; AMDGPU-DISABLED:       for.cond.cleanup:
-; AMDGPU-DISABLED-NEXT:    call void @spmd_amenable() #[[ATTR7]]
-; AMDGPU-DISABLED-NEXT:    ret void
-; AMDGPU-DISABLED:       for.body:
-; AMDGPU-DISABLED-NEXT:    store ptr addrspacecast (ptr addrspace(3) @x_shared to ptr), ptr [[CAPTURED_VARS_ADDRS]], align 8, !tbaa [[TBAA26:![0-9]+]]
-; AMDGPU-DISABLED-NEXT:    [[TMP0:%.*]] = load i32, ptr [[DOTGLOBAL_TID_]], align 4, !tbaa [[TBAA18]]
-; AMDGPU-DISABLED-NEXT:    call void @__kmpc_parallel_51(ptr @[[GLOB1]], i32 [[TMP0]], i32 1, i32 -1, i32 -1, ptr @__omp_outlined__5, ptr @__omp_outlined__5_wrapper.ID, ptr [[CAPTURED_VARS_ADDRS]], i64 1)
-; AMDGPU-DISABLED-NEXT:    [[INC]] = add nsw i32 [[I_0]], 1
-; AMDGPU-DISABLED-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP28:![0-9]+]]
-; NVPTX-DISABLED-LABEL: define {{[^@]+}}@__omp_outlined__4
-; NVPTX-DISABLED-SAME: (ptr noalias [[DOTGLOBAL_TID_:%.*]], ptr noalias [[DOTBOUND_TID_:%.*]]) #[[ATTR1]] {
-; NVPTX-DISABLED-NEXT:  entry:
-; NVPTX-DISABLED-NEXT:    [[CAPTURED_VARS_ADDRS:%.*]] = alloca [1 x ptr], align 8
-; NVPTX-DISABLED-NEXT:    br label [[FOR_COND:%.*]]
-; NVPTX-DISABLED:       for.cond:
-; NVPTX-DISABLED-NEXT:    [[I_0:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[INC:%.*]], [[FOR_BODY:%.*]] ]
-; NVPTX-DISABLED-NEXT:    [[CMP:%.*]] = icmp slt i32 [[I_0]], 100
-; NVPTX-DISABLED-NEXT:    br i1 [[CMP]], label [[FOR_BODY]], label [[FOR_COND_CLEANUP:%.*]]
-; NVPTX-DISABLED:       for.cond.cleanup:
-; NVPTX-DISABLED-NEXT:    call void @spmd_amenable() #[[ATTR7]]
-; NVPTX-DISABLED-NEXT:    ret void
-; NVPTX-DISABLED:       for.body:
-; NVPTX-DISABLED-NEXT:    store ptr addrspacecast (ptr addrspace(3) @x_shared to ptr), ptr [[CAPTURED_VARS_ADDRS]], align 8, !tbaa [[TBAA26:![0-9]+]]
-; NVPTX-DISABLED-NEXT:    [[TMP0:%.*]] = load i32, ptr [[DOTGLOBAL_TID_]], align 4, !tbaa [[TBAA18]]
-; NVPTX-DISABLED-NEXT:    call void @__kmpc_parallel_51(ptr @[[GLOB1]], i32 [[TMP0]], i32 1, i32 -1, i32 -1, ptr @__omp_outlined__5, ptr @__omp_outlined__5_wrapper.ID, ptr [[CAPTURED_VARS_ADDRS]], i64 1)
-; NVPTX-DISABLED-NEXT:    [[INC]] = add nsw i32 [[I_0]], 1
-; NVPTX-DISABLED-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP28:![0-9]+]]
+; NVPTX-DISABLED2-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP22:![0-9]+]]
+;
 entry:
   %captured_vars_addrs = alloca [1 x ptr], align 8
   %x = call align 4 ptr @__kmpc_alloc_shared(i64 4)
@@ -2167,73 +1444,57 @@ define internal void @__omp_outlined__5(ptr noalias %.global_tid., ptr noalias %
 ; AMDGPU-LABEL: define {{[^@]+}}@__omp_outlined__5
 ; AMDGPU-SAME: (ptr noalias [[DOTGLOBAL_TID_:%.*]], ptr noalias [[DOTBOUND_TID_:%.*]], ptr nonnull align 4 dereferenceable(4) [[X:%.*]]) {
 ; AMDGPU-NEXT:  entry:
-; AMDGPU-NEXT:    [[TMP0:%.*]] = load i32, ptr [[X]], align 4, !tbaa [[TBAA18]]
+; AMDGPU-NEXT:    [[TMP0:%.*]] = load i32, ptr [[X]], align 4, !tbaa [[TBAA12]]
 ; AMDGPU-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP0]], 1
-; AMDGPU-NEXT:    store i32 [[INC]], ptr [[X]], align 4, !tbaa [[TBAA18]]
+; AMDGPU-NEXT:    store i32 [[INC]], ptr [[X]], align 4, !tbaa [[TBAA12]]
 ; AMDGPU-NEXT:    call void @unknown() #[[ATTR8]]
 ; AMDGPU-NEXT:    ret void
 ;
 ; NVPTX-LABEL: define {{[^@]+}}@__omp_outlined__5
 ; NVPTX-SAME: (ptr noalias [[DOTGLOBAL_TID_:%.*]], ptr noalias [[DOTBOUND_TID_:%.*]], ptr nonnull align 4 dereferenceable(4) [[X:%.*]]) {
 ; NVPTX-NEXT:  entry:
-; NVPTX-NEXT:    [[TMP0:%.*]] = load i32, ptr [[X]], align 4, !tbaa [[TBAA18]]
+; NVPTX-NEXT:    [[TMP0:%.*]] = load i32, ptr [[X]], align 4, !tbaa [[TBAA12]]
 ; NVPTX-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP0]], 1
-; NVPTX-NEXT:    store i32 [[INC]], ptr [[X]], align 4, !tbaa [[TBAA18]]
+; NVPTX-NEXT:    store i32 [[INC]], ptr [[X]], align 4, !tbaa [[TBAA12]]
 ; NVPTX-NEXT:    call void @unknown() #[[ATTR8]]
 ; NVPTX-NEXT:    ret void
 ;
 ; AMDGPU-DISABLED1-LABEL: define {{[^@]+}}@__omp_outlined__5
 ; AMDGPU-DISABLED1-SAME: (ptr noalias [[DOTGLOBAL_TID_:%.*]], ptr noalias [[DOTBOUND_TID_:%.*]], ptr nonnull align 4 dereferenceable(4) [[X:%.*]]) {
 ; AMDGPU-DISABLED1-NEXT:  entry:
-; AMDGPU-DISABLED1-NEXT:    [[TMP0:%.*]] = load i32, ptr [[X]], align 4, !tbaa [[TBAA18]]
+; AMDGPU-DISABLED1-NEXT:    [[TMP0:%.*]] = load i32, ptr [[X]], align 4, !tbaa [[TBAA12]]
 ; AMDGPU-DISABLED1-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP0]], 1
-; AMDGPU-DISABLED1-NEXT:    store i32 [[INC]], ptr [[X]], align 4, !tbaa [[TBAA18]]
+; AMDGPU-DISABLED1-NEXT:    store i32 [[INC]], ptr [[X]], align 4, !tbaa [[TBAA12]]
 ; AMDGPU-DISABLED1-NEXT:    call void @unknown() #[[ATTR8]]
 ; AMDGPU-DISABLED1-NEXT:    ret void
 ;
 ; AMDGPU-DISABLED2-LABEL: define {{[^@]+}}@__omp_outlined__5
 ; AMDGPU-DISABLED2-SAME: (ptr noalias [[DOTGLOBAL_TID_:%.*]], ptr noalias [[DOTBOUND_TID_:%.*]], ptr nonnull align 4 dereferenceable(4) [[X:%.*]]) {
 ; AMDGPU-DISABLED2-NEXT:  entry:
-; AMDGPU-DISABLED2-NEXT:    [[TMP0:%.*]] = load i32, ptr [[X]], align 4, !tbaa [[TBAA18]]
+; AMDGPU-DISABLED2-NEXT:    [[TMP0:%.*]] = load i32, ptr [[X]], align 4, !tbaa [[TBAA12]]
 ; AMDGPU-DISABLED2-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP0]], 1
-; AMDGPU-DISABLED2-NEXT:    store i32 [[INC]], ptr [[X]], align 4, !tbaa [[TBAA18]]
+; AMDGPU-DISABLED2-NEXT:    store i32 [[INC]], ptr [[X]], align 4, !tbaa [[TBAA12]]
 ; AMDGPU-DISABLED2-NEXT:    call void @unknown() #[[ATTR8]]
 ; AMDGPU-DISABLED2-NEXT:    ret void
 ;
 ; NVPTX-DISABLED1-LABEL: define {{[^@]+}}@__omp_outlined__5
 ; NVPTX-DISABLED1-SAME: (ptr noalias [[DOTGLOBAL_TID_:%.*]], ptr noalias [[DOTBOUND_TID_:%.*]], ptr nonnull align 4 dereferenceable(4) [[X:%.*]]) {
 ; NVPTX-DISABLED1-NEXT:  entry:
-; NVPTX-DISABLED1-NEXT:    [[TMP0:%.*]] = load i32, ptr [[X]], align 4, !tbaa [[TBAA18]]
+; NVPTX-DISABLED1-NEXT:    [[TMP0:%.*]] = load i32, ptr [[X]], align 4, !tbaa [[TBAA12]]
 ; NVPTX-DISABLED1-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP0]], 1
-; NVPTX-DISABLED1-NEXT:    store i32 [[INC]], ptr [[X]], align 4, !tbaa [[TBAA18]]
+; NVPTX-DISABLED1-NEXT:    store i32 [[INC]], ptr [[X]], align 4, !tbaa [[TBAA12]]
 ; NVPTX-DISABLED1-NEXT:    call void @unknown() #[[ATTR8]]
 ; NVPTX-DISABLED1-NEXT:    ret void
 ;
 ; NVPTX-DISABLED2-LABEL: define {{[^@]+}}@__omp_outlined__5
 ; NVPTX-DISABLED2-SAME: (ptr noalias [[DOTGLOBAL_TID_:%.*]], ptr noalias [[DOTBOUND_TID_:%.*]], ptr nonnull align 4 dereferenceable(4) [[X:%.*]]) {
 ; NVPTX-DISABLED2-NEXT:  entry:
-; NVPTX-DISABLED2-NEXT:    [[TMP0:%.*]] = load i32, ptr [[X]], align 4, !tbaa [[TBAA18]]
+; NVPTX-DISABLED2-NEXT:    [[TMP0:%.*]] = load i32, ptr [[X]], align 4, !tbaa [[TBAA12]]
 ; NVPTX-DISABLED2-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP0]], 1
-; NVPTX-DISABLED2-NEXT:    store i32 [[INC]], ptr [[X]], align 4, !tbaa [[TBAA18]]
+; NVPTX-DISABLED2-NEXT:    store i32 [[INC]], ptr [[X]], align 4, !tbaa [[TBAA12]]
 ; NVPTX-DISABLED2-NEXT:    call void @unknown() #[[ATTR8]]
 ; NVPTX-DISABLED2-NEXT:    ret void
 ;
-; AMDGPU-DISABLED-LABEL: define {{[^@]+}}@__omp_outlined__5
-; AMDGPU-DISABLED-SAME: (ptr noalias [[DOTGLOBAL_TID_:%.*]], ptr noalias [[DOTBOUND_TID_:%.*]], ptr nonnull align 4 dereferenceable(4) [[X:%.*]]) {
-; AMDGPU-DISABLED-NEXT:  entry:
-; AMDGPU-DISABLED-NEXT:    [[TMP0:%.*]] = load i32, ptr [[X]], align 4, !tbaa [[TBAA18]]
-; AMDGPU-DISABLED-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP0]], 1
-; AMDGPU-DISABLED-NEXT:    store i32 [[INC]], ptr [[X]], align 4, !tbaa [[TBAA18]]
-; AMDGPU-DISABLED-NEXT:    call void @unknown() #[[ATTR8]]
-; AMDGPU-DISABLED-NEXT:    ret void
-; NVPTX-DISABLED-LABEL: define {{[^@]+}}@__omp_outlined__5
-; NVPTX-DISABLED-SAME: (ptr noalias [[DOTGLOBAL_TID_:%.*]], ptr noalias [[DOTBOUND_TID_:%.*]], ptr nonnull align 4 dereferenceable(4) [[X:%.*]]) {
-; NVPTX-DISABLED-NEXT:  entry:
-; NVPTX-DISABLED-NEXT:    [[TMP0:%.*]] = load i32, ptr [[X]], align 4, !tbaa [[TBAA18]]
-; NVPTX-DISABLED-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP0]], 1
-; NVPTX-DISABLED-NEXT:    store i32 [[INC]], ptr [[X]], align 4, !tbaa [[TBAA18]]
-; NVPTX-DISABLED-NEXT:    call void @unknown() #[[ATTR8]]
-; NVPTX-DISABLED-NEXT:    ret void
 entry:
   %0 = load i32, ptr %x, align 4, !tbaa !18
   %inc = add nsw i32 %0, 1
@@ -2252,7 +1513,7 @@ define internal void @__omp_outlined__5_wrapper(i16 zeroext %0, i32 %1) #3 {
 ; AMDGPU-NEXT:    [[GLOBAL_ARGS:%.*]] = alloca ptr, align 8
 ; AMDGPU-NEXT:    call void @__kmpc_get_shared_variables(ptr [[GLOBAL_ARGS]])
 ; AMDGPU-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[GLOBAL_ARGS]], align 8
-; AMDGPU-NEXT:    [[TMP3:%.*]] = load ptr, ptr [[TMP2]], align 8, !tbaa [[TBAA26]]
+; AMDGPU-NEXT:    [[TMP3:%.*]] = load ptr, ptr [[TMP2]], align 8, !tbaa [[TBAA20]]
 ; AMDGPU-NEXT:    call void @__omp_outlined__5(ptr [[DOTADDR1]], ptr [[DOTZERO_ADDR]], ptr [[TMP3]]) #[[ATTR4]]
 ; AMDGPU-NEXT:    ret void
 ;
@@ -2264,7 +1525,7 @@ define internal void @__omp_outlined__5_wrapper(i16 zeroext %0, i32 %1) #3 {
 ; NVPTX-NEXT:    [[GLOBAL_ARGS:%.*]] = alloca ptr, align 8
 ; NVPTX-NEXT:    call void @__kmpc_get_shared_variables(ptr [[GLOBAL_ARGS]])
 ; NVPTX-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[GLOBAL_ARGS]], align 8
-; NVPTX-NEXT:    [[TMP3:%.*]] = load ptr, ptr [[TMP2]], align 8, !tbaa [[TBAA26]]
+; NVPTX-NEXT:    [[TMP3:%.*]] = load ptr, ptr [[TMP2]], align 8, !tbaa [[TBAA20]]
 ; NVPTX-NEXT:    call void @__omp_outlined__5(ptr [[DOTADDR1]], ptr [[DOTZERO_ADDR]], ptr [[TMP3]]) #[[ATTR4]]
 ; NVPTX-NEXT:    ret void
 ;
@@ -2276,7 +1537,7 @@ define internal void @__omp_outlined__5_wrapper(i16 zeroext %0, i32 %1) #3 {
 ; AMDGPU-DISABLED1-NEXT:    [[GLOBAL_ARGS:%.*]] = alloca ptr, align 8
 ; AMDGPU-DISABLED1-NEXT:    call void @__kmpc_get_shared_variables(ptr [[GLOBAL_ARGS]])
 ; AMDGPU-DISABLED1-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[GLOBAL_ARGS]], align 8
-; AMDGPU-DISABLED1-NEXT:    [[TMP3:%.*]] = load ptr, ptr [[TMP2]], align 8, !tbaa [[TBAA26]]
+; AMDGPU-DISABLED1-NEXT:    [[TMP3:%.*]] = load ptr, ptr [[TMP2]], align 8, !tbaa [[TBAA20]]
 ; AMDGPU-DISABLED1-NEXT:    call void @__omp_outlined__5(ptr [[DOTADDR1]], ptr [[DOTZERO_ADDR]], ptr [[TMP3]]) #[[ATTR4]]
 ; AMDGPU-DISABLED1-NEXT:    ret void
 ;
@@ -2288,7 +1549,7 @@ define internal void @__omp_outlined__5_wrapper(i16 zeroext %0, i32 %1) #3 {
 ; AMDGPU-DISABLED2-NEXT:    [[GLOBAL_ARGS:%.*]] = alloca ptr, align 8
 ; AMDGPU-DISABLED2-NEXT:    call void @__kmpc_get_shared_variables(ptr [[GLOBAL_ARGS]])
 ; AMDGPU-DISABLED2-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[GLOBAL_ARGS]], align 8
-; AMDGPU-DISABLED2-NEXT:    [[TMP3:%.*]] = load ptr, ptr [[TMP2]], align 8, !tbaa [[TBAA26]]
+; AMDGPU-DISABLED2-NEXT:    [[TMP3:%.*]] = load ptr, ptr [[TMP2]], align 8, !tbaa [[TBAA20]]
 ; AMDGPU-DISABLED2-NEXT:    call void @__omp_outlined__5(ptr [[DOTADDR1]], ptr [[DOTZERO_ADDR]], ptr [[TMP3]]) #[[ATTR4]]
 ; AMDGPU-DISABLED2-NEXT:    ret void
 ;
@@ -2300,7 +1561,7 @@ define internal void @__omp_outlined__5_wrapper(i16 zeroext %0, i32 %1) #3 {
 ; NVPTX-DISABLED1-NEXT:    [[GLOBAL_ARGS:%.*]] = alloca ptr, align 8
 ; NVPTX-DISABLED1-NEXT:    call void @__kmpc_get_shared_variables(ptr [[GLOBAL_ARGS]])
 ; NVPTX-DISABLED1-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[GLOBAL_ARGS]], align 8
-; NVPTX-DISABLED1-NEXT:    [[TMP3:%.*]] = load ptr, ptr [[TMP2]], align 8, !tbaa [[TBAA26]]
+; NVPTX-DISABLED1-NEXT:    [[TMP3:%.*]] = load ptr, ptr [[TMP2]], align 8, !tbaa [[TBAA20]]
 ; NVPTX-DISABLED1-NEXT:    call void @__omp_outlined__5(ptr [[DOTADDR1]], ptr [[DOTZERO_ADDR]], ptr [[TMP3]]) #[[ATTR4]]
 ; NVPTX-DISABLED1-NEXT:    ret void
 ;
@@ -2312,32 +1573,10 @@ define internal void @__omp_outlined__5_wrapper(i16 zeroext %0, i32 %1) #3 {
 ; NVPTX-DISABLED2-NEXT:    [[GLOBAL_ARGS:%.*]] = alloca ptr, align 8
 ; NVPTX-DISABLED2-NEXT:    call void @__kmpc_get_shared_variables(ptr [[GLOBAL_ARGS]])
 ; NVPTX-DISABLED2-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[GLOBAL_ARGS]], align 8
-; NVPTX-DISABLED2-NEXT:    [[TMP3:%.*]] = load ptr, ptr [[TMP2]], align 8, !tbaa [[TBAA26]]
+; NVPTX-DISABLED2-NEXT:    [[TMP3:%.*]] = load ptr, ptr [[TMP2]], align 8, !tbaa [[TBAA20]]
 ; NVPTX-DISABLED2-NEXT:    call void @__omp_outlined__5(ptr [[DOTADDR1]], ptr [[DOTZERO_ADDR]], ptr [[TMP3]]) #[[ATTR4]]
 ; NVPTX-DISABLED2-NEXT:    ret void
 ;
-; AMDGPU-DISABLED-LABEL: define {{[^@]+}}@__omp_outlined__5_wrapper
-; AMDGPU-DISABLED-SAME: (i16 zeroext [[TMP0:%.*]], i32 [[TMP1:%.*]]) #[[ATTR2]] {
-; AMDGPU-DISABLED-NEXT:  entry:
-; AMDGPU-DISABLED-NEXT:    [[DOTADDR1:%.*]] = alloca i32, align 4
-; AMDGPU-DISABLED-NEXT:    [[DOTZERO_ADDR:%.*]] = alloca i32, align 4
-; AMDGPU-DISABLED-NEXT:    [[GLOBAL_ARGS:%.*]] = alloca ptr, align 8
-; AMDGPU-DISABLED-NEXT:    call void @__kmpc_get_shared_variables(ptr [[GLOBAL_ARGS]])
-; AMDGPU-DISABLED-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[GLOBAL_ARGS]], align 8
-; AMDGPU-DISABLED-NEXT:    [[TMP3:%.*]] = load ptr, ptr [[TMP2]], align 8, !tbaa [[TBAA26]]
-; AMDGPU-DISABLED-NEXT:    call void @__omp_outlined__5(ptr [[DOTADDR1]], ptr [[DOTZERO_ADDR]], ptr [[TMP3]]) #[[ATTR4]]
-; AMDGPU-DISABLED-NEXT:    ret void
-; NVPTX-DISABLED-LABEL: define {{[^@]+}}@__omp_outlined__5_wrapper
-; NVPTX-DISABLED-SAME: (i16 zeroext [[TMP0:%.*]], i32 [[TMP1:%.*]]) #[[ATTR2]] {
-; NVPTX-DISABLED-NEXT:  entry:
-; NVPTX-DISABLED-NEXT:    [[DOTADDR1:%.*]] = alloca i32, align 4
-; NVPTX-DISABLED-NEXT:    [[DOTZERO_ADDR:%.*]] = alloca i32, align 4
-; NVPTX-DISABLED-NEXT:    [[GLOBAL_ARGS:%.*]] = alloca ptr, align 8
-; NVPTX-DISABLED-NEXT:    call void @__kmpc_get_shared_variables(ptr [[GLOBAL_ARGS]])
-; NVPTX-DISABLED-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[GLOBAL_ARGS]], align 8
-; NVPTX-DISABLED-NEXT:    [[TMP3:%.*]] = load ptr, ptr [[TMP2]], align 8, !tbaa [[TBAA26]]
-; NVPTX-DISABLED-NEXT:    call void @__omp_outlined__5(ptr [[DOTADDR1]], ptr [[DOTZERO_ADDR]], ptr [[TMP3]]) #[[ATTR4]]
-; NVPTX-DISABLED-NEXT:    ret void
 entry:
   %.addr1 = alloca i32, align 4
   %.zero.addr = alloca i32, align 4
@@ -2352,7 +1591,7 @@ entry:
 }
 
 ; Function Attrs: alwaysinline convergent norecurse nounwind
-define weak void @__omp_offloading_fd02_2044372e_sequential_loop_to_shared_var_guarded_l50() #0 {
+define weak ptx_kernel void @__omp_offloading_fd02_2044372e_sequential_loop_to_shared_var_guarded_l50() #0 {
 ; AMDGPU-LABEL: define {{[^@]+}}@__omp_offloading_fd02_2044372e_sequential_loop_to_shared_var_guarded_l50
 ; AMDGPU-SAME: () #[[ATTR0]] {
 ; AMDGPU-NEXT:  entry:
@@ -2365,7 +1604,7 @@ define weak void @__omp_offloading_fd02_2044372e_sequential_loop_to_shared_var_g
 ; AMDGPU-NEXT:    ret void
 ; AMDGPU:       user_code.entry:
 ; AMDGPU-NEXT:    [[TMP1:%.*]] = call i32 @__kmpc_global_thread_num(ptr @[[GLOB1]]) #[[ATTR4]]
-; AMDGPU-NEXT:    store i32 [[TMP1]], ptr [[DOTTHREADID_TEMP_]], align 4, !tbaa [[TBAA18]]
+; AMDGPU-NEXT:    store i32 [[TMP1]], ptr [[DOTTHREADID_TEMP_]], align 4, !tbaa [[TBAA12]]
 ; AMDGPU-NEXT:    call void @__omp_outlined__6(ptr [[DOTTHREADID_TEMP_]], ptr [[DOTZERO_ADDR]]) #[[ATTR4]]
 ; AMDGPU-NEXT:    call void @__kmpc_target_deinit()
 ; AMDGPU-NEXT:    br label [[COMMON_RET]]
@@ -2382,7 +1621,7 @@ define weak void @__omp_offloading_fd02_2044372e_sequential_loop_to_shared_var_g
 ; NVPTX-NEXT:    ret void
 ; NVPTX:       user_code.entry:
 ; NVPTX-NEXT:    [[TMP1:%.*]] = call i32 @__kmpc_global_thread_num(ptr @[[GLOB1]]) #[[ATTR4]]
-; NVPTX-NEXT:    store i32 [[TMP1]], ptr [[DOTTHREADID_TEMP_]], align 4, !tbaa [[TBAA18]]
+; NVPTX-NEXT:    store i32 [[TMP1]], ptr [[DOTTHREADID_TEMP_]], align 4, !tbaa [[TBAA12]]
 ; NVPTX-NEXT:    call void @__omp_outlined__6(ptr [[DOTTHREADID_TEMP_]], ptr [[DOTZERO_ADDR]]) #[[ATTR4]]
 ; NVPTX-NEXT:    call void @__kmpc_target_deinit()
 ; NVPTX-NEXT:    br label [[COMMON_RET]]
@@ -2433,7 +1672,7 @@ define weak void @__omp_offloading_fd02_2044372e_sequential_loop_to_shared_var_g
 ; AMDGPU-DISABLED1-NEXT:    ret void
 ; AMDGPU-DISABLED1:       user_code.entry:
 ; AMDGPU-DISABLED1-NEXT:    [[TMP1:%.*]] = call i32 @__kmpc_global_thread_num(ptr @[[GLOB1]]) #[[ATTR4]]
-; AMDGPU-DISABLED1-NEXT:    store i32 [[TMP1]], ptr [[DOTTHREADID_TEMP_]], align 4, !tbaa [[TBAA18]]
+; AMDGPU-DISABLED1-NEXT:    store i32 [[TMP1]], ptr [[DOTTHREADID_TEMP_]], align 4, !tbaa [[TBAA12]]
 ; AMDGPU-DISABLED1-NEXT:    call void @__omp_outlined__6(ptr [[DOTTHREADID_TEMP_]], ptr [[DOTZERO_ADDR]]) #[[ATTR4]]
 ; AMDGPU-DISABLED1-NEXT:    call void @__kmpc_target_deinit()
 ; AMDGPU-DISABLED1-NEXT:    br label [[COMMON_RET]]
@@ -2441,50 +1680,16 @@ define weak void @__omp_offloading_fd02_2044372e_sequential_loop_to_shared_var_g
 ; AMDGPU-DISABLED2-LABEL: define {{[^@]+}}@__omp_offloading_fd02_2044372e_sequential_loop_to_shared_var_guarded_l50
 ; AMDGPU-DISABLED2-SAME: () #[[ATTR0]] {
 ; AMDGPU-DISABLED2-NEXT:  entry:
-; AMDGPU-DISABLED2-NEXT:    [[WORKER_WORK_FN_ADDR:%.*]] = alloca ptr, align 8, addrspace(5)
 ; AMDGPU-DISABLED2-NEXT:    [[DOTZERO_ADDR:%.*]] = alloca i32, align 4
 ; AMDGPU-DISABLED2-NEXT:    [[DOTTHREADID_TEMP_:%.*]] = alloca i32, align 4
 ; AMDGPU-DISABLED2-NEXT:    [[TMP0:%.*]] = call i32 @__kmpc_target_init(ptr @__omp_offloading_fd02_2044372e_sequential_loop_to_shared_var_guarded_l50_kernel_environment, ptr null)
-; AMDGPU-DISABLED2-NEXT:    [[THREAD_IS_WORKER:%.*]] = icmp ne i32 [[TMP0]], -1
-; AMDGPU-DISABLED2-NEXT:    br i1 [[THREAD_IS_WORKER]], label [[IS_WORKER_CHECK:%.*]], label [[THREAD_USER_CODE_CHECK:%.*]]
-; AMDGPU-DISABLED2:       is_worker_check:
-; AMDGPU-DISABLED2-NEXT:    [[BLOCK_HW_SIZE:%.*]] = call i32 @__kmpc_get_hardware_num_threads_in_block()
-; AMDGPU-DISABLED2-NEXT:    [[WARP_SIZE:%.*]] = call i32 @__kmpc_get_warp_size()
-; AMDGPU-DISABLED2-NEXT:    [[BLOCK_SIZE:%.*]] = sub i32 [[BLOCK_HW_SIZE]], [[WARP_SIZE]]
-; AMDGPU-DISABLED2-NEXT:    [[THREAD_IS_MAIN_OR_WORKER:%.*]] = icmp slt i32 [[TMP0]], [[BLOCK_SIZE]]
-; AMDGPU-DISABLED2-NEXT:    br i1 [[THREAD_IS_MAIN_OR_WORKER]], label [[WORKER_STATE_MACHINE_BEGIN:%.*]], label [[WORKER_STATE_MACHINE_FINISHED:%.*]]
-; AMDGPU-DISABLED2:       worker_state_machine.begin:
-; AMDGPU-DISABLED2-NEXT:    call void @__kmpc_barrier_simple_generic(ptr @[[GLOB1]], i32 [[TMP0]])
-; AMDGPU-DISABLED2-NEXT:    [[WORKER_WORK_FN_ADDR_GENERIC:%.*]] = addrspacecast ptr addrspace(5) [[WORKER_WORK_FN_ADDR]] to ptr
-; AMDGPU-DISABLED2-NEXT:    [[WORKER_IS_ACTIVE:%.*]] = call i1 @__kmpc_kernel_parallel(ptr [[WORKER_WORK_FN_ADDR_GENERIC]])
-; AMDGPU-DISABLED2-NEXT:    [[WORKER_WORK_FN:%.*]] = load ptr, ptr [[WORKER_WORK_FN_ADDR_GENERIC]], align 8
-; AMDGPU-DISABLED2-NEXT:    [[WORKER_IS_DONE:%.*]] = icmp eq ptr [[WORKER_WORK_FN]], null
-; AMDGPU-DISABLED2-NEXT:    br i1 [[WORKER_IS_DONE]], label [[WORKER_STATE_MACHINE_FINISHED]], label [[WORKER_STATE_MACHINE_IS_ACTIVE_CHECK:%.*]]
-; AMDGPU-DISABLED2:       worker_state_machine.finished:
-; AMDGPU-DISABLED2-NEXT:    ret void
-; AMDGPU-DISABLED2:       worker_state_machine.is_active.check:
-; AMDGPU-DISABLED2-NEXT:    br i1 [[WORKER_IS_ACTIVE]], label [[WORKER_STATE_MACHINE_PARALLEL_REGION_CHECK:%.*]], label [[WORKER_STATE_MACHINE_DONE_BARRIER:%.*]]
-; AMDGPU-DISABLED2:       worker_state_machine.parallel_region.check:
-; AMDGPU-DISABLED2-NEXT:    br i1 true, label [[WORKER_STATE_MACHINE_PARALLEL_REGION_EXECUTE:%.*]], label [[WORKER_STATE_MACHINE_PARALLEL_REGION_CHECK1:%.*]]
-; AMDGPU-DISABLED2:       worker_state_machine.parallel_region.execute:
-; AMDGPU-DISABLED2-NEXT:    call void @__omp_outlined__7_wrapper(i16 0, i32 [[TMP0]])
-; AMDGPU-DISABLED2-NEXT:    br label [[WORKER_STATE_MACHINE_PARALLEL_REGION_END:%.*]]
-; AMDGPU-DISABLED2:       worker_state_machine.parallel_region.check1:
-; AMDGPU-DISABLED2-NEXT:    br label [[WORKER_STATE_MACHINE_PARALLEL_REGION_END]]
-; AMDGPU-DISABLED2:       worker_state_machine.parallel_region.end:
-; AMDGPU-DISABLED2-NEXT:    call void @__kmpc_kernel_end_parallel()
-; AMDGPU-DISABLED2-NEXT:    br label [[WORKER_STATE_MACHINE_DONE_BARRIER]]
-; AMDGPU-DISABLED2:       worker_state_machine.done.barrier:
-; AMDGPU-DISABLED2-NEXT:    call void @__kmpc_barrier_simple_generic(ptr @[[GLOB1]], i32 [[TMP0]])
-; AMDGPU-DISABLED2-NEXT:    br label [[WORKER_STATE_MACHINE_BEGIN]]
-; AMDGPU-DISABLED2:       thread.user_code.check:
 ; AMDGPU-DISABLED2-NEXT:    [[EXEC_USER_CODE:%.*]] = icmp eq i32 [[TMP0]], -1
 ; AMDGPU-DISABLED2-NEXT:    br i1 [[EXEC_USER_CODE]], label [[USER_CODE_ENTRY:%.*]], label [[COMMON_RET:%.*]]
 ; AMDGPU-DISABLED2:       common.ret:
 ; AMDGPU-DISABLED2-NEXT:    ret void
 ; AMDGPU-DISABLED2:       user_code.entry:
 ; AMDGPU-DISABLED2-NEXT:    [[TMP1:%.*]] = call i32 @__kmpc_global_thread_num(ptr @[[GLOB1]]) #[[ATTR4]]
-; AMDGPU-DISABLED2-NEXT:    store i32 [[TMP1]], ptr [[DOTTHREADID_TEMP_]], align 4, !tbaa [[TBAA18]]
+; AMDGPU-DISABLED2-NEXT:    store i32 [[TMP1]], ptr [[DOTTHREADID_TEMP_]], align 4, !tbaa [[TBAA12]]
 ; AMDGPU-DISABLED2-NEXT:    call void @__omp_outlined__6(ptr [[DOTTHREADID_TEMP_]], ptr [[DOTZERO_ADDR]]) #[[ATTR4]]
 ; AMDGPU-DISABLED2-NEXT:    call void @__kmpc_target_deinit()
 ; AMDGPU-DISABLED2-NEXT:    br label [[COMMON_RET]]
@@ -2534,7 +1739,7 @@ define weak void @__omp_offloading_fd02_2044372e_sequential_loop_to_shared_var_g
 ; NVPTX-DISABLED1-NEXT:    ret void
 ; NVPTX-DISABLED1:       user_code.entry:
 ; NVPTX-DISABLED1-NEXT:    [[TMP1:%.*]] = call i32 @__kmpc_global_thread_num(ptr @[[GLOB1]]) #[[ATTR4]]
-; NVPTX-DISABLED1-NEXT:    store i32 [[TMP1]], ptr [[DOTTHREADID_TEMP_]], align 4, !tbaa [[TBAA18]]
+; NVPTX-DISABLED1-NEXT:    store i32 [[TMP1]], ptr [[DOTTHREADID_TEMP_]], align 4, !tbaa [[TBAA12]]
 ; NVPTX-DISABLED1-NEXT:    call void @__omp_outlined__6(ptr [[DOTTHREADID_TEMP_]], ptr [[DOTZERO_ADDR]]) #[[ATTR4]]
 ; NVPTX-DISABLED1-NEXT:    call void @__kmpc_target_deinit()
 ; NVPTX-DISABLED1-NEXT:    br label [[COMMON_RET]]
@@ -2542,156 +1747,20 @@ define weak void @__omp_offloading_fd02_2044372e_sequential_loop_to_shared_var_g
 ; NVPTX-DISABLED2-LABEL: define {{[^@]+}}@__omp_offloading_fd02_2044372e_sequential_loop_to_shared_var_guarded_l50
 ; NVPTX-DISABLED2-SAME: () #[[ATTR0]] {
 ; NVPTX-DISABLED2-NEXT:  entry:
-; NVPTX-DISABLED2-NEXT:    [[WORKER_WORK_FN_ADDR:%.*]] = alloca ptr, align 8
 ; NVPTX-DISABLED2-NEXT:    [[DOTZERO_ADDR:%.*]] = alloca i32, align 4
 ; NVPTX-DISABLED2-NEXT:    [[DOTTHREADID_TEMP_:%.*]] = alloca i32, align 4
 ; NVPTX-DISABLED2-NEXT:    [[TMP0:%.*]] = call i32 @__kmpc_target_init(ptr @__omp_offloading_fd02_2044372e_sequential_loop_to_shared_var_guarded_l50_kernel_environment, ptr null)
-; NVPTX-DISABLED2-NEXT:    [[THREAD_IS_WORKER:%.*]] = icmp ne i32 [[TMP0]], -1
-; NVPTX-DISABLED2-NEXT:    br i1 [[THREAD_IS_WORKER]], label [[IS_WORKER_CHECK:%.*]], label [[THREAD_USER_CODE_CHECK:%.*]]
-; NVPTX-DISABLED2:       is_worker_check:
-; NVPTX-DISABLED2-NEXT:    [[BLOCK_HW_SIZE:%.*]] = call i32 @__kmpc_get_hardware_num_threads_in_block()
-; NVPTX-DISABLED2-NEXT:    [[WARP_SIZE:%.*]] = call i32 @__kmpc_get_warp_size()
-; NVPTX-DISABLED2-NEXT:    [[BLOCK_SIZE:%.*]] = sub i32 [[BLOCK_HW_SIZE]], [[WARP_SIZE]]
-; NVPTX-DISABLED2-NEXT:    [[THREAD_IS_MAIN_OR_WORKER:%.*]] = icmp slt i32 [[TMP0]], [[BLOCK_SIZE]]
-; NVPTX-DISABLED2-NEXT:    br i1 [[THREAD_IS_MAIN_OR_WORKER]], label [[WORKER_STATE_MACHINE_BEGIN:%.*]], label [[WORKER_STATE_MACHINE_FINISHED:%.*]]
-; NVPTX-DISABLED2:       worker_state_machine.begin:
-; NVPTX-DISABLED2-NEXT:    call void @__kmpc_barrier_simple_generic(ptr @[[GLOB1]], i32 [[TMP0]])
-; NVPTX-DISABLED2-NEXT:    [[WORKER_IS_ACTIVE:%.*]] = call i1 @__kmpc_kernel_parallel(ptr [[WORKER_WORK_FN_ADDR]])
-; NVPTX-DISABLED2-NEXT:    [[WORKER_WORK_FN:%.*]] = load ptr, ptr [[WORKER_WORK_FN_ADDR]], align 8
-; NVPTX-DISABLED2-NEXT:    [[WORKER_IS_DONE:%.*]] = icmp eq ptr [[WORKER_WORK_FN]], null
-; NVPTX-DISABLED2-NEXT:    br i1 [[WORKER_IS_DONE]], label [[WORKER_STATE_MACHINE_FINISHED]], label [[WORKER_STATE_MACHINE_IS_ACTIVE_CHECK:%.*]]
-; NVPTX-DISABLED2:       worker_state_machine.finished:
-; NVPTX-DISABLED2-NEXT:    ret void
-; NVPTX-DISABLED2:       worker_state_machine.is_active.check:
-; NVPTX-DISABLED2-NEXT:    br i1 [[WORKER_IS_ACTIVE]], label [[WORKER_STATE_MACHINE_PARALLEL_REGION_CHECK:%.*]], label [[WORKER_STATE_MACHINE_DONE_BARRIER:%.*]]
-; NVPTX-DISABLED2:       worker_state_machine.parallel_region.check:
-; NVPTX-DISABLED2-NEXT:    br i1 true, label [[WORKER_STATE_MACHINE_PARALLEL_REGION_EXECUTE:%.*]], label [[WORKER_STATE_MACHINE_PARALLEL_REGION_CHECK1:%.*]]
-; NVPTX-DISABLED2:       worker_state_machine.parallel_region.execute:
-; NVPTX-DISABLED2-NEXT:    call void @__omp_outlined__7_wrapper(i16 0, i32 [[TMP0]])
-; NVPTX-DISABLED2-NEXT:    br label [[WORKER_STATE_MACHINE_PARALLEL_REGION_END:%.*]]
-; NVPTX-DISABLED2:       worker_state_machine.parallel_region.check1:
-; NVPTX-DISABLED2-NEXT:    br label [[WORKER_STATE_MACHINE_PARALLEL_REGION_END]]
-; NVPTX-DISABLED2:       worker_state_machine.parallel_region.end:
-; NVPTX-DISABLED2-NEXT:    call void @__kmpc_kernel_end_parallel()
-; NVPTX-DISABLED2-NEXT:    br label [[WORKER_STATE_MACHINE_DONE_BARRIER]]
-; NVPTX-DISABLED2:       worker_state_machine.done.barrier:
-; NVPTX-DISABLED2-NEXT:    call void @__kmpc_barrier_simple_generic(ptr @[[GLOB1]], i32 [[TMP0]])
-; NVPTX-DISABLED2-NEXT:    br label [[WORKER_STATE_MACHINE_BEGIN]]
-; NVPTX-DISABLED2:       thread.user_code.check:
 ; NVPTX-DISABLED2-NEXT:    [[EXEC_USER_CODE:%.*]] = icmp eq i32 [[TMP0]], -1
 ; NVPTX-DISABLED2-NEXT:    br i1 [[EXEC_USER_CODE]], label [[USER_CODE_ENTRY:%.*]], label [[COMMON_RET:%.*]]
 ; NVPTX-DISABLED2:       common.ret:
 ; NVPTX-DISABLED2-NEXT:    ret void
 ; NVPTX-DISABLED2:       user_code.entry:
 ; NVPTX-DISABLED2-NEXT:    [[TMP1:%.*]] = call i32 @__kmpc_global_thread_num(ptr @[[GLOB1]]) #[[ATTR4]]
-; NVPTX-DISABLED2-NEXT:    store i32 [[TMP1]], ptr [[DOTTHREADID_TEMP_]], align 4, !tbaa [[TBAA18]]
+; NVPTX-DISABLED2-NEXT:    store i32 [[TMP1]], ptr [[DOTTHREADID_TEMP_]], align 4, !tbaa [[TBAA12]]
 ; NVPTX-DISABLED2-NEXT:    call void @__omp_outlined__6(ptr [[DOTTHREADID_TEMP_]], ptr [[DOTZERO_ADDR]]) #[[ATTR4]]
 ; NVPTX-DISABLED2-NEXT:    call void @__kmpc_target_deinit()
 ; NVPTX-DISABLED2-NEXT:    br label [[COMMON_RET]]
 ;
-; AMDGPU-DISABLED-LABEL: define {{[^@]+}}@__omp_offloading_fd02_2044372e_sequential_loop_to_shared_var_guarded_l50
-; AMDGPU-DISABLED-SAME: () #[[ATTR0]] {
-; AMDGPU-DISABLED-NEXT:  entry:
-; AMDGPU-DISABLED-NEXT:    [[WORKER_WORK_FN_ADDR:%.*]] = alloca ptr, align 8, addrspace(5)
-; AMDGPU-DISABLED-NEXT:    [[DOTZERO_ADDR:%.*]] = alloca i32, align 4
-; AMDGPU-DISABLED-NEXT:    [[DOTTHREADID_TEMP_:%.*]] = alloca i32, align 4
-; AMDGPU-DISABLED-NEXT:    [[TMP0:%.*]] = call i32 @__kmpc_target_init(ptr @__omp_offloading_fd02_2044372e_sequential_loop_to_shared_var_guarded_l50_kernel_environment)
-; AMDGPU-DISABLED-NEXT:    [[THREAD_IS_WORKER:%.*]] = icmp ne i32 [[TMP0]], -1
-; AMDGPU-DISABLED-NEXT:    br i1 [[THREAD_IS_WORKER]], label [[IS_WORKER_CHECK:%.*]], label [[THREAD_USER_CODE_CHECK:%.*]]
-; AMDGPU-DISABLED:       is_worker_check:
-; AMDGPU-DISABLED-NEXT:    [[BLOCK_HW_SIZE:%.*]] = call i32 @__kmpc_get_hardware_num_threads_in_block()
-; AMDGPU-DISABLED-NEXT:    [[WARP_SIZE:%.*]] = call i32 @__kmpc_get_warp_size()
-; AMDGPU-DISABLED-NEXT:    [[BLOCK_SIZE:%.*]] = sub i32 [[BLOCK_HW_SIZE]], [[WARP_SIZE]]
-; AMDGPU-DISABLED-NEXT:    [[THREAD_IS_MAIN_OR_WORKER:%.*]] = icmp slt i32 [[TMP0]], [[BLOCK_SIZE]]
-; AMDGPU-DISABLED-NEXT:    br i1 [[THREAD_IS_MAIN_OR_WORKER]], label [[WORKER_STATE_MACHINE_BEGIN:%.*]], label [[WORKER_STATE_MACHINE_FINISHED:%.*]]
-; AMDGPU-DISABLED:       worker_state_machine.begin:
-; AMDGPU-DISABLED-NEXT:    call void @__kmpc_barrier_simple_generic(ptr @[[GLOB1]], i32 [[TMP0]])
-; AMDGPU-DISABLED-NEXT:    [[WORKER_WORK_FN_ADDR_GENERIC:%.*]] = addrspacecast ptr addrspace(5) [[WORKER_WORK_FN_ADDR]] to ptr
-; AMDGPU-DISABLED-NEXT:    [[WORKER_IS_ACTIVE:%.*]] = call i1 @__kmpc_kernel_parallel(ptr [[WORKER_WORK_FN_ADDR_GENERIC]])
-; AMDGPU-DISABLED-NEXT:    [[WORKER_WORK_FN:%.*]] = load ptr, ptr [[WORKER_WORK_FN_ADDR_GENERIC]], align 8
-; AMDGPU-DISABLED-NEXT:    [[WORKER_IS_DONE:%.*]] = icmp eq ptr [[WORKER_WORK_FN]], null
-; AMDGPU-DISABLED-NEXT:    br i1 [[WORKER_IS_DONE]], label [[WORKER_STATE_MACHINE_FINISHED]], label [[WORKER_STATE_MACHINE_IS_ACTIVE_CHECK:%.*]]
-; AMDGPU-DISABLED:       worker_state_machine.finished:
-; AMDGPU-DISABLED-NEXT:    ret void
-; AMDGPU-DISABLED:       worker_state_machine.is_active.check:
-; AMDGPU-DISABLED-NEXT:    br i1 [[WORKER_IS_ACTIVE]], label [[WORKER_STATE_MACHINE_PARALLEL_REGION_CHECK:%.*]], label [[WORKER_STATE_MACHINE_DONE_BARRIER:%.*]]
-; AMDGPU-DISABLED:       worker_state_machine.parallel_region.check:
-; AMDGPU-DISABLED-NEXT:    [[WORKER_CHECK_PARALLEL_REGION:%.*]] = icmp eq ptr [[WORKER_WORK_FN]], @__omp_outlined__7_wrapper.ID
-; AMDGPU-DISABLED-NEXT:    br i1 [[WORKER_CHECK_PARALLEL_REGION]], label [[WORKER_STATE_MACHINE_PARALLEL_REGION_EXECUTE:%.*]], label [[WORKER_STATE_MACHINE_PARALLEL_REGION_FALLBACK_EXECUTE:%.*]]
-; AMDGPU-DISABLED:       worker_state_machine.parallel_region.execute:
-; AMDGPU-DISABLED-NEXT:    call void @__omp_outlined__7_wrapper(i16 0, i32 [[TMP0]])
-; AMDGPU-DISABLED-NEXT:    br label [[WORKER_STATE_MACHINE_PARALLEL_REGION_END:%.*]]
-; AMDGPU-DISABLED:       worker_state_machine.parallel_region.fallback.execute:
-; AMDGPU-DISABLED-NEXT:    call void [[WORKER_WORK_FN]](i16 0, i32 [[TMP0]])
-; AMDGPU-DISABLED-NEXT:    br label [[WORKER_STATE_MACHINE_PARALLEL_REGION_END]]
-; AMDGPU-DISABLED:       worker_state_machine.parallel_region.end:
-; AMDGPU-DISABLED-NEXT:    call void @__kmpc_kernel_end_parallel()
-; AMDGPU-DISABLED-NEXT:    br label [[WORKER_STATE_MACHINE_DONE_BARRIER]]
-; AMDGPU-DISABLED:       worker_state_machine.done.barrier:
-; AMDGPU-DISABLED-NEXT:    call void @__kmpc_barrier_simple_generic(ptr @[[GLOB1]], i32 [[TMP0]])
-; AMDGPU-DISABLED-NEXT:    br label [[WORKER_STATE_MACHINE_BEGIN]]
-; AMDGPU-DISABLED:       thread.user_code.check:
-; AMDGPU-DISABLED-NEXT:    [[EXEC_USER_CODE:%.*]] = icmp eq i32 [[TMP0]], -1
-; AMDGPU-DISABLED-NEXT:    br i1 [[EXEC_USER_CODE]], label [[USER_CODE_ENTRY:%.*]], label [[COMMON_RET:%.*]]
-; AMDGPU-DISABLED:       common.ret:
-; AMDGPU-DISABLED-NEXT:    ret void
-; AMDGPU-DISABLED:       user_code.entry:
-; AMDGPU-DISABLED-NEXT:    [[TMP1:%.*]] = call i32 @__kmpc_global_thread_num(ptr @[[GLOB1]]) #[[ATTR4]]
-; AMDGPU-DISABLED-NEXT:    store i32 [[TMP1]], ptr [[DOTTHREADID_TEMP_]], align 4, !tbaa [[TBAA18]]
-; AMDGPU-DISABLED-NEXT:    call void @__omp_outlined__6(ptr [[DOTTHREADID_TEMP_]], ptr [[DOTZERO_ADDR]]) #[[ATTR5]]
-; AMDGPU-DISABLED-NEXT:    call void @__kmpc_target_deinit()
-; AMDGPU-DISABLED-NEXT:    br label [[COMMON_RET]]
-; NVPTX-DISABLED-LABEL: define {{[^@]+}}@__omp_offloading_fd02_2044372e_sequential_loop_to_shared_var_guarded_l50
-; NVPTX-DISABLED-SAME: () #[[ATTR0]] {
-; NVPTX-DISABLED-NEXT:  entry:
-; NVPTX-DISABLED-NEXT:    [[WORKER_WORK_FN_ADDR:%.*]] = alloca ptr, align 8
-; NVPTX-DISABLED-NEXT:    [[DOTZERO_ADDR:%.*]] = alloca i32, align 4
-; NVPTX-DISABLED-NEXT:    [[DOTTHREADID_TEMP_:%.*]] = alloca i32, align 4
-; NVPTX-DISABLED-NEXT:    [[TMP0:%.*]] = call i32 @__kmpc_target_init(ptr @__omp_offloading_fd02_2044372e_sequential_loop_to_shared_var_guarded_l50_kernel_environment)
-; NVPTX-DISABLED-NEXT:    [[THREAD_IS_WORKER:%.*]] = icmp ne i32 [[TMP0]], -1
-; NVPTX-DISABLED-NEXT:    br i1 [[THREAD_IS_WORKER]], label [[IS_WORKER_CHECK:%.*]], label [[THREAD_USER_CODE_CHECK:%.*]]
-; NVPTX-DISABLED:       is_worker_check:
-; NVPTX-DISABLED-NEXT:    [[BLOCK_HW_SIZE:%.*]] = call i32 @__kmpc_get_hardware_num_threads_in_block()
-; NVPTX-DISABLED-NEXT:    [[WARP_SIZE:%.*]] = call i32 @__kmpc_get_warp_size()
-; NVPTX-DISABLED-NEXT:    [[BLOCK_SIZE:%.*]] = sub i32 [[BLOCK_HW_SIZE]], [[WARP_SIZE]]
-; NVPTX-DISABLED-NEXT:    [[THREAD_IS_MAIN_OR_WORKER:%.*]] = icmp slt i32 [[TMP0]], [[BLOCK_SIZE]]
-; NVPTX-DISABLED-NEXT:    br i1 [[THREAD_IS_MAIN_OR_WORKER]], label [[WORKER_STATE_MACHINE_BEGIN:%.*]], label [[WORKER_STATE_MACHINE_FINISHED:%.*]]
-; NVPTX-DISABLED:       worker_state_machine.begin:
-; NVPTX-DISABLED-NEXT:    call void @__kmpc_barrier_simple_generic(ptr @[[GLOB1]], i32 [[TMP0]])
-; NVPTX-DISABLED-NEXT:    [[WORKER_IS_ACTIVE:%.*]] = call i1 @__kmpc_kernel_parallel(ptr [[WORKER_WORK_FN_ADDR]])
-; NVPTX-DISABLED-NEXT:    [[WORKER_WORK_FN:%.*]] = load ptr, ptr [[WORKER_WORK_FN_ADDR]], align 8
-; NVPTX-DISABLED-NEXT:    [[WORKER_IS_DONE:%.*]] = icmp eq ptr [[WORKER_WORK_FN]], null
-; NVPTX-DISABLED-NEXT:    br i1 [[WORKER_IS_DONE]], label [[WORKER_STATE_MACHINE_FINISHED]], label [[WORKER_STATE_MACHINE_IS_ACTIVE_CHECK:%.*]]
-; NVPTX-DISABLED:       worker_state_machine.finished:
-; NVPTX-DISABLED-NEXT:    ret void
-; NVPTX-DISABLED:       worker_state_machine.is_active.check:
-; NVPTX-DISABLED-NEXT:    br i1 [[WORKER_IS_ACTIVE]], label [[WORKER_STATE_MACHINE_PARALLEL_REGION_CHECK:%.*]], label [[WORKER_STATE_MACHINE_DONE_BARRIER:%.*]]
-; NVPTX-DISABLED:       worker_state_machine.parallel_region.check:
-; NVPTX-DISABLED-NEXT:    [[WORKER_CHECK_PARALLEL_REGION:%.*]] = icmp eq ptr [[WORKER_WORK_FN]], @__omp_outlined__7_wrapper.ID
-; NVPTX-DISABLED-NEXT:    br i1 [[WORKER_CHECK_PARALLEL_REGION]], label [[WORKER_STATE_MACHINE_PARALLEL_REGION_EXECUTE:%.*]], label [[WORKER_STATE_MACHINE_PARALLEL_REGION_FALLBACK_EXECUTE:%.*]]
-; NVPTX-DISABLED:       worker_state_machine.parallel_region.execute:
-; NVPTX-DISABLED-NEXT:    call void @__omp_outlined__7_wrapper(i16 0, i32 [[TMP0]])
-; NVPTX-DISABLED-NEXT:    br label [[WORKER_STATE_MACHINE_PARALLEL_REGION_END:%.*]]
-; NVPTX-DISABLED:       worker_state_machine.parallel_region.fallback.execute:
-; NVPTX-DISABLED-NEXT:    call void [[WORKER_WORK_FN]](i16 0, i32 [[TMP0]])
-; NVPTX-DISABLED-NEXT:    br label [[WORKER_STATE_MACHINE_PARALLEL_REGION_END]]
-; NVPTX-DISABLED:       worker_state_machine.parallel_region.end:
-; NVPTX-DISABLED-NEXT:    call void @__kmpc_kernel_end_parallel()
-; NVPTX-DISABLED-NEXT:    br label [[WORKER_STATE_MACHINE_DONE_BARRIER]]
-; NVPTX-DISABLED:       worker_state_machine.done.barrier:
-; NVPTX-DISABLED-NEXT:    call void @__kmpc_barrier_simple_generic(ptr @[[GLOB1]], i32 [[TMP0]])
-; NVPTX-DISABLED-NEXT:    br label [[WORKER_STATE_MACHINE_BEGIN]]
-; NVPTX-DISABLED:       thread.user_code.check:
-; NVPTX-DISABLED-NEXT:    [[EXEC_USER_CODE:%.*]] = icmp eq i32 [[TMP0]], -1
-; NVPTX-DISABLED-NEXT:    br i1 [[EXEC_USER_CODE]], label [[USER_CODE_ENTRY:%.*]], label [[COMMON_RET:%.*]]
-; NVPTX-DISABLED:       common.ret:
-; NVPTX-DISABLED-NEXT:    ret void
-; NVPTX-DISABLED:       user_code.entry:
-; NVPTX-DISABLED-NEXT:    [[TMP1:%.*]] = call i32 @__kmpc_global_thread_num(ptr @[[GLOB1]]) #[[ATTR4]]
-; NVPTX-DISABLED-NEXT:    store i32 [[TMP1]], ptr [[DOTTHREADID_TEMP_]], align 4, !tbaa [[TBAA18]]
-; NVPTX-DISABLED-NEXT:    call void @__omp_outlined__6(ptr [[DOTTHREADID_TEMP_]], ptr [[DOTZERO_ADDR]]) #[[ATTR5]]
-; NVPTX-DISABLED-NEXT:    call void @__kmpc_target_deinit()
-; NVPTX-DISABLED-NEXT:    br label [[COMMON_RET]]
 entry:
   %.zero.addr = alloca i32, align 4
   %.threadid_temp. = alloca i32, align 4
@@ -2723,7 +1792,7 @@ define internal void @__omp_outlined__6(ptr noalias %.global_tid., ptr noalias %
 ; AMDGPU-NEXT:    [[TMP1:%.*]] = icmp eq i32 [[TMP0]], 0
 ; AMDGPU-NEXT:    br i1 [[TMP1]], label [[REGION_GUARDED:%.*]], label [[REGION_BARRIER:%.*]]
 ; AMDGPU:       region.guarded:
-; AMDGPU-NEXT:    store i32 42, ptr addrspacecast (ptr addrspace(3) @x_shared.1 to ptr), align 4, !tbaa [[TBAA18]]
+; AMDGPU-NEXT:    store i32 42, ptr addrspacecast (ptr addrspace(3) @x_shared.1 to ptr), align 4, !tbaa [[TBAA12]]
 ; AMDGPU-NEXT:    br label [[REGION_GUARDED_END:%.*]]
 ; AMDGPU:       region.guarded.end:
 ; AMDGPU-NEXT:    br label [[REGION_BARRIER]]
@@ -2740,11 +1809,11 @@ define internal void @__omp_outlined__6(ptr noalias %.global_tid., ptr noalias %
 ; AMDGPU-NEXT:    call void @spmd_amenable() #[[ATTR7]]
 ; AMDGPU-NEXT:    ret void
 ; AMDGPU:       for.body:
-; AMDGPU-NEXT:    store ptr addrspacecast (ptr addrspace(3) @x_shared.1 to ptr), ptr [[CAPTURED_VARS_ADDRS]], align 8, !tbaa [[TBAA26]]
-; AMDGPU-NEXT:    [[TMP2:%.*]] = load i32, ptr [[DOTGLOBAL_TID_]], align 4, !tbaa [[TBAA18]]
+; AMDGPU-NEXT:    store ptr addrspacecast (ptr addrspace(3) @x_shared.1 to ptr), ptr [[CAPTURED_VARS_ADDRS]], align 8, !tbaa [[TBAA20]]
+; AMDGPU-NEXT:    [[TMP2:%.*]] = load i32, ptr [[DOTGLOBAL_TID_]], align 4, !tbaa [[TBAA12]]
 ; AMDGPU-NEXT:    call void @__kmpc_parallel_51(ptr @[[GLOB1]], i32 [[TMP2]], i32 1, i32 -1, i32 -1, ptr @__omp_outlined__7, ptr @__omp_outlined__7_wrapper, ptr [[CAPTURED_VARS_ADDRS]], i64 1)
 ; AMDGPU-NEXT:    [[INC]] = add nsw i32 [[I_0]], 1
-; AMDGPU-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP29:![0-9]+]]
+; AMDGPU-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP23:![0-9]+]]
 ;
 ; NVPTX-LABEL: define {{[^@]+}}@__omp_outlined__6
 ; NVPTX-SAME: (ptr noalias [[DOTGLOBAL_TID_:%.*]], ptr noalias [[DOTBOUND_TID_:%.*]]) #[[ATTR1]] {
@@ -2756,7 +1825,7 @@ define internal void @__omp_outlined__6(ptr noalias %.global_tid., ptr noalias %
 ; NVPTX-NEXT:    [[TMP1:%.*]] = icmp eq i32 [[TMP0]], 0
 ; NVPTX-NEXT:    br i1 [[TMP1]], label [[REGION_GUARDED:%.*]], label [[REGION_BARRIER:%.*]]
 ; NVPTX:       region.guarded:
-; NVPTX-NEXT:    store i32 42, ptr addrspacecast (ptr addrspace(3) @x_shared1 to ptr), align 4, !tbaa [[TBAA18]]
+; NVPTX-NEXT:    store i32 42, ptr addrspacecast (ptr addrspace(3) @x_shared1 to ptr), align 4, !tbaa [[TBAA12]]
 ; NVPTX-NEXT:    br label [[REGION_GUARDED_END:%.*]]
 ; NVPTX:       region.guarded.end:
 ; NVPTX-NEXT:    br label [[REGION_BARRIER]]
@@ -2773,17 +1842,17 @@ define internal void @__omp_outlined__6(ptr noalias %.global_tid., ptr noalias %
 ; NVPTX-NEXT:    call void @spmd_amenable() #[[ATTR7]]
 ; NVPTX-NEXT:    ret void
 ; NVPTX:       for.body:
-; NVPTX-NEXT:    store ptr addrspacecast (ptr addrspace(3) @x_shared1 to ptr), ptr [[CAPTURED_VARS_ADDRS]], align 8, !tbaa [[TBAA26]]
-; NVPTX-NEXT:    [[TMP2:%.*]] = load i32, ptr [[DOTGLOBAL_TID_]], align 4, !tbaa [[TBAA18]]
+; NVPTX-NEXT:    store ptr addrspacecast (ptr addrspace(3) @x_shared1 to ptr), ptr [[CAPTURED_VARS_ADDRS]], align 8, !tbaa [[TBAA20]]
+; NVPTX-NEXT:    [[TMP2:%.*]] = load i32, ptr [[DOTGLOBAL_TID_]], align 4, !tbaa [[TBAA12]]
 ; NVPTX-NEXT:    call void @__kmpc_parallel_51(ptr @[[GLOB1]], i32 [[TMP2]], i32 1, i32 -1, i32 -1, ptr @__omp_outlined__7, ptr @__omp_outlined__7_wrapper, ptr [[CAPTURED_VARS_ADDRS]], i64 1)
 ; NVPTX-NEXT:    [[INC]] = add nsw i32 [[I_0]], 1
-; NVPTX-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP29:![0-9]+]]
+; NVPTX-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP23:![0-9]+]]
 ;
 ; AMDGPU-DISABLED1-LABEL: define {{[^@]+}}@__omp_outlined__6
 ; AMDGPU-DISABLED1-SAME: (ptr noalias [[DOTGLOBAL_TID_:%.*]], ptr noalias [[DOTBOUND_TID_:%.*]]) #[[ATTR1]] {
 ; AMDGPU-DISABLED1-NEXT:  entry:
 ; AMDGPU-DISABLED1-NEXT:    [[CAPTURED_VARS_ADDRS:%.*]] = alloca [1 x ptr], align 8
-; AMDGPU-DISABLED1-NEXT:    store i32 42, ptr addrspacecast (ptr addrspace(3) @x_shared.1 to ptr), align 4, !tbaa [[TBAA18]]
+; AMDGPU-DISABLED1-NEXT:    store i32 42, ptr addrspacecast (ptr addrspace(3) @x_shared.1 to ptr), align 4, !tbaa [[TBAA12]]
 ; AMDGPU-DISABLED1-NEXT:    br label [[FOR_COND:%.*]]
 ; AMDGPU-DISABLED1:       for.cond:
 ; AMDGPU-DISABLED1-NEXT:    [[I_0:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[INC:%.*]], [[FOR_BODY:%.*]] ]
@@ -2793,17 +1862,17 @@ define internal void @__omp_outlined__6(ptr noalias %.global_tid., ptr noalias %
 ; AMDGPU-DISABLED1-NEXT:    call void @spmd_amenable() #[[ATTR7]]
 ; AMDGPU-DISABLED1-NEXT:    ret void
 ; AMDGPU-DISABLED1:       for.body:
-; AMDGPU-DISABLED1-NEXT:    store ptr addrspacecast (ptr addrspace(3) @x_shared.1 to ptr), ptr [[CAPTURED_VARS_ADDRS]], align 8, !tbaa [[TBAA26]]
-; AMDGPU-DISABLED1-NEXT:    [[TMP0:%.*]] = load i32, ptr [[DOTGLOBAL_TID_]], align 4, !tbaa [[TBAA18]]
+; AMDGPU-DISABLED1-NEXT:    store ptr addrspacecast (ptr addrspace(3) @x_shared.1 to ptr), ptr [[CAPTURED_VARS_ADDRS]], align 8, !tbaa [[TBAA20]]
+; AMDGPU-DISABLED1-NEXT:    [[TMP0:%.*]] = load i32, ptr [[DOTGLOBAL_TID_]], align 4, !tbaa [[TBAA12]]
 ; AMDGPU-DISABLED1-NEXT:    call void @__kmpc_parallel_51(ptr @[[GLOB1]], i32 [[TMP0]], i32 1, i32 -1, i32 -1, ptr @__omp_outlined__7, ptr @__omp_outlined__7_wrapper.ID, ptr [[CAPTURED_VARS_ADDRS]], i64 1)
 ; AMDGPU-DISABLED1-NEXT:    [[INC]] = add nsw i32 [[I_0]], 1
-; AMDGPU-DISABLED1-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP29:![0-9]+]]
+; AMDGPU-DISABLED1-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP23:![0-9]+]]
 ;
 ; AMDGPU-DISABLED2-LABEL: define {{[^@]+}}@__omp_outlined__6
 ; AMDGPU-DISABLED2-SAME: (ptr noalias [[DOTGLOBAL_TID_:%.*]], ptr noalias [[DOTBOUND_TID_:%.*]]) #[[ATTR1]] {
 ; AMDGPU-DISABLED2-NEXT:  entry:
 ; AMDGPU-DISABLED2-NEXT:    [[CAPTURED_VARS_ADDRS:%.*]] = alloca [1 x ptr], align 8
-; AMDGPU-DISABLED2-NEXT:    store i32 42, ptr addrspacecast (ptr addrspace(3) @x_shared.1 to ptr), align 4, !tbaa [[TBAA18]]
+; AMDGPU-DISABLED2-NEXT:    store i32 42, ptr addrspacecast (ptr addrspace(3) @x_shared.1 to ptr), align 4, !tbaa [[TBAA12]]
 ; AMDGPU-DISABLED2-NEXT:    br label [[FOR_COND:%.*]]
 ; AMDGPU-DISABLED2:       for.cond:
 ; AMDGPU-DISABLED2-NEXT:    [[I_0:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[INC:%.*]], [[FOR_BODY:%.*]] ]
@@ -2813,17 +1882,17 @@ define internal void @__omp_outlined__6(ptr noalias %.global_tid., ptr noalias %
 ; AMDGPU-DISABLED2-NEXT:    call void @spmd_amenable() #[[ATTR7]]
 ; AMDGPU-DISABLED2-NEXT:    ret void
 ; AMDGPU-DISABLED2:       for.body:
-; AMDGPU-DISABLED2-NEXT:    store ptr addrspacecast (ptr addrspace(3) @x_shared.1 to ptr), ptr [[CAPTURED_VARS_ADDRS]], align 8, !tbaa [[TBAA26]]
-; AMDGPU-DISABLED2-NEXT:    [[TMP0:%.*]] = load i32, ptr [[DOTGLOBAL_TID_]], align 4, !tbaa [[TBAA18]]
-; AMDGPU-DISABLED2-NEXT:    call void @__kmpc_parallel_51(ptr @[[GLOB1]], i32 [[TMP0]], i32 1, i32 -1, i32 -1, ptr @__omp_outlined__7, ptr @__omp_outlined__7_wrapper.ID, ptr [[CAPTURED_VARS_ADDRS]], i64 1)
+; AMDGPU-DISABLED2-NEXT:    store ptr addrspacecast (ptr addrspace(3) @x_shared.1 to ptr), ptr [[CAPTURED_VARS_ADDRS]], align 8, !tbaa [[TBAA20]]
+; AMDGPU-DISABLED2-NEXT:    [[TMP0:%.*]] = load i32, ptr [[DOTGLOBAL_TID_]], align 4, !tbaa [[TBAA12]]
+; AMDGPU-DISABLED2-NEXT:    call void @__kmpc_parallel_51(ptr @[[GLOB1]], i32 [[TMP0]], i32 1, i32 -1, i32 -1, ptr @__omp_outlined__7, ptr @__omp_outlined__7_wrapper, ptr [[CAPTURED_VARS_ADDRS]], i64 1)
 ; AMDGPU-DISABLED2-NEXT:    [[INC]] = add nsw i32 [[I_0]], 1
-; AMDGPU-DISABLED2-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP29:![0-9]+]]
+; AMDGPU-DISABLED2-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP23:![0-9]+]]
 ;
 ; NVPTX-DISABLED1-LABEL: define {{[^@]+}}@__omp_outlined__6
 ; NVPTX-DISABLED1-SAME: (ptr noalias [[DOTGLOBAL_TID_:%.*]], ptr noalias [[DOTBOUND_TID_:%.*]]) #[[ATTR1]] {
 ; NVPTX-DISABLED1-NEXT:  entry:
 ; NVPTX-DISABLED1-NEXT:    [[CAPTURED_VARS_ADDRS:%.*]] = alloca [1 x ptr], align 8
-; NVPTX-DISABLED1-NEXT:    store i32 42, ptr addrspacecast (ptr addrspace(3) @x_shared1 to ptr), align 4, !tbaa [[TBAA18]]
+; NVPTX-DISABLED1-NEXT:    store i32 42, ptr addrspacecast (ptr addrspace(3) @x_shared1 to ptr), align 4, !tbaa [[TBAA12]]
 ; NVPTX-DISABLED1-NEXT:    br label [[FOR_COND:%.*]]
 ; NVPTX-DISABLED1:       for.cond:
 ; NVPTX-DISABLED1-NEXT:    [[I_0:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[INC:%.*]], [[FOR_BODY:%.*]] ]
@@ -2833,17 +1902,17 @@ define internal void @__omp_outlined__6(ptr noalias %.global_tid., ptr noalias %
 ; NVPTX-DISABLED1-NEXT:    call void @spmd_amenable() #[[ATTR7]]
 ; NVPTX-DISABLED1-NEXT:    ret void
 ; NVPTX-DISABLED1:       for.body:
-; NVPTX-DISABLED1-NEXT:    store ptr addrspacecast (ptr addrspace(3) @x_shared1 to ptr), ptr [[CAPTURED_VARS_ADDRS]], align 8, !tbaa [[TBAA26]]
-; NVPTX-DISABLED1-NEXT:    [[TMP0:%.*]] = load i32, ptr [[DOTGLOBAL_TID_]], align 4, !tbaa [[TBAA18]]
+; NVPTX-DISABLED1-NEXT:    store ptr addrspacecast (ptr addrspace(3) @x_shared1 to ptr), ptr [[CAPTURED_VARS_ADDRS]], align 8, !tbaa [[TBAA20]]
+; NVPTX-DISABLED1-NEXT:    [[TMP0:%.*]] = load i32, ptr [[DOTGLOBAL_TID_]], align 4, !tbaa [[TBAA12]]
 ; NVPTX-DISABLED1-NEXT:    call void @__kmpc_parallel_51(ptr @[[GLOB1]], i32 [[TMP0]], i32 1, i32 -1, i32 -1, ptr @__omp_outlined__7, ptr @__omp_outlined__7_wrapper.ID, ptr [[CAPTURED_VARS_ADDRS]], i64 1)
 ; NVPTX-DISABLED1-NEXT:    [[INC]] = add nsw i32 [[I_0]], 1
-; NVPTX-DISABLED1-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP29:![0-9]+]]
+; NVPTX-DISABLED1-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP23:![0-9]+]]
 ;
 ; NVPTX-DISABLED2-LABEL: define {{[^@]+}}@__omp_outlined__6
 ; NVPTX-DISABLED2-SAME: (ptr noalias [[DOTGLOBAL_TID_:%.*]], ptr noalias [[DOTBOUND_TID_:%.*]]) #[[ATTR1]] {
 ; NVPTX-DISABLED2-NEXT:  entry:
 ; NVPTX-DISABLED2-NEXT:    [[CAPTURED_VARS_ADDRS:%.*]] = alloca [1 x ptr], align 8
-; NVPTX-DISABLED2-NEXT:    store i32 42, ptr addrspacecast (ptr addrspace(3) @x_shared1 to ptr), align 4, !tbaa [[TBAA18]]
+; NVPTX-DISABLED2-NEXT:    store i32 42, ptr addrspacecast (ptr addrspace(3) @x_shared1 to ptr), align 4, !tbaa [[TBAA12]]
 ; NVPTX-DISABLED2-NEXT:    br label [[FOR_COND:%.*]]
 ; NVPTX-DISABLED2:       for.cond:
 ; NVPTX-DISABLED2-NEXT:    [[I_0:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[INC:%.*]], [[FOR_BODY:%.*]] ]
@@ -2853,50 +1922,12 @@ define internal void @__omp_outlined__6(ptr noalias %.global_tid., ptr noalias %
 ; NVPTX-DISABLED2-NEXT:    call void @spmd_amenable() #[[ATTR7]]
 ; NVPTX-DISABLED2-NEXT:    ret void
 ; NVPTX-DISABLED2:       for.body:
-; NVPTX-DISABLED2-NEXT:    store ptr addrspacecast (ptr addrspace(3) @x_shared1 to ptr), ptr [[CAPTURED_VARS_ADDRS]], align 8, !tbaa [[TBAA26]]
-; NVPTX-DISABLED2-NEXT:    [[TMP0:%.*]] = load i32, ptr [[DOTGLOBAL_TID_]], align 4, !tbaa [[TBAA18]]
-; NVPTX-DISABLED2-NEXT:    call void @__kmpc_parallel_51(ptr @[[GLOB1]], i32 [[TMP0]], i32 1, i32 -1, i32 -1, ptr @__omp_outlined__7, ptr @__omp_outlined__7_wrapper.ID, ptr [[CAPTURED_VARS_ADDRS]], i64 1)
+; NVPTX-DISABLED2-NEXT:    store ptr addrspacecast (ptr addrspace(3) @x_shared1 to ptr), ptr [[CAPTURED_VARS_ADDRS]], align 8, !tbaa [[TBAA20]]
+; NVPTX-DISABLED2-NEXT:    [[TMP0:%.*]] = load i32, ptr [[DOTGLOBAL_TID_]], align 4, !tbaa [[TBAA12]]
+; NVPTX-DISABLED2-NEXT:    call void @__kmpc_parallel_51(ptr @[[GLOB1]], i32 [[TMP0]], i32 1, i32 -1, i32 -1, ptr @__omp_outlined__7, ptr @__omp_outlined__7_wrapper, ptr [[CAPTURED_VARS_ADDRS]], i64 1)
 ; NVPTX-DISABLED2-NEXT:    [[INC]] = add nsw i32 [[I_0]], 1
-; NVPTX-DISABLED2-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP29:![0-9]+]]
-;
-; AMDGPU-DISABLED-LABEL: define {{[^@]+}}@__omp_outlined__6
-; AMDGPU-DISABLED-SAME: (ptr noalias [[DOTGLOBAL_TID_:%.*]], ptr noalias [[DOTBOUND_TID_:%.*]]) #[[ATTR1]] {
-; AMDGPU-DISABLED-NEXT:  entry:
-; AMDGPU-DISABLED-NEXT:    [[CAPTURED_VARS_ADDRS:%.*]] = alloca [1 x ptr], align 8
-; AMDGPU-DISABLED-NEXT:    store i32 42, ptr addrspacecast (ptr addrspace(3) @x_shared.1 to ptr), align 4, !tbaa [[TBAA18]]
-; AMDGPU-DISABLED-NEXT:    br label [[FOR_COND:%.*]]
-; AMDGPU-DISABLED:       for.cond:
-; AMDGPU-DISABLED-NEXT:    [[I_0:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[INC:%.*]], [[FOR_BODY:%.*]] ]
-; AMDGPU-DISABLED-NEXT:    [[CMP:%.*]] = icmp slt i32 [[I_0]], 100
-; AMDGPU-DISABLED-NEXT:    br i1 [[CMP]], label [[FOR_BODY]], label [[FOR_COND_CLEANUP:%.*]]
-; AMDGPU-DISABLED:       for.cond.cleanup:
-; AMDGPU-DISABLED-NEXT:    call void @spmd_amenable() #[[ATTR7]]
-; AMDGPU-DISABLED-NEXT:    ret void
-; AMDGPU-DISABLED:       for.body:
-; AMDGPU-DISABLED-NEXT:    store ptr addrspacecast (ptr addrspace(3) @x_shared.1 to ptr), ptr [[CAPTURED_VARS_ADDRS]], align 8, !tbaa [[TBAA26]]
-; AMDGPU-DISABLED-NEXT:    [[TMP0:%.*]] = load i32, ptr [[DOTGLOBAL_TID_]], align 4, !tbaa [[TBAA18]]
-; AMDGPU-DISABLED-NEXT:    call void @__kmpc_parallel_51(ptr @[[GLOB1]], i32 [[TMP0]], i32 1, i32 -1, i32 -1, ptr @__omp_outlined__7, ptr @__omp_outlined__7_wrapper.ID, ptr [[CAPTURED_VARS_ADDRS]], i64 1)
-; AMDGPU-DISABLED-NEXT:    [[INC]] = add nsw i32 [[I_0]], 1
-; AMDGPU-DISABLED-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP29:![0-9]+]]
-; NVPTX-DISABLED-LABEL: define {{[^@]+}}@__omp_outlined__6
-; NVPTX-DISABLED-SAME: (ptr noalias [[DOTGLOBAL_TID_:%.*]], ptr noalias [[DOTBOUND_TID_:%.*]]) #[[ATTR1]] {
-; NVPTX-DISABLED-NEXT:  entry:
-; NVPTX-DISABLED-NEXT:    [[CAPTURED_VARS_ADDRS:%.*]] = alloca [1 x ptr], align 8
-; NVPTX-DISABLED-NEXT:    store i32 42, ptr addrspacecast (ptr addrspace(3) @x_shared1 to ptr), align 4, !tbaa [[TBAA18]]
-; NVPTX-DISABLED-NEXT:    br label [[FOR_COND:%.*]]
-; NVPTX-DISABLED:       for.cond:
-; NVPTX-DISABLED-NEXT:    [[I_0:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[INC:%.*]], [[FOR_BODY:%.*]] ]
-; NVPTX-DISABLED-NEXT:    [[CMP:%.*]] = icmp slt i32 [[I_0]], 100
-; NVPTX-DISABLED-NEXT:    br i1 [[CMP]], label [[FOR_BODY]], label [[FOR_COND_CLEANUP:%.*]]
-; NVPTX-DISABLED:       for.cond.cleanup:
-; NVPTX-DISABLED-NEXT:    call void @spmd_amenable() #[[ATTR7]]
-; NVPTX-DISABLED-NEXT:    ret void
-; NVPTX-DISABLED:       for.body:
-; NVPTX-DISABLED-NEXT:    store ptr addrspacecast (ptr addrspace(3) @x_shared1 to ptr), ptr [[CAPTURED_VARS_ADDRS]], align 8, !tbaa [[TBAA26]]
-; NVPTX-DISABLED-NEXT:    [[TMP0:%.*]] = load i32, ptr [[DOTGLOBAL_TID_]], align 4, !tbaa [[TBAA18]]
-; NVPTX-DISABLED-NEXT:    call void @__kmpc_parallel_51(ptr @[[GLOB1]], i32 [[TMP0]], i32 1, i32 -1, i32 -1, ptr @__omp_outlined__7, ptr @__omp_outlined__7_wrapper.ID, ptr [[CAPTURED_VARS_ADDRS]], i64 1)
-; NVPTX-DISABLED-NEXT:    [[INC]] = add nsw i32 [[I_0]], 1
-; NVPTX-DISABLED-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP29:![0-9]+]]
+; NVPTX-DISABLED2-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP23:![0-9]+]]
+;
 entry:
   %captured_vars_addrs = alloca [1 x ptr], align 8
   %x = call align 4 ptr @__kmpc_alloc_shared(i64 4)
@@ -2926,73 +1957,57 @@ define internal void @__omp_outlined__7(ptr noalias %.global_tid., ptr noalias %
 ; AMDGPU-LABEL: define {{[^@]+}}@__omp_outlined__7
 ; AMDGPU-SAME: (ptr noalias [[DOTGLOBAL_TID_:%.*]], ptr noalias [[DOTBOUND_TID_:%.*]], ptr nonnull align 4 dereferenceable(4) [[X:%.*]]) {
 ; AMDGPU-NEXT:  entry:
-; AMDGPU-NEXT:    [[TMP0:%.*]] = load i32, ptr [[X]], align 4, !tbaa [[TBAA18]]
+; AMDGPU-NEXT:    [[TMP0:%.*]] = load i32, ptr [[X]], align 4, !tbaa [[TBAA12]]
 ; AMDGPU-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP0]], 1
-; AMDGPU-NEXT:    store i32 [[INC]], ptr [[X]], align 4, !tbaa [[TBAA18]]
+; AMDGPU-NEXT:    store i32 [[INC]], ptr [[X]], align 4, !tbaa [[TBAA12]]
 ; AMDGPU-NEXT:    call void @unknowni32p(ptr [[X]]) #[[ATTR8]]
 ; AMDGPU-NEXT:    ret void
 ;
 ; NVPTX-LABEL: define {{[^@]+}}@__omp_outlined__7
 ; NVPTX-SAME: (ptr noalias [[DOTGLOBAL_TID_:%.*]], ptr noalias [[DOTBOUND_TID_:%.*]], ptr nonnull align 4 dereferenceable(4) [[X:%.*]]) {
 ; NVPTX-NEXT:  entry:
-; NVPTX-NEXT:    [[TMP0:%.*]] = load i32, ptr [[X]], align 4, !tbaa [[TBAA18]]
+; NVPTX-NEXT:    [[TMP0:%.*]] = load i32, ptr [[X]], align 4, !tbaa [[TBAA12]]
 ; NVPTX-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP0]], 1
-; NVPTX-NEXT:    store i32 [[INC]], ptr [[X]], align 4, !tbaa [[TBAA18]]
+; NVPTX-NEXT:    store i32 [[INC]], ptr [[X]], align 4, !tbaa [[TBAA12]]
 ; NVPTX-NEXT:    call void @unknowni32p(ptr [[X]]) #[[ATTR8]]
 ; NVPTX-NEXT:    ret void
 ;
 ; AMDGPU-DISABLED1-LABEL: define {{[^@]+}}@__omp_outlined__7
 ; AMDGPU-DISABLED1-SAME: (ptr noalias [[DOTGLOBAL_TID_:%.*]], ptr noalias [[DOTBOUND_TID_:%.*]], ptr nonnull align 4 dereferenceable(4) [[X:%.*]]) {
 ; AMDGPU-DISABLED1-NEXT:  entry:
-; AMDGPU-DISABLED1-NEXT:    [[TMP0:%.*]] = load i32, ptr [[X]], align 4, !tbaa [[TBAA18]]
+; AMDGPU-DISABLED1-NEXT:    [[TMP0:%.*]] = load i32, ptr [[X]], align 4, !tbaa [[TBAA12]]
 ; AMDGPU-DISABLED1-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP0]], 1
-; AMDGPU-DISABLED1-NEXT:    store i32 [[INC]], ptr [[X]], align 4, !tbaa [[TBAA18]]
+; AMDGPU-DISABLED1-NEXT:    store i32 [[INC]], ptr [[X]], align 4, !tbaa [[TBAA12]]
 ; AMDGPU-DISABLED1-NEXT:    call void @unknowni32p(ptr [[X]]) #[[ATTR8]]
 ; AMDGPU-DISABLED1-NEXT:    ret void
 ;
 ; AMDGPU-DISABLED2-LABEL: define {{[^@]+}}@__omp_outlined__7
 ; AMDGPU-DISABLED2-SAME: (ptr noalias [[DOTGLOBAL_TID_:%.*]], ptr noalias [[DOTBOUND_TID_:%.*]], ptr nonnull align 4 dereferenceable(4) [[X:%.*]]) {
 ; AMDGPU-DISABLED2-NEXT:  entry:
-; AMDGPU-DISABLED2-NEXT:    [[TMP0:%.*]] = load i32, ptr [[X]], align 4, !tbaa [[TBAA18]]
+; AMDGPU-DISABLED2-NEXT:    [[TMP0:%.*]] = load i32, ptr [[X]], align 4, !tbaa [[TBAA12]]
 ; AMDGPU-DISABLED2-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP0]], 1
-; AMDGPU-DISABLED2-NEXT:    store i32 [[INC]], ptr [[X]], align 4, !tbaa [[TBAA18]]
+; AMDGPU-DISABLED2-NEXT:    store i32 [[INC]], ptr [[X]], align 4, !tbaa [[TBAA12]]
 ; AMDGPU-DISABLED2-NEXT:    call void @unknowni32p(ptr [[X]]) #[[ATTR8]]
 ; AMDGPU-DISABLED2-NEXT:    ret void
 ;
 ; NVPTX-DISABLED1-LABEL: define {{[^@]+}}@__omp_outlined__7
 ; NVPTX-DISABLED1-SAME: (ptr noalias [[DOTGLOBAL_TID_:%.*]], ptr noalias [[DOTBOUND_TID_:%.*]], ptr nonnull align 4 dereferenceable(4) [[X:%.*]]) {
 ; NVPTX-DISABLED1-NEXT:  entry:
-; NVPTX-DISABLED1-NEXT:    [[TMP0:%.*]] = load i32, ptr [[X]], align 4, !tbaa [[TBAA18]]
+; NVPTX-DISABLED1-NEXT:    [[TMP0:%.*]] = load i32, ptr [[X]], align 4, !tbaa [[TBAA12]]
 ; NVPTX-DISABLED1-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP0]], 1
-; NVPTX-DISABLED1-NEXT:    store i32 [[INC]], ptr [[X]], align 4, !tbaa [[TBAA18]]
+; NVPTX-DISABLED1-NEXT:    store i32 [[INC]], ptr [[X]], align 4, !tbaa [[TBAA12]]
 ; NVPTX-DISABLED1-NEXT:    call void @unknowni32p(ptr [[X]]) #[[ATTR8]]
 ; NVPTX-DISABLED1-NEXT:    ret void
 ;
 ; NVPTX-DISABLED2-LABEL: define {{[^@]+}}@__omp_outlined__7
 ; NVPTX-DISABLED2-SAME: (ptr noalias [[DOTGLOBAL_TID_:%.*]], ptr noalias [[DOTBOUND_TID_:%.*]], ptr nonnull align 4 dereferenceable(4) [[X:%.*]]) {
 ; NVPTX-DISABLED2-NEXT:  entry:
-; NVPTX-DISABLED2-NEXT:    [[TMP0:%.*]] = load i32, ptr [[X]], align 4, !tbaa [[TBAA18]]
+; NVPTX-DISABLED2-NEXT:    [[TMP0:%.*]] = load i32, ptr [[X]], align 4, !tbaa [[TBAA12]]
 ; NVPTX-DISABLED2-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP0]], 1
-; NVPTX-DISABLED2-NEXT:    store i32 [[INC]], ptr [[X]], align 4, !tbaa [[TBAA18]]
+; NVPTX-DISABLED2-NEXT:    store i32 [[INC]], ptr [[X]], align 4, !tbaa [[TBAA12]]
 ; NVPTX-DISABLED2-NEXT:    call void @unknowni32p(ptr [[X]]) #[[ATTR8]]
 ; NVPTX-DISABLED2-NEXT:    ret void
 ;
-; AMDGPU-DISABLED-LABEL: define {{[^@]+}}@__omp_outlined__7
-; AMDGPU-DISABLED-SAME: (ptr noalias [[DOTGLOBAL_TID_:%.*]], ptr noalias [[DOTBOUND_TID_:%.*]], ptr nonnull align 4 dereferenceable(4) [[X:%.*]]) {
-; AMDGPU-DISABLED-NEXT:  entry:
-; AMDGPU-DISABLED-NEXT:    [[TMP0:%.*]] = load i32, ptr [[X]], align 4, !tbaa [[TBAA18]]
-; AMDGPU-DISABLED-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP0]], 1
-; AMDGPU-DISABLED-NEXT:    store i32 [[INC]], ptr [[X]], align 4, !tbaa [[TBAA18]]
-; AMDGPU-DISABLED-NEXT:    call void @unknowni32p(ptr [[X]]) #[[ATTR8]]
-; AMDGPU-DISABLED-NEXT:    ret void
-; NVPTX-DISABLED-LABEL: define {{[^@]+}}@__omp_outlined__7
-; NVPTX-DISABLED-SAME: (ptr noalias [[DOTGLOBAL_TID_:%.*]], ptr noalias [[DOTBOUND_TID_:%.*]], ptr nonnull align 4 dereferenceable(4) [[X:%.*]]) {
-; NVPTX-DISABLED-NEXT:  entry:
-; NVPTX-DISABLED-NEXT:    [[TMP0:%.*]] = load i32, ptr [[X]], align 4, !tbaa [[TBAA18]]
-; NVPTX-DISABLED-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP0]], 1
-; NVPTX-DISABLED-NEXT:    store i32 [[INC]], ptr [[X]], align 4, !tbaa [[TBAA18]]
-; NVPTX-DISABLED-NEXT:    call void @unknowni32p(ptr [[X]]) #[[ATTR8]]
-; NVPTX-DISABLED-NEXT:    ret void
 entry:
   %0 = load i32, ptr %x, align 4, !tbaa !18
   %inc = add nsw i32 %0, 1
@@ -3011,7 +2026,7 @@ define internal void @__omp_outlined__7_wrapper(i16 zeroext %0, i32 %1) #3 {
 ; AMDGPU-NEXT:    [[GLOBAL_ARGS:%.*]] = alloca ptr, align 8
 ; AMDGPU-NEXT:    call void @__kmpc_get_shared_variables(ptr [[GLOBAL_ARGS]])
 ; AMDGPU-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[GLOBAL_ARGS]], align 8
-; AMDGPU-NEXT:    [[TMP3:%.*]] = load ptr, ptr [[TMP2]], align 8, !tbaa [[TBAA26]]
+; AMDGPU-NEXT:    [[TMP3:%.*]] = load ptr, ptr [[TMP2]], align 8, !tbaa [[TBAA20]]
 ; AMDGPU-NEXT:    call void @__omp_outlined__7(ptr [[DOTADDR1]], ptr [[DOTZERO_ADDR]], ptr [[TMP3]]) #[[ATTR4]]
 ; AMDGPU-NEXT:    ret void
 ;
@@ -3023,7 +2038,7 @@ define internal void @__omp_outlined__7_wrapper(i16 zeroext %0, i32 %1) #3 {
 ; NVPTX-NEXT:    [[GLOBAL_ARGS:%.*]] = alloca ptr, align 8
 ; NVPTX-NEXT:    call void @__kmpc_get_shared_variables(ptr [[GLOBAL_ARGS]])
 ; NVPTX-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[GLOBAL_ARGS]], align 8
-; NVPTX-NEXT:    [[TMP3:%.*]] = load ptr, ptr [[TMP2]], align 8, !tbaa [[TBAA26]]
+; NVPTX-NEXT:    [[TMP3:%.*]] = load ptr, ptr [[TMP2]], align 8, !tbaa [[TBAA20]]
 ; NVPTX-NEXT:    call void @__omp_outlined__7(ptr [[DOTADDR1]], ptr [[DOTZERO_ADDR]], ptr [[TMP3]]) #[[ATTR4]]
 ; NVPTX-NEXT:    ret void
 ;
@@ -3035,7 +2050,7 @@ define internal void @__omp_outlined__7_wrapper(i16 zeroext %0, i32 %1) #3 {
 ; AMDGPU-DISABLED1-NEXT:    [[GLOBAL_ARGS:%.*]] = alloca ptr, align 8
 ; AMDGPU-DISABLED1-NEXT:    call void @__kmpc_get_shared_variables(ptr [[GLOBAL_ARGS]])
 ; AMDGPU-DISABLED1-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[GLOBAL_ARGS]], align 8
-; AMDGPU-DISABLED1-NEXT:    [[TMP3:%.*]] = load ptr, ptr [[TMP2]], align 8, !tbaa [[TBAA26]]
+; AMDGPU-DISABLED1-NEXT:    [[TMP3:%.*]] = load ptr, ptr [[TMP2]], align 8, !tbaa [[TBAA20]]
 ; AMDGPU-DISABLED1-NEXT:    call void @__omp_outlined__7(ptr [[DOTADDR1]], ptr [[DOTZERO_ADDR]], ptr [[TMP3]]) #[[ATTR4]]
 ; AMDGPU-DISABLED1-NEXT:    ret void
 ;
@@ -3047,7 +2062,7 @@ define internal void @__omp_outlined__7_wrapper(i16 zeroext %0, i32 %1) #3 {
 ; AMDGPU-DISABLED2-NEXT:    [[GLOBAL_ARGS:%.*]] = alloca ptr, align 8
 ; AMDGPU-DISABLED2-NEXT:    call void @__kmpc_get_shared_variables(ptr [[GLOBAL_ARGS]])
 ; AMDGPU-DISABLED2-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[GLOBAL_ARGS]], align 8
-; AMDGPU-DISABLED2-NEXT:    [[TMP3:%.*]] = load ptr, ptr [[TMP2]], align 8, !tbaa [[TBAA26]]
+; AMDGPU-DISABLED2-NEXT:    [[TMP3:%.*]] = load ptr, ptr [[TMP2]], align 8, !tbaa [[TBAA20]]
 ; AMDGPU-DISABLED2-NEXT:    call void @__omp_outlined__7(ptr [[DOTADDR1]], ptr [[DOTZERO_ADDR]], ptr [[TMP3]]) #[[ATTR4]]
 ; AMDGPU-DISABLED2-NEXT:    ret void
 ;
@@ -3059,7 +2074,7 @@ define internal void @__omp_outlined__7_wrapper(i16 zeroext %0, i32 %1) #3 {
 ; NVPTX-DISABLED1-NEXT:    [[GLOBAL_ARGS:%.*]] = alloca ptr, align 8
 ; NVPTX-DISABLED1-NEXT:    call void @__kmpc_get_shared_variables(ptr [[GLOBAL_ARGS]])
 ; NVPTX-DISABLED1-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[GLOBAL_ARGS]], align 8
-; NVPTX-DISABLED1-NEXT:    [[TMP3:%.*]] = load ptr, ptr [[TMP2]], align 8, !tbaa [[TBAA26]]
+; NVPTX-DISABLED1-NEXT:    [[TMP3:%.*]] = load ptr, ptr [[TMP2]], align 8, !tbaa [[TBAA20]]
 ; NVPTX-DISABLED1-NEXT:    call void @__omp_outlined__7(ptr [[DOTADDR1]], ptr [[DOTZERO_ADDR]], ptr [[TMP3]]) #[[ATTR4]]
 ; NVPTX-DISABLED1-NEXT:    ret void
 ;
@@ -3071,32 +2086,10 @@ define internal void @__omp_outlined__7_wrapper(i16 zeroext %0, i32 %1) #3 {
 ; NVPTX-DISABLED2-NEXT:    [[GLOBAL_ARGS:%.*]] = alloca ptr, align 8
 ; NVPTX-DISABLED2-NEXT:    call void @__kmpc_get_shared_variables(ptr [[GLOBAL_ARGS]])
 ; NVPTX-DISABLED2-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[GLOBAL_ARGS]], align 8
-; NVPTX-DISABLED2-NEXT:    [[TMP3:%.*]] = load ptr, ptr [[TMP2]], align 8, !tbaa [[TBAA26]]
+; NVPTX-DISABLED2-NEXT:    [[TMP3:%.*]] = load ptr, ptr [[TMP2]], align 8, !tbaa [[TBAA20]]
 ; NVPTX-DISABLED2-NEXT:    call void @__omp_outlined__7(ptr [[DOTADDR1]], ptr [[DOTZERO_ADDR]], ptr [[TMP3]]) #[[ATTR4]]
 ; NVPTX-DISABLED2-NEXT:    ret void
 ;
-; AMDGPU-DISABLED-LABEL: define {{[^@]+}}@__omp_outlined__7_wrapper
-; AMDGPU-DISABLED-SAME: (i16 zeroext [[TMP0:%.*]], i32 [[TMP1:%.*]]) #[[ATTR2]] {
-; AMDGPU-DISABLED-NEXT:  entry:
-; AMDGPU-DISABLED-NEXT:    [[DOTADDR1:%.*]] = alloca i32, align 4
-; AMDGPU-DISABLED-NEXT:    [[DOTZERO_ADDR:%.*]] = alloca i32, align 4
-; AMDGPU-DISABLED-NEXT:    [[GLOBAL_ARGS:%.*]] = alloca ptr, align 8
-; AMDGPU-DISABLED-NEXT:    call void @__kmpc_get_shared_variables(ptr [[GLOBAL_ARGS]])
-; AMDGPU-DISABLED-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[GLOBAL_ARGS]], align 8
-; AMDGPU-DISABLED-NEXT:    [[TMP3:%.*]] = load ptr, ptr [[TMP2]], align 8, !tbaa [[TBAA26]]
-; AMDGPU-DISABLED-NEXT:    call void @__omp_outlined__7(ptr [[DOTADDR1]], ptr [[DOTZERO_ADDR]], ptr [[TMP3]]) #[[ATTR4]]
-; AMDGPU-DISABLED-NEXT:    ret void
-; NVPTX-DISABLED-LABEL: define {{[^@]+}}@__omp_outlined__7_wrapper
-; NVPTX-DISABLED-SAME: (i16 zeroext [[TMP0:%.*]], i32 [[TMP1:%.*]]) #[[ATTR2]] {
-; NVPTX-DISABLED-NEXT:  entry:
-; NVPTX-DISABLED-NEXT:    [[DOTADDR1:%.*]] = alloca i32, align 4
-; NVPTX-DISABLED-NEXT:    [[DOTZERO_ADDR:%.*]] = alloca i32, align 4
-; NVPTX-DISABLED-NEXT:    [[GLOBAL_ARGS:%.*]] = alloca ptr, align 8
-; NVPTX-DISABLED-NEXT:    call void @__kmpc_get_shared_variables(ptr [[GLOBAL_ARGS]])
-; NVPTX-DISABLED-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[GLOBAL_ARGS]], align 8
-; NVPTX-DISABLED-NEXT:    [[TMP3:%.*]] = load ptr, ptr [[TMP2]], align 8, !tbaa [[TBAA26]]
-; NVPTX-DISABLED-NEXT:    call void @__omp_outlined__7(ptr [[DOTADDR1]], ptr [[DOTZERO_ADDR]], ptr [[TMP3]]) #[[ATTR4]]
-; NVPTX-DISABLED-NEXT:    ret void
 entry:
   %.addr1 = alloca i32, align 4
   %.zero.addr = alloca i32, align 4
@@ -3111,7 +2104,7 @@ entry:
 }
 
 ; Function Attrs: alwaysinline convergent norecurse nounwind
-define weak void @__omp_offloading_fd02_2044372e_do_not_spmdize_target_l65() #0 {
+define weak ptx_kernel void @__omp_offloading_fd02_2044372e_do_not_spmdize_target_l65() #0 {
 ; AMDGPU-LABEL: define {{[^@]+}}@__omp_offloading_fd02_2044372e_do_not_spmdize_target_l65
 ; AMDGPU-SAME: () #[[ATTR0]] {
 ; AMDGPU-NEXT:  entry:
@@ -3252,39 +2245,9 @@ define weak void @__omp_offloading_fd02_2044372e_do_not_spmdize_target_l65() #0
 ; AMDGPU-DISABLED2-LABEL: define {{[^@]+}}@__omp_offloading_fd02_2044372e_do_not_spmdize_target_l65
 ; AMDGPU-DISABLED2-SAME: () #[[ATTR0]] {
 ; AMDGPU-DISABLED2-NEXT:  entry:
-; AMDGPU-DISABLED2-NEXT:    [[WORKER_WORK_FN_ADDR:%.*]] = alloca ptr, align 8, addrspace(5)
 ; AMDGPU-DISABLED2-NEXT:    [[DOTZERO_ADDR:%.*]] = alloca i32, align 4
 ; AMDGPU-DISABLED2-NEXT:    [[DOTTHREADID_TEMP_:%.*]] = alloca i32, align 4
 ; AMDGPU-DISABLED2-NEXT:    [[TMP0:%.*]] = call i32 @__kmpc_target_init(ptr @__omp_offloading_fd02_2044372e_do_not_spmdize_target_l65_kernel_environment, ptr null)
-; AMDGPU-DISABLED2-NEXT:    [[THREAD_IS_WORKER:%.*]] = icmp ne i32 [[TMP0]], -1
-; AMDGPU-DISABLED2-NEXT:    br i1 [[THREAD_IS_WORKER]], label [[IS_WORKER_CHECK:%.*]], label [[THREAD_USER_CODE_CHECK:%.*]]
-; AMDGPU-DISABLED2:       is_worker_check:
-; AMDGPU-DISABLED2-NEXT:    [[BLOCK_HW_SIZE:%.*]] = call i32 @__kmpc_get_hardware_num_threads_in_block()
-; AMDGPU-DISABLED2-NEXT:    [[WARP_SIZE:%.*]] = call i32 @__kmpc_get_warp_size()
-; AMDGPU-DISABLED2-NEXT:    [[BLOCK_SIZE:%.*]] = sub i32 [[BLOCK_HW_SIZE]], [[WARP_SIZE]]
-; AMDGPU-DISABLED2-NEXT:    [[THREAD_IS_MAIN_OR_WORKER:%.*]] = icmp slt i32 [[TMP0]], [[BLOCK_SIZE]]
-; AMDGPU-DISABLED2-NEXT:    br i1 [[THREAD_IS_MAIN_OR_WORKER]], label [[WORKER_STATE_MACHINE_BEGIN:%.*]], label [[WORKER_STATE_MACHINE_FINISHED:%.*]]
-; AMDGPU-DISABLED2:       worker_state_machine.begin:
-; AMDGPU-DISABLED2-NEXT:    call void @__kmpc_barrier_simple_generic(ptr @[[GLOB1]], i32 [[TMP0]])
-; AMDGPU-DISABLED2-NEXT:    [[WORKER_WORK_FN_ADDR_GENERIC:%.*]] = addrspacecast ptr addrspace(5) [[WORKER_WORK_FN_ADDR]] to ptr
-; AMDGPU-DISABLED2-NEXT:    [[WORKER_IS_ACTIVE:%.*]] = call i1 @__kmpc_kernel_parallel(ptr [[WORKER_WORK_FN_ADDR_GENERIC]])
-; AMDGPU-DISABLED2-NEXT:    [[WORKER_WORK_FN:%.*]] = load ptr, ptr [[WORKER_WORK_FN_ADDR_GENERIC]], align 8
-; AMDGPU-DISABLED2-NEXT:    [[WORKER_IS_DONE:%.*]] = icmp eq ptr [[WORKER_WORK_FN]], null
-; AMDGPU-DISABLED2-NEXT:    br i1 [[WORKER_IS_DONE]], label [[WORKER_STATE_MACHINE_FINISHED]], label [[WORKER_STATE_MACHINE_IS_ACTIVE_CHECK:%.*]]
-; AMDGPU-DISABLED2:       worker_state_machine.finished:
-; AMDGPU-DISABLED2-NEXT:    ret void
-; AMDGPU-DISABLED2:       worker_state_machine.is_active.check:
-; AMDGPU-DISABLED2-NEXT:    br i1 [[WORKER_IS_ACTIVE]], label [[WORKER_STATE_MACHINE_PARALLEL_REGION_FALLBACK_EXECUTE:%.*]], label [[WORKER_STATE_MACHINE_DONE_BARRIER:%.*]]
-; AMDGPU-DISABLED2:       worker_state_machine.parallel_region.fallback.execute:
-; AMDGPU-DISABLED2-NEXT:    call void [[WORKER_WORK_FN]](i16 0, i32 [[TMP0]])
-; AMDGPU-DISABLED2-NEXT:    br label [[WORKER_STATE_MACHINE_PARALLEL_REGION_END:%.*]]
-; AMDGPU-DISABLED2:       worker_state_machine.parallel_region.end:
-; AMDGPU-DISABLED2-NEXT:    call void @__kmpc_kernel_end_parallel()
-; AMDGPU-DISABLED2-NEXT:    br label [[WORKER_STATE_MACHINE_DONE_BARRIER]]
-; AMDGPU-DISABLED2:       worker_state_machine.done.barrier:
-; AMDGPU-DISABLED2-NEXT:    call void @__kmpc_barrier_simple_generic(ptr @[[GLOB1]], i32 [[TMP0]])
-; AMDGPU-DISABLED2-NEXT:    br label [[WORKER_STATE_MACHINE_BEGIN]]
-; AMDGPU-DISABLED2:       thread.user_code.check:
 ; AMDGPU-DISABLED2-NEXT:    [[EXEC_USER_CODE:%.*]] = icmp eq i32 [[TMP0]], -1
 ; AMDGPU-DISABLED2-NEXT:    br i1 [[EXEC_USER_CODE]], label [[USER_CODE_ENTRY:%.*]], label [[COMMON_RET:%.*]]
 ; AMDGPU-DISABLED2:       common.ret:
@@ -3343,38 +2306,9 @@ define weak void @__omp_offloading_fd02_2044372e_do_not_spmdize_target_l65() #0
 ; NVPTX-DISABLED2-LABEL: define {{[^@]+}}@__omp_offloading_fd02_2044372e_do_not_spmdize_target_l65
 ; NVPTX-DISABLED2-SAME: () #[[ATTR0]] {
 ; NVPTX-DISABLED2-NEXT:  entry:
-; NVPTX-DISABLED2-NEXT:    [[WORKER_WORK_FN_ADDR:%.*]] = alloca ptr, align 8
 ; NVPTX-DISABLED2-NEXT:    [[DOTZERO_ADDR:%.*]] = alloca i32, align 4
 ; NVPTX-DISABLED2-NEXT:    [[DOTTHREADID_TEMP_:%.*]] = alloca i32, align 4
 ; NVPTX-DISABLED2-NEXT:    [[TMP0:%.*]] = call i32 @__kmpc_target_init(ptr @__omp_offloading_fd02_2044372e_do_not_spmdize_target_l65_kernel_environment, ptr null)
-; NVPTX-DISABLED2-NEXT:    [[THREAD_IS_WORKER:%.*]] = icmp ne i32 [[TMP0]], -1
-; NVPTX-DISABLED2-NEXT:    br i1 [[THREAD_IS_WORKER]], label [[IS_WORKER_CHECK:%.*]], label [[THREAD_USER_CODE_CHECK:%.*]]
-; NVPTX-DISABLED2:       is_worker_check:
-; NVPTX-DISABLED2-NEXT:    [[BLOCK_HW_SIZE:%.*]] = call i32 @__kmpc_get_hardware_num_threads_in_block()
-; NVPTX-DISABLED2-NEXT:    [[WARP_SIZE:%.*]] = call i32 @__kmpc_get_warp_size()
-; NVPTX-DISABLED2-NEXT:    [[BLOCK_SIZE:%.*]] = sub i32 [[BLOCK_HW_SIZE]], [[WARP_SIZE]]
-; NVPTX-DISABLED2-NEXT:    [[THREAD_IS_MAIN_OR_WORKER:%.*]] = icmp slt i32 [[TMP0]], [[BLOCK_SIZE]]
-; NVPTX-DISABLED2-NEXT:    br i1 [[THREAD_IS_MAIN_OR_WORKER]], label [[WORKER_STATE_MACHINE_BEGIN:%.*]], label [[WORKER_STATE_MACHINE_FINISHED:%.*]]
-; NVPTX-DISABLED2:       worker_state_machine.begin:
-; NVPTX-DISABLED2-NEXT:    call void @__kmpc_barrier_simple_generic(ptr @[[GLOB1]], i32 [[TMP0]])
-; NVPTX-DISABLED2-NEXT:    [[WORKER_IS_ACTIVE:%.*]] = call i1 @__kmpc_kernel_parallel(ptr [[WORKER_WORK_FN_ADDR]])
-; NVPTX-DISABLED2-NEXT:    [[WORKER_WORK_FN:%.*]] = load ptr, ptr [[WORKER_WORK_FN_ADDR]], align 8
-; NVPTX-DISABLED2-NEXT:    [[WORKER_IS_DONE:%.*]] = icmp eq ptr [[WORKER_WORK_FN]], null
-; NVPTX-DISABLED2-NEXT:    br i1 [[WORKER_IS_DONE]], label [[WORKER_STATE_MACHINE_FINISHED]], label [[WORKER_STATE_MACHINE_IS_ACTIVE_CHECK:%.*]]
-; NVPTX-DISABLED2:       worker_state_machine.finished:
-; NVPTX-DISABLED2-NEXT:    ret void
-; NVPTX-DISABLED2:       worker_state_machine.is_active.check:
-; NVPTX-DISABLED2-NEXT:    br i1 [[WORKER_IS_ACTIVE]], label [[WORKER_STATE_MACHINE_PARALLEL_REGION_FALLBACK_EXECUTE:%.*]], label [[WORKER_STATE_MACHINE_DONE_BARRIER:%.*]]
-; NVPTX-DISABLED2:       worker_state_machine.parallel_region.fallback.execute:
-; NVPTX-DISABLED2-NEXT:    call void [[WORKER_WORK_FN]](i16 0, i32 [[TMP0]])
-; NVPTX-DISABLED2-NEXT:    br label [[WORKER_STATE_MACHINE_PARALLEL_REGION_END:%.*]]
-; NVPTX-DISABLED2:       worker_state_machine.parallel_region.end:
-; NVPTX-DISABLED2-NEXT:    call void @__kmpc_kernel_end_parallel()
-; NVPTX-DISABLED2-NEXT:    br label [[WORKER_STATE_MACHINE_DONE_BARRIER]]
-; NVPTX-DISABLED2:       worker_state_machine.done.barrier:
-; NVPTX-DISABLED2-NEXT:    call void @__kmpc_barrier_simple_generic(ptr @[[GLOB1]], i32 [[TMP0]])
-; NVPTX-DISABLED2-NEXT:    br label [[WORKER_STATE_MACHINE_BEGIN]]
-; NVPTX-DISABLED2:       thread.user_code.check:
 ; NVPTX-DISABLED2-NEXT:    [[EXEC_USER_CODE:%.*]] = icmp eq i32 [[TMP0]], -1
 ; NVPTX-DISABLED2-NEXT:    br i1 [[EXEC_USER_CODE]], label [[USER_CODE_ENTRY:%.*]], label [[COMMON_RET:%.*]]
 ; NVPTX-DISABLED2:       common.ret:
@@ -3385,95 +2319,6 @@ define weak void @__omp_offloading_fd02_2044372e_do_not_spmdize_target_l65() #0
 ; NVPTX-DISABLED2-NEXT:    call void @__kmpc_target_deinit()
 ; NVPTX-DISABLED2-NEXT:    br label [[COMMON_RET]]
 ;
-; AMDGPU-DISABLED-LABEL: define {{[^@]+}}@__omp_offloading_fd02_2044372e_do_not_spmdize_target_l65
-; AMDGPU-DISABLED-SAME: () #[[ATTR0]] {
-; AMDGPU-DISABLED-NEXT:  entry:
-; AMDGPU-DISABLED-NEXT:    [[WORKER_WORK_FN_ADDR:%.*]] = alloca ptr, align 8, addrspace(5)
-; AMDGPU-DISABLED-NEXT:    [[DOTZERO_ADDR:%.*]] = alloca i32, align 4
-; AMDGPU-DISABLED-NEXT:    [[DOTTHREADID_TEMP_:%.*]] = alloca i32, align 4
-; AMDGPU-DISABLED-NEXT:    [[TMP0:%.*]] = call i32 @__kmpc_target_init(ptr @__omp_offloading_fd02_2044372e_do_not_spmdize_target_l65_kernel_environment)
-; AMDGPU-DISABLED-NEXT:    [[THREAD_IS_WORKER:%.*]] = icmp ne i32 [[TMP0]], -1
-; AMDGPU-DISABLED-NEXT:    br i1 [[THREAD_IS_WORKER]], label [[IS_WORKER_CHECK:%.*]], label [[THREAD_USER_CODE_CHECK:%.*]]
-; AMDGPU-DISABLED:       is_worker_check:
-; AMDGPU-DISABLED-NEXT:    [[BLOCK_HW_SIZE:%.*]] = call i32 @__kmpc_get_hardware_num_threads_in_block()
-; AMDGPU-DISABLED-NEXT:    [[WARP_SIZE:%.*]] = call i32 @__kmpc_get_warp_size()
-; AMDGPU-DISABLED-NEXT:    [[BLOCK_SIZE:%.*]] = sub i32 [[BLOCK_HW_SIZE]], [[WARP_SIZE]]
-; AMDGPU-DISABLED-NEXT:    [[THREAD_IS_MAIN_OR_WORKER:%.*]] = icmp slt i32 [[TMP0]], [[BLOCK_SIZE]]
-; AMDGPU-DISABLED-NEXT:    br i1 [[THREAD_IS_MAIN_OR_WORKER]], label [[WORKER_STATE_MACHINE_BEGIN:%.*]], label [[WORKER_STATE_MACHINE_FINISHED:%.*]]
-; AMDGPU-DISABLED:       worker_state_machine.begin:
-; AMDGPU-DISABLED-NEXT:    call void @__kmpc_barrier_simple_generic(ptr @[[GLOB1]], i32 [[TMP0]])
-; AMDGPU-DISABLED-NEXT:    [[WORKER_WORK_FN_ADDR_GENERIC:%.*]] = addrspacecast ptr addrspace(5) [[WORKER_WORK_FN_ADDR]] to ptr
-; AMDGPU-DISABLED-NEXT:    [[WORKER_IS_ACTIVE:%.*]] = call i1 @__kmpc_kernel_parallel(ptr [[WORKER_WORK_FN_ADDR_GENERIC]])
-; AMDGPU-DISABLED-NEXT:    [[WORKER_WORK_FN:%.*]] = load ptr, ptr [[WORKER_WORK_FN_ADDR_GENERIC]], align 8
-; AMDGPU-DISABLED-NEXT:    [[WORKER_IS_DONE:%.*]] = icmp eq ptr [[WORKER_WORK_FN]], null
-; AMDGPU-DISABLED-NEXT:    br i1 [[WORKER_IS_DONE]], label [[WORKER_STATE_MACHINE_FINISHED]], label [[WORKER_STATE_MACHINE_IS_ACTIVE_CHECK:%.*]]
-; AMDGPU-DISABLED:       worker_state_machine.finished:
-; AMDGPU-DISABLED-NEXT:    ret void
-; AMDGPU-DISABLED:       worker_state_machine.is_active.check:
-; AMDGPU-DISABLED-NEXT:    br i1 [[WORKER_IS_ACTIVE]], label [[WORKER_STATE_MACHINE_PARALLEL_REGION_FALLBACK_EXECUTE:%.*]], label [[WORKER_STATE_MACHINE_DONE_BARRIER:%.*]]
-; AMDGPU-DISABLED:       worker_state_machine.parallel_region.fallback.execute:
-; AMDGPU-DISABLED-NEXT:    call void [[WORKER_WORK_FN]](i16 0, i32 [[TMP0]])
-; AMDGPU-DISABLED-NEXT:    br label [[WORKER_STATE_MACHINE_PARALLEL_REGION_END:%.*]]
-; AMDGPU-DISABLED:       worker_state_machine.parallel_region.end:
-; AMDGPU-DISABLED-NEXT:    call void @__kmpc_kernel_end_parallel()
-; AMDGPU-DISABLED-NEXT:    br label [[WORKER_STATE_MACHINE_DONE_BARRIER]]
-; AMDGPU-DISABLED:       worker_state_machine.done.barrier:
-; AMDGPU-DISABLED-NEXT:    call void @__kmpc_barrier_simple_generic(ptr @[[GLOB1]], i32 [[TMP0]])
-; AMDGPU-DISABLED-NEXT:    br label [[WORKER_STATE_MACHINE_BEGIN]]
-; AMDGPU-DISABLED:       thread.user_code.check:
-; AMDGPU-DISABLED-NEXT:    [[EXEC_USER_CODE:%.*]] = icmp eq i32 [[TMP0]], -1
-; AMDGPU-DISABLED-NEXT:    br i1 [[EXEC_USER_CODE]], label [[USER_CODE_ENTRY:%.*]], label [[COMMON_RET:%.*]]
-; AMDGPU-DISABLED:       common.ret:
-; AMDGPU-DISABLED-NEXT:    ret void
-; AMDGPU-DISABLED:       user_code.entry:
-; AMDGPU-DISABLED-NEXT:    [[TMP1:%.*]] = call i32 @__kmpc_global_thread_num(ptr @[[GLOB1]]) #[[ATTR5]]
-; AMDGPU-DISABLED-NEXT:    call void @__omp_outlined__8(ptr [[DOTTHREADID_TEMP_]], ptr [[DOTZERO_ADDR]]) #[[ATTR5]]
-; AMDGPU-DISABLED-NEXT:    call void @__kmpc_target_deinit()
-; AMDGPU-DISABLED-NEXT:    br label [[COMMON_RET]]
-; NVPTX-DISABLED-LABEL: define {{[^@]+}}@__omp_offloading_fd02_2044372e_do_not_spmdize_target_l65
-; NVPTX-DISABLED-SAME: () #[[ATTR0]] {
-; NVPTX-DISABLED-NEXT:  entry:
-; NVPTX-DISABLED-NEXT:    [[WORKER_WORK_FN_ADDR:%.*]] = alloca ptr, align 8
-; NVPTX-DISABLED-NEXT:    [[DOTZERO_ADDR:%.*]] = alloca i32, align 4
-; NVPTX-DISABLED-NEXT:    [[DOTTHREADID_TEMP_:%.*]] = alloca i32, align 4
-; NVPTX-DISABLED-NEXT:    [[TMP0:%.*]] = call i32 @__kmpc_target_init(ptr @__omp_offloading_fd02_2044372e_do_not_spmdize_target_l65_kernel_environment)
-; NVPTX-DISABLED-NEXT:    [[THREAD_IS_WORKER:%.*]] = icmp ne i32 [[TMP0]], -1
-; NVPTX-DISABLED-NEXT:    br i1 [[THREAD_IS_WORKER]], label [[IS_WORKER_CHECK:%.*]], label [[THREAD_USER_CODE_CHECK:%.*]]
-; NVPTX-DISABLED:       is_worker_check:
-; NVPTX-DISABLED-NEXT:    [[BLOCK_HW_SIZE:%.*]] = call i32 @__kmpc_get_hardware_num_threads_in_block()
-; NVPTX-DISABLED-NEXT:    [[WARP_SIZE:%.*]] = call i32 @__kmpc_get_warp_size()
-; NVPTX-DISABLED-NEXT:    [[BLOCK_SIZE:%.*]] = sub i32 [[BLOCK_HW_SIZE]], [[WARP_SIZE]]
-; NVPTX-DISABLED-NEXT:    [[THREAD_IS_MAIN_OR_WORKER:%.*]] = icmp slt i32 [[TMP0]], [[BLOCK_SIZE]]
-; NVPTX-DISABLED-NEXT:    br i1 [[THREAD_IS_MAIN_OR_WORKER]], label [[WORKER_STATE_MACHINE_BEGIN:%.*]], label [[WORKER_STATE_MACHINE_FINISHED:%.*]]
-; NVPTX-DISABLED:       worker_state_machine.begin:
-; NVPTX-DISABLED-NEXT:    call void @__kmpc_barrier_simple_generic(ptr @[[GLOB1]], i32 [[TMP0]])
-; NVPTX-DISABLED-NEXT:    [[WORKER_IS_ACTIVE:%.*]] = call i1 @__kmpc_kernel_parallel(ptr [[WORKER_WORK_FN_ADDR]])
-; NVPTX-DISABLED-NEXT:    [[WORKER_WORK_FN:%.*]] = load ptr, ptr [[WORKER_WORK_FN_ADDR]], align 8
-; NVPTX-DISABLED-NEXT:    [[WORKER_IS_DONE:%.*]] = icmp eq ptr [[WORKER_WORK_FN]], null
-; NVPTX-DISABLED-NEXT:    br i1 [[WORKER_IS_DONE]], label [[WORKER_STATE_MACHINE_FINISHED]], label [[WORKER_STATE_MACHINE_IS_ACTIVE_CHECK:%.*]]
-; NVPTX-DISABLED:       worker_state_machine.finished:
-; NVPTX-DISABLED-NEXT:    ret void
-; NVPTX-DISABLED:       worker_state_machine.is_active.check:
-; NVPTX-DISABLED-NEXT:    br i1 [[WORKER_IS_ACTIVE]], label [[WORKER_STATE_MACHINE_PARALLEL_REGION_FALLBACK_EXECUTE:%.*]], label [[WORKER_STATE_MACHINE_DONE_BARRIER:%.*]]
-; NVPTX-DISABLED:       worker_state_machine.parallel_region.fallback.execute:
-; NVPTX-DISABLED-NEXT:    call void [[WORKER_WORK_FN]](i16 0, i32 [[TMP0]])
-; NVPTX-DISABLED-NEXT:    br label [[WORKER_STATE_MACHINE_PARALLEL_REGION_END:%.*]]
-; NVPTX-DISABLED:       worker_state_machine.parallel_region.end:
-; NVPTX-DISABLED-NEXT:    call void @__kmpc_kernel_end_parallel()
-; NVPTX-DISABLED-NEXT:    br label [[WORKER_STATE_MACHINE_DONE_BARRIER]]
-; NVPTX-DISABLED:       worker_state_machine.done.barrier:
-; NVPTX-DISABLED-NEXT:    call void @__kmpc_barrier_simple_generic(ptr @[[GLOB1]], i32 [[TMP0]])
-; NVPTX-DISABLED-NEXT:    br label [[WORKER_STATE_MACHINE_BEGIN]]
-; NVPTX-DISABLED:       thread.user_code.check:
-; NVPTX-DISABLED-NEXT:    [[EXEC_USER_CODE:%.*]] = icmp eq i32 [[TMP0]], -1
-; NVPTX-DISABLED-NEXT:    br i1 [[EXEC_USER_CODE]], label [[USER_CODE_ENTRY:%.*]], label [[COMMON_RET:%.*]]
-; NVPTX-DISABLED:       common.ret:
-; NVPTX-DISABLED-NEXT:    ret void
-; NVPTX-DISABLED:       user_code.entry:
-; NVPTX-DISABLED-NEXT:    [[TMP1:%.*]] = call i32 @__kmpc_global_thread_num(ptr @[[GLOB1]]) #[[ATTR5]]
-; NVPTX-DISABLED-NEXT:    call void @__omp_outlined__8(ptr [[DOTTHREADID_TEMP_]], ptr [[DOTZERO_ADDR]]) #[[ATTR5]]
-; NVPTX-DISABLED-NEXT:    call void @__kmpc_target_deinit()
-; NVPTX-DISABLED-NEXT:    br label [[COMMON_RET]]
 entry:
   %.zero.addr = alloca i32, align 4
   %.threadid_temp. = alloca i32, align 4
@@ -3531,23 +2376,13 @@ define internal void @__omp_outlined__8(ptr noalias %.global_tid., ptr noalias %
 ; NVPTX-DISABLED2-NEXT:    call void @unknown() #[[ATTR8]]
 ; NVPTX-DISABLED2-NEXT:    ret void
 ;
-; AMDGPU-DISABLED-LABEL: define {{[^@]+}}@__omp_outlined__8
-; AMDGPU-DISABLED-SAME: (ptr noalias [[DOTGLOBAL_TID_:%.*]], ptr noalias [[DOTBOUND_TID_:%.*]]) {
-; AMDGPU-DISABLED-NEXT:  entry:
-; AMDGPU-DISABLED-NEXT:    call void @unknown() #[[ATTR8]]
-; AMDGPU-DISABLED-NEXT:    ret void
-; NVPTX-DISABLED-LABEL: define {{[^@]+}}@__omp_outlined__8
-; NVPTX-DISABLED-SAME: (ptr noalias [[DOTGLOBAL_TID_:%.*]], ptr noalias [[DOTBOUND_TID_:%.*]]) {
-; NVPTX-DISABLED-NEXT:  entry:
-; NVPTX-DISABLED-NEXT:    call void @unknown() #[[ATTR8]]
-; NVPTX-DISABLED-NEXT:    ret void
 entry:
   call void @unknown() #11
   ret void
 }
 
 ; Function Attrs: alwaysinline convergent norecurse nounwind
-define weak void @__omp_offloading_fd02_2044372e_do_not_spmdize_task_l74() #0 {
+define weak ptx_kernel void @__omp_offloading_fd02_2044372e_do_not_spmdize_task_l74() #0 {
 ; AMDGPU-LABEL: define {{[^@]+}}@__omp_offloading_fd02_2044372e_do_not_spmdize_task_l74
 ; AMDGPU-SAME: () #[[ATTR0]] {
 ; AMDGPU-NEXT:  entry:
@@ -3709,44 +2544,8 @@ define weak void @__omp_offloading_fd02_2044372e_do_not_spmdize_task_l74() #0 {
 ; AMDGPU-DISABLED2-LABEL: define {{[^@]+}}@__omp_offloading_fd02_2044372e_do_not_spmdize_task_l74
 ; AMDGPU-DISABLED2-SAME: () #[[ATTR0]] {
 ; AMDGPU-DISABLED2-NEXT:  entry:
-; AMDGPU-DISABLED2-NEXT:    [[WORKER_WORK_FN_ADDR:%.*]] = alloca ptr, align 8, addrspace(5)
 ; AMDGPU-DISABLED2-NEXT:    [[CAPTURED_VARS_ADDRS:%.*]] = alloca [0 x ptr], align 8
 ; AMDGPU-DISABLED2-NEXT:    [[TMP0:%.*]] = call i32 @__kmpc_target_init(ptr @__omp_offloading_fd02_2044372e_do_not_spmdize_task_l74_kernel_environment, ptr null)
-; AMDGPU-DISABLED2-NEXT:    [[THREAD_IS_WORKER:%.*]] = icmp ne i32 [[TMP0]], -1
-; AMDGPU-DISABLED2-NEXT:    br i1 [[THREAD_IS_WORKER]], label [[IS_WORKER_CHECK:%.*]], label [[THREAD_USER_CODE_CHECK:%.*]]
-; AMDGPU-DISABLED2:       is_worker_check:
-; AMDGPU-DISABLED2-NEXT:    [[BLOCK_HW_SIZE:%.*]] = call i32 @__kmpc_get_hardware_num_threads_in_block()
-; AMDGPU-DISABLED2-NEXT:    [[WARP_SIZE:%.*]] = call i32 @__kmpc_get_warp_size()
-; AMDGPU-DISABLED2-NEXT:    [[BLOCK_SIZE:%.*]] = sub i32 [[BLOCK_HW_SIZE]], [[WARP_SIZE]]
-; AMDGPU-DISABLED2-NEXT:    [[THREAD_IS_MAIN_OR_WORKER:%.*]] = icmp slt i32 [[TMP0]], [[BLOCK_SIZE]]
-; AMDGPU-DISABLED2-NEXT:    br i1 [[THREAD_IS_MAIN_OR_WORKER]], label [[WORKER_STATE_MACHINE_BEGIN:%.*]], label [[WORKER_STATE_MACHINE_FINISHED:%.*]]
-; AMDGPU-DISABLED2:       worker_state_machine.begin:
-; AMDGPU-DISABLED2-NEXT:    call void @__kmpc_barrier_simple_generic(ptr @[[GLOB1]], i32 [[TMP0]])
-; AMDGPU-DISABLED2-NEXT:    [[WORKER_WORK_FN_ADDR_GENERIC:%.*]] = addrspacecast ptr addrspace(5) [[WORKER_WORK_FN_ADDR]] to ptr
-; AMDGPU-DISABLED2-NEXT:    [[WORKER_IS_ACTIVE:%.*]] = call i1 @__kmpc_kernel_parallel(ptr [[WORKER_WORK_FN_ADDR_GENERIC]])
-; AMDGPU-DISABLED2-NEXT:    [[WORKER_WORK_FN:%.*]] = load ptr, ptr [[WORKER_WORK_FN_ADDR_GENERIC]], align 8
-; AMDGPU-DISABLED2-NEXT:    [[WORKER_IS_DONE:%.*]] = icmp eq ptr [[WORKER_WORK_FN]], null
-; AMDGPU-DISABLED2-NEXT:    br i1 [[WORKER_IS_DONE]], label [[WORKER_STATE_MACHINE_FINISHED]], label [[WORKER_STATE_MACHINE_IS_ACTIVE_CHECK:%.*]]
-; AMDGPU-DISABLED2:       worker_state_machine.finished:
-; AMDGPU-DISABLED2-NEXT:    ret void
-; AMDGPU-DISABLED2:       worker_state_machine.is_active.check:
-; AMDGPU-DISABLED2-NEXT:    br i1 [[WORKER_IS_ACTIVE]], label [[WORKER_STATE_MACHINE_PARALLEL_REGION_CHECK:%.*]], label [[WORKER_STATE_MACHINE_DONE_BARRIER:%.*]]
-; AMDGPU-DISABLED2:       worker_state_machine.parallel_region.check:
-; AMDGPU-DISABLED2-NEXT:    [[WORKER_CHECK_PARALLEL_REGION:%.*]] = icmp eq ptr [[WORKER_WORK_FN]], @__omp_outlined__9_wrapper.ID
-; AMDGPU-DISABLED2-NEXT:    br i1 [[WORKER_CHECK_PARALLEL_REGION]], label [[WORKER_STATE_MACHINE_PARALLEL_REGION_EXECUTE:%.*]], label [[WORKER_STATE_MACHINE_PARALLEL_REGION_FALLBACK_EXECUTE:%.*]]
-; AMDGPU-DISABLED2:       worker_state_machine.parallel_region.execute:
-; AMDGPU-DISABLED2-NEXT:    call void @__omp_outlined__9_wrapper(i16 0, i32 [[TMP0]])
-; AMDGPU-DISABLED2-NEXT:    br label [[WORKER_STATE_MACHINE_PARALLEL_REGION_END:%.*]]
-; AMDGPU-DISABLED2:       worker_state_machine.parallel_region.fallback.execute:
-; AMDGPU-DISABLED2-NEXT:    call void [[WORKER_WORK_FN]](i16 0, i32 [[TMP0]])
-; AMDGPU-DISABLED2-NEXT:    br label [[WORKER_STATE_MACHINE_PARALLEL_REGION_END]]
-; AMDGPU-DISABLED2:       worker_state_machine.parallel_region.end:
-; AMDGPU-DISABLED2-NEXT:    call void @__kmpc_kernel_end_parallel()
-; AMDGPU-DISABLED2-NEXT:    br label [[WORKER_STATE_MACHINE_DONE_BARRIER]]
-; AMDGPU-DISABLED2:       worker_state_machine.done.barrier:
-; AMDGPU-DISABLED2-NEXT:    call void @__kmpc_barrier_simple_generic(ptr @[[GLOB1]], i32 [[TMP0]])
-; AMDGPU-DISABLED2-NEXT:    br label [[WORKER_STATE_MACHINE_BEGIN]]
-; AMDGPU-DISABLED2:       thread.user_code.check:
 ; AMDGPU-DISABLED2-NEXT:    [[EXEC_USER_CODE:%.*]] = icmp eq i32 [[TMP0]], -1
 ; AMDGPU-DISABLED2-NEXT:    br i1 [[EXEC_USER_CODE]], label [[USER_CODE_ENTRY:%.*]], label [[COMMON_RET:%.*]]
 ; AMDGPU-DISABLED2:       common.ret:
@@ -3755,7 +2554,7 @@ define weak void @__omp_offloading_fd02_2044372e_do_not_spmdize_task_l74() #0 {
 ; AMDGPU-DISABLED2-NEXT:    [[TMP1:%.*]] = call i32 @__kmpc_global_thread_num(ptr @[[GLOB1]]) #[[ATTR4]]
 ; AMDGPU-DISABLED2-NEXT:    [[TMP2:%.*]] = call ptr @__kmpc_omp_task_alloc(ptr @[[GLOB1]], i32 [[TMP1]], i32 1, i64 40, i64 0, ptr @"_omp_task_entry$") #[[ATTR4]]
 ; AMDGPU-DISABLED2-NEXT:    [[TMP3:%.*]] = call i32 @__kmpc_omp_task(ptr @[[GLOB1]], i32 [[TMP1]], ptr [[TMP2]]) #[[ATTR4]]
-; AMDGPU-DISABLED2-NEXT:    call void @__kmpc_parallel_51(ptr @[[GLOB1]], i32 [[TMP1]], i32 1, i32 -1, i32 -1, ptr @__omp_outlined__9, ptr @__omp_outlined__9_wrapper.ID, ptr [[CAPTURED_VARS_ADDRS]], i64 0)
+; AMDGPU-DISABLED2-NEXT:    call void @__kmpc_parallel_51(ptr @[[GLOB1]], i32 [[TMP1]], i32 1, i32 -1, i32 -1, ptr @__omp_outlined__9, ptr @__omp_outlined__9_wrapper, ptr [[CAPTURED_VARS_ADDRS]], i64 0)
 ; AMDGPU-DISABLED2-NEXT:    call void @__kmpc_target_deinit()
 ; AMDGPU-DISABLED2-NEXT:    br label [[COMMON_RET]]
 ;
@@ -3814,43 +2613,8 @@ define weak void @__omp_offloading_fd02_2044372e_do_not_spmdize_task_l74() #0 {
 ; NVPTX-DISABLED2-LABEL: define {{[^@]+}}@__omp_offloading_fd02_2044372e_do_not_spmdize_task_l74
 ; NVPTX-DISABLED2-SAME: () #[[ATTR0]] {
 ; NVPTX-DISABLED2-NEXT:  entry:
-; NVPTX-DISABLED2-NEXT:    [[WORKER_WORK_FN_ADDR:%.*]] = alloca ptr, align 8
 ; NVPTX-DISABLED2-NEXT:    [[CAPTURED_VARS_ADDRS:%.*]] = alloca [0 x ptr], align 8
 ; NVPTX-DISABLED2-NEXT:    [[TMP0:%.*]] = call i32 @__kmpc_target_init(ptr @__omp_offloading_fd02_2044372e_do_not_spmdize_task_l74_kernel_environment, ptr null)
-; NVPTX-DISABLED2-NEXT:    [[THREAD_IS_WORKER:%.*]] = icmp ne i32 [[TMP0]], -1
-; NVPTX-DISABLED2-NEXT:    br i1 [[THREAD_IS_WORKER]], label [[IS_WORKER_CHECK:%.*]], label [[THREAD_USER_CODE_CHECK:%.*]]
-; NVPTX-DISABLED2:       is_worker_check:
-; NVPTX-DISABLED2-NEXT:    [[BLOCK_HW_SIZE:%.*]] = call i32 @__kmpc_get_hardware_num_threads_in_block()
-; NVPTX-DISABLED2-NEXT:    [[WARP_SIZE:%.*]] = call i32 @__kmpc_get_warp_size()
-; NVPTX-DISABLED2-NEXT:    [[BLOCK_SIZE:%.*]] = sub i32 [[BLOCK_HW_SIZE]], [[WARP_SIZE]]
-; NVPTX-DISABLED2-NEXT:    [[THREAD_IS_MAIN_OR_WORKER:%.*]] = icmp slt i32 [[TMP0]], [[BLOCK_SIZE]]
-; NVPTX-DISABLED2-NEXT:    br i1 [[THREAD_IS_MAIN_OR_WORKER]], label [[WORKER_STATE_MACHINE_BEGIN:%.*]], label [[WORKER_STATE_MACHINE_FINISHED:%.*]]
-; NVPTX-DISABLED2:       worker_state_machine.begin:
-; NVPTX-DISABLED2-NEXT:    call void @__kmpc_barrier_simple_generic(ptr @[[GLOB1]], i32 [[TMP0]])
-; NVPTX-DISABLED2-NEXT:    [[WORKER_IS_ACTIVE:%.*]] = call i1 @__kmpc_kernel_parallel(ptr [[WORKER_WORK_FN_ADDR]])
-; NVPTX-DISABLED2-NEXT:    [[WORKER_WORK_FN:%.*]] = load ptr, ptr [[WORKER_WORK_FN_ADDR]], align 8
-; NVPTX-DISABLED2-NEXT:    [[WORKER_IS_DONE:%.*]] = icmp eq ptr [[WORKER_WORK_FN]], null
-; NVPTX-DISABLED2-NEXT:    br i1 [[WORKER_IS_DONE]], label [[WORKER_STATE_MACHINE_FINISHED]], label [[WORKER_STATE_MACHINE_IS_ACTIVE_CHECK:%.*]]
-; NVPTX-DISABLED2:       worker_state_machine.finished:
-; NVPTX-DISABLED2-NEXT:    ret void
-; NVPTX-DISABLED2:       worker_state_machine.is_active.check:
-; NVPTX-DISABLED2-NEXT:    br i1 [[WORKER_IS_ACTIVE]], label [[WORKER_STATE_MACHINE_PARALLEL_REGION_CHECK:%.*]], label [[WORKER_STATE_MACHINE_DONE_BARRIER:%.*]]
-; NVPTX-DISABLED2:       worker_state_machine.parallel_region.check:
-; NVPTX-DISABLED2-NEXT:    [[WORKER_CHECK_PARALLEL_REGION:%.*]] = icmp eq ptr [[WORKER_WORK_FN]], @__omp_outlined__9_wrapper.ID
-; NVPTX-DISABLED2-NEXT:    br i1 [[WORKER_CHECK_PARALLEL_REGION]], label [[WORKER_STATE_MACHINE_PARALLEL_REGION_EXECUTE:%.*]], label [[WORKER_STATE_MACHINE_PARALLEL_REGION_FALLBACK_EXECUTE:%.*]]
-; NVPTX-DISABLED2:       worker_state_machine.parallel_region.execute:
-; NVPTX-DISABLED2-NEXT:    call void @__omp_outlined__9_wrapper(i16 0, i32 [[TMP0]])
-; NVPTX-DISABLED2-NEXT:    br label [[WORKER_STATE_MACHINE_PARALLEL_REGION_END:%.*]]
-; NVPTX-DISABLED2:       worker_state_machine.parallel_region.fallback.execute:
-; NVPTX-DISABLED2-NEXT:    call void [[WORKER_WORK_FN]](i16 0, i32 [[TMP0]])
-; NVPTX-DISABLED2-NEXT:    br label [[WORKER_STATE_MACHINE_PARALLEL_REGION_END]]
-; NVPTX-DISABLED2:       worker_state_machine.parallel_region.end:
-; NVPTX-DISABLED2-NEXT:    call void @__kmpc_kernel_end_parallel()
-; NVPTX-DISABLED2-NEXT:    br label [[WORKER_STATE_MACHINE_DONE_BARRIER]]
-; NVPTX-DISABLED2:       worker_state_machine.done.barrier:
-; NVPTX-DISABLED2-NEXT:    call void @__kmpc_barrier_simple_generic(ptr @[[GLOB1]], i32 [[TMP0]])
-; NVPTX-DISABLED2-NEXT:    br label [[WORKER_STATE_MACHINE_BEGIN]]
-; NVPTX-DISABLED2:       thread.user_code.check:
 ; NVPTX-DISABLED2-NEXT:    [[EXEC_USER_CODE:%.*]] = icmp eq i32 [[TMP0]], -1
 ; NVPTX-DISABLED2-NEXT:    br i1 [[EXEC_USER_CODE]], label [[USER_CODE_ENTRY:%.*]], label [[COMMON_RET:%.*]]
 ; NVPTX-DISABLED2:       common.ret:
@@ -3859,113 +2623,10 @@ define weak void @__omp_offloading_fd02_2044372e_do_not_spmdize_task_l74() #0 {
 ; NVPTX-DISABLED2-NEXT:    [[TMP1:%.*]] = call i32 @__kmpc_global_thread_num(ptr @[[GLOB1]]) #[[ATTR4]]
 ; NVPTX-DISABLED2-NEXT:    [[TMP2:%.*]] = call ptr @__kmpc_omp_task_alloc(ptr @[[GLOB1]], i32 [[TMP1]], i32 1, i64 40, i64 0, ptr @"_omp_task_entry$") #[[ATTR4]]
 ; NVPTX-DISABLED2-NEXT:    [[TMP3:%.*]] = call i32 @__kmpc_omp_task(ptr @[[GLOB1]], i32 [[TMP1]], ptr [[TMP2]]) #[[ATTR4]]
-; NVPTX-DISABLED2-NEXT:    call void @__kmpc_parallel_51(ptr @[[GLOB1]], i32 [[TMP1]], i32 1, i32 -1, i32 -1, ptr @__omp_outlined__9, ptr @__omp_outlined__9_wrapper.ID, ptr [[CAPTURED_VARS_ADDRS]], i64 0)
+; NVPTX-DISABLED2-NEXT:    call void @__kmpc_parallel_51(ptr @[[GLOB1]], i32 [[TMP1]], i32 1, i32 -1, i32 -1, ptr @__omp_outlined__9, ptr @__omp_outlined__9_wrapper, ptr [[CAPTURED_VARS_ADDRS]], i64 0)
 ; NVPTX-DISABLED2-NEXT:    call void @__kmpc_target_deinit()
 ; NVPTX-DISABLED2-NEXT:    br label [[COMMON_RET]]
 ;
-; AMDGPU-DISABLED-LABEL: define {{[^@]+}}@__omp_offloading_fd02_2044372e_do_not_spmdize_task_l74
-; AMDGPU-DISABLED-SAME: () #[[ATTR0]] {
-; AMDGPU-DISABLED-NEXT:  entry:
-; AMDGPU-DISABLED-NEXT:    [[WORKER_WORK_FN_ADDR:%.*]] = alloca ptr, align 8, addrspace(5)
-; AMDGPU-DISABLED-NEXT:    [[CAPTURED_VARS_ADDRS:%.*]] = alloca [0 x ptr], align 8
-; AMDGPU-DISABLED-NEXT:    [[TMP0:%.*]] = call i32 @__kmpc_target_init(ptr @__omp_offloading_fd02_2044372e_do_not_spmdize_task_l74_kernel_environment)
-; AMDGPU-DISABLED-NEXT:    [[THREAD_IS_WORKER:%.*]] = icmp ne i32 [[TMP0]], -1
-; AMDGPU-DISABLED-NEXT:    br i1 [[THREAD_IS_WORKER]], label [[IS_WORKER_CHECK:%.*]], label [[THREAD_USER_CODE_CHECK:%.*]]
-; AMDGPU-DISABLED:       is_worker_check:
-; AMDGPU-DISABLED-NEXT:    [[BLOCK_HW_SIZE:%.*]] = call i32 @__kmpc_get_hardware_num_threads_in_block()
-; AMDGPU-DISABLED-NEXT:    [[WARP_SIZE:%.*]] = call i32 @__kmpc_get_warp_size()
-; AMDGPU-DISABLED-NEXT:    [[BLOCK_SIZE:%.*]] = sub i32 [[BLOCK_HW_SIZE]], [[WARP_SIZE]]
-; AMDGPU-DISABLED-NEXT:    [[THREAD_IS_MAIN_OR_WORKER:%.*]] = icmp slt i32 [[TMP0]], [[BLOCK_SIZE]]
-; AMDGPU-DISABLED-NEXT:    br i1 [[THREAD_IS_MAIN_OR_WORKER]], label [[WORKER_STATE_MACHINE_BEGIN:%.*]], label [[WORKER_STATE_MACHINE_FINISHED:%.*]]
-; AMDGPU-DISABLED:       worker_state_machine.begin:
-; AMDGPU-DISABLED-NEXT:    call void @__kmpc_barrier_simple_generic(ptr @[[GLOB1]], i32 [[TMP0]])
-; AMDGPU-DISABLED-NEXT:    [[WORKER_WORK_FN_ADDR_GENERIC:%.*]] = addrspacecast ptr addrspace(5) [[WORKER_WORK_FN_ADDR]] to ptr
-; AMDGPU-DISABLED-NEXT:    [[WORKER_IS_ACTIVE:%.*]] = call i1 @__kmpc_kernel_parallel(ptr [[WORKER_WORK_FN_ADDR_GENERIC]])
-; AMDGPU-DISABLED-NEXT:    [[WORKER_WORK_FN:%.*]] = load ptr, ptr [[WORKER_WORK_FN_ADDR_GENERIC]], align 8
-; AMDGPU-DISABLED-NEXT:    [[WORKER_IS_DONE:%.*]] = icmp eq ptr [[WORKER_WORK_FN]], null
-; AMDGPU-DISABLED-NEXT:    br i1 [[WORKER_IS_DONE]], label [[WORKER_STATE_MACHINE_FINISHED]], label [[WORKER_STATE_MACHINE_IS_ACTIVE_CHECK:%.*]]
-; AMDGPU-DISABLED:       worker_state_machine.finished:
-; AMDGPU-DISABLED-NEXT:    ret void
-; AMDGPU-DISABLED:       worker_state_machine.is_active.check:
-; AMDGPU-DISABLED-NEXT:    br i1 [[WORKER_IS_ACTIVE]], label [[WORKER_STATE_MACHINE_PARALLEL_REGION_CHECK:%.*]], label [[WORKER_STATE_MACHINE_DONE_BARRIER:%.*]]
-; AMDGPU-DISABLED:       worker_state_machine.parallel_region.check:
-; AMDGPU-DISABLED-NEXT:    [[WORKER_CHECK_PARALLEL_REGION:%.*]] = icmp eq ptr [[WORKER_WORK_FN]], @__omp_outlined__9_wrapper.ID
-; AMDGPU-DISABLED-NEXT:    br i1 [[WORKER_CHECK_PARALLEL_REGION]], label [[WORKER_STATE_MACHINE_PARALLEL_REGION_EXECUTE:%.*]], label [[WORKER_STATE_MACHINE_PARALLEL_REGION_FALLBACK_EXECUTE:%.*]]
-; AMDGPU-DISABLED:       worker_state_machine.parallel_region.execute:
-; AMDGPU-DISABLED-NEXT:    call void @__omp_outlined__9_wrapper(i16 0, i32 [[TMP0]])
-; AMDGPU-DISABLED-NEXT:    br label [[WORKER_STATE_MACHINE_PARALLEL_REGION_END:%.*]]
-; AMDGPU-DISABLED:       worker_state_machine.parallel_region.fallback.execute:
-; AMDGPU-DISABLED-NEXT:    call void [[WORKER_WORK_FN]](i16 0, i32 [[TMP0]])
-; AMDGPU-DISABLED-NEXT:    br label [[WORKER_STATE_MACHINE_PARALLEL_REGION_END]]
-; AMDGPU-DISABLED:       worker_state_machine.parallel_region.end:
-; AMDGPU-DISABLED-NEXT:    call void @__kmpc_kernel_end_parallel()
-; AMDGPU-DISABLED-NEXT:    br label [[WORKER_STATE_MACHINE_DONE_BARRIER]]
-; AMDGPU-DISABLED:       worker_state_machine.done.barrier:
-; AMDGPU-DISABLED-NEXT:    call void @__kmpc_barrier_simple_generic(ptr @[[GLOB1]], i32 [[TMP0]])
-; AMDGPU-DISABLED-NEXT:    br label [[WORKER_STATE_MACHINE_BEGIN]]
-; AMDGPU-DISABLED:       thread.user_code.check:
-; AMDGPU-DISABLED-NEXT:    [[EXEC_USER_CODE:%.*]] = icmp eq i32 [[TMP0]], -1
-; AMDGPU-DISABLED-NEXT:    br i1 [[EXEC_USER_CODE]], label [[USER_CODE_ENTRY:%.*]], label [[COMMON_RET:%.*]]
-; AMDGPU-DISABLED:       common.ret:
-; AMDGPU-DISABLED-NEXT:    ret void
-; AMDGPU-DISABLED:       user_code.entry:
-; AMDGPU-DISABLED-NEXT:    [[TMP1:%.*]] = call i32 @__kmpc_global_thread_num(ptr @[[GLOB1]]) #[[ATTR4]]
-; AMDGPU-DISABLED-NEXT:    [[TMP2:%.*]] = call ptr @__kmpc_omp_task_alloc(ptr @[[GLOB1]], i32 [[TMP1]], i32 1, i64 40, i64 0, ptr @"_omp_task_entry$") #[[ATTR4]]
-; AMDGPU-DISABLED-NEXT:    [[TMP3:%.*]] = call i32 @__kmpc_omp_task(ptr @[[GLOB1]], i32 [[TMP1]], ptr [[TMP2]]) #[[ATTR4]]
-; AMDGPU-DISABLED-NEXT:    call void @__kmpc_parallel_51(ptr @[[GLOB1]], i32 [[TMP1]], i32 1, i32 -1, i32 -1, ptr @__omp_outlined__9, ptr @__omp_outlined__9_wrapper.ID, ptr [[CAPTURED_VARS_ADDRS]], i64 0)
-; AMDGPU-DISABLED-NEXT:    call void @__kmpc_target_deinit()
-; AMDGPU-DISABLED-NEXT:    br label [[COMMON_RET]]
-; NVPTX-DISABLED-LABEL: define {{[^@]+}}@__omp_offloading_fd02_2044372e_do_not_spmdize_task_l74
-; NVPTX-DISABLED-SAME: () #[[ATTR0]] {
-; NVPTX-DISABLED-NEXT:  entry:
-; NVPTX-DISABLED-NEXT:    [[WORKER_WORK_FN_ADDR:%.*]] = alloca ptr, align 8
-; NVPTX-DISABLED-NEXT:    [[CAPTURED_VARS_ADDRS:%.*]] = alloca [0 x ptr], align 8
-; NVPTX-DISABLED-NEXT:    [[TMP0:%.*]] = call i32 @__kmpc_target_init(ptr @__omp_offloading_fd02_2044372e_do_not_spmdize_task_l74_kernel_environment)
-; NVPTX-DISABLED-NEXT:    [[THREAD_IS_WORKER:%.*]] = icmp ne i32 [[TMP0]], -1
-; NVPTX-DISABLED-NEXT:    br i1 [[THREAD_IS_WORKER]], label [[IS_WORKER_CHECK:%.*]], label [[THREAD_USER_CODE_CHECK:%.*]]
-; NVPTX-DISABLED:       is_worker_check:
-; NVPTX-DISABLED-NEXT:    [[BLOCK_HW_SIZE:%.*]] = call i32 @__kmpc_get_hardware_num_threads_in_block()
-; NVPTX-DISABLED-NEXT:    [[WARP_SIZE:%.*]] = call i32 @__kmpc_get_warp_size()
-; NVPTX-DISABLED-NEXT:    [[BLOCK_SIZE:%.*]] = sub i32 [[BLOCK_HW_SIZE]], [[WARP_SIZE]]
-; NVPTX-DISABLED-NEXT:    [[THREAD_IS_MAIN_OR_WORKER:%.*]] = icmp slt i32 [[TMP0]], [[BLOCK_SIZE]]
-; NVPTX-DISABLED-NEXT:    br i1 [[THREAD_IS_MAIN_OR_WORKER]], label [[WORKER_STATE_MACHINE_BEGIN:%.*]], label [[WORKER_STATE_MACHINE_FINISHED:%.*]]
-; NVPTX-DISABLED:       worker_state_machine.begin:
-; NVPTX-DISABLED-NEXT:    call void @__kmpc_barrier_simple_generic(ptr @[[GLOB1]], i32 [[TMP0]])
-; NVPTX-DISABLED-NEXT:    [[WORKER_IS_ACTIVE:%.*]] = call i1 @__kmpc_kernel_parallel(ptr [[WORKER_WORK_FN_ADDR]])
-; NVPTX-DISABLED-NEXT:    [[WORKER_WORK_FN:%.*]] = load ptr, ptr [[WORKER_WORK_FN_ADDR]], align 8
-; NVPTX-DISABLED-NEXT:    [[WORKER_IS_DONE:%.*]] = icmp eq ptr [[WORKER_WORK_FN]], null
-; NVPTX-DISABLED-NEXT:    br i1 [[WORKER_IS_DONE]], label [[WORKER_STATE_MACHINE_FINISHED]], label [[WORKER_STATE_MACHINE_IS_ACTIVE_CHECK:%.*]]
-; NVPTX-DISABLED:       worker_state_machine.finished:
-; NVPTX-DISABLED-NEXT:    ret void
-; NVPTX-DISABLED:       worker_state_machine.is_active.check:
-; NVPTX-DISABLED-NEXT:    br i1 [[WORKER_IS_ACTIVE]], label [[WORKER_STATE_MACHINE_PARALLEL_REGION_CHECK:%.*]], label [[WORKER_STATE_MACHINE_DONE_BARRIER:%.*]]
-; NVPTX-DISABLED:       worker_state_machine.parallel_region.check:
-; NVPTX-DISABLED-NEXT:    [[WORKER_CHECK_PARALLEL_REGION:%.*]] = icmp eq ptr [[WORKER_WORK_FN]], @__omp_outlined__9_wrapper.ID
-; NVPTX-DISABLED-NEXT:    br i1 [[WORKER_CHECK_PARALLEL_REGION]], label [[WORKER_STATE_MACHINE_PARALLEL_REGION_EXECUTE:%.*]], label [[WORKER_STATE_MACHINE_PARALLEL_REGION_FALLBACK_EXECUTE:%.*]]
-; NVPTX-DISABLED:       worker_state_machine.parallel_region.execute:
-; NVPTX-DISABLED-NEXT:    call void @__omp_outlined__9_wrapper(i16 0, i32 [[TMP0]])
-; NVPTX-DISABLED-NEXT:    br label [[WORKER_STATE_MACHINE_PARALLEL_REGION_END:%.*]]
-; NVPTX-DISABLED:       worker_state_machine.parallel_region.fallback.execute:
-; NVPTX-DISABLED-NEXT:    call void [[WORKER_WORK_FN]](i16 0, i32 [[TMP0]])
-; NVPTX-DISABLED-NEXT:    br label [[WORKER_STATE_MACHINE_PARALLEL_REGION_END]]
-; NVPTX-DISABLED:       worker_state_machine.parallel_region.end:
-; NVPTX-DISABLED-NEXT:    call void @__kmpc_kernel_end_parallel()
-; NVPTX-DISABLED-NEXT:    br label [[WORKER_STATE_MACHINE_DONE_BARRIER]]
-; NVPTX-DISABLED:       worker_state_machine.done.barrier:
-; NVPTX-DISABLED-NEXT:    call void @__kmpc_barrier_simple_generic(ptr @[[GLOB1]], i32 [[TMP0]])
-; NVPTX-DISABLED-NEXT:    br label [[WORKER_STATE_MACHINE_BEGIN]]
-; NVPTX-DISABLED:       thread.user_code.check:
-; NVPTX-DISABLED-NEXT:    [[EXEC_USER_CODE:%.*]] = icmp eq i32 [[TMP0]], -1
-; NVPTX-DISABLED-NEXT:    br i1 [[EXEC_USER_CODE]], label [[USER_CODE_ENTRY:%.*]], label [[COMMON_RET:%.*]]
-; NVPTX-DISABLED:       common.ret:
-; NVPTX-DISABLED-NEXT:    ret void
-; NVPTX-DISABLED:       user_code.entry:
-; NVPTX-DISABLED-NEXT:    [[TMP1:%.*]] = call i32 @__kmpc_global_thread_num(ptr @[[GLOB1]]) #[[ATTR4]]
-; NVPTX-DISABLED-NEXT:    [[TMP2:%.*]] = call ptr @__kmpc_omp_task_alloc(ptr @[[GLOB1]], i32 [[TMP1]], i32 1, i64 40, i64 0, ptr @"_omp_task_entry$") #[[ATTR4]]
-; NVPTX-DISABLED-NEXT:    [[TMP3:%.*]] = call i32 @__kmpc_omp_task(ptr @[[GLOB1]], i32 [[TMP1]], ptr [[TMP2]]) #[[ATTR4]]
-; NVPTX-DISABLED-NEXT:    call void @__kmpc_parallel_51(ptr @[[GLOB1]], i32 [[TMP1]], i32 1, i32 -1, i32 -1, ptr @__omp_outlined__9, ptr @__omp_outlined__9_wrapper.ID, ptr [[CAPTURED_VARS_ADDRS]], i64 0)
-; NVPTX-DISABLED-NEXT:    call void @__kmpc_target_deinit()
-; NVPTX-DISABLED-NEXT:    br label [[COMMON_RET]]
 entry:
   %captured_vars_addrs = alloca [0 x ptr], align 8
   %0 = call i32 @__kmpc_target_init(ptr @__omp_offloading_fd02_2044372e_do_not_spmdize_task_l74_kernel_environment, ptr null)
@@ -4022,16 +2683,6 @@ define internal void @.omp_outlined.(i32 %.global_tid., ptr noalias %.part_id.,
 ; NVPTX-DISABLED2-NEXT:    call void @spmd_amenable() #[[ATTR7]]
 ; NVPTX-DISABLED2-NEXT:    ret void
 ;
-; AMDGPU-DISABLED-LABEL: define {{[^@]+}}@.omp_outlined.
-; AMDGPU-DISABLED-SAME: (i32 [[DOTGLOBAL_TID_:%.*]], ptr noalias [[DOTPART_ID_:%.*]], ptr noalias [[DOTPRIVATES_:%.*]], ptr noalias [[DOTCOPY_FN_:%.*]], ptr [[DOTTASK_T_:%.*]], ptr noalias [[__CONTEXT:%.*]]) #[[ATTR3:[0-9]+]] {
-; AMDGPU-DISABLED-NEXT:  entry:
-; AMDGPU-DISABLED-NEXT:    call void @spmd_amenable() #[[ATTR7]]
-; AMDGPU-DISABLED-NEXT:    ret void
-; NVPTX-DISABLED-LABEL: define {{[^@]+}}@.omp_outlined.
-; NVPTX-DISABLED-SAME: (i32 [[DOTGLOBAL_TID_:%.*]], ptr noalias [[DOTPART_ID_:%.*]], ptr noalias [[DOTPRIVATES_:%.*]], ptr noalias [[DOTCOPY_FN_:%.*]], ptr [[DOTTASK_T_:%.*]], ptr noalias [[__CONTEXT:%.*]]) #[[ATTR3:[0-9]+]] {
-; NVPTX-DISABLED-NEXT:  entry:
-; NVPTX-DISABLED-NEXT:    call void @spmd_amenable() #[[ATTR7]]
-; NVPTX-DISABLED-NEXT:    ret void
 entry:
   call void @spmd_amenable() #10
   ret void
@@ -4094,12 +2745,6 @@ define weak i32 @__kmpc_target_init(ptr, ptr) {
 ; NVPTX-DISABLED2-SAME: (ptr [[TMP0:%.*]], ptr [[TMP1:%.*]]) {
 ; NVPTX-DISABLED2-NEXT:    ret i32 0
 ;
-; AMDGPU-DISABLED-LABEL: define {{[^@]+}}@__kmpc_target_init
-; AMDGPU-DISABLED-SAME: (ptr [[TMP0:%.*]]) {
-; AMDGPU-DISABLED-NEXT:    ret i32 0
-; NVPTX-DISABLED-LABEL: define {{[^@]+}}@__kmpc_target_init
-; NVPTX-DISABLED-SAME: (ptr [[TMP0:%.*]]) {
-; NVPTX-DISABLED-NEXT:    ret i32 0
   ret i32 0
 }
 
@@ -4158,16 +2803,6 @@ define internal void @__omp_outlined__9(ptr noalias %.global_tid., ptr noalias %
 ; NVPTX-DISABLED2-NEXT:    call void @unknown() #[[ATTR8]]
 ; NVPTX-DISABLED2-NEXT:    ret void
 ;
-; AMDGPU-DISABLED-LABEL: define {{[^@]+}}@__omp_outlined__9
-; AMDGPU-DISABLED-SAME: (ptr noalias [[DOTGLOBAL_TID_:%.*]], ptr noalias [[DOTBOUND_TID_:%.*]]) {
-; AMDGPU-DISABLED-NEXT:  entry:
-; AMDGPU-DISABLED-NEXT:    call void @unknown() #[[ATTR8]]
-; AMDGPU-DISABLED-NEXT:    ret void
-; NVPTX-DISABLED-LABEL: define {{[^@]+}}@__omp_outlined__9
-; NVPTX-DISABLED-SAME: (ptr noalias [[DOTGLOBAL_TID_:%.*]], ptr noalias [[DOTBOUND_TID_:%.*]]) {
-; NVPTX-DISABLED-NEXT:  entry:
-; NVPTX-DISABLED-NEXT:    call void @unknown() #[[ATTR8]]
-; NVPTX-DISABLED-NEXT:    ret void
 entry:
   call void @unknown() #11
   ret void
@@ -4235,24 +2870,6 @@ define internal void @__omp_outlined__9_wrapper(i16 zeroext %0, i32 %1) #3 {
 ; NVPTX-DISABLED2-NEXT:    call void @__omp_outlined__9(ptr [[DOTADDR1]], ptr [[DOTZERO_ADDR]]) #[[ATTR4]]
 ; NVPTX-DISABLED2-NEXT:    ret void
 ;
-; AMDGPU-DISABLED-LABEL: define {{[^@]+}}@__omp_outlined__9_wrapper
-; AMDGPU-DISABLED-SAME: (i16 zeroext [[TMP0:%.*]], i32 [[TMP1:%.*]]) #[[ATTR2]] {
-; AMDGPU-DISABLED-NEXT:  entry:
-; AMDGPU-DISABLED-NEXT:    [[DOTADDR1:%.*]] = alloca i32, align 4
-; AMDGPU-DISABLED-NEXT:    [[DOTZERO_ADDR:%.*]] = alloca i32, align 4
-; AMDGPU-DISABLED-NEXT:    [[GLOBAL_ARGS:%.*]] = alloca ptr, align 8
-; AMDGPU-DISABLED-NEXT:    call void @__kmpc_get_shared_variables(ptr [[GLOBAL_ARGS]])
-; AMDGPU-DISABLED-NEXT:    call void @__omp_outlined__9(ptr [[DOTADDR1]], ptr [[DOTZERO_ADDR]]) #[[ATTR4]]
-; AMDGPU-DISABLED-NEXT:    ret void
-; NVPTX-DISABLED-LABEL: define {{[^@]+}}@__omp_outlined__9_wrapper
-; NVPTX-DISABLED-SAME: (i16 zeroext [[TMP0:%.*]], i32 [[TMP1:%.*]]) #[[ATTR2]] {
-; NVPTX-DISABLED-NEXT:  entry:
-; NVPTX-DISABLED-NEXT:    [[DOTADDR1:%.*]] = alloca i32, align 4
-; NVPTX-DISABLED-NEXT:    [[DOTZERO_ADDR:%.*]] = alloca i32, align 4
-; NVPTX-DISABLED-NEXT:    [[GLOBAL_ARGS:%.*]] = alloca ptr, align 8
-; NVPTX-DISABLED-NEXT:    call void @__kmpc_get_shared_variables(ptr [[GLOBAL_ARGS]])
-; NVPTX-DISABLED-NEXT:    call void @__omp_outlined__9(ptr [[DOTADDR1]], ptr [[DOTZERO_ADDR]]) #[[ATTR4]]
-; NVPTX-DISABLED-NEXT:    ret void
 entry:
   %.addr1 = alloca i32, align 4
   %.zero.addr = alloca i32, align 4
@@ -4280,7 +2897,6 @@ attributes #10 = { convergent "llvm.assume"="ompx_spmd_amenable" }
 attributes #11 = { convergent }
 
 !omp_offload.info = !{!0, !1, !2, !3, !4, !5}
-!nvvm.annotations = !{!6, !7, !8, !9, !10, !11}
 !llvm.module.flags = !{!12, !13, !14, !15, !16}
 !llvm.ident = !{!17}
 
@@ -4290,12 +2906,6 @@ attributes #11 = { convergent }
 !3 = !{i32 0, i32 64770, i32 541341486, !"sequential_loop_to_shared_var", i32 35, i32 2}
 !4 = !{i32 0, i32 64770, i32 541341486, !"do_not_spmdize_target", i32 65, i32 4}
 !5 = !{i32 0, i32 64770, i32 541341486, !"sequential_loop_to_shared_var_guarded", i32 50, i32 3}
-!6 = !{ptr @__omp_offloading_fd02_2044372e_sequential_loop_l5, !"kernel", i32 1}
-!7 = !{ptr @__omp_offloading_fd02_2044372e_sequential_loop_to_stack_var_l20, !"kernel", i32 1}
-!8 = !{ptr @__omp_offloading_fd02_2044372e_sequential_loop_to_shared_var_l35, !"kernel", i32 1}
-!9 = !{ptr @__omp_offloading_fd02_2044372e_sequential_loop_to_shared_var_guarded_l50, !"kernel", i32 1}
-!10 = !{ptr @__omp_offloading_fd02_2044372e_do_not_spmdize_target_l65, !"kernel", i32 1}
-!11 = !{ptr @__omp_offloading_fd02_2044372e_do_not_spmdize_task_l74, !"kernel", i32 1}
 !12 = !{i32 1, !"wchar_size", i32 4}
 !13 = !{i32 7, !"openmp", i32 50}
 !14 = !{i32 7, !"openmp-device", i32 50}
@@ -4317,92 +2927,6 @@ attributes #11 = { convergent }
 !30 = !{!31, !27, i64 0}
 !31 = !{!"kmp_task_t_with_privates", !32, i64 0}
 !32 = !{!"kmp_task_t", !27, i64 0, !27, i64 8, !19, i64 16, !20, i64 24, !20, i64 32}
-; AMDGPU-DISABLED: attributes #[[ATTR0]] = { alwaysinline convergent norecurse nounwind }
-; AMDGPU-DISABLED: attributes #[[ATTR1]] = { norecurse }
-; AMDGPU-DISABLED: attributes #[[ATTR2]] = { convergent norecurse nounwind }
-; AMDGPU-DISABLED: attributes #[[ATTR3]] = { alwaysinline norecurse nounwind }
-; AMDGPU-DISABLED: attributes #[[ATTR4]] = { alwaysinline convergent nounwind }
-; AMDGPU-DISABLED: attributes #[[ATTR5]] = { nounwind }
-; AMDGPU-DISABLED: attributes #[[ATTR6:[0-9]+]] = { nosync nounwind }
-; AMDGPU-DISABLED: attributes #[[ATTR7:[0-9]+]] = { nofree nosync nounwind allocsize(0) }
-; AMDGPU-DISABLED: attributes #[[ATTR8]] = { convergent "llvm.assume"="ompx_spmd_amenable" }
-; AMDGPU-DISABLED: attributes #[[ATTR9]] = { convergent }
-; AMDGPU-DISABLED: attributes #[[ATTR10:[0-9]+]] = { nocallback nofree nosync nounwind willreturn memory(argmem: readwrite) }
-; AMDGPU-DISABLED: attributes #[[ATTR11:[0-9]+]] = { alwaysinline }
-; AMDGPU-DISABLED: attributes #[[ATTR12:[0-9]+]] = { convergent nounwind }
-; NVPTX-DISABLED: attributes #[[ATTR0]] = { alwaysinline convergent norecurse nounwind }
-; NVPTX-DISABLED: attributes #[[ATTR1]] = { norecurse }
-; NVPTX-DISABLED: attributes #[[ATTR2]] = { convergent norecurse nounwind }
-; NVPTX-DISABLED: attributes #[[ATTR3]] = { alwaysinline norecurse nounwind }
-; NVPTX-DISABLED: attributes #[[ATTR4]] = { alwaysinline convergent nounwind }
-; NVPTX-DISABLED: attributes #[[ATTR5]] = { nounwind }
-; NVPTX-DISABLED: attributes #[[ATTR6:[0-9]+]] = { nosync nounwind }
-; NVPTX-DISABLED: attributes #[[ATTR7:[0-9]+]] = { nofree nosync nounwind allocsize(0) }
-; NVPTX-DISABLED: attributes #[[ATTR8]] = { convergent "llvm.assume"="ompx_spmd_amenable" }
-; NVPTX-DISABLED: attributes #[[ATTR9]] = { convergent }
-; NVPTX-DISABLED: attributes #[[ATTR10:[0-9]+]] = { nocallback nofree nosync nounwind willreturn memory(argmem: readwrite) }
-; NVPTX-DISABLED: attributes #[[ATTR11:[0-9]+]] = { alwaysinline }
-; NVPTX-DISABLED: attributes #[[ATTR12:[0-9]+]] = { convergent nounwind }
-; AMDGPU-DISABLED: [[META0:![0-9]+]] = !{i32 0, i32 64770, i32 541341486, !"do_not_spmdize_task", i32 74, i32 5}
-; AMDGPU-DISABLED: [[META1:![0-9]+]] = !{i32 0, i32 64770, i32 541341486, !"sequential_loop_to_stack_var", i32 20, i32 1}
-; AMDGPU-DISABLED: [[META2:![0-9]+]] = !{i32 0, i32 64770, i32 541341486, !"sequential_loop", i32 5, i32 0}
-; AMDGPU-DISABLED: [[META3:![0-9]+]] = !{i32 0, i32 64770, i32 541341486, !"sequential_loop_to_shared_var", i32 35, i32 2}
-; AMDGPU-DISABLED: [[META4:![0-9]+]] = !{i32 0, i32 64770, i32 541341486, !"do_not_spmdize_target", i32 65, i32 4}
-; AMDGPU-DISABLED: [[META5:![0-9]+]] = !{i32 0, i32 64770, i32 541341486, !"sequential_loop_to_shared_var_guarded", i32 50, i32 3}
-; AMDGPU-DISABLED: [[META6:![0-9]+]] = !{ptr @__omp_offloading_fd02_2044372e_sequential_loop_l5, !"kernel", i32 1}
-; AMDGPU-DISABLED: [[META7:![0-9]+]] = !{ptr @__omp_offloading_fd02_2044372e_sequential_loop_to_stack_var_l20, !"kernel", i32 1}
-; AMDGPU-DISABLED: [[META8:![0-9]+]] = !{ptr @__omp_offloading_fd02_2044372e_sequential_loop_to_shared_var_l35, !"kernel", i32 1}
-; AMDGPU-DISABLED: [[META9:![0-9]+]] = !{ptr @__omp_offloading_fd02_2044372e_sequential_loop_to_shared_var_guarded_l50, !"kernel", i32 1}
-; AMDGPU-DISABLED: [[META10:![0-9]+]] = !{ptr @__omp_offloading_fd02_2044372e_do_not_spmdize_target_l65, !"kernel", i32 1}
-; AMDGPU-DISABLED: [[META11:![0-9]+]] = !{ptr @__omp_offloading_fd02_2044372e_do_not_spmdize_task_l74, !"kernel", i32 1}
-; AMDGPU-DISABLED: [[META12:![0-9]+]] = !{i32 1, !"wchar_size", i32 4}
-; AMDGPU-DISABLED: [[META13:![0-9]+]] = !{i32 7, !"openmp", i32 50}
-; AMDGPU-DISABLED: [[META14:![0-9]+]] = !{i32 7, !"openmp-device", i32 50}
-; AMDGPU-DISABLED: [[META15:![0-9]+]] = !{i32 8, !"PIC Level", i32 2}
-; AMDGPU-DISABLED: [[META16:![0-9]+]] = !{i32 7, !"frame-pointer", i32 2}
-; AMDGPU-DISABLED: [[META17:![0-9]+]] = !{!"clang version 14.0.0"}
-; AMDGPU-DISABLED: [[TBAA18]] = !{!19, !19, i64 0}
-; AMDGPU-DISABLED: [[META19:![0-9]+]] = !{!"int", !20, i64 0}
-; AMDGPU-DISABLED: [[META20:![0-9]+]] = !{!"omnipotent char", !21, i64 0}
-; AMDGPU-DISABLED: [[META21:![0-9]+]] = !{!"Simple C/C++ TBAA"}
-; AMDGPU-DISABLED: [[LOOP22]] = distinct !{!22, !23, !24}
-; AMDGPU-DISABLED: [[META23:![0-9]+]] = !{!"llvm.loop.mustprogress"}
-; AMDGPU-DISABLED: [[META24:![0-9]+]] = !{!"llvm.loop.unroll.disable"}
-; AMDGPU-DISABLED: [[LOOP25]] = distinct !{!25, !23, !24}
-; AMDGPU-DISABLED: [[TBAA26]] = !{!27, !27, i64 0}
-; AMDGPU-DISABLED: [[META27:![0-9]+]] = !{!"any pointer", !20, i64 0}
-; AMDGPU-DISABLED: [[LOOP28]] = distinct !{!28, !23, !24}
-; AMDGPU-DISABLED: [[LOOP29]] = distinct !{!29, !23, !24}
-; NVPTX-DISABLED: [[META0:![0-9]+]] = !{i32 0, i32 64770, i32 541341486, !"do_not_spmdize_task", i32 74, i32 5}
-; NVPTX-DISABLED: [[META1:![0-9]+]] = !{i32 0, i32 64770, i32 541341486, !"sequential_loop_to_stack_var", i32 20, i32 1}
-; NVPTX-DISABLED: [[META2:![0-9]+]] = !{i32 0, i32 64770, i32 541341486, !"sequential_loop", i32 5, i32 0}
-; NVPTX-DISABLED: [[META3:![0-9]+]] = !{i32 0, i32 64770, i32 541341486, !"sequential_loop_to_shared_var", i32 35, i32 2}
-; NVPTX-DISABLED: [[META4:![0-9]+]] = !{i32 0, i32 64770, i32 541341486, !"do_not_spmdize_target", i32 65, i32 4}
-; NVPTX-DISABLED: [[META5:![0-9]+]] = !{i32 0, i32 64770, i32 541341486, !"sequential_loop_to_shared_var_guarded", i32 50, i32 3}
-; NVPTX-DISABLED: [[META6:![0-9]+]] = !{ptr @__omp_offloading_fd02_2044372e_sequential_loop_l5, !"kernel", i32 1}
-; NVPTX-DISABLED: [[META7:![0-9]+]] = !{ptr @__omp_offloading_fd02_2044372e_sequential_loop_to_stack_var_l20, !"kernel", i32 1}
-; NVPTX-DISABLED: [[META8:![0-9]+]] = !{ptr @__omp_offloading_fd02_2044372e_sequential_loop_to_shared_var_l35, !"kernel", i32 1}
-; NVPTX-DISABLED: [[META9:![0-9]+]] = !{ptr @__omp_offloading_fd02_2044372e_sequential_loop_to_shared_var_guarded_l50, !"kernel", i32 1}
-; NVPTX-DISABLED: [[META10:![0-9]+]] = !{ptr @__omp_offloading_fd02_2044372e_do_not_spmdize_target_l65, !"kernel", i32 1}
-; NVPTX-DISABLED: [[META11:![0-9]+]] = !{ptr @__omp_offloading_fd02_2044372e_do_not_spmdize_task_l74, !"kernel", i32 1}
-; NVPTX-DISABLED: [[META12:![0-9]+]] = !{i32 1, !"wchar_size", i32 4}
-; NVPTX-DISABLED: [[META13:![0-9]+]] = !{i32 7, !"openmp", i32 50}
-; NVPTX-DISABLED: [[META14:![0-9]+]] = !{i32 7, !"openmp-device", i32 50}
-; NVPTX-DISABLED: [[META15:![0-9]+]] = !{i32 8, !"PIC Level", i32 2}
-; NVPTX-DISABLED: [[META16:![0-9]+]] = !{i32 7, !"frame-pointer", i32 2}
-; NVPTX-DISABLED: [[META17:![0-9]+]] = !{!"clang version 14.0.0"}
-; NVPTX-DISABLED: [[TBAA18]] = !{!19, !19, i64 0}
-; NVPTX-DISABLED: [[META19:![0-9]+]] = !{!"int", !20, i64 0}
-; NVPTX-DISABLED: [[META20:![0-9]+]] = !{!"omnipotent char", !21, i64 0}
-; NVPTX-DISABLED: [[META21:![0-9]+]] = !{!"Simple C/C++ TBAA"}
-; NVPTX-DISABLED: [[LOOP22]] = distinct !{!22, !23, !24}
-; NVPTX-DISABLED: [[META23:![0-9]+]] = !{!"llvm.loop.mustprogress"}
-; NVPTX-DISABLED: [[META24:![0-9]+]] = !{!"llvm.loop.unroll.disable"}
-; NVPTX-DISABLED: [[LOOP25]] = distinct !{!25, !23, !24}
-; NVPTX-DISABLED: [[TBAA26]] = !{!27, !27, i64 0}
-; NVPTX-DISABLED: [[META27:![0-9]+]] = !{!"any pointer", !20, i64 0}
-; NVPTX-DISABLED: [[LOOP28]] = distinct !{!28, !23, !24}
-; NVPTX-DISABLED: [[LOOP29]] = distinct !{!29, !23, !24}
 ;.
 ; AMDGPU: attributes #[[ATTR0]] = { alwaysinline convergent norecurse nounwind "kernel" }
 ; AMDGPU: attributes #[[ATTR1]] = { norecurse }
@@ -4454,7 +2978,6 @@ attributes #11 = { convergent }
 ; AMDGPU-DISABLED2: attributes #[[ATTR8]] = { convergent }
 ; AMDGPU-DISABLED2: attributes #[[ATTR9:[0-9]+]] = { nocallback nofree nosync nounwind willreturn memory(argmem: readwrite) }
 ; AMDGPU-DISABLED2: attributes #[[ATTR10:[0-9]+]] = { alwaysinline }
-; AMDGPU-DISABLED2: attributes #[[ATTR11:[0-9]+]] = { convergent nounwind }
 ;.
 ; NVPTX-DISABLED1: attributes #[[ATTR0]] = { alwaysinline convergent norecurse nounwind "kernel" }
 ; NVPTX-DISABLED1: attributes #[[ATTR1]] = { norecurse }
@@ -4480,7 +3003,6 @@ attributes #11 = { convergent }
 ; NVPTX-DISABLED2: attributes #[[ATTR8]] = { convergent }
 ; NVPTX-DISABLED2: attributes #[[ATTR9:[0-9]+]] = { nocallback nofree nosync nounwind willreturn memory(argmem: readwrite) }
 ; NVPTX-DISABLED2: attributes #[[ATTR10:[0-9]+]] = { alwaysinline }
-; NVPTX-DISABLED2: attributes #[[ATTR11:[0-9]+]] = { convergent nounwind }
 ;.
 ; AMDGPU: [[META0:![0-9]+]] = !{i32 0, i32 64770, i32 541341486, !"do_not_spmdize_task", i32 74, i32 5}
 ; AMDGPU: [[META1:![0-9]+]] = !{i32 0, i32 64770, i32 541341486, !"sequential_loop_to_stack_var", i32 20, i32 1}
@@ -4488,30 +3010,24 @@ attributes #11 = { convergent }
 ; AMDGPU: [[META3:![0-9]+]] = !{i32 0, i32 64770, i32 541341486, !"sequential_loop_to_shared_var", i32 35, i32 2}
 ; AMDGPU: [[META4:![0-9]+]] = !{i32 0, i32 64770, i32 541341486, !"do_not_spmdize_target", i32 65, i32 4}
 ; AMDGPU: [[META5:![0-9]+]] = !{i32 0, i32 64770, i32 541341486, !"sequential_loop_to_shared_var_guarded", i32 50, i32 3}
-; AMDGPU: [[META6:![0-9]+]] = !{ptr @__omp_offloading_fd02_2044372e_sequential_loop_l5, !"kernel", i32 1}
-; AMDGPU: [[META7:![0-9]+]] = !{ptr @__omp_offloading_fd02_2044372e_sequential_loop_to_stack_var_l20, !"kernel", i32 1}
-; AMDGPU: [[META8:![0-9]+]] = !{ptr @__omp_offloading_fd02_2044372e_sequential_loop_to_shared_var_l35, !"kernel", i32 1}
-; AMDGPU: [[META9:![0-9]+]] = !{ptr @__omp_offloading_fd02_2044372e_sequential_loop_to_shared_var_guarded_l50, !"kernel", i32 1}
-; AMDGPU: [[META10:![0-9]+]] = !{ptr @__omp_offloading_fd02_2044372e_do_not_spmdize_target_l65, !"kernel", i32 1}
-; AMDGPU: [[META11:![0-9]+]] = !{ptr @__omp_offloading_fd02_2044372e_do_not_spmdize_task_l74, !"kernel", i32 1}
-; AMDGPU: [[META12:![0-9]+]] = !{i32 1, !"wchar_size", i32 4}
-; AMDGPU: [[META13:![0-9]+]] = !{i32 7, !"openmp", i32 50}
-; AMDGPU: [[META14:![0-9]+]] = !{i32 7, !"openmp-device", i32 50}
-; AMDGPU: [[META15:![0-9]+]] = !{i32 8, !"PIC Level", i32 2}
-; AMDGPU: [[META16:![0-9]+]] = !{i32 7, !"frame-pointer", i32 2}
-; AMDGPU: [[META17:![0-9]+]] = !{!"{{.*}}clang version {{.*}}"}
-; AMDGPU: [[TBAA18]] = !{[[META19:![0-9]+]], [[META19]], i64 0}
-; AMDGPU: [[META19]] = !{!"int", [[META20:![0-9]+]], i64 0}
-; AMDGPU: [[META20]] = !{!"omnipotent char", [[META21:![0-9]+]], i64 0}
-; AMDGPU: [[META21]] = !{!"Simple C/C++ TBAA"}
-; AMDGPU: [[LOOP22]] = distinct !{[[LOOP22]], [[META23:![0-9]+]], [[META24:![0-9]+]]}
-; AMDGPU: [[META23]] = !{!"llvm.loop.mustprogress"}
-; AMDGPU: [[META24]] = !{!"llvm.loop.unroll.disable"}
-; AMDGPU: [[LOOP25]] = distinct !{[[LOOP25]], [[META23]], [[META24]]}
-; AMDGPU: [[TBAA26]] = !{[[META27:![0-9]+]], [[META27]], i64 0}
-; AMDGPU: [[META27]] = !{!"any pointer", [[META20]], i64 0}
-; AMDGPU: [[LOOP28]] = distinct !{[[LOOP28]], [[META23]], [[META24]]}
-; AMDGPU: [[LOOP29]] = distinct !{[[LOOP29]], [[META23]], [[META24]]}
+; AMDGPU: [[META6:![0-9]+]] = !{i32 1, !"wchar_size", i32 4}
+; AMDGPU: [[META7:![0-9]+]] = !{i32 7, !"openmp", i32 50}
+; AMDGPU: [[META8:![0-9]+]] = !{i32 7, !"openmp-device", i32 50}
+; AMDGPU: [[META9:![0-9]+]] = !{i32 8, !"PIC Level", i32 2}
+; AMDGPU: [[META10:![0-9]+]] = !{i32 7, !"frame-pointer", i32 2}
+; AMDGPU: [[META11:![0-9]+]] = !{!"{{.*}}clang version {{.*}}"}
+; AMDGPU: [[TBAA12]] = !{[[META13:![0-9]+]], [[META13]], i64 0}
+; AMDGPU: [[META13]] = !{!"int", [[META14:![0-9]+]], i64 0}
+; AMDGPU: [[META14]] = !{!"omnipotent char", [[META15:![0-9]+]], i64 0}
+; AMDGPU: [[META15]] = !{!"Simple C/C++ TBAA"}
+; AMDGPU: [[LOOP16]] = distinct !{[[LOOP16]], [[META17:![0-9]+]], [[META18:![0-9]+]]}
+; AMDGPU: [[META17]] = !{!"llvm.loop.mustprogress"}
+; AMDGPU: [[META18]] = !{!"llvm.loop.unroll.disable"}
+; AMDGPU: [[LOOP19]] = distinct !{[[LOOP19]], [[META17]], [[META18]]}
+; AMDGPU: [[TBAA20]] = !{[[META21:![0-9]+]], [[META21]], i64 0}
+; AMDGPU: [[META21]] = !{!"any pointer", [[META14]], i64 0}
+; AMDGPU: [[LOOP22]] = distinct !{[[LOOP22]], [[META17]], [[META18]]}
+; AMDGPU: [[LOOP23]] = distinct !{[[LOOP23]], [[META17]], [[META18]]}
 ;.
 ; NVPTX: [[META0:![0-9]+]] = !{i32 0, i32 64770, i32 541341486, !"do_not_spmdize_task", i32 74, i32 5}
 ; NVPTX: [[META1:![0-9]+]] = !{i32 0, i32 64770, i32 541341486, !"sequential_loop_to_stack_var", i32 20, i32 1}
@@ -4519,30 +3035,24 @@ attributes #11 = { convergent }
 ; NVPTX: [[META3:![0-9]+]] = !{i32 0, i32 64770, i32 541341486, !"sequential_loop_to_shared_var", i32 35, i32 2}
 ; NVPTX: [[META4:![0-9]+]] = !{i32 0, i32 64770, i32 541341486, !"do_not_spmdize_target", i32 65, i32 4}
 ; NVPTX: [[META5:![0-9]+]] = !{i32 0, i32 64770, i32 541341486, !"sequential_loop_to_shared_var_guarded", i32 50, i32 3}
-; NVPTX: [[META6:![0-9]+]] = !{ptr @__omp_offloading_fd02_2044372e_sequential_loop_l5, !"kernel", i32 1}
-; NVPTX: [[META7:![0-9]+]] = !{ptr @__omp_offloading_fd02_2044372e_sequential_loop_to_stack_var_l20, !"kernel", i32 1}
-; NVPTX: [[META8:![0-9]+]] = !{ptr @__omp_offloading_fd02_2044372e_sequential_loop_to_shared_var_l35, !"kernel", i32 1}
-; NVPTX: [[META9:![0-9]+]] = !{ptr @__omp_offloading_fd02_2044372e_sequential_loop_to_shared_var_guarded_l50, !"kernel", i32 1}
-; NVPTX: [[META10:![0-9]+]] = !{ptr @__omp_offloading_fd02_2044372e_do_not_spmdize_target_l65, !"kernel", i32 1}
-; NVPTX: [[META11:![0-9]+]] = !{ptr @__omp_offloading_fd02_2044372e_do_not_spmdize_task_l74, !"kernel", i32 1}
-; NVPTX: [[META12:![0-9]+]] = !{i32 1, !"wchar_size", i32 4}
-; NVPTX: [[META13:![0-9]+]] = !{i32 7, !"openmp", i32 50}
-; NVPTX: [[META14:![0-9]+]] = !{i32 7, !"openmp-device", i32 50}
-; NVPTX: [[META15:![0-9]+]] = !{i32 8, !"PIC Level", i32 2}
-; NVPTX: [[META16:![0-9]+]] = !{i32 7, !"frame-pointer", i32 2}
-; NVPTX: [[META17:![0-9]+]] = !{!"{{.*}}clang version {{.*}}"}
-; NVPTX: [[TBAA18]] = !{[[META19:![0-9]+]], [[META19]], i64 0}
-; NVPTX: [[META19]] = !{!"int", [[META20:![0-9]+]], i64 0}
-; NVPTX: [[META20]] = !{!"omnipotent char", [[META21:![0-9]+]], i64 0}
-; NVPTX: [[META21]] = !{!"Simple C/C++ TBAA"}
-; NVPTX: [[LOOP22]] = distinct !{[[LOOP22]], [[META23:![0-9]+]], [[META24:![0-9]+]]}
-; NVPTX: [[META23]] = !{!"llvm.loop.mustprogress"}
-; NVPTX: [[META24]] = !{!"llvm.loop.unroll.disable"}
-; NVPTX: [[LOOP25]] = distinct !{[[LOOP25]], [[META23]], [[META24]]}
-; NVPTX: [[TBAA26]] = !{[[META27:![0-9]+]], [[META27]], i64 0}
-; NVPTX: [[META27]] = !{!"any pointer", [[META20]], i64 0}
-; NVPTX: [[LOOP28]] = distinct !{[[LOOP28]], [[META23]], [[META24]]}
-; NVPTX: [[LOOP29]] = distinct !{[[LOOP29]], [[META23]], [[META24]]}
+; NVPTX: [[META6:![0-9]+]] = !{i32 1, !"wchar_size", i32 4}
+; NVPTX: [[META7:![0-9]+]] = !{i32 7, !"openmp", i32 50}
+; NVPTX: [[META8:![0-9]+]] = !{i32 7, !"openmp-device", i32 50}
+; NVPTX: [[META9:![0-9]+]] = !{i32 8, !"PIC Level", i32 2}
+; NVPTX: [[META10:![0-9]+]] = !{i32 7, !"frame-pointer", i32 2}
+; NVPTX: [[META11:![0-9]+]] = !{!"{{.*}}clang version {{.*}}"}
+; NVPTX: [[TBAA12]] = !{[[META13:![0-9]+]], [[META13]], i64 0}
+; NVPTX: [[META13]] = !{!"int", [[META14:![0-9]+]], i64 0}
+; NVPTX: [[META14]] = !{!"omnipotent char", [[META15:![0-9]+]], i64 0}
+; NVPTX: [[META15]] = !{!"Simple C/C++ TBAA"}
+; NVPTX: [[LOOP16]] = distinct !{[[LOOP16]], [[META17:![0-9]+]], [[META18:![0-9]+]]}
+; NVPTX: [[META17]] = !{!"llvm.loop.mustprogress"}
+; NVPTX: [[META18]] = !{!"llvm.loop.unroll.disable"}
+; NVPTX: [[LOOP19]] = distinct !{[[LOOP19]], [[META17]], [[META18]]}
+; NVPTX: [[TBAA20]] = !{[[META21:![0-9]+]], [[META21]], i64 0}
+; NVPTX: [[META21]] = !{!"any pointer", [[META14]], i64 0}
+; NVPTX: [[LOOP22]] = distinct !{[[LOOP22]], [[META17]], [[META18]]}
+; NVPTX: [[LOOP23]] = distinct !{[[LOOP23]], [[META17]], [[META18]]}
 ;.
 ; AMDGPU-DISABLED1: [[META0:![0-9]+]] = !{i32 0, i32 64770, i32 541341486, !"do_not_spmdize_task", i32 74, i32 5}
 ; AMDGPU-DISABLED1: [[META1:![0-9]+]] = !{i32 0, i32 64770, i32 541341486, !"sequential_loop_to_stack_var", i32 20, i32 1}
@@ -4550,30 +3060,24 @@ attributes #11 = { convergent }
 ; AMDGPU-DISABLED1: [[META3:![0-9]+]] = !{i32 0, i32 64770, i32 541341486, !"sequential_loop_to_shared_var", i32 35, i32 2}
 ; AMDGPU-DISABLED1: [[META4:![0-9]+]] = !{i32 0, i32 64770, i32 541341486, !"do_not_spmdize_target", i32 65, i32 4}
 ; AMDGPU-DISABLED1: [[META5:![0-9]+]] = !{i32 0, i32 64770, i32 541341486, !"sequential_loop_to_shared_var_guarded", i32 50, i32 3}
-; AMDGPU-DISABLED1: [[META6:![0-9]+]] = !{ptr @__omp_offloading_fd02_2044372e_sequential_loop_l5, !"kernel", i32 1}
-; AMDGPU-DISABLED1: [[META7:![0-9]+]] = !{ptr @__omp_offloading_fd02_2044372e_sequential_loop_to_stack_var_l20, !"kernel", i32 1}
-; AMDGPU-DISABLED1: [[META8:![0-9]+]] = !{ptr @__omp_offloading_fd02_2044372e_sequential_loop_to_shared_var_l35, !"kernel", i32 1}
-; AMDGPU-DISABLED1: [[META9:![0-9]+]] = !{ptr @__omp_offloading_fd02_2044372e_sequential_loop_to_shared_var_guarded_l50, !"kernel", i32 1}
-; AMDGPU-DISABLED1: [[META10:![0-9]+]] = !{ptr @__omp_offloading_fd02_2044372e_do_not_spmdize_target_l65, !"kernel", i32 1}
-; AMDGPU-DISABLED1: [[META11:![0-9]+]] = !{ptr @__omp_offloading_fd02_2044372e_do_not_spmdize_task_l74, !"kernel", i32 1}
-; AMDGPU-DISABLED1: [[META12:![0-9]+]] = !{i32 1, !"wchar_size", i32 4}
-; AMDGPU-DISABLED1: [[META13:![0-9]+]] = !{i32 7, !"openmp", i32 50}
-; AMDGPU-DISABLED1: [[META14:![0-9]+]] = !{i32 7, !"openmp-device", i32 50}
-; AMDGPU-DISABLED1: [[META15:![0-9]+]] = !{i32 8, !"PIC Level", i32 2}
-; AMDGPU-DISABLED1: [[META16:![0-9]+]] = !{i32 7, !"frame-pointer", i32 2}
-; AMDGPU-DISABLED1: [[META17:![0-9]+]] = !{!"{{.*}}clang version {{.*}}"}
-; AMDGPU-DISABLED1: [[TBAA18]] = !{[[META19:![0-9]+]], [[META19]], i64 0}
-; AMDGPU-DISABLED1: [[META19]] = !{!"int", [[META20:![0-9]+]], i64 0}
-; AMDGPU-DISABLED1: [[META20]] = !{!"omnipotent char", [[META21:![0-9]+]], i64 0}
-; AMDGPU-DISABLED1: [[META21]] = !{!"Simple C/C++ TBAA"}
-; AMDGPU-DISABLED1: [[LOOP22]] = distinct !{[[LOOP22]], [[META23:![0-9]+]], [[META24:![0-9]+]]}
-; AMDGPU-DISABLED1: [[META23]] = !{!"llvm.loop.mustprogress"}
-; AMDGPU-DISABLED1: [[META24]] = !{!"llvm.loop.unroll.disable"}
-; AMDGPU-DISABLED1: [[LOOP25]] = distinct !{[[LOOP25]], [[META23]], [[META24]]}
-; AMDGPU-DISABLED1: [[TBAA26]] = !{[[META27:![0-9]+]], [[META27]], i64 0}
-; AMDGPU-DISABLED1: [[META27]] = !{!"any pointer", [[META20]], i64 0}
-; AMDGPU-DISABLED1: [[LOOP28]] = distinct !{[[LOOP28]], [[META23]], [[META24]]}
-; AMDGPU-DISABLED1: [[LOOP29]] = distinct !{[[LOOP29]], [[META23]], [[META24]]}
+; AMDGPU-DISABLED1: [[META6:![0-9]+]] = !{i32 1, !"wchar_size", i32 4}
+; AMDGPU-DISABLED1: [[META7:![0-9]+]] = !{i32 7, !"openmp", i32 50}
+; AMDGPU-DISABLED1: [[META8:![0-9]+]] = !{i32 7, !"openmp-device", i32 50}
+; AMDGPU-DISABLED1: [[META9:![0-9]+]] = !{i32 8, !"PIC Level", i32 2}
+; AMDGPU-DISABLED1: [[META10:![0-9]+]] = !{i32 7, !"frame-pointer", i32 2}
+; AMDGPU-DISABLED1: [[META11:![0-9]+]] = !{!"{{.*}}clang version {{.*}}"}
+; AMDGPU-DISABLED1: [[TBAA12]] = !{[[META13:![0-9]+]], [[META13]], i64 0}
+; AMDGPU-DISABLED1: [[META13]] = !{!"int", [[META14:![0-9]+]], i64 0}
+; AMDGPU-DISABLED1: [[META14]] = !{!"omnipotent char", [[META15:![0-9]+]], i64 0}
+; AMDGPU-DISABLED1: [[META15]] = !{!"Simple C/C++ TBAA"}
+; AMDGPU-DISABLED1: [[LOOP16]] = distinct !{[[LOOP16]], [[META17:![0-9]+]], [[META18:![0-9]+]]}
+; AMDGPU-DISABLED1: [[META17]] = !{!"llvm.loop.mustprogress"}
+; AMDGPU-DISABLED1: [[META18]] = !{!"llvm.loop.unroll.disable"}
+; AMDGPU-DISABLED1: [[LOOP19]] = distinct !{[[LOOP19]], [[META17]], [[META18]]}
+; AMDGPU-DISABLED1: [[TBAA20]] = !{[[META21:![0-9]+]], [[META21]], i64 0}
+; AMDGPU-DISABLED1: [[META21]] = !{!"any pointer", [[META14]], i64 0}
+; AMDGPU-DISABLED1: [[LOOP22]] = distinct !{[[LOOP22]], [[META17]], [[META18]]}
+; AMDGPU-DISABLED1: [[LOOP23]] = distinct !{[[LOOP23]], [[META17]], [[META18]]}
 ;.
 ; AMDGPU-DISABLED2: [[META0:![0-9]+]] = !{i32 0, i32 64770, i32 541341486, !"do_not_spmdize_task", i32 74, i32 5}
 ; AMDGPU-DISABLED2: [[META1:![0-9]+]] = !{i32 0, i32 64770, i32 541341486, !"sequential_loop_to_stack_var", i32 20, i32 1}
@@ -4581,30 +3085,24 @@ attributes #11 = { convergent }
 ; AMDGPU-DISABLED2: [[META3:![0-9]+]] = !{i32 0, i32 64770, i32 541341486, !"sequential_loop_to_shared_var", i32 35, i32 2}
 ; AMDGPU-DISABLED2: [[META4:![0-9]+]] = !{i32 0, i32 64770, i32 541341486, !"do_not_spmdize_target", i32 65, i32 4}
 ; AMDGPU-DISABLED2: [[META5:![0-9]+]] = !{i32 0, i32 64770, i32 541341486, !"sequential_loop_to_shared_var_guarded", i32 50, i32 3}
-; AMDGPU-DISABLED2: [[META6:![0-9]+]] = !{ptr @__omp_offloading_fd02_2044372e_sequential_loop_l5, !"kernel", i32 1}
-; AMDGPU-DISABLED2: [[META7:![0-9]+]] = !{ptr @__omp_offloading_fd02_2044372e_sequential_loop_to_stack_var_l20, !"kernel", i32 1}
-; AMDGPU-DISABLED2: [[META8:![0-9]+]] = !{ptr @__omp_offloading_fd02_2044372e_sequential_loop_to_shared_var_l35, !"kernel", i32 1}
-; AMDGPU-DISABLED2: [[META9:![0-9]+]] = !{ptr @__omp_offloading_fd02_2044372e_sequential_loop_to_shared_var_guarded_l50, !"kernel", i32 1}
-; AMDGPU-DISABLED2: [[META10:![0-9]+]] = !{ptr @__omp_offloading_fd02_2044372e_do_not_spmdize_target_l65, !"kernel", i32 1}
-; AMDGPU-DISABLED2: [[META11:![0-9]+]] = !{ptr @__omp_offloading_fd02_2044372e_do_not_spmdize_task_l74, !"kernel", i32 1}
-; AMDGPU-DISABLED2: [[META12:![0-9]+]] = !{i32 1, !"wchar_size", i32 4}
-; AMDGPU-DISABLED2: [[META13:![0-9]+]] = !{i32 7, !"openmp", i32 50}
-; AMDGPU-DISABLED2: [[META14:![0-9]+]] = !{i32 7, !"openmp-device", i32 50}
-; AMDGPU-DISABLED2: [[META15:![0-9]+]] = !{i32 8, !"PIC Level", i32 2}
-; AMDGPU-DISABLED2: [[META16:![0-9]+]] = !{i32 7, !"frame-pointer", i32 2}
-; AMDGPU-DISABLED2: [[META17:![0-9]+]] = !{!"{{.*}}clang version {{.*}}"}
-; AMDGPU-DISABLED2: [[TBAA18]] = !{[[META19:![0-9]+]], [[META19]], i64 0}
-; AMDGPU-DISABLED2: [[META19]] = !{!"int", [[META20:![0-9]+]], i64 0}
-; AMDGPU-DISABLED2: [[META20]] = !{!"omnipotent char", [[META21:![0-9]+]], i64 0}
-; AMDGPU-DISABLED2: [[META21]] = !{!"Simple C/C++ TBAA"}
-; AMDGPU-DISABLED2: [[LOOP22]] = distinct !{[[LOOP22]], [[META23:![0-9]+]], [[META24:![0-9]+]]}
-; AMDGPU-DISABLED2: [[META23]] = !{!"llvm.loop.mustprogress"}
-; AMDGPU-DISABLED2: [[META24]] = !{!"llvm.loop.unroll.disable"}
-; AMDGPU-DISABLED2: [[LOOP25]] = distinct !{[[LOOP25]], [[META23]], [[META24]]}
-; AMDGPU-DISABLED2: [[TBAA26]] = !{[[META27:![0-9]+]], [[META27]], i64 0}
-; AMDGPU-DISABLED2: [[META27]] = !{!"any pointer", [[META20]], i64 0}
-; AMDGPU-DISABLED2: [[LOOP28]] = distinct !{[[LOOP28]], [[META23]], [[META24]]}
-; AMDGPU-DISABLED2: [[LOOP29]] = distinct !{[[LOOP29]], [[META23]], [[META24]]}
+; AMDGPU-DISABLED2: [[META6:![0-9]+]] = !{i32 1, !"wchar_size", i32 4}
+; AMDGPU-DISABLED2: [[META7:![0-9]+]] = !{i32 7, !"openmp", i32 50}
+; AMDGPU-DISABLED2: [[META8:![0-9]+]] = !{i32 7, !"openmp-device", i32 50}
+; AMDGPU-DISABLED2: [[META9:![0-9]+]] = !{i32 8, !"PIC Level", i32 2}
+; AMDGPU-DISABLED2: [[META10:![0-9]+]] = !{i32 7, !"frame-pointer", i32 2}
+; AMDGPU-DISABLED2: [[META11:![0-9]+]] = !{!"{{.*}}clang version {{.*}}"}
+; AMDGPU-DISABLED2: [[TBAA12]] = !{[[META13:![0-9]+]], [[META13]], i64 0}
+; AMDGPU-DISABLED2: [[META13]] = !{!"int", [[META14:![0-9]+]], i64 0}
+; AMDGPU-DISABLED2: [[META14]] = !{!"omnipotent char", [[META15:![0-9]+]], i64 0}
+; AMDGPU-DISABLED2: [[META15]] = !{!"Simple C/C++ TBAA"}
+; AMDGPU-DISABLED2: [[LOOP16]] = distinct !{[[LOOP16]], [[META17:![0-9]+]], [[META18:![0-9]+]]}
+; AMDGPU-DISABLED2: [[META17]] = !{!"llvm.loop.mustprogress"}
+; AMDGPU-DISABLED2: [[META18]] = !{!"llvm.loop.unroll.disable"}
+; AMDGPU-DISABLED2: [[LOOP19]] = distinct !{[[LOOP19]], [[META17]], [[META18]]}
+; AMDGPU-DISABLED2: [[TBAA20]] = !{[[META21:![0-9]+]], [[META21]], i64 0}
+; AMDGPU-DISABLED2: [[META21]] = !{!"any pointer", [[META14]], i64 0}
+; AMDGPU-DISABLED2: [[LOOP22]] = distinct !{[[LOOP22]], [[META17]], [[META18]]}
+; AMDGPU-DISABLED2: [[LOOP23]] = distinct !{[[LOOP23]], [[META17]], [[META18]]}
 ;.
 ; NVPTX-DISABLED1: [[META0:![0-9]+]] = !{i32 0, i32 64770, i32 541341486, !"do_not_spmdize_task", i32 74, i32 5}
 ; NVPTX-DISABLED1: [[META1:![0-9]+]] = !{i32 0, i32 64770, i32 541341486, !"sequential_loop_to_stack_var", i32 20, i32 1}
@@ -4612,30 +3110,24 @@ attributes #11 = { convergent }
 ; NVPTX-DISABLED1: [[META3:![0-9]+]] = !{i32 0, i32 64770, i32 541341486, !"sequential_loop_to_shared_var", i32 35, i32 2}
 ; NVPTX-DISABLED1: [[META4:![0-9]+]] = !{i32 0, i32 64770, i32 541341486, !"do_not_spmdize_target", i32 65, i32 4}
 ; NVPTX-DISABLED1: [[META5:![0-9]+]] = !{i32 0, i32 64770, i32 541341486, !"sequential_loop_to_shared_var_guarded", i32 50, i32 3}
-; NVPTX-DISABLED1: [[META6:![0-9]+]] = !{ptr @__omp_offloading_fd02_2044372e_sequential_loop_l5, !"kernel", i32 1}
-; NVPTX-DISABLED1: [[META7:![0-9]+]] = !{ptr @__omp_offloading_fd02_2044372e_sequential_loop_to_stack_var_l20, !"kernel", i32 1}
-; NVPTX-DISABLED1: [[META8:![0-9]+]] = !{ptr @__omp_offloading_fd02_2044372e_sequential_loop_to_shared_var_l35, !"kernel", i32 1}
-; NVPTX-DISABLED1: [[META9:![0-9]+]] = !{ptr @__omp_offloading_fd02_2044372e_sequential_loop_to_shared_var_guarded_l50, !"kernel", i32 1}
-; NVPTX-DISABLED1: [[META10:![0-9]+]] = !{ptr @__omp_offloading_fd02_2044372e_do_not_spmdize_target_l65, !"kernel", i32 1}
-; NVPTX-DISABLED1: [[META11:![0-9]+]] = !{ptr @__omp_offloading_fd02_2044372e_do_not_spmdize_task_l74, !"kernel", i32 1}
-; NVPTX-DISABLED1: [[META12:![0-9]+]] = !{i32 1, !"wchar_size", i32 4}
-; NVPTX-DISABLED1: [[META13:![0-9]+]] = !{i32 7, !"openmp", i32 50}
-; NVPTX-DISABLED1: [[META14:![0-9]+]] = !{i32 7, !"openmp-device", i32 50}
-; NVPTX-DISABLED1: [[META15:![0-9]+]] = !{i32 8, !"PIC Level", i32 2}
-; NVPTX-DISABLED1: [[META16:![0-9]+]] = !{i32 7, !"frame-pointer", i32 2}
-; NVPTX-DISABLED1: [[META17:![0-9]+]] = !{!"{{.*}}clang version {{.*}}"}
-; NVPTX-DISABLED1: [[TBAA18]] = !{[[META19:![0-9]+]], [[META19]], i64 0}
-; NVPTX-DISABLED1: [[META19]] = !{!"int", [[META20:![0-9]+]], i64 0}
-; NVPTX-DISABLED1: [[META20]] = !{!"omnipotent char", [[META21:![0-9]+]], i64 0}
-; NVPTX-DISABLED1: [[META21]] = !{!"Simple C/C++ TBAA"}
-; NVPTX-DISABLED1: [[LOOP22]] = distinct !{[[LOOP22]], [[META23:![0-9]+]], [[META24:![0-9]+]]}
-; NVPTX-DISABLED1: [[META23]] = !{!"llvm.loop.mustprogress"}
-; NVPTX-DISABLED1: [[META24]] = !{!"llvm.loop.unroll.disable"}
-; NVPTX-DISABLED1: [[LOOP25]] = distinct !{[[LOOP25]], [[META23]], [[META24]]}
-; NVPTX-DISABLED1: [[TBAA26]] = !{[[META27:![0-9]+]], [[META27]], i64 0}
-; NVPTX-DISABLED1: [[META27]] = !{!"any pointer", [[META20]], i64 0}
-; NVPTX-DISABLED1: [[LOOP28]] = distinct !{[[LOOP28]], [[META23]], [[META24]]}
-; NVPTX-DISABLED1: [[LOOP29]] = distinct !{[[LOOP29]], [[META23]], [[META24]]}
+; NVPTX-DISABLED1: [[META6:![0-9]+]] = !{i32 1, !"wchar_size", i32 4}
+; NVPTX-DISABLED1: [[META7:![0-9]+]] = !{i32 7, !"openmp", i32 50}
+; NVPTX-DISABLED1: [[META8:![0-9]+]] = !{i32 7, !"openmp-device", i32 50}
+; NVPTX-DISABLED1: [[META9:![0-9]+]] = !{i32 8, !"PIC Level", i32 2}
+; NVPTX-DISABLED1: [[META10:![0-9]+]] = !{i32 7, !"frame-pointer", i32 2}
+; NVPTX-DISABLED1: [[META11:![0-9]+]] = !{!"{{.*}}clang version {{.*}}"}
+; NVPTX-DISABLED1: [[TBAA12]] = !{[[META13:![0-9]+]], [[META13]], i64 0}
+; NVPTX-DISABLED1: [[META13]] = !{!"int", [[META14:![0-9]+]], i64 0}
+; NVPTX-DISABLED1: [[META14]] = !{!"omnipotent char", [[META15:![0-9]+]], i64 0}
+; NVPTX-DISABLED1: [[META15]] = !{!"Simple C/C++ TBAA"}
+; NVPTX-DISABLED1: [[LOOP16]] = distinct !{[[LOOP16]], [[META17:![0-9]+]], [[META18:![0-9]+]]}
+; NVPTX-DISABLED1: [[META17]] = !{!"llvm.loop.mustprogress"}
+; NVPTX-DISABLED1: [[META18]] = !{!"llvm.loop.unroll.disable"}
+; NVPTX-DISABLED1: [[LOOP19]] = distinct !{[[LOOP19]], [[META17]], [[META18]]}
+; NVPTX-DISABLED1: [[TBAA20]] = !{[[META21:![0-9]+]], [[META21]], i64 0}
+; NVPTX-DISABLED1: [[META21]] = !{!"any pointer", [[META14]], i64 0}
+; NVPTX-DISABLED1: [[LOOP22]] = distinct !{[[LOOP22]], [[META17]], [[META18]]}
+; NVPTX-DISABLED1: [[LOOP23]] = distinct !{[[LOOP23]], [[META17]], [[META18]]}
 ;.
 ; NVPTX-DISABLED2: [[META0:![0-9]+]] = !{i32 0, i32 64770, i32 541341486, !"do_not_spmdize_task", i32 74, i32 5}
 ; NVPTX-DISABLED2: [[META1:![0-9]+]] = !{i32 0, i32 64770, i32 541341486, !"sequential_loop_to_stack_var", i32 20, i32 1}
@@ -4643,28 +3135,22 @@ attributes #11 = { convergent }
 ; NVPTX-DISABLED2: [[META3:![0-9]+]] = !{i32 0, i32 64770, i32 541341486, !"sequential_loop_to_shared_var", i32 35, i32 2}
 ; NVPTX-DISABLED2: [[META4:![0-9]+]] = !{i32 0, i32 64770, i32 541341486, !"do_not_spmdize_target", i32 65, i32 4}
 ; NVPTX-DISABLED2: [[META5:![0-9]+]] = !{i32 0, i32 64770, i32 541341486, !"sequential_loop_to_shared_var_guarded", i32 50, i32 3}
-; NVPTX-DISABLED2: [[META6:![0-9]+]] = !{ptr @__omp_offloading_fd02_2044372e_sequential_loop_l5, !"kernel", i32 1}
-; NVPTX-DISABLED2: [[META7:![0-9]+]] = !{ptr @__omp_offloading_fd02_2044372e_sequential_loop_to_stack_var_l20, !"kernel", i32 1}
-; NVPTX-DISABLED2: [[META8:![0-9]+]] = !{ptr @__omp_offloading_fd02_2044372e_sequential_loop_to_shared_var_l35, !"kernel", i32 1}
-; NVPTX-DISABLED2: [[META9:![0-9]+]] = !{ptr @__omp_offloading_fd02_2044372e_sequential_loop_to_shared_var_guarded_l50, !"kernel", i32 1}
-; NVPTX-DISABLED2: [[META10:![0-9]+]] = !{ptr @__omp_offloading_fd02_2044372e_do_not_spmdize_target_l65, !"kernel", i32 1}
-; NVPTX-DISABLED2: [[META11:![0-9]+]] = !{ptr @__omp_offloading_fd02_2044372e_do_not_spmdize_task_l74, !"kernel", i32 1}
-; NVPTX-DISABLED2: [[META12:![0-9]+]] = !{i32 1, !"wchar_size", i32 4}
-; NVPTX-DISABLED2: [[META13:![0-9]+]] = !{i32 7, !"openmp", i32 50}
-; NVPTX-DISABLED2: [[META14:![0-9]+]] = !{i32 7, !"openmp-device", i32 50}
-; NVPTX-DISABLED2: [[META15:![0-9]+]] = !{i32 8, !"PIC Level", i32 2}
-; NVPTX-DISABLED2: [[META16:![0-9]+]] = !{i32 7, !"frame-pointer", i32 2}
-; NVPTX-DISABLED2: [[META17:![0-9]+]] = !{!"{{.*}}clang version {{.*}}"}
-; NVPTX-DISABLED2: [[TBAA18]] = !{[[META19:![0-9]+]], [[META19]], i64 0}
-; NVPTX-DISABLED2: [[META19]] = !{!"int", [[META20:![0-9]+]], i64 0}
-; NVPTX-DISABLED2: [[META20]] = !{!"omnipotent char", [[META21:![0-9]+]], i64 0}
-; NVPTX-DISABLED2: [[META21]] = !{!"Simple C/C++ TBAA"}
-; NVPTX-DISABLED2: [[LOOP22]] = distinct !{[[LOOP22]], [[META23:![0-9]+]], [[META24:![0-9]+]]}
-; NVPTX-DISABLED2: [[META23]] = !{!"llvm.loop.mustprogress"}
-; NVPTX-DISABLED2: [[META24]] = !{!"llvm.loop.unroll.disable"}
-; NVPTX-DISABLED2: [[LOOP25]] = distinct !{[[LOOP25]], [[META23]], [[META24]]}
-; NVPTX-DISABLED2: [[TBAA26]] = !{[[META27:![0-9]+]], [[META27]], i64 0}
-; NVPTX-DISABLED2: [[META27]] = !{!"any pointer", [[META20]], i64 0}
-; NVPTX-DISABLED2: [[LOOP28]] = distinct !{[[LOOP28]], [[META23]], [[META24]]}
-; NVPTX-DISABLED2: [[LOOP29]] = distinct !{[[LOOP29]], [[META23]], [[META24]]}
+; NVPTX-DISABLED2: [[META6:![0-9]+]] = !{i32 1, !"wchar_size", i32 4}
+; NVPTX-DISABLED2: [[META7:![0-9]+]] = !{i32 7, !"openmp", i32 50}
+; NVPTX-DISABLED2: [[META8:![0-9]+]] = !{i32 7, !"openmp-device", i32 50}
+; NVPTX-DISABLED2: [[META9:![0-9]+]] = !{i32 8, !"PIC Level", i32 2}
+; NVPTX-DISABLED2: [[META10:![0-9]+]] = !{i32 7, !"frame-pointer", i32 2}
+; NVPTX-DISABLED2: [[META11:![0-9]+]] = !{!"{{.*}}clang version {{.*}}"}
+; NVPTX-DISABLED2: [[TBAA12]] = !{[[META13:![0-9]+]], [[META13]], i64 0}
+; NVPTX-DISABLED2: [[META13]] = !{!"int", [[META14:![0-9]+]], i64 0}
+; NVPTX-DISABLED2: [[META14]] = !{!"omnipotent char", [[META15:![0-9]+]], i64 0}
+; NVPTX-DISABLED2: [[META15]] = !{!"Simple C/C++ TBAA"}
+; NVPTX-DISABLED2: [[LOOP16]] = distinct !{[[LOOP16]], [[META17:![0-9]+]], [[META18:![0-9]+]]}
+; NVPTX-DISABLED2: [[META17]] = !{!"llvm.loop.mustprogress"}
+; NVPTX-DISABLED2: [[META18]] = !{!"llvm.loop.unroll.disable"}
+; NVPTX-DISABLED2: [[LOOP19]] = distinct !{[[LOOP19]], [[META17]], [[META18]]}
+; NVPTX-DISABLED2: [[TBAA20]] = !{[[META21:![0-9]+]], [[META21]], i64 0}
+; NVPTX-DISABLED2: [[META21]] = !{!"any pointer", [[META14]], i64 0}
+; NVPTX-DISABLED2: [[LOOP22]] = distinct !{[[LOOP22]], [[META17]], [[META18]]}
+; NVPTX-DISABLED2: [[LOOP23]] = distinct !{[[LOOP23]], [[META17]], [[META18]]}
 ;.
diff --git a/llvm/test/Transforms/OpenMP/spmdization_assumes.ll b/llvm/test/Transforms/OpenMP/spmdization_assumes.ll
index 2f43a4e4286a2..59e2499ead2ad 100644
--- a/llvm/test/Transforms/OpenMP/spmdization_assumes.ll
+++ b/llvm/test/Transforms/OpenMP/spmdization_assumes.ll
@@ -28,7 +28,7 @@ target triple = "nvptx64"
 ; CHECK: @__omp_offloading_fd02_404433c2_main_l5_kernel_environment = local_unnamed_addr constant %struct.KernelEnvironmentTy { %struct.ConfigurationEnvironmentTy { i8 0, i8 0, i8 3, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0 }, ptr @[[GLOB1]], ptr null }
 ; CHECK: @[[GLOB2:[0-9]+]] = private unnamed_addr constant %struct.ident_t { i32 0, i32 2, i32 0, i32 22, ptr @[[GLOB0]] }, align 8
 ;.
-define weak void @__omp_offloading_fd02_404433c2_main_l5(ptr %dyn, ptr nonnull align 8 dereferenceable(8) %x) local_unnamed_addr #0 {
+define weak ptx_kernel void @__omp_offloading_fd02_404433c2_main_l5(ptr %dyn, ptr nonnull align 8 dereferenceable(8) %x) local_unnamed_addr #0 {
 ; CHECK-LABEL: define {{[^@]+}}@__omp_offloading_fd02_404433c2_main_l5
 ; CHECK-SAME: (ptr [[DYN:%.*]], ptr nonnull align 8 dereferenceable(8) [[X:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
 ; CHECK-NEXT:  entry:
@@ -47,7 +47,7 @@ define weak void @__omp_offloading_fd02_404433c2_main_l5(ptr %dyn, ptr nonnull a
 ; CHECK-NEXT:    [[TMP3:%.*]] = icmp eq i32 [[TMP2]], 0
 ; CHECK-NEXT:    br i1 [[TMP3]], label [[REGION_GUARDED:%.*]], label [[REGION_BARRIER:%.*]]
 ; CHECK:       region.guarded:
-; CHECK-NEXT:    store double [[CALL_I]], ptr [[X]], align 8, !tbaa [[TBAA8:![0-9]+]]
+; CHECK-NEXT:    store double [[CALL_I]], ptr [[X]], align 8, !tbaa [[TBAA7:![0-9]+]]
 ; CHECK-NEXT:    br label [[REGION_GUARDED_END:%.*]]
 ; CHECK:       region.guarded.end:
 ; CHECK-NEXT:    br label [[REGION_BARRIER]]
@@ -82,7 +82,7 @@ declare i32 @__kmpc_target_init(ptr, ptr) local_unnamed_addr
 ; Function Attrs: alwaysinline mustprogress nofree norecurse nosync nounwind readnone willreturn
 define internal void @__omp_outlined__(ptr noalias nocapture %.global_tid., ptr noalias nocapture %.bound_tid.) #1 {
 ; CHECK-LABEL: define {{[^@]+}}@__omp_outlined__
-; CHECK-SAME: (ptr noalias nocapture [[DOTGLOBAL_TID_:%.*]], ptr noalias nocapture [[DOTBOUND_TID_:%.*]]) #[[ATTR1:[0-9]+]] {
+; CHECK-SAME: (ptr noalias captures(none) [[DOTGLOBAL_TID_:%.*]], ptr noalias captures(none) [[DOTBOUND_TID_:%.*]]) #[[ATTR1:[0-9]+]] {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    ret void
 ;
@@ -127,12 +127,10 @@ attributes #5 = { convergent "frame-pointer"="all" "no-trapping-math"="true" "st
 attributes #6 = { convergent nounwind "llvm.assume"="ompx_spmd_amenable" }
 
 !omp_offload.info = !{!0}
-!nvvm.annotations = !{!1}
 !llvm.module.flags = !{!2, !3, !4, !5, !6}
 !llvm.ident = !{!7}
 
 !0 = !{i32 0, i32 64770, i32 1078211522, !"main", i32 5, i32 0}
-!1 = !{ptr @__omp_offloading_fd02_404433c2_main_l5, !"kernel", i32 1}
 !2 = !{i32 1, !"wchar_size", i32 4}
 !3 = !{i32 7, !"openmp", i32 50}
 !4 = !{i32 7, !"openmp-device", i32 50}
@@ -154,15 +152,14 @@ attributes #6 = { convergent nounwind "llvm.assume"="ompx_spmd_amenable" }
 ; CHECK: attributes #[[ATTR7]] = { convergent nounwind "llvm.assume"="ompx_spmd_amenable" }
 ;.
 ; CHECK: [[META0:![0-9]+]] = !{i32 0, i32 64770, i32 1078211522, !"main", i32 5, i32 0}
-; CHECK: [[META1:![0-9]+]] = !{ptr @__omp_offloading_fd02_404433c2_main_l5, !"kernel", i32 1}
-; CHECK: [[META2:![0-9]+]] = !{i32 1, !"wchar_size", i32 4}
-; CHECK: [[META3:![0-9]+]] = !{i32 7, !"openmp", i32 50}
-; CHECK: [[META4:![0-9]+]] = !{i32 7, !"openmp-device", i32 50}
-; CHECK: [[META5:![0-9]+]] = !{i32 8, !"PIC Level", i32 2}
-; CHECK: [[META6:![0-9]+]] = !{i32 7, !"frame-pointer", i32 2}
-; CHECK: [[META7:![0-9]+]] = !{!"{{.*}}clang version {{.*}}"}
-; CHECK: [[TBAA8]] = !{[[META9:![0-9]+]], [[META9]], i64 0}
-; CHECK: [[META9]] = !{!"double", [[META10:![0-9]+]], i64 0}
-; CHECK: [[META10]] = !{!"omnipotent char", [[META11:![0-9]+]], i64 0}
-; CHECK: [[META11]] = !{!"Simple C/C++ TBAA"}
+; CHECK: [[META1:![0-9]+]] = !{i32 1, !"wchar_size", i32 4}
+; CHECK: [[META2:![0-9]+]] = !{i32 7, !"openmp", i32 50}
+; CHECK: [[META3:![0-9]+]] = !{i32 7, !"openmp-device", i32 50}
+; CHECK: [[META4:![0-9]+]] = !{i32 8, !"PIC Level", i32 2}
+; CHECK: [[META5:![0-9]+]] = !{i32 7, !"frame-pointer", i32 2}
+; CHECK: [[META6:![0-9]+]] = !{!"{{.*}}clang version {{.*}}"}
+; CHECK: [[TBAA7]] = !{[[META8:![0-9]+]], [[META8]], i64 0}
+; CHECK: [[META8]] = !{!"double", [[META9:![0-9]+]], i64 0}
+; CHECK: [[META9]] = !{!"omnipotent char", [[META10:![0-9]+]], i64 0}
+; CHECK: [[META10]] = !{!"Simple C/C++ TBAA"}
 ;.
diff --git a/llvm/test/Transforms/OpenMP/spmdization_constant_prop.ll b/llvm/test/Transforms/OpenMP/spmdization_constant_prop.ll
index 75e01f3295fe2..953ecb2ddd8a6 100644
--- a/llvm/test/Transforms/OpenMP/spmdization_constant_prop.ll
+++ b/llvm/test/Transforms/OpenMP/spmdization_constant_prop.ll
@@ -297,12 +297,10 @@ attributes #14 = { convergent nounwind "llvm.assume"="ompx_aligned_barrier,ompx_
 attributes #15 = { convergent nounwind }
 
 !omp_offload.info = !{!0}
-!nvvm.annotations = !{!1}
 !llvm.module.flags = !{!2, !3, !4, !5}
 !llvm.ident = !{!6}
 
 !0 = !{i32 0, i32 32, i32 18757968, !"main", i32 12, i32 0}
-!1 = !{ptr @__omp_offloading_20_11e3950_main_l12, !"kernel", i32 1}
 !2 = !{i32 1, !"wchar_size", i32 4}
 !3 = !{i32 7, !"openmp", i32 50}
 !4 = !{i32 7, !"openmp-device", i32 50}
diff --git a/llvm/test/Transforms/OpenMP/spmdization_guarding.ll b/llvm/test/Transforms/OpenMP/spmdization_guarding.ll
index 229a49d784559..2f1aadc073142 100644
--- a/llvm/test/Transforms/OpenMP/spmdization_guarding.ll
+++ b/llvm/test/Transforms/OpenMP/spmdization_guarding.ll
@@ -58,7 +58,7 @@ target triple = "nvptx64"
 ; CHECK-DISABLED: @__omp_offloading_2a_fbfa7a_sequential_loop_l6_kernel_environment = local_unnamed_addr constant %struct.KernelEnvironmentTy { %struct.ConfigurationEnvironmentTy { i8 0, i8 0, i8 1, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0 }, ptr @[[GLOB1]], ptr null }
 ; CHECK-DISABLED: @__omp_outlined__1_wrapper.ID = private constant i8 undef
 ;.
-define weak void @__omp_offloading_2a_fbfa7a_sequential_loop_l6(ptr %dyn, ptr %x, i64 %N) #0 {
+define weak ptx_kernel void @__omp_offloading_2a_fbfa7a_sequential_loop_l6(ptr %dyn, ptr %x, i64 %N) #0 {
 ; CHECK-LABEL: define {{[^@]+}}@__omp_offloading_2a_fbfa7a_sequential_loop_l6
 ; CHECK-SAME: (ptr [[DYN:%.*]], ptr [[X:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
 ; CHECK-NEXT:  entry:
@@ -84,9 +84,9 @@ define weak void @__omp_offloading_2a_fbfa7a_sequential_loop_l6(ptr %dyn, ptr %x
 ; CHECK-NEXT:    [[TMP3:%.*]] = icmp eq i32 [[TMP2]], 0
 ; CHECK-NEXT:    br i1 [[TMP3]], label [[REGION_GUARDED:%.*]], label [[REGION_BARRIER:%.*]]
 ; CHECK:       region.guarded:
-; CHECK-NEXT:    store i32 0, ptr [[X]], align 4, !noalias [[META8:![0-9]+]]
-; CHECK-NEXT:    store i32 1, ptr [[ARRAYIDX1_I]], align 4, !noalias [[META8]]
-; CHECK-NEXT:    store i32 [[N_ADDR_SROA_0_0_EXTRACT_TRUNC]], ptr [[ARRAYIDX2_I]], align 4, !noalias [[META8]]
+; CHECK-NEXT:    store i32 0, ptr [[X]], align 4, !noalias [[META7:![0-9]+]]
+; CHECK-NEXT:    store i32 1, ptr [[ARRAYIDX1_I]], align 4, !noalias [[META7]]
+; CHECK-NEXT:    store i32 [[N_ADDR_SROA_0_0_EXTRACT_TRUNC]], ptr [[ARRAYIDX2_I]], align 4, !noalias [[META7]]
 ; CHECK-NEXT:    br label [[REGION_GUARDED_END:%.*]]
 ; CHECK:       region.guarded.end:
 ; CHECK-NEXT:    br label [[REGION_BARRIER]]
@@ -94,7 +94,7 @@ define weak void @__omp_offloading_2a_fbfa7a_sequential_loop_l6(ptr %dyn, ptr %x
 ; CHECK-NEXT:    call void @__kmpc_barrier_simple_spmd(ptr @[[GLOB2]], i32 [[TMP2]])
 ; CHECK-NEXT:    br label [[REGION_EXIT:%.*]]
 ; CHECK:       region.exit:
-; CHECK-NEXT:    call void @usei8ptr(ptr nocapture [[HEAP2STACK_H2S]]) #[[ATTR9:[0-9]+]]
+; CHECK-NEXT:    call void @usei8ptr(ptr captures(none) [[HEAP2STACK_H2S]]) #[[ATTR9:[0-9]+]]
 ; CHECK-NEXT:    br label [[FOR_COND_I:%.*]]
 ; CHECK:       for.cond.i:
 ; CHECK-NEXT:    [[I_0_I:%.*]] = phi i32 [ 2, [[REGION_EXIT]] ], [ [[INC_I:%.*]], [[REGION_EXIT3:%.*]] ]
@@ -111,7 +111,7 @@ define weak void @__omp_offloading_2a_fbfa7a_sequential_loop_l6(ptr %dyn, ptr %x
 ; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq i32 [[TMP4]], 0
 ; CHECK-NEXT:    br i1 [[TMP5]], label [[REGION_GUARDED4:%.*]], label [[REGION_BARRIER2:%.*]]
 ; CHECK:       region.guarded4:
-; CHECK-NEXT:    store i32 [[SUB3_I]], ptr [[ARRAYIDX5_I]], align 4, !noalias [[META8]]
+; CHECK-NEXT:    store i32 [[SUB3_I]], ptr [[ARRAYIDX5_I]], align 4, !noalias [[META7]]
 ; CHECK-NEXT:    br label [[REGION_GUARDED_END1:%.*]]
 ; CHECK:       region.guarded.end1:
 ; CHECK-NEXT:    br label [[REGION_BARRIER2]]
@@ -120,10 +120,10 @@ define weak void @__omp_offloading_2a_fbfa7a_sequential_loop_l6(ptr %dyn, ptr %x
 ; CHECK-NEXT:    br label [[REGION_EXIT3]]
 ; CHECK:       region.exit3:
 ; CHECK-NEXT:    [[INC_I]] = add nuw nsw i32 [[I_0_I]], 1
-; CHECK-NEXT:    br label [[FOR_COND_I]], !llvm.loop [[LOOP11:![0-9]+]]
+; CHECK-NEXT:    br label [[FOR_COND_I]], !llvm.loop [[LOOP10:![0-9]+]]
 ; CHECK:       __omp_outlined__.exit:
 ; CHECK-NEXT:    call void @__kmpc_parallel_51(ptr null, i32 0, i32 1, i32 -1, i32 -1, ptr @__omp_outlined__1, ptr @__omp_outlined__1_wrapper, ptr null, i64 0)
-; CHECK-NEXT:    [[CALL_I:%.*]] = call i32 @no_openmp(ptr nonnull [[X]]) #[[ATTR10:[0-9]+]], !noalias [[META8]]
+; CHECK-NEXT:    [[CALL_I:%.*]] = call i32 @no_openmp(ptr nonnull [[X]]) #[[ATTR10:[0-9]+]], !noalias [[META7]]
 ; CHECK-NEXT:    [[IDXPROM6_I:%.*]] = sext i32 [[CALL_I]] to i64
 ; CHECK-NEXT:    [[ARRAYIDX7_I:%.*]] = getelementptr inbounds i32, ptr [[X]], i64 [[IDXPROM6_I]]
 ; CHECK-NEXT:    br label [[REGION_CHECK_TID10:%.*]]
@@ -132,7 +132,7 @@ define weak void @__omp_offloading_2a_fbfa7a_sequential_loop_l6(ptr %dyn, ptr %x
 ; CHECK-NEXT:    [[TMP7:%.*]] = icmp eq i32 [[TMP6]], 0
 ; CHECK-NEXT:    br i1 [[TMP7]], label [[REGION_GUARDED9:%.*]], label [[REGION_BARRIER7:%.*]]
 ; CHECK:       region.guarded9:
-; CHECK-NEXT:    store i32 [[CALL_I]], ptr [[ARRAYIDX7_I]], align 4, !noalias [[META8]]
+; CHECK-NEXT:    store i32 [[CALL_I]], ptr [[ARRAYIDX7_I]], align 4, !noalias [[META7]]
 ; CHECK-NEXT:    br label [[REGION_GUARDED_END6:%.*]]
 ; CHECK:       region.guarded.end6:
 ; CHECK-NEXT:    br label [[REGION_BARRIER7]]
@@ -140,7 +140,7 @@ define weak void @__omp_offloading_2a_fbfa7a_sequential_loop_l6(ptr %dyn, ptr %x
 ; CHECK-NEXT:    call void @__kmpc_barrier_simple_spmd(ptr @[[GLOB2]], i32 [[TMP6]])
 ; CHECK-NEXT:    br label [[REGION_EXIT8:%.*]]
 ; CHECK:       region.exit8:
-; CHECK-NEXT:    [[CALL8_I:%.*]] = call i32 @no_openmp(ptr nonnull [[X]]) #[[ATTR10]], !noalias [[META8]]
+; CHECK-NEXT:    [[CALL8_I:%.*]] = call i32 @no_openmp(ptr nonnull [[X]]) #[[ATTR10]], !noalias [[META7]]
 ; CHECK-NEXT:    [[IDXPROM9_I:%.*]] = sext i32 [[CALL8_I]] to i64
 ; CHECK-NEXT:    [[ARRAYIDX10_I:%.*]] = getelementptr inbounds i32, ptr [[X]], i64 [[IDXPROM9_I]]
 ; CHECK-NEXT:    br label [[REGION_CHECK_TID15:%.*]]
@@ -149,7 +149,7 @@ define weak void @__omp_offloading_2a_fbfa7a_sequential_loop_l6(ptr %dyn, ptr %x
 ; CHECK-NEXT:    [[TMP9:%.*]] = icmp eq i32 [[TMP8]], 0
 ; CHECK-NEXT:    br i1 [[TMP9]], label [[REGION_GUARDED14:%.*]], label [[REGION_BARRIER12:%.*]]
 ; CHECK:       region.guarded14:
-; CHECK-NEXT:    store i32 [[CALL8_I]], ptr [[ARRAYIDX10_I]], align 4, !noalias [[META8]]
+; CHECK-NEXT:    store i32 [[CALL8_I]], ptr [[ARRAYIDX10_I]], align 4, !noalias [[META7]]
 ; CHECK-NEXT:    br label [[REGION_GUARDED_END11:%.*]]
 ; CHECK:       region.guarded.end11:
 ; CHECK-NEXT:    br label [[REGION_BARRIER12]]
@@ -157,7 +157,7 @@ define weak void @__omp_offloading_2a_fbfa7a_sequential_loop_l6(ptr %dyn, ptr %x
 ; CHECK-NEXT:    call void @__kmpc_barrier_simple_spmd(ptr @[[GLOB2]], i32 [[TMP8]])
 ; CHECK-NEXT:    br label [[REGION_EXIT13:%.*]]
 ; CHECK:       region.exit13:
-; CHECK-NEXT:    [[CALL11_I:%.*]] = call i32 @no_openmp(ptr nonnull [[X]]) #[[ATTR10]], !noalias [[META8]]
+; CHECK-NEXT:    [[CALL11_I:%.*]] = call i32 @no_openmp(ptr nonnull [[X]]) #[[ATTR10]], !noalias [[META7]]
 ; CHECK-NEXT:    [[IDXPROM12_I:%.*]] = sext i32 [[CALL11_I]] to i64
 ; CHECK-NEXT:    [[ARRAYIDX13_I:%.*]] = getelementptr inbounds i32, ptr [[X]], i64 [[IDXPROM12_I]]
 ; CHECK-NEXT:    br label [[REGION_CHECK_TID20:%.*]]
@@ -166,7 +166,7 @@ define weak void @__omp_offloading_2a_fbfa7a_sequential_loop_l6(ptr %dyn, ptr %x
 ; CHECK-NEXT:    [[TMP11:%.*]] = icmp eq i32 [[TMP10]], 0
 ; CHECK-NEXT:    br i1 [[TMP11]], label [[REGION_GUARDED19:%.*]], label [[REGION_BARRIER17:%.*]]
 ; CHECK:       region.guarded19:
-; CHECK-NEXT:    store i32 [[CALL11_I]], ptr [[ARRAYIDX13_I]], align 4, !noalias [[META8]]
+; CHECK-NEXT:    store i32 [[CALL11_I]], ptr [[ARRAYIDX13_I]], align 4, !noalias [[META7]]
 ; CHECK-NEXT:    br label [[REGION_GUARDED_END16:%.*]]
 ; CHECK:       region.guarded.end16:
 ; CHECK-NEXT:    br label [[REGION_BARRIER17]]
@@ -174,9 +174,9 @@ define weak void @__omp_offloading_2a_fbfa7a_sequential_loop_l6(ptr %dyn, ptr %x
 ; CHECK-NEXT:    call void @__kmpc_barrier_simple_spmd(ptr @[[GLOB2]], i32 [[TMP10]])
 ; CHECK-NEXT:    br label [[REGION_EXIT18:%.*]]
 ; CHECK:       region.exit18:
-; CHECK-NEXT:    [[CALL14_I:%.*]] = call i32 @no_openmp(ptr nonnull [[X]]) #[[ATTR10]], !noalias [[META8]]
-; CHECK-NEXT:    [[CALL15_I:%.*]] = call i32 @no_openmp(ptr nonnull [[X]]) #[[ATTR10]], !noalias [[META8]]
-; CHECK-NEXT:    [[CALL16_I:%.*]] = call i32 @no_openmp(ptr nonnull [[X]]) #[[ATTR10]], !noalias [[META8]]
+; CHECK-NEXT:    [[CALL14_I:%.*]] = call i32 @no_openmp(ptr nonnull [[X]]) #[[ATTR10]], !noalias [[META7]]
+; CHECK-NEXT:    [[CALL15_I:%.*]] = call i32 @no_openmp(ptr nonnull [[X]]) #[[ATTR10]], !noalias [[META7]]
+; CHECK-NEXT:    [[CALL16_I:%.*]] = call i32 @no_openmp(ptr nonnull [[X]]) #[[ATTR10]], !noalias [[META7]]
 ; CHECK-NEXT:    call void @__kmpc_target_deinit() #[[ATTR6]]
 ; CHECK-NEXT:    ret void
 ; CHECK:       worker.exit:
@@ -230,14 +230,14 @@ define weak void @__omp_offloading_2a_fbfa7a_sequential_loop_l6(ptr %dyn, ptr %x
 ; CHECK-DISABLED-NEXT:    [[SELECT:%.*]] = select i1 [[C]], ptr [[AL32]], ptr addrspacecast (ptr addrspace(5) @LocGlob to ptr)
 ; CHECK-DISABLED-NEXT:    store ptr [[SELECT]], ptr [[LOC]], align 8
 ; CHECK-DISABLED-NEXT:    [[TMP1:%.*]] = call i32 @__kmpc_global_thread_num(ptr nonnull @[[GLOB1]]) #[[ATTR6]]
-; CHECK-DISABLED-NEXT:    store i32 0, ptr [[X]], align 4, !noalias [[META8:![0-9]+]]
+; CHECK-DISABLED-NEXT:    store i32 0, ptr [[X]], align 4, !noalias [[META7:![0-9]+]]
 ; CHECK-DISABLED-NEXT:    [[ARRAYIDX1_I:%.*]] = getelementptr inbounds i32, ptr [[X]], i64 1
-; CHECK-DISABLED-NEXT:    store i32 1, ptr [[ARRAYIDX1_I]], align 4, !noalias [[META8]]
+; CHECK-DISABLED-NEXT:    store i32 1, ptr [[ARRAYIDX1_I]], align 4, !noalias [[META7]]
 ; CHECK-DISABLED-NEXT:    [[SEXT:%.*]] = shl i64 [[N]], 32
 ; CHECK-DISABLED-NEXT:    [[IDXPROM_I:%.*]] = ashr exact i64 [[SEXT]], 32
 ; CHECK-DISABLED-NEXT:    [[ARRAYIDX2_I:%.*]] = getelementptr inbounds i32, ptr [[X]], i64 [[IDXPROM_I]]
-; CHECK-DISABLED-NEXT:    store i32 [[N_ADDR_SROA_0_0_EXTRACT_TRUNC]], ptr [[ARRAYIDX2_I]], align 4, !noalias [[META8]]
-; CHECK-DISABLED-NEXT:    call void @usei8ptr(ptr nocapture [[HEAP2STACK_H2S]]) #[[ATTR9:[0-9]+]]
+; CHECK-DISABLED-NEXT:    store i32 [[N_ADDR_SROA_0_0_EXTRACT_TRUNC]], ptr [[ARRAYIDX2_I]], align 4, !noalias [[META7]]
+; CHECK-DISABLED-NEXT:    call void @usei8ptr(ptr captures(none) [[HEAP2STACK_H2S]]) #[[ATTR9:[0-9]+]]
 ; CHECK-DISABLED-NEXT:    br label [[FOR_COND_I:%.*]]
 ; CHECK-DISABLED:       for.cond.i:
 ; CHECK-DISABLED-NEXT:    [[I_0_I:%.*]] = phi i32 [ 2, [[USER_CODE_ENTRY]] ], [ [[INC_I:%.*]], [[FOR_BODY_I:%.*]] ]
@@ -248,26 +248,26 @@ define weak void @__omp_offloading_2a_fbfa7a_sequential_loop_l6(ptr %dyn, ptr %x
 ; CHECK-DISABLED-NEXT:    [[SUB3_I:%.*]] = add nsw i32 [[I_0_I]], -1
 ; CHECK-DISABLED-NEXT:    [[IDXPROM4_I:%.*]] = zext i32 [[I_0_I]] to i64
 ; CHECK-DISABLED-NEXT:    [[ARRAYIDX5_I:%.*]] = getelementptr inbounds i32, ptr [[X]], i64 [[IDXPROM4_I]]
-; CHECK-DISABLED-NEXT:    store i32 [[SUB3_I]], ptr [[ARRAYIDX5_I]], align 4, !noalias [[META8]]
+; CHECK-DISABLED-NEXT:    store i32 [[SUB3_I]], ptr [[ARRAYIDX5_I]], align 4, !noalias [[META7]]
 ; CHECK-DISABLED-NEXT:    [[INC_I]] = add nuw nsw i32 [[I_0_I]], 1
-; CHECK-DISABLED-NEXT:    br label [[FOR_COND_I]], !llvm.loop [[LOOP11:![0-9]+]]
+; CHECK-DISABLED-NEXT:    br label [[FOR_COND_I]], !llvm.loop [[LOOP10:![0-9]+]]
 ; CHECK-DISABLED:       __omp_outlined__.exit:
 ; CHECK-DISABLED-NEXT:    call void @__kmpc_parallel_51(ptr null, i32 0, i32 1, i32 -1, i32 -1, ptr @__omp_outlined__1, ptr @__omp_outlined__1_wrapper.ID, ptr null, i64 0)
-; CHECK-DISABLED-NEXT:    [[CALL_I:%.*]] = call i32 @no_openmp(ptr nonnull [[X]]) #[[ATTR10:[0-9]+]], !noalias [[META8]]
+; CHECK-DISABLED-NEXT:    [[CALL_I:%.*]] = call i32 @no_openmp(ptr nonnull [[X]]) #[[ATTR10:[0-9]+]], !noalias [[META7]]
 ; CHECK-DISABLED-NEXT:    [[IDXPROM6_I:%.*]] = sext i32 [[CALL_I]] to i64
 ; CHECK-DISABLED-NEXT:    [[ARRAYIDX7_I:%.*]] = getelementptr inbounds i32, ptr [[X]], i64 [[IDXPROM6_I]]
-; CHECK-DISABLED-NEXT:    store i32 [[CALL_I]], ptr [[ARRAYIDX7_I]], align 4, !noalias [[META8]]
-; CHECK-DISABLED-NEXT:    [[CALL8_I:%.*]] = call i32 @no_openmp(ptr nonnull [[X]]) #[[ATTR10]], !noalias [[META8]]
+; CHECK-DISABLED-NEXT:    store i32 [[CALL_I]], ptr [[ARRAYIDX7_I]], align 4, !noalias [[META7]]
+; CHECK-DISABLED-NEXT:    [[CALL8_I:%.*]] = call i32 @no_openmp(ptr nonnull [[X]]) #[[ATTR10]], !noalias [[META7]]
 ; CHECK-DISABLED-NEXT:    [[IDXPROM9_I:%.*]] = sext i32 [[CALL8_I]] to i64
 ; CHECK-DISABLED-NEXT:    [[ARRAYIDX10_I:%.*]] = getelementptr inbounds i32, ptr [[X]], i64 [[IDXPROM9_I]]
-; CHECK-DISABLED-NEXT:    store i32 [[CALL8_I]], ptr [[ARRAYIDX10_I]], align 4, !noalias [[META8]]
-; CHECK-DISABLED-NEXT:    [[CALL11_I:%.*]] = call i32 @no_openmp(ptr nonnull [[X]]) #[[ATTR10]], !noalias [[META8]]
+; CHECK-DISABLED-NEXT:    store i32 [[CALL8_I]], ptr [[ARRAYIDX10_I]], align 4, !noalias [[META7]]
+; CHECK-DISABLED-NEXT:    [[CALL11_I:%.*]] = call i32 @no_openmp(ptr nonnull [[X]]) #[[ATTR10]], !noalias [[META7]]
 ; CHECK-DISABLED-NEXT:    [[IDXPROM12_I:%.*]] = sext i32 [[CALL11_I]] to i64
 ; CHECK-DISABLED-NEXT:    [[ARRAYIDX13_I:%.*]] = getelementptr inbounds i32, ptr [[X]], i64 [[IDXPROM12_I]]
-; CHECK-DISABLED-NEXT:    store i32 [[CALL11_I]], ptr [[ARRAYIDX13_I]], align 4, !noalias [[META8]]
-; CHECK-DISABLED-NEXT:    [[CALL14_I:%.*]] = call i32 @no_openmp(ptr nonnull [[X]]) #[[ATTR10]], !noalias [[META8]]
-; CHECK-DISABLED-NEXT:    [[CALL15_I:%.*]] = call i32 @no_openmp(ptr nonnull [[X]]) #[[ATTR10]], !noalias [[META8]]
-; CHECK-DISABLED-NEXT:    [[CALL16_I:%.*]] = call i32 @no_openmp(ptr nonnull [[X]]) #[[ATTR10]], !noalias [[META8]]
+; CHECK-DISABLED-NEXT:    store i32 [[CALL11_I]], ptr [[ARRAYIDX13_I]], align 4, !noalias [[META7]]
+; CHECK-DISABLED-NEXT:    [[CALL14_I:%.*]] = call i32 @no_openmp(ptr nonnull [[X]]) #[[ATTR10]], !noalias [[META7]]
+; CHECK-DISABLED-NEXT:    [[CALL15_I:%.*]] = call i32 @no_openmp(ptr nonnull [[X]]) #[[ATTR10]], !noalias [[META7]]
+; CHECK-DISABLED-NEXT:    [[CALL16_I:%.*]] = call i32 @no_openmp(ptr nonnull [[X]]) #[[ATTR10]], !noalias [[META7]]
 ; CHECK-DISABLED-NEXT:    call void @__kmpc_target_deinit() #[[ATTR6]]
 ; CHECK-DISABLED-NEXT:    ret void
 ; CHECK-DISABLED:       worker.exit:
@@ -404,12 +404,10 @@ attributes #4 = { inaccessiblememonly nofree nosync nounwind willreturn }
 attributes #5 = { convergent nounwind "llvm.assume"="omp_no_openmp,ompx_spmd_amenable" }
 
 !omp_offload.info = !{!0}
-!nvvm.annotations = !{!1}
 !llvm.module.flags = !{!2, !3, !4, !5, !6}
 !llvm.ident = !{!7}
 
 !0 = !{i32 0, i32 42, i32 16513658, !"sequential_loop", i32 6, i32 0}
-!1 = !{ptr @__omp_offloading_2a_fbfa7a_sequential_loop_l6, !"kernel", i32 1}
 !2 = !{i32 1, !"wchar_size", i32 4}
 !3 = !{i32 7, !"openmp", i32 50}
 !4 = !{i32 7, !"openmp-device", i32 50}
@@ -447,30 +445,28 @@ attributes #5 = { convergent nounwind "llvm.assume"="omp_no_openmp,ompx_spmd_ame
 ; CHECK-DISABLED: attributes #[[ATTR10]] = { convergent nounwind "llvm.assume"="omp_no_openmp,ompx_spmd_amenable" }
 ;.
 ; CHECK: [[META0:![0-9]+]] = !{i32 0, i32 42, i32 16513658, !"sequential_loop", i32 6, i32 0}
-; CHECK: [[META1:![0-9]+]] = !{ptr @__omp_offloading_2a_fbfa7a_sequential_loop_l6, !"kernel", i32 1}
-; CHECK: [[META2:![0-9]+]] = !{i32 1, !"wchar_size", i32 4}
-; CHECK: [[META3:![0-9]+]] = !{i32 7, !"openmp", i32 50}
-; CHECK: [[META4:![0-9]+]] = !{i32 7, !"openmp-device", i32 50}
-; CHECK: [[META5:![0-9]+]] = !{i32 8, !"PIC Level", i32 2}
-; CHECK: [[META6:![0-9]+]] = !{i32 7, !"frame-pointer", i32 2}
-; CHECK: [[META7:![0-9]+]] = !{!"{{.*}}clang version {{.*}}"}
-; CHECK: [[META8]] = !{[[META9:![0-9]+]]}
-; CHECK: [[META9]] = distinct !{[[META9]], [[META10:![0-9]+]], !"__omp_outlined__: %__context"}
-; CHECK: [[META10]] = distinct !{[[META10]], !"__omp_outlined__"}
-; CHECK: [[LOOP11]] = distinct !{[[LOOP11]], [[META12:![0-9]+]]}
-; CHECK: [[META12]] = !{!"llvm.loop.mustprogress"}
+; CHECK: [[META1:![0-9]+]] = !{i32 1, !"wchar_size", i32 4}
+; CHECK: [[META2:![0-9]+]] = !{i32 7, !"openmp", i32 50}
+; CHECK: [[META3:![0-9]+]] = !{i32 7, !"openmp-device", i32 50}
+; CHECK: [[META4:![0-9]+]] = !{i32 8, !"PIC Level", i32 2}
+; CHECK: [[META5:![0-9]+]] = !{i32 7, !"frame-pointer", i32 2}
+; CHECK: [[META6:![0-9]+]] = !{!"{{.*}}clang version {{.*}}"}
+; CHECK: [[META7]] = !{[[META8:![0-9]+]]}
+; CHECK: [[META8]] = distinct !{[[META8]], [[META9:![0-9]+]], !"__omp_outlined__: %__context"}
+; CHECK: [[META9]] = distinct !{[[META9]], !"__omp_outlined__"}
+; CHECK: [[LOOP10]] = distinct !{[[LOOP10]], [[META11:![0-9]+]]}
+; CHECK: [[META11]] = !{!"llvm.loop.mustprogress"}
 ;.
 ; CHECK-DISABLED: [[META0:![0-9]+]] = !{i32 0, i32 42, i32 16513658, !"sequential_loop", i32 6, i32 0}
-; CHECK-DISABLED: [[META1:![0-9]+]] = !{ptr @__omp_offloading_2a_fbfa7a_sequential_loop_l6, !"kernel", i32 1}
-; CHECK-DISABLED: [[META2:![0-9]+]] = !{i32 1, !"wchar_size", i32 4}
-; CHECK-DISABLED: [[META3:![0-9]+]] = !{i32 7, !"openmp", i32 50}
-; CHECK-DISABLED: [[META4:![0-9]+]] = !{i32 7, !"openmp-device", i32 50}
-; CHECK-DISABLED: [[META5:![0-9]+]] = !{i32 8, !"PIC Level", i32 2}
-; CHECK-DISABLED: [[META6:![0-9]+]] = !{i32 7, !"frame-pointer", i32 2}
-; CHECK-DISABLED: [[META7:![0-9]+]] = !{!"{{.*}}clang version {{.*}}"}
-; CHECK-DISABLED: [[META8]] = !{[[META9:![0-9]+]]}
-; CHECK-DISABLED: [[META9]] = distinct !{[[META9]], [[META10:![0-9]+]], !"__omp_outlined__: %__context"}
-; CHECK-DISABLED: [[META10]] = distinct !{[[META10]], !"__omp_outlined__"}
-; CHECK-DISABLED: [[LOOP11]] = distinct !{[[LOOP11]], [[META12:![0-9]+]]}
-; CHECK-DISABLED: [[META12]] = !{!"llvm.loop.mustprogress"}
+; CHECK-DISABLED: [[META1:![0-9]+]] = !{i32 1, !"wchar_size", i32 4}
+; CHECK-DISABLED: [[META2:![0-9]+]] = !{i32 7, !"openmp", i32 50}
+; CHECK-DISABLED: [[META3:![0-9]+]] = !{i32 7, !"openmp-device", i32 50}
+; CHECK-DISABLED: [[META4:![0-9]+]] = !{i32 8, !"PIC Level", i32 2}
+; CHECK-DISABLED: [[META5:![0-9]+]] = !{i32 7, !"frame-pointer", i32 2}
+; CHECK-DISABLED: [[META6:![0-9]+]] = !{!"{{.*}}clang version {{.*}}"}
+; CHECK-DISABLED: [[META7]] = !{[[META8:![0-9]+]]}
+; CHECK-DISABLED: [[META8]] = distinct !{[[META8]], [[META9:![0-9]+]], !"__omp_outlined__: %__context"}
+; CHECK-DISABLED: [[META9]] = distinct !{[[META9]], !"__omp_outlined__"}
+; CHECK-DISABLED: [[LOOP10]] = distinct !{[[LOOP10]], [[META11:![0-9]+]]}
+; CHECK-DISABLED: [[META11]] = !{!"llvm.loop.mustprogress"}
 ;.
diff --git a/llvm/test/Transforms/OpenMP/spmdization_guarding_two_reaching_kernels.ll b/llvm/test/Transforms/OpenMP/spmdization_guarding_two_reaching_kernels.ll
index 11405b7eb447c..a644fe1b2f821 100644
--- a/llvm/test/Transforms/OpenMP/spmdization_guarding_two_reaching_kernels.ll
+++ b/llvm/test/Transforms/OpenMP/spmdization_guarding_two_reaching_kernels.ll
@@ -56,7 +56,7 @@ target triple = "nvptx64"
 ; CHECK-DISABLE-SPMDIZATION: @__omp_offloading_2b_10393b5_spmd_l12_kernel_environment = local_unnamed_addr constant %struct.KernelEnvironmentTy { %struct.ConfigurationEnvironmentTy { i8 0, i8 1, i8 1, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0 }, ptr @[[GLOB1]], ptr null }
 ; CHECK-DISABLE-SPMDIZATION: @__omp_offloading_2b_10393b5_generic_l20_kernel_environment = local_unnamed_addr constant %struct.KernelEnvironmentTy { %struct.ConfigurationEnvironmentTy { i8 0, i8 0, i8 1, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0 }, ptr @[[GLOB1]], ptr null }
 ;.
-define weak void @__omp_offloading_2b_10393b5_spmd_l12(ptr %dyn) "kernel" #0 {
+define weak ptx_kernel void @__omp_offloading_2b_10393b5_spmd_l12(ptr %dyn) "kernel" #0 {
 ; CHECK-LABEL: define {{[^@]+}}@__omp_offloading_2b_10393b5_spmd_l12
 ; CHECK-SAME: (ptr [[DYN:%.*]]) #[[ATTR0:[0-9]+]] {
 ; CHECK-NEXT:  entry:
@@ -113,7 +113,7 @@ define weak i32 @__kmpc_target_init(ptr, ptr) {
 declare void @__kmpc_target_deinit()
 
 ; Function Attrs: convergent noinline norecurse nounwind
-define weak void @__omp_offloading_2b_10393b5_generic_l20(ptr %dyn) #0 {
+define weak ptx_kernel void @__omp_offloading_2b_10393b5_generic_l20(ptr %dyn) #0 {
 ; CHECK-LABEL: define {{[^@]+}}@__omp_offloading_2b_10393b5_generic_l20
 ; CHECK-SAME: (ptr [[DYN:%.*]]) #[[ATTR0]] {
 ; CHECK-NEXT:  entry:
@@ -321,14 +321,12 @@ attributes #4 = { alwaysinline }
 attributes #5 = { convergent }
 
 !omp_offload.info = !{!0, !1}
-!nvvm.annotations = !{!2, !3}
 !llvm.module.flags = !{!4, !5, !6, !7, !8}
 !llvm.ident = !{!9}
 
 !0 = !{i32 0, i32 43, i32 17011637, !"spmd", i32 12, i32 0}
 !1 = !{i32 0, i32 43, i32 17011637, !"generic", i32 20, i32 1}
-!2 = !{ptr @__omp_offloading_2b_10393b5_spmd_l12, !"kernel", i32 1}
-!3 = !{ptr @__omp_offloading_2b_10393b5_generic_l20, !"kernel", i32 1}
+
 !4 = !{i32 1, !"wchar_size", i32 4}
 !5 = !{i32 7, !"openmp", i32 50}
 !6 = !{i32 7, !"openmp-device", i32 50}
@@ -358,23 +356,19 @@ attributes #5 = { convergent }
 ;.
 ; CHECK: [[META0:![0-9]+]] = !{i32 0, i32 43, i32 17011637, !"spmd", i32 12, i32 0}
 ; CHECK: [[META1:![0-9]+]] = !{i32 0, i32 43, i32 17011637, !"generic", i32 20, i32 1}
-; CHECK: [[META2:![0-9]+]] = !{ptr @__omp_offloading_2b_10393b5_spmd_l12, !"kernel", i32 1}
-; CHECK: [[META3:![0-9]+]] = !{ptr @__omp_offloading_2b_10393b5_generic_l20, !"kernel", i32 1}
-; CHECK: [[META4:![0-9]+]] = !{i32 1, !"wchar_size", i32 4}
-; CHECK: [[META5:![0-9]+]] = !{i32 7, !"openmp", i32 50}
-; CHECK: [[META6:![0-9]+]] = !{i32 7, !"openmp-device", i32 50}
-; CHECK: [[META7:![0-9]+]] = !{i32 8, !"PIC Level", i32 2}
-; CHECK: [[META8:![0-9]+]] = !{i32 7, !"frame-pointer", i32 2}
-; CHECK: [[META9:![0-9]+]] = !{!"{{.*}}clang version {{.*}}"}
+; CHECK: [[META2:![0-9]+]] = !{i32 1, !"wchar_size", i32 4}
+; CHECK: [[META3:![0-9]+]] = !{i32 7, !"openmp", i32 50}
+; CHECK: [[META4:![0-9]+]] = !{i32 7, !"openmp-device", i32 50}
+; CHECK: [[META5:![0-9]+]] = !{i32 8, !"PIC Level", i32 2}
+; CHECK: [[META6:![0-9]+]] = !{i32 7, !"frame-pointer", i32 2}
+; CHECK: [[META7:![0-9]+]] = !{!"{{.*}}clang version {{.*}}"}
 ;.
 ; CHECK-DISABLE-SPMDIZATION: [[META0:![0-9]+]] = !{i32 0, i32 43, i32 17011637, !"spmd", i32 12, i32 0}
 ; CHECK-DISABLE-SPMDIZATION: [[META1:![0-9]+]] = !{i32 0, i32 43, i32 17011637, !"generic", i32 20, i32 1}
-; CHECK-DISABLE-SPMDIZATION: [[META2:![0-9]+]] = !{ptr @__omp_offloading_2b_10393b5_spmd_l12, !"kernel", i32 1}
-; CHECK-DISABLE-SPMDIZATION: [[META3:![0-9]+]] = !{ptr @__omp_offloading_2b_10393b5_generic_l20, !"kernel", i32 1}
-; CHECK-DISABLE-SPMDIZATION: [[META4:![0-9]+]] = !{i32 1, !"wchar_size", i32 4}
-; CHECK-DISABLE-SPMDIZATION: [[META5:![0-9]+]] = !{i32 7, !"openmp", i32 50}
-; CHECK-DISABLE-SPMDIZATION: [[META6:![0-9]+]] = !{i32 7, !"openmp-device", i32 50}
-; CHECK-DISABLE-SPMDIZATION: [[META7:![0-9]+]] = !{i32 8, !"PIC Level", i32 2}
-; CHECK-DISABLE-SPMDIZATION: [[META8:![0-9]+]] = !{i32 7, !"frame-pointer", i32 2}
-; CHECK-DISABLE-SPMDIZATION: [[META9:![0-9]+]] = !{!"{{.*}}clang version {{.*}}"}
+; CHECK-DISABLE-SPMDIZATION: [[META2:![0-9]+]] = !{i32 1, !"wchar_size", i32 4}
+; CHECK-DISABLE-SPMDIZATION: [[META3:![0-9]+]] = !{i32 7, !"openmp", i32 50}
+; CHECK-DISABLE-SPMDIZATION: [[META4:![0-9]+]] = !{i32 7, !"openmp-device", i32 50}
+; CHECK-DISABLE-SPMDIZATION: [[META5:![0-9]+]] = !{i32 8, !"PIC Level", i32 2}
+; CHECK-DISABLE-SPMDIZATION: [[META6:![0-9]+]] = !{i32 7, !"frame-pointer", i32 2}
+; CHECK-DISABLE-SPMDIZATION: [[META7:![0-9]+]] = !{!"{{.*}}clang version {{.*}}"}
 ;.
diff --git a/llvm/test/Transforms/OpenMP/spmdization_indirect.ll b/llvm/test/Transforms/OpenMP/spmdization_indirect.ll
index f348825446c63..ef8caf48e57b7 100644
--- a/llvm/test/Transforms/OpenMP/spmdization_indirect.ll
+++ b/llvm/test/Transforms/OpenMP/spmdization_indirect.ll
@@ -28,7 +28,7 @@
 ; NVPTX: @spmd_and_non_spmd_callees_metadata_kernel_environment = local_unnamed_addr constant %struct.KernelEnvironmentTy { %struct.ConfigurationEnvironmentTy { i8 0, i8 0, i8 1, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0 }, ptr @[[GLOB1]], ptr null }
 ; NVPTX: @spmd_and_non_spmd_callee_kernel_environment = local_unnamed_addr constant %struct.KernelEnvironmentTy { %struct.ConfigurationEnvironmentTy { i8 0, i8 0, i8 1, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0 }, ptr @[[GLOB1]], ptr null }
 ;.
-define weak void @spmd_callees(i1 %c) #0 {
+define weak ptx_kernel void @spmd_callees(i1 %c) #0 {
 ; AMDGPU-LABEL: define {{[^@]+}}@spmd_callees
 ; AMDGPU-SAME: (i1 [[C:%.*]]) #[[ATTR0:[0-9]+]] {
 ; AMDGPU-NEXT:    call void @spmd_callees__debug(i1 [[C]])
@@ -57,7 +57,7 @@ define internal void @spmd_callees__debug(i1 %c) {
 ; AMDGPU:       user_code.entry:
 ; AMDGPU-NEXT:    [[TMP1:%.*]] = call i32 @__kmpc_global_thread_num(ptr @[[GLOB1]]) #[[ATTR10:[0-9]+]]
 ; AMDGPU-NEXT:    store i32 0, ptr [[DOTZERO_ADDR]], align 4
-; AMDGPU-NEXT:    store i32 [[TMP1]], ptr [[DOTTHREADID_TEMP_]], align 4, !tbaa [[TBAA17:![0-9]+]]
+; AMDGPU-NEXT:    store i32 [[TMP1]], ptr [[DOTTHREADID_TEMP_]], align 4, !tbaa [[TBAA12:![0-9]+]]
 ; AMDGPU-NEXT:    [[FP:%.*]] = select i1 [[C]], ptr @__omp_outlined_spmd_amenable1, ptr @__omp_outlined_spmd_amenable2
 ; AMDGPU-NEXT:    [[TMP2:%.*]] = icmp eq ptr [[FP]], @__omp_outlined_spmd_amenable2
 ; AMDGPU-NEXT:    br i1 [[TMP2]], label [[TMP3:%.*]], label [[TMP4:%.*]]
@@ -88,7 +88,7 @@ define internal void @spmd_callees__debug(i1 %c) {
 ; NVPTX:       user_code.entry:
 ; NVPTX-NEXT:    [[TMP1:%.*]] = call i32 @__kmpc_global_thread_num(ptr @[[GLOB1]]) #[[ATTR10:[0-9]+]]
 ; NVPTX-NEXT:    store i32 0, ptr [[DOTZERO_ADDR]], align 4
-; NVPTX-NEXT:    store i32 [[TMP1]], ptr [[DOTTHREADID_TEMP_]], align 4, !tbaa [[TBAA17:![0-9]+]]
+; NVPTX-NEXT:    store i32 [[TMP1]], ptr [[DOTTHREADID_TEMP_]], align 4, !tbaa [[TBAA12:![0-9]+]]
 ; NVPTX-NEXT:    [[FP:%.*]] = select i1 [[C]], ptr @__omp_outlined_spmd_amenable1, ptr @__omp_outlined_spmd_amenable2
 ; NVPTX-NEXT:    [[TMP2:%.*]] = icmp eq ptr [[FP]], @__omp_outlined_spmd_amenable2
 ; NVPTX-NEXT:    br i1 [[TMP2]], label [[TMP3:%.*]], label [[TMP4:%.*]]
@@ -143,10 +143,10 @@ define internal void @__omp_outlined_spmd_amenable1(ptr noalias %.global_tid., p
 ; AMDGPU-NEXT:    call void @spmd_amenable() #[[ATTR6:[0-9]+]]
 ; AMDGPU-NEXT:    ret void
 ; AMDGPU:       for.body:
-; AMDGPU-NEXT:    [[TMP0:%.*]] = load i32, ptr [[DOTGLOBAL_TID_]], align 4, !tbaa [[TBAA17]]
+; AMDGPU-NEXT:    [[TMP0:%.*]] = load i32, ptr [[DOTGLOBAL_TID_]], align 4, !tbaa [[TBAA12]]
 ; AMDGPU-NEXT:    call void @__kmpc_parallel_51(ptr @[[GLOB1]], i32 [[TMP0]], i32 1, i32 -1, i32 -1, ptr @__omp_outlined__1, ptr @__omp_outlined__1_wrapper, ptr [[CAPTURED_VARS_ADDRS]], i64 0)
 ; AMDGPU-NEXT:    [[INC]] = add nsw i32 [[I_0]], 1
-; AMDGPU-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP21:![0-9]+]]
+; AMDGPU-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP16:![0-9]+]]
 ;
 ; NVPTX-LABEL: define {{[^@]+}}@__omp_outlined_spmd_amenable1
 ; NVPTX-SAME: (ptr noalias [[DOTGLOBAL_TID_:%.*]], ptr noalias [[DOTBOUND_TID_:%.*]]) {
@@ -161,10 +161,10 @@ define internal void @__omp_outlined_spmd_amenable1(ptr noalias %.global_tid., p
 ; NVPTX-NEXT:    call void @spmd_amenable() #[[ATTR6:[0-9]+]]
 ; NVPTX-NEXT:    ret void
 ; NVPTX:       for.body:
-; NVPTX-NEXT:    [[TMP0:%.*]] = load i32, ptr [[DOTGLOBAL_TID_]], align 4, !tbaa [[TBAA17]]
+; NVPTX-NEXT:    [[TMP0:%.*]] = load i32, ptr [[DOTGLOBAL_TID_]], align 4, !tbaa [[TBAA12]]
 ; NVPTX-NEXT:    call void @__kmpc_parallel_51(ptr @[[GLOB1]], i32 [[TMP0]], i32 1, i32 -1, i32 -1, ptr @__omp_outlined__1, ptr @__omp_outlined__1_wrapper, ptr [[CAPTURED_VARS_ADDRS]], i64 0)
 ; NVPTX-NEXT:    [[INC]] = add nsw i32 [[I_0]], 1
-; NVPTX-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP21:![0-9]+]]
+; NVPTX-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP16:![0-9]+]]
 ;
 entry:
   %captured_vars_addrs = alloca [0 x ptr], align 8
@@ -252,7 +252,7 @@ define internal void @__omp_outlined_spmd_amenable2(ptr noalias %.global_tid., p
 ; AMDGPU-NEXT:    [[X_H2S:%.*]] = alloca i8, i64 4, align 4, addrspace(5)
 ; AMDGPU-NEXT:    [[CAPTURED_VARS_ADDRS:%.*]] = alloca [0 x ptr], align 8
 ; AMDGPU-NEXT:    [[MALLOC_CAST:%.*]] = addrspacecast ptr addrspace(5) [[X_H2S]] to ptr
-; AMDGPU-NEXT:    call void @use(ptr nocapture [[MALLOC_CAST]]) #[[ATTR6]]
+; AMDGPU-NEXT:    call void @use(ptr captures(none) [[MALLOC_CAST]]) #[[ATTR6]]
 ; AMDGPU-NEXT:    br label [[FOR_COND:%.*]]
 ; AMDGPU:       for.cond:
 ; AMDGPU-NEXT:    [[I_0:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[INC:%.*]], [[FOR_BODY:%.*]] ]
@@ -262,17 +262,17 @@ define internal void @__omp_outlined_spmd_amenable2(ptr noalias %.global_tid., p
 ; AMDGPU-NEXT:    call void @spmd_amenable() #[[ATTR6]]
 ; AMDGPU-NEXT:    ret void
 ; AMDGPU:       for.body:
-; AMDGPU-NEXT:    [[TMP0:%.*]] = load i32, ptr [[DOTGLOBAL_TID_]], align 4, !tbaa [[TBAA17]]
+; AMDGPU-NEXT:    [[TMP0:%.*]] = load i32, ptr [[DOTGLOBAL_TID_]], align 4, !tbaa [[TBAA12]]
 ; AMDGPU-NEXT:    call void @__kmpc_parallel_51(ptr @[[GLOB1]], i32 [[TMP0]], i32 1, i32 -1, i32 -1, ptr @__omp_outlined__3, ptr @__omp_outlined__3_wrapper, ptr [[CAPTURED_VARS_ADDRS]], i64 0)
 ; AMDGPU-NEXT:    [[INC]] = add nsw i32 [[I_0]], 1
-; AMDGPU-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP24:![0-9]+]]
+; AMDGPU-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP19:![0-9]+]]
 ;
 ; NVPTX-LABEL: define {{[^@]+}}@__omp_outlined_spmd_amenable2
 ; NVPTX-SAME: (ptr noalias [[DOTGLOBAL_TID_:%.*]], ptr noalias [[DOTBOUND_TID_:%.*]]) {
 ; NVPTX-NEXT:  entry:
 ; NVPTX-NEXT:    [[X_H2S:%.*]] = alloca i8, i64 4, align 4
 ; NVPTX-NEXT:    [[CAPTURED_VARS_ADDRS:%.*]] = alloca [0 x ptr], align 8
-; NVPTX-NEXT:    call void @use(ptr nocapture [[X_H2S]]) #[[ATTR6]]
+; NVPTX-NEXT:    call void @use(ptr captures(none) [[X_H2S]]) #[[ATTR6]]
 ; NVPTX-NEXT:    br label [[FOR_COND:%.*]]
 ; NVPTX:       for.cond:
 ; NVPTX-NEXT:    [[I_0:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[INC:%.*]], [[FOR_BODY:%.*]] ]
@@ -282,10 +282,10 @@ define internal void @__omp_outlined_spmd_amenable2(ptr noalias %.global_tid., p
 ; NVPTX-NEXT:    call void @spmd_amenable() #[[ATTR6]]
 ; NVPTX-NEXT:    ret void
 ; NVPTX:       for.body:
-; NVPTX-NEXT:    [[TMP0:%.*]] = load i32, ptr [[DOTGLOBAL_TID_]], align 4, !tbaa [[TBAA17]]
+; NVPTX-NEXT:    [[TMP0:%.*]] = load i32, ptr [[DOTGLOBAL_TID_]], align 4, !tbaa [[TBAA12]]
 ; NVPTX-NEXT:    call void @__kmpc_parallel_51(ptr @[[GLOB1]], i32 [[TMP0]], i32 1, i32 -1, i32 -1, ptr @__omp_outlined__3, ptr @__omp_outlined__3_wrapper, ptr [[CAPTURED_VARS_ADDRS]], i64 0)
 ; NVPTX-NEXT:    [[INC]] = add nsw i32 [[I_0]], 1
-; NVPTX-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP24:![0-9]+]]
+; NVPTX-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP19:![0-9]+]]
 ;
 entry:
   %captured_vars_addrs = alloca [0 x ptr], align 8
@@ -367,7 +367,7 @@ entry:
 
 
 ; Function Attrs: alwaysinline convergent norecurse nounwind
-define weak void @spmd_and_non_spmd_callee(i1 %c) #0 {
+define weak ptx_kernel void @spmd_and_non_spmd_callee(i1 %c) #0 {
 ;
 ;
 ; AMDGPU-LABEL: define {{[^@]+}}@spmd_and_non_spmd_callee
@@ -413,7 +413,7 @@ define weak void @spmd_and_non_spmd_callee(i1 %c) #0 {
 ; AMDGPU:       user_code.entry:
 ; AMDGPU-NEXT:    [[TMP1:%.*]] = call i32 @__kmpc_global_thread_num(ptr @[[GLOB1]]) #[[ATTR10]]
 ; AMDGPU-NEXT:    store i32 0, ptr [[DOTZERO_ADDR]], align 4
-; AMDGPU-NEXT:    store i32 [[TMP1]], ptr [[DOTTHREADID_TEMP_]], align 4, !tbaa [[TBAA17]]
+; AMDGPU-NEXT:    store i32 [[TMP1]], ptr [[DOTTHREADID_TEMP_]], align 4, !tbaa [[TBAA12]]
 ; AMDGPU-NEXT:    [[FP:%.*]] = select i1 [[C]], ptr @__omp_outlined_spmd_amenable3, ptr @__omp_outlined_not_spmd_amenable
 ; AMDGPU-NEXT:    [[TMP2:%.*]] = icmp eq ptr [[FP]], @__omp_outlined_not_spmd_amenable
 ; AMDGPU-NEXT:    br i1 [[TMP2]], label [[TMP3:%.*]], label [[TMP4:%.*]]
@@ -473,7 +473,7 @@ define weak void @spmd_and_non_spmd_callee(i1 %c) #0 {
 ; NVPTX:       user_code.entry:
 ; NVPTX-NEXT:    [[TMP1:%.*]] = call i32 @__kmpc_global_thread_num(ptr @[[GLOB1]]) #[[ATTR10]]
 ; NVPTX-NEXT:    store i32 0, ptr [[DOTZERO_ADDR]], align 4
-; NVPTX-NEXT:    store i32 [[TMP1]], ptr [[DOTTHREADID_TEMP_]], align 4, !tbaa [[TBAA17]]
+; NVPTX-NEXT:    store i32 [[TMP1]], ptr [[DOTTHREADID_TEMP_]], align 4, !tbaa [[TBAA12]]
 ; NVPTX-NEXT:    [[FP:%.*]] = select i1 [[C]], ptr @__omp_outlined_spmd_amenable3, ptr @__omp_outlined_not_spmd_amenable
 ; NVPTX-NEXT:    [[TMP2:%.*]] = icmp eq ptr [[FP]], @__omp_outlined_not_spmd_amenable
 ; NVPTX-NEXT:    br i1 [[TMP2]], label [[TMP3:%.*]], label [[TMP4:%.*]]
@@ -530,11 +530,11 @@ define internal void @__omp_outlined_spmd_amenable3(ptr noalias %.global_tid., p
 ; AMDGPU-NEXT:    call void @__kmpc_free_shared(ptr [[X]], i64 4) #[[ATTR10]]
 ; AMDGPU-NEXT:    ret void
 ; AMDGPU:       for.body:
-; AMDGPU-NEXT:    store ptr [[X]], ptr [[CAPTURED_VARS_ADDRS]], align 8, !tbaa [[TBAA25:![0-9]+]]
-; AMDGPU-NEXT:    [[TMP0:%.*]] = load i32, ptr [[DOTGLOBAL_TID_]], align 4, !tbaa [[TBAA17]]
+; AMDGPU-NEXT:    store ptr [[X]], ptr [[CAPTURED_VARS_ADDRS]], align 8, !tbaa [[TBAA20:![0-9]+]]
+; AMDGPU-NEXT:    [[TMP0:%.*]] = load i32, ptr [[DOTGLOBAL_TID_]], align 4, !tbaa [[TBAA12]]
 ; AMDGPU-NEXT:    call void @__kmpc_parallel_51(ptr @[[GLOB1]], i32 [[TMP0]], i32 1, i32 -1, i32 -1, ptr @__omp_outlined__5, ptr @__omp_outlined__5_wrapper, ptr [[CAPTURED_VARS_ADDRS]], i64 1)
 ; AMDGPU-NEXT:    [[INC]] = add nsw i32 [[I_0]], 1
-; AMDGPU-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP27:![0-9]+]]
+; AMDGPU-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP22:![0-9]+]]
 ;
 ; NVPTX-LABEL: define {{[^@]+}}@__omp_outlined_spmd_amenable3
 ; NVPTX-SAME: (ptr noalias [[DOTGLOBAL_TID_:%.*]], ptr noalias [[DOTBOUND_TID_:%.*]]) {
@@ -551,11 +551,11 @@ define internal void @__omp_outlined_spmd_amenable3(ptr noalias %.global_tid., p
 ; NVPTX-NEXT:    call void @__kmpc_free_shared(ptr [[X]], i64 4) #[[ATTR10]]
 ; NVPTX-NEXT:    ret void
 ; NVPTX:       for.body:
-; NVPTX-NEXT:    store ptr [[X]], ptr [[CAPTURED_VARS_ADDRS]], align 8, !tbaa [[TBAA25:![0-9]+]]
-; NVPTX-NEXT:    [[TMP0:%.*]] = load i32, ptr [[DOTGLOBAL_TID_]], align 4, !tbaa [[TBAA17]]
+; NVPTX-NEXT:    store ptr [[X]], ptr [[CAPTURED_VARS_ADDRS]], align 8, !tbaa [[TBAA20:![0-9]+]]
+; NVPTX-NEXT:    [[TMP0:%.*]] = load i32, ptr [[DOTGLOBAL_TID_]], align 4, !tbaa [[TBAA12]]
 ; NVPTX-NEXT:    call void @__kmpc_parallel_51(ptr @[[GLOB1]], i32 [[TMP0]], i32 1, i32 -1, i32 -1, ptr @__omp_outlined__5, ptr @__omp_outlined__5_wrapper, ptr [[CAPTURED_VARS_ADDRS]], i64 1)
 ; NVPTX-NEXT:    [[INC]] = add nsw i32 [[I_0]], 1
-; NVPTX-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP27:![0-9]+]]
+; NVPTX-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP22:![0-9]+]]
 ;
 entry:
   %captured_vars_addrs = alloca [1 x ptr], align 8
@@ -587,18 +587,18 @@ define internal void @__omp_outlined__5(ptr noalias %.global_tid., ptr noalias %
 ; AMDGPU-LABEL: define {{[^@]+}}@__omp_outlined__5
 ; AMDGPU-SAME: (ptr noalias [[DOTGLOBAL_TID_:%.*]], ptr noalias [[DOTBOUND_TID_:%.*]], ptr nonnull align 4 dereferenceable(4) [[X:%.*]]) {
 ; AMDGPU-NEXT:  entry:
-; AMDGPU-NEXT:    [[TMP0:%.*]] = load i32, ptr [[X]], align 4, !tbaa [[TBAA17]]
+; AMDGPU-NEXT:    [[TMP0:%.*]] = load i32, ptr [[X]], align 4, !tbaa [[TBAA12]]
 ; AMDGPU-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP0]], 1
-; AMDGPU-NEXT:    store i32 [[INC]], ptr [[X]], align 4, !tbaa [[TBAA17]]
+; AMDGPU-NEXT:    store i32 [[INC]], ptr [[X]], align 4, !tbaa [[TBAA12]]
 ; AMDGPU-NEXT:    call void @unknown() #[[ATTR7]]
 ; AMDGPU-NEXT:    ret void
 ;
 ; NVPTX-LABEL: define {{[^@]+}}@__omp_outlined__5
 ; NVPTX-SAME: (ptr noalias [[DOTGLOBAL_TID_:%.*]], ptr noalias [[DOTBOUND_TID_:%.*]], ptr nonnull align 4 dereferenceable(4) [[X:%.*]]) {
 ; NVPTX-NEXT:  entry:
-; NVPTX-NEXT:    [[TMP0:%.*]] = load i32, ptr [[X]], align 4, !tbaa [[TBAA17]]
+; NVPTX-NEXT:    [[TMP0:%.*]] = load i32, ptr [[X]], align 4, !tbaa [[TBAA12]]
 ; NVPTX-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP0]], 1
-; NVPTX-NEXT:    store i32 [[INC]], ptr [[X]], align 4, !tbaa [[TBAA17]]
+; NVPTX-NEXT:    store i32 [[INC]], ptr [[X]], align 4, !tbaa [[TBAA12]]
 ; NVPTX-NEXT:    call void @unknown() #[[ATTR7]]
 ; NVPTX-NEXT:    ret void
 ;
@@ -622,7 +622,7 @@ define internal void @__omp_outlined__5_wrapper(i16 zeroext %0, i32 %1) #3 {
 ; AMDGPU-NEXT:    [[GLOBAL_ARGS:%.*]] = alloca ptr, align 8
 ; AMDGPU-NEXT:    call void @__kmpc_get_shared_variables(ptr [[GLOBAL_ARGS]])
 ; AMDGPU-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[GLOBAL_ARGS]], align 8
-; AMDGPU-NEXT:    [[TMP3:%.*]] = load ptr, ptr [[TMP2]], align 8, !tbaa [[TBAA25]]
+; AMDGPU-NEXT:    [[TMP3:%.*]] = load ptr, ptr [[TMP2]], align 8, !tbaa [[TBAA20]]
 ; AMDGPU-NEXT:    call void @__omp_outlined__5(ptr [[DOTADDR1]], ptr [[DOTZERO_ADDR]], ptr [[TMP3]]) #[[ATTR10]]
 ; AMDGPU-NEXT:    ret void
 ;
@@ -634,7 +634,7 @@ define internal void @__omp_outlined__5_wrapper(i16 zeroext %0, i32 %1) #3 {
 ; NVPTX-NEXT:    [[GLOBAL_ARGS:%.*]] = alloca ptr, align 8
 ; NVPTX-NEXT:    call void @__kmpc_get_shared_variables(ptr [[GLOBAL_ARGS]])
 ; NVPTX-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[GLOBAL_ARGS]], align 8
-; NVPTX-NEXT:    [[TMP3:%.*]] = load ptr, ptr [[TMP2]], align 8, !tbaa [[TBAA25]]
+; NVPTX-NEXT:    [[TMP3:%.*]] = load ptr, ptr [[TMP2]], align 8, !tbaa [[TBAA20]]
 ; NVPTX-NEXT:    call void @__omp_outlined__5(ptr [[DOTADDR1]], ptr [[DOTZERO_ADDR]], ptr [[TMP3]]) #[[ATTR10]]
 ; NVPTX-NEXT:    ret void
 ;
@@ -652,7 +652,7 @@ entry:
 }
 
 ; Function Attrs: alwaysinline convergent norecurse nounwind
-define weak void @spmd_callees_metadata(ptr %fp) #0 {
+define weak ptx_kernel void @spmd_callees_metadata(ptr %fp) #0 {
 ;
 ;
 ; AMDGPU-LABEL: define {{[^@]+}}@spmd_callees_metadata
@@ -668,7 +668,7 @@ define weak void @spmd_callees_metadata(ptr %fp) #0 {
 ; AMDGPU:       user_code.entry:
 ; AMDGPU-NEXT:    [[TMP1:%.*]] = call i32 @__kmpc_global_thread_num(ptr @[[GLOB1]]) #[[ATTR10]]
 ; AMDGPU-NEXT:    store i32 0, ptr [[DOTZERO_ADDR]], align 4
-; AMDGPU-NEXT:    store i32 [[TMP1]], ptr [[DOTTHREADID_TEMP_]], align 4, !tbaa [[TBAA17]]
+; AMDGPU-NEXT:    store i32 [[TMP1]], ptr [[DOTTHREADID_TEMP_]], align 4, !tbaa [[TBAA12]]
 ; AMDGPU-NEXT:    call void @__omp_outlined_spmd_amenable_external(ptr [[DOTTHREADID_TEMP_]], ptr [[DOTZERO_ADDR]])
 ; AMDGPU-NEXT:    call void @__kmpc_target_deinit()
 ; AMDGPU-NEXT:    br label [[COMMON_RET]]
@@ -686,7 +686,7 @@ define weak void @spmd_callees_metadata(ptr %fp) #0 {
 ; NVPTX:       user_code.entry:
 ; NVPTX-NEXT:    [[TMP1:%.*]] = call i32 @__kmpc_global_thread_num(ptr @[[GLOB1]]) #[[ATTR10]]
 ; NVPTX-NEXT:    store i32 0, ptr [[DOTZERO_ADDR]], align 4
-; NVPTX-NEXT:    store i32 [[TMP1]], ptr [[DOTTHREADID_TEMP_]], align 4, !tbaa [[TBAA17]]
+; NVPTX-NEXT:    store i32 [[TMP1]], ptr [[DOTTHREADID_TEMP_]], align 4, !tbaa [[TBAA12]]
 ; NVPTX-NEXT:    call void @__omp_outlined_spmd_amenable_external(ptr [[DOTTHREADID_TEMP_]], ptr [[DOTZERO_ADDR]])
 ; NVPTX-NEXT:    call void @__kmpc_target_deinit()
 ; NVPTX-NEXT:    br label [[COMMON_RET]]
@@ -711,7 +711,7 @@ user_code.entry:                                  ; preds = %entry
 }
 
 ; Function Attrs: alwaysinline convergent norecurse nounwind
-define weak void @spmd_and_non_spmd_callees_metadata(ptr %fp) #0 {
+define weak ptx_kernel void @spmd_and_non_spmd_callees_metadata(ptr %fp) #0 {
 ;
 ;
 ; AMDGPU-LABEL: define {{[^@]+}}@spmd_and_non_spmd_callees_metadata
@@ -757,7 +757,7 @@ define weak void @spmd_and_non_spmd_callees_metadata(ptr %fp) #0 {
 ; AMDGPU:       user_code.entry:
 ; AMDGPU-NEXT:    [[TMP1:%.*]] = call i32 @__kmpc_global_thread_num(ptr @[[GLOB1]]) #[[ATTR10]]
 ; AMDGPU-NEXT:    store i32 0, ptr [[DOTZERO_ADDR]], align 4
-; AMDGPU-NEXT:    store i32 [[TMP1]], ptr [[DOTTHREADID_TEMP_]], align 4, !tbaa [[TBAA17]]
+; AMDGPU-NEXT:    store i32 [[TMP1]], ptr [[DOTTHREADID_TEMP_]], align 4, !tbaa [[TBAA12]]
 ; AMDGPU-NEXT:    [[TMP2:%.*]] = icmp eq ptr [[FP]], @__omp_outlined_spmd_amenable_external
 ; AMDGPU-NEXT:    br i1 [[TMP2]], label [[TMP3:%.*]], label [[TMP4:%.*]]
 ; AMDGPU:       3:
@@ -816,7 +816,7 @@ define weak void @spmd_and_non_spmd_callees_metadata(ptr %fp) #0 {
 ; NVPTX:       user_code.entry:
 ; NVPTX-NEXT:    [[TMP1:%.*]] = call i32 @__kmpc_global_thread_num(ptr @[[GLOB1]]) #[[ATTR10]]
 ; NVPTX-NEXT:    store i32 0, ptr [[DOTZERO_ADDR]], align 4
-; NVPTX-NEXT:    store i32 [[TMP1]], ptr [[DOTTHREADID_TEMP_]], align 4, !tbaa [[TBAA17]]
+; NVPTX-NEXT:    store i32 [[TMP1]], ptr [[DOTTHREADID_TEMP_]], align 4, !tbaa [[TBAA12]]
 ; NVPTX-NEXT:    [[TMP2:%.*]] = icmp eq ptr [[FP]], @__omp_outlined_spmd_amenable_external
 ; NVPTX-NEXT:    br i1 [[TMP2]], label [[TMP3:%.*]], label [[TMP4:%.*]]
 ; NVPTX:       3:
@@ -868,10 +868,10 @@ define void @__omp_outlined_spmd_amenable_external(ptr noalias %.global_tid., pt
 ; AMDGPU-NEXT:    call void @spmd_amenable() #[[ATTR6]]
 ; AMDGPU-NEXT:    ret void
 ; AMDGPU:       for.body:
-; AMDGPU-NEXT:    [[TMP0:%.*]] = load i32, ptr [[DOTGLOBAL_TID_]], align 4, !tbaa [[TBAA17]]
+; AMDGPU-NEXT:    [[TMP0:%.*]] = load i32, ptr [[DOTGLOBAL_TID_]], align 4, !tbaa [[TBAA12]]
 ; AMDGPU-NEXT:    call void @__kmpc_parallel_51(ptr @[[GLOB1]], i32 [[TMP0]], i32 1, i32 -1, i32 -1, ptr @__omp_outlined__7, ptr @__omp_outlined__7_wrapper, ptr undef, i64 0)
 ; AMDGPU-NEXT:    [[INC]] = add nsw i32 [[I_0]], 1
-; AMDGPU-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP28:![0-9]+]]
+; AMDGPU-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP23:![0-9]+]]
 ;
 ; NVPTX-LABEL: define {{[^@]+}}@__omp_outlined_spmd_amenable_external
 ; NVPTX-SAME: (ptr noalias [[DOTGLOBAL_TID_:%.*]], ptr noalias [[DOTBOUND_TID_:%.*]]) {
@@ -885,10 +885,10 @@ define void @__omp_outlined_spmd_amenable_external(ptr noalias %.global_tid., pt
 ; NVPTX-NEXT:    call void @spmd_amenable() #[[ATTR6]]
 ; NVPTX-NEXT:    ret void
 ; NVPTX:       for.body:
-; NVPTX-NEXT:    [[TMP0:%.*]] = load i32, ptr [[DOTGLOBAL_TID_]], align 4, !tbaa [[TBAA17]]
+; NVPTX-NEXT:    [[TMP0:%.*]] = load i32, ptr [[DOTGLOBAL_TID_]], align 4, !tbaa [[TBAA12]]
 ; NVPTX-NEXT:    call void @__kmpc_parallel_51(ptr @[[GLOB1]], i32 [[TMP0]], i32 1, i32 -1, i32 -1, ptr @__omp_outlined__7, ptr @__omp_outlined__7_wrapper, ptr undef, i64 0)
 ; NVPTX-NEXT:    [[INC]] = add nsw i32 [[I_0]], 1
-; NVPTX-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP28:![0-9]+]]
+; NVPTX-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP23:![0-9]+]]
 ;
 entry:
   br label %for.cond
@@ -1069,7 +1069,6 @@ attributes #10 = { convergent "llvm.assume"="ompx_spmd_amenable" }
 attributes #11 = { convergent }
 
 !omp_offload.info = !{!0, !1, !2, !3, !4, !5}
-!nvvm.annotations = !{!6, !7, !8, !9, !10, !11}
 !llvm.module.flags = !{!12, !13, !14, !15, !16}
 !llvm.ident = !{!17}
 
@@ -1079,12 +1078,6 @@ attributes #11 = { convergent }
 !3 = !{i32 0, i32 64770, i32 541341486, !"sequential_loop_to_shared_var", i32 35, i32 2}
 !4 = !{i32 0, i32 64770, i32 541341486, !"do_not_spmdize_target", i32 65, i32 4}
 !5 = !{i32 0, i32 64770, i32 541341486, !"sequential_loop_to_shared_var_guarded", i32 50, i32 3}
-!6 = !{ptr @spmd_callees, !"kernel", i32 1}
-!7 = !{ptr @spmd_and_non_spmd_callees_metadata, !"kernel", i32 1}
-!8 = !{ptr @spmd_and_non_spmd_callee, !"kernel", i32 1}
-!9 = !{ptr @spmd_callees_metadata, !"kernel", i32 1}
-!10 = !{i32 1}
-!11 = !{i32 1}
 !12 = !{i32 1, !"wchar_size", i32 4}
 !13 = !{i32 7, !"openmp", i32 50}
 !14 = !{i32 7, !"openmp-device", i32 50}
@@ -1139,29 +1132,24 @@ attributes #11 = { convergent }
 ; AMDGPU: [[META3:![0-9]+]] = !{i32 0, i32 64770, i32 541341486, !"sequential_loop_to_shared_var", i32 35, i32 2}
 ; AMDGPU: [[META4:![0-9]+]] = !{i32 0, i32 64770, i32 541341486, !"do_not_spmdize_target", i32 65, i32 4}
 ; AMDGPU: [[META5:![0-9]+]] = !{i32 0, i32 64770, i32 541341486, !"sequential_loop_to_shared_var_guarded", i32 50, i32 3}
-; AMDGPU: [[META6:![0-9]+]] = !{ptr @spmd_callees, !"kernel", i32 1}
-; AMDGPU: [[META7:![0-9]+]] = !{ptr @spmd_and_non_spmd_callees_metadata, !"kernel", i32 1}
-; AMDGPU: [[META8:![0-9]+]] = !{ptr @spmd_and_non_spmd_callee, !"kernel", i32 1}
-; AMDGPU: [[META9:![0-9]+]] = !{ptr @spmd_callees_metadata, !"kernel", i32 1}
-; AMDGPU: [[META10:![0-9]+]] = !{i32 1}
-; AMDGPU: [[META11:![0-9]+]] = !{i32 1, !"wchar_size", i32 4}
-; AMDGPU: [[META12:![0-9]+]] = !{i32 7, !"openmp", i32 50}
-; AMDGPU: [[META13:![0-9]+]] = !{i32 7, !"openmp-device", i32 50}
-; AMDGPU: [[META14:![0-9]+]] = !{i32 8, !"PIC Level", i32 2}
-; AMDGPU: [[META15:![0-9]+]] = !{i32 7, !"frame-pointer", i32 2}
-; AMDGPU: [[META16:![0-9]+]] = !{!"{{.*}}clang version {{.*}}"}
-; AMDGPU: [[TBAA17]] = !{[[META18:![0-9]+]], [[META18]], i64 0}
-; AMDGPU: [[META18]] = !{!"int", [[META19:![0-9]+]], i64 0}
-; AMDGPU: [[META19]] = !{!"omnipotent char", [[META20:![0-9]+]], i64 0}
-; AMDGPU: [[META20]] = !{!"Simple C/C++ TBAA"}
-; AMDGPU: [[LOOP21]] = distinct !{[[LOOP21]], [[META22:![0-9]+]], [[META23:![0-9]+]]}
-; AMDGPU: [[META22]] = !{!"llvm.loop.mustprogress"}
-; AMDGPU: [[META23]] = !{!"llvm.loop.unroll.disable"}
-; AMDGPU: [[LOOP24]] = distinct !{[[LOOP24]], [[META22]], [[META23]]}
-; AMDGPU: [[TBAA25]] = !{[[META26:![0-9]+]], [[META26]], i64 0}
-; AMDGPU: [[META26]] = !{!"any pointer", [[META19]], i64 0}
-; AMDGPU: [[LOOP27]] = distinct !{[[LOOP27]], [[META22]], [[META23]]}
-; AMDGPU: [[LOOP28]] = distinct !{[[LOOP28]], [[META22]], [[META23]]}
+; AMDGPU: [[META6:![0-9]+]] = !{i32 1, !"wchar_size", i32 4}
+; AMDGPU: [[META7:![0-9]+]] = !{i32 7, !"openmp", i32 50}
+; AMDGPU: [[META8:![0-9]+]] = !{i32 7, !"openmp-device", i32 50}
+; AMDGPU: [[META9:![0-9]+]] = !{i32 8, !"PIC Level", i32 2}
+; AMDGPU: [[META10:![0-9]+]] = !{i32 7, !"frame-pointer", i32 2}
+; AMDGPU: [[META11:![0-9]+]] = !{!"{{.*}}clang version {{.*}}"}
+; AMDGPU: [[TBAA12]] = !{[[META13:![0-9]+]], [[META13]], i64 0}
+; AMDGPU: [[META13]] = !{!"int", [[META14:![0-9]+]], i64 0}
+; AMDGPU: [[META14]] = !{!"omnipotent char", [[META15:![0-9]+]], i64 0}
+; AMDGPU: [[META15]] = !{!"Simple C/C++ TBAA"}
+; AMDGPU: [[LOOP16]] = distinct !{[[LOOP16]], [[META17:![0-9]+]], [[META18:![0-9]+]]}
+; AMDGPU: [[META17]] = !{!"llvm.loop.mustprogress"}
+; AMDGPU: [[META18]] = !{!"llvm.loop.unroll.disable"}
+; AMDGPU: [[LOOP19]] = distinct !{[[LOOP19]], [[META17]], [[META18]]}
+; AMDGPU: [[TBAA20]] = !{[[META21:![0-9]+]], [[META21]], i64 0}
+; AMDGPU: [[META21]] = !{!"any pointer", [[META14]], i64 0}
+; AMDGPU: [[LOOP22]] = distinct !{[[LOOP22]], [[META17]], [[META18]]}
+; AMDGPU: [[LOOP23]] = distinct !{[[LOOP23]], [[META17]], [[META18]]}
 ;.
 ; NVPTX: [[META0:![0-9]+]] = !{i32 0, i32 64770, i32 541341486, !"", i32 74, i32 5}
 ; NVPTX: [[META1:![0-9]+]] = !{i32 0, i32 64770, i32 541341486, !"sequential_loop_to_stack_var", i32 20, i32 1}
@@ -1169,27 +1157,22 @@ attributes #11 = { convergent }
 ; NVPTX: [[META3:![0-9]+]] = !{i32 0, i32 64770, i32 541341486, !"sequential_loop_to_shared_var", i32 35, i32 2}
 ; NVPTX: [[META4:![0-9]+]] = !{i32 0, i32 64770, i32 541341486, !"do_not_spmdize_target", i32 65, i32 4}
 ; NVPTX: [[META5:![0-9]+]] = !{i32 0, i32 64770, i32 541341486, !"sequential_loop_to_shared_var_guarded", i32 50, i32 3}
-; NVPTX: [[META6:![0-9]+]] = !{ptr @spmd_callees, !"kernel", i32 1}
-; NVPTX: [[META7:![0-9]+]] = !{ptr @spmd_and_non_spmd_callees_metadata, !"kernel", i32 1}
-; NVPTX: [[META8:![0-9]+]] = !{ptr @spmd_and_non_spmd_callee, !"kernel", i32 1}
-; NVPTX: [[META9:![0-9]+]] = !{ptr @spmd_callees_metadata, !"kernel", i32 1}
-; NVPTX: [[META10:![0-9]+]] = !{i32 1}
-; NVPTX: [[META11:![0-9]+]] = !{i32 1, !"wchar_size", i32 4}
-; NVPTX: [[META12:![0-9]+]] = !{i32 7, !"openmp", i32 50}
-; NVPTX: [[META13:![0-9]+]] = !{i32 7, !"openmp-device", i32 50}
-; NVPTX: [[META14:![0-9]+]] = !{i32 8, !"PIC Level", i32 2}
-; NVPTX: [[META15:![0-9]+]] = !{i32 7, !"frame-pointer", i32 2}
-; NVPTX: [[META16:![0-9]+]] = !{!"{{.*}}clang version {{.*}}"}
-; NVPTX: [[TBAA17]] = !{[[META18:![0-9]+]], [[META18]], i64 0}
-; NVPTX: [[META18]] = !{!"int", [[META19:![0-9]+]], i64 0}
-; NVPTX: [[META19]] = !{!"omnipotent char", [[META20:![0-9]+]], i64 0}
-; NVPTX: [[META20]] = !{!"Simple C/C++ TBAA"}
-; NVPTX: [[LOOP21]] = distinct !{[[LOOP21]], [[META22:![0-9]+]], [[META23:![0-9]+]]}
-; NVPTX: [[META22]] = !{!"llvm.loop.mustprogress"}
-; NVPTX: [[META23]] = !{!"llvm.loop.unroll.disable"}
-; NVPTX: [[LOOP24]] = distinct !{[[LOOP24]], [[META22]], [[META23]]}
-; NVPTX: [[TBAA25]] = !{[[META26:![0-9]+]], [[META26]], i64 0}
-; NVPTX: [[META26]] = !{!"any pointer", [[META19]], i64 0}
-; NVPTX: [[LOOP27]] = distinct !{[[LOOP27]], [[META22]], [[META23]]}
-; NVPTX: [[LOOP28]] = distinct !{[[LOOP28]], [[META22]], [[META23]]}
+; NVPTX: [[META6:![0-9]+]] = !{i32 1, !"wchar_size", i32 4}
+; NVPTX: [[META7:![0-9]+]] = !{i32 7, !"openmp", i32 50}
+; NVPTX: [[META8:![0-9]+]] = !{i32 7, !"openmp-device", i32 50}
+; NVPTX: [[META9:![0-9]+]] = !{i32 8, !"PIC Level", i32 2}
+; NVPTX: [[META10:![0-9]+]] = !{i32 7, !"frame-pointer", i32 2}
+; NVPTX: [[META11:![0-9]+]] = !{!"{{.*}}clang version {{.*}}"}
+; NVPTX: [[TBAA12]] = !{[[META13:![0-9]+]], [[META13]], i64 0}
+; NVPTX: [[META13]] = !{!"int", [[META14:![0-9]+]], i64 0}
+; NVPTX: [[META14]] = !{!"omnipotent char", [[META15:![0-9]+]], i64 0}
+; NVPTX: [[META15]] = !{!"Simple C/C++ TBAA"}
+; NVPTX: [[LOOP16]] = distinct !{[[LOOP16]], [[META17:![0-9]+]], [[META18:![0-9]+]]}
+; NVPTX: [[META17]] = !{!"llvm.loop.mustprogress"}
+; NVPTX: [[META18]] = !{!"llvm.loop.unroll.disable"}
+; NVPTX: [[LOOP19]] = distinct !{[[LOOP19]], [[META17]], [[META18]]}
+; NVPTX: [[TBAA20]] = !{[[META21:![0-9]+]], [[META21]], i64 0}
+; NVPTX: [[META21]] = !{!"any pointer", [[META14]], i64 0}
+; NVPTX: [[LOOP22]] = distinct !{[[LOOP22]], [[META17]], [[META18]]}
+; NVPTX: [[LOOP23]] = distinct !{[[LOOP23]], [[META17]], [[META18]]}
 ;.
diff --git a/llvm/test/Transforms/OpenMP/spmdization_no_guarding_two_reaching_kernels.ll b/llvm/test/Transforms/OpenMP/spmdization_no_guarding_two_reaching_kernels.ll
index f28f61e053275..1cfce147ac81e 100644
--- a/llvm/test/Transforms/OpenMP/spmdization_no_guarding_two_reaching_kernels.ll
+++ b/llvm/test/Transforms/OpenMP/spmdization_no_guarding_two_reaching_kernels.ll
@@ -57,7 +57,7 @@ target triple = "nvptx64"
 ; CHECK-DISABLE-SPMDIZATION: @__omp_offloading_2b_10393b5_generic_l20_kernel_environment = local_unnamed_addr constant %struct.KernelEnvironmentTy { %struct.ConfigurationEnvironmentTy { i8 0, i8 0, i8 1, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0 }, ptr @[[GLOB1]], ptr null }
 ; CHECK-DISABLE-SPMDIZATION: @__omp_outlined___wrapper.ID = private constant i8 undef
 ;.
-define weak void @__omp_offloading_2b_10393b5_spmd_l12(ptr %dyn) #0 {
+define weak ptx_kernel void @__omp_offloading_2b_10393b5_spmd_l12(ptr %dyn) #0 {
 ; CHECK-LABEL: define {{[^@]+}}@__omp_offloading_2b_10393b5_spmd_l12
 ; CHECK-SAME: (ptr [[DYN:%.*]]) #[[ATTR0:[0-9]+]] {
 ; CHECK-NEXT:  entry:
@@ -147,7 +147,7 @@ define weak i32 @__kmpc_target_init(ptr, ptr) {
 declare void @__kmpc_target_deinit()
 
 ; Function Attrs: convergent noinline norecurse nounwind
-define weak void @__omp_offloading_2b_10393b5_generic_l20(ptr %dyn) #0 {
+define weak ptx_kernel void @__omp_offloading_2b_10393b5_generic_l20(ptr %dyn) #0 {
 ; CHECK-LABEL: define {{[^@]+}}@__omp_offloading_2b_10393b5_generic_l20
 ; CHECK-SAME: (ptr [[DYN:%.*]]) #[[ATTR0]] {
 ; CHECK-NEXT:  entry:
@@ -397,14 +397,11 @@ attributes #4 = { alwaysinline }
 attributes #5 = { convergent }
 
 !omp_offload.info = !{!0, !1}
-!nvvm.annotations = !{!2, !3}
 !llvm.module.flags = !{!4, !5, !6, !7, !8}
 !llvm.ident = !{!9}
 
 !0 = !{i32 0, i32 43, i32 17011637, !"spmd", i32 12, i32 0}
 !1 = !{i32 0, i32 43, i32 17011637, !"generic", i32 20, i32 1}
-!2 = !{ptr @__omp_offloading_2b_10393b5_spmd_l12, !"kernel", i32 1}
-!3 = !{ptr @__omp_offloading_2b_10393b5_generic_l20, !"kernel", i32 1}
 !4 = !{i32 1, !"wchar_size", i32 4}
 !5 = !{i32 7, !"openmp", i32 50}
 !6 = !{i32 7, !"openmp-device", i32 50}
@@ -434,23 +431,19 @@ attributes #5 = { convergent }
 ;.
 ; CHECK: [[META0:![0-9]+]] = !{i32 0, i32 43, i32 17011637, !"spmd", i32 12, i32 0}
 ; CHECK: [[META1:![0-9]+]] = !{i32 0, i32 43, i32 17011637, !"generic", i32 20, i32 1}
-; CHECK: [[META2:![0-9]+]] = !{ptr @__omp_offloading_2b_10393b5_spmd_l12, !"kernel", i32 1}
-; CHECK: [[META3:![0-9]+]] = !{ptr @__omp_offloading_2b_10393b5_generic_l20, !"kernel", i32 1}
-; CHECK: [[META4:![0-9]+]] = !{i32 1, !"wchar_size", i32 4}
-; CHECK: [[META5:![0-9]+]] = !{i32 7, !"openmp", i32 50}
-; CHECK: [[META6:![0-9]+]] = !{i32 7, !"openmp-device", i32 50}
-; CHECK: [[META7:![0-9]+]] = !{i32 8, !"PIC Level", i32 2}
-; CHECK: [[META8:![0-9]+]] = !{i32 7, !"frame-pointer", i32 2}
-; CHECK: [[META9:![0-9]+]] = !{!"{{.*}}clang version {{.*}}"}
+; CHECK: [[META2:![0-9]+]] = !{i32 1, !"wchar_size", i32 4}
+; CHECK: [[META3:![0-9]+]] = !{i32 7, !"openmp", i32 50}
+; CHECK: [[META4:![0-9]+]] = !{i32 7, !"openmp-device", i32 50}
+; CHECK: [[META5:![0-9]+]] = !{i32 8, !"PIC Level", i32 2}
+; CHECK: [[META6:![0-9]+]] = !{i32 7, !"frame-pointer", i32 2}
+; CHECK: [[META7:![0-9]+]] = !{!"{{.*}}clang version {{.*}}"}
 ;.
 ; CHECK-DISABLE-SPMDIZATION: [[META0:![0-9]+]] = !{i32 0, i32 43, i32 17011637, !"spmd", i32 12, i32 0}
 ; CHECK-DISABLE-SPMDIZATION: [[META1:![0-9]+]] = !{i32 0, i32 43, i32 17011637, !"generic", i32 20, i32 1}
-; CHECK-DISABLE-SPMDIZATION: [[META2:![0-9]+]] = !{ptr @__omp_offloading_2b_10393b5_spmd_l12, !"kernel", i32 1}
-; CHECK-DISABLE-SPMDIZATION: [[META3:![0-9]+]] = !{ptr @__omp_offloading_2b_10393b5_generic_l20, !"kernel", i32 1}
-; CHECK-DISABLE-SPMDIZATION: [[META4:![0-9]+]] = !{i32 1, !"wchar_size", i32 4}
-; CHECK-DISABLE-SPMDIZATION: [[META5:![0-9]+]] = !{i32 7, !"openmp", i32 50}
-; CHECK-DISABLE-SPMDIZATION: [[META6:![0-9]+]] = !{i32 7, !"openmp-device", i32 50}
-; CHECK-DISABLE-SPMDIZATION: [[META7:![0-9]+]] = !{i32 8, !"PIC Level", i32 2}
-; CHECK-DISABLE-SPMDIZATION: [[META8:![0-9]+]] = !{i32 7, !"frame-pointer", i32 2}
-; CHECK-DISABLE-SPMDIZATION: [[META9:![0-9]+]] = !{!"{{.*}}clang version {{.*}}"}
+; CHECK-DISABLE-SPMDIZATION: [[META2:![0-9]+]] = !{i32 1, !"wchar_size", i32 4}
+; CHECK-DISABLE-SPMDIZATION: [[META3:![0-9]+]] = !{i32 7, !"openmp", i32 50}
+; CHECK-DISABLE-SPMDIZATION: [[META4:![0-9]+]] = !{i32 7, !"openmp-device", i32 50}
+; CHECK-DISABLE-SPMDIZATION: [[META5:![0-9]+]] = !{i32 8, !"PIC Level", i32 2}
+; CHECK-DISABLE-SPMDIZATION: [[META6:![0-9]+]] = !{i32 7, !"frame-pointer", i32 2}
+; CHECK-DISABLE-SPMDIZATION: [[META7:![0-9]+]] = !{!"{{.*}}clang version {{.*}}"}
 ;.
diff --git a/llvm/test/Transforms/OpenMP/spmdization_remarks.ll b/llvm/test/Transforms/OpenMP/spmdization_remarks.ll
index f5a4cea9a841c..ef36937bc5734 100644
--- a/llvm/test/Transforms/OpenMP/spmdization_remarks.ll
+++ b/llvm/test/Transforms/OpenMP/spmdization_remarks.ll
@@ -62,7 +62,7 @@ target triple = "nvptx64"
 
 
 ; Function Attrs: convergent norecurse nounwind
-define weak void @__omp_offloading_2a_d80d3d_test_fallback_l11(ptr %dyn) local_unnamed_addr #0 !dbg !15 {
+define weak ptx_kernel void @__omp_offloading_2a_d80d3d_test_fallback_l11(ptr %dyn) local_unnamed_addr #0 !dbg !15 {
 entry:
   %captured_vars_addrs.i.i = alloca [0 x ptr], align 8
   %0 = call i32 @__kmpc_target_init(ptr nonnull @__omp_offloading_2a_d80d3d_test_fallback_l11_kernel_environment, ptr %dyn) #3, !dbg !18
@@ -107,7 +107,7 @@ declare i32 @__kmpc_global_thread_num(ptr) local_unnamed_addr #3
 declare void @__kmpc_target_deinit() local_unnamed_addr
 
 ; Function Attrs: norecurse nounwind
-define weak void @__omp_offloading_2a_d80d3d_test_no_fallback_l20(ptr %dyn) local_unnamed_addr #4 !dbg !32 {
+define weak ptx_kernel void @__omp_offloading_2a_d80d3d_test_no_fallback_l20(ptr %dyn) local_unnamed_addr #4 !dbg !32 {
 entry:
   %captured_vars_addrs.i2.i = alloca [0 x ptr], align 8
   %0 = call i32 @__kmpc_target_init(ptr nonnull @__omp_offloading_2a_d80d3d_test_no_fallback_l20_kernel_environment, ptr %dyn) #3, !dbg !33
@@ -175,7 +175,6 @@ attributes #7 = { "llvm.assume"="ompx_spmd_amenable" }
 
 !llvm.dbg.cu = !{!0}
 !omp_offload.info = !{!3, !4}
-!nvvm.annotations = !{!5, !6}
 !llvm.module.flags = !{!7, !8, !9, !10, !11, !12, !13}
 !llvm.ident = !{!14}
 
@@ -184,8 +183,6 @@ attributes #7 = { "llvm.assume"="ompx_spmd_amenable" }
 !2 = !{}
 !3 = !{i32 0, i32 42, i32 14159165, !"test_no_fallback", i32 20, i32 1}
 !4 = !{i32 0, i32 42, i32 14159165, !"test_fallback", i32 11, i32 0}
-!5 = !{ptr @__omp_offloading_2a_d80d3d_test_fallback_l11, !"kernel", i32 1}
-!6 = !{ptr @__omp_offloading_2a_d80d3d_test_no_fallback_l20, !"kernel", i32 1}
 !7 = !{i32 7, !"Dwarf Version", i32 2}
 !8 = !{i32 2, !"Debug Info Version", i32 3}
 !9 = !{i32 1, !"wchar_size", i32 4}
diff --git a/llvm/test/Transforms/OpenMP/value-simplify-openmp-opt.ll b/llvm/test/Transforms/OpenMP/value-simplify-openmp-opt.ll
index 5e2abbae1811c..2842dfd030b11 100644
--- a/llvm/test/Transforms/OpenMP/value-simplify-openmp-opt.ll
+++ b/llvm/test/Transforms/OpenMP/value-simplify-openmp-opt.ll
@@ -53,7 +53,7 @@ target triple = "amdgcn-amd-amdhsa"
 ; CHECK: @str = private unnamed_addr addrspace(4) constant [1 x i8] zeroinitializer, align 1
 ; CHECK: @kernel_kernel_environment = local_unnamed_addr constant %struct.KernelEnvironmentTy { %struct.ConfigurationEnvironmentTy { i8 0, i8 0, i8 1, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0 }, ptr null, ptr null }
 ;.
-define void @kernel(ptr %dyn) "kernel" {
+define amdgpu_kernel void @kernel(ptr %dyn) "kernel" {
 ;
 ; TUNIT: Function Attrs: norecurse
 ; TUNIT-LABEL: define {{[^@]+}}@kernel
@@ -144,7 +144,7 @@ define void @test_assume() {
 }
 
 ; We can't ignore the sync, hence this might store 2 into %p
-define void @kernel2(ptr %p) "kernel" {
+define amdgpu_kernel void @kernel2(ptr %p) "kernel" {
 ; CHECK-LABEL: define {{[^@]+}}@kernel2
 ; CHECK-SAME: (ptr [[P:%.*]]) #[[ATTR1:[0-9]+]] {
 ; CHECK-NEXT:    store i32 1, ptr addrspace(3) @X, align 4
@@ -163,7 +163,7 @@ define void @kernel2(ptr %p) "kernel" {
 }
 
 ; We can't ignore the sync, hence this might store 2 into %p
-define void @kernel3(ptr %p) "kernel" {
+define amdgpu_kernel void @kernel3(ptr %p) "kernel" {
 ; TUNIT-LABEL: define {{[^@]+}}@kernel3
 ; TUNIT-SAME: (ptr [[P:%.*]]) #[[ATTR1]] {
 ; TUNIT-NEXT:    store i32 1, ptr addrspace(3) @X, align 4
@@ -199,7 +199,7 @@ define void @sync_def() {
   ret void
 }
 
-define void @kernel4a1(i1 %c) "kernel" {
+define amdgpu_kernel void @kernel4a1(i1 %c) "kernel" {
 ; TUNIT-LABEL: define {{[^@]+}}@kernel4a1
 ; TUNIT-SAME: (i1 [[C:%.*]]) #[[ATTR1]] {
 ; TUNIT-NEXT:    store i32 0, ptr addrspace(3) @QA1, align 4
@@ -242,7 +242,7 @@ S:
 }
 
 ; We should not replace the load or delete the second store.
-define void @kernel4b1(i1 %c) "kernel" {
+define amdgpu_kernel void @kernel4b1(i1 %c) "kernel" {
 ; TUNIT-LABEL: define {{[^@]+}}@kernel4b1
 ; TUNIT-SAME: (i1 [[C:%.*]]) #[[ATTR1]] {
 ; TUNIT-NEXT:    store i32 0, ptr addrspace(3) @QB1, align 4
@@ -281,7 +281,7 @@ S:
   ret void
 }
 
-define void @kernel4a2(i1 %c) "kernel" {
+define amdgpu_kernel void @kernel4a2(i1 %c) "kernel" {
 ; TUNIT-LABEL: define {{[^@]+}}@kernel4a2
 ; TUNIT-SAME: (i1 [[C:%.*]]) #[[ATTR1]] {
 ; TUNIT-NEXT:    br i1 [[C]], label [[S:%.*]], label [[L:%.*]]
@@ -317,7 +317,7 @@ S:
 }
 
 ; FIXME: We should not replace the load with undef.
-define void @kernel4b2(i1 %c) "kernel" {
+define amdgpu_kernel void @kernel4b2(i1 %c) "kernel" {
 ; TUNIT-LABEL: define {{[^@]+}}@kernel4b2
 ; TUNIT-SAME: (i1 [[C:%.*]]) #[[ATTR1]] {
 ; TUNIT-NEXT:    br i1 [[C]], label [[S:%.*]], label [[L:%.*]]
@@ -349,7 +349,7 @@ S:
   ret void
 }
 
-define void @kernel4a3(i1 %c) "kernel" {
+define amdgpu_kernel void @kernel4a3(i1 %c) "kernel" {
 ; TUNIT-LABEL: define {{[^@]+}}@kernel4a3
 ; TUNIT-SAME: (i1 [[C:%.*]]) #[[ATTR1]] {
 ; TUNIT-NEXT:    store i32 0, ptr addrspace(3) @QA3, align 4
@@ -401,7 +401,7 @@ S:
 }
 
 ; The load of QB3 should not be simplified to 0.
-define void @kernel4b3(i1 %c) "kernel" {
+define amdgpu_kernel void @kernel4b3(i1 %c) "kernel" {
 ; TUNIT-LABEL: define {{[^@]+}}@kernel4b3
 ; TUNIT-SAME: (i1 [[C:%.*]]) #[[ATTR1]] {
 ; TUNIT-NEXT:    store i32 0, ptr addrspace(3) @QB3, align 4
@@ -453,7 +453,7 @@ S:
 }
 
 
-define void @kernel4c1(i1 %c) "kernel" {
+define amdgpu_kernel void @kernel4c1(i1 %c) "kernel" {
 ; TUNIT: Function Attrs: norecurse
 ; TUNIT-LABEL: define {{[^@]+}}@kernel4c1
 ; TUNIT-SAME: (i1 [[C:%.*]]) #[[ATTR0]] {
@@ -488,7 +488,7 @@ S:
 }
 
 ; We should not replace the load or delete the second store.
-define void @kernel4d1(i1 %c) "kernel" {
+define amdgpu_kernel void @kernel4d1(i1 %c) "kernel" {
 ; TUNIT: Function Attrs: norecurse
 ; TUNIT-LABEL: define {{[^@]+}}@kernel4d1
 ; TUNIT-SAME: (i1 [[C:%.*]]) #[[ATTR0]] {
@@ -529,7 +529,7 @@ S:
   ret void
 }
 
-define void @kernel4c2(i1 %c) "kernel" {
+define amdgpu_kernel void @kernel4c2(i1 %c) "kernel" {
 ; TUNIT: Function Attrs: norecurse
 ; TUNIT-LABEL: define {{[^@]+}}@kernel4c2
 ; TUNIT-SAME: (i1 [[C:%.*]]) #[[ATTR0]] {
@@ -563,7 +563,7 @@ S:
 }
 
 ; We should not replace the load with undef.
-define void @kernel4d2(i1 %c) "kernel" {
+define amdgpu_kernel void @kernel4d2(i1 %c) "kernel" {
 ; TUNIT: Function Attrs: norecurse
 ; TUNIT-LABEL: define {{[^@]+}}@kernel4d2
 ; TUNIT-SAME: (i1 [[C:%.*]]) #[[ATTR0]] {
@@ -595,7 +595,7 @@ S:
   ret void
 }
 
-define void @kernel4c3(i1 %c) "kernel" {
+define amdgpu_kernel void @kernel4c3(i1 %c) "kernel" {
 ; TUNIT: Function Attrs: norecurse
 ; TUNIT-LABEL: define {{[^@]+}}@kernel4c3
 ; TUNIT-SAME: (i1 [[C:%.*]]) #[[ATTR0]] {
@@ -629,7 +629,7 @@ S:
 }
 
 ; We should not replace the load with undef.
-define void @kernel4d3(i1 %c) "kernel" {
+define amdgpu_kernel void @kernel4d3(i1 %c) "kernel" {
 ; TUNIT: Function Attrs: norecurse
 ; TUNIT-LABEL: define {{[^@]+}}@kernel4d3
 ; TUNIT-SAME: (i1 [[C:%.*]]) #[[ATTR0]] {
@@ -661,7 +661,7 @@ S:
   ret void
 }
 
-define void @kernel_unknown_and_aligned1(i1 %c) "kernel" {
+define amdgpu_kernel void @kernel_unknown_and_aligned1(i1 %c) "kernel" {
 ; TUNIT-LABEL: define {{[^@]+}}@kernel_unknown_and_aligned1
 ; TUNIT-SAME: (i1 [[C:%.*]]) #[[ATTR1]] {
 ; TUNIT-NEXT:    br i1 [[C]], label [[S:%.*]], label [[L:%.*]]
@@ -700,7 +700,7 @@ S:
   ret void
 }
 
-define void @kernel_unknown_and_aligned2(i1 %c) "kernel" {
+define amdgpu_kernel void @kernel_unknown_and_aligned2(i1 %c) "kernel" {
 ; TUNIT-LABEL: define {{[^@]+}}@kernel_unknown_and_aligned2
 ; TUNIT-SAME: (i1 [[C:%.*]]) #[[ATTR1]] {
 ; TUNIT-NEXT:    br i1 [[C]], label [[S:%.*]], label [[L:%.*]]
@@ -741,7 +741,7 @@ S:
   ret void
 }
 
-define void @kernel_unknown_and_aligned3(i1 %c) "kernel" {
+define amdgpu_kernel void @kernel_unknown_and_aligned3(i1 %c) "kernel" {
 ; TUNIT-LABEL: define {{[^@]+}}@kernel_unknown_and_aligned3
 ; TUNIT-SAME: (i1 [[C:%.*]]) #[[ATTR1]] {
 ; TUNIT-NEXT:    br i1 [[C]], label [[S:%.*]], label [[L:%.*]]
@@ -782,7 +782,7 @@ S:
   ret void
 }
 
-define void @kernel_unknown_and_not_aligned1(i1 %c) "kernel" {
+define amdgpu_kernel void @kernel_unknown_and_not_aligned1(i1 %c) "kernel" {
 ; TUNIT-LABEL: define {{[^@]+}}@kernel_unknown_and_not_aligned1
 ; TUNIT-SAME: (i1 [[C:%.*]]) #[[ATTR1]] {
 ; TUNIT-NEXT:    br i1 [[C]], label [[S:%.*]], label [[L:%.*]]
@@ -828,29 +828,9 @@ declare void @__kmpc_target_deinit() nocallback
 declare void @llvm.assume(i1)
 
 !llvm.module.flags = !{!0, !1}
-!nvvm.annotations = !{!2, !3, !4, !5, !6, !7, !8, !9, !10, !11, !12, !13, !14, !15, !16, !17, !18, !19, !20}
 
 !0 = !{i32 7, !"openmp", i32 50}
 !1 = !{i32 7, !"openmp-device", i32 50}
-!2 = !{ptr @kernel, !"kernel", i32 1}
-!3 = !{ptr @kernel2, !"kernel", i32 1}
-!4 = !{ptr @kernel3, !"kernel", i32 1}
-!5 = !{ptr @kernel4a1, !"kernel", i32 1}
-!6 = !{ptr @kernel4b1, !"kernel", i32 1}
-!7 = !{ptr @kernel4a2, !"kernel", i32 1}
-!8 = !{ptr @kernel4b2, !"kernel", i32 1}
-!9 = !{ptr @kernel4a3, !"kernel", i32 1}
-!10 = !{ptr @kernel4b3, !"kernel", i32 1}
-!11 = !{ptr @kernel4c1, !"kernel", i32 1}
-!12 = !{ptr @kernel4d1, !"kernel", i32 1}
-!13 = !{ptr @kernel4c2, !"kernel", i32 1}
-!14 = !{ptr @kernel4d2, !"kernel", i32 1}
-!15 = !{ptr @kernel4c3, !"kernel", i32 1}
-!16 = !{ptr @kernel4d3, !"kernel", i32 1}
-!17 = !{ptr @kernel_unknown_and_aligned1, !"kernel", i32 1}
-!18 = !{ptr @kernel_unknown_and_aligned2, !"kernel", i32 1}
-!19 = !{ptr @kernel_unknown_and_aligned3, !"kernel", i32 1}
-!20 = !{ptr @kernel_unknown_and_not_aligned1, !"kernel", i32 1}
 
 ;.
 ; TUNIT: attributes #[[ATTR0]] = { norecurse "kernel" }
@@ -872,45 +852,7 @@ declare void @llvm.assume(i1)
 ;.
 ; TUNIT: [[META0:![0-9]+]] = !{i32 7, !"openmp", i32 50}
 ; TUNIT: [[META1:![0-9]+]] = !{i32 7, !"openmp-device", i32 50}
-; TUNIT: [[META2:![0-9]+]] = !{ptr @kernel, !"kernel", i32 1}
-; TUNIT: [[META3:![0-9]+]] = !{ptr @kernel2, !"kernel", i32 1}
-; TUNIT: [[META4:![0-9]+]] = !{ptr @kernel3, !"kernel", i32 1}
-; TUNIT: [[META5:![0-9]+]] = !{ptr @kernel4a1, !"kernel", i32 1}
-; TUNIT: [[META6:![0-9]+]] = !{ptr @kernel4b1, !"kernel", i32 1}
-; TUNIT: [[META7:![0-9]+]] = !{ptr @kernel4a2, !"kernel", i32 1}
-; TUNIT: [[META8:![0-9]+]] = !{ptr @kernel4b2, !"kernel", i32 1}
-; TUNIT: [[META9:![0-9]+]] = !{ptr @kernel4a3, !"kernel", i32 1}
-; TUNIT: [[META10:![0-9]+]] = !{ptr @kernel4b3, !"kernel", i32 1}
-; TUNIT: [[META11:![0-9]+]] = !{ptr @kernel4c1, !"kernel", i32 1}
-; TUNIT: [[META12:![0-9]+]] = !{ptr @kernel4d1, !"kernel", i32 1}
-; TUNIT: [[META13:![0-9]+]] = !{ptr @kernel4c2, !"kernel", i32 1}
-; TUNIT: [[META14:![0-9]+]] = !{ptr @kernel4d2, !"kernel", i32 1}
-; TUNIT: [[META15:![0-9]+]] = !{ptr @kernel4c3, !"kernel", i32 1}
-; TUNIT: [[META16:![0-9]+]] = !{ptr @kernel4d3, !"kernel", i32 1}
-; TUNIT: [[META17:![0-9]+]] = !{ptr @kernel_unknown_and_aligned1, !"kernel", i32 1}
-; TUNIT: [[META18:![0-9]+]] = !{ptr @kernel_unknown_and_aligned2, !"kernel", i32 1}
-; TUNIT: [[META19:![0-9]+]] = !{ptr @kernel_unknown_and_aligned3, !"kernel", i32 1}
-; TUNIT: [[META20:![0-9]+]] = !{ptr @kernel_unknown_and_not_aligned1, !"kernel", i32 1}
 ;.
 ; CGSCC: [[META0:![0-9]+]] = !{i32 7, !"openmp", i32 50}
 ; CGSCC: [[META1:![0-9]+]] = !{i32 7, !"openmp-device", i32 50}
-; CGSCC: [[META2:![0-9]+]] = !{ptr @kernel, !"kernel", i32 1}
-; CGSCC: [[META3:![0-9]+]] = !{ptr @kernel2, !"kernel", i32 1}
-; CGSCC: [[META4:![0-9]+]] = !{ptr @kernel3, !"kernel", i32 1}
-; CGSCC: [[META5:![0-9]+]] = !{ptr @kernel4a1, !"kernel", i32 1}
-; CGSCC: [[META6:![0-9]+]] = !{ptr @kernel4b1, !"kernel", i32 1}
-; CGSCC: [[META7:![0-9]+]] = !{ptr @kernel4a2, !"kernel", i32 1}
-; CGSCC: [[META8:![0-9]+]] = !{ptr @kernel4b2, !"kernel", i32 1}
-; CGSCC: [[META9:![0-9]+]] = !{ptr @kernel4a3, !"kernel", i32 1}
-; CGSCC: [[META10:![0-9]+]] = !{ptr @kernel4b3, !"kernel", i32 1}
-; CGSCC: [[META11:![0-9]+]] = !{ptr @kernel4c1, !"kernel", i32 1}
-; CGSCC: [[META12:![0-9]+]] = !{ptr @kernel4d1, !"kernel", i32 1}
-; CGSCC: [[META13:![0-9]+]] = !{ptr @kernel4c2, !"kernel", i32 1}
-; CGSCC: [[META14:![0-9]+]] = !{ptr @kernel4d2, !"kernel", i32 1}
-; CGSCC: [[META15:![0-9]+]] = !{ptr @kernel4c3, !"kernel", i32 1}
-; CGSCC: [[META16:![0-9]+]] = !{ptr @kernel4d3, !"kernel", i32 1}
-; CGSCC: [[META17:![0-9]+]] = !{ptr @kernel_unknown_and_aligned1, !"kernel", i32 1}
-; CGSCC: [[META18:![0-9]+]] = !{ptr @kernel_unknown_and_aligned2, !"kernel", i32 1}
-; CGSCC: [[META19:![0-9]+]] = !{ptr @kernel_unknown_and_aligned3, !"kernel", i32 1}
-; CGSCC: [[META20:![0-9]+]] = !{ptr @kernel_unknown_and_not_aligned1, !"kernel", i32 1}
 ;.
diff --git a/llvm/test/Transforms/PGOProfile/memprof-dump-matched-call-sites.ll b/llvm/test/Transforms/PGOProfile/memprof-dump-matched-call-sites.ll
new file mode 100644
index 0000000000000..a5302895d0593
--- /dev/null
+++ b/llvm/test/Transforms/PGOProfile/memprof-dump-matched-call-sites.ll
@@ -0,0 +1,114 @@
+; Tests that the compiler dumps call site matches upon request.
+;
+; The test case is generated from:
+;
+; // main
+; // |
+; // f1 (noinline)
+; // |
+; // f2
+; // |
+; // f3 (noinline)
+; // |
+; // new
+;
+; __attribute__((noinline)) char *f3() { return ::new char[4]; }
+;
+; static char *f2() { return f3(); }
+;
+; __attribute__((noinline)) static char *f1() { return f2(); }
+;
+; int main() {
+;   f1();
+;   return 0;
+; }
+;
+; Here we expect to match two inline call stacks:
+;
+; - [main]
+; - [f1, f2]
+;
+; Note that f3 is considered to be an allocation site, not a call site, because
+; it directly calls new after inlining.
+
+; REQUIRES: x86_64-linux
+; RUN: split-file %s %t
+; RUN: llvm-profdata merge %t/memprof-dump-matched-call-site.yaml -o %t/memprof-dump-matched-call-site.memprofdata
+; RUN: opt < %t/memprof-dump-matched-call-site.ll -passes='memprof-use<profile-filename=%t/memprof-dump-matched-call-site.memprofdata>' -memprof-print-match-info -S 2>&1 | FileCheck %s
+
+;--- memprof-dump-matched-call-site.yaml
+---
+HeapProfileRecords:
+  - GUID:            main
+    AllocSites:      []
+    CallSites:
+      - - { Function: main, LineOffset: 1, Column: 3, IsInlineFrame: false }
+  - GUID:            _ZL2f1v
+    AllocSites:      []
+    CallSites:
+      - - { Function: _ZL2f2v, LineOffset: 0, Column: 28, IsInlineFrame: true }
+        - { Function: _ZL2f1v, LineOffset: 0, Column: 54, IsInlineFrame: false }
+  - GUID:            _ZL2f2v
+    AllocSites:      []
+    CallSites:
+      - - { Function: _ZL2f2v, LineOffset: 0, Column: 28, IsInlineFrame: true }
+        - { Function: _ZL2f1v, LineOffset: 0, Column: 54, IsInlineFrame: false }
+  - GUID:            _Z2f3v
+    AllocSites:
+      - Callstack:
+          - { Function: _Z2f3v, LineOffset: 0, Column: 47, IsInlineFrame: false }
+          - { Function: _ZL2f2v, LineOffset: 0, Column: 28, IsInlineFrame: true }
+          - { Function: _ZL2f1v, LineOffset: 0, Column: 54, IsInlineFrame: false }
+          - { Function: main, LineOffset: 1, Column: 3, IsInlineFrame: false }
+        MemInfoBlock:
+          AllocCount:      1
+          TotalSize:       4
+          TotalLifetime:   0
+          TotalLifetimeAccessDensity: 0
+    CallSites:       []
+...
+;--- memprof-dump-matched-call-site.ll
+; CHECK: MemProf notcold context with id 3894143216621363392 has total profiled size 4 is matched
+; CHECK: MemProf callsite match for inline call stack 4745611964195289084 10616861955219347331
+; CHECK: MemProf callsite match for inline call stack 5401059281181789382
+
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+define ptr @_Z2f3v() {
+entry:
+  %call = call ptr @_Znam(i64 0), !dbg !3
+  ret ptr null
+}
+
+declare ptr @_Znam(i64)
+
+define i32 @main() {
+entry:
+  call void @_ZL2f1v(), !dbg !7
+  ret i32 0
+}
+
+define void @_ZL2f1v() {
+entry:
+  %call.i = call ptr @_Z2f3v(), !dbg !9
+  ret void
+}
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!2}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus_14, file: !1)
+!1 = !DIFile(filename: "match.cc", directory: "/")
+!2 = !{i32 2, !"Debug Info Version", i32 3}
+!3 = !DILocation(line: 11, column: 47, scope: !4)
+!4 = distinct !DISubprogram(name: "f3", linkageName: "_Z2f3v", scope: !1, file: !1, line: 11, type: !5, scopeLine: 11, flags: DIFlagPrototyped | DIFlagAllCallsDescribed, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0)
+!5 = !DISubroutineType(types: !6)
+!6 = !{}
+!7 = !DILocation(line: 18, column: 3, scope: !8)
+!8 = distinct !DISubprogram(name: "main", scope: !1, file: !1, line: 17, type: !5, scopeLine: 17, flags: DIFlagPrototyped | DIFlagAllCallsDescribed, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0)
+!9 = !DILocation(line: 13, column: 28, scope: !10, inlinedAt: !11)
+!10 = distinct !DISubprogram(name: "f2", linkageName: "_ZL2f2v", scope: !1, file: !1, line: 13, type: !5, scopeLine: 13, flags: DIFlagPrototyped | DIFlagAllCallsDescribed, spFlags: DISPFlagLocalToUnit | DISPFlagDefinition | DISPFlagOptimized, unit: !0)
+!11 = distinct !DILocation(line: 15, column: 54, scope: !12)
+!12 = distinct !DISubprogram(name: "f1", linkageName: "_ZL2f1v", scope: !1, file: !1, line: 15, type: !13, scopeLine: 15, flags: DIFlagPrototyped | DIFlagAllCallsDescribed, spFlags: DISPFlagLocalToUnit | DISPFlagDefinition | DISPFlagOptimized, unit: !0)
+!13 = !DISubroutineType(cc: DW_CC_nocall, types: !6)
diff --git a/llvm/test/Transforms/PGOProfile/memprof.ll b/llvm/test/Transforms/PGOProfile/memprof.ll
index c0e44cccbf16f..5a958de5f7f8d 100644
--- a/llvm/test/Transforms/PGOProfile/memprof.ll
+++ b/llvm/test/Transforms/PGOProfile/memprof.ll
@@ -64,14 +64,14 @@
 ; RUN: opt < %s -passes='pgo-instr-use,memprof-use<profile-filename=%t.pgomemprofdata>' -pgo-test-profile-file=%t.pgomemprofdata -pgo-warn-missing-function -S 2>&1 | FileCheck %s --check-prefixes=MEMPROF,ALL,PGO
 
 ;; Check that the total sizes are reported if requested.
-; RUN: opt < %s -passes='memprof-use<profile-filename=%t.memprofdata>' -pgo-warn-missing-function -S -memprof-report-hinted-sizes 2>&1 | FileCheck %s --check-prefixes=TOTALSIZESSINGLE,TOTALSIZES
+; RUN: opt < %s -passes='memprof-use<profile-filename=%t.memprofdata>' -pgo-warn-missing-function -S -memprof-report-hinted-sizes -memprof-keep-all-not-cold-contexts 2>&1 | FileCheck %s --check-prefixes=TOTALSIZESSINGLE,TOTALSIZES
 
 ;; Check that we hint additional allocations with a threshold < 100%
 ; RUN: opt < %s -passes='memprof-use<profile-filename=%t.memprofdata>' -pgo-warn-missing-function -S -memprof-report-hinted-sizes -memprof-matching-cold-threshold=60 2>&1 | FileCheck %s --check-prefixes=TOTALSIZESSINGLE,TOTALSIZESTHRESH60
 
 ;; Make sure that the -memprof-cloning-cold-threshold flag is enough to cause
 ;; the size metadata to be generated for the LTO link.
-; RUN: opt < %s -passes='memprof-use<profile-filename=%t.memprofdata>' -pgo-warn-missing-function -S -memprof-cloning-cold-threshold=80 2>&1 | FileCheck %s --check-prefixes=TOTALSIZES
+; RUN: opt < %s -passes='memprof-use<profile-filename=%t.memprofdata>' -pgo-warn-missing-function -S -memprof-cloning-cold-threshold=80 -memprof-keep-all-not-cold-contexts 2>&1 | FileCheck %s --check-prefixes=TOTALSIZES
 
 ;; Make sure we emit a random hotness seed if requested.
 ; RUN: llvm-profdata merge -memprof-random-hotness %S/Inputs/memprof.memprofraw --profiled-binary %S/Inputs/memprof.exe -o %t.memprofdatarand 2>&1 | FileCheck %s --check-prefix=RAND
@@ -85,6 +85,14 @@
 ; RAND2: random hotness seed = 1730170724
 ; RUN: opt < %s -passes='memprof-use<profile-filename=%t.memprofdatarand2>' -pgo-warn-missing-function -S -stats 2>&1 | FileCheck %s --check-prefixes=MEMPROFRAND2,ALL,MEMPROFONLY,MEMPROFSTATS
 
+;; With the hot access density threshold set to 0, and hot hints enabled,
+;; the unconditionally notcold call to new should instead get a hot attribute.
+; RUN: opt < %s -passes='memprof-use<profile-filename=%t.memprofdata>' -pgo-warn-missing-function -S -memprof-print-match-info -stats -memprof-min-ave-lifetime-access-density-hot-threshold=0 -memprof-use-hot-hints 2>&1 | FileCheck %s --check-prefixes=MEMPROFHOT,ALL
+
+;; However, with the same threshold, but hot hints not enabled, it should be
+;; notcold again.
+; RUN: opt < %s -passes='memprof-use<profile-filename=%t.memprofdata>' -pgo-warn-missing-function -S -memprof-min-ave-lifetime-access-density-hot-threshold=0 2>&1 | FileCheck %s --check-prefixes=MEMPROF,ALL
+
 ; MEMPROFMATCHINFO: MemProf notcold context with id 1093248920606587996 has total profiled size 10 is matched
 ; MEMPROFMATCHINFO: MemProf notcold context with id 5725971306423925017 has total profiled size 10 is matched
 ; MEMPROFMATCHINFO: MemProf notcold context with id 6792096022461663180 has total profiled size 10 is matched
@@ -93,6 +101,16 @@
 ; MEMPROFMATCHINFO: MemProf cold context with id 15737101490731057601 has total profiled size 10 is matched
 ; MEMPROFMATCHINFO: MemProf cold context with id 16342802530253093571 has total profiled size 10 is matched
 ; MEMPROFMATCHINFO: MemProf cold context with id 18254812774972004394 has total profiled size 10 is matched
+; MEMPROFMATCHINFO: MemProf callsite match for inline call stack 748269490701775343
+; MEMPROFMATCHINFO: MemProf callsite match for inline call stack 1544787832369987002
+; MEMPROFMATCHINFO: MemProf callsite match for inline call stack 2061451396820446691
+; MEMPROFMATCHINFO: MemProf callsite match for inline call stack 2104812325165620841
+; MEMPROFMATCHINFO: MemProf callsite match for inline call stack 6281715513834610934
+; MEMPROFMATCHINFO: MemProf callsite match for inline call stack 8467819354083268568
+; MEMPROFMATCHINFO: MemProf callsite match for inline call stack 8690657650969109624
+; MEMPROFMATCHINFO: MemProf callsite match for inline call stack 9086428284934609951
+; MEMPROFMATCHINFO: MemProf callsite match for inline call stack 12481870273128938184
+; MEMPROFMATCHINFO: MemProf callsite match for inline call stack 12699492813229484831
 
 ; ModuleID = 'memprof.cc'
 source_filename = "memprof.cc"
@@ -192,6 +210,7 @@ entry:
   store ptr %argv, ptr %argv.addr, align 8
   ; MEMPROF: call {{.*}} @_Znam{{.*}} #[[A1:[0-9]+]]
   ; MEMPROFNOCOLINFO: call {{.*}} @_Znam{{.*}} #[[A1:[0-9]+]]
+  ; MEMPROFHOT: call {{.*}} @_Znam{{.*}} #[[A1:[0-9]+]]
   %call = call noalias noundef nonnull ptr @_Znam(i64 noundef 10) #6, !dbg !35
   store ptr %call, ptr %a, align 8, !dbg !36
   ; MEMPROF: call {{.*}} @_Znam{{.*}} #[[A2:[0-9]+]]
@@ -330,7 +349,7 @@ for.end:                                          ; preds = %for.cond
 
 ; MEMPROF: #[[A1]] = { builtin allocsize(0) "memprof"="notcold" }
 ; MEMPROF: #[[A2]] = { builtin allocsize(0) "memprof"="cold" }
-; MEMPROF: ![[M1]] = !{![[MIB1:[0-9]+]], ![[MIB2:[0-9]+]], ![[MIB3:[0-9]+]], ![[MIB4:[0-9]+]], ![[MIB5:[0-9]+]]}
+; MEMPROF: ![[M1]] = !{![[MIB1:[0-9]+]], ![[MIB2:[0-9]+]], ![[MIB3:[0-9]+]], ![[MIB4:[0-9]+]]}
 ; MEMPROF: ![[MIB1]] = !{![[STACK1:[0-9]+]], !"cold"}
 ; MEMPROF: ![[STACK1]] = !{i64 2732490490862098848, i64 748269490701775343}
 ; MEMPROF: ![[MIB2]] = !{![[STACK2:[0-9]+]], !"cold"}
@@ -339,8 +358,6 @@ for.end:                                          ; preds = %for.cond
 ; MEMPROF: ![[STACK3]] = !{i64 2732490490862098848, i64 2104812325165620841, i64 6281715513834610934, i64 6281715513834610934, i64 6281715513834610934, i64 6281715513834610934}
 ; MEMPROF: ![[MIB4]] = !{![[STACK4:[0-9]+]], !"cold"}
 ; MEMPROF: ![[STACK4]] = !{i64 2732490490862098848, i64 8467819354083268568}
-; MEMPROF: ![[MIB5]] = !{![[STACK5:[0-9]+]], !"notcold"}
-; MEMPROF: ![[STACK5]] = !{i64 2732490490862098848, i64 8690657650969109624}
 ; MEMPROF: ![[C1]] = !{i64 2732490490862098848}
 ; MEMPROF: ![[C2]] = !{i64 8467819354083268568}
 ; MEMPROF: ![[C3]] = !{i64 9086428284934609951}
@@ -381,17 +398,15 @@ for.end:                                          ; preds = %for.cond
 
 ; MEMPROFNOCOLINFO: #[[A1]] = { builtin allocsize(0) "memprof"="notcold" }
 ; MEMPROFNOCOLINFO: #[[A2]] = { builtin allocsize(0) "memprof"="cold" }
-; MEMPROFNOCOLINFO: ![[M1]] = !{![[MIB1:[0-9]+]], ![[MIB2:[0-9]+]], ![[MIB3:[0-9]+]], ![[MIB4:[0-9]+]], ![[MIB5:[0-9]+]]}
+; MEMPROFNOCOLINFO: ![[M1]] = !{![[MIB1:[0-9]+]], ![[MIB2:[0-9]+]], ![[MIB3:[0-9]+]], ![[MIB4:[0-9]+]]}
 ; MEMPROFNOCOLINFO: ![[MIB1]] = !{![[STACK1:[0-9]+]], !"cold"}
 ; MEMPROFNOCOLINFO: ![[STACK1]] = !{i64 5281664982037379640, i64 6362220161075421157, i64 -5772587307814069790, i64 -5772587307814069790, i64 -5772587307814069790, i64 3577763375057267810}
 ; MEMPROFNOCOLINFO: ![[MIB2]] = !{![[STACK2:[0-9]+]], !"notcold"}
 ; MEMPROFNOCOLINFO: ![[STACK2]] = !{i64 5281664982037379640, i64 6362220161075421157, i64 -5772587307814069790, i64 -5772587307814069790, i64 -5772587307814069790, i64 -5772587307814069790}
-; MEMPROFNOCOLINFO: ![[MIB3]] = !{![[STACK3:[0-9]+]], !"notcold"}
-; MEMPROFNOCOLINFO: ![[STACK3]] = !{i64 5281664982037379640, i64 -6896091699916449732}
+; MEMPROFNOCOLINFO: ![[MIB3]] = !{![[STACK3:[0-9]+]], !"cold"}
+; MEMPROFNOCOLINFO: ![[STACK3]] = !{i64 5281664982037379640, i64 -6871734214936418908}
 ; MEMPROFNOCOLINFO: ![[MIB4]] = !{![[STACK4:[0-9]+]], !"cold"}
-; MEMPROFNOCOLINFO: ![[STACK4]] = !{i64 5281664982037379640, i64 -6871734214936418908}
-; MEMPROFNOCOLINFO: ![[MIB5]] = !{![[STACK5:[0-9]+]], !"cold"}
-; MEMPROFNOCOLINFO: ![[STACK5]] = !{i64 5281664982037379640, i64 -6201180255894224618}
+; MEMPROFNOCOLINFO: ![[STACK4]] = !{i64 5281664982037379640, i64 -6201180255894224618}
 ; MEMPROFNOCOLINFO: ![[C1]] = !{i64 5281664982037379640}
 ; MEMPROFNOCOLINFO: ![[C2]] = !{i64 -6871734214936418908}
 ; MEMPROFNOCOLINFO: ![[C3]] = !{i64 -5588766871448036195}
@@ -404,12 +419,13 @@ for.end:                                          ; preds = %for.cond
 ; MEMPROFNOCOLINFO: ![[C10]] = !{i64 -4535090212904553409}
 ; MEMPROFNOCOLINFO: ![[C11]] = !{i64 3577763375057267810}
 
+; MEMPROFHOT: #[[A1]] = { builtin allocsize(0) "memprof"="hot" }
+
 ;; For the specific random seed, this is the expected order of hotness
 ; MEMPROFRAND2: !"cold"
 ; MEMPROFRAND2: !"cold"
 ; MEMPROFRAND2: !"cold"
-; MEMPROFRAND2: !"hot"
-; MEMPROFRAND2: !"hot"
+; MEMPROFRAND2: !"notcold"
 
 ; MEMPROFSTATS:  8 memprof - Number of alloc contexts in memory profile.
 ; MEMPROFSTATS: 10 memprof - Number of callsites in memory profile.
diff --git a/llvm/test/Transforms/PGOProfile/memprof_loop_unroll.ll b/llvm/test/Transforms/PGOProfile/memprof_loop_unroll.ll
index 9bc1282ab4529..2461ca32e9821 100644
--- a/llvm/test/Transforms/PGOProfile/memprof_loop_unroll.ll
+++ b/llvm/test/Transforms/PGOProfile/memprof_loop_unroll.ll
@@ -10,7 +10,9 @@
 ;; $ clang++ -gmlt -fdebug-info-for-profiling -S %S/Inputs/memprof_loop_unroll_b.cc -emit-llvm
 
 ; RUN: llvm-profdata merge %S/Inputs/memprof_loop_unroll.memprofraw --profiled-binary %S/Inputs/memprof_loop_unroll.exe -o %t.memprofdata
-; RUN: opt < %s -passes='memprof-use<profile-filename=%t.memprofdata>' -S -memprof-report-hinted-sizes 2>&1 | FileCheck %s
+;; Set the minimum lifetime threshold to 0 to ensure that one context is
+;; considered cold (the other will be notcold).
+; RUN: opt < %s -passes='memprof-use<profile-filename=%t.memprofdata>' -S -memprof-report-hinted-sizes -memprof-ave-lifetime-cold-threshold=0 2>&1 | FileCheck %s
 
 ;; Conservatively annotate as not cold. We get two messages as there are two
 ;; unrolled copies of the allocation.
diff --git a/llvm/test/Transforms/PGOProfile/memprof_match_hot_cold_new_calls.ll b/llvm/test/Transforms/PGOProfile/memprof_match_hot_cold_new_calls.ll
index 83753ed6376d3..4aa0511622662 100644
--- a/llvm/test/Transforms/PGOProfile/memprof_match_hot_cold_new_calls.ll
+++ b/llvm/test/Transforms/PGOProfile/memprof_match_hot_cold_new_calls.ll
@@ -105,7 +105,7 @@ entry:
 
 declare noundef ptr @_ZnwmSt11align_val_tRKSt9nothrow_t12__hot_cold_t(i64 noundef, i64 noundef, ptr noundef nonnull align 1 dereferenceable(1), i8 noundef zeroext)
 
-; MEMPROF: ![[M1]] = !{![[MIB1:[0-9]+]], ![[MIB2:[0-9]+]], ![[MIB3:[0-9]+]], ![[MIB4:[0-9]+]], ![[MIB5:[0-9]+]]}
+; MEMPROF: ![[M1]] = !{![[MIB1:[0-9]+]], ![[MIB2:[0-9]+]], ![[MIB3:[0-9]+]], ![[MIB4:[0-9]+]]}
 ; MEMPROF: ![[MIB1]] = !{![[STACK1:[0-9]+]], !"cold"}
 ; MEMPROF: ![[STACK1]] = !{i64 2732490490862098848, i64 748269490701775343}
 ; MEMPROF: ![[MIB2]] = !{![[STACK2:[0-9]+]], !"cold"}
@@ -114,8 +114,6 @@ declare noundef ptr @_ZnwmSt11align_val_tRKSt9nothrow_t12__hot_cold_t(i64 nounde
 ; MEMPROF: ![[STACK3]] = !{i64 2732490490862098848, i64 2104812325165620841, i64 6281715513834610934, i64 6281715513834610934, i64 6281715513834610934, i64 6281715513834610934}
 ; MEMPROF: ![[MIB4]] = !{![[STACK4:[0-9]+]], !"cold"}
 ; MEMPROF: ![[STACK4]] = !{i64 2732490490862098848, i64 8467819354083268568}
-; MEMPROF: ![[MIB5]] = !{![[STACK5:[0-9]+]], !"notcold"}
-; MEMPROF: ![[STACK5]] = !{i64 2732490490862098848, i64 8690657650969109624}
 ; MEMPROF: ![[C1]] = !{i64 2732490490862098848}
 
 !llvm.dbg.cu = !{!0}
diff --git a/llvm/test/Transforms/PhaseOrdering/AArch64/block_scaling_decompr_8bit.ll b/llvm/test/Transforms/PhaseOrdering/AArch64/block_scaling_decompr_8bit.ll
index 9f3e09d7420d1..e01dba328a3a1 100644
--- a/llvm/test/Transforms/PhaseOrdering/AArch64/block_scaling_decompr_8bit.ll
+++ b/llvm/test/Transforms/PhaseOrdering/AArch64/block_scaling_decompr_8bit.ll
@@ -9,7 +9,7 @@ target triple = "aarch64"
 
 define dso_local noundef i32 @_Z33block_scaling_decompr_8bitjPK27compressed_data_8bitP20cmplx_int16_tPKS2_(i32 noundef %n_prb, ptr noundef %src, ptr noundef %dst, ptr noundef %scale) #0 {
 ; CHECK-LABEL: define dso_local noundef i32 @_Z33block_scaling_decompr_8bitjPK27compressed_data_8bitP20cmplx_int16_tPKS2_(
-; CHECK-SAME: i32 noundef [[N_PRB:%.*]], ptr nocapture noundef readonly [[SRC:%.*]], ptr nocapture noundef writeonly [[DST:%.*]], ptr noundef readonly [[SCALE:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
+; CHECK-SAME: i32 noundef [[N_PRB:%.*]], ptr noundef readonly captures(none) [[SRC:%.*]], ptr noundef writeonly captures(none) [[DST:%.*]], ptr noundef readonly [[SCALE:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
 ; CHECK-NEXT:  [[ENTRY:.*:]]
 ; CHECK-NEXT:    [[CMP47_NOT:%.*]] = icmp eq i32 [[N_PRB]], 0
 ; CHECK-NEXT:    br i1 [[CMP47_NOT]], label %[[FOR_END:.*]], label %[[FOR_BODY_LR_PH:.*]]
diff --git a/llvm/test/Transforms/PhaseOrdering/AArch64/constraint-elimination-placement.ll b/llvm/test/Transforms/PhaseOrdering/AArch64/constraint-elimination-placement.ll
index 04bed80b55de2..bbdbd95c6017a 100644
--- a/llvm/test/Transforms/PhaseOrdering/AArch64/constraint-elimination-placement.ll
+++ b/llvm/test/Transforms/PhaseOrdering/AArch64/constraint-elimination-placement.ll
@@ -6,7 +6,7 @@ target triple = "arm64-apple-macosx"
 
 define i1 @test_order_1(ptr %this, ptr noalias %other, i1 %tobool9.not, i32 %call) {
 ; CHECK-LABEL: define noundef i1 @test_order_1(
-; CHECK-SAME: ptr nocapture writeonly [[THIS:%.*]], ptr noalias [[OTHER:%.*]], i1 [[TOBOOL9_NOT:%.*]], i32 [[CALL:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
+; CHECK-SAME: ptr writeonly captures(none) [[THIS:%.*]], ptr noalias [[OTHER:%.*]], i1 [[TOBOOL9_NOT:%.*]], i32 [[CALL:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    br i1 [[TOBOOL9_NOT]], label [[EXIT:%.*]], label [[FOR_COND_PREHEADER:%.*]]
 ; CHECK:       for.cond.preheader:
@@ -94,7 +94,7 @@ declare i64 @strlen(ptr)
 
 define void @test2(ptr %this) #0 {
 ; CHECK-LABEL: define void @test2(
-; CHECK-SAME: ptr nocapture writeonly [[THIS:%.*]]) local_unnamed_addr #[[ATTR2:[0-9]+]] {
+; CHECK-SAME: ptr writeonly captures(none) [[THIS:%.*]]) local_unnamed_addr #[[ATTR2:[0-9]+]] {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[CALL1_I_I:%.*]] = tail call i1 @test2_fn4(i8 undef)
 ; CHECK-NEXT:    [[CALL2_I_I:%.*]] = load i64, ptr inttoptr (i64 8 to ptr), align 8
@@ -153,7 +153,7 @@ if.else21:                                        ; preds = %entry
 
 define i1 @test2_fn2(ptr %__rhs) #0 {
 ; CHECK-LABEL: define noundef i1 @test2_fn2(
-; CHECK-SAME: ptr nocapture readonly [[__RHS:%.*]]) local_unnamed_addr #[[ATTR3:[0-9]+]] {
+; CHECK-SAME: ptr readonly captures(none) [[__RHS:%.*]]) local_unnamed_addr #[[ATTR3:[0-9]+]] {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[CALL:%.*]] = tail call i64 @strlen(ptr noundef nonnull dereferenceable(1) [[__RHS]])
 ; CHECK-NEXT:    [[CALL1_I:%.*]] = tail call i1 @test2_fn4(i8 undef)
diff --git a/llvm/test/Transforms/PhaseOrdering/AArch64/extra-unroll-simplifications.ll b/llvm/test/Transforms/PhaseOrdering/AArch64/extra-unroll-simplifications.ll
index 13ea35a87c312..d8fc42bfaaec2 100644
--- a/llvm/test/Transforms/PhaseOrdering/AArch64/extra-unroll-simplifications.ll
+++ b/llvm/test/Transforms/PhaseOrdering/AArch64/extra-unroll-simplifications.ll
@@ -6,7 +6,7 @@ target triple = "arm64-apple-macosx11.0.0"
 
 define void @partial_unroll_forced(i32 %N, ptr %src, ptr noalias %dst) {
 ; CHECK-LABEL: define void @partial_unroll_forced(
-; CHECK-SAME: i32 [[N:%.*]], ptr nocapture readonly [[SRC:%.*]], ptr noalias nocapture writeonly [[DST:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
+; CHECK-SAME: i32 [[N:%.*]], ptr readonly captures(none) [[SRC:%.*]], ptr noalias writeonly captures(none) [[DST:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[CMP141:%.*]] = icmp sgt i32 [[N]], 0
 ; CHECK-NEXT:    br i1 [[CMP141]], label [[LOOP_LATCH_PREHEADER:%.*]], label [[EXIT:%.*]]
@@ -74,7 +74,7 @@ exit:
 
 define void @cse_matching_load_from_previous_unrolled_iteration(i32 %N, ptr %src, ptr noalias %dst) {
 ; CHECK-LABEL: define void @cse_matching_load_from_previous_unrolled_iteration(
-; CHECK-SAME: i32 [[N:%.*]], ptr nocapture readonly [[SRC:%.*]], ptr noalias nocapture writeonly [[DST:%.*]]) local_unnamed_addr #[[ATTR0]] {
+; CHECK-SAME: i32 [[N:%.*]], ptr readonly captures(none) [[SRC:%.*]], ptr noalias writeonly captures(none) [[DST:%.*]]) local_unnamed_addr #[[ATTR0]] {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[SRC_4:%.*]] = getelementptr i8, ptr [[SRC]], i64 4
 ; CHECK-NEXT:    [[SRC_12:%.*]] = getelementptr i8, ptr [[SRC]], i64 12
diff --git a/llvm/test/Transforms/PhaseOrdering/AArch64/hoist-runtime-checks.ll b/llvm/test/Transforms/PhaseOrdering/AArch64/hoist-runtime-checks.ll
index a38413f26ec97..b4b12da3244b2 100644
--- a/llvm/test/Transforms/PhaseOrdering/AArch64/hoist-runtime-checks.ll
+++ b/llvm/test/Transforms/PhaseOrdering/AArch64/hoist-runtime-checks.ll
@@ -6,7 +6,7 @@ target triple = "arm64-apple-macosx11.0.0"
 
 define i32 @read_only_loop_with_runtime_check(ptr noundef %array, i32 noundef %count, i32 noundef %n) {
 ; CHECK-LABEL: define i32 @read_only_loop_with_runtime_check(
-; CHECK-SAME: ptr nocapture noundef readonly [[ARRAY:%.*]], i32 noundef [[COUNT:%.*]], i32 noundef [[N:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
+; CHECK-SAME: ptr noundef readonly captures(none) [[ARRAY:%.*]], i32 noundef [[COUNT:%.*]], i32 noundef [[N:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[CMP6_NOT:%.*]] = icmp eq i32 [[N]], 0
 ; CHECK-NEXT:    br i1 [[CMP6_NOT]], label [[FOR_COND_CLEANUP:%.*]], label [[FOR_BODY_PREHEADER:%.*]]
@@ -122,7 +122,7 @@ for.end:                                          ; preds = %for.cond.cleanup
 
 define dso_local noundef i32 @sum_prefix_with_sum(ptr %s.coerce0, i64 %s.coerce1, i64 noundef %n) {
 ; CHECK-LABEL: define dso_local noundef i32 @sum_prefix_with_sum(
-; CHECK-SAME: ptr nocapture readonly [[S_COERCE0:%.*]], i64 [[S_COERCE1:%.*]], i64 noundef [[N:%.*]]) local_unnamed_addr #[[ATTR0]] {
+; CHECK-SAME: ptr readonly captures(none) [[S_COERCE0:%.*]], i64 [[S_COERCE1:%.*]], i64 noundef [[N:%.*]]) local_unnamed_addr #[[ATTR0]] {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[CMP5_NOT:%.*]] = icmp eq i64 [[N]], 0
 ; CHECK-NEXT:    br i1 [[CMP5_NOT]], label [[FOR_COND_CLEANUP:%.*]], label [[FOR_BODY_PREHEADER:%.*]]
@@ -223,7 +223,7 @@ for.end:                                          ; preds = %for.cond.cleanup
 
 define hidden noundef nonnull align 4 dereferenceable(4) ptr @span_checked_access(ptr noundef nonnull align 8 dereferenceable(16) %this, i64 noundef %__idx) {
 ; CHECK-LABEL: define hidden noundef nonnull align 4 dereferenceable(4) ptr @span_checked_access(
-; CHECK-SAME: ptr nocapture noundef nonnull readonly align 8 dereferenceable(16) [[THIS:%.*]], i64 noundef [[__IDX:%.*]]) local_unnamed_addr #[[ATTR0]] {
+; CHECK-SAME: ptr noundef nonnull readonly align 8 captures(none) dereferenceable(16) [[THIS:%.*]], i64 noundef [[__IDX:%.*]]) local_unnamed_addr #[[ATTR0]] {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[__SIZE__I:%.*]] = getelementptr inbounds nuw i8, ptr [[THIS]], i64 8
 ; CHECK-NEXT:    [[TMP0:%.*]] = load i64, ptr [[__SIZE__I]], align 8
@@ -268,7 +268,7 @@ cond.end:                                         ; preds = %cond.false, %cond.t
 
 define hidden noundef i64 @span_access(ptr noundef nonnull align 8 dereferenceable(16) %this) {
 ; CHECK-LABEL: define hidden noundef i64 @span_access(
-; CHECK-SAME: ptr nocapture noundef nonnull readonly align 8 dereferenceable(16) [[THIS:%.*]]) local_unnamed_addr #[[ATTR1:[0-9]+]] {
+; CHECK-SAME: ptr noundef nonnull readonly align 8 captures(none) dereferenceable(16) [[THIS:%.*]]) local_unnamed_addr #[[ATTR1:[0-9]+]] {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[__SIZE_:%.*]] = getelementptr inbounds nuw i8, ptr [[THIS]], i64 8
 ; CHECK-NEXT:    [[TMP0:%.*]] = load i64, ptr [[__SIZE_]], align 8
diff --git a/llvm/test/Transforms/PhaseOrdering/AArch64/indvars-vectorization.ll b/llvm/test/Transforms/PhaseOrdering/AArch64/indvars-vectorization.ll
index 801a8a05d82ed..b056f44a6c469 100644
--- a/llvm/test/Transforms/PhaseOrdering/AArch64/indvars-vectorization.ll
+++ b/llvm/test/Transforms/PhaseOrdering/AArch64/indvars-vectorization.ll
@@ -6,7 +6,7 @@ target triple = "arm64-apple-macosx14.0.0"
 
 define void @s172(i32 noundef %xa, i32 noundef %xb, ptr noundef %a, ptr noundef %b) {
 ; CHECK-LABEL: define void @s172(
-; CHECK-SAME: i32 noundef [[XA:%.*]], i32 noundef [[XB:%.*]], ptr nocapture noundef [[A:%.*]], ptr nocapture noundef readonly [[B:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
+; CHECK-SAME: i32 noundef [[XA:%.*]], i32 noundef [[XB:%.*]], ptr noundef captures(none) [[A:%.*]], ptr noundef readonly captures(none) [[B:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[CMP1:%.*]] = icmp slt i32 [[XA]], 32001
 ; CHECK-NEXT:    br i1 [[CMP1]], label [[FOR_BODY_PREHEADER:%.*]], label [[FOR_END:%.*]]
@@ -76,7 +76,7 @@ define void @s172(i32 noundef %xa, i32 noundef %xb, ptr noundef %a, ptr noundef
 ; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[TMP8]], [[N_VEC]]
 ; CHECK-NEXT:    br i1 [[CMP_N]], label [[FOR_END]], label [[FOR_BODY_PREHEADER13]]
 ; CHECK:       for.body.preheader13:
-; CHECK-NEXT:    [[INDVARS_IV_PH:%.*]] = phi i64 [ [[TMP0]], [[FOR_BODY_PREHEADER]] ], [ [[TMP0]], [[VECTOR_MEMCHECK]] ], [ [[IND_END]], [[MIDDLE_BLOCK]] ]
+; CHECK-NEXT:    [[INDVARS_IV_PH:%.*]] = phi i64 [ [[TMP0]], [[VECTOR_MEMCHECK]] ], [ [[TMP0]], [[FOR_BODY_PREHEADER]] ], [ [[IND_END]], [[MIDDLE_BLOCK]] ]
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK:       for.body:
 ; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ], [ [[INDVARS_IV_PH]], [[FOR_BODY_PREHEADER13]] ]
diff --git a/llvm/test/Transforms/PhaseOrdering/AArch64/infer-align-from-assumption.ll b/llvm/test/Transforms/PhaseOrdering/AArch64/infer-align-from-assumption.ll
index 632e3a56aacac..889f25c79c10f 100644
--- a/llvm/test/Transforms/PhaseOrdering/AArch64/infer-align-from-assumption.ll
+++ b/llvm/test/Transforms/PhaseOrdering/AArch64/infer-align-from-assumption.ll
@@ -7,7 +7,7 @@ declare void @llvm.assume(i1 noundef)
 
 define i32 @earlycse_entry(ptr %p) {
 ; CHECK-LABEL: define i32 @earlycse_entry(
-; CHECK-SAME: ptr nocapture [[P:%.*]]) local_unnamed_addr {
+; CHECK-SAME: ptr captures(none) [[P:%.*]]) local_unnamed_addr {
 ; CHECK-NEXT:    [[L_I:%.*]] = load ptr, ptr [[P]], align 8
 ; CHECK-NEXT:    call void @llvm.assume(i1 true) [ "align"(ptr [[L_I]], i64 4) ]
 ; CHECK-NEXT:    [[L_ASSUME_ALIGNED_I_I:%.*]] = load i32, ptr [[L_I]], align 4
@@ -30,7 +30,7 @@ define i32 @earlycse_entry(ptr %p) {
 
 define i32 @earlycse_fn1(ptr %p) {
 ; CHECK-LABEL: define i32 @earlycse_fn1(
-; CHECK-SAME: ptr nocapture [[P:%.*]]) local_unnamed_addr {
+; CHECK-SAME: ptr captures(none) [[P:%.*]]) local_unnamed_addr {
 ; CHECK-NEXT:    [[L:%.*]] = load ptr, ptr [[P]], align 8
 ; CHECK-NEXT:    call void @llvm.assume(i1 true) [ "align"(ptr [[L]], i64 4) ]
 ; CHECK-NEXT:    [[L_ASSUME_ALIGNED_I:%.*]] = load i32, ptr [[L]], align 4
@@ -82,7 +82,7 @@ define void @sroa_align_entry(ptr %p) {
 
 define ptr @sroa_fn1(ptr %p) {
 ; CHECK-LABEL: define ptr @sroa_fn1(
-; CHECK-SAME: ptr nocapture readonly [[P:%.*]]) local_unnamed_addr #[[ATTR2:[0-9]+]] {
+; CHECK-SAME: ptr readonly captures(none) [[P:%.*]]) local_unnamed_addr #[[ATTR2:[0-9]+]] {
 ; CHECK-NEXT:    [[L:%.*]] = load ptr, ptr [[P]], align 8
 ; CHECK-NEXT:    call void @llvm.assume(i1 true) [ "align"(ptr [[L]], i64 8) ]
 ; CHECK-NEXT:    [[L_FN3_I_I:%.*]] = load i64, ptr [[L]], align 8
diff --git a/llvm/test/Transforms/PhaseOrdering/AArch64/matrix-extract-insert.ll b/llvm/test/Transforms/PhaseOrdering/AArch64/matrix-extract-insert.ll
index 886e7a758d053..089511d5b2e57 100644
--- a/llvm/test/Transforms/PhaseOrdering/AArch64/matrix-extract-insert.ll
+++ b/llvm/test/Transforms/PhaseOrdering/AArch64/matrix-extract-insert.ll
@@ -147,7 +147,7 @@ define void @matrix_extract_insert_loop(i32 %i, ptr nonnull align 8 dereferencea
 ; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N_VEC]], [[CONV6]]
 ; CHECK-NEXT:    br i1 [[CMP_N]], label [[VECTOR_MEMCHECK_1:%.*]], label [[FOR_BODY4_US_PREHEADER]]
 ; CHECK:       for.body4.us.preheader:
-; CHECK-NEXT:    [[INDVARS_IV_PH:%.*]] = phi i64 [ 0, [[VECTOR_MEMCHECK]] ], [ 0, [[FOR_COND1_PREHEADER_US_PREHEADER]] ], [ [[N_VEC]], [[MIDDLE_BLOCK]] ]
+; CHECK-NEXT:    [[INDVARS_IV_PH:%.*]] = phi i64 [ 0, [[FOR_COND1_PREHEADER_US_PREHEADER]] ], [ 0, [[VECTOR_MEMCHECK]] ], [ [[N_VEC]], [[MIDDLE_BLOCK]] ]
 ; CHECK-NEXT:    br label [[FOR_BODY4_US:%.*]]
 ; CHECK:       for.body4.us:
 ; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY4_US]] ], [ [[INDVARS_IV_PH]], [[FOR_BODY4_US_PREHEADER]] ]
@@ -223,7 +223,7 @@ define void @matrix_extract_insert_loop(i32 %i, ptr nonnull align 8 dereferencea
 ; CHECK-NEXT:    [[CMP_N_1:%.*]] = icmp eq i64 [[N_VEC_1]], [[CONV6]]
 ; CHECK-NEXT:    br i1 [[CMP_N_1]], label [[VECTOR_MEMCHECK_2:%.*]], label [[FOR_BODY4_US_PREHEADER_1]]
 ; CHECK:       for.body4.us.preheader.1:
-; CHECK-NEXT:    [[INDVARS_IV_PH_1:%.*]] = phi i64 [ 0, [[VECTOR_MEMCHECK_1]] ], [ 0, [[FOR_COND1_FOR_COND_CLEANUP3_CRIT_EDGE_US]] ], [ [[N_VEC_1]], [[MIDDLE_BLOCK_1]] ]
+; CHECK-NEXT:    [[INDVARS_IV_PH_1:%.*]] = phi i64 [ 0, [[FOR_COND1_FOR_COND_CLEANUP3_CRIT_EDGE_US]] ], [ 0, [[VECTOR_MEMCHECK_1]] ], [ [[N_VEC_1]], [[MIDDLE_BLOCK_1]] ]
 ; CHECK-NEXT:    br label [[FOR_BODY4_US_1:%.*]]
 ; CHECK:       for.body4.us.1:
 ; CHECK-NEXT:    [[INDVARS_IV_1:%.*]] = phi i64 [ [[INDVARS_IV_NEXT_1:%.*]], [[FOR_BODY4_US_1]] ], [ [[INDVARS_IV_PH_1]], [[FOR_BODY4_US_PREHEADER_1]] ]
@@ -300,7 +300,7 @@ define void @matrix_extract_insert_loop(i32 %i, ptr nonnull align 8 dereferencea
 ; CHECK-NEXT:    [[CMP_N_2:%.*]] = icmp eq i64 [[N_VEC_2]], [[CONV6]]
 ; CHECK-NEXT:    br i1 [[CMP_N_2]], label [[VECTOR_MEMCHECK_3:%.*]], label [[FOR_BODY4_US_PREHEADER_2]]
 ; CHECK:       for.body4.us.preheader.2:
-; CHECK-NEXT:    [[INDVARS_IV_PH_2:%.*]] = phi i64 [ 0, [[VECTOR_MEMCHECK_2]] ], [ 0, [[FOR_COND1_FOR_COND_CLEANUP3_CRIT_EDGE_US_1]] ], [ [[N_VEC_2]], [[MIDDLE_BLOCK_2]] ]
+; CHECK-NEXT:    [[INDVARS_IV_PH_2:%.*]] = phi i64 [ 0, [[FOR_COND1_FOR_COND_CLEANUP3_CRIT_EDGE_US_1]] ], [ 0, [[VECTOR_MEMCHECK_2]] ], [ [[N_VEC_2]], [[MIDDLE_BLOCK_2]] ]
 ; CHECK-NEXT:    br label [[FOR_BODY4_US_2:%.*]]
 ; CHECK:       for.body4.us.2:
 ; CHECK-NEXT:    [[INDVARS_IV_2:%.*]] = phi i64 [ [[INDVARS_IV_NEXT_2:%.*]], [[FOR_BODY4_US_2]] ], [ [[INDVARS_IV_PH_2]], [[FOR_BODY4_US_PREHEADER_2]] ]
@@ -377,7 +377,7 @@ define void @matrix_extract_insert_loop(i32 %i, ptr nonnull align 8 dereferencea
 ; CHECK-NEXT:    [[CMP_N_3:%.*]] = icmp eq i64 [[N_VEC_3]], [[CONV6]]
 ; CHECK-NEXT:    br i1 [[CMP_N_3]], label [[FOR_COND_CLEANUP]], label [[FOR_BODY4_US_PREHEADER_3]]
 ; CHECK:       for.body4.us.preheader.3:
-; CHECK-NEXT:    [[INDVARS_IV_PH_3:%.*]] = phi i64 [ 0, [[VECTOR_MEMCHECK_3]] ], [ 0, [[FOR_COND1_FOR_COND_CLEANUP3_CRIT_EDGE_US_2]] ], [ [[N_VEC_3]], [[MIDDLE_BLOCK_3]] ]
+; CHECK-NEXT:    [[INDVARS_IV_PH_3:%.*]] = phi i64 [ 0, [[FOR_COND1_FOR_COND_CLEANUP3_CRIT_EDGE_US_2]] ], [ 0, [[VECTOR_MEMCHECK_3]] ], [ [[N_VEC_3]], [[MIDDLE_BLOCK_3]] ]
 ; CHECK-NEXT:    br label [[FOR_BODY4_US_3:%.*]]
 ; CHECK:       for.body4.us.3:
 ; CHECK-NEXT:    [[INDVARS_IV_3:%.*]] = phi i64 [ [[INDVARS_IV_NEXT_3:%.*]], [[FOR_BODY4_US_3]] ], [ [[INDVARS_IV_PH_3]], [[FOR_BODY4_US_PREHEADER_3]] ]
diff --git a/llvm/test/Transforms/PhaseOrdering/AArch64/memcpy-constant-size.ll b/llvm/test/Transforms/PhaseOrdering/AArch64/memcpy-constant-size.ll
index d34063824fe79..315b38d621d6e 100644
--- a/llvm/test/Transforms/PhaseOrdering/AArch64/memcpy-constant-size.ll
+++ b/llvm/test/Transforms/PhaseOrdering/AArch64/memcpy-constant-size.ll
@@ -9,7 +9,7 @@ declare ptr @__memset_chk(ptr, i32, i64, i64)
 
 define void @callee_memcpy(ptr %dst, ptr %src, i64 %size) {
 ; CHECK-LABEL: define void @callee_memcpy
-; CHECK-SAME: (ptr [[DST:%.*]], ptr nocapture readonly [[SRC:%.*]], i64 [[SIZE:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
+; CHECK-SAME: (ptr [[DST:%.*]], ptr readonly captures(none) [[SRC:%.*]], i64 [[SIZE:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
 ; CHECK-NEXT:    tail call void @llvm.memcpy.p0.p0.i64(ptr align 1 [[DST]], ptr align 1 [[SRC]], i64 [[SIZE]], i1 false)
 ; CHECK-NEXT:    ret void
 ;
@@ -53,7 +53,7 @@ define void @callee_memset(ptr %dst, i64 %size) {
 
 define void @caller_memcpy(ptr %dst, ptr %src) {
 ; CHECK-LABEL: define void @caller_memcpy
-; CHECK-SAME: (ptr [[DST:%.*]], ptr nocapture readonly [[SRC:%.*]]) local_unnamed_addr #[[ATTR0]] {
+; CHECK-SAME: (ptr [[DST:%.*]], ptr readonly captures(none) [[SRC:%.*]]) local_unnamed_addr #[[ATTR0]] {
 ; CHECK-NEXT:    [[TMP1:%.*]] = load i32, ptr [[SRC]], align 1
 ; CHECK-NEXT:    store i32 [[TMP1]], ptr [[DST]], align 1
 ; CHECK-NEXT:    ret void
diff --git a/llvm/test/Transforms/PhaseOrdering/AArch64/predicated-reduction.ll b/llvm/test/Transforms/PhaseOrdering/AArch64/predicated-reduction.ll
index 260567723aad0..4e0f41d19948f 100644
--- a/llvm/test/Transforms/PhaseOrdering/AArch64/predicated-reduction.ll
+++ b/llvm/test/Transforms/PhaseOrdering/AArch64/predicated-reduction.ll
@@ -6,7 +6,7 @@ target triple = "aarch64"
 
 define nofpclass(nan inf) double @monte_simple(i32 noundef %nblocks, i32 noundef %RAND_BLOCK_LENGTH, ptr noundef %samples, double noundef nofpclass(nan inf) %Y, double noundef nofpclass(nan inf) %Z) {
 ; CHECK-LABEL: define nofpclass(nan inf) double @monte_simple(
-; CHECK-SAME: i32 noundef [[NBLOCKS:%.*]], i32 noundef [[RAND_BLOCK_LENGTH:%.*]], ptr nocapture noundef readonly [[SAMPLES:%.*]], double noundef nofpclass(nan inf) [[Y:%.*]], double noundef nofpclass(nan inf) [[Z:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
+; CHECK-SAME: i32 noundef [[NBLOCKS:%.*]], i32 noundef [[RAND_BLOCK_LENGTH:%.*]], ptr noundef readonly captures(none) [[SAMPLES:%.*]], double noundef nofpclass(nan inf) [[Y:%.*]], double noundef nofpclass(nan inf) [[Z:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
 ; CHECK-NEXT:  [[ENTRY:.*]]:
 ; CHECK-NEXT:    [[CMP8:%.*]] = icmp sgt i32 [[RAND_BLOCK_LENGTH]], 0
 ; CHECK-NEXT:    br i1 [[CMP8]], label %[[FOR_BODY_PREHEADER:.*]], label %[[FOR_END:.*]]
diff --git a/llvm/test/Transforms/PhaseOrdering/AArch64/quant_4x4.ll b/llvm/test/Transforms/PhaseOrdering/AArch64/quant_4x4.ll
index eaced404bba28..09a59de44c745 100644
--- a/llvm/test/Transforms/PhaseOrdering/AArch64/quant_4x4.ll
+++ b/llvm/test/Transforms/PhaseOrdering/AArch64/quant_4x4.ll
@@ -8,7 +8,7 @@ target triple = "aarch64"
 
 define i32 @quant_4x4(ptr noundef %dct, ptr noundef %mf, ptr noundef %bias) {
 ; CHECK-LABEL: define range(i32 0, 2) i32 @quant_4x4
-; CHECK-SAME: (ptr nocapture noundef [[DCT:%.*]], ptr nocapture noundef readonly [[MF:%.*]], ptr nocapture noundef readonly [[BIAS:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
+; CHECK-SAME: (ptr noundef captures(none) [[DCT:%.*]], ptr noundef readonly captures(none) [[MF:%.*]], ptr noundef readonly captures(none) [[BIAS:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[SCEVGEP:%.*]] = getelementptr i8, ptr [[DCT]], i64 32
 ; CHECK-NEXT:    [[SCEVGEP23:%.*]] = getelementptr i8, ptr [[BIAS]], i64 32
diff --git a/llvm/test/Transforms/PhaseOrdering/AArch64/sinking-vs-if-conversion.ll b/llvm/test/Transforms/PhaseOrdering/AArch64/sinking-vs-if-conversion.ll
index 600e30f6853e6..eda54d999a79f 100644
--- a/llvm/test/Transforms/PhaseOrdering/AArch64/sinking-vs-if-conversion.ll
+++ b/llvm/test/Transforms/PhaseOrdering/AArch64/sinking-vs-if-conversion.ll
@@ -10,7 +10,7 @@ target triple = "arm64-apple-macosx"
 
 define void @test_find_min(ptr noundef nonnull align 8 dereferenceable(24) %this) {
 ; CHECK-LABEL: define void @test_find_min(
-; CHECK-SAME: ptr nocapture noundef nonnull align 8 dereferenceable(24) [[THIS:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
+; CHECK-SAME: ptr noundef nonnull align 8 captures(none) dereferenceable(24) [[THIS:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[NUM_:%.*]] = getelementptr inbounds nuw i8, ptr [[THIS]], i64 16
 ; CHECK-NEXT:    [[TMP0:%.*]] = load i32, ptr [[NUM_]], align 8
@@ -138,7 +138,7 @@ for.end:                                          ; preds = %for.cond.cleanup
 
 define void @cond_select_loop(ptr noalias nocapture noundef readonly %a, ptr noalias nocapture noundef readonly %b, ptr noalias nocapture noundef writeonly %c) {
 ; CHECK-LABEL: define void @cond_select_loop(
-; CHECK-SAME: ptr noalias nocapture noundef readonly [[A:%.*]], ptr noalias nocapture noundef readonly [[B:%.*]], ptr noalias nocapture noundef writeonly [[C:%.*]]) local_unnamed_addr #[[ATTR1:[0-9]+]] {
+; CHECK-SAME: ptr noalias noundef readonly captures(none) [[A:%.*]], ptr noalias noundef readonly captures(none) [[B:%.*]], ptr noalias noundef writeonly captures(none) [[C:%.*]]) local_unnamed_addr #[[ATTR1:[0-9]+]] {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK:       for.body:
diff --git a/llvm/test/Transforms/PhaseOrdering/AArch64/slpordering.ll b/llvm/test/Transforms/PhaseOrdering/AArch64/slpordering.ll
index 1cf20cb1fd64d..a2019836098e8 100644
--- a/llvm/test/Transforms/PhaseOrdering/AArch64/slpordering.ll
+++ b/llvm/test/Transforms/PhaseOrdering/AArch64/slpordering.ll
@@ -12,7 +12,7 @@ target triple = "aarch64"
 ; Function Attrs: nounwind uwtable
 define i32 @slpordering(ptr noundef %p1, i32 noundef %ip1, ptr noundef %p2, i32 noundef %ip2) #0 {
 ; CHECK-LABEL: define range(i32 0, 65536) i32 @slpordering
-; CHECK-SAME: (ptr nocapture noundef readonly [[P1:%.*]], i32 noundef [[IP1:%.*]], ptr nocapture noundef readonly [[P2:%.*]], i32 noundef [[IP2:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
+; CHECK-SAME: (ptr noundef readonly captures(none) [[P1:%.*]], i32 noundef [[IP1:%.*]], ptr noundef readonly captures(none) [[P2:%.*]], i32 noundef [[IP2:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[IDX_EXT:%.*]] = sext i32 [[IP1]] to i64
 ; CHECK-NEXT:    [[IDX_EXT63:%.*]] = sext i32 [[IP2]] to i64
diff --git a/llvm/test/Transforms/PhaseOrdering/AArch64/sve-interleave-vectorization.ll b/llvm/test/Transforms/PhaseOrdering/AArch64/sve-interleave-vectorization.ll
new file mode 100644
index 0000000000000..362ec22600f92
--- /dev/null
+++ b/llvm/test/Transforms/PhaseOrdering/AArch64/sve-interleave-vectorization.ll
@@ -0,0 +1,135 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -passes=loop-vectorize,interleaved-access -mattr=+sve -S -o - %s | FileCheck %s
+
+target datalayout = "e-m:o-i64:64-i128:128-n32:64-S128"
+target triple = "aarch64"
+
+%struct.xyzt = type { i32, i32, i32, i32 }
+; for (int i = 0; i < 1024; ++i) {
+;   dst[i].x = a[i].x + b[i].x;
+;   dst[i].y = a[i].y - b[i].y;
+;   dst[i].z = a[i].z << b[i].z;
+;   dst[i].t = a[i].t >> b[i].t;
+; }
+
+define void @interleave_deinterleave(ptr noalias %dst, ptr %a, ptr %b) {
+; CHECK-LABEL: @interleave_deinterleave(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 4
+; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP1]]
+; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[FOR_BODY_PREHEADER:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK:       vector.ph:
+; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 4
+; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]]
+; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
+; CHECK-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 4
+; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK:       vector.body:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP6:%.*]] = add i64 [[INDEX]], 0
+; CHECK-NEXT:    [[TMP7:%.*]] = getelementptr inbounds [[STRUCT_XYZT:%.*]], ptr [[A:%.*]], i64 [[TMP6]]
+; CHECK-NEXT:    [[LDN:%.*]] = call { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.aarch64.sve.ld4.sret.nxv4i32(<vscale x 4 x i1> splat (i1 true), ptr [[TMP7]])
+; CHECK-NEXT:    [[TMP9:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[LDN]], 0
+; CHECK-NEXT:    [[TMP10:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[LDN]], 1
+; CHECK-NEXT:    [[TMP11:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[LDN]], 2
+; CHECK-NEXT:    [[TMP12:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[LDN]], 3
+; CHECK-NEXT:    [[TMP13:%.*]] = getelementptr inbounds [[STRUCT_XYZT]], ptr [[B:%.*]], i64 [[TMP6]]
+; CHECK-NEXT:    [[LDN9:%.*]] = call { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.aarch64.sve.ld4.sret.nxv4i32(<vscale x 4 x i1> splat (i1 true), ptr [[TMP13]])
+; CHECK-NEXT:    [[TMP16:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[LDN9]], 0
+; CHECK-NEXT:    [[TMP17:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[LDN9]], 1
+; CHECK-NEXT:    [[TMP18:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[LDN9]], 2
+; CHECK-NEXT:    [[TMP19:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[LDN9]], 3
+; CHECK-NEXT:    [[TMP20:%.*]] = add nsw <vscale x 4 x i32> [[TMP16]], [[TMP9]]
+; CHECK-NEXT:    [[TMP21:%.*]] = getelementptr inbounds [[STRUCT_XYZT]], ptr [[DST:%.*]], i64 [[TMP6]]
+; CHECK-NEXT:    [[TMP22:%.*]] = sub nsw <vscale x 4 x i32> [[TMP10]], [[TMP17]]
+; CHECK-NEXT:    [[TMP23:%.*]] = shl <vscale x 4 x i32> [[TMP11]], [[TMP18]]
+; CHECK-NEXT:    [[TMP24:%.*]] = ashr <vscale x 4 x i32> [[TMP12]], [[TMP19]]
+; CHECK-NEXT:    call void @llvm.aarch64.sve.st4.nxv4i32(<vscale x 4 x i32> [[TMP20]], <vscale x 4 x i32> [[TMP22]], <vscale x 4 x i32> [[TMP23]], <vscale x 4 x i32> [[TMP24]], <vscale x 4 x i1> splat (i1 true), ptr [[TMP21]])
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
+; CHECK-NEXT:    [[TMP25:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[TMP25]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK:       middle.block:
+; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 1024, [[N_VEC]]
+; CHECK-NEXT:    br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[FOR_BODY_PREHEADER]]
+; CHECK:       scalar.ph:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK:       for.body:
+; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[FOR_BODY_PREHEADER]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [[STRUCT_XYZT]], ptr [[A]], i64 [[INDVARS_IV]]
+; CHECK-NEXT:    [[TMP31:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
+; CHECK-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds [[STRUCT_XYZT]], ptr [[B]], i64 [[INDVARS_IV]]
+; CHECK-NEXT:    [[TMP32:%.*]] = load i32, ptr [[ARRAYIDX2]], align 4
+; CHECK-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP32]], [[TMP31]]
+; CHECK-NEXT:    [[ARRAYIDX5:%.*]] = getelementptr inbounds [[STRUCT_XYZT]], ptr [[DST]], i64 [[INDVARS_IV]]
+; CHECK-NEXT:    store i32 [[ADD]], ptr [[ARRAYIDX5]], align 4
+; CHECK-NEXT:    [[Y:%.*]] = getelementptr inbounds nuw i8, ptr [[ARRAYIDX]], i64 4
+; CHECK-NEXT:    [[TMP33:%.*]] = load i32, ptr [[Y]], align 4
+; CHECK-NEXT:    [[Y11:%.*]] = getelementptr inbounds nuw i8, ptr [[ARRAYIDX2]], i64 4
+; CHECK-NEXT:    [[TMP26:%.*]] = load i32, ptr [[Y11]], align 4
+; CHECK-NEXT:    [[SUB:%.*]] = sub nsw i32 [[TMP33]], [[TMP26]]
+; CHECK-NEXT:    [[Y14:%.*]] = getelementptr inbounds nuw i8, ptr [[ARRAYIDX5]], i64 4
+; CHECK-NEXT:    store i32 [[SUB]], ptr [[Y14]], align 4
+; CHECK-NEXT:    [[Z:%.*]] = getelementptr inbounds nuw i8, ptr [[ARRAYIDX]], i64 8
+; CHECK-NEXT:    [[TMP27:%.*]] = load i32, ptr [[Z]], align 4
+; CHECK-NEXT:    [[Z19:%.*]] = getelementptr inbounds nuw i8, ptr [[ARRAYIDX2]], i64 8
+; CHECK-NEXT:    [[TMP28:%.*]] = load i32, ptr [[Z19]], align 4
+; CHECK-NEXT:    [[SHL:%.*]] = shl i32 [[TMP27]], [[TMP28]]
+; CHECK-NEXT:    [[Z22:%.*]] = getelementptr inbounds nuw i8, ptr [[ARRAYIDX5]], i64 8
+; CHECK-NEXT:    store i32 [[SHL]], ptr [[Z22]], align 4
+; CHECK-NEXT:    [[T:%.*]] = getelementptr inbounds nuw i8, ptr [[ARRAYIDX]], i64 12
+; CHECK-NEXT:    [[TMP29:%.*]] = load i32, ptr [[T]], align 4
+; CHECK-NEXT:    [[T27:%.*]] = getelementptr inbounds nuw i8, ptr [[ARRAYIDX2]], i64 12
+; CHECK-NEXT:    [[TMP30:%.*]] = load i32, ptr [[T27]], align 4
+; CHECK-NEXT:    [[SHR:%.*]] = ashr i32 [[TMP29]], [[TMP30]]
+; CHECK-NEXT:    [[T30:%.*]] = getelementptr inbounds nuw i8, ptr [[ARRAYIDX5]], i64 12
+; CHECK-NEXT:    store i32 [[SHR]], ptr [[T30]], align 4
+; CHECK-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
+; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], 1024
+; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK:       for.end:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %for.body
+
+for.body:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+  %gep.a = getelementptr inbounds %struct.xyzt, ptr %a, i64 %iv
+  %a.0 = load i32, ptr %gep.a, align 4
+  %gep.b = getelementptr inbounds %struct.xyzt, ptr %b, i64 %iv
+  %b.0 = load i32, ptr %gep.b, align 4
+  %add = add nsw i32 %b.0, %a.0
+  %gep.dst = getelementptr inbounds %struct.xyzt, ptr %dst, i64 %iv
+  store i32 %add, ptr %gep.dst, align 4
+  %gep.a.1 = getelementptr inbounds nuw i8, ptr %gep.a, i64 4
+  %a.1 = load i32, ptr %gep.a.1, align 4
+  %gep.b.1 = getelementptr inbounds nuw i8, ptr %gep.b, i64 4
+  %b.1 = load i32, ptr %gep.b.1, align 4
+  %sub = sub nsw i32 %a.1, %b.1
+  %gep.dst.1 = getelementptr inbounds nuw i8, ptr %gep.dst, i64 4
+  store i32 %sub, ptr %gep.dst.1, align 4
+  %gep.a.2 = getelementptr inbounds nuw i8, ptr %gep.a, i64 8
+  %a.2 = load i32, ptr %gep.a.2, align 4
+  %gep.b.2 = getelementptr inbounds nuw i8, ptr %gep.b, i64 8
+  %b.2 = load i32, ptr %gep.b.2, align 4
+  %shl = shl i32 %a.2, %b.2
+  %gep.dst.2 = getelementptr inbounds nuw i8, ptr %gep.dst, i64 8
+  store i32 %shl, ptr %gep.dst.2, align 4
+  %gep.a.3 = getelementptr inbounds nuw i8, ptr %gep.a, i64 12
+  %a.3 = load i32, ptr %gep.a.3, align 4
+  %gep.b.3 = getelementptr inbounds nuw i8, ptr %gep.b, i64 12
+  %b.3 = load i32, ptr %gep.b.3, align 4
+  %shr = ashr i32 %a.3, %b.3
+  %gep.dst.3 = getelementptr inbounds nuw i8, ptr %gep.dst, i64 12
+  store i32 %shr, ptr %gep.dst.3, align 4
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond.not = icmp eq i64 %iv.next, 1024
+  br i1 %exitcond.not, label %for.end, label %for.body
+
+for.end:
+  ret void
+}
diff --git a/llvm/test/Transforms/PhaseOrdering/AArch64/udotabd.ll b/llvm/test/Transforms/PhaseOrdering/AArch64/udotabd.ll
new file mode 100644
index 0000000000000..3496520c232aa
--- /dev/null
+++ b/llvm/test/Transforms/PhaseOrdering/AArch64/udotabd.ll
@@ -0,0 +1,499 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -S -O3 < %s | FileCheck %s --check-prefixes=CHECK-O3
+; RUN: opt -S -passes="default<O3>,default<O3>" < %s | FileCheck %s --check-prefixes=CHECK-LTO
+
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128-Fn32"
+target triple = "aarch64"
+
+define dso_local i32 @test(ptr noundef %p1, i32 noundef %s_p1, ptr noundef %p2, i32 noundef %s_p2) #0 {
+; CHECK-O3-LABEL: define dso_local i32 @test(
+; CHECK-O3-SAME: ptr noundef readonly captures(none) [[P1:%.*]], i32 noundef [[S_P1:%.*]], ptr noundef readonly captures(none) [[P2:%.*]], i32 noundef [[S_P2:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
+; CHECK-O3-NEXT:  [[ENTRY:.*:]]
+; CHECK-O3-NEXT:    [[IDX_EXT8:%.*]] = sext i32 [[S_P2]] to i64
+; CHECK-O3-NEXT:    [[IDX_EXT:%.*]] = sext i32 [[S_P1]] to i64
+; CHECK-O3-NEXT:    [[TMP0:%.*]] = load <16 x i8>, ptr [[P1]], align 1, !tbaa [[TBAA0:![0-9]+]]
+; CHECK-O3-NEXT:    [[TMP1:%.*]] = zext <16 x i8> [[TMP0]] to <16 x i16>
+; CHECK-O3-NEXT:    [[TMP2:%.*]] = load <16 x i8>, ptr [[P2]], align 1, !tbaa [[TBAA0]]
+; CHECK-O3-NEXT:    [[TMP3:%.*]] = zext <16 x i8> [[TMP2]] to <16 x i16>
+; CHECK-O3-NEXT:    [[TMP4:%.*]] = sub nsw <16 x i16> [[TMP1]], [[TMP3]]
+; CHECK-O3-NEXT:    [[TMP5:%.*]] = tail call <16 x i16> @llvm.abs.v16i16(<16 x i16> [[TMP4]], i1 false)
+; CHECK-O3-NEXT:    [[TMP6:%.*]] = zext <16 x i16> [[TMP5]] to <16 x i32>
+; CHECK-O3-NEXT:    [[TMP7:%.*]] = tail call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> [[TMP6]])
+; CHECK-O3-NEXT:    [[ADD_PTR:%.*]] = getelementptr inbounds i8, ptr [[P1]], i64 [[IDX_EXT]]
+; CHECK-O3-NEXT:    [[ADD_PTR9:%.*]] = getelementptr inbounds i8, ptr [[P2]], i64 [[IDX_EXT8]]
+; CHECK-O3-NEXT:    [[TMP8:%.*]] = load <16 x i8>, ptr [[ADD_PTR]], align 1, !tbaa [[TBAA0]]
+; CHECK-O3-NEXT:    [[TMP9:%.*]] = zext <16 x i8> [[TMP8]] to <16 x i16>
+; CHECK-O3-NEXT:    [[TMP10:%.*]] = load <16 x i8>, ptr [[ADD_PTR9]], align 1, !tbaa [[TBAA0]]
+; CHECK-O3-NEXT:    [[TMP11:%.*]] = zext <16 x i8> [[TMP10]] to <16 x i16>
+; CHECK-O3-NEXT:    [[TMP12:%.*]] = sub nsw <16 x i16> [[TMP9]], [[TMP11]]
+; CHECK-O3-NEXT:    [[TMP13:%.*]] = tail call <16 x i16> @llvm.abs.v16i16(<16 x i16> [[TMP12]], i1 false)
+; CHECK-O3-NEXT:    [[TMP14:%.*]] = zext <16 x i16> [[TMP13]] to <16 x i32>
+; CHECK-O3-NEXT:    [[TMP15:%.*]] = tail call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> [[TMP14]])
+; CHECK-O3-NEXT:    [[OP_RDX_1:%.*]] = add i32 [[TMP15]], [[TMP7]]
+; CHECK-O3-NEXT:    [[ADD_PTR_1:%.*]] = getelementptr inbounds i8, ptr [[ADD_PTR]], i64 [[IDX_EXT]]
+; CHECK-O3-NEXT:    [[ADD_PTR9_1:%.*]] = getelementptr inbounds i8, ptr [[ADD_PTR9]], i64 [[IDX_EXT8]]
+; CHECK-O3-NEXT:    [[TMP16:%.*]] = load <16 x i8>, ptr [[ADD_PTR_1]], align 1, !tbaa [[TBAA0]]
+; CHECK-O3-NEXT:    [[TMP17:%.*]] = zext <16 x i8> [[TMP16]] to <16 x i16>
+; CHECK-O3-NEXT:    [[TMP18:%.*]] = load <16 x i8>, ptr [[ADD_PTR9_1]], align 1, !tbaa [[TBAA0]]
+; CHECK-O3-NEXT:    [[TMP19:%.*]] = zext <16 x i8> [[TMP18]] to <16 x i16>
+; CHECK-O3-NEXT:    [[TMP20:%.*]] = sub nsw <16 x i16> [[TMP17]], [[TMP19]]
+; CHECK-O3-NEXT:    [[TMP21:%.*]] = tail call <16 x i16> @llvm.abs.v16i16(<16 x i16> [[TMP20]], i1 false)
+; CHECK-O3-NEXT:    [[TMP22:%.*]] = zext <16 x i16> [[TMP21]] to <16 x i32>
+; CHECK-O3-NEXT:    [[TMP23:%.*]] = tail call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> [[TMP22]])
+; CHECK-O3-NEXT:    [[OP_RDX_2:%.*]] = add i32 [[TMP23]], [[OP_RDX_1]]
+; CHECK-O3-NEXT:    [[ADD_PTR_2:%.*]] = getelementptr inbounds i8, ptr [[ADD_PTR_1]], i64 [[IDX_EXT]]
+; CHECK-O3-NEXT:    [[ADD_PTR9_2:%.*]] = getelementptr inbounds i8, ptr [[ADD_PTR9_1]], i64 [[IDX_EXT8]]
+; CHECK-O3-NEXT:    [[TMP24:%.*]] = load <16 x i8>, ptr [[ADD_PTR_2]], align 1, !tbaa [[TBAA0]]
+; CHECK-O3-NEXT:    [[TMP25:%.*]] = zext <16 x i8> [[TMP24]] to <16 x i16>
+; CHECK-O3-NEXT:    [[TMP26:%.*]] = load <16 x i8>, ptr [[ADD_PTR9_2]], align 1, !tbaa [[TBAA0]]
+; CHECK-O3-NEXT:    [[TMP27:%.*]] = zext <16 x i8> [[TMP26]] to <16 x i16>
+; CHECK-O3-NEXT:    [[TMP28:%.*]] = sub nsw <16 x i16> [[TMP25]], [[TMP27]]
+; CHECK-O3-NEXT:    [[TMP29:%.*]] = tail call <16 x i16> @llvm.abs.v16i16(<16 x i16> [[TMP28]], i1 false)
+; CHECK-O3-NEXT:    [[TMP30:%.*]] = zext <16 x i16> [[TMP29]] to <16 x i32>
+; CHECK-O3-NEXT:    [[TMP31:%.*]] = tail call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> [[TMP30]])
+; CHECK-O3-NEXT:    [[OP_RDX_3:%.*]] = add i32 [[TMP31]], [[OP_RDX_2]]
+; CHECK-O3-NEXT:    [[ADD_PTR_3:%.*]] = getelementptr inbounds i8, ptr [[ADD_PTR_2]], i64 [[IDX_EXT]]
+; CHECK-O3-NEXT:    [[ADD_PTR9_3:%.*]] = getelementptr inbounds i8, ptr [[ADD_PTR9_2]], i64 [[IDX_EXT8]]
+; CHECK-O3-NEXT:    [[TMP32:%.*]] = load <16 x i8>, ptr [[ADD_PTR_3]], align 1, !tbaa [[TBAA0]]
+; CHECK-O3-NEXT:    [[TMP33:%.*]] = zext <16 x i8> [[TMP32]] to <16 x i16>
+; CHECK-O3-NEXT:    [[TMP34:%.*]] = load <16 x i8>, ptr [[ADD_PTR9_3]], align 1, !tbaa [[TBAA0]]
+; CHECK-O3-NEXT:    [[TMP35:%.*]] = zext <16 x i8> [[TMP34]] to <16 x i16>
+; CHECK-O3-NEXT:    [[TMP36:%.*]] = sub nsw <16 x i16> [[TMP33]], [[TMP35]]
+; CHECK-O3-NEXT:    [[TMP37:%.*]] = tail call <16 x i16> @llvm.abs.v16i16(<16 x i16> [[TMP36]], i1 false)
+; CHECK-O3-NEXT:    [[TMP38:%.*]] = zext <16 x i16> [[TMP37]] to <16 x i32>
+; CHECK-O3-NEXT:    [[TMP39:%.*]] = tail call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> [[TMP38]])
+; CHECK-O3-NEXT:    [[OP_RDX_4:%.*]] = add i32 [[TMP39]], [[OP_RDX_3]]
+; CHECK-O3-NEXT:    [[ADD_PTR_4:%.*]] = getelementptr inbounds i8, ptr [[ADD_PTR_3]], i64 [[IDX_EXT]]
+; CHECK-O3-NEXT:    [[ADD_PTR9_4:%.*]] = getelementptr inbounds i8, ptr [[ADD_PTR9_3]], i64 [[IDX_EXT8]]
+; CHECK-O3-NEXT:    [[TMP40:%.*]] = load <16 x i8>, ptr [[ADD_PTR_4]], align 1, !tbaa [[TBAA0]]
+; CHECK-O3-NEXT:    [[TMP41:%.*]] = zext <16 x i8> [[TMP40]] to <16 x i16>
+; CHECK-O3-NEXT:    [[TMP42:%.*]] = load <16 x i8>, ptr [[ADD_PTR9_4]], align 1, !tbaa [[TBAA0]]
+; CHECK-O3-NEXT:    [[TMP43:%.*]] = zext <16 x i8> [[TMP42]] to <16 x i16>
+; CHECK-O3-NEXT:    [[TMP44:%.*]] = sub nsw <16 x i16> [[TMP41]], [[TMP43]]
+; CHECK-O3-NEXT:    [[TMP45:%.*]] = tail call <16 x i16> @llvm.abs.v16i16(<16 x i16> [[TMP44]], i1 false)
+; CHECK-O3-NEXT:    [[TMP46:%.*]] = zext <16 x i16> [[TMP45]] to <16 x i32>
+; CHECK-O3-NEXT:    [[TMP47:%.*]] = tail call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> [[TMP46]])
+; CHECK-O3-NEXT:    [[OP_RDX_5:%.*]] = add i32 [[TMP47]], [[OP_RDX_4]]
+; CHECK-O3-NEXT:    [[ADD_PTR_5:%.*]] = getelementptr inbounds i8, ptr [[ADD_PTR_4]], i64 [[IDX_EXT]]
+; CHECK-O3-NEXT:    [[ADD_PTR9_5:%.*]] = getelementptr inbounds i8, ptr [[ADD_PTR9_4]], i64 [[IDX_EXT8]]
+; CHECK-O3-NEXT:    [[TMP48:%.*]] = load <16 x i8>, ptr [[ADD_PTR_5]], align 1, !tbaa [[TBAA0]]
+; CHECK-O3-NEXT:    [[TMP49:%.*]] = zext <16 x i8> [[TMP48]] to <16 x i16>
+; CHECK-O3-NEXT:    [[TMP50:%.*]] = load <16 x i8>, ptr [[ADD_PTR9_5]], align 1, !tbaa [[TBAA0]]
+; CHECK-O3-NEXT:    [[TMP51:%.*]] = zext <16 x i8> [[TMP50]] to <16 x i16>
+; CHECK-O3-NEXT:    [[TMP52:%.*]] = sub nsw <16 x i16> [[TMP49]], [[TMP51]]
+; CHECK-O3-NEXT:    [[TMP53:%.*]] = tail call <16 x i16> @llvm.abs.v16i16(<16 x i16> [[TMP52]], i1 false)
+; CHECK-O3-NEXT:    [[TMP54:%.*]] = zext <16 x i16> [[TMP53]] to <16 x i32>
+; CHECK-O3-NEXT:    [[TMP55:%.*]] = tail call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> [[TMP54]])
+; CHECK-O3-NEXT:    [[OP_RDX_6:%.*]] = add i32 [[TMP55]], [[OP_RDX_5]]
+; CHECK-O3-NEXT:    [[ADD_PTR_6:%.*]] = getelementptr inbounds i8, ptr [[ADD_PTR_5]], i64 [[IDX_EXT]]
+; CHECK-O3-NEXT:    [[ADD_PTR9_6:%.*]] = getelementptr inbounds i8, ptr [[ADD_PTR9_5]], i64 [[IDX_EXT8]]
+; CHECK-O3-NEXT:    [[TMP56:%.*]] = load <16 x i8>, ptr [[ADD_PTR_6]], align 1, !tbaa [[TBAA0]]
+; CHECK-O3-NEXT:    [[TMP57:%.*]] = zext <16 x i8> [[TMP56]] to <16 x i16>
+; CHECK-O3-NEXT:    [[TMP58:%.*]] = load <16 x i8>, ptr [[ADD_PTR9_6]], align 1, !tbaa [[TBAA0]]
+; CHECK-O3-NEXT:    [[TMP59:%.*]] = zext <16 x i8> [[TMP58]] to <16 x i16>
+; CHECK-O3-NEXT:    [[TMP60:%.*]] = sub nsw <16 x i16> [[TMP57]], [[TMP59]]
+; CHECK-O3-NEXT:    [[TMP61:%.*]] = tail call <16 x i16> @llvm.abs.v16i16(<16 x i16> [[TMP60]], i1 false)
+; CHECK-O3-NEXT:    [[TMP62:%.*]] = zext <16 x i16> [[TMP61]] to <16 x i32>
+; CHECK-O3-NEXT:    [[TMP63:%.*]] = tail call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> [[TMP62]])
+; CHECK-O3-NEXT:    [[OP_RDX_7:%.*]] = add i32 [[TMP63]], [[OP_RDX_6]]
+; CHECK-O3-NEXT:    [[ADD_PTR_7:%.*]] = getelementptr inbounds i8, ptr [[ADD_PTR_6]], i64 [[IDX_EXT]]
+; CHECK-O3-NEXT:    [[ADD_PTR9_7:%.*]] = getelementptr inbounds i8, ptr [[ADD_PTR9_6]], i64 [[IDX_EXT8]]
+; CHECK-O3-NEXT:    [[TMP64:%.*]] = load <16 x i8>, ptr [[ADD_PTR_7]], align 1, !tbaa [[TBAA0]]
+; CHECK-O3-NEXT:    [[TMP65:%.*]] = zext <16 x i8> [[TMP64]] to <16 x i16>
+; CHECK-O3-NEXT:    [[TMP66:%.*]] = load <16 x i8>, ptr [[ADD_PTR9_7]], align 1, !tbaa [[TBAA0]]
+; CHECK-O3-NEXT:    [[TMP67:%.*]] = zext <16 x i8> [[TMP66]] to <16 x i16>
+; CHECK-O3-NEXT:    [[TMP68:%.*]] = sub nsw <16 x i16> [[TMP65]], [[TMP67]]
+; CHECK-O3-NEXT:    [[TMP69:%.*]] = tail call <16 x i16> @llvm.abs.v16i16(<16 x i16> [[TMP68]], i1 false)
+; CHECK-O3-NEXT:    [[TMP70:%.*]] = zext <16 x i16> [[TMP69]] to <16 x i32>
+; CHECK-O3-NEXT:    [[TMP71:%.*]] = tail call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> [[TMP70]])
+; CHECK-O3-NEXT:    [[OP_RDX_8:%.*]] = add i32 [[TMP71]], [[OP_RDX_7]]
+; CHECK-O3-NEXT:    [[ADD_PTR_8:%.*]] = getelementptr inbounds i8, ptr [[ADD_PTR_7]], i64 [[IDX_EXT]]
+; CHECK-O3-NEXT:    [[ADD_PTR9_8:%.*]] = getelementptr inbounds i8, ptr [[ADD_PTR9_7]], i64 [[IDX_EXT8]]
+; CHECK-O3-NEXT:    [[TMP72:%.*]] = load <16 x i8>, ptr [[ADD_PTR_8]], align 1, !tbaa [[TBAA0]]
+; CHECK-O3-NEXT:    [[TMP73:%.*]] = zext <16 x i8> [[TMP72]] to <16 x i16>
+; CHECK-O3-NEXT:    [[TMP74:%.*]] = load <16 x i8>, ptr [[ADD_PTR9_8]], align 1, !tbaa [[TBAA0]]
+; CHECK-O3-NEXT:    [[TMP75:%.*]] = zext <16 x i8> [[TMP74]] to <16 x i16>
+; CHECK-O3-NEXT:    [[TMP76:%.*]] = sub nsw <16 x i16> [[TMP73]], [[TMP75]]
+; CHECK-O3-NEXT:    [[TMP77:%.*]] = tail call <16 x i16> @llvm.abs.v16i16(<16 x i16> [[TMP76]], i1 false)
+; CHECK-O3-NEXT:    [[TMP78:%.*]] = zext <16 x i16> [[TMP77]] to <16 x i32>
+; CHECK-O3-NEXT:    [[TMP79:%.*]] = tail call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> [[TMP78]])
+; CHECK-O3-NEXT:    [[OP_RDX_9:%.*]] = add i32 [[TMP79]], [[OP_RDX_8]]
+; CHECK-O3-NEXT:    [[ADD_PTR_9:%.*]] = getelementptr inbounds i8, ptr [[ADD_PTR_8]], i64 [[IDX_EXT]]
+; CHECK-O3-NEXT:    [[ADD_PTR9_9:%.*]] = getelementptr inbounds i8, ptr [[ADD_PTR9_8]], i64 [[IDX_EXT8]]
+; CHECK-O3-NEXT:    [[TMP80:%.*]] = load <16 x i8>, ptr [[ADD_PTR_9]], align 1, !tbaa [[TBAA0]]
+; CHECK-O3-NEXT:    [[TMP81:%.*]] = zext <16 x i8> [[TMP80]] to <16 x i16>
+; CHECK-O3-NEXT:    [[TMP82:%.*]] = load <16 x i8>, ptr [[ADD_PTR9_9]], align 1, !tbaa [[TBAA0]]
+; CHECK-O3-NEXT:    [[TMP83:%.*]] = zext <16 x i8> [[TMP82]] to <16 x i16>
+; CHECK-O3-NEXT:    [[TMP84:%.*]] = sub nsw <16 x i16> [[TMP81]], [[TMP83]]
+; CHECK-O3-NEXT:    [[TMP85:%.*]] = tail call <16 x i16> @llvm.abs.v16i16(<16 x i16> [[TMP84]], i1 false)
+; CHECK-O3-NEXT:    [[TMP86:%.*]] = zext <16 x i16> [[TMP85]] to <16 x i32>
+; CHECK-O3-NEXT:    [[TMP87:%.*]] = tail call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> [[TMP86]])
+; CHECK-O3-NEXT:    [[OP_RDX_10:%.*]] = add i32 [[TMP87]], [[OP_RDX_9]]
+; CHECK-O3-NEXT:    [[ADD_PTR_10:%.*]] = getelementptr inbounds i8, ptr [[ADD_PTR_9]], i64 [[IDX_EXT]]
+; CHECK-O3-NEXT:    [[ADD_PTR9_10:%.*]] = getelementptr inbounds i8, ptr [[ADD_PTR9_9]], i64 [[IDX_EXT8]]
+; CHECK-O3-NEXT:    [[TMP88:%.*]] = load <16 x i8>, ptr [[ADD_PTR_10]], align 1, !tbaa [[TBAA0]]
+; CHECK-O3-NEXT:    [[TMP89:%.*]] = zext <16 x i8> [[TMP88]] to <16 x i16>
+; CHECK-O3-NEXT:    [[TMP90:%.*]] = load <16 x i8>, ptr [[ADD_PTR9_10]], align 1, !tbaa [[TBAA0]]
+; CHECK-O3-NEXT:    [[TMP91:%.*]] = zext <16 x i8> [[TMP90]] to <16 x i16>
+; CHECK-O3-NEXT:    [[TMP92:%.*]] = sub nsw <16 x i16> [[TMP89]], [[TMP91]]
+; CHECK-O3-NEXT:    [[TMP93:%.*]] = tail call <16 x i16> @llvm.abs.v16i16(<16 x i16> [[TMP92]], i1 false)
+; CHECK-O3-NEXT:    [[TMP94:%.*]] = zext <16 x i16> [[TMP93]] to <16 x i32>
+; CHECK-O3-NEXT:    [[TMP95:%.*]] = tail call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> [[TMP94]])
+; CHECK-O3-NEXT:    [[OP_RDX_11:%.*]] = add i32 [[TMP95]], [[OP_RDX_10]]
+; CHECK-O3-NEXT:    [[ADD_PTR_11:%.*]] = getelementptr inbounds i8, ptr [[ADD_PTR_10]], i64 [[IDX_EXT]]
+; CHECK-O3-NEXT:    [[ADD_PTR9_11:%.*]] = getelementptr inbounds i8, ptr [[ADD_PTR9_10]], i64 [[IDX_EXT8]]
+; CHECK-O3-NEXT:    [[TMP96:%.*]] = load <16 x i8>, ptr [[ADD_PTR_11]], align 1, !tbaa [[TBAA0]]
+; CHECK-O3-NEXT:    [[TMP97:%.*]] = zext <16 x i8> [[TMP96]] to <16 x i16>
+; CHECK-O3-NEXT:    [[TMP98:%.*]] = load <16 x i8>, ptr [[ADD_PTR9_11]], align 1, !tbaa [[TBAA0]]
+; CHECK-O3-NEXT:    [[TMP99:%.*]] = zext <16 x i8> [[TMP98]] to <16 x i16>
+; CHECK-O3-NEXT:    [[TMP100:%.*]] = sub nsw <16 x i16> [[TMP97]], [[TMP99]]
+; CHECK-O3-NEXT:    [[TMP101:%.*]] = tail call <16 x i16> @llvm.abs.v16i16(<16 x i16> [[TMP100]], i1 false)
+; CHECK-O3-NEXT:    [[TMP102:%.*]] = zext <16 x i16> [[TMP101]] to <16 x i32>
+; CHECK-O3-NEXT:    [[TMP103:%.*]] = tail call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> [[TMP102]])
+; CHECK-O3-NEXT:    [[OP_RDX_12:%.*]] = add i32 [[TMP103]], [[OP_RDX_11]]
+; CHECK-O3-NEXT:    [[ADD_PTR_12:%.*]] = getelementptr inbounds i8, ptr [[ADD_PTR_11]], i64 [[IDX_EXT]]
+; CHECK-O3-NEXT:    [[ADD_PTR9_12:%.*]] = getelementptr inbounds i8, ptr [[ADD_PTR9_11]], i64 [[IDX_EXT8]]
+; CHECK-O3-NEXT:    [[TMP104:%.*]] = load <16 x i8>, ptr [[ADD_PTR_12]], align 1, !tbaa [[TBAA0]]
+; CHECK-O3-NEXT:    [[TMP105:%.*]] = zext <16 x i8> [[TMP104]] to <16 x i16>
+; CHECK-O3-NEXT:    [[TMP106:%.*]] = load <16 x i8>, ptr [[ADD_PTR9_12]], align 1, !tbaa [[TBAA0]]
+; CHECK-O3-NEXT:    [[TMP107:%.*]] = zext <16 x i8> [[TMP106]] to <16 x i16>
+; CHECK-O3-NEXT:    [[TMP108:%.*]] = sub nsw <16 x i16> [[TMP105]], [[TMP107]]
+; CHECK-O3-NEXT:    [[TMP109:%.*]] = tail call <16 x i16> @llvm.abs.v16i16(<16 x i16> [[TMP108]], i1 false)
+; CHECK-O3-NEXT:    [[TMP110:%.*]] = zext <16 x i16> [[TMP109]] to <16 x i32>
+; CHECK-O3-NEXT:    [[TMP111:%.*]] = tail call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> [[TMP110]])
+; CHECK-O3-NEXT:    [[OP_RDX_13:%.*]] = add i32 [[TMP111]], [[OP_RDX_12]]
+; CHECK-O3-NEXT:    [[ADD_PTR_13:%.*]] = getelementptr inbounds i8, ptr [[ADD_PTR_12]], i64 [[IDX_EXT]]
+; CHECK-O3-NEXT:    [[ADD_PTR9_13:%.*]] = getelementptr inbounds i8, ptr [[ADD_PTR9_12]], i64 [[IDX_EXT8]]
+; CHECK-O3-NEXT:    [[TMP112:%.*]] = load <16 x i8>, ptr [[ADD_PTR_13]], align 1, !tbaa [[TBAA0]]
+; CHECK-O3-NEXT:    [[TMP113:%.*]] = zext <16 x i8> [[TMP112]] to <16 x i16>
+; CHECK-O3-NEXT:    [[TMP114:%.*]] = load <16 x i8>, ptr [[ADD_PTR9_13]], align 1, !tbaa [[TBAA0]]
+; CHECK-O3-NEXT:    [[TMP115:%.*]] = zext <16 x i8> [[TMP114]] to <16 x i16>
+; CHECK-O3-NEXT:    [[TMP116:%.*]] = sub nsw <16 x i16> [[TMP113]], [[TMP115]]
+; CHECK-O3-NEXT:    [[TMP117:%.*]] = tail call <16 x i16> @llvm.abs.v16i16(<16 x i16> [[TMP116]], i1 false)
+; CHECK-O3-NEXT:    [[TMP118:%.*]] = zext <16 x i16> [[TMP117]] to <16 x i32>
+; CHECK-O3-NEXT:    [[TMP119:%.*]] = tail call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> [[TMP118]])
+; CHECK-O3-NEXT:    [[OP_RDX_14:%.*]] = add i32 [[TMP119]], [[OP_RDX_13]]
+; CHECK-O3-NEXT:    [[ADD_PTR_14:%.*]] = getelementptr inbounds i8, ptr [[ADD_PTR_13]], i64 [[IDX_EXT]]
+; CHECK-O3-NEXT:    [[ADD_PTR9_14:%.*]] = getelementptr inbounds i8, ptr [[ADD_PTR9_13]], i64 [[IDX_EXT8]]
+; CHECK-O3-NEXT:    [[TMP120:%.*]] = load <16 x i8>, ptr [[ADD_PTR_14]], align 1, !tbaa [[TBAA0]]
+; CHECK-O3-NEXT:    [[TMP121:%.*]] = zext <16 x i8> [[TMP120]] to <16 x i16>
+; CHECK-O3-NEXT:    [[TMP122:%.*]] = load <16 x i8>, ptr [[ADD_PTR9_14]], align 1, !tbaa [[TBAA0]]
+; CHECK-O3-NEXT:    [[TMP123:%.*]] = zext <16 x i8> [[TMP122]] to <16 x i16>
+; CHECK-O3-NEXT:    [[TMP124:%.*]] = sub nsw <16 x i16> [[TMP121]], [[TMP123]]
+; CHECK-O3-NEXT:    [[TMP125:%.*]] = tail call <16 x i16> @llvm.abs.v16i16(<16 x i16> [[TMP124]], i1 false)
+; CHECK-O3-NEXT:    [[TMP126:%.*]] = zext <16 x i16> [[TMP125]] to <16 x i32>
+; CHECK-O3-NEXT:    [[TMP127:%.*]] = tail call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> [[TMP126]])
+; CHECK-O3-NEXT:    [[OP_RDX_15:%.*]] = add i32 [[TMP127]], [[OP_RDX_14]]
+; CHECK-O3-NEXT:    ret i32 [[OP_RDX_15]]
+;
+; CHECK-LTO-LABEL: define dso_local i32 @test(
+; CHECK-LTO-SAME: ptr noundef readonly captures(none) [[P1:%.*]], i32 noundef [[S_P1:%.*]], ptr noundef readonly captures(none) [[P2:%.*]], i32 noundef [[S_P2:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
+; CHECK-LTO-NEXT:  [[ENTRY:.*:]]
+; CHECK-LTO-NEXT:    [[IDX_EXT8:%.*]] = sext i32 [[S_P2]] to i64
+; CHECK-LTO-NEXT:    [[IDX_EXT:%.*]] = sext i32 [[S_P1]] to i64
+; CHECK-LTO-NEXT:    [[TMP0:%.*]] = load <16 x i8>, ptr [[P1]], align 1, !tbaa [[TBAA0:![0-9]+]]
+; CHECK-LTO-NEXT:    [[TMP1:%.*]] = zext <16 x i8> [[TMP0]] to <16 x i16>
+; CHECK-LTO-NEXT:    [[TMP2:%.*]] = load <16 x i8>, ptr [[P2]], align 1, !tbaa [[TBAA0]]
+; CHECK-LTO-NEXT:    [[TMP3:%.*]] = zext <16 x i8> [[TMP2]] to <16 x i16>
+; CHECK-LTO-NEXT:    [[TMP4:%.*]] = sub nsw <16 x i16> [[TMP1]], [[TMP3]]
+; CHECK-LTO-NEXT:    [[TMP5:%.*]] = tail call <16 x i16> @llvm.abs.v16i16(<16 x i16> [[TMP4]], i1 true)
+; CHECK-LTO-NEXT:    [[TMP36:%.*]] = zext nneg <16 x i16> [[TMP5]] to <16 x i32>
+; CHECK-LTO-NEXT:    [[TMP44:%.*]] = tail call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> [[TMP36]])
+; CHECK-LTO-NEXT:    [[ADD_PTR:%.*]] = getelementptr inbounds i8, ptr [[P1]], i64 [[IDX_EXT]]
+; CHECK-LTO-NEXT:    [[ADD_PTR9:%.*]] = getelementptr inbounds i8, ptr [[P2]], i64 [[IDX_EXT8]]
+; CHECK-LTO-NEXT:    [[TMP6:%.*]] = load <16 x i8>, ptr [[ADD_PTR]], align 1, !tbaa [[TBAA0]]
+; CHECK-LTO-NEXT:    [[TMP7:%.*]] = zext <16 x i8> [[TMP6]] to <16 x i16>
+; CHECK-LTO-NEXT:    [[TMP8:%.*]] = load <16 x i8>, ptr [[ADD_PTR9]], align 1, !tbaa [[TBAA0]]
+; CHECK-LTO-NEXT:    [[TMP9:%.*]] = zext <16 x i8> [[TMP8]] to <16 x i16>
+; CHECK-LTO-NEXT:    [[TMP10:%.*]] = sub nsw <16 x i16> [[TMP7]], [[TMP9]]
+; CHECK-LTO-NEXT:    [[TMP11:%.*]] = tail call <16 x i16> @llvm.abs.v16i16(<16 x i16> [[TMP10]], i1 true)
+; CHECK-LTO-NEXT:    [[TMP52:%.*]] = zext nneg <16 x i16> [[TMP11]] to <16 x i32>
+; CHECK-LTO-NEXT:    [[TMP60:%.*]] = tail call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> [[TMP52]])
+; CHECK-LTO-NEXT:    [[OP_RDX_1:%.*]] = add i32 [[TMP60]], [[TMP44]]
+; CHECK-LTO-NEXT:    [[ADD_PTR_1:%.*]] = getelementptr inbounds i8, ptr [[ADD_PTR]], i64 [[IDX_EXT]]
+; CHECK-LTO-NEXT:    [[ADD_PTR9_1:%.*]] = getelementptr inbounds i8, ptr [[ADD_PTR9]], i64 [[IDX_EXT8]]
+; CHECK-LTO-NEXT:    [[TMP12:%.*]] = load <16 x i8>, ptr [[ADD_PTR_1]], align 1, !tbaa [[TBAA0]]
+; CHECK-LTO-NEXT:    [[TMP13:%.*]] = zext <16 x i8> [[TMP12]] to <16 x i16>
+; CHECK-LTO-NEXT:    [[TMP14:%.*]] = load <16 x i8>, ptr [[ADD_PTR9_1]], align 1, !tbaa [[TBAA0]]
+; CHECK-LTO-NEXT:    [[TMP15:%.*]] = zext <16 x i8> [[TMP14]] to <16 x i16>
+; CHECK-LTO-NEXT:    [[TMP16:%.*]] = sub nsw <16 x i16> [[TMP13]], [[TMP15]]
+; CHECK-LTO-NEXT:    [[TMP17:%.*]] = tail call <16 x i16> @llvm.abs.v16i16(<16 x i16> [[TMP16]], i1 true)
+; CHECK-LTO-NEXT:    [[TMP68:%.*]] = zext nneg <16 x i16> [[TMP17]] to <16 x i32>
+; CHECK-LTO-NEXT:    [[TMP76:%.*]] = tail call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> [[TMP68]])
+; CHECK-LTO-NEXT:    [[OP_RDX_2:%.*]] = add i32 [[OP_RDX_1]], [[TMP76]]
+; CHECK-LTO-NEXT:    [[ADD_PTR_2:%.*]] = getelementptr inbounds i8, ptr [[ADD_PTR_1]], i64 [[IDX_EXT]]
+; CHECK-LTO-NEXT:    [[ADD_PTR9_2:%.*]] = getelementptr inbounds i8, ptr [[ADD_PTR9_1]], i64 [[IDX_EXT8]]
+; CHECK-LTO-NEXT:    [[TMP18:%.*]] = load <16 x i8>, ptr [[ADD_PTR_2]], align 1, !tbaa [[TBAA0]]
+; CHECK-LTO-NEXT:    [[TMP19:%.*]] = zext <16 x i8> [[TMP18]] to <16 x i16>
+; CHECK-LTO-NEXT:    [[TMP20:%.*]] = load <16 x i8>, ptr [[ADD_PTR9_2]], align 1, !tbaa [[TBAA0]]
+; CHECK-LTO-NEXT:    [[TMP21:%.*]] = zext <16 x i8> [[TMP20]] to <16 x i16>
+; CHECK-LTO-NEXT:    [[TMP22:%.*]] = sub nsw <16 x i16> [[TMP19]], [[TMP21]]
+; CHECK-LTO-NEXT:    [[TMP23:%.*]] = tail call <16 x i16> @llvm.abs.v16i16(<16 x i16> [[TMP22]], i1 true)
+; CHECK-LTO-NEXT:    [[TMP84:%.*]] = zext nneg <16 x i16> [[TMP23]] to <16 x i32>
+; CHECK-LTO-NEXT:    [[TMP92:%.*]] = tail call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> [[TMP84]])
+; CHECK-LTO-NEXT:    [[OP_RDX_3:%.*]] = add i32 [[OP_RDX_2]], [[TMP92]]
+; CHECK-LTO-NEXT:    [[ADD_PTR_3:%.*]] = getelementptr inbounds i8, ptr [[ADD_PTR_2]], i64 [[IDX_EXT]]
+; CHECK-LTO-NEXT:    [[ADD_PTR9_3:%.*]] = getelementptr inbounds i8, ptr [[ADD_PTR9_2]], i64 [[IDX_EXT8]]
+; CHECK-LTO-NEXT:    [[TMP24:%.*]] = load <16 x i8>, ptr [[ADD_PTR_3]], align 1, !tbaa [[TBAA0]]
+; CHECK-LTO-NEXT:    [[TMP25:%.*]] = zext <16 x i8> [[TMP24]] to <16 x i16>
+; CHECK-LTO-NEXT:    [[TMP26:%.*]] = load <16 x i8>, ptr [[ADD_PTR9_3]], align 1, !tbaa [[TBAA0]]
+; CHECK-LTO-NEXT:    [[TMP27:%.*]] = zext <16 x i8> [[TMP26]] to <16 x i16>
+; CHECK-LTO-NEXT:    [[TMP28:%.*]] = sub nsw <16 x i16> [[TMP25]], [[TMP27]]
+; CHECK-LTO-NEXT:    [[TMP29:%.*]] = tail call <16 x i16> @llvm.abs.v16i16(<16 x i16> [[TMP28]], i1 true)
+; CHECK-LTO-NEXT:    [[TMP100:%.*]] = zext nneg <16 x i16> [[TMP29]] to <16 x i32>
+; CHECK-LTO-NEXT:    [[TMP108:%.*]] = tail call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> [[TMP100]])
+; CHECK-LTO-NEXT:    [[OP_RDX_4:%.*]] = add i32 [[OP_RDX_3]], [[TMP108]]
+; CHECK-LTO-NEXT:    [[ADD_PTR_4:%.*]] = getelementptr inbounds i8, ptr [[ADD_PTR_3]], i64 [[IDX_EXT]]
+; CHECK-LTO-NEXT:    [[ADD_PTR9_4:%.*]] = getelementptr inbounds i8, ptr [[ADD_PTR9_3]], i64 [[IDX_EXT8]]
+; CHECK-LTO-NEXT:    [[TMP30:%.*]] = load <16 x i8>, ptr [[ADD_PTR_4]], align 1, !tbaa [[TBAA0]]
+; CHECK-LTO-NEXT:    [[TMP31:%.*]] = zext <16 x i8> [[TMP30]] to <16 x i16>
+; CHECK-LTO-NEXT:    [[TMP32:%.*]] = load <16 x i8>, ptr [[ADD_PTR9_4]], align 1, !tbaa [[TBAA0]]
+; CHECK-LTO-NEXT:    [[TMP33:%.*]] = zext <16 x i8> [[TMP32]] to <16 x i16>
+; CHECK-LTO-NEXT:    [[TMP34:%.*]] = sub nsw <16 x i16> [[TMP31]], [[TMP33]]
+; CHECK-LTO-NEXT:    [[TMP35:%.*]] = tail call <16 x i16> @llvm.abs.v16i16(<16 x i16> [[TMP34]], i1 true)
+; CHECK-LTO-NEXT:    [[TMP116:%.*]] = zext nneg <16 x i16> [[TMP35]] to <16 x i32>
+; CHECK-LTO-NEXT:    [[TMP117:%.*]] = tail call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> [[TMP116]])
+; CHECK-LTO-NEXT:    [[OP_RDX_5:%.*]] = add i32 [[OP_RDX_4]], [[TMP117]]
+; CHECK-LTO-NEXT:    [[ADD_PTR_5:%.*]] = getelementptr inbounds i8, ptr [[ADD_PTR_4]], i64 [[IDX_EXT]]
+; CHECK-LTO-NEXT:    [[ADD_PTR9_5:%.*]] = getelementptr inbounds i8, ptr [[ADD_PTR9_4]], i64 [[IDX_EXT8]]
+; CHECK-LTO-NEXT:    [[TMP37:%.*]] = load <16 x i8>, ptr [[ADD_PTR_5]], align 1, !tbaa [[TBAA0]]
+; CHECK-LTO-NEXT:    [[TMP38:%.*]] = zext <16 x i8> [[TMP37]] to <16 x i16>
+; CHECK-LTO-NEXT:    [[TMP39:%.*]] = load <16 x i8>, ptr [[ADD_PTR9_5]], align 1, !tbaa [[TBAA0]]
+; CHECK-LTO-NEXT:    [[TMP40:%.*]] = zext <16 x i8> [[TMP39]] to <16 x i16>
+; CHECK-LTO-NEXT:    [[TMP41:%.*]] = sub nsw <16 x i16> [[TMP38]], [[TMP40]]
+; CHECK-LTO-NEXT:    [[TMP42:%.*]] = tail call <16 x i16> @llvm.abs.v16i16(<16 x i16> [[TMP41]], i1 true)
+; CHECK-LTO-NEXT:    [[TMP43:%.*]] = zext nneg <16 x i16> [[TMP42]] to <16 x i32>
+; CHECK-LTO-NEXT:    [[TMP118:%.*]] = tail call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> [[TMP43]])
+; CHECK-LTO-NEXT:    [[OP_RDX_6:%.*]] = add i32 [[OP_RDX_5]], [[TMP118]]
+; CHECK-LTO-NEXT:    [[ADD_PTR_6:%.*]] = getelementptr inbounds i8, ptr [[ADD_PTR_5]], i64 [[IDX_EXT]]
+; CHECK-LTO-NEXT:    [[ADD_PTR9_6:%.*]] = getelementptr inbounds i8, ptr [[ADD_PTR9_5]], i64 [[IDX_EXT8]]
+; CHECK-LTO-NEXT:    [[TMP45:%.*]] = load <16 x i8>, ptr [[ADD_PTR_6]], align 1, !tbaa [[TBAA0]]
+; CHECK-LTO-NEXT:    [[TMP46:%.*]] = zext <16 x i8> [[TMP45]] to <16 x i16>
+; CHECK-LTO-NEXT:    [[TMP47:%.*]] = load <16 x i8>, ptr [[ADD_PTR9_6]], align 1, !tbaa [[TBAA0]]
+; CHECK-LTO-NEXT:    [[TMP48:%.*]] = zext <16 x i8> [[TMP47]] to <16 x i16>
+; CHECK-LTO-NEXT:    [[TMP49:%.*]] = sub nsw <16 x i16> [[TMP46]], [[TMP48]]
+; CHECK-LTO-NEXT:    [[TMP50:%.*]] = tail call <16 x i16> @llvm.abs.v16i16(<16 x i16> [[TMP49]], i1 true)
+; CHECK-LTO-NEXT:    [[TMP51:%.*]] = zext nneg <16 x i16> [[TMP50]] to <16 x i32>
+; CHECK-LTO-NEXT:    [[TMP120:%.*]] = tail call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> [[TMP51]])
+; CHECK-LTO-NEXT:    [[OP_RDX_7:%.*]] = add i32 [[OP_RDX_6]], [[TMP120]]
+; CHECK-LTO-NEXT:    [[ADD_PTR_7:%.*]] = getelementptr inbounds i8, ptr [[ADD_PTR_6]], i64 [[IDX_EXT]]
+; CHECK-LTO-NEXT:    [[ADD_PTR9_7:%.*]] = getelementptr inbounds i8, ptr [[ADD_PTR9_6]], i64 [[IDX_EXT8]]
+; CHECK-LTO-NEXT:    [[TMP53:%.*]] = load <16 x i8>, ptr [[ADD_PTR_7]], align 1, !tbaa [[TBAA0]]
+; CHECK-LTO-NEXT:    [[TMP54:%.*]] = zext <16 x i8> [[TMP53]] to <16 x i16>
+; CHECK-LTO-NEXT:    [[TMP55:%.*]] = load <16 x i8>, ptr [[ADD_PTR9_7]], align 1, !tbaa [[TBAA0]]
+; CHECK-LTO-NEXT:    [[TMP56:%.*]] = zext <16 x i8> [[TMP55]] to <16 x i16>
+; CHECK-LTO-NEXT:    [[TMP57:%.*]] = sub nsw <16 x i16> [[TMP54]], [[TMP56]]
+; CHECK-LTO-NEXT:    [[TMP58:%.*]] = tail call <16 x i16> @llvm.abs.v16i16(<16 x i16> [[TMP57]], i1 true)
+; CHECK-LTO-NEXT:    [[TMP59:%.*]] = zext nneg <16 x i16> [[TMP58]] to <16 x i32>
+; CHECK-LTO-NEXT:    [[TMP121:%.*]] = tail call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> [[TMP59]])
+; CHECK-LTO-NEXT:    [[OP_RDX_8:%.*]] = add i32 [[OP_RDX_7]], [[TMP121]]
+; CHECK-LTO-NEXT:    [[ADD_PTR_8:%.*]] = getelementptr inbounds i8, ptr [[ADD_PTR_7]], i64 [[IDX_EXT]]
+; CHECK-LTO-NEXT:    [[ADD_PTR9_8:%.*]] = getelementptr inbounds i8, ptr [[ADD_PTR9_7]], i64 [[IDX_EXT8]]
+; CHECK-LTO-NEXT:    [[TMP61:%.*]] = load <16 x i8>, ptr [[ADD_PTR_8]], align 1, !tbaa [[TBAA0]]
+; CHECK-LTO-NEXT:    [[TMP62:%.*]] = zext <16 x i8> [[TMP61]] to <16 x i16>
+; CHECK-LTO-NEXT:    [[TMP63:%.*]] = load <16 x i8>, ptr [[ADD_PTR9_8]], align 1, !tbaa [[TBAA0]]
+; CHECK-LTO-NEXT:    [[TMP64:%.*]] = zext <16 x i8> [[TMP63]] to <16 x i16>
+; CHECK-LTO-NEXT:    [[TMP65:%.*]] = sub nsw <16 x i16> [[TMP62]], [[TMP64]]
+; CHECK-LTO-NEXT:    [[TMP66:%.*]] = tail call <16 x i16> @llvm.abs.v16i16(<16 x i16> [[TMP65]], i1 true)
+; CHECK-LTO-NEXT:    [[TMP67:%.*]] = zext nneg <16 x i16> [[TMP66]] to <16 x i32>
+; CHECK-LTO-NEXT:    [[TMP122:%.*]] = tail call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> [[TMP67]])
+; CHECK-LTO-NEXT:    [[OP_RDX_9:%.*]] = add i32 [[OP_RDX_8]], [[TMP122]]
+; CHECK-LTO-NEXT:    [[ADD_PTR_9:%.*]] = getelementptr inbounds i8, ptr [[ADD_PTR_8]], i64 [[IDX_EXT]]
+; CHECK-LTO-NEXT:    [[ADD_PTR9_9:%.*]] = getelementptr inbounds i8, ptr [[ADD_PTR9_8]], i64 [[IDX_EXT8]]
+; CHECK-LTO-NEXT:    [[TMP69:%.*]] = load <16 x i8>, ptr [[ADD_PTR_9]], align 1, !tbaa [[TBAA0]]
+; CHECK-LTO-NEXT:    [[TMP70:%.*]] = zext <16 x i8> [[TMP69]] to <16 x i16>
+; CHECK-LTO-NEXT:    [[TMP71:%.*]] = load <16 x i8>, ptr [[ADD_PTR9_9]], align 1, !tbaa [[TBAA0]]
+; CHECK-LTO-NEXT:    [[TMP72:%.*]] = zext <16 x i8> [[TMP71]] to <16 x i16>
+; CHECK-LTO-NEXT:    [[TMP73:%.*]] = sub nsw <16 x i16> [[TMP70]], [[TMP72]]
+; CHECK-LTO-NEXT:    [[TMP74:%.*]] = tail call <16 x i16> @llvm.abs.v16i16(<16 x i16> [[TMP73]], i1 true)
+; CHECK-LTO-NEXT:    [[TMP75:%.*]] = zext nneg <16 x i16> [[TMP74]] to <16 x i32>
+; CHECK-LTO-NEXT:    [[TMP123:%.*]] = tail call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> [[TMP75]])
+; CHECK-LTO-NEXT:    [[OP_RDX_10:%.*]] = add i32 [[OP_RDX_9]], [[TMP123]]
+; CHECK-LTO-NEXT:    [[ADD_PTR_10:%.*]] = getelementptr inbounds i8, ptr [[ADD_PTR_9]], i64 [[IDX_EXT]]
+; CHECK-LTO-NEXT:    [[ADD_PTR9_10:%.*]] = getelementptr inbounds i8, ptr [[ADD_PTR9_9]], i64 [[IDX_EXT8]]
+; CHECK-LTO-NEXT:    [[TMP77:%.*]] = load <16 x i8>, ptr [[ADD_PTR_10]], align 1, !tbaa [[TBAA0]]
+; CHECK-LTO-NEXT:    [[TMP78:%.*]] = zext <16 x i8> [[TMP77]] to <16 x i16>
+; CHECK-LTO-NEXT:    [[TMP79:%.*]] = load <16 x i8>, ptr [[ADD_PTR9_10]], align 1, !tbaa [[TBAA0]]
+; CHECK-LTO-NEXT:    [[TMP80:%.*]] = zext <16 x i8> [[TMP79]] to <16 x i16>
+; CHECK-LTO-NEXT:    [[TMP81:%.*]] = sub nsw <16 x i16> [[TMP78]], [[TMP80]]
+; CHECK-LTO-NEXT:    [[TMP82:%.*]] = tail call <16 x i16> @llvm.abs.v16i16(<16 x i16> [[TMP81]], i1 true)
+; CHECK-LTO-NEXT:    [[TMP83:%.*]] = zext nneg <16 x i16> [[TMP82]] to <16 x i32>
+; CHECK-LTO-NEXT:    [[TMP124:%.*]] = tail call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> [[TMP83]])
+; CHECK-LTO-NEXT:    [[OP_RDX_11:%.*]] = add i32 [[OP_RDX_10]], [[TMP124]]
+; CHECK-LTO-NEXT:    [[ADD_PTR_11:%.*]] = getelementptr inbounds i8, ptr [[ADD_PTR_10]], i64 [[IDX_EXT]]
+; CHECK-LTO-NEXT:    [[ADD_PTR9_11:%.*]] = getelementptr inbounds i8, ptr [[ADD_PTR9_10]], i64 [[IDX_EXT8]]
+; CHECK-LTO-NEXT:    [[TMP85:%.*]] = load <16 x i8>, ptr [[ADD_PTR_11]], align 1, !tbaa [[TBAA0]]
+; CHECK-LTO-NEXT:    [[TMP86:%.*]] = zext <16 x i8> [[TMP85]] to <16 x i16>
+; CHECK-LTO-NEXT:    [[TMP87:%.*]] = load <16 x i8>, ptr [[ADD_PTR9_11]], align 1, !tbaa [[TBAA0]]
+; CHECK-LTO-NEXT:    [[TMP88:%.*]] = zext <16 x i8> [[TMP87]] to <16 x i16>
+; CHECK-LTO-NEXT:    [[TMP89:%.*]] = sub nsw <16 x i16> [[TMP86]], [[TMP88]]
+; CHECK-LTO-NEXT:    [[TMP90:%.*]] = tail call <16 x i16> @llvm.abs.v16i16(<16 x i16> [[TMP89]], i1 true)
+; CHECK-LTO-NEXT:    [[TMP91:%.*]] = zext nneg <16 x i16> [[TMP90]] to <16 x i32>
+; CHECK-LTO-NEXT:    [[TMP125:%.*]] = tail call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> [[TMP91]])
+; CHECK-LTO-NEXT:    [[OP_RDX_12:%.*]] = add i32 [[OP_RDX_11]], [[TMP125]]
+; CHECK-LTO-NEXT:    [[ADD_PTR_12:%.*]] = getelementptr inbounds i8, ptr [[ADD_PTR_11]], i64 [[IDX_EXT]]
+; CHECK-LTO-NEXT:    [[ADD_PTR9_12:%.*]] = getelementptr inbounds i8, ptr [[ADD_PTR9_11]], i64 [[IDX_EXT8]]
+; CHECK-LTO-NEXT:    [[TMP93:%.*]] = load <16 x i8>, ptr [[ADD_PTR_12]], align 1, !tbaa [[TBAA0]]
+; CHECK-LTO-NEXT:    [[TMP94:%.*]] = zext <16 x i8> [[TMP93]] to <16 x i16>
+; CHECK-LTO-NEXT:    [[TMP95:%.*]] = load <16 x i8>, ptr [[ADD_PTR9_12]], align 1, !tbaa [[TBAA0]]
+; CHECK-LTO-NEXT:    [[TMP96:%.*]] = zext <16 x i8> [[TMP95]] to <16 x i16>
+; CHECK-LTO-NEXT:    [[TMP97:%.*]] = sub nsw <16 x i16> [[TMP94]], [[TMP96]]
+; CHECK-LTO-NEXT:    [[TMP98:%.*]] = tail call <16 x i16> @llvm.abs.v16i16(<16 x i16> [[TMP97]], i1 true)
+; CHECK-LTO-NEXT:    [[TMP99:%.*]] = zext nneg <16 x i16> [[TMP98]] to <16 x i32>
+; CHECK-LTO-NEXT:    [[TMP126:%.*]] = tail call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> [[TMP99]])
+; CHECK-LTO-NEXT:    [[OP_RDX_13:%.*]] = add i32 [[OP_RDX_12]], [[TMP126]]
+; CHECK-LTO-NEXT:    [[ADD_PTR_13:%.*]] = getelementptr inbounds i8, ptr [[ADD_PTR_12]], i64 [[IDX_EXT]]
+; CHECK-LTO-NEXT:    [[ADD_PTR9_13:%.*]] = getelementptr inbounds i8, ptr [[ADD_PTR9_12]], i64 [[IDX_EXT8]]
+; CHECK-LTO-NEXT:    [[TMP101:%.*]] = load <16 x i8>, ptr [[ADD_PTR_13]], align 1, !tbaa [[TBAA0]]
+; CHECK-LTO-NEXT:    [[TMP102:%.*]] = zext <16 x i8> [[TMP101]] to <16 x i16>
+; CHECK-LTO-NEXT:    [[TMP103:%.*]] = load <16 x i8>, ptr [[ADD_PTR9_13]], align 1, !tbaa [[TBAA0]]
+; CHECK-LTO-NEXT:    [[TMP104:%.*]] = zext <16 x i8> [[TMP103]] to <16 x i16>
+; CHECK-LTO-NEXT:    [[TMP105:%.*]] = sub nsw <16 x i16> [[TMP102]], [[TMP104]]
+; CHECK-LTO-NEXT:    [[TMP106:%.*]] = tail call <16 x i16> @llvm.abs.v16i16(<16 x i16> [[TMP105]], i1 true)
+; CHECK-LTO-NEXT:    [[TMP107:%.*]] = zext nneg <16 x i16> [[TMP106]] to <16 x i32>
+; CHECK-LTO-NEXT:    [[TMP119:%.*]] = tail call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> [[TMP107]])
+; CHECK-LTO-NEXT:    [[OP_RDX_14:%.*]] = add i32 [[OP_RDX_13]], [[TMP119]]
+; CHECK-LTO-NEXT:    [[ADD_PTR_14:%.*]] = getelementptr inbounds i8, ptr [[ADD_PTR_13]], i64 [[IDX_EXT]]
+; CHECK-LTO-NEXT:    [[ADD_PTR9_14:%.*]] = getelementptr inbounds i8, ptr [[ADD_PTR9_13]], i64 [[IDX_EXT8]]
+; CHECK-LTO-NEXT:    [[TMP109:%.*]] = load <16 x i8>, ptr [[ADD_PTR_14]], align 1, !tbaa [[TBAA0]]
+; CHECK-LTO-NEXT:    [[TMP110:%.*]] = zext <16 x i8> [[TMP109]] to <16 x i16>
+; CHECK-LTO-NEXT:    [[TMP111:%.*]] = load <16 x i8>, ptr [[ADD_PTR9_14]], align 1, !tbaa [[TBAA0]]
+; CHECK-LTO-NEXT:    [[TMP112:%.*]] = zext <16 x i8> [[TMP111]] to <16 x i16>
+; CHECK-LTO-NEXT:    [[TMP113:%.*]] = sub nsw <16 x i16> [[TMP110]], [[TMP112]]
+; CHECK-LTO-NEXT:    [[TMP114:%.*]] = tail call <16 x i16> @llvm.abs.v16i16(<16 x i16> [[TMP113]], i1 true)
+; CHECK-LTO-NEXT:    [[TMP115:%.*]] = zext nneg <16 x i16> [[TMP114]] to <16 x i32>
+; CHECK-LTO-NEXT:    [[TMP127:%.*]] = tail call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> [[TMP115]])
+; CHECK-LTO-NEXT:    [[OP_RDX_15:%.*]] = add i32 [[OP_RDX_14]], [[TMP127]]
+; CHECK-LTO-NEXT:    ret i32 [[OP_RDX_15]]
+;
+entry:
+  %p1.addr = alloca ptr, align 8
+  %s_p1.addr = alloca i32, align 4
+  %p2.addr = alloca ptr, align 8
+  %s_p2.addr = alloca i32, align 4
+  %i_sum = alloca i32, align 4
+  %y = alloca i32, align 4
+  %cleanup.dest.slot = alloca i32, align 4
+  %x = alloca i32, align 4
+  store ptr %p1, ptr %p1.addr, align 8, !tbaa !4
+  store i32 %s_p1, ptr %s_p1.addr, align 4, !tbaa !9
+  store ptr %p2, ptr %p2.addr, align 8, !tbaa !4
+  store i32 %s_p2, ptr %s_p2.addr, align 4, !tbaa !9
+  call void @llvm.lifetime.start.p0(i64 4, ptr %i_sum) #3
+  store i32 0, ptr %i_sum, align 4, !tbaa !9
+  call void @llvm.lifetime.start.p0(i64 4, ptr %y) #3
+  store i32 0, ptr %y, align 4, !tbaa !9
+  br label %for.cond
+
+for.cond:                                         ; preds = %for.inc10, %entry
+  %0 = load i32, ptr %y, align 4, !tbaa !9
+  %cmp = icmp slt i32 %0, 16
+  br i1 %cmp, label %for.body, label %for.cond.cleanup
+
+for.cond.cleanup:                                 ; preds = %for.cond
+  store i32 2, ptr %cleanup.dest.slot, align 4
+  call void @llvm.lifetime.end.p0(i64 4, ptr %y) #3
+  br label %for.end12
+
+for.body:                                         ; preds = %for.cond
+  call void @llvm.lifetime.start.p0(i64 4, ptr %x) #3
+  store i32 0, ptr %x, align 4, !tbaa !9
+  br label %for.cond1
+
+for.cond1:                                        ; preds = %for.inc, %for.body
+  %1 = load i32, ptr %x, align 4, !tbaa !9
+  %cmp2 = icmp slt i32 %1, 16
+  br i1 %cmp2, label %for.body4, label %for.cond.cleanup3
+
+for.cond.cleanup3:                                ; preds = %for.cond1
+  store i32 5, ptr %cleanup.dest.slot, align 4
+  call void @llvm.lifetime.end.p0(i64 4, ptr %x) #3
+  br label %for.end
+
+for.body4:                                        ; preds = %for.cond1
+  %2 = load ptr, ptr %p1.addr, align 8, !tbaa !4
+  %3 = load i32, ptr %x, align 4, !tbaa !9
+  %idxprom = sext i32 %3 to i64
+  %arrayidx = getelementptr inbounds i8, ptr %2, i64 %idxprom
+  %4 = load i8, ptr %arrayidx, align 1, !tbaa !11
+  %conv = zext i8 %4 to i32
+  %5 = load ptr, ptr %p2.addr, align 8, !tbaa !4
+  %6 = load i32, ptr %x, align 4, !tbaa !9
+  %idxprom5 = sext i32 %6 to i64
+  %arrayidx6 = getelementptr inbounds i8, ptr %5, i64 %idxprom5
+  %7 = load i8, ptr %arrayidx6, align 1, !tbaa !11
+  %conv7 = zext i8 %7 to i32
+  %sub = sub nsw i32 %conv, %conv7
+  %8 = call i32 @llvm.abs.i32(i32 %sub, i1 true)
+  %9 = load i32, ptr %i_sum, align 4, !tbaa !9
+  %add = add nsw i32 %9, %8
+  store i32 %add, ptr %i_sum, align 4, !tbaa !9
+  br label %for.inc
+
+for.inc:                                          ; preds = %for.body4
+  %10 = load i32, ptr %x, align 4, !tbaa !9
+  %inc = add nsw i32 %10, 1
+  store i32 %inc, ptr %x, align 4, !tbaa !9
+  br label %for.cond1, !llvm.loop !12
+
+for.end:                                          ; preds = %for.cond.cleanup3
+  %11 = load i32, ptr %s_p1.addr, align 4, !tbaa !9
+  %12 = load ptr, ptr %p1.addr, align 8, !tbaa !4
+  %idx.ext = sext i32 %11 to i64
+  %add.ptr = getelementptr inbounds i8, ptr %12, i64 %idx.ext
+  store ptr %add.ptr, ptr %p1.addr, align 8, !tbaa !4
+  %13 = load i32, ptr %s_p2.addr, align 4, !tbaa !9
+  %14 = load ptr, ptr %p2.addr, align 8, !tbaa !4
+  %idx.ext8 = sext i32 %13 to i64
+  %add.ptr9 = getelementptr inbounds i8, ptr %14, i64 %idx.ext8
+  store ptr %add.ptr9, ptr %p2.addr, align 8, !tbaa !4
+  br label %for.inc10
+
+for.inc10:                                        ; preds = %for.end
+  %15 = load i32, ptr %y, align 4, !tbaa !9
+  %inc11 = add nsw i32 %15, 1
+  store i32 %inc11, ptr %y, align 4, !tbaa !9
+  br label %for.cond, !llvm.loop !14
+
+for.end12:                                        ; preds = %for.cond.cleanup
+  %16 = load i32, ptr %i_sum, align 4, !tbaa !9
+  store i32 1, ptr %cleanup.dest.slot, align 4
+  call void @llvm.lifetime.end.p0(i64 4, ptr %i_sum) #3
+  ret i32 %16
+}
+
+; Function Attrs: nocallback nofree nosync nounwind willreturn memory(argmem: readwrite)
+declare void @llvm.lifetime.start.p0(i64 immarg, ptr) #1
+
+; Function Attrs: nocallback nofree nosync nounwind speculatable willreturn memory(none)
+declare i32 @llvm.abs.i32(i32, i1 immarg) #2
+
+; Function Attrs: nocallback nofree nosync nounwind willreturn memory(argmem: readwrite)
+declare void @llvm.lifetime.end.p0(i64 immarg, ptr) #1
+
+attributes #0 = { nounwind uwtable vscale_range(1,16) "frame-pointer"="non-leaf" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="generic" "target-features"="+bf16,+bti,+ccidx,+complxnum,+crc,+dit,+dotprod,+ete,+flagm,+fp-armv8,+fp16fml,+fullfp16,+i8mm,+jsconv,+lse,+mte,+neon,+pauth,+perfmon,+predres,+rand,+ras,+rcpc,+rdm,+sb,+spe,+ssbs,+sve,+sve-bitperm,+sve2,+trbe,+v8.1a,+v8.2a,+v8.3a,+v8.4a,+v8.5a,+v8a,+v9a,-fmv" }
+attributes #1 = { nocallback nofree nosync nounwind willreturn memory(argmem: readwrite) }
+attributes #2 = { nocallback nofree nosync nounwind speculatable willreturn memory(none) }
+attributes #3 = { nounwind }
+
+
+!4 = !{!5, !5, i64 0}
+!5 = !{!"p1 omnipotent char", !6, i64 0}
+!6 = !{!"any pointer", !7, i64 0}
+!7 = !{!"omnipotent char", !8, i64 0}
+!8 = !{!"Simple C/C++ TBAA"}
+!9 = !{!10, !10, i64 0}
+!10 = !{!"int", !7, i64 0}
+!11 = !{!7, !7, i64 0}
+!12 = distinct !{!12, !13}
+!13 = !{!"llvm.loop.mustprogress"}
+!14 = distinct !{!14, !13}
+;.
+; CHECK-O3: [[TBAA0]] = !{[[META1:![0-9]+]], [[META1]], i64 0}
+; CHECK-O3: [[META1]] = !{!"omnipotent char", [[META2:![0-9]+]], i64 0}
+; CHECK-O3: [[META2]] = !{!"Simple C/C++ TBAA"}
+;.
+; CHECK-LTO: [[TBAA0]] = !{[[META1:![0-9]+]], [[META1]], i64 0}
+; CHECK-LTO: [[META1]] = !{!"omnipotent char", [[META2:![0-9]+]], i64 0}
+; CHECK-LTO: [[META2]] = !{!"Simple C/C++ TBAA"}
+;.
diff --git a/llvm/test/Transforms/PhaseOrdering/X86/addsub-inseltpoison.ll b/llvm/test/Transforms/PhaseOrdering/X86/addsub-inseltpoison.ll
index 84eba813b179d..a3af048c4e442 100644
--- a/llvm/test/Transforms/PhaseOrdering/X86/addsub-inseltpoison.ll
+++ b/llvm/test/Transforms/PhaseOrdering/X86/addsub-inseltpoison.ll
@@ -1,9 +1,12 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -O3 -S                                        | FileCheck %s
-; RUN: opt < %s -passes="default<O3>" -S | FileCheck %s
-
-target triple = "x86_64--"
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+; RUN: opt < %s -O3 -S -mtriple=x86_64-- -mcpu=x86-64    | FileCheck %s --check-prefixes=CHECK,SSE
+; RUN: opt < %s -O3 -S -mtriple=x86_64-- -mcpu=x86-64-v2 | FileCheck %s --check-prefixes=CHECK,SSE
+; RUN: opt < %s -O3 -S -mtriple=x86_64-- -mcpu=x86-64-v3 | FileCheck %s --check-prefixes=CHECK,AVX
+; RUN: opt < %s -O3 -S -mtriple=x86_64-- -mcpu=x86-64-v4 | FileCheck %s --check-prefixes=CHECK,AVX
+; RUN: opt < %s -passes="default<O3>" -S -mtriple=x86_64-- -mcpu=x86-64    | FileCheck %s --check-prefixes=CHECK,SSE
+; RUN: opt < %s -passes="default<O3>" -S -mtriple=x86_64-- -mcpu=x86-64-v2 | FileCheck %s --check-prefixes=CHECK,SSE
+; RUN: opt < %s -passes="default<O3>" -S -mtriple=x86_64-- -mcpu=x86-64-v3 | FileCheck %s --check-prefixes=CHECK,AVX
+; RUN: opt < %s -passes="default<O3>" -S -mtriple=x86_64-- -mcpu=x86-64-v4 | FileCheck %s --check-prefixes=CHECK,AVX
 
 ; Ideally, this should reach the backend with 1 fsub, 1 fadd, and 1 shuffle.
 ; That may require some coordination between VectorCombine, SLP, and other passes.
@@ -97,3 +100,44 @@ define void @add_aggregate_store(<2 x float> %a0, <2 x float> %a1, <2 x float> %
   store float %add10, ptr %r3, align 4
   ret void
 }
+
+; PR58139
+define <2 x double> @_mm_complexmult_pd_naive(<2 x double> %a, <2 x double> %b) {
+; SSE-LABEL: @_mm_complexmult_pd_naive(
+; SSE-NEXT:    [[B1:%.*]] = extractelement <2 x double> [[B:%.*]], i64 1
+; SSE-NEXT:    [[TMP1:%.*]] = fneg double [[B1]]
+; SSE-NEXT:    [[TMP2:%.*]] = shufflevector <2 x double> [[A:%.*]], <2 x double> poison, <2 x i32> <i32 1, i32 1>
+; SSE-NEXT:    [[TMP3:%.*]] = shufflevector <2 x double> [[B]], <2 x double> poison, <2 x i32> <i32 poison, i32 0>
+; SSE-NEXT:    [[TMP4:%.*]] = insertelement <2 x double> [[TMP3]], double [[TMP1]], i64 0
+; SSE-NEXT:    [[TMP5:%.*]] = fmul <2 x double> [[TMP2]], [[TMP4]]
+; SSE-NEXT:    [[TMP6:%.*]] = shufflevector <2 x double> [[A]], <2 x double> poison, <2 x i32> zeroinitializer
+; SSE-NEXT:    [[TMP7:%.*]] = tail call <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[TMP6]], <2 x double> [[B]], <2 x double> [[TMP5]])
+; SSE-NEXT:    ret <2 x double> [[TMP7]]
+;
+; AVX-LABEL: @_mm_complexmult_pd_naive(
+; AVX-NEXT:    [[A0:%.*]] = extractelement <2 x double> [[A:%.*]], i64 0
+; AVX-NEXT:    [[A1:%.*]] = extractelement <2 x double> [[A]], i64 1
+; AVX-NEXT:    [[B0:%.*]] = extractelement <2 x double> [[B:%.*]], i64 0
+; AVX-NEXT:    [[B1:%.*]] = extractelement <2 x double> [[B]], i64 1
+; AVX-NEXT:    [[MUL10:%.*]] = fmul double [[A1]], [[B0]]
+; AVX-NEXT:    [[TMP1:%.*]] = fneg double [[B1]]
+; AVX-NEXT:    [[NEG11:%.*]] = fmul double [[A1]], [[TMP1]]
+; AVX-NEXT:    [[MADD0:%.*]] = tail call double @llvm.fmuladd.f64(double [[A0]], double [[B0]], double [[NEG11]])
+; AVX-NEXT:    [[MADD1:%.*]] = tail call double @llvm.fmuladd.f64(double [[A0]], double [[B1]], double [[MUL10]])
+; AVX-NEXT:    [[RES0:%.*]] = insertelement <2 x double> poison, double [[MADD0]], i64 0
+; AVX-NEXT:    [[RES1:%.*]] = insertelement <2 x double> [[RES0]], double [[MADD1]], i64 1
+; AVX-NEXT:    ret <2 x double> [[RES1]]
+;
+  %a0 = extractelement <2 x double> %a, i32 0
+  %a1 = extractelement <2 x double> %a, i32 1
+  %b0 = extractelement <2 x double> %b, i32 0
+  %b1 = extractelement <2 x double> %b, i32 1
+  %mul10 = fmul double %a1, %b0
+  %mul11 = fmul double %a1, %b1
+  %neg11 = fneg double %mul11
+  %madd0 = call double @llvm.fmuladd.f64(double %a0, double %b0, double %neg11)
+  %madd1 = call double @llvm.fmuladd.f64(double %a0, double %b1, double %mul10)
+  %res0 = insertelement <2 x double> poison, double %madd0, i32 0
+  %res1 = insertelement <2 x double> %res0, double %madd1, i32 1
+  ret <2 x double> %res1
+}
diff --git a/llvm/test/Transforms/PhaseOrdering/X86/addsub.ll b/llvm/test/Transforms/PhaseOrdering/X86/addsub.ll
index 870ae4ec20e4c..40dc2aaeced57 100644
--- a/llvm/test/Transforms/PhaseOrdering/X86/addsub.ll
+++ b/llvm/test/Transforms/PhaseOrdering/X86/addsub.ll
@@ -1,9 +1,12 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -O3 -S                                        | FileCheck %s
-; RUN: opt < %s -passes="default<O3>" -S | FileCheck %s
-
-target triple = "x86_64--"
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+; RUN: opt < %s -O3 -S -mtriple=x86_64-- -mcpu=x86-64    | FileCheck %s --check-prefixes=CHECK,SSE
+; RUN: opt < %s -O3 -S -mtriple=x86_64-- -mcpu=x86-64-v2 | FileCheck %s --check-prefixes=CHECK,SSE
+; RUN: opt < %s -O3 -S -mtriple=x86_64-- -mcpu=x86-64-v3 | FileCheck %s --check-prefixes=CHECK,AVX
+; RUN: opt < %s -O3 -S -mtriple=x86_64-- -mcpu=x86-64-v4 | FileCheck %s --check-prefixes=CHECK,AVX
+; RUN: opt < %s -passes="default<O3>" -S -mtriple=x86_64-- -mcpu=x86-64    | FileCheck %s --check-prefixes=CHECK,SSE
+; RUN: opt < %s -passes="default<O3>" -S -mtriple=x86_64-- -mcpu=x86-64-v2 | FileCheck %s --check-prefixes=CHECK,SSE
+; RUN: opt < %s -passes="default<O3>" -S -mtriple=x86_64-- -mcpu=x86-64-v3 | FileCheck %s --check-prefixes=CHECK,AVX
+; RUN: opt < %s -passes="default<O3>" -S -mtriple=x86_64-- -mcpu=x86-64-v4 | FileCheck %s --check-prefixes=CHECK,AVX
 
 ; Ideally, this should reach the backend with 1 fsub, 1 fadd, and 1 shuffle.
 ; That may require some coordination between VectorCombine, SLP, and other passes.
@@ -97,3 +100,44 @@ define void @add_aggregate_store(<2 x float> %a0, <2 x float> %a1, <2 x float> %
   store float %add10, ptr %r3, align 4
   ret void
 }
+
+; PR58139
+define <2 x double> @_mm_complexmult_pd_naive(<2 x double> %a, <2 x double> %b) {
+; SSE-LABEL: @_mm_complexmult_pd_naive(
+; SSE-NEXT:    [[B1:%.*]] = extractelement <2 x double> [[B:%.*]], i64 1
+; SSE-NEXT:    [[TMP1:%.*]] = fneg double [[B1]]
+; SSE-NEXT:    [[TMP2:%.*]] = shufflevector <2 x double> [[A:%.*]], <2 x double> poison, <2 x i32> <i32 1, i32 1>
+; SSE-NEXT:    [[TMP3:%.*]] = shufflevector <2 x double> [[B]], <2 x double> poison, <2 x i32> <i32 poison, i32 0>
+; SSE-NEXT:    [[TMP4:%.*]] = insertelement <2 x double> [[TMP3]], double [[TMP1]], i64 0
+; SSE-NEXT:    [[TMP5:%.*]] = fmul <2 x double> [[TMP2]], [[TMP4]]
+; SSE-NEXT:    [[TMP6:%.*]] = shufflevector <2 x double> [[A]], <2 x double> poison, <2 x i32> zeroinitializer
+; SSE-NEXT:    [[TMP7:%.*]] = tail call <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[TMP6]], <2 x double> [[B]], <2 x double> [[TMP5]])
+; SSE-NEXT:    ret <2 x double> [[TMP7]]
+;
+; AVX-LABEL: @_mm_complexmult_pd_naive(
+; AVX-NEXT:    [[A0:%.*]] = extractelement <2 x double> [[A:%.*]], i64 0
+; AVX-NEXT:    [[A1:%.*]] = extractelement <2 x double> [[A]], i64 1
+; AVX-NEXT:    [[B0:%.*]] = extractelement <2 x double> [[B:%.*]], i64 0
+; AVX-NEXT:    [[B1:%.*]] = extractelement <2 x double> [[B]], i64 1
+; AVX-NEXT:    [[MUL10:%.*]] = fmul double [[A1]], [[B0]]
+; AVX-NEXT:    [[TMP1:%.*]] = fneg double [[B1]]
+; AVX-NEXT:    [[NEG11:%.*]] = fmul double [[A1]], [[TMP1]]
+; AVX-NEXT:    [[MADD0:%.*]] = tail call double @llvm.fmuladd.f64(double [[A0]], double [[B0]], double [[NEG11]])
+; AVX-NEXT:    [[MADD1:%.*]] = tail call double @llvm.fmuladd.f64(double [[A0]], double [[B1]], double [[MUL10]])
+; AVX-NEXT:    [[RES0:%.*]] = insertelement <2 x double> poison, double [[MADD0]], i64 0
+; AVX-NEXT:    [[RES1:%.*]] = insertelement <2 x double> [[RES0]], double [[MADD1]], i64 1
+; AVX-NEXT:    ret <2 x double> [[RES1]]
+;
+  %a0 = extractelement <2 x double> %a, i32 0
+  %a1 = extractelement <2 x double> %a, i32 1
+  %b0 = extractelement <2 x double> %b, i32 0
+  %b1 = extractelement <2 x double> %b, i32 1
+  %mul10 = fmul double %a1, %b0
+  %mul11 = fmul double %a1, %b1
+  %neg11 = fneg double %mul11
+  %madd0 = call double @llvm.fmuladd.f64(double %a0, double %b0, double %neg11)
+  %madd1 = call double @llvm.fmuladd.f64(double %a0, double %b1, double %mul10)
+  %res0 = insertelement <2 x double> undef, double %madd0, i32 0
+  %res1 = insertelement <2 x double> %res0, double %madd1, i32 1
+  ret <2 x double> %res1
+}
diff --git a/llvm/test/Transforms/PhaseOrdering/X86/hadd.ll b/llvm/test/Transforms/PhaseOrdering/X86/hadd.ll
index 4b1234fda0e18..28b48bd3ce6d9 100644
--- a/llvm/test/Transforms/PhaseOrdering/X86/hadd.ll
+++ b/llvm/test/Transforms/PhaseOrdering/X86/hadd.ll
@@ -59,18 +59,12 @@ define <8 x i16> @add_v8i16_01234567(<8 x i16> %a, <8 x i16> %b) {
 
 define <8 x i16> @add_v8i16_u1234567(<8 x i16> %a, <8 x i16> %b) {
 ; SSE2-LABEL: @add_v8i16_u1234567(
-; SSE2-NEXT:    [[A2:%.*]] = extractelement <8 x i16> [[A:%.*]], i64 2
-; SSE2-NEXT:    [[A3:%.*]] = extractelement <8 x i16> [[A]], i64 3
-; SSE2-NEXT:    [[A4:%.*]] = extractelement <8 x i16> [[A]], i64 4
-; SSE2-NEXT:    [[A5:%.*]] = extractelement <8 x i16> [[A]], i64 5
-; SSE2-NEXT:    [[A6:%.*]] = extractelement <8 x i16> [[A]], i64 6
-; SSE2-NEXT:    [[A7:%.*]] = extractelement <8 x i16> [[A]], i64 7
-; SSE2-NEXT:    [[A23:%.*]] = add i16 [[A2]], [[A3]]
-; SSE2-NEXT:    [[A45:%.*]] = add i16 [[A4]], [[A5]]
-; SSE2-NEXT:    [[A67:%.*]] = add i16 [[A6]], [[A7]]
-; SSE2-NEXT:    [[HADD1:%.*]] = insertelement <8 x i16> poison, i16 [[A23]], i64 1
-; SSE2-NEXT:    [[HADD2:%.*]] = insertelement <8 x i16> [[HADD1]], i16 [[A45]], i64 2
-; SSE2-NEXT:    [[HADD3:%.*]] = insertelement <8 x i16> [[HADD2]], i16 [[A67]], i64 3
+; SSE2-NEXT:    [[SHIFT3:%.*]] = shufflevector <8 x i16> [[A:%.*]], <8 x i16> poison, <8 x i32> <i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 7, i32 poison>
+; SSE2-NEXT:    [[TMP6:%.*]] = add <8 x i16> [[A]], [[SHIFT3]]
+; SSE2-NEXT:    [[TMP7:%.*]] = shufflevector <8 x i16> [[A]], <8 x i16> poison, <8 x i32> <i32 poison, i32 3, i32 4, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE2-NEXT:    [[TMP4:%.*]] = shufflevector <8 x i16> [[A]], <8 x i16> poison, <8 x i32> <i32 poison, i32 2, i32 5, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE2-NEXT:    [[HADD1:%.*]] = add <8 x i16> [[TMP7]], [[TMP4]]
+; SSE2-NEXT:    [[HADD3:%.*]] = shufflevector <8 x i16> [[HADD1]], <8 x i16> [[TMP6]], <8 x i32> <i32 poison, i32 1, i32 2, i32 14, i32 poison, i32 poison, i32 poison, i32 poison>
 ; SSE2-NEXT:    [[TMP1:%.*]] = shufflevector <8 x i16> [[B:%.*]], <8 x i16> poison, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 poison, i32 poison, i32 poison, i32 poison>
 ; SSE2-NEXT:    [[TMP2:%.*]] = shufflevector <8 x i16> [[B]], <8 x i16> poison, <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 poison, i32 poison, i32 poison, i32 poison>
 ; SSE2-NEXT:    [[TMP3:%.*]] = add <8 x i16> [[TMP1]], [[TMP2]]
@@ -125,6 +119,395 @@ define <8 x i16> @add_v8i16_u1234567(<8 x i16> %a, <8 x i16> %b) {
   ret <8 x i16> %result
 }
 
+define <8 x i16> @add_v8i16_76u43210(<8 x i16> %a, <8 x i16> %b) {
+; SSE2-LABEL: @add_v8i16_76u43210(
+; SSE2-NEXT:    [[SHIFT:%.*]] = shufflevector <8 x i16> [[A:%.*]], <8 x i16> poison, <8 x i32> <i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE2-NEXT:    [[TMP1:%.*]] = add <8 x i16> [[A]], [[SHIFT]]
+; SSE2-NEXT:    [[SHIFT2:%.*]] = shufflevector <8 x i16> [[B:%.*]], <8 x i16> poison, <8 x i32> <i32 poison, i32 poison, i32 poison, i32 poison, i32 5, i32 poison, i32 poison, i32 poison>
+; SSE2-NEXT:    [[TMP2:%.*]] = add <8 x i16> [[B]], [[SHIFT2]]
+; SSE2-NEXT:    [[SHIFT3:%.*]] = shufflevector <8 x i16> [[B]], <8 x i16> poison, <8 x i32> <i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 6>
+; SSE2-NEXT:    [[TMP3:%.*]] = add <8 x i16> [[SHIFT3]], [[B]]
+; SSE2-NEXT:    [[TMP4:%.*]] = shufflevector <8 x i16> [[A]], <8 x i16> [[B]], <8 x i32> <i32 2, i32 4, i32 6, i32 8, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE2-NEXT:    [[TMP5:%.*]] = shufflevector <8 x i16> [[A]], <8 x i16> [[B]], <8 x i32> <i32 3, i32 5, i32 7, i32 9, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE2-NEXT:    [[TMP6:%.*]] = add <8 x i16> [[TMP4]], [[TMP5]]
+; SSE2-NEXT:    [[HADD41:%.*]] = shufflevector <8 x i16> [[TMP1]], <8 x i16> [[TMP6]], <8 x i32> <i32 0, i32 8, i32 9, i32 10, i32 11, i32 poison, i32 poison, i32 poison>
+; SSE2-NEXT:    [[HADD6:%.*]] = shufflevector <8 x i16> [[HADD41]], <8 x i16> [[TMP2]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 poison, i32 12, i32 poison>
+; SSE2-NEXT:    [[RESULT:%.*]] = shufflevector <8 x i16> [[TMP3]], <8 x i16> [[HADD6]], <8 x i32> <i32 7, i32 14, i32 poison, i32 12, i32 11, i32 10, i32 9, i32 8>
+; SSE2-NEXT:    ret <8 x i16> [[RESULT]]
+;
+; SSE4-LABEL: @add_v8i16_76u43210(
+; SSE4-NEXT:    [[TMP1:%.*]] = shufflevector <8 x i16> [[A:%.*]], <8 x i16> poison, <8 x i32> <i32 0, i32 3, i32 4, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE4-NEXT:    [[TMP2:%.*]] = shufflevector <8 x i16> [[A]], <8 x i16> poison, <8 x i32> <i32 1, i32 2, i32 5, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE4-NEXT:    [[HADD22:%.*]] = add <8 x i16> [[TMP1]], [[TMP2]]
+; SSE4-NEXT:    [[TMP3:%.*]] = shufflevector <8 x i16> [[A]], <8 x i16> [[B:%.*]], <8 x i32> <i32 6, i32 8, i32 poison, i32 12, i32 14, i32 poison, i32 poison, i32 poison>
+; SSE4-NEXT:    [[TMP4:%.*]] = shufflevector <8 x i16> [[A]], <8 x i16> [[B]], <8 x i32> <i32 7, i32 9, i32 poison, i32 13, i32 15, i32 poison, i32 poison, i32 poison>
+; SSE4-NEXT:    [[TMP5:%.*]] = add <8 x i16> [[TMP3]], [[TMP4]]
+; SSE4-NEXT:    [[RESULT:%.*]] = shufflevector <8 x i16> [[TMP5]], <8 x i16> [[HADD22]], <8 x i32> <i32 4, i32 3, i32 poison, i32 1, i32 0, i32 10, i32 9, i32 8>
+; SSE4-NEXT:    ret <8 x i16> [[RESULT]]
+;
+; AVX-LABEL: @add_v8i16_76u43210(
+; AVX-NEXT:    [[TMP1:%.*]] = shufflevector <8 x i16> [[A:%.*]], <8 x i16> poison, <8 x i32> <i32 0, i32 3, i32 4, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX-NEXT:    [[TMP2:%.*]] = shufflevector <8 x i16> [[A]], <8 x i16> poison, <8 x i32> <i32 1, i32 2, i32 5, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX-NEXT:    [[HADD22:%.*]] = add <8 x i16> [[TMP1]], [[TMP2]]
+; AVX-NEXT:    [[TMP3:%.*]] = shufflevector <8 x i16> [[A]], <8 x i16> [[B:%.*]], <8 x i32> <i32 6, i32 8, i32 poison, i32 12, i32 14, i32 poison, i32 poison, i32 poison>
+; AVX-NEXT:    [[TMP4:%.*]] = shufflevector <8 x i16> [[A]], <8 x i16> [[B]], <8 x i32> <i32 7, i32 9, i32 poison, i32 13, i32 15, i32 poison, i32 poison, i32 poison>
+; AVX-NEXT:    [[TMP5:%.*]] = add <8 x i16> [[TMP3]], [[TMP4]]
+; AVX-NEXT:    [[RESULT:%.*]] = shufflevector <8 x i16> [[TMP5]], <8 x i16> [[HADD22]], <8 x i32> <i32 4, i32 3, i32 poison, i32 1, i32 0, i32 10, i32 9, i32 8>
+; AVX-NEXT:    ret <8 x i16> [[RESULT]]
+;
+  %a0 = extractelement <8 x i16> %a, i32 0
+  %a1 = extractelement <8 x i16> %a, i32 1
+  %a2 = extractelement <8 x i16> %a, i32 2
+  %a3 = extractelement <8 x i16> %a, i32 3
+  %a4 = extractelement <8 x i16> %a, i32 4
+  %a5 = extractelement <8 x i16> %a, i32 5
+  %a6 = extractelement <8 x i16> %a, i32 6
+  %a7 = extractelement <8 x i16> %a, i32 7
+  %a01 = add i16 %a0, %a1
+  %a23 = add i16 %a2, %a3
+  %a45 = add i16 %a4, %a5
+  %a67 = add i16 %a6, %a7
+  %b0 = extractelement <8 x i16> %b, i32 0
+  %b1 = extractelement <8 x i16> %b, i32 1
+  %b2 = extractelement <8 x i16> %b, i32 2
+  %b3 = extractelement <8 x i16> %b, i32 3
+  %b4 = extractelement <8 x i16> %b, i32 4
+  %b5 = extractelement <8 x i16> %b, i32 5
+  %b6 = extractelement <8 x i16> %b, i32 6
+  %b7 = extractelement <8 x i16> %b, i32 7
+  %b01 = add i16 %b0, %b1
+  %b23 = add i16 %b2, %b3
+  %b45 = add i16 %b4, %b5
+  %b67 = add i16 %b6, %b7
+  %hadd0 = insertelement <8 x i16> poison, i16 %a01, i32 0
+  %hadd1 = insertelement <8 x i16> %hadd0, i16 %a23, i32 1
+  %hadd2 = insertelement <8 x i16> %hadd1, i16 %a45, i32 2
+  %hadd3 = insertelement <8 x i16> %hadd2, i16 %a67, i32 3
+  %hadd4 = insertelement <8 x i16> %hadd3, i16 %b01, i32 4
+  %hadd5 = insertelement <8 x i16> %hadd4, i16 %b23, i32 5
+  %hadd6 = insertelement <8 x i16> %hadd5, i16 %b45, i32 6
+  %hadd7 = insertelement <8 x i16> %hadd6, i16 %b67, i32 7
+  %result = shufflevector <8 x i16> %hadd7, <8 x i16> %a, <8 x i32> <i32 7, i32 6, i32 poison, i32 4, i32 3, i32 2, i32 1, i32 0>
+  ret <8 x i16> %result
+}
+
+;
+; v16i16
+;
+
+define <16 x i16> @add_v16i16_0123456789ABCDEF(<16 x i16> %a, <16 x i16> %b) {
+; CHECK-LABEL: @add_v16i16_0123456789ABCDEF(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <16 x i16> [[A:%.*]], <16 x i16> [[B:%.*]], <16 x i32> <i32 0, i32 2, i32 4, i32 6, i32 16, i32 18, i32 20, i32 22, i32 8, i32 10, i32 12, i32 14, i32 24, i32 26, i32 28, i32 30>
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <16 x i16> [[A]], <16 x i16> [[B]], <16 x i32> <i32 1, i32 3, i32 5, i32 7, i32 17, i32 19, i32 21, i32 23, i32 9, i32 11, i32 13, i32 15, i32 25, i32 27, i32 29, i32 31>
+; CHECK-NEXT:    [[TMP3:%.*]] = add <16 x i16> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    ret <16 x i16> [[TMP3]]
+;
+  %a0 = extractelement <16 x i16> %a, i32 0
+  %a1 = extractelement <16 x i16> %a, i32 1
+  %a2 = extractelement <16 x i16> %a, i32 2
+  %a3 = extractelement <16 x i16> %a, i32 3
+  %a4 = extractelement <16 x i16> %a, i32 4
+  %a5 = extractelement <16 x i16> %a, i32 5
+  %a6 = extractelement <16 x i16> %a, i32 6
+  %a7 = extractelement <16 x i16> %a, i32 7
+  %a8 = extractelement <16 x i16> %a, i32 8
+  %a9 = extractelement <16 x i16> %a, i32 9
+  %aA = extractelement <16 x i16> %a, i32 10
+  %aB = extractelement <16 x i16> %a, i32 11
+  %aC = extractelement <16 x i16> %a, i32 12
+  %aD = extractelement <16 x i16> %a, i32 13
+  %aE = extractelement <16 x i16> %a, i32 14
+  %aF = extractelement <16 x i16> %a, i32 15
+  %a01 = add i16 %a0, %a1
+  %a23 = add i16 %a2, %a3
+  %a45 = add i16 %a4, %a5
+  %a67 = add i16 %a6, %a7
+  %a89 = add i16 %a8, %a9
+  %aAB = add i16 %aA, %aB
+  %aCD = add i16 %aC, %aD
+  %aEF = add i16 %aE, %aF
+  %b0 = extractelement <16 x i16> %b, i32 0
+  %b1 = extractelement <16 x i16> %b, i32 1
+  %b2 = extractelement <16 x i16> %b, i32 2
+  %b3 = extractelement <16 x i16> %b, i32 3
+  %b4 = extractelement <16 x i16> %b, i32 4
+  %b5 = extractelement <16 x i16> %b, i32 5
+  %b6 = extractelement <16 x i16> %b, i32 6
+  %b7 = extractelement <16 x i16> %b, i32 7
+  %b8 = extractelement <16 x i16> %b, i32 8
+  %b9 = extractelement <16 x i16> %b, i32 9
+  %bA = extractelement <16 x i16> %b, i32 10
+  %bB = extractelement <16 x i16> %b, i32 11
+  %bC = extractelement <16 x i16> %b, i32 12
+  %bD = extractelement <16 x i16> %b, i32 13
+  %bE = extractelement <16 x i16> %b, i32 14
+  %bF = extractelement <16 x i16> %b, i32 15
+  %b01 = add i16 %b0, %b1
+  %b23 = add i16 %b2, %b3
+  %b45 = add i16 %b4, %b5
+  %b67 = add i16 %b6, %b7
+  %b89 = add i16 %b8, %b9
+  %bAB = add i16 %bA, %bB
+  %bCD = add i16 %bC, %bD
+  %bEF = add i16 %bE, %bF
+  %hadd0 = insertelement <16 x i16> poison, i16 %a01, i32 0
+  %hadd1 = insertelement <16 x i16> %hadd0, i16 %a23, i32 1
+  %hadd2 = insertelement <16 x i16> %hadd1, i16 %a45, i32 2
+  %hadd3 = insertelement <16 x i16> %hadd2, i16 %a67, i32 3
+  %hadd4 = insertelement <16 x i16> %hadd3, i16 %b01, i32 4
+  %hadd5 = insertelement <16 x i16> %hadd4, i16 %b23, i32 5
+  %hadd6 = insertelement <16 x i16> %hadd5, i16 %b45, i32 6
+  %hadd7 = insertelement <16 x i16> %hadd6, i16 %b67, i32 7
+  %hadd8 = insertelement <16 x i16> %hadd7, i16 %a89, i32 8
+  %hadd9 = insertelement <16 x i16> %hadd8, i16 %aAB, i32 9
+  %haddA = insertelement <16 x i16> %hadd9, i16 %aCD, i32 10
+  %haddB = insertelement <16 x i16> %haddA, i16 %aEF, i32 11
+  %haddC = insertelement <16 x i16> %haddB, i16 %b89, i32 12
+  %haddD = insertelement <16 x i16> %haddC, i16 %bAB, i32 13
+  %haddE = insertelement <16 x i16> %haddD, i16 %bCD, i32 14
+  %haddF = insertelement <16 x i16> %haddE, i16 %bEF, i32 15
+  %result = shufflevector <16 x i16> %haddF, <16 x i16> %a, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+  ret <16 x i16> %result
+}
+
+define <16 x i16> @add_v16i16_0123u56789uBCDEF(<16 x i16> %a, <16 x i16> %b) {
+; SSE2-LABEL: @add_v16i16_0123u56789uBCDEF(
+; SSE2-NEXT:    [[BC:%.*]] = extractelement <16 x i16> [[B:%.*]], i64 12
+; SSE2-NEXT:    [[BD:%.*]] = extractelement <16 x i16> [[B]], i64 13
+; SSE2-NEXT:    [[BE:%.*]] = extractelement <16 x i16> [[B]], i64 14
+; SSE2-NEXT:    [[BF:%.*]] = extractelement <16 x i16> [[B]], i64 15
+; SSE2-NEXT:    [[BCD:%.*]] = add i16 [[BC]], [[BD]]
+; SSE2-NEXT:    [[BEF:%.*]] = add i16 [[BE]], [[BF]]
+; SSE2-NEXT:    [[TMP1:%.*]] = shufflevector <16 x i16> [[A:%.*]], <16 x i16> [[B]], <16 x i32> <i32 0, i32 2, i32 4, i32 6, i32 poison, i32 18, i32 20, i32 22, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE2-NEXT:    [[TMP2:%.*]] = shufflevector <16 x i16> [[A]], <16 x i16> [[B]], <16 x i32> <i32 1, i32 3, i32 5, i32 7, i32 poison, i32 19, i32 21, i32 23, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE2-NEXT:    [[TMP3:%.*]] = shufflevector <16 x i16> [[TMP1]], <16 x i16> [[A]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 5, i32 6, i32 7, i32 24, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE2-NEXT:    [[TMP7:%.*]] = shufflevector <16 x i16> [[TMP2]], <16 x i16> [[A]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 5, i32 6, i32 7, i32 25, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE2-NEXT:    [[HADD8:%.*]] = add <16 x i16> [[TMP3]], [[TMP7]]
+; SSE2-NEXT:    [[TMP4:%.*]] = shufflevector <16 x i16> [[A]], <16 x i16> [[B]], <16 x i32> <i32 10, i32 poison, i32 14, i32 24, i32 26, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE2-NEXT:    [[TMP5:%.*]] = shufflevector <16 x i16> [[A]], <16 x i16> [[B]], <16 x i32> <i32 11, i32 poison, i32 15, i32 25, i32 27, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE2-NEXT:    [[TMP6:%.*]] = add <16 x i16> [[TMP4]], [[TMP5]]
+; SSE2-NEXT:    [[HADDD1:%.*]] = shufflevector <16 x i16> [[HADD8]], <16 x i16> [[TMP6]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 5, i32 6, i32 7, i32 8, i32 16, i32 poison, i32 18, i32 19, i32 20, i32 poison, i32 poison>
+; SSE2-NEXT:    [[HADDE:%.*]] = insertelement <16 x i16> [[HADDD1]], i16 [[BCD]], i64 14
+; SSE2-NEXT:    [[HADDF:%.*]] = insertelement <16 x i16> [[HADDE]], i16 [[BEF]], i64 15
+; SSE2-NEXT:    [[RESULT:%.*]] = shufflevector <16 x i16> [[HADDF]], <16 x i16> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 5, i32 6, i32 7, i32 8, i32 9, i32 poison, i32 11, i32 12, i32 13, i32 14, i32 15>
+; SSE2-NEXT:    ret <16 x i16> [[RESULT]]
+;
+; SSE4-LABEL: @add_v16i16_0123u56789uBCDEF(
+; SSE4-NEXT:    [[TMP1:%.*]] = shufflevector <16 x i16> [[A:%.*]], <16 x i16> [[B:%.*]], <16 x i32> <i32 0, i32 2, i32 4, i32 6, i32 poison, i32 18, i32 20, i32 22, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE4-NEXT:    [[TMP2:%.*]] = shufflevector <16 x i16> [[A]], <16 x i16> [[B]], <16 x i32> <i32 1, i32 3, i32 5, i32 7, i32 poison, i32 19, i32 21, i32 23, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE4-NEXT:    [[TMP3:%.*]] = shufflevector <16 x i16> [[TMP1]], <16 x i16> [[A]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 5, i32 6, i32 7, i32 24, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE4-NEXT:    [[TMP4:%.*]] = shufflevector <16 x i16> [[TMP2]], <16 x i16> [[A]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 5, i32 6, i32 7, i32 25, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE4-NEXT:    [[TMP5:%.*]] = shufflevector <16 x i16> [[TMP3]], <16 x i16> [[A]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 5, i32 6, i32 7, i32 8, i32 26, i32 poison, i32 30, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE4-NEXT:    [[TMP6:%.*]] = shufflevector <16 x i16> [[TMP4]], <16 x i16> [[A]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 5, i32 6, i32 7, i32 8, i32 27, i32 poison, i32 31, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE4-NEXT:    [[TMP7:%.*]] = shufflevector <16 x i16> [[TMP5]], <16 x i16> [[B]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 5, i32 6, i32 7, i32 8, i32 9, i32 poison, i32 11, i32 24, i32 26, i32 28, i32 30>
+; SSE4-NEXT:    [[TMP8:%.*]] = shufflevector <16 x i16> [[TMP6]], <16 x i16> [[B]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 5, i32 6, i32 7, i32 8, i32 9, i32 poison, i32 11, i32 25, i32 27, i32 29, i32 31>
+; SSE4-NEXT:    [[TMP9:%.*]] = add <16 x i16> [[TMP7]], [[TMP8]]
+; SSE4-NEXT:    ret <16 x i16> [[TMP9]]
+;
+; AVX2-LABEL: @add_v16i16_0123u56789uBCDEF(
+; AVX2-NEXT:    [[TMP1:%.*]] = shufflevector <16 x i16> [[A:%.*]], <16 x i16> [[B:%.*]], <16 x i32> <i32 0, i32 2, i32 4, i32 6, i32 poison, i32 18, i32 20, i32 22, i32 9, i32 10, i32 poison, i32 14, i32 24, i32 26, i32 28, i32 30>
+; AVX2-NEXT:    [[TMP2:%.*]] = shufflevector <16 x i16> [[A]], <16 x i16> [[B]], <16 x i32> <i32 1, i32 3, i32 5, i32 7, i32 poison, i32 19, i32 21, i32 23, i32 8, i32 11, i32 poison, i32 15, i32 25, i32 27, i32 29, i32 31>
+; AVX2-NEXT:    [[RESULT:%.*]] = add <16 x i16> [[TMP1]], [[TMP2]]
+; AVX2-NEXT:    ret <16 x i16> [[RESULT]]
+;
+; AVX512-LABEL: @add_v16i16_0123u56789uBCDEF(
+; AVX512-NEXT:    [[TMP1:%.*]] = shufflevector <16 x i16> [[A:%.*]], <16 x i16> [[B:%.*]], <16 x i32> <i32 0, i32 2, i32 4, i32 6, i32 poison, i32 18, i32 20, i32 22, i32 8, i32 10, i32 poison, i32 14, i32 24, i32 26, i32 28, i32 30>
+; AVX512-NEXT:    [[TMP2:%.*]] = shufflevector <16 x i16> [[A]], <16 x i16> [[B]], <16 x i32> <i32 1, i32 3, i32 5, i32 7, i32 poison, i32 19, i32 21, i32 23, i32 9, i32 11, i32 poison, i32 15, i32 25, i32 27, i32 29, i32 31>
+; AVX512-NEXT:    [[RESULT:%.*]] = add <16 x i16> [[TMP1]], [[TMP2]]
+; AVX512-NEXT:    ret <16 x i16> [[RESULT]]
+;
+  %a0 = extractelement <16 x i16> %a, i32 0
+  %a1 = extractelement <16 x i16> %a, i32 1
+  %a2 = extractelement <16 x i16> %a, i32 2
+  %a3 = extractelement <16 x i16> %a, i32 3
+  %a4 = extractelement <16 x i16> %a, i32 4
+  %a5 = extractelement <16 x i16> %a, i32 5
+  %a6 = extractelement <16 x i16> %a, i32 6
+  %a7 = extractelement <16 x i16> %a, i32 7
+  %a8 = extractelement <16 x i16> %a, i32 8
+  %a9 = extractelement <16 x i16> %a, i32 9
+  %aA = extractelement <16 x i16> %a, i32 10
+  %aB = extractelement <16 x i16> %a, i32 11
+  %aC = extractelement <16 x i16> %a, i32 12
+  %aD = extractelement <16 x i16> %a, i32 13
+  %aE = extractelement <16 x i16> %a, i32 14
+  %aF = extractelement <16 x i16> %a, i32 15
+  %a01 = add i16 %a0, %a1
+  %a23 = add i16 %a2, %a3
+  %a45 = add i16 %a4, %a5
+  %a67 = add i16 %a6, %a7
+  %a89 = add i16 %a8, %a9
+  %aAB = add i16 %aA, %aB
+  %aCD = add i16 %aC, %aD
+  %aEF = add i16 %aE, %aF
+  %b0 = extractelement <16 x i16> %b, i32 0
+  %b1 = extractelement <16 x i16> %b, i32 1
+  %b2 = extractelement <16 x i16> %b, i32 2
+  %b3 = extractelement <16 x i16> %b, i32 3
+  %b4 = extractelement <16 x i16> %b, i32 4
+  %b5 = extractelement <16 x i16> %b, i32 5
+  %b6 = extractelement <16 x i16> %b, i32 6
+  %b7 = extractelement <16 x i16> %b, i32 7
+  %b8 = extractelement <16 x i16> %b, i32 8
+  %b9 = extractelement <16 x i16> %b, i32 9
+  %bA = extractelement <16 x i16> %b, i32 10
+  %bB = extractelement <16 x i16> %b, i32 11
+  %bC = extractelement <16 x i16> %b, i32 12
+  %bD = extractelement <16 x i16> %b, i32 13
+  %bE = extractelement <16 x i16> %b, i32 14
+  %bF = extractelement <16 x i16> %b, i32 15
+  %b01 = add i16 %b0, %b1
+  %b23 = add i16 %b2, %b3
+  %b45 = add i16 %b4, %b5
+  %b67 = add i16 %b6, %b7
+  %b89 = add i16 %b8, %b9
+  %bAB = add i16 %bA, %bB
+  %bCD = add i16 %bC, %bD
+  %bEF = add i16 %bE, %bF
+  %hadd0 = insertelement <16 x i16> poison, i16 %a01, i32 0
+  %hadd1 = insertelement <16 x i16> %hadd0, i16 %a23, i32 1
+  %hadd2 = insertelement <16 x i16> %hadd1, i16 %a45, i32 2
+  %hadd3 = insertelement <16 x i16> %hadd2, i16 %a67, i32 3
+  %hadd4 = insertelement <16 x i16> %hadd3, i16 %b01, i32 4
+  %hadd5 = insertelement <16 x i16> %hadd4, i16 %b23, i32 5
+  %hadd6 = insertelement <16 x i16> %hadd5, i16 %b45, i32 6
+  %hadd7 = insertelement <16 x i16> %hadd6, i16 %b67, i32 7
+  %hadd8 = insertelement <16 x i16> %hadd7, i16 %a89, i32 8
+  %hadd9 = insertelement <16 x i16> %hadd8, i16 %aAB, i32 9
+  %haddA = insertelement <16 x i16> %hadd9, i16 %aCD, i32 10
+  %haddB = insertelement <16 x i16> %haddA, i16 %aEF, i32 11
+  %haddC = insertelement <16 x i16> %haddB, i16 %b89, i32 12
+  %haddD = insertelement <16 x i16> %haddC, i16 %bAB, i32 13
+  %haddE = insertelement <16 x i16> %haddD, i16 %bCD, i32 14
+  %haddF = insertelement <16 x i16> %haddE, i16 %bEF, i32 15
+  %result = shufflevector <16 x i16> %haddF, <16 x i16> %a, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 5, i32 6, i32 7, i32 8, i32 9, i32 poison, i32 11, i32 12, i32 13, i32 14, i32 15>
+  ret <16 x i16> %result
+}
+
+define <16 x i16> @add_v16i16_FEuCBA98765432u0(<16 x i16> %a, <16 x i16> %b) {
+; SSE2-LABEL: @add_v16i16_FEuCBA98765432u0(
+; SSE2-NEXT:    [[BC:%.*]] = extractelement <16 x i16> [[B:%.*]], i64 12
+; SSE2-NEXT:    [[BD:%.*]] = extractelement <16 x i16> [[B]], i64 13
+; SSE2-NEXT:    [[BE:%.*]] = extractelement <16 x i16> [[B]], i64 14
+; SSE2-NEXT:    [[BF:%.*]] = extractelement <16 x i16> [[B]], i64 15
+; SSE2-NEXT:    [[BCD:%.*]] = add i16 [[BC]], [[BD]]
+; SSE2-NEXT:    [[BEF:%.*]] = add i16 [[BE]], [[BF]]
+; SSE2-NEXT:    [[TMP1:%.*]] = shufflevector <16 x i16> [[A:%.*]], <16 x i16> [[B]], <16 x i32> <i32 0, i32 poison, i32 4, i32 6, i32 16, i32 18, i32 20, i32 22, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE2-NEXT:    [[TMP2:%.*]] = shufflevector <16 x i16> [[A]], <16 x i16> [[B]], <16 x i32> <i32 1, i32 poison, i32 5, i32 7, i32 17, i32 19, i32 21, i32 23, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE2-NEXT:    [[TMP3:%.*]] = shufflevector <16 x i16> [[TMP1]], <16 x i16> [[A]], <16 x i32> <i32 0, i32 poison, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 24, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE2-NEXT:    [[TMP7:%.*]] = shufflevector <16 x i16> [[TMP2]], <16 x i16> [[A]], <16 x i32> <i32 0, i32 poison, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 25, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE2-NEXT:    [[HADD8:%.*]] = add <16 x i16> [[TMP3]], [[TMP7]]
+; SSE2-NEXT:    [[TMP4:%.*]] = shufflevector <16 x i16> [[A]], <16 x i16> [[B]], <16 x i32> <i32 10, i32 12, i32 14, i32 24, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE2-NEXT:    [[TMP5:%.*]] = shufflevector <16 x i16> [[A]], <16 x i16> [[B]], <16 x i32> <i32 11, i32 13, i32 15, i32 25, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE2-NEXT:    [[TMP6:%.*]] = add <16 x i16> [[TMP4]], [[TMP5]]
+; SSE2-NEXT:    [[HADDC1:%.*]] = shufflevector <16 x i16> [[HADD8]], <16 x i16> [[TMP6]], <16 x i32> <i32 0, i32 poison, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 16, i32 17, i32 18, i32 19, i32 poison, i32 poison, i32 poison>
+; SSE2-NEXT:    [[HADDE:%.*]] = insertelement <16 x i16> [[HADDC1]], i16 [[BCD]], i64 14
+; SSE2-NEXT:    [[HADDF:%.*]] = insertelement <16 x i16> [[HADDE]], i16 [[BEF]], i64 15
+; SSE2-NEXT:    [[RESULT:%.*]] = shufflevector <16 x i16> [[HADDF]], <16 x i16> poison, <16 x i32> <i32 15, i32 14, i32 poison, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 poison, i32 0>
+; SSE2-NEXT:    ret <16 x i16> [[RESULT]]
+;
+; SSE4-LABEL: @add_v16i16_FEuCBA98765432u0(
+; SSE4-NEXT:    [[TMP1:%.*]] = shufflevector <16 x i16> [[A:%.*]], <16 x i16> [[B:%.*]], <16 x i32> <i32 0, i32 poison, i32 4, i32 6, i32 16, i32 18, i32 20, i32 22, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE4-NEXT:    [[TMP2:%.*]] = shufflevector <16 x i16> [[A]], <16 x i16> [[B]], <16 x i32> <i32 1, i32 poison, i32 5, i32 7, i32 17, i32 19, i32 21, i32 23, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE4-NEXT:    [[TMP3:%.*]] = shufflevector <16 x i16> [[TMP1]], <16 x i16> [[A]], <16 x i32> <i32 0, i32 poison, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 24, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE4-NEXT:    [[TMP10:%.*]] = shufflevector <16 x i16> [[TMP2]], <16 x i16> [[A]], <16 x i32> <i32 0, i32 poison, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 25, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE4-NEXT:    [[TMP4:%.*]] = shufflevector <16 x i16> [[TMP3]], <16 x i16> [[A]], <16 x i32> <i32 0, i32 poison, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 poison, i32 27, i32 28, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE4-NEXT:    [[TMP5:%.*]] = shufflevector <16 x i16> [[TMP10]], <16 x i16> [[A]], <16 x i32> <i32 0, i32 poison, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 poison, i32 26, i32 29, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE4-NEXT:    [[TMP6:%.*]] = add <16 x i16> [[TMP4]], [[TMP5]]
+; SSE4-NEXT:    [[TMP7:%.*]] = shufflevector <16 x i16> [[A]], <16 x i16> [[B]], <16 x i32> <i32 14, i32 24, i32 28, i32 30, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE4-NEXT:    [[TMP8:%.*]] = shufflevector <16 x i16> [[A]], <16 x i16> [[B]], <16 x i32> <i32 15, i32 25, i32 29, i32 31, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE4-NEXT:    [[TMP9:%.*]] = add <16 x i16> [[TMP7]], [[TMP8]]
+; SSE4-NEXT:    [[RESULT:%.*]] = shufflevector <16 x i16> [[TMP9]], <16 x i16> [[TMP6]], <16 x i32> <i32 3, i32 2, i32 poison, i32 1, i32 0, i32 27, i32 26, i32 24, i32 23, i32 22, i32 21, i32 20, i32 19, i32 18, i32 poison, i32 16>
+; SSE4-NEXT:    ret <16 x i16> [[RESULT]]
+;
+; AVX2-LABEL: @add_v16i16_FEuCBA98765432u0(
+; AVX2-NEXT:    [[TMP1:%.*]] = shufflevector <16 x i16> [[A:%.*]], <16 x i16> [[B:%.*]], <16 x i32> <i32 0, i32 poison, i32 4, i32 6, i32 16, i32 18, i32 20, i32 22, i32 9, i32 10, i32 12, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX2-NEXT:    [[TMP2:%.*]] = shufflevector <16 x i16> [[A]], <16 x i16> [[B]], <16 x i32> <i32 1, i32 poison, i32 5, i32 7, i32 17, i32 19, i32 21, i32 23, i32 8, i32 11, i32 13, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX2-NEXT:    [[HADDA:%.*]] = add <16 x i16> [[TMP1]], [[TMP2]]
+; AVX2-NEXT:    [[TMP3:%.*]] = shufflevector <16 x i16> [[A]], <16 x i16> [[B]], <16 x i32> <i32 14, i32 24, i32 poison, i32 28, i32 30, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX2-NEXT:    [[TMP4:%.*]] = shufflevector <16 x i16> [[A]], <16 x i16> [[B]], <16 x i32> <i32 15, i32 25, i32 poison, i32 29, i32 31, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX2-NEXT:    [[TMP5:%.*]] = add <16 x i16> [[TMP3]], [[TMP4]]
+; AVX2-NEXT:    [[RESULT:%.*]] = shufflevector <16 x i16> [[TMP5]], <16 x i16> [[HADDA]], <16 x i32> <i32 4, i32 3, i32 poison, i32 1, i32 0, i32 26, i32 25, i32 24, i32 23, i32 22, i32 21, i32 20, i32 19, i32 18, i32 poison, i32 16>
+; AVX2-NEXT:    ret <16 x i16> [[RESULT]]
+;
+; AVX512-LABEL: @add_v16i16_FEuCBA98765432u0(
+; AVX512-NEXT:    [[TMP1:%.*]] = shufflevector <16 x i16> [[A:%.*]], <16 x i16> [[B:%.*]], <16 x i32> <i32 0, i32 poison, i32 4, i32 6, i32 16, i32 18, i32 20, i32 22, i32 8, i32 11, i32 12, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512-NEXT:    [[TMP2:%.*]] = shufflevector <16 x i16> [[A]], <16 x i16> [[B]], <16 x i32> <i32 1, i32 poison, i32 5, i32 7, i32 17, i32 19, i32 21, i32 23, i32 9, i32 10, i32 13, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512-NEXT:    [[HADDA2:%.*]] = add <16 x i16> [[TMP1]], [[TMP2]]
+; AVX512-NEXT:    [[TMP3:%.*]] = shufflevector <16 x i16> [[A]], <16 x i16> [[B]], <16 x i32> <i32 14, i32 24, i32 poison, i32 28, i32 30, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512-NEXT:    [[TMP4:%.*]] = shufflevector <16 x i16> [[A]], <16 x i16> [[B]], <16 x i32> <i32 15, i32 25, i32 poison, i32 29, i32 31, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512-NEXT:    [[TMP5:%.*]] = add <16 x i16> [[TMP3]], [[TMP4]]
+; AVX512-NEXT:    [[RESULT:%.*]] = shufflevector <16 x i16> [[TMP5]], <16 x i16> [[HADDA2]], <16 x i32> <i32 4, i32 3, i32 poison, i32 1, i32 0, i32 26, i32 25, i32 24, i32 23, i32 22, i32 21, i32 20, i32 19, i32 18, i32 poison, i32 16>
+; AVX512-NEXT:    ret <16 x i16> [[RESULT]]
+;
+  %a0 = extractelement <16 x i16> %a, i32 0
+  %a1 = extractelement <16 x i16> %a, i32 1
+  %a2 = extractelement <16 x i16> %a, i32 2
+  %a3 = extractelement <16 x i16> %a, i32 3
+  %a4 = extractelement <16 x i16> %a, i32 4
+  %a5 = extractelement <16 x i16> %a, i32 5
+  %a6 = extractelement <16 x i16> %a, i32 6
+  %a7 = extractelement <16 x i16> %a, i32 7
+  %a8 = extractelement <16 x i16> %a, i32 8
+  %a9 = extractelement <16 x i16> %a, i32 9
+  %aA = extractelement <16 x i16> %a, i32 10
+  %aB = extractelement <16 x i16> %a, i32 11
+  %aC = extractelement <16 x i16> %a, i32 12
+  %aD = extractelement <16 x i16> %a, i32 13
+  %aE = extractelement <16 x i16> %a, i32 14
+  %aF = extractelement <16 x i16> %a, i32 15
+  %a01 = add i16 %a0, %a1
+  %a23 = add i16 %a2, %a3
+  %a45 = add i16 %a4, %a5
+  %a67 = add i16 %a6, %a7
+  %a89 = add i16 %a8, %a9
+  %aAB = add i16 %aA, %aB
+  %aCD = add i16 %aC, %aD
+  %aEF = add i16 %aE, %aF
+  %b0 = extractelement <16 x i16> %b, i32 0
+  %b1 = extractelement <16 x i16> %b, i32 1
+  %b2 = extractelement <16 x i16> %b, i32 2
+  %b3 = extractelement <16 x i16> %b, i32 3
+  %b4 = extractelement <16 x i16> %b, i32 4
+  %b5 = extractelement <16 x i16> %b, i32 5
+  %b6 = extractelement <16 x i16> %b, i32 6
+  %b7 = extractelement <16 x i16> %b, i32 7
+  %b8 = extractelement <16 x i16> %b, i32 8
+  %b9 = extractelement <16 x i16> %b, i32 9
+  %bA = extractelement <16 x i16> %b, i32 10
+  %bB = extractelement <16 x i16> %b, i32 11
+  %bC = extractelement <16 x i16> %b, i32 12
+  %bD = extractelement <16 x i16> %b, i32 13
+  %bE = extractelement <16 x i16> %b, i32 14
+  %bF = extractelement <16 x i16> %b, i32 15
+  %b01 = add i16 %b0, %b1
+  %b23 = add i16 %b2, %b3
+  %b45 = add i16 %b4, %b5
+  %b67 = add i16 %b6, %b7
+  %b89 = add i16 %b8, %b9
+  %bAB = add i16 %bA, %bB
+  %bCD = add i16 %bC, %bD
+  %bEF = add i16 %bE, %bF
+  %hadd0 = insertelement <16 x i16> poison, i16 %a01, i32 0
+  %hadd1 = insertelement <16 x i16> %hadd0, i16 %a23, i32 1
+  %hadd2 = insertelement <16 x i16> %hadd1, i16 %a45, i32 2
+  %hadd3 = insertelement <16 x i16> %hadd2, i16 %a67, i32 3
+  %hadd4 = insertelement <16 x i16> %hadd3, i16 %b01, i32 4
+  %hadd5 = insertelement <16 x i16> %hadd4, i16 %b23, i32 5
+  %hadd6 = insertelement <16 x i16> %hadd5, i16 %b45, i32 6
+  %hadd7 = insertelement <16 x i16> %hadd6, i16 %b67, i32 7
+  %hadd8 = insertelement <16 x i16> %hadd7, i16 %a89, i32 8
+  %hadd9 = insertelement <16 x i16> %hadd8, i16 %aAB, i32 9
+  %haddA = insertelement <16 x i16> %hadd9, i16 %aCD, i32 10
+  %haddB = insertelement <16 x i16> %haddA, i16 %aEF, i32 11
+  %haddC = insertelement <16 x i16> %haddB, i16 %b89, i32 12
+  %haddD = insertelement <16 x i16> %haddC, i16 %bAB, i32 13
+  %haddE = insertelement <16 x i16> %haddD, i16 %bCD, i32 14
+  %haddF = insertelement <16 x i16> %haddE, i16 %bEF, i32 15
+  %result = shufflevector <16 x i16> %haddF, <16 x i16> %a, <16 x i32> <i32 15, i32 14, i32 poison, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 poison, i32 0>
+  ret <16 x i16> %result
+}
+
 ;
 ; v4i32
 ;
@@ -336,6 +719,51 @@ define <4 x i32> @add_v4i32_01uu(<4 x i32> %a, <4 x i32> %b) {
   ret <4 x i32> %result
 }
 
+define <4 x i32> @add_v4i32_32u0(<4 x i32> %a, <4 x i32> %b) {
+; SSE2-LABEL: @add_v4i32_32u0(
+; SSE2-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i32> [[B:%.*]], <4 x i32> [[A:%.*]], <4 x i32> <i32 2, i32 0, i32 poison, i32 4>
+; SSE2-NEXT:    [[TMP2:%.*]] = shufflevector <4 x i32> [[B]], <4 x i32> [[A]], <4 x i32> <i32 3, i32 1, i32 poison, i32 5>
+; SSE2-NEXT:    [[RESULT1:%.*]] = add <4 x i32> [[TMP1]], [[TMP2]]
+; SSE2-NEXT:    ret <4 x i32> [[RESULT1]]
+;
+; SSE4-LABEL: @add_v4i32_32u0(
+; SSE4-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i32> [[B:%.*]], <4 x i32> [[A:%.*]], <4 x i32> <i32 2, i32 0, i32 poison, i32 4>
+; SSE4-NEXT:    [[TMP2:%.*]] = shufflevector <4 x i32> [[B]], <4 x i32> [[A]], <4 x i32> <i32 3, i32 1, i32 poison, i32 5>
+; SSE4-NEXT:    [[RESULT:%.*]] = add <4 x i32> [[TMP1]], [[TMP2]]
+; SSE4-NEXT:    ret <4 x i32> [[RESULT]]
+;
+; AVX2-LABEL: @add_v4i32_32u0(
+; AVX2-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i32> [[B:%.*]], <4 x i32> [[A:%.*]], <4 x i32> <i32 2, i32 0, i32 poison, i32 4>
+; AVX2-NEXT:    [[TMP2:%.*]] = shufflevector <4 x i32> [[B]], <4 x i32> [[A]], <4 x i32> <i32 3, i32 1, i32 poison, i32 5>
+; AVX2-NEXT:    [[RESULT:%.*]] = add <4 x i32> [[TMP1]], [[TMP2]]
+; AVX2-NEXT:    ret <4 x i32> [[RESULT]]
+;
+; AVX512-LABEL: @add_v4i32_32u0(
+; AVX512-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i32> [[B:%.*]], <4 x i32> [[A:%.*]], <4 x i32> <i32 2, i32 0, i32 poison, i32 4>
+; AVX512-NEXT:    [[TMP2:%.*]] = shufflevector <4 x i32> [[B]], <4 x i32> [[A]], <4 x i32> <i32 3, i32 1, i32 poison, i32 5>
+; AVX512-NEXT:    [[RESULT1:%.*]] = add <4 x i32> [[TMP1]], [[TMP2]]
+; AVX512-NEXT:    ret <4 x i32> [[RESULT1]]
+;
+  %a0 = extractelement <4 x i32> %a, i32 0
+  %a1 = extractelement <4 x i32> %a, i32 1
+  %a2 = extractelement <4 x i32> %a, i32 2
+  %a3 = extractelement <4 x i32> %a, i32 3
+  %a01 = add i32 %a0, %a1
+  %a23 = add i32 %a2, %a3
+  %b0 = extractelement <4 x i32> %b, i32 0
+  %b1 = extractelement <4 x i32> %b, i32 1
+  %b2 = extractelement <4 x i32> %b, i32 2
+  %b3 = extractelement <4 x i32> %b, i32 3
+  %b01 = add i32 %b0, %b1
+  %b23 = add i32 %b2, %b3
+  %hadd0 = insertelement <4 x i32> poison, i32 %a01, i32 0
+  %hadd1 = insertelement <4 x i32> %hadd0, i32 %a23, i32 1
+  %hadd2 = insertelement <4 x i32> %hadd1, i32 %b01, i32 2
+  %hadd3 = insertelement <4 x i32> %hadd2, i32 %b23, i32 3
+  %result = shufflevector <4 x i32> %hadd3, <4 x i32> %a, <4 x i32> <i32 3, i32 2, i32 poison, i32 0>
+  ret <4 x i32> %result
+}
+
 ;
 ; v8i32
 ;
@@ -384,31 +812,11 @@ define <8 x i32> @add_v8i32_01234567(<8 x i32> %a, <8 x i32> %b) {
 }
 
 define <8 x i32> @add_v8i32_01234u67(<8 x i32> %a, <8 x i32> %b) {
-; SSE2-LABEL: @add_v8i32_01234u67(
-; SSE2-NEXT:    [[TMP2:%.*]] = shufflevector <8 x i32> [[A:%.*]], <8 x i32> [[B:%.*]], <8 x i32> <i32 0, i32 2, i32 8, i32 10, i32 4, i32 poison, i32 13, i32 14>
-; SSE2-NEXT:    [[TMP3:%.*]] = shufflevector <8 x i32> [[A]], <8 x i32> [[B]], <8 x i32> <i32 1, i32 3, i32 9, i32 11, i32 5, i32 poison, i32 12, i32 15>
-; SSE2-NEXT:    [[TMP4:%.*]] = add <8 x i32> [[TMP2]], [[TMP3]]
-; SSE2-NEXT:    ret <8 x i32> [[TMP4]]
-;
-; SSE4-LABEL: @add_v8i32_01234u67(
-; SSE4-NEXT:    [[A4:%.*]] = extractelement <8 x i32> [[A:%.*]], i64 4
-; SSE4-NEXT:    [[A5:%.*]] = extractelement <8 x i32> [[A]], i64 5
-; SSE4-NEXT:    [[A45:%.*]] = add i32 [[A4]], [[A5]]
-; SSE4-NEXT:    [[TMP2:%.*]] = shufflevector <8 x i32> [[A]], <8 x i32> [[B:%.*]], <8 x i32> <i32 0, i32 2, i32 8, i32 10, i32 poison, i32 poison, i32 poison, i32 poison>
-; SSE4-NEXT:    [[TMP5:%.*]] = shufflevector <8 x i32> [[A]], <8 x i32> [[B]], <8 x i32> <i32 1, i32 3, i32 9, i32 11, i32 poison, i32 poison, i32 poison, i32 poison>
-; SSE4-NEXT:    [[TMP3:%.*]] = add <8 x i32> [[TMP2]], [[TMP5]]
-; SSE4-NEXT:    [[HADD4:%.*]] = insertelement <8 x i32> [[TMP3]], i32 [[A45]], i64 4
-; SSE4-NEXT:    [[TMP4:%.*]] = shufflevector <8 x i32> [[B]], <8 x i32> poison, <8 x i32> <i32 5, i32 6, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; SSE4-NEXT:    [[TMP7:%.*]] = shufflevector <8 x i32> [[B]], <8 x i32> poison, <8 x i32> <i32 4, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; SSE4-NEXT:    [[TMP6:%.*]] = add <8 x i32> [[TMP4]], [[TMP7]]
-; SSE4-NEXT:    [[RESULT:%.*]] = shufflevector <8 x i32> [[HADD4]], <8 x i32> [[TMP6]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 poison, i32 8, i32 9>
-; SSE4-NEXT:    ret <8 x i32> [[RESULT]]
-;
-; AVX-LABEL: @add_v8i32_01234u67(
-; AVX-NEXT:    [[TMP5:%.*]] = shufflevector <8 x i32> [[A:%.*]], <8 x i32> [[B:%.*]], <8 x i32> <i32 0, i32 2, i32 8, i32 10, i32 4, i32 poison, i32 13, i32 14>
-; AVX-NEXT:    [[TMP6:%.*]] = shufflevector <8 x i32> [[A]], <8 x i32> [[B]], <8 x i32> <i32 1, i32 3, i32 9, i32 11, i32 5, i32 poison, i32 12, i32 15>
-; AVX-NEXT:    [[TMP7:%.*]] = add <8 x i32> [[TMP5]], [[TMP6]]
-; AVX-NEXT:    ret <8 x i32> [[TMP7]]
+; CHECK-LABEL: @add_v8i32_01234u67(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <8 x i32> [[A:%.*]], <8 x i32> [[B:%.*]], <8 x i32> <i32 0, i32 2, i32 8, i32 10, i32 4, i32 poison, i32 13, i32 14>
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <8 x i32> [[A]], <8 x i32> [[B]], <8 x i32> <i32 1, i32 3, i32 9, i32 11, i32 5, i32 poison, i32 12, i32 15>
+; CHECK-NEXT:    [[RESULT:%.*]] = add <8 x i32> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    ret <8 x i32> [[RESULT]]
 ;
   %a0 = extractelement <8 x i32> %a, i32 0
   %a1 = extractelement <8 x i32> %a, i32 1
@@ -605,8 +1013,8 @@ define <4 x float> @add_v4f32_012u(<4 x float> %a, <4 x float> %b) {
 
 define <4 x float> @add_v4f32_uu23(<4 x float> %a, <4 x float> %b) {
 ; CHECK-LABEL: @add_v4f32_uu23(
-; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <4 x float> [[TMP1:%.*]], <4 x float> poison, <4 x i32> <i32 poison, i32 poison, i32 1, i32 2>
-; CHECK-NEXT:    [[RESULT1:%.*]] = shufflevector <4 x float> [[TMP1]], <4 x float> poison, <4 x i32> <i32 poison, i32 poison, i32 0, i32 3>
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <4 x float> [[B:%.*]], <4 x float> poison, <4 x i32> <i32 poison, i32 poison, i32 1, i32 2>
+; CHECK-NEXT:    [[RESULT1:%.*]] = shufflevector <4 x float> [[B]], <4 x float> poison, <4 x i32> <i32 poison, i32 poison, i32 0, i32 3>
 ; CHECK-NEXT:    [[RESULT2:%.*]] = fadd <4 x float> [[TMP2]], [[RESULT1]]
 ; CHECK-NEXT:    ret <4 x float> [[RESULT2]]
 ;
@@ -632,8 +1040,8 @@ define <4 x float> @add_v4f32_uu23(<4 x float> %a, <4 x float> %b) {
 
 define <4 x float> @add_v4f32_01uu(<4 x float> %a, <4 x float> %b) {
 ; CHECK-LABEL: @add_v4f32_01uu(
-; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <4 x float> [[TMP1:%.*]], <4 x float> poison, <4 x i32> <i32 1, i32 2, i32 poison, i32 poison>
-; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <4 x float> [[TMP1]], <4 x float> poison, <4 x i32> <i32 0, i32 3, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <4 x float> [[A:%.*]], <4 x float> poison, <4 x i32> <i32 1, i32 2, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <4 x float> [[A]], <4 x float> poison, <4 x i32> <i32 0, i32 3, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP4:%.*]] = fadd <4 x float> [[TMP2]], [[TMP3]]
 ; CHECK-NEXT:    ret <4 x float> [[TMP4]]
 ;
@@ -706,33 +1114,21 @@ define <8 x float> @add_v8f32_01234567(<8 x float> %a, <8 x float> %b) {
 
 define <8 x float> @add_v8f32_012u4567(<8 x float> %a, <8 x float> %b) {
 ; SSE2-LABEL: @add_v8f32_012u4567(
-; SSE2-NEXT:    [[A6:%.*]] = extractelement <8 x float> [[A:%.*]], i64 6
-; SSE2-NEXT:    [[A7:%.*]] = extractelement <8 x float> [[A]], i64 7
-; SSE2-NEXT:    [[A67:%.*]] = fadd float [[A6]], [[A7]]
 ; SSE2-NEXT:    [[TMP1:%.*]] = shufflevector <8 x float> [[B:%.*]], <8 x float> poison, <2 x i32> <i32 5, i32 6>
 ; SSE2-NEXT:    [[TMP2:%.*]] = shufflevector <8 x float> [[B]], <8 x float> poison, <2 x i32> <i32 4, i32 7>
 ; SSE2-NEXT:    [[TMP3:%.*]] = fadd <2 x float> [[TMP1]], [[TMP2]]
-; SSE2-NEXT:    [[TMP5:%.*]] = shufflevector <8 x float> [[A]], <8 x float> [[B]], <8 x i32> <i32 0, i32 2, i32 8, i32 poison, i32 4, i32 poison, i32 poison, i32 poison>
-; SSE2-NEXT:    [[TMP8:%.*]] = shufflevector <8 x float> [[A]], <8 x float> [[B]], <8 x i32> <i32 1, i32 3, i32 9, i32 poison, i32 5, i32 poison, i32 poison, i32 poison>
+; SSE2-NEXT:    [[TMP5:%.*]] = shufflevector <8 x float> [[A:%.*]], <8 x float> [[B]], <8 x i32> <i32 0, i32 2, i32 8, i32 poison, i32 4, i32 6, i32 poison, i32 poison>
+; SSE2-NEXT:    [[TMP8:%.*]] = shufflevector <8 x float> [[A]], <8 x float> [[B]], <8 x i32> <i32 1, i32 3, i32 9, i32 poison, i32 5, i32 7, i32 poison, i32 poison>
 ; SSE2-NEXT:    [[TMP6:%.*]] = fadd <8 x float> [[TMP5]], [[TMP8]]
-; SSE2-NEXT:    [[HADD5:%.*]] = insertelement <8 x float> [[TMP6]], float [[A67]], i64 5
 ; SSE2-NEXT:    [[TMP7:%.*]] = shufflevector <2 x float> [[TMP3]], <2 x float> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; SSE2-NEXT:    [[RESULT:%.*]] = shufflevector <8 x float> [[HADD5]], <8 x float> [[TMP7]], <8 x i32> <i32 0, i32 1, i32 2, i32 poison, i32 4, i32 5, i32 8, i32 9>
+; SSE2-NEXT:    [[RESULT:%.*]] = shufflevector <8 x float> [[TMP6]], <8 x float> [[TMP7]], <8 x i32> <i32 0, i32 1, i32 2, i32 poison, i32 4, i32 5, i32 8, i32 9>
 ; SSE2-NEXT:    ret <8 x float> [[RESULT]]
 ;
 ; SSE4-LABEL: @add_v8f32_012u4567(
-; SSE4-NEXT:    [[A6:%.*]] = extractelement <8 x float> [[A:%.*]], i64 6
-; SSE4-NEXT:    [[A7:%.*]] = extractelement <8 x float> [[A]], i64 7
-; SSE4-NEXT:    [[A67:%.*]] = fadd float [[A6]], [[A7]]
-; SSE4-NEXT:    [[TMP2:%.*]] = shufflevector <8 x float> [[A]], <8 x float> [[B:%.*]], <8 x i32> <i32 0, i32 2, i32 8, i32 poison, i32 4, i32 poison, i32 poison, i32 poison>
-; SSE4-NEXT:    [[TMP5:%.*]] = shufflevector <8 x float> [[A]], <8 x float> [[B]], <8 x i32> <i32 1, i32 3, i32 9, i32 poison, i32 5, i32 poison, i32 poison, i32 poison>
-; SSE4-NEXT:    [[TMP3:%.*]] = fadd <8 x float> [[TMP2]], [[TMP5]]
-; SSE4-NEXT:    [[HADD5:%.*]] = insertelement <8 x float> [[TMP3]], float [[A67]], i64 5
-; SSE4-NEXT:    [[TMP4:%.*]] = shufflevector <8 x float> [[B]], <8 x float> poison, <8 x i32> <i32 5, i32 6, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; SSE4-NEXT:    [[TMP7:%.*]] = shufflevector <8 x float> [[B]], <8 x float> poison, <8 x i32> <i32 4, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; SSE4-NEXT:    [[TMP6:%.*]] = fadd <8 x float> [[TMP4]], [[TMP7]]
-; SSE4-NEXT:    [[RESULT:%.*]] = shufflevector <8 x float> [[HADD5]], <8 x float> [[TMP6]], <8 x i32> <i32 0, i32 1, i32 2, i32 poison, i32 4, i32 5, i32 8, i32 9>
-; SSE4-NEXT:    ret <8 x float> [[RESULT]]
+; SSE4-NEXT:    [[TMP8:%.*]] = shufflevector <8 x float> [[A:%.*]], <8 x float> [[B:%.*]], <8 x i32> <i32 0, i32 2, i32 8, i32 poison, i32 4, i32 6, i32 13, i32 14>
+; SSE4-NEXT:    [[TMP5:%.*]] = shufflevector <8 x float> [[A]], <8 x float> [[B]], <8 x i32> <i32 1, i32 3, i32 9, i32 poison, i32 5, i32 7, i32 12, i32 15>
+; SSE4-NEXT:    [[TMP9:%.*]] = fadd <8 x float> [[TMP8]], [[TMP5]]
+; SSE4-NEXT:    ret <8 x float> [[TMP9]]
 ;
 ; AVX-LABEL: @add_v8f32_012u4567(
 ; AVX-NEXT:    [[TMP5:%.*]] = shufflevector <8 x float> [[A:%.*]], <8 x float> [[B:%.*]], <8 x i32> <i32 0, i32 2, i32 8, i32 poison, i32 4, i32 6, i32 13, i32 14>
@@ -776,6 +1172,71 @@ define <8 x float> @add_v8f32_012u4567(<8 x float> %a, <8 x float> %b) {
   ret <8 x float> %result
 }
 
+define <8 x float> @add_v8f32_76u43210(<8 x float> %a, <8 x float> %b) {
+; SSE2-LABEL: @add_v8f32_76u43210(
+; SSE2-NEXT:    [[TMP1:%.*]] = shufflevector <8 x float> [[A:%.*]], <8 x float> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+; SSE2-NEXT:    [[TMP2:%.*]] = shufflevector <8 x float> [[A]], <8 x float> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+; SSE2-NEXT:    [[TMP3:%.*]] = fadd <4 x float> [[TMP1]], [[TMP2]]
+; SSE2-NEXT:    [[B0:%.*]] = extractelement <8 x float> [[B:%.*]], i64 0
+; SSE2-NEXT:    [[B1:%.*]] = extractelement <8 x float> [[B]], i64 1
+; SSE2-NEXT:    [[B01:%.*]] = fadd float [[B0]], [[B1]]
+; SSE2-NEXT:    [[TMP4:%.*]] = shufflevector <8 x float> [[B]], <8 x float> poison, <2 x i32> <i32 5, i32 6>
+; SSE2-NEXT:    [[TMP5:%.*]] = shufflevector <8 x float> [[B]], <8 x float> poison, <2 x i32> <i32 4, i32 7>
+; SSE2-NEXT:    [[TMP6:%.*]] = fadd <2 x float> [[TMP4]], [[TMP5]]
+; SSE2-NEXT:    [[TMP7:%.*]] = shufflevector <4 x float> [[TMP3]], <4 x float> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE2-NEXT:    [[HADD4:%.*]] = insertelement <8 x float> [[TMP7]], float [[B01]], i64 4
+; SSE2-NEXT:    [[TMP8:%.*]] = shufflevector <2 x float> [[TMP6]], <2 x float> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE2-NEXT:    [[RESULT:%.*]] = shufflevector <8 x float> [[TMP8]], <8 x float> [[HADD4]], <8 x i32> <i32 1, i32 0, i32 poison, i32 12, i32 11, i32 10, i32 9, i32 8>
+; SSE2-NEXT:    ret <8 x float> [[RESULT]]
+;
+; SSE4-LABEL: @add_v8f32_76u43210(
+; SSE4-NEXT:    [[TMP4:%.*]] = shufflevector <8 x float> [[B:%.*]], <8 x float> [[A:%.*]], <8 x i32> <i32 6, i32 5, i32 poison, i32 0, i32 14, i32 12, i32 10, i32 8>
+; SSE4-NEXT:    [[TMP5:%.*]] = shufflevector <8 x float> [[B]], <8 x float> [[A]], <8 x i32> <i32 7, i32 4, i32 poison, i32 1, i32 15, i32 13, i32 11, i32 9>
+; SSE4-NEXT:    [[TMP6:%.*]] = fadd <8 x float> [[TMP4]], [[TMP5]]
+; SSE4-NEXT:    ret <8 x float> [[TMP6]]
+;
+; AVX-LABEL: @add_v8f32_76u43210(
+; AVX-NEXT:    [[TMP1:%.*]] = shufflevector <8 x float> [[B:%.*]], <8 x float> [[A:%.*]], <8 x i32> <i32 6, i32 5, i32 poison, i32 0, i32 14, i32 12, i32 10, i32 8>
+; AVX-NEXT:    [[TMP2:%.*]] = shufflevector <8 x float> [[B]], <8 x float> [[A]], <8 x i32> <i32 7, i32 4, i32 poison, i32 1, i32 15, i32 13, i32 11, i32 9>
+; AVX-NEXT:    [[RESULT:%.*]] = fadd <8 x float> [[TMP1]], [[TMP2]]
+; AVX-NEXT:    ret <8 x float> [[RESULT]]
+;
+  %a0 = extractelement <8 x float> %a, i32 0
+  %a1 = extractelement <8 x float> %a, i32 1
+  %a2 = extractelement <8 x float> %a, i32 2
+  %a3 = extractelement <8 x float> %a, i32 3
+  %a4 = extractelement <8 x float> %a, i32 4
+  %a5 = extractelement <8 x float> %a, i32 5
+  %a6 = extractelement <8 x float> %a, i32 6
+  %a7 = extractelement <8 x float> %a, i32 7
+  %a01 = fadd float %a0, %a1
+  %a23 = fadd float %a2, %a3
+  %a45 = fadd float %a4, %a5
+  %a67 = fadd float %a6, %a7
+  %b0 = extractelement <8 x float> %b, i32 0
+  %b1 = extractelement <8 x float> %b, i32 1
+  %b2 = extractelement <8 x float> %b, i32 2
+  %b3 = extractelement <8 x float> %b, i32 3
+  %b4 = extractelement <8 x float> %b, i32 4
+  %b5 = extractelement <8 x float> %b, i32 5
+  %b6 = extractelement <8 x float> %b, i32 6
+  %b7 = extractelement <8 x float> %b, i32 7
+  %b01 = fadd float %b0, %b1
+  %b23 = fadd float %b2, %b3
+  %b45 = fadd float %b4, %b5
+  %b67 = fadd float %b6, %b7
+  %hadd0 = insertelement <8 x float> poison, float %a01, i32 0
+  %hadd1 = insertelement <8 x float> %hadd0, float %a23, i32 1
+  %hadd2 = insertelement <8 x float> %hadd1, float %a45, i32 2
+  %hadd3 = insertelement <8 x float> %hadd2, float %a67, i32 3
+  %hadd4 = insertelement <8 x float> %hadd3, float %b01, i32 4
+  %hadd5 = insertelement <8 x float> %hadd4, float %b23, i32 5
+  %hadd6 = insertelement <8 x float> %hadd5, float %b45, i32 6
+  %hadd7 = insertelement <8 x float> %hadd6, float %b67, i32 7
+  %result = shufflevector <8 x float> %hadd7, <8 x float> %a, <8 x i32> <i32 7, i32 6, i32 poison, i32 4, i32 3, i32 2, i32 1, i32 0>
+  ret <8 x float> %result
+}
+
 ;
 ; v2f64
 ;
@@ -879,14 +1340,10 @@ define <4 x double> @add_v4f64_u123(<4 x double> %a, <4 x double> %b) {
 ; SSE2-NEXT:    ret <4 x double> [[RESULT]]
 ;
 ; SSE4-LABEL: @add_v4f64_u123(
-; SSE4-NEXT:    [[B2:%.*]] = extractelement <4 x double> [[B:%.*]], i64 2
-; SSE4-NEXT:    [[B3:%.*]] = extractelement <4 x double> [[B]], i64 3
-; SSE4-NEXT:    [[B23:%.*]] = fadd double [[B2]], [[B3]]
-; SSE4-NEXT:    [[TMP2:%.*]] = shufflevector <4 x double> [[B]], <4 x double> [[A:%.*]], <4 x i32> <i32 poison, i32 0, i32 6, i32 poison>
-; SSE4-NEXT:    [[TMP4:%.*]] = shufflevector <4 x double> [[B]], <4 x double> [[A]], <4 x i32> <i32 poison, i32 1, i32 7, i32 poison>
+; SSE4-NEXT:    [[TMP2:%.*]] = shufflevector <4 x double> [[B:%.*]], <4 x double> [[A:%.*]], <4 x i32> <i32 poison, i32 0, i32 6, i32 2>
+; SSE4-NEXT:    [[TMP4:%.*]] = shufflevector <4 x double> [[B]], <4 x double> [[A]], <4 x i32> <i32 poison, i32 1, i32 7, i32 3>
 ; SSE4-NEXT:    [[TMP3:%.*]] = fadd <4 x double> [[TMP2]], [[TMP4]]
-; SSE4-NEXT:    [[RESULT:%.*]] = insertelement <4 x double> [[TMP3]], double [[B23]], i64 3
-; SSE4-NEXT:    ret <4 x double> [[RESULT]]
+; SSE4-NEXT:    ret <4 x double> [[TMP3]]
 ;
 ; AVX-LABEL: @add_v4f64_u123(
 ; AVX-NEXT:    [[TMP2:%.*]] = shufflevector <4 x double> [[B:%.*]], <4 x double> [[A:%.*]], <4 x i32> <i32 poison, i32 0, i32 6, i32 2>
@@ -927,14 +1384,10 @@ define <4 x double> @add_v4f64_0u23(<4 x double> %a, <4 x double> %b) {
 ; SSE2-NEXT:    ret <4 x double> [[RESULT]]
 ;
 ; SSE4-LABEL: @add_v4f64_0u23(
-; SSE4-NEXT:    [[B2:%.*]] = extractelement <4 x double> [[B:%.*]], i64 2
-; SSE4-NEXT:    [[B3:%.*]] = extractelement <4 x double> [[B]], i64 3
-; SSE4-NEXT:    [[B23:%.*]] = fadd double [[B2]], [[B3]]
-; SSE4-NEXT:    [[TMP2:%.*]] = shufflevector <4 x double> [[TMP1:%.*]], <4 x double> poison, <4 x i32> <i32 1, i32 poison, i32 2, i32 poison>
-; SSE4-NEXT:    [[TMP3:%.*]] = shufflevector <4 x double> [[TMP1]], <4 x double> poison, <4 x i32> <i32 0, i32 poison, i32 3, i32 poison>
+; SSE4-NEXT:    [[TMP2:%.*]] = shufflevector <4 x double> [[A:%.*]], <4 x double> [[B:%.*]], <4 x i32> <i32 1, i32 poison, i32 2, i32 6>
+; SSE4-NEXT:    [[TMP3:%.*]] = shufflevector <4 x double> [[A]], <4 x double> [[B]], <4 x i32> <i32 0, i32 poison, i32 3, i32 7>
 ; SSE4-NEXT:    [[TMP4:%.*]] = fadd <4 x double> [[TMP2]], [[TMP3]]
-; SSE4-NEXT:    [[RESULT:%.*]] = insertelement <4 x double> [[TMP4]], double [[B23]], i64 3
-; SSE4-NEXT:    ret <4 x double> [[RESULT]]
+; SSE4-NEXT:    ret <4 x double> [[TMP4]]
 ;
 ; AVX-LABEL: @add_v4f64_0u23(
 ; AVX-NEXT:    [[TMP2:%.*]] = shufflevector <4 x double> [[A:%.*]], <4 x double> [[B:%.*]], <4 x i32> <i32 1, i32 poison, i32 2, i32 6>
@@ -975,14 +1428,10 @@ define <4 x double> @add_v4f64_01u3(<4 x double> %a, <4 x double> %b) {
 ; SSE2-NEXT:    ret <4 x double> [[RESULT]]
 ;
 ; SSE4-LABEL: @add_v4f64_01u3(
-; SSE4-NEXT:    [[B2:%.*]] = extractelement <4 x double> [[B:%.*]], i64 2
-; SSE4-NEXT:    [[B3:%.*]] = extractelement <4 x double> [[B]], i64 3
-; SSE4-NEXT:    [[B23:%.*]] = fadd double [[B2]], [[B3]]
-; SSE4-NEXT:    [[TMP2:%.*]] = shufflevector <4 x double> [[A:%.*]], <4 x double> [[B]], <4 x i32> <i32 0, i32 4, i32 poison, i32 poison>
-; SSE4-NEXT:    [[TMP4:%.*]] = shufflevector <4 x double> [[A]], <4 x double> [[B]], <4 x i32> <i32 1, i32 5, i32 poison, i32 poison>
+; SSE4-NEXT:    [[TMP2:%.*]] = shufflevector <4 x double> [[A:%.*]], <4 x double> [[B:%.*]], <4 x i32> <i32 0, i32 4, i32 poison, i32 6>
+; SSE4-NEXT:    [[TMP4:%.*]] = shufflevector <4 x double> [[A]], <4 x double> [[B]], <4 x i32> <i32 1, i32 5, i32 poison, i32 7>
 ; SSE4-NEXT:    [[TMP3:%.*]] = fadd <4 x double> [[TMP2]], [[TMP4]]
-; SSE4-NEXT:    [[RESULT:%.*]] = insertelement <4 x double> [[TMP3]], double [[B23]], i64 3
-; SSE4-NEXT:    ret <4 x double> [[RESULT]]
+; SSE4-NEXT:    ret <4 x double> [[TMP3]]
 ;
 ; AVX-LABEL: @add_v4f64_01u3(
 ; AVX-NEXT:    [[TMP2:%.*]] = shufflevector <4 x double> [[A:%.*]], <4 x double> [[B:%.*]], <4 x i32> <i32 0, i32 4, i32 poison, i32 6>
@@ -1023,14 +1472,10 @@ define <4 x double> @add_v4f64_012u(<4 x double> %a, <4 x double> %b) {
 ; SSE2-NEXT:    ret <4 x double> [[RESULT]]
 ;
 ; SSE4-LABEL: @add_v4f64_012u(
-; SSE4-NEXT:    [[A2:%.*]] = extractelement <4 x double> [[A:%.*]], i64 2
-; SSE4-NEXT:    [[A3:%.*]] = extractelement <4 x double> [[A]], i64 3
-; SSE4-NEXT:    [[A23:%.*]] = fadd double [[A2]], [[A3]]
-; SSE4-NEXT:    [[TMP2:%.*]] = shufflevector <4 x double> [[A]], <4 x double> [[B:%.*]], <4 x i32> <i32 0, i32 4, i32 poison, i32 poison>
-; SSE4-NEXT:    [[TMP4:%.*]] = shufflevector <4 x double> [[A]], <4 x double> [[B]], <4 x i32> <i32 1, i32 5, i32 poison, i32 poison>
+; SSE4-NEXT:    [[TMP2:%.*]] = shufflevector <4 x double> [[A:%.*]], <4 x double> [[B:%.*]], <4 x i32> <i32 0, i32 4, i32 2, i32 poison>
+; SSE4-NEXT:    [[TMP4:%.*]] = shufflevector <4 x double> [[A]], <4 x double> [[B]], <4 x i32> <i32 1, i32 5, i32 3, i32 poison>
 ; SSE4-NEXT:    [[TMP3:%.*]] = fadd <4 x double> [[TMP2]], [[TMP4]]
-; SSE4-NEXT:    [[RESULT:%.*]] = insertelement <4 x double> [[TMP3]], double [[A23]], i64 2
-; SSE4-NEXT:    ret <4 x double> [[RESULT]]
+; SSE4-NEXT:    ret <4 x double> [[TMP3]]
 ;
 ; AVX-LABEL: @add_v4f64_012u(
 ; AVX-NEXT:    [[TMP2:%.*]] = shufflevector <4 x double> [[A:%.*]], <4 x double> [[B:%.*]], <4 x i32> <i32 0, i32 4, i32 2, i32 poison>
@@ -1137,3 +1582,47 @@ define <4 x double> @add_v4f64_01uu(<4 x double> %a, <4 x double> %b) {
   %result = shufflevector <4 x double> %hadd3, <4 x double> %a, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
   ret <4 x double> %result
 }
+
+define <4 x double> @add_v4f64_32u0(<4 x double> %a, <4 x double> %b) {
+; SSE2-LABEL: @add_v4f64_32u0(
+; SSE2-NEXT:    [[A0:%.*]] = extractelement <4 x double> [[A:%.*]], i64 0
+; SSE2-NEXT:    [[A1:%.*]] = extractelement <4 x double> [[A]], i64 1
+; SSE2-NEXT:    [[A01:%.*]] = fadd double [[A0]], [[A1]]
+; SSE2-NEXT:    [[TMP1:%.*]] = shufflevector <4 x double> [[B:%.*]], <4 x double> poison, <2 x i32> <i32 2, i32 0>
+; SSE2-NEXT:    [[TMP2:%.*]] = shufflevector <4 x double> [[B]], <4 x double> poison, <2 x i32> <i32 3, i32 1>
+; SSE2-NEXT:    [[TMP3:%.*]] = fadd <2 x double> [[TMP1]], [[TMP2]]
+; SSE2-NEXT:    [[TMP4:%.*]] = shufflevector <2 x double> [[TMP3]], <2 x double> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; SSE2-NEXT:    [[RESULT:%.*]] = insertelement <4 x double> [[TMP4]], double [[A01]], i64 3
+; SSE2-NEXT:    ret <4 x double> [[RESULT]]
+;
+; SSE4-LABEL: @add_v4f64_32u0(
+; SSE4-NEXT:    [[TMP1:%.*]] = shufflevector <4 x double> [[B:%.*]], <4 x double> [[A:%.*]], <4 x i32> <i32 2, i32 0, i32 poison, i32 4>
+; SSE4-NEXT:    [[TMP2:%.*]] = shufflevector <4 x double> [[B]], <4 x double> [[A]], <4 x i32> <i32 3, i32 1, i32 poison, i32 5>
+; SSE4-NEXT:    [[RESULT:%.*]] = fadd <4 x double> [[TMP1]], [[TMP2]]
+; SSE4-NEXT:    ret <4 x double> [[RESULT]]
+;
+; AVX-LABEL: @add_v4f64_32u0(
+; AVX-NEXT:    [[TMP1:%.*]] = shufflevector <4 x double> [[B:%.*]], <4 x double> [[A:%.*]], <4 x i32> <i32 2, i32 0, i32 poison, i32 4>
+; AVX-NEXT:    [[TMP2:%.*]] = shufflevector <4 x double> [[B]], <4 x double> [[A]], <4 x i32> <i32 3, i32 1, i32 poison, i32 5>
+; AVX-NEXT:    [[RESULT:%.*]] = fadd <4 x double> [[TMP1]], [[TMP2]]
+; AVX-NEXT:    ret <4 x double> [[RESULT]]
+;
+  %a0 = extractelement <4 x double> %a, i32 0
+  %a1 = extractelement <4 x double> %a, i32 1
+  %a2 = extractelement <4 x double> %a, i32 2
+  %a3 = extractelement <4 x double> %a, i32 3
+  %a01 = fadd double %a0, %a1
+  %a23 = fadd double %a2, %a3
+  %b0 = extractelement <4 x double> %b, i32 0
+  %b1 = extractelement <4 x double> %b, i32 1
+  %b2 = extractelement <4 x double> %b, i32 2
+  %b3 = extractelement <4 x double> %b, i32 3
+  %b01 = fadd double %b0, %b1
+  %b23 = fadd double %b2, %b3
+  %hadd0 = insertelement <4 x double> poison, double %a01, i32 0
+  %hadd1 = insertelement <4 x double> %hadd0, double %a23, i32 1
+  %hadd2 = insertelement <4 x double> %hadd1, double %b01, i32 2
+  %hadd3 = insertelement <4 x double> %hadd2, double %b23, i32 3
+  %result = shufflevector <4 x double> %hadd3, <4 x double> %a, <4 x i32> <i32 3, i32 2, i32 poison, i32 0>
+  ret <4 x double> %result
+}
diff --git a/llvm/test/Transforms/PhaseOrdering/X86/hoist-load-of-baseptr.ll b/llvm/test/Transforms/PhaseOrdering/X86/hoist-load-of-baseptr.ll
index 8bc40cfc5cc8b..dfad53411aa55 100644
--- a/llvm/test/Transforms/PhaseOrdering/X86/hoist-load-of-baseptr.ll
+++ b/llvm/test/Transforms/PhaseOrdering/X86/hoist-load-of-baseptr.ll
@@ -15,7 +15,7 @@ $_ZNSt6vectorIiSaIiEEixEm = comdat any
 
 define dso_local void @_Z7computeRSt6vectorIiSaIiEEy(ptr noundef nonnull align 8 dereferenceable(24) %data, i64 noundef %numElems) {
 ; O1-LABEL: define {{[^@]+}}@_Z7computeRSt6vectorIiSaIiEEy
-; O1-SAME: (ptr nocapture noundef nonnull readonly align 8 dereferenceable(24) [[DATA:%.*]], i64 noundef [[NUMELEMS:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
+; O1-SAME: (ptr noundef nonnull readonly align 8 captures(none) dereferenceable(24) [[DATA:%.*]], i64 noundef [[NUMELEMS:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
 ; O1-NEXT:  entry:
 ; O1-NEXT:    [[CMP24_NOT:%.*]] = icmp eq i64 [[NUMELEMS]], 0
 ; O1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[DATA]], align 8
@@ -40,7 +40,7 @@ define dso_local void @_Z7computeRSt6vectorIiSaIiEEy(ptr noundef nonnull align 8
 ; O1-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_COND_CLEANUP3]], label [[FOR_BODY4]], !llvm.loop [[LOOP6:![0-9]+]]
 ;
 ; O2-LABEL: define {{[^@]+}}@_Z7computeRSt6vectorIiSaIiEEy
-; O2-SAME: (ptr nocapture noundef nonnull readonly align 8 dereferenceable(24) [[DATA:%.*]], i64 noundef [[NUMELEMS:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
+; O2-SAME: (ptr noundef nonnull readonly align 8 captures(none) dereferenceable(24) [[DATA:%.*]], i64 noundef [[NUMELEMS:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
 ; O2-NEXT:  entry:
 ; O2-NEXT:    [[CMP24_NOT:%.*]] = icmp eq i64 [[NUMELEMS]], 0
 ; O2-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[DATA]], align 8
@@ -88,7 +88,7 @@ define dso_local void @_Z7computeRSt6vectorIiSaIiEEy(ptr noundef nonnull align 8
 ; O2-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_COND_CLEANUP3]], label [[FOR_BODY4]], !llvm.loop [[LOOP9:![0-9]+]]
 ;
 ; O3-LABEL: define {{[^@]+}}@_Z7computeRSt6vectorIiSaIiEEy
-; O3-SAME: (ptr nocapture noundef nonnull readonly align 8 dereferenceable(24) [[DATA:%.*]], i64 noundef [[NUMELEMS:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
+; O3-SAME: (ptr noundef nonnull readonly align 8 captures(none) dereferenceable(24) [[DATA:%.*]], i64 noundef [[NUMELEMS:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
 ; O3-NEXT:  entry:
 ; O3-NEXT:    [[CMP24_NOT:%.*]] = icmp eq i64 [[NUMELEMS]], 0
 ; O3-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[DATA]], align 8
diff --git a/llvm/test/Transforms/PhaseOrdering/X86/hsub.ll b/llvm/test/Transforms/PhaseOrdering/X86/hsub.ll
index c9cba0a4cc0ff..0062527b678c9 100644
--- a/llvm/test/Transforms/PhaseOrdering/X86/hsub.ll
+++ b/llvm/test/Transforms/PhaseOrdering/X86/hsub.ll
@@ -59,18 +59,12 @@ define <8 x i16> @sub_v8i16_01234567(<8 x i16> %a, <8 x i16> %b) {
 
 define <8 x i16> @sub_v8i16_u1234567(<8 x i16> %a, <8 x i16> %b) {
 ; SSE2-LABEL: @sub_v8i16_u1234567(
-; SSE2-NEXT:    [[A2:%.*]] = extractelement <8 x i16> [[A:%.*]], i64 2
-; SSE2-NEXT:    [[A3:%.*]] = extractelement <8 x i16> [[A]], i64 3
-; SSE2-NEXT:    [[A4:%.*]] = extractelement <8 x i16> [[A]], i64 4
-; SSE2-NEXT:    [[A5:%.*]] = extractelement <8 x i16> [[A]], i64 5
-; SSE2-NEXT:    [[A6:%.*]] = extractelement <8 x i16> [[A]], i64 6
-; SSE2-NEXT:    [[A7:%.*]] = extractelement <8 x i16> [[A]], i64 7
-; SSE2-NEXT:    [[A23:%.*]] = sub i16 [[A2]], [[A3]]
-; SSE2-NEXT:    [[A45:%.*]] = sub i16 [[A4]], [[A5]]
-; SSE2-NEXT:    [[A67:%.*]] = sub i16 [[A6]], [[A7]]
-; SSE2-NEXT:    [[HSUB1:%.*]] = insertelement <8 x i16> poison, i16 [[A23]], i64 1
-; SSE2-NEXT:    [[HSUB2:%.*]] = insertelement <8 x i16> [[HSUB1]], i16 [[A45]], i64 2
-; SSE2-NEXT:    [[HSUB3:%.*]] = insertelement <8 x i16> [[HSUB2]], i16 [[A67]], i64 3
+; SSE2-NEXT:    [[SHIFT3:%.*]] = shufflevector <8 x i16> [[A:%.*]], <8 x i16> poison, <8 x i32> <i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 7, i32 poison>
+; SSE2-NEXT:    [[TMP6:%.*]] = sub <8 x i16> [[A]], [[SHIFT3]]
+; SSE2-NEXT:    [[TMP4:%.*]] = shufflevector <8 x i16> [[A]], <8 x i16> poison, <8 x i32> <i32 poison, i32 2, i32 4, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE2-NEXT:    [[TMP5:%.*]] = shufflevector <8 x i16> [[A]], <8 x i16> poison, <8 x i32> <i32 poison, i32 3, i32 5, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE2-NEXT:    [[HSUB22:%.*]] = sub <8 x i16> [[TMP4]], [[TMP5]]
+; SSE2-NEXT:    [[HSUB3:%.*]] = shufflevector <8 x i16> [[HSUB22]], <8 x i16> [[TMP6]], <8 x i32> <i32 poison, i32 1, i32 2, i32 14, i32 poison, i32 poison, i32 poison, i32 poison>
 ; SSE2-NEXT:    [[TMP1:%.*]] = shufflevector <8 x i16> [[B:%.*]], <8 x i16> poison, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 poison, i32 poison, i32 poison, i32 poison>
 ; SSE2-NEXT:    [[TMP2:%.*]] = shufflevector <8 x i16> [[B]], <8 x i16> poison, <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 poison, i32 poison, i32 poison, i32 poison>
 ; SSE2-NEXT:    [[TMP3:%.*]] = sub <8 x i16> [[TMP1]], [[TMP2]]
@@ -125,6 +119,389 @@ define <8 x i16> @sub_v8i16_u1234567(<8 x i16> %a, <8 x i16> %b) {
   ret <8 x i16> %result
 }
 
+define <8 x i16> @sub_v8i16_76u43210(<8 x i16> %a, <8 x i16> %b) {
+; SSE2-LABEL: @sub_v8i16_76u43210(
+; SSE2-NEXT:    [[SHIFT:%.*]] = shufflevector <8 x i16> [[A:%.*]], <8 x i16> poison, <8 x i32> <i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE2-NEXT:    [[TMP1:%.*]] = sub <8 x i16> [[A]], [[SHIFT]]
+; SSE2-NEXT:    [[SHIFT2:%.*]] = shufflevector <8 x i16> [[B:%.*]], <8 x i16> poison, <8 x i32> <i32 poison, i32 poison, i32 poison, i32 poison, i32 5, i32 poison, i32 poison, i32 poison>
+; SSE2-NEXT:    [[TMP2:%.*]] = sub <8 x i16> [[B]], [[SHIFT2]]
+; SSE2-NEXT:    [[SHIFT3:%.*]] = shufflevector <8 x i16> [[B]], <8 x i16> poison, <8 x i32> <i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 6>
+; SSE2-NEXT:    [[TMP3:%.*]] = sub <8 x i16> [[SHIFT3]], [[B]]
+; SSE2-NEXT:    [[TMP4:%.*]] = shufflevector <8 x i16> [[A]], <8 x i16> [[B]], <8 x i32> <i32 2, i32 4, i32 6, i32 8, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE2-NEXT:    [[TMP5:%.*]] = shufflevector <8 x i16> [[A]], <8 x i16> [[B]], <8 x i32> <i32 3, i32 5, i32 7, i32 9, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE2-NEXT:    [[TMP6:%.*]] = sub <8 x i16> [[TMP4]], [[TMP5]]
+; SSE2-NEXT:    [[HSUB41:%.*]] = shufflevector <8 x i16> [[TMP1]], <8 x i16> [[TMP6]], <8 x i32> <i32 0, i32 8, i32 9, i32 10, i32 11, i32 poison, i32 poison, i32 poison>
+; SSE2-NEXT:    [[HSUB6:%.*]] = shufflevector <8 x i16> [[HSUB41]], <8 x i16> [[TMP2]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 poison, i32 12, i32 poison>
+; SSE2-NEXT:    [[RESULT:%.*]] = shufflevector <8 x i16> [[TMP3]], <8 x i16> [[HSUB6]], <8 x i32> <i32 7, i32 14, i32 poison, i32 12, i32 11, i32 10, i32 9, i32 8>
+; SSE2-NEXT:    ret <8 x i16> [[RESULT]]
+;
+; SSE4-LABEL: @sub_v8i16_76u43210(
+; SSE4-NEXT:    [[TMP1:%.*]] = shufflevector <8 x i16> [[A:%.*]], <8 x i16> poison, <8 x i32> <i32 0, i32 2, i32 4, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE4-NEXT:    [[TMP2:%.*]] = shufflevector <8 x i16> [[A]], <8 x i16> poison, <8 x i32> <i32 1, i32 3, i32 5, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE4-NEXT:    [[HSUB22:%.*]] = sub <8 x i16> [[TMP1]], [[TMP2]]
+; SSE4-NEXT:    [[TMP3:%.*]] = shufflevector <8 x i16> [[A]], <8 x i16> [[B:%.*]], <8 x i32> <i32 6, i32 8, i32 poison, i32 12, i32 14, i32 poison, i32 poison, i32 poison>
+; SSE4-NEXT:    [[TMP4:%.*]] = shufflevector <8 x i16> [[A]], <8 x i16> [[B]], <8 x i32> <i32 7, i32 9, i32 poison, i32 13, i32 15, i32 poison, i32 poison, i32 poison>
+; SSE4-NEXT:    [[TMP5:%.*]] = sub <8 x i16> [[TMP3]], [[TMP4]]
+; SSE4-NEXT:    [[RESULT:%.*]] = shufflevector <8 x i16> [[TMP5]], <8 x i16> [[HSUB22]], <8 x i32> <i32 4, i32 3, i32 poison, i32 1, i32 0, i32 10, i32 9, i32 8>
+; SSE4-NEXT:    ret <8 x i16> [[RESULT]]
+;
+; AVX-LABEL: @sub_v8i16_76u43210(
+; AVX-NEXT:    [[TMP1:%.*]] = shufflevector <8 x i16> [[A:%.*]], <8 x i16> poison, <8 x i32> <i32 0, i32 2, i32 4, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX-NEXT:    [[TMP2:%.*]] = shufflevector <8 x i16> [[A]], <8 x i16> poison, <8 x i32> <i32 1, i32 3, i32 5, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX-NEXT:    [[HSUB22:%.*]] = sub <8 x i16> [[TMP1]], [[TMP2]]
+; AVX-NEXT:    [[TMP3:%.*]] = shufflevector <8 x i16> [[A]], <8 x i16> [[B:%.*]], <8 x i32> <i32 6, i32 8, i32 poison, i32 12, i32 14, i32 poison, i32 poison, i32 poison>
+; AVX-NEXT:    [[TMP4:%.*]] = shufflevector <8 x i16> [[A]], <8 x i16> [[B]], <8 x i32> <i32 7, i32 9, i32 poison, i32 13, i32 15, i32 poison, i32 poison, i32 poison>
+; AVX-NEXT:    [[TMP5:%.*]] = sub <8 x i16> [[TMP3]], [[TMP4]]
+; AVX-NEXT:    [[RESULT:%.*]] = shufflevector <8 x i16> [[TMP5]], <8 x i16> [[HSUB22]], <8 x i32> <i32 4, i32 3, i32 poison, i32 1, i32 0, i32 10, i32 9, i32 8>
+; AVX-NEXT:    ret <8 x i16> [[RESULT]]
+;
+  %a0 = extractelement <8 x i16> %a, i32 0
+  %a1 = extractelement <8 x i16> %a, i32 1
+  %a2 = extractelement <8 x i16> %a, i32 2
+  %a3 = extractelement <8 x i16> %a, i32 3
+  %a4 = extractelement <8 x i16> %a, i32 4
+  %a5 = extractelement <8 x i16> %a, i32 5
+  %a6 = extractelement <8 x i16> %a, i32 6
+  %a7 = extractelement <8 x i16> %a, i32 7
+  %a01 = sub i16 %a0, %a1
+  %a23 = sub i16 %a2, %a3
+  %a45 = sub i16 %a4, %a5
+  %a67 = sub i16 %a6, %a7
+  %b0 = extractelement <8 x i16> %b, i32 0
+  %b1 = extractelement <8 x i16> %b, i32 1
+  %b2 = extractelement <8 x i16> %b, i32 2
+  %b3 = extractelement <8 x i16> %b, i32 3
+  %b4 = extractelement <8 x i16> %b, i32 4
+  %b5 = extractelement <8 x i16> %b, i32 5
+  %b6 = extractelement <8 x i16> %b, i32 6
+  %b7 = extractelement <8 x i16> %b, i32 7
+  %b01 = sub i16 %b0, %b1
+  %b23 = sub i16 %b2, %b3
+  %b45 = sub i16 %b4, %b5
+  %b67 = sub i16 %b6, %b7
+  %hsub0 = insertelement <8 x i16> poison, i16 %a01, i32 0
+  %hsub1 = insertelement <8 x i16> %hsub0, i16 %a23, i32 1
+  %hsub2 = insertelement <8 x i16> %hsub1, i16 %a45, i32 2
+  %hsub3 = insertelement <8 x i16> %hsub2, i16 %a67, i32 3
+  %hsub4 = insertelement <8 x i16> %hsub3, i16 %b01, i32 4
+  %hsub5 = insertelement <8 x i16> %hsub4, i16 %b23, i32 5
+  %hsub6 = insertelement <8 x i16> %hsub5, i16 %b45, i32 6
+  %hsub7 = insertelement <8 x i16> %hsub6, i16 %b67, i32 7
+  %result = shufflevector <8 x i16> %hsub7, <8 x i16> %a, <8 x i32> <i32 7, i32 6, i32 poison, i32 4, i32 3, i32 2, i32 1, i32 0>
+  ret <8 x i16> %result
+}
+
+;
+; v16i16
+;
+
+define <16 x i16> @sub_v16i16_0123456789ABCDEF(<16 x i16> %a, <16 x i16> %b) {
+; CHECK-LABEL: @sub_v16i16_0123456789ABCDEF(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <16 x i16> [[A:%.*]], <16 x i16> [[B:%.*]], <16 x i32> <i32 0, i32 2, i32 4, i32 6, i32 16, i32 18, i32 20, i32 22, i32 8, i32 10, i32 12, i32 14, i32 24, i32 26, i32 28, i32 30>
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <16 x i16> [[A]], <16 x i16> [[B]], <16 x i32> <i32 1, i32 3, i32 5, i32 7, i32 17, i32 19, i32 21, i32 23, i32 9, i32 11, i32 13, i32 15, i32 25, i32 27, i32 29, i32 31>
+; CHECK-NEXT:    [[TMP3:%.*]] = sub <16 x i16> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    ret <16 x i16> [[TMP3]]
+;
+  %a0 = extractelement <16 x i16> %a, i32 0
+  %a1 = extractelement <16 x i16> %a, i32 1
+  %a2 = extractelement <16 x i16> %a, i32 2
+  %a3 = extractelement <16 x i16> %a, i32 3
+  %a4 = extractelement <16 x i16> %a, i32 4
+  %a5 = extractelement <16 x i16> %a, i32 5
+  %a6 = extractelement <16 x i16> %a, i32 6
+  %a7 = extractelement <16 x i16> %a, i32 7
+  %a8 = extractelement <16 x i16> %a, i32 8
+  %a9 = extractelement <16 x i16> %a, i32 9
+  %aA = extractelement <16 x i16> %a, i32 10
+  %aB = extractelement <16 x i16> %a, i32 11
+  %aC = extractelement <16 x i16> %a, i32 12
+  %aD = extractelement <16 x i16> %a, i32 13
+  %aE = extractelement <16 x i16> %a, i32 14
+  %aF = extractelement <16 x i16> %a, i32 15
+  %a01 = sub i16 %a0, %a1
+  %a23 = sub i16 %a2, %a3
+  %a45 = sub i16 %a4, %a5
+  %a67 = sub i16 %a6, %a7
+  %a89 = sub i16 %a8, %a9
+  %aAB = sub i16 %aA, %aB
+  %aCD = sub i16 %aC, %aD
+  %aEF = sub i16 %aE, %aF
+  %b0 = extractelement <16 x i16> %b, i32 0
+  %b1 = extractelement <16 x i16> %b, i32 1
+  %b2 = extractelement <16 x i16> %b, i32 2
+  %b3 = extractelement <16 x i16> %b, i32 3
+  %b4 = extractelement <16 x i16> %b, i32 4
+  %b5 = extractelement <16 x i16> %b, i32 5
+  %b6 = extractelement <16 x i16> %b, i32 6
+  %b7 = extractelement <16 x i16> %b, i32 7
+  %b8 = extractelement <16 x i16> %b, i32 8
+  %b9 = extractelement <16 x i16> %b, i32 9
+  %bA = extractelement <16 x i16> %b, i32 10
+  %bB = extractelement <16 x i16> %b, i32 11
+  %bC = extractelement <16 x i16> %b, i32 12
+  %bD = extractelement <16 x i16> %b, i32 13
+  %bE = extractelement <16 x i16> %b, i32 14
+  %bF = extractelement <16 x i16> %b, i32 15
+  %b01 = sub i16 %b0, %b1
+  %b23 = sub i16 %b2, %b3
+  %b45 = sub i16 %b4, %b5
+  %b67 = sub i16 %b6, %b7
+  %b89 = sub i16 %b8, %b9
+  %bAB = sub i16 %bA, %bB
+  %bCD = sub i16 %bC, %bD
+  %bEF = sub i16 %bE, %bF
+  %hsub0 = insertelement <16 x i16> poison, i16 %a01, i32 0
+  %hsub1 = insertelement <16 x i16> %hsub0, i16 %a23, i32 1
+  %hsub2 = insertelement <16 x i16> %hsub1, i16 %a45, i32 2
+  %hsub3 = insertelement <16 x i16> %hsub2, i16 %a67, i32 3
+  %hsub4 = insertelement <16 x i16> %hsub3, i16 %b01, i32 4
+  %hsub5 = insertelement <16 x i16> %hsub4, i16 %b23, i32 5
+  %hsub6 = insertelement <16 x i16> %hsub5, i16 %b45, i32 6
+  %hsub7 = insertelement <16 x i16> %hsub6, i16 %b67, i32 7
+  %hsub8 = insertelement <16 x i16> %hsub7, i16 %a89, i32 8
+  %hsub9 = insertelement <16 x i16> %hsub8, i16 %aAB, i32 9
+  %hsubA = insertelement <16 x i16> %hsub9, i16 %aCD, i32 10
+  %hsubB = insertelement <16 x i16> %hsubA, i16 %aEF, i32 11
+  %hsubC = insertelement <16 x i16> %hsubB, i16 %b89, i32 12
+  %hsubD = insertelement <16 x i16> %hsubC, i16 %bAB, i32 13
+  %hsubE = insertelement <16 x i16> %hsubD, i16 %bCD, i32 14
+  %hsubF = insertelement <16 x i16> %hsubE, i16 %bEF, i32 15
+  %result = shufflevector <16 x i16> %hsubF, <16 x i16> %a, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+  ret <16 x i16> %result
+}
+
+define <16 x i16> @sub_v16i16_0123u56789uBCDEF(<16 x i16> %a, <16 x i16> %b) {
+; SSE2-LABEL: @sub_v16i16_0123u56789uBCDEF(
+; SSE2-NEXT:    [[BC:%.*]] = extractelement <16 x i16> [[B:%.*]], i64 12
+; SSE2-NEXT:    [[BD:%.*]] = extractelement <16 x i16> [[B]], i64 13
+; SSE2-NEXT:    [[BE:%.*]] = extractelement <16 x i16> [[B]], i64 14
+; SSE2-NEXT:    [[BF:%.*]] = extractelement <16 x i16> [[B]], i64 15
+; SSE2-NEXT:    [[BCD:%.*]] = sub i16 [[BC]], [[BD]]
+; SSE2-NEXT:    [[BEF:%.*]] = sub i16 [[BE]], [[BF]]
+; SSE2-NEXT:    [[TMP1:%.*]] = shufflevector <16 x i16> [[A:%.*]], <16 x i16> [[B]], <16 x i32> <i32 0, i32 2, i32 4, i32 6, i32 poison, i32 18, i32 20, i32 22, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE2-NEXT:    [[TMP2:%.*]] = shufflevector <16 x i16> [[A]], <16 x i16> [[B]], <16 x i32> <i32 1, i32 3, i32 5, i32 7, i32 poison, i32 19, i32 21, i32 23, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE2-NEXT:    [[TMP3:%.*]] = shufflevector <16 x i16> [[TMP1]], <16 x i16> [[A]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 5, i32 6, i32 7, i32 24, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE2-NEXT:    [[TMP7:%.*]] = shufflevector <16 x i16> [[TMP2]], <16 x i16> [[A]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 5, i32 6, i32 7, i32 25, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE2-NEXT:    [[HSUB8:%.*]] = sub <16 x i16> [[TMP3]], [[TMP7]]
+; SSE2-NEXT:    [[TMP4:%.*]] = shufflevector <16 x i16> [[A]], <16 x i16> [[B]], <16 x i32> <i32 10, i32 poison, i32 14, i32 24, i32 26, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE2-NEXT:    [[TMP5:%.*]] = shufflevector <16 x i16> [[A]], <16 x i16> [[B]], <16 x i32> <i32 11, i32 poison, i32 15, i32 25, i32 27, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE2-NEXT:    [[TMP6:%.*]] = sub <16 x i16> [[TMP4]], [[TMP5]]
+; SSE2-NEXT:    [[HSUBD1:%.*]] = shufflevector <16 x i16> [[HSUB8]], <16 x i16> [[TMP6]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 5, i32 6, i32 7, i32 8, i32 16, i32 poison, i32 18, i32 19, i32 20, i32 poison, i32 poison>
+; SSE2-NEXT:    [[HSUBE:%.*]] = insertelement <16 x i16> [[HSUBD1]], i16 [[BCD]], i64 14
+; SSE2-NEXT:    [[HSUBF:%.*]] = insertelement <16 x i16> [[HSUBE]], i16 [[BEF]], i64 15
+; SSE2-NEXT:    [[RESULT:%.*]] = shufflevector <16 x i16> [[HSUBF]], <16 x i16> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 5, i32 6, i32 7, i32 8, i32 9, i32 poison, i32 11, i32 12, i32 13, i32 14, i32 15>
+; SSE2-NEXT:    ret <16 x i16> [[RESULT]]
+;
+; SSE4-LABEL: @sub_v16i16_0123u56789uBCDEF(
+; SSE4-NEXT:    [[TMP1:%.*]] = shufflevector <16 x i16> [[A:%.*]], <16 x i16> [[B:%.*]], <16 x i32> <i32 0, i32 2, i32 4, i32 6, i32 poison, i32 18, i32 20, i32 22, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE4-NEXT:    [[TMP2:%.*]] = shufflevector <16 x i16> [[A]], <16 x i16> [[B]], <16 x i32> <i32 1, i32 3, i32 5, i32 7, i32 poison, i32 19, i32 21, i32 23, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE4-NEXT:    [[TMP3:%.*]] = shufflevector <16 x i16> [[TMP1]], <16 x i16> [[A]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 5, i32 6, i32 7, i32 24, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE4-NEXT:    [[TMP4:%.*]] = shufflevector <16 x i16> [[TMP2]], <16 x i16> [[A]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 5, i32 6, i32 7, i32 25, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE4-NEXT:    [[TMP5:%.*]] = shufflevector <16 x i16> [[TMP3]], <16 x i16> [[A]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 5, i32 6, i32 7, i32 8, i32 26, i32 poison, i32 30, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE4-NEXT:    [[TMP6:%.*]] = shufflevector <16 x i16> [[TMP4]], <16 x i16> [[A]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 5, i32 6, i32 7, i32 8, i32 27, i32 poison, i32 31, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE4-NEXT:    [[TMP7:%.*]] = shufflevector <16 x i16> [[TMP5]], <16 x i16> [[B]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 5, i32 6, i32 7, i32 8, i32 9, i32 poison, i32 11, i32 24, i32 26, i32 28, i32 30>
+; SSE4-NEXT:    [[TMP8:%.*]] = shufflevector <16 x i16> [[TMP6]], <16 x i16> [[B]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 5, i32 6, i32 7, i32 8, i32 9, i32 poison, i32 11, i32 25, i32 27, i32 29, i32 31>
+; SSE4-NEXT:    [[TMP9:%.*]] = sub <16 x i16> [[TMP7]], [[TMP8]]
+; SSE4-NEXT:    ret <16 x i16> [[TMP9]]
+;
+; AVX-LABEL: @sub_v16i16_0123u56789uBCDEF(
+; AVX-NEXT:    [[TMP1:%.*]] = shufflevector <16 x i16> [[A:%.*]], <16 x i16> [[B:%.*]], <16 x i32> <i32 0, i32 2, i32 4, i32 6, i32 poison, i32 18, i32 20, i32 22, i32 8, i32 10, i32 poison, i32 14, i32 24, i32 26, i32 28, i32 30>
+; AVX-NEXT:    [[TMP2:%.*]] = shufflevector <16 x i16> [[A]], <16 x i16> [[B]], <16 x i32> <i32 1, i32 3, i32 5, i32 7, i32 poison, i32 19, i32 21, i32 23, i32 9, i32 11, i32 poison, i32 15, i32 25, i32 27, i32 29, i32 31>
+; AVX-NEXT:    [[RESULT:%.*]] = sub <16 x i16> [[TMP1]], [[TMP2]]
+; AVX-NEXT:    ret <16 x i16> [[RESULT]]
+;
+  %a0 = extractelement <16 x i16> %a, i32 0
+  %a1 = extractelement <16 x i16> %a, i32 1
+  %a2 = extractelement <16 x i16> %a, i32 2
+  %a3 = extractelement <16 x i16> %a, i32 3
+  %a4 = extractelement <16 x i16> %a, i32 4
+  %a5 = extractelement <16 x i16> %a, i32 5
+  %a6 = extractelement <16 x i16> %a, i32 6
+  %a7 = extractelement <16 x i16> %a, i32 7
+  %a8 = extractelement <16 x i16> %a, i32 8
+  %a9 = extractelement <16 x i16> %a, i32 9
+  %aA = extractelement <16 x i16> %a, i32 10
+  %aB = extractelement <16 x i16> %a, i32 11
+  %aC = extractelement <16 x i16> %a, i32 12
+  %aD = extractelement <16 x i16> %a, i32 13
+  %aE = extractelement <16 x i16> %a, i32 14
+  %aF = extractelement <16 x i16> %a, i32 15
+  %a01 = sub i16 %a0, %a1
+  %a23 = sub i16 %a2, %a3
+  %a45 = sub i16 %a4, %a5
+  %a67 = sub i16 %a6, %a7
+  %a89 = sub i16 %a8, %a9
+  %aAB = sub i16 %aA, %aB
+  %aCD = sub i16 %aC, %aD
+  %aEF = sub i16 %aE, %aF
+  %b0 = extractelement <16 x i16> %b, i32 0
+  %b1 = extractelement <16 x i16> %b, i32 1
+  %b2 = extractelement <16 x i16> %b, i32 2
+  %b3 = extractelement <16 x i16> %b, i32 3
+  %b4 = extractelement <16 x i16> %b, i32 4
+  %b5 = extractelement <16 x i16> %b, i32 5
+  %b6 = extractelement <16 x i16> %b, i32 6
+  %b7 = extractelement <16 x i16> %b, i32 7
+  %b8 = extractelement <16 x i16> %b, i32 8
+  %b9 = extractelement <16 x i16> %b, i32 9
+  %bA = extractelement <16 x i16> %b, i32 10
+  %bB = extractelement <16 x i16> %b, i32 11
+  %bC = extractelement <16 x i16> %b, i32 12
+  %bD = extractelement <16 x i16> %b, i32 13
+  %bE = extractelement <16 x i16> %b, i32 14
+  %bF = extractelement <16 x i16> %b, i32 15
+  %b01 = sub i16 %b0, %b1
+  %b23 = sub i16 %b2, %b3
+  %b45 = sub i16 %b4, %b5
+  %b67 = sub i16 %b6, %b7
+  %b89 = sub i16 %b8, %b9
+  %bAB = sub i16 %bA, %bB
+  %bCD = sub i16 %bC, %bD
+  %bEF = sub i16 %bE, %bF
+  %hsub0 = insertelement <16 x i16> poison, i16 %a01, i32 0
+  %hsub1 = insertelement <16 x i16> %hsub0, i16 %a23, i32 1
+  %hsub2 = insertelement <16 x i16> %hsub1, i16 %a45, i32 2
+  %hsub3 = insertelement <16 x i16> %hsub2, i16 %a67, i32 3
+  %hsub4 = insertelement <16 x i16> %hsub3, i16 %b01, i32 4
+  %hsub5 = insertelement <16 x i16> %hsub4, i16 %b23, i32 5
+  %hsub6 = insertelement <16 x i16> %hsub5, i16 %b45, i32 6
+  %hsub7 = insertelement <16 x i16> %hsub6, i16 %b67, i32 7
+  %hsub8 = insertelement <16 x i16> %hsub7, i16 %a89, i32 8
+  %hsub9 = insertelement <16 x i16> %hsub8, i16 %aAB, i32 9
+  %hsubA = insertelement <16 x i16> %hsub9, i16 %aCD, i32 10
+  %hsubB = insertelement <16 x i16> %hsubA, i16 %aEF, i32 11
+  %hsubC = insertelement <16 x i16> %hsubB, i16 %b89, i32 12
+  %hsubD = insertelement <16 x i16> %hsubC, i16 %bAB, i32 13
+  %hsubE = insertelement <16 x i16> %hsubD, i16 %bCD, i32 14
+  %hsubF = insertelement <16 x i16> %hsubE, i16 %bEF, i32 15
+  %result = shufflevector <16 x i16> %hsubF, <16 x i16> %a, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 5, i32 6, i32 7, i32 8, i32 9, i32 poison, i32 11, i32 12, i32 13, i32 14, i32 15>
+  ret <16 x i16> %result
+}
+
+define <16 x i16> @sub_v16i16_FEuCBA98765432u0(<16 x i16> %a, <16 x i16> %b) {
+; SSE2-LABEL: @sub_v16i16_FEuCBA98765432u0(
+; SSE2-NEXT:    [[BC:%.*]] = extractelement <16 x i16> [[B:%.*]], i64 12
+; SSE2-NEXT:    [[BD:%.*]] = extractelement <16 x i16> [[B]], i64 13
+; SSE2-NEXT:    [[BE:%.*]] = extractelement <16 x i16> [[B]], i64 14
+; SSE2-NEXT:    [[BF:%.*]] = extractelement <16 x i16> [[B]], i64 15
+; SSE2-NEXT:    [[BCD:%.*]] = sub i16 [[BC]], [[BD]]
+; SSE2-NEXT:    [[BEF:%.*]] = sub i16 [[BE]], [[BF]]
+; SSE2-NEXT:    [[TMP1:%.*]] = shufflevector <16 x i16> [[A:%.*]], <16 x i16> [[B]], <16 x i32> <i32 0, i32 poison, i32 4, i32 6, i32 16, i32 18, i32 20, i32 22, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE2-NEXT:    [[TMP2:%.*]] = shufflevector <16 x i16> [[A]], <16 x i16> [[B]], <16 x i32> <i32 1, i32 poison, i32 5, i32 7, i32 17, i32 19, i32 21, i32 23, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE2-NEXT:    [[TMP3:%.*]] = shufflevector <16 x i16> [[TMP1]], <16 x i16> [[A]], <16 x i32> <i32 0, i32 poison, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 24, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE2-NEXT:    [[TMP7:%.*]] = shufflevector <16 x i16> [[TMP2]], <16 x i16> [[A]], <16 x i32> <i32 0, i32 poison, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 25, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE2-NEXT:    [[HSUB8:%.*]] = sub <16 x i16> [[TMP3]], [[TMP7]]
+; SSE2-NEXT:    [[TMP4:%.*]] = shufflevector <16 x i16> [[A]], <16 x i16> [[B]], <16 x i32> <i32 10, i32 12, i32 14, i32 24, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE2-NEXT:    [[TMP5:%.*]] = shufflevector <16 x i16> [[A]], <16 x i16> [[B]], <16 x i32> <i32 11, i32 13, i32 15, i32 25, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE2-NEXT:    [[TMP6:%.*]] = sub <16 x i16> [[TMP4]], [[TMP5]]
+; SSE2-NEXT:    [[HSUBC1:%.*]] = shufflevector <16 x i16> [[HSUB8]], <16 x i16> [[TMP6]], <16 x i32> <i32 0, i32 poison, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 16, i32 17, i32 18, i32 19, i32 poison, i32 poison, i32 poison>
+; SSE2-NEXT:    [[HSUBE:%.*]] = insertelement <16 x i16> [[HSUBC1]], i16 [[BCD]], i64 14
+; SSE2-NEXT:    [[HSUBF:%.*]] = insertelement <16 x i16> [[HSUBE]], i16 [[BEF]], i64 15
+; SSE2-NEXT:    [[RESULT:%.*]] = shufflevector <16 x i16> [[HSUBF]], <16 x i16> poison, <16 x i32> <i32 15, i32 14, i32 poison, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 poison, i32 0>
+; SSE2-NEXT:    ret <16 x i16> [[RESULT]]
+;
+; SSE4-LABEL: @sub_v16i16_FEuCBA98765432u0(
+; SSE4-NEXT:    [[TMP1:%.*]] = shufflevector <16 x i16> [[A:%.*]], <16 x i16> [[B:%.*]], <16 x i32> <i32 0, i32 poison, i32 4, i32 6, i32 16, i32 18, i32 20, i32 22, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE4-NEXT:    [[TMP2:%.*]] = shufflevector <16 x i16> [[A]], <16 x i16> [[B]], <16 x i32> <i32 1, i32 poison, i32 5, i32 7, i32 17, i32 19, i32 21, i32 23, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE4-NEXT:    [[TMP3:%.*]] = shufflevector <16 x i16> [[TMP1]], <16 x i16> [[A]], <16 x i32> <i32 0, i32 poison, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 24, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE4-NEXT:    [[TMP10:%.*]] = shufflevector <16 x i16> [[TMP2]], <16 x i16> [[A]], <16 x i32> <i32 0, i32 poison, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 25, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE4-NEXT:    [[TMP4:%.*]] = shufflevector <16 x i16> [[TMP3]], <16 x i16> [[A]], <16 x i32> <i32 0, i32 poison, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 poison, i32 26, i32 28, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE4-NEXT:    [[TMP5:%.*]] = shufflevector <16 x i16> [[TMP10]], <16 x i16> [[A]], <16 x i32> <i32 0, i32 poison, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 poison, i32 27, i32 29, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE4-NEXT:    [[TMP6:%.*]] = sub <16 x i16> [[TMP4]], [[TMP5]]
+; SSE4-NEXT:    [[TMP7:%.*]] = shufflevector <16 x i16> [[A]], <16 x i16> [[B]], <16 x i32> <i32 14, i32 24, i32 28, i32 30, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE4-NEXT:    [[TMP8:%.*]] = shufflevector <16 x i16> [[A]], <16 x i16> [[B]], <16 x i32> <i32 15, i32 25, i32 29, i32 31, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE4-NEXT:    [[TMP9:%.*]] = sub <16 x i16> [[TMP7]], [[TMP8]]
+; SSE4-NEXT:    [[RESULT:%.*]] = shufflevector <16 x i16> [[TMP9]], <16 x i16> [[TMP6]], <16 x i32> <i32 3, i32 2, i32 poison, i32 1, i32 0, i32 27, i32 26, i32 24, i32 23, i32 22, i32 21, i32 20, i32 19, i32 18, i32 poison, i32 16>
+; SSE4-NEXT:    ret <16 x i16> [[RESULT]]
+;
+; AVX2-LABEL: @sub_v16i16_FEuCBA98765432u0(
+; AVX2-NEXT:    [[TMP1:%.*]] = shufflevector <16 x i16> [[A:%.*]], <16 x i16> [[B:%.*]], <16 x i32> <i32 0, i32 poison, i32 4, i32 6, i32 16, i32 18, i32 20, i32 22, i32 8, i32 10, i32 12, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX2-NEXT:    [[TMP2:%.*]] = shufflevector <16 x i16> [[A]], <16 x i16> [[B]], <16 x i32> <i32 1, i32 poison, i32 5, i32 7, i32 17, i32 19, i32 21, i32 23, i32 9, i32 11, i32 13, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX2-NEXT:    [[HSUBA:%.*]] = sub <16 x i16> [[TMP1]], [[TMP2]]
+; AVX2-NEXT:    [[TMP3:%.*]] = shufflevector <16 x i16> [[A]], <16 x i16> [[B]], <16 x i32> <i32 14, i32 24, i32 poison, i32 28, i32 30, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX2-NEXT:    [[TMP4:%.*]] = shufflevector <16 x i16> [[A]], <16 x i16> [[B]], <16 x i32> <i32 15, i32 25, i32 poison, i32 29, i32 31, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX2-NEXT:    [[TMP5:%.*]] = sub <16 x i16> [[TMP3]], [[TMP4]]
+; AVX2-NEXT:    [[RESULT:%.*]] = shufflevector <16 x i16> [[TMP5]], <16 x i16> [[HSUBA]], <16 x i32> <i32 4, i32 3, i32 poison, i32 1, i32 0, i32 26, i32 25, i32 24, i32 23, i32 22, i32 21, i32 20, i32 19, i32 18, i32 poison, i32 16>
+; AVX2-NEXT:    ret <16 x i16> [[RESULT]]
+;
+; AVX512-LABEL: @sub_v16i16_FEuCBA98765432u0(
+; AVX512-NEXT:    [[TMP1:%.*]] = shufflevector <16 x i16> [[A:%.*]], <16 x i16> [[B:%.*]], <16 x i32> <i32 0, i32 poison, i32 4, i32 6, i32 16, i32 18, i32 20, i32 22, i32 8, i32 10, i32 12, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512-NEXT:    [[TMP2:%.*]] = shufflevector <16 x i16> [[A]], <16 x i16> [[B]], <16 x i32> <i32 1, i32 poison, i32 5, i32 7, i32 17, i32 19, i32 21, i32 23, i32 9, i32 11, i32 13, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512-NEXT:    [[HSUBA2:%.*]] = sub <16 x i16> [[TMP1]], [[TMP2]]
+; AVX512-NEXT:    [[TMP3:%.*]] = shufflevector <16 x i16> [[A]], <16 x i16> [[B]], <16 x i32> <i32 14, i32 24, i32 poison, i32 28, i32 30, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512-NEXT:    [[TMP4:%.*]] = shufflevector <16 x i16> [[A]], <16 x i16> [[B]], <16 x i32> <i32 15, i32 25, i32 poison, i32 29, i32 31, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512-NEXT:    [[TMP5:%.*]] = sub <16 x i16> [[TMP3]], [[TMP4]]
+; AVX512-NEXT:    [[RESULT:%.*]] = shufflevector <16 x i16> [[TMP5]], <16 x i16> [[HSUBA2]], <16 x i32> <i32 4, i32 3, i32 poison, i32 1, i32 0, i32 26, i32 25, i32 24, i32 23, i32 22, i32 21, i32 20, i32 19, i32 18, i32 poison, i32 16>
+; AVX512-NEXT:    ret <16 x i16> [[RESULT]]
+;
+  %a0 = extractelement <16 x i16> %a, i32 0
+  %a1 = extractelement <16 x i16> %a, i32 1
+  %a2 = extractelement <16 x i16> %a, i32 2
+  %a3 = extractelement <16 x i16> %a, i32 3
+  %a4 = extractelement <16 x i16> %a, i32 4
+  %a5 = extractelement <16 x i16> %a, i32 5
+  %a6 = extractelement <16 x i16> %a, i32 6
+  %a7 = extractelement <16 x i16> %a, i32 7
+  %a8 = extractelement <16 x i16> %a, i32 8
+  %a9 = extractelement <16 x i16> %a, i32 9
+  %aA = extractelement <16 x i16> %a, i32 10
+  %aB = extractelement <16 x i16> %a, i32 11
+  %aC = extractelement <16 x i16> %a, i32 12
+  %aD = extractelement <16 x i16> %a, i32 13
+  %aE = extractelement <16 x i16> %a, i32 14
+  %aF = extractelement <16 x i16> %a, i32 15
+  %a01 = sub i16 %a0, %a1
+  %a23 = sub i16 %a2, %a3
+  %a45 = sub i16 %a4, %a5
+  %a67 = sub i16 %a6, %a7
+  %a89 = sub i16 %a8, %a9
+  %aAB = sub i16 %aA, %aB
+  %aCD = sub i16 %aC, %aD
+  %aEF = sub i16 %aE, %aF
+  %b0 = extractelement <16 x i16> %b, i32 0
+  %b1 = extractelement <16 x i16> %b, i32 1
+  %b2 = extractelement <16 x i16> %b, i32 2
+  %b3 = extractelement <16 x i16> %b, i32 3
+  %b4 = extractelement <16 x i16> %b, i32 4
+  %b5 = extractelement <16 x i16> %b, i32 5
+  %b6 = extractelement <16 x i16> %b, i32 6
+  %b7 = extractelement <16 x i16> %b, i32 7
+  %b8 = extractelement <16 x i16> %b, i32 8
+  %b9 = extractelement <16 x i16> %b, i32 9
+  %bA = extractelement <16 x i16> %b, i32 10
+  %bB = extractelement <16 x i16> %b, i32 11
+  %bC = extractelement <16 x i16> %b, i32 12
+  %bD = extractelement <16 x i16> %b, i32 13
+  %bE = extractelement <16 x i16> %b, i32 14
+  %bF = extractelement <16 x i16> %b, i32 15
+  %b01 = sub i16 %b0, %b1
+  %b23 = sub i16 %b2, %b3
+  %b45 = sub i16 %b4, %b5
+  %b67 = sub i16 %b6, %b7
+  %b89 = sub i16 %b8, %b9
+  %bAB = sub i16 %bA, %bB
+  %bCD = sub i16 %bC, %bD
+  %bEF = sub i16 %bE, %bF
+  %hsub0 = insertelement <16 x i16> poison, i16 %a01, i32 0
+  %hsub1 = insertelement <16 x i16> %hsub0, i16 %a23, i32 1
+  %hsub2 = insertelement <16 x i16> %hsub1, i16 %a45, i32 2
+  %hsub3 = insertelement <16 x i16> %hsub2, i16 %a67, i32 3
+  %hsub4 = insertelement <16 x i16> %hsub3, i16 %b01, i32 4
+  %hsub5 = insertelement <16 x i16> %hsub4, i16 %b23, i32 5
+  %hsub6 = insertelement <16 x i16> %hsub5, i16 %b45, i32 6
+  %hsub7 = insertelement <16 x i16> %hsub6, i16 %b67, i32 7
+  %hsub8 = insertelement <16 x i16> %hsub7, i16 %a89, i32 8
+  %hsub9 = insertelement <16 x i16> %hsub8, i16 %aAB, i32 9
+  %hsubA = insertelement <16 x i16> %hsub9, i16 %aCD, i32 10
+  %hsubB = insertelement <16 x i16> %hsubA, i16 %aEF, i32 11
+  %hsubC = insertelement <16 x i16> %hsubB, i16 %b89, i32 12
+  %hsubD = insertelement <16 x i16> %hsubC, i16 %bAB, i32 13
+  %hsubE = insertelement <16 x i16> %hsubD, i16 %bCD, i32 14
+  %hsubF = insertelement <16 x i16> %hsubE, i16 %bEF, i32 15
+  %result = shufflevector <16 x i16> %hsubF, <16 x i16> %a, <16 x i32> <i32 15, i32 14, i32 poison, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 poison, i32 0>
+  ret <16 x i16> %result
+}
+
 ;
 ; v4i32
 ;
@@ -336,6 +713,51 @@ define <4 x i32> @sub_v4i32_01uu(<4 x i32> %a, <4 x i32> %b) {
   ret <4 x i32> %result
 }
 
+define <4 x i32> @sub_v4i32_32u0(<4 x i32> %a, <4 x i32> %b) {
+; SSE2-LABEL: @sub_v4i32_32u0(
+; SSE2-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i32> [[B:%.*]], <4 x i32> [[A:%.*]], <4 x i32> <i32 2, i32 0, i32 poison, i32 4>
+; SSE2-NEXT:    [[TMP2:%.*]] = shufflevector <4 x i32> [[B]], <4 x i32> [[A]], <4 x i32> <i32 3, i32 1, i32 poison, i32 5>
+; SSE2-NEXT:    [[RESULT1:%.*]] = sub <4 x i32> [[TMP1]], [[TMP2]]
+; SSE2-NEXT:    ret <4 x i32> [[RESULT1]]
+;
+; SSE4-LABEL: @sub_v4i32_32u0(
+; SSE4-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i32> [[B:%.*]], <4 x i32> [[A:%.*]], <4 x i32> <i32 2, i32 0, i32 poison, i32 4>
+; SSE4-NEXT:    [[TMP2:%.*]] = shufflevector <4 x i32> [[B]], <4 x i32> [[A]], <4 x i32> <i32 3, i32 1, i32 poison, i32 5>
+; SSE4-NEXT:    [[RESULT:%.*]] = sub <4 x i32> [[TMP1]], [[TMP2]]
+; SSE4-NEXT:    ret <4 x i32> [[RESULT]]
+;
+; AVX2-LABEL: @sub_v4i32_32u0(
+; AVX2-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i32> [[B:%.*]], <4 x i32> [[A:%.*]], <4 x i32> <i32 2, i32 0, i32 poison, i32 4>
+; AVX2-NEXT:    [[TMP2:%.*]] = shufflevector <4 x i32> [[B]], <4 x i32> [[A]], <4 x i32> <i32 3, i32 1, i32 poison, i32 5>
+; AVX2-NEXT:    [[RESULT:%.*]] = sub <4 x i32> [[TMP1]], [[TMP2]]
+; AVX2-NEXT:    ret <4 x i32> [[RESULT]]
+;
+; AVX512-LABEL: @sub_v4i32_32u0(
+; AVX512-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i32> [[B:%.*]], <4 x i32> [[A:%.*]], <4 x i32> <i32 2, i32 0, i32 poison, i32 4>
+; AVX512-NEXT:    [[TMP2:%.*]] = shufflevector <4 x i32> [[B]], <4 x i32> [[A]], <4 x i32> <i32 3, i32 1, i32 poison, i32 5>
+; AVX512-NEXT:    [[RESULT1:%.*]] = sub <4 x i32> [[TMP1]], [[TMP2]]
+; AVX512-NEXT:    ret <4 x i32> [[RESULT1]]
+;
+  %a0 = extractelement <4 x i32> %a, i32 0
+  %a1 = extractelement <4 x i32> %a, i32 1
+  %a2 = extractelement <4 x i32> %a, i32 2
+  %a3 = extractelement <4 x i32> %a, i32 3
+  %a01 = sub i32 %a0, %a1
+  %a23 = sub i32 %a2, %a3
+  %b0 = extractelement <4 x i32> %b, i32 0
+  %b1 = extractelement <4 x i32> %b, i32 1
+  %b2 = extractelement <4 x i32> %b, i32 2
+  %b3 = extractelement <4 x i32> %b, i32 3
+  %b01 = sub i32 %b0, %b1
+  %b23 = sub i32 %b2, %b3
+  %hsub0 = insertelement <4 x i32> poison, i32 %a01, i32 0
+  %hsub1 = insertelement <4 x i32> %hsub0, i32 %a23, i32 1
+  %hsub2 = insertelement <4 x i32> %hsub1, i32 %b01, i32 2
+  %hsub3 = insertelement <4 x i32> %hsub2, i32 %b23, i32 3
+  %result = shufflevector <4 x i32> %hsub3, <4 x i32> %a, <4 x i32> <i32 3, i32 2, i32 poison, i32 0>
+  ret <4 x i32> %result
+}
+
 ;
 ; v8i32
 ;
@@ -384,31 +806,11 @@ define <8 x i32> @sub_v8i32_01234567(<8 x i32> %a, <8 x i32> %b) {
 }
 
 define <8 x i32> @sub_v8i32_01234u67(<8 x i32> %a, <8 x i32> %b) {
-; SSE2-LABEL: @sub_v8i32_01234u67(
-; SSE2-NEXT:    [[TMP2:%.*]] = shufflevector <8 x i32> [[A:%.*]], <8 x i32> [[B:%.*]], <8 x i32> <i32 0, i32 2, i32 8, i32 10, i32 4, i32 poison, i32 12, i32 14>
-; SSE2-NEXT:    [[TMP3:%.*]] = shufflevector <8 x i32> [[A]], <8 x i32> [[B]], <8 x i32> <i32 1, i32 3, i32 9, i32 11, i32 5, i32 poison, i32 13, i32 15>
-; SSE2-NEXT:    [[TMP4:%.*]] = sub <8 x i32> [[TMP2]], [[TMP3]]
-; SSE2-NEXT:    ret <8 x i32> [[TMP4]]
-;
-; SSE4-LABEL: @sub_v8i32_01234u67(
-; SSE4-NEXT:    [[A4:%.*]] = extractelement <8 x i32> [[A:%.*]], i64 4
-; SSE4-NEXT:    [[A5:%.*]] = extractelement <8 x i32> [[A]], i64 5
-; SSE4-NEXT:    [[A45:%.*]] = sub i32 [[A4]], [[A5]]
-; SSE4-NEXT:    [[TMP2:%.*]] = shufflevector <8 x i32> [[A]], <8 x i32> [[B:%.*]], <8 x i32> <i32 0, i32 2, i32 8, i32 10, i32 poison, i32 poison, i32 poison, i32 poison>
-; SSE4-NEXT:    [[TMP4:%.*]] = shufflevector <8 x i32> [[A]], <8 x i32> [[B]], <8 x i32> <i32 1, i32 3, i32 9, i32 11, i32 poison, i32 poison, i32 poison, i32 poison>
-; SSE4-NEXT:    [[TMP3:%.*]] = sub <8 x i32> [[TMP2]], [[TMP4]]
-; SSE4-NEXT:    [[HSUB4:%.*]] = insertelement <8 x i32> [[TMP3]], i32 [[A45]], i64 4
-; SSE4-NEXT:    [[TMP7:%.*]] = shufflevector <8 x i32> [[B]], <8 x i32> poison, <8 x i32> <i32 4, i32 6, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; SSE4-NEXT:    [[TMP5:%.*]] = shufflevector <8 x i32> [[B]], <8 x i32> poison, <8 x i32> <i32 5, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; SSE4-NEXT:    [[TMP6:%.*]] = sub <8 x i32> [[TMP7]], [[TMP5]]
-; SSE4-NEXT:    [[RESULT:%.*]] = shufflevector <8 x i32> [[HSUB4]], <8 x i32> [[TMP6]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 poison, i32 8, i32 9>
-; SSE4-NEXT:    ret <8 x i32> [[RESULT]]
-;
-; AVX-LABEL: @sub_v8i32_01234u67(
-; AVX-NEXT:    [[TMP5:%.*]] = shufflevector <8 x i32> [[A:%.*]], <8 x i32> [[B:%.*]], <8 x i32> <i32 0, i32 2, i32 8, i32 10, i32 4, i32 poison, i32 12, i32 14>
-; AVX-NEXT:    [[TMP6:%.*]] = shufflevector <8 x i32> [[A]], <8 x i32> [[B]], <8 x i32> <i32 1, i32 3, i32 9, i32 11, i32 5, i32 poison, i32 13, i32 15>
-; AVX-NEXT:    [[TMP7:%.*]] = sub <8 x i32> [[TMP5]], [[TMP6]]
-; AVX-NEXT:    ret <8 x i32> [[TMP7]]
+; CHECK-LABEL: @sub_v8i32_01234u67(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <8 x i32> [[A:%.*]], <8 x i32> [[B:%.*]], <8 x i32> <i32 0, i32 2, i32 8, i32 10, i32 4, i32 poison, i32 12, i32 14>
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <8 x i32> [[A]], <8 x i32> [[B]], <8 x i32> <i32 1, i32 3, i32 9, i32 11, i32 5, i32 poison, i32 13, i32 15>
+; CHECK-NEXT:    [[RESULT:%.*]] = sub <8 x i32> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    ret <8 x i32> [[RESULT]]
 ;
   %a0 = extractelement <8 x i32> %a, i32 0
   %a1 = extractelement <8 x i32> %a, i32 1
@@ -605,8 +1007,8 @@ define <4 x float> @sub_v4f32_012u(<4 x float> %a, <4 x float> %b) {
 
 define <4 x float> @sub_v4f32_uu23(<4 x float> %a, <4 x float> %b) {
 ; CHECK-LABEL: @sub_v4f32_uu23(
-; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <4 x float> [[TMP1:%.*]], <4 x float> poison, <4 x i32> <i32 poison, i32 poison, i32 0, i32 2>
-; CHECK-NEXT:    [[RESULT1:%.*]] = shufflevector <4 x float> [[TMP1]], <4 x float> poison, <4 x i32> <i32 poison, i32 poison, i32 1, i32 3>
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <4 x float> [[B:%.*]], <4 x float> poison, <4 x i32> <i32 poison, i32 poison, i32 0, i32 2>
+; CHECK-NEXT:    [[RESULT1:%.*]] = shufflevector <4 x float> [[B]], <4 x float> poison, <4 x i32> <i32 poison, i32 poison, i32 1, i32 3>
 ; CHECK-NEXT:    [[RESULT2:%.*]] = fsub <4 x float> [[TMP2]], [[RESULT1]]
 ; CHECK-NEXT:    ret <4 x float> [[RESULT2]]
 ;
@@ -632,8 +1034,8 @@ define <4 x float> @sub_v4f32_uu23(<4 x float> %a, <4 x float> %b) {
 
 define <4 x float> @sub_v4f32_01uu(<4 x float> %a, <4 x float> %b) {
 ; CHECK-LABEL: @sub_v4f32_01uu(
-; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <4 x float> [[TMP1:%.*]], <4 x float> poison, <4 x i32> <i32 0, i32 2, i32 poison, i32 poison>
-; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <4 x float> [[TMP1]], <4 x float> poison, <4 x i32> <i32 1, i32 3, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <4 x float> [[A:%.*]], <4 x float> poison, <4 x i32> <i32 0, i32 2, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <4 x float> [[A]], <4 x float> poison, <4 x i32> <i32 1, i32 3, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP4:%.*]] = fsub <4 x float> [[TMP2]], [[TMP3]]
 ; CHECK-NEXT:    ret <4 x float> [[TMP4]]
 ;
@@ -706,33 +1108,21 @@ define <8 x float> @sub_v8f32_01234567(<8 x float> %a, <8 x float> %b) {
 
 define <8 x float> @sub_v8f32_012u4567(<8 x float> %a, <8 x float> %b) {
 ; SSE2-LABEL: @sub_v8f32_012u4567(
-; SSE2-NEXT:    [[A6:%.*]] = extractelement <8 x float> [[A:%.*]], i64 6
-; SSE2-NEXT:    [[A7:%.*]] = extractelement <8 x float> [[A]], i64 7
-; SSE2-NEXT:    [[A67:%.*]] = fsub float [[A6]], [[A7]]
 ; SSE2-NEXT:    [[TMP1:%.*]] = shufflevector <8 x float> [[B:%.*]], <8 x float> poison, <2 x i32> <i32 4, i32 6>
 ; SSE2-NEXT:    [[TMP2:%.*]] = shufflevector <8 x float> [[B]], <8 x float> poison, <2 x i32> <i32 5, i32 7>
 ; SSE2-NEXT:    [[TMP3:%.*]] = fsub <2 x float> [[TMP1]], [[TMP2]]
-; SSE2-NEXT:    [[TMP5:%.*]] = shufflevector <8 x float> [[A]], <8 x float> [[B]], <8 x i32> <i32 0, i32 2, i32 8, i32 poison, i32 4, i32 poison, i32 poison, i32 poison>
-; SSE2-NEXT:    [[TMP8:%.*]] = shufflevector <8 x float> [[A]], <8 x float> [[B]], <8 x i32> <i32 1, i32 3, i32 9, i32 poison, i32 5, i32 poison, i32 poison, i32 poison>
+; SSE2-NEXT:    [[TMP5:%.*]] = shufflevector <8 x float> [[A:%.*]], <8 x float> [[B]], <8 x i32> <i32 0, i32 2, i32 8, i32 poison, i32 4, i32 6, i32 poison, i32 poison>
+; SSE2-NEXT:    [[TMP8:%.*]] = shufflevector <8 x float> [[A]], <8 x float> [[B]], <8 x i32> <i32 1, i32 3, i32 9, i32 poison, i32 5, i32 7, i32 poison, i32 poison>
 ; SSE2-NEXT:    [[TMP6:%.*]] = fsub <8 x float> [[TMP5]], [[TMP8]]
-; SSE2-NEXT:    [[HSUB5:%.*]] = insertelement <8 x float> [[TMP6]], float [[A67]], i64 5
 ; SSE2-NEXT:    [[TMP7:%.*]] = shufflevector <2 x float> [[TMP3]], <2 x float> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; SSE2-NEXT:    [[RESULT:%.*]] = shufflevector <8 x float> [[HSUB5]], <8 x float> [[TMP7]], <8 x i32> <i32 0, i32 1, i32 2, i32 poison, i32 4, i32 5, i32 8, i32 9>
+; SSE2-NEXT:    [[RESULT:%.*]] = shufflevector <8 x float> [[TMP6]], <8 x float> [[TMP7]], <8 x i32> <i32 0, i32 1, i32 2, i32 poison, i32 4, i32 5, i32 8, i32 9>
 ; SSE2-NEXT:    ret <8 x float> [[RESULT]]
 ;
 ; SSE4-LABEL: @sub_v8f32_012u4567(
-; SSE4-NEXT:    [[A6:%.*]] = extractelement <8 x float> [[A:%.*]], i64 6
-; SSE4-NEXT:    [[A7:%.*]] = extractelement <8 x float> [[A]], i64 7
-; SSE4-NEXT:    [[A67:%.*]] = fsub float [[A6]], [[A7]]
-; SSE4-NEXT:    [[TMP2:%.*]] = shufflevector <8 x float> [[A]], <8 x float> [[B:%.*]], <8 x i32> <i32 0, i32 2, i32 8, i32 poison, i32 4, i32 poison, i32 poison, i32 poison>
-; SSE4-NEXT:    [[TMP5:%.*]] = shufflevector <8 x float> [[A]], <8 x float> [[B]], <8 x i32> <i32 1, i32 3, i32 9, i32 poison, i32 5, i32 poison, i32 poison, i32 poison>
-; SSE4-NEXT:    [[TMP3:%.*]] = fsub <8 x float> [[TMP2]], [[TMP5]]
-; SSE4-NEXT:    [[HSUB5:%.*]] = insertelement <8 x float> [[TMP3]], float [[A67]], i64 5
-; SSE4-NEXT:    [[TMP4:%.*]] = shufflevector <8 x float> [[B]], <8 x float> poison, <8 x i32> <i32 4, i32 6, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; SSE4-NEXT:    [[TMP7:%.*]] = shufflevector <8 x float> [[B]], <8 x float> poison, <8 x i32> <i32 5, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; SSE4-NEXT:    [[TMP6:%.*]] = fsub <8 x float> [[TMP4]], [[TMP7]]
-; SSE4-NEXT:    [[RESULT:%.*]] = shufflevector <8 x float> [[HSUB5]], <8 x float> [[TMP6]], <8 x i32> <i32 0, i32 1, i32 2, i32 poison, i32 4, i32 5, i32 8, i32 9>
-; SSE4-NEXT:    ret <8 x float> [[RESULT]]
+; SSE4-NEXT:    [[TMP8:%.*]] = shufflevector <8 x float> [[A:%.*]], <8 x float> [[B:%.*]], <8 x i32> <i32 0, i32 2, i32 8, i32 poison, i32 4, i32 6, i32 12, i32 14>
+; SSE4-NEXT:    [[TMP5:%.*]] = shufflevector <8 x float> [[A]], <8 x float> [[B]], <8 x i32> <i32 1, i32 3, i32 9, i32 poison, i32 5, i32 7, i32 13, i32 15>
+; SSE4-NEXT:    [[TMP9:%.*]] = fsub <8 x float> [[TMP8]], [[TMP5]]
+; SSE4-NEXT:    ret <8 x float> [[TMP9]]
 ;
 ; AVX-LABEL: @sub_v8f32_012u4567(
 ; AVX-NEXT:    [[TMP5:%.*]] = shufflevector <8 x float> [[A:%.*]], <8 x float> [[B:%.*]], <8 x i32> <i32 0, i32 2, i32 8, i32 poison, i32 4, i32 6, i32 12, i32 14>
@@ -776,6 +1166,71 @@ define <8 x float> @sub_v8f32_012u4567(<8 x float> %a, <8 x float> %b) {
   ret <8 x float> %result
 }
 
+define <8 x float> @sub_v8f32_76u43210(<8 x float> %a, <8 x float> %b) {
+; SSE2-LABEL: @sub_v8f32_76u43210(
+; SSE2-NEXT:    [[TMP1:%.*]] = shufflevector <8 x float> [[A:%.*]], <8 x float> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+; SSE2-NEXT:    [[TMP2:%.*]] = shufflevector <8 x float> [[A]], <8 x float> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+; SSE2-NEXT:    [[TMP3:%.*]] = fsub <4 x float> [[TMP1]], [[TMP2]]
+; SSE2-NEXT:    [[B0:%.*]] = extractelement <8 x float> [[B:%.*]], i64 0
+; SSE2-NEXT:    [[B1:%.*]] = extractelement <8 x float> [[B]], i64 1
+; SSE2-NEXT:    [[B01:%.*]] = fsub float [[B0]], [[B1]]
+; SSE2-NEXT:    [[TMP4:%.*]] = shufflevector <8 x float> [[B]], <8 x float> poison, <2 x i32> <i32 4, i32 6>
+; SSE2-NEXT:    [[TMP5:%.*]] = shufflevector <8 x float> [[B]], <8 x float> poison, <2 x i32> <i32 5, i32 7>
+; SSE2-NEXT:    [[TMP6:%.*]] = fsub <2 x float> [[TMP4]], [[TMP5]]
+; SSE2-NEXT:    [[TMP7:%.*]] = shufflevector <4 x float> [[TMP3]], <4 x float> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE2-NEXT:    [[HSUB4:%.*]] = insertelement <8 x float> [[TMP7]], float [[B01]], i64 4
+; SSE2-NEXT:    [[TMP8:%.*]] = shufflevector <2 x float> [[TMP6]], <2 x float> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE2-NEXT:    [[RESULT:%.*]] = shufflevector <8 x float> [[TMP8]], <8 x float> [[HSUB4]], <8 x i32> <i32 1, i32 0, i32 poison, i32 12, i32 11, i32 10, i32 9, i32 8>
+; SSE2-NEXT:    ret <8 x float> [[RESULT]]
+;
+; SSE4-LABEL: @sub_v8f32_76u43210(
+; SSE4-NEXT:    [[TMP4:%.*]] = shufflevector <8 x float> [[B:%.*]], <8 x float> [[A:%.*]], <8 x i32> <i32 6, i32 4, i32 poison, i32 0, i32 14, i32 12, i32 10, i32 8>
+; SSE4-NEXT:    [[TMP5:%.*]] = shufflevector <8 x float> [[B]], <8 x float> [[A]], <8 x i32> <i32 7, i32 5, i32 poison, i32 1, i32 15, i32 13, i32 11, i32 9>
+; SSE4-NEXT:    [[TMP6:%.*]] = fsub <8 x float> [[TMP4]], [[TMP5]]
+; SSE4-NEXT:    ret <8 x float> [[TMP6]]
+;
+; AVX-LABEL: @sub_v8f32_76u43210(
+; AVX-NEXT:    [[TMP1:%.*]] = shufflevector <8 x float> [[B:%.*]], <8 x float> [[A:%.*]], <8 x i32> <i32 6, i32 4, i32 poison, i32 0, i32 14, i32 12, i32 10, i32 8>
+; AVX-NEXT:    [[TMP2:%.*]] = shufflevector <8 x float> [[B]], <8 x float> [[A]], <8 x i32> <i32 7, i32 5, i32 poison, i32 1, i32 15, i32 13, i32 11, i32 9>
+; AVX-NEXT:    [[RESULT:%.*]] = fsub <8 x float> [[TMP1]], [[TMP2]]
+; AVX-NEXT:    ret <8 x float> [[RESULT]]
+;
+  %a0 = extractelement <8 x float> %a, i32 0
+  %a1 = extractelement <8 x float> %a, i32 1
+  %a2 = extractelement <8 x float> %a, i32 2
+  %a3 = extractelement <8 x float> %a, i32 3
+  %a4 = extractelement <8 x float> %a, i32 4
+  %a5 = extractelement <8 x float> %a, i32 5
+  %a6 = extractelement <8 x float> %a, i32 6
+  %a7 = extractelement <8 x float> %a, i32 7
+  %a01 = fsub float %a0, %a1
+  %a23 = fsub float %a2, %a3
+  %a45 = fsub float %a4, %a5
+  %a67 = fsub float %a6, %a7
+  %b0 = extractelement <8 x float> %b, i32 0
+  %b1 = extractelement <8 x float> %b, i32 1
+  %b2 = extractelement <8 x float> %b, i32 2
+  %b3 = extractelement <8 x float> %b, i32 3
+  %b4 = extractelement <8 x float> %b, i32 4
+  %b5 = extractelement <8 x float> %b, i32 5
+  %b6 = extractelement <8 x float> %b, i32 6
+  %b7 = extractelement <8 x float> %b, i32 7
+  %b01 = fsub float %b0, %b1
+  %b23 = fsub float %b2, %b3
+  %b45 = fsub float %b4, %b5
+  %b67 = fsub float %b6, %b7
+  %hsub0 = insertelement <8 x float> poison, float %a01, i32 0
+  %hsub1 = insertelement <8 x float> %hsub0, float %a23, i32 1
+  %hsub2 = insertelement <8 x float> %hsub1, float %a45, i32 2
+  %hsub3 = insertelement <8 x float> %hsub2, float %a67, i32 3
+  %hsub4 = insertelement <8 x float> %hsub3, float %b01, i32 4
+  %hsub5 = insertelement <8 x float> %hsub4, float %b23, i32 5
+  %hsub6 = insertelement <8 x float> %hsub5, float %b45, i32 6
+  %hsub7 = insertelement <8 x float> %hsub6, float %b67, i32 7
+  %result = shufflevector <8 x float> %hsub7, <8 x float> %a, <8 x i32> <i32 7, i32 6, i32 poison, i32 4, i32 3, i32 2, i32 1, i32 0>
+  ret <8 x float> %result
+}
+
 ;
 ; v2f64
 ;
@@ -879,14 +1334,10 @@ define <4 x double> @sub_v4f64_u123(<4 x double> %a, <4 x double> %b) {
 ; SSE2-NEXT:    ret <4 x double> [[RESULT]]
 ;
 ; SSE4-LABEL: @sub_v4f64_u123(
-; SSE4-NEXT:    [[B2:%.*]] = extractelement <4 x double> [[B:%.*]], i64 2
-; SSE4-NEXT:    [[B3:%.*]] = extractelement <4 x double> [[B]], i64 3
-; SSE4-NEXT:    [[B23:%.*]] = fsub double [[B2]], [[B3]]
-; SSE4-NEXT:    [[TMP2:%.*]] = shufflevector <4 x double> [[B]], <4 x double> [[A:%.*]], <4 x i32> <i32 poison, i32 0, i32 6, i32 poison>
-; SSE4-NEXT:    [[TMP4:%.*]] = shufflevector <4 x double> [[B]], <4 x double> [[A]], <4 x i32> <i32 poison, i32 1, i32 7, i32 poison>
+; SSE4-NEXT:    [[TMP2:%.*]] = shufflevector <4 x double> [[B:%.*]], <4 x double> [[A:%.*]], <4 x i32> <i32 poison, i32 0, i32 6, i32 2>
+; SSE4-NEXT:    [[TMP4:%.*]] = shufflevector <4 x double> [[B]], <4 x double> [[A]], <4 x i32> <i32 poison, i32 1, i32 7, i32 3>
 ; SSE4-NEXT:    [[TMP3:%.*]] = fsub <4 x double> [[TMP2]], [[TMP4]]
-; SSE4-NEXT:    [[RESULT:%.*]] = insertelement <4 x double> [[TMP3]], double [[B23]], i64 3
-; SSE4-NEXT:    ret <4 x double> [[RESULT]]
+; SSE4-NEXT:    ret <4 x double> [[TMP3]]
 ;
 ; AVX-LABEL: @sub_v4f64_u123(
 ; AVX-NEXT:    [[TMP2:%.*]] = shufflevector <4 x double> [[B:%.*]], <4 x double> [[A:%.*]], <4 x i32> <i32 poison, i32 0, i32 6, i32 2>
@@ -927,13 +1378,10 @@ define <4 x double> @sub_v4f64_0u23(<4 x double> %a, <4 x double> %b) {
 ; SSE2-NEXT:    ret <4 x double> [[RESULT]]
 ;
 ; SSE4-LABEL: @sub_v4f64_0u23(
-; SSE4-NEXT:    [[B2:%.*]] = extractelement <4 x double> [[B:%.*]], i64 2
-; SSE4-NEXT:    [[B3:%.*]] = extractelement <4 x double> [[B]], i64 3
-; SSE4-NEXT:    [[B23:%.*]] = fsub double [[B2]], [[B3]]
-; SSE4-NEXT:    [[TMP3:%.*]] = shufflevector <4 x double> [[TMP1:%.*]], <4 x double> poison, <4 x i32> <i32 1, i32 poison, i32 3, i32 poison>
-; SSE4-NEXT:    [[TMP2:%.*]] = fsub <4 x double> [[TMP1]], [[TMP3]]
-; SSE4-NEXT:    [[RESULT:%.*]] = insertelement <4 x double> [[TMP2]], double [[B23]], i64 3
-; SSE4-NEXT:    ret <4 x double> [[RESULT]]
+; SSE4-NEXT:    [[TMP2:%.*]] = shufflevector <4 x double> [[A:%.*]], <4 x double> [[B:%.*]], <4 x i32> <i32 0, i32 1, i32 2, i32 6>
+; SSE4-NEXT:    [[TMP3:%.*]] = shufflevector <4 x double> [[A]], <4 x double> [[B]], <4 x i32> <i32 1, i32 poison, i32 3, i32 7>
+; SSE4-NEXT:    [[TMP4:%.*]] = fsub <4 x double> [[TMP2]], [[TMP3]]
+; SSE4-NEXT:    ret <4 x double> [[TMP4]]
 ;
 ; AVX-LABEL: @sub_v4f64_0u23(
 ; AVX-NEXT:    [[TMP2:%.*]] = shufflevector <4 x double> [[A:%.*]], <4 x double> [[B:%.*]], <4 x i32> <i32 0, i32 1, i32 2, i32 6>
@@ -974,14 +1422,10 @@ define <4 x double> @sub_v4f64_01u3(<4 x double> %a, <4 x double> %b) {
 ; SSE2-NEXT:    ret <4 x double> [[RESULT]]
 ;
 ; SSE4-LABEL: @sub_v4f64_01u3(
-; SSE4-NEXT:    [[B2:%.*]] = extractelement <4 x double> [[B:%.*]], i64 2
-; SSE4-NEXT:    [[B3:%.*]] = extractelement <4 x double> [[B]], i64 3
-; SSE4-NEXT:    [[B23:%.*]] = fsub double [[B2]], [[B3]]
-; SSE4-NEXT:    [[TMP2:%.*]] = shufflevector <4 x double> [[A:%.*]], <4 x double> [[B]], <4 x i32> <i32 0, i32 4, i32 poison, i32 poison>
-; SSE4-NEXT:    [[TMP4:%.*]] = shufflevector <4 x double> [[A]], <4 x double> [[B]], <4 x i32> <i32 1, i32 5, i32 poison, i32 poison>
+; SSE4-NEXT:    [[TMP2:%.*]] = shufflevector <4 x double> [[A:%.*]], <4 x double> [[B:%.*]], <4 x i32> <i32 0, i32 4, i32 poison, i32 6>
+; SSE4-NEXT:    [[TMP4:%.*]] = shufflevector <4 x double> [[A]], <4 x double> [[B]], <4 x i32> <i32 1, i32 5, i32 poison, i32 7>
 ; SSE4-NEXT:    [[TMP3:%.*]] = fsub <4 x double> [[TMP2]], [[TMP4]]
-; SSE4-NEXT:    [[RESULT:%.*]] = insertelement <4 x double> [[TMP3]], double [[B23]], i64 3
-; SSE4-NEXT:    ret <4 x double> [[RESULT]]
+; SSE4-NEXT:    ret <4 x double> [[TMP3]]
 ;
 ; AVX-LABEL: @sub_v4f64_01u3(
 ; AVX-NEXT:    [[TMP2:%.*]] = shufflevector <4 x double> [[A:%.*]], <4 x double> [[B:%.*]], <4 x i32> <i32 0, i32 4, i32 poison, i32 6>
@@ -1022,14 +1466,10 @@ define <4 x double> @sub_v4f64_012u(<4 x double> %a, <4 x double> %b) {
 ; SSE2-NEXT:    ret <4 x double> [[RESULT]]
 ;
 ; SSE4-LABEL: @sub_v4f64_012u(
-; SSE4-NEXT:    [[A2:%.*]] = extractelement <4 x double> [[A:%.*]], i64 2
-; SSE4-NEXT:    [[A3:%.*]] = extractelement <4 x double> [[A]], i64 3
-; SSE4-NEXT:    [[A23:%.*]] = fsub double [[A2]], [[A3]]
-; SSE4-NEXT:    [[TMP2:%.*]] = shufflevector <4 x double> [[A]], <4 x double> [[B:%.*]], <4 x i32> <i32 0, i32 4, i32 poison, i32 poison>
-; SSE4-NEXT:    [[TMP4:%.*]] = shufflevector <4 x double> [[A]], <4 x double> [[B]], <4 x i32> <i32 1, i32 5, i32 poison, i32 poison>
+; SSE4-NEXT:    [[TMP2:%.*]] = shufflevector <4 x double> [[A:%.*]], <4 x double> [[B:%.*]], <4 x i32> <i32 0, i32 4, i32 2, i32 poison>
+; SSE4-NEXT:    [[TMP4:%.*]] = shufflevector <4 x double> [[A]], <4 x double> [[B]], <4 x i32> <i32 1, i32 5, i32 3, i32 poison>
 ; SSE4-NEXT:    [[TMP3:%.*]] = fsub <4 x double> [[TMP2]], [[TMP4]]
-; SSE4-NEXT:    [[RESULT:%.*]] = insertelement <4 x double> [[TMP3]], double [[A23]], i64 2
-; SSE4-NEXT:    ret <4 x double> [[RESULT]]
+; SSE4-NEXT:    ret <4 x double> [[TMP3]]
 ;
 ; AVX-LABEL: @sub_v4f64_012u(
 ; AVX-NEXT:    [[TMP2:%.*]] = shufflevector <4 x double> [[A:%.*]], <4 x double> [[B:%.*]], <4 x i32> <i32 0, i32 4, i32 2, i32 poison>
@@ -1136,3 +1576,47 @@ define <4 x double> @sub_v4f64_01uu(<4 x double> %a, <4 x double> %b) {
   %result = shufflevector <4 x double> %hsub3, <4 x double> %a, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
   ret <4 x double> %result
 }
+
+define <4 x double> @sub_v4f64_32u0(<4 x double> %a, <4 x double> %b) {
+; SSE2-LABEL: @sub_v4f64_32u0(
+; SSE2-NEXT:    [[A0:%.*]] = extractelement <4 x double> [[A:%.*]], i64 0
+; SSE2-NEXT:    [[A1:%.*]] = extractelement <4 x double> [[A]], i64 1
+; SSE2-NEXT:    [[A01:%.*]] = fsub double [[A0]], [[A1]]
+; SSE2-NEXT:    [[TMP1:%.*]] = shufflevector <4 x double> [[B:%.*]], <4 x double> poison, <2 x i32> <i32 2, i32 0>
+; SSE2-NEXT:    [[TMP2:%.*]] = shufflevector <4 x double> [[B]], <4 x double> poison, <2 x i32> <i32 3, i32 1>
+; SSE2-NEXT:    [[TMP3:%.*]] = fsub <2 x double> [[TMP1]], [[TMP2]]
+; SSE2-NEXT:    [[TMP4:%.*]] = shufflevector <2 x double> [[TMP3]], <2 x double> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; SSE2-NEXT:    [[RESULT:%.*]] = insertelement <4 x double> [[TMP4]], double [[A01]], i64 3
+; SSE2-NEXT:    ret <4 x double> [[RESULT]]
+;
+; SSE4-LABEL: @sub_v4f64_32u0(
+; SSE4-NEXT:    [[TMP1:%.*]] = shufflevector <4 x double> [[B:%.*]], <4 x double> [[A:%.*]], <4 x i32> <i32 2, i32 0, i32 poison, i32 4>
+; SSE4-NEXT:    [[TMP2:%.*]] = shufflevector <4 x double> [[B]], <4 x double> [[A]], <4 x i32> <i32 3, i32 1, i32 poison, i32 5>
+; SSE4-NEXT:    [[RESULT:%.*]] = fsub <4 x double> [[TMP1]], [[TMP2]]
+; SSE4-NEXT:    ret <4 x double> [[RESULT]]
+;
+; AVX-LABEL: @sub_v4f64_32u0(
+; AVX-NEXT:    [[TMP1:%.*]] = shufflevector <4 x double> [[B:%.*]], <4 x double> [[A:%.*]], <4 x i32> <i32 2, i32 0, i32 poison, i32 4>
+; AVX-NEXT:    [[TMP2:%.*]] = shufflevector <4 x double> [[B]], <4 x double> [[A]], <4 x i32> <i32 3, i32 1, i32 poison, i32 5>
+; AVX-NEXT:    [[RESULT:%.*]] = fsub <4 x double> [[TMP1]], [[TMP2]]
+; AVX-NEXT:    ret <4 x double> [[RESULT]]
+;
+  %a0 = extractelement <4 x double> %a, i32 0
+  %a1 = extractelement <4 x double> %a, i32 1
+  %a2 = extractelement <4 x double> %a, i32 2
+  %a3 = extractelement <4 x double> %a, i32 3
+  %a01 = fsub double %a0, %a1
+  %a23 = fsub double %a2, %a3
+  %b0 = extractelement <4 x double> %b, i32 0
+  %b1 = extractelement <4 x double> %b, i32 1
+  %b2 = extractelement <4 x double> %b, i32 2
+  %b3 = extractelement <4 x double> %b, i32 3
+  %b01 = fsub double %b0, %b1
+  %b23 = fsub double %b2, %b3
+  %hsub0 = insertelement <4 x double> poison, double %a01, i32 0
+  %hsub1 = insertelement <4 x double> %hsub0, double %a23, i32 1
+  %hsub2 = insertelement <4 x double> %hsub1, double %b01, i32 2
+  %hsub3 = insertelement <4 x double> %hsub2, double %b23, i32 3
+  %result = shufflevector <4 x double> %hsub3, <4 x double> %a, <4 x i32> <i32 3, i32 2, i32 poison, i32 0>
+  ret <4 x double> %result
+}
diff --git a/llvm/test/Transforms/PhaseOrdering/X86/loop-vectorizer-noalias.ll b/llvm/test/Transforms/PhaseOrdering/X86/loop-vectorizer-noalias.ll
index 5c85c0d21f59f..1a4af5ab3cb4b 100644
--- a/llvm/test/Transforms/PhaseOrdering/X86/loop-vectorizer-noalias.ll
+++ b/llvm/test/Transforms/PhaseOrdering/X86/loop-vectorizer-noalias.ll
@@ -16,7 +16,7 @@ entry:
 ; This loop should not get vectorized.
 define void @accsum(ptr noundef %vals, i64 noundef %num) #0 {
 ; CHECK-LABEL: define void @accsum(
-; CHECK-SAME: ptr nocapture noundef [[VALS:%.*]], i64 noundef [[NUM:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
+; CHECK-SAME: ptr noundef captures(none) [[VALS:%.*]], i64 noundef [[NUM:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[CMP1:%.*]] = icmp ugt i64 [[NUM]], 1
 ; CHECK-NEXT:    br i1 [[CMP1]], label [[FOR_BODY_PREHEADER:%.*]], label [[FOR_END:%.*]]
diff --git a/llvm/test/Transforms/PhaseOrdering/X86/pr48844-br-to-switch-vectorization.ll b/llvm/test/Transforms/PhaseOrdering/X86/pr48844-br-to-switch-vectorization.ll
index 9196e3c96462a..2fe420183c683 100644
--- a/llvm/test/Transforms/PhaseOrdering/X86/pr48844-br-to-switch-vectorization.ll
+++ b/llvm/test/Transforms/PhaseOrdering/X86/pr48844-br-to-switch-vectorization.ll
@@ -109,7 +109,7 @@ define dso_local void @test(ptr %start, ptr %end) #0 {
 ; AVX2-NEXT:    [[CMP_N17:%.*]] = icmp eq i64 [[TMP3]], [[N_VEC10]]
 ; AVX2-NEXT:    br i1 [[CMP_N17]], label [[EXIT]], label [[BB12_PREHEADER1]]
 ; AVX2:       bb12.preheader:
-; AVX2-NEXT:    [[PTR2_PH:%.*]] = phi ptr [ [[IND_END11]], [[VEC_EPILOG_ITER_CHECK]] ], [ [[START]], [[BB12_PREHEADER]] ], [ [[IND_END]], [[VEC_EPILOG_MIDDLE_BLOCK]] ]
+; AVX2-NEXT:    [[PTR2_PH:%.*]] = phi ptr [ [[START]], [[BB12_PREHEADER]] ], [ [[IND_END11]], [[VEC_EPILOG_ITER_CHECK]] ], [ [[IND_END]], [[VEC_EPILOG_MIDDLE_BLOCK]] ]
 ; AVX2-NEXT:    br label [[BB13:%.*]]
 ; AVX2:       bb12:
 ; AVX2-NEXT:    [[PTR2:%.*]] = phi ptr [ [[PTR_NEXT:%.*]], [[LATCH:%.*]] ], [ [[PTR2_PH]], [[BB12_PREHEADER1]] ]
diff --git a/llvm/test/Transforms/PhaseOrdering/X86/pr50392.ll b/llvm/test/Transforms/PhaseOrdering/X86/pr50392.ll
index d92df9741644b..842cf42505673 100644
--- a/llvm/test/Transforms/PhaseOrdering/X86/pr50392.ll
+++ b/llvm/test/Transforms/PhaseOrdering/X86/pr50392.ll
@@ -21,14 +21,10 @@ define <4 x double> @PR50392(<4 x double> %a, <4 x double> %b) {
 ; SSE2-NEXT:    ret <4 x double> [[SHUFFLE]]
 ;
 ; SSE4-LABEL: @PR50392(
-; SSE4-NEXT:    [[TMP1:%.*]] = shufflevector <4 x double> [[A:%.*]], <4 x double> [[B:%.*]], <4 x i32> <i32 0, i32 poison, i32 4, i32 poison>
-; SSE4-NEXT:    [[TMP2:%.*]] = shufflevector <4 x double> [[A]], <4 x double> [[B]], <4 x i32> <i32 1, i32 poison, i32 5, i32 poison>
+; SSE4-NEXT:    [[TMP1:%.*]] = shufflevector <4 x double> [[A:%.*]], <4 x double> [[B:%.*]], <4 x i32> <i32 0, i32 poison, i32 4, i32 6>
+; SSE4-NEXT:    [[TMP2:%.*]] = shufflevector <4 x double> [[A]], <4 x double> [[B]], <4 x i32> <i32 1, i32 poison, i32 5, i32 7>
 ; SSE4-NEXT:    [[TMP3:%.*]] = fadd <4 x double> [[TMP1]], [[TMP2]]
-; SSE4-NEXT:    [[VECEXT10:%.*]] = extractelement <4 x double> [[B]], i64 2
-; SSE4-NEXT:    [[VECEXT11:%.*]] = extractelement <4 x double> [[B]], i64 3
-; SSE4-NEXT:    [[ADD12:%.*]] = fadd double [[VECEXT10]], [[VECEXT11]]
-; SSE4-NEXT:    [[SHUFFLE:%.*]] = insertelement <4 x double> [[TMP3]], double [[ADD12]], i64 3
-; SSE4-NEXT:    ret <4 x double> [[SHUFFLE]]
+; SSE4-NEXT:    ret <4 x double> [[TMP3]]
 ;
 ; AVX-LABEL: @PR50392(
 ; AVX-NEXT:    [[B:%.*]] = shufflevector <4 x double> [[A:%.*]], <4 x double> [[B1:%.*]], <4 x i32> <i32 0, i32 poison, i32 4, i32 6>
diff --git a/llvm/test/Transforms/PhaseOrdering/X86/pr88239.ll b/llvm/test/Transforms/PhaseOrdering/X86/pr88239.ll
index b3625094f07ea..c98e7d349e6c0 100644
--- a/llvm/test/Transforms/PhaseOrdering/X86/pr88239.ll
+++ b/llvm/test/Transforms/PhaseOrdering/X86/pr88239.ll
@@ -6,7 +6,7 @@ target triple = "x86_64-unknown-linux-gnu"
 
 define void @foo(ptr noalias noundef %0, ptr noalias noundef %1) optsize {
 ; CHECK-LABEL: define void @foo(
-; CHECK-SAME: ptr noalias nocapture noundef readonly [[TMP0:%.*]], ptr noalias nocapture noundef writeonly [[TMP1:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
+; CHECK-SAME: ptr noalias noundef readonly captures(none) [[TMP0:%.*]], ptr noalias noundef writeonly captures(none) [[TMP1:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
 ; CHECK-NEXT:  vector.ph:
 ; CHECK-NEXT:    [[INVARIANT_GEP:%.*]] = getelementptr i8, ptr [[TMP0]], i64 -28
 ; CHECK-NEXT:    br label [[TMP4:%.*]]
diff --git a/llvm/test/Transforms/PhaseOrdering/X86/preserve-access-group.ll b/llvm/test/Transforms/PhaseOrdering/X86/preserve-access-group.ll
index 7bb22e2e9f5b5..be7f4c2a941a0 100644
--- a/llvm/test/Transforms/PhaseOrdering/X86/preserve-access-group.ll
+++ b/llvm/test/Transforms/PhaseOrdering/X86/preserve-access-group.ll
@@ -8,7 +8,7 @@ target triple = "x86_64-unknown-linux-gnu"
 ; End-to-end test for https://github.com/llvm/llvm-project/issues/115595.
 define void @test(i32 noundef %nface, i32 noundef %ncell, ptr noalias noundef %face_cell, ptr noalias noundef %x, ptr noalias noundef %y) #0 {
 ; CHECK-LABEL: define void @test(
-; CHECK-SAME: i32 noundef [[NFACE:%.*]], i32 noundef [[NCELL:%.*]], ptr noalias nocapture noundef readonly [[FACE_CELL:%.*]], ptr noalias nocapture noundef readonly [[X:%.*]], ptr noalias nocapture noundef [[Y:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
+; CHECK-SAME: i32 noundef [[NFACE:%.*]], i32 noundef [[NCELL:%.*]], ptr noalias noundef readonly captures(none) [[FACE_CELL:%.*]], ptr noalias noundef readonly captures(none) [[X:%.*]], ptr noalias noundef captures(none) [[Y:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
 ; CHECK-NEXT:  [[ENTRY:.*:]]
 ; CHECK-NEXT:    [[CMP8:%.*]] = icmp sgt i32 [[NFACE]], 0
 ; CHECK-NEXT:    br i1 [[CMP8]], label %[[FOR_BODY_PREHEADER:.*]], label %[[FOR_COND_CLEANUP:.*]]
diff --git a/llvm/test/Transforms/PhaseOrdering/X86/spurious-peeling.ll b/llvm/test/Transforms/PhaseOrdering/X86/spurious-peeling.ll
index a715736df887f..438a93c735796 100644
--- a/llvm/test/Transforms/PhaseOrdering/X86/spurious-peeling.ll
+++ b/llvm/test/Transforms/PhaseOrdering/X86/spurious-peeling.ll
@@ -14,7 +14,7 @@ $_ZN12FloatVecPair6vecIncEv = comdat any
 
 define dso_local void @_Z13vecIncFromPtrP12FloatVecPair(ptr %FVP) {
 ; O1-LABEL: define {{[^@]+}}@_Z13vecIncFromPtrP12FloatVecPair
-; O1-SAME: (ptr nocapture readonly [[FVP:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
+; O1-SAME: (ptr readonly captures(none) [[FVP:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
 ; O1-NEXT:  entry:
 ; O1-NEXT:    [[VSRC23_I:%.*]] = getelementptr inbounds nuw i8, ptr [[FVP]], i64 16
 ; O1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VSRC23_I]], align 8, !tbaa [[TBAA0:![0-9]+]]
@@ -44,7 +44,7 @@ define dso_local void @_Z13vecIncFromPtrP12FloatVecPair(ptr %FVP) {
 ; O1-NEXT:    ret void
 ;
 ; O23-LABEL: define {{[^@]+}}@_Z13vecIncFromPtrP12FloatVecPair
-; O23-SAME: (ptr nocapture readonly [[FVP:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
+; O23-SAME: (ptr readonly captures(none) [[FVP:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
 ; O23-NEXT:  entry:
 ; O23-NEXT:    [[VSRC23_I:%.*]] = getelementptr inbounds nuw i8, ptr [[FVP]], i64 16
 ; O23-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VSRC23_I]], align 8, !tbaa [[TBAA0:![0-9]+]]
diff --git a/llvm/test/Transforms/PhaseOrdering/X86/unroll-vectorizer.ll b/llvm/test/Transforms/PhaseOrdering/X86/unroll-vectorizer.ll
index ec0c2b40640f4..9b3a6ecf8bc8b 100644
--- a/llvm/test/Transforms/PhaseOrdering/X86/unroll-vectorizer.ll
+++ b/llvm/test/Transforms/PhaseOrdering/X86/unroll-vectorizer.ll
@@ -8,7 +8,7 @@ target triple = "x86_64-unknown-linux-gnu"
 
 define void @foo(ptr %a, <32 x i8> %_0) #0 {
 ; CHECK-LABEL: define void @foo(
-; CHECK-SAME: ptr nocapture writeonly initializes((0, 32)) [[A:%.*]], <32 x i8> [[_0:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
+; CHECK-SAME: ptr writeonly captures(none) initializes((0, 32)) [[A:%.*]], <32 x i8> [[_0:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
 ; CHECK-NEXT:  start:
 ; CHECK-NEXT:    store <32 x i8> [[_0]], ptr [[A]], align 1
 ; CHECK-NEXT:    ret void
diff --git a/llvm/test/Transforms/PhaseOrdering/X86/vdiv.ll b/llvm/test/Transforms/PhaseOrdering/X86/vdiv.ll
index 703a53949a063..7817c23e6a3ec 100644
--- a/llvm/test/Transforms/PhaseOrdering/X86/vdiv.ll
+++ b/llvm/test/Transforms/PhaseOrdering/X86/vdiv.ll
@@ -90,7 +90,7 @@ define void @vdiv(ptr %x, ptr %y, double %a, i32 %N) #0 {
 ; CHECK-NEXT:    [[CMP_N17:%.*]] = icmp eq i64 [[N_VEC11]], [[WIDE_TRIP_COUNT]]
 ; CHECK-NEXT:    br i1 [[CMP_N17]], label [[FOR_END]], label [[FOR_BODY_PREHEADER9]]
 ; CHECK:       for.body.preheader:
-; CHECK-NEXT:    [[INDVARS_IV_PH:%.*]] = phi i64 [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[FOR_BODY_PREHEADER]] ], [ [[N_VEC11]], [[VEC_EPILOG_MIDDLE_BLOCK]] ]
+; CHECK-NEXT:    [[INDVARS_IV_PH:%.*]] = phi i64 [ 0, [[FOR_BODY_PREHEADER]] ], [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ [[N_VEC11]], [[VEC_EPILOG_MIDDLE_BLOCK]] ]
 ; CHECK-NEXT:    [[TMP43:%.*]] = sub nsw i64 [[WIDE_TRIP_COUNT]], [[INDVARS_IV_PH]]
 ; CHECK-NEXT:    [[XTRAITER:%.*]] = and i64 [[TMP43]], 7
 ; CHECK-NEXT:    [[LCMP_MOD_NOT:%.*]] = icmp eq i64 [[XTRAITER]], 0
diff --git a/llvm/test/Transforms/PhaseOrdering/early-arg-attrs-inference.ll b/llvm/test/Transforms/PhaseOrdering/early-arg-attrs-inference.ll
index ba4583160b137..93a8c803aba37 100644
--- a/llvm/test/Transforms/PhaseOrdering/early-arg-attrs-inference.ll
+++ b/llvm/test/Transforms/PhaseOrdering/early-arg-attrs-inference.ll
@@ -3,7 +3,7 @@
 
 define i32 @f(ptr noalias %p, i32 %c) {
 ; CHECK-LABEL: define noundef i32 @f
-; CHECK-SAME: (ptr noalias nocapture readonly [[P:%.*]], i32 [[C:%.*]]) local_unnamed_addr {
+; CHECK-SAME: (ptr noalias readonly captures(none) [[P:%.*]], i32 [[C:%.*]]) local_unnamed_addr {
 ; CHECK-NEXT:    tail call void @g()
 ; CHECK-NEXT:    tail call void @g()
 ; CHECK-NEXT:    tail call void @g()
diff --git a/llvm/test/Transforms/PhaseOrdering/gvn-replacement-vs-hoist.ll b/llvm/test/Transforms/PhaseOrdering/gvn-replacement-vs-hoist.ll
index e156111bd445e..ae98fe602f3fc 100644
--- a/llvm/test/Transforms/PhaseOrdering/gvn-replacement-vs-hoist.ll
+++ b/llvm/test/Transforms/PhaseOrdering/gvn-replacement-vs-hoist.ll
@@ -3,7 +3,7 @@
 
 define void @test(ptr noundef %a, i32 noundef %beam) {
 ; CHECK-LABEL: define void @test
-; CHECK-SAME: (ptr nocapture noundef writeonly [[A:%.*]], i32 noundef [[BEAM:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
+; CHECK-SAME: (ptr noundef writeonly captures(none) [[A:%.*]], i32 noundef [[BEAM:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[MUL:%.*]] = shl nuw nsw i32 [[BEAM]], 1
 ; CHECK-NEXT:    [[IDXPROM:%.*]] = zext nneg i32 [[MUL]] to i64
diff --git a/llvm/test/Transforms/PhaseOrdering/loop-access-checks.ll b/llvm/test/Transforms/PhaseOrdering/loop-access-checks.ll
index 394204e9409a1..e53b8687af915 100644
--- a/llvm/test/Transforms/PhaseOrdering/loop-access-checks.ll
+++ b/llvm/test/Transforms/PhaseOrdering/loop-access-checks.ll
@@ -131,7 +131,7 @@ declare void @llvm.lifetime.end.p0(i64 immarg, ptr nocapture)
 
 define void @foo(ptr noundef nonnull align 8 dereferenceable(24) noalias %vec) #0 {
 ; CHECK-LABEL: define void @foo
-; CHECK-SAME: (ptr noalias nocapture noundef nonnull readonly align 8 dereferenceable(24) [[VEC:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
+; CHECK-SAME: (ptr noalias noundef nonnull readonly align 8 captures(none) dereferenceable(24) [[VEC:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[_M_FINISH_I_I:%.*]] = getelementptr inbounds nuw i8, ptr [[VEC]], i64 8
 ; CHECK-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[_M_FINISH_I_I]], align 8, !tbaa [[TBAA0:![0-9]+]]
@@ -270,7 +270,7 @@ declare void @abort()
 
 define void @loop_with_signed_induction(ptr noundef nonnull align 8 dereferenceable(24) %vec) {
 ; CHECK-LABEL: define void @loop_with_signed_induction
-; CHECK-SAME: (ptr nocapture noundef nonnull readonly align 8 dereferenceable(24) [[VEC:%.*]]) local_unnamed_addr #[[ATTR0]] {
+; CHECK-SAME: (ptr noundef nonnull readonly align 8 captures(none) dereferenceable(24) [[VEC:%.*]]) local_unnamed_addr #[[ATTR0]] {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[_M_FINISH_I_I:%.*]] = getelementptr inbounds nuw i8, ptr [[VEC]], i64 8
 ; CHECK-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[_M_FINISH_I_I]], align 8, !tbaa [[TBAA0]]
@@ -343,7 +343,7 @@ for.end:
 
 define void @monkey(ptr noundef %arr, i32 noundef %len) {
 ; CHECK-LABEL: define void @monkey
-; CHECK-SAME: (ptr nocapture noundef [[ARR:%.*]], i32 noundef [[LEN:%.*]]) local_unnamed_addr #[[ATTR1:[0-9]+]] {
+; CHECK-SAME: (ptr noundef captures(none) [[ARR:%.*]], i32 noundef [[LEN:%.*]]) local_unnamed_addr #[[ATTR1:[0-9]+]] {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[CMP8:%.*]] = icmp ugt i32 [[LEN]], 1
 ; CHECK-NEXT:    br i1 [[CMP8]], label [[FOR_BODY4_PREHEADER:%.*]], label [[FOR_COND_CLEANUP:%.*]]
diff --git a/llvm/test/Transforms/PhaseOrdering/lto-argpromotion-ipsccp.ll b/llvm/test/Transforms/PhaseOrdering/lto-argpromotion-ipsccp.ll
index 5435bc2eb8b37..bc42b0b354ab7 100644
--- a/llvm/test/Transforms/PhaseOrdering/lto-argpromotion-ipsccp.ll
+++ b/llvm/test/Transforms/PhaseOrdering/lto-argpromotion-ipsccp.ll
@@ -5,7 +5,7 @@
 
 define void @parent(ptr %p) {
 ; CHECK-LABEL: define void @parent(
-; CHECK-SAME: ptr nocapture [[P:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
+; CHECK-SAME: ptr captures(none) [[P:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
 ; CHECK-NEXT:    tail call fastcc void @child(ptr [[P]])
 ; CHECK-NEXT:    ret void
 ;
@@ -19,7 +19,7 @@ define void @parent(ptr %p) {
 
 define internal void @child(ptr %p, ptr %n, ptr %c) noinline {
 ; CHECK-LABEL: define internal fastcc void @child(
-; CHECK-SAME: ptr nocapture [[P:%.*]]) unnamed_addr #[[ATTR1:[0-9]+]] {
+; CHECK-SAME: ptr captures(none) [[P:%.*]]) unnamed_addr #[[ATTR1:[0-9]+]] {
 ; CHECK-NEXT:  [[ENTRY:.*]]:
 ; CHECK-NEXT:    br label %[[FOR_COND:.*]]
 ; CHECK:       [[FOR_COND]]:
diff --git a/llvm/test/Transforms/PhaseOrdering/memcpy-offset.ll b/llvm/test/Transforms/PhaseOrdering/memcpy-offset.ll
index 5e6eab9d80736..87b8217e01db0 100644
--- a/llvm/test/Transforms/PhaseOrdering/memcpy-offset.ll
+++ b/llvm/test/Transforms/PhaseOrdering/memcpy-offset.ll
@@ -10,7 +10,7 @@ define void @memcpy_forward_back_with_offset(ptr %arg) {
 ; CUSTOM-NEXT:    ret void
 ;
 ; O2-LABEL: define void @memcpy_forward_back_with_offset(
-; O2-SAME: ptr nocapture writeonly initializes((0, 1)) [[ARG:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
+; O2-SAME: ptr writeonly captures(none) initializes((0, 1)) [[ARG:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
 ; O2-NEXT:    store i8 1, ptr [[ARG]], align 1
 ; O2-NEXT:    ret void
 ;
diff --git a/llvm/test/Transforms/PhaseOrdering/pr95152.ll b/llvm/test/Transforms/PhaseOrdering/pr95152.ll
index 6941ea2aece92..1e28856f32bd4 100644
--- a/llvm/test/Transforms/PhaseOrdering/pr95152.ll
+++ b/llvm/test/Transforms/PhaseOrdering/pr95152.ll
@@ -33,7 +33,7 @@ define void @h(ptr %p) {
 
 define void @g(ptr dead_on_unwind noalias writable dereferenceable(8) align 8 %p) minsize {
 ; CHECK-LABEL: define void @g(
-; CHECK-SAME: ptr dead_on_unwind noalias nocapture writable writeonly align 8 dereferenceable(8) initializes((0, 8)) [[P:%.*]]) local_unnamed_addr #[[ATTR1:[0-9]+]] {
+; CHECK-SAME: ptr dead_on_unwind noalias writable writeonly align 8 captures(none) dereferenceable(8) initializes((0, 8)) [[P:%.*]]) local_unnamed_addr #[[ATTR1:[0-9]+]] {
 ; CHECK-NEXT:    tail call void @h(ptr nonnull [[P]])
 ; CHECK-NEXT:    ret void
 ;
diff --git a/llvm/test/Transforms/PhaseOrdering/pr98799-inline-simplifycfg-ub.ll b/llvm/test/Transforms/PhaseOrdering/pr98799-inline-simplifycfg-ub.ll
index bad07c19f14b3..2bb78dd5ec773 100644
--- a/llvm/test/Transforms/PhaseOrdering/pr98799-inline-simplifycfg-ub.ll
+++ b/llvm/test/Transforms/PhaseOrdering/pr98799-inline-simplifycfg-ub.ll
@@ -32,7 +32,7 @@ define i32 @foo(ptr %arg, i1 %arg1) {
 ; CUSTOM-NEXT:    ret i32 [[I3]]
 ;
 ; O2-LABEL: define i32 @foo(
-; O2-SAME: ptr nocapture [[ARG:%.*]], i1 [[ARG1:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
+; O2-SAME: ptr captures(none) [[ARG:%.*]], i1 [[ARG1:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
 ; O2-NEXT:  [[BB:.*:]]
 ; O2-NEXT:    [[TMP0:%.*]] = xor i1 [[ARG1]], true
 ; O2-NEXT:    tail call void @llvm.assume(i1 [[TMP0]])
diff --git a/llvm/test/Transforms/PhaseOrdering/switch_with_geps.ll b/llvm/test/Transforms/PhaseOrdering/switch_with_geps.ll
index 33266caf70923..6b11df5513668 100644
--- a/llvm/test/Transforms/PhaseOrdering/switch_with_geps.ll
+++ b/llvm/test/Transforms/PhaseOrdering/switch_with_geps.ll
@@ -12,7 +12,7 @@ target datalayout = "n64"
 
 define i32 @test(ptr %ptr) {
 ; CHECK-LABEL: define i32 @test(
-; CHECK-SAME: ptr nocapture readonly [[PTR:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
+; CHECK-SAME: ptr readonly captures(none) [[PTR:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
 ; CHECK-NEXT:  start:
 ; CHECK-NEXT:    [[PHI:%.*]] = getelementptr inbounds nuw i8, ptr [[PTR]], i64 4
 ; CHECK-NEXT:    [[RET:%.*]] = load i32, ptr [[PHI]], align 4
@@ -56,7 +56,7 @@ exit:
 
 define void @test2(ptr %self, i64 %v, i64 %ix) {
 ; CHECK-LABEL: define void @test2(
-; CHECK-SAME: ptr nocapture writeonly [[SELF:%.*]], i64 [[V:%.*]], i64 [[IX:%.*]]) local_unnamed_addr #[[ATTR1:[0-9]+]] {
+; CHECK-SAME: ptr writeonly captures(none) [[SELF:%.*]], i64 [[V:%.*]], i64 [[IX:%.*]]) local_unnamed_addr #[[ATTR1:[0-9]+]] {
 ; CHECK-NEXT:  start:
 ; CHECK-NEXT:    [[SWITCH_TABLEIDX:%.*]] = shl i64 [[IX]], 3
 ; CHECK-NEXT:    [[GEP5:%.*]] = getelementptr inbounds nuw i8, ptr [[SELF]], i64 [[SWITCH_TABLEIDX]]
diff --git a/llvm/test/Transforms/PhaseOrdering/vector-trunc-inseltpoison.ll b/llvm/test/Transforms/PhaseOrdering/vector-trunc-inseltpoison.ll
index b9cfc53bdac74..2daf15064bf01 100644
--- a/llvm/test/Transforms/PhaseOrdering/vector-trunc-inseltpoison.ll
+++ b/llvm/test/Transforms/PhaseOrdering/vector-trunc-inseltpoison.ll
@@ -1,6 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt -O2                   -S -data-layout="e" < %s | FileCheck %s
-; RUN: opt -passes='default<O2>' -S -data-layout="e" < %s | FileCheck %s
+; RUN: opt -passes="default<O2>" -S -data-layout="e" < %s | FileCheck %s
 
 define <4 x i16> @truncate(<4 x i32> %x) {
 ; CHECK-LABEL: @truncate(
diff --git a/llvm/test/Transforms/PhaseOrdering/vector-trunc.ll b/llvm/test/Transforms/PhaseOrdering/vector-trunc.ll
index 3e3bf56a8c8d6..13e76a15c0c7f 100644
--- a/llvm/test/Transforms/PhaseOrdering/vector-trunc.ll
+++ b/llvm/test/Transforms/PhaseOrdering/vector-trunc.ll
@@ -1,6 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt -O2                   -S -data-layout="e" < %s | FileCheck %s
-; RUN: opt -passes='default<O2>' -S -data-layout="e" < %s | FileCheck %s
+; RUN: opt -passes="default<O2>" -S -data-layout="e" < %s | FileCheck %s
 
 define <4 x i16> @truncate(<4 x i32> %x) {
 ; CHECK-LABEL: @truncate(
diff --git a/llvm/test/Transforms/PreISelIntrinsicLowering/X86/memset-pattern.ll b/llvm/test/Transforms/PreISelIntrinsicLowering/X86/memset-pattern.ll
new file mode 100644
index 0000000000000..7cfdcb8578809
--- /dev/null
+++ b/llvm/test/Transforms/PreISelIntrinsicLowering/X86/memset-pattern.ll
@@ -0,0 +1,156 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals all --version 5
+; RUN: opt -mtriple=x86_64-apple-darwin10.0.0 -passes=pre-isel-intrinsic-lowering -S -o - %s | FileCheck %s
+
+;.
+; CHECK: @.memset_pattern = private unnamed_addr constant [2 x i64] [i64 -6148895925951734307, i64 -6148895925951734307], align 16
+; CHECK: @.memset_pattern.1 = private unnamed_addr constant [2 x i64] [i64 4614256656552045848, i64 4614256656552045848], align 16
+; CHECK: @.memset_pattern.2 = private unnamed_addr constant [8 x i16] [i16 -21555, i16 -21555, i16 -21555, i16 -21555, i16 -21555, i16 -21555, i16 -21555, i16 -21555], align 16
+; CHECK: @.memset_pattern.3 = private unnamed_addr constant i128 -113427455635030943652277463699152839203, align 16
+; CHECK: @.memset_pattern.4 = private unnamed_addr constant i128 -113427455635030943652277463699152839203, align 16
+; CHECK: @.memset_pattern.5 = private unnamed_addr constant i128 -113427455635030943652277463699152839203, align 16
+; CHECK: @.memset_pattern.6 = private unnamed_addr constant i128 -113427455635030943652277463699152839203, align 16
+;.
+define void @memset_pattern_i128_1_dynvalue(ptr %a, i128 %value) nounwind {
+; CHECK-LABEL: define void @memset_pattern_i128_1_dynvalue(
+; CHECK-SAME: ptr [[A:%.*]], i128 [[VALUE:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT:    br i1 false, label %[[SPLIT:.*]], label %[[LOADSTORELOOP:.*]]
+; CHECK:       [[LOADSTORELOOP]]:
+; CHECK-NEXT:    [[TMP1:%.*]] = phi i64 [ 0, [[TMP0:%.*]] ], [ [[TMP3:%.*]], %[[LOADSTORELOOP]] ]
+; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i128, ptr [[A]], i64 [[TMP1]]
+; CHECK-NEXT:    store i128 [[VALUE]], ptr [[TMP2]], align 1
+; CHECK-NEXT:    [[TMP3]] = add i64 [[TMP1]], 1
+; CHECK-NEXT:    [[TMP4:%.*]] = icmp ult i64 [[TMP3]], 1
+; CHECK-NEXT:    br i1 [[TMP4]], label %[[LOADSTORELOOP]], label %[[SPLIT]]
+; CHECK:       [[SPLIT]]:
+; CHECK-NEXT:    ret void
+;
+  tail call void @llvm.experimental.memset.pattern(ptr %a, i128 %value, i64 1, i1 false)
+  ret void
+}
+
+define void @memset_pattern_i128_1(ptr %a, i128 %value) nounwind {
+; CHECK-LABEL: define void @memset_pattern_i128_1(
+; CHECK-SAME: ptr [[A:%.*]], i128 [[VALUE:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    call void @memset_pattern16(ptr [[A]], ptr @.memset_pattern.3, i64 16)
+; CHECK-NEXT:    ret void
+;
+  tail call void @llvm.experimental.memset.pattern(ptr %a, i128 u0xaaaaaaaabbbbbbbbccccccccdddddddd, i64 1, i1 false)
+  ret void
+}
+
+define void @memset_pattern_i128_1_nz_as(ptr addrspace(1) %a, i128 %value) nounwind {
+; CHECK-LABEL: define void @memset_pattern_i128_1_nz_as(
+; CHECK-SAME: ptr addrspace(1) [[A:%.*]], i128 [[VALUE:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    br i1 false, label %[[SPLIT:.*]], label %[[LOADSTORELOOP:.*]]
+; CHECK:       [[LOADSTORELOOP]]:
+; CHECK-NEXT:    [[TMP1:%.*]] = phi i64 [ 0, [[TMP0:%.*]] ], [ [[TMP3:%.*]], %[[LOADSTORELOOP]] ]
+; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i128, ptr addrspace(1) [[A]], i64 [[TMP1]]
+; CHECK-NEXT:    store i128 -113427455635030943652277463699152839203, ptr addrspace(1) [[TMP2]], align 1
+; CHECK-NEXT:    [[TMP3]] = add i64 [[TMP1]], 1
+; CHECK-NEXT:    [[TMP4:%.*]] = icmp ult i64 [[TMP3]], 1
+; CHECK-NEXT:    br i1 [[TMP4]], label %[[LOADSTORELOOP]], label %[[SPLIT]]
+; CHECK:       [[SPLIT]]:
+; CHECK-NEXT:    ret void
+;
+  tail call void @llvm.experimental.memset.pattern(ptr addrspace(1) %a, i128 u0xaaaaaaaabbbbbbbbccccccccdddddddd, i64 1, i1 false)
+  ret void
+}
+
+define void @memset_pattern_i128_1_align_attr(ptr align(16) %a, i128 %value) nounwind {
+; CHECK-LABEL: define void @memset_pattern_i128_1_align_attr(
+; CHECK-SAME: ptr align 16 [[A:%.*]], i128 [[VALUE:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    call void @memset_pattern16(ptr align 16 [[A]], ptr @.memset_pattern.4, i64 16)
+; CHECK-NEXT:    ret void
+;
+  tail call void @llvm.experimental.memset.pattern(ptr align(16) %a, i128 u0xaaaaaaaabbbbbbbbccccccccdddddddd, i64 1, i1 false)
+  ret void
+}
+
+define void @memset_pattern_i128_16(ptr %a) nounwind {
+; CHECK-LABEL: define void @memset_pattern_i128_16(
+; CHECK-SAME: ptr [[A:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    call void @memset_pattern16(ptr [[A]], ptr @.memset_pattern.5, i64 256)
+; CHECK-NEXT:    ret void
+;
+  tail call void @llvm.experimental.memset.pattern(ptr %a, i128 u0xaaaaaaaabbbbbbbbccccccccdddddddd, i64 16, i1 false)
+  ret void
+}
+
+define void @memset_pattern_i128_x(ptr %a, i64 %x) nounwind {
+; CHECK-LABEL: define void @memset_pattern_i128_x(
+; CHECK-SAME: ptr [[A:%.*]], i64 [[X:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = mul i64 16, [[X]]
+; CHECK-NEXT:    call void @memset_pattern16(ptr [[A]], ptr @.memset_pattern.6, i64 [[TMP1]])
+; CHECK-NEXT:    ret void
+;
+  tail call void @llvm.experimental.memset.pattern(ptr %a, i128 u0xaaaaaaaabbbbbbbbccccccccdddddddd, i64 %x, i1 false)
+  ret void
+}
+
+define void @memset_pattern_i128_x_nonzero_as(ptr addrspace(10) %a, i64 %x) nounwind {
+; CHECK-LABEL: define void @memset_pattern_i128_x_nonzero_as(
+; CHECK-SAME: ptr addrspace(10) [[A:%.*]], i64 [[X:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp eq i64 0, [[X]]
+; CHECK-NEXT:    br i1 [[TMP1]], label %[[SPLIT:.*]], label %[[LOADSTORELOOP:.*]]
+; CHECK:       [[LOADSTORELOOP]]:
+; CHECK-NEXT:    [[TMP2:%.*]] = phi i64 [ 0, [[TMP0:%.*]] ], [ [[TMP4:%.*]], %[[LOADSTORELOOP]] ]
+; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i128, ptr addrspace(10) [[A]], i64 [[TMP2]]
+; CHECK-NEXT:    store i128 -113427455635030943652277463699152839203, ptr addrspace(10) [[TMP3]], align 1
+; CHECK-NEXT:    [[TMP4]] = add i64 [[TMP2]], 1
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp ult i64 [[TMP4]], [[X]]
+; CHECK-NEXT:    br i1 [[TMP5]], label %[[LOADSTORELOOP]], label %[[SPLIT]]
+; CHECK:       [[SPLIT]]:
+; CHECK-NEXT:    ret void
+;
+  tail call void @llvm.experimental.memset.pattern(ptr addrspace(10) %a, i128 u0xaaaaaaaabbbbbbbbccccccccdddddddd, i64 %x, i1 false)
+  ret void
+}
+
+define void @memset_pattern_i16_x(ptr %a, i64 %x) nounwind {
+; CHECK-LABEL: define void @memset_pattern_i16_x(
+; CHECK-SAME: ptr [[A:%.*]], i64 [[X:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = mul i64 2, [[X]]
+; CHECK-NEXT:    call void @memset_pattern16(ptr [[A]], ptr @.memset_pattern.2, i64 [[TMP1]])
+; CHECK-NEXT:    ret void
+;
+  tail call void @llvm.experimental.memset.pattern(ptr %a, i16 u0xabcd, i64 %x, i1 false)
+  ret void
+}
+
+define void @memset_pattern_i64_x(ptr %a, i64 %x) nounwind {
+; CHECK-LABEL: define void @memset_pattern_i64_x(
+; CHECK-SAME: ptr [[A:%.*]], i64 [[X:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = mul i64 8, [[X]]
+; CHECK-NEXT:    call void @memset_pattern16(ptr [[A]], ptr @.memset_pattern, i64 [[TMP1]])
+; CHECK-NEXT:    ret void
+;
+  tail call void @llvm.experimental.memset.pattern(ptr %a, i64 u0xaaaabbbbccccdddd, i64 %x, i1 false)
+  ret void
+}
+
+; Demonstrate that TBAA metadata is preserved.
+define void @memset_pattern_i64_128_tbaa(ptr %a) nounwind {
+; CHECK-LABEL: define void @memset_pattern_i64_128_tbaa(
+; CHECK-SAME: ptr [[A:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    call void @memset_pattern16(ptr [[A]], ptr @.memset_pattern.1, i64 1024), !tbaa [[TBAA0:![0-9]+]]
+; CHECK-NEXT:    ret void
+;
+  tail call void @llvm.experimental.memset.pattern(ptr %a, i64 u0x400921fb54442d18, i64 128, i1 false), !tbaa !5
+  ret void
+}
+
+!5 = !{!6, !6, i64 0}
+!6 = !{!"double", !7, i64 0}
+!7 = !{!"omnipotent char", !8, i64 0}
+!8 = !{!"Simple C++ TBAA"}
+
+;.
+; CHECK: attributes #[[ATTR0]] = { nounwind }
+; CHECK: attributes #[[ATTR1:[0-9]+]] = { nocallback nofree nounwind willreturn memory(argmem: write) }
+; CHECK: attributes #[[ATTR2:[0-9]+]] = { nofree nounwind willreturn memory(argmem: readwrite) }
+;.
+; CHECK: [[TBAA0]] = !{[[META1:![0-9]+]], [[META1]], i64 0}
+; CHECK: [[META1]] = !{!"double", [[META2:![0-9]+]], i64 0}
+; CHECK: [[META2]] = !{!"omnipotent char", [[META3:![0-9]+]], i64 0}
+; CHECK: [[META3]] = !{!"Simple C++ TBAA"}
+;.
diff --git a/llvm/test/Transforms/RewriteStatepointsForGC/call-argument-attributes.ll b/llvm/test/Transforms/RewriteStatepointsForGC/call-argument-attributes.ll
index 4a7088f95329f..6a35bb3812baa 100644
--- a/llvm/test/Transforms/RewriteStatepointsForGC/call-argument-attributes.ll
+++ b/llvm/test/Transforms/RewriteStatepointsForGC/call-argument-attributes.ll
@@ -6,7 +6,7 @@ declare i8 @callee(ptr, i8, float, ptr)
 define i8 @test(ptr %arg) gc "statepoint-example" {
 ; CHECK-LABEL: define i8 @test(
 ; CHECK-SAME: ptr [[ARG:%.*]]) gc "statepoint-example" {
-; CHECK-NEXT:    [[STATEPOINT_TOKEN:%.*]] = call token (i64, i32, ptr, i32, i32, ...) @llvm.experimental.gc.statepoint.p0(i64 2882400000, i32 0, ptr elementtype(i8 (ptr, i8, float, ptr)) @callee, i32 4, i32 0, ptr nocapture sret({ i64, i64 }) align 8 null, i8 signext 8, float inreg 1.000000e+00, ptr [[ARG]], i32 0, i32 0)
+; CHECK-NEXT:    [[STATEPOINT_TOKEN:%.*]] = call token (i64, i32, ptr, i32, i32, ...) @llvm.experimental.gc.statepoint.p0(i64 2882400000, i32 0, ptr elementtype(i8 (ptr, i8, float, ptr)) @callee, i32 4, i32 0, ptr sret({ i64, i64 }) align 8 captures(none) null, i8 signext 8, float inreg 1.000000e+00, ptr [[ARG]], i32 0, i32 0)
 ; CHECK-NEXT:    [[R1:%.*]] = call zeroext i8 @llvm.experimental.gc.result.i8(token [[STATEPOINT_TOKEN]])
 ; CHECK-NEXT:    ret i8 [[R1]]
 ;
@@ -19,14 +19,14 @@ declare i32 @personality_function()
 define i8 @test_invoke(ptr %arg) gc "statepoint-example" personality ptr @personality_function {
 ; CHECK-LABEL: define i8 @test_invoke(
 ; CHECK-SAME: ptr [[ARG:%.*]]) gc "statepoint-example" personality ptr @personality_function {
-; CHECK-NEXT:    [[STATEPOINT_TOKEN:%.*]] = invoke token (i64, i32, ptr, i32, i32, ...) @llvm.experimental.gc.statepoint.p0(i64 2882400000, i32 0, ptr elementtype(i8 (ptr, i8, float, ptr)) @callee, i32 4, i32 0, ptr nocapture sret({ i64, i64 }) align 8 null, i8 signext 8, float inreg 1.000000e+00, ptr [[ARG]], i32 0, i32 0)
-; CHECK-NEXT:    to label [[NORMAL_RETURN:%.*]] unwind label [[EXCEPTIONAL_RETURN:%.*]]
+; CHECK-NEXT:    [[STATEPOINT_TOKEN:%.*]] = invoke token (i64, i32, ptr, i32, i32, ...) @llvm.experimental.gc.statepoint.p0(i64 2882400000, i32 0, ptr elementtype(i8 (ptr, i8, float, ptr)) @callee, i32 4, i32 0, ptr sret({ i64, i64 }) align 8 captures(none) null, i8 signext 8, float inreg 1.000000e+00, ptr [[ARG]], i32 0, i32 0)
+; CHECK-NEXT:            to label [[NORMAL_RETURN:%.*]] unwind label [[EXCEPTIONAL_RETURN:%.*]]
 ; CHECK:       normal_return:
 ; CHECK-NEXT:    [[R1:%.*]] = call zeroext i8 @llvm.experimental.gc.result.i8(token [[STATEPOINT_TOKEN]])
 ; CHECK-NEXT:    ret i8 [[R1]]
 ; CHECK:       exceptional_return:
 ; CHECK-NEXT:    [[LANDING_PAD4:%.*]] = landingpad token
-; CHECK-NEXT:    cleanup
+; CHECK-NEXT:            cleanup
 ; CHECK-NEXT:    ret i8 0
 ;
   %r = invoke zeroext i8 @callee(ptr sret({i64, i64}) noalias align 8 nocapture null, i8 signext 8, float inreg 1.0, ptr writeonly %arg)
diff --git a/llvm/test/Transforms/SCCP/pr45185-range-predinfo.ll b/llvm/test/Transforms/SCCP/pr45185-range-predinfo.ll
index 111a1c35f414b..ef0f43f0848d8 100644
--- a/llvm/test/Transforms/SCCP/pr45185-range-predinfo.ll
+++ b/llvm/test/Transforms/SCCP/pr45185-range-predinfo.ll
@@ -26,7 +26,7 @@ bb:
 ; Make sure we do not incorrectly eliminate the checks in @blam.
 define internal void @blam(i32 %arg, ptr nocapture %arg1) {
 ; CHECK-LABEL: define internal void @blam(
-; CHECK-SAME: i32 range(i32 0, 3) [[ARG:%.*]], ptr nocapture [[ARG1:%.*]]) {
+; CHECK-SAME: i32 range(i32 0, 3) [[ARG:%.*]], ptr captures(none) [[ARG1:%.*]]) {
 ; CHECK-NEXT:  [[BB:.*:]]
 ; CHECK-NEXT:    [[TMP:%.*]] = icmp eq i32 [[ARG]], 0
 ; CHECK-NEXT:    br i1 [[TMP]], label %[[BB2:.*]], label %[[BB3:.*]]
diff --git a/llvm/test/Transforms/SLPVectorizer/AArch64/accelerate-vector-functions-inseltpoison.ll b/llvm/test/Transforms/SLPVectorizer/AArch64/accelerate-vector-functions-inseltpoison.ll
index e1fc7e056e097..0d80affcbb4f4 100644
--- a/llvm/test/Transforms/SLPVectorizer/AArch64/accelerate-vector-functions-inseltpoison.ll
+++ b/llvm/test/Transforms/SLPVectorizer/AArch64/accelerate-vector-functions-inseltpoison.ll
@@ -24,10 +24,12 @@ define <4 x float> @int_sin_4x(ptr %a) {
 ; NOACCELERATE-NEXT:    [[VECEXT_1:%.*]] = extractelement <4 x float> [[TMP0]], i32 1
 ; NOACCELERATE-NEXT:    [[TMP2:%.*]] = tail call fast float @llvm.sin.f32(float [[VECEXT_1]])
 ; NOACCELERATE-NEXT:    [[VECINS_1:%.*]] = insertelement <4 x float> [[VECINS]], float [[TMP2]], i32 1
-; NOACCELERATE-NEXT:    [[TMP3:%.*]] = shufflevector <4 x float> [[TMP0]], <4 x float> poison, <2 x i32> <i32 2, i32 3>
-; NOACCELERATE-NEXT:    [[TMP4:%.*]] = call fast <2 x float> @llvm.sin.v2f32(<2 x float> [[TMP3]])
-; NOACCELERATE-NEXT:    [[TMP5:%.*]] = shufflevector <2 x float> [[TMP4]], <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
-; NOACCELERATE-NEXT:    [[VECINS_31:%.*]] = shufflevector <4 x float> [[VECINS_1]], <4 x float> [[TMP5]], <4 x i32> <i32 0, i32 1, i32 4, i32 5>
+; NOACCELERATE-NEXT:    [[VECEXT_2:%.*]] = extractelement <4 x float> [[TMP0]], i32 2
+; NOACCELERATE-NEXT:    [[TMP3:%.*]] = tail call fast float @llvm.sin.f32(float [[VECEXT_2]])
+; NOACCELERATE-NEXT:    [[VECINS_2:%.*]] = insertelement <4 x float> [[VECINS_1]], float [[TMP3]], i32 2
+; NOACCELERATE-NEXT:    [[VECEXT_3:%.*]] = extractelement <4 x float> [[TMP0]], i32 3
+; NOACCELERATE-NEXT:    [[TMP4:%.*]] = tail call fast float @llvm.sin.f32(float [[VECEXT_3]])
+; NOACCELERATE-NEXT:    [[VECINS_31:%.*]] = insertelement <4 x float> [[VECINS_2]], float [[TMP4]], i32 3
 ; NOACCELERATE-NEXT:    ret <4 x float> [[VECINS_31]]
 ;
 entry:
@@ -217,10 +219,12 @@ define <4 x float> @exp_4x(ptr %a) {
 ; NOACCELERATE-NEXT:    [[VECEXT_1:%.*]] = extractelement <4 x float> [[TMP0]], i32 1
 ; NOACCELERATE-NEXT:    [[TMP2:%.*]] = tail call fast float @expf(float [[VECEXT_1]])
 ; NOACCELERATE-NEXT:    [[VECINS_1:%.*]] = insertelement <4 x float> [[VECINS]], float [[TMP2]], i32 1
-; NOACCELERATE-NEXT:    [[TMP3:%.*]] = shufflevector <4 x float> [[TMP0]], <4 x float> poison, <2 x i32> <i32 2, i32 3>
-; NOACCELERATE-NEXT:    [[TMP4:%.*]] = call fast <2 x float> @llvm.exp.v2f32(<2 x float> [[TMP3]])
-; NOACCELERATE-NEXT:    [[TMP5:%.*]] = shufflevector <2 x float> [[TMP4]], <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
-; NOACCELERATE-NEXT:    [[VECINS_31:%.*]] = shufflevector <4 x float> [[VECINS_1]], <4 x float> [[TMP5]], <4 x i32> <i32 0, i32 1, i32 4, i32 5>
+; NOACCELERATE-NEXT:    [[VECEXT_2:%.*]] = extractelement <4 x float> [[TMP0]], i32 2
+; NOACCELERATE-NEXT:    [[TMP3:%.*]] = tail call fast float @expf(float [[VECEXT_2]])
+; NOACCELERATE-NEXT:    [[VECINS_2:%.*]] = insertelement <4 x float> [[VECINS_1]], float [[TMP3]], i32 2
+; NOACCELERATE-NEXT:    [[VECEXT_3:%.*]] = extractelement <4 x float> [[TMP0]], i32 3
+; NOACCELERATE-NEXT:    [[TMP4:%.*]] = tail call fast float @expf(float [[VECEXT_3]])
+; NOACCELERATE-NEXT:    [[VECINS_31:%.*]] = insertelement <4 x float> [[VECINS_2]], float [[TMP4]], i32 3
 ; NOACCELERATE-NEXT:    ret <4 x float> [[VECINS_31]]
 ;
 entry:
@@ -297,10 +301,12 @@ define <4 x float> @log_4x(ptr %a) {
 ; NOACCELERATE-NEXT:    [[VECEXT_1:%.*]] = extractelement <4 x float> [[TMP0]], i32 1
 ; NOACCELERATE-NEXT:    [[TMP2:%.*]] = tail call fast float @logf(float [[VECEXT_1]])
 ; NOACCELERATE-NEXT:    [[VECINS_1:%.*]] = insertelement <4 x float> [[VECINS]], float [[TMP2]], i32 1
-; NOACCELERATE-NEXT:    [[TMP3:%.*]] = shufflevector <4 x float> [[TMP0]], <4 x float> poison, <2 x i32> <i32 2, i32 3>
-; NOACCELERATE-NEXT:    [[TMP4:%.*]] = call fast <2 x float> @llvm.log.v2f32(<2 x float> [[TMP3]])
-; NOACCELERATE-NEXT:    [[TMP5:%.*]] = shufflevector <2 x float> [[TMP4]], <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
-; NOACCELERATE-NEXT:    [[VECINS_31:%.*]] = shufflevector <4 x float> [[VECINS_1]], <4 x float> [[TMP5]], <4 x i32> <i32 0, i32 1, i32 4, i32 5>
+; NOACCELERATE-NEXT:    [[VECEXT_2:%.*]] = extractelement <4 x float> [[TMP0]], i32 2
+; NOACCELERATE-NEXT:    [[TMP3:%.*]] = tail call fast float @logf(float [[VECEXT_2]])
+; NOACCELERATE-NEXT:    [[VECINS_2:%.*]] = insertelement <4 x float> [[VECINS_1]], float [[TMP3]], i32 2
+; NOACCELERATE-NEXT:    [[VECEXT_3:%.*]] = extractelement <4 x float> [[TMP0]], i32 3
+; NOACCELERATE-NEXT:    [[TMP4:%.*]] = tail call fast float @logf(float [[VECEXT_3]])
+; NOACCELERATE-NEXT:    [[VECINS_31:%.*]] = insertelement <4 x float> [[VECINS_2]], float [[TMP4]], i32 3
 ; NOACCELERATE-NEXT:    ret <4 x float> [[VECINS_31]]
 ;
 entry:
@@ -470,10 +476,12 @@ define <4 x float> @sin_4x(ptr %a) {
 ; NOACCELERATE-NEXT:    [[VECEXT_1:%.*]] = extractelement <4 x float> [[TMP0]], i32 1
 ; NOACCELERATE-NEXT:    [[TMP2:%.*]] = tail call fast float @sinf(float [[VECEXT_1]])
 ; NOACCELERATE-NEXT:    [[VECINS_1:%.*]] = insertelement <4 x float> [[VECINS]], float [[TMP2]], i32 1
-; NOACCELERATE-NEXT:    [[TMP3:%.*]] = shufflevector <4 x float> [[TMP0]], <4 x float> poison, <2 x i32> <i32 2, i32 3>
-; NOACCELERATE-NEXT:    [[TMP4:%.*]] = call fast <2 x float> @llvm.sin.v2f32(<2 x float> [[TMP3]])
-; NOACCELERATE-NEXT:    [[TMP5:%.*]] = shufflevector <2 x float> [[TMP4]], <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
-; NOACCELERATE-NEXT:    [[VECINS_31:%.*]] = shufflevector <4 x float> [[VECINS_1]], <4 x float> [[TMP5]], <4 x i32> <i32 0, i32 1, i32 4, i32 5>
+; NOACCELERATE-NEXT:    [[VECEXT_2:%.*]] = extractelement <4 x float> [[TMP0]], i32 2
+; NOACCELERATE-NEXT:    [[TMP3:%.*]] = tail call fast float @sinf(float [[VECEXT_2]])
+; NOACCELERATE-NEXT:    [[VECINS_2:%.*]] = insertelement <4 x float> [[VECINS_1]], float [[TMP3]], i32 2
+; NOACCELERATE-NEXT:    [[VECEXT_3:%.*]] = extractelement <4 x float> [[TMP0]], i32 3
+; NOACCELERATE-NEXT:    [[TMP4:%.*]] = tail call fast float @sinf(float [[VECEXT_3]])
+; NOACCELERATE-NEXT:    [[VECINS_31:%.*]] = insertelement <4 x float> [[VECINS_2]], float [[TMP4]], i32 3
 ; NOACCELERATE-NEXT:    ret <4 x float> [[VECINS_31]]
 ;
 entry:
@@ -509,10 +517,12 @@ define <4 x float> @cos_4x(ptr %a) {
 ; NOACCELERATE-NEXT:    [[VECEXT_1:%.*]] = extractelement <4 x float> [[TMP0]], i32 1
 ; NOACCELERATE-NEXT:    [[TMP2:%.*]] = tail call fast float @cosf(float [[VECEXT_1]])
 ; NOACCELERATE-NEXT:    [[VECINS_1:%.*]] = insertelement <4 x float> [[VECINS]], float [[TMP2]], i32 1
-; NOACCELERATE-NEXT:    [[TMP3:%.*]] = shufflevector <4 x float> [[TMP0]], <4 x float> poison, <2 x i32> <i32 2, i32 3>
-; NOACCELERATE-NEXT:    [[TMP4:%.*]] = call fast <2 x float> @llvm.cos.v2f32(<2 x float> [[TMP3]])
-; NOACCELERATE-NEXT:    [[TMP5:%.*]] = shufflevector <2 x float> [[TMP4]], <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
-; NOACCELERATE-NEXT:    [[VECINS_31:%.*]] = shufflevector <4 x float> [[VECINS_1]], <4 x float> [[TMP5]], <4 x i32> <i32 0, i32 1, i32 4, i32 5>
+; NOACCELERATE-NEXT:    [[VECEXT_2:%.*]] = extractelement <4 x float> [[TMP0]], i32 2
+; NOACCELERATE-NEXT:    [[TMP3:%.*]] = tail call fast float @cosf(float [[VECEXT_2]])
+; NOACCELERATE-NEXT:    [[VECINS_2:%.*]] = insertelement <4 x float> [[VECINS_1]], float [[TMP3]], i32 2
+; NOACCELERATE-NEXT:    [[VECEXT_3:%.*]] = extractelement <4 x float> [[TMP0]], i32 3
+; NOACCELERATE-NEXT:    [[TMP4:%.*]] = tail call fast float @cosf(float [[VECEXT_3]])
+; NOACCELERATE-NEXT:    [[VECINS_31:%.*]] = insertelement <4 x float> [[VECINS_2]], float [[TMP4]], i32 3
 ; NOACCELERATE-NEXT:    ret <4 x float> [[VECINS_31]]
 ;
 entry:
@@ -548,10 +558,12 @@ define <4 x float> @tan_4x(ptr %a) {
 ; NOACCELERATE-NEXT:    [[VECEXT_1:%.*]] = extractelement <4 x float> [[TMP0]], i32 1
 ; NOACCELERATE-NEXT:    [[TMP2:%.*]] = tail call fast float @tanf(float [[VECEXT_1]])
 ; NOACCELERATE-NEXT:    [[VECINS_1:%.*]] = insertelement <4 x float> [[VECINS]], float [[TMP2]], i32 1
-; NOACCELERATE-NEXT:    [[TMP3:%.*]] = shufflevector <4 x float> [[TMP0]], <4 x float> poison, <2 x i32> <i32 2, i32 3>
-; NOACCELERATE-NEXT:    [[TMP4:%.*]] = call fast <2 x float> @llvm.tan.v2f32(<2 x float> [[TMP3]])
-; NOACCELERATE-NEXT:    [[TMP5:%.*]] = shufflevector <2 x float> [[TMP4]], <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
-; NOACCELERATE-NEXT:    [[VECINS_31:%.*]] = shufflevector <4 x float> [[VECINS_1]], <4 x float> [[TMP5]], <4 x i32> <i32 0, i32 1, i32 4, i32 5>
+; NOACCELERATE-NEXT:    [[VECEXT_2:%.*]] = extractelement <4 x float> [[TMP0]], i32 2
+; NOACCELERATE-NEXT:    [[TMP3:%.*]] = tail call fast float @tanf(float [[VECEXT_2]])
+; NOACCELERATE-NEXT:    [[VECINS_2:%.*]] = insertelement <4 x float> [[VECINS_1]], float [[TMP3]], i32 2
+; NOACCELERATE-NEXT:    [[VECEXT_3:%.*]] = extractelement <4 x float> [[TMP0]], i32 3
+; NOACCELERATE-NEXT:    [[TMP4:%.*]] = tail call fast float @tanf(float [[VECEXT_3]])
+; NOACCELERATE-NEXT:    [[VECINS_31:%.*]] = insertelement <4 x float> [[VECINS_2]], float [[TMP4]], i32 3
 ; NOACCELERATE-NEXT:    ret <4 x float> [[VECINS_31]]
 ;
 entry:
@@ -587,10 +599,12 @@ define <4 x float> @asin_4x(ptr %a) {
 ; NOACCELERATE-NEXT:    [[VECEXT_1:%.*]] = extractelement <4 x float> [[TMP0]], i32 1
 ; NOACCELERATE-NEXT:    [[TMP2:%.*]] = tail call fast float @asinf(float [[VECEXT_1]])
 ; NOACCELERATE-NEXT:    [[VECINS_1:%.*]] = insertelement <4 x float> [[VECINS]], float [[TMP2]], i32 1
-; NOACCELERATE-NEXT:    [[TMP3:%.*]] = shufflevector <4 x float> [[TMP0]], <4 x float> poison, <2 x i32> <i32 2, i32 3>
-; NOACCELERATE-NEXT:    [[TMP4:%.*]] = call fast <2 x float> @llvm.asin.v2f32(<2 x float> [[TMP3]])
-; NOACCELERATE-NEXT:    [[TMP5:%.*]] = shufflevector <2 x float> [[TMP4]], <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
-; NOACCELERATE-NEXT:    [[VECINS_31:%.*]] = shufflevector <4 x float> [[VECINS_1]], <4 x float> [[TMP5]], <4 x i32> <i32 0, i32 1, i32 4, i32 5>
+; NOACCELERATE-NEXT:    [[VECEXT_2:%.*]] = extractelement <4 x float> [[TMP0]], i32 2
+; NOACCELERATE-NEXT:    [[TMP3:%.*]] = tail call fast float @asinf(float [[VECEXT_2]])
+; NOACCELERATE-NEXT:    [[VECINS_2:%.*]] = insertelement <4 x float> [[VECINS_1]], float [[TMP3]], i32 2
+; NOACCELERATE-NEXT:    [[VECEXT_3:%.*]] = extractelement <4 x float> [[TMP0]], i32 3
+; NOACCELERATE-NEXT:    [[TMP4:%.*]] = tail call fast float @asinf(float [[VECEXT_3]])
+; NOACCELERATE-NEXT:    [[VECINS_31:%.*]] = insertelement <4 x float> [[VECINS_2]], float [[TMP4]], i32 3
 ; NOACCELERATE-NEXT:    ret <4 x float> [[VECINS_31]]
 ;
 entry:
@@ -625,10 +639,12 @@ define <4 x float> @int_asin_4x(ptr %a) {
 ; NOACCELERATE-NEXT:    [[VECEXT_1:%.*]] = extractelement <4 x float> [[TMP0]], i32 1
 ; NOACCELERATE-NEXT:    [[TMP2:%.*]] = tail call fast float @llvm.asin.f32(float [[VECEXT_1]])
 ; NOACCELERATE-NEXT:    [[VECINS_1:%.*]] = insertelement <4 x float> [[VECINS]], float [[TMP2]], i32 1
-; NOACCELERATE-NEXT:    [[TMP3:%.*]] = shufflevector <4 x float> [[TMP0]], <4 x float> poison, <2 x i32> <i32 2, i32 3>
-; NOACCELERATE-NEXT:    [[TMP4:%.*]] = call fast <2 x float> @llvm.asin.v2f32(<2 x float> [[TMP3]])
-; NOACCELERATE-NEXT:    [[TMP5:%.*]] = shufflevector <2 x float> [[TMP4]], <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
-; NOACCELERATE-NEXT:    [[VECINS_31:%.*]] = shufflevector <4 x float> [[VECINS_1]], <4 x float> [[TMP5]], <4 x i32> <i32 0, i32 1, i32 4, i32 5>
+; NOACCELERATE-NEXT:    [[VECEXT_2:%.*]] = extractelement <4 x float> [[TMP0]], i32 2
+; NOACCELERATE-NEXT:    [[TMP3:%.*]] = tail call fast float @llvm.asin.f32(float [[VECEXT_2]])
+; NOACCELERATE-NEXT:    [[VECINS_2:%.*]] = insertelement <4 x float> [[VECINS_1]], float [[TMP3]], i32 2
+; NOACCELERATE-NEXT:    [[VECEXT_3:%.*]] = extractelement <4 x float> [[TMP0]], i32 3
+; NOACCELERATE-NEXT:    [[TMP4:%.*]] = tail call fast float @llvm.asin.f32(float [[VECEXT_3]])
+; NOACCELERATE-NEXT:    [[VECINS_31:%.*]] = insertelement <4 x float> [[VECINS_2]], float [[TMP4]], i32 3
 ; NOACCELERATE-NEXT:    ret <4 x float> [[VECINS_31]]
 ;
 entry:
@@ -664,10 +680,12 @@ define <4 x float> @acos_4x(ptr %a) {
 ; NOACCELERATE-NEXT:    [[VECEXT_1:%.*]] = extractelement <4 x float> [[TMP0]], i32 1
 ; NOACCELERATE-NEXT:    [[TMP2:%.*]] = tail call fast float @acosf(float [[VECEXT_1]])
 ; NOACCELERATE-NEXT:    [[VECINS_1:%.*]] = insertelement <4 x float> [[VECINS]], float [[TMP2]], i32 1
-; NOACCELERATE-NEXT:    [[TMP3:%.*]] = shufflevector <4 x float> [[TMP0]], <4 x float> poison, <2 x i32> <i32 2, i32 3>
-; NOACCELERATE-NEXT:    [[TMP4:%.*]] = call fast <2 x float> @llvm.acos.v2f32(<2 x float> [[TMP3]])
-; NOACCELERATE-NEXT:    [[TMP5:%.*]] = shufflevector <2 x float> [[TMP4]], <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
-; NOACCELERATE-NEXT:    [[VECINS_31:%.*]] = shufflevector <4 x float> [[VECINS_1]], <4 x float> [[TMP5]], <4 x i32> <i32 0, i32 1, i32 4, i32 5>
+; NOACCELERATE-NEXT:    [[VECEXT_2:%.*]] = extractelement <4 x float> [[TMP0]], i32 2
+; NOACCELERATE-NEXT:    [[TMP3:%.*]] = tail call fast float @acosf(float [[VECEXT_2]])
+; NOACCELERATE-NEXT:    [[VECINS_2:%.*]] = insertelement <4 x float> [[VECINS_1]], float [[TMP3]], i32 2
+; NOACCELERATE-NEXT:    [[VECEXT_3:%.*]] = extractelement <4 x float> [[TMP0]], i32 3
+; NOACCELERATE-NEXT:    [[TMP4:%.*]] = tail call fast float @acosf(float [[VECEXT_3]])
+; NOACCELERATE-NEXT:    [[VECINS_31:%.*]] = insertelement <4 x float> [[VECINS_2]], float [[TMP4]], i32 3
 ; NOACCELERATE-NEXT:    ret <4 x float> [[VECINS_31]]
 ;
 entry:
@@ -702,10 +720,12 @@ define <4 x float> @int_acos_4x(ptr %a) {
 ; NOACCELERATE-NEXT:    [[VECEXT_1:%.*]] = extractelement <4 x float> [[TMP0]], i32 1
 ; NOACCELERATE-NEXT:    [[TMP2:%.*]] = tail call fast float @llvm.acos.f32(float [[VECEXT_1]])
 ; NOACCELERATE-NEXT:    [[VECINS_1:%.*]] = insertelement <4 x float> [[VECINS]], float [[TMP2]], i32 1
-; NOACCELERATE-NEXT:    [[TMP3:%.*]] = shufflevector <4 x float> [[TMP0]], <4 x float> poison, <2 x i32> <i32 2, i32 3>
-; NOACCELERATE-NEXT:    [[TMP4:%.*]] = call fast <2 x float> @llvm.acos.v2f32(<2 x float> [[TMP3]])
-; NOACCELERATE-NEXT:    [[TMP5:%.*]] = shufflevector <2 x float> [[TMP4]], <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
-; NOACCELERATE-NEXT:    [[VECINS_31:%.*]] = shufflevector <4 x float> [[VECINS_1]], <4 x float> [[TMP5]], <4 x i32> <i32 0, i32 1, i32 4, i32 5>
+; NOACCELERATE-NEXT:    [[VECEXT_2:%.*]] = extractelement <4 x float> [[TMP0]], i32 2
+; NOACCELERATE-NEXT:    [[TMP3:%.*]] = tail call fast float @llvm.acos.f32(float [[VECEXT_2]])
+; NOACCELERATE-NEXT:    [[VECINS_2:%.*]] = insertelement <4 x float> [[VECINS_1]], float [[TMP3]], i32 2
+; NOACCELERATE-NEXT:    [[VECEXT_3:%.*]] = extractelement <4 x float> [[TMP0]], i32 3
+; NOACCELERATE-NEXT:    [[TMP4:%.*]] = tail call fast float @llvm.acos.f32(float [[VECEXT_3]])
+; NOACCELERATE-NEXT:    [[VECINS_31:%.*]] = insertelement <4 x float> [[VECINS_2]], float [[TMP4]], i32 3
 ; NOACCELERATE-NEXT:    ret <4 x float> [[VECINS_31]]
 ;
 entry:
@@ -741,10 +761,12 @@ define <4 x float> @atan_4x(ptr %a) {
 ; NOACCELERATE-NEXT:    [[VECEXT_1:%.*]] = extractelement <4 x float> [[TMP0]], i32 1
 ; NOACCELERATE-NEXT:    [[TMP2:%.*]] = tail call fast float @atanf(float [[VECEXT_1]])
 ; NOACCELERATE-NEXT:    [[VECINS_1:%.*]] = insertelement <4 x float> [[VECINS]], float [[TMP2]], i32 1
-; NOACCELERATE-NEXT:    [[TMP3:%.*]] = shufflevector <4 x float> [[TMP0]], <4 x float> poison, <2 x i32> <i32 2, i32 3>
-; NOACCELERATE-NEXT:    [[TMP4:%.*]] = call fast <2 x float> @llvm.atan.v2f32(<2 x float> [[TMP3]])
-; NOACCELERATE-NEXT:    [[TMP5:%.*]] = shufflevector <2 x float> [[TMP4]], <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
-; NOACCELERATE-NEXT:    [[VECINS_31:%.*]] = shufflevector <4 x float> [[VECINS_1]], <4 x float> [[TMP5]], <4 x i32> <i32 0, i32 1, i32 4, i32 5>
+; NOACCELERATE-NEXT:    [[VECEXT_2:%.*]] = extractelement <4 x float> [[TMP0]], i32 2
+; NOACCELERATE-NEXT:    [[TMP3:%.*]] = tail call fast float @atanf(float [[VECEXT_2]])
+; NOACCELERATE-NEXT:    [[VECINS_2:%.*]] = insertelement <4 x float> [[VECINS_1]], float [[TMP3]], i32 2
+; NOACCELERATE-NEXT:    [[VECEXT_3:%.*]] = extractelement <4 x float> [[TMP0]], i32 3
+; NOACCELERATE-NEXT:    [[TMP4:%.*]] = tail call fast float @atanf(float [[VECEXT_3]])
+; NOACCELERATE-NEXT:    [[VECINS_31:%.*]] = insertelement <4 x float> [[VECINS_2]], float [[TMP4]], i32 3
 ; NOACCELERATE-NEXT:    ret <4 x float> [[VECINS_31]]
 ;
 entry:
@@ -779,10 +801,12 @@ define <4 x float> @int_atan_4x(ptr %a) {
 ; NOACCELERATE-NEXT:    [[VECEXT_1:%.*]] = extractelement <4 x float> [[TMP0]], i32 1
 ; NOACCELERATE-NEXT:    [[TMP2:%.*]] = tail call fast float @llvm.atan.f32(float [[VECEXT_1]])
 ; NOACCELERATE-NEXT:    [[VECINS_1:%.*]] = insertelement <4 x float> [[VECINS]], float [[TMP2]], i32 1
-; NOACCELERATE-NEXT:    [[TMP3:%.*]] = shufflevector <4 x float> [[TMP0]], <4 x float> poison, <2 x i32> <i32 2, i32 3>
-; NOACCELERATE-NEXT:    [[TMP4:%.*]] = call fast <2 x float> @llvm.atan.v2f32(<2 x float> [[TMP3]])
-; NOACCELERATE-NEXT:    [[TMP5:%.*]] = shufflevector <2 x float> [[TMP4]], <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
-; NOACCELERATE-NEXT:    [[VECINS_31:%.*]] = shufflevector <4 x float> [[VECINS_1]], <4 x float> [[TMP5]], <4 x i32> <i32 0, i32 1, i32 4, i32 5>
+; NOACCELERATE-NEXT:    [[VECEXT_2:%.*]] = extractelement <4 x float> [[TMP0]], i32 2
+; NOACCELERATE-NEXT:    [[TMP3:%.*]] = tail call fast float @llvm.atan.f32(float [[VECEXT_2]])
+; NOACCELERATE-NEXT:    [[VECINS_2:%.*]] = insertelement <4 x float> [[VECINS_1]], float [[TMP3]], i32 2
+; NOACCELERATE-NEXT:    [[VECEXT_3:%.*]] = extractelement <4 x float> [[TMP0]], i32 3
+; NOACCELERATE-NEXT:    [[TMP4:%.*]] = tail call fast float @llvm.atan.f32(float [[VECEXT_3]])
+; NOACCELERATE-NEXT:    [[VECINS_31:%.*]] = insertelement <4 x float> [[VECINS_2]], float [[TMP4]], i32 3
 ; NOACCELERATE-NEXT:    ret <4 x float> [[VECINS_31]]
 ;
 entry:
@@ -822,11 +846,14 @@ define <4 x float> @atan2_4x(ptr %a, ptr %b) {
 ; NOACCELERATE-NEXT:    [[VECEXTB_1:%.*]] = extractelement <4 x float> [[BB]], i32 1
 ; NOACCELERATE-NEXT:    [[TMP2:%.*]] = tail call fast float @atan2f(float [[VECEXT_1]], float [[VECEXTB_1]])
 ; NOACCELERATE-NEXT:    [[VECINS_1:%.*]] = insertelement <4 x float> [[VECINS]], float [[TMP2]], i32 1
-; NOACCELERATE-NEXT:    [[TMP3:%.*]] = shufflevector <4 x float> [[TMP0]], <4 x float> poison, <2 x i32> <i32 2, i32 3>
-; NOACCELERATE-NEXT:    [[TMP4:%.*]] = shufflevector <4 x float> [[BB]], <4 x float> poison, <2 x i32> <i32 2, i32 3>
-; NOACCELERATE-NEXT:    [[TMP5:%.*]] = call fast <2 x float> @llvm.atan2.v2f32(<2 x float> [[TMP3]], <2 x float> [[TMP4]])
-; NOACCELERATE-NEXT:    [[TMP6:%.*]] = shufflevector <2 x float> [[TMP5]], <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
-; NOACCELERATE-NEXT:    [[VECINS_3:%.*]] = shufflevector <4 x float> [[VECINS_1]], <4 x float> [[TMP6]], <4 x i32> <i32 0, i32 1, i32 4, i32 5>
+; NOACCELERATE-NEXT:    [[VECEXT_2:%.*]] = extractelement <4 x float> [[TMP0]], i32 2
+; NOACCELERATE-NEXT:    [[VECEXTB_2:%.*]] = extractelement <4 x float> [[BB]], i32 2
+; NOACCELERATE-NEXT:    [[TMP3:%.*]] = tail call fast float @atan2f(float [[VECEXT_2]], float [[VECEXTB_2]])
+; NOACCELERATE-NEXT:    [[VECINS_2:%.*]] = insertelement <4 x float> [[VECINS_1]], float [[TMP3]], i32 2
+; NOACCELERATE-NEXT:    [[VECEXT_3:%.*]] = extractelement <4 x float> [[TMP0]], i32 3
+; NOACCELERATE-NEXT:    [[VECEXTB_3:%.*]] = extractelement <4 x float> [[BB]], i32 3
+; NOACCELERATE-NEXT:    [[TMP4:%.*]] = tail call fast float @atan2f(float [[VECEXT_3]], float [[VECEXTB_3]])
+; NOACCELERATE-NEXT:    [[VECINS_3:%.*]] = insertelement <4 x float> [[VECINS_2]], float [[TMP4]], i32 3
 ; NOACCELERATE-NEXT:    ret <4 x float> [[VECINS_3]]
 ;
 entry:
@@ -870,11 +897,14 @@ define <4 x float> @int_atan2_4x(ptr %a, ptr %b) {
 ; NOACCELERATE-NEXT:    [[VECEXTB_1:%.*]] = extractelement <4 x float> [[BB]], i32 1
 ; NOACCELERATE-NEXT:    [[TMP2:%.*]] = tail call fast float @llvm.atan2.f32(float [[VECEXT_1]], float [[VECEXTB_1]])
 ; NOACCELERATE-NEXT:    [[VECINS_1:%.*]] = insertelement <4 x float> [[VECINS]], float [[TMP2]], i32 1
-; NOACCELERATE-NEXT:    [[TMP3:%.*]] = shufflevector <4 x float> [[TMP0]], <4 x float> poison, <2 x i32> <i32 2, i32 3>
-; NOACCELERATE-NEXT:    [[TMP4:%.*]] = shufflevector <4 x float> [[BB]], <4 x float> poison, <2 x i32> <i32 2, i32 3>
-; NOACCELERATE-NEXT:    [[TMP5:%.*]] = call fast <2 x float> @llvm.atan2.v2f32(<2 x float> [[TMP3]], <2 x float> [[TMP4]])
-; NOACCELERATE-NEXT:    [[TMP6:%.*]] = shufflevector <2 x float> [[TMP5]], <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
-; NOACCELERATE-NEXT:    [[VECINS_31:%.*]] = shufflevector <4 x float> [[VECINS_1]], <4 x float> [[TMP6]], <4 x i32> <i32 0, i32 1, i32 4, i32 5>
+; NOACCELERATE-NEXT:    [[VECEXT_2:%.*]] = extractelement <4 x float> [[TMP0]], i32 2
+; NOACCELERATE-NEXT:    [[VECEXTB_2:%.*]] = extractelement <4 x float> [[BB]], i32 2
+; NOACCELERATE-NEXT:    [[TMP3:%.*]] = tail call fast float @llvm.atan2.f32(float [[VECEXT_2]], float [[VECEXTB_2]])
+; NOACCELERATE-NEXT:    [[VECINS_2:%.*]] = insertelement <4 x float> [[VECINS_1]], float [[TMP3]], i32 2
+; NOACCELERATE-NEXT:    [[VECEXT_3:%.*]] = extractelement <4 x float> [[TMP0]], i32 3
+; NOACCELERATE-NEXT:    [[VECEXTB_3:%.*]] = extractelement <4 x float> [[BB]], i32 3
+; NOACCELERATE-NEXT:    [[TMP4:%.*]] = tail call fast float @llvm.atan2.f32(float [[VECEXT_3]], float [[VECEXTB_3]])
+; NOACCELERATE-NEXT:    [[VECINS_31:%.*]] = insertelement <4 x float> [[VECINS_2]], float [[TMP4]], i32 3
 ; NOACCELERATE-NEXT:    ret <4 x float> [[VECINS_31]]
 ;
 entry:
@@ -915,10 +945,12 @@ define <4 x float> @sinh_4x(ptr %a) {
 ; NOACCELERATE-NEXT:    [[VECEXT_1:%.*]] = extractelement <4 x float> [[TMP0]], i32 1
 ; NOACCELERATE-NEXT:    [[TMP2:%.*]] = tail call fast float @sinhf(float [[VECEXT_1]])
 ; NOACCELERATE-NEXT:    [[VECINS_1:%.*]] = insertelement <4 x float> [[VECINS]], float [[TMP2]], i32 1
-; NOACCELERATE-NEXT:    [[TMP3:%.*]] = shufflevector <4 x float> [[TMP0]], <4 x float> poison, <2 x i32> <i32 2, i32 3>
-; NOACCELERATE-NEXT:    [[TMP4:%.*]] = call fast <2 x float> @llvm.sinh.v2f32(<2 x float> [[TMP3]])
-; NOACCELERATE-NEXT:    [[TMP5:%.*]] = shufflevector <2 x float> [[TMP4]], <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
-; NOACCELERATE-NEXT:    [[VECINS_31:%.*]] = shufflevector <4 x float> [[VECINS_1]], <4 x float> [[TMP5]], <4 x i32> <i32 0, i32 1, i32 4, i32 5>
+; NOACCELERATE-NEXT:    [[VECEXT_2:%.*]] = extractelement <4 x float> [[TMP0]], i32 2
+; NOACCELERATE-NEXT:    [[TMP3:%.*]] = tail call fast float @sinhf(float [[VECEXT_2]])
+; NOACCELERATE-NEXT:    [[VECINS_2:%.*]] = insertelement <4 x float> [[VECINS_1]], float [[TMP3]], i32 2
+; NOACCELERATE-NEXT:    [[VECEXT_3:%.*]] = extractelement <4 x float> [[TMP0]], i32 3
+; NOACCELERATE-NEXT:    [[TMP4:%.*]] = tail call fast float @sinhf(float [[VECEXT_3]])
+; NOACCELERATE-NEXT:    [[VECINS_31:%.*]] = insertelement <4 x float> [[VECINS_2]], float [[TMP4]], i32 3
 ; NOACCELERATE-NEXT:    ret <4 x float> [[VECINS_31]]
 ;
 entry:
@@ -953,10 +985,12 @@ define <4 x float> @int_sinh_4x(ptr %a) {
 ; NOACCELERATE-NEXT:    [[VECEXT_1:%.*]] = extractelement <4 x float> [[TMP0]], i32 1
 ; NOACCELERATE-NEXT:    [[TMP2:%.*]] = tail call fast float @llvm.sinh.f32(float [[VECEXT_1]])
 ; NOACCELERATE-NEXT:    [[VECINS_1:%.*]] = insertelement <4 x float> [[VECINS]], float [[TMP2]], i32 1
-; NOACCELERATE-NEXT:    [[TMP3:%.*]] = shufflevector <4 x float> [[TMP0]], <4 x float> poison, <2 x i32> <i32 2, i32 3>
-; NOACCELERATE-NEXT:    [[TMP4:%.*]] = call fast <2 x float> @llvm.sinh.v2f32(<2 x float> [[TMP3]])
-; NOACCELERATE-NEXT:    [[TMP5:%.*]] = shufflevector <2 x float> [[TMP4]], <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
-; NOACCELERATE-NEXT:    [[VECINS_31:%.*]] = shufflevector <4 x float> [[VECINS_1]], <4 x float> [[TMP5]], <4 x i32> <i32 0, i32 1, i32 4, i32 5>
+; NOACCELERATE-NEXT:    [[VECEXT_2:%.*]] = extractelement <4 x float> [[TMP0]], i32 2
+; NOACCELERATE-NEXT:    [[TMP3:%.*]] = tail call fast float @llvm.sinh.f32(float [[VECEXT_2]])
+; NOACCELERATE-NEXT:    [[VECINS_2:%.*]] = insertelement <4 x float> [[VECINS_1]], float [[TMP3]], i32 2
+; NOACCELERATE-NEXT:    [[VECEXT_3:%.*]] = extractelement <4 x float> [[TMP0]], i32 3
+; NOACCELERATE-NEXT:    [[TMP4:%.*]] = tail call fast float @llvm.sinh.f32(float [[VECEXT_3]])
+; NOACCELERATE-NEXT:    [[VECINS_31:%.*]] = insertelement <4 x float> [[VECINS_2]], float [[TMP4]], i32 3
 ; NOACCELERATE-NEXT:    ret <4 x float> [[VECINS_31]]
 ;
 entry:
@@ -992,10 +1026,12 @@ define <4 x float> @cosh_4x(ptr %a) {
 ; NOACCELERATE-NEXT:    [[VECEXT_1:%.*]] = extractelement <4 x float> [[TMP0]], i32 1
 ; NOACCELERATE-NEXT:    [[TMP2:%.*]] = tail call fast float @coshf(float [[VECEXT_1]])
 ; NOACCELERATE-NEXT:    [[VECINS_1:%.*]] = insertelement <4 x float> [[VECINS]], float [[TMP2]], i32 1
-; NOACCELERATE-NEXT:    [[TMP3:%.*]] = shufflevector <4 x float> [[TMP0]], <4 x float> poison, <2 x i32> <i32 2, i32 3>
-; NOACCELERATE-NEXT:    [[TMP4:%.*]] = call fast <2 x float> @llvm.cosh.v2f32(<2 x float> [[TMP3]])
-; NOACCELERATE-NEXT:    [[TMP5:%.*]] = shufflevector <2 x float> [[TMP4]], <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
-; NOACCELERATE-NEXT:    [[VECINS_31:%.*]] = shufflevector <4 x float> [[VECINS_1]], <4 x float> [[TMP5]], <4 x i32> <i32 0, i32 1, i32 4, i32 5>
+; NOACCELERATE-NEXT:    [[VECEXT_2:%.*]] = extractelement <4 x float> [[TMP0]], i32 2
+; NOACCELERATE-NEXT:    [[TMP3:%.*]] = tail call fast float @coshf(float [[VECEXT_2]])
+; NOACCELERATE-NEXT:    [[VECINS_2:%.*]] = insertelement <4 x float> [[VECINS_1]], float [[TMP3]], i32 2
+; NOACCELERATE-NEXT:    [[VECEXT_3:%.*]] = extractelement <4 x float> [[TMP0]], i32 3
+; NOACCELERATE-NEXT:    [[TMP4:%.*]] = tail call fast float @coshf(float [[VECEXT_3]])
+; NOACCELERATE-NEXT:    [[VECINS_31:%.*]] = insertelement <4 x float> [[VECINS_2]], float [[TMP4]], i32 3
 ; NOACCELERATE-NEXT:    ret <4 x float> [[VECINS_31]]
 ;
 entry:
@@ -1030,10 +1066,12 @@ define <4 x float> @int_cosh_4x(ptr %a) {
 ; NOACCELERATE-NEXT:    [[VECEXT_1:%.*]] = extractelement <4 x float> [[TMP0]], i32 1
 ; NOACCELERATE-NEXT:    [[TMP2:%.*]] = tail call fast float @llvm.cosh.f32(float [[VECEXT_1]])
 ; NOACCELERATE-NEXT:    [[VECINS_1:%.*]] = insertelement <4 x float> [[VECINS]], float [[TMP2]], i32 1
-; NOACCELERATE-NEXT:    [[TMP3:%.*]] = shufflevector <4 x float> [[TMP0]], <4 x float> poison, <2 x i32> <i32 2, i32 3>
-; NOACCELERATE-NEXT:    [[TMP4:%.*]] = call fast <2 x float> @llvm.cosh.v2f32(<2 x float> [[TMP3]])
-; NOACCELERATE-NEXT:    [[TMP5:%.*]] = shufflevector <2 x float> [[TMP4]], <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
-; NOACCELERATE-NEXT:    [[VECINS_31:%.*]] = shufflevector <4 x float> [[VECINS_1]], <4 x float> [[TMP5]], <4 x i32> <i32 0, i32 1, i32 4, i32 5>
+; NOACCELERATE-NEXT:    [[VECEXT_2:%.*]] = extractelement <4 x float> [[TMP0]], i32 2
+; NOACCELERATE-NEXT:    [[TMP3:%.*]] = tail call fast float @llvm.cosh.f32(float [[VECEXT_2]])
+; NOACCELERATE-NEXT:    [[VECINS_2:%.*]] = insertelement <4 x float> [[VECINS_1]], float [[TMP3]], i32 2
+; NOACCELERATE-NEXT:    [[VECEXT_3:%.*]] = extractelement <4 x float> [[TMP0]], i32 3
+; NOACCELERATE-NEXT:    [[TMP4:%.*]] = tail call fast float @llvm.cosh.f32(float [[VECEXT_3]])
+; NOACCELERATE-NEXT:    [[VECINS_31:%.*]] = insertelement <4 x float> [[VECINS_2]], float [[TMP4]], i32 3
 ; NOACCELERATE-NEXT:    ret <4 x float> [[VECINS_31]]
 ;
 entry:
@@ -1069,10 +1107,12 @@ define <4 x float> @tanh_4x(ptr %a) {
 ; NOACCELERATE-NEXT:    [[VECEXT_1:%.*]] = extractelement <4 x float> [[TMP0]], i32 1
 ; NOACCELERATE-NEXT:    [[TMP2:%.*]] = tail call fast float @tanhf(float [[VECEXT_1]])
 ; NOACCELERATE-NEXT:    [[VECINS_1:%.*]] = insertelement <4 x float> [[VECINS]], float [[TMP2]], i32 1
-; NOACCELERATE-NEXT:    [[TMP3:%.*]] = shufflevector <4 x float> [[TMP0]], <4 x float> poison, <2 x i32> <i32 2, i32 3>
-; NOACCELERATE-NEXT:    [[TMP4:%.*]] = call fast <2 x float> @llvm.tanh.v2f32(<2 x float> [[TMP3]])
-; NOACCELERATE-NEXT:    [[TMP5:%.*]] = shufflevector <2 x float> [[TMP4]], <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
-; NOACCELERATE-NEXT:    [[VECINS_31:%.*]] = shufflevector <4 x float> [[VECINS_1]], <4 x float> [[TMP5]], <4 x i32> <i32 0, i32 1, i32 4, i32 5>
+; NOACCELERATE-NEXT:    [[VECEXT_2:%.*]] = extractelement <4 x float> [[TMP0]], i32 2
+; NOACCELERATE-NEXT:    [[TMP3:%.*]] = tail call fast float @tanhf(float [[VECEXT_2]])
+; NOACCELERATE-NEXT:    [[VECINS_2:%.*]] = insertelement <4 x float> [[VECINS_1]], float [[TMP3]], i32 2
+; NOACCELERATE-NEXT:    [[VECEXT_3:%.*]] = extractelement <4 x float> [[TMP0]], i32 3
+; NOACCELERATE-NEXT:    [[TMP4:%.*]] = tail call fast float @tanhf(float [[VECEXT_3]])
+; NOACCELERATE-NEXT:    [[VECINS_31:%.*]] = insertelement <4 x float> [[VECINS_2]], float [[TMP4]], i32 3
 ; NOACCELERATE-NEXT:    ret <4 x float> [[VECINS_31]]
 ;
 entry:
@@ -1107,10 +1147,12 @@ define <4 x float> @int_tanh_4x(ptr %a) {
 ; NOACCELERATE-NEXT:    [[VECEXT_1:%.*]] = extractelement <4 x float> [[TMP0]], i32 1
 ; NOACCELERATE-NEXT:    [[TMP2:%.*]] = tail call fast float @llvm.tanh.f32(float [[VECEXT_1]])
 ; NOACCELERATE-NEXT:    [[VECINS_1:%.*]] = insertelement <4 x float> [[VECINS]], float [[TMP2]], i32 1
-; NOACCELERATE-NEXT:    [[TMP3:%.*]] = shufflevector <4 x float> [[TMP0]], <4 x float> poison, <2 x i32> <i32 2, i32 3>
-; NOACCELERATE-NEXT:    [[TMP4:%.*]] = call fast <2 x float> @llvm.tanh.v2f32(<2 x float> [[TMP3]])
-; NOACCELERATE-NEXT:    [[TMP5:%.*]] = shufflevector <2 x float> [[TMP4]], <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
-; NOACCELERATE-NEXT:    [[VECINS_31:%.*]] = shufflevector <4 x float> [[VECINS_1]], <4 x float> [[TMP5]], <4 x i32> <i32 0, i32 1, i32 4, i32 5>
+; NOACCELERATE-NEXT:    [[VECEXT_2:%.*]] = extractelement <4 x float> [[TMP0]], i32 2
+; NOACCELERATE-NEXT:    [[TMP3:%.*]] = tail call fast float @llvm.tanh.f32(float [[VECEXT_2]])
+; NOACCELERATE-NEXT:    [[VECINS_2:%.*]] = insertelement <4 x float> [[VECINS_1]], float [[TMP3]], i32 2
+; NOACCELERATE-NEXT:    [[VECEXT_3:%.*]] = extractelement <4 x float> [[TMP0]], i32 3
+; NOACCELERATE-NEXT:    [[TMP4:%.*]] = tail call fast float @llvm.tanh.f32(float [[VECEXT_3]])
+; NOACCELERATE-NEXT:    [[VECINS_31:%.*]] = insertelement <4 x float> [[VECINS_2]], float [[TMP4]], i32 3
 ; NOACCELERATE-NEXT:    ret <4 x float> [[VECINS_31]]
 ;
 entry:
@@ -1308,10 +1350,12 @@ define <4 x float> @int_cos_4x(ptr %a) {
 ; NOACCELERATE-NEXT:    [[VECEXT_1:%.*]] = extractelement <4 x float> [[TMP0]], i32 1
 ; NOACCELERATE-NEXT:    [[TMP2:%.*]] = tail call fast float @llvm.cos.f32(float [[VECEXT_1]])
 ; NOACCELERATE-NEXT:    [[VECINS_1:%.*]] = insertelement <4 x float> [[VECINS]], float [[TMP2]], i32 1
-; NOACCELERATE-NEXT:    [[TMP3:%.*]] = shufflevector <4 x float> [[TMP0]], <4 x float> poison, <2 x i32> <i32 2, i32 3>
-; NOACCELERATE-NEXT:    [[TMP4:%.*]] = call fast <2 x float> @llvm.cos.v2f32(<2 x float> [[TMP3]])
-; NOACCELERATE-NEXT:    [[TMP5:%.*]] = shufflevector <2 x float> [[TMP4]], <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
-; NOACCELERATE-NEXT:    [[VECINS_31:%.*]] = shufflevector <4 x float> [[VECINS_1]], <4 x float> [[TMP5]], <4 x i32> <i32 0, i32 1, i32 4, i32 5>
+; NOACCELERATE-NEXT:    [[VECEXT_2:%.*]] = extractelement <4 x float> [[TMP0]], i32 2
+; NOACCELERATE-NEXT:    [[TMP3:%.*]] = tail call fast float @llvm.cos.f32(float [[VECEXT_2]])
+; NOACCELERATE-NEXT:    [[VECINS_2:%.*]] = insertelement <4 x float> [[VECINS_1]], float [[TMP3]], i32 2
+; NOACCELERATE-NEXT:    [[VECEXT_3:%.*]] = extractelement <4 x float> [[TMP0]], i32 3
+; NOACCELERATE-NEXT:    [[TMP4:%.*]] = tail call fast float @llvm.cos.f32(float [[VECEXT_3]])
+; NOACCELERATE-NEXT:    [[VECINS_31:%.*]] = insertelement <4 x float> [[VECINS_2]], float [[TMP4]], i32 3
 ; NOACCELERATE-NEXT:    ret <4 x float> [[VECINS_31]]
 ;
 entry:
diff --git a/llvm/test/Transforms/SLPVectorizer/AArch64/accelerate-vector-functions.ll b/llvm/test/Transforms/SLPVectorizer/AArch64/accelerate-vector-functions.ll
index 058b68b8f6c78..34d65a307016f 100644
--- a/llvm/test/Transforms/SLPVectorizer/AArch64/accelerate-vector-functions.ll
+++ b/llvm/test/Transforms/SLPVectorizer/AArch64/accelerate-vector-functions.ll
@@ -20,21 +20,23 @@ define <4 x float> @int_sin_4x(ptr %a) {
 ; NOACCELERATE-NEXT:    [[TMP0:%.*]] = load <4 x float>, ptr [[A:%.*]], align 16
 ; NOACCELERATE-NEXT:    [[VECEXT:%.*]] = extractelement <4 x float> [[TMP0]], i32 0
 ; NOACCELERATE-NEXT:    [[TMP1:%.*]] = tail call fast float @llvm.sin.f32(float [[VECEXT]])
-; NOACCELERATE-NEXT:    [[VECINS:%.*]] = insertelement <4 x float> undef, float [[TMP1]], i32 0
+; NOACCELERATE-NEXT:    [[VECINS:%.*]] = insertelement <4 x float> zeroinitializer, float [[TMP1]], i32 0
 ; NOACCELERATE-NEXT:    [[VECEXT_1:%.*]] = extractelement <4 x float> [[TMP0]], i32 1
 ; NOACCELERATE-NEXT:    [[TMP2:%.*]] = tail call fast float @llvm.sin.f32(float [[VECEXT_1]])
 ; NOACCELERATE-NEXT:    [[VECINS_1:%.*]] = insertelement <4 x float> [[VECINS]], float [[TMP2]], i32 1
-; NOACCELERATE-NEXT:    [[TMP3:%.*]] = shufflevector <4 x float> [[TMP0]], <4 x float> poison, <2 x i32> <i32 2, i32 3>
-; NOACCELERATE-NEXT:    [[TMP4:%.*]] = call fast <2 x float> @llvm.sin.v2f32(<2 x float> [[TMP3]])
-; NOACCELERATE-NEXT:    [[TMP5:%.*]] = shufflevector <2 x float> [[TMP4]], <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
-; NOACCELERATE-NEXT:    [[VECINS_31:%.*]] = shufflevector <4 x float> [[VECINS_1]], <4 x float> [[TMP5]], <4 x i32> <i32 0, i32 1, i32 4, i32 5>
+; NOACCELERATE-NEXT:    [[VECEXT_2:%.*]] = extractelement <4 x float> [[TMP0]], i32 2
+; NOACCELERATE-NEXT:    [[TMP3:%.*]] = tail call fast float @llvm.sin.f32(float [[VECEXT_2]])
+; NOACCELERATE-NEXT:    [[VECINS_2:%.*]] = insertelement <4 x float> [[VECINS_1]], float [[TMP3]], i32 2
+; NOACCELERATE-NEXT:    [[VECEXT_3:%.*]] = extractelement <4 x float> [[TMP0]], i32 3
+; NOACCELERATE-NEXT:    [[TMP4:%.*]] = tail call fast float @llvm.sin.f32(float [[VECEXT_3]])
+; NOACCELERATE-NEXT:    [[VECINS_31:%.*]] = insertelement <4 x float> [[VECINS_2]], float [[TMP4]], i32 3
 ; NOACCELERATE-NEXT:    ret <4 x float> [[VECINS_31]]
 ;
 entry:
   %0 = load <4 x float>, ptr %a, align 16
   %vecext = extractelement <4 x float> %0, i32 0
   %1 = tail call fast float @llvm.sin.f32(float %vecext)
-  %vecins = insertelement <4 x float> undef, float %1, i32 0
+  %vecins = insertelement <4 x float> zeroinitializer, float %1, i32 0
   %vecext.1 = extractelement <4 x float> %0, i32 1
   %2 = tail call fast float @llvm.sin.f32(float %vecext.1)
   %vecins.1 = insertelement <4 x float> %vecins, float %2, i32 1
@@ -66,7 +68,7 @@ entry:
   %0 = load <4 x float>, ptr %a, align 16
   %vecext = extractelement <4 x float> %0, i32 0
   %1 = tail call fast float @ceilf(float %vecext)
-  %vecins = insertelement <4 x float> undef, float %1, i32 0
+  %vecins = insertelement <4 x float> zeroinitializer, float %1, i32 0
   %vecext.1 = extractelement <4 x float> %0, i32 1
   %2 = tail call fast float @ceilf(float %vecext.1)
   %vecins.1 = insertelement <4 x float> %vecins, float %2, i32 1
@@ -98,7 +100,7 @@ entry:
   %0 = load <4 x float>, ptr %a, align 16
   %vecext = extractelement <4 x float> %0, i32 0
   %1 = tail call fast float @fabsf(float %vecext)
-  %vecins = insertelement <4 x float> undef, float %1, i32 0
+  %vecins = insertelement <4 x float> zeroinitializer, float %1, i32 0
   %vecext.1 = extractelement <4 x float> %0, i32 1
   %2 = tail call fast float @fabsf(float %vecext.1)
   %vecins.1 = insertelement <4 x float> %vecins, float %2, i32 1
@@ -128,7 +130,7 @@ entry:
   %0 = load <4 x float>, ptr %a, align 16
   %vecext = extractelement <4 x float> %0, i32 0
   %1 = tail call fast float @llvm.fabs.f32(float %vecext)
-  %vecins = insertelement <4 x float> undef, float %1, i32 0
+  %vecins = insertelement <4 x float> zeroinitializer, float %1, i32 0
   %vecext.1 = extractelement <4 x float> %0, i32 1
   %2 = tail call fast float @llvm.fabs.f32(float %vecext.1)
   %vecins.1 = insertelement <4 x float> %vecins, float %2, i32 1
@@ -158,7 +160,7 @@ entry:
   %0 = load <4 x float>, ptr %a, align 16
   %vecext = extractelement <4 x float> %0, i32 0
   %1 = tail call fast float @floorf(float %vecext)
-  %vecins = insertelement <4 x float> undef, float %1, i32 0
+  %vecins = insertelement <4 x float> zeroinitializer, float %1, i32 0
   %vecext.1 = extractelement <4 x float> %0, i32 1
   %2 = tail call fast float @floorf(float %vecext.1)
   %vecins.1 = insertelement <4 x float> %vecins, float %2, i32 1
@@ -188,7 +190,7 @@ entry:
   %0 = load <4 x float>, ptr %a, align 16
   %vecext = extractelement <4 x float> %0, i32 0
   %1 = tail call fast float @sqrtf(float %vecext)
-  %vecins = insertelement <4 x float> undef, float %1, i32 0
+  %vecins = insertelement <4 x float> zeroinitializer, float %1, i32 0
   %vecext.1 = extractelement <4 x float> %0, i32 1
   %2 = tail call fast float @sqrtf(float %vecext.1)
   %vecins.1 = insertelement <4 x float> %vecins, float %2, i32 1
@@ -213,21 +215,23 @@ define <4 x float> @exp_4x(ptr %a) {
 ; NOACCELERATE-NEXT:    [[TMP0:%.*]] = load <4 x float>, ptr [[A:%.*]], align 16
 ; NOACCELERATE-NEXT:    [[VECEXT:%.*]] = extractelement <4 x float> [[TMP0]], i32 0
 ; NOACCELERATE-NEXT:    [[TMP1:%.*]] = tail call fast float @expf(float [[VECEXT]])
-; NOACCELERATE-NEXT:    [[VECINS:%.*]] = insertelement <4 x float> undef, float [[TMP1]], i32 0
+; NOACCELERATE-NEXT:    [[VECINS:%.*]] = insertelement <4 x float> zeroinitializer, float [[TMP1]], i32 0
 ; NOACCELERATE-NEXT:    [[VECEXT_1:%.*]] = extractelement <4 x float> [[TMP0]], i32 1
 ; NOACCELERATE-NEXT:    [[TMP2:%.*]] = tail call fast float @expf(float [[VECEXT_1]])
 ; NOACCELERATE-NEXT:    [[VECINS_1:%.*]] = insertelement <4 x float> [[VECINS]], float [[TMP2]], i32 1
-; NOACCELERATE-NEXT:    [[TMP3:%.*]] = shufflevector <4 x float> [[TMP0]], <4 x float> poison, <2 x i32> <i32 2, i32 3>
-; NOACCELERATE-NEXT:    [[TMP4:%.*]] = call fast <2 x float> @llvm.exp.v2f32(<2 x float> [[TMP3]])
-; NOACCELERATE-NEXT:    [[TMP5:%.*]] = shufflevector <2 x float> [[TMP4]], <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
-; NOACCELERATE-NEXT:    [[VECINS_31:%.*]] = shufflevector <4 x float> [[VECINS_1]], <4 x float> [[TMP5]], <4 x i32> <i32 0, i32 1, i32 4, i32 5>
+; NOACCELERATE-NEXT:    [[VECEXT_2:%.*]] = extractelement <4 x float> [[TMP0]], i32 2
+; NOACCELERATE-NEXT:    [[TMP3:%.*]] = tail call fast float @expf(float [[VECEXT_2]])
+; NOACCELERATE-NEXT:    [[VECINS_2:%.*]] = insertelement <4 x float> [[VECINS_1]], float [[TMP3]], i32 2
+; NOACCELERATE-NEXT:    [[VECEXT_3:%.*]] = extractelement <4 x float> [[TMP0]], i32 3
+; NOACCELERATE-NEXT:    [[TMP4:%.*]] = tail call fast float @expf(float [[VECEXT_3]])
+; NOACCELERATE-NEXT:    [[VECINS_31:%.*]] = insertelement <4 x float> [[VECINS_2]], float [[TMP4]], i32 3
 ; NOACCELERATE-NEXT:    ret <4 x float> [[VECINS_31]]
 ;
 entry:
   %0 = load <4 x float>, ptr %a, align 16
   %vecext = extractelement <4 x float> %0, i32 0
   %1 = tail call fast float @expf(float %vecext)
-  %vecins = insertelement <4 x float> undef, float %1, i32 0
+  %vecins = insertelement <4 x float> zeroinitializer, float %1, i32 0
   %vecext.1 = extractelement <4 x float> %0, i32 1
   %2 = tail call fast float @expf(float %vecext.1)
   %vecins.1 = insertelement <4 x float> %vecins, float %2, i32 1
@@ -252,7 +256,7 @@ define <4 x float> @expm1_4x(ptr %a) {
 ; NOACCELERATE-NEXT:    [[TMP0:%.*]] = load <4 x float>, ptr [[A:%.*]], align 16
 ; NOACCELERATE-NEXT:    [[VECEXT:%.*]] = extractelement <4 x float> [[TMP0]], i32 0
 ; NOACCELERATE-NEXT:    [[TMP1:%.*]] = tail call fast float @expm1f(float [[VECEXT]])
-; NOACCELERATE-NEXT:    [[VECINS:%.*]] = insertelement <4 x float> undef, float [[TMP1]], i32 0
+; NOACCELERATE-NEXT:    [[VECINS:%.*]] = insertelement <4 x float> zeroinitializer, float [[TMP1]], i32 0
 ; NOACCELERATE-NEXT:    [[VECEXT_1:%.*]] = extractelement <4 x float> [[TMP0]], i32 1
 ; NOACCELERATE-NEXT:    [[TMP2:%.*]] = tail call fast float @expm1f(float [[VECEXT_1]])
 ; NOACCELERATE-NEXT:    [[VECINS_1:%.*]] = insertelement <4 x float> [[VECINS]], float [[TMP2]], i32 1
@@ -268,7 +272,7 @@ entry:
   %0 = load <4 x float>, ptr %a, align 16
   %vecext = extractelement <4 x float> %0, i32 0
   %1 = tail call fast float @expm1f(float %vecext)
-  %vecins = insertelement <4 x float> undef, float %1, i32 0
+  %vecins = insertelement <4 x float> zeroinitializer, float %1, i32 0
   %vecext.1 = extractelement <4 x float> %0, i32 1
   %2 = tail call fast float @expm1f(float %vecext.1)
   %vecins.1 = insertelement <4 x float> %vecins, float %2, i32 1
@@ -293,21 +297,23 @@ define <4 x float> @log_4x(ptr %a) {
 ; NOACCELERATE-NEXT:    [[TMP0:%.*]] = load <4 x float>, ptr [[A:%.*]], align 16
 ; NOACCELERATE-NEXT:    [[VECEXT:%.*]] = extractelement <4 x float> [[TMP0]], i32 0
 ; NOACCELERATE-NEXT:    [[TMP1:%.*]] = tail call fast float @logf(float [[VECEXT]])
-; NOACCELERATE-NEXT:    [[VECINS:%.*]] = insertelement <4 x float> undef, float [[TMP1]], i32 0
+; NOACCELERATE-NEXT:    [[VECINS:%.*]] = insertelement <4 x float> zeroinitializer, float [[TMP1]], i32 0
 ; NOACCELERATE-NEXT:    [[VECEXT_1:%.*]] = extractelement <4 x float> [[TMP0]], i32 1
 ; NOACCELERATE-NEXT:    [[TMP2:%.*]] = tail call fast float @logf(float [[VECEXT_1]])
 ; NOACCELERATE-NEXT:    [[VECINS_1:%.*]] = insertelement <4 x float> [[VECINS]], float [[TMP2]], i32 1
-; NOACCELERATE-NEXT:    [[TMP3:%.*]] = shufflevector <4 x float> [[TMP0]], <4 x float> poison, <2 x i32> <i32 2, i32 3>
-; NOACCELERATE-NEXT:    [[TMP4:%.*]] = call fast <2 x float> @llvm.log.v2f32(<2 x float> [[TMP3]])
-; NOACCELERATE-NEXT:    [[TMP5:%.*]] = shufflevector <2 x float> [[TMP4]], <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
-; NOACCELERATE-NEXT:    [[VECINS_31:%.*]] = shufflevector <4 x float> [[VECINS_1]], <4 x float> [[TMP5]], <4 x i32> <i32 0, i32 1, i32 4, i32 5>
+; NOACCELERATE-NEXT:    [[VECEXT_2:%.*]] = extractelement <4 x float> [[TMP0]], i32 2
+; NOACCELERATE-NEXT:    [[TMP3:%.*]] = tail call fast float @logf(float [[VECEXT_2]])
+; NOACCELERATE-NEXT:    [[VECINS_2:%.*]] = insertelement <4 x float> [[VECINS_1]], float [[TMP3]], i32 2
+; NOACCELERATE-NEXT:    [[VECEXT_3:%.*]] = extractelement <4 x float> [[TMP0]], i32 3
+; NOACCELERATE-NEXT:    [[TMP4:%.*]] = tail call fast float @logf(float [[VECEXT_3]])
+; NOACCELERATE-NEXT:    [[VECINS_31:%.*]] = insertelement <4 x float> [[VECINS_2]], float [[TMP4]], i32 3
 ; NOACCELERATE-NEXT:    ret <4 x float> [[VECINS_31]]
 ;
 entry:
   %0 = load <4 x float>, ptr %a, align 16
   %vecext = extractelement <4 x float> %0, i32 0
   %1 = tail call fast float @logf(float %vecext)
-  %vecins = insertelement <4 x float> undef, float %1, i32 0
+  %vecins = insertelement <4 x float> zeroinitializer, float %1, i32 0
   %vecext.1 = extractelement <4 x float> %0, i32 1
   %2 = tail call fast float @logf(float %vecext.1)
   %vecins.1 = insertelement <4 x float> %vecins, float %2, i32 1
@@ -332,7 +338,7 @@ define <4 x float> @log1p_4x(ptr %a) {
 ; NOACCELERATE-NEXT:    [[TMP0:%.*]] = load <4 x float>, ptr [[A:%.*]], align 16
 ; NOACCELERATE-NEXT:    [[VECEXT:%.*]] = extractelement <4 x float> [[TMP0]], i32 0
 ; NOACCELERATE-NEXT:    [[TMP1:%.*]] = tail call fast float @log1pf(float [[VECEXT]])
-; NOACCELERATE-NEXT:    [[VECINS:%.*]] = insertelement <4 x float> undef, float [[TMP1]], i32 0
+; NOACCELERATE-NEXT:    [[VECINS:%.*]] = insertelement <4 x float> zeroinitializer, float [[TMP1]], i32 0
 ; NOACCELERATE-NEXT:    [[VECEXT_1:%.*]] = extractelement <4 x float> [[TMP0]], i32 1
 ; NOACCELERATE-NEXT:    [[TMP2:%.*]] = tail call fast float @log1pf(float [[VECEXT_1]])
 ; NOACCELERATE-NEXT:    [[VECINS_1:%.*]] = insertelement <4 x float> [[VECINS]], float [[TMP2]], i32 1
@@ -348,7 +354,7 @@ entry:
   %0 = load <4 x float>, ptr %a, align 16
   %vecext = extractelement <4 x float> %0, i32 0
   %1 = tail call fast float @log1pf(float %vecext)
-  %vecins = insertelement <4 x float> undef, float %1, i32 0
+  %vecins = insertelement <4 x float> zeroinitializer, float %1, i32 0
   %vecext.1 = extractelement <4 x float> %0, i32 1
   %2 = tail call fast float @log1pf(float %vecext.1)
   %vecins.1 = insertelement <4 x float> %vecins, float %2, i32 1
@@ -367,7 +373,7 @@ define <4 x float> @log10p_4x(ptr %a) {
 ; CHECK-NEXT:    [[TMP0:%.*]] = load <4 x float>, ptr [[A:%.*]], align 16
 ; CHECK-NEXT:    [[VECEXT:%.*]] = extractelement <4 x float> [[TMP0]], i32 0
 ; CHECK-NEXT:    [[TMP1:%.*]] = tail call fast float @log10pf(float [[VECEXT]])
-; CHECK-NEXT:    [[VECINS:%.*]] = insertelement <4 x float> undef, float [[TMP1]], i32 0
+; CHECK-NEXT:    [[VECINS:%.*]] = insertelement <4 x float> zeroinitializer, float [[TMP1]], i32 0
 ; CHECK-NEXT:    [[VECEXT_1:%.*]] = extractelement <4 x float> [[TMP0]], i32 1
 ; CHECK-NEXT:    [[TMP2:%.*]] = tail call fast float @log10pf(float [[VECEXT_1]])
 ; CHECK-NEXT:    [[VECINS_1:%.*]] = insertelement <4 x float> [[VECINS]], float [[TMP2]], i32 1
@@ -384,7 +390,7 @@ define <4 x float> @log10p_4x(ptr %a) {
 ; NOACCELERATE-NEXT:    [[TMP0:%.*]] = load <4 x float>, ptr [[A:%.*]], align 16
 ; NOACCELERATE-NEXT:    [[VECEXT:%.*]] = extractelement <4 x float> [[TMP0]], i32 0
 ; NOACCELERATE-NEXT:    [[TMP1:%.*]] = tail call fast float @log10pf(float [[VECEXT]])
-; NOACCELERATE-NEXT:    [[VECINS:%.*]] = insertelement <4 x float> undef, float [[TMP1]], i32 0
+; NOACCELERATE-NEXT:    [[VECINS:%.*]] = insertelement <4 x float> zeroinitializer, float [[TMP1]], i32 0
 ; NOACCELERATE-NEXT:    [[VECEXT_1:%.*]] = extractelement <4 x float> [[TMP0]], i32 1
 ; NOACCELERATE-NEXT:    [[TMP2:%.*]] = tail call fast float @log10pf(float [[VECEXT_1]])
 ; NOACCELERATE-NEXT:    [[VECINS_1:%.*]] = insertelement <4 x float> [[VECINS]], float [[TMP2]], i32 1
@@ -400,7 +406,7 @@ entry:
   %0 = load <4 x float>, ptr %a, align 16
   %vecext = extractelement <4 x float> %0, i32 0
   %1 = tail call fast float @log10pf(float %vecext)
-  %vecins = insertelement <4 x float> undef, float %1, i32 0
+  %vecins = insertelement <4 x float> zeroinitializer, float %1, i32 0
   %vecext.1 = extractelement <4 x float> %0, i32 1
   %2 = tail call fast float @log10pf(float %vecext.1)
   %vecins.1 = insertelement <4 x float> %vecins, float %2, i32 1
@@ -425,7 +431,7 @@ define <4 x float> @logb_4x(ptr %a) {
 ; NOACCELERATE-NEXT:    [[TMP0:%.*]] = load <4 x float>, ptr [[A:%.*]], align 16
 ; NOACCELERATE-NEXT:    [[VECEXT:%.*]] = extractelement <4 x float> [[TMP0]], i32 0
 ; NOACCELERATE-NEXT:    [[TMP1:%.*]] = tail call fast float @logbf(float [[VECEXT]])
-; NOACCELERATE-NEXT:    [[VECINS:%.*]] = insertelement <4 x float> undef, float [[TMP1]], i32 0
+; NOACCELERATE-NEXT:    [[VECINS:%.*]] = insertelement <4 x float> zeroinitializer, float [[TMP1]], i32 0
 ; NOACCELERATE-NEXT:    [[VECEXT_1:%.*]] = extractelement <4 x float> [[TMP0]], i32 1
 ; NOACCELERATE-NEXT:    [[TMP2:%.*]] = tail call fast float @logbf(float [[VECEXT_1]])
 ; NOACCELERATE-NEXT:    [[VECINS_1:%.*]] = insertelement <4 x float> [[VECINS]], float [[TMP2]], i32 1
@@ -441,7 +447,7 @@ entry:
   %0 = load <4 x float>, ptr %a, align 16
   %vecext = extractelement <4 x float> %0, i32 0
   %1 = tail call fast float @logbf(float %vecext)
-  %vecins = insertelement <4 x float> undef, float %1, i32 0
+  %vecins = insertelement <4 x float> zeroinitializer, float %1, i32 0
   %vecext.1 = extractelement <4 x float> %0, i32 1
   %2 = tail call fast float @logbf(float %vecext.1)
   %vecins.1 = insertelement <4 x float> %vecins, float %2, i32 1
@@ -466,21 +472,23 @@ define <4 x float> @sin_4x(ptr %a) {
 ; NOACCELERATE-NEXT:    [[TMP0:%.*]] = load <4 x float>, ptr [[A:%.*]], align 16
 ; NOACCELERATE-NEXT:    [[VECEXT:%.*]] = extractelement <4 x float> [[TMP0]], i32 0
 ; NOACCELERATE-NEXT:    [[TMP1:%.*]] = tail call fast float @sinf(float [[VECEXT]])
-; NOACCELERATE-NEXT:    [[VECINS:%.*]] = insertelement <4 x float> undef, float [[TMP1]], i32 0
+; NOACCELERATE-NEXT:    [[VECINS:%.*]] = insertelement <4 x float> zeroinitializer, float [[TMP1]], i32 0
 ; NOACCELERATE-NEXT:    [[VECEXT_1:%.*]] = extractelement <4 x float> [[TMP0]], i32 1
 ; NOACCELERATE-NEXT:    [[TMP2:%.*]] = tail call fast float @sinf(float [[VECEXT_1]])
 ; NOACCELERATE-NEXT:    [[VECINS_1:%.*]] = insertelement <4 x float> [[VECINS]], float [[TMP2]], i32 1
-; NOACCELERATE-NEXT:    [[TMP3:%.*]] = shufflevector <4 x float> [[TMP0]], <4 x float> poison, <2 x i32> <i32 2, i32 3>
-; NOACCELERATE-NEXT:    [[TMP4:%.*]] = call fast <2 x float> @llvm.sin.v2f32(<2 x float> [[TMP3]])
-; NOACCELERATE-NEXT:    [[TMP5:%.*]] = shufflevector <2 x float> [[TMP4]], <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
-; NOACCELERATE-NEXT:    [[VECINS_31:%.*]] = shufflevector <4 x float> [[VECINS_1]], <4 x float> [[TMP5]], <4 x i32> <i32 0, i32 1, i32 4, i32 5>
+; NOACCELERATE-NEXT:    [[VECEXT_2:%.*]] = extractelement <4 x float> [[TMP0]], i32 2
+; NOACCELERATE-NEXT:    [[TMP3:%.*]] = tail call fast float @sinf(float [[VECEXT_2]])
+; NOACCELERATE-NEXT:    [[VECINS_2:%.*]] = insertelement <4 x float> [[VECINS_1]], float [[TMP3]], i32 2
+; NOACCELERATE-NEXT:    [[VECEXT_3:%.*]] = extractelement <4 x float> [[TMP0]], i32 3
+; NOACCELERATE-NEXT:    [[TMP4:%.*]] = tail call fast float @sinf(float [[VECEXT_3]])
+; NOACCELERATE-NEXT:    [[VECINS_31:%.*]] = insertelement <4 x float> [[VECINS_2]], float [[TMP4]], i32 3
 ; NOACCELERATE-NEXT:    ret <4 x float> [[VECINS_31]]
 ;
 entry:
   %0 = load <4 x float>, ptr %a, align 16
   %vecext = extractelement <4 x float> %0, i32 0
   %1 = tail call fast float @sinf(float %vecext)
-  %vecins = insertelement <4 x float> undef, float %1, i32 0
+  %vecins = insertelement <4 x float> zeroinitializer, float %1, i32 0
   %vecext.1 = extractelement <4 x float> %0, i32 1
   %2 = tail call fast float @sinf(float %vecext.1)
   %vecins.1 = insertelement <4 x float> %vecins, float %2, i32 1
@@ -505,21 +513,23 @@ define <4 x float> @cos_4x(ptr %a) {
 ; NOACCELERATE-NEXT:    [[TMP0:%.*]] = load <4 x float>, ptr [[A:%.*]], align 16
 ; NOACCELERATE-NEXT:    [[VECEXT:%.*]] = extractelement <4 x float> [[TMP0]], i32 0
 ; NOACCELERATE-NEXT:    [[TMP1:%.*]] = tail call fast float @cosf(float [[VECEXT]])
-; NOACCELERATE-NEXT:    [[VECINS:%.*]] = insertelement <4 x float> undef, float [[TMP1]], i32 0
+; NOACCELERATE-NEXT:    [[VECINS:%.*]] = insertelement <4 x float> zeroinitializer, float [[TMP1]], i32 0
 ; NOACCELERATE-NEXT:    [[VECEXT_1:%.*]] = extractelement <4 x float> [[TMP0]], i32 1
 ; NOACCELERATE-NEXT:    [[TMP2:%.*]] = tail call fast float @cosf(float [[VECEXT_1]])
 ; NOACCELERATE-NEXT:    [[VECINS_1:%.*]] = insertelement <4 x float> [[VECINS]], float [[TMP2]], i32 1
-; NOACCELERATE-NEXT:    [[TMP3:%.*]] = shufflevector <4 x float> [[TMP0]], <4 x float> poison, <2 x i32> <i32 2, i32 3>
-; NOACCELERATE-NEXT:    [[TMP4:%.*]] = call fast <2 x float> @llvm.cos.v2f32(<2 x float> [[TMP3]])
-; NOACCELERATE-NEXT:    [[TMP5:%.*]] = shufflevector <2 x float> [[TMP4]], <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
-; NOACCELERATE-NEXT:    [[VECINS_31:%.*]] = shufflevector <4 x float> [[VECINS_1]], <4 x float> [[TMP5]], <4 x i32> <i32 0, i32 1, i32 4, i32 5>
+; NOACCELERATE-NEXT:    [[VECEXT_2:%.*]] = extractelement <4 x float> [[TMP0]], i32 2
+; NOACCELERATE-NEXT:    [[TMP3:%.*]] = tail call fast float @cosf(float [[VECEXT_2]])
+; NOACCELERATE-NEXT:    [[VECINS_2:%.*]] = insertelement <4 x float> [[VECINS_1]], float [[TMP3]], i32 2
+; NOACCELERATE-NEXT:    [[VECEXT_3:%.*]] = extractelement <4 x float> [[TMP0]], i32 3
+; NOACCELERATE-NEXT:    [[TMP4:%.*]] = tail call fast float @cosf(float [[VECEXT_3]])
+; NOACCELERATE-NEXT:    [[VECINS_31:%.*]] = insertelement <4 x float> [[VECINS_2]], float [[TMP4]], i32 3
 ; NOACCELERATE-NEXT:    ret <4 x float> [[VECINS_31]]
 ;
 entry:
   %0 = load <4 x float>, ptr %a, align 16
   %vecext = extractelement <4 x float> %0, i32 0
   %1 = tail call fast float @cosf(float %vecext)
-  %vecins = insertelement <4 x float> undef, float %1, i32 0
+  %vecins = insertelement <4 x float> zeroinitializer, float %1, i32 0
   %vecext.1 = extractelement <4 x float> %0, i32 1
   %2 = tail call fast float @cosf(float %vecext.1)
   %vecins.1 = insertelement <4 x float> %vecins, float %2, i32 1
@@ -544,21 +554,23 @@ define <4 x float> @tan_4x(ptr %a) {
 ; NOACCELERATE-NEXT:    [[TMP0:%.*]] = load <4 x float>, ptr [[A:%.*]], align 16
 ; NOACCELERATE-NEXT:    [[VECEXT:%.*]] = extractelement <4 x float> [[TMP0]], i32 0
 ; NOACCELERATE-NEXT:    [[TMP1:%.*]] = tail call fast float @tanf(float [[VECEXT]])
-; NOACCELERATE-NEXT:    [[VECINS:%.*]] = insertelement <4 x float> undef, float [[TMP1]], i32 0
+; NOACCELERATE-NEXT:    [[VECINS:%.*]] = insertelement <4 x float> zeroinitializer, float [[TMP1]], i32 0
 ; NOACCELERATE-NEXT:    [[VECEXT_1:%.*]] = extractelement <4 x float> [[TMP0]], i32 1
 ; NOACCELERATE-NEXT:    [[TMP2:%.*]] = tail call fast float @tanf(float [[VECEXT_1]])
 ; NOACCELERATE-NEXT:    [[VECINS_1:%.*]] = insertelement <4 x float> [[VECINS]], float [[TMP2]], i32 1
-; NOACCELERATE-NEXT:    [[TMP3:%.*]] = shufflevector <4 x float> [[TMP0]], <4 x float> poison, <2 x i32> <i32 2, i32 3>
-; NOACCELERATE-NEXT:    [[TMP4:%.*]] = call fast <2 x float> @llvm.tan.v2f32(<2 x float> [[TMP3]])
-; NOACCELERATE-NEXT:    [[TMP5:%.*]] = shufflevector <2 x float> [[TMP4]], <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
-; NOACCELERATE-NEXT:    [[VECINS_31:%.*]] = shufflevector <4 x float> [[VECINS_1]], <4 x float> [[TMP5]], <4 x i32> <i32 0, i32 1, i32 4, i32 5>
+; NOACCELERATE-NEXT:    [[VECEXT_2:%.*]] = extractelement <4 x float> [[TMP0]], i32 2
+; NOACCELERATE-NEXT:    [[TMP3:%.*]] = tail call fast float @tanf(float [[VECEXT_2]])
+; NOACCELERATE-NEXT:    [[VECINS_2:%.*]] = insertelement <4 x float> [[VECINS_1]], float [[TMP3]], i32 2
+; NOACCELERATE-NEXT:    [[VECEXT_3:%.*]] = extractelement <4 x float> [[TMP0]], i32 3
+; NOACCELERATE-NEXT:    [[TMP4:%.*]] = tail call fast float @tanf(float [[VECEXT_3]])
+; NOACCELERATE-NEXT:    [[VECINS_31:%.*]] = insertelement <4 x float> [[VECINS_2]], float [[TMP4]], i32 3
 ; NOACCELERATE-NEXT:    ret <4 x float> [[VECINS_31]]
 ;
 entry:
   %0 = load <4 x float>, ptr %a, align 16
   %vecext = extractelement <4 x float> %0, i32 0
   %1 = tail call fast float @tanf(float %vecext)
-  %vecins = insertelement <4 x float> undef, float %1, i32 0
+  %vecins = insertelement <4 x float> zeroinitializer, float %1, i32 0
   %vecext.1 = extractelement <4 x float> %0, i32 1
   %2 = tail call fast float @tanf(float %vecext.1)
   %vecins.1 = insertelement <4 x float> %vecins, float %2, i32 1
@@ -583,21 +595,23 @@ define <4 x float> @asin_4x(ptr %a) {
 ; NOACCELERATE-NEXT:    [[TMP0:%.*]] = load <4 x float>, ptr [[A:%.*]], align 16
 ; NOACCELERATE-NEXT:    [[VECEXT:%.*]] = extractelement <4 x float> [[TMP0]], i32 0
 ; NOACCELERATE-NEXT:    [[TMP1:%.*]] = tail call fast float @asinf(float [[VECEXT]])
-; NOACCELERATE-NEXT:    [[VECINS:%.*]] = insertelement <4 x float> undef, float [[TMP1]], i32 0
+; NOACCELERATE-NEXT:    [[VECINS:%.*]] = insertelement <4 x float> zeroinitializer, float [[TMP1]], i32 0
 ; NOACCELERATE-NEXT:    [[VECEXT_1:%.*]] = extractelement <4 x float> [[TMP0]], i32 1
 ; NOACCELERATE-NEXT:    [[TMP2:%.*]] = tail call fast float @asinf(float [[VECEXT_1]])
 ; NOACCELERATE-NEXT:    [[VECINS_1:%.*]] = insertelement <4 x float> [[VECINS]], float [[TMP2]], i32 1
-; NOACCELERATE-NEXT:    [[TMP3:%.*]] = shufflevector <4 x float> [[TMP0]], <4 x float> poison, <2 x i32> <i32 2, i32 3>
-; NOACCELERATE-NEXT:    [[TMP4:%.*]] = call fast <2 x float> @llvm.asin.v2f32(<2 x float> [[TMP3]])
-; NOACCELERATE-NEXT:    [[TMP5:%.*]] = shufflevector <2 x float> [[TMP4]], <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
-; NOACCELERATE-NEXT:    [[VECINS_31:%.*]] = shufflevector <4 x float> [[VECINS_1]], <4 x float> [[TMP5]], <4 x i32> <i32 0, i32 1, i32 4, i32 5>
+; NOACCELERATE-NEXT:    [[VECEXT_2:%.*]] = extractelement <4 x float> [[TMP0]], i32 2
+; NOACCELERATE-NEXT:    [[TMP3:%.*]] = tail call fast float @asinf(float [[VECEXT_2]])
+; NOACCELERATE-NEXT:    [[VECINS_2:%.*]] = insertelement <4 x float> [[VECINS_1]], float [[TMP3]], i32 2
+; NOACCELERATE-NEXT:    [[VECEXT_3:%.*]] = extractelement <4 x float> [[TMP0]], i32 3
+; NOACCELERATE-NEXT:    [[TMP4:%.*]] = tail call fast float @asinf(float [[VECEXT_3]])
+; NOACCELERATE-NEXT:    [[VECINS_31:%.*]] = insertelement <4 x float> [[VECINS_2]], float [[TMP4]], i32 3
 ; NOACCELERATE-NEXT:    ret <4 x float> [[VECINS_31]]
 ;
 entry:
   %0 = load <4 x float>, ptr %a, align 16
   %vecext = extractelement <4 x float> %0, i32 0
   %1 = tail call fast float @asinf(float %vecext)
-  %vecins = insertelement <4 x float> undef, float %1, i32 0
+  %vecins = insertelement <4 x float> zeroinitializer, float %1, i32 0
   %vecext.1 = extractelement <4 x float> %0, i32 1
   %2 = tail call fast float @asinf(float %vecext.1)
   %vecins.1 = insertelement <4 x float> %vecins, float %2, i32 1
@@ -621,21 +635,23 @@ define <4 x float> @int_asin_4x(ptr %a) {
 ; NOACCELERATE-NEXT:    [[TMP0:%.*]] = load <4 x float>, ptr [[A:%.*]], align 16
 ; NOACCELERATE-NEXT:    [[VECEXT:%.*]] = extractelement <4 x float> [[TMP0]], i32 0
 ; NOACCELERATE-NEXT:    [[TMP1:%.*]] = tail call fast float @llvm.asin.f32(float [[VECEXT]])
-; NOACCELERATE-NEXT:    [[VECINS:%.*]] = insertelement <4 x float> undef, float [[TMP1]], i32 0
+; NOACCELERATE-NEXT:    [[VECINS:%.*]] = insertelement <4 x float> zeroinitializer, float [[TMP1]], i32 0
 ; NOACCELERATE-NEXT:    [[VECEXT_1:%.*]] = extractelement <4 x float> [[TMP0]], i32 1
 ; NOACCELERATE-NEXT:    [[TMP2:%.*]] = tail call fast float @llvm.asin.f32(float [[VECEXT_1]])
 ; NOACCELERATE-NEXT:    [[VECINS_1:%.*]] = insertelement <4 x float> [[VECINS]], float [[TMP2]], i32 1
-; NOACCELERATE-NEXT:    [[TMP3:%.*]] = shufflevector <4 x float> [[TMP0]], <4 x float> poison, <2 x i32> <i32 2, i32 3>
-; NOACCELERATE-NEXT:    [[TMP4:%.*]] = call fast <2 x float> @llvm.asin.v2f32(<2 x float> [[TMP3]])
-; NOACCELERATE-NEXT:    [[TMP5:%.*]] = shufflevector <2 x float> [[TMP4]], <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
-; NOACCELERATE-NEXT:    [[VECINS_31:%.*]] = shufflevector <4 x float> [[VECINS_1]], <4 x float> [[TMP5]], <4 x i32> <i32 0, i32 1, i32 4, i32 5>
+; NOACCELERATE-NEXT:    [[VECEXT_2:%.*]] = extractelement <4 x float> [[TMP0]], i32 2
+; NOACCELERATE-NEXT:    [[TMP3:%.*]] = tail call fast float @llvm.asin.f32(float [[VECEXT_2]])
+; NOACCELERATE-NEXT:    [[VECINS_2:%.*]] = insertelement <4 x float> [[VECINS_1]], float [[TMP3]], i32 2
+; NOACCELERATE-NEXT:    [[VECEXT_3:%.*]] = extractelement <4 x float> [[TMP0]], i32 3
+; NOACCELERATE-NEXT:    [[TMP4:%.*]] = tail call fast float @llvm.asin.f32(float [[VECEXT_3]])
+; NOACCELERATE-NEXT:    [[VECINS_31:%.*]] = insertelement <4 x float> [[VECINS_2]], float [[TMP4]], i32 3
 ; NOACCELERATE-NEXT:    ret <4 x float> [[VECINS_31]]
 ;
 entry:
   %0 = load <4 x float>, ptr %a, align 16
   %vecext = extractelement <4 x float> %0, i32 0
   %1 = tail call fast float @llvm.asin.f32(float %vecext)
-  %vecins = insertelement <4 x float> undef, float %1, i32 0
+  %vecins = insertelement <4 x float> zeroinitializer, float %1, i32 0
   %vecext.1 = extractelement <4 x float> %0, i32 1
   %2 = tail call fast float @llvm.asin.f32(float %vecext.1)
   %vecins.1 = insertelement <4 x float> %vecins, float %2, i32 1
@@ -660,21 +676,23 @@ define <4 x float> @acos_4x(ptr %a) {
 ; NOACCELERATE-NEXT:    [[TMP0:%.*]] = load <4 x float>, ptr [[A:%.*]], align 16
 ; NOACCELERATE-NEXT:    [[VECEXT:%.*]] = extractelement <4 x float> [[TMP0]], i32 0
 ; NOACCELERATE-NEXT:    [[TMP1:%.*]] = tail call fast float @acosf(float [[VECEXT]])
-; NOACCELERATE-NEXT:    [[VECINS:%.*]] = insertelement <4 x float> undef, float [[TMP1]], i32 0
+; NOACCELERATE-NEXT:    [[VECINS:%.*]] = insertelement <4 x float> zeroinitializer, float [[TMP1]], i32 0
 ; NOACCELERATE-NEXT:    [[VECEXT_1:%.*]] = extractelement <4 x float> [[TMP0]], i32 1
 ; NOACCELERATE-NEXT:    [[TMP2:%.*]] = tail call fast float @acosf(float [[VECEXT_1]])
 ; NOACCELERATE-NEXT:    [[VECINS_1:%.*]] = insertelement <4 x float> [[VECINS]], float [[TMP2]], i32 1
-; NOACCELERATE-NEXT:    [[TMP3:%.*]] = shufflevector <4 x float> [[TMP0]], <4 x float> poison, <2 x i32> <i32 2, i32 3>
-; NOACCELERATE-NEXT:    [[TMP4:%.*]] = call fast <2 x float> @llvm.acos.v2f32(<2 x float> [[TMP3]])
-; NOACCELERATE-NEXT:    [[TMP5:%.*]] = shufflevector <2 x float> [[TMP4]], <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
-; NOACCELERATE-NEXT:    [[VECINS_31:%.*]] = shufflevector <4 x float> [[VECINS_1]], <4 x float> [[TMP5]], <4 x i32> <i32 0, i32 1, i32 4, i32 5>
+; NOACCELERATE-NEXT:    [[VECEXT_2:%.*]] = extractelement <4 x float> [[TMP0]], i32 2
+; NOACCELERATE-NEXT:    [[TMP3:%.*]] = tail call fast float @acosf(float [[VECEXT_2]])
+; NOACCELERATE-NEXT:    [[VECINS_2:%.*]] = insertelement <4 x float> [[VECINS_1]], float [[TMP3]], i32 2
+; NOACCELERATE-NEXT:    [[VECEXT_3:%.*]] = extractelement <4 x float> [[TMP0]], i32 3
+; NOACCELERATE-NEXT:    [[TMP4:%.*]] = tail call fast float @acosf(float [[VECEXT_3]])
+; NOACCELERATE-NEXT:    [[VECINS_31:%.*]] = insertelement <4 x float> [[VECINS_2]], float [[TMP4]], i32 3
 ; NOACCELERATE-NEXT:    ret <4 x float> [[VECINS_31]]
 ;
 entry:
   %0 = load <4 x float>, ptr %a, align 16
   %vecext = extractelement <4 x float> %0, i32 0
   %1 = tail call fast float @acosf(float %vecext)
-  %vecins = insertelement <4 x float> undef, float %1, i32 0
+  %vecins = insertelement <4 x float> zeroinitializer, float %1, i32 0
   %vecext.1 = extractelement <4 x float> %0, i32 1
   %2 = tail call fast float @acosf(float %vecext.1)
   %vecins.1 = insertelement <4 x float> %vecins, float %2, i32 1
@@ -698,21 +716,23 @@ define <4 x float> @int_acos_4x(ptr %a) {
 ; NOACCELERATE-NEXT:    [[TMP0:%.*]] = load <4 x float>, ptr [[A:%.*]], align 16
 ; NOACCELERATE-NEXT:    [[VECEXT:%.*]] = extractelement <4 x float> [[TMP0]], i32 0
 ; NOACCELERATE-NEXT:    [[TMP1:%.*]] = tail call fast float @llvm.acos.f32(float [[VECEXT]])
-; NOACCELERATE-NEXT:    [[VECINS:%.*]] = insertelement <4 x float> undef, float [[TMP1]], i32 0
+; NOACCELERATE-NEXT:    [[VECINS:%.*]] = insertelement <4 x float> zeroinitializer, float [[TMP1]], i32 0
 ; NOACCELERATE-NEXT:    [[VECEXT_1:%.*]] = extractelement <4 x float> [[TMP0]], i32 1
 ; NOACCELERATE-NEXT:    [[TMP2:%.*]] = tail call fast float @llvm.acos.f32(float [[VECEXT_1]])
 ; NOACCELERATE-NEXT:    [[VECINS_1:%.*]] = insertelement <4 x float> [[VECINS]], float [[TMP2]], i32 1
-; NOACCELERATE-NEXT:    [[TMP3:%.*]] = shufflevector <4 x float> [[TMP0]], <4 x float> poison, <2 x i32> <i32 2, i32 3>
-; NOACCELERATE-NEXT:    [[TMP4:%.*]] = call fast <2 x float> @llvm.acos.v2f32(<2 x float> [[TMP3]])
-; NOACCELERATE-NEXT:    [[TMP5:%.*]] = shufflevector <2 x float> [[TMP4]], <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
-; NOACCELERATE-NEXT:    [[VECINS_31:%.*]] = shufflevector <4 x float> [[VECINS_1]], <4 x float> [[TMP5]], <4 x i32> <i32 0, i32 1, i32 4, i32 5>
+; NOACCELERATE-NEXT:    [[VECEXT_2:%.*]] = extractelement <4 x float> [[TMP0]], i32 2
+; NOACCELERATE-NEXT:    [[TMP3:%.*]] = tail call fast float @llvm.acos.f32(float [[VECEXT_2]])
+; NOACCELERATE-NEXT:    [[VECINS_2:%.*]] = insertelement <4 x float> [[VECINS_1]], float [[TMP3]], i32 2
+; NOACCELERATE-NEXT:    [[VECEXT_3:%.*]] = extractelement <4 x float> [[TMP0]], i32 3
+; NOACCELERATE-NEXT:    [[TMP4:%.*]] = tail call fast float @llvm.acos.f32(float [[VECEXT_3]])
+; NOACCELERATE-NEXT:    [[VECINS_31:%.*]] = insertelement <4 x float> [[VECINS_2]], float [[TMP4]], i32 3
 ; NOACCELERATE-NEXT:    ret <4 x float> [[VECINS_31]]
 ;
 entry:
   %0 = load <4 x float>, ptr %a, align 16
   %vecext = extractelement <4 x float> %0, i32 0
   %1 = tail call fast float @llvm.acos.f32(float %vecext)
-  %vecins = insertelement <4 x float> undef, float %1, i32 0
+  %vecins = insertelement <4 x float> zeroinitializer, float %1, i32 0
   %vecext.1 = extractelement <4 x float> %0, i32 1
   %2 = tail call fast float @llvm.acos.f32(float %vecext.1)
   %vecins.1 = insertelement <4 x float> %vecins, float %2, i32 1
@@ -737,21 +757,23 @@ define <4 x float> @atan_4x(ptr %a) {
 ; NOACCELERATE-NEXT:    [[TMP0:%.*]] = load <4 x float>, ptr [[A:%.*]], align 16
 ; NOACCELERATE-NEXT:    [[VECEXT:%.*]] = extractelement <4 x float> [[TMP0]], i32 0
 ; NOACCELERATE-NEXT:    [[TMP1:%.*]] = tail call fast float @atanf(float [[VECEXT]])
-; NOACCELERATE-NEXT:    [[VECINS:%.*]] = insertelement <4 x float> undef, float [[TMP1]], i32 0
+; NOACCELERATE-NEXT:    [[VECINS:%.*]] = insertelement <4 x float> zeroinitializer, float [[TMP1]], i32 0
 ; NOACCELERATE-NEXT:    [[VECEXT_1:%.*]] = extractelement <4 x float> [[TMP0]], i32 1
 ; NOACCELERATE-NEXT:    [[TMP2:%.*]] = tail call fast float @atanf(float [[VECEXT_1]])
 ; NOACCELERATE-NEXT:    [[VECINS_1:%.*]] = insertelement <4 x float> [[VECINS]], float [[TMP2]], i32 1
-; NOACCELERATE-NEXT:    [[TMP3:%.*]] = shufflevector <4 x float> [[TMP0]], <4 x float> poison, <2 x i32> <i32 2, i32 3>
-; NOACCELERATE-NEXT:    [[TMP4:%.*]] = call fast <2 x float> @llvm.atan.v2f32(<2 x float> [[TMP3]])
-; NOACCELERATE-NEXT:    [[TMP5:%.*]] = shufflevector <2 x float> [[TMP4]], <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
-; NOACCELERATE-NEXT:    [[VECINS_31:%.*]] = shufflevector <4 x float> [[VECINS_1]], <4 x float> [[TMP5]], <4 x i32> <i32 0, i32 1, i32 4, i32 5>
+; NOACCELERATE-NEXT:    [[VECEXT_2:%.*]] = extractelement <4 x float> [[TMP0]], i32 2
+; NOACCELERATE-NEXT:    [[TMP3:%.*]] = tail call fast float @atanf(float [[VECEXT_2]])
+; NOACCELERATE-NEXT:    [[VECINS_2:%.*]] = insertelement <4 x float> [[VECINS_1]], float [[TMP3]], i32 2
+; NOACCELERATE-NEXT:    [[VECEXT_3:%.*]] = extractelement <4 x float> [[TMP0]], i32 3
+; NOACCELERATE-NEXT:    [[TMP4:%.*]] = tail call fast float @atanf(float [[VECEXT_3]])
+; NOACCELERATE-NEXT:    [[VECINS_31:%.*]] = insertelement <4 x float> [[VECINS_2]], float [[TMP4]], i32 3
 ; NOACCELERATE-NEXT:    ret <4 x float> [[VECINS_31]]
 ;
 entry:
   %0 = load <4 x float>, ptr %a, align 16
   %vecext = extractelement <4 x float> %0, i32 0
   %1 = tail call fast float @atanf(float %vecext)
-  %vecins = insertelement <4 x float> undef, float %1, i32 0
+  %vecins = insertelement <4 x float> zeroinitializer, float %1, i32 0
   %vecext.1 = extractelement <4 x float> %0, i32 1
   %2 = tail call fast float @atanf(float %vecext.1)
   %vecins.1 = insertelement <4 x float> %vecins, float %2, i32 1
@@ -775,21 +797,23 @@ define <4 x float> @int_atan_4x(ptr %a) {
 ; NOACCELERATE-NEXT:    [[TMP0:%.*]] = load <4 x float>, ptr [[A:%.*]], align 16
 ; NOACCELERATE-NEXT:    [[VECEXT:%.*]] = extractelement <4 x float> [[TMP0]], i32 0
 ; NOACCELERATE-NEXT:    [[TMP1:%.*]] = tail call fast float @llvm.atan.f32(float [[VECEXT]])
-; NOACCELERATE-NEXT:    [[VECINS:%.*]] = insertelement <4 x float> undef, float [[TMP1]], i32 0
+; NOACCELERATE-NEXT:    [[VECINS:%.*]] = insertelement <4 x float> zeroinitializer, float [[TMP1]], i32 0
 ; NOACCELERATE-NEXT:    [[VECEXT_1:%.*]] = extractelement <4 x float> [[TMP0]], i32 1
 ; NOACCELERATE-NEXT:    [[TMP2:%.*]] = tail call fast float @llvm.atan.f32(float [[VECEXT_1]])
 ; NOACCELERATE-NEXT:    [[VECINS_1:%.*]] = insertelement <4 x float> [[VECINS]], float [[TMP2]], i32 1
-; NOACCELERATE-NEXT:    [[TMP3:%.*]] = shufflevector <4 x float> [[TMP0]], <4 x float> poison, <2 x i32> <i32 2, i32 3>
-; NOACCELERATE-NEXT:    [[TMP4:%.*]] = call fast <2 x float> @llvm.atan.v2f32(<2 x float> [[TMP3]])
-; NOACCELERATE-NEXT:    [[TMP5:%.*]] = shufflevector <2 x float> [[TMP4]], <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
-; NOACCELERATE-NEXT:    [[VECINS_31:%.*]] = shufflevector <4 x float> [[VECINS_1]], <4 x float> [[TMP5]], <4 x i32> <i32 0, i32 1, i32 4, i32 5>
+; NOACCELERATE-NEXT:    [[VECEXT_2:%.*]] = extractelement <4 x float> [[TMP0]], i32 2
+; NOACCELERATE-NEXT:    [[TMP3:%.*]] = tail call fast float @llvm.atan.f32(float [[VECEXT_2]])
+; NOACCELERATE-NEXT:    [[VECINS_2:%.*]] = insertelement <4 x float> [[VECINS_1]], float [[TMP3]], i32 2
+; NOACCELERATE-NEXT:    [[VECEXT_3:%.*]] = extractelement <4 x float> [[TMP0]], i32 3
+; NOACCELERATE-NEXT:    [[TMP4:%.*]] = tail call fast float @llvm.atan.f32(float [[VECEXT_3]])
+; NOACCELERATE-NEXT:    [[VECINS_31:%.*]] = insertelement <4 x float> [[VECINS_2]], float [[TMP4]], i32 3
 ; NOACCELERATE-NEXT:    ret <4 x float> [[VECINS_31]]
 ;
 entry:
   %0 = load <4 x float>, ptr %a, align 16
   %vecext = extractelement <4 x float> %0, i32 0
   %1 = tail call fast float @llvm.atan.f32(float %vecext)
-  %vecins = insertelement <4 x float> undef, float %1, i32 0
+  %vecins = insertelement <4 x float> zeroinitializer, float %1, i32 0
   %vecext.1 = extractelement <4 x float> %0, i32 1
   %2 = tail call fast float @llvm.atan.f32(float %vecext.1)
   %vecins.1 = insertelement <4 x float> %vecins, float %2, i32 1
@@ -817,16 +841,19 @@ define <4 x float> @atan2_4x(ptr %a, ptr %b) {
 ; NOACCELERATE-NEXT:    [[VECEXT:%.*]] = extractelement <4 x float> [[TMP0]], i32 0
 ; NOACCELERATE-NEXT:    [[VECEXTB:%.*]] = extractelement <4 x float> [[BB]], i32 0
 ; NOACCELERATE-NEXT:    [[TMP1:%.*]] = tail call fast float @atan2f(float [[VECEXT]], float [[VECEXTB]])
-; NOACCELERATE-NEXT:    [[VECINS:%.*]] = insertelement <4 x float> undef, float [[TMP1]], i32 0
+; NOACCELERATE-NEXT:    [[VECINS:%.*]] = insertelement <4 x float> zeroinitializer, float [[TMP1]], i32 0
 ; NOACCELERATE-NEXT:    [[VECEXT_1:%.*]] = extractelement <4 x float> [[TMP0]], i32 1
 ; NOACCELERATE-NEXT:    [[VECEXTB_1:%.*]] = extractelement <4 x float> [[BB]], i32 1
 ; NOACCELERATE-NEXT:    [[TMP2:%.*]] = tail call fast float @atan2f(float [[VECEXT_1]], float [[VECEXTB_1]])
 ; NOACCELERATE-NEXT:    [[VECINS_1:%.*]] = insertelement <4 x float> [[VECINS]], float [[TMP2]], i32 1
-; NOACCELERATE-NEXT:    [[TMP3:%.*]] = shufflevector <4 x float> [[TMP0]], <4 x float> poison, <2 x i32> <i32 2, i32 3>
-; NOACCELERATE-NEXT:    [[TMP4:%.*]] = shufflevector <4 x float> [[BB]], <4 x float> poison, <2 x i32> <i32 2, i32 3>
-; NOACCELERATE-NEXT:    [[TMP5:%.*]] = call fast <2 x float> @llvm.atan2.v2f32(<2 x float> [[TMP3]], <2 x float> [[TMP4]])
-; NOACCELERATE-NEXT:    [[TMP6:%.*]] = shufflevector <2 x float> [[TMP5]], <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
-; NOACCELERATE-NEXT:    [[VECINS_3:%.*]] = shufflevector <4 x float> [[VECINS_1]], <4 x float> [[TMP6]], <4 x i32> <i32 0, i32 1, i32 4, i32 5>
+; NOACCELERATE-NEXT:    [[VECEXT_2:%.*]] = extractelement <4 x float> [[TMP0]], i32 2
+; NOACCELERATE-NEXT:    [[VECEXTB_2:%.*]] = extractelement <4 x float> [[BB]], i32 2
+; NOACCELERATE-NEXT:    [[TMP3:%.*]] = tail call fast float @atan2f(float [[VECEXT_2]], float [[VECEXTB_2]])
+; NOACCELERATE-NEXT:    [[VECINS_2:%.*]] = insertelement <4 x float> [[VECINS_1]], float [[TMP3]], i32 2
+; NOACCELERATE-NEXT:    [[VECEXT_3:%.*]] = extractelement <4 x float> [[TMP0]], i32 3
+; NOACCELERATE-NEXT:    [[VECEXTB_3:%.*]] = extractelement <4 x float> [[BB]], i32 3
+; NOACCELERATE-NEXT:    [[TMP4:%.*]] = tail call fast float @atan2f(float [[VECEXT_3]], float [[VECEXTB_3]])
+; NOACCELERATE-NEXT:    [[VECINS_3:%.*]] = insertelement <4 x float> [[VECINS_2]], float [[TMP4]], i32 3
 ; NOACCELERATE-NEXT:    ret <4 x float> [[VECINS_3]]
 ;
 entry:
@@ -835,7 +862,7 @@ entry:
   %vecext = extractelement <4 x float> %0, i32 0
   %vecextb = extractelement <4 x float> %bb, i32 0
   %1 = tail call fast float @atan2f(float %vecext, float %vecextb)
-  %vecins = insertelement <4 x float> undef, float %1, i32 0
+  %vecins = insertelement <4 x float> zeroinitializer, float %1, i32 0
   %vecext.1 = extractelement <4 x float> %0, i32 1
   %vecextb.1 = extractelement <4 x float> %bb, i32 1
   %2 = tail call fast float @atan2f(float %vecext.1, float %vecextb.1)
@@ -865,16 +892,19 @@ define <4 x float> @int_atan2_4x(ptr %a, ptr %b) {
 ; NOACCELERATE-NEXT:    [[VECEXT:%.*]] = extractelement <4 x float> [[TMP0]], i32 0
 ; NOACCELERATE-NEXT:    [[VECEXTB:%.*]] = extractelement <4 x float> [[BB]], i32 0
 ; NOACCELERATE-NEXT:    [[TMP1:%.*]] = tail call fast float @llvm.atan2.f32(float [[VECEXT]], float [[VECEXTB]])
-; NOACCELERATE-NEXT:    [[VECINS:%.*]] = insertelement <4 x float> undef, float [[TMP1]], i32 0
+; NOACCELERATE-NEXT:    [[VECINS:%.*]] = insertelement <4 x float> zeroinitializer, float [[TMP1]], i32 0
 ; NOACCELERATE-NEXT:    [[VECEXT_1:%.*]] = extractelement <4 x float> [[TMP0]], i32 1
 ; NOACCELERATE-NEXT:    [[VECEXTB_1:%.*]] = extractelement <4 x float> [[BB]], i32 1
 ; NOACCELERATE-NEXT:    [[TMP2:%.*]] = tail call fast float @llvm.atan2.f32(float [[VECEXT_1]], float [[VECEXTB_1]])
 ; NOACCELERATE-NEXT:    [[VECINS_1:%.*]] = insertelement <4 x float> [[VECINS]], float [[TMP2]], i32 1
-; NOACCELERATE-NEXT:    [[TMP3:%.*]] = shufflevector <4 x float> [[TMP0]], <4 x float> poison, <2 x i32> <i32 2, i32 3>
-; NOACCELERATE-NEXT:    [[TMP4:%.*]] = shufflevector <4 x float> [[BB]], <4 x float> poison, <2 x i32> <i32 2, i32 3>
-; NOACCELERATE-NEXT:    [[TMP5:%.*]] = call fast <2 x float> @llvm.atan2.v2f32(<2 x float> [[TMP3]], <2 x float> [[TMP4]])
-; NOACCELERATE-NEXT:    [[TMP6:%.*]] = shufflevector <2 x float> [[TMP5]], <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
-; NOACCELERATE-NEXT:    [[VECINS_31:%.*]] = shufflevector <4 x float> [[VECINS_1]], <4 x float> [[TMP6]], <4 x i32> <i32 0, i32 1, i32 4, i32 5>
+; NOACCELERATE-NEXT:    [[VECEXT_2:%.*]] = extractelement <4 x float> [[TMP0]], i32 2
+; NOACCELERATE-NEXT:    [[VECEXTB_2:%.*]] = extractelement <4 x float> [[BB]], i32 2
+; NOACCELERATE-NEXT:    [[TMP3:%.*]] = tail call fast float @llvm.atan2.f32(float [[VECEXT_2]], float [[VECEXTB_2]])
+; NOACCELERATE-NEXT:    [[VECINS_2:%.*]] = insertelement <4 x float> [[VECINS_1]], float [[TMP3]], i32 2
+; NOACCELERATE-NEXT:    [[VECEXT_3:%.*]] = extractelement <4 x float> [[TMP0]], i32 3
+; NOACCELERATE-NEXT:    [[VECEXTB_3:%.*]] = extractelement <4 x float> [[BB]], i32 3
+; NOACCELERATE-NEXT:    [[TMP4:%.*]] = tail call fast float @llvm.atan2.f32(float [[VECEXT_3]], float [[VECEXTB_3]])
+; NOACCELERATE-NEXT:    [[VECINS_31:%.*]] = insertelement <4 x float> [[VECINS_2]], float [[TMP4]], i32 3
 ; NOACCELERATE-NEXT:    ret <4 x float> [[VECINS_31]]
 ;
 entry:
@@ -883,7 +913,7 @@ entry:
   %vecext = extractelement <4 x float> %0, i32 0
   %vecextb = extractelement <4 x float> %bb, i32 0
   %1 = tail call fast float @llvm.atan2.f32(float %vecext, float %vecextb)
-  %vecins = insertelement <4 x float> undef, float %1, i32 0
+  %vecins = insertelement <4 x float> zeroinitializer, float %1, i32 0
   %vecext.1 = extractelement <4 x float> %0, i32 1
   %vecextb.1 = extractelement <4 x float> %bb, i32 1
   %2 = tail call fast float @llvm.atan2.f32(float %vecext.1, float %vecextb.1)
@@ -911,21 +941,23 @@ define <4 x float> @sinh_4x(ptr %a) {
 ; NOACCELERATE-NEXT:    [[TMP0:%.*]] = load <4 x float>, ptr [[A:%.*]], align 16
 ; NOACCELERATE-NEXT:    [[VECEXT:%.*]] = extractelement <4 x float> [[TMP0]], i32 0
 ; NOACCELERATE-NEXT:    [[TMP1:%.*]] = tail call fast float @sinhf(float [[VECEXT]])
-; NOACCELERATE-NEXT:    [[VECINS:%.*]] = insertelement <4 x float> undef, float [[TMP1]], i32 0
+; NOACCELERATE-NEXT:    [[VECINS:%.*]] = insertelement <4 x float> zeroinitializer, float [[TMP1]], i32 0
 ; NOACCELERATE-NEXT:    [[VECEXT_1:%.*]] = extractelement <4 x float> [[TMP0]], i32 1
 ; NOACCELERATE-NEXT:    [[TMP2:%.*]] = tail call fast float @sinhf(float [[VECEXT_1]])
 ; NOACCELERATE-NEXT:    [[VECINS_1:%.*]] = insertelement <4 x float> [[VECINS]], float [[TMP2]], i32 1
-; NOACCELERATE-NEXT:    [[TMP3:%.*]] = shufflevector <4 x float> [[TMP0]], <4 x float> poison, <2 x i32> <i32 2, i32 3>
-; NOACCELERATE-NEXT:    [[TMP4:%.*]] = call fast <2 x float> @llvm.sinh.v2f32(<2 x float> [[TMP3]])
-; NOACCELERATE-NEXT:    [[TMP5:%.*]] = shufflevector <2 x float> [[TMP4]], <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
-; NOACCELERATE-NEXT:    [[VECINS_31:%.*]] = shufflevector <4 x float> [[VECINS_1]], <4 x float> [[TMP5]], <4 x i32> <i32 0, i32 1, i32 4, i32 5>
+; NOACCELERATE-NEXT:    [[VECEXT_2:%.*]] = extractelement <4 x float> [[TMP0]], i32 2
+; NOACCELERATE-NEXT:    [[TMP3:%.*]] = tail call fast float @sinhf(float [[VECEXT_2]])
+; NOACCELERATE-NEXT:    [[VECINS_2:%.*]] = insertelement <4 x float> [[VECINS_1]], float [[TMP3]], i32 2
+; NOACCELERATE-NEXT:    [[VECEXT_3:%.*]] = extractelement <4 x float> [[TMP0]], i32 3
+; NOACCELERATE-NEXT:    [[TMP4:%.*]] = tail call fast float @sinhf(float [[VECEXT_3]])
+; NOACCELERATE-NEXT:    [[VECINS_31:%.*]] = insertelement <4 x float> [[VECINS_2]], float [[TMP4]], i32 3
 ; NOACCELERATE-NEXT:    ret <4 x float> [[VECINS_31]]
 ;
 entry:
   %0 = load <4 x float>, ptr %a, align 16
   %vecext = extractelement <4 x float> %0, i32 0
   %1 = tail call fast float @sinhf(float %vecext)
-  %vecins = insertelement <4 x float> undef, float %1, i32 0
+  %vecins = insertelement <4 x float> zeroinitializer, float %1, i32 0
   %vecext.1 = extractelement <4 x float> %0, i32 1
   %2 = tail call fast float @sinhf(float %vecext.1)
   %vecins.1 = insertelement <4 x float> %vecins, float %2, i32 1
@@ -949,21 +981,23 @@ define <4 x float> @int_sinh_4x(ptr %a) {
 ; NOACCELERATE-NEXT:    [[TMP0:%.*]] = load <4 x float>, ptr [[A:%.*]], align 16
 ; NOACCELERATE-NEXT:    [[VECEXT:%.*]] = extractelement <4 x float> [[TMP0]], i32 0
 ; NOACCELERATE-NEXT:    [[TMP1:%.*]] = tail call fast float @llvm.sinh.f32(float [[VECEXT]])
-; NOACCELERATE-NEXT:    [[VECINS:%.*]] = insertelement <4 x float> undef, float [[TMP1]], i32 0
+; NOACCELERATE-NEXT:    [[VECINS:%.*]] = insertelement <4 x float> zeroinitializer, float [[TMP1]], i32 0
 ; NOACCELERATE-NEXT:    [[VECEXT_1:%.*]] = extractelement <4 x float> [[TMP0]], i32 1
 ; NOACCELERATE-NEXT:    [[TMP2:%.*]] = tail call fast float @llvm.sinh.f32(float [[VECEXT_1]])
 ; NOACCELERATE-NEXT:    [[VECINS_1:%.*]] = insertelement <4 x float> [[VECINS]], float [[TMP2]], i32 1
-; NOACCELERATE-NEXT:    [[TMP3:%.*]] = shufflevector <4 x float> [[TMP0]], <4 x float> poison, <2 x i32> <i32 2, i32 3>
-; NOACCELERATE-NEXT:    [[TMP4:%.*]] = call fast <2 x float> @llvm.sinh.v2f32(<2 x float> [[TMP3]])
-; NOACCELERATE-NEXT:    [[TMP5:%.*]] = shufflevector <2 x float> [[TMP4]], <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
-; NOACCELERATE-NEXT:    [[VECINS_31:%.*]] = shufflevector <4 x float> [[VECINS_1]], <4 x float> [[TMP5]], <4 x i32> <i32 0, i32 1, i32 4, i32 5>
+; NOACCELERATE-NEXT:    [[VECEXT_2:%.*]] = extractelement <4 x float> [[TMP0]], i32 2
+; NOACCELERATE-NEXT:    [[TMP3:%.*]] = tail call fast float @llvm.sinh.f32(float [[VECEXT_2]])
+; NOACCELERATE-NEXT:    [[VECINS_2:%.*]] = insertelement <4 x float> [[VECINS_1]], float [[TMP3]], i32 2
+; NOACCELERATE-NEXT:    [[VECEXT_3:%.*]] = extractelement <4 x float> [[TMP0]], i32 3
+; NOACCELERATE-NEXT:    [[TMP4:%.*]] = tail call fast float @llvm.sinh.f32(float [[VECEXT_3]])
+; NOACCELERATE-NEXT:    [[VECINS_31:%.*]] = insertelement <4 x float> [[VECINS_2]], float [[TMP4]], i32 3
 ; NOACCELERATE-NEXT:    ret <4 x float> [[VECINS_31]]
 ;
 entry:
   %0 = load <4 x float>, ptr %a, align 16
   %vecext = extractelement <4 x float> %0, i32 0
   %1 = tail call fast float @llvm.sinh.f32(float %vecext)
-  %vecins = insertelement <4 x float> undef, float %1, i32 0
+  %vecins = insertelement <4 x float> zeroinitializer, float %1, i32 0
   %vecext.1 = extractelement <4 x float> %0, i32 1
   %2 = tail call fast float @llvm.sinh.f32(float %vecext.1)
   %vecins.1 = insertelement <4 x float> %vecins, float %2, i32 1
@@ -988,21 +1022,23 @@ define <4 x float> @cosh_4x(ptr %a) {
 ; NOACCELERATE-NEXT:    [[TMP0:%.*]] = load <4 x float>, ptr [[A:%.*]], align 16
 ; NOACCELERATE-NEXT:    [[VECEXT:%.*]] = extractelement <4 x float> [[TMP0]], i32 0
 ; NOACCELERATE-NEXT:    [[TMP1:%.*]] = tail call fast float @coshf(float [[VECEXT]])
-; NOACCELERATE-NEXT:    [[VECINS:%.*]] = insertelement <4 x float> undef, float [[TMP1]], i32 0
+; NOACCELERATE-NEXT:    [[VECINS:%.*]] = insertelement <4 x float> zeroinitializer, float [[TMP1]], i32 0
 ; NOACCELERATE-NEXT:    [[VECEXT_1:%.*]] = extractelement <4 x float> [[TMP0]], i32 1
 ; NOACCELERATE-NEXT:    [[TMP2:%.*]] = tail call fast float @coshf(float [[VECEXT_1]])
 ; NOACCELERATE-NEXT:    [[VECINS_1:%.*]] = insertelement <4 x float> [[VECINS]], float [[TMP2]], i32 1
-; NOACCELERATE-NEXT:    [[TMP3:%.*]] = shufflevector <4 x float> [[TMP0]], <4 x float> poison, <2 x i32> <i32 2, i32 3>
-; NOACCELERATE-NEXT:    [[TMP4:%.*]] = call fast <2 x float> @llvm.cosh.v2f32(<2 x float> [[TMP3]])
-; NOACCELERATE-NEXT:    [[TMP5:%.*]] = shufflevector <2 x float> [[TMP4]], <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
-; NOACCELERATE-NEXT:    [[VECINS_31:%.*]] = shufflevector <4 x float> [[VECINS_1]], <4 x float> [[TMP5]], <4 x i32> <i32 0, i32 1, i32 4, i32 5>
+; NOACCELERATE-NEXT:    [[VECEXT_2:%.*]] = extractelement <4 x float> [[TMP0]], i32 2
+; NOACCELERATE-NEXT:    [[TMP3:%.*]] = tail call fast float @coshf(float [[VECEXT_2]])
+; NOACCELERATE-NEXT:    [[VECINS_2:%.*]] = insertelement <4 x float> [[VECINS_1]], float [[TMP3]], i32 2
+; NOACCELERATE-NEXT:    [[VECEXT_3:%.*]] = extractelement <4 x float> [[TMP0]], i32 3
+; NOACCELERATE-NEXT:    [[TMP4:%.*]] = tail call fast float @coshf(float [[VECEXT_3]])
+; NOACCELERATE-NEXT:    [[VECINS_31:%.*]] = insertelement <4 x float> [[VECINS_2]], float [[TMP4]], i32 3
 ; NOACCELERATE-NEXT:    ret <4 x float> [[VECINS_31]]
 ;
 entry:
   %0 = load <4 x float>, ptr %a, align 16
   %vecext = extractelement <4 x float> %0, i32 0
   %1 = tail call fast float @coshf(float %vecext)
-  %vecins = insertelement <4 x float> undef, float %1, i32 0
+  %vecins = insertelement <4 x float> zeroinitializer, float %1, i32 0
   %vecext.1 = extractelement <4 x float> %0, i32 1
   %2 = tail call fast float @coshf(float %vecext.1)
   %vecins.1 = insertelement <4 x float> %vecins, float %2, i32 1
@@ -1026,21 +1062,23 @@ define <4 x float> @int_cosh_4x(ptr %a) {
 ; NOACCELERATE-NEXT:    [[TMP0:%.*]] = load <4 x float>, ptr [[A:%.*]], align 16
 ; NOACCELERATE-NEXT:    [[VECEXT:%.*]] = extractelement <4 x float> [[TMP0]], i32 0
 ; NOACCELERATE-NEXT:    [[TMP1:%.*]] = tail call fast float @llvm.cosh.f32(float [[VECEXT]])
-; NOACCELERATE-NEXT:    [[VECINS:%.*]] = insertelement <4 x float> undef, float [[TMP1]], i32 0
+; NOACCELERATE-NEXT:    [[VECINS:%.*]] = insertelement <4 x float> zeroinitializer, float [[TMP1]], i32 0
 ; NOACCELERATE-NEXT:    [[VECEXT_1:%.*]] = extractelement <4 x float> [[TMP0]], i32 1
 ; NOACCELERATE-NEXT:    [[TMP2:%.*]] = tail call fast float @llvm.cosh.f32(float [[VECEXT_1]])
 ; NOACCELERATE-NEXT:    [[VECINS_1:%.*]] = insertelement <4 x float> [[VECINS]], float [[TMP2]], i32 1
-; NOACCELERATE-NEXT:    [[TMP3:%.*]] = shufflevector <4 x float> [[TMP0]], <4 x float> poison, <2 x i32> <i32 2, i32 3>
-; NOACCELERATE-NEXT:    [[TMP4:%.*]] = call fast <2 x float> @llvm.cosh.v2f32(<2 x float> [[TMP3]])
-; NOACCELERATE-NEXT:    [[TMP5:%.*]] = shufflevector <2 x float> [[TMP4]], <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
-; NOACCELERATE-NEXT:    [[VECINS_31:%.*]] = shufflevector <4 x float> [[VECINS_1]], <4 x float> [[TMP5]], <4 x i32> <i32 0, i32 1, i32 4, i32 5>
+; NOACCELERATE-NEXT:    [[VECEXT_2:%.*]] = extractelement <4 x float> [[TMP0]], i32 2
+; NOACCELERATE-NEXT:    [[TMP3:%.*]] = tail call fast float @llvm.cosh.f32(float [[VECEXT_2]])
+; NOACCELERATE-NEXT:    [[VECINS_2:%.*]] = insertelement <4 x float> [[VECINS_1]], float [[TMP3]], i32 2
+; NOACCELERATE-NEXT:    [[VECEXT_3:%.*]] = extractelement <4 x float> [[TMP0]], i32 3
+; NOACCELERATE-NEXT:    [[TMP4:%.*]] = tail call fast float @llvm.cosh.f32(float [[VECEXT_3]])
+; NOACCELERATE-NEXT:    [[VECINS_31:%.*]] = insertelement <4 x float> [[VECINS_2]], float [[TMP4]], i32 3
 ; NOACCELERATE-NEXT:    ret <4 x float> [[VECINS_31]]
 ;
 entry:
   %0 = load <4 x float>, ptr %a, align 16
   %vecext = extractelement <4 x float> %0, i32 0
   %1 = tail call fast float @llvm.cosh.f32(float %vecext)
-  %vecins = insertelement <4 x float> undef, float %1, i32 0
+  %vecins = insertelement <4 x float> zeroinitializer, float %1, i32 0
   %vecext.1 = extractelement <4 x float> %0, i32 1
   %2 = tail call fast float @llvm.cosh.f32(float %vecext.1)
   %vecins.1 = insertelement <4 x float> %vecins, float %2, i32 1
@@ -1065,21 +1103,23 @@ define <4 x float> @tanh_4x(ptr %a) {
 ; NOACCELERATE-NEXT:    [[TMP0:%.*]] = load <4 x float>, ptr [[A:%.*]], align 16
 ; NOACCELERATE-NEXT:    [[VECEXT:%.*]] = extractelement <4 x float> [[TMP0]], i32 0
 ; NOACCELERATE-NEXT:    [[TMP1:%.*]] = tail call fast float @tanhf(float [[VECEXT]])
-; NOACCELERATE-NEXT:    [[VECINS:%.*]] = insertelement <4 x float> undef, float [[TMP1]], i32 0
+; NOACCELERATE-NEXT:    [[VECINS:%.*]] = insertelement <4 x float> zeroinitializer, float [[TMP1]], i32 0
 ; NOACCELERATE-NEXT:    [[VECEXT_1:%.*]] = extractelement <4 x float> [[TMP0]], i32 1
 ; NOACCELERATE-NEXT:    [[TMP2:%.*]] = tail call fast float @tanhf(float [[VECEXT_1]])
 ; NOACCELERATE-NEXT:    [[VECINS_1:%.*]] = insertelement <4 x float> [[VECINS]], float [[TMP2]], i32 1
-; NOACCELERATE-NEXT:    [[TMP3:%.*]] = shufflevector <4 x float> [[TMP0]], <4 x float> poison, <2 x i32> <i32 2, i32 3>
-; NOACCELERATE-NEXT:    [[TMP4:%.*]] = call fast <2 x float> @llvm.tanh.v2f32(<2 x float> [[TMP3]])
-; NOACCELERATE-NEXT:    [[TMP5:%.*]] = shufflevector <2 x float> [[TMP4]], <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
-; NOACCELERATE-NEXT:    [[VECINS_31:%.*]] = shufflevector <4 x float> [[VECINS_1]], <4 x float> [[TMP5]], <4 x i32> <i32 0, i32 1, i32 4, i32 5>
+; NOACCELERATE-NEXT:    [[VECEXT_2:%.*]] = extractelement <4 x float> [[TMP0]], i32 2
+; NOACCELERATE-NEXT:    [[TMP3:%.*]] = tail call fast float @tanhf(float [[VECEXT_2]])
+; NOACCELERATE-NEXT:    [[VECINS_2:%.*]] = insertelement <4 x float> [[VECINS_1]], float [[TMP3]], i32 2
+; NOACCELERATE-NEXT:    [[VECEXT_3:%.*]] = extractelement <4 x float> [[TMP0]], i32 3
+; NOACCELERATE-NEXT:    [[TMP4:%.*]] = tail call fast float @tanhf(float [[VECEXT_3]])
+; NOACCELERATE-NEXT:    [[VECINS_31:%.*]] = insertelement <4 x float> [[VECINS_2]], float [[TMP4]], i32 3
 ; NOACCELERATE-NEXT:    ret <4 x float> [[VECINS_31]]
 ;
 entry:
   %0 = load <4 x float>, ptr %a, align 16
   %vecext = extractelement <4 x float> %0, i32 0
   %1 = tail call fast float @tanhf(float %vecext)
-  %vecins = insertelement <4 x float> undef, float %1, i32 0
+  %vecins = insertelement <4 x float> zeroinitializer, float %1, i32 0
   %vecext.1 = extractelement <4 x float> %0, i32 1
   %2 = tail call fast float @tanhf(float %vecext.1)
   %vecins.1 = insertelement <4 x float> %vecins, float %2, i32 1
@@ -1103,21 +1143,23 @@ define <4 x float> @int_tanh_4x(ptr %a) {
 ; NOACCELERATE-NEXT:    [[TMP0:%.*]] = load <4 x float>, ptr [[A:%.*]], align 16
 ; NOACCELERATE-NEXT:    [[VECEXT:%.*]] = extractelement <4 x float> [[TMP0]], i32 0
 ; NOACCELERATE-NEXT:    [[TMP1:%.*]] = tail call fast float @llvm.tanh.f32(float [[VECEXT]])
-; NOACCELERATE-NEXT:    [[VECINS:%.*]] = insertelement <4 x float> undef, float [[TMP1]], i32 0
+; NOACCELERATE-NEXT:    [[VECINS:%.*]] = insertelement <4 x float> zeroinitializer, float [[TMP1]], i32 0
 ; NOACCELERATE-NEXT:    [[VECEXT_1:%.*]] = extractelement <4 x float> [[TMP0]], i32 1
 ; NOACCELERATE-NEXT:    [[TMP2:%.*]] = tail call fast float @llvm.tanh.f32(float [[VECEXT_1]])
 ; NOACCELERATE-NEXT:    [[VECINS_1:%.*]] = insertelement <4 x float> [[VECINS]], float [[TMP2]], i32 1
-; NOACCELERATE-NEXT:    [[TMP3:%.*]] = shufflevector <4 x float> [[TMP0]], <4 x float> poison, <2 x i32> <i32 2, i32 3>
-; NOACCELERATE-NEXT:    [[TMP4:%.*]] = call fast <2 x float> @llvm.tanh.v2f32(<2 x float> [[TMP3]])
-; NOACCELERATE-NEXT:    [[TMP5:%.*]] = shufflevector <2 x float> [[TMP4]], <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
-; NOACCELERATE-NEXT:    [[VECINS_31:%.*]] = shufflevector <4 x float> [[VECINS_1]], <4 x float> [[TMP5]], <4 x i32> <i32 0, i32 1, i32 4, i32 5>
+; NOACCELERATE-NEXT:    [[VECEXT_2:%.*]] = extractelement <4 x float> [[TMP0]], i32 2
+; NOACCELERATE-NEXT:    [[TMP3:%.*]] = tail call fast float @llvm.tanh.f32(float [[VECEXT_2]])
+; NOACCELERATE-NEXT:    [[VECINS_2:%.*]] = insertelement <4 x float> [[VECINS_1]], float [[TMP3]], i32 2
+; NOACCELERATE-NEXT:    [[VECEXT_3:%.*]] = extractelement <4 x float> [[TMP0]], i32 3
+; NOACCELERATE-NEXT:    [[TMP4:%.*]] = tail call fast float @llvm.tanh.f32(float [[VECEXT_3]])
+; NOACCELERATE-NEXT:    [[VECINS_31:%.*]] = insertelement <4 x float> [[VECINS_2]], float [[TMP4]], i32 3
 ; NOACCELERATE-NEXT:    ret <4 x float> [[VECINS_31]]
 ;
 entry:
   %0 = load <4 x float>, ptr %a, align 16
   %vecext = extractelement <4 x float> %0, i32 0
   %1 = tail call fast float @llvm.tanh.f32(float %vecext)
-  %vecins = insertelement <4 x float> undef, float %1, i32 0
+  %vecins = insertelement <4 x float> zeroinitializer, float %1, i32 0
   %vecext.1 = extractelement <4 x float> %0, i32 1
   %2 = tail call fast float @llvm.tanh.f32(float %vecext.1)
   %vecins.1 = insertelement <4 x float> %vecins, float %2, i32 1
@@ -1142,7 +1184,7 @@ define <4 x float> @asinh_4x(ptr %a) {
 ; NOACCELERATE-NEXT:    [[TMP0:%.*]] = load <4 x float>, ptr [[A:%.*]], align 16
 ; NOACCELERATE-NEXT:    [[VECEXT:%.*]] = extractelement <4 x float> [[TMP0]], i32 0
 ; NOACCELERATE-NEXT:    [[TMP1:%.*]] = tail call fast float @asinhf(float [[VECEXT]])
-; NOACCELERATE-NEXT:    [[VECINS:%.*]] = insertelement <4 x float> undef, float [[TMP1]], i32 0
+; NOACCELERATE-NEXT:    [[VECINS:%.*]] = insertelement <4 x float> zeroinitializer, float [[TMP1]], i32 0
 ; NOACCELERATE-NEXT:    [[VECEXT_1:%.*]] = extractelement <4 x float> [[TMP0]], i32 1
 ; NOACCELERATE-NEXT:    [[TMP2:%.*]] = tail call fast float @asinhf(float [[VECEXT_1]])
 ; NOACCELERATE-NEXT:    [[VECINS_1:%.*]] = insertelement <4 x float> [[VECINS]], float [[TMP2]], i32 1
@@ -1158,7 +1200,7 @@ entry:
   %0 = load <4 x float>, ptr %a, align 16
   %vecext = extractelement <4 x float> %0, i32 0
   %1 = tail call fast float @asinhf(float %vecext)
-  %vecins = insertelement <4 x float> undef, float %1, i32 0
+  %vecins = insertelement <4 x float> zeroinitializer, float %1, i32 0
   %vecext.1 = extractelement <4 x float> %0, i32 1
   %2 = tail call fast float @asinhf(float %vecext.1)
   %vecins.1 = insertelement <4 x float> %vecins, float %2, i32 1
@@ -1183,7 +1225,7 @@ define <4 x float> @acosh_4x(ptr %a) {
 ; NOACCELERATE-NEXT:    [[TMP0:%.*]] = load <4 x float>, ptr [[A:%.*]], align 16
 ; NOACCELERATE-NEXT:    [[VECEXT:%.*]] = extractelement <4 x float> [[TMP0]], i32 0
 ; NOACCELERATE-NEXT:    [[TMP1:%.*]] = tail call fast float @acoshf(float [[VECEXT]])
-; NOACCELERATE-NEXT:    [[VECINS:%.*]] = insertelement <4 x float> undef, float [[TMP1]], i32 0
+; NOACCELERATE-NEXT:    [[VECINS:%.*]] = insertelement <4 x float> zeroinitializer, float [[TMP1]], i32 0
 ; NOACCELERATE-NEXT:    [[VECEXT_1:%.*]] = extractelement <4 x float> [[TMP0]], i32 1
 ; NOACCELERATE-NEXT:    [[TMP2:%.*]] = tail call fast float @acoshf(float [[VECEXT_1]])
 ; NOACCELERATE-NEXT:    [[VECINS_1:%.*]] = insertelement <4 x float> [[VECINS]], float [[TMP2]], i32 1
@@ -1199,7 +1241,7 @@ entry:
   %0 = load <4 x float>, ptr %a, align 16
   %vecext = extractelement <4 x float> %0, i32 0
   %1 = tail call fast float @acoshf(float %vecext)
-  %vecins = insertelement <4 x float> undef, float %1, i32 0
+  %vecins = insertelement <4 x float> zeroinitializer, float %1, i32 0
   %vecext.1 = extractelement <4 x float> %0, i32 1
   %2 = tail call fast float @acoshf(float %vecext.1)
   %vecins.1 = insertelement <4 x float> %vecins, float %2, i32 1
@@ -1224,7 +1266,7 @@ define <4 x float> @atanh_4x(ptr %a) {
 ; NOACCELERATE-NEXT:    [[TMP0:%.*]] = load <4 x float>, ptr [[A:%.*]], align 16
 ; NOACCELERATE-NEXT:    [[VECEXT:%.*]] = extractelement <4 x float> [[TMP0]], i32 0
 ; NOACCELERATE-NEXT:    [[TMP1:%.*]] = tail call fast float @atanhf(float [[VECEXT]])
-; NOACCELERATE-NEXT:    [[VECINS:%.*]] = insertelement <4 x float> undef, float [[TMP1]], i32 0
+; NOACCELERATE-NEXT:    [[VECINS:%.*]] = insertelement <4 x float> zeroinitializer, float [[TMP1]], i32 0
 ; NOACCELERATE-NEXT:    [[VECEXT_1:%.*]] = extractelement <4 x float> [[TMP0]], i32 1
 ; NOACCELERATE-NEXT:    [[TMP2:%.*]] = tail call fast float @atanhf(float [[VECEXT_1]])
 ; NOACCELERATE-NEXT:    [[VECINS_1:%.*]] = insertelement <4 x float> [[VECINS]], float [[TMP2]], i32 1
@@ -1240,7 +1282,7 @@ entry:
   %0 = load <4 x float>, ptr %a, align 16
   %vecext = extractelement <4 x float> %0, i32 0
   %1 = tail call fast float @atanhf(float %vecext)
-  %vecins = insertelement <4 x float> undef, float %1, i32 0
+  %vecins = insertelement <4 x float> zeroinitializer, float %1, i32 0
   %vecext.1 = extractelement <4 x float> %0, i32 1
   %2 = tail call fast float @atanhf(float %vecext.1)
   %vecins.1 = insertelement <4 x float> %vecins, float %2, i32 1
@@ -1260,7 +1302,7 @@ define <2 x float> @sin_2x(ptr %a) {
 ; CHECK-NEXT:    [[TMP0:%.*]] = load <2 x float>, ptr [[A:%.*]], align 16
 ; CHECK-NEXT:    [[VECEXT:%.*]] = extractelement <2 x float> [[TMP0]], i32 0
 ; CHECK-NEXT:    [[TMP1:%.*]] = tail call fast float @llvm.sin.f32(float [[VECEXT]]) #[[ATTR2:[0-9]+]]
-; CHECK-NEXT:    [[VECINS:%.*]] = insertelement <2 x float> undef, float [[TMP1]], i32 0
+; CHECK-NEXT:    [[VECINS:%.*]] = insertelement <2 x float> zeroinitializer, float [[TMP1]], i32 0
 ; CHECK-NEXT:    [[VECEXT_1:%.*]] = extractelement <2 x float> [[TMP0]], i32 1
 ; CHECK-NEXT:    [[TMP2:%.*]] = tail call fast float @llvm.sin.f32(float [[VECEXT_1]]) #[[ATTR2]]
 ; CHECK-NEXT:    [[VECINS_1:%.*]] = insertelement <2 x float> [[VECINS]], float [[TMP2]], i32 1
@@ -1271,7 +1313,7 @@ define <2 x float> @sin_2x(ptr %a) {
 ; NOACCELERATE-NEXT:    [[TMP0:%.*]] = load <2 x float>, ptr [[A:%.*]], align 16
 ; NOACCELERATE-NEXT:    [[VECEXT:%.*]] = extractelement <2 x float> [[TMP0]], i32 0
 ; NOACCELERATE-NEXT:    [[TMP1:%.*]] = tail call fast float @llvm.sin.f32(float [[VECEXT]])
-; NOACCELERATE-NEXT:    [[VECINS:%.*]] = insertelement <2 x float> undef, float [[TMP1]], i32 0
+; NOACCELERATE-NEXT:    [[VECINS:%.*]] = insertelement <2 x float> zeroinitializer, float [[TMP1]], i32 0
 ; NOACCELERATE-NEXT:    [[VECEXT_1:%.*]] = extractelement <2 x float> [[TMP0]], i32 1
 ; NOACCELERATE-NEXT:    [[TMP2:%.*]] = tail call fast float @llvm.sin.f32(float [[VECEXT_1]])
 ; NOACCELERATE-NEXT:    [[VECINS_1:%.*]] = insertelement <2 x float> [[VECINS]], float [[TMP2]], i32 1
@@ -1281,7 +1323,7 @@ entry:
   %0 = load <2 x float>, ptr %a, align 16
   %vecext = extractelement <2 x float> %0, i32 0
   %1 = tail call fast float @llvm.sin.f32(float %vecext)
-  %vecins = insertelement <2 x float> undef, float %1, i32 0
+  %vecins = insertelement <2 x float> zeroinitializer, float %1, i32 0
   %vecext.1 = extractelement <2 x float> %0, i32 1
   %2 = tail call fast float @llvm.sin.f32(float %vecext.1)
   %vecins.1 = insertelement <2 x float> %vecins, float %2, i32 1
@@ -1304,21 +1346,23 @@ define <4 x float> @int_cos_4x(ptr %a) {
 ; NOACCELERATE-NEXT:    [[TMP0:%.*]] = load <4 x float>, ptr [[A:%.*]], align 16
 ; NOACCELERATE-NEXT:    [[VECEXT:%.*]] = extractelement <4 x float> [[TMP0]], i32 0
 ; NOACCELERATE-NEXT:    [[TMP1:%.*]] = tail call fast float @llvm.cos.f32(float [[VECEXT]])
-; NOACCELERATE-NEXT:    [[VECINS:%.*]] = insertelement <4 x float> undef, float [[TMP1]], i32 0
+; NOACCELERATE-NEXT:    [[VECINS:%.*]] = insertelement <4 x float> zeroinitializer, float [[TMP1]], i32 0
 ; NOACCELERATE-NEXT:    [[VECEXT_1:%.*]] = extractelement <4 x float> [[TMP0]], i32 1
 ; NOACCELERATE-NEXT:    [[TMP2:%.*]] = tail call fast float @llvm.cos.f32(float [[VECEXT_1]])
 ; NOACCELERATE-NEXT:    [[VECINS_1:%.*]] = insertelement <4 x float> [[VECINS]], float [[TMP2]], i32 1
-; NOACCELERATE-NEXT:    [[TMP3:%.*]] = shufflevector <4 x float> [[TMP0]], <4 x float> poison, <2 x i32> <i32 2, i32 3>
-; NOACCELERATE-NEXT:    [[TMP4:%.*]] = call fast <2 x float> @llvm.cos.v2f32(<2 x float> [[TMP3]])
-; NOACCELERATE-NEXT:    [[TMP5:%.*]] = shufflevector <2 x float> [[TMP4]], <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
-; NOACCELERATE-NEXT:    [[VECINS_31:%.*]] = shufflevector <4 x float> [[VECINS_1]], <4 x float> [[TMP5]], <4 x i32> <i32 0, i32 1, i32 4, i32 5>
+; NOACCELERATE-NEXT:    [[VECEXT_2:%.*]] = extractelement <4 x float> [[TMP0]], i32 2
+; NOACCELERATE-NEXT:    [[TMP3:%.*]] = tail call fast float @llvm.cos.f32(float [[VECEXT_2]])
+; NOACCELERATE-NEXT:    [[VECINS_2:%.*]] = insertelement <4 x float> [[VECINS_1]], float [[TMP3]], i32 2
+; NOACCELERATE-NEXT:    [[VECEXT_3:%.*]] = extractelement <4 x float> [[TMP0]], i32 3
+; NOACCELERATE-NEXT:    [[TMP4:%.*]] = tail call fast float @llvm.cos.f32(float [[VECEXT_3]])
+; NOACCELERATE-NEXT:    [[VECINS_31:%.*]] = insertelement <4 x float> [[VECINS_2]], float [[TMP4]], i32 3
 ; NOACCELERATE-NEXT:    ret <4 x float> [[VECINS_31]]
 ;
 entry:
   %0 = load <4 x float>, ptr %a, align 16
   %vecext = extractelement <4 x float> %0, i32 0
   %1 = tail call fast float @llvm.cos.f32(float %vecext)
-  %vecins = insertelement <4 x float> undef, float %1, i32 0
+  %vecins = insertelement <4 x float> zeroinitializer, float %1, i32 0
   %vecext.1 = extractelement <4 x float> %0, i32 1
   %2 = tail call fast float @llvm.cos.f32(float %vecext.1)
   %vecins.1 = insertelement <4 x float> %vecins, float %2, i32 1
@@ -1338,7 +1382,7 @@ define <2 x float> @cos_2x(ptr %a) {
 ; CHECK-NEXT:    [[TMP0:%.*]] = load <2 x float>, ptr [[A:%.*]], align 16
 ; CHECK-NEXT:    [[VECEXT:%.*]] = extractelement <2 x float> [[TMP0]], i32 0
 ; CHECK-NEXT:    [[TMP1:%.*]] = tail call fast float @llvm.cos.f32(float [[VECEXT]]) #[[ATTR3:[0-9]+]]
-; CHECK-NEXT:    [[VECINS:%.*]] = insertelement <2 x float> undef, float [[TMP1]], i32 0
+; CHECK-NEXT:    [[VECINS:%.*]] = insertelement <2 x float> zeroinitializer, float [[TMP1]], i32 0
 ; CHECK-NEXT:    [[VECEXT_1:%.*]] = extractelement <2 x float> [[TMP0]], i32 1
 ; CHECK-NEXT:    [[TMP2:%.*]] = tail call fast float @llvm.cos.f32(float [[VECEXT_1]]) #[[ATTR3]]
 ; CHECK-NEXT:    [[VECINS_1:%.*]] = insertelement <2 x float> [[VECINS]], float [[TMP2]], i32 1
@@ -1349,7 +1393,7 @@ define <2 x float> @cos_2x(ptr %a) {
 ; NOACCELERATE-NEXT:    [[TMP0:%.*]] = load <2 x float>, ptr [[A:%.*]], align 16
 ; NOACCELERATE-NEXT:    [[VECEXT:%.*]] = extractelement <2 x float> [[TMP0]], i32 0
 ; NOACCELERATE-NEXT:    [[TMP1:%.*]] = tail call fast float @llvm.cos.f32(float [[VECEXT]])
-; NOACCELERATE-NEXT:    [[VECINS:%.*]] = insertelement <2 x float> undef, float [[TMP1]], i32 0
+; NOACCELERATE-NEXT:    [[VECINS:%.*]] = insertelement <2 x float> zeroinitializer, float [[TMP1]], i32 0
 ; NOACCELERATE-NEXT:    [[VECEXT_1:%.*]] = extractelement <2 x float> [[TMP0]], i32 1
 ; NOACCELERATE-NEXT:    [[TMP2:%.*]] = tail call fast float @llvm.cos.f32(float [[VECEXT_1]])
 ; NOACCELERATE-NEXT:    [[VECINS_1:%.*]] = insertelement <2 x float> [[VECINS]], float [[TMP2]], i32 1
@@ -1359,7 +1403,7 @@ entry:
   %0 = load <2 x float>, ptr %a, align 16
   %vecext = extractelement <2 x float> %0, i32 0
   %1 = tail call fast float @llvm.cos.f32(float %vecext)
-  %vecins = insertelement <2 x float> undef, float %1, i32 0
+  %vecins = insertelement <2 x float> zeroinitializer, float %1, i32 0
   %vecext.1 = extractelement <2 x float> %0, i32 1
   %2 = tail call fast float @llvm.cos.f32(float %vecext.1)
   %vecins.1 = insertelement <2 x float> %vecins, float %2, i32 1
diff --git a/llvm/test/Transforms/SLPVectorizer/AArch64/div.ll b/llvm/test/Transforms/SLPVectorizer/AArch64/div.ll
index 29bd81998cdb2..bb88edff11634 100644
--- a/llvm/test/Transforms/SLPVectorizer/AArch64/div.ll
+++ b/llvm/test/Transforms/SLPVectorizer/AArch64/div.ll
@@ -607,35 +607,13 @@ define <2 x i32> @sdiv_v2i32_unknown_divisor(<2 x i32> %a, <2 x i32> %x, <2 x i3
 
 ; computes (a/const + x - y) * z
 define <2 x i32> @sdiv_v2i32_const_divisor(<2 x i32> %a, <2 x i32> %x, <2 x i32> %y, <2 x i32> %z)
-; NO-SVE-LABEL: define <2 x i32> @sdiv_v2i32_const_divisor(
-; NO-SVE-SAME: <2 x i32> [[A:%.*]], <2 x i32> [[X:%.*]], <2 x i32> [[Y:%.*]], <2 x i32> [[Z:%.*]]) #[[ATTR0]] {
-; NO-SVE-NEXT:    [[A0:%.*]] = extractelement <2 x i32> [[A]], i64 0
-; NO-SVE-NEXT:    [[A1:%.*]] = extractelement <2 x i32> [[A]], i64 1
-; NO-SVE-NEXT:    [[TMP1:%.*]] = sdiv i32 [[A0]], 2
-; NO-SVE-NEXT:    [[TMP2:%.*]] = sdiv i32 [[A1]], 4
-; NO-SVE-NEXT:    [[X0:%.*]] = extractelement <2 x i32> [[X]], i64 0
-; NO-SVE-NEXT:    [[X1:%.*]] = extractelement <2 x i32> [[X]], i64 1
-; NO-SVE-NEXT:    [[TMP3:%.*]] = add i32 [[TMP1]], [[X0]]
-; NO-SVE-NEXT:    [[TMP4:%.*]] = add i32 [[TMP2]], [[X1]]
-; NO-SVE-NEXT:    [[Y0:%.*]] = extractelement <2 x i32> [[Y]], i64 0
-; NO-SVE-NEXT:    [[Y1:%.*]] = extractelement <2 x i32> [[Y]], i64 1
-; NO-SVE-NEXT:    [[TMP5:%.*]] = sub i32 [[TMP3]], [[Y0]]
-; NO-SVE-NEXT:    [[TMP6:%.*]] = sub i32 [[TMP4]], [[Y1]]
-; NO-SVE-NEXT:    [[Z0:%.*]] = extractelement <2 x i32> [[Z]], i64 0
-; NO-SVE-NEXT:    [[Z1:%.*]] = extractelement <2 x i32> [[Z]], i64 1
-; NO-SVE-NEXT:    [[TMP7:%.*]] = mul i32 [[TMP5]], [[Z0]]
-; NO-SVE-NEXT:    [[TMP8:%.*]] = mul i32 [[TMP6]], [[Z1]]
-; NO-SVE-NEXT:    [[RES0:%.*]] = insertelement <2 x i32> poison, i32 [[TMP7]], i32 0
-; NO-SVE-NEXT:    [[RES1:%.*]] = insertelement <2 x i32> [[RES0]], i32 [[TMP8]], i32 1
-; NO-SVE-NEXT:    ret <2 x i32> [[RES1]]
-;
-; SVE-LABEL: define <2 x i32> @sdiv_v2i32_const_divisor(
-; SVE-SAME: <2 x i32> [[A:%.*]], <2 x i32> [[X:%.*]], <2 x i32> [[Y:%.*]], <2 x i32> [[Z:%.*]]) #[[ATTR0]] {
-; SVE-NEXT:    [[TMP1:%.*]] = sdiv <2 x i32> [[A]], <i32 2, i32 4>
-; SVE-NEXT:    [[TMP2:%.*]] = add <2 x i32> [[TMP1]], [[X]]
-; SVE-NEXT:    [[TMP3:%.*]] = sub <2 x i32> [[TMP2]], [[Y]]
-; SVE-NEXT:    [[TMP4:%.*]] = mul <2 x i32> [[TMP3]], [[Z]]
-; SVE-NEXT:    ret <2 x i32> [[TMP4]]
+; CHECK-LABEL: define <2 x i32> @sdiv_v2i32_const_divisor(
+; CHECK-SAME: <2 x i32> [[A:%.*]], <2 x i32> [[X:%.*]], <2 x i32> [[Y:%.*]], <2 x i32> [[Z:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = sdiv <2 x i32> [[A]], <i32 2, i32 4>
+; CHECK-NEXT:    [[TMP2:%.*]] = add <2 x i32> [[TMP1]], [[X]]
+; CHECK-NEXT:    [[TMP3:%.*]] = sub <2 x i32> [[TMP2]], [[Y]]
+; CHECK-NEXT:    [[TMP4:%.*]] = mul <2 x i32> [[TMP3]], [[Z]]
+; CHECK-NEXT:    ret <2 x i32> [[TMP4]]
 ;
 {
   %a0 = extractelement <2 x i32> %a, i64 0
diff --git a/llvm/test/Transforms/SLPVectorizer/AArch64/reduce-fadd.ll b/llvm/test/Transforms/SLPVectorizer/AArch64/reduce-fadd.ll
index 6576cbe075b74..00a4417ba7aff 100644
--- a/llvm/test/Transforms/SLPVectorizer/AArch64/reduce-fadd.ll
+++ b/llvm/test/Transforms/SLPVectorizer/AArch64/reduce-fadd.ll
@@ -716,29 +716,29 @@ define float @reduce_float_case3(ptr %a) {
 ; CHECK-NEXT:    [[GEP5:%.*]] = getelementptr inbounds float, ptr [[A]], i32 5
 ; CHECK-NEXT:    [[GEP6:%.*]] = getelementptr inbounds float, ptr [[A]], i32 6
 ; CHECK-NEXT:    [[GEP7:%.*]] = getelementptr inbounds float, ptr [[A]], i32 7
-; CHECK-NEXT:    [[LOAD2:%.*]] = load float, ptr [[A]], align 4
-; CHECK-NEXT:    [[LOAD3:%.*]] = load float, ptr [[GEP1]], align 4
-; CHECK-NEXT:    [[LOAD4:%.*]] = load float, ptr [[GEP2]], align 4
-; CHECK-NEXT:    [[LOAD5:%.*]] = load float, ptr [[GEP3]], align 4
-; CHECK-NEXT:    [[LOAD6:%.*]] = load float, ptr [[GEP4]], align 4
-; CHECK-NEXT:    [[LOAD7:%.*]] = load float, ptr [[GEP5]], align 4
-; CHECK-NEXT:    [[LOAD8:%.*]] = load float, ptr [[GEP6]], align 4
-; CHECK-NEXT:    [[LOAD9:%.*]] = load float, ptr [[GEP7]], align 4
+; CHECK-NEXT:    [[LOAD:%.*]] = load float, ptr [[A]], align 4
+; CHECK-NEXT:    [[LOAD1:%.*]] = load float, ptr [[GEP1]], align 4
+; CHECK-NEXT:    [[LOAD2:%.*]] = load float, ptr [[GEP2]], align 4
+; CHECK-NEXT:    [[LOAD3:%.*]] = load float, ptr [[GEP3]], align 4
+; CHECK-NEXT:    [[LOAD4:%.*]] = load float, ptr [[GEP4]], align 4
+; CHECK-NEXT:    [[LOAD5:%.*]] = load float, ptr [[GEP5]], align 4
+; CHECK-NEXT:    [[LOAD6:%.*]] = load float, ptr [[GEP6]], align 4
+; CHECK-NEXT:    [[LOAD7:%.*]] = load float, ptr [[GEP7]], align 4
+; CHECK-NEXT:    [[LOG:%.*]] = call float @llvm.log.f32(float [[LOAD]])
+; CHECK-NEXT:    [[LOG1:%.*]] = call float @llvm.log.f32(float [[LOAD1]])
 ; CHECK-NEXT:    [[LOG2:%.*]] = call float @llvm.log.f32(float [[LOAD2]])
 ; CHECK-NEXT:    [[LOG3:%.*]] = call float @llvm.log.f32(float [[LOAD3]])
 ; CHECK-NEXT:    [[LOG4:%.*]] = call float @llvm.log.f32(float [[LOAD4]])
 ; CHECK-NEXT:    [[LOG5:%.*]] = call float @llvm.log.f32(float [[LOAD5]])
 ; CHECK-NEXT:    [[LOG6:%.*]] = call float @llvm.log.f32(float [[LOAD6]])
 ; CHECK-NEXT:    [[LOG7:%.*]] = call float @llvm.log.f32(float [[LOAD7]])
-; CHECK-NEXT:    [[LOG8:%.*]] = call float @llvm.log.f32(float [[LOAD8]])
-; CHECK-NEXT:    [[LOG9:%.*]] = call float @llvm.log.f32(float [[LOAD9]])
-; CHECK-NEXT:    [[ADD3:%.*]] = fadd float [[LOG2]], [[LOG3]]
+; CHECK-NEXT:    [[ADD1:%.*]] = fadd float [[LOG]], [[LOG1]]
+; CHECK-NEXT:    [[ADD2:%.*]] = fadd float [[ADD1]], [[LOG2]]
+; CHECK-NEXT:    [[ADD3:%.*]] = fadd float [[ADD2]], [[LOG3]]
 ; CHECK-NEXT:    [[ADD4:%.*]] = fadd float [[ADD3]], [[LOG4]]
 ; CHECK-NEXT:    [[ADD5:%.*]] = fadd float [[ADD4]], [[LOG5]]
 ; CHECK-NEXT:    [[ADD6:%.*]] = fadd float [[ADD5]], [[LOG6]]
-; CHECK-NEXT:    [[ADD8:%.*]] = fadd float [[ADD6]], [[LOG7]]
-; CHECK-NEXT:    [[ADD9:%.*]] = fadd float [[ADD8]], [[LOG8]]
-; CHECK-NEXT:    [[ADD7:%.*]] = fadd float [[ADD9]], [[LOG9]]
+; CHECK-NEXT:    [[ADD7:%.*]] = fadd float [[ADD6]], [[LOG7]]
 ; CHECK-NEXT:    ret float [[ADD7]]
 ;
 entry:
diff --git a/llvm/test/Transforms/SLPVectorizer/AMDGPU/min_max.ll b/llvm/test/Transforms/SLPVectorizer/AMDGPU/min_max.ll
index a3be8f5e935c9..46c6c10125b95 100644
--- a/llvm/test/Transforms/SLPVectorizer/AMDGPU/min_max.ll
+++ b/llvm/test/Transforms/SLPVectorizer/AMDGPU/min_max.ll
@@ -358,12 +358,12 @@ define <4 x i16> @uadd_sat_v4i16(<4 x i16> %arg0, <4 x i16> %arg1) {
 ; GFX8-NEXT:    [[ARG1_1:%.*]] = extractelement <4 x i16> [[ARG1]], i64 1
 ; GFX8-NEXT:    [[ADD_0:%.*]] = call i16 @llvm.umin.i16(i16 [[ARG0_0]], i16 [[ARG1_0]])
 ; GFX8-NEXT:    [[ADD_1:%.*]] = call i16 @llvm.umin.i16(i16 [[ARG0_1]], i16 [[ARG1_1]])
-; GFX8-NEXT:    [[TMP3:%.*]] = call <4 x i16> @llvm.umin.v4i16(<4 x i16> [[ARG0]], <4 x i16> [[ARG1]])
-; GFX8-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i16> [[TMP3]], <4 x i16> poison, <2 x i32> <i32 2, i32 3>
+; GFX8-NEXT:    [[TMP0:%.*]] = call <4 x i16> @llvm.umin.v4i16(<4 x i16> [[ARG0]], <4 x i16> [[ARG1]])
+; GFX8-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i16> [[TMP0]], <4 x i16> poison, <2 x i32> <i32 2, i32 3>
 ; GFX8-NEXT:    [[INS_0:%.*]] = insertelement <4 x i16> poison, i16 [[ADD_0]], i64 0
-; GFX8-NEXT:    [[TMP0:%.*]] = insertelement <4 x i16> [[INS_0]], i16 [[ADD_1]], i64 1
+; GFX8-NEXT:    [[INS_1:%.*]] = insertelement <4 x i16> [[INS_0]], i16 [[ADD_1]], i64 1
 ; GFX8-NEXT:    [[TMP2:%.*]] = shufflevector <2 x i16> [[TMP1]], <2 x i16> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
-; GFX8-NEXT:    [[INS_31:%.*]] = shufflevector <4 x i16> [[TMP0]], <4 x i16> [[TMP2]], <4 x i32> <i32 0, i32 1, i32 4, i32 5>
+; GFX8-NEXT:    [[INS_31:%.*]] = shufflevector <4 x i16> [[INS_1]], <4 x i16> [[TMP2]], <4 x i32> <i32 0, i32 1, i32 4, i32 5>
 ; GFX8-NEXT:    ret <4 x i16> [[INS_31]]
 ;
 ; GFX9-LABEL: @uadd_sat_v4i16(
diff --git a/llvm/test/Transforms/SLPVectorizer/ARM/main-gep-with-non-matching-reqs.ll b/llvm/test/Transforms/SLPVectorizer/ARM/main-gep-with-non-matching-reqs.ll
new file mode 100644
index 0000000000000..cf1aec7dde8ac
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/ARM/main-gep-with-non-matching-reqs.ll
@@ -0,0 +1,22 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -S --passes=slp-vectorizer -mtriple=thumb7 -mcpu=swift < %s | FileCheck %s
+
+define i32 @test(ptr  %0) {
+; CHECK-LABEL: define i32 @test(
+; CHECK-SAME: ptr [[TMP0:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT:  [[CONT2:.*:]]
+; CHECK-NEXT:    [[ARRAYIDX518:%.*]] = getelementptr i16, ptr [[TMP0]], i32 0
+; CHECK-NEXT:    [[CMP524_NOT:%.*]] = icmp eq ptr [[ARRAYIDX518]], null
+; CHECK-NEXT:    [[ARRAYIDX482_2:%.*]] = getelementptr [4 x i16], ptr null, i32 0, i32 0
+; CHECK-NEXT:    [[CMP487_NOT_2:%.*]] = icmp eq ptr [[ARRAYIDX482_2]], null
+; CHECK-NEXT:    ret i32 0
+;
+cont2:
+  %arrayidx518 = getelementptr i16, ptr  %0, i32 0
+  %cmp524.not = icmp eq ptr  %arrayidx518, null
+  %cmp487.not.3 = icmp eq ptr  null, null
+  %arrayidx482.2 = getelementptr [4 x i16], ptr  null, i32 0, i32 0
+  %cmp487.not.2 = icmp eq ptr  %arrayidx482.2, null
+  %cmp487.not.1 = icmp eq ptr  null, null
+  ret i32 0
+}
diff --git a/llvm/test/Transforms/SLPVectorizer/NVPTX/vectorizable-intrinsic.ll b/llvm/test/Transforms/SLPVectorizer/NVPTX/vectorizable-intrinsic.ll
index e6ad2adba1759..3a562586d69d8 100644
--- a/llvm/test/Transforms/SLPVectorizer/NVPTX/vectorizable-intrinsic.ll
+++ b/llvm/test/Transforms/SLPVectorizer/NVPTX/vectorizable-intrinsic.ll
@@ -1,19 +1,26 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
 ; RUN: opt < %s -passes=slp-vectorizer -o - -S -slp-threshold=-1000 | FileCheck %s
 
 target datalayout = "e-p:32:32-i64:64-v16:16-v32:32-n16:32:64"
 target triple = "nvptx--nvidiacl"
 
-; Test that CTLZ can be vectorized currently even though the second argument is a scalar
-
+; Vector versions of the intrinsics are scalarized, so keep them scalar
 define <2 x i8> @cltz_test(<2 x i8> %x) #0 {
-; CHECK-LABEL: @cltz_test(
-;      CHECK: [[VEC:%.*]] = call <2 x i8> @llvm.ctlz.v2i8(<2 x i8> %{{.*}}, i1 false)
-; CHECK-NEXT: ret <2 x i8> [[VEC]]
+; CHECK-LABEL: define <2 x i8> @cltz_test(
+; CHECK-SAME: <2 x i8> [[X:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[TMP0:%.*]] = extractelement <2 x i8> [[X]], i32 0
+; CHECK-NEXT:    [[CALL_I:%.*]] = call i8 @llvm.ctlz.i8(i8 [[TMP0]], i1 false)
+; CHECK-NEXT:    [[VECINIT:%.*]] = insertelement <2 x i8> zeroinitializer, i8 [[CALL_I]], i32 0
+; CHECK-NEXT:    [[TMP1:%.*]] = extractelement <2 x i8> [[X]], i32 1
+; CHECK-NEXT:    [[CALL_I4:%.*]] = call i8 @llvm.ctlz.i8(i8 [[TMP1]], i1 false)
+; CHECK-NEXT:    [[VEC:%.*]] = insertelement <2 x i8> [[VECINIT]], i8 [[CALL_I4]], i32 1
+; CHECK-NEXT:    ret <2 x i8> [[VEC]]
 ;
 entry:
   %0 = extractelement <2 x i8> %x, i32 0
   %call.i = call i8 @llvm.ctlz.i8(i8 %0, i1 false)
-  %vecinit = insertelement <2 x i8> undef, i8 %call.i, i32 0
+  %vecinit = insertelement <2 x i8> zeroinitializer, i8 %call.i, i32 0
   %1 = extractelement <2 x i8> %x, i32 1
   %call.i4 = call i8 @llvm.ctlz.i8(i8 %1, i1 false)
   %vecinit2 = insertelement <2 x i8> %vecinit, i8 %call.i4, i32 1
@@ -22,9 +29,16 @@ entry:
 
 
 define <2 x i8> @cltz_test_poison(<2 x i8> %x) #0 {
-; CHECK-LABEL: @cltz_test_poison(
-;      CHECK: [[VEC:%.*]] = call <2 x i8> @llvm.ctlz.v2i8(<2 x i8> %{{.*}}, i1 false)
-; CHECK-NEXT: ret <2 x i8> [[VEC]]
+; CHECK-LABEL: define <2 x i8> @cltz_test_poison(
+; CHECK-SAME: <2 x i8> [[X:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[TMP0:%.*]] = extractelement <2 x i8> [[X]], i32 0
+; CHECK-NEXT:    [[CALL_I:%.*]] = call i8 @llvm.ctlz.i8(i8 [[TMP0]], i1 false)
+; CHECK-NEXT:    [[VECINIT:%.*]] = insertelement <2 x i8> poison, i8 [[CALL_I]], i32 0
+; CHECK-NEXT:    [[TMP1:%.*]] = extractelement <2 x i8> [[X]], i32 1
+; CHECK-NEXT:    [[CALL_I4:%.*]] = call i8 @llvm.ctlz.i8(i8 [[TMP1]], i1 false)
+; CHECK-NEXT:    [[VEC:%.*]] = insertelement <2 x i8> [[VECINIT]], i8 [[CALL_I4]], i32 1
+; CHECK-NEXT:    ret <2 x i8> [[VEC]]
 ;
 entry:
   %0 = extractelement <2 x i8> %x, i32 0
diff --git a/llvm/test/Transforms/SLPVectorizer/RISCV/math-function.ll b/llvm/test/Transforms/SLPVectorizer/RISCV/math-function.ll
index 6fbd05aaedfe5..a9eb786eb436d 100644
--- a/llvm/test/Transforms/SLPVectorizer/RISCV/math-function.ll
+++ b/llvm/test/Transforms/SLPVectorizer/RISCV/math-function.ll
@@ -26,7 +26,7 @@ entry:
   %0 = load <4 x float>, ptr %a, align 16
   %vecext = extractelement <4 x float> %0, i32 0
   %1 = tail call fast float @fabsf(float %vecext)
-  %vecins = insertelement <4 x float> undef, float %1, i32 0
+  %vecins = insertelement <4 x float> poison, float %1, i32 0
   %vecext.1 = extractelement <4 x float> %0, i32 1
   %2 = tail call fast float @fabsf(float %vecext.1)
   %vecins.1 = insertelement <4 x float> %vecins, float %2, i32 1
@@ -60,7 +60,7 @@ entry:
   %0 = load <4 x float>, ptr %a, align 16
   %vecext = extractelement <4 x float> %0, i32 0
   %1 = tail call fast float @llvm.fabs.f32(float %vecext)
-  %vecins = insertelement <4 x float> undef, float %1, i32 0
+  %vecins = insertelement <4 x float> poison, float %1, i32 0
   %vecext.1 = extractelement <4 x float> %0, i32 1
   %2 = tail call fast float @llvm.fabs.f32(float %vecext.1)
   %vecins.1 = insertelement <4 x float> %vecins, float %2, i32 1
@@ -94,7 +94,7 @@ entry:
   %0 = load <4 x float>, ptr %a, align 16
   %vecext = extractelement <4 x float> %0, i32 0
   %1 = tail call fast float @sqrtf(float %vecext)
-  %vecins = insertelement <4 x float> undef, float %1, i32 0
+  %vecins = insertelement <4 x float> poison, float %1, i32 0
   %vecext.1 = extractelement <4 x float> %0, i32 1
   %2 = tail call fast float @sqrtf(float %vecext.1)
   %vecins.1 = insertelement <4 x float> %vecins, float %2, i32 1
@@ -128,7 +128,7 @@ entry:
   %0 = load <4 x float>, ptr %a, align 16
   %vecext = extractelement <4 x float> %0, i32 0
   %1 = tail call fast float @llvm.sqrt.f32(float %vecext)
-  %vecins = insertelement <4 x float> undef, float %1, i32 0
+  %vecins = insertelement <4 x float> poison, float %1, i32 0
   %vecext.1 = extractelement <4 x float> %0, i32 1
   %2 = tail call fast float @llvm.sqrt.f32(float %vecext.1)
   %vecins.1 = insertelement <4 x float> %vecins, float %2, i32 1
@@ -151,7 +151,7 @@ define <4 x float> @exp_4x(ptr %a) {
 ; CHECK-NEXT:    [[TMP0:%.*]] = load <4 x float>, ptr [[A]], align 16
 ; CHECK-NEXT:    [[VECEXT:%.*]] = extractelement <4 x float> [[TMP0]], i32 0
 ; CHECK-NEXT:    [[TMP1:%.*]] = tail call fast float @expf(float [[VECEXT]])
-; CHECK-NEXT:    [[VECINS:%.*]] = insertelement <4 x float> undef, float [[TMP1]], i32 0
+; CHECK-NEXT:    [[VECINS:%.*]] = insertelement <4 x float> poison, float [[TMP1]], i32 0
 ; CHECK-NEXT:    [[VECEXT_1:%.*]] = extractelement <4 x float> [[TMP0]], i32 1
 ; CHECK-NEXT:    [[TMP2:%.*]] = tail call fast float @expf(float [[VECEXT_1]])
 ; CHECK-NEXT:    [[VECINS_1:%.*]] = insertelement <4 x float> [[VECINS]], float [[TMP2]], i32 1
@@ -169,7 +169,7 @@ define <4 x float> @exp_4x(ptr %a) {
 ; DEFAULT-NEXT:    [[TMP0:%.*]] = load <4 x float>, ptr [[A]], align 16
 ; DEFAULT-NEXT:    [[VECEXT:%.*]] = extractelement <4 x float> [[TMP0]], i32 0
 ; DEFAULT-NEXT:    [[TMP1:%.*]] = tail call fast float @expf(float [[VECEXT]])
-; DEFAULT-NEXT:    [[VECINS:%.*]] = insertelement <4 x float> undef, float [[TMP1]], i32 0
+; DEFAULT-NEXT:    [[VECINS:%.*]] = insertelement <4 x float> poison, float [[TMP1]], i32 0
 ; DEFAULT-NEXT:    [[VECEXT_1:%.*]] = extractelement <4 x float> [[TMP0]], i32 1
 ; DEFAULT-NEXT:    [[TMP2:%.*]] = tail call fast float @expf(float [[VECEXT_1]])
 ; DEFAULT-NEXT:    [[VECINS_1:%.*]] = insertelement <4 x float> [[VECINS]], float [[TMP2]], i32 1
@@ -185,7 +185,7 @@ entry:
   %0 = load <4 x float>, ptr %a, align 16
   %vecext = extractelement <4 x float> %0, i32 0
   %1 = tail call fast float @expf(float %vecext)
-  %vecins = insertelement <4 x float> undef, float %1, i32 0
+  %vecins = insertelement <4 x float> poison, float %1, i32 0
   %vecext.1 = extractelement <4 x float> %0, i32 1
   %2 = tail call fast float @expf(float %vecext.1)
   %vecins.1 = insertelement <4 x float> %vecins, float %2, i32 1
@@ -208,7 +208,7 @@ define <4 x float> @int_exp_4x(ptr %a) {
 ; CHECK-NEXT:    [[TMP0:%.*]] = load <4 x float>, ptr [[A]], align 16
 ; CHECK-NEXT:    [[VECEXT:%.*]] = extractelement <4 x float> [[TMP0]], i32 0
 ; CHECK-NEXT:    [[TMP1:%.*]] = tail call fast float @llvm.exp.f32(float [[VECEXT]])
-; CHECK-NEXT:    [[VECINS:%.*]] = insertelement <4 x float> undef, float [[TMP1]], i32 0
+; CHECK-NEXT:    [[VECINS:%.*]] = insertelement <4 x float> poison, float [[TMP1]], i32 0
 ; CHECK-NEXT:    [[VECEXT_1:%.*]] = extractelement <4 x float> [[TMP0]], i32 1
 ; CHECK-NEXT:    [[TMP2:%.*]] = tail call fast float @llvm.exp.f32(float [[VECEXT_1]])
 ; CHECK-NEXT:    [[VECINS_1:%.*]] = insertelement <4 x float> [[VECINS]], float [[TMP2]], i32 1
@@ -226,7 +226,7 @@ define <4 x float> @int_exp_4x(ptr %a) {
 ; DEFAULT-NEXT:    [[TMP0:%.*]] = load <4 x float>, ptr [[A]], align 16
 ; DEFAULT-NEXT:    [[VECEXT:%.*]] = extractelement <4 x float> [[TMP0]], i32 0
 ; DEFAULT-NEXT:    [[TMP1:%.*]] = tail call fast float @llvm.exp.f32(float [[VECEXT]])
-; DEFAULT-NEXT:    [[VECINS:%.*]] = insertelement <4 x float> undef, float [[TMP1]], i32 0
+; DEFAULT-NEXT:    [[VECINS:%.*]] = insertelement <4 x float> poison, float [[TMP1]], i32 0
 ; DEFAULT-NEXT:    [[VECEXT_1:%.*]] = extractelement <4 x float> [[TMP0]], i32 1
 ; DEFAULT-NEXT:    [[TMP2:%.*]] = tail call fast float @llvm.exp.f32(float [[VECEXT_1]])
 ; DEFAULT-NEXT:    [[VECINS_1:%.*]] = insertelement <4 x float> [[VECINS]], float [[TMP2]], i32 1
@@ -242,7 +242,7 @@ entry:
   %0 = load <4 x float>, ptr %a, align 16
   %vecext = extractelement <4 x float> %0, i32 0
   %1 = tail call fast float @llvm.exp.f32(float %vecext)
-  %vecins = insertelement <4 x float> undef, float %1, i32 0
+  %vecins = insertelement <4 x float> poison, float %1, i32 0
   %vecext.1 = extractelement <4 x float> %0, i32 1
   %2 = tail call fast float @llvm.exp.f32(float %vecext.1)
   %vecins.1 = insertelement <4 x float> %vecins, float %2, i32 1
@@ -265,7 +265,7 @@ define <4 x float> @log_4x(ptr %a) {
 ; CHECK-NEXT:    [[TMP0:%.*]] = load <4 x float>, ptr [[A]], align 16
 ; CHECK-NEXT:    [[VECEXT:%.*]] = extractelement <4 x float> [[TMP0]], i32 0
 ; CHECK-NEXT:    [[TMP1:%.*]] = tail call fast float @logf(float [[VECEXT]])
-; CHECK-NEXT:    [[VECINS:%.*]] = insertelement <4 x float> undef, float [[TMP1]], i32 0
+; CHECK-NEXT:    [[VECINS:%.*]] = insertelement <4 x float> poison, float [[TMP1]], i32 0
 ; CHECK-NEXT:    [[VECEXT_1:%.*]] = extractelement <4 x float> [[TMP0]], i32 1
 ; CHECK-NEXT:    [[TMP2:%.*]] = tail call fast float @logf(float [[VECEXT_1]])
 ; CHECK-NEXT:    [[VECINS_1:%.*]] = insertelement <4 x float> [[VECINS]], float [[TMP2]], i32 1
@@ -283,7 +283,7 @@ define <4 x float> @log_4x(ptr %a) {
 ; DEFAULT-NEXT:    [[TMP0:%.*]] = load <4 x float>, ptr [[A]], align 16
 ; DEFAULT-NEXT:    [[VECEXT:%.*]] = extractelement <4 x float> [[TMP0]], i32 0
 ; DEFAULT-NEXT:    [[TMP1:%.*]] = tail call fast float @logf(float [[VECEXT]])
-; DEFAULT-NEXT:    [[VECINS:%.*]] = insertelement <4 x float> undef, float [[TMP1]], i32 0
+; DEFAULT-NEXT:    [[VECINS:%.*]] = insertelement <4 x float> poison, float [[TMP1]], i32 0
 ; DEFAULT-NEXT:    [[VECEXT_1:%.*]] = extractelement <4 x float> [[TMP0]], i32 1
 ; DEFAULT-NEXT:    [[TMP2:%.*]] = tail call fast float @logf(float [[VECEXT_1]])
 ; DEFAULT-NEXT:    [[VECINS_1:%.*]] = insertelement <4 x float> [[VECINS]], float [[TMP2]], i32 1
@@ -299,7 +299,7 @@ entry:
   %0 = load <4 x float>, ptr %a, align 16
   %vecext = extractelement <4 x float> %0, i32 0
   %1 = tail call fast float @logf(float %vecext)
-  %vecins = insertelement <4 x float> undef, float %1, i32 0
+  %vecins = insertelement <4 x float> poison, float %1, i32 0
   %vecext.1 = extractelement <4 x float> %0, i32 1
   %2 = tail call fast float @logf(float %vecext.1)
   %vecins.1 = insertelement <4 x float> %vecins, float %2, i32 1
@@ -322,7 +322,7 @@ define <4 x float> @int_log_4x(ptr %a) {
 ; CHECK-NEXT:    [[TMP0:%.*]] = load <4 x float>, ptr [[A]], align 16
 ; CHECK-NEXT:    [[VECEXT:%.*]] = extractelement <4 x float> [[TMP0]], i32 0
 ; CHECK-NEXT:    [[TMP1:%.*]] = tail call fast float @llvm.log.f32(float [[VECEXT]])
-; CHECK-NEXT:    [[VECINS:%.*]] = insertelement <4 x float> undef, float [[TMP1]], i32 0
+; CHECK-NEXT:    [[VECINS:%.*]] = insertelement <4 x float> poison, float [[TMP1]], i32 0
 ; CHECK-NEXT:    [[VECEXT_1:%.*]] = extractelement <4 x float> [[TMP0]], i32 1
 ; CHECK-NEXT:    [[TMP2:%.*]] = tail call fast float @llvm.log.f32(float [[VECEXT_1]])
 ; CHECK-NEXT:    [[VECINS_1:%.*]] = insertelement <4 x float> [[VECINS]], float [[TMP2]], i32 1
@@ -340,7 +340,7 @@ define <4 x float> @int_log_4x(ptr %a) {
 ; DEFAULT-NEXT:    [[TMP0:%.*]] = load <4 x float>, ptr [[A]], align 16
 ; DEFAULT-NEXT:    [[VECEXT:%.*]] = extractelement <4 x float> [[TMP0]], i32 0
 ; DEFAULT-NEXT:    [[TMP1:%.*]] = tail call fast float @llvm.log.f32(float [[VECEXT]])
-; DEFAULT-NEXT:    [[VECINS:%.*]] = insertelement <4 x float> undef, float [[TMP1]], i32 0
+; DEFAULT-NEXT:    [[VECINS:%.*]] = insertelement <4 x float> poison, float [[TMP1]], i32 0
 ; DEFAULT-NEXT:    [[VECEXT_1:%.*]] = extractelement <4 x float> [[TMP0]], i32 1
 ; DEFAULT-NEXT:    [[TMP2:%.*]] = tail call fast float @llvm.log.f32(float [[VECEXT_1]])
 ; DEFAULT-NEXT:    [[VECINS_1:%.*]] = insertelement <4 x float> [[VECINS]], float [[TMP2]], i32 1
@@ -356,7 +356,7 @@ entry:
   %0 = load <4 x float>, ptr %a, align 16
   %vecext = extractelement <4 x float> %0, i32 0
   %1 = tail call fast float @llvm.log.f32(float %vecext)
-  %vecins = insertelement <4 x float> undef, float %1, i32 0
+  %vecins = insertelement <4 x float> poison, float %1, i32 0
   %vecext.1 = extractelement <4 x float> %0, i32 1
   %2 = tail call fast float @llvm.log.f32(float %vecext.1)
   %vecins.1 = insertelement <4 x float> %vecins, float %2, i32 1
@@ -379,7 +379,7 @@ define <4 x float> @sin_4x(ptr %a) {
 ; CHECK-NEXT:    [[TMP0:%.*]] = load <4 x float>, ptr [[A]], align 16
 ; CHECK-NEXT:    [[VECEXT:%.*]] = extractelement <4 x float> [[TMP0]], i32 0
 ; CHECK-NEXT:    [[TMP1:%.*]] = tail call fast float @sinf(float [[VECEXT]])
-; CHECK-NEXT:    [[VECINS:%.*]] = insertelement <4 x float> undef, float [[TMP1]], i32 0
+; CHECK-NEXT:    [[VECINS:%.*]] = insertelement <4 x float> poison, float [[TMP1]], i32 0
 ; CHECK-NEXT:    [[VECEXT_1:%.*]] = extractelement <4 x float> [[TMP0]], i32 1
 ; CHECK-NEXT:    [[TMP2:%.*]] = tail call fast float @sinf(float [[VECEXT_1]])
 ; CHECK-NEXT:    [[VECINS_1:%.*]] = insertelement <4 x float> [[VECINS]], float [[TMP2]], i32 1
@@ -397,7 +397,7 @@ define <4 x float> @sin_4x(ptr %a) {
 ; DEFAULT-NEXT:    [[TMP0:%.*]] = load <4 x float>, ptr [[A]], align 16
 ; DEFAULT-NEXT:    [[VECEXT:%.*]] = extractelement <4 x float> [[TMP0]], i32 0
 ; DEFAULT-NEXT:    [[TMP1:%.*]] = tail call fast float @sinf(float [[VECEXT]])
-; DEFAULT-NEXT:    [[VECINS:%.*]] = insertelement <4 x float> undef, float [[TMP1]], i32 0
+; DEFAULT-NEXT:    [[VECINS:%.*]] = insertelement <4 x float> poison, float [[TMP1]], i32 0
 ; DEFAULT-NEXT:    [[VECEXT_1:%.*]] = extractelement <4 x float> [[TMP0]], i32 1
 ; DEFAULT-NEXT:    [[TMP2:%.*]] = tail call fast float @sinf(float [[VECEXT_1]])
 ; DEFAULT-NEXT:    [[VECINS_1:%.*]] = insertelement <4 x float> [[VECINS]], float [[TMP2]], i32 1
@@ -413,7 +413,7 @@ entry:
   %0 = load <4 x float>, ptr %a, align 16
   %vecext = extractelement <4 x float> %0, i32 0
   %1 = tail call fast float @sinf(float %vecext)
-  %vecins = insertelement <4 x float> undef, float %1, i32 0
+  %vecins = insertelement <4 x float> poison, float %1, i32 0
   %vecext.1 = extractelement <4 x float> %0, i32 1
   %2 = tail call fast float @sinf(float %vecext.1)
   %vecins.1 = insertelement <4 x float> %vecins, float %2, i32 1
@@ -436,7 +436,7 @@ define <4 x float> @int_sin_4x(ptr %a) {
 ; CHECK-NEXT:    [[TMP0:%.*]] = load <4 x float>, ptr [[A]], align 16
 ; CHECK-NEXT:    [[VECEXT:%.*]] = extractelement <4 x float> [[TMP0]], i32 0
 ; CHECK-NEXT:    [[TMP1:%.*]] = tail call fast float @llvm.sin.f32(float [[VECEXT]])
-; CHECK-NEXT:    [[VECINS:%.*]] = insertelement <4 x float> undef, float [[TMP1]], i32 0
+; CHECK-NEXT:    [[VECINS:%.*]] = insertelement <4 x float> poison, float [[TMP1]], i32 0
 ; CHECK-NEXT:    [[VECEXT_1:%.*]] = extractelement <4 x float> [[TMP0]], i32 1
 ; CHECK-NEXT:    [[TMP2:%.*]] = tail call fast float @llvm.sin.f32(float [[VECEXT_1]])
 ; CHECK-NEXT:    [[VECINS_1:%.*]] = insertelement <4 x float> [[VECINS]], float [[TMP2]], i32 1
@@ -454,7 +454,7 @@ define <4 x float> @int_sin_4x(ptr %a) {
 ; DEFAULT-NEXT:    [[TMP0:%.*]] = load <4 x float>, ptr [[A]], align 16
 ; DEFAULT-NEXT:    [[VECEXT:%.*]] = extractelement <4 x float> [[TMP0]], i32 0
 ; DEFAULT-NEXT:    [[TMP1:%.*]] = tail call fast float @llvm.sin.f32(float [[VECEXT]])
-; DEFAULT-NEXT:    [[VECINS:%.*]] = insertelement <4 x float> undef, float [[TMP1]], i32 0
+; DEFAULT-NEXT:    [[VECINS:%.*]] = insertelement <4 x float> poison, float [[TMP1]], i32 0
 ; DEFAULT-NEXT:    [[VECEXT_1:%.*]] = extractelement <4 x float> [[TMP0]], i32 1
 ; DEFAULT-NEXT:    [[TMP2:%.*]] = tail call fast float @llvm.sin.f32(float [[VECEXT_1]])
 ; DEFAULT-NEXT:    [[VECINS_1:%.*]] = insertelement <4 x float> [[VECINS]], float [[TMP2]], i32 1
@@ -470,7 +470,7 @@ entry:
   %0 = load <4 x float>, ptr %a, align 16
   %vecext = extractelement <4 x float> %0, i32 0
   %1 = tail call fast float @llvm.sin.f32(float %vecext)
-  %vecins = insertelement <4 x float> undef, float %1, i32 0
+  %vecins = insertelement <4 x float> poison, float %1, i32 0
   %vecext.1 = extractelement <4 x float> %0, i32 1
   %2 = tail call fast float @llvm.sin.f32(float %vecext.1)
   %vecins.1 = insertelement <4 x float> %vecins, float %2, i32 1
@@ -493,7 +493,7 @@ define <4 x float> @asin_4x(ptr %a) {
 ; CHECK-NEXT:    [[TMP0:%.*]] = load <4 x float>, ptr [[A]], align 16
 ; CHECK-NEXT:    [[VECEXT:%.*]] = extractelement <4 x float> [[TMP0]], i32 0
 ; CHECK-NEXT:    [[TMP1:%.*]] = tail call fast float @asinf(float [[VECEXT]])
-; CHECK-NEXT:    [[VECINS:%.*]] = insertelement <4 x float> undef, float [[TMP1]], i32 0
+; CHECK-NEXT:    [[VECINS:%.*]] = insertelement <4 x float> poison, float [[TMP1]], i32 0
 ; CHECK-NEXT:    [[VECEXT_1:%.*]] = extractelement <4 x float> [[TMP0]], i32 1
 ; CHECK-NEXT:    [[TMP2:%.*]] = tail call fast float @asinf(float [[VECEXT_1]])
 ; CHECK-NEXT:    [[VECINS_1:%.*]] = insertelement <4 x float> [[VECINS]], float [[TMP2]], i32 1
@@ -511,7 +511,7 @@ define <4 x float> @asin_4x(ptr %a) {
 ; DEFAULT-NEXT:    [[TMP0:%.*]] = load <4 x float>, ptr [[A]], align 16
 ; DEFAULT-NEXT:    [[VECEXT:%.*]] = extractelement <4 x float> [[TMP0]], i32 0
 ; DEFAULT-NEXT:    [[TMP1:%.*]] = tail call fast float @asinf(float [[VECEXT]])
-; DEFAULT-NEXT:    [[VECINS:%.*]] = insertelement <4 x float> undef, float [[TMP1]], i32 0
+; DEFAULT-NEXT:    [[VECINS:%.*]] = insertelement <4 x float> poison, float [[TMP1]], i32 0
 ; DEFAULT-NEXT:    [[VECEXT_1:%.*]] = extractelement <4 x float> [[TMP0]], i32 1
 ; DEFAULT-NEXT:    [[TMP2:%.*]] = tail call fast float @asinf(float [[VECEXT_1]])
 ; DEFAULT-NEXT:    [[VECINS_1:%.*]] = insertelement <4 x float> [[VECINS]], float [[TMP2]], i32 1
@@ -527,7 +527,7 @@ entry:
   %0 = load <4 x float>, ptr %a, align 16
   %vecext = extractelement <4 x float> %0, i32 0
   %1 = tail call fast float @asinf(float %vecext)
-  %vecins = insertelement <4 x float> undef, float %1, i32 0
+  %vecins = insertelement <4 x float> poison, float %1, i32 0
   %vecext.1 = extractelement <4 x float> %0, i32 1
   %2 = tail call fast float @asinf(float %vecext.1)
   %vecins.1 = insertelement <4 x float> %vecins, float %2, i32 1
@@ -550,7 +550,7 @@ define <4 x float> @int_asin_4x(ptr %a) {
 ; CHECK-NEXT:    [[TMP0:%.*]] = load <4 x float>, ptr [[A]], align 16
 ; CHECK-NEXT:    [[VECEXT:%.*]] = extractelement <4 x float> [[TMP0]], i32 0
 ; CHECK-NEXT:    [[TMP1:%.*]] = tail call fast float @llvm.asin.f32(float [[VECEXT]])
-; CHECK-NEXT:    [[VECINS:%.*]] = insertelement <4 x float> undef, float [[TMP1]], i32 0
+; CHECK-NEXT:    [[VECINS:%.*]] = insertelement <4 x float> poison, float [[TMP1]], i32 0
 ; CHECK-NEXT:    [[VECEXT_1:%.*]] = extractelement <4 x float> [[TMP0]], i32 1
 ; CHECK-NEXT:    [[TMP2:%.*]] = tail call fast float @llvm.asin.f32(float [[VECEXT_1]])
 ; CHECK-NEXT:    [[VECINS_1:%.*]] = insertelement <4 x float> [[VECINS]], float [[TMP2]], i32 1
@@ -568,7 +568,7 @@ define <4 x float> @int_asin_4x(ptr %a) {
 ; DEFAULT-NEXT:    [[TMP0:%.*]] = load <4 x float>, ptr [[A]], align 16
 ; DEFAULT-NEXT:    [[VECEXT:%.*]] = extractelement <4 x float> [[TMP0]], i32 0
 ; DEFAULT-NEXT:    [[TMP1:%.*]] = tail call fast float @llvm.asin.f32(float [[VECEXT]])
-; DEFAULT-NEXT:    [[VECINS:%.*]] = insertelement <4 x float> undef, float [[TMP1]], i32 0
+; DEFAULT-NEXT:    [[VECINS:%.*]] = insertelement <4 x float> poison, float [[TMP1]], i32 0
 ; DEFAULT-NEXT:    [[VECEXT_1:%.*]] = extractelement <4 x float> [[TMP0]], i32 1
 ; DEFAULT-NEXT:    [[TMP2:%.*]] = tail call fast float @llvm.asin.f32(float [[VECEXT_1]])
 ; DEFAULT-NEXT:    [[VECINS_1:%.*]] = insertelement <4 x float> [[VECINS]], float [[TMP2]], i32 1
@@ -584,7 +584,7 @@ entry:
   %0 = load <4 x float>, ptr %a, align 16
   %vecext = extractelement <4 x float> %0, i32 0
   %1 = tail call fast float @llvm.asin.f32(float %vecext)
-  %vecins = insertelement <4 x float> undef, float %1, i32 0
+  %vecins = insertelement <4 x float> poison, float %1, i32 0
   %vecext.1 = extractelement <4 x float> %0, i32 1
   %2 = tail call fast float @llvm.asin.f32(float %vecext.1)
   %vecins.1 = insertelement <4 x float> %vecins, float %2, i32 1
@@ -607,7 +607,7 @@ define <4 x float> @cos_4x(ptr %a) {
 ; CHECK-NEXT:    [[TMP0:%.*]] = load <4 x float>, ptr [[A]], align 16
 ; CHECK-NEXT:    [[VECEXT:%.*]] = extractelement <4 x float> [[TMP0]], i32 0
 ; CHECK-NEXT:    [[TMP1:%.*]] = tail call fast float @cosf(float [[VECEXT]])
-; CHECK-NEXT:    [[VECINS:%.*]] = insertelement <4 x float> undef, float [[TMP1]], i32 0
+; CHECK-NEXT:    [[VECINS:%.*]] = insertelement <4 x float> poison, float [[TMP1]], i32 0
 ; CHECK-NEXT:    [[VECEXT_1:%.*]] = extractelement <4 x float> [[TMP0]], i32 1
 ; CHECK-NEXT:    [[TMP2:%.*]] = tail call fast float @cosf(float [[VECEXT_1]])
 ; CHECK-NEXT:    [[VECINS_1:%.*]] = insertelement <4 x float> [[VECINS]], float [[TMP2]], i32 1
@@ -625,7 +625,7 @@ define <4 x float> @cos_4x(ptr %a) {
 ; DEFAULT-NEXT:    [[TMP0:%.*]] = load <4 x float>, ptr [[A]], align 16
 ; DEFAULT-NEXT:    [[VECEXT:%.*]] = extractelement <4 x float> [[TMP0]], i32 0
 ; DEFAULT-NEXT:    [[TMP1:%.*]] = tail call fast float @cosf(float [[VECEXT]])
-; DEFAULT-NEXT:    [[VECINS:%.*]] = insertelement <4 x float> undef, float [[TMP1]], i32 0
+; DEFAULT-NEXT:    [[VECINS:%.*]] = insertelement <4 x float> poison, float [[TMP1]], i32 0
 ; DEFAULT-NEXT:    [[VECEXT_1:%.*]] = extractelement <4 x float> [[TMP0]], i32 1
 ; DEFAULT-NEXT:    [[TMP2:%.*]] = tail call fast float @cosf(float [[VECEXT_1]])
 ; DEFAULT-NEXT:    [[VECINS_1:%.*]] = insertelement <4 x float> [[VECINS]], float [[TMP2]], i32 1
@@ -641,7 +641,7 @@ entry:
   %0 = load <4 x float>, ptr %a, align 16
   %vecext = extractelement <4 x float> %0, i32 0
   %1 = tail call fast float @cosf(float %vecext)
-  %vecins = insertelement <4 x float> undef, float %1, i32 0
+  %vecins = insertelement <4 x float> poison, float %1, i32 0
   %vecext.1 = extractelement <4 x float> %0, i32 1
   %2 = tail call fast float @cosf(float %vecext.1)
   %vecins.1 = insertelement <4 x float> %vecins, float %2, i32 1
@@ -664,7 +664,7 @@ define <4 x float> @int_cos_4x(ptr %a) {
 ; CHECK-NEXT:    [[TMP0:%.*]] = load <4 x float>, ptr [[A]], align 16
 ; CHECK-NEXT:    [[VECEXT:%.*]] = extractelement <4 x float> [[TMP0]], i32 0
 ; CHECK-NEXT:    [[TMP1:%.*]] = tail call fast float @llvm.cos.f32(float [[VECEXT]])
-; CHECK-NEXT:    [[VECINS:%.*]] = insertelement <4 x float> undef, float [[TMP1]], i32 0
+; CHECK-NEXT:    [[VECINS:%.*]] = insertelement <4 x float> poison, float [[TMP1]], i32 0
 ; CHECK-NEXT:    [[VECEXT_1:%.*]] = extractelement <4 x float> [[TMP0]], i32 1
 ; CHECK-NEXT:    [[TMP2:%.*]] = tail call fast float @llvm.cos.f32(float [[VECEXT_1]])
 ; CHECK-NEXT:    [[VECINS_1:%.*]] = insertelement <4 x float> [[VECINS]], float [[TMP2]], i32 1
@@ -682,7 +682,7 @@ define <4 x float> @int_cos_4x(ptr %a) {
 ; DEFAULT-NEXT:    [[TMP0:%.*]] = load <4 x float>, ptr [[A]], align 16
 ; DEFAULT-NEXT:    [[VECEXT:%.*]] = extractelement <4 x float> [[TMP0]], i32 0
 ; DEFAULT-NEXT:    [[TMP1:%.*]] = tail call fast float @llvm.cos.f32(float [[VECEXT]])
-; DEFAULT-NEXT:    [[VECINS:%.*]] = insertelement <4 x float> undef, float [[TMP1]], i32 0
+; DEFAULT-NEXT:    [[VECINS:%.*]] = insertelement <4 x float> poison, float [[TMP1]], i32 0
 ; DEFAULT-NEXT:    [[VECEXT_1:%.*]] = extractelement <4 x float> [[TMP0]], i32 1
 ; DEFAULT-NEXT:    [[TMP2:%.*]] = tail call fast float @llvm.cos.f32(float [[VECEXT_1]])
 ; DEFAULT-NEXT:    [[VECINS_1:%.*]] = insertelement <4 x float> [[VECINS]], float [[TMP2]], i32 1
@@ -698,7 +698,7 @@ entry:
   %0 = load <4 x float>, ptr %a, align 16
   %vecext = extractelement <4 x float> %0, i32 0
   %1 = tail call fast float @llvm.cos.f32(float %vecext)
-  %vecins = insertelement <4 x float> undef, float %1, i32 0
+  %vecins = insertelement <4 x float> poison, float %1, i32 0
   %vecext.1 = extractelement <4 x float> %0, i32 1
   %2 = tail call fast float @llvm.cos.f32(float %vecext.1)
   %vecins.1 = insertelement <4 x float> %vecins, float %2, i32 1
@@ -721,7 +721,7 @@ define <4 x float> @acos_4x(ptr %a) {
 ; CHECK-NEXT:    [[TMP0:%.*]] = load <4 x float>, ptr [[A]], align 16
 ; CHECK-NEXT:    [[VECEXT:%.*]] = extractelement <4 x float> [[TMP0]], i32 0
 ; CHECK-NEXT:    [[TMP1:%.*]] = tail call fast float @acosf(float [[VECEXT]])
-; CHECK-NEXT:    [[VECINS:%.*]] = insertelement <4 x float> undef, float [[TMP1]], i32 0
+; CHECK-NEXT:    [[VECINS:%.*]] = insertelement <4 x float> poison, float [[TMP1]], i32 0
 ; CHECK-NEXT:    [[VECEXT_1:%.*]] = extractelement <4 x float> [[TMP0]], i32 1
 ; CHECK-NEXT:    [[TMP2:%.*]] = tail call fast float @acosf(float [[VECEXT_1]])
 ; CHECK-NEXT:    [[VECINS_1:%.*]] = insertelement <4 x float> [[VECINS]], float [[TMP2]], i32 1
@@ -739,7 +739,7 @@ define <4 x float> @acos_4x(ptr %a) {
 ; DEFAULT-NEXT:    [[TMP0:%.*]] = load <4 x float>, ptr [[A]], align 16
 ; DEFAULT-NEXT:    [[VECEXT:%.*]] = extractelement <4 x float> [[TMP0]], i32 0
 ; DEFAULT-NEXT:    [[TMP1:%.*]] = tail call fast float @acosf(float [[VECEXT]])
-; DEFAULT-NEXT:    [[VECINS:%.*]] = insertelement <4 x float> undef, float [[TMP1]], i32 0
+; DEFAULT-NEXT:    [[VECINS:%.*]] = insertelement <4 x float> poison, float [[TMP1]], i32 0
 ; DEFAULT-NEXT:    [[VECEXT_1:%.*]] = extractelement <4 x float> [[TMP0]], i32 1
 ; DEFAULT-NEXT:    [[TMP2:%.*]] = tail call fast float @acosf(float [[VECEXT_1]])
 ; DEFAULT-NEXT:    [[VECINS_1:%.*]] = insertelement <4 x float> [[VECINS]], float [[TMP2]], i32 1
@@ -755,7 +755,7 @@ entry:
   %0 = load <4 x float>, ptr %a, align 16
   %vecext = extractelement <4 x float> %0, i32 0
   %1 = tail call fast float @acosf(float %vecext)
-  %vecins = insertelement <4 x float> undef, float %1, i32 0
+  %vecins = insertelement <4 x float> poison, float %1, i32 0
   %vecext.1 = extractelement <4 x float> %0, i32 1
   %2 = tail call fast float @acosf(float %vecext.1)
   %vecins.1 = insertelement <4 x float> %vecins, float %2, i32 1
@@ -778,7 +778,7 @@ define <4 x float> @int_acos_4x(ptr %a) {
 ; CHECK-NEXT:    [[TMP0:%.*]] = load <4 x float>, ptr [[A]], align 16
 ; CHECK-NEXT:    [[VECEXT:%.*]] = extractelement <4 x float> [[TMP0]], i32 0
 ; CHECK-NEXT:    [[TMP1:%.*]] = tail call fast float @llvm.acos.f32(float [[VECEXT]])
-; CHECK-NEXT:    [[VECINS:%.*]] = insertelement <4 x float> undef, float [[TMP1]], i32 0
+; CHECK-NEXT:    [[VECINS:%.*]] = insertelement <4 x float> poison, float [[TMP1]], i32 0
 ; CHECK-NEXT:    [[VECEXT_1:%.*]] = extractelement <4 x float> [[TMP0]], i32 1
 ; CHECK-NEXT:    [[TMP2:%.*]] = tail call fast float @llvm.acos.f32(float [[VECEXT_1]])
 ; CHECK-NEXT:    [[VECINS_1:%.*]] = insertelement <4 x float> [[VECINS]], float [[TMP2]], i32 1
@@ -796,7 +796,7 @@ define <4 x float> @int_acos_4x(ptr %a) {
 ; DEFAULT-NEXT:    [[TMP0:%.*]] = load <4 x float>, ptr [[A]], align 16
 ; DEFAULT-NEXT:    [[VECEXT:%.*]] = extractelement <4 x float> [[TMP0]], i32 0
 ; DEFAULT-NEXT:    [[TMP1:%.*]] = tail call fast float @llvm.acos.f32(float [[VECEXT]])
-; DEFAULT-NEXT:    [[VECINS:%.*]] = insertelement <4 x float> undef, float [[TMP1]], i32 0
+; DEFAULT-NEXT:    [[VECINS:%.*]] = insertelement <4 x float> poison, float [[TMP1]], i32 0
 ; DEFAULT-NEXT:    [[VECEXT_1:%.*]] = extractelement <4 x float> [[TMP0]], i32 1
 ; DEFAULT-NEXT:    [[TMP2:%.*]] = tail call fast float @llvm.acos.f32(float [[VECEXT_1]])
 ; DEFAULT-NEXT:    [[VECINS_1:%.*]] = insertelement <4 x float> [[VECINS]], float [[TMP2]], i32 1
@@ -812,7 +812,7 @@ entry:
   %0 = load <4 x float>, ptr %a, align 16
   %vecext = extractelement <4 x float> %0, i32 0
   %1 = tail call fast float @llvm.acos.f32(float %vecext)
-  %vecins = insertelement <4 x float> undef, float %1, i32 0
+  %vecins = insertelement <4 x float> poison, float %1, i32 0
   %vecext.1 = extractelement <4 x float> %0, i32 1
   %2 = tail call fast float @llvm.acos.f32(float %vecext.1)
   %vecins.1 = insertelement <4 x float> %vecins, float %2, i32 1
@@ -835,7 +835,7 @@ define <4 x float> @tan_4x(ptr %a) {
 ; CHECK-NEXT:    [[TMP0:%.*]] = load <4 x float>, ptr [[A]], align 16
 ; CHECK-NEXT:    [[VECEXT:%.*]] = extractelement <4 x float> [[TMP0]], i32 0
 ; CHECK-NEXT:    [[TMP1:%.*]] = tail call fast float @tanf(float [[VECEXT]])
-; CHECK-NEXT:    [[VECINS:%.*]] = insertelement <4 x float> undef, float [[TMP1]], i32 0
+; CHECK-NEXT:    [[VECINS:%.*]] = insertelement <4 x float> poison, float [[TMP1]], i32 0
 ; CHECK-NEXT:    [[VECEXT_1:%.*]] = extractelement <4 x float> [[TMP0]], i32 1
 ; CHECK-NEXT:    [[TMP2:%.*]] = tail call fast float @tanf(float [[VECEXT_1]])
 ; CHECK-NEXT:    [[VECINS_1:%.*]] = insertelement <4 x float> [[VECINS]], float [[TMP2]], i32 1
@@ -853,7 +853,7 @@ define <4 x float> @tan_4x(ptr %a) {
 ; DEFAULT-NEXT:    [[TMP0:%.*]] = load <4 x float>, ptr [[A]], align 16
 ; DEFAULT-NEXT:    [[VECEXT:%.*]] = extractelement <4 x float> [[TMP0]], i32 0
 ; DEFAULT-NEXT:    [[TMP1:%.*]] = tail call fast float @tanf(float [[VECEXT]])
-; DEFAULT-NEXT:    [[VECINS:%.*]] = insertelement <4 x float> undef, float [[TMP1]], i32 0
+; DEFAULT-NEXT:    [[VECINS:%.*]] = insertelement <4 x float> poison, float [[TMP1]], i32 0
 ; DEFAULT-NEXT:    [[VECEXT_1:%.*]] = extractelement <4 x float> [[TMP0]], i32 1
 ; DEFAULT-NEXT:    [[TMP2:%.*]] = tail call fast float @tanf(float [[VECEXT_1]])
 ; DEFAULT-NEXT:    [[VECINS_1:%.*]] = insertelement <4 x float> [[VECINS]], float [[TMP2]], i32 1
@@ -869,7 +869,7 @@ entry:
   %0 = load <4 x float>, ptr %a, align 16
   %vecext = extractelement <4 x float> %0, i32 0
   %1 = tail call fast float @tanf(float %vecext)
-  %vecins = insertelement <4 x float> undef, float %1, i32 0
+  %vecins = insertelement <4 x float> poison, float %1, i32 0
   %vecext.1 = extractelement <4 x float> %0, i32 1
   %2 = tail call fast float @tanf(float %vecext.1)
   %vecins.1 = insertelement <4 x float> %vecins, float %2, i32 1
@@ -892,7 +892,7 @@ define <4 x float> @int_tan_4x(ptr %a) {
 ; CHECK-NEXT:    [[TMP0:%.*]] = load <4 x float>, ptr [[A]], align 16
 ; CHECK-NEXT:    [[VECEXT:%.*]] = extractelement <4 x float> [[TMP0]], i32 0
 ; CHECK-NEXT:    [[TMP1:%.*]] = tail call fast float @llvm.tan.f32(float [[VECEXT]])
-; CHECK-NEXT:    [[VECINS:%.*]] = insertelement <4 x float> undef, float [[TMP1]], i32 0
+; CHECK-NEXT:    [[VECINS:%.*]] = insertelement <4 x float> poison, float [[TMP1]], i32 0
 ; CHECK-NEXT:    [[VECEXT_1:%.*]] = extractelement <4 x float> [[TMP0]], i32 1
 ; CHECK-NEXT:    [[TMP2:%.*]] = tail call fast float @llvm.tan.f32(float [[VECEXT_1]])
 ; CHECK-NEXT:    [[VECINS_1:%.*]] = insertelement <4 x float> [[VECINS]], float [[TMP2]], i32 1
@@ -910,7 +910,7 @@ define <4 x float> @int_tan_4x(ptr %a) {
 ; DEFAULT-NEXT:    [[TMP0:%.*]] = load <4 x float>, ptr [[A]], align 16
 ; DEFAULT-NEXT:    [[VECEXT:%.*]] = extractelement <4 x float> [[TMP0]], i32 0
 ; DEFAULT-NEXT:    [[TMP1:%.*]] = tail call fast float @llvm.tan.f32(float [[VECEXT]])
-; DEFAULT-NEXT:    [[VECINS:%.*]] = insertelement <4 x float> undef, float [[TMP1]], i32 0
+; DEFAULT-NEXT:    [[VECINS:%.*]] = insertelement <4 x float> poison, float [[TMP1]], i32 0
 ; DEFAULT-NEXT:    [[VECEXT_1:%.*]] = extractelement <4 x float> [[TMP0]], i32 1
 ; DEFAULT-NEXT:    [[TMP2:%.*]] = tail call fast float @llvm.tan.f32(float [[VECEXT_1]])
 ; DEFAULT-NEXT:    [[VECINS_1:%.*]] = insertelement <4 x float> [[VECINS]], float [[TMP2]], i32 1
@@ -926,7 +926,7 @@ entry:
   %0 = load <4 x float>, ptr %a, align 16
   %vecext = extractelement <4 x float> %0, i32 0
   %1 = tail call fast float @llvm.tan.f32(float %vecext)
-  %vecins = insertelement <4 x float> undef, float %1, i32 0
+  %vecins = insertelement <4 x float> poison, float %1, i32 0
   %vecext.1 = extractelement <4 x float> %0, i32 1
   %2 = tail call fast float @llvm.tan.f32(float %vecext.1)
   %vecins.1 = insertelement <4 x float> %vecins, float %2, i32 1
@@ -949,7 +949,7 @@ define <4 x float> @atan_4x(ptr %a) {
 ; CHECK-NEXT:    [[TMP0:%.*]] = load <4 x float>, ptr [[A]], align 16
 ; CHECK-NEXT:    [[VECEXT:%.*]] = extractelement <4 x float> [[TMP0]], i32 0
 ; CHECK-NEXT:    [[TMP1:%.*]] = tail call fast float @atanf(float [[VECEXT]])
-; CHECK-NEXT:    [[VECINS:%.*]] = insertelement <4 x float> undef, float [[TMP1]], i32 0
+; CHECK-NEXT:    [[VECINS:%.*]] = insertelement <4 x float> poison, float [[TMP1]], i32 0
 ; CHECK-NEXT:    [[VECEXT_1:%.*]] = extractelement <4 x float> [[TMP0]], i32 1
 ; CHECK-NEXT:    [[TMP2:%.*]] = tail call fast float @atanf(float [[VECEXT_1]])
 ; CHECK-NEXT:    [[VECINS_1:%.*]] = insertelement <4 x float> [[VECINS]], float [[TMP2]], i32 1
@@ -967,7 +967,7 @@ define <4 x float> @atan_4x(ptr %a) {
 ; DEFAULT-NEXT:    [[TMP0:%.*]] = load <4 x float>, ptr [[A]], align 16
 ; DEFAULT-NEXT:    [[VECEXT:%.*]] = extractelement <4 x float> [[TMP0]], i32 0
 ; DEFAULT-NEXT:    [[TMP1:%.*]] = tail call fast float @atanf(float [[VECEXT]])
-; DEFAULT-NEXT:    [[VECINS:%.*]] = insertelement <4 x float> undef, float [[TMP1]], i32 0
+; DEFAULT-NEXT:    [[VECINS:%.*]] = insertelement <4 x float> poison, float [[TMP1]], i32 0
 ; DEFAULT-NEXT:    [[VECEXT_1:%.*]] = extractelement <4 x float> [[TMP0]], i32 1
 ; DEFAULT-NEXT:    [[TMP2:%.*]] = tail call fast float @atanf(float [[VECEXT_1]])
 ; DEFAULT-NEXT:    [[VECINS_1:%.*]] = insertelement <4 x float> [[VECINS]], float [[TMP2]], i32 1
@@ -983,7 +983,7 @@ entry:
   %0 = load <4 x float>, ptr %a, align 16
   %vecext = extractelement <4 x float> %0, i32 0
   %1 = tail call fast float @atanf(float %vecext)
-  %vecins = insertelement <4 x float> undef, float %1, i32 0
+  %vecins = insertelement <4 x float> poison, float %1, i32 0
   %vecext.1 = extractelement <4 x float> %0, i32 1
   %2 = tail call fast float @atanf(float %vecext.1)
   %vecins.1 = insertelement <4 x float> %vecins, float %2, i32 1
@@ -1006,7 +1006,7 @@ define <4 x float> @int_atan_4x(ptr %a) {
 ; CHECK-NEXT:    [[TMP0:%.*]] = load <4 x float>, ptr [[A]], align 16
 ; CHECK-NEXT:    [[VECEXT:%.*]] = extractelement <4 x float> [[TMP0]], i32 0
 ; CHECK-NEXT:    [[TMP1:%.*]] = tail call fast float @llvm.atan.f32(float [[VECEXT]])
-; CHECK-NEXT:    [[VECINS:%.*]] = insertelement <4 x float> undef, float [[TMP1]], i32 0
+; CHECK-NEXT:    [[VECINS:%.*]] = insertelement <4 x float> poison, float [[TMP1]], i32 0
 ; CHECK-NEXT:    [[VECEXT_1:%.*]] = extractelement <4 x float> [[TMP0]], i32 1
 ; CHECK-NEXT:    [[TMP2:%.*]] = tail call fast float @llvm.atan.f32(float [[VECEXT_1]])
 ; CHECK-NEXT:    [[VECINS_1:%.*]] = insertelement <4 x float> [[VECINS]], float [[TMP2]], i32 1
@@ -1024,7 +1024,7 @@ define <4 x float> @int_atan_4x(ptr %a) {
 ; DEFAULT-NEXT:    [[TMP0:%.*]] = load <4 x float>, ptr [[A]], align 16
 ; DEFAULT-NEXT:    [[VECEXT:%.*]] = extractelement <4 x float> [[TMP0]], i32 0
 ; DEFAULT-NEXT:    [[TMP1:%.*]] = tail call fast float @llvm.atan.f32(float [[VECEXT]])
-; DEFAULT-NEXT:    [[VECINS:%.*]] = insertelement <4 x float> undef, float [[TMP1]], i32 0
+; DEFAULT-NEXT:    [[VECINS:%.*]] = insertelement <4 x float> poison, float [[TMP1]], i32 0
 ; DEFAULT-NEXT:    [[VECEXT_1:%.*]] = extractelement <4 x float> [[TMP0]], i32 1
 ; DEFAULT-NEXT:    [[TMP2:%.*]] = tail call fast float @llvm.atan.f32(float [[VECEXT_1]])
 ; DEFAULT-NEXT:    [[VECINS_1:%.*]] = insertelement <4 x float> [[VECINS]], float [[TMP2]], i32 1
@@ -1040,7 +1040,7 @@ entry:
   %0 = load <4 x float>, ptr %a, align 16
   %vecext = extractelement <4 x float> %0, i32 0
   %1 = tail call fast float @llvm.atan.f32(float %vecext)
-  %vecins = insertelement <4 x float> undef, float %1, i32 0
+  %vecins = insertelement <4 x float> poison, float %1, i32 0
   %vecext.1 = extractelement <4 x float> %0, i32 1
   %2 = tail call fast float @llvm.atan.f32(float %vecext.1)
   %vecins.1 = insertelement <4 x float> %vecins, float %2, i32 1
@@ -1063,7 +1063,7 @@ define <4 x float> @sinh_4x(ptr %a) {
 ; CHECK-NEXT:    [[TMP0:%.*]] = load <4 x float>, ptr [[A]], align 16
 ; CHECK-NEXT:    [[VECEXT:%.*]] = extractelement <4 x float> [[TMP0]], i32 0
 ; CHECK-NEXT:    [[TMP1:%.*]] = tail call fast float @sinhf(float [[VECEXT]])
-; CHECK-NEXT:    [[VECINS:%.*]] = insertelement <4 x float> undef, float [[TMP1]], i32 0
+; CHECK-NEXT:    [[VECINS:%.*]] = insertelement <4 x float> poison, float [[TMP1]], i32 0
 ; CHECK-NEXT:    [[VECEXT_1:%.*]] = extractelement <4 x float> [[TMP0]], i32 1
 ; CHECK-NEXT:    [[TMP2:%.*]] = tail call fast float @sinhf(float [[VECEXT_1]])
 ; CHECK-NEXT:    [[VECINS_1:%.*]] = insertelement <4 x float> [[VECINS]], float [[TMP2]], i32 1
@@ -1081,7 +1081,7 @@ define <4 x float> @sinh_4x(ptr %a) {
 ; DEFAULT-NEXT:    [[TMP0:%.*]] = load <4 x float>, ptr [[A]], align 16
 ; DEFAULT-NEXT:    [[VECEXT:%.*]] = extractelement <4 x float> [[TMP0]], i32 0
 ; DEFAULT-NEXT:    [[TMP1:%.*]] = tail call fast float @sinhf(float [[VECEXT]])
-; DEFAULT-NEXT:    [[VECINS:%.*]] = insertelement <4 x float> undef, float [[TMP1]], i32 0
+; DEFAULT-NEXT:    [[VECINS:%.*]] = insertelement <4 x float> poison, float [[TMP1]], i32 0
 ; DEFAULT-NEXT:    [[VECEXT_1:%.*]] = extractelement <4 x float> [[TMP0]], i32 1
 ; DEFAULT-NEXT:    [[TMP2:%.*]] = tail call fast float @sinhf(float [[VECEXT_1]])
 ; DEFAULT-NEXT:    [[VECINS_1:%.*]] = insertelement <4 x float> [[VECINS]], float [[TMP2]], i32 1
@@ -1097,7 +1097,7 @@ entry:
   %0 = load <4 x float>, ptr %a, align 16
   %vecext = extractelement <4 x float> %0, i32 0
   %1 = tail call fast float @sinhf(float %vecext)
-  %vecins = insertelement <4 x float> undef, float %1, i32 0
+  %vecins = insertelement <4 x float> poison, float %1, i32 0
   %vecext.1 = extractelement <4 x float> %0, i32 1
   %2 = tail call fast float @sinhf(float %vecext.1)
   %vecins.1 = insertelement <4 x float> %vecins, float %2, i32 1
@@ -1120,7 +1120,7 @@ define <4 x float> @int_sinh_4x(ptr %a) {
 ; CHECK-NEXT:    [[TMP0:%.*]] = load <4 x float>, ptr [[A]], align 16
 ; CHECK-NEXT:    [[VECEXT:%.*]] = extractelement <4 x float> [[TMP0]], i32 0
 ; CHECK-NEXT:    [[TMP1:%.*]] = tail call fast float @llvm.sinh.f32(float [[VECEXT]])
-; CHECK-NEXT:    [[VECINS:%.*]] = insertelement <4 x float> undef, float [[TMP1]], i32 0
+; CHECK-NEXT:    [[VECINS:%.*]] = insertelement <4 x float> poison, float [[TMP1]], i32 0
 ; CHECK-NEXT:    [[VECEXT_1:%.*]] = extractelement <4 x float> [[TMP0]], i32 1
 ; CHECK-NEXT:    [[TMP2:%.*]] = tail call fast float @llvm.sinh.f32(float [[VECEXT_1]])
 ; CHECK-NEXT:    [[VECINS_1:%.*]] = insertelement <4 x float> [[VECINS]], float [[TMP2]], i32 1
@@ -1138,7 +1138,7 @@ define <4 x float> @int_sinh_4x(ptr %a) {
 ; DEFAULT-NEXT:    [[TMP0:%.*]] = load <4 x float>, ptr [[A]], align 16
 ; DEFAULT-NEXT:    [[VECEXT:%.*]] = extractelement <4 x float> [[TMP0]], i32 0
 ; DEFAULT-NEXT:    [[TMP1:%.*]] = tail call fast float @llvm.sinh.f32(float [[VECEXT]])
-; DEFAULT-NEXT:    [[VECINS:%.*]] = insertelement <4 x float> undef, float [[TMP1]], i32 0
+; DEFAULT-NEXT:    [[VECINS:%.*]] = insertelement <4 x float> poison, float [[TMP1]], i32 0
 ; DEFAULT-NEXT:    [[VECEXT_1:%.*]] = extractelement <4 x float> [[TMP0]], i32 1
 ; DEFAULT-NEXT:    [[TMP2:%.*]] = tail call fast float @llvm.sinh.f32(float [[VECEXT_1]])
 ; DEFAULT-NEXT:    [[VECINS_1:%.*]] = insertelement <4 x float> [[VECINS]], float [[TMP2]], i32 1
@@ -1154,7 +1154,7 @@ entry:
   %0 = load <4 x float>, ptr %a, align 16
   %vecext = extractelement <4 x float> %0, i32 0
   %1 = tail call fast float @llvm.sinh.f32(float %vecext)
-  %vecins = insertelement <4 x float> undef, float %1, i32 0
+  %vecins = insertelement <4 x float> poison, float %1, i32 0
   %vecext.1 = extractelement <4 x float> %0, i32 1
   %2 = tail call fast float @llvm.sinh.f32(float %vecext.1)
   %vecins.1 = insertelement <4 x float> %vecins, float %2, i32 1
@@ -1177,7 +1177,7 @@ define <4 x float> @asinh_4x(ptr %a) {
 ; CHECK-NEXT:    [[TMP0:%.*]] = load <4 x float>, ptr [[A]], align 16
 ; CHECK-NEXT:    [[VECEXT:%.*]] = extractelement <4 x float> [[TMP0]], i32 0
 ; CHECK-NEXT:    [[TMP1:%.*]] = tail call fast float @asinhf(float [[VECEXT]])
-; CHECK-NEXT:    [[VECINS:%.*]] = insertelement <4 x float> undef, float [[TMP1]], i32 0
+; CHECK-NEXT:    [[VECINS:%.*]] = insertelement <4 x float> poison, float [[TMP1]], i32 0
 ; CHECK-NEXT:    [[VECEXT_1:%.*]] = extractelement <4 x float> [[TMP0]], i32 1
 ; CHECK-NEXT:    [[TMP2:%.*]] = tail call fast float @asinhf(float [[VECEXT_1]])
 ; CHECK-NEXT:    [[VECINS_1:%.*]] = insertelement <4 x float> [[VECINS]], float [[TMP2]], i32 1
@@ -1195,7 +1195,7 @@ define <4 x float> @asinh_4x(ptr %a) {
 ; DEFAULT-NEXT:    [[TMP0:%.*]] = load <4 x float>, ptr [[A]], align 16
 ; DEFAULT-NEXT:    [[VECEXT:%.*]] = extractelement <4 x float> [[TMP0]], i32 0
 ; DEFAULT-NEXT:    [[TMP1:%.*]] = tail call fast float @asinhf(float [[VECEXT]])
-; DEFAULT-NEXT:    [[VECINS:%.*]] = insertelement <4 x float> undef, float [[TMP1]], i32 0
+; DEFAULT-NEXT:    [[VECINS:%.*]] = insertelement <4 x float> poison, float [[TMP1]], i32 0
 ; DEFAULT-NEXT:    [[VECEXT_1:%.*]] = extractelement <4 x float> [[TMP0]], i32 1
 ; DEFAULT-NEXT:    [[TMP2:%.*]] = tail call fast float @asinhf(float [[VECEXT_1]])
 ; DEFAULT-NEXT:    [[VECINS_1:%.*]] = insertelement <4 x float> [[VECINS]], float [[TMP2]], i32 1
@@ -1211,7 +1211,7 @@ entry:
   %0 = load <4 x float>, ptr %a, align 16
   %vecext = extractelement <4 x float> %0, i32 0
   %1 = tail call fast float @asinhf(float %vecext)
-  %vecins = insertelement <4 x float> undef, float %1, i32 0
+  %vecins = insertelement <4 x float> poison, float %1, i32 0
   %vecext.1 = extractelement <4 x float> %0, i32 1
   %2 = tail call fast float @asinhf(float %vecext.1)
   %vecins.1 = insertelement <4 x float> %vecins, float %2, i32 1
@@ -1234,7 +1234,7 @@ define <4 x float> @int_asinh_4x(ptr %a) {
 ; CHECK-NEXT:    [[TMP0:%.*]] = load <4 x float>, ptr [[A]], align 16
 ; CHECK-NEXT:    [[VECEXT:%.*]] = extractelement <4 x float> [[TMP0]], i32 0
 ; CHECK-NEXT:    [[TMP1:%.*]] = tail call fast float @llvm.asinh.f32(float [[VECEXT]])
-; CHECK-NEXT:    [[VECINS:%.*]] = insertelement <4 x float> undef, float [[TMP1]], i32 0
+; CHECK-NEXT:    [[VECINS:%.*]] = insertelement <4 x float> poison, float [[TMP1]], i32 0
 ; CHECK-NEXT:    [[VECEXT_1:%.*]] = extractelement <4 x float> [[TMP0]], i32 1
 ; CHECK-NEXT:    [[TMP2:%.*]] = tail call fast float @llvm.asinh.f32(float [[VECEXT_1]])
 ; CHECK-NEXT:    [[VECINS_1:%.*]] = insertelement <4 x float> [[VECINS]], float [[TMP2]], i32 1
@@ -1252,7 +1252,7 @@ define <4 x float> @int_asinh_4x(ptr %a) {
 ; DEFAULT-NEXT:    [[TMP0:%.*]] = load <4 x float>, ptr [[A]], align 16
 ; DEFAULT-NEXT:    [[VECEXT:%.*]] = extractelement <4 x float> [[TMP0]], i32 0
 ; DEFAULT-NEXT:    [[TMP1:%.*]] = tail call fast float @llvm.asinh.f32(float [[VECEXT]])
-; DEFAULT-NEXT:    [[VECINS:%.*]] = insertelement <4 x float> undef, float [[TMP1]], i32 0
+; DEFAULT-NEXT:    [[VECINS:%.*]] = insertelement <4 x float> poison, float [[TMP1]], i32 0
 ; DEFAULT-NEXT:    [[VECEXT_1:%.*]] = extractelement <4 x float> [[TMP0]], i32 1
 ; DEFAULT-NEXT:    [[TMP2:%.*]] = tail call fast float @llvm.asinh.f32(float [[VECEXT_1]])
 ; DEFAULT-NEXT:    [[VECINS_1:%.*]] = insertelement <4 x float> [[VECINS]], float [[TMP2]], i32 1
@@ -1268,7 +1268,7 @@ entry:
   %0 = load <4 x float>, ptr %a, align 16
   %vecext = extractelement <4 x float> %0, i32 0
   %1 = tail call fast float @llvm.asinh.f32(float %vecext)
-  %vecins = insertelement <4 x float> undef, float %1, i32 0
+  %vecins = insertelement <4 x float> poison, float %1, i32 0
   %vecext.1 = extractelement <4 x float> %0, i32 1
   %2 = tail call fast float @llvm.asinh.f32(float %vecext.1)
   %vecins.1 = insertelement <4 x float> %vecins, float %2, i32 1
@@ -1291,7 +1291,7 @@ define <4 x float> @cosh_4x(ptr %a) {
 ; CHECK-NEXT:    [[TMP0:%.*]] = load <4 x float>, ptr [[A]], align 16
 ; CHECK-NEXT:    [[VECEXT:%.*]] = extractelement <4 x float> [[TMP0]], i32 0
 ; CHECK-NEXT:    [[TMP1:%.*]] = tail call fast float @coshf(float [[VECEXT]])
-; CHECK-NEXT:    [[VECINS:%.*]] = insertelement <4 x float> undef, float [[TMP1]], i32 0
+; CHECK-NEXT:    [[VECINS:%.*]] = insertelement <4 x float> poison, float [[TMP1]], i32 0
 ; CHECK-NEXT:    [[VECEXT_1:%.*]] = extractelement <4 x float> [[TMP0]], i32 1
 ; CHECK-NEXT:    [[TMP2:%.*]] = tail call fast float @coshf(float [[VECEXT_1]])
 ; CHECK-NEXT:    [[VECINS_1:%.*]] = insertelement <4 x float> [[VECINS]], float [[TMP2]], i32 1
@@ -1309,7 +1309,7 @@ define <4 x float> @cosh_4x(ptr %a) {
 ; DEFAULT-NEXT:    [[TMP0:%.*]] = load <4 x float>, ptr [[A]], align 16
 ; DEFAULT-NEXT:    [[VECEXT:%.*]] = extractelement <4 x float> [[TMP0]], i32 0
 ; DEFAULT-NEXT:    [[TMP1:%.*]] = tail call fast float @coshf(float [[VECEXT]])
-; DEFAULT-NEXT:    [[VECINS:%.*]] = insertelement <4 x float> undef, float [[TMP1]], i32 0
+; DEFAULT-NEXT:    [[VECINS:%.*]] = insertelement <4 x float> poison, float [[TMP1]], i32 0
 ; DEFAULT-NEXT:    [[VECEXT_1:%.*]] = extractelement <4 x float> [[TMP0]], i32 1
 ; DEFAULT-NEXT:    [[TMP2:%.*]] = tail call fast float @coshf(float [[VECEXT_1]])
 ; DEFAULT-NEXT:    [[VECINS_1:%.*]] = insertelement <4 x float> [[VECINS]], float [[TMP2]], i32 1
@@ -1325,7 +1325,7 @@ entry:
   %0 = load <4 x float>, ptr %a, align 16
   %vecext = extractelement <4 x float> %0, i32 0
   %1 = tail call fast float @coshf(float %vecext)
-  %vecins = insertelement <4 x float> undef, float %1, i32 0
+  %vecins = insertelement <4 x float> poison, float %1, i32 0
   %vecext.1 = extractelement <4 x float> %0, i32 1
   %2 = tail call fast float @coshf(float %vecext.1)
   %vecins.1 = insertelement <4 x float> %vecins, float %2, i32 1
@@ -1348,7 +1348,7 @@ define <4 x float> @int_cosh_4x(ptr %a) {
 ; CHECK-NEXT:    [[TMP0:%.*]] = load <4 x float>, ptr [[A]], align 16
 ; CHECK-NEXT:    [[VECEXT:%.*]] = extractelement <4 x float> [[TMP0]], i32 0
 ; CHECK-NEXT:    [[TMP1:%.*]] = tail call fast float @llvm.cosh.f32(float [[VECEXT]])
-; CHECK-NEXT:    [[VECINS:%.*]] = insertelement <4 x float> undef, float [[TMP1]], i32 0
+; CHECK-NEXT:    [[VECINS:%.*]] = insertelement <4 x float> poison, float [[TMP1]], i32 0
 ; CHECK-NEXT:    [[VECEXT_1:%.*]] = extractelement <4 x float> [[TMP0]], i32 1
 ; CHECK-NEXT:    [[TMP2:%.*]] = tail call fast float @llvm.cosh.f32(float [[VECEXT_1]])
 ; CHECK-NEXT:    [[VECINS_1:%.*]] = insertelement <4 x float> [[VECINS]], float [[TMP2]], i32 1
@@ -1366,7 +1366,7 @@ define <4 x float> @int_cosh_4x(ptr %a) {
 ; DEFAULT-NEXT:    [[TMP0:%.*]] = load <4 x float>, ptr [[A]], align 16
 ; DEFAULT-NEXT:    [[VECEXT:%.*]] = extractelement <4 x float> [[TMP0]], i32 0
 ; DEFAULT-NEXT:    [[TMP1:%.*]] = tail call fast float @llvm.cosh.f32(float [[VECEXT]])
-; DEFAULT-NEXT:    [[VECINS:%.*]] = insertelement <4 x float> undef, float [[TMP1]], i32 0
+; DEFAULT-NEXT:    [[VECINS:%.*]] = insertelement <4 x float> poison, float [[TMP1]], i32 0
 ; DEFAULT-NEXT:    [[VECEXT_1:%.*]] = extractelement <4 x float> [[TMP0]], i32 1
 ; DEFAULT-NEXT:    [[TMP2:%.*]] = tail call fast float @llvm.cosh.f32(float [[VECEXT_1]])
 ; DEFAULT-NEXT:    [[VECINS_1:%.*]] = insertelement <4 x float> [[VECINS]], float [[TMP2]], i32 1
@@ -1382,7 +1382,7 @@ entry:
   %0 = load <4 x float>, ptr %a, align 16
   %vecext = extractelement <4 x float> %0, i32 0
   %1 = tail call fast float @llvm.cosh.f32(float %vecext)
-  %vecins = insertelement <4 x float> undef, float %1, i32 0
+  %vecins = insertelement <4 x float> poison, float %1, i32 0
   %vecext.1 = extractelement <4 x float> %0, i32 1
   %2 = tail call fast float @llvm.cosh.f32(float %vecext.1)
   %vecins.1 = insertelement <4 x float> %vecins, float %2, i32 1
@@ -1405,7 +1405,7 @@ define <4 x float> @acosh_4x(ptr %a) {
 ; CHECK-NEXT:    [[TMP0:%.*]] = load <4 x float>, ptr [[A]], align 16
 ; CHECK-NEXT:    [[VECEXT:%.*]] = extractelement <4 x float> [[TMP0]], i32 0
 ; CHECK-NEXT:    [[TMP1:%.*]] = tail call fast float @acoshf(float [[VECEXT]])
-; CHECK-NEXT:    [[VECINS:%.*]] = insertelement <4 x float> undef, float [[TMP1]], i32 0
+; CHECK-NEXT:    [[VECINS:%.*]] = insertelement <4 x float> poison, float [[TMP1]], i32 0
 ; CHECK-NEXT:    [[VECEXT_1:%.*]] = extractelement <4 x float> [[TMP0]], i32 1
 ; CHECK-NEXT:    [[TMP2:%.*]] = tail call fast float @acoshf(float [[VECEXT_1]])
 ; CHECK-NEXT:    [[VECINS_1:%.*]] = insertelement <4 x float> [[VECINS]], float [[TMP2]], i32 1
@@ -1423,7 +1423,7 @@ define <4 x float> @acosh_4x(ptr %a) {
 ; DEFAULT-NEXT:    [[TMP0:%.*]] = load <4 x float>, ptr [[A]], align 16
 ; DEFAULT-NEXT:    [[VECEXT:%.*]] = extractelement <4 x float> [[TMP0]], i32 0
 ; DEFAULT-NEXT:    [[TMP1:%.*]] = tail call fast float @acoshf(float [[VECEXT]])
-; DEFAULT-NEXT:    [[VECINS:%.*]] = insertelement <4 x float> undef, float [[TMP1]], i32 0
+; DEFAULT-NEXT:    [[VECINS:%.*]] = insertelement <4 x float> poison, float [[TMP1]], i32 0
 ; DEFAULT-NEXT:    [[VECEXT_1:%.*]] = extractelement <4 x float> [[TMP0]], i32 1
 ; DEFAULT-NEXT:    [[TMP2:%.*]] = tail call fast float @acoshf(float [[VECEXT_1]])
 ; DEFAULT-NEXT:    [[VECINS_1:%.*]] = insertelement <4 x float> [[VECINS]], float [[TMP2]], i32 1
@@ -1439,7 +1439,7 @@ entry:
   %0 = load <4 x float>, ptr %a, align 16
   %vecext = extractelement <4 x float> %0, i32 0
   %1 = tail call fast float @acoshf(float %vecext)
-  %vecins = insertelement <4 x float> undef, float %1, i32 0
+  %vecins = insertelement <4 x float> poison, float %1, i32 0
   %vecext.1 = extractelement <4 x float> %0, i32 1
   %2 = tail call fast float @acoshf(float %vecext.1)
   %vecins.1 = insertelement <4 x float> %vecins, float %2, i32 1
@@ -1462,7 +1462,7 @@ define <4 x float> @int_acosh_4x(ptr %a) {
 ; CHECK-NEXT:    [[TMP0:%.*]] = load <4 x float>, ptr [[A]], align 16
 ; CHECK-NEXT:    [[VECEXT:%.*]] = extractelement <4 x float> [[TMP0]], i32 0
 ; CHECK-NEXT:    [[TMP1:%.*]] = tail call fast float @llvm.acosh.f32(float [[VECEXT]])
-; CHECK-NEXT:    [[VECINS:%.*]] = insertelement <4 x float> undef, float [[TMP1]], i32 0
+; CHECK-NEXT:    [[VECINS:%.*]] = insertelement <4 x float> poison, float [[TMP1]], i32 0
 ; CHECK-NEXT:    [[VECEXT_1:%.*]] = extractelement <4 x float> [[TMP0]], i32 1
 ; CHECK-NEXT:    [[TMP2:%.*]] = tail call fast float @llvm.acosh.f32(float [[VECEXT_1]])
 ; CHECK-NEXT:    [[VECINS_1:%.*]] = insertelement <4 x float> [[VECINS]], float [[TMP2]], i32 1
@@ -1480,7 +1480,7 @@ define <4 x float> @int_acosh_4x(ptr %a) {
 ; DEFAULT-NEXT:    [[TMP0:%.*]] = load <4 x float>, ptr [[A]], align 16
 ; DEFAULT-NEXT:    [[VECEXT:%.*]] = extractelement <4 x float> [[TMP0]], i32 0
 ; DEFAULT-NEXT:    [[TMP1:%.*]] = tail call fast float @llvm.acosh.f32(float [[VECEXT]])
-; DEFAULT-NEXT:    [[VECINS:%.*]] = insertelement <4 x float> undef, float [[TMP1]], i32 0
+; DEFAULT-NEXT:    [[VECINS:%.*]] = insertelement <4 x float> poison, float [[TMP1]], i32 0
 ; DEFAULT-NEXT:    [[VECEXT_1:%.*]] = extractelement <4 x float> [[TMP0]], i32 1
 ; DEFAULT-NEXT:    [[TMP2:%.*]] = tail call fast float @llvm.acosh.f32(float [[VECEXT_1]])
 ; DEFAULT-NEXT:    [[VECINS_1:%.*]] = insertelement <4 x float> [[VECINS]], float [[TMP2]], i32 1
@@ -1496,7 +1496,7 @@ entry:
   %0 = load <4 x float>, ptr %a, align 16
   %vecext = extractelement <4 x float> %0, i32 0
   %1 = tail call fast float @llvm.acosh.f32(float %vecext)
-  %vecins = insertelement <4 x float> undef, float %1, i32 0
+  %vecins = insertelement <4 x float> poison, float %1, i32 0
   %vecext.1 = extractelement <4 x float> %0, i32 1
   %2 = tail call fast float @llvm.acosh.f32(float %vecext.1)
   %vecins.1 = insertelement <4 x float> %vecins, float %2, i32 1
@@ -1519,7 +1519,7 @@ define <4 x float> @tanh_4x(ptr %a) {
 ; CHECK-NEXT:    [[TMP0:%.*]] = load <4 x float>, ptr [[A]], align 16
 ; CHECK-NEXT:    [[VECEXT:%.*]] = extractelement <4 x float> [[TMP0]], i32 0
 ; CHECK-NEXT:    [[TMP1:%.*]] = tail call fast float @tanhf(float [[VECEXT]])
-; CHECK-NEXT:    [[VECINS:%.*]] = insertelement <4 x float> undef, float [[TMP1]], i32 0
+; CHECK-NEXT:    [[VECINS:%.*]] = insertelement <4 x float> poison, float [[TMP1]], i32 0
 ; CHECK-NEXT:    [[VECEXT_1:%.*]] = extractelement <4 x float> [[TMP0]], i32 1
 ; CHECK-NEXT:    [[TMP2:%.*]] = tail call fast float @tanhf(float [[VECEXT_1]])
 ; CHECK-NEXT:    [[VECINS_1:%.*]] = insertelement <4 x float> [[VECINS]], float [[TMP2]], i32 1
@@ -1537,7 +1537,7 @@ define <4 x float> @tanh_4x(ptr %a) {
 ; DEFAULT-NEXT:    [[TMP0:%.*]] = load <4 x float>, ptr [[A]], align 16
 ; DEFAULT-NEXT:    [[VECEXT:%.*]] = extractelement <4 x float> [[TMP0]], i32 0
 ; DEFAULT-NEXT:    [[TMP1:%.*]] = tail call fast float @tanhf(float [[VECEXT]])
-; DEFAULT-NEXT:    [[VECINS:%.*]] = insertelement <4 x float> undef, float [[TMP1]], i32 0
+; DEFAULT-NEXT:    [[VECINS:%.*]] = insertelement <4 x float> poison, float [[TMP1]], i32 0
 ; DEFAULT-NEXT:    [[VECEXT_1:%.*]] = extractelement <4 x float> [[TMP0]], i32 1
 ; DEFAULT-NEXT:    [[TMP2:%.*]] = tail call fast float @tanhf(float [[VECEXT_1]])
 ; DEFAULT-NEXT:    [[VECINS_1:%.*]] = insertelement <4 x float> [[VECINS]], float [[TMP2]], i32 1
@@ -1553,7 +1553,7 @@ entry:
   %0 = load <4 x float>, ptr %a, align 16
   %vecext = extractelement <4 x float> %0, i32 0
   %1 = tail call fast float @tanhf(float %vecext)
-  %vecins = insertelement <4 x float> undef, float %1, i32 0
+  %vecins = insertelement <4 x float> poison, float %1, i32 0
   %vecext.1 = extractelement <4 x float> %0, i32 1
   %2 = tail call fast float @tanhf(float %vecext.1)
   %vecins.1 = insertelement <4 x float> %vecins, float %2, i32 1
@@ -1576,7 +1576,7 @@ define <4 x float> @int_tanh_4x(ptr %a) {
 ; CHECK-NEXT:    [[TMP0:%.*]] = load <4 x float>, ptr [[A]], align 16
 ; CHECK-NEXT:    [[VECEXT:%.*]] = extractelement <4 x float> [[TMP0]], i32 0
 ; CHECK-NEXT:    [[TMP1:%.*]] = tail call fast float @llvm.tanh.f32(float [[VECEXT]])
-; CHECK-NEXT:    [[VECINS:%.*]] = insertelement <4 x float> undef, float [[TMP1]], i32 0
+; CHECK-NEXT:    [[VECINS:%.*]] = insertelement <4 x float> poison, float [[TMP1]], i32 0
 ; CHECK-NEXT:    [[VECEXT_1:%.*]] = extractelement <4 x float> [[TMP0]], i32 1
 ; CHECK-NEXT:    [[TMP2:%.*]] = tail call fast float @llvm.tanh.f32(float [[VECEXT_1]])
 ; CHECK-NEXT:    [[VECINS_1:%.*]] = insertelement <4 x float> [[VECINS]], float [[TMP2]], i32 1
@@ -1594,7 +1594,7 @@ define <4 x float> @int_tanh_4x(ptr %a) {
 ; DEFAULT-NEXT:    [[TMP0:%.*]] = load <4 x float>, ptr [[A]], align 16
 ; DEFAULT-NEXT:    [[VECEXT:%.*]] = extractelement <4 x float> [[TMP0]], i32 0
 ; DEFAULT-NEXT:    [[TMP1:%.*]] = tail call fast float @llvm.tanh.f32(float [[VECEXT]])
-; DEFAULT-NEXT:    [[VECINS:%.*]] = insertelement <4 x float> undef, float [[TMP1]], i32 0
+; DEFAULT-NEXT:    [[VECINS:%.*]] = insertelement <4 x float> poison, float [[TMP1]], i32 0
 ; DEFAULT-NEXT:    [[VECEXT_1:%.*]] = extractelement <4 x float> [[TMP0]], i32 1
 ; DEFAULT-NEXT:    [[TMP2:%.*]] = tail call fast float @llvm.tanh.f32(float [[VECEXT_1]])
 ; DEFAULT-NEXT:    [[VECINS_1:%.*]] = insertelement <4 x float> [[VECINS]], float [[TMP2]], i32 1
@@ -1610,7 +1610,7 @@ entry:
   %0 = load <4 x float>, ptr %a, align 16
   %vecext = extractelement <4 x float> %0, i32 0
   %1 = tail call fast float @llvm.tanh.f32(float %vecext)
-  %vecins = insertelement <4 x float> undef, float %1, i32 0
+  %vecins = insertelement <4 x float> poison, float %1, i32 0
   %vecext.1 = extractelement <4 x float> %0, i32 1
   %2 = tail call fast float @llvm.tanh.f32(float %vecext.1)
   %vecins.1 = insertelement <4 x float> %vecins, float %2, i32 1
@@ -1633,7 +1633,7 @@ define <4 x float> @atanh_4x(ptr %a) {
 ; CHECK-NEXT:    [[TMP0:%.*]] = load <4 x float>, ptr [[A]], align 16
 ; CHECK-NEXT:    [[VECEXT:%.*]] = extractelement <4 x float> [[TMP0]], i32 0
 ; CHECK-NEXT:    [[TMP1:%.*]] = tail call fast float @atanhf(float [[VECEXT]])
-; CHECK-NEXT:    [[VECINS:%.*]] = insertelement <4 x float> undef, float [[TMP1]], i32 0
+; CHECK-NEXT:    [[VECINS:%.*]] = insertelement <4 x float> poison, float [[TMP1]], i32 0
 ; CHECK-NEXT:    [[VECEXT_1:%.*]] = extractelement <4 x float> [[TMP0]], i32 1
 ; CHECK-NEXT:    [[TMP2:%.*]] = tail call fast float @atanhf(float [[VECEXT_1]])
 ; CHECK-NEXT:    [[VECINS_1:%.*]] = insertelement <4 x float> [[VECINS]], float [[TMP2]], i32 1
@@ -1651,7 +1651,7 @@ define <4 x float> @atanh_4x(ptr %a) {
 ; DEFAULT-NEXT:    [[TMP0:%.*]] = load <4 x float>, ptr [[A]], align 16
 ; DEFAULT-NEXT:    [[VECEXT:%.*]] = extractelement <4 x float> [[TMP0]], i32 0
 ; DEFAULT-NEXT:    [[TMP1:%.*]] = tail call fast float @atanhf(float [[VECEXT]])
-; DEFAULT-NEXT:    [[VECINS:%.*]] = insertelement <4 x float> undef, float [[TMP1]], i32 0
+; DEFAULT-NEXT:    [[VECINS:%.*]] = insertelement <4 x float> poison, float [[TMP1]], i32 0
 ; DEFAULT-NEXT:    [[VECEXT_1:%.*]] = extractelement <4 x float> [[TMP0]], i32 1
 ; DEFAULT-NEXT:    [[TMP2:%.*]] = tail call fast float @atanhf(float [[VECEXT_1]])
 ; DEFAULT-NEXT:    [[VECINS_1:%.*]] = insertelement <4 x float> [[VECINS]], float [[TMP2]], i32 1
@@ -1667,7 +1667,7 @@ entry:
   %0 = load <4 x float>, ptr %a, align 16
   %vecext = extractelement <4 x float> %0, i32 0
   %1 = tail call fast float @atanhf(float %vecext)
-  %vecins = insertelement <4 x float> undef, float %1, i32 0
+  %vecins = insertelement <4 x float> poison, float %1, i32 0
   %vecext.1 = extractelement <4 x float> %0, i32 1
   %2 = tail call fast float @atanhf(float %vecext.1)
   %vecins.1 = insertelement <4 x float> %vecins, float %2, i32 1
@@ -1690,7 +1690,7 @@ define <4 x float> @int_atanh_4x(ptr %a) {
 ; CHECK-NEXT:    [[TMP0:%.*]] = load <4 x float>, ptr [[A]], align 16
 ; CHECK-NEXT:    [[VECEXT:%.*]] = extractelement <4 x float> [[TMP0]], i32 0
 ; CHECK-NEXT:    [[TMP1:%.*]] = tail call fast float @llvm.atanh.f32(float [[VECEXT]])
-; CHECK-NEXT:    [[VECINS:%.*]] = insertelement <4 x float> undef, float [[TMP1]], i32 0
+; CHECK-NEXT:    [[VECINS:%.*]] = insertelement <4 x float> poison, float [[TMP1]], i32 0
 ; CHECK-NEXT:    [[VECEXT_1:%.*]] = extractelement <4 x float> [[TMP0]], i32 1
 ; CHECK-NEXT:    [[TMP2:%.*]] = tail call fast float @llvm.atanh.f32(float [[VECEXT_1]])
 ; CHECK-NEXT:    [[VECINS_1:%.*]] = insertelement <4 x float> [[VECINS]], float [[TMP2]], i32 1
@@ -1708,7 +1708,7 @@ define <4 x float> @int_atanh_4x(ptr %a) {
 ; DEFAULT-NEXT:    [[TMP0:%.*]] = load <4 x float>, ptr [[A]], align 16
 ; DEFAULT-NEXT:    [[VECEXT:%.*]] = extractelement <4 x float> [[TMP0]], i32 0
 ; DEFAULT-NEXT:    [[TMP1:%.*]] = tail call fast float @llvm.atanh.f32(float [[VECEXT]])
-; DEFAULT-NEXT:    [[VECINS:%.*]] = insertelement <4 x float> undef, float [[TMP1]], i32 0
+; DEFAULT-NEXT:    [[VECINS:%.*]] = insertelement <4 x float> poison, float [[TMP1]], i32 0
 ; DEFAULT-NEXT:    [[VECEXT_1:%.*]] = extractelement <4 x float> [[TMP0]], i32 1
 ; DEFAULT-NEXT:    [[TMP2:%.*]] = tail call fast float @llvm.atanh.f32(float [[VECEXT_1]])
 ; DEFAULT-NEXT:    [[VECINS_1:%.*]] = insertelement <4 x float> [[VECINS]], float [[TMP2]], i32 1
@@ -1724,7 +1724,7 @@ entry:
   %0 = load <4 x float>, ptr %a, align 16
   %vecext = extractelement <4 x float> %0, i32 0
   %1 = tail call fast float @llvm.atanh.f32(float %vecext)
-  %vecins = insertelement <4 x float> undef, float %1, i32 0
+  %vecins = insertelement <4 x float> poison, float %1, i32 0
   %vecext.1 = extractelement <4 x float> %0, i32 1
   %2 = tail call fast float @llvm.atanh.f32(float %vecext.1)
   %vecins.1 = insertelement <4 x float> %vecins, float %2, i32 1
diff --git a/llvm/test/Transforms/SLPVectorizer/RISCV/partial-vec-invalid-cost.ll b/llvm/test/Transforms/SLPVectorizer/RISCV/partial-vec-invalid-cost.ll
index 6388cc2dedc73..085d7a64fc9ac 100644
--- a/llvm/test/Transforms/SLPVectorizer/RISCV/partial-vec-invalid-cost.ll
+++ b/llvm/test/Transforms/SLPVectorizer/RISCV/partial-vec-invalid-cost.ll
@@ -7,9 +7,17 @@ define void @partial_vec_invalid_cost() #0 {
 ; CHECK-LABEL: define void @partial_vec_invalid_cost(
 ; CHECK-SAME: ) #[[ATTR0:[0-9]+]] {
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[TMP0:%.*]] = call i32 @llvm.vector.reduce.or.v4i32(<4 x i32> zeroinitializer)
+; CHECK-NEXT:    [[LSHR_1:%.*]] = lshr i96 0, 0
+; CHECK-NEXT:    [[LSHR_2:%.*]] = lshr i96 0, 0
+; CHECK-NEXT:    [[TRUNC_I96_1:%.*]] = trunc i96 [[LSHR_1]] to i32
+; CHECK-NEXT:    [[TRUNC_I96_2:%.*]] = trunc i96 [[LSHR_2]] to i32
+; CHECK-NEXT:    [[TRUNC_I96_3:%.*]] = trunc i96 0 to i32
+; CHECK-NEXT:    [[TRUNC_I96_4:%.*]] = trunc i96 0 to i32
 ; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @llvm.vector.reduce.or.v4i32(<4 x i32> zeroinitializer)
-; CHECK-NEXT:    [[OP_RDX3:%.*]] = or i32 [[TMP0]], [[TMP1]]
+; CHECK-NEXT:    [[OP_RDX:%.*]] = or i32 [[TMP1]], [[TRUNC_I96_1]]
+; CHECK-NEXT:    [[OP_RDX1:%.*]] = or i32 [[TRUNC_I96_2]], [[TRUNC_I96_3]]
+; CHECK-NEXT:    [[OP_RDX2:%.*]] = or i32 [[OP_RDX]], [[OP_RDX1]]
+; CHECK-NEXT:    [[OP_RDX3:%.*]] = or i32 [[OP_RDX2]], [[TRUNC_I96_4]]
 ; CHECK-NEXT:    [[STORE_THIS:%.*]] = zext i32 [[OP_RDX3]] to i96
 ; CHECK-NEXT:    store i96 [[STORE_THIS]], ptr null, align 16
 ; CHECK-NEXT:    ret void
diff --git a/llvm/test/Transforms/SLPVectorizer/RISCV/revec-getGatherCost.ll b/llvm/test/Transforms/SLPVectorizer/RISCV/revec-getGatherCost.ll
index 2565d5be32e20..258b0ec0bcfc7 100644
--- a/llvm/test/Transforms/SLPVectorizer/RISCV/revec-getGatherCost.ll
+++ b/llvm/test/Transforms/SLPVectorizer/RISCV/revec-getGatherCost.ll
@@ -8,7 +8,7 @@
 ; YAML: Function:        test1
 ; YAML: Args:
 ; YAML:   - String:          'Stores SLP vectorized with cost '
-; YAML:   - Cost:            '4'
+; YAML:   - Cost:            '5'
 ; YAML:   - String:          ' and with tree size '
 ; YAML:   - TreeSize:        '5'
 
@@ -46,7 +46,7 @@ declare <4 x float> @llvm.fmuladd.v4f32(<4 x float>, <4 x float>, <4 x float>)
 ; YAML: Function:        test2
 ; YAML: Args:
 ; YAML:   - String:          'Stores SLP vectorized with cost '
-; YAML:   - Cost:            '12'
+; YAML:   - Cost:            '14'
 ; YAML:   - String:          ' and with tree size '
 ; YAML:   - TreeSize:        '5'
 
diff --git a/llvm/test/Transforms/SLPVectorizer/RISCV/revec.ll b/llvm/test/Transforms/SLPVectorizer/RISCV/revec.ll
index 61ff4f5766d30..dd7a21198ac1f 100644
--- a/llvm/test/Transforms/SLPVectorizer/RISCV/revec.ll
+++ b/llvm/test/Transforms/SLPVectorizer/RISCV/revec.ll
@@ -231,3 +231,40 @@ define ptr @test4() {
   %28 = tail call float @llvm.sqrt.f32(float %26)
   ret ptr null
 }
+
+define i32 @test5() {
+; CHECK-LABEL: @test5(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = call <4 x double> @llvm.vector.insert.v4f64.v2f64(<4 x double> poison, <2 x double> zeroinitializer, i64 0)
+; CHECK-NEXT:    [[TMP1:%.*]] = call <4 x double> @llvm.vector.insert.v4f64.v2f64(<4 x double> [[TMP0]], <2 x double> zeroinitializer, i64 2)
+; CHECK-NEXT:    [[TMP2:%.*]] = fdiv <4 x double> [[TMP1]], [[TMP1]]
+; CHECK-NEXT:    [[TMP3:%.*]] = call <8 x double> @llvm.vector.insert.v8f64.v2f64(<8 x double> poison, <2 x double> zeroinitializer, i64 0)
+; CHECK-NEXT:    [[TMP4:%.*]] = call <8 x double> @llvm.vector.insert.v8f64.v2f64(<8 x double> [[TMP3]], <2 x double> zeroinitializer, i64 2)
+; CHECK-NEXT:    [[TMP5:%.*]] = call <8 x double> @llvm.vector.insert.v8f64.v2f64(<8 x double> [[TMP4]], <2 x double> zeroinitializer, i64 4)
+; CHECK-NEXT:    [[TMP6:%.*]] = call <8 x double> @llvm.vector.insert.v8f64.v2f64(<8 x double> [[TMP5]], <2 x double> zeroinitializer, i64 6)
+; CHECK-NEXT:    [[TMP7:%.*]] = call <8 x double> @llvm.vector.insert.v8f64.v2f64(<8 x double> poison, <2 x double> zeroinitializer, i64 2)
+; CHECK-NEXT:    [[TMP8:%.*]] = call <8 x double> @llvm.vector.insert.v8f64.v2f64(<8 x double> [[TMP7]], <2 x double> zeroinitializer, i64 6)
+; CHECK-NEXT:    [[TMP9:%.*]] = call <8 x double> @llvm.vector.insert.v8f64.v4f64(<8 x double> poison, <4 x double> [[TMP2]], i64 0)
+; CHECK-NEXT:    [[TMP10:%.*]] = shufflevector <8 x double> [[TMP9]], <8 x double> [[TMP8]], <8 x i32> <i32 0, i32 1, i32 10, i32 11, i32 2, i32 3, i32 14, i32 15>
+; CHECK-NEXT:    [[TMP11:%.*]] = fadd <8 x double> [[TMP6]], [[TMP10]]
+; CHECK-NEXT:    br label [[FOR_END47:%.*]]
+; CHECK:       for.end47:
+; CHECK-NEXT:    [[TMP12:%.*]] = phi <8 x double> [ [[TMP11]], [[ENTRY:%.*]] ]
+; CHECK-NEXT:    ret i32 0
+;
+entry:
+  %div0 = fdiv <2 x double> zeroinitializer, zeroinitializer
+  %div1 = fdiv <2 x double> zeroinitializer, zeroinitializer
+  %add0 = fadd <2 x double> zeroinitializer, %div0
+  %add1 = fadd <2 x double> zeroinitializer, zeroinitializer
+  %add2 = fadd <2 x double> %div1, zeroinitializer
+  %add3 = fadd <2 x double> zeroinitializer, zeroinitializer
+  br label %for.end47
+
+for.end47:                                        ; preds = %entry
+  %add0.lcssa = phi <2 x double> [ %add0, %entry ]
+  %add1.lcssa = phi <2 x double> [ %add1, %entry ]
+  %add2.lcssa = phi <2 x double> [ %add2, %entry ]
+  %add3.lcssa = phi <2 x double> [ %add3, %entry ]
+  ret i32 0
+}
diff --git a/llvm/test/Transforms/SLPVectorizer/SystemZ/reductions-fadd.ll b/llvm/test/Transforms/SLPVectorizer/SystemZ/reductions-fadd.ll
index fa0587f1da931..5f8dfa370dacb 100644
--- a/llvm/test/Transforms/SLPVectorizer/SystemZ/reductions-fadd.ll
+++ b/llvm/test/Transforms/SLPVectorizer/SystemZ/reductions-fadd.ll
@@ -7,7 +7,7 @@
 
 define double @fadd_double_4_addends_seq(ptr nocapture noundef readonly %x) {
 ; CHECK-LABEL: define double @fadd_double_4_addends_seq(
-; CHECK-SAME: ptr nocapture noundef readonly [[X:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-SAME: ptr noundef readonly captures(none) [[X:%.*]]) #[[ATTR0:[0-9]+]] {
 ; CHECK-NEXT:  [[ENTRY:.*:]]
 ; CHECK-NEXT:    [[TMP0:%.*]] = load <4 x double>, ptr [[X]], align 8
 ; CHECK-NEXT:    [[TMP1:%.*]] = call reassoc nsz arcp contract afn double @llvm.vector.reduce.fadd.v4f64(double 0.000000e+00, <4 x double> [[TMP0]])
@@ -29,7 +29,7 @@ entry:
 
 define double @fadd_double_8_addends_nonseq(ptr nocapture noundef readonly %x)  {
 ; CHECK-LABEL: define double @fadd_double_8_addends_nonseq(
-; CHECK-SAME: ptr nocapture noundef readonly [[X:%.*]]) #[[ATTR0]] {
+; CHECK-SAME: ptr noundef readonly captures(none) [[X:%.*]]) #[[ATTR0]] {
 ; CHECK-NEXT:  [[ENTRY:.*:]]
 ; CHECK-NEXT:    [[TMP0:%.*]] = load double, ptr [[X]], align 8
 ; CHECK-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds double, ptr [[X]], i64 2
@@ -85,7 +85,7 @@ entry:
 
 define float @fadd_float_16_addends_nonseq(ptr nocapture noundef readonly %x)  {
 ; CHECK-LABEL: define float @fadd_float_16_addends_nonseq(
-; CHECK-SAME: ptr nocapture noundef readonly [[X:%.*]]) #[[ATTR0]] {
+; CHECK-SAME: ptr noundef readonly captures(none) [[X:%.*]]) #[[ATTR0]] {
 ; CHECK-NEXT:  [[ENTRY:.*:]]
 ; CHECK-NEXT:    [[TMP0:%.*]] = load float, ptr [[X]], align 4
 ; CHECK-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds float, ptr [[X]], i64 2
diff --git a/llvm/test/Transforms/SLPVectorizer/SystemZ/reductions-fmin-fmax.ll b/llvm/test/Transforms/SLPVectorizer/SystemZ/reductions-fmin-fmax.ll
index 5ea777e1c9a10..1b05f31cfe4b1 100644
--- a/llvm/test/Transforms/SLPVectorizer/SystemZ/reductions-fmin-fmax.ll
+++ b/llvm/test/Transforms/SLPVectorizer/SystemZ/reductions-fmin-fmax.ll
@@ -7,7 +7,7 @@
 
 define double @fmin_double_4_nums_seq(ptr nocapture noundef readonly %x) {
 ; CHECK-LABEL: define double @fmin_double_4_nums_seq(
-; CHECK-SAME: ptr nocapture noundef readonly [[X:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-SAME: ptr noundef readonly captures(none) [[X:%.*]]) #[[ATTR0:[0-9]+]] {
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x double>, ptr [[X]], align 4
 ; CHECK-NEXT:    [[TMP2:%.*]] = call fast double @llvm.vector.reduce.fmin.v4f64(<4 x double> [[TMP1]])
 ; CHECK-NEXT:    ret double [[TMP2]]
@@ -27,7 +27,7 @@ define double @fmin_double_4_nums_seq(ptr nocapture noundef readonly %x) {
 
 define double @fmin_double_16_nums_nonseq(ptr nocapture noundef readonly %x) {
 ; CHECK-LABEL: define double @fmin_double_16_nums_nonseq(
-; CHECK-SAME: ptr nocapture noundef readonly [[X:%.*]]) #[[ATTR0]] {
+; CHECK-SAME: ptr noundef readonly captures(none) [[X:%.*]]) #[[ATTR0]] {
 ; CHECK-NEXT:    [[G1:%.*]] = getelementptr inbounds double, ptr [[X]], i64 2
 ; CHECK-NEXT:    [[G2:%.*]] = getelementptr inbounds double, ptr [[X]], i64 4
 ; CHECK-NEXT:    [[G3:%.*]] = getelementptr inbounds double, ptr [[X]], i64 6
@@ -129,7 +129,7 @@ define double @fmin_double_16_nums_nonseq(ptr nocapture noundef readonly %x) {
 
 define float @fmin_float_12_nums_nonseq(ptr nocapture noundef readonly %x) {
 ; CHECK-LABEL: define float @fmin_float_12_nums_nonseq(
-; CHECK-SAME: ptr nocapture noundef readonly [[X:%.*]]) #[[ATTR0]] {
+; CHECK-SAME: ptr noundef readonly captures(none) [[X:%.*]]) #[[ATTR0]] {
 ; CHECK-NEXT:    [[G1:%.*]] = getelementptr inbounds float, ptr [[X]], i64 2
 ; CHECK-NEXT:    [[G2:%.*]] = getelementptr inbounds float, ptr [[X]], i64 4
 ; CHECK-NEXT:    [[G3:%.*]] = getelementptr inbounds float, ptr [[X]], i64 6
@@ -207,7 +207,7 @@ define float @fmin_float_12_nums_nonseq(ptr nocapture noundef readonly %x) {
 
 define double @fmax_double_4_nums_seq(ptr nocapture noundef readonly %x) {
 ; CHECK-LABEL: define double @fmax_double_4_nums_seq(
-; CHECK-SAME: ptr nocapture noundef readonly [[X:%.*]]) #[[ATTR0]] {
+; CHECK-SAME: ptr noundef readonly captures(none) [[X:%.*]]) #[[ATTR0]] {
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x double>, ptr [[X]], align 4
 ; CHECK-NEXT:    [[TMP2:%.*]] = call fast double @llvm.vector.reduce.fmax.v4f64(<4 x double> [[TMP1]])
 ; CHECK-NEXT:    ret double [[TMP2]]
@@ -227,7 +227,7 @@ define double @fmax_double_4_nums_seq(ptr nocapture noundef readonly %x) {
 
 define double @fmax_double_16_nums_nonseq(ptr nocapture noundef readonly %x) {
 ; CHECK-LABEL: define double @fmax_double_16_nums_nonseq(
-; CHECK-SAME: ptr nocapture noundef readonly [[X:%.*]]) #[[ATTR0]] {
+; CHECK-SAME: ptr noundef readonly captures(none) [[X:%.*]]) #[[ATTR0]] {
 ; CHECK-NEXT:    [[G1:%.*]] = getelementptr inbounds double, ptr [[X]], i64 2
 ; CHECK-NEXT:    [[G2:%.*]] = getelementptr inbounds double, ptr [[X]], i64 4
 ; CHECK-NEXT:    [[G3:%.*]] = getelementptr inbounds double, ptr [[X]], i64 6
@@ -329,7 +329,7 @@ define double @fmax_double_16_nums_nonseq(ptr nocapture noundef readonly %x) {
 
 define float @fmax_float_12_nums_nonseq(ptr nocapture noundef readonly %x) {
 ; CHECK-LABEL: define float @fmax_float_12_nums_nonseq(
-; CHECK-SAME: ptr nocapture noundef readonly [[X:%.*]]) #[[ATTR0]] {
+; CHECK-SAME: ptr noundef readonly captures(none) [[X:%.*]]) #[[ATTR0]] {
 ; CHECK-NEXT:    [[G1:%.*]] = getelementptr inbounds float, ptr [[X]], i64 2
 ; CHECK-NEXT:    [[G2:%.*]] = getelementptr inbounds float, ptr [[X]], i64 4
 ; CHECK-NEXT:    [[G3:%.*]] = getelementptr inbounds float, ptr [[X]], i64 6
diff --git a/llvm/test/Transforms/SLPVectorizer/SystemZ/reductions-fmul.ll b/llvm/test/Transforms/SLPVectorizer/SystemZ/reductions-fmul.ll
index e08b38c69a840..fbdca10ca6ced 100644
--- a/llvm/test/Transforms/SLPVectorizer/SystemZ/reductions-fmul.ll
+++ b/llvm/test/Transforms/SLPVectorizer/SystemZ/reductions-fmul.ll
@@ -7,7 +7,7 @@
 
 define double @fmul_double_4_factors_seq(ptr nocapture noundef readonly %x) {
 ; CHECK-LABEL: define double @fmul_double_4_factors_seq(
-; CHECK-SAME: ptr nocapture noundef readonly [[X:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-SAME: ptr noundef readonly captures(none) [[X:%.*]]) #[[ATTR0:[0-9]+]] {
 ; CHECK-NEXT:  [[ENTRY:.*:]]
 ; CHECK-NEXT:    [[TMP0:%.*]] = load <4 x double>, ptr [[X]], align 8
 ; CHECK-NEXT:    [[TMP1:%.*]] = call reassoc nsz arcp contract afn double @llvm.vector.reduce.fmul.v4f64(double 1.000000e+00, <4 x double> [[TMP0]])
@@ -29,7 +29,7 @@ entry:
 
 define double @fmul_double_8_factors_nonseq(ptr nocapture noundef readonly %x) {
 ; CHECK-LABEL: define double @fmul_double_8_factors_nonseq(
-; CHECK-SAME: ptr nocapture noundef readonly [[X:%.*]]) #[[ATTR0]] {
+; CHECK-SAME: ptr noundef readonly captures(none) [[X:%.*]]) #[[ATTR0]] {
 ; CHECK-NEXT:  [[ENTRY:.*:]]
 ; CHECK-NEXT:    [[TMP0:%.*]] = load double, ptr [[X]], align 8
 ; CHECK-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds double, ptr [[X]], i64 2
@@ -85,7 +85,7 @@ entry:
 
 define float @fmul_float_16_factors_nonseq(float noundef %m, ptr nocapture noundef readonly %x) {
 ; CHECK-LABEL: define float @fmul_float_16_factors_nonseq(
-; CHECK-SAME: float noundef [[M:%.*]], ptr nocapture noundef readonly [[X:%.*]]) #[[ATTR0]] {
+; CHECK-SAME: float noundef [[M:%.*]], ptr noundef readonly captures(none) [[X:%.*]]) #[[ATTR0]] {
 ; CHECK-NEXT:  [[ENTRY:.*:]]
 ; CHECK-NEXT:    [[TMP0:%.*]] = load float, ptr [[X]], align 4
 ; CHECK-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds float, ptr [[X]], i64 2
diff --git a/llvm/test/Transforms/SLPVectorizer/SystemZ/reuse-non-power-of-2-reorder.ll b/llvm/test/Transforms/SLPVectorizer/SystemZ/reuse-non-power-of-2-reorder.ll
new file mode 100644
index 0000000000000..5681fb7346124
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/SystemZ/reuse-non-power-of-2-reorder.ll
@@ -0,0 +1,155 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -S --passes=slp-vectorizer -mtriple=systemz -mcpu=z15 < %s -slp-threshold=-100 | FileCheck %s
+
+define void @test(i32 %0, i64 %1, i32 %2, i32 %3, ptr %4) {
+; CHECK-LABEL: define void @test(
+; CHECK-SAME: i32 [[TMP0:%.*]], i64 [[TMP1:%.*]], i32 [[TMP2:%.*]], i32 [[TMP3:%.*]], ptr [[TMP4:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT:    [[TMP56:%.*]] = trunc i64 [[TMP1]] to i32
+; CHECK-NEXT:    [[TMP7:%.*]] = insertelement <2 x i32> poison, i32 [[TMP56]], i32 0
+; CHECK-NEXT:    [[TMP8:%.*]] = shufflevector <2 x i32> [[TMP7]], <2 x i32> poison, <2 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP73:%.*]] = trunc i64 [[TMP1]] to i32
+; CHECK-NEXT:    [[TMP98:%.*]] = trunc i64 [[TMP1]] to i32
+; CHECK-NEXT:    [[TMP11:%.*]] = insertelement <60 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>, i32 [[TMP0]], i32 7
+; CHECK-NEXT:    [[TMP12:%.*]] = shufflevector <60 x i32> [[TMP11]], <60 x i32> poison, <60 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 7, i32 7, i32 7, i32 7, i32 7>
+; CHECK-NEXT:    [[TMP13:%.*]] = insertelement <60 x i32> poison, i32 [[TMP3]], i32 1
+; CHECK-NEXT:    [[TMP14:%.*]] = insertelement <60 x i32> [[TMP13]], i32 [[TMP2]], i32 7
+; CHECK-NEXT:    [[TMP15:%.*]] = insertelement <60 x i32> [[TMP14]], i32 [[TMP98]], i32 0
+; CHECK-NEXT:    [[TMP16:%.*]] = insertelement <60 x i32> [[TMP15]], i32 [[TMP73]], i32 6
+; CHECK-NEXT:    [[TMP17:%.*]] = shufflevector <60 x i32> [[TMP16]], <60 x i32> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 6, i32 7>
+; CHECK-NEXT:    [[TMP18:%.*]] = call <8 x i32> @llvm.vector.insert.v8i32.v2i32(<8 x i32> [[TMP17]], <2 x i32> [[TMP8]], i64 2)
+; CHECK-NEXT:    [[TMP19:%.*]] = call <8 x i32> @llvm.vector.insert.v8i32.v2i32(<8 x i32> [[TMP18]], <2 x i32> [[TMP8]], i64 4)
+; CHECK-NEXT:    [[TMP20:%.*]] = shufflevector <8 x i32> [[TMP19]], <8 x i32> poison, <60 x i32> <i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 4, i32 4, i32 4, i32 4, i32 4, i32 4, i32 4, i32 4, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 6, i32 6, i32 6, i32 6, i32 6, i32 6, i32 6, i32 6, i32 7, i32 7, i32 7, i32 7>
+; CHECK-NEXT:    [[TMP21:%.*]] = xor <60 x i32> [[TMP12]], [[TMP20]]
+; CHECK-NEXT:    [[TMP130:%.*]] = call i32 @llvm.vector.reduce.or.v60i32(<60 x i32> [[TMP21]])
+; CHECK-NEXT:    store i32 [[TMP130]], ptr [[TMP4]], align 4
+; CHECK-NEXT:    ret void
+;
+  %6 = trunc i64 %1 to i32
+  %7 = xor i32 %6, 1
+  %8 = xor i32 %6, 1
+  %9 = or i32 %7, %8
+  %10 = xor i32 %6, 1
+  %11 = or i32 %9, %10
+  %12 = xor i32 %6, 1
+  %13 = or i32 %11, %12
+  %14 = xor i32 %6, 1
+  %15 = or i32 %13, %14
+  %16 = xor i32 %6, 1
+  %17 = or i32 %15, %16
+  %18 = xor i32 %6, 1
+  %19 = or i32 %17, %18
+  %20 = xor i32 %6, 1
+  %21 = or i32 %19, %20
+  %22 = trunc i64 %1 to i32
+  %23 = xor i32 %22, 1
+  %24 = or i32 %23, %21
+  %25 = xor i32 %22, 1
+  %26 = or i32 %24, %25
+  %27 = xor i32 %22, 1
+  %28 = or i32 %26, %27
+  %29 = xor i32 %22, 1
+  %30 = or i32 %28, %29
+  %31 = xor i32 %22, 1
+  %32 = or i32 %30, %31
+  %33 = xor i32 %22, 1
+  %34 = or i32 %32, %33
+  %35 = xor i32 %22, 1
+  %36 = or i32 %34, %35
+  %37 = xor i32 %22, 1
+  %38 = or i32 %36, %37
+  %39 = trunc i64 %1 to i32
+  %40 = xor i32 %39, 1
+  %41 = or i32 %40, %38
+  %42 = xor i32 %39, 1
+  %43 = or i32 %41, %42
+  %44 = xor i32 %39, 1
+  %45 = or i32 %43, %44
+  %46 = xor i32 %39, 1
+  %47 = or i32 %45, %46
+  %48 = xor i32 %39, 1
+  %49 = or i32 %47, %48
+  %50 = xor i32 %39, 1
+  %51 = or i32 %49, %50
+  %52 = xor i32 %39, 1
+  %53 = or i32 %51, %52
+  %54 = xor i32 %39, 1
+  %55 = or i32 %53, %54
+  %56 = trunc i64 %1 to i32
+  %57 = xor i32 %56, 1
+  %58 = or i32 %57, %55
+  %59 = xor i32 %56, 1
+  %60 = or i32 %58, %59
+  %61 = xor i32 %56, 1
+  %62 = or i32 %60, %61
+  %63 = xor i32 %56, 1
+  %64 = or i32 %62, %63
+  %65 = xor i32 %56, 1
+  %66 = or i32 %64, %65
+  %67 = xor i32 %56, 1
+  %68 = or i32 %66, %67
+  %69 = xor i32 %56, 1
+  %70 = or i32 %68, %69
+  %71 = xor i32 %56, 1
+  %72 = or i32 %70, %71
+  %73 = trunc i64 %1 to i32
+  %74 = xor i32 %73, 1
+  %75 = or i32 %74, %72
+  %76 = xor i32 %73, 1
+  %77 = or i32 %75, %76
+  %78 = xor i32 %73, 1
+  %79 = or i32 %77, %78
+  %80 = xor i32 %73, 1
+  %81 = or i32 %79, %80
+  %82 = xor i32 %73, 1
+  %83 = or i32 %81, %82
+  %84 = xor i32 %73, 1
+  %85 = or i32 %83, %84
+  %86 = xor i32 %73, 1
+  %87 = or i32 %85, %86
+  %88 = xor i32 %0, %73
+  %89 = or i32 %87, %88
+  %90 = xor i32 %0, %2
+  %91 = or i32 %90, %89
+  %92 = xor i32 %0, %2
+  %93 = or i32 %91, %92
+  %94 = xor i32 %0, %2
+  %95 = or i32 %93, %94
+  %96 = xor i32 %0, %2
+  %97 = or i32 %95, %96
+  %98 = trunc i64 %1 to i32
+  %99 = xor i32 %98, 1
+  %100 = xor i32 %98, 1
+  %101 = or i32 %99, %100
+  %102 = xor i32 %98, 1
+  %103 = or i32 %101, %102
+  %104 = xor i32 %98, 1
+  %105 = or i32 %103, %104
+  %106 = xor i32 %98, 1
+  %107 = or i32 %105, %106
+  %108 = xor i32 %98, 1
+  %109 = or i32 %107, %108
+  %110 = xor i32 %98, 1
+  %111 = or i32 %109, %110
+  %112 = xor i32 %0, %98
+  %113 = or i32 %111, %112
+  %114 = xor i32 %0, %3
+  %115 = or i32 %113, %114
+  %116 = xor i32 %0, %3
+  %117 = or i32 %115, %116
+  %118 = xor i32 %0, %3
+  %119 = or i32 %117, %118
+  %120 = xor i32 %0, %3
+  %121 = or i32 %119, %120
+  %122 = xor i32 %0, %3
+  %123 = or i32 %121, %122
+  %124 = xor i32 %0, %3
+  %125 = or i32 %123, %124
+  %126 = xor i32 %0, %3
+  %127 = or i32 %125, %126
+  %128 = xor i32 %0, %3
+  %129 = or i32 %127, %128
+  %130 = or i32 %129, %97
+  store i32 %130, ptr %4, align 4
+  ret void
+}
+
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/alternate-calls-inseltpoison.ll b/llvm/test/Transforms/SLPVectorizer/X86/alternate-calls-inseltpoison.ll
index 78dd39be777cd..b790e6f3c99c6 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/alternate-calls-inseltpoison.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/alternate-calls-inseltpoison.ll
@@ -9,23 +9,29 @@
 define <8 x float> @ceil_floor(<8 x float> %a) {
 ; SSE-LABEL: @ceil_floor(
 ; SSE-NEXT:    [[A0:%.*]] = extractelement <8 x float> [[A:%.*]], i64 0
+; SSE-NEXT:    [[A1:%.*]] = extractelement <8 x float> [[A]], i64 1
+; SSE-NEXT:    [[A2:%.*]] = extractelement <8 x float> [[A]], i64 2
 ; SSE-NEXT:    [[A3:%.*]] = extractelement <8 x float> [[A]], i64 3
+; SSE-NEXT:    [[A4:%.*]] = extractelement <8 x float> [[A]], i64 4
+; SSE-NEXT:    [[A5:%.*]] = extractelement <8 x float> [[A]], i64 5
+; SSE-NEXT:    [[A6:%.*]] = extractelement <8 x float> [[A]], i64 6
+; SSE-NEXT:    [[A7:%.*]] = extractelement <8 x float> [[A]], i64 7
 ; SSE-NEXT:    [[AB0:%.*]] = call float @llvm.ceil.f32(float [[A0]])
-; SSE-NEXT:    [[TMP1:%.*]] = shufflevector <8 x float> [[A]], <8 x float> poison, <2 x i32> <i32 1, i32 2>
-; SSE-NEXT:    [[TMP2:%.*]] = call <2 x float> @llvm.floor.v2f32(<2 x float> [[TMP1]])
+; SSE-NEXT:    [[AB1:%.*]] = call float @llvm.floor.f32(float [[A1]])
+; SSE-NEXT:    [[AB2:%.*]] = call float @llvm.floor.f32(float [[A2]])
 ; SSE-NEXT:    [[AB3:%.*]] = call float @llvm.ceil.f32(float [[A3]])
-; SSE-NEXT:    [[TMP3:%.*]] = shufflevector <8 x float> [[A]], <8 x float> poison, <2 x i32> <i32 4, i32 5>
-; SSE-NEXT:    [[TMP4:%.*]] = call <2 x float> @llvm.ceil.v2f32(<2 x float> [[TMP3]])
-; SSE-NEXT:    [[TMP5:%.*]] = shufflevector <8 x float> [[A]], <8 x float> poison, <2 x i32> <i32 6, i32 7>
-; SSE-NEXT:    [[TMP6:%.*]] = call <2 x float> @llvm.floor.v2f32(<2 x float> [[TMP5]])
+; SSE-NEXT:    [[AB4:%.*]] = call float @llvm.ceil.f32(float [[A4]])
+; SSE-NEXT:    [[AB5:%.*]] = call float @llvm.ceil.f32(float [[A5]])
+; SSE-NEXT:    [[AB6:%.*]] = call float @llvm.floor.f32(float [[A6]])
+; SSE-NEXT:    [[AB7:%.*]] = call float @llvm.floor.f32(float [[A7]])
 ; SSE-NEXT:    [[R0:%.*]] = insertelement <8 x float> poison, float [[AB0]], i64 0
-; SSE-NEXT:    [[TMP7:%.*]] = shufflevector <2 x float> [[TMP2]], <2 x float> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; SSE-NEXT:    [[R23:%.*]] = shufflevector <8 x float> [[R0]], <8 x float> [[TMP7]], <8 x i32> <i32 0, i32 8, i32 9, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE-NEXT:    [[R1:%.*]] = insertelement <8 x float> [[R0]], float [[AB1]], i64 1
+; SSE-NEXT:    [[R23:%.*]] = insertelement <8 x float> [[R1]], float [[AB2]], i64 2
 ; SSE-NEXT:    [[R3:%.*]] = insertelement <8 x float> [[R23]], float [[AB3]], i64 3
-; SSE-NEXT:    [[TMP8:%.*]] = shufflevector <2 x float> [[TMP4]], <2 x float> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; SSE-NEXT:    [[R52:%.*]] = shufflevector <8 x float> [[R3]], <8 x float> [[TMP8]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 poison, i32 poison>
-; SSE-NEXT:    [[TMP9:%.*]] = shufflevector <2 x float> [[TMP6]], <2 x float> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; SSE-NEXT:    [[R71:%.*]] = shufflevector <8 x float> [[R52]], <8 x float> [[TMP9]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 8, i32 9>
+; SSE-NEXT:    [[R4:%.*]] = insertelement <8 x float> [[R3]], float [[AB4]], i64 4
+; SSE-NEXT:    [[R5:%.*]] = insertelement <8 x float> [[R4]], float [[AB5]], i64 5
+; SSE-NEXT:    [[R6:%.*]] = insertelement <8 x float> [[R5]], float [[AB6]], i64 6
+; SSE-NEXT:    [[R71:%.*]] = insertelement <8 x float> [[R6]], float [[AB7]], i64 7
 ; SSE-NEXT:    ret <8 x float> [[R71]]
 ;
 ; SLM-LABEL: @ceil_floor(
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/alternate-calls.ll b/llvm/test/Transforms/SLPVectorizer/X86/alternate-calls.ll
index bfa5c56e935ad..ef1a67032c237 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/alternate-calls.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/alternate-calls.ll
@@ -9,23 +9,29 @@
 define <8 x float> @ceil_floor(<8 x float> %a) {
 ; SSE-LABEL: @ceil_floor(
 ; SSE-NEXT:    [[A0:%.*]] = extractelement <8 x float> [[A:%.*]], i64 0
+; SSE-NEXT:    [[A1:%.*]] = extractelement <8 x float> [[A]], i64 1
+; SSE-NEXT:    [[A2:%.*]] = extractelement <8 x float> [[A]], i64 2
 ; SSE-NEXT:    [[A3:%.*]] = extractelement <8 x float> [[A]], i64 3
+; SSE-NEXT:    [[A4:%.*]] = extractelement <8 x float> [[A]], i64 4
+; SSE-NEXT:    [[A5:%.*]] = extractelement <8 x float> [[A]], i64 5
+; SSE-NEXT:    [[A6:%.*]] = extractelement <8 x float> [[A]], i64 6
+; SSE-NEXT:    [[A7:%.*]] = extractelement <8 x float> [[A]], i64 7
 ; SSE-NEXT:    [[AB0:%.*]] = call float @llvm.ceil.f32(float [[A0]])
-; SSE-NEXT:    [[TMP1:%.*]] = shufflevector <8 x float> [[A]], <8 x float> poison, <2 x i32> <i32 1, i32 2>
-; SSE-NEXT:    [[TMP2:%.*]] = call <2 x float> @llvm.floor.v2f32(<2 x float> [[TMP1]])
+; SSE-NEXT:    [[AB1:%.*]] = call float @llvm.floor.f32(float [[A1]])
+; SSE-NEXT:    [[AB2:%.*]] = call float @llvm.floor.f32(float [[A2]])
 ; SSE-NEXT:    [[AB3:%.*]] = call float @llvm.ceil.f32(float [[A3]])
-; SSE-NEXT:    [[TMP3:%.*]] = shufflevector <8 x float> [[A]], <8 x float> poison, <2 x i32> <i32 4, i32 5>
-; SSE-NEXT:    [[TMP4:%.*]] = call <2 x float> @llvm.ceil.v2f32(<2 x float> [[TMP3]])
-; SSE-NEXT:    [[TMP5:%.*]] = shufflevector <8 x float> [[A]], <8 x float> poison, <2 x i32> <i32 6, i32 7>
-; SSE-NEXT:    [[TMP6:%.*]] = call <2 x float> @llvm.floor.v2f32(<2 x float> [[TMP5]])
+; SSE-NEXT:    [[AB4:%.*]] = call float @llvm.ceil.f32(float [[A4]])
+; SSE-NEXT:    [[AB5:%.*]] = call float @llvm.ceil.f32(float [[A5]])
+; SSE-NEXT:    [[AB6:%.*]] = call float @llvm.floor.f32(float [[A6]])
+; SSE-NEXT:    [[AB7:%.*]] = call float @llvm.floor.f32(float [[A7]])
 ; SSE-NEXT:    [[R0:%.*]] = insertelement <8 x float> poison, float [[AB0]], i64 0
-; SSE-NEXT:    [[TMP7:%.*]] = shufflevector <2 x float> [[TMP2]], <2 x float> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; SSE-NEXT:    [[R23:%.*]] = shufflevector <8 x float> [[R0]], <8 x float> [[TMP7]], <8 x i32> <i32 0, i32 8, i32 9, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE-NEXT:    [[R1:%.*]] = insertelement <8 x float> [[R0]], float [[AB1]], i64 1
+; SSE-NEXT:    [[R23:%.*]] = insertelement <8 x float> [[R1]], float [[AB2]], i64 2
 ; SSE-NEXT:    [[R3:%.*]] = insertelement <8 x float> [[R23]], float [[AB3]], i64 3
-; SSE-NEXT:    [[TMP8:%.*]] = shufflevector <2 x float> [[TMP4]], <2 x float> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; SSE-NEXT:    [[R52:%.*]] = shufflevector <8 x float> [[R3]], <8 x float> [[TMP8]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 poison, i32 poison>
-; SSE-NEXT:    [[TMP9:%.*]] = shufflevector <2 x float> [[TMP6]], <2 x float> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; SSE-NEXT:    [[R71:%.*]] = shufflevector <8 x float> [[R52]], <8 x float> [[TMP9]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 8, i32 9>
+; SSE-NEXT:    [[R4:%.*]] = insertelement <8 x float> [[R3]], float [[AB4]], i64 4
+; SSE-NEXT:    [[R5:%.*]] = insertelement <8 x float> [[R4]], float [[AB5]], i64 5
+; SSE-NEXT:    [[R6:%.*]] = insertelement <8 x float> [[R5]], float [[AB6]], i64 6
+; SSE-NEXT:    [[R71:%.*]] = insertelement <8 x float> [[R6]], float [[AB7]], i64 7
 ; SSE-NEXT:    ret <8 x float> [[R71]]
 ;
 ; SLM-LABEL: @ceil_floor(
@@ -107,7 +113,7 @@ define <8 x float> @ceil_floor(<8 x float> %a) {
   %ab5 = call float @llvm.ceil.f32(float %a5)
   %ab6 = call float @llvm.floor.f32(float %a6)
   %ab7 = call float @llvm.floor.f32(float %a7)
-  %r0 = insertelement <8 x float> undef, float %ab0, i32 0
+  %r0 = insertelement <8 x float> zeroinitializer, float %ab0, i32 0
   %r1 = insertelement <8 x float>   %r0, float %ab1, i32 1
   %r2 = insertelement <8 x float>   %r1, float %ab2, i32 2
   %r3 = insertelement <8 x float>   %r2, float %ab3, i32 3
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/buildvector-with-reuses.ll b/llvm/test/Transforms/SLPVectorizer/X86/buildvector-with-reuses.ll
index 579239bc659bd..75a413ffc1fb1 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/buildvector-with-reuses.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/buildvector-with-reuses.ll
@@ -10,15 +10,18 @@ define <4 x double> @test(ptr %ia, ptr %ib, ptr %ic, ptr %id, ptr %ie, ptr %x) {
 ; CHECK-NEXT:    [[I4275:%.*]] = load double, ptr [[ID]], align 8
 ; CHECK-NEXT:    [[I4277:%.*]] = load double, ptr [[IE]], align 8
 ; CHECK-NEXT:    [[I4326:%.*]] = load <4 x double>, ptr [[X]], align 8
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x double> [[I4326]], <4 x double> poison, <2 x i32> <i32 0, i32 poison>
-; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <2 x double> [[TMP1]], double [[I4275]], i32 1
-; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <2 x double> [[TMP2]], <2 x double> poison, <4 x i32> <i32 0, i32 0, i32 0, i32 1>
-; CHECK-NEXT:    [[TMP4:%.*]] = insertelement <4 x double> poison, double [[I4238]], i32 0
-; CHECK-NEXT:    [[TMP5:%.*]] = insertelement <4 x double> [[TMP4]], double [[I4252]], i32 1
-; CHECK-NEXT:    [[TMP6:%.*]] = insertelement <4 x double> [[TMP5]], double [[I4264]], i32 2
-; CHECK-NEXT:    [[TMP7:%.*]] = insertelement <4 x double> [[TMP6]], double [[I4277]], i32 3
-; CHECK-NEXT:    [[TMP8:%.*]] = fmul fast <4 x double> [[TMP3]], [[TMP7]]
-; CHECK-NEXT:    ret <4 x double> [[TMP8]]
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x double> [[I4326]], <4 x double> poison, <2 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <2 x double> poison, double [[I4238]], i32 0
+; CHECK-NEXT:    [[TMP3:%.*]] = insertelement <2 x double> [[TMP2]], double [[I4252]], i32 1
+; CHECK-NEXT:    [[TMP4:%.*]] = fmul fast <2 x double> [[TMP1]], [[TMP3]]
+; CHECK-NEXT:    [[TMP5:%.*]] = insertelement <2 x double> [[TMP1]], double [[I4275]], i32 1
+; CHECK-NEXT:    [[TMP6:%.*]] = insertelement <2 x double> poison, double [[I4264]], i32 0
+; CHECK-NEXT:    [[TMP7:%.*]] = insertelement <2 x double> [[TMP6]], double [[I4277]], i32 1
+; CHECK-NEXT:    [[TMP8:%.*]] = fmul fast <2 x double> [[TMP5]], [[TMP7]]
+; CHECK-NEXT:    [[TMP9:%.*]] = shufflevector <2 x double> [[TMP4]], <2 x double> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP10:%.*]] = shufflevector <2 x double> [[TMP8]], <2 x double> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; CHECK-NEXT:    [[I44281:%.*]] = shufflevector <4 x double> [[TMP9]], <4 x double> [[TMP10]], <4 x i32> <i32 0, i32 1, i32 4, i32 5>
+; CHECK-NEXT:    ret <4 x double> [[I44281]]
 ;
   %i4238 = load double, ptr %ia, align 8
   %i4252 = load double, ptr %ib, align 8
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/call.ll b/llvm/test/Transforms/SLPVectorizer/X86/call.ll
index 8835e3b144be6..548a63c3a8cb1 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/call.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/call.ll
@@ -15,9 +15,14 @@ declare i64 @round(i64) nounwind willreturn
 
 define void @sin_libm(ptr %a, ptr %b) {
 ; CHECK-LABEL: @sin_libm(
-; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x double>, ptr [[A:%.*]], align 8
-; CHECK-NEXT:    [[TMP3:%.*]] = call <2 x double> @llvm.sin.v2f64(<2 x double> [[TMP2]])
-; CHECK-NEXT:    store <2 x double> [[TMP3]], ptr [[B:%.*]], align 8
+; CHECK-NEXT:    [[A0:%.*]] = load double, ptr [[A:%.*]], align 8
+; CHECK-NEXT:    [[IDX1:%.*]] = getelementptr inbounds double, ptr [[A]], i64 1
+; CHECK-NEXT:    [[A1:%.*]] = load double, ptr [[IDX1]], align 8
+; CHECK-NEXT:    [[SIN1:%.*]] = tail call double @sin(double [[A0]]) #[[ATTR3:[0-9]+]]
+; CHECK-NEXT:    [[SIN2:%.*]] = tail call double @sin(double [[A1]]) #[[ATTR3]]
+; CHECK-NEXT:    store double [[SIN1]], ptr [[B:%.*]], align 8
+; CHECK-NEXT:    [[IDX2:%.*]] = getelementptr inbounds double, ptr [[B]], i64 1
+; CHECK-NEXT:    store double [[SIN2]], ptr [[IDX2]], align 8
 ; CHECK-NEXT:    ret void
 ;
   %a0 = load double, ptr %a, align 8
@@ -33,9 +38,14 @@ define void @sin_libm(ptr %a, ptr %b) {
 
 define void @cos_libm(ptr %a, ptr %b) {
 ; CHECK-LABEL: @cos_libm(
-; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x double>, ptr [[A:%.*]], align 8
-; CHECK-NEXT:    [[TMP3:%.*]] = call <2 x double> @llvm.cos.v2f64(<2 x double> [[TMP2]])
-; CHECK-NEXT:    store <2 x double> [[TMP3]], ptr [[B:%.*]], align 8
+; CHECK-NEXT:    [[A0:%.*]] = load double, ptr [[A:%.*]], align 8
+; CHECK-NEXT:    [[IDX1:%.*]] = getelementptr inbounds double, ptr [[A]], i64 1
+; CHECK-NEXT:    [[A1:%.*]] = load double, ptr [[IDX1]], align 8
+; CHECK-NEXT:    [[COS1:%.*]] = tail call double @cos(double [[A0]]) #[[ATTR3]]
+; CHECK-NEXT:    [[COS2:%.*]] = tail call double @cos(double [[A1]]) #[[ATTR3]]
+; CHECK-NEXT:    store double [[COS1]], ptr [[B:%.*]], align 8
+; CHECK-NEXT:    [[IDX2:%.*]] = getelementptr inbounds double, ptr [[B]], i64 1
+; CHECK-NEXT:    store double [[COS2]], ptr [[IDX2]], align 8
 ; CHECK-NEXT:    ret void
 ;
   %a0 = load double, ptr %a, align 8
@@ -51,9 +61,14 @@ define void @cos_libm(ptr %a, ptr %b) {
 
 define void @tan_libm(ptr %a, ptr %b) {
 ; CHECK-LABEL: @tan_libm(
-; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x double>, ptr [[A:%.*]], align 8
-; CHECK-NEXT:    [[TMP3:%.*]] = call <2 x double> @llvm.tan.v2f64(<2 x double> [[TMP2]])
-; CHECK-NEXT:    store <2 x double> [[TMP3]], ptr [[B:%.*]], align 8
+; CHECK-NEXT:    [[A0:%.*]] = load double, ptr [[A:%.*]], align 8
+; CHECK-NEXT:    [[IDX1:%.*]] = getelementptr inbounds double, ptr [[A]], i64 1
+; CHECK-NEXT:    [[A1:%.*]] = load double, ptr [[IDX1]], align 8
+; CHECK-NEXT:    [[TAN1:%.*]] = tail call double @tan(double [[A0]]) #[[ATTR3]]
+; CHECK-NEXT:    [[TAN2:%.*]] = tail call double @tan(double [[A1]]) #[[ATTR3]]
+; CHECK-NEXT:    store double [[TAN1]], ptr [[B:%.*]], align 8
+; CHECK-NEXT:    [[IDX2:%.*]] = getelementptr inbounds double, ptr [[B]], i64 1
+; CHECK-NEXT:    store double [[TAN2]], ptr [[IDX2]], align 8
 ; CHECK-NEXT:    ret void
 ;
   %a0 = load double, ptr %a, align 8
@@ -69,9 +84,14 @@ define void @tan_libm(ptr %a, ptr %b) {
 
 define void @pow_libm(ptr %a, ptr %b) {
 ; CHECK-LABEL: @pow_libm(
-; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x double>, ptr [[A:%.*]], align 8
-; CHECK-NEXT:    [[TMP3:%.*]] = call <2 x double> @llvm.pow.v2f64(<2 x double> [[TMP2]], <2 x double> [[TMP2]])
-; CHECK-NEXT:    store <2 x double> [[TMP3]], ptr [[B:%.*]], align 8
+; CHECK-NEXT:    [[A0:%.*]] = load double, ptr [[A:%.*]], align 8
+; CHECK-NEXT:    [[IDX1:%.*]] = getelementptr inbounds double, ptr [[A]], i64 1
+; CHECK-NEXT:    [[A1:%.*]] = load double, ptr [[IDX1]], align 8
+; CHECK-NEXT:    [[POW1:%.*]] = tail call double @pow(double [[A0]], double [[A0]]) #[[ATTR3]]
+; CHECK-NEXT:    [[POW2:%.*]] = tail call double @pow(double [[A1]], double [[A1]]) #[[ATTR3]]
+; CHECK-NEXT:    store double [[POW1]], ptr [[B:%.*]], align 8
+; CHECK-NEXT:    [[IDX2:%.*]] = getelementptr inbounds double, ptr [[B]], i64 1
+; CHECK-NEXT:    store double [[POW2]], ptr [[IDX2]], align 8
 ; CHECK-NEXT:    ret void
 ;
   %a0 = load double, ptr %a, align 8
@@ -87,9 +107,14 @@ define void @pow_libm(ptr %a, ptr %b) {
 
 define void @exp_libm(ptr %a, ptr %b) {
 ; CHECK-LABEL: @exp_libm(
-; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x double>, ptr [[A:%.*]], align 8
-; CHECK-NEXT:    [[TMP3:%.*]] = call <2 x double> @llvm.exp2.v2f64(<2 x double> [[TMP2]])
-; CHECK-NEXT:    store <2 x double> [[TMP3]], ptr [[B:%.*]], align 8
+; CHECK-NEXT:    [[A0:%.*]] = load double, ptr [[A:%.*]], align 8
+; CHECK-NEXT:    [[IDX1:%.*]] = getelementptr inbounds double, ptr [[A]], i64 1
+; CHECK-NEXT:    [[A1:%.*]] = load double, ptr [[IDX1]], align 8
+; CHECK-NEXT:    [[EXP1:%.*]] = tail call double @exp2(double [[A0]]) #[[ATTR3]]
+; CHECK-NEXT:    [[EXP2:%.*]] = tail call double @exp2(double [[A1]]) #[[ATTR3]]
+; CHECK-NEXT:    store double [[EXP1]], ptr [[B:%.*]], align 8
+; CHECK-NEXT:    [[IDX2:%.*]] = getelementptr inbounds double, ptr [[B]], i64 1
+; CHECK-NEXT:    store double [[EXP2]], ptr [[IDX2]], align 8
 ; CHECK-NEXT:    ret void
 ;
   %a0 = load double, ptr %a, align 8
@@ -134,8 +159,8 @@ define void @sqrt_libm_errno(ptr %a, ptr %b) {
 ; CHECK-NEXT:    [[A0:%.*]] = load double, ptr [[A:%.*]], align 8
 ; CHECK-NEXT:    [[IDX1:%.*]] = getelementptr inbounds double, ptr [[A]], i64 1
 ; CHECK-NEXT:    [[A1:%.*]] = load double, ptr [[IDX1]], align 8
-; CHECK-NEXT:    [[SQRT1:%.*]] = tail call nnan double @sqrt(double [[A0]]) #[[ATTR3:[0-9]+]]
-; CHECK-NEXT:    [[SQRT2:%.*]] = tail call nnan double @sqrt(double [[A1]]) #[[ATTR3]]
+; CHECK-NEXT:    [[SQRT1:%.*]] = tail call nnan double @sqrt(double [[A0]]) #[[ATTR4:[0-9]+]]
+; CHECK-NEXT:    [[SQRT2:%.*]] = tail call nnan double @sqrt(double [[A1]]) #[[ATTR4]]
 ; CHECK-NEXT:    store double [[SQRT1]], ptr [[B:%.*]], align 8
 ; CHECK-NEXT:    [[IDX2:%.*]] = getelementptr inbounds double, ptr [[B]], i64 1
 ; CHECK-NEXT:    store double [[SQRT2]], ptr [[IDX2]], align 8
@@ -158,8 +183,8 @@ define void @round_custom(ptr %a, ptr %b) {
 ; CHECK-NEXT:    [[A0:%.*]] = load i64, ptr [[A:%.*]], align 8
 ; CHECK-NEXT:    [[IDX1:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 1
 ; CHECK-NEXT:    [[A1:%.*]] = load i64, ptr [[IDX1]], align 8
-; CHECK-NEXT:    [[ROUND1:%.*]] = tail call i64 @round(i64 [[A0]]) #[[ATTR4:[0-9]+]]
-; CHECK-NEXT:    [[ROUND2:%.*]] = tail call i64 @round(i64 [[A1]]) #[[ATTR4]]
+; CHECK-NEXT:    [[ROUND1:%.*]] = tail call i64 @round(i64 [[A0]]) #[[ATTR3]]
+; CHECK-NEXT:    [[ROUND2:%.*]] = tail call i64 @round(i64 [[A1]]) #[[ATTR3]]
 ; CHECK-NEXT:    store i64 [[ROUND1]], ptr [[B:%.*]], align 8
 ; CHECK-NEXT:    [[IDX2:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 1
 ; CHECK-NEXT:    store i64 [[ROUND2]], ptr [[IDX2]], align 8
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/crash_clear_undefs.ll b/llvm/test/Transforms/SLPVectorizer/X86/crash_clear_undefs.ll
index de99654d84eb8..c2369a6a89ec1 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/crash_clear_undefs.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/crash_clear_undefs.ll
@@ -9,7 +9,7 @@ target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16
 ; YAML-NEXT:  Function:        foo
 ; YAML-NEXT:  Args:
 ; YAML-NEXT:    - String:          'SLP vectorized with cost '
-; YAML-NEXT:    - Cost:            '-3'
+; YAML-NEXT:    - Cost:            '-4'
 ; YAML-NEXT:    - String:          ' and with tree size '
 ; YAML-NEXT:    - TreeSize:        '10'
 ; YAML-NEXT:  ...
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/ctlz.ll b/llvm/test/Transforms/SLPVectorizer/X86/ctlz.ll
index 8a22e45fe1ca5..9bf2ade3176d6 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/ctlz.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/ctlz.ll
@@ -1,6 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -mtriple=x86_64-unknown -passes=slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,SSE,SSE2
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=corei7 -passes=slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,SSE,SSE4
+; RUN: opt < %s -mtriple=x86_64-unknown -passes=slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,SSE
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=corei7 -passes=slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,SSE
 ; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=corei7-avx -passes=slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX,AVX1
 ; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=core-avx2 -passes=slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX,AVX2
 ; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=icelake-server -passes=slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX,AVX512
@@ -136,32 +136,47 @@ define void @ctlz_4i64() #0 {
 }
 
 define void @ctlz_4i32() #0 {
-; SSE2-LABEL: @ctlz_4i32(
-; SSE2-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr @src32, align 4
-; SSE2-NEXT:    [[TMP2:%.*]] = call <4 x i32> @llvm.ctlz.v4i32(<4 x i32> [[TMP1]], i1 false)
-; SSE2-NEXT:    store <4 x i32> [[TMP2]], ptr @dst32, align 4
-; SSE2-NEXT:    ret void
+; SSE-LABEL: @ctlz_4i32(
+; SSE-NEXT:    [[LD0:%.*]] = load i32, ptr @src32, align 4
+; SSE-NEXT:    [[LD1:%.*]] = load i32, ptr getelementptr inbounds ([8 x i32], ptr @src32, i32 0, i64 1), align 4
+; SSE-NEXT:    [[LD2:%.*]] = load i32, ptr getelementptr inbounds ([8 x i32], ptr @src32, i32 0, i64 2), align 4
+; SSE-NEXT:    [[LD3:%.*]] = load i32, ptr getelementptr inbounds ([8 x i32], ptr @src32, i32 0, i64 3), align 4
+; SSE-NEXT:    [[CTLZ0:%.*]] = call i32 @llvm.ctlz.i32(i32 [[LD0]], i1 false)
+; SSE-NEXT:    [[CTLZ1:%.*]] = call i32 @llvm.ctlz.i32(i32 [[LD1]], i1 false)
+; SSE-NEXT:    [[CTLZ2:%.*]] = call i32 @llvm.ctlz.i32(i32 [[LD2]], i1 false)
+; SSE-NEXT:    [[CTLZ3:%.*]] = call i32 @llvm.ctlz.i32(i32 [[LD3]], i1 false)
+; SSE-NEXT:    store i32 [[CTLZ0]], ptr @dst32, align 4
+; SSE-NEXT:    store i32 [[CTLZ1]], ptr getelementptr inbounds ([8 x i32], ptr @dst32, i32 0, i64 1), align 4
+; SSE-NEXT:    store i32 [[CTLZ2]], ptr getelementptr inbounds ([8 x i32], ptr @dst32, i32 0, i64 2), align 4
+; SSE-NEXT:    store i32 [[CTLZ3]], ptr getelementptr inbounds ([8 x i32], ptr @dst32, i32 0, i64 3), align 4
+; SSE-NEXT:    ret void
+;
+; AVX1-LABEL: @ctlz_4i32(
+; AVX1-NEXT:    [[LD0:%.*]] = load i32, ptr @src32, align 4
+; AVX1-NEXT:    [[LD1:%.*]] = load i32, ptr getelementptr inbounds ([8 x i32], ptr @src32, i32 0, i64 1), align 4
+; AVX1-NEXT:    [[LD2:%.*]] = load i32, ptr getelementptr inbounds ([8 x i32], ptr @src32, i32 0, i64 2), align 4
+; AVX1-NEXT:    [[LD3:%.*]] = load i32, ptr getelementptr inbounds ([8 x i32], ptr @src32, i32 0, i64 3), align 4
+; AVX1-NEXT:    [[CTLZ0:%.*]] = call i32 @llvm.ctlz.i32(i32 [[LD0]], i1 false)
+; AVX1-NEXT:    [[CTLZ1:%.*]] = call i32 @llvm.ctlz.i32(i32 [[LD1]], i1 false)
+; AVX1-NEXT:    [[CTLZ2:%.*]] = call i32 @llvm.ctlz.i32(i32 [[LD2]], i1 false)
+; AVX1-NEXT:    [[CTLZ3:%.*]] = call i32 @llvm.ctlz.i32(i32 [[LD3]], i1 false)
+; AVX1-NEXT:    store i32 [[CTLZ0]], ptr @dst32, align 4
+; AVX1-NEXT:    store i32 [[CTLZ1]], ptr getelementptr inbounds ([8 x i32], ptr @dst32, i32 0, i64 1), align 4
+; AVX1-NEXT:    store i32 [[CTLZ2]], ptr getelementptr inbounds ([8 x i32], ptr @dst32, i32 0, i64 2), align 4
+; AVX1-NEXT:    store i32 [[CTLZ3]], ptr getelementptr inbounds ([8 x i32], ptr @dst32, i32 0, i64 3), align 4
+; AVX1-NEXT:    ret void
 ;
-; SSE4-LABEL: @ctlz_4i32(
-; SSE4-NEXT:    [[LD0:%.*]] = load i32, ptr @src32, align 4
-; SSE4-NEXT:    [[LD1:%.*]] = load i32, ptr getelementptr inbounds ([8 x i32], ptr @src32, i32 0, i64 1), align 4
-; SSE4-NEXT:    [[LD2:%.*]] = load i32, ptr getelementptr inbounds ([8 x i32], ptr @src32, i32 0, i64 2), align 4
-; SSE4-NEXT:    [[LD3:%.*]] = load i32, ptr getelementptr inbounds ([8 x i32], ptr @src32, i32 0, i64 3), align 4
-; SSE4-NEXT:    [[CTLZ0:%.*]] = call i32 @llvm.ctlz.i32(i32 [[LD0]], i1 false)
-; SSE4-NEXT:    [[CTLZ1:%.*]] = call i32 @llvm.ctlz.i32(i32 [[LD1]], i1 false)
-; SSE4-NEXT:    [[CTLZ2:%.*]] = call i32 @llvm.ctlz.i32(i32 [[LD2]], i1 false)
-; SSE4-NEXT:    [[CTLZ3:%.*]] = call i32 @llvm.ctlz.i32(i32 [[LD3]], i1 false)
-; SSE4-NEXT:    store i32 [[CTLZ0]], ptr @dst32, align 4
-; SSE4-NEXT:    store i32 [[CTLZ1]], ptr getelementptr inbounds ([8 x i32], ptr @dst32, i32 0, i64 1), align 4
-; SSE4-NEXT:    store i32 [[CTLZ2]], ptr getelementptr inbounds ([8 x i32], ptr @dst32, i32 0, i64 2), align 4
-; SSE4-NEXT:    store i32 [[CTLZ3]], ptr getelementptr inbounds ([8 x i32], ptr @dst32, i32 0, i64 3), align 4
-; SSE4-NEXT:    ret void
+; AVX2-LABEL: @ctlz_4i32(
+; AVX2-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr @src32, align 4
+; AVX2-NEXT:    [[TMP2:%.*]] = call <4 x i32> @llvm.ctlz.v4i32(<4 x i32> [[TMP1]], i1 false)
+; AVX2-NEXT:    store <4 x i32> [[TMP2]], ptr @dst32, align 4
+; AVX2-NEXT:    ret void
 ;
-; AVX-LABEL: @ctlz_4i32(
-; AVX-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr @src32, align 4
-; AVX-NEXT:    [[TMP2:%.*]] = call <4 x i32> @llvm.ctlz.v4i32(<4 x i32> [[TMP1]], i1 false)
-; AVX-NEXT:    store <4 x i32> [[TMP2]], ptr @dst32, align 4
-; AVX-NEXT:    ret void
+; AVX512-LABEL: @ctlz_4i32(
+; AVX512-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr @src32, align 4
+; AVX512-NEXT:    [[TMP2:%.*]] = call <4 x i32> @llvm.ctlz.v4i32(<4 x i32> [[TMP1]], i1 false)
+; AVX512-NEXT:    store <4 x i32> [[TMP2]], ptr @dst32, align 4
+; AVX512-NEXT:    ret void
 ;
   %ld0 = load i32, ptr @src32, align 4
   %ld1 = load i32, ptr getelementptr inbounds ([8 x i32], ptr @src32, i32 0, i64 1), align 4
@@ -179,47 +194,71 @@ define void @ctlz_4i32() #0 {
 }
 
 define void @ctlz_8i32() #0 {
-; SSE2-LABEL: @ctlz_8i32(
-; SSE2-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr @src32, align 2
-; SSE2-NEXT:    [[TMP2:%.*]] = call <4 x i32> @llvm.ctlz.v4i32(<4 x i32> [[TMP1]], i1 false)
-; SSE2-NEXT:    store <4 x i32> [[TMP2]], ptr @dst32, align 2
-; SSE2-NEXT:    [[TMP3:%.*]] = load <4 x i32>, ptr getelementptr inbounds ([8 x i32], ptr @src32, i32 0, i64 4), align 2
-; SSE2-NEXT:    [[TMP4:%.*]] = call <4 x i32> @llvm.ctlz.v4i32(<4 x i32> [[TMP3]], i1 false)
-; SSE2-NEXT:    store <4 x i32> [[TMP4]], ptr getelementptr inbounds ([8 x i32], ptr @dst32, i32 0, i64 4), align 2
-; SSE2-NEXT:    ret void
+; SSE-LABEL: @ctlz_8i32(
+; SSE-NEXT:    [[LD0:%.*]] = load i32, ptr @src32, align 2
+; SSE-NEXT:    [[LD1:%.*]] = load i32, ptr getelementptr inbounds ([8 x i32], ptr @src32, i32 0, i64 1), align 2
+; SSE-NEXT:    [[LD2:%.*]] = load i32, ptr getelementptr inbounds ([8 x i32], ptr @src32, i32 0, i64 2), align 2
+; SSE-NEXT:    [[LD3:%.*]] = load i32, ptr getelementptr inbounds ([8 x i32], ptr @src32, i32 0, i64 3), align 2
+; SSE-NEXT:    [[LD4:%.*]] = load i32, ptr getelementptr inbounds ([8 x i32], ptr @src32, i32 0, i64 4), align 2
+; SSE-NEXT:    [[LD5:%.*]] = load i32, ptr getelementptr inbounds ([8 x i32], ptr @src32, i32 0, i64 5), align 2
+; SSE-NEXT:    [[LD6:%.*]] = load i32, ptr getelementptr inbounds ([8 x i32], ptr @src32, i32 0, i64 6), align 2
+; SSE-NEXT:    [[LD7:%.*]] = load i32, ptr getelementptr inbounds ([8 x i32], ptr @src32, i32 0, i64 7), align 2
+; SSE-NEXT:    [[CTLZ0:%.*]] = call i32 @llvm.ctlz.i32(i32 [[LD0]], i1 false)
+; SSE-NEXT:    [[CTLZ1:%.*]] = call i32 @llvm.ctlz.i32(i32 [[LD1]], i1 false)
+; SSE-NEXT:    [[CTLZ2:%.*]] = call i32 @llvm.ctlz.i32(i32 [[LD2]], i1 false)
+; SSE-NEXT:    [[CTLZ3:%.*]] = call i32 @llvm.ctlz.i32(i32 [[LD3]], i1 false)
+; SSE-NEXT:    [[CTLZ4:%.*]] = call i32 @llvm.ctlz.i32(i32 [[LD4]], i1 false)
+; SSE-NEXT:    [[CTLZ5:%.*]] = call i32 @llvm.ctlz.i32(i32 [[LD5]], i1 false)
+; SSE-NEXT:    [[CTLZ6:%.*]] = call i32 @llvm.ctlz.i32(i32 [[LD6]], i1 false)
+; SSE-NEXT:    [[CTLZ7:%.*]] = call i32 @llvm.ctlz.i32(i32 [[LD7]], i1 false)
+; SSE-NEXT:    store i32 [[CTLZ0]], ptr @dst32, align 2
+; SSE-NEXT:    store i32 [[CTLZ1]], ptr getelementptr inbounds ([8 x i32], ptr @dst32, i32 0, i64 1), align 2
+; SSE-NEXT:    store i32 [[CTLZ2]], ptr getelementptr inbounds ([8 x i32], ptr @dst32, i32 0, i64 2), align 2
+; SSE-NEXT:    store i32 [[CTLZ3]], ptr getelementptr inbounds ([8 x i32], ptr @dst32, i32 0, i64 3), align 2
+; SSE-NEXT:    store i32 [[CTLZ4]], ptr getelementptr inbounds ([8 x i32], ptr @dst32, i32 0, i64 4), align 2
+; SSE-NEXT:    store i32 [[CTLZ5]], ptr getelementptr inbounds ([8 x i32], ptr @dst32, i32 0, i64 5), align 2
+; SSE-NEXT:    store i32 [[CTLZ6]], ptr getelementptr inbounds ([8 x i32], ptr @dst32, i32 0, i64 6), align 2
+; SSE-NEXT:    store i32 [[CTLZ7]], ptr getelementptr inbounds ([8 x i32], ptr @dst32, i32 0, i64 7), align 2
+; SSE-NEXT:    ret void
+;
+; AVX1-LABEL: @ctlz_8i32(
+; AVX1-NEXT:    [[LD0:%.*]] = load i32, ptr @src32, align 2
+; AVX1-NEXT:    [[LD1:%.*]] = load i32, ptr getelementptr inbounds ([8 x i32], ptr @src32, i32 0, i64 1), align 2
+; AVX1-NEXT:    [[LD2:%.*]] = load i32, ptr getelementptr inbounds ([8 x i32], ptr @src32, i32 0, i64 2), align 2
+; AVX1-NEXT:    [[LD3:%.*]] = load i32, ptr getelementptr inbounds ([8 x i32], ptr @src32, i32 0, i64 3), align 2
+; AVX1-NEXT:    [[LD4:%.*]] = load i32, ptr getelementptr inbounds ([8 x i32], ptr @src32, i32 0, i64 4), align 2
+; AVX1-NEXT:    [[LD5:%.*]] = load i32, ptr getelementptr inbounds ([8 x i32], ptr @src32, i32 0, i64 5), align 2
+; AVX1-NEXT:    [[LD6:%.*]] = load i32, ptr getelementptr inbounds ([8 x i32], ptr @src32, i32 0, i64 6), align 2
+; AVX1-NEXT:    [[LD7:%.*]] = load i32, ptr getelementptr inbounds ([8 x i32], ptr @src32, i32 0, i64 7), align 2
+; AVX1-NEXT:    [[CTLZ0:%.*]] = call i32 @llvm.ctlz.i32(i32 [[LD0]], i1 false)
+; AVX1-NEXT:    [[CTLZ1:%.*]] = call i32 @llvm.ctlz.i32(i32 [[LD1]], i1 false)
+; AVX1-NEXT:    [[CTLZ2:%.*]] = call i32 @llvm.ctlz.i32(i32 [[LD2]], i1 false)
+; AVX1-NEXT:    [[CTLZ3:%.*]] = call i32 @llvm.ctlz.i32(i32 [[LD3]], i1 false)
+; AVX1-NEXT:    [[CTLZ4:%.*]] = call i32 @llvm.ctlz.i32(i32 [[LD4]], i1 false)
+; AVX1-NEXT:    [[CTLZ5:%.*]] = call i32 @llvm.ctlz.i32(i32 [[LD5]], i1 false)
+; AVX1-NEXT:    [[CTLZ6:%.*]] = call i32 @llvm.ctlz.i32(i32 [[LD6]], i1 false)
+; AVX1-NEXT:    [[CTLZ7:%.*]] = call i32 @llvm.ctlz.i32(i32 [[LD7]], i1 false)
+; AVX1-NEXT:    store i32 [[CTLZ0]], ptr @dst32, align 2
+; AVX1-NEXT:    store i32 [[CTLZ1]], ptr getelementptr inbounds ([8 x i32], ptr @dst32, i32 0, i64 1), align 2
+; AVX1-NEXT:    store i32 [[CTLZ2]], ptr getelementptr inbounds ([8 x i32], ptr @dst32, i32 0, i64 2), align 2
+; AVX1-NEXT:    store i32 [[CTLZ3]], ptr getelementptr inbounds ([8 x i32], ptr @dst32, i32 0, i64 3), align 2
+; AVX1-NEXT:    store i32 [[CTLZ4]], ptr getelementptr inbounds ([8 x i32], ptr @dst32, i32 0, i64 4), align 2
+; AVX1-NEXT:    store i32 [[CTLZ5]], ptr getelementptr inbounds ([8 x i32], ptr @dst32, i32 0, i64 5), align 2
+; AVX1-NEXT:    store i32 [[CTLZ6]], ptr getelementptr inbounds ([8 x i32], ptr @dst32, i32 0, i64 6), align 2
+; AVX1-NEXT:    store i32 [[CTLZ7]], ptr getelementptr inbounds ([8 x i32], ptr @dst32, i32 0, i64 7), align 2
+; AVX1-NEXT:    ret void
 ;
-; SSE4-LABEL: @ctlz_8i32(
-; SSE4-NEXT:    [[LD0:%.*]] = load i32, ptr @src32, align 2
-; SSE4-NEXT:    [[LD1:%.*]] = load i32, ptr getelementptr inbounds ([8 x i32], ptr @src32, i32 0, i64 1), align 2
-; SSE4-NEXT:    [[LD2:%.*]] = load i32, ptr getelementptr inbounds ([8 x i32], ptr @src32, i32 0, i64 2), align 2
-; SSE4-NEXT:    [[LD3:%.*]] = load i32, ptr getelementptr inbounds ([8 x i32], ptr @src32, i32 0, i64 3), align 2
-; SSE4-NEXT:    [[LD4:%.*]] = load i32, ptr getelementptr inbounds ([8 x i32], ptr @src32, i32 0, i64 4), align 2
-; SSE4-NEXT:    [[LD5:%.*]] = load i32, ptr getelementptr inbounds ([8 x i32], ptr @src32, i32 0, i64 5), align 2
-; SSE4-NEXT:    [[LD6:%.*]] = load i32, ptr getelementptr inbounds ([8 x i32], ptr @src32, i32 0, i64 6), align 2
-; SSE4-NEXT:    [[LD7:%.*]] = load i32, ptr getelementptr inbounds ([8 x i32], ptr @src32, i32 0, i64 7), align 2
-; SSE4-NEXT:    [[CTLZ0:%.*]] = call i32 @llvm.ctlz.i32(i32 [[LD0]], i1 false)
-; SSE4-NEXT:    [[CTLZ1:%.*]] = call i32 @llvm.ctlz.i32(i32 [[LD1]], i1 false)
-; SSE4-NEXT:    [[CTLZ2:%.*]] = call i32 @llvm.ctlz.i32(i32 [[LD2]], i1 false)
-; SSE4-NEXT:    [[CTLZ3:%.*]] = call i32 @llvm.ctlz.i32(i32 [[LD3]], i1 false)
-; SSE4-NEXT:    [[CTLZ4:%.*]] = call i32 @llvm.ctlz.i32(i32 [[LD4]], i1 false)
-; SSE4-NEXT:    [[CTLZ5:%.*]] = call i32 @llvm.ctlz.i32(i32 [[LD5]], i1 false)
-; SSE4-NEXT:    [[CTLZ6:%.*]] = call i32 @llvm.ctlz.i32(i32 [[LD6]], i1 false)
-; SSE4-NEXT:    [[CTLZ7:%.*]] = call i32 @llvm.ctlz.i32(i32 [[LD7]], i1 false)
-; SSE4-NEXT:    store i32 [[CTLZ0]], ptr @dst32, align 2
-; SSE4-NEXT:    store i32 [[CTLZ1]], ptr getelementptr inbounds ([8 x i32], ptr @dst32, i32 0, i64 1), align 2
-; SSE4-NEXT:    store i32 [[CTLZ2]], ptr getelementptr inbounds ([8 x i32], ptr @dst32, i32 0, i64 2), align 2
-; SSE4-NEXT:    store i32 [[CTLZ3]], ptr getelementptr inbounds ([8 x i32], ptr @dst32, i32 0, i64 3), align 2
-; SSE4-NEXT:    store i32 [[CTLZ4]], ptr getelementptr inbounds ([8 x i32], ptr @dst32, i32 0, i64 4), align 2
-; SSE4-NEXT:    store i32 [[CTLZ5]], ptr getelementptr inbounds ([8 x i32], ptr @dst32, i32 0, i64 5), align 2
-; SSE4-NEXT:    store i32 [[CTLZ6]], ptr getelementptr inbounds ([8 x i32], ptr @dst32, i32 0, i64 6), align 2
-; SSE4-NEXT:    store i32 [[CTLZ7]], ptr getelementptr inbounds ([8 x i32], ptr @dst32, i32 0, i64 7), align 2
-; SSE4-NEXT:    ret void
+; AVX2-LABEL: @ctlz_8i32(
+; AVX2-NEXT:    [[TMP1:%.*]] = load <8 x i32>, ptr @src32, align 2
+; AVX2-NEXT:    [[TMP2:%.*]] = call <8 x i32> @llvm.ctlz.v8i32(<8 x i32> [[TMP1]], i1 false)
+; AVX2-NEXT:    store <8 x i32> [[TMP2]], ptr @dst32, align 2
+; AVX2-NEXT:    ret void
 ;
-; AVX-LABEL: @ctlz_8i32(
-; AVX-NEXT:    [[TMP1:%.*]] = load <8 x i32>, ptr @src32, align 2
-; AVX-NEXT:    [[TMP2:%.*]] = call <8 x i32> @llvm.ctlz.v8i32(<8 x i32> [[TMP1]], i1 false)
-; AVX-NEXT:    store <8 x i32> [[TMP2]], ptr @dst32, align 2
-; AVX-NEXT:    ret void
+; AVX512-LABEL: @ctlz_8i32(
+; AVX512-NEXT:    [[TMP1:%.*]] = load <8 x i32>, ptr @src32, align 2
+; AVX512-NEXT:    [[TMP2:%.*]] = call <8 x i32> @llvm.ctlz.v8i32(<8 x i32> [[TMP1]], i1 false)
+; AVX512-NEXT:    store <8 x i32> [[TMP2]], ptr @dst32, align 2
+; AVX512-NEXT:    ret void
 ;
   %ld0 = load i32, ptr @src32, align 2
   %ld1 = load i32, ptr getelementptr inbounds ([8 x i32], ptr @src32, i32 0, i64 1), align 2
@@ -1063,3 +1102,6 @@ define void @ctlz_undef_32i8() #0 {
 }
 
 attributes #0 = { nounwind }
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; SSE2: {{.*}}
+; SSE4: {{.*}}
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/cttz.ll b/llvm/test/Transforms/SLPVectorizer/X86/cttz.ll
index 22f0c3f936509..896be6f2fe213 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/cttz.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/cttz.ll
@@ -142,11 +142,32 @@ define void @cttz_4i32() #0 {
 ; SSE-NEXT:    store i32 [[CTTZ3]], ptr getelementptr inbounds ([8 x i32], ptr @dst32, i32 0, i64 3), align 4
 ; SSE-NEXT:    ret void
 ;
-; AVX-LABEL: @cttz_4i32(
-; AVX-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr @src32, align 4
-; AVX-NEXT:    [[TMP2:%.*]] = call <4 x i32> @llvm.cttz.v4i32(<4 x i32> [[TMP1]], i1 false)
-; AVX-NEXT:    store <4 x i32> [[TMP2]], ptr @dst32, align 4
-; AVX-NEXT:    ret void
+; AVX1-LABEL: @cttz_4i32(
+; AVX1-NEXT:    [[LD0:%.*]] = load i32, ptr @src32, align 4
+; AVX1-NEXT:    [[LD1:%.*]] = load i32, ptr getelementptr inbounds ([8 x i32], ptr @src32, i32 0, i64 1), align 4
+; AVX1-NEXT:    [[LD2:%.*]] = load i32, ptr getelementptr inbounds ([8 x i32], ptr @src32, i32 0, i64 2), align 4
+; AVX1-NEXT:    [[LD3:%.*]] = load i32, ptr getelementptr inbounds ([8 x i32], ptr @src32, i32 0, i64 3), align 4
+; AVX1-NEXT:    [[CTTZ0:%.*]] = call i32 @llvm.cttz.i32(i32 [[LD0]], i1 false)
+; AVX1-NEXT:    [[CTTZ1:%.*]] = call i32 @llvm.cttz.i32(i32 [[LD1]], i1 false)
+; AVX1-NEXT:    [[CTTZ2:%.*]] = call i32 @llvm.cttz.i32(i32 [[LD2]], i1 false)
+; AVX1-NEXT:    [[CTTZ3:%.*]] = call i32 @llvm.cttz.i32(i32 [[LD3]], i1 false)
+; AVX1-NEXT:    store i32 [[CTTZ0]], ptr @dst32, align 4
+; AVX1-NEXT:    store i32 [[CTTZ1]], ptr getelementptr inbounds ([8 x i32], ptr @dst32, i32 0, i64 1), align 4
+; AVX1-NEXT:    store i32 [[CTTZ2]], ptr getelementptr inbounds ([8 x i32], ptr @dst32, i32 0, i64 2), align 4
+; AVX1-NEXT:    store i32 [[CTTZ3]], ptr getelementptr inbounds ([8 x i32], ptr @dst32, i32 0, i64 3), align 4
+; AVX1-NEXT:    ret void
+;
+; AVX2-LABEL: @cttz_4i32(
+; AVX2-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr @src32, align 4
+; AVX2-NEXT:    [[TMP2:%.*]] = call <4 x i32> @llvm.cttz.v4i32(<4 x i32> [[TMP1]], i1 false)
+; AVX2-NEXT:    store <4 x i32> [[TMP2]], ptr @dst32, align 4
+; AVX2-NEXT:    ret void
+;
+; AVX512-LABEL: @cttz_4i32(
+; AVX512-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr @src32, align 4
+; AVX512-NEXT:    [[TMP2:%.*]] = call <4 x i32> @llvm.cttz.v4i32(<4 x i32> [[TMP1]], i1 false)
+; AVX512-NEXT:    store <4 x i32> [[TMP2]], ptr @dst32, align 4
+; AVX512-NEXT:    ret void
 ;
   %ld0 = load i32, ptr @src32, align 4
   %ld1 = load i32, ptr getelementptr inbounds ([8 x i32], ptr @src32, i32 0, i64 1), align 4
@@ -191,11 +212,44 @@ define void @cttz_8i32() #0 {
 ; SSE-NEXT:    store i32 [[CTTZ7]], ptr getelementptr inbounds ([8 x i32], ptr @dst32, i32 0, i64 7), align 2
 ; SSE-NEXT:    ret void
 ;
-; AVX-LABEL: @cttz_8i32(
-; AVX-NEXT:    [[TMP1:%.*]] = load <8 x i32>, ptr @src32, align 2
-; AVX-NEXT:    [[TMP2:%.*]] = call <8 x i32> @llvm.cttz.v8i32(<8 x i32> [[TMP1]], i1 false)
-; AVX-NEXT:    store <8 x i32> [[TMP2]], ptr @dst32, align 2
-; AVX-NEXT:    ret void
+; AVX1-LABEL: @cttz_8i32(
+; AVX1-NEXT:    [[LD0:%.*]] = load i32, ptr @src32, align 2
+; AVX1-NEXT:    [[LD1:%.*]] = load i32, ptr getelementptr inbounds ([8 x i32], ptr @src32, i32 0, i64 1), align 2
+; AVX1-NEXT:    [[LD2:%.*]] = load i32, ptr getelementptr inbounds ([8 x i32], ptr @src32, i32 0, i64 2), align 2
+; AVX1-NEXT:    [[LD3:%.*]] = load i32, ptr getelementptr inbounds ([8 x i32], ptr @src32, i32 0, i64 3), align 2
+; AVX1-NEXT:    [[LD4:%.*]] = load i32, ptr getelementptr inbounds ([8 x i32], ptr @src32, i32 0, i64 4), align 2
+; AVX1-NEXT:    [[LD5:%.*]] = load i32, ptr getelementptr inbounds ([8 x i32], ptr @src32, i32 0, i64 5), align 2
+; AVX1-NEXT:    [[LD6:%.*]] = load i32, ptr getelementptr inbounds ([8 x i32], ptr @src32, i32 0, i64 6), align 2
+; AVX1-NEXT:    [[LD7:%.*]] = load i32, ptr getelementptr inbounds ([8 x i32], ptr @src32, i32 0, i64 7), align 2
+; AVX1-NEXT:    [[CTTZ0:%.*]] = call i32 @llvm.cttz.i32(i32 [[LD0]], i1 false)
+; AVX1-NEXT:    [[CTTZ1:%.*]] = call i32 @llvm.cttz.i32(i32 [[LD1]], i1 false)
+; AVX1-NEXT:    [[CTTZ2:%.*]] = call i32 @llvm.cttz.i32(i32 [[LD2]], i1 false)
+; AVX1-NEXT:    [[CTTZ3:%.*]] = call i32 @llvm.cttz.i32(i32 [[LD3]], i1 false)
+; AVX1-NEXT:    [[CTTZ4:%.*]] = call i32 @llvm.cttz.i32(i32 [[LD4]], i1 false)
+; AVX1-NEXT:    [[CTTZ5:%.*]] = call i32 @llvm.cttz.i32(i32 [[LD5]], i1 false)
+; AVX1-NEXT:    [[CTTZ6:%.*]] = call i32 @llvm.cttz.i32(i32 [[LD6]], i1 false)
+; AVX1-NEXT:    [[CTTZ7:%.*]] = call i32 @llvm.cttz.i32(i32 [[LD7]], i1 false)
+; AVX1-NEXT:    store i32 [[CTTZ0]], ptr @dst32, align 2
+; AVX1-NEXT:    store i32 [[CTTZ1]], ptr getelementptr inbounds ([8 x i32], ptr @dst32, i32 0, i64 1), align 2
+; AVX1-NEXT:    store i32 [[CTTZ2]], ptr getelementptr inbounds ([8 x i32], ptr @dst32, i32 0, i64 2), align 2
+; AVX1-NEXT:    store i32 [[CTTZ3]], ptr getelementptr inbounds ([8 x i32], ptr @dst32, i32 0, i64 3), align 2
+; AVX1-NEXT:    store i32 [[CTTZ4]], ptr getelementptr inbounds ([8 x i32], ptr @dst32, i32 0, i64 4), align 2
+; AVX1-NEXT:    store i32 [[CTTZ5]], ptr getelementptr inbounds ([8 x i32], ptr @dst32, i32 0, i64 5), align 2
+; AVX1-NEXT:    store i32 [[CTTZ6]], ptr getelementptr inbounds ([8 x i32], ptr @dst32, i32 0, i64 6), align 2
+; AVX1-NEXT:    store i32 [[CTTZ7]], ptr getelementptr inbounds ([8 x i32], ptr @dst32, i32 0, i64 7), align 2
+; AVX1-NEXT:    ret void
+;
+; AVX2-LABEL: @cttz_8i32(
+; AVX2-NEXT:    [[TMP1:%.*]] = load <8 x i32>, ptr @src32, align 2
+; AVX2-NEXT:    [[TMP2:%.*]] = call <8 x i32> @llvm.cttz.v8i32(<8 x i32> [[TMP1]], i1 false)
+; AVX2-NEXT:    store <8 x i32> [[TMP2]], ptr @dst32, align 2
+; AVX2-NEXT:    ret void
+;
+; AVX512-LABEL: @cttz_8i32(
+; AVX512-NEXT:    [[TMP1:%.*]] = load <8 x i32>, ptr @src32, align 2
+; AVX512-NEXT:    [[TMP2:%.*]] = call <8 x i32> @llvm.cttz.v8i32(<8 x i32> [[TMP1]], i1 false)
+; AVX512-NEXT:    store <8 x i32> [[TMP2]], ptr @dst32, align 2
+; AVX512-NEXT:    ret void
 ;
   %ld0 = load i32, ptr @src32, align 2
   %ld1 = load i32, ptr getelementptr inbounds ([8 x i32], ptr @src32, i32 0, i64 1), align 2
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/extract_in_tree_user.ll b/llvm/test/Transforms/SLPVectorizer/X86/extract_in_tree_user.ll
index 6033e8def3436..e7d7ce8ee4a3f 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/extract_in_tree_user.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/extract_in_tree_user.ll
@@ -6,7 +6,7 @@ target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
 @a = common global ptr null, align 8
 
 ; Function Attrs: nounwind ssp uwtable
-define i32 @fn1() {
+define void @fn1() {
 ; CHECK-LABEL: @fn1(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[TMP0:%.*]] = load ptr, ptr @a, align 8
@@ -16,7 +16,7 @@ define i32 @fn1() {
 ; CHECK-NEXT:    [[ADD_PTR:%.*]] = getelementptr inbounds i64, ptr [[TMP0]], i64 11
 ; CHECK-NEXT:    [[TMP4:%.*]] = ptrtoint <2 x ptr> [[TMP3]] to <2 x i64>
 ; CHECK-NEXT:    store <2 x i64> [[TMP4]], ptr [[ADD_PTR]], align 8
-; CHECK-NEXT:    ret i32 undef
+; CHECK-NEXT:    ret void
 ;
 entry:
   %0 = load ptr, ptr @a, align 8
@@ -27,20 +27,46 @@ entry:
   %2 = ptrtoint ptr %add.ptr1 to i64
   %arrayidx2 = getelementptr inbounds i64, ptr %0, i64 12
   store i64 %2, ptr %arrayidx2, align 8
-  ret i32 undef
+  ret void
 }
 
 declare float @llvm.powi.f32.i32(float, i32)
 define void @fn2(ptr %a, ptr %b, ptr %c) {
 ; CHECK-LABEL: @fn2(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[TMP0:%.*]] = load <4 x i32>, ptr [[A:%.*]], align 4
-; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr [[B:%.*]], align 4
-; CHECK-NEXT:    [[TMP2:%.*]] = add <4 x i32> [[TMP0]], [[TMP1]]
-; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <4 x i32> [[TMP2]], i32 0
-; CHECK-NEXT:    [[TMP4:%.*]] = sitofp <4 x i32> [[TMP2]] to <4 x float>
-; CHECK-NEXT:    [[TMP5:%.*]] = call <4 x float> @llvm.powi.v4f32.i32(<4 x float> [[TMP4]], i32 [[TMP3]])
-; CHECK-NEXT:    store <4 x float> [[TMP5]], ptr [[C:%.*]], align 4
+; CHECK-NEXT:    [[I0:%.*]] = load i32, ptr [[A:%.*]], align 4
+; CHECK-NEXT:    [[I1:%.*]] = load i32, ptr [[B:%.*]], align 4
+; CHECK-NEXT:    [[ADD1:%.*]] = add i32 [[I0]], [[I1]]
+; CHECK-NEXT:    [[FP1:%.*]] = sitofp i32 [[ADD1]] to float
+; CHECK-NEXT:    [[CALL1:%.*]] = tail call float @llvm.powi.f32.i32(float [[FP1]], i32 [[ADD1]]) #[[ATTR2:[0-9]+]]
+; CHECK-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds i32, ptr [[A]], i32 1
+; CHECK-NEXT:    [[I2:%.*]] = load i32, ptr [[ARRAYIDX2]], align 4
+; CHECK-NEXT:    [[ARRAYIDX3:%.*]] = getelementptr inbounds i32, ptr [[B]], i32 1
+; CHECK-NEXT:    [[I3:%.*]] = load i32, ptr [[ARRAYIDX3]], align 4
+; CHECK-NEXT:    [[ADD2:%.*]] = add i32 [[I2]], [[I3]]
+; CHECK-NEXT:    [[FP2:%.*]] = sitofp i32 [[ADD2]] to float
+; CHECK-NEXT:    [[CALL2:%.*]] = tail call float @llvm.powi.f32.i32(float [[FP2]], i32 [[ADD1]]) #[[ATTR2]]
+; CHECK-NEXT:    [[ARRAYIDX4:%.*]] = getelementptr inbounds i32, ptr [[A]], i32 2
+; CHECK-NEXT:    [[I4:%.*]] = load i32, ptr [[ARRAYIDX4]], align 4
+; CHECK-NEXT:    [[ARRAYIDX5:%.*]] = getelementptr inbounds i32, ptr [[B]], i32 2
+; CHECK-NEXT:    [[I5:%.*]] = load i32, ptr [[ARRAYIDX5]], align 4
+; CHECK-NEXT:    [[ADD3:%.*]] = add i32 [[I4]], [[I5]]
+; CHECK-NEXT:    [[FP3:%.*]] = sitofp i32 [[ADD3]] to float
+; CHECK-NEXT:    [[CALL3:%.*]] = tail call float @llvm.powi.f32.i32(float [[FP3]], i32 [[ADD1]]) #[[ATTR2]]
+; CHECK-NEXT:    [[ARRAYIDX6:%.*]] = getelementptr inbounds i32, ptr [[A]], i32 3
+; CHECK-NEXT:    [[I6:%.*]] = load i32, ptr [[ARRAYIDX6]], align 4
+; CHECK-NEXT:    [[ARRAYIDX7:%.*]] = getelementptr inbounds i32, ptr [[B]], i32 3
+; CHECK-NEXT:    [[I7:%.*]] = load i32, ptr [[ARRAYIDX7]], align 4
+; CHECK-NEXT:    [[ADD4:%.*]] = add i32 [[I6]], [[I7]]
+; CHECK-NEXT:    [[FP4:%.*]] = sitofp i32 [[ADD4]] to float
+; CHECK-NEXT:    [[CALL4:%.*]] = tail call float @llvm.powi.f32.i32(float [[FP4]], i32 [[ADD1]]) #[[ATTR2]]
+; CHECK-NEXT:    store float [[CALL1]], ptr [[C:%.*]], align 4
+; CHECK-NEXT:    [[ARRAYIDX8:%.*]] = getelementptr inbounds float, ptr [[C]], i32 1
+; CHECK-NEXT:    store float [[CALL2]], ptr [[ARRAYIDX8]], align 4
+; CHECK-NEXT:    [[ARRAYIDX9:%.*]] = getelementptr inbounds float, ptr [[C]], i32 2
+; CHECK-NEXT:    store float [[CALL3]], ptr [[ARRAYIDX9]], align 4
+; CHECK-NEXT:    [[ARRAYIDX10:%.*]] = getelementptr inbounds float, ptr [[C]], i32 3
+; CHECK-NEXT:    store float [[CALL4]], ptr [[ARRAYIDX10]], align 4
 ; CHECK-NEXT:    ret void
 ;
 entry:
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/extracts-non-extendable.ll b/llvm/test/Transforms/SLPVectorizer/X86/extracts-non-extendable.ll
new file mode 100644
index 0000000000000..d87c40511fcf7
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/extracts-non-extendable.ll
@@ -0,0 +1,82 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -S --passes=slp-vectorizer -mtriple=x86_64-grtev4-linux-gnu -mattr="+aes,+avx,+cmov,+crc32,+cx16,+cx8,+fxsr,+mmx,+pclmul,+popcnt,+prfchw,+sse,+sse2,+sse3,+sse4.1,+sse4.2,+ssse3,+x87,+xsave" < %s | FileCheck %s
+
+define void @test(i64 %v) {
+; CHECK-LABEL: define void @test(
+; CHECK-SAME: i64 [[V:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT:  [[BB:.*:]]
+; CHECK-NEXT:    [[TMP0:%.*]] = insertelement <2 x i64> <i64 0, i64 poison>, i64 [[V]], i32 1
+; CHECK-NEXT:    [[TMP1:%.*]] = or <2 x i64> zeroinitializer, [[TMP0]]
+; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <2 x i64> [[TMP1]], i32 1
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp eq i64 0, [[TMP2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = icmp eq i64 0, 0
+; CHECK-NEXT:    [[TMP5:%.*]] = and i1 [[TMP3]], [[TMP4]]
+; CHECK-NEXT:    [[TMP6:%.*]] = icmp eq i64 0, 0
+; CHECK-NEXT:    [[TMP7:%.*]] = and i1 [[TMP5]], [[TMP6]]
+; CHECK-NEXT:    [[TMP8:%.*]] = icmp eq i64 0, 0
+; CHECK-NEXT:    [[TMP9:%.*]] = and i1 [[TMP7]], [[TMP8]]
+; CHECK-NEXT:    [[TMP10:%.*]] = and i1 [[TMP9]], false
+; CHECK-NEXT:    [[TMP11:%.*]] = icmp eq i64 0, [[TMP2]]
+; CHECK-NEXT:    [[TMP12:%.*]] = and i1 [[TMP10]], [[TMP11]]
+; CHECK-NEXT:    [[TMP13:%.*]] = icmp eq i64 0, 0
+; CHECK-NEXT:    [[TMP14:%.*]] = and i1 [[TMP12]], [[TMP13]]
+; CHECK-NEXT:    [[TMP15:%.*]] = icmp eq i64 0, 0
+; CHECK-NEXT:    [[TMP16:%.*]] = and i1 [[TMP14]], [[TMP15]]
+; CHECK-NEXT:    [[TMP17:%.*]] = icmp eq i64 0, 0
+; CHECK-NEXT:    [[TMP18:%.*]] = and i1 [[TMP16]], [[TMP17]]
+; CHECK-NEXT:    [[TMP19:%.*]] = icmp ult i64 0, 0
+; CHECK-NEXT:    [[TMP20:%.*]] = select i1 [[TMP19]], i1 [[TMP18]], i1 false
+; CHECK-NEXT:    br i1 [[TMP20]], label %[[BB_I107_PREHEADER:.*]], label %[[BB_I27_I_PREHEADER:.*]]
+; CHECK:       [[BB_I107_PREHEADER]]:
+; CHECK-NEXT:    [[TMP21:%.*]] = extractelement <2 x i64> [[TMP1]], i32 0
+; CHECK-NEXT:    [[DOTSROA_1278_10_EXTRACT_SHIFT83_I1622_1:%.*]] = xor i64 0, [[TMP21]]
+; CHECK-NEXT:    [[TMP22:%.*]] = xor <2 x i64> zeroinitializer, [[TMP1]]
+; CHECK-NEXT:    [[TMP23:%.*]] = or <2 x i64> [[TMP22]], zeroinitializer
+; CHECK-NEXT:    [[TMP24:%.*]] = or <2 x i64> splat (i64 1), [[TMP23]]
+; CHECK-NEXT:    [[TMP25:%.*]] = and <2 x i64> [[TMP24]], zeroinitializer
+; CHECK-NEXT:    [[TMP26:%.*]] = icmp eq <2 x i64> [[TMP25]], zeroinitializer
+; CHECK-NEXT:    ret void
+; CHECK:       [[BB_I27_I_PREHEADER]]:
+; CHECK-NEXT:    unreachable
+;
+bb:
+  %.sroa.82529.14.insert.insert = or i64 0, 0
+  %.sroa.02528.sroa.0.0.insert.insert = or i64 %v, 0
+  %0 = icmp eq i64 0, %.sroa.02528.sroa.0.0.insert.insert
+  %1 = icmp eq i64 0, 0
+  %2 = and i1 %0, %1
+  %3 = icmp eq i64 0, 0
+  %4 = and i1 %2, %3
+  %5 = icmp eq i64 0, 0
+  %6 = and i1 %4, %5
+  %7 = and i1 %6, false
+  %8 = icmp eq i64 0, %.sroa.02528.sroa.0.0.insert.insert
+  %9 = and i1 %7, %8
+  %10 = icmp eq i64 0, 0
+  %11 = and i1 %9, %10
+  %12 = icmp eq i64 0, 0
+  %13 = and i1 %11, %12
+  %14 = icmp eq i64 0, 0
+  %15 = and i1 %13, %14
+  %16 = icmp ult i64 0, 0
+  %17 = select i1 %16, i1 %15, i1 false
+  br i1 %17, label %bb.i107.preheader, label %bb.i27.i.preheader
+
+bb.i107.preheader: ; preds = %bb
+  %.sroa.1278.10.extract.shift83.i1622.1 = xor i64 0, %.sroa.82529.14.insert.insert
+  %.sroa.076.2.extract.shift80.i1619.4 = xor i64 0, %.sroa.02528.sroa.0.0.insert.insert
+  %.sroa.071.2.extract.shift86.i1625.4 = or i64 %.sroa.076.2.extract.shift80.i1619.4, 0
+  %.sroa.1278.10.extract.shift83.i1622.7 = xor i64 0, %.sroa.82529.14.insert.insert
+  %.sroa.12.10.extract.shift89.i1634.7 = or i64 %.sroa.1278.10.extract.shift83.i1622.7, 0
+  %.sroa.02756.2.extract.shift6530 = or i64 %.sroa.071.2.extract.shift86.i1625.4, 1
+  %18 = and i64 %.sroa.02756.2.extract.shift6530, 0
+  %19 = icmp eq i64 %18, 0
+  %20 = or i64 1, %.sroa.12.10.extract.shift89.i1634.7
+  %21 = and i64 %20, 0
+  %22 = icmp eq i64 %21, 0
+  ret void
+
+bb.i27.i.preheader: ; preds = %bb
+  unreachable
+}
+
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/gathered-shuffle-resized.ll b/llvm/test/Transforms/SLPVectorizer/X86/gathered-shuffle-resized.ll
new file mode 100644
index 0000000000000..48b04201d1acc
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/gathered-shuffle-resized.ll
@@ -0,0 +1,182 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -S --passes=slp-vectorizer -mtriple=x86_64-grtev4-linux-gnu < %s | FileCheck %s
+
+define ptr @test(ptr %0, ptr %args_gep) {
+; CHECK-LABEL: define ptr @test(
+; CHECK-SAME: ptr [[TMP0:%.*]], ptr [[ARGS_GEP:%.*]]) {
+; CHECK-NEXT:  [[BB:.*:]]
+; CHECK-NEXT:    [[ARG1:%.*]] = load ptr, ptr [[ARGS_GEP]], align 8
+; CHECK-NEXT:    [[ARG26:%.*]] = load ptr, ptr [[TMP0]], align 8
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i32>, ptr [[ARG1]], align 32
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <2 x i32> [[TMP1]], <2 x i32> poison, <16 x i32> <i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 0, i32 0, i32 0, i32 0>
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp eq <16 x i32> [[TMP2]], zeroinitializer
+; CHECK-NEXT:    [[TMP4:%.*]] = zext <16 x i1> [[TMP3]] to <16 x i8>
+; CHECK-NEXT:    store <16 x i8> [[TMP4]], ptr [[ARG26]], align 32
+; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr i8, ptr [[ARG26]], i64 17
+; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr i8, ptr [[ARG1]], i64 8
+; CHECK-NEXT:    [[TMP7:%.*]] = getelementptr i8, ptr [[ARG1]], i64 12
+; CHECK-NEXT:    [[TMP8:%.*]] = load i32, ptr [[TMP7]], align 4, !noalias [[META0:![0-9]+]]
+; CHECK-NEXT:    [[TMP9:%.*]] = load <2 x i32>, ptr [[TMP6]], align 8, !noalias [[META0]]
+; CHECK-NEXT:    [[TMP10:%.*]] = shufflevector <2 x i32> [[TMP1]], <2 x i32> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP11:%.*]] = shufflevector <2 x i32> [[TMP9]], <2 x i32> poison, <16 x i32> <i32 poison, i32 poison, i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP12:%.*]] = shufflevector <4 x i32> [[TMP10]], <4 x i32> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP13:%.*]] = shufflevector <2 x i32> [[TMP1]], <2 x i32> [[TMP9]], <16 x i32> <i32 0, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 3, i32 3, i32 3>
+; CHECK-NEXT:    [[TMP14:%.*]] = icmp eq <16 x i32> [[TMP13]], zeroinitializer
+; CHECK-NEXT:    [[TMP15:%.*]] = zext <16 x i1> [[TMP14]] to <16 x i8>
+; CHECK-NEXT:    store <16 x i8> [[TMP15]], ptr [[TMP5]], align 1
+; CHECK-NEXT:    [[TMP16:%.*]] = icmp eq i32 [[TMP8]], 0
+; CHECK-NEXT:    [[TMP17:%.*]] = zext i1 [[TMP16]] to i8
+; CHECK-NEXT:    store i8 [[TMP17]], ptr [[TMP0]], align 1
+; CHECK-NEXT:    ret ptr null
+;
+bb:
+  %arg1 = load ptr, ptr %args_gep, align 8
+  %arg26 = load ptr, ptr %0, align 8
+  %1 = load i32, ptr %arg1, align 32
+  %2 = icmp eq i32 %1, 0
+  %3 = zext i1 %2 to i8
+  store i8 %3, ptr %arg26, align 32, !alias.scope !0
+  %4 = icmp eq i32 %1, 0
+  %5 = zext i1 %4 to i8
+  %6 = getelementptr i8, ptr %arg26, i64 1
+  store i8 %5, ptr %6, align 1, !alias.scope !0
+  %7 = icmp eq i32 %1, 0
+  %8 = zext i1 %7 to i8
+  %9 = getelementptr i8, ptr %arg26, i64 2
+  store i8 %8, ptr %9, align 2, !alias.scope !0
+  %10 = icmp eq i32 %1, 0
+  %11 = zext i1 %10 to i8
+  %12 = getelementptr i8, ptr %arg26, i64 3
+  store i8 %11, ptr %12, align 1, !alias.scope !0
+  %13 = icmp eq i32 %1, 0
+  %14 = zext i1 %13 to i8
+  %15 = getelementptr i8, ptr %arg26, i64 4
+  store i8 %14, ptr %15, align 4, !alias.scope !0
+  %16 = icmp eq i32 %1, 0
+  %17 = zext i1 %16 to i8
+  %18 = getelementptr i8, ptr %arg26, i64 5
+  store i8 %17, ptr %18, align 1, !alias.scope !0
+  %19 = getelementptr i8, ptr %arg1, i64 4
+  %20 = load i32, ptr %19, align 4, !noalias !0
+  %21 = icmp eq i32 %20, 0
+  %22 = zext i1 %21 to i8
+  %23 = getelementptr i8, ptr %arg26, i64 6
+  store i8 %22, ptr %23, align 2
+  %24 = icmp eq i32 %20, 0
+  %25 = zext i1 %24 to i8
+  %26 = getelementptr i8, ptr %arg26, i64 7
+  store i8 %25, ptr %26, align 1
+  %27 = icmp eq i32 %20, 0
+  %28 = zext i1 %27 to i8
+  %29 = getelementptr i8, ptr %arg26, i64 8
+  store i8 %28, ptr %29, align 8
+  %30 = icmp eq i32 %20, 0
+  %31 = zext i1 %30 to i8
+  %32 = getelementptr i8, ptr %arg26, i64 9
+  store i8 %31, ptr %32, align 1
+  %33 = icmp eq i32 %20, 0
+  %34 = zext i1 %33 to i8
+  %35 = getelementptr i8, ptr %arg26, i64 10
+  store i8 %34, ptr %35, align 2
+  %36 = icmp eq i32 %20, 0
+  %37 = zext i1 %36 to i8
+  %38 = getelementptr i8, ptr %arg26, i64 11
+  store i8 %37, ptr %38, align 1
+  %39 = icmp eq i32 %1, 0
+  %40 = zext i1 %39 to i8
+  %41 = getelementptr i8, ptr %arg26, i64 12
+  store i8 %40, ptr %41, align 4
+  %42 = icmp eq i32 %1, 0
+  %43 = zext i1 %42 to i8
+  %44 = getelementptr i8, ptr %arg26, i64 13
+  store i8 %43, ptr %44, align 1
+  %45 = icmp eq i32 %1, 0
+  %46 = zext i1 %45 to i8
+  %47 = getelementptr i8, ptr %arg26, i64 14
+  store i8 %46, ptr %47, align 2
+  %48 = icmp eq i32 %1, 0
+  %49 = zext i1 %48 to i8
+  %50 = getelementptr i8, ptr %arg26, i64 15
+  store i8 %49, ptr %50, align 1
+  %51 = icmp eq i32 %1, 0
+  %52 = zext i1 %51 to i8
+  %53 = getelementptr i8, ptr %arg26, i64 17
+  store i8 %52, ptr %53, align 1, !alias.scope !0
+  %54 = icmp eq i32 %20, 0
+  %55 = zext i1 %54 to i8
+  %56 = getelementptr i8, ptr %arg26, i64 18
+  store i8 %55, ptr %56, align 2, !alias.scope !0
+  %57 = icmp eq i32 %20, 0
+  %58 = zext i1 %57 to i8
+  %59 = getelementptr i8, ptr %arg26, i64 19
+  store i8 %58, ptr %59, align 1, !alias.scope !0
+  %60 = icmp eq i32 %20, 0
+  %61 = zext i1 %60 to i8
+  %62 = getelementptr i8, ptr %arg26, i64 20
+  store i8 %61, ptr %62, align 4, !alias.scope !0
+  %63 = icmp eq i32 %20, 0
+  %64 = zext i1 %63 to i8
+  %65 = getelementptr i8, ptr %arg26, i64 21
+  store i8 %64, ptr %65, align 1, !alias.scope !0
+  %66 = icmp eq i32 %20, 0
+  %67 = zext i1 %66 to i8
+  %68 = getelementptr i8, ptr %arg26, i64 22
+  store i8 %67, ptr %68, align 2, !alias.scope !0
+  %69 = icmp eq i32 %20, 0
+  %70 = zext i1 %69 to i8
+  %71 = getelementptr i8, ptr %arg26, i64 23
+  store i8 %70, ptr %71, align 1, !alias.scope !0
+  %72 = getelementptr i8, ptr %arg1, i64 8
+  %73 = load i32, ptr %72, align 8, !noalias !0
+  %74 = icmp eq i32 %73, 0
+  %75 = zext i1 %74 to i8
+  %76 = getelementptr i8, ptr %arg26, i64 24
+  store i8 %75, ptr %76, align 8, !alias.scope !0
+  %77 = icmp eq i32 %73, 0
+  %78 = zext i1 %77 to i8
+  %79 = getelementptr i8, ptr %arg26, i64 25
+  store i8 %78, ptr %79, align 1, !alias.scope !0
+  %80 = icmp eq i32 %73, 0
+  %81 = zext i1 %80 to i8
+  %82 = getelementptr i8, ptr %arg26, i64 26
+  store i8 %81, ptr %82, align 2, !alias.scope !0
+  %83 = icmp eq i32 %73, 0
+  %84 = zext i1 %83 to i8
+  %85 = getelementptr i8, ptr %arg26, i64 27
+  store i8 %84, ptr %85, align 1, !alias.scope !0
+  %86 = icmp eq i32 %73, 0
+  %87 = zext i1 %86 to i8
+  %88 = getelementptr i8, ptr %arg26, i64 28
+  store i8 %87, ptr %88, align 4, !alias.scope !0
+  %89 = icmp eq i32 %73, 0
+  %90 = zext i1 %89 to i8
+  %91 = getelementptr i8, ptr %arg26, i64 29
+  store i8 %90, ptr %91, align 1, !alias.scope !0
+  %92 = getelementptr i8, ptr %arg1, i64 12
+  %93 = load i32, ptr %92, align 4, !noalias !0
+  %94 = icmp eq i32 %93, 0
+  %95 = zext i1 %94 to i8
+  %96 = getelementptr i8, ptr %arg26, i64 30
+  store i8 %95, ptr %96, align 2
+  %97 = icmp eq i32 %93, 0
+  %98 = zext i1 %97 to i8
+  %99 = getelementptr i8, ptr %arg26, i64 31
+  store i8 %98, ptr %99, align 1
+  %100 = icmp eq i32 %93, 0
+  %101 = zext i1 %100 to i8
+  %102 = getelementptr i8, ptr %arg26, i64 32
+  store i8 %101, ptr %102, align 32
+  %103 = icmp eq i32 %93, 0
+  %104 = zext i1 %103 to i8
+  store i8 %104, ptr %0, align 1
+  ret ptr null
+}
+
+!0 = !{!1}
+!1 = !{!"result slice: {}", !2}
+!2 = !{!"fn AA domain"}
+;.
+; CHECK: [[META0]] = !{[[META1:![0-9]+]]}
+; CHECK: [[META1]] = !{!"result slice: {}", [[META2:![0-9]+]]}
+; CHECK: [[META2]] = !{!"fn AA domain"}
+;.
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/intrinsic.ll b/llvm/test/Transforms/SLPVectorizer/X86/intrinsic.ll
index 0eaf89da36601..bb7d54c8a78c3 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/intrinsic.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/intrinsic.ll
@@ -378,11 +378,35 @@ declare float @llvm.powi.f32.i32(float, i32)
 define void @vec_powi_f32(ptr %a, ptr %b, ptr %c, i32 %P) {
 ; CHECK-LABEL: @vec_powi_f32(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x float>, ptr [[A:%.*]], align 4
-; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x float>, ptr [[B:%.*]], align 4
-; CHECK-NEXT:    [[TMP4:%.*]] = fadd <4 x float> [[TMP1]], [[TMP3]]
-; CHECK-NEXT:    [[TMP5:%.*]] = call <4 x float> @llvm.powi.v4f32.i32(<4 x float> [[TMP4]], i32 [[P:%.*]])
-; CHECK-NEXT:    store <4 x float> [[TMP5]], ptr [[C:%.*]], align 4
+; CHECK-NEXT:    [[I0:%.*]] = load float, ptr [[A:%.*]], align 4
+; CHECK-NEXT:    [[I1:%.*]] = load float, ptr [[B:%.*]], align 4
+; CHECK-NEXT:    [[ADD1:%.*]] = fadd float [[I0]], [[I1]]
+; CHECK-NEXT:    [[CALL1:%.*]] = tail call float @llvm.powi.f32.i32(float [[ADD1]], i32 [[P:%.*]]) #[[ATTR3]]
+; CHECK-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds float, ptr [[A]], i32 1
+; CHECK-NEXT:    [[I2:%.*]] = load float, ptr [[ARRAYIDX2]], align 4
+; CHECK-NEXT:    [[ARRAYIDX3:%.*]] = getelementptr inbounds float, ptr [[B]], i32 1
+; CHECK-NEXT:    [[I3:%.*]] = load float, ptr [[ARRAYIDX3]], align 4
+; CHECK-NEXT:    [[ADD2:%.*]] = fadd float [[I2]], [[I3]]
+; CHECK-NEXT:    [[CALL2:%.*]] = tail call float @llvm.powi.f32.i32(float [[ADD2]], i32 [[P]]) #[[ATTR3]]
+; CHECK-NEXT:    [[ARRAYIDX4:%.*]] = getelementptr inbounds float, ptr [[A]], i32 2
+; CHECK-NEXT:    [[I4:%.*]] = load float, ptr [[ARRAYIDX4]], align 4
+; CHECK-NEXT:    [[ARRAYIDX5:%.*]] = getelementptr inbounds float, ptr [[B]], i32 2
+; CHECK-NEXT:    [[I5:%.*]] = load float, ptr [[ARRAYIDX5]], align 4
+; CHECK-NEXT:    [[ADD3:%.*]] = fadd float [[I4]], [[I5]]
+; CHECK-NEXT:    [[CALL3:%.*]] = tail call float @llvm.powi.f32.i32(float [[ADD3]], i32 [[P]]) #[[ATTR3]]
+; CHECK-NEXT:    [[ARRAYIDX6:%.*]] = getelementptr inbounds float, ptr [[A]], i32 3
+; CHECK-NEXT:    [[I6:%.*]] = load float, ptr [[ARRAYIDX6]], align 4
+; CHECK-NEXT:    [[ARRAYIDX7:%.*]] = getelementptr inbounds float, ptr [[B]], i32 3
+; CHECK-NEXT:    [[I7:%.*]] = load float, ptr [[ARRAYIDX7]], align 4
+; CHECK-NEXT:    [[ADD4:%.*]] = fadd float [[I6]], [[I7]]
+; CHECK-NEXT:    [[CALL4:%.*]] = tail call float @llvm.powi.f32.i32(float [[ADD4]], i32 [[P]]) #[[ATTR3]]
+; CHECK-NEXT:    store float [[CALL1]], ptr [[C:%.*]], align 4
+; CHECK-NEXT:    [[ARRAYIDX8:%.*]] = getelementptr inbounds float, ptr [[C]], i32 1
+; CHECK-NEXT:    store float [[CALL2]], ptr [[ARRAYIDX8]], align 4
+; CHECK-NEXT:    [[ARRAYIDX9:%.*]] = getelementptr inbounds float, ptr [[C]], i32 2
+; CHECK-NEXT:    store float [[CALL3]], ptr [[ARRAYIDX9]], align 4
+; CHECK-NEXT:    [[ARRAYIDX10:%.*]] = getelementptr inbounds float, ptr [[C]], i32 3
+; CHECK-NEXT:    store float [[CALL4]], ptr [[ARRAYIDX10]], align 4
 ; CHECK-NEXT:    ret void
 ;
 entry:
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/intrinsic_with_scalar_param.ll b/llvm/test/Transforms/SLPVectorizer/X86/intrinsic_with_scalar_param.ll
index f87a713faf5f5..f8f2459fd9c43 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/intrinsic_with_scalar_param.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/intrinsic_with_scalar_param.ll
@@ -5,9 +5,24 @@ declare float @llvm.powi.f32.i32(float, i32)
 define void @vec_powi_f32(ptr %a, ptr %c, i32 %P) {
 ; CHECK-LABEL: @vec_powi_f32(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x float>, ptr [[A:%.*]], align 4
-; CHECK-NEXT:    [[TMP2:%.*]] = call <4 x float> @llvm.powi.v4f32.i32(<4 x float> [[TMP1]], i32 [[P:%.*]])
-; CHECK-NEXT:    store <4 x float> [[TMP2]], ptr [[C:%.*]], align 4
+; CHECK-NEXT:    [[I0:%.*]] = load float, ptr [[A:%.*]], align 4
+; CHECK-NEXT:    [[CALL1:%.*]] = tail call float @llvm.powi.f32.i32(float [[I0]], i32 [[P:%.*]])
+; CHECK-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds float, ptr [[A]], i32 1
+; CHECK-NEXT:    [[I2:%.*]] = load float, ptr [[ARRAYIDX2]], align 4
+; CHECK-NEXT:    [[CALL2:%.*]] = tail call float @llvm.powi.f32.i32(float [[I2]], i32 [[P]])
+; CHECK-NEXT:    [[ARRAYIDX4:%.*]] = getelementptr inbounds float, ptr [[A]], i32 2
+; CHECK-NEXT:    [[I4:%.*]] = load float, ptr [[ARRAYIDX4]], align 4
+; CHECK-NEXT:    [[CALL3:%.*]] = tail call float @llvm.powi.f32.i32(float [[I4]], i32 [[P]])
+; CHECK-NEXT:    [[ARRAYIDX6:%.*]] = getelementptr inbounds float, ptr [[A]], i32 3
+; CHECK-NEXT:    [[I6:%.*]] = load float, ptr [[ARRAYIDX6]], align 4
+; CHECK-NEXT:    [[CALL4:%.*]] = tail call float @llvm.powi.f32.i32(float [[I6]], i32 [[P]])
+; CHECK-NEXT:    store float [[CALL1]], ptr [[C:%.*]], align 4
+; CHECK-NEXT:    [[ARRAYIDX8:%.*]] = getelementptr inbounds float, ptr [[C]], i32 1
+; CHECK-NEXT:    store float [[CALL2]], ptr [[ARRAYIDX8]], align 4
+; CHECK-NEXT:    [[ARRAYIDX9:%.*]] = getelementptr inbounds float, ptr [[C]], i32 2
+; CHECK-NEXT:    store float [[CALL3]], ptr [[ARRAYIDX9]], align 4
+; CHECK-NEXT:    [[ARRAYIDX10:%.*]] = getelementptr inbounds float, ptr [[C]], i32 3
+; CHECK-NEXT:    store float [[CALL4]], ptr [[ARRAYIDX10]], align 4
 ; CHECK-NEXT:    ret void
 ;
 entry:
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/load-partial-vector-shuffle.ll b/llvm/test/Transforms/SLPVectorizer/X86/load-partial-vector-shuffle.ll
index 8d37665224e7e..f18a72b0bf776 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/load-partial-vector-shuffle.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/load-partial-vector-shuffle.ll
@@ -10,24 +10,16 @@ define <2 x i64> @load_00123456(ptr nocapture noundef readonly %data) {
 ; SSE-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds i16, ptr [[DATA:%.*]], i64 1
 ; SSE-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds i16, ptr [[DATA]], i64 2
 ; SSE-NEXT:    [[ARRAYIDX3:%.*]] = getelementptr inbounds i16, ptr [[DATA]], i64 3
-; SSE-NEXT:    [[ARRAYIDX4:%.*]] = getelementptr inbounds i16, ptr [[DATA]], i64 4
-; SSE-NEXT:    [[ARRAYIDX5:%.*]] = getelementptr inbounds i16, ptr [[DATA]], i64 5
-; SSE-NEXT:    [[ARRAYIDX6:%.*]] = getelementptr inbounds i16, ptr [[DATA]], i64 6
 ; SSE-NEXT:    [[T0:%.*]] = load i16, ptr [[DATA]], align 2
 ; SSE-NEXT:    [[T1:%.*]] = load i16, ptr [[ARRAYIDX1]], align 2
 ; SSE-NEXT:    [[T2:%.*]] = load i16, ptr [[ARRAYIDX2]], align 2
-; SSE-NEXT:    [[T3:%.*]] = load i16, ptr [[ARRAYIDX3]], align 2
-; SSE-NEXT:    [[T4:%.*]] = load i16, ptr [[ARRAYIDX4]], align 2
-; SSE-NEXT:    [[T5:%.*]] = load i16, ptr [[ARRAYIDX5]], align 2
-; SSE-NEXT:    [[T6:%.*]] = load i16, ptr [[ARRAYIDX6]], align 2
+; SSE-NEXT:    [[TMP1:%.*]] = load <4 x i16>, ptr [[ARRAYIDX3]], align 2
 ; SSE-NEXT:    [[VECINIT0_I_I:%.*]] = insertelement <8 x i16> undef, i16 [[T0]], i64 0
 ; SSE-NEXT:    [[VECINIT1_I_I:%.*]] = insertelement <8 x i16> [[VECINIT0_I_I]], i16 [[T0]], i64 1
 ; SSE-NEXT:    [[VECINIT2_I_I:%.*]] = insertelement <8 x i16> [[VECINIT1_I_I]], i16 [[T1]], i64 2
 ; SSE-NEXT:    [[VECINIT3_I_I:%.*]] = insertelement <8 x i16> [[VECINIT2_I_I]], i16 [[T2]], i64 3
-; SSE-NEXT:    [[VECINIT4_I_I:%.*]] = insertelement <8 x i16> [[VECINIT3_I_I]], i16 [[T3]], i64 4
-; SSE-NEXT:    [[VECINIT5_I_I:%.*]] = insertelement <8 x i16> [[VECINIT4_I_I]], i16 [[T4]], i64 5
-; SSE-NEXT:    [[VECINIT6_I_I:%.*]] = insertelement <8 x i16> [[VECINIT5_I_I]], i16 [[T5]], i64 6
-; SSE-NEXT:    [[VECINIT7_I_I:%.*]] = insertelement <8 x i16> [[VECINIT6_I_I]], i16 [[T6]], i64 7
+; SSE-NEXT:    [[TMP2:%.*]] = shufflevector <4 x i16> [[TMP1]], <4 x i16> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE-NEXT:    [[VECINIT7_I_I:%.*]] = shufflevector <8 x i16> [[VECINIT3_I_I]], <8 x i16> [[TMP2]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
 ; SSE-NEXT:    [[T7:%.*]] = bitcast <8 x i16> [[VECINIT7_I_I]] to <2 x i64>
 ; SSE-NEXT:    ret <2 x i64> [[T7]]
 ;
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/powi-regression.ll b/llvm/test/Transforms/SLPVectorizer/X86/powi-regression.ll
index 0eeef0ddf154d..0cf0d77e3f29b 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/powi-regression.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/powi-regression.ll
@@ -6,13 +6,18 @@
 define <2 x double> @PR53887_v2f64(<2 x double> noundef %x) {
 ; CHECK-LABEL: @PR53887_v2f64(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[TMP0:%.*]] = call fast <2 x double> @llvm.powi.v2f64.i32(<2 x double> [[X:%.*]], i32 6)
+; CHECK-NEXT:    [[VECEXT:%.*]] = extractelement <2 x double> [[X:%.*]], i64 0
+; CHECK-NEXT:    [[TMP2:%.*]] = tail call fast double @llvm.powi.f64.i32(double [[VECEXT]], i32 6)
+; CHECK-NEXT:    [[VECINIT:%.*]] = insertelement <2 x double> zeroinitializer, double [[TMP2]], i64 0
+; CHECK-NEXT:    [[VECEXT1:%.*]] = extractelement <2 x double> [[X]], i64 1
+; CHECK-NEXT:    [[TMP1:%.*]] = tail call fast double @llvm.powi.f64.i32(double [[VECEXT1]], i32 6)
+; CHECK-NEXT:    [[TMP0:%.*]] = insertelement <2 x double> [[VECINIT]], double [[TMP1]], i64 1
 ; CHECK-NEXT:    ret <2 x double> [[TMP0]]
 ;
 entry:
   %vecext = extractelement <2 x double> %x, i64 0
   %0 = tail call fast double @llvm.powi.f64.i32(double %vecext, i32 6)
-  %vecinit = insertelement <2 x double> undef, double %0, i64 0
+  %vecinit = insertelement <2 x double> zeroinitializer, double %0, i64 0
   %vecext1 = extractelement <2 x double> %x, i64 1
   %1 = tail call fast double @llvm.powi.f64.i32(double %vecext1, i32 6)
   %vecinit3 = insertelement <2 x double> %vecinit, double %1, i64 1
@@ -22,13 +27,24 @@ entry:
 define <4 x double> @PR53887_v4f64(<4 x double> noundef %x) {
 ; CHECK-LABEL: @PR53887_v4f64(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[TMP0:%.*]] = call fast <4 x double> @llvm.powi.v4f64.i32(<4 x double> [[X:%.*]], i32 6)
+; CHECK-NEXT:    [[VECEXT:%.*]] = extractelement <4 x double> [[X:%.*]], i64 0
+; CHECK-NEXT:    [[TMP4:%.*]] = tail call fast double @llvm.powi.f64.i32(double [[VECEXT]], i32 6)
+; CHECK-NEXT:    [[VECINIT:%.*]] = insertelement <4 x double> zeroinitializer, double [[TMP4]], i64 0
+; CHECK-NEXT:    [[VECEXT1:%.*]] = extractelement <4 x double> [[X]], i64 1
+; CHECK-NEXT:    [[TMP1:%.*]] = tail call fast double @llvm.powi.f64.i32(double [[VECEXT1]], i32 6)
+; CHECK-NEXT:    [[VECINIT3:%.*]] = insertelement <4 x double> [[VECINIT]], double [[TMP1]], i64 1
+; CHECK-NEXT:    [[VECEXT4:%.*]] = extractelement <4 x double> [[X]], i64 2
+; CHECK-NEXT:    [[TMP2:%.*]] = tail call fast double @llvm.powi.f64.i32(double [[VECEXT4]], i32 6)
+; CHECK-NEXT:    [[VECINIT6:%.*]] = insertelement <4 x double> [[VECINIT3]], double [[TMP2]], i64 2
+; CHECK-NEXT:    [[VECEXT7:%.*]] = extractelement <4 x double> [[X]], i64 3
+; CHECK-NEXT:    [[TMP3:%.*]] = tail call fast double @llvm.powi.f64.i32(double [[VECEXT7]], i32 6)
+; CHECK-NEXT:    [[TMP0:%.*]] = insertelement <4 x double> [[VECINIT6]], double [[TMP3]], i64 3
 ; CHECK-NEXT:    ret <4 x double> [[TMP0]]
 ;
 entry:
   %vecext = extractelement <4 x double> %x, i64 0
   %0 = tail call fast double @llvm.powi.f64.i32(double %vecext, i32 6) #2
-  %vecinit = insertelement <4 x double> undef, double %0, i64 0
+  %vecinit = insertelement <4 x double> zeroinitializer, double %0, i64 0
   %vecext1 = extractelement <4 x double> %x, i64 1
   %1 = tail call fast double @llvm.powi.f64.i32(double %vecext1, i32 6) #2
   %vecinit3 = insertelement <4 x double> %vecinit, double %1, i64 1
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/powi.ll b/llvm/test/Transforms/SLPVectorizer/X86/powi.ll
index ce5f912f140b4..a177b46ffde14 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/powi.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/powi.ll
@@ -6,7 +6,12 @@
 
 define <2 x double> @buildvector_powi_2f64_6(<2 x double> %a) {
 ; CHECK-LABEL: @buildvector_powi_2f64_6(
-; CHECK-NEXT:    [[TMP1:%.*]] = call <2 x double> @llvm.powi.v2f64.i32(<2 x double> [[A:%.*]], i32 6)
+; CHECK-NEXT:    [[A0:%.*]] = extractelement <2 x double> [[A:%.*]], i32 0
+; CHECK-NEXT:    [[A1:%.*]] = extractelement <2 x double> [[A]], i32 1
+; CHECK-NEXT:    [[C0:%.*]] = call double @llvm.powi.f64.i32(double [[A0]], i32 6)
+; CHECK-NEXT:    [[C1:%.*]] = call double @llvm.powi.f64.i32(double [[A1]], i32 6)
+; CHECK-NEXT:    [[R0:%.*]] = insertelement <2 x double> poison, double [[C0]], i32 0
+; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <2 x double> [[R0]], double [[C1]], i32 1
 ; CHECK-NEXT:    ret <2 x double> [[TMP1]]
 ;
   %a0 = extractelement <2 x double> %a, i32 0
@@ -39,7 +44,18 @@ define <2 x double> @buildvector_powi_2f64_var(<2 x double> %a, i32 %b) {
 
 define <4 x float> @buildvector_powi_4f32_3(<4 x float> %a) {
 ; CHECK-LABEL: @buildvector_powi_4f32_3(
-; CHECK-NEXT:    [[TMP1:%.*]] = call <4 x float> @llvm.powi.v4f32.i32(<4 x float> [[A:%.*]], i32 3)
+; CHECK-NEXT:    [[A0:%.*]] = extractelement <4 x float> [[A:%.*]], i32 0
+; CHECK-NEXT:    [[A1:%.*]] = extractelement <4 x float> [[A]], i32 1
+; CHECK-NEXT:    [[A2:%.*]] = extractelement <4 x float> [[A]], i32 2
+; CHECK-NEXT:    [[A3:%.*]] = extractelement <4 x float> [[A]], i32 3
+; CHECK-NEXT:    [[C0:%.*]] = call float @llvm.powi.f32.i32(float [[A0]], i32 3)
+; CHECK-NEXT:    [[C1:%.*]] = call float @llvm.powi.f32.i32(float [[A1]], i32 3)
+; CHECK-NEXT:    [[C2:%.*]] = call float @llvm.powi.f32.i32(float [[A2]], i32 3)
+; CHECK-NEXT:    [[C3:%.*]] = call float @llvm.powi.f32.i32(float [[A3]], i32 3)
+; CHECK-NEXT:    [[R0:%.*]] = insertelement <4 x float> poison, float [[C0]], i32 0
+; CHECK-NEXT:    [[R1:%.*]] = insertelement <4 x float> [[R0]], float [[C1]], i32 1
+; CHECK-NEXT:    [[R2:%.*]] = insertelement <4 x float> [[R1]], float [[C2]], i32 2
+; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <4 x float> [[R2]], float [[C3]], i32 3
 ; CHECK-NEXT:    ret <4 x float> [[TMP1]]
 ;
   %a0 = extractelement <4 x float> %a, i32 0
@@ -63,7 +79,18 @@ define <4 x float> @buildvector_powi_4f32_3(<4 x float> %a) {
 
 define <4 x double> @buildvector_powi_4f64_16(<4 x double> %a) {
 ; CHECK-LABEL: @buildvector_powi_4f64_16(
-; CHECK-NEXT:    [[TMP1:%.*]] = call <4 x double> @llvm.powi.v4f64.i32(<4 x double> [[A:%.*]], i32 16)
+; CHECK-NEXT:    [[A0:%.*]] = extractelement <4 x double> [[A:%.*]], i32 0
+; CHECK-NEXT:    [[A1:%.*]] = extractelement <4 x double> [[A]], i32 1
+; CHECK-NEXT:    [[A2:%.*]] = extractelement <4 x double> [[A]], i32 2
+; CHECK-NEXT:    [[A3:%.*]] = extractelement <4 x double> [[A]], i32 3
+; CHECK-NEXT:    [[C0:%.*]] = call double @llvm.powi.f64.i32(double [[A0]], i32 16)
+; CHECK-NEXT:    [[C1:%.*]] = call double @llvm.powi.f64.i32(double [[A1]], i32 16)
+; CHECK-NEXT:    [[C2:%.*]] = call double @llvm.powi.f64.i32(double [[A2]], i32 16)
+; CHECK-NEXT:    [[C3:%.*]] = call double @llvm.powi.f64.i32(double [[A3]], i32 16)
+; CHECK-NEXT:    [[R0:%.*]] = insertelement <4 x double> poison, double [[C0]], i32 0
+; CHECK-NEXT:    [[R1:%.*]] = insertelement <4 x double> [[R0]], double [[C1]], i32 1
+; CHECK-NEXT:    [[R2:%.*]] = insertelement <4 x double> [[R1]], double [[C2]], i32 2
+; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <4 x double> [[R2]], double [[C3]], i32 3
 ; CHECK-NEXT:    ret <4 x double> [[TMP1]]
 ;
   %a0 = extractelement <4 x double> %a, i32 0
@@ -83,7 +110,30 @@ define <4 x double> @buildvector_powi_4f64_16(<4 x double> %a) {
 
 define <8 x float> @buildvector_powi_8f32_4(<8 x float> %a) {
 ; CHECK-LABEL: @buildvector_powi_8f32_4(
-; CHECK-NEXT:    [[TMP1:%.*]] = call <8 x float> @llvm.powi.v8f32.i32(<8 x float> [[A:%.*]], i32 4)
+; CHECK-NEXT:    [[A0:%.*]] = extractelement <8 x float> [[A:%.*]], i32 0
+; CHECK-NEXT:    [[A1:%.*]] = extractelement <8 x float> [[A]], i32 1
+; CHECK-NEXT:    [[A2:%.*]] = extractelement <8 x float> [[A]], i32 2
+; CHECK-NEXT:    [[A3:%.*]] = extractelement <8 x float> [[A]], i32 3
+; CHECK-NEXT:    [[A4:%.*]] = extractelement <8 x float> [[A]], i32 4
+; CHECK-NEXT:    [[A5:%.*]] = extractelement <8 x float> [[A]], i32 5
+; CHECK-NEXT:    [[A6:%.*]] = extractelement <8 x float> [[A]], i32 6
+; CHECK-NEXT:    [[A7:%.*]] = extractelement <8 x float> [[A]], i32 7
+; CHECK-NEXT:    [[C0:%.*]] = call float @llvm.powi.f32.i32(float [[A0]], i32 4)
+; CHECK-NEXT:    [[C1:%.*]] = call float @llvm.powi.f32.i32(float [[A1]], i32 4)
+; CHECK-NEXT:    [[C2:%.*]] = call float @llvm.powi.f32.i32(float [[A2]], i32 4)
+; CHECK-NEXT:    [[C3:%.*]] = call float @llvm.powi.f32.i32(float [[A3]], i32 4)
+; CHECK-NEXT:    [[C4:%.*]] = call float @llvm.powi.f32.i32(float [[A4]], i32 4)
+; CHECK-NEXT:    [[C5:%.*]] = call float @llvm.powi.f32.i32(float [[A5]], i32 4)
+; CHECK-NEXT:    [[C6:%.*]] = call float @llvm.powi.f32.i32(float [[A6]], i32 4)
+; CHECK-NEXT:    [[C7:%.*]] = call float @llvm.powi.f32.i32(float [[A7]], i32 4)
+; CHECK-NEXT:    [[R0:%.*]] = insertelement <8 x float> poison, float [[C0]], i32 0
+; CHECK-NEXT:    [[R1:%.*]] = insertelement <8 x float> [[R0]], float [[C1]], i32 1
+; CHECK-NEXT:    [[R2:%.*]] = insertelement <8 x float> [[R1]], float [[C2]], i32 2
+; CHECK-NEXT:    [[R3:%.*]] = insertelement <8 x float> [[R2]], float [[C3]], i32 3
+; CHECK-NEXT:    [[R4:%.*]] = insertelement <8 x float> [[R3]], float [[C4]], i32 4
+; CHECK-NEXT:    [[R5:%.*]] = insertelement <8 x float> [[R4]], float [[C5]], i32 5
+; CHECK-NEXT:    [[R6:%.*]] = insertelement <8 x float> [[R5]], float [[C6]], i32 6
+; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <8 x float> [[R6]], float [[C7]], i32 7
 ; CHECK-NEXT:    ret <8 x float> [[TMP1]]
 ;
   %a0 = extractelement <8 x float> %a, i32 0
@@ -119,7 +169,30 @@ define <8 x float> @buildvector_powi_8f32_4(<8 x float> %a) {
 
 define <8 x double> @buildvector_powi_8f64_5(<8 x double> %a) {
 ; CHECK-LABEL: @buildvector_powi_8f64_5(
-; CHECK-NEXT:    [[TMP1:%.*]] = call <8 x double> @llvm.powi.v8f64.i32(<8 x double> [[A:%.*]], i32 5)
+; CHECK-NEXT:    [[A0:%.*]] = extractelement <8 x double> [[A:%.*]], i32 0
+; CHECK-NEXT:    [[A1:%.*]] = extractelement <8 x double> [[A]], i32 1
+; CHECK-NEXT:    [[A2:%.*]] = extractelement <8 x double> [[A]], i32 2
+; CHECK-NEXT:    [[A3:%.*]] = extractelement <8 x double> [[A]], i32 3
+; CHECK-NEXT:    [[A4:%.*]] = extractelement <8 x double> [[A]], i32 4
+; CHECK-NEXT:    [[A5:%.*]] = extractelement <8 x double> [[A]], i32 5
+; CHECK-NEXT:    [[A6:%.*]] = extractelement <8 x double> [[A]], i32 6
+; CHECK-NEXT:    [[A7:%.*]] = extractelement <8 x double> [[A]], i32 7
+; CHECK-NEXT:    [[C0:%.*]] = call double @llvm.powi.f64.i32(double [[A0]], i32 5)
+; CHECK-NEXT:    [[C1:%.*]] = call double @llvm.powi.f64.i32(double [[A1]], i32 5)
+; CHECK-NEXT:    [[C2:%.*]] = call double @llvm.powi.f64.i32(double [[A2]], i32 5)
+; CHECK-NEXT:    [[C3:%.*]] = call double @llvm.powi.f64.i32(double [[A3]], i32 5)
+; CHECK-NEXT:    [[C4:%.*]] = call double @llvm.powi.f64.i32(double [[A4]], i32 5)
+; CHECK-NEXT:    [[C5:%.*]] = call double @llvm.powi.f64.i32(double [[A5]], i32 5)
+; CHECK-NEXT:    [[C6:%.*]] = call double @llvm.powi.f64.i32(double [[A6]], i32 5)
+; CHECK-NEXT:    [[C7:%.*]] = call double @llvm.powi.f64.i32(double [[A7]], i32 5)
+; CHECK-NEXT:    [[R0:%.*]] = insertelement <8 x double> poison, double [[C0]], i32 0
+; CHECK-NEXT:    [[R1:%.*]] = insertelement <8 x double> [[R0]], double [[C1]], i32 1
+; CHECK-NEXT:    [[R2:%.*]] = insertelement <8 x double> [[R1]], double [[C2]], i32 2
+; CHECK-NEXT:    [[R3:%.*]] = insertelement <8 x double> [[R2]], double [[C3]], i32 3
+; CHECK-NEXT:    [[R4:%.*]] = insertelement <8 x double> [[R3]], double [[C4]], i32 4
+; CHECK-NEXT:    [[R5:%.*]] = insertelement <8 x double> [[R4]], double [[C5]], i32 5
+; CHECK-NEXT:    [[R6:%.*]] = insertelement <8 x double> [[R5]], double [[C6]], i32 6
+; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <8 x double> [[R6]], double [[C7]], i32 7
 ; CHECK-NEXT:    ret <8 x double> [[TMP1]]
 ;
   %a0 = extractelement <8 x double> %a, i32 0
@@ -206,7 +279,54 @@ define <8 x double> @buildvector_powi_8f64_mismatch(<8 x double> %a) {
 
 define <16 x float> @buildvector_powi_16f32_n13(<16 x float> %a) {
 ; CHECK-LABEL: @buildvector_powi_16f32_n13(
-; CHECK-NEXT:    [[TMP1:%.*]] = call <16 x float> @llvm.powi.v16f32.i32(<16 x float> [[A:%.*]], i32 -13)
+; CHECK-NEXT:    [[A0:%.*]] = extractelement <16 x float> [[A:%.*]], i32 0
+; CHECK-NEXT:    [[A1:%.*]] = extractelement <16 x float> [[A]], i32 1
+; CHECK-NEXT:    [[A2:%.*]] = extractelement <16 x float> [[A]], i32 2
+; CHECK-NEXT:    [[A3:%.*]] = extractelement <16 x float> [[A]], i32 3
+; CHECK-NEXT:    [[A4:%.*]] = extractelement <16 x float> [[A]], i32 4
+; CHECK-NEXT:    [[A5:%.*]] = extractelement <16 x float> [[A]], i32 5
+; CHECK-NEXT:    [[A6:%.*]] = extractelement <16 x float> [[A]], i32 6
+; CHECK-NEXT:    [[A7:%.*]] = extractelement <16 x float> [[A]], i32 7
+; CHECK-NEXT:    [[A8:%.*]] = extractelement <16 x float> [[A]], i32 8
+; CHECK-NEXT:    [[A9:%.*]] = extractelement <16 x float> [[A]], i32 9
+; CHECK-NEXT:    [[A10:%.*]] = extractelement <16 x float> [[A]], i32 10
+; CHECK-NEXT:    [[A11:%.*]] = extractelement <16 x float> [[A]], i32 11
+; CHECK-NEXT:    [[A12:%.*]] = extractelement <16 x float> [[A]], i32 12
+; CHECK-NEXT:    [[A13:%.*]] = extractelement <16 x float> [[A]], i32 13
+; CHECK-NEXT:    [[A14:%.*]] = extractelement <16 x float> [[A]], i32 14
+; CHECK-NEXT:    [[A15:%.*]] = extractelement <16 x float> [[A]], i32 15
+; CHECK-NEXT:    [[C0:%.*]] = call float @llvm.powi.f32.i32(float [[A0]], i32 -13)
+; CHECK-NEXT:    [[C1:%.*]] = call float @llvm.powi.f32.i32(float [[A1]], i32 -13)
+; CHECK-NEXT:    [[C2:%.*]] = call float @llvm.powi.f32.i32(float [[A2]], i32 -13)
+; CHECK-NEXT:    [[C3:%.*]] = call float @llvm.powi.f32.i32(float [[A3]], i32 -13)
+; CHECK-NEXT:    [[C4:%.*]] = call float @llvm.powi.f32.i32(float [[A4]], i32 -13)
+; CHECK-NEXT:    [[C5:%.*]] = call float @llvm.powi.f32.i32(float [[A5]], i32 -13)
+; CHECK-NEXT:    [[C6:%.*]] = call float @llvm.powi.f32.i32(float [[A6]], i32 -13)
+; CHECK-NEXT:    [[C7:%.*]] = call float @llvm.powi.f32.i32(float [[A7]], i32 -13)
+; CHECK-NEXT:    [[C8:%.*]] = call float @llvm.powi.f32.i32(float [[A8]], i32 -13)
+; CHECK-NEXT:    [[C9:%.*]] = call float @llvm.powi.f32.i32(float [[A9]], i32 -13)
+; CHECK-NEXT:    [[C10:%.*]] = call float @llvm.powi.f32.i32(float [[A10]], i32 -13)
+; CHECK-NEXT:    [[C11:%.*]] = call float @llvm.powi.f32.i32(float [[A11]], i32 -13)
+; CHECK-NEXT:    [[C12:%.*]] = call float @llvm.powi.f32.i32(float [[A12]], i32 -13)
+; CHECK-NEXT:    [[C13:%.*]] = call float @llvm.powi.f32.i32(float [[A13]], i32 -13)
+; CHECK-NEXT:    [[C14:%.*]] = call float @llvm.powi.f32.i32(float [[A14]], i32 -13)
+; CHECK-NEXT:    [[C15:%.*]] = call float @llvm.powi.f32.i32(float [[A15]], i32 -13)
+; CHECK-NEXT:    [[R0:%.*]] = insertelement <16 x float> poison, float [[C0]], i32 0
+; CHECK-NEXT:    [[R1:%.*]] = insertelement <16 x float> [[R0]], float [[C1]], i32 1
+; CHECK-NEXT:    [[R2:%.*]] = insertelement <16 x float> [[R1]], float [[C2]], i32 2
+; CHECK-NEXT:    [[R3:%.*]] = insertelement <16 x float> [[R2]], float [[C3]], i32 3
+; CHECK-NEXT:    [[R4:%.*]] = insertelement <16 x float> [[R3]], float [[C4]], i32 4
+; CHECK-NEXT:    [[R5:%.*]] = insertelement <16 x float> [[R4]], float [[C5]], i32 5
+; CHECK-NEXT:    [[R6:%.*]] = insertelement <16 x float> [[R5]], float [[C6]], i32 6
+; CHECK-NEXT:    [[R7:%.*]] = insertelement <16 x float> [[R6]], float [[C7]], i32 7
+; CHECK-NEXT:    [[R8:%.*]] = insertelement <16 x float> [[R7]], float [[C8]], i32 8
+; CHECK-NEXT:    [[R9:%.*]] = insertelement <16 x float> [[R8]], float [[C9]], i32 9
+; CHECK-NEXT:    [[R10:%.*]] = insertelement <16 x float> [[R9]], float [[C10]], i32 10
+; CHECK-NEXT:    [[R11:%.*]] = insertelement <16 x float> [[R10]], float [[C11]], i32 11
+; CHECK-NEXT:    [[R12:%.*]] = insertelement <16 x float> [[R11]], float [[C12]], i32 12
+; CHECK-NEXT:    [[R13:%.*]] = insertelement <16 x float> [[R12]], float [[C13]], i32 13
+; CHECK-NEXT:    [[R14:%.*]] = insertelement <16 x float> [[R13]], float [[C14]], i32 14
+; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <16 x float> [[R14]], float [[C15]], i32 15
 ; CHECK-NEXT:    ret <16 x float> [[TMP1]]
 ;
   %a0  = extractelement <16 x float> %a, i32 0
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/reduction-transpose.ll b/llvm/test/Transforms/SLPVectorizer/X86/reduction-transpose.ll
index 2cdbd5cff4468..cb4783010965e 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/reduction-transpose.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/reduction-transpose.ll
@@ -49,11 +49,24 @@ define i32 @reduce_and4(i32 %acc, <4 x i32> %v1, <4 x i32> %v2, <4 x i32> %v3, <
 ;
 ; AVX512-LABEL: @reduce_and4(
 ; AVX512-NEXT:  entry:
-; AVX512-NEXT:    [[TMP0:%.*]] = shufflevector <4 x i32> [[V4:%.*]], <4 x i32> [[V3:%.*]], <8 x i32> <i32 1, i32 0, i32 2, i32 3, i32 5, i32 4, i32 6, i32 7>
-; AVX512-NEXT:    [[TMP1:%.*]] = call i32 @llvm.vector.reduce.and.v8i32(<8 x i32> [[TMP0]])
-; AVX512-NEXT:    [[TMP2:%.*]] = shufflevector <4 x i32> [[V2:%.*]], <4 x i32> [[V1:%.*]], <8 x i32> <i32 1, i32 0, i32 2, i32 3, i32 5, i32 4, i32 6, i32 7>
-; AVX512-NEXT:    [[TMP3:%.*]] = call i32 @llvm.vector.reduce.and.v8i32(<8 x i32> [[TMP2]])
-; AVX512-NEXT:    [[OP_RDX:%.*]] = and i32 [[TMP1]], [[TMP3]]
+; AVX512-NEXT:    [[VECEXT:%.*]] = extractelement <4 x i32> [[V1:%.*]], i64 0
+; AVX512-NEXT:    [[VECEXT1:%.*]] = extractelement <4 x i32> [[V1]], i64 1
+; AVX512-NEXT:    [[VECEXT2:%.*]] = extractelement <4 x i32> [[V1]], i64 2
+; AVX512-NEXT:    [[VECEXT4:%.*]] = extractelement <4 x i32> [[V1]], i64 3
+; AVX512-NEXT:    [[VECEXT7:%.*]] = extractelement <4 x i32> [[V2:%.*]], i64 0
+; AVX512-NEXT:    [[VECEXT8:%.*]] = extractelement <4 x i32> [[V2]], i64 1
+; AVX512-NEXT:    [[VECEXT10:%.*]] = extractelement <4 x i32> [[V2]], i64 2
+; AVX512-NEXT:    [[VECEXT12:%.*]] = extractelement <4 x i32> [[V2]], i64 3
+; AVX512-NEXT:    [[TMP0:%.*]] = shufflevector <4 x i32> [[V4:%.*]], <4 x i32> [[V3:%.*]], <16 x i32> <i32 1, i32 0, i32 2, i32 3, i32 5, i32 4, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512-NEXT:    [[TMP1:%.*]] = insertelement <16 x i32> [[TMP0]], i32 [[VECEXT8]], i32 8
+; AVX512-NEXT:    [[TMP2:%.*]] = insertelement <16 x i32> [[TMP1]], i32 [[VECEXT7]], i32 9
+; AVX512-NEXT:    [[TMP3:%.*]] = insertelement <16 x i32> [[TMP2]], i32 [[VECEXT10]], i32 10
+; AVX512-NEXT:    [[TMP4:%.*]] = insertelement <16 x i32> [[TMP3]], i32 [[VECEXT12]], i32 11
+; AVX512-NEXT:    [[TMP5:%.*]] = insertelement <16 x i32> [[TMP4]], i32 [[VECEXT1]], i32 12
+; AVX512-NEXT:    [[TMP6:%.*]] = insertelement <16 x i32> [[TMP5]], i32 [[VECEXT]], i32 13
+; AVX512-NEXT:    [[TMP7:%.*]] = insertelement <16 x i32> [[TMP6]], i32 [[VECEXT2]], i32 14
+; AVX512-NEXT:    [[TMP8:%.*]] = insertelement <16 x i32> [[TMP7]], i32 [[VECEXT4]], i32 15
+; AVX512-NEXT:    [[OP_RDX:%.*]] = call i32 @llvm.vector.reduce.and.v16i32(<16 x i32> [[TMP8]])
 ; AVX512-NEXT:    [[OP_RDX1:%.*]] = and i32 [[OP_RDX]], [[ACC:%.*]]
 ; AVX512-NEXT:    ret i32 [[OP_RDX1]]
 ;
@@ -131,11 +144,24 @@ define i32 @reduce_and4_transpose(i32 %acc, <4 x i32> %v1, <4 x i32> %v2, <4 x i
 ; AVX2-NEXT:    ret i32 [[OP_RDX]]
 ;
 ; AVX512-LABEL: @reduce_and4_transpose(
-; AVX512-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i32> [[V4:%.*]], <4 x i32> [[V3:%.*]], <8 x i32> <i32 3, i32 2, i32 1, i32 0, i32 7, i32 6, i32 5, i32 4>
-; AVX512-NEXT:    [[TMP2:%.*]] = call i32 @llvm.vector.reduce.and.v8i32(<8 x i32> [[TMP1]])
-; AVX512-NEXT:    [[TMP3:%.*]] = shufflevector <4 x i32> [[V2:%.*]], <4 x i32> [[V1:%.*]], <8 x i32> <i32 3, i32 2, i32 1, i32 0, i32 7, i32 6, i32 5, i32 4>
-; AVX512-NEXT:    [[TMP4:%.*]] = call i32 @llvm.vector.reduce.and.v8i32(<8 x i32> [[TMP3]])
-; AVX512-NEXT:    [[OP_RDX:%.*]] = and i32 [[TMP2]], [[TMP4]]
+; AVX512-NEXT:    [[VECEXT:%.*]] = extractelement <4 x i32> [[V1:%.*]], i64 0
+; AVX512-NEXT:    [[VECEXT1:%.*]] = extractelement <4 x i32> [[V2:%.*]], i64 0
+; AVX512-NEXT:    [[VECEXT7:%.*]] = extractelement <4 x i32> [[V1]], i64 1
+; AVX512-NEXT:    [[VECEXT8:%.*]] = extractelement <4 x i32> [[V2]], i64 1
+; AVX512-NEXT:    [[VECEXT15:%.*]] = extractelement <4 x i32> [[V1]], i64 2
+; AVX512-NEXT:    [[VECEXT16:%.*]] = extractelement <4 x i32> [[V2]], i64 2
+; AVX512-NEXT:    [[VECEXT23:%.*]] = extractelement <4 x i32> [[V1]], i64 3
+; AVX512-NEXT:    [[VECEXT24:%.*]] = extractelement <4 x i32> [[V2]], i64 3
+; AVX512-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i32> [[V4:%.*]], <4 x i32> [[V3:%.*]], <16 x i32> <i32 3, i32 2, i32 1, i32 0, i32 7, i32 6, i32 5, i32 4, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512-NEXT:    [[TMP2:%.*]] = insertelement <16 x i32> [[TMP1]], i32 [[VECEXT24]], i32 8
+; AVX512-NEXT:    [[TMP3:%.*]] = insertelement <16 x i32> [[TMP2]], i32 [[VECEXT16]], i32 9
+; AVX512-NEXT:    [[TMP4:%.*]] = insertelement <16 x i32> [[TMP3]], i32 [[VECEXT8]], i32 10
+; AVX512-NEXT:    [[TMP5:%.*]] = insertelement <16 x i32> [[TMP4]], i32 [[VECEXT1]], i32 11
+; AVX512-NEXT:    [[TMP6:%.*]] = insertelement <16 x i32> [[TMP5]], i32 [[VECEXT23]], i32 12
+; AVX512-NEXT:    [[TMP7:%.*]] = insertelement <16 x i32> [[TMP6]], i32 [[VECEXT15]], i32 13
+; AVX512-NEXT:    [[TMP8:%.*]] = insertelement <16 x i32> [[TMP7]], i32 [[VECEXT7]], i32 14
+; AVX512-NEXT:    [[TMP9:%.*]] = insertelement <16 x i32> [[TMP8]], i32 [[VECEXT]], i32 15
+; AVX512-NEXT:    [[OP_RDX:%.*]] = call i32 @llvm.vector.reduce.and.v16i32(<16 x i32> [[TMP9]])
 ; AVX512-NEXT:    [[OP_RDX1:%.*]] = and i32 [[OP_RDX]], [[ACC:%.*]]
 ; AVX512-NEXT:    ret i32 [[OP_RDX1]]
 ;
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/resized-bv-values-non-power-of2-node.ll b/llvm/test/Transforms/SLPVectorizer/X86/resized-bv-values-non-power-of2-node.ll
new file mode 100644
index 0000000000000..e56131b4681e3
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/resized-bv-values-non-power-of2-node.ll
@@ -0,0 +1,269 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -S --passes=slp-vectorizer -mtriple=x86_64-unknown-linux < %s | FileCheck %s
+
+define <16 x half> @test(i32 %0, float %1, i32 %2) {
+; CHECK-LABEL: define <16 x half> @test(
+; CHECK-SAME: i32 [[TMP0:%.*]], float [[TMP1:%.*]], i32 [[TMP2:%.*]]) {
+; CHECK-NEXT:    [[TMP4:%.*]] = insertelement <16 x float> <float 0.000000e+00, float 0.000000e+00, float 0.000000e+00, float 0.000000e+00, float 0.000000e+00, float 0.000000e+00, float 0.000000e+00, float 0.000000e+00, float 0.000000e+00, float 0.000000e+00, float 0.000000e+00, float 0.000000e+00, float 0.000000e+00, float poison, float 0.000000e+00, float 0.000000e+00>, float [[TMP1]], i32 13
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <16 x float> [[TMP4]] to <16 x i32>
+; CHECK-NEXT:    [[TMP6:%.*]] = call <2 x float> @llvm.fmuladd.v2f32(<2 x float> zeroinitializer, <2 x float> zeroinitializer, <2 x float> zeroinitializer)
+; CHECK-NEXT:    [[TMP7:%.*]] = icmp ugt i32 [[TMP2]], 0
+; CHECK-NEXT:    [[TMP8:%.*]] = icmp eq i32 [[TMP0]], [[TMP0]]
+; CHECK-NEXT:    [[TMP9:%.*]] = extractelement <2 x float> [[TMP6]], i32 0
+; CHECK-NEXT:    [[TMP10:%.*]] = fcmp ogt float [[TMP9]], 0.000000e+00
+; CHECK-NEXT:    [[TMP11:%.*]] = icmp eq i32 [[TMP0]], 0
+; CHECK-NEXT:    [[TMP12:%.*]] = icmp eq i32 [[TMP0]], 0
+; CHECK-NEXT:    [[TMP13:%.*]] = extractelement <16 x i32> [[TMP5]], i32 10
+; CHECK-NEXT:    [[TMP14:%.*]] = icmp eq i32 [[TMP13]], 0
+; CHECK-NEXT:    [[TMP15:%.*]] = bitcast float 0.000000e+00 to i32
+; CHECK-NEXT:    [[TMP16:%.*]] = icmp eq i32 [[TMP15]], 0
+; CHECK-NEXT:    [[TMP17:%.*]] = icmp eq i32 [[TMP0]], 0
+; CHECK-NEXT:    [[TMP18:%.*]] = icmp ult i32 0, 0
+; CHECK-NEXT:    [[TMP19:%.*]] = icmp ult i32 0, 0
+; CHECK-NEXT:    [[TMP20:%.*]] = icmp ult i32 0, 0
+; CHECK-NEXT:    [[TMP21:%.*]] = icmp eq i32 0, 0
+; CHECK-NEXT:    [[TMP22:%.*]] = extractelement <16 x i32> [[TMP5]], i32 4
+; CHECK-NEXT:    [[TMP23:%.*]] = icmp eq i32 [[TMP22]], 0
+; CHECK-NEXT:    [[TMP24:%.*]] = icmp eq i32 [[TMP0]], [[TMP0]]
+; CHECK-NEXT:    [[TMP25:%.*]] = bitcast float 0.000000e+00 to i32
+; CHECK-NEXT:    [[TMP26:%.*]] = icmp eq i32 [[TMP25]], 0
+; CHECK-NEXT:    [[TMP27:%.*]] = icmp ult <16 x i32> [[TMP5]], zeroinitializer
+; CHECK-NEXT:    [[TMP28:%.*]] = select <16 x i1> [[TMP27]], <16 x i32> zeroinitializer, <16 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP29:%.*]] = sitofp <16 x i32> [[TMP28]] to <16 x float>
+; CHECK-NEXT:    [[TMP30:%.*]] = call <16 x float> @llvm.fmuladd.v16f32(<16 x float> [[TMP29]], <16 x float> zeroinitializer, <16 x float> zeroinitializer)
+; CHECK-NEXT:    [[TMP31:%.*]] = fadd <16 x float> [[TMP30]], zeroinitializer
+; CHECK-NEXT:    [[TMP32:%.*]] = call <12 x i1> @llvm.vector.insert.v12i1.v2i1(<12 x i1> poison, <2 x i1> zeroinitializer, i64 0)
+; CHECK-NEXT:    [[TMP33:%.*]] = shufflevector <12 x i1> [[TMP32]], <12 x i1> <i1 poison, i1 false, i1 false, i1 poison, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false>, <12 x i32> <i32 0, i32 13, i32 14, i32 1, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23>
+; CHECK-NEXT:    [[TMP34:%.*]] = shufflevector <12 x i1> [[TMP33]], <12 x i1> poison, <16 x i32> <i32 0, i32 1, i32 0, i32 0, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 0, i32 10, i32 11, i32 0>
+; CHECK-NEXT:    [[TMP35:%.*]] = select <16 x i1> [[TMP34]], <16 x float> zeroinitializer, <16 x float> [[TMP31]]
+; CHECK-NEXT:    [[TMP36:%.*]] = bitcast <16 x float> [[TMP35]] to <16 x i32>
+; CHECK-NEXT:    [[TMP37:%.*]] = and <16 x i32> [[TMP36]], zeroinitializer
+; CHECK-NEXT:    [[TMP38:%.*]] = bitcast <16 x i32> [[TMP37]] to <16 x float>
+; CHECK-NEXT:    [[TMP39:%.*]] = call <16 x float> @llvm.vector.insert.v16f32.v2f32(<16 x float> <float 0.000000e+00, float 0.000000e+00, float 0.000000e+00, float 0.000000e+00, float 0.000000e+00, float 0.000000e+00, float 0.000000e+00, float 0.000000e+00, float 0.000000e+00, float 0.000000e+00, float 0.000000e+00, float 0.000000e+00, float 0.000000e+00, float 0.000000e+00, float poison, float poison>, <2 x float> [[TMP6]], i64 14)
+; CHECK-NEXT:    [[TMP40:%.*]] = call <16 x float> @llvm.fmuladd.v16f32(<16 x float> zeroinitializer, <16 x float> [[TMP38]], <16 x float> [[TMP39]])
+; CHECK-NEXT:    [[TMP41:%.*]] = extractelement <16 x float> [[TMP29]], i32 0
+; CHECK-NEXT:    [[TMP42:%.*]] = fcmp olt float [[TMP41]], 0.000000e+00
+; CHECK-NEXT:    [[TMP43:%.*]] = extractelement <16 x float> [[TMP29]], i32 14
+; CHECK-NEXT:    [[TMP44:%.*]] = fcmp ogt float [[TMP43]], 0.000000e+00
+; CHECK-NEXT:    [[TMP45:%.*]] = fcmp olt float [[TMP43]], 0.000000e+00
+; CHECK-NEXT:    [[TMP46:%.*]] = extractelement <16 x float> [[TMP29]], i32 13
+; CHECK-NEXT:    [[TMP47:%.*]] = fcmp ogt float [[TMP46]], 0.000000e+00
+; CHECK-NEXT:    [[TMP48:%.*]] = fcmp olt float [[TMP46]], 0.000000e+00
+; CHECK-NEXT:    [[TMP49:%.*]] = fcmp olt float [[TMP41]], 0.000000e+00
+; CHECK-NEXT:    [[TMP50:%.*]] = extractelement <16 x float> [[TMP29]], i32 1
+; CHECK-NEXT:    [[TMP51:%.*]] = fcmp ogt float [[TMP50]], 0.000000e+00
+; CHECK-NEXT:    [[TMP52:%.*]] = fcmp oeq <16 x float> [[TMP40]], zeroinitializer
+; CHECK-NEXT:    ret <16 x half> zeroinitializer
+;
+  %4 = bitcast float 0.000000e+00 to i32
+  %5 = fcmp olt float 0.000000e+00, 0.000000e+00
+  %6 = icmp ult i32 %4, 0
+  %7 = select i1 %6, i32 0, i32 0
+  %8 = sitofp i32 %7 to float
+  %9 = tail call float @llvm.fmuladd.f32(float %8, float 0.000000e+00, float 0.000000e+00)
+  %10 = fadd float %9, 0.000000e+00
+  %11 = select i1 %5, float 0.000000e+00, float %10
+  %12 = bitcast float %11 to i32
+  %13 = and i32 %12, 0
+  %14 = bitcast i32 %13 to float
+  %15 = tail call float @llvm.fmuladd.f32(float 0.000000e+00, float 0.000000e+00, float 0.000000e+00)
+  %16 = tail call float @llvm.fmuladd.f32(float 0.000000e+00, float %14, float %15)
+  %17 = fcmp oeq float %16, 0.000000e+00
+  %18 = fcmp olt float %8, 0.000000e+00
+  %19 = icmp ugt i32 %2, 0
+  %20 = bitcast float 0.000000e+00 to i32
+  %21 = icmp eq i32 %0, %0
+  %22 = icmp ult i32 %20, 0
+  %23 = select i1 %22, i32 0, i32 0
+  %24 = sitofp i32 %23 to float
+  %25 = tail call float @llvm.fmuladd.f32(float %24, float 0.000000e+00, float 0.000000e+00)
+  %26 = fadd float %25, 0.000000e+00
+  %27 = select i1 false, float 0.000000e+00, float %26
+  %28 = bitcast float %27 to i32
+  %29 = and i32 %28, 0
+  %30 = bitcast i32 %29 to float
+  %31 = tail call float @llvm.fmuladd.f32(float 0.000000e+00, float 0.000000e+00, float 0.000000e+00)
+  %32 = tail call float @llvm.fmuladd.f32(float 0.000000e+00, float %30, float %31)
+  %33 = fcmp ogt float %24, 0.000000e+00
+  %34 = fcmp oeq float %32, 0.000000e+00
+  %35 = fcmp ogt float %31, 0.000000e+00
+  %36 = fcmp olt float %24, 0.000000e+00
+  %37 = bitcast float %1 to i32
+  %38 = icmp ult i32 %37, 0
+  %39 = select i1 %38, i32 0, i32 0
+  %40 = sitofp i32 %39 to float
+  %41 = tail call float @llvm.fmuladd.f32(float %40, float 0.000000e+00, float 0.000000e+00)
+  %42 = fadd float %41, 0.000000e+00
+  %43 = select i1 false, float 0.000000e+00, float %42
+  %44 = bitcast float %43 to i32
+  %45 = and i32 %44, 0
+  %46 = bitcast i32 %45 to float
+  %47 = tail call float @llvm.fmuladd.f32(float 0.000000e+00, float %46, float 0.000000e+00)
+  %48 = fadd float 0.000000e+00, 0.000000e+00
+  %49 = fcmp ogt float %40, 0.000000e+00
+  %50 = fcmp oeq float %47, 0.000000e+00
+  %51 = fcmp ogt float %48, 0.000000e+00
+  %52 = fcmp olt float %40, 0.000000e+00
+  %53 = icmp eq i32 0, 0
+  %54 = bitcast float 0.000000e+00 to i32
+  %55 = icmp eq i32 0, 0
+  %56 = icmp ult i32 %54, 0
+  %57 = select i1 %56, i32 0, i32 0
+  %58 = sitofp i32 %57 to float
+  %59 = tail call float @llvm.fmuladd.f32(float %58, float 0.000000e+00, float 0.000000e+00)
+  %60 = fadd float %59, 0.000000e+00
+  %61 = select i1 %5, float 0.000000e+00, float %60
+  %62 = bitcast float %61 to i32
+  %63 = and i32 %62, 0
+  %64 = bitcast i32 %63 to float
+  %65 = tail call float @llvm.fmuladd.f32(float 0.000000e+00, float %64, float 0.000000e+00)
+  %66 = fcmp oeq float %65, 0.000000e+00
+  %67 = bitcast float 0.000000e+00 to i32
+  %68 = icmp eq i32 %0, 0
+  %69 = icmp ult i32 %67, 0
+  %70 = select i1 %69, i32 0, i32 0
+  %71 = sitofp i32 %70 to float
+  %72 = tail call float @llvm.fmuladd.f32(float %71, float 0.000000e+00, float 0.000000e+00)
+  %73 = fadd float %72, 0.000000e+00
+  %74 = select i1 false, float 0.000000e+00, float %73
+  %75 = bitcast float %74 to i32
+  %76 = and i32 %75, 0
+  %77 = bitcast i32 %76 to float
+  %78 = tail call float @llvm.fmuladd.f32(float 0.000000e+00, float %77, float 0.000000e+00)
+  %79 = fadd float 0.000000e+00, 0.000000e+00
+  %80 = fcmp oeq float %78, 0.000000e+00
+  %81 = fcmp ogt float %79, 0.000000e+00
+  %82 = icmp eq i32 %0, 0
+  %83 = bitcast float 0.000000e+00 to i32
+  %84 = icmp eq i32 %83, 0
+  %85 = icmp ult i32 %83, 0
+  %86 = select i1 %85, i32 0, i32 0
+  %87 = sitofp i32 %86 to float
+  %88 = tail call float @llvm.fmuladd.f32(float %87, float 0.000000e+00, float 0.000000e+00)
+  %89 = fadd float %88, 0.000000e+00
+  %90 = select i1 false, float 0.000000e+00, float %89
+  %91 = bitcast float %90 to i32
+  %92 = and i32 %91, 0
+  %93 = bitcast i32 %92 to float
+  %94 = tail call float @llvm.fmuladd.f32(float 0.000000e+00, float %93, float 0.000000e+00)
+  %95 = fcmp oeq float %94, 0.000000e+00
+  %96 = bitcast float 0.000000e+00 to i32
+  %97 = bitcast float 0.000000e+00 to i32
+  %98 = icmp ult i32 %97, 0
+  %99 = select i1 %98, i32 0, i32 0
+  %100 = sitofp i32 %99 to float
+  %101 = tail call float @llvm.fmuladd.f32(float %100, float 0.000000e+00, float 0.000000e+00)
+  %102 = fadd float %101, 0.000000e+00
+  %103 = select i1 false, float 0.000000e+00, float %102
+  %104 = bitcast float %103 to i32
+  %105 = and i32 %104, 0
+  %106 = bitcast i32 %105 to float
+  %107 = tail call float @llvm.fmuladd.f32(float 0.000000e+00, float %106, float 0.000000e+00)
+  %108 = fcmp oeq float %107, 0.000000e+00
+  %109 = icmp eq i32 %96, 0
+  %110 = icmp eq i32 %0, 0
+  %111 = icmp ult i32 0, 0
+  %112 = bitcast float 0.000000e+00 to i32
+  %113 = icmp ult i32 %112, 0
+  %114 = select i1 %113, i32 0, i32 0
+  %115 = sitofp i32 %114 to float
+  %116 = tail call float @llvm.fmuladd.f32(float %115, float 0.000000e+00, float 0.000000e+00)
+  %117 = fadd float %116, 0.000000e+00
+  %118 = select i1 false, float 0.000000e+00, float %117
+  %119 = bitcast float %118 to i32
+  %120 = and i32 %119, 0
+  %121 = bitcast i32 %120 to float
+  %122 = tail call float @llvm.fmuladd.f32(float 0.000000e+00, float %121, float 0.000000e+00)
+  %123 = fadd float 0.000000e+00, 0.000000e+00
+  %124 = fcmp oeq float %122, 0.000000e+00
+  %125 = fcmp ogt float %123, 0.000000e+00
+  %126 = icmp ult i32 0, 0
+  %127 = bitcast float 0.000000e+00 to i32
+  %128 = icmp ult i32 %127, 0
+  %129 = select i1 %128, i32 0, i32 0
+  %130 = sitofp i32 %129 to float
+  %131 = tail call float @llvm.fmuladd.f32(float %130, float 0.000000e+00, float 0.000000e+00)
+  %132 = fadd float %131, 0.000000e+00
+  %133 = select i1 false, float 0.000000e+00, float %132
+  %134 = bitcast float %133 to i32
+  %135 = and i32 %134, 0
+  %136 = bitcast i32 %135 to float
+  %137 = tail call float @llvm.fmuladd.f32(float 0.000000e+00, float %136, float 0.000000e+00)
+  %138 = fcmp oeq float %137, 0.000000e+00
+  %139 = icmp ult i32 0, 0
+  %140 = bitcast float 0.000000e+00 to i32
+  %141 = icmp eq i32 0, 0
+  %142 = icmp ult i32 %140, 0
+  %143 = select i1 %142, i32 0, i32 0
+  %144 = sitofp i32 %143 to float
+  %145 = tail call float @llvm.fmuladd.f32(float %144, float 0.000000e+00, float 0.000000e+00)
+  %146 = fadd float %145, 0.000000e+00
+  %147 = select i1 false, float 0.000000e+00, float %146
+  %148 = bitcast float %147 to i32
+  %149 = and i32 %148, 0
+  %150 = bitcast i32 %149 to float
+  %151 = tail call float @llvm.fmuladd.f32(float 0.000000e+00, float %150, float 0.000000e+00)
+  %152 = fcmp oeq float %151, 0.000000e+00
+  %153 = fcmp olt float 0.000000e+00, 0.000000e+00
+  %154 = select i1 %153, float 0.000000e+00, float %10
+  %155 = bitcast float %154 to i32
+  %156 = and i32 %155, 0
+  %157 = bitcast i32 %156 to float
+  %158 = tail call float @llvm.fmuladd.f32(float 0.000000e+00, float %157, float 0.000000e+00)
+  %159 = fcmp oeq float %158, 0.000000e+00
+  %160 = bitcast float 0.000000e+00 to i32
+  %161 = icmp eq i32 %160, 0
+  %162 = icmp ult i32 %160, 0
+  %163 = select i1 %162, i32 0, i32 0
+  %164 = sitofp i32 %163 to float
+  %165 = tail call float @llvm.fmuladd.f32(float %164, float 0.000000e+00, float 0.000000e+00)
+  %166 = fadd float %165, 0.000000e+00
+  %167 = select i1 false, float 0.000000e+00, float %166
+  %168 = bitcast float %167 to i32
+  %169 = and i32 %168, 0
+  %170 = bitcast i32 %169 to float
+  %171 = tail call float @llvm.fmuladd.f32(float 0.000000e+00, float %170, float 0.000000e+00)
+  %172 = fcmp oeq float %171, 0.000000e+00
+  %173 = tail call float @llvm.fmuladd.f32(float %8, float 0.000000e+00, float 0.000000e+00)
+  %174 = fadd float %173, 0.000000e+00
+  %175 = select i1 %5, float 0.000000e+00, float %174
+  %176 = bitcast float %175 to i32
+  %177 = and i32 %176, 0
+  %178 = bitcast i32 %177 to float
+  %179 = tail call float @llvm.fmuladd.f32(float 0.000000e+00, float %178, float 0.000000e+00)
+  %180 = fadd float 0.000000e+00, 0.000000e+00
+  %181 = fcmp oeq float %179, 0.000000e+00
+  %182 = fcmp ogt float %180, 0.000000e+00
+  %183 = fcmp olt float %8, 0.000000e+00
+  %184 = bitcast float 0.000000e+00 to i32
+  %185 = icmp eq i32 %0, %0
+  %186 = icmp ult i32 %184, 0
+  %187 = select i1 %186, i32 0, i32 0
+  %188 = sitofp i32 %187 to float
+  %189 = tail call float @llvm.fmuladd.f32(float %188, float 0.000000e+00, float 0.000000e+00)
+  %190 = fadd float %189, 0.000000e+00
+  %191 = select i1 %5, float 0.000000e+00, float %190
+  %192 = bitcast float %191 to i32
+  %193 = and i32 %192, 0
+  %194 = bitcast i32 %193 to float
+  %195 = tail call float @llvm.fmuladd.f32(float 0.000000e+00, float %194, float 0.000000e+00)
+  %196 = fcmp oeq float %195, 0.000000e+00
+  %197 = bitcast float 0.000000e+00 to i32
+  %198 = icmp eq i32 %197, 0
+  %199 = icmp ult i32 %197, 0
+  %200 = select i1 %199, i32 0, i32 0
+  %201 = sitofp i32 %200 to float
+  %202 = tail call float @llvm.fmuladd.f32(float %201, float 0.000000e+00, float 0.000000e+00)
+  %203 = fadd float %202, 0.000000e+00
+  %204 = select i1 false, float 0.000000e+00, float %203
+  %205 = bitcast float %204 to i32
+  %206 = and i32 %205, 0
+  %207 = bitcast i32 %206 to float
+  %208 = tail call float @llvm.fmuladd.f32(float 0.000000e+00, float %207, float 0.000000e+00)
+  %209 = fcmp oeq float %208, 0.000000e+00
+  %210 = fcmp ogt float %201, 0.000000e+00
+  %211 = tail call float @llvm.fmuladd.f32(float 0.000000e+00, float %14, float 0.000000e+00)
+  %212 = fcmp oeq float %211, 0.000000e+00
+  ret <16 x half> zeroinitializer
+}
+
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/revectorized_rdx_crash.ll b/llvm/test/Transforms/SLPVectorizer/X86/revectorized_rdx_crash.ll
index 2a3029bf325ca..dab37a7b05294 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/revectorized_rdx_crash.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/revectorized_rdx_crash.ll
@@ -15,10 +15,10 @@
 define void @test(i1 %arg, ptr %p) {
 ; CHECK-LABEL: @test(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    br i1 %arg, label [[IF_END:%.*]], label [[FOR_COND_PREHEADER:%.*]]
+; CHECK-NEXT:    br i1 [[ARG:%.*]], label [[IF_END:%.*]], label [[FOR_COND_PREHEADER:%.*]]
 ; CHECK:       for.cond.preheader:
-; CHECK-NEXT:    [[I:%.*]] = getelementptr inbounds [100 x i32], ptr %p, i64 0, i64 2
-; CHECK-NEXT:    [[I1:%.*]] = getelementptr inbounds [100 x i32], ptr %p, i64 0, i64 3
+; CHECK-NEXT:    [[I:%.*]] = getelementptr inbounds [100 x i32], ptr [[P:%.*]], i64 0, i64 2
+; CHECK-NEXT:    [[I1:%.*]] = getelementptr inbounds [100 x i32], ptr [[P]], i64 0, i64 3
 ; CHECK-NEXT:    [[TMP0:%.*]] = load <4 x i32>, ptr [[I]], align 8
 ; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP0]])
 ; CHECK-NEXT:    [[OP_RDX3:%.*]] = add i32 [[TMP1]], 0
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/root-node-reordered-reused.ll b/llvm/test/Transforms/SLPVectorizer/X86/root-node-reordered-reused.ll
new file mode 100644
index 0000000000000..64b43ebf3e140
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/root-node-reordered-reused.ll
@@ -0,0 +1,106 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -S --passes=slp-vectorizer -mtriple=x86_64-unknown-linux-gnu < %s | FileCheck %s
+
+@d = global i32 0, align 4
+@c = global i32 0, align 4
+@f = global i32 0, align 4
+@b = global i32 0, align 4
+@a = global i32 0, align 4
+@e = global i32 0, align 4
+
+define i32 @test() {
+; CHECK-LABEL: define i32 @test() {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i32, ptr @d, align 4
+; CHECK-NEXT:    [[TOBOOL_NOT:%.*]] = icmp eq i32 [[TMP0]], 0
+; CHECK-NEXT:    br i1 [[TOBOOL_NOT]], label %[[IF_END6:.*]], label %[[IF_THEN:.*]]
+; CHECK:       [[IF_THEN]]:
+; CHECK-NEXT:    [[TMP1:%.*]] = load i32, ptr @c, align 4
+; CHECK-NEXT:    [[TMP2:%.*]] = load volatile i32, ptr @f, align 4
+; CHECK-NEXT:    [[TOBOOL1_NOT:%.*]] = icmp eq i32 [[TMP2]], 0
+; CHECK-NEXT:    br i1 [[TOBOOL1_NOT]], label %[[IF_END:.*]], label %[[I:.*]]
+; CHECK:       [[IF_END]]:
+; CHECK-NEXT:    [[TMP3:%.*]] = load volatile i32, ptr @f, align 4
+; CHECK-NEXT:    [[TOBOOL3_NOT:%.*]] = icmp eq i32 [[TMP3]], 0
+; CHECK-NEXT:    br i1 [[TOBOOL3_NOT]], label %[[IF_END6]], label %[[I]]
+; CHECK:       [[IF_END6]]:
+; CHECK-NEXT:    [[H_0:%.*]] = phi i32 [ [[TMP1]], %[[IF_END]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT:    [[TMP4:%.*]] = load i32, ptr @b, align 4
+; CHECK-NEXT:    [[TOBOOL7_NOT:%.*]] = icmp eq i32 [[TMP4]], 0
+; CHECK-NEXT:    br i1 [[TOBOOL7_NOT]], label %[[IF_END13:.*]], label %[[I]]
+; CHECK:       [[LAND_RHS:.*]]:
+; CHECK-NEXT:    [[TMP5:%.*]] = load volatile i32, ptr @f, align 4
+; CHECK-NEXT:    br label %[[IF_END13]]
+; CHECK:       [[IF_END13]]:
+; CHECK-NEXT:    [[H_2:%.*]] = phi i32 [ [[H_0]], %[[IF_END6]] ], [ [[H_1:%.*]], %[[LAND_RHS]] ], [ [[H_1]], %[[I]] ]
+; CHECK-NEXT:    [[J_1:%.*]] = phi i32 [ [[TMP0]], %[[IF_END6]] ], [ [[J_0:%.*]], %[[LAND_RHS]] ], [ [[J_0]], %[[I]] ]
+; CHECK-NEXT:    [[K_1:%.*]] = phi i32 [ [[TMP0]], %[[IF_END6]] ], [ [[K_0:%.*]], %[[LAND_RHS]] ], [ [[K_0]], %[[I]] ]
+; CHECK-NEXT:    [[L_1:%.*]] = phi i32 [ [[TMP0]], %[[IF_END6]] ], [ [[L_0:%.*]], %[[LAND_RHS]] ], [ [[L_0]], %[[I]] ]
+; CHECK-NEXT:    [[TMP6:%.*]] = load i32, ptr @a, align 4
+; CHECK-NEXT:    [[TMP7:%.*]] = load i32, ptr @e, align 4
+; CHECK-NEXT:    [[CMP_NOT:%.*]] = icmp eq i32 [[TMP6]], [[TMP7]]
+; CHECK-NEXT:    br i1 [[CMP_NOT]], label %[[IF_END15:.*]], label %[[I]]
+; CHECK:       [[I]]:
+; CHECK-NEXT:    [[J_0]] = phi i32 [ 0, %[[IF_THEN]] ], [ [[J_1]], %[[IF_END13]] ], [ 0, %[[IF_END]] ], [ [[TMP0]], %[[IF_END6]] ]
+; CHECK-NEXT:    [[K_0]] = phi i32 [ 0, %[[IF_THEN]] ], [ [[K_1]], %[[IF_END13]] ], [ 0, %[[IF_END]] ], [ [[TMP0]], %[[IF_END6]] ]
+; CHECK-NEXT:    [[L_0]] = phi i32 [ 0, %[[IF_THEN]] ], [ [[L_1]], %[[IF_END13]] ], [ 0, %[[IF_END]] ], [ [[TMP0]], %[[IF_END6]] ]
+; CHECK-NEXT:    [[H_1]] = phi i32 [ [[TMP1]], %[[IF_THEN]] ], [ [[H_2]], %[[IF_END13]] ], [ [[TMP1]], %[[IF_END]] ], [ [[H_0]], %[[IF_END6]] ]
+; CHECK-NEXT:    [[OR:%.*]] = or i32 [[J_0]], [[H_1]]
+; CHECK-NEXT:    [[OR9:%.*]] = or i32 [[OR]], [[K_0]]
+; CHECK-NEXT:    [[OR10:%.*]] = or i32 [[OR9]], [[L_0]]
+; CHECK-NEXT:    [[TOBOOL11_NOT:%.*]] = icmp eq i32 [[OR10]], 0
+; CHECK-NEXT:    br i1 [[TOBOOL11_NOT]], label %[[IF_END13]], label %[[LAND_RHS]]
+; CHECK:       [[IF_END15]]:
+; CHECK-NEXT:    ret i32 0
+;
+entry:
+  %0 = load i32, ptr @d, align 4
+  %tobool.not = icmp eq i32 %0, 0
+  br i1 %tobool.not, label %if.end6, label %if.then
+
+if.then:
+  %1 = load i32, ptr @c, align 4
+  %2 = load volatile i32, ptr @f, align 4
+  %tobool1.not = icmp eq i32 %2, 0
+  br i1 %tobool1.not, label %if.end, label %i
+
+if.end:
+  %3 = load volatile i32, ptr @f, align 4
+  %tobool3.not = icmp eq i32 %3, 0
+  br i1 %tobool3.not, label %if.end6, label %i
+
+if.end6:
+  %h.0 = phi i32 [ %1, %if.end ], [ 0, %entry ]
+  %4 = load i32, ptr @b, align 4
+  %tobool7.not = icmp eq i32 %4, 0
+  br i1 %tobool7.not, label %if.end13, label %i
+
+land.rhs:
+  %5 = load volatile i32, ptr @f, align 4
+  br label %if.end13
+
+if.end13:
+  %h.2 = phi i32 [ %h.0, %if.end6 ], [ %h.1, %land.rhs ], [ %h.1, %i ]
+  %j.1 = phi i32 [ %0, %if.end6 ], [ %j.0, %land.rhs ], [ %j.0, %i ]
+  %k.1 = phi i32 [ %0, %if.end6 ], [ %k.0, %land.rhs ], [ %k.0, %i ]
+  %l.1 = phi i32 [ %0, %if.end6 ], [ %l.0, %land.rhs ], [ %l.0, %i ]
+  %6 = load i32, ptr @a, align 4
+  %7 = load i32, ptr @e, align 4
+  %cmp.not = icmp eq i32 %6, %7
+  br i1 %cmp.not, label %if.end15, label %i
+
+i:
+  %j.0 = phi i32 [ 0, %if.then ], [ %j.1, %if.end13 ], [ 0, %if.end ], [ %0, %if.end6 ]
+  %k.0 = phi i32 [ 0, %if.then ], [ %k.1, %if.end13 ], [ 0, %if.end ], [ %0, %if.end6 ]
+  %l.0 = phi i32 [ 0, %if.then ], [ %l.1, %if.end13 ], [ 0, %if.end ], [ %0, %if.end6 ]
+  %h.1 = phi i32 [ %1, %if.then ], [ %h.2, %if.end13 ], [ %1, %if.end ], [ %h.0, %if.end6 ]
+  %or = or i32 %j.0, %h.1
+  %or9 = or i32 %or, %k.0
+  %or10 = or i32 %or9, %l.0
+  %tobool11.not = icmp eq i32 %or10, 0
+  br i1 %tobool11.not, label %if.end13, label %land.rhs
+
+if.end15:
+  ret i32 0
+}
+
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/scalarize-ctlz.ll b/llvm/test/Transforms/SLPVectorizer/X86/scalarize-ctlz.ll
new file mode 100644
index 0000000000000..0f9b2e9ba86fd
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/scalarize-ctlz.ll
@@ -0,0 +1,203 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -S --passes=slp-vectorizer -mtriple=x86_64-- -mcpu=x86-64 %s | FileCheck %s --check-prefixes=SSE,SSE2
+; RUN: opt -S --passes=slp-vectorizer -mtriple=x86_64-- -mcpu=x86-64-v2 %s | FileCheck %s --check-prefixes=SSE,SSE4
+; RUN: opt -S --passes=slp-vectorizer -mtriple=x86_64-- -mcpu=x86-64-v3 %s | FileCheck %s --check-prefixes=AVX2
+; RUN: opt -S --passes=slp-vectorizer -mtriple=x86_64-- -mcpu=x86-64-v4 %s | FileCheck %s --check-prefixes=AVX512
+
+; PR124993 - ensure scalarized CTLZ calls remain scalarized unless there is a definite cost improvement, the cost of scalarization was being over estimated.
+
+define <2 x i64> @scalarize_ctlz_v2i64(<2 x i64> %v)  {
+; SSE-LABEL: define <2 x i64> @scalarize_ctlz_v2i64(
+; SSE-SAME: <2 x i64> [[V:%.*]]) #[[ATTR0:[0-9]+]] {
+; SSE-NEXT:    [[V0:%.*]] = extractelement <2 x i64> [[V]], i64 0
+; SSE-NEXT:    [[V1:%.*]] = extractelement <2 x i64> [[V]], i64 1
+; SSE-NEXT:    [[C0:%.*]] = tail call range(i64 0, 65) i64 @llvm.ctlz.i64(i64 [[V0]], i1 false)
+; SSE-NEXT:    [[C1:%.*]] = tail call range(i64 0, 65) i64 @llvm.ctlz.i64(i64 [[V1]], i1 false)
+; SSE-NEXT:    [[R0:%.*]] = insertelement <2 x i64> poison, i64 [[C0]], i64 0
+; SSE-NEXT:    [[R1:%.*]] = insertelement <2 x i64> [[R0]], i64 [[C1]], i64 1
+; SSE-NEXT:    ret <2 x i64> [[R1]]
+;
+; AVX2-LABEL: define <2 x i64> @scalarize_ctlz_v2i64(
+; AVX2-SAME: <2 x i64> [[V:%.*]]) #[[ATTR0:[0-9]+]] {
+; AVX2-NEXT:    [[V0:%.*]] = extractelement <2 x i64> [[V]], i64 0
+; AVX2-NEXT:    [[V1:%.*]] = extractelement <2 x i64> [[V]], i64 1
+; AVX2-NEXT:    [[C0:%.*]] = tail call range(i64 0, 65) i64 @llvm.ctlz.i64(i64 [[V0]], i1 false)
+; AVX2-NEXT:    [[C1:%.*]] = tail call range(i64 0, 65) i64 @llvm.ctlz.i64(i64 [[V1]], i1 false)
+; AVX2-NEXT:    [[R0:%.*]] = insertelement <2 x i64> poison, i64 [[C0]], i64 0
+; AVX2-NEXT:    [[R1:%.*]] = insertelement <2 x i64> [[R0]], i64 [[C1]], i64 1
+; AVX2-NEXT:    ret <2 x i64> [[R1]]
+;
+; AVX512-LABEL: define <2 x i64> @scalarize_ctlz_v2i64(
+; AVX512-SAME: <2 x i64> [[V:%.*]]) #[[ATTR0:[0-9]+]] {
+; AVX512-NEXT:    [[TMP1:%.*]] = call <2 x i64> @llvm.ctlz.v2i64(<2 x i64> [[V]], i1 false)
+; AVX512-NEXT:    ret <2 x i64> [[TMP1]]
+;
+  %v0 = extractelement <2 x i64> %v, i64 0
+  %v1 = extractelement <2 x i64> %v, i64 1
+  %c0 = tail call range(i64 0, 65) i64 @llvm.ctlz.i64(i64 %v0, i1 false)
+  %c1 = tail call range(i64 0, 65) i64 @llvm.ctlz.i64(i64 %v1, i1 false)
+  %r0 = insertelement <2 x i64> poison, i64 %c0, i64 0
+  %r1 = insertelement <2 x i64> %r0, i64 %c1, i64 1
+  ret <2 x i64> %r1
+}
+
+define <4 x i64> @scalarize_ctlz_v4i64(<4 x i64> %v)  {
+; SSE-LABEL: define <4 x i64> @scalarize_ctlz_v4i64(
+; SSE-SAME: <4 x i64> [[V:%.*]]) #[[ATTR0]] {
+; SSE-NEXT:    [[V0:%.*]] = extractelement <4 x i64> [[V]], i64 0
+; SSE-NEXT:    [[V1:%.*]] = extractelement <4 x i64> [[V]], i64 1
+; SSE-NEXT:    [[V2:%.*]] = extractelement <4 x i64> [[V]], i64 2
+; SSE-NEXT:    [[V3:%.*]] = extractelement <4 x i64> [[V]], i64 3
+; SSE-NEXT:    [[C0:%.*]] = tail call range(i64 0, 65) i64 @llvm.ctlz.i64(i64 [[V0]], i1 false)
+; SSE-NEXT:    [[C1:%.*]] = tail call range(i64 0, 65) i64 @llvm.ctlz.i64(i64 [[V1]], i1 false)
+; SSE-NEXT:    [[C2:%.*]] = tail call range(i64 0, 65) i64 @llvm.ctlz.i64(i64 [[V2]], i1 false)
+; SSE-NEXT:    [[C3:%.*]] = tail call range(i64 0, 65) i64 @llvm.ctlz.i64(i64 [[V3]], i1 false)
+; SSE-NEXT:    [[R0:%.*]] = insertelement <4 x i64> poison, i64 [[C0]], i64 0
+; SSE-NEXT:    [[R1:%.*]] = insertelement <4 x i64> [[R0]], i64 [[C1]], i64 1
+; SSE-NEXT:    [[R2:%.*]] = insertelement <4 x i64> [[R1]], i64 [[C2]], i64 2
+; SSE-NEXT:    [[R3:%.*]] = insertelement <4 x i64> [[R2]], i64 [[C3]], i64 3
+; SSE-NEXT:    ret <4 x i64> [[R3]]
+;
+; AVX2-LABEL: define <4 x i64> @scalarize_ctlz_v4i64(
+; AVX2-SAME: <4 x i64> [[V:%.*]]) #[[ATTR0]] {
+; AVX2-NEXT:    [[V0:%.*]] = extractelement <4 x i64> [[V]], i64 0
+; AVX2-NEXT:    [[V1:%.*]] = extractelement <4 x i64> [[V]], i64 1
+; AVX2-NEXT:    [[V2:%.*]] = extractelement <4 x i64> [[V]], i64 2
+; AVX2-NEXT:    [[V3:%.*]] = extractelement <4 x i64> [[V]], i64 3
+; AVX2-NEXT:    [[C0:%.*]] = tail call range(i64 0, 65) i64 @llvm.ctlz.i64(i64 [[V0]], i1 false)
+; AVX2-NEXT:    [[C1:%.*]] = tail call range(i64 0, 65) i64 @llvm.ctlz.i64(i64 [[V1]], i1 false)
+; AVX2-NEXT:    [[C2:%.*]] = tail call range(i64 0, 65) i64 @llvm.ctlz.i64(i64 [[V2]], i1 false)
+; AVX2-NEXT:    [[C3:%.*]] = tail call range(i64 0, 65) i64 @llvm.ctlz.i64(i64 [[V3]], i1 false)
+; AVX2-NEXT:    [[R0:%.*]] = insertelement <4 x i64> poison, i64 [[C0]], i64 0
+; AVX2-NEXT:    [[R1:%.*]] = insertelement <4 x i64> [[R0]], i64 [[C1]], i64 1
+; AVX2-NEXT:    [[R2:%.*]] = insertelement <4 x i64> [[R1]], i64 [[C2]], i64 2
+; AVX2-NEXT:    [[R3:%.*]] = insertelement <4 x i64> [[R2]], i64 [[C3]], i64 3
+; AVX2-NEXT:    ret <4 x i64> [[R3]]
+;
+; AVX512-LABEL: define <4 x i64> @scalarize_ctlz_v4i64(
+; AVX512-SAME: <4 x i64> [[V:%.*]]) #[[ATTR0]] {
+; AVX512-NEXT:    [[TMP1:%.*]] = call <4 x i64> @llvm.ctlz.v4i64(<4 x i64> [[V]], i1 false)
+; AVX512-NEXT:    ret <4 x i64> [[TMP1]]
+;
+  %v0 = extractelement <4 x i64> %v, i64 0
+  %v1 = extractelement <4 x i64> %v, i64 1
+  %v2 = extractelement <4 x i64> %v, i64 2
+  %v3 = extractelement <4 x i64> %v, i64 3
+  %c0 = tail call range(i64 0, 65) i64 @llvm.ctlz.i64(i64 %v0, i1 false)
+  %c1 = tail call range(i64 0, 65) i64 @llvm.ctlz.i64(i64 %v1, i1 false)
+  %c2 = tail call range(i64 0, 65) i64 @llvm.ctlz.i64(i64 %v2, i1 false)
+  %c3 = tail call range(i64 0, 65) i64 @llvm.ctlz.i64(i64 %v3, i1 false)
+  %r0 = insertelement <4 x i64> poison, i64 %c0, i64 0
+  %r1 = insertelement <4 x i64> %r0, i64 %c1, i64 1
+  %r2 = insertelement <4 x i64> %r1, i64 %c2, i64 2
+  %r3 = insertelement <4 x i64> %r2, i64 %c3, i64 3
+  ret <4 x i64> %r3
+}
+
+define <8 x i64> @scalarize_ctlz_v8i64(<8 x i64> %v)  {
+; SSE2-LABEL: define <8 x i64> @scalarize_ctlz_v8i64(
+; SSE2-SAME: <8 x i64> [[V:%.*]]) #[[ATTR0]] {
+; SSE2-NEXT:    [[TMP1:%.*]] = shufflevector <8 x i64> [[V]], <8 x i64> poison, <2 x i32> <i32 0, i32 1>
+; SSE2-NEXT:    [[TMP2:%.*]] = call <2 x i64> @llvm.ctlz.v2i64(<2 x i64> [[TMP1]], i1 false)
+; SSE2-NEXT:    [[TMP3:%.*]] = shufflevector <8 x i64> [[V]], <8 x i64> poison, <2 x i32> <i32 2, i32 3>
+; SSE2-NEXT:    [[TMP4:%.*]] = call <2 x i64> @llvm.ctlz.v2i64(<2 x i64> [[TMP3]], i1 false)
+; SSE2-NEXT:    [[TMP5:%.*]] = shufflevector <8 x i64> [[V]], <8 x i64> poison, <2 x i32> <i32 4, i32 5>
+; SSE2-NEXT:    [[TMP6:%.*]] = call <2 x i64> @llvm.ctlz.v2i64(<2 x i64> [[TMP5]], i1 false)
+; SSE2-NEXT:    [[TMP7:%.*]] = shufflevector <8 x i64> [[V]], <8 x i64> poison, <2 x i32> <i32 6, i32 7>
+; SSE2-NEXT:    [[TMP8:%.*]] = call <2 x i64> @llvm.ctlz.v2i64(<2 x i64> [[TMP7]], i1 false)
+; SSE2-NEXT:    [[TMP9:%.*]] = shufflevector <2 x i64> [[TMP2]], <2 x i64> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE2-NEXT:    [[TMP10:%.*]] = shufflevector <2 x i64> [[TMP4]], <2 x i64> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE2-NEXT:    [[R31:%.*]] = shufflevector <8 x i64> [[TMP9]], <8 x i64> [[TMP10]], <8 x i32> <i32 0, i32 1, i32 8, i32 9, i32 4, i32 5, i32 6, i32 7>
+; SSE2-NEXT:    [[TMP11:%.*]] = shufflevector <2 x i64> [[TMP6]], <2 x i64> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE2-NEXT:    [[R52:%.*]] = shufflevector <8 x i64> [[R31]], <8 x i64> [[TMP11]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 6, i32 7>
+; SSE2-NEXT:    [[TMP12:%.*]] = shufflevector <2 x i64> [[TMP8]], <2 x i64> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE2-NEXT:    [[R73:%.*]] = shufflevector <8 x i64> [[R52]], <8 x i64> [[TMP12]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 8, i32 9>
+; SSE2-NEXT:    ret <8 x i64> [[R73]]
+;
+; SSE4-LABEL: define <8 x i64> @scalarize_ctlz_v8i64(
+; SSE4-SAME: <8 x i64> [[V:%.*]]) #[[ATTR0]] {
+; SSE4-NEXT:    [[V0:%.*]] = extractelement <8 x i64> [[V]], i64 0
+; SSE4-NEXT:    [[V1:%.*]] = extractelement <8 x i64> [[V]], i64 1
+; SSE4-NEXT:    [[V2:%.*]] = extractelement <8 x i64> [[V]], i64 2
+; SSE4-NEXT:    [[V3:%.*]] = extractelement <8 x i64> [[V]], i64 3
+; SSE4-NEXT:    [[V4:%.*]] = extractelement <8 x i64> [[V]], i64 4
+; SSE4-NEXT:    [[V5:%.*]] = extractelement <8 x i64> [[V]], i64 5
+; SSE4-NEXT:    [[V6:%.*]] = extractelement <8 x i64> [[V]], i64 6
+; SSE4-NEXT:    [[V7:%.*]] = extractelement <8 x i64> [[V]], i64 7
+; SSE4-NEXT:    [[C0:%.*]] = tail call range(i64 0, 65) i64 @llvm.ctlz.i64(i64 [[V0]], i1 false)
+; SSE4-NEXT:    [[C1:%.*]] = tail call range(i64 0, 65) i64 @llvm.ctlz.i64(i64 [[V1]], i1 false)
+; SSE4-NEXT:    [[C2:%.*]] = tail call range(i64 0, 65) i64 @llvm.ctlz.i64(i64 [[V2]], i1 false)
+; SSE4-NEXT:    [[C3:%.*]] = tail call range(i64 0, 65) i64 @llvm.ctlz.i64(i64 [[V3]], i1 false)
+; SSE4-NEXT:    [[C4:%.*]] = tail call range(i64 0, 65) i64 @llvm.ctlz.i64(i64 [[V4]], i1 false)
+; SSE4-NEXT:    [[C5:%.*]] = tail call range(i64 0, 65) i64 @llvm.ctlz.i64(i64 [[V5]], i1 false)
+; SSE4-NEXT:    [[C6:%.*]] = tail call range(i64 0, 65) i64 @llvm.ctlz.i64(i64 [[V6]], i1 false)
+; SSE4-NEXT:    [[C7:%.*]] = tail call range(i64 0, 65) i64 @llvm.ctlz.i64(i64 [[V7]], i1 false)
+; SSE4-NEXT:    [[R0:%.*]] = insertelement <8 x i64> poison, i64 [[C0]], i64 0
+; SSE4-NEXT:    [[R1:%.*]] = insertelement <8 x i64> [[R0]], i64 [[C1]], i64 1
+; SSE4-NEXT:    [[R2:%.*]] = insertelement <8 x i64> [[R1]], i64 [[C2]], i64 2
+; SSE4-NEXT:    [[R3:%.*]] = insertelement <8 x i64> [[R2]], i64 [[C3]], i64 3
+; SSE4-NEXT:    [[R4:%.*]] = insertelement <8 x i64> [[R3]], i64 [[C4]], i64 4
+; SSE4-NEXT:    [[R5:%.*]] = insertelement <8 x i64> [[R4]], i64 [[C5]], i64 5
+; SSE4-NEXT:    [[R6:%.*]] = insertelement <8 x i64> [[R5]], i64 [[C6]], i64 6
+; SSE4-NEXT:    [[R7:%.*]] = insertelement <8 x i64> [[R6]], i64 [[C7]], i64 7
+; SSE4-NEXT:    ret <8 x i64> [[R7]]
+;
+; AVX2-LABEL: define <8 x i64> @scalarize_ctlz_v8i64(
+; AVX2-SAME: <8 x i64> [[V:%.*]]) #[[ATTR0]] {
+; AVX2-NEXT:    [[V0:%.*]] = extractelement <8 x i64> [[V]], i64 0
+; AVX2-NEXT:    [[V1:%.*]] = extractelement <8 x i64> [[V]], i64 1
+; AVX2-NEXT:    [[V2:%.*]] = extractelement <8 x i64> [[V]], i64 2
+; AVX2-NEXT:    [[V3:%.*]] = extractelement <8 x i64> [[V]], i64 3
+; AVX2-NEXT:    [[V4:%.*]] = extractelement <8 x i64> [[V]], i64 4
+; AVX2-NEXT:    [[V5:%.*]] = extractelement <8 x i64> [[V]], i64 5
+; AVX2-NEXT:    [[V6:%.*]] = extractelement <8 x i64> [[V]], i64 6
+; AVX2-NEXT:    [[V7:%.*]] = extractelement <8 x i64> [[V]], i64 7
+; AVX2-NEXT:    [[C0:%.*]] = tail call range(i64 0, 65) i64 @llvm.ctlz.i64(i64 [[V0]], i1 false)
+; AVX2-NEXT:    [[C1:%.*]] = tail call range(i64 0, 65) i64 @llvm.ctlz.i64(i64 [[V1]], i1 false)
+; AVX2-NEXT:    [[C2:%.*]] = tail call range(i64 0, 65) i64 @llvm.ctlz.i64(i64 [[V2]], i1 false)
+; AVX2-NEXT:    [[C3:%.*]] = tail call range(i64 0, 65) i64 @llvm.ctlz.i64(i64 [[V3]], i1 false)
+; AVX2-NEXT:    [[C4:%.*]] = tail call range(i64 0, 65) i64 @llvm.ctlz.i64(i64 [[V4]], i1 false)
+; AVX2-NEXT:    [[C5:%.*]] = tail call range(i64 0, 65) i64 @llvm.ctlz.i64(i64 [[V5]], i1 false)
+; AVX2-NEXT:    [[C6:%.*]] = tail call range(i64 0, 65) i64 @llvm.ctlz.i64(i64 [[V6]], i1 false)
+; AVX2-NEXT:    [[C7:%.*]] = tail call range(i64 0, 65) i64 @llvm.ctlz.i64(i64 [[V7]], i1 false)
+; AVX2-NEXT:    [[R0:%.*]] = insertelement <8 x i64> poison, i64 [[C0]], i64 0
+; AVX2-NEXT:    [[R1:%.*]] = insertelement <8 x i64> [[R0]], i64 [[C1]], i64 1
+; AVX2-NEXT:    [[R2:%.*]] = insertelement <8 x i64> [[R1]], i64 [[C2]], i64 2
+; AVX2-NEXT:    [[R3:%.*]] = insertelement <8 x i64> [[R2]], i64 [[C3]], i64 3
+; AVX2-NEXT:    [[R4:%.*]] = insertelement <8 x i64> [[R3]], i64 [[C4]], i64 4
+; AVX2-NEXT:    [[R5:%.*]] = insertelement <8 x i64> [[R4]], i64 [[C5]], i64 5
+; AVX2-NEXT:    [[R6:%.*]] = insertelement <8 x i64> [[R5]], i64 [[C6]], i64 6
+; AVX2-NEXT:    [[R7:%.*]] = insertelement <8 x i64> [[R6]], i64 [[C7]], i64 7
+; AVX2-NEXT:    ret <8 x i64> [[R7]]
+;
+; AVX512-LABEL: define <8 x i64> @scalarize_ctlz_v8i64(
+; AVX512-SAME: <8 x i64> [[V:%.*]]) #[[ATTR0]] {
+; AVX512-NEXT:    [[TMP1:%.*]] = call <8 x i64> @llvm.ctlz.v8i64(<8 x i64> [[V]], i1 false)
+; AVX512-NEXT:    ret <8 x i64> [[TMP1]]
+;
+  %v0 = extractelement <8 x i64> %v, i64 0
+  %v1 = extractelement <8 x i64> %v, i64 1
+  %v2 = extractelement <8 x i64> %v, i64 2
+  %v3 = extractelement <8 x i64> %v, i64 3
+  %v4 = extractelement <8 x i64> %v, i64 4
+  %v5 = extractelement <8 x i64> %v, i64 5
+  %v6 = extractelement <8 x i64> %v, i64 6
+  %v7 = extractelement <8 x i64> %v, i64 7
+  %c0 = tail call range(i64 0, 65) i64 @llvm.ctlz.i64(i64 %v0, i1 false)
+  %c1 = tail call range(i64 0, 65) i64 @llvm.ctlz.i64(i64 %v1, i1 false)
+  %c2 = tail call range(i64 0, 65) i64 @llvm.ctlz.i64(i64 %v2, i1 false)
+  %c3 = tail call range(i64 0, 65) i64 @llvm.ctlz.i64(i64 %v3, i1 false)
+  %c4 = tail call range(i64 0, 65) i64 @llvm.ctlz.i64(i64 %v4, i1 false)
+  %c5 = tail call range(i64 0, 65) i64 @llvm.ctlz.i64(i64 %v5, i1 false)
+  %c6 = tail call range(i64 0, 65) i64 @llvm.ctlz.i64(i64 %v6, i1 false)
+  %c7 = tail call range(i64 0, 65) i64 @llvm.ctlz.i64(i64 %v7, i1 false)
+  %r0 = insertelement <8 x i64> poison, i64 %c0, i64 0
+  %r1 = insertelement <8 x i64> %r0, i64 %c1, i64 1
+  %r2 = insertelement <8 x i64> %r1, i64 %c2, i64 2
+  %r3 = insertelement <8 x i64> %r2, i64 %c3, i64 3
+  %r4 = insertelement <8 x i64> %r3, i64 %c4, i64 4
+  %r5 = insertelement <8 x i64> %r4, i64 %c5, i64 5
+  %r6 = insertelement <8 x i64> %r5, i64 %c6, i64 6
+  %r7 = insertelement <8 x i64> %r6, i64 %c7, i64 7
+  ret <8 x i64> %r7
+}
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/sin-sqrt.ll b/llvm/test/Transforms/SLPVectorizer/X86/sin-sqrt.ll
index 68fb6c6202ce7..e1e80d96d416d 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/sin-sqrt.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/sin-sqrt.ll
@@ -11,16 +11,25 @@ declare double @llvm.sin.f64(double)
 
 define void @test() {
 ; CHECK-LABEL: @test(
-; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x double>, ptr @src, align 8
-; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <8 x double> [[TMP1]], <8 x double> poison, <2 x i32> <i32 2, i32 6>
-; CHECK-NEXT:    [[TMP3:%.*]] = call fast <2 x double> @llvm.sin.v2f64(<2 x double> [[TMP2]])
-; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <8 x double> [[TMP1]], <8 x double> poison, <2 x i32> <i32 3, i32 7>
-; CHECK-NEXT:    [[TMP5:%.*]] = call fast <2 x double> @llvm.sin.v2f64(<2 x double> [[TMP4]])
-; CHECK-NEXT:    [[TMP6:%.*]] = shufflevector <8 x double> [[TMP1]], <8 x double> poison, <2 x i32> <i32 0, i32 4>
+; CHECK-NEXT:    [[A2:%.*]] = load double, ptr getelementptr inbounds ([8 x double], ptr @src, i32 0, i64 2), align 8
+; CHECK-NEXT:    [[A3:%.*]] = load double, ptr getelementptr inbounds ([8 x double], ptr @src, i32 0, i64 3), align 8
+; CHECK-NEXT:    [[A6:%.*]] = load double, ptr getelementptr inbounds ([8 x double], ptr @src, i32 0, i64 6), align 8
+; CHECK-NEXT:    [[A7:%.*]] = load double, ptr getelementptr inbounds ([8 x double], ptr @src, i32 0, i64 7), align 8
+; CHECK-NEXT:    [[SIN0:%.*]] = call fast double @llvm.sin.f64(double [[A2]])
+; CHECK-NEXT:    [[SIN1:%.*]] = call fast double @llvm.sin.f64(double [[A3]])
+; CHECK-NEXT:    [[SIN2:%.*]] = call fast double @llvm.sin.f64(double [[A6]])
+; CHECK-NEXT:    [[SIN3:%.*]] = call fast double @llvm.sin.f64(double [[A7]])
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x double>, ptr @src, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x double>, ptr getelementptr inbounds ([8 x double], ptr @src, i32 0, i64 4), align 8
+; CHECK-NEXT:    [[TMP6:%.*]] = shufflevector <2 x double> [[TMP1]], <2 x double> [[TMP2]], <2 x i32> <i32 0, i32 2>
 ; CHECK-NEXT:    [[TMP7:%.*]] = call fast <2 x double> @llvm.sqrt.v2f64(<2 x double> [[TMP6]])
-; CHECK-NEXT:    [[TMP8:%.*]] = shufflevector <8 x double> [[TMP1]], <8 x double> poison, <2 x i32> <i32 1, i32 5>
+; CHECK-NEXT:    [[TMP8:%.*]] = shufflevector <2 x double> [[TMP1]], <2 x double> [[TMP2]], <2 x i32> <i32 1, i32 3>
 ; CHECK-NEXT:    [[TMP9:%.*]] = call fast <2 x double> @llvm.sqrt.v2f64(<2 x double> [[TMP8]])
+; CHECK-NEXT:    [[TMP13:%.*]] = insertelement <2 x double> poison, double [[SIN1]], i32 0
+; CHECK-NEXT:    [[TMP5:%.*]] = insertelement <2 x double> [[TMP13]], double [[SIN3]], i32 1
 ; CHECK-NEXT:    [[TMP10:%.*]] = fadd fast <2 x double> [[TMP7]], [[TMP5]]
+; CHECK-NEXT:    [[TMP14:%.*]] = insertelement <2 x double> poison, double [[SIN0]], i32 0
+; CHECK-NEXT:    [[TMP3:%.*]] = insertelement <2 x double> [[TMP14]], double [[SIN2]], i32 1
 ; CHECK-NEXT:    [[TMP11:%.*]] = fadd fast <2 x double> [[TMP3]], [[TMP9]]
 ; CHECK-NEXT:    [[TMP12:%.*]] = fadd fast <2 x double> [[TMP10]], [[TMP11]]
 ; CHECK-NEXT:    store <2 x double> [[TMP12]], ptr @dst, align 8
diff --git a/llvm/test/Transforms/SLPVectorizer/reduction-gather-non-scheduled-extracts.ll b/llvm/test/Transforms/SLPVectorizer/reduction-gather-non-scheduled-extracts.ll
index f1034f3971135..ae5018a63e214 100644
--- a/llvm/test/Transforms/SLPVectorizer/reduction-gather-non-scheduled-extracts.ll
+++ b/llvm/test/Transforms/SLPVectorizer/reduction-gather-non-scheduled-extracts.ll
@@ -1,22 +1,43 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 3
-; RUN: %if x86-registered-target %{ opt -S --passes=slp-vectorizer -mtriple=x86_64-sie-ps5 < %s | FileCheck %s %}
-; RUN: %if aarch64-registered-target %{ opt -S --passes=slp-vectorizer -mtriple=aarch64-unknown-linux-gnu < %s | FileCheck %s %}
+; RUN: %if x86-registered-target %{ opt -S --passes=slp-vectorizer -mtriple=x86_64-sie-ps5 < %s | FileCheck %s --check-prefix=X86 %}
+; RUN: %if aarch64-registered-target %{ opt -S --passes=slp-vectorizer -mtriple=aarch64-unknown-linux-gnu < %s | FileCheck %s --check-prefix=AARCH64 %}
 
 define void @tes() {
-; CHECK-LABEL: define void @tes() {
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[TMP0:%.*]] = fcmp ole <2 x double> zeroinitializer, zeroinitializer
-; CHECK-NEXT:    br label [[TMP1:%.*]]
-; CHECK:       1:
-; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <2 x i1> zeroinitializer, <2 x i1> [[TMP0]], <4 x i32> <i32 0, i32 0, i32 0, i32 2>
-; CHECK-NEXT:    [[TMP4:%.*]] = call i1 @llvm.vector.reduce.and.v4i1(<4 x i1> [[TMP3]])
-; CHECK-NEXT:    [[OP_RDX:%.*]] = select i1 false, i1 [[TMP4]], i1 false
-; CHECK-NEXT:    [[OP_RDX1:%.*]] = select i1 false, i1 [[OP_RDX]], i1 false
-; CHECK-NEXT:    br i1 [[OP_RDX1]], label [[TMP6:%.*]], label [[TMP5:%.*]]
-; CHECK:       4:
-; CHECK-NEXT:    ret void
-; CHECK:       5:
-; CHECK-NEXT:    ret void
+; X86-LABEL: define void @tes() {
+; X86-NEXT:  entry:
+; X86-NEXT:    [[TMP0:%.*]] = fcmp ole <2 x double> zeroinitializer, zeroinitializer
+; X86-NEXT:    br label [[TMP1:%.*]]
+; X86:       1:
+; X86-NEXT:    [[TMP2:%.*]] = shufflevector <2 x i1> zeroinitializer, <2 x i1> [[TMP0]], <4 x i32> <i32 0, i32 0, i32 0, i32 2>
+; X86-NEXT:    [[TMP3:%.*]] = call i1 @llvm.vector.reduce.and.v4i1(<4 x i1> [[TMP2]])
+; X86-NEXT:    [[OP_RDX:%.*]] = select i1 false, i1 [[TMP3]], i1 false
+; X86-NEXT:    [[OP_RDX1:%.*]] = select i1 false, i1 [[OP_RDX]], i1 false
+; X86-NEXT:    br i1 [[OP_RDX1]], label [[TMP4:%.*]], label [[TMP5:%.*]]
+; X86:       4:
+; X86-NEXT:    ret void
+; X86:       5:
+; X86-NEXT:    ret void
+;
+; AARCH64-LABEL: define void @tes() {
+; AARCH64-NEXT:  entry:
+; AARCH64-NEXT:    [[TMP0:%.*]] = extractelement <2 x i1> zeroinitializer, i64 0
+; AARCH64-NEXT:    [[TMP1:%.*]] = extractelement <2 x i1> zeroinitializer, i64 0
+; AARCH64-NEXT:    [[TMP2:%.*]] = fcmp ole <2 x double> zeroinitializer, zeroinitializer
+; AARCH64-NEXT:    [[TMP3:%.*]] = extractelement <2 x i1> [[TMP2]], i64 0
+; AARCH64-NEXT:    [[TMP4:%.*]] = extractelement <2 x i1> zeroinitializer, i64 0
+; AARCH64-NEXT:    br label [[TMP5:%.*]]
+; AARCH64:       5:
+; AARCH64-NEXT:    [[TMP6:%.*]] = select i1 false, i1 false, i1 false
+; AARCH64-NEXT:    [[TMP7:%.*]] = select i1 [[TMP6]], i1 [[TMP0]], i1 false
+; AARCH64-NEXT:    [[TMP8:%.*]] = select i1 [[TMP7]], i1 [[TMP1]], i1 false
+; AARCH64-NEXT:    [[TMP9:%.*]] = select i1 [[TMP8]], i1 false, i1 false
+; AARCH64-NEXT:    [[TMP10:%.*]] = select i1 [[TMP9]], i1 [[TMP3]], i1 false
+; AARCH64-NEXT:    [[TMP11:%.*]] = select i1 [[TMP10]], i1 [[TMP4]], i1 false
+; AARCH64-NEXT:    br i1 [[TMP11]], label [[TMP12:%.*]], label [[TMP13:%.*]]
+; AARCH64:       12:
+; AARCH64-NEXT:    ret void
+; AARCH64:       13:
+; AARCH64-NEXT:    ret void
 ;
 entry:
   %0 = extractelement <2 x i1> zeroinitializer, i64 0
diff --git a/llvm/test/Transforms/SLPVectorizer/revec-shufflevector.ll b/llvm/test/Transforms/SLPVectorizer/revec-shufflevector.ll
index a2673d81068d8..d6c09bc224a7d 100644
--- a/llvm/test/Transforms/SLPVectorizer/revec-shufflevector.ll
+++ b/llvm/test/Transforms/SLPVectorizer/revec-shufflevector.ll
@@ -121,3 +121,117 @@ entry:
   store <4 x i32> %1, ptr %3, align 4
   ret void
 }
+
+define void @test6(ptr %in0, ptr %in1, ptr %in2) {
+; CHECK-LABEL: @test6(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[GEP1:%.*]] = getelementptr inbounds nuw i8, ptr [[IN0:%.*]], i64 32
+; CHECK-NEXT:    [[LOAD2:%.*]] = load <4 x float>, ptr [[GEP1]], align 16
+; CHECK-NEXT:    [[TMP0:%.*]] = load <8 x float>, ptr [[IN0]], align 16
+; CHECK-NEXT:    [[TMP1:%.*]] = load <32 x i8>, ptr [[IN1:%.*]], align 1
+; CHECK-NEXT:    [[TMP2:%.*]] = uitofp <32 x i8> [[TMP1]] to <32 x float>
+; CHECK-NEXT:    [[TMP14:%.*]] = shufflevector <8 x float> [[TMP0]], <8 x float> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP15:%.*]] = shufflevector <4 x float> [[LOAD2]], <4 x float> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP16:%.*]] = shufflevector <16 x float> [[TMP14]], <16 x float> [[TMP15]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <16 x float> [[TMP16]], <16 x float> poison, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:    [[TMP4:%.*]] = fmul <32 x float> [[TMP3]], [[TMP2]]
+; CHECK-NEXT:    store <32 x float> [[TMP4]], ptr [[IN2:%.*]], align 16
+; CHECK-NEXT:    [[GEP10:%.*]] = getelementptr inbounds nuw i8, ptr [[IN1]], i64 32
+; CHECK-NEXT:    [[LOAD5:%.*]] = load <16 x i8>, ptr [[GEP10]], align 1
+; CHECK-NEXT:    [[TMP5:%.*]] = shufflevector <8 x float> [[TMP0]], <8 x float> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:    [[GEP11:%.*]] = getelementptr inbounds nuw i8, ptr [[IN2]], i64 128
+; CHECK-NEXT:    [[TMP6:%.*]] = uitofp <16 x i8> [[LOAD5]] to <16 x float>
+; CHECK-NEXT:    [[TMP7:%.*]] = shufflevector <4 x float> [[LOAD2]], <4 x float> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP8:%.*]] = shufflevector <8 x float> [[TMP0]], <8 x float> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP9:%.*]] = shufflevector <16 x float> [[TMP7]], <16 x float> [[TMP8]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 16, i32 17, i32 18, i32 19, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP10:%.*]] = shufflevector <4 x float> [[TMP5]], <4 x float> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP11:%.*]] = shufflevector <16 x float> [[TMP9]], <16 x float> [[TMP10]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP12:%.*]] = shufflevector <16 x float> [[TMP11]], <16 x float> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[TMP13:%.*]] = fmul <16 x float> [[TMP12]], [[TMP6]]
+; CHECK-NEXT:    store <16 x float> [[TMP13]], ptr [[GEP11]], align 16
+; CHECK-NEXT:    ret void
+;
+entry:
+  %gep0 = getelementptr inbounds i8, ptr %in0, i64 16
+  %gep1 = getelementptr inbounds i8, ptr %in0, i64 32
+  %load0 = load <4 x float>, ptr %in0, align 16
+  %load1 = load <4 x float>, ptr %gep0, align 16
+  %load2 = load <4 x float>, ptr %gep1, align 16
+  %gep2 = getelementptr inbounds i8, ptr %in1, i64 16
+  %load3 = load <16 x i8>, ptr %in1, align 1
+  %load4 = load <16 x i8>, ptr %gep2, align 1
+  %shufflevector0 = shufflevector <16 x i8> %load3, <16 x i8> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+  %shufflevector1 = shufflevector <16 x i8> %load3, <16 x i8> poison, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+  %shufflevector2 = shufflevector <16 x i8> %load4, <16 x i8> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+  %shufflevector3 = shufflevector <16 x i8> %load4, <16 x i8> poison, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+  %zext0 = zext <8 x i8> %shufflevector0 to <8 x i16>
+  %zext1 = zext <8 x i8> %shufflevector1 to <8 x i16>
+  %zext2 = zext <8 x i8> %shufflevector2 to <8 x i16>
+  %zext3 = zext <8 x i8> %shufflevector3 to <8 x i16>
+  %shufflevector4 = shufflevector <8 x i16> %zext0, <8 x i16> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %shufflevector5 = shufflevector <8 x i16> %zext0, <8 x i16> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+  %shufflevector6 = shufflevector <8 x i16> %zext1, <8 x i16> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %shufflevector7 = shufflevector <8 x i16> %zext1, <8 x i16> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+  %shufflevector8 = shufflevector <8 x i16> %zext2, <8 x i16> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %shufflevector9 = shufflevector <8 x i16> %zext2, <8 x i16> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+  %shufflevector10 = shufflevector <8 x i16> %zext3, <8 x i16> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %shufflevector11 = shufflevector <8 x i16> %zext3, <8 x i16> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+  %uitofp0 = uitofp nneg <4 x i16> %shufflevector4 to <4 x float>
+  %uitofp1 = uitofp nneg <4 x i16> %shufflevector5 to <4 x float>
+  %uitofp2 = uitofp nneg <4 x i16> %shufflevector6 to <4 x float>
+  %uitofp3 = uitofp nneg <4 x i16> %shufflevector7 to <4 x float>
+  %uitofp4 = uitofp nneg <4 x i16> %shufflevector8 to <4 x float>
+  %uitofp5 = uitofp nneg <4 x i16> %shufflevector9 to <4 x float>
+  %uitofp6 = uitofp nneg <4 x i16> %shufflevector10 to <4 x float>
+  %uitofp7 = uitofp nneg <4 x i16> %shufflevector11 to <4 x float>
+  %fmul0 = fmul <4 x float> %load0, %uitofp0
+  %fmul1 = fmul <4 x float> %load1, %uitofp1
+  %fmul2 = fmul <4 x float> %load2, %uitofp2
+  %fmul3 = fmul <4 x float> %load0, %uitofp3
+  %fmul4 = fmul <4 x float> %load1, %uitofp4
+  %fmul5 = fmul <4 x float> %load2, %uitofp5
+  %fmul6 = fmul <4 x float> %load0, %uitofp6
+  %fmul7 = fmul <4 x float> %load1, %uitofp7
+  %gep3 = getelementptr inbounds i8, ptr %in2, i64 16
+  %gep4 = getelementptr inbounds i8, ptr %in2, i64 32
+  %gep5 = getelementptr inbounds i8, ptr %in2, i64 48
+  %gep6 = getelementptr inbounds i8, ptr %in2, i64 64
+  %gep7 = getelementptr inbounds i8, ptr %in2, i64 80
+  %gep8 = getelementptr inbounds i8, ptr %in2, i64 96
+  %gep9 = getelementptr inbounds i8, ptr %in2, i64 112
+  store <4 x float> %fmul0, ptr %in2, align 16
+  store <4 x float> %fmul1, ptr %gep3, align 16
+  store <4 x float> %fmul2, ptr %gep4, align 16
+  store <4 x float> %fmul3, ptr %gep5, align 16
+  store <4 x float> %fmul4, ptr %gep6, align 16
+  store <4 x float> %fmul5, ptr %gep7, align 16
+  store <4 x float> %fmul6, ptr %gep8, align 16
+  store <4 x float> %fmul7, ptr %gep9, align 16
+  %gep10 = getelementptr inbounds i8, ptr %in1, i64 32
+  %load5 = load <16 x i8>, ptr %gep10, align 1
+  %shufflevector12 = shufflevector <16 x i8> %load5, <16 x i8> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+  %shufflevector13 = shufflevector <16 x i8> %load5, <16 x i8> poison, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+  %zext4 = zext <8 x i8> %shufflevector12 to <8 x i16>
+  %zext5 = zext <8 x i8> %shufflevector13 to <8 x i16>
+  %shufflevector14 = shufflevector <8 x i16> %zext4, <8 x i16> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %shufflevector15 = shufflevector <8 x i16> %zext4, <8 x i16> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+  %shufflevector16 = shufflevector <8 x i16> %zext5, <8 x i16> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %shufflevector17 = shufflevector <8 x i16> %zext5, <8 x i16> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+  %uitofp8 = uitofp nneg <4 x i16> %shufflevector14 to <4 x float>
+  %uitofp9 = uitofp nneg <4 x i16> %shufflevector15 to <4 x float>
+  %uitofp10 = uitofp nneg <4 x i16> %shufflevector16 to <4 x float>
+  %uitofp11 = uitofp nneg <4 x i16> %shufflevector17 to <4 x float>
+  %fmul8 = fmul <4 x float> %load2, %uitofp8
+  %fmul9 = fmul <4 x float> %load0, %uitofp9
+  %fmul10 = fmul <4 x float> %load1, %uitofp10
+  %fmul11 = fmul <4 x float> %load2, %uitofp11
+  %gep11 = getelementptr inbounds i8, ptr %in2, i64 128
+  %gep12 = getelementptr inbounds i8, ptr %in2, i64 144
+  %gep13 = getelementptr inbounds i8, ptr %in2, i64 160
+  %gep14 = getelementptr inbounds i8, ptr %in2, i64 176
+  store <4 x float> %fmul8, ptr %gep11, align 16
+  store <4 x float> %fmul9, ptr %gep12, align 16
+  store <4 x float> %fmul10, ptr %gep13, align 16
+  store <4 x float> %fmul11, ptr %gep14, align 16
+  ret void
+}
diff --git a/llvm/test/Transforms/SLPVectorizer/scalarazied-result.ll b/llvm/test/Transforms/SLPVectorizer/scalarazied-result.ll
index 2570cdb45e1e7..0a6b86c953ee0 100644
--- a/llvm/test/Transforms/SLPVectorizer/scalarazied-result.ll
+++ b/llvm/test/Transforms/SLPVectorizer/scalarazied-result.ll
@@ -1,11 +1,19 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: %if x86-registered-target %{ opt -passes=slp-vectorizer -mtriple=x86_64-unknown-linux-gnu -S < %s | FileCheck %s %}
-; RUN: %if aarch64-registered-target %{ opt -passes=slp-vectorizer -mtriple=aarch64-unknown-linux-gnu -S < %s | FileCheck %s %}
+; RUN: %if x86-registered-target %{ opt -passes=slp-vectorizer -mtriple=x86_64-unknown-linux-gnu -S < %s | FileCheck %s --check-prefix=X86 %}
+; RUN: %if aarch64-registered-target %{ opt -passes=slp-vectorizer -mtriple=aarch64-unknown-linux-gnu -S < %s | FileCheck %s --check-prefix=AARCH64 %}
 
 define void @test() {
-; CHECK-LABEL: @test(
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    ret void
+; X86-LABEL: @test(
+; X86-NEXT:  entry:
+; X86-NEXT:    ret void
+;
+; AARCH64-LABEL: @test(
+; AARCH64-NEXT:  entry:
+; AARCH64-NEXT:    [[TMP0:%.*]] = extractelement <8 x half> zeroinitializer, i64 1
+; AARCH64-NEXT:    [[TOBOOL:%.*]] = fcmp une half [[TMP0]], 0xH0000
+; AARCH64-NEXT:    [[TMP1:%.*]] = extractelement <8 x half> zeroinitializer, i64 1
+; AARCH64-NEXT:    [[TOBOOL3:%.*]] = fcmp une half [[TMP1]], 0xH0000
+; AARCH64-NEXT:    ret void
 ;
 entry:
   %0 = extractelement <8 x half> zeroinitializer, i64 1
diff --git a/llvm/test/Transforms/SROA/non-capturing-call-readonly.ll b/llvm/test/Transforms/SROA/non-capturing-call-readonly.ll
index cc57abe391aa8..a37f02df46c75 100644
--- a/llvm/test/Transforms/SROA/non-capturing-call-readonly.ll
+++ b/llvm/test/Transforms/SROA/non-capturing-call-readonly.ll
@@ -104,7 +104,7 @@ define i32 @alloca_not_captured_as_per_operand_attr(ptr %data, i64 %n) {
 ; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp ne i64 [[INDVARS_IV_NEXT]], [[N:%.*]]
 ; CHECK-NEXT:    br i1 [[EXITCOND]], label [[LOOP]], label [[EXIT:%.*]]
 ; CHECK:       exit:
-; CHECK-NEXT:    [[I0:%.*]] = call i32 @capture_of_alloca(ptr nocapture [[RETVAL]])
+; CHECK-NEXT:    [[I0:%.*]] = call i32 @capture_of_alloca(ptr captures(none) [[RETVAL]])
 ; CHECK-NEXT:    [[I1:%.*]] = load i32, ptr [[RETVAL]], align 4
 ; CHECK-NEXT:    ret i32 [[I1]]
 ;
@@ -147,7 +147,7 @@ define i32 @alloca_not_captured_and_readonly_as_per_operand_attr(ptr %data, i64
 ; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp ne i64 [[INDVARS_IV_NEXT]], [[N:%.*]]
 ; CHECK-NEXT:    br i1 [[EXITCOND]], label [[LOOP]], label [[EXIT:%.*]]
 ; CHECK:       exit:
-; CHECK-NEXT:    [[I0:%.*]] = call i32 @capture_of_alloca(ptr nocapture readonly [[RETVAL]])
+; CHECK-NEXT:    [[I0:%.*]] = call i32 @capture_of_alloca(ptr readonly captures(none) [[RETVAL]])
 ; CHECK-NEXT:    ret i32 [[RDX_INC]]
 ;
 entry:
@@ -189,7 +189,7 @@ define i32 @alloca_not_captured_as_per_operand_attr_and_readonly_as_per_callbase
 ; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp ne i64 [[INDVARS_IV_NEXT]], [[N:%.*]]
 ; CHECK-NEXT:    br i1 [[EXITCOND]], label [[LOOP]], label [[EXIT:%.*]]
 ; CHECK:       exit:
-; CHECK-NEXT:    [[I0:%.*]] = call i32 @capture_of_alloca(ptr nocapture [[RETVAL]]) #[[ATTR2:[0-9]+]]
+; CHECK-NEXT:    [[I0:%.*]] = call i32 @capture_of_alloca(ptr captures(none) [[RETVAL]]) #[[ATTR2:[0-9]+]]
 ; CHECK-NEXT:    [[I1:%.*]] = load i32, ptr [[RETVAL]], align 4
 ; CHECK-NEXT:    ret i32 [[I1]]
 ;
diff --git a/llvm/test/Transforms/SampleProfile/pseudo-probe-callee-profile-mismatch.ll b/llvm/test/Transforms/SampleProfile/pseudo-probe-callee-profile-mismatch.ll
index 43be142e7cf98..dd86c7b81c5a3 100644
--- a/llvm/test/Transforms/SampleProfile/pseudo-probe-callee-profile-mismatch.ll
+++ b/llvm/test/Transforms/SampleProfile/pseudo-probe-callee-profile-mismatch.ll
@@ -1,6 +1,6 @@
 ; REQUIRES: x86_64-linux
 ; REQUIRES: asserts
-; RUN: opt < %s -passes='thinlto<O2>' -pgo-kind=pgo-sample-use-pipeline  -sample-profile-file=%S/Inputs/pseudo-probe-callee-profile-mismatch.prof --salvage-stale-profile -S --debug-only=sample-profile,sample-profile-matcher,sample-profile-impl  -pass-remarks=inline 2>&1 | FileCheck %s
+; RUN: opt < %s -passes='thinlto<O2>' -pgo-kind=pgo-sample-use-pipeline  -sample-profile-file=%S/Inputs/pseudo-probe-callee-profile-mismatch.prof --salvage-stale-profile --salvage-unused-profile=false -S --debug-only=sample-profile,sample-profile-matcher,sample-profile-impl  -pass-remarks=inline 2>&1 | FileCheck %s
 
 ; There is no profile-checksum-mismatch attr, even the checksum is mismatched in the pseudo_probe_desc, it doesn't run the matching.
 ; CHECK-NOT: Run stale profile matching for main
diff --git a/llvm/test/Transforms/SampleProfile/pseudo-probe-profile-mismatch-thinlto.ll b/llvm/test/Transforms/SampleProfile/pseudo-probe-profile-mismatch-thinlto.ll
index bf33df2481044..16616c39401b3 100644
--- a/llvm/test/Transforms/SampleProfile/pseudo-probe-profile-mismatch-thinlto.ll
+++ b/llvm/test/Transforms/SampleProfile/pseudo-probe-profile-mismatch-thinlto.ll
@@ -1,5 +1,5 @@
 ; REQUIRES: x86_64-linux
-; RUN: opt < %S/pseudo-probe-stale-profile-matching-lto.ll -passes='thinlto<O2>' -pgo-kind=pgo-sample-use-pipeline -sample-profile-file=%S/Inputs/pseudo-probe-stale-profile-matching-lto.prof -report-profile-staleness -persist-profile-staleness  -S 2>%t -o %t.ll
+; RUN: opt < %S/pseudo-probe-stale-profile-matching-lto.ll -passes='thinlto<O2>' -pgo-kind=pgo-sample-use-pipeline  --salvage-unused-profile=false -sample-profile-file=%S/Inputs/pseudo-probe-stale-profile-matching-lto.prof -report-profile-staleness -persist-profile-staleness  -S 2>%t -o %t.ll
 ; RUN: FileCheck %s --input-file %t
 ; RUN: FileCheck %s --input-file %t.ll -check-prefix=CHECK-MD
 
diff --git a/llvm/test/Transforms/SampleProfile/pseudo-probe-profile-mismatch.ll b/llvm/test/Transforms/SampleProfile/pseudo-probe-profile-mismatch.ll
index 6efd9f511e137..22317e60cd081 100644
--- a/llvm/test/Transforms/SampleProfile/pseudo-probe-profile-mismatch.ll
+++ b/llvm/test/Transforms/SampleProfile/pseudo-probe-profile-mismatch.ll
@@ -1,12 +1,12 @@
 ; REQUIRES: x86_64-linux
-; RUN: opt < %s -passes=sample-profile -sample-profile-file=%S/Inputs/pseudo-probe-profile-mismatch.prof -report-profile-staleness -persist-profile-staleness -S 2>%t -o %t.ll
+; RUN: opt < %s -passes=sample-profile -sample-profile-file=%S/Inputs/pseudo-probe-profile-mismatch.prof --salvage-unused-profile=false -report-profile-staleness -persist-profile-staleness -S 2>%t -o %t.ll
 ; RUN: FileCheck %s --input-file %t
 ; RUN: FileCheck %s --input-file %t.ll -check-prefix=CHECK-MD
 ; RUN: llc < %t.ll -filetype=obj -o %t.obj
 ; RUN: llvm-objdump --section-headers %t.obj | FileCheck %s --check-prefix=CHECK-OBJ
 ; RUN: llc < %t.ll -filetype=asm -o - | FileCheck %s --check-prefix=CHECK-ASM
 
-; RUN: opt < %s -passes=sample-profile -sample-profile-file=%S/Inputs/pseudo-probe-profile-mismatch-nested.prof -report-profile-staleness -persist-profile-staleness -S 2>&1 | FileCheck %s --check-prefix=CHECK-NESTED
+; RUN: opt < %s -passes=sample-profile -sample-profile-file=%S/Inputs/pseudo-probe-profile-mismatch-nested.prof --salvage-unused-profile=false -report-profile-staleness -persist-profile-staleness -S 2>&1 | FileCheck %s --check-prefix=CHECK-NESTED
 
 
 ; CHECK: (2/3) of functions' profile are invalid and (40/50) of samples are discarded due to function hash mismatch.
diff --git a/llvm/test/Transforms/SampleProfile/section-accurate-samplepgo.ll b/llvm/test/Transforms/SampleProfile/section-accurate-samplepgo.ll
index ef2ddbc33cee4..af4b875818f6f 100644
--- a/llvm/test/Transforms/SampleProfile/section-accurate-samplepgo.ll
+++ b/llvm/test/Transforms/SampleProfile/section-accurate-samplepgo.ll
@@ -36,11 +36,11 @@ attributes #1 = { "use-sample-profile" }
 
 ; CHECK: ![[NOPROFILE_ID]] = !{!"function_entry_count", i64 -1}
 ; CHECK: ![[ZERO_ID]] = !{!"function_entry_count", i64 0}
-; CHECK: ![[COLD_ID]] = !{!"function_section_prefix", !"unlikely"}
+; CHECK: ![[COLD_ID]] = !{!"section_prefix", !"unlikely"}
 ; UNKNOWN: ![[NOPROFILE_ID]] = !{!"function_entry_count", i64 -1}
-; UNKNOWN: ![[UNKNOWN_ID]] = !{!"function_section_prefix", !"unknown"}
+; UNKNOWN: ![[UNKNOWN_ID]] = !{!"section_prefix", !"unknown"}
 ; ACCURATE: ![[ZERO_ID]] = !{!"function_entry_count", i64 0}
-; ACCURATE: ![[COLD_ID]] = !{!"function_section_prefix", !"unlikely"}
+; ACCURATE: ![[COLD_ID]] = !{!"section_prefix", !"unlikely"}
 !llvm.module.flags = !{!1}
 !1 = !{i32 1, !"ProfileSummary", !2}
 !2 = !{!3, !4, !5, !6, !7, !8, !9, !10}
diff --git a/llvm/test/Transforms/SandboxVectorizer/X86/lit.local.cfg b/llvm/test/Transforms/SandboxVectorizer/X86/lit.local.cfg
new file mode 100644
index 0000000000000..42bf50dcc13c3
--- /dev/null
+++ b/llvm/test/Transforms/SandboxVectorizer/X86/lit.local.cfg
@@ -0,0 +1,2 @@
+if not "X86" in config.root.targets:
+    config.unsupported = True
diff --git a/llvm/test/Transforms/SandboxVectorizer/X86/simple_cost_test.ll b/llvm/test/Transforms/SandboxVectorizer/X86/simple_cost_test.ll
new file mode 100644
index 0000000000000..f1df52bd88ad7
--- /dev/null
+++ b/llvm/test/Transforms/SandboxVectorizer/X86/simple_cost_test.ll
@@ -0,0 +1,91 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -passes=sandbox-vectorizer -mtriple=x86_64-- -mattr=+sse4.1 %s -S -sbvec-cost-threshold=0 | FileCheck %s --check-prefix=THRESHOLD_0
+; RUN: opt -passes=sandbox-vectorizer -mtriple=x86_64-- -mattr=+sse4.1 %s -S -sbvec-cost-threshold=99 | FileCheck %s --check-prefix=THRESHOLD_99
+
+define void @simple_cost_test(ptr %ptr) {
+; THRESHOLD_0-LABEL: define void @simple_cost_test(
+; THRESHOLD_0-SAME: ptr [[PTR:%.*]]) #[[ATTR0:[0-9]+]] {
+; THRESHOLD_0-NEXT:    [[PTR0:%.*]] = getelementptr double, ptr [[PTR]], i32 0
+; THRESHOLD_0-NEXT:    [[VECL:%.*]] = load <2 x double>, ptr [[PTR0]], align 8, !sandboxvec [[META0:![0-9]+]]
+; THRESHOLD_0-NEXT:    store <2 x double> [[VECL]], ptr [[PTR0]], align 8, !sandboxvec [[META0]]
+; THRESHOLD_0-NEXT:    ret void
+;
+; THRESHOLD_99-LABEL: define void @simple_cost_test(
+; THRESHOLD_99-SAME: ptr [[PTR:%.*]]) #[[ATTR0:[0-9]+]] {
+; THRESHOLD_99-NEXT:    [[PTR0:%.*]] = getelementptr double, ptr [[PTR]], i32 0
+; THRESHOLD_99-NEXT:    [[PTR1:%.*]] = getelementptr double, ptr [[PTR]], i32 1, !sandboxvec [[META0:![0-9]+]]
+; THRESHOLD_99-NEXT:    [[LD0:%.*]] = load double, ptr [[PTR0]], align 8, !sandboxvec [[META0]]
+; THRESHOLD_99-NEXT:    [[LD1:%.*]] = load double, ptr [[PTR1]], align 8, !sandboxvec [[META0]]
+; THRESHOLD_99-NEXT:    store double [[LD0]], ptr [[PTR0]], align 8, !sandboxvec [[META0]]
+; THRESHOLD_99-NEXT:    store double [[LD1]], ptr [[PTR1]], align 8, !sandboxvec [[META0]]
+; THRESHOLD_99-NEXT:    ret void
+;
+  %ptr0 = getelementptr double, ptr %ptr, i32 0
+  %ptr1 = getelementptr double, ptr %ptr, i32 1
+  %ld0 = load double, ptr %ptr0
+  %ld1 = load double, ptr %ptr1
+  store double %ld0, ptr %ptr0
+  store double %ld1, ptr %ptr1
+  ret void
+}
+
+define void @pack_cost_test_(ptr %ptr) {
+; THRESHOLD_0-LABEL: define void @pack_cost_test_(
+; THRESHOLD_0-SAME: ptr [[PTR:%.*]]) #[[ATTR0]] {
+; THRESHOLD_0-NEXT:    [[PTR0:%.*]] = getelementptr float, ptr [[PTR]], i32 0
+; THRESHOLD_0-NEXT:    [[PTR1:%.*]] = getelementptr float, ptr [[PTR]], i32 1
+; THRESHOLD_0-NEXT:    [[LD0:%.*]] = load float, ptr [[PTR0]], align 4
+; THRESHOLD_0-NEXT:    [[LD1:%.*]] = load float, ptr [[PTR1]], align 4
+; THRESHOLD_0-NEXT:    [[PACK4:%.*]] = insertelement <4 x float> poison, float [[LD0]], i32 0, !sandboxvec [[META1:![0-9]+]]
+; THRESHOLD_0-NEXT:    [[PACK5:%.*]] = insertelement <4 x float> [[PACK4]], float [[LD1]], i32 1, !sandboxvec [[META1]]
+; THRESHOLD_0-NEXT:    [[PACK6:%.*]] = insertelement <4 x float> [[PACK5]], float [[LD0]], i32 2, !sandboxvec [[META1]]
+; THRESHOLD_0-NEXT:    [[PACK7:%.*]] = insertelement <4 x float> [[PACK6]], float [[LD1]], i32 3, !sandboxvec [[META1]]
+; THRESHOLD_0-NEXT:    [[PACK:%.*]] = insertelement <4 x float> poison, float [[LD0]], i32 0, !sandboxvec [[META1]]
+; THRESHOLD_0-NEXT:    [[PACK1:%.*]] = insertelement <4 x float> [[PACK]], float [[LD1]], i32 1, !sandboxvec [[META1]]
+; THRESHOLD_0-NEXT:    [[PACK2:%.*]] = insertelement <4 x float> [[PACK1]], float [[LD0]], i32 2, !sandboxvec [[META1]]
+; THRESHOLD_0-NEXT:    [[PACK3:%.*]] = insertelement <4 x float> [[PACK2]], float [[LD1]], i32 3, !sandboxvec [[META1]]
+; THRESHOLD_0-NEXT:    [[VEC:%.*]] = fmul <4 x float> [[PACK3]], [[PACK7]], !sandboxvec [[META1]]
+; THRESHOLD_0-NEXT:    store <4 x float> [[VEC]], ptr [[PTR0]], align 4, !sandboxvec [[META1]]
+; THRESHOLD_0-NEXT:    ret void
+;
+; THRESHOLD_99-LABEL: define void @pack_cost_test_(
+; THRESHOLD_99-SAME: ptr [[PTR:%.*]]) #[[ATTR0]] {
+; THRESHOLD_99-NEXT:    [[PTR0:%.*]] = getelementptr float, ptr [[PTR]], i32 0
+; THRESHOLD_99-NEXT:    [[PTR1:%.*]] = getelementptr float, ptr [[PTR]], i32 1
+; THRESHOLD_99-NEXT:    [[PTR2:%.*]] = getelementptr float, ptr [[PTR]], i32 2, !sandboxvec [[META1:![0-9]+]]
+; THRESHOLD_99-NEXT:    [[PTR3:%.*]] = getelementptr float, ptr [[PTR]], i32 3, !sandboxvec [[META1]]
+; THRESHOLD_99-NEXT:    [[LD0:%.*]] = load float, ptr [[PTR0]], align 4
+; THRESHOLD_99-NEXT:    [[LD1:%.*]] = load float, ptr [[PTR1]], align 4
+; THRESHOLD_99-NEXT:    [[MUL0:%.*]] = fmul float [[LD0]], [[LD0]], !sandboxvec [[META1]]
+; THRESHOLD_99-NEXT:    [[MUL1:%.*]] = fmul float [[LD1]], [[LD1]], !sandboxvec [[META1]]
+; THRESHOLD_99-NEXT:    [[MUL2:%.*]] = fmul float [[LD0]], [[LD0]], !sandboxvec [[META1]]
+; THRESHOLD_99-NEXT:    [[MUL3:%.*]] = fmul float [[LD1]], [[LD1]], !sandboxvec [[META1]]
+; THRESHOLD_99-NEXT:    store float [[MUL0]], ptr [[PTR0]], align 4, !sandboxvec [[META1]]
+; THRESHOLD_99-NEXT:    store float [[MUL1]], ptr [[PTR1]], align 4, !sandboxvec [[META1]]
+; THRESHOLD_99-NEXT:    store float [[MUL2]], ptr [[PTR2]], align 4, !sandboxvec [[META1]]
+; THRESHOLD_99-NEXT:    store float [[MUL3]], ptr [[PTR3]], align 4, !sandboxvec [[META1]]
+; THRESHOLD_99-NEXT:    ret void
+;
+  %ptr0 = getelementptr float, ptr %ptr, i32 0
+  %ptr1 = getelementptr float, ptr %ptr, i32 1
+  %ptr2 = getelementptr float, ptr %ptr, i32 2
+  %ptr3 = getelementptr float, ptr %ptr, i32 3
+  %ld0 = load float, ptr %ptr0
+  %ld1 = load float, ptr %ptr1
+  %mul0 = fmul float %ld0, %ld0
+  %mul1 = fmul float %ld1, %ld1
+  %mul2 = fmul float %ld0, %ld0
+  %mul3 = fmul float %ld1, %ld1
+  store float %mul0, ptr %ptr0
+  store float %mul1, ptr %ptr1
+  store float %mul2, ptr %ptr2
+  store float %mul3, ptr %ptr3
+  ret void
+}
+;.
+; THRESHOLD_0: [[META0]] = distinct !{!"sandboxregion"}
+; THRESHOLD_0: [[META1]] = distinct !{!"sandboxregion"}
+;.
+; THRESHOLD_99: [[META0]] = distinct !{!"sandboxregion"}
+; THRESHOLD_99: [[META1]] = distinct !{!"sandboxregion"}
+;.
diff --git a/llvm/test/Transforms/SandboxVectorizer/bottomup_basic.ll b/llvm/test/Transforms/SandboxVectorizer/bottomup_basic.ll
index ee5a3a514b3c5..ee8592c04b62c 100644
--- a/llvm/test/Transforms/SandboxVectorizer/bottomup_basic.ll
+++ b/llvm/test/Transforms/SandboxVectorizer/bottomup_basic.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
-; RUN: opt -passes=sandbox-vectorizer -sbvec-vec-reg-bits=1024 -sbvec-allow-non-pow2 -sbvec-passes="bottom-up-vec<>" %s -S | FileCheck %s
+; RUN: opt -passes=sandbox-vectorizer -sbvec-vec-reg-bits=1024 -sbvec-allow-non-pow2 -sbvec-passes="bottom-up-vec<tr-accept>" %s -S | FileCheck %s
 
 define void @store_load(ptr %ptr) {
 ; CHECK-LABEL: define void @store_load(
diff --git a/llvm/test/Transforms/SandboxVectorizer/bottomup_seed_slice.ll b/llvm/test/Transforms/SandboxVectorizer/bottomup_seed_slice.ll
index 8459c3addaa83..202b5a6fbd6c9 100644
--- a/llvm/test/Transforms/SandboxVectorizer/bottomup_seed_slice.ll
+++ b/llvm/test/Transforms/SandboxVectorizer/bottomup_seed_slice.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
-; RUN: opt -passes=sandbox-vectorizer -sbvec-vec-reg-bits=1024 -sbvec-allow-non-pow2 -sbvec-passes="bottom-up-vec<>" %s -S | FileCheck %s
+; RUN: opt -passes=sandbox-vectorizer -sbvec-vec-reg-bits=1024 -sbvec-allow-non-pow2 -sbvec-passes="bottom-up-vec<tr-accept>" %s -S | FileCheck %s
 
 
 declare void @foo()
diff --git a/llvm/test/Transforms/SandboxVectorizer/bottomup_seed_slice_pow2.ll b/llvm/test/Transforms/SandboxVectorizer/bottomup_seed_slice_pow2.ll
index e186d5fa86e4a..f1c6e3297d79c 100644
--- a/llvm/test/Transforms/SandboxVectorizer/bottomup_seed_slice_pow2.ll
+++ b/llvm/test/Transforms/SandboxVectorizer/bottomup_seed_slice_pow2.ll
@@ -1,6 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
-; RUN: opt -passes=sandbox-vectorizer -sbvec-vec-reg-bits=1024 -sbvec-allow-non-pow2=false -sbvec-passes="bottom-up-vec<>" %s -S | FileCheck %s --check-prefix=POW2
-; RUN: opt -passes=sandbox-vectorizer -sbvec-vec-reg-bits=1024 -sbvec-allow-non-pow2=true -sbvec-passes="bottom-up-vec<>" %s -S | FileCheck %s --check-prefix=NON-POW2
+; RUN: opt -passes=sandbox-vectorizer -sbvec-vec-reg-bits=1024 -sbvec-allow-non-pow2=false -sbvec-passes="bottom-up-vec<tr-accept>" %s -S | FileCheck %s --check-prefix=POW2
+; RUN: opt -passes=sandbox-vectorizer -sbvec-vec-reg-bits=1024 -sbvec-allow-non-pow2=true -sbvec-passes="bottom-up-vec<tr-accept>" %s -S | FileCheck %s --check-prefix=NON-POW2
 
 define void @pow2(ptr %ptr, float %val) {
 ; POW2-LABEL: define void @pow2(
diff --git a/llvm/test/Transforms/SandboxVectorizer/cross_bbs.ll b/llvm/test/Transforms/SandboxVectorizer/cross_bbs.ll
index e913fc5913ba7..ff1604173c317 100644
--- a/llvm/test/Transforms/SandboxVectorizer/cross_bbs.ll
+++ b/llvm/test/Transforms/SandboxVectorizer/cross_bbs.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
-; RUN: opt -passes=sandbox-vectorizer -sbvec-vec-reg-bits=1024 -sbvec-allow-non-pow2 -sbvec-passes="bottom-up-vec<>" %s -S | FileCheck %s
+; RUN: opt -passes=sandbox-vectorizer -sbvec-vec-reg-bits=1024 -sbvec-allow-non-pow2 -sbvec-passes="bottom-up-vec<tr-accept>" %s -S | FileCheck %s
 
 define void @cross_bbs(ptr %ptr) {
 ; CHECK-LABEL: define void @cross_bbs(
@@ -8,10 +8,10 @@ define void @cross_bbs(ptr %ptr) {
 ; CHECK-NEXT:    [[PTR1:%.*]] = getelementptr i8, ptr [[PTR]], i32 1
 ; CHECK-NEXT:    [[L0:%.*]] = load i8, ptr [[PTR0]], align 1
 ; CHECK-NEXT:    [[L1:%.*]] = load i8, ptr [[PTR1]], align 1
-; CHECK-NEXT:    [[PACK:%.*]] = insertelement <2 x i8> poison, i8 [[L0]], i32 0
-; CHECK-NEXT:    [[PACK1:%.*]] = insertelement <2 x i8> [[PACK]], i8 [[L1]], i32 1
 ; CHECK-NEXT:    br label %[[BB:.*]]
 ; CHECK:       [[BB]]:
+; CHECK-NEXT:    [[PACK:%.*]] = insertelement <2 x i8> poison, i8 [[L0]], i32 0
+; CHECK-NEXT:    [[PACK1:%.*]] = insertelement <2 x i8> [[PACK]], i8 [[L1]], i32 1
 ; CHECK-NEXT:    store <2 x i8> [[PACK1]], ptr [[PTR0]], align 1
 ; CHECK-NEXT:    ret void
 ;
diff --git a/llvm/test/Transforms/SandboxVectorizer/default_pass_pipeline.ll b/llvm/test/Transforms/SandboxVectorizer/default_pass_pipeline.ll
index 1d7be43336c87..10de4338caf23 100644
--- a/llvm/test/Transforms/SandboxVectorizer/default_pass_pipeline.ll
+++ b/llvm/test/Transforms/SandboxVectorizer/default_pass_pipeline.ll
@@ -4,8 +4,10 @@
 
 ; This checks the default pass pipeline for the sandbox vectorizer.
 define void @pipeline() {
+; CHECK: fpm
 ; CHECK: bottom-up-vec
 ; CHECK: rpm
+; CHECK: tr-accept-or-revert
 ; CHECK-EMPTY:
   ret void
 }
diff --git a/llvm/test/Transforms/SandboxVectorizer/pack.ll b/llvm/test/Transforms/SandboxVectorizer/pack.ll
index 373ab743fb890..da41036e3a58b 100644
--- a/llvm/test/Transforms/SandboxVectorizer/pack.ll
+++ b/llvm/test/Transforms/SandboxVectorizer/pack.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
-; RUN: opt -passes=sandbox-vectorizer -sbvec-vec-reg-bits=1024 -sbvec-allow-non-pow2 -sbvec-passes="bottom-up-vec<>" %s -S | FileCheck %s
+; RUN: opt -passes=sandbox-vectorizer -sbvec-vec-reg-bits=1024 -sbvec-allow-non-pow2 -sbvec-passes="bottom-up-vec<tr-accept>" %s -S | FileCheck %s
 
 define void @pack_constants(ptr %ptr) {
 ; CHECK-LABEL: define void @pack_constants(
@@ -59,12 +59,12 @@ define void @packFromOtherBB(ptr %ptr, i8 %val) {
 ; CHECK-NEXT:  [[ENTRY:.*]]:
 ; CHECK-NEXT:    [[ADD0:%.*]] = add i8 [[VAL]], 0
 ; CHECK-NEXT:    [[MUL1:%.*]] = mul i8 [[VAL]], 1
-; CHECK-NEXT:    [[PACK:%.*]] = insertelement <2 x i8> poison, i8 [[ADD0]], i32 0
-; CHECK-NEXT:    [[PACK1:%.*]] = insertelement <2 x i8> [[PACK]], i8 [[MUL1]], i32 1
 ; CHECK-NEXT:    br label %[[LOOP:.*]]
 ; CHECK:       [[LOOP]]:
 ; CHECK-NEXT:    [[PHI0:%.*]] = phi i8 [ 0, %[[ENTRY]] ], [ 1, %[[LOOP]] ]
 ; CHECK-NEXT:    [[PHI1:%.*]] = phi i8 [ 0, %[[ENTRY]] ], [ 1, %[[LOOP]] ]
+; CHECK-NEXT:    [[PACK:%.*]] = insertelement <2 x i8> poison, i8 [[ADD0]], i32 0
+; CHECK-NEXT:    [[PACK1:%.*]] = insertelement <2 x i8> [[PACK]], i8 [[MUL1]], i32 1
 ; CHECK-NEXT:    [[GEP0:%.*]] = getelementptr i8, ptr [[PTR]], i64 0
 ; CHECK-NEXT:    store <2 x i8> [[PACK1]], ptr [[GEP0]], align 1
 ; CHECK-NEXT:    br label %[[LOOP]]
@@ -88,3 +88,30 @@ loop:
 exit:
   ret void
 }
+
+define void @packFromDiffBBs(ptr %ptr, i8 %v) {
+; CHECK-LABEL: define void @packFromDiffBBs(
+; CHECK-SAME: ptr [[PTR:%.*]], i8 [[V:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[ADD0:%.*]] = add i8 [[V]], 1
+; CHECK-NEXT:    br label %[[BB:.*]]
+; CHECK:       [[BB]]:
+; CHECK-NEXT:    [[ADD1:%.*]] = add i8 [[V]], 2
+; CHECK-NEXT:    [[PACK:%.*]] = insertelement <2 x i8> poison, i8 [[ADD0]], i32 0
+; CHECK-NEXT:    [[PACK1:%.*]] = insertelement <2 x i8> [[PACK]], i8 [[ADD1]], i32 1
+; CHECK-NEXT:    [[GEP0:%.*]] = getelementptr i8, ptr [[PTR]], i64 0
+; CHECK-NEXT:    store <2 x i8> [[PACK1]], ptr [[GEP0]], align 1
+; CHECK-NEXT:    ret void
+;
+entry:
+  %add0 = add i8 %v, 1
+  br label %bb
+
+bb:
+  %add1 = add i8 %v, 2
+  %gep0 = getelementptr i8, ptr %ptr, i64 0
+  %gep1 = getelementptr i8, ptr %ptr, i64 1
+  store i8 %add0, ptr %gep0
+  store i8 %add1, ptr %gep1
+  ret void
+}
diff --git a/llvm/test/Transforms/SandboxVectorizer/repeated_instrs.ll b/llvm/test/Transforms/SandboxVectorizer/repeated_instrs.ll
new file mode 100644
index 0000000000000..25d9d79154d35
--- /dev/null
+++ b/llvm/test/Transforms/SandboxVectorizer/repeated_instrs.ll
@@ -0,0 +1,65 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -passes=sandbox-vectorizer -sbvec-vec-reg-bits=1024 -sbvec-allow-non-pow2 -sbvec-passes="bottom-up-vec<tr-accept>" %s -S | FileCheck %s
+
+define i32 @repeated_splat(ptr %ptr, i32 %v) #0 {
+; CHECK-LABEL: define i32 @repeated_splat(
+; CHECK-SAME: ptr [[PTR:%.*]], i32 [[V:%.*]]) {
+; CHECK-NEXT:    [[GEP0:%.*]] = getelementptr inbounds i32, ptr [[PTR]], i64 0
+; CHECK-NEXT:    [[VECL:%.*]] = load <2 x i32>, ptr [[GEP0]], align 4
+; CHECK-NEXT:    [[SPLAT:%.*]] = add i32 [[V]], 0
+; CHECK-NEXT:    [[PACK:%.*]] = insertelement <2 x i32> poison, i32 [[SPLAT]], i32 0
+; CHECK-NEXT:    [[PACK1:%.*]] = insertelement <2 x i32> [[PACK]], i32 [[SPLAT]], i32 1
+; CHECK-NEXT:    [[VEC:%.*]] = mul <2 x i32> [[VECL]], [[PACK1]]
+; CHECK-NEXT:    store <2 x i32> [[VEC]], ptr [[GEP0]], align 4
+; CHECK-NEXT:    ret i32 0
+;
+  %gep0 = getelementptr inbounds i32, ptr %ptr, i64 0
+  %gep1 = getelementptr inbounds i32, ptr %ptr, i64 1
+  %ld0 = load i32, ptr %gep0, align 4
+  %ld1 = load i32, ptr %gep1, align 4
+  %splat = add i32 %v, 0
+  %add0 = mul i32 %ld0, %splat
+  %add1 = mul i32 %ld1, %splat
+  store i32 %add0, ptr %gep0, align 4
+  store i32 %add1, ptr %gep1, align 4
+  ret i32 0
+}
+
+define i32 @repeated_partial(ptr %ptr, i32 %v) #0 {
+; CHECK-LABEL: define i32 @repeated_partial(
+; CHECK-SAME: ptr [[PTR:%.*]], i32 [[V:%.*]]) {
+; CHECK-NEXT:    [[GEP0:%.*]] = getelementptr inbounds i32, ptr [[PTR]], i64 0
+; CHECK-NEXT:    [[GEP1:%.*]] = getelementptr inbounds i32, ptr [[PTR]], i64 1
+; CHECK-NEXT:    [[GEP3:%.*]] = getelementptr inbounds i32, ptr [[PTR]], i64 3
+; CHECK-NEXT:    [[LD0:%.*]] = load i32, ptr [[GEP0]], align 4
+; CHECK-NEXT:    [[LD1:%.*]] = load i32, ptr [[GEP1]], align 4
+; CHECK-NEXT:    [[LD3:%.*]] = load i32, ptr [[GEP3]], align 4
+; CHECK-NEXT:    [[PACK:%.*]] = insertelement <4 x i32> poison, i32 [[LD0]], i32 0
+; CHECK-NEXT:    [[PACK1:%.*]] = insertelement <4 x i32> [[PACK]], i32 [[LD1]], i32 1
+; CHECK-NEXT:    [[PACK2:%.*]] = insertelement <4 x i32> [[PACK1]], i32 [[LD1]], i32 2
+; CHECK-NEXT:    [[PACK3:%.*]] = insertelement <4 x i32> [[PACK2]], i32 [[LD3]], i32 3
+; CHECK-NEXT:    [[VECL:%.*]] = load <4 x i32>, ptr [[GEP0]], align 4
+; CHECK-NEXT:    [[SPLAT:%.*]] = add i32 [[V]], 0
+; CHECK-NEXT:    [[VEC:%.*]] = mul <4 x i32> [[VECL]], [[PACK3]]
+; CHECK-NEXT:    store <4 x i32> [[VEC]], ptr [[GEP0]], align 4
+; CHECK-NEXT:    ret i32 0
+;
+  %gep0 = getelementptr inbounds i32, ptr %ptr, i64 0
+  %gep1 = getelementptr inbounds i32, ptr %ptr, i64 1
+  %gep2 = getelementptr inbounds i32, ptr %ptr, i64 2
+  %gep3 = getelementptr inbounds i32, ptr %ptr, i64 3
+  %ld0 = load i32, ptr %gep0, align 4
+  %ld1 = load i32, ptr %gep1, align 4
+  %ld2 = load i32, ptr %gep2, align 4
+  %ld3 = load i32, ptr %gep3, align 4
+  %splat = add i32 %v, 0
+  %add0 = mul i32 %ld0, %ld0
+  %add1 = mul i32 %ld1, %ld1
+  %add2 = mul i32 %ld2, %ld1
+  %add3 = mul i32 %ld3, %ld3
+  store i32 %add0, ptr %gep0, align 4
+  store i32 %add1, ptr %gep1, align 4
+  store i32 %add2, ptr %gep2, align 4
+  store i32 %add3, ptr %gep3, align 4
+  ret i32 0
+}
diff --git a/llvm/test/Transforms/SandboxVectorizer/scheduler.ll b/llvm/test/Transforms/SandboxVectorizer/scheduler.ll
new file mode 100644
index 0000000000000..92a78a979192b
--- /dev/null
+++ b/llvm/test/Transforms/SandboxVectorizer/scheduler.ll
@@ -0,0 +1,51 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -passes=sandbox-vectorizer -sbvec-vec-reg-bits=1024 -sbvec-allow-non-pow2 -sbvec-passes="bottom-up-vec<tr-accept>" %s -S | FileCheck %s
+
+; This used to crash because the newly added pack   instructions would not update
+; the DAG and scheduler, leading to def-after-use.
+define void @check_dag_scheduler_update(ptr noalias %p, ptr noalias %p1) {
+; CHECK-LABEL: define void @check_dag_scheduler_update(
+; CHECK-SAME: ptr noalias [[P:%.*]], ptr noalias [[P1:%.*]]) {
+; CHECK-NEXT:    [[I:%.*]] = load i32, ptr [[P]], align 4
+; CHECK-NEXT:    [[ARRAYIDX4:%.*]] = getelementptr i32, ptr [[P]], i64 32
+; CHECK-NEXT:    [[I2:%.*]] = load i32, ptr [[ARRAYIDX4]], align 4
+; CHECK-NEXT:    [[ARRAYIDX11:%.*]] = getelementptr i32, ptr [[P]], i64 33
+; CHECK-NEXT:    [[I4:%.*]] = load i32, ptr [[ARRAYIDX11]], align 4
+; CHECK-NEXT:    [[ARRAYIDX18:%.*]] = getelementptr i32, ptr [[P]], i64 34
+; CHECK-NEXT:    [[I6:%.*]] = load i32, ptr [[ARRAYIDX18]], align 4
+; CHECK-NEXT:    [[PACK:%.*]] = insertelement <4 x i32> poison, i32 [[I]], i32 0
+; CHECK-NEXT:    [[PACK1:%.*]] = insertelement <4 x i32> [[PACK]], i32 [[I2]], i32 1
+; CHECK-NEXT:    [[PACK2:%.*]] = insertelement <4 x i32> [[PACK1]], i32 [[I4]], i32 2
+; CHECK-NEXT:    [[PACK3:%.*]] = insertelement <4 x i32> [[PACK2]], i32 [[I6]], i32 3
+; CHECK-NEXT:    [[VECL:%.*]] = load <4 x i32>, ptr [[P]], align 4
+; CHECK-NEXT:    [[VEC:%.*]] = add nsw <4 x i32> [[PACK3]], [[VECL]]
+; CHECK-NEXT:    store <4 x i32> [[VEC]], ptr [[P1]], align 4
+; CHECK-NEXT:    ret void
+;
+  %i = load i32, ptr %p
+  %i1 = load i32, ptr %p
+  %add = add nsw i32 %i, %i1
+  store i32 %add, ptr %p1
+  %arrayidx4 = getelementptr i32, ptr %p, i64 32
+  %i2 = load i32, ptr %arrayidx4
+  %arrayidx6 = getelementptr i32, ptr %p, i64 1
+  %i3 = load i32, ptr %arrayidx6
+  %add7 = add nsw i32 %i2, %i3
+  %arrayidx9 = getelementptr i32, ptr %p1, i64 1
+  store i32 %add7, ptr %arrayidx9
+  %arrayidx11 = getelementptr i32, ptr %p, i64 33
+  %i4 = load i32, ptr %arrayidx11
+  %arrayidx13 = getelementptr i32, ptr %p, i64 2
+  %i5 = load i32, ptr %arrayidx13
+  %add14 = add nsw i32 %i4, %i5
+  %arrayidx16 = getelementptr i32, ptr %p1, i64 2
+  store i32 %add14, ptr %arrayidx16
+  %arrayidx18 = getelementptr i32, ptr %p, i64 34
+  %i6 = load i32, ptr %arrayidx18
+  %arrayidx19 = getelementptr i32, ptr %p, i64 3
+  %i7 = load i32, ptr %arrayidx19
+  %add21 = add nsw i32 %i6, %i7
+  %arrayidx23 = getelementptr i32, ptr %p1, i64 3
+  store i32 %add21, ptr %arrayidx23
+  ret void
+}
diff --git a/llvm/test/Transforms/SandboxVectorizer/special_opcodes.ll b/llvm/test/Transforms/SandboxVectorizer/special_opcodes.ll
new file mode 100644
index 0000000000000..e8fe8b4fa88e3
--- /dev/null
+++ b/llvm/test/Transforms/SandboxVectorizer/special_opcodes.ll
@@ -0,0 +1,97 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -passes=sandbox-vectorizer -sbvec-vec-reg-bits=1024 -sbvec-allow-non-pow2 -sbvec-passes="bottom-up-vec<tr-accept>" %s -S | FileCheck %s
+
+; This file includes tests for opcodes that need special checks.
+
+; TODO: Selects with conditions of diff number of lanes than the instruction itself need special treatment.
+define void @selects_with_diff_cond_lanes(ptr %ptr, i1 %cond0, i1 %cond1, <2 x i8> %op0, <2 x i8> %op1) {
+; CHECK-LABEL: define void @selects_with_diff_cond_lanes(
+; CHECK-SAME: ptr [[PTR:%.*]], i1 [[COND0:%.*]], i1 [[COND1:%.*]], <2 x i8> [[OP0:%.*]], <2 x i8> [[OP1:%.*]]) {
+; CHECK-NEXT:    [[PTR0:%.*]] = getelementptr <2 x i8>, ptr [[PTR]], i32 0
+; CHECK-NEXT:    [[PTR1:%.*]] = getelementptr <2 x i8>, ptr [[PTR]], i32 1
+; CHECK-NEXT:    [[LD0:%.*]] = load <2 x i8>, ptr [[PTR0]], align 2
+; CHECK-NEXT:    [[LD1:%.*]] = load <2 x i8>, ptr [[PTR1]], align 2
+; CHECK-NEXT:    [[SEL0:%.*]] = select i1 [[COND0]], <2 x i8> [[LD0]], <2 x i8> [[LD0]]
+; CHECK-NEXT:    [[SEL1:%.*]] = select i1 [[COND1]], <2 x i8> [[LD1]], <2 x i8> [[LD1]]
+; CHECK-NEXT:    [[VPACK:%.*]] = extractelement <2 x i8> [[SEL0]], i32 0
+; CHECK-NEXT:    [[VPACK1:%.*]] = insertelement <4 x i8> poison, i8 [[VPACK]], i32 0
+; CHECK-NEXT:    [[VPACK2:%.*]] = extractelement <2 x i8> [[SEL0]], i32 1
+; CHECK-NEXT:    [[VPACK3:%.*]] = insertelement <4 x i8> [[VPACK1]], i8 [[VPACK2]], i32 1
+; CHECK-NEXT:    [[VPACK4:%.*]] = extractelement <2 x i8> [[SEL1]], i32 0
+; CHECK-NEXT:    [[VPACK5:%.*]] = insertelement <4 x i8> [[VPACK3]], i8 [[VPACK4]], i32 2
+; CHECK-NEXT:    [[VPACK6:%.*]] = extractelement <2 x i8> [[SEL1]], i32 1
+; CHECK-NEXT:    [[VPACK7:%.*]] = insertelement <4 x i8> [[VPACK5]], i8 [[VPACK6]], i32 3
+; CHECK-NEXT:    store <4 x i8> [[VPACK7]], ptr [[PTR0]], align 2
+; CHECK-NEXT:    ret void
+;
+  %ptr0 = getelementptr <2 x i8>, ptr %ptr, i32 0
+  %ptr1 = getelementptr <2 x i8>, ptr %ptr, i32 1
+  %ld0 = load <2 x i8>, ptr %ptr0
+  %ld1 = load <2 x i8>, ptr %ptr1
+  %sel0 = select i1 %cond0, <2 x i8> %ld0, <2 x i8> %ld0
+  %sel1 = select i1 %cond1, <2 x i8> %ld1, <2 x i8> %ld1
+  store <2 x i8> %sel0, ptr %ptr0
+  store <2 x i8> %sel1, ptr %ptr1
+  ret void
+}
+
+; TODO: Selects that share the same condition need special treatment.
+define void @selects_with_common_condition_but_diff_lanes(ptr %ptr, i1 %cond, <2 x i8> %op0, <2 x i8> %op1) {
+; CHECK-LABEL: define void @selects_with_common_condition_but_diff_lanes(
+; CHECK-SAME: ptr [[PTR:%.*]], i1 [[COND:%.*]], <2 x i8> [[OP0:%.*]], <2 x i8> [[OP1:%.*]]) {
+; CHECK-NEXT:    [[PTR0:%.*]] = getelementptr <2 x i8>, ptr [[PTR]], i32 0
+; CHECK-NEXT:    [[PTR1:%.*]] = getelementptr <2 x i8>, ptr [[PTR]], i32 1
+; CHECK-NEXT:    [[LD0:%.*]] = load <2 x i8>, ptr [[PTR0]], align 2
+; CHECK-NEXT:    [[LD1:%.*]] = load <2 x i8>, ptr [[PTR1]], align 2
+; CHECK-NEXT:    [[SEL0:%.*]] = select i1 [[COND]], <2 x i8> [[LD0]], <2 x i8> [[LD0]]
+; CHECK-NEXT:    [[SEL1:%.*]] = select i1 [[COND]], <2 x i8> [[LD1]], <2 x i8> [[LD1]]
+; CHECK-NEXT:    [[VPACK:%.*]] = extractelement <2 x i8> [[SEL0]], i32 0
+; CHECK-NEXT:    [[VPACK1:%.*]] = insertelement <4 x i8> poison, i8 [[VPACK]], i32 0
+; CHECK-NEXT:    [[VPACK2:%.*]] = extractelement <2 x i8> [[SEL0]], i32 1
+; CHECK-NEXT:    [[VPACK3:%.*]] = insertelement <4 x i8> [[VPACK1]], i8 [[VPACK2]], i32 1
+; CHECK-NEXT:    [[VPACK4:%.*]] = extractelement <2 x i8> [[SEL1]], i32 0
+; CHECK-NEXT:    [[VPACK5:%.*]] = insertelement <4 x i8> [[VPACK3]], i8 [[VPACK4]], i32 2
+; CHECK-NEXT:    [[VPACK6:%.*]] = extractelement <2 x i8> [[SEL1]], i32 1
+; CHECK-NEXT:    [[VPACK7:%.*]] = insertelement <4 x i8> [[VPACK5]], i8 [[VPACK6]], i32 3
+; CHECK-NEXT:    store <4 x i8> [[VPACK7]], ptr [[PTR0]], align 2
+; CHECK-NEXT:    ret void
+;
+  %ptr0 = getelementptr <2 x i8>, ptr %ptr, i32 0
+  %ptr1 = getelementptr <2 x i8>, ptr %ptr, i32 1
+  %ld0 = load <2 x i8>, ptr %ptr0
+  %ld1 = load <2 x i8>, ptr %ptr1
+  %sel0 = select i1 %cond, <2 x i8> %ld0, <2 x i8> %ld0
+  %sel1 = select i1 %cond, <2 x i8> %ld1, <2 x i8> %ld1
+  store <2 x i8> %sel0, ptr %ptr0
+  store <2 x i8> %sel1, ptr %ptr1
+  ret void
+}
+
+; Selects with conditions of the same number of lanes as the instruction itself be vectorized as usual.
+define void @selects_same_cond_lanes(ptr %ptr, <2 x i1> %cond0, <2 x i1> %cond1, <2 x i8> %op0, <2 x i8> %op1) {
+; CHECK-LABEL: define void @selects_same_cond_lanes(
+; CHECK-SAME: ptr [[PTR:%.*]], <2 x i1> [[COND0:%.*]], <2 x i1> [[COND1:%.*]], <2 x i8> [[OP0:%.*]], <2 x i8> [[OP1:%.*]]) {
+; CHECK-NEXT:    [[PTR0:%.*]] = getelementptr <2 x i8>, ptr [[PTR]], i32 0
+; CHECK-NEXT:    [[VPACK:%.*]] = extractelement <2 x i1> [[COND0]], i32 0
+; CHECK-NEXT:    [[VPACK1:%.*]] = insertelement <4 x i1> poison, i1 [[VPACK]], i32 0
+; CHECK-NEXT:    [[VPACK2:%.*]] = extractelement <2 x i1> [[COND0]], i32 1
+; CHECK-NEXT:    [[VPACK3:%.*]] = insertelement <4 x i1> [[VPACK1]], i1 [[VPACK2]], i32 1
+; CHECK-NEXT:    [[VPACK4:%.*]] = extractelement <2 x i1> [[COND1]], i32 0
+; CHECK-NEXT:    [[VPACK5:%.*]] = insertelement <4 x i1> [[VPACK3]], i1 [[VPACK4]], i32 2
+; CHECK-NEXT:    [[VPACK6:%.*]] = extractelement <2 x i1> [[COND1]], i32 1
+; CHECK-NEXT:    [[VPACK7:%.*]] = insertelement <4 x i1> [[VPACK5]], i1 [[VPACK6]], i32 3
+; CHECK-NEXT:    [[VECL:%.*]] = load <4 x i8>, ptr [[PTR0]], align 2
+; CHECK-NEXT:    [[VEC:%.*]] = select <4 x i1> [[VPACK7]], <4 x i8> [[VECL]], <4 x i8> [[VECL]]
+; CHECK-NEXT:    store <4 x i8> [[VEC]], ptr [[PTR0]], align 2
+; CHECK-NEXT:    ret void
+;
+  %ptr0 = getelementptr <2 x i8>, ptr %ptr, i32 0
+  %ptr1 = getelementptr <2 x i8>, ptr %ptr, i32 1
+  %ld0 = load <2 x i8>, ptr %ptr0
+  %ld1 = load <2 x i8>, ptr %ptr1
+  %sel0 = select <2 x i1> %cond0, <2 x i8> %ld0, <2 x i8> %ld0
+  %sel1 = select <2 x i1> %cond1, <2 x i8> %ld1, <2 x i8> %ld1
+  store <2 x i8> %sel0, ptr %ptr0
+  store <2 x i8> %sel1, ptr %ptr1
+  ret void
+}
diff --git a/llvm/test/Transforms/SeparateConstOffsetFromGEP/AMDGPU/split-gep-and-gvn-addrspace-addressing-modes.ll b/llvm/test/Transforms/SeparateConstOffsetFromGEP/AMDGPU/split-gep-and-gvn-addrspace-addressing-modes.ll
index 83e36eba36c84..2ee36d712a94b 100644
--- a/llvm/test/Transforms/SeparateConstOffsetFromGEP/AMDGPU/split-gep-and-gvn-addrspace-addressing-modes.ll
+++ b/llvm/test/Transforms/SeparateConstOffsetFromGEP/AMDGPU/split-gep-and-gvn-addrspace-addressing-modes.ll
@@ -7,7 +7,7 @@ target datalayout = "e-p:32:32-p1:64:64-p2:64:64-p3:32:32-p4:64:64-p5:32:32-p24:
 
 define amdgpu_kernel void @sum_of_array(i32 %x, i32 %y, ptr addrspace(1) nocapture %output) {
 ; IR-LABEL: define amdgpu_kernel void @sum_of_array(
-; IR-SAME: i32 [[X:%.*]], i32 [[Y:%.*]], ptr addrspace(1) nocapture [[OUTPUT:%.*]]) {
+; IR-SAME: i32 [[X:%.*]], i32 [[Y:%.*]], ptr addrspace(1) captures(none) [[OUTPUT:%.*]]) {
 ; IR-NEXT:    [[TMP:%.*]] = sext i32 [[Y]] to i64
 ; IR-NEXT:    [[TMP1:%.*]] = sext i32 [[X]] to i64
 ; IR-NEXT:    [[TMP2:%.*]] = getelementptr [4096 x [32 x float]], ptr addrspace(4) @array, i64 0, i64 [[TMP1]], i64 [[TMP]]
@@ -45,7 +45,7 @@ define amdgpu_kernel void @sum_of_array(i32 %x, i32 %y, ptr addrspace(1) nocaptu
 
 define amdgpu_kernel void @sum_of_array_over_max_mubuf_offset(i32 %x, i32 %y, ptr addrspace(1) nocapture %output) {
 ; IR-LABEL: define amdgpu_kernel void @sum_of_array_over_max_mubuf_offset(
-; IR-SAME: i32 [[X:%.*]], i32 [[Y:%.*]], ptr addrspace(1) nocapture [[OUTPUT:%.*]]) {
+; IR-SAME: i32 [[X:%.*]], i32 [[Y:%.*]], ptr addrspace(1) captures(none) [[OUTPUT:%.*]]) {
 ; IR-NEXT:    [[TMP:%.*]] = sext i32 [[Y]] to i64
 ; IR-NEXT:    [[TMP1:%.*]] = sext i32 [[X]] to i64
 ; IR-NEXT:    [[TMP2:%.*]] = getelementptr [4096 x [4 x float]], ptr addrspace(4) @array2, i64 0, i64 [[TMP1]], i64 [[TMP]]
@@ -87,7 +87,7 @@ define amdgpu_kernel void @sum_of_array_over_max_mubuf_offset(i32 %x, i32 %y, pt
 ; DS instructions have a larger immediate offset, so make sure these are OK.
 define amdgpu_kernel void @sum_of_lds_array_over_max_mubuf_offset(i32 %x, i32 %y, ptr addrspace(1) nocapture %output) {
 ; IR-LABEL: define amdgpu_kernel void @sum_of_lds_array_over_max_mubuf_offset(
-; IR-SAME: i32 [[X:%.*]], i32 [[Y:%.*]], ptr addrspace(1) nocapture [[OUTPUT:%.*]]) {
+; IR-SAME: i32 [[X:%.*]], i32 [[Y:%.*]], ptr addrspace(1) captures(none) [[OUTPUT:%.*]]) {
 ; IR-NEXT:    [[TMP2:%.*]] = getelementptr [4096 x [4 x float]], ptr addrspace(3) @lds_array, i32 0, i32 [[X]], i32 [[Y]]
 ; IR-NEXT:    [[TMP4:%.*]] = load float, ptr addrspace(3) [[TMP2]], align 4
 ; IR-NEXT:    [[TMP5:%.*]] = fadd float [[TMP4]], 0.000000e+00
diff --git a/llvm/test/Transforms/SeparateConstOffsetFromGEP/AMDGPU/split-gep-and-gvn.ll b/llvm/test/Transforms/SeparateConstOffsetFromGEP/AMDGPU/split-gep-and-gvn.ll
index b53c048187853..79940041ed5f0 100644
--- a/llvm/test/Transforms/SeparateConstOffsetFromGEP/AMDGPU/split-gep-and-gvn.ll
+++ b/llvm/test/Transforms/SeparateConstOffsetFromGEP/AMDGPU/split-gep-and-gvn.ll
@@ -18,7 +18,7 @@
 
 define void @sum_of_array(i32 %x, i32 %y, ptr nocapture %output) {
 ; IR-LABEL: define void @sum_of_array
-; IR-SAME: (i32 [[X:%.*]], i32 [[Y:%.*]], ptr nocapture [[OUTPUT:%.*]]) #[[ATTR0:[0-9]+]] {
+; IR-SAME: (i32 [[X:%.*]], i32 [[Y:%.*]], ptr captures(none) [[OUTPUT:%.*]]) #[[ATTR0:[0-9]+]] {
 ; IR-NEXT:  .preheader:
 ; IR-NEXT:    [[I:%.*]] = sext i32 [[Y]] to i64
 ; IR-NEXT:    [[I1:%.*]] = sext i32 [[X]] to i64
@@ -80,7 +80,7 @@ define void @sum_of_array(i32 %x, i32 %y, ptr nocapture %output) {
 ; able to extract constant offsets from both forms.
 define void @sum_of_array2(i32 %x, i32 %y, ptr nocapture %output) {
 ; IR-LABEL: define void @sum_of_array2
-; IR-SAME: (i32 [[X:%.*]], i32 [[Y:%.*]], ptr nocapture [[OUTPUT:%.*]]) #[[ATTR0]] {
+; IR-SAME: (i32 [[X:%.*]], i32 [[Y:%.*]], ptr captures(none) [[OUTPUT:%.*]]) #[[ATTR0]] {
 ; IR-NEXT:  .preheader:
 ; IR-NEXT:    [[I:%.*]] = sext i32 [[Y]] to i64
 ; IR-NEXT:    [[I1:%.*]] = sext i32 [[X]] to i64
@@ -141,7 +141,7 @@ define void @sum_of_array2(i32 %x, i32 %y, ptr nocapture %output) {
 
 define void @sum_of_array3(i32 %x, i32 %y, ptr nocapture %output) {
 ; IR-LABEL: define void @sum_of_array3
-; IR-SAME: (i32 [[X:%.*]], i32 [[Y:%.*]], ptr nocapture [[OUTPUT:%.*]]) #[[ATTR0]] {
+; IR-SAME: (i32 [[X:%.*]], i32 [[Y:%.*]], ptr captures(none) [[OUTPUT:%.*]]) #[[ATTR0]] {
 ; IR-NEXT:  .preheader:
 ; IR-NEXT:    [[I:%.*]] = zext i32 [[Y]] to i64
 ; IR-NEXT:    [[I1:%.*]] = zext i32 [[X]] to i64
@@ -201,7 +201,7 @@ define void @sum_of_array3(i32 %x, i32 %y, ptr nocapture %output) {
 ; &array[zext(x)][zext(y)]. See the expected IR and PTX for details.
 define void @sum_of_array4(i32 %x, i32 %y, ptr nocapture %output) {
 ; IR-LABEL: define void @sum_of_array4
-; IR-SAME: (i32 [[X:%.*]], i32 [[Y:%.*]], ptr nocapture [[OUTPUT:%.*]]) #[[ATTR0]] {
+; IR-SAME: (i32 [[X:%.*]], i32 [[Y:%.*]], ptr captures(none) [[OUTPUT:%.*]]) #[[ATTR0]] {
 ; IR-NEXT:  .preheader:
 ; IR-NEXT:    [[I:%.*]] = zext i32 [[Y]] to i64
 ; IR-NEXT:    [[I1:%.*]] = zext i32 [[X]] to i64
diff --git a/llvm/test/Transforms/SeparateConstOffsetFromGEP/NVPTX/split-gep-and-gvn.ll b/llvm/test/Transforms/SeparateConstOffsetFromGEP/NVPTX/split-gep-and-gvn.ll
index 79398a80ac659..e03f5cc6ad24b 100644
--- a/llvm/test/Transforms/SeparateConstOffsetFromGEP/NVPTX/split-gep-and-gvn.ll
+++ b/llvm/test/Transforms/SeparateConstOffsetFromGEP/NVPTX/split-gep-and-gvn.ll
@@ -20,7 +20,7 @@
 
 define void @sum_of_array(i32 %x, i32 %y, ptr nocapture %output) {
 ; IR-LABEL: define void @sum_of_array(
-; IR-SAME: i32 [[X:%.*]], i32 [[Y:%.*]], ptr nocapture [[OUTPUT:%.*]]) {
+; IR-SAME: i32 [[X:%.*]], i32 [[Y:%.*]], ptr captures(none) [[OUTPUT:%.*]]) {
 ; IR-NEXT:  .preheader:
 ; IR-NEXT:    [[TMP0:%.*]] = sext i32 [[Y]] to i64
 ; IR-NEXT:    [[TMP1:%.*]] = sext i32 [[X]] to i64
@@ -86,7 +86,7 @@ define void @sum_of_array(i32 %x, i32 %y, ptr nocapture %output) {
 ; able to extract constant offsets from both forms.
 define void @sum_of_array2(i32 %x, i32 %y, ptr nocapture %output) {
 ; IR-LABEL: define void @sum_of_array2(
-; IR-SAME: i32 [[X:%.*]], i32 [[Y:%.*]], ptr nocapture [[OUTPUT:%.*]]) {
+; IR-SAME: i32 [[X:%.*]], i32 [[Y:%.*]], ptr captures(none) [[OUTPUT:%.*]]) {
 ; IR-NEXT:  .preheader:
 ; IR-NEXT:    [[TMP0:%.*]] = sext i32 [[Y]] to i64
 ; IR-NEXT:    [[TMP1:%.*]] = sext i32 [[X]] to i64
@@ -153,7 +153,7 @@ define void @sum_of_array2(i32 %x, i32 %y, ptr nocapture %output) {
 ;    may be invalid.
 define void @sum_of_array3(i32 %x, i32 %y, ptr nocapture %output) {
 ; IR-LABEL: define void @sum_of_array3(
-; IR-SAME: i32 [[X:%.*]], i32 [[Y:%.*]], ptr nocapture [[OUTPUT:%.*]]) {
+; IR-SAME: i32 [[X:%.*]], i32 [[Y:%.*]], ptr captures(none) [[OUTPUT:%.*]]) {
 ; IR-NEXT:  .preheader:
 ; IR-NEXT:    [[TMP0:%.*]] = zext i32 [[Y]] to i64
 ; IR-NEXT:    [[TMP1:%.*]] = zext i32 [[X]] to i64
@@ -220,7 +220,7 @@ define void @sum_of_array3(i32 %x, i32 %y, ptr nocapture %output) {
 ; &array[zext(x)][zext(y)]. See the expected IR and PTX for details.
 define void @sum_of_array4(i32 %x, i32 %y, ptr nocapture %output) {
 ; IR-LABEL: define void @sum_of_array4(
-; IR-SAME: i32 [[X:%.*]], i32 [[Y:%.*]], ptr nocapture [[OUTPUT:%.*]]) {
+; IR-SAME: i32 [[X:%.*]], i32 [[Y:%.*]], ptr captures(none) [[OUTPUT:%.*]]) {
 ; IR-NEXT:  .preheader:
 ; IR-NEXT:    [[TMP0:%.*]] = zext i32 [[Y]] to i64
 ; IR-NEXT:    [[TMP1:%.*]] = zext i32 [[X]] to i64
diff --git a/llvm/test/Transforms/SimpleLoopUnswitch/AMDGPU/uniform-unswitch.ll b/llvm/test/Transforms/SimpleLoopUnswitch/AMDGPU/uniform-unswitch.ll
index 2199d6a07981a..cdd0fcab59b7d 100644
--- a/llvm/test/Transforms/SimpleLoopUnswitch/AMDGPU/uniform-unswitch.ll
+++ b/llvm/test/Transforms/SimpleLoopUnswitch/AMDGPU/uniform-unswitch.ll
@@ -20,7 +20,7 @@
 
 define amdgpu_kernel void @uniform_unswitch(ptr nocapture %out, i32 %n, i32 %x) {
 ; CHECK-LABEL: define amdgpu_kernel void @uniform_unswitch(
-; CHECK-SAME: ptr nocapture writeonly [[OUT:%.*]], i32 [[N:%.*]], i32 [[X:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
+; CHECK-SAME: ptr writeonly captures(none) [[OUT:%.*]], i32 [[N:%.*]], i32 [[X:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[OUT_GLOBAL:%.*]] = addrspacecast ptr [[OUT]] to ptr addrspace(1)
 ; CHECK-NEXT:    [[CMP6:%.*]] = icmp sgt i32 [[N]], 0
diff --git a/llvm/test/Transforms/StructurizeCFG/simple-structurizecfg-crash.ll b/llvm/test/Transforms/StructurizeCFG/simple-structurizecfg-crash.ll
new file mode 100644
index 0000000000000..6edc0b883f51e
--- /dev/null
+++ b/llvm/test/Transforms/StructurizeCFG/simple-structurizecfg-crash.ll
@@ -0,0 +1,20 @@
+; RUN: opt -S -passes=structurizecfg %s -o -
+; REQUIRES: asserts
+; XFAIL: *
+
+; Issue tracking: https://github.com/llvm/llvm-project/issues/126534.
+; FIXME: This test is expected to crash. Generate checklines after the crash is fixed.
+
+define void @foo() {
+entry:
+  br i1 false, label %cond.true, label %cond.false
+
+cond.true:                                        ; preds = %entry
+  br label %cond.end
+
+cond.false:                                       ; preds = %entry
+  br label %cond.end
+
+cond.end:                                         ; preds = %cond.false, %cond.true
+  ret void
+}
diff --git a/llvm/test/Transforms/TailCallElim/accum_recursion.ll b/llvm/test/Transforms/TailCallElim/accum_recursion.ll
index c093f026e4678..e2d962e412c62 100644
--- a/llvm/test/Transforms/TailCallElim/accum_recursion.ll
+++ b/llvm/test/Transforms/TailCallElim/accum_recursion.ll
@@ -145,7 +145,7 @@ declare i32 @test4_helper()
 
 define i32 @test5_base_case_load(ptr nocapture %A, i32 %n) local_unnamed_addr {
 ; CHECK-LABEL: define i32 @test5_base_case_load(
-; CHECK-SAME: ptr nocapture [[A:%.*]], i32 [[N:%.*]]) local_unnamed_addr {
+; CHECK-SAME: ptr captures(none) [[A:%.*]], i32 [[N:%.*]]) local_unnamed_addr {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    br label [[TAILRECURSE:%.*]]
 ; CHECK:       tailrecurse:
diff --git a/llvm/test/Transforms/TailCallElim/tre-byval-parameter-2.ll b/llvm/test/Transforms/TailCallElim/tre-byval-parameter-2.ll
index 9a563f771b6ca..325db79ebbd7e 100644
--- a/llvm/test/Transforms/TailCallElim/tre-byval-parameter-2.ll
+++ b/llvm/test/Transforms/TailCallElim/tre-byval-parameter-2.ll
@@ -25,7 +25,7 @@
 ; Function Attrs: noinline nounwind uwtable
 define dso_local void @_Z7dostuff1AS_i(ptr nocapture byval(%struct.A) align 8 %a, ptr nocapture readonly byval(%struct.A) align 8 %b, i32 %i) local_unnamed_addr #0 {
 ; CHECK-LABEL: define {{[^@]+}}@_Z7dostuff1AS_i
-; CHECK-SAME: (ptr nocapture byval([[STRUCT_A:%.*]]) align 8 [[A:%.*]], ptr nocapture byval([[STRUCT_A]]) align 8 [[B:%.*]], i32 [[I:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
+; CHECK-SAME: (ptr byval([[STRUCT_A:%.*]]) align 8 captures(none) [[A:%.*]], ptr byval([[STRUCT_A]]) align 8 captures(none) [[B:%.*]], i32 [[I:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[AGG_TMP52:%.*]] = alloca [[STRUCT_A]], align 8
 ; CHECK-NEXT:    [[AGG_TMP1:%.*]] = alloca [[STRUCT_A]], align 8
diff --git a/llvm/test/Transforms/TailCallElim/tre-byval-parameter.ll b/llvm/test/Transforms/TailCallElim/tre-byval-parameter.ll
index 72f83b37a2d9f..256fb04b3ba1d 100644
--- a/llvm/test/Transforms/TailCallElim/tre-byval-parameter.ll
+++ b/llvm/test/Transforms/TailCallElim/tre-byval-parameter.ll
@@ -25,7 +25,7 @@
 ; Function Attrs: uwtable
 define dso_local i32 @_Z3fooi1S(i32 %count, ptr nocapture readonly byval(%struct.S) align 8 %p1) local_unnamed_addr #0 {
 ; CHECK-LABEL: define {{[^@]+}}@_Z3fooi1S
-; CHECK-SAME: (i32 [[COUNT:%.*]], ptr nocapture byval([[STRUCT_S:%.*]]) align 8 [[P1:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
+; CHECK-SAME: (i32 [[COUNT:%.*]], ptr byval([[STRUCT_S:%.*]]) align 8 captures(none) [[P1:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[AGG_TMP_I1:%.*]] = alloca [[STRUCT_S]], align 8
 ; CHECK-NEXT:    [[AGG_TMP_I:%.*]] = alloca [[STRUCT_S]], align 8
diff --git a/llvm/test/Transforms/Util/strip-nonlinetable-debuginfo-pr125116.ll b/llvm/test/Transforms/Util/strip-nonlinetable-debuginfo-pr125116.ll
new file mode 100644
index 0000000000000..dafee60a491e7
--- /dev/null
+++ b/llvm/test/Transforms/Util/strip-nonlinetable-debuginfo-pr125116.ll
@@ -0,0 +1,88 @@
+; Test if StripNonLineTableDebugInfo crashes or produces invalid IR,
+; this test contains a slightly complex debug info structure,
+; which may trigger the bug mentioned in pr#125116
+;
+; RUN: opt < %s -p=strip-nonlinetable-debuginfo -S | FileCheck %s
+;
+; CHECK-NOT: DIBasicType
+; CHECK-NOT: DIDerivedType
+; CHECK-NOT: DICompositeType
+; CHECK-NOT: DILocation(line: 604, column: 1, scope: null)
+
+define void @main() !dbg !34 {
+  ret void, !dbg !68
+}
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!31, !32}
+!llvm.ident = !{!33}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus, file: !1, producer: "clang version 3.7 (tags/RELEASE_370/final)", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, enums: !2, retainedTypes: !3, globals: !2)
+!1 = !DIFile(filename: "no filename", directory: "")
+!2 = !{}
+!3 = !{!4, !22}
+!4 = !DIDerivedType(tag: DW_TAG_typedef, name: "float3x3", file: !1, line: 361, baseType: !5)
+!5 = !DICompositeType(tag: DW_TAG_class_type, name: "matrix<float, 3, 3>", file: !1, line: 246, size: 288, align: 32, elements: !6, templateParams: !17)
+!6 = !{!7, !9, !10, !11, !12, !13, !14, !15, !16}
+!7 = !DIDerivedType(tag: DW_TAG_member, name: "_11", scope: !5, file: !1, line: 246, baseType: !8, size: 32, align: 32, flags: DIFlagPublic)
+!8 = !DIBasicType(name: "float", size: 32, align: 32, encoding: DW_ATE_float)
+!9 = !DIDerivedType(tag: DW_TAG_member, name: "_12", scope: !5, file: !1, line: 246, baseType: !8, size: 32, align: 32, offset: 32, flags: DIFlagPublic)
+!10 = !DIDerivedType(tag: DW_TAG_member, name: "_13", scope: !5, file: !1, line: 246, baseType: !8, size: 32, align: 32, offset: 64, flags: DIFlagPublic)
+!11 = !DIDerivedType(tag: DW_TAG_member, name: "_21", scope: !5, file: !1, line: 246, baseType: !8, size: 32, align: 32, offset: 96, flags: DIFlagPublic)
+!12 = !DIDerivedType(tag: DW_TAG_member, name: "_22", scope: !5, file: !1, line: 246, baseType: !8, size: 32, align: 32, offset: 128, flags: DIFlagPublic)
+!13 = !DIDerivedType(tag: DW_TAG_member, name: "_23", scope: !5, file: !1, line: 246, baseType: !8, size: 32, align: 32, offset: 160, flags: DIFlagPublic)
+!14 = !DIDerivedType(tag: DW_TAG_member, name: "_31", scope: !5, file: !1, line: 246, baseType: !8, size: 32, align: 32, offset: 192, flags: DIFlagPublic)
+!15 = !DIDerivedType(tag: DW_TAG_member, name: "_32", scope: !5, file: !1, line: 246, baseType: !8, size: 32, align: 32, offset: 224, flags: DIFlagPublic)
+!16 = !DIDerivedType(tag: DW_TAG_member, name: "_33", scope: !5, file: !1, line: 246, baseType: !8, size: 32, align: 32, offset: 256, flags: DIFlagPublic)
+!17 = !{!18, !19, !21}
+!18 = !DITemplateTypeParameter(name: "element", type: !8)
+!19 = !DITemplateValueParameter(name: "row_count", type: !20, value: i32 3)
+!20 = !DIBasicType(name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!21 = !DITemplateValueParameter(name: "col_count", type: !20, value: i32 3)
+!22 = !DIDerivedType(tag: DW_TAG_typedef, name: "float4", file: !1, baseType: !23)
+!23 = !DICompositeType(tag: DW_TAG_class_type, name: "vector<float, 4>", file: !1, size: 128, align: 32, elements: !24, templateParams: !29)
+!24 = !{!25, !26, !27, !28}
+!25 = !DIDerivedType(tag: DW_TAG_member, name: "x", scope: !23, file: !1, baseType: !8, size: 32, align: 32, flags: DIFlagPublic)
+!26 = !DIDerivedType(tag: DW_TAG_member, name: "y", scope: !23, file: !1, baseType: !8, size: 32, align: 32, offset: 32, flags: DIFlagPublic)
+!27 = !DIDerivedType(tag: DW_TAG_member, name: "z", scope: !23, file: !1, baseType: !8, size: 32, align: 32, offset: 64, flags: DIFlagPublic)
+!28 = !DIDerivedType(tag: DW_TAG_member, name: "w", scope: !23, file: !1, baseType: !8, size: 32, align: 32, offset: 96, flags: DIFlagPublic)
+!29 = !{!18, !30}
+!30 = !DITemplateValueParameter(name: "element_count", type: !20, value: i32 4)
+!31 = !{i32 2, !"Dwarf Version", i32 4}
+!32 = !{i32 2, !"Debug Info Version", i32 3}
+!33 = !{!"clang version 3.7 (tags/RELEASE_370/final)"}
+!34 = distinct !DISubprogram(name: "main", scope: !1, file: !1, line: 581, type: !35, scopeLine: 582, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition, unit: !0)
+!35 = !DISubroutineType(types: !36)
+!36 = !{null, !37, !58}
+!37 = !DICompositeType(tag: DW_TAG_structure_type, name: "VertexInput", file: !1, line: 254, size: 416, align: 32, elements: !38)
+!38 = !{!39, !40, !48, !57}
+!39 = !DIDerivedType(tag: DW_TAG_member, name: "Position", scope: !37, file: !1, line: 256, baseType: !22, size: 128, align: 32)
+!40 = !DIDerivedType(tag: DW_TAG_member, name: "TexCoord", scope: !37, file: !1, line: 257, baseType: !41, size: 64, align: 32, offset: 128)
+!41 = !DIDerivedType(tag: DW_TAG_typedef, name: "float2", file: !1, baseType: !42)
+!42 = !DICompositeType(tag: DW_TAG_class_type, name: "vector<float, 2>", file: !1, size: 64, align: 32, elements: !43, templateParams: !46)
+!43 = !{!44, !45}
+!44 = !DIDerivedType(tag: DW_TAG_member, name: "x", scope: !42, file: !1, baseType: !8, size: 32, align: 32, flags: DIFlagPublic)
+!45 = !DIDerivedType(tag: DW_TAG_member, name: "y", scope: !42, file: !1, baseType: !8, size: 32, align: 32, offset: 32, flags: DIFlagPublic)
+!46 = !{!18, !47}
+!47 = !DITemplateValueParameter(name: "element_count", type: !20, value: i32 2)
+!48 = !DIDerivedType(tag: DW_TAG_member, name: "Normal", scope: !37, file: !1, line: 258, baseType: !49, size: 96, align: 32, offset: 192)
+!49 = !DIDerivedType(tag: DW_TAG_typedef, name: "float3", file: !1, baseType: !50)
+!50 = !DICompositeType(tag: DW_TAG_class_type, name: "vector<float, 3>", file: !1, size: 96, align: 32, elements: !51, templateParams: !55)
+!51 = !{!52, !53, !54}
+!52 = !DIDerivedType(tag: DW_TAG_member, name: "x", scope: !50, file: !1, baseType: !8, size: 32, align: 32, flags: DIFlagPublic)
+!53 = !DIDerivedType(tag: DW_TAG_member, name: "y", scope: !50, file: !1, baseType: !8, size: 32, align: 32, offset: 32, flags: DIFlagPublic)
+!54 = !DIDerivedType(tag: DW_TAG_member, name: "z", scope: !50, file: !1, baseType: !8, size: 32, align: 32, offset: 64, flags: DIFlagPublic)
+!55 = !{!18, !56}
+!56 = !DITemplateValueParameter(name: "element_count", type: !20, value: i32 3)
+!57 = !DIDerivedType(tag: DW_TAG_member, name: "Tangent", scope: !37, file: !1, line: 259, baseType: !22, size: 128, align: 32, offset: 288)
+!58 = !DIDerivedType(tag: DW_TAG_restrict_type, baseType: !59)
+!59 = !DICompositeType(tag: DW_TAG_structure_type, name: "VertexOutput", file: !1, line: 269, size: 672, align: 32, elements: !60)
+!60 = !{!61, !62, !63, !64, !65, !66, !67}
+!61 = !DIDerivedType(tag: DW_TAG_member, name: "Position", scope: !59, file: !1, line: 271, baseType: !22, size: 128, align: 32)
+!62 = !DIDerivedType(tag: DW_TAG_member, name: "TexCoord", scope: !59, file: !1, line: 272, baseType: !41, size: 64, align: 32, offset: 128)
+!63 = !DIDerivedType(tag: DW_TAG_member, name: "TangentInView", scope: !59, file: !1, line: 273, baseType: !49, size: 96, align: 32, offset: 192)
+!64 = !DIDerivedType(tag: DW_TAG_member, name: "BitangentInView", scope: !59, file: !1, line: 274, baseType: !49, size: 96, align: 32, offset: 288)
+!65 = !DIDerivedType(tag: DW_TAG_member, name: "NormalInView", scope: !59, file: !1, line: 275, baseType: !49, size: 96, align: 32, offset: 384)
+!66 = !DIDerivedType(tag: DW_TAG_member, name: "EyeDirectionInView", scope: !59, file: !1, line: 276, baseType: !49, size: 96, align: 32, offset: 480)
+!67 = !DIDerivedType(tag: DW_TAG_member, name: "PositionInView", scope: !59, file: !1, line: 277, baseType: !49, size: 96, align: 32, offset: 576)
+!68 = !DILocation(line: 604, column: 1, scope: !34)
diff --git a/llvm/test/Transforms/VectorCombine/RISCV/vector-interleave2-splat-e64.ll b/llvm/test/Transforms/VectorCombine/RISCV/vector-interleave2-splat-e64.ll
new file mode 100644
index 0000000000000..26a5d2e7f849b
--- /dev/null
+++ b/llvm/test/Transforms/VectorCombine/RISCV/vector-interleave2-splat-e64.ll
@@ -0,0 +1,17 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -S -mtriple=riscv64 -mattr=+v %s -passes=vector-combine | FileCheck %s
+; RUN: opt -S -mtriple=riscv32 -mattr=+v %s -passes=vector-combine | FileCheck %s
+
+; We should not form a i128 vector.
+
+define void @interleave2_const_splat_nxv8i64(ptr %dst) {
+; CHECK-LABEL: define void @interleave2_const_splat_nxv8i64(
+; CHECK-SAME: ptr [[DST:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT:    [[INTERLEAVE2:%.*]] = call <vscale x 8 x i64> @llvm.vector.interleave2.nxv8i64(<vscale x 4 x i64> splat (i64 666), <vscale x 4 x i64> splat (i64 777))
+; CHECK-NEXT:    call void @llvm.vp.store.nxv8i64.p0(<vscale x 8 x i64> [[INTERLEAVE2]], ptr [[DST]], <vscale x 8 x i1> splat (i1 true), i32 88)
+; CHECK-NEXT:    ret void
+;
+  %interleave2 = call <vscale x 8 x i64> @llvm.vector.interleave2.nxv8i64(<vscale x 4 x i64> splat (i64 666), <vscale x 4 x i64> splat (i64 777))
+  call void @llvm.vp.store.nxv8i64.p0(<vscale x 8 x i64> %interleave2, ptr %dst, <vscale x 8 x i1> splat (i1 true), i32 88)
+  ret void
+}
diff --git a/llvm/test/Transforms/VectorCombine/RISCV/vector-interleave2-splat.ll b/llvm/test/Transforms/VectorCombine/RISCV/vector-interleave2-splat.ll
new file mode 100644
index 0000000000000..477a593ec51e9
--- /dev/null
+++ b/llvm/test/Transforms/VectorCombine/RISCV/vector-interleave2-splat.ll
@@ -0,0 +1,21 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -S -mtriple=riscv64 -mattr=+v %s -passes=vector-combine | FileCheck %s
+; RUN: opt -S -mtriple=riscv32 -mattr=+v %s -passes=vector-combine | FileCheck %s
+; RUN: opt -S -mtriple=riscv64 -mattr=+zve32x %s -passes=vector-combine | FileCheck %s --check-prefix=ZVE32X
+
+define void @interleave2_const_splat_nxv16i32(ptr %dst) {
+; CHECK-LABEL: define void @interleave2_const_splat_nxv16i32(
+; CHECK-SAME: ptr [[DST:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT:    call void @llvm.vp.store.nxv16i32.p0(<vscale x 16 x i32> bitcast (<vscale x 8 x i64> splat (i64 3337189589658) to <vscale x 16 x i32>), ptr [[DST]], <vscale x 16 x i1> splat (i1 true), i32 88)
+; CHECK-NEXT:    ret void
+;
+; ZVE32X-LABEL: define void @interleave2_const_splat_nxv16i32(
+; ZVE32X-SAME: ptr [[DST:%.*]]) #[[ATTR0:[0-9]+]] {
+; ZVE32X-NEXT:    [[INTERLEAVE2:%.*]] = call <vscale x 16 x i32> @llvm.vector.interleave2.nxv16i32(<vscale x 8 x i32> splat (i32 666), <vscale x 8 x i32> splat (i32 777))
+; ZVE32X-NEXT:    call void @llvm.vp.store.nxv16i32.p0(<vscale x 16 x i32> [[INTERLEAVE2]], ptr [[DST]], <vscale x 16 x i1> splat (i1 true), i32 88)
+; ZVE32X-NEXT:    ret void
+;
+  %interleave2 = call <vscale x 16 x i32> @llvm.vector.interleave2.nxv16i32(<vscale x 8 x i32> splat (i32 666), <vscale x 8 x i32> splat (i32 777))
+  call void @llvm.vp.store.nxv16i32.p0(<vscale x 16 x i32> %interleave2, ptr %dst, <vscale x 16 x i1> splat (i1 true), i32 88)
+  ret void
+}
diff --git a/llvm/test/Transforms/VectorCombine/X86/extract-binop-inseltpoison.ll b/llvm/test/Transforms/VectorCombine/X86/extract-binop-inseltpoison.ll
index f3b7f7b72ee42..d369279c15db4 100644
--- a/llvm/test/Transforms/VectorCombine/X86/extract-binop-inseltpoison.ll
+++ b/llvm/test/Transforms/VectorCombine/X86/extract-binop-inseltpoison.ll
@@ -252,17 +252,11 @@ define i8 @ext1_ext1_add_uses2(<16 x i8> %x, <16 x i8> %y) {
 }
 
 define i8 @ext0_ext1_add(<16 x i8> %x, <16 x i8> %y) {
-; SSE-LABEL: @ext0_ext1_add(
-; SSE-NEXT:    [[E0:%.*]] = extractelement <16 x i8> [[X:%.*]], i32 0
-; SSE-NEXT:    [[E1:%.*]] = extractelement <16 x i8> [[Y:%.*]], i32 1
-; SSE-NEXT:    [[R:%.*]] = add nuw i8 [[E0]], [[E1]]
-; SSE-NEXT:    ret i8 [[R]]
-;
-; AVX-LABEL: @ext0_ext1_add(
-; AVX-NEXT:    [[SHIFT:%.*]] = shufflevector <16 x i8> [[Y:%.*]], <16 x i8> poison, <16 x i32> <i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; AVX-NEXT:    [[TMP1:%.*]] = add nuw <16 x i8> [[X:%.*]], [[SHIFT]]
-; AVX-NEXT:    [[R:%.*]] = extractelement <16 x i8> [[TMP1]], i32 0
-; AVX-NEXT:    ret i8 [[R]]
+; CHECK-LABEL: @ext0_ext1_add(
+; CHECK-NEXT:    [[SHIFT:%.*]] = shufflevector <16 x i8> [[Y:%.*]], <16 x i8> poison, <16 x i32> <i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP1:%.*]] = add nuw <16 x i8> [[X:%.*]], [[SHIFT]]
+; CHECK-NEXT:    [[R:%.*]] = extractelement <16 x i8> [[TMP1]], i32 0
+; CHECK-NEXT:    ret i8 [[R]]
 ;
   %e0 = extractelement <16 x i8> %x, i32 0
   %e1 = extractelement <16 x i8> %y, i32 1
@@ -271,17 +265,11 @@ define i8 @ext0_ext1_add(<16 x i8> %x, <16 x i8> %y) {
 }
 
 define i8 @ext5_ext0_add(<16 x i8> %x, <16 x i8> %y) {
-; SSE-LABEL: @ext5_ext0_add(
-; SSE-NEXT:    [[E0:%.*]] = extractelement <16 x i8> [[X:%.*]], i32 5
-; SSE-NEXT:    [[E1:%.*]] = extractelement <16 x i8> [[Y:%.*]], i32 0
-; SSE-NEXT:    [[R:%.*]] = sub nsw i8 [[E0]], [[E1]]
-; SSE-NEXT:    ret i8 [[R]]
-;
-; AVX-LABEL: @ext5_ext0_add(
-; AVX-NEXT:    [[SHIFT:%.*]] = shufflevector <16 x i8> [[X:%.*]], <16 x i8> poison, <16 x i32> <i32 5, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; AVX-NEXT:    [[TMP1:%.*]] = sub nsw <16 x i8> [[SHIFT]], [[Y:%.*]]
-; AVX-NEXT:    [[R:%.*]] = extractelement <16 x i8> [[TMP1]], i64 0
-; AVX-NEXT:    ret i8 [[R]]
+; CHECK-LABEL: @ext5_ext0_add(
+; CHECK-NEXT:    [[SHIFT:%.*]] = shufflevector <16 x i8> [[X:%.*]], <16 x i8> poison, <16 x i32> <i32 5, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP1:%.*]] = sub nsw <16 x i8> [[SHIFT]], [[Y:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = extractelement <16 x i8> [[TMP1]], i64 0
+; CHECK-NEXT:    ret i8 [[R]]
 ;
   %e0 = extractelement <16 x i8> %x, i32 5
   %e1 = extractelement <16 x i8> %y, i32 0
@@ -290,17 +278,11 @@ define i8 @ext5_ext0_add(<16 x i8> %x, <16 x i8> %y) {
 }
 
 define i8 @ext1_ext6_add(<16 x i8> %x, <16 x i8> %y) {
-; SSE-LABEL: @ext1_ext6_add(
-; SSE-NEXT:    [[E0:%.*]] = extractelement <16 x i8> [[X:%.*]], i32 1
-; SSE-NEXT:    [[E1:%.*]] = extractelement <16 x i8> [[Y:%.*]], i32 6
-; SSE-NEXT:    [[R:%.*]] = and i8 [[E0]], [[E1]]
-; SSE-NEXT:    ret i8 [[R]]
-;
-; AVX-LABEL: @ext1_ext6_add(
-; AVX-NEXT:    [[SHIFT:%.*]] = shufflevector <16 x i8> [[Y:%.*]], <16 x i8> poison, <16 x i32> <i32 poison, i32 6, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; AVX-NEXT:    [[TMP1:%.*]] = and <16 x i8> [[X:%.*]], [[SHIFT]]
-; AVX-NEXT:    [[R:%.*]] = extractelement <16 x i8> [[TMP1]], i32 1
-; AVX-NEXT:    ret i8 [[R]]
+; CHECK-LABEL: @ext1_ext6_add(
+; CHECK-NEXT:    [[SHIFT:%.*]] = shufflevector <16 x i8> [[Y:%.*]], <16 x i8> poison, <16 x i32> <i32 poison, i32 6, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP1:%.*]] = and <16 x i8> [[X:%.*]], [[SHIFT]]
+; CHECK-NEXT:    [[R:%.*]] = extractelement <16 x i8> [[TMP1]], i32 1
+; CHECK-NEXT:    ret i8 [[R]]
 ;
   %e0 = extractelement <16 x i8> %x, i32 1
   %e1 = extractelement <16 x i8> %y, i32 6
diff --git a/llvm/test/Transforms/VectorCombine/X86/extract-binop.ll b/llvm/test/Transforms/VectorCombine/X86/extract-binop.ll
index c125b73fccddf..7cbe1c6cec906 100644
--- a/llvm/test/Transforms/VectorCombine/X86/extract-binop.ll
+++ b/llvm/test/Transforms/VectorCombine/X86/extract-binop.ll
@@ -252,17 +252,11 @@ define i8 @ext1_ext1_add_uses2(<16 x i8> %x, <16 x i8> %y) {
 }
 
 define i8 @ext0_ext1_add(<16 x i8> %x, <16 x i8> %y) {
-; SSE-LABEL: @ext0_ext1_add(
-; SSE-NEXT:    [[E0:%.*]] = extractelement <16 x i8> [[X:%.*]], i32 0
-; SSE-NEXT:    [[E1:%.*]] = extractelement <16 x i8> [[Y:%.*]], i32 1
-; SSE-NEXT:    [[R:%.*]] = add nuw i8 [[E0]], [[E1]]
-; SSE-NEXT:    ret i8 [[R]]
-;
-; AVX-LABEL: @ext0_ext1_add(
-; AVX-NEXT:    [[SHIFT:%.*]] = shufflevector <16 x i8> [[Y:%.*]], <16 x i8> poison, <16 x i32> <i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; AVX-NEXT:    [[TMP1:%.*]] = add nuw <16 x i8> [[X:%.*]], [[SHIFT]]
-; AVX-NEXT:    [[R:%.*]] = extractelement <16 x i8> [[TMP1]], i32 0
-; AVX-NEXT:    ret i8 [[R]]
+; CHECK-LABEL: @ext0_ext1_add(
+; CHECK-NEXT:    [[SHIFT:%.*]] = shufflevector <16 x i8> [[Y:%.*]], <16 x i8> poison, <16 x i32> <i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP1:%.*]] = add nuw <16 x i8> [[X:%.*]], [[SHIFT]]
+; CHECK-NEXT:    [[R:%.*]] = extractelement <16 x i8> [[TMP1]], i32 0
+; CHECK-NEXT:    ret i8 [[R]]
 ;
   %e0 = extractelement <16 x i8> %x, i32 0
   %e1 = extractelement <16 x i8> %y, i32 1
@@ -271,17 +265,11 @@ define i8 @ext0_ext1_add(<16 x i8> %x, <16 x i8> %y) {
 }
 
 define i8 @ext5_ext0_add(<16 x i8> %x, <16 x i8> %y) {
-; SSE-LABEL: @ext5_ext0_add(
-; SSE-NEXT:    [[E0:%.*]] = extractelement <16 x i8> [[X:%.*]], i32 5
-; SSE-NEXT:    [[E1:%.*]] = extractelement <16 x i8> [[Y:%.*]], i32 0
-; SSE-NEXT:    [[R:%.*]] = sub nsw i8 [[E0]], [[E1]]
-; SSE-NEXT:    ret i8 [[R]]
-;
-; AVX-LABEL: @ext5_ext0_add(
-; AVX-NEXT:    [[SHIFT:%.*]] = shufflevector <16 x i8> [[X:%.*]], <16 x i8> poison, <16 x i32> <i32 5, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; AVX-NEXT:    [[TMP1:%.*]] = sub nsw <16 x i8> [[SHIFT]], [[Y:%.*]]
-; AVX-NEXT:    [[R:%.*]] = extractelement <16 x i8> [[TMP1]], i64 0
-; AVX-NEXT:    ret i8 [[R]]
+; CHECK-LABEL: @ext5_ext0_add(
+; CHECK-NEXT:    [[SHIFT:%.*]] = shufflevector <16 x i8> [[X:%.*]], <16 x i8> poison, <16 x i32> <i32 5, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP1:%.*]] = sub nsw <16 x i8> [[SHIFT]], [[Y:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = extractelement <16 x i8> [[TMP1]], i64 0
+; CHECK-NEXT:    ret i8 [[R]]
 ;
   %e0 = extractelement <16 x i8> %x, i32 5
   %e1 = extractelement <16 x i8> %y, i32 0
@@ -290,17 +278,11 @@ define i8 @ext5_ext0_add(<16 x i8> %x, <16 x i8> %y) {
 }
 
 define i8 @ext1_ext6_add(<16 x i8> %x, <16 x i8> %y) {
-; SSE-LABEL: @ext1_ext6_add(
-; SSE-NEXT:    [[E0:%.*]] = extractelement <16 x i8> [[X:%.*]], i32 1
-; SSE-NEXT:    [[E1:%.*]] = extractelement <16 x i8> [[Y:%.*]], i32 6
-; SSE-NEXT:    [[R:%.*]] = and i8 [[E0]], [[E1]]
-; SSE-NEXT:    ret i8 [[R]]
-;
-; AVX-LABEL: @ext1_ext6_add(
-; AVX-NEXT:    [[SHIFT:%.*]] = shufflevector <16 x i8> [[Y:%.*]], <16 x i8> poison, <16 x i32> <i32 poison, i32 6, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; AVX-NEXT:    [[TMP1:%.*]] = and <16 x i8> [[X:%.*]], [[SHIFT]]
-; AVX-NEXT:    [[R:%.*]] = extractelement <16 x i8> [[TMP1]], i32 1
-; AVX-NEXT:    ret i8 [[R]]
+; CHECK-LABEL: @ext1_ext6_add(
+; CHECK-NEXT:    [[SHIFT:%.*]] = shufflevector <16 x i8> [[Y:%.*]], <16 x i8> poison, <16 x i32> <i32 poison, i32 6, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP1:%.*]] = and <16 x i8> [[X:%.*]], [[SHIFT]]
+; CHECK-NEXT:    [[R:%.*]] = extractelement <16 x i8> [[TMP1]], i32 1
+; CHECK-NEXT:    ret i8 [[R]]
 ;
   %e0 = extractelement <16 x i8> %x, i32 1
   %e1 = extractelement <16 x i8> %y, i32 6
diff --git a/llvm/test/Transforms/VectorCombine/X86/extract-cmp-binop.ll b/llvm/test/Transforms/VectorCombine/X86/extract-cmp-binop.ll
index 2b0855dda6dc2..6907e12158337 100644
--- a/llvm/test/Transforms/VectorCombine/X86/extract-cmp-binop.ll
+++ b/llvm/test/Transforms/VectorCombine/X86/extract-cmp-binop.ll
@@ -85,20 +85,12 @@ define i1 @icmp_samesign_xor_v4i32(<4 x i32> %a) {
 ; add is not canonical (should be xor), but that is ok.
 
 define i1 @icmp_add_v8i32(<8 x i32> %a) {
-; SSE-LABEL: @icmp_add_v8i32(
-; SSE-NEXT:    [[E1:%.*]] = extractelement <8 x i32> [[A:%.*]], i32 7
-; SSE-NEXT:    [[E2:%.*]] = extractelement <8 x i32> [[A]], i32 2
-; SSE-NEXT:    [[CMP1:%.*]] = icmp eq i32 [[E1]], 42
-; SSE-NEXT:    [[CMP2:%.*]] = icmp eq i32 [[E2]], -8
-; SSE-NEXT:    [[R:%.*]] = add i1 [[CMP1]], [[CMP2]]
-; SSE-NEXT:    ret i1 [[R]]
-;
-; AVX-LABEL: @icmp_add_v8i32(
-; AVX-NEXT:    [[TMP1:%.*]] = icmp eq <8 x i32> [[A:%.*]], <i32 poison, i32 poison, i32 -8, i32 poison, i32 poison, i32 poison, i32 poison, i32 42>
-; AVX-NEXT:    [[SHIFT:%.*]] = shufflevector <8 x i1> [[TMP1]], <8 x i1> poison, <8 x i32> <i32 poison, i32 poison, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; AVX-NEXT:    [[TMP2:%.*]] = add <8 x i1> [[SHIFT]], [[TMP1]]
-; AVX-NEXT:    [[R:%.*]] = extractelement <8 x i1> [[TMP2]], i64 2
-; AVX-NEXT:    ret i1 [[R]]
+; CHECK-LABEL: @icmp_add_v8i32(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp eq <8 x i32> [[A:%.*]], <i32 poison, i32 poison, i32 -8, i32 poison, i32 poison, i32 poison, i32 poison, i32 42>
+; CHECK-NEXT:    [[SHIFT:%.*]] = shufflevector <8 x i1> [[TMP1]], <8 x i1> poison, <8 x i32> <i32 poison, i32 poison, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP2:%.*]] = add <8 x i1> [[SHIFT]], [[TMP1]]
+; CHECK-NEXT:    [[R:%.*]] = extractelement <8 x i1> [[TMP2]], i64 2
+; CHECK-NEXT:    ret i1 [[R]]
 ;
   %e1 = extractelement <8 x i32> %a, i32 7
   %e2 = extractelement <8 x i32> %a, i32 2
diff --git a/llvm/test/Transforms/VectorCombine/X86/extract-insert-poison.ll b/llvm/test/Transforms/VectorCombine/X86/extract-insert-poison.ll
new file mode 100644
index 0000000000000..642d07a8f3253
--- /dev/null
+++ b/llvm/test/Transforms/VectorCombine/X86/extract-insert-poison.ll
@@ -0,0 +1,201 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -passes=vector-combine -S -mtriple=x86_64-- -mattr=SSE2 | FileCheck %s --check-prefixes=CHECK,SSE
+; RUN: opt < %s -passes=vector-combine -S -mtriple=x86_64-- -mattr=AVX2 | FileCheck %s --check-prefixes=CHECK,AVX
+
+
+define <4 x double> @src_ins0_v4f64_ext0_v2f64(<4 x double> %a, <2 x double> %b) #0 {
+; CHECK-LABEL: @src_ins0_v4f64_ext0_v2f64(
+; CHECK-NEXT:    [[EXT:%.*]] = extractelement <2 x double> [[B:%.*]], i32 0
+; CHECK-NEXT:    [[INS:%.*]] = insertelement <4 x double> poison, double [[EXT]], i32 0
+; CHECK-NEXT:    ret <4 x double> [[INS]]
+;
+  %ext = extractelement <2 x double> %b, i32 0
+  %ins = insertelement <4 x double> poison, double %ext, i32 0
+  ret <4 x double> %ins
+}
+
+define <4 x double> @src_ins1_v4f64_ext0_v2f64(<4 x double> %a, <2 x double> %b) #0 {
+; CHECK-LABEL: @src_ins1_v4f64_ext0_v2f64(
+; CHECK-NEXT:    [[EXT:%.*]] = extractelement <2 x double> [[B:%.*]], i32 0
+; CHECK-NEXT:    [[INS:%.*]] = insertelement <4 x double> poison, double [[EXT]], i32 1
+; CHECK-NEXT:    ret <4 x double> [[INS]]
+;
+  %ext = extractelement <2 x double> %b, i32 0
+  %ins = insertelement <4 x double> poison, double %ext, i32 1
+  ret <4 x double> %ins
+}
+
+define <4 x double> @src_ins2_v4f64_ext0_v2f64(<4 x double> %a, <2 x double> %b) #0 {
+; SSE-LABEL: @src_ins2_v4f64_ext0_v2f64(
+; SSE-NEXT:    [[EXT:%.*]] = extractelement <2 x double> [[B:%.*]], i32 0
+; SSE-NEXT:    [[INS:%.*]] = insertelement <4 x double> poison, double [[EXT]], i32 2
+; SSE-NEXT:    ret <4 x double> [[INS]]
+;
+; AVX-LABEL: @src_ins2_v4f64_ext0_v2f64(
+; AVX-NEXT:    [[TMP1:%.*]] = shufflevector <2 x double> [[B:%.*]], <2 x double> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+; AVX-NEXT:    [[INS:%.*]] = shufflevector <4 x double> [[TMP1]], <4 x double> poison, <4 x i32> <i32 poison, i32 poison, i32 0, i32 poison>
+; AVX-NEXT:    ret <4 x double> [[INS]]
+;
+  %ext = extractelement <2 x double> %b, i32 0
+  %ins = insertelement <4 x double> poison, double %ext, i32 2
+  ret <4 x double> %ins
+}
+
+define <4 x double> @src_ins3_v4f64_ext0_v2f64(<4 x double> %a, <2 x double> %b) #0 {
+; SSE-LABEL: @src_ins3_v4f64_ext0_v2f64(
+; SSE-NEXT:    [[EXT:%.*]] = extractelement <2 x double> [[B:%.*]], i32 0
+; SSE-NEXT:    [[INS:%.*]] = insertelement <4 x double> poison, double [[EXT]], i32 3
+; SSE-NEXT:    ret <4 x double> [[INS]]
+;
+; AVX-LABEL: @src_ins3_v4f64_ext0_v2f64(
+; AVX-NEXT:    [[TMP1:%.*]] = shufflevector <2 x double> [[B:%.*]], <2 x double> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+; AVX-NEXT:    [[INS:%.*]] = shufflevector <4 x double> [[TMP1]], <4 x double> poison, <4 x i32> <i32 poison, i32 poison, i32 poison, i32 0>
+; AVX-NEXT:    ret <4 x double> [[INS]]
+;
+  %ext = extractelement <2 x double> %b, i32 0
+  %ins = insertelement <4 x double> poison, double %ext, i32 3
+  ret <4 x double> %ins
+}
+
+define <4 x double> @src_ins0_v4f64_ext1_v2f64(<4 x double> %a, <2 x double> %b) #0 {
+; CHECK-LABEL: @src_ins0_v4f64_ext1_v2f64(
+; CHECK-NEXT:    [[EXT:%.*]] = extractelement <2 x double> [[B:%.*]], i32 1
+; CHECK-NEXT:    [[INS:%.*]] = insertelement <4 x double> poison, double [[EXT]], i32 0
+; CHECK-NEXT:    ret <4 x double> [[INS]]
+;
+  %ext = extractelement <2 x double> %b, i32 1
+  %ins = insertelement <4 x double> poison, double %ext, i32 0
+  ret <4 x double> %ins
+}
+
+define <4 x double> @src_ins1_v4f64_ext1_v2f64(<4 x double> %a, <2 x double> %b) #0 {
+; CHECK-LABEL: @src_ins1_v4f64_ext1_v2f64(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <2 x double> [[B:%.*]], <2 x double> poison, <4 x i32> <i32 poison, i32 1, i32 poison, i32 poison>
+; CHECK-NEXT:    [[INS:%.*]] = shufflevector <4 x double> [[TMP1]], <4 x double> poison, <4 x i32> <i32 poison, i32 1, i32 poison, i32 poison>
+; CHECK-NEXT:    ret <4 x double> [[INS]]
+;
+  %ext = extractelement <2 x double> %b, i32 1
+  %ins = insertelement <4 x double> poison, double %ext, i32 1
+  ret <4 x double> %ins
+}
+
+define <4 x double> @src_ins2_v4f64_ext1_v2f64(<4 x double> %a, <2 x double> %b) #0 {
+; SSE-LABEL: @src_ins2_v4f64_ext1_v2f64(
+; SSE-NEXT:    [[EXT:%.*]] = extractelement <2 x double> [[B:%.*]], i32 1
+; SSE-NEXT:    [[INS:%.*]] = insertelement <4 x double> poison, double [[EXT]], i32 2
+; SSE-NEXT:    ret <4 x double> [[INS]]
+;
+; AVX-LABEL: @src_ins2_v4f64_ext1_v2f64(
+; AVX-NEXT:    [[TMP1:%.*]] = shufflevector <2 x double> [[B:%.*]], <2 x double> poison, <4 x i32> <i32 poison, i32 1, i32 poison, i32 poison>
+; AVX-NEXT:    [[INS:%.*]] = shufflevector <4 x double> [[TMP1]], <4 x double> poison, <4 x i32> <i32 poison, i32 poison, i32 1, i32 poison>
+; AVX-NEXT:    ret <4 x double> [[INS]]
+;
+  %ext = extractelement <2 x double> %b, i32 1
+  %ins = insertelement <4 x double> poison, double %ext, i32 2
+  ret <4 x double> %ins
+}
+
+define <4 x double> @src_ins3_v4f64_ext1_v2f64(<4 x double> %a, <2 x double> %b) #0 {
+; CHECK-LABEL: @src_ins3_v4f64_ext1_v2f64(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <2 x double> [[B:%.*]], <2 x double> poison, <4 x i32> <i32 poison, i32 1, i32 poison, i32 poison>
+; CHECK-NEXT:    [[INS:%.*]] = shufflevector <4 x double> [[TMP1]], <4 x double> poison, <4 x i32> <i32 poison, i32 poison, i32 poison, i32 1>
+; CHECK-NEXT:    ret <4 x double> [[INS]]
+;
+  %ext = extractelement <2 x double> %b, i32 1
+  %ins = insertelement <4 x double> poison, double %ext, i32 3
+  ret <4 x double> %ins
+}
+
+define <2 x double> @src_ins0_v2f64_ext0_v4f64(<2 x double> %a, <4 x double> %b) {
+; CHECK-LABEL: @src_ins0_v2f64_ext0_v4f64(
+; CHECK-NEXT:    [[EXT:%.*]] = extractelement <4 x double> [[B:%.*]], i32 0
+; CHECK-NEXT:    [[INS:%.*]] = insertelement <2 x double> poison, double [[EXT]], i32 0
+; CHECK-NEXT:    ret <2 x double> [[INS]]
+;
+  %ext = extractelement <4 x double> %b, i32 0
+  %ins = insertelement <2 x double> poison, double %ext, i32 0
+  ret <2 x double> %ins
+}
+
+define <2 x double> @src_ins0_v2f64_ext1_v4f64(<2 x double> %a, <4 x double> %b) {
+; CHECK-LABEL: @src_ins0_v2f64_ext1_v4f64(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x double> [[B:%.*]], <4 x double> poison, <2 x i32> <i32 poison, i32 1>
+; CHECK-NEXT:    [[INS:%.*]] = shufflevector <2 x double> [[TMP1]], <2 x double> poison, <2 x i32> <i32 1, i32 poison>
+; CHECK-NEXT:    ret <2 x double> [[INS]]
+;
+  %ext = extractelement <4 x double> %b, i32 1
+  %ins = insertelement <2 x double> poison, double %ext, i32 0
+  ret <2 x double> %ins
+}
+
+define <2 x double> @src_ins0_v2f64_ext2_v4f64(<2 x double> %a, <4 x double> %b) {
+; CHECK-LABEL: @src_ins0_v2f64_ext2_v4f64(
+; CHECK-NEXT:    [[EXT:%.*]] = extractelement <4 x double> [[B:%.*]], i32 2
+; CHECK-NEXT:    [[INS:%.*]] = insertelement <2 x double> poison, double [[EXT]], i32 0
+; CHECK-NEXT:    ret <2 x double> [[INS]]
+;
+  %ext = extractelement <4 x double> %b, i32 2
+  %ins = insertelement <2 x double> poison, double %ext, i32 0
+  ret <2 x double> %ins
+}
+
+define <2 x double> @src_ins0_v2f64_ext3_v4f64(<2 x double> %a, <4 x double> %b) {
+; CHECK-LABEL: @src_ins0_v2f64_ext3_v4f64(
+; CHECK-NEXT:    [[EXT:%.*]] = extractelement <4 x double> [[B:%.*]], i32 3
+; CHECK-NEXT:    [[INS:%.*]] = insertelement <2 x double> poison, double [[EXT]], i32 0
+; CHECK-NEXT:    ret <2 x double> [[INS]]
+;
+  %ext = extractelement <4 x double> %b, i32 3
+  %ins = insertelement <2 x double> poison, double %ext, i32 0
+  ret <2 x double> %ins
+}
+
+define <2 x double> @src_ins1_v2f64_ext0_v4f64(<2 x double> %a, <4 x double> %b) {
+; CHECK-LABEL: @src_ins1_v2f64_ext0_v4f64(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x double> [[B:%.*]], <4 x double> poison, <2 x i32> <i32 0, i32 poison>
+; CHECK-NEXT:    [[INS:%.*]] = shufflevector <2 x double> [[TMP1]], <2 x double> poison, <2 x i32> <i32 poison, i32 0>
+; CHECK-NEXT:    ret <2 x double> [[INS]]
+;
+  %ext = extractelement <4 x double> %b, i32 0
+  %ins = insertelement <2 x double> poison, double %ext, i32 1
+  ret <2 x double> %ins
+}
+
+define <2 x double> @src_ins1_v2f64_ext1_v4f64(<2 x double> %a, <4 x double> %b) {
+; CHECK-LABEL: @src_ins1_v2f64_ext1_v4f64(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x double> [[B:%.*]], <4 x double> poison, <2 x i32> <i32 poison, i32 1>
+; CHECK-NEXT:    [[INS:%.*]] = shufflevector <2 x double> [[TMP1]], <2 x double> poison, <2 x i32> <i32 poison, i32 1>
+; CHECK-NEXT:    ret <2 x double> [[INS]]
+;
+  %ext = extractelement <4 x double> %b, i32 1
+  %ins = insertelement <2 x double> poison, double %ext, i32 1
+  ret <2 x double> %ins
+}
+
+define <2 x double> @src_ins1_v2f64_ext2_v4f64(<2 x double> %a, <4 x double> %b) {
+; SSE-LABEL: @src_ins1_v2f64_ext2_v4f64(
+; SSE-NEXT:    [[TMP1:%.*]] = shufflevector <4 x double> [[B:%.*]], <4 x double> poison, <2 x i32> <i32 2, i32 poison>
+; SSE-NEXT:    [[INS:%.*]] = shufflevector <2 x double> [[TMP1]], <2 x double> poison, <2 x i32> <i32 poison, i32 0>
+; SSE-NEXT:    ret <2 x double> [[INS]]
+;
+; AVX-LABEL: @src_ins1_v2f64_ext2_v4f64(
+; AVX-NEXT:    [[EXT:%.*]] = extractelement <4 x double> [[B:%.*]], i32 2
+; AVX-NEXT:    [[INS:%.*]] = insertelement <2 x double> poison, double [[EXT]], i32 1
+; AVX-NEXT:    ret <2 x double> [[INS]]
+;
+  %ext = extractelement <4 x double> %b, i32 2
+  %ins = insertelement <2 x double> poison, double %ext, i32 1
+  ret <2 x double> %ins
+}
+
+define <2 x double> @src_ins1_v2f64_ext3_v4f64(<2 x double> %a, <4 x double> %b) {
+; CHECK-LABEL: @src_ins1_v2f64_ext3_v4f64(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x double> [[B:%.*]], <4 x double> poison, <2 x i32> <i32 3, i32 poison>
+; CHECK-NEXT:    [[INS:%.*]] = shufflevector <2 x double> [[TMP1]], <2 x double> poison, <2 x i32> <i32 poison, i32 0>
+; CHECK-NEXT:    ret <2 x double> [[INS]]
+;
+  %ext = extractelement <4 x double> %b, i32 3
+  %ins = insertelement <2 x double> poison, double %ext, i32 1
+  ret <2 x double> %ins
+}
+
diff --git a/llvm/test/Transforms/VectorCombine/X86/extract-insert.ll b/llvm/test/Transforms/VectorCombine/X86/extract-insert.ll
new file mode 100644
index 0000000000000..193ad36616a4a
--- /dev/null
+++ b/llvm/test/Transforms/VectorCombine/X86/extract-insert.ll
@@ -0,0 +1,196 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -passes=vector-combine -S -mtriple=x86_64-- -mattr=SSE2 | FileCheck %s --check-prefixes=CHECK,SSE
+; RUN: opt < %s -passes=vector-combine -S -mtriple=x86_64-- -mattr=AVX2 | FileCheck %s --check-prefixes=CHECK,AVX
+
+
+define <4 x double> @src_ins0_v4f64_ext0_v2f64(<4 x double> %a, <2 x double> %b) #0 {
+; CHECK-LABEL: @src_ins0_v4f64_ext0_v2f64(
+; CHECK-NEXT:    [[EXT:%.*]] = extractelement <2 x double> [[B:%.*]], i32 0
+; CHECK-NEXT:    [[INS:%.*]] = insertelement <4 x double> [[A:%.*]], double [[EXT]], i32 0
+; CHECK-NEXT:    ret <4 x double> [[INS]]
+;
+  %ext = extractelement <2 x double> %b, i32 0
+  %ins = insertelement <4 x double> %a, double %ext, i32 0
+  ret <4 x double> %ins
+}
+
+define <4 x double> @src_ins1_v4f64_ext0_v2f64(<4 x double> %a, <2 x double> %b) #0 {
+; CHECK-LABEL: @src_ins1_v4f64_ext0_v2f64(
+; CHECK-NEXT:    [[EXT:%.*]] = extractelement <2 x double> [[B:%.*]], i32 0
+; CHECK-NEXT:    [[INS:%.*]] = insertelement <4 x double> [[A:%.*]], double [[EXT]], i32 1
+; CHECK-NEXT:    ret <4 x double> [[INS]]
+;
+  %ext = extractelement <2 x double> %b, i32 0
+  %ins = insertelement <4 x double> %a, double %ext, i32 1
+  ret <4 x double> %ins
+}
+
+define <4 x double> @src_ins2_v4f64_ext0_v2f64(<4 x double> %a, <2 x double> %b) #0 {
+; CHECK-LABEL: @src_ins2_v4f64_ext0_v2f64(
+; CHECK-NEXT:    [[EXT:%.*]] = extractelement <2 x double> [[B:%.*]], i32 0
+; CHECK-NEXT:    [[INS:%.*]] = insertelement <4 x double> [[A:%.*]], double [[EXT]], i32 2
+; CHECK-NEXT:    ret <4 x double> [[INS]]
+;
+  %ext = extractelement <2 x double> %b, i32 0
+  %ins = insertelement <4 x double> %a, double %ext, i32 2
+  ret <4 x double> %ins
+}
+
+define <4 x double> @src_ins3_v4f64_ext0_v2f64(<4 x double> %a, <2 x double> %b) #0 {
+; CHECK-LABEL: @src_ins3_v4f64_ext0_v2f64(
+; CHECK-NEXT:    [[EXT:%.*]] = extractelement <2 x double> [[B:%.*]], i32 0
+; CHECK-NEXT:    [[INS:%.*]] = insertelement <4 x double> [[A:%.*]], double [[EXT]], i32 3
+; CHECK-NEXT:    ret <4 x double> [[INS]]
+;
+  %ext = extractelement <2 x double> %b, i32 0
+  %ins = insertelement <4 x double> %a, double %ext, i32 3
+  ret <4 x double> %ins
+}
+
+define <4 x double> @src_ins0_v4f64_ext1_v2f64(<4 x double> %a, <2 x double> %b) #0 {
+; SSE-LABEL: @src_ins0_v4f64_ext1_v2f64(
+; SSE-NEXT:    [[TMP1:%.*]] = shufflevector <2 x double> [[B:%.*]], <2 x double> poison, <4 x i32> <i32 poison, i32 1, i32 poison, i32 poison>
+; SSE-NEXT:    [[INS:%.*]] = shufflevector <4 x double> [[A:%.*]], <4 x double> [[TMP1]], <4 x i32> <i32 5, i32 1, i32 2, i32 3>
+; SSE-NEXT:    ret <4 x double> [[INS]]
+;
+; AVX-LABEL: @src_ins0_v4f64_ext1_v2f64(
+; AVX-NEXT:    [[EXT:%.*]] = extractelement <2 x double> [[B:%.*]], i32 1
+; AVX-NEXT:    [[INS:%.*]] = insertelement <4 x double> [[A:%.*]], double [[EXT]], i32 0
+; AVX-NEXT:    ret <4 x double> [[INS]]
+;
+  %ext = extractelement <2 x double> %b, i32 1
+  %ins = insertelement <4 x double> %a, double %ext, i32 0
+  ret <4 x double> %ins
+}
+
+define <4 x double> @src_ins1_v4f64_ext1_v2f64(<4 x double> %a, <2 x double> %b) #0 {
+; CHECK-LABEL: @src_ins1_v4f64_ext1_v2f64(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <2 x double> [[B:%.*]], <2 x double> poison, <4 x i32> <i32 poison, i32 1, i32 poison, i32 poison>
+; CHECK-NEXT:    [[INS:%.*]] = shufflevector <4 x double> [[A:%.*]], <4 x double> [[TMP1]], <4 x i32> <i32 0, i32 5, i32 2, i32 3>
+; CHECK-NEXT:    ret <4 x double> [[INS]]
+;
+  %ext = extractelement <2 x double> %b, i32 1
+  %ins = insertelement <4 x double> %a, double %ext, i32 1
+  ret <4 x double> %ins
+}
+
+define <4 x double> @src_ins2_v4f64_ext1_v2f64(<4 x double> %a, <2 x double> %b) #0 {
+; CHECK-LABEL: @src_ins2_v4f64_ext1_v2f64(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <2 x double> [[B:%.*]], <2 x double> poison, <4 x i32> <i32 poison, i32 1, i32 poison, i32 poison>
+; CHECK-NEXT:    [[INS:%.*]] = shufflevector <4 x double> [[A:%.*]], <4 x double> [[TMP1]], <4 x i32> <i32 0, i32 1, i32 5, i32 3>
+; CHECK-NEXT:    ret <4 x double> [[INS]]
+;
+  %ext = extractelement <2 x double> %b, i32 1
+  %ins = insertelement <4 x double> %a, double %ext, i32 2
+  ret <4 x double> %ins
+}
+
+define <4 x double> @src_ins3_v4f64_ext1_v2f64(<4 x double> %a, <2 x double> %b) #0 {
+; CHECK-LABEL: @src_ins3_v4f64_ext1_v2f64(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <2 x double> [[B:%.*]], <2 x double> poison, <4 x i32> <i32 poison, i32 1, i32 poison, i32 poison>
+; CHECK-NEXT:    [[INS:%.*]] = shufflevector <4 x double> [[A:%.*]], <4 x double> [[TMP1]], <4 x i32> <i32 0, i32 1, i32 2, i32 5>
+; CHECK-NEXT:    ret <4 x double> [[INS]]
+;
+  %ext = extractelement <2 x double> %b, i32 1
+  %ins = insertelement <4 x double> %a, double %ext, i32 3
+  ret <4 x double> %ins
+}
+
+define <2 x double> @src_ins0_v2f64_ext0_v4f64(<2 x double> %a, <4 x double> %b) {
+; CHECK-LABEL: @src_ins0_v2f64_ext0_v4f64(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x double> [[B:%.*]], <4 x double> poison, <2 x i32> <i32 0, i32 poison>
+; CHECK-NEXT:    [[INS:%.*]] = shufflevector <2 x double> [[A:%.*]], <2 x double> [[TMP1]], <2 x i32> <i32 2, i32 1>
+; CHECK-NEXT:    ret <2 x double> [[INS]]
+;
+  %ext = extractelement <4 x double> %b, i32 0
+  %ins = insertelement <2 x double> %a, double %ext, i32 0
+  ret <2 x double> %ins
+}
+
+define <2 x double> @src_ins0_v2f64_ext1_v4f64(<2 x double> %a, <4 x double> %b) {
+; CHECK-LABEL: @src_ins0_v2f64_ext1_v4f64(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x double> [[B:%.*]], <4 x double> poison, <2 x i32> <i32 poison, i32 1>
+; CHECK-NEXT:    [[INS:%.*]] = shufflevector <2 x double> [[A:%.*]], <2 x double> [[TMP1]], <2 x i32> <i32 3, i32 1>
+; CHECK-NEXT:    ret <2 x double> [[INS]]
+;
+  %ext = extractelement <4 x double> %b, i32 1
+  %ins = insertelement <2 x double> %a, double %ext, i32 0
+  ret <2 x double> %ins
+}
+
+define <2 x double> @src_ins0_v2f64_ext2_v4f64(<2 x double> %a, <4 x double> %b) {
+; SSE-LABEL: @src_ins0_v2f64_ext2_v4f64(
+; SSE-NEXT:    [[TMP1:%.*]] = shufflevector <4 x double> [[B:%.*]], <4 x double> poison, <2 x i32> <i32 2, i32 poison>
+; SSE-NEXT:    [[INS:%.*]] = shufflevector <2 x double> [[A:%.*]], <2 x double> [[TMP1]], <2 x i32> <i32 2, i32 1>
+; SSE-NEXT:    ret <2 x double> [[INS]]
+;
+; AVX-LABEL: @src_ins0_v2f64_ext2_v4f64(
+; AVX-NEXT:    [[EXT:%.*]] = extractelement <4 x double> [[B:%.*]], i32 2
+; AVX-NEXT:    [[INS:%.*]] = insertelement <2 x double> [[A:%.*]], double [[EXT]], i32 0
+; AVX-NEXT:    ret <2 x double> [[INS]]
+;
+  %ext = extractelement <4 x double> %b, i32 2
+  %ins = insertelement <2 x double> %a, double %ext, i32 0
+  ret <2 x double> %ins
+}
+
+define <2 x double> @src_ins0_v2f64_ext3_v4f64(<2 x double> %a, <4 x double> %b) {
+; CHECK-LABEL: @src_ins0_v2f64_ext3_v4f64(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x double> [[B:%.*]], <4 x double> poison, <2 x i32> <i32 3, i32 poison>
+; CHECK-NEXT:    [[INS:%.*]] = shufflevector <2 x double> [[A:%.*]], <2 x double> [[TMP1]], <2 x i32> <i32 2, i32 1>
+; CHECK-NEXT:    ret <2 x double> [[INS]]
+;
+  %ext = extractelement <4 x double> %b, i32 3
+  %ins = insertelement <2 x double> %a, double %ext, i32 0
+  ret <2 x double> %ins
+}
+
+define <2 x double> @src_ins1_v2f64_ext0_v4f64(<2 x double> %a, <4 x double> %b) {
+; CHECK-LABEL: @src_ins1_v2f64_ext0_v4f64(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x double> [[B:%.*]], <4 x double> poison, <2 x i32> <i32 0, i32 poison>
+; CHECK-NEXT:    [[INS:%.*]] = shufflevector <2 x double> [[A:%.*]], <2 x double> [[TMP1]], <2 x i32> <i32 0, i32 2>
+; CHECK-NEXT:    ret <2 x double> [[INS]]
+;
+  %ext = extractelement <4 x double> %b, i32 0
+  %ins = insertelement <2 x double> %a, double %ext, i32 1
+  ret <2 x double> %ins
+}
+
+define <2 x double> @src_ins1_v2f64_ext1_v4f64(<2 x double> %a, <4 x double> %b) {
+; CHECK-LABEL: @src_ins1_v2f64_ext1_v4f64(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x double> [[B:%.*]], <4 x double> poison, <2 x i32> <i32 poison, i32 1>
+; CHECK-NEXT:    [[INS:%.*]] = shufflevector <2 x double> [[A:%.*]], <2 x double> [[TMP1]], <2 x i32> <i32 0, i32 3>
+; CHECK-NEXT:    ret <2 x double> [[INS]]
+;
+  %ext = extractelement <4 x double> %b, i32 1
+  %ins = insertelement <2 x double> %a, double %ext, i32 1
+  ret <2 x double> %ins
+}
+
+define <2 x double> @src_ins1_v2f64_ext2_v4f64(<2 x double> %a, <4 x double> %b) {
+; SSE-LABEL: @src_ins1_v2f64_ext2_v4f64(
+; SSE-NEXT:    [[TMP1:%.*]] = shufflevector <4 x double> [[B:%.*]], <4 x double> poison, <2 x i32> <i32 2, i32 poison>
+; SSE-NEXT:    [[INS:%.*]] = shufflevector <2 x double> [[A:%.*]], <2 x double> [[TMP1]], <2 x i32> <i32 0, i32 2>
+; SSE-NEXT:    ret <2 x double> [[INS]]
+;
+; AVX-LABEL: @src_ins1_v2f64_ext2_v4f64(
+; AVX-NEXT:    [[EXT:%.*]] = extractelement <4 x double> [[B:%.*]], i32 2
+; AVX-NEXT:    [[INS:%.*]] = insertelement <2 x double> [[A:%.*]], double [[EXT]], i32 1
+; AVX-NEXT:    ret <2 x double> [[INS]]
+;
+  %ext = extractelement <4 x double> %b, i32 2
+  %ins = insertelement <2 x double> %a, double %ext, i32 1
+  ret <2 x double> %ins
+}
+
+define <2 x double> @src_ins1_v2f64_ext3_v4f64(<2 x double> %a, <4 x double> %b) {
+; CHECK-LABEL: @src_ins1_v2f64_ext3_v4f64(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x double> [[B:%.*]], <4 x double> poison, <2 x i32> <i32 3, i32 poison>
+; CHECK-NEXT:    [[INS:%.*]] = shufflevector <2 x double> [[A:%.*]], <2 x double> [[TMP1]], <2 x i32> <i32 0, i32 2>
+; CHECK-NEXT:    ret <2 x double> [[INS]]
+;
+  %ext = extractelement <4 x double> %b, i32 3
+  %ins = insertelement <2 x double> %a, double %ext, i32 1
+  ret <2 x double> %ins
+}
+
diff --git a/llvm/test/Transforms/VectorCombine/X86/insert-binop-vector.ll b/llvm/test/Transforms/VectorCombine/X86/insert-binop-vector.ll
new file mode 100644
index 0000000000000..3a356f0027336
--- /dev/null
+++ b/llvm/test/Transforms/VectorCombine/X86/insert-binop-vector.ll
@@ -0,0 +1,155 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -passes=vector-combine -S -mtriple=x86_64-- -mcpu=x86-64    | FileCheck %s --check-prefixes=CHECK,SSE,SSE2
+; RUN: opt < %s -passes=vector-combine -S -mtriple=x86_64-- -mcpu=x86-64-v2 | FileCheck %s --check-prefixes=CHECK,SSE,SSE4
+; RUN: opt < %s -passes=vector-combine -S -mtriple=x86_64-- -mcpu=x86-64-v3 | FileCheck %s --check-prefixes=CHECK,AVX
+; RUN: opt < %s -passes=vector-combine -S -mtriple=x86_64-- -mcpu=x86-64-v4 | FileCheck %s --check-prefixes=CHECK,AVX
+
+define <2 x double> @insert1_v2f64_f64_fdiv(<2 x double> %v0, <2 x double> %v1, double %s0, double %s1) {
+; CHECK-LABEL: @insert1_v2f64_f64_fdiv(
+; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <2 x double> [[V0:%.*]], double [[S0:%.*]], i64 1
+; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <2 x double> [[V1:%.*]], double [[S1:%.*]], i64 1
+; CHECK-NEXT:    [[R:%.*]] = fdiv <2 x double> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    ret <2 x double> [[R]]
+;
+  %s = fdiv double %s0, %s1
+  %v = fdiv <2 x double> %v0, %v1
+  %r = insertelement <2 x double> %v, double %s, i32 1
+  ret <2 x double> %r
+}
+
+; SSE2 has no fast v4i32 insertelement
+define <4 x i32> @insert0_v4i32_i32_add(<4 x i32> %v0, <4 x i32> %v1, i32 %s0, i32 %s1) {
+; SSE2-LABEL: @insert0_v4i32_i32_add(
+; SSE2-NEXT:    [[S:%.*]] = add i32 [[S0:%.*]], [[S1:%.*]]
+; SSE2-NEXT:    [[V:%.*]] = add <4 x i32> [[V0:%.*]], [[V1:%.*]]
+; SSE2-NEXT:    [[R:%.*]] = insertelement <4 x i32> [[V]], i32 [[S]], i32 0
+; SSE2-NEXT:    ret <4 x i32> [[R]]
+;
+; SSE4-LABEL: @insert0_v4i32_i32_add(
+; SSE4-NEXT:    [[TMP1:%.*]] = insertelement <4 x i32> [[V0:%.*]], i32 [[S0:%.*]], i64 0
+; SSE4-NEXT:    [[TMP2:%.*]] = insertelement <4 x i32> [[V1:%.*]], i32 [[S1:%.*]], i64 0
+; SSE4-NEXT:    [[R:%.*]] = add <4 x i32> [[TMP1]], [[TMP2]]
+; SSE4-NEXT:    ret <4 x i32> [[R]]
+;
+; AVX-LABEL: @insert0_v4i32_i32_add(
+; AVX-NEXT:    [[TMP1:%.*]] = insertelement <4 x i32> [[V0:%.*]], i32 [[S0:%.*]], i64 0
+; AVX-NEXT:    [[TMP2:%.*]] = insertelement <4 x i32> [[V1:%.*]], i32 [[S1:%.*]], i64 0
+; AVX-NEXT:    [[R:%.*]] = add <4 x i32> [[TMP1]], [[TMP2]]
+; AVX-NEXT:    ret <4 x i32> [[R]]
+;
+  %s = add i32 %s0, %s1
+  %v = add <4 x i32> %v0, %v1
+  %r = insertelement <4 x i32> %v, i32 %s, i32 0
+  ret <4 x i32> %r
+}
+
+; AVX expensive insertion into upper 128-bit vector
+define <16 x i16> @insert9_v16i16_i16_add(<16 x i16> %v0, <16 x i16> %v1, i16 %s0, i16 %s1) {
+; SSE-LABEL: @insert9_v16i16_i16_add(
+; SSE-NEXT:    [[TMP1:%.*]] = insertelement <16 x i16> [[V0:%.*]], i16 [[S0:%.*]], i64 9
+; SSE-NEXT:    [[TMP2:%.*]] = insertelement <16 x i16> [[V1:%.*]], i16 [[S1:%.*]], i64 9
+; SSE-NEXT:    [[R:%.*]] = add <16 x i16> [[TMP1]], [[TMP2]]
+; SSE-NEXT:    ret <16 x i16> [[R]]
+;
+; AVX-LABEL: @insert9_v16i16_i16_add(
+; AVX-NEXT:    [[S:%.*]] = add i16 [[S0:%.*]], [[S1:%.*]]
+; AVX-NEXT:    [[V:%.*]] = add <16 x i16> [[V0:%.*]], [[V1:%.*]]
+; AVX-NEXT:    [[R:%.*]] = insertelement <16 x i16> [[V]], i16 [[S]], i32 9
+; AVX-NEXT:    ret <16 x i16> [[R]]
+;
+  %s = add i16 %s0, %s1
+  %v = add <16 x i16> %v0, %v1
+  %r = insertelement <16 x i16> %v, i16 %s, i32 9
+  ret <16 x i16> %r
+}
+
+; Merge flags
+define <4 x float> @insert0_v4f32_f32_fadd_common_flags(<4 x float> %v0, <4 x float> %v1, float %s0, float %s1) {
+; CHECK-LABEL: @insert0_v4f32_f32_fadd_common_flags(
+; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <4 x float> [[V0:%.*]], float [[S0:%.*]], i64 0
+; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <4 x float> [[V1:%.*]], float [[S1:%.*]], i64 0
+; CHECK-NEXT:    [[R:%.*]] = fadd fast <4 x float> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    ret <4 x float> [[R]]
+;
+  %s = fadd fast float %s0, %s1
+  %v = fadd fast <4 x float> %v0, %v1
+  %r = insertelement <4 x float> %v, float %s, i32 0
+  ret <4 x float> %r
+}
+
+; Merge (shared) flags
+define <4 x float> @insert1_v4f32_f32_fsub_mixed_flags(<4 x float> %v0, <4 x float> %v1, float %s0, float %s1) {
+; CHECK-LABEL: @insert1_v4f32_f32_fsub_mixed_flags(
+; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <4 x float> [[V0:%.*]], float [[S0:%.*]], i64 1
+; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <4 x float> [[V1:%.*]], float [[S1:%.*]], i64 1
+; CHECK-NEXT:    [[R:%.*]] = fsub nnan <4 x float> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    ret <4 x float> [[R]]
+;
+  %s = fsub nnan nsz float %s0, %s1
+  %v = fsub nnan ninf <4 x float> %v0, %v1
+  %r = insertelement <4 x float> %v, float %s, i32 1
+  ret <4 x float> %r
+}
+
+; TODO: Fold equivalent opcodes
+define <4 x i32> @insert0_v4i32_i32_or_disjoint_add(<4 x i32> %v0, <4 x i32> %v1, i32 %s0, i32 %s1) {
+; CHECK-LABEL: @insert0_v4i32_i32_or_disjoint_add(
+; CHECK-NEXT:    [[S:%.*]] = add i32 [[S0:%.*]], [[S1:%.*]]
+; CHECK-NEXT:    [[V:%.*]] = or disjoint <4 x i32> [[V0:%.*]], [[V1:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = insertelement <4 x i32> [[V]], i32 [[S]], i32 0
+; CHECK-NEXT:    ret <4 x i32> [[R]]
+;
+  %s = add i32 %s0, %s1
+  %v = or disjoint <4 x i32> %v0, %v1
+  %r = insertelement <4 x i32> %v, i32 %s, i32 0
+  ret <4 x i32> %r
+}
+
+; Negative - multi use
+define <2 x double> @insert0_v2f64_f64_fmul_multiuse(<2 x double> %v0, <2 x double> %v1, double %s0, double %s1) {
+; CHECK-LABEL: @insert0_v2f64_f64_fmul_multiuse(
+; CHECK-NEXT:    [[S:%.*]] = fmul double [[S0:%.*]], [[S1:%.*]]
+; CHECK-NEXT:    [[V:%.*]] = fmul <2 x double> [[V0:%.*]], [[V1:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = insertelement <2 x double> [[V]], double [[S]], i32 0
+; CHECK-NEXT:    call void @use_f64(double [[S]])
+; CHECK-NEXT:    ret <2 x double> [[R]]
+;
+  %s = fmul double %s0, %s1
+  %v = fmul <2 x double> %v0, %v1
+  %r = insertelement <2 x double> %v, double %s, i32 0
+  call void @use_f64(double %s)
+  ret <2 x double> %r
+}
+declare void @use_f64(<2 x double>)
+
+; Negative - multi use
+define <2 x i64> @insert0_v2i64_i64_add_multiuse(<2 x i64> %v0, <2 x i64> %v1, i64 %s0, i64 %s1) {
+; CHECK-LABEL: @insert0_v2i64_i64_add_multiuse(
+; CHECK-NEXT:    [[S:%.*]] = add i64 [[S0:%.*]], [[S1:%.*]]
+; CHECK-NEXT:    [[V:%.*]] = add <2 x i64> [[V0:%.*]], [[V1:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = insertelement <2 x i64> [[V]], i64 [[S]], i32 0
+; CHECK-NEXT:    call void @use_v2i64(<2 x i64> [[V]])
+; CHECK-NEXT:    ret <2 x i64> [[R]]
+;
+  %s = add i64 %s0, %s1
+  %v = add <2 x i64> %v0, %v1
+  %r = insertelement <2 x i64> %v, i64 %s, i32 0
+  call void @use_v2i64(<2 x i64> %v)
+  ret <2 x i64> %r
+}
+declare void @use_v2i64(<2 x i64>)
+
+; Negative - binop mismatch
+define <2 x double> @insert0_v2f64_f64_fadd_fsub(<2 x double> %v0, <2 x double> %v1, double %s0, double %s1) {
+; CHECK-LABEL: @insert0_v2f64_f64_fadd_fsub(
+; CHECK-NEXT:    [[S:%.*]] = fsub double [[S0:%.*]], [[S1:%.*]]
+; CHECK-NEXT:    [[V:%.*]] = fadd <2 x double> [[V0:%.*]], [[V1:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = insertelement <2 x double> [[V]], double [[S]], i32 0
+; CHECK-NEXT:    ret <2 x double> [[R]]
+;
+  %s = fsub double %s0, %s1
+  %v = fadd <2 x double> %v0, %v1
+  %r = insertelement <2 x double> %v, double %s, i32 0
+  ret <2 x double> %r
+}
+
diff --git a/llvm/test/Transforms/VectorCombine/X86/load-inseltpoison.ll b/llvm/test/Transforms/VectorCombine/X86/load-inseltpoison.ll
index 2db1e21b3e95a..73476308916fb 100644
--- a/llvm/test/Transforms/VectorCombine/X86/load-inseltpoison.ll
+++ b/llvm/test/Transforms/VectorCombine/X86/load-inseltpoison.ll
@@ -1,6 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -passes=vector-combine -S -mtriple=x86_64-- -mattr=sse2 | FileCheck %s --check-prefixes=CHECK,SSE2
-; RUN: opt < %s -passes=vector-combine -S -mtriple=x86_64-- -mattr=avx2 | FileCheck %s --check-prefixes=CHECK,AVX2
+; RUN: opt < %s -passes=vector-combine -S -mtriple=x86_64-- -mattr=sse2 | FileCheck %s
+; RUN: opt < %s -passes=vector-combine -S -mtriple=x86_64-- -mattr=avx2 | FileCheck %s
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 
@@ -265,16 +265,10 @@ define <8 x i16> @gep01_load_i16_insert_v8i16(ptr align 16 dereferenceable(18) %
 ; Can't safely load the offset vector, but can load+shuffle if it is profitable.
 
 define <8 x i16> @gep01_load_i16_insert_v8i16_deref(ptr align 16 dereferenceable(17) %p) nofree nosync {
-; SSE2-LABEL: @gep01_load_i16_insert_v8i16_deref(
-; SSE2-NEXT:    [[GEP:%.*]] = getelementptr inbounds <8 x i16>, ptr [[P:%.*]], i64 0, i64 1
-; SSE2-NEXT:    [[S:%.*]] = load i16, ptr [[GEP]], align 2
-; SSE2-NEXT:    [[R:%.*]] = insertelement <8 x i16> poison, i16 [[S]], i64 0
-; SSE2-NEXT:    ret <8 x i16> [[R]]
-;
-; AVX2-LABEL: @gep01_load_i16_insert_v8i16_deref(
-; AVX2-NEXT:    [[TMP1:%.*]] = load <8 x i16>, ptr [[P:%.*]], align 16
-; AVX2-NEXT:    [[R:%.*]] = shufflevector <8 x i16> [[TMP1]], <8 x i16> poison, <8 x i32> <i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; AVX2-NEXT:    ret <8 x i16> [[R]]
+; CHECK-LABEL: @gep01_load_i16_insert_v8i16_deref(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i16>, ptr [[P:%.*]], align 16
+; CHECK-NEXT:    [[R:%.*]] = shufflevector <8 x i16> [[TMP1]], <8 x i16> poison, <8 x i32> <i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:    ret <8 x i16> [[R]]
 ;
   %gep = getelementptr inbounds <8 x i16>, ptr %p, i64 0, i64 1
   %s = load i16, ptr %gep, align 2
@@ -285,16 +279,10 @@ define <8 x i16> @gep01_load_i16_insert_v8i16_deref(ptr align 16 dereferenceable
 ; Verify that alignment of the new load is not over-specified.
 
 define <8 x i16> @gep01_load_i16_insert_v8i16_deref_minalign(ptr align 2 dereferenceable(16) %p) nofree nosync {
-; SSE2-LABEL: @gep01_load_i16_insert_v8i16_deref_minalign(
-; SSE2-NEXT:    [[GEP:%.*]] = getelementptr inbounds <8 x i16>, ptr [[P:%.*]], i64 0, i64 1
-; SSE2-NEXT:    [[S:%.*]] = load i16, ptr [[GEP]], align 8
-; SSE2-NEXT:    [[R:%.*]] = insertelement <8 x i16> poison, i16 [[S]], i64 0
-; SSE2-NEXT:    ret <8 x i16> [[R]]
-;
-; AVX2-LABEL: @gep01_load_i16_insert_v8i16_deref_minalign(
-; AVX2-NEXT:    [[TMP1:%.*]] = load <8 x i16>, ptr [[P:%.*]], align 2
-; AVX2-NEXT:    [[R:%.*]] = shufflevector <8 x i16> [[TMP1]], <8 x i16> poison, <8 x i32> <i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; AVX2-NEXT:    ret <8 x i16> [[R]]
+; CHECK-LABEL: @gep01_load_i16_insert_v8i16_deref_minalign(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i16>, ptr [[P:%.*]], align 2
+; CHECK-NEXT:    [[R:%.*]] = shufflevector <8 x i16> [[TMP1]], <8 x i16> poison, <8 x i32> <i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:    ret <8 x i16> [[R]]
 ;
   %gep = getelementptr inbounds <8 x i16>, ptr %p, i64 0, i64 1
   %s = load i16, ptr %gep, align 8
@@ -589,8 +577,8 @@ define <8 x i32> @load_v1i32_extract_insert_v8i32_extra_use(ptr align 16 derefer
 ; CHECK-LABEL: @load_v1i32_extract_insert_v8i32_extra_use(
 ; CHECK-NEXT:    [[L:%.*]] = load <1 x i32>, ptr [[P:%.*]], align 4
 ; CHECK-NEXT:    store <1 x i32> [[L]], ptr [[STORE_PTR:%.*]], align 4
-; CHECK-NEXT:    [[S:%.*]] = extractelement <1 x i32> [[L]], i32 0
-; CHECK-NEXT:    [[R:%.*]] = insertelement <8 x i32> poison, i32 [[S]], i32 0
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <1 x i32> [[L]], <1 x i32> poison, <8 x i32> <i32 0, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:    [[R:%.*]] = shufflevector <8 x i32> [[TMP1]], <8 x i32> poison, <8 x i32> <i32 0, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    ret <8 x i32> [[R]]
 ;
   %l = load <1 x i32>, ptr %p, align 4
@@ -603,17 +591,10 @@ define <8 x i32> @load_v1i32_extract_insert_v8i32_extra_use(ptr align 16 derefer
 ; Can't safely load the offset vector, but can load+shuffle if it is profitable.
 
 define <8 x i16> @gep1_load_v2i16_extract_insert_v8i16(ptr align 1 dereferenceable(16) %p) nofree nosync {
-; SSE2-LABEL: @gep1_load_v2i16_extract_insert_v8i16(
-; SSE2-NEXT:    [[GEP:%.*]] = getelementptr inbounds <2 x i16>, ptr [[P:%.*]], i64 1
-; SSE2-NEXT:    [[TMP1:%.*]] = getelementptr inbounds <2 x i16>, ptr [[GEP]], i32 0, i32 0
-; SSE2-NEXT:    [[S:%.*]] = load i16, ptr [[TMP1]], align 8
-; SSE2-NEXT:    [[R:%.*]] = insertelement <8 x i16> poison, i16 [[S]], i64 0
-; SSE2-NEXT:    ret <8 x i16> [[R]]
-;
-; AVX2-LABEL: @gep1_load_v2i16_extract_insert_v8i16(
-; AVX2-NEXT:    [[TMP1:%.*]] = load <8 x i16>, ptr [[P:%.*]], align 4
-; AVX2-NEXT:    [[R:%.*]] = shufflevector <8 x i16> [[TMP1]], <8 x i16> poison, <8 x i32> <i32 2, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; AVX2-NEXT:    ret <8 x i16> [[R]]
+; CHECK-LABEL: @gep1_load_v2i16_extract_insert_v8i16(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i16>, ptr [[P:%.*]], align 4
+; CHECK-NEXT:    [[R:%.*]] = shufflevector <8 x i16> [[TMP1]], <8 x i16> poison, <8 x i32> <i32 2, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:    ret <8 x i16> [[R]]
 ;
   %gep = getelementptr inbounds <2 x i16>, ptr %p, i64 1
   %l = load <2 x i16>, ptr %gep, align 8
diff --git a/llvm/test/Transforms/VectorCombine/X86/load.ll b/llvm/test/Transforms/VectorCombine/X86/load.ll
index b12104c5c673e..84c223be88621 100644
--- a/llvm/test/Transforms/VectorCombine/X86/load.ll
+++ b/llvm/test/Transforms/VectorCombine/X86/load.ll
@@ -574,8 +574,8 @@ define <8 x i32> @load_v1i32_extract_insert_v8i32_extra_use(ptr align 16 derefer
 ; CHECK-LABEL: @load_v1i32_extract_insert_v8i32_extra_use(
 ; CHECK-NEXT:    [[L:%.*]] = load <1 x i32>, ptr [[P:%.*]], align 4
 ; CHECK-NEXT:    store <1 x i32> [[L]], ptr [[STORE_PTR:%.*]], align 4
-; CHECK-NEXT:    [[S:%.*]] = extractelement <1 x i32> [[L]], i32 0
-; CHECK-NEXT:    [[R:%.*]] = insertelement <8 x i32> undef, i32 [[S]], i32 0
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <1 x i32> [[L]], <1 x i32> poison, <8 x i32> <i32 0, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:    [[R:%.*]] = shufflevector <8 x i32> [[TMP1]], <8 x i32> undef, <8 x i32> <i32 0, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; CHECK-NEXT:    ret <8 x i32> [[R]]
 ;
   %l = load <1 x i32>, ptr %p, align 4
diff --git a/llvm/test/Transforms/VectorCombine/X86/pr126085.ll b/llvm/test/Transforms/VectorCombine/X86/pr126085.ll
new file mode 100644
index 0000000000000..f596807027db6
--- /dev/null
+++ b/llvm/test/Transforms/VectorCombine/X86/pr126085.ll
@@ -0,0 +1,20 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 4
+; RUN: opt < %s -passes=vector-combine -S -mtriple=x86_64-- | FileCheck %s
+
+define i32 @test(ptr %a0) {
+; CHECK-LABEL: define i32 @test(
+; CHECK-SAME: ptr [[A0:%.*]]) {
+; CHECK-NEXT:    [[LOAD:%.*]] = load <16 x i8>, ptr [[A0]], align 1
+; CHECK-NEXT:    [[SHUF:%.*]] = shufflevector <16 x i8> [[LOAD]], <16 x i8> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:    [[ELT:%.*]] = extractelement <16 x i8> [[LOAD]], i64 11
+; CHECK-NEXT:    [[INS:%.*]] = insertelement <4 x i8> [[SHUF]], i8 [[ELT]], i64 1
+; CHECK-NEXT:    [[RES:%.*]] = bitcast <4 x i8> [[INS]] to i32
+; CHECK-NEXT:    ret i32 [[RES]]
+;
+  %load = load <16 x i8>, ptr %a0, align 1
+  %shuf = shufflevector <16 x i8> %load, <16 x i8> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+  %elt = extractelement <16 x i8> %load, i64 11
+  %ins = insertelement <4 x i8> %shuf, i8 %elt, i64 1
+  %res = bitcast <4 x i8> %ins to i32
+  ret i32 %res
+}
diff --git a/llvm/test/Transforms/InstCombine/fold-binop-of-reductions.ll b/llvm/test/Transforms/VectorCombine/fold-binop-of-reductions.ll
similarity index 100%
rename from llvm/test/Transforms/InstCombine/fold-binop-of-reductions.ll
rename to llvm/test/Transforms/VectorCombine/fold-binop-of-reductions.ll
diff --git a/llvm/test/Transforms/lower-builtin-allow-check.ll b/llvm/test/Transforms/lower-builtin-allow-check.ll
index bcd9722d2b289..fb87269429928 100644
--- a/llvm/test/Transforms/lower-builtin-allow-check.ll
+++ b/llvm/test/Transforms/lower-builtin-allow-check.ll
@@ -2,6 +2,14 @@
 ; RUN: opt < %s -passes='function(lower-allow-check)' -S | FileCheck %s --check-prefixes=NOPROFILE
 ; RUN: opt < %s -passes='function(lower-allow-check)' -lower-allow-check-random-rate=0 -S | FileCheck %s --check-prefixes=NONE
 ; RUN: opt < %s -passes='function(lower-allow-check)' -lower-allow-check-random-rate=1 -S | FileCheck %s --check-prefixes=ALL
+;
+; RUN: opt < %s -passes='require<profile-summary>,function(lower-allow-check<cutoffs[22]=990000>)' -S | FileCheck %s --check-prefixes=HOT99
+; RUN: opt < %s -passes='require<profile-summary>,function(lower-allow-check<cutoffs[22]=700000>)' -S | FileCheck %s --check-prefixes=HOT70
+; RUN: opt < %s -passes='require<profile-summary>,function(lower-allow-check<cutoffs[22]=990000>)' -lower-allow-check-random-rate=0 -S | FileCheck %s --check-prefixes=NONE99
+; RUN: opt < %s -passes='require<profile-summary>,function(lower-allow-check<cutoffs[22]=700000>)' -lower-allow-check-random-rate=1 -S | FileCheck %s --check-prefixes=ALL70
+;
+; -lower-allow-check-percentile-cutoff is deprecated and will be removed in the future;
+; use the cutoffs parameter to the lower-allow-check pass, as shown above.
 ; RUN: opt < %s -passes='require<profile-summary>,function(lower-allow-check)' -lower-allow-check-percentile-cutoff-hot=990000 -S | FileCheck %s --check-prefixes=HOT99
 ; RUN: opt < %s -passes='require<profile-summary>,function(lower-allow-check)' -lower-allow-check-percentile-cutoff-hot=700000 -S | FileCheck %s --check-prefixes=HOT70
 ; RUN: opt < %s -passes='require<profile-summary>,function(lower-allow-check)' -lower-allow-check-random-rate=0 -lower-allow-check-percentile-cutoff-hot=990000 -S | FileCheck %s --check-prefixes=NONE99
diff --git a/llvm/test/lit.cfg.py b/llvm/test/lit.cfg.py
index b17d41fa11af7..aad7a088551b2 100644
--- a/llvm/test/lit.cfg.py
+++ b/llvm/test/lit.cfg.py
@@ -311,6 +311,9 @@ def enable_ptxas(ptxas_executable):
             (12, 2),
             (12, 3),
             (12, 4),
+            (12, 5),
+            (12, 6),
+            (12, 8),
         ]
 
         def version_int(ver):
@@ -563,6 +566,54 @@ def have_ld64_plugin_support():
 if have_ld64_plugin_support():
     config.available_features.add("ld64_plugin")
 
+def host_unwind_supports_jit():
+    # Do we expect the host machine to support JIT registration of clang's
+    # default unwind info format for the host (e.g. eh-frames, compact-unwind,
+    # etc.).
+
+    # Linux and the BSDs use DWARF eh-frames and all known unwinders support
+    # register_frame at minimum.
+    if platform.system() in [ "Linux", "FreeBSD", "NetBSD" ]:
+        return True
+
+    # Windows does not support frame info without the ORC runtime.
+    if platform.system() == "Windows":
+        return False
+
+    # On Darwin/x86-64 clang produces both eh-frames and compact-unwind, and
+    # libunwind supports register_frame. On Darwin/arm64 clang produces
+    # compact-unwind only, and JIT'd registration is not available before
+    # macOS 14.0.
+    if platform.system() == "Darwin":
+
+        assert (
+            "arm64" in config.host_triple
+            or "x86_64" in config.host_triple
+        )
+
+        if "x86_64" in config.host_triple:
+            return True
+
+        # Must be arm64. Check the macOS version.
+        try:
+            osx_version = subprocess.check_output(
+                ["sw_vers", "-productVersion"], universal_newlines=True
+            )
+            osx_version = tuple(int(x) for x in osx_version.split("."))
+            if len(osx_version) == 2:
+                osx_version = (osx_version[0], osx_version[1], 0)
+            if osx_version >= (14, 0):
+                return True
+        except:
+            pass
+
+        return False
+
+    return False
+
+if host_unwind_supports_jit():
+    config.available_features.add("host-unwind-supports-jit")
+
 # Ask llvm-config about asserts
 llvm_config.feature_config(
     [
diff --git a/llvm/test/tools/UpdateTestChecks/update_analyze_test_checks/Inputs/loop-distribute.ll b/llvm/test/tools/UpdateTestChecks/update_analyze_test_checks/Inputs/loop-distribute.ll
index 48f80533c6379..548aa0ab2673b 100644
--- a/llvm/test/tools/UpdateTestChecks/update_analyze_test_checks/Inputs/loop-distribute.ll
+++ b/llvm/test/tools/UpdateTestChecks/update_analyze_test_checks/Inputs/loop-distribute.ll
@@ -1,7 +1,7 @@
 ; RUN: opt -passes=loop-distribute -enable-loop-distribute \
 ; RUN:   -debug-only=loop-distribute -disable-output 2>&1 %s | FileCheck %s
 
-define void @ldist(i1 %c, ptr %A, ptr %B, ptr %C) {
+define void @ldist(i1 %cond, ptr %A, ptr %B, ptr %C) {
 entry:
   br label %for.body
 
@@ -9,7 +9,7 @@ for.body:                                         ; preds = %if.end, %entry
   %iv = phi i16 [ 0, %entry ], [ %iv.next, %if.end ]
   %lv = load i16, ptr %A, align 1
   store i16 %lv, ptr %A, align 1
-  br i1 %c, label %if.then, label %if.end
+  br i1 %cond, label %if.then, label %if.end
 
 if.then:                                          ; preds = %for.body
   %lv2 = load i16, ptr %A, align 1
@@ -18,6 +18,8 @@ if.then:                                          ; preds = %for.body
 if.end:                                           ; preds = %if.then, %for.body
   %c.sink = phi ptr [ %B, %if.then ], [ %C, %for.body ]
   %lv3 = load i16, ptr %c.sink
+  %add = add i16 %lv3, 10
+  store i16 %add, ptr %c.sink, align 1
   %iv.next = add nuw nsw i16 %iv, 1
   %tobool.not = icmp eq i16 %iv.next, 1000
   br i1 %tobool.not, label %for.end.loopexit, label %for.body
diff --git a/llvm/test/tools/UpdateTestChecks/update_analyze_test_checks/Inputs/loop-distribute.ll.expected b/llvm/test/tools/UpdateTestChecks/update_analyze_test_checks/Inputs/loop-distribute.ll.expected
index baef851b84ee5..eba378c175091 100644
--- a/llvm/test/tools/UpdateTestChecks/update_analyze_test_checks/Inputs/loop-distribute.ll.expected
+++ b/llvm/test/tools/UpdateTestChecks/update_analyze_test_checks/Inputs/loop-distribute.ll.expected
@@ -2,51 +2,55 @@
 ; RUN: opt -passes=loop-distribute -enable-loop-distribute \
 ; RUN:   -debug-only=loop-distribute -disable-output 2>&1 %s | FileCheck %s
 
-define void @ldist(i1 %c, ptr %A, ptr %B, ptr %C) {
+define void @ldist(i1 %cond, ptr %A, ptr %B, ptr %C) {
 ; CHECK-LABEL: 'ldist'
 ; CHECK-NEXT:  LDist: Found a candidate loop: for.body
 ; CHECK-NEXT:  LDist: Backward dependences:
 ; CHECK-NEXT:    Unknown:
-; CHECK-NEXT:        %lv = load i16, ptr %A, align 1 ->
-; CHECK-NEXT:        store i16 %lv, ptr %A, align 1
+; CHECK-NEXT:        %lv3 = load i16, ptr %c.sink, align 2 ->
+; CHECK-NEXT:        store i16 %add, ptr %c.sink, align 1
 ; CHECK-NEXT:    Unknown:
-; CHECK-NEXT:        store i16 %lv, ptr %A, align 1 ->
-; CHECK-NEXT:        %lv2 = load i16, ptr %A, align 1
+; CHECK-NEXT:        %lv3 = load i16, ptr %c.sink, align 2 ->
+; CHECK-NEXT:        store i16 %add, ptr %c.sink, align 1
 ; CHECK-NEXT:  LDist: Seeded partitions:
-; CHECK-NEXT:  LDist: Partition 0: (cycle)
+; CHECK-NEXT:  LDist: Partition 0:
 ; CHECK-NEXT:    for.body: %lv = load i16, ptr %A, align 1
-; CHECK-NEXT:    for.body: store i16 %lv, ptr %A, align 1
-; CHECK-NEXT:    if.then: %lv2 = load i16, ptr %A, align 1
 ; CHECK-NEXT:  LDist: Partition 1:
-; CHECK-NEXT:    if.end: %lv3 = load i16, ptr %c.sink, align 2
+; CHECK-NEXT:    for.body: store i16 %lv, ptr %A, align 1
 ; CHECK-NEXT:  LDist: Partition 2:
+; CHECK-NEXT:    if.then: %lv2 = load i16, ptr %A, align 1
+; CHECK-NEXT:  LDist: Partition 3: (cycle)
 ; CHECK-NEXT:    if.end: %lv3 = load i16, ptr %c.sink, align 2
+; CHECK-NEXT:    if.end: store i16 %add, ptr %c.sink, align 1
 ; CHECK-NEXT:  LDist: Merged partitions:
-; CHECK-NEXT:  LDist: Partition 0: (cycle)
+; CHECK-NEXT:  LDist: Partition 0:
 ; CHECK-NEXT:    for.body: %lv = load i16, ptr %A, align 1
 ; CHECK-NEXT:    for.body: store i16 %lv, ptr %A, align 1
 ; CHECK-NEXT:    if.then: %lv2 = load i16, ptr %A, align 1
-; CHECK-NEXT:  LDist: Partition 1:
+; CHECK-NEXT:  LDist: Partition 1: (cycle)
 ; CHECK-NEXT:    if.end: %lv3 = load i16, ptr %c.sink, align 2
+; CHECK-NEXT:    if.end: store i16 %add, ptr %c.sink, align 1
 ; CHECK-NEXT:  LDist: Populated partitions:
-; CHECK-NEXT:  LDist: Partition 0: (cycle)
+; CHECK-NEXT:  LDist: Partition 0:
 ; CHECK-NEXT:    for.body: %lv = load i16, ptr %A, align 1
 ; CHECK-NEXT:    for.body: store i16 %lv, ptr %A, align 1
 ; CHECK-NEXT:    if.then: %lv2 = load i16, ptr %A, align 1
-; CHECK-NEXT:    for.body: br i1 %c, label %if.then, label %if.end
+; CHECK-NEXT:    for.body: br i1 %cond, label %if.then, label %if.end
 ; CHECK-NEXT:    if.then: br label %if.end
 ; CHECK-NEXT:    if.end: br i1 %tobool.not, label %for.end.loopexit, label %for.body
 ; CHECK-NEXT:    if.end: %tobool.not = icmp eq i16 %iv.next, 1000
 ; CHECK-NEXT:    if.end: %iv.next = add nuw nsw i16 %iv, 1
 ; CHECK-NEXT:    for.body: %iv = phi i16 [ 0, %entry ], [ %iv.next, %if.end ]
-; CHECK-NEXT:  LDist: Partition 1:
+; CHECK-NEXT:  LDist: Partition 1: (cycle)
 ; CHECK-NEXT:    if.end: %lv3 = load i16, ptr %c.sink, align 2
-; CHECK-NEXT:    for.body: br i1 %c, label %if.then, label %if.end
+; CHECK-NEXT:    if.end: store i16 %add, ptr %c.sink, align 1
+; CHECK-NEXT:    for.body: br i1 %cond, label %if.then, label %if.end
 ; CHECK-NEXT:    if.then: br label %if.end
 ; CHECK-NEXT:    if.end: br i1 %tobool.not, label %for.end.loopexit, label %for.body
 ; CHECK-NEXT:    if.end: %tobool.not = icmp eq i16 %iv.next, 1000
 ; CHECK-NEXT:    if.end: %iv.next = add nuw nsw i16 %iv, 1
 ; CHECK-NEXT:    for.body: %iv = phi i16 [ 0, %entry ], [ %iv.next, %if.end ]
+; CHECK-NEXT:    if.end: %add = add i16 %lv3, 10
 ; CHECK-NEXT:    if.end: %c.sink = phi ptr [ %B, %if.then ], [ %C, %for.body ]
 ; CHECK-NEXT:  LDist: Distributing loop: for.body
 ; CHECK-NEXT:  LDist: Pointers:
@@ -56,19 +60,21 @@ define void @ldist(i1 %c, ptr %A, ptr %B, ptr %C) {
 ; CHECK-NEXT:    ptr %A
 ; CHECK-NEXT:    Against group ([[GRP2:0x[0-9a-f]+]]):
 ; CHECK-NEXT:    ptr %C
+; CHECK-NEXT:    ptr %C
 ; CHECK-NEXT:  Check 1:
 ; CHECK-NEXT:    Comparing group ([[GRP1]]):
 ; CHECK-NEXT:    ptr %A
 ; CHECK-NEXT:    ptr %A
 ; CHECK-NEXT:    Against group ([[GRP3:0x[0-9a-f]+]]):
 ; CHECK-NEXT:    ptr %B
+; CHECK-NEXT:    ptr %B
 ; CHECK-NEXT:  LDist: After removing unused Instrs:
 ; CHECK-NEXT:  LDist: Partition 0:
 ; CHECK-NEXT:  for.body.ldist1: ; preds = %if.end.ldist1, %for.body.ph.ldist1
 ; CHECK-NEXT:    %iv.ldist1 = phi i16 [ 0, %for.body.ph.ldist1 ], [ %iv.next.ldist1, %if.end.ldist1 ]
 ; CHECK-NEXT:    %lv.ldist1 = load i16, ptr %A, align 1, !alias.scope !0, !noalias !3
 ; CHECK-NEXT:    store i16 %lv.ldist1, ptr %A, align 1, !alias.scope !0, !noalias !3
-; CHECK-NEXT:    br i1 %c, label %if.then.ldist1, label %if.end.ldist1
+; CHECK-NEXT:    br i1 %cond, label %if.then.ldist1, label %if.end.ldist1
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  if.then.ldist1: ; preds = %for.body.ldist1
 ; CHECK-NEXT:    %lv2.ldist1 = load i16, ptr %A, align 1, !alias.scope !0, !noalias !3
@@ -81,7 +87,7 @@ define void @ldist(i1 %c, ptr %A, ptr %B, ptr %C) {
 ; CHECK-NEXT:  LDist: Partition 1:
 ; CHECK-NEXT:  for.body: ; preds = %if.end, %for.body.ph
 ; CHECK-NEXT:    %iv = phi i16 [ 0, %for.body.ph ], [ %iv.next, %if.end ]
-; CHECK-NEXT:    br i1 %c, label %if.then, label %if.end
+; CHECK-NEXT:    br i1 %cond, label %if.then, label %if.end
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  if.then: ; preds = %for.body
 ; CHECK-NEXT:    br label %if.end
@@ -89,6 +95,8 @@ define void @ldist(i1 %c, ptr %A, ptr %B, ptr %C) {
 ; CHECK-NEXT:  if.end: ; preds = %if.then, %for.body
 ; CHECK-NEXT:    %c.sink = phi ptr [ %B, %if.then ], [ %C, %for.body ]
 ; CHECK-NEXT:    %lv3 = load i16, ptr %c.sink, align 2
+; CHECK-NEXT:    %add = add i16 %lv3, 10
+; CHECK-NEXT:    store i16 %add, ptr %c.sink, align 1
 ; CHECK-NEXT:    %iv.next = add nuw nsw i16 %iv, 1
 ; CHECK-NEXT:    %tobool.not = icmp eq i16 %iv.next, 1000
 ; CHECK-NEXT:    br i1 %tobool.not, label %for.end.loopexit.loopexit6, label %for.body
@@ -100,7 +108,7 @@ for.body:                                         ; preds = %if.end, %entry
   %iv = phi i16 [ 0, %entry ], [ %iv.next, %if.end ]
   %lv = load i16, ptr %A, align 1
   store i16 %lv, ptr %A, align 1
-  br i1 %c, label %if.then, label %if.end
+  br i1 %cond, label %if.then, label %if.end
 
 if.then:                                          ; preds = %for.body
   %lv2 = load i16, ptr %A, align 1
@@ -109,6 +117,8 @@ if.then:                                          ; preds = %for.body
 if.end:                                           ; preds = %if.then, %for.body
   %c.sink = phi ptr [ %B, %if.then ], [ %C, %for.body ]
   %lv3 = load i16, ptr %c.sink
+  %add = add i16 %lv3, 10
+  store i16 %add, ptr %c.sink, align 1
   %iv.next = add nuw nsw i16 %iv, 1
   %tobool.not = icmp eq i16 %iv.next, 1000
   br i1 %tobool.not, label %for.end.loopexit, label %for.body
diff --git a/llvm/test/tools/UpdateTestChecks/update_llc_test_checks/Inputs/x86-basic.ll.expected b/llvm/test/tools/UpdateTestChecks/update_llc_test_checks/Inputs/x86-basic.ll.expected
index e113303f57a37..9bfabdd018667 100644
--- a/llvm/test/tools/UpdateTestChecks/update_llc_test_checks/Inputs/x86-basic.ll.expected
+++ b/llvm/test/tools/UpdateTestChecks/update_llc_test_checks/Inputs/x86-basic.ll.expected
@@ -58,7 +58,9 @@ define dso_local i32 @dsolocal() {
 ; WIN-NEXT:    .seh_endprologue
 ; WIN-NEXT:    callq ext
 ; WIN-NEXT:    movl $2, %eax
+; WIN-NEXT:    .seh_startepilogue
 ; WIN-NEXT:    addq $40, %rsp
+; WIN-NEXT:    .seh_endepilogue
 ; WIN-NEXT:    retq
 ; WIN-NEXT:    .seh_endproc
 entry:
diff --git a/llvm/test/tools/llvm-dwarfdump/AArch64/DW_AT_APPLE_enum_kind.s b/llvm/test/tools/llvm-dwarfdump/AArch64/DW_AT_APPLE_enum_kind.s
new file mode 100644
index 0000000000000..5b391f82dac32
--- /dev/null
+++ b/llvm/test/tools/llvm-dwarfdump/AArch64/DW_AT_APPLE_enum_kind.s
@@ -0,0 +1,44 @@
+;; Demonstrate dumping DW_AT_APPLE_enum_kind.
+; RUN: llvm-mc -triple=aarch64--darwin -filetype=obj < %s | \
+; RUN:     llvm-dwarfdump -v - | FileCheck %s
+
+; CHECK: .debug_abbrev contents:
+; CHECK: DW_AT_APPLE_enum_kind DW_FORM_data1
+; CHECK: .debug_info contents:
+; CHECK: DW_AT_APPLE_enum_kind [DW_FORM_data1] (DW_APPLE_ENUM_KIND_Closed)
+; CHECK: DW_AT_APPLE_enum_kind [DW_FORM_data1] (DW_APPLE_ENUM_KIND_Open)
+
+	.section	__DWARF,__debug_abbrev,regular,debug
+Lsection_abbrev:
+	.byte	1                               ; Abbreviation Code
+	.byte	17                              ; DW_TAG_compile_unit
+	.byte	1                               ; DW_CHILDREN_yes
+	.byte	0                               ; EOM(1)
+	.byte	0                               ; EOM(2)
+	.byte	2                               ; Abbreviation Code
+	.byte	4                               ; DW_TAG_enumeration_type
+	.byte	0                               ; DW_CHILDREN_no
+	.ascii	"\361\177"                      ; DW_AT_APPLE_enum_kind
+	.byte	11                              ; DW_FORM_data1
+	.byte	0                               ; EOM(1)
+	.byte	0                               ; EOM(2)
+	.byte	0                               ; EOM(3)
+	.section	__DWARF,__debug_info,regular,debug
+Lsection_info:
+Lcu_begin0:
+.set Lset0, Ldebug_info_end0-Ldebug_info_start0 ; Length of Unit
+	.long	Lset0
+Ldebug_info_start0:
+	.short	5                               ; DWARF version number
+	.byte	1                               ; DWARF Unit Type
+	.byte	8                               ; Address Size (in bytes)
+.set Lset1, Lsection_abbrev-Lsection_abbrev ; Offset Into Abbrev. Section
+	.long	Lset1
+	.byte	1                               ; Abbrev [1] 0xc:0x40 DW_TAG_compile_unit
+	.byte	2                               ; Abbrev [3] 0x2a:0x9 DW_TAG_enumeration_type
+	.byte	0                               ; DW_APPLE_ENUM_KIND_Closed
+	.byte	2                               ; Abbrev [3] 0x42:0x9 DW_TAG_enumeration_type
+	.byte	1                               ; DW_APPLE_ENUM_KIND_Open
+	.byte	0                               ; End Of Children Mark
+Ldebug_info_end0:
+
diff --git a/llvm/test/tools/llvm-dwarfdump/X86/dwarf-verify-good-json-output.s b/llvm/test/tools/llvm-dwarfdump/X86/dwarf-verify-good-json-output.s
new file mode 100644
index 0000000000000..6f4b1cdedf83a
--- /dev/null
+++ b/llvm/test/tools/llvm-dwarfdump/X86/dwarf-verify-good-json-output.s
@@ -0,0 +1,32 @@
+# RUN: llvm-mc %s -filetype obj -triple x86_64-apple-darwin -o - | llvm-dwarfdump -verify --verify-json=%t.json -
+# RUN: FileCheck %s --input-file %t.json
+
+# CHECK: {"error-categories":{},"error-count":0}
+
+# This test is meant to verify that the -verify option
+# in llvm-dwarfdump doesn't produce any .apple_names related
+# output when there's no such section in the object.
+# The test was manually modified to exclude the
+# .apple_names section from the apple_names_verify_num_atoms.s
+# test file in the same directory.
+
+  .section  __TEXT,__text,regular,pure_instructions
+  .file 1 "basic.c"
+  .comm _i,4,2                  ## @i
+  .comm _j,4,2                  ## @j
+  .section  __DWARF,__debug_str,regular,debug
+Linfo_string:
+  .asciz  "Apple LLVM version 8.1.0 (clang-802.0.35)" ## string offset=0
+  .asciz  "basic.c"               ## string offset=42
+  .asciz  "/Users/sgravani/Development/tests" ## string offset=50
+  .asciz  "i"                     ## string offset=84
+  .asciz  "int"                   ## string offset=86
+  .asciz  "j"                     ## string offset=90
+
+  .section  __DWARF,__debug_info,regular,debug
+Lsection_info:
+
+.subsections_via_symbols
+  .section  __DWARF,__debug_line,regular,debug
+Lsection_line:
+Lline_table_start0:
diff --git a/llvm/test/tools/llvm-dwarfdump/X86/statistics.ll b/llvm/test/tools/llvm-dwarfdump/X86/statistics.ll
index 9f3a00df2ffe7..a454bf14c3353 100644
--- a/llvm/test/tools/llvm-dwarfdump/X86/statistics.ll
+++ b/llvm/test/tools/llvm-dwarfdump/X86/statistics.ll
@@ -55,8 +55,8 @@
 ; CHECK:      "#bytes within functions": [[FUNCSIZE:[0-9]+]]
 ; CHECK:      "#bytes within inlined functions": [[INLINESIZE:[0-9]+]]
 ; CHECK:      "#bytes in __debug_loc": 35,
-; CHECK-NEXT: "#bytes in __debug_abbrev": 386,
-; CHECK-NEXT: "#bytes in __debug_info": 463,
+; CHECK-NEXT: "#bytes in __debug_abbrev": 384,
+; CHECK-NEXT: "#bytes in __debug_info": 459,
 ; CHECK-NEXT: "#bytes in __debug_str": 231,
 ; CHECK-NEXT: "#bytes in __apple_names": 348,
 ; CHECK-NEXT: "#bytes in __apple_objc": 36,
diff --git a/llvm/test/tools/llvm-mca/ARM/m55-fp.s b/llvm/test/tools/llvm-mca/ARM/m55-fp.s
index 6318cfa9d6e9c..1668f58c7937f 100644
--- a/llvm/test/tools/llvm-mca/ARM/m55-fp.s
+++ b/llvm/test/tools/llvm-mca/ARM/m55-fp.s
@@ -21,30 +21,30 @@ vcmpe.f32 s1, #0.0
 vcmpe.f64 d1, #0.0
 vcvt.f32.f64 s1, d2
 vcvt.f64.f32 d1, s1
-vcvt.f16.u16 s1, s2, #8
-vcvt.f16.s16 s1, s2, #8
-vcvt.f16.u32 s1, s2, #8
-vcvt.f16.s32 s1, s2, #8
-vcvt.u16.f16 s1, s2, #8
-vcvt.s16.f16 s1, s2, #8
-vcvt.u32.f16 s1, s2, #8
-vcvt.s32.f16 s1, s2, #8
-vcvt.f32.u16 s1, s2, #8
-vcvt.f32.s16 s1, s2, #8
-vcvt.f32.u32 s1, s2, #8
-vcvt.f32.s32 s1, s2, #8
-vcvt.u16.f32 s1, s2, #8
-vcvt.s16.f32 s1, s2, #8
-vcvt.u32.f32 s1, s2, #8
-vcvt.s32.f32 s1, s2, #8
-vcvt.f64.u16 d1, d2, #8
-vcvt.f64.s16 d1, d2, #8
-vcvt.f64.u32 d1, d2, #8
-vcvt.f64.s32 d1, d2, #8
-vcvt.u16.f64 d1, d2, #8
-vcvt.s16.f64 d1, d2, #8
-vcvt.u32.f64 d1, d2, #8
-vcvt.s32.f64 d1, d2, #8
+vcvt.f16.u16 s1, s1, #8
+vcvt.f16.s16 s1, s1, #8
+vcvt.f16.u32 s1, s1, #8
+vcvt.f16.s32 s1, s1, #8
+vcvt.u16.f16 s1, s1, #8
+vcvt.s16.f16 s1, s1, #8
+vcvt.u32.f16 s1, s1, #8
+vcvt.s32.f16 s1, s1, #8
+vcvt.f32.u16 s1, s1, #8
+vcvt.f32.s16 s1, s1, #8
+vcvt.f32.u32 s1, s1, #8
+vcvt.f32.s32 s1, s1, #8
+vcvt.u16.f32 s1, s1, #8
+vcvt.s16.f32 s1, s1, #8
+vcvt.u32.f32 s1, s1, #8
+vcvt.s32.f32 s1, s1, #8
+vcvt.f64.u16 d1, d1, #8
+vcvt.f64.s16 d1, d1, #8
+vcvt.f64.u32 d1, d1, #8
+vcvt.f64.s32 d1, d1, #8
+vcvt.u16.f64 d1, d1, #8
+vcvt.s16.f64 d1, d1, #8
+vcvt.u32.f64 d1, d1, #8
+vcvt.s32.f64 d1, d1, #8
 vcvt.u32.f16 s1, s2
 vcvt.s32.f16 s1, s2
 vcvt.u32.f32 s1, s2
diff --git a/llvm/test/tools/llvm-mca/ARM/m7-fp.s b/llvm/test/tools/llvm-mca/ARM/m7-fp.s
index dcf9723461dec..dba7ff92f30cb 100644
--- a/llvm/test/tools/llvm-mca/ARM/m7-fp.s
+++ b/llvm/test/tools/llvm-mca/ARM/m7-fp.s
@@ -9,22 +9,22 @@ vcmp.f32 s1, s2
 vcmp.f64 d1, d2
 vcvt.f32.f64 s1, d2
 vcvt.f64.f32 d1, s1
-vcvt.f32.u16 s1, s2, #8
-vcvt.f32.s16 s1, s2, #8
-vcvt.f32.u32 s1, s2, #8
-vcvt.f32.s32 s1, s2, #8
-vcvt.u16.f32 s1, s2, #8
-vcvt.s16.f32 s1, s2, #8
-vcvt.u32.f32 s1, s2, #8
-vcvt.s32.f32 s1, s2, #8
-vcvt.f64.u16 d1, d2, #8
-vcvt.f64.s16 d1, d2, #8
-vcvt.f64.u32 d1, d2, #8
-vcvt.f64.s32 d1, d2, #8
-vcvt.u16.f64 d1, d2, #8
-vcvt.s16.f64 d1, d2, #8
-vcvt.u32.f64 d1, d2, #8
-vcvt.s32.f64 d1, d2, #8
+vcvt.f32.u16 s1, s1, #8
+vcvt.f32.s16 s1, s1, #8
+vcvt.f32.u32 s1, s1, #8
+vcvt.f32.s32 s1, s1, #8
+vcvt.u16.f32 s1, s1, #8
+vcvt.s16.f32 s1, s1, #8
+vcvt.u32.f32 s1, s1, #8
+vcvt.s32.f32 s1, s1, #8
+vcvt.f64.u16 d1, d1, #8
+vcvt.f64.s16 d1, d1, #8
+vcvt.f64.u32 d1, d1, #8
+vcvt.f64.s32 d1, d1, #8
+vcvt.u16.f64 d1, d1, #8
+vcvt.s16.f64 d1, d1, #8
+vcvt.u32.f64 d1, d1, #8
+vcvt.s32.f64 d1, d1, #8
 vcvt.u32.f32 s1, s2
 vcvt.s32.f32 s1, s2
 vcvt.u32.f64 s1, d2
diff --git a/llvm/test/tools/llvm-mca/ARM/m85-fp.s b/llvm/test/tools/llvm-mca/ARM/m85-fp.s
index edc46060fe0f3..0fc1b394de2dc 100644
--- a/llvm/test/tools/llvm-mca/ARM/m85-fp.s
+++ b/llvm/test/tools/llvm-mca/ARM/m85-fp.s
@@ -21,30 +21,30 @@ vcmpe.f32 s1, #0.0
 vcmpe.f64 d1, #0.0
 vcvt.f32.f64 s1, d2
 vcvt.f64.f32 d1, s1
-vcvt.f16.u16 s1, s2, #8
-vcvt.f16.s16 s1, s2, #8
-vcvt.f16.u32 s1, s2, #8
-vcvt.f16.s32 s1, s2, #8
-vcvt.u16.f16 s1, s2, #8
-vcvt.s16.f16 s1, s2, #8
-vcvt.u32.f16 s1, s2, #8
-vcvt.s32.f16 s1, s2, #8
-vcvt.f32.u16 s1, s2, #8
-vcvt.f32.s16 s1, s2, #8
-vcvt.f32.u32 s1, s2, #8
-vcvt.f32.s32 s1, s2, #8
-vcvt.u16.f32 s1, s2, #8
-vcvt.s16.f32 s1, s2, #8
-vcvt.u32.f32 s1, s2, #8
-vcvt.s32.f32 s1, s2, #8
-vcvt.f64.u16 d1, d2, #8
-vcvt.f64.s16 d1, d2, #8
-vcvt.f64.u32 d1, d2, #8
-vcvt.f64.s32 d1, d2, #8
-vcvt.u16.f64 d1, d2, #8
-vcvt.s16.f64 d1, d2, #8
-vcvt.u32.f64 d1, d2, #8
-vcvt.s32.f64 d1, d2, #8
+vcvt.f16.u16 s1, s1, #8
+vcvt.f16.s16 s1, s1, #8
+vcvt.f16.u32 s1, s1, #8
+vcvt.f16.s32 s1, s1, #8
+vcvt.u16.f16 s1, s1, #8
+vcvt.s16.f16 s1, s1, #8
+vcvt.u32.f16 s1, s1, #8
+vcvt.s32.f16 s1, s1, #8
+vcvt.f32.u16 s1, s1, #8
+vcvt.f32.s16 s1, s1, #8
+vcvt.f32.u32 s1, s1, #8
+vcvt.f32.s32 s1, s1, #8
+vcvt.u16.f32 s1, s1, #8
+vcvt.s16.f32 s1, s1, #8
+vcvt.u32.f32 s1, s1, #8
+vcvt.s32.f32 s1, s1, #8
+vcvt.f64.u16 d1, d1, #8
+vcvt.f64.s16 d1, d1, #8
+vcvt.f64.u32 d1, d1, #8
+vcvt.f64.s32 d1, d1, #8
+vcvt.u16.f64 d1, d1, #8
+vcvt.s16.f64 d1, d1, #8
+vcvt.u32.f64 d1, d1, #8
+vcvt.s32.f64 d1, d1, #8
 vcvt.u32.f16 s1, s2
 vcvt.s32.f16 s1, s2
 vcvt.u32.f32 s1, s2
diff --git a/llvm/test/tools/llvm-mca/RISCV/SiFiveP400/vlseg-vsseg.s b/llvm/test/tools/llvm-mca/RISCV/SiFiveP400/vlseg-vsseg.s
index aba6b2f3e24a5..9ba461acef0e3 100644
--- a/llvm/test/tools/llvm-mca/RISCV/SiFiveP400/vlseg-vsseg.s
+++ b/llvm/test/tools/llvm-mca/RISCV/SiFiveP400/vlseg-vsseg.s
@@ -13,10 +13,6 @@ vsetvli zero, zero, e8, m2, tu, mu
 vlseg2e8.v  v8,(a0)
 vsetvli zero, zero, e8, m4, tu, mu
 vlseg2e8.v  v8,(a0)
-vsetvli zero, zero, e8, m8, tu, mu
-vlseg2e8.v  v8,(a0)
-vsetvli zero, zero, e16, mf8, tu, mu
-vlseg2e16.v v8,(a0)
 vsetvli zero, zero, e16, mf4, tu, mu
 vlseg2e16.v v8,(a0)
 vsetvli zero, zero, e16, mf2, tu, mu
@@ -27,12 +23,6 @@ vsetvli zero, zero, e16, m2, tu, mu
 vlseg2e16.v v8,(a0)
 vsetvli zero, zero, e16, m4, tu, mu
 vlseg2e16.v v8,(a0)
-vsetvli zero, zero, e16, m8, tu, mu
-vlseg2e16.v v8,(a0)
-vsetvli zero, zero, e32, mf8, tu, mu
-vlseg2e32.v v8,(a0)
-vsetvli zero, zero, e32, mf4, tu, mu
-vlseg2e32.v v8,(a0)
 vsetvli zero, zero, e32, mf2, tu, mu
 vlseg2e32.v v8,(a0)
 vsetvli zero, zero, e32, m1, tu, mu
@@ -41,22 +31,12 @@ vsetvli zero, zero, e32, m2, tu, mu
 vlseg2e32.v v8,(a0)
 vsetvli zero, zero, e32, m4, tu, mu
 vlseg2e32.v v8,(a0)
-vsetvli zero, zero, e32, m8, tu, mu
-vlseg2e32.v v8,(a0)
-vsetvli zero, zero, e64, mf8, tu, mu
-vlseg2e64.v v8,(a0)
-vsetvli zero, zero, e64, mf4, tu, mu
-vlseg2e64.v v8,(a0)
-vsetvli zero, zero, e64, mf2, tu, mu
-vlseg2e64.v v8,(a0)
 vsetvli zero, zero, e64, m1, tu, mu
 vlseg2e64.v v8,(a0)
 vsetvli zero, zero, e64, m2, tu, mu
 vlseg2e64.v v8,(a0)
 vsetvli zero, zero, e64, m4, tu, mu
 vlseg2e64.v v8,(a0)
-vsetvli zero, zero, e64, m8, tu, mu
-vlseg2e64.v v8,(a0)
 
 vsetvli zero, zero, e8, mf8, tu, mu
 vlseg3e8.v  v8,(a0)
@@ -68,12 +48,6 @@ vsetvli zero, zero, e8, m1, tu, mu
 vlseg3e8.v  v8,(a0)
 vsetvli zero, zero, e8, m2, tu, mu
 vlseg3e8.v  v8,(a0)
-vsetvli zero, zero, e8, m4, tu, mu
-vlseg3e8.v  v8,(a0)
-vsetvli zero, zero, e8, m8, tu, mu
-vlseg3e8.v  v8,(a0)
-vsetvli zero, zero, e16, mf8, tu, mu
-vlseg3e16.v v8,(a0)
 vsetvli zero, zero, e16, mf4, tu, mu
 vlseg3e16.v v8,(a0)
 vsetvli zero, zero, e16, mf2, tu, mu
@@ -82,38 +56,16 @@ vsetvli zero, zero, e16, m1, tu, mu
 vlseg3e16.v v8,(a0)
 vsetvli zero, zero, e16, m2, tu, mu
 vlseg3e16.v v8,(a0)
-vsetvli zero, zero, e16, m4, tu, mu
-vlseg3e16.v v8,(a0)
-vsetvli zero, zero, e16, m8, tu, mu
-vlseg3e16.v v8,(a0)
-vsetvli zero, zero, e32, mf8, tu, mu
-vlseg3e32.v v8,(a0)
-vsetvli zero, zero, e32, mf4, tu, mu
-vlseg3e32.v v8,(a0)
 vsetvli zero, zero, e32, mf2, tu, mu
 vlseg3e32.v v8,(a0)
 vsetvli zero, zero, e32, m1, tu, mu
 vlseg3e32.v v8,(a0)
 vsetvli zero, zero, e32, m2, tu, mu
 vlseg3e32.v v8,(a0)
-vsetvli zero, zero, e32, m4, tu, mu
-vlseg3e32.v v8,(a0)
-vsetvli zero, zero, e32, m8, tu, mu
-vlseg3e32.v v8,(a0)
-vsetvli zero, zero, e64, mf8, tu, mu
-vlseg3e64.v v8,(a0)
-vsetvli zero, zero, e64, mf4, tu, mu
-vlseg3e64.v v8,(a0)
-vsetvli zero, zero, e64, mf2, tu, mu
-vlseg3e64.v v8,(a0)
 vsetvli zero, zero, e64, m1, tu, mu
 vlseg3e64.v v8,(a0)
 vsetvli zero, zero, e64, m2, tu, mu
 vlseg3e64.v v8,(a0)
-vsetvli zero, zero, e64, m4, tu, mu
-vlseg3e64.v v8,(a0)
-vsetvli zero, zero, e64, m8, tu, mu
-vlseg3e64.v v8,(a0)
 
 vsetvli zero, zero, e8, mf8, tu, mu
 vlseg4e8.v  v8,(a0)
@@ -125,12 +77,6 @@ vsetvli zero, zero, e8, m1, tu, mu
 vlseg4e8.v  v8,(a0)
 vsetvli zero, zero, e8, m2, tu, mu
 vlseg4e8.v  v8,(a0)
-vsetvli zero, zero, e8, m4, tu, mu
-vlseg4e8.v  v8,(a0)
-vsetvli zero, zero, e8, m8, tu, mu
-vlseg4e8.v  v8,(a0)
-vsetvli zero, zero, e16, mf8, tu, mu
-vlseg4e16.v v8,(a0)
 vsetvli zero, zero, e16, mf4, tu, mu
 vlseg4e16.v v8,(a0)
 vsetvli zero, zero, e16, mf2, tu, mu
@@ -139,38 +85,16 @@ vsetvli zero, zero, e16, m1, tu, mu
 vlseg4e16.v v8,(a0)
 vsetvli zero, zero, e16, m2, tu, mu
 vlseg4e16.v v8,(a0)
-vsetvli zero, zero, e16, m4, tu, mu
-vlseg4e16.v v8,(a0)
-vsetvli zero, zero, e16, m8, tu, mu
-vlseg4e16.v v8,(a0)
-vsetvli zero, zero, e32, mf8, tu, mu
-vlseg4e32.v v8,(a0)
-vsetvli zero, zero, e32, mf4, tu, mu
-vlseg4e32.v v8,(a0)
 vsetvli zero, zero, e32, mf2, tu, mu
 vlseg4e32.v v8,(a0)
 vsetvli zero, zero, e32, m1, tu, mu
 vlseg4e32.v v8,(a0)
 vsetvli zero, zero, e32, m2, tu, mu
 vlseg4e32.v v8,(a0)
-vsetvli zero, zero, e32, m4, tu, mu
-vlseg4e32.v v8,(a0)
-vsetvli zero, zero, e32, m8, tu, mu
-vlseg4e32.v v8,(a0)
-vsetvli zero, zero, e64, mf8, tu, mu
-vlseg4e64.v v8,(a0)
-vsetvli zero, zero, e64, mf4, tu, mu
-vlseg4e64.v v8,(a0)
-vsetvli zero, zero, e64, mf2, tu, mu
-vlseg4e64.v v8,(a0)
 vsetvli zero, zero, e64, m1, tu, mu
 vlseg4e64.v v8,(a0)
 vsetvli zero, zero, e64, m2, tu, mu
 vlseg4e64.v v8,(a0)
-vsetvli zero, zero, e64, m4, tu, mu
-vlseg4e64.v v8,(a0)
-vsetvli zero, zero, e64, m8, tu, mu
-vlseg4e64.v v8,(a0)
 
 vsetvli zero, zero, e8, mf8, tu, mu
 vlseg5e8.v  v8,(a0)
@@ -180,46 +104,18 @@ vsetvli zero, zero, e8, mf2, tu, mu
 vlseg5e8.v  v8,(a0)
 vsetvli zero, zero, e8, m1, tu, mu
 vlseg5e8.v  v8,(a0)
-vsetvli zero, zero, e8, m2, tu, mu
-vlseg5e8.v  v8,(a0)
-vsetvli zero, zero, e8, m4, tu, mu
-vlseg5e8.v  v8,(a0)
-vsetvli zero, zero, e16, mf8, tu, mu
-vlseg5e16.v v8,(a0)
 vsetvli zero, zero, e16, mf4, tu, mu
 vlseg5e16.v v8,(a0)
 vsetvli zero, zero, e16, mf2, tu, mu
 vlseg5e16.v v8,(a0)
 vsetvli zero, zero, e16, m1, tu, mu
 vlseg5e16.v v8,(a0)
-vsetvli zero, zero, e16, m2, tu, mu
-vlseg5e16.v v8,(a0)
-vsetvli zero, zero, e16, m4, tu, mu
-vlseg5e16.v v8,(a0)
-vsetvli zero, zero, e32, mf8, tu, mu
-vlseg5e32.v v8,(a0)
-vsetvli zero, zero, e32, mf4, tu, mu
-vlseg5e32.v v8,(a0)
 vsetvli zero, zero, e32, mf2, tu, mu
 vlseg5e32.v v8,(a0)
 vsetvli zero, zero, e32, m1, tu, mu
 vlseg5e32.v v8,(a0)
-vsetvli zero, zero, e32, m2, tu, mu
-vlseg5e32.v v8,(a0)
-vsetvli zero, zero, e32, m4, tu, mu
-vlseg5e32.v v8,(a0)
-vsetvli zero, zero, e64, mf8, tu, mu
-vlseg5e64.v v8,(a0)
-vsetvli zero, zero, e64, mf4, tu, mu
-vlseg5e64.v v8,(a0)
-vsetvli zero, zero, e64, mf2, tu, mu
-vlseg5e64.v v8,(a0)
 vsetvli zero, zero, e64, m1, tu, mu
 vlseg5e64.v v8,(a0)
-vsetvli zero, zero, e64, m2, tu, mu
-vlseg5e64.v v8,(a0)
-vsetvli zero, zero, e64, m4, tu, mu
-vlseg5e64.v v8,(a0)
 
 vsetvli zero, zero, e8, mf8, tu, mu
 vlseg6e8.v  v8,(a0)
@@ -229,46 +125,18 @@ vsetvli zero, zero, e8, mf2, tu, mu
 vlseg6e8.v  v8,(a0)
 vsetvli zero, zero, e8, m1, tu, mu
 vlseg6e8.v  v8,(a0)
-vsetvli zero, zero, e8, m2, tu, mu
-vlseg6e8.v  v8,(a0)
-vsetvli zero, zero, e8, m4, tu, mu
-vlseg6e8.v  v8,(a0)
-vsetvli zero, zero, e16, mf8, tu, mu
-vlseg6e16.v v8,(a0)
 vsetvli zero, zero, e16, mf4, tu, mu
 vlseg6e16.v v8,(a0)
 vsetvli zero, zero, e16, mf2, tu, mu
 vlseg6e16.v v8,(a0)
 vsetvli zero, zero, e16, m1, tu, mu
 vlseg6e16.v v8,(a0)
-vsetvli zero, zero, e16, m2, tu, mu
-vlseg6e16.v v8,(a0)
-vsetvli zero, zero, e16, m4, tu, mu
-vlseg6e16.v v8,(a0)
-vsetvli zero, zero, e32, mf8, tu, mu
-vlseg6e32.v v8,(a0)
-vsetvli zero, zero, e32, mf4, tu, mu
-vlseg6e32.v v8,(a0)
 vsetvli zero, zero, e32, mf2, tu, mu
 vlseg6e32.v v8,(a0)
 vsetvli zero, zero, e32, m1, tu, mu
 vlseg6e32.v v8,(a0)
-vsetvli zero, zero, e32, m2, tu, mu
-vlseg6e32.v v8,(a0)
-vsetvli zero, zero, e32, m4, tu, mu
-vlseg6e32.v v8,(a0)
-vsetvli zero, zero, e64, mf8, tu, mu
-vlseg6e64.v v8,(a0)
-vsetvli zero, zero, e64, mf4, tu, mu
-vlseg6e64.v v8,(a0)
-vsetvli zero, zero, e64, mf2, tu, mu
-vlseg6e64.v v8,(a0)
 vsetvli zero, zero, e64, m1, tu, mu
 vlseg6e64.v v8,(a0)
-vsetvli zero, zero, e64, m2, tu, mu
-vlseg6e64.v v8,(a0)
-vsetvli zero, zero, e64, m4, tu, mu
-vlseg6e64.v v8,(a0)
 
 vsetvli zero, zero, e8, mf8, tu, mu
 vlseg7e8.v  v8,(a0)
@@ -278,46 +146,18 @@ vsetvli zero, zero, e8, mf2, tu, mu
 vlseg7e8.v  v8,(a0)
 vsetvli zero, zero, e8, m1, tu, mu
 vlseg7e8.v  v8,(a0)
-vsetvli zero, zero, e8, m2, tu, mu
-vlseg7e8.v  v8,(a0)
-vsetvli zero, zero, e8, m4, tu, mu
-vlseg7e8.v  v8,(a0)
-vsetvli zero, zero, e16, mf8, tu, mu
-vlseg7e16.v v8,(a0)
 vsetvli zero, zero, e16, mf4, tu, mu
 vlseg7e16.v v8,(a0)
 vsetvli zero, zero, e16, mf2, tu, mu
 vlseg7e16.v v8,(a0)
 vsetvli zero, zero, e16, m1, tu, mu
 vlseg7e16.v v8,(a0)
-vsetvli zero, zero, e16, m2, tu, mu
-vlseg7e16.v v8,(a0)
-vsetvli zero, zero, e16, m4, tu, mu
-vlseg7e16.v v8,(a0)
-vsetvli zero, zero, e32, mf8, tu, mu
-vlseg7e32.v v8,(a0)
-vsetvli zero, zero, e32, mf4, tu, mu
-vlseg7e32.v v8,(a0)
 vsetvli zero, zero, e32, mf2, tu, mu
 vlseg7e32.v v8,(a0)
 vsetvli zero, zero, e32, m1, tu, mu
 vlseg7e32.v v8,(a0)
-vsetvli zero, zero, e32, m2, tu, mu
-vlseg7e32.v v8,(a0)
-vsetvli zero, zero, e32, m4, tu, mu
-vlseg7e32.v v8,(a0)
-vsetvli zero, zero, e64, mf8, tu, mu
-vlseg7e64.v v8,(a0)
-vsetvli zero, zero, e64, mf4, tu, mu
-vlseg7e64.v v8,(a0)
-vsetvli zero, zero, e64, mf2, tu, mu
-vlseg7e64.v v8,(a0)
 vsetvli zero, zero, e64, m1, tu, mu
 vlseg7e64.v v8,(a0)
-vsetvli zero, zero, e64, m2, tu, mu
-vlseg7e64.v v8,(a0)
-vsetvli zero, zero, e64, m4, tu, mu
-vlseg7e64.v v8,(a0)
 
 vsetvli zero, zero, e8, mf8, tu, mu
 vlseg8e8.v  v8,(a0)
@@ -327,46 +167,18 @@ vsetvli zero, zero, e8, mf2, tu, mu
 vlseg8e8.v  v8,(a0)
 vsetvli zero, zero, e8, m1, tu, mu
 vlseg8e8.v  v8,(a0)
-vsetvli zero, zero, e8, m2, tu, mu
-vlseg8e8.v  v8,(a0)
-vsetvli zero, zero, e8, m4, tu, mu
-vlseg8e8.v  v8,(a0)
-vsetvli zero, zero, e16, mf8, tu, mu
-vlseg8e16.v v8,(a0)
 vsetvli zero, zero, e16, mf4, tu, mu
 vlseg8e16.v v8,(a0)
 vsetvli zero, zero, e16, mf2, tu, mu
 vlseg8e16.v v8,(a0)
 vsetvli zero, zero, e16, m1, tu, mu
 vlseg8e16.v v8,(a0)
-vsetvli zero, zero, e16, m2, tu, mu
-vlseg8e16.v v8,(a0)
-vsetvli zero, zero, e16, m4, tu, mu
-vlseg8e16.v v8,(a0)
-vsetvli zero, zero, e32, mf8, tu, mu
-vlseg8e32.v v8,(a0)
-vsetvli zero, zero, e32, mf4, tu, mu
-vlseg8e32.v v8,(a0)
 vsetvli zero, zero, e32, mf2, tu, mu
 vlseg8e32.v v8,(a0)
 vsetvli zero, zero, e32, m1, tu, mu
 vlseg8e32.v v8,(a0)
-vsetvli zero, zero, e32, m2, tu, mu
-vlseg8e32.v v8,(a0)
-vsetvli zero, zero, e32, m4, tu, mu
-vlseg8e32.v v8,(a0)
-vsetvli zero, zero, e64, mf8, tu, mu
-vlseg8e64.v v8,(a0)
-vsetvli zero, zero, e64, mf4, tu, mu
-vlseg8e64.v v8,(a0)
-vsetvli zero, zero, e64, mf2, tu, mu
-vlseg8e64.v v8,(a0)
 vsetvli zero, zero, e64, m1, tu, mu
 vlseg8e64.v v8,(a0)
-vsetvli zero, zero, e64, m2, tu, mu
-vlseg8e64.v v8,(a0)
-vsetvli zero, zero, e64, m4, tu, mu
-vlseg8e64.v v8,(a0)
 
 vsetvli zero, zero, e8, mf8, tu, mu
 vsseg2e8.v  v8,(a0)
@@ -380,10 +192,6 @@ vsetvli zero, zero, e8, m2, tu, mu
 vsseg2e8.v  v8,(a0)
 vsetvli zero, zero, e8, m4, tu, mu
 vsseg2e8.v  v8,(a0)
-vsetvli zero, zero, e8, m8, tu, mu
-vsseg2e8.v  v8,(a0)
-vsetvli zero, zero, e16, mf8, tu, mu
-vsseg2e16.v v8,(a0)
 vsetvli zero, zero, e16, mf4, tu, mu
 vsseg2e16.v v8,(a0)
 vsetvli zero, zero, e16, mf2, tu, mu
@@ -394,12 +202,6 @@ vsetvli zero, zero, e16, m2, tu, mu
 vsseg2e16.v v8,(a0)
 vsetvli zero, zero, e16, m4, tu, mu
 vsseg2e16.v v8,(a0)
-vsetvli zero, zero, e16, m8, tu, mu
-vsseg2e16.v v8,(a0)
-vsetvli zero, zero, e32, mf8, tu, mu
-vsseg2e32.v v8,(a0)
-vsetvli zero, zero, e32, mf4, tu, mu
-vsseg2e32.v v8,(a0)
 vsetvli zero, zero, e32, mf2, tu, mu
 vsseg2e32.v v8,(a0)
 vsetvli zero, zero, e32, m1, tu, mu
@@ -408,22 +210,12 @@ vsetvli zero, zero, e32, m2, tu, mu
 vsseg2e32.v v8,(a0)
 vsetvli zero, zero, e32, m4, tu, mu
 vsseg2e32.v v8,(a0)
-vsetvli zero, zero, e32, m8, tu, mu
-vsseg2e32.v v8,(a0)
-vsetvli zero, zero, e64, mf8, tu, mu
-vsseg2e64.v v8,(a0)
-vsetvli zero, zero, e64, mf4, tu, mu
-vsseg2e64.v v8,(a0)
-vsetvli zero, zero, e64, mf2, tu, mu
-vsseg2e64.v v8,(a0)
 vsetvli zero, zero, e64, m1, tu, mu
 vsseg2e64.v v8,(a0)
 vsetvli zero, zero, e64, m2, tu, mu
 vsseg2e64.v v8,(a0)
 vsetvli zero, zero, e64, m4, tu, mu
 vsseg2e64.v v8,(a0)
-vsetvli zero, zero, e64, m8, tu, mu
-vsseg2e64.v v8,(a0)
 
 vsetvli zero, zero, e8, mf8, tu, mu
 vsseg3e8.v  v8,(a0)
@@ -435,12 +227,6 @@ vsetvli zero, zero, e8, m1, tu, mu
 vsseg3e8.v  v8,(a0)
 vsetvli zero, zero, e8, m2, tu, mu
 vsseg3e8.v  v8,(a0)
-vsetvli zero, zero, e8, m4, tu, mu
-vsseg3e8.v  v8,(a0)
-vsetvli zero, zero, e8, m8, tu, mu
-vsseg3e8.v  v8,(a0)
-vsetvli zero, zero, e16, mf8, tu, mu
-vsseg3e16.v v8,(a0)
 vsetvli zero, zero, e16, mf4, tu, mu
 vsseg3e16.v v8,(a0)
 vsetvli zero, zero, e16, mf2, tu, mu
@@ -449,38 +235,16 @@ vsetvli zero, zero, e16, m1, tu, mu
 vsseg3e16.v v8,(a0)
 vsetvli zero, zero, e16, m2, tu, mu
 vsseg3e16.v v8,(a0)
-vsetvli zero, zero, e16, m4, tu, mu
-vsseg3e16.v v8,(a0)
-vsetvli zero, zero, e16, m8, tu, mu
-vsseg3e16.v v8,(a0)
-vsetvli zero, zero, e32, mf8, tu, mu
-vsseg3e32.v v8,(a0)
-vsetvli zero, zero, e32, mf4, tu, mu
-vsseg3e32.v v8,(a0)
 vsetvli zero, zero, e32, mf2, tu, mu
 vsseg3e32.v v8,(a0)
 vsetvli zero, zero, e32, m1, tu, mu
 vsseg3e32.v v8,(a0)
 vsetvli zero, zero, e32, m2, tu, mu
 vsseg3e32.v v8,(a0)
-vsetvli zero, zero, e32, m4, tu, mu
-vsseg3e32.v v8,(a0)
-vsetvli zero, zero, e32, m8, tu, mu
-vsseg3e32.v v8,(a0)
-vsetvli zero, zero, e64, mf8, tu, mu
-vsseg3e64.v v8,(a0)
-vsetvli zero, zero, e64, mf4, tu, mu
-vsseg3e64.v v8,(a0)
-vsetvli zero, zero, e64, mf2, tu, mu
-vsseg3e64.v v8,(a0)
 vsetvli zero, zero, e64, m1, tu, mu
 vsseg3e64.v v8,(a0)
 vsetvli zero, zero, e64, m2, tu, mu
 vsseg3e64.v v8,(a0)
-vsetvli zero, zero, e64, m4, tu, mu
-vsseg3e64.v v8,(a0)
-vsetvli zero, zero, e64, m8, tu, mu
-vsseg3e64.v v8,(a0)
 
 vsetvli zero, zero, e8, mf8, tu, mu
 vsseg4e8.v  v8,(a0)
@@ -492,12 +256,6 @@ vsetvli zero, zero, e8, m1, tu, mu
 vsseg4e8.v  v8,(a0)
 vsetvli zero, zero, e8, m2, tu, mu
 vsseg4e8.v  v8,(a0)
-vsetvli zero, zero, e8, m4, tu, mu
-vsseg4e8.v  v8,(a0)
-vsetvli zero, zero, e8, m8, tu, mu
-vsseg4e8.v  v8,(a0)
-vsetvli zero, zero, e16, mf8, tu, mu
-vsseg4e16.v v8,(a0)
 vsetvli zero, zero, e16, mf4, tu, mu
 vsseg4e16.v v8,(a0)
 vsetvli zero, zero, e16, mf2, tu, mu
@@ -506,38 +264,16 @@ vsetvli zero, zero, e16, m1, tu, mu
 vsseg4e16.v v8,(a0)
 vsetvli zero, zero, e16, m2, tu, mu
 vsseg4e16.v v8,(a0)
-vsetvli zero, zero, e16, m4, tu, mu
-vsseg4e16.v v8,(a0)
-vsetvli zero, zero, e16, m8, tu, mu
-vsseg4e16.v v8,(a0)
-vsetvli zero, zero, e32, mf8, tu, mu
-vsseg4e32.v v8,(a0)
-vsetvli zero, zero, e32, mf4, tu, mu
-vsseg4e32.v v8,(a0)
 vsetvli zero, zero, e32, mf2, tu, mu
 vsseg4e32.v v8,(a0)
 vsetvli zero, zero, e32, m1, tu, mu
 vsseg4e32.v v8,(a0)
 vsetvli zero, zero, e32, m2, tu, mu
 vsseg4e32.v v8,(a0)
-vsetvli zero, zero, e32, m4, tu, mu
-vsseg4e32.v v8,(a0)
-vsetvli zero, zero, e32, m8, tu, mu
-vsseg4e32.v v8,(a0)
-vsetvli zero, zero, e64, mf8, tu, mu
-vsseg4e64.v v8,(a0)
-vsetvli zero, zero, e64, mf4, tu, mu
-vsseg4e64.v v8,(a0)
-vsetvli zero, zero, e64, mf2, tu, mu
-vsseg4e64.v v8,(a0)
 vsetvli zero, zero, e64, m1, tu, mu
 vsseg4e64.v v8,(a0)
 vsetvli zero, zero, e64, m2, tu, mu
 vsseg4e64.v v8,(a0)
-vsetvli zero, zero, e64, m4, tu, mu
-vsseg4e64.v v8,(a0)
-vsetvli zero, zero, e64, m8, tu, mu
-vsseg4e64.v v8,(a0)
 
 vsetvli zero, zero, e8, mf8, tu, mu
 vsseg5e8.v  v8,(a0)
@@ -547,46 +283,18 @@ vsetvli zero, zero, e8, mf2, tu, mu
 vsseg5e8.v  v8,(a0)
 vsetvli zero, zero, e8, m1, tu, mu
 vsseg5e8.v  v8,(a0)
-vsetvli zero, zero, e8, m2, tu, mu
-vsseg5e8.v  v8,(a0)
-vsetvli zero, zero, e8, m4, tu, mu
-vsseg5e8.v  v8,(a0)
-vsetvli zero, zero, e16, mf8, tu, mu
-vsseg5e16.v v8,(a0)
 vsetvli zero, zero, e16, mf4, tu, mu
 vsseg5e16.v v8,(a0)
 vsetvli zero, zero, e16, mf2, tu, mu
 vsseg5e16.v v8,(a0)
 vsetvli zero, zero, e16, m1, tu, mu
 vsseg5e16.v v8,(a0)
-vsetvli zero, zero, e16, m2, tu, mu
-vsseg5e16.v v8,(a0)
-vsetvli zero, zero, e16, m4, tu, mu
-vsseg5e16.v v8,(a0)
-vsetvli zero, zero, e32, mf8, tu, mu
-vsseg5e32.v v8,(a0)
-vsetvli zero, zero, e32, mf4, tu, mu
-vsseg5e32.v v8,(a0)
 vsetvli zero, zero, e32, mf2, tu, mu
 vsseg5e32.v v8,(a0)
 vsetvli zero, zero, e32, m1, tu, mu
 vsseg5e32.v v8,(a0)
-vsetvli zero, zero, e32, m2, tu, mu
-vsseg5e32.v v8,(a0)
-vsetvli zero, zero, e32, m4, tu, mu
-vsseg5e32.v v8,(a0)
-vsetvli zero, zero, e64, mf8, tu, mu
-vsseg5e64.v v8,(a0)
-vsetvli zero, zero, e64, mf4, tu, mu
-vsseg5e64.v v8,(a0)
-vsetvli zero, zero, e64, mf2, tu, mu
-vsseg5e64.v v8,(a0)
 vsetvli zero, zero, e64, m1, tu, mu
 vsseg5e64.v v8,(a0)
-vsetvli zero, zero, e64, m2, tu, mu
-vsseg5e64.v v8,(a0)
-vsetvli zero, zero, e64, m4, tu, mu
-vsseg5e64.v v8,(a0)
 
 vsetvli zero, zero, e8, mf8, tu, mu
 vsseg6e8.v  v8,(a0)
@@ -596,46 +304,18 @@ vsetvli zero, zero, e8, mf2, tu, mu
 vsseg6e8.v  v8,(a0)
 vsetvli zero, zero, e8, m1, tu, mu
 vsseg6e8.v  v8,(a0)
-vsetvli zero, zero, e8, m2, tu, mu
-vsseg6e8.v  v8,(a0)
-vsetvli zero, zero, e8, m4, tu, mu
-vsseg6e8.v  v8,(a0)
-vsetvli zero, zero, e16, mf8, tu, mu
-vsseg6e16.v v8,(a0)
 vsetvli zero, zero, e16, mf4, tu, mu
 vsseg6e16.v v8,(a0)
 vsetvli zero, zero, e16, mf2, tu, mu
 vsseg6e16.v v8,(a0)
 vsetvli zero, zero, e16, m1, tu, mu
 vsseg6e16.v v8,(a0)
-vsetvli zero, zero, e16, m2, tu, mu
-vsseg6e16.v v8,(a0)
-vsetvli zero, zero, e16, m4, tu, mu
-vsseg6e16.v v8,(a0)
-vsetvli zero, zero, e32, mf8, tu, mu
-vsseg6e32.v v8,(a0)
-vsetvli zero, zero, e32, mf4, tu, mu
-vsseg6e32.v v8,(a0)
 vsetvli zero, zero, e32, mf2, tu, mu
 vsseg6e32.v v8,(a0)
 vsetvli zero, zero, e32, m1, tu, mu
 vsseg6e32.v v8,(a0)
-vsetvli zero, zero, e32, m2, tu, mu
-vsseg6e32.v v8,(a0)
-vsetvli zero, zero, e32, m4, tu, mu
-vsseg6e32.v v8,(a0)
-vsetvli zero, zero, e64, mf8, tu, mu
-vsseg6e64.v v8,(a0)
-vsetvli zero, zero, e64, mf4, tu, mu
-vsseg6e64.v v8,(a0)
-vsetvli zero, zero, e64, mf2, tu, mu
-vsseg6e64.v v8,(a0)
 vsetvli zero, zero, e64, m1, tu, mu
 vsseg6e64.v v8,(a0)
-vsetvli zero, zero, e64, m2, tu, mu
-vsseg6e64.v v8,(a0)
-vsetvli zero, zero, e64, m4, tu, mu
-vsseg6e64.v v8,(a0)
 
 vsetvli zero, zero, e8, mf8, tu, mu
 vsseg7e8.v  v8,(a0)
@@ -645,46 +325,18 @@ vsetvli zero, zero, e8, mf2, tu, mu
 vsseg7e8.v  v8,(a0)
 vsetvli zero, zero, e8, m1, tu, mu
 vsseg7e8.v  v8,(a0)
-vsetvli zero, zero, e8, m2, tu, mu
-vsseg7e8.v  v8,(a0)
-vsetvli zero, zero, e8, m4, tu, mu
-vsseg7e8.v  v8,(a0)
-vsetvli zero, zero, e16, mf8, tu, mu
-vsseg7e16.v v8,(a0)
 vsetvli zero, zero, e16, mf4, tu, mu
 vsseg7e16.v v8,(a0)
 vsetvli zero, zero, e16, mf2, tu, mu
 vsseg7e16.v v8,(a0)
 vsetvli zero, zero, e16, m1, tu, mu
 vsseg7e16.v v8,(a0)
-vsetvli zero, zero, e16, m2, tu, mu
-vsseg7e16.v v8,(a0)
-vsetvli zero, zero, e16, m4, tu, mu
-vsseg7e16.v v8,(a0)
-vsetvli zero, zero, e32, mf8, tu, mu
-vsseg7e32.v v8,(a0)
-vsetvli zero, zero, e32, mf4, tu, mu
-vsseg7e32.v v8,(a0)
 vsetvli zero, zero, e32, mf2, tu, mu
 vsseg7e32.v v8,(a0)
 vsetvli zero, zero, e32, m1, tu, mu
 vsseg7e32.v v8,(a0)
-vsetvli zero, zero, e32, m2, tu, mu
-vsseg7e32.v v8,(a0)
-vsetvli zero, zero, e32, m4, tu, mu
-vsseg7e32.v v8,(a0)
-vsetvli zero, zero, e64, mf8, tu, mu
-vsseg7e64.v v8,(a0)
-vsetvli zero, zero, e64, mf4, tu, mu
-vsseg7e64.v v8,(a0)
-vsetvli zero, zero, e64, mf2, tu, mu
-vsseg7e64.v v8,(a0)
 vsetvli zero, zero, e64, m1, tu, mu
 vsseg7e64.v v8,(a0)
-vsetvli zero, zero, e64, m2, tu, mu
-vsseg7e64.v v8,(a0)
-vsetvli zero, zero, e64, m4, tu, mu
-vsseg7e64.v v8,(a0)
 
 vsetvli zero, zero, e8, mf8, tu, mu
 vsseg8e8.v  v8,(a0)
@@ -694,46 +346,18 @@ vsetvli zero, zero, e8, mf2, tu, mu
 vsseg8e8.v  v8,(a0)
 vsetvli zero, zero, e8, m1, tu, mu
 vsseg8e8.v  v8,(a0)
-vsetvli zero, zero, e8, m2, tu, mu
-vsseg8e8.v  v8,(a0)
-vsetvli zero, zero, e8, m4, tu, mu
-vsseg8e8.v  v8,(a0)
-vsetvli zero, zero, e16, mf8, tu, mu
-vsseg8e16.v v8,(a0)
 vsetvli zero, zero, e16, mf4, tu, mu
 vsseg8e16.v v8,(a0)
 vsetvli zero, zero, e16, mf2, tu, mu
 vsseg8e16.v v8,(a0)
 vsetvli zero, zero, e16, m1, tu, mu
 vsseg8e16.v v8,(a0)
-vsetvli zero, zero, e16, m2, tu, mu
-vsseg8e16.v v8,(a0)
-vsetvli zero, zero, e16, m4, tu, mu
-vsseg8e16.v v8,(a0)
-vsetvli zero, zero, e32, mf8, tu, mu
-vsseg8e32.v v8,(a0)
-vsetvli zero, zero, e32, mf4, tu, mu
-vsseg8e32.v v8,(a0)
-vsetvli zero, zero, e32, mf2, tu, mu
+vsetvli zero, zero, e32, mf2, tu, mu
 vsseg8e32.v v8,(a0)
 vsetvli zero, zero, e32, m1, tu, mu
 vsseg8e32.v v8,(a0)
-vsetvli zero, zero, e32, m2, tu, mu
-vsseg8e32.v v8,(a0)
-vsetvli zero, zero, e32, m4, tu, mu
-vsseg8e32.v v8,(a0)
-vsetvli zero, zero, e64, mf8, tu, mu
-vsseg8e64.v v8,(a0)
-vsetvli zero, zero, e64, mf4, tu, mu
-vsseg8e64.v v8,(a0)
-vsetvli zero, zero, e64, mf2, tu, mu
-vsseg8e64.v v8,(a0)
 vsetvli zero, zero, e64, m1, tu, mu
 vsseg8e64.v v8,(a0)
-vsetvli zero, zero, e64, m2, tu, mu
-vsseg8e64.v v8,(a0)
-vsetvli zero, zero, e64, m4, tu, mu
-vsseg8e64.v v8,(a0)
 
 vsetvli zero, zero, e8, mf8, tu, mu
 vlsseg2e8.v v8, (a0), a1
@@ -747,10 +371,6 @@ vsetvli zero, zero, e8, m2, tu, mu
 vlsseg2e8.v v8, (a0), a1
 vsetvli zero, zero, e8, m4, tu, mu
 vlsseg2e8.v v8, (a0), a1
-vsetvli zero, zero, e8, m8, tu, mu
-vlsseg2e8.v v8, (a0), a1
-vsetvli zero, zero, e16, mf8, tu, mu
-vlsseg2e16.v  v8, (a0), a1
 vsetvli zero, zero, e16, mf4, tu, mu
 vlsseg2e16.v  v8, (a0), a1
 vsetvli zero, zero, e16, mf2, tu, mu
@@ -761,12 +381,6 @@ vsetvli zero, zero, e16, m2, tu, mu
 vlsseg2e16.v  v8, (a0), a1
 vsetvli zero, zero, e16, m4, tu, mu
 vlsseg2e16.v  v8, (a0), a1
-vsetvli zero, zero, e16, m8, tu, mu
-vlsseg2e16.v  v8, (a0), a1
-vsetvli zero, zero, e32, mf8, tu, mu
-vlsseg2e32.v  v8, (a0), a1
-vsetvli zero, zero, e32, mf4, tu, mu
-vlsseg2e32.v  v8, (a0), a1
 vsetvli zero, zero, e32, mf2, tu, mu
 vlsseg2e32.v  v8, (a0), a1
 vsetvli zero, zero, e32, m1, tu, mu
@@ -775,22 +389,12 @@ vsetvli zero, zero, e32, m2, tu, mu
 vlsseg2e32.v  v8, (a0), a1
 vsetvli zero, zero, e32, m4, tu, mu
 vlsseg2e32.v  v8, (a0), a1
-vsetvli zero, zero, e32, m8, tu, mu
-vlsseg2e32.v  v8, (a0), a1
-vsetvli zero, zero, e64, mf8, tu, mu
-vlsseg2e64.v  v8, (a0), a1
-vsetvli zero, zero, e64, mf4, tu, mu
-vlsseg2e64.v  v8, (a0), a1
-vsetvli zero, zero, e64, mf2, tu, mu
-vlsseg2e64.v  v8, (a0), a1
 vsetvli zero, zero, e64, m1, tu, mu
 vlsseg2e64.v  v8, (a0), a1
 vsetvli zero, zero, e64, m2, tu, mu
 vlsseg2e64.v  v8, (a0), a1
 vsetvli zero, zero, e64, m4, tu, mu
 vlsseg2e64.v  v8, (a0), a1
-vsetvli zero, zero, e64, m8, tu, mu
-vlsseg2e64.v  v8, (a0), a1
 
 vsetvli zero, zero, e8, mf8, tu, mu
 vlsseg3e8.v v8, (a0), a1
@@ -802,12 +406,6 @@ vsetvli zero, zero, e8, m1, tu, mu
 vlsseg3e8.v v8, (a0), a1
 vsetvli zero, zero, e8, m2, tu, mu
 vlsseg3e8.v v8, (a0), a1
-vsetvli zero, zero, e8, m4, tu, mu
-vlsseg3e8.v v8, (a0), a1
-vsetvli zero, zero, e8, m8, tu, mu
-vlsseg3e8.v v8, (a0), a1
-vsetvli zero, zero, e16, mf8, tu, mu
-vlsseg3e16.v  v8, (a0), a1
 vsetvli zero, zero, e16, mf4, tu, mu
 vlsseg3e16.v  v8, (a0), a1
 vsetvli zero, zero, e16, mf2, tu, mu
@@ -816,38 +414,16 @@ vsetvli zero, zero, e16, m1, tu, mu
 vlsseg3e16.v  v8, (a0), a1
 vsetvli zero, zero, e16, m2, tu, mu
 vlsseg3e16.v  v8, (a0), a1
-vsetvli zero, zero, e16, m4, tu, mu
-vlsseg3e16.v  v8, (a0), a1
-vsetvli zero, zero, e16, m8, tu, mu
-vlsseg3e16.v  v8, (a0), a1
-vsetvli zero, zero, e32, mf8, tu, mu
-vlsseg3e32.v  v8, (a0), a1
-vsetvli zero, zero, e32, mf4, tu, mu
-vlsseg3e32.v  v8, (a0), a1
 vsetvli zero, zero, e32, mf2, tu, mu
 vlsseg3e32.v  v8, (a0), a1
 vsetvli zero, zero, e32, m1, tu, mu
 vlsseg3e32.v  v8, (a0), a1
 vsetvli zero, zero, e32, m2, tu, mu
 vlsseg3e32.v  v8, (a0), a1
-vsetvli zero, zero, e32, m4, tu, mu
-vlsseg3e32.v  v8, (a0), a1
-vsetvli zero, zero, e32, m8, tu, mu
-vlsseg3e32.v  v8, (a0), a1
-vsetvli zero, zero, e64, mf8, tu, mu
-vlsseg3e64.v  v8, (a0), a1
-vsetvli zero, zero, e64, mf4, tu, mu
-vlsseg3e64.v  v8, (a0), a1
-vsetvli zero, zero, e64, mf2, tu, mu
-vlsseg3e64.v  v8, (a0), a1
 vsetvli zero, zero, e64, m1, tu, mu
 vlsseg3e64.v  v8, (a0), a1
 vsetvli zero, zero, e64, m2, tu, mu
 vlsseg3e64.v  v8, (a0), a1
-vsetvli zero, zero, e64, m4, tu, mu
-vlsseg3e64.v  v8, (a0), a1
-vsetvli zero, zero, e64, m8, tu, mu
-vlsseg3e64.v  v8, (a0), a1
 
 vsetvli zero, zero, e8, mf8, tu, mu
 vlsseg4e8.v v8, (a0), a1
@@ -859,12 +435,6 @@ vsetvli zero, zero, e8, m1, tu, mu
 vlsseg4e8.v v8, (a0), a1
 vsetvli zero, zero, e8, m2, tu, mu
 vlsseg4e8.v v8, (a0), a1
-vsetvli zero, zero, e8, m4, tu, mu
-vlsseg4e8.v v8, (a0), a1
-vsetvli zero, zero, e8, m8, tu, mu
-vlsseg4e8.v v8, (a0), a1
-vsetvli zero, zero, e16, mf8, tu, mu
-vlsseg4e16.v  v8, (a0), a1
 vsetvli zero, zero, e16, mf4, tu, mu
 vlsseg4e16.v  v8, (a0), a1
 vsetvli zero, zero, e16, mf2, tu, mu
@@ -873,38 +443,16 @@ vsetvli zero, zero, e16, m1, tu, mu
 vlsseg4e16.v  v8, (a0), a1
 vsetvli zero, zero, e16, m2, tu, mu
 vlsseg4e16.v  v8, (a0), a1
-vsetvli zero, zero, e16, m4, tu, mu
-vlsseg4e16.v  v8, (a0), a1
-vsetvli zero, zero, e16, m8, tu, mu
-vlsseg4e16.v  v8, (a0), a1
-vsetvli zero, zero, e32, mf8, tu, mu
-vlsseg4e32.v  v8, (a0), a1
-vsetvli zero, zero, e32, mf4, tu, mu
-vlsseg4e32.v  v8, (a0), a1
 vsetvli zero, zero, e32, mf2, tu, mu
 vlsseg4e32.v  v8, (a0), a1
 vsetvli zero, zero, e32, m1, tu, mu
 vlsseg4e32.v  v8, (a0), a1
 vsetvli zero, zero, e32, m2, tu, mu
 vlsseg4e32.v  v8, (a0), a1
-vsetvli zero, zero, e32, m4, tu, mu
-vlsseg4e32.v  v8, (a0), a1
-vsetvli zero, zero, e32, m8, tu, mu
-vlsseg4e32.v  v8, (a0), a1
-vsetvli zero, zero, e64, mf8, tu, mu
-vlsseg4e64.v  v8, (a0), a1
-vsetvli zero, zero, e64, mf4, tu, mu
-vlsseg4e64.v  v8, (a0), a1
-vsetvli zero, zero, e64, mf2, tu, mu
-vlsseg4e64.v  v8, (a0), a1
 vsetvli zero, zero, e64, m1, tu, mu
 vlsseg4e64.v  v8, (a0), a1
 vsetvli zero, zero, e64, m2, tu, mu
 vlsseg4e64.v  v8, (a0), a1
-vsetvli zero, zero, e64, m4, tu, mu
-vlsseg4e64.v  v8, (a0), a1
-vsetvli zero, zero, e64, m8, tu, mu
-vlsseg4e64.v  v8, (a0), a1
 
 vsetvli zero, zero, e8, mf8, tu, mu
 vlsseg5e8.v v8, (a0), a1
@@ -914,46 +462,18 @@ vsetvli zero, zero, e8, mf2, tu, mu
 vlsseg5e8.v v8, (a0), a1
 vsetvli zero, zero, e8, m1, tu, mu
 vlsseg5e8.v v8, (a0), a1
-vsetvli zero, zero, e8, m2, tu, mu
-vlsseg5e8.v v8, (a0), a1
-vsetvli zero, zero, e8, m4, tu, mu
-vlsseg5e8.v v8, (a0), a1
-vsetvli zero, zero, e16, mf8, tu, mu
-vlsseg5e16.v  v8, (a0), a1
 vsetvli zero, zero, e16, mf4, tu, mu
 vlsseg5e16.v  v8, (a0), a1
 vsetvli zero, zero, e16, mf2, tu, mu
 vlsseg5e16.v  v8, (a0), a1
 vsetvli zero, zero, e16, m1, tu, mu
 vlsseg5e16.v  v8, (a0), a1
-vsetvli zero, zero, e16, m2, tu, mu
-vlsseg5e16.v  v8, (a0), a1
-vsetvli zero, zero, e16, m4, tu, mu
-vlsseg5e16.v  v8, (a0), a1
-vsetvli zero, zero, e32, mf8, tu, mu
-vlsseg5e32.v  v8, (a0), a1
-vsetvli zero, zero, e32, mf4, tu, mu
-vlsseg5e32.v  v8, (a0), a1
 vsetvli zero, zero, e32, mf2, tu, mu
 vlsseg5e32.v  v8, (a0), a1
 vsetvli zero, zero, e32, m1, tu, mu
 vlsseg5e32.v  v8, (a0), a1
-vsetvli zero, zero, e32, m2, tu, mu
-vlsseg5e32.v  v8, (a0), a1
-vsetvli zero, zero, e32, m4, tu, mu
-vlsseg5e32.v  v8, (a0), a1
-vsetvli zero, zero, e64, mf8, tu, mu
-vlsseg5e64.v  v8, (a0), a1
-vsetvli zero, zero, e64, mf4, tu, mu
-vlsseg5e64.v  v8, (a0), a1
-vsetvli zero, zero, e64, mf2, tu, mu
-vlsseg5e64.v  v8, (a0), a1
 vsetvli zero, zero, e64, m1, tu, mu
 vlsseg5e64.v  v8, (a0), a1
-vsetvli zero, zero, e64, m2, tu, mu
-vlsseg5e64.v  v8, (a0), a1
-vsetvli zero, zero, e64, m4, tu, mu
-vlsseg5e64.v  v8, (a0), a1
 
 vsetvli zero, zero, e8, mf8, tu, mu
 vlsseg6e8.v v8, (a0), a1
@@ -963,46 +483,18 @@ vsetvli zero, zero, e8, mf2, tu, mu
 vlsseg6e8.v v8, (a0), a1
 vsetvli zero, zero, e8, m1, tu, mu
 vlsseg6e8.v v8, (a0), a1
-vsetvli zero, zero, e8, m2, tu, mu
-vlsseg6e8.v v8, (a0), a1
-vsetvli zero, zero, e8, m4, tu, mu
-vlsseg6e8.v v8, (a0), a1
-vsetvli zero, zero, e16, mf8, tu, mu
-vlsseg6e16.v  v8, (a0), a1
 vsetvli zero, zero, e16, mf4, tu, mu
 vlsseg6e16.v  v8, (a0), a1
 vsetvli zero, zero, e16, mf2, tu, mu
 vlsseg6e16.v  v8, (a0), a1
 vsetvli zero, zero, e16, m1, tu, mu
 vlsseg6e16.v  v8, (a0), a1
-vsetvli zero, zero, e16, m2, tu, mu
-vlsseg6e16.v  v8, (a0), a1
-vsetvli zero, zero, e16, m4, tu, mu
-vlsseg6e16.v  v8, (a0), a1
-vsetvli zero, zero, e32, mf8, tu, mu
-vlsseg6e32.v  v8, (a0), a1
-vsetvli zero, zero, e32, mf4, tu, mu
-vlsseg6e32.v  v8, (a0), a1
 vsetvli zero, zero, e32, mf2, tu, mu
 vlsseg6e32.v  v8, (a0), a1
 vsetvli zero, zero, e32, m1, tu, mu
 vlsseg6e32.v  v8, (a0), a1
-vsetvli zero, zero, e32, m2, tu, mu
-vlsseg6e32.v  v8, (a0), a1
-vsetvli zero, zero, e32, m4, tu, mu
-vlsseg6e32.v  v8, (a0), a1
-vsetvli zero, zero, e64, mf8, tu, mu
-vlsseg6e64.v  v8, (a0), a1
-vsetvli zero, zero, e64, mf4, tu, mu
-vlsseg6e64.v  v8, (a0), a1
-vsetvli zero, zero, e64, mf2, tu, mu
-vlsseg6e64.v  v8, (a0), a1
 vsetvli zero, zero, e64, m1, tu, mu
 vlsseg6e64.v  v8, (a0), a1
-vsetvli zero, zero, e64, m2, tu, mu
-vlsseg6e64.v  v8, (a0), a1
-vsetvli zero, zero, e64, m4, tu, mu
-vlsseg6e64.v  v8, (a0), a1
 
 vsetvli zero, zero, e8, mf8, tu, mu
 vlsseg7e8.v v8, (a0), a1
@@ -1012,46 +504,18 @@ vsetvli zero, zero, e8, mf2, tu, mu
 vlsseg7e8.v v8, (a0), a1
 vsetvli zero, zero, e8, m1, tu, mu
 vlsseg7e8.v v8, (a0), a1
-vsetvli zero, zero, e8, m2, tu, mu
-vlsseg7e8.v v8, (a0), a1
-vsetvli zero, zero, e8, m4, tu, mu
-vlsseg7e8.v v8, (a0), a1
-vsetvli zero, zero, e16, mf8, tu, mu
-vlsseg7e16.v  v8, (a0), a1
 vsetvli zero, zero, e16, mf4, tu, mu
 vlsseg7e16.v  v8, (a0), a1
 vsetvli zero, zero, e16, mf2, tu, mu
 vlsseg7e16.v  v8, (a0), a1
 vsetvli zero, zero, e16, m1, tu, mu
 vlsseg7e16.v  v8, (a0), a1
-vsetvli zero, zero, e16, m2, tu, mu
-vlsseg7e16.v  v8, (a0), a1
-vsetvli zero, zero, e16, m4, tu, mu
-vlsseg7e16.v  v8, (a0), a1
-vsetvli zero, zero, e32, mf8, tu, mu
-vlsseg7e32.v  v8, (a0), a1
-vsetvli zero, zero, e32, mf4, tu, mu
-vlsseg7e32.v  v8, (a0), a1
 vsetvli zero, zero, e32, mf2, tu, mu
 vlsseg7e32.v  v8, (a0), a1
 vsetvli zero, zero, e32, m1, tu, mu
 vlsseg7e32.v  v8, (a0), a1
-vsetvli zero, zero, e32, m2, tu, mu
-vlsseg7e32.v  v8, (a0), a1
-vsetvli zero, zero, e32, m4, tu, mu
-vlsseg7e32.v  v8, (a0), a1
-vsetvli zero, zero, e64, mf8, tu, mu
-vlsseg7e64.v  v8, (a0), a1
-vsetvli zero, zero, e64, mf4, tu, mu
-vlsseg7e64.v  v8, (a0), a1
-vsetvli zero, zero, e64, mf2, tu, mu
-vlsseg7e64.v  v8, (a0), a1
 vsetvli zero, zero, e64, m1, tu, mu
 vlsseg7e64.v  v8, (a0), a1
-vsetvli zero, zero, e64, m2, tu, mu
-vlsseg7e64.v  v8, (a0), a1
-vsetvli zero, zero, e64, m4, tu, mu
-vlsseg7e64.v  v8, (a0), a1
 
 vsetvli zero, zero, e8, mf8, tu, mu
 vlsseg8e8.v v8, (a0), a1
@@ -1061,46 +525,18 @@ vsetvli zero, zero, e8, mf2, tu, mu
 vlsseg8e8.v v8, (a0), a1
 vsetvli zero, zero, e8, m1, tu, mu
 vlsseg8e8.v v8, (a0), a1
-vsetvli zero, zero, e8, m2, tu, mu
-vlsseg8e8.v v8, (a0), a1
-vsetvli zero, zero, e8, m4, tu, mu
-vlsseg8e8.v v8, (a0), a1
-vsetvli zero, zero, e16, mf8, tu, mu
-vlsseg8e16.v  v8, (a0), a1
 vsetvli zero, zero, e16, mf4, tu, mu
 vlsseg8e16.v  v8, (a0), a1
 vsetvli zero, zero, e16, mf2, tu, mu
 vlsseg8e16.v  v8, (a0), a1
 vsetvli zero, zero, e16, m1, tu, mu
 vlsseg8e16.v  v8, (a0), a1
-vsetvli zero, zero, e16, m2, tu, mu
-vlsseg8e16.v  v8, (a0), a1
-vsetvli zero, zero, e16, m4, tu, mu
-vlsseg8e16.v  v8, (a0), a1
-vsetvli zero, zero, e32, mf8, tu, mu
-vlsseg8e32.v  v8, (a0), a1
-vsetvli zero, zero, e32, mf4, tu, mu
-vlsseg8e32.v  v8, (a0), a1
 vsetvli zero, zero, e32, mf2, tu, mu
 vlsseg8e32.v  v8, (a0), a1
 vsetvli zero, zero, e32, m1, tu, mu
 vlsseg8e32.v  v8, (a0), a1
-vsetvli zero, zero, e32, m2, tu, mu
-vlsseg8e32.v  v8, (a0), a1
-vsetvli zero, zero, e32, m4, tu, mu
-vlsseg8e32.v  v8, (a0), a1
-vsetvli zero, zero, e64, mf8, tu, mu
-vlsseg8e64.v  v8, (a0), a1
-vsetvli zero, zero, e64, mf4, tu, mu
-vlsseg8e64.v  v8, (a0), a1
-vsetvli zero, zero, e64, mf2, tu, mu
-vlsseg8e64.v  v8, (a0), a1
 vsetvli zero, zero, e64, m1, tu, mu
 vlsseg8e64.v  v8, (a0), a1
-vsetvli zero, zero, e64, m2, tu, mu
-vlsseg8e64.v  v8, (a0), a1
-vsetvli zero, zero, e64, m4, tu, mu
-vlsseg8e64.v  v8, (a0), a1
 
 vsetvli zero, zero, e8, mf8, tu, mu
 vssseg2e8.v v8, (a0), a1
@@ -1114,10 +550,6 @@ vsetvli zero, zero, e8, m2, tu, mu
 vssseg2e8.v v8, (a0), a1
 vsetvli zero, zero, e8, m4, tu, mu
 vssseg2e8.v v8, (a0), a1
-vsetvli zero, zero, e8, m8, tu, mu
-vssseg2e8.v v8, (a0), a1
-vsetvli zero, zero, e16, mf8, tu, mu
-vssseg2e16.v  v8, (a0), a1
 vsetvli zero, zero, e16, mf4, tu, mu
 vssseg2e16.v  v8, (a0), a1
 vsetvli zero, zero, e16, mf2, tu, mu
@@ -1128,12 +560,6 @@ vsetvli zero, zero, e16, m2, tu, mu
 vssseg2e16.v  v8, (a0), a1
 vsetvli zero, zero, e16, m4, tu, mu
 vssseg2e16.v  v8, (a0), a1
-vsetvli zero, zero, e16, m8, tu, mu
-vssseg2e16.v  v8, (a0), a1
-vsetvli zero, zero, e32, mf8, tu, mu
-vssseg2e32.v  v8, (a0), a1
-vsetvli zero, zero, e32, mf4, tu, mu
-vssseg2e32.v  v8, (a0), a1
 vsetvli zero, zero, e32, mf2, tu, mu
 vssseg2e32.v  v8, (a0), a1
 vsetvli zero, zero, e32, m1, tu, mu
@@ -1142,22 +568,12 @@ vsetvli zero, zero, e32, m2, tu, mu
 vssseg2e32.v  v8, (a0), a1
 vsetvli zero, zero, e32, m4, tu, mu
 vssseg2e32.v  v8, (a0), a1
-vsetvli zero, zero, e32, m8, tu, mu
-vssseg2e32.v  v8, (a0), a1
-vsetvli zero, zero, e64, mf8, tu, mu
-vssseg2e64.v  v8, (a0), a1
-vsetvli zero, zero, e64, mf4, tu, mu
-vssseg2e64.v  v8, (a0), a1
-vsetvli zero, zero, e64, mf2, tu, mu
-vssseg2e64.v  v8, (a0), a1
 vsetvli zero, zero, e64, m1, tu, mu
 vssseg2e64.v  v8, (a0), a1
 vsetvli zero, zero, e64, m2, tu, mu
 vssseg2e64.v  v8, (a0), a1
 vsetvli zero, zero, e64, m4, tu, mu
 vssseg2e64.v  v8, (a0), a1
-vsetvli zero, zero, e64, m8, tu, mu
-vssseg2e64.v  v8, (a0), a1
 
 vsetvli zero, zero, e8, mf8, tu, mu
 vssseg3e8.v v8, (a0), a1
@@ -1169,12 +585,6 @@ vsetvli zero, zero, e8, m1, tu, mu
 vssseg3e8.v v8, (a0), a1
 vsetvli zero, zero, e8, m2, tu, mu
 vssseg3e8.v v8, (a0), a1
-vsetvli zero, zero, e8, m4, tu, mu
-vssseg3e8.v v8, (a0), a1
-vsetvli zero, zero, e8, m8, tu, mu
-vssseg3e8.v v8, (a0), a1
-vsetvli zero, zero, e16, mf8, tu, mu
-vssseg3e16.v  v8, (a0), a1
 vsetvli zero, zero, e16, mf4, tu, mu
 vssseg3e16.v  v8, (a0), a1
 vsetvli zero, zero, e16, mf2, tu, mu
@@ -1183,38 +593,16 @@ vsetvli zero, zero, e16, m1, tu, mu
 vssseg3e16.v  v8, (a0), a1
 vsetvli zero, zero, e16, m2, tu, mu
 vssseg3e16.v  v8, (a0), a1
-vsetvli zero, zero, e16, m4, tu, mu
-vssseg3e16.v  v8, (a0), a1
-vsetvli zero, zero, e16, m8, tu, mu
-vssseg3e16.v  v8, (a0), a1
-vsetvli zero, zero, e32, mf8, tu, mu
-vssseg3e32.v  v8, (a0), a1
-vsetvli zero, zero, e32, mf4, tu, mu
-vssseg3e32.v  v8, (a0), a1
 vsetvli zero, zero, e32, mf2, tu, mu
 vssseg3e32.v  v8, (a0), a1
 vsetvli zero, zero, e32, m1, tu, mu
 vssseg3e32.v  v8, (a0), a1
 vsetvli zero, zero, e32, m2, tu, mu
 vssseg3e32.v  v8, (a0), a1
-vsetvli zero, zero, e32, m4, tu, mu
-vssseg3e32.v  v8, (a0), a1
-vsetvli zero, zero, e32, m8, tu, mu
-vssseg3e32.v  v8, (a0), a1
-vsetvli zero, zero, e64, mf8, tu, mu
-vssseg3e64.v  v8, (a0), a1
-vsetvli zero, zero, e64, mf4, tu, mu
-vssseg3e64.v  v8, (a0), a1
-vsetvli zero, zero, e64, mf2, tu, mu
-vssseg3e64.v  v8, (a0), a1
 vsetvli zero, zero, e64, m1, tu, mu
 vssseg3e64.v  v8, (a0), a1
 vsetvli zero, zero, e64, m2, tu, mu
 vssseg3e64.v  v8, (a0), a1
-vsetvli zero, zero, e64, m4, tu, mu
-vssseg3e64.v  v8, (a0), a1
-vsetvli zero, zero, e64, m8, tu, mu
-vssseg3e64.v  v8, (a0), a1
 
 vsetvli zero, zero, e8, mf8, tu, mu
 vssseg4e8.v v8, (a0), a1
@@ -1226,12 +614,6 @@ vsetvli zero, zero, e8, m1, tu, mu
 vssseg4e8.v v8, (a0), a1
 vsetvli zero, zero, e8, m2, tu, mu
 vssseg4e8.v v8, (a0), a1
-vsetvli zero, zero, e8, m4, tu, mu
-vssseg4e8.v v8, (a0), a1
-vsetvli zero, zero, e8, m8, tu, mu
-vssseg4e8.v v8, (a0), a1
-vsetvli zero, zero, e16, mf8, tu, mu
-vssseg4e16.v  v8, (a0), a1
 vsetvli zero, zero, e16, mf4, tu, mu
 vssseg4e16.v  v8, (a0), a1
 vsetvli zero, zero, e16, mf2, tu, mu
@@ -1240,38 +622,16 @@ vsetvli zero, zero, e16, m1, tu, mu
 vssseg4e16.v  v8, (a0), a1
 vsetvli zero, zero, e16, m2, tu, mu
 vssseg4e16.v  v8, (a0), a1
-vsetvli zero, zero, e16, m4, tu, mu
-vssseg4e16.v  v8, (a0), a1
-vsetvli zero, zero, e16, m8, tu, mu
-vssseg4e16.v  v8, (a0), a1
-vsetvli zero, zero, e32, mf8, tu, mu
-vssseg4e32.v  v8, (a0), a1
-vsetvli zero, zero, e32, mf4, tu, mu
-vssseg4e32.v  v8, (a0), a1
 vsetvli zero, zero, e32, mf2, tu, mu
 vssseg4e32.v  v8, (a0), a1
 vsetvli zero, zero, e32, m1, tu, mu
 vssseg4e32.v  v8, (a0), a1
 vsetvli zero, zero, e32, m2, tu, mu
 vssseg4e32.v  v8, (a0), a1
-vsetvli zero, zero, e32, m4, tu, mu
-vssseg4e32.v  v8, (a0), a1
-vsetvli zero, zero, e32, m8, tu, mu
-vssseg4e32.v  v8, (a0), a1
-vsetvli zero, zero, e64, mf8, tu, mu
-vssseg4e64.v  v8, (a0), a1
-vsetvli zero, zero, e64, mf4, tu, mu
-vssseg4e64.v  v8, (a0), a1
-vsetvli zero, zero, e64, mf2, tu, mu
-vssseg4e64.v  v8, (a0), a1
 vsetvli zero, zero, e64, m1, tu, mu
 vssseg4e64.v  v8, (a0), a1
 vsetvli zero, zero, e64, m2, tu, mu
 vssseg4e64.v  v8, (a0), a1
-vsetvli zero, zero, e64, m4, tu, mu
-vssseg4e64.v  v8, (a0), a1
-vsetvli zero, zero, e64, m8, tu, mu
-vssseg4e64.v  v8, (a0), a1
 
 vsetvli zero, zero, e8, mf8, tu, mu
 vssseg5e8.v v8, (a0), a1
@@ -1281,46 +641,18 @@ vsetvli zero, zero, e8, mf2, tu, mu
 vssseg5e8.v v8, (a0), a1
 vsetvli zero, zero, e8, m1, tu, mu
 vssseg5e8.v v8, (a0), a1
-vsetvli zero, zero, e8, m2, tu, mu
-vssseg5e8.v v8, (a0), a1
-vsetvli zero, zero, e8, m4, tu, mu
-vssseg5e8.v v8, (a0), a1
-vsetvli zero, zero, e16, mf8, tu, mu
-vssseg5e16.v  v8, (a0), a1
 vsetvli zero, zero, e16, mf4, tu, mu
 vssseg5e16.v  v8, (a0), a1
 vsetvli zero, zero, e16, mf2, tu, mu
 vssseg5e16.v  v8, (a0), a1
 vsetvli zero, zero, e16, m1, tu, mu
 vssseg5e16.v  v8, (a0), a1
-vsetvli zero, zero, e16, m2, tu, mu
-vssseg5e16.v  v8, (a0), a1
-vsetvli zero, zero, e16, m4, tu, mu
-vssseg5e16.v  v8, (a0), a1
-vsetvli zero, zero, e32, mf8, tu, mu
-vssseg5e32.v  v8, (a0), a1
-vsetvli zero, zero, e32, mf4, tu, mu
-vssseg5e32.v  v8, (a0), a1
 vsetvli zero, zero, e32, mf2, tu, mu
 vssseg5e32.v  v8, (a0), a1
 vsetvli zero, zero, e32, m1, tu, mu
 vssseg5e32.v  v8, (a0), a1
-vsetvli zero, zero, e32, m2, tu, mu
-vssseg5e32.v  v8, (a0), a1
-vsetvli zero, zero, e32, m4, tu, mu
-vssseg5e32.v  v8, (a0), a1
-vsetvli zero, zero, e64, mf8, tu, mu
-vssseg5e64.v  v8, (a0), a1
-vsetvli zero, zero, e64, mf4, tu, mu
-vssseg5e64.v  v8, (a0), a1
-vsetvli zero, zero, e64, mf2, tu, mu
-vssseg5e64.v  v8, (a0), a1
 vsetvli zero, zero, e64, m1, tu, mu
 vssseg5e64.v  v8, (a0), a1
-vsetvli zero, zero, e64, m2, tu, mu
-vssseg5e64.v  v8, (a0), a1
-vsetvli zero, zero, e64, m4, tu, mu
-vssseg5e64.v  v8, (a0), a1
 
 vsetvli zero, zero, e8, mf8, tu, mu
 vssseg6e8.v v8, (a0), a1
@@ -1330,46 +662,18 @@ vsetvli zero, zero, e8, mf2, tu, mu
 vssseg6e8.v v8, (a0), a1
 vsetvli zero, zero, e8, m1, tu, mu
 vssseg6e8.v v8, (a0), a1
-vsetvli zero, zero, e8, m2, tu, mu
-vssseg6e8.v v8, (a0), a1
-vsetvli zero, zero, e8, m4, tu, mu
-vssseg6e8.v v8, (a0), a1
-vsetvli zero, zero, e16, mf8, tu, mu
-vssseg6e16.v  v8, (a0), a1
 vsetvli zero, zero, e16, mf4, tu, mu
 vssseg6e16.v  v8, (a0), a1
 vsetvli zero, zero, e16, mf2, tu, mu
 vssseg6e16.v  v8, (a0), a1
 vsetvli zero, zero, e16, m1, tu, mu
 vssseg6e16.v  v8, (a0), a1
-vsetvli zero, zero, e16, m2, tu, mu
-vssseg6e16.v  v8, (a0), a1
-vsetvli zero, zero, e16, m4, tu, mu
-vssseg6e16.v  v8, (a0), a1
-vsetvli zero, zero, e32, mf8, tu, mu
-vssseg6e32.v  v8, (a0), a1
-vsetvli zero, zero, e32, mf4, tu, mu
-vssseg6e32.v  v8, (a0), a1
 vsetvli zero, zero, e32, mf2, tu, mu
 vssseg6e32.v  v8, (a0), a1
 vsetvli zero, zero, e32, m1, tu, mu
 vssseg6e32.v  v8, (a0), a1
-vsetvli zero, zero, e32, m2, tu, mu
-vssseg6e32.v  v8, (a0), a1
-vsetvli zero, zero, e32, m4, tu, mu
-vssseg6e32.v  v8, (a0), a1
-vsetvli zero, zero, e64, mf8, tu, mu
-vssseg6e64.v  v8, (a0), a1
-vsetvli zero, zero, e64, mf4, tu, mu
-vssseg6e64.v  v8, (a0), a1
-vsetvli zero, zero, e64, mf2, tu, mu
-vssseg6e64.v  v8, (a0), a1
 vsetvli zero, zero, e64, m1, tu, mu
 vssseg6e64.v  v8, (a0), a1
-vsetvli zero, zero, e64, m2, tu, mu
-vssseg6e64.v  v8, (a0), a1
-vsetvli zero, zero, e64, m4, tu, mu
-vssseg6e64.v  v8, (a0), a1
 
 vsetvli zero, zero, e8, mf8, tu, mu
 vssseg7e8.v v8, (a0), a1
@@ -1379,46 +683,18 @@ vsetvli zero, zero, e8, mf2, tu, mu
 vssseg7e8.v v8, (a0), a1
 vsetvli zero, zero, e8, m1, tu, mu
 vssseg7e8.v v8, (a0), a1
-vsetvli zero, zero, e8, m2, tu, mu
-vssseg7e8.v v8, (a0), a1
-vsetvli zero, zero, e8, m4, tu, mu
-vssseg7e8.v v8, (a0), a1
-vsetvli zero, zero, e16, mf8, tu, mu
-vssseg7e16.v  v8, (a0), a1
 vsetvli zero, zero, e16, mf4, tu, mu
 vssseg7e16.v  v8, (a0), a1
 vsetvli zero, zero, e16, mf2, tu, mu
 vssseg7e16.v  v8, (a0), a1
 vsetvli zero, zero, e16, m1, tu, mu
 vssseg7e16.v  v8, (a0), a1
-vsetvli zero, zero, e16, m2, tu, mu
-vssseg7e16.v  v8, (a0), a1
-vsetvli zero, zero, e16, m4, tu, mu
-vssseg7e16.v  v8, (a0), a1
-vsetvli zero, zero, e32, mf8, tu, mu
-vssseg7e32.v  v8, (a0), a1
-vsetvli zero, zero, e32, mf4, tu, mu
-vssseg7e32.v  v8, (a0), a1
 vsetvli zero, zero, e32, mf2, tu, mu
 vssseg7e32.v  v8, (a0), a1
 vsetvli zero, zero, e32, m1, tu, mu
 vssseg7e32.v  v8, (a0), a1
-vsetvli zero, zero, e32, m2, tu, mu
-vssseg7e32.v  v8, (a0), a1
-vsetvli zero, zero, e32, m4, tu, mu
-vssseg7e32.v  v8, (a0), a1
-vsetvli zero, zero, e64, mf8, tu, mu
-vssseg7e64.v  v8, (a0), a1
-vsetvli zero, zero, e64, mf4, tu, mu
-vssseg7e64.v  v8, (a0), a1
-vsetvli zero, zero, e64, mf2, tu, mu
-vssseg7e64.v  v8, (a0), a1
 vsetvli zero, zero, e64, m1, tu, mu
 vssseg7e64.v  v8, (a0), a1
-vsetvli zero, zero, e64, m2, tu, mu
-vssseg7e64.v  v8, (a0), a1
-vsetvli zero, zero, e64, m4, tu, mu
-vssseg7e64.v  v8, (a0), a1
 
 vsetvli zero, zero, e8, mf8, tu, mu
 vssseg8e8.v v8, (a0), a1
@@ -1428,46 +704,18 @@ vsetvli zero, zero, e8, mf2, tu, mu
 vssseg8e8.v v8, (a0), a1
 vsetvli zero, zero, e8, m1, tu, mu
 vssseg8e8.v v8, (a0), a1
-vsetvli zero, zero, e8, m2, tu, mu
-vssseg8e8.v v8, (a0), a1
-vsetvli zero, zero, e8, m4, tu, mu
-vssseg8e8.v v8, (a0), a1
-vsetvli zero, zero, e16, mf8, tu, mu
-vssseg8e16.v  v8, (a0), a1
 vsetvli zero, zero, e16, mf4, tu, mu
 vssseg8e16.v  v8, (a0), a1
 vsetvli zero, zero, e16, mf2, tu, mu
 vssseg8e16.v  v8, (a0), a1
 vsetvli zero, zero, e16, m1, tu, mu
 vssseg8e16.v  v8, (a0), a1
-vsetvli zero, zero, e16, m2, tu, mu
-vssseg8e16.v  v8, (a0), a1
-vsetvli zero, zero, e16, m4, tu, mu
-vssseg8e16.v  v8, (a0), a1
-vsetvli zero, zero, e32, mf8, tu, mu
-vssseg8e32.v  v8, (a0), a1
-vsetvli zero, zero, e32, mf4, tu, mu
-vssseg8e32.v  v8, (a0), a1
 vsetvli zero, zero, e32, mf2, tu, mu
 vssseg8e32.v  v8, (a0), a1
 vsetvli zero, zero, e32, m1, tu, mu
 vssseg8e32.v  v8, (a0), a1
-vsetvli zero, zero, e32, m2, tu, mu
-vssseg8e32.v  v8, (a0), a1
-vsetvli zero, zero, e32, m4, tu, mu
-vssseg8e32.v  v8, (a0), a1
-vsetvli zero, zero, e64, mf8, tu, mu
-vssseg8e64.v  v8, (a0), a1
-vsetvli zero, zero, e64, mf4, tu, mu
-vssseg8e64.v  v8, (a0), a1
-vsetvli zero, zero, e64, mf2, tu, mu
-vssseg8e64.v  v8, (a0), a1
 vsetvli zero, zero, e64, m1, tu, mu
 vssseg8e64.v  v8, (a0), a1
-vsetvli zero, zero, e64, m2, tu, mu
-vssseg8e64.v  v8, (a0), a1
-vsetvli zero, zero, e64, m4, tu, mu
-vssseg8e64.v  v8, (a0), a1
 
 vsetvli zero, zero, e8, mf8, tu, mu
 vlseg2e8ff.v  v8,(a0)
@@ -1481,10 +729,6 @@ vsetvli zero, zero, e8, m2, tu, mu
 vlseg2e8ff.v  v8,(a0)
 vsetvli zero, zero, e8, m4, tu, mu
 vlseg2e8ff.v  v8,(a0)
-vsetvli zero, zero, e8, m8, tu, mu
-vlseg2e8ff.v  v8,(a0)
-vsetvli zero, zero, e16, mf8, tu, mu
-vlseg2e16ff.v v8,(a0)
 vsetvli zero, zero, e16, mf4, tu, mu
 vlseg2e16ff.v v8,(a0)
 vsetvli zero, zero, e16, mf2, tu, mu
@@ -1495,12 +739,6 @@ vsetvli zero, zero, e16, m2, tu, mu
 vlseg2e16ff.v v8,(a0)
 vsetvli zero, zero, e16, m4, tu, mu
 vlseg2e16ff.v v8,(a0)
-vsetvli zero, zero, e16, m8, tu, mu
-vlseg2e16ff.v v8,(a0)
-vsetvli zero, zero, e32, mf8, tu, mu
-vlseg2e32ff.v v8,(a0)
-vsetvli zero, zero, e32, mf4, tu, mu
-vlseg2e32ff.v v8,(a0)
 vsetvli zero, zero, e32, mf2, tu, mu
 vlseg2e32ff.v v8,(a0)
 vsetvli zero, zero, e32, m1, tu, mu
@@ -1509,22 +747,12 @@ vsetvli zero, zero, e32, m2, tu, mu
 vlseg2e32ff.v v8,(a0)
 vsetvli zero, zero, e32, m4, tu, mu
 vlseg2e32ff.v v8,(a0)
-vsetvli zero, zero, e32, m8, tu, mu
-vlseg2e32ff.v v8,(a0)
-vsetvli zero, zero, e64, mf8, tu, mu
-vlseg2e64ff.v v8,(a0)
-vsetvli zero, zero, e64, mf4, tu, mu
-vlseg2e64ff.v v8,(a0)
-vsetvli zero, zero, e64, mf2, tu, mu
-vlseg2e64ff.v v8,(a0)
 vsetvli zero, zero, e64, m1, tu, mu
 vlseg2e64ff.v v8,(a0)
 vsetvli zero, zero, e64, m2, tu, mu
 vlseg2e64ff.v v8,(a0)
 vsetvli zero, zero, e64, m4, tu, mu
 vlseg2e64ff.v v8,(a0)
-vsetvli zero, zero, e64, m8, tu, mu
-vlseg2e64ff.v v8,(a0)
 
 vsetvli zero, zero, e8, mf8, tu, mu
 vlseg3e8ff.v  v8,(a0)
@@ -1536,12 +764,6 @@ vsetvli zero, zero, e8, m1, tu, mu
 vlseg3e8ff.v  v8,(a0)
 vsetvli zero, zero, e8, m2, tu, mu
 vlseg3e8ff.v  v8,(a0)
-vsetvli zero, zero, e8, m4, tu, mu
-vlseg3e8ff.v  v8,(a0)
-vsetvli zero, zero, e8, m8, tu, mu
-vlseg3e8ff.v  v8,(a0)
-vsetvli zero, zero, e16, mf8, tu, mu
-vlseg3e16ff.v v8,(a0)
 vsetvli zero, zero, e16, mf4, tu, mu
 vlseg3e16ff.v v8,(a0)
 vsetvli zero, zero, e16, mf2, tu, mu
@@ -1550,38 +772,16 @@ vsetvli zero, zero, e16, m1, tu, mu
 vlseg3e16ff.v v8,(a0)
 vsetvli zero, zero, e16, m2, tu, mu
 vlseg3e16ff.v v8,(a0)
-vsetvli zero, zero, e16, m4, tu, mu
-vlseg3e16ff.v v8,(a0)
-vsetvli zero, zero, e16, m8, tu, mu
-vlseg3e16ff.v v8,(a0)
-vsetvli zero, zero, e32, mf8, tu, mu
-vlseg3e32ff.v v8,(a0)
-vsetvli zero, zero, e32, mf4, tu, mu
-vlseg3e32ff.v v8,(a0)
 vsetvli zero, zero, e32, mf2, tu, mu
 vlseg3e32ff.v v8,(a0)
 vsetvli zero, zero, e32, m1, tu, mu
 vlseg3e32ff.v v8,(a0)
 vsetvli zero, zero, e32, m2, tu, mu
 vlseg3e32ff.v v8,(a0)
-vsetvli zero, zero, e32, m4, tu, mu
-vlseg3e32ff.v v8,(a0)
-vsetvli zero, zero, e32, m8, tu, mu
-vlseg3e32ff.v v8,(a0)
-vsetvli zero, zero, e64, mf8, tu, mu
-vlseg3e64ff.v v8,(a0)
-vsetvli zero, zero, e64, mf4, tu, mu
-vlseg3e64ff.v v8,(a0)
-vsetvli zero, zero, e64, mf2, tu, mu
-vlseg3e64ff.v v8,(a0)
 vsetvli zero, zero, e64, m1, tu, mu
 vlseg3e64ff.v v8,(a0)
 vsetvli zero, zero, e64, m2, tu, mu
 vlseg3e64ff.v v8,(a0)
-vsetvli zero, zero, e64, m4, tu, mu
-vlseg3e64ff.v v8,(a0)
-vsetvli zero, zero, e64, m8, tu, mu
-vlseg3e64ff.v v8,(a0)
 
 vsetvli zero, zero, e8, mf8, tu, mu
 vlseg4e8ff.v  v8,(a0)
@@ -1593,12 +793,6 @@ vsetvli zero, zero, e8, m1, tu, mu
 vlseg4e8ff.v  v8,(a0)
 vsetvli zero, zero, e8, m2, tu, mu
 vlseg4e8ff.v  v8,(a0)
-vsetvli zero, zero, e8, m4, tu, mu
-vlseg4e8ff.v  v8,(a0)
-vsetvli zero, zero, e8, m8, tu, mu
-vlseg4e8ff.v  v8,(a0)
-vsetvli zero, zero, e16, mf8, tu, mu
-vlseg4e16ff.v v8,(a0)
 vsetvli zero, zero, e16, mf4, tu, mu
 vlseg4e16ff.v v8,(a0)
 vsetvli zero, zero, e16, mf2, tu, mu
@@ -1607,38 +801,16 @@ vsetvli zero, zero, e16, m1, tu, mu
 vlseg4e16ff.v v8,(a0)
 vsetvli zero, zero, e16, m2, tu, mu
 vlseg4e16ff.v v8,(a0)
-vsetvli zero, zero, e16, m4, tu, mu
-vlseg4e16ff.v v8,(a0)
-vsetvli zero, zero, e16, m8, tu, mu
-vlseg4e16ff.v v8,(a0)
-vsetvli zero, zero, e32, mf8, tu, mu
-vlseg4e32ff.v v8,(a0)
-vsetvli zero, zero, e32, mf4, tu, mu
-vlseg4e32ff.v v8,(a0)
 vsetvli zero, zero, e32, mf2, tu, mu
 vlseg4e32ff.v v8,(a0)
 vsetvli zero, zero, e32, m1, tu, mu
 vlseg4e32ff.v v8,(a0)
 vsetvli zero, zero, e32, m2, tu, mu
 vlseg4e32ff.v v8,(a0)
-vsetvli zero, zero, e32, m4, tu, mu
-vlseg4e32ff.v v8,(a0)
-vsetvli zero, zero, e32, m8, tu, mu
-vlseg4e32ff.v v8,(a0)
-vsetvli zero, zero, e64, mf8, tu, mu
-vlseg4e64ff.v v8,(a0)
-vsetvli zero, zero, e64, mf4, tu, mu
-vlseg4e64ff.v v8,(a0)
-vsetvli zero, zero, e64, mf2, tu, mu
-vlseg4e64ff.v v8,(a0)
 vsetvli zero, zero, e64, m1, tu, mu
 vlseg4e64ff.v v8,(a0)
 vsetvli zero, zero, e64, m2, tu, mu
 vlseg4e64ff.v v8,(a0)
-vsetvli zero, zero, e64, m4, tu, mu
-vlseg4e64ff.v v8,(a0)
-vsetvli zero, zero, e64, m8, tu, mu
-vlseg4e64ff.v v8,(a0)
 
 vsetvli zero, zero, e8, mf8, tu, mu
 vlseg5e8ff.v  v8,(a0)
@@ -1648,46 +820,18 @@ vsetvli zero, zero, e8, mf2, tu, mu
 vlseg5e8ff.v  v8,(a0)
 vsetvli zero, zero, e8, m1, tu, mu
 vlseg5e8ff.v  v8,(a0)
-vsetvli zero, zero, e8, m2, tu, mu
-vlseg5e8ff.v  v8,(a0)
-vsetvli zero, zero, e8, m4, tu, mu
-vlseg5e8ff.v  v8,(a0)
-vsetvli zero, zero, e16, mf8, tu, mu
-vlseg5e16ff.v v8,(a0)
 vsetvli zero, zero, e16, mf4, tu, mu
 vlseg5e16ff.v v8,(a0)
 vsetvli zero, zero, e16, mf2, tu, mu
 vlseg5e16ff.v v8,(a0)
 vsetvli zero, zero, e16, m1, tu, mu
 vlseg5e16ff.v v8,(a0)
-vsetvli zero, zero, e16, m2, tu, mu
-vlseg5e16ff.v v8,(a0)
-vsetvli zero, zero, e16, m4, tu, mu
-vlseg5e16ff.v v8,(a0)
-vsetvli zero, zero, e32, mf8, tu, mu
-vlseg5e32ff.v v8,(a0)
-vsetvli zero, zero, e32, mf4, tu, mu
-vlseg5e32ff.v v8,(a0)
 vsetvli zero, zero, e32, mf2, tu, mu
 vlseg5e32ff.v v8,(a0)
 vsetvli zero, zero, e32, m1, tu, mu
 vlseg5e32ff.v v8,(a0)
-vsetvli zero, zero, e32, m2, tu, mu
-vlseg5e32ff.v v8,(a0)
-vsetvli zero, zero, e32, m4, tu, mu
-vlseg5e32ff.v v8,(a0)
-vsetvli zero, zero, e64, mf8, tu, mu
-vlseg5e64ff.v v8,(a0)
-vsetvli zero, zero, e64, mf4, tu, mu
-vlseg5e64ff.v v8,(a0)
-vsetvli zero, zero, e64, mf2, tu, mu
-vlseg5e64ff.v v8,(a0)
 vsetvli zero, zero, e64, m1, tu, mu
 vlseg5e64ff.v v8,(a0)
-vsetvli zero, zero, e64, m2, tu, mu
-vlseg5e64ff.v v8,(a0)
-vsetvli zero, zero, e64, m4, tu, mu
-vlseg5e64ff.v v8,(a0)
 
 vsetvli zero, zero, e8, mf8, tu, mu
 vlseg6e8ff.v  v8,(a0)
@@ -1697,46 +841,18 @@ vsetvli zero, zero, e8, mf2, tu, mu
 vlseg6e8ff.v  v8,(a0)
 vsetvli zero, zero, e8, m1, tu, mu
 vlseg6e8ff.v  v8,(a0)
-vsetvli zero, zero, e8, m2, tu, mu
-vlseg6e8ff.v  v8,(a0)
-vsetvli zero, zero, e8, m4, tu, mu
-vlseg6e8ff.v  v8,(a0)
-vsetvli zero, zero, e16, mf8, tu, mu
-vlseg6e16ff.v v8,(a0)
 vsetvli zero, zero, e16, mf4, tu, mu
 vlseg6e16ff.v v8,(a0)
 vsetvli zero, zero, e16, mf2, tu, mu
 vlseg6e16ff.v v8,(a0)
 vsetvli zero, zero, e16, m1, tu, mu
 vlseg6e16ff.v v8,(a0)
-vsetvli zero, zero, e16, m2, tu, mu
-vlseg6e16ff.v v8,(a0)
-vsetvli zero, zero, e16, m4, tu, mu
-vlseg6e16ff.v v8,(a0)
-vsetvli zero, zero, e32, mf8, tu, mu
-vlseg6e32ff.v v8,(a0)
-vsetvli zero, zero, e32, mf4, tu, mu
-vlseg6e32ff.v v8,(a0)
 vsetvli zero, zero, e32, mf2, tu, mu
 vlseg6e32ff.v v8,(a0)
 vsetvli zero, zero, e32, m1, tu, mu
 vlseg6e32ff.v v8,(a0)
-vsetvli zero, zero, e32, m2, tu, mu
-vlseg6e32ff.v v8,(a0)
-vsetvli zero, zero, e32, m4, tu, mu
-vlseg6e32ff.v v8,(a0)
-vsetvli zero, zero, e64, mf8, tu, mu
-vlseg6e64ff.v v8,(a0)
-vsetvli zero, zero, e64, mf4, tu, mu
-vlseg6e64ff.v v8,(a0)
-vsetvli zero, zero, e64, mf2, tu, mu
-vlseg6e64ff.v v8,(a0)
 vsetvli zero, zero, e64, m1, tu, mu
 vlseg6e64ff.v v8,(a0)
-vsetvli zero, zero, e64, m2, tu, mu
-vlseg6e64ff.v v8,(a0)
-vsetvli zero, zero, e64, m4, tu, mu
-vlseg6e64ff.v v8,(a0)
 
 vsetvli zero, zero, e8, mf8, tu, mu
 vlseg7e8ff.v  v8,(a0)
@@ -1746,46 +862,18 @@ vsetvli zero, zero, e8, mf2, tu, mu
 vlseg7e8ff.v  v8,(a0)
 vsetvli zero, zero, e8, m1, tu, mu
 vlseg7e8ff.v  v8,(a0)
-vsetvli zero, zero, e8, m2, tu, mu
-vlseg7e8ff.v  v8,(a0)
-vsetvli zero, zero, e8, m4, tu, mu
-vlseg7e8ff.v  v8,(a0)
-vsetvli zero, zero, e16, mf8, tu, mu
-vlseg7e16ff.v v8,(a0)
 vsetvli zero, zero, e16, mf4, tu, mu
 vlseg7e16ff.v v8,(a0)
 vsetvli zero, zero, e16, mf2, tu, mu
 vlseg7e16ff.v v8,(a0)
 vsetvli zero, zero, e16, m1, tu, mu
 vlseg7e16ff.v v8,(a0)
-vsetvli zero, zero, e16, m2, tu, mu
-vlseg7e16ff.v v8,(a0)
-vsetvli zero, zero, e16, m4, tu, mu
-vlseg7e16ff.v v8,(a0)
-vsetvli zero, zero, e32, mf8, tu, mu
-vlseg7e32ff.v v8,(a0)
-vsetvli zero, zero, e32, mf4, tu, mu
-vlseg7e32ff.v v8,(a0)
 vsetvli zero, zero, e32, mf2, tu, mu
 vlseg7e32ff.v v8,(a0)
 vsetvli zero, zero, e32, m1, tu, mu
 vlseg7e32ff.v v8,(a0)
-vsetvli zero, zero, e32, m2, tu, mu
-vlseg7e32ff.v v8,(a0)
-vsetvli zero, zero, e32, m4, tu, mu
-vlseg7e32ff.v v8,(a0)
-vsetvli zero, zero, e64, mf8, tu, mu
-vlseg7e64ff.v v8,(a0)
-vsetvli zero, zero, e64, mf4, tu, mu
-vlseg7e64ff.v v8,(a0)
-vsetvli zero, zero, e64, mf2, tu, mu
-vlseg7e64ff.v v8,(a0)
 vsetvli zero, zero, e64, m1, tu, mu
 vlseg7e64ff.v v8,(a0)
-vsetvli zero, zero, e64, m2, tu, mu
-vlseg7e64ff.v v8,(a0)
-vsetvli zero, zero, e64, m4, tu, mu
-vlseg7e64ff.v v8,(a0)
 
 vsetvli zero, zero, e8, mf8, tu, mu
 vlseg8e8ff.v  v8,(a0)
@@ -1795,46 +883,18 @@ vsetvli zero, zero, e8, mf2, tu, mu
 vlseg8e8ff.v  v8,(a0)
 vsetvli zero, zero, e8, m1, tu, mu
 vlseg8e8ff.v  v8,(a0)
-vsetvli zero, zero, e8, m2, tu, mu
-vlseg8e8ff.v  v8,(a0)
-vsetvli zero, zero, e8, m4, tu, mu
-vlseg8e8ff.v  v8,(a0)
-vsetvli zero, zero, e16, mf8, tu, mu
-vlseg8e16ff.v v8,(a0)
 vsetvli zero, zero, e16, mf4, tu, mu
 vlseg8e16ff.v v8,(a0)
 vsetvli zero, zero, e16, mf2, tu, mu
 vlseg8e16ff.v v8,(a0)
 vsetvli zero, zero, e16, m1, tu, mu
 vlseg8e16ff.v v8,(a0)
-vsetvli zero, zero, e16, m2, tu, mu
-vlseg8e16ff.v v8,(a0)
-vsetvli zero, zero, e16, m4, tu, mu
-vlseg8e16ff.v v8,(a0)
-vsetvli zero, zero, e32, mf8, tu, mu
-vlseg8e32ff.v v8,(a0)
-vsetvli zero, zero, e32, mf4, tu, mu
-vlseg8e32ff.v v8,(a0)
 vsetvli zero, zero, e32, mf2, tu, mu
 vlseg8e32ff.v v8,(a0)
 vsetvli zero, zero, e32, m1, tu, mu
 vlseg8e32ff.v v8,(a0)
-vsetvli zero, zero, e32, m2, tu, mu
-vlseg8e32ff.v v8,(a0)
-vsetvli zero, zero, e32, m4, tu, mu
-vlseg8e32ff.v v8,(a0)
-vsetvli zero, zero, e64, mf8, tu, mu
-vlseg8e64ff.v v8,(a0)
-vsetvli zero, zero, e64, mf4, tu, mu
-vlseg8e64ff.v v8,(a0)
-vsetvli zero, zero, e64, mf2, tu, mu
-vlseg8e64ff.v v8,(a0)
 vsetvli zero, zero, e64, m1, tu, mu
 vlseg8e64ff.v v8,(a0)
-vsetvli zero, zero, e64, m2, tu, mu
-vlseg8e64ff.v v8,(a0)
-vsetvli zero, zero, e64, m4, tu, mu
-vlseg8e64ff.v v8,(a0)
 
 vsetvli zero, zero, e8, mf8, tu, mu
 vluxseg2ei8.v v8, (a0), v16
@@ -1848,10 +908,6 @@ vsetvli zero, zero, e8, m2, tu, mu
 vluxseg2ei8.v v8, (a0), v16
 vsetvli zero, zero, e8, m4, tu, mu
 vluxseg2ei8.v v8, (a0), v16
-vsetvli zero, zero, e8, m8, tu, mu
-vluxseg2ei8.v v8, (a0), v16
-vsetvli zero, zero, e16, mf8, tu, mu
-vluxseg2ei16.v  v8, (a0), v16
 vsetvli zero, zero, e16, mf4, tu, mu
 vluxseg2ei16.v  v8, (a0), v16
 vsetvli zero, zero, e16, mf2, tu, mu
@@ -1862,12 +918,6 @@ vsetvli zero, zero, e16, m2, tu, mu
 vluxseg2ei16.v  v8, (a0), v16
 vsetvli zero, zero, e16, m4, tu, mu
 vluxseg2ei16.v  v8, (a0), v16
-vsetvli zero, zero, e16, m8, tu, mu
-vluxseg2ei16.v  v8, (a0), v16
-vsetvli zero, zero, e32, mf8, tu, mu
-vluxseg2ei32.v  v8, (a0), v16
-vsetvli zero, zero, e32, mf4, tu, mu
-vluxseg2ei32.v  v8, (a0), v16
 vsetvli zero, zero, e32, mf2, tu, mu
 vluxseg2ei32.v  v8, (a0), v16
 vsetvli zero, zero, e32, m1, tu, mu
@@ -1876,22 +926,12 @@ vsetvli zero, zero, e32, m2, tu, mu
 vluxseg2ei32.v  v8, (a0), v16
 vsetvli zero, zero, e32, m4, tu, mu
 vluxseg2ei32.v  v8, (a0), v16
-vsetvli zero, zero, e32, m8, tu, mu
-vluxseg2ei32.v  v8, (a0), v16
-vsetvli zero, zero, e64, mf8, tu, mu
-vluxseg2ei64.v  v8, (a0), v16
-vsetvli zero, zero, e64, mf4, tu, mu
-vluxseg2ei64.v  v8, (a0), v16
-vsetvli zero, zero, e64, mf2, tu, mu
-vluxseg2ei64.v  v8, (a0), v16
 vsetvli zero, zero, e64, m1, tu, mu
 vluxseg2ei64.v  v8, (a0), v16
 vsetvli zero, zero, e64, m2, tu, mu
 vluxseg2ei64.v  v8, (a0), v16
 vsetvli zero, zero, e64, m4, tu, mu
 vluxseg2ei64.v  v8, (a0), v16
-vsetvli zero, zero, e64, m8, tu, mu
-vluxseg2ei64.v  v8, (a0), v16
 
 vsetvli zero, zero, e8, mf8, tu, mu
 vluxseg3ei8.v v8, (a0), v16
@@ -1903,12 +943,6 @@ vsetvli zero, zero, e8, m1, tu, mu
 vluxseg3ei8.v v8, (a0), v16
 vsetvli zero, zero, e8, m2, tu, mu
 vluxseg3ei8.v v8, (a0), v16
-vsetvli zero, zero, e8, m4, tu, mu
-vluxseg3ei8.v v8, (a0), v16
-vsetvli zero, zero, e8, m8, tu, mu
-vluxseg3ei8.v v8, (a0), v16
-vsetvli zero, zero, e16, mf8, tu, mu
-vluxseg3ei16.v  v8, (a0), v16
 vsetvli zero, zero, e16, mf4, tu, mu
 vluxseg3ei16.v  v8, (a0), v16
 vsetvli zero, zero, e16, mf2, tu, mu
@@ -1917,38 +951,16 @@ vsetvli zero, zero, e16, m1, tu, mu
 vluxseg3ei16.v  v8, (a0), v16
 vsetvli zero, zero, e16, m2, tu, mu
 vluxseg3ei16.v  v8, (a0), v16
-vsetvli zero, zero, e16, m4, tu, mu
-vluxseg3ei16.v  v8, (a0), v16
-vsetvli zero, zero, e16, m8, tu, mu
-vluxseg3ei16.v  v8, (a0), v16
-vsetvli zero, zero, e32, mf8, tu, mu
-vluxseg3ei32.v  v8, (a0), v16
-vsetvli zero, zero, e32, mf4, tu, mu
-vluxseg3ei32.v  v8, (a0), v16
 vsetvli zero, zero, e32, mf2, tu, mu
 vluxseg3ei32.v  v8, (a0), v16
 vsetvli zero, zero, e32, m1, tu, mu
 vluxseg3ei32.v  v8, (a0), v16
 vsetvli zero, zero, e32, m2, tu, mu
 vluxseg3ei32.v  v8, (a0), v16
-vsetvli zero, zero, e32, m4, tu, mu
-vluxseg3ei32.v  v8, (a0), v16
-vsetvli zero, zero, e32, m8, tu, mu
-vluxseg3ei32.v  v8, (a0), v16
-vsetvli zero, zero, e64, mf8, tu, mu
-vluxseg3ei64.v  v8, (a0), v16
-vsetvli zero, zero, e64, mf4, tu, mu
-vluxseg3ei64.v  v8, (a0), v16
-vsetvli zero, zero, e64, mf2, tu, mu
-vluxseg3ei64.v  v8, (a0), v16
 vsetvli zero, zero, e64, m1, tu, mu
 vluxseg3ei64.v  v8, (a0), v16
 vsetvli zero, zero, e64, m2, tu, mu
 vluxseg3ei64.v  v8, (a0), v16
-vsetvli zero, zero, e64, m4, tu, mu
-vluxseg3ei64.v  v8, (a0), v16
-vsetvli zero, zero, e64, m8, tu, mu
-vluxseg3ei64.v  v8, (a0), v16
 
 vsetvli zero, zero, e8, mf8, tu, mu
 vluxseg4ei8.v v8, (a0), v16
@@ -1960,12 +972,6 @@ vsetvli zero, zero, e8, m1, tu, mu
 vluxseg4ei8.v v8, (a0), v16
 vsetvli zero, zero, e8, m2, tu, mu
 vluxseg4ei8.v v8, (a0), v16
-vsetvli zero, zero, e8, m4, tu, mu
-vluxseg4ei8.v v8, (a0), v16
-vsetvli zero, zero, e8, m8, tu, mu
-vluxseg4ei8.v v8, (a0), v16
-vsetvli zero, zero, e16, mf8, tu, mu
-vluxseg4ei16.v  v8, (a0), v16
 vsetvli zero, zero, e16, mf4, tu, mu
 vluxseg4ei16.v  v8, (a0), v16
 vsetvli zero, zero, e16, mf2, tu, mu
@@ -1974,38 +980,16 @@ vsetvli zero, zero, e16, m1, tu, mu
 vluxseg4ei16.v  v8, (a0), v16
 vsetvli zero, zero, e16, m2, tu, mu
 vluxseg4ei16.v  v8, (a0), v16
-vsetvli zero, zero, e16, m4, tu, mu
-vluxseg4ei16.v  v8, (a0), v16
-vsetvli zero, zero, e16, m8, tu, mu
-vluxseg4ei16.v  v8, (a0), v16
-vsetvli zero, zero, e32, mf8, tu, mu
-vluxseg4ei32.v  v8, (a0), v16
-vsetvli zero, zero, e32, mf4, tu, mu
-vluxseg4ei32.v  v8, (a0), v16
 vsetvli zero, zero, e32, mf2, tu, mu
 vluxseg4ei32.v  v8, (a0), v16
 vsetvli zero, zero, e32, m1, tu, mu
 vluxseg4ei32.v  v8, (a0), v16
 vsetvli zero, zero, e32, m2, tu, mu
 vluxseg4ei32.v  v8, (a0), v16
-vsetvli zero, zero, e32, m4, tu, mu
-vluxseg4ei32.v  v8, (a0), v16
-vsetvli zero, zero, e32, m8, tu, mu
-vluxseg4ei32.v  v8, (a0), v16
-vsetvli zero, zero, e64, mf8, tu, mu
-vluxseg4ei64.v  v8, (a0), v16
-vsetvli zero, zero, e64, mf4, tu, mu
-vluxseg4ei64.v  v8, (a0), v16
-vsetvli zero, zero, e64, mf2, tu, mu
-vluxseg4ei64.v  v8, (a0), v16
 vsetvli zero, zero, e64, m1, tu, mu
 vluxseg4ei64.v  v8, (a0), v16
 vsetvli zero, zero, e64, m2, tu, mu
 vluxseg4ei64.v  v8, (a0), v16
-vsetvli zero, zero, e64, m4, tu, mu
-vluxseg4ei64.v  v8, (a0), v16
-vsetvli zero, zero, e64, m8, tu, mu
-vluxseg4ei64.v  v8, (a0), v16
 
 vsetvli zero, zero, e8, mf8, tu, mu
 vluxseg5ei8.v v8, (a0), v16
@@ -2015,46 +999,18 @@ vsetvli zero, zero, e8, mf2, tu, mu
 vluxseg5ei8.v v8, (a0), v16
 vsetvli zero, zero, e8, m1, tu, mu
 vluxseg5ei8.v v8, (a0), v16
-vsetvli zero, zero, e8, m2, tu, mu
-vluxseg5ei8.v v8, (a0), v16
-vsetvli zero, zero, e8, m4, tu, mu
-vluxseg5ei8.v v8, (a0), v16
-vsetvli zero, zero, e16, mf8, tu, mu
-vluxseg5ei16.v  v8, (a0), v16
 vsetvli zero, zero, e16, mf4, tu, mu
 vluxseg5ei16.v  v8, (a0), v16
 vsetvli zero, zero, e16, mf2, tu, mu
 vluxseg5ei16.v  v8, (a0), v16
 vsetvli zero, zero, e16, m1, tu, mu
 vluxseg5ei16.v  v8, (a0), v16
-vsetvli zero, zero, e16, m2, tu, mu
-vluxseg5ei16.v  v8, (a0), v16
-vsetvli zero, zero, e16, m4, tu, mu
-vluxseg5ei16.v  v8, (a0), v16
-vsetvli zero, zero, e32, mf8, tu, mu
-vluxseg5ei32.v  v8, (a0), v16
-vsetvli zero, zero, e32, mf4, tu, mu
-vluxseg5ei32.v  v8, (a0), v16
 vsetvli zero, zero, e32, mf2, tu, mu
 vluxseg5ei32.v  v8, (a0), v16
 vsetvli zero, zero, e32, m1, tu, mu
 vluxseg5ei32.v  v8, (a0), v16
-vsetvli zero, zero, e32, m2, tu, mu
-vluxseg5ei32.v  v8, (a0), v16
-vsetvli zero, zero, e32, m4, tu, mu
-vluxseg5ei32.v  v8, (a0), v16
-vsetvli zero, zero, e64, mf8, tu, mu
-vluxseg5ei64.v  v8, (a0), v16
-vsetvli zero, zero, e64, mf4, tu, mu
-vluxseg5ei64.v  v8, (a0), v16
-vsetvli zero, zero, e64, mf2, tu, mu
-vluxseg5ei64.v  v8, (a0), v16
 vsetvli zero, zero, e64, m1, tu, mu
 vluxseg5ei64.v  v8, (a0), v16
-vsetvli zero, zero, e64, m2, tu, mu
-vluxseg5ei64.v  v8, (a0), v16
-vsetvli zero, zero, e64, m4, tu, mu
-vluxseg5ei64.v  v8, (a0), v16
 
 vsetvli zero, zero, e8, mf8, tu, mu
 vluxseg6ei8.v v8, (a0), v16
@@ -2064,46 +1020,18 @@ vsetvli zero, zero, e8, mf2, tu, mu
 vluxseg6ei8.v v8, (a0), v16
 vsetvli zero, zero, e8, m1, tu, mu
 vluxseg6ei8.v v8, (a0), v16
-vsetvli zero, zero, e8, m2, tu, mu
-vluxseg6ei8.v v8, (a0), v16
-vsetvli zero, zero, e8, m4, tu, mu
-vluxseg6ei8.v v8, (a0), v16
-vsetvli zero, zero, e16, mf8, tu, mu
-vluxseg6ei16.v  v8, (a0), v16
 vsetvli zero, zero, e16, mf4, tu, mu
 vluxseg6ei16.v  v8, (a0), v16
 vsetvli zero, zero, e16, mf2, tu, mu
 vluxseg6ei16.v  v8, (a0), v16
 vsetvli zero, zero, e16, m1, tu, mu
 vluxseg6ei16.v  v8, (a0), v16
-vsetvli zero, zero, e16, m2, tu, mu
-vluxseg6ei16.v  v8, (a0), v16
-vsetvli zero, zero, e16, m4, tu, mu
-vluxseg6ei16.v  v8, (a0), v16
-vsetvli zero, zero, e32, mf8, tu, mu
-vluxseg6ei32.v  v8, (a0), v16
-vsetvli zero, zero, e32, mf4, tu, mu
-vluxseg6ei32.v  v8, (a0), v16
 vsetvli zero, zero, e32, mf2, tu, mu
 vluxseg6ei32.v  v8, (a0), v16
 vsetvli zero, zero, e32, m1, tu, mu
 vluxseg6ei32.v  v8, (a0), v16
-vsetvli zero, zero, e32, m2, tu, mu
-vluxseg6ei32.v  v8, (a0), v16
-vsetvli zero, zero, e32, m4, tu, mu
-vluxseg6ei32.v  v8, (a0), v16
-vsetvli zero, zero, e64, mf8, tu, mu
-vluxseg6ei64.v  v8, (a0), v16
-vsetvli zero, zero, e64, mf4, tu, mu
-vluxseg6ei64.v  v8, (a0), v16
-vsetvli zero, zero, e64, mf2, tu, mu
-vluxseg6ei64.v  v8, (a0), v16
 vsetvli zero, zero, e64, m1, tu, mu
 vluxseg6ei64.v  v8, (a0), v16
-vsetvli zero, zero, e64, m2, tu, mu
-vluxseg6ei64.v  v8, (a0), v16
-vsetvli zero, zero, e64, m4, tu, mu
-vluxseg6ei64.v  v8, (a0), v16
 
 vsetvli zero, zero, e8, mf8, tu, mu
 vluxseg7ei8.v v8, (a0), v16
@@ -2113,46 +1041,18 @@ vsetvli zero, zero, e8, mf2, tu, mu
 vluxseg7ei8.v v8, (a0), v16
 vsetvli zero, zero, e8, m1, tu, mu
 vluxseg7ei8.v v8, (a0), v16
-vsetvli zero, zero, e8, m2, tu, mu
-vluxseg7ei8.v v8, (a0), v16
-vsetvli zero, zero, e8, m4, tu, mu
-vluxseg7ei8.v v8, (a0), v16
-vsetvli zero, zero, e16, mf8, tu, mu
-vluxseg7ei16.v  v8, (a0), v16
 vsetvli zero, zero, e16, mf4, tu, mu
 vluxseg7ei16.v  v8, (a0), v16
 vsetvli zero, zero, e16, mf2, tu, mu
 vluxseg7ei16.v  v8, (a0), v16
 vsetvli zero, zero, e16, m1, tu, mu
 vluxseg7ei16.v  v8, (a0), v16
-vsetvli zero, zero, e16, m2, tu, mu
-vluxseg7ei16.v  v8, (a0), v16
-vsetvli zero, zero, e16, m4, tu, mu
-vluxseg7ei16.v  v8, (a0), v16
-vsetvli zero, zero, e32, mf8, tu, mu
-vluxseg7ei32.v  v8, (a0), v16
-vsetvli zero, zero, e32, mf4, tu, mu
-vluxseg7ei32.v  v8, (a0), v16
 vsetvli zero, zero, e32, mf2, tu, mu
 vluxseg7ei32.v  v8, (a0), v16
 vsetvli zero, zero, e32, m1, tu, mu
 vluxseg7ei32.v  v8, (a0), v16
-vsetvli zero, zero, e32, m2, tu, mu
-vluxseg7ei32.v  v8, (a0), v16
-vsetvli zero, zero, e32, m4, tu, mu
-vluxseg7ei32.v  v8, (a0), v16
-vsetvli zero, zero, e64, mf8, tu, mu
-vluxseg7ei64.v  v8, (a0), v16
-vsetvli zero, zero, e64, mf4, tu, mu
-vluxseg7ei64.v  v8, (a0), v16
-vsetvli zero, zero, e64, mf2, tu, mu
-vluxseg7ei64.v  v8, (a0), v16
 vsetvli zero, zero, e64, m1, tu, mu
 vluxseg7ei64.v  v8, (a0), v16
-vsetvli zero, zero, e64, m2, tu, mu
-vluxseg7ei64.v  v8, (a0), v16
-vsetvli zero, zero, e64, m4, tu, mu
-vluxseg7ei64.v  v8, (a0), v16
 
 vsetvli zero, zero, e8, mf8, tu, mu
 vluxseg8ei8.v v8, (a0), v16
@@ -2162,46 +1062,18 @@ vsetvli zero, zero, e8, mf2, tu, mu
 vluxseg8ei8.v v8, (a0), v16
 vsetvli zero, zero, e8, m1, tu, mu
 vluxseg8ei8.v v8, (a0), v16
-vsetvli zero, zero, e8, m2, tu, mu
-vluxseg8ei8.v v8, (a0), v16
-vsetvli zero, zero, e8, m4, tu, mu
-vluxseg8ei8.v v8, (a0), v16
-vsetvli zero, zero, e16, mf8, tu, mu
-vluxseg8ei16.v  v8, (a0), v16
 vsetvli zero, zero, e16, mf4, tu, mu
 vluxseg8ei16.v  v8, (a0), v16
 vsetvli zero, zero, e16, mf2, tu, mu
 vluxseg8ei16.v  v8, (a0), v16
 vsetvli zero, zero, e16, m1, tu, mu
 vluxseg8ei16.v  v8, (a0), v16
-vsetvli zero, zero, e16, m2, tu, mu
-vluxseg8ei16.v  v8, (a0), v16
-vsetvli zero, zero, e16, m4, tu, mu
-vluxseg8ei16.v  v8, (a0), v16
-vsetvli zero, zero, e32, mf8, tu, mu
-vluxseg8ei32.v  v8, (a0), v16
-vsetvli zero, zero, e32, mf4, tu, mu
-vluxseg8ei32.v  v8, (a0), v16
 vsetvli zero, zero, e32, mf2, tu, mu
 vluxseg8ei32.v  v8, (a0), v16
 vsetvli zero, zero, e32, m1, tu, mu
 vluxseg8ei32.v  v8, (a0), v16
-vsetvli zero, zero, e32, m2, tu, mu
-vluxseg8ei32.v  v8, (a0), v16
-vsetvli zero, zero, e32, m4, tu, mu
-vluxseg8ei32.v  v8, (a0), v16
-vsetvli zero, zero, e64, mf8, tu, mu
-vluxseg8ei64.v  v8, (a0), v16
-vsetvli zero, zero, e64, mf4, tu, mu
-vluxseg8ei64.v  v8, (a0), v16
-vsetvli zero, zero, e64, mf2, tu, mu
-vluxseg8ei64.v  v8, (a0), v16
 vsetvli zero, zero, e64, m1, tu, mu
 vluxseg8ei64.v  v8, (a0), v16
-vsetvli zero, zero, e64, m2, tu, mu
-vluxseg8ei64.v  v8, (a0), v16
-vsetvli zero, zero, e64, m4, tu, mu
-vluxseg8ei64.v  v8, (a0), v16
 
 vsetvli zero, zero, e8, mf8, tu, mu
 vloxseg2ei8.v v8, (a0), v16
@@ -2215,10 +1087,6 @@ vsetvli zero, zero, e8, m2, tu, mu
 vloxseg2ei8.v v8, (a0), v16
 vsetvli zero, zero, e8, m4, tu, mu
 vloxseg2ei8.v v8, (a0), v16
-vsetvli zero, zero, e8, m8, tu, mu
-vloxseg2ei8.v v8, (a0), v16
-vsetvli zero, zero, e16, mf8, tu, mu
-vloxseg2ei16.v  v8, (a0), v16
 vsetvli zero, zero, e16, mf4, tu, mu
 vloxseg2ei16.v  v8, (a0), v16
 vsetvli zero, zero, e16, mf2, tu, mu
@@ -2229,12 +1097,6 @@ vsetvli zero, zero, e16, m2, tu, mu
 vloxseg2ei16.v  v8, (a0), v16
 vsetvli zero, zero, e16, m4, tu, mu
 vloxseg2ei16.v  v8, (a0), v16
-vsetvli zero, zero, e16, m8, tu, mu
-vloxseg2ei16.v  v8, (a0), v16
-vsetvli zero, zero, e32, mf8, tu, mu
-vloxseg2ei32.v  v8, (a0), v16
-vsetvli zero, zero, e32, mf4, tu, mu
-vloxseg2ei32.v  v8, (a0), v16
 vsetvli zero, zero, e32, mf2, tu, mu
 vloxseg2ei32.v  v8, (a0), v16
 vsetvli zero, zero, e32, m1, tu, mu
@@ -2243,22 +1105,12 @@ vsetvli zero, zero, e32, m2, tu, mu
 vloxseg2ei32.v  v8, (a0), v16
 vsetvli zero, zero, e32, m4, tu, mu
 vloxseg2ei32.v  v8, (a0), v16
-vsetvli zero, zero, e32, m8, tu, mu
-vloxseg2ei32.v  v8, (a0), v16
-vsetvli zero, zero, e64, mf8, tu, mu
-vloxseg2ei64.v  v8, (a0), v16
-vsetvli zero, zero, e64, mf4, tu, mu
-vloxseg2ei64.v  v8, (a0), v16
-vsetvli zero, zero, e64, mf2, tu, mu
-vloxseg2ei64.v  v8, (a0), v16
 vsetvli zero, zero, e64, m1, tu, mu
 vloxseg2ei64.v  v8, (a0), v16
 vsetvli zero, zero, e64, m2, tu, mu
 vloxseg2ei64.v  v8, (a0), v16
 vsetvli zero, zero, e64, m4, tu, mu
 vloxseg2ei64.v  v8, (a0), v16
-vsetvli zero, zero, e64, m8, tu, mu
-vloxseg2ei64.v  v8, (a0), v16
 
 vsetvli zero, zero, e8, mf8, tu, mu
 vloxseg3ei8.v v8, (a0), v16
@@ -2270,12 +1122,6 @@ vsetvli zero, zero, e8, m1, tu, mu
 vloxseg3ei8.v v8, (a0), v16
 vsetvli zero, zero, e8, m2, tu, mu
 vloxseg3ei8.v v8, (a0), v16
-vsetvli zero, zero, e8, m4, tu, mu
-vloxseg3ei8.v v8, (a0), v16
-vsetvli zero, zero, e8, m8, tu, mu
-vloxseg3ei8.v v8, (a0), v16
-vsetvli zero, zero, e16, mf8, tu, mu
-vloxseg3ei16.v  v8, (a0), v16
 vsetvli zero, zero, e16, mf4, tu, mu
 vloxseg3ei16.v  v8, (a0), v16
 vsetvli zero, zero, e16, mf2, tu, mu
@@ -2284,38 +1130,16 @@ vsetvli zero, zero, e16, m1, tu, mu
 vloxseg3ei16.v  v8, (a0), v16
 vsetvli zero, zero, e16, m2, tu, mu
 vloxseg3ei16.v  v8, (a0), v16
-vsetvli zero, zero, e16, m4, tu, mu
-vloxseg3ei16.v  v8, (a0), v16
-vsetvli zero, zero, e16, m8, tu, mu
-vloxseg3ei16.v  v8, (a0), v16
-vsetvli zero, zero, e32, mf8, tu, mu
-vloxseg3ei32.v  v8, (a0), v16
-vsetvli zero, zero, e32, mf4, tu, mu
-vloxseg3ei32.v  v8, (a0), v16
 vsetvli zero, zero, e32, mf2, tu, mu
 vloxseg3ei32.v  v8, (a0), v16
 vsetvli zero, zero, e32, m1, tu, mu
 vloxseg3ei32.v  v8, (a0), v16
 vsetvli zero, zero, e32, m2, tu, mu
 vloxseg3ei32.v  v8, (a0), v16
-vsetvli zero, zero, e32, m4, tu, mu
-vloxseg3ei32.v  v8, (a0), v16
-vsetvli zero, zero, e32, m8, tu, mu
-vloxseg3ei32.v  v8, (a0), v16
-vsetvli zero, zero, e64, mf8, tu, mu
-vloxseg3ei64.v  v8, (a0), v16
-vsetvli zero, zero, e64, mf4, tu, mu
-vloxseg3ei64.v  v8, (a0), v16
-vsetvli zero, zero, e64, mf2, tu, mu
-vloxseg3ei64.v  v8, (a0), v16
 vsetvli zero, zero, e64, m1, tu, mu
 vloxseg3ei64.v  v8, (a0), v16
 vsetvli zero, zero, e64, m2, tu, mu
 vloxseg3ei64.v  v8, (a0), v16
-vsetvli zero, zero, e64, m4, tu, mu
-vloxseg3ei64.v  v8, (a0), v16
-vsetvli zero, zero, e64, m8, tu, mu
-vloxseg3ei64.v  v8, (a0), v16
 
 vsetvli zero, zero, e8, mf8, tu, mu
 vloxseg4ei8.v v8, (a0), v16
@@ -2327,12 +1151,6 @@ vsetvli zero, zero, e8, m1, tu, mu
 vloxseg4ei8.v v8, (a0), v16
 vsetvli zero, zero, e8, m2, tu, mu
 vloxseg4ei8.v v8, (a0), v16
-vsetvli zero, zero, e8, m4, tu, mu
-vloxseg4ei8.v v8, (a0), v16
-vsetvli zero, zero, e8, m8, tu, mu
-vloxseg4ei8.v v8, (a0), v16
-vsetvli zero, zero, e16, mf8, tu, mu
-vloxseg4ei16.v  v8, (a0), v16
 vsetvli zero, zero, e16, mf4, tu, mu
 vloxseg4ei16.v  v8, (a0), v16
 vsetvli zero, zero, e16, mf2, tu, mu
@@ -2341,38 +1159,16 @@ vsetvli zero, zero, e16, m1, tu, mu
 vloxseg4ei16.v  v8, (a0), v16
 vsetvli zero, zero, e16, m2, tu, mu
 vloxseg4ei16.v  v8, (a0), v16
-vsetvli zero, zero, e16, m4, tu, mu
-vloxseg4ei16.v  v8, (a0), v16
-vsetvli zero, zero, e16, m8, tu, mu
-vloxseg4ei16.v  v8, (a0), v16
-vsetvli zero, zero, e32, mf8, tu, mu
-vloxseg4ei32.v  v8, (a0), v16
-vsetvli zero, zero, e32, mf4, tu, mu
-vloxseg4ei32.v  v8, (a0), v16
 vsetvli zero, zero, e32, mf2, tu, mu
 vloxseg4ei32.v  v8, (a0), v16
 vsetvli zero, zero, e32, m1, tu, mu
 vloxseg4ei32.v  v8, (a0), v16
 vsetvli zero, zero, e32, m2, tu, mu
 vloxseg4ei32.v  v8, (a0), v16
-vsetvli zero, zero, e32, m4, tu, mu
-vloxseg4ei32.v  v8, (a0), v16
-vsetvli zero, zero, e32, m8, tu, mu
-vloxseg4ei32.v  v8, (a0), v16
-vsetvli zero, zero, e64, mf8, tu, mu
-vloxseg4ei64.v  v8, (a0), v16
-vsetvli zero, zero, e64, mf4, tu, mu
-vloxseg4ei64.v  v8, (a0), v16
-vsetvli zero, zero, e64, mf2, tu, mu
-vloxseg4ei64.v  v8, (a0), v16
 vsetvli zero, zero, e64, m1, tu, mu
 vloxseg4ei64.v  v8, (a0), v16
 vsetvli zero, zero, e64, m2, tu, mu
 vloxseg4ei64.v  v8, (a0), v16
-vsetvli zero, zero, e64, m4, tu, mu
-vloxseg4ei64.v  v8, (a0), v16
-vsetvli zero, zero, e64, m8, tu, mu
-vloxseg4ei64.v  v8, (a0), v16
 
 vsetvli zero, zero, e8, mf8, tu, mu
 vloxseg5ei8.v v8, (a0), v16
@@ -2382,46 +1178,18 @@ vsetvli zero, zero, e8, mf2, tu, mu
 vloxseg5ei8.v v8, (a0), v16
 vsetvli zero, zero, e8, m1, tu, mu
 vloxseg5ei8.v v8, (a0), v16
-vsetvli zero, zero, e8, m2, tu, mu
-vloxseg5ei8.v v8, (a0), v16
-vsetvli zero, zero, e8, m4, tu, mu
-vloxseg5ei8.v v8, (a0), v16
-vsetvli zero, zero, e16, mf8, tu, mu
-vloxseg5ei16.v  v8, (a0), v16
 vsetvli zero, zero, e16, mf4, tu, mu
 vloxseg5ei16.v  v8, (a0), v16
 vsetvli zero, zero, e16, mf2, tu, mu
 vloxseg5ei16.v  v8, (a0), v16
 vsetvli zero, zero, e16, m1, tu, mu
 vloxseg5ei16.v  v8, (a0), v16
-vsetvli zero, zero, e16, m2, tu, mu
-vloxseg5ei16.v  v8, (a0), v16
-vsetvli zero, zero, e16, m4, tu, mu
-vloxseg5ei16.v  v8, (a0), v16
-vsetvli zero, zero, e32, mf8, tu, mu
-vloxseg5ei32.v  v8, (a0), v16
-vsetvli zero, zero, e32, mf4, tu, mu
-vloxseg5ei32.v  v8, (a0), v16
 vsetvli zero, zero, e32, mf2, tu, mu
 vloxseg5ei32.v  v8, (a0), v16
 vsetvli zero, zero, e32, m1, tu, mu
 vloxseg5ei32.v  v8, (a0), v16
-vsetvli zero, zero, e32, m2, tu, mu
-vloxseg5ei32.v  v8, (a0), v16
-vsetvli zero, zero, e32, m4, tu, mu
-vloxseg5ei32.v  v8, (a0), v16
-vsetvli zero, zero, e64, mf8, tu, mu
-vloxseg5ei64.v  v8, (a0), v16
-vsetvli zero, zero, e64, mf4, tu, mu
-vloxseg5ei64.v  v8, (a0), v16
-vsetvli zero, zero, e64, mf2, tu, mu
-vloxseg5ei64.v  v8, (a0), v16
 vsetvli zero, zero, e64, m1, tu, mu
 vloxseg5ei64.v  v8, (a0), v16
-vsetvli zero, zero, e64, m2, tu, mu
-vloxseg5ei64.v  v8, (a0), v16
-vsetvli zero, zero, e64, m4, tu, mu
-vloxseg5ei64.v  v8, (a0), v16
 
 vsetvli zero, zero, e8, mf8, tu, mu
 vloxseg6ei8.v v8, (a0), v16
@@ -2431,46 +1199,18 @@ vsetvli zero, zero, e8, mf2, tu, mu
 vloxseg6ei8.v v8, (a0), v16
 vsetvli zero, zero, e8, m1, tu, mu
 vloxseg6ei8.v v8, (a0), v16
-vsetvli zero, zero, e8, m2, tu, mu
-vloxseg6ei8.v v8, (a0), v16
-vsetvli zero, zero, e8, m4, tu, mu
-vloxseg6ei8.v v8, (a0), v16
-vsetvli zero, zero, e16, mf8, tu, mu
-vloxseg6ei16.v  v8, (a0), v16
 vsetvli zero, zero, e16, mf4, tu, mu
 vloxseg6ei16.v  v8, (a0), v16
 vsetvli zero, zero, e16, mf2, tu, mu
 vloxseg6ei16.v  v8, (a0), v16
 vsetvli zero, zero, e16, m1, tu, mu
 vloxseg6ei16.v  v8, (a0), v16
-vsetvli zero, zero, e16, m2, tu, mu
-vloxseg6ei16.v  v8, (a0), v16
-vsetvli zero, zero, e16, m4, tu, mu
-vloxseg6ei16.v  v8, (a0), v16
-vsetvli zero, zero, e32, mf8, tu, mu
-vloxseg6ei32.v  v8, (a0), v16
-vsetvli zero, zero, e32, mf4, tu, mu
-vloxseg6ei32.v  v8, (a0), v16
 vsetvli zero, zero, e32, mf2, tu, mu
 vloxseg6ei32.v  v8, (a0), v16
 vsetvli zero, zero, e32, m1, tu, mu
 vloxseg6ei32.v  v8, (a0), v16
-vsetvli zero, zero, e32, m2, tu, mu
-vloxseg6ei32.v  v8, (a0), v16
-vsetvli zero, zero, e32, m4, tu, mu
-vloxseg6ei32.v  v8, (a0), v16
-vsetvli zero, zero, e64, mf8, tu, mu
-vloxseg6ei64.v  v8, (a0), v16
-vsetvli zero, zero, e64, mf4, tu, mu
-vloxseg6ei64.v  v8, (a0), v16
-vsetvli zero, zero, e64, mf2, tu, mu
-vloxseg6ei64.v  v8, (a0), v16
 vsetvli zero, zero, e64, m1, tu, mu
 vloxseg6ei64.v  v8, (a0), v16
-vsetvli zero, zero, e64, m2, tu, mu
-vloxseg6ei64.v  v8, (a0), v16
-vsetvli zero, zero, e64, m4, tu, mu
-vloxseg6ei64.v  v8, (a0), v16
 
 vsetvli zero, zero, e8, mf8, tu, mu
 vloxseg7ei8.v v8, (a0), v16
@@ -2480,46 +1220,18 @@ vsetvli zero, zero, e8, mf2, tu, mu
 vloxseg7ei8.v v8, (a0), v16
 vsetvli zero, zero, e8, m1, tu, mu
 vloxseg7ei8.v v8, (a0), v16
-vsetvli zero, zero, e8, m2, tu, mu
-vloxseg7ei8.v v8, (a0), v16
-vsetvli zero, zero, e8, m4, tu, mu
-vloxseg7ei8.v v8, (a0), v16
-vsetvli zero, zero, e16, mf8, tu, mu
-vloxseg7ei16.v  v8, (a0), v16
 vsetvli zero, zero, e16, mf4, tu, mu
 vloxseg7ei16.v  v8, (a0), v16
 vsetvli zero, zero, e16, mf2, tu, mu
 vloxseg7ei16.v  v8, (a0), v16
 vsetvli zero, zero, e16, m1, tu, mu
 vloxseg7ei16.v  v8, (a0), v16
-vsetvli zero, zero, e16, m2, tu, mu
-vloxseg7ei16.v  v8, (a0), v16
-vsetvli zero, zero, e16, m4, tu, mu
-vloxseg7ei16.v  v8, (a0), v16
-vsetvli zero, zero, e32, mf8, tu, mu
-vloxseg7ei32.v  v8, (a0), v16
-vsetvli zero, zero, e32, mf4, tu, mu
-vloxseg7ei32.v  v8, (a0), v16
 vsetvli zero, zero, e32, mf2, tu, mu
 vloxseg7ei32.v  v8, (a0), v16
 vsetvli zero, zero, e32, m1, tu, mu
 vloxseg7ei32.v  v8, (a0), v16
-vsetvli zero, zero, e32, m2, tu, mu
-vloxseg7ei32.v  v8, (a0), v16
-vsetvli zero, zero, e32, m4, tu, mu
-vloxseg7ei32.v  v8, (a0), v16
-vsetvli zero, zero, e64, mf8, tu, mu
-vloxseg7ei64.v  v8, (a0), v16
-vsetvli zero, zero, e64, mf4, tu, mu
-vloxseg7ei64.v  v8, (a0), v16
-vsetvli zero, zero, e64, mf2, tu, mu
-vloxseg7ei64.v  v8, (a0), v16
 vsetvli zero, zero, e64, m1, tu, mu
 vloxseg7ei64.v  v8, (a0), v16
-vsetvli zero, zero, e64, m2, tu, mu
-vloxseg7ei64.v  v8, (a0), v16
-vsetvli zero, zero, e64, m4, tu, mu
-vloxseg7ei64.v  v8, (a0), v16
 
 vsetvli zero, zero, e8, mf8, tu, mu
 vloxseg8ei8.v v8, (a0), v16
@@ -2529,46 +1241,18 @@ vsetvli zero, zero, e8, mf2, tu, mu
 vloxseg8ei8.v v8, (a0), v16
 vsetvli zero, zero, e8, m1, tu, mu
 vloxseg8ei8.v v8, (a0), v16
-vsetvli zero, zero, e8, m2, tu, mu
-vloxseg8ei8.v v8, (a0), v16
-vsetvli zero, zero, e8, m4, tu, mu
-vloxseg8ei8.v v8, (a0), v16
-vsetvli zero, zero, e16, mf8, tu, mu
-vloxseg8ei16.v  v8, (a0), v16
 vsetvli zero, zero, e16, mf4, tu, mu
 vloxseg8ei16.v  v8, (a0), v16
 vsetvli zero, zero, e16, mf2, tu, mu
 vloxseg8ei16.v  v8, (a0), v16
 vsetvli zero, zero, e16, m1, tu, mu
 vloxseg8ei16.v  v8, (a0), v16
-vsetvli zero, zero, e16, m2, tu, mu
-vloxseg8ei16.v  v8, (a0), v16
-vsetvli zero, zero, e16, m4, tu, mu
-vloxseg8ei16.v  v8, (a0), v16
-vsetvli zero, zero, e32, mf8, tu, mu
-vloxseg8ei32.v  v8, (a0), v16
-vsetvli zero, zero, e32, mf4, tu, mu
-vloxseg8ei32.v  v8, (a0), v16
 vsetvli zero, zero, e32, mf2, tu, mu
 vloxseg8ei32.v  v8, (a0), v16
 vsetvli zero, zero, e32, m1, tu, mu
 vloxseg8ei32.v  v8, (a0), v16
-vsetvli zero, zero, e32, m2, tu, mu
-vloxseg8ei32.v  v8, (a0), v16
-vsetvli zero, zero, e32, m4, tu, mu
-vloxseg8ei32.v  v8, (a0), v16
-vsetvli zero, zero, e64, mf8, tu, mu
-vloxseg8ei64.v  v8, (a0), v16
-vsetvli zero, zero, e64, mf4, tu, mu
-vloxseg8ei64.v  v8, (a0), v16
-vsetvli zero, zero, e64, mf2, tu, mu
-vloxseg8ei64.v  v8, (a0), v16
 vsetvli zero, zero, e64, m1, tu, mu
 vloxseg8ei64.v  v8, (a0), v16
-vsetvli zero, zero, e64, m2, tu, mu
-vloxseg8ei64.v  v8, (a0), v16
-vsetvli zero, zero, e64, m4, tu, mu
-vloxseg8ei64.v  v8, (a0), v16
 
 vsetvli zero, zero, e8, mf8, tu, mu
 vsuxseg2ei8.v v8, (a0), v16
@@ -2580,12 +1264,6 @@ vsetvli zero, zero, e8, m1, tu, mu
 vsuxseg2ei8.v v8, (a0), v16
 vsetvli zero, zero, e8, m2, tu, mu
 vsuxseg2ei8.v v8, (a0), v16
-vsetvli zero, zero, e8, m4, tu, mu
-vsuxseg2ei8.v v8, (a0), v16
-vsetvli zero, zero, e8, m8, tu, mu
-vsuxseg2ei8.v v8, (a0), v16
-vsetvli zero, zero, e16, mf8, tu, mu
-vsuxseg2ei16.v  v8, (a0), v16
 vsetvli zero, zero, e16, mf4, tu, mu
 vsuxseg2ei16.v  v8, (a0), v16
 vsetvli zero, zero, e16, mf2, tu, mu
@@ -2594,38 +1272,16 @@ vsetvli zero, zero, e16, m1, tu, mu
 vsuxseg2ei16.v  v8, (a0), v16
 vsetvli zero, zero, e16, m2, tu, mu
 vsuxseg2ei16.v  v8, (a0), v16
-vsetvli zero, zero, e16, m4, tu, mu
-vsuxseg2ei16.v  v8, (a0), v16
-vsetvli zero, zero, e16, m8, tu, mu
-vsuxseg2ei16.v  v8, (a0), v16
-vsetvli zero, zero, e32, mf8, tu, mu
-vsuxseg2ei32.v  v8, (a0), v16
-vsetvli zero, zero, e32, mf4, tu, mu
-vsuxseg2ei32.v  v8, (a0), v16
 vsetvli zero, zero, e32, mf2, tu, mu
 vsuxseg2ei32.v  v8, (a0), v16
 vsetvli zero, zero, e32, m1, tu, mu
 vsuxseg2ei32.v  v8, (a0), v16
 vsetvli zero, zero, e32, m2, tu, mu
 vsuxseg2ei32.v  v8, (a0), v16
-vsetvli zero, zero, e32, m4, tu, mu
-vsuxseg2ei32.v  v8, (a0), v16
-vsetvli zero, zero, e32, m8, tu, mu
-vsuxseg2ei32.v  v8, (a0), v16
-vsetvli zero, zero, e64, mf8, tu, mu
-vsuxseg2ei64.v  v8, (a0), v16
-vsetvli zero, zero, e64, mf4, tu, mu
-vsuxseg2ei64.v  v8, (a0), v16
-vsetvli zero, zero, e64, mf2, tu, mu
-vsuxseg2ei64.v  v8, (a0), v16
 vsetvli zero, zero, e64, m1, tu, mu
 vsuxseg2ei64.v  v8, (a0), v16
 vsetvli zero, zero, e64, m2, tu, mu
 vsuxseg2ei64.v  v8, (a0), v16
-vsetvli zero, zero, e64, m4, tu, mu
-vsuxseg2ei64.v  v8, (a0), v16
-vsetvli zero, zero, e64, m8, tu, mu
-vsuxseg2ei64.v  v8, (a0), v16
 
 vsetvli zero, zero, e8, mf8, tu, mu
 vsuxseg3ei8.v v8, (a0), v16
@@ -2637,12 +1293,6 @@ vsetvli zero, zero, e8, m1, tu, mu
 vsuxseg3ei8.v v8, (a0), v16
 vsetvli zero, zero, e8, m2, tu, mu
 vsuxseg3ei8.v v8, (a0), v16
-vsetvli zero, zero, e8, m4, tu, mu
-vsuxseg3ei8.v v8, (a0), v16
-vsetvli zero, zero, e8, m8, tu, mu
-vsuxseg3ei8.v v8, (a0), v16
-vsetvli zero, zero, e16, mf8, tu, mu
-vsuxseg3ei16.v  v8, (a0), v16
 vsetvli zero, zero, e16, mf4, tu, mu
 vsuxseg3ei16.v  v8, (a0), v16
 vsetvli zero, zero, e16, mf2, tu, mu
@@ -2651,38 +1301,16 @@ vsetvli zero, zero, e16, m1, tu, mu
 vsuxseg3ei16.v  v8, (a0), v16
 vsetvli zero, zero, e16, m2, tu, mu
 vsuxseg3ei16.v  v8, (a0), v16
-vsetvli zero, zero, e16, m4, tu, mu
-vsuxseg3ei16.v  v8, (a0), v16
-vsetvli zero, zero, e16, m8, tu, mu
-vsuxseg3ei16.v  v8, (a0), v16
-vsetvli zero, zero, e32, mf8, tu, mu
-vsuxseg3ei32.v  v8, (a0), v16
-vsetvli zero, zero, e32, mf4, tu, mu
-vsuxseg3ei32.v  v8, (a0), v16
 vsetvli zero, zero, e32, mf2, tu, mu
 vsuxseg3ei32.v  v8, (a0), v16
 vsetvli zero, zero, e32, m1, tu, mu
 vsuxseg3ei32.v  v8, (a0), v16
 vsetvli zero, zero, e32, m2, tu, mu
 vsuxseg3ei32.v  v8, (a0), v16
-vsetvli zero, zero, e32, m4, tu, mu
-vsuxseg3ei32.v  v8, (a0), v16
-vsetvli zero, zero, e32, m8, tu, mu
-vsuxseg3ei32.v  v8, (a0), v16
-vsetvli zero, zero, e64, mf8, tu, mu
-vsuxseg3ei64.v  v8, (a0), v16
-vsetvli zero, zero, e64, mf4, tu, mu
-vsuxseg3ei64.v  v8, (a0), v16
-vsetvli zero, zero, e64, mf2, tu, mu
-vsuxseg3ei64.v  v8, (a0), v16
 vsetvli zero, zero, e64, m1, tu, mu
 vsuxseg3ei64.v  v8, (a0), v16
 vsetvli zero, zero, e64, m2, tu, mu
 vsuxseg3ei64.v  v8, (a0), v16
-vsetvli zero, zero, e64, m4, tu, mu
-vsuxseg3ei64.v  v8, (a0), v16
-vsetvli zero, zero, e64, m8, tu, mu
-vsuxseg3ei64.v  v8, (a0), v16
 
 vsetvli zero, zero, e8, mf8, tu, mu
 vsuxseg4ei8.v v8, (a0), v16
@@ -2694,12 +1322,6 @@ vsetvli zero, zero, e8, m1, tu, mu
 vsuxseg4ei8.v v8, (a0), v16
 vsetvli zero, zero, e8, m2, tu, mu
 vsuxseg4ei8.v v8, (a0), v16
-vsetvli zero, zero, e8, m4, tu, mu
-vsuxseg4ei8.v v8, (a0), v16
-vsetvli zero, zero, e8, m8, tu, mu
-vsuxseg4ei8.v v8, (a0), v16
-vsetvli zero, zero, e16, mf8, tu, mu
-vsuxseg4ei16.v  v8, (a0), v16
 vsetvli zero, zero, e16, mf4, tu, mu
 vsuxseg4ei16.v  v8, (a0), v16
 vsetvli zero, zero, e16, mf2, tu, mu
@@ -2708,38 +1330,16 @@ vsetvli zero, zero, e16, m1, tu, mu
 vsuxseg4ei16.v  v8, (a0), v16
 vsetvli zero, zero, e16, m2, tu, mu
 vsuxseg4ei16.v  v8, (a0), v16
-vsetvli zero, zero, e16, m4, tu, mu
-vsuxseg4ei16.v  v8, (a0), v16
-vsetvli zero, zero, e16, m8, tu, mu
-vsuxseg4ei16.v  v8, (a0), v16
-vsetvli zero, zero, e32, mf8, tu, mu
-vsuxseg4ei32.v  v8, (a0), v16
-vsetvli zero, zero, e32, mf4, tu, mu
-vsuxseg4ei32.v  v8, (a0), v16
 vsetvli zero, zero, e32, mf2, tu, mu
 vsuxseg4ei32.v  v8, (a0), v16
 vsetvli zero, zero, e32, m1, tu, mu
 vsuxseg4ei32.v  v8, (a0), v16
 vsetvli zero, zero, e32, m2, tu, mu
 vsuxseg4ei32.v  v8, (a0), v16
-vsetvli zero, zero, e32, m4, tu, mu
-vsuxseg4ei32.v  v8, (a0), v16
-vsetvli zero, zero, e32, m8, tu, mu
-vsuxseg4ei32.v  v8, (a0), v16
-vsetvli zero, zero, e64, mf8, tu, mu
-vsuxseg4ei64.v  v8, (a0), v16
-vsetvli zero, zero, e64, mf4, tu, mu
-vsuxseg4ei64.v  v8, (a0), v16
-vsetvli zero, zero, e64, mf2, tu, mu
-vsuxseg4ei64.v  v8, (a0), v16
 vsetvli zero, zero, e64, m1, tu, mu
 vsuxseg4ei64.v  v8, (a0), v16
 vsetvli zero, zero, e64, m2, tu, mu
 vsuxseg4ei64.v  v8, (a0), v16
-vsetvli zero, zero, e64, m4, tu, mu
-vsuxseg4ei64.v  v8, (a0), v16
-vsetvli zero, zero, e64, m8, tu, mu
-vsuxseg4ei64.v  v8, (a0), v16
 
 vsetvli zero, zero, e8, mf8, tu, mu
 vsuxseg5ei8.v v8, (a0), v16
@@ -2749,46 +1349,18 @@ vsetvli zero, zero, e8, mf2, tu, mu
 vsuxseg5ei8.v v8, (a0), v16
 vsetvli zero, zero, e8, m1, tu, mu
 vsuxseg5ei8.v v8, (a0), v16
-vsetvli zero, zero, e8, m2, tu, mu
-vsuxseg5ei8.v v8, (a0), v16
-vsetvli zero, zero, e8, m4, tu, mu
-vsuxseg5ei8.v v8, (a0), v16
-vsetvli zero, zero, e16, mf8, tu, mu
-vsuxseg5ei16.v  v8, (a0), v16
 vsetvli zero, zero, e16, mf4, tu, mu
 vsuxseg5ei16.v  v8, (a0), v16
 vsetvli zero, zero, e16, mf2, tu, mu
 vsuxseg5ei16.v  v8, (a0), v16
 vsetvli zero, zero, e16, m1, tu, mu
 vsuxseg5ei16.v  v8, (a0), v16
-vsetvli zero, zero, e16, m2, tu, mu
-vsuxseg5ei16.v  v8, (a0), v16
-vsetvli zero, zero, e16, m4, tu, mu
-vsuxseg5ei16.v  v8, (a0), v16
-vsetvli zero, zero, e32, mf8, tu, mu
-vsuxseg5ei32.v  v8, (a0), v16
-vsetvli zero, zero, e32, mf4, tu, mu
-vsuxseg5ei32.v  v8, (a0), v16
 vsetvli zero, zero, e32, mf2, tu, mu
 vsuxseg5ei32.v  v8, (a0), v16
 vsetvli zero, zero, e32, m1, tu, mu
 vsuxseg5ei32.v  v8, (a0), v16
-vsetvli zero, zero, e32, m2, tu, mu
-vsuxseg5ei32.v  v8, (a0), v16
-vsetvli zero, zero, e32, m4, tu, mu
-vsuxseg5ei32.v  v8, (a0), v16
-vsetvli zero, zero, e64, mf8, tu, mu
-vsuxseg5ei64.v  v8, (a0), v16
-vsetvli zero, zero, e64, mf4, tu, mu
-vsuxseg5ei64.v  v8, (a0), v16
-vsetvli zero, zero, e64, mf2, tu, mu
-vsuxseg5ei64.v  v8, (a0), v16
 vsetvli zero, zero, e64, m1, tu, mu
 vsuxseg5ei64.v  v8, (a0), v16
-vsetvli zero, zero, e64, m2, tu, mu
-vsuxseg5ei64.v  v8, (a0), v16
-vsetvli zero, zero, e64, m4, tu, mu
-vsuxseg5ei64.v  v8, (a0), v16
 
 vsetvli zero, zero, e8, mf8, tu, mu
 vsuxseg6ei8.v v8, (a0), v16
@@ -2798,46 +1370,18 @@ vsetvli zero, zero, e8, mf2, tu, mu
 vsuxseg6ei8.v v8, (a0), v16
 vsetvli zero, zero, e8, m1, tu, mu
 vsuxseg6ei8.v v8, (a0), v16
-vsetvli zero, zero, e8, m2, tu, mu
-vsuxseg6ei8.v v8, (a0), v16
-vsetvli zero, zero, e8, m4, tu, mu
-vsuxseg6ei8.v v8, (a0), v16
-vsetvli zero, zero, e16, mf8, tu, mu
-vsuxseg6ei16.v  v8, (a0), v16
 vsetvli zero, zero, e16, mf4, tu, mu
 vsuxseg6ei16.v  v8, (a0), v16
 vsetvli zero, zero, e16, mf2, tu, mu
 vsuxseg6ei16.v  v8, (a0), v16
 vsetvli zero, zero, e16, m1, tu, mu
 vsuxseg6ei16.v  v8, (a0), v16
-vsetvli zero, zero, e16, m2, tu, mu
-vsuxseg6ei16.v  v8, (a0), v16
-vsetvli zero, zero, e16, m4, tu, mu
-vsuxseg6ei16.v  v8, (a0), v16
-vsetvli zero, zero, e32, mf8, tu, mu
-vsuxseg6ei32.v  v8, (a0), v16
-vsetvli zero, zero, e32, mf4, tu, mu
-vsuxseg6ei32.v  v8, (a0), v16
 vsetvli zero, zero, e32, mf2, tu, mu
 vsuxseg6ei32.v  v8, (a0), v16
 vsetvli zero, zero, e32, m1, tu, mu
 vsuxseg6ei32.v  v8, (a0), v16
-vsetvli zero, zero, e32, m2, tu, mu
-vsuxseg6ei32.v  v8, (a0), v16
-vsetvli zero, zero, e32, m4, tu, mu
-vsuxseg6ei32.v  v8, (a0), v16
-vsetvli zero, zero, e64, mf8, tu, mu
-vsuxseg6ei64.v  v8, (a0), v16
-vsetvli zero, zero, e64, mf4, tu, mu
-vsuxseg6ei64.v  v8, (a0), v16
-vsetvli zero, zero, e64, mf2, tu, mu
-vsuxseg6ei64.v  v8, (a0), v16
 vsetvli zero, zero, e64, m1, tu, mu
 vsuxseg6ei64.v  v8, (a0), v16
-vsetvli zero, zero, e64, m2, tu, mu
-vsuxseg6ei64.v  v8, (a0), v16
-vsetvli zero, zero, e64, m4, tu, mu
-vsuxseg6ei64.v  v8, (a0), v16
 
 vsetvli zero, zero, e8, mf8, tu, mu
 vsuxseg7ei8.v v8, (a0), v16
@@ -2847,46 +1391,18 @@ vsetvli zero, zero, e8, mf2, tu, mu
 vsuxseg7ei8.v v8, (a0), v16
 vsetvli zero, zero, e8, m1, tu, mu
 vsuxseg7ei8.v v8, (a0), v16
-vsetvli zero, zero, e8, m2, tu, mu
-vsuxseg7ei8.v v8, (a0), v16
-vsetvli zero, zero, e8, m4, tu, mu
-vsuxseg7ei8.v v8, (a0), v16
-vsetvli zero, zero, e16, mf8, tu, mu
-vsuxseg7ei16.v  v8, (a0), v16
 vsetvli zero, zero, e16, mf4, tu, mu
 vsuxseg7ei16.v  v8, (a0), v16
 vsetvli zero, zero, e16, mf2, tu, mu
 vsuxseg7ei16.v  v8, (a0), v16
 vsetvli zero, zero, e16, m1, tu, mu
 vsuxseg7ei16.v  v8, (a0), v16
-vsetvli zero, zero, e16, m2, tu, mu
-vsuxseg7ei16.v  v8, (a0), v16
-vsetvli zero, zero, e16, m4, tu, mu
-vsuxseg7ei16.v  v8, (a0), v16
-vsetvli zero, zero, e32, mf8, tu, mu
-vsuxseg7ei32.v  v8, (a0), v16
-vsetvli zero, zero, e32, mf4, tu, mu
-vsuxseg7ei32.v  v8, (a0), v16
 vsetvli zero, zero, e32, mf2, tu, mu
 vsuxseg7ei32.v  v8, (a0), v16
 vsetvli zero, zero, e32, m1, tu, mu
 vsuxseg7ei32.v  v8, (a0), v16
-vsetvli zero, zero, e32, m2, tu, mu
-vsuxseg7ei32.v  v8, (a0), v16
-vsetvli zero, zero, e32, m4, tu, mu
-vsuxseg7ei32.v  v8, (a0), v16
-vsetvli zero, zero, e64, mf8, tu, mu
-vsuxseg7ei64.v  v8, (a0), v16
-vsetvli zero, zero, e64, mf4, tu, mu
-vsuxseg7ei64.v  v8, (a0), v16
-vsetvli zero, zero, e64, mf2, tu, mu
-vsuxseg7ei64.v  v8, (a0), v16
 vsetvli zero, zero, e64, m1, tu, mu
 vsuxseg7ei64.v  v8, (a0), v16
-vsetvli zero, zero, e64, m2, tu, mu
-vsuxseg7ei64.v  v8, (a0), v16
-vsetvli zero, zero, e64, m4, tu, mu
-vsuxseg7ei64.v  v8, (a0), v16
 
 vsetvli zero, zero, e8, mf8, tu, mu
 vsuxseg8ei8.v v8, (a0), v16
@@ -2896,46 +1412,18 @@ vsetvli zero, zero, e8, mf2, tu, mu
 vsuxseg8ei8.v v8, (a0), v16
 vsetvli zero, zero, e8, m1, tu, mu
 vsuxseg8ei8.v v8, (a0), v16
-vsetvli zero, zero, e8, m2, tu, mu
-vsuxseg8ei8.v v8, (a0), v16
-vsetvli zero, zero, e8, m4, tu, mu
-vsuxseg8ei8.v v8, (a0), v16
-vsetvli zero, zero, e16, mf8, tu, mu
-vsuxseg8ei16.v  v8, (a0), v16
 vsetvli zero, zero, e16, mf4, tu, mu
 vsuxseg8ei16.v  v8, (a0), v16
 vsetvli zero, zero, e16, mf2, tu, mu
 vsuxseg8ei16.v  v8, (a0), v16
 vsetvli zero, zero, e16, m1, tu, mu
 vsuxseg8ei16.v  v8, (a0), v16
-vsetvli zero, zero, e16, m2, tu, mu
-vsuxseg8ei16.v  v8, (a0), v16
-vsetvli zero, zero, e16, m4, tu, mu
-vsuxseg8ei16.v  v8, (a0), v16
-vsetvli zero, zero, e32, mf8, tu, mu
-vsuxseg8ei32.v  v8, (a0), v16
-vsetvli zero, zero, e32, mf4, tu, mu
-vsuxseg8ei32.v  v8, (a0), v16
 vsetvli zero, zero, e32, mf2, tu, mu
 vsuxseg8ei32.v  v8, (a0), v16
 vsetvli zero, zero, e32, m1, tu, mu
 vsuxseg8ei32.v  v8, (a0), v16
-vsetvli zero, zero, e32, m2, tu, mu
-vsuxseg8ei32.v  v8, (a0), v16
-vsetvli zero, zero, e32, m4, tu, mu
-vsuxseg8ei32.v  v8, (a0), v16
-vsetvli zero, zero, e64, mf8, tu, mu
-vsuxseg8ei64.v  v8, (a0), v16
-vsetvli zero, zero, e64, mf4, tu, mu
-vsuxseg8ei64.v  v8, (a0), v16
-vsetvli zero, zero, e64, mf2, tu, mu
-vsuxseg8ei64.v  v8, (a0), v16
 vsetvli zero, zero, e64, m1, tu, mu
 vsuxseg8ei64.v  v8, (a0), v16
-vsetvli zero, zero, e64, m2, tu, mu
-vsuxseg8ei64.v  v8, (a0), v16
-vsetvli zero, zero, e64, m4, tu, mu
-vsuxseg8ei64.v  v8, (a0), v16
 
 vsetvli zero, zero, e8, mf8, tu, mu
 vsoxseg2ei8.v v8, (a0), v16
@@ -2949,10 +1437,6 @@ vsetvli zero, zero, e8, m2, tu, mu
 vsoxseg2ei8.v v8, (a0), v16
 vsetvli zero, zero, e8, m4, tu, mu
 vsoxseg2ei8.v v8, (a0), v16
-vsetvli zero, zero, e8, m8, tu, mu
-vsoxseg2ei8.v v8, (a0), v16
-vsetvli zero, zero, e16, mf8, tu, mu
-vsoxseg2ei16.v  v8, (a0), v16
 vsetvli zero, zero, e16, mf4, tu, mu
 vsoxseg2ei16.v  v8, (a0), v16
 vsetvli zero, zero, e16, mf2, tu, mu
@@ -2963,12 +1447,6 @@ vsetvli zero, zero, e16, m2, tu, mu
 vsoxseg2ei16.v  v8, (a0), v16
 vsetvli zero, zero, e16, m4, tu, mu
 vsoxseg2ei16.v  v8, (a0), v16
-vsetvli zero, zero, e16, m8, tu, mu
-vsoxseg2ei16.v  v8, (a0), v16
-vsetvli zero, zero, e32, mf8, tu, mu
-vsoxseg2ei32.v  v8, (a0), v16
-vsetvli zero, zero, e32, mf4, tu, mu
-vsoxseg2ei32.v  v8, (a0), v16
 vsetvli zero, zero, e32, mf2, tu, mu
 vsoxseg2ei32.v  v8, (a0), v16
 vsetvli zero, zero, e32, m1, tu, mu
@@ -2977,22 +1455,12 @@ vsetvli zero, zero, e32, m2, tu, mu
 vsoxseg2ei32.v  v8, (a0), v16
 vsetvli zero, zero, e32, m4, tu, mu
 vsoxseg2ei32.v  v8, (a0), v16
-vsetvli zero, zero, e32, m8, tu, mu
-vsoxseg2ei32.v  v8, (a0), v16
-vsetvli zero, zero, e64, mf8, tu, mu
-vsoxseg2ei64.v  v8, (a0), v16
-vsetvli zero, zero, e64, mf4, tu, mu
-vsoxseg2ei64.v  v8, (a0), v16
-vsetvli zero, zero, e64, mf2, tu, mu
-vsoxseg2ei64.v  v8, (a0), v16
 vsetvli zero, zero, e64, m1, tu, mu
 vsoxseg2ei64.v  v8, (a0), v16
 vsetvli zero, zero, e64, m2, tu, mu
 vsoxseg2ei64.v  v8, (a0), v16
 vsetvli zero, zero, e64, m4, tu, mu
 vsoxseg2ei64.v  v8, (a0), v16
-vsetvli zero, zero, e64, m8, tu, mu
-vsoxseg2ei64.v  v8, (a0), v16
 
 vsetvli zero, zero, e8, mf8, tu, mu
 vsoxseg3ei8.v v8, (a0), v16
@@ -3004,12 +1472,6 @@ vsetvli zero, zero, e8, m1, tu, mu
 vsoxseg3ei8.v v8, (a0), v16
 vsetvli zero, zero, e8, m2, tu, mu
 vsoxseg3ei8.v v8, (a0), v16
-vsetvli zero, zero, e8, m4, tu, mu
-vsoxseg3ei8.v v8, (a0), v16
-vsetvli zero, zero, e8, m8, tu, mu
-vsoxseg3ei8.v v8, (a0), v16
-vsetvli zero, zero, e16, mf8, tu, mu
-vsoxseg3ei16.v  v8, (a0), v16
 vsetvli zero, zero, e16, mf4, tu, mu
 vsoxseg3ei16.v  v8, (a0), v16
 vsetvli zero, zero, e16, mf2, tu, mu
@@ -3018,38 +1480,16 @@ vsetvli zero, zero, e16, m1, tu, mu
 vsoxseg3ei16.v  v8, (a0), v16
 vsetvli zero, zero, e16, m2, tu, mu
 vsoxseg3ei16.v  v8, (a0), v16
-vsetvli zero, zero, e16, m4, tu, mu
-vsoxseg3ei16.v  v8, (a0), v16
-vsetvli zero, zero, e16, m8, tu, mu
-vsoxseg3ei16.v  v8, (a0), v16
-vsetvli zero, zero, e32, mf8, tu, mu
-vsoxseg3ei32.v  v8, (a0), v16
-vsetvli zero, zero, e32, mf4, tu, mu
-vsoxseg3ei32.v  v8, (a0), v16
 vsetvli zero, zero, e32, mf2, tu, mu
 vsoxseg3ei32.v  v8, (a0), v16
 vsetvli zero, zero, e32, m1, tu, mu
 vsoxseg3ei32.v  v8, (a0), v16
 vsetvli zero, zero, e32, m2, tu, mu
 vsoxseg3ei32.v  v8, (a0), v16
-vsetvli zero, zero, e32, m4, tu, mu
-vsoxseg3ei32.v  v8, (a0), v16
-vsetvli zero, zero, e32, m8, tu, mu
-vsoxseg3ei32.v  v8, (a0), v16
-vsetvli zero, zero, e64, mf8, tu, mu
-vsoxseg3ei64.v  v8, (a0), v16
-vsetvli zero, zero, e64, mf4, tu, mu
-vsoxseg3ei64.v  v8, (a0), v16
-vsetvli zero, zero, e64, mf2, tu, mu
-vsoxseg3ei64.v  v8, (a0), v16
 vsetvli zero, zero, e64, m1, tu, mu
 vsoxseg3ei64.v  v8, (a0), v16
 vsetvli zero, zero, e64, m2, tu, mu
 vsoxseg3ei64.v  v8, (a0), v16
-vsetvli zero, zero, e64, m4, tu, mu
-vsoxseg3ei64.v  v8, (a0), v16
-vsetvli zero, zero, e64, m8, tu, mu
-vsoxseg3ei64.v  v8, (a0), v16
 
 vsetvli zero, zero, e8, mf8, tu, mu
 vsoxseg4ei8.v v8, (a0), v16
@@ -3061,12 +1501,6 @@ vsetvli zero, zero, e8, m1, tu, mu
 vsoxseg4ei8.v v8, (a0), v16
 vsetvli zero, zero, e8, m2, tu, mu
 vsoxseg4ei8.v v8, (a0), v16
-vsetvli zero, zero, e8, m4, tu, mu
-vsoxseg4ei8.v v8, (a0), v16
-vsetvli zero, zero, e8, m8, tu, mu
-vsoxseg4ei8.v v8, (a0), v16
-vsetvli zero, zero, e16, mf8, tu, mu
-vsoxseg4ei16.v  v8, (a0), v16
 vsetvli zero, zero, e16, mf4, tu, mu
 vsoxseg4ei16.v  v8, (a0), v16
 vsetvli zero, zero, e16, mf2, tu, mu
@@ -3075,38 +1509,16 @@ vsetvli zero, zero, e16, m1, tu, mu
 vsoxseg4ei16.v  v8, (a0), v16
 vsetvli zero, zero, e16, m2, tu, mu
 vsoxseg4ei16.v  v8, (a0), v16
-vsetvli zero, zero, e16, m4, tu, mu
-vsoxseg4ei16.v  v8, (a0), v16
-vsetvli zero, zero, e16, m8, tu, mu
-vsoxseg4ei16.v  v8, (a0), v16
-vsetvli zero, zero, e32, mf8, tu, mu
-vsoxseg4ei32.v  v8, (a0), v16
-vsetvli zero, zero, e32, mf4, tu, mu
-vsoxseg4ei32.v  v8, (a0), v16
 vsetvli zero, zero, e32, mf2, tu, mu
 vsoxseg4ei32.v  v8, (a0), v16
 vsetvli zero, zero, e32, m1, tu, mu
 vsoxseg4ei32.v  v8, (a0), v16
 vsetvli zero, zero, e32, m2, tu, mu
 vsoxseg4ei32.v  v8, (a0), v16
-vsetvli zero, zero, e32, m4, tu, mu
-vsoxseg4ei32.v  v8, (a0), v16
-vsetvli zero, zero, e32, m8, tu, mu
-vsoxseg4ei32.v  v8, (a0), v16
-vsetvli zero, zero, e64, mf8, tu, mu
-vsoxseg4ei64.v  v8, (a0), v16
-vsetvli zero, zero, e64, mf4, tu, mu
-vsoxseg4ei64.v  v8, (a0), v16
-vsetvli zero, zero, e64, mf2, tu, mu
-vsoxseg4ei64.v  v8, (a0), v16
 vsetvli zero, zero, e64, m1, tu, mu
 vsoxseg4ei64.v  v8, (a0), v16
 vsetvli zero, zero, e64, m2, tu, mu
 vsoxseg4ei64.v  v8, (a0), v16
-vsetvli zero, zero, e64, m4, tu, mu
-vsoxseg4ei64.v  v8, (a0), v16
-vsetvli zero, zero, e64, m8, tu, mu
-vsoxseg4ei64.v  v8, (a0), v16
 
 vsetvli zero, zero, e8, mf8, tu, mu
 vsoxseg5ei8.v v8, (a0), v16
@@ -3116,46 +1528,18 @@ vsetvli zero, zero, e8, mf2, tu, mu
 vsoxseg5ei8.v v8, (a0), v16
 vsetvli zero, zero, e8, m1, tu, mu
 vsoxseg5ei8.v v8, (a0), v16
-vsetvli zero, zero, e8, m2, tu, mu
-vsoxseg5ei8.v v8, (a0), v16
-vsetvli zero, zero, e8, m4, tu, mu
-vsoxseg5ei8.v v8, (a0), v16
-vsetvli zero, zero, e16, mf8, tu, mu
-vsoxseg5ei16.v  v8, (a0), v16
 vsetvli zero, zero, e16, mf4, tu, mu
 vsoxseg5ei16.v  v8, (a0), v16
 vsetvli zero, zero, e16, mf2, tu, mu
 vsoxseg5ei16.v  v8, (a0), v16
 vsetvli zero, zero, e16, m1, tu, mu
 vsoxseg5ei16.v  v8, (a0), v16
-vsetvli zero, zero, e16, m2, tu, mu
-vsoxseg5ei16.v  v8, (a0), v16
-vsetvli zero, zero, e16, m4, tu, mu
-vsoxseg5ei16.v  v8, (a0), v16
-vsetvli zero, zero, e32, mf8, tu, mu
-vsoxseg5ei32.v  v8, (a0), v16
-vsetvli zero, zero, e32, mf4, tu, mu
-vsoxseg5ei32.v  v8, (a0), v16
 vsetvli zero, zero, e32, mf2, tu, mu
 vsoxseg5ei32.v  v8, (a0), v16
 vsetvli zero, zero, e32, m1, tu, mu
 vsoxseg5ei32.v  v8, (a0), v16
-vsetvli zero, zero, e32, m2, tu, mu
-vsoxseg5ei32.v  v8, (a0), v16
-vsetvli zero, zero, e32, m4, tu, mu
-vsoxseg5ei32.v  v8, (a0), v16
-vsetvli zero, zero, e64, mf8, tu, mu
-vsoxseg5ei64.v  v8, (a0), v16
-vsetvli zero, zero, e64, mf4, tu, mu
-vsoxseg5ei64.v  v8, (a0), v16
-vsetvli zero, zero, e64, mf2, tu, mu
-vsoxseg5ei64.v  v8, (a0), v16
 vsetvli zero, zero, e64, m1, tu, mu
 vsoxseg5ei64.v  v8, (a0), v16
-vsetvli zero, zero, e64, m2, tu, mu
-vsoxseg5ei64.v  v8, (a0), v16
-vsetvli zero, zero, e64, m4, tu, mu
-vsoxseg5ei64.v  v8, (a0), v16
 
 vsetvli zero, zero, e8, mf8, tu, mu
 vsoxseg6ei8.v v8, (a0), v16
@@ -3165,46 +1549,18 @@ vsetvli zero, zero, e8, mf2, tu, mu
 vsoxseg6ei8.v v8, (a0), v16
 vsetvli zero, zero, e8, m1, tu, mu
 vsoxseg6ei8.v v8, (a0), v16
-vsetvli zero, zero, e8, m2, tu, mu
-vsoxseg6ei8.v v8, (a0), v16
-vsetvli zero, zero, e8, m4, tu, mu
-vsoxseg6ei8.v v8, (a0), v16
-vsetvli zero, zero, e16, mf8, tu, mu
-vsoxseg6ei16.v  v8, (a0), v16
 vsetvli zero, zero, e16, mf4, tu, mu
 vsoxseg6ei16.v  v8, (a0), v16
 vsetvli zero, zero, e16, mf2, tu, mu
 vsoxseg6ei16.v  v8, (a0), v16
 vsetvli zero, zero, e16, m1, tu, mu
 vsoxseg6ei16.v  v8, (a0), v16
-vsetvli zero, zero, e16, m2, tu, mu
-vsoxseg6ei16.v  v8, (a0), v16
-vsetvli zero, zero, e16, m4, tu, mu
-vsoxseg6ei16.v  v8, (a0), v16
-vsetvli zero, zero, e32, mf8, tu, mu
-vsoxseg6ei32.v  v8, (a0), v16
-vsetvli zero, zero, e32, mf4, tu, mu
-vsoxseg6ei32.v  v8, (a0), v16
 vsetvli zero, zero, e32, mf2, tu, mu
 vsoxseg6ei32.v  v8, (a0), v16
 vsetvli zero, zero, e32, m1, tu, mu
 vsoxseg6ei32.v  v8, (a0), v16
-vsetvli zero, zero, e32, m2, tu, mu
-vsoxseg6ei32.v  v8, (a0), v16
-vsetvli zero, zero, e32, m4, tu, mu
-vsoxseg6ei32.v  v8, (a0), v16
-vsetvli zero, zero, e64, mf8, tu, mu
-vsoxseg6ei64.v  v8, (a0), v16
-vsetvli zero, zero, e64, mf4, tu, mu
-vsoxseg6ei64.v  v8, (a0), v16
-vsetvli zero, zero, e64, mf2, tu, mu
-vsoxseg6ei64.v  v8, (a0), v16
 vsetvli zero, zero, e64, m1, tu, mu
 vsoxseg6ei64.v  v8, (a0), v16
-vsetvli zero, zero, e64, m2, tu, mu
-vsoxseg6ei64.v  v8, (a0), v16
-vsetvli zero, zero, e64, m4, tu, mu
-vsoxseg6ei64.v  v8, (a0), v16
 
 vsetvli zero, zero, e8, mf8, tu, mu
 vsoxseg7ei8.v v8, (a0), v16
@@ -3214,46 +1570,18 @@ vsetvli zero, zero, e8, mf2, tu, mu
 vsoxseg7ei8.v v8, (a0), v16
 vsetvli zero, zero, e8, m1, tu, mu
 vsoxseg7ei8.v v8, (a0), v16
-vsetvli zero, zero, e8, m2, tu, mu
-vsoxseg7ei8.v v8, (a0), v16
-vsetvli zero, zero, e8, m4, tu, mu
-vsoxseg7ei8.v v8, (a0), v16
-vsetvli zero, zero, e16, mf8, tu, mu
-vsoxseg7ei16.v  v8, (a0), v16
 vsetvli zero, zero, e16, mf4, tu, mu
 vsoxseg7ei16.v  v8, (a0), v16
 vsetvli zero, zero, e16, mf2, tu, mu
 vsoxseg7ei16.v  v8, (a0), v16
 vsetvli zero, zero, e16, m1, tu, mu
 vsoxseg7ei16.v  v8, (a0), v16
-vsetvli zero, zero, e16, m2, tu, mu
-vsoxseg7ei16.v  v8, (a0), v16
-vsetvli zero, zero, e16, m4, tu, mu
-vsoxseg7ei16.v  v8, (a0), v16
-vsetvli zero, zero, e32, mf8, tu, mu
-vsoxseg7ei32.v  v8, (a0), v16
-vsetvli zero, zero, e32, mf4, tu, mu
-vsoxseg7ei32.v  v8, (a0), v16
 vsetvli zero, zero, e32, mf2, tu, mu
 vsoxseg7ei32.v  v8, (a0), v16
 vsetvli zero, zero, e32, m1, tu, mu
 vsoxseg7ei32.v  v8, (a0), v16
-vsetvli zero, zero, e32, m2, tu, mu
-vsoxseg7ei32.v  v8, (a0), v16
-vsetvli zero, zero, e32, m4, tu, mu
-vsoxseg7ei32.v  v8, (a0), v16
-vsetvli zero, zero, e64, mf8, tu, mu
-vsoxseg7ei64.v  v8, (a0), v16
-vsetvli zero, zero, e64, mf4, tu, mu
-vsoxseg7ei64.v  v8, (a0), v16
-vsetvli zero, zero, e64, mf2, tu, mu
-vsoxseg7ei64.v  v8, (a0), v16
 vsetvli zero, zero, e64, m1, tu, mu
 vsoxseg7ei64.v  v8, (a0), v16
-vsetvli zero, zero, e64, m2, tu, mu
-vsoxseg7ei64.v  v8, (a0), v16
-vsetvli zero, zero, e64, m4, tu, mu
-vsoxseg7ei64.v  v8, (a0), v16
 
 vsetvli zero, zero, e8, mf8, tu, mu
 vsoxseg8ei8.v v8, (a0), v16
@@ -3263,56 +1591,28 @@ vsetvli zero, zero, e8, mf2, tu, mu
 vsoxseg8ei8.v v8, (a0), v16
 vsetvli zero, zero, e8, m1, tu, mu
 vsoxseg8ei8.v v8, (a0), v16
-vsetvli zero, zero, e8, m2, tu, mu
-vsoxseg8ei8.v v8, (a0), v16
-vsetvli zero, zero, e8, m4, tu, mu
-vsoxseg8ei8.v v8, (a0), v16
-vsetvli zero, zero, e16, mf8, tu, mu
-vsoxseg8ei16.v  v8, (a0), v16
 vsetvli zero, zero, e16, mf4, tu, mu
 vsoxseg8ei16.v  v8, (a0), v16
 vsetvli zero, zero, e16, mf2, tu, mu
 vsoxseg8ei16.v  v8, (a0), v16
 vsetvli zero, zero, e16, m1, tu, mu
 vsoxseg8ei16.v  v8, (a0), v16
-vsetvli zero, zero, e16, m2, tu, mu
-vsoxseg8ei16.v  v8, (a0), v16
-vsetvli zero, zero, e16, m4, tu, mu
-vsoxseg8ei16.v  v8, (a0), v16
-vsetvli zero, zero, e32, mf8, tu, mu
-vsoxseg8ei32.v  v8, (a0), v16
-vsetvli zero, zero, e32, mf4, tu, mu
-vsoxseg8ei32.v  v8, (a0), v16
 vsetvli zero, zero, e32, mf2, tu, mu
 vsoxseg8ei32.v  v8, (a0), v16
 vsetvli zero, zero, e32, m1, tu, mu
 vsoxseg8ei32.v  v8, (a0), v16
-vsetvli zero, zero, e32, m2, tu, mu
-vsoxseg8ei32.v  v8, (a0), v16
-vsetvli zero, zero, e32, m4, tu, mu
-vsoxseg8ei32.v  v8, (a0), v16
-vsetvli zero, zero, e64, mf8, tu, mu
-vsoxseg8ei64.v  v8, (a0), v16
-vsetvli zero, zero, e64, mf4, tu, mu
-vsoxseg8ei64.v  v8, (a0), v16
-vsetvli zero, zero, e64, mf2, tu, mu
-vsoxseg8ei64.v  v8, (a0), v16
 vsetvli zero, zero, e64, m1, tu, mu
 vsoxseg8ei64.v  v8, (a0), v16
-vsetvli zero, zero, e64, m2, tu, mu
-vsoxseg8ei64.v  v8, (a0), v16
-vsetvli zero, zero, e64, m4, tu, mu
-vsoxseg8ei64.v  v8, (a0), v16
 
 # CHECK:      Iterations:        1
-# CHECK-NEXT: Instructions:      3240
-# CHECK-NEXT: Total Cycles:      217825
-# CHECK-NEXT: Total uOps:        3240
+# CHECK-NEXT: Instructions:      1540
+# CHECK-NEXT: Total Cycles:      29967
+# CHECK-NEXT: Total uOps:        1540
 
 # CHECK:      Dispatch Width:    3
-# CHECK-NEXT: uOps Per Cycle:    0.01
-# CHECK-NEXT: IPC:               0.01
-# CHECK-NEXT: Block RThroughput: 124596.0
+# CHECK-NEXT: uOps Per Cycle:    0.05
+# CHECK-NEXT: IPC:               0.05
+# CHECK-NEXT: Block RThroughput: 17896.0
 
 # CHECK:      Instruction Info:
 # CHECK-NEXT: [1]: #uOps
@@ -3323,3246 +1623,1546 @@ vsoxseg8ei64.v  v8, (a0), v16
 # CHECK-NEXT: [6]: HasSideEffects (U)
 
 # CHECK:      [1]    [2]    [3]    [4]    [5]    [6]    Instructions:
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf8, tu, mu
-# CHECK-NEXT:  1      16    16.00   *                   vlseg2e8.v  v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf4, tu, mu
-# CHECK-NEXT:  1      20    20.00   *                   vlseg2e8.v  v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf2, tu, mu
-# CHECK-NEXT:  1      28    28.00   *                   vlseg2e8.v  v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m1, tu, mu
-# CHECK-NEXT:  1      44    44.00   *                   vlseg2e8.v  v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m2, tu, mu
-# CHECK-NEXT:  1      76    76.00   *                   vlseg2e8.v  v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m4, tu, mu
-# CHECK-NEXT:  1      140   140.00  *                   vlseg2e8.v  v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m8, tu, mu
-# CHECK-NEXT:  1      268   268.00  *                   vlseg2e8.v  v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf8, tu, mu
-# CHECK-NEXT:  1      140   140.00  *                   vlseg2e16.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf4, tu, mu
-# CHECK-NEXT:  1      16    16.00   *                   vlseg2e16.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf2, tu, mu
-# CHECK-NEXT:  1      20    20.00   *                   vlseg2e16.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m1, tu, mu
-# CHECK-NEXT:  1      28    28.00   *                   vlseg2e16.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m2, tu, mu
-# CHECK-NEXT:  1      44    44.00   *                   vlseg2e16.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m4, tu, mu
-# CHECK-NEXT:  1      76    76.00   *                   vlseg2e16.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m8, tu, mu
-# CHECK-NEXT:  1      140   140.00  *                   vlseg2e16.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf8, tu, mu
-# CHECK-NEXT:  1      76    76.00   *                   vlseg2e32.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf4, tu, mu
-# CHECK-NEXT:  1      76    76.00   *                   vlseg2e32.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf2, tu, mu
-# CHECK-NEXT:  1      16    16.00   *                   vlseg2e32.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m1, tu, mu
-# CHECK-NEXT:  1      20    20.00   *                   vlseg2e32.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m2, tu, mu
-# CHECK-NEXT:  1      28    28.00   *                   vlseg2e32.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m4, tu, mu
-# CHECK-NEXT:  1      44    44.00   *                   vlseg2e32.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m8, tu, mu
-# CHECK-NEXT:  1      76    76.00   *                   vlseg2e32.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf8, tu, mu
-# CHECK-NEXT:  1      44    44.00   *                   vlseg2e64.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf4, tu, mu
-# CHECK-NEXT:  1      44    44.00   *                   vlseg2e64.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf2, tu, mu
-# CHECK-NEXT:  1      44    44.00   *                   vlseg2e64.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m1, tu, mu
-# CHECK-NEXT:  1      16    16.00   *                   vlseg2e64.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m2, tu, mu
-# CHECK-NEXT:  1      20    20.00   *                   vlseg2e64.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m4, tu, mu
-# CHECK-NEXT:  1      28    28.00   *                   vlseg2e64.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m8, tu, mu
-# CHECK-NEXT:  1      44    44.00   *                   vlseg2e64.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf8, tu, mu
-# CHECK-NEXT:  1      18    18.00   *                   vlseg3e8.v  v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf4, tu, mu
-# CHECK-NEXT:  1      24    24.00   *                   vlseg3e8.v  v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf2, tu, mu
-# CHECK-NEXT:  1      36    36.00   *                   vlseg3e8.v  v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m1, tu, mu
-# CHECK-NEXT:  1      60    60.00   *                   vlseg3e8.v  v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m2, tu, mu
-# CHECK-NEXT:  1      108   108.00  *                   vlseg3e8.v  v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m4, tu, mu
-# CHECK-NEXT:  1      396   396.00  *                   vlseg3e8.v  v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m8, tu, mu
-# CHECK-NEXT:  1      396   396.00  *                   vlseg3e8.v  v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf8, tu, mu
-# CHECK-NEXT:  1      204   204.00  *                   vlseg3e16.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf4, tu, mu
-# CHECK-NEXT:  1      18    18.00   *                   vlseg3e16.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf2, tu, mu
-# CHECK-NEXT:  1      24    24.00   *                   vlseg3e16.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m1, tu, mu
-# CHECK-NEXT:  1      36    36.00   *                   vlseg3e16.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m2, tu, mu
-# CHECK-NEXT:  1      60    60.00   *                   vlseg3e16.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m4, tu, mu
-# CHECK-NEXT:  1      204   204.00  *                   vlseg3e16.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m8, tu, mu
-# CHECK-NEXT:  1      204   204.00  *                   vlseg3e16.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf8, tu, mu
-# CHECK-NEXT:  1      108   108.00  *                   vlseg3e32.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf4, tu, mu
-# CHECK-NEXT:  1      108   108.00  *                   vlseg3e32.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf2, tu, mu
-# CHECK-NEXT:  1      18    18.00   *                   vlseg3e32.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m1, tu, mu
-# CHECK-NEXT:  1      24    24.00   *                   vlseg3e32.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m2, tu, mu
-# CHECK-NEXT:  1      36    36.00   *                   vlseg3e32.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m4, tu, mu
-# CHECK-NEXT:  1      108   108.00  *                   vlseg3e32.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m8, tu, mu
-# CHECK-NEXT:  1      108   108.00  *                   vlseg3e32.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf8, tu, mu
-# CHECK-NEXT:  1      60    60.00   *                   vlseg3e64.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf4, tu, mu
-# CHECK-NEXT:  1      60    60.00   *                   vlseg3e64.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf2, tu, mu
-# CHECK-NEXT:  1      60    60.00   *                   vlseg3e64.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m1, tu, mu
-# CHECK-NEXT:  1      18    18.00   *                   vlseg3e64.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m2, tu, mu
-# CHECK-NEXT:  1      24    24.00   *                   vlseg3e64.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m4, tu, mu
-# CHECK-NEXT:  1      60    60.00   *                   vlseg3e64.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m8, tu, mu
-# CHECK-NEXT:  1      60    60.00   *                   vlseg3e64.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf8, tu, mu
-# CHECK-NEXT:  1      20    20.00   *                   vlseg4e8.v  v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf4, tu, mu
-# CHECK-NEXT:  1      28    28.00   *                   vlseg4e8.v  v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf2, tu, mu
-# CHECK-NEXT:  1      44    44.00   *                   vlseg4e8.v  v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m1, tu, mu
-# CHECK-NEXT:  1      76    76.00   *                   vlseg4e8.v  v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m2, tu, mu
-# CHECK-NEXT:  1      140   140.00  *                   vlseg4e8.v  v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m4, tu, mu
-# CHECK-NEXT:  1      524   524.00  *                   vlseg4e8.v  v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m8, tu, mu
-# CHECK-NEXT:  1      524   524.00  *                   vlseg4e8.v  v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf8, tu, mu
-# CHECK-NEXT:  1      268   268.00  *                   vlseg4e16.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf4, tu, mu
-# CHECK-NEXT:  1      20    20.00   *                   vlseg4e16.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf2, tu, mu
-# CHECK-NEXT:  1      28    28.00   *                   vlseg4e16.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m1, tu, mu
-# CHECK-NEXT:  1      44    44.00   *                   vlseg4e16.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m2, tu, mu
-# CHECK-NEXT:  1      76    76.00   *                   vlseg4e16.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m4, tu, mu
-# CHECK-NEXT:  1      268   268.00  *                   vlseg4e16.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m8, tu, mu
-# CHECK-NEXT:  1      268   268.00  *                   vlseg4e16.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf8, tu, mu
-# CHECK-NEXT:  1      140   140.00  *                   vlseg4e32.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf4, tu, mu
-# CHECK-NEXT:  1      140   140.00  *                   vlseg4e32.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf2, tu, mu
-# CHECK-NEXT:  1      20    20.00   *                   vlseg4e32.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m1, tu, mu
-# CHECK-NEXT:  1      28    28.00   *                   vlseg4e32.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m2, tu, mu
-# CHECK-NEXT:  1      44    44.00   *                   vlseg4e32.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m4, tu, mu
-# CHECK-NEXT:  1      140   140.00  *                   vlseg4e32.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m8, tu, mu
-# CHECK-NEXT:  1      140   140.00  *                   vlseg4e32.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf8, tu, mu
-# CHECK-NEXT:  1      76    76.00   *                   vlseg4e64.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf4, tu, mu
-# CHECK-NEXT:  1      76    76.00   *                   vlseg4e64.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf2, tu, mu
-# CHECK-NEXT:  1      76    76.00   *                   vlseg4e64.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m1, tu, mu
-# CHECK-NEXT:  1      20    20.00   *                   vlseg4e64.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m2, tu, mu
-# CHECK-NEXT:  1      28    28.00   *                   vlseg4e64.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m4, tu, mu
-# CHECK-NEXT:  1      76    76.00   *                   vlseg4e64.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m8, tu, mu
-# CHECK-NEXT:  1      76    76.00   *                   vlseg4e64.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf8, tu, mu
-# CHECK-NEXT:  1      22    22.00   *                   vlseg5e8.v  v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf4, tu, mu
-# CHECK-NEXT:  1      32    32.00   *                   vlseg5e8.v  v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf2, tu, mu
-# CHECK-NEXT:  1      52    52.00   *                   vlseg5e8.v  v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m1, tu, mu
-# CHECK-NEXT:  1      92    92.00   *                   vlseg5e8.v  v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m2, tu, mu
-# CHECK-NEXT:  1      652   652.00  *                   vlseg5e8.v  v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m4, tu, mu
-# CHECK-NEXT:  1      652   652.00  *                   vlseg5e8.v  v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf8, tu, mu
-# CHECK-NEXT:  1      332   332.00  *                   vlseg5e16.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf4, tu, mu
-# CHECK-NEXT:  1      22    22.00   *                   vlseg5e16.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf2, tu, mu
-# CHECK-NEXT:  1      32    32.00   *                   vlseg5e16.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m1, tu, mu
-# CHECK-NEXT:  1      52    52.00   *                   vlseg5e16.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m2, tu, mu
-# CHECK-NEXT:  1      332   332.00  *                   vlseg5e16.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m4, tu, mu
-# CHECK-NEXT:  1      332   332.00  *                   vlseg5e16.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf8, tu, mu
-# CHECK-NEXT:  1      172   172.00  *                   vlseg5e32.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf4, tu, mu
-# CHECK-NEXT:  1      172   172.00  *                   vlseg5e32.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf2, tu, mu
-# CHECK-NEXT:  1      22    22.00   *                   vlseg5e32.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m1, tu, mu
-# CHECK-NEXT:  1      32    32.00   *                   vlseg5e32.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m2, tu, mu
-# CHECK-NEXT:  1      172   172.00  *                   vlseg5e32.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m4, tu, mu
-# CHECK-NEXT:  1      172   172.00  *                   vlseg5e32.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf8, tu, mu
-# CHECK-NEXT:  1      92    92.00   *                   vlseg5e64.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf4, tu, mu
-# CHECK-NEXT:  1      92    92.00   *                   vlseg5e64.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf2, tu, mu
-# CHECK-NEXT:  1      92    92.00   *                   vlseg5e64.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m1, tu, mu
-# CHECK-NEXT:  1      22    22.00   *                   vlseg5e64.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m2, tu, mu
-# CHECK-NEXT:  1      92    92.00   *                   vlseg5e64.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m4, tu, mu
-# CHECK-NEXT:  1      92    92.00   *                   vlseg5e64.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf8, tu, mu
-# CHECK-NEXT:  1      24    24.00   *                   vlseg6e8.v  v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf4, tu, mu
-# CHECK-NEXT:  1      36    36.00   *                   vlseg6e8.v  v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf2, tu, mu
-# CHECK-NEXT:  1      60    60.00   *                   vlseg6e8.v  v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m1, tu, mu
-# CHECK-NEXT:  1      108   108.00  *                   vlseg6e8.v  v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m2, tu, mu
-# CHECK-NEXT:  1      780   780.00  *                   vlseg6e8.v  v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m4, tu, mu
-# CHECK-NEXT:  1      780   780.00  *                   vlseg6e8.v  v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf8, tu, mu
-# CHECK-NEXT:  1      396   396.00  *                   vlseg6e16.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf4, tu, mu
-# CHECK-NEXT:  1      24    24.00   *                   vlseg6e16.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf2, tu, mu
-# CHECK-NEXT:  1      36    36.00   *                   vlseg6e16.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m1, tu, mu
-# CHECK-NEXT:  1      60    60.00   *                   vlseg6e16.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m2, tu, mu
-# CHECK-NEXT:  1      396   396.00  *                   vlseg6e16.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m4, tu, mu
-# CHECK-NEXT:  1      396   396.00  *                   vlseg6e16.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf8, tu, mu
-# CHECK-NEXT:  1      204   204.00  *                   vlseg6e32.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf4, tu, mu
-# CHECK-NEXT:  1      204   204.00  *                   vlseg6e32.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf2, tu, mu
-# CHECK-NEXT:  1      24    24.00   *                   vlseg6e32.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m1, tu, mu
-# CHECK-NEXT:  1      36    36.00   *                   vlseg6e32.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m2, tu, mu
-# CHECK-NEXT:  1      204   204.00  *                   vlseg6e32.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m4, tu, mu
-# CHECK-NEXT:  1      204   204.00  *                   vlseg6e32.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf8, tu, mu
-# CHECK-NEXT:  1      108   108.00  *                   vlseg6e64.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf4, tu, mu
-# CHECK-NEXT:  1      108   108.00  *                   vlseg6e64.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf2, tu, mu
-# CHECK-NEXT:  1      108   108.00  *                   vlseg6e64.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m1, tu, mu
-# CHECK-NEXT:  1      24    24.00   *                   vlseg6e64.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m2, tu, mu
-# CHECK-NEXT:  1      108   108.00  *                   vlseg6e64.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m4, tu, mu
-# CHECK-NEXT:  1      108   108.00  *                   vlseg6e64.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf8, tu, mu
-# CHECK-NEXT:  1      26    26.00   *                   vlseg7e8.v  v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf4, tu, mu
-# CHECK-NEXT:  1      40    40.00   *                   vlseg7e8.v  v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf2, tu, mu
-# CHECK-NEXT:  1      68    68.00   *                   vlseg7e8.v  v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m1, tu, mu
-# CHECK-NEXT:  1      124   124.00  *                   vlseg7e8.v  v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m2, tu, mu
-# CHECK-NEXT:  1      908   908.00  *                   vlseg7e8.v  v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m4, tu, mu
-# CHECK-NEXT:  1      908   908.00  *                   vlseg7e8.v  v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf8, tu, mu
-# CHECK-NEXT:  1      460   460.00  *                   vlseg7e16.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf4, tu, mu
-# CHECK-NEXT:  1      26    26.00   *                   vlseg7e16.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf2, tu, mu
-# CHECK-NEXT:  1      40    40.00   *                   vlseg7e16.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m1, tu, mu
-# CHECK-NEXT:  1      68    68.00   *                   vlseg7e16.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m2, tu, mu
-# CHECK-NEXT:  1      460   460.00  *                   vlseg7e16.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m4, tu, mu
-# CHECK-NEXT:  1      460   460.00  *                   vlseg7e16.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf8, tu, mu
-# CHECK-NEXT:  1      236   236.00  *                   vlseg7e32.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf4, tu, mu
-# CHECK-NEXT:  1      236   236.00  *                   vlseg7e32.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf2, tu, mu
-# CHECK-NEXT:  1      26    26.00   *                   vlseg7e32.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m1, tu, mu
-# CHECK-NEXT:  1      40    40.00   *                   vlseg7e32.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m2, tu, mu
-# CHECK-NEXT:  1      236   236.00  *                   vlseg7e32.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m4, tu, mu
-# CHECK-NEXT:  1      236   236.00  *                   vlseg7e32.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf8, tu, mu
-# CHECK-NEXT:  1      124   124.00  *                   vlseg7e64.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf4, tu, mu
-# CHECK-NEXT:  1      124   124.00  *                   vlseg7e64.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf2, tu, mu
-# CHECK-NEXT:  1      124   124.00  *                   vlseg7e64.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m1, tu, mu
-# CHECK-NEXT:  1      26    26.00   *                   vlseg7e64.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m2, tu, mu
-# CHECK-NEXT:  1      124   124.00  *                   vlseg7e64.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m4, tu, mu
-# CHECK-NEXT:  1      124   124.00  *                   vlseg7e64.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf8, tu, mu
-# CHECK-NEXT:  1      28    28.00   *                   vlseg8e8.v  v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf4, tu, mu
-# CHECK-NEXT:  1      44    44.00   *                   vlseg8e8.v  v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf2, tu, mu
-# CHECK-NEXT:  1      76    76.00   *                   vlseg8e8.v  v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m1, tu, mu
-# CHECK-NEXT:  1      140   140.00  *                   vlseg8e8.v  v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m2, tu, mu
-# CHECK-NEXT:  1      1036   1036.00  *                   vlseg8e8.v  v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m4, tu, mu
-# CHECK-NEXT:  1      1036   1036.00  *                   vlseg8e8.v  v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf8, tu, mu
-# CHECK-NEXT:  1      524   524.00  *                   vlseg8e16.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf4, tu, mu
-# CHECK-NEXT:  1      28    28.00   *                   vlseg8e16.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf2, tu, mu
-# CHECK-NEXT:  1      44    44.00   *                   vlseg8e16.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m1, tu, mu
-# CHECK-NEXT:  1      76    76.00   *                   vlseg8e16.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m2, tu, mu
-# CHECK-NEXT:  1      524   524.00  *                   vlseg8e16.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m4, tu, mu
-# CHECK-NEXT:  1      524   524.00  *                   vlseg8e16.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf8, tu, mu
-# CHECK-NEXT:  1      268   268.00  *                   vlseg8e32.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf4, tu, mu
-# CHECK-NEXT:  1      268   268.00  *                   vlseg8e32.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf2, tu, mu
-# CHECK-NEXT:  1      28    28.00   *                   vlseg8e32.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m1, tu, mu
-# CHECK-NEXT:  1      44    44.00   *                   vlseg8e32.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m2, tu, mu
-# CHECK-NEXT:  1      268   268.00  *                   vlseg8e32.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m4, tu, mu
-# CHECK-NEXT:  1      268   268.00  *                   vlseg8e32.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf8, tu, mu
-# CHECK-NEXT:  1      140   140.00  *                   vlseg8e64.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf4, tu, mu
-# CHECK-NEXT:  1      140   140.00  *                   vlseg8e64.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf2, tu, mu
-# CHECK-NEXT:  1      140   140.00  *                   vlseg8e64.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m1, tu, mu
-# CHECK-NEXT:  1      28    28.00   *                   vlseg8e64.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m2, tu, mu
-# CHECK-NEXT:  1      140   140.00  *                   vlseg8e64.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m4, tu, mu
-# CHECK-NEXT:  1      140   140.00  *                   vlseg8e64.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf8, tu, mu
-# CHECK-NEXT:  1      5     16.00          *            vsseg2e8.v  v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf4, tu, mu
-# CHECK-NEXT:  1      9     20.00          *            vsseg2e8.v  v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf2, tu, mu
-# CHECK-NEXT:  1      17    28.00          *            vsseg2e8.v  v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m1, tu, mu
-# CHECK-NEXT:  1      33    44.00          *            vsseg2e8.v  v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m2, tu, mu
-# CHECK-NEXT:  1      65    76.00          *            vsseg2e8.v  v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m4, tu, mu
-# CHECK-NEXT:  1      129   140.00         *            vsseg2e8.v  v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m8, tu, mu
-# CHECK-NEXT:  1      257   268.00         *            vsseg2e8.v  v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf8, tu, mu
-# CHECK-NEXT:  1      129   140.00         *            vsseg2e16.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf4, tu, mu
-# CHECK-NEXT:  1      5     16.00          *            vsseg2e16.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf2, tu, mu
-# CHECK-NEXT:  1      9     20.00          *            vsseg2e16.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m1, tu, mu
-# CHECK-NEXT:  1      17    28.00          *            vsseg2e16.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m2, tu, mu
-# CHECK-NEXT:  1      33    44.00          *            vsseg2e16.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m4, tu, mu
-# CHECK-NEXT:  1      65    76.00          *            vsseg2e16.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m8, tu, mu
-# CHECK-NEXT:  1      129   140.00         *            vsseg2e16.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf8, tu, mu
-# CHECK-NEXT:  1      65    76.00          *            vsseg2e32.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf4, tu, mu
-# CHECK-NEXT:  1      65    76.00          *            vsseg2e32.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf2, tu, mu
-# CHECK-NEXT:  1      5     16.00          *            vsseg2e32.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m1, tu, mu
-# CHECK-NEXT:  1      9     20.00          *            vsseg2e32.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m2, tu, mu
-# CHECK-NEXT:  1      17    28.00          *            vsseg2e32.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m4, tu, mu
-# CHECK-NEXT:  1      33    44.00          *            vsseg2e32.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m8, tu, mu
-# CHECK-NEXT:  1      65    76.00          *            vsseg2e32.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf8, tu, mu
-# CHECK-NEXT:  1      33    44.00          *            vsseg2e64.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf4, tu, mu
-# CHECK-NEXT:  1      33    44.00          *            vsseg2e64.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf2, tu, mu
-# CHECK-NEXT:  1      33    44.00          *            vsseg2e64.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m1, tu, mu
-# CHECK-NEXT:  1      5     16.00          *            vsseg2e64.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m2, tu, mu
-# CHECK-NEXT:  1      9     20.00          *            vsseg2e64.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m4, tu, mu
-# CHECK-NEXT:  1      17    28.00          *            vsseg2e64.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m8, tu, mu
-# CHECK-NEXT:  1      33    44.00          *            vsseg2e64.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf8, tu, mu
-# CHECK-NEXT:  1      7     18.00          *            vsseg3e8.v  v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf4, tu, mu
-# CHECK-NEXT:  1      13    24.00          *            vsseg3e8.v  v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf2, tu, mu
-# CHECK-NEXT:  1      25    36.00          *            vsseg3e8.v  v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m1, tu, mu
-# CHECK-NEXT:  1      49    60.00          *            vsseg3e8.v  v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m2, tu, mu
-# CHECK-NEXT:  1      97    108.00         *            vsseg3e8.v  v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m4, tu, mu
-# CHECK-NEXT:  1      385   396.00         *            vsseg3e8.v  v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m8, tu, mu
-# CHECK-NEXT:  1      385   396.00         *            vsseg3e8.v  v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf8, tu, mu
-# CHECK-NEXT:  1      193   204.00         *            vsseg3e16.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf4, tu, mu
-# CHECK-NEXT:  1      7     18.00          *            vsseg3e16.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf2, tu, mu
-# CHECK-NEXT:  1      13    24.00          *            vsseg3e16.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m1, tu, mu
-# CHECK-NEXT:  1      25    36.00          *            vsseg3e16.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m2, tu, mu
-# CHECK-NEXT:  1      49    60.00          *            vsseg3e16.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m4, tu, mu
-# CHECK-NEXT:  1      193   204.00         *            vsseg3e16.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m8, tu, mu
-# CHECK-NEXT:  1      193   204.00         *            vsseg3e16.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf8, tu, mu
-# CHECK-NEXT:  1      97    108.00         *            vsseg3e32.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf4, tu, mu
-# CHECK-NEXT:  1      97    108.00         *            vsseg3e32.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf2, tu, mu
-# CHECK-NEXT:  1      7     18.00          *            vsseg3e32.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m1, tu, mu
-# CHECK-NEXT:  1      13    24.00          *            vsseg3e32.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m2, tu, mu
-# CHECK-NEXT:  1      25    36.00          *            vsseg3e32.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m4, tu, mu
-# CHECK-NEXT:  1      97    108.00         *            vsseg3e32.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m8, tu, mu
-# CHECK-NEXT:  1      97    108.00         *            vsseg3e32.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf8, tu, mu
-# CHECK-NEXT:  1      49    60.00          *            vsseg3e64.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf4, tu, mu
-# CHECK-NEXT:  1      49    60.00          *            vsseg3e64.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf2, tu, mu
-# CHECK-NEXT:  1      49    60.00          *            vsseg3e64.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m1, tu, mu
-# CHECK-NEXT:  1      7     18.00          *            vsseg3e64.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m2, tu, mu
-# CHECK-NEXT:  1      13    24.00          *            vsseg3e64.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m4, tu, mu
-# CHECK-NEXT:  1      49    60.00          *            vsseg3e64.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m8, tu, mu
-# CHECK-NEXT:  1      49    60.00          *            vsseg3e64.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf8, tu, mu
-# CHECK-NEXT:  1      9     20.00          *            vsseg4e8.v  v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf4, tu, mu
-# CHECK-NEXT:  1      17    28.00          *            vsseg4e8.v  v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf2, tu, mu
-# CHECK-NEXT:  1      33    44.00          *            vsseg4e8.v  v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m1, tu, mu
-# CHECK-NEXT:  1      65    76.00          *            vsseg4e8.v  v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m2, tu, mu
-# CHECK-NEXT:  1      129   140.00         *            vsseg4e8.v  v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m4, tu, mu
-# CHECK-NEXT:  1      513   524.00         *            vsseg4e8.v  v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m8, tu, mu
-# CHECK-NEXT:  1      513   524.00         *            vsseg4e8.v  v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf8, tu, mu
-# CHECK-NEXT:  1      257   268.00         *            vsseg4e16.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf4, tu, mu
-# CHECK-NEXT:  1      9     20.00          *            vsseg4e16.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf2, tu, mu
-# CHECK-NEXT:  1      17    28.00          *            vsseg4e16.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m1, tu, mu
-# CHECK-NEXT:  1      33    44.00          *            vsseg4e16.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m2, tu, mu
-# CHECK-NEXT:  1      65    76.00          *            vsseg4e16.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m4, tu, mu
-# CHECK-NEXT:  1      257   268.00         *            vsseg4e16.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m8, tu, mu
-# CHECK-NEXT:  1      257   268.00         *            vsseg4e16.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf8, tu, mu
-# CHECK-NEXT:  1      129   140.00         *            vsseg4e32.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf4, tu, mu
-# CHECK-NEXT:  1      129   140.00         *            vsseg4e32.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf2, tu, mu
-# CHECK-NEXT:  1      9     20.00          *            vsseg4e32.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m1, tu, mu
-# CHECK-NEXT:  1      17    28.00          *            vsseg4e32.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m2, tu, mu
-# CHECK-NEXT:  1      33    44.00          *            vsseg4e32.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m4, tu, mu
-# CHECK-NEXT:  1      129   140.00         *            vsseg4e32.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m8, tu, mu
-# CHECK-NEXT:  1      129   140.00         *            vsseg4e32.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf8, tu, mu
-# CHECK-NEXT:  1      65    76.00          *            vsseg4e64.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf4, tu, mu
-# CHECK-NEXT:  1      65    76.00          *            vsseg4e64.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf2, tu, mu
-# CHECK-NEXT:  1      65    76.00          *            vsseg4e64.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m1, tu, mu
-# CHECK-NEXT:  1      9     20.00          *            vsseg4e64.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m2, tu, mu
-# CHECK-NEXT:  1      17    28.00          *            vsseg4e64.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m4, tu, mu
-# CHECK-NEXT:  1      65    76.00          *            vsseg4e64.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m8, tu, mu
-# CHECK-NEXT:  1      65    76.00          *            vsseg4e64.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf8, tu, mu
-# CHECK-NEXT:  1      11    22.00          *            vsseg5e8.v  v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf4, tu, mu
-# CHECK-NEXT:  1      21    32.00          *            vsseg5e8.v  v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf2, tu, mu
-# CHECK-NEXT:  1      41    52.00          *            vsseg5e8.v  v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m1, tu, mu
-# CHECK-NEXT:  1      81    92.00          *            vsseg5e8.v  v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m2, tu, mu
-# CHECK-NEXT:  1      641   652.00         *            vsseg5e8.v  v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m4, tu, mu
-# CHECK-NEXT:  1      641   652.00         *            vsseg5e8.v  v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf8, tu, mu
-# CHECK-NEXT:  1      321   332.00         *            vsseg5e16.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf4, tu, mu
-# CHECK-NEXT:  1      11    22.00          *            vsseg5e16.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf2, tu, mu
-# CHECK-NEXT:  1      21    32.00          *            vsseg5e16.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m1, tu, mu
-# CHECK-NEXT:  1      41    52.00          *            vsseg5e16.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m2, tu, mu
-# CHECK-NEXT:  1      321   332.00         *            vsseg5e16.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m4, tu, mu
-# CHECK-NEXT:  1      321   332.00         *            vsseg5e16.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf8, tu, mu
-# CHECK-NEXT:  1      161   172.00         *            vsseg5e32.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf4, tu, mu
-# CHECK-NEXT:  1      161   172.00         *            vsseg5e32.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf2, tu, mu
-# CHECK-NEXT:  1      11    22.00          *            vsseg5e32.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m1, tu, mu
-# CHECK-NEXT:  1      21    32.00          *            vsseg5e32.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m2, tu, mu
-# CHECK-NEXT:  1      161   172.00         *            vsseg5e32.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m4, tu, mu
-# CHECK-NEXT:  1      161   172.00         *            vsseg5e32.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf8, tu, mu
-# CHECK-NEXT:  1      81    92.00          *            vsseg5e64.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf4, tu, mu
-# CHECK-NEXT:  1      81    92.00          *            vsseg5e64.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf2, tu, mu
-# CHECK-NEXT:  1      81    92.00          *            vsseg5e64.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m1, tu, mu
-# CHECK-NEXT:  1      11    22.00          *            vsseg5e64.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m2, tu, mu
-# CHECK-NEXT:  1      81    92.00          *            vsseg5e64.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m4, tu, mu
-# CHECK-NEXT:  1      81    92.00          *            vsseg5e64.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf8, tu, mu
-# CHECK-NEXT:  1      13    24.00          *            vsseg6e8.v  v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf4, tu, mu
-# CHECK-NEXT:  1      25    36.00          *            vsseg6e8.v  v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf2, tu, mu
-# CHECK-NEXT:  1      49    60.00          *            vsseg6e8.v  v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m1, tu, mu
-# CHECK-NEXT:  1      97    108.00         *            vsseg6e8.v  v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m2, tu, mu
-# CHECK-NEXT:  1      769   780.00         *            vsseg6e8.v  v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m4, tu, mu
-# CHECK-NEXT:  1      769   780.00         *            vsseg6e8.v  v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf8, tu, mu
-# CHECK-NEXT:  1      385   396.00         *            vsseg6e16.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf4, tu, mu
-# CHECK-NEXT:  1      13    24.00          *            vsseg6e16.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf2, tu, mu
-# CHECK-NEXT:  1      25    36.00          *            vsseg6e16.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m1, tu, mu
-# CHECK-NEXT:  1      49    60.00          *            vsseg6e16.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m2, tu, mu
-# CHECK-NEXT:  1      385   396.00         *            vsseg6e16.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m4, tu, mu
-# CHECK-NEXT:  1      385   396.00         *            vsseg6e16.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf8, tu, mu
-# CHECK-NEXT:  1      193   204.00         *            vsseg6e32.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf4, tu, mu
-# CHECK-NEXT:  1      193   204.00         *            vsseg6e32.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf2, tu, mu
-# CHECK-NEXT:  1      13    24.00          *            vsseg6e32.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m1, tu, mu
-# CHECK-NEXT:  1      25    36.00          *            vsseg6e32.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m2, tu, mu
-# CHECK-NEXT:  1      193   204.00         *            vsseg6e32.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m4, tu, mu
-# CHECK-NEXT:  1      193   204.00         *            vsseg6e32.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf8, tu, mu
-# CHECK-NEXT:  1      97    108.00         *            vsseg6e64.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf4, tu, mu
-# CHECK-NEXT:  1      97    108.00         *            vsseg6e64.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf2, tu, mu
-# CHECK-NEXT:  1      97    108.00         *            vsseg6e64.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m1, tu, mu
-# CHECK-NEXT:  1      13    24.00          *            vsseg6e64.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m2, tu, mu
-# CHECK-NEXT:  1      97    108.00         *            vsseg6e64.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m4, tu, mu
-# CHECK-NEXT:  1      97    108.00         *            vsseg6e64.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf8, tu, mu
-# CHECK-NEXT:  1      15    26.00          *            vsseg7e8.v  v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf4, tu, mu
-# CHECK-NEXT:  1      29    40.00          *            vsseg7e8.v  v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf2, tu, mu
-# CHECK-NEXT:  1      57    68.00          *            vsseg7e8.v  v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m1, tu, mu
-# CHECK-NEXT:  1      113   124.00         *            vsseg7e8.v  v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m2, tu, mu
-# CHECK-NEXT:  1      897   908.00         *            vsseg7e8.v  v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m4, tu, mu
-# CHECK-NEXT:  1      897   908.00         *            vsseg7e8.v  v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf8, tu, mu
-# CHECK-NEXT:  1      449   460.00         *            vsseg7e16.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf4, tu, mu
-# CHECK-NEXT:  1      15    26.00          *            vsseg7e16.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf2, tu, mu
-# CHECK-NEXT:  1      29    40.00          *            vsseg7e16.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m1, tu, mu
-# CHECK-NEXT:  1      57    68.00          *            vsseg7e16.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m2, tu, mu
-# CHECK-NEXT:  1      449   460.00         *            vsseg7e16.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m4, tu, mu
-# CHECK-NEXT:  1      449   460.00         *            vsseg7e16.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf8, tu, mu
-# CHECK-NEXT:  1      225   236.00         *            vsseg7e32.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf4, tu, mu
-# CHECK-NEXT:  1      225   236.00         *            vsseg7e32.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf2, tu, mu
-# CHECK-NEXT:  1      15    26.00          *            vsseg7e32.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m1, tu, mu
-# CHECK-NEXT:  1      29    40.00          *            vsseg7e32.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m2, tu, mu
-# CHECK-NEXT:  1      225   236.00         *            vsseg7e32.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m4, tu, mu
-# CHECK-NEXT:  1      225   236.00         *            vsseg7e32.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf8, tu, mu
-# CHECK-NEXT:  1      113   124.00         *            vsseg7e64.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf4, tu, mu
-# CHECK-NEXT:  1      113   124.00         *            vsseg7e64.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf2, tu, mu
-# CHECK-NEXT:  1      113   124.00         *            vsseg7e64.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m1, tu, mu
-# CHECK-NEXT:  1      15    26.00          *            vsseg7e64.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m2, tu, mu
-# CHECK-NEXT:  1      113   124.00         *            vsseg7e64.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m4, tu, mu
-# CHECK-NEXT:  1      113   124.00         *            vsseg7e64.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf8, tu, mu
-# CHECK-NEXT:  1      17    28.00          *            vsseg8e8.v  v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf4, tu, mu
-# CHECK-NEXT:  1      33    44.00          *            vsseg8e8.v  v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf2, tu, mu
-# CHECK-NEXT:  1      65    76.00          *            vsseg8e8.v  v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m1, tu, mu
-# CHECK-NEXT:  1      129   140.00         *            vsseg8e8.v  v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m2, tu, mu
-# CHECK-NEXT:  1      1025   1036.00         *            vsseg8e8.v  v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m4, tu, mu
-# CHECK-NEXT:  1      1025   1036.00         *            vsseg8e8.v  v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf8, tu, mu
-# CHECK-NEXT:  1      513   524.00         *            vsseg8e16.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf4, tu, mu
-# CHECK-NEXT:  1      17    28.00          *            vsseg8e16.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf2, tu, mu
-# CHECK-NEXT:  1      33    44.00          *            vsseg8e16.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m1, tu, mu
-# CHECK-NEXT:  1      65    76.00          *            vsseg8e16.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m2, tu, mu
-# CHECK-NEXT:  1      513   524.00         *            vsseg8e16.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m4, tu, mu
-# CHECK-NEXT:  1      513   524.00         *            vsseg8e16.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf8, tu, mu
-# CHECK-NEXT:  1      257   268.00         *            vsseg8e32.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf4, tu, mu
-# CHECK-NEXT:  1      257   268.00         *            vsseg8e32.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf2, tu, mu
-# CHECK-NEXT:  1      17    28.00          *            vsseg8e32.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m1, tu, mu
-# CHECK-NEXT:  1      33    44.00          *            vsseg8e32.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m2, tu, mu
-# CHECK-NEXT:  1      257   268.00         *            vsseg8e32.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m4, tu, mu
-# CHECK-NEXT:  1      257   268.00         *            vsseg8e32.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf8, tu, mu
-# CHECK-NEXT:  1      129   140.00         *            vsseg8e64.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf4, tu, mu
-# CHECK-NEXT:  1      129   140.00         *            vsseg8e64.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf2, tu, mu
-# CHECK-NEXT:  1      129   140.00         *            vsseg8e64.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m1, tu, mu
-# CHECK-NEXT:  1      17    28.00          *            vsseg8e64.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m2, tu, mu
-# CHECK-NEXT:  1      129   140.00         *            vsseg8e64.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m4, tu, mu
-# CHECK-NEXT:  1      129   140.00         *            vsseg8e64.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf8, tu, mu
-# CHECK-NEXT:  1      16    16.00   *                   vlsseg2e8.v v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf4, tu, mu
-# CHECK-NEXT:  1      20    20.00   *                   vlsseg2e8.v v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf2, tu, mu
-# CHECK-NEXT:  1      28    28.00   *                   vlsseg2e8.v v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m1, tu, mu
-# CHECK-NEXT:  1      44    44.00   *                   vlsseg2e8.v v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m2, tu, mu
-# CHECK-NEXT:  1      76    76.00   *                   vlsseg2e8.v v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m4, tu, mu
-# CHECK-NEXT:  1      140   140.00  *                   vlsseg2e8.v v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m8, tu, mu
-# CHECK-NEXT:  1      268   268.00  *                   vlsseg2e8.v v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf8, tu, mu
-# CHECK-NEXT:  1      140   140.00  *                   vlsseg2e16.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf4, tu, mu
-# CHECK-NEXT:  1      16    16.00   *                   vlsseg2e16.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf2, tu, mu
-# CHECK-NEXT:  1      20    20.00   *                   vlsseg2e16.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m1, tu, mu
-# CHECK-NEXT:  1      28    28.00   *                   vlsseg2e16.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m2, tu, mu
-# CHECK-NEXT:  1      44    44.00   *                   vlsseg2e16.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m4, tu, mu
-# CHECK-NEXT:  1      76    76.00   *                   vlsseg2e16.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m8, tu, mu
-# CHECK-NEXT:  1      140   140.00  *                   vlsseg2e16.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf8, tu, mu
-# CHECK-NEXT:  1      76    76.00   *                   vlsseg2e32.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf4, tu, mu
-# CHECK-NEXT:  1      76    76.00   *                   vlsseg2e32.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf2, tu, mu
-# CHECK-NEXT:  1      16    16.00   *                   vlsseg2e32.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m1, tu, mu
-# CHECK-NEXT:  1      20    20.00   *                   vlsseg2e32.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m2, tu, mu
-# CHECK-NEXT:  1      28    28.00   *                   vlsseg2e32.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m4, tu, mu
-# CHECK-NEXT:  1      44    44.00   *                   vlsseg2e32.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m8, tu, mu
-# CHECK-NEXT:  1      76    76.00   *                   vlsseg2e32.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf8, tu, mu
-# CHECK-NEXT:  1      44    44.00   *                   vlsseg2e64.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf4, tu, mu
-# CHECK-NEXT:  1      44    44.00   *                   vlsseg2e64.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf2, tu, mu
-# CHECK-NEXT:  1      44    44.00   *                   vlsseg2e64.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m1, tu, mu
-# CHECK-NEXT:  1      16    16.00   *                   vlsseg2e64.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m2, tu, mu
-# CHECK-NEXT:  1      20    20.00   *                   vlsseg2e64.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m4, tu, mu
-# CHECK-NEXT:  1      28    28.00   *                   vlsseg2e64.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m8, tu, mu
-# CHECK-NEXT:  1      44    44.00   *                   vlsseg2e64.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf8, tu, mu
-# CHECK-NEXT:  1      18    18.00   *                   vlsseg3e8.v v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf4, tu, mu
-# CHECK-NEXT:  1      24    24.00   *                   vlsseg3e8.v v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf2, tu, mu
-# CHECK-NEXT:  1      36    36.00   *                   vlsseg3e8.v v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m1, tu, mu
-# CHECK-NEXT:  1      60    60.00   *                   vlsseg3e8.v v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m2, tu, mu
-# CHECK-NEXT:  1      108   108.00  *                   vlsseg3e8.v v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m4, tu, mu
-# CHECK-NEXT:  1      396   396.00  *                   vlsseg3e8.v v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m8, tu, mu
-# CHECK-NEXT:  1      396   396.00  *                   vlsseg3e8.v v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf8, tu, mu
-# CHECK-NEXT:  1      204   204.00  *                   vlsseg3e16.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf4, tu, mu
-# CHECK-NEXT:  1      18    18.00   *                   vlsseg3e16.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf2, tu, mu
-# CHECK-NEXT:  1      24    24.00   *                   vlsseg3e16.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m1, tu, mu
-# CHECK-NEXT:  1      36    36.00   *                   vlsseg3e16.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m2, tu, mu
-# CHECK-NEXT:  1      60    60.00   *                   vlsseg3e16.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m4, tu, mu
-# CHECK-NEXT:  1      204   204.00  *                   vlsseg3e16.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m8, tu, mu
-# CHECK-NEXT:  1      204   204.00  *                   vlsseg3e16.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf8, tu, mu
-# CHECK-NEXT:  1      108   108.00  *                   vlsseg3e32.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf4, tu, mu
-# CHECK-NEXT:  1      108   108.00  *                   vlsseg3e32.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf2, tu, mu
-# CHECK-NEXT:  1      18    18.00   *                   vlsseg3e32.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m1, tu, mu
-# CHECK-NEXT:  1      24    24.00   *                   vlsseg3e32.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m2, tu, mu
-# CHECK-NEXT:  1      36    36.00   *                   vlsseg3e32.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m4, tu, mu
-# CHECK-NEXT:  1      108   108.00  *                   vlsseg3e32.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m8, tu, mu
-# CHECK-NEXT:  1      108   108.00  *                   vlsseg3e32.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf8, tu, mu
-# CHECK-NEXT:  1      60    60.00   *                   vlsseg3e64.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf4, tu, mu
-# CHECK-NEXT:  1      60    60.00   *                   vlsseg3e64.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf2, tu, mu
-# CHECK-NEXT:  1      60    60.00   *                   vlsseg3e64.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m1, tu, mu
-# CHECK-NEXT:  1      18    18.00   *                   vlsseg3e64.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m2, tu, mu
-# CHECK-NEXT:  1      24    24.00   *                   vlsseg3e64.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m4, tu, mu
-# CHECK-NEXT:  1      60    60.00   *                   vlsseg3e64.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m8, tu, mu
-# CHECK-NEXT:  1      60    60.00   *                   vlsseg3e64.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf8, tu, mu
-# CHECK-NEXT:  1      20    20.00   *                   vlsseg4e8.v v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf4, tu, mu
-# CHECK-NEXT:  1      28    28.00   *                   vlsseg4e8.v v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf2, tu, mu
-# CHECK-NEXT:  1      44    44.00   *                   vlsseg4e8.v v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m1, tu, mu
-# CHECK-NEXT:  1      76    76.00   *                   vlsseg4e8.v v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m2, tu, mu
-# CHECK-NEXT:  1      140   140.00  *                   vlsseg4e8.v v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m4, tu, mu
-# CHECK-NEXT:  1      524   524.00  *                   vlsseg4e8.v v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m8, tu, mu
-# CHECK-NEXT:  1      524   524.00  *                   vlsseg4e8.v v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf8, tu, mu
-# CHECK-NEXT:  1      268   268.00  *                   vlsseg4e16.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf4, tu, mu
-# CHECK-NEXT:  1      20    20.00   *                   vlsseg4e16.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf2, tu, mu
-# CHECK-NEXT:  1      28    28.00   *                   vlsseg4e16.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m1, tu, mu
-# CHECK-NEXT:  1      44    44.00   *                   vlsseg4e16.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m2, tu, mu
-# CHECK-NEXT:  1      76    76.00   *                   vlsseg4e16.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m4, tu, mu
-# CHECK-NEXT:  1      268   268.00  *                   vlsseg4e16.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m8, tu, mu
-# CHECK-NEXT:  1      268   268.00  *                   vlsseg4e16.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf8, tu, mu
-# CHECK-NEXT:  1      140   140.00  *                   vlsseg4e32.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf4, tu, mu
-# CHECK-NEXT:  1      140   140.00  *                   vlsseg4e32.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf2, tu, mu
-# CHECK-NEXT:  1      20    20.00   *                   vlsseg4e32.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m1, tu, mu
-# CHECK-NEXT:  1      28    28.00   *                   vlsseg4e32.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m2, tu, mu
-# CHECK-NEXT:  1      44    44.00   *                   vlsseg4e32.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m4, tu, mu
-# CHECK-NEXT:  1      140   140.00  *                   vlsseg4e32.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m8, tu, mu
-# CHECK-NEXT:  1      140   140.00  *                   vlsseg4e32.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf8, tu, mu
-# CHECK-NEXT:  1      76    76.00   *                   vlsseg4e64.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf4, tu, mu
-# CHECK-NEXT:  1      76    76.00   *                   vlsseg4e64.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf2, tu, mu
-# CHECK-NEXT:  1      76    76.00   *                   vlsseg4e64.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m1, tu, mu
-# CHECK-NEXT:  1      20    20.00   *                   vlsseg4e64.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m2, tu, mu
-# CHECK-NEXT:  1      28    28.00   *                   vlsseg4e64.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m4, tu, mu
-# CHECK-NEXT:  1      76    76.00   *                   vlsseg4e64.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m8, tu, mu
-# CHECK-NEXT:  1      76    76.00   *                   vlsseg4e64.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf8, tu, mu
-# CHECK-NEXT:  1      22    22.00   *                   vlsseg5e8.v v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf4, tu, mu
-# CHECK-NEXT:  1      32    32.00   *                   vlsseg5e8.v v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf2, tu, mu
-# CHECK-NEXT:  1      52    52.00   *                   vlsseg5e8.v v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m1, tu, mu
-# CHECK-NEXT:  1      92    92.00   *                   vlsseg5e8.v v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m2, tu, mu
-# CHECK-NEXT:  1      652   652.00  *                   vlsseg5e8.v v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m4, tu, mu
-# CHECK-NEXT:  1      652   652.00  *                   vlsseg5e8.v v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf8, tu, mu
-# CHECK-NEXT:  1      332   332.00  *                   vlsseg5e16.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf4, tu, mu
-# CHECK-NEXT:  1      22    22.00   *                   vlsseg5e16.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf2, tu, mu
-# CHECK-NEXT:  1      32    32.00   *                   vlsseg5e16.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m1, tu, mu
-# CHECK-NEXT:  1      52    52.00   *                   vlsseg5e16.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m2, tu, mu
-# CHECK-NEXT:  1      332   332.00  *                   vlsseg5e16.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m4, tu, mu
-# CHECK-NEXT:  1      332   332.00  *                   vlsseg5e16.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf8, tu, mu
-# CHECK-NEXT:  1      172   172.00  *                   vlsseg5e32.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf4, tu, mu
-# CHECK-NEXT:  1      172   172.00  *                   vlsseg5e32.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf2, tu, mu
-# CHECK-NEXT:  1      22    22.00   *                   vlsseg5e32.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m1, tu, mu
-# CHECK-NEXT:  1      32    32.00   *                   vlsseg5e32.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m2, tu, mu
-# CHECK-NEXT:  1      172   172.00  *                   vlsseg5e32.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m4, tu, mu
-# CHECK-NEXT:  1      172   172.00  *                   vlsseg5e32.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf8, tu, mu
-# CHECK-NEXT:  1      92    92.00   *                   vlsseg5e64.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf4, tu, mu
-# CHECK-NEXT:  1      92    92.00   *                   vlsseg5e64.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf2, tu, mu
-# CHECK-NEXT:  1      92    92.00   *                   vlsseg5e64.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m1, tu, mu
-# CHECK-NEXT:  1      22    22.00   *                   vlsseg5e64.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m2, tu, mu
-# CHECK-NEXT:  1      92    92.00   *                   vlsseg5e64.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m4, tu, mu
-# CHECK-NEXT:  1      92    92.00   *                   vlsseg5e64.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf8, tu, mu
-# CHECK-NEXT:  1      24    24.00   *                   vlsseg6e8.v v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf4, tu, mu
-# CHECK-NEXT:  1      36    36.00   *                   vlsseg6e8.v v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf2, tu, mu
-# CHECK-NEXT:  1      60    60.00   *                   vlsseg6e8.v v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m1, tu, mu
-# CHECK-NEXT:  1      108   108.00  *                   vlsseg6e8.v v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m2, tu, mu
-# CHECK-NEXT:  1      780   780.00  *                   vlsseg6e8.v v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m4, tu, mu
-# CHECK-NEXT:  1      780   780.00  *                   vlsseg6e8.v v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf8, tu, mu
-# CHECK-NEXT:  1      396   396.00  *                   vlsseg6e16.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf4, tu, mu
-# CHECK-NEXT:  1      24    24.00   *                   vlsseg6e16.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf2, tu, mu
-# CHECK-NEXT:  1      36    36.00   *                   vlsseg6e16.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m1, tu, mu
-# CHECK-NEXT:  1      60    60.00   *                   vlsseg6e16.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m2, tu, mu
-# CHECK-NEXT:  1      396   396.00  *                   vlsseg6e16.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m4, tu, mu
-# CHECK-NEXT:  1      396   396.00  *                   vlsseg6e16.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf8, tu, mu
-# CHECK-NEXT:  1      204   204.00  *                   vlsseg6e32.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf4, tu, mu
-# CHECK-NEXT:  1      204   204.00  *                   vlsseg6e32.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf2, tu, mu
-# CHECK-NEXT:  1      24    24.00   *                   vlsseg6e32.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m1, tu, mu
-# CHECK-NEXT:  1      36    36.00   *                   vlsseg6e32.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m2, tu, mu
-# CHECK-NEXT:  1      204   204.00  *                   vlsseg6e32.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m4, tu, mu
-# CHECK-NEXT:  1      204   204.00  *                   vlsseg6e32.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf8, tu, mu
-# CHECK-NEXT:  1      108   108.00  *                   vlsseg6e64.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf4, tu, mu
-# CHECK-NEXT:  1      108   108.00  *                   vlsseg6e64.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf2, tu, mu
-# CHECK-NEXT:  1      108   108.00  *                   vlsseg6e64.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m1, tu, mu
-# CHECK-NEXT:  1      24    24.00   *                   vlsseg6e64.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m2, tu, mu
-# CHECK-NEXT:  1      108   108.00  *                   vlsseg6e64.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m4, tu, mu
-# CHECK-NEXT:  1      108   108.00  *                   vlsseg6e64.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf8, tu, mu
-# CHECK-NEXT:  1      26    26.00   *                   vlsseg7e8.v v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf4, tu, mu
-# CHECK-NEXT:  1      40    40.00   *                   vlsseg7e8.v v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf2, tu, mu
-# CHECK-NEXT:  1      68    68.00   *                   vlsseg7e8.v v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m1, tu, mu
-# CHECK-NEXT:  1      124   124.00  *                   vlsseg7e8.v v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m2, tu, mu
-# CHECK-NEXT:  1      908   908.00  *                   vlsseg7e8.v v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m4, tu, mu
-# CHECK-NEXT:  1      908   908.00  *                   vlsseg7e8.v v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf8, tu, mu
-# CHECK-NEXT:  1      460   460.00  *                   vlsseg7e16.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf4, tu, mu
-# CHECK-NEXT:  1      26    26.00   *                   vlsseg7e16.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf2, tu, mu
-# CHECK-NEXT:  1      40    40.00   *                   vlsseg7e16.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m1, tu, mu
-# CHECK-NEXT:  1      68    68.00   *                   vlsseg7e16.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m2, tu, mu
-# CHECK-NEXT:  1      460   460.00  *                   vlsseg7e16.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m4, tu, mu
-# CHECK-NEXT:  1      460   460.00  *                   vlsseg7e16.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf8, tu, mu
-# CHECK-NEXT:  1      236   236.00  *                   vlsseg7e32.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf4, tu, mu
-# CHECK-NEXT:  1      236   236.00  *                   vlsseg7e32.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf2, tu, mu
-# CHECK-NEXT:  1      26    26.00   *                   vlsseg7e32.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m1, tu, mu
-# CHECK-NEXT:  1      40    40.00   *                   vlsseg7e32.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m2, tu, mu
-# CHECK-NEXT:  1      236   236.00  *                   vlsseg7e32.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m4, tu, mu
-# CHECK-NEXT:  1      236   236.00  *                   vlsseg7e32.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf8, tu, mu
-# CHECK-NEXT:  1      124   124.00  *                   vlsseg7e64.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf4, tu, mu
-# CHECK-NEXT:  1      124   124.00  *                   vlsseg7e64.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf2, tu, mu
-# CHECK-NEXT:  1      124   124.00  *                   vlsseg7e64.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m1, tu, mu
-# CHECK-NEXT:  1      26    26.00   *                   vlsseg7e64.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m2, tu, mu
-# CHECK-NEXT:  1      124   124.00  *                   vlsseg7e64.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m4, tu, mu
-# CHECK-NEXT:  1      124   124.00  *                   vlsseg7e64.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf8, tu, mu
-# CHECK-NEXT:  1      28    28.00   *                   vlsseg8e8.v v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf4, tu, mu
-# CHECK-NEXT:  1      44    44.00   *                   vlsseg8e8.v v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf2, tu, mu
-# CHECK-NEXT:  1      76    76.00   *                   vlsseg8e8.v v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m1, tu, mu
-# CHECK-NEXT:  1      140   140.00  *                   vlsseg8e8.v v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m2, tu, mu
-# CHECK-NEXT:  1      1036   1036.00  *                   vlsseg8e8.v v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m4, tu, mu
-# CHECK-NEXT:  1      1036   1036.00  *                   vlsseg8e8.v v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf8, tu, mu
-# CHECK-NEXT:  1      524   524.00  *                   vlsseg8e16.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf4, tu, mu
-# CHECK-NEXT:  1      28    28.00   *                   vlsseg8e16.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf2, tu, mu
-# CHECK-NEXT:  1      44    44.00   *                   vlsseg8e16.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m1, tu, mu
-# CHECK-NEXT:  1      76    76.00   *                   vlsseg8e16.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m2, tu, mu
-# CHECK-NEXT:  1      524   524.00  *                   vlsseg8e16.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m4, tu, mu
-# CHECK-NEXT:  1      524   524.00  *                   vlsseg8e16.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf8, tu, mu
-# CHECK-NEXT:  1      268   268.00  *                   vlsseg8e32.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf4, tu, mu
-# CHECK-NEXT:  1      268   268.00  *                   vlsseg8e32.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf2, tu, mu
-# CHECK-NEXT:  1      28    28.00   *                   vlsseg8e32.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m1, tu, mu
-# CHECK-NEXT:  1      44    44.00   *                   vlsseg8e32.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m2, tu, mu
-# CHECK-NEXT:  1      268   268.00  *                   vlsseg8e32.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m4, tu, mu
-# CHECK-NEXT:  1      268   268.00  *                   vlsseg8e32.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf8, tu, mu
-# CHECK-NEXT:  1      140   140.00  *                   vlsseg8e64.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf4, tu, mu
-# CHECK-NEXT:  1      140   140.00  *                   vlsseg8e64.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf2, tu, mu
-# CHECK-NEXT:  1      140   140.00  *                   vlsseg8e64.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m1, tu, mu
-# CHECK-NEXT:  1      28    28.00   *                   vlsseg8e64.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m2, tu, mu
-# CHECK-NEXT:  1      140   140.00  *                   vlsseg8e64.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m4, tu, mu
-# CHECK-NEXT:  1      140   140.00  *                   vlsseg8e64.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf8, tu, mu
-# CHECK-NEXT:  1      5     16.00          *            vssseg2e8.v v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf4, tu, mu
-# CHECK-NEXT:  1      9     20.00          *            vssseg2e8.v v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf2, tu, mu
-# CHECK-NEXT:  1      17    28.00          *            vssseg2e8.v v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m1, tu, mu
-# CHECK-NEXT:  1      33    44.00          *            vssseg2e8.v v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m2, tu, mu
-# CHECK-NEXT:  1      65    76.00          *            vssseg2e8.v v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m4, tu, mu
-# CHECK-NEXT:  1      129   140.00         *            vssseg2e8.v v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m8, tu, mu
-# CHECK-NEXT:  1      257   268.00         *            vssseg2e8.v v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf8, tu, mu
-# CHECK-NEXT:  1      129   140.00         *            vssseg2e16.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf4, tu, mu
-# CHECK-NEXT:  1      5     16.00          *            vssseg2e16.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf2, tu, mu
-# CHECK-NEXT:  1      9     20.00          *            vssseg2e16.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m1, tu, mu
-# CHECK-NEXT:  1      17    28.00          *            vssseg2e16.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m2, tu, mu
-# CHECK-NEXT:  1      33    44.00          *            vssseg2e16.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m4, tu, mu
-# CHECK-NEXT:  1      65    76.00          *            vssseg2e16.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m8, tu, mu
-# CHECK-NEXT:  1      129   140.00         *            vssseg2e16.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf8, tu, mu
-# CHECK-NEXT:  1      65    76.00          *            vssseg2e32.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf4, tu, mu
-# CHECK-NEXT:  1      65    76.00          *            vssseg2e32.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf2, tu, mu
-# CHECK-NEXT:  1      5     16.00          *            vssseg2e32.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m1, tu, mu
-# CHECK-NEXT:  1      9     20.00          *            vssseg2e32.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m2, tu, mu
-# CHECK-NEXT:  1      17    28.00          *            vssseg2e32.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m4, tu, mu
-# CHECK-NEXT:  1      33    44.00          *            vssseg2e32.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m8, tu, mu
-# CHECK-NEXT:  1      65    76.00          *            vssseg2e32.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf8, tu, mu
-# CHECK-NEXT:  1      33    44.00          *            vssseg2e64.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf4, tu, mu
-# CHECK-NEXT:  1      33    44.00          *            vssseg2e64.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf2, tu, mu
-# CHECK-NEXT:  1      33    44.00          *            vssseg2e64.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m1, tu, mu
-# CHECK-NEXT:  1      5     16.00          *            vssseg2e64.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m2, tu, mu
-# CHECK-NEXT:  1      9     20.00          *            vssseg2e64.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m4, tu, mu
-# CHECK-NEXT:  1      17    28.00          *            vssseg2e64.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m8, tu, mu
-# CHECK-NEXT:  1      33    44.00          *            vssseg2e64.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf8, tu, mu
-# CHECK-NEXT:  1      7     18.00          *            vssseg3e8.v v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf4, tu, mu
-# CHECK-NEXT:  1      13    24.00          *            vssseg3e8.v v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf2, tu, mu
-# CHECK-NEXT:  1      25    36.00          *            vssseg3e8.v v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m1, tu, mu
-# CHECK-NEXT:  1      49    60.00          *            vssseg3e8.v v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m2, tu, mu
-# CHECK-NEXT:  1      97    108.00         *            vssseg3e8.v v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m4, tu, mu
-# CHECK-NEXT:  1      385   396.00         *            vssseg3e8.v v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m8, tu, mu
-# CHECK-NEXT:  1      385   396.00         *            vssseg3e8.v v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf8, tu, mu
-# CHECK-NEXT:  1      193   204.00         *            vssseg3e16.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf4, tu, mu
-# CHECK-NEXT:  1      7     18.00          *            vssseg3e16.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf2, tu, mu
-# CHECK-NEXT:  1      13    24.00          *            vssseg3e16.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m1, tu, mu
-# CHECK-NEXT:  1      25    36.00          *            vssseg3e16.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m2, tu, mu
-# CHECK-NEXT:  1      49    60.00          *            vssseg3e16.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m4, tu, mu
-# CHECK-NEXT:  1      193   204.00         *            vssseg3e16.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m8, tu, mu
-# CHECK-NEXT:  1      193   204.00         *            vssseg3e16.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf8, tu, mu
-# CHECK-NEXT:  1      97    108.00         *            vssseg3e32.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf4, tu, mu
-# CHECK-NEXT:  1      97    108.00         *            vssseg3e32.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf2, tu, mu
-# CHECK-NEXT:  1      7     18.00          *            vssseg3e32.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m1, tu, mu
-# CHECK-NEXT:  1      13    24.00          *            vssseg3e32.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m2, tu, mu
-# CHECK-NEXT:  1      25    36.00          *            vssseg3e32.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m4, tu, mu
-# CHECK-NEXT:  1      97    108.00         *            vssseg3e32.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m8, tu, mu
-# CHECK-NEXT:  1      97    108.00         *            vssseg3e32.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf8, tu, mu
-# CHECK-NEXT:  1      49    60.00          *            vssseg3e64.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf4, tu, mu
-# CHECK-NEXT:  1      49    60.00          *            vssseg3e64.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf2, tu, mu
-# CHECK-NEXT:  1      49    60.00          *            vssseg3e64.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m1, tu, mu
-# CHECK-NEXT:  1      7     18.00          *            vssseg3e64.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m2, tu, mu
-# CHECK-NEXT:  1      13    24.00          *            vssseg3e64.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m4, tu, mu
-# CHECK-NEXT:  1      49    60.00          *            vssseg3e64.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m8, tu, mu
-# CHECK-NEXT:  1      49    60.00          *            vssseg3e64.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf8, tu, mu
-# CHECK-NEXT:  1      9     20.00          *            vssseg4e8.v v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf4, tu, mu
-# CHECK-NEXT:  1      17    28.00          *            vssseg4e8.v v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf2, tu, mu
-# CHECK-NEXT:  1      33    44.00          *            vssseg4e8.v v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m1, tu, mu
-# CHECK-NEXT:  1      65    76.00          *            vssseg4e8.v v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m2, tu, mu
-# CHECK-NEXT:  1      129   140.00         *            vssseg4e8.v v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m4, tu, mu
-# CHECK-NEXT:  1      513   524.00         *            vssseg4e8.v v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m8, tu, mu
-# CHECK-NEXT:  1      513   524.00         *            vssseg4e8.v v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf8, tu, mu
-# CHECK-NEXT:  1      257   268.00         *            vssseg4e16.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf4, tu, mu
-# CHECK-NEXT:  1      9     20.00          *            vssseg4e16.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf2, tu, mu
-# CHECK-NEXT:  1      17    28.00          *            vssseg4e16.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m1, tu, mu
-# CHECK-NEXT:  1      33    44.00          *            vssseg4e16.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m2, tu, mu
-# CHECK-NEXT:  1      65    76.00          *            vssseg4e16.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m4, tu, mu
-# CHECK-NEXT:  1      257   268.00         *            vssseg4e16.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m8, tu, mu
-# CHECK-NEXT:  1      257   268.00         *            vssseg4e16.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf8, tu, mu
-# CHECK-NEXT:  1      129   140.00         *            vssseg4e32.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf4, tu, mu
-# CHECK-NEXT:  1      129   140.00         *            vssseg4e32.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf2, tu, mu
-# CHECK-NEXT:  1      9     20.00          *            vssseg4e32.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m1, tu, mu
-# CHECK-NEXT:  1      17    28.00          *            vssseg4e32.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m2, tu, mu
-# CHECK-NEXT:  1      33    44.00          *            vssseg4e32.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m4, tu, mu
-# CHECK-NEXT:  1      129   140.00         *            vssseg4e32.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m8, tu, mu
-# CHECK-NEXT:  1      129   140.00         *            vssseg4e32.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf8, tu, mu
-# CHECK-NEXT:  1      65    76.00          *            vssseg4e64.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf4, tu, mu
-# CHECK-NEXT:  1      65    76.00          *            vssseg4e64.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf2, tu, mu
-# CHECK-NEXT:  1      65    76.00          *            vssseg4e64.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m1, tu, mu
-# CHECK-NEXT:  1      9     20.00          *            vssseg4e64.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m2, tu, mu
-# CHECK-NEXT:  1      17    28.00          *            vssseg4e64.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m4, tu, mu
-# CHECK-NEXT:  1      65    76.00          *            vssseg4e64.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m8, tu, mu
-# CHECK-NEXT:  1      65    76.00          *            vssseg4e64.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf8, tu, mu
-# CHECK-NEXT:  1      11    22.00          *            vssseg5e8.v v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf4, tu, mu
-# CHECK-NEXT:  1      21    32.00          *            vssseg5e8.v v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf2, tu, mu
-# CHECK-NEXT:  1      41    52.00          *            vssseg5e8.v v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m1, tu, mu
-# CHECK-NEXT:  1      81    92.00          *            vssseg5e8.v v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m2, tu, mu
-# CHECK-NEXT:  1      641   652.00         *            vssseg5e8.v v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m4, tu, mu
-# CHECK-NEXT:  1      641   652.00         *            vssseg5e8.v v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf8, tu, mu
-# CHECK-NEXT:  1      321   332.00         *            vssseg5e16.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf4, tu, mu
-# CHECK-NEXT:  1      11    22.00          *            vssseg5e16.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf2, tu, mu
-# CHECK-NEXT:  1      21    32.00          *            vssseg5e16.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m1, tu, mu
-# CHECK-NEXT:  1      41    52.00          *            vssseg5e16.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m2, tu, mu
-# CHECK-NEXT:  1      321   332.00         *            vssseg5e16.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m4, tu, mu
-# CHECK-NEXT:  1      321   332.00         *            vssseg5e16.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf8, tu, mu
-# CHECK-NEXT:  1      161   172.00         *            vssseg5e32.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf4, tu, mu
-# CHECK-NEXT:  1      161   172.00         *            vssseg5e32.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf2, tu, mu
-# CHECK-NEXT:  1      11    22.00          *            vssseg5e32.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m1, tu, mu
-# CHECK-NEXT:  1      21    32.00          *            vssseg5e32.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m2, tu, mu
-# CHECK-NEXT:  1      161   172.00         *            vssseg5e32.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m4, tu, mu
-# CHECK-NEXT:  1      161   172.00         *            vssseg5e32.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf8, tu, mu
-# CHECK-NEXT:  1      81    92.00          *            vssseg5e64.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf4, tu, mu
-# CHECK-NEXT:  1      81    92.00          *            vssseg5e64.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf2, tu, mu
-# CHECK-NEXT:  1      81    92.00          *            vssseg5e64.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m1, tu, mu
-# CHECK-NEXT:  1      11    22.00          *            vssseg5e64.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m2, tu, mu
-# CHECK-NEXT:  1      81    92.00          *            vssseg5e64.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m4, tu, mu
-# CHECK-NEXT:  1      81    92.00          *            vssseg5e64.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf8, tu, mu
-# CHECK-NEXT:  1      13    24.00          *            vssseg6e8.v v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf4, tu, mu
-# CHECK-NEXT:  1      25    36.00          *            vssseg6e8.v v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf2, tu, mu
-# CHECK-NEXT:  1      49    60.00          *            vssseg6e8.v v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m1, tu, mu
-# CHECK-NEXT:  1      97    108.00         *            vssseg6e8.v v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m2, tu, mu
-# CHECK-NEXT:  1      769   780.00         *            vssseg6e8.v v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m4, tu, mu
-# CHECK-NEXT:  1      769   780.00         *            vssseg6e8.v v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf8, tu, mu
-# CHECK-NEXT:  1      385   396.00         *            vssseg6e16.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf4, tu, mu
-# CHECK-NEXT:  1      13    24.00          *            vssseg6e16.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf2, tu, mu
-# CHECK-NEXT:  1      25    36.00          *            vssseg6e16.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m1, tu, mu
-# CHECK-NEXT:  1      49    60.00          *            vssseg6e16.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m2, tu, mu
-# CHECK-NEXT:  1      385   396.00         *            vssseg6e16.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m4, tu, mu
-# CHECK-NEXT:  1      385   396.00         *            vssseg6e16.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf8, tu, mu
-# CHECK-NEXT:  1      193   204.00         *            vssseg6e32.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf4, tu, mu
-# CHECK-NEXT:  1      193   204.00         *            vssseg6e32.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf2, tu, mu
-# CHECK-NEXT:  1      13    24.00          *            vssseg6e32.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m1, tu, mu
-# CHECK-NEXT:  1      25    36.00          *            vssseg6e32.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m2, tu, mu
-# CHECK-NEXT:  1      193   204.00         *            vssseg6e32.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m4, tu, mu
-# CHECK-NEXT:  1      193   204.00         *            vssseg6e32.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf8, tu, mu
-# CHECK-NEXT:  1      97    108.00         *            vssseg6e64.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf4, tu, mu
-# CHECK-NEXT:  1      97    108.00         *            vssseg6e64.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf2, tu, mu
-# CHECK-NEXT:  1      97    108.00         *            vssseg6e64.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m1, tu, mu
-# CHECK-NEXT:  1      13    24.00          *            vssseg6e64.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m2, tu, mu
-# CHECK-NEXT:  1      97    108.00         *            vssseg6e64.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m4, tu, mu
-# CHECK-NEXT:  1      97    108.00         *            vssseg6e64.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf8, tu, mu
-# CHECK-NEXT:  1      15    26.00          *            vssseg7e8.v v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf4, tu, mu
-# CHECK-NEXT:  1      29    40.00          *            vssseg7e8.v v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf2, tu, mu
-# CHECK-NEXT:  1      57    68.00          *            vssseg7e8.v v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m1, tu, mu
-# CHECK-NEXT:  1      113   124.00         *            vssseg7e8.v v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m2, tu, mu
-# CHECK-NEXT:  1      897   908.00         *            vssseg7e8.v v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m4, tu, mu
-# CHECK-NEXT:  1      897   908.00         *            vssseg7e8.v v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf8, tu, mu
-# CHECK-NEXT:  1      449   460.00         *            vssseg7e16.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf4, tu, mu
-# CHECK-NEXT:  1      15    26.00          *            vssseg7e16.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf2, tu, mu
-# CHECK-NEXT:  1      29    40.00          *            vssseg7e16.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m1, tu, mu
-# CHECK-NEXT:  1      57    68.00          *            vssseg7e16.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m2, tu, mu
-# CHECK-NEXT:  1      449   460.00         *            vssseg7e16.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m4, tu, mu
-# CHECK-NEXT:  1      449   460.00         *            vssseg7e16.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf8, tu, mu
-# CHECK-NEXT:  1      225   236.00         *            vssseg7e32.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf4, tu, mu
-# CHECK-NEXT:  1      225   236.00         *            vssseg7e32.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf2, tu, mu
-# CHECK-NEXT:  1      15    26.00          *            vssseg7e32.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m1, tu, mu
-# CHECK-NEXT:  1      29    40.00          *            vssseg7e32.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m2, tu, mu
-# CHECK-NEXT:  1      225   236.00         *            vssseg7e32.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m4, tu, mu
-# CHECK-NEXT:  1      225   236.00         *            vssseg7e32.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf8, tu, mu
-# CHECK-NEXT:  1      113   124.00         *            vssseg7e64.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf4, tu, mu
-# CHECK-NEXT:  1      113   124.00         *            vssseg7e64.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf2, tu, mu
-# CHECK-NEXT:  1      113   124.00         *            vssseg7e64.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m1, tu, mu
-# CHECK-NEXT:  1      15    26.00          *            vssseg7e64.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m2, tu, mu
-# CHECK-NEXT:  1      113   124.00         *            vssseg7e64.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m4, tu, mu
-# CHECK-NEXT:  1      113   124.00         *            vssseg7e64.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf8, tu, mu
-# CHECK-NEXT:  1      17    28.00          *            vssseg8e8.v v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf4, tu, mu
-# CHECK-NEXT:  1      33    44.00          *            vssseg8e8.v v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf2, tu, mu
-# CHECK-NEXT:  1      65    76.00          *            vssseg8e8.v v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m1, tu, mu
-# CHECK-NEXT:  1      129   140.00         *            vssseg8e8.v v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m2, tu, mu
-# CHECK-NEXT:  1      1025   1036.00         *            vssseg8e8.v v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m4, tu, mu
-# CHECK-NEXT:  1      1025   1036.00         *            vssseg8e8.v v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf8, tu, mu
-# CHECK-NEXT:  1      513   524.00         *            vssseg8e16.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf4, tu, mu
-# CHECK-NEXT:  1      17    28.00          *            vssseg8e16.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf2, tu, mu
-# CHECK-NEXT:  1      33    44.00          *            vssseg8e16.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m1, tu, mu
-# CHECK-NEXT:  1      65    76.00          *            vssseg8e16.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m2, tu, mu
-# CHECK-NEXT:  1      513   524.00         *            vssseg8e16.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m4, tu, mu
-# CHECK-NEXT:  1      513   524.00         *            vssseg8e16.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf8, tu, mu
-# CHECK-NEXT:  1      257   268.00         *            vssseg8e32.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf4, tu, mu
-# CHECK-NEXT:  1      257   268.00         *            vssseg8e32.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf2, tu, mu
-# CHECK-NEXT:  1      17    28.00          *            vssseg8e32.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m1, tu, mu
-# CHECK-NEXT:  1      33    44.00          *            vssseg8e32.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m2, tu, mu
-# CHECK-NEXT:  1      257   268.00         *            vssseg8e32.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m4, tu, mu
-# CHECK-NEXT:  1      257   268.00         *            vssseg8e32.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf8, tu, mu
-# CHECK-NEXT:  1      129   140.00         *            vssseg8e64.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf4, tu, mu
-# CHECK-NEXT:  1      129   140.00         *            vssseg8e64.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf2, tu, mu
-# CHECK-NEXT:  1      129   140.00         *            vssseg8e64.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m1, tu, mu
-# CHECK-NEXT:  1      17    28.00          *            vssseg8e64.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m2, tu, mu
-# CHECK-NEXT:  1      129   140.00         *            vssseg8e64.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m4, tu, mu
-# CHECK-NEXT:  1      129   140.00         *            vssseg8e64.v  v8, (a0), a1
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf8, tu, mu
-# CHECK-NEXT:  1      16    16.00   *                   vlseg2e8ff.v  v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf4, tu, mu
-# CHECK-NEXT:  1      20    20.00   *                   vlseg2e8ff.v  v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf2, tu, mu
-# CHECK-NEXT:  1      28    28.00   *                   vlseg2e8ff.v  v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m1, tu, mu
-# CHECK-NEXT:  1      44    44.00   *                   vlseg2e8ff.v  v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m2, tu, mu
-# CHECK-NEXT:  1      76    76.00   *                   vlseg2e8ff.v  v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m4, tu, mu
-# CHECK-NEXT:  1      140   140.00  *                   vlseg2e8ff.v  v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m8, tu, mu
-# CHECK-NEXT:  1      268   268.00  *                   vlseg2e8ff.v  v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf8, tu, mu
-# CHECK-NEXT:  1      140   140.00  *                   vlseg2e16ff.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf4, tu, mu
-# CHECK-NEXT:  1      16    16.00   *                   vlseg2e16ff.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf2, tu, mu
-# CHECK-NEXT:  1      20    20.00   *                   vlseg2e16ff.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m1, tu, mu
-# CHECK-NEXT:  1      28    28.00   *                   vlseg2e16ff.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m2, tu, mu
-# CHECK-NEXT:  1      44    44.00   *                   vlseg2e16ff.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m4, tu, mu
-# CHECK-NEXT:  1      76    76.00   *                   vlseg2e16ff.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m8, tu, mu
-# CHECK-NEXT:  1      140   140.00  *                   vlseg2e16ff.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf8, tu, mu
-# CHECK-NEXT:  1      76    76.00   *                   vlseg2e32ff.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf4, tu, mu
-# CHECK-NEXT:  1      76    76.00   *                   vlseg2e32ff.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf2, tu, mu
-# CHECK-NEXT:  1      16    16.00   *                   vlseg2e32ff.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m1, tu, mu
-# CHECK-NEXT:  1      20    20.00   *                   vlseg2e32ff.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m2, tu, mu
-# CHECK-NEXT:  1      28    28.00   *                   vlseg2e32ff.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m4, tu, mu
-# CHECK-NEXT:  1      44    44.00   *                   vlseg2e32ff.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m8, tu, mu
-# CHECK-NEXT:  1      76    76.00   *                   vlseg2e32ff.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf8, tu, mu
-# CHECK-NEXT:  1      44    44.00   *                   vlseg2e64ff.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf4, tu, mu
-# CHECK-NEXT:  1      44    44.00   *                   vlseg2e64ff.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf2, tu, mu
-# CHECK-NEXT:  1      44    44.00   *                   vlseg2e64ff.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m1, tu, mu
-# CHECK-NEXT:  1      16    16.00   *                   vlseg2e64ff.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m2, tu, mu
-# CHECK-NEXT:  1      20    20.00   *                   vlseg2e64ff.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m4, tu, mu
-# CHECK-NEXT:  1      28    28.00   *                   vlseg2e64ff.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m8, tu, mu
-# CHECK-NEXT:  1      44    44.00   *                   vlseg2e64ff.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf8, tu, mu
-# CHECK-NEXT:  1      18    18.00   *                   vlseg3e8ff.v  v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf4, tu, mu
-# CHECK-NEXT:  1      24    24.00   *                   vlseg3e8ff.v  v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf2, tu, mu
-# CHECK-NEXT:  1      36    36.00   *                   vlseg3e8ff.v  v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m1, tu, mu
-# CHECK-NEXT:  1      60    60.00   *                   vlseg3e8ff.v  v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m2, tu, mu
-# CHECK-NEXT:  1      108   108.00  *                   vlseg3e8ff.v  v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m4, tu, mu
-# CHECK-NEXT:  1      396   396.00  *                   vlseg3e8ff.v  v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m8, tu, mu
-# CHECK-NEXT:  1      396   396.00  *                   vlseg3e8ff.v  v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf8, tu, mu
-# CHECK-NEXT:  1      204   204.00  *                   vlseg3e16ff.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf4, tu, mu
-# CHECK-NEXT:  1      18    18.00   *                   vlseg3e16ff.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf2, tu, mu
-# CHECK-NEXT:  1      24    24.00   *                   vlseg3e16ff.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m1, tu, mu
-# CHECK-NEXT:  1      36    36.00   *                   vlseg3e16ff.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m2, tu, mu
-# CHECK-NEXT:  1      60    60.00   *                   vlseg3e16ff.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m4, tu, mu
-# CHECK-NEXT:  1      204   204.00  *                   vlseg3e16ff.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m8, tu, mu
-# CHECK-NEXT:  1      204   204.00  *                   vlseg3e16ff.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf8, tu, mu
-# CHECK-NEXT:  1      108   108.00  *                   vlseg3e32ff.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf4, tu, mu
-# CHECK-NEXT:  1      108   108.00  *                   vlseg3e32ff.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf2, tu, mu
-# CHECK-NEXT:  1      18    18.00   *                   vlseg3e32ff.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m1, tu, mu
-# CHECK-NEXT:  1      24    24.00   *                   vlseg3e32ff.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m2, tu, mu
-# CHECK-NEXT:  1      36    36.00   *                   vlseg3e32ff.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m4, tu, mu
-# CHECK-NEXT:  1      108   108.00  *                   vlseg3e32ff.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m8, tu, mu
-# CHECK-NEXT:  1      108   108.00  *                   vlseg3e32ff.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf8, tu, mu
-# CHECK-NEXT:  1      60    60.00   *                   vlseg3e64ff.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf4, tu, mu
-# CHECK-NEXT:  1      60    60.00   *                   vlseg3e64ff.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf2, tu, mu
-# CHECK-NEXT:  1      60    60.00   *                   vlseg3e64ff.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m1, tu, mu
-# CHECK-NEXT:  1      18    18.00   *                   vlseg3e64ff.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m2, tu, mu
-# CHECK-NEXT:  1      24    24.00   *                   vlseg3e64ff.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m4, tu, mu
-# CHECK-NEXT:  1      60    60.00   *                   vlseg3e64ff.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m8, tu, mu
-# CHECK-NEXT:  1      60    60.00   *                   vlseg3e64ff.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf8, tu, mu
-# CHECK-NEXT:  1      20    20.00   *                   vlseg4e8ff.v  v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf4, tu, mu
-# CHECK-NEXT:  1      28    28.00   *                   vlseg4e8ff.v  v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf2, tu, mu
-# CHECK-NEXT:  1      44    44.00   *                   vlseg4e8ff.v  v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m1, tu, mu
-# CHECK-NEXT:  1      76    76.00   *                   vlseg4e8ff.v  v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m2, tu, mu
-# CHECK-NEXT:  1      140   140.00  *                   vlseg4e8ff.v  v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m4, tu, mu
-# CHECK-NEXT:  1      524   524.00  *                   vlseg4e8ff.v  v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m8, tu, mu
-# CHECK-NEXT:  1      524   524.00  *                   vlseg4e8ff.v  v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf8, tu, mu
-# CHECK-NEXT:  1      268   268.00  *                   vlseg4e16ff.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf4, tu, mu
-# CHECK-NEXT:  1      20    20.00   *                   vlseg4e16ff.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf2, tu, mu
-# CHECK-NEXT:  1      28    28.00   *                   vlseg4e16ff.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m1, tu, mu
-# CHECK-NEXT:  1      44    44.00   *                   vlseg4e16ff.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m2, tu, mu
-# CHECK-NEXT:  1      76    76.00   *                   vlseg4e16ff.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m4, tu, mu
-# CHECK-NEXT:  1      268   268.00  *                   vlseg4e16ff.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m8, tu, mu
-# CHECK-NEXT:  1      268   268.00  *                   vlseg4e16ff.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf8, tu, mu
-# CHECK-NEXT:  1      140   140.00  *                   vlseg4e32ff.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf4, tu, mu
-# CHECK-NEXT:  1      140   140.00  *                   vlseg4e32ff.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf2, tu, mu
-# CHECK-NEXT:  1      20    20.00   *                   vlseg4e32ff.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m1, tu, mu
-# CHECK-NEXT:  1      28    28.00   *                   vlseg4e32ff.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m2, tu, mu
-# CHECK-NEXT:  1      44    44.00   *                   vlseg4e32ff.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m4, tu, mu
-# CHECK-NEXT:  1      140   140.00  *                   vlseg4e32ff.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m8, tu, mu
-# CHECK-NEXT:  1      140   140.00  *                   vlseg4e32ff.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf8, tu, mu
-# CHECK-NEXT:  1      76    76.00   *                   vlseg4e64ff.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf4, tu, mu
-# CHECK-NEXT:  1      76    76.00   *                   vlseg4e64ff.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf2, tu, mu
-# CHECK-NEXT:  1      76    76.00   *                   vlseg4e64ff.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m1, tu, mu
-# CHECK-NEXT:  1      20    20.00   *                   vlseg4e64ff.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m2, tu, mu
-# CHECK-NEXT:  1      28    28.00   *                   vlseg4e64ff.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m4, tu, mu
-# CHECK-NEXT:  1      76    76.00   *                   vlseg4e64ff.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m8, tu, mu
-# CHECK-NEXT:  1      76    76.00   *                   vlseg4e64ff.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf8, tu, mu
-# CHECK-NEXT:  1      22    22.00   *                   vlseg5e8ff.v  v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf4, tu, mu
-# CHECK-NEXT:  1      32    32.00   *                   vlseg5e8ff.v  v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf2, tu, mu
-# CHECK-NEXT:  1      52    52.00   *                   vlseg5e8ff.v  v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m1, tu, mu
-# CHECK-NEXT:  1      92    92.00   *                   vlseg5e8ff.v  v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m2, tu, mu
-# CHECK-NEXT:  1      652   652.00  *                   vlseg5e8ff.v  v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m4, tu, mu
-# CHECK-NEXT:  1      652   652.00  *                   vlseg5e8ff.v  v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf8, tu, mu
-# CHECK-NEXT:  1      332   332.00  *                   vlseg5e16ff.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf4, tu, mu
-# CHECK-NEXT:  1      22    22.00   *                   vlseg5e16ff.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf2, tu, mu
-# CHECK-NEXT:  1      32    32.00   *                   vlseg5e16ff.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m1, tu, mu
-# CHECK-NEXT:  1      52    52.00   *                   vlseg5e16ff.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m2, tu, mu
-# CHECK-NEXT:  1      332   332.00  *                   vlseg5e16ff.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m4, tu, mu
-# CHECK-NEXT:  1      332   332.00  *                   vlseg5e16ff.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf8, tu, mu
-# CHECK-NEXT:  1      172   172.00  *                   vlseg5e32ff.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf4, tu, mu
-# CHECK-NEXT:  1      172   172.00  *                   vlseg5e32ff.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf2, tu, mu
-# CHECK-NEXT:  1      22    22.00   *                   vlseg5e32ff.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m1, tu, mu
-# CHECK-NEXT:  1      32    32.00   *                   vlseg5e32ff.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m2, tu, mu
-# CHECK-NEXT:  1      172   172.00  *                   vlseg5e32ff.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m4, tu, mu
-# CHECK-NEXT:  1      172   172.00  *                   vlseg5e32ff.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf8, tu, mu
-# CHECK-NEXT:  1      92    92.00   *                   vlseg5e64ff.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf4, tu, mu
-# CHECK-NEXT:  1      92    92.00   *                   vlseg5e64ff.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf2, tu, mu
-# CHECK-NEXT:  1      92    92.00   *                   vlseg5e64ff.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m1, tu, mu
-# CHECK-NEXT:  1      22    22.00   *                   vlseg5e64ff.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m2, tu, mu
-# CHECK-NEXT:  1      92    92.00   *                   vlseg5e64ff.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m4, tu, mu
-# CHECK-NEXT:  1      92    92.00   *                   vlseg5e64ff.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf8, tu, mu
-# CHECK-NEXT:  1      24    24.00   *                   vlseg6e8ff.v  v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf4, tu, mu
-# CHECK-NEXT:  1      36    36.00   *                   vlseg6e8ff.v  v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf2, tu, mu
-# CHECK-NEXT:  1      60    60.00   *                   vlseg6e8ff.v  v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m1, tu, mu
-# CHECK-NEXT:  1      108   108.00  *                   vlseg6e8ff.v  v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m2, tu, mu
-# CHECK-NEXT:  1      780   780.00  *                   vlseg6e8ff.v  v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m4, tu, mu
-# CHECK-NEXT:  1      780   780.00  *                   vlseg6e8ff.v  v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf8, tu, mu
-# CHECK-NEXT:  1      396   396.00  *                   vlseg6e16ff.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf4, tu, mu
-# CHECK-NEXT:  1      24    24.00   *                   vlseg6e16ff.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf2, tu, mu
-# CHECK-NEXT:  1      36    36.00   *                   vlseg6e16ff.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m1, tu, mu
-# CHECK-NEXT:  1      60    60.00   *                   vlseg6e16ff.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m2, tu, mu
-# CHECK-NEXT:  1      396   396.00  *                   vlseg6e16ff.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m4, tu, mu
-# CHECK-NEXT:  1      396   396.00  *                   vlseg6e16ff.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf8, tu, mu
-# CHECK-NEXT:  1      204   204.00  *                   vlseg6e32ff.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf4, tu, mu
-# CHECK-NEXT:  1      204   204.00  *                   vlseg6e32ff.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf2, tu, mu
-# CHECK-NEXT:  1      24    24.00   *                   vlseg6e32ff.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m1, tu, mu
-# CHECK-NEXT:  1      36    36.00   *                   vlseg6e32ff.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m2, tu, mu
-# CHECK-NEXT:  1      204   204.00  *                   vlseg6e32ff.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m4, tu, mu
-# CHECK-NEXT:  1      204   204.00  *                   vlseg6e32ff.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf8, tu, mu
-# CHECK-NEXT:  1      108   108.00  *                   vlseg6e64ff.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf4, tu, mu
-# CHECK-NEXT:  1      108   108.00  *                   vlseg6e64ff.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf2, tu, mu
-# CHECK-NEXT:  1      108   108.00  *                   vlseg6e64ff.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m1, tu, mu
-# CHECK-NEXT:  1      24    24.00   *                   vlseg6e64ff.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m2, tu, mu
-# CHECK-NEXT:  1      108   108.00  *                   vlseg6e64ff.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m4, tu, mu
-# CHECK-NEXT:  1      108   108.00  *                   vlseg6e64ff.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf8, tu, mu
-# CHECK-NEXT:  1      26    26.00   *                   vlseg7e8ff.v  v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf4, tu, mu
-# CHECK-NEXT:  1      40    40.00   *                   vlseg7e8ff.v  v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf2, tu, mu
-# CHECK-NEXT:  1      68    68.00   *                   vlseg7e8ff.v  v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m1, tu, mu
-# CHECK-NEXT:  1      124   124.00  *                   vlseg7e8ff.v  v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m2, tu, mu
-# CHECK-NEXT:  1      908   908.00  *                   vlseg7e8ff.v  v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m4, tu, mu
-# CHECK-NEXT:  1      908   908.00  *                   vlseg7e8ff.v  v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf8, tu, mu
-# CHECK-NEXT:  1      460   460.00  *                   vlseg7e16ff.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf4, tu, mu
-# CHECK-NEXT:  1      26    26.00   *                   vlseg7e16ff.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf2, tu, mu
-# CHECK-NEXT:  1      40    40.00   *                   vlseg7e16ff.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m1, tu, mu
-# CHECK-NEXT:  1      68    68.00   *                   vlseg7e16ff.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m2, tu, mu
-# CHECK-NEXT:  1      460   460.00  *                   vlseg7e16ff.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m4, tu, mu
-# CHECK-NEXT:  1      460   460.00  *                   vlseg7e16ff.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf8, tu, mu
-# CHECK-NEXT:  1      236   236.00  *                   vlseg7e32ff.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf4, tu, mu
-# CHECK-NEXT:  1      236   236.00  *                   vlseg7e32ff.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf2, tu, mu
-# CHECK-NEXT:  1      26    26.00   *                   vlseg7e32ff.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m1, tu, mu
-# CHECK-NEXT:  1      40    40.00   *                   vlseg7e32ff.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m2, tu, mu
-# CHECK-NEXT:  1      236   236.00  *                   vlseg7e32ff.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m4, tu, mu
-# CHECK-NEXT:  1      236   236.00  *                   vlseg7e32ff.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf8, tu, mu
-# CHECK-NEXT:  1      124   124.00  *                   vlseg7e64ff.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf4, tu, mu
-# CHECK-NEXT:  1      124   124.00  *                   vlseg7e64ff.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf2, tu, mu
-# CHECK-NEXT:  1      124   124.00  *                   vlseg7e64ff.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m1, tu, mu
-# CHECK-NEXT:  1      26    26.00   *                   vlseg7e64ff.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m2, tu, mu
-# CHECK-NEXT:  1      124   124.00  *                   vlseg7e64ff.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m4, tu, mu
-# CHECK-NEXT:  1      124   124.00  *                   vlseg7e64ff.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf8, tu, mu
-# CHECK-NEXT:  1      28    28.00   *                   vlseg8e8ff.v  v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf4, tu, mu
-# CHECK-NEXT:  1      44    44.00   *                   vlseg8e8ff.v  v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf2, tu, mu
-# CHECK-NEXT:  1      76    76.00   *                   vlseg8e8ff.v  v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m1, tu, mu
-# CHECK-NEXT:  1      140   140.00  *                   vlseg8e8ff.v  v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m2, tu, mu
-# CHECK-NEXT:  1      1036   1036.00  *                   vlseg8e8ff.v  v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m4, tu, mu
-# CHECK-NEXT:  1      1036   1036.00  *                   vlseg8e8ff.v  v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf8, tu, mu
-# CHECK-NEXT:  1      524   524.00  *                   vlseg8e16ff.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf4, tu, mu
-# CHECK-NEXT:  1      28    28.00   *                   vlseg8e16ff.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf2, tu, mu
-# CHECK-NEXT:  1      44    44.00   *                   vlseg8e16ff.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m1, tu, mu
-# CHECK-NEXT:  1      76    76.00   *                   vlseg8e16ff.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m2, tu, mu
-# CHECK-NEXT:  1      524   524.00  *                   vlseg8e16ff.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m4, tu, mu
-# CHECK-NEXT:  1      524   524.00  *                   vlseg8e16ff.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf8, tu, mu
-# CHECK-NEXT:  1      268   268.00  *                   vlseg8e32ff.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf4, tu, mu
-# CHECK-NEXT:  1      268   268.00  *                   vlseg8e32ff.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf2, tu, mu
-# CHECK-NEXT:  1      28    28.00   *                   vlseg8e32ff.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m1, tu, mu
-# CHECK-NEXT:  1      44    44.00   *                   vlseg8e32ff.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m2, tu, mu
-# CHECK-NEXT:  1      268   268.00  *                   vlseg8e32ff.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m4, tu, mu
-# CHECK-NEXT:  1      268   268.00  *                   vlseg8e32ff.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf8, tu, mu
-# CHECK-NEXT:  1      140   140.00  *                   vlseg8e64ff.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf4, tu, mu
-# CHECK-NEXT:  1      140   140.00  *                   vlseg8e64ff.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf2, tu, mu
-# CHECK-NEXT:  1      140   140.00  *                   vlseg8e64ff.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m1, tu, mu
-# CHECK-NEXT:  1      28    28.00   *                   vlseg8e64ff.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m2, tu, mu
-# CHECK-NEXT:  1      140   140.00  *                   vlseg8e64ff.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m4, tu, mu
-# CHECK-NEXT:  1      140   140.00  *                   vlseg8e64ff.v v8, (a0)
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf8, tu, mu
-# CHECK-NEXT:  1      16    16.00   *                   vluxseg2ei8.v v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf4, tu, mu
-# CHECK-NEXT:  1      20    20.00   *                   vluxseg2ei8.v v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf2, tu, mu
-# CHECK-NEXT:  1      28    28.00   *                   vluxseg2ei8.v v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m1, tu, mu
-# CHECK-NEXT:  1      44    44.00   *                   vluxseg2ei8.v v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m2, tu, mu
-# CHECK-NEXT:  1      44    44.00   *                   vluxseg2ei8.v v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m4, tu, mu
-# CHECK-NEXT:  1      44    44.00   *                   vluxseg2ei8.v v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m8, tu, mu
-# CHECK-NEXT:  1      268   268.00  *                   vluxseg2ei8.v v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf8, tu, mu
-# CHECK-NEXT:  1      16    16.00   *                   vluxseg2ei16.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf4, tu, mu
-# CHECK-NEXT:  1      20    20.00   *                   vluxseg2ei16.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf2, tu, mu
-# CHECK-NEXT:  1      28    28.00   *                   vluxseg2ei16.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m1, tu, mu
-# CHECK-NEXT:  1      28    28.00   *                   vluxseg2ei16.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m2, tu, mu
-# CHECK-NEXT:  1      28    28.00   *                   vluxseg2ei16.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m4, tu, mu
-# CHECK-NEXT:  1      28    28.00   *                   vluxseg2ei16.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m8, tu, mu
-# CHECK-NEXT:  1      140   140.00  *                   vluxseg2ei16.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf8, tu, mu
-# CHECK-NEXT:  1      16    16.00   *                   vluxseg2ei32.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf4, tu, mu
-# CHECK-NEXT:  1      20    20.00   *                   vluxseg2ei32.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf2, tu, mu
-# CHECK-NEXT:  1      20    20.00   *                   vluxseg2ei32.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m1, tu, mu
-# CHECK-NEXT:  1      20    20.00   *                   vluxseg2ei32.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m2, tu, mu
-# CHECK-NEXT:  1      20    20.00   *                   vluxseg2ei32.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m4, tu, mu
-# CHECK-NEXT:  1      28    28.00   *                   vluxseg2ei32.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m8, tu, mu
-# CHECK-NEXT:  1      76    76.00   *                   vluxseg2ei32.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf8, tu, mu
-# CHECK-NEXT:  1      16    16.00   *                   vluxseg2ei64.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf4, tu, mu
-# CHECK-NEXT:  1      16    16.00   *                   vluxseg2ei64.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf2, tu, mu
-# CHECK-NEXT:  1      16    16.00   *                   vluxseg2ei64.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m1, tu, mu
-# CHECK-NEXT:  1      16    16.00   *                   vluxseg2ei64.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m2, tu, mu
-# CHECK-NEXT:  1      20    20.00   *                   vluxseg2ei64.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m4, tu, mu
-# CHECK-NEXT:  1      28    28.00   *                   vluxseg2ei64.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m8, tu, mu
-# CHECK-NEXT:  1      44    44.00   *                   vluxseg2ei64.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf8, tu, mu
-# CHECK-NEXT:  1      18    18.00   *                   vluxseg3ei8.v v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf4, tu, mu
-# CHECK-NEXT:  1      24    24.00   *                   vluxseg3ei8.v v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf2, tu, mu
-# CHECK-NEXT:  1      36    36.00   *                   vluxseg3ei8.v v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m1, tu, mu
-# CHECK-NEXT:  1      60    60.00   *                   vluxseg3ei8.v v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m2, tu, mu
-# CHECK-NEXT:  1      60    60.00   *                   vluxseg3ei8.v v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m4, tu, mu
-# CHECK-NEXT:  1      396   396.00  *                   vluxseg3ei8.v v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m8, tu, mu
-# CHECK-NEXT:  1      396   396.00  *                   vluxseg3ei8.v v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf8, tu, mu
-# CHECK-NEXT:  1      18    18.00   *                   vluxseg3ei16.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf4, tu, mu
-# CHECK-NEXT:  1      24    24.00   *                   vluxseg3ei16.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf2, tu, mu
-# CHECK-NEXT:  1      36    36.00   *                   vluxseg3ei16.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m1, tu, mu
-# CHECK-NEXT:  1      36    36.00   *                   vluxseg3ei16.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m2, tu, mu
-# CHECK-NEXT:  1      36    36.00   *                   vluxseg3ei16.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m4, tu, mu
-# CHECK-NEXT:  1      204   204.00  *                   vluxseg3ei16.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m8, tu, mu
-# CHECK-NEXT:  1      204   204.00  *                   vluxseg3ei16.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf8, tu, mu
-# CHECK-NEXT:  1      18    18.00   *                   vluxseg3ei32.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf4, tu, mu
-# CHECK-NEXT:  1      24    24.00   *                   vluxseg3ei32.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf2, tu, mu
-# CHECK-NEXT:  1      24    24.00   *                   vluxseg3ei32.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m1, tu, mu
-# CHECK-NEXT:  1      24    24.00   *                   vluxseg3ei32.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m2, tu, mu
-# CHECK-NEXT:  1      24    24.00   *                   vluxseg3ei32.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m4, tu, mu
-# CHECK-NEXT:  1      108   108.00  *                   vluxseg3ei32.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m8, tu, mu
-# CHECK-NEXT:  1      108   108.00  *                   vluxseg3ei32.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf8, tu, mu
-# CHECK-NEXT:  1      18    18.00   *                   vluxseg3ei64.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf4, tu, mu
-# CHECK-NEXT:  1      18    18.00   *                   vluxseg3ei64.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf2, tu, mu
-# CHECK-NEXT:  1      18    18.00   *                   vluxseg3ei64.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m1, tu, mu
-# CHECK-NEXT:  1      18    18.00   *                   vluxseg3ei64.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m2, tu, mu
-# CHECK-NEXT:  1      24    24.00   *                   vluxseg3ei64.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m4, tu, mu
-# CHECK-NEXT:  1      60    60.00   *                   vluxseg3ei64.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m8, tu, mu
-# CHECK-NEXT:  1      60    60.00   *                   vluxseg3ei64.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf8, tu, mu
-# CHECK-NEXT:  1      20    20.00   *                   vluxseg4ei8.v v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf4, tu, mu
-# CHECK-NEXT:  1      28    28.00   *                   vluxseg4ei8.v v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf2, tu, mu
-# CHECK-NEXT:  1      44    44.00   *                   vluxseg4ei8.v v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m1, tu, mu
-# CHECK-NEXT:  1      76    76.00   *                   vluxseg4ei8.v v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m2, tu, mu
-# CHECK-NEXT:  1      76    76.00   *                   vluxseg4ei8.v v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m4, tu, mu
-# CHECK-NEXT:  1      524   524.00  *                   vluxseg4ei8.v v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m8, tu, mu
-# CHECK-NEXT:  1      524   524.00  *                   vluxseg4ei8.v v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf8, tu, mu
-# CHECK-NEXT:  1      20    20.00   *                   vluxseg4ei16.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf4, tu, mu
-# CHECK-NEXT:  1      28    28.00   *                   vluxseg4ei16.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf2, tu, mu
-# CHECK-NEXT:  1      44    44.00   *                   vluxseg4ei16.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m1, tu, mu
-# CHECK-NEXT:  1      44    44.00   *                   vluxseg4ei16.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m2, tu, mu
-# CHECK-NEXT:  1      44    44.00   *                   vluxseg4ei16.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m4, tu, mu
-# CHECK-NEXT:  1      268   268.00  *                   vluxseg4ei16.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m8, tu, mu
-# CHECK-NEXT:  1      268   268.00  *                   vluxseg4ei16.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf8, tu, mu
-# CHECK-NEXT:  1      20    20.00   *                   vluxseg4ei32.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf4, tu, mu
-# CHECK-NEXT:  1      28    28.00   *                   vluxseg4ei32.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf2, tu, mu
-# CHECK-NEXT:  1      28    28.00   *                   vluxseg4ei32.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m1, tu, mu
-# CHECK-NEXT:  1      28    28.00   *                   vluxseg4ei32.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m2, tu, mu
-# CHECK-NEXT:  1      28    28.00   *                   vluxseg4ei32.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m4, tu, mu
-# CHECK-NEXT:  1      140   140.00  *                   vluxseg4ei32.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m8, tu, mu
-# CHECK-NEXT:  1      140   140.00  *                   vluxseg4ei32.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf8, tu, mu
-# CHECK-NEXT:  1      20    20.00   *                   vluxseg4ei64.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf4, tu, mu
-# CHECK-NEXT:  1      20    20.00   *                   vluxseg4ei64.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf2, tu, mu
-# CHECK-NEXT:  1      20    20.00   *                   vluxseg4ei64.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m1, tu, mu
-# CHECK-NEXT:  1      20    20.00   *                   vluxseg4ei64.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m2, tu, mu
-# CHECK-NEXT:  1      28    28.00   *                   vluxseg4ei64.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m4, tu, mu
-# CHECK-NEXT:  1      76    76.00   *                   vluxseg4ei64.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m8, tu, mu
-# CHECK-NEXT:  1      76    76.00   *                   vluxseg4ei64.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf8, tu, mu
-# CHECK-NEXT:  1      22    22.00   *                   vluxseg5ei8.v v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf4, tu, mu
-# CHECK-NEXT:  1      32    32.00   *                   vluxseg5ei8.v v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf2, tu, mu
-# CHECK-NEXT:  1      52    52.00   *                   vluxseg5ei8.v v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m1, tu, mu
-# CHECK-NEXT:  1      92    92.00   *                   vluxseg5ei8.v v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m2, tu, mu
-# CHECK-NEXT:  1      652   652.00  *                   vluxseg5ei8.v v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m4, tu, mu
-# CHECK-NEXT:  1      652   652.00  *                   vluxseg5ei8.v v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf8, tu, mu
-# CHECK-NEXT:  1      22    22.00   *                   vluxseg5ei16.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf4, tu, mu
-# CHECK-NEXT:  1      32    32.00   *                   vluxseg5ei16.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf2, tu, mu
-# CHECK-NEXT:  1      52    52.00   *                   vluxseg5ei16.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m1, tu, mu
-# CHECK-NEXT:  1      52    52.00   *                   vluxseg5ei16.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m2, tu, mu
-# CHECK-NEXT:  1      332   332.00  *                   vluxseg5ei16.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m4, tu, mu
-# CHECK-NEXT:  1      332   332.00  *                   vluxseg5ei16.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf8, tu, mu
-# CHECK-NEXT:  1      22    22.00   *                   vluxseg5ei32.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf4, tu, mu
-# CHECK-NEXT:  1      32    32.00   *                   vluxseg5ei32.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf2, tu, mu
-# CHECK-NEXT:  1      32    32.00   *                   vluxseg5ei32.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m1, tu, mu
-# CHECK-NEXT:  1      32    32.00   *                   vluxseg5ei32.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m2, tu, mu
-# CHECK-NEXT:  1      172   172.00  *                   vluxseg5ei32.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m4, tu, mu
-# CHECK-NEXT:  1      172   172.00  *                   vluxseg5ei32.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf8, tu, mu
-# CHECK-NEXT:  1      22    22.00   *                   vluxseg5ei64.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf4, tu, mu
-# CHECK-NEXT:  1      22    22.00   *                   vluxseg5ei64.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf2, tu, mu
-# CHECK-NEXT:  1      22    22.00   *                   vluxseg5ei64.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m1, tu, mu
-# CHECK-NEXT:  1      22    22.00   *                   vluxseg5ei64.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m2, tu, mu
-# CHECK-NEXT:  1      92    92.00   *                   vluxseg5ei64.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m4, tu, mu
-# CHECK-NEXT:  1      92    92.00   *                   vluxseg5ei64.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf8, tu, mu
-# CHECK-NEXT:  1      24    24.00   *                   vluxseg6ei8.v v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf4, tu, mu
-# CHECK-NEXT:  1      36    36.00   *                   vluxseg6ei8.v v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf2, tu, mu
-# CHECK-NEXT:  1      60    60.00   *                   vluxseg6ei8.v v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m1, tu, mu
-# CHECK-NEXT:  1      108   108.00  *                   vluxseg6ei8.v v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m2, tu, mu
-# CHECK-NEXT:  1      780   780.00  *                   vluxseg6ei8.v v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m4, tu, mu
-# CHECK-NEXT:  1      780   780.00  *                   vluxseg6ei8.v v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf8, tu, mu
-# CHECK-NEXT:  1      24    24.00   *                   vluxseg6ei16.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf4, tu, mu
-# CHECK-NEXT:  1      36    36.00   *                   vluxseg6ei16.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf2, tu, mu
-# CHECK-NEXT:  1      60    60.00   *                   vluxseg6ei16.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m1, tu, mu
-# CHECK-NEXT:  1      60    60.00   *                   vluxseg6ei16.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m2, tu, mu
-# CHECK-NEXT:  1      396   396.00  *                   vluxseg6ei16.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m4, tu, mu
-# CHECK-NEXT:  1      396   396.00  *                   vluxseg6ei16.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf8, tu, mu
-# CHECK-NEXT:  1      24    24.00   *                   vluxseg6ei32.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf4, tu, mu
-# CHECK-NEXT:  1      36    36.00   *                   vluxseg6ei32.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf2, tu, mu
-# CHECK-NEXT:  1      36    36.00   *                   vluxseg6ei32.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m1, tu, mu
-# CHECK-NEXT:  1      36    36.00   *                   vluxseg6ei32.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m2, tu, mu
-# CHECK-NEXT:  1      204   204.00  *                   vluxseg6ei32.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m4, tu, mu
-# CHECK-NEXT:  1      204   204.00  *                   vluxseg6ei32.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf8, tu, mu
-# CHECK-NEXT:  1      24    24.00   *                   vluxseg6ei64.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf4, tu, mu
-# CHECK-NEXT:  1      24    24.00   *                   vluxseg6ei64.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf2, tu, mu
-# CHECK-NEXT:  1      24    24.00   *                   vluxseg6ei64.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m1, tu, mu
-# CHECK-NEXT:  1      24    24.00   *                   vluxseg6ei64.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m2, tu, mu
-# CHECK-NEXT:  1      108   108.00  *                   vluxseg6ei64.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m4, tu, mu
-# CHECK-NEXT:  1      108   108.00  *                   vluxseg6ei64.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf8, tu, mu
-# CHECK-NEXT:  1      26    26.00   *                   vluxseg7ei8.v v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf4, tu, mu
-# CHECK-NEXT:  1      40    40.00   *                   vluxseg7ei8.v v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf2, tu, mu
-# CHECK-NEXT:  1      68    68.00   *                   vluxseg7ei8.v v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m1, tu, mu
-# CHECK-NEXT:  1      124   124.00  *                   vluxseg7ei8.v v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m2, tu, mu
-# CHECK-NEXT:  1      908   908.00  *                   vluxseg7ei8.v v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m4, tu, mu
-# CHECK-NEXT:  1      908   908.00  *                   vluxseg7ei8.v v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf8, tu, mu
-# CHECK-NEXT:  1      26    26.00   *                   vluxseg7ei16.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf4, tu, mu
-# CHECK-NEXT:  1      40    40.00   *                   vluxseg7ei16.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf2, tu, mu
-# CHECK-NEXT:  1      68    68.00   *                   vluxseg7ei16.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m1, tu, mu
-# CHECK-NEXT:  1      68    68.00   *                   vluxseg7ei16.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m2, tu, mu
-# CHECK-NEXT:  1      460   460.00  *                   vluxseg7ei16.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m4, tu, mu
-# CHECK-NEXT:  1      460   460.00  *                   vluxseg7ei16.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf8, tu, mu
-# CHECK-NEXT:  1      26    26.00   *                   vluxseg7ei32.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf4, tu, mu
-# CHECK-NEXT:  1      40    40.00   *                   vluxseg7ei32.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf2, tu, mu
-# CHECK-NEXT:  1      40    40.00   *                   vluxseg7ei32.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m1, tu, mu
-# CHECK-NEXT:  1      40    40.00   *                   vluxseg7ei32.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m2, tu, mu
-# CHECK-NEXT:  1      236   236.00  *                   vluxseg7ei32.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m4, tu, mu
-# CHECK-NEXT:  1      236   236.00  *                   vluxseg7ei32.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf8, tu, mu
-# CHECK-NEXT:  1      26    26.00   *                   vluxseg7ei64.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf4, tu, mu
-# CHECK-NEXT:  1      26    26.00   *                   vluxseg7ei64.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf2, tu, mu
-# CHECK-NEXT:  1      26    26.00   *                   vluxseg7ei64.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m1, tu, mu
-# CHECK-NEXT:  1      26    26.00   *                   vluxseg7ei64.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m2, tu, mu
-# CHECK-NEXT:  1      124   124.00  *                   vluxseg7ei64.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m4, tu, mu
-# CHECK-NEXT:  1      124   124.00  *                   vluxseg7ei64.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf8, tu, mu
-# CHECK-NEXT:  1      28    28.00   *                   vluxseg8ei8.v v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf4, tu, mu
-# CHECK-NEXT:  1      44    44.00   *                   vluxseg8ei8.v v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf2, tu, mu
-# CHECK-NEXT:  1      76    76.00   *                   vluxseg8ei8.v v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m1, tu, mu
-# CHECK-NEXT:  1      140   140.00  *                   vluxseg8ei8.v v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m2, tu, mu
-# CHECK-NEXT:  1      1036   1036.00  *                   vluxseg8ei8.v v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m4, tu, mu
-# CHECK-NEXT:  1      1036   1036.00  *                   vluxseg8ei8.v v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf8, tu, mu
-# CHECK-NEXT:  1      28    28.00   *                   vluxseg8ei16.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf4, tu, mu
-# CHECK-NEXT:  1      44    44.00   *                   vluxseg8ei16.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf2, tu, mu
-# CHECK-NEXT:  1      76    76.00   *                   vluxseg8ei16.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m1, tu, mu
-# CHECK-NEXT:  1      76    76.00   *                   vluxseg8ei16.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m2, tu, mu
-# CHECK-NEXT:  1      524   524.00  *                   vluxseg8ei16.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m4, tu, mu
-# CHECK-NEXT:  1      524   524.00  *                   vluxseg8ei16.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf8, tu, mu
-# CHECK-NEXT:  1      28    28.00   *                   vluxseg8ei32.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf4, tu, mu
-# CHECK-NEXT:  1      44    44.00   *                   vluxseg8ei32.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf2, tu, mu
-# CHECK-NEXT:  1      44    44.00   *                   vluxseg8ei32.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m1, tu, mu
-# CHECK-NEXT:  1      44    44.00   *                   vluxseg8ei32.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m2, tu, mu
-# CHECK-NEXT:  1      268   268.00  *                   vluxseg8ei32.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m4, tu, mu
-# CHECK-NEXT:  1      268   268.00  *                   vluxseg8ei32.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf8, tu, mu
-# CHECK-NEXT:  1      28    28.00   *                   vluxseg8ei64.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf4, tu, mu
-# CHECK-NEXT:  1      28    28.00   *                   vluxseg8ei64.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf2, tu, mu
-# CHECK-NEXT:  1      28    28.00   *                   vluxseg8ei64.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m1, tu, mu
-# CHECK-NEXT:  1      28    28.00   *                   vluxseg8ei64.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m2, tu, mu
-# CHECK-NEXT:  1      140   140.00  *                   vluxseg8ei64.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m4, tu, mu
-# CHECK-NEXT:  1      140   140.00  *                   vluxseg8ei64.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf8, tu, mu
-# CHECK-NEXT:  1      16    16.00   *                   vloxseg2ei8.v v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf4, tu, mu
-# CHECK-NEXT:  1      20    20.00   *                   vloxseg2ei8.v v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf2, tu, mu
-# CHECK-NEXT:  1      28    28.00   *                   vloxseg2ei8.v v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m1, tu, mu
-# CHECK-NEXT:  1      44    44.00   *                   vloxseg2ei8.v v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m2, tu, mu
-# CHECK-NEXT:  1      44    44.00   *                   vloxseg2ei8.v v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m4, tu, mu
-# CHECK-NEXT:  1      44    44.00   *                   vloxseg2ei8.v v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m8, tu, mu
-# CHECK-NEXT:  1      268   268.00  *                   vloxseg2ei8.v v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf8, tu, mu
-# CHECK-NEXT:  1      16    16.00   *                   vloxseg2ei16.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf4, tu, mu
-# CHECK-NEXT:  1      20    20.00   *                   vloxseg2ei16.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf2, tu, mu
-# CHECK-NEXT:  1      28    28.00   *                   vloxseg2ei16.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m1, tu, mu
-# CHECK-NEXT:  1      28    28.00   *                   vloxseg2ei16.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m2, tu, mu
-# CHECK-NEXT:  1      28    28.00   *                   vloxseg2ei16.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m4, tu, mu
-# CHECK-NEXT:  1      28    28.00   *                   vloxseg2ei16.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m8, tu, mu
-# CHECK-NEXT:  1      140   140.00  *                   vloxseg2ei16.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf8, tu, mu
-# CHECK-NEXT:  1      16    16.00   *                   vloxseg2ei32.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf4, tu, mu
-# CHECK-NEXT:  1      20    20.00   *                   vloxseg2ei32.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf2, tu, mu
-# CHECK-NEXT:  1      20    20.00   *                   vloxseg2ei32.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m1, tu, mu
-# CHECK-NEXT:  1      20    20.00   *                   vloxseg2ei32.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m2, tu, mu
-# CHECK-NEXT:  1      20    20.00   *                   vloxseg2ei32.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m4, tu, mu
-# CHECK-NEXT:  1      28    28.00   *                   vloxseg2ei32.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m8, tu, mu
-# CHECK-NEXT:  1      76    76.00   *                   vloxseg2ei32.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf8, tu, mu
-# CHECK-NEXT:  1      16    16.00   *                   vloxseg2ei64.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf4, tu, mu
-# CHECK-NEXT:  1      16    16.00   *                   vloxseg2ei64.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf2, tu, mu
-# CHECK-NEXT:  1      16    16.00   *                   vloxseg2ei64.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m1, tu, mu
-# CHECK-NEXT:  1      16    16.00   *                   vloxseg2ei64.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m2, tu, mu
-# CHECK-NEXT:  1      20    20.00   *                   vloxseg2ei64.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m4, tu, mu
-# CHECK-NEXT:  1      28    28.00   *                   vloxseg2ei64.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m8, tu, mu
-# CHECK-NEXT:  1      44    44.00   *                   vloxseg2ei64.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf8, tu, mu
-# CHECK-NEXT:  1      18    18.00   *                   vloxseg3ei8.v v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf4, tu, mu
-# CHECK-NEXT:  1      24    24.00   *                   vloxseg3ei8.v v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf2, tu, mu
-# CHECK-NEXT:  1      36    36.00   *                   vloxseg3ei8.v v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m1, tu, mu
-# CHECK-NEXT:  1      60    60.00   *                   vloxseg3ei8.v v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m2, tu, mu
-# CHECK-NEXT:  1      60    60.00   *                   vloxseg3ei8.v v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m4, tu, mu
-# CHECK-NEXT:  1      396   396.00  *                   vloxseg3ei8.v v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m8, tu, mu
-# CHECK-NEXT:  1      396   396.00  *                   vloxseg3ei8.v v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf8, tu, mu
-# CHECK-NEXT:  1      18    18.00   *                   vloxseg3ei16.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf4, tu, mu
-# CHECK-NEXT:  1      24    24.00   *                   vloxseg3ei16.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf2, tu, mu
-# CHECK-NEXT:  1      36    36.00   *                   vloxseg3ei16.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m1, tu, mu
-# CHECK-NEXT:  1      36    36.00   *                   vloxseg3ei16.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m2, tu, mu
-# CHECK-NEXT:  1      36    36.00   *                   vloxseg3ei16.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m4, tu, mu
-# CHECK-NEXT:  1      204   204.00  *                   vloxseg3ei16.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m8, tu, mu
-# CHECK-NEXT:  1      204   204.00  *                   vloxseg3ei16.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf8, tu, mu
-# CHECK-NEXT:  1      18    18.00   *                   vloxseg3ei32.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf4, tu, mu
-# CHECK-NEXT:  1      24    24.00   *                   vloxseg3ei32.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf2, tu, mu
-# CHECK-NEXT:  1      24    24.00   *                   vloxseg3ei32.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m1, tu, mu
-# CHECK-NEXT:  1      24    24.00   *                   vloxseg3ei32.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m2, tu, mu
-# CHECK-NEXT:  1      24    24.00   *                   vloxseg3ei32.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m4, tu, mu
-# CHECK-NEXT:  1      108   108.00  *                   vloxseg3ei32.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m8, tu, mu
-# CHECK-NEXT:  1      108   108.00  *                   vloxseg3ei32.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf8, tu, mu
-# CHECK-NEXT:  1      18    18.00   *                   vloxseg3ei64.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf4, tu, mu
-# CHECK-NEXT:  1      18    18.00   *                   vloxseg3ei64.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf2, tu, mu
-# CHECK-NEXT:  1      18    18.00   *                   vloxseg3ei64.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m1, tu, mu
-# CHECK-NEXT:  1      18    18.00   *                   vloxseg3ei64.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m2, tu, mu
-# CHECK-NEXT:  1      24    24.00   *                   vloxseg3ei64.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m4, tu, mu
-# CHECK-NEXT:  1      60    60.00   *                   vloxseg3ei64.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m8, tu, mu
-# CHECK-NEXT:  1      60    60.00   *                   vloxseg3ei64.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf8, tu, mu
-# CHECK-NEXT:  1      20    20.00   *                   vloxseg4ei8.v v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf4, tu, mu
-# CHECK-NEXT:  1      28    28.00   *                   vloxseg4ei8.v v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf2, tu, mu
-# CHECK-NEXT:  1      44    44.00   *                   vloxseg4ei8.v v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m1, tu, mu
-# CHECK-NEXT:  1      76    76.00   *                   vloxseg4ei8.v v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m2, tu, mu
-# CHECK-NEXT:  1      76    76.00   *                   vloxseg4ei8.v v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m4, tu, mu
-# CHECK-NEXT:  1      524   524.00  *                   vloxseg4ei8.v v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m8, tu, mu
-# CHECK-NEXT:  1      524   524.00  *                   vloxseg4ei8.v v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf8, tu, mu
-# CHECK-NEXT:  1      20    20.00   *                   vloxseg4ei16.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf4, tu, mu
-# CHECK-NEXT:  1      28    28.00   *                   vloxseg4ei16.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf2, tu, mu
-# CHECK-NEXT:  1      44    44.00   *                   vloxseg4ei16.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m1, tu, mu
-# CHECK-NEXT:  1      44    44.00   *                   vloxseg4ei16.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m2, tu, mu
-# CHECK-NEXT:  1      44    44.00   *                   vloxseg4ei16.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m4, tu, mu
-# CHECK-NEXT:  1      268   268.00  *                   vloxseg4ei16.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m8, tu, mu
-# CHECK-NEXT:  1      268   268.00  *                   vloxseg4ei16.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf8, tu, mu
-# CHECK-NEXT:  1      20    20.00   *                   vloxseg4ei32.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf4, tu, mu
-# CHECK-NEXT:  1      28    28.00   *                   vloxseg4ei32.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf2, tu, mu
-# CHECK-NEXT:  1      28    28.00   *                   vloxseg4ei32.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m1, tu, mu
-# CHECK-NEXT:  1      28    28.00   *                   vloxseg4ei32.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m2, tu, mu
-# CHECK-NEXT:  1      28    28.00   *                   vloxseg4ei32.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m4, tu, mu
-# CHECK-NEXT:  1      140   140.00  *                   vloxseg4ei32.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m8, tu, mu
-# CHECK-NEXT:  1      140   140.00  *                   vloxseg4ei32.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf8, tu, mu
-# CHECK-NEXT:  1      20    20.00   *                   vloxseg4ei64.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf4, tu, mu
-# CHECK-NEXT:  1      20    20.00   *                   vloxseg4ei64.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf2, tu, mu
-# CHECK-NEXT:  1      20    20.00   *                   vloxseg4ei64.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m1, tu, mu
-# CHECK-NEXT:  1      20    20.00   *                   vloxseg4ei64.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m2, tu, mu
-# CHECK-NEXT:  1      28    28.00   *                   vloxseg4ei64.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m4, tu, mu
-# CHECK-NEXT:  1      76    76.00   *                   vloxseg4ei64.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m8, tu, mu
-# CHECK-NEXT:  1      76    76.00   *                   vloxseg4ei64.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf8, tu, mu
-# CHECK-NEXT:  1      22    22.00   *                   vloxseg5ei8.v v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf4, tu, mu
-# CHECK-NEXT:  1      32    32.00   *                   vloxseg5ei8.v v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf2, tu, mu
-# CHECK-NEXT:  1      52    52.00   *                   vloxseg5ei8.v v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m1, tu, mu
-# CHECK-NEXT:  1      92    92.00   *                   vloxseg5ei8.v v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m2, tu, mu
-# CHECK-NEXT:  1      652   652.00  *                   vloxseg5ei8.v v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m4, tu, mu
-# CHECK-NEXT:  1      652   652.00  *                   vloxseg5ei8.v v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf8, tu, mu
-# CHECK-NEXT:  1      22    22.00   *                   vloxseg5ei16.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf4, tu, mu
-# CHECK-NEXT:  1      32    32.00   *                   vloxseg5ei16.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf2, tu, mu
-# CHECK-NEXT:  1      52    52.00   *                   vloxseg5ei16.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m1, tu, mu
-# CHECK-NEXT:  1      52    52.00   *                   vloxseg5ei16.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m2, tu, mu
-# CHECK-NEXT:  1      332   332.00  *                   vloxseg5ei16.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m4, tu, mu
-# CHECK-NEXT:  1      332   332.00  *                   vloxseg5ei16.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf8, tu, mu
-# CHECK-NEXT:  1      22    22.00   *                   vloxseg5ei32.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf4, tu, mu
-# CHECK-NEXT:  1      32    32.00   *                   vloxseg5ei32.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf2, tu, mu
-# CHECK-NEXT:  1      32    32.00   *                   vloxseg5ei32.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m1, tu, mu
-# CHECK-NEXT:  1      32    32.00   *                   vloxseg5ei32.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m2, tu, mu
-# CHECK-NEXT:  1      172   172.00  *                   vloxseg5ei32.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m4, tu, mu
-# CHECK-NEXT:  1      172   172.00  *                   vloxseg5ei32.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf8, tu, mu
-# CHECK-NEXT:  1      22    22.00   *                   vloxseg5ei64.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf4, tu, mu
-# CHECK-NEXT:  1      22    22.00   *                   vloxseg5ei64.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf2, tu, mu
-# CHECK-NEXT:  1      22    22.00   *                   vloxseg5ei64.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m1, tu, mu
-# CHECK-NEXT:  1      22    22.00   *                   vloxseg5ei64.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m2, tu, mu
-# CHECK-NEXT:  1      92    92.00   *                   vloxseg5ei64.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m4, tu, mu
-# CHECK-NEXT:  1      92    92.00   *                   vloxseg5ei64.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf8, tu, mu
-# CHECK-NEXT:  1      24    24.00   *                   vloxseg6ei8.v v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf4, tu, mu
-# CHECK-NEXT:  1      36    36.00   *                   vloxseg6ei8.v v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf2, tu, mu
-# CHECK-NEXT:  1      60    60.00   *                   vloxseg6ei8.v v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m1, tu, mu
-# CHECK-NEXT:  1      108   108.00  *                   vloxseg6ei8.v v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m2, tu, mu
-# CHECK-NEXT:  1      780   780.00  *                   vloxseg6ei8.v v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m4, tu, mu
-# CHECK-NEXT:  1      780   780.00  *                   vloxseg6ei8.v v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf8, tu, mu
-# CHECK-NEXT:  1      24    24.00   *                   vloxseg6ei16.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf4, tu, mu
-# CHECK-NEXT:  1      36    36.00   *                   vloxseg6ei16.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf2, tu, mu
-# CHECK-NEXT:  1      60    60.00   *                   vloxseg6ei16.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m1, tu, mu
-# CHECK-NEXT:  1      60    60.00   *                   vloxseg6ei16.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m2, tu, mu
-# CHECK-NEXT:  1      396   396.00  *                   vloxseg6ei16.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m4, tu, mu
-# CHECK-NEXT:  1      396   396.00  *                   vloxseg6ei16.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf8, tu, mu
-# CHECK-NEXT:  1      24    24.00   *                   vloxseg6ei32.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf4, tu, mu
-# CHECK-NEXT:  1      36    36.00   *                   vloxseg6ei32.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf2, tu, mu
-# CHECK-NEXT:  1      36    36.00   *                   vloxseg6ei32.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m1, tu, mu
-# CHECK-NEXT:  1      36    36.00   *                   vloxseg6ei32.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m2, tu, mu
-# CHECK-NEXT:  1      204   204.00  *                   vloxseg6ei32.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m4, tu, mu
-# CHECK-NEXT:  1      204   204.00  *                   vloxseg6ei32.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf8, tu, mu
-# CHECK-NEXT:  1      24    24.00   *                   vloxseg6ei64.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf4, tu, mu
-# CHECK-NEXT:  1      24    24.00   *                   vloxseg6ei64.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf2, tu, mu
-# CHECK-NEXT:  1      24    24.00   *                   vloxseg6ei64.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m1, tu, mu
-# CHECK-NEXT:  1      24    24.00   *                   vloxseg6ei64.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m2, tu, mu
-# CHECK-NEXT:  1      108   108.00  *                   vloxseg6ei64.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m4, tu, mu
-# CHECK-NEXT:  1      108   108.00  *                   vloxseg6ei64.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf8, tu, mu
-# CHECK-NEXT:  1      26    26.00   *                   vloxseg7ei8.v v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf4, tu, mu
-# CHECK-NEXT:  1      40    40.00   *                   vloxseg7ei8.v v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf2, tu, mu
-# CHECK-NEXT:  1      68    68.00   *                   vloxseg7ei8.v v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m1, tu, mu
-# CHECK-NEXT:  1      124   124.00  *                   vloxseg7ei8.v v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m2, tu, mu
-# CHECK-NEXT:  1      908   908.00  *                   vloxseg7ei8.v v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m4, tu, mu
-# CHECK-NEXT:  1      908   908.00  *                   vloxseg7ei8.v v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf8, tu, mu
-# CHECK-NEXT:  1      26    26.00   *                   vloxseg7ei16.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf4, tu, mu
-# CHECK-NEXT:  1      40    40.00   *                   vloxseg7ei16.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf2, tu, mu
-# CHECK-NEXT:  1      68    68.00   *                   vloxseg7ei16.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m1, tu, mu
-# CHECK-NEXT:  1      68    68.00   *                   vloxseg7ei16.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m2, tu, mu
-# CHECK-NEXT:  1      460   460.00  *                   vloxseg7ei16.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m4, tu, mu
-# CHECK-NEXT:  1      460   460.00  *                   vloxseg7ei16.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf8, tu, mu
-# CHECK-NEXT:  1      26    26.00   *                   vloxseg7ei32.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf4, tu, mu
-# CHECK-NEXT:  1      40    40.00   *                   vloxseg7ei32.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf2, tu, mu
-# CHECK-NEXT:  1      40    40.00   *                   vloxseg7ei32.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m1, tu, mu
-# CHECK-NEXT:  1      40    40.00   *                   vloxseg7ei32.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m2, tu, mu
-# CHECK-NEXT:  1      236   236.00  *                   vloxseg7ei32.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m4, tu, mu
-# CHECK-NEXT:  1      236   236.00  *                   vloxseg7ei32.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf8, tu, mu
-# CHECK-NEXT:  1      26    26.00   *                   vloxseg7ei64.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf4, tu, mu
-# CHECK-NEXT:  1      26    26.00   *                   vloxseg7ei64.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf2, tu, mu
-# CHECK-NEXT:  1      26    26.00   *                   vloxseg7ei64.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m1, tu, mu
-# CHECK-NEXT:  1      26    26.00   *                   vloxseg7ei64.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m2, tu, mu
-# CHECK-NEXT:  1      124   124.00  *                   vloxseg7ei64.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m4, tu, mu
-# CHECK-NEXT:  1      124   124.00  *                   vloxseg7ei64.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf8, tu, mu
-# CHECK-NEXT:  1      28    28.00   *                   vloxseg8ei8.v v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf4, tu, mu
-# CHECK-NEXT:  1      44    44.00   *                   vloxseg8ei8.v v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf2, tu, mu
-# CHECK-NEXT:  1      76    76.00   *                   vloxseg8ei8.v v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m1, tu, mu
-# CHECK-NEXT:  1      140   140.00  *                   vloxseg8ei8.v v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m2, tu, mu
-# CHECK-NEXT:  1      1036   1036.00  *                   vloxseg8ei8.v v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m4, tu, mu
-# CHECK-NEXT:  1      1036   1036.00  *                   vloxseg8ei8.v v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf8, tu, mu
-# CHECK-NEXT:  1      28    28.00   *                   vloxseg8ei16.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf4, tu, mu
-# CHECK-NEXT:  1      44    44.00   *                   vloxseg8ei16.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf2, tu, mu
-# CHECK-NEXT:  1      76    76.00   *                   vloxseg8ei16.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m1, tu, mu
-# CHECK-NEXT:  1      76    76.00   *                   vloxseg8ei16.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m2, tu, mu
-# CHECK-NEXT:  1      524   524.00  *                   vloxseg8ei16.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m4, tu, mu
-# CHECK-NEXT:  1      524   524.00  *                   vloxseg8ei16.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf8, tu, mu
-# CHECK-NEXT:  1      28    28.00   *                   vloxseg8ei32.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf4, tu, mu
-# CHECK-NEXT:  1      44    44.00   *                   vloxseg8ei32.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf2, tu, mu
-# CHECK-NEXT:  1      44    44.00   *                   vloxseg8ei32.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m1, tu, mu
-# CHECK-NEXT:  1      44    44.00   *                   vloxseg8ei32.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m2, tu, mu
-# CHECK-NEXT:  1      268   268.00  *                   vloxseg8ei32.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m4, tu, mu
-# CHECK-NEXT:  1      268   268.00  *                   vloxseg8ei32.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf8, tu, mu
-# CHECK-NEXT:  1      28    28.00   *                   vloxseg8ei64.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf4, tu, mu
-# CHECK-NEXT:  1      28    28.00   *                   vloxseg8ei64.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf2, tu, mu
-# CHECK-NEXT:  1      28    28.00   *                   vloxseg8ei64.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m1, tu, mu
-# CHECK-NEXT:  1      28    28.00   *                   vloxseg8ei64.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m2, tu, mu
-# CHECK-NEXT:  1      140   140.00  *                   vloxseg8ei64.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m4, tu, mu
-# CHECK-NEXT:  1      140   140.00  *                   vloxseg8ei64.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf8, tu, mu
-# CHECK-NEXT:  1      5     16.00          *            vsuxseg2ei8.v v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf4, tu, mu
-# CHECK-NEXT:  1      9     20.00          *            vsuxseg2ei8.v v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf2, tu, mu
-# CHECK-NEXT:  1      17    28.00          *            vsuxseg2ei8.v v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m1, tu, mu
-# CHECK-NEXT:  1      33    44.00          *            vsuxseg2ei8.v v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m2, tu, mu
-# CHECK-NEXT:  1      65    76.00          *            vsuxseg2ei8.v v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m4, tu, mu
-# CHECK-NEXT:  1      129   140.00         *            vsuxseg2ei8.v v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m8, tu, mu
-# CHECK-NEXT:  1      257   268.00         *            vsuxseg2ei8.v v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf8, tu, mu
-# CHECK-NEXT:  1      3     14.00          *            vsuxseg2ei16.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf4, tu, mu
-# CHECK-NEXT:  1      5     16.00          *            vsuxseg2ei16.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf2, tu, mu
-# CHECK-NEXT:  1      9     20.00          *            vsuxseg2ei16.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m1, tu, mu
-# CHECK-NEXT:  1      17    28.00          *            vsuxseg2ei16.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m2, tu, mu
-# CHECK-NEXT:  1      33    44.00          *            vsuxseg2ei16.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m4, tu, mu
-# CHECK-NEXT:  1      65    76.00          *            vsuxseg2ei16.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m8, tu, mu
-# CHECK-NEXT:  1      129   140.00         *            vsuxseg2ei16.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf8, tu, mu
-# CHECK-NEXT:  1      1     12.00          *            vsuxseg2ei32.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf4, tu, mu
-# CHECK-NEXT:  1      3     14.00          *            vsuxseg2ei32.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf2, tu, mu
-# CHECK-NEXT:  1      5     16.00          *            vsuxseg2ei32.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m1, tu, mu
-# CHECK-NEXT:  1      9     20.00          *            vsuxseg2ei32.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m2, tu, mu
-# CHECK-NEXT:  1      17    28.00          *            vsuxseg2ei32.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m4, tu, mu
-# CHECK-NEXT:  1      33    44.00          *            vsuxseg2ei32.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m8, tu, mu
-# CHECK-NEXT:  1      65    76.00          *            vsuxseg2ei32.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf8, tu, mu
-# CHECK-NEXT:  1      1     12.00          *            vsuxseg2ei64.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf4, tu, mu
-# CHECK-NEXT:  1      1     12.00          *            vsuxseg2ei64.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf2, tu, mu
-# CHECK-NEXT:  1      3     14.00          *            vsuxseg2ei64.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m1, tu, mu
-# CHECK-NEXT:  1      5     16.00          *            vsuxseg2ei64.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m2, tu, mu
-# CHECK-NEXT:  1      9     20.00          *            vsuxseg2ei64.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m4, tu, mu
-# CHECK-NEXT:  1      17    28.00          *            vsuxseg2ei64.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m8, tu, mu
-# CHECK-NEXT:  1      33    44.00          *            vsuxseg2ei64.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf8, tu, mu
-# CHECK-NEXT:  1      7     18.00          *            vsuxseg3ei8.v v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf4, tu, mu
-# CHECK-NEXT:  1      13    24.00          *            vsuxseg3ei8.v v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf2, tu, mu
-# CHECK-NEXT:  1      25    36.00          *            vsuxseg3ei8.v v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m1, tu, mu
-# CHECK-NEXT:  1      49    60.00          *            vsuxseg3ei8.v v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m2, tu, mu
-# CHECK-NEXT:  1      97    108.00         *            vsuxseg3ei8.v v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m4, tu, mu
-# CHECK-NEXT:  1      385   396.00         *            vsuxseg3ei8.v v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m8, tu, mu
-# CHECK-NEXT:  1      385   396.00         *            vsuxseg3ei8.v v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf8, tu, mu
-# CHECK-NEXT:  1      4     15.00          *            vsuxseg3ei16.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf4, tu, mu
-# CHECK-NEXT:  1      7     18.00          *            vsuxseg3ei16.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf2, tu, mu
-# CHECK-NEXT:  1      13    24.00          *            vsuxseg3ei16.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m1, tu, mu
-# CHECK-NEXT:  1      25    36.00          *            vsuxseg3ei16.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m2, tu, mu
-# CHECK-NEXT:  1      49    60.00          *            vsuxseg3ei16.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m4, tu, mu
-# CHECK-NEXT:  1      193   204.00         *            vsuxseg3ei16.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m8, tu, mu
-# CHECK-NEXT:  1      193   204.00         *            vsuxseg3ei16.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf8, tu, mu
-# CHECK-NEXT:  1      1     12.00          *            vsuxseg3ei32.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf4, tu, mu
-# CHECK-NEXT:  1      4     15.00          *            vsuxseg3ei32.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf2, tu, mu
-# CHECK-NEXT:  1      7     18.00          *            vsuxseg3ei32.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m1, tu, mu
-# CHECK-NEXT:  1      13    24.00          *            vsuxseg3ei32.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m2, tu, mu
-# CHECK-NEXT:  1      25    36.00          *            vsuxseg3ei32.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m4, tu, mu
-# CHECK-NEXT:  1      97    108.00         *            vsuxseg3ei32.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m8, tu, mu
-# CHECK-NEXT:  1      97    108.00         *            vsuxseg3ei32.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf8, tu, mu
-# CHECK-NEXT:  1      1     12.00          *            vsuxseg3ei64.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf4, tu, mu
-# CHECK-NEXT:  1      1     12.00          *            vsuxseg3ei64.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf2, tu, mu
-# CHECK-NEXT:  1      4     15.00          *            vsuxseg3ei64.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m1, tu, mu
-# CHECK-NEXT:  1      7     18.00          *            vsuxseg3ei64.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m2, tu, mu
-# CHECK-NEXT:  1      13    24.00          *            vsuxseg3ei64.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m4, tu, mu
-# CHECK-NEXT:  1      49    60.00          *            vsuxseg3ei64.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m8, tu, mu
-# CHECK-NEXT:  1      49    60.00          *            vsuxseg3ei64.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf8, tu, mu
-# CHECK-NEXT:  1      9     20.00          *            vsuxseg4ei8.v v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf4, tu, mu
-# CHECK-NEXT:  1      17    28.00          *            vsuxseg4ei8.v v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf2, tu, mu
-# CHECK-NEXT:  1      33    44.00          *            vsuxseg4ei8.v v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m1, tu, mu
-# CHECK-NEXT:  1      65    76.00          *            vsuxseg4ei8.v v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m2, tu, mu
-# CHECK-NEXT:  1      129   140.00         *            vsuxseg4ei8.v v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m4, tu, mu
-# CHECK-NEXT:  1      513   524.00         *            vsuxseg4ei8.v v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m8, tu, mu
-# CHECK-NEXT:  1      513   524.00         *            vsuxseg4ei8.v v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf8, tu, mu
-# CHECK-NEXT:  1      5     16.00          *            vsuxseg4ei16.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf4, tu, mu
-# CHECK-NEXT:  1      9     20.00          *            vsuxseg4ei16.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf2, tu, mu
-# CHECK-NEXT:  1      17    28.00          *            vsuxseg4ei16.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m1, tu, mu
-# CHECK-NEXT:  1      33    44.00          *            vsuxseg4ei16.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m2, tu, mu
-# CHECK-NEXT:  1      65    76.00          *            vsuxseg4ei16.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m4, tu, mu
-# CHECK-NEXT:  1      257   268.00         *            vsuxseg4ei16.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m8, tu, mu
-# CHECK-NEXT:  1      257   268.00         *            vsuxseg4ei16.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf8, tu, mu
-# CHECK-NEXT:  1      1     12.00          *            vsuxseg4ei32.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf4, tu, mu
-# CHECK-NEXT:  1      5     16.00          *            vsuxseg4ei32.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf2, tu, mu
-# CHECK-NEXT:  1      9     20.00          *            vsuxseg4ei32.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m1, tu, mu
-# CHECK-NEXT:  1      17    28.00          *            vsuxseg4ei32.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m2, tu, mu
-# CHECK-NEXT:  1      33    44.00          *            vsuxseg4ei32.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m4, tu, mu
-# CHECK-NEXT:  1      129   140.00         *            vsuxseg4ei32.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m8, tu, mu
-# CHECK-NEXT:  1      129   140.00         *            vsuxseg4ei32.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf8, tu, mu
-# CHECK-NEXT:  1      1     12.00          *            vsuxseg4ei64.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf4, tu, mu
-# CHECK-NEXT:  1      1     12.00          *            vsuxseg4ei64.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf2, tu, mu
-# CHECK-NEXT:  1      5     16.00          *            vsuxseg4ei64.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m1, tu, mu
-# CHECK-NEXT:  1      9     20.00          *            vsuxseg4ei64.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m2, tu, mu
-# CHECK-NEXT:  1      17    28.00          *            vsuxseg4ei64.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m4, tu, mu
-# CHECK-NEXT:  1      65    76.00          *            vsuxseg4ei64.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m8, tu, mu
-# CHECK-NEXT:  1      65    76.00          *            vsuxseg4ei64.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf8, tu, mu
-# CHECK-NEXT:  1      11    22.00          *            vsuxseg5ei8.v v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf4, tu, mu
-# CHECK-NEXT:  1      21    32.00          *            vsuxseg5ei8.v v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf2, tu, mu
-# CHECK-NEXT:  1      41    52.00          *            vsuxseg5ei8.v v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m1, tu, mu
-# CHECK-NEXT:  1      81    92.00          *            vsuxseg5ei8.v v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m2, tu, mu
-# CHECK-NEXT:  1      641   652.00         *            vsuxseg5ei8.v v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m4, tu, mu
-# CHECK-NEXT:  1      641   652.00         *            vsuxseg5ei8.v v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf8, tu, mu
-# CHECK-NEXT:  1      6     17.00          *            vsuxseg5ei16.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf4, tu, mu
-# CHECK-NEXT:  1      11    22.00          *            vsuxseg5ei16.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf2, tu, mu
-# CHECK-NEXT:  1      21    32.00          *            vsuxseg5ei16.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m1, tu, mu
-# CHECK-NEXT:  1      41    52.00          *            vsuxseg5ei16.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m2, tu, mu
-# CHECK-NEXT:  1      321   332.00         *            vsuxseg5ei16.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m4, tu, mu
-# CHECK-NEXT:  1      321   332.00         *            vsuxseg5ei16.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf8, tu, mu
-# CHECK-NEXT:  1      1     12.00          *            vsuxseg5ei32.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf4, tu, mu
-# CHECK-NEXT:  1      6     17.00          *            vsuxseg5ei32.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf2, tu, mu
-# CHECK-NEXT:  1      11    22.00          *            vsuxseg5ei32.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m1, tu, mu
-# CHECK-NEXT:  1      21    32.00          *            vsuxseg5ei32.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m2, tu, mu
-# CHECK-NEXT:  1      161   172.00         *            vsuxseg5ei32.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m4, tu, mu
-# CHECK-NEXT:  1      161   172.00         *            vsuxseg5ei32.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf8, tu, mu
-# CHECK-NEXT:  1      1     12.00          *            vsuxseg5ei64.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf4, tu, mu
-# CHECK-NEXT:  1      1     12.00          *            vsuxseg5ei64.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf2, tu, mu
-# CHECK-NEXT:  1      6     17.00          *            vsuxseg5ei64.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m1, tu, mu
-# CHECK-NEXT:  1      11    22.00          *            vsuxseg5ei64.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m2, tu, mu
-# CHECK-NEXT:  1      81    92.00          *            vsuxseg5ei64.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m4, tu, mu
-# CHECK-NEXT:  1      81    92.00          *            vsuxseg5ei64.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf8, tu, mu
-# CHECK-NEXT:  1      13    24.00          *            vsuxseg6ei8.v v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf4, tu, mu
-# CHECK-NEXT:  1      25    36.00          *            vsuxseg6ei8.v v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf2, tu, mu
-# CHECK-NEXT:  1      49    60.00          *            vsuxseg6ei8.v v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m1, tu, mu
-# CHECK-NEXT:  1      97    108.00         *            vsuxseg6ei8.v v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m2, tu, mu
-# CHECK-NEXT:  1      769   780.00         *            vsuxseg6ei8.v v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m4, tu, mu
-# CHECK-NEXT:  1      769   780.00         *            vsuxseg6ei8.v v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf8, tu, mu
-# CHECK-NEXT:  1      7     18.00          *            vsuxseg6ei16.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf4, tu, mu
-# CHECK-NEXT:  1      13    24.00          *            vsuxseg6ei16.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf2, tu, mu
-# CHECK-NEXT:  1      25    36.00          *            vsuxseg6ei16.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m1, tu, mu
-# CHECK-NEXT:  1      49    60.00          *            vsuxseg6ei16.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m2, tu, mu
-# CHECK-NEXT:  1      385   396.00         *            vsuxseg6ei16.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m4, tu, mu
-# CHECK-NEXT:  1      385   396.00         *            vsuxseg6ei16.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf8, tu, mu
-# CHECK-NEXT:  1      1     12.00          *            vsuxseg6ei32.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf4, tu, mu
-# CHECK-NEXT:  1      7     18.00          *            vsuxseg6ei32.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf2, tu, mu
-# CHECK-NEXT:  1      13    24.00          *            vsuxseg6ei32.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m1, tu, mu
-# CHECK-NEXT:  1      25    36.00          *            vsuxseg6ei32.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m2, tu, mu
-# CHECK-NEXT:  1      193   204.00         *            vsuxseg6ei32.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m4, tu, mu
-# CHECK-NEXT:  1      193   204.00         *            vsuxseg6ei32.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf8, tu, mu
-# CHECK-NEXT:  1      1     12.00          *            vsuxseg6ei64.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf4, tu, mu
-# CHECK-NEXT:  1      1     12.00          *            vsuxseg6ei64.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf2, tu, mu
-# CHECK-NEXT:  1      7     18.00          *            vsuxseg6ei64.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m1, tu, mu
-# CHECK-NEXT:  1      13    24.00          *            vsuxseg6ei64.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m2, tu, mu
-# CHECK-NEXT:  1      97    108.00         *            vsuxseg6ei64.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m4, tu, mu
-# CHECK-NEXT:  1      97    108.00         *            vsuxseg6ei64.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf8, tu, mu
-# CHECK-NEXT:  1      15    26.00          *            vsuxseg7ei8.v v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf4, tu, mu
-# CHECK-NEXT:  1      29    40.00          *            vsuxseg7ei8.v v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf2, tu, mu
-# CHECK-NEXT:  1      57    68.00          *            vsuxseg7ei8.v v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m1, tu, mu
-# CHECK-NEXT:  1      113   124.00         *            vsuxseg7ei8.v v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m2, tu, mu
-# CHECK-NEXT:  1      897   908.00         *            vsuxseg7ei8.v v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m4, tu, mu
-# CHECK-NEXT:  1      897   908.00         *            vsuxseg7ei8.v v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf8, tu, mu
-# CHECK-NEXT:  1      8     19.00          *            vsuxseg7ei16.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf4, tu, mu
-# CHECK-NEXT:  1      15    26.00          *            vsuxseg7ei16.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf2, tu, mu
-# CHECK-NEXT:  1      29    40.00          *            vsuxseg7ei16.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m1, tu, mu
-# CHECK-NEXT:  1      57    68.00          *            vsuxseg7ei16.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m2, tu, mu
-# CHECK-NEXT:  1      449   460.00         *            vsuxseg7ei16.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m4, tu, mu
-# CHECK-NEXT:  1      449   460.00         *            vsuxseg7ei16.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf8, tu, mu
-# CHECK-NEXT:  1      1     12.00          *            vsuxseg7ei32.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf4, tu, mu
-# CHECK-NEXT:  1      8     19.00          *            vsuxseg7ei32.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf2, tu, mu
-# CHECK-NEXT:  1      15    26.00          *            vsuxseg7ei32.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m1, tu, mu
-# CHECK-NEXT:  1      29    40.00          *            vsuxseg7ei32.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m2, tu, mu
-# CHECK-NEXT:  1      225   236.00         *            vsuxseg7ei32.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m4, tu, mu
-# CHECK-NEXT:  1      225   236.00         *            vsuxseg7ei32.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf8, tu, mu
-# CHECK-NEXT:  1      1     12.00          *            vsuxseg7ei64.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf4, tu, mu
-# CHECK-NEXT:  1      1     12.00          *            vsuxseg7ei64.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf2, tu, mu
-# CHECK-NEXT:  1      8     19.00          *            vsuxseg7ei64.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m1, tu, mu
-# CHECK-NEXT:  1      15    26.00          *            vsuxseg7ei64.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m2, tu, mu
-# CHECK-NEXT:  1      113   124.00         *            vsuxseg7ei64.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m4, tu, mu
-# CHECK-NEXT:  1      113   124.00         *            vsuxseg7ei64.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf8, tu, mu
-# CHECK-NEXT:  1      17    28.00          *            vsuxseg8ei8.v v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf4, tu, mu
-# CHECK-NEXT:  1      33    44.00          *            vsuxseg8ei8.v v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf2, tu, mu
-# CHECK-NEXT:  1      65    76.00          *            vsuxseg8ei8.v v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m1, tu, mu
-# CHECK-NEXT:  1      129   140.00         *            vsuxseg8ei8.v v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m2, tu, mu
-# CHECK-NEXT:  1      1025   1036.00         *            vsuxseg8ei8.v v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m4, tu, mu
-# CHECK-NEXT:  1      1025   1036.00         *            vsuxseg8ei8.v v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf8, tu, mu
-# CHECK-NEXT:  1      9     20.00          *            vsuxseg8ei16.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf4, tu, mu
-# CHECK-NEXT:  1      17    28.00          *            vsuxseg8ei16.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf2, tu, mu
-# CHECK-NEXT:  1      33    44.00          *            vsuxseg8ei16.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m1, tu, mu
-# CHECK-NEXT:  1      65    76.00          *            vsuxseg8ei16.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m2, tu, mu
-# CHECK-NEXT:  1      513   524.00         *            vsuxseg8ei16.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m4, tu, mu
-# CHECK-NEXT:  1      513   524.00         *            vsuxseg8ei16.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf8, tu, mu
-# CHECK-NEXT:  1      1     12.00          *            vsuxseg8ei32.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf4, tu, mu
-# CHECK-NEXT:  1      9     20.00          *            vsuxseg8ei32.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf2, tu, mu
-# CHECK-NEXT:  1      17    28.00          *            vsuxseg8ei32.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m1, tu, mu
-# CHECK-NEXT:  1      33    44.00          *            vsuxseg8ei32.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m2, tu, mu
-# CHECK-NEXT:  1      257   268.00         *            vsuxseg8ei32.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m4, tu, mu
-# CHECK-NEXT:  1      257   268.00         *            vsuxseg8ei32.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf8, tu, mu
-# CHECK-NEXT:  1      1     12.00          *            vsuxseg8ei64.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf4, tu, mu
-# CHECK-NEXT:  1      1     12.00          *            vsuxseg8ei64.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf2, tu, mu
-# CHECK-NEXT:  1      9     20.00          *            vsuxseg8ei64.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m1, tu, mu
-# CHECK-NEXT:  1      17    28.00          *            vsuxseg8ei64.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m2, tu, mu
-# CHECK-NEXT:  1      129   140.00         *            vsuxseg8ei64.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m4, tu, mu
-# CHECK-NEXT:  1      129   140.00         *            vsuxseg8ei64.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf8, tu, mu
-# CHECK-NEXT:  1      5     16.00          *            vsoxseg2ei8.v v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf4, tu, mu
-# CHECK-NEXT:  1      9     20.00          *            vsoxseg2ei8.v v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf2, tu, mu
-# CHECK-NEXT:  1      17    28.00          *            vsoxseg2ei8.v v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m1, tu, mu
-# CHECK-NEXT:  1      33    44.00          *            vsoxseg2ei8.v v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m2, tu, mu
-# CHECK-NEXT:  1      65    76.00          *            vsoxseg2ei8.v v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m4, tu, mu
-# CHECK-NEXT:  1      129   140.00         *            vsoxseg2ei8.v v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m8, tu, mu
-# CHECK-NEXT:  1      257   268.00         *            vsoxseg2ei8.v v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf8, tu, mu
-# CHECK-NEXT:  1      3     14.00          *            vsoxseg2ei16.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf4, tu, mu
-# CHECK-NEXT:  1      5     16.00          *            vsoxseg2ei16.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf2, tu, mu
-# CHECK-NEXT:  1      9     20.00          *            vsoxseg2ei16.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m1, tu, mu
-# CHECK-NEXT:  1      17    28.00          *            vsoxseg2ei16.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m2, tu, mu
-# CHECK-NEXT:  1      33    44.00          *            vsoxseg2ei16.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m4, tu, mu
-# CHECK-NEXT:  1      65    76.00          *            vsoxseg2ei16.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m8, tu, mu
-# CHECK-NEXT:  1      129   140.00         *            vsoxseg2ei16.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf8, tu, mu
-# CHECK-NEXT:  1      1     12.00          *            vsoxseg2ei32.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf4, tu, mu
-# CHECK-NEXT:  1      3     14.00          *            vsoxseg2ei32.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf2, tu, mu
-# CHECK-NEXT:  1      5     16.00          *            vsoxseg2ei32.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m1, tu, mu
-# CHECK-NEXT:  1      9     20.00          *            vsoxseg2ei32.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m2, tu, mu
-# CHECK-NEXT:  1      17    28.00          *            vsoxseg2ei32.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m4, tu, mu
-# CHECK-NEXT:  1      33    44.00          *            vsoxseg2ei32.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m8, tu, mu
-# CHECK-NEXT:  1      65    76.00          *            vsoxseg2ei32.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf8, tu, mu
-# CHECK-NEXT:  1      1     12.00          *            vsoxseg2ei64.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf4, tu, mu
-# CHECK-NEXT:  1      1     12.00          *            vsoxseg2ei64.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf2, tu, mu
-# CHECK-NEXT:  1      3     14.00          *            vsoxseg2ei64.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m1, tu, mu
-# CHECK-NEXT:  1      5     16.00          *            vsoxseg2ei64.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m2, tu, mu
-# CHECK-NEXT:  1      9     20.00          *            vsoxseg2ei64.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m4, tu, mu
-# CHECK-NEXT:  1      17    28.00          *            vsoxseg2ei64.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m8, tu, mu
-# CHECK-NEXT:  1      33    44.00          *            vsoxseg2ei64.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf8, tu, mu
-# CHECK-NEXT:  1      7     18.00          *            vsoxseg3ei8.v v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf4, tu, mu
-# CHECK-NEXT:  1      13    24.00          *            vsoxseg3ei8.v v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf2, tu, mu
-# CHECK-NEXT:  1      25    36.00          *            vsoxseg3ei8.v v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m1, tu, mu
-# CHECK-NEXT:  1      49    60.00          *            vsoxseg3ei8.v v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m2, tu, mu
-# CHECK-NEXT:  1      97    108.00         *            vsoxseg3ei8.v v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m4, tu, mu
-# CHECK-NEXT:  1      385   396.00         *            vsoxseg3ei8.v v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m8, tu, mu
-# CHECK-NEXT:  1      385   396.00         *            vsoxseg3ei8.v v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf8, tu, mu
-# CHECK-NEXT:  1      4     15.00          *            vsoxseg3ei16.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf4, tu, mu
-# CHECK-NEXT:  1      7     18.00          *            vsoxseg3ei16.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf2, tu, mu
-# CHECK-NEXT:  1      13    24.00          *            vsoxseg3ei16.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m1, tu, mu
-# CHECK-NEXT:  1      25    36.00          *            vsoxseg3ei16.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m2, tu, mu
-# CHECK-NEXT:  1      49    60.00          *            vsoxseg3ei16.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m4, tu, mu
-# CHECK-NEXT:  1      193   204.00         *            vsoxseg3ei16.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m8, tu, mu
-# CHECK-NEXT:  1      193   204.00         *            vsoxseg3ei16.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf8, tu, mu
-# CHECK-NEXT:  1      1     12.00          *            vsoxseg3ei32.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf4, tu, mu
-# CHECK-NEXT:  1      4     15.00          *            vsoxseg3ei32.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf2, tu, mu
-# CHECK-NEXT:  1      7     18.00          *            vsoxseg3ei32.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m1, tu, mu
-# CHECK-NEXT:  1      13    24.00          *            vsoxseg3ei32.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m2, tu, mu
-# CHECK-NEXT:  1      25    36.00          *            vsoxseg3ei32.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m4, tu, mu
-# CHECK-NEXT:  1      97    108.00         *            vsoxseg3ei32.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m8, tu, mu
-# CHECK-NEXT:  1      97    108.00         *            vsoxseg3ei32.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf8, tu, mu
-# CHECK-NEXT:  1      1     12.00          *            vsoxseg3ei64.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf4, tu, mu
-# CHECK-NEXT:  1      1     12.00          *            vsoxseg3ei64.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf2, tu, mu
-# CHECK-NEXT:  1      4     15.00          *            vsoxseg3ei64.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m1, tu, mu
-# CHECK-NEXT:  1      7     18.00          *            vsoxseg3ei64.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m2, tu, mu
-# CHECK-NEXT:  1      13    24.00          *            vsoxseg3ei64.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m4, tu, mu
-# CHECK-NEXT:  1      49    60.00          *            vsoxseg3ei64.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m8, tu, mu
-# CHECK-NEXT:  1      49    60.00          *            vsoxseg3ei64.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf8, tu, mu
-# CHECK-NEXT:  1      9     20.00          *            vsoxseg4ei8.v v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf4, tu, mu
-# CHECK-NEXT:  1      17    28.00          *            vsoxseg4ei8.v v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf2, tu, mu
-# CHECK-NEXT:  1      33    44.00          *            vsoxseg4ei8.v v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m1, tu, mu
-# CHECK-NEXT:  1      65    76.00          *            vsoxseg4ei8.v v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m2, tu, mu
-# CHECK-NEXT:  1      129   140.00         *            vsoxseg4ei8.v v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m4, tu, mu
-# CHECK-NEXT:  1      513   524.00         *            vsoxseg4ei8.v v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m8, tu, mu
-# CHECK-NEXT:  1      513   524.00         *            vsoxseg4ei8.v v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf8, tu, mu
-# CHECK-NEXT:  1      5     16.00          *            vsoxseg4ei16.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf4, tu, mu
-# CHECK-NEXT:  1      9     20.00          *            vsoxseg4ei16.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf2, tu, mu
-# CHECK-NEXT:  1      17    28.00          *            vsoxseg4ei16.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m1, tu, mu
-# CHECK-NEXT:  1      33    44.00          *            vsoxseg4ei16.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m2, tu, mu
-# CHECK-NEXT:  1      65    76.00          *            vsoxseg4ei16.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m4, tu, mu
-# CHECK-NEXT:  1      257   268.00         *            vsoxseg4ei16.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m8, tu, mu
-# CHECK-NEXT:  1      257   268.00         *            vsoxseg4ei16.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf8, tu, mu
-# CHECK-NEXT:  1      1     12.00          *            vsoxseg4ei32.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf4, tu, mu
-# CHECK-NEXT:  1      5     16.00          *            vsoxseg4ei32.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf2, tu, mu
-# CHECK-NEXT:  1      9     20.00          *            vsoxseg4ei32.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m1, tu, mu
-# CHECK-NEXT:  1      17    28.00          *            vsoxseg4ei32.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m2, tu, mu
-# CHECK-NEXT:  1      33    44.00          *            vsoxseg4ei32.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m4, tu, mu
-# CHECK-NEXT:  1      129   140.00         *            vsoxseg4ei32.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m8, tu, mu
-# CHECK-NEXT:  1      129   140.00         *            vsoxseg4ei32.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf8, tu, mu
-# CHECK-NEXT:  1      1     12.00          *            vsoxseg4ei64.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf4, tu, mu
-# CHECK-NEXT:  1      1     12.00          *            vsoxseg4ei64.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf2, tu, mu
-# CHECK-NEXT:  1      5     16.00          *            vsoxseg4ei64.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m1, tu, mu
-# CHECK-NEXT:  1      9     20.00          *            vsoxseg4ei64.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m2, tu, mu
-# CHECK-NEXT:  1      17    28.00          *            vsoxseg4ei64.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m4, tu, mu
-# CHECK-NEXT:  1      65    76.00          *            vsoxseg4ei64.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m8, tu, mu
-# CHECK-NEXT:  1      65    76.00          *            vsoxseg4ei64.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf8, tu, mu
-# CHECK-NEXT:  1      11    22.00          *            vsoxseg5ei8.v v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf4, tu, mu
-# CHECK-NEXT:  1      21    32.00          *            vsoxseg5ei8.v v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf2, tu, mu
-# CHECK-NEXT:  1      41    52.00          *            vsoxseg5ei8.v v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m1, tu, mu
-# CHECK-NEXT:  1      81    92.00          *            vsoxseg5ei8.v v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m2, tu, mu
-# CHECK-NEXT:  1      641   652.00         *            vsoxseg5ei8.v v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m4, tu, mu
-# CHECK-NEXT:  1      641   652.00         *            vsoxseg5ei8.v v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf8, tu, mu
-# CHECK-NEXT:  1      6     17.00          *            vsoxseg5ei16.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf4, tu, mu
-# CHECK-NEXT:  1      11    22.00          *            vsoxseg5ei16.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf2, tu, mu
-# CHECK-NEXT:  1      21    32.00          *            vsoxseg5ei16.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m1, tu, mu
-# CHECK-NEXT:  1      41    52.00          *            vsoxseg5ei16.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m2, tu, mu
-# CHECK-NEXT:  1      321   332.00         *            vsoxseg5ei16.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m4, tu, mu
-# CHECK-NEXT:  1      321   332.00         *            vsoxseg5ei16.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf8, tu, mu
-# CHECK-NEXT:  1      1     12.00          *            vsoxseg5ei32.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf4, tu, mu
-# CHECK-NEXT:  1      6     17.00          *            vsoxseg5ei32.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf2, tu, mu
-# CHECK-NEXT:  1      11    22.00          *            vsoxseg5ei32.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m1, tu, mu
-# CHECK-NEXT:  1      21    32.00          *            vsoxseg5ei32.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m2, tu, mu
-# CHECK-NEXT:  1      161   172.00         *            vsoxseg5ei32.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m4, tu, mu
-# CHECK-NEXT:  1      161   172.00         *            vsoxseg5ei32.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf8, tu, mu
-# CHECK-NEXT:  1      1     12.00          *            vsoxseg5ei64.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf4, tu, mu
-# CHECK-NEXT:  1      1     12.00          *            vsoxseg5ei64.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf2, tu, mu
-# CHECK-NEXT:  1      6     17.00          *            vsoxseg5ei64.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m1, tu, mu
-# CHECK-NEXT:  1      11    22.00          *            vsoxseg5ei64.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m2, tu, mu
-# CHECK-NEXT:  1      81    92.00          *            vsoxseg5ei64.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m4, tu, mu
-# CHECK-NEXT:  1      81    92.00          *            vsoxseg5ei64.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf8, tu, mu
-# CHECK-NEXT:  1      13    24.00          *            vsoxseg6ei8.v v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf4, tu, mu
-# CHECK-NEXT:  1      25    36.00          *            vsoxseg6ei8.v v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf2, tu, mu
-# CHECK-NEXT:  1      49    60.00          *            vsoxseg6ei8.v v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m1, tu, mu
-# CHECK-NEXT:  1      97    108.00         *            vsoxseg6ei8.v v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m2, tu, mu
-# CHECK-NEXT:  1      769   780.00         *            vsoxseg6ei8.v v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m4, tu, mu
-# CHECK-NEXT:  1      769   780.00         *            vsoxseg6ei8.v v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf8, tu, mu
-# CHECK-NEXT:  1      7     18.00          *            vsoxseg6ei16.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf4, tu, mu
-# CHECK-NEXT:  1      13    24.00          *            vsoxseg6ei16.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf2, tu, mu
-# CHECK-NEXT:  1      25    36.00          *            vsoxseg6ei16.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m1, tu, mu
-# CHECK-NEXT:  1      49    60.00          *            vsoxseg6ei16.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m2, tu, mu
-# CHECK-NEXT:  1      385   396.00         *            vsoxseg6ei16.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m4, tu, mu
-# CHECK-NEXT:  1      385   396.00         *            vsoxseg6ei16.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf8, tu, mu
-# CHECK-NEXT:  1      1     12.00          *            vsoxseg6ei32.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf4, tu, mu
-# CHECK-NEXT:  1      7     18.00          *            vsoxseg6ei32.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf2, tu, mu
-# CHECK-NEXT:  1      13    24.00          *            vsoxseg6ei32.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m1, tu, mu
-# CHECK-NEXT:  1      25    36.00          *            vsoxseg6ei32.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m2, tu, mu
-# CHECK-NEXT:  1      193   204.00         *            vsoxseg6ei32.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m4, tu, mu
-# CHECK-NEXT:  1      193   204.00         *            vsoxseg6ei32.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf8, tu, mu
-# CHECK-NEXT:  1      1     12.00          *            vsoxseg6ei64.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf4, tu, mu
-# CHECK-NEXT:  1      1     12.00          *            vsoxseg6ei64.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf2, tu, mu
-# CHECK-NEXT:  1      7     18.00          *            vsoxseg6ei64.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m1, tu, mu
-# CHECK-NEXT:  1      13    24.00          *            vsoxseg6ei64.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m2, tu, mu
-# CHECK-NEXT:  1      97    108.00         *            vsoxseg6ei64.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m4, tu, mu
-# CHECK-NEXT:  1      97    108.00         *            vsoxseg6ei64.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf8, tu, mu
-# CHECK-NEXT:  1      15    26.00          *            vsoxseg7ei8.v v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf4, tu, mu
-# CHECK-NEXT:  1      29    40.00          *            vsoxseg7ei8.v v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf2, tu, mu
-# CHECK-NEXT:  1      57    68.00          *            vsoxseg7ei8.v v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m1, tu, mu
-# CHECK-NEXT:  1      113   124.00         *            vsoxseg7ei8.v v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m2, tu, mu
-# CHECK-NEXT:  1      897   908.00         *            vsoxseg7ei8.v v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m4, tu, mu
-# CHECK-NEXT:  1      897   908.00         *            vsoxseg7ei8.v v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf8, tu, mu
-# CHECK-NEXT:  1      8     19.00          *            vsoxseg7ei16.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf4, tu, mu
-# CHECK-NEXT:  1      15    26.00          *            vsoxseg7ei16.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf2, tu, mu
-# CHECK-NEXT:  1      29    40.00          *            vsoxseg7ei16.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m1, tu, mu
-# CHECK-NEXT:  1      57    68.00          *            vsoxseg7ei16.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m2, tu, mu
-# CHECK-NEXT:  1      449   460.00         *            vsoxseg7ei16.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m4, tu, mu
-# CHECK-NEXT:  1      449   460.00         *            vsoxseg7ei16.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf8, tu, mu
-# CHECK-NEXT:  1      1     12.00          *            vsoxseg7ei32.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf4, tu, mu
-# CHECK-NEXT:  1      8     19.00          *            vsoxseg7ei32.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf2, tu, mu
-# CHECK-NEXT:  1      15    26.00          *            vsoxseg7ei32.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m1, tu, mu
-# CHECK-NEXT:  1      29    40.00          *            vsoxseg7ei32.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m2, tu, mu
-# CHECK-NEXT:  1      225   236.00         *            vsoxseg7ei32.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m4, tu, mu
-# CHECK-NEXT:  1      225   236.00         *            vsoxseg7ei32.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf8, tu, mu
-# CHECK-NEXT:  1      1     12.00          *            vsoxseg7ei64.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf4, tu, mu
-# CHECK-NEXT:  1      1     12.00          *            vsoxseg7ei64.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf2, tu, mu
-# CHECK-NEXT:  1      8     19.00          *            vsoxseg7ei64.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m1, tu, mu
-# CHECK-NEXT:  1      15    26.00          *            vsoxseg7ei64.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m2, tu, mu
-# CHECK-NEXT:  1      113   124.00         *            vsoxseg7ei64.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m4, tu, mu
-# CHECK-NEXT:  1      113   124.00         *            vsoxseg7ei64.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf8, tu, mu
-# CHECK-NEXT:  1      17    28.00          *            vsoxseg8ei8.v v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf4, tu, mu
-# CHECK-NEXT:  1      33    44.00          *            vsoxseg8ei8.v v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf2, tu, mu
-# CHECK-NEXT:  1      65    76.00          *            vsoxseg8ei8.v v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m1, tu, mu
-# CHECK-NEXT:  1      129   140.00         *            vsoxseg8ei8.v v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m2, tu, mu
-# CHECK-NEXT:  1      1025   1036.00         *            vsoxseg8ei8.v v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m4, tu, mu
-# CHECK-NEXT:  1      1025   1036.00         *            vsoxseg8ei8.v v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf8, tu, mu
-# CHECK-NEXT:  1      9     20.00          *            vsoxseg8ei16.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf4, tu, mu
-# CHECK-NEXT:  1      17    28.00          *            vsoxseg8ei16.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf2, tu, mu
-# CHECK-NEXT:  1      33    44.00          *            vsoxseg8ei16.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m1, tu, mu
-# CHECK-NEXT:  1      65    76.00          *            vsoxseg8ei16.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m2, tu, mu
-# CHECK-NEXT:  1      513   524.00         *            vsoxseg8ei16.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m4, tu, mu
-# CHECK-NEXT:  1      513   524.00         *            vsoxseg8ei16.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf8, tu, mu
-# CHECK-NEXT:  1      1     12.00          *            vsoxseg8ei32.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf4, tu, mu
-# CHECK-NEXT:  1      9     20.00          *            vsoxseg8ei32.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf2, tu, mu
-# CHECK-NEXT:  1      17    28.00          *            vsoxseg8ei32.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m1, tu, mu
-# CHECK-NEXT:  1      33    44.00          *            vsoxseg8ei32.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m2, tu, mu
-# CHECK-NEXT:  1      257   268.00         *            vsoxseg8ei32.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m4, tu, mu
-# CHECK-NEXT:  1      257   268.00         *            vsoxseg8ei32.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf8, tu, mu
-# CHECK-NEXT:  1      1     12.00          *            vsoxseg8ei64.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf4, tu, mu
-# CHECK-NEXT:  1      1     12.00          *            vsoxseg8ei64.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf2, tu, mu
-# CHECK-NEXT:  1      9     20.00          *            vsoxseg8ei64.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m1, tu, mu
-# CHECK-NEXT:  1      17    28.00          *            vsoxseg8ei64.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m2, tu, mu
-# CHECK-NEXT:  1      129   140.00         *            vsoxseg8ei64.v  v8, (a0), v16
-# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m4, tu, mu
-# CHECK-NEXT:  1      129   140.00         *            vsoxseg8ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  1      16    16.00   *                   vlseg2e8.v	v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  1      20    20.00   *                   vlseg2e8.v	v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  1      28    28.00   *                   vlseg2e8.v	v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  1      44    44.00   *                   vlseg2e8.v	v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  1      76    76.00   *                   vlseg2e8.v	v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  1      140   140.00  *                   vlseg2e8.v	v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  1      16    16.00   *                   vlseg2e16.v	v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  1      20    20.00   *                   vlseg2e16.v	v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  1      28    28.00   *                   vlseg2e16.v	v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  1      44    44.00   *                   vlseg2e16.v	v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  1      76    76.00   *                   vlseg2e16.v	v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  1      16    16.00   *                   vlseg2e32.v	v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  1      20    20.00   *                   vlseg2e32.v	v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  1      28    28.00   *                   vlseg2e32.v	v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  1      44    44.00   *                   vlseg2e32.v	v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  1      16    16.00   *                   vlseg2e64.v	v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  1      20    20.00   *                   vlseg2e64.v	v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  1      28    28.00   *                   vlseg2e64.v	v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  1      18    18.00   *                   vlseg3e8.v	v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  1      24    24.00   *                   vlseg3e8.v	v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  1      36    36.00   *                   vlseg3e8.v	v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  1      60    60.00   *                   vlseg3e8.v	v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  1      108   108.00  *                   vlseg3e8.v	v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  1      18    18.00   *                   vlseg3e16.v	v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  1      24    24.00   *                   vlseg3e16.v	v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  1      36    36.00   *                   vlseg3e16.v	v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  1      60    60.00   *                   vlseg3e16.v	v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  1      18    18.00   *                   vlseg3e32.v	v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  1      24    24.00   *                   vlseg3e32.v	v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  1      36    36.00   *                   vlseg3e32.v	v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  1      18    18.00   *                   vlseg3e64.v	v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  1      24    24.00   *                   vlseg3e64.v	v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  1      20    20.00   *                   vlseg4e8.v	v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  1      28    28.00   *                   vlseg4e8.v	v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  1      44    44.00   *                   vlseg4e8.v	v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  1      76    76.00   *                   vlseg4e8.v	v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  1      140   140.00  *                   vlseg4e8.v	v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  1      20    20.00   *                   vlseg4e16.v	v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  1      28    28.00   *                   vlseg4e16.v	v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  1      44    44.00   *                   vlseg4e16.v	v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  1      76    76.00   *                   vlseg4e16.v	v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  1      20    20.00   *                   vlseg4e32.v	v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  1      28    28.00   *                   vlseg4e32.v	v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  1      44    44.00   *                   vlseg4e32.v	v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  1      20    20.00   *                   vlseg4e64.v	v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  1      28    28.00   *                   vlseg4e64.v	v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  1      22    22.00   *                   vlseg5e8.v	v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  1      32    32.00   *                   vlseg5e8.v	v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  1      52    52.00   *                   vlseg5e8.v	v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  1      92    92.00   *                   vlseg5e8.v	v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  1      22    22.00   *                   vlseg5e16.v	v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  1      32    32.00   *                   vlseg5e16.v	v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  1      52    52.00   *                   vlseg5e16.v	v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  1      22    22.00   *                   vlseg5e32.v	v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  1      32    32.00   *                   vlseg5e32.v	v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  1      22    22.00   *                   vlseg5e64.v	v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  1      24    24.00   *                   vlseg6e8.v	v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  1      36    36.00   *                   vlseg6e8.v	v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  1      60    60.00   *                   vlseg6e8.v	v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  1      108   108.00  *                   vlseg6e8.v	v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  1      24    24.00   *                   vlseg6e16.v	v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  1      36    36.00   *                   vlseg6e16.v	v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  1      60    60.00   *                   vlseg6e16.v	v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  1      24    24.00   *                   vlseg6e32.v	v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  1      36    36.00   *                   vlseg6e32.v	v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  1      24    24.00   *                   vlseg6e64.v	v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  1      26    26.00   *                   vlseg7e8.v	v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  1      40    40.00   *                   vlseg7e8.v	v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  1      68    68.00   *                   vlseg7e8.v	v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  1      124   124.00  *                   vlseg7e8.v	v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  1      26    26.00   *                   vlseg7e16.v	v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  1      40    40.00   *                   vlseg7e16.v	v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  1      68    68.00   *                   vlseg7e16.v	v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  1      26    26.00   *                   vlseg7e32.v	v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  1      40    40.00   *                   vlseg7e32.v	v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  1      26    26.00   *                   vlseg7e64.v	v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  1      28    28.00   *                   vlseg8e8.v	v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  1      44    44.00   *                   vlseg8e8.v	v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  1      76    76.00   *                   vlseg8e8.v	v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  1      140   140.00  *                   vlseg8e8.v	v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  1      28    28.00   *                   vlseg8e16.v	v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  1      44    44.00   *                   vlseg8e16.v	v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  1      76    76.00   *                   vlseg8e16.v	v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  1      28    28.00   *                   vlseg8e32.v	v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  1      44    44.00   *                   vlseg8e32.v	v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  1      28    28.00   *                   vlseg8e64.v	v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  1      5     16.00          *            vsseg2e8.v	v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  1      9     20.00          *            vsseg2e8.v	v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  1      17    28.00          *            vsseg2e8.v	v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  1      33    44.00          *            vsseg2e8.v	v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  1      65    76.00          *            vsseg2e8.v	v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  1      129   140.00         *            vsseg2e8.v	v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  1      5     16.00          *            vsseg2e16.v	v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  1      9     20.00          *            vsseg2e16.v	v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  1      17    28.00          *            vsseg2e16.v	v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  1      33    44.00          *            vsseg2e16.v	v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  1      65    76.00          *            vsseg2e16.v	v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  1      5     16.00          *            vsseg2e32.v	v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  1      9     20.00          *            vsseg2e32.v	v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  1      17    28.00          *            vsseg2e32.v	v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  1      33    44.00          *            vsseg2e32.v	v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  1      5     16.00          *            vsseg2e64.v	v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  1      9     20.00          *            vsseg2e64.v	v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  1      17    28.00          *            vsseg2e64.v	v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  1      7     18.00          *            vsseg3e8.v	v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  1      13    24.00          *            vsseg3e8.v	v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  1      25    36.00          *            vsseg3e8.v	v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  1      49    60.00          *            vsseg3e8.v	v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  1      97    108.00         *            vsseg3e8.v	v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  1      7     18.00          *            vsseg3e16.v	v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  1      13    24.00          *            vsseg3e16.v	v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  1      25    36.00          *            vsseg3e16.v	v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  1      49    60.00          *            vsseg3e16.v	v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  1      7     18.00          *            vsseg3e32.v	v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  1      13    24.00          *            vsseg3e32.v	v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  1      25    36.00          *            vsseg3e32.v	v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  1      7     18.00          *            vsseg3e64.v	v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  1      13    24.00          *            vsseg3e64.v	v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  1      9     20.00          *            vsseg4e8.v	v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  1      17    28.00          *            vsseg4e8.v	v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  1      33    44.00          *            vsseg4e8.v	v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  1      65    76.00          *            vsseg4e8.v	v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  1      129   140.00         *            vsseg4e8.v	v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  1      9     20.00          *            vsseg4e16.v	v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  1      17    28.00          *            vsseg4e16.v	v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  1      33    44.00          *            vsseg4e16.v	v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  1      65    76.00          *            vsseg4e16.v	v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  1      9     20.00          *            vsseg4e32.v	v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  1      17    28.00          *            vsseg4e32.v	v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  1      33    44.00          *            vsseg4e32.v	v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  1      9     20.00          *            vsseg4e64.v	v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  1      17    28.00          *            vsseg4e64.v	v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  1      11    22.00          *            vsseg5e8.v	v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  1      21    32.00          *            vsseg5e8.v	v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  1      41    52.00          *            vsseg5e8.v	v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  1      81    92.00          *            vsseg5e8.v	v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  1      11    22.00          *            vsseg5e16.v	v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  1      21    32.00          *            vsseg5e16.v	v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  1      41    52.00          *            vsseg5e16.v	v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  1      11    22.00          *            vsseg5e32.v	v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  1      21    32.00          *            vsseg5e32.v	v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  1      11    22.00          *            vsseg5e64.v	v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  1      13    24.00          *            vsseg6e8.v	v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  1      25    36.00          *            vsseg6e8.v	v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  1      49    60.00          *            vsseg6e8.v	v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  1      97    108.00         *            vsseg6e8.v	v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  1      13    24.00          *            vsseg6e16.v	v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  1      25    36.00          *            vsseg6e16.v	v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  1      49    60.00          *            vsseg6e16.v	v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  1      13    24.00          *            vsseg6e32.v	v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  1      25    36.00          *            vsseg6e32.v	v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  1      13    24.00          *            vsseg6e64.v	v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  1      15    26.00          *            vsseg7e8.v	v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  1      29    40.00          *            vsseg7e8.v	v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  1      57    68.00          *            vsseg7e8.v	v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  1      113   124.00         *            vsseg7e8.v	v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  1      15    26.00          *            vsseg7e16.v	v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  1      29    40.00          *            vsseg7e16.v	v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  1      57    68.00          *            vsseg7e16.v	v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  1      15    26.00          *            vsseg7e32.v	v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  1      29    40.00          *            vsseg7e32.v	v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  1      15    26.00          *            vsseg7e64.v	v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  1      17    28.00          *            vsseg8e8.v	v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  1      33    44.00          *            vsseg8e8.v	v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  1      65    76.00          *            vsseg8e8.v	v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  1      129   140.00         *            vsseg8e8.v	v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  1      17    28.00          *            vsseg8e16.v	v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  1      33    44.00          *            vsseg8e16.v	v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  1      65    76.00          *            vsseg8e16.v	v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  1      17    28.00          *            vsseg8e32.v	v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  1      33    44.00          *            vsseg8e32.v	v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  1      17    28.00          *            vsseg8e64.v	v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  1      16    16.00   *                   vlsseg2e8.v	v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  1      20    20.00   *                   vlsseg2e8.v	v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  1      28    28.00   *                   vlsseg2e8.v	v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  1      44    44.00   *                   vlsseg2e8.v	v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  1      76    76.00   *                   vlsseg2e8.v	v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  1      140   140.00  *                   vlsseg2e8.v	v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  1      16    16.00   *                   vlsseg2e16.v	v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  1      20    20.00   *                   vlsseg2e16.v	v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  1      28    28.00   *                   vlsseg2e16.v	v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  1      44    44.00   *                   vlsseg2e16.v	v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  1      76    76.00   *                   vlsseg2e16.v	v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  1      16    16.00   *                   vlsseg2e32.v	v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  1      20    20.00   *                   vlsseg2e32.v	v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  1      28    28.00   *                   vlsseg2e32.v	v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  1      44    44.00   *                   vlsseg2e32.v	v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  1      16    16.00   *                   vlsseg2e64.v	v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  1      20    20.00   *                   vlsseg2e64.v	v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  1      28    28.00   *                   vlsseg2e64.v	v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  1      18    18.00   *                   vlsseg3e8.v	v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  1      24    24.00   *                   vlsseg3e8.v	v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  1      36    36.00   *                   vlsseg3e8.v	v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  1      60    60.00   *                   vlsseg3e8.v	v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  1      108   108.00  *                   vlsseg3e8.v	v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  1      18    18.00   *                   vlsseg3e16.v	v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  1      24    24.00   *                   vlsseg3e16.v	v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  1      36    36.00   *                   vlsseg3e16.v	v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  1      60    60.00   *                   vlsseg3e16.v	v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  1      18    18.00   *                   vlsseg3e32.v	v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  1      24    24.00   *                   vlsseg3e32.v	v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  1      36    36.00   *                   vlsseg3e32.v	v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  1      18    18.00   *                   vlsseg3e64.v	v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  1      24    24.00   *                   vlsseg3e64.v	v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  1      20    20.00   *                   vlsseg4e8.v	v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  1      28    28.00   *                   vlsseg4e8.v	v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  1      44    44.00   *                   vlsseg4e8.v	v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  1      76    76.00   *                   vlsseg4e8.v	v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  1      140   140.00  *                   vlsseg4e8.v	v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  1      20    20.00   *                   vlsseg4e16.v	v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  1      28    28.00   *                   vlsseg4e16.v	v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  1      44    44.00   *                   vlsseg4e16.v	v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  1      76    76.00   *                   vlsseg4e16.v	v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  1      20    20.00   *                   vlsseg4e32.v	v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  1      28    28.00   *                   vlsseg4e32.v	v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  1      44    44.00   *                   vlsseg4e32.v	v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  1      20    20.00   *                   vlsseg4e64.v	v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  1      28    28.00   *                   vlsseg4e64.v	v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  1      22    22.00   *                   vlsseg5e8.v	v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  1      32    32.00   *                   vlsseg5e8.v	v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  1      52    52.00   *                   vlsseg5e8.v	v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  1      92    92.00   *                   vlsseg5e8.v	v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  1      22    22.00   *                   vlsseg5e16.v	v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  1      32    32.00   *                   vlsseg5e16.v	v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  1      52    52.00   *                   vlsseg5e16.v	v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  1      22    22.00   *                   vlsseg5e32.v	v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  1      32    32.00   *                   vlsseg5e32.v	v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  1      22    22.00   *                   vlsseg5e64.v	v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  1      24    24.00   *                   vlsseg6e8.v	v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  1      36    36.00   *                   vlsseg6e8.v	v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  1      60    60.00   *                   vlsseg6e8.v	v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  1      108   108.00  *                   vlsseg6e8.v	v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  1      24    24.00   *                   vlsseg6e16.v	v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  1      36    36.00   *                   vlsseg6e16.v	v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  1      60    60.00   *                   vlsseg6e16.v	v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  1      24    24.00   *                   vlsseg6e32.v	v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  1      36    36.00   *                   vlsseg6e32.v	v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  1      24    24.00   *                   vlsseg6e64.v	v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  1      26    26.00   *                   vlsseg7e8.v	v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  1      40    40.00   *                   vlsseg7e8.v	v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  1      68    68.00   *                   vlsseg7e8.v	v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  1      124   124.00  *                   vlsseg7e8.v	v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  1      26    26.00   *                   vlsseg7e16.v	v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  1      40    40.00   *                   vlsseg7e16.v	v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  1      68    68.00   *                   vlsseg7e16.v	v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  1      26    26.00   *                   vlsseg7e32.v	v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  1      40    40.00   *                   vlsseg7e32.v	v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  1      26    26.00   *                   vlsseg7e64.v	v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  1      28    28.00   *                   vlsseg8e8.v	v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  1      44    44.00   *                   vlsseg8e8.v	v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  1      76    76.00   *                   vlsseg8e8.v	v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  1      140   140.00  *                   vlsseg8e8.v	v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  1      28    28.00   *                   vlsseg8e16.v	v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  1      44    44.00   *                   vlsseg8e16.v	v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  1      76    76.00   *                   vlsseg8e16.v	v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  1      28    28.00   *                   vlsseg8e32.v	v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  1      44    44.00   *                   vlsseg8e32.v	v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  1      28    28.00   *                   vlsseg8e64.v	v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  1      5     16.00          *            vssseg2e8.v	v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  1      9     20.00          *            vssseg2e8.v	v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  1      17    28.00          *            vssseg2e8.v	v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  1      33    44.00          *            vssseg2e8.v	v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  1      65    76.00          *            vssseg2e8.v	v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  1      129   140.00         *            vssseg2e8.v	v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  1      5     16.00          *            vssseg2e16.v	v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  1      9     20.00          *            vssseg2e16.v	v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  1      17    28.00          *            vssseg2e16.v	v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  1      33    44.00          *            vssseg2e16.v	v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  1      65    76.00          *            vssseg2e16.v	v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  1      5     16.00          *            vssseg2e32.v	v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  1      9     20.00          *            vssseg2e32.v	v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  1      17    28.00          *            vssseg2e32.v	v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  1      33    44.00          *            vssseg2e32.v	v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  1      5     16.00          *            vssseg2e64.v	v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  1      9     20.00          *            vssseg2e64.v	v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  1      17    28.00          *            vssseg2e64.v	v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  1      7     18.00          *            vssseg3e8.v	v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  1      13    24.00          *            vssseg3e8.v	v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  1      25    36.00          *            vssseg3e8.v	v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  1      49    60.00          *            vssseg3e8.v	v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  1      97    108.00         *            vssseg3e8.v	v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  1      7     18.00          *            vssseg3e16.v	v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  1      13    24.00          *            vssseg3e16.v	v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  1      25    36.00          *            vssseg3e16.v	v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  1      49    60.00          *            vssseg3e16.v	v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  1      7     18.00          *            vssseg3e32.v	v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  1      13    24.00          *            vssseg3e32.v	v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  1      25    36.00          *            vssseg3e32.v	v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  1      7     18.00          *            vssseg3e64.v	v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  1      13    24.00          *            vssseg3e64.v	v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  1      9     20.00          *            vssseg4e8.v	v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  1      17    28.00          *            vssseg4e8.v	v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  1      33    44.00          *            vssseg4e8.v	v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  1      65    76.00          *            vssseg4e8.v	v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  1      129   140.00         *            vssseg4e8.v	v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  1      9     20.00          *            vssseg4e16.v	v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  1      17    28.00          *            vssseg4e16.v	v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  1      33    44.00          *            vssseg4e16.v	v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  1      65    76.00          *            vssseg4e16.v	v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  1      9     20.00          *            vssseg4e32.v	v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  1      17    28.00          *            vssseg4e32.v	v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  1      33    44.00          *            vssseg4e32.v	v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  1      9     20.00          *            vssseg4e64.v	v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  1      17    28.00          *            vssseg4e64.v	v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  1      11    22.00          *            vssseg5e8.v	v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  1      21    32.00          *            vssseg5e8.v	v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  1      41    52.00          *            vssseg5e8.v	v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  1      81    92.00          *            vssseg5e8.v	v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  1      11    22.00          *            vssseg5e16.v	v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  1      21    32.00          *            vssseg5e16.v	v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  1      41    52.00          *            vssseg5e16.v	v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  1      11    22.00          *            vssseg5e32.v	v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  1      21    32.00          *            vssseg5e32.v	v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  1      11    22.00          *            vssseg5e64.v	v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  1      13    24.00          *            vssseg6e8.v	v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  1      25    36.00          *            vssseg6e8.v	v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  1      49    60.00          *            vssseg6e8.v	v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  1      97    108.00         *            vssseg6e8.v	v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  1      13    24.00          *            vssseg6e16.v	v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  1      25    36.00          *            vssseg6e16.v	v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  1      49    60.00          *            vssseg6e16.v	v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  1      13    24.00          *            vssseg6e32.v	v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  1      25    36.00          *            vssseg6e32.v	v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  1      13    24.00          *            vssseg6e64.v	v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  1      15    26.00          *            vssseg7e8.v	v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  1      29    40.00          *            vssseg7e8.v	v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  1      57    68.00          *            vssseg7e8.v	v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  1      113   124.00         *            vssseg7e8.v	v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  1      15    26.00          *            vssseg7e16.v	v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  1      29    40.00          *            vssseg7e16.v	v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  1      57    68.00          *            vssseg7e16.v	v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  1      15    26.00          *            vssseg7e32.v	v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  1      29    40.00          *            vssseg7e32.v	v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  1      15    26.00          *            vssseg7e64.v	v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  1      17    28.00          *            vssseg8e8.v	v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  1      33    44.00          *            vssseg8e8.v	v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  1      65    76.00          *            vssseg8e8.v	v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  1      129   140.00         *            vssseg8e8.v	v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  1      17    28.00          *            vssseg8e16.v	v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  1      33    44.00          *            vssseg8e16.v	v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  1      65    76.00          *            vssseg8e16.v	v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  1      17    28.00          *            vssseg8e32.v	v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  1      33    44.00          *            vssseg8e32.v	v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  1      17    28.00          *            vssseg8e64.v	v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  1      16    16.00   *                   vlseg2e8ff.v	v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  1      20    20.00   *                   vlseg2e8ff.v	v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  1      28    28.00   *                   vlseg2e8ff.v	v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  1      44    44.00   *                   vlseg2e8ff.v	v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  1      76    76.00   *                   vlseg2e8ff.v	v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  1      140   140.00  *                   vlseg2e8ff.v	v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  1      16    16.00   *                   vlseg2e16ff.v	v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  1      20    20.00   *                   vlseg2e16ff.v	v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  1      28    28.00   *                   vlseg2e16ff.v	v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  1      44    44.00   *                   vlseg2e16ff.v	v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  1      76    76.00   *                   vlseg2e16ff.v	v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  1      16    16.00   *                   vlseg2e32ff.v	v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  1      20    20.00   *                   vlseg2e32ff.v	v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  1      28    28.00   *                   vlseg2e32ff.v	v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  1      44    44.00   *                   vlseg2e32ff.v	v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  1      16    16.00   *                   vlseg2e64ff.v	v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  1      20    20.00   *                   vlseg2e64ff.v	v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  1      28    28.00   *                   vlseg2e64ff.v	v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  1      18    18.00   *                   vlseg3e8ff.v	v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  1      24    24.00   *                   vlseg3e8ff.v	v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  1      36    36.00   *                   vlseg3e8ff.v	v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  1      60    60.00   *                   vlseg3e8ff.v	v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  1      108   108.00  *                   vlseg3e8ff.v	v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  1      18    18.00   *                   vlseg3e16ff.v	v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  1      24    24.00   *                   vlseg3e16ff.v	v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  1      36    36.00   *                   vlseg3e16ff.v	v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  1      60    60.00   *                   vlseg3e16ff.v	v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  1      18    18.00   *                   vlseg3e32ff.v	v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  1      24    24.00   *                   vlseg3e32ff.v	v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  1      36    36.00   *                   vlseg3e32ff.v	v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  1      18    18.00   *                   vlseg3e64ff.v	v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  1      24    24.00   *                   vlseg3e64ff.v	v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  1      20    20.00   *                   vlseg4e8ff.v	v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  1      28    28.00   *                   vlseg4e8ff.v	v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  1      44    44.00   *                   vlseg4e8ff.v	v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  1      76    76.00   *                   vlseg4e8ff.v	v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  1      140   140.00  *                   vlseg4e8ff.v	v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  1      20    20.00   *                   vlseg4e16ff.v	v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  1      28    28.00   *                   vlseg4e16ff.v	v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  1      44    44.00   *                   vlseg4e16ff.v	v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  1      76    76.00   *                   vlseg4e16ff.v	v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  1      20    20.00   *                   vlseg4e32ff.v	v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  1      28    28.00   *                   vlseg4e32ff.v	v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  1      44    44.00   *                   vlseg4e32ff.v	v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  1      20    20.00   *                   vlseg4e64ff.v	v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  1      28    28.00   *                   vlseg4e64ff.v	v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  1      22    22.00   *                   vlseg5e8ff.v	v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  1      32    32.00   *                   vlseg5e8ff.v	v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  1      52    52.00   *                   vlseg5e8ff.v	v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  1      92    92.00   *                   vlseg5e8ff.v	v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  1      22    22.00   *                   vlseg5e16ff.v	v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  1      32    32.00   *                   vlseg5e16ff.v	v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  1      52    52.00   *                   vlseg5e16ff.v	v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  1      22    22.00   *                   vlseg5e32ff.v	v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  1      32    32.00   *                   vlseg5e32ff.v	v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  1      22    22.00   *                   vlseg5e64ff.v	v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  1      24    24.00   *                   vlseg6e8ff.v	v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  1      36    36.00   *                   vlseg6e8ff.v	v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  1      60    60.00   *                   vlseg6e8ff.v	v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  1      108   108.00  *                   vlseg6e8ff.v	v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  1      24    24.00   *                   vlseg6e16ff.v	v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  1      36    36.00   *                   vlseg6e16ff.v	v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  1      60    60.00   *                   vlseg6e16ff.v	v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  1      24    24.00   *                   vlseg6e32ff.v	v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  1      36    36.00   *                   vlseg6e32ff.v	v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  1      24    24.00   *                   vlseg6e64ff.v	v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  1      26    26.00   *                   vlseg7e8ff.v	v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  1      40    40.00   *                   vlseg7e8ff.v	v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  1      68    68.00   *                   vlseg7e8ff.v	v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  1      124   124.00  *                   vlseg7e8ff.v	v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  1      26    26.00   *                   vlseg7e16ff.v	v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  1      40    40.00   *                   vlseg7e16ff.v	v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  1      68    68.00   *                   vlseg7e16ff.v	v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  1      26    26.00   *                   vlseg7e32ff.v	v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  1      40    40.00   *                   vlseg7e32ff.v	v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  1      26    26.00   *                   vlseg7e64ff.v	v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  1      28    28.00   *                   vlseg8e8ff.v	v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  1      44    44.00   *                   vlseg8e8ff.v	v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  1      76    76.00   *                   vlseg8e8ff.v	v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  1      140   140.00  *                   vlseg8e8ff.v	v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  1      28    28.00   *                   vlseg8e16ff.v	v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  1      44    44.00   *                   vlseg8e16ff.v	v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  1      76    76.00   *                   vlseg8e16ff.v	v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  1      28    28.00   *                   vlseg8e32ff.v	v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  1      44    44.00   *                   vlseg8e32ff.v	v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  1      28    28.00   *                   vlseg8e64ff.v	v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  1      16    16.00   *                   vluxseg2ei8.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  1      20    20.00   *                   vluxseg2ei8.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  1      28    28.00   *                   vluxseg2ei8.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  1      44    44.00   *                   vluxseg2ei8.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  1      44    44.00   *                   vluxseg2ei8.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  1      44    44.00   *                   vluxseg2ei8.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  1      20    20.00   *                   vluxseg2ei16.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  1      28    28.00   *                   vluxseg2ei16.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  1      28    28.00   *                   vluxseg2ei16.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  1      28    28.00   *                   vluxseg2ei16.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  1      28    28.00   *                   vluxseg2ei16.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  1      20    20.00   *                   vluxseg2ei32.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  1      20    20.00   *                   vluxseg2ei32.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  1      20    20.00   *                   vluxseg2ei32.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  1      28    28.00   *                   vluxseg2ei32.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  1      16    16.00   *                   vluxseg2ei64.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  1      20    20.00   *                   vluxseg2ei64.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  1      28    28.00   *                   vluxseg2ei64.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  1      18    18.00   *                   vluxseg3ei8.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  1      24    24.00   *                   vluxseg3ei8.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  1      36    36.00   *                   vluxseg3ei8.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  1      60    60.00   *                   vluxseg3ei8.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  1      60    60.00   *                   vluxseg3ei8.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  1      24    24.00   *                   vluxseg3ei16.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  1      36    36.00   *                   vluxseg3ei16.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  1      36    36.00   *                   vluxseg3ei16.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  1      36    36.00   *                   vluxseg3ei16.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  1      24    24.00   *                   vluxseg3ei32.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  1      24    24.00   *                   vluxseg3ei32.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  1      24    24.00   *                   vluxseg3ei32.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  1      18    18.00   *                   vluxseg3ei64.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  1      24    24.00   *                   vluxseg3ei64.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  1      20    20.00   *                   vluxseg4ei8.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  1      28    28.00   *                   vluxseg4ei8.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  1      44    44.00   *                   vluxseg4ei8.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  1      76    76.00   *                   vluxseg4ei8.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  1      76    76.00   *                   vluxseg4ei8.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  1      28    28.00   *                   vluxseg4ei16.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  1      44    44.00   *                   vluxseg4ei16.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  1      44    44.00   *                   vluxseg4ei16.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  1      44    44.00   *                   vluxseg4ei16.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  1      28    28.00   *                   vluxseg4ei32.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  1      28    28.00   *                   vluxseg4ei32.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  1      28    28.00   *                   vluxseg4ei32.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  1      20    20.00   *                   vluxseg4ei64.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  1      28    28.00   *                   vluxseg4ei64.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  1      22    22.00   *                   vluxseg5ei8.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  1      32    32.00   *                   vluxseg5ei8.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  1      52    52.00   *                   vluxseg5ei8.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  1      92    92.00   *                   vluxseg5ei8.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  1      32    32.00   *                   vluxseg5ei16.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  1      52    52.00   *                   vluxseg5ei16.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  1      52    52.00   *                   vluxseg5ei16.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  1      32    32.00   *                   vluxseg5ei32.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  1      32    32.00   *                   vluxseg5ei32.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  1      22    22.00   *                   vluxseg5ei64.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  1      24    24.00   *                   vluxseg6ei8.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  1      36    36.00   *                   vluxseg6ei8.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  1      60    60.00   *                   vluxseg6ei8.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  1      108   108.00  *                   vluxseg6ei8.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  1      36    36.00   *                   vluxseg6ei16.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  1      60    60.00   *                   vluxseg6ei16.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  1      60    60.00   *                   vluxseg6ei16.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  1      36    36.00   *                   vluxseg6ei32.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  1      36    36.00   *                   vluxseg6ei32.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  1      24    24.00   *                   vluxseg6ei64.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  1      26    26.00   *                   vluxseg7ei8.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  1      40    40.00   *                   vluxseg7ei8.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  1      68    68.00   *                   vluxseg7ei8.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  1      124   124.00  *                   vluxseg7ei8.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  1      40    40.00   *                   vluxseg7ei16.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  1      68    68.00   *                   vluxseg7ei16.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  1      68    68.00   *                   vluxseg7ei16.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  1      40    40.00   *                   vluxseg7ei32.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  1      40    40.00   *                   vluxseg7ei32.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  1      26    26.00   *                   vluxseg7ei64.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  1      28    28.00   *                   vluxseg8ei8.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  1      44    44.00   *                   vluxseg8ei8.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  1      76    76.00   *                   vluxseg8ei8.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  1      140   140.00  *                   vluxseg8ei8.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  1      44    44.00   *                   vluxseg8ei16.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  1      76    76.00   *                   vluxseg8ei16.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  1      76    76.00   *                   vluxseg8ei16.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  1      44    44.00   *                   vluxseg8ei32.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  1      44    44.00   *                   vluxseg8ei32.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  1      28    28.00   *                   vluxseg8ei64.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  1      16    16.00   *                   vloxseg2ei8.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  1      20    20.00   *                   vloxseg2ei8.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  1      28    28.00   *                   vloxseg2ei8.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  1      44    44.00   *                   vloxseg2ei8.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  1      44    44.00   *                   vloxseg2ei8.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  1      44    44.00   *                   vloxseg2ei8.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  1      20    20.00   *                   vloxseg2ei16.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  1      28    28.00   *                   vloxseg2ei16.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  1      28    28.00   *                   vloxseg2ei16.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  1      28    28.00   *                   vloxseg2ei16.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  1      28    28.00   *                   vloxseg2ei16.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  1      20    20.00   *                   vloxseg2ei32.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  1      20    20.00   *                   vloxseg2ei32.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  1      20    20.00   *                   vloxseg2ei32.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  1      28    28.00   *                   vloxseg2ei32.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  1      16    16.00   *                   vloxseg2ei64.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  1      20    20.00   *                   vloxseg2ei64.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  1      28    28.00   *                   vloxseg2ei64.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  1      18    18.00   *                   vloxseg3ei8.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  1      24    24.00   *                   vloxseg3ei8.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  1      36    36.00   *                   vloxseg3ei8.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  1      60    60.00   *                   vloxseg3ei8.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  1      60    60.00   *                   vloxseg3ei8.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  1      24    24.00   *                   vloxseg3ei16.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  1      36    36.00   *                   vloxseg3ei16.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  1      36    36.00   *                   vloxseg3ei16.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  1      36    36.00   *                   vloxseg3ei16.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  1      24    24.00   *                   vloxseg3ei32.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  1      24    24.00   *                   vloxseg3ei32.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  1      24    24.00   *                   vloxseg3ei32.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  1      18    18.00   *                   vloxseg3ei64.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  1      24    24.00   *                   vloxseg3ei64.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  1      20    20.00   *                   vloxseg4ei8.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  1      28    28.00   *                   vloxseg4ei8.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  1      44    44.00   *                   vloxseg4ei8.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  1      76    76.00   *                   vloxseg4ei8.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  1      76    76.00   *                   vloxseg4ei8.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  1      28    28.00   *                   vloxseg4ei16.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  1      44    44.00   *                   vloxseg4ei16.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  1      44    44.00   *                   vloxseg4ei16.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  1      44    44.00   *                   vloxseg4ei16.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  1      28    28.00   *                   vloxseg4ei32.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  1      28    28.00   *                   vloxseg4ei32.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  1      28    28.00   *                   vloxseg4ei32.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  1      20    20.00   *                   vloxseg4ei64.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  1      28    28.00   *                   vloxseg4ei64.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  1      22    22.00   *                   vloxseg5ei8.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  1      32    32.00   *                   vloxseg5ei8.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  1      52    52.00   *                   vloxseg5ei8.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  1      92    92.00   *                   vloxseg5ei8.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  1      32    32.00   *                   vloxseg5ei16.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  1      52    52.00   *                   vloxseg5ei16.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  1      52    52.00   *                   vloxseg5ei16.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  1      32    32.00   *                   vloxseg5ei32.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  1      32    32.00   *                   vloxseg5ei32.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  1      22    22.00   *                   vloxseg5ei64.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  1      24    24.00   *                   vloxseg6ei8.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  1      36    36.00   *                   vloxseg6ei8.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  1      60    60.00   *                   vloxseg6ei8.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  1      108   108.00  *                   vloxseg6ei8.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  1      36    36.00   *                   vloxseg6ei16.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  1      60    60.00   *                   vloxseg6ei16.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  1      60    60.00   *                   vloxseg6ei16.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  1      36    36.00   *                   vloxseg6ei32.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  1      36    36.00   *                   vloxseg6ei32.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  1      24    24.00   *                   vloxseg6ei64.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  1      26    26.00   *                   vloxseg7ei8.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  1      40    40.00   *                   vloxseg7ei8.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  1      68    68.00   *                   vloxseg7ei8.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  1      124   124.00  *                   vloxseg7ei8.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  1      40    40.00   *                   vloxseg7ei16.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  1      68    68.00   *                   vloxseg7ei16.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  1      68    68.00   *                   vloxseg7ei16.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  1      40    40.00   *                   vloxseg7ei32.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  1      40    40.00   *                   vloxseg7ei32.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  1      26    26.00   *                   vloxseg7ei64.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  1      28    28.00   *                   vloxseg8ei8.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  1      44    44.00   *                   vloxseg8ei8.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  1      76    76.00   *                   vloxseg8ei8.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  1      140   140.00  *                   vloxseg8ei8.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  1      44    44.00   *                   vloxseg8ei16.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  1      76    76.00   *                   vloxseg8ei16.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  1      76    76.00   *                   vloxseg8ei16.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  1      44    44.00   *                   vloxseg8ei32.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  1      44    44.00   *                   vloxseg8ei32.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  1      28    28.00   *                   vloxseg8ei64.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  1      5     16.00          *            vsuxseg2ei8.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  1      9     20.00          *            vsuxseg2ei8.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  1      17    28.00          *            vsuxseg2ei8.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  1      33    44.00          *            vsuxseg2ei8.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  1      65    76.00          *            vsuxseg2ei8.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  1      5     16.00          *            vsuxseg2ei16.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  1      9     20.00          *            vsuxseg2ei16.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  1      17    28.00          *            vsuxseg2ei16.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  1      33    44.00          *            vsuxseg2ei16.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  1      5     16.00          *            vsuxseg2ei32.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  1      9     20.00          *            vsuxseg2ei32.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  1      17    28.00          *            vsuxseg2ei32.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  1      5     16.00          *            vsuxseg2ei64.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  1      9     20.00          *            vsuxseg2ei64.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  1      7     18.00          *            vsuxseg3ei8.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  1      13    24.00          *            vsuxseg3ei8.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  1      25    36.00          *            vsuxseg3ei8.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  1      49    60.00          *            vsuxseg3ei8.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  1      97    108.00         *            vsuxseg3ei8.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  1      7     18.00          *            vsuxseg3ei16.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  1      13    24.00          *            vsuxseg3ei16.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  1      25    36.00          *            vsuxseg3ei16.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  1      49    60.00          *            vsuxseg3ei16.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  1      7     18.00          *            vsuxseg3ei32.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  1      13    24.00          *            vsuxseg3ei32.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  1      25    36.00          *            vsuxseg3ei32.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  1      7     18.00          *            vsuxseg3ei64.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  1      13    24.00          *            vsuxseg3ei64.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  1      9     20.00          *            vsuxseg4ei8.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  1      17    28.00          *            vsuxseg4ei8.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  1      33    44.00          *            vsuxseg4ei8.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  1      65    76.00          *            vsuxseg4ei8.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  1      129   140.00         *            vsuxseg4ei8.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  1      9     20.00          *            vsuxseg4ei16.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  1      17    28.00          *            vsuxseg4ei16.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  1      33    44.00          *            vsuxseg4ei16.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  1      65    76.00          *            vsuxseg4ei16.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  1      9     20.00          *            vsuxseg4ei32.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  1      17    28.00          *            vsuxseg4ei32.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  1      33    44.00          *            vsuxseg4ei32.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  1      9     20.00          *            vsuxseg4ei64.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  1      17    28.00          *            vsuxseg4ei64.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  1      11    22.00          *            vsuxseg5ei8.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  1      21    32.00          *            vsuxseg5ei8.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  1      41    52.00          *            vsuxseg5ei8.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  1      81    92.00          *            vsuxseg5ei8.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  1      11    22.00          *            vsuxseg5ei16.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  1      21    32.00          *            vsuxseg5ei16.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  1      41    52.00          *            vsuxseg5ei16.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  1      11    22.00          *            vsuxseg5ei32.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  1      21    32.00          *            vsuxseg5ei32.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  1      11    22.00          *            vsuxseg5ei64.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  1      13    24.00          *            vsuxseg6ei8.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  1      25    36.00          *            vsuxseg6ei8.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  1      49    60.00          *            vsuxseg6ei8.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  1      97    108.00         *            vsuxseg6ei8.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  1      13    24.00          *            vsuxseg6ei16.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  1      25    36.00          *            vsuxseg6ei16.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  1      49    60.00          *            vsuxseg6ei16.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  1      13    24.00          *            vsuxseg6ei32.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  1      25    36.00          *            vsuxseg6ei32.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  1      13    24.00          *            vsuxseg6ei64.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  1      15    26.00          *            vsuxseg7ei8.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  1      29    40.00          *            vsuxseg7ei8.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  1      57    68.00          *            vsuxseg7ei8.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  1      113   124.00         *            vsuxseg7ei8.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  1      15    26.00          *            vsuxseg7ei16.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  1      29    40.00          *            vsuxseg7ei16.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  1      57    68.00          *            vsuxseg7ei16.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  1      15    26.00          *            vsuxseg7ei32.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  1      29    40.00          *            vsuxseg7ei32.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  1      15    26.00          *            vsuxseg7ei64.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  1      17    28.00          *            vsuxseg8ei8.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  1      33    44.00          *            vsuxseg8ei8.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  1      65    76.00          *            vsuxseg8ei8.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  1      129   140.00         *            vsuxseg8ei8.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  1      17    28.00          *            vsuxseg8ei16.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  1      33    44.00          *            vsuxseg8ei16.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  1      65    76.00          *            vsuxseg8ei16.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  1      17    28.00          *            vsuxseg8ei32.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  1      33    44.00          *            vsuxseg8ei32.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  1      17    28.00          *            vsuxseg8ei64.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  1      5     16.00          *            vsoxseg2ei8.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  1      9     20.00          *            vsoxseg2ei8.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  1      17    28.00          *            vsoxseg2ei8.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  1      33    44.00          *            vsoxseg2ei8.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  1      65    76.00          *            vsoxseg2ei8.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  1      129   140.00         *            vsoxseg2ei8.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  1      5     16.00          *            vsoxseg2ei16.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  1      9     20.00          *            vsoxseg2ei16.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  1      17    28.00          *            vsoxseg2ei16.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  1      33    44.00          *            vsoxseg2ei16.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  1      65    76.00          *            vsoxseg2ei16.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  1      5     16.00          *            vsoxseg2ei32.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  1      9     20.00          *            vsoxseg2ei32.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  1      17    28.00          *            vsoxseg2ei32.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  1      33    44.00          *            vsoxseg2ei32.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  1      5     16.00          *            vsoxseg2ei64.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  1      9     20.00          *            vsoxseg2ei64.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  1      17    28.00          *            vsoxseg2ei64.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  1      7     18.00          *            vsoxseg3ei8.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  1      13    24.00          *            vsoxseg3ei8.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  1      25    36.00          *            vsoxseg3ei8.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  1      49    60.00          *            vsoxseg3ei8.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  1      97    108.00         *            vsoxseg3ei8.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  1      7     18.00          *            vsoxseg3ei16.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  1      13    24.00          *            vsoxseg3ei16.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  1      25    36.00          *            vsoxseg3ei16.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  1      49    60.00          *            vsoxseg3ei16.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  1      7     18.00          *            vsoxseg3ei32.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  1      13    24.00          *            vsoxseg3ei32.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  1      25    36.00          *            vsoxseg3ei32.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  1      7     18.00          *            vsoxseg3ei64.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  1      13    24.00          *            vsoxseg3ei64.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  1      9     20.00          *            vsoxseg4ei8.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  1      17    28.00          *            vsoxseg4ei8.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  1      33    44.00          *            vsoxseg4ei8.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  1      65    76.00          *            vsoxseg4ei8.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  1      129   140.00         *            vsoxseg4ei8.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  1      9     20.00          *            vsoxseg4ei16.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  1      17    28.00          *            vsoxseg4ei16.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  1      33    44.00          *            vsoxseg4ei16.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  1      65    76.00          *            vsoxseg4ei16.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  1      9     20.00          *            vsoxseg4ei32.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  1      17    28.00          *            vsoxseg4ei32.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  1      33    44.00          *            vsoxseg4ei32.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  1      9     20.00          *            vsoxseg4ei64.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  1      17    28.00          *            vsoxseg4ei64.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  1      11    22.00          *            vsoxseg5ei8.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  1      21    32.00          *            vsoxseg5ei8.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  1      41    52.00          *            vsoxseg5ei8.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  1      81    92.00          *            vsoxseg5ei8.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  1      11    22.00          *            vsoxseg5ei16.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  1      21    32.00          *            vsoxseg5ei16.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  1      41    52.00          *            vsoxseg5ei16.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  1      11    22.00          *            vsoxseg5ei32.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  1      21    32.00          *            vsoxseg5ei32.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  1      11    22.00          *            vsoxseg5ei64.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  1      13    24.00          *            vsoxseg6ei8.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  1      25    36.00          *            vsoxseg6ei8.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  1      49    60.00          *            vsoxseg6ei8.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  1      97    108.00         *            vsoxseg6ei8.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  1      13    24.00          *            vsoxseg6ei16.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  1      25    36.00          *            vsoxseg6ei16.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  1      49    60.00          *            vsoxseg6ei16.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  1      13    24.00          *            vsoxseg6ei32.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  1      25    36.00          *            vsoxseg6ei32.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  1      13    24.00          *            vsoxseg6ei64.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  1      15    26.00          *            vsoxseg7ei8.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  1      29    40.00          *            vsoxseg7ei8.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  1      57    68.00          *            vsoxseg7ei8.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  1      113   124.00         *            vsoxseg7ei8.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  1      15    26.00          *            vsoxseg7ei16.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  1      29    40.00          *            vsoxseg7ei16.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  1      57    68.00          *            vsoxseg7ei16.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  1      15    26.00          *            vsoxseg7ei32.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  1      29    40.00          *            vsoxseg7ei32.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  1      15    26.00          *            vsoxseg7ei64.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  1      17    28.00          *            vsoxseg8ei8.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  1      33    44.00          *            vsoxseg8ei8.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  1      65    76.00          *            vsoxseg8ei8.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  1      129   140.00         *            vsoxseg8ei8.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  1      17    28.00          *            vsoxseg8ei16.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  1      33    44.00          *            vsoxseg8ei16.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  1      65    76.00          *            vsoxseg8ei16.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  1      17    28.00          *            vsoxseg8ei32.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  1      33    44.00          *            vsoxseg8ei32.v	v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli	zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  1      17    28.00          *            vsoxseg8ei64.v	v8, (a0), v16
 
 # CHECK:      Resources:
 # CHECK-NEXT: [0]   - SiFiveP400Div
@@ -6581,3247 +3181,1547 @@ vsoxseg8ei64.v  v8, (a0), v16
 
 # CHECK:      Resource pressure per iteration:
 # CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]   [12]
-# CHECK-NEXT:  -      -      -      -     1620.00  -     -      -      -      -      -     124596.00 96850.00
+# CHECK-NEXT:  -      -      -      -     770.00  -      -      -      -      -      -     17896.00 14240.00
 
 # CHECK:      Resource pressure by instruction:
 # CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]   [12]   Instructions:
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     16.00   -     vlseg2e8.v v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     20.00   -     vlseg2e8.v v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vlseg2e8.v v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     44.00   -     vlseg2e8.v v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     76.00   -     vlseg2e8.v v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     140.00  -     vlseg2e8.v v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     268.00  -     vlseg2e8.v v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     140.00  -     vlseg2e16.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     16.00   -     vlseg2e16.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     20.00   -     vlseg2e16.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vlseg2e16.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     44.00   -     vlseg2e16.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     76.00   -     vlseg2e16.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     140.00  -     vlseg2e16.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     76.00   -     vlseg2e32.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     76.00   -     vlseg2e32.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     16.00   -     vlseg2e32.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     20.00   -     vlseg2e32.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vlseg2e32.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     44.00   -     vlseg2e32.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     76.00   -     vlseg2e32.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     44.00   -     vlseg2e64.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     44.00   -     vlseg2e64.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     44.00   -     vlseg2e64.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     16.00   -     vlseg2e64.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     20.00   -     vlseg2e64.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vlseg2e64.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     44.00   -     vlseg2e64.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     18.00   -     vlseg3e8.v v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     24.00   -     vlseg3e8.v v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     36.00   -     vlseg3e8.v v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     60.00   -     vlseg3e8.v v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     108.00  -     vlseg3e8.v v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     396.00  -     vlseg3e8.v v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     396.00  -     vlseg3e8.v v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     204.00  -     vlseg3e16.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     18.00   -     vlseg3e16.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     24.00   -     vlseg3e16.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     36.00   -     vlseg3e16.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     60.00   -     vlseg3e16.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     204.00  -     vlseg3e16.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     204.00  -     vlseg3e16.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     108.00  -     vlseg3e32.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     108.00  -     vlseg3e32.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     18.00   -     vlseg3e32.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     24.00   -     vlseg3e32.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     36.00   -     vlseg3e32.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     108.00  -     vlseg3e32.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     108.00  -     vlseg3e32.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     60.00   -     vlseg3e64.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     60.00   -     vlseg3e64.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     60.00   -     vlseg3e64.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     18.00   -     vlseg3e64.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     24.00   -     vlseg3e64.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     60.00   -     vlseg3e64.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     60.00   -     vlseg3e64.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     20.00   -     vlseg4e8.v v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vlseg4e8.v v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     44.00   -     vlseg4e8.v v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     76.00   -     vlseg4e8.v v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     140.00  -     vlseg4e8.v v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     524.00  -     vlseg4e8.v v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     524.00  -     vlseg4e8.v v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     268.00  -     vlseg4e16.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     20.00   -     vlseg4e16.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vlseg4e16.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     44.00   -     vlseg4e16.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     76.00   -     vlseg4e16.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     268.00  -     vlseg4e16.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     268.00  -     vlseg4e16.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     140.00  -     vlseg4e32.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     140.00  -     vlseg4e32.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     20.00   -     vlseg4e32.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vlseg4e32.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     44.00   -     vlseg4e32.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     140.00  -     vlseg4e32.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     140.00  -     vlseg4e32.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     76.00   -     vlseg4e64.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     76.00   -     vlseg4e64.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     76.00   -     vlseg4e64.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     20.00   -     vlseg4e64.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vlseg4e64.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     76.00   -     vlseg4e64.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     76.00   -     vlseg4e64.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     22.00   -     vlseg5e8.v v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     32.00   -     vlseg5e8.v v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     52.00   -     vlseg5e8.v v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     92.00   -     vlseg5e8.v v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     652.00  -     vlseg5e8.v v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     652.00  -     vlseg5e8.v v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     332.00  -     vlseg5e16.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     22.00   -     vlseg5e16.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     32.00   -     vlseg5e16.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     52.00   -     vlseg5e16.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     332.00  -     vlseg5e16.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     332.00  -     vlseg5e16.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     172.00  -     vlseg5e32.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     172.00  -     vlseg5e32.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     22.00   -     vlseg5e32.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     32.00   -     vlseg5e32.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     172.00  -     vlseg5e32.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     172.00  -     vlseg5e32.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     92.00   -     vlseg5e64.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     92.00   -     vlseg5e64.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     92.00   -     vlseg5e64.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     22.00   -     vlseg5e64.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     92.00   -     vlseg5e64.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     92.00   -     vlseg5e64.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     24.00   -     vlseg6e8.v v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     36.00   -     vlseg6e8.v v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     60.00   -     vlseg6e8.v v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     108.00  -     vlseg6e8.v v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     780.00  -     vlseg6e8.v v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     780.00  -     vlseg6e8.v v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     396.00  -     vlseg6e16.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     24.00   -     vlseg6e16.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     36.00   -     vlseg6e16.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     60.00   -     vlseg6e16.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     396.00  -     vlseg6e16.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     396.00  -     vlseg6e16.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     204.00  -     vlseg6e32.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     204.00  -     vlseg6e32.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     24.00   -     vlseg6e32.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     36.00   -     vlseg6e32.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     204.00  -     vlseg6e32.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     204.00  -     vlseg6e32.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     108.00  -     vlseg6e64.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     108.00  -     vlseg6e64.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     108.00  -     vlseg6e64.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     24.00   -     vlseg6e64.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     108.00  -     vlseg6e64.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     108.00  -     vlseg6e64.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     26.00   -     vlseg7e8.v v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     40.00   -     vlseg7e8.v v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     68.00   -     vlseg7e8.v v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     124.00  -     vlseg7e8.v v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     908.00  -     vlseg7e8.v v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     908.00  -     vlseg7e8.v v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     460.00  -     vlseg7e16.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     26.00   -     vlseg7e16.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     40.00   -     vlseg7e16.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     68.00   -     vlseg7e16.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     460.00  -     vlseg7e16.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     460.00  -     vlseg7e16.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     236.00  -     vlseg7e32.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     236.00  -     vlseg7e32.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     26.00   -     vlseg7e32.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     40.00   -     vlseg7e32.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     236.00  -     vlseg7e32.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     236.00  -     vlseg7e32.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     124.00  -     vlseg7e64.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     124.00  -     vlseg7e64.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     124.00  -     vlseg7e64.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     26.00   -     vlseg7e64.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     124.00  -     vlseg7e64.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     124.00  -     vlseg7e64.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vlseg8e8.v v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     44.00   -     vlseg8e8.v v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     76.00   -     vlseg8e8.v v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     140.00  -     vlseg8e8.v v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1036.00  -    vlseg8e8.v v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1036.00  -    vlseg8e8.v v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     524.00  -     vlseg8e16.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vlseg8e16.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     44.00   -     vlseg8e16.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     76.00   -     vlseg8e16.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     524.00  -     vlseg8e16.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     524.00  -     vlseg8e16.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     268.00  -     vlseg8e32.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     268.00  -     vlseg8e32.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vlseg8e32.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     44.00   -     vlseg8e32.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     268.00  -     vlseg8e32.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     268.00  -     vlseg8e32.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     140.00  -     vlseg8e64.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     140.00  -     vlseg8e64.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     140.00  -     vlseg8e64.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vlseg8e64.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     140.00  -     vlseg8e64.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     140.00  -     vlseg8e64.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     16.00  vsseg2e8.v v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     20.00  vsseg2e8.v v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     28.00  vsseg2e8.v v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     44.00  vsseg2e8.v v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     76.00  vsseg2e8.v v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     140.00 vsseg2e8.v v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     268.00 vsseg2e8.v v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     140.00 vsseg2e16.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     16.00  vsseg2e16.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     20.00  vsseg2e16.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     28.00  vsseg2e16.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     44.00  vsseg2e16.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     76.00  vsseg2e16.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     140.00 vsseg2e16.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     76.00  vsseg2e32.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     76.00  vsseg2e32.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     16.00  vsseg2e32.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     20.00  vsseg2e32.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     28.00  vsseg2e32.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     44.00  vsseg2e32.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     76.00  vsseg2e32.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     44.00  vsseg2e64.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     44.00  vsseg2e64.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     44.00  vsseg2e64.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     16.00  vsseg2e64.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     20.00  vsseg2e64.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     28.00  vsseg2e64.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     44.00  vsseg2e64.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     18.00  vsseg3e8.v v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     24.00  vsseg3e8.v v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     36.00  vsseg3e8.v v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     60.00  vsseg3e8.v v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     108.00 vsseg3e8.v v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     396.00 vsseg3e8.v v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     396.00 vsseg3e8.v v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     204.00 vsseg3e16.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     18.00  vsseg3e16.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     24.00  vsseg3e16.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     36.00  vsseg3e16.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     60.00  vsseg3e16.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     204.00 vsseg3e16.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     204.00 vsseg3e16.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     108.00 vsseg3e32.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     108.00 vsseg3e32.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     18.00  vsseg3e32.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     24.00  vsseg3e32.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     36.00  vsseg3e32.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     108.00 vsseg3e32.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     108.00 vsseg3e32.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     60.00  vsseg3e64.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     60.00  vsseg3e64.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     60.00  vsseg3e64.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     18.00  vsseg3e64.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     24.00  vsseg3e64.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     60.00  vsseg3e64.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     60.00  vsseg3e64.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     20.00  vsseg4e8.v v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     28.00  vsseg4e8.v v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     44.00  vsseg4e8.v v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     76.00  vsseg4e8.v v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     140.00 vsseg4e8.v v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     524.00 vsseg4e8.v v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     524.00 vsseg4e8.v v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     268.00 vsseg4e16.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     20.00  vsseg4e16.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     28.00  vsseg4e16.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     44.00  vsseg4e16.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     76.00  vsseg4e16.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     268.00 vsseg4e16.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     268.00 vsseg4e16.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     140.00 vsseg4e32.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     140.00 vsseg4e32.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     20.00  vsseg4e32.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     28.00  vsseg4e32.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     44.00  vsseg4e32.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     140.00 vsseg4e32.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     140.00 vsseg4e32.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     76.00  vsseg4e64.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     76.00  vsseg4e64.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     76.00  vsseg4e64.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     20.00  vsseg4e64.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     28.00  vsseg4e64.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     76.00  vsseg4e64.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     76.00  vsseg4e64.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     22.00  vsseg5e8.v v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     32.00  vsseg5e8.v v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     52.00  vsseg5e8.v v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     92.00  vsseg5e8.v v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     652.00 vsseg5e8.v v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     652.00 vsseg5e8.v v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     332.00 vsseg5e16.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     22.00  vsseg5e16.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     32.00  vsseg5e16.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     52.00  vsseg5e16.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     332.00 vsseg5e16.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     332.00 vsseg5e16.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     172.00 vsseg5e32.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     172.00 vsseg5e32.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     22.00  vsseg5e32.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     32.00  vsseg5e32.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     172.00 vsseg5e32.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     172.00 vsseg5e32.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     92.00  vsseg5e64.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     92.00  vsseg5e64.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     92.00  vsseg5e64.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     22.00  vsseg5e64.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     92.00  vsseg5e64.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     92.00  vsseg5e64.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     24.00  vsseg6e8.v v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     36.00  vsseg6e8.v v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     60.00  vsseg6e8.v v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     108.00 vsseg6e8.v v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     780.00 vsseg6e8.v v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     780.00 vsseg6e8.v v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     396.00 vsseg6e16.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     24.00  vsseg6e16.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     36.00  vsseg6e16.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     60.00  vsseg6e16.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     396.00 vsseg6e16.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     396.00 vsseg6e16.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     204.00 vsseg6e32.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     204.00 vsseg6e32.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     24.00  vsseg6e32.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     36.00  vsseg6e32.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     204.00 vsseg6e32.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     204.00 vsseg6e32.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     108.00 vsseg6e64.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     108.00 vsseg6e64.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     108.00 vsseg6e64.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     24.00  vsseg6e64.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     108.00 vsseg6e64.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     108.00 vsseg6e64.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     26.00  vsseg7e8.v v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     40.00  vsseg7e8.v v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     68.00  vsseg7e8.v v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     124.00 vsseg7e8.v v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     908.00 vsseg7e8.v v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     908.00 vsseg7e8.v v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     460.00 vsseg7e16.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     26.00  vsseg7e16.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     40.00  vsseg7e16.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     68.00  vsseg7e16.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     460.00 vsseg7e16.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     460.00 vsseg7e16.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     236.00 vsseg7e32.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     236.00 vsseg7e32.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     26.00  vsseg7e32.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     40.00  vsseg7e32.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     236.00 vsseg7e32.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     236.00 vsseg7e32.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     124.00 vsseg7e64.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     124.00 vsseg7e64.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     124.00 vsseg7e64.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     26.00  vsseg7e64.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     124.00 vsseg7e64.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     124.00 vsseg7e64.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     28.00  vsseg8e8.v v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     44.00  vsseg8e8.v v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     76.00  vsseg8e8.v v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     140.00 vsseg8e8.v v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     1036.00 vsseg8e8.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     1036.00 vsseg8e8.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     524.00 vsseg8e16.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     28.00  vsseg8e16.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     44.00  vsseg8e16.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     76.00  vsseg8e16.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     524.00 vsseg8e16.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     524.00 vsseg8e16.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     268.00 vsseg8e32.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     268.00 vsseg8e32.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     28.00  vsseg8e32.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     44.00  vsseg8e32.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     268.00 vsseg8e32.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     268.00 vsseg8e32.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     140.00 vsseg8e64.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     140.00 vsseg8e64.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     140.00 vsseg8e64.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     28.00  vsseg8e64.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     140.00 vsseg8e64.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     140.00 vsseg8e64.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     16.00   -     vlsseg2e8.v  v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     20.00   -     vlsseg2e8.v  v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vlsseg2e8.v  v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     44.00   -     vlsseg2e8.v  v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     76.00   -     vlsseg2e8.v  v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     140.00  -     vlsseg2e8.v  v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     268.00  -     vlsseg2e8.v  v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     140.00  -     vlsseg2e16.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     16.00   -     vlsseg2e16.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     20.00   -     vlsseg2e16.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vlsseg2e16.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     44.00   -     vlsseg2e16.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     76.00   -     vlsseg2e16.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     140.00  -     vlsseg2e16.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     76.00   -     vlsseg2e32.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     76.00   -     vlsseg2e32.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     16.00   -     vlsseg2e32.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     20.00   -     vlsseg2e32.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vlsseg2e32.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     44.00   -     vlsseg2e32.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     76.00   -     vlsseg2e32.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     44.00   -     vlsseg2e64.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     44.00   -     vlsseg2e64.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     44.00   -     vlsseg2e64.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     16.00   -     vlsseg2e64.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     20.00   -     vlsseg2e64.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vlsseg2e64.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     44.00   -     vlsseg2e64.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     18.00   -     vlsseg3e8.v  v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     24.00   -     vlsseg3e8.v  v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     36.00   -     vlsseg3e8.v  v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     60.00   -     vlsseg3e8.v  v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     108.00  -     vlsseg3e8.v  v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     396.00  -     vlsseg3e8.v  v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     396.00  -     vlsseg3e8.v  v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     204.00  -     vlsseg3e16.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     18.00   -     vlsseg3e16.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     24.00   -     vlsseg3e16.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     36.00   -     vlsseg3e16.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     60.00   -     vlsseg3e16.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     204.00  -     vlsseg3e16.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     204.00  -     vlsseg3e16.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     108.00  -     vlsseg3e32.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     108.00  -     vlsseg3e32.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     18.00   -     vlsseg3e32.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     24.00   -     vlsseg3e32.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     36.00   -     vlsseg3e32.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     108.00  -     vlsseg3e32.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     108.00  -     vlsseg3e32.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     60.00   -     vlsseg3e64.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     60.00   -     vlsseg3e64.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     60.00   -     vlsseg3e64.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     18.00   -     vlsseg3e64.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     24.00   -     vlsseg3e64.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     60.00   -     vlsseg3e64.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     60.00   -     vlsseg3e64.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     20.00   -     vlsseg4e8.v  v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vlsseg4e8.v  v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     44.00   -     vlsseg4e8.v  v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     76.00   -     vlsseg4e8.v  v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     140.00  -     vlsseg4e8.v  v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     524.00  -     vlsseg4e8.v  v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     524.00  -     vlsseg4e8.v  v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     268.00  -     vlsseg4e16.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     20.00   -     vlsseg4e16.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vlsseg4e16.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     44.00   -     vlsseg4e16.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     76.00   -     vlsseg4e16.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     268.00  -     vlsseg4e16.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     268.00  -     vlsseg4e16.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     140.00  -     vlsseg4e32.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     140.00  -     vlsseg4e32.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     20.00   -     vlsseg4e32.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vlsseg4e32.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     44.00   -     vlsseg4e32.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     140.00  -     vlsseg4e32.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     140.00  -     vlsseg4e32.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     76.00   -     vlsseg4e64.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     76.00   -     vlsseg4e64.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     76.00   -     vlsseg4e64.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     20.00   -     vlsseg4e64.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vlsseg4e64.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     76.00   -     vlsseg4e64.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     76.00   -     vlsseg4e64.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     22.00   -     vlsseg5e8.v  v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     32.00   -     vlsseg5e8.v  v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     52.00   -     vlsseg5e8.v  v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     92.00   -     vlsseg5e8.v  v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     652.00  -     vlsseg5e8.v  v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     652.00  -     vlsseg5e8.v  v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     332.00  -     vlsseg5e16.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     22.00   -     vlsseg5e16.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     32.00   -     vlsseg5e16.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     52.00   -     vlsseg5e16.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     332.00  -     vlsseg5e16.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     332.00  -     vlsseg5e16.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     172.00  -     vlsseg5e32.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     172.00  -     vlsseg5e32.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     22.00   -     vlsseg5e32.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     32.00   -     vlsseg5e32.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     172.00  -     vlsseg5e32.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     172.00  -     vlsseg5e32.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     92.00   -     vlsseg5e64.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     92.00   -     vlsseg5e64.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     92.00   -     vlsseg5e64.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     22.00   -     vlsseg5e64.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     92.00   -     vlsseg5e64.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     92.00   -     vlsseg5e64.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     24.00   -     vlsseg6e8.v  v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     36.00   -     vlsseg6e8.v  v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     60.00   -     vlsseg6e8.v  v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     108.00  -     vlsseg6e8.v  v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     780.00  -     vlsseg6e8.v  v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     780.00  -     vlsseg6e8.v  v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     396.00  -     vlsseg6e16.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     24.00   -     vlsseg6e16.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     36.00   -     vlsseg6e16.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     60.00   -     vlsseg6e16.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     396.00  -     vlsseg6e16.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     396.00  -     vlsseg6e16.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     204.00  -     vlsseg6e32.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     204.00  -     vlsseg6e32.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     24.00   -     vlsseg6e32.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     36.00   -     vlsseg6e32.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     204.00  -     vlsseg6e32.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     204.00  -     vlsseg6e32.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     108.00  -     vlsseg6e64.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     108.00  -     vlsseg6e64.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     108.00  -     vlsseg6e64.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     24.00   -     vlsseg6e64.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     108.00  -     vlsseg6e64.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     108.00  -     vlsseg6e64.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     26.00   -     vlsseg7e8.v  v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     40.00   -     vlsseg7e8.v  v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     68.00   -     vlsseg7e8.v  v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     124.00  -     vlsseg7e8.v  v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     908.00  -     vlsseg7e8.v  v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     908.00  -     vlsseg7e8.v  v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     460.00  -     vlsseg7e16.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     26.00   -     vlsseg7e16.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     40.00   -     vlsseg7e16.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     68.00   -     vlsseg7e16.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     460.00  -     vlsseg7e16.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     460.00  -     vlsseg7e16.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     236.00  -     vlsseg7e32.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     236.00  -     vlsseg7e32.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     26.00   -     vlsseg7e32.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     40.00   -     vlsseg7e32.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     236.00  -     vlsseg7e32.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     236.00  -     vlsseg7e32.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     124.00  -     vlsseg7e64.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     124.00  -     vlsseg7e64.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     124.00  -     vlsseg7e64.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     26.00   -     vlsseg7e64.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     124.00  -     vlsseg7e64.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     124.00  -     vlsseg7e64.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vlsseg8e8.v  v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     44.00   -     vlsseg8e8.v  v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     76.00   -     vlsseg8e8.v  v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     140.00  -     vlsseg8e8.v  v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1036.00  -    vlsseg8e8.v  v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1036.00  -    vlsseg8e8.v  v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     524.00  -     vlsseg8e16.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vlsseg8e16.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     44.00   -     vlsseg8e16.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     76.00   -     vlsseg8e16.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     524.00  -     vlsseg8e16.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     524.00  -     vlsseg8e16.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     268.00  -     vlsseg8e32.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     268.00  -     vlsseg8e32.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vlsseg8e32.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     44.00   -     vlsseg8e32.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     268.00  -     vlsseg8e32.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     268.00  -     vlsseg8e32.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     140.00  -     vlsseg8e64.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     140.00  -     vlsseg8e64.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     140.00  -     vlsseg8e64.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vlsseg8e64.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     140.00  -     vlsseg8e64.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     140.00  -     vlsseg8e64.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     16.00  vssseg2e8.v  v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     20.00  vssseg2e8.v  v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     28.00  vssseg2e8.v  v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     44.00  vssseg2e8.v  v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     76.00  vssseg2e8.v  v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     140.00 vssseg2e8.v  v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     268.00 vssseg2e8.v  v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     140.00 vssseg2e16.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     16.00  vssseg2e16.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     20.00  vssseg2e16.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     28.00  vssseg2e16.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     44.00  vssseg2e16.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     76.00  vssseg2e16.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     140.00 vssseg2e16.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     76.00  vssseg2e32.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     76.00  vssseg2e32.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     16.00  vssseg2e32.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     20.00  vssseg2e32.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     28.00  vssseg2e32.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     44.00  vssseg2e32.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     76.00  vssseg2e32.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     44.00  vssseg2e64.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     44.00  vssseg2e64.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     44.00  vssseg2e64.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     16.00  vssseg2e64.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     20.00  vssseg2e64.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     28.00  vssseg2e64.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     44.00  vssseg2e64.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     18.00  vssseg3e8.v  v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     24.00  vssseg3e8.v  v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     36.00  vssseg3e8.v  v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     60.00  vssseg3e8.v  v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     108.00 vssseg3e8.v  v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     396.00 vssseg3e8.v  v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     396.00 vssseg3e8.v  v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     204.00 vssseg3e16.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     18.00  vssseg3e16.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     24.00  vssseg3e16.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     36.00  vssseg3e16.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     60.00  vssseg3e16.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     204.00 vssseg3e16.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     204.00 vssseg3e16.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     108.00 vssseg3e32.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     108.00 vssseg3e32.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     18.00  vssseg3e32.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     24.00  vssseg3e32.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     36.00  vssseg3e32.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     108.00 vssseg3e32.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     108.00 vssseg3e32.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     60.00  vssseg3e64.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     60.00  vssseg3e64.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     60.00  vssseg3e64.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     18.00  vssseg3e64.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     24.00  vssseg3e64.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     60.00  vssseg3e64.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     60.00  vssseg3e64.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     20.00  vssseg4e8.v  v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     28.00  vssseg4e8.v  v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     44.00  vssseg4e8.v  v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     76.00  vssseg4e8.v  v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     140.00 vssseg4e8.v  v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     524.00 vssseg4e8.v  v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     524.00 vssseg4e8.v  v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     268.00 vssseg4e16.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     20.00  vssseg4e16.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     28.00  vssseg4e16.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     44.00  vssseg4e16.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     76.00  vssseg4e16.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     268.00 vssseg4e16.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     268.00 vssseg4e16.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     140.00 vssseg4e32.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     140.00 vssseg4e32.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     20.00  vssseg4e32.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     28.00  vssseg4e32.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     44.00  vssseg4e32.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     140.00 vssseg4e32.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     140.00 vssseg4e32.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     76.00  vssseg4e64.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     76.00  vssseg4e64.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     76.00  vssseg4e64.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     20.00  vssseg4e64.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     28.00  vssseg4e64.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     76.00  vssseg4e64.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     76.00  vssseg4e64.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     22.00  vssseg5e8.v  v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     32.00  vssseg5e8.v  v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     52.00  vssseg5e8.v  v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     92.00  vssseg5e8.v  v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     652.00 vssseg5e8.v  v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     652.00 vssseg5e8.v  v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     332.00 vssseg5e16.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     22.00  vssseg5e16.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     32.00  vssseg5e16.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     52.00  vssseg5e16.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     332.00 vssseg5e16.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     332.00 vssseg5e16.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     172.00 vssseg5e32.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     172.00 vssseg5e32.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     22.00  vssseg5e32.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     32.00  vssseg5e32.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     172.00 vssseg5e32.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     172.00 vssseg5e32.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     92.00  vssseg5e64.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     92.00  vssseg5e64.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     92.00  vssseg5e64.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     22.00  vssseg5e64.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     92.00  vssseg5e64.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     92.00  vssseg5e64.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     24.00  vssseg6e8.v  v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     36.00  vssseg6e8.v  v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     60.00  vssseg6e8.v  v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     108.00 vssseg6e8.v  v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     780.00 vssseg6e8.v  v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     780.00 vssseg6e8.v  v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     396.00 vssseg6e16.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     24.00  vssseg6e16.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     36.00  vssseg6e16.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     60.00  vssseg6e16.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     396.00 vssseg6e16.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     396.00 vssseg6e16.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     204.00 vssseg6e32.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     204.00 vssseg6e32.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     24.00  vssseg6e32.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     36.00  vssseg6e32.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     204.00 vssseg6e32.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     204.00 vssseg6e32.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     108.00 vssseg6e64.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     108.00 vssseg6e64.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     108.00 vssseg6e64.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     24.00  vssseg6e64.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     108.00 vssseg6e64.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     108.00 vssseg6e64.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     26.00  vssseg7e8.v  v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     40.00  vssseg7e8.v  v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     68.00  vssseg7e8.v  v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     124.00 vssseg7e8.v  v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     908.00 vssseg7e8.v  v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     908.00 vssseg7e8.v  v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     460.00 vssseg7e16.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     26.00  vssseg7e16.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     40.00  vssseg7e16.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     68.00  vssseg7e16.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     460.00 vssseg7e16.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     460.00 vssseg7e16.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     236.00 vssseg7e32.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     236.00 vssseg7e32.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     26.00  vssseg7e32.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     40.00  vssseg7e32.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     236.00 vssseg7e32.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     236.00 vssseg7e32.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     124.00 vssseg7e64.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     124.00 vssseg7e64.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     124.00 vssseg7e64.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     26.00  vssseg7e64.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     124.00 vssseg7e64.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     124.00 vssseg7e64.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     28.00  vssseg8e8.v  v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     44.00  vssseg8e8.v  v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     76.00  vssseg8e8.v  v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     140.00 vssseg8e8.v  v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     1036.00 vssseg8e8.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     1036.00 vssseg8e8.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     524.00 vssseg8e16.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     28.00  vssseg8e16.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     44.00  vssseg8e16.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     76.00  vssseg8e16.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     524.00 vssseg8e16.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     524.00 vssseg8e16.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     268.00 vssseg8e32.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     268.00 vssseg8e32.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     28.00  vssseg8e32.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     44.00  vssseg8e32.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     268.00 vssseg8e32.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     268.00 vssseg8e32.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     140.00 vssseg8e64.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     140.00 vssseg8e64.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     140.00 vssseg8e64.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     28.00  vssseg8e64.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     140.00 vssseg8e64.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     140.00 vssseg8e64.v v8, (a0), a1
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     16.00   -     vlseg2e8ff.v v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     20.00   -     vlseg2e8ff.v v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vlseg2e8ff.v v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     44.00   -     vlseg2e8ff.v v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     76.00   -     vlseg2e8ff.v v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     140.00  -     vlseg2e8ff.v v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     268.00  -     vlseg2e8ff.v v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     140.00  -     vlseg2e16ff.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     16.00   -     vlseg2e16ff.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     20.00   -     vlseg2e16ff.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vlseg2e16ff.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     44.00   -     vlseg2e16ff.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     76.00   -     vlseg2e16ff.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     140.00  -     vlseg2e16ff.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     76.00   -     vlseg2e32ff.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     76.00   -     vlseg2e32ff.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     16.00   -     vlseg2e32ff.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     20.00   -     vlseg2e32ff.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vlseg2e32ff.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     44.00   -     vlseg2e32ff.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     76.00   -     vlseg2e32ff.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     44.00   -     vlseg2e64ff.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     44.00   -     vlseg2e64ff.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     44.00   -     vlseg2e64ff.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     16.00   -     vlseg2e64ff.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     20.00   -     vlseg2e64ff.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vlseg2e64ff.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     44.00   -     vlseg2e64ff.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     18.00   -     vlseg3e8ff.v v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     24.00   -     vlseg3e8ff.v v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     36.00   -     vlseg3e8ff.v v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     60.00   -     vlseg3e8ff.v v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     108.00  -     vlseg3e8ff.v v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     396.00  -     vlseg3e8ff.v v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     396.00  -     vlseg3e8ff.v v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     204.00  -     vlseg3e16ff.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     18.00   -     vlseg3e16ff.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     24.00   -     vlseg3e16ff.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     36.00   -     vlseg3e16ff.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     60.00   -     vlseg3e16ff.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     204.00  -     vlseg3e16ff.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     204.00  -     vlseg3e16ff.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     108.00  -     vlseg3e32ff.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     108.00  -     vlseg3e32ff.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     18.00   -     vlseg3e32ff.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     24.00   -     vlseg3e32ff.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     36.00   -     vlseg3e32ff.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     108.00  -     vlseg3e32ff.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     108.00  -     vlseg3e32ff.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     60.00   -     vlseg3e64ff.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     60.00   -     vlseg3e64ff.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     60.00   -     vlseg3e64ff.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     18.00   -     vlseg3e64ff.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     24.00   -     vlseg3e64ff.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     60.00   -     vlseg3e64ff.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     60.00   -     vlseg3e64ff.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     20.00   -     vlseg4e8ff.v v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vlseg4e8ff.v v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     44.00   -     vlseg4e8ff.v v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     76.00   -     vlseg4e8ff.v v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     140.00  -     vlseg4e8ff.v v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     524.00  -     vlseg4e8ff.v v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     524.00  -     vlseg4e8ff.v v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     268.00  -     vlseg4e16ff.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     20.00   -     vlseg4e16ff.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vlseg4e16ff.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     44.00   -     vlseg4e16ff.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     76.00   -     vlseg4e16ff.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     268.00  -     vlseg4e16ff.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     268.00  -     vlseg4e16ff.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     140.00  -     vlseg4e32ff.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     140.00  -     vlseg4e32ff.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     20.00   -     vlseg4e32ff.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vlseg4e32ff.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     44.00   -     vlseg4e32ff.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     140.00  -     vlseg4e32ff.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     140.00  -     vlseg4e32ff.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     76.00   -     vlseg4e64ff.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     76.00   -     vlseg4e64ff.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     76.00   -     vlseg4e64ff.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     20.00   -     vlseg4e64ff.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vlseg4e64ff.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     76.00   -     vlseg4e64ff.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     76.00   -     vlseg4e64ff.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     22.00   -     vlseg5e8ff.v v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     32.00   -     vlseg5e8ff.v v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     52.00   -     vlseg5e8ff.v v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     92.00   -     vlseg5e8ff.v v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     652.00  -     vlseg5e8ff.v v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     652.00  -     vlseg5e8ff.v v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     332.00  -     vlseg5e16ff.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     22.00   -     vlseg5e16ff.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     32.00   -     vlseg5e16ff.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     52.00   -     vlseg5e16ff.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     332.00  -     vlseg5e16ff.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     332.00  -     vlseg5e16ff.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     172.00  -     vlseg5e32ff.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     172.00  -     vlseg5e32ff.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     22.00   -     vlseg5e32ff.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     32.00   -     vlseg5e32ff.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     172.00  -     vlseg5e32ff.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     172.00  -     vlseg5e32ff.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     92.00   -     vlseg5e64ff.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     92.00   -     vlseg5e64ff.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     92.00   -     vlseg5e64ff.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     22.00   -     vlseg5e64ff.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     92.00   -     vlseg5e64ff.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     92.00   -     vlseg5e64ff.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     24.00   -     vlseg6e8ff.v v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     36.00   -     vlseg6e8ff.v v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     60.00   -     vlseg6e8ff.v v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     108.00  -     vlseg6e8ff.v v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     780.00  -     vlseg6e8ff.v v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     780.00  -     vlseg6e8ff.v v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     396.00  -     vlseg6e16ff.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     24.00   -     vlseg6e16ff.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     36.00   -     vlseg6e16ff.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     60.00   -     vlseg6e16ff.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     396.00  -     vlseg6e16ff.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     396.00  -     vlseg6e16ff.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     204.00  -     vlseg6e32ff.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     204.00  -     vlseg6e32ff.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     24.00   -     vlseg6e32ff.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     36.00   -     vlseg6e32ff.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     204.00  -     vlseg6e32ff.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     204.00  -     vlseg6e32ff.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     108.00  -     vlseg6e64ff.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     108.00  -     vlseg6e64ff.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     108.00  -     vlseg6e64ff.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     24.00   -     vlseg6e64ff.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     108.00  -     vlseg6e64ff.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     108.00  -     vlseg6e64ff.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     26.00   -     vlseg7e8ff.v v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     40.00   -     vlseg7e8ff.v v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     68.00   -     vlseg7e8ff.v v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     124.00  -     vlseg7e8ff.v v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     908.00  -     vlseg7e8ff.v v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     908.00  -     vlseg7e8ff.v v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     460.00  -     vlseg7e16ff.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     26.00   -     vlseg7e16ff.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     40.00   -     vlseg7e16ff.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     68.00   -     vlseg7e16ff.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     460.00  -     vlseg7e16ff.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     460.00  -     vlseg7e16ff.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     236.00  -     vlseg7e32ff.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     236.00  -     vlseg7e32ff.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     26.00   -     vlseg7e32ff.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     40.00   -     vlseg7e32ff.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     236.00  -     vlseg7e32ff.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     236.00  -     vlseg7e32ff.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     124.00  -     vlseg7e64ff.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     124.00  -     vlseg7e64ff.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     124.00  -     vlseg7e64ff.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     26.00   -     vlseg7e64ff.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     124.00  -     vlseg7e64ff.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     124.00  -     vlseg7e64ff.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vlseg8e8ff.v v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     44.00   -     vlseg8e8ff.v v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     76.00   -     vlseg8e8ff.v v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     140.00  -     vlseg8e8ff.v v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1036.00  -    vlseg8e8ff.v v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1036.00  -    vlseg8e8ff.v v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     524.00  -     vlseg8e16ff.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vlseg8e16ff.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     44.00   -     vlseg8e16ff.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     76.00   -     vlseg8e16ff.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     524.00  -     vlseg8e16ff.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     524.00  -     vlseg8e16ff.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     268.00  -     vlseg8e32ff.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     268.00  -     vlseg8e32ff.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vlseg8e32ff.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     44.00   -     vlseg8e32ff.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     268.00  -     vlseg8e32ff.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     268.00  -     vlseg8e32ff.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     140.00  -     vlseg8e64ff.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     140.00  -     vlseg8e64ff.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     140.00  -     vlseg8e64ff.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vlseg8e64ff.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     140.00  -     vlseg8e64ff.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     140.00  -     vlseg8e64ff.v  v8, (a0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     16.00   -     vluxseg2ei8.v  v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     20.00   -     vluxseg2ei8.v  v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vluxseg2ei8.v  v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     44.00   -     vluxseg2ei8.v  v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     44.00   -     vluxseg2ei8.v  v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     44.00   -     vluxseg2ei8.v  v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     268.00  -     vluxseg2ei8.v  v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     16.00   -     vluxseg2ei16.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     20.00   -     vluxseg2ei16.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vluxseg2ei16.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vluxseg2ei16.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vluxseg2ei16.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vluxseg2ei16.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     140.00  -     vluxseg2ei16.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     16.00   -     vluxseg2ei32.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     20.00   -     vluxseg2ei32.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     20.00   -     vluxseg2ei32.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     20.00   -     vluxseg2ei32.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     20.00   -     vluxseg2ei32.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vluxseg2ei32.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     76.00   -     vluxseg2ei32.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     16.00   -     vluxseg2ei64.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     16.00   -     vluxseg2ei64.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     16.00   -     vluxseg2ei64.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     16.00   -     vluxseg2ei64.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     20.00   -     vluxseg2ei64.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vluxseg2ei64.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     44.00   -     vluxseg2ei64.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     18.00   -     vluxseg3ei8.v  v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     24.00   -     vluxseg3ei8.v  v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     36.00   -     vluxseg3ei8.v  v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     60.00   -     vluxseg3ei8.v  v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     60.00   -     vluxseg3ei8.v  v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     396.00  -     vluxseg3ei8.v  v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     396.00  -     vluxseg3ei8.v  v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     18.00   -     vluxseg3ei16.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     24.00   -     vluxseg3ei16.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     36.00   -     vluxseg3ei16.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     36.00   -     vluxseg3ei16.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     36.00   -     vluxseg3ei16.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     204.00  -     vluxseg3ei16.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     204.00  -     vluxseg3ei16.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     18.00   -     vluxseg3ei32.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     24.00   -     vluxseg3ei32.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     24.00   -     vluxseg3ei32.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     24.00   -     vluxseg3ei32.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     24.00   -     vluxseg3ei32.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     108.00  -     vluxseg3ei32.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     108.00  -     vluxseg3ei32.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     18.00   -     vluxseg3ei64.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     18.00   -     vluxseg3ei64.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     18.00   -     vluxseg3ei64.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     18.00   -     vluxseg3ei64.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     24.00   -     vluxseg3ei64.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     60.00   -     vluxseg3ei64.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     60.00   -     vluxseg3ei64.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     20.00   -     vluxseg4ei8.v  v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vluxseg4ei8.v  v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     44.00   -     vluxseg4ei8.v  v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     76.00   -     vluxseg4ei8.v  v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     76.00   -     vluxseg4ei8.v  v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     524.00  -     vluxseg4ei8.v  v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     524.00  -     vluxseg4ei8.v  v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     20.00   -     vluxseg4ei16.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vluxseg4ei16.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     44.00   -     vluxseg4ei16.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     44.00   -     vluxseg4ei16.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     44.00   -     vluxseg4ei16.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     268.00  -     vluxseg4ei16.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     268.00  -     vluxseg4ei16.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     20.00   -     vluxseg4ei32.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vluxseg4ei32.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vluxseg4ei32.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vluxseg4ei32.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vluxseg4ei32.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     140.00  -     vluxseg4ei32.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     140.00  -     vluxseg4ei32.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     20.00   -     vluxseg4ei64.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     20.00   -     vluxseg4ei64.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     20.00   -     vluxseg4ei64.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     20.00   -     vluxseg4ei64.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vluxseg4ei64.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     76.00   -     vluxseg4ei64.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     76.00   -     vluxseg4ei64.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     22.00   -     vluxseg5ei8.v  v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     32.00   -     vluxseg5ei8.v  v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     52.00   -     vluxseg5ei8.v  v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     92.00   -     vluxseg5ei8.v  v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     652.00  -     vluxseg5ei8.v  v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     652.00  -     vluxseg5ei8.v  v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     22.00   -     vluxseg5ei16.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     32.00   -     vluxseg5ei16.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     52.00   -     vluxseg5ei16.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     52.00   -     vluxseg5ei16.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     332.00  -     vluxseg5ei16.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     332.00  -     vluxseg5ei16.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     22.00   -     vluxseg5ei32.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     32.00   -     vluxseg5ei32.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     32.00   -     vluxseg5ei32.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     32.00   -     vluxseg5ei32.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     172.00  -     vluxseg5ei32.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     172.00  -     vluxseg5ei32.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     22.00   -     vluxseg5ei64.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     22.00   -     vluxseg5ei64.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     22.00   -     vluxseg5ei64.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     22.00   -     vluxseg5ei64.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     92.00   -     vluxseg5ei64.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     92.00   -     vluxseg5ei64.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     24.00   -     vluxseg6ei8.v  v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     36.00   -     vluxseg6ei8.v  v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     60.00   -     vluxseg6ei8.v  v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     108.00  -     vluxseg6ei8.v  v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     780.00  -     vluxseg6ei8.v  v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     780.00  -     vluxseg6ei8.v  v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     24.00   -     vluxseg6ei16.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     36.00   -     vluxseg6ei16.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     60.00   -     vluxseg6ei16.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     60.00   -     vluxseg6ei16.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     396.00  -     vluxseg6ei16.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     396.00  -     vluxseg6ei16.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     24.00   -     vluxseg6ei32.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     36.00   -     vluxseg6ei32.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     36.00   -     vluxseg6ei32.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     36.00   -     vluxseg6ei32.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     204.00  -     vluxseg6ei32.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     204.00  -     vluxseg6ei32.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     24.00   -     vluxseg6ei64.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     24.00   -     vluxseg6ei64.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     24.00   -     vluxseg6ei64.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     24.00   -     vluxseg6ei64.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     108.00  -     vluxseg6ei64.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     108.00  -     vluxseg6ei64.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     26.00   -     vluxseg7ei8.v  v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     40.00   -     vluxseg7ei8.v  v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     68.00   -     vluxseg7ei8.v  v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     124.00  -     vluxseg7ei8.v  v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     908.00  -     vluxseg7ei8.v  v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     908.00  -     vluxseg7ei8.v  v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     26.00   -     vluxseg7ei16.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     40.00   -     vluxseg7ei16.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     68.00   -     vluxseg7ei16.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     68.00   -     vluxseg7ei16.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     460.00  -     vluxseg7ei16.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     460.00  -     vluxseg7ei16.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     26.00   -     vluxseg7ei32.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     40.00   -     vluxseg7ei32.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     40.00   -     vluxseg7ei32.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     40.00   -     vluxseg7ei32.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     236.00  -     vluxseg7ei32.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     236.00  -     vluxseg7ei32.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     26.00   -     vluxseg7ei64.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     26.00   -     vluxseg7ei64.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     26.00   -     vluxseg7ei64.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     26.00   -     vluxseg7ei64.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     124.00  -     vluxseg7ei64.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     124.00  -     vluxseg7ei64.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vluxseg8ei8.v  v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     44.00   -     vluxseg8ei8.v  v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     76.00   -     vluxseg8ei8.v  v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     140.00  -     vluxseg8ei8.v  v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1036.00  -    vluxseg8ei8.v  v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1036.00  -    vluxseg8ei8.v  v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vluxseg8ei16.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     44.00   -     vluxseg8ei16.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     76.00   -     vluxseg8ei16.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     76.00   -     vluxseg8ei16.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     524.00  -     vluxseg8ei16.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     524.00  -     vluxseg8ei16.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vluxseg8ei32.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     44.00   -     vluxseg8ei32.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     44.00   -     vluxseg8ei32.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     44.00   -     vluxseg8ei32.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     268.00  -     vluxseg8ei32.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     268.00  -     vluxseg8ei32.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vluxseg8ei64.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vluxseg8ei64.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vluxseg8ei64.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vluxseg8ei64.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     140.00  -     vluxseg8ei64.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     140.00  -     vluxseg8ei64.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     16.00   -     vloxseg2ei8.v  v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     20.00   -     vloxseg2ei8.v  v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vloxseg2ei8.v  v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     44.00   -     vloxseg2ei8.v  v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     44.00   -     vloxseg2ei8.v  v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     44.00   -     vloxseg2ei8.v  v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     268.00  -     vloxseg2ei8.v  v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     16.00   -     vloxseg2ei16.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     20.00   -     vloxseg2ei16.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vloxseg2ei16.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vloxseg2ei16.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vloxseg2ei16.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vloxseg2ei16.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     140.00  -     vloxseg2ei16.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     16.00   -     vloxseg2ei32.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     20.00   -     vloxseg2ei32.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     20.00   -     vloxseg2ei32.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     20.00   -     vloxseg2ei32.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     20.00   -     vloxseg2ei32.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vloxseg2ei32.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     76.00   -     vloxseg2ei32.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     16.00   -     vloxseg2ei64.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     16.00   -     vloxseg2ei64.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     16.00   -     vloxseg2ei64.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     16.00   -     vloxseg2ei64.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     20.00   -     vloxseg2ei64.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vloxseg2ei64.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     44.00   -     vloxseg2ei64.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     18.00   -     vloxseg3ei8.v  v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     24.00   -     vloxseg3ei8.v  v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     36.00   -     vloxseg3ei8.v  v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     60.00   -     vloxseg3ei8.v  v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     60.00   -     vloxseg3ei8.v  v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     396.00  -     vloxseg3ei8.v  v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     396.00  -     vloxseg3ei8.v  v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     18.00   -     vloxseg3ei16.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     24.00   -     vloxseg3ei16.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     36.00   -     vloxseg3ei16.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     36.00   -     vloxseg3ei16.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     36.00   -     vloxseg3ei16.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     204.00  -     vloxseg3ei16.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     204.00  -     vloxseg3ei16.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     18.00   -     vloxseg3ei32.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     24.00   -     vloxseg3ei32.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     24.00   -     vloxseg3ei32.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     24.00   -     vloxseg3ei32.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     24.00   -     vloxseg3ei32.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     108.00  -     vloxseg3ei32.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     108.00  -     vloxseg3ei32.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     18.00   -     vloxseg3ei64.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     18.00   -     vloxseg3ei64.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     18.00   -     vloxseg3ei64.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     18.00   -     vloxseg3ei64.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     24.00   -     vloxseg3ei64.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     60.00   -     vloxseg3ei64.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     60.00   -     vloxseg3ei64.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     20.00   -     vloxseg4ei8.v  v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vloxseg4ei8.v  v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     44.00   -     vloxseg4ei8.v  v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     76.00   -     vloxseg4ei8.v  v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     76.00   -     vloxseg4ei8.v  v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     524.00  -     vloxseg4ei8.v  v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     524.00  -     vloxseg4ei8.v  v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     20.00   -     vloxseg4ei16.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vloxseg4ei16.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     44.00   -     vloxseg4ei16.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     44.00   -     vloxseg4ei16.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     44.00   -     vloxseg4ei16.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     268.00  -     vloxseg4ei16.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     268.00  -     vloxseg4ei16.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     20.00   -     vloxseg4ei32.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vloxseg4ei32.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vloxseg4ei32.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vloxseg4ei32.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vloxseg4ei32.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     140.00  -     vloxseg4ei32.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     140.00  -     vloxseg4ei32.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     20.00   -     vloxseg4ei64.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     20.00   -     vloxseg4ei64.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     20.00   -     vloxseg4ei64.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     20.00   -     vloxseg4ei64.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vloxseg4ei64.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     76.00   -     vloxseg4ei64.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     76.00   -     vloxseg4ei64.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     22.00   -     vloxseg5ei8.v  v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     32.00   -     vloxseg5ei8.v  v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     52.00   -     vloxseg5ei8.v  v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     92.00   -     vloxseg5ei8.v  v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     652.00  -     vloxseg5ei8.v  v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     652.00  -     vloxseg5ei8.v  v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     22.00   -     vloxseg5ei16.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     32.00   -     vloxseg5ei16.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     52.00   -     vloxseg5ei16.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     52.00   -     vloxseg5ei16.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     332.00  -     vloxseg5ei16.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     332.00  -     vloxseg5ei16.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     22.00   -     vloxseg5ei32.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     32.00   -     vloxseg5ei32.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     32.00   -     vloxseg5ei32.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     32.00   -     vloxseg5ei32.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     172.00  -     vloxseg5ei32.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     172.00  -     vloxseg5ei32.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     22.00   -     vloxseg5ei64.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     22.00   -     vloxseg5ei64.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     22.00   -     vloxseg5ei64.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     22.00   -     vloxseg5ei64.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     92.00   -     vloxseg5ei64.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     92.00   -     vloxseg5ei64.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     24.00   -     vloxseg6ei8.v  v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     36.00   -     vloxseg6ei8.v  v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     60.00   -     vloxseg6ei8.v  v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     108.00  -     vloxseg6ei8.v  v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     780.00  -     vloxseg6ei8.v  v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     780.00  -     vloxseg6ei8.v  v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     24.00   -     vloxseg6ei16.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     36.00   -     vloxseg6ei16.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     60.00   -     vloxseg6ei16.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     60.00   -     vloxseg6ei16.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     396.00  -     vloxseg6ei16.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     396.00  -     vloxseg6ei16.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     24.00   -     vloxseg6ei32.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     36.00   -     vloxseg6ei32.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     36.00   -     vloxseg6ei32.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     36.00   -     vloxseg6ei32.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     204.00  -     vloxseg6ei32.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     204.00  -     vloxseg6ei32.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     24.00   -     vloxseg6ei64.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     24.00   -     vloxseg6ei64.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     24.00   -     vloxseg6ei64.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     24.00   -     vloxseg6ei64.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     108.00  -     vloxseg6ei64.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     108.00  -     vloxseg6ei64.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     26.00   -     vloxseg7ei8.v  v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     40.00   -     vloxseg7ei8.v  v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     68.00   -     vloxseg7ei8.v  v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     124.00  -     vloxseg7ei8.v  v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     908.00  -     vloxseg7ei8.v  v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     908.00  -     vloxseg7ei8.v  v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     26.00   -     vloxseg7ei16.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     40.00   -     vloxseg7ei16.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     68.00   -     vloxseg7ei16.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     68.00   -     vloxseg7ei16.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     460.00  -     vloxseg7ei16.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     460.00  -     vloxseg7ei16.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     26.00   -     vloxseg7ei32.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     40.00   -     vloxseg7ei32.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     40.00   -     vloxseg7ei32.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     40.00   -     vloxseg7ei32.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     236.00  -     vloxseg7ei32.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     236.00  -     vloxseg7ei32.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     26.00   -     vloxseg7ei64.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     26.00   -     vloxseg7ei64.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     26.00   -     vloxseg7ei64.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     26.00   -     vloxseg7ei64.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     124.00  -     vloxseg7ei64.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     124.00  -     vloxseg7ei64.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vloxseg8ei8.v  v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     44.00   -     vloxseg8ei8.v  v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     76.00   -     vloxseg8ei8.v  v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     140.00  -     vloxseg8ei8.v  v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1036.00  -    vloxseg8ei8.v  v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1036.00  -    vloxseg8ei8.v  v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vloxseg8ei16.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     44.00   -     vloxseg8ei16.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     76.00   -     vloxseg8ei16.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     76.00   -     vloxseg8ei16.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     524.00  -     vloxseg8ei16.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     524.00  -     vloxseg8ei16.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vloxseg8ei32.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     44.00   -     vloxseg8ei32.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     44.00   -     vloxseg8ei32.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     44.00   -     vloxseg8ei32.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     268.00  -     vloxseg8ei32.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     268.00  -     vloxseg8ei32.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vloxseg8ei64.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vloxseg8ei64.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vloxseg8ei64.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vloxseg8ei64.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     140.00  -     vloxseg8ei64.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     140.00  -     vloxseg8ei64.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     16.00  vsuxseg2ei8.v  v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     20.00  vsuxseg2ei8.v  v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     28.00  vsuxseg2ei8.v  v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     44.00  vsuxseg2ei8.v  v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     76.00  vsuxseg2ei8.v  v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     140.00 vsuxseg2ei8.v  v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     268.00 vsuxseg2ei8.v  v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     14.00  vsuxseg2ei16.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     16.00  vsuxseg2ei16.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     20.00  vsuxseg2ei16.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     28.00  vsuxseg2ei16.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     44.00  vsuxseg2ei16.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     76.00  vsuxseg2ei16.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     140.00 vsuxseg2ei16.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     12.00  vsuxseg2ei32.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     14.00  vsuxseg2ei32.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     16.00  vsuxseg2ei32.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     20.00  vsuxseg2ei32.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     28.00  vsuxseg2ei32.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     44.00  vsuxseg2ei32.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     76.00  vsuxseg2ei32.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     12.00  vsuxseg2ei64.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     12.00  vsuxseg2ei64.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     14.00  vsuxseg2ei64.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     16.00  vsuxseg2ei64.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     20.00  vsuxseg2ei64.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     28.00  vsuxseg2ei64.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     44.00  vsuxseg2ei64.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     18.00  vsuxseg3ei8.v  v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     24.00  vsuxseg3ei8.v  v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     36.00  vsuxseg3ei8.v  v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     60.00  vsuxseg3ei8.v  v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     108.00 vsuxseg3ei8.v  v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     396.00 vsuxseg3ei8.v  v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     396.00 vsuxseg3ei8.v  v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     15.00  vsuxseg3ei16.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     18.00  vsuxseg3ei16.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     24.00  vsuxseg3ei16.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     36.00  vsuxseg3ei16.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     60.00  vsuxseg3ei16.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     204.00 vsuxseg3ei16.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     204.00 vsuxseg3ei16.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     12.00  vsuxseg3ei32.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     15.00  vsuxseg3ei32.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     18.00  vsuxseg3ei32.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     24.00  vsuxseg3ei32.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     36.00  vsuxseg3ei32.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     108.00 vsuxseg3ei32.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     108.00 vsuxseg3ei32.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     12.00  vsuxseg3ei64.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     12.00  vsuxseg3ei64.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     15.00  vsuxseg3ei64.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     18.00  vsuxseg3ei64.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     24.00  vsuxseg3ei64.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     60.00  vsuxseg3ei64.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     60.00  vsuxseg3ei64.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     20.00  vsuxseg4ei8.v  v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     28.00  vsuxseg4ei8.v  v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     44.00  vsuxseg4ei8.v  v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     76.00  vsuxseg4ei8.v  v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     140.00 vsuxseg4ei8.v  v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     524.00 vsuxseg4ei8.v  v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     524.00 vsuxseg4ei8.v  v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     16.00  vsuxseg4ei16.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     20.00  vsuxseg4ei16.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     28.00  vsuxseg4ei16.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     44.00  vsuxseg4ei16.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     76.00  vsuxseg4ei16.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     268.00 vsuxseg4ei16.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     268.00 vsuxseg4ei16.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     12.00  vsuxseg4ei32.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     16.00  vsuxseg4ei32.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     20.00  vsuxseg4ei32.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     28.00  vsuxseg4ei32.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     44.00  vsuxseg4ei32.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     140.00 vsuxseg4ei32.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     140.00 vsuxseg4ei32.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     12.00  vsuxseg4ei64.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     12.00  vsuxseg4ei64.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     16.00  vsuxseg4ei64.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     20.00  vsuxseg4ei64.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     28.00  vsuxseg4ei64.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     76.00  vsuxseg4ei64.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     76.00  vsuxseg4ei64.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     22.00  vsuxseg5ei8.v  v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     32.00  vsuxseg5ei8.v  v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     52.00  vsuxseg5ei8.v  v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     92.00  vsuxseg5ei8.v  v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     652.00 vsuxseg5ei8.v  v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     652.00 vsuxseg5ei8.v  v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     17.00  vsuxseg5ei16.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     22.00  vsuxseg5ei16.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     32.00  vsuxseg5ei16.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     52.00  vsuxseg5ei16.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     332.00 vsuxseg5ei16.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     332.00 vsuxseg5ei16.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     12.00  vsuxseg5ei32.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     17.00  vsuxseg5ei32.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     22.00  vsuxseg5ei32.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     32.00  vsuxseg5ei32.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     172.00 vsuxseg5ei32.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     172.00 vsuxseg5ei32.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     12.00  vsuxseg5ei64.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     12.00  vsuxseg5ei64.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     17.00  vsuxseg5ei64.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     22.00  vsuxseg5ei64.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     92.00  vsuxseg5ei64.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     92.00  vsuxseg5ei64.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     24.00  vsuxseg6ei8.v  v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     36.00  vsuxseg6ei8.v  v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     60.00  vsuxseg6ei8.v  v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     108.00 vsuxseg6ei8.v  v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     780.00 vsuxseg6ei8.v  v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     780.00 vsuxseg6ei8.v  v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     18.00  vsuxseg6ei16.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     24.00  vsuxseg6ei16.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     36.00  vsuxseg6ei16.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     60.00  vsuxseg6ei16.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     396.00 vsuxseg6ei16.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     396.00 vsuxseg6ei16.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     12.00  vsuxseg6ei32.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     18.00  vsuxseg6ei32.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     24.00  vsuxseg6ei32.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     36.00  vsuxseg6ei32.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     204.00 vsuxseg6ei32.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     204.00 vsuxseg6ei32.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     12.00  vsuxseg6ei64.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     12.00  vsuxseg6ei64.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     18.00  vsuxseg6ei64.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     24.00  vsuxseg6ei64.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     108.00 vsuxseg6ei64.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     108.00 vsuxseg6ei64.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     26.00  vsuxseg7ei8.v  v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     40.00  vsuxseg7ei8.v  v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     68.00  vsuxseg7ei8.v  v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     124.00 vsuxseg7ei8.v  v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     908.00 vsuxseg7ei8.v  v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     908.00 vsuxseg7ei8.v  v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     19.00  vsuxseg7ei16.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     26.00  vsuxseg7ei16.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     40.00  vsuxseg7ei16.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     68.00  vsuxseg7ei16.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     460.00 vsuxseg7ei16.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     460.00 vsuxseg7ei16.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     12.00  vsuxseg7ei32.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     19.00  vsuxseg7ei32.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     26.00  vsuxseg7ei32.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     40.00  vsuxseg7ei32.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     236.00 vsuxseg7ei32.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     236.00 vsuxseg7ei32.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     12.00  vsuxseg7ei64.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     12.00  vsuxseg7ei64.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     19.00  vsuxseg7ei64.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     26.00  vsuxseg7ei64.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     124.00 vsuxseg7ei64.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     124.00 vsuxseg7ei64.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     28.00  vsuxseg8ei8.v  v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     44.00  vsuxseg8ei8.v  v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     76.00  vsuxseg8ei8.v  v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     140.00 vsuxseg8ei8.v  v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     1036.00 vsuxseg8ei8.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     1036.00 vsuxseg8ei8.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     20.00  vsuxseg8ei16.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     28.00  vsuxseg8ei16.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     44.00  vsuxseg8ei16.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     76.00  vsuxseg8ei16.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     524.00 vsuxseg8ei16.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     524.00 vsuxseg8ei16.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     12.00  vsuxseg8ei32.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     20.00  vsuxseg8ei32.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     28.00  vsuxseg8ei32.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     44.00  vsuxseg8ei32.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     268.00 vsuxseg8ei32.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     268.00 vsuxseg8ei32.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     12.00  vsuxseg8ei64.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     12.00  vsuxseg8ei64.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     20.00  vsuxseg8ei64.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     28.00  vsuxseg8ei64.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     140.00 vsuxseg8ei64.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     140.00 vsuxseg8ei64.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     16.00  vsoxseg2ei8.v  v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     20.00  vsoxseg2ei8.v  v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     28.00  vsoxseg2ei8.v  v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     44.00  vsoxseg2ei8.v  v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     76.00  vsoxseg2ei8.v  v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     140.00 vsoxseg2ei8.v  v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     268.00 vsoxseg2ei8.v  v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     14.00  vsoxseg2ei16.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     16.00  vsoxseg2ei16.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     20.00  vsoxseg2ei16.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     28.00  vsoxseg2ei16.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     44.00  vsoxseg2ei16.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     76.00  vsoxseg2ei16.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     140.00 vsoxseg2ei16.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     12.00  vsoxseg2ei32.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     14.00  vsoxseg2ei32.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     16.00  vsoxseg2ei32.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     20.00  vsoxseg2ei32.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     28.00  vsoxseg2ei32.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     44.00  vsoxseg2ei32.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     76.00  vsoxseg2ei32.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     12.00  vsoxseg2ei64.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     12.00  vsoxseg2ei64.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     14.00  vsoxseg2ei64.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     16.00  vsoxseg2ei64.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     20.00  vsoxseg2ei64.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     28.00  vsoxseg2ei64.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     44.00  vsoxseg2ei64.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     18.00  vsoxseg3ei8.v  v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     24.00  vsoxseg3ei8.v  v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     36.00  vsoxseg3ei8.v  v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     60.00  vsoxseg3ei8.v  v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     108.00 vsoxseg3ei8.v  v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     396.00 vsoxseg3ei8.v  v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     396.00 vsoxseg3ei8.v  v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     15.00  vsoxseg3ei16.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     18.00  vsoxseg3ei16.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     24.00  vsoxseg3ei16.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     36.00  vsoxseg3ei16.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     60.00  vsoxseg3ei16.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     204.00 vsoxseg3ei16.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     204.00 vsoxseg3ei16.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     12.00  vsoxseg3ei32.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     15.00  vsoxseg3ei32.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     18.00  vsoxseg3ei32.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     24.00  vsoxseg3ei32.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     36.00  vsoxseg3ei32.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     108.00 vsoxseg3ei32.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     108.00 vsoxseg3ei32.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     12.00  vsoxseg3ei64.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     12.00  vsoxseg3ei64.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     15.00  vsoxseg3ei64.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     18.00  vsoxseg3ei64.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     24.00  vsoxseg3ei64.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     60.00  vsoxseg3ei64.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     60.00  vsoxseg3ei64.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     20.00  vsoxseg4ei8.v  v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     28.00  vsoxseg4ei8.v  v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     44.00  vsoxseg4ei8.v  v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     76.00  vsoxseg4ei8.v  v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     140.00 vsoxseg4ei8.v  v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     524.00 vsoxseg4ei8.v  v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     524.00 vsoxseg4ei8.v  v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     16.00  vsoxseg4ei16.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     20.00  vsoxseg4ei16.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     28.00  vsoxseg4ei16.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     44.00  vsoxseg4ei16.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     76.00  vsoxseg4ei16.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     268.00 vsoxseg4ei16.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     268.00 vsoxseg4ei16.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     12.00  vsoxseg4ei32.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     16.00  vsoxseg4ei32.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     20.00  vsoxseg4ei32.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     28.00  vsoxseg4ei32.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     44.00  vsoxseg4ei32.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     140.00 vsoxseg4ei32.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     140.00 vsoxseg4ei32.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     12.00  vsoxseg4ei64.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     12.00  vsoxseg4ei64.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     16.00  vsoxseg4ei64.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     20.00  vsoxseg4ei64.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     28.00  vsoxseg4ei64.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     76.00  vsoxseg4ei64.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     76.00  vsoxseg4ei64.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     22.00  vsoxseg5ei8.v  v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     32.00  vsoxseg5ei8.v  v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     52.00  vsoxseg5ei8.v  v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     92.00  vsoxseg5ei8.v  v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     652.00 vsoxseg5ei8.v  v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     652.00 vsoxseg5ei8.v  v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     17.00  vsoxseg5ei16.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     22.00  vsoxseg5ei16.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     32.00  vsoxseg5ei16.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     52.00  vsoxseg5ei16.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     332.00 vsoxseg5ei16.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     332.00 vsoxseg5ei16.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     12.00  vsoxseg5ei32.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     17.00  vsoxseg5ei32.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     22.00  vsoxseg5ei32.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     32.00  vsoxseg5ei32.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     172.00 vsoxseg5ei32.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     172.00 vsoxseg5ei32.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     12.00  vsoxseg5ei64.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     12.00  vsoxseg5ei64.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     17.00  vsoxseg5ei64.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     22.00  vsoxseg5ei64.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     92.00  vsoxseg5ei64.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     92.00  vsoxseg5ei64.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     24.00  vsoxseg6ei8.v  v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     36.00  vsoxseg6ei8.v  v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     60.00  vsoxseg6ei8.v  v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     108.00 vsoxseg6ei8.v  v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     780.00 vsoxseg6ei8.v  v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     780.00 vsoxseg6ei8.v  v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     18.00  vsoxseg6ei16.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     24.00  vsoxseg6ei16.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     36.00  vsoxseg6ei16.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     60.00  vsoxseg6ei16.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     396.00 vsoxseg6ei16.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     396.00 vsoxseg6ei16.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     12.00  vsoxseg6ei32.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     18.00  vsoxseg6ei32.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     24.00  vsoxseg6ei32.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     36.00  vsoxseg6ei32.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     204.00 vsoxseg6ei32.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     204.00 vsoxseg6ei32.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     12.00  vsoxseg6ei64.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     12.00  vsoxseg6ei64.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     18.00  vsoxseg6ei64.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     24.00  vsoxseg6ei64.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     108.00 vsoxseg6ei64.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     108.00 vsoxseg6ei64.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     26.00  vsoxseg7ei8.v  v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     40.00  vsoxseg7ei8.v  v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     68.00  vsoxseg7ei8.v  v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     124.00 vsoxseg7ei8.v  v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     908.00 vsoxseg7ei8.v  v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     908.00 vsoxseg7ei8.v  v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     19.00  vsoxseg7ei16.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     26.00  vsoxseg7ei16.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     40.00  vsoxseg7ei16.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     68.00  vsoxseg7ei16.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     460.00 vsoxseg7ei16.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     460.00 vsoxseg7ei16.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     12.00  vsoxseg7ei32.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     19.00  vsoxseg7ei32.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     26.00  vsoxseg7ei32.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     40.00  vsoxseg7ei32.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     236.00 vsoxseg7ei32.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     236.00 vsoxseg7ei32.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     12.00  vsoxseg7ei64.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     12.00  vsoxseg7ei64.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     19.00  vsoxseg7ei64.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     26.00  vsoxseg7ei64.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     124.00 vsoxseg7ei64.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     124.00 vsoxseg7ei64.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     28.00  vsoxseg8ei8.v  v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     44.00  vsoxseg8ei8.v  v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     76.00  vsoxseg8ei8.v  v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     140.00 vsoxseg8ei8.v  v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     1036.00 vsoxseg8ei8.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     1036.00 vsoxseg8ei8.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     20.00  vsoxseg8ei16.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     28.00  vsoxseg8ei16.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     44.00  vsoxseg8ei16.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     76.00  vsoxseg8ei16.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     524.00 vsoxseg8ei16.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     524.00 vsoxseg8ei16.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     12.00  vsoxseg8ei32.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     20.00  vsoxseg8ei32.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     28.00  vsoxseg8ei32.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     44.00  vsoxseg8ei32.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     268.00 vsoxseg8ei32.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     268.00 vsoxseg8ei32.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     12.00  vsoxseg8ei64.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     12.00  vsoxseg8ei64.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     20.00  vsoxseg8ei64.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     28.00  vsoxseg8ei64.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     140.00 vsoxseg8ei64.v v8, (a0), v16
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     140.00 vsoxseg8ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     16.00   -     vlseg2e8.v	v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     20.00   -     vlseg2e8.v	v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vlseg2e8.v	v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     44.00   -     vlseg2e8.v	v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     76.00   -     vlseg2e8.v	v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     140.00  -     vlseg2e8.v	v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     16.00   -     vlseg2e16.v	v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     20.00   -     vlseg2e16.v	v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vlseg2e16.v	v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     44.00   -     vlseg2e16.v	v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     76.00   -     vlseg2e16.v	v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     16.00   -     vlseg2e32.v	v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     20.00   -     vlseg2e32.v	v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vlseg2e32.v	v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     44.00   -     vlseg2e32.v	v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     16.00   -     vlseg2e64.v	v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     20.00   -     vlseg2e64.v	v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vlseg2e64.v	v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     18.00   -     vlseg3e8.v	v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     24.00   -     vlseg3e8.v	v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     36.00   -     vlseg3e8.v	v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     60.00   -     vlseg3e8.v	v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     108.00  -     vlseg3e8.v	v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     18.00   -     vlseg3e16.v	v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     24.00   -     vlseg3e16.v	v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     36.00   -     vlseg3e16.v	v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     60.00   -     vlseg3e16.v	v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     18.00   -     vlseg3e32.v	v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     24.00   -     vlseg3e32.v	v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     36.00   -     vlseg3e32.v	v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     18.00   -     vlseg3e64.v	v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     24.00   -     vlseg3e64.v	v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     20.00   -     vlseg4e8.v	v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vlseg4e8.v	v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     44.00   -     vlseg4e8.v	v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     76.00   -     vlseg4e8.v	v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     140.00  -     vlseg4e8.v	v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     20.00   -     vlseg4e16.v	v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vlseg4e16.v	v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     44.00   -     vlseg4e16.v	v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     76.00   -     vlseg4e16.v	v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     20.00   -     vlseg4e32.v	v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vlseg4e32.v	v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     44.00   -     vlseg4e32.v	v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     20.00   -     vlseg4e64.v	v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vlseg4e64.v	v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     22.00   -     vlseg5e8.v	v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     32.00   -     vlseg5e8.v	v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     52.00   -     vlseg5e8.v	v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     92.00   -     vlseg5e8.v	v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     22.00   -     vlseg5e16.v	v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     32.00   -     vlseg5e16.v	v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     52.00   -     vlseg5e16.v	v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     22.00   -     vlseg5e32.v	v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     32.00   -     vlseg5e32.v	v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     22.00   -     vlseg5e64.v	v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     24.00   -     vlseg6e8.v	v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     36.00   -     vlseg6e8.v	v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     60.00   -     vlseg6e8.v	v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     108.00  -     vlseg6e8.v	v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     24.00   -     vlseg6e16.v	v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     36.00   -     vlseg6e16.v	v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     60.00   -     vlseg6e16.v	v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     24.00   -     vlseg6e32.v	v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     36.00   -     vlseg6e32.v	v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     24.00   -     vlseg6e64.v	v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     26.00   -     vlseg7e8.v	v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     40.00   -     vlseg7e8.v	v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     68.00   -     vlseg7e8.v	v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     124.00  -     vlseg7e8.v	v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     26.00   -     vlseg7e16.v	v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     40.00   -     vlseg7e16.v	v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     68.00   -     vlseg7e16.v	v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     26.00   -     vlseg7e32.v	v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     40.00   -     vlseg7e32.v	v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     26.00   -     vlseg7e64.v	v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vlseg8e8.v	v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     44.00   -     vlseg8e8.v	v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     76.00   -     vlseg8e8.v	v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     140.00  -     vlseg8e8.v	v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vlseg8e16.v	v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     44.00   -     vlseg8e16.v	v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     76.00   -     vlseg8e16.v	v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vlseg8e32.v	v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     44.00   -     vlseg8e32.v	v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vlseg8e64.v	v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     16.00  vsseg2e8.v	v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     20.00  vsseg2e8.v	v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     28.00  vsseg2e8.v	v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     44.00  vsseg2e8.v	v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     76.00  vsseg2e8.v	v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     140.00 vsseg2e8.v	v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     16.00  vsseg2e16.v	v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     20.00  vsseg2e16.v	v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     28.00  vsseg2e16.v	v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     44.00  vsseg2e16.v	v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     76.00  vsseg2e16.v	v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     16.00  vsseg2e32.v	v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     20.00  vsseg2e32.v	v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     28.00  vsseg2e32.v	v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     44.00  vsseg2e32.v	v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     16.00  vsseg2e64.v	v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     20.00  vsseg2e64.v	v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     28.00  vsseg2e64.v	v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     18.00  vsseg3e8.v	v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     24.00  vsseg3e8.v	v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     36.00  vsseg3e8.v	v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     60.00  vsseg3e8.v	v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     108.00 vsseg3e8.v	v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     18.00  vsseg3e16.v	v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     24.00  vsseg3e16.v	v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     36.00  vsseg3e16.v	v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     60.00  vsseg3e16.v	v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     18.00  vsseg3e32.v	v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     24.00  vsseg3e32.v	v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     36.00  vsseg3e32.v	v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     18.00  vsseg3e64.v	v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     24.00  vsseg3e64.v	v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     20.00  vsseg4e8.v	v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     28.00  vsseg4e8.v	v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     44.00  vsseg4e8.v	v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     76.00  vsseg4e8.v	v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     140.00 vsseg4e8.v	v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     20.00  vsseg4e16.v	v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     28.00  vsseg4e16.v	v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     44.00  vsseg4e16.v	v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     76.00  vsseg4e16.v	v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     20.00  vsseg4e32.v	v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     28.00  vsseg4e32.v	v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     44.00  vsseg4e32.v	v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     20.00  vsseg4e64.v	v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     28.00  vsseg4e64.v	v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     22.00  vsseg5e8.v	v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     32.00  vsseg5e8.v	v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     52.00  vsseg5e8.v	v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     92.00  vsseg5e8.v	v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     22.00  vsseg5e16.v	v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     32.00  vsseg5e16.v	v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     52.00  vsseg5e16.v	v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     22.00  vsseg5e32.v	v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     32.00  vsseg5e32.v	v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     22.00  vsseg5e64.v	v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     24.00  vsseg6e8.v	v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     36.00  vsseg6e8.v	v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     60.00  vsseg6e8.v	v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     108.00 vsseg6e8.v	v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     24.00  vsseg6e16.v	v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     36.00  vsseg6e16.v	v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     60.00  vsseg6e16.v	v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     24.00  vsseg6e32.v	v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     36.00  vsseg6e32.v	v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     24.00  vsseg6e64.v	v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     26.00  vsseg7e8.v	v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     40.00  vsseg7e8.v	v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     68.00  vsseg7e8.v	v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     124.00 vsseg7e8.v	v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     26.00  vsseg7e16.v	v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     40.00  vsseg7e16.v	v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     68.00  vsseg7e16.v	v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     26.00  vsseg7e32.v	v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     40.00  vsseg7e32.v	v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     26.00  vsseg7e64.v	v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     28.00  vsseg8e8.v	v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     44.00  vsseg8e8.v	v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     76.00  vsseg8e8.v	v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     140.00 vsseg8e8.v	v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     28.00  vsseg8e16.v	v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     44.00  vsseg8e16.v	v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     76.00  vsseg8e16.v	v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     28.00  vsseg8e32.v	v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     44.00  vsseg8e32.v	v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     28.00  vsseg8e64.v	v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     16.00   -     vlsseg2e8.v	v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     20.00   -     vlsseg2e8.v	v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vlsseg2e8.v	v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     44.00   -     vlsseg2e8.v	v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     76.00   -     vlsseg2e8.v	v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     140.00  -     vlsseg2e8.v	v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     16.00   -     vlsseg2e16.v	v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     20.00   -     vlsseg2e16.v	v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vlsseg2e16.v	v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     44.00   -     vlsseg2e16.v	v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     76.00   -     vlsseg2e16.v	v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     16.00   -     vlsseg2e32.v	v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     20.00   -     vlsseg2e32.v	v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vlsseg2e32.v	v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     44.00   -     vlsseg2e32.v	v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     16.00   -     vlsseg2e64.v	v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     20.00   -     vlsseg2e64.v	v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vlsseg2e64.v	v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     18.00   -     vlsseg3e8.v	v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     24.00   -     vlsseg3e8.v	v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     36.00   -     vlsseg3e8.v	v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     60.00   -     vlsseg3e8.v	v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     108.00  -     vlsseg3e8.v	v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     18.00   -     vlsseg3e16.v	v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     24.00   -     vlsseg3e16.v	v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     36.00   -     vlsseg3e16.v	v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     60.00   -     vlsseg3e16.v	v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     18.00   -     vlsseg3e32.v	v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     24.00   -     vlsseg3e32.v	v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     36.00   -     vlsseg3e32.v	v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     18.00   -     vlsseg3e64.v	v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     24.00   -     vlsseg3e64.v	v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     20.00   -     vlsseg4e8.v	v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vlsseg4e8.v	v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     44.00   -     vlsseg4e8.v	v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     76.00   -     vlsseg4e8.v	v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     140.00  -     vlsseg4e8.v	v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     20.00   -     vlsseg4e16.v	v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vlsseg4e16.v	v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     44.00   -     vlsseg4e16.v	v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     76.00   -     vlsseg4e16.v	v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     20.00   -     vlsseg4e32.v	v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vlsseg4e32.v	v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     44.00   -     vlsseg4e32.v	v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     20.00   -     vlsseg4e64.v	v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vlsseg4e64.v	v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     22.00   -     vlsseg5e8.v	v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     32.00   -     vlsseg5e8.v	v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     52.00   -     vlsseg5e8.v	v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     92.00   -     vlsseg5e8.v	v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     22.00   -     vlsseg5e16.v	v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     32.00   -     vlsseg5e16.v	v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     52.00   -     vlsseg5e16.v	v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     22.00   -     vlsseg5e32.v	v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     32.00   -     vlsseg5e32.v	v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     22.00   -     vlsseg5e64.v	v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     24.00   -     vlsseg6e8.v	v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     36.00   -     vlsseg6e8.v	v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     60.00   -     vlsseg6e8.v	v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     108.00  -     vlsseg6e8.v	v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     24.00   -     vlsseg6e16.v	v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     36.00   -     vlsseg6e16.v	v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     60.00   -     vlsseg6e16.v	v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     24.00   -     vlsseg6e32.v	v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     36.00   -     vlsseg6e32.v	v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     24.00   -     vlsseg6e64.v	v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     26.00   -     vlsseg7e8.v	v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     40.00   -     vlsseg7e8.v	v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     68.00   -     vlsseg7e8.v	v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     124.00  -     vlsseg7e8.v	v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     26.00   -     vlsseg7e16.v	v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     40.00   -     vlsseg7e16.v	v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     68.00   -     vlsseg7e16.v	v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     26.00   -     vlsseg7e32.v	v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     40.00   -     vlsseg7e32.v	v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     26.00   -     vlsseg7e64.v	v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vlsseg8e8.v	v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     44.00   -     vlsseg8e8.v	v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     76.00   -     vlsseg8e8.v	v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     140.00  -     vlsseg8e8.v	v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vlsseg8e16.v	v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     44.00   -     vlsseg8e16.v	v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     76.00   -     vlsseg8e16.v	v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vlsseg8e32.v	v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     44.00   -     vlsseg8e32.v	v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vlsseg8e64.v	v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     16.00  vssseg2e8.v	v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     20.00  vssseg2e8.v	v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     28.00  vssseg2e8.v	v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     44.00  vssseg2e8.v	v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     76.00  vssseg2e8.v	v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     140.00 vssseg2e8.v	v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     16.00  vssseg2e16.v	v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     20.00  vssseg2e16.v	v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     28.00  vssseg2e16.v	v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     44.00  vssseg2e16.v	v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     76.00  vssseg2e16.v	v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     16.00  vssseg2e32.v	v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     20.00  vssseg2e32.v	v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     28.00  vssseg2e32.v	v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     44.00  vssseg2e32.v	v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     16.00  vssseg2e64.v	v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     20.00  vssseg2e64.v	v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     28.00  vssseg2e64.v	v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     18.00  vssseg3e8.v	v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     24.00  vssseg3e8.v	v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     36.00  vssseg3e8.v	v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     60.00  vssseg3e8.v	v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     108.00 vssseg3e8.v	v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     18.00  vssseg3e16.v	v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     24.00  vssseg3e16.v	v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     36.00  vssseg3e16.v	v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     60.00  vssseg3e16.v	v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     18.00  vssseg3e32.v	v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     24.00  vssseg3e32.v	v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     36.00  vssseg3e32.v	v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     18.00  vssseg3e64.v	v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     24.00  vssseg3e64.v	v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     20.00  vssseg4e8.v	v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     28.00  vssseg4e8.v	v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     44.00  vssseg4e8.v	v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     76.00  vssseg4e8.v	v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     140.00 vssseg4e8.v	v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     20.00  vssseg4e16.v	v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     28.00  vssseg4e16.v	v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     44.00  vssseg4e16.v	v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     76.00  vssseg4e16.v	v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     20.00  vssseg4e32.v	v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     28.00  vssseg4e32.v	v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     44.00  vssseg4e32.v	v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     20.00  vssseg4e64.v	v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     28.00  vssseg4e64.v	v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     22.00  vssseg5e8.v	v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     32.00  vssseg5e8.v	v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     52.00  vssseg5e8.v	v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     92.00  vssseg5e8.v	v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     22.00  vssseg5e16.v	v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     32.00  vssseg5e16.v	v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     52.00  vssseg5e16.v	v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     22.00  vssseg5e32.v	v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     32.00  vssseg5e32.v	v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     22.00  vssseg5e64.v	v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     24.00  vssseg6e8.v	v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     36.00  vssseg6e8.v	v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     60.00  vssseg6e8.v	v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     108.00 vssseg6e8.v	v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     24.00  vssseg6e16.v	v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     36.00  vssseg6e16.v	v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     60.00  vssseg6e16.v	v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     24.00  vssseg6e32.v	v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     36.00  vssseg6e32.v	v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     24.00  vssseg6e64.v	v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     26.00  vssseg7e8.v	v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     40.00  vssseg7e8.v	v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     68.00  vssseg7e8.v	v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     124.00 vssseg7e8.v	v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     26.00  vssseg7e16.v	v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     40.00  vssseg7e16.v	v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     68.00  vssseg7e16.v	v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     26.00  vssseg7e32.v	v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     40.00  vssseg7e32.v	v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     26.00  vssseg7e64.v	v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     28.00  vssseg8e8.v	v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     44.00  vssseg8e8.v	v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     76.00  vssseg8e8.v	v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     140.00 vssseg8e8.v	v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     28.00  vssseg8e16.v	v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     44.00  vssseg8e16.v	v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     76.00  vssseg8e16.v	v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     28.00  vssseg8e32.v	v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     44.00  vssseg8e32.v	v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     28.00  vssseg8e64.v	v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     16.00   -     vlseg2e8ff.v	v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     20.00   -     vlseg2e8ff.v	v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vlseg2e8ff.v	v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     44.00   -     vlseg2e8ff.v	v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     76.00   -     vlseg2e8ff.v	v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     140.00  -     vlseg2e8ff.v	v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     16.00   -     vlseg2e16ff.v	v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     20.00   -     vlseg2e16ff.v	v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vlseg2e16ff.v	v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     44.00   -     vlseg2e16ff.v	v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     76.00   -     vlseg2e16ff.v	v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     16.00   -     vlseg2e32ff.v	v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     20.00   -     vlseg2e32ff.v	v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vlseg2e32ff.v	v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     44.00   -     vlseg2e32ff.v	v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     16.00   -     vlseg2e64ff.v	v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     20.00   -     vlseg2e64ff.v	v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vlseg2e64ff.v	v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     18.00   -     vlseg3e8ff.v	v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     24.00   -     vlseg3e8ff.v	v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     36.00   -     vlseg3e8ff.v	v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     60.00   -     vlseg3e8ff.v	v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     108.00  -     vlseg3e8ff.v	v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     18.00   -     vlseg3e16ff.v	v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     24.00   -     vlseg3e16ff.v	v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     36.00   -     vlseg3e16ff.v	v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     60.00   -     vlseg3e16ff.v	v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     18.00   -     vlseg3e32ff.v	v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     24.00   -     vlseg3e32ff.v	v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     36.00   -     vlseg3e32ff.v	v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     18.00   -     vlseg3e64ff.v	v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     24.00   -     vlseg3e64ff.v	v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     20.00   -     vlseg4e8ff.v	v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vlseg4e8ff.v	v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     44.00   -     vlseg4e8ff.v	v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     76.00   -     vlseg4e8ff.v	v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     140.00  -     vlseg4e8ff.v	v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     20.00   -     vlseg4e16ff.v	v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vlseg4e16ff.v	v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     44.00   -     vlseg4e16ff.v	v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     76.00   -     vlseg4e16ff.v	v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     20.00   -     vlseg4e32ff.v	v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vlseg4e32ff.v	v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     44.00   -     vlseg4e32ff.v	v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     20.00   -     vlseg4e64ff.v	v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vlseg4e64ff.v	v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     22.00   -     vlseg5e8ff.v	v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     32.00   -     vlseg5e8ff.v	v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     52.00   -     vlseg5e8ff.v	v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     92.00   -     vlseg5e8ff.v	v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     22.00   -     vlseg5e16ff.v	v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     32.00   -     vlseg5e16ff.v	v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     52.00   -     vlseg5e16ff.v	v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     22.00   -     vlseg5e32ff.v	v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     32.00   -     vlseg5e32ff.v	v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     22.00   -     vlseg5e64ff.v	v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     24.00   -     vlseg6e8ff.v	v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     36.00   -     vlseg6e8ff.v	v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     60.00   -     vlseg6e8ff.v	v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     108.00  -     vlseg6e8ff.v	v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     24.00   -     vlseg6e16ff.v	v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     36.00   -     vlseg6e16ff.v	v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     60.00   -     vlseg6e16ff.v	v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     24.00   -     vlseg6e32ff.v	v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     36.00   -     vlseg6e32ff.v	v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     24.00   -     vlseg6e64ff.v	v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     26.00   -     vlseg7e8ff.v	v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     40.00   -     vlseg7e8ff.v	v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     68.00   -     vlseg7e8ff.v	v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     124.00  -     vlseg7e8ff.v	v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     26.00   -     vlseg7e16ff.v	v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     40.00   -     vlseg7e16ff.v	v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     68.00   -     vlseg7e16ff.v	v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     26.00   -     vlseg7e32ff.v	v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     40.00   -     vlseg7e32ff.v	v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     26.00   -     vlseg7e64ff.v	v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vlseg8e8ff.v	v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     44.00   -     vlseg8e8ff.v	v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     76.00   -     vlseg8e8ff.v	v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     140.00  -     vlseg8e8ff.v	v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vlseg8e16ff.v	v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     44.00   -     vlseg8e16ff.v	v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     76.00   -     vlseg8e16ff.v	v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vlseg8e32ff.v	v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     44.00   -     vlseg8e32ff.v	v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vlseg8e64ff.v	v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     16.00   -     vluxseg2ei8.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     20.00   -     vluxseg2ei8.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vluxseg2ei8.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     44.00   -     vluxseg2ei8.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     44.00   -     vluxseg2ei8.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     44.00   -     vluxseg2ei8.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     20.00   -     vluxseg2ei16.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vluxseg2ei16.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vluxseg2ei16.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vluxseg2ei16.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vluxseg2ei16.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     20.00   -     vluxseg2ei32.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     20.00   -     vluxseg2ei32.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     20.00   -     vluxseg2ei32.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vluxseg2ei32.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     16.00   -     vluxseg2ei64.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     20.00   -     vluxseg2ei64.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vluxseg2ei64.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     18.00   -     vluxseg3ei8.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     24.00   -     vluxseg3ei8.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     36.00   -     vluxseg3ei8.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     60.00   -     vluxseg3ei8.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     60.00   -     vluxseg3ei8.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     24.00   -     vluxseg3ei16.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     36.00   -     vluxseg3ei16.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     36.00   -     vluxseg3ei16.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     36.00   -     vluxseg3ei16.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     24.00   -     vluxseg3ei32.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     24.00   -     vluxseg3ei32.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     24.00   -     vluxseg3ei32.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     18.00   -     vluxseg3ei64.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     24.00   -     vluxseg3ei64.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     20.00   -     vluxseg4ei8.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vluxseg4ei8.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     44.00   -     vluxseg4ei8.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     76.00   -     vluxseg4ei8.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     76.00   -     vluxseg4ei8.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vluxseg4ei16.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     44.00   -     vluxseg4ei16.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     44.00   -     vluxseg4ei16.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     44.00   -     vluxseg4ei16.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vluxseg4ei32.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vluxseg4ei32.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vluxseg4ei32.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     20.00   -     vluxseg4ei64.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vluxseg4ei64.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     22.00   -     vluxseg5ei8.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     32.00   -     vluxseg5ei8.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     52.00   -     vluxseg5ei8.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     92.00   -     vluxseg5ei8.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     32.00   -     vluxseg5ei16.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     52.00   -     vluxseg5ei16.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     52.00   -     vluxseg5ei16.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     32.00   -     vluxseg5ei32.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     32.00   -     vluxseg5ei32.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     22.00   -     vluxseg5ei64.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     24.00   -     vluxseg6ei8.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     36.00   -     vluxseg6ei8.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     60.00   -     vluxseg6ei8.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     108.00  -     vluxseg6ei8.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     36.00   -     vluxseg6ei16.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     60.00   -     vluxseg6ei16.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     60.00   -     vluxseg6ei16.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     36.00   -     vluxseg6ei32.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     36.00   -     vluxseg6ei32.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     24.00   -     vluxseg6ei64.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     26.00   -     vluxseg7ei8.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     40.00   -     vluxseg7ei8.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     68.00   -     vluxseg7ei8.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     124.00  -     vluxseg7ei8.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     40.00   -     vluxseg7ei16.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     68.00   -     vluxseg7ei16.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     68.00   -     vluxseg7ei16.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     40.00   -     vluxseg7ei32.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     40.00   -     vluxseg7ei32.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     26.00   -     vluxseg7ei64.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vluxseg8ei8.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     44.00   -     vluxseg8ei8.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     76.00   -     vluxseg8ei8.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     140.00  -     vluxseg8ei8.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     44.00   -     vluxseg8ei16.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     76.00   -     vluxseg8ei16.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     76.00   -     vluxseg8ei16.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     44.00   -     vluxseg8ei32.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     44.00   -     vluxseg8ei32.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vluxseg8ei64.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     16.00   -     vloxseg2ei8.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     20.00   -     vloxseg2ei8.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vloxseg2ei8.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     44.00   -     vloxseg2ei8.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     44.00   -     vloxseg2ei8.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     44.00   -     vloxseg2ei8.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     20.00   -     vloxseg2ei16.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vloxseg2ei16.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vloxseg2ei16.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vloxseg2ei16.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vloxseg2ei16.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     20.00   -     vloxseg2ei32.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     20.00   -     vloxseg2ei32.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     20.00   -     vloxseg2ei32.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vloxseg2ei32.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     16.00   -     vloxseg2ei64.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     20.00   -     vloxseg2ei64.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vloxseg2ei64.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     18.00   -     vloxseg3ei8.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     24.00   -     vloxseg3ei8.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     36.00   -     vloxseg3ei8.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     60.00   -     vloxseg3ei8.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     60.00   -     vloxseg3ei8.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     24.00   -     vloxseg3ei16.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     36.00   -     vloxseg3ei16.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     36.00   -     vloxseg3ei16.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     36.00   -     vloxseg3ei16.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     24.00   -     vloxseg3ei32.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     24.00   -     vloxseg3ei32.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     24.00   -     vloxseg3ei32.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     18.00   -     vloxseg3ei64.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     24.00   -     vloxseg3ei64.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     20.00   -     vloxseg4ei8.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vloxseg4ei8.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     44.00   -     vloxseg4ei8.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     76.00   -     vloxseg4ei8.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     76.00   -     vloxseg4ei8.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vloxseg4ei16.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     44.00   -     vloxseg4ei16.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     44.00   -     vloxseg4ei16.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     44.00   -     vloxseg4ei16.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vloxseg4ei32.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vloxseg4ei32.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vloxseg4ei32.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     20.00   -     vloxseg4ei64.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vloxseg4ei64.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     22.00   -     vloxseg5ei8.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     32.00   -     vloxseg5ei8.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     52.00   -     vloxseg5ei8.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     92.00   -     vloxseg5ei8.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     32.00   -     vloxseg5ei16.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     52.00   -     vloxseg5ei16.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     52.00   -     vloxseg5ei16.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     32.00   -     vloxseg5ei32.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     32.00   -     vloxseg5ei32.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     22.00   -     vloxseg5ei64.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     24.00   -     vloxseg6ei8.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     36.00   -     vloxseg6ei8.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     60.00   -     vloxseg6ei8.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     108.00  -     vloxseg6ei8.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     36.00   -     vloxseg6ei16.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     60.00   -     vloxseg6ei16.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     60.00   -     vloxseg6ei16.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     36.00   -     vloxseg6ei32.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     36.00   -     vloxseg6ei32.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     24.00   -     vloxseg6ei64.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     26.00   -     vloxseg7ei8.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     40.00   -     vloxseg7ei8.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     68.00   -     vloxseg7ei8.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     124.00  -     vloxseg7ei8.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     40.00   -     vloxseg7ei16.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     68.00   -     vloxseg7ei16.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     68.00   -     vloxseg7ei16.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     40.00   -     vloxseg7ei32.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     40.00   -     vloxseg7ei32.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     26.00   -     vloxseg7ei64.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vloxseg8ei8.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     44.00   -     vloxseg8ei8.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     76.00   -     vloxseg8ei8.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     140.00  -     vloxseg8ei8.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     44.00   -     vloxseg8ei16.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     76.00   -     vloxseg8ei16.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     76.00   -     vloxseg8ei16.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     44.00   -     vloxseg8ei32.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     44.00   -     vloxseg8ei32.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vloxseg8ei64.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     16.00  vsuxseg2ei8.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     20.00  vsuxseg2ei8.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     28.00  vsuxseg2ei8.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     44.00  vsuxseg2ei8.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     76.00  vsuxseg2ei8.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     16.00  vsuxseg2ei16.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     20.00  vsuxseg2ei16.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     28.00  vsuxseg2ei16.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     44.00  vsuxseg2ei16.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     16.00  vsuxseg2ei32.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     20.00  vsuxseg2ei32.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     28.00  vsuxseg2ei32.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     16.00  vsuxseg2ei64.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     20.00  vsuxseg2ei64.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     18.00  vsuxseg3ei8.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     24.00  vsuxseg3ei8.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     36.00  vsuxseg3ei8.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     60.00  vsuxseg3ei8.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     108.00 vsuxseg3ei8.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     18.00  vsuxseg3ei16.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     24.00  vsuxseg3ei16.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     36.00  vsuxseg3ei16.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     60.00  vsuxseg3ei16.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     18.00  vsuxseg3ei32.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     24.00  vsuxseg3ei32.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     36.00  vsuxseg3ei32.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     18.00  vsuxseg3ei64.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     24.00  vsuxseg3ei64.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     20.00  vsuxseg4ei8.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     28.00  vsuxseg4ei8.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     44.00  vsuxseg4ei8.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     76.00  vsuxseg4ei8.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     140.00 vsuxseg4ei8.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     20.00  vsuxseg4ei16.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     28.00  vsuxseg4ei16.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     44.00  vsuxseg4ei16.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     76.00  vsuxseg4ei16.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     20.00  vsuxseg4ei32.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     28.00  vsuxseg4ei32.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     44.00  vsuxseg4ei32.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     20.00  vsuxseg4ei64.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     28.00  vsuxseg4ei64.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     22.00  vsuxseg5ei8.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     32.00  vsuxseg5ei8.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     52.00  vsuxseg5ei8.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     92.00  vsuxseg5ei8.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     22.00  vsuxseg5ei16.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     32.00  vsuxseg5ei16.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     52.00  vsuxseg5ei16.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     22.00  vsuxseg5ei32.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     32.00  vsuxseg5ei32.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     22.00  vsuxseg5ei64.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     24.00  vsuxseg6ei8.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     36.00  vsuxseg6ei8.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     60.00  vsuxseg6ei8.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     108.00 vsuxseg6ei8.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     24.00  vsuxseg6ei16.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     36.00  vsuxseg6ei16.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     60.00  vsuxseg6ei16.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     24.00  vsuxseg6ei32.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     36.00  vsuxseg6ei32.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     24.00  vsuxseg6ei64.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     26.00  vsuxseg7ei8.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     40.00  vsuxseg7ei8.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     68.00  vsuxseg7ei8.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     124.00 vsuxseg7ei8.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     26.00  vsuxseg7ei16.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     40.00  vsuxseg7ei16.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     68.00  vsuxseg7ei16.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     26.00  vsuxseg7ei32.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     40.00  vsuxseg7ei32.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     26.00  vsuxseg7ei64.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     28.00  vsuxseg8ei8.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     44.00  vsuxseg8ei8.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     76.00  vsuxseg8ei8.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     140.00 vsuxseg8ei8.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     28.00  vsuxseg8ei16.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     44.00  vsuxseg8ei16.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     76.00  vsuxseg8ei16.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     28.00  vsuxseg8ei32.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     44.00  vsuxseg8ei32.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     28.00  vsuxseg8ei64.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     16.00  vsoxseg2ei8.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     20.00  vsoxseg2ei8.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     28.00  vsoxseg2ei8.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     44.00  vsoxseg2ei8.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     76.00  vsoxseg2ei8.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     140.00 vsoxseg2ei8.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     16.00  vsoxseg2ei16.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     20.00  vsoxseg2ei16.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     28.00  vsoxseg2ei16.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     44.00  vsoxseg2ei16.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     76.00  vsoxseg2ei16.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     16.00  vsoxseg2ei32.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     20.00  vsoxseg2ei32.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     28.00  vsoxseg2ei32.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     44.00  vsoxseg2ei32.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     16.00  vsoxseg2ei64.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     20.00  vsoxseg2ei64.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     28.00  vsoxseg2ei64.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     18.00  vsoxseg3ei8.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     24.00  vsoxseg3ei8.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     36.00  vsoxseg3ei8.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     60.00  vsoxseg3ei8.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     108.00 vsoxseg3ei8.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     18.00  vsoxseg3ei16.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     24.00  vsoxseg3ei16.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     36.00  vsoxseg3ei16.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     60.00  vsoxseg3ei16.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     18.00  vsoxseg3ei32.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     24.00  vsoxseg3ei32.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     36.00  vsoxseg3ei32.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     18.00  vsoxseg3ei64.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     24.00  vsoxseg3ei64.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     20.00  vsoxseg4ei8.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     28.00  vsoxseg4ei8.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     44.00  vsoxseg4ei8.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     76.00  vsoxseg4ei8.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     140.00 vsoxseg4ei8.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     20.00  vsoxseg4ei16.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     28.00  vsoxseg4ei16.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     44.00  vsoxseg4ei16.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     76.00  vsoxseg4ei16.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     20.00  vsoxseg4ei32.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     28.00  vsoxseg4ei32.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     44.00  vsoxseg4ei32.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     20.00  vsoxseg4ei64.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     28.00  vsoxseg4ei64.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     22.00  vsoxseg5ei8.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     32.00  vsoxseg5ei8.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     52.00  vsoxseg5ei8.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     92.00  vsoxseg5ei8.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     22.00  vsoxseg5ei16.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     32.00  vsoxseg5ei16.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     52.00  vsoxseg5ei16.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     22.00  vsoxseg5ei32.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     32.00  vsoxseg5ei32.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     22.00  vsoxseg5ei64.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     24.00  vsoxseg6ei8.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     36.00  vsoxseg6ei8.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     60.00  vsoxseg6ei8.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     108.00 vsoxseg6ei8.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     24.00  vsoxseg6ei16.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     36.00  vsoxseg6ei16.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     60.00  vsoxseg6ei16.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     24.00  vsoxseg6ei32.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     36.00  vsoxseg6ei32.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     24.00  vsoxseg6ei64.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     26.00  vsoxseg7ei8.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     40.00  vsoxseg7ei8.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     68.00  vsoxseg7ei8.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     124.00 vsoxseg7ei8.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     26.00  vsoxseg7ei16.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     40.00  vsoxseg7ei16.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     68.00  vsoxseg7ei16.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     26.00  vsoxseg7ei32.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     40.00  vsoxseg7ei32.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     26.00  vsoxseg7ei64.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     28.00  vsoxseg8ei8.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     44.00  vsoxseg8ei8.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     76.00  vsoxseg8ei8.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     140.00 vsoxseg8ei8.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     28.00  vsoxseg8ei16.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     44.00  vsoxseg8ei16.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     76.00  vsoxseg8ei16.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     28.00  vsoxseg8ei32.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     44.00  vsoxseg8ei32.v	v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli	zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     28.00  vsoxseg8ei64.v	v8, (a0), v16
diff --git a/llvm/test/tools/llvm-mca/RISCV/SiFiveP500/alu.s b/llvm/test/tools/llvm-mca/RISCV/SiFiveP500/alu.s
new file mode 100644
index 0000000000000..d7791b73bf1bf
--- /dev/null
+++ b/llvm/test/tools/llvm-mca/RISCV/SiFiveP500/alu.s
@@ -0,0 +1,200 @@
+# NOTE: Assertions have been autogenerated by utils/update_mca_test_checks.py
+# RUN: llvm-mca -mtriple=riscv64 -mcpu=sifive-p550 -iterations=1 < %s | FileCheck %s
+
+add t0, t0, t0
+addw t2, t2, t2
+sub t1, t1, t1
+subw t3, t3, t3
+
+addi t0, t0, 100
+addiw t2, t2, 200
+
+and t2, t2, t2
+or  t3, t3, t3
+xor t4, t4, t4
+
+andi t2, t2, 10
+ori  t3, t3, 20
+xori t4, t4, 30
+
+sll t5, t5, t6
+srl t0, s0, t6
+sra s1, s1, t6
+
+slli t5, t5, 1
+srli t0, s0, 2
+srai s1, s1, 3
+
+mul s6, s6, s7
+mulw s4, s4, a2
+
+div a0, a0, a0
+divw a1, a1, a1
+rem a2, a2, a2
+remw a2, a2, a2
+
+rol t5, t5, t6
+ror t6, t6, t5
+rori t5, t5, 5
+
+rolw t5, t5, t6
+rorw t6, t6, t5
+roriw t5, t5, 5
+
+andn a0, a0, a1
+orn a1, a2, a3
+xnor a2, a3, a4
+
+min a3, a4, a5
+max a4, a5, a6
+minu a5, a6, a7
+maxu a6, a7, s0
+
+orc.b s0, s1
+rev8 s1, s2
+
+cpop t1, t1
+cpopw t2, t2
+
+sh1add t0, t1, t2
+sh2add t0, t1, t2
+sh3add t0, t1, t2
+
+add.uw    s0, s1, s2
+sh1add.uw t0, t1, t2
+sh2add.uw t0, t1, t2
+sh3add.uw t0, t1, t2
+
+# CHECK:      Iterations:        1
+# CHECK-NEXT: Instructions:      48
+# CHECK-NEXT: Total Cycles:      120
+# CHECK-NEXT: Total uOps:        48
+
+# CHECK:      Dispatch Width:    3
+# CHECK-NEXT: uOps Per Cycle:    0.40
+# CHECK-NEXT: IPC:               0.40
+# CHECK-NEXT: Block RThroughput: 106.0
+
+# CHECK:      Instruction Info:
+# CHECK-NEXT: [1]: #uOps
+# CHECK-NEXT: [2]: Latency
+# CHECK-NEXT: [3]: RThroughput
+# CHECK-NEXT: [4]: MayLoad
+# CHECK-NEXT: [5]: MayStore
+# CHECK-NEXT: [6]: HasSideEffects (U)
+
+# CHECK:      [1]    [2]    [3]    [4]    [5]    [6]    Instructions:
+# CHECK-NEXT:  1      1     0.33                        add	t0, t0, t0
+# CHECK-NEXT:  1      1     0.33                        addw	t2, t2, t2
+# CHECK-NEXT:  1      1     0.33                        sub	t1, t1, t1
+# CHECK-NEXT:  1      1     0.33                        subw	t3, t3, t3
+# CHECK-NEXT:  1      1     0.33                        addi	t0, t0, 100
+# CHECK-NEXT:  1      1     0.33                        addiw	t2, t2, 200
+# CHECK-NEXT:  1      1     0.33                        and	t2, t2, t2
+# CHECK-NEXT:  1      1     0.33                        or	t3, t3, t3
+# CHECK-NEXT:  1      1     0.33                        xor	t4, t4, t4
+# CHECK-NEXT:  1      1     0.33                        andi	t2, t2, 10
+# CHECK-NEXT:  1      1     0.33                        ori	t3, t3, 20
+# CHECK-NEXT:  1      1     0.33                        xori	t4, t4, 30
+# CHECK-NEXT:  1      1     0.33                        sll	t5, t5, t6
+# CHECK-NEXT:  1      1     0.33                        srl	t0, s0, t6
+# CHECK-NEXT:  1      1     0.33                        sra	s1, s1, t6
+# CHECK-NEXT:  1      1     0.33                        slli	t5, t5, 1
+# CHECK-NEXT:  1      1     0.33                        srli	t0, s0, 2
+# CHECK-NEXT:  1      1     0.33                        srai	s1, s1, 3
+# CHECK-NEXT:  1      3     1.00                        mul	s6, s6, s7
+# CHECK-NEXT:  1      3     1.00                        mulw	s4, s4, a2
+# CHECK-NEXT:  1      35    34.00                       div	a0, a0, a0
+# CHECK-NEXT:  1      20    19.00                       divw	a1, a1, a1
+# CHECK-NEXT:  1      35    34.00                       rem	a2, a2, a2
+# CHECK-NEXT:  1      20    19.00                       remw	a2, a2, a2
+# CHECK-NEXT:  1      1     0.33                        rol	t5, t5, t6
+# CHECK-NEXT:  1      1     0.33                        ror	t6, t6, t5
+# CHECK-NEXT:  1      1     0.33                        rori	t5, t5, 5
+# CHECK-NEXT:  1      1     0.33                        rolw	t5, t5, t6
+# CHECK-NEXT:  1      1     0.33                        rorw	t6, t6, t5
+# CHECK-NEXT:  1      1     0.33                        roriw	t5, t5, 5
+# CHECK-NEXT:  1      1     0.33                        andn	a0, a0, a1
+# CHECK-NEXT:  1      1     0.33                        orn	a1, a2, a3
+# CHECK-NEXT:  1      1     0.33                        xnor	a2, a3, a4
+# CHECK-NEXT:  1      1     0.33                        min	a3, a4, a5
+# CHECK-NEXT:  1      1     0.33                        max	a4, a5, a6
+# CHECK-NEXT:  1      1     0.33                        minu	a5, a6, a7
+# CHECK-NEXT:  1      1     0.33                        maxu	a6, a7, s0
+# CHECK-NEXT:  1      1     0.33                        orc.b	s0, s1
+# CHECK-NEXT:  1      1     0.33                        rev8	s1, s2
+# CHECK-NEXT:  1      3     1.00                        cpop	t1, t1
+# CHECK-NEXT:  1      3     1.00                        cpopw	t2, t2
+# CHECK-NEXT:  1      1     0.33                        sh1add	t0, t1, t2
+# CHECK-NEXT:  1      1     0.33                        sh2add	t0, t1, t2
+# CHECK-NEXT:  1      1     0.33                        sh3add	t0, t1, t2
+# CHECK-NEXT:  1      1     0.33                        add.uw	s0, s1, s2
+# CHECK-NEXT:  1      1     0.33                        sh1add.uw	t0, t1, t2
+# CHECK-NEXT:  1      1     0.33                        sh2add.uw	t0, t1, t2
+# CHECK-NEXT:  1      1     0.33                        sh3add.uw	t0, t1, t2
+
+# CHECK:      Resources:
+# CHECK-NEXT: [0]   - SiFiveP500Div
+# CHECK-NEXT: [1]   - SiFiveP500FEXQ0
+# CHECK-NEXT: [2]   - SiFiveP500FEXQ1
+# CHECK-NEXT: [3]   - SiFiveP500FloatDiv
+# CHECK-NEXT: [4]   - SiFiveP500IEXQ0
+# CHECK-NEXT: [5]   - SiFiveP500IEXQ1
+# CHECK-NEXT: [6]   - SiFiveP500IEXQ2
+# CHECK-NEXT: [7]   - SiFiveP500Load
+# CHECK-NEXT: [8]   - SiFiveP500Store
+
+# CHECK:      Resource pressure per iteration:
+# CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]
+# CHECK-NEXT: 106.00  -      -      -     14.00  15.00  19.00   -      -
+
+# CHECK:      Resource pressure by instruction:
+# CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    Instructions:
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     add	t0, t0, t0
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -      -     addw	t2, t2, t2
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -     sub	t1, t1, t1
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     subw	t3, t3, t3
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -      -     addi	t0, t0, 100
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -     addiw	t2, t2, 200
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     and	t2, t2, t2
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -      -     or	t3, t3, t3
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -     xor	t4, t4, t4
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     andi	t2, t2, 10
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -      -     ori	t3, t3, 20
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -     xori	t4, t4, 30
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sll	t5, t5, t6
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -      -     srl	t0, s0, t6
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -     sra	s1, s1, t6
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     slli	t5, t5, 1
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -      -     srli	t0, s0, 2
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -     srai	s1, s1, 3
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     mul	s6, s6, s7
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     mulw	s4, s4, a2
+# CHECK-NEXT: 34.00   -      -      -      -      -     1.00    -      -     div	a0, a0, a0
+# CHECK-NEXT: 19.00   -      -      -      -      -     1.00    -      -     divw	a1, a1, a1
+# CHECK-NEXT: 34.00   -      -      -      -      -     1.00    -      -     rem	a2, a2, a2
+# CHECK-NEXT: 19.00   -      -      -      -      -     1.00    -      -     remw	a2, a2, a2
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -      -     rol	t5, t5, t6
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -     ror	t6, t6, t5
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -      -     rori	t5, t5, 5
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -     rolw	t5, t5, t6
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -      -     rorw	t6, t6, t5
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -     roriw	t5, t5, 5
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -      -     andn	a0, a0, a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -     orn	a1, a2, a3
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     xnor	a2, a3, a4
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     min	a3, a4, a5
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -      -     max	a4, a5, a6
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -     minu	a5, a6, a7
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     maxu	a6, a7, s0
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -      -     orc.b	s0, s1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -     rev8	s1, s2
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     cpop	t1, t1
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     cpopw	t2, t2
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -     sh1add	t0, t1, t2
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -      -     sh2add	t0, t1, t2
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sh3add	t0, t1, t2
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -      -     add.uw	s0, s1, s2
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -      -     sh1add.uw	t0, t1, t2
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -     sh2add.uw	t0, t1, t2
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sh3add.uw	t0, t1, t2
diff --git a/llvm/test/tools/llvm-mca/RISCV/SiFiveP500/fp.s b/llvm/test/tools/llvm-mca/RISCV/SiFiveP500/fp.s
new file mode 100644
index 0000000000000..8eb64405b7911
--- /dev/null
+++ b/llvm/test/tools/llvm-mca/RISCV/SiFiveP500/fp.s
@@ -0,0 +1,155 @@
+# NOTE: Assertions have been autogenerated by utils/update_mca_test_checks.py
+# RUN: llvm-mca -mtriple=riscv64 -mcpu=sifive-p550 -iterations=1 < %s | FileCheck %s
+
+fmin.s ft0, fa0, fa1
+fmax.s ft1, fa0, fa1
+fmadd.s fs0, fs0, fs8, fs9
+fmsub.s fs1, fs1, fs8, fs9
+fmul.s fs3, fs3, fs4
+fadd.s fs4, fs4, fs5
+fdiv.s fs2, fs3, fs4
+fsqrt.s ft1, fa2
+
+fcvt.s.w ft2, a0
+fcvt.s.l ft3, a1
+fcvt.s.wu ft2, a0
+fcvt.s.lu ft3, a1
+fcvt.w.s a0, ft2
+fcvt.l.s a1, fs2
+fcvt.wu.s a0, ft2
+fcvt.lu.s a1, fs2
+
+feq.s a2, fa0, fa1
+fclass.s a3, fa2
+
+fmin.d ft2, ft4, ft5
+fmax.d ft3, ft4, ft5
+fmadd.d fs0, fs0, fs8, fs9
+fmsub.d fs1, fs1, fs8, fs9
+fmul.d ft4, ft4, ft5
+fadd.d ft4, ft5, ft6
+fdiv.d fs4, fa3, ft5
+fsqrt.d ft2, fa3
+
+fcvt.d.w ft2, a0
+fcvt.d.l ft3, a1
+fcvt.d.w ft2, a0
+fcvt.d.l ft3, a1
+fcvt.w.d a0, ft2
+fcvt.l.d a1, fs2
+
+feq.d a2, fa0, fa1
+fclass.d a3, fa2
+
+fcvt.d.s ft1, ft2
+fcvt.s.d ft3, ft4
+
+# CHECK:      Iterations:        1
+# CHECK-NEXT: Instructions:      36
+# CHECK-NEXT: Total Cycles:      106
+# CHECK-NEXT: Total uOps:        36
+
+# CHECK:      Dispatch Width:    3
+# CHECK-NEXT: uOps Per Cycle:    0.34
+# CHECK-NEXT: IPC:               0.34
+# CHECK-NEXT: Block RThroughput: 99.0
+
+# CHECK:      Instruction Info:
+# CHECK-NEXT: [1]: #uOps
+# CHECK-NEXT: [2]: Latency
+# CHECK-NEXT: [3]: RThroughput
+# CHECK-NEXT: [4]: MayLoad
+# CHECK-NEXT: [5]: MayStore
+# CHECK-NEXT: [6]: HasSideEffects (U)
+
+# CHECK:      [1]    [2]    [3]    [4]    [5]    [6]    Instructions:
+# CHECK-NEXT:  1      2     0.50                        fmin.s	ft0, fa0, fa1
+# CHECK-NEXT:  1      2     0.50                        fmax.s	ft1, fa0, fa1
+# CHECK-NEXT:  1      4     0.50                        fmadd.s	fs0, fs0, fs8, fs9
+# CHECK-NEXT:  1      4     0.50                        fmsub.s	fs1, fs1, fs8, fs9
+# CHECK-NEXT:  1      4     0.50                        fmul.s	fs3, fs3, fs4
+# CHECK-NEXT:  1      4     0.50                        fadd.s	fs4, fs4, fs5
+# CHECK-NEXT:  1      19    18.00                       fdiv.s	fs2, fs3, fs4
+# CHECK-NEXT:  1      18    17.00                       fsqrt.s	ft1, fa2
+# CHECK-NEXT:  1      2     1.00                        fcvt.s.w	ft2, a0
+# CHECK-NEXT:  1      2     1.00                        fcvt.s.l	ft3, a1
+# CHECK-NEXT:  1      2     1.00                        fcvt.s.wu	ft2, a0
+# CHECK-NEXT:  1      2     1.00                        fcvt.s.lu	ft3, a1
+# CHECK-NEXT:  1      2     1.00                        fcvt.w.s	a0, ft2
+# CHECK-NEXT:  1      2     1.00                        fcvt.l.s	a1, fs2
+# CHECK-NEXT:  1      2     1.00                        fcvt.wu.s	a0, ft2
+# CHECK-NEXT:  1      2     1.00                        fcvt.lu.s	a1, fs2
+# CHECK-NEXT:  1      2     1.00                        feq.s	a2, fa0, fa1
+# CHECK-NEXT:  1      2     1.00                        fclass.s	a3, fa2
+# CHECK-NEXT:  1      2     0.50                        fmin.d	ft2, ft4, ft5
+# CHECK-NEXT:  1      2     0.50                        fmax.d	ft3, ft4, ft5
+# CHECK-NEXT:  1      4     0.50                        fmadd.d	fs0, fs0, fs8, fs9
+# CHECK-NEXT:  1      4     0.50                        fmsub.d	fs1, fs1, fs8, fs9
+# CHECK-NEXT:  1      4     0.50                        fmul.d	ft4, ft4, ft5
+# CHECK-NEXT:  1      4     0.50                        fadd.d	ft4, ft5, ft6
+# CHECK-NEXT:  1      33    32.00                       fdiv.d	fs4, fa3, ft5
+# CHECK-NEXT:  1      33    32.00                       fsqrt.d	ft2, fa3
+# CHECK-NEXT:  1      2     1.00                        fcvt.d.w	ft2, a0
+# CHECK-NEXT:  1      2     1.00                        fcvt.d.l	ft3, a1
+# CHECK-NEXT:  1      2     1.00                        fcvt.d.w	ft2, a0
+# CHECK-NEXT:  1      2     1.00                        fcvt.d.l	ft3, a1
+# CHECK-NEXT:  1      2     1.00                        fcvt.w.d	a0, ft2
+# CHECK-NEXT:  1      2     1.00                        fcvt.l.d	a1, fs2
+# CHECK-NEXT:  1      2     1.00                        feq.d	a2, fa0, fa1
+# CHECK-NEXT:  1      2     1.00                        fclass.d	a3, fa2
+# CHECK-NEXT:  1      2     0.50                        fcvt.d.s	ft1, ft2
+# CHECK-NEXT:  1      2     0.50                        fcvt.s.d	ft3, ft4
+
+# CHECK:      Resources:
+# CHECK-NEXT: [0]   - SiFiveP500Div
+# CHECK-NEXT: [1]   - SiFiveP500FEXQ0
+# CHECK-NEXT: [2]   - SiFiveP500FEXQ1
+# CHECK-NEXT: [3]   - SiFiveP500FloatDiv
+# CHECK-NEXT: [4]   - SiFiveP500IEXQ0
+# CHECK-NEXT: [5]   - SiFiveP500IEXQ1
+# CHECK-NEXT: [6]   - SiFiveP500IEXQ2
+# CHECK-NEXT: [7]   - SiFiveP500Load
+# CHECK-NEXT: [8]   - SiFiveP500Store
+
+# CHECK:      Resource pressure per iteration:
+# CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]
+# CHECK-NEXT:  -     14.00  14.00  99.00   -      -     8.00    -      -
+
+# CHECK:      Resource pressure by instruction:
+# CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    Instructions:
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -      -     fmin.s	ft0, fa0, fa1
+# CHECK-NEXT:  -     1.00    -      -      -      -      -      -      -     fmax.s	ft1, fa0, fa1
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -      -     fmadd.s	fs0, fs0, fs8, fs9
+# CHECK-NEXT:  -     1.00    -      -      -      -      -      -      -     fmsub.s	fs1, fs1, fs8, fs9
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -      -     fmul.s	fs3, fs3, fs4
+# CHECK-NEXT:  -     1.00    -      -      -      -      -      -      -     fadd.s	fs4, fs4, fs5
+# CHECK-NEXT:  -      -     1.00   18.00   -      -      -      -      -     fdiv.s	fs2, fs3, fs4
+# CHECK-NEXT:  -      -     1.00   17.00   -      -      -      -      -     fsqrt.s	ft1, fa2
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcvt.s.w	ft2, a0
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcvt.s.l	ft3, a1
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcvt.s.wu	ft2, a0
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcvt.s.lu	ft3, a1
+# CHECK-NEXT:  -     1.00    -      -      -      -      -      -      -     fcvt.w.s	a0, ft2
+# CHECK-NEXT:  -     1.00    -      -      -      -      -      -      -     fcvt.l.s	a1, fs2
+# CHECK-NEXT:  -     1.00    -      -      -      -      -      -      -     fcvt.wu.s	a0, ft2
+# CHECK-NEXT:  -     1.00    -      -      -      -      -      -      -     fcvt.lu.s	a1, fs2
+# CHECK-NEXT:  -     1.00    -      -      -      -      -      -      -     feq.s	a2, fa0, fa1
+# CHECK-NEXT:  -     1.00    -      -      -      -      -      -      -     fclass.s	a3, fa2
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -      -     fmin.d	ft2, ft4, ft5
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -      -     fmax.d	ft3, ft4, ft5
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -      -     fmadd.d	fs0, fs0, fs8, fs9
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -      -     fmsub.d	fs1, fs1, fs8, fs9
+# CHECK-NEXT:  -     1.00    -      -      -      -      -      -      -     fmul.d	ft4, ft4, ft5
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -      -     fadd.d	ft4, ft5, ft6
+# CHECK-NEXT:  -      -     1.00   32.00   -      -      -      -      -     fdiv.d	fs4, fa3, ft5
+# CHECK-NEXT:  -      -     1.00   32.00   -      -      -      -      -     fsqrt.d	ft2, fa3
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcvt.d.w	ft2, a0
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcvt.d.l	ft3, a1
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcvt.d.w	ft2, a0
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcvt.d.l	ft3, a1
+# CHECK-NEXT:  -     1.00    -      -      -      -      -      -      -     fcvt.w.d	a0, ft2
+# CHECK-NEXT:  -     1.00    -      -      -      -      -      -      -     fcvt.l.d	a1, fs2
+# CHECK-NEXT:  -     1.00    -      -      -      -      -      -      -     feq.d	a2, fa0, fa1
+# CHECK-NEXT:  -     1.00    -      -      -      -      -      -      -     fclass.d	a3, fa2
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -      -     fcvt.d.s	ft1, ft2
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -      -     fcvt.s.d	ft3, ft4
diff --git a/llvm/test/tools/llvm-mca/RISCV/SiFiveP500/load.s b/llvm/test/tools/llvm-mca/RISCV/SiFiveP500/load.s
new file mode 100644
index 0000000000000..2b7df9215e0cb
--- /dev/null
+++ b/llvm/test/tools/llvm-mca/RISCV/SiFiveP500/load.s
@@ -0,0 +1,54 @@
+# NOTE: Assertions have been autogenerated by utils/update_mca_test_checks.py
+# RUN: llvm-mca -mtriple=riscv64 -mcpu=sifive-p550 -iterations=1 < %s | FileCheck %s
+
+lw t0, 0(a0)
+ld t0, 0(a0)
+
+flw ft0, 0(a0)
+fld ft0, 0(a0)
+
+# CHECK:      Iterations:        1
+# CHECK-NEXT: Instructions:      4
+# CHECK-NEXT: Total Cycles:      12
+# CHECK-NEXT: Total uOps:        4
+
+# CHECK:      Dispatch Width:    3
+# CHECK-NEXT: uOps Per Cycle:    0.33
+# CHECK-NEXT: IPC:               0.33
+# CHECK-NEXT: Block RThroughput: 4.0
+
+# CHECK:      Instruction Info:
+# CHECK-NEXT: [1]: #uOps
+# CHECK-NEXT: [2]: Latency
+# CHECK-NEXT: [3]: RThroughput
+# CHECK-NEXT: [4]: MayLoad
+# CHECK-NEXT: [5]: MayStore
+# CHECK-NEXT: [6]: HasSideEffects (U)
+
+# CHECK:      [1]    [2]    [3]    [4]    [5]    [6]    Instructions:
+# CHECK-NEXT:  1      4     1.00    *                   lw	t0, 0(a0)
+# CHECK-NEXT:  1      4     1.00    *                   ld	t0, 0(a0)
+# CHECK-NEXT:  1      6     1.00    *                   flw	ft0, 0(a0)
+# CHECK-NEXT:  1      6     1.00    *                   fld	ft0, 0(a0)
+
+# CHECK:      Resources:
+# CHECK-NEXT: [0]   - SiFiveP500Div
+# CHECK-NEXT: [1]   - SiFiveP500FEXQ0
+# CHECK-NEXT: [2]   - SiFiveP500FEXQ1
+# CHECK-NEXT: [3]   - SiFiveP500FloatDiv
+# CHECK-NEXT: [4]   - SiFiveP500IEXQ0
+# CHECK-NEXT: [5]   - SiFiveP500IEXQ1
+# CHECK-NEXT: [6]   - SiFiveP500IEXQ2
+# CHECK-NEXT: [7]   - SiFiveP500Load
+# CHECK-NEXT: [8]   - SiFiveP500Store
+
+# CHECK:      Resource pressure per iteration:
+# CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]
+# CHECK-NEXT:  -      -      -      -      -      -      -     4.00    -
+
+# CHECK:      Resource pressure by instruction:
+# CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    Instructions:
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     lw	t0, 0(a0)
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     ld	t0, 0(a0)
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     flw	ft0, 0(a0)
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     fld	ft0, 0(a0)
diff --git a/llvm/test/tools/llvm-mca/RISCV/SiFiveX280/vector-integer-arithmetic.s b/llvm/test/tools/llvm-mca/RISCV/SiFiveX280/vector-integer-arithmetic.s
index 3b6fd7e150137..82da484202c9c 100644
--- a/llvm/test/tools/llvm-mca/RISCV/SiFiveX280/vector-integer-arithmetic.s
+++ b/llvm/test/tools/llvm-mca/RISCV/SiFiveX280/vector-integer-arithmetic.s
@@ -775,7 +775,7 @@ vmv.v.v v4, v12
 
 # CHECK:      Iterations:        1
 # CHECK-NEXT: Instructions:      727
-# CHECK-NEXT: Total Cycles:      12018
+# CHECK-NEXT: Total Cycles:      12174
 # CHECK-NEXT: Total uOps:        727
 
 # CHECK:      Dispatch Width:    2
@@ -1032,49 +1032,49 @@ vmv.v.v v4, v12
 # CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e64, m8, tu, mu
 # CHECK-NEXT:  1      4     17.00                       vor.vv	v4, v8, v12
 # CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e8, mf8, tu, mu
-# CHECK-NEXT:  1      4     2.00                        vsll.vv	v4, v8, v12
+# CHECK-NEXT:  1      8     2.00                        vsll.vv	v4, v8, v12
 # CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e8, mf4, tu, mu
-# CHECK-NEXT:  1      4     2.00                        vsll.vx	v4, v8, a0
+# CHECK-NEXT:  1      8     2.00                        vsll.vx	v4, v8, a0
 # CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e8, mf2, tu, mu
-# CHECK-NEXT:  1      4     2.00                        vsll.vi	v4, v8, 0
+# CHECK-NEXT:  1      8     2.00                        vsll.vi	v4, v8, 0
 # CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e8, m1, tu, mu
-# CHECK-NEXT:  1      4     3.00                        vsrl.vv	v4, v8, v12
+# CHECK-NEXT:  1      8     3.00                        vsrl.vv	v4, v8, v12
 # CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e8, m2, tu, mu
-# CHECK-NEXT:  1      4     5.00                        vsrl.vx	v4, v8, a0
+# CHECK-NEXT:  1      8     5.00                        vsrl.vx	v4, v8, a0
 # CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e8, m4, tu, mu
-# CHECK-NEXT:  1      4     9.00                        vsrl.vi	v4, v8, 0
+# CHECK-NEXT:  1      8     9.00                        vsrl.vi	v4, v8, 0
 # CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e8, m8, tu, mu
-# CHECK-NEXT:  1      4     17.00                       vsra.vv	v4, v8, v12
+# CHECK-NEXT:  1      8     17.00                       vsra.vv	v4, v8, v12
 # CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e16, mf4, tu, mu
-# CHECK-NEXT:  1      4     2.00                        vsra.vx	v4, v8, a0
+# CHECK-NEXT:  1      8     2.00                        vsra.vx	v4, v8, a0
 # CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e16, mf2, tu, mu
-# CHECK-NEXT:  1      4     2.00                        vsra.vi	v4, v8, 0
+# CHECK-NEXT:  1      8     2.00                        vsra.vi	v4, v8, 0
 # CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e16, m1, tu, mu
-# CHECK-NEXT:  1      4     3.00                        vsll.vv	v4, v8, v12
+# CHECK-NEXT:  1      8     3.00                        vsll.vv	v4, v8, v12
 # CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e16, m2, tu, mu
-# CHECK-NEXT:  1      4     5.00                        vsll.vx	v4, v8, a0
+# CHECK-NEXT:  1      8     5.00                        vsll.vx	v4, v8, a0
 # CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e16, m4, tu, mu
-# CHECK-NEXT:  1      4     9.00                        vsll.vi	v4, v8, 0
+# CHECK-NEXT:  1      8     9.00                        vsll.vi	v4, v8, 0
 # CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e16, m8, tu, mu
-# CHECK-NEXT:  1      4     17.00                       vsrl.vv	v4, v8, v12
+# CHECK-NEXT:  1      8     17.00                       vsrl.vv	v4, v8, v12
 # CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e32, mf2, tu, mu
-# CHECK-NEXT:  1      4     2.00                        vsrl.vx	v4, v8, a0
+# CHECK-NEXT:  1      8     2.00                        vsrl.vx	v4, v8, a0
 # CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e32, m1, tu, mu
-# CHECK-NEXT:  1      4     3.00                        vsrl.vi	v4, v8, 0
+# CHECK-NEXT:  1      8     3.00                        vsrl.vi	v4, v8, 0
 # CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e32, m2, tu, mu
-# CHECK-NEXT:  1      4     5.00                        vsra.vv	v4, v8, v12
+# CHECK-NEXT:  1      8     5.00                        vsra.vv	v4, v8, v12
 # CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e32, m4, tu, mu
-# CHECK-NEXT:  1      4     9.00                        vsra.vx	v4, v8, a0
+# CHECK-NEXT:  1      8     9.00                        vsra.vx	v4, v8, a0
 # CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e32, m8, tu, mu
-# CHECK-NEXT:  1      4     17.00                       vsra.vi	v4, v8, 0
+# CHECK-NEXT:  1      8     17.00                       vsra.vi	v4, v8, 0
 # CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e64, m1, tu, mu
-# CHECK-NEXT:  1      4     3.00                        vsll.vv	v4, v8, v12
+# CHECK-NEXT:  1      8     3.00                        vsll.vv	v4, v8, v12
 # CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e64, m2, tu, mu
-# CHECK-NEXT:  1      4     5.00                        vsll.vx	v4, v8, a0
+# CHECK-NEXT:  1      8     5.00                        vsll.vx	v4, v8, a0
 # CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e64, m4, tu, mu
-# CHECK-NEXT:  1      4     9.00                        vsll.vi	v4, v8, 0
+# CHECK-NEXT:  1      8     9.00                        vsll.vi	v4, v8, 0
 # CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e64, m8, tu, mu
-# CHECK-NEXT:  1      4     17.00                       vsrl.vv	v4, v8, v12
+# CHECK-NEXT:  1      8     17.00                       vsrl.vv	v4, v8, v12
 # CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e8, mf8, tu, mu
 # CHECK-NEXT:  1      8     2.00                        vnsrl.wv	v4, v8, v12
 # CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e8, mf4, tu, mu
@@ -1228,49 +1228,49 @@ vmv.v.v v4, v12
 # CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e64, m8, tu, mu
 # CHECK-NEXT:  1      4     17.00                       vmaxu.vx	v4, v8, a0
 # CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e8, mf8, tu, mu
-# CHECK-NEXT:  1      4     2.00                        vmul.vv	v4, v8, v12
+# CHECK-NEXT:  1      8     2.00                        vmul.vv	v4, v8, v12
 # CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e8, mf4, tu, mu
-# CHECK-NEXT:  1      4     2.00                        vmul.vx	v4, v8, a0
+# CHECK-NEXT:  1      8     2.00                        vmul.vx	v4, v8, a0
 # CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e8, mf2, tu, mu
-# CHECK-NEXT:  1      4     2.00                        vmulh.vv	v4, v8, v12
+# CHECK-NEXT:  1      8     2.00                        vmulh.vv	v4, v8, v12
 # CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e8, m1, tu, mu
-# CHECK-NEXT:  1      4     3.00                        vmulh.vx	v4, v8, a0
+# CHECK-NEXT:  1      8     3.00                        vmulh.vx	v4, v8, a0
 # CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e8, m2, tu, mu
-# CHECK-NEXT:  1      4     5.00                        vmulhu.vv	v4, v8, v12
+# CHECK-NEXT:  1      8     5.00                        vmulhu.vv	v4, v8, v12
 # CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e8, m4, tu, mu
-# CHECK-NEXT:  1      4     9.00                        vmulhu.vx	v4, v8, a0
+# CHECK-NEXT:  1      8     9.00                        vmulhu.vx	v4, v8, a0
 # CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e8, m8, tu, mu
-# CHECK-NEXT:  1      4     17.00                       vmulhsu.vv	v4, v8, v12
+# CHECK-NEXT:  1      8     17.00                       vmulhsu.vv	v4, v8, v12
 # CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e16, mf4, tu, mu
-# CHECK-NEXT:  1      4     2.00                        vmulhsu.vx	v4, v8, a0
+# CHECK-NEXT:  1      8     2.00                        vmulhsu.vx	v4, v8, a0
 # CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e16, mf2, tu, mu
-# CHECK-NEXT:  1      4     2.00                        vmul.vv	v4, v8, v12
+# CHECK-NEXT:  1      8     2.00                        vmul.vv	v4, v8, v12
 # CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e16, m1, tu, mu
-# CHECK-NEXT:  1      4     3.00                        vmul.vx	v4, v8, a0
+# CHECK-NEXT:  1      8     3.00                        vmul.vx	v4, v8, a0
 # CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e16, m2, tu, mu
-# CHECK-NEXT:  1      4     5.00                        vmulh.vv	v4, v8, v12
+# CHECK-NEXT:  1      8     5.00                        vmulh.vv	v4, v8, v12
 # CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e16, m4, tu, mu
-# CHECK-NEXT:  1      4     9.00                        vmulh.vx	v4, v8, a0
+# CHECK-NEXT:  1      8     9.00                        vmulh.vx	v4, v8, a0
 # CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e16, m8, tu, mu
-# CHECK-NEXT:  1      4     17.00                       vmulhu.vv	v4, v8, v12
+# CHECK-NEXT:  1      8     17.00                       vmulhu.vv	v4, v8, v12
 # CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e32, mf2, tu, mu
-# CHECK-NEXT:  1      4     2.00                        vmulhu.vx	v4, v8, a0
+# CHECK-NEXT:  1      8     2.00                        vmulhu.vx	v4, v8, a0
 # CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e32, m1, tu, mu
-# CHECK-NEXT:  1      4     3.00                        vmulhsu.vv	v4, v8, v12
+# CHECK-NEXT:  1      8     3.00                        vmulhsu.vv	v4, v8, v12
 # CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e32, m2, tu, mu
-# CHECK-NEXT:  1      4     5.00                        vmulhsu.vx	v4, v8, a0
+# CHECK-NEXT:  1      8     5.00                        vmulhsu.vx	v4, v8, a0
 # CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e32, m4, tu, mu
-# CHECK-NEXT:  1      4     9.00                        vmul.vv	v4, v8, v12
+# CHECK-NEXT:  1      8     9.00                        vmul.vv	v4, v8, v12
 # CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e32, m8, tu, mu
-# CHECK-NEXT:  1      4     17.00                       vmul.vx	v4, v8, a0
+# CHECK-NEXT:  1      8     17.00                       vmul.vx	v4, v8, a0
 # CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e64, m1, tu, mu
-# CHECK-NEXT:  1      4     3.00                        vmulh.vv	v4, v8, v12
+# CHECK-NEXT:  1      8     3.00                        vmulh.vv	v4, v8, v12
 # CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e64, m2, tu, mu
-# CHECK-NEXT:  1      4     5.00                        vmulh.vx	v4, v8, a0
+# CHECK-NEXT:  1      8     5.00                        vmulh.vx	v4, v8, a0
 # CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e64, m4, tu, mu
-# CHECK-NEXT:  1      4     9.00                        vmulhu.vv	v4, v8, v12
+# CHECK-NEXT:  1      8     9.00                        vmulhu.vv	v4, v8, v12
 # CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e64, m8, tu, mu
-# CHECK-NEXT:  1      4     17.00                       vmulhu.vx	v4, v8, a0
+# CHECK-NEXT:  1      8     17.00                       vmulhu.vx	v4, v8, a0
 # CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e8, mf8, tu, mu
 # CHECK-NEXT:  1      30    31.00                       vdivu.vv	v4, v8, v12
 # CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e8, mf4, tu, mu
@@ -1352,49 +1352,49 @@ vmv.v.v v4, v12
 # CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e32, m8, tu, mu
 # CHECK-NEXT:  1      8     9.00                        vwmulsu.vx	v4, v8, a0
 # CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e8, mf8, tu, mu
-# CHECK-NEXT:  1      4     2.00                        vmacc.vv	v4, v12, v8
+# CHECK-NEXT:  1      8     2.00                        vmacc.vv	v4, v12, v8
 # CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e8, mf4, tu, mu
-# CHECK-NEXT:  1      4     2.00                        vmacc.vx	v4, a0, v8
+# CHECK-NEXT:  1      8     2.00                        vmacc.vx	v4, a0, v8
 # CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e8, mf2, tu, mu
-# CHECK-NEXT:  1      4     2.00                        vnmsac.vv	v4, v12, v8
+# CHECK-NEXT:  1      8     2.00                        vnmsac.vv	v4, v12, v8
 # CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e8, m1, tu, mu
-# CHECK-NEXT:  1      4     3.00                        vnmsac.vx	v4, a0, v8
+# CHECK-NEXT:  1      8     3.00                        vnmsac.vx	v4, a0, v8
 # CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e8, m2, tu, mu
-# CHECK-NEXT:  1      4     5.00                        vmadd.vv	v4, v12, v8
+# CHECK-NEXT:  1      8     5.00                        vmadd.vv	v4, v12, v8
 # CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e8, m4, tu, mu
-# CHECK-NEXT:  1      4     9.00                        vmadd.vx	v4, a0, v8
+# CHECK-NEXT:  1      8     9.00                        vmadd.vx	v4, a0, v8
 # CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e8, m8, tu, mu
-# CHECK-NEXT:  1      4     17.00                       vnmsub.vv	v4, v12, v8
+# CHECK-NEXT:  1      8     17.00                       vnmsub.vv	v4, v12, v8
 # CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e16, mf4, tu, mu
-# CHECK-NEXT:  1      4     2.00                        vnmsub.vx	v4, a0, v8
+# CHECK-NEXT:  1      8     2.00                        vnmsub.vx	v4, a0, v8
 # CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e16, mf2, tu, mu
-# CHECK-NEXT:  1      4     2.00                        vmacc.vv	v4, v12, v8
+# CHECK-NEXT:  1      8     2.00                        vmacc.vv	v4, v12, v8
 # CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e16, m1, tu, mu
-# CHECK-NEXT:  1      4     3.00                        vmacc.vx	v4, a0, v8
+# CHECK-NEXT:  1      8     3.00                        vmacc.vx	v4, a0, v8
 # CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e16, m2, tu, mu
-# CHECK-NEXT:  1      4     5.00                        vnmsac.vv	v4, v12, v8
+# CHECK-NEXT:  1      8     5.00                        vnmsac.vv	v4, v12, v8
 # CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e16, m4, tu, mu
-# CHECK-NEXT:  1      4     9.00                        vnmsac.vx	v4, a0, v8
+# CHECK-NEXT:  1      8     9.00                        vnmsac.vx	v4, a0, v8
 # CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e16, m8, tu, mu
-# CHECK-NEXT:  1      4     17.00                       vmadd.vv	v4, v12, v8
+# CHECK-NEXT:  1      8     17.00                       vmadd.vv	v4, v12, v8
 # CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e32, mf2, tu, mu
-# CHECK-NEXT:  1      4     2.00                        vmadd.vx	v4, a0, v8
+# CHECK-NEXT:  1      8     2.00                        vmadd.vx	v4, a0, v8
 # CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e32, m1, tu, mu
-# CHECK-NEXT:  1      4     3.00                        vnmsub.vv	v4, v12, v8
+# CHECK-NEXT:  1      8     3.00                        vnmsub.vv	v4, v12, v8
 # CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e32, m2, tu, mu
-# CHECK-NEXT:  1      4     5.00                        vnmsub.vx	v4, a0, v8
+# CHECK-NEXT:  1      8     5.00                        vnmsub.vx	v4, a0, v8
 # CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e32, m4, tu, mu
-# CHECK-NEXT:  1      4     9.00                        vmacc.vv	v4, v12, v8
+# CHECK-NEXT:  1      8     9.00                        vmacc.vv	v4, v12, v8
 # CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e32, m8, tu, mu
-# CHECK-NEXT:  1      4     17.00                       vmacc.vx	v4, a0, v8
+# CHECK-NEXT:  1      8     17.00                       vmacc.vx	v4, a0, v8
 # CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e64, m1, tu, mu
-# CHECK-NEXT:  1      4     3.00                        vnmsac.vv	v4, v12, v8
+# CHECK-NEXT:  1      8     3.00                        vnmsac.vv	v4, v12, v8
 # CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e64, m2, tu, mu
-# CHECK-NEXT:  1      4     5.00                        vnmsac.vx	v4, a0, v8
+# CHECK-NEXT:  1      8     5.00                        vnmsac.vx	v4, a0, v8
 # CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e64, m4, tu, mu
-# CHECK-NEXT:  1      4     9.00                        vmadd.vv	v4, v12, v8
+# CHECK-NEXT:  1      8     9.00                        vmadd.vv	v4, v12, v8
 # CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e64, m8, tu, mu
-# CHECK-NEXT:  1      4     17.00                       vmadd.vx	v4, a0, v8
+# CHECK-NEXT:  1      8     17.00                       vmadd.vx	v4, a0, v8
 # CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e8, mf8, tu, mu
 # CHECK-NEXT:  1      8     2.00                        vwmaccu.vv	v4, v12, v8
 # CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e8, mf4, tu, mu
diff --git a/llvm/test/tools/llvm-objcopy/ELF/dump-section.test b/llvm/test/tools/llvm-objcopy/ELF/dump-section.test
index 037ec86090e55..2dbbcc0ca568e 100644
--- a/llvm/test/tools/llvm-objcopy/ELF/dump-section.test
+++ b/llvm/test/tools/llvm-objcopy/ELF/dump-section.test
@@ -64,3 +64,7 @@ ProgramHeaders:
 # RUN: not llvm-objcopy --dump-section .text= %t /dev/null 2>&1 | FileCheck %s --check-prefix=ERR2
 
 # ERR2: error: bad format for --dump-section, expected section=file
+
+# RUN: not llvm-objcopy --dump-section .text=not_exists/text-section %t 2>&1 \
+# RUN:   | FileCheck -DMSG=%errc_ENOENT %s -DINPUT=%t --check-prefix=NO-SUCH-PATH
+# NO-SUCH-PATH: error: 'not_exists/text-section': [[MSG]]
diff --git a/llvm/test/tools/llvm-objcopy/ELF/remove-note.test b/llvm/test/tools/llvm-objcopy/ELF/remove-note.test
index f8936bf9ea731..e15f934dfe2da 100644
--- a/llvm/test/tools/llvm-objcopy/ELF/remove-note.test
+++ b/llvm/test/tools/llvm-objcopy/ELF/remove-note.test
@@ -1,198 +1,198 @@
-## Check incompatible options.
-# RUN: not llvm-objcopy --remove-note=1 --remove-section=.test - 2>&1 | FileCheck %s --check-prefix=ERR-REMSEC
-# RUN: not llvm-objcopy --remove-note=1 --add-section=.test=%s - 2>&1 | FileCheck %s --check-prefix=ERR-ADDSEC
-# RUN: not llvm-objcopy --remove-note=1 --update-section=.test=%s - 2>&1 | FileCheck %s --check-prefix=ERR-UPDSEC
-
-# ERR-REMSEC: error: cannot specify both --remove-note and --remove-section
-# ERR-ADDSEC: error: cannot specify both --remove-note and --add-section
-# ERR-UPDSEC: error: cannot specify both --remove-note and --update-section
-
-## Check invalid argument formats.
-# RUN: not llvm-objcopy --remove-note= - 2>&1 | FileCheck %s --check-prefix=ERR-NOTYPEID
-# RUN: not llvm-objcopy --remove-note=CORE/ - 2>&1 | FileCheck %s --check-prefix=ERR-NOTYPEID
-# RUN: not llvm-objcopy --remove-note=/1 - 2>&1 | FileCheck %s --check-prefix=ERR-EMPTYNAME
-# RUN: not llvm-objcopy --remove-note=CORE/1/2 - 2>&1 | FileCheck %s --check-prefix=ERR-INVNUM1
-# RUN: not llvm-objcopy --remove-note=Notanumber - 2>&1 | FileCheck %s --check-prefix=ERR-INVNUM2
-# RUN: not llvm-objcopy --remove-note=CORE/Notanumber - 2>&1 | FileCheck %s --check-prefix=ERR-INVNUM2
-
-# ERR-NOTYPEID: error: bad format for --remove-note, missing type_id
-# ERR-EMPTYNAME: error: bad format for --remove-note, note name is empty
-# ERR-INVNUM1: error: bad note type_id for --remove-note: '1/2'
-# ERR-INVNUM2: error: bad note type_id for --remove-note: 'Notanumber'
-
-## Check deleting notes:
-## * --remove-note=1 will remove note "CORE/1" and "LINUX/1",
-## * --remove-note=DUMMY/2 will not remove any notes because there are no notes with this owner,
-## * --remove-note=CORE/3 will remove "CORE/3" but preserve "LINUX/3".
-# RUN: yaml2obj --docnum=1 -D ALIGN=8 -D ELFCLASS=64 -D ENDIANNESS=LSB %s -o %t8.64.lsb
-# RUN: llvm-objcopy --remove-note=0x01 --remove-note=DUMMY/2 --remove-note=CORE/0x03 %t8.64.lsb %t8.64.lsb.o
-# RUN: llvm-readobj --segments --sections --notes %t8.64.lsb.o | \
-# RUN:   FileCheck %s -D#SIZE0=32 -D#SIZE1=64
-
-# RUN: yaml2obj --docnum=1 -D ALIGN=4 -D ELFCLASS=64 -D ENDIANNESS=MSB %s -o %t4.64.msb
-# RUN: llvm-objcopy --remove-note=0x01 --remove-note=DUMMY/0x02 --remove-note=CORE/3 %t4.64.msb %t4.64.msb.o
-# RUN: llvm-readobj --segments --sections --notes %t4.64.msb.o | \
-# RUN:   FileCheck %s -D#SIZE0=24 -D#SIZE1=48
-
-# RUN: yaml2obj --docnum=1 -D ALIGN=4 -D ELFCLASS=32 -D ENDIANNESS=LSB %s -o %t4.32.lsb
-# RUN: llvm-objcopy --remove-note=1 --remove-note=DUMMY/0x02 --remove-note=CORE/3 %t4.32.lsb %t4.32.lsb.o
-# RUN: llvm-readobj --segments --sections --notes %t4.32.lsb.o | \
-# RUN:   FileCheck %s -D#SIZE0=24 -D#SIZE1=48
-
-# CHECK:      Sections [
-# CHECK:        Section {
-# CHECK:          Name: .note0
-# CHECK-NEXT:     Type: SHT_NOTE
-# CHECK-NEXT:     Flags [
-# CHECK-NEXT:     ]
-# CHECK-NEXT:     Address:
-# CHECK-NEXT:     Offset:
-# CHECK-NEXT:     Size: [[#%d,SIZE0]]
-# CHECK:          Name: .note1
-# CHECK-NEXT:     Type: SHT_NOTE
-# CHECK-NEXT:     Flags [
-# CHECK-NEXT:     ]
-# CHECK-NEXT:     Address:
-# CHECK-NEXT:     Offset:
-# CHECK-NEXT:     Size: [[#%d,SIZE1]]
-# CHECK:          Name: .note2
-# CHECK-NEXT:     Type: SHT_NOTE
-# CHECK-NEXT:     Flags [
-# CHECK-NEXT:     ]
-# CHECK-NEXT:     Address:
-# CHECK-NEXT:     Offset:
-# CHECK-NEXT:     Size: 0
-
-# CHECK:      NoteSections [
-# CHECK-NEXT:   NoteSection {
-# CHECK-NEXT:     Name: .note0
-# CHECK-NEXT:     Offset:
-# CHECK-NEXT:     Size: 0x[[#%X,SIZE0]]
-# CHECK-NEXT:     Notes [
-# CHECK-NEXT:       {
-# CHECK-NEXT:         Owner: CORE
-# CHECK-NEXT:         Data size: 0x2
-# CHECK-NEXT:         Type: NT_ARCH
-# CHECK-NEXT:         Description data (
-# CHECK-NEXT:           0000: 0201
-# CHECK-NEXT:         )
-# CHECK-NEXT:       }
-# CHECK-NEXT:     ]
-# CHECK-NEXT:   }
-# CHECK-NEXT:   NoteSection {
-# CHECK-NEXT:     Name: .note1
-# CHECK-NEXT:     Offset:
-# CHECK-NEXT:     Size: 0x[[#%X,SIZE1]]
-# CHECK-NEXT:     Notes [
-# CHECK-NEXT:       {
-# CHECK-NEXT:         Owner: LINUX
-# CHECK-NEXT:         Data size: 0x2
-# CHECK-NEXT:         Type: Unknown (0x00000003)
-# CHECK-NEXT:         Description data (
-# CHECK-NEXT:           0000: 0301
-# CHECK-NEXT:         )
-# CHECK-NEXT:       }
-# CHECK-NEXT:       {
-# CHECK-NEXT:         Owner: CORE
-# CHECK-NEXT:         Data size: 0x2
-# CHECK-NEXT:         Type: Unknown (0x00000004)
-# CHECK-NEXT:         Description data (
-# CHECK-NEXT:           0000: 0401
-# CHECK-NEXT:         )
-# CHECK-NEXT:       }
-# CHECK-NEXT:     ]
-# CHECK-NEXT:   }
-# CHECK-NEXT:   NoteSection {
-# CHECK-NEXT:     Name: .note2
-# CHECK-NEXT:     Offset:
-# CHECK-NEXT:     Size: 0x0
-# CHECK-NEXT:     Notes [
-# CHECK-NEXT:     ]
-# CHECK-NEXT:   }
-
---- !ELF
-FileHeader:
-  Class:          ELFCLASS[[ELFCLASS]]
-  Data:           ELFDATA2[[ENDIANNESS]]
-  Type:           ET_REL
-  Machine:        EM_X86_64
-Sections:
-  - Name:         .note0
-    Type:         SHT_NOTE
-    AddressAlign: [[ALIGN]]
-    Notes:
-      - Name:   CORE
-        Type:   0x01
-        Desc:   0101
-      - Name:   CORE
-        Type:   0x02
-        Desc:   0201
-  - Name:         .note1
-    Type:         SHT_NOTE
-    AddressAlign: [[ALIGN]]
-    Notes:
-      - Name:   LINUX
-        Type:   0x03
-        Desc:   0301
-      - Name:   CORE
-        Type:   0x03
-        Desc:   0302
-      - Name:   CORE
-        Type:   0x04
-        Desc:   0401
-  - Name:         .note2
-    Type:         SHT_NOTE
-    AddressAlign: [[ALIGN]]
-    Notes:
-      - Name:   LINUX
-        Type:   0x01
-        Desc:   0102
-
-# RUN: yaml2obj --docnum=2 %s -o %t2
-# RUN: llvm-objcopy --remove-note=1 %t2 %t2o 2>&1 | FileCheck %s --check-prefix=TEST2
-# TEST2: warning: note segments are not supported
-# TEST2-NOT: note segments are not supported
-
---- !ELF
-FileHeader:
-  Class:          ELFCLASS64
-  Data:           ELFDATA2LSB
-  Type:           ET_CORE
-  Machine:        EM_X86_64
-ProgramHeaders:
-  - Type:         PT_NOTE
-    FirstSec:     .data0
-    LastSec:      .data0
-  - Type:         PT_NOTE
-    FirstSec:     .data1
-    LastSec:      .data1
-Sections:
-  - Name:         .data0
-    Type:         Fill
-    Size:         8
-  - Name:         .data1
-    Type:         Fill
-    Size:         8
-
-# RUN: yaml2obj --docnum=3 %s -o %t3
-# RUN: llvm-objcopy --remove-note=1 %t3 %t3o 2>&1 | FileCheck %s --check-prefix=TEST3
-# TEST3: warning: cannot remove note(s) from .note: sections in segments are not supported
-
---- !ELF
-FileHeader:
-  Class:          ELFCLASS64
-  Data:           ELFDATA2LSB
-  Type:           ET_EXEC
-  Machine:        EM_X86_64
-ProgramHeaders:
-  - Type:         PT_LOAD
-    FirstSec:     .note
-    LastSec:      .note
-Sections:
-  - Name:         .note
-    Type:         SHT_NOTE
-    AddressAlign: 4
-    Notes:
-      - Name:     ABC
-        Type:     1
-        Desc:     0102
+## Check incompatible options.
+# RUN: not llvm-objcopy --remove-note=1 --remove-section=.test - 2>&1 | FileCheck %s --check-prefix=ERR-REMSEC
+# RUN: not llvm-objcopy --remove-note=1 --add-section=.test=%s - 2>&1 | FileCheck %s --check-prefix=ERR-ADDSEC
+# RUN: not llvm-objcopy --remove-note=1 --update-section=.test=%s - 2>&1 | FileCheck %s --check-prefix=ERR-UPDSEC
+
+# ERR-REMSEC: error: cannot specify both --remove-note and --remove-section
+# ERR-ADDSEC: error: cannot specify both --remove-note and --add-section
+# ERR-UPDSEC: error: cannot specify both --remove-note and --update-section
+
+## Check invalid argument formats.
+# RUN: not llvm-objcopy --remove-note= - 2>&1 | FileCheck %s --check-prefix=ERR-NOTYPEID
+# RUN: not llvm-objcopy --remove-note=CORE/ - 2>&1 | FileCheck %s --check-prefix=ERR-NOTYPEID
+# RUN: not llvm-objcopy --remove-note=/1 - 2>&1 | FileCheck %s --check-prefix=ERR-EMPTYNAME
+# RUN: not llvm-objcopy --remove-note=CORE/1/2 - 2>&1 | FileCheck %s --check-prefix=ERR-INVNUM1
+# RUN: not llvm-objcopy --remove-note=Notanumber - 2>&1 | FileCheck %s --check-prefix=ERR-INVNUM2
+# RUN: not llvm-objcopy --remove-note=CORE/Notanumber - 2>&1 | FileCheck %s --check-prefix=ERR-INVNUM2
+
+# ERR-NOTYPEID: error: bad format for --remove-note, missing type_id
+# ERR-EMPTYNAME: error: bad format for --remove-note, note name is empty
+# ERR-INVNUM1: error: bad note type_id for --remove-note: '1/2'
+# ERR-INVNUM2: error: bad note type_id for --remove-note: 'Notanumber'
+
+## Check deleting notes:
+## * --remove-note=1 will remove note "CORE/1" and "LINUX/1",
+## * --remove-note=DUMMY/2 will not remove any notes because there are no notes with this owner,
+## * --remove-note=CORE/3 will remove "CORE/3" but preserve "LINUX/3".
+# RUN: yaml2obj --docnum=1 -D ALIGN=8 -D ELFCLASS=64 -D ENDIANNESS=LSB %s -o %t8.64.lsb
+# RUN: llvm-objcopy --remove-note=0x01 --remove-note=DUMMY/2 --remove-note=CORE/0x03 %t8.64.lsb %t8.64.lsb.o
+# RUN: llvm-readobj --segments --sections --notes %t8.64.lsb.o | \
+# RUN:   FileCheck %s -D#SIZE0=32 -D#SIZE1=64
+
+# RUN: yaml2obj --docnum=1 -D ALIGN=4 -D ELFCLASS=64 -D ENDIANNESS=MSB %s -o %t4.64.msb
+# RUN: llvm-objcopy --remove-note=0x01 --remove-note=DUMMY/0x02 --remove-note=CORE/3 %t4.64.msb %t4.64.msb.o
+# RUN: llvm-readobj --segments --sections --notes %t4.64.msb.o | \
+# RUN:   FileCheck %s -D#SIZE0=24 -D#SIZE1=48
+
+# RUN: yaml2obj --docnum=1 -D ALIGN=4 -D ELFCLASS=32 -D ENDIANNESS=LSB %s -o %t4.32.lsb
+# RUN: llvm-objcopy --remove-note=1 --remove-note=DUMMY/0x02 --remove-note=CORE/3 %t4.32.lsb %t4.32.lsb.o
+# RUN: llvm-readobj --segments --sections --notes %t4.32.lsb.o | \
+# RUN:   FileCheck %s -D#SIZE0=24 -D#SIZE1=48
+
+# CHECK:      Sections [
+# CHECK:        Section {
+# CHECK:          Name: .note0
+# CHECK-NEXT:     Type: SHT_NOTE
+# CHECK-NEXT:     Flags [
+# CHECK-NEXT:     ]
+# CHECK-NEXT:     Address:
+# CHECK-NEXT:     Offset:
+# CHECK-NEXT:     Size: [[#%d,SIZE0]]
+# CHECK:          Name: .note1
+# CHECK-NEXT:     Type: SHT_NOTE
+# CHECK-NEXT:     Flags [
+# CHECK-NEXT:     ]
+# CHECK-NEXT:     Address:
+# CHECK-NEXT:     Offset:
+# CHECK-NEXT:     Size: [[#%d,SIZE1]]
+# CHECK:          Name: .note2
+# CHECK-NEXT:     Type: SHT_NOTE
+# CHECK-NEXT:     Flags [
+# CHECK-NEXT:     ]
+# CHECK-NEXT:     Address:
+# CHECK-NEXT:     Offset:
+# CHECK-NEXT:     Size: 0
+
+# CHECK:      NoteSections [
+# CHECK-NEXT:   NoteSection {
+# CHECK-NEXT:     Name: .note0
+# CHECK-NEXT:     Offset:
+# CHECK-NEXT:     Size: 0x[[#%X,SIZE0]]
+# CHECK-NEXT:     Notes [
+# CHECK-NEXT:       {
+# CHECK-NEXT:         Owner: CORE
+# CHECK-NEXT:         Data size: 0x2
+# CHECK-NEXT:         Type: NT_ARCH
+# CHECK-NEXT:         Description data (
+# CHECK-NEXT:           0000: 0201
+# CHECK-NEXT:         )
+# CHECK-NEXT:       }
+# CHECK-NEXT:     ]
+# CHECK-NEXT:   }
+# CHECK-NEXT:   NoteSection {
+# CHECK-NEXT:     Name: .note1
+# CHECK-NEXT:     Offset:
+# CHECK-NEXT:     Size: 0x[[#%X,SIZE1]]
+# CHECK-NEXT:     Notes [
+# CHECK-NEXT:       {
+# CHECK-NEXT:         Owner: LINUX
+# CHECK-NEXT:         Data size: 0x2
+# CHECK-NEXT:         Type: Unknown (0x00000003)
+# CHECK-NEXT:         Description data (
+# CHECK-NEXT:           0000: 0301
+# CHECK-NEXT:         )
+# CHECK-NEXT:       }
+# CHECK-NEXT:       {
+# CHECK-NEXT:         Owner: CORE
+# CHECK-NEXT:         Data size: 0x2
+# CHECK-NEXT:         Type: Unknown (0x00000004)
+# CHECK-NEXT:         Description data (
+# CHECK-NEXT:           0000: 0401
+# CHECK-NEXT:         )
+# CHECK-NEXT:       }
+# CHECK-NEXT:     ]
+# CHECK-NEXT:   }
+# CHECK-NEXT:   NoteSection {
+# CHECK-NEXT:     Name: .note2
+# CHECK-NEXT:     Offset:
+# CHECK-NEXT:     Size: 0x0
+# CHECK-NEXT:     Notes [
+# CHECK-NEXT:     ]
+# CHECK-NEXT:   }
+
+--- !ELF
+FileHeader:
+  Class:          ELFCLASS[[ELFCLASS]]
+  Data:           ELFDATA2[[ENDIANNESS]]
+  Type:           ET_REL
+  Machine:        EM_X86_64
+Sections:
+  - Name:         .note0
+    Type:         SHT_NOTE
+    AddressAlign: [[ALIGN]]
+    Notes:
+      - Name:   CORE
+        Type:   0x01
+        Desc:   0101
+      - Name:   CORE
+        Type:   0x02
+        Desc:   0201
+  - Name:         .note1
+    Type:         SHT_NOTE
+    AddressAlign: [[ALIGN]]
+    Notes:
+      - Name:   LINUX
+        Type:   0x03
+        Desc:   0301
+      - Name:   CORE
+        Type:   0x03
+        Desc:   0302
+      - Name:   CORE
+        Type:   0x04
+        Desc:   0401
+  - Name:         .note2
+    Type:         SHT_NOTE
+    AddressAlign: [[ALIGN]]
+    Notes:
+      - Name:   LINUX
+        Type:   0x01
+        Desc:   0102
+
+# RUN: yaml2obj --docnum=2 %s -o %t2
+# RUN: llvm-objcopy --remove-note=1 %t2 %t2o 2>&1 | FileCheck %s --check-prefix=TEST2
+# TEST2: warning: note segments are not supported
+# TEST2-NOT: note segments are not supported
+
+--- !ELF
+FileHeader:
+  Class:          ELFCLASS64
+  Data:           ELFDATA2LSB
+  Type:           ET_CORE
+  Machine:        EM_X86_64
+ProgramHeaders:
+  - Type:         PT_NOTE
+    FirstSec:     .data0
+    LastSec:      .data0
+  - Type:         PT_NOTE
+    FirstSec:     .data1
+    LastSec:      .data1
+Sections:
+  - Name:         .data0
+    Type:         Fill
+    Size:         8
+  - Name:         .data1
+    Type:         Fill
+    Size:         8
+
+# RUN: yaml2obj --docnum=3 %s -o %t3
+# RUN: llvm-objcopy --remove-note=1 %t3 %t3o 2>&1 | FileCheck %s --check-prefix=TEST3
+# TEST3: warning: cannot remove note(s) from .note: sections in segments are not supported
+
+--- !ELF
+FileHeader:
+  Class:          ELFCLASS64
+  Data:           ELFDATA2LSB
+  Type:           ET_EXEC
+  Machine:        EM_X86_64
+ProgramHeaders:
+  - Type:         PT_LOAD
+    FirstSec:     .note
+    LastSec:      .note
+Sections:
+  - Name:         .note
+    Type:         SHT_NOTE
+    AddressAlign: 4
+    Notes:
+      - Name:     ABC
+        Type:     1
+        Desc:     0102
diff --git a/llvm/test/tools/llvm-objcopy/MachO/dump-section.test b/llvm/test/tools/llvm-objcopy/MachO/dump-section.test
index 9a1227cdbbda1..d54a50b557bb7 100644
--- a/llvm/test/tools/llvm-objcopy/MachO/dump-section.test
+++ b/llvm/test/tools/llvm-objcopy/MachO/dump-section.test
@@ -21,6 +21,10 @@
 # RUN:   | FileCheck %s -DINPUT=%t --check-prefix=NO-SUCH-SECTION
 # NO-SUCH-SECTION: error: '[[INPUT]]': section '__TEXT,__foo' not found
 
+# RUN: not llvm-objcopy --dump-section __TEXT,__text=not_exists/text-section %t 2>&1 \
+# RUN:   | FileCheck -DMSG=%errc_ENOENT %s -DINPUT=%t --check-prefix=NO-SUCH-PATH
+# NO-SUCH-PATH: error: 'not_exists/text-section': [[MSG]]
+
 --- !mach-o
 FileHeader:
   magic:           0xFEEDFACF
diff --git a/llvm/test/tools/llvm-objcopy/wasm/dump-section.test b/llvm/test/tools/llvm-objcopy/wasm/dump-section.test
index 983a581e03fe2..2d1533f06df10 100644
--- a/llvm/test/tools/llvm-objcopy/wasm/dump-section.test
+++ b/llvm/test/tools/llvm-objcopy/wasm/dump-section.test
@@ -28,6 +28,10 @@
 
 # REMOVED-NOT: producers
 
+# RUN: not llvm-objcopy --dump-section producers=not_exists/text-section %t 2>&1 \
+# RUN:   | FileCheck -DMSG=%errc_ENOENT %s -DINPUT=%t --check-prefix=NO-SUCH-PATH
+# NO-SUCH-PATH: error: 'not_exists/text-section': [[MSG]]
+
 --- !WASM
 FileHeader:
   Version: 0x00000001
diff --git a/llvm/test/tools/llvm-objdump/ELF/AMDGPU/kd-gfx11.s b/llvm/test/tools/llvm-objdump/ELF/AMDGPU/kd-gfx11.s
index 750809128189f..3cd7a0503e301 100644
--- a/llvm/test/tools/llvm-objdump/ELF/AMDGPU/kd-gfx11.s
+++ b/llvm/test/tools/llvm-objdump/ELF/AMDGPU/kd-gfx11.s
@@ -13,7 +13,7 @@
 ; CHECK-NEXT: .amdhsa_private_segment_fixed_size 0
 ; CHECK-NEXT: .amdhsa_kernarg_size 0
 ; CHECK-NEXT: ; SHARED_VGPR_COUNT 0
-; CHECK-NEXT: ; INST_PREF_SIZE 0
+; CHECK-NEXT: .amdhsa_inst_pref_size 0
 ; CHECK-NEXT: ; TRAP_ON_START 0
 ; CHECK-NEXT: ; TRAP_ON_END 0
 ; CHECK-NEXT: ; IMAGE_OP 0
@@ -70,7 +70,7 @@
 ; CHECK-NEXT: .amdhsa_private_segment_fixed_size 0
 ; CHECK-NEXT: .amdhsa_kernarg_size 0
 ; CHECK-NEXT: .amdhsa_shared_vgpr_count 0
-; CHECK-NEXT: ; INST_PREF_SIZE 0
+; CHECK-NEXT: .amdhsa_inst_pref_size 0
 ; CHECK-NEXT: ; TRAP_ON_START 0
 ; CHECK-NEXT: ; TRAP_ON_END 0
 ; CHECK-NEXT: ; IMAGE_OP 0
@@ -114,6 +114,7 @@
   .amdhsa_next_free_vgpr 32
   .amdhsa_next_free_sgpr 32
   .amdhsa_shared_vgpr_count 0
+  .amdhsa_inst_pref_size 0
 .end_amdhsa_kernel
 
 ;--- 3.s
@@ -127,7 +128,7 @@
 ; CHECK-NEXT: .amdhsa_private_segment_fixed_size 0
 ; CHECK-NEXT: .amdhsa_kernarg_size 0
 ; CHECK-NEXT: .amdhsa_shared_vgpr_count 1
-; CHECK-NEXT: ; INST_PREF_SIZE 0
+; CHECK-NEXT: .amdhsa_inst_pref_size 63
 ; CHECK-NEXT: ; TRAP_ON_START 0
 ; CHECK-NEXT: ; TRAP_ON_END 0
 ; CHECK-NEXT: ; IMAGE_OP 0
@@ -171,6 +172,7 @@
   .amdhsa_next_free_vgpr 32
   .amdhsa_next_free_sgpr 32
   .amdhsa_shared_vgpr_count 1
+  .amdhsa_inst_pref_size 63
 .end_amdhsa_kernel
 
 ;--- 4.s
@@ -184,7 +186,7 @@
 ; CHECK-NEXT: .amdhsa_private_segment_fixed_size 0
 ; CHECK-NEXT: .amdhsa_kernarg_size 0
 ; CHECK-NEXT: .amdhsa_shared_vgpr_count 1
-; CHECK-NEXT: ; INST_PREF_SIZE 0
+; CHECK-NEXT: .amdhsa_inst_pref_size 63
 ; CHECK-NEXT: ; TRAP_ON_START 0
 ; CHECK-NEXT: ; TRAP_ON_END 0
 ; CHECK-NEXT: ; IMAGE_OP 0
@@ -228,5 +230,6 @@
   .amdhsa_next_free_vgpr 32
   .amdhsa_next_free_sgpr 32
   .amdhsa_shared_vgpr_count 1
+  .amdhsa_inst_pref_size 63
   .amdhsa_wavefront_size32 0
 .end_amdhsa_kernel
diff --git a/llvm/test/tools/llvm-objdump/ELF/AMDGPU/kd-gfx12.s b/llvm/test/tools/llvm-objdump/ELF/AMDGPU/kd-gfx12.s
index c644e15efc8d7..ed2b87d9885c6 100644
--- a/llvm/test/tools/llvm-objdump/ELF/AMDGPU/kd-gfx12.s
+++ b/llvm/test/tools/llvm-objdump/ELF/AMDGPU/kd-gfx12.s
@@ -12,7 +12,7 @@
 ; CHECK-NEXT: .amdhsa_group_segment_fixed_size 0
 ; CHECK-NEXT: .amdhsa_private_segment_fixed_size 0
 ; CHECK-NEXT: .amdhsa_kernarg_size 0
-; CHECK-NEXT: ; INST_PREF_SIZE 0
+; CHECK-NEXT: .amdhsa_inst_pref_size 0
 ; CHECK-NEXT: ; GLG_EN 0
 ; CHECK-NEXT: ; IMAGE_OP 0
 ; CHECK-NEXT: .amdhsa_next_free_vgpr 32
@@ -66,7 +66,7 @@
 ; CHECK-NEXT: .amdhsa_group_segment_fixed_size 0
 ; CHECK-NEXT: .amdhsa_private_segment_fixed_size 0
 ; CHECK-NEXT: .amdhsa_kernarg_size 0
-; CHECK-NEXT: ; INST_PREF_SIZE 0
+; CHECK-NEXT: .amdhsa_inst_pref_size 255
 ; CHECK-NEXT: ; GLG_EN 0
 ; CHECK-NEXT: ; IMAGE_OP 0
 ; CHECK-NEXT: .amdhsa_next_free_vgpr 32
@@ -108,4 +108,5 @@
   .amdhsa_next_free_vgpr 32
   .amdhsa_next_free_sgpr 32
   .amdhsa_wavefront_size32 0
+  .amdhsa_inst_pref_size 255
 .end_amdhsa_kernel
diff --git a/llvm/test/tools/llvm-objdump/ELF/Hexagon/plt.test b/llvm/test/tools/llvm-objdump/ELF/Hexagon/plt.test
new file mode 100644
index 0000000000000..5d2b8576b9b84
--- /dev/null
+++ b/llvm/test/tools/llvm-objdump/ELF/Hexagon/plt.test
@@ -0,0 +1,42 @@
+# RUN: yaml2obj %s | llvm-objdump -d - | FileCheck %s
+
+#      CHECK: 00000310 <printf@plt>:
+# CHECK-NEXT:      310:       36 40 00 00     00004036 {      immext(#0xd80)
+# CHECK-NEXT:      314:       0e de 49 6a     6a49de0e        r14 = add(pc,##0xdbc) }
+# CHECK-NEXT:      318:       1c c0 8e 91     918ec01c {      r28 = memw(r14+#0x0) }
+# CHECK-NEXT:      31c:       00 c0 9c 52     529cc000 {      jumpr r28 }
+
+--- !ELF
+FileHeader:
+  Class:           ELFCLASS32
+  Data:            ELFDATA2LSB
+  Type:            ET_DYN
+  Machine:         EM_HEXAGON
+Sections:
+  - Name:            .rela.plt
+    Type:            SHT_RELA
+    Flags:           [ SHF_ALLOC ]
+    Info:            .got.plt
+    Relocations:
+      - Offset:          0x10CC
+        Symbol:          printf
+        Type:            R_HEX_JMP_SLOT
+  - Name:            .plt
+    Type:            SHT_PROGBITS
+    Flags:           [ SHF_ALLOC, SHF_EXECINSTR ]
+    Address:         0x2B0
+    Content:         384000001CC0496A0E429CE24F409C913CC09C910E420E8C00C09C520000000000000000000000000000000000000000374000000ED0496A1CC08E9100C09C52374000000ECA496A1CC08E9100C09C52374000000EC4496A1CC08E9100C09C52364000000EDE496A1CC08E9100C09C52
+  - Name:            .text
+    Type:            SHT_PROGBITS
+    Flags:           [ SHF_ALLOC, SHF_EXECINSTR ]
+    Address:         0x320
+    Content:         0240096AC97FFF0F01C6007802C221F3FF7FFF0F80C7007800C002F300C0809100C0007500C05F5300C0096ADAFFFF5901C09DA082FFFEBF00C0423C0140000000D4496AD6FFFF5B00C000781EC01E96
+  - Name:            .got.plt
+    Type:            SHT_PROGBITS
+    Flags:           [ SHF_WRITE, SHF_ALLOC ]
+    Address:         0x10B0
+    Content:         00000000000000000000000000000000B0020000B0020000B0020000B0020000
+Symbols:
+  - Name:            printf
+    Binding:         STB_GLOBAL
+...
diff --git a/llvm/test/tools/llvm-readobj/ELF/RISCV/note-gnu-property.yaml b/llvm/test/tools/llvm-readobj/ELF/RISCV/note-gnu-property.yaml
new file mode 100644
index 0000000000000..f87ed762038c1
--- /dev/null
+++ b/llvm/test/tools/llvm-readobj/ELF/RISCV/note-gnu-property.yaml
@@ -0,0 +1,45 @@
+# RUN: yaml2obj %s -DBITS=32 -DPR_PADDING= -o %t1
+# RUN: llvm-readelf --notes %t1 | FileCheck %s --check-prefix=GNU
+# RUN: llvm-readobj --notes %t1 | FileCheck %s --check-prefix=LLVM
+
+# RUN: yaml2obj %s -DBITS=64 -DPR_PADDING=00000000 -o %t2
+# RUN: llvm-readelf --notes %t2 | FileCheck %s --check-prefix=GNU
+# RUN: llvm-readobj --notes %t2 | FileCheck %s --check-prefix=LLVM
+
+# GNU: Displaying notes found in: .note.gnu.property
+# GNU-NEXT:   Owner                 Data size 	Description
+# GNU-NEXT:   GNU                   0x{{([0-9a-z]{8})}}	NT_GNU_PROPERTY_TYPE_0 (property note)
+# GNU-NEXT:     Properties:    RISC-V feature: ZICFILP-unlabeled, ZICFISS, ZICFILP-func-sig
+
+# LLVM:      NoteSections [
+# LLVM-NEXT:   NoteSection {
+# LLVM-NEXT:     Name: .note.gnu.property
+# LLVM-NEXT:     Offset:
+# LLVM-NEXT:     Size:
+# LLVM-NEXT:     Notes [
+# LLVM-NEXT:       {
+# LLVM-NEXT:         Owner: GNU
+# LLVM-NEXT:         Data size:
+# LLVM-NEXT:         Type: NT_GNU_PROPERTY_TYPE_0 (property note)
+# LLVM-NEXT:         Property [
+# LLVM-NEXT:           RISC-V feature: ZICFILP-unlabeled, ZICFISS, ZICFILP-func-sig
+# LLVM-NEXT:         ]
+# LLVM-NEXT:       }
+# LLVM-NEXT:     ]
+# LLVM-NEXT:   }
+# LLVM-NEXT: ]
+
+--- !ELF
+FileHeader:
+  Class:   ELFCLASS[[BITS]]
+  Data:    ELFDATA2LSB
+  Type:    ET_REL
+  Machine: EM_RISCV
+Sections:
+  - Name:  .note.gnu.property
+    Type:  SHT_NOTE
+    Flags: [ SHF_ALLOC ]
+    Notes:
+      - Name: 'GNU'
+        Desc: '000000c00400000007000000[[PR_PADDING]]'
+        Type: 5 # NT_GNU_PROPERTY_TYPE_0
diff --git a/llvm/test/tools/llvm-readobj/ELF/bb-addr-map-relocatable.test b/llvm/test/tools/llvm-readobj/ELF/bb-addr-map-relocatable.test
index e7f78491a9473..325a956e78591 100644
--- a/llvm/test/tools/llvm-readobj/ELF/bb-addr-map-relocatable.test
+++ b/llvm/test/tools/llvm-readobj/ELF/bb-addr-map-relocatable.test
@@ -1,7 +1,8 @@
 ## This test checks how we handle the --bb-addr-map option on relocatable
 ## object files.
 
-# RUN: yaml2obj %s -o %t1.o
+# RUN: yaml2obj -D RELOCATION_SECTION_NAME=.rela.llvm_bb_addr_map \
+# RUN:   -D RELOCATION_SECTION_TYPE=SHT_RELA %s -o %t1.o
 # RUN: llvm-readobj %t1.o --bb-addr-map | FileCheck %s
 
 # CHECK:      BBAddrMap [
@@ -77,8 +78,8 @@ Sections:
                AddressOffset:   0x0
                Size:            0x11
                Metadata:        0x8
-  - Name:  .rela.llvm_bb_addr_map
-    Type:  SHT_RELA
+  - Name:  [[RELOCATION_SECTION_NAME]]
+    Type:  [[RELOCATION_SECTION_TYPE]]
     Flags: [ SHF_INFO_LINK ]
     Link:  .symtab
     Info:  .llvm_bb_addr_map
@@ -228,3 +229,55 @@ Sections:
 # ET-DYN-NO-WARNING:     ]
 # ET-DYN-NO-WARNING:   }
 # ET-DYN-NO-WARNING: ]
+
+## Check that we can correctly decode a BBAddrMap in a reloctable object file
+## with CREL relocations.
+
+# RUN: yaml2obj -D RELOCATION_SECTION_NAME=.crel.llvm_bb_addr_map \
+# RUN:   -D RELOCATION_SECTION_TYPE=SHT_CREL %s -o %t6.o
+# RUN: llvm-readobj %t6.o --bb-addr-map | FileCheck %s --check-prefix=CREL
+
+# CREL:      BBAddrMap [
+# CREL-NEXT:   Function {
+# CREL-NEXT:     At: 0x0
+# CREL-NEXT:     Name: <?>
+# CREL-NEXT:     BB Ranges [
+# CREL-NEXT:       {
+# CREL-NEXT:         Base Address: 0x0
+# CREL-NEXT:         BB Entries [
+# CREL-NEXT:           {
+# CREL-NEXT:             ID: 0
+# CREL-NEXT:             Offset: 0x0
+# CREL-NEXT:             Size: 0xF
+# CREL-NEXT:             HasReturn: Yes
+# CREL-NEXT:             HasTailCall: No
+# CREL-NEXT:             IsEHPad: No
+# CREL-NEXT:             CanFallThrough: No
+# CREL-NEXT:             HasIndirectBranch: No
+# CREL-NEXT:           }
+# CREL-NEXT:         ]
+# CREL-NEXT:       }
+# CREL-NEXT:     ]
+# CREL-NEXT:   }
+# CREL-NEXT:   Function {
+# CREL-NEXT:     At: 0x10
+# CREL-NEXT:     Name: <?>
+# CREL-NEXT:     BB Ranges [
+# CREL-NEXT:       {
+# CREL-NEXT:         Base Address: 0x10
+# CREL-NEXT:         BB Entries [
+# CREL-NEXT:           {
+# CREL-NEXT:             ID: 0
+# CREL-NEXT:             Offset: 0x0
+# CREL-NEXT:             Size: 0x11
+# CREL-NEXT:             HasReturn: No
+# CREL-NEXT:             HasTailCall: No
+# CREL-NEXT:             IsEHPad: No
+# CREL-NEXT:             CanFallThrough: Yes
+# CREL-NEXT:             HasIndirectBranch: No
+# CREL-NEXT:           }
+# CREL-NEXT:         ]
+# CREL-NEXT:       }
+# CREL-NEXT:     ]
+# CREL-NEXT:   }
+# CREL-NEXT: ]
diff --git a/llvm/test/tools/llvm-readobj/XCOFF/symbols.test b/llvm/test/tools/llvm-readobj/XCOFF/symbols.test
index 71347a85f1ba5..64e0feb966ee1 100644
--- a/llvm/test/tools/llvm-readobj/XCOFF/symbols.test
+++ b/llvm/test/tools/llvm-readobj/XCOFF/symbols.test
@@ -163,6 +163,73 @@ Symbols:
         StabInfoIndex:          44
         StabSectNum:            55
 
+  - Name:               bstat
+    Value:              0x0
+    Section:            N_DEBUG
+    Type:               0x00
+    StorageClass:       C_BSTAT
+
+  - Name:               cfun
+    Value:              0x0
+    Section:            N_DEBUG
+    Type:               0x00
+    StorageClass:       C_FUN
+
+  - Name:               stsym
+    Value:              0x05
+    Section:            N_DEBUG
+    Type:               0x00
+    StorageClass:       C_STSYM
+
+  - Name:               bincl
+    Value:              0x06
+    Section:            .text
+    Type:               0x00
+    StorageClass:       C_BINCL
+
+  - Name:               eincl
+    Value:              0x06
+    Section:            .text
+    Type:               0x00
+    StorageClass:       C_EINCL
+
+  - Name:               lsym
+    Value:              0x07
+    Section:            N_DEBUG
+    Type:               0x00
+    StorageClass:       C_LSYM
+
+  - Name:               psym
+    Value:              0x07
+    Section:            N_DEBUG
+    Type:               0x00
+    StorageClass:       C_PSYM
+
+  - Name:               rsym
+    Value:              0x08
+    Section:            N_DEBUG
+    Type:               0x00
+    StorageClass:       C_RSYM
+
+  - Name:               rpsym
+    Value:              0x08
+    Section:            N_DEBUG
+    Type:               0x00
+    StorageClass:       C_RPSYM
+
+  - Name:               ecoml
+    Value:              0x09
+    Section:            .text
+    Type:               0x00
+    StorageClass:       C_ECOML
+
+  - Name:               cinfo
+    Value:              0x02
+    Section:            .text
+    Type:               0x00
+    StorageClass:       C_INFO
+
+
 # SYMBOL32:      Symbols [
 # SYMBOL32-NEXT:   Symbol {
 # SYMBOL32-NEXT:     Index: 0
@@ -368,4 +435,103 @@ Symbols:
 # SYMBOL32-NEXT:       StabSectNum: 0x37
 # SYMBOL32-NEXT:     }
 # SYMBOL32-NEXT:   }
+# SYMBOL32-NEXT:   Symbol {
+# SYMBOL32-NEXT:     Index: 25
+# SYMBOL32-NEXT:     Name: Unimplemented Debug Name
+# SYMBOL32-NEXT:     Value (SymbolTableIndex): 0x0
+# SYMBOL32-NEXT:     Section: N_DEBUG
+# SYMBOL32-NEXT:     Type: 0x0
+# SYMBOL32-NEXT:     StorageClass: C_BSTAT (0x8F)
+# SYMBOL32-NEXT:     NumberOfAuxEntries: 0
+# SYMBOL32-NEXT:   }
+# SYMBOL32-NEXT:   Symbol {
+# SYMBOL32-NEXT:     Index: 26
+# SYMBOL32-NEXT:     Name: Unimplemented Debug Name
+# SYMBOL32-NEXT:     Value (OffsetInCSect): 0x0
+# SYMBOL32-NEXT:     Section: N_DEBUG
+# SYMBOL32-NEXT:     Type: 0x0
+# SYMBOL32-NEXT:     StorageClass: C_FUN (0x8E)
+# SYMBOL32-NEXT:     NumberOfAuxEntries: 0
+# SYMBOL32-NEXT:   }
+# SYMBOL32-NEXT:   Symbol {
+# SYMBOL32-NEXT:     Index: 27
+# SYMBOL32-NEXT:     Name: Unimplemented Debug Name
+# SYMBOL32-NEXT:     Value (OffsetInCSect): 0x5
+# SYMBOL32-NEXT:     Section: N_DEBUG
+# SYMBOL32-NEXT:     Type: 0x0
+# SYMBOL32-NEXT:     StorageClass: C_STSYM (0x85)
+# SYMBOL32-NEXT:     NumberOfAuxEntries: 0
+# SYMBOL32-NEXT:   }
+# SYMBOL32-NEXT:   Symbol {
+# SYMBOL32-NEXT:     Index: 28
+# SYMBOL32-NEXT:     Name: bincl
+# SYMBOL32-NEXT:     Value (OffsetInFile): 0x6
+# SYMBOL32-NEXT:     Section: .text
+# SYMBOL32-NEXT:     Type: 0x0
+# SYMBOL32-NEXT:     StorageClass: C_BINCL (0x6C)
+# SYMBOL32-NEXT:     NumberOfAuxEntries: 0
+# SYMBOL32-NEXT:   }
+# SYMBOL32-NEXT:   Symbol {
+# SYMBOL32-NEXT:     Index: 29
+# SYMBOL32-NEXT:     Name: eincl
+# SYMBOL32-NEXT:     Value (OffsetInFile): 0x6
+# SYMBOL32-NEXT:     Section: .text
+# SYMBOL32-NEXT:     Type: 0x0
+# SYMBOL32-NEXT:     StorageClass: C_EINCL (0x6D)
+# SYMBOL32-NEXT:     NumberOfAuxEntries: 0
+# SYMBOL32-NEXT:   }
+# SYMBOL32-NEXT:   Symbol {
+# SYMBOL32-NEXT:     Index: 30
+# SYMBOL32-NEXT:     Name: Unimplemented Debug Name
+# SYMBOL32-NEXT:     Value (OffsetRelToStackFrame): 0x7
+# SYMBOL32-NEXT:     Section: N_DEBUG
+# SYMBOL32-NEXT:     Type: 0x0
+# SYMBOL32-NEXT:     StorageClass: C_LSYM (0x81)
+# SYMBOL32-NEXT:     NumberOfAuxEntries: 0
+# SYMBOL32-NEXT:   }
+# SYMBOL32-NEXT:   Symbol {
+# SYMBOL32-NEXT:     Index: 31
+# SYMBOL32-NEXT:     Name: Unimplemented Debug Name
+# SYMBOL32-NEXT:     Value (OffsetRelToStackFrame): 0x7
+# SYMBOL32-NEXT:     Section: N_DEBUG
+# SYMBOL32-NEXT:     Type: 0x0
+# SYMBOL32-NEXT:     StorageClass: C_PSYM (0x82)
+# SYMBOL32-NEXT:     NumberOfAuxEntries: 0
+# SYMBOL32-NEXT:   }
+# SYMBOL32-NEXT:   Symbol {
+# SYMBOL32-NEXT:     Index: 32
+# SYMBOL32-NEXT:     Name: Unimplemented Debug Name
+# SYMBOL32-NEXT:     Value (RegisterNumber): 0x8
+# SYMBOL32-NEXT:     Section: N_DEBUG
+# SYMBOL32-NEXT:     Type: 0x0
+# SYMBOL32-NEXT:     StorageClass: C_RSYM (0x83)
+# SYMBOL32-NEXT:     NumberOfAuxEntries: 0
+# SYMBOL32-NEXT:   }
+# SYMBOL32-NEXT:   Symbol {
+# SYMBOL32-NEXT:     Index: 33
+# SYMBOL32-NEXT:     Name: Unimplemented Debug Name
+# SYMBOL32-NEXT:     Value (RegisterNumber): 0x8
+# SYMBOL32-NEXT:     Section: N_DEBUG
+# SYMBOL32-NEXT:     Type: 0x0
+# SYMBOL32-NEXT:     StorageClass: C_RPSYM (0x84)
+# SYMBOL32-NEXT:     NumberOfAuxEntries: 0
+# SYMBOL32-NEXT:   }
+# SYMBOL32-NEXT:   Symbol {
+# SYMBOL32-NEXT:     Index: 34
+# SYMBOL32-NEXT:     Name: Unimplemented Debug Name
+# SYMBOL32-NEXT:     Value (OffsetInCommBlock): 0x9
+# SYMBOL32-NEXT:     Section: .text
+# SYMBOL32-NEXT:     Type: 0x0
+# SYMBOL32-NEXT:     StorageClass: C_ECOML (0x88)
+# SYMBOL32-NEXT:     NumberOfAuxEntries: 0
+# SYMBOL32-NEXT:   }
+# SYMBOL32-NEXT:   Symbol {
+# SYMBOL32-NEXT:     Index: 35
+# SYMBOL32-NEXT:     Name: cinfo
+# SYMBOL32-NEXT:     Value (OffsetInCommentSection): 0x2
+# SYMBOL32-NEXT:     Section: .text
+# SYMBOL32-NEXT:     Type: 0x0
+# SYMBOL32-NEXT:     StorageClass: C_INFO (0x6E)
+# SYMBOL32-NEXT:     NumberOfAuxEntries: 0
+# SYMBOL32-NEXT:   }
 # SYMBOL32-NEXT: ]
diff --git a/llvm/test/tools/llvm-readobj/XCOFF/symbols64.test b/llvm/test/tools/llvm-readobj/XCOFF/symbols64.test
index 2b9edb3829af8..462ac5f4a3161 100644
--- a/llvm/test/tools/llvm-readobj/XCOFF/symbols64.test
+++ b/llvm/test/tools/llvm-readobj/XCOFF/symbols64.test
@@ -138,6 +138,73 @@ Symbols:
       - Type:    AUX_SYM
         LineNum: 3
 
+  - Name:               bstat
+    Value:              0x0
+    Section:            N_DEBUG
+    Type:               0x00
+    StorageClass:       C_BSTAT
+
+  - Name:               cfun
+    Value:              0x0
+    Section:            N_DEBUG
+    Type:               0x00
+    StorageClass:       C_FUN
+
+  - Name:               stsym
+    Value:              0x05
+    Section:            N_DEBUG
+    Type:               0x00
+    StorageClass:       C_STSYM
+
+  - Name:               bincl
+    Value:              0x06
+    Section:            .text
+    Type:               0x00
+    StorageClass:       C_BINCL
+
+  - Name:               eincl
+    Value:              0x06
+    Section:            .text
+    Type:               0x00
+    StorageClass:       C_EINCL
+
+  - Name:               lsym
+    Value:              0x07
+    Section:            N_DEBUG
+    Type:               0x00
+    StorageClass:       C_LSYM
+
+  - Name:               psym
+    Value:              0x07
+    Section:            N_DEBUG
+    Type:               0x00
+    StorageClass:       C_PSYM
+
+  - Name:               rsym
+    Value:              0x08
+    Section:            N_DEBUG
+    Type:               0x00
+    StorageClass:       C_RSYM
+
+  - Name:               rpsym
+    Value:              0x08
+    Section:            N_DEBUG
+    Type:               0x00
+    StorageClass:       C_RPSYM
+
+  - Name:               ecoml
+    Value:              0x09
+    Section:            .text
+    Type:               0x00
+    StorageClass:       C_ECOML
+
+  - Name:               cinfo
+    Value:              0x02
+    Section:            .text
+    Type:               0x00
+    StorageClass:       C_INFO
+
+
 # SYMBOL64:      Symbols [
 # SYMBOL64-NEXT:   Symbol {
 # SYMBOL64-NEXT:     Index: 0
@@ -326,4 +393,103 @@ Symbols:
 # SYMBOL64-NEXT:       Auxiliary Type: AUX_SYM (0xFD)
 # SYMBOL64-NEXT:     }
 # SYMBOL64-NEXT:   }
+# SYMBOL64-NEXT:   Symbol {
+# SYMBOL64-NEXT:     Index: 23
+# SYMBOL64-NEXT:     Name: Unimplemented Debug Name
+# SYMBOL64-NEXT:     Value (SymbolTableIndex): 0x0
+# SYMBOL64-NEXT:     Section: N_DEBUG
+# SYMBOL64-NEXT:     Type: 0x0
+# SYMBOL64-NEXT:     StorageClass: C_BSTAT (0x8F)
+# SYMBOL64-NEXT:     NumberOfAuxEntries: 0
+# SYMBOL64-NEXT:   }
+# SYMBOL64-NEXT:   Symbol {
+# SYMBOL64-NEXT:     Index: 24
+# SYMBOL64-NEXT:     Name: Unimplemented Debug Name
+# SYMBOL64-NEXT:     Value (OffsetInCSect): 0x0
+# SYMBOL64-NEXT:     Section: N_DEBUG
+# SYMBOL64-NEXT:     Type: 0x0
+# SYMBOL64-NEXT:     StorageClass: C_FUN (0x8E)
+# SYMBOL64-NEXT:     NumberOfAuxEntries: 0
+# SYMBOL64-NEXT:   }
+# SYMBOL64-NEXT:   Symbol {
+# SYMBOL64-NEXT:     Index: 25
+# SYMBOL64-NEXT:     Name: Unimplemented Debug Name
+# SYMBOL64-NEXT:     Value (OffsetInCSect): 0x5
+# SYMBOL64-NEXT:     Section: N_DEBUG
+# SYMBOL64-NEXT:     Type: 0x0
+# SYMBOL64-NEXT:     StorageClass: C_STSYM (0x85)
+# SYMBOL64-NEXT:     NumberOfAuxEntries: 0
+# SYMBOL64-NEXT:   }
+# SYMBOL64-NEXT:   Symbol {
+# SYMBOL64-NEXT:     Index: 26
+# SYMBOL64-NEXT:     Name: bincl
+# SYMBOL64-NEXT:     Value (OffsetInFile): 0x6
+# SYMBOL64-NEXT:     Section: .text
+# SYMBOL64-NEXT:     Type: 0x0
+# SYMBOL64-NEXT:     StorageClass: C_BINCL (0x6C)
+# SYMBOL64-NEXT:     NumberOfAuxEntries: 0
+# SYMBOL64-NEXT:   }
+# SYMBOL64-NEXT:   Symbol {
+# SYMBOL64-NEXT:     Index: 27
+# SYMBOL64-NEXT:     Name: eincl
+# SYMBOL64-NEXT:     Value (OffsetInFile): 0x6
+# SYMBOL64-NEXT:     Section: .text
+# SYMBOL64-NEXT:     Type: 0x0
+# SYMBOL64-NEXT:     StorageClass: C_EINCL (0x6D)
+# SYMBOL64-NEXT:     NumberOfAuxEntries: 0
+# SYMBOL64-NEXT:   }
+# SYMBOL64-NEXT:   Symbol {
+# SYMBOL64-NEXT:     Index: 28
+# SYMBOL64-NEXT:     Name: Unimplemented Debug Name
+# SYMBOL64-NEXT:     Value (OffsetRelToStackFrame): 0x7
+# SYMBOL64-NEXT:     Section: N_DEBUG
+# SYMBOL64-NEXT:     Type: 0x0
+# SYMBOL64-NEXT:     StorageClass: C_LSYM (0x81)
+# SYMBOL64-NEXT:     NumberOfAuxEntries: 0
+# SYMBOL64-NEXT:   }
+# SYMBOL64-NEXT:   Symbol {
+# SYMBOL64-NEXT:     Index: 29
+# SYMBOL64-NEXT:     Name: Unimplemented Debug Name
+# SYMBOL64-NEXT:     Value (OffsetRelToStackFrame): 0x7
+# SYMBOL64-NEXT:     Section: N_DEBUG
+# SYMBOL64-NEXT:     Type: 0x0
+# SYMBOL64-NEXT:     StorageClass: C_PSYM (0x82)
+# SYMBOL64-NEXT:     NumberOfAuxEntries: 0
+# SYMBOL64-NEXT:   }
+# SYMBOL64-NEXT:   Symbol {
+# SYMBOL64-NEXT:     Index: 30
+# SYMBOL64-NEXT:     Name: Unimplemented Debug Name
+# SYMBOL64-NEXT:     Value (RegisterNumber): 0x8
+# SYMBOL64-NEXT:     Section: N_DEBUG
+# SYMBOL64-NEXT:     Type: 0x0
+# SYMBOL64-NEXT:     StorageClass: C_RSYM (0x83)
+# SYMBOL64-NEXT:     NumberOfAuxEntries: 0
+# SYMBOL64-NEXT:   }
+# SYMBOL64-NEXT:   Symbol {
+# SYMBOL64-NEXT:     Index: 31
+# SYMBOL64-NEXT:     Name: Unimplemented Debug Name
+# SYMBOL64-NEXT:     Value (RegisterNumber): 0x8
+# SYMBOL64-NEXT:     Section: N_DEBUG
+# SYMBOL64-NEXT:     Type: 0x0
+# SYMBOL64-NEXT:     StorageClass: C_RPSYM (0x84)
+# SYMBOL64-NEXT:     NumberOfAuxEntries: 0
+# SYMBOL64-NEXT:   }
+# SYMBOL64-NEXT:   Symbol {
+# SYMBOL64-NEXT:     Index: 32
+# SYMBOL64-NEXT:     Name: Unimplemented Debug Name
+# SYMBOL64-NEXT:     Value (OffsetInCommBlock): 0x9
+# SYMBOL64-NEXT:     Section: .text
+# SYMBOL64-NEXT:     Type: 0x0
+# SYMBOL64-NEXT:     StorageClass: C_ECOML (0x88)
+# SYMBOL64-NEXT:     NumberOfAuxEntries: 0
+# SYMBOL64-NEXT:   }
+# SYMBOL64-NEXT:   Symbol {
+# SYMBOL64-NEXT:     Index: 33
+# SYMBOL64-NEXT:     Name: cinfo
+# SYMBOL64-NEXT:     Value (OffsetInCommentSection): 0x2
+# SYMBOL64-NEXT:     Section: .text
+# SYMBOL64-NEXT:     Type: 0x0
+# SYMBOL64-NEXT:     StorageClass: C_INFO (0x6E)
+# SYMBOL64-NEXT:     NumberOfAuxEntries: 0
+# SYMBOL64-NEXT:   }
 # SYMBOL64-NEXT: ]
diff --git a/llvm/test/tools/llvm-symbolizer/skip-line-zero.s b/llvm/test/tools/llvm-symbolizer/skip-line-zero.s
index e9fbea558e0eb..74dfb5cdc1aae 100644
--- a/llvm/test/tools/llvm-symbolizer/skip-line-zero.s
+++ b/llvm/test/tools/llvm-symbolizer/skip-line-zero.s
@@ -20,13 +20,13 @@
 ## Check that without '--skip-line-zero', line zero is displayed for a line-table entry which has no source correspondence.
 # RUN: llvm-symbolizer --obj=%t.o -f=none 0x16d4 | FileCheck --strict-whitespace --match-full-lines --check-prefix=DISABLE %s
 
-# DISABLE:main.c:0:0
+# DISABLE:??:0:0
 
 ## Check that the '--skip-line-zero' does not cross sequence boundaries.
 ## If it fails to find in the current sequence then line zero is returned for the queried address.
 # RUN: llvm-symbolizer --obj=%t.o -f=none --skip-line-zero 0x16c0 | FileCheck --strict-whitespace --match-full-lines --check-prefix=FAIL-ACROSS-SEQ %s
 
-# FAIL-ACROSS-SEQ:main.c:0:0
+# FAIL-ACROSS-SEQ:??:0:0
 
 ## Check that with '--skip-line-zero', the last non-zero line in the current sequence is displayed.
 # RUN: llvm-symbolizer --obj=%t.o -f=none --skip-line-zero 0x1717 | FileCheck --strict-whitespace --match-full-lines --check-prefix=WITHIN-SEQ %s
diff --git a/llvm/test/tools/llvm-symbolizer/sym-verbose.test b/llvm/test/tools/llvm-symbolizer/sym-verbose.test
index 831fd6c7f0507..224c317f558a1 100644
--- a/llvm/test/tools/llvm-symbolizer/sym-verbose.test
+++ b/llvm/test/tools/llvm-symbolizer/sym-verbose.test
@@ -50,13 +50,12 @@ CHECK-NEXT: Column: 0
 
 CHECK: 0x4005ad
 CHECK-NEXT: foo
-CHECK-NEXT: Filename: /tmp{{[\\/]}}discrim.c
+CHECK-NEXT: Filename: ??
 CHECK-NEXT: Function start filename: /tmp{{[\\/]}}discrim.c
 CHECK-NEXT: Function start line: 4
 CHECK-NEXT: Function start address: 0x400590
 CHECK-NEXT: Line: 0
-CHECK-NEXT: Column: 30
-CHECK-NEXT: Discriminator: 4
+CHECK-NEXT: Column: 0
 CHECK-NEXT: main
 CHECK-NEXT: Filename: /tmp{{[\\/]}}discrim.c
 CHECK-NEXT: Function start filename: /tmp{{[\\/]}}discrim.c
diff --git a/llvm/tools/dsymutil/BinaryHolder.cpp b/llvm/tools/dsymutil/BinaryHolder.cpp
index 5daaa6755b295..7588a33eb46b2 100644
--- a/llvm/tools/dsymutil/BinaryHolder.cpp
+++ b/llvm/tools/dsymutil/BinaryHolder.cpp
@@ -176,8 +176,8 @@ BinaryHolder::ArchiveEntry::getObjectEntry(StringRef Filename,
 
   // Try the cache first.
   std::lock_guard<std::mutex> Lock(MemberCacheMutex);
-  if (MemberCache.count(Key))
-    return *MemberCache[Key];
+  if (auto It = MemberCache.find(Key); It != MemberCache.end())
+    return *It->second;
 
   // Create a new ObjectEntry, but don't add it to the cache yet. Loading of
   // the archive members might fail and we don't want to lock the whole archive
@@ -228,8 +228,7 @@ BinaryHolder::ArchiveEntry::getObjectEntry(StringRef Filename,
   if (OE->Objects.empty())
     return errorCodeToError(errc::no_such_file_or_directory);
 
-  MemberCache[Key] = std::move(OE);
-  return *MemberCache[Key];
+  return *(MemberCache[Key] = std::move(OE));
 }
 
 Expected<const BinaryHolder::ObjectEntry &>
diff --git a/llvm/tools/lli/lli.cpp b/llvm/tools/lli/lli.cpp
index 448660a539a0b..19246f0394167 100644
--- a/llvm/tools/lli/lli.cpp
+++ b/llvm/tools/lli/lli.cpp
@@ -27,9 +27,7 @@
 #include "llvm/ExecutionEngine/Orc/AbsoluteSymbols.h"
 #include "llvm/ExecutionEngine/Orc/DebugUtils.h"
 #include "llvm/ExecutionEngine/Orc/Debugging/DebuggerSupport.h"
-#include "llvm/ExecutionEngine/Orc/EHFrameRegistrationPlugin.h"
 #include "llvm/ExecutionEngine/Orc/EPCDynamicLibrarySearchGenerator.h"
-#include "llvm/ExecutionEngine/Orc/EPCEHFrameRegistrar.h"
 #include "llvm/ExecutionEngine/Orc/EPCGenericRTDyldMemoryManager.h"
 #include "llvm/ExecutionEngine/Orc/ExecutionUtils.h"
 #include "llvm/ExecutionEngine/Orc/IRPartitionLayer.h"
@@ -1033,14 +1031,10 @@ int runOrcJIT(const char *ProgName) {
     Builder.getJITTargetMachineBuilder()
         ->setRelocationModel(Reloc::PIC_)
         .setCodeModel(CodeModel::Small);
-    Builder.setObjectLinkingLayerCreator([&P](orc::ExecutionSession &ES,
-                                              const Triple &TT) {
-      auto L = std::make_unique<orc::ObjectLinkingLayer>(ES);
-      if (P != LLJITPlatform::ExecutorNative)
-        L->addPlugin(std::make_unique<orc::EHFrameRegistrationPlugin>(
-            ES, ExitOnErr(orc::EPCEHFrameRegistrar::Create(ES))));
-      return L;
-    });
+    Builder.setObjectLinkingLayerCreator(
+        [&](orc::ExecutionSession &ES, const Triple &TT) {
+          return std::make_unique<orc::ObjectLinkingLayer>(ES);
+        });
   }
 
   auto J = ExitOnErr(Builder.create());
diff --git a/llvm/tools/llvm-cov/CoverageSummaryInfo.cpp b/llvm/tools/llvm-cov/CoverageSummaryInfo.cpp
index 5c002a694f66a..be9aef8416e8d 100644
--- a/llvm/tools/llvm-cov/CoverageSummaryInfo.cpp
+++ b/llvm/tools/llvm-cov/CoverageSummaryInfo.cpp
@@ -53,10 +53,11 @@ auto sumMCDCPairs(const ArrayRef<MCDCRecord> &Records) {
   for (const auto &Record : Records) {
     const auto NumConditions = Record.getNumConditions();
     for (unsigned C = 0; C < NumConditions; C++) {
-      if (!Record.isCondFolded(C))
+      if (!Record.isCondFolded(C)) {
         ++NumPairs;
-      if (Record.isConditionIndependencePairCovered(C))
-        ++CoveredPairs;
+        if (Record.isConditionIndependencePairCovered(C))
+          ++CoveredPairs;
+      }
     }
   }
   return MCDCCoverageInfo(CoveredPairs, NumPairs);
diff --git a/llvm/tools/llvm-jitlink/llvm-jitlink-executor/llvm-jitlink-executor.cpp b/llvm/tools/llvm-jitlink/llvm-jitlink-executor/llvm-jitlink-executor.cpp
index 86b89a38c1760..dbdec77327774 100644
--- a/llvm/tools/llvm-jitlink/llvm-jitlink-executor/llvm-jitlink-executor.cpp
+++ b/llvm/tools/llvm-jitlink/llvm-jitlink-executor/llvm-jitlink-executor.cpp
@@ -12,6 +12,7 @@
 
 #include "llvm/ADT/StringRef.h"
 #include "llvm/Config/llvm-config.h" // for LLVM_ON_UNIX, LLVM_ENABLE_THREADS
+#include "llvm/ExecutionEngine/Orc/TargetProcess/DefaultHostBootstrapValues.h"
 #include "llvm/ExecutionEngine/Orc/TargetProcess/ExecutorSharedMemoryMapperService.h"
 #include "llvm/ExecutionEngine/Orc/TargetProcess/JITLoaderGDB.h"
 #include "llvm/ExecutionEngine/Orc/TargetProcess/RegisterEHFrames.h"
@@ -187,6 +188,8 @@ int main(int argc, char *argv[]) {
                 std::make_unique<SimpleRemoteEPCServer::ThreadDispatcher>());
             S.bootstrapSymbols() =
                 SimpleRemoteEPCServer::defaultBootstrapSymbols();
+            addDefaultBootstrapValuesForHostProcess(S.bootstrapMap(),
+                                                    S.bootstrapSymbols());
             S.services().push_back(
                 std::make_unique<rt_bootstrap::SimpleExecutorMemoryManager>());
             S.services().push_back(
diff --git a/llvm/tools/llvm-mca/Views/BottleneckAnalysis.h b/llvm/tools/llvm-mca/Views/BottleneckAnalysis.h
index 529090cf543fc..d4c07f521e6b1 100644
--- a/llvm/tools/llvm-mca/Views/BottleneckAnalysis.h
+++ b/llvm/tools/llvm-mca/Views/BottleneckAnalysis.h
@@ -228,7 +228,10 @@ class DependencyGraph {
     unsigned Depth;
 
     DependencyEdge CriticalPredecessor;
-    SmallVector<DependencyEdge, 8> OutgoingEdges;
+    // Measurements show that more than 90% of nodes have no outgoing edges. To
+    // minimize memory consumption we use SmallVector with zero inline elements
+    // that is preferred version of std::vector.
+    SmallVector<DependencyEdge, 0> OutgoingEdges;
   };
   SmallVector<DGNode, 16> Nodes;
 
diff --git a/llvm/tools/llvm-mca/Views/ResourcePressureView.cpp b/llvm/tools/llvm-mca/Views/ResourcePressureView.cpp
index f39350f3b4585..556c911b53f0f 100644
--- a/llvm/tools/llvm-mca/Views/ResourcePressureView.cpp
+++ b/llvm/tools/llvm-mca/Views/ResourcePressureView.cpp
@@ -37,8 +37,16 @@ ResourcePressureView::ResourcePressureView(const llvm::MCSubtargetInfo &sti,
   }
 
   NumResourceUnits = R2VIndex;
-  ResourceUsage.resize(NumResourceUnits * (getSource().size() + 1));
-  std::fill(ResourceUsage.begin(), ResourceUsage.end(), 0.0);
+  ResourceUsage.resize(getSource().size());
+
+  ResourceReleaseAtCycles InitValue{0, 0};
+  auto Generator = [&InitValue]() {
+    ResourceReleaseAtCycles Old = InitValue;
+    ++InitValue.ResourceIdx;
+    return Old;
+  };
+  std::generate_n(std::back_inserter(CommonResourceUsage), NumResourceUnits,
+                  Generator);
 }
 
 void ResourcePressureView::onEvent(const HWInstructionEvent &Event) {
@@ -60,8 +68,19 @@ void ResourcePressureView::onEvent(const HWInstructionEvent &Event) {
     assert(Resource2VecIndex.contains(RR.first));
     unsigned R2VIndex = Resource2VecIndex[RR.first];
     R2VIndex += llvm::countr_zero(RR.second);
-    ResourceUsage[R2VIndex + NumResourceUnits * SourceIdx] += Use.second;
-    ResourceUsage[R2VIndex + NumResourceUnits * Source.size()] += Use.second;
+
+    InstResourceUsage &RU = ResourceUsage[SourceIdx];
+    ResourceReleaseAtCycles NewUsage{R2VIndex, Use.second};
+    auto ResCyclesIt =
+        lower_bound(RU, NewUsage, [](const auto &L, const auto &R) {
+          return L.ResourceIdx < R.ResourceIdx;
+        });
+    if (ResCyclesIt != RU.end() && ResCyclesIt->ResourceIdx == R2VIndex)
+      ResCyclesIt->Cycles += NewUsage.Cycles;
+    else
+      RU.insert(ResCyclesIt, std::move(NewUsage));
+
+    CommonResourceUsage[R2VIndex].Cycles += NewUsage.Cycles;
   }
 }
 
@@ -135,10 +154,17 @@ void ResourcePressureView::printResourcePressurePerIter(raw_ostream &OS) const {
 
   ArrayRef<llvm::MCInst> Source = getSource();
   const unsigned Executions = LastInstructionIdx / Source.size() + 1;
+  auto UsageEntryEnd = CommonResourceUsage.end();
+  auto UsageEntryIt = CommonResourceUsage.begin();
   for (unsigned I = 0, E = NumResourceUnits; I < E; ++I) {
-    double Usage = ResourceUsage[I + Source.size() * E];
-    printResourcePressure(FOS, Usage / Executions, (I + 1) * 7);
+    double Pressure = 0.0;
+    if (UsageEntryIt != UsageEntryEnd && UsageEntryIt->ResourceIdx == I) {
+      Pressure = UsageEntryIt->Cycles / Executions;
+      ++UsageEntryIt;
+    }
+    printResourcePressure(FOS, Pressure, (I + 1) * 7);
   }
+  assert(UsageEntryIt == UsageEntryEnd);
 
   FOS.flush();
   OS << Buffer;
@@ -157,11 +183,17 @@ void ResourcePressureView::printResourcePressurePerInst(raw_ostream &OS) const {
   ArrayRef<llvm::MCInst> Source = getSource();
   const unsigned Executions = LastInstructionIdx / Source.size() + 1;
   for (const MCInst &MCI : Source) {
-    unsigned BaseEltIdx = InstrIndex * NumResourceUnits;
+    auto UsageEntryEnd = ResourceUsage[InstrIndex].end();
+    auto UsageEntryIt = ResourceUsage[InstrIndex].begin();
     for (unsigned J = 0; J < NumResourceUnits; ++J) {
-      double Usage = ResourceUsage[J + BaseEltIdx];
-      printResourcePressure(FOS, Usage / Executions, (J + 1) * 7);
+      double Pressure = 0.0;
+      if (UsageEntryIt != UsageEntryEnd && UsageEntryIt->ResourceIdx == J) {
+        Pressure = UsageEntryIt->Cycles / Executions;
+        ++UsageEntryIt;
+      }
+      printResourcePressure(FOS, Pressure, (J + 1) * 7);
     }
+    assert(UsageEntryIt == UsageEntryEnd);
 
     FOS << printInstructionString(MCI) << '\n';
     FOS.flush();
@@ -180,17 +212,22 @@ json::Value ResourcePressureView::toJSON() const {
   // non-zero values.
   ArrayRef<llvm::MCInst> Source = getSource();
   const unsigned Executions = LastInstructionIdx / Source.size() + 1;
-  for (const auto &R : enumerate(ResourceUsage)) {
-    const ReleaseAtCycles &RU = R.value();
-    if (RU.getNumerator() == 0)
-      continue;
-    unsigned InstructionIndex = R.index() / NumResourceUnits;
-    unsigned ResourceIndex = R.index() % NumResourceUnits;
-    double Usage = RU / Executions;
+
+  auto AddToJSON = [&ResourcePressureInfo, Executions](
+                       const ResourceReleaseAtCycles &RU, unsigned InstIndex) {
+    assert(RU.Cycles.getNumerator() != 0);
+    double Usage = RU.Cycles / Executions;
     ResourcePressureInfo.push_back(
-        json::Object({{"InstructionIndex", InstructionIndex},
-                      {"ResourceIndex", ResourceIndex},
+        json::Object({{"InstructionIndex", InstIndex},
+                      {"ResourceIndex", RU.ResourceIdx},
                       {"ResourceUsage", Usage}}));
+  };
+  for (const auto &[InstIndex, Usages] : enumerate(ResourceUsage))
+    for (const auto &RU : Usages)
+      AddToJSON(RU, InstIndex);
+  for (const auto &RU : CommonResourceUsage) {
+    if (RU.Cycles.getNumerator() != 0)
+      AddToJSON(RU, Source.size());
   }
 
   json::Object JO({{"ResourcePressureInfo", std::move(ResourcePressureInfo)}});
diff --git a/llvm/tools/llvm-mca/Views/ResourcePressureView.h b/llvm/tools/llvm-mca/Views/ResourcePressureView.h
index be8ad04102fd0..3deb9e8a21d5e 100644
--- a/llvm/tools/llvm-mca/Views/ResourcePressureView.h
+++ b/llvm/tools/llvm-mca/Views/ResourcePressureView.h
@@ -77,8 +77,13 @@ class ResourcePressureView : public InstructionView {
   // resource ID.
   llvm::DenseMap<unsigned, unsigned> Resource2VecIndex;
 
-  // Table of resources used by instructions.
-  std::vector<ReleaseAtCycles> ResourceUsage;
+  struct ResourceReleaseAtCycles {
+    unsigned ResourceIdx;
+    ReleaseAtCycles Cycles;
+  };
+  using InstResourceUsage = std::vector<ResourceReleaseAtCycles>;
+  std::vector<InstResourceUsage> ResourceUsage;
+  InstResourceUsage CommonResourceUsage;
   unsigned NumResourceUnits;
 
   void printResourcePressurePerIter(llvm::raw_ostream &OS) const;
diff --git a/llvm/tools/llvm-profgen/MissingFrameInferrer.cpp b/llvm/tools/llvm-profgen/MissingFrameInferrer.cpp
index ee49950f39ca4..eefe38cd3fa00 100644
--- a/llvm/tools/llvm-profgen/MissingFrameInferrer.cpp
+++ b/llvm/tools/llvm-profgen/MissingFrameInferrer.cpp
@@ -206,11 +206,12 @@ uint64_t MissingFrameInferrer::computeUniqueTailCallPath(
 
 uint64_t MissingFrameInferrer::computeUniqueTailCallPath(
     uint64_t From, BinaryFunction *To, SmallVectorImpl<uint64_t> &Path) {
-  if (!TailCallEdgesF.count(From))
+  auto It = TailCallEdgesF.find(From);
+  if (It == TailCallEdgesF.end())
     return 0;
   Path.push_back(From);
   uint64_t NumPaths = 0;
-  for (auto Target : TailCallEdgesF[From]) {
+  for (auto Target : It->second) {
     NumPaths += computeUniqueTailCallPath(Target, To, Path);
     // Stop analyzing the remaining if we are already seeing more than one
     // reachable paths.
diff --git a/llvm/tools/llvm-readobj/ELFDumper.cpp b/llvm/tools/llvm-readobj/ELFDumper.cpp
index bfca65aad52b4..2128a95510f22 100644
--- a/llvm/tools/llvm-readobj/ELFDumper.cpp
+++ b/llvm/tools/llvm-readobj/ELFDumper.cpp
@@ -5333,7 +5333,8 @@ static bool printAArch64PAuthABICoreInfo(raw_ostream &OS, uint32_t DataSize,
 
 template <typename ELFT>
 static std::string getGNUProperty(uint32_t Type, uint32_t DataSize,
-                                  ArrayRef<uint8_t> Data) {
+                                  ArrayRef<uint8_t> Data,
+                                  typename ELFT::Half EMachine) {
   std::string str;
   raw_string_ostream OS(str);
   uint32_t PrData;
@@ -5366,8 +5367,25 @@ static std::string getGNUProperty(uint32_t Type, uint32_t DataSize,
     return str;
   case GNU_PROPERTY_AARCH64_FEATURE_1_AND:
   case GNU_PROPERTY_X86_FEATURE_1_AND:
-    OS << ((Type == GNU_PROPERTY_AARCH64_FEATURE_1_AND) ? "aarch64 feature: "
-                                                        : "x86 feature: ");
+    static_assert(GNU_PROPERTY_AARCH64_FEATURE_1_AND ==
+                      GNU_PROPERTY_RISCV_FEATURE_1_AND,
+                  "GNU_PROPERTY_RISCV_FEATURE_1_AND should equal "
+                  "GNU_PROPERTY_AARCH64_FEATURE_1_AND, otherwise "
+                  "GNU_PROPERTY_RISCV_FEATURE_1_AND would be skipped!");
+
+    if (EMachine == EM_AARCH64 && Type == GNU_PROPERTY_AARCH64_FEATURE_1_AND) {
+      OS << "aarch64 feature: ";
+    } else if (EMachine == EM_RISCV &&
+               Type == GNU_PROPERTY_RISCV_FEATURE_1_AND) {
+      OS << "RISC-V feature: ";
+    } else if ((EMachine == EM_386 || EMachine == EM_X86_64) &&
+               Type == GNU_PROPERTY_X86_FEATURE_1_AND) {
+      OS << "x86 feature: ";
+    } else {
+      OS << format("<application-specific type 0x%x>", Type);
+      return str;
+    }
+
     if (DataSize != 4) {
       OS << format("<corrupt length: 0x%x>", DataSize);
       return str;
@@ -5377,10 +5395,16 @@ static std::string getGNUProperty(uint32_t Type, uint32_t DataSize,
       OS << "<None>";
       return str;
     }
-    if (Type == GNU_PROPERTY_AARCH64_FEATURE_1_AND) {
+
+    if (EMachine == EM_AARCH64) {
       DumpBit(GNU_PROPERTY_AARCH64_FEATURE_1_BTI, "BTI");
       DumpBit(GNU_PROPERTY_AARCH64_FEATURE_1_PAC, "PAC");
       DumpBit(GNU_PROPERTY_AARCH64_FEATURE_1_GCS, "GCS");
+    } else if (EMachine == EM_RISCV) {
+      DumpBit(GNU_PROPERTY_RISCV_FEATURE_1_CFI_LP_UNLABELED,
+              "ZICFILP-unlabeled");
+      DumpBit(GNU_PROPERTY_RISCV_FEATURE_1_CFI_SS, "ZICFISS");
+      DumpBit(GNU_PROPERTY_RISCV_FEATURE_1_CFI_LP_FUNC_SIG, "ZICFILP-func-sig");
     } else {
       DumpBit(GNU_PROPERTY_X86_FEATURE_1_IBT, "IBT");
       DumpBit(GNU_PROPERTY_X86_FEATURE_1_SHSTK, "SHSTK");
@@ -5441,7 +5465,8 @@ static std::string getGNUProperty(uint32_t Type, uint32_t DataSize,
 }
 
 template <typename ELFT>
-static SmallVector<std::string, 4> getGNUPropertyList(ArrayRef<uint8_t> Arr) {
+static SmallVector<std::string, 4>
+getGNUPropertyList(ArrayRef<uint8_t> Arr, typename ELFT::Half EMachine) {
   using Elf_Word = typename ELFT::Word;
 
   SmallVector<std::string, 4> Properties;
@@ -5459,8 +5484,8 @@ static SmallVector<std::string, 4> getGNUPropertyList(ArrayRef<uint8_t> Arr) {
       Properties.push_back(str);
       break;
     }
-    Properties.push_back(
-        getGNUProperty<ELFT>(Type, DataSize, Arr.take_front(PaddedSize)));
+    Properties.push_back(getGNUProperty<ELFT>(
+        Type, DataSize, Arr.take_front(PaddedSize), EMachine));
     Arr = Arr.drop_front(PaddedSize);
   }
 
@@ -5512,7 +5537,7 @@ static StringRef getDescAsStringRef(ArrayRef<uint8_t> Desc) {
 
 template <typename ELFT>
 static bool printGNUNote(raw_ostream &OS, uint32_t NoteType,
-                         ArrayRef<uint8_t> Desc) {
+                         ArrayRef<uint8_t> Desc, typename ELFT::Half EMachine) {
   // Return true if we were able to pretty-print the note, false otherwise.
   switch (NoteType) {
   default:
@@ -5534,7 +5559,7 @@ static bool printGNUNote(raw_ostream &OS, uint32_t NoteType,
     break;
   case ELF::NT_GNU_PROPERTY_TYPE_0:
     OS << "    Properties:";
-    for (const std::string &Property : getGNUPropertyList<ELFT>(Desc))
+    for (const std::string &Property : getGNUPropertyList<ELFT>(Desc, EMachine))
       OS << "    " << Property << "\n";
     break;
   }
@@ -6223,10 +6248,12 @@ template <class ELFT> void GNUELFDumper<ELFT>::printNotes() {
     else
       OS << "Unknown note type: (" << format_hex(Type, 10) << ")\n";
 
+    const typename ELFT::Half EMachine = this->Obj.getHeader().e_machine;
+
     // Print the description, or fallback to printing raw bytes for unknown
     // owners/if we fail to pretty-print the contents.
     if (Name == "GNU") {
-      if (printGNUNote<ELFT>(OS, Type, Descriptor))
+      if (printGNUNote<ELFT>(OS, Type, Descriptor, EMachine))
         return Error::success();
     } else if (Name == "FreeBSD") {
       if (std::optional<FreeBSDNote> N =
@@ -6806,6 +6833,16 @@ void ELFDumper<ELFT>::printRelocatableStackSizes(
       continue;
     }
 
+    // We might end up with relocations in CREL here. If we do, report a
+    // warning since we do not currently support them.
+    if (RelocSec->sh_type == ELF::SHT_CREL) {
+      reportWarning(createError(".stack_sizes (" + describe(*StackSizesELFSec) +
+                                ") has a corresponding CREL relocation "
+                                "section, which is not currently supported"),
+                    FileName);
+      continue;
+    }
+
     // A .stack_sizes section header's sh_link field is supposed to point
     // to the section that contains the functions whose stack sizes are
     // described in it.
@@ -7913,7 +7950,8 @@ template <class ELFT> void LLVMELFDumper<ELFT>::printAddrsig() {
 
 template <typename ELFT>
 static bool printGNUNoteLLVMStyle(uint32_t NoteType, ArrayRef<uint8_t> Desc,
-                                  ScopedPrinter &W) {
+                                  ScopedPrinter &W,
+                                  typename ELFT::Half EMachine) {
   // Return true if we were able to pretty-print the note, false otherwise.
   switch (NoteType) {
   default:
@@ -7938,7 +7976,7 @@ static bool printGNUNoteLLVMStyle(uint32_t NoteType, ArrayRef<uint8_t> Desc,
     break;
   case ELF::NT_GNU_PROPERTY_TYPE_0:
     ListScope D(W, "Property");
-    for (const std::string &Property : getGNUPropertyList<ELFT>(Desc))
+    for (const std::string &Property : getGNUPropertyList<ELFT>(Desc, EMachine))
       W.printString(Property);
     break;
   }
@@ -8057,10 +8095,11 @@ template <class ELFT> void LLVMELFDumper<ELFT>::printNotes() {
       W.printString("Type",
                     "Unknown (" + to_string(format_hex(Type, 10)) + ")");
 
+    const typename ELFT::Half EMachine = this->Obj.getHeader().e_machine;
     // Print the description, or fallback to printing raw bytes for unknown
     // owners/if we fail to pretty-print the contents.
     if (Name == "GNU") {
-      if (printGNUNoteLLVMStyle<ELFT>(Type, Descriptor, W))
+      if (printGNUNoteLLVMStyle<ELFT>(Type, Descriptor, W, EMachine))
         return Error::success();
     } else if (Name == "FreeBSD") {
       if (std::optional<FreeBSDNote> N =
diff --git a/llvm/tools/llvm-readobj/XCOFFDumper.cpp b/llvm/tools/llvm-readobj/XCOFFDumper.cpp
index 6a099c08e1aca..03764e9ba483d 100644
--- a/llvm/tools/llvm-readobj/XCOFFDumper.cpp
+++ b/llvm/tools/llvm-readobj/XCOFFDumper.cpp
@@ -692,22 +692,26 @@ static StringRef GetSymbolValueName(XCOFF::StorageClass SC) {
   case XCOFF::C_BLOCK:
     return "Value (RelocatableAddress)";
   case XCOFF::C_FILE:
+  case XCOFF::C_BSTAT:
     return "Value (SymbolTableIndex)";
   case XCOFF::C_DWARF:
     return "Value (OffsetInDWARF)";
   case XCOFF::C_FUN:
   case XCOFF::C_STSYM:
+    return "Value (OffsetInCSect)";
   case XCOFF::C_BINCL:
   case XCOFF::C_EINCL:
+    return "Value (OffsetInFile)";
   case XCOFF::C_INFO:
-  case XCOFF::C_BSTAT:
+    return "Value (OffsetInCommentSection)";
   case XCOFF::C_LSYM:
   case XCOFF::C_PSYM:
+    return "Value (OffsetRelToStackFrame)";
   case XCOFF::C_RPSYM:
   case XCOFF::C_RSYM:
+    return "Value (RegisterNumber)";
   case XCOFF::C_ECOML:
-    assert(false && "This StorageClass for the symbol is not yet implemented.");
-    return "";
+    return "Value (OffsetInCommBlock)";
   default:
     return "Value";
   }
diff --git a/llvm/tools/obj2yaml/dxcontainer2yaml.cpp b/llvm/tools/obj2yaml/dxcontainer2yaml.cpp
index 06966b1883586..54a912d9438af 100644
--- a/llvm/tools/obj2yaml/dxcontainer2yaml.cpp
+++ b/llvm/tools/obj2yaml/dxcontainer2yaml.cpp
@@ -153,6 +153,11 @@ dumpDXContainer(MemoryBufferRef Source) {
       break;
     case dxbc::PartType::Unknown:
       break;
+    case dxbc::PartType::RTS0:
+      std::optional<DirectX::RootSignature> RS = Container.getRootSignature();
+      if (RS.has_value())
+        NewPart.RootSignature = DXContainerYAML::RootSignatureDesc(*RS);
+      break;
     }
   }
 
diff --git a/llvm/tools/opt/NewPMDriver.cpp b/llvm/tools/opt/NewPMDriver.cpp
index ea300fb3dbeb1..5ed42b054316b 100644
--- a/llvm/tools/opt/NewPMDriver.cpp
+++ b/llvm/tools/opt/NewPMDriver.cpp
@@ -122,6 +122,11 @@ static cl::opt<std::string> VectorizerStartEPPipeline(
     cl::desc("A textual description of the function pass pipeline inserted at "
              "the VectorizerStart extension point into default pipelines"),
     cl::Hidden);
+static cl::opt<std::string> VectorizerEndEPPipeline(
+    "passes-ep-vectorizer-end",
+    cl::desc("A textual description of the function pass pipeline inserted at "
+             "the VectorizerEnd extension point into default pipelines"),
+    cl::Hidden);
 static cl::opt<std::string> PipelineStartEPPipeline(
     "passes-ep-pipeline-start",
     cl::desc("A textual description of the module pass pipeline inserted at "
@@ -285,6 +290,12 @@ static void registerEPCallbacks(PassBuilder &PB) {
           ExitOnError Err("Unable to parse VectorizerStartEP pipeline: ");
           Err(PB.parsePassPipeline(PM, VectorizerStartEPPipeline));
         });
+  if (tryParsePipelineText<FunctionPassManager>(PB, VectorizerEndEPPipeline))
+    PB.registerVectorizerEndEPCallback(
+        [&PB](FunctionPassManager &PM, OptimizationLevel Level) {
+          ExitOnError Err("Unable to parse VectorizerEndEP pipeline: ");
+          Err(PB.parsePassPipeline(PM, VectorizerEndEPPipeline));
+        });
   if (tryParsePipelineText<ModulePassManager>(PB, PipelineStartEPPipeline))
     PB.registerPipelineStartEPCallback(
         [&PB](ModulePassManager &PM, OptimizationLevel) {
diff --git a/llvm/tools/spirv-tools/CMakeLists.txt b/llvm/tools/spirv-tools/CMakeLists.txt
index a47c1f894d7a4..c2c0f3e3c2e42 100644
--- a/llvm/tools/spirv-tools/CMakeLists.txt
+++ b/llvm/tools/spirv-tools/CMakeLists.txt
@@ -9,10 +9,10 @@ if (NOT "SPIRV" IN_LIST LLVM_TARGETS_TO_BUILD)
   message(FATAL_ERROR "Building SPIRV-Tools tests is unsupported without the SPIR-V target")
 endif ()
 
-# SPIRV_DIS and SPIRV_VAL variables can be used to provide paths to existing
-# spirv-dis and spirv-val binaries, respectively. Otherwise, build them from
+# SPIRV_DIS, SPIRV_VAL, SPIRV_AS and SPIRV_LINK variables can be used to provide paths to existing
+# spirv-dis, spirv-val, spirv-as, and spirv-link binaries, respectively. Otherwise, build them from
 # SPIRV-Tools source.
-if (NOT SPIRV_DIS OR NOT SPIRV_VAL OR NOT SPIRV_AS)
+if (NOT SPIRV_DIS OR NOT SPIRV_VAL OR NOT SPIRV_AS OR NOT SPIRV_LINK)
   include(ExternalProject)
 
   set(BINARY_DIR ${CMAKE_CURRENT_BINARY_DIR}/SPIRVTools-bin)
@@ -21,8 +21,8 @@ if (NOT SPIRV_DIS OR NOT SPIRV_VAL OR NOT SPIRV_AS)
     GIT_REPOSITORY https://github.com/KhronosGroup/SPIRV-Tools.git
     GIT_TAG main
     BINARY_DIR ${BINARY_DIR}
-    BUILD_COMMAND ${CMAKE_COMMAND} --build ${BINARY_DIR} --target spirv-dis spirv-val spirv-as
-    BUILD_BYPRODUCTS ${BINARY_DIR}/tools/spirv-dis ${BINARY_DIR}/tools/spirv-val ${BINARY_DIR}/tools/spirv-as
+    BUILD_COMMAND ${CMAKE_COMMAND} --build ${BINARY_DIR} --target spirv-dis spirv-val spirv-as spirv-link
+    BUILD_BYPRODUCTS ${BINARY_DIR}/tools/spirv-dis ${BINARY_DIR}/tools/spirv-val ${BINARY_DIR}/tools/spirv-as ${BINARY_DIR}/tools/spirv-link
     DOWNLOAD_COMMAND git clone https://github.com/KhronosGroup/SPIRV-Tools.git SPIRVTools &&
       cd SPIRVTools &&
       ${Python3_EXECUTABLE} utils/git-sync-deps
@@ -43,7 +43,7 @@ else ()
   set(LLVM_LINK_OR_COPY copy)
 endif ()
 
-# Link the provided or just built spirv-dis and spirv-val binaries.
+# Link the provided or just built binaries.
 if (SPIRV_DIS)
   add_custom_target(spirv-dis
     COMMAND ${CMAKE_COMMAND} -E ${LLVM_LINK_OR_COPY} "${SPIRV_DIS}" "${LLVM_RUNTIME_OUTPUT_INTDIR}/spirv-dis")
@@ -73,3 +73,13 @@ else ()
     DEPENDS SPIRVTools
     )
 endif ()
+
+if (SPIRV_LINK)
+  add_custom_target(spirv-link
+    COMMAND ${CMAKE_COMMAND} -E ${LLVM_LINK_OR_COPY} "${SPIRV_LINK}" "${LLVM_RUNTIME_OUTPUT_INTDIR}/spirv-link")
+else ()
+  add_custom_target(spirv-link
+    COMMAND ${CMAKE_COMMAND} -E ${LLVM_LINK_OR_COPY} "${BINARY_DIR}/tools/spirv-link${CMAKE_EXECUTABLE_SUFFIX}" "${LLVM_RUNTIME_OUTPUT_INTDIR}/spirv-link${CMAKE_EXECUTABLE_SUFFIX}"
+    DEPENDS SPIRVTools
+    )
+endif ()
diff --git a/llvm/unittests/ADT/StringSwitchTest.cpp b/llvm/unittests/ADT/StringSwitchTest.cpp
index 2ce6cdca8d36a..2953f4b0a381b 100644
--- a/llvm/unittests/ADT/StringSwitchTest.cpp
+++ b/llvm/unittests/ADT/StringSwitchTest.cpp
@@ -205,3 +205,28 @@ TEST(StringSwitchTest, CasesLower) {
   EXPECT_EQ(OSType::Unknown, Translate("wind"));
   EXPECT_EQ(OSType::Unknown, Translate(""));
 }
+
+TEST(StringSwitchTest, CasesCopies) {
+  struct Copyable {
+    unsigned &NumCopies;
+    Copyable(unsigned &Value) : NumCopies(Value) {}
+    Copyable(const Copyable &Other) : NumCopies(Other.NumCopies) {
+      ++NumCopies;
+    }
+    Copyable &operator=(const Copyable &Other) {
+      ++NumCopies;
+      return *this;
+    }
+  };
+
+  // Check that evaluating multiple cases does not cause unnecessary copies.
+  unsigned NumCopies = 0;
+  llvm::StringSwitch<Copyable, void>("baz").Cases("foo", "bar", "baz", "qux",
+                                                  Copyable{NumCopies});
+  EXPECT_EQ(NumCopies, 1u);
+
+  NumCopies = 0;
+  llvm::StringSwitch<Copyable, void>("baz").CasesLower(
+      "Foo", "Bar", "Baz", "Qux", Copyable{NumCopies});
+  EXPECT_EQ(NumCopies, 1u);
+}
diff --git a/llvm/unittests/Analysis/MemoryProfileInfoTest.cpp b/llvm/unittests/Analysis/MemoryProfileInfoTest.cpp
index 4c177ae844690..637fff93cc1ad 100644
--- a/llvm/unittests/Analysis/MemoryProfileInfoTest.cpp
+++ b/llvm/unittests/Analysis/MemoryProfileInfoTest.cpp
@@ -15,6 +15,7 @@
 #include "llvm/IR/ModuleSummaryIndex.h"
 #include "llvm/Support/CommandLine.h"
 #include "llvm/Support/SourceMgr.h"
+#include "gmock/gmock.h"
 #include "gtest/gtest.h"
 #include <cstring>
 #include <sys/types.h>
@@ -25,6 +26,7 @@ using namespace llvm::memprof;
 extern cl::opt<float> MemProfLifetimeAccessDensityColdThreshold;
 extern cl::opt<unsigned> MemProfAveLifetimeColdThreshold;
 extern cl::opt<unsigned> MemProfMinAveLifetimeAccessDensityHotThreshold;
+extern cl::opt<bool> MemProfUseHotHints;
 
 namespace {
 
@@ -81,14 +83,23 @@ TEST_F(MemoryProfileInfoTest, GetAllocType) {
   //    MemProfMinAveLifetimeAccessDensityHotThreshold
   // so compute the HotTotalLifetimeAccessDensityThreshold  at the threshold.
   const uint64_t HotTotalLifetimeAccessDensityThreshold =
-      (uint64_t)(MemProfMinAveLifetimeAccessDensityHotThreshold * AllocCount * 100);  
-   
-  
+      (uint64_t)(MemProfMinAveLifetimeAccessDensityHotThreshold * AllocCount *
+                 100);
+
+  // Make sure the option for detecting hot allocations is set.
+  MemProfUseHotHints = true;
   // Test Hot
   // More accesses per byte per sec than hot threshold is hot.
   EXPECT_EQ(getAllocType(HotTotalLifetimeAccessDensityThreshold + 1, AllocCount,
                          ColdTotalLifetimeThreshold + 1),
-            AllocationType::Hot);  
+            AllocationType::Hot);
+  // Undo the manual set of the option above.
+  cl::ResetAllOptionOccurrences();
+
+  // Without MemProfUseHotHints (default) we should treat simply as NotCold.
+  EXPECT_EQ(getAllocType(HotTotalLifetimeAccessDensityThreshold + 1, AllocCount,
+                         ColdTotalLifetimeThreshold + 1),
+            AllocationType::NotCold);
 
   // Test Cold
   // Long lived with less accesses per byte per sec than cold threshold is cold.
@@ -155,6 +166,8 @@ define i32* @test() {
   %1 = bitcast i8* %call2 to i32*
   %call3 = call noalias dereferenceable_or_null(40) i8* @malloc(i64 noundef 40)
   %2 = bitcast i8* %call3 to i32*  
+  %call4 = call noalias dereferenceable_or_null(40) i8* @malloc(i64 noundef 40)
+  %3 = bitcast i8* %call4 to i32*
   ret i32* %1
 }
 declare dso_local noalias noundef i8* @malloc(i64 noundef)
@@ -194,6 +207,60 @@ declare dso_local noalias noundef i8* @malloc(i64 noundef)
   EXPECT_FALSE(Call3->hasMetadata(LLVMContext::MD_memprof));
   EXPECT_TRUE(Call3->hasFnAttr("memprof"));
   EXPECT_EQ(Call3->getFnAttr("memprof").getValueAsString(), "hot");
+
+  // Fourth call has hot and non-cold contexts. These should be treated as
+  // notcold and given a notcold attribute.
+  CallStackTrie Trie4;
+  Trie4.addCallStack(AllocationType::Hot, {5, 6});
+  Trie4.addCallStack(AllocationType::NotCold, {5, 7, 8});
+  CallBase *Call4 = findCall(*Func, "call4");
+  Trie4.buildAndAttachMIBMetadata(Call4);
+
+  EXPECT_FALSE(Call4->hasMetadata(LLVMContext::MD_memprof));
+  EXPECT_TRUE(Call4->hasFnAttr("memprof"));
+  EXPECT_EQ(Call4->getFnAttr("memprof").getValueAsString(), "notcold");
+}
+
+// TODO: Use this matcher in existing tests.
+// ExpectedVals should be a vector of expected MIBs and their allocation type
+// and stack id contents in order, of type:
+//  std::vector<std::pair<AllocationType, std::vector<unsigned>>>
+MATCHER_P(MemprofMetadataEquals, ExpectedVals, "Matching !memprof contents") {
+  auto PrintAndFail = [&]() {
+    std::string Buffer;
+    llvm::raw_string_ostream OS(Buffer);
+    OS << "Expected:\n";
+    for (auto &[ExpectedAllocType, ExpectedStackIds] : ExpectedVals) {
+      OS << "\t" << getAllocTypeAttributeString(ExpectedAllocType) << " { ";
+      for (auto Id : ExpectedStackIds)
+        OS << Id << " ";
+      OS << "}\n";
+    }
+    OS << "Got:\n";
+    arg->printTree(OS);
+    *result_listener << "!memprof metadata differs!\n" << Buffer;
+    return false;
+  };
+
+  if (ExpectedVals.size() != arg->getNumOperands())
+    return PrintAndFail();
+
+  for (size_t I = 0; I < ExpectedVals.size(); I++) {
+    const auto &[ExpectedAllocType, ExpectedStackIds] = ExpectedVals[I];
+    MDNode *MIB = dyn_cast<MDNode>(arg->getOperand(I));
+    if (getMIBAllocType(MIB) != ExpectedAllocType)
+      return PrintAndFail();
+    MDNode *StackMD = getMIBStackNode(MIB);
+    EXPECT_NE(StackMD, nullptr);
+    if (StackMD->getNumOperands() != ExpectedStackIds.size())
+      return PrintAndFail();
+    for (size_t J = 0; J < ExpectedStackIds.size(); J++) {
+      auto *StackId = mdconst::dyn_extract<ConstantInt>(StackMD->getOperand(J));
+      if (StackId->getZExtValue() != ExpectedStackIds[J])
+        return PrintAndFail();
+    }
+  }
+  return true;
 }
 
 // Test CallStackTrie::addCallStack interface taking allocation type and list of
@@ -289,56 +356,8 @@ declare dso_local noalias noundef i8* @malloc(i64 noundef)
       EXPECT_EQ(getMIBAllocType(MIB), AllocationType::Cold);
     else {
       ASSERT_EQ(StackId->getZExtValue(), 3u);
-      EXPECT_EQ(getMIBAllocType(MIB), AllocationType::Hot);
-    }
-  }
-}
-
-// Test CallStackTrie::addCallStack interface taking allocation type and list of
-// call stack ids.
-// Test that an allocation call reached by both non cold and hot call stacks
-// gets memprof metadata representing the different allocation type contexts.
-TEST_F(MemoryProfileInfoTest, NotColdAndHotMIB) {
-  LLVMContext C;
-  std::unique_ptr<Module> M = makeLLVMModule(C,
-                                             R"IR(
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
-target triple = "x86_64-pc-linux-gnu"
-define i32* @test() {
-entry:
-  %call = call noalias dereferenceable_or_null(40) i8* @malloc(i64 noundef 40)
-  %0 = bitcast i8* %call to i32*
-  ret i32* %0
-}
-declare dso_local noalias noundef i8* @malloc(i64 noundef)
-)IR");
-
-  Function *Func = M->getFunction("test");
-
-  CallStackTrie Trie;
-  Trie.addCallStack(AllocationType::NotCold, {1, 2});
-  Trie.addCallStack(AllocationType::Hot, {1, 3});
-
-  CallBase *Call = findCall(*Func, "call");
-  Trie.buildAndAttachMIBMetadata(Call);
-
-  EXPECT_FALSE(Call->hasFnAttr("memprof"));
-  EXPECT_TRUE(Call->hasMetadata(LLVMContext::MD_memprof));
-  MDNode *MemProfMD = Call->getMetadata(LLVMContext::MD_memprof);
-  ASSERT_EQ(MemProfMD->getNumOperands(), 2u);
-  for (auto &MIBOp : MemProfMD->operands()) {
-    MDNode *MIB = dyn_cast<MDNode>(MIBOp);
-    MDNode *StackMD = getMIBStackNode(MIB);
-    ASSERT_NE(StackMD, nullptr);
-    ASSERT_EQ(StackMD->getNumOperands(), 2u);
-    auto *StackId = mdconst::dyn_extract<ConstantInt>(StackMD->getOperand(0));
-    ASSERT_EQ(StackId->getZExtValue(), 1u);
-    StackId = mdconst::dyn_extract<ConstantInt>(StackMD->getOperand(1));
-    if (StackId->getZExtValue() == 2u)
+      // Hot contexts are converted to NotCold when building the metadata.
       EXPECT_EQ(getMIBAllocType(MIB), AllocationType::NotCold);
-    else {
-      ASSERT_EQ(StackId->getZExtValue(), 3u);
-      EXPECT_EQ(getMIBAllocType(MIB), AllocationType::Hot);
     }
   }
 }
@@ -368,6 +387,8 @@ declare dso_local noalias noundef i8* @malloc(i64 noundef)
   CallStackTrie Trie;
   Trie.addCallStack(AllocationType::Cold, {1, 2});
   Trie.addCallStack(AllocationType::NotCold, {1, 3});
+  // This will be pruned as it is unnecessary to determine how to clone the
+  // cold allocation.
   Trie.addCallStack(AllocationType::Hot, {1, 4});
 
   CallBase *Call = findCall(*Func, "call");
@@ -376,7 +397,7 @@ declare dso_local noalias noundef i8* @malloc(i64 noundef)
   EXPECT_FALSE(Call->hasFnAttr("memprof"));
   EXPECT_TRUE(Call->hasMetadata(LLVMContext::MD_memprof));
   MDNode *MemProfMD = Call->getMetadata(LLVMContext::MD_memprof);
-  ASSERT_EQ(MemProfMD->getNumOperands(), 3u);
+  ASSERT_EQ(MemProfMD->getNumOperands(), 2u);
   for (auto &MIBOp : MemProfMD->operands()) {
     MDNode *MIB = dyn_cast<MDNode>(MIBOp);
     MDNode *StackMD = getMIBStackNode(MIB);
@@ -389,9 +410,6 @@ declare dso_local noalias noundef i8* @malloc(i64 noundef)
       EXPECT_EQ(getMIBAllocType(MIB), AllocationType::Cold);
     } else if (StackId->getZExtValue() == 3u) {
       EXPECT_EQ(getMIBAllocType(MIB), AllocationType::NotCold);
-    } else {
-      ASSERT_EQ(StackId->getZExtValue(), 4u);
-      EXPECT_EQ(getMIBAllocType(MIB), AllocationType::Hot);
     }
   }
 }
@@ -427,8 +445,8 @@ declare dso_local noalias noundef i8* @malloc(i64 noundef)
   // with the non-cold context {1, 5}.
   Trie.addCallStack(AllocationType::NotCold, {1, 5, 6});
   Trie.addCallStack(AllocationType::NotCold, {1, 5, 7});
-  // We should be able to trim the following two and combine into a single MIB
-  // with the hot context {1, 8}.
+  // These will be pruned as they are unnecessary to determine how to clone the
+  // cold allocation.
   Trie.addCallStack(AllocationType::Hot, {1, 8, 9});
   Trie.addCallStack(AllocationType::Hot, {1, 8, 10});
 
@@ -438,7 +456,7 @@ declare dso_local noalias noundef i8* @malloc(i64 noundef)
   EXPECT_FALSE(Call->hasFnAttr("memprof"));
   EXPECT_TRUE(Call->hasMetadata(LLVMContext::MD_memprof));
   MDNode *MemProfMD = Call->getMetadata(LLVMContext::MD_memprof);
-  ASSERT_EQ(MemProfMD->getNumOperands(), 3u);
+  ASSERT_EQ(MemProfMD->getNumOperands(), 2u);
   for (auto &MIBOp : MemProfMD->operands()) {
     MDNode *MIB = dyn_cast<MDNode>(MIBOp);
     MDNode *StackMD = getMIBStackNode(MIB);
@@ -451,13 +469,67 @@ declare dso_local noalias noundef i8* @malloc(i64 noundef)
       EXPECT_EQ(getMIBAllocType(MIB), AllocationType::Cold);
     else if (StackId->getZExtValue() == 5u)
       EXPECT_EQ(getMIBAllocType(MIB), AllocationType::NotCold);
-    else {
-      ASSERT_EQ(StackId->getZExtValue(), 8u);
-      EXPECT_EQ(getMIBAllocType(MIB), AllocationType::Hot);
-    }
   }
 }
 
+// Test to ensure that we prune NotCold contexts that are unneeded for
+// determining where Cold contexts need to be cloned to enable correct hinting.
+TEST_F(MemoryProfileInfoTest, PruneUnneededNotColdContexts) {
+  LLVMContext C;
+  std::unique_ptr<Module> M = makeLLVMModule(C,
+                                             R"IR(
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-pc-linux-gnu"
+define i32* @test() {
+entry:
+  %call = call noalias dereferenceable_or_null(40) i8* @malloc(i64 noundef 40)
+  %0 = bitcast i8* %call to i32*
+  ret i32* %0
+}
+declare dso_local noalias noundef i8* @malloc(i64 noundef)
+)IR");
+
+  Function *Func = M->getFunction("test");
+
+  CallStackTrie Trie;
+
+  Trie.addCallStack(AllocationType::Cold, {1, 2, 3, 4});
+  Trie.addCallStack(AllocationType::Cold, {1, 2, 3, 5, 6, 7});
+  // This NotCold context is needed to know where the above two Cold contexts
+  // must be cloned from:
+  Trie.addCallStack(AllocationType::NotCold, {1, 2, 3, 5, 6, 13});
+
+  Trie.addCallStack(AllocationType::Cold, {1, 2, 3, 8, 9, 10});
+  // This NotCold context is needed to know where the above Cold context must be
+  // cloned from:
+  Trie.addCallStack(AllocationType::NotCold, {1, 2, 3, 8, 9, 14});
+  // This NotCold context is not needed since the above is sufficient (we pick
+  // the first in sorted order).
+  Trie.addCallStack(AllocationType::NotCold, {1, 2, 3, 8, 9, 15});
+
+  // None of these NotCold contexts are needed as the Cold contexts they
+  // overlap with are covered by longer overlapping NotCold contexts.
+  Trie.addCallStack(AllocationType::NotCold, {1, 2, 3, 12});
+  Trie.addCallStack(AllocationType::NotCold, {1, 2, 11});
+  Trie.addCallStack(AllocationType::NotCold, {1, 16});
+
+  std::vector<std::pair<AllocationType, std::vector<unsigned>>> ExpectedVals = {
+      {AllocationType::Cold, {1, 2, 3, 4}},
+      {AllocationType::Cold, {1, 2, 3, 5, 6, 7}},
+      {AllocationType::NotCold, {1, 2, 3, 5, 6, 13}},
+      {AllocationType::Cold, {1, 2, 3, 8, 9, 10}},
+      {AllocationType::NotCold, {1, 2, 3, 8, 9, 14}}};
+
+  CallBase *Call = findCall(*Func, "call");
+  ASSERT_NE(Call, nullptr);
+  Trie.buildAndAttachMIBMetadata(Call);
+
+  EXPECT_FALSE(Call->hasFnAttr("memprof"));
+  EXPECT_TRUE(Call->hasMetadata(LLVMContext::MD_memprof));
+  MDNode *MemProfMD = Call->getMetadata(LLVMContext::MD_memprof);
+  EXPECT_THAT(MemProfMD, MemprofMetadataEquals(ExpectedVals));
+}
+
 // Test CallStackTrie::addCallStack interface taking memprof MIB metadata.
 // Check that allocations annotated with memprof metadata with a single
 // allocation type get simplified to an attribute.
@@ -577,11 +649,13 @@ declare dso_local noalias noundef i8* @malloc(i64 noundef)
   // with the cold context {1, 2}.
   // We should be able to trim the second two and combine into a single MIB
   // with the non-cold context {1, 5}.
+  // The hot allocations will be converted to NotCold and pruned as they
+  // are unnecessary to determine how to clone the cold allocation.
 
   EXPECT_FALSE(Call->hasFnAttr("memprof"));
   EXPECT_TRUE(Call->hasMetadata(LLVMContext::MD_memprof));
   MemProfMD = Call->getMetadata(LLVMContext::MD_memprof);
-  ASSERT_EQ(MemProfMD->getNumOperands(), 3u);
+  ASSERT_EQ(MemProfMD->getNumOperands(), 2u);
   for (auto &MIBOp : MemProfMD->operands()) {
     MDNode *MIB = dyn_cast<MDNode>(MIBOp);
     MDNode *StackMD = getMIBStackNode(MIB);
@@ -594,10 +668,6 @@ declare dso_local noalias noundef i8* @malloc(i64 noundef)
       EXPECT_EQ(getMIBAllocType(MIB), AllocationType::Cold);
     else if (StackId->getZExtValue() == 5u)
       EXPECT_EQ(getMIBAllocType(MIB), AllocationType::NotCold);
-    else {
-      ASSERT_EQ(StackId->getZExtValue(), 8u);
-      EXPECT_EQ(getMIBAllocType(MIB), AllocationType::Hot);
-    }
   }
 }
 
diff --git a/llvm/unittests/Analysis/ValueTrackingTest.cpp b/llvm/unittests/Analysis/ValueTrackingTest.cpp
index ee44aac45594d..50e5e0e6b2ff5 100644
--- a/llvm/unittests/Analysis/ValueTrackingTest.cpp
+++ b/llvm/unittests/Analysis/ValueTrackingTest.cpp
@@ -2679,6 +2679,41 @@ TEST_F(ComputeKnownBitsTest, ComputeKnownBitsAbsoluteSymbol) {
   EXPECT_EQ(0u, Known_0_256_Align8.countMinTrailingOnes());
 }
 
+TEST_F(ComputeKnownBitsTest, ComputeKnownBitsGEPExtendBeforeMul) {
+  // The index should be extended before multiplying with the scale.
+  parseAssembly(R"(
+    target datalayout = "p:16:16:16"
+
+    define void @test(i16 %arg) {
+      %and = and i16 %arg, u0x8000
+      %base = inttoptr i16 %and to ptr
+      %A = getelementptr i32, ptr %base, i8 80
+      ret void
+    }
+    )");
+  KnownBits Known = computeKnownBits(A, M->getDataLayout());
+  EXPECT_EQ(~320 & 0x7fff, Known.Zero);
+  EXPECT_EQ(320, Known.One);
+}
+
+TEST_F(ComputeKnownBitsTest, ComputeKnownBitsGEPOnlyIndexBits) {
+  // GEP should only affect the index width.
+  parseAssembly(R"(
+    target datalayout = "p:16:16:16:8"
+
+    define void @test(i16 %arg) {
+      %and = and i16 %arg, u0x8000
+      %or = or i16 %and, u0x00ff
+      %base = inttoptr i16 %or to ptr
+      %A = getelementptr i8, ptr %base, i8 1
+      ret void
+    }
+    )");
+  KnownBits Known = computeKnownBits(A, M->getDataLayout());
+  EXPECT_EQ(0x7fff, Known.Zero);
+  EXPECT_EQ(0, Known.One);
+}
+
 TEST_F(ValueTrackingTest, HaveNoCommonBitsSet) {
   {
     // Check for an inverted mask: (X & ~M) op (Y & M).
diff --git a/llvm/unittests/CMakeLists.txt b/llvm/unittests/CMakeLists.txt
index 81abce51b8939..12e229b1c3498 100644
--- a/llvm/unittests/CMakeLists.txt
+++ b/llvm/unittests/CMakeLists.txt
@@ -63,7 +63,9 @@ add_subdirectory(Support)
 add_subdirectory(TableGen)
 add_subdirectory(Target)
 add_subdirectory(TargetParser)
-add_subdirectory(Telemetry)
+if (LLVM_BUILD_TELEMETRY)
+  add_subdirectory(Telemetry)
+endif()
 add_subdirectory(Testing)
 add_subdirectory(TextAPI)
 add_subdirectory(Transforms)
diff --git a/llvm/unittests/CodeGen/MachineOperandTest.cpp b/llvm/unittests/CodeGen/MachineOperandTest.cpp
index 714d6b5c50265..3f3f48fcc7c58 100644
--- a/llvm/unittests/CodeGen/MachineOperandTest.cpp
+++ b/llvm/unittests/CodeGen/MachineOperandTest.cpp
@@ -69,10 +69,10 @@ TEST(MachineOperandTest, PrintRegisterMask) {
   ASSERT_TRUE(MO.getRegMask() == Dummy);
 
   // Print a MachineOperand containing a RegMask. Here we check that without a
-  // TRI and IntrinsicInfo we still print a less detailed regmask.
+  // TRI we still print a less detailed regmask.
   std::string str;
   raw_string_ostream OS(str);
-  MO.print(OS, /*TRI=*/nullptr, /*IntrinsicInfo=*/nullptr);
+  MO.print(OS, /*TRI=*/nullptr);
   ASSERT_TRUE(str == "<regmask ...>");
 }
 
@@ -90,10 +90,10 @@ TEST(MachineOperandTest, PrintSubReg) {
   ASSERT_TRUE(MO.getSubReg() == 5);
 
   // Print a MachineOperand containing a SubReg. Here we check that without a
-  // TRI and IntrinsicInfo we can still print the subreg index.
+  // TRI we can still print the subreg index.
   std::string str;
   raw_string_ostream OS(str);
-  MO.print(OS, /*TRI=*/nullptr, /*IntrinsicInfo=*/nullptr);
+  MO.print(OS, /*TRI=*/nullptr);
   ASSERT_TRUE(str == "$physreg1.subreg5");
 }
 
@@ -112,10 +112,10 @@ TEST(MachineOperandTest, PrintCImm) {
   ASSERT_TRUE(MO.getCImm()->getValue() == Int);
 
   // Print a MachineOperand containing a SubReg. Here we check that without a
-  // TRI and IntrinsicInfo we can still print the subreg index.
+  // TRI we can still print the subreg index.
   std::string str;
   raw_string_ostream OS(str);
-  MO.print(OS, /*TRI=*/nullptr, /*IntrinsicInfo=*/nullptr);
+  MO.print(OS, /*TRI=*/nullptr);
   ASSERT_TRUE(str == "i128 18446744073709551616");
 }
 
@@ -129,7 +129,7 @@ TEST(MachineOperandTest, PrintSubRegIndex) {
   ASSERT_TRUE(MO.getImm() == 3);
 
   // Print a MachineOperand containing a SubRegIdx. Here we check that without a
-  // TRI and IntrinsicInfo we can print the operand as a subreg index.
+  // TRI we can print the operand as a subreg index.
   std::string str;
   raw_string_ostream OS(str);
   MachineOperand::printSubRegIdx(OS, MO.getImm(), nullptr);
@@ -151,7 +151,7 @@ TEST(MachineOperandTest, PrintCPI) {
   std::string str;
   {
     raw_string_ostream OS(str);
-    MO.print(OS, /*TRI=*/nullptr, /*IntrinsicInfo=*/nullptr);
+    MO.print(OS, /*TRI=*/nullptr);
     ASSERT_TRUE(str == "%const.0 + 8");
   }
 
@@ -163,7 +163,7 @@ TEST(MachineOperandTest, PrintCPI) {
   // offset.
   {
     raw_string_ostream OS(str);
-    MO.print(OS, /*TRI=*/nullptr, /*IntrinsicInfo=*/nullptr);
+    MO.print(OS, /*TRI=*/nullptr);
     ASSERT_TRUE(str == "%const.0 - 12");
   }
 }
@@ -182,7 +182,7 @@ TEST(MachineOperandTest, PrintTargetIndexName) {
   std::string str;
   {
     raw_string_ostream OS(str);
-    MO.print(OS, /*TRI=*/nullptr, /*IntrinsicInfo=*/nullptr);
+    MO.print(OS, /*TRI=*/nullptr);
     ASSERT_TRUE(str == "target-index(<unknown>) + 8");
   }
 
@@ -193,7 +193,7 @@ TEST(MachineOperandTest, PrintTargetIndexName) {
   // Print a MachineOperand containing a target index and a negative offset.
   {
     raw_string_ostream OS(str);
-    MO.print(OS, /*TRI=*/nullptr, /*IntrinsicInfo=*/nullptr);
+    MO.print(OS, /*TRI=*/nullptr);
     ASSERT_TRUE(str == "target-index(<unknown>) - 12");
   }
 }
@@ -210,7 +210,7 @@ TEST(MachineOperandTest, PrintJumpTableIndex) {
   // Print a MachineOperand containing a jump-table index.
   std::string str;
   raw_string_ostream OS(str);
-  MO.print(OS, /*TRI=*/nullptr, /*IntrinsicInfo=*/nullptr);
+  MO.print(OS, /*TRI=*/nullptr);
   ASSERT_TRUE(str == "%jump-table.3");
 }
 
@@ -227,7 +227,7 @@ TEST(MachineOperandTest, PrintExternalSymbol) {
   std::string str;
   {
     raw_string_ostream OS(str);
-    MO.print(OS, /*TRI=*/nullptr, /*IntrinsicInfo=*/nullptr);
+    MO.print(OS, /*TRI=*/nullptr);
     ASSERT_TRUE(str == "&foo");
   }
 
@@ -237,7 +237,7 @@ TEST(MachineOperandTest, PrintExternalSymbol) {
   // Print a MachineOperand containing an external symbol and a positive offset.
   {
     raw_string_ostream OS(str);
-    MO.print(OS, /*TRI=*/nullptr, /*IntrinsicInfo=*/nullptr);
+    MO.print(OS, /*TRI=*/nullptr);
     ASSERT_TRUE(str == "&foo + 12");
   }
 
@@ -247,7 +247,7 @@ TEST(MachineOperandTest, PrintExternalSymbol) {
   // Print a MachineOperand containing an external symbol and a negative offset.
   {
     raw_string_ostream OS(str);
-    MO.print(OS, /*TRI=*/nullptr, /*IntrinsicInfo=*/nullptr);
+    MO.print(OS, /*TRI=*/nullptr);
     ASSERT_TRUE(str == "&foo - 12");
   }
 }
@@ -273,7 +273,7 @@ TEST(MachineOperandTest, PrintGlobalAddress) {
   // Print a MachineOperand containing a global address and a positive offset.
   {
     raw_string_ostream OS(str);
-    MO.print(OS, /*TRI=*/nullptr, /*IntrinsicInfo=*/nullptr);
+    MO.print(OS, /*TRI=*/nullptr);
     ASSERT_TRUE(str == "@foo + 12");
   }
 
@@ -283,7 +283,7 @@ TEST(MachineOperandTest, PrintGlobalAddress) {
   // Print a MachineOperand containing a global address and a negative offset.
   {
     raw_string_ostream OS(str);
-    MO.print(OS, /*TRI=*/nullptr, /*IntrinsicInfo=*/nullptr);
+    MO.print(OS, /*TRI=*/nullptr);
     ASSERT_TRUE(str == "@foo - 12");
   }
 }
@@ -301,7 +301,7 @@ TEST(MachineOperandTest, PrintRegisterLiveOut) {
   std::string str;
   // Print a MachineOperand containing a register live out list without a TRI.
   raw_string_ostream OS(str);
-  MO.print(OS, /*TRI=*/nullptr, /*IntrinsicInfo=*/nullptr);
+  MO.print(OS, /*TRI=*/nullptr);
   ASSERT_TRUE(str == "liveout(<unknown>)");
 }
 
@@ -325,9 +325,9 @@ TEST(MachineOperandTest, PrintMetadata) {
   std::string str;
   // Print a MachineOperand containing a metadata node.
   raw_string_ostream OS(str);
-  MO.print(OS, MST, LLT{}, /*OpIdx*/~0U, /*PrintDef=*/false, /*IsStandalone=*/false,
-           /*ShouldPrintRegisterTies=*/false, 0, /*TRI=*/nullptr,
-           /*IntrinsicInfo=*/nullptr);
+  MO.print(OS, MST, LLT{}, /*OpIdx*/ ~0U, /*PrintDef=*/false,
+           /*IsStandalone=*/false,
+           /*ShouldPrintRegisterTies=*/false, 0, /*TRI=*/nullptr);
   ASSERT_TRUE(str == "!0");
 }
 
@@ -348,7 +348,7 @@ TEST(MachineOperandTest, PrintMCSymbol) {
   std::string str;
   // Print a MachineOperand containing a metadata node.
   raw_string_ostream OS(str);
-  MO.print(OS, /*TRI=*/nullptr, /*IntrinsicInfo=*/nullptr);
+  MO.print(OS, /*TRI=*/nullptr);
   ASSERT_TRUE(str == "<mcsymbol foo>");
 }
 
@@ -365,7 +365,7 @@ TEST(MachineOperandTest, PrintCFI) {
   // Print a MachineOperand containing a CFI Index node but no machine function
   // attached to it.
   raw_string_ostream OS(str);
-  MO.print(OS, /*TRI=*/nullptr, /*IntrinsicInfo=*/nullptr);
+  MO.print(OS, /*TRI=*/nullptr);
   ASSERT_TRUE(str == "<cfi directive>");
 }
 
@@ -382,7 +382,7 @@ TEST(MachineOperandTest, PrintIntrinsicID) {
   {
     // Print a MachineOperand containing a generic intrinsic ID.
     raw_string_ostream OS(str);
-    MO.print(OS, /*TRI=*/nullptr, /*IntrinsicInfo=*/nullptr);
+    MO.print(OS, /*TRI=*/nullptr);
     ASSERT_TRUE(str == "intrinsic(@llvm.bswap)");
   }
 
@@ -393,7 +393,7 @@ TEST(MachineOperandTest, PrintIntrinsicID) {
     // Print a MachineOperand containing a target-specific intrinsic ID but not
     // IntrinsicInfo.
     raw_string_ostream OS(str);
-    MO.print(OS, /*TRI=*/nullptr, /*IntrinsicInfo=*/nullptr);
+    MO.print(OS, /*TRI=*/nullptr);
     ASSERT_TRUE(str == "intrinsic(4294967295)");
   }
 }
@@ -410,7 +410,7 @@ TEST(MachineOperandTest, PrintPredicate) {
   std::string str;
   // Print a MachineOperand containing a int predicate ICMP_EQ.
   raw_string_ostream OS(str);
-  MO.print(OS, /*TRI=*/nullptr, /*IntrinsicInfo=*/nullptr);
+  MO.print(OS, /*TRI=*/nullptr);
   ASSERT_TRUE(str == "intpred(eq)");
 }
 
diff --git a/llvm/unittests/DebugInfo/DWARF/DWARFDebugLineTest.cpp b/llvm/unittests/DebugInfo/DWARF/DWARFDebugLineTest.cpp
index e549128031744..2fe52600df923 100644
--- a/llvm/unittests/DebugInfo/DWARF/DWARFDebugLineTest.cpp
+++ b/llvm/unittests/DebugInfo/DWARF/DWARFDebugLineTest.cpp
@@ -6,10 +6,10 @@
 //
 //===----------------------------------------------------------------------===//
 
+#include "llvm/DebugInfo/DWARF/DWARFDebugLine.h"
 #include "DwarfGenerator.h"
 #include "DwarfUtils.h"
 #include "llvm/DebugInfo/DWARF/DWARFContext.h"
-#include "llvm/DebugInfo/DWARF/DWARFDebugLine.h"
 #include "llvm/Object/ObjectFile.h"
 #include "llvm/Testing/Support/Error.h"
 #include "gtest/gtest.h"
@@ -2035,4 +2035,117 @@ TEST_F(DebugLineBasicFixture, PrintPathsProperly) {
   EXPECT_THAT(Result.c_str(), MatchesRegex("a dir.b dir.b file"));
 }
 
+/// Test that lookupAddressRange correctly filters rows based on
+/// a statement-sequence offset (simulating DW_AT_LLVM_stmt_sequence).
+///
+/// This test verifies that:
+/// 1. When a statement-sequence offset is provided, lookupAddressRange
+///    only returns rows from the sequence starting at that offset.
+/// 2. When an invalid statement-sequence offset is provided, no rows
+///    are returned.
+/// 3. When no statement-sequence offset is provided, all matching rows
+///    in the table are returned.
+///
+/// We build a line table with two sequences at the same address range
+/// but different line numbers. Then we try lookups with various statement-
+/// sequence offsets to check the filtering logic.
+TEST_F(DebugLineBasicFixture, LookupAddressRangeWithStmtSequenceOffset) {
+  if (!setupGenerator())
+    GTEST_SKIP();
+
+  // Create a line table that has two sequences covering [0x1000, 0x1004).
+  // Each sequence has two rows: addresses at 0x1000 and 0x1004, but
+  // they differ by line numbers (100 vs. 200, etc.).
+  //
+  // We'll pretend the first sequence starts at offset 0x2e in the line table,
+  // the second at 0x42, and we'll also test an invalid offset 0x66.
+
+  LineTable &LT = Gen->addLineTable();
+
+  // First sequence at offset 0x2e: addresses 0x1000(Ln=100), 0x1004(Ln=101)
+  LT.addExtendedOpcode(9, DW_LNE_set_address, {{0x1000U, LineTable::Quad}});
+  LT.addStandardOpcode(DW_LNS_set_prologue_end, {});
+  // Advance the line register by 99 (so line=100) and copy.
+  LT.addStandardOpcode(DW_LNS_advance_line, {{99, LineTable::SLEB}});
+  LT.addStandardOpcode(DW_LNS_copy, {});
+  // 0x4b is a special opcode: address += 4, line += 1 (so line=101).
+  LT.addByte(0x4b);
+  // End this sequence.
+  LT.addExtendedOpcode(1, DW_LNE_end_sequence, {});
+
+  // Second sequence at offset 0x42: addresses 0x1000(Ln=200), 0x1004(Ln=201)
+  LT.addExtendedOpcode(9, DW_LNE_set_address, {{0x1000U, LineTable::Quad}});
+  LT.addStandardOpcode(DW_LNS_set_prologue_end, {});
+  LT.addStandardOpcode(DW_LNS_advance_line, {{199, LineTable::SLEB}});
+  LT.addStandardOpcode(DW_LNS_copy, {});
+  // 0x4b again: address += 4, line += 1 (so line=201).
+  LT.addByte(0x4b);
+  // End this second sequence.
+  LT.addExtendedOpcode(1, DW_LNE_end_sequence, {});
+
+  // Generate the DWARF data.
+  generate();
+
+  // Parse the line table.
+  auto ExpectedLineTable =
+      Line.getOrParseLineTable(LineData, /*Offset=*/0, *Context,
+                               /*DwarfUnit=*/nullptr, RecordRecoverable);
+  ASSERT_THAT_EXPECTED(ExpectedLineTable, Succeeded());
+  const auto *Table = *ExpectedLineTable;
+
+  // The table should have two sequences, each starting at our chosen offsets.
+  ASSERT_EQ(Table->Sequences.size(), 2u);
+
+  // 1) Try looking up with an invalid offset (simulating an invalid
+  //    DW_AT_LLVM_stmt_sequence). We expect no rows.
+  {
+    std::vector<uint32_t> Rows;
+    bool Found = Table->lookupAddressRange(
+        {0x1000, object::SectionedAddress::UndefSection}, /*Size=*/1, Rows,
+        /*StmtSequenceOffset=*/0x66); // invalid offset
+    EXPECT_FALSE(Found);
+    EXPECT_TRUE(Rows.empty());
+  }
+
+  // 2) Look up using the offset 0x2e (our first sequence). We expect
+  //    to get the rows from that sequence only (which for 0x1000 is row #0).
+  {
+    std::vector<uint32_t> Rows;
+    bool Found = Table->lookupAddressRange(
+        {0x1000, object::SectionedAddress::UndefSection}, /*Size=*/1, Rows,
+        /*StmtSequenceOffset=*/0x2e);
+    EXPECT_TRUE(Found);
+    ASSERT_EQ(Rows.size(), 1u);
+    // The first sequence's first row is index 0.
+    EXPECT_EQ(Rows[0], 0u);
+  }
+
+  // 3) Look up using the offset 0x42 (second sequence). For address 0x1000
+  //    in that second sequence, we should see row #2.
+  {
+    std::vector<uint32_t> Rows;
+    bool Found = Table->lookupAddressRange(
+        {0x1000, object::SectionedAddress::UndefSection}, /*Size=*/1, Rows,
+        /*StmtSequenceOffset=*/0x42);
+    EXPECT_TRUE(Found);
+    ASSERT_EQ(Rows.size(), 1u);
+    // The second sequence's first row is index 2 in the table.
+    EXPECT_EQ(Rows[0], 3u);
+  }
+
+  // 4) Look up with no statement-sequence offset specified.
+  //    We should get rows from both sequences for address 0x1000.
+  {
+    std::vector<uint32_t> Rows;
+    bool Found = Table->lookupAddressRange(
+        {0x1000, object::SectionedAddress::UndefSection}, /*Size=*/1, Rows,
+        std::nullopt /* no filter */);
+    EXPECT_TRUE(Found);
+    // The first sequence's row is #0, second's row is #2, so both should
+    // appear.
+    ASSERT_EQ(Rows.size(), 2u);
+    EXPECT_EQ(Rows[0], 0u);
+    EXPECT_EQ(Rows[1], 3u);
+  }
+}
 } // end anonymous namespace
diff --git a/llvm/unittests/ExecutionEngine/JITLink/LinkGraphTests.cpp b/llvm/unittests/ExecutionEngine/JITLink/LinkGraphTests.cpp
index fb60acddf7821..c760873493528 100644
--- a/llvm/unittests/ExecutionEngine/JITLink/LinkGraphTests.cpp
+++ b/llvm/unittests/ExecutionEngine/JITLink/LinkGraphTests.cpp
@@ -54,6 +54,36 @@ TEST(LinkGraphTest, AddressAccess) {
   EXPECT_EQ(B1.getFixupAddress(E1), B1Addr + 8) << "Incorrect fixup address";
 }
 
+TEST(LinkGraphTest, DefinedSymbolProperties) {
+  // Check that Section::empty behaves as expected.
+  LinkGraph G("foo", std::make_shared<orc::SymbolStringPool>(),
+              Triple("x86_64-apple-darwin"), SubtargetFeatures(),
+              getGenericEdgeKindName);
+  auto &Sec =
+      G.createSection("__data", orc::MemProt::Read | orc::MemProt::Write);
+  auto &B =
+      G.createContentBlock(Sec, BlockContent, orc::ExecutorAddr(0x1000), 8, 0);
+  auto &S = G.addDefinedSymbol(B, 0, "sym", 4, Linkage::Strong, Scope::Default,
+                               false, false);
+
+  EXPECT_TRUE(S.hasName());
+  EXPECT_EQ(*S.getName(), "sym");
+  EXPECT_TRUE(S.isDefined());
+  EXPECT_FALSE(S.isLive());
+  EXPECT_FALSE(S.isCallable());
+  EXPECT_FALSE(S.isExternal());
+  EXPECT_FALSE(S.isAbsolute());
+  EXPECT_EQ(&S.getBlock(), &B);
+  EXPECT_EQ(&S.getSection(), &Sec);
+  EXPECT_EQ(S.getOffset(), 0U);
+  EXPECT_EQ(S.getSize(), 4U);
+  EXPECT_EQ(S.getRange(), orc::ExecutorAddrRange(B.getAddress(), 4));
+  EXPECT_EQ(S.getSymbolContent(), BlockContent.slice(0, 4));
+  EXPECT_EQ(S.getLinkage(), Linkage::Strong);
+  EXPECT_EQ(S.getScope(), Scope::Default);
+  EXPECT_EQ(S.getTargetFlags(), 0U);
+}
+
 TEST(LinkGraphTest, SectionEmpty) {
   // Check that Section::empty behaves as expected.
   LinkGraph G("foo", std::make_shared<orc::SymbolStringPool>(),
diff --git a/llvm/unittests/Frontend/OpenMPContextTest.cpp b/llvm/unittests/Frontend/OpenMPContextTest.cpp
index 1c999a2d7bf85..f9683ae56e933 100644
--- a/llvm/unittests/Frontend/OpenMPContextTest.cpp
+++ b/llvm/unittests/Frontend/OpenMPContextTest.cpp
@@ -33,9 +33,11 @@ TEST_F(OpenMPContextTest, RoundTripAndAssociation) {
 #define OMP_TRAIT_SELECTOR(Enum, TraitSetEnum, Str, RequiresProperty)          \
   EXPECT_EQ(TraitSelector::Enum,                                               \
             getOpenMPContextTraitSelectorKind(                                 \
-                getOpenMPContextTraitSelectorName(TraitSelector::Enum)));      \
+                getOpenMPContextTraitSelectorName(TraitSelector::Enum),        \
+                TraitSet::TraitSetEnum));                                      \
   EXPECT_EQ(Str, getOpenMPContextTraitSelectorName(                            \
-                     getOpenMPContextTraitSelectorKind(Str)));
+                     getOpenMPContextTraitSelectorKind(                        \
+                         Str, TraitSet::TraitSetEnum)));
 #define OMP_TRAIT_PROPERTY(Enum, TraitSetEnum, TraitSelectorEnum, Str)         \
   EXPECT_EQ(TraitProperty::Enum,                                               \
             getOpenMPContextTraitPropertyKind(                                 \
@@ -68,10 +70,10 @@ TEST_F(OpenMPContextTest, ValidNesting) {
 }
 
 TEST_F(OpenMPContextTest, ApplicabilityNonConstruct) {
-  OMPContext HostLinux(false, Triple("x86_64-unknown-linux"));
-  OMPContext DeviceLinux(true, Triple("x86_64-unknown-linux"));
-  OMPContext HostNVPTX(false, Triple("nvptx64-nvidia-cuda"));
-  OMPContext DeviceNVPTX(true, Triple("nvptx64-nvidia-cuda"));
+  OMPContext HostLinux(false, Triple("x86_64-unknown-linux"), Triple(), -1);
+  OMPContext DeviceLinux(true, Triple("x86_64-unknown-linux"), Triple(), -1);
+  OMPContext HostNVPTX(false, Triple("nvptx64-nvidia-cuda"), Triple(), -1);
+  OMPContext DeviceNVPTX(true, Triple("nvptx64-nvidia-cuda"), Triple(), -1);
 
   VariantMatchInfo Empty;
   EXPECT_TRUE(isVariantApplicableInContext(Empty, HostLinux));
@@ -129,19 +131,21 @@ TEST_F(OpenMPContextTest, ApplicabilityNonConstruct) {
 }
 
 TEST_F(OpenMPContextTest, ApplicabilityAllTraits) {
-  OMPContext HostLinuxParallelParallel(false, Triple("x86_64-unknown-linux"));
+  OMPContext HostLinuxParallelParallel(false, Triple("x86_64-unknown-linux"),
+                                       Triple(), -1);
   HostLinuxParallelParallel.addTrait(
       TraitProperty::construct_parallel_parallel);
   HostLinuxParallelParallel.addTrait(
       TraitProperty::construct_parallel_parallel);
-  OMPContext DeviceLinuxTargetParallel(true, Triple("x86_64-unknown-linux"));
+  OMPContext DeviceLinuxTargetParallel(true, Triple("x86_64-unknown-linux"),
+                                       Triple(), -1);
   DeviceLinuxTargetParallel.addTrait(TraitProperty::construct_target_target);
   DeviceLinuxTargetParallel.addTrait(
       TraitProperty::construct_parallel_parallel);
-  OMPContext HostNVPTXFor(false, Triple("nvptx64-nvidia-cuda"));
+  OMPContext HostNVPTXFor(false, Triple("nvptx64-nvidia-cuda"), Triple(), -1);
   HostNVPTXFor.addTrait(TraitProperty::construct_for_for);
-  OMPContext DeviceNVPTXTargetTeamsParallel(true,
-                                            Triple("nvptx64-nvidia-cuda"));
+  OMPContext DeviceNVPTXTargetTeamsParallel(true, Triple("nvptx64-nvidia-cuda"),
+                                            Triple(), -1);
   DeviceNVPTXTargetTeamsParallel.addTrait(
       TraitProperty::construct_target_target);
   DeviceNVPTXTargetTeamsParallel.addTrait(TraitProperty::construct_teams_teams);
diff --git a/llvm/unittests/Frontend/OpenMPIRBuilderTest.cpp b/llvm/unittests/Frontend/OpenMPIRBuilderTest.cpp
index 42616155f0cc3..83c8f7e932b2b 100644
--- a/llvm/unittests/Frontend/OpenMPIRBuilderTest.cpp
+++ b/llvm/unittests/Frontend/OpenMPIRBuilderTest.cpp
@@ -4643,9 +4643,11 @@ TEST_F(OpenMPIRBuilderTest, CreateTeamsWithThreadLimit) {
       dyn_cast<BranchInst>(PushNumTeamsCallInst->getNextNonDebugInstruction());
   ASSERT_NE(BrInst, nullptr);
   ASSERT_EQ(BrInst->getNumSuccessors(), 1U);
-  Instruction *NextInstruction =
+  BasicBlock::iterator NextInstruction =
       BrInst->getSuccessor(0)->getFirstNonPHIOrDbgOrLifetime();
-  CallInst *ForkTeamsCI = dyn_cast_if_present<CallInst>(NextInstruction);
+  CallInst *ForkTeamsCI = nullptr;
+  if (NextInstruction != BrInst->getSuccessor(0)->end())
+    ForkTeamsCI = dyn_cast_if_present<CallInst>(NextInstruction);
   ASSERT_NE(ForkTeamsCI, nullptr);
   EXPECT_EQ(ForkTeamsCI->getCalledFunction(),
             OMPBuilder.getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_fork_teams));
@@ -4698,9 +4700,11 @@ TEST_F(OpenMPIRBuilderTest, CreateTeamsWithNumTeamsUpper) {
       dyn_cast<BranchInst>(PushNumTeamsCallInst->getNextNonDebugInstruction());
   ASSERT_NE(BrInst, nullptr);
   ASSERT_EQ(BrInst->getNumSuccessors(), 1U);
-  Instruction *NextInstruction =
+  BasicBlock::iterator NextInstruction =
       BrInst->getSuccessor(0)->getFirstNonPHIOrDbgOrLifetime();
-  CallInst *ForkTeamsCI = dyn_cast_if_present<CallInst>(NextInstruction);
+  CallInst *ForkTeamsCI = nullptr;
+  if (NextInstruction != BrInst->getSuccessor(0)->end())
+    ForkTeamsCI = dyn_cast_if_present<CallInst>(NextInstruction);
   ASSERT_NE(ForkTeamsCI, nullptr);
   EXPECT_EQ(ForkTeamsCI->getCalledFunction(),
             OMPBuilder.getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_fork_teams));
@@ -4756,9 +4760,11 @@ TEST_F(OpenMPIRBuilderTest, CreateTeamsWithNumTeamsBoth) {
       dyn_cast<BranchInst>(PushNumTeamsCallInst->getNextNonDebugInstruction());
   ASSERT_NE(BrInst, nullptr);
   ASSERT_EQ(BrInst->getNumSuccessors(), 1U);
-  Instruction *NextInstruction =
+  BasicBlock::iterator NextInstruction =
       BrInst->getSuccessor(0)->getFirstNonPHIOrDbgOrLifetime();
-  CallInst *ForkTeamsCI = dyn_cast_if_present<CallInst>(NextInstruction);
+  CallInst *ForkTeamsCI = nullptr;
+  if (NextInstruction != BrInst->getSuccessor(0)->end())
+    ForkTeamsCI = dyn_cast_if_present<CallInst>(NextInstruction);
   ASSERT_NE(ForkTeamsCI, nullptr);
   EXPECT_EQ(ForkTeamsCI->getCalledFunction(),
             OMPBuilder.getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_fork_teams));
@@ -4820,9 +4826,11 @@ TEST_F(OpenMPIRBuilderTest, CreateTeamsWithNumTeamsAndThreadLimit) {
       dyn_cast<BranchInst>(PushNumTeamsCallInst->getNextNonDebugInstruction());
   ASSERT_NE(BrInst, nullptr);
   ASSERT_EQ(BrInst->getNumSuccessors(), 1U);
-  Instruction *NextInstruction =
+  BasicBlock::iterator NextInstruction =
       BrInst->getSuccessor(0)->getFirstNonPHIOrDbgOrLifetime();
-  CallInst *ForkTeamsCI = dyn_cast_if_present<CallInst>(NextInstruction);
+  CallInst *ForkTeamsCI = nullptr;
+  if (NextInstruction != BrInst->getSuccessor(0)->end())
+    ForkTeamsCI = dyn_cast_if_present<CallInst>(NextInstruction);
   ASSERT_NE(ForkTeamsCI, nullptr);
   EXPECT_EQ(ForkTeamsCI->getCalledFunction(),
             OMPBuilder.getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_fork_teams));
@@ -6172,6 +6180,7 @@ TEST_F(OpenMPIRBuilderTest, TargetRegion) {
   F->addFnAttr("target-features", "+mmx,+sse");
   IRBuilder<> Builder(BB);
   auto *Int32Ty = Builder.getInt32Ty();
+  Builder.SetCurrentDebugLocation(DL);
 
   AllocaInst *APtr = Builder.CreateAlloca(Int32Ty, nullptr, "a_ptr");
   AllocaInst *BPtr = Builder.CreateAlloca(Int32Ty, nullptr, "b_ptr");
@@ -6181,6 +6190,8 @@ TEST_F(OpenMPIRBuilderTest, TargetRegion) {
   Builder.CreateStore(Builder.getInt32(20), BPtr);
   auto BodyGenCB = [&](InsertPointTy AllocaIP,
                        InsertPointTy CodeGenIP) -> InsertPointTy {
+    IRBuilderBase::InsertPointGuard guard(Builder);
+    Builder.SetCurrentDebugLocation(llvm::DebugLoc());
     Builder.restoreIP(CodeGenIP);
     LoadInst *AVal = Builder.CreateLoad(Int32Ty, APtr);
     LoadInst *BVal = Builder.CreateLoad(Int32Ty, BPtr);
@@ -6198,6 +6209,8 @@ TEST_F(OpenMPIRBuilderTest, TargetRegion) {
       [&](llvm::Argument &Arg, llvm::Value *Input, llvm::Value *&RetVal,
           llvm::OpenMPIRBuilder::InsertPointTy AllocaIP,
           llvm::OpenMPIRBuilder::InsertPointTy CodeGenIP) {
+        IRBuilderBase::InsertPointGuard guard(Builder);
+        Builder.SetCurrentDebugLocation(llvm::DebugLoc());
         if (!OMPBuilder.Config.isTargetDevice()) {
           RetVal = cast<llvm::Value>(&Arg);
           return CodeGenIP;
@@ -6244,6 +6257,7 @@ TEST_F(OpenMPIRBuilderTest, TargetRegion) {
                               Builder.saveIP(), EntryInfo, DefaultAttrs,
                               RuntimeAttrs, /*IfCond=*/nullptr, Inputs,
                               GenMapInfoCB, BodyGenCB, SimpleArgAccessorCB));
+  EXPECT_EQ(DL, Builder.getCurrentDebugLocation());
   Builder.restoreIP(AfterIP);
 
   OMPBuilder.finalize();
@@ -6342,6 +6356,7 @@ TEST_F(OpenMPIRBuilderTest, TargetRegionDevice) {
   F->addFnAttr("target-features", "+gfx9-insts,+wavefrontsize64");
   IRBuilder<> Builder(BB);
   OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});
+  Builder.SetCurrentDebugLocation(DL);
 
   LoadInst *Value = nullptr;
   StoreInst *TargetStore = nullptr;
@@ -6353,6 +6368,8 @@ TEST_F(OpenMPIRBuilderTest, TargetRegionDevice) {
       [&](llvm::Argument &Arg, llvm::Value *Input, llvm::Value *&RetVal,
           llvm::OpenMPIRBuilder::InsertPointTy AllocaIP,
           llvm::OpenMPIRBuilder::InsertPointTy CodeGenIP) {
+        IRBuilderBase::InsertPointGuard guard(Builder);
+        Builder.SetCurrentDebugLocation(llvm::DebugLoc());
         if (!OMPBuilder.Config.isTargetDevice()) {
           RetVal = cast<llvm::Value>(&Arg);
           return CodeGenIP;
@@ -6386,6 +6403,8 @@ TEST_F(OpenMPIRBuilderTest, TargetRegionDevice) {
   auto BodyGenCB = [&](OpenMPIRBuilder::InsertPointTy AllocaIP,
                        OpenMPIRBuilder::InsertPointTy CodeGenIP)
       -> OpenMPIRBuilder::InsertPointTy {
+    IRBuilderBase::InsertPointGuard guard(Builder);
+    Builder.SetCurrentDebugLocation(llvm::DebugLoc());
     Builder.restoreIP(CodeGenIP);
     Value = Builder.CreateLoad(Type::getInt32Ty(Ctx), CapturedArgs[0]);
     TargetStore = Builder.CreateStore(Value, CapturedArgs[1]);
@@ -6407,6 +6426,7 @@ TEST_F(OpenMPIRBuilderTest, TargetRegionDevice) {
                               EntryInfo, DefaultAttrs, RuntimeAttrs,
                               /*IfCond=*/nullptr, CapturedArgs, GenMapInfoCB,
                               BodyGenCB, SimpleArgAccessorCB));
+  EXPECT_EQ(DL, Builder.getCurrentDebugLocation());
   Builder.restoreIP(AfterIP);
 
   Builder.CreateRetVoid();
@@ -6714,6 +6734,7 @@ TEST_F(OpenMPIRBuilderTest, ConstantAllocaRaise) {
   F->setName("func");
   IRBuilder<> Builder(BB);
   OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});
+  Builder.SetCurrentDebugLocation(DL);
 
   LoadInst *Value = nullptr;
   StoreInst *TargetStore = nullptr;
@@ -6724,6 +6745,8 @@ TEST_F(OpenMPIRBuilderTest, ConstantAllocaRaise) {
       [&](llvm::Argument &Arg, llvm::Value *Input, llvm::Value *&RetVal,
           llvm::OpenMPIRBuilder::InsertPointTy AllocaIP,
           llvm::OpenMPIRBuilder::InsertPointTy CodeGenIP) {
+        IRBuilderBase::InsertPointGuard guard(Builder);
+        Builder.SetCurrentDebugLocation(llvm::DebugLoc());
         if (!OMPBuilder.Config.isTargetDevice()) {
           RetVal = cast<llvm::Value>(&Arg);
           return CodeGenIP;
@@ -6759,6 +6782,8 @@ TEST_F(OpenMPIRBuilderTest, ConstantAllocaRaise) {
   auto BodyGenCB = [&](OpenMPIRBuilder::InsertPointTy AllocaIP,
                        OpenMPIRBuilder::InsertPointTy CodeGenIP)
       -> OpenMPIRBuilder::InsertPointTy {
+    IRBuilderBase::InsertPointGuard guard(Builder);
+    Builder.SetCurrentDebugLocation(llvm::DebugLoc());
     Builder.restoreIP(CodeGenIP);
     RaiseAlloca = Builder.CreateAlloca(Builder.getInt32Ty());
     Value = Builder.CreateLoad(Type::getInt32Ty(Ctx), CapturedArgs[0]);
@@ -6781,6 +6806,7 @@ TEST_F(OpenMPIRBuilderTest, ConstantAllocaRaise) {
                               EntryInfo, DefaultAttrs, RuntimeAttrs,
                               /*IfCond=*/nullptr, CapturedArgs, GenMapInfoCB,
                               BodyGenCB, SimpleArgAccessorCB));
+  EXPECT_EQ(DL, Builder.getCurrentDebugLocation());
   Builder.restoreIP(AfterIP);
 
   Builder.CreateRetVoid();
@@ -7622,29 +7648,25 @@ TEST_F(OpenMPIRBuilderTest, createGPUOffloadEntry) {
                                 /* Size = */ 0,
                                 /* Flags = */ 0, GlobalValue::WeakAnyLinkage);
 
-  // Check nvvm.annotations only created for GPU kernels
-  NamedMDNode *MD = M->getNamedMetadata("nvvm.annotations");
-  EXPECT_NE(MD, nullptr);
-  EXPECT_EQ(MD->getNumOperands(), 1u);
-
-  MDNode *Annotations = MD->getOperand(0);
-  EXPECT_EQ(Annotations->getNumOperands(), 3u);
-
-  Constant *ConstVal =
-      dyn_cast<ConstantAsMetadata>(Annotations->getOperand(0))->getValue();
-  EXPECT_TRUE(isa<Function>(Fn));
-  EXPECT_EQ(ConstVal, cast<Function>(Fn));
-
-  EXPECT_TRUE(Annotations->getOperand(1).equalsStr("kernel"));
-
-  EXPECT_TRUE(mdconst::hasa<ConstantInt>(Annotations->getOperand(2)));
-  APInt IntVal =
-      mdconst::extract<ConstantInt>(Annotations->getOperand(2))->getValue();
-  EXPECT_EQ(IntVal, 1);
-
   // Check kernel attributes
   EXPECT_TRUE(Fn->hasFnAttribute("kernel"));
   EXPECT_TRUE(Fn->hasFnAttribute(Attribute::MustProgress));
 }
 
+TEST_F(OpenMPIRBuilderTest, splitBB) {
+  OpenMPIRBuilder OMPBuilder(*M);
+  OMPBuilder.Config.IsTargetDevice = false;
+  OMPBuilder.initialize();
+  F->setName("func");
+  IRBuilder<> Builder(BB);
+
+  Builder.SetCurrentDebugLocation(DL);
+  AllocaInst *alloc = Builder.CreateAlloca(Builder.getInt32Ty());
+  EXPECT_TRUE(DL == alloc->getStableDebugLoc());
+  BasicBlock *AllocaBB = Builder.GetInsertBlock();
+  splitBB(Builder, /*CreateBranch=*/true, "test");
+  if (AllocaBB->getTerminator())
+    EXPECT_TRUE(DL == AllocaBB->getTerminator()->getStableDebugLoc());
+}
+
 } // namespace
diff --git a/llvm/unittests/IR/DebugTypeODRUniquingTest.cpp b/llvm/unittests/IR/DebugTypeODRUniquingTest.cpp
index 7180b8183c50a..7c673b0166188 100644
--- a/llvm/unittests/IR/DebugTypeODRUniquingTest.cpp
+++ b/llvm/unittests/IR/DebugTypeODRUniquingTest.cpp
@@ -30,8 +30,8 @@ TEST(DebugTypeODRUniquingTest, getODRType) {
   // Without a type map, this should return null.
   EXPECT_FALSE(DICompositeType::getODRType(
       Context, UUID, dwarf::DW_TAG_class_type, nullptr, nullptr, 0, nullptr,
-      nullptr, 0, 0, 0, nullptr, 0, DINode::FlagZero, nullptr, 0, nullptr,
-      nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr));
+      nullptr, 0, 0, 0, nullptr, 0, DINode::FlagZero, nullptr, 0, std::nullopt,
+      nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr));
 
   // Enable the mapping.  There still shouldn't be a type.
   Context.enableDebugTypeODRUniquing();
@@ -40,8 +40,8 @@ TEST(DebugTypeODRUniquingTest, getODRType) {
   // Create some ODR-uniqued type.
   auto &CT = *DICompositeType::getODRType(
       Context, UUID, dwarf::DW_TAG_class_type, nullptr, nullptr, 0, nullptr,
-      nullptr, 0, 0, 0, nullptr, 0, DINode::FlagZero, nullptr, 0, nullptr,
-      nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr);
+      nullptr, 0, 0, 0, nullptr, 0, DINode::FlagZero, nullptr, 0, std::nullopt,
+      nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr);
   EXPECT_EQ(UUID.getString(), CT.getIdentifier());
 
   // Check that we get it back, even if we change a field.
@@ -49,14 +49,14 @@ TEST(DebugTypeODRUniquingTest, getODRType) {
   EXPECT_EQ(&CT, DICompositeType::getODRType(
                      Context, UUID, dwarf::DW_TAG_class_type, nullptr, nullptr,
                      0, nullptr, nullptr, 0, 0, 0, nullptr, 0, DINode::FlagZero,
-                     nullptr, 0, nullptr, nullptr, nullptr, nullptr, nullptr,
+                     nullptr, 0, std::nullopt, nullptr, nullptr, nullptr,
+                     nullptr, nullptr, nullptr, nullptr, nullptr));
+  EXPECT_EQ(&CT, DICompositeType::getODRType(
+                     Context, UUID, dwarf::DW_TAG_class_type,
+                     MDString::get(Context, "name"), nullptr, 0, nullptr,
+                     nullptr, 0, 0, 0, nullptr, 0, DINode::FlagZero, nullptr, 0,
+                     std::nullopt, nullptr, nullptr, nullptr, nullptr, nullptr,
                      nullptr, nullptr, nullptr));
-  EXPECT_EQ(&CT,
-            DICompositeType::getODRType(
-                Context, UUID, dwarf::DW_TAG_class_type,
-                MDString::get(Context, "name"), nullptr, 0, nullptr, nullptr, 0,
-                0, 0, nullptr, 0, DINode::FlagZero, nullptr, 0, nullptr,
-                nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr));
 
   // Check that it's discarded with the type map.
   Context.disableDebugTypeODRUniquing();
@@ -75,43 +75,48 @@ TEST(DebugTypeODRUniquingTest, buildODRType) {
   MDString &UUID = *MDString::get(Context, "Type");
   auto &CT = *DICompositeType::buildODRType(
       Context, UUID, dwarf::DW_TAG_class_type, nullptr, nullptr, 0, nullptr,
-      nullptr, 0, 0, 0, nullptr, 0, DINode::FlagFwdDecl, nullptr, 0, nullptr,
-      nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr);
+      nullptr, 0, 0, 0, nullptr, 0, DINode::FlagFwdDecl, nullptr, 0,
+      std::nullopt, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr,
+      nullptr, nullptr);
   EXPECT_EQ(&CT, DICompositeType::getODRTypeIfExists(Context, UUID));
   EXPECT_EQ(dwarf::DW_TAG_class_type, CT.getTag());
 
   // Update with another forward decl.  This should be a no-op.
-  EXPECT_EQ(&CT, DICompositeType::buildODRType(
-                     Context, UUID, dwarf::DW_TAG_class_type, nullptr, nullptr,
-                     0, nullptr, nullptr, 0, 0, 0, nullptr, 0,
-                     DINode::FlagFwdDecl, nullptr, 0, nullptr, nullptr, nullptr,
-                     nullptr, nullptr, nullptr, nullptr, nullptr));
+  EXPECT_EQ(&CT,
+            DICompositeType::buildODRType(
+                Context, UUID, dwarf::DW_TAG_class_type, nullptr, nullptr, 0,
+                nullptr, nullptr, 0, 0, 0, nullptr, 0, DINode::FlagFwdDecl,
+                nullptr, 0, std::nullopt, nullptr, nullptr, nullptr, nullptr,
+                nullptr, nullptr, nullptr, nullptr));
 
   EXPECT_FALSE(DICompositeType::buildODRType(
       Context, UUID, dwarf::DW_TAG_structure_type, nullptr, nullptr, 0, nullptr,
-      nullptr, 0, 0, 0, nullptr, 0, DINode::FlagFwdDecl, nullptr, 0, nullptr,
-      nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr));
+      nullptr, 0, 0, 0, nullptr, 0, DINode::FlagFwdDecl, nullptr, 0,
+      std::nullopt, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr,
+      nullptr, nullptr));
 
   // Update with a definition.  This time we should see a change.
   EXPECT_EQ(&CT, DICompositeType::buildODRType(
                      Context, UUID, dwarf::DW_TAG_class_type, nullptr, nullptr,
                      0, nullptr, nullptr, 0, 0, 0, nullptr, 0, DINode::FlagZero,
-                     nullptr, 0, nullptr, nullptr, nullptr, nullptr, nullptr,
-                     nullptr, nullptr, nullptr));
+                     nullptr, 0, std::nullopt, nullptr, nullptr, nullptr,
+                     nullptr, nullptr, nullptr, nullptr, nullptr));
   EXPECT_FALSE(CT.isForwardDecl());
 
   // Further updates should be ignored.
-  EXPECT_EQ(&CT, DICompositeType::buildODRType(
-                     Context, UUID, dwarf::DW_TAG_class_type, nullptr, nullptr,
-                     0, nullptr, nullptr, 0, 0, 0, nullptr, 0,
-                     DINode::FlagFwdDecl, nullptr, 0, nullptr, nullptr, nullptr,
-                     nullptr, nullptr, nullptr, nullptr, nullptr));
+  EXPECT_EQ(&CT,
+            DICompositeType::buildODRType(
+                Context, UUID, dwarf::DW_TAG_class_type, nullptr, nullptr, 0,
+                nullptr, nullptr, 0, 0, 0, nullptr, 0, DINode::FlagFwdDecl,
+                nullptr, 0, std::nullopt, nullptr, nullptr, nullptr, nullptr,
+                nullptr, nullptr, nullptr, nullptr));
   EXPECT_FALSE(CT.isForwardDecl());
-  EXPECT_EQ(&CT, DICompositeType::buildODRType(
-                     Context, UUID, dwarf::DW_TAG_class_type, nullptr, nullptr,
-                     111u, nullptr, nullptr, 0, 0, 0, nullptr, 0,
-                     DINode::FlagZero, nullptr, 0, nullptr, nullptr, nullptr,
-                     nullptr, nullptr, nullptr, nullptr, nullptr));
+  EXPECT_EQ(&CT,
+            DICompositeType::buildODRType(
+                Context, UUID, dwarf::DW_TAG_class_type, nullptr, nullptr, 111u,
+                nullptr, nullptr, 0, 0, 0, nullptr, 0, DINode::FlagZero,
+                nullptr, 0, std::nullopt, nullptr, nullptr, nullptr, nullptr,
+                nullptr, nullptr, nullptr, nullptr));
   EXPECT_NE(111u, CT.getLine());
 }
 
@@ -123,8 +128,8 @@ TEST(DebugTypeODRUniquingTest, buildODRTypeFields) {
   MDString &UUID = *MDString::get(Context, "UUID");
   auto &CT = *DICompositeType::buildODRType(
       Context, UUID, 0, nullptr, nullptr, 0, nullptr, nullptr, 0, 0, 0, nullptr,
-      0, DINode::FlagFwdDecl, nullptr, 0, nullptr, nullptr, nullptr, nullptr,
-      nullptr, nullptr, nullptr, nullptr);
+      0, DINode::FlagFwdDecl, nullptr, 0, std::nullopt, nullptr, nullptr,
+      nullptr, nullptr, nullptr, nullptr, nullptr, nullptr);
 
 // Create macros for running through all the fields except Identifier and Flags.
 #define FOR_EACH_MDFIELD()                                                     \
@@ -141,7 +146,8 @@ TEST(DebugTypeODRUniquingTest, buildODRTypeFields) {
   DO_FOR_FIELD(AlignInBits)                                                    \
   DO_FOR_FIELD(OffsetInBits)                                                   \
   DO_FOR_FIELD(NumExtraInhabitants)                                            \
-  DO_FOR_FIELD(RuntimeLang)
+  DO_FOR_FIELD(RuntimeLang)                                                    \
+  DO_FOR_FIELD(EnumKind)
 
 // Create all the fields.
 #define DO_FOR_FIELD(X) auto *X = MDString::get(Context, #X);
@@ -157,8 +163,8 @@ TEST(DebugTypeODRUniquingTest, buildODRTypeFields) {
                      Context, UUID, 0, Name, File, Line, Scope, BaseType,
                      SizeInBits, AlignInBits, OffsetInBits, nullptr,
                      NumExtraInhabitants, DINode::FlagArtificial, Elements,
-                     RuntimeLang, VTableHolder, TemplateParams, nullptr,
-                     nullptr, nullptr, nullptr, nullptr, nullptr));
+                     RuntimeLang, EnumKind, VTableHolder, TemplateParams,
+                     nullptr, nullptr, nullptr, nullptr, nullptr, nullptr));
 
   // Confirm that all the right fields got updated.
 #define DO_FOR_FIELD(X) EXPECT_EQ(X, CT.getRaw##X());
diff --git a/llvm/unittests/IR/MetadataTest.cpp b/llvm/unittests/IR/MetadataTest.cpp
index 628221339c89b..8fe40a94ee546 100644
--- a/llvm/unittests/IR/MetadataTest.cpp
+++ b/llvm/unittests/IR/MetadataTest.cpp
@@ -115,9 +115,10 @@ class MetadataTest : public testing::Test {
     return ConstantAsMetadata::get(getConstant());
   }
   DIType *getCompositeType() {
-    return DICompositeType::getDistinct(
-        Context, dwarf::DW_TAG_structure_type, "", nullptr, 0, nullptr, nullptr,
-        32, 32, 0, DINode::FlagZero, nullptr, 0, nullptr, nullptr, "");
+    return DICompositeType::getDistinct(Context, dwarf::DW_TAG_structure_type,
+                                        "", nullptr, 0, nullptr, nullptr, 32,
+                                        32, 0, DINode::FlagZero, nullptr, 0,
+                                        std::nullopt, nullptr, nullptr, "");
   }
   Function *getFunction(StringRef Name) {
     return Function::Create(
@@ -2001,11 +2002,12 @@ TEST_F(DICompositeTypeTest, get) {
   DIType *VTableHolder = getCompositeType();
   MDTuple *TemplateParams = getTuple();
   StringRef Identifier = "some id";
+  std::optional<uint32_t> EnumKind = 1;
 
-  auto *N = DICompositeType::get(Context, Tag, Name, File, Line, Scope,
-                                 BaseType, SizeInBits, AlignInBits,
-                                 OffsetInBits, Flags, Elements, RuntimeLang,
-                                 VTableHolder, TemplateParams, Identifier);
+  auto *N = DICompositeType::get(
+      Context, Tag, Name, File, Line, Scope, BaseType, SizeInBits, AlignInBits,
+      OffsetInBits, Flags, Elements, RuntimeLang, EnumKind, VTableHolder,
+      TemplateParams, Identifier);
   EXPECT_EQ(Tag, N->getTag());
   EXPECT_EQ(Name, N->getName());
   EXPECT_EQ(File, N->getFile());
@@ -2021,84 +2023,91 @@ TEST_F(DICompositeTypeTest, get) {
   EXPECT_EQ(VTableHolder, N->getVTableHolder());
   EXPECT_EQ(TemplateParams, N->getTemplateParams().get());
   EXPECT_EQ(Identifier, N->getIdentifier());
+  EXPECT_EQ(EnumKind, N->getEnumKind());
 
-  EXPECT_EQ(N, DICompositeType::get(Context, Tag, Name, File, Line, Scope,
-                                    BaseType, SizeInBits, AlignInBits,
-                                    OffsetInBits, Flags, Elements, RuntimeLang,
-                                    VTableHolder, TemplateParams, Identifier));
+  EXPECT_EQ(N, DICompositeType::get(
+                   Context, Tag, Name, File, Line, Scope, BaseType, SizeInBits,
+                   AlignInBits, OffsetInBits, Flags, Elements, RuntimeLang,
+                   EnumKind, VTableHolder, TemplateParams, Identifier));
 
   EXPECT_NE(N, DICompositeType::get(Context, Tag + 1, Name, File, Line, Scope,
                                     BaseType, SizeInBits, AlignInBits,
                                     OffsetInBits, Flags, Elements, RuntimeLang,
-                                    VTableHolder, TemplateParams, Identifier));
-  EXPECT_NE(N, DICompositeType::get(Context, Tag, "abc", File, Line, Scope,
-                                    BaseType, SizeInBits, AlignInBits,
-                                    OffsetInBits, Flags, Elements, RuntimeLang,
-                                    VTableHolder, TemplateParams, Identifier));
+                                    EnumKind, VTableHolder, TemplateParams,
+                                    Identifier));
+  EXPECT_NE(N, DICompositeType::get(
+                   Context, Tag, "abc", File, Line, Scope, BaseType, SizeInBits,
+                   AlignInBits, OffsetInBits, Flags, Elements, RuntimeLang,
+                   EnumKind, VTableHolder, TemplateParams, Identifier));
   EXPECT_NE(N, DICompositeType::get(Context, Tag, Name, getFile(), Line, Scope,
                                     BaseType, SizeInBits, AlignInBits,
                                     OffsetInBits, Flags, Elements, RuntimeLang,
-                                    VTableHolder, TemplateParams, Identifier));
+                                    EnumKind, VTableHolder, TemplateParams,
+                                    Identifier));
   EXPECT_NE(N, DICompositeType::get(Context, Tag, Name, File, Line + 1, Scope,
                                     BaseType, SizeInBits, AlignInBits,
                                     OffsetInBits, Flags, Elements, RuntimeLang,
-                                    VTableHolder, TemplateParams, Identifier));
-  EXPECT_NE(N, DICompositeType::get(
-                   Context, Tag, Name, File, Line, getSubprogram(), BaseType,
-                   SizeInBits, AlignInBits, OffsetInBits, Flags, Elements,
-                   RuntimeLang, VTableHolder, TemplateParams, Identifier));
-  EXPECT_NE(N, DICompositeType::get(
-                   Context, Tag, Name, File, Line, Scope, getBasicType("other"),
-                   SizeInBits, AlignInBits, OffsetInBits, Flags, Elements,
-                   RuntimeLang, VTableHolder, TemplateParams, Identifier));
+                                    EnumKind, VTableHolder, TemplateParams,
+                                    Identifier));
+  EXPECT_NE(N, DICompositeType::get(Context, Tag, Name, File, Line,
+                                    getSubprogram(), BaseType, SizeInBits,
+                                    AlignInBits, OffsetInBits, Flags, Elements,
+                                    RuntimeLang, EnumKind, VTableHolder,
+                                    TemplateParams, Identifier));
   EXPECT_NE(N, DICompositeType::get(Context, Tag, Name, File, Line, Scope,
-                                    BaseType, SizeInBits + 1, AlignInBits,
-                                    OffsetInBits, Flags, Elements, RuntimeLang,
-                                    VTableHolder, TemplateParams, Identifier));
+                                    getBasicType("other"), SizeInBits,
+                                    AlignInBits, OffsetInBits, Flags, Elements,
+                                    RuntimeLang, EnumKind, VTableHolder,
+                                    TemplateParams, Identifier));
   EXPECT_NE(N, DICompositeType::get(Context, Tag, Name, File, Line, Scope,
-                                    BaseType, SizeInBits, AlignInBits + 1,
+                                    BaseType, SizeInBits + 1, AlignInBits,
                                     OffsetInBits, Flags, Elements, RuntimeLang,
-                                    VTableHolder, TemplateParams, Identifier));
+                                    EnumKind, VTableHolder, TemplateParams,
+                                    Identifier));
+  EXPECT_NE(N, DICompositeType::get(
+                   Context, Tag, Name, File, Line, Scope, BaseType, SizeInBits,
+                   AlignInBits + 1, OffsetInBits, Flags, Elements, RuntimeLang,
+                   EnumKind, VTableHolder, TemplateParams, Identifier));
   EXPECT_NE(N, DICompositeType::get(
                    Context, Tag, Name, File, Line, Scope, BaseType, SizeInBits,
                    AlignInBits, OffsetInBits + 1, Flags, Elements, RuntimeLang,
-                   VTableHolder, TemplateParams, Identifier));
+                   EnumKind, VTableHolder, TemplateParams, Identifier));
   DINode::DIFlags FlagsPOne = static_cast<DINode::DIFlags>(Flags + 1);
   EXPECT_NE(N, DICompositeType::get(
                    Context, Tag, Name, File, Line, Scope, BaseType, SizeInBits,
                    AlignInBits, OffsetInBits, FlagsPOne, Elements, RuntimeLang,
-                   VTableHolder, TemplateParams, Identifier));
+                   EnumKind, VTableHolder, TemplateParams, Identifier));
   EXPECT_NE(N, DICompositeType::get(
                    Context, Tag, Name, File, Line, Scope, BaseType, SizeInBits,
                    AlignInBits, OffsetInBits, Flags, getTuple(), RuntimeLang,
-                   VTableHolder, TemplateParams, Identifier));
+                   EnumKind, VTableHolder, TemplateParams, Identifier));
   EXPECT_NE(N, DICompositeType::get(
                    Context, Tag, Name, File, Line, Scope, BaseType, SizeInBits,
                    AlignInBits, OffsetInBits, Flags, Elements, RuntimeLang + 1,
-                   VTableHolder, TemplateParams, Identifier));
+                   EnumKind, VTableHolder, TemplateParams, Identifier));
   EXPECT_NE(N, DICompositeType::get(
                    Context, Tag, Name, File, Line, Scope, BaseType, SizeInBits,
                    AlignInBits, OffsetInBits, Flags, Elements, RuntimeLang,
-                   getCompositeType(), TemplateParams, Identifier));
-  EXPECT_NE(N, DICompositeType::get(Context, Tag, Name, File, Line, Scope,
-                                    BaseType, SizeInBits, AlignInBits,
-                                    OffsetInBits, Flags, Elements, RuntimeLang,
-                                    VTableHolder, getTuple(), Identifier));
-  EXPECT_NE(N, DICompositeType::get(Context, Tag, Name, File, Line, Scope,
-                                    BaseType, SizeInBits, AlignInBits,
-                                    OffsetInBits, Flags, Elements, RuntimeLang,
-                                    VTableHolder, TemplateParams, "other"));
+                   EnumKind, getCompositeType(), TemplateParams, Identifier));
+  EXPECT_NE(N, DICompositeType::get(
+                   Context, Tag, Name, File, Line, Scope, BaseType, SizeInBits,
+                   AlignInBits, OffsetInBits, Flags, Elements, RuntimeLang,
+                   EnumKind, VTableHolder, getTuple(), Identifier));
+  EXPECT_NE(N, DICompositeType::get(
+                   Context, Tag, Name, File, Line, Scope, BaseType, SizeInBits,
+                   AlignInBits, OffsetInBits, Flags, Elements, RuntimeLang,
+                   EnumKind, VTableHolder, TemplateParams, "other"));
 
   // Be sure that missing identifiers get null pointers.
   EXPECT_FALSE(DICompositeType::get(Context, Tag, Name, File, Line, Scope,
                                     BaseType, SizeInBits, AlignInBits,
                                     OffsetInBits, Flags, Elements, RuntimeLang,
-                                    VTableHolder, TemplateParams, "")
+                                    EnumKind, VTableHolder, TemplateParams, "")
                    ->getRawIdentifier());
   EXPECT_FALSE(DICompositeType::get(Context, Tag, Name, File, Line, Scope,
                                     BaseType, SizeInBits, AlignInBits,
                                     OffsetInBits, Flags, Elements, RuntimeLang,
-                                    VTableHolder, TemplateParams)
+                                    EnumKind, VTableHolder, TemplateParams)
                    ->getRawIdentifier());
 
   TempDICompositeType Temp = N->clone();
@@ -2118,14 +2127,15 @@ TEST_F(DICompositeTypeTest, getWithLargeValues) {
   DINode::DIFlags Flags = static_cast<DINode::DIFlags>(5);
   MDTuple *Elements = getTuple();
   unsigned RuntimeLang = 6;
+  std::optional<uint32_t> EnumKind = 1;
   DIType *VTableHolder = getCompositeType();
   MDTuple *TemplateParams = getTuple();
   StringRef Identifier = "some id";
 
-  auto *N = DICompositeType::get(Context, Tag, Name, File, Line, Scope,
-                                 BaseType, SizeInBits, AlignInBits,
-                                 OffsetInBits, Flags, Elements, RuntimeLang,
-                                 VTableHolder, TemplateParams, Identifier);
+  auto *N = DICompositeType::get(
+      Context, Tag, Name, File, Line, Scope, BaseType, SizeInBits, AlignInBits,
+      OffsetInBits, Flags, Elements, RuntimeLang, EnumKind, VTableHolder,
+      TemplateParams, Identifier);
   EXPECT_EQ(SizeInBits, N->getSizeInBits());
   EXPECT_EQ(AlignInBits, N->getAlignInBits());
   EXPECT_EQ(OffsetInBits, N->getOffsetInBits());
@@ -2143,11 +2153,13 @@ TEST_F(DICompositeTypeTest, replaceOperands) {
   uint64_t OffsetInBits = 4;
   DINode::DIFlags Flags = static_cast<DINode::DIFlags>(5);
   unsigned RuntimeLang = 6;
+  std::optional<uint32_t> EnumKind = 1;
   StringRef Identifier = "some id";
 
-  auto *N = DICompositeType::get(
-      Context, Tag, Name, File, Line, Scope, BaseType, SizeInBits, AlignInBits,
-      OffsetInBits, Flags, nullptr, RuntimeLang, nullptr, nullptr, Identifier);
+  auto *N = DICompositeType::get(Context, Tag, Name, File, Line, Scope,
+                                 BaseType, SizeInBits, AlignInBits,
+                                 OffsetInBits, Flags, nullptr, RuntimeLang,
+                                 EnumKind, nullptr, nullptr, Identifier);
 
   auto *Elements = MDTuple::getDistinct(Context, {});
   EXPECT_EQ(nullptr, N->getElements().get());
@@ -2188,6 +2200,7 @@ TEST_F(DICompositeTypeTest, variant_part) {
   uint64_t OffsetInBits = 4;
   DINode::DIFlags Flags = static_cast<DINode::DIFlags>(5);
   unsigned RuntimeLang = 6;
+  std::optional<uint32_t> EnumKind = 1;
   StringRef Identifier = "some id";
   DIDerivedType *Discriminator = cast<DIDerivedType>(getDerivedType());
   DIDerivedType *Discriminator2 = cast<DIDerivedType>(getDerivedType());
@@ -2196,22 +2209,22 @@ TEST_F(DICompositeTypeTest, variant_part) {
 
   auto *N = DICompositeType::get(
       Context, Tag, Name, File, Line, Scope, BaseType, SizeInBits, AlignInBits,
-      OffsetInBits, Flags, nullptr, RuntimeLang, nullptr, nullptr, Identifier,
-      Discriminator);
+      OffsetInBits, Flags, nullptr, RuntimeLang, EnumKind, nullptr, nullptr,
+      Identifier, Discriminator);
 
   // Test the hashing.
   auto *Same = DICompositeType::get(
       Context, Tag, Name, File, Line, Scope, BaseType, SizeInBits, AlignInBits,
-      OffsetInBits, Flags, nullptr, RuntimeLang, nullptr, nullptr, Identifier,
-      Discriminator);
+      OffsetInBits, Flags, nullptr, RuntimeLang, EnumKind, nullptr, nullptr,
+      Identifier, Discriminator);
   auto *Other = DICompositeType::get(
       Context, Tag, Name, File, Line, Scope, BaseType, SizeInBits, AlignInBits,
-      OffsetInBits, Flags, nullptr, RuntimeLang, nullptr, nullptr, Identifier,
-      Discriminator2);
+      OffsetInBits, Flags, nullptr, RuntimeLang, EnumKind, nullptr, nullptr,
+      Identifier, Discriminator2);
   auto *NoDisc = DICompositeType::get(
       Context, Tag, Name, File, Line, Scope, BaseType, SizeInBits, AlignInBits,
-      OffsetInBits, Flags, nullptr, RuntimeLang, nullptr, nullptr, Identifier,
-      nullptr);
+      OffsetInBits, Flags, nullptr, RuntimeLang, EnumKind, nullptr, nullptr,
+      Identifier, nullptr);
 
   EXPECT_EQ(N, Same);
   EXPECT_NE(Same, Other);
@@ -2233,6 +2246,7 @@ TEST_F(DICompositeTypeTest, dynamicArray) {
   uint64_t OffsetInBits = 4;
   DINode::DIFlags Flags = static_cast<DINode::DIFlags>(3);
   unsigned RuntimeLang = 6;
+  std::optional<uint32_t> EnumKind = 1;
   StringRef Identifier = "some id";
   DIType *Type = getDerivedType();
   Metadata *DlVar1 = DILocalVariable::get(Context, Scope, "dl_var1", File, 8,
@@ -2257,18 +2271,18 @@ TEST_F(DICompositeTypeTest, dynamicArray) {
   ConstantAsMetadata *RankConst2 = ConstantAsMetadata::get(RankInt2);
   auto *N1 = DICompositeType::get(
       Context, Tag, Name, File, Line, Scope, BaseType, SizeInBits, AlignInBits,
-      OffsetInBits, Flags, nullptr, RuntimeLang, nullptr, nullptr, Identifier,
-      nullptr, DlVar1);
+      OffsetInBits, Flags, nullptr, RuntimeLang, EnumKind, nullptr, nullptr,
+      Identifier, nullptr, DlVar1);
 
   auto *Same1 = DICompositeType::get(
       Context, Tag, Name, File, Line, Scope, BaseType, SizeInBits, AlignInBits,
-      OffsetInBits, Flags, nullptr, RuntimeLang, nullptr, nullptr, Identifier,
-      nullptr, DlVar1);
+      OffsetInBits, Flags, nullptr, RuntimeLang, EnumKind, nullptr, nullptr,
+      Identifier, nullptr, DlVar1);
 
   auto *Other1 = DICompositeType::get(
       Context, Tag, Name, File, Line, Scope, BaseType, SizeInBits, AlignInBits,
-      OffsetInBits, Flags, nullptr, RuntimeLang, nullptr, nullptr, Identifier,
-      nullptr, DlVar2);
+      OffsetInBits, Flags, nullptr, RuntimeLang, EnumKind, nullptr, nullptr,
+      Identifier, nullptr, DlVar2);
 
   EXPECT_EQ(N1, Same1);
   EXPECT_NE(Same1, Other1);
@@ -2276,18 +2290,18 @@ TEST_F(DICompositeTypeTest, dynamicArray) {
 
   auto *N2 = DICompositeType::get(
       Context, Tag, Name, File, Line, Scope, BaseType, SizeInBits, AlignInBits,
-      OffsetInBits, Flags, nullptr, RuntimeLang, nullptr, nullptr, Identifier,
-      nullptr, DataLocation1);
+      OffsetInBits, Flags, nullptr, RuntimeLang, EnumKind, nullptr, nullptr,
+      Identifier, nullptr, DataLocation1);
 
   auto *Same2 = DICompositeType::get(
       Context, Tag, Name, File, Line, Scope, BaseType, SizeInBits, AlignInBits,
-      OffsetInBits, Flags, nullptr, RuntimeLang, nullptr, nullptr, Identifier,
-      nullptr, DataLocation1);
+      OffsetInBits, Flags, nullptr, RuntimeLang, EnumKind, nullptr, nullptr,
+      Identifier, nullptr, DataLocation1);
 
   auto *Other2 = DICompositeType::get(
       Context, Tag, Name, File, Line, Scope, BaseType, SizeInBits, AlignInBits,
-      OffsetInBits, Flags, nullptr, RuntimeLang, nullptr, nullptr, Identifier,
-      nullptr, DataLocation2);
+      OffsetInBits, Flags, nullptr, RuntimeLang, EnumKind, nullptr, nullptr,
+      Identifier, nullptr, DataLocation2);
 
   EXPECT_EQ(N2, Same2);
   EXPECT_NE(Same2, Other2);
@@ -2295,18 +2309,18 @@ TEST_F(DICompositeTypeTest, dynamicArray) {
 
   auto *N3 = DICompositeType::get(
       Context, Tag, Name, File, Line, Scope, BaseType, SizeInBits, AlignInBits,
-      OffsetInBits, Flags, nullptr, RuntimeLang, nullptr, nullptr, Identifier,
-      nullptr, DataLocation1, nullptr, nullptr, Rank1);
+      OffsetInBits, Flags, nullptr, RuntimeLang, EnumKind, nullptr, nullptr,
+      Identifier, nullptr, DataLocation1, nullptr, nullptr, Rank1);
 
   auto *Same3 = DICompositeType::get(
       Context, Tag, Name, File, Line, Scope, BaseType, SizeInBits, AlignInBits,
-      OffsetInBits, Flags, nullptr, RuntimeLang, nullptr, nullptr, Identifier,
-      nullptr, DataLocation1, nullptr, nullptr, Rank1);
+      OffsetInBits, Flags, nullptr, RuntimeLang, EnumKind, nullptr, nullptr,
+      Identifier, nullptr, DataLocation1, nullptr, nullptr, Rank1);
 
   auto *Other3 = DICompositeType::get(
       Context, Tag, Name, File, Line, Scope, BaseType, SizeInBits, AlignInBits,
-      OffsetInBits, Flags, nullptr, RuntimeLang, nullptr, nullptr, Identifier,
-      nullptr, DataLocation1, nullptr, nullptr, Rank2);
+      OffsetInBits, Flags, nullptr, RuntimeLang, EnumKind, nullptr, nullptr,
+      Identifier, nullptr, DataLocation1, nullptr, nullptr, Rank2);
 
   EXPECT_EQ(N3, Same3);
   EXPECT_NE(Same3, Other3);
@@ -2314,18 +2328,18 @@ TEST_F(DICompositeTypeTest, dynamicArray) {
 
   auto *N4 = DICompositeType::get(
       Context, Tag, Name, File, Line, Scope, BaseType, SizeInBits, AlignInBits,
-      OffsetInBits, Flags, nullptr, RuntimeLang, nullptr, nullptr, Identifier,
-      nullptr, DataLocation1, nullptr, nullptr, RankConst1);
+      OffsetInBits, Flags, nullptr, RuntimeLang, EnumKind, nullptr, nullptr,
+      Identifier, nullptr, DataLocation1, nullptr, nullptr, RankConst1);
 
   auto *Same4 = DICompositeType::get(
       Context, Tag, Name, File, Line, Scope, BaseType, SizeInBits, AlignInBits,
-      OffsetInBits, Flags, nullptr, RuntimeLang, nullptr, nullptr, Identifier,
-      nullptr, DataLocation1, nullptr, nullptr, RankConst1);
+      OffsetInBits, Flags, nullptr, RuntimeLang, EnumKind, nullptr, nullptr,
+      Identifier, nullptr, DataLocation1, nullptr, nullptr, RankConst1);
 
   auto *Other4 = DICompositeType::get(
       Context, Tag, Name, File, Line, Scope, BaseType, SizeInBits, AlignInBits,
-      OffsetInBits, Flags, nullptr, RuntimeLang, nullptr, nullptr, Identifier,
-      nullptr, DataLocation1, nullptr, nullptr, RankConst2);
+      OffsetInBits, Flags, nullptr, RuntimeLang, EnumKind, nullptr, nullptr,
+      Identifier, nullptr, DataLocation1, nullptr, nullptr, RankConst2);
 
   EXPECT_EQ(N4, Same4);
   EXPECT_NE(Same4, Other4);
diff --git a/llvm/unittests/MIR/MachineMetadata.cpp b/llvm/unittests/MIR/MachineMetadata.cpp
index b16f70be6efe5..51204f23945c1 100644
--- a/llvm/unittests/MIR/MachineMetadata.cpp
+++ b/llvm/unittests/MIR/MachineMetadata.cpp
@@ -131,8 +131,7 @@ TEST_F(MachineMetadataTest, TrivialHook) {
               MO.print(OS, MST, LLT{}, /*OpIdx*/ ~0U, /*PrintDef=*/false,
                        /*IsStandalone=*/false,
                        /*ShouldPrintRegisterTies=*/false, /*TiedOperandIdx=*/0,
-                       /*TRI=*/nullptr,
-                       /*IntrinsicInfo=*/nullptr);
+                       /*TRI=*/nullptr);
             }));
   // Print the definition of that metadata node.
   EXPECT_EQ("!0 = !{!\"foo\"}",
@@ -169,8 +168,7 @@ TEST_F(MachineMetadataTest, BasicHook) {
               MO.print(OS, MST, LLT{}, /*OpIdx*/ ~0U, /*PrintDef=*/false,
                        /*IsStandalone=*/false,
                        /*ShouldPrintRegisterTies=*/false, /*TiedOperandIdx=*/0,
-                       /*TRI=*/nullptr,
-                       /*IntrinsicInfo=*/nullptr);
+                       /*TRI=*/nullptr);
             }));
   // Print the definition of these unnamed metadata nodes.
   EXPECT_EQ("!0 = !{!\"bar\"}",
diff --git a/llvm/unittests/Object/DXContainerTest.cpp b/llvm/unittests/Object/DXContainerTest.cpp
index 5a2c852d6aef9..e7b491103d2d0 100644
--- a/llvm/unittests/Object/DXContainerTest.cpp
+++ b/llvm/unittests/Object/DXContainerTest.cpp
@@ -821,3 +821,73 @@ TEST(DXCFile, MalformedSignature) {
                           "the end of the part data"));
   }
 }
+
+TEST(RootSignature, ParseRootFlags) {
+  {
+    uint8_t Buffer[] = {
+        0x44, 0x58, 0x42, 0x43, 0x32, 0x9A, 0x53, 0xD8, 0xEC, 0xBE, 0x35, 0x6F,
+        0x05, 0x39, 0xE1, 0xFE, 0x31, 0x20, 0xF0, 0xC1, 0x01, 0x00, 0x00, 0x00,
+        0x44, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x24, 0x00, 0x00, 0x00,
+        0x52, 0x54, 0x53, 0x30, 0x18, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00,
+        0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+        0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00,
+    };
+    DXContainer C =
+        llvm::cantFail(DXContainer::create(getMemoryBuffer<68>(Buffer)));
+
+    const auto &RS = C.getRootSignature();
+    ASSERT_TRUE(RS.has_value());
+    ASSERT_EQ(RS->getVersion(), 2u);
+    ASSERT_EQ(RS->getNumParameters(), 0u);
+    ASSERT_EQ(RS->getRootParametersOffset(), 0u);
+    ASSERT_EQ(RS->getNumStaticSamplers(), 0u);
+    ASSERT_EQ(RS->getStaticSamplersOffset(), 0u);
+    ASSERT_EQ(RS->getFlags(), 0x01u);
+  }
+
+  {
+    // this parameter has the root signature definition missing some values.
+    uint8_t Buffer[] = {
+        0x44, 0x58, 0x42, 0x43, 0x32, 0x9A, 0x53, 0xD8, 0xEC, 0xBE, 0x35,
+        0x6F, 0x05, 0x39, 0xE1, 0xFE, 0x31, 0x20, 0xF0, 0xC1, 0x01, 0x00,
+        0x00, 0x00, 0x44, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x24,
+        0x00, 0x00, 0x00, 0x52, 0x54, 0x53, 0x30, 0x18, 0x00, 0x00, 0x00,
+        0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+        0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+    };
+    EXPECT_THAT_EXPECTED(
+        DXContainer::create(getMemoryBuffer<64>(Buffer)),
+        FailedWithMessage(
+            "Invalid root signature, insufficient space for header."));
+  }
+  {
+    // Version has been changed to an invalid number.
+    uint8_t Buffer[] = {
+        0x44, 0x58, 0x42, 0x43, 0x32, 0x9A, 0x53, 0xD8, 0xEC, 0xBE, 0x35, 0x6F,
+        0x05, 0x39, 0xE1, 0xFE, 0x31, 0x20, 0xF0, 0xC1, 0x01, 0x00, 0x00, 0x00,
+        0x44, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x24, 0x00, 0x00, 0x00,
+        0x52, 0x54, 0x53, 0x30, 0x18, 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00,
+        0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+        0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00,
+    };
+    EXPECT_THAT_EXPECTED(
+        DXContainer::create(getMemoryBuffer<68>(Buffer)),
+        FailedWithMessage("Stream Error: An unspecified error has occurred.  "
+                          "Invalid Root Signature Version"));
+  }
+  {
+    // Flag has been set to an invalid value
+    uint8_t Buffer[] = {
+        0x44, 0x58, 0x42, 0x43, 0x32, 0x9A, 0x53, 0xD8, 0xEC, 0xBE, 0x35, 0x6F,
+        0x05, 0x39, 0xE1, 0xFE, 0x31, 0x20, 0xF0, 0xC1, 0x01, 0x00, 0x00, 0x00,
+        0x44, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x24, 0x00, 0x00, 0x00,
+        0x52, 0x54, 0x53, 0x30, 0x18, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00,
+        0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+        0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0xFF,
+    };
+    EXPECT_THAT_EXPECTED(
+        DXContainer::create(getMemoryBuffer<68>(Buffer)),
+        FailedWithMessage("Stream Error: An unspecified error has occurred.  "
+                          "Invalid Root Signature flag"));
+  }
+}
diff --git a/llvm/unittests/Object/ELFObjectFileTest.cpp b/llvm/unittests/Object/ELFObjectFileTest.cpp
index 2a0921690914b..493e673d6a07d 100644
--- a/llvm/unittests/Object/ELFObjectFileTest.cpp
+++ b/llvm/unittests/Object/ELFObjectFileTest.cpp
@@ -8,6 +8,7 @@
 
 #include "llvm/Object/ELFObjectFile.h"
 #include "llvm/ADT/STLExtras.h"
+#include "llvm/BinaryFormat/ELF.h"
 #include "llvm/ObjectYAML/yaml2obj.h"
 #include "llvm/Support/BlockFrequency.h"
 #include "llvm/Support/MemoryBuffer.h"
@@ -1433,7 +1434,8 @@ TEST(ELFObjectFileTest, GetSectionAndRelocations) {
     // Basic verification to make sure we have the correct section types.
     for (auto const &[Sec, RelaSec] : *SecToRelocMapOrErr) {
       ASSERT_EQ(Sec->sh_type, ELF::SHT_PROGBITS);
-      ASSERT_EQ(RelaSec->sh_type, ELF::SHT_RELA);
+      ASSERT_TRUE(RelaSec->sh_type == ELF::SHT_RELA ||
+                  RelaSec->sh_type == ELF::SHT_CREL);
     }
   };
 
@@ -1503,6 +1505,19 @@ TEST(ELFObjectFileTest, GetSectionAndRelocations) {
   DoCheckFails(MissingRelocatableContent, DefaultMatcher,
                "SHT_RELA section with index 1: failed to get a "
                "relocated section: invalid section index: 255");
+
+  StringRef OneTextSectionCREL = R"(
+Sections:
+  - Name: .text
+    Type: SHT_PROGBITS
+    Flags: [ SHF_ALLOC, SHF_EXECINSTR ]
+  - Name: .crel.tex
+    Type: SHT_CREL
+    Flags: [ SHF_INFO_LINK ]
+    Info: .text
+)";
+
+  DoCheckSucceeds(OneTextSectionCREL, DefaultMatcher);
 }
 
 TEST(ELFObjectFileTest, ELFSymbolRefLess) {
diff --git a/llvm/unittests/Object/SymbolicFileTest.cpp b/llvm/unittests/Object/SymbolicFileTest.cpp
index 38875ce7b8cd9..c3813b12b4476 100644
--- a/llvm/unittests/Object/SymbolicFileTest.cpp
+++ b/llvm/unittests/Object/SymbolicFileTest.cpp
@@ -7,8 +7,10 @@
 //===----------------------------------------------------------------------===//
 
 #include "llvm/Object/SymbolicFile.h"
+#include "llvm/ADT/STLExtras.h"
 #include "llvm/Support/raw_ostream.h"
 #include "llvm/TargetParser/Host.h"
+#include "gmock/gmock.h"
 #include "gtest/gtest.h"
 #include <sstream>
 
@@ -38,3 +40,22 @@ TEST(Object, DataRefImplOstream) {
 
   EXPECT_EQ(Expected, s);
 }
+
+struct ProxyContent {
+  unsigned Index = 0;
+  ProxyContent(unsigned Index) : Index(Index) {};
+  void moveNext() { ++Index; }
+
+  bool operator==(const ProxyContent &Another) const {
+    return Index == Another.Index;
+  }
+};
+
+TEST(Object, ContentIterator) {
+  using Iter = llvm::object::content_iterator<ProxyContent>;
+  auto Sequence = llvm::make_range(Iter(0u), Iter(10u));
+  auto EvenSequence = llvm::make_filter_range(
+      Sequence, [](auto &&PC) { return PC.Index % 2 == 0; });
+
+  EXPECT_THAT(EvenSequence, testing::ElementsAre(0u, 2u, 4u, 6u, 8u));
+}
diff --git a/llvm/unittests/ObjectYAML/DXContainerYAMLTest.cpp b/llvm/unittests/ObjectYAML/DXContainerYAMLTest.cpp
index d4232295c8584..b48cd9ce53987 100644
--- a/llvm/unittests/ObjectYAML/DXContainerYAMLTest.cpp
+++ b/llvm/unittests/ObjectYAML/DXContainerYAMLTest.cpp
@@ -107,3 +107,42 @@ TEST(DXCFile, ParseEmptyParts) {
   EXPECT_EQ(Storage.size(), 116u);
   EXPECT_TRUE(memcmp(Buffer, Storage.data(), 116) == 0);
 }
+
+TEST(RootSignature, ParseRootFlags) {
+  SmallString<128> Storage;
+
+  // First read a fully explicit yaml with all sizes and offsets provided
+  ASSERT_TRUE(convert(Storage, R"(--- !dxcontainer
+  Header:
+    Hash:            [ 0x32, 0x9A, 0x53, 0xD8, 0xEC, 0xBE, 0x35, 0x6F, 0x5, 
+                      0x39, 0xE1, 0xFE, 0x31, 0x20, 0xF0, 0xC1 ]
+    Version:
+      Major:           1
+      Minor:           0
+    FileSize:        68
+    PartCount:       1
+    PartOffsets:     [ 36 ]
+  Parts:
+    - Name:            RTS0
+      Size:            24
+      RootSignature:
+        Version:         2
+        NumParameters:   0
+        RootParametersOffset: 0
+        NumStaticSamplers: 0
+        StaticSamplersOffset: 0
+        AllowInputAssemblerInputLayout: true
+    )"));
+
+  uint8_t Buffer[] = {
+      0x44, 0x58, 0x42, 0x43, 0x32, 0x9A, 0x53, 0xD8, 0xEC, 0xBE, 0x35, 0x6F,
+      0x05, 0x39, 0xE1, 0xFE, 0x31, 0x20, 0xF0, 0xC1, 0x01, 0x00, 0x00, 0x00,
+      0x44, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x24, 0x00, 0x00, 0x00,
+      0x52, 0x54, 0x53, 0x30, 0x18, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00,
+      0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+      0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00,
+  };
+
+  EXPECT_EQ(Storage.size(), 68u);
+  EXPECT_TRUE(memcmp(Buffer, Storage.data(), 68u) == 0);
+}
diff --git a/llvm/unittests/SandboxIR/PassTest.cpp b/llvm/unittests/SandboxIR/PassTest.cpp
index 751aedefd8fe2..19fce94563e48 100644
--- a/llvm/unittests/SandboxIR/PassTest.cpp
+++ b/llvm/unittests/SandboxIR/PassTest.cpp
@@ -7,6 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "llvm/SandboxIR/Pass.h"
+#include "llvm/Analysis/TargetTransformInfo.h"
 #include "llvm/AsmParser/Parser.h"
 #include "llvm/IR/Module.h"
 #include "llvm/SandboxIR/Constant.h"
@@ -23,10 +24,13 @@ struct PassTest : public testing::Test {
   llvm::LLVMContext LLVMCtx;
   std::unique_ptr<llvm::Module> LLVMM;
   std::unique_ptr<Context> Ctx;
+  std::unique_ptr<llvm::TargetTransformInfo> TTI;
 
   Function *parseFunction(const char *IR, const char *FuncName) {
     llvm::SMDiagnostic Err;
     LLVMM = parseAssemblyString(IR, Err, LLVMCtx);
+    TTI = std::make_unique<llvm::TargetTransformInfo>(LLVMM->getDataLayout());
+
     if (!LLVMM)
       Err.print("PassTest", llvm::errs());
     Ctx = std::make_unique<Context>(LLVMCtx);
@@ -119,7 +123,7 @@ define i8 @foo(i8 %v0, i8 %v1) {
   EXPECT_EQ(TPass.getName(), "test-pass");
   // Check runOnRegion();
   llvm::SmallVector<std::unique_ptr<Region>> Regions =
-      Region::createRegionsFromMD(*F);
+      Region::createRegionsFromMD(*F, *TTI);
   ASSERT_EQ(Regions.size(), 1u);
   TPass.runOnRegion(*Regions[0], Analyses::emptyForTesting());
   EXPECT_EQ(InstCount, 2u);
@@ -242,7 +246,7 @@ define i8 @foo(i8 %v0, i8 %v1) {
   RPM.addPass(std::make_unique<TestPass2>(InstCount2));
   // Check runOnRegion().
   llvm::SmallVector<std::unique_ptr<Region>> Regions =
-      Region::createRegionsFromMD(*F);
+      Region::createRegionsFromMD(*F, *TTI);
   ASSERT_EQ(Regions.size(), 1u);
   RPM.runOnRegion(*Regions[0], Analyses::emptyForTesting());
   EXPECT_EQ(InstCount1, 2u);
diff --git a/llvm/unittests/SandboxIR/RegionTest.cpp b/llvm/unittests/SandboxIR/RegionTest.cpp
index 47368f93a32c0..1ee72d127daa4 100644
--- a/llvm/unittests/SandboxIR/RegionTest.cpp
+++ b/llvm/unittests/SandboxIR/RegionTest.cpp
@@ -7,6 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "llvm/SandboxIR/Region.h"
+#include "llvm/Analysis/TargetTransformInfo.h"
 #include "llvm/AsmParser/Parser.h"
 #include "llvm/SandboxIR/Context.h"
 #include "llvm/SandboxIR/Function.h"
@@ -20,10 +21,12 @@ using namespace llvm;
 struct RegionTest : public testing::Test {
   LLVMContext C;
   std::unique_ptr<Module> M;
+  std::unique_ptr<TargetTransformInfo> TTI;
 
   void parseIR(LLVMContext &C, const char *IR) {
     SMDiagnostic Err;
     M = parseAssemblyString(IR, Err, C);
+    TTI = std::make_unique<TargetTransformInfo>(M->getDataLayout());
     if (!M)
       Err.print("RegionTest", errs());
   }
@@ -45,7 +48,7 @@ define i8 @foo(i8 %v0, i8 %v1) {
   auto *T0 = cast<sandboxir::Instruction>(&*It++);
   auto *T1 = cast<sandboxir::Instruction>(&*It++);
   auto *Ret = cast<sandboxir::Instruction>(&*It++);
-  sandboxir::Region Rgn(Ctx);
+  sandboxir::Region Rgn(Ctx, *TTI);
 
   // Check getContext.
   EXPECT_EQ(&Ctx, &Rgn.getContext());
@@ -73,7 +76,7 @@ define i8 @foo(i8 %v0, i8 %v1) {
 #ifndef NDEBUG
   // Check equality comparison. Insert in reverse order into `Other` to check
   // that comparison is order-independent.
-  sandboxir::Region Other(Ctx);
+  sandboxir::Region Other(Ctx, *TTI);
   Other.add(Ret);
   EXPECT_NE(Rgn, Other);
   Other.add(T1);
@@ -98,7 +101,7 @@ define i8 @foo(i8 %v0, i8 %v1, ptr %ptr) {
   auto *T0 = cast<sandboxir::Instruction>(&*It++);
   auto *T1 = cast<sandboxir::Instruction>(&*It++);
   auto *Ret = cast<sandboxir::Instruction>(&*It++);
-  sandboxir::Region Rgn(Ctx);
+  sandboxir::Region Rgn(Ctx, *TTI);
   Rgn.add(T0);
   Rgn.add(T1);
 
@@ -134,7 +137,7 @@ define i8 @foo(i8 %v0, i8 %v1) {
   auto *T2 = cast<sandboxir::Instruction>(&*It++);
 
   SmallVector<std::unique_ptr<sandboxir::Region>> Regions =
-      sandboxir::Region::createRegionsFromMD(*F);
+      sandboxir::Region::createRegionsFromMD(*F, *TTI);
   EXPECT_THAT(Regions[0]->insts(), testing::UnorderedElementsAre(T0));
   EXPECT_THAT(Regions[1]->insts(), testing::UnorderedElementsAre(T1, T2));
 }
@@ -160,7 +163,7 @@ define i8 @foo(i8 %v0, i8 %v1) {
   auto *T2 = cast<sandboxir::Instruction>(&*It++);
 
   SmallVector<std::unique_ptr<sandboxir::Region>> Regions =
-      sandboxir::Region::createRegionsFromMD(*F);
+      sandboxir::Region::createRegionsFromMD(*F, *TTI);
   EXPECT_THAT(Regions[0]->insts(), testing::UnorderedElementsAre(T0, T2));
 }
 
@@ -182,9 +185,9 @@ define i8 @foo(i8 %v0, i8 %v1) {
   [[maybe_unused]] auto *T1 = cast<sandboxir::Instruction>(&*It++);
   auto *T2 = cast<sandboxir::Instruction>(&*It++);
   [[maybe_unused]] auto *Ret = cast<sandboxir::Instruction>(&*It++);
-  sandboxir::Region Rgn(Ctx);
+  sandboxir::Region Rgn(Ctx, *TTI);
   Rgn.add(T0);
-  sandboxir::Region Rgn2(Ctx);
+  sandboxir::Region Rgn2(Ctx, *TTI);
   Rgn2.add(T2);
 
   std::string output;
@@ -226,14 +229,66 @@ define i8 @foo(i8 %v0, i8 %v1) {
   auto *T0 = cast<sandboxir::Instruction>(&*It++);
   auto *T1 = cast<sandboxir::Instruction>(&*It++);
 
-  sandboxir::Region Rgn(Ctx);
+  sandboxir::Region Rgn(Ctx, *TTI);
   Rgn.add(T0);
   Rgn.add(T1);
 
   SmallVector<std::unique_ptr<sandboxir::Region>> Regions =
-      sandboxir::Region::createRegionsFromMD(*F);
+      sandboxir::Region::createRegionsFromMD(*F, *TTI);
   ASSERT_EQ(1U, Regions.size());
 #ifndef NDEBUG
   EXPECT_EQ(Rgn, *Regions[0].get());
 #endif
 }
+
+TEST_F(RegionTest, RegionCost) {
+  parseIR(C, R"IR(
+define void @foo(i8 %v0, i8 %v1, i8 %v2) {
+  %add0 = add i8 %v0, 1
+  %add1 = add i8 %v1, 2
+  %add2 = add i8 %v2, 3
+  ret void
+}
+)IR");
+  llvm::Function *LLVMF = &*M->getFunction("foo");
+  auto *LLVMBB = &*LLVMF->begin();
+  auto LLVMIt = LLVMBB->begin();
+  auto *LLVMAdd0 = &*LLVMIt++;
+  auto *LLVMAdd1 = &*LLVMIt++;
+  auto *LLVMAdd2 = &*LLVMIt++;
+
+  sandboxir::Context Ctx(C);
+  auto *F = Ctx.createFunction(LLVMF);
+  auto *BB = &*F->begin();
+  auto It = BB->begin();
+  auto *Add0 = cast<sandboxir::Instruction>(&*It++);
+  auto *Add1 = cast<sandboxir::Instruction>(&*It++);
+  auto *Add2 = cast<sandboxir::Instruction>(&*It++);
+
+  sandboxir::Region Rgn(Ctx, *TTI);
+  const auto &SB = Rgn.getScoreboard();
+  EXPECT_EQ(SB.getAfterCost(), 0);
+  EXPECT_EQ(SB.getBeforeCost(), 0);
+
+  auto GetCost = [this](llvm::Instruction *LLVMI) {
+    constexpr static TTI::TargetCostKind CostKind = TTI::TCK_RecipThroughput;
+    SmallVector<const llvm::Value *> Operands(LLVMI->operands());
+    return TTI->getInstructionCost(LLVMI, Operands, CostKind);
+  };
+  // Add `Add0` to the region, should be counted in "After".
+  Rgn.add(Add0);
+  EXPECT_EQ(SB.getBeforeCost(), 0);
+  EXPECT_EQ(SB.getAfterCost(), GetCost(LLVMAdd0));
+  // Same for `Add1`.
+  Rgn.add(Add1);
+  EXPECT_EQ(SB.getBeforeCost(), 0);
+  EXPECT_EQ(SB.getAfterCost(), GetCost(LLVMAdd0) + GetCost(LLVMAdd1));
+  // Remove `Add0`, should be subtracted from "After".
+  Rgn.remove(Add0);
+  EXPECT_EQ(SB.getBeforeCost(), 0);
+  EXPECT_EQ(SB.getAfterCost(), GetCost(LLVMAdd1));
+  // Remove `Add2` which was never in the region, should counted in "Before".
+  Rgn.remove(Add2);
+  EXPECT_EQ(SB.getBeforeCost(), GetCost(LLVMAdd2));
+  EXPECT_EQ(SB.getAfterCost(), GetCost(LLVMAdd1));
+}
diff --git a/llvm/unittests/SandboxIR/SandboxIRTest.cpp b/llvm/unittests/SandboxIR/SandboxIRTest.cpp
index 9d1c86a9b9c72..9eeac9b60372f 100644
--- a/llvm/unittests/SandboxIR/SandboxIRTest.cpp
+++ b/llvm/unittests/SandboxIR/SandboxIRTest.cpp
@@ -6080,3 +6080,23 @@ TEST_F(SandboxIRTest, InstructionCallbacks) {
   EXPECT_THAT(Removed, testing::IsEmpty());
   EXPECT_THAT(Moved, testing::IsEmpty());
 }
+
+TEST_F(SandboxIRTest, FunctionObjectAlreadyExists) {
+  parseIR(C, R"IR(
+define void @foo() {
+  call void @bar()
+  ret void
+}
+define void @bar() {
+  ret void
+}
+)IR");
+  Function &LLVMFoo = *M->getFunction("foo");
+  Function &LLVMBar = *M->getFunction("bar");
+  sandboxir::Context Ctx(C);
+  // This will create a Function object for @bar().
+  Ctx.createFunction(&LLVMFoo);
+  EXPECT_NE(Ctx.getValue(&LLVMBar), nullptr);
+  // This should not crash, even though there is already a value for LLVMBar.
+  Ctx.createFunction(&LLVMBar);
+}
diff --git a/llvm/unittests/SandboxIR/TrackerTest.cpp b/llvm/unittests/SandboxIR/TrackerTest.cpp
index 4eedab124bfa0..9c18247b6b96d 100644
--- a/llvm/unittests/SandboxIR/TrackerTest.cpp
+++ b/llvm/unittests/SandboxIR/TrackerTest.cpp
@@ -52,6 +52,9 @@ define void @foo(ptr %ptr) {
   auto *F = Ctx.createFunction(&LLVMF);
   auto *BB = &*F->begin();
   auto &Tracker = Ctx.getTracker();
+  // Check empty().
+  EXPECT_TRUE(Ctx.getTracker().empty());
+
   Tracker.save();
   auto It = BB->begin();
   auto *Gep0 = &*It++;
@@ -65,6 +68,9 @@ define void @foo(ptr %ptr) {
   EXPECT_EQ(St->getOperand(1), Gep1);
   EXPECT_EQ(Ld->getOperand(0), Gep1);
 
+  // Check empty().
+  EXPECT_FALSE(Ctx.getTracker().empty());
+
   Ctx.getTracker().revert();
   EXPECT_NE(St->getOperand(0), Ld);
   EXPECT_EQ(St->getOperand(1), Gep0);
diff --git a/llvm/unittests/Support/ErrorTest.cpp b/llvm/unittests/Support/ErrorTest.cpp
index 98d19e8d2a15a..00c562ecc059d 100644
--- a/llvm/unittests/Support/ErrorTest.cpp
+++ b/llvm/unittests/Support/ErrorTest.cpp
@@ -976,6 +976,17 @@ TEST(Error, FileErrorTest) {
   handleAllErrors(std::move(FE6), [](std::unique_ptr<FileError> F) {
     EXPECT_EQ(F->messageWithoutFileInfo(), "CustomError {6}");
   });
+
+  Error FE7 =
+      createFileError("file.bin", make_error_code(std::errc::invalid_argument),
+                      "invalid argument");
+  EXPECT_EQ(toString(std::move(FE7)), "'file.bin': invalid argument");
+
+  StringRef Argument = "arg";
+  Error FE8 =
+      createFileError("file.bin", make_error_code(std::errc::invalid_argument),
+                      "invalid argument '%s'", Argument.str().c_str());
+  EXPECT_EQ(toString(std::move(FE8)), "'file.bin': invalid argument 'arg'");
 }
 
 TEST(Error, FileErrorErrorCode) {
diff --git a/llvm/unittests/Target/SPIRV/CMakeLists.txt b/llvm/unittests/Target/SPIRV/CMakeLists.txt
index af7d9395605d0..d7f0290089c4c 100644
--- a/llvm/unittests/Target/SPIRV/CMakeLists.txt
+++ b/llvm/unittests/Target/SPIRV/CMakeLists.txt
@@ -11,6 +11,7 @@ set(LLVM_LINK_COMPONENTS
   SPIRVCodeGen
   SPIRVAnalysis
   Support
+  TargetParser
   )
 
 add_llvm_target_unittest(SPIRVTests
diff --git a/llvm/unittests/Target/SPIRV/SPIRVAPITest.cpp b/llvm/unittests/Target/SPIRV/SPIRVAPITest.cpp
index f0b4a2f55c151..5eae9277c4438 100644
--- a/llvm/unittests/Target/SPIRV/SPIRVAPITest.cpp
+++ b/llvm/unittests/Target/SPIRV/SPIRVAPITest.cpp
@@ -16,6 +16,7 @@
 #include "llvm/BinaryFormat/Magic.h"
 #include "llvm/IR/Module.h"
 #include "llvm/Support/SourceMgr.h"
+#include "llvm/TargetParser/Triple.h"
 #include "gtest/gtest.h"
 #include <gmock/gmock.h>
 #include <string>
@@ -25,6 +26,11 @@ using ::testing::StartsWith;
 
 namespace llvm {
 
+extern "C" LLVM_EXTERNAL_VISIBILITY bool
+SPIRVTranslate(Module *M, std::string &SpirvObj, std::string &ErrMsg,
+               const std::vector<std::string> &AllowExtNames,
+               llvm::CodeGenOptLevel OLevel, Triple TargetTriple);
+
 extern "C" bool
 SPIRVTranslateModule(Module *M, std::string &SpirvObj, std::string &ErrMsg,
                      const std::vector<std::string> &AllowExtNames,
@@ -34,7 +40,27 @@ class SPIRVAPITest : public testing::Test {
 protected:
   bool toSpirv(StringRef Assembly, std::string &Result, std::string &ErrMsg,
                const std::vector<std::string> &AllowExtNames,
-               const std::vector<std::string> &Opts) {
+               llvm::CodeGenOptLevel OLevel, Triple TargetTriple) {
+    SMDiagnostic ParseError;
+    LLVMContext Context;
+    std::unique_ptr<Module> M =
+        parseAssemblyString(Assembly, ParseError, Context);
+    if (!M) {
+      ParseError.print("IR parsing failed: ", errs());
+      report_fatal_error("Can't parse input assembly.");
+    }
+    bool Status = SPIRVTranslate(M.get(), Result, ErrMsg, AllowExtNames, OLevel,
+                                 TargetTriple);
+    if (!Status)
+      errs() << ErrMsg;
+    return Status;
+  }
+  // TODO: Remove toSpirvLegacy() and related tests after existing clients
+  // switch into a newer implementation of SPIRVTranslate().
+  bool toSpirvLegacy(StringRef Assembly, std::string &Result,
+                     std::string &ErrMsg,
+                     const std::vector<std::string> &AllowExtNames,
+                     const std::vector<std::string> &Opts) {
     SMDiagnostic ParseError;
     LLVMContext Context;
     std::unique_ptr<Module> M =
@@ -77,48 +103,21 @@ class SPIRVAPITest : public testing::Test {
 
 TEST_F(SPIRVAPITest, checkTranslateOk) {
   StringRef Assemblies[] = {"", OkAssembly};
-  // Those command line arguments that overlap with registered by llc/codegen
-  // are to be started with the ' ' symbol.
-  std::vector<std::string> SetOfOpts[] = {
-      {}, {"--spirv-mtriple=spirv32-unknown-unknown"}};
-  for (const auto &Opts : SetOfOpts) {
-    for (StringRef &Assembly : Assemblies) {
-      std::string Result, Error;
-      bool Status = toSpirv(Assembly, Result, Error, {}, Opts);
-      EXPECT_TRUE(Status && Error.empty() && !Result.empty());
-      EXPECT_EQ(identify_magic(Result), file_magic::spirv_object);
-    }
+  for (StringRef &Assembly : Assemblies) {
+    std::string Result, Error;
+    bool Status = toSpirv(Assembly, Result, Error, {}, CodeGenOptLevel::Default,
+                          Triple("spirv32-unknown-unknown"));
+    EXPECT_TRUE(Status && Error.empty() && !Result.empty());
+    EXPECT_EQ(identify_magic(Result), file_magic::spirv_object);
   }
 }
 
-TEST_F(SPIRVAPITest, checkTranslateError) {
-  std::string Result, Error;
-  bool Status = toSpirv(OkAssembly, Result, Error, {},
-                        {"-mtriple=spirv32-unknown-unknown"});
-  EXPECT_FALSE(Status);
-  EXPECT_TRUE(Result.empty());
-  EXPECT_THAT(Error,
-              StartsWith("SPIRVTranslateModule: Unknown command line argument "
-                         "'-mtriple=spirv32-unknown-unknown'"));
-  Status = toSpirv(OkAssembly, Result, Error, {}, {"--spirv-O 5"});
-  EXPECT_FALSE(Status);
-  EXPECT_TRUE(Result.empty());
-  EXPECT_EQ(Error, "Invalid optimization level!");
-}
-
-TEST_F(SPIRVAPITest, checkTranslateSupportExtensionByOpts) {
-  std::string Result, Error;
-  std::vector<std::string> Opts{
-      "--spirv-ext=+SPV_KHR_uniform_group_instructions"};
-  bool Status = toSpirv(ExtensionAssembly, Result, Error, {}, Opts);
-  EXPECT_TRUE(Status && Error.empty() && !Result.empty());
-  EXPECT_EQ(identify_magic(Result), file_magic::spirv_object);
-}
-
 TEST_F(SPIRVAPITest, checkTranslateSupportExtensionByArg) {
   std::string Result, Error;
   std::vector<std::string> ExtNames{"SPV_KHR_uniform_group_instructions"};
-  bool Status = toSpirv(ExtensionAssembly, Result, Error, ExtNames, {});
+  bool Status =
+      toSpirv(ExtensionAssembly, Result, Error, ExtNames,
+              CodeGenOptLevel::Aggressive, Triple("spirv64-unknown-unknown"));
   EXPECT_TRUE(Status && Error.empty() && !Result.empty());
   EXPECT_EQ(identify_magic(Result), file_magic::spirv_object);
 }
@@ -128,15 +127,19 @@ TEST_F(SPIRVAPITest, checkTranslateSupportExtensionByArgList) {
   std::vector<std::string> ExtNames{"SPV_KHR_subgroup_rotate",
                                     "SPV_KHR_uniform_group_instructions",
                                     "SPV_KHR_subgroup_rotate"};
-  bool Status = toSpirv(ExtensionAssembly, Result, Error, ExtNames, {});
+  bool Status =
+      toSpirv(ExtensionAssembly, Result, Error, ExtNames,
+              CodeGenOptLevel::Aggressive, Triple("spirv64-unknown-unknown"));
   EXPECT_TRUE(Status && Error.empty() && !Result.empty());
   EXPECT_EQ(identify_magic(Result), file_magic::spirv_object);
 }
 
 TEST_F(SPIRVAPITest, checkTranslateAllExtensions) {
   std::string Result, Error;
-  std::vector<std::string> Opts{"--spirv-ext=all"};
-  bool Status = toSpirv(ExtensionAssembly, Result, Error, {}, Opts);
+  std::vector<std::string> ExtNames{"all"};
+  bool Status =
+      toSpirv(ExtensionAssembly, Result, Error, ExtNames,
+              CodeGenOptLevel::Aggressive, Triple("spirv64-unknown-unknown"));
   EXPECT_TRUE(Status && Error.empty() && !Result.empty());
   EXPECT_EQ(identify_magic(Result), file_magic::spirv_object);
 }
@@ -144,7 +147,9 @@ TEST_F(SPIRVAPITest, checkTranslateAllExtensions) {
 TEST_F(SPIRVAPITest, checkTranslateUnknownExtensionByArg) {
   std::string Result, Error;
   std::vector<std::string> ExtNames{"SPV_XYZ_my_unknown_extension"};
-  bool Status = toSpirv(ExtensionAssembly, Result, Error, ExtNames, {});
+  bool Status =
+      toSpirv(ExtensionAssembly, Result, Error, ExtNames,
+              CodeGenOptLevel::Aggressive, Triple("spirv64-unknown-unknown"));
   EXPECT_FALSE(Status);
   EXPECT_TRUE(Result.empty());
   EXPECT_EQ(Error, "Unknown SPIR-V extension: SPV_XYZ_my_unknown_extension");
@@ -153,35 +158,118 @@ TEST_F(SPIRVAPITest, checkTranslateUnknownExtensionByArg) {
 #if !defined(NDEBUG) && GTEST_HAS_DEATH_TEST
 TEST_F(SPIRVAPITest, checkTranslateExtensionError) {
   std::string Result, Error;
-  std::vector<std::string> Opts;
   EXPECT_DEATH_IF_SUPPORTED(
-      { toSpirv(ExtensionAssembly, Result, Error, {}, Opts); },
+      {
+        toSpirv(ExtensionAssembly, Result, Error, {},
+                CodeGenOptLevel::Aggressive, Triple("spirv64-unknown-unknown"));
+      },
       "LLVM ERROR: __spirv_GroupBitwiseAndKHR: the builtin requires the "
       "following SPIR-V extension: SPV_KHR_uniform_group_instructions");
 }
 
-TEST_F(SPIRVAPITest, checkTranslateUnknownExtensionByOpts) {
+TEST_F(SPIRVAPITest, checkTranslateWrongExtensionByArg) {
   std::string Result, Error;
-  std::vector<std::string> Opts{"--spirv-ext=+SPV_XYZ_my_unknown_extension"};
+  std::vector<std::string> ExtNames{"SPV_KHR_subgroup_rotate"};
   EXPECT_DEATH_IF_SUPPORTED(
-      { toSpirv(ExtensionAssembly, Result, Error, {}, Opts); },
-      "SPIRVTranslateModule: for the --spirv-ext option: Unknown SPIR-V");
+      {
+        toSpirv(ExtensionAssembly, Result, Error, ExtNames,
+                CodeGenOptLevel::Aggressive, Triple("spirv64-unknown-unknown"));
+      },
+      "LLVM ERROR: __spirv_GroupBitwiseAndKHR: the builtin requires the "
+      "following SPIR-V extension: SPV_KHR_uniform_group_instructions");
 }
+#endif
 
-TEST_F(SPIRVAPITest, checkTranslateWrongExtensionByOpts) {
+// Legacy API calls. TODO: Remove after transition into a newer API.
+TEST_F(SPIRVAPITest, checkTranslateStringOptsOk) {
+  StringRef Assemblies[] = {"", OkAssembly};
+  std::vector<std::string> SetOfOpts[] = {{}, {"spirv32-unknown-unknown"}};
+  for (const auto &Opts : SetOfOpts) {
+    for (StringRef &Assembly : Assemblies) {
+      std::string Result, Error;
+      bool Status = toSpirvLegacy(Assembly, Result, Error, {}, Opts);
+      EXPECT_TRUE(Status && Error.empty() && !Result.empty());
+      EXPECT_EQ(identify_magic(Result), file_magic::spirv_object);
+    }
+  }
+}
+
+TEST_F(SPIRVAPITest, checkTranslateStringOptsError) {
+  std::string Result, Error;
+  bool Status = toSpirvLegacy(OkAssembly, Result, Error, {},
+                              {"spirv64v1.6-unknown-unknown", "5"});
+  EXPECT_FALSE(Status);
+  EXPECT_TRUE(Result.empty());
+  EXPECT_EQ(Error, "Invalid optimization level!");
+}
+
+TEST_F(SPIRVAPITest, checkTranslateStringOptsErrorOk) {
+  {
+    std::string Result, Error;
+    bool Status = toSpirvLegacy(OkAssembly, Result, Error, {},
+                                {"spirv64v1.6-unknown-unknown", "5"});
+    EXPECT_FALSE(Status);
+    EXPECT_TRUE(Result.empty());
+    EXPECT_EQ(Error, "Invalid optimization level!");
+  }
+  {
+    std::string Result, Error;
+    bool Status = toSpirvLegacy(OkAssembly, Result, Error, {},
+                                {"spirv64v1.6-unknown-unknown", "3"});
+    EXPECT_TRUE(Status && Error.empty() && !Result.empty());
+    EXPECT_EQ(identify_magic(Result), file_magic::spirv_object);
+  }
+}
+
+TEST_F(SPIRVAPITest, checkTranslateStringOptsSupportExtensionByArg) {
+  std::string Result, Error;
+  std::vector<std::string> ExtNames{"SPV_KHR_uniform_group_instructions"};
+  bool Status = toSpirvLegacy(ExtensionAssembly, Result, Error, ExtNames, {});
+  EXPECT_TRUE(Status && Error.empty() && !Result.empty());
+  EXPECT_EQ(identify_magic(Result), file_magic::spirv_object);
+}
+
+TEST_F(SPIRVAPITest, checkTranslateStringOptsSupportExtensionByArgList) {
+  std::string Result, Error;
+  std::vector<std::string> ExtNames{"SPV_KHR_subgroup_rotate",
+                                    "SPV_KHR_uniform_group_instructions",
+                                    "SPV_KHR_subgroup_rotate"};
+  bool Status = toSpirvLegacy(ExtensionAssembly, Result, Error, ExtNames, {});
+  EXPECT_TRUE(Status && Error.empty() && !Result.empty());
+  EXPECT_EQ(identify_magic(Result), file_magic::spirv_object);
+}
+
+TEST_F(SPIRVAPITest, checkTranslateStringOptsAllExtensions) {
+  std::string Result, Error;
+  std::vector<std::string> ExtNames{"all"};
+  bool Status = toSpirvLegacy(ExtensionAssembly, Result, Error, ExtNames, {});
+  EXPECT_TRUE(Status && Error.empty() && !Result.empty());
+  EXPECT_EQ(identify_magic(Result), file_magic::spirv_object);
+}
+
+TEST_F(SPIRVAPITest, checkTranslateStringOptsUnknownExtensionByArg) {
+  std::string Result, Error;
+  std::vector<std::string> ExtNames{"SPV_XYZ_my_unknown_extension"};
+  bool Status = toSpirvLegacy(ExtensionAssembly, Result, Error, ExtNames, {});
+  EXPECT_FALSE(Status);
+  EXPECT_TRUE(Result.empty());
+  EXPECT_EQ(Error, "Unknown SPIR-V extension: SPV_XYZ_my_unknown_extension");
+}
+
+#if !defined(NDEBUG) && GTEST_HAS_DEATH_TEST
+TEST_F(SPIRVAPITest, checkTranslateStringOptsExtensionError) {
   std::string Result, Error;
-  std::vector<std::string> Opts{"--spirv-ext=+SPV_KHR_subgroup_rotate"};
   EXPECT_DEATH_IF_SUPPORTED(
-      { toSpirv(ExtensionAssembly, Result, Error, {}, Opts); },
+      { toSpirvLegacy(ExtensionAssembly, Result, Error, {}, {}); },
       "LLVM ERROR: __spirv_GroupBitwiseAndKHR: the builtin requires the "
       "following SPIR-V extension: SPV_KHR_uniform_group_instructions");
 }
 
-TEST_F(SPIRVAPITest, checkTranslateWrongExtensionByArg) {
+TEST_F(SPIRVAPITest, checkTranslateStringOptsWrongExtensionByArg) {
   std::string Result, Error;
   std::vector<std::string> ExtNames{"SPV_KHR_subgroup_rotate"};
   EXPECT_DEATH_IF_SUPPORTED(
-      { toSpirv(ExtensionAssembly, Result, Error, ExtNames, {}); },
+      { toSpirvLegacy(ExtensionAssembly, Result, Error, ExtNames, {}); },
       "LLVM ERROR: __spirv_GroupBitwiseAndKHR: the builtin requires the "
       "following SPIR-V extension: SPV_KHR_uniform_group_instructions");
 }
diff --git a/llvm/unittests/TargetParser/Host.cpp b/llvm/unittests/TargetParser/Host.cpp
index c5b96e1df904e..6eb13649a5904 100644
--- a/llvm/unittests/TargetParser/Host.cpp
+++ b/llvm/unittests/TargetParser/Host.cpp
@@ -405,6 +405,19 @@ uarch           : sifive,u74-mc
   EXPECT_EQ(
       sys::detail::getHostCPUNameForRISCV("uarch           : sifive,bullet0\n"),
       "sifive-u74");
+
+  const StringRef SifiveP550MCProcCPUInfo = R"(
+processor       : 0
+hart            : 2
+isa             : rv64imafdch_zicsr_zifencei_zba_zbb_sscofpmf
+mmu             : sv48
+uarch           : eswin,eic770x
+)";
+  EXPECT_EQ(sys::detail::getHostCPUNameForRISCV(SifiveP550MCProcCPUInfo),
+            "sifive-p550");
+  EXPECT_EQ(
+      sys::detail::getHostCPUNameForRISCV("uarch           : eswin,eic770x\n"),
+      "sifive-p550");
 }
 
 static bool runAndGetCommandOutput(
diff --git a/llvm/unittests/TargetParser/RISCVISAInfoTest.cpp b/llvm/unittests/TargetParser/RISCVISAInfoTest.cpp
index 14a60c1857f24..7ebfcf915a7c5 100644
--- a/llvm/unittests/TargetParser/RISCVISAInfoTest.cpp
+++ b/llvm/unittests/TargetParser/RISCVISAInfoTest.cpp
@@ -654,7 +654,7 @@ TEST(ParseArchString, RejectsConflictingExtensions) {
   }
 
   for (StringRef Input :
-       {"rv64i_xqcisls0p2", "rv64i_xqcia0p2", "rv64i_xqciac0p2",
+       {"rv64i_xqcisls0p2", "rv64i_xqcia0p2", "rv64i_xqciac0p3",
         "rv64i_xqcicsr0p2", "rv64i_xqcilsm0p2", "rv64i_xqcicm0p2",
         "rv64i_xqcics0p2", "rv64i_xqcicli0p2", "rv64i_xqciint0p2",
         "rv64i_xqcilo0p2"}) {
@@ -1083,6 +1083,8 @@ R"(All available -march extensions for RISC-V
     xcvmac               1.0
     xcvmem               1.0
     xcvsimd              1.0
+    xmipscmove           1.0
+    xmipslsp             1.0
     xsfcease             1.0
     xsfvcp               1.0
     xsfvfnrclipxfqf      1.0
@@ -1117,7 +1119,7 @@ Experimental extensions
     ssctr                1.0
     svukte               0.3
     xqcia                0.2
-    xqciac               0.2
+    xqciac               0.3
     xqcicli              0.2
     xqcicm               0.2
     xqcics               0.2
diff --git a/llvm/unittests/TargetParser/TargetParserTest.cpp b/llvm/unittests/TargetParser/TargetParserTest.cpp
index c594d38b50b22..11d552bfa328b 100644
--- a/llvm/unittests/TargetParser/TargetParserTest.cpp
+++ b/llvm/unittests/TargetParser/TargetParserTest.cpp
@@ -10,6 +10,7 @@
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/StringExtras.h"
 #include "llvm/ADT/StringMap.h"
+#include "llvm/ADT/StringRef.h"
 #include "llvm/Support/ARMBuildAttributes.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/FormatVariadic.h"
@@ -1814,7 +1815,7 @@ AArch64ExtensionDependenciesBaseArchTestParams
         {AArch64::ARMV9_6A, {"nofp", "fprcvt"}, {"fp-armv8", "fprcvt"}, {}},
         {AArch64::ARMV9_6A, {"fprcvt", "nofp"}, {}, {"fp-armv8", "fprcvt"}},
 
-        // simd -> {aes, sha2, sha3, sm4, f8f16mm, f8f32mm, fp8dot4, fp8dot2}
+        // simd -> {aes, sha2, sha3, sm4, f8f16mm, f8f32mm, faminmax, lut, fp8}
         {AArch64::ARMV8A, {"nosimd", "aes"}, {"neon", "aes"}, {}},
         {AArch64::ARMV8A, {"aes", "nosimd"}, {}, {"neon", "aes"}},
         {AArch64::ARMV8A, {"nosimd", "sha2"}, {"neon", "sha2"}, {}},
@@ -1827,10 +1828,18 @@ AArch64ExtensionDependenciesBaseArchTestParams
         {AArch64::ARMV9_6A, {"f8f16mm", "nosimd"}, {}, {"neon", "f8f16mm"}},
         {AArch64::ARMV9_6A, {"nosimd", "f8f32mm"}, {"neon", "f8f32mm"}, {}},
         {AArch64::ARMV9_6A, {"f8f32mm", "nosimd"}, {}, {"neon", "f8f32mm"}},
-        {AArch64::ARMV9_6A, {"nosimd", "fp8dot4"}, {"neon", "fp8dot4"}, {}},
-        {AArch64::ARMV9_6A, {"fp8dot4", "nosimd"}, {}, {"neon", "fp8dot4"}},
-        {AArch64::ARMV9_6A, {"nosimd", "fp8dot2"}, {"neon", "fp8dot2"}, {}},
-        {AArch64::ARMV9_6A, {"fp8dot2", "nosimd"}, {}, {"neon", "fp8dot2"}},
+        {AArch64::ARMV9_6A, {"faminmax", "nosimd"}, {}, {"neon", "faminmax"}},
+        {AArch64::ARMV9_6A, {"nosimd", "faminmax"}, {"neon", "faminmax"}, {}},
+        {AArch64::ARMV9_6A, {"lut", "nosimd"}, {}, {"neon", "lut"}},
+        {AArch64::ARMV9_6A, {"nosimd", "lut"}, {"neon", "lut"}, {}},
+        {AArch64::ARMV9_6A, {"fp8", "nosimd"}, {}, {"neon", "fp8"}},
+        {AArch64::ARMV9_6A, {"nosimd", "fp8"}, {"neon", "fp8"}, {}},
+
+        // fp8 -> {fp8dot4, fp8dot2}
+        {AArch64::ARMV9_6A, {"nofp8", "fp8dot4"}, {"fp8", "fp8dot4"}, {}},
+        {AArch64::ARMV9_6A, {"fp8dot4", "nofp8"}, {}, {"fp8", "fp8dot4"}},
+        {AArch64::ARMV9_6A, {"nofp8", "fp8dot2"}, {"fp8", "fp8dot2"}, {}},
+        {AArch64::ARMV9_6A, {"fp8dot2", "nofp8"}, {}, {"fp8", "fp8dot2"}},
 
         // simd -> {rdm, dotprod, fcma}
         {AArch64::ARMV8A, {"nosimd", "rdm"}, {"neon", "rdm"}, {}},
@@ -2085,4 +2094,54 @@ INSTANTIATE_TEST_SUITE_P(
     AArch64ExtensionDependenciesBaseCPUTestFixture,
     ::testing::ValuesIn(AArch64ExtensionDependenciesCPUData));
 
+struct CheckFindSinglePrecisionFpuTest {
+  StringRef Cpu;
+  ARM::ArchKind Arch;
+  StringRef Archext;
+  std::vector<StringRef> Features;
+  ARM::FPUKind Fpu;
+  ARM::FPUKind Output;
+};
+
+TEST(TargetParserTest, checkFindSinglePrecisionFPU) {
+  CheckFindSinglePrecisionFpuTest tests[] = {
+      {"cortex-r4f",
+       ARM::ArchKind::ARMV7R,
+       "nofp.dp",
+       {},
+       ARM::FK_INVALID,
+       ARM::FK_VFPV3XD},
+      {"cortex-r7",
+       ARM::ArchKind::ARMV7R,
+       "nofp.dp",
+       {},
+       ARM::FK_INVALID,
+       ARM::FK_VFPV3XD_FP16},
+      {"cortex-a7",
+       ARM::ArchKind::ARMV7A,
+       "nofp.dp",
+       {},
+       ARM::FK_INVALID,
+       ARM::FK_FPV4_SP_D16},
+      {"cortex-r52",
+       ARM::ArchKind::ARMV8R,
+       "nofp.dp",
+       {},
+       ARM::FK_INVALID,
+       ARM::FK_FPV5_SP_D16},
+      {"cortex-m55",
+       ARM::ArchKind::ARMV8_1MMainline,
+       "nofp.dp",
+       {},
+       ARM::FK_INVALID,
+       ARM::FK_FP_ARMV8_FULLFP16_SP_D16}};
+
+  for (auto X : tests) {
+    ARM::FPUKind FPU = X.Fpu;
+    EXPECT_TRUE(
+        ARM::appendArchExtFeatures(X.Cpu, X.Arch, X.Archext, X.Features, FPU));
+    EXPECT_EQ(FPU, X.Output);
+  }
+}
+
 } // namespace
diff --git a/llvm/unittests/Transforms/Utils/CloningTest.cpp b/llvm/unittests/Transforms/Utils/CloningTest.cpp
index 96e863b257c95..f2b73c282b764 100644
--- a/llvm/unittests/Transforms/Utils/CloningTest.cpp
+++ b/llvm/unittests/Transforms/Utils/CloningTest.cpp
@@ -173,7 +173,8 @@ TEST_F(CloneInstruction, Attributes) {
   Function *F2 = Function::Create(FT1, Function::ExternalLinkage);
 
   Argument *A = &*F1->arg_begin();
-  A->addAttr(Attribute::NoCapture);
+  A->addAttr(
+      Attribute::getWithCaptureInfo(A->getContext(), CaptureInfo::none()));
 
   SmallVector<ReturnInst*, 4> Returns;
   ValueToValueMapTy VMap;
@@ -514,7 +515,8 @@ class CloneFunc : public ::testing::Test {
     // cloning).
     auto *StructType = DICompositeType::getDistinct(
         C, dwarf::DW_TAG_structure_type, "some_struct", nullptr, 0, nullptr,
-        nullptr, 32, 32, 0, DINode::FlagZero, nullptr, 0, nullptr, nullptr);
+        nullptr, 32, 32, 0, DINode::FlagZero, nullptr, 0, std::nullopt, nullptr,
+        nullptr);
     auto *InlinedSP = DBuilder.createFunction(
         CU, "inlined", "inlined", File, 8, FuncType, 9, DINode::FlagZero,
         DISubprogram::SPFlagLocalToUnit | DISubprogram::SPFlagDefinition);
diff --git a/llvm/unittests/Transforms/Utils/UnrollLoopTest.cpp b/llvm/unittests/Transforms/Utils/UnrollLoopTest.cpp
index b49e37d9eee98..fb02c89c77a10 100644
--- a/llvm/unittests/Transforms/Utils/UnrollLoopTest.cpp
+++ b/llvm/unittests/Transforms/Utils/UnrollLoopTest.cpp
@@ -73,6 +73,6 @@ while.end:                                        ; preds = %while.cond
 
   bool ret =
       UnrollRuntimeLoopRemainder(L, 4, true, false, false, false, &LI, &SE, &DT,
-                                 &AC, /*TTI=*/nullptr, PreserveLCSSA, 4);
+                                 &AC, /*TTI=*/nullptr, PreserveLCSSA, 4, false);
   EXPECT_FALSE(ret);
 }
diff --git a/llvm/unittests/Transforms/Vectorize/SandboxVectorizer/CMakeLists.txt b/llvm/unittests/Transforms/Vectorize/SandboxVectorizer/CMakeLists.txt
index bbfbcc730a4cb..104a27975cfc0 100644
--- a/llvm/unittests/Transforms/Vectorize/SandboxVectorizer/CMakeLists.txt
+++ b/llvm/unittests/Transforms/Vectorize/SandboxVectorizer/CMakeLists.txt
@@ -12,6 +12,7 @@ add_llvm_unittest(SandboxVectorizerTests
   InstrMapsTest.cpp
   IntervalTest.cpp
   LegalityTest.cpp
+  SandboxVectorizerTest.cpp
   SchedulerTest.cpp
   SeedCollectorTest.cpp	
   VecUtilsTest.cpp
diff --git a/llvm/unittests/Transforms/Vectorize/SandboxVectorizer/DependencyGraphTest.cpp b/llvm/unittests/Transforms/Vectorize/SandboxVectorizer/DependencyGraphTest.cpp
index 3fa4de501f3f5..f1e9afefb4531 100644
--- a/llvm/unittests/Transforms/Vectorize/SandboxVectorizer/DependencyGraphTest.cpp
+++ b/llvm/unittests/Transforms/Vectorize/SandboxVectorizer/DependencyGraphTest.cpp
@@ -313,6 +313,39 @@ define i8 @foo(i8 %v0, i8 %v1) {
   EXPECT_EQ(RetN->getNumUnscheduledSuccs(), 0u);
 }
 
+// Make sure we don't get null predecessors even if they are outside the DAG.
+TEST_F(DependencyGraphTest, NonNullPreds) {
+  parseIR(C, R"IR(
+define void @foo(ptr %ptr, i8 %val) {
+  %gep = getelementptr i8, ptr %ptr, i32 0
+  store i8 %val, ptr %gep
+  ret void
+}
+)IR");
+  llvm::Function *LLVMF = &*M->getFunction("foo");
+  sandboxir::Context Ctx(C);
+  auto *F = Ctx.createFunction(LLVMF);
+  auto *BB = &*F->begin();
+  auto It = BB->begin();
+  [[maybe_unused]] auto *GEP = cast<sandboxir::GetElementPtrInst>(&*It++);
+  auto *S0 = cast<sandboxir::StoreInst>(&*It++);
+  auto *Ret = cast<sandboxir::ReturnInst>(&*It++);
+
+  sandboxir::DependencyGraph DAG(getAA(*LLVMF), Ctx);
+  // The DAG doesn't include GEP.
+  DAG.extend({S0, Ret});
+
+  auto *S0N = DAG.getNode(S0);
+  // S0 has one operand (the GEP) that is outside the DAG and no memory
+  // predecessors. So pred_begin() should be == pred_end().
+  auto PredIt = S0N->preds_begin(DAG);
+  auto PredItE = S0N->preds_end(DAG);
+  EXPECT_EQ(PredIt, PredItE);
+  // Check preds().
+  for (auto *PredN : S0N->preds(DAG))
+    EXPECT_NE(PredN, nullptr);
+}
+
 TEST_F(DependencyGraphTest, MemDGNode_getPrevNode_getNextNode) {
   parseIR(C, R"IR(
 define void @foo(ptr %ptr, i8 %v0, i8 %v1) {
@@ -799,9 +832,10 @@ define void @foo(ptr %ptr, i8 %v1, i8 %v2, i8 %v3, i8 %v4, i8 %v5) {
   }
 }
 
+// Check that the DAG gets updated when we create a new instruction.
 TEST_F(DependencyGraphTest, CreateInstrCallback) {
   parseIR(C, R"IR(
-define void @foo(ptr %ptr, ptr noalias %ptr2, i8 %v1, i8 %v2, i8 %v3, i8 %arg) {
+define void @foo(ptr %ptr, i8 %v1, i8 %v2, i8 %v3, i8 %new1, i8 %new2) {
   store i8 %v1, ptr %ptr
   store i8 %v2, ptr %ptr
   store i8 %v3, ptr %ptr
@@ -818,42 +852,52 @@ define void @foo(ptr %ptr, ptr noalias %ptr2, i8 %v1, i8 %v2, i8 %v3, i8 %arg) {
   auto *S3 = cast<sandboxir::StoreInst>(&*It++);
   auto *Ret = cast<sandboxir::ReturnInst>(&*It++);
 
-  // Check new instruction callback.
   sandboxir::DependencyGraph DAG(getAA(*LLVMF), Ctx);
-  DAG.extend({S1, Ret});
-  auto *Arg = F->getArg(3);
+  // Create a DAG spanning S1 to S3.
+  DAG.extend({S1, S3});
+  auto *ArgNew1 = F->getArg(4);
+  auto *ArgNew2 = F->getArg(5);
   auto *Ptr = S1->getPointerOperand();
+
+  auto *S1MemN = cast<sandboxir::MemDGNode>(DAG.getNode(S1));
+  auto *S2MemN = cast<sandboxir::MemDGNode>(DAG.getNode(S2));
+  auto *S3MemN = cast<sandboxir::MemDGNode>(DAG.getNode(S3));
+  sandboxir::MemDGNode *New1MemN = nullptr;
+  sandboxir::MemDGNode *New2MemN = nullptr;
   {
+    // Create a new store before S3 (within the span of the DAG).
     sandboxir::StoreInst *NewS =
-        sandboxir::StoreInst::create(Arg, Ptr, Align(8), S3->getIterator(),
+        sandboxir::StoreInst::create(ArgNew1, Ptr, Align(8), S3->getIterator(),
                                      /*IsVolatile=*/true, Ctx);
-    auto *NewSN = DAG.getNode(NewS);
-    EXPECT_TRUE(NewSN != nullptr);
-
     // Check the MemDGNode chain.
-    auto *S2MemN = cast<sandboxir::MemDGNode>(DAG.getNode(S2));
-    auto *NewMemSN = cast<sandboxir::MemDGNode>(NewSN);
-    auto *S3MemN = cast<sandboxir::MemDGNode>(DAG.getNode(S3));
-    EXPECT_EQ(S2MemN->getNextNode(), NewMemSN);
-    EXPECT_EQ(NewMemSN->getPrevNode(), S2MemN);
-    EXPECT_EQ(NewMemSN->getNextNode(), S3MemN);
-    EXPECT_EQ(S3MemN->getPrevNode(), NewMemSN);
+    New1MemN = cast<sandboxir::MemDGNode>(DAG.getNode(NewS));
+    EXPECT_EQ(S2MemN->getNextNode(), New1MemN);
+    EXPECT_EQ(New1MemN->getPrevNode(), S2MemN);
+    EXPECT_EQ(New1MemN->getNextNode(), S3MemN);
+    EXPECT_EQ(S3MemN->getPrevNode(), New1MemN);
+
+    // Check dependencies.
+    EXPECT_TRUE(memDependency(S1MemN, New1MemN));
+    EXPECT_TRUE(memDependency(S2MemN, New1MemN));
+    EXPECT_TRUE(memDependency(New1MemN, S3MemN));
   }
-
   {
-    // Also check if new node is at the end of the BB, after Ret.
+    // Create a new store before Ret (outside the current DAG).
     sandboxir::StoreInst *NewS =
-        sandboxir::StoreInst::create(Arg, Ptr, Align(8), BB->end(),
+        sandboxir::StoreInst::create(ArgNew2, Ptr, Align(8), Ret->getIterator(),
                                      /*IsVolatile=*/true, Ctx);
     // Check the MemDGNode chain.
-    auto *S3MemN = cast<sandboxir::MemDGNode>(DAG.getNode(S3));
-    auto *NewMemSN = cast<sandboxir::MemDGNode>(DAG.getNode(NewS));
-    EXPECT_EQ(S3MemN->getNextNode(), NewMemSN);
-    EXPECT_EQ(NewMemSN->getPrevNode(), S3MemN);
-    EXPECT_EQ(NewMemSN->getNextNode(), nullptr);
+    New2MemN = cast<sandboxir::MemDGNode>(DAG.getNode(NewS));
+    EXPECT_EQ(S3MemN->getNextNode(), New2MemN);
+    EXPECT_EQ(New2MemN->getPrevNode(), S3MemN);
+    EXPECT_EQ(New2MemN->getNextNode(), nullptr);
+
+    // Check dependencies.
+    EXPECT_TRUE(memDependency(S1MemN, New2MemN));
+    EXPECT_TRUE(memDependency(S2MemN, New2MemN));
+    EXPECT_TRUE(memDependency(New1MemN, New2MemN));
+    EXPECT_TRUE(memDependency(S3MemN, New2MemN));
   }
-
-  // TODO: Check the dependencies to/from NewSN after they land.
 }
 
 TEST_F(DependencyGraphTest, EraseInstrCallback) {
@@ -926,3 +970,46 @@ define void @foo(ptr %ptr, ptr %ptr2, i8 %v1, i8 %v2, i8 %v3, i8 %arg) {
   EXPECT_EQ(LdN->getPrevNode(), S1N);
   EXPECT_EQ(LdN->getNextNode(), S2N);
 }
+
+// Check that the mem chain is maintained correctly when the move destination is
+// not a mem node.
+TEST_F(DependencyGraphTest, MoveInstrCallbackWithNonMemInstrs) {
+  parseIR(C, R"IR(
+define void @foo(ptr %ptr, i8 %v1, i8 %v2, i8 %arg) {
+  %ld = load i8, ptr %ptr
+  %zext1 = zext i8 %arg to i32
+  %zext2 = zext i8 %arg to i32
+  store i8 %v1, ptr %ptr
+  store i8 %v2, ptr %ptr
+  ret void
+}
+)IR");
+  llvm::Function *LLVMF = &*M->getFunction("foo");
+  sandboxir::Context Ctx(C);
+  auto *F = Ctx.createFunction(LLVMF);
+  auto *BB = &*F->begin();
+  auto It = BB->begin();
+  auto *Ld = cast<sandboxir::LoadInst>(&*It++);
+  [[maybe_unused]] auto *Zext1 = cast<sandboxir::CastInst>(&*It++);
+  auto *Zext2 = cast<sandboxir::CastInst>(&*It++);
+  auto *S1 = cast<sandboxir::StoreInst>(&*It++);
+  auto *S2 = cast<sandboxir::StoreInst>(&*It++);
+  auto *Ret = cast<sandboxir::ReturnInst>(&*It++);
+
+  sandboxir::DependencyGraph DAG(getAA(*LLVMF), Ctx);
+  DAG.extend({Ld, S2});
+  auto *LdN = cast<sandboxir::MemDGNode>(DAG.getNode(Ld));
+  auto *S1N = cast<sandboxir::MemDGNode>(DAG.getNode(S1));
+  auto *S2N = cast<sandboxir::MemDGNode>(DAG.getNode(S2));
+  EXPECT_EQ(LdN->getNextNode(), S1N);
+  EXPECT_EQ(S1N->getNextNode(), S2N);
+
+  S1->moveBefore(Zext2);
+  EXPECT_EQ(LdN->getNextNode(), S1N);
+  EXPECT_EQ(S1N->getNextNode(), S2N);
+
+  // Try move right after the end of the DAGInterval.
+  S1->moveBefore(Ret);
+  EXPECT_EQ(S2N->getNextNode(), S1N);
+  EXPECT_EQ(S1N->getNextNode(), nullptr);
+}
diff --git a/llvm/unittests/Transforms/Vectorize/SandboxVectorizer/IntervalTest.cpp b/llvm/unittests/Transforms/Vectorize/SandboxVectorizer/IntervalTest.cpp
index 32521ed79a314..59498371b4d73 100644
--- a/llvm/unittests/Transforms/Vectorize/SandboxVectorizer/IntervalTest.cpp
+++ b/llvm/unittests/Transforms/Vectorize/SandboxVectorizer/IntervalTest.cpp
@@ -87,6 +87,15 @@ define void @foo(i8 %v0) {
   EXPECT_FALSE(One.contains(I1));
   EXPECT_FALSE(One.contains(I2));
   EXPECT_FALSE(One.contains(Ret));
+  // Check touches().
+  {
+    sandboxir::Interval<sandboxir::Instruction> Intvl(I2, I2);
+    EXPECT_TRUE(Intvl.touches(I1));
+    EXPECT_TRUE(Intvl.contains(I2));
+    EXPECT_FALSE(Intvl.touches(I2));
+    EXPECT_TRUE(Intvl.touches(Ret));
+    EXPECT_FALSE(Intvl.touches(I0));
+  }
   // Check iterator.
   auto BBIt = BB->begin();
   for (auto &I : Intvl)
diff --git a/llvm/unittests/Transforms/Vectorize/SandboxVectorizer/LegalityTest.cpp b/llvm/unittests/Transforms/Vectorize/SandboxVectorizer/LegalityTest.cpp
index b421d08bc6b02..15f8166b705fc 100644
--- a/llvm/unittests/Transforms/Vectorize/SandboxVectorizer/LegalityTest.cpp
+++ b/llvm/unittests/Transforms/Vectorize/SandboxVectorizer/LegalityTest.cpp
@@ -57,11 +57,24 @@ struct LegalityTest : public testing::Test {
   }
 };
 
+static sandboxir::BasicBlock *getBasicBlockByName(sandboxir::Function *F,
+                                                  StringRef Name) {
+  for (sandboxir::BasicBlock &BB : *F)
+    if (BB.getName() == Name)
+      return &BB;
+  llvm_unreachable("Expected to find basic block!");
+}
+
 TEST_F(LegalityTest, LegalitySkipSchedule) {
   parseIR(C, R"IR(
-define void @foo(ptr %ptr, <2 x float> %vec2, <3 x float> %vec3, i8 %arg, float %farg0, float %farg1, i64 %v0, i64 %v1, i32 %v2) {
+define void @foo(ptr %ptr, <2 x float> %vec2, <3 x float> %vec3, i8 %arg, float %farg0, float %farg1, i64 %v0, i64 %v1, i32 %v2, i1 %c0, i1 %c1) {
+entry:
   %gep0 = getelementptr float, ptr %ptr, i32 0
   %gep1 = getelementptr float, ptr %ptr, i32 1
+  store float %farg0, ptr %gep1
+  br label %bb
+
+bb:
   %gep3 = getelementptr float, ptr %ptr, i32 3
   %ld0 = load float, ptr %gep0
   %ld0b = load float, ptr %gep0
@@ -80,6 +93,8 @@ define void @foo(ptr %ptr, <2 x float> %vec2, <3 x float> %vec3, i8 %arg, float
   %trunc32to8 = trunc i32 %v2 to i8
   %cmpSLT = icmp slt i64 %v0, %v1
   %cmpSGT = icmp sgt i64 %v0, %v1
+  %sel0 = select i1 %c0, <2 x float> %vec2, <2 x float> %vec2
+  %sel1 = select i1 %c1, <2 x float> %vec2, <2 x float> %vec2
   ret void
 }
 )IR");
@@ -89,10 +104,14 @@ define void @foo(ptr %ptr, <2 x float> %vec2, <3 x float> %vec3, i8 %arg, float
 
   sandboxir::Context Ctx(C);
   auto *F = Ctx.createFunction(LLVMF);
-  auto *BB = &*F->begin();
-  auto It = BB->begin();
+  auto *EntryBB = getBasicBlockByName(F, "entry");
+  auto It = EntryBB->begin();
   [[maybe_unused]] auto *Gep0 = cast<sandboxir::GetElementPtrInst>(&*It++);
   [[maybe_unused]] auto *Gep1 = cast<sandboxir::GetElementPtrInst>(&*It++);
+  auto *St1Entry = cast<sandboxir::StoreInst>(&*It++);
+
+  auto *BB = getBasicBlockByName(F, "bb");
+  It = BB->begin();
   [[maybe_unused]] auto *Gep3 = cast<sandboxir::GetElementPtrInst>(&*It++);
   auto *Ld0 = cast<sandboxir::LoadInst>(&*It++);
   auto *Ld0b = cast<sandboxir::LoadInst>(&*It++);
@@ -111,6 +130,8 @@ define void @foo(ptr %ptr, <2 x float> %vec2, <3 x float> %vec3, i8 %arg, float
   auto *Trunc32to8 = cast<sandboxir::TruncInst>(&*It++);
   auto *CmpSLT = cast<sandboxir::CmpInst>(&*It++);
   auto *CmpSGT = cast<sandboxir::CmpInst>(&*It++);
+  auto *Sel0 = cast<sandboxir::SelectInst>(&*It++);
+  auto *Sel1 = cast<sandboxir::SelectInst>(&*It++);
 
   llvm::sandboxir::InstrMaps IMaps(Ctx);
   sandboxir::LegalityAnalysis Legality(*AA, *SE, DL, Ctx, IMaps);
@@ -162,6 +183,14 @@ define void @foo(ptr %ptr, <2 x float> %vec2, <3 x float> %vec3, i8 %arg, float
     EXPECT_EQ(cast<sandboxir::Pack>(Result).getReason(),
               sandboxir::ResultReason::DiffWrapFlags);
   }
+  {
+    // Check DiffBBs
+    const auto &Result =
+        Legality.canVectorize({St0, St1Entry}, /*SkipScheduling=*/true);
+    EXPECT_TRUE(isa<sandboxir::Pack>(Result));
+    EXPECT_EQ(cast<sandboxir::Pack>(Result).getReason(),
+              sandboxir::ResultReason::DiffBBs);
+  }
   {
     // Check DiffTypes for unary operands that have a different type.
     const auto &Result = Legality.canVectorize({Trunc64to8, Trunc32to8},
@@ -200,6 +229,31 @@ define void @foo(ptr %ptr, <2 x float> %vec2, <3 x float> %vec3, i8 %arg, float
         Legality.canVectorize({Ld0, Ld1}, /*SkipScheduling=*/true);
     EXPECT_TRUE(isa<sandboxir::Widen>(Result));
   }
+  {
+    // Check Repeated instructions (splat)
+    const auto &Result =
+        Legality.canVectorize({Ld0, Ld0}, /*SkipScheduling=*/true);
+    EXPECT_TRUE(isa<sandboxir::Pack>(Result));
+    EXPECT_EQ(cast<sandboxir::Pack>(Result).getReason(),
+              sandboxir::ResultReason::RepeatedInstrs);
+  }
+  {
+    // Check Repeated instructions (not splat)
+    const auto &Result =
+        Legality.canVectorize({Ld0, Ld1, Ld0}, /*SkipScheduling=*/true);
+    EXPECT_TRUE(isa<sandboxir::Pack>(Result));
+    EXPECT_EQ(cast<sandboxir::Pack>(Result).getReason(),
+              sandboxir::ResultReason::RepeatedInstrs);
+  }
+  {
+    // For now don't vectorize Selects when the number of elements of conditions
+    // doesn't match the operands.
+    const auto &Result =
+        Legality.canVectorize({Sel0, Sel1}, /*SkipScheduling=*/true);
+    EXPECT_TRUE(isa<sandboxir::Pack>(Result));
+    EXPECT_EQ(cast<sandboxir::Pack>(Result).getReason(),
+              sandboxir::ResultReason::Unimplemented);
+  }
 }
 
 TEST_F(LegalityTest, LegalitySchedule) {
diff --git a/llvm/unittests/Transforms/Vectorize/SandboxVectorizer/SandboxVectorizerTest.cpp b/llvm/unittests/Transforms/Vectorize/SandboxVectorizer/SandboxVectorizerTest.cpp
new file mode 100644
index 0000000000000..e9b304618a06c
--- /dev/null
+++ b/llvm/unittests/Transforms/Vectorize/SandboxVectorizer/SandboxVectorizerTest.cpp
@@ -0,0 +1,63 @@
+//===- SandboxVectorizerTest.cpp ------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/Transforms/Vectorize/SandboxVectorizer/SandboxVectorizer.h"
+#include "llvm/Analysis/AssumptionCache.h"
+#include "llvm/Analysis/BasicAliasAnalysis.h"
+#include "llvm/Analysis/LoopInfo.h"
+#include "llvm/Analysis/ScalarEvolution.h"
+#include "llvm/Analysis/TargetLibraryInfo.h"
+#include "llvm/Analysis/TargetTransformInfo.h"
+#include "llvm/AsmParser/Parser.h"
+#include "llvm/IR/DataLayout.h"
+#include "llvm/IR/Dominators.h"
+#include "llvm/IR/PassInstrumentation.h"
+#include "llvm/SandboxIR/Function.h"
+#include "llvm/SandboxIR/Instruction.h"
+#include "llvm/Support/SourceMgr.h"
+#include "gmock/gmock.h"
+#include "gtest/gtest.h"
+
+using namespace llvm;
+
+struct SandboxVectorizerTest : public testing::Test {
+  LLVMContext C;
+  std::unique_ptr<Module> M;
+
+  void parseIR(LLVMContext &C, const char *IR) {
+    SMDiagnostic Err;
+    M = parseAssemblyString(IR, Err, C);
+    if (!M)
+      Err.print("SandboxVectorizerTest", errs());
+  }
+};
+
+// Check that we can run the pass on the same function more than once without
+// issues. This basically checks that Sandbox IR Context gets cleared after we
+// run the function pass.
+TEST_F(SandboxVectorizerTest, ContextCleared) {
+  parseIR(C, R"IR(
+define void @foo() {
+  ret void
+}
+)IR");
+  auto &LLVMF = *M->getFunction("foo");
+  SandboxVectorizerPass SVecPass;
+  FunctionAnalysisManager AM;
+  AM.registerPass([] { return TargetIRAnalysis(); });
+  AM.registerPass([] { return AAManager(); });
+  AM.registerPass([] { return ScalarEvolutionAnalysis(); });
+  AM.registerPass([] { return PassInstrumentationAnalysis(); });
+  AM.registerPass([] { return TargetLibraryAnalysis(); });
+  AM.registerPass([] { return AssumptionAnalysis(); });
+  AM.registerPass([] { return DominatorTreeAnalysis(); });
+  AM.registerPass([] { return LoopAnalysis(); });
+  SVecPass.run(LLVMF, AM);
+  // This shouldn't crash.
+  SVecPass.run(LLVMF, AM);
+}
diff --git a/llvm/unittests/Transforms/Vectorize/SandboxVectorizer/SchedulerTest.cpp b/llvm/unittests/Transforms/Vectorize/SandboxVectorizer/SchedulerTest.cpp
index c5e44a97976a7..0d5d86acaee89 100644
--- a/llvm/unittests/Transforms/Vectorize/SandboxVectorizer/SchedulerTest.cpp
+++ b/llvm/unittests/Transforms/Vectorize/SandboxVectorizer/SchedulerTest.cpp
@@ -51,6 +51,14 @@ struct SchedulerTest : public testing::Test {
   }
 };
 
+static sandboxir::BasicBlock *getBasicBlockByName(sandboxir::Function *F,
+                                                  StringRef Name) {
+  for (sandboxir::BasicBlock &BB : *F)
+    if (BB.getName() == Name)
+      return &BB;
+  llvm_unreachable("Expected to find basic block!");
+}
+
 TEST_F(SchedulerTest, SchedBundle) {
   parseIR(C, R"IR(
 define void @foo(ptr %ptr, i8 %v0, i8 %v1) {
@@ -237,3 +245,225 @@ define void @foo(ptr noalias %ptr0, ptr noalias %ptr1) {
   EXPECT_TRUE(Sched.trySchedule({Add0, Add1}));
   EXPECT_TRUE(Sched.trySchedule({L0, L1}));
 }
+
+TEST_F(SchedulerTest, DontCrossBBs) {
+  parseIR(C, R"IR(
+define void @foo(ptr noalias %ptr0, ptr noalias %ptr1, i8 %v0, i8 %v1) {
+bb0:
+  %add0 = add i8 %v0, 0
+  %add1 = add i8 %v1, 1
+  br label %bb1
+
+bb1:
+  store i8 %add0, ptr %ptr0
+  store i8 %add1, ptr %ptr1
+  ret void
+}
+)IR");
+  llvm::Function *LLVMF = &*M->getFunction("foo");
+  sandboxir::Context Ctx(C);
+  auto *F = Ctx.createFunction(LLVMF);
+  auto *BB0 = getBasicBlockByName(F, "bb0");
+  auto *BB1 = getBasicBlockByName(F, "bb1");
+  auto It = BB0->begin();
+  auto *Add0 = &*It++;
+  auto *Add1 = &*It++;
+
+  It = BB1->begin();
+  auto *S0 = cast<sandboxir::StoreInst>(&*It++);
+  auto *S1 = cast<sandboxir::StoreInst>(&*It++);
+  auto *Ret = cast<sandboxir::ReturnInst>(&*It++);
+
+  {
+    // Schedule bottom-up
+    sandboxir::Scheduler Sched(getAA(*LLVMF), Ctx);
+    EXPECT_TRUE(Sched.trySchedule({Ret}));
+    EXPECT_TRUE(Sched.trySchedule({S0, S1}));
+    // Scheduling across blocks should fail.
+    EXPECT_FALSE(Sched.trySchedule({Add0, Add1}));
+  }
+  {
+    // Schedule top-down
+    sandboxir::Scheduler Sched(getAA(*LLVMF), Ctx);
+    EXPECT_TRUE(Sched.trySchedule({Add0, Add1}));
+    // Scheduling across blocks should fail.
+    EXPECT_FALSE(Sched.trySchedule({S0, S1}));
+  }
+}
+
+TEST_F(SchedulerTest, NotifyCreateInst) {
+  parseIR(C, R"IR(
+define void @foo(ptr noalias %ptr, ptr noalias %ptr1, ptr noalias %ptr2) {
+  %ld0 = load i8, ptr %ptr
+  store i8 %ld0, ptr %ptr
+  ret void
+}
+)IR");
+  llvm::Function *LLVMF = &*M->getFunction("foo");
+  sandboxir::Context Ctx(C);
+  auto *F = Ctx.createFunction(LLVMF);
+  auto *BB = &*F->begin();
+  auto It = BB->begin();
+  auto *L0 = cast<sandboxir::LoadInst>(&*It++);
+  auto *S0 = cast<sandboxir::StoreInst>(&*It++);
+  auto *Ret = cast<sandboxir::ReturnInst>(&*It++);
+  auto *Ptr1 = F->getArg(1);
+  auto *Ptr2 = F->getArg(2);
+
+  sandboxir::Scheduler Sched(getAA(*LLVMF), Ctx);
+  // Schedule Ret and S0. The top of schedule should be at S0.
+  EXPECT_TRUE(Sched.trySchedule({Ret}));
+  EXPECT_TRUE(Sched.trySchedule({S0}));
+  auto &DAG = sandboxir::SchedulerInternalsAttorney::getDAG(Sched);
+  DAG.extend({L0});
+  auto *L0N = DAG.getNode(L0);
+  EXPECT_EQ(L0N->getNumUnscheduledSuccs(), 0u);
+  // We should have DAG nodes for all instructions at this point
+
+  // Now create a new instruction below S0.
+  sandboxir::StoreInst *NewS1 =
+      sandboxir::StoreInst::create(L0, Ptr1, Align(8), Ret->getIterator(),
+                                   /*IsVolatile=*/false, Ctx);
+  // Check that it is marked as "scheduled".
+  auto *NewS1N = DAG.getNode(NewS1);
+  EXPECT_TRUE(NewS1N->scheduled());
+  // Check that L0's UnscheduledSuccs are still == 0 since NewS1 is "scheduled".
+  EXPECT_EQ(L0N->getNumUnscheduledSuccs(), 0u);
+
+  // Now create a new instruction above S0.
+  sandboxir::StoreInst *NewS2 =
+      sandboxir::StoreInst::create(L0, Ptr2, Align(8), S0->getIterator(),
+                                   /*IsVolatile=*/false, Ctx);
+  // Check that it is not marked as "scheduled".
+  auto *NewS2N = DAG.getNode(NewS2);
+  EXPECT_FALSE(NewS2N->scheduled());
+  // Check that L0's UnscheduledSuccs got updated because of NewS2.
+  EXPECT_EQ(L0N->getNumUnscheduledSuccs(), 1u);
+
+  sandboxir::ReadyListContainer ReadyList;
+  // Check empty().
+  EXPECT_TRUE(ReadyList.empty());
+}
+
+TEST_F(SchedulerTest, ReadyList) {
+  parseIR(C, R"IR(
+define void @foo(ptr %ptr) {
+  %ld0 = load i8, ptr %ptr
+  store i8 %ld0, ptr %ptr
+  ret void
+}
+)IR");
+  llvm::Function *LLVMF = &*M->getFunction("foo");
+  sandboxir::Context Ctx(C);
+  auto *F = Ctx.createFunction(LLVMF);
+  auto *BB = &*F->begin();
+  auto It = BB->begin();
+  auto *L0 = cast<sandboxir::LoadInst>(&*It++);
+  auto *S0 = cast<sandboxir::StoreInst>(&*It++);
+  auto *Ret = cast<sandboxir::ReturnInst>(&*It++);
+
+  sandboxir::DependencyGraph DAG(getAA(*LLVMF), Ctx);
+  DAG.extend({&*BB->begin(), BB->getTerminator()});
+  auto *L0N = DAG.getNode(L0);
+  auto *S0N = DAG.getNode(S0);
+  auto *RetN = DAG.getNode(Ret);
+
+  sandboxir::ReadyListContainer ReadyList;
+  // Check empty().
+  EXPECT_TRUE(ReadyList.empty());
+  // Check insert(), pop().
+  ReadyList.insert(L0N);
+  EXPECT_FALSE(ReadyList.empty());
+  EXPECT_EQ(ReadyList.pop(), L0N);
+  // Check clear().
+  ReadyList.insert(L0N);
+  EXPECT_FALSE(ReadyList.empty());
+  ReadyList.clear();
+  EXPECT_TRUE(ReadyList.empty());
+  // Check remove().
+  EXPECT_TRUE(ReadyList.empty());
+  ReadyList.remove(L0N); // Removing a non-existing node should be valid.
+  ReadyList.insert(L0N);
+  ReadyList.insert(S0N);
+  ReadyList.insert(RetN);
+  ReadyList.remove(S0N);
+  DenseSet<sandboxir::DGNode *> Nodes;
+  Nodes.insert(ReadyList.pop());
+  Nodes.insert(ReadyList.pop());
+  EXPECT_TRUE(ReadyList.empty());
+  EXPECT_THAT(Nodes, testing::UnorderedElementsAre(L0N, RetN));
+}
+
+TEST_F(SchedulerTest, ReadyListPriorities) {
+  parseIR(C, R"IR(
+define void @foo(ptr %ptr) {
+bb0:
+  br label %bb1
+
+bb1:
+  %phi0 = phi i8 [0, %bb0], [1, %bb1]
+  %phi1 = phi i8 [0, %bb0], [1, %bb1]
+  %ld0 = load i8, ptr %ptr
+  store i8 %ld0, ptr %ptr
+  ret void
+}
+)IR");
+  llvm::Function *LLVMF = &*M->getFunction("foo");
+  sandboxir::Context Ctx(C);
+  auto *F = Ctx.createFunction(LLVMF);
+  auto *BB1 = getBasicBlockByName(F, "bb1");
+  auto It = BB1->begin();
+  auto *Phi0 = cast<sandboxir::PHINode>(&*It++);
+  auto *Phi1 = cast<sandboxir::PHINode>(&*It++);
+  auto *L0 = cast<sandboxir::LoadInst>(&*It++);
+  auto *S0 = cast<sandboxir::StoreInst>(&*It++);
+  auto *Ret = cast<sandboxir::ReturnInst>(&*It++);
+
+  sandboxir::DependencyGraph DAG(getAA(*LLVMF), Ctx);
+  DAG.extend({&*BB1->begin(), BB1->getTerminator()});
+  auto *Phi0N = DAG.getNode(Phi0);
+  auto *Phi1N = DAG.getNode(Phi1);
+  auto *L0N = DAG.getNode(L0);
+  auto *S0N = DAG.getNode(S0);
+  auto *RetN = DAG.getNode(Ret);
+
+  sandboxir::ReadyListContainer ReadyList;
+  // Check PHI vs non-PHI.
+  ReadyList.insert(S0N);
+  ReadyList.insert(Phi0N);
+  EXPECT_EQ(ReadyList.pop(), Phi0N);
+  EXPECT_EQ(ReadyList.pop(), S0N);
+  ReadyList.insert(Phi0N);
+  ReadyList.insert(S0N);
+  EXPECT_EQ(ReadyList.pop(), Phi0N);
+  EXPECT_EQ(ReadyList.pop(), S0N);
+  // Check PHI vs terminator.
+  ReadyList.insert(RetN);
+  ReadyList.insert(Phi1N);
+  EXPECT_EQ(ReadyList.pop(), Phi1N);
+  EXPECT_EQ(ReadyList.pop(), RetN);
+  ReadyList.insert(Phi1N);
+  ReadyList.insert(RetN);
+  EXPECT_EQ(ReadyList.pop(), Phi1N);
+  EXPECT_EQ(ReadyList.pop(), RetN);
+  // Check terminator vs non-terminator.
+  ReadyList.insert(RetN);
+  ReadyList.insert(L0N);
+  EXPECT_EQ(ReadyList.pop(), L0N);
+  EXPECT_EQ(ReadyList.pop(), RetN);
+  ReadyList.insert(L0N);
+  ReadyList.insert(RetN);
+  EXPECT_EQ(ReadyList.pop(), L0N);
+  EXPECT_EQ(ReadyList.pop(), RetN);
+  // Check all, program order.
+  ReadyList.insert(RetN);
+  ReadyList.insert(L0N);
+  ReadyList.insert(Phi1N);
+  ReadyList.insert(S0N);
+  ReadyList.insert(Phi0N);
+  EXPECT_EQ(ReadyList.pop(), Phi0N);
+  EXPECT_EQ(ReadyList.pop(), Phi1N);
+  EXPECT_EQ(ReadyList.pop(), L0N);
+  EXPECT_EQ(ReadyList.pop(), S0N);
+  EXPECT_EQ(ReadyList.pop(), RetN);
+}
diff --git a/llvm/unittests/Transforms/Vectorize/SandboxVectorizer/VecUtilsTest.cpp b/llvm/unittests/Transforms/Vectorize/SandboxVectorizer/VecUtilsTest.cpp
index a46e47afea3c7..5c062df8112f6 100644
--- a/llvm/unittests/Transforms/Vectorize/SandboxVectorizer/VecUtilsTest.cpp
+++ b/llvm/unittests/Transforms/Vectorize/SandboxVectorizer/VecUtilsTest.cpp
@@ -461,24 +461,33 @@ define void @foo(i8 %v) {
 
   // Check getLowest(ArrayRef<Value *>)
   SmallVector<sandboxir::Value *> C1Only({C1});
-  EXPECT_EQ(sandboxir::VecUtils::getLowest(C1Only), nullptr);
+  EXPECT_EQ(sandboxir::VecUtils::getLowest(C1Only, &BB), nullptr);
+  EXPECT_EQ(sandboxir::VecUtils::getLowest(C1Only, &BB0), nullptr);
   SmallVector<sandboxir::Value *> AOnly({IA});
-  EXPECT_EQ(sandboxir::VecUtils::getLowest(AOnly), IA);
+  EXPECT_EQ(sandboxir::VecUtils::getLowest(AOnly, &BB), IA);
+  EXPECT_EQ(sandboxir::VecUtils::getLowest(AOnly, &BB0), nullptr);
   SmallVector<sandboxir::Value *> AC1({IA, C1});
-  EXPECT_EQ(sandboxir::VecUtils::getLowest(AC1), IA);
+  EXPECT_EQ(sandboxir::VecUtils::getLowest(AC1, &BB), IA);
+  EXPECT_EQ(sandboxir::VecUtils::getLowest(AC1, &BB0), nullptr);
   SmallVector<sandboxir::Value *> C1A({C1, IA});
-  EXPECT_EQ(sandboxir::VecUtils::getLowest(C1A), IA);
+  EXPECT_EQ(sandboxir::VecUtils::getLowest(C1A, &BB), IA);
+  EXPECT_EQ(sandboxir::VecUtils::getLowest(C1A, &BB0), nullptr);
   SmallVector<sandboxir::Value *> AC1B({IA, C1, IB});
-  EXPECT_EQ(sandboxir::VecUtils::getLowest(AC1B), IB);
+  EXPECT_EQ(sandboxir::VecUtils::getLowest(AC1B, &BB), IB);
+  EXPECT_EQ(sandboxir::VecUtils::getLowest(AC1B, &BB0), nullptr);
   SmallVector<sandboxir::Value *> ABC1({IA, IB, C1});
-  EXPECT_EQ(sandboxir::VecUtils::getLowest(ABC1), IB);
+  EXPECT_EQ(sandboxir::VecUtils::getLowest(ABC1, &BB), IB);
+  EXPECT_EQ(sandboxir::VecUtils::getLowest(ABC1, &BB0), nullptr);
   SmallVector<sandboxir::Value *> AC1C2({IA, C1, C2});
-  EXPECT_EQ(sandboxir::VecUtils::getLowest(AC1C2), IA);
+  EXPECT_EQ(sandboxir::VecUtils::getLowest(AC1C2, &BB), IA);
+  EXPECT_EQ(sandboxir::VecUtils::getLowest(AC1C2, &BB0), nullptr);
   SmallVector<sandboxir::Value *> C1C2C3({C1, C2, C3});
-  EXPECT_EQ(sandboxir::VecUtils::getLowest(C1C2C3), nullptr);
+  EXPECT_EQ(sandboxir::VecUtils::getLowest(C1C2C3, &BB), nullptr);
+  EXPECT_EQ(sandboxir::VecUtils::getLowest(C1C2C3, &BB0), nullptr);
 
   SmallVector<sandboxir::Value *> DiffBBs({BB0I, IA});
-  EXPECT_EQ(sandboxir::VecUtils::getLowest(DiffBBs), nullptr);
+  EXPECT_EQ(sandboxir::VecUtils::getLowest(DiffBBs, &BB0), BB0I);
+  EXPECT_EQ(sandboxir::VecUtils::getLowest(DiffBBs, &BB), IA);
 }
 
 TEST_F(VecUtilsTest, GetLastPHIOrSelf) {
diff --git a/llvm/unittests/Transforms/Vectorize/VPlanHCFGTest.cpp b/llvm/unittests/Transforms/Vectorize/VPlanHCFGTest.cpp
index 19c2483d34ed1..d787a6c977194 100644
--- a/llvm/unittests/Transforms/Vectorize/VPlanHCFGTest.cpp
+++ b/llvm/unittests/Transforms/Vectorize/VPlanHCFGTest.cpp
@@ -48,16 +48,19 @@ TEST_F(VPlanHCFGTest, testBuildHCFGInnerLoop) {
   EXPECT_EQ(0u, Entry->getNumPredecessors());
   EXPECT_EQ(1u, Entry->getNumSuccessors());
 
-  // Check that the region following the preheader is a single basic-block
-  // region (loop).
+  // Check that the region following the preheader consists of a block for the
+  // original header and a separate latch.
   VPBasicBlock *VecBB = Plan->getVectorLoopRegion()->getEntryBasicBlock();
-  EXPECT_EQ(8u, VecBB->size());
+  EXPECT_EQ(7u, VecBB->size());
   EXPECT_EQ(0u, VecBB->getNumPredecessors());
-  EXPECT_EQ(0u, VecBB->getNumSuccessors());
+  EXPECT_EQ(1u, VecBB->getNumSuccessors());
   EXPECT_EQ(VecBB->getParent()->getEntryBasicBlock(), VecBB);
-  EXPECT_EQ(VecBB->getParent()->getExitingBasicBlock(), VecBB);
   EXPECT_EQ(&*Plan, VecBB->getPlan());
 
+  VPBlockBase *VecLatch = VecBB->getSingleSuccessor();
+  EXPECT_EQ(VecLatch->getParent()->getExitingBasicBlock(), VecLatch);
+  EXPECT_EQ(0u, VecLatch->getNumSuccessors());
+
   auto Iter = VecBB->begin();
   VPWidenPHIRecipe *Phi = dyn_cast<VPWidenPHIRecipe>(&*Iter++);
   EXPECT_NE(nullptr, Phi);
@@ -127,29 +130,33 @@ compound=true
       "  EMIT store ir\<%res\>, ir\<%arr.idx\>\l" +
       "  EMIT ir\<%indvars.iv.next\> = add ir\<%indvars.iv\>, ir\<1\>\l" +
       "  EMIT ir\<%exitcond\> = icmp ir\<%indvars.iv.next\>, ir\<%N\>\l" +
-      "  EMIT branch-on-cond ir\<%exitcond\>\l" +
+      "Successor(s): vector.latch\l"
+    ]
+    N2 -> N4 [ label=""]
+    N4 [label =
+      "vector.latch:\l" +
       "No successors\l"
     ]
   }
-  N2 -> N4 [ label="" ltail=cluster_N3]
-  N4 [label =
+  N4 -> N5 [ label="" ltail=cluster_N3]
+  N5 [label =
     "middle.block:\l" +
     "  EMIT vp\<%cmp.n\> = icmp eq ir\<%N\>, vp\<%0\>\l" +
     "  EMIT branch-on-cond vp\<%cmp.n\>\l" +
     "Successor(s): ir-bb\<for.end\>, scalar.ph\l"
   ]
-  N4 -> N5 [ label="T"]
-  N4 -> N6 [ label="F"]
-  N5 [label =
+  N5 -> N6 [ label="T"]
+  N5 -> N7 [ label="F"]
+  N6 [label =
     "ir-bb\<for.end\>:\l" +
     "No successors\l"
   ]
-  N6 [label =
+  N7 [label =
     "scalar.ph:\l" +
     "Successor(s): ir-bb\<for.body\>\l"
   ]
-  N6 -> N7 [ label=""]
-  N7 [label =
+  N7 -> N8 [ label=""]
+  N8 [label =
     "ir-bb\<for.body\>:\l" +
     "  IR   %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]\l" +
     "  IR   %arr.idx = getelementptr inbounds i32, ptr %A, i64 %indvars.iv\l" +
@@ -204,14 +211,17 @@ TEST_F(VPlanHCFGTest, testVPInstructionToVPRecipesInner) {
   EXPECT_EQ(0u, Entry->getNumPredecessors());
   EXPECT_EQ(1u, Entry->getNumSuccessors());
 
-  // Check that the region following the preheader is a single basic-block
-  // region (loop).
+  // Check that the region following the preheader consists of a block for the
+  // original header and a separate latch.
   VPBasicBlock *VecBB = Plan->getVectorLoopRegion()->getEntryBasicBlock();
-  EXPECT_EQ(8u, VecBB->size());
+  EXPECT_EQ(7u, VecBB->size());
   EXPECT_EQ(0u, VecBB->getNumPredecessors());
-  EXPECT_EQ(0u, VecBB->getNumSuccessors());
+  EXPECT_EQ(1u, VecBB->getNumSuccessors());
   EXPECT_EQ(VecBB->getParent()->getEntryBasicBlock(), VecBB);
-  EXPECT_EQ(VecBB->getParent()->getExitingBasicBlock(), VecBB);
+
+  VPBlockBase *VecLatch = VecBB->getSingleSuccessor();
+  EXPECT_EQ(VecLatch->getParent()->getExitingBasicBlock(), VecLatch);
+  EXPECT_EQ(0u, VecLatch->getNumSuccessors());
 
   auto Iter = VecBB->begin();
   EXPECT_NE(nullptr, dyn_cast<VPWidenPHIRecipe>(&*Iter++));
@@ -221,9 +231,117 @@ TEST_F(VPlanHCFGTest, testVPInstructionToVPRecipesInner) {
   EXPECT_NE(nullptr, dyn_cast<VPWidenMemoryRecipe>(&*Iter++));
   EXPECT_NE(nullptr, dyn_cast<VPWidenRecipe>(&*Iter++));
   EXPECT_NE(nullptr, dyn_cast<VPWidenRecipe>(&*Iter++));
-  EXPECT_NE(nullptr, dyn_cast<VPInstruction>(&*Iter++));
   EXPECT_EQ(VecBB->end(), Iter);
 }
 
+TEST_F(VPlanHCFGTest, testBuildHCFGInnerLoopMultiExit) {
+  const char *ModuleString =
+      "define void @f(ptr %A, i64 %N) {\n"
+      "entry:\n"
+      "  br label %loop.header\n"
+      "loop.header:\n"
+      "  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop.latch ]\n"
+      "  %arr.idx = getelementptr inbounds i32, ptr %A, i64 %iv\n"
+      "  %l1 = load i32, ptr %arr.idx, align 4\n"
+      "  %c = icmp eq i32 %l1, 0\n"
+      "  br i1 %c, label %exit.1, label %loop.latch\n"
+      "loop.latch:\n"
+      "  %res = add i32 %l1, 10\n"
+      "  store i32 %res, ptr %arr.idx, align 4\n"
+      "  %iv.next = add i64 %iv, 1\n"
+      "  %exitcond = icmp ne i64 %iv.next, %N\n"
+      "  br i1 %exitcond, label %loop.header, label %exit.2\n"
+      "exit.1:\n"
+      "  ret void\n"
+      "exit.2:\n"
+      "  ret void\n"
+      "}\n";
+
+  Module &M = parseModule(ModuleString);
+
+  Function *F = M.getFunction("f");
+  BasicBlock *LoopHeader = F->getEntryBlock().getSingleSuccessor();
+  auto Plan = buildHCFG(LoopHeader);
+
+#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
+  // Add an external value to check we do not print the list of external values,
+  // as this is not required with the new printing.
+  Plan->getOrAddLiveIn(&*F->arg_begin());
+  std::string FullDump;
+  raw_string_ostream OS(FullDump);
+  Plan->printDOT(OS);
+  const char *ExpectedStr = R"(digraph VPlan {
+graph [labelloc=t, fontsize=30; label="Vectorization Plan\n for UF\>=1\nLive-in vp\<%0\> = vector-trip-count\nLive-in ir\<%N\> = original trip-count\n"]
+node [shape=rect, fontname=Courier, fontsize=30]
+edge [fontname=Courier, fontsize=30]
+compound=true
+  N0 [label =
+    "ir-bb\<entry\>:\l" +
+    "Successor(s): vector.ph\l"
+  ]
+  N0 -> N1 [ label=""]
+  N1 [label =
+    "vector.ph:\l" +
+    "Successor(s): vector loop\l"
+  ]
+  N1 -> N2 [ label="" lhead=cluster_N3]
+  subgraph cluster_N3 {
+    fontname=Courier
+    label="\<x1\> vector loop"
+    N2 [label =
+      "vector.body:\l" +
+      "  WIDEN-PHI ir\<%iv\> = phi ir\<0\>, ir\<%iv.next\>\l" +
+      "  EMIT ir\<%arr.idx\> = getelementptr ir\<%A\>, ir\<%iv\>\l" +
+      "  EMIT ir\<%l1\> = load ir\<%arr.idx\>\l" +
+      "  EMIT ir\<%c\> = icmp ir\<%l1\>, ir\<0\>\l" +
+      "Successor(s): loop.latch\l"
+    ]
+    N2 -> N4 [ label=""]
+    N4 [label =
+      "loop.latch:\l" +
+      "  EMIT ir\<%res\> = add ir\<%l1\>, ir\<10\>\l" +
+      "  EMIT store ir\<%res\>, ir\<%arr.idx\>\l" +
+      "  EMIT ir\<%iv.next\> = add ir\<%iv\>, ir\<1\>\l" +
+      "  EMIT ir\<%exitcond\> = icmp ir\<%iv.next\>, ir\<%N\>\l" +
+      "Successor(s): vector.latch\l"
+    ]
+    N4 -> N5 [ label=""]
+    N5 [label =
+      "vector.latch:\l" +
+      "No successors\l"
+    ]
+  }
+  N5 -> N6 [ label="" ltail=cluster_N3]
+  N6 [label =
+    "middle.block:\l" +
+    "  EMIT vp\<%cmp.n\> = icmp eq ir\<%N\>, vp\<%0\>\l" +
+    "  EMIT branch-on-cond vp\<%cmp.n\>\l" +
+    "Successor(s): ir-bb\<exit.2\>, scalar.ph\l"
+  ]
+  N6 -> N7 [ label="T"]
+  N6 -> N8 [ label="F"]
+  N7 [label =
+    "ir-bb\<exit.2\>:\l" +
+    "No successors\l"
+  ]
+  N8 [label =
+    "scalar.ph:\l" +
+    "Successor(s): ir-bb\<loop.header\>\l"
+  ]
+  N8 -> N9 [ label=""]
+  N9 [label =
+    "ir-bb\<loop.header\>:\l" +
+    "  IR   %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop.latch ]\l" +
+    "  IR   %arr.idx = getelementptr inbounds i32, ptr %A, i64 %iv\l" +
+    "  IR   %l1 = load i32, ptr %arr.idx, align 4\l" +
+    "  IR   %c = icmp eq i32 %l1, 0\l" +
+    "No successors\l"
+  ]
+}
+)";
+  EXPECT_EQ(ExpectedStr, FullDump);
+#endif
+}
+
 } // namespace
 } // namespace llvm
diff --git a/llvm/unittests/Transforms/Vectorize/VPlanSlpTest.cpp b/llvm/unittests/Transforms/Vectorize/VPlanSlpTest.cpp
index e3c542ec5cac8..3a2658ea1e8dc 100644
--- a/llvm/unittests/Transforms/Vectorize/VPlanSlpTest.cpp
+++ b/llvm/unittests/Transforms/Vectorize/VPlanSlpTest.cpp
@@ -6,6 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
+#include "../lib/Transforms/Vectorize/VPlanSLP.h"
 #include "../lib/Transforms/Vectorize/VPlan.h"
 #include "../lib/Transforms/Vectorize/VPlanHCFGBuilder.h"
 #include "VPlanTestBase.h"
diff --git a/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp b/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp
index e7987a95f1ca2..2f37c08bd9f11 100644
--- a/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp
+++ b/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp
@@ -9,6 +9,7 @@
 
 #include "../lib/Transforms/Vectorize/VPlan.h"
 #include "../lib/Transforms/Vectorize/VPlanCFG.h"
+#include "../lib/Transforms/Vectorize/VPlanHelpers.h"
 #include "VPlanTestBase.h"
 #include "llvm/ADT/DepthFirstIterator.h"
 #include "llvm/ADT/PostOrderIterator.h"
@@ -659,6 +660,49 @@ TEST_F(VPBasicBlockTest, TraversingIteratorTest) {
   }
 }
 
+TEST_F(VPBasicBlockTest, reassociateBlocks) {
+  {
+    // Ensure that when we reassociate a basic block, we make sure to update any
+    // references to it in VPWidenPHIRecipes' incoming blocks.
+    VPlan &Plan = getPlan();
+    VPBasicBlock *VPBB1 = Plan.createVPBasicBlock("VPBB1");
+    VPBasicBlock *VPBB2 = Plan.createVPBasicBlock("VPBB2");
+    VPBlockUtils::connectBlocks(VPBB1, VPBB2);
+
+    auto *WidenPhi = new VPWidenPHIRecipe(nullptr);
+    IntegerType *Int32 = IntegerType::get(C, 32);
+    VPValue *Val = Plan.getOrAddLiveIn(ConstantInt::get(Int32, 1));
+    WidenPhi->addIncoming(Val, VPBB1);
+    VPBB2->appendRecipe(WidenPhi);
+
+    VPBasicBlock *VPBBNew = Plan.createVPBasicBlock("VPBBNew");
+    VPBlockUtils::reassociateBlocks(VPBB1, VPBBNew);
+    EXPECT_EQ(VPBB2->getSinglePredecessor(), VPBBNew);
+    EXPECT_EQ(WidenPhi->getIncomingBlock(0), VPBBNew);
+  }
+
+  {
+    // Ensure that we update VPWidenPHIRecipes that are nested inside a
+    // VPRegionBlock.
+    VPlan &Plan = getPlan();
+    VPBasicBlock *VPBB1 = Plan.createVPBasicBlock("VPBB1");
+    VPBasicBlock *VPBB2 = Plan.createVPBasicBlock("VPBB2");
+    VPRegionBlock *R1 = Plan.createVPRegionBlock(VPBB2, VPBB2, "R1");
+    VPBlockUtils::connectBlocks(VPBB1, R1);
+
+    auto *WidenPhi = new VPWidenPHIRecipe(nullptr);
+    IntegerType *Int32 = IntegerType::get(C, 32);
+    VPValue *Val = Plan.getOrAddLiveIn(ConstantInt::get(Int32, 1));
+    WidenPhi->addIncoming(Val, VPBB1);
+    VPBB2->appendRecipe(WidenPhi);
+
+    VPBasicBlock *VPBBNew = Plan.createVPBasicBlock("VPBBNew");
+    VPBlockUtils::reassociateBlocks(VPBB1, VPBBNew);
+    EXPECT_EQ(R1->getSinglePredecessor(), VPBBNew);
+    EXPECT_EQ(WidenPhi->getIncomingBlock(0), VPBBNew);
+  }
+}
+
 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
 TEST_F(VPBasicBlockTest, print) {
   VPInstruction *TC = new VPInstruction(Instruction::Add, {});
diff --git a/llvm/unittests/tools/llvm-exegesis/PowerPC/TargetTest.cpp b/llvm/unittests/tools/llvm-exegesis/PowerPC/TargetTest.cpp
index c3107ee4ec0eb..da24ae8cae2a4 100644
--- a/llvm/unittests/tools/llvm-exegesis/PowerPC/TargetTest.cpp
+++ b/llvm/unittests/tools/llvm-exegesis/PowerPC/TargetTest.cpp
@@ -12,6 +12,7 @@
 #include <memory>
 
 #include "MCTargetDesc/PPCMCTargetDesc.h"
+#include "TestBase.h"
 #include "llvm/MC/TargetRegistry.h"
 #include "llvm/Support/TargetSelect.h"
 #include "gmock/gmock.h"
@@ -30,7 +31,7 @@ using testing::Not;
 
 constexpr const char kTriple[] = "powerpc64le-unknown-linux";
 
-class PowerPCTargetTest : public ::testing::Test {
+class PowerPCTargetTest : public PPCTestBase {
 protected:
   PowerPCTargetTest()
       : ExegesisTarget_(ExegesisTarget::lookup(Triple(kTriple))) {
@@ -39,12 +40,6 @@ class PowerPCTargetTest : public ::testing::Test {
     Target_ = TargetRegistry::lookupTarget(kTriple, error);
     EXPECT_THAT(Target_, NotNull());
   }
-  static void SetUpTestCase() {
-    LLVMInitializePowerPCTargetInfo();
-    LLVMInitializePowerPCTarget();
-    LLVMInitializePowerPCTargetMC();
-    InitializePowerPCExegesisTarget();
-  }
 
   const Target *Target_;
   const ExegesisTarget *const ExegesisTarget_;
diff --git a/llvm/unittests/tools/llvm-exegesis/X86/TargetTest.cpp b/llvm/unittests/tools/llvm-exegesis/X86/TargetTest.cpp
index 3dff50c44798d..846729c6f85ee 100644
--- a/llvm/unittests/tools/llvm-exegesis/X86/TargetTest.cpp
+++ b/llvm/unittests/tools/llvm-exegesis/X86/TargetTest.cpp
@@ -14,6 +14,7 @@
 #include "MCTargetDesc/X86MCTargetDesc.h"
 #include "MmapUtils.h"
 #include "SubprocessMemory.h"
+#include "TestBase.h"
 #include "llvm/MC/TargetRegistry.h"
 #include "llvm/Support/TargetSelect.h"
 #include "gmock/gmock.h"
@@ -112,19 +113,9 @@ Matcher<MCInst> IsStackDeallocate(unsigned Size) {
                ElementsAre(IsReg(X86::RSP), IsReg(X86::RSP), IsImm(Size)));
 }
 
-constexpr const char kTriple[] = "x86_64-unknown-linux";
-
-class X86TargetTest : public ::testing::Test {
+class X86TargetTest : public X86TestBase {
 protected:
-  X86TargetTest(const char *Features)
-      : State(cantFail(LLVMState::Create(kTriple, "core2", Features))) {}
-
-  static void SetUpTestCase() {
-    LLVMInitializeX86TargetInfo();
-    LLVMInitializeX86Target();
-    LLVMInitializeX86TargetMC();
-    InitializeX86ExegesisTarget();
-  }
+  X86TargetTest(const char *Features) : X86TestBase("core2", Features) {}
 
   std::vector<MCInst> setRegTo(unsigned Reg, const APInt &Value) {
     return State.getExegesisTarget().setRegTo(State.getSubtargetInfo(), Reg,
@@ -134,8 +125,6 @@ class X86TargetTest : public ::testing::Test {
   const Instruction &getInstr(unsigned OpCode) {
     return State.getIC().getInstr(OpCode);
   }
-
-  LLVMState State;
 };
 
 class X86Core2TargetTest : public X86TargetTest {
diff --git a/llvm/unittests/tools/llvm-exegesis/X86/TestBase.h b/llvm/unittests/tools/llvm-exegesis/X86/TestBase.h
index ea8063ee44d2c..4122726aef94a 100644
--- a/llvm/unittests/tools/llvm-exegesis/X86/TestBase.h
+++ b/llvm/unittests/tools/llvm-exegesis/X86/TestBase.h
@@ -22,10 +22,12 @@ namespace exegesis {
 
 void InitializeX86ExegesisTarget();
 
+constexpr const char kTriple[] = "x86_64-unknown-linux";
+
 class X86TestBase : public ::testing::Test {
 protected:
-  X86TestBase()
-      : State(cantFail(LLVMState::Create("x86_64-unknown-linux", "haswell"))) {}
+  X86TestBase(std::string CPUName = "haswell", const char *Features = "")
+      : State(cantFail(LLVMState::Create(kTriple, CPUName, Features))) {}
 
   static void SetUpTestCase() {
     LLVMInitializeX86TargetInfo();
diff --git a/llvm/utils/TableGen/AsmMatcherEmitter.cpp b/llvm/utils/TableGen/AsmMatcherEmitter.cpp
index 7684387d80fe2..fcc82e598c437 100644
--- a/llvm/utils/TableGen/AsmMatcherEmitter.cpp
+++ b/llvm/utils/TableGen/AsmMatcherEmitter.cpp
@@ -125,7 +125,7 @@ using namespace llvm;
 
 #define DEBUG_TYPE "asm-matcher-emitter"
 
-cl::OptionCategory AsmMatcherEmitterCat("Options for -gen-asm-matcher");
+static cl::OptionCategory AsmMatcherEmitterCat("Options for -gen-asm-matcher");
 
 static cl::opt<std::string>
     MatchPrefix("match-prefix", cl::init(""),
diff --git a/llvm/utils/TableGen/AsmWriterEmitter.cpp b/llvm/utils/TableGen/AsmWriterEmitter.cpp
index e7606b9df4626..795185c677e30 100644
--- a/llvm/utils/TableGen/AsmWriterEmitter.cpp
+++ b/llvm/utils/TableGen/AsmWriterEmitter.cpp
@@ -967,12 +967,11 @@ void AsmWriterEmitter::EmitPrintAliasInstruction(raw_ostream &O) {
             IAP.addOperand(ROName, MIOpNum, PrintMethodIdx);
 
             // There might be an additional predicate on the MCOperand
-            unsigned Entry = MCOpPredicateMap[Rec];
+            unsigned &Entry = MCOpPredicateMap[Rec];
             if (!Entry) {
               if (!Rec->isValueUnset("MCOperandPredicate")) {
                 MCOpPredicates.push_back(Rec);
                 Entry = MCOpPredicates.size();
-                MCOpPredicateMap[Rec] = Entry;
               } else
                 break; // No conditions on this operand at all
             }
diff --git a/llvm/utils/TableGen/Basic/IntrinsicEmitter.cpp b/llvm/utils/TableGen/Basic/IntrinsicEmitter.cpp
index fc2b8908a35b8..269d2a9544379 100644
--- a/llvm/utils/TableGen/Basic/IntrinsicEmitter.cpp
+++ b/llvm/utils/TableGen/Basic/IntrinsicEmitter.cpp
@@ -252,8 +252,7 @@ void IntrinsicEmitter::EmitIntrinsicToNameTable(
 
 )";
 
-  Table.EmitStringLiteralDef(OS, "static constexpr char IntrinsicNameTable[]",
-                             /*Indent=*/"");
+  Table.EmitStringTableDef(OS, "IntrinsicNameTable", /*Indent=*/"");
 
   OS << R"(
 static constexpr unsigned IntrinsicNameOffsetTable[] = {
@@ -473,7 +472,7 @@ struct AttributeComparator {
 static StringRef getArgAttrEnumName(CodeGenIntrinsic::ArgAttrKind Kind) {
   switch (Kind) {
   case CodeGenIntrinsic::NoCapture:
-    return "NoCapture";
+    llvm_unreachable("Handled separately");
   case CodeGenIntrinsic::NoAlias:
     return "NoAlias";
   case CodeGenIntrinsic::NoUndef:
@@ -526,6 +525,11 @@ static AttributeSet getIntrinsicArgAttributeSet(LLVMContext &C, unsigned ID) {
 )",
                     ID);
       for (const CodeGenIntrinsic::ArgAttribute &Attr : Attrs) {
+        if (Attr.Kind == CodeGenIntrinsic::NoCapture) {
+          OS << "      Attribute::getWithCaptureInfo(C, "
+                "CaptureInfo::none()),\n";
+          continue;
+        }
         StringRef AttrName = getArgAttrEnumName(Attr.Kind);
         if (Attr.Kind == CodeGenIntrinsic::Alignment ||
             Attr.Kind == CodeGenIntrinsic::Dereferenceable)
@@ -759,13 +763,13 @@ Intrinsic::getIntrinsicFor{}Builtin(StringRef TargetPrefix,
   }
 
   if (!Table.empty()) {
-    Table.EmitStringLiteralDef(OS, "static constexpr char BuiltinNames[]");
+    Table.EmitStringTableDef(OS, "BuiltinNames");
 
     OS << R"(
   struct BuiltinEntry {
     ID IntrinsicID;
     unsigned StrTabOffset;
-    const char *getName() const { return &BuiltinNames[StrTabOffset]; }
+    const char *getName() const { return BuiltinNames[StrTabOffset].data(); }
     bool operator<(StringRef RHS) const {
       return strncmp(getName(), RHS.data(), RHS.size()) < 0;
     }
diff --git a/llvm/utils/TableGen/CodeGenMapTable.cpp b/llvm/utils/TableGen/CodeGenMapTable.cpp
index 8d22c0013dda8..2641e713c0c85 100644
--- a/llvm/utils/TableGen/CodeGenMapTable.cpp
+++ b/llvm/utils/TableGen/CodeGenMapTable.cpp
@@ -78,6 +78,7 @@
 #include "Common/CodeGenInstruction.h"
 #include "Common/CodeGenTarget.h"
 #include "TableGenBackends.h"
+#include "llvm/ADT/SetVector.h"
 #include "llvm/ADT/StringExtras.h"
 #include "llvm/TableGen/Error.h"
 #include "llvm/TableGen/Record.h"
@@ -361,45 +362,38 @@ unsigned MapTableEmitter::emitBinSearchTable(raw_ostream &OS) {
   StringRef Namespace = Target.getInstNamespace();
   ArrayRef<const ListInit *> ValueCols = InstrMapDesc.getValueCols();
   unsigned NumCol = ValueCols.size();
-  unsigned TotalNumInstr = NumberedInstructions.size();
   unsigned TableSize = 0;
 
-  OS << "static const uint16_t " << InstrMapDesc.getName();
+  OS << "  using namespace " << Namespace << ";\n";
   // Number of columns in the table are NumCol+1 because key instructions are
   // emitted as first column.
-  OS << "Table[][" << NumCol + 1 << "] = {\n";
-  for (unsigned I = 0; I < TotalNumInstr; I++) {
-    const Record *CurInstr = NumberedInstructions[I]->TheDef;
+  for (const CodeGenInstruction *Inst : NumberedInstructions) {
+    const Record *CurInstr = Inst->TheDef;
     ArrayRef<const Record *> ColInstrs = MapTable[CurInstr];
+    if (ColInstrs.empty())
+      continue;
     std::string OutStr;
-    unsigned RelExists = 0;
-    if (!ColInstrs.empty()) {
-      for (unsigned J = 0; J < NumCol; J++) {
-        if (ColInstrs[J] != nullptr) {
-          RelExists = 1;
-          OutStr += ", ";
-          OutStr += Namespace;
-          OutStr += "::";
-          OutStr += ColInstrs[J]->getName();
-        } else {
-          OutStr += ", (uint16_t)-1U";
-        }
+    bool RelExists = false;
+    for (const Record *ColInstr : ColInstrs) {
+      if (ColInstr) {
+        RelExists = true;
+        OutStr += ", ";
+        OutStr += ColInstr->getName();
+      } else {
+        OutStr += ", (uint16_t)-1U";
       }
+    }
 
-      if (RelExists) {
-        OS << "  { " << Namespace << "::" << CurInstr->getName();
-        OS << OutStr << " },\n";
-        TableSize++;
-      }
+    if (RelExists) {
+      if (TableSize == 0)
+        OS << "  static constexpr uint16_t Table[][" << NumCol + 1 << "] = {\n";
+      OS << "    { " << CurInstr->getName() << OutStr << " },\n";
+      ++TableSize;
     }
   }
-  if (!TableSize) {
-    OS << "  { " << Namespace << "::"
-       << "INSTRUCTION_LIST_END, ";
-    OS << Namespace << "::"
-       << "INSTRUCTION_LIST_END }";
-  }
-  OS << "}; // End of " << InstrMapDesc.getName() << "Table\n\n";
+
+  if (TableSize != 0)
+    OS << "  }; // End of Table\n\n";
   return TableSize;
 }
 
@@ -409,15 +403,19 @@ unsigned MapTableEmitter::emitBinSearchTable(raw_ostream &OS) {
 //===----------------------------------------------------------------------===//
 
 void MapTableEmitter::emitBinSearch(raw_ostream &OS, unsigned TableSize) {
+  if (TableSize == 0) {
+    OS << "  return -1;\n";
+    return;
+  }
+
   OS << "  unsigned mid;\n";
   OS << "  unsigned start = 0;\n";
   OS << "  unsigned end = " << TableSize << ";\n";
   OS << "  while (start < end) {\n";
   OS << "    mid = start + (end - start) / 2;\n";
-  OS << "    if (Opcode == " << InstrMapDesc.getName() << "Table[mid][0]) {\n";
+  OS << "    if (Opcode == Table[mid][0]) \n";
   OS << "      break;\n";
-  OS << "    }\n";
-  OS << "    if (Opcode < " << InstrMapDesc.getName() << "Table[mid][0])\n";
+  OS << "    if (Opcode < Table[mid][0])\n";
   OS << "      end = mid;\n";
   OS << "    else\n";
   OS << "      start = mid + 1;\n";
@@ -431,7 +429,6 @@ void MapTableEmitter::emitBinSearch(raw_ostream &OS, unsigned TableSize) {
 //===----------------------------------------------------------------------===//
 
 void MapTableEmitter::emitMapFuncBody(raw_ostream &OS, unsigned TableSize) {
-
   const ListInit *ColFields = InstrMapDesc.getColFields();
   ArrayRef<const ListInit *> ValueCols = InstrMapDesc.getValueCols();
 
@@ -439,6 +436,8 @@ void MapTableEmitter::emitMapFuncBody(raw_ostream &OS, unsigned TableSize) {
   // relation table. If found, return opcode value from the appropriate column
   // of the table.
   emitBinSearch(OS, TableSize);
+  if (TableSize == 0)
+    return;
 
   if (ValueCols.size() > 1) {
     for (unsigned I = 0, E = ValueCols.size(); I < E; I++) {
@@ -453,14 +452,12 @@ void MapTableEmitter::emitMapFuncBody(raw_ostream &OS, unsigned TableSize) {
           OS << " && ";
       }
       OS << ")\n";
-      OS << "    return " << InstrMapDesc.getName();
-      OS << "Table[mid][" << I + 1 << "];\n";
+      OS << "    return Table[mid][" << I + 1 << "];\n";
     }
     OS << "  return -1;";
-  } else
-    OS << "  return " << InstrMapDesc.getName() << "Table[mid][1];\n";
-
-  OS << "}\n\n";
+  } else {
+    OS << "  return Table[mid][1];\n";
+  }
 }
 
 //===----------------------------------------------------------------------===//
@@ -468,7 +465,6 @@ void MapTableEmitter::emitMapFuncBody(raw_ostream &OS, unsigned TableSize) {
 //===----------------------------------------------------------------------===//
 
 void MapTableEmitter::emitTablesWithFunc(raw_ostream &OS) {
-
   // Emit function name and the input parameters : mostly opcode value of the
   // current instruction. However, if a table has multiple columns (more than 2
   // since first column is used for the key instructions), then we also need
@@ -491,6 +487,8 @@ void MapTableEmitter::emitTablesWithFunc(raw_ostream &OS) {
 
   // Emit rest of the function body.
   emitMapFuncBody(OS, TableSize);
+
+  OS << "}\n\n";
 }
 
 //===----------------------------------------------------------------------===//
@@ -498,7 +496,7 @@ void MapTableEmitter::emitTablesWithFunc(raw_ostream &OS) {
 //===----------------------------------------------------------------------===//
 
 static void emitEnums(raw_ostream &OS, const RecordKeeper &Records) {
-  std::map<std::string, std::vector<const Init *>> ColFieldValueMap;
+  std::map<std::string, SetVector<const Init *>> ColFieldValueMap;
 
   // Iterate over all InstrMapping records and create a map between column
   // fields and their possible values across all records.
@@ -507,10 +505,9 @@ static void emitEnums(raw_ostream &OS, const RecordKeeper &Records) {
     const ListInit *ColFields = CurMap->getValueAsListInit("ColFields");
     const ListInit *List = CurMap->getValueAsListInit("ValueCols");
     std::vector<const ListInit *> ValueCols;
-    unsigned ListSize = List->size();
 
-    for (unsigned J = 0; J < ListSize; J++) {
-      const auto *ListJ = cast<ListInit>(List->getElement(J));
+    for (const Init *Elem : *List) {
+      const auto *ListJ = cast<ListInit>(Elem);
 
       if (ListJ->size() != ColFields->size())
         PrintFatalError("Record `" + CurMap->getName() +
@@ -521,37 +518,26 @@ static void emitEnums(raw_ostream &OS, const RecordKeeper &Records) {
     }
 
     for (unsigned J = 0, EndCf = ColFields->size(); J < EndCf; J++) {
-      for (unsigned K = 0; K < ListSize; K++) {
-        std::string ColName = ColFields->getElement(J)->getAsUnquotedString();
-        ColFieldValueMap[ColName].push_back((ValueCols[K])->getElement(J));
-      }
+      std::string ColName = ColFields->getElement(J)->getAsUnquotedString();
+      auto &MapEntry = ColFieldValueMap[ColName];
+      for (const ListInit *List : ValueCols)
+        MapEntry.insert(List->getElement(J));
     }
   }
 
   for (auto &[EnumName, FieldValues] : ColFieldValueMap) {
-    // Delete duplicate entries from ColFieldValueMap
-    for (unsigned i = 0; i < FieldValues.size() - 1; i++) {
-      const Init *CurVal = FieldValues[i];
-      for (unsigned j = i + 1; j < FieldValues.size(); j++) {
-        if (CurVal == FieldValues[j]) {
-          FieldValues.erase(FieldValues.begin() + j);
-          --j;
-        }
-      }
-    }
-
     // Emit enumerated values for the column fields.
     OS << "enum " << EnumName << " {\n";
     ListSeparator LS(",\n");
     for (const Init *Field : FieldValues)
-      OS << LS << "\t" << EnumName << "_" << Field->getAsUnquotedString();
+      OS << LS << "  " << EnumName << "_" << Field->getAsUnquotedString();
     OS << "\n};\n\n";
   }
 }
 
 //===----------------------------------------------------------------------===//
 // Parse 'InstrMapping' records and use the information to form relationship
-// between instructions. These relations are emitted as a tables along with the
+// between instructions. These relations are emitted as tables along with the
 // functions to query them.
 //===----------------------------------------------------------------------===//
 void llvm::EmitMapTable(const RecordKeeper &Records, raw_ostream &OS) {
@@ -565,8 +551,7 @@ void llvm::EmitMapTable(const RecordKeeper &Records, raw_ostream &OS) {
 
   OS << "#ifdef GET_INSTRMAP_INFO\n";
   OS << "#undef GET_INSTRMAP_INFO\n";
-  OS << "namespace llvm {\n\n";
-  OS << "namespace " << NameSpace << " {\n\n";
+  OS << "namespace llvm::" << NameSpace << " {\n\n";
 
   // Emit coulumn field names and their values as enums.
   emitEnums(OS, Records);
@@ -589,7 +574,6 @@ void llvm::EmitMapTable(const RecordKeeper &Records, raw_ostream &OS) {
     // Emit map tables and the functions to query them.
     IMap.emitTablesWithFunc(OS);
   }
-  OS << "} // end namespace " << NameSpace << "\n";
-  OS << "} // end namespace llvm\n";
+  OS << "} // end namespace llvm::" << NameSpace << '\n';
   OS << "#endif // GET_INSTRMAP_INFO\n\n";
 }
diff --git a/llvm/utils/TableGen/Common/CodeGenDAGPatterns.h b/llvm/utils/TableGen/Common/CodeGenDAGPatterns.h
index 6a6f1a6ac437c..328700cd0d163 100644
--- a/llvm/utils/TableGen/Common/CodeGenDAGPatterns.h
+++ b/llvm/utils/TableGen/Common/CodeGenDAGPatterns.h
@@ -393,6 +393,9 @@ struct SDTypeConstraint {
                         const SDTypeConstraint &RHS);
 };
 
+bool operator==(const SDTypeConstraint &LHS, const SDTypeConstraint &RHS);
+bool operator<(const SDTypeConstraint &LHS, const SDTypeConstraint &RHS);
+
 /// ScopedName - A name of a node associated with a "scope" that indicates
 /// the context (e.g. instance of Pattern or PatFrag) in which the name was
 /// used. This enables substitution of pattern fragments while keeping track
diff --git a/llvm/utils/TableGen/Common/CodeGenHwModes.cpp b/llvm/utils/TableGen/Common/CodeGenHwModes.cpp
index c744691ae9e08..9996b5a4451f0 100644
--- a/llvm/utils/TableGen/Common/CodeGenHwModes.cpp
+++ b/llvm/utils/TableGen/Common/CodeGenHwModes.cpp
@@ -41,14 +41,6 @@ void HwMode::dump() const { dbgs() << Name << ": " << Features << '\n'; }
 HwModeSelect::HwModeSelect(const Record *R, CodeGenHwModes &CGH) {
   std::vector<const Record *> Modes = R->getValueAsListOfDefs("Modes");
   std::vector<const Record *> Objects = R->getValueAsListOfDefs("Objects");
-  if (Modes.size() != Objects.size()) {
-    PrintError(
-        R->getLoc(),
-        "in record " + R->getName() +
-            " derived from HwModeSelect: the lists Modes and Objects should "
-            "have the same size");
-    report_fatal_error("error in target description.");
-  }
   for (auto [Mode, Object] : zip_equal(Modes, Objects)) {
     unsigned ModeId = CGH.getHwModeId(Mode);
     Items.emplace_back(ModeId, Object);
diff --git a/llvm/utils/TableGen/Common/CodeGenSchedule.cpp b/llvm/utils/TableGen/Common/CodeGenSchedule.cpp
index e84b4fd77a6c1..8e8b3196c91b0 100644
--- a/llvm/utils/TableGen/Common/CodeGenSchedule.cpp
+++ b/llvm/utils/TableGen/Common/CodeGenSchedule.cpp
@@ -413,9 +413,9 @@ void CodeGenSchedModels::collectSTIPredicates() {
   for (const Record *R : Records.getAllDerivedDefinitions("STIPredicate")) {
     const Record *Decl = R->getValueAsDef("Declaration");
 
-    const auto It = Decl2Index.find(Decl);
-    if (It == Decl2Index.end()) {
-      Decl2Index[Decl] = STIPredicates.size();
+    const auto [It, Inserted] =
+        Decl2Index.try_emplace(Decl, STIPredicates.size());
+    if (Inserted) {
       STIPredicateFunction Predicate(Decl);
       Predicate.addDefinition(R);
       STIPredicates.emplace_back(std::move(Predicate));
diff --git a/llvm/utils/TableGen/Common/CodeGenSchedule.h b/llvm/utils/TableGen/Common/CodeGenSchedule.h
index 0dce1fa308118..981782c17c48c 100644
--- a/llvm/utils/TableGen/Common/CodeGenSchedule.h
+++ b/llvm/utils/TableGen/Common/CodeGenSchedule.h
@@ -480,11 +480,8 @@ class CodeGenSchedModels {
   iterator_range<const_class_iterator> classes() const {
     return make_range(classes_begin(), classes_end());
   }
-  iterator_range<class_iterator> explicit_classes() {
-    return make_range(classes_begin(), classes_begin() + NumInstrSchedClasses);
-  }
-  iterator_range<const_class_iterator> explicit_classes() const {
-    return make_range(classes_begin(), classes_begin() + NumInstrSchedClasses);
+  ArrayRef<CodeGenSchedClass> explicit_classes() const {
+    return ArrayRef(SchedClasses).take_front(NumInstrSchedClasses);
   }
 
   const Record *getModelOrItinDef(const Record *ProcDef) const {
diff --git a/llvm/utils/TableGen/Common/CodeGenTarget.cpp b/llvm/utils/TableGen/Common/CodeGenTarget.cpp
index 96829a185a30b..e8286d295587f 100644
--- a/llvm/utils/TableGen/Common/CodeGenTarget.cpp
+++ b/llvm/utils/TableGen/Common/CodeGenTarget.cpp
@@ -28,8 +28,8 @@
 #include <tuple>
 using namespace llvm;
 
-cl::OptionCategory AsmParserCat("Options for -gen-asm-parser");
-cl::OptionCategory AsmWriterCat("Options for -gen-asm-writer");
+static cl::OptionCategory AsmParserCat("Options for -gen-asm-parser");
+static cl::OptionCategory AsmWriterCat("Options for -gen-asm-writer");
 
 static cl::opt<unsigned>
     AsmParserNum("asmparsernum", cl::init(0),
@@ -64,9 +64,9 @@ StringRef llvm::getEnumName(MVT::SimpleValueType T) {
 std::string llvm::getQualifiedName(const Record *R) {
   std::string Namespace;
   if (R->getValue("Namespace"))
-    Namespace = std::string(R->getValueAsString("Namespace"));
+    Namespace = R->getValueAsString("Namespace").str();
   if (Namespace.empty())
-    return std::string(R->getName());
+    return R->getName().str();
   return Namespace + "::" + R->getName().str();
 }
 
@@ -166,14 +166,15 @@ CodeGenRegBank &CodeGenTarget::getRegBank() const {
 const CodeGenRegisterClass *CodeGenTarget::getSuperRegForSubReg(
     const ValueTypeByHwMode &ValueTy, CodeGenRegBank &RegBank,
     const CodeGenSubRegIndex *SubIdx, bool MustBeAllocatable) const {
-  std::vector<CodeGenRegisterClass *> Candidates;
+  std::vector<const CodeGenRegisterClass *> Candidates;
   auto &RegClasses = RegBank.getRegClasses();
 
   // Try to find a register class which supports ValueTy, and also contains
   // SubIdx.
-  for (CodeGenRegisterClass &RC : RegClasses) {
+  for (const CodeGenRegisterClass &RC : RegClasses) {
     // Is there a subclass of this class which contains this subregister index?
-    CodeGenRegisterClass *SubClassWithSubReg = RC.getSubClassWithSubReg(SubIdx);
+    const CodeGenRegisterClass *SubClassWithSubReg =
+        RC.getSubClassWithSubReg(SubIdx);
     if (!SubClassWithSubReg)
       continue;
 
@@ -261,39 +262,40 @@ void CodeGenTarget::ReadInstructions() const {
 
   // Parse the instructions defined in the .td file.
   for (const Record *R : Insts) {
-    Instructions[R] = std::make_unique<CodeGenInstruction>(R);
-    if (Instructions[R]->isVariableLengthEncoding())
+    auto &Inst = Instructions[R];
+    Inst = std::make_unique<CodeGenInstruction>(R);
+    if (Inst->isVariableLengthEncoding())
       HasVariableLengthEncodings = true;
   }
 }
 
 static const CodeGenInstruction *GetInstByName(
-    const char *Name,
+    StringRef Name,
     const DenseMap<const Record *, std::unique_ptr<CodeGenInstruction>> &Insts,
     const RecordKeeper &Records) {
   const Record *Rec = Records.getDef(Name);
 
   const auto I = Insts.find(Rec);
   if (!Rec || I == Insts.end())
-    PrintFatalError(Twine("Could not find '") + Name + "' instruction!");
+    PrintFatalError("Could not find '" + Name + "' instruction!");
   return I->second.get();
 }
 
 static const char *FixedInstrs[] = {
 #define HANDLE_TARGET_OPCODE(OPC) #OPC,
 #include "llvm/Support/TargetOpcodes.def"
-    nullptr};
+};
 
 unsigned CodeGenTarget::getNumFixedInstructions() {
-  return std::size(FixedInstrs) - 1;
+  return std::size(FixedInstrs);
 }
 
 /// Return all of the instructions defined by the target, ordered by
 /// their enum value.
 void CodeGenTarget::ComputeInstrsByEnum() const {
   const auto &Insts = getInstructions();
-  for (const char *const *p = FixedInstrs; *p; ++p) {
-    const CodeGenInstruction *Instr = GetInstByName(*p, Insts, Records);
+  for (const char *Name : FixedInstrs) {
+    const CodeGenInstruction *Instr = GetInstByName(Name, Insts, Records);
     assert(Instr && "Missing target independent instruction");
     assert(Instr->Namespace == "TargetOpcode" && "Bad namespace");
     InstrsByEnum.push_back(Instr);
@@ -324,9 +326,8 @@ void CodeGenTarget::ComputeInstrsByEnum() const {
       });
 
   // Assign an enum value to each instruction according to the sorted order.
-  unsigned Num = 0;
-  for (const CodeGenInstruction *Inst : InstrsByEnum)
-    Inst->EnumVal = Num++;
+  for (const auto &[Idx, Inst] : enumerate(InstrsByEnum))
+    Inst->EnumVal = Idx;
 }
 
 /// isLittleEndianEncoding - Return whether this target encodes its instruction
diff --git a/llvm/utils/TableGen/Common/GlobalISel/GlobalISelMatchTable.cpp b/llvm/utils/TableGen/Common/GlobalISel/GlobalISelMatchTable.cpp
index f0cd98dd2dee0..21db090f62cce 100644
--- a/llvm/utils/TableGen/Common/GlobalISel/GlobalISelMatchTable.cpp
+++ b/llvm/utils/TableGen/Common/GlobalISel/GlobalISelMatchTable.cpp
@@ -227,26 +227,12 @@ MatchTableRecord MatchTable::NamedValue(unsigned NumBytes,
                           MatchTableRecord::MTRF_CommaFollows);
 }
 
-MatchTableRecord MatchTable::NamedValue(unsigned NumBytes, StringRef NamedValue,
-                                        int64_t RawValue) {
-  return MatchTableRecord(std::nullopt, NamedValue, NumBytes,
-                          MatchTableRecord::MTRF_CommaFollows, RawValue);
-}
-
 MatchTableRecord MatchTable::NamedValue(unsigned NumBytes, StringRef Namespace,
                                         StringRef NamedValue) {
   return MatchTableRecord(std::nullopt, (Namespace + "::" + NamedValue).str(),
                           NumBytes, MatchTableRecord::MTRF_CommaFollows);
 }
 
-MatchTableRecord MatchTable::NamedValue(unsigned NumBytes, StringRef Namespace,
-                                        StringRef NamedValue,
-                                        int64_t RawValue) {
-  return MatchTableRecord(std::nullopt, (Namespace + "::" + NamedValue).str(),
-                          NumBytes, MatchTableRecord::MTRF_CommaFollows,
-                          RawValue);
-}
-
 MatchTableRecord MatchTable::IntValue(unsigned NumBytes, int64_t IntValue) {
   assert(isUIntN(NumBytes * 8, IntValue) || isIntN(NumBytes * 8, IntValue));
   auto Str = llvm::to_string(IntValue);
@@ -300,11 +286,28 @@ MatchTableRecord MatchTable::JumpTarget(unsigned LabelID) {
 void MatchTable::emitUse(raw_ostream &OS) const { OS << "MatchTable" << ID; }
 
 void MatchTable::emitDeclaration(raw_ostream &OS) const {
-  unsigned Indentation = 4;
+  static constexpr unsigned BaseIndent = 4;
+  unsigned Indentation = 0;
   OS << "  constexpr static uint8_t MatchTable" << ID << "[] = {";
   LineBreak.emit(OS, true, *this);
-  OS << std::string(Indentation, ' ');
 
+  // We want to display the table index of each line in a consistent
+  // manner. It has to appear as a column on the left side of the table.
+  // To determine how wide the column needs to be, check how many characters
+  // we need to fit the largest possible index in the current table.
+  const unsigned NumColsForIdx = llvm::to_string(CurrentSize).size();
+
+  unsigned CurIndex = 0;
+  const auto BeginLine = [&]() {
+    OS.indent(BaseIndent);
+    std::string IdxStr = llvm::to_string(CurIndex);
+    // Pad the string with spaces to keep the size of the prefix consistent.
+    OS << " /* ";
+    OS.indent(NumColsForIdx - IdxStr.size()) << IdxStr << " */ ";
+    OS.indent(Indentation);
+  };
+
+  BeginLine();
   for (auto I = Contents.begin(), E = Contents.end(); I != E; ++I) {
     bool LineBreakIsNext = false;
     const auto &NextI = std::next(I);
@@ -320,11 +323,14 @@ void MatchTable::emitDeclaration(raw_ostream &OS) const {
 
     I->emit(OS, LineBreakIsNext, *this);
     if (I->Flags & MatchTableRecord::MTRF_LineBreakFollows)
-      OS << std::string(Indentation, ' ');
+      BeginLine();
 
     if (I->Flags & MatchTableRecord::MTRF_Outdent)
       Indentation -= 2;
+
+    CurIndex += I->size();
   }
+  assert(CurIndex == CurrentSize);
   OS << "}; // Size: " << CurrentSize << " bytes\n";
 }
 
@@ -651,8 +657,8 @@ void SwitchMatcher::emit(MatchTable &Table) {
                 [&Table]() { return Table.allocateLabelID(); });
   const unsigned Default = Table.allocateLabelID();
 
-  const int64_t LowerBound = Values.begin()->getRawValue();
-  const int64_t UpperBound = Values.rbegin()->getRawValue() + 1;
+  const int64_t LowerBound = Values.begin()->RawValue;
+  const int64_t UpperBound = Values.rbegin()->RawValue + 1;
 
   emitPredicateSpecificOpcodes(*Condition, Table);
 
@@ -664,10 +670,11 @@ void SwitchMatcher::emit(MatchTable &Table) {
   auto VI = Values.begin();
   for (unsigned I = 0, E = Values.size(); I < E; ++I) {
     auto V = *VI++;
-    while (J++ < V.getRawValue())
+    while (J++ < V.RawValue)
       Table << MatchTable::IntValue(4, 0);
-    V.turnIntoComment();
-    Table << MatchTable::LineBreak << V << MatchTable::JumpTarget(LabelIDs[I]);
+    V.Record.turnIntoComment();
+    Table << MatchTable::LineBreak << V.Record
+          << MatchTable::JumpTarget(LabelIDs[I]);
   }
   Table << MatchTable::LineBreak;
 
@@ -1145,11 +1152,11 @@ void SameOperandMatcher::emitPredicateOpcodes(MatchTable &Table,
 
 std::map<LLTCodeGen, unsigned> LLTOperandMatcher::TypeIDValues;
 
-MatchTableRecord LLTOperandMatcher::getValue() const {
+RecordAndValue LLTOperandMatcher::getValue() const {
   const auto VI = TypeIDValues.find(Ty);
   if (VI == TypeIDValues.end())
     return MatchTable::NamedValue(1, getTy().getCxxEnumValue());
-  return MatchTable::NamedValue(1, getTy().getCxxEnumValue(), VI->second);
+  return {MatchTable::NamedValue(1, getTy().getCxxEnumValue()), VI->second};
 }
 
 bool LLTOperandMatcher::hasValue() const {
@@ -1167,7 +1174,8 @@ void LLTOperandMatcher::emitPredicateOpcodes(MatchTable &Table,
           << MatchTable::ULEB128Value(InsnVarID);
   }
   Table << MatchTable::Comment("Op") << MatchTable::ULEB128Value(OpIdx)
-        << MatchTable::Comment("Type") << getValue() << MatchTable::LineBreak;
+        << MatchTable::Comment("Type") << getValue().Record
+        << MatchTable::LineBreak;
 }
 
 //===- PointerToAnyOperandMatcher -----------------------------------------===//
@@ -1411,12 +1419,12 @@ Error OperandMatcher::addTypeCheckPredicate(const TypeSetByHwMode &VTy,
 DenseMap<const CodeGenInstruction *, unsigned>
     InstructionOpcodeMatcher::OpcodeValues;
 
-MatchTableRecord
+RecordAndValue
 InstructionOpcodeMatcher::getInstValue(const CodeGenInstruction *I) const {
   const auto VI = OpcodeValues.find(I);
   if (VI != OpcodeValues.end())
-    return MatchTable::NamedValue(2, I->Namespace, I->TheDef->getName(),
-                                  VI->second);
+    return {MatchTable::NamedValue(2, I->Namespace, I->TheDef->getName()),
+            VI->second};
   return MatchTable::NamedValue(2, I->Namespace, I->TheDef->getName());
 }
 
@@ -1428,14 +1436,14 @@ void InstructionOpcodeMatcher::initOpcodeValuesMap(
     OpcodeValues[I] = Target.getInstrIntValue(I->TheDef);
 }
 
-MatchTableRecord InstructionOpcodeMatcher::getValue() const {
+RecordAndValue InstructionOpcodeMatcher::getValue() const {
   assert(Insts.size() == 1);
 
   const CodeGenInstruction *I = Insts[0];
   const auto VI = OpcodeValues.find(I);
   if (VI != OpcodeValues.end())
-    return MatchTable::NamedValue(2, I->Namespace, I->TheDef->getName(),
-                                  VI->second);
+    return {MatchTable::NamedValue(2, I->Namespace, I->TheDef->getName()),
+            VI->second};
   return MatchTable::NamedValue(2, I->Namespace, I->TheDef->getName());
 }
 
@@ -1447,7 +1455,7 @@ void InstructionOpcodeMatcher::emitPredicateOpcodes(MatchTable &Table,
         << MatchTable::ULEB128Value(InsnVarID);
 
   for (const CodeGenInstruction *I : Insts)
-    Table << getInstValue(I);
+    Table << getInstValue(I).Record;
   Table << MatchTable::LineBreak;
 }
 
diff --git a/llvm/utils/TableGen/Common/GlobalISel/GlobalISelMatchTable.h b/llvm/utils/TableGen/Common/GlobalISel/GlobalISelMatchTable.h
index e7914a613973b..77c8bc290faaf 100644
--- a/llvm/utils/TableGen/Common/GlobalISel/GlobalISelMatchTable.h
+++ b/llvm/utils/TableGen/Common/GlobalISel/GlobalISelMatchTable.h
@@ -145,14 +145,10 @@ struct MatchTableRecord {
   /// A bitfield of RecordFlagsBits flags.
   unsigned Flags;
 
-  /// The actual run-time value, if known
-  int64_t RawValue;
-
   MatchTableRecord(std::optional<unsigned> LabelID_, StringRef EmitStr,
-                   unsigned NumElements, unsigned Flags,
-                   int64_t RawValue = std::numeric_limits<int64_t>::min())
+                   unsigned NumElements, unsigned Flags)
       : LabelID(LabelID_.value_or(~0u)), EmitStr(EmitStr),
-        NumElements(NumElements), Flags(Flags), RawValue(RawValue) {
+        NumElements(NumElements), Flags(Flags) {
     assert((!LabelID_ || LabelID != ~0u) &&
            "This value is reserved for non-labels");
   }
@@ -166,12 +162,6 @@ struct MatchTableRecord {
     NumElements = 0;
   }
 
-  /// For Jump Table generation purposes
-  bool operator<(const MatchTableRecord &Other) const {
-    return RawValue < Other.RawValue;
-  }
-  int64_t getRawValue() const { return RawValue; }
-
   void emit(raw_ostream &OS, bool LineBreakNextAfterThis,
             const MatchTable &Table) const;
   unsigned size() const { return NumElements; }
@@ -202,12 +192,8 @@ class MatchTable {
   static MatchTableRecord Comment(StringRef Comment);
   static MatchTableRecord Opcode(StringRef Opcode, int IndentAdjust = 0);
   static MatchTableRecord NamedValue(unsigned NumBytes, StringRef NamedValue);
-  static MatchTableRecord NamedValue(unsigned NumBytes, StringRef NamedValue,
-                                     int64_t RawValue);
   static MatchTableRecord NamedValue(unsigned NumBytes, StringRef Namespace,
                                      StringRef NamedValue);
-  static MatchTableRecord NamedValue(unsigned NumBytes, StringRef Namespace,
-                                     StringRef NamedValue, int64_t RawValue);
   static MatchTableRecord IntValue(unsigned NumBytes, int64_t IntValue);
   static MatchTableRecord ULEB128Value(uint64_t IntValue);
   static MatchTableRecord Label(unsigned LabelID);
@@ -400,6 +386,20 @@ class GroupMatcher final : public Matcher {
   bool candidateConditionMatches(const PredicateMatcher &Predicate) const;
 };
 
+/// MatchTableRecord and associated value, for jump table generation.
+struct RecordAndValue {
+  MatchTableRecord Record;
+  int64_t RawValue;
+
+  RecordAndValue(MatchTableRecord Record,
+                 int64_t RawValue = std::numeric_limits<int64_t>::min())
+      : Record(std::move(Record)), RawValue(RawValue) {}
+
+  bool operator<(const RecordAndValue &Other) const {
+    return RawValue < Other.RawValue;
+  }
+};
+
 class SwitchMatcher : public Matcher {
   /// All the nested matchers, representing distinct switch-cases. The first
   /// conditions (as Matcher::getFirstCondition() reports) of all the nested
@@ -414,7 +414,7 @@ class SwitchMatcher : public Matcher {
 
   /// Temporary set used to check that the case values don't repeat within the
   /// same switch.
-  std::set<MatchTableRecord> Values;
+  std::set<RecordAndValue> Values;
 
   /// An owning collection for any auxiliary matchers created while optimizing
   /// nested matchers contained.
@@ -874,7 +874,7 @@ class PredicateMatcher {
     return hasValue() && PredicateMatcher::isIdentical(B);
   }
 
-  virtual MatchTableRecord getValue() const {
+  virtual RecordAndValue getValue() const {
     assert(hasValue() && "Can not get a value of a value-less predicate!");
     llvm_unreachable("Not implemented yet");
   }
@@ -968,7 +968,7 @@ class LLTOperandMatcher : public OperandPredicateMatcher {
            Ty == cast<LLTOperandMatcher>(&B)->Ty;
   }
 
-  MatchTableRecord getValue() const override;
+  RecordAndValue getValue() const override;
   bool hasValue() const override;
 
   LLTCodeGen getTy() const { return Ty; }
@@ -1378,7 +1378,7 @@ class InstructionOpcodeMatcher : public InstructionPredicateMatcher {
 
   static DenseMap<const CodeGenInstruction *, unsigned> OpcodeValues;
 
-  MatchTableRecord getInstValue(const CodeGenInstruction *I) const;
+  RecordAndValue getInstValue(const CodeGenInstruction *I) const;
 
 public:
   static void initOpcodeValuesMap(const CodeGenTarget &Target);
@@ -1405,7 +1405,7 @@ class InstructionOpcodeMatcher : public InstructionPredicateMatcher {
 
   // TODO: This is used for the SwitchMatcher optimization. We should be able to
   // return a list of the opcodes to match.
-  MatchTableRecord getValue() const override;
+  RecordAndValue getValue() const override;
 
   void emitPredicateOpcodes(MatchTable &Table,
                             RuleMatcher &Rule) const override;
diff --git a/llvm/utils/TableGen/DAGISelMatcherGen.cpp b/llvm/utils/TableGen/DAGISelMatcherGen.cpp
index e1c25075e384d..55213b74ffb0d 100644
--- a/llvm/utils/TableGen/DAGISelMatcherGen.cpp
+++ b/llvm/utils/TableGen/DAGISelMatcherGen.cpp
@@ -580,8 +580,8 @@ bool MatcherGen::EmitMatcherCode(unsigned Variant) {
   // checks (e.g. addrmode matches).  We emit this after the structural match
   // because they are generally more expensive to evaluate and more difficult to
   // factor.
-  for (unsigned i = 0, e = MatchedComplexPatterns.size(); i != e; ++i) {
-    auto &N = *MatchedComplexPatterns[i].first;
+  for (const auto &MCP : MatchedComplexPatterns) {
+    auto &N = *MCP.first;
 
     // Remember where the results of this match get stuck.
     if (N.isLeaf()) {
@@ -595,7 +595,7 @@ bool MatcherGen::EmitMatcherCode(unsigned Variant) {
     }
 
     // Get the slot we recorded the value in from the name on the node.
-    unsigned RecNodeEntry = MatchedComplexPatterns[i].second;
+    unsigned RecNodeEntry = MCP.second;
 
     const ComplexPattern *CP = N.getComplexPatternInfo(CGP);
     assert(CP && "Not a valid ComplexPattern!");
diff --git a/llvm/utils/TableGen/DAGISelMatcherOpt.cpp b/llvm/utils/TableGen/DAGISelMatcherOpt.cpp
index f747944543cfd..9028a60027ffe 100644
--- a/llvm/utils/TableGen/DAGISelMatcherOpt.cpp
+++ b/llvm/utils/TableGen/DAGISelMatcherOpt.cpp
@@ -22,162 +22,300 @@ using namespace llvm;
 
 /// ContractNodes - Turn multiple matcher node patterns like 'MoveChild+Record'
 /// into single compound nodes like RecordChild.
-static void ContractNodes(std::unique_ptr<Matcher> &MatcherPtr,
+static void ContractNodes(std::unique_ptr<Matcher> &InputMatcherPtr,
                           const CodeGenDAGPatterns &CGP) {
-  // If we reached the end of the chain, we're done.
-  Matcher *N = MatcherPtr.get();
-  if (!N)
-    return;
+  std::unique_ptr<Matcher> *MatcherPtr = &InputMatcherPtr;
+  while (true) {
+    Matcher *N = MatcherPtr->get();
+
+    // If we have a scope node, walk down all of the children.
+    if (auto *Scope = dyn_cast<ScopeMatcher>(N)) {
+      for (unsigned i = 0, e = Scope->getNumChildren(); i != e; ++i) {
+        std::unique_ptr<Matcher> Child(Scope->takeChild(i));
+        ContractNodes(Child, CGP);
+        Scope->resetChild(i, Child.release());
+      }
+      return;
+    }
 
-  // If we have a scope node, walk down all of the children.
-  if (ScopeMatcher *Scope = dyn_cast<ScopeMatcher>(N)) {
-    for (unsigned i = 0, e = Scope->getNumChildren(); i != e; ++i) {
-      std::unique_ptr<Matcher> Child(Scope->takeChild(i));
-      ContractNodes(Child, CGP);
-      Scope->resetChild(i, Child.release());
+    // If we found a movechild node with a node that comes in a 'foochild' form,
+    // transform it.
+    if (MoveChildMatcher *MC = dyn_cast<MoveChildMatcher>(N)) {
+      Matcher *New = nullptr;
+      if (RecordMatcher *RM = dyn_cast<RecordMatcher>(MC->getNext()))
+        if (MC->getChildNo() < 8) // Only have RecordChild0...7
+          New = new RecordChildMatcher(MC->getChildNo(), RM->getWhatFor(),
+                                       RM->getResultNo());
+
+      if (CheckTypeMatcher *CT = dyn_cast<CheckTypeMatcher>(MC->getNext()))
+        if (MC->getChildNo() < 8 && // Only have CheckChildType0...7
+            CT->getResNo() == 0)    // CheckChildType checks res #0
+          New = new CheckChildTypeMatcher(MC->getChildNo(), CT->getType());
+
+      if (CheckSameMatcher *CS = dyn_cast<CheckSameMatcher>(MC->getNext()))
+        if (MC->getChildNo() < 4) // Only have CheckChildSame0...3
+          New =
+              new CheckChildSameMatcher(MC->getChildNo(), CS->getMatchNumber());
+
+      if (CheckIntegerMatcher *CI =
+              dyn_cast<CheckIntegerMatcher>(MC->getNext()))
+        if (MC->getChildNo() < 5) // Only have CheckChildInteger0...4
+          New = new CheckChildIntegerMatcher(MC->getChildNo(), CI->getValue());
+
+      if (auto *CCC = dyn_cast<CheckCondCodeMatcher>(MC->getNext()))
+        if (MC->getChildNo() == 2) // Only have CheckChild2CondCode
+          New = new CheckChild2CondCodeMatcher(CCC->getCondCodeName());
+
+      if (New) {
+        // Insert the new node.
+        New->setNext(MatcherPtr->release());
+        MatcherPtr->reset(New);
+        // Remove the old one.
+        MC->setNext(MC->getNext()->takeNext());
+        continue;
+      }
     }
-    return;
-  }
 
-  // If we found a movechild node with a node that comes in a 'foochild' form,
-  // transform it.
-  if (MoveChildMatcher *MC = dyn_cast<MoveChildMatcher>(N)) {
-    Matcher *New = nullptr;
-    if (RecordMatcher *RM = dyn_cast<RecordMatcher>(MC->getNext()))
-      if (MC->getChildNo() < 8) // Only have RecordChild0...7
-        New = new RecordChildMatcher(MC->getChildNo(), RM->getWhatFor(),
-                                     RM->getResultNo());
-
-    if (CheckTypeMatcher *CT = dyn_cast<CheckTypeMatcher>(MC->getNext()))
-      if (MC->getChildNo() < 8 && // Only have CheckChildType0...7
-          CT->getResNo() == 0)    // CheckChildType checks res #0
-        New = new CheckChildTypeMatcher(MC->getChildNo(), CT->getType());
-
-    if (CheckSameMatcher *CS = dyn_cast<CheckSameMatcher>(MC->getNext()))
-      if (MC->getChildNo() < 4) // Only have CheckChildSame0...3
-        New = new CheckChildSameMatcher(MC->getChildNo(), CS->getMatchNumber());
-
-    if (CheckIntegerMatcher *CI = dyn_cast<CheckIntegerMatcher>(MC->getNext()))
-      if (MC->getChildNo() < 5) // Only have CheckChildInteger0...4
-        New = new CheckChildIntegerMatcher(MC->getChildNo(), CI->getValue());
-
-    if (auto *CCC = dyn_cast<CheckCondCodeMatcher>(MC->getNext()))
-      if (MC->getChildNo() == 2) // Only have CheckChild2CondCode
-        New = new CheckChild2CondCodeMatcher(CCC->getCondCodeName());
-
-    if (New) {
-      // Insert the new node.
-      New->setNext(MatcherPtr.release());
-      MatcherPtr.reset(New);
-      // Remove the old one.
-      MC->setNext(MC->getNext()->takeNext());
-      return ContractNodes(MatcherPtr, CGP);
+    // Turn MoveParent->MoveChild into MoveSibling.
+    if (auto *MP = dyn_cast<MoveParentMatcher>(N)) {
+      if (auto *MC = dyn_cast<MoveChildMatcher>(MP->getNext())) {
+        auto *MS = new MoveSiblingMatcher(MC->getChildNo());
+        MS->setNext(MC->takeNext());
+        MatcherPtr->reset(MS);
+        continue;
+      }
     }
-  }
 
-  // Zap movechild -> moveparent.
-  if (MoveChildMatcher *MC = dyn_cast<MoveChildMatcher>(N))
-    if (MoveParentMatcher *MP = dyn_cast<MoveParentMatcher>(MC->getNext())) {
-      MatcherPtr.reset(MP->takeNext());
-      return ContractNodes(MatcherPtr, CGP);
+    // Uncontract MoveSibling if it will help form other child operations.
+    if (auto *MS = dyn_cast<MoveSiblingMatcher>(N)) {
+      if (auto *RM = dyn_cast<RecordMatcher>(MS->getNext())) {
+        // Turn MoveSibling->Record->MoveParent into MoveParent->RecordChild.
+        if (auto *MP = dyn_cast<MoveParentMatcher>(RM->getNext())) {
+          if (MS->getSiblingNo() < 8) { // Only have RecordChild0...7
+            auto *NewMP = new MoveParentMatcher();
+            auto *NewRCM = new RecordChildMatcher(
+                MS->getSiblingNo(), RM->getWhatFor(), RM->getResultNo());
+            NewMP->setNext(NewRCM);
+            NewRCM->setNext(MP->takeNext());
+            MatcherPtr->reset(NewMP);
+            continue;
+          }
+        }
+
+        // Turn MoveSibling->Record->CheckType->MoveParent into
+        // MoveParent->RecordChild->CheckChildType.
+        if (auto *CT = dyn_cast<CheckTypeMatcher>(RM->getNext())) {
+          if (auto *MP = dyn_cast<MoveParentMatcher>(CT->getNext())) {
+            if (MS->getSiblingNo() < 8 && // Only have CheckChildType0...7
+                CT->getResNo() == 0) {    // CheckChildType checks res #0
+              auto *NewMP = new MoveParentMatcher();
+              auto *NewRCM = new RecordChildMatcher(
+                  MS->getSiblingNo(), RM->getWhatFor(), RM->getResultNo());
+              auto *NewCCT =
+                  new CheckChildTypeMatcher(MS->getSiblingNo(), CT->getType());
+              NewMP->setNext(NewRCM);
+              NewRCM->setNext(NewCCT);
+              NewCCT->setNext(MP->takeNext());
+              MatcherPtr->reset(NewMP);
+              continue;
+            }
+          }
+        }
+      }
+
+      // Turn MoveSibling->CheckType->MoveParent into
+      // MoveParent->CheckChildType.
+      if (auto *CT = dyn_cast<CheckTypeMatcher>(MS->getNext())) {
+        if (auto *MP = dyn_cast<MoveParentMatcher>(CT->getNext())) {
+          if (MS->getSiblingNo() < 8 && // Only have CheckChildType0...7
+              CT->getResNo() == 0) {    // CheckChildType checks res #0
+            auto *NewMP = new MoveParentMatcher();
+            auto *NewCCT =
+                new CheckChildTypeMatcher(MS->getSiblingNo(), CT->getType());
+            NewMP->setNext(NewCCT);
+            NewCCT->setNext(MP->takeNext());
+            MatcherPtr->reset(NewMP);
+            continue;
+          }
+        }
+      }
+
+      // Turn MoveSibling->CheckInteger->MoveParent into
+      // MoveParent->CheckChildInteger.
+      if (auto *CI = dyn_cast<CheckIntegerMatcher>(MS->getNext())) {
+        if (auto *MP = dyn_cast<MoveParentMatcher>(CI->getNext())) {
+          if (MS->getSiblingNo() < 5) { // Only have CheckChildInteger0...4
+            auto *NewMP = new MoveParentMatcher();
+            auto *NewCCI = new CheckChildIntegerMatcher(MS->getSiblingNo(),
+                                                        CI->getValue());
+            NewMP->setNext(NewCCI);
+            NewCCI->setNext(MP->takeNext());
+            MatcherPtr->reset(NewMP);
+            continue;
+          }
+        }
+
+        // Turn MoveSibling->CheckInteger->CheckType->MoveParent into
+        // MoveParent->CheckChildInteger->CheckType.
+        if (auto *CT = dyn_cast<CheckTypeMatcher>(CI->getNext())) {
+          if (auto *MP = dyn_cast<MoveParentMatcher>(CT->getNext())) {
+            if (MS->getSiblingNo() < 5 && // Only have CheckChildInteger0...4
+                CT->getResNo() == 0) {    // CheckChildType checks res #0
+              auto *NewMP = new MoveParentMatcher();
+              auto *NewCCI = new CheckChildIntegerMatcher(MS->getSiblingNo(),
+                                                          CI->getValue());
+              auto *NewCCT =
+                  new CheckChildTypeMatcher(MS->getSiblingNo(), CT->getType());
+              NewMP->setNext(NewCCI);
+              NewCCI->setNext(NewCCT);
+              NewCCT->setNext(MP->takeNext());
+              MatcherPtr->reset(NewMP);
+              continue;
+            }
+          }
+        }
+      }
+
+      // Turn MoveSibling->CheckCondCode->MoveParent into
+      // MoveParent->CheckChild2CondCode.
+      if (auto *CCC = dyn_cast<CheckCondCodeMatcher>(MS->getNext())) {
+        if (auto *MP = dyn_cast<MoveParentMatcher>(CCC->getNext())) {
+          if (MS->getSiblingNo() == 2) { // Only have CheckChild2CondCode
+            auto *NewMP = new MoveParentMatcher();
+            auto *NewCCCC =
+                new CheckChild2CondCodeMatcher(CCC->getCondCodeName());
+            NewMP->setNext(NewCCCC);
+            NewCCCC->setNext(MP->takeNext());
+            MatcherPtr->reset(NewMP);
+            continue;
+          }
+        }
+      }
+
+      // Turn MoveSibling->CheckSame->MoveParent into
+      // MoveParent->CheckChildSame.
+      if (auto *CS = dyn_cast<CheckSameMatcher>(MS->getNext())) {
+        if (auto *MP = dyn_cast<MoveParentMatcher>(CS->getNext())) {
+          if (MS->getSiblingNo() < 4) { // Only have CheckChildSame0...3
+            auto *NewMP = new MoveParentMatcher();
+            auto *NewCCS = new CheckChildSameMatcher(MS->getSiblingNo(),
+                                                     CS->getMatchNumber());
+            NewMP->setNext(NewCCS);
+            NewCCS->setNext(MP->takeNext());
+            MatcherPtr->reset(NewMP);
+            continue;
+          }
+        }
+
+        // Turn MoveSibling->CheckSame->CheckType->MoveParent into
+        // MoveParent->CheckChildSame->CheckChildType.
+        if (auto *CT = dyn_cast<CheckTypeMatcher>(CS->getNext())) {
+          if (auto *MP = dyn_cast<MoveParentMatcher>(CT->getNext())) {
+            if (MS->getSiblingNo() < 4 && // Only have CheckChildSame0...3
+                CT->getResNo() == 0) {    // CheckChildType checks res #0
+              auto *NewMP = new MoveParentMatcher();
+              auto *NewCCS = new CheckChildSameMatcher(MS->getSiblingNo(),
+                                                       CS->getMatchNumber());
+              auto *NewCCT =
+                  new CheckChildTypeMatcher(MS->getSiblingNo(), CT->getType());
+              NewMP->setNext(NewCCS);
+              NewCCS->setNext(NewCCT);
+              NewCCT->setNext(MP->takeNext());
+              MatcherPtr->reset(NewMP);
+              continue;
+            }
+          }
+        }
+      }
+
+      // Turn MoveSibling->MoveParent into MoveParent.
+      if (isa<MoveParentMatcher>(MS->getNext())) {
+        MatcherPtr->reset(MS->takeNext());
+        continue;
+      }
     }
 
-  // Turn EmitNode->CompleteMatch into MorphNodeTo if we can.
-  if (EmitNodeMatcher *EN = dyn_cast<EmitNodeMatcher>(N))
-    if (CompleteMatchMatcher *CM =
-            dyn_cast<CompleteMatchMatcher>(EN->getNext())) {
-      // We can only use MorphNodeTo if the result values match up.
-      unsigned RootResultFirst = EN->getFirstResultSlot();
-      bool ResultsMatch = true;
-      for (unsigned i = 0, e = CM->getNumResults(); i != e; ++i)
-        if (CM->getResult(i) != RootResultFirst + i)
+    // Zap movechild -> moveparent.
+    if (MoveChildMatcher *MC = dyn_cast<MoveChildMatcher>(N))
+      if (MoveParentMatcher *MP = dyn_cast<MoveParentMatcher>(MC->getNext())) {
+        MatcherPtr->reset(MP->takeNext());
+        continue;
+      }
+
+    // Turn EmitNode->CompleteMatch into MorphNodeTo if we can.
+    if (EmitNodeMatcher *EN = dyn_cast<EmitNodeMatcher>(N)) {
+      if (CompleteMatchMatcher *CM =
+              dyn_cast<CompleteMatchMatcher>(EN->getNext())) {
+        // We can only use MorphNodeTo if the result values match up.
+        unsigned RootResultFirst = EN->getFirstResultSlot();
+        bool ResultsMatch = true;
+        for (unsigned i = 0, e = CM->getNumResults(); i != e; ++i)
+          if (CM->getResult(i) != RootResultFirst + i)
+            ResultsMatch = false;
+
+        // If the selected node defines a subset of the glue/chain results, we
+        // can't use MorphNodeTo.  For example, we can't use MorphNodeTo if the
+        // matched pattern has a chain but the root node doesn't.
+        const PatternToMatch &Pattern = CM->getPattern();
+
+        if (!EN->hasChain() &&
+            Pattern.getSrcPattern().NodeHasProperty(SDNPHasChain, CGP))
           ResultsMatch = false;
 
-      // If the selected node defines a subset of the glue/chain results, we
-      // can't use MorphNodeTo.  For example, we can't use MorphNodeTo if the
-      // matched pattern has a chain but the root node doesn't.
-      const PatternToMatch &Pattern = CM->getPattern();
-
-      if (!EN->hasChain() &&
-          Pattern.getSrcPattern().NodeHasProperty(SDNPHasChain, CGP))
-        ResultsMatch = false;
-
-      // If the matched node has glue and the output root doesn't, we can't
-      // use MorphNodeTo.
-      //
-      // NOTE: Strictly speaking, we don't have to check for glue here
-      // because the code in the pattern generator doesn't handle it right.  We
-      // do it anyway for thoroughness.
-      if (!EN->hasOutGlue() &&
-          Pattern.getSrcPattern().NodeHasProperty(SDNPOutGlue, CGP))
-        ResultsMatch = false;
+        // If the matched node has glue and the output root doesn't, we can't
+        // use MorphNodeTo.
+        //
+        // NOTE: Strictly speaking, we don't have to check for glue here
+        // because the code in the pattern generator doesn't handle it right. We
+        // do it anyway for thoroughness.
+        if (!EN->hasOutGlue() &&
+            Pattern.getSrcPattern().NodeHasProperty(SDNPOutGlue, CGP))
+          ResultsMatch = false;
 
 #if 0
-      // If the root result node defines more results than the source root node
-      // *and* has a chain or glue input, then we can't match it because it
-      // would end up replacing the extra result with the chain/glue.
-      if ((EN->hasGlue() || EN->hasChain()) &&
-          EN->getNumNonChainGlueVTs() > ... need to get no results reliably ...)
-        ResultMatch = false;
+        // If the root result node defines more results than the source root
+        // node *and* has a chain or glue input, then we can't match it because
+        // it would end up replacing the extra result with the chain/glue.
+        if ((EN->hasGlue() || EN->hasChain()) &&
+            EN->getNumNonChainGlueVTs() > ...need to get no results reliably...)
+          ResultMatch = false;
 #endif
 
-      if (ResultsMatch) {
-        const SmallVectorImpl<MVT::SimpleValueType> &VTs = EN->getVTList();
-        const SmallVectorImpl<unsigned> &Operands = EN->getOperandList();
-        MatcherPtr.reset(new MorphNodeToMatcher(
-            EN->getInstruction(), VTs, Operands, EN->hasChain(),
-            EN->hasInGlue(), EN->hasOutGlue(), EN->hasMemRefs(),
-            EN->getNumFixedArityOperands(), Pattern));
-        return;
+        if (ResultsMatch) {
+          const SmallVectorImpl<MVT::SimpleValueType> &VTs = EN->getVTList();
+          const SmallVectorImpl<unsigned> &Operands = EN->getOperandList();
+          MatcherPtr->reset(new MorphNodeToMatcher(
+              EN->getInstruction(), VTs, Operands, EN->hasChain(),
+              EN->hasInGlue(), EN->hasOutGlue(), EN->hasMemRefs(),
+              EN->getNumFixedArityOperands(), Pattern));
+          return;
+        }
       }
-
-      // FIXME2: Kill off all the SelectionDAG::SelectNodeTo and getMachineNode
-      // variants.
     }
 
-  ContractNodes(N->getNextPtr(), CGP);
-
-  // If we have a CheckType/CheckChildType/Record node followed by a
-  // CheckOpcode, invert the two nodes.  We prefer to do structural checks
-  // before type checks, as this opens opportunities for factoring on targets
-  // like X86 where many operations are valid on multiple types.
-  if ((isa<CheckTypeMatcher>(N) || isa<CheckChildTypeMatcher>(N) ||
-       isa<RecordMatcher>(N)) &&
-      isa<CheckOpcodeMatcher>(N->getNext())) {
+  // If we have a Record node followed by a CheckOpcode, invert the two nodes.
+  // We prefer to do structural checks before type checks, as this opens
+  // opportunities for factoring on targets like X86 where many operations are
+  // valid on multiple types.
+  if (isa<RecordMatcher>(N) && isa<CheckOpcodeMatcher>(N->getNext())) {
     // Unlink the two nodes from the list.
-    Matcher *CheckType = MatcherPtr.release();
+    Matcher *CheckType = MatcherPtr->release();
     Matcher *CheckOpcode = CheckType->takeNext();
     Matcher *Tail = CheckOpcode->takeNext();
 
     // Relink them.
-    MatcherPtr.reset(CheckOpcode);
+    MatcherPtr->reset(CheckOpcode);
     CheckOpcode->setNext(CheckType);
     CheckType->setNext(Tail);
-    return ContractNodes(MatcherPtr, CGP);
+    continue;
   }
 
-  // If we have a MoveParent followed by a MoveChild, we convert it to
-  // MoveSibling.
-  if (auto *MP = dyn_cast<MoveParentMatcher>(N)) {
-    if (auto *MC = dyn_cast<MoveChildMatcher>(MP->getNext())) {
-      auto *MS = new MoveSiblingMatcher(MC->getChildNo());
-      MS->setNext(MC->takeNext());
-      MatcherPtr.reset(MS);
-      return ContractNodes(MatcherPtr, CGP);
-    }
-    if (auto *RC = dyn_cast<RecordChildMatcher>(MP->getNext())) {
-      if (auto *MC = dyn_cast<MoveChildMatcher>(RC->getNext())) {
-        if (RC->getChildNo() == MC->getChildNo()) {
-          auto *MS = new MoveSiblingMatcher(MC->getChildNo());
-          auto *RM = new RecordMatcher(RC->getWhatFor(), RC->getResultNo());
-          // Insert the new node.
-          RM->setNext(MC->takeNext());
-          MS->setNext(RM);
-          MatcherPtr.reset(MS);
-          return ContractNodes(MatcherPtr, CGP);
-        }
-      }
-    }
+  // No contractions were performed, go to next node.
+  MatcherPtr = &(MatcherPtr->get()->getNextPtr());
+
+  // If we reached the end of the chain, we're done.
+  if (!MatcherPtr->get())
+    return;
   }
 }
 
@@ -191,7 +329,9 @@ static Matcher *FindNodeWithKind(Matcher *M, Matcher::KindTy Kind) {
   return nullptr;
 }
 
-/// FactorNodes - Turn matches like this:
+static void FactorNodes(std::unique_ptr<Matcher> &InputMatcherPtr);
+
+/// Turn matches like this:
 ///   Scope
 ///     OPC_CheckType i32
 ///       ABC
@@ -203,22 +343,8 @@ static Matcher *FindNodeWithKind(Matcher *M, Matcher::KindTy Kind) {
 ///       ABC
 ///       XYZ
 ///
-static void FactorNodes(std::unique_ptr<Matcher> &InputMatcherPtr) {
-  // Look for a push node. Iterates instead of recurses to reduce stack usage.
-  ScopeMatcher *Scope = nullptr;
-  std::unique_ptr<Matcher> *RebindableMatcherPtr = &InputMatcherPtr;
-  while (!Scope) {
-    // If we reached the end of the chain, we're done.
-    Matcher *N = RebindableMatcherPtr->get();
-    if (!N)
-      return;
-
-    // If this is not a push node, just scan for one.
-    Scope = dyn_cast<ScopeMatcher>(N);
-    if (!Scope)
-      RebindableMatcherPtr = &(N->getNextPtr());
-  }
-  std::unique_ptr<Matcher> &MatcherPtr = *RebindableMatcherPtr;
+static void FactorScope(std::unique_ptr<Matcher> &MatcherPtr) {
+  ScopeMatcher *Scope = cast<ScopeMatcher>(MatcherPtr.get());
 
   // Okay, pull together the children of the scope node into a vector so we can
   // inspect it more easily.
@@ -320,13 +446,8 @@ static void FactorNodes(std::unique_ptr<Matcher> &InputMatcherPtr) {
 
     // If we removed any equal matchers, we may need to slide the rest of the
     // elements down for the next iteration of the outer loop.
-    if (J != K) {
-      while (J != E)
-        *K++ = *J++;
-
-      // Update end pointer for outer loop.
-      E = K;
-    }
+    if (J != K)
+      E = std::copy(J, E, K);
 
     // If we only found one option starting with this matcher, no factoring is
     // possible. Put the Matcher back in OptionsToMatch.
@@ -353,7 +474,7 @@ static void FactorNodes(std::unique_ptr<Matcher> &InputMatcherPtr) {
       Shared->setNext(new ScopeMatcher(std::move(EqualMatchers)));
 
       // Recursively factor the newly created node.
-      FactorNodes(Shared->getNextPtr());
+      FactorScope(Shared->getNextPtr());
     }
 
     // Put the new Matcher where we started in OptionsToMatch.
@@ -361,8 +482,7 @@ static void FactorNodes(std::unique_ptr<Matcher> &InputMatcherPtr) {
   }
 
   // Trim the array to match the updated end.
-  if (E != OptionsToMatch.end())
-    OptionsToMatch.erase(E, OptionsToMatch.end());
+  OptionsToMatch.erase(E, OptionsToMatch.end());
 
   // If we're down to a single pattern to match, then we don't need this scope
   // anymore.
@@ -382,13 +502,13 @@ static void FactorNodes(std::unique_ptr<Matcher> &InputMatcherPtr) {
   // Check to see if all of the leading entries are now opcode checks.  If so,
   // we can convert this Scope to be a OpcodeSwitch instead.
   bool AllOpcodeChecks = true, AllTypeChecks = true;
-  for (unsigned i = 0, e = OptionsToMatch.size(); i != e; ++i) {
+  for (Matcher *Optn : OptionsToMatch) {
     // Check to see if this breaks a series of CheckOpcodeMatchers.
-    if (AllOpcodeChecks && !isa<CheckOpcodeMatcher>(OptionsToMatch[i])) {
+    if (AllOpcodeChecks && !isa<CheckOpcodeMatcher>(Optn)) {
 #if 0
       if (i > 3) {
         errs() << "FAILING OPC #" << i << "\n";
-        OptionsToMatch[i]->dump();
+        Optn->dump();
       }
 #endif
       AllOpcodeChecks = false;
@@ -397,7 +517,7 @@ static void FactorNodes(std::unique_ptr<Matcher> &InputMatcherPtr) {
     // Check to see if this breaks a series of CheckTypeMatcher's.
     if (AllTypeChecks) {
       CheckTypeMatcher *CTM = cast_or_null<CheckTypeMatcher>(
-          FindNodeWithKind(OptionsToMatch[i], Matcher::CheckType));
+          FindNodeWithKind(Optn, Matcher::CheckType));
       if (!CTM ||
           // iPTR checks could alias any other case without us knowing, don't
           // bother with them.
@@ -406,12 +526,11 @@ static void FactorNodes(std::unique_ptr<Matcher> &InputMatcherPtr) {
           CTM->getResNo() != 0 ||
           // If the CheckType isn't at the start of the list, see if we can move
           // it there.
-          !CTM->canMoveBefore(OptionsToMatch[i])) {
+          !CTM->canMoveBefore(Optn)) {
 #if 0
         if (i > 3 && AllTypeChecks) {
           errs() << "FAILING TYPE #" << i << "\n";
-          OptionsToMatch[i]->dump();
-        }
+          Optn->dump(); }
 #endif
         AllTypeChecks = false;
       }
@@ -422,8 +541,8 @@ static void FactorNodes(std::unique_ptr<Matcher> &InputMatcherPtr) {
   if (AllOpcodeChecks) {
     StringSet<> Opcodes;
     SmallVector<std::pair<const SDNodeInfo *, Matcher *>, 8> Cases;
-    for (unsigned i = 0, e = OptionsToMatch.size(); i != e; ++i) {
-      CheckOpcodeMatcher *COM = cast<CheckOpcodeMatcher>(OptionsToMatch[i]);
+    for (Matcher *Optn : OptionsToMatch) {
+      CheckOpcodeMatcher *COM = cast<CheckOpcodeMatcher>(Optn);
       assert(Opcodes.insert(COM->getOpcode().getEnumName()).second &&
              "Duplicate opcodes not factored?");
       Cases.emplace_back(&COM->getOpcode(), COM->takeNext());
@@ -438,12 +557,12 @@ static void FactorNodes(std::unique_ptr<Matcher> &InputMatcherPtr) {
   if (AllTypeChecks) {
     DenseMap<unsigned, unsigned> TypeEntry;
     SmallVector<std::pair<MVT::SimpleValueType, Matcher *>, 8> Cases;
-    for (unsigned i = 0, e = OptionsToMatch.size(); i != e; ++i) {
-      Matcher *M = FindNodeWithKind(OptionsToMatch[i], Matcher::CheckType);
+    for (Matcher *Optn : OptionsToMatch) {
+      Matcher *M = FindNodeWithKind(Optn, Matcher::CheckType);
       assert(M && isa<CheckTypeMatcher>(M) && "Unknown Matcher type");
 
       auto *CTM = cast<CheckTypeMatcher>(M);
-      Matcher *MatcherWithoutCTM = OptionsToMatch[i]->unlinkNode(CTM);
+      Matcher *MatcherWithoutCTM = Optn->unlinkNode(CTM);
       MVT::SimpleValueType CTMTy = CTM->getType();
       delete CTM;
 
@@ -470,7 +589,7 @@ static void FactorNodes(std::unique_ptr<Matcher> &InputMatcherPtr) {
     for (auto &M : Cases) {
       if (ScopeMatcher *SM = dyn_cast<ScopeMatcher>(M.second)) {
         std::unique_ptr<Matcher> Scope(SM);
-        FactorNodes(Scope);
+        FactorScope(Scope);
         M.second = Scope.release();
         assert(M.second && "null matcher");
       }
@@ -492,6 +611,20 @@ static void FactorNodes(std::unique_ptr<Matcher> &InputMatcherPtr) {
     Scope->resetChild(i, OptionsToMatch[i]);
 }
 
+/// Search a ScopeMatcher to factor with FactorScope.
+static void FactorNodes(std::unique_ptr<Matcher> &InputMatcherPtr) {
+  // Look for a scope matcher. Iterates instead of recurses to reduce stack
+  // usage.
+  std::unique_ptr<Matcher> *MatcherPtr = &InputMatcherPtr;
+  do {
+    if (isa<ScopeMatcher>(*MatcherPtr))
+      return FactorScope(*MatcherPtr);
+
+    // If this is not a scope matcher, go to the next node.
+    MatcherPtr = &(MatcherPtr->get()->getNextPtr());
+  } while (MatcherPtr->get());
+}
+
 void llvm::OptimizeMatcher(std::unique_ptr<Matcher> &MatcherPtr,
                            const CodeGenDAGPatterns &CGP) {
   ContractNodes(MatcherPtr, CGP);
diff --git a/llvm/utils/TableGen/GlobalISelEmitter.cpp b/llvm/utils/TableGen/GlobalISelEmitter.cpp
index 5466d315c05a4..2969dd9156ccb 100644
--- a/llvm/utils/TableGen/GlobalISelEmitter.cpp
+++ b/llvm/utils/TableGen/GlobalISelEmitter.cpp
@@ -2346,6 +2346,20 @@ void GlobalISelEmitter::emitRunCustomAction(raw_ostream &OS) {
      << "}\n";
 }
 
+bool hasBFloatType(const TreePatternNode &Node) {
+  for (unsigned I = 0, E = Node.getNumTypes(); I < E; I++) {
+    auto Ty = Node.getType(I);
+    for (auto T : Ty)
+      if (T.second == MVT::bf16 ||
+          (T.second.isVector() && T.second.getScalarType() == MVT::bf16))
+        return true;
+  }
+  for (const TreePatternNode &C : Node.children())
+    if (hasBFloatType(C))
+      return true;
+  return false;
+}
+
 void GlobalISelEmitter::run(raw_ostream &OS) {
   if (!UseCoverageFile.empty()) {
     RuleCoverage = CodeGenCoverage();
@@ -2382,6 +2396,13 @@ void GlobalISelEmitter::run(raw_ostream &OS) {
 
     if (Pat.getGISelShouldIgnore())
       continue; // skip without warning
+
+    // Skip any patterns containing BF16 types, as GISel cannot currently tell
+    // the difference between fp16 and bf16. FIXME: This can be removed once
+    // BF16 is supported properly.
+    if (hasBFloatType(Pat.getSrcPattern()))
+      continue;
+
     auto MatcherOrErr = runOnPattern(Pat);
 
     // The pattern analysis can fail, indicating an unsupported pattern.
diff --git a/llvm/utils/TableGen/InstrInfoEmitter.cpp b/llvm/utils/TableGen/InstrInfoEmitter.cpp
index 12401a2f246a1..a46a83ea12284 100644
--- a/llvm/utils/TableGen/InstrInfoEmitter.cpp
+++ b/llvm/utils/TableGen/InstrInfoEmitter.cpp
@@ -40,7 +40,7 @@
 
 using namespace llvm;
 
-cl::OptionCategory InstrInfoEmitterCat("Options for -gen-instr-info");
+static cl::OptionCategory InstrInfoEmitterCat("Options for -gen-instr-info");
 static cl::opt<bool> ExpandMIOperandInfo(
     "instr-info-expand-mi-operand-info",
     cl::desc("Expand operand's MIOperandInfo DAG into suboperands"),
@@ -61,19 +61,13 @@ class InstrInfoEmitter {
   void run(raw_ostream &OS);
 
 private:
-  void emitEnums(raw_ostream &OS);
+  void emitEnums(raw_ostream &OS,
+                 ArrayRef<const CodeGenInstruction *> NumberedInstructions);
 
   typedef std::vector<std::string> OperandInfoTy;
   typedef std::vector<OperandInfoTy> OperandInfoListTy;
   typedef std::map<OperandInfoTy, unsigned> OperandInfoMapTy;
 
-  /// The keys of this map are maps which have OpName enum values as their keys
-  /// and instruction operand indices as their values.  The values of this map
-  /// are lists of instruction names.
-  typedef std::map<std::map<unsigned, unsigned>, std::vector<std::string>>
-      OpNameMapTy;
-  typedef std::map<std::string, unsigned>::iterator StrUintMapIter;
-
   /// Generate member functions in the target-specific GenInstrInfo class.
   ///
   /// This method is used to custom expand TIIPredicate definitions.
@@ -95,21 +89,12 @@ class InstrInfoEmitter {
   void emitOperandTypeMappings(
       raw_ostream &OS, const CodeGenTarget &Target,
       ArrayRef<const CodeGenInstruction *> NumberedInstructions);
-  void
-  initOperandMapData(ArrayRef<const CodeGenInstruction *> NumberedInstructions,
-                     StringRef Namespace,
-                     std::map<std::string, unsigned> &Operands,
-                     OpNameMapTy &OperandMap);
   void emitOperandNameMappings(
       raw_ostream &OS, const CodeGenTarget &Target,
       ArrayRef<const CodeGenInstruction *> NumberedInstructions);
-
   void emitLogicalOperandSizeMappings(
       raw_ostream &OS, StringRef Namespace,
       ArrayRef<const CodeGenInstruction *> NumberedInstructions);
-  void emitLogicalOperandTypeMappings(
-      raw_ostream &OS, StringRef Namespace,
-      ArrayRef<const CodeGenInstruction *> NumberedInstructions);
 
   // Operand information.
   unsigned CollectOperandInfo(OperandInfoListTy &OperandInfoList,
@@ -133,10 +118,9 @@ InstrInfoEmitter::GetOperandInfo(const CodeGenInstruction &Inst) {
     // either case into a list of operands for this op.
     std::vector<CGIOperandList::OperandInfo> OperandList;
 
-    // This might be a multiple operand thing.  Targets like X86 have
-    // registers in their multi-operand operands.  It may also be an anonymous
-    // operand, which has a single operand, but no declared class for the
-    // operand.
+    // This might be a multiple operand thing. Targets like X86 have registers
+    // in their multi-operand operands. It may also be an anonymous operand,
+    // which has a single operand, but no declared class for the operand.
     const DagInit *MIOI = Op.MIOperandInfo;
 
     if (!MIOI || MIOI->getNumArgs() == 0) {
@@ -151,8 +135,9 @@ InstrInfoEmitter::GetOperandInfo(const CodeGenInstruction &Inst) {
       }
     }
 
-    for (unsigned j = 0, e = OperandList.size(); j != e; ++j) {
-      const Record *OpR = OperandList[j].Rec;
+    for (const auto &[OpInfo, Constraint] :
+         zip_equal(OperandList, Op.Constraints)) {
+      const Record *OpR = OpInfo.Rec;
       std::string Res;
 
       if (OpR->isSubClassOf("RegisterOperand"))
@@ -195,12 +180,11 @@ InstrInfoEmitter::GetOperandInfo(const CodeGenInstruction &Inst) {
       // Fill in constraint info.
       Res += ", ";
 
-      const CGIOperandList::ConstraintInfo &Constraint = Op.Constraints[j];
-      if (Constraint.isNone())
+      if (Constraint.isNone()) {
         Res += "0";
-      else if (Constraint.isEarlyClobber())
+      } else if (Constraint.isEarlyClobber()) {
         Res += "MCOI_EARLY_CLOBBER";
-      else {
+      } else {
         assert(Constraint.isTied());
         Res += "MCOI_TIED_TO(" + utostr(Constraint.getTiedOperand()) + ")";
       }
@@ -239,35 +223,6 @@ void InstrInfoEmitter::EmitOperandInfo(raw_ostream &OS,
   }
 }
 
-/// Initialize data structures for generating operand name mappings.
-///
-/// \param Operands [out] A map used to generate the OpName enum with operand
-///        names as its keys and operand enum values as its values.
-/// \param OperandMap [out] A map for representing the operand name mappings for
-///        each instructions.  This is used to generate the OperandMap table as
-///        well as the getNamedOperandIdx() function.
-void InstrInfoEmitter::initOperandMapData(
-    ArrayRef<const CodeGenInstruction *> NumberedInstructions,
-    StringRef Namespace, std::map<std::string, unsigned> &Operands,
-    OpNameMapTy &OperandMap) {
-  unsigned NumOperands = 0;
-  for (const CodeGenInstruction *Inst : NumberedInstructions) {
-    if (!Inst->TheDef->getValueAsBit("UseNamedOperandTable"))
-      continue;
-    std::map<unsigned, unsigned> OpList;
-    for (const auto &Info : Inst->Operands) {
-      StrUintMapIter I = Operands.find(Info.Name);
-
-      if (I == Operands.end()) {
-        I = Operands.insert(Operands.begin(), {Info.Name, NumOperands++});
-      }
-      OpList[I->second] = Info.MIOperandNo;
-    }
-    OperandMap[OpList].push_back(Namespace.str() +
-                                 "::" + Inst->TheDef->getName().str());
-  }
-}
-
 /// Generate a table and function for looking up the indices of operands by
 /// name.
 ///
@@ -283,22 +238,57 @@ void InstrInfoEmitter::emitOperandNameMappings(
     raw_ostream &OS, const CodeGenTarget &Target,
     ArrayRef<const CodeGenInstruction *> NumberedInstructions) {
   StringRef Namespace = Target.getInstNamespace();
-  // Map of operand names to their enumeration value.  This will be used to
-  // generate the OpName enum.
-  std::map<std::string, unsigned> Operands;
-  OpNameMapTy OperandMap;
 
-  initOperandMapData(NumberedInstructions, Namespace, Operands, OperandMap);
+  /// To facilitate assigning OpName enum values in the sorted alphabetical
+  /// order, we go through an indirection from OpName -> ID, and Enum -> ID.
+  /// This allows us to build the OpList and assign IDs to OpNames in a single
+  /// scan of the instructions below.
+
+  // Map of operand names to their ID.
+  std::map<StringRef, unsigned> OperandNameToID;
+  // Map from operand name enum value -> ID.
+  std::vector<unsigned> OperandEnumToID;
+
+  /// The keys of this map is a map which have OpName ID values as their keys
+  /// and instruction operand indices as their values. The values of this map
+  /// are lists of instruction names. This map helps to unique entries among
+  /// instructions that have identical OpName -> Operand index mapping.
+  std::map<std::map<unsigned, unsigned>, std::vector<StringRef>> OperandMap;
+
+  // Max operand index seen.
+  unsigned MaxOperandNo = 0;
+
+  // Fixed/Predefined instructions do not have UseNamedOperandTable enabled, so
+  // we can just skip them.
+  const unsigned NumFixedInsts = Target.getNumFixedInstructions();
+  for (const CodeGenInstruction *Inst :
+       NumberedInstructions.drop_front(NumFixedInsts)) {
+    if (!Inst->TheDef->getValueAsBit("UseNamedOperandTable"))
+      continue;
+    std::map<unsigned, unsigned> OpList;
+    for (const auto &Info : Inst->Operands) {
+      unsigned ID =
+          OperandNameToID.try_emplace(Info.Name, OperandNameToID.size())
+              .first->second;
+      OpList[ID] = Info.MIOperandNo;
+      MaxOperandNo = std::max(MaxOperandNo, Info.MIOperandNo);
+    }
+    OperandMap[OpList].push_back(Inst->TheDef->getName());
+  }
+
+  const size_t NumOperandNames = OperandNameToID.size();
+  OperandEnumToID.reserve(NumOperandNames);
+  for (const auto &Op : OperandNameToID)
+    OperandEnumToID.push_back(Op.second);
 
   OS << "#ifdef GET_INSTRINFO_OPERAND_ENUM\n";
   OS << "#undef GET_INSTRINFO_OPERAND_ENUM\n";
   OS << "namespace llvm::" << Namespace << "::OpName {\n";
   OS << "enum {\n";
-  for (const auto &Op : Operands)
-    OS << "  " << Op.first << " = " << Op.second << ",\n";
-
-  OS << "  OPERAND_LAST";
-  OS << "\n};\n";
+  for (const auto &[I, Op] : enumerate(OperandNameToID))
+    OS << "  " << Op.first << " = " << I << ",\n";
+  OS << "  OPERAND_LAST = " << NumOperandNames << ",\n";
+  OS << "};\n";
   OS << "} // end namespace llvm::" << Namespace << "::OpName\n";
   OS << "#endif //GET_INSTRINFO_OPERAND_ENUM\n\n";
 
@@ -307,28 +297,30 @@ void InstrInfoEmitter::emitOperandNameMappings(
   OS << "namespace llvm::" << Namespace << " {\n";
   OS << "LLVM_READONLY\n";
   OS << "int16_t getNamedOperandIdx(uint16_t Opcode, uint16_t NamedIdx) {\n";
-  if (!Operands.empty()) {
-    OS << "  static const int16_t OperandMap [][" << Operands.size()
+  if (NumOperandNames != 0) {
+    assert(MaxOperandNo <= INT16_MAX &&
+           "Too many operands for the operand name -> index table");
+    StringRef Type = MaxOperandNo <= INT8_MAX ? "int8_t" : "int16_t";
+    OS << "  static constexpr " << Type << " OperandMap[][" << NumOperandNames
        << "] = {\n";
     for (const auto &Entry : OperandMap) {
       const std::map<unsigned, unsigned> &OpList = Entry.first;
-      OS << "{";
-
-      // Emit a row of the OperandMap table
-      for (unsigned i = 0, e = Operands.size(); i != e; ++i)
-        OS << (OpList.count(i) == 0 ? -1 : (int)OpList.find(i)->second) << ", ";
 
+      // Emit a row of the OperandMap table.
+      OS << "    {";
+      for (unsigned ID : OperandEnumToID) {
+        auto Iter = OpList.find(ID);
+        OS << (Iter != OpList.end() ? (int)Iter->second : -1) << ", ";
+      }
       OS << "},\n";
     }
-    OS << "};\n";
+    OS << "  };\n";
 
     OS << "  switch(Opcode) {\n";
-    unsigned TableIndex = 0;
-    for (const auto &Entry : OperandMap) {
-      for (const std::string &Name : Entry.second)
-        OS << "  case " << Name << ":\n";
-
-      OS << "    return OperandMap[" << TableIndex++ << "][NamedIdx];\n";
+    for (const auto &[TableIndex, Entry] : enumerate(OperandMap)) {
+      for (StringRef Name : Entry.second)
+        OS << "  case " << Namespace << "::" << Name << ":\n";
+      OS << "    return OperandMap[" << TableIndex << "][NamedIdx];\n";
     }
     OS << "  default: return -1;\n";
     OS << "  }\n";
@@ -337,18 +329,25 @@ void InstrInfoEmitter::emitOperandNameMappings(
     OS << "  return -1;\n";
   }
   OS << "}\n";
-  OS << "} // end namespace llvm::" << Namespace << "\n";
+  OS << "} // end namespace llvm::" << Namespace << '\n';
   OS << "#endif //GET_INSTRINFO_NAMED_OPS\n\n";
 }
 
 /// Generate an enum for all the operand types for this target, under the
 /// llvm::TargetNamespace::OpTypes namespace.
 /// Operand types are all definitions derived of the Operand Target.td class.
+///
 void InstrInfoEmitter::emitOperandTypeMappings(
     raw_ostream &OS, const CodeGenTarget &Target,
     ArrayRef<const CodeGenInstruction *> NumberedInstructions) {
-
   StringRef Namespace = Target.getInstNamespace();
+
+  // These generated functions are used only by the X86 target
+  // (in bolt/lib/Target/X86/X86MCPlusBuilder.cpp). So emit them only
+  // for X86.
+  if (Namespace != "X86")
+    return;
+
   ArrayRef<const Record *> Operands =
       Records.getAllDerivedDefinitions("Operand");
   ArrayRef<const Record *> RegisterOperands =
@@ -385,75 +384,68 @@ void InstrInfoEmitter::emitOperandTypeMappings(
     return NumberedInstructions[I]->TheDef->getName();
   };
   // TODO: Factor out duplicate operand lists to compress the tables.
-  if (!NumberedInstructions.empty()) {
-    std::vector<int> OperandOffsets;
-    std::vector<const Record *> OperandRecords;
-    int CurrentOffset = 0;
-    for (const CodeGenInstruction *Inst : NumberedInstructions) {
-      OperandOffsets.push_back(CurrentOffset);
-      for (const auto &Op : Inst->Operands) {
-        const DagInit *MIOI = Op.MIOperandInfo;
-        if (!ExpandMIOperandInfo || !MIOI || MIOI->getNumArgs() == 0) {
-          // Single, anonymous, operand.
-          OperandRecords.push_back(Op.Rec);
+  std::vector<size_t> OperandOffsets;
+  std::vector<const Record *> OperandRecords;
+  size_t CurrentOffset = 0;
+  for (const CodeGenInstruction *Inst : NumberedInstructions) {
+    OperandOffsets.push_back(CurrentOffset);
+    for (const auto &Op : Inst->Operands) {
+      const DagInit *MIOI = Op.MIOperandInfo;
+      if (!ExpandMIOperandInfo || !MIOI || MIOI->getNumArgs() == 0) {
+        // Single, anonymous, operand.
+        OperandRecords.push_back(Op.Rec);
+        ++CurrentOffset;
+      } else {
+        for (const Init *Arg : MIOI->getArgs()) {
+          OperandRecords.push_back(cast<DefInit>(Arg)->getDef());
           ++CurrentOffset;
-        } else {
-          for (const Init *Arg : MIOI->getArgs()) {
-            OperandRecords.push_back(cast<DefInit>(Arg)->getDef());
-            ++CurrentOffset;
-          }
         }
       }
     }
+  }
 
-    // Emit the table of offsets (indexes) into the operand type table.
-    // Size the unsigned integer offset to save space.
-    assert(OperandRecords.size() <= UINT32_MAX &&
-           "Too many operands for offset table");
-    OS << "  static const " << getMinimalTypeForRange(OperandRecords.size());
-    OS << " Offsets[] = {\n";
-    for (int I = 0, E = OperandOffsets.size(); I != E; ++I) {
-      OS << "    /* " << getInstrName(I) << " */\n";
-      OS << "    " << OperandOffsets[I] << ",\n";
-    }
-    OS << "  };\n";
+  // Emit the table of offsets (indexes) into the operand type table.
+  // Size the unsigned integer offset to save space.
+  assert(OperandRecords.size() <= UINT32_MAX &&
+         "Too many operands for offset table");
+  OS << "  static constexpr " << getMinimalTypeForRange(OperandRecords.size());
+  OS << " Offsets[] = {\n";
+  for (const auto &[Idx, Offset] : enumerate(OperandOffsets))
+    OS << "    " << Offset << ", // " << getInstrName(Idx) << '\n';
+  OS << "  };\n";
 
-    // Add an entry for the end so that we don't need to special case it below.
-    OperandOffsets.push_back(OperandRecords.size());
-
-    // Emit the actual operand types in a flat table.
-    // Size the signed integer operand type to save space.
-    assert(EnumVal <= INT16_MAX &&
-           "Too many operand types for operand types table");
-    OS << "\n  using namespace OpTypes;\n";
-    OS << "  static";
-    OS << ((EnumVal <= INT8_MAX) ? " const int8_t" : " const int16_t");
-    OS << " OpcodeOperandTypes[] = {\n    ";
-    for (int I = 0, E = OperandRecords.size(), CurOffset = 0; I != E; ++I) {
-      // We print each Opcode's operands in its own row.
-      if (I == OperandOffsets[CurOffset]) {
-        OS << "\n    /* " << getInstrName(CurOffset) << " */\n    ";
-        while (OperandOffsets[++CurOffset] == I)
-          OS << "/* " << getInstrName(CurOffset) << " */\n    ";
-      }
-      const Record *OpR = OperandRecords[I];
-      if ((OpR->isSubClassOf("Operand") ||
-           OpR->isSubClassOf("RegisterOperand") ||
-           OpR->isSubClassOf("RegisterClass")) &&
-          !OpR->isAnonymous())
-        OS << OpR->getName();
-      else
-        OS << -1;
-      OS << ", ";
+  // Add an entry for the end so that we don't need to special case it below.
+  OperandOffsets.push_back(OperandRecords.size());
+
+  // Emit the actual operand types in a flat table.
+  // Size the signed integer operand type to save space.
+  assert(EnumVal <= INT16_MAX &&
+         "Too many operand types for operand types table");
+  OS << "\n  using namespace OpTypes;\n";
+  OS << "  static";
+  OS << (EnumVal <= INT8_MAX ? " constexpr int8_t" : " constexpr int16_t");
+  OS << " OpcodeOperandTypes[] = {";
+  size_t CurOffset = 0;
+  for (auto [Idx, OpR] : enumerate(OperandRecords)) {
+    // We print each Opcode's operands in its own row.
+    if (Idx == OperandOffsets[CurOffset]) {
+      OS << "\n    /* " << getInstrName(CurOffset) << " */\n    ";
+      while (OperandOffsets[++CurOffset] == Idx)
+        OS << "/* " << getInstrName(CurOffset) << " */\n    ";
     }
-    OS << "\n  };\n";
-
-    OS << "  return OpcodeOperandTypes[Offsets[Opcode] + OpIdx];\n";
-  } else {
-    OS << "  llvm_unreachable(\"No instructions defined\");\n";
+    if ((OpR->isSubClassOf("Operand") || OpR->isSubClassOf("RegisterOperand") ||
+         OpR->isSubClassOf("RegisterClass")) &&
+        !OpR->isAnonymous())
+      OS << OpR->getName();
+    else
+      OS << -1;
+    OS << ", ";
   }
+  OS << "\n  };\n";
+
+  OS << "  return OpcodeOperandTypes[Offsets[Opcode] + OpIdx];\n";
   OS << "}\n";
-  OS << "} // end namespace llvm::" << Namespace << "\n";
+  OS << "} // end namespace llvm::" << Namespace << '\n';
   OS << "#endif // GET_INSTRINFO_OPERAND_TYPE\n\n";
 
   OS << "#ifdef GET_INSTRINFO_MEM_OPERAND_SIZE\n";
@@ -470,13 +462,13 @@ void InstrInfoEmitter::emitOperandTypeMappings(
       SizeToOperandName[Size].push_back(Op->getName());
   }
   OS << "  default: return 0;\n";
-  for (const auto &KV : SizeToOperandName) {
-    for (const StringRef &OperandName : KV.second)
+  for (const auto &[Size, OperandNames] : SizeToOperandName) {
+    for (const StringRef &OperandName : OperandNames)
       OS << "  case OpTypes::" << OperandName << ":\n";
-    OS << "    return " << KV.first << ";\n\n";
+    OS << "    return " << Size << ";\n\n";
   }
   OS << "  }\n}\n";
-  OS << "} // end namespace llvm::" << Namespace << "\n";
+  OS << "} // end namespace llvm::" << Namespace << '\n';
   OS << "#endif // GET_INSTRINFO_MEM_OPERAND_SIZE\n\n";
 }
 
@@ -484,12 +476,15 @@ void InstrInfoEmitter::emitLogicalOperandSizeMappings(
     raw_ostream &OS, StringRef Namespace,
     ArrayRef<const CodeGenInstruction *> NumberedInstructions) {
   std::map<std::vector<unsigned>, unsigned> LogicalOpSizeMap;
-
   std::map<unsigned, std::vector<std::string>> InstMap;
 
   size_t LogicalOpListSize = 0U;
   std::vector<unsigned> LogicalOpList;
-  for (const auto *Inst : NumberedInstructions) {
+
+  // Fixed/Predefined instructions do not have UseLogicalOperandMappings
+  // enabled, so we can just skip them.
+  const unsigned NumFixedInsts = CDP.getTargetInfo().getNumFixedInstructions();
+  for (const auto *Inst : NumberedInstructions.drop_front(NumFixedInsts)) {
     if (!Inst->TheDef->getValueAsBit("UseLogicalOperandMappings"))
       continue;
 
@@ -532,8 +527,7 @@ void InstrInfoEmitter::emitLogicalOperandSizeMappings(
       for (; i < static_cast<int>(LogicalOpListSize); ++i) {
         OS << "0, ";
       }
-      OS << "}, ";
-      OS << "\n";
+      OS << "}, \n";
     }
     OS << "  };\n";
 
@@ -561,97 +555,10 @@ void InstrInfoEmitter::emitLogicalOperandSizeMappings(
   OS << "  return S;\n";
   OS << "}\n";
 
-  OS << "} // end namespace llvm::" << Namespace << "\n";
+  OS << "} // end namespace llvm::" << Namespace << '\n';
   OS << "#endif // GET_INSTRINFO_LOGICAL_OPERAND_SIZE_MAP\n\n";
 }
 
-void InstrInfoEmitter::emitLogicalOperandTypeMappings(
-    raw_ostream &OS, StringRef Namespace,
-    ArrayRef<const CodeGenInstruction *> NumberedInstructions) {
-  std::map<std::vector<std::string>, unsigned> LogicalOpTypeMap;
-
-  std::map<unsigned, std::vector<std::string>> InstMap;
-
-  size_t OpTypeListSize = 0U;
-  std::vector<std::string> LogicalOpTypeList;
-  for (const auto *Inst : NumberedInstructions) {
-    if (!Inst->TheDef->getValueAsBit("UseLogicalOperandMappings"))
-      continue;
-
-    LogicalOpTypeList.clear();
-    for (const auto &Op : Inst->Operands) {
-      auto *OpR = Op.Rec;
-      if ((OpR->isSubClassOf("Operand") ||
-           OpR->isSubClassOf("RegisterOperand") ||
-           OpR->isSubClassOf("RegisterClass")) &&
-          !OpR->isAnonymous()) {
-        LogicalOpTypeList.push_back(
-            (Namespace + "::OpTypes::" + Op.Rec->getName()).str());
-      } else {
-        LogicalOpTypeList.push_back("-1");
-      }
-    }
-    OpTypeListSize = std::max(LogicalOpTypeList.size(), OpTypeListSize);
-
-    auto I =
-        LogicalOpTypeMap.insert({LogicalOpTypeList, LogicalOpTypeMap.size()})
-            .first;
-    InstMap[I->second].push_back(
-        (Namespace + "::" + Inst->TheDef->getName()).str());
-  }
-
-  OS << "#ifdef GET_INSTRINFO_LOGICAL_OPERAND_TYPE_MAP\n";
-  OS << "#undef GET_INSTRINFO_LOGICAL_OPERAND_TYPE_MAP\n";
-  OS << "namespace llvm::" << Namespace << " {\n";
-  OS << "LLVM_READONLY static int\n";
-  OS << "getLogicalOperandType(uint16_t Opcode, uint16_t LogicalOpIdx) {\n";
-  if (!InstMap.empty()) {
-    std::vector<const std::vector<std::string> *> LogicalOpTypeList(
-        LogicalOpTypeMap.size());
-    for (auto &P : LogicalOpTypeMap) {
-      LogicalOpTypeList[P.second] = &P.first;
-    }
-    OS << "  static const int TypeMap[][" << OpTypeListSize << "] = {\n";
-    for (int r = 0, rs = LogicalOpTypeList.size(); r < rs; ++r) {
-      const auto &Row = *LogicalOpTypeList[r];
-      OS << "   {";
-      int i, s = Row.size();
-      for (i = 0; i < s; ++i) {
-        if (i > 0)
-          OS << ", ";
-        OS << Row[i];
-      }
-      for (; i < static_cast<int>(OpTypeListSize); ++i) {
-        if (i > 0)
-          OS << ", ";
-        OS << "-1";
-      }
-      OS << "}";
-      if (r != rs - 1)
-        OS << ",";
-      OS << "\n";
-    }
-    OS << "  };\n";
-
-    OS << "  switch (Opcode) {\n";
-    OS << "  default: return -1;\n";
-    for (auto &P : InstMap) {
-      auto OpMapIdx = P.first;
-      const auto &Insts = P.second;
-      for (const auto &Inst : Insts) {
-        OS << "  case " << Inst << ":\n";
-      }
-      OS << "    return TypeMap[" << OpMapIdx << "][LogicalOpIdx];\n";
-    }
-    OS << "  }\n";
-  } else {
-    OS << "  return -1;\n";
-  }
-  OS << "}\n";
-  OS << "} // end namespace llvm::" << Namespace << "\n";
-  OS << "#endif // GET_INSTRINFO_LOGICAL_OPERAND_TYPE_MAP\n\n";
-}
-
 void InstrInfoEmitter::emitMCIIHelperMethods(raw_ostream &OS,
                                              StringRef TargetName) {
   ArrayRef<const Record *> TIIPredicates =
@@ -797,7 +704,7 @@ void InstrInfoEmitter::emitFeatureVerifier(raw_ostream &OS,
     }
     if (!NumPredicates)
       OS << "_None";
-    OS << ", // " << Inst->TheDef->getName() << " = " << InstIdx << "\n";
+    OS << ", // " << Inst->TheDef->getName() << " = " << InstIdx << '\n';
     InstIdx++;
   }
   OS << "  };\n\n"
@@ -899,15 +806,21 @@ void InstrInfoEmitter::emitTIIHelperMethods(raw_ostream &OS,
 
 // run - Emit the main instruction description records for the target...
 void InstrInfoEmitter::run(raw_ostream &OS) {
+  TGTimer &Timer = Records.getTimer();
+  Timer.startTimer("Analyze DAG patterns");
+
   emitSourceFileHeader("Target Instruction Enum Values and Descriptors", OS);
-  emitEnums(OS);
 
   const CodeGenTarget &Target = CDP.getTargetInfo();
-  const std::string &TargetName = std::string(Target.getName());
+  ArrayRef<const CodeGenInstruction *> NumberedInstructions =
+      Target.getInstructionsByEnumValue();
+
+  emitEnums(OS, NumberedInstructions);
+
+  StringRef TargetName = Target.getName();
   const Record *InstrInfo = Target.getInstructionSet();
 
   // Collect all of the operand info records.
-  TGTimer &Timer = Records.getTimer();
   Timer.startTimer("Collect operand info");
   OperandInfoListTy OperandInfoList;
   OperandInfoMapTy OperandInfoMap;
@@ -915,21 +828,30 @@ void InstrInfoEmitter::run(raw_ostream &OS) {
       CollectOperandInfo(OperandInfoList, OperandInfoMap);
 
   // Collect all of the instruction's implicit uses and defs.
+  // Also collect which features are enabled by instructions to control
+  // emission of various mappings.
+
+  bool HasUseLogicalOperandMappings = false;
+  bool HasUseNamedOperandTable = false;
+
   Timer.startTimer("Collect uses/defs");
   std::map<std::vector<const Record *>, unsigned> EmittedLists;
   std::vector<std::vector<const Record *>> ImplicitLists;
   unsigned ImplicitListSize = 0;
-  for (const CodeGenInstruction *II : Target.getInstructionsByEnumValue()) {
-    std::vector<const Record *> ImplicitOps = II->ImplicitUses;
-    llvm::append_range(ImplicitOps, II->ImplicitDefs);
+  for (const CodeGenInstruction *Inst : NumberedInstructions) {
+    HasUseLogicalOperandMappings |=
+        Inst->TheDef->getValueAsBit("UseLogicalOperandMappings");
+    HasUseNamedOperandTable |=
+        Inst->TheDef->getValueAsBit("UseNamedOperandTable");
+
+    std::vector<const Record *> ImplicitOps = Inst->ImplicitUses;
+    llvm::append_range(ImplicitOps, Inst->ImplicitDefs);
     if (EmittedLists.insert({ImplicitOps, ImplicitListSize}).second) {
       ImplicitLists.push_back(ImplicitOps);
       ImplicitListSize += ImplicitOps.size();
     }
   }
 
-  ArrayRef<const CodeGenInstruction *> NumberedInstructions =
-      Target.getInstructionsByEnumValue();
   OS << "#if defined(GET_INSTRINFO_MC_DESC) || "
         "defined(GET_INSTRINFO_CTOR_DTOR)\n";
   OS << "namespace llvm {\n\n";
@@ -960,11 +882,11 @@ void InstrInfoEmitter::run(raw_ostream &OS) {
 
   OS << "extern const " << TargetName << "InstrTable " << TargetName
      << "Descs = {\n  {\n";
-  SequenceToOffsetTable<std::string> InstrNames;
+  SequenceToOffsetTable<StringRef> InstrNames;
   unsigned Num = NumberedInstructions.size();
   for (const CodeGenInstruction *Inst : reverse(NumberedInstructions)) {
     // Keep a list of the instruction names.
-    InstrNames.add(std::string(Inst->TheDef->getName()));
+    InstrNames.add(Inst->TheDef->getName());
     // Emit the record into the table.
     emitRecord(*Inst, --Num, InstrInfo, EmittedLists, OperandInfoMap, OS);
   }
@@ -1000,7 +922,7 @@ void InstrInfoEmitter::run(raw_ostream &OS) {
     // Newline every eight entries.
     if (Num % 8 == 0)
       OS << "\n    ";
-    OS << InstrNames.get(std::string(Inst->TheDef->getName())) << "U, ";
+    OS << InstrNames.get(Inst->TheDef->getName()) << "U, ";
     ++Num;
   }
   OS << "\n};\n\n";
@@ -1073,7 +995,7 @@ void InstrInfoEmitter::run(raw_ostream &OS) {
   OS << "#ifdef GET_INSTRINFO_HEADER\n";
   OS << "#undef GET_INSTRINFO_HEADER\n";
 
-  std::string ClassName = TargetName + "GenInstrInfo";
+  Twine ClassName = TargetName + "GenInstrInfo";
   OS << "namespace llvm {\n";
   OS << "struct " << ClassName << " : public TargetInstrInfo {\n"
      << "  explicit " << ClassName
@@ -1088,7 +1010,7 @@ void InstrInfoEmitter::run(raw_ostream &OS) {
   OS << "#ifdef GET_INSTRINFO_HELPER_DECLS\n";
   OS << "#undef GET_INSTRINFO_HELPER_DECLS\n\n";
   emitTIIHelperMethods(OS, TargetName, /* ExpandDefinition = */ false);
-  OS << "\n";
+  OS << '\n';
   OS << "#endif // GET_INSTRINFO_HELPER_DECLS\n\n";
 
   OS << "#ifdef GET_INSTRINFO_HELPERS\n";
@@ -1130,23 +1052,27 @@ void InstrInfoEmitter::run(raw_ostream &OS) {
 
   OS << "#endif // GET_INSTRINFO_CTOR_DTOR\n\n";
 
-  Timer.startTimer("Emit operand name mappings");
-  emitOperandNameMappings(OS, Target, NumberedInstructions);
+  if (HasUseNamedOperandTable) {
+    Timer.startTimer("Emit operand name mappings");
+    emitOperandNameMappings(OS, Target, NumberedInstructions);
+  }
 
   Timer.startTimer("Emit operand type mappings");
   emitOperandTypeMappings(OS, Target, NumberedInstructions);
 
-  Timer.startTimer("Emit logical operand size mappings");
-  emitLogicalOperandSizeMappings(OS, TargetName, NumberedInstructions);
-
-  Timer.startTimer("Emit logical operand type mappings");
-  emitLogicalOperandTypeMappings(OS, TargetName, NumberedInstructions);
+  if (HasUseLogicalOperandMappings) {
+    Timer.startTimer("Emit logical operand size mappings");
+    emitLogicalOperandSizeMappings(OS, TargetName, NumberedInstructions);
+  }
 
   Timer.startTimer("Emit helper methods");
   emitMCIIHelperMethods(OS, TargetName);
 
   Timer.startTimer("Emit verifier methods");
   emitFeatureVerifier(OS, Target);
+
+  Timer.startTimer("Emit map table");
+  EmitMapTable(Records, OS);
 }
 
 void InstrInfoEmitter::emitRecord(
@@ -1283,11 +1209,13 @@ void InstrInfoEmitter::emitRecord(
   OS.write_hex(Value);
   OS << "ULL";
 
-  OS << " },  // Inst #" << Num << " = " << Inst.TheDef->getName() << "\n";
+  OS << " },  // Inst #" << Num << " = " << Inst.TheDef->getName() << '\n';
 }
 
 // emitEnums - Print out enum values for all of the instructions.
-void InstrInfoEmitter::emitEnums(raw_ostream &OS) {
+void InstrInfoEmitter::emitEnums(
+    raw_ostream &OS,
+    ArrayRef<const CodeGenInstruction *> NumberedInstructions) {
   OS << "#ifdef GET_INSTRINFO_ENUM\n";
   OS << "#undef GET_INSTRINFO_ENUM\n";
 
@@ -1300,36 +1228,27 @@ void InstrInfoEmitter::emitEnums(raw_ostream &OS) {
   OS << "namespace llvm::" << Namespace << " {\n";
 
   OS << "  enum {\n";
-  unsigned Num = 0;
-  for (const CodeGenInstruction *Inst : Target.getInstructionsByEnumValue())
+  for (const CodeGenInstruction *Inst : NumberedInstructions)
     OS << "    " << Inst->TheDef->getName()
-       << "\t= " << (Num = Target.getInstrIntValue(Inst->TheDef)) << ",\n";
-  OS << "    INSTRUCTION_LIST_END = " << Num + 1 << "\n";
+       << "\t= " << Target.getInstrIntValue(Inst->TheDef) << ",\n";
+  OS << "    INSTRUCTION_LIST_END = " << NumberedInstructions.size() << '\n';
   OS << "  };\n\n";
-  OS << "} // end namespace llvm::" << Namespace << "\n";
+  OS << "} // end namespace llvm::" << Namespace << '\n';
   OS << "#endif // GET_INSTRINFO_ENUM\n\n";
 
   OS << "#ifdef GET_INSTRINFO_SCHED_ENUM\n";
   OS << "#undef GET_INSTRINFO_SCHED_ENUM\n";
   OS << "namespace llvm::" << Namespace << "::Sched {\n\n";
   OS << "  enum {\n";
-  Num = 0;
-  for (const auto &Class : SchedModels.explicit_classes())
-    OS << "    " << Class.Name << "\t= " << Num++ << ",\n";
-  OS << "    SCHED_LIST_END = " << Num << "\n";
+  auto ExplictClasses = SchedModels.explicit_classes();
+  for (const auto &[Idx, Class] : enumerate(ExplictClasses))
+    OS << "    " << Class.Name << "\t= " << Idx << ",\n";
+  OS << "    SCHED_LIST_END = " << ExplictClasses.size() << '\n';
   OS << "  };\n";
   OS << "} // end namespace llvm::" << Namespace << "::Sched\n";
 
   OS << "#endif // GET_INSTRINFO_SCHED_ENUM\n\n";
 }
 
-static void EmitInstrInfo(const RecordKeeper &Records, raw_ostream &OS) {
-  TGTimer &Timer = Records.getTimer();
-  Timer.startTimer("Analyze DAG patterns");
-  InstrInfoEmitter(Records).run(OS);
-  Timer.startTimer("Emit map table");
-  EmitMapTable(Records, OS);
-}
-
-static TableGen::Emitter::Opt X("gen-instr-info", EmitInstrInfo,
-                                "Generate instruction descriptions");
+static TableGen::Emitter::OptClass<InstrInfoEmitter>
+    X("gen-instr-info", "Generate instruction descriptions");
diff --git a/llvm/utils/TableGen/OptionParserEmitter.cpp b/llvm/utils/TableGen/OptionParserEmitter.cpp
index 35a452890b0ec..d17cad41e6a7e 100644
--- a/llvm/utils/TableGen/OptionParserEmitter.cpp
+++ b/llvm/utils/TableGen/OptionParserEmitter.cpp
@@ -287,10 +287,6 @@ static void emitOptionParser(const RecordKeeper &Records, raw_ostream &OS) {
   array_pod_sort(PrefixesUnion.begin(), PrefixesUnion.end());
 
   llvm::StringToOffsetTable Table;
-  // Make sure the empty string is the zero-th one in the table. This both makes
-  // it easy to check for empty strings (zero offset == empty) and makes
-  // initialization cheaper for empty strings.
-  Table.GetOrAddStringOffset("");
   // We can add all the prefixes via the union.
   for (const auto &Prefix : PrefixesUnion)
     Table.GetOrAddStringOffset(Prefix);
@@ -303,9 +299,7 @@ static void emitOptionParser(const RecordKeeper &Records, raw_ostream &OS) {
   OS << "/////////\n";
   OS << "// String table\n\n";
   OS << "#ifdef OPTTABLE_STR_TABLE_CODE\n";
-  Table.EmitStringLiteralDef(
-      OS, "static constexpr llvm::StringTable OptionStrTable",
-      /*Indent=*/"");
+  Table.EmitStringTableDef(OS, "OptionStrTable", /*Indent=*/"");
   OS << "#endif // OPTTABLE_STR_TABLE_CODE\n\n";
 
   // Dump prefixes.
diff --git a/llvm/utils/TableGen/PseudoLoweringEmitter.cpp b/llvm/utils/TableGen/PseudoLoweringEmitter.cpp
index bcbc6ea20751f..6d39d1496c09e 100644
--- a/llvm/utils/TableGen/PseudoLoweringEmitter.cpp
+++ b/llvm/utils/TableGen/PseudoLoweringEmitter.cpp
@@ -79,8 +79,9 @@ unsigned PseudoLoweringEmitter::addDagOperandMapping(
       // "zero_reg" definition.
       if (DI->getDef()->isSubClassOf("Register") ||
           DI->getDef()->getName() == "zero_reg") {
-        OperandMap[BaseIdx + i].Kind = OpData::Reg;
-        OperandMap[BaseIdx + i].Data.Reg = DI->getDef();
+        auto &Entry = OperandMap[BaseIdx + i];
+        Entry.Kind = OpData::Reg;
+        Entry.Data.Reg = DI->getDef();
         ++OpsAdded;
         continue;
       }
@@ -105,12 +106,14 @@ unsigned PseudoLoweringEmitter::addDagOperandMapping(
         OperandMap[BaseIdx + i + I].Kind = OpData::Operand;
       OpsAdded += Insn.Operands[i].MINumOperands;
     } else if (const IntInit *II = dyn_cast<IntInit>(Dag->getArg(i))) {
-      OperandMap[BaseIdx + i].Kind = OpData::Imm;
-      OperandMap[BaseIdx + i].Data.Imm = II->getValue();
+      auto &Entry = OperandMap[BaseIdx + i];
+      Entry.Kind = OpData::Imm;
+      Entry.Data.Imm = II->getValue();
       ++OpsAdded;
     } else if (const auto *BI = dyn_cast<BitsInit>(Dag->getArg(i))) {
-      OperandMap[BaseIdx + i].Kind = OpData::Imm;
-      OperandMap[BaseIdx + i].Data.Imm = *BI->convertInitializerToInt();
+      auto &Entry = OperandMap[BaseIdx + i];
+      Entry.Kind = OpData::Imm;
+      Entry.Data.Imm = *BI->convertInitializerToInt();
       ++OpsAdded;
     } else if (const DagInit *SubDag = dyn_cast<DagInit>(Dag->getArg(i))) {
       // Just add the operands recursively. This is almost certainly
diff --git a/llvm/utils/TableGen/RegisterInfoEmitter.cpp b/llvm/utils/TableGen/RegisterInfoEmitter.cpp
index 8247b2d8f5a40..01e83f4d9d41d 100644
--- a/llvm/utils/TableGen/RegisterInfoEmitter.cpp
+++ b/llvm/utils/TableGen/RegisterInfoEmitter.cpp
@@ -1608,7 +1608,7 @@ void RegisterInfoEmitter::runTargetDesc(raw_ostream &OS) {
       }
       OS << "  };\n\n"
             "  assert(Reg < ArrayRef(Mapping).size());\n"
-            "  unsigned RCID = Mapping[Reg];\n"
+            "  unsigned RCID = Mapping[Reg.id()];\n"
             "  if (RCID == InvalidRegClassID)\n"
             "    return nullptr;\n"
             "  return RegisterClasses[RCID];\n"
diff --git a/llvm/utils/TableGen/SDNodeInfoEmitter.cpp b/llvm/utils/TableGen/SDNodeInfoEmitter.cpp
index cb971b089f5a4..63ee0deb87110 100644
--- a/llvm/utils/TableGen/SDNodeInfoEmitter.cpp
+++ b/llvm/utils/TableGen/SDNodeInfoEmitter.cpp
@@ -167,9 +167,7 @@ std::vector<unsigned> SDNodeInfoEmitter::emitNodeNames(raw_ostream &OS) const {
     NameOffsets.push_back(NameTable.GetOrAddStringOffset(DebugName));
   }
 
-  NameTable.EmitStringLiteralDef(
-      OS, "static const char " + Target.getName() + "SDNodeNames[]",
-      /*Indent=*/"");
+  NameTable.EmitStringTableDef(OS, Target.getName() + "SDNodeNames");
   OS << '\n';
 
   return NameOffsets;
diff --git a/llvm/utils/TableGen/SubtargetEmitter.cpp b/llvm/utils/TableGen/SubtargetEmitter.cpp
index 3db3ae65cc555..49362ff5ef655 100644
--- a/llvm/utils/TableGen/SubtargetEmitter.cpp
+++ b/llvm/utils/TableGen/SubtargetEmitter.cpp
@@ -2082,6 +2082,7 @@ void SubtargetEmitter::run(raw_ostream &OS) {
   OS << "\n#ifdef GET_SUBTARGETINFO_TARGET_DESC\n";
   OS << "#undef GET_SUBTARGETINFO_TARGET_DESC\n\n";
 
+  OS << "#include \"llvm/ADT/BitmaskEnum.h\"\n";
   OS << "#include \"llvm/Support/Debug.h\"\n";
   OS << "#include \"llvm/Support/raw_ostream.h\"\n\n";
   if (Target == "AArch64")
@@ -2113,7 +2114,26 @@ void SubtargetEmitter::run(raw_ostream &OS) {
      << " unsigned CPUID) const override;\n"
      << "  DFAPacketizer *createDFAPacketizer(const InstrItineraryData *IID)"
      << " const;\n";
-  if (TGT.getHwModes().getNumModeIds() > 1) {
+
+  const CodeGenHwModes &CGH = TGT.getHwModes();
+  if (CGH.getNumModeIds() > 1) {
+    OS << "  enum class " << Target << "HwModeBits : unsigned {\n";
+    for (unsigned M = 0, NumModes = CGH.getNumModeIds(); M != NumModes; ++M) {
+      StringRef ModeName = CGH.getModeName(M, /*IncludeDefault=*/true);
+      OS << "    " << ModeName << " = ";
+      if (M == 0)
+        OS << "0";
+      else
+        OS << "(1 << " << (M - 1) << ")";
+      OS << ",\n";
+      if (M == NumModes - 1) {
+        OS << "\n";
+        OS << "    LLVM_MARK_AS_BITMASK_ENUM(/*LargestValue=*/" << ModeName
+           << "),\n";
+      }
+    }
+    OS << "  };\n";
+
     OS << "  unsigned getHwModeSet() const override;\n";
     OS << "  unsigned getHwMode(enum HwModeType type = HwMode_Default) const "
           "override;\n";
diff --git a/llvm/utils/TableGen/X86InstrMappingEmitter.cpp b/llvm/utils/TableGen/X86InstrMappingEmitter.cpp
index 1ee79aa27fa98..df43f39e0e9be 100644
--- a/llvm/utils/TableGen/X86InstrMappingEmitter.cpp
+++ b/llvm/utils/TableGen/X86InstrMappingEmitter.cpp
@@ -228,8 +228,9 @@ void X86InstrMappingEmitter::emitCompressEVEXTable(
       // For each pre-compression instruction look for a match in the
       // appropriate vector (instructions with the same opcode) using function
       // object IsMatch.
-      auto Match = llvm::find_if(CompressedInsts[Opcode], IsMatch(Inst));
-      if (Match != CompressedInsts[Opcode].end())
+      const auto &Insts = CompressedInsts[Opcode];
+      auto Match = llvm::find_if(Insts, IsMatch(Inst));
+      if (Match != Insts.end())
         NewInst = *Match;
     }
 
diff --git a/llvm/utils/UpdateTestChecks/common.py b/llvm/utils/UpdateTestChecks/common.py
index 1a875c2b523e4..6611dc2fd3ced 100644
--- a/llvm/utils/UpdateTestChecks/common.py
+++ b/llvm/utils/UpdateTestChecks/common.py
@@ -35,6 +35,7 @@
 SUPPORTED_ANALYSES = {
     "Branch Probability Analysis",
     "Cost Model Analysis",
+    "Dependence Analysis",
     "Loop Access Analysis",
     "Scalar Evolution Analysis",
 }
diff --git a/llvm/utils/git/code-format-helper.py b/llvm/utils/git/code-format-helper.py
index 48a338aca9c8e..cb1e56859d083 100755
--- a/llvm/utils/git/code-format-helper.py
+++ b/llvm/utils/git/code-format-helper.py
@@ -353,6 +353,8 @@ def pr_comment_text_for_diff(self, diff: str) -> str:
 
     def format_run(self, changed_files: List[str], args: FormatArgs) -> Optional[str]:
         files = self.filter_changed_files(changed_files)
+        if not files:
+            return None
 
         # Use git to find files that have had a change in the number of undefs
         regex = "([^a-zA-Z0-9#_-]undef[^a-zA-Z0-9_-]|UndefValue::get)"
@@ -379,10 +381,6 @@ def format_run(self, changed_files: List[str], args: FormatArgs) -> Optional[str
         # Each file is prefixed like:
         # diff --git a/file b/file
         for file in re.split("^diff --git ", stdout, 0, re.MULTILINE):
-            # We skip checking in MIR files as undef is a valid token and not
-            # going away.
-            if file.endswith(".mir"):
-                continue
             # search for additions of undef
             if re.search(r"^[+](?!\s*#\s*).*(\bundef\b|UndefValue::get)", file, re.MULTILINE):
                 files.append(re.match("a/([^ ]+)", file.splitlines()[0])[1])
diff --git a/llvm/utils/git/github-automation.py b/llvm/utils/git/github-automation.py
index da467f46b4dd3..6e847f6cea59d 100755
--- a/llvm/utils/git/github-automation.py
+++ b/llvm/utils/git/github-automation.py
@@ -17,6 +17,7 @@
 import re
 import requests
 import sys
+import textwrap
 import time
 from typing import List, Optional
 
@@ -209,6 +210,14 @@ def _get_current_team(self) -> Optional[github.Team.Team]:
         return None
 
 
+def get_top_values(values: dict, top: int = 3) -> list:
+    return [v for v in sorted(values.items(), key=lambda x: x[1], reverse=True)][:top]
+
+
+def get_user_values_str(values: list) -> str:
+    return ", ".join([f"@{v[0]} ({v[1]})" for v in values])
+
+
 class PRGreeter:
     COMMENT_TAG = "<!--LLVM NEW CONTRIBUTOR COMMENT-->\n"
 
@@ -241,6 +250,60 @@ def run(self) -> bool:
         return True
 
 
+class CommitRequestGreeter:
+    def __init__(self, token: str, repo: str, issue_number: int):
+        self.repo = github.Github(token).get_repo(repo)
+        self.issue = self.repo.get_issue(issue_number)
+
+    def run(self) -> bool:
+        # Post greeter comment:
+        comment = textwrap.dedent(
+            f"""
+            @{self.issue.user.login} thank you for applying for commit access.  Please  review the project's [code review policy](https://llvm.org/docs/CodeReview.html).
+        """
+        )
+        self.issue.create_comment(comment)
+
+        # Post activity summary:
+        total_prs = 0
+        merged_prs = 0
+        merged_by = {}
+        reviewed_by = {}
+        for i in self.repo.get_issues(creator=self.issue.user.login, state="all"):
+            issue_reviewed_by = set()
+            try:
+                pr = i.as_pull_request()
+                total_prs += 1
+                for c in pr.get_review_comments():
+                    if c.user.login == self.issue.user.login:
+                        continue
+                    issue_reviewed_by.add(c.user.login)
+                for r in issue_reviewed_by:
+                    if r not in reviewed_by:
+                        reviewed_by[r] = 1
+                    else:
+                        reviewed_by[r] += 1
+                if pr.is_merged():
+                    merged_prs += 1
+                    merger = pr.merged_by.login
+                    if merger not in merged_by:
+                        merged_by[merger] = 1
+                    else:
+                        merged_by[merger] += 1
+                    continue
+
+            except github.GithubException:
+                continue
+
+        comment = f"""
+            ### Activity Summary:
+            * [{total_prs} Pull Requests](https://github.com/llvm/llvm-project/pulls/{self.issue.user.login}) ({merged_prs} merged)
+            * Top 3 Committers: {get_user_values_str(get_top_values(merged_by))}
+            * Top 3 Reviewers: {get_user_values_str(get_top_values(reviewed_by))}
+        """
+        self.issue.create_comment(textwrap.dedent(comment))
+
+
 class PRBuildbotInformation:
     COMMENT_TAG = "<!--LLVM BUILDBOT INFORMATION COMMENT-->\n"
 
@@ -409,6 +472,98 @@ def release_branch_for_issue(self) -> Optional[str]:
             return m.group(1)
         return None
 
+    def update_issue_project_status(self) -> None:
+        """
+        A common workflow is to merge a PR and then use the /cherry-pick
+        command in a pull request comment to backport the change to the
+        release branch.  In this case, once the new PR for the backport has
+        been created, we want to mark the project status for the original PR
+        as Done.  This is becuase we can track progress now on the new PR
+        which is targeting the release branch.
+
+        """
+
+        pr = self.issue.as_pull_request()
+        if not pr:
+            return
+
+        if pr.state != "closed":
+            return
+
+        gh = github.Github(login_or_token=self.token)
+        query = """
+            query($node_id: ID!) {
+              node(id: $node_id) {
+                ... on PullRequest {
+                  url
+                  projectItems(first:100){
+                    nodes {
+                      id
+                      project {
+                        id
+                        number
+                        field(name: "Status") {
+                          ... on ProjectV2SingleSelectField {
+                            id
+                            name
+                            options {
+                              id
+                              name
+                            }
+                          }
+                        }
+                      }
+                    }
+                  }
+                }
+              }
+            }
+        """
+        variables = {"node_id": pr.node_id}
+        res_header, res_data = gh._Github__requester.graphql_query(
+            query=query, variables=variables
+        )
+        print(res_header)
+
+        llvm_release_status_project_number = 3
+        for item in res_data["data"]["node"]["projectItems"]["nodes"]:
+            project = item["project"]
+            if project["number"] != llvm_release_status_project_number:
+                continue
+            status_field = project["field"]
+            for option in status_field["options"]:
+                if option["name"] != "Done":
+                    continue
+                variables = {
+                    "project": project["id"],
+                    "item": item["id"],
+                    "status_field": status_field["id"],
+                    "status_value": option["id"],
+                }
+
+                query = """
+                  mutation($project: ID!, $item: ID!, $status_field: ID!, $status_value: String!) {
+                    set_status:
+                    updateProjectV2ItemFieldValue(input: {
+                      projectId: $project
+                      itemId: $item
+                      fieldId: $status_field
+                      value: {
+                        singleSelectOptionId: $status_value
+                      }
+                    }) {
+                      projectV2Item {
+                        id
+                      }
+                    }
+                  }
+                """
+
+                res_header, res_data = gh._Github__requester.graphql_query(
+                    query=query, variables=variables
+                )
+                print(res_header)
+
     def print_release_branch(self) -> None:
         print(self.release_branch_for_issue)
 
@@ -601,6 +756,7 @@ def create_pull_request(
 
         self.issue_notify_pull_request(pull)
         self.issue_remove_cherry_pick_failed_label()
+        self.update_issue_project_status()
 
         # TODO(tstellar): Do you really want to always return True?
         return True
@@ -676,6 +832,9 @@ def request_release_note(token: str, repo_name: str, pr_number: int):
 pr_greeter_parser = subparsers.add_parser("pr-greeter")
 pr_greeter_parser.add_argument("--issue-number", type=int, required=True)
 
+commit_request_greeter = subparsers.add_parser("commit-request-greeter")
+commit_request_greeter.add_argument("--issue-number", type=int, required=True)
+
 pr_buildbot_information_parser = subparsers.add_parser("pr-buildbot-information")
 pr_buildbot_information_parser.add_argument("--issue-number", type=int, required=True)
 pr_buildbot_information_parser.add_argument("--author", type=str, required=True)
@@ -746,6 +905,9 @@ def request_release_note(token: str, repo_name: str, pr_number: int):
 elif args.command == "pr-greeter":
     pr_greeter = PRGreeter(args.token, args.repo, args.issue_number)
     pr_greeter.run()
+elif args.command == "commit-request-greeter":
+    commit_greeter = CommitRequestGreeter(args.token, args.repo, args.issue_number)
+    commit_greeter.run()
 elif args.command == "pr-buildbot-information":
     pr_buildbot_information = PRBuildbotInformation(
         args.token, args.repo, args.issue_number, args.author
diff --git a/llvm/utils/gn/secondary/BUILD.gn b/llvm/utils/gn/secondary/BUILD.gn
index 5590a5ba195e2..5188bb80d17fa 100644
--- a/llvm/utils/gn/secondary/BUILD.gn
+++ b/llvm/utils/gn/secondary/BUILD.gn
@@ -1,4 +1,3 @@
-import("//clang/lib/ARCMigrate/enable.gni")
 import("//clang/lib/StaticAnalyzer/Frontend/enable.gni")
 import("//llvm/utils/gn/build/toolchain/compiler.gni")
 
@@ -15,8 +14,7 @@ group("default") {
     "//lldb/test",
     "//llvm/test",
   ]
-  # FIXME: Reenable once libc hand-in-hand is figured out.
-  if (false && current_os == "linux") {
+  if (current_os == "linux") {
     deps += [
       "//libcxx",
       "//libcxxabi",
diff --git a/llvm/utils/gn/secondary/clang/include/clang/Basic/BUILD.gn b/llvm/utils/gn/secondary/clang/include/clang/Basic/BUILD.gn
index e7e145c2b273e..941db48a2dc12 100644
--- a/llvm/utils/gn/secondary/clang/include/clang/Basic/BUILD.gn
+++ b/llvm/utils/gn/secondary/clang/include/clang/Basic/BUILD.gn
@@ -12,6 +12,7 @@ write_cmake_config("version") {
     "CLANG_VERSION_MAJOR=$llvm_version_major",
     "CLANG_VERSION_MINOR=$llvm_version_minor",
     "CLANG_VERSION_PATCHLEVEL=$llvm_version_patch",
+    "MAX_CLANG_ABI_COMPAT_VERSION=$llvm_version_major",
   ]
 }
 
@@ -112,6 +113,14 @@ clang_tablegen("BuiltinsBPF") {
   args = [ "-gen-clang-builtins" ]
 }
 
+clang_tablegen("BuiltinsHexagon") {
+  args = [ "-gen-clang-builtins" ]
+}
+
+clang_tablegen("BuiltinsNVPTX") {
+  args = [ "-gen-clang-builtins" ]
+}
+
 clang_tablegen("BuiltinsRISCV") {
   args = [ "-gen-clang-builtins" ]
 }
diff --git a/llvm/utils/gn/secondary/clang/include/clang/Config/BUILD.gn b/llvm/utils/gn/secondary/clang/include/clang/Config/BUILD.gn
index bc0631dc269ac..9678bce7c0459 100644
--- a/llvm/utils/gn/secondary/clang/include/clang/Config/BUILD.gn
+++ b/llvm/utils/gn/secondary/clang/include/clang/Config/BUILD.gn
@@ -1,4 +1,3 @@
-import("//clang/lib/ARCMigrate/enable.gni")
 import("//clang/lib/StaticAnalyzer/Frontend/enable.gni")
 import("//llvm/utils/gn/build/libs/xml/enable.gni")
 import("//llvm/utils/gn/build/write_cmake_config.gni")
@@ -32,12 +31,6 @@ write_cmake_config("Config") {
     "PPC_LINUX_DEFAULT_IEEELONGDOUBLE=",
   ]
 
-  if (clang_enable_arcmt) {
-    values += [ "CLANG_ENABLE_ARCMT=1" ]
-  } else {
-    values += [ "CLANG_ENABLE_ARCMT=" ]
-  }
-
   if (clang_enable_static_analyzer) {
     values += [ "CLANG_ENABLE_STATIC_ANALYZER=1" ]
   } else {
diff --git a/llvm/utils/gn/secondary/clang/lib/ARCMigrate/BUILD.gn b/llvm/utils/gn/secondary/clang/lib/ARCMigrate/BUILD.gn
deleted file mode 100644
index 8a790c0875ac0..0000000000000
--- a/llvm/utils/gn/secondary/clang/lib/ARCMigrate/BUILD.gn
+++ /dev/null
@@ -1,39 +0,0 @@
-static_library("ARCMigrate") {
-  output_name = "clangARCMigrate"
-  configs += [ "//llvm/utils/gn/build:clang_code" ]
-  deps = [
-    "//clang/lib/AST",
-    "//clang/lib/Analysis",
-    "//clang/lib/Basic",
-    "//clang/lib/Edit",
-    "//clang/lib/Frontend",
-    "//clang/lib/Lex",
-    "//clang/lib/Rewrite",
-    "//clang/lib/Sema",
-    "//clang/lib/Serialization",
-    "//llvm/lib/Support",
-    "//llvm/lib/TargetParser",
-  ]
-  sources = [
-    "ARCMT.cpp",
-    "ARCMTActions.cpp",
-    "FileRemapper.cpp",
-    "ObjCMT.cpp",
-    "PlistReporter.cpp",
-    "TransAPIUses.cpp",
-    "TransARCAssign.cpp",
-    "TransAutoreleasePool.cpp",
-    "TransBlockObjCVariable.cpp",
-    "TransEmptyStatementsAndDealloc.cpp",
-    "TransGCAttrs.cpp",
-    "TransGCCalls.cpp",
-    "TransProperties.cpp",
-    "TransProtectedScope.cpp",
-    "TransRetainReleaseDealloc.cpp",
-    "TransUnbridgedCasts.cpp",
-    "TransUnusedInitDelegate.cpp",
-    "TransZeroOutPropsInDealloc.cpp",
-    "TransformActions.cpp",
-    "Transforms.cpp",
-  ]
-}
diff --git a/llvm/utils/gn/secondary/clang/lib/ARCMigrate/enable.gni b/llvm/utils/gn/secondary/clang/lib/ARCMigrate/enable.gni
deleted file mode 100644
index 7a7a42edbfd42..0000000000000
--- a/llvm/utils/gn/secondary/clang/lib/ARCMigrate/enable.gni
+++ /dev/null
@@ -1,4 +0,0 @@
-declare_args() {
-  # Whether to include the arc migrate tool in the clang binary.
-  clang_enable_arcmt = true
-}
diff --git a/llvm/utils/gn/secondary/clang/lib/Basic/BUILD.gn b/llvm/utils/gn/secondary/clang/lib/Basic/BUILD.gn
index bbfac67b2a6e6..fd671bbdf7c65 100644
--- a/llvm/utils/gn/secondary/clang/lib/Basic/BUILD.gn
+++ b/llvm/utils/gn/secondary/clang/lib/Basic/BUILD.gn
@@ -26,6 +26,8 @@ static_library("Basic") {
     "//clang/include/clang/Basic:AttrSubMatchRulesList",
     "//clang/include/clang/Basic:Builtins",
     "//clang/include/clang/Basic:BuiltinsBPF",
+    "//clang/include/clang/Basic:BuiltinsHexagon",
+    "//clang/include/clang/Basic:BuiltinsNVPTX",
     "//clang/include/clang/Basic:BuiltinsRISCV",
     "//clang/include/clang/Basic:BuiltinsSPIRV",
     "//clang/include/clang/Basic:BuiltinsX86",
diff --git a/llvm/utils/gn/secondary/clang/lib/FrontendTool/BUILD.gn b/llvm/utils/gn/secondary/clang/lib/FrontendTool/BUILD.gn
index 15d838a45aed3..9f7140ed391dd 100644
--- a/llvm/utils/gn/secondary/clang/lib/FrontendTool/BUILD.gn
+++ b/llvm/utils/gn/secondary/clang/lib/FrontendTool/BUILD.gn
@@ -1,9 +1,5 @@
-import("//clang/lib/ARCMigrate/enable.gni")
 import("//clang/lib/StaticAnalyzer/Frontend/enable.gni")
 
-assert(clang_enable_static_analyzer || !clang_enable_arcmt,
-       "Cannot disable static analyzer while enabling ARCMT")
-
 static_library("FrontendTool") {
   output_name = "clangFrontendTool"
   configs += [ "//llvm/utils/gn/build:clang_code" ]
@@ -18,9 +14,6 @@ static_library("FrontendTool") {
     "//llvm/lib/Option",
     "//llvm/lib/Support",
   ]
-  if (clang_enable_arcmt) {
-    deps += [ "//clang/lib/ARCMigrate" ]
-  }
   if (clang_enable_static_analyzer) {
     deps += [ "//clang/lib/StaticAnalyzer/Frontend" ]
   }
diff --git a/llvm/utils/gn/secondary/clang/lib/Sema/BUILD.gn b/llvm/utils/gn/secondary/clang/lib/Sema/BUILD.gn
index 306ef0adda708..73d3326ba5fab 100644
--- a/llvm/utils/gn/secondary/clang/lib/Sema/BUILD.gn
+++ b/llvm/utils/gn/secondary/clang/lib/Sema/BUILD.gn
@@ -95,6 +95,7 @@ static_library("Sema") {
     "SemaObjC.cpp",
     "SemaObjCProperty.cpp",
     "SemaOpenACC.cpp",
+    "SemaOpenACCAtomic.cpp",
     "SemaOpenACCClause.cpp",
     "SemaOpenCL.cpp",
     "SemaOpenMP.cpp",
diff --git a/llvm/utils/gn/secondary/clang/lib/StaticAnalyzer/Checkers/BUILD.gn b/llvm/utils/gn/secondary/clang/lib/StaticAnalyzer/Checkers/BUILD.gn
index e1c773630de70..d9c3257536639 100644
--- a/llvm/utils/gn/secondary/clang/lib/StaticAnalyzer/Checkers/BUILD.gn
+++ b/llvm/utils/gn/secondary/clang/lib/StaticAnalyzer/Checkers/BUILD.gn
@@ -17,7 +17,6 @@ static_library("Checkers") {
     "AnalysisOrderChecker.cpp",
     "AnalyzerStatsChecker.cpp",
     "ArrayBoundChecker.cpp",
-    "ArrayBoundCheckerV2.cpp",
     "BasicObjCFoundationChecks.cpp",
     "BitwiseShiftChecker.cpp",
     "BlockInCriticalSectionChecker.cpp",
diff --git a/llvm/utils/gn/secondary/clang/test/BUILD.gn b/llvm/utils/gn/secondary/clang/test/BUILD.gn
index 41edc072a55e7..926407fbea2af 100644
--- a/llvm/utils/gn/secondary/clang/test/BUILD.gn
+++ b/llvm/utils/gn/secondary/clang/test/BUILD.gn
@@ -1,4 +1,3 @@
-import("//clang/lib/ARCMigrate/enable.gni")
 import("//clang/lib/StaticAnalyzer/Frontend/enable.gni")
 import("//llvm/include/llvm/Config/config.gni")
 import("//llvm/lib/Target/targets.gni")
@@ -51,6 +50,7 @@ write_lit_config("lit_site_cfg") {
     "CLANG_DEFAULT_CXX_STDLIB=",  # Empty string means "default value" here.
     "CLANG_DEFAULT_PIE_ON_LINUX=0",
     "CLANG_ENABLE_CIR=0",
+    "CLANG_ENABLE_OBJC_REWRITER=1",  # FIXME: flag?
     "CLANG_PLUGIN_SUPPORT=0",  # FIXME: Analysis/plugins need global -fPIC
     "CURRENT_TOOLS_DIR=" + rebase_path("$root_out_dir/bin", dir),
     "CLANG_VENDOR_UTI=org.llvm.clang",
@@ -67,6 +67,7 @@ write_lit_config("lit_site_cfg") {
     "ENABLE_SHARED=0",
     "LLVM_EXTERNAL_LIT=",
     "LLVM_HOST_TRIPLE=$llvm_current_triple",
+    "LLVM_INCLUDE_SPIRV_TOOLS_TESTS=0",
     "LLVM_LIT_TOOLS_DIR=",  # Intentionally empty, matches cmake build.
     "LLVM_TOOL_LLVM_DRIVER_BUILD=0",  # FIXME: Add actual support for this.
     "LLVM_USE_SANITIZER=",
@@ -76,12 +77,6 @@ write_lit_config("lit_site_cfg") {
     "PPC_LINUX_DEFAULT_IEEELONGDOUBLE=0",
   ]
 
-  if (clang_enable_arcmt) {
-    extra_values += [ "CLANG_ENABLE_ARCMT=1" ]
-  } else {
-    extra_values += [ "CLANG_ENABLE_ARCMT=0" ]
-  }
-
   if (clang_enable_static_analyzer) {
     extra_values += [ "CLANG_ENABLE_STATIC_ANALYZER=1" ]
   } else {
@@ -210,12 +205,6 @@ group("test") {
     "//llvm/utils/not",
     "//llvm/utils/split-file",
   ]
-  if (clang_enable_arcmt) {
-    deps += [
-      "//clang/tools/arcmt-test",
-      "//clang/tools/c-arcmt-test",
-    ]
-  }
   if (clang_enable_static_analyzer) {
     deps += [
       "//clang/tools/clang-check",
diff --git a/llvm/utils/gn/secondary/clang/tools/arcmt-test/BUILD.gn b/llvm/utils/gn/secondary/clang/tools/arcmt-test/BUILD.gn
deleted file mode 100644
index 8d7fece1bc024..0000000000000
--- a/llvm/utils/gn/secondary/clang/tools/arcmt-test/BUILD.gn
+++ /dev/null
@@ -1,11 +0,0 @@
-executable("arcmt-test") {
-  configs += [ "//llvm/utils/gn/build:clang_code" ]
-  deps = [
-    "//clang/lib/ARCMigrate",
-    "//clang/lib/Basic",
-    "//clang/lib/Frontend",
-    "//clang/lib/Lex",
-    "//llvm/lib/Support",
-  ]
-  sources = [ "arcmt-test.cpp" ]
-}
diff --git a/llvm/utils/gn/secondary/clang/tools/c-arcmt-test/BUILD.gn b/llvm/utils/gn/secondary/clang/tools/c-arcmt-test/BUILD.gn
deleted file mode 100644
index a1c8fe10b5904..0000000000000
--- a/llvm/utils/gn/secondary/clang/tools/c-arcmt-test/BUILD.gn
+++ /dev/null
@@ -1,11 +0,0 @@
-executable("c-arcmt-test") {
-  configs += [ "//llvm/utils/gn/build:clang_code" ]
-  deps = [ "//clang/tools/libclang" ]
-  sources = [ "c-arcmt-test.c" ]
-
-  # See comment at top of clang/tools/libclang/BUILD.gn for why this isn't
-  # needed on Linux.
-  if (host_os == "mac") {
-    ldflags = [ "-Wl,-rpath,@loader_path/../lib" ]
-  }
-}
diff --git a/llvm/utils/gn/secondary/clang/tools/libclang/BUILD.gn b/llvm/utils/gn/secondary/clang/tools/libclang/BUILD.gn
index 9ec7dc975721a..8f7beea152ab7 100644
--- a/llvm/utils/gn/secondary/clang/tools/libclang/BUILD.gn
+++ b/llvm/utils/gn/secondary/clang/tools/libclang/BUILD.gn
@@ -1,4 +1,3 @@
-import("//clang/lib/ARCMigrate/enable.gni")
 import("//llvm/utils/gn/build/symbol_exports.gni")
 import("//llvm/version.gni")
 
@@ -48,9 +47,6 @@ shared_library("libclang") {
     ldflags =
         [ "-Wl,--version-script," + rebase_path(inputs[0], root_build_dir) ]
   }
-  if (clang_enable_arcmt) {
-    deps += [ "//clang/lib/ARCMigrate" ]
-  }
 
   defines = []
 
@@ -60,7 +56,6 @@ shared_library("libclang") {
 
   sources = [
     "../../include/clang-c/Index.h",
-    "ARCMigrate.cpp",
     "BuildSystem.cpp",
     "CIndex.cpp",
     "CIndexCXX.cpp",
diff --git a/llvm/utils/gn/secondary/clang/unittests/StaticAnalyzer/BUILD.gn b/llvm/utils/gn/secondary/clang/unittests/StaticAnalyzer/BUILD.gn
index 9ec69b519c30a..2dbb2ce7ab0e8 100644
--- a/llvm/utils/gn/secondary/clang/unittests/StaticAnalyzer/BUILD.gn
+++ b/llvm/utils/gn/secondary/clang/unittests/StaticAnalyzer/BUILD.gn
@@ -23,6 +23,7 @@ unittest("StaticAnalysisTests") {
     "IsCLibraryFunctionTest.cpp",
     "MemRegionDescriptiveNameTest.cpp",
     "NoStateChangeFuncVisitorTest.cpp",
+    "ObjcBug-124477.cpp",
     "ParamRegionTest.cpp",
     "RangeSetTest.cpp",
     "RegisterCustomCheckersTest.cpp",
diff --git a/llvm/utils/gn/secondary/libc/BUILD.gn b/llvm/utils/gn/secondary/libc/BUILD.gn
new file mode 100644
index 0000000000000..9d77afe638da4
--- /dev/null
+++ b/llvm/utils/gn/secondary/libc/BUILD.gn
@@ -0,0 +1,8 @@
+config("common_utils_config") {
+  include_dirs = [ "." ]
+  defines = [ "LIBC_NAMESPACE=__llvm_libc_common_utils" ]
+}
+
+group("common_utils") {
+  public_configs = [ ":common_utils_config" ]
+}
diff --git a/llvm/utils/gn/secondary/libcxx/include/BUILD.gn b/llvm/utils/gn/secondary/libcxx/include/BUILD.gn
index 90303821eb09f..7faeb203870a2 100644
--- a/llvm/utils/gn/secondary/libcxx/include/BUILD.gn
+++ b/llvm/utils/gn/secondary/libcxx/include/BUILD.gn
@@ -342,6 +342,7 @@ if (current_toolchain == default_toolchain) {
       "__chrono/steady_clock.h",
       "__chrono/sys_info.h",
       "__chrono/system_clock.h",
+      "__chrono/tai_clock.h",
       "__chrono/time_point.h",
       "__chrono/time_zone.h",
       "__chrono/time_zone_link.h",
@@ -1325,8 +1326,11 @@ if (current_toolchain == default_toolchain) {
       "__filesystem/space_info.h",
       "__filesystem/u8path.h",
       "__flat_map/flat_map.h",
+      "__flat_map/flat_multimap.h",
       "__flat_map/key_value_iterator.h",
+      "__flat_map/sorted_equivalent.h",
       "__flat_map/sorted_unique.h",
+      "__flat_map/utils.h",
       "__format/buffer.h",
       "__format/concepts.h",
       "__format/container_adaptor.h",
@@ -1516,12 +1520,14 @@ if (current_toolchain == default_toolchain) {
       "__memory/compressed_pair.h",
       "__memory/concepts.h",
       "__memory/construct_at.h",
+      "__memory/destroy.h",
       "__memory/destruct_n.h",
       "__memory/inout_ptr.h",
       "__memory/noexcept_move_assign_container.h",
       "__memory/out_ptr.h",
       "__memory/pointer_traits.h",
       "__memory/ranges_construct_at.h",
+      "__memory/ranges_destroy.h",
       "__memory/ranges_uninitialized_algorithms.h",
       "__memory/raw_storage_iterator.h",
       "__memory/shared_count.h",
diff --git a/llvm/utils/gn/secondary/libcxx/src/BUILD.gn b/llvm/utils/gn/secondary/libcxx/src/BUILD.gn
index 2bbe9e8e0a7d3..2bc851a99afb1 100644
--- a/llvm/utils/gn/secondary/libcxx/src/BUILD.gn
+++ b/llvm/utils/gn/secondary/libcxx/src/BUILD.gn
@@ -247,6 +247,7 @@ if (libcxx_enable_shared) {
     include_dirs = [ "." ]
     deps = [
       "//compiler-rt/lib/builtins",
+      "//libc:common_utils",
       "//libcxx/include",
       "//libcxxabi/src:cxxabi_shared",
       "//libunwind/src:unwind_shared",
@@ -298,6 +299,7 @@ if (libcxx_enable_static) {
     }
     deps = [
       "//compiler-rt/lib/builtins",
+      "//libc:common_utils",
       "//libcxx/include",
       "//libcxxabi/src:cxxabi_static",
       "//libunwind/src:unwind_static",
diff --git a/llvm/utils/gn/secondary/lld/Common/BUILD.gn b/llvm/utils/gn/secondary/lld/Common/BUILD.gn
index c0b1c451f2ad5..b5286545f867d 100644
--- a/llvm/utils/gn/secondary/lld/Common/BUILD.gn
+++ b/llvm/utils/gn/secondary/lld/Common/BUILD.gn
@@ -33,7 +33,6 @@ static_library("Common") {
   ]
   sources = [
     "Args.cpp",
-    "BPSectionOrdererBase.cpp",
     "CommonLinkerContext.cpp",
     "DWARF.cpp",
     "DriverDispatcher.cpp",
diff --git a/llvm/utils/gn/secondary/lld/ELF/BUILD.gn b/llvm/utils/gn/secondary/lld/ELF/BUILD.gn
index d9037256e142a..d23b979eaecad 100644
--- a/llvm/utils/gn/secondary/lld/ELF/BUILD.gn
+++ b/llvm/utils/gn/secondary/lld/ELF/BUILD.gn
@@ -42,6 +42,7 @@ static_library("ELF") {
     "Arch/SystemZ.cpp",
     "Arch/X86.cpp",
     "Arch/X86_64.cpp",
+    "BPSectionOrderer.cpp",
     "CallGraphSort.cpp",
     "DWARF.cpp",
     "Driver.cpp",
diff --git a/llvm/utils/gn/secondary/lldb/source/Core/BUILD.gn b/llvm/utils/gn/secondary/lldb/source/Core/BUILD.gn
index 7b690b5c8782a..f7f52468eff00 100644
--- a/llvm/utils/gn/secondary/lldb/source/Core/BUILD.gn
+++ b/llvm/utils/gn/secondary/lldb/source/Core/BUILD.gn
@@ -78,6 +78,7 @@ static_library("Core") {
     "SourceLocationSpec.cpp",
     "SourceManager.cpp",
     "StreamAsynchronousIO.cpp",
+    "Telemetry.cpp",
     "ThreadedCommunication.cpp",
     "UserSettingsController.cpp",
     "Value.cpp",
diff --git a/llvm/utils/gn/secondary/lldb/source/ValueObject/BUILD.gn b/llvm/utils/gn/secondary/lldb/source/ValueObject/BUILD.gn
index 110f0a6761461..5d36cf0e1e053 100644
--- a/llvm/utils/gn/secondary/lldb/source/ValueObject/BUILD.gn
+++ b/llvm/utils/gn/secondary/lldb/source/ValueObject/BUILD.gn
@@ -24,6 +24,7 @@ static_library("ValueObject") {
   ]
   include_dirs = [ ".." ]
   sources = [
+    "DILLexer.cpp",
     "ValueObject.cpp",
     "ValueObjectCast.cpp",
     "ValueObjectChild.cpp",
@@ -37,7 +38,7 @@ static_library("ValueObject") {
     "ValueObjectRegister.cpp",
     "ValueObjectSyntheticFilter.cpp",
     "ValueObjectUpdater.cpp",
-    "ValueObjectVariable.cpp",
     "ValueObjectVTable.cpp",
+    "ValueObjectVariable.cpp",
   ]
 }
diff --git a/llvm/utils/gn/secondary/llvm/lib/Analysis/BUILD.gn b/llvm/utils/gn/secondary/llvm/lib/Analysis/BUILD.gn
index ba78c2cf9e75f..4939b6eeff4d1 100644
--- a/llvm/utils/gn/secondary/llvm/lib/Analysis/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/lib/Analysis/BUILD.gn
@@ -68,6 +68,7 @@ static_library("Analysis") {
     "InstructionPrecedenceTracking.cpp",
     "InstructionSimplify.cpp",
     "InteractiveModelRunner.cpp",
+    "KernelInfo.cpp",
     "LastRunTrackingAnalysis.cpp",
     "LazyBlockFrequencyInfo.cpp",
     "LazyBranchProbabilityInfo.cpp",
diff --git a/llvm/utils/gn/secondary/llvm/lib/ExecutionEngine/JITLink/BUILD.gn b/llvm/utils/gn/secondary/llvm/lib/ExecutionEngine/JITLink/BUILD.gn
index b9399faa45e2e..96db1ab43e76e 100644
--- a/llvm/utils/gn/secondary/llvm/lib/ExecutionEngine/JITLink/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/lib/ExecutionEngine/JITLink/BUILD.gn
@@ -22,6 +22,7 @@ static_library("JITLink") {
     "COFFDirectiveParser.cpp",
     "COFFLinkGraphBuilder.cpp",
     "COFF_x86_64.cpp",
+    "CompactUnwindSupport.cpp",
     "DWARFRecordSectionSplitter.cpp",
     "EHFrameSupport.cpp",
     "ELF.cpp",
diff --git a/llvm/utils/gn/secondary/llvm/lib/ExecutionEngine/Orc/BUILD.gn b/llvm/utils/gn/secondary/llvm/lib/ExecutionEngine/Orc/BUILD.gn
index a13c3cd22e72c..b0d4d94c6ac80 100644
--- a/llvm/utils/gn/secondary/llvm/lib/ExecutionEngine/Orc/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/lib/ExecutionEngine/Orc/BUILD.gn
@@ -68,5 +68,6 @@ static_library("Orc") {
     "Speculation.cpp",
     "TaskDispatch.cpp",
     "ThreadSafeModule.cpp",
+    "UnwindInfoRegistrationPlugin.cpp",
   ]
 }
diff --git a/llvm/utils/gn/secondary/llvm/lib/ExecutionEngine/Orc/TargetProcess/BUILD.gn b/llvm/utils/gn/secondary/llvm/lib/ExecutionEngine/Orc/TargetProcess/BUILD.gn
index d62f5042c9463..0104684e07d18 100644
--- a/llvm/utils/gn/secondary/llvm/lib/ExecutionEngine/Orc/TargetProcess/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/lib/ExecutionEngine/Orc/TargetProcess/BUILD.gn
@@ -6,6 +6,7 @@ static_library("TargetProcess") {
     "//llvm/lib/TargetParser",
   ]
   sources = [
+    "DefaultHostBootstrapValues.cpp",
     "ExecutorSharedMemoryMapperService.cpp",
     "JITLoaderGDB.cpp",
     "JITLoaderPerf.cpp",
@@ -16,6 +17,7 @@ static_library("TargetProcess") {
     "SimpleExecutorMemoryManager.cpp",
     "SimpleRemoteEPCServer.cpp",
     "TargetExecutionUtils.cpp",
+    "UnwindInfoManager.cpp",
   ]
   if (current_os == "linux") {
     libs = [ "rt" ]
diff --git a/llvm/utils/gn/secondary/llvm/lib/Frontend/Offloading/BUILD.gn b/llvm/utils/gn/secondary/llvm/lib/Frontend/Offloading/BUILD.gn
index 3fa4ffab82cf8..1c839b1f3090f 100644
--- a/llvm/utils/gn/secondary/llvm/lib/Frontend/Offloading/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/lib/Frontend/Offloading/BUILD.gn
@@ -2,6 +2,7 @@ static_library("Offloading") {
   output_name = "LLVMFrontendOffloading"
   deps = [
     "//llvm/lib/IR",
+    "//llvm/lib/ObjectYAML",
     "//llvm/lib/Support",
     "//llvm/lib/Transforms/Utils",
   ]
diff --git a/llvm/utils/gn/secondary/llvm/lib/MC/BUILD.gn b/llvm/utils/gn/secondary/llvm/lib/MC/BUILD.gn
index 274ab154c441b..c76962ae92c1c 100644
--- a/llvm/utils/gn/secondary/llvm/lib/MC/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/lib/MC/BUILD.gn
@@ -14,6 +14,7 @@ static_library("MC") {
   sources = [
     "ConstantPools.cpp",
     "DXContainerPSVInfo.cpp",
+    "DXContainerRootSignature.cpp",
     "ELFObjectWriter.cpp",
     "GOFFObjectWriter.cpp",
     "MCAsmBackend.cpp",
diff --git a/llvm/utils/gn/secondary/llvm/lib/Support/BUILD.gn b/llvm/utils/gn/secondary/llvm/lib/Support/BUILD.gn
index 5146d4141f29b..ba13bb16731af 100644
--- a/llvm/utils/gn/secondary/llvm/lib/Support/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/lib/Support/BUILD.gn
@@ -41,7 +41,7 @@ static_library("Support") {
     "APInt.cpp",
     "APSInt.cpp",
     "ARMAttributeParser.cpp",
-    "ARMBuildAttrs.cpp",
+    "ARMBuildAttributes.cpp",
     "ARMWinEH.cpp",
     "Allocator.cpp",
     "AutoConvert.cpp",
diff --git a/llvm/utils/gn/secondary/llvm/lib/Target/AMDGPU/BUILD.gn b/llvm/utils/gn/secondary/llvm/lib/Target/AMDGPU/BUILD.gn
index a4a17a22c1f2c..b152fc844a493 100644
--- a/llvm/utils/gn/secondary/llvm/lib/Target/AMDGPU/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/lib/Target/AMDGPU/BUILD.gn
@@ -200,6 +200,7 @@ static_library("LLVMAMDGPUCodeGen") {
     "AMDGPUTargetTransformInfo.cpp",
     "AMDGPUUnifyDivergentExitNodes.cpp",
     "AMDGPUUnifyMetadata.cpp",
+    "AMDGPUWaitSGPRHazards.cpp",
     "GCNCreateVOPD.cpp",
     "GCNDPPCombine.cpp",
     "GCNHazardRecognizer.cpp",
diff --git a/llvm/utils/gn/secondary/llvm/lib/Target/BUILD.gn b/llvm/utils/gn/secondary/llvm/lib/Target/BUILD.gn
index 01b004aad97cd..3439d25f16b2f 100644
--- a/llvm/utils/gn/secondary/llvm/lib/Target/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/lib/Target/BUILD.gn
@@ -40,7 +40,6 @@ static_library("Target") {
   ]
   sources = [
     "Target.cpp",
-    "TargetIntrinsicInfo.cpp",
     "TargetLoweringObjectFile.cpp",
     "TargetMachine.cpp",
     "TargetMachineC.cpp",
diff --git a/llvm/utils/gn/secondary/llvm/lib/Transforms/Vectorize/BUILD.gn b/llvm/utils/gn/secondary/llvm/lib/Transforms/Vectorize/BUILD.gn
index f59b6446f0dea..433a7f43bb780 100644
--- a/llvm/utils/gn/secondary/llvm/lib/Transforms/Vectorize/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/lib/Transforms/Vectorize/BUILD.gn
@@ -19,6 +19,7 @@ static_library("Vectorize") {
     "SandboxVectorizer/Legality.cpp",
     "SandboxVectorizer/Passes/BottomUpVec.cpp",
     "SandboxVectorizer/Passes/RegionsFromMetadata.cpp",
+    "SandboxVectorizer/Passes/TransactionAcceptOrRevert.cpp",
     "SandboxVectorizer/SandboxVectorizer.cpp",
     "SandboxVectorizer/SandboxVectorizerPassBuilder.cpp",
     "SandboxVectorizer/Scheduler.cpp",
diff --git a/llvm/utils/gn/secondary/llvm/unittests/Transforms/Vectorize/SandboxVectorizer/BUILD.gn b/llvm/utils/gn/secondary/llvm/unittests/Transforms/Vectorize/SandboxVectorizer/BUILD.gn
index 919512919cacc..dfa126102e0c1 100644
--- a/llvm/utils/gn/secondary/llvm/unittests/Transforms/Vectorize/SandboxVectorizer/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/unittests/Transforms/Vectorize/SandboxVectorizer/BUILD.gn
@@ -14,6 +14,7 @@ unittest("SandboxVectorizerTests") {
     "InstrMapsTest.cpp",
     "IntervalTest.cpp",
     "LegalityTest.cpp",
+    "SandboxVectorizerTest.cpp",
     "SchedulerTest.cpp",
     "VecUtilsTest.cpp",
   ]
diff --git a/llvm/utils/gn/secondary/llvm/version.gni b/llvm/utils/gn/secondary/llvm/version.gni
index e0107cdeae76d..2b1a9076afe4a 100644
--- a/llvm/utils/gn/secondary/llvm/version.gni
+++ b/llvm/utils/gn/secondary/llvm/version.gni
@@ -1,4 +1,4 @@
-llvm_version_major = 20
+llvm_version_major = 21
 llvm_version_minor = 0
 llvm_version_patch = 0
 llvm_version = "$llvm_version_major.$llvm_version_minor.$llvm_version_patch"
diff --git a/llvm/utils/lit/lit/__init__.py b/llvm/utils/lit/lit/__init__.py
index ea56d192fb394..b5aa8edc03dc7 100644
--- a/llvm/utils/lit/lit/__init__.py
+++ b/llvm/utils/lit/lit/__init__.py
@@ -2,7 +2,7 @@
 
 __author__ = "Daniel Dunbar"
 __email__ = "daniel@minormatter.com"
-__versioninfo__ = (20, 0, 0)
+__versioninfo__ = (21, 0, 0)
 __version__ = ".".join(str(v) for v in __versioninfo__) + "dev"
 
 __all__ = []
diff --git a/llvm/utils/lit/tests/xunit-output.py b/llvm/utils/lit/tests/xunit-output.py
index 392cded4653fe..c6cf3dfc24c81 100644
--- a/llvm/utils/lit/tests/xunit-output.py
+++ b/llvm/utils/lit/tests/xunit-output.py
@@ -8,19 +8,19 @@
 # RUN: FileCheck < %t.xunit.xml %s
 
 # CHECK:      <?xml version="1.0" encoding="UTF-8"?>
-# CHECK-NEXT: <testsuites time="{{[0-9.]+}}">
-# CHECK-NEXT: <testsuite name="test-data" tests="5" failures="1" skipped="3" time="{{[0-9.]+}}">
-# CHECK-NEXT: <testcase classname="test-data.test-data" name="bad&amp;name.ini" time="{{[0-1]\.[0-9]+}}">
+# CHECK-NEXT: <testsuites time="{{[0-9]+\.[0-9]+}}">
+# CHECK-NEXT: <testsuite name="test-data" tests="5" failures="1" skipped="3" time="{{[0-9]+\.[0-9]+}}">
+# CHECK-NEXT: <testcase classname="test-data.test-data" name="bad&amp;name.ini" time="{{[0-9]+\.[0-9]+}}">
 # CHECK-NEXT:   <failure><![CDATA[& < > ]]]]><![CDATA[> &"]]></failure>
 # CHECK-NEXT: </testcase>
-# CHECK-NEXT: <testcase classname="test-data.test-data" name="excluded.ini" time="{{[0-1]\.[0-9]+}}">
+# CHECK-NEXT: <testcase classname="test-data.test-data" name="excluded.ini" time="{{[0-9]+\.[0-9]+}}">
 # CHECK-NEXT:   <skipped message="Test not selected (--filter, --max-tests)"/>
 # CHECK-NEXT: </testcase>
-# CHECK-NEXT: <testcase classname="test-data.test-data" name="missing_feature.ini" time="{{[0-1]\.[0-9]+}}">
+# CHECK-NEXT: <testcase classname="test-data.test-data" name="missing_feature.ini" time="{{[0-9]+\.[0-9]+}}">
 # CHECK-NEXT:   <skipped message="Missing required feature(s): dummy_feature"/>
 # CHECK-NEXT: </testcase>
-# CHECK-NEXT: <testcase classname="test-data.test-data" name="pass.ini" time="{{[0-1]\.[0-9]+}}"/>
-# CHECK-NEXT: <testcase classname="test-data.test-data" name="unsupported.ini" time="{{[0-1]\.[0-9]+}}">
+# CHECK-NEXT: <testcase classname="test-data.test-data" name="pass.ini" time="{{[0-9]+\.[0-9]+}}"/>
+# CHECK-NEXT: <testcase classname="test-data.test-data" name="unsupported.ini" time="{{[0-9]+\.[0-9]+}}">
 # CHECK-NEXT:   <skipped message="Unsupported configuration"/>
 # CHECK-NEXT: </testcase>
 # CHECK-NEXT: </testsuite>
diff --git a/llvm/utils/update_mir_test_checks.py b/llvm/utils/update_mir_test_checks.py
index 86adf77714a11..2ee156dd7faf7 100755
--- a/llvm/utils/update_mir_test_checks.py
+++ b/llvm/utils/update_mir_test_checks.py
@@ -36,7 +36,7 @@
 VREG_RE = re.compile(r"(%[0-9]+)(?:\.[a-z0-9_]+)?(?::[a-z0-9_]+)?(?:\([<>a-z0-9 ]+\))?")
 MI_FLAGS_STR = (
     r"(frame-setup |frame-destroy |nnan |ninf |nsz |arcp |contract |afn "
-    r"|reassoc |nuw |nsw |exact |nofpexcept |nomerge )*"
+    r"|reassoc |nuw |nsw |exact |nofpexcept |nomerge |disjoint )*"
 )
 VREG_DEF_FLAGS_STR = r"(?:dead |undef )*"
 VREG_DEF_RE = re.compile(
diff --git a/mlir/cmake/modules/AddMLIRPython.cmake b/mlir/cmake/modules/AddMLIRPython.cmake
index a23de004eb014..c2616ea57acef 100644
--- a/mlir/cmake/modules/AddMLIRPython.cmake
+++ b/mlir/cmake/modules/AddMLIRPython.cmake
@@ -709,7 +709,7 @@ function(add_mlir_python_extension libname extname)
       # NanobindAdaptors.h uses PyClassMethod_New to build `pure_subclass`es but nanobind
       # doesn't declare this API as undefined in its linker flags. So we need to declare it as such
       # for downstream users that do not do something like `-undefined dynamic_lookup`.
-      set(CMAKE_MODULE_LINKER_FLAGS "${CMAKE_MODULE_LINKER_FLAGS} -Wl,-U -Wl,_PyClassMethod_New")
+      target_link_options(${libname} PUBLIC "LINKER:-U,_PyClassMethod_New")
     endif()
   endif()
 
diff --git a/mlir/cmake/modules/MLIRConfig.cmake.in b/mlir/cmake/modules/MLIRConfig.cmake.in
index 7076d94a32f2b..c695b5787af66 100644
--- a/mlir/cmake/modules/MLIRConfig.cmake.in
+++ b/mlir/cmake/modules/MLIRConfig.cmake.in
@@ -16,10 +16,6 @@ set(MLIR_INSTALL_AGGREGATE_OBJECTS "@MLIR_INSTALL_AGGREGATE_OBJECTS@")
 set(MLIR_ENABLE_BINDINGS_PYTHON "@MLIR_ENABLE_BINDINGS_PYTHON@")
 set(MLIR_ENABLE_EXECUTION_ENGINE "@MLIR_ENABLE_EXECUTION_ENGINE@")
 
-# For mlir_tablegen()
-set(MLIR_INCLUDE_DIR "@MLIR_INCLUDE_DIR@")
-set(MLIR_MAIN_SRC_DIR "@MLIR_MAIN_SRC_DIR@")
-
 set_property(GLOBAL PROPERTY MLIR_ALL_LIBS "@MLIR_ALL_LIBS@")
 set_property(GLOBAL PROPERTY MLIR_DIALECT_LIBS "@MLIR_DIALECT_LIBS@")
 set_property(GLOBAL PROPERTY MLIR_CONVERSION_LIBS "@MLIR_CONVERSION_LIBS@")
diff --git a/mlir/docs/Canonicalization.md b/mlir/docs/Canonicalization.md
index 03fd174229afe..6e59a4128093a 100644
--- a/mlir/docs/Canonicalization.md
+++ b/mlir/docs/Canonicalization.md
@@ -71,7 +71,7 @@ For example, a pattern that transform
       outs(%init1 : tensor<2x1x3xf32>)
       dimensions = [1, 0, 2]
   %out = linalg.transpose
-      ins(%tranpose: tensor<2x1x3xf32>)
+      ins(%transpose: tensor<2x1x3xf32>)
       outs(%init2 : tensor<3x1x2xf32>)
       permutation = [2, 1, 0]
 ```
diff --git a/mlir/docs/Interfaces.md b/mlir/docs/Interfaces.md
index 51747db546bb7..b7e9e64d23d77 100644
--- a/mlir/docs/Interfaces.md
+++ b/mlir/docs/Interfaces.md
@@ -753,10 +753,15 @@ interface section goes as follows:
     -   (`C++ class` -- `ODS class`(if applicable))
 
 ##### CallInterfaces
-
 *   `CallOpInterface` - Used to represent operations like 'call'
     -   `CallInterfaceCallable getCallableForCallee()`
     -   `void setCalleeFromCallable(CallInterfaceCallable)`
+    -   `ArrayAttr getArgAttrsAttr()`
+    -   `ArrayAttr getResAttrsAttr()`
+    -   `void setArgAttrsAttr(ArrayAttr)`
+    -   `void setResAttrsAttr(ArrayAttr)`
+    -   `Attribute removeArgAttrsAttr()`
+    -   `Attribute removeResAttrsAttr()`
 *   `CallableOpInterface` - Used to represent the target callee of call.
     -   `Region * getCallableRegion()`
     -   `ArrayRef<Type> getArgumentTypes()`
diff --git a/mlir/docs/Tutorials/Toy/Ch-4.md b/mlir/docs/Tutorials/Toy/Ch-4.md
index b753ee7a5332f..39e9ecdd92221 100644
--- a/mlir/docs/Tutorials/Toy/Ch-4.md
+++ b/mlir/docs/Tutorials/Toy/Ch-4.md
@@ -91,7 +91,7 @@ struct ToyInlinerInterface : public DialectInlinerInterface {
   /// previously returned by the call operation with the operands of the
   /// return.
   void handleTerminator(Operation *op,
-                        MutableArrayRef<Value> valuesToRepl) const final {
+                        ValueRange valuesToRepl) const final {
     // Only "toy.return" needs to be handled here.
     auto returnOp = cast<ReturnOp>(op);
 
@@ -147,7 +147,7 @@ and add it to the traits list of `GenericCallOp`:
 
 ```tablegen
 def FuncOp : Toy_Op<"func",
-    [DeclareOpInterfaceMethods<CallableOpInterface>]> {
+    [FunctionOpInterface, IsolatedFromAbove]> {
   ...
 }
 
@@ -159,7 +159,8 @@ def GenericCallOp : Toy_Op<"generic_call",
 
 In the above we also use the `DeclareOpInterfaceMethods` directive to
 auto-declare all of the interface methods in the class declaration of
-GenericCallOp. This means that we just need to provide a definition:
+GenericCallOp. We have already provided the definition in the `extraClassDeclaration`
+field of the `FuncOp` class:
 
 ```c++
 /// Returns the region on the function operation that is callable.
@@ -170,7 +171,7 @@ Region *FuncOp::getCallableRegion() { return &getBody(); }
 /// Return the callee of the generic call operation, this is required by the
 /// call interface.
 CallInterfaceCallable GenericCallOp::getCallableForCallee() {
-  return getAttrOfType<SymbolRefAttr>("callee");
+  return (*this)->getAttrOfType<SymbolRefAttr>("callee");
 }
 
 /// Set the callee for the generic call operation, this is required by the call
@@ -181,7 +182,13 @@ void GenericCallOp::setCalleeFromCallable(CallInterfaceCallable callee) {
 
 /// Get the argument operands to the called function, this is required by the
 /// call interface.
-Operation::operand_range GenericCallOp::getArgOperands() { return inputs(); }
+Operation::operand_range GenericCallOp::getArgOperands() { return getInputs(); }
+
+/// Get the argument operands to the called function as a mutable range, this is
+/// required by the call interface.
+MutableOperandRange GenericCallOp::getArgOperandsMutable() {
+  return getInputsMutable();
+}
 ```
 
 Now that the inliner has been informed about the Toy dialect, we can add the
@@ -255,8 +262,8 @@ bool CastOp::areCastCompatible(TypeRange inputs, TypeRange outputs) {
   if (inputs.size() != 1 || outputs.size() != 1)
     return false;
   // The inputs must be Tensors with the same element type.
-  TensorType input = inputs.front().dyn_cast<TensorType>();
-  TensorType output = outputs.front().dyn_cast<TensorType>();
+  TensorType input = llvm::dyn_cast<TensorType>(inputs.front());
+  TensorType output = llvm::dyn_cast<TensorType>(outputs.front());
   if (!input || !output || input.getElementType() != output.getElementType())
     return false;
   // The shape is required to match if both types are ranked.
diff --git a/mlir/examples/toy/Ch4/include/toy/Ops.td b/mlir/examples/toy/Ch4/include/toy/Ops.td
index 075fd1a9cd473..4441e48ca53c0 100644
--- a/mlir/examples/toy/Ch4/include/toy/Ops.td
+++ b/mlir/examples/toy/Ch4/include/toy/Ops.td
@@ -215,7 +215,12 @@ def GenericCallOp : Toy_Op<"generic_call",
 
   // The generic call operation takes a symbol reference attribute as the
   // callee, and inputs for the call.
-  let arguments = (ins FlatSymbolRefAttr:$callee, Variadic<F64Tensor>:$inputs);
+  let arguments = (ins
+    FlatSymbolRefAttr:$callee,
+    Variadic<F64Tensor>:$inputs,
+    OptionalAttr<DictArrayAttr>:$arg_attrs,
+    OptionalAttr<DictArrayAttr>:$res_attrs
+  );
 
   // The generic call operation returns a single value of TensorType.
   let results = (outs F64Tensor);
diff --git a/mlir/examples/toy/Ch4/mlir/Dialect.cpp b/mlir/examples/toy/Ch4/mlir/Dialect.cpp
index 6c6cdd934cea8..076a75a26619b 100644
--- a/mlir/examples/toy/Ch4/mlir/Dialect.cpp
+++ b/mlir/examples/toy/Ch4/mlir/Dialect.cpp
@@ -333,7 +333,7 @@ CallInterfaceCallable GenericCallOp::getCallableForCallee() {
 /// Set the callee for the generic call operation, this is required by the call
 /// interface.
 void GenericCallOp::setCalleeFromCallable(CallInterfaceCallable callee) {
-  (*this)->setAttr("callee", callee.get<SymbolRefAttr>());
+  (*this)->setAttr("callee", cast<SymbolRefAttr>(callee));
 }
 
 /// Get the argument operands to the called function, this is required by the
diff --git a/mlir/examples/toy/Ch4/mlir/ShapeInferencePass.cpp b/mlir/examples/toy/Ch4/mlir/ShapeInferencePass.cpp
index a9e995ed91bff..2522abe1a46f9 100644
--- a/mlir/examples/toy/Ch4/mlir/ShapeInferencePass.cpp
+++ b/mlir/examples/toy/Ch4/mlir/ShapeInferencePass.cpp
@@ -55,6 +55,7 @@ namespace {
 struct ShapeInferencePass
     : public mlir::PassWrapper<ShapeInferencePass, OperationPass<toy::FuncOp>> {
   MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(ShapeInferencePass)
+  StringRef getArgument() const override { return "toy-shape-inference"; }
 
   void runOnOperation() override {
     auto f = getOperation();
diff --git a/mlir/examples/toy/Ch5/include/toy/Ops.td b/mlir/examples/toy/Ch5/include/toy/Ops.td
index ec6762ff406e8..5b7c966de6f08 100644
--- a/mlir/examples/toy/Ch5/include/toy/Ops.td
+++ b/mlir/examples/toy/Ch5/include/toy/Ops.td
@@ -214,7 +214,12 @@ def GenericCallOp : Toy_Op<"generic_call",
 
   // The generic call operation takes a symbol reference attribute as the
   // callee, and inputs for the call.
-  let arguments = (ins FlatSymbolRefAttr:$callee, Variadic<F64Tensor>:$inputs);
+  let arguments = (ins
+    FlatSymbolRefAttr:$callee,
+    Variadic<F64Tensor>:$inputs,
+    OptionalAttr<DictArrayAttr>:$arg_attrs,
+    OptionalAttr<DictArrayAttr>:$res_attrs
+  );
 
   // The generic call operation returns a single value of TensorType.
   let results = (outs F64Tensor);
diff --git a/mlir/examples/toy/Ch5/mlir/Dialect.cpp b/mlir/examples/toy/Ch5/mlir/Dialect.cpp
index 72072f9188bf3..fb7c742a01802 100644
--- a/mlir/examples/toy/Ch5/mlir/Dialect.cpp
+++ b/mlir/examples/toy/Ch5/mlir/Dialect.cpp
@@ -333,7 +333,7 @@ CallInterfaceCallable GenericCallOp::getCallableForCallee() {
 /// Set the callee for the generic call operation, this is required by the call
 /// interface.
 void GenericCallOp::setCalleeFromCallable(CallInterfaceCallable callee) {
-  (*this)->setAttr("callee", callee.get<SymbolRefAttr>());
+  (*this)->setAttr("callee", cast<SymbolRefAttr>(callee));
 }
 
 /// Get the argument operands to the called function, this is required by the
diff --git a/mlir/examples/toy/Ch5/mlir/LowerToAffineLoops.cpp b/mlir/examples/toy/Ch5/mlir/LowerToAffineLoops.cpp
index 741321432b452..bf2bc43301a33 100644
--- a/mlir/examples/toy/Ch5/mlir/LowerToAffineLoops.cpp
+++ b/mlir/examples/toy/Ch5/mlir/LowerToAffineLoops.cpp
@@ -328,6 +328,7 @@ namespace {
 struct ToyToAffineLoweringPass
     : public PassWrapper<ToyToAffineLoweringPass, OperationPass<ModuleOp>> {
   MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(ToyToAffineLoweringPass)
+  StringRef getArgument() const override { return "toy-to-affine"; }
 
   void getDependentDialects(DialectRegistry &registry) const override {
     registry.insert<affine::AffineDialect, func::FuncDialect,
diff --git a/mlir/examples/toy/Ch5/mlir/ShapeInferencePass.cpp b/mlir/examples/toy/Ch5/mlir/ShapeInferencePass.cpp
index a9e995ed91bff..2522abe1a46f9 100644
--- a/mlir/examples/toy/Ch5/mlir/ShapeInferencePass.cpp
+++ b/mlir/examples/toy/Ch5/mlir/ShapeInferencePass.cpp
@@ -55,6 +55,7 @@ namespace {
 struct ShapeInferencePass
     : public mlir::PassWrapper<ShapeInferencePass, OperationPass<toy::FuncOp>> {
   MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(ShapeInferencePass)
+  StringRef getArgument() const override { return "toy-shape-inference"; }
 
   void runOnOperation() override {
     auto f = getOperation();
diff --git a/mlir/examples/toy/Ch6/include/toy/Ops.td b/mlir/examples/toy/Ch6/include/toy/Ops.td
index a52bebc8b67b8..fdbc239a171df 100644
--- a/mlir/examples/toy/Ch6/include/toy/Ops.td
+++ b/mlir/examples/toy/Ch6/include/toy/Ops.td
@@ -214,7 +214,12 @@ def GenericCallOp : Toy_Op<"generic_call",
 
   // The generic call operation takes a symbol reference attribute as the
   // callee, and inputs for the call.
-  let arguments = (ins FlatSymbolRefAttr:$callee, Variadic<F64Tensor>:$inputs);
+  let arguments = (ins
+    FlatSymbolRefAttr:$callee,
+    Variadic<F64Tensor>:$inputs,
+    OptionalAttr<DictArrayAttr>:$arg_attrs,
+    OptionalAttr<DictArrayAttr>:$res_attrs
+  );
 
   // The generic call operation returns a single value of TensorType.
   let results = (outs F64Tensor);
diff --git a/mlir/examples/toy/Ch6/mlir/Dialect.cpp b/mlir/examples/toy/Ch6/mlir/Dialect.cpp
index 72072f9188bf3..fb7c742a01802 100644
--- a/mlir/examples/toy/Ch6/mlir/Dialect.cpp
+++ b/mlir/examples/toy/Ch6/mlir/Dialect.cpp
@@ -333,7 +333,7 @@ CallInterfaceCallable GenericCallOp::getCallableForCallee() {
 /// Set the callee for the generic call operation, this is required by the call
 /// interface.
 void GenericCallOp::setCalleeFromCallable(CallInterfaceCallable callee) {
-  (*this)->setAttr("callee", callee.get<SymbolRefAttr>());
+  (*this)->setAttr("callee", cast<SymbolRefAttr>(callee));
 }
 
 /// Get the argument operands to the called function, this is required by the
diff --git a/mlir/examples/toy/Ch6/mlir/LowerToAffineLoops.cpp b/mlir/examples/toy/Ch6/mlir/LowerToAffineLoops.cpp
index 741321432b452..bf2bc43301a33 100644
--- a/mlir/examples/toy/Ch6/mlir/LowerToAffineLoops.cpp
+++ b/mlir/examples/toy/Ch6/mlir/LowerToAffineLoops.cpp
@@ -328,6 +328,7 @@ namespace {
 struct ToyToAffineLoweringPass
     : public PassWrapper<ToyToAffineLoweringPass, OperationPass<ModuleOp>> {
   MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(ToyToAffineLoweringPass)
+  StringRef getArgument() const override { return "toy-to-affine"; }
 
   void getDependentDialects(DialectRegistry &registry) const override {
     registry.insert<affine::AffineDialect, func::FuncDialect,
diff --git a/mlir/examples/toy/Ch6/mlir/LowerToLLVM.cpp b/mlir/examples/toy/Ch6/mlir/LowerToLLVM.cpp
index 3ad70e7279692..de1eafc4d8678 100644
--- a/mlir/examples/toy/Ch6/mlir/LowerToLLVM.cpp
+++ b/mlir/examples/toy/Ch6/mlir/LowerToLLVM.cpp
@@ -184,6 +184,7 @@ namespace {
 struct ToyToLLVMLoweringPass
     : public PassWrapper<ToyToLLVMLoweringPass, OperationPass<ModuleOp>> {
   MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(ToyToLLVMLoweringPass)
+  StringRef getArgument() const override { return "toy-to-llvm"; }
 
   void getDependentDialects(DialectRegistry &registry) const override {
     registry.insert<LLVM::LLVMDialect, scf::SCFDialect>();
diff --git a/mlir/examples/toy/Ch6/mlir/ShapeInferencePass.cpp b/mlir/examples/toy/Ch6/mlir/ShapeInferencePass.cpp
index a9e995ed91bff..2522abe1a46f9 100644
--- a/mlir/examples/toy/Ch6/mlir/ShapeInferencePass.cpp
+++ b/mlir/examples/toy/Ch6/mlir/ShapeInferencePass.cpp
@@ -55,6 +55,7 @@ namespace {
 struct ShapeInferencePass
     : public mlir::PassWrapper<ShapeInferencePass, OperationPass<toy::FuncOp>> {
   MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(ShapeInferencePass)
+  StringRef getArgument() const override { return "toy-shape-inference"; }
 
   void runOnOperation() override {
     auto f = getOperation();
diff --git a/mlir/examples/toy/Ch7/include/toy/Ops.td b/mlir/examples/toy/Ch7/include/toy/Ops.td
index cfd6859eb27bf..71ab7b0aeebb9 100644
--- a/mlir/examples/toy/Ch7/include/toy/Ops.td
+++ b/mlir/examples/toy/Ch7/include/toy/Ops.td
@@ -237,7 +237,12 @@ def GenericCallOp : Toy_Op<"generic_call",
 
   // The generic call operation takes a symbol reference attribute as the
   // callee, and inputs for the call.
-  let arguments = (ins FlatSymbolRefAttr:$callee, Variadic<Toy_Type>:$inputs);
+  let arguments = (ins
+    FlatSymbolRefAttr:$callee,
+    Variadic<Toy_Type>:$inputs,
+    OptionalAttr<DictArrayAttr>:$arg_attrs,
+    OptionalAttr<DictArrayAttr>:$res_attrs
+  );
 
   // The generic call operation returns a single value of TensorType or
   // StructType.
@@ -250,7 +255,8 @@ def GenericCallOp : Toy_Op<"generic_call",
 
   // Add custom build methods for the generic call operation.
   let builders = [
-    OpBuilder<(ins "StringRef":$callee, "ArrayRef<Value>":$arguments)>
+    OpBuilder<(ins "Type":$result_type, "StringRef":$callee,
+              "ArrayRef<Value>":$arguments)>
   ];
 }
 
diff --git a/mlir/examples/toy/Ch7/mlir/Dialect.cpp b/mlir/examples/toy/Ch7/mlir/Dialect.cpp
index 7e030ffc5488c..52881db87d86b 100644
--- a/mlir/examples/toy/Ch7/mlir/Dialect.cpp
+++ b/mlir/examples/toy/Ch7/mlir/Dialect.cpp
@@ -350,9 +350,9 @@ void FuncOp::print(mlir::OpAsmPrinter &p) {
 //===----------------------------------------------------------------------===//
 
 void GenericCallOp::build(mlir::OpBuilder &builder, mlir::OperationState &state,
-                          StringRef callee, ArrayRef<mlir::Value> arguments) {
-  // Generic call always returns an unranked Tensor initially.
-  state.addTypes(UnrankedTensorType::get(builder.getF64Type()));
+                          mlir::Type resultType, StringRef callee,
+                          ArrayRef<mlir::Value> arguments) {
+  state.addTypes(resultType);
   state.addOperands(arguments);
   state.addAttribute("callee",
                      mlir::SymbolRefAttr::get(builder.getContext(), callee));
@@ -367,7 +367,7 @@ CallInterfaceCallable GenericCallOp::getCallableForCallee() {
 /// Set the callee for the generic call operation, this is required by the call
 /// interface.
 void GenericCallOp::setCalleeFromCallable(CallInterfaceCallable callee) {
-  (*this)->setAttr("callee", callee.get<SymbolRefAttr>());
+  (*this)->setAttr("callee", cast<SymbolRefAttr>(callee));
 }
 
 /// Get the argument operands to the called function, this is required by the
diff --git a/mlir/examples/toy/Ch7/mlir/LowerToAffineLoops.cpp b/mlir/examples/toy/Ch7/mlir/LowerToAffineLoops.cpp
index 741321432b452..bf2bc43301a33 100644
--- a/mlir/examples/toy/Ch7/mlir/LowerToAffineLoops.cpp
+++ b/mlir/examples/toy/Ch7/mlir/LowerToAffineLoops.cpp
@@ -328,6 +328,7 @@ namespace {
 struct ToyToAffineLoweringPass
     : public PassWrapper<ToyToAffineLoweringPass, OperationPass<ModuleOp>> {
   MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(ToyToAffineLoweringPass)
+  StringRef getArgument() const override { return "toy-to-affine"; }
 
   void getDependentDialects(DialectRegistry &registry) const override {
     registry.insert<affine::AffineDialect, func::FuncDialect,
diff --git a/mlir/examples/toy/Ch7/mlir/LowerToLLVM.cpp b/mlir/examples/toy/Ch7/mlir/LowerToLLVM.cpp
index 123d114ae1635..a20a2c888a175 100644
--- a/mlir/examples/toy/Ch7/mlir/LowerToLLVM.cpp
+++ b/mlir/examples/toy/Ch7/mlir/LowerToLLVM.cpp
@@ -184,6 +184,7 @@ namespace {
 struct ToyToLLVMLoweringPass
     : public PassWrapper<ToyToLLVMLoweringPass, OperationPass<ModuleOp>> {
   MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(ToyToLLVMLoweringPass)
+  StringRef getArgument() const override { return "toy-to-llvm"; }
 
   void getDependentDialects(DialectRegistry &registry) const override {
     registry.insert<LLVM::LLVMDialect, scf::SCFDialect>();
diff --git a/mlir/examples/toy/Ch7/mlir/MLIRGen.cpp b/mlir/examples/toy/Ch7/mlir/MLIRGen.cpp
index 090e5ff914604..e554e375209f1 100644
--- a/mlir/examples/toy/Ch7/mlir/MLIRGen.cpp
+++ b/mlir/examples/toy/Ch7/mlir/MLIRGen.cpp
@@ -535,8 +535,7 @@ class MLIRGenImpl {
     }
     mlir::toy::FuncOp calledFunc = calledFuncIt->second;
     return builder.create<GenericCallOp>(
-        location, calledFunc.getFunctionType().getResult(0),
-        mlir::SymbolRefAttr::get(builder.getContext(), callee), operands);
+        location, calledFunc.getFunctionType().getResult(0), callee, operands);
   }
 
   /// Emit a print expression. It emits specific operations for two builtins:
diff --git a/mlir/examples/toy/Ch7/mlir/ShapeInferencePass.cpp b/mlir/examples/toy/Ch7/mlir/ShapeInferencePass.cpp
index a9e995ed91bff..2522abe1a46f9 100644
--- a/mlir/examples/toy/Ch7/mlir/ShapeInferencePass.cpp
+++ b/mlir/examples/toy/Ch7/mlir/ShapeInferencePass.cpp
@@ -55,6 +55,7 @@ namespace {
 struct ShapeInferencePass
     : public mlir::PassWrapper<ShapeInferencePass, OperationPass<toy::FuncOp>> {
   MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(ShapeInferencePass)
+  StringRef getArgument() const override { return "toy-shape-inference"; }
 
   void runOnOperation() override {
     auto f = getOperation();
diff --git a/mlir/include/mlir-c/BuiltinAttributes.h b/mlir/include/mlir-c/BuiltinAttributes.h
index 7c8c84e55b962..1d0edf9ea809d 100644
--- a/mlir/include/mlir-c/BuiltinAttributes.h
+++ b/mlir/include/mlir-c/BuiltinAttributes.h
@@ -556,6 +556,8 @@ MLIR_CAPI_EXPORTED int64_t
 mlirDenseElementsAttrGetInt64Value(MlirAttribute attr, intptr_t pos);
 MLIR_CAPI_EXPORTED uint64_t
 mlirDenseElementsAttrGetUInt64Value(MlirAttribute attr, intptr_t pos);
+MLIR_CAPI_EXPORTED uint64_t
+mlirDenseElementsAttrGetIndexValue(MlirAttribute attr, intptr_t pos);
 MLIR_CAPI_EXPORTED float mlirDenseElementsAttrGetFloatValue(MlirAttribute attr,
                                                             intptr_t pos);
 MLIR_CAPI_EXPORTED double
diff --git a/mlir/include/mlir-c/Dialect/Func.h b/mlir/include/mlir-c/Dialect/Func.h
index 1df759f0e8e3f..001f915af1054 100644
--- a/mlir/include/mlir-c/Dialect/Func.h
+++ b/mlir/include/mlir-c/Dialect/Func.h
@@ -35,6 +35,10 @@ MLIR_CAPI_EXPORTED void mlirFuncSetArgAttr(MlirOperation op, intptr_t pos,
                                            MlirStringRef name,
                                            MlirAttribute attr);
 
+MLIR_CAPI_EXPORTED void mlirFuncSetResultAttr(MlirOperation op, intptr_t pos,
+                                              MlirStringRef name,
+                                              MlirAttribute attr);
+
 #ifdef __cplusplus
 }
 #endif
diff --git a/mlir/include/mlir-c/Dialect/LLVM.h b/mlir/include/mlir-c/Dialect/LLVM.h
index 26c4140757c3c..65b14254e4492 100644
--- a/mlir/include/mlir-c/Dialect/LLVM.h
+++ b/mlir/include/mlir-c/Dialect/LLVM.h
@@ -52,6 +52,9 @@ MLIR_CAPI_EXPORTED intptr_t mlirLLVMFunctionTypeGetNumInputs(MlirType type);
 MLIR_CAPI_EXPORTED MlirType mlirLLVMFunctionTypeGetInput(MlirType type,
                                                          intptr_t pos);
 
+/// Returns the return type of the function type.
+MLIR_CAPI_EXPORTED MlirType mlirLLVMFunctionTypeGetReturnType(MlirType type);
+
 /// Returns `true` if the type is an LLVM dialect struct type.
 MLIR_CAPI_EXPORTED bool mlirTypeIsALLVMStructType(MlirType type);
 
diff --git a/mlir/include/mlir-c/Target/LLVMIR.h b/mlir/include/mlir-c/Target/LLVMIR.h
index effa74b905ce6..b5f948961e898 100644
--- a/mlir/include/mlir-c/Target/LLVMIR.h
+++ b/mlir/include/mlir-c/Target/LLVMIR.h
@@ -16,6 +16,7 @@
 
 #include "mlir-c/IR.h"
 #include "mlir-c/Support.h"
+#include "llvm-c/Core.h"
 #include "llvm-c/Support.h"
 
 #ifdef __cplusplus
@@ -32,6 +33,48 @@ extern "C" {
 MLIR_CAPI_EXPORTED LLVMModuleRef
 mlirTranslateModuleToLLVMIR(MlirOperation module, LLVMContextRef context);
 
+struct MlirTypeFromLLVMIRTranslator {
+  void *ptr;
+};
+
+typedef struct MlirTypeFromLLVMIRTranslator MlirTypeFromLLVMIRTranslator;
+
+/// Create an LLVM::TypeFromLLVMIRTranslator and transfer ownership to the
+/// caller.
+MLIR_CAPI_EXPORTED MlirTypeFromLLVMIRTranslator
+mlirTypeFromLLVMIRTranslatorCreate(MlirContext ctx);
+
+/// Takes an LLVM::TypeFromLLVMIRTranslator owned by the caller and destroys it.
+/// It is the responsibility of the user to only pass an
+/// LLVM::TypeFromLLVMIRTranslator class.
+MLIR_CAPI_EXPORTED void
+mlirTypeFromLLVMIRTranslatorDestroy(MlirTypeFromLLVMIRTranslator translator);
+
+/// Translates the given LLVM IR type to the MLIR LLVM dialect.
+MLIR_CAPI_EXPORTED MlirType mlirTypeFromLLVMIRTranslatorTranslateType(
+    MlirTypeFromLLVMIRTranslator translator, LLVMTypeRef llvmType);
+
+struct MlirTypeToLLVMIRTranslator {
+  void *ptr;
+};
+
+typedef struct MlirTypeToLLVMIRTranslator MlirTypeToLLVMIRTranslator;
+
+/// Create an LLVM::TypeToLLVMIRTranslator and transfer ownership to the
+/// caller.
+MLIR_CAPI_EXPORTED MlirTypeToLLVMIRTranslator
+mlirTypeToLLVMIRTranslatorCreate(LLVMContextRef ctx);
+
+/// Takes an LLVM::TypeToLLVMIRTranslator owned by the caller and destroys it.
+/// It is the responsibility of the user to only pass an
+/// LLVM::TypeToLLVMIRTranslator class.
+MLIR_CAPI_EXPORTED void
+mlirTypeToLLVMIRTranslatorDestroy(MlirTypeToLLVMIRTranslator translator);
+
+/// Translates the given MLIR LLVM dialect to the LLVM IR type.
+MLIR_CAPI_EXPORTED LLVMTypeRef mlirTypeToLLVMIRTranslatorTranslateType(
+    MlirTypeToLLVMIRTranslator translator, MlirType mlirType);
+
 #ifdef __cplusplus
 }
 #endif
diff --git a/mlir/include/mlir/Analysis/DataFlow/SparseAnalysis.h b/mlir/include/mlir/Analysis/DataFlow/SparseAnalysis.h
index 387b9ee707179..b9cb549a0e438 100644
--- a/mlir/include/mlir/Analysis/DataFlow/SparseAnalysis.h
+++ b/mlir/include/mlir/Analysis/DataFlow/SparseAnalysis.h
@@ -255,10 +255,15 @@ class AbstractSparseForwardDataFlowAnalysis : public DataFlowAnalysis {
   /// operation `branch`, which can either be the entry block of one of the
   /// regions or the parent operation itself, and set either the argument or
   /// parent result lattices.
-  void visitRegionSuccessors(ProgramPoint *point,
-                             RegionBranchOpInterface branch,
-                             RegionBranchPoint successor,
-                             ArrayRef<AbstractSparseLattice *> lattices);
+  /// This method can be overridden to control precisely how the region
+  /// successors of `branch` are visited. For example in order to precisely
+  /// control the order in which predecessor operand lattices are propagated
+  /// from. An override is responsible for visiting all the known predecessors
+  /// and propagating therefrom.
+  virtual void
+  visitRegionSuccessors(ProgramPoint *point, RegionBranchOpInterface branch,
+                        RegionBranchPoint successor,
+                        ArrayRef<AbstractSparseLattice *> lattices);
 };
 
 //===----------------------------------------------------------------------===//
diff --git a/mlir/include/mlir/Analysis/DataFlowFramework.h b/mlir/include/mlir/Analysis/DataFlowFramework.h
index b6d10ba0bea2d..a3714c4332fbb 100644
--- a/mlir/include/mlir/Analysis/DataFlowFramework.h
+++ b/mlir/include/mlir/Analysis/DataFlowFramework.h
@@ -401,6 +401,8 @@ class DataFlowSolver {
 
   /// Propagate an update to an analysis state if it changed by pushing
   /// dependent work items to the back of the queue.
+  /// This should only be used when DataFlowSolver is running.
+  /// Otherwise, the solver won't process the work items.
   void propagateIfChanged(AnalysisState *state, ChangeResult changed);
 
   /// Get the configuration of the solver.
@@ -410,6 +412,9 @@ class DataFlowSolver {
   /// Configuration of the dataflow solver.
   DataFlowConfig config;
 
+  /// The solver is working on the worklist.
+  bool isRunning = false;
+
   /// The solver's work queue. Work items can be inserted to the front of the
   /// queue to be processed greedily, speeding up computations that otherwise
   /// quickly degenerate to quadratic due to propagation of state updates.
diff --git a/mlir/include/mlir/Bytecode/BytecodeWriter.h b/mlir/include/mlir/Bytecode/BytecodeWriter.h
index 0287e004bb993..c6cff0bc81314 100644
--- a/mlir/include/mlir/Bytecode/BytecodeWriter.h
+++ b/mlir/include/mlir/Bytecode/BytecodeWriter.h
@@ -82,6 +82,7 @@ class BytecodeWriterConfig {
   /// printers for the fallback resources within the map.
   BytecodeWriterConfig(FallbackAsmResourceMap &map,
                        StringRef producer = "MLIR" LLVM_VERSION_STRING);
+  BytecodeWriterConfig(BytecodeWriterConfig &&);
   ~BytecodeWriterConfig();
 
   /// An internal implementation class that contains the state of the
diff --git a/mlir/include/mlir/Conversion/ComplexToLLVM/ComplexToLLVM.h b/mlir/include/mlir/Conversion/ComplexToLLVM/ComplexToLLVM.h
index 65d6f165c7448..8266442cf5db8 100644
--- a/mlir/include/mlir/Conversion/ComplexToLLVM/ComplexToLLVM.h
+++ b/mlir/include/mlir/Conversion/ComplexToLLVM/ComplexToLLVM.h
@@ -24,8 +24,8 @@ class ComplexStructBuilder : public StructBuilder {
   /// Construct a helper for the given complex number value.
   using StructBuilder::StructBuilder;
   /// Build IR creating an `undef` value of the complex number type.
-  static ComplexStructBuilder undef(OpBuilder &builder, Location loc,
-                                    Type type);
+  static ComplexStructBuilder poison(OpBuilder &builder, Location loc,
+                                     Type type);
 
   // Build IR extracting the real value from the complex number struct.
   Value real(OpBuilder &builder, Location loc);
diff --git a/mlir/include/mlir/Conversion/ConvertToSPIRV/ConvertToSPIRVPass.h b/mlir/include/mlir/Conversion/ConvertToSPIRV/ConvertToSPIRVPass.h
deleted file mode 100644
index 3852782247527..0000000000000
--- a/mlir/include/mlir/Conversion/ConvertToSPIRV/ConvertToSPIRVPass.h
+++ /dev/null
@@ -1,22 +0,0 @@
-//===- ConvertToSPIRVPass.h - Conversion to SPIR-V pass ---*- C++ -*-=========//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef MLIR_CONVERSION_CONVERTTOSPIRV_CONVERTTOSPIRVPASS_H
-#define MLIR_CONVERSION_CONVERTTOSPIRV_CONVERTTOSPIRVPASS_H
-
-#include <memory>
-
-namespace mlir {
-class Pass;
-
-#define GEN_PASS_DECL_CONVERTTOSPIRVPASS
-#include "mlir/Conversion/Passes.h.inc"
-
-} // namespace mlir
-
-#endif // MLIR_CONVERSION_CONVERTTOSPIRV_CONVERTTOSPIRVPASS_H
diff --git a/mlir/include/mlir/Conversion/LLVMCommon/MemRefBuilder.h b/mlir/include/mlir/Conversion/LLVMCommon/MemRefBuilder.h
index e97675f37e10c..d5055f023cdc8 100644
--- a/mlir/include/mlir/Conversion/LLVMCommon/MemRefBuilder.h
+++ b/mlir/include/mlir/Conversion/LLVMCommon/MemRefBuilder.h
@@ -34,9 +34,9 @@ class MemRefDescriptor : public StructBuilder {
 public:
   /// Construct a helper for the given descriptor value.
   explicit MemRefDescriptor(Value descriptor);
-  /// Builds IR creating an `undef` value of the descriptor type.
-  static MemRefDescriptor undef(OpBuilder &builder, Location loc,
-                                Type descriptorType);
+  /// Builds IR creating a `poison` value of the descriptor type.
+  static MemRefDescriptor poison(OpBuilder &builder, Location loc,
+                                 Type descriptorType);
   /// Builds IR creating a MemRef descriptor that represents `type` and
   /// populates it with static shape and stride information extracted from the
   /// type.
@@ -104,7 +104,7 @@ class MemRefDescriptor : public StructBuilder {
   /// - aligned pointer;
   /// - offset;
   /// - <rank> sizes;
-  /// - <rank> shapes;
+  /// - <rank> strides;
   /// where <rank> is the MemRef rank as provided in `type`.
   static Value pack(OpBuilder &builder, Location loc,
                     const LLVMTypeConverter &converter, MemRefType type,
@@ -160,8 +160,8 @@ class UnrankedMemRefDescriptor : public StructBuilder {
   /// Construct a helper for the given descriptor value.
   explicit UnrankedMemRefDescriptor(Value descriptor);
   /// Builds IR creating an `undef` value of the descriptor type.
-  static UnrankedMemRefDescriptor undef(OpBuilder &builder, Location loc,
-                                        Type descriptorType);
+  static UnrankedMemRefDescriptor poison(OpBuilder &builder, Location loc,
+                                         Type descriptorType);
 
   /// Builds IR extracting the rank from the descriptor
   Value rank(OpBuilder &builder, Location loc) const;
diff --git a/mlir/include/mlir/Conversion/LLVMCommon/PrintCallHelper.h b/mlir/include/mlir/Conversion/LLVMCommon/PrintCallHelper.h
index c2742b6fc1d73..33402301115b7 100644
--- a/mlir/include/mlir/Conversion/LLVMCommon/PrintCallHelper.h
+++ b/mlir/include/mlir/Conversion/LLVMCommon/PrintCallHelper.h
@@ -23,11 +23,10 @@ namespace LLVM {
 /// Generate IR that prints the given string to stdout.
 /// If a custom runtime function is defined via `runtimeFunctionName`, it must
 /// have the signature void(char const*). The default function is `printString`.
-void createPrintStrCall(OpBuilder &builder, Location loc, ModuleOp moduleOp,
-                        StringRef symbolName, StringRef string,
-                        const LLVMTypeConverter &typeConverter,
-                        bool addNewline = true,
-                        std::optional<StringRef> runtimeFunctionName = {});
+LogicalResult createPrintStrCall(
+    OpBuilder &builder, Location loc, ModuleOp moduleOp, StringRef symbolName,
+    StringRef string, const LLVMTypeConverter &typeConverter,
+    bool addNewline = true, std::optional<StringRef> runtimeFunctionName = {});
 } // namespace LLVM
 
 } // namespace mlir
diff --git a/mlir/include/mlir/Conversion/LLVMCommon/StructBuilder.h b/mlir/include/mlir/Conversion/LLVMCommon/StructBuilder.h
index 1a5b97eb92830..b6ff1d13abf1d 100644
--- a/mlir/include/mlir/Conversion/LLVMCommon/StructBuilder.h
+++ b/mlir/include/mlir/Conversion/LLVMCommon/StructBuilder.h
@@ -27,9 +27,9 @@ class StructBuilder {
 public:
   /// Construct a helper for the given value.
   explicit StructBuilder(Value v);
-  /// Builds IR creating an `undef` value of the descriptor type.
-  static StructBuilder undef(OpBuilder &builder, Location loc,
-                             Type descriptorType);
+  /// Builds IR creating a `poison` value of the descriptor type.
+  static StructBuilder poison(OpBuilder &builder, Location loc,
+                              Type descriptorType);
 
   /*implicit*/ operator Value() { return value; }
 
diff --git a/mlir/include/mlir/Conversion/MathToEmitC/MathToEmitC.h b/mlir/include/mlir/Conversion/MathToEmitC/MathToEmitC.h
index 0fc33bf790be4..c61773026ca53 100644
--- a/mlir/include/mlir/Conversion/MathToEmitC/MathToEmitC.h
+++ b/mlir/include/mlir/Conversion/MathToEmitC/MathToEmitC.h
@@ -8,7 +8,6 @@
 
 #ifndef MLIR_CONVERSION_MATHTOEMITC_MATHTOEMITC_H
 #define MLIR_CONVERSION_MATHTOEMITC_MATHTOEMITC_H
-#include "mlir/Dialect/EmitC/IR/EmitC.h"
 namespace mlir {
 class RewritePatternSet;
 namespace emitc {
diff --git a/mlir/include/mlir/Conversion/Passes.h b/mlir/include/mlir/Conversion/Passes.h
index e9761c20642c0..ccd862f67c068 100644
--- a/mlir/include/mlir/Conversion/Passes.h
+++ b/mlir/include/mlir/Conversion/Passes.h
@@ -30,7 +30,6 @@
 #include "mlir/Conversion/ControlFlowToSPIRV/ControlFlowToSPIRV.h"
 #include "mlir/Conversion/ControlFlowToSPIRV/ControlFlowToSPIRVPass.h"
 #include "mlir/Conversion/ConvertToLLVM/ToLLVMPass.h"
-#include "mlir/Conversion/ConvertToSPIRV/ConvertToSPIRVPass.h"
 #include "mlir/Conversion/FuncToEmitC/FuncToEmitCPass.h"
 #include "mlir/Conversion/FuncToLLVM/ConvertFuncToLLVMPass.h"
 #include "mlir/Conversion/FuncToSPIRV/FuncToSPIRVPass.h"
diff --git a/mlir/include/mlir/Conversion/Passes.td b/mlir/include/mlir/Conversion/Passes.td
index b547839d76738..ff79a1226c047 100644
--- a/mlir/include/mlir/Conversion/Passes.td
+++ b/mlir/include/mlir/Conversion/Passes.td
@@ -39,35 +39,6 @@ def ConvertToLLVMPass : Pass<"convert-to-llvm"> {
   ];
 }
 
-//===----------------------------------------------------------------------===//
-// ToSPIRV
-//===----------------------------------------------------------------------===//
-
-def ConvertToSPIRVPass : Pass<"convert-to-spirv"> {
-  let summary = "Convert to SPIR-V";
-  let description = [{
-    This is a generic pass to convert to SPIR-V.
-  }];
-  let dependentDialects = [
-    "spirv::SPIRVDialect",
-    "vector::VectorDialect",
-  ];
-  let options = [
-    Option<"runSignatureConversion", "run-signature-conversion", "bool",
-    /*default=*/"true",
-    "Run function signature conversion to convert vector types">,
-    Option<"runVectorUnrolling", "run-vector-unrolling", "bool",
-    /*default=*/"true",
-    "Run vector unrolling to convert vector types in function bodies">,
-    Option<"convertGPUModules", "convert-gpu-modules", "bool",
-    /*default=*/"false",
-    "Clone and convert GPU modules">,
-    Option<"nestInGPUModule", "nest-in-gpu-module", "bool",
-    /*default=*/"false",
-    "Put converted SPIR-V module inside the gpu.module instead of alongside it.">,
-  ];
-}
-
 //===----------------------------------------------------------------------===//
 // AffineToStandard
 //===----------------------------------------------------------------------===//
@@ -1454,7 +1425,10 @@ def ConvertVectorToLLVMPass : Pass<"convert-vector-to-llvm"> {
 def ConvertVectorToSPIRV : Pass<"convert-vector-to-spirv"> {
   let summary = "Convert Vector dialect to SPIR-V dialect";
   let constructor = "mlir::createConvertVectorToSPIRVPass()";
-  let dependentDialects = ["spirv::SPIRVDialect"];
+  let dependentDialects = [
+    "spirv::SPIRVDialect",
+    "ub::UBDialect"
+  ];
 }
 
 //===----------------------------------------------------------------------===//
diff --git a/mlir/include/mlir/Conversion/VectorToLLVM/ConvertVectorToLLVM.h b/mlir/include/mlir/Conversion/VectorToLLVM/ConvertVectorToLLVM.h
index 5fda62e3584c7..1e29bfeb9c392 100644
--- a/mlir/include/mlir/Conversion/VectorToLLVM/ConvertVectorToLLVM.h
+++ b/mlir/include/mlir/Conversion/VectorToLLVM/ConvertVectorToLLVM.h
@@ -24,6 +24,9 @@ void populateVectorToLLVMConversionPatterns(
     const LLVMTypeConverter &converter, RewritePatternSet &patterns,
     bool reassociateFPReductions = false, bool force32BitVectorIndices = false);
 
+namespace vector {
+void registerConvertVectorToLLVMInterface(DialectRegistry &registry);
+}
 } // namespace mlir
 
 #endif // MLIR_CONVERSION_VECTORTOLLVM_CONVERTVECTORTOLLVM_H_
diff --git a/mlir/include/mlir/Dialect/Affine/Analysis/Utils.h b/mlir/include/mlir/Dialect/Affine/Analysis/Utils.h
index b8f354892ee60..5b386868cb004 100644
--- a/mlir/include/mlir/Dialect/Affine/Analysis/Utils.h
+++ b/mlir/include/mlir/Dialect/Affine/Analysis/Utils.h
@@ -37,9 +37,16 @@ struct MemRefAccess;
 // was encountered in the loop nest.
 struct LoopNestStateCollector {
   SmallVector<AffineForOp, 4> forOps;
+  // Affine loads.
   SmallVector<Operation *, 4> loadOpInsts;
+  // Affine stores.
   SmallVector<Operation *, 4> storeOpInsts;
-  bool hasNonAffineRegionOp = false;
+  // Non-affine loads.
+  SmallVector<Operation *, 4> memrefLoads;
+  // Non-affine stores.
+  SmallVector<Operation *, 4> memrefStores;
+  // Free operations.
+  SmallVector<Operation *, 4> memrefFrees;
 
   // Collects load and store operations, and whether or not a region holding op
   // other than ForOp and IfOp was encountered in the loop nest.
@@ -47,9 +54,11 @@ struct LoopNestStateCollector {
 };
 
 // MemRefDependenceGraph is a graph data structure where graph nodes are
-// top-level operations in a `Block` which contain load/store ops, and edges
-// are memref dependences between the nodes.
-// TODO: Add a more flexible dependence graph representation.
+// top-level operations in a `Block` and edges are memref dependences or SSA
+// dependences (on memrefs) between the nodes. Nodes are created for all
+// top-level operations except in certain cases (see `init` method). Edges are
+// created between nodes with a dependence (see `Edge` documentation). Edges
+// aren't created from/to nodes that have no memory effects.
 struct MemRefDependenceGraph {
 public:
   // Node represents a node in the graph. A Node is either an entire loop nest
@@ -60,10 +69,18 @@ struct MemRefDependenceGraph {
     unsigned id;
     // The top-level statement which is (or contains) a load/store.
     Operation *op;
-    // List of load operations.
+    // List of affine loads.
     SmallVector<Operation *, 4> loads;
-    // List of store op insts.
+    // List of non-affine loads.
+    SmallVector<Operation *, 4> memrefLoads;
+    // List of affine store ops.
     SmallVector<Operation *, 4> stores;
+    // List of non-affine stores.
+    SmallVector<Operation *, 4> memrefStores;
+    // List of free operations.
+    SmallVector<Operation *, 4> memrefFrees;
+    // Set of private memrefs used in this node.
+    DenseSet<Value> privateMemrefs;
 
     Node(unsigned id, Operation *op) : id(id), op(op) {}
 
@@ -73,6 +90,13 @@ struct MemRefDependenceGraph {
     // Returns the store op count for 'memref'.
     unsigned getStoreOpCount(Value memref) const;
 
+    /// Returns true if there exists an operation with a write memory effect to
+    /// `memref` in this node.
+    unsigned hasStore(Value memref) const;
+
+    // Returns true if the node has a free op on `memref`.
+    unsigned hasFree(Value memref) const;
+
     // Returns all store ops in 'storeOps' which access 'memref'.
     void getStoreOpsForMemref(Value memref,
                               SmallVectorImpl<Operation *> *storeOps) const;
@@ -86,7 +110,16 @@ struct MemRefDependenceGraph {
     void getLoadAndStoreMemrefSet(DenseSet<Value> *loadAndStoreMemrefSet) const;
   };
 
-  // Edge represents a data dependence between nodes in the graph.
+  // Edge represents a data dependence between nodes in the graph. It can either
+  // be a memory dependence or an SSA dependence. In the former case, it
+  // corresponds to a pair of memory accesses to the same memref or aliasing
+  // memrefs where at least one of them has a write or free memory effect. The
+  // memory accesses need not be affine load/store operations. Operations are
+  // checked for read/write effects and edges may be added conservatively. Edges
+  // are not created to/from nodes that have no memory effect. An exception to
+  // this are SSA dependences between operations that define memrefs (like
+  // alloc's, view-like ops) and their memory-effecting users that are enclosed
+  // in loops.
   struct Edge {
     // The id of the node at the other end of the edge.
     // If this edge is stored in Edge = Node.inEdges[i], then
@@ -106,9 +139,11 @@ struct MemRefDependenceGraph {
 
   // Map from node id to Node.
   DenseMap<unsigned, Node> nodes;
-  // Map from node id to list of input edges.
+  // Map from node id to list of input edges. The absence of an entry for a key
+  // is also equivalent to the absence of any edges.
   DenseMap<unsigned, SmallVector<Edge, 2>> inEdges;
-  // Map from node id to list of output edges.
+  // Map from node id to list of output edges. The absence of an entry for a
+  // node is also equivalent to the absence of any edges.
   DenseMap<unsigned, SmallVector<Edge, 2>> outEdges;
   // Map from memref to a count on the dependence edges associated with that
   // memref.
@@ -123,10 +158,21 @@ struct MemRefDependenceGraph {
   bool init();
 
   // Returns the graph node for 'id'.
-  Node *getNode(unsigned id);
+  const Node *getNode(unsigned id) const;
+  Node *getNode(unsigned id) {
+    return const_cast<Node *>(
+        static_cast<const MemRefDependenceGraph *>(this)->getNode(id));
+  }
+
+  // Returns true if the graph has node with ID `id`.
+  bool hasNode(unsigned id) const { return nodes.contains(id); }
 
   // Returns the graph node for 'forOp'.
-  Node *getForOpNode(AffineForOp forOp);
+  const Node *getForOpNode(AffineForOp forOp) const;
+  Node *getForOpNode(AffineForOp forOp) {
+    return const_cast<Node *>(
+        static_cast<const MemRefDependenceGraph *>(this)->getForOpNode(forOp));
+  }
 
   // Adds a node with 'op' to the graph and returns its unique identifier.
   unsigned addNode(Operation *op);
@@ -136,12 +182,12 @@ struct MemRefDependenceGraph {
 
   // Returns true if node 'id' writes to any memref which escapes (or is an
   // argument to) the block. Returns false otherwise.
-  bool writesToLiveInOrEscapingMemrefs(unsigned id);
+  bool writesToLiveInOrEscapingMemrefs(unsigned id) const;
 
   // Returns true iff there is an edge from node 'srcId' to node 'dstId' which
   // is for 'value' if non-null, or for any value otherwise. Returns false
   // otherwise.
-  bool hasEdge(unsigned srcId, unsigned dstId, Value value = nullptr);
+  bool hasEdge(unsigned srcId, unsigned dstId, Value value = nullptr) const;
 
   // Adds an edge from node 'srcId' to node 'dstId' for 'value'.
   void addEdge(unsigned srcId, unsigned dstId, Value value);
@@ -152,23 +198,25 @@ struct MemRefDependenceGraph {
   // Returns true if there is a path in the dependence graph from node 'srcId'
   // to node 'dstId'. Returns false otherwise. `srcId`, `dstId`, and the
   // operations that the edges connected are expected to be from the same block.
-  bool hasDependencePath(unsigned srcId, unsigned dstId);
+  bool hasDependencePath(unsigned srcId, unsigned dstId) const;
 
   // Returns the input edge count for node 'id' and 'memref' from src nodes
   // which access 'memref' with a store operation.
-  unsigned getIncomingMemRefAccesses(unsigned id, Value memref);
+  unsigned getIncomingMemRefAccesses(unsigned id, Value memref) const;
 
   // Returns the output edge count for node 'id' and 'memref' (if non-null),
   // otherwise returns the total output edge count from node 'id'.
-  unsigned getOutEdgeCount(unsigned id, Value memref = nullptr);
+  unsigned getOutEdgeCount(unsigned id, Value memref = nullptr) const;
 
   /// Return all nodes which define SSA values used in node 'id'.
-  void gatherDefiningNodes(unsigned id, DenseSet<unsigned> &definingNodes);
+  void gatherDefiningNodes(unsigned id,
+                           DenseSet<unsigned> &definingNodes) const;
 
   // Computes and returns an insertion point operation, before which the
   // the fused <srcId, dstId> loop nest can be inserted while preserving
   // dependences. Returns nullptr if no such insertion point is found.
-  Operation *getFusedLoopNestInsertionPoint(unsigned srcId, unsigned dstId);
+  Operation *getFusedLoopNestInsertionPoint(unsigned srcId,
+                                            unsigned dstId) const;
 
   // Updates edge mappings from node 'srcId' to node 'dstId' after fusing them,
   // taking into account that:
@@ -182,9 +230,12 @@ struct MemRefDependenceGraph {
   // of sibling node 'sibId' into node 'dstId'.
   void updateEdges(unsigned sibId, unsigned dstId);
 
-  // Adds ops in 'loads' and 'stores' to node at 'id'.
-  void addToNode(unsigned id, const SmallVectorImpl<Operation *> &loads,
-                 const SmallVectorImpl<Operation *> &stores);
+  // Adds the specified ops to lists of node at 'id'.
+  void addToNode(unsigned id, ArrayRef<Operation *> loads,
+                 ArrayRef<Operation *> stores,
+                 ArrayRef<Operation *> memrefLoads,
+                 ArrayRef<Operation *> memrefStores,
+                 ArrayRef<Operation *> memrefFrees);
 
   void clearNodeLoadAndStores(unsigned id);
 
@@ -574,6 +625,14 @@ FailureOr<AffineValueMap>
 simplifyConstrainedMinMaxOp(Operation *op,
                             FlatAffineValueConstraints constraints);
 
+/// Find the innermost common `Block` of `a` and `b` in the affine scope
+/// that `a` and `b` are part of. Return nullptr if they belong to different
+/// affine scopes. Also, return nullptr if they do not have a common `Block`
+/// ancestor (for eg., when they are part of the `then` and `else` regions
+/// of an op that itself starts an affine scope.
+mlir::Block *findInnermostCommonBlockInScope(mlir::Operation *a,
+                                             mlir::Operation *b);
+
 } // namespace affine
 } // namespace mlir
 
diff --git a/mlir/include/mlir/Dialect/Affine/Passes.h b/mlir/include/mlir/Dialect/Affine/Passes.h
index e152101236dc7..bc29d04287ac4 100644
--- a/mlir/include/mlir/Dialect/Affine/Passes.h
+++ b/mlir/include/mlir/Dialect/Affine/Passes.h
@@ -44,10 +44,6 @@ createSimplifyAffineStructuresPass();
 std::unique_ptr<OperationPass<func::FuncOp>>
 createAffineLoopInvariantCodeMotionPass();
 
-/// Creates a pass to convert all parallel affine.for's into 1-d affine.parallel
-/// ops.
-std::unique_ptr<OperationPass<func::FuncOp>> createAffineParallelizePass();
-
 /// Apply normalization transformations to affine loop-like ops. If
 /// `promoteSingleIter` is true, single iteration loops are promoted (i.e., the
 /// loop is replaced by its loop body).
diff --git a/mlir/include/mlir/Dialect/Affine/Passes.td b/mlir/include/mlir/Dialect/Affine/Passes.td
index 77073aa29da73..d7c7897c65730 100644
--- a/mlir/include/mlir/Dialect/Affine/Passes.td
+++ b/mlir/include/mlir/Dialect/Affine/Passes.td
@@ -370,7 +370,6 @@ def AffineVectorize : Pass<"affine-super-vectorize", "func::FuncOp"> {
 
 def AffineParallelize : Pass<"affine-parallelize", "func::FuncOp"> {
   let summary = "Convert affine.for ops into 1-D affine.parallel";
-  let constructor = "mlir::affine::createAffineParallelizePass()";
   let options = [
     Option<"maxNested", "max-nested", "unsigned", /*default=*/"-1u",
            "Maximum number of nested parallel loops to produce. "
diff --git a/mlir/include/mlir/Dialect/Affine/Utils.h b/mlir/include/mlir/Dialect/Affine/Utils.h
index 0f801ebb6f589..ff1900bc8f2eb 100644
--- a/mlir/include/mlir/Dialect/Affine/Utils.h
+++ b/mlir/include/mlir/Dialect/Affine/Utils.h
@@ -31,6 +31,7 @@ class FuncOp;
 
 namespace memref {
 class AllocOp;
+class AllocaOp;
 } // namespace memref
 
 namespace affine {
@@ -245,7 +246,12 @@ LogicalResult replaceAllMemRefUsesWith(Value oldMemRef, Value newMemRef,
 /// Rewrites the memref defined by this alloc op to have an identity layout map
 /// and updates all its indexing uses. Returns failure if any of its uses
 /// escape (while leaving the IR in a valid state).
-LogicalResult normalizeMemRef(memref::AllocOp *op);
+template <typename AllocLikeOp>
+LogicalResult normalizeMemRef(AllocLikeOp *op);
+extern template LogicalResult
+normalizeMemRef<memref::AllocaOp>(memref::AllocaOp *op);
+extern template LogicalResult
+normalizeMemRef<memref::AllocOp>(memref::AllocOp *op);
 
 /// Normalizes `memrefType` so that the affine layout map of the memref is
 /// transformed to an identity map with a new shape being computed for the
diff --git a/mlir/include/mlir/Dialect/Arith/IR/ArithOps.td b/mlir/include/mlir/Dialect/Arith/IR/ArithOps.td
index ea9b0f6509b80..d50b6aeca15c9 100644
--- a/mlir/include/mlir/Dialect/Arith/IR/ArithOps.td
+++ b/mlir/include/mlir/Dialect/Arith/IR/ArithOps.td
@@ -144,13 +144,6 @@ class Arith_CompareOp<string mnemonic, list<Trait> traits = []> :
   let assemblyFormat = "$predicate `,` $lhs `,` $rhs attr-dict `:` type($lhs)";
 }
 
-// Just like `Arith_CompareOp` but also admits 0-D vectors. Introduced
-// temporarily to allow gradual transition to 0-D vectors.
-class Arith_CompareOpOfAnyRank<string mnemonic, list<Trait> traits = []> :
-    Arith_CompareOp<mnemonic, traits> {
-  let results = (outs BoolLikeOfAnyRank:$result);
-}
-
 class Arith_IntBinaryOpWithOverflowFlags<string mnemonic, list<Trait> traits = []> :
     Arith_BinaryOp<mnemonic, traits #
       [Pure, DeclareOpInterfaceMethods<InferIntRangeInterface, ["inferResultRanges"]>,
@@ -1426,9 +1419,9 @@ def Arith_BitcastOp : Arith_CastOp<"bitcast", BitcastTypeConstraint,
 // CmpIOp
 //===----------------------------------------------------------------------===//
 
-def Arith_CmpIOp
-  : Arith_CompareOpOfAnyRank<"cmpi",
-                             [DeclareOpInterfaceMethods<InferIntRangeInterface, ["inferResultRanges"]>]> {
+def Arith_CmpIOp : Arith_CompareOp<"cmpi",
+    [DeclareOpInterfaceMethods<InferIntRangeInterface,
+                               ["inferResultRanges"]>]> {
   let summary = "integer comparison operation";
   let description = [{
     The `cmpi` operation is a generic comparison for integer-like types. Its two
@@ -1495,8 +1488,8 @@ def Arith_CmpIOp
   }];
 
   let arguments = (ins Arith_CmpIPredicateAttr:$predicate,
-                       SignlessIntegerOrIndexLikeOfAnyRank:$lhs,
-                       SignlessIntegerOrIndexLikeOfAnyRank:$rhs);
+                       SignlessIntegerOrIndexLike:$lhs,
+                       SignlessIntegerOrIndexLike:$rhs);
 
   let hasFolder = 1;
   let hasCanonicalizer = 1;
diff --git a/mlir/include/mlir/Dialect/Async/IR/AsyncOps.td b/mlir/include/mlir/Dialect/Async/IR/AsyncOps.td
index a08f5d6e714ef..3d29d5bc7dbb6 100644
--- a/mlir/include/mlir/Dialect/Async/IR/AsyncOps.td
+++ b/mlir/include/mlir/Dialect/Async/IR/AsyncOps.td
@@ -208,7 +208,13 @@ def Async_CallOp : Async_Op<"call",
     ```
   }];
 
-  let arguments = (ins FlatSymbolRefAttr:$callee, Variadic<AnyType>:$operands);
+  let arguments = (ins
+    FlatSymbolRefAttr:$callee,
+    Variadic<AnyType>:$operands,
+    OptionalAttr<DictArrayAttr>:$arg_attrs,
+    OptionalAttr<DictArrayAttr>:$res_attrs
+  );
+
   let results = (outs Variadic<Async_AnyValueOrTokenType>);
 
   let builders = [
diff --git a/mlir/include/mlir/Dialect/EmitC/IR/EmitC.td b/mlir/include/mlir/Dialect/EmitC/IR/EmitC.td
index abcc00feb5816..4fbce995ce5b8 100644
--- a/mlir/include/mlir/Dialect/EmitC/IR/EmitC.td
+++ b/mlir/include/mlir/Dialect/EmitC/IR/EmitC.td
@@ -551,7 +551,13 @@ def EmitC_CallOp : EmitC_Op<"call",
     %2 = emitc.call @my_add(%0, %1) : (f32, f32) -> f32
     ```
   }];
-  let arguments = (ins FlatSymbolRefAttr:$callee, Variadic<EmitCType>:$operands);
+  let arguments = (ins 
+    FlatSymbolRefAttr:$callee,
+    Variadic<EmitCType>:$operands,
+    OptionalAttr<DictArrayAttr>:$arg_attrs,
+    OptionalAttr<DictArrayAttr>:$res_attrs
+  );
+
   let results = (outs Variadic<EmitCType>);
 
   let builders = [
diff --git a/mlir/include/mlir/Dialect/Func/IR/FuncOps.td b/mlir/include/mlir/Dialect/Func/IR/FuncOps.td
index 4da0efcb13ddf..cdaeb6461afb4 100644
--- a/mlir/include/mlir/Dialect/Func/IR/FuncOps.td
+++ b/mlir/include/mlir/Dialect/Func/IR/FuncOps.td
@@ -49,8 +49,14 @@ def CallOp : Func_Op<"call",
     ```
   }];
 
-  let arguments = (ins FlatSymbolRefAttr:$callee, Variadic<AnyType>:$operands,
-                       UnitAttr:$no_inline);
+  let arguments = (ins
+    FlatSymbolRefAttr:$callee,
+    Variadic<AnyType>:$operands,
+    OptionalAttr<DictArrayAttr>:$arg_attrs,
+    OptionalAttr<DictArrayAttr>:$res_attrs, 
+    UnitAttr:$no_inline
+  );
+
   let results = (outs Variadic<AnyType>);
 
   let builders = [
@@ -73,6 +79,18 @@ def CallOp : Func_Op<"call",
       CArg<"ValueRange", "{}">:$operands), [{
       build($_builder, $_state, StringAttr::get($_builder.getContext(), callee),
             results, operands);
+    }]>,
+    OpBuilder<(ins "TypeRange":$results, "FlatSymbolRefAttr":$callee,
+      CArg<"ValueRange", "{}">:$operands), [{
+      build($_builder, $_state, callee, results, operands);
+    }]>,
+    OpBuilder<(ins "TypeRange":$results, "StringAttr":$callee,
+      CArg<"ValueRange", "{}">:$operands), [{
+      build($_builder, $_state, callee, results, operands);
+    }]>,
+    OpBuilder<(ins "TypeRange":$results, "StringRef":$callee,
+      CArg<"ValueRange", "{}">:$operands), [{
+      build($_builder, $_state, callee, results, operands);
     }]>];
 
   let extraClassDeclaration = [{
@@ -136,8 +154,13 @@ def CallIndirectOp : Func_Op<"call_indirect", [
     ```
   }];
 
-  let arguments = (ins FunctionType:$callee,
-                       Variadic<AnyType>:$callee_operands);
+  let arguments = (ins
+    FunctionType:$callee,
+    Variadic<AnyType>:$callee_operands,
+    OptionalAttr<DictArrayAttr>:$arg_attrs,
+    OptionalAttr<DictArrayAttr>:$res_attrs
+  );
+
   let results = (outs Variadic<AnyType>:$results);
 
   let builders = [
diff --git a/mlir/include/mlir/Dialect/GPU/IR/GPUOps.td b/mlir/include/mlir/Dialect/GPU/IR/GPUOps.td
index 3adfd5f4f2c43..2b1ce573effd0 100644
--- a/mlir/include/mlir/Dialect/GPU/IR/GPUOps.td
+++ b/mlir/include/mlir/Dialect/GPU/IR/GPUOps.td
@@ -794,9 +794,9 @@ def GPU_LaunchFuncOp :GPU_Op<"launch_func", [
 }
 
 def GPU_LaunchOp : GPU_Op<"launch", [
-      AutomaticAllocationScope, AttrSizedOperandSegments, GPU_AsyncOpInterface,
+      AffineScope, AutomaticAllocationScope, AttrSizedOperandSegments,
       DeclareOpInterfaceMethods<InferIntRangeInterface, ["inferResultRanges"]>,
-      RecursiveMemoryEffects]>,
+      GPU_AsyncOpInterface, RecursiveMemoryEffects]>,
     Arguments<(ins Variadic<GPU_AsyncToken>:$asyncDependencies,
                Index:$gridSizeX, Index:$gridSizeY, Index:$gridSizeZ,
                Index:$blockSizeX, Index:$blockSizeY, Index:$blockSizeZ,
diff --git a/mlir/include/mlir/Dialect/IRDL/IR/CMakeLists.txt b/mlir/include/mlir/Dialect/IRDL/IR/CMakeLists.txt
index ec3e06bedae03..861db0ca96671 100644
--- a/mlir/include/mlir/Dialect/IRDL/IR/CMakeLists.txt
+++ b/mlir/include/mlir/Dialect/IRDL/IR/CMakeLists.txt
@@ -1,5 +1,5 @@
 add_mlir_dialect(IRDL irdl)
-add_mlir_doc(IRDLOps IRDL Dialects/ -gen-dialect-doc)
+add_mlir_doc(IRDLOps IRDL Dialects/ -gen-dialect-doc -dialect=irdl)
 
 # Add IRDL interfaces
 set(LLVM_TARGET_DEFINITIONS IRDLInterfaces.td)
diff --git a/mlir/include/mlir/Dialect/LLVMIR/FunctionCallUtils.h b/mlir/include/mlir/Dialect/LLVMIR/FunctionCallUtils.h
index 852490cf7428f..05e9fe9d58859 100644
--- a/mlir/include/mlir/Dialect/LLVMIR/FunctionCallUtils.h
+++ b/mlir/include/mlir/Dialect/LLVMIR/FunctionCallUtils.h
@@ -16,7 +16,6 @@
 
 #include "mlir/IR/Operation.h"
 #include "mlir/Support/LLVM.h"
-#include <optional>
 
 namespace mlir {
 class Location;
@@ -29,42 +28,47 @@ class ValueRange;
 namespace LLVM {
 class LLVMFuncOp;
 
-/// Helper functions to lookup or create the declaration for commonly used
+/// Helper functions to look up or create the declaration for commonly used
 /// external C function calls. The list of functions provided here must be
 /// implemented separately (e.g. as part of a support runtime library or as part
 /// of the libc).
-LLVM::LLVMFuncOp lookupOrCreatePrintI64Fn(Operation *moduleOp);
-LLVM::LLVMFuncOp lookupOrCreatePrintU64Fn(Operation *moduleOp);
-LLVM::LLVMFuncOp lookupOrCreatePrintF16Fn(Operation *moduleOp);
-LLVM::LLVMFuncOp lookupOrCreatePrintBF16Fn(Operation *moduleOp);
-LLVM::LLVMFuncOp lookupOrCreatePrintF32Fn(Operation *moduleOp);
-LLVM::LLVMFuncOp lookupOrCreatePrintF64Fn(Operation *moduleOp);
+/// Failure if an unexpected version of function is found.
+FailureOr<LLVM::LLVMFuncOp> lookupOrCreatePrintI64Fn(Operation *moduleOp);
+FailureOr<LLVM::LLVMFuncOp> lookupOrCreatePrintU64Fn(Operation *moduleOp);
+FailureOr<LLVM::LLVMFuncOp> lookupOrCreatePrintF16Fn(Operation *moduleOp);
+FailureOr<LLVM::LLVMFuncOp> lookupOrCreatePrintBF16Fn(Operation *moduleOp);
+FailureOr<LLVM::LLVMFuncOp> lookupOrCreatePrintF32Fn(Operation *moduleOp);
+FailureOr<LLVM::LLVMFuncOp> lookupOrCreatePrintF64Fn(Operation *moduleOp);
 /// Declares a function to print a C-string.
 /// If a custom runtime function is defined via `runtimeFunctionName`, it must
 /// have the signature void(char const*). The default function is `printString`.
-LLVM::LLVMFuncOp
+FailureOr<LLVM::LLVMFuncOp>
 lookupOrCreatePrintStringFn(Operation *moduleOp,
                             std::optional<StringRef> runtimeFunctionName = {});
-LLVM::LLVMFuncOp lookupOrCreatePrintOpenFn(Operation *moduleOp);
-LLVM::LLVMFuncOp lookupOrCreatePrintCloseFn(Operation *moduleOp);
-LLVM::LLVMFuncOp lookupOrCreatePrintCommaFn(Operation *moduleOp);
-LLVM::LLVMFuncOp lookupOrCreatePrintNewlineFn(Operation *moduleOp);
-LLVM::LLVMFuncOp lookupOrCreateMallocFn(Operation *moduleOp, Type indexType);
-LLVM::LLVMFuncOp lookupOrCreateAlignedAllocFn(Operation *moduleOp,
-                                              Type indexType);
-LLVM::LLVMFuncOp lookupOrCreateFreeFn(Operation *moduleOp);
-LLVM::LLVMFuncOp lookupOrCreateGenericAllocFn(Operation *moduleOp,
-                                              Type indexType);
-LLVM::LLVMFuncOp lookupOrCreateGenericAlignedAllocFn(Operation *moduleOp,
-                                                     Type indexType);
-LLVM::LLVMFuncOp lookupOrCreateGenericFreeFn(Operation *moduleOp);
-LLVM::LLVMFuncOp lookupOrCreateMemRefCopyFn(Operation *moduleOp, Type indexType,
-                                            Type unrankedDescriptorType);
+FailureOr<LLVM::LLVMFuncOp> lookupOrCreatePrintOpenFn(Operation *moduleOp);
+FailureOr<LLVM::LLVMFuncOp> lookupOrCreatePrintCloseFn(Operation *moduleOp);
+FailureOr<LLVM::LLVMFuncOp> lookupOrCreatePrintCommaFn(Operation *moduleOp);
+FailureOr<LLVM::LLVMFuncOp> lookupOrCreatePrintNewlineFn(Operation *moduleOp);
+FailureOr<LLVM::LLVMFuncOp> lookupOrCreateMallocFn(Operation *moduleOp,
+                                                   Type indexType);
+FailureOr<LLVM::LLVMFuncOp> lookupOrCreateAlignedAllocFn(Operation *moduleOp,
+                                                         Type indexType);
+FailureOr<LLVM::LLVMFuncOp> lookupOrCreateFreeFn(Operation *moduleOp);
+FailureOr<LLVM::LLVMFuncOp> lookupOrCreateGenericAllocFn(Operation *moduleOp,
+                                                         Type indexType);
+FailureOr<LLVM::LLVMFuncOp>
+lookupOrCreateGenericAlignedAllocFn(Operation *moduleOp, Type indexType);
+FailureOr<LLVM::LLVMFuncOp> lookupOrCreateGenericFreeFn(Operation *moduleOp);
+FailureOr<LLVM::LLVMFuncOp>
+lookupOrCreateMemRefCopyFn(Operation *moduleOp, Type indexType,
+                           Type unrankedDescriptorType);
 
 /// Create a FuncOp with signature `resultType`(`paramTypes`)` and name `name`.
-LLVM::LLVMFuncOp lookupOrCreateFn(Operation *moduleOp, StringRef name,
-                                  ArrayRef<Type> paramTypes = {},
-                                  Type resultType = {}, bool isVarArg = false);
+/// Return a failure if the FuncOp found has unexpected signature.
+FailureOr<LLVM::LLVMFuncOp>
+lookupOrCreateFn(Operation *moduleOp, StringRef name,
+                 ArrayRef<Type> paramTypes = {}, Type resultType = {},
+                 bool isVarArg = false, bool isReserved = false);
 
 } // namespace LLVM
 } // namespace mlir
diff --git a/mlir/include/mlir/Dialect/LLVMIR/LLVMIntrinsicOps.td b/mlir/include/mlir/Dialect/LLVMIR/LLVMIntrinsicOps.td
index a7d683438ee8a..72fae1bdbf461 100644
--- a/mlir/include/mlir/Dialect/LLVMIR/LLVMIntrinsicOps.td
+++ b/mlir/include/mlir/Dialect/LLVMIR/LLVMIntrinsicOps.td
@@ -107,7 +107,6 @@ def LLVM_IsFPClass : LLVM_OneResultIntrOp<"is.fpclass", [], [0], [Pure],
 }
 
 def LLVM_CopySignOp : LLVM_BinarySameArgsIntrOpF<"copysign">;
-def LLVM_CosOp : LLVM_UnaryIntrOpF<"cos">;
 def LLVM_ExpOp : LLVM_UnaryIntrOpF<"exp">;
 def LLVM_Exp2Op : LLVM_UnaryIntrOpF<"exp2">;
 def LLVM_FAbsOp : LLVM_UnaryIntrOpF<"fabs">;
@@ -125,7 +124,6 @@ def LLVM_Prefetch : LLVM_ZeroResultIntrOp<"prefetch", [0],
 > {
   let arguments = (ins LLVM_AnyPointer:$addr, I32Attr:$rw, I32Attr:$hint, I32Attr:$cache);
 }
-def LLVM_SinOp : LLVM_UnaryIntrOpF<"sin">;
 def LLVM_RoundEvenOp : LLVM_UnaryIntrOpF<"roundeven">;
 def LLVM_RoundOp : LLVM_UnaryIntrOpF<"round">;
 def LLVM_FTruncOp : LLVM_UnaryIntrOpF<"trunc">;
@@ -167,6 +165,11 @@ def LLVM_SMaxOp : LLVM_BinarySameArgsIntrOpI<"smax">;
 def LLVM_SMinOp : LLVM_BinarySameArgsIntrOpI<"smin">;
 def LLVM_UMaxOp : LLVM_BinarySameArgsIntrOpI<"umax">;
 def LLVM_UMinOp : LLVM_BinarySameArgsIntrOpI<"umin">;
+
+def LLVM_SinOp : LLVM_UnaryIntrOpF<"sin">;
+def LLVM_CosOp : LLVM_UnaryIntrOpF<"cos">;
+def LLVM_TanOp : LLVM_UnaryIntrOpF<"tan">;
+
 def LLVM_SinhOp : LLVM_UnaryIntrOpF<"sinh">;
 def LLVM_CoshOp : LLVM_UnaryIntrOpF<"cosh">;
 def LLVM_TanhOp : LLVM_UnaryIntrOpF<"tanh">;
diff --git a/mlir/include/mlir/Dialect/LLVMIR/LLVMOps.td b/mlir/include/mlir/Dialect/LLVMIR/LLVMOps.td
index b2281536aa40b..eed3b2cdb91ea 100644
--- a/mlir/include/mlir/Dialect/LLVMIR/LLVMOps.td
+++ b/mlir/include/mlir/Dialect/LLVMIR/LLVMOps.td
@@ -633,6 +633,8 @@ def LLVM_InvokeOp : LLVM_Op<"invoke", [
                    OptionalAttr<TypeAttrOf<LLVM_FunctionType>>:$var_callee_type,
                    OptionalAttr<FlatSymbolRefAttr>:$callee,
                    Variadic<LLVM_Type>:$callee_operands,
+                   OptionalAttr<DictArrayAttr>:$arg_attrs,
+                   OptionalAttr<DictArrayAttr>:$res_attrs,
                    Variadic<LLVM_Type>:$normalDestOperands,
                    Variadic<LLVM_Type>:$unwindDestOperands,
                    OptionalAttr<DenseI32ArrayAttr>:$branch_weights,
@@ -755,7 +757,9 @@ def LLVM_CallOp : LLVM_MemAccessOpBase<"call",
                   VariadicOfVariadic<LLVM_Type,
                                      "op_bundle_sizes">:$op_bundle_operands,
                   DenseI32ArrayAttr:$op_bundle_sizes,
-                  OptionalAttr<ArrayAttr>:$op_bundle_tags);
+                  OptionalAttr<ArrayAttr>:$op_bundle_tags,
+                  OptionalAttr<DictArrayAttr>:$arg_attrs,
+                  OptionalAttr<DictArrayAttr>:$res_attrs);
   // Append the aliasing related attributes defined in LLVM_MemAccessOpBase.
   let arguments = !con(args, aliasAttrs);
   let results = (outs Optional<LLVM_Type>:$result);
@@ -1163,12 +1167,12 @@ def LLVM_AddressOfOp : LLVM_Op<"mlir.addressof",
   let arguments = (ins FlatSymbolRefAttr:$global_name);
   let results = (outs LLVM_AnyPointer:$res);
 
-  let summary = "Creates a pointer pointing to a global or a function";
+  let summary = "Creates a pointer pointing to a global, alias or a function";
 
   let description = [{
-    Creates an SSA value containing a pointer to a global variable or constant
-    defined by `llvm.mlir.global`. The global value can be defined after its
-    first referenced. If the global value is a constant, storing into it is not
+    Creates an SSA value containing a pointer to a global value (function,
+    variable or alias). The global value can be defined after its first
+    referenced. If the global value is a constant, storing into it is not
     allowed.
 
     Examples:
@@ -1186,10 +1190,19 @@ def LLVM_AddressOfOp : LLVM_Op<"mlir.addressof",
 
       // The function address can be used for indirect calls.
       llvm.call %2() : !llvm.ptr, () -> ()
+
+      // Get the address of an aliased global.
+      %3 = llvm.mlir.addressof @const_alias : !llvm.ptr
     }
 
     // Define the global.
     llvm.mlir.global @const(42 : i32) : i32
+
+    // Define an alias.
+    llvm.mlir.alias @const_alias : i32 {
+      %0 = llvm.mlir.addressof @const : !llvm.ptr
+      llvm.return %0 : !llvm.ptr
+    }
     ```
   }];
 
@@ -1208,6 +1221,14 @@ def LLVM_AddressOfOp : LLVM_Op<"mlir.addressof",
       build($_builder, $_state,
             LLVM::LLVMPointerType::get($_builder.getContext()), func.getName());
       $_state.addAttributes(attrs);
+    }]>,
+    OpBuilder<(ins "AliasOp":$alias,
+      CArg<"ArrayRef<NamedAttribute>", "{}">:$attrs),
+    [{
+      build($_builder, $_state,
+            LLVM::LLVMPointerType::get($_builder.getContext(), alias.getAddrSpace()),
+            alias.getSymName());
+      $_state.addAttributes(attrs);
     }]>
   ];
 
@@ -1218,6 +1239,10 @@ def LLVM_AddressOfOp : LLVM_Op<"mlir.addressof",
 
     /// Return the llvm.func operation that is referenced here.
     LLVMFuncOp getFunction(SymbolTableCollection &symbolTable);
+
+    /// Return the llvm.mlir.alias operation that defined the value referenced
+    /// here.
+    AliasOp getAlias(SymbolTableCollection &symbolTable);
   }];
 
   let assemblyFormat = "$global_name attr-dict `:` qualified(type($res))";
@@ -1443,6 +1468,85 @@ def LLVM_GlobalDtorsOp : LLVM_Op<"mlir.global_dtors", [
   let hasVerifier = 1;
 }
 
+def LLVM_AliasOp : LLVM_Op<"mlir.alias",
+    [IsolatedFromAbove, SingleBlockImplicitTerminator<"ReturnOp">, Symbol]> {
+  let arguments = (ins
+    TypeAttr:$alias_type,
+    StrAttr:$sym_name,
+    Linkage:$linkage,
+    UnitAttr:$dso_local,
+    UnitAttr:$thread_local_,
+    OptionalAttr<UnnamedAddr>:$unnamed_addr,
+    DefaultValuedAttr<Visibility, "mlir::LLVM::Visibility::Default">:$visibility_
+  );
+  let summary = "LLVM dialect alias.";
+  let description = [{
+    `llvm.mlir.alias` is a top level operation that defines a global alias for
+    global variables and functions. The operation is always initialized by
+    using a initializer region which could be a direct map to another global
+    value or contain some address computation on top of it.
+
+    It uses a symbol for its value, which will be uniqued by the module
+    with respect to other symbols in it.
+
+    Similarly to functions and globals, they can also have a linkage attribute.
+    This attribute is placed between `llvm.mlir.alias` and the symbol name. If
+    the attribute is omitted, `external` linkage is assumed by default.
+
+    Examples:
+
+    ```mlir
+    // Global alias use @-identifiers.
+    llvm.mlir.alias external @foo_alias {addr_space = 0 : i32} : !llvm.ptr {
+      %0 = llvm.mlir.addressof @some_function : !llvm.ptr
+      llvm.return %0 : !llvm.ptr
+    }
+
+    // More complex initialization.
+    llvm.mlir.alias linkonce_odr hidden @glob
+    {addr_space = 0 : i32, dso_local} : !llvm.array<32 x i32> {
+      %0 = llvm.mlir.constant(1234 : i64) : i64
+      %1 = llvm.mlir.addressof @glob.private : !llvm.ptr
+      %2 = llvm.ptrtoint %1 : !llvm.ptr to i64
+      %3 = llvm.add %2, %0 : i64
+      %4 = llvm.inttoptr %3 : i64 to !llvm.ptr
+      llvm.return %4 : !llvm.ptr
+    }
+    ```
+  }];
+  let regions = (region SizedRegion<1>:$initializer);
+
+  let builders = [
+    OpBuilder<(ins "Type":$type, "Linkage":$linkage,
+      "StringRef":$name,
+      CArg<"bool", "false">:$dsoLocal,
+      CArg<"bool", "false">:$thread_local_,
+      CArg<"ArrayRef<NamedAttribute>", "{}">:$attrs)>
+  ];
+
+  let extraClassDeclaration = [{
+    /// Return the LLVM type of the global alias.
+    Type getType() {
+      return getAliasType();
+    }
+    /// Return the initializer region. It's always present and terminates
+    /// with an `llvm.return` op with the initializer value.
+    Region &getInitializerRegion() {
+      return getOperation()->getRegion(0);
+    }
+    Block &getInitializerBlock() {
+      return getInitializerRegion().front();
+    }
+    // Retrieve address space information from the initializer block
+    // result.
+    unsigned getAddrSpace();
+  }];
+
+  let hasCustomAssemblyFormat = 1;
+  let hasVerifier = 1;
+  let hasRegionVerifier = 1;
+}
+
 def LLVM_ComdatSelectorOp : LLVM_Op<"comdat_selector", [Symbol]> {
   let arguments = (ins
     SymbolNameAttr:$sym_name,
diff --git a/mlir/include/mlir/Dialect/LLVMIR/LLVMTypes.td b/mlir/include/mlir/Dialect/LLVMIR/LLVMTypes.td
index 77c8035ce3d71..0b9e4b9c55738 100644
--- a/mlir/include/mlir/Dialect/LLVMIR/LLVMTypes.td
+++ b/mlir/include/mlir/Dialect/LLVMIR/LLVMTypes.td
@@ -291,9 +291,9 @@ def LLVMPointerType : LLVMType<"LLVMPointer", "ptr", [
 def LLVMFixedVectorType : LLVMType<"LLVMFixedVector", "vec"> {
   let summary = "LLVM fixed vector type";
   let description = [{
-    LLVM dialect scalable vector type, represents a sequence of elements of
-    unknown length that is known to be divisible by some constant. These
-    elements can be processed as one in SIMD context.
+    LLVM dialect vector type that supports all element types that are supported
+    in LLVM vectors but that are not supported by the builtin MLIR vector type.
+    E.g., LLVMFixedVectorType supports LLVM pointers as element type.
   }];
 
   let typeName = "llvm.fixed_vec";
diff --git a/mlir/include/mlir/Dialect/LLVMIR/NVVMDialect.h b/mlir/include/mlir/Dialect/LLVMIR/NVVMDialect.h
index d474ba8485d5d..a9270c6f52344 100644
--- a/mlir/include/mlir/Dialect/LLVMIR/NVVMDialect.h
+++ b/mlir/include/mlir/Dialect/LLVMIR/NVVMDialect.h
@@ -39,7 +39,11 @@ enum NVVMMemorySpace {
   /// Shared memory space identifier.
   kSharedMemorySpace = 3,
   /// Constant memory space identifier.
-  kConstantMemorySpace = 4
+  kConstantMemorySpace = 4,
+  /// Tensor memory space identifier.
+  /// Tensor memory is available only in arch-accelerated
+  /// variants from sm100 onwards.
+  kTensorMemorySpace = 6
 };
 
 /// Return the element type and number of elements associated with a wmma matrix
diff --git a/mlir/include/mlir/Dialect/LLVMIR/NVVMOps.td b/mlir/include/mlir/Dialect/LLVMIR/NVVMOps.td
index 8c8e44a054a62..fe15a524ec3b5 100644
--- a/mlir/include/mlir/Dialect/LLVMIR/NVVMOps.td
+++ b/mlir/include/mlir/Dialect/LLVMIR/NVVMOps.td
@@ -23,6 +23,7 @@ include "mlir/Interfaces/InferIntRangeInterface.td"
 def LLVM_PointerGeneric : LLVM_PointerInAddressSpace<0>;
 def LLVM_PointerGlobal : LLVM_PointerInAddressSpace<1>;
 def LLVM_PointerShared : LLVM_PointerInAddressSpace<3>;
+def LLVM_PointerTensor : LLVM_PointerInAddressSpace<6>;
 
 //===----------------------------------------------------------------------===//
 // NVVM dialect definitions
@@ -462,7 +463,17 @@ def NVVM_BarrierOp : NVVM_Op<"barrier", [AttrSizedOperandSegments]> {
     }
   }];
   let hasVerifier = 1;
+
   let assemblyFormat = "(`id` `=` $barrierId^)? (`number_of_threads` `=` $numberOfThreads^)? attr-dict";
+
+  let builders = [
+    OpBuilder<(ins), [{
+      return build($_builder, $_state, Value{}, Value{});
+    }]>,
+    OpBuilder<(ins "Value":$barrierId), [{
+      return build($_builder, $_state, barrierId, Value{});
+    }]>
+  ];
 }
 
 def NVVM_BarrierArriveOp : NVVM_PTXBuilder_Op<"barrier.arrive"> 
@@ -476,8 +487,7 @@ def NVVM_BarrierArriveOp : NVVM_PTXBuilder_Op<"barrier.arrive">
     The default barrier id is 0 that is similar to `nvvm.barrier` Op. When 
     `barrierId` is not present, the default barrier id is used. 
 
-    [For more information, see PTX ISA]
-    (https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-bar)
+    [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-bar)
   }];
   
   let assemblyFormat = "(`id` `=` $barrierId^)? `number_of_threads` `=` $numberOfThreads attr-dict";
@@ -503,8 +513,7 @@ def NVVM_ClusterArriveOp : NVVM_Op<"cluster.arrive"> {
 
     The `aligned` attribute, when provided, generates the .aligned version of the PTX instruction.
 
-    [For more information, see PTX ISA]
-    (https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-barrier-cluster)
+    [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-barrier-cluster)
   }];
 
   string llvmBuilder = [{
@@ -530,8 +539,7 @@ def NVVM_ClusterArriveRelaxedOp : NVVM_Op<"cluster.arrive.relaxed"> {
     ordering and visibility guarantees provided for the memory accesses performed prior to
     `cluster.arrive`.
 
-    [For more information, see PTX ISA]
-    (https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-barrier-cluster)
+    [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-barrier-cluster)
   }];
 
   string llvmBuilder = [{
@@ -552,8 +560,7 @@ def NVVM_ClusterWaitOp : NVVM_Op<"cluster.wait"> {
     of the cluster to perform `cluster.arrive`. The `aligned` attribute, when provided,
     generates the .aligned version of the PTX instruction.
 
-    [For more information, see PTX ISA]
-    (https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-barrier-cluster)
+    [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-barrier-cluster)
   }];
 
   string llvmBuilder = [{
@@ -605,8 +612,8 @@ def NVVM_FenceProxyOp : NVVM_PTXBuilder_Op<"fence.proxy">,
   let description = [{
     Fence operation with proxy to establish an ordering between memory accesses
     that may happen through different proxies.
-    [For more information, see PTX ISA]
-    (https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-membar)
+
+    [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-membar)
   }];
   
   let assemblyFormat = "attr-dict";
@@ -656,8 +663,8 @@ def NVVM_FenceProxyAcquireOp : NVVM_Op<"fence.proxy.acquire">,
     value for the `size` operand is 128 and must be an immediate. Generic Addressing
     is used unconditionally, and the address specified by the operand `addr` must
     fall within the `.global` state space. Otherwise, the behavior is undefined
-    [For more information, see PTX ISA]
-    (https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-membar)
+    
+    [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-membar)
   }];
 
   let assemblyFormat = "$scope $addr `,` $size (`from_proxy` `=` $fromProxy^)? (`to_proxy` `=` $toProxy^)? attr-dict";
@@ -684,8 +691,8 @@ def NVVM_FenceProxyReleaseOp : NVVM_Op<"fence.proxy.release">,
     subsequent memory access performed via the tensormap proxy. `fence.proxy.release`
     operation can form a release sequence that synchronizes with an acquire
     sequence that contains the fence.proxy.acquire proxy fence operation
-    [For more information, see PTX ISA]
-    (https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-membar)
+    
+    [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-membar)
   }];
 
   let assemblyFormat = "$scope (`from_proxy` `=` $fromProxy^)? (`to_proxy` `=` $toProxy^)? attr-dict";
@@ -723,8 +730,8 @@ def NVVM_FenceMbarrierInitOp : NVVM_PTXBuilder_Op<"fence.mbarrier.init"> {
   let arguments = (ins );
     let description = [{
     Fence operation that applies on the prior nvvm.mbarrier.init
-    [For more information, see PTX ISA]
-    (https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-membar)
+    
+    [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-membar)
   }];
   
   let assemblyFormat = "attr-dict";
@@ -767,8 +774,8 @@ def NVVM_ShflOp :
     the source. The `mask_and_clamp` contains two packed values specifying
     a mask for logically splitting warps into sub-segments and an upper bound
     for clamping the source lane index.
-    [For more information, refer PTX ISA]
-    (https://docs.nvidia.com/cuda/parallel-thread-execution/#data-movement-and-conversion-instructions-shfl-sync)
+    
+    [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/#data-movement-and-conversion-instructions-shfl-sync)
   }];
   string llvmBuilder = [{
       auto intId = getShflIntrinsicId(
@@ -813,8 +820,7 @@ def NVVM_ElectSyncOp : NVVM_Op<"elect.sync">
     of this Op. The predicate result is set to `True` for the
     leader thread, and `False` for all other threads.
 
-    [For more information, see PTX ISA]
-    (https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-elect-sync)
+    [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-elect-sync)
   }];
 
   let results = (outs I1:$pred);
@@ -898,8 +904,8 @@ def NVVM_CpAsyncMBarrierArriveOp : NVVM_Op<"cp.async.mbarrier.arrive"> {
     The `addr` operand specifies the address of the mbarrier object
     in generic address space. The `noinc` attr impacts how the
     mbarrier's state is updated.
-    [For more information, refer PTX ISA]
-    (https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-cp-async-mbarrier-arrive)
+    
+    [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-cp-async-mbarrier-arrive)
   }];
   let assemblyFormat = "$addr attr-dict `:` type(operands)";
 
@@ -922,8 +928,9 @@ def NVVM_CpAsyncMBarrierArriveSharedOp : NVVM_Op<"cp.async.mbarrier.arrive.share
     track all prior cp.async operations initiated by the executing thread.
     The `addr` operand specifies the address of the mbarrier object in
     shared memory. The `noinc` attr impacts how the mbarrier's state
-    is updated. [For more information, refer PTX ISA]
-    (https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-cp-async-mbarrier-arrive)
+    is updated. 
+    
+    [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-cp-async-mbarrier-arrive)
   }];
   let assemblyFormat = "$addr attr-dict `:` type(operands)";
 
@@ -981,8 +988,8 @@ def NVVM_CvtFloatToTF32Op : NVVM_Op<"cvt.float.to.tf32"> {
     The `relu` attribute, when set, lowers to the '.relu' variant of
     the cvt instruction. The `rnd` and `sat` attributes specify the
     the rounding and saturation modes respectively.
-    [For more information, see PTX ISA]
-    (https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cvt)
+    
+    [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cvt)
   }];
 
   let hasVerifier = 1;
@@ -1632,8 +1639,8 @@ def NVVM_StMatrixOp: NVVM_PTXBuilder_Op<"stmatrix">,
   let description = [{
     Collectively store one or more matrices across all threads in a warp to the
     location indicated by the address operand $ptr in shared memory.
-    [For more information, see PTX ISA]
-    (https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#warp-level-matrix-store-instruction-stmatrix)
+    
+    [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#warp-level-matrix-store-instruction-stmatrix)
   }];
   
   let assemblyFormat = "$ptr `,` $sources attr-dict `:` type(operands)";
@@ -1845,8 +1852,7 @@ def NVVM_CpAsyncBulkCommitGroupOp : NVVM_Op<"cp.async.bulk.commit.group">,
     This Op commits all prior initiated but uncommitted cp.async.bulk
     instructions into a cp.async.bulk-group.
 
-    [For more information, see PTX ISA]
-    (https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cp-async-bulk-commit-group)
+    [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cp-async-bulk-commit-group)
   }];
 
   string llvmBuilder = [{
@@ -1870,8 +1876,7 @@ def NVVM_CpAsyncBulkWaitGroupOp : NVVM_Op<"cp.async.bulk.wait_group">,
     async operations in the specified bulk async-group have completed reading 
     from their source locations.
 
-    [For more information, see PTX ISA]
-    (https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cp-async-bulk-wait-group)
+    [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cp-async-bulk-wait-group)
   }];
   
   string llvmBuilder = [{
@@ -1916,8 +1921,7 @@ def NVVM_CpAsyncBulkTensorGlobalToSharedClusterOp :
     The `l2CacheHint` operand is optional, and it is used to specify cache 
     eviction policy that may be used during the memory access.
     
-    [For more information, see PTX ISA]
-    (https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cp-async-bulk-tensor)
+    [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cp-async-bulk-tensor)
   }];
 
   let assemblyFormat = [{ 
@@ -2033,8 +2037,7 @@ def NVVM_CpAsyncBulkTensorPrefetchOp :
     The `l2CacheHint` operand is optional, and it is used to specify cache
     eviction policy that may be used during the memory access.
 
-    [For more information, see PTX ISA]
-    (https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cp-async-bulk-prefetch-tensor)
+    [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cp-async-bulk-prefetch-tensor)
   }];
 
   let assemblyFormat = [{
@@ -2133,8 +2136,7 @@ def NVVM_CpAsyncBulkTensorReduceOp :
     The `l2CacheHint` operand is optional, and it is used to specify cache
     eviction policy that may be used during the memory access.
 
-    [For more information, see PTX ISA]
-    (https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cp-reduce-async-bulk-tensor)
+    [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cp-reduce-async-bulk-tensor)
   }];
 
   let assemblyFormat = [{
@@ -2193,8 +2195,8 @@ def NVVM_CpAsyncBulkGlobalToSharedClusterOp :
 
     The `l2CacheHint` operand is optional, and it is used to specify cache
     eviction policy that may be used during the memory access.
-    [For more information, see PTX ISA]
-    (https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cp-async-bulk)
+    
+    [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cp-async-bulk)
   }];
 
   let arguments = (ins
@@ -2251,8 +2253,7 @@ def NVVM_CpAsyncBulkSharedCTAToSharedClusterOp :
     Initiates an asynchronous copy operation from Shared CTA memory to Shared
     cluster memory.
 
-    [For more information, see PTX ISA]
-    (https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cp-async-bulk)
+    [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cp-async-bulk)
   }];
 
   let arguments = (ins
@@ -2282,8 +2283,8 @@ def NVVM_CpAsyncBulkSharedCTAToGlobalOp :
 
     The `l2CacheHint` operand is optional, and it is used to specify cache
     eviction policy that may be used during the memory access.
-    [For more information, see PTX ISA]
-    (https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cp-async-bulk)
+    
+    [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cp-async-bulk)
   }];
 
   let arguments = (ins
@@ -2512,6 +2513,59 @@ def NVVM_WgmmaMmaAsyncOp : NVVM_Op<"wgmma.mma_async",
   }];
 }
 
+//===----------------------------------------------------------------------===//
+// NVVM Griddepcontrol Ops
+//===----------------------------------------------------------------------===//
+
+def NVVM_GriddepcontrolWaitOp : NVVM_IntrOp<"griddepcontrol.wait", [], 0> {
+  let assemblyFormat = "attr-dict";
+
+  let description = [{
+    Causes the executing thread to wait until all prerequisite grids in flight 
+    have completed and all the memory operations from the prerequisite grids 
+    are performed and made visible to the current grid.
+
+
+    [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/#parallel-synchronization-and-communication-instructions-griddepcontrol)
+  }];
+}
+
+def NVVM_GriddepcontrolLaunchDependentsOp
+    : NVVM_IntrOp<"griddepcontrol.launch.dependents", [], 0> {
+  let assemblyFormat = "attr-dict";
+
+  let description = [{
+    Signals that specific dependents the runtime system designated to react to 
+    this instruction can be scheduled as soon as all other CTAs in the grid 
+    issue the same instruction or have completed.
+
+
+    [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/#parallel-synchronization-and-communication-instructions-griddepcontrol)
+  }];
+}
+
+//===----------------------------------------------------------------------===//
+// NVVM Mapa Op
+//===----------------------------------------------------------------------===//
+
+def NVVM_MapaOp: NVVM_Op<"mapa",
+    [TypesMatchWith<"`res` and `a` should have the same type",
+                    "a", "res", "$_self">]> {
+  let results = (outs AnyTypeOf<[LLVM_PointerGeneric, LLVM_PointerShared]>:$res);
+  let arguments = (ins AnyTypeOf<[LLVM_PointerGeneric, LLVM_PointerShared]>:$a, I32:$b);
+
+  string llvmBuilder = [{
+    int addrSpace = llvm::cast<LLVMPointerType>(op.getA().getType()).getAddressSpace();
+    
+    bool isSharedMemory = addrSpace == NVVM::NVVMMemorySpace::kSharedMemorySpace;
+
+    auto intId = isSharedMemory? llvm::Intrinsic::nvvm_mapa_shared_cluster : llvm::Intrinsic::nvvm_mapa;
+    $res = createIntrinsicCall(builder, intId, {$a, $b});
+  }];
+  
+  let assemblyFormat = "$a`,` $b attr-dict `:` type($a) `->` type($res)";
+}
+
 def NVVM_Exit : NVVM_Op<"exit"> {
   let summary = "Exit Op";
   let description = [{
@@ -2543,6 +2597,219 @@ def NVVM_Breakpoint : NVVM_Op<"breakpoint"> {
   let assemblyFormat = "attr-dict";
 }
 
+//===----------------------------------------------------------------------===//
+// NVVM TCGEN05 Ops
+//===----------------------------------------------------------------------===//
+// Num CTAs in a group participating in the TCGEN05 operation.
+// This corresponds to the "cta_group::1", "cta_group::2"
+// modifiers in the PTX instructions.
+def Tcgen05GroupCTA_1 : I32EnumAttrCase<"CTA_1", 0, "cta_1">;
+def Tcgen05GroupCTA_2 : I32EnumAttrCase<"CTA_2", 1, "cta_2">;
+
+def Tcgen05GroupKind : I32EnumAttr<"Tcgen05GroupKind",
+                            "NVVM Tcgen05 group kind",
+  [Tcgen05GroupCTA_1, Tcgen05GroupCTA_2]> {
+  let genSpecializedAttr = 0;
+  let cppNamespace = "::mlir::NVVM";
+}
+def Tcgen05GroupKindAttr :
+  EnumAttr<NVVM_Dialect, Tcgen05GroupKind, "tcgen05_group"> {
+  let assemblyFormat = "`<` $value `>`";
+}
+
+def Tcgen05FenceBefore : I32EnumAttrCase<"BEFORE_THREAD_SYNC", 0, "before">;
+def Tcgen05FenceAfter  : I32EnumAttrCase<"AFTER_THREAD_SYNC",  1, "after">;
+def Tcgen05FenceKind : I32EnumAttr<"Tcgen05FenceKind", "NVVM Tcgen05 fence kind",
+  [Tcgen05FenceBefore, Tcgen05FenceAfter]> {
+  let genSpecializedAttr = 0;
+  let cppNamespace = "::mlir::NVVM";
+}
+def Tcgen05FenceKindAttr :
+  EnumAttr<NVVM_Dialect, Tcgen05FenceKind, "tcgen05_fence"> {
+  let assemblyFormat = "`<` $value `>`";
+}
+
+def Tcgen05WaitLoad  : I32EnumAttrCase<"LOAD",  0, "load">;
+def Tcgen05WaitStore : I32EnumAttrCase<"STORE", 1, "store">;
+def Tcgen05WaitKind : I32EnumAttr<"Tcgen05WaitKind", "NVVM Tcgen05 wait kind",
+  [Tcgen05WaitLoad, Tcgen05WaitStore]> {
+  let genSpecializedAttr = 0;
+  let cppNamespace = "::mlir::NVVM";
+}
+def Tcgen05WaitKindAttr :
+  EnumAttr<NVVM_Dialect, Tcgen05WaitKind, "tcgen05_wait"> {
+  let assemblyFormat = "`<` $value `>`";
+}
+
+def NVVM_Tcgen05AllocOp : NVVM_Op<"tcgen05.alloc"> {
+  let summary = "Tcgen05 alloc operation";
+  let description = [{
+    The `tcgen05.alloc` Op allocates tensor core memory for
+    the amount specified by `nCols` and writes the destination
+    address to the `addr` argument. The `nCols` operand specifies the
+    number of columns to be allocated and it must be a power-of-two.
+    [For more information, refer to the PTX ISA]
+    (https://docs.nvidia.com/cuda/parallel-thread-execution/#tcgen05-memory-alloc-manage-instructions)
+  }];
+
+  let arguments = (ins
+    AnyTypeOf<[LLVM_AnyPointer, LLVM_PointerShared]>:$addr,
+    I32:$nCols,
+    DefaultValuedAttr<Tcgen05GroupKindAttr, "Tcgen05GroupKind::CTA_1">:$group);
+
+  let assemblyFormat = "$addr `,` $nCols attr-dict `:` type(operands)";
+
+  let extraClassDeclaration = [{
+    static llvm::Intrinsic::ID
+      getIntrinsicIDAndArgs(Operation &op, LLVM::ModuleTranslation &mt,
+                            llvm::SmallVector<llvm::Value *> &args);
+  }];
+  string llvmBuilder = [{
+    llvm::SmallVector<llvm::Value *> args;
+    auto id = NVVM::Tcgen05AllocOp::getIntrinsicIDAndArgs(
+      *op, moduleTranslation, args);
+    createIntrinsicCall(builder, id, args);
+  }];
+}
+
+def NVVM_Tcgen05DeallocOp : NVVM_Op<"tcgen05.dealloc"> {
+  let summary = "Tcgen05 dealloc operation";
+  let description = [{
+    The `tcgen05.dealloc` Op de-allocates the tensor core memory
+    specified by `tmemAddr`, which must be from a previous tensor
+    memory allocation. The `nCols` operand specifies the number
+    of columns to be de-allocated, and it must be a power-of-two.
+    [For more information, refer to the PTX ISA]
+    (https://docs.nvidia.com/cuda/parallel-thread-execution/#tcgen05-memory-alloc-manage-instructions)
+  }];
+
+  let arguments = (ins LLVM_PointerTensor:$taddr, I32:$nCols,
+    DefaultValuedAttr<Tcgen05GroupKindAttr, "Tcgen05GroupKind::CTA_1">:$group);
+
+  let assemblyFormat = "$taddr `,` $nCols attr-dict `:` type(operands)";
+
+  let extraClassDeclaration = [{
+    static llvm::Intrinsic::ID
+      getIntrinsicIDAndArgs(Operation &op, LLVM::ModuleTranslation &mt,
+                            llvm::SmallVector<llvm::Value *> &args);
+  }];
+  string llvmBuilder = [{
+    llvm::SmallVector<llvm::Value *> args;
+    auto id = NVVM::Tcgen05DeallocOp::getIntrinsicIDAndArgs(
+      *op, moduleTranslation, args);
+    createIntrinsicCall(builder, id, args);
+  }];
+}
+
+def NVVM_Tcgen05RelinquishAllocPermitOp : NVVM_Op<"tcgen05.relinquish_alloc_permit"> {
+  let summary = "Tcgen05 Op to relinquish the right to allocate";
+  let description = [{
+    The `tcgen05.relinquish_alloc_permit` Op specifies that the CTA
+    of the executing thread is relinquishing the right to allocate
+    Tensor Memory. So, it is illegal for a CTA to perform `tcgen05.alloc`
+    after any of its constituent threads execute `tcgen05.relinquish_alloc_permit`.
+    [For more information, refer to the PTX ISA]
+    (https://docs.nvidia.com/cuda/parallel-thread-execution/#tcgen05-memory-alloc-manage-instructions)
+  }];
+
+  let arguments = (ins
+    DefaultValuedAttr<Tcgen05GroupKindAttr, "Tcgen05GroupKind::CTA_1">:$group);
+
+  let assemblyFormat = "attr-dict";
+
+  string llvmBuilder = [{
+    auto id = ($group == NVVM::Tcgen05GroupKind::CTA_1) ?
+      llvm::Intrinsic::nvvm_tcgen05_relinq_alloc_permit_cg1 :
+      llvm::Intrinsic::nvvm_tcgen05_relinq_alloc_permit_cg2;
+    createIntrinsicCall(builder, id);
+  }];
+}
+
+def NVVM_Tcgen05FenceOp : NVVM_Op<"tcgen05.fence"> {
+  let summary = "Tcgen05 fence operations";
+  let description = [{
+    The `tcgen05.fence<before>` orders all prior async tcgen05 operations
+    with respect to the subsequent tcgen05 and execution ordering operations.
+    The `tcgen05.fence<after>` orders all subsequent async tcgen05 operations
+    with respect to the prior tcgen05 and execution ordering operations.
+
+    [For more information refer to the PTX ISA]
+    (https://docs.nvidia.com/cuda/parallel-thread-execution/#tensorcore-5th-generation-instructions-tcgen05-fence)
+  }];
+
+  let arguments = (ins Tcgen05FenceKindAttr:$kind);
+  let assemblyFormat = "$kind attr-dict";
+
+  string llvmBuilder = [{
+    auto id = ($kind == NVVM::Tcgen05FenceKind::BEFORE_THREAD_SYNC)
+      ? llvm::Intrinsic::nvvm_tcgen05_fence_before_thread_sync
+      : llvm::Intrinsic::nvvm_tcgen05_fence_after_thread_sync;
+    createIntrinsicCall(builder, id);
+  }];
+}
+
+def NVVM_Tcgen05WaitOp : NVVM_Op<"tcgen05.wait"> {
+  let summary = "Tcgen05 wait operations";
+  let description = [{
+    The `tcgen05.wait<load>` causes the executing thread to block until
+    all prior `tcgen05.ld` operations issued by the executing thread
+    have completed. Similarly, the `tcgen05.wait<store>` causes the executing
+    thread to block until all prior `tcgen05.st` operations issued by the
+    executing thread have completed.
+    [For more information refer PTX ISA]
+    (https://docs.nvidia.com/cuda/parallel-thread-execution/#tcgen05-instructions-tcgen05-wait)
+  }];
+
+  let arguments = (ins Tcgen05WaitKindAttr:$kind);
+  let assemblyFormat = "$kind attr-dict";
+
+  string llvmBuilder = [{
+    auto id = ($kind == NVVM::Tcgen05WaitKind::LOAD)
+      ? llvm::Intrinsic::nvvm_tcgen05_wait_ld
+      : llvm::Intrinsic::nvvm_tcgen05_wait_st;
+    createIntrinsicCall(builder, id);
+  }];
+}
+
+def NVVM_Tcgen05CommitOp : NVVM_Op<"tcgen05.commit"> {
+  let summary = "Tcgen05 commit operations";
+  let description = [{
+    The `tcgen05.commit` makes the mbarrier object, specified by
+    the operand `addr`, track the completion of all the prior
+    async-tcgen05 operations initiated by the executing thread.
+    The multicast variants allow signaling on the mbarrier objects
+    of multiple CTAs within the cluster. Operand `multicastMask`,
+    when present, specifies the destination CTAs in the cluster such
+    that each bit position in the 16-bit `multicastMask` operand
+    corresponds to the `nvvm.read.ptx.sreg.ctaid` of the destination CTA.
+    [For more information refer PTX ISA]
+    (https://docs.nvidia.com/cuda/parallel-thread-execution/#tcgen-async-sync-operations-commit)
+  }];
+
+  let arguments = (ins
+    AnyTypeOf<[LLVM_AnyPointer, LLVM_PointerShared]>:$addr,
+    Optional<I16>:$multicastMask,
+    DefaultValuedAttr<Tcgen05GroupKindAttr, "Tcgen05GroupKind::CTA_1">:$group);
+
+  let assemblyFormat = [{
+    $addr (`,` `multicast_mask` `=` $multicastMask^)?
+    attr-dict `:` type(operands)
+  }];
+
+  let extraClassDeclaration = [{
+    static llvm::Intrinsic::ID
+      getIntrinsicIDAndArgs(Operation &op, LLVM::ModuleTranslation &mt,
+                            llvm::SmallVector<llvm::Value *> &args);
+  }];
+
+  string llvmBuilder = [{
+    llvm::SmallVector<llvm::Value *> args;
+    auto id = NVVM::Tcgen05CommitOp::getIntrinsicIDAndArgs(
+      *op, moduleTranslation, args);
+    createIntrinsicCall(builder, id, args);
+  }];
+}
+
 //===----------------------------------------------------------------------===//
 // NVVM target attribute.
 //===----------------------------------------------------------------------===//
diff --git a/mlir/include/mlir/Dialect/LLVMIR/ROCDLOps.td b/mlir/include/mlir/Dialect/LLVMIR/ROCDLOps.td
index 95fbe7ed66a43..7efa4ffa2aa6f 100644
--- a/mlir/include/mlir/Dialect/LLVMIR/ROCDLOps.td
+++ b/mlir/include/mlir/Dialect/LLVMIR/ROCDLOps.td
@@ -100,6 +100,13 @@ class ROCDL_IntrOp<string mnemonic, list<int> overloadedResults,
     overloadedOperands, traits, numResults, requiresAccessGroup,
     requiresAliasAnalysis, 0, 0, immArgPositions, immArgAttrNames>;
 
+// Subclass to save typing and ease readibility when there aren't overloaded
+// operands or memory accesses.
+class ROCDL_ConcreteNonMemIntrOp<string mnemonic, list<Trait> traits,
+    int numResults, list<int> immArgPositions = [],
+    list<string> immArgNames = []>
+  : ROCDL_IntrOp<mnemonic, [], [], traits, numResults, 0, 0,
+      immArgPositions, immArgNames>;
 //===----------------------------------------------------------------------===//
 // ROCDL special register op definitions
 //===----------------------------------------------------------------------===//
@@ -150,37 +157,26 @@ class ROCDL_MbcntOp<string mnemonic> :
 def ROCDL_MbcntLoOp : ROCDL_MbcntOp<"lo">;
 def ROCDL_MbcntHiOp : ROCDL_MbcntOp<"hi">;
 
-def ROCDL_DsSwizzleOp :
-ROCDL_Op<"ds_swizzle">,
-Results<(outs I32:$res)>,
-Arguments<(ins I32:$src,
-               I32:$offset)>
-{
-  string llvmBuilder = [{
-    $res = createIntrinsicCall(builder, llvm::Intrinsic::amdgcn_ds_swizzle, {$src, $offset});
-  }];
+def ROCDL_DsSwizzleOp : ROCDL_ConcreteNonMemIntrOp<"ds_swizzle", [], 1>,
+  Arguments<(ins I32:$src,
+                 I32:$offset)> {
+  let results = (outs I32:$res);
   let assemblyFormat = [{
     $src `,` $offset  attr-dict `:` `(` type($src) `,` type($offset) `)` `->` type($res)
    }];
 }
 
-def ROCDL_DsBpermuteOp :
-ROCDL_Op<"ds_bpermute">,
-Results<(outs I32:$res)>,
-Arguments<(ins I32:$index,
-               I32:$src)>
-{
-  string llvmBuilder = [{
-    $res = createIntrinsicCall(builder, llvm::Intrinsic::amdgcn_ds_bpermute, {$index, $src});
-  }];
+def ROCDL_DsBpermuteOp : ROCDL_ConcreteNonMemIntrOp<"ds_bpermute", [], 1>,
+  Arguments<(ins I32:$index,
+                 I32:$src)> {
+  let results = (outs I32:$res);
   let assemblyFormat = [{
     $index `,` $src  attr-dict `:` `(` type($index) `,` type($src) `)` `->` type($res)
    }];
 }
 
 def ROCDL_BallotOp :
-  ROCDL_Op<"ballot">,
-  Results<(outs LLVM_Type:$res)>,
+  ROCDL_IntrOp<"ballot", [0], [], [], 1>,
   Arguments<(ins I1:$pred)> {
   let summary = "Vote across thread group";
 
@@ -189,11 +185,6 @@ def ROCDL_BallotOp :
       The nth bit of the result contains the 1 bit contributed by the nth warp lane.
   }];
 
-  string llvmBuilder = [{
-      $res = createIntrinsicCall(builder,
-            llvm::Intrinsic::amdgcn_ballot, {$pred}, {$_resultType});
-  }];
-
   let assemblyFormat = "$pred attr-dict `:` type($res)";
 }
 
@@ -249,18 +240,12 @@ def ROCDL_GridDimZOp : ROCDL_DimGetterFunctionOp<"grid.dim.z",
 
 // Emits the waintcnt instruction. The bitfield's semantics depend
 // on the target chipset
-def ROCDL_WaitcntOp : ROCDL_Op<"waitcnt">, Arguments<(ins I32Attr:$bitfield)> {
-  string llvmBuilder = [{
-    createIntrinsicCall(builder, llvm::Intrinsic::amdgcn_s_waitcnt,
-      {builder.getInt32($bitfield)});
-  }];
+def ROCDL_SWaitcntOp : ROCDL_ConcreteNonMemIntrOp<"s.waitcnt", [], 0, [0], ["bitfield"]>,
+  Arguments<(ins I32Attr:$bitfield)> {
   let assemblyFormat = "attr-dict $bitfield";
 }
 
-def ROCDL_SBarrierOp : ROCDL_Op<"s.barrier"> {
-  string llvmBuilder = [{
-    createIntrinsicCall(builder, llvm::Intrinsic::amdgcn_s_barrier);
-  }];
+def ROCDL_SBarrierOp : ROCDL_ConcreteNonMemIntrOp<"s.barrier", [], 0> {
   let assemblyFormat = "attr-dict";
 }
 
@@ -276,68 +261,51 @@ def ROCDL_BarrierOp : ROCDL_Op<"barrier"> {
   let assemblyFormat = "attr-dict";
 }
 
-def ROCDL_BarrierSignalOp : ROCDL_IntrOp<"s.barrier.signal", [], [], [], 0, 0, 0, [0], ["id"]>,
+def ROCDL_BarrierSignalOp : ROCDL_ConcreteNonMemIntrOp<"s.barrier.signal", [], 0, [0], ["id"]>,
   Arguments<(ins I32Attr:$id)> {
   let results = (outs);
   let assemblyFormat = "$id attr-dict";
 }
 
-def ROCDL_BarrierWaitOp : ROCDL_IntrOp<"s.barrier.wait", [], [], [], 0, 0, 0, [0], ["id"]>,
+def ROCDL_BarrierWaitOp : ROCDL_ConcreteNonMemIntrOp<"s.barrier.wait", [], 0, [0], ["id"]>,
   Arguments<(ins I16Attr:$id)> {
   let results = (outs);
   let assemblyFormat = "$id attr-dict";
-  string llvmBuilder =
-    "createIntrinsicCall(builder, llvm::Intrinsic::amdgcn_s_barrier_wait,builder.getInt16(op.getId()));";
 }
 
-def ROCDL_WaitDscntOp: ROCDL_IntrOp<"s.wait.dscnt", [], [], [], 0, 0, 0, [0], ["id"]>,
+def ROCDL_WaitDscntOp: ROCDL_ConcreteNonMemIntrOp<"s.wait.dscnt", [], 0, [0], ["id"]>,
   Arguments<(ins I16Attr:$id)> {
   let results = (outs);
   let assemblyFormat = "$id attr-dict";
 }
 
-def ROCDL_SetPrioOp : ROCDL_IntrOp<"s.setprio", [], [], [], 0>,
+def ROCDL_SetPrioOp : ROCDL_ConcreteNonMemIntrOp<"s.setprio", [], 0, [0], ["priority"]>,
   Arguments<(ins I16Attr:$priority)> {
-  let results = (outs);
   let assemblyFormat = "$priority attr-dict";
-  string llvmBuilder =
-    "createIntrinsicCall(builder, llvm::Intrinsic::amdgcn_s_setprio,builder.getInt16(op.getPriority()));";
 }
 
-def ROCDL_SchedBarrier : ROCDL_IntrOp<"sched.barrier", [], [], [], 0>,
+def ROCDL_SchedBarrier : ROCDL_ConcreteNonMemIntrOp<"sched.barrier", [], 0, [0],["mask"]>,
   Arguments<(ins I32Attr:$mask)> {
-  let results = (outs);
   let assemblyFormat = "$mask attr-dict";
-  string llvmBuilder =
-    "createIntrinsicCall(builder, llvm::Intrinsic::amdgcn_sched_barrier,builder.getInt32(op.getMask()));";
 }
 
-def ROCDL_SchedGroupBarrier : ROCDL_IntrOp<"sched.group.barrier", [], [], [], 0>,
-  Arguments<(ins I32Attr:$mask, I32Attr:$size, I32Attr:$groupId)> {
-  let results = (outs);
+def ROCDL_SchedGroupBarrier
+  : ROCDL_ConcreteNonMemIntrOp<"sched.group.barrier", [], 0,
+      [0, 1, 2], ["mask", "size", "groupId"]>,
+    Arguments<(ins I32Attr:$mask, I32Attr:$size, I32Attr:$groupId)> {
   let assemblyFormat = "$mask `,` $size `,` $groupId attr-dict";
-  string llvmBuilder = [{
-    createIntrinsicCall(builder,
-      llvm::Intrinsic::amdgcn_sched_group_barrier,
-      {builder.getInt32(op.getMask()), builder.getInt32(op.getSize()), builder.getInt32(op.getGroupId())});
-  }];
 }
 
-def ROCDL_IglpOpt : ROCDL_IntrOp<"iglp.opt", [], [], [], 0>,
+def ROCDL_IglpOpt : ROCDL_ConcreteNonMemIntrOp<"iglp.opt", [], 0, [0], ["variant"]>,
   Arguments<(ins I32Attr:$variant)> {
-  let results = (outs);
   let assemblyFormat = "$variant attr-dict";
-  string llvmBuilder =
-    "createIntrinsicCall(builder, llvm::Intrinsic::amdgcn_iglp_opt,builder.getInt32(op.getVariant()));";
 }
 
 //===---------------------------------------------------------------------===//
 // Xdlops intrinsics
 
 class ROCDL_Mfma_IntrOp<string mnemonic, list<Trait> traits = []> :
-  LLVM_IntrOpBase<ROCDL_Dialect, mnemonic,
-                  "amdgcn_" # !subst(".","_", mnemonic),
-                  [], [], traits, 1>,
+  ROCDL_IntrOp<mnemonic, [], [], traits, 1>,
   Arguments<(ins Variadic<LLVM_Type>:$args)> {
   let assemblyFormat =
     "$args attr-dict `:` functional-type($args, $res)";
@@ -347,9 +315,7 @@ class ROCDL_Mfma_IntrOp<string mnemonic, list<Trait> traits = []> :
 // MFMA intrinsics with overloaded operands
 class ROCDL_Mfma_OO_IntrOp<string mnemonic, list<int> overloadedOperands,
                         list<Trait> traits = []> :
-  LLVM_IntrOpBase<ROCDL_Dialect, mnemonic,
-                  "amdgcn_" # !subst(".","_", mnemonic),
-                  [], overloadedOperands, traits, 1>,
+  ROCDL_IntrOp<mnemonic, [], overloadedOperands, traits, 1>,
   Arguments<(ins Variadic<LLVM_Type>:$args)> {
   let assemblyFormat =
     "$args attr-dict `:` functional-type($args, $res)";
@@ -408,13 +374,29 @@ def ROCDL_mfma_i32_32x32x32_i8 : ROCDL_Mfma_IntrOp<"mfma.i32.32x32x32.i8">;
 def ROCDL_mfma_f32_32x32x16_f16 : ROCDL_Mfma_IntrOp<"mfma.f32.32x32x16.f16">;
 def ROCDL_mfma_scale_f32_16x16x128_f8f6f4 : ROCDL_Mfma_OO_IntrOp<"mfma.scale.f32.16x16x128.f8f6f4", [0,1]>;
 def ROCDL_mfma_scale_f32_32x32x64_f8f6f4 : ROCDL_Mfma_OO_IntrOp<"mfma.scale.f32.32x32x64.f8f6f4", [0,1]>;
+
+// 2:4 Sparsity ops (GFX940)
+def ROCDL_smfmac_f32_16x16x32_f16 : ROCDL_Mfma_IntrOp<"smfmac.f32.16x16x32.f16">;
+def ROCDL_smfmac_f32_32x32x16_f16 : ROCDL_Mfma_IntrOp<"smfmac.f32.32x32x16.f16">;
+def ROCDL_smfmac_f32_16x16x32_bf16 : ROCDL_Mfma_IntrOp<"smfmac.f32.16x16x32.bf16">;
+def ROCDL_smfmac_f32_32x32x16_bf16 : ROCDL_Mfma_IntrOp<"smfmac.f32.32x32x16.bf16">;
+def ROCDL_smfmac_i32_16x16x64_i8 : ROCDL_Mfma_IntrOp<"smfmac.i32.16x16x64.i8">;
+def ROCDL_smfmac_i32_32x32x32_i8 : ROCDL_Mfma_IntrOp<"smfmac.i32.32x32x32.i8">;
+def ROCDL_smfmac_f32_16x16x64_bf8_bf8 : ROCDL_Mfma_IntrOp<"smfmac.f32.16x16x64.bf8.bf8">;
+def ROCDL_smfmac_f32_16x16x64_bf8_fp8 : ROCDL_Mfma_IntrOp<"smfmac.f32.16x16x64.bf8.fp8">;
+def ROCDL_smfmac_f32_16x16x64_fp8_bf8 : ROCDL_Mfma_IntrOp<"smfmac.f32.16x16x64.fp8.bf8">;
+def ROCDL_smfmac_f32_16x16x64_fp8_fp8 : ROCDL_Mfma_IntrOp<"smfmac.f32.16x16x64.fp8.fp8">;
+def ROCDL_smfmac_f32_32x32x32_bf8_bf8 : ROCDL_Mfma_IntrOp<"smfmac.f32.32x32x32.bf8.bf8">;
+def ROCDL_smfmac_f32_32x32x32_bf8_fp8 : ROCDL_Mfma_IntrOp<"smfmac.f32.32x32x32.bf8.fp8">;
+def ROCDL_smfmac_f32_32x32x32_fp8_bf8 : ROCDL_Mfma_IntrOp<"smfmac.f32.32x32x32.fp8.bf8">;
+def ROCDL_smfmac_f32_32x32x32_fp8_fp8 : ROCDL_Mfma_IntrOp<"smfmac.f32.32x32x32.fp8.fp8">;
+
+
 //===---------------------------------------------------------------------===//
 // WMMA intrinsics
 class ROCDL_Wmma_IntrOp<string mnemonic, list<int> overloadedOperands,
                         list<Trait> traits = []> :
-  LLVM_IntrOpBase<ROCDL_Dialect, mnemonic,
-                  "amdgcn_" # !subst(".","_", mnemonic),
-                  [0], overloadedOperands, traits, 1>,
+  ROCDL_IntrOp<mnemonic, [0], overloadedOperands, traits, 1>,
   Arguments<(ins Variadic<LLVM_Type>:$args)> {
   let assemblyFormat =
     "$args attr-dict `:` functional-type($args, $res)";
@@ -554,50 +536,32 @@ def ROCDL_RawPtrBufferAtomicFaddOp : ROCDL_RawPtrBufferAtomicNoRet<"fadd">;
 // Raw buffer load/store intrinsics
 
 def ROCDL_RawBufferLoadOp :
-  ROCDL_Op<"raw.buffer.load">,
-  Results<(outs LLVM_Type:$res)>,
+  ROCDL_IntrOp<"raw.buffer.load", [0], [], [], 1>,
   Arguments<(ins LLVM_Type:$rsrc,
                  LLVM_Type:$offset,
                  LLVM_Type:$soffset,
                  LLVM_Type:$aux)> {
-  string llvmBuilder = [{
-      $res = createIntrinsicCall(builder,
-          llvm::Intrinsic::amdgcn_raw_buffer_load, {$rsrc, $offset,
-          $soffset, $aux}, {$_resultType});
-  }];
   let hasCustomAssemblyFormat = 1;
 }
 
 def ROCDL_RawBufferStoreOp :
-  ROCDL_Op<"raw.buffer.store">,
+  ROCDL_IntrOp<"raw.buffer.store", [], [0], [], 0>,
   Arguments<(ins LLVM_Type:$vdata,
                  LLVM_Type:$rsrc,
                  LLVM_Type:$offset,
                  LLVM_Type:$soffset,
                  LLVM_Type:$aux)>{
-  string llvmBuilder = [{
-    auto vdataType = moduleTranslation.convertType(op.getVdata().getType());
-    createIntrinsicCall(builder,
-          llvm::Intrinsic::amdgcn_raw_buffer_store, {$vdata, $rsrc,
-          $offset, $soffset, $aux}, {vdataType});
-  }];
   let hasCustomAssemblyFormat = 1;
 }
 
 def ROCDL_RawBufferAtomicCmpSwap :
-  ROCDL_Op<"raw.buffer.atomic.cmpswap", [AllTypesMatch<["res", "src", "cmp"]>]>,
-  Results<(outs LLVM_Type:$res)>,
+  ROCDL_IntrOp<"raw.buffer.atomic.cmpswap", [], [0], [AllTypesMatch<["res", "src", "cmp"]>], 1>,
   Arguments<(ins LLVM_Type:$src,
                  LLVM_Type:$cmp,
                  LLVM_Type:$rsrc,
                  I32:$offset,
                  I32:$soffset,
                  I32:$aux)>{
-  string llvmBuilder = [{
-      $res = createIntrinsicCall(builder,
-          llvm::Intrinsic::amdgcn_raw_buffer_atomic_cmpswap, {$src, $cmp, $rsrc,
-            $offset, $soffset, $aux}, {$_resultType});
-  }];
   let assemblyFormat = [{
     attr-dict `(` operands `)` `:` type($res) `,` type($rsrc)
   }];
@@ -607,18 +571,12 @@ def ROCDL_RawBufferAtomicCmpSwap :
 // MI-100 and MI-200 buffer atomic floating point add intrinsic
 
 def ROCDL_RawBufferAtomicFAddOp :
-  ROCDL_Op<"raw.buffer.atomic.fadd">,
+  ROCDL_IntrOp<"raw.buffer.atomic.fadd", [], [0], [], 0>,
   Arguments<(ins LLVM_Type:$vdata,
                  LLVM_Type:$rsrc,
                  LLVM_Type:$offset,
                  LLVM_Type:$soffset,
                  LLVM_Type:$aux)>{
-  string llvmBuilder = [{
-      auto vdataType = moduleTranslation.convertType(op.getVdata().getType());
-      createIntrinsicCall(builder,
-          llvm::Intrinsic::amdgcn_raw_buffer_atomic_fadd, {$vdata, $rsrc,
-            $offset, $soffset, $aux}, {vdataType});
-  }];
   let hasCustomAssemblyFormat = 1;
 }
 
@@ -626,18 +584,12 @@ def ROCDL_RawBufferAtomicFAddOp :
 // Buffer atomic floating point max intrinsic. GFX9 does not support fp32.
 
 def ROCDL_RawBufferAtomicFMaxOp :
-  ROCDL_Op<"raw.buffer.atomic.fmax">,
+  ROCDL_IntrOp<"raw.buffer.atomic.fmax", [], [0], [], 0>,
   Arguments<(ins LLVM_Type:$vdata,
                  LLVM_Type:$rsrc,
                  LLVM_Type:$offset,
                  LLVM_Type:$soffset,
                  LLVM_Type:$aux)>{
-  string llvmBuilder = [{
-      auto vdataType = moduleTranslation.convertType(op.getVdata().getType());
-      createIntrinsicCall(builder,
-          llvm::Intrinsic::amdgcn_raw_buffer_atomic_fmax, {$vdata, $rsrc,
-            $offset, $soffset, $aux}, {vdataType});
-  }];
   let hasCustomAssemblyFormat = 1;
 }
 
@@ -645,18 +597,12 @@ def ROCDL_RawBufferAtomicFMaxOp :
 // Buffer atomic signed integer max intrinsic.
 
 def ROCDL_RawBufferAtomicSMaxOp :
-  ROCDL_Op<"raw.buffer.atomic.smax">,
+  ROCDL_IntrOp<"raw.buffer.atomic.smax", [], [0], [], 0>,
   Arguments<(ins LLVM_Type:$vdata,
                  LLVM_Type:$rsrc,
                  LLVM_Type:$offset,
                  LLVM_Type:$soffset,
                  LLVM_Type:$aux)>{
-  string llvmBuilder = [{
-      auto vdataType = moduleTranslation.convertType(op.getVdata().getType());
-      createIntrinsicCall(builder,
-          llvm::Intrinsic::amdgcn_raw_buffer_atomic_smax, {$vdata, $rsrc,
-            $offset, $soffset, $aux}, {vdataType});
-  }];
   let hasCustomAssemblyFormat = 1;
 }
 
@@ -664,43 +610,25 @@ def ROCDL_RawBufferAtomicSMaxOp :
 // Buffer atomic unsigned integer min intrinsic.
 
 def ROCDL_RawBufferAtomicUMinOp :
-  ROCDL_Op<"raw.buffer.atomic.umin">,
+  ROCDL_IntrOp<"raw.buffer.atomic.umin", [], [0], [], 0>,
   Arguments<(ins LLVM_Type:$vdata,
                  LLVM_Type:$rsrc,
                  LLVM_Type:$offset,
                  LLVM_Type:$soffset,
                  LLVM_Type:$aux)>{
-  string llvmBuilder = [{
-      auto vdataType = moduleTranslation.convertType(op.getVdata().getType());
-      createIntrinsicCall(builder,
-          llvm::Intrinsic::amdgcn_raw_buffer_atomic_umin, {$vdata, $rsrc,
-            $offset, $soffset, $aux}, {vdataType});
-  }];
   let hasCustomAssemblyFormat = 1;
 }
 
 // DPP Update intrinsic
 def ROCDL_DPPUpdateOp : ROCDL_IntrOp<"update.dpp", [], [0],
-    [AllTypesMatch<["res", "src", "old"]>], 1>,
+    [AllTypesMatch<["res", "src", "old"]>], 1, 0, 0,
+      [2, 3, 4, 5], ["dppCtrl", "rowMask", "bankMask", "boundCtrl"]>,
   Arguments<(ins LLVM_Type:$old, LLVM_Type:$src, I32Attr:$dppCtrl, I32Attr:$rowMask,
       I32Attr:$bankMask, I1Attr:$boundCtrl)> {
   let results = (outs LLVM_Type:$res);
   let assemblyFormat = [{
     attr-dict $old `,` $src `with` $dppCtrl `,` $rowMask `,` $bankMask `,` $boundCtrl `:` type($src)
   }];
-  string llvmBuilder = [{
-      auto vdataType = moduleTranslation.convertType(op.getSrc().getType());
-      llvm::Value *args[] = {
-        moduleTranslation.lookupValue(op.getOld()),
-        moduleTranslation.lookupValue(op.getSrc()),
-          builder.getInt32(op.getDppCtrl()),
-          builder.getInt32(op.getRowMask()),
-          builder.getInt32(op.getBankMask()),
-          builder.getInt1(op.getBoundCtrl())
-      };
-      $res = createIntrinsicCall(builder,
-        llvm::Intrinsic::amdgcn_update_dpp, args, {vdataType});
-  }];
 }
 
 //===---------------------------------------------------------------------===//
@@ -718,6 +646,93 @@ def ROCDL_CvtPkRtz:
   }];
 }
 
+//===---------------------------------------------------------------------===//
+// 32-bit float intrinsics
+//===---------------------------------------------------------------------===//
+def ROCDL_CvtScalePkF32Fp8 :
+    ROCDL_IntrOp<"cvt.scalef32.pk.f32.fp8", [], [], [Pure], 1>,
+    Arguments<(ins I32:$src, F32: $scale, I1:$wordSel)> {
+  let summary = "Scale and convert packed fp8 to packed f32";
+  let description = [{
+    Scale `src` by the exponent in `scale` then convert to packed fp32.
+    Store the result in low/high word based on $wordSel, preserving the other word.
+  }];
+  let assemblyFormat = [{
+    attr-dict $src `[` $wordSel `]` `,` $scale `:` type($res)
+  }];
+}
+def ROCDL_CvtScalePkF32Bf8 :
+    ROCDL_IntrOp<"cvt.scalef32.pk.f32.bf8", [], [], [Pure], 1>,
+    Arguments<(ins I32:$src, F32: $scale, I1:$wordSel)> {
+  let summary = "Scale and convert packed bf8 to packed f32";
+  let description = [{
+    Scale `src` by the exponent in `scale` then convert to packed fp32.
+    Store the result in low/high word based on $wordSel, preserving the other word.
+  }];
+  let assemblyFormat = [{
+    attr-dict $src `[` $wordSel `]` `,` $scale `:` type($res)
+  }];
+}
+//===---------------------------------------------------------------------===//
+// 8-bit float scale intrinsics
+//===---------------------------------------------------------------------===//
+def ROCDL_V2I16Type : FixedVectorOfLengthAndType<[2], [I16]>,
+                        BuildableType<"::mlir::VectorType::get("
+                          "{2},$_builder.getI16Type())">;
+
+def ROCDL_CvtScaleF32PkFp8F32:
+    ROCDL_IntrOp<"cvt.scalef32.pk.fp8.f32", [], [], [Pure], 1>,
+    Arguments<(ins ROCDL_V2I16Type:$old, F32:$srcA, F32:$srcB, F32:$scale, I1:$wordSel)> {
+  let summary = "Scale and convert two f32's to packed fp8";
+  let description = [{
+    Scale `srcA` and `srcB` by the exponent in `scale` then convert to packed fp8
+    and store into the low/high word of `old`, preserving the other word.
+  }];
+  let assemblyFormat = [{
+    attr-dict $srcA `,` $srcB `,` $scale `->` $old `[` $wordSel `]` `:` type($res)
+  }];
+}
+    
+def ROCDL_CvtScaleF32PkBf8F32:
+    ROCDL_IntrOp<"cvt.scalef32.pk.bf8.f32", [], [], [Pure], 1>,
+    Arguments<(ins ROCDL_V2I16Type:$old, F32:$srcA, F32:$srcB, F32: $scale, I1:$wordSel)> {
+  let summary = "Scale and convert two f32's to packed bf8";
+  let description = [{
+    Scale `srcA` and `srcB` by the exponent in `scale` then convert to packed bf8
+    and store into the low/high word of `old`, preserving the other word.
+  }];
+  let assemblyFormat = [{
+    attr-dict $srcA `,` $srcB `,` $scale `->` $old `[` $wordSel `]` `:` type($res)
+  }];
+}
+
+def ROCDL_CvtScaleF32SrFp8F32:
+    ROCDL_IntrOp<"cvt.scalef32.sr.fp8.f32", [], [], [Pure], 1>,
+    Arguments<(ins I32:$old, F32:$src, I32:$seed, F32: $scale, I32:$byteSel)> {
+    let summary = "Scale and convert f32 to fp8 using stochastic rounding";
+    let description = [{
+       Scale `src` by the exponent in `scale` then convert to fp8 with stochastic rounding
+       using seed data in `seed`. store into the `byteSel`th byte of `old`, preserving the others.
+    }];
+  let assemblyFormat = [{
+    attr-dict $src `,` $seed `,` $scale `->` $old `[` $byteSel `]` `:` type($res)
+  }];
+}
+
+
+def ROCDL_CvtScaleF32SrBf8F32:
+    ROCDL_IntrOp<"cvt.scalef32.sr.bf8.f32", [], [], [Pure], 1>,
+    Arguments<(ins I32:$old, F32:$src, I32:$seed, F32: $scale, I32:$byteSel)> {
+    let summary = "Scale and convert f32 to bf8 using stochastic rounding";
+    let description = [{
+       Scale `src` by the exponent in `scale` then convert to bf8 with stochastic rounding
+       using seed data in `seed`. store into the `byteSel`th byte of `old`, preserving the others.
+    }];
+  let assemblyFormat = [{
+    attr-dict $src `,` $seed `,` $scale `->` $old `[` $byteSel `]` `:` type($res)
+  }];
+}
+
 //===---------------------------------------------------------------------===//
 // 8-bit float intrinsics
 //===---------------------------------------------------------------------===//
@@ -733,6 +748,20 @@ def ROCDL_CvtF32Bf8Op :
   }];
 }
 
+def ROCDL_CvtScaleF32Bf8Op :
+    ROCDL_IntrOp<"cvt.scalef32.f32.bf8", [], [], [Pure], 1>,
+    Arguments<(ins I32:$src, F32: $scale, I32:$byteSel)> {
+  let summary = "Scale and convert bf8 to f32";
+  let description = [{
+    Scale `src` by the exponent in `scale` then convert 8-bit bf8 value 
+    from the `byteSel`th bit of `src` to fp32.
+  }];
+  let assemblyFormat = [{
+    attr-dict $src `[` $byteSel `]` `,` $scale `:` type($res)
+  }];
+}
+
+
 def ROCDL_CvtF32Fp8Op :
     ROCDL_IntrOp<"cvt.f32.fp8", [], [], [Pure], 1>,
     Arguments<(ins I32:$srcA, I32:$byteSel)> {
@@ -745,6 +774,22 @@ def ROCDL_CvtF32Fp8Op :
   }];
 }
 
+
+def ROCDL_CvtScaleF32Fp8Op :
+    ROCDL_IntrOp<"cvt.scalef32.f32.fp8", [], [], [Pure], 1>,
+    Arguments<(ins I32:$src, F32: $scale, I32:$byteSel)> {
+  let summary = "Scale and convert fp8 to f32";
+  let description = [{
+    Scale `src` by the exponent in `scale` then convert 8-bit fp8 value
+    from the `byteSel`th bit of `src` to fp32.
+
+  }];
+  let assemblyFormat = [{
+    attr-dict $src `[` $byteSel `]` `,` $scale `:` type($res)
+  }];
+}
+
+
 def ROCDL_CvtPkBf8F32Op :
     ROCDL_IntrOp<"cvt.pk.bf8.f32", [], [], [Pure], 1>,
     Arguments<(ins F32:$srcA, F32:$srcB, I32:$old, I1:$wordSel)> {
diff --git a/mlir/include/mlir/Dialect/Linalg/IR/LinalgInterfaces.td b/mlir/include/mlir/Dialect/Linalg/IR/LinalgInterfaces.td
index 244db23925ab3..98a5fd278a997 100644
--- a/mlir/include/mlir/Dialect/Linalg/IR/LinalgInterfaces.td
+++ b/mlir/include/mlir/Dialect/Linalg/IR/LinalgInterfaces.td
@@ -710,7 +710,10 @@ def LinalgStructuredInterface
     >,
     InterfaceMethod<
       /*desc=*/[{
-        Return true if the user has supplied an explicit indexing maps for this op.
+        Returns true if the user has supplied explicit indexing maps that are
+        different from default indexing maps for this op. Returns `false` otherwise.
+        Note, if the user define maps that are identical to the default maps,
+        this method returns `false`.
       }],
       /*retTy=*/"bool",
       /*methodName=*/"hasUserDefinedMaps",
diff --git a/mlir/include/mlir/Dialect/Linalg/IR/LinalgNamedStructuredOps.yaml b/mlir/include/mlir/Dialect/Linalg/IR/LinalgNamedStructuredOps.yaml
index b0ea1f7695581..496a323249e85 100644
--- a/mlir/include/mlir/Dialect/Linalg/IR/LinalgNamedStructuredOps.yaml
+++ b/mlir/include/mlir/Dialect/Linalg/IR/LinalgNamedStructuredOps.yaml
@@ -1472,75 +1472,6 @@ structured_op: !LinalgStructuredOpConfig
                 - !ScalarExpression
                   scalar_arg: rhs
 --- !LinalgOpConfig
-metadata: !LinalgOpMetadata
-  name: batch_matmul
-  cpp_class_name: BatchMatmulOp
-  doc: |-
-    Performs a batched matrix multiplication of two 3D inputs.
-
-    Numeric casting is performed on the operands to the inner multiply, promoting
-    them to the same data type as the accumulator/output.
-  implements:
-  - LinalgContractionOpInterface
-structured_op: !LinalgStructuredOpConfig
-  args:
-  - !LinalgOperandDefConfig
-    name: A
-    kind: input_tensor
-    type_var: T1
-    shape_map: affine_map<()[s0, s1, s2, s3] -> (s0, s1, s2)>
-  - !LinalgOperandDefConfig
-    name: B
-    kind: input_tensor
-    type_var: T2
-    shape_map: affine_map<()[s0, s1, s2, s3] -> (s0, s2, s3)>
-  - !LinalgOperandDefConfig
-    name: C
-    kind: output_tensor
-    type_var: U
-    shape_map: affine_map<()[s0, s1, s2, s3] -> (s0, s1, s3)>
-  indexing_maps: !LinalgIndexingMapsConfig
-    static_indexing_maps:
-    - affine_map<(d0, d1, d2, d3)[s0, s1, s2, s3] -> (d0, d1, d3)>
-    - affine_map<(d0, d1, d2, d3)[s0, s1, s2, s3] -> (d0, d3, d2)>
-    - affine_map<(d0, d1, d2, d3)[s0, s1, s2, s3] -> (d0, d1, d2)>
-  iterator_types:
-  - parallel
-  - parallel
-  - parallel
-  - reduction
-  assignments:
-  - !ScalarAssign
-    arg: C
-    value: !ScalarExpression
-      scalar_fn:
-        kind: binary
-        fn_name: add
-        operands:
-        - !ScalarExpression
-          scalar_arg: C
-        - !ScalarExpression
-          scalar_fn:
-            kind: binary
-            fn_name: mul
-            operands:
-            - !ScalarExpression
-              scalar_fn:
-                kind: type
-                fn_name: cast_signed
-                type_var: U
-                operands:
-                - !ScalarExpression
-                  scalar_arg: A
-            - !ScalarExpression
-              scalar_fn:
-                kind: type
-                fn_name: cast_signed
-                type_var: U
-                operands:
-                - !ScalarExpression
-                  scalar_arg: B
---- !LinalgOpConfig
 metadata: !LinalgOpMetadata
   name: batch_matmul_transpose_a
   cpp_class_name: BatchMatmulTransposeAOp
diff --git a/mlir/include/mlir/Dialect/Linalg/IR/LinalgOps.td b/mlir/include/mlir/Dialect/Linalg/IR/LinalgOps.td
index e42fd5d2ce13c..f8df828f74851 100644
--- a/mlir/include/mlir/Dialect/Linalg/IR/LinalgOps.td
+++ b/mlir/include/mlir/Dialect/Linalg/IR/LinalgOps.td
@@ -155,7 +155,7 @@ def Linalg_SoftmaxOp : Linalg_Op<"softmax",
 }
 
 def Linalg_WinogradFilterTransformOp : Linalg_Op<"winograd_filter_transform",
-    [AllElementTypesMatch<["filter", "output"]>,
+    [AllElementTypesMatch<["filter", "output"]>, DestinationStyleOpInterface,
      DeclareOpInterfaceMethods<TilingInterface,
       ["getIterationDomain",
        "getLoopIteratorTypes",
@@ -220,12 +220,13 @@ def Linalg_WinogradFilterTransformOp : Linalg_Op<"winograd_filter_transform",
     int64_t getFilterCDim() {
       return 3;
     }
+    MutableOperandRange getDpsInitsMutable() { return getOutputMutable(); }
   }];
   let hasVerifier = 1;
 }
 
 def Linalg_WinogradInputTransformOp : Linalg_Op<"winograd_input_transform",
-    [AllElementTypesMatch<["input", "output"]>,
+    [AllElementTypesMatch<["input", "output"]>, DestinationStyleOpInterface,
      DeclareOpInterfaceMethods<TilingInterface,
       ["getIterationDomain",
        "getLoopIteratorTypes",
@@ -308,6 +309,7 @@ def Linalg_WinogradInputTransformOp : Linalg_Op<"winograd_input_transform",
     int64_t getOutputCDim() {
       return 5;
     }
+    MutableOperandRange getDpsInitsMutable() { return getOutputMutable(); }
   }];
   let hasVerifier = 1;
 }
diff --git a/mlir/include/mlir/Dialect/Linalg/IR/LinalgStructuredOps.td b/mlir/include/mlir/Dialect/Linalg/IR/LinalgStructuredOps.td
index fff4048ee125e..29cb8035b583b 100644
--- a/mlir/include/mlir/Dialect/Linalg/IR/LinalgStructuredOps.td
+++ b/mlir/include/mlir/Dialect/Linalg/IR/LinalgStructuredOps.td
@@ -606,7 +606,10 @@ def MatmulOp : LinalgStructuredBase_Op<"matmul", [
     let arguments = (ins
       Variadic<AnyType>:$inputs,
       Variadic<AnyShaped>:$outputs,
-      DefaultValuedOptionalAttr<AffineMapArrayAttr, "{}">:$indexing_maps,
+      DefaultValuedOptionalAttr<
+        AffineMapArrayAttr,
+        "MatmulOp::getDefaultIndexingMaps($_builder.getContext())"
+      >:$indexing_maps,
       DefaultValuedOptionalAttr<TypeFnAttr, "TypeFn::cast_signed">:$cast
     );
     let results = (outs Variadic<AnyRankedTensor>:$result_tensors);
@@ -674,12 +677,271 @@ def MatmulOp : LinalgStructuredBase_Op<"matmul", [
       static unsigned getNumRegionArgs();
       std::string getLibraryCallName();
       bool hasDynamicIndexingMaps();
-      /// Check if the op has broadcast and/or transpose semantic. Returns true if the
-      /// user defined indexing maps are not equal to default map.
+      /// Returns true if the user defined indexing maps are not equal to default maps.
       bool hasUserDefinedMaps();
     }];
 }
 
+//===----------------------------------------------------------------------===//
+// Contract op.
+//===----------------------------------------------------------------------===//
+
+def ContractOp : LinalgStructuredBase_Op<"contract", [
+               AttrSizedOperandSegments,
+               LinalgContractionOpInterface]> {
+  let summary = [{
+    Perform a contraction on two inputs, accumulating into the third.
+  }];
+  let description = [{
+    The semantics of contracting inputs `A` and `B` on top of `C` to produce
+    output `D` is given by
+
+      `D[H] = (SUM_{(I ∪ J) \ H} A[I] * B[J]) + C[H]`
+
+    where `I`, `J`, and `H` are tuples of (pairwise distinct) dimension
+    identifiers - meant to range over valid indices - corresponding to the
+    results of the mandatory (projected permutation) `indexing_maps` for `A`,
+    `B` and `C`. `SUM_{dims}` means reduce over all valid indices for the
+    dimensions in the set `dims` (with `I`, `J`, and `K` treated as _sets_ of
+    dim identifiers).
+
+    The iteration space consists of all dimensions in `I`, `J` and `H`, i.e. the
+    domain of each of the `affine_map`s. Like for einsums, the iteration type of
+    each dim is inferred and is either:
+
+    - reduction: the dim is used to index into `A` and `B` but not `C`. Per the
+      above semantics, these dims will be contracted, i.e. reduced over.
+
+    - parallel: the dim is used to index into `C` and at least one of `A` and
+      `B`, and - deriving from matmul terminology - is either an "M-like" dim
+      (if used on `A` and `C`), an "N-like" dim (if used on `B` and `C`) or a
+      "batch"-dim (if used to index into `A`, `B`, and `C`).
+
+    For example, batch-matmul is given by `I = ⟨ b, m, k ⟩`, `J = ⟨ b, k, n ⟩`,
+    `H = ⟨ b, m, n ⟩` (with `k` as a contracting reduction-dimension while `m`,
+    `n` and `b` have parallel iteration-type) and gets represented as:
+
+    ```
+    %D = linalg.contract
+        indexing_maps = [affine_map<(batch, m, n, k) -> (batch, m, k)>,
+                         affine_map<(batch, m, n, k) -> (batch, k, n)>,
+                         affine_map<(batch, m, n, k) -> (batch, m, n)>]
+        ins(%A, %B: tensor<?x?x?xf32>, tensor<?x?x?xf32>)
+        outs(%C: tensor<?x?x?xf32>) -> tensor<?x?x?xf32>
+    ```
+
+    Note that by permuting dims in the `affine_map`s' results, accesses to
+    to the inputs and output can be arbitrarily transposed. Similarly, arbitrary
+    broadcasts can be achieved through leaving out dims on either input operand.
+    For example, the following is a variant of batch-matmul with a transposition
+    applied to `A` while `B`'s 2D-matrix gets broadcasted along the batch dim:
+
+    ```
+    linalg.contract
+        indexing_maps = [affine_map<(batch, m, n, k) -> (batch, k, m)>,
+                         affine_map<(batch, m, n, k) -> (k, n)>,
+                         affine_map<(batch, m, n, k) -> (batch, m, n)>]
+        ins(%A, %B: memref<?x?x?xf32>, memref<?x?xf32>)
+        outs(%C: memref<?x?x?xf32>)
+    ```
+
+    Numeric casting is performed on the operands to the inner multiplication,
+    promoting/truncating them to the same data type as the accumulator/output.
+
+    TODO: Allow control over the combining/accumulating op and possibly the
+          multiplication op.
+  }];
+
+  let arguments = (ins
+    Variadic<AnyType>:$inputs,
+    Variadic<AnyShaped>:$outputs,
+    AffineMapArrayAttr:$indexing_maps,
+    DefaultValuedOptionalAttr<TypeFnAttr, "TypeFn::cast_signed">:$cast
+  );
+  let results = (outs Variadic<AnyShaped>:$result_tensors);
+  // NB: The only reason this op has a region - and it get populated at op build
+  //     time - is that currently the LinalgOp interface exposes methods that
+  //     assume a relevant region is available to be queried at any time.
+  let regions = (region SizedRegion<1>:$combiner);
+
+  let skipDefaultBuilders = 1;
+  let builders = [
+    OpBuilder<(ins "TypeRange":$resultTensorTypes, "ValueRange":$inputs,
+      "ValueRange":$outputs, "ArrayAttr":$indexingMaps,
+      CArg<"ArrayRef<NamedAttribute>", "{}">:$attributes),
+      [{
+        $_state.addAttribute("indexing_maps", indexingMaps);
+        buildStructuredOp($_builder, $_state, resultTensorTypes, inputs,
+                          outputs, attributes, regionBuilder);
+      }]>,
+    OpBuilder<(ins "ValueRange":$inputs, "ValueRange":$outputs,
+      "ArrayAttr":$indexingMaps,
+      CArg<"ArrayRef<NamedAttribute>", "{}">:$attributes),
+      [{
+        $_state.addAttribute("indexing_maps", indexingMaps);
+        buildStructuredOp($_builder, $_state, std::nullopt, inputs, outputs,
+                          attributes, regionBuilder);
+      }]>
+  ];
+  let hasCustomAssemblyFormat = 1;
+  let hasFolder = 1;
+  let hasVerifier = 1;
+
+  let extraClassDeclaration = structuredOpsBaseDecls # [{
+    // Declare/implement functions necessary for LinalgStructuredInterface.
+
+    /// Infer iterator types for each dim in the domain of IndexingMaps.
+    SmallVector<utils::IteratorType> getIteratorTypesArray();
+
+    /// IndexingMaps always depends on attr associated to current Op instance.
+    bool hasDynamicIndexingMaps() { return true; };
+    bool hasUserDefinedMaps() { return true; };
+
+    static unsigned getNumRegionArgs();
+
+    static void regionBuilder(ImplicitLocOpBuilder &b,
+                              Block &block, ArrayRef<NamedAttribute> attrs);
+
+    static std::function<void(ImplicitLocOpBuilder &,
+                              Block &, ArrayRef<NamedAttribute>)>
+    getRegionBuilder() {
+      return regionBuilder;
+    }
+
+    std::string getLibraryCallName() {
+      return "op_has_no_registered_library_name";
+    }
+
+    // Implement function necessary for DestinationStyleOpInterface.
+    ::mlir::MutableOperandRange getDpsInitsMutable() {
+      return getOutputsMutable();
+    }
+  }];
+}
+
+//===----------------------------------------------------------------------===//
+// Op definition for BatchMatmulOp
+//===----------------------------------------------------------------------===//
+
+def BatchMatmulOp : LinalgStructuredBase_Op<"batch_matmul", !listconcat([AttrSizedOperandSegments],
+  /*extraInterfaces=*/[LinalgContractionOpInterface])> {
+    
+  let summary = [{Performs a batched matrix multiplication of two 3D inputs.}];
+  let description = [{Numeric casting is performed on the operands to the inner multiply, promoting
+    them to the same data type as the accumulator/output.
+
+    Broadcast and Transpose semantics can be appiled by specifying the explicit attribute
+    'indexing_maps' as shown below. This is a list attribute, so must include maps for all
+    arguments if specified.
+
+    Example Transpose:
+    ```
+    linalg.batch_matmul indexing_maps = [
+                   affine_map<(d0, d1, d2, d3) -> (d0, d3, d1)>, // transpose
+                   affine_map<(d0, d1, d2, d3) -> (d0, d3, d2)>,
+                   affine_map<(d0, d1, d2, d3) -> (d0, d1, d2)>
+                   ]
+                   ins(%arg0, %arg1 : memref<2x5x3xf32>,memref<2x5x7xf32>)
+                   outs(%arg2: memref<2x3x7xf32>)
+    ```
+
+    Example Broadcast:
+    ```
+    linalg.batch_matmul indexing_maps = [
+                       affine_map<(d0, d1, d2, d3) -> (d3)>,         // broadcast
+                       affine_map<(d0, d1, d2, d3) -> (d0, d3, d2)>,
+                       affine_map<(d0, d1, d2, d3) -> (d0, d1, d2)>
+                     ]
+                     ins(%arg0, %arg1 : memref<5xf32>, memref<2x5x7xf32>)
+                     outs(%arg2: memref<2x3x7xf32>)
+    ```
+
+    Example Broadcast and Transpose:
+    ```
+    linalg.batch_matmul indexing_maps = [
+                       affine_map<(d0, d1, d2, d3) -> (d1, d3)>,     // broadcast
+                       affine_map<(d0, d1, d2, d3) -> (d0, d2, d3)>, // transpose
+                       affine_map<(d0, d1, d2, d3) -> (d0, d1, d2)>
+                     ]
+                     ins(%arg0, %arg1 : memref<3x5xf32>, memref<2x7x5xf32>)
+                     outs(%arg2: memref<2x3x7xf32>)
+    ```
+}];
+
+    let arguments = (ins
+      Variadic<AnyType>:$inputs,
+      Variadic<AnyShaped>:$outputs,
+      DefaultValuedOptionalAttr<AffineMapArrayAttr, "{}">:$indexing_maps
+    );
+    let results = (outs Variadic<AnyRankedTensor>:$result_tensors);
+    let regions = (region AnyRegion:$region);
+
+    let skipDefaultBuilders = 1;
+    let builders = [
+      OpBuilder<
+      (ins "ValueRange":$inputs, "ValueRange":$outputs,
+            CArg<"ArrayRef<NamedAttribute>", "{}">:$attributes),
+      [{
+        buildBatchMatmulOp($_builder, $_state, std::nullopt, inputs, outputs,
+          attributes, BatchMatmulOp::getRegionBuilder(),
+          BatchMatmulOp::getDefaultIndexingMaps($_builder.getContext()));
+      }]>,
+      OpBuilder<
+      (ins "TypeRange":$resultTensorTypes, "ValueRange":$inputs,
+            "ValueRange":$outputs,
+            CArg<"ArrayRef<NamedAttribute>", "{}">:$attributes),
+      [{
+        buildBatchMatmulOp($_builder, $_state, resultTensorTypes,
+          inputs, outputs, attributes, BatchMatmulOp::getRegionBuilder(),
+          BatchMatmulOp::getDefaultIndexingMaps($_builder.getContext()));
+      }]>,
+      OpBuilder<
+      (ins "TypeRange":$resultTensorTypes, "ValueRange":$operands,
+            CArg<"ArrayRef<NamedAttribute>", "{}">:$attributes),
+      [{
+        $_state.addOperands(operands);
+        $_state.addAttributes(attributes);
+        $_state.addTypes(resultTensorTypes);
+        (void)$_state.addRegion(),
+        BatchMatmulOp::getDefaultIndexingMaps($_builder.getContext());
+      }]>
+      
+    ];
+    let hasCustomAssemblyFormat = 1;
+    let hasFolder = 1;
+    let hasVerifier = 1;
+
+    let extraClassDeclaration = structuredOpsBaseDecls # [{
+
+      SmallVector<utils::IteratorType> getIteratorTypesArray();
+      static void regionBuilder(ImplicitLocOpBuilder &b,
+                                Block &block, ArrayRef<NamedAttribute> attrs);
+      static std::function<void(ImplicitLocOpBuilder &,
+                                Block &, ArrayRef<NamedAttribute>)>
+      getRegionBuilder() {
+        return regionBuilder;
+      }
+
+      /// Returns a list with default AffineMap(s), i.e. without broadcasts and transpositions.
+      static SmallVector<AffineMap> getDefaultIndexingMaps(MLIRContext *context);
+
+      /// Returns true if the given broadcast map \p bcastMap is valid for this op.
+      bool isValidLhsRhsBroadcastMap(AffineMap bcastMap, bool isLHS = true);
+
+      ::mlir::MutableOperandRange getDpsInitsMutable() {
+        return getOutputsMutable();
+      }
+
+      // Generic methods.
+      static unsigned getNumRegionArgs();
+      bool hasDynamicIndexingMaps() { return true; }
+      std::string getLibraryCallName();
+      /// Returns true if the user defined indexing maps are not equal to default maps.
+      bool hasUserDefinedMaps();
+    }];
+}
+
+
 //===----------------------------------------------------------------------===//
 // Named Linalg ops, implemented as a declarative configurations of generic ops.
 //===----------------------------------------------------------------------===//
diff --git a/mlir/include/mlir/Dialect/Linalg/TransformOps/LinalgTransformOps.td b/mlir/include/mlir/Dialect/Linalg/TransformOps/LinalgTransformOps.td
index 081bf9b6d3b23..e86d175489775 100644
--- a/mlir/include/mlir/Dialect/Linalg/TransformOps/LinalgTransformOps.td
+++ b/mlir/include/mlir/Dialect/Linalg/TransformOps/LinalgTransformOps.td
@@ -1007,7 +1007,7 @@ def PackTransposeOp : Op<Transform_Dialect, "structured.pack_transpose", [
 
     This operation may produce a silenceableFailure if the transpose spec is
     ill-formed (i.e. `outer_perm` or `inner_perm` are not permutations of the
-    proper rank) or if the tranposition of all involved operations fails for any
+    proper rank) or if the transposition of all involved operations fails for any
     reason.
 
     This operation returns 3 handles, one to the transformed LinalgOp, one to
diff --git a/mlir/include/mlir/Dialect/Linalg/Transforms/Transforms.h b/mlir/include/mlir/Dialect/Linalg/Transforms/Transforms.h
index eed279b6be34a..2a3ce0519ab53 100644
--- a/mlir/include/mlir/Dialect/Linalg/Transforms/Transforms.h
+++ b/mlir/include/mlir/Dialect/Linalg/Transforms/Transforms.h
@@ -1395,6 +1395,14 @@ FailureOr<Operation *>
 decomposeWinogradOutputTransformOp(RewriterBase &rewriter,
                                    linalg::WinogradOutputTransformOp op);
 
+/// Method to deduplicate operands and remove dead results of `linalg.generic`
+/// operations. This is effectively DCE for a linalg.generic op. If there is
+/// deduplication of operands orremoval of results, replaces the `genericOp`
+/// with a new op and returns it. Returns the same operation if there is no
+/// deduplication/removal.
+FailureOr<linalg::GenericOp> deduplicateOperandsAndRemoveDeadResults(
+    RewriterBase &rewriter, linalg::GenericOp genericOp, bool removeOutputs);
+
 //===----------------------------------------------------------------------===//
 // Rewrite patterns wrapping transformations.
 // TODO: every single such pattern should be a close to noop wrapper around a
@@ -1723,11 +1731,6 @@ void populateDecomposePadPatterns(RewritePatternSet &patterns);
 /// \see rewriteInIm2Col for more details.
 void populateConvertConv2DToImg2ColPatterns(RewritePatternSet &patterns);
 
-/// Populates `patterns` with vectorisation patterns for tensor.insert_slice.
-/// TODO: Avoid having a dedicated `populate{}` for one pattern. Instead, either
-/// expand or merge with other `populate{}`.
-void populateInsertSliceVectorizationPatterns(RewritePatternSet &patterns);
-
 /// Populates `patterns` with patterns that vectorize tensor.pad.
 /// These patterns are meant to apply in a complementary fashion. Benefits
 /// are used to encode a certain ordering of pattern application. To avoid
@@ -1786,7 +1789,7 @@ void populateDataLayoutPropagationPatterns(
     const ControlPropagationFn &controlPackUnPackPropagation);
 
 /// Pattern to remove dead operands and results of `linalg.generic` operations.
-/// This is effectively DCE for a linalg op.
+/// This is a pattern wrapper for `deduplicateOperandsAndRemoveDeadResults`.
 void populateEraseUnusedOperandsAndResultsPatterns(RewritePatternSet &patterns);
 
 /// Patterns to promote inputs to outputs and remove unused inputs of
diff --git a/mlir/include/mlir/Dialect/MPI/IR/MPI.td b/mlir/include/mlir/Dialect/MPI/IR/MPI.td
index 643612e1e2ee8..7c84443e5520d 100644
--- a/mlir/include/mlir/Dialect/MPI/IR/MPI.td
+++ b/mlir/include/mlir/Dialect/MPI/IR/MPI.td
@@ -215,4 +215,43 @@ def MPI_ErrorClassAttr : EnumAttr<MPI_Dialect, MPI_ErrorClassEnum, "errclass"> {
   let assemblyFormat = "`<` $value `>`";
 }
 
+def MPI_OpNull : I32EnumAttrCase<"MPI_OP_NULL", 0, "MPI_OP_NULL">;
+def MPI_OpMax : I32EnumAttrCase<"MPI_MAX", 1, "MPI_MAX">;
+def MPI_OpMin : I32EnumAttrCase<"MPI_MIN", 2, "MPI_MIN">;
+def MPI_OpSum : I32EnumAttrCase<"MPI_SUM", 3, "MPI_SUM">;
+def MPI_OpProd : I32EnumAttrCase<"MPI_PROD", 4, "MPI_PROD">;
+def MPI_OpLand : I32EnumAttrCase<"MPI_LAND", 5, "MPI_LAND">;
+def MPI_OpBand : I32EnumAttrCase<"MPI_BAND", 6, "MPI_BAND">;
+def MPI_OpLor : I32EnumAttrCase<"MPI_LOR", 7, "MPI_LOR">;
+def MPI_OpBor : I32EnumAttrCase<"MPI_BOR", 8, "MPI_BOR">;
+def MPI_OpLxor : I32EnumAttrCase<"MPI_LXOR", 9, "MPI_LXOR">;
+def MPI_OpBxor : I32EnumAttrCase<"MPI_BXOR", 10, "MPI_BXOR">;
+def MPI_OpMinloc : I32EnumAttrCase<"MPI_MINLOC", 11, "MPI_MINLOC">;
+def MPI_OpMaxloc : I32EnumAttrCase<"MPI_MAXLOC", 12, "MPI_MAXLOC">;
+def MPI_OpReplace : I32EnumAttrCase<"MPI_REPLACE", 13, "MPI_REPLACE">;
+
+def MPI_OpClassEnum : I32EnumAttr<"MPI_OpClassEnum", "MPI operation class", [
+      MPI_OpNull,
+      MPI_OpMax,
+      MPI_OpMin,
+      MPI_OpSum,
+      MPI_OpProd,
+      MPI_OpLand,
+      MPI_OpBand,
+      MPI_OpLor,
+      MPI_OpBor,
+      MPI_OpLxor,
+      MPI_OpBxor,
+      MPI_OpMinloc,
+      MPI_OpMaxloc,
+      MPI_OpReplace
+    ]> {
+  let genSpecializedAttr = 0;
+  let cppNamespace = "::mlir::mpi";
+}
+
+def MPI_OpClassAttr : EnumAttr<MPI_Dialect, MPI_OpClassEnum, "opclass"> {
+  let assemblyFormat = "`<` $value `>`";
+}
+
 #endif // MLIR_DIALECT_MPI_IR_MPI_TD
diff --git a/mlir/include/mlir/Dialect/MPI/IR/MPIOps.td b/mlir/include/mlir/Dialect/MPI/IR/MPIOps.td
index 240fac5104c34..284ba72af9768 100644
--- a/mlir/include/mlir/Dialect/MPI/IR/MPIOps.td
+++ b/mlir/include/mlir/Dialect/MPI/IR/MPIOps.td
@@ -59,6 +59,28 @@ def MPI_CommRankOp : MPI_Op<"comm_rank", []> {
   let assemblyFormat = "attr-dict `:` type(results)";
 }
 
+//===----------------------------------------------------------------------===//
+// CommSizeOp
+//===----------------------------------------------------------------------===//
+
+def MPI_CommSizeOp : MPI_Op<"comm_size", []> {
+  let summary = "Get the size of the group associated to the communicator, "
+                "equivalent to `MPI_Comm_size(MPI_COMM_WORLD, &size)`";
+  let description = [{
+    Communicators other than `MPI_COMM_WORLD` are not supported for now.
+
+    This operation can optionally return an `!mpi.retval` value that can be used
+    to check for errors.
+  }];
+
+  let results = (
+    outs Optional<MPI_Retval> : $retval,
+    I32 : $size
+  );
+
+  let assemblyFormat = "attr-dict `:` type(results)";
+}
+
 //===----------------------------------------------------------------------===//
 // SendOp
 //===----------------------------------------------------------------------===//
@@ -71,13 +93,17 @@ def MPI_SendOp : MPI_Op<"send", []> {
     `dest`. The `tag` value and communicator enables the library to determine 
     the matching of multiple sends and receives between the same ranks.
 
-    Communicators other than `MPI_COMM_WORLD` are not supprted for now.
+    Communicators other than `MPI_COMM_WORLD` are not supported for now.
 
     This operation can optionally return an `!mpi.retval` value that can be used
     to check for errors.
   }];
 
-  let arguments = (ins AnyMemRef : $ref, I32 : $tag, I32 : $rank);
+  let arguments = (
+    ins AnyMemRef : $ref,
+    I32 : $tag,
+    I32 : $rank
+  );
 
   let results = (outs Optional<MPI_Retval>:$retval);
 
@@ -87,6 +113,42 @@ def MPI_SendOp : MPI_Op<"send", []> {
   let hasCanonicalizer = 1;
 }
 
+//===----------------------------------------------------------------------===//
+// ISendOp
+//===----------------------------------------------------------------------===//
+
+def MPI_ISendOp : MPI_Op<"isend", []> {
+  let summary =
+      "Equivalent to `MPI_Isend(ptr, size, dtype, dest, tag, MPI_COMM_WORLD)`";
+  let description = [{
+    MPI_Isend begins a non-blocking send of `size` elements of type `dtype` to
+    rank `dest`. The `tag` value and communicator enables the library to
+    determine the matching of multiple sends and receives between the same
+    ranks.
+
+    Communicators other than `MPI_COMM_WORLD` are not supported for now.
+
+    This operation can optionally return an `!mpi.retval` value that can be used
+    to check for errors.
+  }];
+
+  let arguments = (
+    ins AnyMemRef : $ref,
+    I32 : $tag,
+    I32 : $rank
+  );
+
+  let results = (
+    outs Optional<MPI_Retval>:$retval,
+    MPI_Request : $req
+  );
+
+  let assemblyFormat = "`(` $ref `,` $tag `,` $rank `)` attr-dict "
+                       "`:` type($ref) `,` type($tag) `,` type($rank) "
+                       "`->` type(results)";
+  let hasCanonicalizer = 1;
+}
+
 //===----------------------------------------------------------------------===//
 // RecvOp
 //===----------------------------------------------------------------------===//
@@ -100,7 +162,7 @@ def MPI_RecvOp : MPI_Op<"recv", []> {
     determine the matching of multiple sends and receives between the same 
     ranks.
 
-    Communicators other than `MPI_COMM_WORLD` are not supprted for now.
+    Communicators other than `MPI_COMM_WORLD` are not supported for now.
     The MPI_Status is set to `MPI_STATUS_IGNORE`, as the status object 
     is not yet ported to MLIR.
 
@@ -108,16 +170,134 @@ def MPI_RecvOp : MPI_Op<"recv", []> {
     to check for errors.
   }];
 
-  let arguments = (ins AnyMemRef : $ref, I32 : $tag, I32 : $rank);
+  let arguments = (
+    ins AnyMemRef : $ref,
+    I32 : $tag, I32 : $rank
+  );
 
   let results = (outs Optional<MPI_Retval>:$retval);
 
-  let assemblyFormat = "`(` $ref `,` $tag `,` $rank `)` attr-dict `:` "
+  let assemblyFormat = "`(` $ref `,` $tag `,` $rank `)` attr-dict `:`"
                        "type($ref) `,` type($tag) `,` type($rank)"
                        "(`->` type($retval)^)?";
   let hasCanonicalizer = 1;
 }
 
+//===----------------------------------------------------------------------===//
+// IRecvOp
+//===----------------------------------------------------------------------===//
+
+def MPI_IRecvOp : MPI_Op<"irecv", []> {
+  let summary = "Equivalent to `MPI_Irecv(ptr, size, dtype, dest, tag, "
+                "MPI_COMM_WORLD, &req)`";
+  let description = [{
+    MPI_Irecv begins a non-blocking receive of `size` elements of type `dtype` 
+    from rank `dest`. The `tag` value and communicator enables the library to 
+    determine the matching of multiple sends and receives between the same 
+    ranks.
+
+    Communicators other than `MPI_COMM_WORLD` are not supported for now.
+
+    This operation can optionally return an `!mpi.retval` value that can be used
+    to check for errors.
+  }];
+
+  let arguments = (
+    ins AnyMemRef : $ref,
+    I32 : $tag,
+    I32 : $rank
+  );
+
+  let results = (
+    outs Optional<MPI_Retval>:$retval,
+    MPI_Request : $req
+  );
+
+  let assemblyFormat = "`(` $ref `,` $tag `,` $rank `)` attr-dict `:`"
+                       "type($ref) `,` type($tag) `,` type($rank) `->`"
+                       "type(results)";
+  let hasCanonicalizer = 1;
+}
+
+//===----------------------------------------------------------------------===//
+// AllReduceOp
+//===----------------------------------------------------------------------===//
+
+def MPI_AllReduceOp : MPI_Op<"allreduce", []> {
+  let summary = "Equivalent to `MPI_Allreduce(sendbuf, recvbuf, op, "
+                "MPI_COMM_WORLD)`";
+  let description = [{
+    MPI_Allreduce performs a reduction operation on the values in the sendbuf
+    array and stores the result in the recvbuf array. The operation is 
+    performed across all processes in the communicator.
+
+    The `op` attribute specifies the reduction operation to be performed.
+    Currently only the `MPI_Op` predefined in the standard (e.g. `MPI_SUM`) are
+    supported.
+
+    Communicators other than `MPI_COMM_WORLD` are not supported for now.
+
+    This operation can optionally return an `!mpi.retval` value that can be used
+    to check for errors.
+  }];
+
+  let arguments = (
+    ins AnyMemRef : $sendbuf,
+    AnyMemRef : $recvbuf,
+    MPI_OpClassAttr : $op
+  );
+
+  let results = (outs Optional<MPI_Retval>:$retval);
+
+  let assemblyFormat = "`(` $sendbuf `,` $recvbuf `,` $op `)` attr-dict `:`"
+                       "type($sendbuf) `,` type($recvbuf)"
+                       "(`->` type($retval)^)?";
+}
+
+//===----------------------------------------------------------------------===//
+// BarrierOp
+//===----------------------------------------------------------------------===//
+
+def MPI_Barrier : MPI_Op<"barrier", []> {
+  let summary = "Equivalent to `MPI_Barrier(MPI_COMM_WORLD)`";
+  let description = [{
+    MPI_Barrier blocks execution until all processes in the communicator have
+    reached this routine.
+
+    Communicators other than `MPI_COMM_WORLD` are not supported for now.
+
+    This operation can optionally return an `!mpi.retval` value that can be used
+    to check for errors.
+  }];
+
+  let results = (outs Optional<MPI_Retval>:$retval);
+
+  let assemblyFormat = "attr-dict (`:` type($retval) ^)?";
+}
+
+//===----------------------------------------------------------------------===//
+// WaitOp
+//===----------------------------------------------------------------------===//
+
+def MPI_Wait : MPI_Op<"wait", []> {
+  let summary = "Equivalent to `MPI_Wait(req, MPI_STATUS_IGNORE)`";
+  let description = [{
+    MPI_Wait blocks execution until the request has completed.
+
+    The MPI_Status is set to `MPI_STATUS_IGNORE`, as the status object 
+    is not yet ported to MLIR.
+
+    This operation can optionally return an `!mpi.retval` value that can be used
+    to check for errors.
+  }];
+
+  let arguments = (ins MPI_Request : $req);
+
+  let results = (outs Optional<MPI_Retval>:$retval);
+
+  let assemblyFormat = "`(` $req `)` attr-dict `:` type($req) "
+                       "(`->` type($retval) ^)?";
+}
 
 //===----------------------------------------------------------------------===//
 // FinalizeOp
@@ -139,7 +319,6 @@ def MPI_FinalizeOp : MPI_Op<"finalize", []> {
   let assemblyFormat = "attr-dict (`:` type($retval)^)?";
 }
 
-
 //===----------------------------------------------------------------------===//
 // RetvalCheckOp
 //===----------------------------------------------------------------------===//
@@ -163,10 +342,8 @@ def MPI_RetvalCheckOp : MPI_Op<"retval_check", []> {
   let assemblyFormat = "$val `=` $errclass attr-dict `:` type($res)";
 }
 
-
-
 //===----------------------------------------------------------------------===//
-// RetvalCheckOp
+// ErrorClassOp
 //===----------------------------------------------------------------------===//
 
 def MPI_ErrorClassOp : MPI_Op<"error_class", []> {
diff --git a/mlir/include/mlir/Dialect/MPI/IR/MPITypes.td b/mlir/include/mlir/Dialect/MPI/IR/MPITypes.td
index 87eefa719d45c..fafea0eac8bb7 100644
--- a/mlir/include/mlir/Dialect/MPI/IR/MPITypes.td
+++ b/mlir/include/mlir/Dialect/MPI/IR/MPITypes.td
@@ -40,4 +40,26 @@ def MPI_Retval : MPI_Type<"Retval", "retval"> {
   }];
 }
 
+//===----------------------------------------------------------------------===//
+// mpi::RequestType
+//===----------------------------------------------------------------------===//
+
+def MPI_Request : MPI_Type<"Request", "request"> {
+  let summary = "MPI asynchronous request handler";
+  let description = [{
+    This type represents a handler to an asynchronous request.
+  }];
+}
+
+//===----------------------------------------------------------------------===//
+// mpi::StatusType
+//===----------------------------------------------------------------------===//
+
+def MPI_Status : MPI_Type<"Status", "status"> {
+  let summary = "MPI reception operation status type";
+  let description = [{
+    This type represents the status of a reception operation.
+  }];
+}
+
 #endif // MLIR_DIALECT_MPI_IR_MPITYPES_TD
diff --git a/mlir/include/mlir/Dialect/Math/Transforms/Passes.h b/mlir/include/mlir/Dialect/Math/Transforms/Passes.h
index f0f17c6adcb08..ea7a556297a76 100644
--- a/mlir/include/mlir/Dialect/Math/Transforms/Passes.h
+++ b/mlir/include/mlir/Dialect/Math/Transforms/Passes.h
@@ -48,6 +48,25 @@ struct MathPolynomialApproximationOptions {
 void populatePolynomialApproximateTanhPattern(RewritePatternSet &patterns);
 void populatePolynomialApproximateErfPattern(RewritePatternSet &patterns);
 
+// Adds patterns to convert to f32 around math functions for which `predicate`
+// returns true.
+void populateMathF32ExpansionPatterns(
+    RewritePatternSet &patterns, llvm::function_ref<bool(StringRef)> predicate);
+
+// Adds patterns to enable polynomial approximations for math functions for
+// which `predicate` returns true.
+void populateMathPolynomialApproximationPatterns(
+    RewritePatternSet &patterns, llvm::function_ref<bool(StringRef)> predicate);
+
+// Legacy. Calls both populateMathF32ExpansionPatterns and
+// populateMathPolynomialApproximationPatterns with predicates enabling a
+// certain set of math function rewrites, that probably can't be changed for
+// compatibility reasons. Notice that unlike
+// populateMathPolynomialApproximationPatterns(patterns, predicate), this
+// overload also calls populateMathF32ExpansionPatterns.
+// Prefer calling these functions directly:
+// * populateMathF32ExpansionPatterns(patterns, predicate)
+// * populateMathPolynomialApproximationPatterns(patterns, predicate)
 void populateMathPolynomialApproximationPatterns(
     RewritePatternSet &patterns,
     const MathPolynomialApproximationOptions &options = {});
diff --git a/mlir/include/mlir/Dialect/MemRef/IR/MemRef.h b/mlir/include/mlir/Dialect/MemRef/IR/MemRef.h
index 72463dca715ca..ac383ab46e7a5 100644
--- a/mlir/include/mlir/Dialect/MemRef/IR/MemRef.h
+++ b/mlir/include/mlir/Dialect/MemRef/IR/MemRef.h
@@ -17,6 +17,7 @@
 #include "mlir/Interfaces/CastInterfaces.h"
 #include "mlir/Interfaces/ControlFlowInterfaces.h"
 #include "mlir/Interfaces/CopyOpInterface.h"
+#include "mlir/Interfaces/InferIntRangeInterface.h"
 #include "mlir/Interfaces/InferTypeOpInterface.h"
 #include "mlir/Interfaces/MemorySlotInterfaces.h"
 #include "mlir/Interfaces/ShapedOpInterfaces.h"
diff --git a/mlir/include/mlir/Dialect/MemRef/IR/MemRefOps.td b/mlir/include/mlir/Dialect/MemRef/IR/MemRefOps.td
index a0d8d34f38237..c3ee3968abc16 100644
--- a/mlir/include/mlir/Dialect/MemRef/IR/MemRefOps.td
+++ b/mlir/include/mlir/Dialect/MemRef/IR/MemRefOps.td
@@ -14,6 +14,7 @@ include "mlir/Dialect/MemRef/IR/MemRefBase.td"
 include "mlir/Interfaces/CastInterfaces.td"
 include "mlir/Interfaces/ControlFlowInterfaces.td"
 include "mlir/Interfaces/CopyOpInterface.td"
+include "mlir/Interfaces/InferIntRangeInterface.td"
 include "mlir/Interfaces/InferTypeOpInterface.td"
 include "mlir/Interfaces/MemorySlotInterfaces.td"
 include "mlir/Interfaces/ShapedOpInterfaces.td"
@@ -577,7 +578,8 @@ def MemRef_DimOp : MemRef_Op<"dim", [
     DeclareOpInterfaceMethods<OpAsmOpInterface, ["getAsmResultNames"]>,
     MemRefsNormalizable,
     ConditionallySpeculatable, NoMemoryEffect,
-    ShapedDimOpInterface]> {
+    ShapedDimOpInterface,
+    DeclareOpInterfaceMethods<InferIntRangeInterface, ["inferResultRangesFromOptional"]>]> {
   let summary = "dimension index operation";
   let description = [{
     The `dim` operation takes a memref and a dimension operand of type `index`.
@@ -1675,7 +1677,7 @@ def MemRef_ExpandShapeOp : MemRef_ReassociativeReshapeOp<"expand_shape", [
     }]>,
 
     // Builder that infers the result layout map. The result shape must be
-    // specified. Otherwise, the op may be ambiguous. The output shape for 
+    // specified. Otherwise, the op may be ambiguous. The output shape for
     // the op will be inferred using the inferOutputShape() method.
     OpBuilder<(ins "ArrayRef<int64_t>":$resultShape, "Value":$src,
                "ArrayRef<ReassociationIndices>":$reassociation)>,
diff --git a/mlir/include/mlir/Dialect/NVGPU/IR/NVGPUDialect.h b/mlir/include/mlir/Dialect/NVGPU/IR/NVGPUDialect.h
index aad2ac6f4dd2b..db4c63b3390eb 100644
--- a/mlir/include/mlir/Dialect/NVGPU/IR/NVGPUDialect.h
+++ b/mlir/include/mlir/Dialect/NVGPU/IR/NVGPUDialect.h
@@ -22,8 +22,20 @@
 
 #include "mlir/Dialect/NVGPU/IR/NVGPUEnums.h.inc"
 
+// Maximum warp size
 constexpr int kWarpSize = 32;
 
+// Maximum number of threads in a block and block in a grid
+// https://docs.nvidia.com/cuda/cuda-c-programming-guide/#features-and-technical-specifications-technical-specifications-per-compute-capability
+constexpr int kMaxTotalBlockdim = 1024;
+constexpr int kMaxBlockdimx = 1024;
+constexpr int kMaxBlockdimy = 1024;
+constexpr int kMaxBlockdimz = 64;
+constexpr int kMaxTotalGriddim = 2147483647;
+constexpr int kMaxGriddimx = 2147483647;
+constexpr int kMaxGriddimy = 65535;
+constexpr int kMaxGriddimz = 65535;
+
 /// M size of wgmma.mma_async instruction
 constexpr int kWgmmaSizeM = 64;
 
diff --git a/mlir/include/mlir/Dialect/OpenACC/OpenACCOps.td b/mlir/include/mlir/Dialect/OpenACC/OpenACCOps.td
index c60eb5cc620a7..42da20251c190 100644
--- a/mlir/include/mlir/Dialect/OpenACC/OpenACCOps.td
+++ b/mlir/include/mlir/Dialect/OpenACC/OpenACCOps.td
@@ -65,6 +65,74 @@ def OpenACC_ReductionOperatorAttr : EnumAttr<OpenACC_Dialect,
   let assemblyFormat = [{ ```<` $value `>` }];
 }
 
+// OpenACC variable type categorization. This is needed because OpenACC
+// dialect is used with other dialects, and each dialect defines its own
+// types. Thus, in order to be able to classify types and apply right semantics,
+// it is needed to ensure the types can be categorized.
+def OpenACC_VariableTypeUncategorized : I32BitEnumAttrCaseNone<"uncategorized">;
+
+// The OpenACC spec definition of scalar type is as follows (from 3.3 spec,
+// line 5454):
+// Scalar datatype - an intrinsic or built-in datatype that is not an array or
+// aggregate datatype. In Fortran, scalar datatypes are integer, real, double
+// precision, complex, or logical. In C, scalar datatypes are char (signed or
+// unsigned), int (signed or unsigned, with optional short, long or long long
+// attribute), enum, float, double, long double, Complex (with optional float
+// or long attribute), or any pointer datatype. In C++, scalar datatypes are
+// char (signed or unsigned), wchar t, int (signed or unsigned, with optional
+// short, long or long long attribute), enum, bool, float, double, long double,
+// or any pointer datatype. Not all implementations or targets will support all
+// of these datatypes.
+// From an MLIR type perspective, the types that those language types map to
+// will be categorized as scalar.
+def OpenACC_VariableTypeScalar : I32BitEnumAttrCaseBit<"scalar", 0>;
+
+// Not in OpenACC spec glossary as its own definition but used throughout the
+// spec. One definition of array that can be assumed for purposes of type
+// categorization is that it is a collection of elements of same type.
+def OpenACC_VariableTypeArray : I32BitEnumAttrCaseBit<"array", 1>;
+
+// The OpenACC spec definition of composite type is as follows (from 3.3 spec,
+// line 5354):
+// Composite datatype - a derived type in Fortran, or a struct or union type in
+// C, or a class, struct, or union type in C++. (This is different from the use
+// of the term composite data type in the C and C++ languages.)
+def OpenACC_VariableTypeComposite : I32BitEnumAttrCaseBit<"composite", 2>;
+
+// The OpenACC spec uses the type category "aggregate" to capture both arrays
+// and composite types. However, it includes types which do not fall in either
+// of those categories. Thus create a case for the others.
+// For example, reading the definition of "Aggregate Variables" in the 3.3
+// spec line 5346 shows this distinction:
+// Aggregate variables - a variable of any non-scalar datatype, including array
+// or composite variables. In Fortran, this includes any variable with
+// allocatable or pointer attribute and character variables
+def OpenACC_VariableTypeOtherNonScalar : I32BitEnumAttrCaseBit<"nonscalar", 3>;
+
+// The OpenACC spec definition of aggregate type is as follows (from 3.3 spec,
+// line 5342):
+// Aggregate datatype - any non-scalar datatype such as array and composite
+// datatypes. In Fortran, aggregate datatypes include arrays, derived types,
+// character types. In C, aggregate datatypes include arrays, targets of
+// pointers, structs, and unions. In C++, aggregate datatypes include arrays,
+// targets of pointers, classes, structs, and unions.
+def OpenACC_VariableTypeAggregate : I32BitEnumAttrCaseGroup<"aggregate",
+  [OpenACC_VariableTypeArray, OpenACC_VariableTypeComposite,
+  OpenACC_VariableTypeOtherNonScalar]>;
+
+def OpenACC_VariableTypeCategory : I32BitEnumAttr<
+    "VariableTypeCategory",
+    "Captures different type categories described in OpenACC spec",
+    [
+      OpenACC_VariableTypeUncategorized, OpenACC_VariableTypeScalar,
+      OpenACC_VariableTypeArray, OpenACC_VariableTypeComposite,
+      OpenACC_VariableTypeOtherNonScalar, OpenACC_VariableTypeAggregate]> {
+  let separator = ",";
+  let cppNamespace = "::mlir::acc";
+  let genSpecializedAttr = 0;
+  let printBitEnumPrimaryGroups = 1;
+}
+
 // Type used in operation below.
 def IntOrIndex : AnyTypeOf<[AnyInteger, Index]>;
 
@@ -859,9 +927,9 @@ class OpenACC_DataExitOpWithVarPtr<string mnemonic, string clause>
           "- `varPtr`: The address of variable to copy back to.",
           [MemoryEffects<[MemRead<OpenACC_RuntimeCounters>,
                           MemWrite<OpenACC_RuntimeCounters>]>],
-          (ins Arg<OpenACC_PointerLikeTypeInterface,
+          (ins Arg<OpenACC_AnyPointerOrMappableType,
                    "Accelerator mapped variable", [MemRead]>:$accVar,
-              Arg<OpenACC_PointerLikeTypeInterface,
+              Arg<OpenACC_AnyPointerOrMappableType,
                   "Host variable", [MemWrite]>:$var,
               TypeAttr:$varType)> {
   let assemblyFormat = [{
@@ -917,7 +985,7 @@ class OpenACC_DataExitOpNoVarPtr<string mnemonic, string clause> :
     OpenACC_DataExitOp<mnemonic, clause, "",
       [MemoryEffects<[MemRead<OpenACC_RuntimeCounters>,
                     MemWrite<OpenACC_RuntimeCounters>]>],
-      (ins Arg<OpenACC_PointerLikeTypeInterface,"Accelerator mapped variable",
+      (ins Arg<OpenACC_AnyPointerOrMappableType,"Accelerator mapped variable",
            [MemRead]>:$accVar)> {
   let assemblyFormat = [{
     custom<AccVar>($accVar, type($accVar))
@@ -1008,8 +1076,9 @@ def OpenACC_UpdateHostOp : OpenACC_DataExitOpWithVarPtr<"update_host",
 // 2.5.13 private clause
 //===----------------------------------------------------------------------===//
 
-def OpenACC_PrivateRecipeOp : OpenACC_Op<"private.recipe",
-    [IsolatedFromAbove, Symbol, RecipeInterface]> {
+def OpenACC_PrivateRecipeOp
+    : OpenACC_Op<"private.recipe", [IsolatedFromAbove, Symbol, RecipeInterface,
+                                    AutomaticAllocationScope]> {
   let summary = "privatization recipe";
 
   let description = [{
@@ -1065,8 +1134,10 @@ def OpenACC_PrivateRecipeOp : OpenACC_Op<"private.recipe",
 // 2.5.14 firstprivate clause
 //===----------------------------------------------------------------------===//
 
-def OpenACC_FirstprivateRecipeOp : OpenACC_Op<"firstprivate.recipe",
-    [IsolatedFromAbove, Symbol, RecipeInterface]> {
+def OpenACC_FirstprivateRecipeOp
+    : OpenACC_Op<"firstprivate.recipe", [IsolatedFromAbove, Symbol,
+                                         RecipeInterface,
+                                         AutomaticAllocationScope]> {
   let summary = "privatization recipe";
 
   let description = [{
@@ -1131,8 +1202,10 @@ def OpenACC_FirstprivateRecipeOp : OpenACC_Op<"firstprivate.recipe",
 // 2.5.15 reduction clause
 //===----------------------------------------------------------------------===//
 
-def OpenACC_ReductionRecipeOp : OpenACC_Op<"reduction.recipe",
-    [IsolatedFromAbove, Symbol, RecipeInterface]> {
+def OpenACC_ReductionRecipeOp
+    : OpenACC_Op<"reduction.recipe", [IsolatedFromAbove, Symbol,
+                                      RecipeInterface,
+                                      AutomaticAllocationScope]> {
   let summary = "reduction recipe";
 
   let description = [{
diff --git a/mlir/include/mlir/Dialect/OpenACC/OpenACCTypeInterfaces.td b/mlir/include/mlir/Dialect/OpenACC/OpenACCTypeInterfaces.td
index bec46be89f058..fa66a4746f7da 100644
--- a/mlir/include/mlir/Dialect/OpenACC/OpenACCTypeInterfaces.td
+++ b/mlir/include/mlir/Dialect/OpenACC/OpenACCTypeInterfaces.td
@@ -28,6 +28,28 @@ def OpenACC_PointerLikeTypeInterface : TypeInterface<"PointerLikeType"> {
       /*retTy=*/"::mlir::Type",
       /*methodName=*/"getElementType"
     >,
+    InterfaceMethod<
+      /*description=*/[{
+        Returns the type category of the pointee. The `var` is provided because
+        a dialect's type system may be incomplete. For example, consider a
+        dialect which computes interior pointers - so a float array element
+        may be represented as `ptr<f32>`. The type system says the pointee
+        is `f32` but this is not a scalar from the point-of-view of OpenACC.
+        It is an array element and thus the appropriate type category is
+        "array" - therefore being able to look up how a variable is computed
+        is important for a complete type determination.
+        The `varType` is provided in cases where a dialect's type system
+        erased the target type.
+      }],
+      /*retTy=*/"::mlir::acc::VariableTypeCategory",
+      /*methodName=*/"getPointeeTypeCategory",
+      /*args=*/(ins "::mlir::TypedValue<::mlir::acc::PointerLikeType>":$varPtr,
+                    "::mlir::Type":$varType),
+      /*methodBody=*/"",
+      /*defaultImplementation=*/[{
+        return ::mlir::acc::VariableTypeCategory::uncategorized;
+      }]
+    >,
   ];
 }
 
@@ -106,7 +128,7 @@ def OpenACC_MappableTypeInterface : TypeInterface<"MappableType"> {
         return {};
       }]
     >,
-      InterfaceMethod<
+    InterfaceMethod<
       /*description=*/[{
         Returns explicit `acc.bounds` operations that envelop the whole
         data structure. These operations are inserted using the provided builder
@@ -121,6 +143,18 @@ def OpenACC_MappableTypeInterface : TypeInterface<"MappableType"> {
         return {};
       }]
     >,
+    InterfaceMethod<
+      /*description=*/[{
+        Returns the OpenACC type category.
+      }],
+      /*retTy=*/"::mlir::acc::VariableTypeCategory",
+      /*methodName=*/"getTypeCategory",
+      /*args=*/(ins "::mlir::Value":$var),
+      /*methodBody=*/"",
+      /*defaultImplementation=*/[{
+        return ::mlir::acc::VariableTypeCategory::uncategorized;
+      }]
+    >,
   ];
 }
 
diff --git a/mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td b/mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td
index 580c9c6ef6fde..983627027ac9c 100644
--- a/mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td
+++ b/mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td
@@ -34,31 +34,27 @@ def PrivateClauseOp : OpenMP_Op<"private", [IsolatedFromAbove, RecipeInterface]>
   let description = [{
     This operation provides a declaration of how to implement the
     [first]privatization of a variable. The dialect users should provide
-    information about how to create an instance of the type in the alloc region,
-    how to initialize the copy from the original item in the copy region, and if
-    needed, how to deallocate allocated memory in the dealloc region.
+    which type should be allocated for this variable. The allocated (usually by
+    alloca) variable is passed to the initialization region which does everything
+    else (e.g. initialization of Fortran runtime descriptors). Information about
+    how to initialize the copy from the original item should be given in the
+    copy region, and if needed, how to deallocate memory (allocated by the
+    initialization region) in the dealloc region.
 
     Examples:
 
-    * `private(x)` would be emitted as:
+    * `private(x)` would not need any regions because no initialization is
+      required by the standard for i32 variables and this is not firstprivate.
     ```mlir
-    omp.private {type = private} @x.privatizer : !fir.ref<i32> alloc {
-    ^bb0(%arg0: !fir.ref<i32>):
-    %0 = ... allocate proper memory for the private clone ...
-    omp.yield(%0 : !fir.ref<i32>)
-    }
+    omp.private {type = private} @x.privatizer : i32
     ```
 
     * `firstprivate(x)` would be emitted as:
     ```mlir
-    omp.private {type = firstprivate} @x.privatizer : !fir.ref<i32> alloc {
-    ^bb0(%arg0: !fir.ref<i32>):
-    %0 = ... allocate proper memory for the private clone ...
-    omp.yield(%0 : !fir.ref<i32>)
-    } copy {
+    omp.private {type = firstprivate} @x.privatizer : i32 copy {
     ^bb0(%arg0: !fir.ref<i32>, %arg1: !fir.ref<i32>):
-    // %arg0 is the original host variable. Same as for `alloc`.
-    // %arg1 represents the memory allocated in `alloc`.
+    // %arg0 is the original host variable.
+    // %arg1 represents the memory allocated for this private variable.
     ... copy from host to the privatized clone ....
     omp.yield(%arg1 : !fir.ref<i32>)
     }
@@ -66,20 +62,26 @@ def PrivateClauseOp : OpenMP_Op<"private", [IsolatedFromAbove, RecipeInterface]>
 
     * `private(x)` for "allocatables" would be emitted as:
     ```mlir
-    omp.private {type = private} @x.privatizer : !some.type alloc {
-    ^bb0(%arg0: !some.type):
-    %0 = ... allocate proper memory for the private clone ...
-    omp.yield(%0 : !fir.ref<i32>)
+    omp.private {type = private} @x.privatizer : !some.type init {
+    ^bb0(%arg0: !some.pointer<!some.type>, %arg1: !some.pointer<!some.type>):
+    // initialize %arg1, using %arg0 as a mold for allocations.
+    // For example if %arg0 is a heap allocated array with a runtime determined
+    // length and !some.type is a runtime type descriptor, the init region
+    // will read the array length from %arg0, and heap allocate an array of the
+    // right length and initialize %arg1 to contain the array allocation and
+    // length.
+    omp.yield(%arg1 : !some.pointer<!some.type>)
     } dealloc {
-    ^bb0(%arg0: !some.type):
-    ... deallocate allocated memory ...
+    ^bb0(%arg0: !some.pointer<!some.type>):
+    // ... deallocate memory allocated by the init region...
+    // In the example above, this will free the heap allocated array data.
     omp.yield
     }
     ```
 
     There are no restrictions on the body except for:
-    - The `alloc` & `dealloc` regions have a single argument.
-    - The `copy` region has 2 arguments.
+    - The `dealloc` regions has a single argument.
+    - The `init & `copy` regions have 2 arguments.
     - All three regions are terminated by `omp.yield` ops.
     The above restrictions and other obvious restrictions (e.g. verifying the
     type of yielded values) are verified by the custom op verifier. The actual
@@ -91,7 +93,10 @@ def PrivateClauseOp : OpenMP_Op<"private", [IsolatedFromAbove, RecipeInterface]>
     The $sym_name attribute provides a symbol by which the privatizer op can be
     referenced by other dialect ops.
 
-    The $type attribute is the type of the value being privatized.
+    The $type attribute is the type of the value being privatized. This type
+    will be implicitly allocated in MLIR->LLVMIR conversion and passed as the
+    second argument to the init region. Therefore the type of arguments to
+    the regions should be a type which represents a pointer to $type.
 
     The $data_sharing_type attribute specifies whether privatizer corresponds
     to a `private` or a `firstprivate` clause.
@@ -101,13 +106,13 @@ def PrivateClauseOp : OpenMP_Op<"private", [IsolatedFromAbove, RecipeInterface]>
                        TypeAttrOf<AnyType>:$type,
                        DataSharingClauseTypeAttr:$data_sharing_type);
 
-  let regions = (region MinSizedRegion<1>:$alloc_region,
+  let regions = (region AnyRegion:$init_region,
                         AnyRegion:$copy_region,
                         AnyRegion:$dealloc_region);
 
   let assemblyFormat = [{
     $data_sharing_type $sym_name `:` $type
-      `alloc` $alloc_region
+      (`init` $init_region^)?
       (`copy` $copy_region^)?
       (`dealloc` $dealloc_region^)?
       attr-dict
@@ -120,8 +125,13 @@ def PrivateClauseOp : OpenMP_Op<"private", [IsolatedFromAbove, RecipeInterface]>
   ];
 
   let extraClassDeclaration = [{
-    BlockArgument getAllocMoldArg() {
-      return getAllocRegion().getArgument(0);
+    BlockArgument getInitMoldArg() {
+      auto &region = getInitRegion();
+      return region.empty() ? nullptr : region.getArgument(0);
+    }
+    BlockArgument getInitPrivateArg() {
+      auto &region = getInitRegion();
+      return region.empty() ? nullptr : region.getArgument(1);
     }
     BlockArgument getCopyMoldArg() {
       auto &region = getCopyRegion();
@@ -141,7 +151,20 @@ def PrivateClauseOp : OpenMP_Op<"private", [IsolatedFromAbove, RecipeInterface]>
     /// when an allocatable is privatized. In such cases, the descriptor is used
     /// in privatization and needs to be mapped on to the device.
     bool needsMap() {
-      return !getAllocMoldArg().use_empty();
+      BlockArgument moldArg = getInitMoldArg();
+      return moldArg ? !moldArg.use_empty() : false;
+    }
+
+    /// Get the type for arguments to nested regions. This should
+    /// generally be either the same as getType() or some pointer
+    /// type (pointing to the type allocated by this op).
+    /// This method will return Type{nullptr} if there are no nested
+    /// regions.
+    Type getArgType() {
+      for (Region *region : getRegions())
+        for (Type ty : region->getArgumentTypes())
+          return ty;
+      return nullptr;
     }
   }];
 
diff --git a/mlir/include/mlir/Dialect/Polynomial/IR/PolynomialTypes.td b/mlir/include/mlir/Dialect/Polynomial/IR/PolynomialTypes.td
index 89e406183e0b0..cf33503764abb 100644
--- a/mlir/include/mlir/Dialect/Polynomial/IR/PolynomialTypes.td
+++ b/mlir/include/mlir/Dialect/Polynomial/IR/PolynomialTypes.td
@@ -26,7 +26,8 @@ def Polynomial_PolynomialType : Polynomial_Type<"Polynomial", "polynomial"> {
   let assemblyFormat = "`<` struct(params) `>`";
 }
 
-def PolynomialLike: TypeOrContainer<Polynomial_PolynomialType, "polynomial-like">;
+def PolynomialLike : TypeOrValueSemanticsContainer<
+    Polynomial_PolynomialType, "polynomial-like">;
 
 
 #endif // POLYNOMIAL_TYPES
diff --git a/mlir/include/mlir/Dialect/SPIRV/IR/SPIRVBase.td b/mlir/include/mlir/Dialect/SPIRV/IR/SPIRVBase.td
index c84677d26a8b6..6b2e4189aea02 100644
--- a/mlir/include/mlir/Dialect/SPIRV/IR/SPIRVBase.td
+++ b/mlir/include/mlir/Dialect/SPIRV/IR/SPIRVBase.td
@@ -4234,8 +4234,13 @@ class SPIRV_CoopMatrixOfType<list<Type> allowedTypes> :
     "::llvm::cast<::mlir::spirv::CooperativeMatrixType>($_self).getElementType()",
     "Cooperative Matrix">;
 
+class SPIRV_MatrixOfType<list<Type> allowedTypes> :
+  ContainerType<AnyTypeOf<allowedTypes>, SPIRV_IsMatrixType,
+    "::llvm::cast<::mlir::spirv::MatrixType>($_self).getElementType()",
+    "Matrix">;
+
 class SPIRV_VectorOf<Type type> :
-    VectorOfLengthAndType<[2, 3, 4, 8,16], [type]>;
+    VectorOfLengthAndType<[2, 3, 4, 8, 16], [type]>;
 
 class SPIRV_ScalarOrVectorOf<Type type> :
     AnyTypeOf<[type, SPIRV_VectorOf<type>]>;
@@ -4248,6 +4253,9 @@ class SPIRV_MatrixOrCoopMatrixOf<Type type> :
     AnyTypeOf<[SPIRV_AnyMatrix,
                SPIRV_CoopMatrixOfType<[type]>]>;
 
+class SPIRV_MatrixOf<Type type> :
+    SPIRV_MatrixOfType<[type]>;
+
 def SPIRV_ScalarOrVector : AnyTypeOf<[SPIRV_Scalar, SPIRV_Vector]>;
 def SPIRV_ScalarOrVectorOrPtr : AnyTypeOf<[SPIRV_ScalarOrVector, SPIRV_AnyPtr]>;
 
@@ -4293,236 +4301,239 @@ class SPIRV_OpCode<string name, int val> {
 
 // Begin opcode section. Generated from SPIR-V spec; DO NOT MODIFY!
 
-def SPIRV_OC_OpNop                          : I32EnumAttrCase<"OpNop", 0>;
-def SPIRV_OC_OpUndef                        : I32EnumAttrCase<"OpUndef", 1>;
-def SPIRV_OC_OpSourceContinued              : I32EnumAttrCase<"OpSourceContinued", 2>;
-def SPIRV_OC_OpSource                       : I32EnumAttrCase<"OpSource", 3>;
-def SPIRV_OC_OpSourceExtension              : I32EnumAttrCase<"OpSourceExtension", 4>;
-def SPIRV_OC_OpName                         : I32EnumAttrCase<"OpName", 5>;
-def SPIRV_OC_OpMemberName                   : I32EnumAttrCase<"OpMemberName", 6>;
-def SPIRV_OC_OpString                       : I32EnumAttrCase<"OpString", 7>;
-def SPIRV_OC_OpLine                         : I32EnumAttrCase<"OpLine", 8>;
-def SPIRV_OC_OpExtension                    : I32EnumAttrCase<"OpExtension", 10>;
-def SPIRV_OC_OpExtInstImport                : I32EnumAttrCase<"OpExtInstImport", 11>;
-def SPIRV_OC_OpExtInst                      : I32EnumAttrCase<"OpExtInst", 12>;
-def SPIRV_OC_OpMemoryModel                  : I32EnumAttrCase<"OpMemoryModel", 14>;
-def SPIRV_OC_OpEntryPoint                   : I32EnumAttrCase<"OpEntryPoint", 15>;
-def SPIRV_OC_OpExecutionMode                : I32EnumAttrCase<"OpExecutionMode", 16>;
-def SPIRV_OC_OpCapability                   : I32EnumAttrCase<"OpCapability", 17>;
-def SPIRV_OC_OpTypeVoid                     : I32EnumAttrCase<"OpTypeVoid", 19>;
-def SPIRV_OC_OpTypeBool                     : I32EnumAttrCase<"OpTypeBool", 20>;
-def SPIRV_OC_OpTypeInt                      : I32EnumAttrCase<"OpTypeInt", 21>;
-def SPIRV_OC_OpTypeFloat                    : I32EnumAttrCase<"OpTypeFloat", 22>;
-def SPIRV_OC_OpTypeVector                   : I32EnumAttrCase<"OpTypeVector", 23>;
-def SPIRV_OC_OpTypeMatrix                   : I32EnumAttrCase<"OpTypeMatrix", 24>;
-def SPIRV_OC_OpTypeImage                    : I32EnumAttrCase<"OpTypeImage", 25>;
-def SPIRV_OC_OpTypeSampledImage             : I32EnumAttrCase<"OpTypeSampledImage", 27>;
-def SPIRV_OC_OpTypeArray                    : I32EnumAttrCase<"OpTypeArray", 28>;
-def SPIRV_OC_OpTypeRuntimeArray             : I32EnumAttrCase<"OpTypeRuntimeArray", 29>;
-def SPIRV_OC_OpTypeStruct                   : I32EnumAttrCase<"OpTypeStruct", 30>;
-def SPIRV_OC_OpTypePointer                  : I32EnumAttrCase<"OpTypePointer", 32>;
-def SPIRV_OC_OpTypeFunction                 : I32EnumAttrCase<"OpTypeFunction", 33>;
-def SPIRV_OC_OpTypeForwardPointer           : I32EnumAttrCase<"OpTypeForwardPointer", 39>;
-def SPIRV_OC_OpConstantTrue                 : I32EnumAttrCase<"OpConstantTrue", 41>;
-def SPIRV_OC_OpConstantFalse                : I32EnumAttrCase<"OpConstantFalse", 42>;
-def SPIRV_OC_OpConstant                     : I32EnumAttrCase<"OpConstant", 43>;
-def SPIRV_OC_OpConstantComposite            : I32EnumAttrCase<"OpConstantComposite", 44>;
-def SPIRV_OC_OpConstantNull                 : I32EnumAttrCase<"OpConstantNull", 46>;
-def SPIRV_OC_OpSpecConstantTrue             : I32EnumAttrCase<"OpSpecConstantTrue", 48>;
-def SPIRV_OC_OpSpecConstantFalse            : I32EnumAttrCase<"OpSpecConstantFalse", 49>;
-def SPIRV_OC_OpSpecConstant                 : I32EnumAttrCase<"OpSpecConstant", 50>;
-def SPIRV_OC_OpSpecConstantComposite        : I32EnumAttrCase<"OpSpecConstantComposite", 51>;
-def SPIRV_OC_OpSpecConstantOp               : I32EnumAttrCase<"OpSpecConstantOp", 52>;
-def SPIRV_OC_OpFunction                     : I32EnumAttrCase<"OpFunction", 54>;
-def SPIRV_OC_OpFunctionParameter            : I32EnumAttrCase<"OpFunctionParameter", 55>;
-def SPIRV_OC_OpFunctionEnd                  : I32EnumAttrCase<"OpFunctionEnd", 56>;
-def SPIRV_OC_OpFunctionCall                 : I32EnumAttrCase<"OpFunctionCall", 57>;
-def SPIRV_OC_OpVariable                     : I32EnumAttrCase<"OpVariable", 59>;
-def SPIRV_OC_OpLoad                         : I32EnumAttrCase<"OpLoad", 61>;
-def SPIRV_OC_OpStore                        : I32EnumAttrCase<"OpStore", 62>;
-def SPIRV_OC_OpCopyMemory                   : I32EnumAttrCase<"OpCopyMemory", 63>;
-def SPIRV_OC_OpAccessChain                  : I32EnumAttrCase<"OpAccessChain", 65>;
-def SPIRV_OC_OpPtrAccessChain               : I32EnumAttrCase<"OpPtrAccessChain", 67>;
-def SPIRV_OC_OpInBoundsPtrAccessChain       : I32EnumAttrCase<"OpInBoundsPtrAccessChain", 70>;
-def SPIRV_OC_OpDecorate                     : I32EnumAttrCase<"OpDecorate", 71>;
-def SPIRV_OC_OpMemberDecorate               : I32EnumAttrCase<"OpMemberDecorate", 72>;
-def SPIRV_OC_OpVectorExtractDynamic         : I32EnumAttrCase<"OpVectorExtractDynamic", 77>;
-def SPIRV_OC_OpVectorInsertDynamic          : I32EnumAttrCase<"OpVectorInsertDynamic", 78>;
-def SPIRV_OC_OpVectorShuffle                : I32EnumAttrCase<"OpVectorShuffle", 79>;
-def SPIRV_OC_OpCompositeConstruct           : I32EnumAttrCase<"OpCompositeConstruct", 80>;
-def SPIRV_OC_OpCompositeExtract             : I32EnumAttrCase<"OpCompositeExtract", 81>;
-def SPIRV_OC_OpCompositeInsert              : I32EnumAttrCase<"OpCompositeInsert", 82>;
-def SPIRV_OC_OpTranspose                    : I32EnumAttrCase<"OpTranspose", 84>;
-def SPIRV_OC_OpImageDrefGather              : I32EnumAttrCase<"OpImageDrefGather", 97>;
-def SPIRV_OC_OpImage                        : I32EnumAttrCase<"OpImage", 100>;
-def SPIRV_OC_OpImageQuerySize               : I32EnumAttrCase<"OpImageQuerySize", 104>;
-def SPIRV_OC_OpConvertFToU                  : I32EnumAttrCase<"OpConvertFToU", 109>;
-def SPIRV_OC_OpConvertFToS                  : I32EnumAttrCase<"OpConvertFToS", 110>;
-def SPIRV_OC_OpConvertSToF                  : I32EnumAttrCase<"OpConvertSToF", 111>;
-def SPIRV_OC_OpConvertUToF                  : I32EnumAttrCase<"OpConvertUToF", 112>;
-def SPIRV_OC_OpUConvert                     : I32EnumAttrCase<"OpUConvert", 113>;
-def SPIRV_OC_OpSConvert                     : I32EnumAttrCase<"OpSConvert", 114>;
-def SPIRV_OC_OpFConvert                     : I32EnumAttrCase<"OpFConvert", 115>;
-def SPIRV_OC_OpConvertPtrToU                : I32EnumAttrCase<"OpConvertPtrToU", 117>;
-def SPIRV_OC_OpConvertUToPtr                : I32EnumAttrCase<"OpConvertUToPtr", 120>;
-def SPIRV_OC_OpPtrCastToGeneric             : I32EnumAttrCase<"OpPtrCastToGeneric", 121>;
-def SPIRV_OC_OpGenericCastToPtr             : I32EnumAttrCase<"OpGenericCastToPtr", 122>;
-def SPIRV_OC_OpGenericCastToPtrExplicit     : I32EnumAttrCase<"OpGenericCastToPtrExplicit", 123>;
-def SPIRV_OC_OpBitcast                      : I32EnumAttrCase<"OpBitcast", 124>;
-def SPIRV_OC_OpSNegate                      : I32EnumAttrCase<"OpSNegate", 126>;
-def SPIRV_OC_OpFNegate                      : I32EnumAttrCase<"OpFNegate", 127>;
-def SPIRV_OC_OpIAdd                         : I32EnumAttrCase<"OpIAdd", 128>;
-def SPIRV_OC_OpFAdd                         : I32EnumAttrCase<"OpFAdd", 129>;
-def SPIRV_OC_OpISub                         : I32EnumAttrCase<"OpISub", 130>;
-def SPIRV_OC_OpFSub                         : I32EnumAttrCase<"OpFSub", 131>;
-def SPIRV_OC_OpIMul                         : I32EnumAttrCase<"OpIMul", 132>;
-def SPIRV_OC_OpFMul                         : I32EnumAttrCase<"OpFMul", 133>;
-def SPIRV_OC_OpUDiv                         : I32EnumAttrCase<"OpUDiv", 134>;
-def SPIRV_OC_OpSDiv                         : I32EnumAttrCase<"OpSDiv", 135>;
-def SPIRV_OC_OpFDiv                         : I32EnumAttrCase<"OpFDiv", 136>;
-def SPIRV_OC_OpUMod                         : I32EnumAttrCase<"OpUMod", 137>;
-def SPIRV_OC_OpSRem                         : I32EnumAttrCase<"OpSRem", 138>;
-def SPIRV_OC_OpSMod                         : I32EnumAttrCase<"OpSMod", 139>;
-def SPIRV_OC_OpFRem                         : I32EnumAttrCase<"OpFRem", 140>;
-def SPIRV_OC_OpFMod                         : I32EnumAttrCase<"OpFMod", 141>;
-def SPIRV_OC_OpVectorTimesScalar            : I32EnumAttrCase<"OpVectorTimesScalar", 142>;
-def SPIRV_OC_OpMatrixTimesScalar            : I32EnumAttrCase<"OpMatrixTimesScalar", 143>;
-def SPIRV_OC_OpMatrixTimesVector           : I32EnumAttrCase<"OpMatrixTimesVector", 145>;
-def SPIRV_OC_OpMatrixTimesMatrix            : I32EnumAttrCase<"OpMatrixTimesMatrix", 146>;
-def SPIRV_OC_OpDot                          : I32EnumAttrCase<"OpDot", 148>;
-def SPIRV_OC_OpIAddCarry                    : I32EnumAttrCase<"OpIAddCarry", 149>;
-def SPIRV_OC_OpISubBorrow                   : I32EnumAttrCase<"OpISubBorrow", 150>;
-def SPIRV_OC_OpUMulExtended                 : I32EnumAttrCase<"OpUMulExtended", 151>;
-def SPIRV_OC_OpSMulExtended                 : I32EnumAttrCase<"OpSMulExtended", 152>;
-def SPIRV_OC_OpIsNan                        : I32EnumAttrCase<"OpIsNan", 156>;
-def SPIRV_OC_OpIsInf                        : I32EnumAttrCase<"OpIsInf", 157>;
-def SPIRV_OC_OpOrdered                      : I32EnumAttrCase<"OpOrdered", 162>;
-def SPIRV_OC_OpUnordered                    : I32EnumAttrCase<"OpUnordered", 163>;
-def SPIRV_OC_OpLogicalEqual                 : I32EnumAttrCase<"OpLogicalEqual", 164>;
-def SPIRV_OC_OpLogicalNotEqual              : I32EnumAttrCase<"OpLogicalNotEqual", 165>;
-def SPIRV_OC_OpLogicalOr                    : I32EnumAttrCase<"OpLogicalOr", 166>;
-def SPIRV_OC_OpLogicalAnd                   : I32EnumAttrCase<"OpLogicalAnd", 167>;
-def SPIRV_OC_OpLogicalNot                   : I32EnumAttrCase<"OpLogicalNot", 168>;
-def SPIRV_OC_OpSelect                       : I32EnumAttrCase<"OpSelect", 169>;
-def SPIRV_OC_OpIEqual                       : I32EnumAttrCase<"OpIEqual", 170>;
-def SPIRV_OC_OpINotEqual                    : I32EnumAttrCase<"OpINotEqual", 171>;
-def SPIRV_OC_OpUGreaterThan                 : I32EnumAttrCase<"OpUGreaterThan", 172>;
-def SPIRV_OC_OpSGreaterThan                 : I32EnumAttrCase<"OpSGreaterThan", 173>;
-def SPIRV_OC_OpUGreaterThanEqual            : I32EnumAttrCase<"OpUGreaterThanEqual", 174>;
-def SPIRV_OC_OpSGreaterThanEqual            : I32EnumAttrCase<"OpSGreaterThanEqual", 175>;
-def SPIRV_OC_OpULessThan                    : I32EnumAttrCase<"OpULessThan", 176>;
-def SPIRV_OC_OpSLessThan                    : I32EnumAttrCase<"OpSLessThan", 177>;
-def SPIRV_OC_OpULessThanEqual               : I32EnumAttrCase<"OpULessThanEqual", 178>;
-def SPIRV_OC_OpSLessThanEqual               : I32EnumAttrCase<"OpSLessThanEqual", 179>;
-def SPIRV_OC_OpFOrdEqual                    : I32EnumAttrCase<"OpFOrdEqual", 180>;
-def SPIRV_OC_OpFUnordEqual                  : I32EnumAttrCase<"OpFUnordEqual", 181>;
-def SPIRV_OC_OpFOrdNotEqual                 : I32EnumAttrCase<"OpFOrdNotEqual", 182>;
-def SPIRV_OC_OpFUnordNotEqual               : I32EnumAttrCase<"OpFUnordNotEqual", 183>;
-def SPIRV_OC_OpFOrdLessThan                 : I32EnumAttrCase<"OpFOrdLessThan", 184>;
-def SPIRV_OC_OpFUnordLessThan               : I32EnumAttrCase<"OpFUnordLessThan", 185>;
-def SPIRV_OC_OpFOrdGreaterThan              : I32EnumAttrCase<"OpFOrdGreaterThan", 186>;
-def SPIRV_OC_OpFUnordGreaterThan            : I32EnumAttrCase<"OpFUnordGreaterThan", 187>;
-def SPIRV_OC_OpFOrdLessThanEqual            : I32EnumAttrCase<"OpFOrdLessThanEqual", 188>;
-def SPIRV_OC_OpFUnordLessThanEqual          : I32EnumAttrCase<"OpFUnordLessThanEqual", 189>;
-def SPIRV_OC_OpFOrdGreaterThanEqual         : I32EnumAttrCase<"OpFOrdGreaterThanEqual", 190>;
-def SPIRV_OC_OpFUnordGreaterThanEqual       : I32EnumAttrCase<"OpFUnordGreaterThanEqual", 191>;
-def SPIRV_OC_OpShiftRightLogical            : I32EnumAttrCase<"OpShiftRightLogical", 194>;
-def SPIRV_OC_OpShiftRightArithmetic         : I32EnumAttrCase<"OpShiftRightArithmetic", 195>;
-def SPIRV_OC_OpShiftLeftLogical             : I32EnumAttrCase<"OpShiftLeftLogical", 196>;
-def SPIRV_OC_OpBitwiseOr                    : I32EnumAttrCase<"OpBitwiseOr", 197>;
-def SPIRV_OC_OpBitwiseXor                   : I32EnumAttrCase<"OpBitwiseXor", 198>;
-def SPIRV_OC_OpBitwiseAnd                   : I32EnumAttrCase<"OpBitwiseAnd", 199>;
-def SPIRV_OC_OpNot                          : I32EnumAttrCase<"OpNot", 200>;
-def SPIRV_OC_OpBitFieldInsert               : I32EnumAttrCase<"OpBitFieldInsert", 201>;
-def SPIRV_OC_OpBitFieldSExtract             : I32EnumAttrCase<"OpBitFieldSExtract", 202>;
-def SPIRV_OC_OpBitFieldUExtract             : I32EnumAttrCase<"OpBitFieldUExtract", 203>;
-def SPIRV_OC_OpBitReverse                   : I32EnumAttrCase<"OpBitReverse", 204>;
-def SPIRV_OC_OpBitCount                     : I32EnumAttrCase<"OpBitCount", 205>;
-def SPIRV_OC_OpEmitVertex                   : I32EnumAttrCase<"OpEmitVertex", 218>;
-def SPIRV_OC_OpEndPrimitive                 : I32EnumAttrCase<"OpEndPrimitive", 219>;
-def SPIRV_OC_OpControlBarrier               : I32EnumAttrCase<"OpControlBarrier", 224>;
-def SPIRV_OC_OpMemoryBarrier                : I32EnumAttrCase<"OpMemoryBarrier", 225>;
-def SPIRV_OC_OpAtomicExchange               : I32EnumAttrCase<"OpAtomicExchange", 229>;
-def SPIRV_OC_OpAtomicCompareExchange        : I32EnumAttrCase<"OpAtomicCompareExchange", 230>;
-def SPIRV_OC_OpAtomicCompareExchangeWeak    : I32EnumAttrCase<"OpAtomicCompareExchangeWeak", 231>;
-def SPIRV_OC_OpAtomicIIncrement             : I32EnumAttrCase<"OpAtomicIIncrement", 232>;
-def SPIRV_OC_OpAtomicIDecrement             : I32EnumAttrCase<"OpAtomicIDecrement", 233>;
-def SPIRV_OC_OpAtomicIAdd                   : I32EnumAttrCase<"OpAtomicIAdd", 234>;
-def SPIRV_OC_OpAtomicISub                   : I32EnumAttrCase<"OpAtomicISub", 235>;
-def SPIRV_OC_OpAtomicSMin                   : I32EnumAttrCase<"OpAtomicSMin", 236>;
-def SPIRV_OC_OpAtomicUMin                   : I32EnumAttrCase<"OpAtomicUMin", 237>;
-def SPIRV_OC_OpAtomicSMax                   : I32EnumAttrCase<"OpAtomicSMax", 238>;
-def SPIRV_OC_OpAtomicUMax                   : I32EnumAttrCase<"OpAtomicUMax", 239>;
-def SPIRV_OC_OpAtomicAnd                    : I32EnumAttrCase<"OpAtomicAnd", 240>;
-def SPIRV_OC_OpAtomicOr                     : I32EnumAttrCase<"OpAtomicOr", 241>;
-def SPIRV_OC_OpAtomicXor                    : I32EnumAttrCase<"OpAtomicXor", 242>;
-def SPIRV_OC_OpPhi                          : I32EnumAttrCase<"OpPhi", 245>;
-def SPIRV_OC_OpLoopMerge                    : I32EnumAttrCase<"OpLoopMerge", 246>;
-def SPIRV_OC_OpSelectionMerge               : I32EnumAttrCase<"OpSelectionMerge", 247>;
-def SPIRV_OC_OpLabel                        : I32EnumAttrCase<"OpLabel", 248>;
-def SPIRV_OC_OpBranch                       : I32EnumAttrCase<"OpBranch", 249>;
-def SPIRV_OC_OpBranchConditional            : I32EnumAttrCase<"OpBranchConditional", 250>;
-def SPIRV_OC_OpReturn                       : I32EnumAttrCase<"OpReturn", 253>;
-def SPIRV_OC_OpReturnValue                  : I32EnumAttrCase<"OpReturnValue", 254>;
-def SPIRV_OC_OpUnreachable                  : I32EnumAttrCase<"OpUnreachable", 255>;
-def SPIRV_OC_OpGroupBroadcast               : I32EnumAttrCase<"OpGroupBroadcast", 263>;
-def SPIRV_OC_OpGroupIAdd                    : I32EnumAttrCase<"OpGroupIAdd", 264>;
-def SPIRV_OC_OpGroupFAdd                    : I32EnumAttrCase<"OpGroupFAdd", 265>;
-def SPIRV_OC_OpGroupFMin                    : I32EnumAttrCase<"OpGroupFMin", 266>;
-def SPIRV_OC_OpGroupUMin                    : I32EnumAttrCase<"OpGroupUMin", 267>;
-def SPIRV_OC_OpGroupSMin                    : I32EnumAttrCase<"OpGroupSMin", 268>;
-def SPIRV_OC_OpGroupFMax                    : I32EnumAttrCase<"OpGroupFMax", 269>;
-def SPIRV_OC_OpGroupUMax                    : I32EnumAttrCase<"OpGroupUMax", 270>;
-def SPIRV_OC_OpGroupSMax                    : I32EnumAttrCase<"OpGroupSMax", 271>;
-def SPIRV_OC_OpNoLine                       : I32EnumAttrCase<"OpNoLine", 317>;
-def SPIRV_OC_OpModuleProcessed              : I32EnumAttrCase<"OpModuleProcessed", 330>;
-def SPIRV_OC_OpGroupNonUniformElect         : I32EnumAttrCase<"OpGroupNonUniformElect", 333>;
-def SPIRV_OC_OpGroupNonUniformBroadcast     : I32EnumAttrCase<"OpGroupNonUniformBroadcast", 337>;
-def SPIRV_OC_OpGroupNonUniformBallot        : I32EnumAttrCase<"OpGroupNonUniformBallot", 339>;
-def SPIRV_OC_OpGroupNonUniformBallotFindLSB : I32EnumAttrCase<"OpGroupNonUniformBallotFindLSB", 343>;
-def SPIRV_OC_OpGroupNonUniformBallotFindMSB : I32EnumAttrCase<"OpGroupNonUniformBallotFindMSB", 344>;
-def SPIRV_OC_OpGroupNonUniformShuffle       : I32EnumAttrCase<"OpGroupNonUniformShuffle", 345>;
-def SPIRV_OC_OpGroupNonUniformShuffleXor    : I32EnumAttrCase<"OpGroupNonUniformShuffleXor", 346>;
-def SPIRV_OC_OpGroupNonUniformShuffleUp     : I32EnumAttrCase<"OpGroupNonUniformShuffleUp", 347>;
-def SPIRV_OC_OpGroupNonUniformShuffleDown   : I32EnumAttrCase<"OpGroupNonUniformShuffleDown", 348>;
-def SPIRV_OC_OpGroupNonUniformIAdd          : I32EnumAttrCase<"OpGroupNonUniformIAdd", 349>;
-def SPIRV_OC_OpGroupNonUniformFAdd          : I32EnumAttrCase<"OpGroupNonUniformFAdd", 350>;
-def SPIRV_OC_OpGroupNonUniformIMul          : I32EnumAttrCase<"OpGroupNonUniformIMul", 351>;
-def SPIRV_OC_OpGroupNonUniformFMul          : I32EnumAttrCase<"OpGroupNonUniformFMul", 352>;
-def SPIRV_OC_OpGroupNonUniformSMin          : I32EnumAttrCase<"OpGroupNonUniformSMin", 353>;
-def SPIRV_OC_OpGroupNonUniformUMin          : I32EnumAttrCase<"OpGroupNonUniformUMin", 354>;
-def SPIRV_OC_OpGroupNonUniformFMin          : I32EnumAttrCase<"OpGroupNonUniformFMin", 355>;
-def SPIRV_OC_OpGroupNonUniformSMax          : I32EnumAttrCase<"OpGroupNonUniformSMax", 356>;
-def SPIRV_OC_OpGroupNonUniformUMax          : I32EnumAttrCase<"OpGroupNonUniformUMax", 357>;
-def SPIRV_OC_OpGroupNonUniformFMax          : I32EnumAttrCase<"OpGroupNonUniformFMax", 358>;
-def SPIRV_OC_OpGroupNonUniformBitwiseAnd    : I32EnumAttrCase<"OpGroupNonUniformBitwiseAnd", 359>;
-def SPIRV_OC_OpGroupNonUniformBitwiseOr     : I32EnumAttrCase<"OpGroupNonUniformBitwiseOr", 360>;
-def SPIRV_OC_OpGroupNonUniformBitwiseXor    : I32EnumAttrCase<"OpGroupNonUniformBitwiseXor", 361>;
-def SPIRV_OC_OpGroupNonUniformLogicalAnd    : I32EnumAttrCase<"OpGroupNonUniformLogicalAnd", 362>;
-def SPIRV_OC_OpGroupNonUniformLogicalOr     : I32EnumAttrCase<"OpGroupNonUniformLogicalOr", 363>;
-def SPIRV_OC_OpGroupNonUniformLogicalXor    : I32EnumAttrCase<"OpGroupNonUniformLogicalXor", 364>;
-def SPIRV_OC_OpSubgroupBallotKHR            : I32EnumAttrCase<"OpSubgroupBallotKHR", 4421>;
-def SPIRV_OC_OpSDot                         : I32EnumAttrCase<"OpSDot", 4450>;
-def SPIRV_OC_OpUDot                         : I32EnumAttrCase<"OpUDot", 4451>;
-def SPIRV_OC_OpSUDot                        : I32EnumAttrCase<"OpSUDot", 4452>;
-def SPIRV_OC_OpSDotAccSat                   : I32EnumAttrCase<"OpSDotAccSat", 4453>;
-def SPIRV_OC_OpUDotAccSat                   : I32EnumAttrCase<"OpUDotAccSat", 4454>;
-def SPIRV_OC_OpSUDotAccSat                  : I32EnumAttrCase<"OpSUDotAccSat", 4455>;
-def SPIRV_OC_OpTypeCooperativeMatrixKHR     : I32EnumAttrCase<"OpTypeCooperativeMatrixKHR", 4456>;
-def SPIRV_OC_OpCooperativeMatrixLoadKHR     : I32EnumAttrCase<"OpCooperativeMatrixLoadKHR", 4457>;
-def SPIRV_OC_OpCooperativeMatrixStoreKHR    : I32EnumAttrCase<"OpCooperativeMatrixStoreKHR", 4458>;
-def SPIRV_OC_OpCooperativeMatrixMulAddKHR   : I32EnumAttrCase<"OpCooperativeMatrixMulAddKHR", 4459>;
-def SPIRV_OC_OpCooperativeMatrixLengthKHR   : I32EnumAttrCase<"OpCooperativeMatrixLengthKHR", 4460>;
-def SPIRV_OC_OpSubgroupBlockReadINTEL       : I32EnumAttrCase<"OpSubgroupBlockReadINTEL", 5575>;
-def SPIRV_OC_OpSubgroupBlockWriteINTEL      : I32EnumAttrCase<"OpSubgroupBlockWriteINTEL", 5576>;
-def SPIRV_OC_OpAssumeTrueKHR                : I32EnumAttrCase<"OpAssumeTrueKHR", 5630>;
-def SPIRV_OC_OpAtomicFAddEXT                : I32EnumAttrCase<"OpAtomicFAddEXT", 6035>;
-def SPIRV_OC_OpConvertFToBF16INTEL          : I32EnumAttrCase<"OpConvertFToBF16INTEL", 6116>;
-def SPIRV_OC_OpConvertBF16ToFINTEL          : I32EnumAttrCase<"OpConvertBF16ToFINTEL", 6117>;
-def SPIRV_OC_OpControlBarrierArriveINTEL    : I32EnumAttrCase<"OpControlBarrierArriveINTEL", 6142>;
-def SPIRV_OC_OpControlBarrierWaitINTEL      : I32EnumAttrCase<"OpControlBarrierWaitINTEL", 6143>;
-def SPIRV_OC_OpGroupIMulKHR                 : I32EnumAttrCase<"OpGroupIMulKHR", 6401>;
-def SPIRV_OC_OpGroupFMulKHR                 : I32EnumAttrCase<"OpGroupFMulKHR", 6402>;
+def SPIRV_OC_OpNop                           : I32EnumAttrCase<"OpNop", 0>;
+def SPIRV_OC_OpUndef                         : I32EnumAttrCase<"OpUndef", 1>;
+def SPIRV_OC_OpSourceContinued               : I32EnumAttrCase<"OpSourceContinued", 2>;
+def SPIRV_OC_OpSource                        : I32EnumAttrCase<"OpSource", 3>;
+def SPIRV_OC_OpSourceExtension               : I32EnumAttrCase<"OpSourceExtension", 4>;
+def SPIRV_OC_OpName                          : I32EnumAttrCase<"OpName", 5>;
+def SPIRV_OC_OpMemberName                    : I32EnumAttrCase<"OpMemberName", 6>;
+def SPIRV_OC_OpString                        : I32EnumAttrCase<"OpString", 7>;
+def SPIRV_OC_OpLine                          : I32EnumAttrCase<"OpLine", 8>;
+def SPIRV_OC_OpExtension                     : I32EnumAttrCase<"OpExtension", 10>;
+def SPIRV_OC_OpExtInstImport                 : I32EnumAttrCase<"OpExtInstImport", 11>;
+def SPIRV_OC_OpExtInst                       : I32EnumAttrCase<"OpExtInst", 12>;
+def SPIRV_OC_OpMemoryModel                   : I32EnumAttrCase<"OpMemoryModel", 14>;
+def SPIRV_OC_OpEntryPoint                    : I32EnumAttrCase<"OpEntryPoint", 15>;
+def SPIRV_OC_OpExecutionMode                 : I32EnumAttrCase<"OpExecutionMode", 16>;
+def SPIRV_OC_OpCapability                    : I32EnumAttrCase<"OpCapability", 17>;
+def SPIRV_OC_OpTypeVoid                      : I32EnumAttrCase<"OpTypeVoid", 19>;
+def SPIRV_OC_OpTypeBool                      : I32EnumAttrCase<"OpTypeBool", 20>;
+def SPIRV_OC_OpTypeInt                       : I32EnumAttrCase<"OpTypeInt", 21>;
+def SPIRV_OC_OpTypeFloat                     : I32EnumAttrCase<"OpTypeFloat", 22>;
+def SPIRV_OC_OpTypeVector                    : I32EnumAttrCase<"OpTypeVector", 23>;
+def SPIRV_OC_OpTypeMatrix                    : I32EnumAttrCase<"OpTypeMatrix", 24>;
+def SPIRV_OC_OpTypeImage                     : I32EnumAttrCase<"OpTypeImage", 25>;
+def SPIRV_OC_OpTypeSampledImage              : I32EnumAttrCase<"OpTypeSampledImage", 27>;
+def SPIRV_OC_OpTypeArray                     : I32EnumAttrCase<"OpTypeArray", 28>;
+def SPIRV_OC_OpTypeRuntimeArray              : I32EnumAttrCase<"OpTypeRuntimeArray", 29>;
+def SPIRV_OC_OpTypeStruct                    : I32EnumAttrCase<"OpTypeStruct", 30>;
+def SPIRV_OC_OpTypePointer                   : I32EnumAttrCase<"OpTypePointer", 32>;
+def SPIRV_OC_OpTypeFunction                  : I32EnumAttrCase<"OpTypeFunction", 33>;
+def SPIRV_OC_OpTypeForwardPointer            : I32EnumAttrCase<"OpTypeForwardPointer", 39>;
+def SPIRV_OC_OpConstantTrue                  : I32EnumAttrCase<"OpConstantTrue", 41>;
+def SPIRV_OC_OpConstantFalse                 : I32EnumAttrCase<"OpConstantFalse", 42>;
+def SPIRV_OC_OpConstant                      : I32EnumAttrCase<"OpConstant", 43>;
+def SPIRV_OC_OpConstantComposite             : I32EnumAttrCase<"OpConstantComposite", 44>;
+def SPIRV_OC_OpConstantNull                  : I32EnumAttrCase<"OpConstantNull", 46>;
+def SPIRV_OC_OpSpecConstantTrue              : I32EnumAttrCase<"OpSpecConstantTrue", 48>;
+def SPIRV_OC_OpSpecConstantFalse             : I32EnumAttrCase<"OpSpecConstantFalse", 49>;
+def SPIRV_OC_OpSpecConstant                  : I32EnumAttrCase<"OpSpecConstant", 50>;
+def SPIRV_OC_OpSpecConstantComposite         : I32EnumAttrCase<"OpSpecConstantComposite", 51>;
+def SPIRV_OC_OpSpecConstantOp                : I32EnumAttrCase<"OpSpecConstantOp", 52>;
+def SPIRV_OC_OpFunction                      : I32EnumAttrCase<"OpFunction", 54>;
+def SPIRV_OC_OpFunctionParameter             : I32EnumAttrCase<"OpFunctionParameter", 55>;
+def SPIRV_OC_OpFunctionEnd                   : I32EnumAttrCase<"OpFunctionEnd", 56>;
+def SPIRV_OC_OpFunctionCall                  : I32EnumAttrCase<"OpFunctionCall", 57>;
+def SPIRV_OC_OpVariable                      : I32EnumAttrCase<"OpVariable", 59>;
+def SPIRV_OC_OpLoad                          : I32EnumAttrCase<"OpLoad", 61>;
+def SPIRV_OC_OpStore                         : I32EnumAttrCase<"OpStore", 62>;
+def SPIRV_OC_OpCopyMemory                    : I32EnumAttrCase<"OpCopyMemory", 63>;
+def SPIRV_OC_OpAccessChain                   : I32EnumAttrCase<"OpAccessChain", 65>;
+def SPIRV_OC_OpPtrAccessChain                : I32EnumAttrCase<"OpPtrAccessChain", 67>;
+def SPIRV_OC_OpInBoundsPtrAccessChain        : I32EnumAttrCase<"OpInBoundsPtrAccessChain", 70>;
+def SPIRV_OC_OpDecorate                      : I32EnumAttrCase<"OpDecorate", 71>;
+def SPIRV_OC_OpMemberDecorate                : I32EnumAttrCase<"OpMemberDecorate", 72>;
+def SPIRV_OC_OpVectorExtractDynamic          : I32EnumAttrCase<"OpVectorExtractDynamic", 77>;
+def SPIRV_OC_OpVectorInsertDynamic           : I32EnumAttrCase<"OpVectorInsertDynamic", 78>;
+def SPIRV_OC_OpVectorShuffle                 : I32EnumAttrCase<"OpVectorShuffle", 79>;
+def SPIRV_OC_OpCompositeConstruct            : I32EnumAttrCase<"OpCompositeConstruct", 80>;
+def SPIRV_OC_OpCompositeExtract              : I32EnumAttrCase<"OpCompositeExtract", 81>;
+def SPIRV_OC_OpCompositeInsert               : I32EnumAttrCase<"OpCompositeInsert", 82>;
+def SPIRV_OC_OpTranspose                     : I32EnumAttrCase<"OpTranspose", 84>;
+def SPIRV_OC_OpImageDrefGather               : I32EnumAttrCase<"OpImageDrefGather", 97>;
+def SPIRV_OC_OpImageWrite                    : I32EnumAttrCase<"OpImageWrite", 99>;
+def SPIRV_OC_OpImage                         : I32EnumAttrCase<"OpImage", 100>;
+def SPIRV_OC_OpImageQuerySize                : I32EnumAttrCase<"OpImageQuerySize", 104>;
+def SPIRV_OC_OpConvertFToU                   : I32EnumAttrCase<"OpConvertFToU", 109>;
+def SPIRV_OC_OpConvertFToS                   : I32EnumAttrCase<"OpConvertFToS", 110>;
+def SPIRV_OC_OpConvertSToF                   : I32EnumAttrCase<"OpConvertSToF", 111>;
+def SPIRV_OC_OpConvertUToF                   : I32EnumAttrCase<"OpConvertUToF", 112>;
+def SPIRV_OC_OpUConvert                      : I32EnumAttrCase<"OpUConvert", 113>;
+def SPIRV_OC_OpSConvert                      : I32EnumAttrCase<"OpSConvert", 114>;
+def SPIRV_OC_OpFConvert                      : I32EnumAttrCase<"OpFConvert", 115>;
+def SPIRV_OC_OpConvertPtrToU                 : I32EnumAttrCase<"OpConvertPtrToU", 117>;
+def SPIRV_OC_OpConvertUToPtr                 : I32EnumAttrCase<"OpConvertUToPtr", 120>;
+def SPIRV_OC_OpPtrCastToGeneric              : I32EnumAttrCase<"OpPtrCastToGeneric", 121>;
+def SPIRV_OC_OpGenericCastToPtr              : I32EnumAttrCase<"OpGenericCastToPtr", 122>;
+def SPIRV_OC_OpGenericCastToPtrExplicit      : I32EnumAttrCase<"OpGenericCastToPtrExplicit", 123>;
+def SPIRV_OC_OpBitcast                       : I32EnumAttrCase<"OpBitcast", 124>;
+def SPIRV_OC_OpSNegate                       : I32EnumAttrCase<"OpSNegate", 126>;
+def SPIRV_OC_OpFNegate                       : I32EnumAttrCase<"OpFNegate", 127>;
+def SPIRV_OC_OpIAdd                          : I32EnumAttrCase<"OpIAdd", 128>;
+def SPIRV_OC_OpFAdd                          : I32EnumAttrCase<"OpFAdd", 129>;
+def SPIRV_OC_OpISub                          : I32EnumAttrCase<"OpISub", 130>;
+def SPIRV_OC_OpFSub                          : I32EnumAttrCase<"OpFSub", 131>;
+def SPIRV_OC_OpIMul                          : I32EnumAttrCase<"OpIMul", 132>;
+def SPIRV_OC_OpFMul                          : I32EnumAttrCase<"OpFMul", 133>;
+def SPIRV_OC_OpUDiv                          : I32EnumAttrCase<"OpUDiv", 134>;
+def SPIRV_OC_OpSDiv                          : I32EnumAttrCase<"OpSDiv", 135>;
+def SPIRV_OC_OpFDiv                          : I32EnumAttrCase<"OpFDiv", 136>;
+def SPIRV_OC_OpUMod                          : I32EnumAttrCase<"OpUMod", 137>;
+def SPIRV_OC_OpSRem                          : I32EnumAttrCase<"OpSRem", 138>;
+def SPIRV_OC_OpSMod                          : I32EnumAttrCase<"OpSMod", 139>;
+def SPIRV_OC_OpFRem                          : I32EnumAttrCase<"OpFRem", 140>;
+def SPIRV_OC_OpFMod                          : I32EnumAttrCase<"OpFMod", 141>;
+def SPIRV_OC_OpVectorTimesScalar             : I32EnumAttrCase<"OpVectorTimesScalar", 142>;
+def SPIRV_OC_OpMatrixTimesScalar             : I32EnumAttrCase<"OpMatrixTimesScalar", 143>;
+def SPIRV_OC_OpVectorTimesMatrix             : I32EnumAttrCase<"OpVectorTimesMatrix", 144>;
+def SPIRV_OC_OpMatrixTimesVector             : I32EnumAttrCase<"OpMatrixTimesVector", 145>;
+def SPIRV_OC_OpMatrixTimesMatrix             : I32EnumAttrCase<"OpMatrixTimesMatrix", 146>;
+def SPIRV_OC_OpDot                           : I32EnumAttrCase<"OpDot", 148>;
+def SPIRV_OC_OpIAddCarry                     : I32EnumAttrCase<"OpIAddCarry", 149>;
+def SPIRV_OC_OpISubBorrow                    : I32EnumAttrCase<"OpISubBorrow", 150>;
+def SPIRV_OC_OpUMulExtended                  : I32EnumAttrCase<"OpUMulExtended", 151>;
+def SPIRV_OC_OpSMulExtended                  : I32EnumAttrCase<"OpSMulExtended", 152>;
+def SPIRV_OC_OpIsNan                         : I32EnumAttrCase<"OpIsNan", 156>;
+def SPIRV_OC_OpIsInf                         : I32EnumAttrCase<"OpIsInf", 157>;
+def SPIRV_OC_OpOrdered                       : I32EnumAttrCase<"OpOrdered", 162>;
+def SPIRV_OC_OpUnordered                     : I32EnumAttrCase<"OpUnordered", 163>;
+def SPIRV_OC_OpLogicalEqual                  : I32EnumAttrCase<"OpLogicalEqual", 164>;
+def SPIRV_OC_OpLogicalNotEqual               : I32EnumAttrCase<"OpLogicalNotEqual", 165>;
+def SPIRV_OC_OpLogicalOr                     : I32EnumAttrCase<"OpLogicalOr", 166>;
+def SPIRV_OC_OpLogicalAnd                    : I32EnumAttrCase<"OpLogicalAnd", 167>;
+def SPIRV_OC_OpLogicalNot                    : I32EnumAttrCase<"OpLogicalNot", 168>;
+def SPIRV_OC_OpSelect                        : I32EnumAttrCase<"OpSelect", 169>;
+def SPIRV_OC_OpIEqual                        : I32EnumAttrCase<"OpIEqual", 170>;
+def SPIRV_OC_OpINotEqual                     : I32EnumAttrCase<"OpINotEqual", 171>;
+def SPIRV_OC_OpUGreaterThan                  : I32EnumAttrCase<"OpUGreaterThan", 172>;
+def SPIRV_OC_OpSGreaterThan                  : I32EnumAttrCase<"OpSGreaterThan", 173>;
+def SPIRV_OC_OpUGreaterThanEqual             : I32EnumAttrCase<"OpUGreaterThanEqual", 174>;
+def SPIRV_OC_OpSGreaterThanEqual             : I32EnumAttrCase<"OpSGreaterThanEqual", 175>;
+def SPIRV_OC_OpULessThan                     : I32EnumAttrCase<"OpULessThan", 176>;
+def SPIRV_OC_OpSLessThan                     : I32EnumAttrCase<"OpSLessThan", 177>;
+def SPIRV_OC_OpULessThanEqual                : I32EnumAttrCase<"OpULessThanEqual", 178>;
+def SPIRV_OC_OpSLessThanEqual                : I32EnumAttrCase<"OpSLessThanEqual", 179>;
+def SPIRV_OC_OpFOrdEqual                     : I32EnumAttrCase<"OpFOrdEqual", 180>;
+def SPIRV_OC_OpFUnordEqual                   : I32EnumAttrCase<"OpFUnordEqual", 181>;
+def SPIRV_OC_OpFOrdNotEqual                  : I32EnumAttrCase<"OpFOrdNotEqual", 182>;
+def SPIRV_OC_OpFUnordNotEqual                : I32EnumAttrCase<"OpFUnordNotEqual", 183>;
+def SPIRV_OC_OpFOrdLessThan                  : I32EnumAttrCase<"OpFOrdLessThan", 184>;
+def SPIRV_OC_OpFUnordLessThan                : I32EnumAttrCase<"OpFUnordLessThan", 185>;
+def SPIRV_OC_OpFOrdGreaterThan               : I32EnumAttrCase<"OpFOrdGreaterThan", 186>;
+def SPIRV_OC_OpFUnordGreaterThan             : I32EnumAttrCase<"OpFUnordGreaterThan", 187>;
+def SPIRV_OC_OpFOrdLessThanEqual             : I32EnumAttrCase<"OpFOrdLessThanEqual", 188>;
+def SPIRV_OC_OpFUnordLessThanEqual           : I32EnumAttrCase<"OpFUnordLessThanEqual", 189>;
+def SPIRV_OC_OpFOrdGreaterThanEqual          : I32EnumAttrCase<"OpFOrdGreaterThanEqual", 190>;
+def SPIRV_OC_OpFUnordGreaterThanEqual        : I32EnumAttrCase<"OpFUnordGreaterThanEqual", 191>;
+def SPIRV_OC_OpShiftRightLogical             : I32EnumAttrCase<"OpShiftRightLogical", 194>;
+def SPIRV_OC_OpShiftRightArithmetic          : I32EnumAttrCase<"OpShiftRightArithmetic", 195>;
+def SPIRV_OC_OpShiftLeftLogical              : I32EnumAttrCase<"OpShiftLeftLogical", 196>;
+def SPIRV_OC_OpBitwiseOr                     : I32EnumAttrCase<"OpBitwiseOr", 197>;
+def SPIRV_OC_OpBitwiseXor                    : I32EnumAttrCase<"OpBitwiseXor", 198>;
+def SPIRV_OC_OpBitwiseAnd                    : I32EnumAttrCase<"OpBitwiseAnd", 199>;
+def SPIRV_OC_OpNot                           : I32EnumAttrCase<"OpNot", 200>;
+def SPIRV_OC_OpBitFieldInsert                : I32EnumAttrCase<"OpBitFieldInsert", 201>;
+def SPIRV_OC_OpBitFieldSExtract              : I32EnumAttrCase<"OpBitFieldSExtract", 202>;
+def SPIRV_OC_OpBitFieldUExtract              : I32EnumAttrCase<"OpBitFieldUExtract", 203>;
+def SPIRV_OC_OpBitReverse                    : I32EnumAttrCase<"OpBitReverse", 204>;
+def SPIRV_OC_OpBitCount                      : I32EnumAttrCase<"OpBitCount", 205>;
+def SPIRV_OC_OpEmitVertex                    : I32EnumAttrCase<"OpEmitVertex", 218>;
+def SPIRV_OC_OpEndPrimitive                  : I32EnumAttrCase<"OpEndPrimitive", 219>;
+def SPIRV_OC_OpControlBarrier                : I32EnumAttrCase<"OpControlBarrier", 224>;
+def SPIRV_OC_OpMemoryBarrier                 : I32EnumAttrCase<"OpMemoryBarrier", 225>;
+def SPIRV_OC_OpAtomicExchange                : I32EnumAttrCase<"OpAtomicExchange", 229>;
+def SPIRV_OC_OpAtomicCompareExchange         : I32EnumAttrCase<"OpAtomicCompareExchange", 230>;
+def SPIRV_OC_OpAtomicCompareExchangeWeak     : I32EnumAttrCase<"OpAtomicCompareExchangeWeak", 231>;
+def SPIRV_OC_OpAtomicIIncrement              : I32EnumAttrCase<"OpAtomicIIncrement", 232>;
+def SPIRV_OC_OpAtomicIDecrement              : I32EnumAttrCase<"OpAtomicIDecrement", 233>;
+def SPIRV_OC_OpAtomicIAdd                    : I32EnumAttrCase<"OpAtomicIAdd", 234>;
+def SPIRV_OC_OpAtomicISub                    : I32EnumAttrCase<"OpAtomicISub", 235>;
+def SPIRV_OC_OpAtomicSMin                    : I32EnumAttrCase<"OpAtomicSMin", 236>;
+def SPIRV_OC_OpAtomicUMin                    : I32EnumAttrCase<"OpAtomicUMin", 237>;
+def SPIRV_OC_OpAtomicSMax                    : I32EnumAttrCase<"OpAtomicSMax", 238>;
+def SPIRV_OC_OpAtomicUMax                    : I32EnumAttrCase<"OpAtomicUMax", 239>;
+def SPIRV_OC_OpAtomicAnd                     : I32EnumAttrCase<"OpAtomicAnd", 240>;
+def SPIRV_OC_OpAtomicOr                      : I32EnumAttrCase<"OpAtomicOr", 241>;
+def SPIRV_OC_OpAtomicXor                     : I32EnumAttrCase<"OpAtomicXor", 242>;
+def SPIRV_OC_OpPhi                           : I32EnumAttrCase<"OpPhi", 245>;
+def SPIRV_OC_OpLoopMerge                     : I32EnumAttrCase<"OpLoopMerge", 246>;
+def SPIRV_OC_OpSelectionMerge                : I32EnumAttrCase<"OpSelectionMerge", 247>;
+def SPIRV_OC_OpLabel                         : I32EnumAttrCase<"OpLabel", 248>;
+def SPIRV_OC_OpBranch                        : I32EnumAttrCase<"OpBranch", 249>;
+def SPIRV_OC_OpBranchConditional             : I32EnumAttrCase<"OpBranchConditional", 250>;
+def SPIRV_OC_OpReturn                        : I32EnumAttrCase<"OpReturn", 253>;
+def SPIRV_OC_OpReturnValue                   : I32EnumAttrCase<"OpReturnValue", 254>;
+def SPIRV_OC_OpUnreachable                   : I32EnumAttrCase<"OpUnreachable", 255>;
+def SPIRV_OC_OpGroupBroadcast                : I32EnumAttrCase<"OpGroupBroadcast", 263>;
+def SPIRV_OC_OpGroupIAdd                     : I32EnumAttrCase<"OpGroupIAdd", 264>;
+def SPIRV_OC_OpGroupFAdd                     : I32EnumAttrCase<"OpGroupFAdd", 265>;
+def SPIRV_OC_OpGroupFMin                     : I32EnumAttrCase<"OpGroupFMin", 266>;
+def SPIRV_OC_OpGroupUMin                     : I32EnumAttrCase<"OpGroupUMin", 267>;
+def SPIRV_OC_OpGroupSMin                     : I32EnumAttrCase<"OpGroupSMin", 268>;
+def SPIRV_OC_OpGroupFMax                     : I32EnumAttrCase<"OpGroupFMax", 269>;
+def SPIRV_OC_OpGroupUMax                     : I32EnumAttrCase<"OpGroupUMax", 270>;
+def SPIRV_OC_OpGroupSMax                     : I32EnumAttrCase<"OpGroupSMax", 271>;
+def SPIRV_OC_OpNoLine                        : I32EnumAttrCase<"OpNoLine", 317>;
+def SPIRV_OC_OpModuleProcessed               : I32EnumAttrCase<"OpModuleProcessed", 330>;
+def SPIRV_OC_OpGroupNonUniformElect          : I32EnumAttrCase<"OpGroupNonUniformElect", 333>;
+def SPIRV_OC_OpGroupNonUniformBroadcast      : I32EnumAttrCase<"OpGroupNonUniformBroadcast", 337>;
+def SPIRV_OC_OpGroupNonUniformBallot         : I32EnumAttrCase<"OpGroupNonUniformBallot", 339>;
+def SPIRV_OC_OpGroupNonUniformBallotBitCount : I32EnumAttrCase<"OpGroupNonUniformBallotBitCount", 342>;
+def SPIRV_OC_OpGroupNonUniformBallotFindLSB  : I32EnumAttrCase<"OpGroupNonUniformBallotFindLSB", 343>;
+def SPIRV_OC_OpGroupNonUniformBallotFindMSB  : I32EnumAttrCase<"OpGroupNonUniformBallotFindMSB", 344>;
+def SPIRV_OC_OpGroupNonUniformShuffle        : I32EnumAttrCase<"OpGroupNonUniformShuffle", 345>;
+def SPIRV_OC_OpGroupNonUniformShuffleXor     : I32EnumAttrCase<"OpGroupNonUniformShuffleXor", 346>;
+def SPIRV_OC_OpGroupNonUniformShuffleUp      : I32EnumAttrCase<"OpGroupNonUniformShuffleUp", 347>;
+def SPIRV_OC_OpGroupNonUniformShuffleDown    : I32EnumAttrCase<"OpGroupNonUniformShuffleDown", 348>;
+def SPIRV_OC_OpGroupNonUniformIAdd           : I32EnumAttrCase<"OpGroupNonUniformIAdd", 349>;
+def SPIRV_OC_OpGroupNonUniformFAdd           : I32EnumAttrCase<"OpGroupNonUniformFAdd", 350>;
+def SPIRV_OC_OpGroupNonUniformIMul           : I32EnumAttrCase<"OpGroupNonUniformIMul", 351>;
+def SPIRV_OC_OpGroupNonUniformFMul           : I32EnumAttrCase<"OpGroupNonUniformFMul", 352>;
+def SPIRV_OC_OpGroupNonUniformSMin           : I32EnumAttrCase<"OpGroupNonUniformSMin", 353>;
+def SPIRV_OC_OpGroupNonUniformUMin           : I32EnumAttrCase<"OpGroupNonUniformUMin", 354>;
+def SPIRV_OC_OpGroupNonUniformFMin           : I32EnumAttrCase<"OpGroupNonUniformFMin", 355>;
+def SPIRV_OC_OpGroupNonUniformSMax           : I32EnumAttrCase<"OpGroupNonUniformSMax", 356>;
+def SPIRV_OC_OpGroupNonUniformUMax           : I32EnumAttrCase<"OpGroupNonUniformUMax", 357>;
+def SPIRV_OC_OpGroupNonUniformFMax           : I32EnumAttrCase<"OpGroupNonUniformFMax", 358>;
+def SPIRV_OC_OpGroupNonUniformBitwiseAnd     : I32EnumAttrCase<"OpGroupNonUniformBitwiseAnd", 359>;
+def SPIRV_OC_OpGroupNonUniformBitwiseOr      : I32EnumAttrCase<"OpGroupNonUniformBitwiseOr", 360>;
+def SPIRV_OC_OpGroupNonUniformBitwiseXor     : I32EnumAttrCase<"OpGroupNonUniformBitwiseXor", 361>;
+def SPIRV_OC_OpGroupNonUniformLogicalAnd     : I32EnumAttrCase<"OpGroupNonUniformLogicalAnd", 362>;
+def SPIRV_OC_OpGroupNonUniformLogicalOr      : I32EnumAttrCase<"OpGroupNonUniformLogicalOr", 363>;
+def SPIRV_OC_OpGroupNonUniformLogicalXor     : I32EnumAttrCase<"OpGroupNonUniformLogicalXor", 364>;
+def SPIRV_OC_OpSubgroupBallotKHR             : I32EnumAttrCase<"OpSubgroupBallotKHR", 4421>;
+def SPIRV_OC_OpSDot                          : I32EnumAttrCase<"OpSDot", 4450>;
+def SPIRV_OC_OpUDot                          : I32EnumAttrCase<"OpUDot", 4451>;
+def SPIRV_OC_OpSUDot                         : I32EnumAttrCase<"OpSUDot", 4452>;
+def SPIRV_OC_OpSDotAccSat                    : I32EnumAttrCase<"OpSDotAccSat", 4453>;
+def SPIRV_OC_OpUDotAccSat                    : I32EnumAttrCase<"OpUDotAccSat", 4454>;
+def SPIRV_OC_OpSUDotAccSat                   : I32EnumAttrCase<"OpSUDotAccSat", 4455>;
+def SPIRV_OC_OpTypeCooperativeMatrixKHR      : I32EnumAttrCase<"OpTypeCooperativeMatrixKHR", 4456>;
+def SPIRV_OC_OpCooperativeMatrixLoadKHR      : I32EnumAttrCase<"OpCooperativeMatrixLoadKHR", 4457>;
+def SPIRV_OC_OpCooperativeMatrixStoreKHR     : I32EnumAttrCase<"OpCooperativeMatrixStoreKHR", 4458>;
+def SPIRV_OC_OpCooperativeMatrixMulAddKHR    : I32EnumAttrCase<"OpCooperativeMatrixMulAddKHR", 4459>;
+def SPIRV_OC_OpCooperativeMatrixLengthKHR    : I32EnumAttrCase<"OpCooperativeMatrixLengthKHR", 4460>;
+def SPIRV_OC_OpSubgroupBlockReadINTEL        : I32EnumAttrCase<"OpSubgroupBlockReadINTEL", 5575>;
+def SPIRV_OC_OpSubgroupBlockWriteINTEL       : I32EnumAttrCase<"OpSubgroupBlockWriteINTEL", 5576>;
+def SPIRV_OC_OpAssumeTrueKHR                 : I32EnumAttrCase<"OpAssumeTrueKHR", 5630>;
+def SPIRV_OC_OpAtomicFAddEXT                 : I32EnumAttrCase<"OpAtomicFAddEXT", 6035>;
+def SPIRV_OC_OpConvertFToBF16INTEL           : I32EnumAttrCase<"OpConvertFToBF16INTEL", 6116>;
+def SPIRV_OC_OpConvertBF16ToFINTEL           : I32EnumAttrCase<"OpConvertBF16ToFINTEL", 6117>;
+def SPIRV_OC_OpControlBarrierArriveINTEL     : I32EnumAttrCase<"OpControlBarrierArriveINTEL", 6142>;
+def SPIRV_OC_OpControlBarrierWaitINTEL       : I32EnumAttrCase<"OpControlBarrierWaitINTEL", 6143>;
+def SPIRV_OC_OpGroupIMulKHR                  : I32EnumAttrCase<"OpGroupIMulKHR", 6401>;
+def SPIRV_OC_OpGroupFMulKHR                  : I32EnumAttrCase<"OpGroupFMulKHR", 6402>;
 
 def SPIRV_OpcodeAttr :
     SPIRV_I32EnumAttr<"Opcode", "valid SPIR-V instructions", "opcode", [
@@ -4549,17 +4560,18 @@ def SPIRV_OpcodeAttr :
       SPIRV_OC_OpVectorInsertDynamic, SPIRV_OC_OpVectorShuffle,
       SPIRV_OC_OpCompositeConstruct, SPIRV_OC_OpCompositeExtract,
       SPIRV_OC_OpCompositeInsert, SPIRV_OC_OpTranspose, SPIRV_OC_OpImageDrefGather,
-      SPIRV_OC_OpImage, SPIRV_OC_OpImageQuerySize, SPIRV_OC_OpConvertFToU,
-      SPIRV_OC_OpConvertFToS, SPIRV_OC_OpConvertSToF, SPIRV_OC_OpConvertUToF,
-      SPIRV_OC_OpUConvert, SPIRV_OC_OpSConvert, SPIRV_OC_OpFConvert,
-      SPIRV_OC_OpConvertPtrToU, SPIRV_OC_OpConvertUToPtr,
+      SPIRV_OC_OpImageWrite, SPIRV_OC_OpImage, SPIRV_OC_OpImageQuerySize,
+      SPIRV_OC_OpConvertFToU, SPIRV_OC_OpConvertFToS, SPIRV_OC_OpConvertSToF,
+      SPIRV_OC_OpConvertUToF, SPIRV_OC_OpUConvert, SPIRV_OC_OpSConvert,
+      SPIRV_OC_OpFConvert, SPIRV_OC_OpConvertPtrToU, SPIRV_OC_OpConvertUToPtr,
       SPIRV_OC_OpPtrCastToGeneric, SPIRV_OC_OpGenericCastToPtr,
       SPIRV_OC_OpGenericCastToPtrExplicit, SPIRV_OC_OpBitcast, SPIRV_OC_OpSNegate,
       SPIRV_OC_OpFNegate, SPIRV_OC_OpIAdd, SPIRV_OC_OpFAdd, SPIRV_OC_OpISub,
       SPIRV_OC_OpFSub, SPIRV_OC_OpIMul, SPIRV_OC_OpFMul, SPIRV_OC_OpUDiv,
       SPIRV_OC_OpSDiv, SPIRV_OC_OpFDiv, SPIRV_OC_OpUMod, SPIRV_OC_OpSRem,
       SPIRV_OC_OpSMod, SPIRV_OC_OpFRem, SPIRV_OC_OpFMod,
-      SPIRV_OC_OpVectorTimesScalar, SPIRV_OC_OpMatrixTimesScalar, SPIRV_OC_OpMatrixTimesVector,
+      SPIRV_OC_OpVectorTimesScalar, SPIRV_OC_OpMatrixTimesScalar,
+      SPIRV_OC_OpVectorTimesMatrix, SPIRV_OC_OpMatrixTimesVector,
       SPIRV_OC_OpMatrixTimesMatrix, SPIRV_OC_OpDot, SPIRV_OC_OpIAddCarry,
       SPIRV_OC_OpISubBorrow, SPIRV_OC_OpUMulExtended, SPIRV_OC_OpSMulExtended,
       SPIRV_OC_OpIsNan, SPIRV_OC_OpIsInf, SPIRV_OC_OpOrdered, SPIRV_OC_OpUnordered,
@@ -4593,6 +4605,7 @@ def SPIRV_OpcodeAttr :
       SPIRV_OC_OpGroupFMax, SPIRV_OC_OpGroupUMax, SPIRV_OC_OpGroupSMax,
       SPIRV_OC_OpNoLine, SPIRV_OC_OpModuleProcessed, SPIRV_OC_OpGroupNonUniformElect,
       SPIRV_OC_OpGroupNonUniformBroadcast, SPIRV_OC_OpGroupNonUniformBallot,
+      SPIRV_OC_OpGroupNonUniformBallotBitCount,
       SPIRV_OC_OpGroupNonUniformBallotFindLSB,
       SPIRV_OC_OpGroupNonUniformBallotFindMSB, SPIRV_OC_OpGroupNonUniformShuffle,
       SPIRV_OC_OpGroupNonUniformShuffleXor, SPIRV_OC_OpGroupNonUniformShuffleUp,
diff --git a/mlir/include/mlir/Dialect/SPIRV/IR/SPIRVControlFlowOps.td b/mlir/include/mlir/Dialect/SPIRV/IR/SPIRVControlFlowOps.td
index 991e753d1b359..cc2f0e4962d8a 100644
--- a/mlir/include/mlir/Dialect/SPIRV/IR/SPIRVControlFlowOps.td
+++ b/mlir/include/mlir/Dialect/SPIRV/IR/SPIRVControlFlowOps.td
@@ -214,13 +214,24 @@ def SPIRV_FunctionCallOp : SPIRV_Op<"FunctionCall", [
 
   let arguments = (ins
     FlatSymbolRefAttr:$callee,
-    Variadic<SPIRV_Type>:$arguments
+    Variadic<SPIRV_Type>:$arguments,
+    OptionalAttr<DictArrayAttr>:$arg_attrs,
+    OptionalAttr<DictArrayAttr>:$res_attrs
   );
 
   let results = (outs
     Optional<SPIRV_Type>:$return_value
   );
 
+  let builders = [
+    OpBuilder<(ins "Type":$returnType, "FlatSymbolRefAttr":$callee,
+      "ValueRange":$arguments),
+    [{
+      build($_builder, $_state, returnType, callee, arguments,
+            /*arg_attrs=*/nullptr, /*res_attrs=*/nullptr);
+    }]>
+  ];
+
   let autogenSerialization = 0;
 
   let assemblyFormat = [{
diff --git a/mlir/include/mlir/Dialect/SPIRV/IR/SPIRVImageOps.td b/mlir/include/mlir/Dialect/SPIRV/IR/SPIRVImageOps.td
index 755d26de9d472..b7d6ec70ce141 100644
--- a/mlir/include/mlir/Dialect/SPIRV/IR/SPIRVImageOps.td
+++ b/mlir/include/mlir/Dialect/SPIRV/IR/SPIRVImageOps.td
@@ -135,6 +135,58 @@ def SPIRV_ImageQuerySizeOp : SPIRV_Op<"ImageQuerySize", [Pure]> {
 
 // -----
 
+def SPIRV_ImageWriteOp : SPIRV_Op<"ImageWrite", []> {
+  let summary = "Write a texel to an image without a sampler.";
+
+  let description = [{
+    Image must be an object whose type is OpTypeImage with a Sampled operand
+    of 0 or 2. If the Arrayed operand is 1, then additional capabilities may
+    be required; e.g., ImageCubeArray, or ImageMSArray. Its Dim operand
+    must not be SubpassData.
+
+    Coordinate must be a scalar or vector of floating-point type or integer
+    type. It contains non-normalized texel coordinates (u[, v] ... [, array
+    layer]) as needed by the definition of Image. See the client API
+    specification for handling of coordinates outside the image.
+
+    Texel is the data to write. It must be a scalar or vector with component
+    type the same as Sampled Type of the OpTypeImage (unless that Sampled
+    Type is OpTypeVoid).
+
+    The Image Format must not be Unknown, unless the
+    StorageImageWriteWithoutFormat Capability was declared.
+
+    Image Operands encodes what operands follow, as per Image Operands.
+
+    <!-- End of AutoGen section -->
+
+    #### Example:
+
+    ```mlir
+    spirv.ImageWrite %0 : !spirv.image<f32, Dim2D, NoDepth, NonArrayed, SingleSampled, NoSampler, Rgba16>, %1 : vector<2xsi32>, %2 : vector<4xf32>
+    ```
+  }];
+
+  let arguments = (ins
+    SPIRV_AnyImage:$image,
+    AnyTypeOf<[SPIRV_ScalarOrVectorOf<SPIRV_Float>, SPIRV_ScalarOrVectorOf<SPIRV_Integer>]>:$coordinate,
+    AnyTypeOf<[SPIRV_ScalarOrVectorOf<SPIRV_Float>, SPIRV_ScalarOrVectorOf<SPIRV_Integer>]>:$texel,
+    OptionalAttr<SPIRV_ImageOperandsAttr>:$image_operands,
+    Variadic<SPIRV_Type>:$operand_arguments
+  );
+
+  let results = (outs);
+
+  let assemblyFormat = [{$image `:` type($image) `,`
+                         $coordinate `:` type($coordinate) `,`
+                         $texel `:` type($texel)
+                         custom<ImageOperands>($image_operands)
+                         ( `(` $operand_arguments^ `:` type($operand_arguments) `)`)?
+                         attr-dict}];
+}
+
+// -----
+
 def SPIRV_ImageOp : SPIRV_Op<"Image",
     [Pure,
      TypesMatchWith<"type of 'result' matches image type of 'sampledimage'",
diff --git a/mlir/include/mlir/Dialect/SPIRV/IR/SPIRVMatrixOps.td b/mlir/include/mlir/Dialect/SPIRV/IR/SPIRVMatrixOps.td
index 5bd99386e0085..f2796861cdf56 100644
--- a/mlir/include/mlir/Dialect/SPIRV/IR/SPIRVMatrixOps.td
+++ b/mlir/include/mlir/Dialect/SPIRV/IR/SPIRVMatrixOps.td
@@ -63,8 +63,7 @@ def SPIRV_MatrixTimesMatrixOp : SPIRV_Op<"MatrixTimesMatrix", [Pure]> {
 
 // -----
 
-def SPIRV_MatrixTimesScalarOp : SPIRV_Op<
-    "MatrixTimesScalar", [Pure, AllTypesMatch<["matrix", "result"]>]> {
+def SPIRV_MatrixTimesScalarOp : SPIRV_Op<"MatrixTimesScalar", [Pure, AllTypesMatch<["matrix", "result"]>]> {
   let summary = "Scale a floating-point matrix.";
 
   let description = [{
@@ -114,8 +113,11 @@ def SPIRV_MatrixTimesScalarOp : SPIRV_Op<
 
 // -----
 
-def SPIRV_MatrixTimesVectorOp : SPIRV_Op<"MatrixTimesVector", [Pure]> {
-  let summary = "Linear-algebraic multiply of matrix X vector.";
+def SPIRV_MatrixTimesVectorOp : SPIRV_Op<"MatrixTimesVector", [
+    Pure,
+    AllElementTypesMatch<["vector", "result"]>
+  ]> {
+  let summary = "Linear-algebraic Matrix X Vector.";
 
   let description = [{
     Result Type must be a vector of floating-point type.
@@ -140,12 +142,12 @@ def SPIRV_MatrixTimesVectorOp : SPIRV_Op<"MatrixTimesVector", [Pure]> {
   ];
 
   let arguments = (ins
-    SPIRV_AnyMatrix:$matrix,
-    SPIRV_AnyVector:$vector
+    SPIRV_MatrixOf<SPIRV_Float>:$matrix,
+    SPIRV_VectorOf<SPIRV_Float>:$vector
   );
 
   let results = (outs
-    SPIRV_AnyVector:$result
+    SPIRV_VectorOf<SPIRV_Float>:$result
   );
 
   let assemblyFormat = [{
@@ -198,4 +200,53 @@ def SPIRV_TransposeOp : SPIRV_Op<"Transpose", [Pure]> {
 
 // -----
 
+def SPIRV_VectorTimesMatrixOp : SPIRV_Op<"VectorTimesMatrix", [
+    Pure,
+    AllElementTypesMatch<["vector", "result"]>
+  ]> {
+  let summary = "Linear-algebraic Vector X Matrix.";
+
+  let description = [{
+    Result Type must be a vector of floating-point type.
+
+    Vector must be a vector with the same Component Type as the Component
+    Type in Result Type. Its number of components must equal the number of
+    components in each column in Matrix.
+
+    Matrix must be a matrix with the same Component Type as the Component
+    Type in Result Type. Its number of columns must equal the number of
+    components in Result Type.
+
+    <!-- End of AutoGen section -->
+
+    #### Example:
+
+    ```mlir
+    %result = spirv.VectorTimesMatrix %vector, %matrix : vector<4xf32>, !spirv.matrix<4 x vector<4xf32>> -> vector<4xf32>
+    ```
+  }];
+
+  let availability = [
+    MinVersion<SPIRV_V_1_0>,
+    MaxVersion<SPIRV_V_1_6>,
+    Extension<[]>,
+    Capability<[SPIRV_C_Matrix]>
+  ];
+
+  let arguments = (ins
+    SPIRV_VectorOf<SPIRV_Float>:$vector,
+    SPIRV_MatrixOf<SPIRV_Float>:$matrix
+  );
+
+  let results = (outs
+    SPIRV_VectorOf<SPIRV_Float>:$result
+  );
+
+  let assemblyFormat = [{
+    operands attr-dict `:` type($vector) `,` type($matrix) `->` type($result)
+  }];
+}
+
+// -----
+
 #endif // MLIR_DIALECT_SPIRV_IR_MATRIX_OPS
diff --git a/mlir/include/mlir/Dialect/SPIRV/IR/SPIRVNonUniformOps.td b/mlir/include/mlir/Dialect/SPIRV/IR/SPIRVNonUniformOps.td
index a1b866387e2ec..98e435c18d3d7 100644
--- a/mlir/include/mlir/Dialect/SPIRV/IR/SPIRVNonUniformOps.td
+++ b/mlir/include/mlir/Dialect/SPIRV/IR/SPIRVNonUniformOps.td
@@ -14,6 +14,15 @@
 #ifndef MLIR_DIALECT_SPIRV_IR_NON_UNIFORM_OPS
 #define MLIR_DIALECT_SPIRV_IR_NON_UNIFORM_OPS
 
+class SPIRV_AttrIs<string operand, string type, string value> : PredOpTrait<
+  operand # " must be " # type # " of value " # value,
+  CPred<"::llvm::cast<::mlir::spirv::" # type # "Attr>(getProperties()." # operand # ").getValue() == ::mlir::spirv::" # type # "::" # value>
+  >;
+
+class SPIRV_ExecutionScopeAttrIs<string operand, string value> : SPIRV_AttrIs<operand, "Scope", value>;
+
+// -----
+
 class SPIRV_GroupNonUniformArithmeticOp<string mnemonic, Type type,
       list<Trait> traits = []> : SPIRV_Op<mnemonic, traits> {
 
@@ -1287,4 +1296,69 @@ def SPIRV_GroupNonUniformLogicalXorOp :
 
 // -----
 
+def SPIRV_GroupNonUniformBallotBitCountOp : SPIRV_Op<"GroupNonUniformBallotBitCount", [
+  SPIRV_ExecutionScopeAttrIs<"execution_scope", "Subgroup">,
+]> {
+  let summary = [{
+    Result is the number of bits that are set to 1 in Value, considering
+    only the bits in Value required to represent all bits of the scope
+    restricted tangle.
+  }];
+
+  let description = [{
+    Result Type must be a scalar of integer type, whose Signedness operand
+    is 0.
+
+    Execution is the scope defining the scope restricted tangle affected by
+    this command. It must be Subgroup.
+
+    The identity I for Operation is 0.
+
+    Value must be a vector of four components of integer type scalar, whose
+    Width operand is 32 and whose Signedness operand is 0.
+
+    Value is a set of bitfields where the first invocation is represented in
+    the lowest bit of the first vector component and the last (up to the
+    size of the scope) is the higher bit number of the last bitmask needed
+    to represent all bits of the invocations in the scope restricted tangle.
+
+    An invocation will not execute a dynamic instance of this instruction
+    (X') until all invocations in its scope restricted tangle have executed
+    all dynamic instances that are program-ordered before X'.
+
+    <!-- End of AutoGen section -->
+
+    #### Example:
+
+    ```mlir
+    %count = spirv.GroupNonUniformBallotBitCount <Subgroup> <Reduce> %val : vector<4xi32> -> i32
+    ```
+  }];
+
+  let availability = [
+    MinVersion<SPIRV_V_1_3>,
+    MaxVersion<SPIRV_V_1_6>,
+    Extension<[]>,
+    Capability<[SPIRV_C_GroupNonUniformBallot]>
+  ];
+
+  let arguments = (ins
+    SPIRV_ScopeAttr:$execution_scope,
+    SPIRV_GroupOperationAttr:$group_operation,
+    SPIRV_Vec4<SignlessOrUnsignedIntOfWidths<[32]>>:$value
+  );
+
+  let results = (outs
+    SPIRV_SignlessOrUnsignedInt:$result
+  );
+
+  let hasVerifier = 0;
+
+  let assemblyFormat = [{
+    $execution_scope $group_operation $value attr-dict `:` type($value) `->` type($result)
+  }];
+}
+
+// -----
+
 #endif // MLIR_DIALECT_SPIRV_IR_NON_UNIFORM_OPS
diff --git a/mlir/include/mlir/Dialect/SparseTensor/Transforms/Passes.h b/mlir/include/mlir/Dialect/SparseTensor/Transforms/Passes.h
index 2e9c297f20182..212f7b6f13c26 100644
--- a/mlir/include/mlir/Dialect/SparseTensor/Transforms/Passes.h
+++ b/mlir/include/mlir/Dialect/SparseTensor/Transforms/Passes.h
@@ -13,10 +13,10 @@
 #ifndef MLIR_DIALECT_SPARSETENSOR_TRANSFORMS_PASSES_H_
 #define MLIR_DIALECT_SPARSETENSOR_TRANSFORMS_PASSES_H_
 
+#include "mlir/Dialect/Bufferization/Transforms/OneShotAnalysis.h"
 #include "mlir/IR/PatternMatch.h"
 #include "mlir/Pass/Pass.h"
 #include "mlir/Transforms/DialectConversion.h"
-#include "mlir/Transforms/OneToNTypeConversion.h"
 
 //===----------------------------------------------------------------------===//
 // Include the generated pass header (which needs some early definitions).
diff --git a/mlir/include/mlir/Dialect/Tensor/IR/CMakeLists.txt b/mlir/include/mlir/Dialect/Tensor/IR/CMakeLists.txt
index cd14fe5c04561..74a05291376b3 100644
--- a/mlir/include/mlir/Dialect/Tensor/IR/CMakeLists.txt
+++ b/mlir/include/mlir/Dialect/Tensor/IR/CMakeLists.txt
@@ -1,2 +1,8 @@
 add_mlir_dialect(TensorOps tensor)
 add_mlir_doc(TensorOps TensorOps Dialects/ -gen-dialect-doc)
+
+set(LLVM_TARGET_DEFINITIONS TensorInterfaces.td)
+mlir_tablegen(TensorInterfaces.h.inc -gen-op-interface-decls)
+mlir_tablegen(TensorInterfaces.cpp.inc -gen-op-interface-defs)
+add_public_tablegen_target(MLIRTensorInterfacesIncGen)
+add_dependencies(mlir-headers MLIRTensorInterfacesIncGen)
diff --git a/mlir/include/mlir/Dialect/Tensor/IR/Tensor.h b/mlir/include/mlir/Dialect/Tensor/IR/Tensor.h
index 0a21c9922b223..1bd0f6553fc8d 100644
--- a/mlir/include/mlir/Dialect/Tensor/IR/Tensor.h
+++ b/mlir/include/mlir/Dialect/Tensor/IR/Tensor.h
@@ -18,6 +18,7 @@
 #include "mlir/Interfaces/CastInterfaces.h"
 #include "mlir/Interfaces/ControlFlowInterfaces.h"
 #include "mlir/Interfaces/DestinationStyleOpInterface.h"
+#include "mlir/Interfaces/InferIntRangeInterface.h"
 #include "mlir/Interfaces/InferTypeOpInterface.h"
 #include "mlir/Interfaces/ParallelCombiningOpInterface.h"
 #include "mlir/Interfaces/ShapedOpInterfaces.h"
@@ -45,6 +46,12 @@ SmallVector<Range, 8> getOrCreateRanges(OffsetSizeAndStrideOpInterface op,
 
 #include "mlir/Dialect/Tensor/IR/TensorOpsDialect.h.inc"
 
+//===----------------------------------------------------------------------===//
+// Tensor Interfaces
+//===----------------------------------------------------------------------===//
+
+#include "mlir/Dialect/Tensor/IR/TensorInterfaces.h.inc"
+
 //===----------------------------------------------------------------------===//
 // Tensor Dialect Operations
 //===----------------------------------------------------------------------===//
diff --git a/mlir/include/mlir/Dialect/Tensor/IR/TensorInterfaces.td b/mlir/include/mlir/Dialect/Tensor/IR/TensorInterfaces.td
new file mode 100644
index 0000000000000..522a9c56f3c92
--- /dev/null
+++ b/mlir/include/mlir/Dialect/Tensor/IR/TensorInterfaces.td
@@ -0,0 +1,33 @@
+//===- TensorInterfaces.td - Tensor Interfaces Declaration -*- tablegen -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This is the definition file for the structured interface sfor Tensor ops.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef TENSOR_IR_TENSORINTERFACES
+#define TENSOR_IR_TENSORINTERFACES
+
+include "mlir/Interfaces/DestinationStyleOpInterface.td"
+include "mlir/IR/OpBase.td"
+
+// TODO: To be moved to LinalgInterfaces.td, see:
+//  * https://github.com/llvm/llvm-project/pull/123902
+//  * https://discourse.llvm.org/t/rfc-move-tensor-pack-and-tensor-unpack-into-linalg/
+def TensorRelayoutOpInterface : OpInterface<"RelayoutOpInterface"> {
+  let description = [{
+    A Tensor (soon to be Linalg) relayout-op is either tensor.pack or
+    tensor.unpack.
+
+    While we could extend this interface with methods from Tensor_RelayoutOp,
+    this is currently not needed and left as a TODO.
+  }];
+  let cppNamespace = "::mlir::tensor";
+}
+
+#endif // TENSOR_IR_TENSORINTERFACES
diff --git a/mlir/include/mlir/Dialect/Tensor/IR/TensorOps.td b/mlir/include/mlir/Dialect/Tensor/IR/TensorOps.td
index 8ad1b23cb2bfe..f6927f5ebcfb8 100644
--- a/mlir/include/mlir/Dialect/Tensor/IR/TensorOps.td
+++ b/mlir/include/mlir/Dialect/Tensor/IR/TensorOps.td
@@ -10,9 +10,11 @@
 #define TENSOR_OPS
 
 include "mlir/Dialect/Tensor/IR/TensorBase.td"
+include "mlir/Dialect/Tensor/IR/TensorInterfaces.td"
 include "mlir/Interfaces/CastInterfaces.td"
 include "mlir/Interfaces/ControlFlowInterfaces.td"
 include "mlir/Interfaces/DestinationStyleOpInterface.td"
+include "mlir/Interfaces/InferIntRangeInterface.td"
 include "mlir/Interfaces/InferTypeOpInterface.td"
 include "mlir/Interfaces/ParallelCombiningOpInterface.td"
 include "mlir/Interfaces/ShapedOpInterfaces.td"
@@ -197,7 +199,8 @@ def Tensor_ConcatOp : Tensor_Op<"concat",
 def Tensor_DimOp : Tensor_Op<"dim", [
     DeclareOpInterfaceMethods<OpAsmOpInterface, ["getAsmResultNames"]>,
     ConditionallySpeculatable, NoMemoryEffect,
-    ShapedDimOpInterface]> {
+    ShapedDimOpInterface,
+    DeclareOpInterfaceMethods<InferIntRangeInterface, ["inferResultRangesFromOptional"]>]> {
   let summary = "dimension index operation";
   let description = [{
     The `tensor.dim` operation takes a tensor and a dimension operand of type
@@ -1165,6 +1168,9 @@ def Tensor_ExpandShapeOp : Tensor_ReassociativeReshapeOp<"expand_shape"> {
   let extraClassDeclaration = commonExtraClassDeclaration # [{
     int64_t getCorrespondingSourceDim(int64_t resultDim);
 
+    // Return output shape as mixes static/dynamic shapes.
+    SmallVector<OpFoldResult> getMixedOutputShape();
+
     // Infer the output shape for a tensor.expand_shape when it is possible
     // to do so.
     static FailureOr<SmallVector<OpFoldResult>> inferOutputShape(
@@ -1828,6 +1834,7 @@ class Tensor_RelayoutOp<string mnemonic, list<Trait> traits = []> :
         DestinationStyleOpInterface,
         ConditionallySpeculatable, NoMemoryEffect,
         DeclareOpInterfaceMethods<ReifyRankedShapedTypeOpInterface>,
+        TensorRelayoutOpInterface,
         TypesMatchWith<"result type matches type of dest",
                    "dest", "result",
                    "$_self">])> {
diff --git a/mlir/include/mlir/Dialect/Tosa/IR/TosaOpBase.td b/mlir/include/mlir/Dialect/Tosa/IR/TosaOpBase.td
index 4975530a9588c..f492bad78e775 100644
--- a/mlir/include/mlir/Dialect/Tosa/IR/TosaOpBase.td
+++ b/mlir/include/mlir/Dialect/Tosa/IR/TosaOpBase.td
@@ -264,4 +264,11 @@ class Tosa_InferShapedTypeOp<string mnemonic, list<Trait> traits = []>
       "operands attr-dict `:` functional-type(operands, results)";
 }
 
+// The "SameVariadicOperandSize" trait allows us to pass optional arguments
+// for multiple zero points in convolution ops.
+class Tosa_ConvOp<string mnemonic, list<Trait> traits = []>
+    : Tosa_InferShapedTypeOp<mnemonic, !listconcat(traits,
+      [SameVariadicOperandSize])> {
+}
+
 #endif // TOSA_OP_BASE
diff --git a/mlir/include/mlir/Dialect/Tosa/IR/TosaOps.h b/mlir/include/mlir/Dialect/Tosa/IR/TosaOps.h
index e4f5d09064cd7..069073bc2d164 100644
--- a/mlir/include/mlir/Dialect/Tosa/IR/TosaOps.h
+++ b/mlir/include/mlir/Dialect/Tosa/IR/TosaOps.h
@@ -14,7 +14,9 @@
 #define MLIR_DIALECT_TOSA_IR_TOSAOPS_H
 
 #include "mlir/Bytecode/BytecodeOpInterface.h"
+#include "mlir/Dialect/Quant/IR/QuantTypes.h"
 #include "mlir/Dialect/Traits.h"
+#include "mlir/IR/Matchers.h"
 #include "mlir/IR/OpDefinition.h"
 #include "mlir/IR/OpImplementation.h"
 #include "mlir/IR/TypeUtilities.h"
@@ -28,6 +30,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "mlir/Dialect/Tosa/IR/TosaOpsDialect.h.inc"
+#include "mlir/Transforms/DialectConversion.h"
 
 namespace mlir {
 class PatternRewriter;
@@ -53,34 +56,43 @@ class MulOperandsAndResultElementType
     : public TraitBase<ConcreteType, MulOperandsAndResultElementType> {
 public:
   static LogicalResult verifyTrait(Operation *op) {
-    auto resElemType = getElementTypeOrSelf(op->getResult(0));
+    // Check we have a single result.
+    if (failed(impl::verifyOneResult(op)))
+      return failure();
+    Type resElemType = getElementTypeOrSelf(op->getResult(0));
 
-    // In cases of floating point type, op requires the same element
-    // type for all operands and result.
-    if (llvm::isa<FloatType>(resElemType))
-      return impl::verifySameOperandsAndResultElementType(op);
+    // Check we have lhs and rhs.
+    if (failed(impl::verifyAtLeastNOperands(op, 2)))
+      return failure();
 
-    if (auto resIntType = dyn_cast<IntegerType>(resElemType)) {
-      IntegerType lhsIntType =
-          cast<IntegerType>(getElementTypeOrSelf(op->getOperand(0)));
-      IntegerType rhsIntType =
-          cast<IntegerType>(getElementTypeOrSelf(op->getOperand(1)));
-      if (lhsIntType != rhsIntType)
-        return op->emitOpError(
-            "requires the same element type for all operands");
+    Type lhsElemType = getElementTypeOrSelf(op->getOperand(0));
+    Type rhsElemType = getElementTypeOrSelf(op->getOperand(1));
+
+    // Check that for i32 a shift has been explicitly provided.
+    if (lhsElemType.isInteger(32) && failed(impl::verifyNOperands(op, 3)))
+      return failure();
 
-      // Though the spec requires the element type of result to be i32, a more
-      // relaxed way is provided at dialect level for easier cooperating with
-      // other dialects.
+    // Verify operands type match (ignoring the shift parameter which will
+    // always be i8).
+    if (lhsElemType != rhsElemType)
+      return op->emitOpError("requires the same element type for all operands");
+
+    // Though the spec requires the element type of result to be i32, a more
+    // relaxed way is provided at dialect level for easier cooperating with
+    // other dialects.
+    if (auto resIntType = dyn_cast<IntegerType>(resElemType)) {
+      auto lhsIntType = cast<IntegerType>(lhsElemType);
       if (lhsIntType.getWidth() > resIntType.getWidth())
         return op->emitOpError("invalid data type size for operands or result");
-
-      return success();
+    } else {
+      // In cases of floating point type or quant types, op requires the same
+      // element type for all operands and result (excluding shift).
+      if (resElemType != lhsElemType)
+        return op->emitOpError(
+            "requires the same element type for all operands and results");
     }
 
-    // In cases of all other types, op requires the same element
-    // type for all operands and result.
-    return impl::verifySameOperandsAndResultElementType(op);
+    return llvm::success();
   }
 };
 
@@ -142,4 +154,120 @@ bool isa_tosa_shape_type(mlir::Type t);
 #define GET_OP_CLASSES
 #include "mlir/Dialect/Tosa/IR/TosaOps.h.inc"
 
+namespace mlir {
+namespace tosa {
+
+// Create a rank-1 const tensor for zero point of the source tensor.
+std::optional<Value> createZeroPointTensor(OpBuilder &builder, Location loc,
+                                           Type srcElemType, int64_t zp = 0);
+
+// Get zero point value from the attribute argument.
+LogicalResult getZeroPoint(ElementsAttr zpAttr, int64_t &zp);
+
+// Verify if zero point falls into valid range.
+template <typename T>
+LogicalResult verifyZeroPoint(Type zpElemType, int64_t zp) {
+  if constexpr (!std::is_same_v<T, Conv2DOp> && !std::is_same_v<T, Conv3DOp> &&
+                !std::is_same_v<T, DepthwiseConv2DOp> &&
+                !std::is_same_v<T, TransposeConv2DOp>) {
+    return failure();
+  }
+
+  if (!zpElemType.isIntOrFloat())
+    return failure();
+
+  if (!zpElemType.isInteger(8) && zp != 0)
+    return failure();
+
+  if (zpElemType.isSignedInteger(8) && (zp < -128 || zp > 127))
+    return failure();
+
+  if (zpElemType.isUnsignedInteger(8) && (zp < 0 || zp > 255))
+    return failure();
+
+  return success();
+}
+
+// Helper type trait to determine if an operation is a tosa convolution.
+template <typename Op>
+struct IsTosaConv : std::false_type {};
+
+template <>
+struct IsTosaConv<tosa::Conv2DOp> : std::true_type {};
+template <>
+struct IsTosaConv<tosa::DepthwiseConv2DOp> : std::true_type {};
+template <>
+struct IsTosaConv<tosa::TransposeConv2DOp> : std::true_type {};
+template <>
+struct IsTosaConv<tosa::Conv3DOp> : std::true_type {};
+
+template <typename Op>
+constexpr bool is_tosa_conv_v = IsTosaConv<Op>::value;
+
+// Helper struct to hold the zero points of a TOSA convolution operation as
+// named 64-bit integer fields.
+struct ConvZpPair {
+  ConvZpPair(std::int64_t inputZp, std::int64_t weightZp)
+      : inputZp(inputZp), weightZp(weightZp) {}
+  std::int64_t inputZp;
+  std::int64_t weightZp;
+};
+
+// Helper function which attempts to extract the zero points from a TOSA
+// convolution by matching them against defining ops which should be tosa.const
+// operations.
+//
+// There are three possible results:
+// 1. Failed to extract the zero-points i.e. they should exist and don't or they
+// do exist but are invalid.
+// 2. Succeeded in extracting zero-points.
+// 3. Zero points are "empty" and meaningless for this op i.e. non-quantized
+// convolution.
+using FailOrMaybeZP = llvm::FailureOr<std::optional<ConvZpPair>>;
+template <typename TosaConvOp>
+std::enable_if_t<is_tosa_conv_v<TosaConvOp>, FailOrMaybeZP>
+extractConvZpPair(TosaConvOp op, PatternRewriter &rewriter) {
+  // Strictly speaking the base TOSA spec requires that for non int8 types
+  // zero points must be zero. However, in the dialect these operands are
+  // optional and only required for int8. They have no semantic meaning for
+  // non-quantized types and can therefore be safely ignored. This is case 3.
+  if (auto opElementTY =
+          cast<ShapedType>(op->getOperand(0).getType()).getElementType();
+      !opElementTY.isInteger(8))
+    return FailOrMaybeZP(std::nullopt);
+
+  // Now we know we should have a zero point check it is valid.
+  if (!op.getInputZp())
+    return rewriter.notifyMatchFailure(op, "missing input zero point");
+
+  // Helper to extract the zero point by matching its definition against a
+  // constant.
+  auto extractZeroPoint = [](Value zpValue) -> std::optional<int64_t> {
+    ElementsAttr zpAttr;
+    if (!matchPattern(zpValue, m_Constant(&zpAttr)))
+      return std::nullopt;
+
+    int64_t zp;
+    if (tosa::getZeroPoint(zpAttr, zp).failed())
+      return std::nullopt;
+
+    return std::make_optional(zp);
+  };
+
+  auto maybeInputZp = extractZeroPoint(op.getInputZp());
+  if (!maybeInputZp)
+    return rewriter.notifyMatchFailure(op, "unable to extract input zp");
+
+  if (!op.getWeightZp())
+    return rewriter.notifyMatchFailure(op, "missing weight zero point");
+
+  auto maybeWeightZp = extractZeroPoint(op.getWeightZp());
+  if (!maybeWeightZp)
+    return rewriter.notifyMatchFailure(op, "unable to extract weight zp");
+
+  return std::make_optional<ConvZpPair>(*maybeInputZp, *maybeWeightZp);
+}
+} // namespace tosa
+} // namespace mlir
+
 #endif // MLIR_DIALECT_TOSA_IR_TOSAOPS_H
diff --git a/mlir/include/mlir/Dialect/Tosa/IR/TosaOps.td b/mlir/include/mlir/Dialect/Tosa/IR/TosaOps.td
index 92ab729f5b933..b8755da8db32e 100644
--- a/mlir/include/mlir/Dialect/Tosa/IR/TosaOps.td
+++ b/mlir/include/mlir/Dialect/Tosa/IR/TosaOps.td
@@ -78,7 +78,8 @@ def Tosa_AvgPool2dOp : Tosa_InferShapedTypeOp<"avg_pool2d"> {
     Tosa_IntArrayAttr2:$stride,
     Tosa_IntArrayAttr4:$pad,
     TypeAttrOf<Tosa_AccType>:$acc_type,
-    OptionalAttr<Tosa_UnaryOpQuantizationAttr>:$quantization_info
+    OptionalAttr<I32Attr>:$input_zp,
+    OptionalAttr<I32Attr>:$output_zp
   );
 
   let results = (outs
@@ -92,7 +93,7 @@ def Tosa_AvgPool2dOp : Tosa_InferShapedTypeOp<"avg_pool2d"> {
 //===----------------------------------------------------------------------===//
 // Operator: conv2d
 //===----------------------------------------------------------------------===//
-def Tosa_Conv2DOp : Tosa_InferShapedTypeOp<"conv2d"> {
+def Tosa_Conv2DOp : Tosa_ConvOp<"conv2d"> {
   let summary = "2D Convolution Operator";
 
   let description = [{
@@ -104,11 +105,12 @@ def Tosa_Conv2DOp : Tosa_InferShapedTypeOp<"conv2d"> {
     Tosa_Tensor4D:$input,
     TosaTensorRankOf<[Tosa_Weight], [4]>:$weight,
     Tosa_Tensor1D:$bias,
+    Optional<Tosa_ScalarTensor>:$input_zp,
+    Optional<Tosa_ScalarTensor>:$weight_zp,
     Tosa_IntArrayAttr4:$pad,
     Tosa_IntArrayAttr2:$stride,
     Tosa_IntArrayAttr2:$dilation,
     TypeAttrOf<Tosa_AccType>:$acc_type,
-    OptionalAttr<Tosa_ConvOpQuantizationAttr>:$quantization_info,
     DefaultValuedOptionalAttr<BoolAttr, "false">:$local_bound
   );
 
@@ -123,7 +125,7 @@ def Tosa_Conv2DOp : Tosa_InferShapedTypeOp<"conv2d"> {
 //===----------------------------------------------------------------------===//
 // Operator: conv3d
 //===----------------------------------------------------------------------===//
-def Tosa_Conv3DOp : Tosa_InferShapedTypeOp<"conv3d"> {
+def Tosa_Conv3DOp : Tosa_ConvOp<"conv3d"> {
   let summary = "3D Convolution operator";
 
   let description = [{
@@ -134,11 +136,12 @@ def Tosa_Conv3DOp : Tosa_InferShapedTypeOp<"conv3d"> {
     Tosa_Tensor5D:$input,
     TosaTensorRankOf<[Tosa_Weight], [5]>:$weight,
     Tosa_Tensor1D:$bias,
+    Optional<Tosa_ScalarTensor>:$input_zp,
+    Optional<Tosa_ScalarTensor>:$weight_zp,
     Tosa_IntArrayAttr6:$pad,
     Tosa_IntArrayAttr3:$stride,
     Tosa_IntArrayAttr3:$dilation,
     TypeAttrOf<Tosa_AccType>:$acc_type,
-    OptionalAttr<Tosa_ConvOpQuantizationAttr>:$quantization_info,
     DefaultValuedOptionalAttr<BoolAttr, "false">:$local_bound
   );
 
@@ -153,7 +156,7 @@ def Tosa_Conv3DOp : Tosa_InferShapedTypeOp<"conv3d"> {
 //===----------------------------------------------------------------------===//
 // Operator: depthwise_conv2d
 //===----------------------------------------------------------------------===//
-def Tosa_DepthwiseConv2DOp : Tosa_InferShapedTypeOp<"depthwise_conv2d"> {
+def Tosa_DepthwiseConv2DOp : Tosa_ConvOp<"depthwise_conv2d"> {
   let summary = "Depthwise 2D Convolution operator";
 
   let description = [{
@@ -165,11 +168,12 @@ def Tosa_DepthwiseConv2DOp : Tosa_InferShapedTypeOp<"depthwise_conv2d"> {
     Tosa_Tensor4D:$input,
     TosaTensorRankOf<[Tosa_Weight], [4]>:$weight,
     Tosa_Tensor1D:$bias,
+    Optional<Tosa_ScalarTensor>:$input_zp,
+    Optional<Tosa_ScalarTensor>:$weight_zp,
     Tosa_IntArrayAttr4:$pad,
     Tosa_IntArrayAttr2:$stride,
     Tosa_IntArrayAttr2:$dilation,
     TypeAttrOf<Tosa_AccType>:$acc_type,
-    OptionalAttr<Tosa_ConvOpQuantizationAttr>:$quantization_info,
     DefaultValuedOptionalAttr<BoolAttr, "false">:$local_bound
   );
 
@@ -234,7 +238,8 @@ def Tosa_FullyConnectedOp : Tosa_InferShapedTypeOp<"fully_connected"> {
     Tosa_Tensor2D:$input,
     TosaTensorRankOf<[Tosa_Weight], [2]>:$weight,
     Tosa_Tensor1D:$bias,
-    OptionalAttr<Tosa_ConvOpQuantizationAttr>:$quantization_info
+    OptionalAttr<I32Attr>:$input_zp,
+    OptionalAttr<I32Attr>:$weight_zp
   );
 
   let results = (outs
@@ -260,7 +265,8 @@ def Tosa_MatMulOp : Tosa_InferShapedTypeOp<"matmul"> {
   let arguments = (ins
     Tosa_Tensor3D:$a,
     Tosa_Tensor3D:$b,
-    OptionalAttr<Tosa_MatMulOpQuantizationAttr>:$quantization_info
+    OptionalAttr<I32Attr>:$a_zp,
+    OptionalAttr<I32Attr>:$b_zp
   );
 
   let results = (outs
@@ -338,7 +344,7 @@ def Tosa_RFFT2dOp : Tosa_InferShapedTypeOp<"rfft2d"> {
 //===----------------------------------------------------------------------===//
 // Operator: transpose_conv2d
 //===----------------------------------------------------------------------===//
-def Tosa_TransposeConv2DOp : Tosa_InferShapedTypeOp<"transpose_conv2d"> {
+def Tosa_TransposeConv2DOp : Tosa_ConvOp<"transpose_conv2d"> {
   let summary = "Transpose 2D Convolution operator.";
 
   let description = [{
@@ -348,13 +354,14 @@ def Tosa_TransposeConv2DOp : Tosa_InferShapedTypeOp<"transpose_conv2d"> {
 
   let arguments = (ins
     Tosa_Tensor4D:$input,
-    TosaTensorRankOf<[Tosa_Weight], [4]>:$filter,
+    TosaTensorRankOf<[Tosa_Weight], [4]>:$weight,
     Tosa_Tensor1D:$bias,
+    Optional<Tosa_ScalarTensor>:$input_zp,
+    Optional<Tosa_ScalarTensor>:$weight_zp,
     Tosa_IntArrayAttr4:$out_pad,
     Tosa_IntArrayAttr2:$stride,
     Tosa_IntArrayAttr4:$out_shape,
     TypeAttrOf<Tosa_AccType>:$acc_type,
-    OptionalAttr<Tosa_ConvOpQuantizationAttr>:$quantization_info,
     DefaultValuedOptionalAttr<BoolAttr, "false">:$local_bound
   );
 
@@ -387,10 +394,8 @@ def Tosa_ClampOp : Tosa_ElementwiseUnaryOp<"clamp"> {
 
   let arguments = (ins
     Tosa_Tensor:$input,
-    I64Attr:$min_int,
-    I64Attr:$max_int,
-    Tosa_FloatAttr:$min_fp,
-    Tosa_FloatAttr:$max_fp,
+    Tosa_IntOrFloatAttr:$min_val,
+    Tosa_IntOrFloatAttr:$max_val,
     DefaultValuedAttr<Tosa_NanPropagationAttr, "\"PROPAGATE\"">:$nan_mode
   );
 
@@ -454,7 +459,7 @@ def Tosa_ErfOp : Tosa_ElementwiseUnaryOp<"erf"> {
   let summary = "Computes gauss error function of input";
 
   let description = [{
-    Gauss error function: $ erf(x) = \frac{2}{\sqrt(\pi)} \int_{0}^{x} e^{-t^2} \,dt $
+    Gauss error function: $ erf(x) = \frac{2}{\sqrt(\pi)} \int_{0}^{x} e^{-t^2} \ dt $
     For quantized integer data types, the TABLE operator should be used instead
     with the following definition.  The erf_table has 513 entries each of
     16-bit/8-bit precision and covering the input range -4.0 to +4.0 in steps of 1/64.
@@ -796,9 +801,11 @@ def MulOperandsAndResultElementType :
 //===----------------------------------------------------------------------===//
 // Operator: mul
 //===----------------------------------------------------------------------===//
-def Tosa_MulOp : Tosa_ElementwiseOp<"mul", [
+def Tosa_MulOp : Tosa_Op<"mul", [
+    DeclareOpInterfaceMethods<InferShapedTypeOpInterface,
+                              ["inferReturnTypeComponents"]>,
     Commutative,
-    MulOperandsAndResultElementType]> {
+    Pure]> {
   let summary = "Multiplication operator";
 
   let description = [{
@@ -810,7 +817,8 @@ def Tosa_MulOp : Tosa_ElementwiseOp<"mul", [
   let arguments = (ins
     Tosa_Tensor:$input1,
     Tosa_Tensor:$input2,
-    I8Attr:$shift
+    // Apply right shift on i32_t input data only
+    Tosa_ScalarInt8Tensor:$shift
   );
 
   let results = (outs
@@ -819,6 +827,9 @@ def Tosa_MulOp : Tosa_ElementwiseOp<"mul", [
 
   let hasFolder = 1;
   let hasVerifier = 1;
+
+  let assemblyFormat =
+      "operands attr-dict `:` functional-type(operands, results)";
 }
 
 //===----------------------------------------------------------------------===//
@@ -1105,7 +1116,8 @@ def Tosa_NegateOp : Tosa_ElementwiseUnaryOp<"negate"> {
 
   let arguments = (ins
       Tosa_Tensor:$input1,
-      OptionalAttr<Tosa_UnaryOpQuantizationAttr>:$quantization_info
+      OptionalAttr<I32Attr>:$input1_zp,
+      OptionalAttr<I32Attr>:$output_zp
   );
 
   let results = (outs
@@ -1579,8 +1591,8 @@ def Tosa_PadOp : Tosa_InferShapedTypeOp<"pad"> {
   let arguments = (ins
     Tosa_RankedTensor:$input1,
     Tosa_Shape:$padding,
-    Optional<Tosa_ScalarTensor>:$pad_const,
-    OptionalAttr<Tosa_PadOpQuantizationAttr>:$quantization_info
+    Optional<Tosa_Rank0Tensor>:$pad_const,
+    OptionalAttr<I32Attr>:$input_zp
   );
 
   let results = (outs
@@ -1612,7 +1624,7 @@ def Tosa_ReshapeOp : Tosa_InferTensorTypeOp<"reshape"> {
 
   let arguments = (ins
     Tosa_Tensor:$input1,
-    DenseI64ArrayAttr:$new_shape
+    Tosa_Shape:$shape
   );
 
   let results = (outs
@@ -1671,8 +1683,8 @@ def Tosa_SliceOp : Tosa_InferShapedTypeOp<"slice"> {
 
   let arguments = (ins
     Tosa_Tensor:$input1,
-    DenseI64ArrayAttr:$start,
-    DenseI64ArrayAttr:$size
+    Tosa_Shape:$start,
+    Tosa_Shape:$size
   );
 
   let results = (outs
@@ -1691,7 +1703,7 @@ def Tosa_TileOp : Tosa_InferShapedTypeOp<"tile"> {
   let summary = "Tile operator";
 
   let description = [{
-    Replicates input 0 multiplies times along each dimension.
+    Replicates input1 multiplies times along each dimension.
   }];
 
   let arguments = (ins
@@ -1886,23 +1898,22 @@ def Tosa_RescaleOp: Tosa_Op<"rescale", [Pure,
   let description = [{
     Rescale quantized values into a new domain. Supported rescalings are:
 
-    | Mode                   | Input | Output | Unsigned | Unsigned |
-    |                        |       |        |  input   |  output  |
-    |------------------------|-------|--------|----------|----------|
-    | signed 8 to 8          | int8  | int8   |  false   |  false   |
-    | signed 8 to 16         | int8  | int16  |  false   |  false   |
-    | signed 8 to 32         | int8  | int32  |  false   |  false   |
-    | signed 16 to 8         | int16 | int8   |  false   |  false   |
-    | signed 16 to 16        | int16 | int16  |  false   |  false   |
-    | signed 16 to 32        | int16 | int32  |  false   |  false   |
-    | signed 32 to 8         | int32 | int8   |  false   |  false   |
-    | signed 32 to 16        | int32 | int16  |  false   |  false   |
-    | signed 32 to 32        | int32 | int32  |  false   |  false   |
-    | signed 48 to 8         | int48 | int8   |  false   |  false   |
-    | signed 48 to 16        | int48 | int16  |  false   |  false   |
-    | signed 48 to 32        | int48 | int32  |  false   |  false   |
-    | unsigned 8 to signed 8 | uint8 | int8   |  true    |  false   |
-    | signed 8 to unsigned 8 | int8  | uint8  |  false   |  true    |
+    | Mode                   | Input | Output | Unsigned input | Unsigned output |
+    |------------------------|-------|--------|----------------|-----------------|
+    | signed 8 to 8          | int8  | int8   |  false         |  false          |
+    | signed 8 to 16         | int8  | int16  |  false         |  false          |
+    | signed 8 to 32         | int8  | int32  |  false         |  false          |
+    | signed 16 to 8         | int16 | int8   |  false         |  false          |
+    | signed 16 to 16        | int16 | int16  |  false         |  false          |
+    | signed 16 to 32        | int16 | int32  |  false         |  false          |
+    | signed 32 to 8         | int32 | int8   |  false         |  false          |
+    | signed 32 to 16        | int32 | int16  |  false         |  false          |
+    | signed 32 to 32        | int32 | int32  |  false         |  false          |
+    | signed 48 to 8         | int48 | int8   |  false         |  false          |
+    | signed 48 to 16        | int48 | int16  |  false         |  false          |
+    | signed 48 to 32        | int48 | int32  |  false         |  false          |
+    | unsigned 8 to signed 8 | uint8 | int8   |  true          |  false          |
+    | signed 8 to unsigned 8 | int8  | uint8  |  false         |  true           |
   }];
 
   let arguments = (ins
diff --git a/mlir/include/mlir/Dialect/Tosa/IR/TosaTypesBase.td b/mlir/include/mlir/Dialect/Tosa/IR/TosaTypesBase.td
index 5693acf3a01db..6457bb8749ee0 100644
--- a/mlir/include/mlir/Dialect/Tosa/IR/TosaTypesBase.td
+++ b/mlir/include/mlir/Dialect/Tosa/IR/TosaTypesBase.td
@@ -93,6 +93,10 @@ def HasNo0Dimensions : And<[
     IsRankedTensorTypePred,
     CPred<"::llvm::all_of(::llvm::cast<::mlir::RankedTensorType>($_self).getShape(), [](auto v) { return v != 0; })">]>;
 
+def AllDimensionsAreSizeOne : And<[
+    IsRankedTensorTypePred,
+    CPred<"::llvm::all_of(::llvm::cast<::mlir::RankedTensorType>($_self).getShape(), [](auto v) { return v == 1; })">]>;
+
 class TosaTensorOf<
     list<Type> allowedTypes, string summary = "tosa-conformant tensor">
     : TensorOf<allowedTypes, [Or<[HasNo0Dimensions, IsUnrankedTensorTypePred]>], summary>;
@@ -109,6 +113,11 @@ class TosaTensorRankOf<list<Type> allowedTypes, list<int> ranks>
       [HasAnyRankOfPred<ranks>],
       !interleave(!foreach(rank, ranks, rank # "D"), "/") # " tensor">;
 
+class TosaScalarTensorOf<list<Type> allowedTypes, list<int> ranks>
+    : TosaRankedTensorOf<allowedTypes,
+      [HasAnyRankOfPred<ranks>, AllDimensionsAreSizeOne],
+      "tosa-conformant scalar tensor">;
+
 //===----------------------------------------------------------------------===//
 // Tensor types
 //===----------------------------------------------------------------------===//
@@ -136,8 +145,10 @@ class Tosa_TensorOfOrNone<list<Type> allowedTypes, string description = ""> :
 // Tensor types with constrained ranks.
 //===----------------------------------------------------------------------===//
 
-// Rank-0 (scalar) tensor
-def Tosa_ScalarTensor : TosaTensorRankOf<[Tosa_AnyNumber], [0]>;
+def Tosa_Rank0Tensor : TosaTensorRankOf<[Tosa_AnyNumber], [0]>;
+
+def Tosa_ScalarTensor : TosaScalarTensorOf<[Tosa_AnyNumber], [1]>;
+def Tosa_ScalarInt8Tensor : TosaScalarTensorOf<[Tosa_Int8], [1]>;
 
 // We include unranked tensors as a supported type for all possible tosa
 // Tensors as unranked does not guarantee invalid. If unranked tensors exist
@@ -202,6 +213,14 @@ def Tosa_FloatAttr : Attr<CPred<"::llvm::isa<::mlir::FloatAttr>($_self)">,
   let returnType = [{ ::mlir::APFloat }];
 }
 
+def Tosa_IntegerAttr : Attr<CPred<"::llvm::isa<::mlir::IntegerAttr>($_self)">,
+                          "arbitrary integer attribute"> {
+  let storageType = [{ ::mlir::IntegerAttr }];
+  let returnType = [{ ::llvm::APInt }];
+}
+
+def Tosa_IntOrFloatAttr : AnyAttrOf<[Tosa_IntegerAttr, Tosa_FloatAttr]>;
+
 //===----------------------------------------------------------------------===//
 // Iterable attributes.
 //===----------------------------------------------------------------------===//
diff --git a/mlir/include/mlir/Dialect/Tosa/Utils/ConversionUtils.h b/mlir/include/mlir/Dialect/Tosa/Utils/ConversionUtils.h
index 78a8828855437..4e2f1b9cb19a9 100644
--- a/mlir/include/mlir/Dialect/Tosa/Utils/ConversionUtils.h
+++ b/mlir/include/mlir/Dialect/Tosa/Utils/ConversionUtils.h
@@ -145,7 +145,7 @@ TosaOp createOpAndInferShape(ImplicitLocOpBuilder &builder, Type resultTy,
 template <typename TosaOp, typename... Args>
 TosaOp CreateOpAndInferShape(ImplicitLocOpBuilder &builder, Type resultTy,
                              Args &&...args) {
-  if (TosaOp::template hasTrait<OpTrait::SameOperandsAndResultRank>()) {
+  if (TosaOp::template hasTrait<::mlir::OpTrait::SameOperandsAndResultRank>()) {
     // op requires same ranks for tensor operands
     if constexpr (sizeof...(Args) == 2) {
       auto argX = std::get<0>(std::tie(args...));
@@ -230,8 +230,11 @@ SmallVector<T> applyTOSAPermutation(ArrayRef<T> input,
 }
 
 // Computes shape value using tosa const_shape op.
+Value getTosaConstShape(ImplicitLocOpBuilder &builder,
+                        llvm::ArrayRef<int64_t> shape);
 Value getTosaConstShape(PatternRewriter &rewriter, Location loc,
                         llvm::ArrayRef<int64_t> shape);
+
 SmallVector<int64_t> convertFromMlirShape(ArrayRef<int64_t> shape);
 
 bool getConstShapeValue(Operation *op,
diff --git a/mlir/include/mlir/Dialect/Tosa/Utils/QuantUtils.h b/mlir/include/mlir/Dialect/Tosa/Utils/QuantUtils.h
index 5e80745777b3b..10dc5dd36cfa9 100644
--- a/mlir/include/mlir/Dialect/Tosa/Utils/QuantUtils.h
+++ b/mlir/include/mlir/Dialect/Tosa/Utils/QuantUtils.h
@@ -35,6 +35,9 @@ void computeMultiplierAndShift(double scale, int32_t &multiplier,
 ConvOpQuantizationAttr buildConvOpQuantizationAttr(OpBuilder &builder,
                                                    Value input, Value weight);
 
+std::pair<Value, Value> createZPsAsConst(OpBuilder &builder, Value input,
+                                         Value weight);
+
 //// Builds MatMulOpQuantizationAttr for MatMul operations from A and B.
 MatMulOpQuantizationAttr buildMatMulOpQuantizationAttr(OpBuilder &builder,
                                                        Value a, Value b);
diff --git a/mlir/include/mlir/Dialect/Transform/IR/TransformOps.td b/mlir/include/mlir/Dialect/Transform/IR/TransformOps.td
index 77ed6b322451e..e4eb67c8e14ce 100644
--- a/mlir/include/mlir/Dialect/Transform/IR/TransformOps.td
+++ b/mlir/include/mlir/Dialect/Transform/IR/TransformOps.td
@@ -886,7 +886,9 @@ def IncludeOp : TransformDialectOp<"include",
 
   let arguments = (ins SymbolRefAttr:$target,
                        FailurePropagationMode:$failure_propagation_mode,
-                       Variadic<Transform_AnyHandleOrParamType>:$operands);
+                       Variadic<Transform_AnyHandleOrParamType>:$operands,
+                       OptionalAttr<DictArrayAttr>:$arg_attrs,
+                       OptionalAttr<DictArrayAttr>:$res_attrs);
   let results = (outs Variadic<Transform_AnyHandleOrParamType>:$results);
 
   let assemblyFormat =
diff --git a/mlir/include/mlir/Dialect/Utils/StaticValueUtils.h b/mlir/include/mlir/Dialect/Utils/StaticValueUtils.h
index d1f7ab1156248..2a3a2defb810d 100644
--- a/mlir/include/mlir/Dialect/Utils/StaticValueUtils.h
+++ b/mlir/include/mlir/Dialect/Utils/StaticValueUtils.h
@@ -144,6 +144,9 @@ bool isEqualConstantIntOrValueArray(ArrayRef<OpFoldResult> ofrs1,
 /// Return a vector of OpFoldResults with the same size a staticValues, but
 /// all elements for which ShapedType::isDynamic is true, will be replaced by
 /// dynamicValues.
+SmallVector<OpFoldResult> getMixedValues(ArrayRef<int64_t> staticValues,
+                                         ValueRange dynamicValues,
+                                         MLIRContext *context);
 SmallVector<OpFoldResult> getMixedValues(ArrayRef<int64_t> staticValues,
                                          ValueRange dynamicValues, Builder &b);
 
diff --git a/mlir/include/mlir/Dialect/Vector/IR/Vector.td b/mlir/include/mlir/Dialect/Vector/IR/Vector.td
index c439ca083e2e0..1922cc63ef353 100644
--- a/mlir/include/mlir/Dialect/Vector/IR/Vector.td
+++ b/mlir/include/mlir/Dialect/Vector/IR/Vector.td
@@ -26,6 +26,15 @@ def Vector_Dialect : Dialect {
 
 // Base class for Vector dialect ops.
 class Vector_Op<string mnemonic, list<Trait> traits = []> :
-    Op<Vector_Dialect, mnemonic, traits>;
+    Op<Vector_Dialect, mnemonic, traits> {
+
+  // Includes definitions for operations that support the use of poison values
+  // within positive index ranges.
+  code extraPoisonClassDeclaration = [{
+    // Integer to represent a poison index within a static and positive integer
+    // range.
+    static constexpr int64_t kPoisonIndex = -1;
+  }];
+}
 
 #endif // MLIR_DIALECT_VECTOR_IR_VECTOR
diff --git a/mlir/include/mlir/Dialect/Vector/IR/VectorOps.td b/mlir/include/mlir/Dialect/Vector/IR/VectorOps.td
index 4331eda166196..c821e7b1527b4 100644
--- a/mlir/include/mlir/Dialect/Vector/IR/VectorOps.td
+++ b/mlir/include/mlir/Dialect/Vector/IR/VectorOps.td
@@ -373,8 +373,9 @@ def Vector_BroadcastOp :
 
     The source operand is duplicated over all the missing leading dimensions
     and stretched over the trailing dimensions where the source has a non-equal
-    dimension of 1. These rules imply that any scalar broadcast (k=0) to any
-    shaped vector with the same element type is always legal.
+    dimension of 1 (stretching a trailing dimension is also referred to as
+    "dim-1" broadcasting). These rules imply that any scalar broadcast (k=0) to
+    any shaped vector with the same element type is always legal.
 
     Example:
 
@@ -396,7 +397,7 @@ def Vector_BroadcastOp :
 
     /// Broadcast `value` to a vector of `dstShape`, knowing that exactly the
     /// `broadcastedDims` dimensions in the dstShape are broadcasted.
-    /// This requires (and asserts) that the broadcast is free of dim-1
+    /// This requires (and asserts) that the broadcast is free of "dim-1"
     /// broadcasting.
     /// Since vector.broadcast only allows expanding leading dimensions, an extra
     /// vector.transpose may be inserted to make the broadcast possible.
@@ -469,10 +470,7 @@ def Vector_ShuffleOp
     ```
   }];
 
-  let extraClassDeclaration = [{
-    // Integer to represent a poison value in a vector shuffle mask.
-    static constexpr int64_t kMaskPoisonValue = -1;
-
+  let extraClassDeclaration = extraPoisonClassDeclaration # [{
     VectorType getV1VectorType() {
       return ::llvm::cast<VectorType>(getV1().getType());
     }
@@ -693,9 +691,10 @@ def Vector_ExtractOp :
     Takes an n-D vector and a k-D position and extracts the (n-k)-D vector at
     the proper position. Degenerates to an element type if n-k is zero.
 
-    Dynamic indices must be greater or equal to zero and less than the size of
-    the corresponding dimension. The result is undefined if any index is
-    out-of-bounds.
+    Static and dynamic indices must be greater or equal to zero and less than
+    the size of the corresponding dimension. The result is undefined if any
+    index is out-of-bounds. The value `-1` represents a poison index, which
+    specifies that the extracted element is poison.
 
     Example:
 
@@ -705,9 +704,8 @@ def Vector_ExtractOp :
     %3 = vector.extract %1[]: vector<f32> from vector<f32>
     %4 = vector.extract %0[%a, %b, %c]: f32 from vector<4x8x16xf32>
     %5 = vector.extract %0[2, %b]: vector<16xf32> from vector<4x8x16xf32>
+    %6 = vector.extract %10[-1, %c]: f32 from vector<4x16xf32>
     ```
-
-    TODO: Implement support for poison indices.
   }];
 
   let arguments = (ins
@@ -724,7 +722,7 @@ def Vector_ExtractOp :
     OpBuilder<(ins "Value":$source, "ArrayRef<OpFoldResult>":$position)>,
   ];
 
-  let extraClassDeclaration = [{
+  let extraClassDeclaration = extraPoisonClassDeclaration # [{
     VectorType getSourceVectorType() {
       return ::llvm::cast<VectorType>(getVector().getType());
     }
@@ -885,9 +883,10 @@ def Vector_InsertOp :
     and inserts the n-D source into the (n+k)-D destination at the proper
     position. Degenerates to a scalar or a 0-d vector source type when n = 0.
 
-    Dynamic indices must be greater or equal to zero and less than the size of
-    the corresponding dimension. The result is undefined if any index is
-    out-of-bounds.
+    Static and dynamic indices must be greater or equal to zero and less than
+    the size of the corresponding dimension. The result is undefined if any
+    index is out-of-bounds. The value `-1` represents a poison index, which
+    specifies that the resulting vector is poison.
 
     Example:
 
@@ -897,9 +896,8 @@ def Vector_InsertOp :
     %8 = vector.insert %6, %7[] : f32 into vector<f32>
     %11 = vector.insert %9, %10[%a, %b, %c] : vector<f32> into vector<4x8x16xf32>
     %12 = vector.insert %4, %10[2, %b] : vector<16xf32> into vector<4x8x16xf32>
+    %13 = vector.insert %20, %1[-1, %c] : f32 into vector<4x16xf32>
     ```
-
-    TODO: Implement support for poison indices.
   }];
 
   let arguments = (ins
@@ -917,7 +915,7 @@ def Vector_InsertOp :
     OpBuilder<(ins "Value":$source, "Value":$dest, "ArrayRef<OpFoldResult>":$position)>,
   ];
 
-  let extraClassDeclaration = [{
+  let extraClassDeclaration = extraPoisonClassDeclaration # [{
     Type getSourceType() { return getSource().getType(); }
     VectorType getDestVectorType() {
       return ::llvm::cast<VectorType>(getDest().getType());
@@ -990,15 +988,13 @@ def Vector_ScalableInsertOp :
     ```mlir
     %2 = vector.scalable.insert %0, %1[5] : vector<4xf32> into vector<[16]xf32>
     ```
-
-    TODO: Implement support for poison indices.
   }];
 
   let assemblyFormat = [{
     $source `,` $dest `[` $pos `]` attr-dict `:` type($source) `into` type($dest)
   }];
 
-  let extraClassDeclaration = [{
+  let extraClassDeclaration = extraPoisonClassDeclaration # [{
     VectorType getSourceVectorType() {
       return ::llvm::cast<VectorType>(getSource().getType());
     }
@@ -1043,15 +1039,13 @@ def Vector_ScalableExtractOp :
     ```mlir
     %1 = vector.scalable.extract %0[5] : vector<4xf32> from vector<[16]xf32>
     ```
-
-    TODO: Implement support for poison indices.
   }];
 
   let assemblyFormat = [{
     $source `[` $pos `]` attr-dict `:` type($res) `from` type($source)
   }];
 
-  let extraClassDeclaration = [{
+  let extraClassDeclaration = extraPoisonClassDeclaration # [{
     VectorType getSourceVectorType() {
       return ::llvm::cast<VectorType>(getSource().getType());
     }
@@ -1089,8 +1083,6 @@ def Vector_InsertStridedSliceOp :
         {offsets = [0, 0, 2], strides = [1, 1]}:
       vector<2x4xf32> into vector<16x4x8xf32>
     ```
-
-    TODO: Implement support for poison indices.
   }];
 
   let assemblyFormat = [{
@@ -2788,7 +2780,7 @@ def Vector_MatmulOp : Vector_Op<"matrix_multiply", [Pure,
     "`:` `(` type($lhs) `,` type($rhs) `)` `->` type($res)";
 }
 
-/// Vector dialect matrix tranposition op that operates on flattened 1-D
+/// Vector dialect matrix transposition op that operates on flattened 1-D
 /// MLIR vectors. This is the counterpart of llvm.matrix.transpose in MLIR.
 /// This may seem redundant with vector.transpose but it serves the purposes of
 /// more progressive lowering and localized type conversion on the path:
@@ -2808,7 +2800,7 @@ def Vector_FlatTransposeOp : Vector_Op<"flat_transpose", [Pure,
   let description = [{
     This is the counterpart of llvm.matrix.transpose in MLIR. It serves
     the purposes of more progressive lowering and localized type conversion.
-    Higher levels typically lower matrix tranpositions into 'vector.transpose'
+    Higher levels typically lower matrix transpositions into 'vector.transpose'
     operations. Subsequent rewriting rule progressively lower these operations
     into 'vector.flat_transpose' operations to bring the operations closer
     to the hardware ISA.
diff --git a/mlir/include/mlir/Dialect/Vector/Transforms/LoweringPatterns.h b/mlir/include/mlir/Dialect/Vector/Transforms/LoweringPatterns.h
index c507b23c6d4de..6aeae30a0a6c0 100644
--- a/mlir/include/mlir/Dialect/Vector/Transforms/LoweringPatterns.h
+++ b/mlir/include/mlir/Dialect/Vector/Transforms/LoweringPatterns.h
@@ -157,12 +157,6 @@ void populateVectorTransposeLoweringPatterns(RewritePatternSet &patterns,
 /// Progressive lowering of transfer_write. This pattern supports lowering of
 /// `vector.transfer_write` to `vector.store`
 ///
-/// [VectorLoadToMemrefLoadLowering]
-/// Replace a 0-d vector.load with a memref.load + vector.broadcast.
-///
-/// [VectorStoreToMemrefStoreLowering]
-/// Replace a 0-d vector.store with a vector.extractelement + memref.store.
-///
 /// These patterns lower transfer ops to simpler ops like `vector.load`,
 /// `vector.store` and `vector.broadcast`. Only transfers with a transfer rank
 /// of a most `maxTransferRank` are lowered. This is useful when combined with
diff --git a/mlir/include/mlir/Dialect/Vector/Transforms/VectorRewritePatterns.h b/mlir/include/mlir/Dialect/Vector/Transforms/VectorRewritePatterns.h
index a59f06f3c1ef1..7de4a6a315750 100644
--- a/mlir/include/mlir/Dialect/Vector/Transforms/VectorRewritePatterns.h
+++ b/mlir/include/mlir/Dialect/Vector/Transforms/VectorRewritePatterns.h
@@ -364,10 +364,12 @@ void populateVectorMaskMaterializationPatterns(RewritePatternSet &patterns,
                                                PatternBenefit benefit = 1);
 
 /// Appends patterns for emulating vector operations over narrow types with ops
-/// over wider types.
+/// over wider types. The `disableAtomicRMW` indicates whether to use a normal
+/// read-modify-write sequence instead of using `memref.generic_atomic_rmw` to
+/// perform subbyte storing.
 void populateVectorNarrowTypeEmulationPatterns(
     const arith::NarrowTypeEmulationConverter &typeConverter,
-    RewritePatternSet &patterns);
+    RewritePatternSet &patterns, bool disableAtomicRMW = false);
 
 /// Rewrite a vector `bitcast(trunci)` to use a more efficient sequence of
 /// vector operations comprising `shuffle` and `bitwise` ops.
diff --git a/mlir/include/mlir/Dialect/X86Vector/X86Vector.td b/mlir/include/mlir/Dialect/X86Vector/X86Vector.td
index fa3f0ee0460b1..16181d7e760db 100644
--- a/mlir/include/mlir/Dialect/X86Vector/X86Vector.td
+++ b/mlir/include/mlir/Dialect/X86Vector/X86Vector.td
@@ -35,17 +35,20 @@ class AVX512_Op<string mnemonic, list<Trait> traits = []> :
   Op<X86Vector_Dialect, "avx512." # mnemonic, traits> {}
 
 // Intrinsic operation used during lowering to LLVM IR.
-class AVX512_IntrOp<string mnemonic, int numResults, list<Trait> traits = []> :
+class AVX512_IntrOp<string mnemonic, int numResults,
+                    list<Trait> traits = [],
+                    string extension = ""> :
   LLVM_IntrOpBase<X86Vector_Dialect, "avx512.intr." # mnemonic,
-                  "x86_avx512_" # !subst(".", "_", mnemonic),
+                  !subst("EXT", extension, "x86_avx512EXT_") # !subst(".", "_", mnemonic),
                   [], [], traits, numResults>;
 
 // Defined by first result overload. May have to be extended for other
 // instructions in the future.
 class AVX512_IntrOverloadedOp<string mnemonic,
-                              list<Trait> traits = []> :
+                              list<Trait> traits = [],
+                              string extension = ""> :
   LLVM_IntrOpBase<X86Vector_Dialect, "avx512.intr." # mnemonic,
-                  "x86_avx512_" # !subst(".", "_", mnemonic),
+                  !subst("EXT", extension, "x86_avx512EXT_") # !subst(".", "_", mnemonic),
                   /*list<int> overloadedResults=*/[0],
                   /*list<int> overloadedOperands=*/[],
                   traits, /*numResults=*/1>;
@@ -271,6 +274,73 @@ def Vp2IntersectQIntrOp : AVX512_IntrOp<"vp2intersect.q.512", 2, [
                    VectorOfLengthAndType<[8], [I64]>:$b);
 }
 
+//----------------------------------------------------------------------------//
+// Dot BF16
+//----------------------------------------------------------------------------//
+
+def DotBF16Op : AVX512_Op<"dot", [Pure,
+  AllTypesMatch<["a", "b"]>,
+  AllTypesMatch<["src", "dst"]>,
+  TypesMatchWith<"`a` has twice an many elements as `src`",
+                 "src", "a",
+                 "VectorType::get({::llvm::cast<VectorType>($_self).getShape()[0] * 2}, "
+                 "BFloat16Type::get($_self.getContext()))">]> {
+  let summary = "Dot BF16 op";
+  let description = [{
+    The `dot` op is an AVX512-BF16 specific op that can lower to the proper
+    LLVMAVX512BF16 operation `llvm.dpbf16ps` depending on the width of MLIR
+    vectors it is applied to.
+
+    #### From the Intel Intrinsics Guide:
+
+    Compute dot-product of BF16 (16-bit) floating-point pairs in `a` and `b`,
+    accumulating the intermediate single-precision (32-bit) floating-point
+    elements with elements in `src`, and store the results in `dst`.
+
+    Example:
+    ```mlir
+    %0 = x86vector.avx512.dot %src, %a, %b : vector<32xbf16> -> vector<16xf32>
+    ```
+  }];
+  let arguments = (ins VectorOfLengthAndType<[4, 8, 16], [F32]>:$src,
+                   VectorOfLengthAndType<[8, 16, 32], [BF16]>:$a,
+                   VectorOfLengthAndType<[8, 16, 32], [BF16]>:$b
+                   );
+  let results = (outs VectorOfLengthAndType<[4, 8, 16], [F32]>:$dst);
+  let assemblyFormat =
+    "$src `,` $a `,` $b attr-dict `:` type($a) `->` type($src)";
+}
+
+def DotBF16Ps128IntrOp : AVX512_IntrOp<"dpbf16ps.128", 1, [Pure,
+    AllTypesMatch<["a", "b"]>,
+    AllTypesMatch<["src", "res"]>],
+    /*extension=*/"bf16"> {
+  let arguments = (ins VectorOfLengthAndType<[4], [F32]>:$src,
+                       VectorOfLengthAndType<[8], [BF16]>:$a,
+                       VectorOfLengthAndType<[8], [BF16]>:$b);
+  let results = (outs VectorOfLengthAndType<[4], [F32]>:$res);
+}
+
+def DotBF16Ps256IntrOp : AVX512_IntrOp<"dpbf16ps.256", 1, [Pure,
+    AllTypesMatch<["a", "b"]>,
+    AllTypesMatch<["src", "res"]>],
+    /*extension=*/"bf16"> {
+  let arguments = (ins VectorOfLengthAndType<[8], [F32]>:$src,
+                       VectorOfLengthAndType<[16], [BF16]>:$a,
+                       VectorOfLengthAndType<[16], [BF16]>:$b);
+  let results = (outs VectorOfLengthAndType<[8], [F32]>:$res);
+}
+
+def DotBF16Ps512IntrOp : AVX512_IntrOp<"dpbf16ps.512", 1, [Pure,
+    AllTypesMatch<["a", "b"]>,
+    AllTypesMatch<["src", "res"]>],
+    /*extension=*/"bf16"> {
+  let arguments = (ins VectorOfLengthAndType<[16], [F32]>:$src,
+                       VectorOfLengthAndType<[32], [BF16]>:$a,
+                       VectorOfLengthAndType<[32], [BF16]>:$b);
+  let results = (outs VectorOfLengthAndType<[16], [F32]>:$res);
+}
+
 //===----------------------------------------------------------------------===//
 // AVX op definitions
 //===----------------------------------------------------------------------===//
diff --git a/mlir/include/mlir/Dialect/XeGPU/IR/XeGPUAttrs.td b/mlir/include/mlir/Dialect/XeGPU/IR/XeGPUAttrs.td
index 4841f94de75f4..0136b18ccfa94 100644
--- a/mlir/include/mlir/Dialect/XeGPU/IR/XeGPUAttrs.td
+++ b/mlir/include/mlir/Dialect/XeGPU/IR/XeGPUAttrs.td
@@ -59,19 +59,29 @@ def XeGPU_BlockTensorDescAttr: XeGPU_TensorDescAttr<"BlockTensorDesc", "block_td
 
 def XeGPU_ScatterTensorDescAttr: XeGPU_TensorDescAttr<"ScatterTensorDesc", "scatter_tdesc_attr"> {
   let summary = [{a composite attribute for `TensorDescType`}];
-  let description = [{`ScatterTensorDesc` (or `scatter_tdesc_attr`) is a composite
-    attribute defined for `TensorDescType` for describing following
-    properties of a `TensorDesc`.
+  let description = [{
+    `ScatterTensorDesc` is a composite attribute defined for `TensorDescType`
+    for describing following properties of a `TensorDesc`:
+
     1. `memory_space`: It describes where the data block described by the
         TensorDesc is located, `Global` device memory or `Shared` local memory.
         It is default to `Global`.
-    2.  `chunk_size`: indicates number of continious elements accessed for each
+
+    2.  `chunk_size`: indicates number of contiguous elements accessed for each
         offset, default is 1. It is used with `scattered` attr only.
   }];
 
   let parameters = (ins
-    OptionalParameter<"MemorySpaceAttr">: $memory_space,
-    OptionalParameter<"IntegerAttr", "1">: $chunk_size
+    DefaultValuedParameter<
+      "MemorySpaceAttr",
+      "MemorySpaceAttr::get($_ctxt, xegpu::MemorySpace::Global)",
+      "Data memory location"
+    >: $memory_space,
+    DefaultValuedParameter<
+      "IntegerAttr",
+      "IntegerAttr::get(IntegerType::get($_ctxt, 64), 1)",
+      "Number of contiguous elements"
+    >: $chunk_size
   );
 
   let builders = [
@@ -80,6 +90,8 @@ def XeGPU_ScatterTensorDescAttr: XeGPU_TensorDescAttr<"ScatterTensorDesc", "scat
       CArg<"int", "1">: $chunk_size
     )>
   ];
+
+  let genVerifyDecl = 1;
  }
 
 //===----------------------------------------------------------------------===//
diff --git a/mlir/include/mlir/Dialect/XeGPU/IR/XeGPUOps.td b/mlir/include/mlir/Dialect/XeGPU/IR/XeGPUOps.td
index a2bfa721f2515..7560ede058faa 100644
--- a/mlir/include/mlir/Dialect/XeGPU/IR/XeGPUOps.td
+++ b/mlir/include/mlir/Dialect/XeGPU/IR/XeGPUOps.td
@@ -276,7 +276,9 @@ def XeGPU_PrefetchNdOp : XeGPU_Op<"prefetch_nd", []> {
 }
 
 
-def XeGPU_LoadNdOp : XeGPU_Op<"load_nd", [AllElementTypesMatch<["value", "TensorDesc"]>]> {
+def XeGPU_LoadNdOp : XeGPU_Op<"load_nd", [
+  AllElementTypesMatch<["value", "TensorDesc"]>, MemoryEffects<[MemRead]>
+  ]> {
   let summary = "loads a n-D block from memory (represented by TensorDesc)"
                 "to registers (represented by vector)";
   let description = [{
@@ -327,7 +329,9 @@ def XeGPU_LoadNdOp : XeGPU_Op<"load_nd", [AllElementTypesMatch<["value", "Tensor
   let hasVerifier = 1;
 }
 
-def XeGPU_StoreNdOp : XeGPU_Op<"store_nd", [AllElementTypesMatch<["value", "TensorDesc"]>]> {
+def XeGPU_StoreNdOp : XeGPU_Op<"store_nd", [
+  AllElementTypesMatch<["value", "TensorDesc"]>, MemoryEffects<[MemWrite]>
+  ]> {
   let summary = "stores a n-D block register region back to memory, currently only supports 2D";
 
   let description = [{
@@ -548,9 +552,9 @@ def XeGPU_PrefetchOp : XeGPU_Op<"prefetch", []> {
   let hasVerifier = 1;
 }
 
-def XeGPU_LoadGatherOp : XeGPU_Op<"load", [AllRanksMatch<["value", "TensorDesc"]>,
-                                    AllElementTypesMatch<["value", "TensorDesc"]>,
-                                   AllElementCountsMatch<["value", "TensorDesc"]>]> {
+def XeGPU_LoadGatherOp : XeGPU_Op<"load", [
+  AllElementTypesMatch<["value", "TensorDesc"]>, MemoryEffects<[MemRead]>
+  ]> {
   let summary = "load a set of scattered data points from memory.";
 
   let description = [{ It (aka. load) load data per each work-item. The output
@@ -620,8 +624,9 @@ def XeGPU_LoadGatherOp : XeGPU_Op<"load", [AllRanksMatch<["value", "TensorDesc"]
   let hasVerifier = 1;
 }
 
-def XeGPU_StoreScatterOp : XeGPU_Op<"store", [AllElementCountsMatch<["value", "TensorDesc"]>,
-                                              AllElementTypesMatch<["value", "TensorDesc"]>]> {
+def XeGPU_StoreScatterOp : XeGPU_Op<"store", [
+  AllElementTypesMatch<["value", "TensorDesc"]>, MemoryEffects<[MemWrite]>
+  ]> {
   let summary = "store data to scattered memory locations.";
   let description = [{ It (aka. store) stores data to scattered memory locations. The value is
   typically a 1D vector. But when the chunk size of the TensorDesc is larger than 1, it will be
diff --git a/mlir/include/mlir/Dialect/XeGPU/IR/XeGPUTypes.td b/mlir/include/mlir/Dialect/XeGPU/IR/XeGPUTypes.td
index d09c5c1870d50..cc2e93fb19a70 100644
--- a/mlir/include/mlir/Dialect/XeGPU/IR/XeGPUTypes.td
+++ b/mlir/include/mlir/Dialect/XeGPU/IR/XeGPUTypes.td
@@ -172,14 +172,14 @@ def XeGPU_TensorDesc: XeGPUTypeDef<"TensorDesc", "tensor_desc",
       auto attr = getEncoding();
       auto scatter_attr = mlir::dyn_cast_if_present<ScatterTensorDescAttr>(attr);
       assert((!attr || scatter_attr) && "invalid on non ScatterTensorDescAttr.");
-      if (scatter_attr && scatter_attr.getChunkSize())
+      if (scatter_attr)
         return scatter_attr.getChunkSize().getInt();
       return 1;
     }
   }];
 
   let hasCustomAssemblyFormat = true;
-
+  let genVerifyDecl = 1;
 }
 
 
diff --git a/mlir/include/mlir/IR/BuiltinDialectBytecode.td b/mlir/include/mlir/IR/BuiltinDialectBytecode.td
index 87da8fd3568fa..0208e8cdbf293 100644
--- a/mlir/include/mlir/IR/BuiltinDialectBytecode.td
+++ b/mlir/include/mlir/IR/BuiltinDialectBytecode.td
@@ -104,7 +104,7 @@ def FileLineColRange : DialectAttribute<(attr
     WithPrinter<"writeFileLineColRangeLocs($_writer, $_name)">>>>:$rawLocData
 )> {
   let cBuilder = "getFileLineColRange(context, filename, rawLocData)";
-  let printerPredicate = "!::llvm::isa<FileLineColLoc>($_val)";
+  let printerPredicate = "!isStrictFileLineColLoc($_val)";
 }
 
 def FileLineColLoc : DialectAttribute<(attr
@@ -112,7 +112,7 @@ def FileLineColLoc : DialectAttribute<(attr
   VarInt:$start_line,
   VarInt:$start_column
 )> {
-  let printerPredicate = "::llvm::isa<FileLineColLoc>($_val)";
+  let printerPredicate = "isStrictFileLineColLoc($_val)";
 }
 }
 
diff --git a/mlir/include/mlir/IR/CMakeLists.txt b/mlir/include/mlir/IR/CMakeLists.txt
index b741eb18d4791..0c7937dfd69e5 100644
--- a/mlir/include/mlir/IR/CMakeLists.txt
+++ b/mlir/include/mlir/IR/CMakeLists.txt
@@ -1,7 +1,14 @@
-add_mlir_interface(OpAsmInterface)
 add_mlir_interface(SymbolInterfaces)
 add_mlir_interface(RegionKindInterface)
 
+set(LLVM_TARGET_DEFINITIONS OpAsmInterface.td)
+mlir_tablegen(OpAsmOpInterface.h.inc -gen-op-interface-decls)
+mlir_tablegen(OpAsmOpInterface.cpp.inc -gen-op-interface-defs)
+mlir_tablegen(OpAsmTypeInterface.h.inc -gen-type-interface-decls)
+mlir_tablegen(OpAsmTypeInterface.cpp.inc -gen-type-interface-defs)
+add_public_tablegen_target(MLIROpAsmInterfaceIncGen)
+add_dependencies(mlir-generic-headers MLIROpAsmInterfaceIncGen)
+
 set(LLVM_TARGET_DEFINITIONS BuiltinAttributes.td)
 mlir_tablegen(BuiltinAttributes.h.inc -gen-attrdef-decls)
 mlir_tablegen(BuiltinAttributes.cpp.inc -gen-attrdef-defs)
diff --git a/mlir/include/mlir/IR/CommonAttrConstraints.td b/mlir/include/mlir/IR/CommonAttrConstraints.td
index 17ca82c510f8a..2beb1e8110afe 100644
--- a/mlir/include/mlir/IR/CommonAttrConstraints.td
+++ b/mlir/include/mlir/IR/CommonAttrConstraints.td
@@ -50,6 +50,9 @@ class Attr<Pred condition, string summary = ""> :
 
   // Default value for attribute.
   // Requires a constBuilderCall defined.
+  //
+  // Format: `$_builder` will be expanded to the relevant builder, e.g. to allow
+  //         access to the current context.
   string defaultValue = ?;
 
   // The value type of this attribute. This corresponds to the mlir::Type that
@@ -334,17 +337,19 @@ class StringBasedAttr<Pred condition, string descr> : Attr<condition, descr> {
   let valueType = NoneType;
 }
 
-def StrAttr : StringBasedAttr<CPred<"::llvm::isa<::mlir::StringAttr>($_self)">,
-                              "string attribute">;
+def StrAttrPred : CPred<"::llvm::isa<::mlir::StringAttr>($_self)">;
+
+def StrAttr : StringBasedAttr<StrAttrPred, "string attribute">;
 
 // A string attribute that represents the name of a symbol.
-def SymbolNameAttr : StringBasedAttr<CPred<"::llvm::isa<::mlir::StringAttr>($_self)">,
-                                     "string attribute">;
+def SymbolNameAttr : StringBasedAttr<StrAttrPred, "string attribute">;
 
 // String attribute that has a specific value type.
 class TypedStrAttr<Type ty>
-    : StringBasedAttr<CPred<"::llvm::isa<::mlir::StringAttr>($_self)">,
-                            "string attribute"> {
+    : StringBasedAttr<And<[StrAttrPred,
+        SubstLeaves<"$_self", "::mlir::cast<StringAttr>($_self).getType()",
+                    ty.predicate>]>,
+        "string attribute of " # ty.summary> {
   let valueType = ty;
 }
 
diff --git a/mlir/include/mlir/IR/CommonTypeConstraints.td b/mlir/include/mlir/IR/CommonTypeConstraints.td
index 82e335e30b6fa..a18b32253d857 100644
--- a/mlir/include/mlir/IR/CommonTypeConstraints.td
+++ b/mlir/include/mlir/IR/CommonTypeConstraints.td
@@ -879,12 +879,6 @@ class NestedTupleOf<list<Type> allowedTypes> :
 //===----------------------------------------------------------------------===//
 // Common type constraints
 //===----------------------------------------------------------------------===//
-// Type constraint for types that are "like" some type or set of types T, that is
-// they're either a T, a vector of Ts, or a tensor of Ts.
-class TypeOrContainer<Type allowedType, string name> : TypeConstraint<Or<[
-  allowedType.predicate,
-  ValueSemanticsContainerOf<[allowedType]>.predicate]>,
-  name>;
 
 // Type constraint for types that are "like" some type or set of types T, that is
 // they're either a T or a mapable container of Ts.
@@ -894,36 +888,23 @@ class TypeOrValueSemanticsContainer<Type allowedType, string name>
   ValueSemanticsContainerOf<[allowedType]>.predicate]>,
   name>;
 
-// Temporary constraint to allow gradual transition to supporting 0-D vectors.
-// TODO: Remove this when all ops support 0-D vectors.
-class TypeOrContainerOfAnyRank<Type allowedType, string name> : TypeConstraint<Or<[
-  allowedType.predicate, VectorOfAnyRankOf<[allowedType]>.predicate,
-  TensorOf<[allowedType]>.predicate]>,
-  name>;
-
-
 // Type constraint for bool-like types: bools, vectors of bools, tensors of
 // bools.
-def BoolLike : TypeOrContainer<I1, "bool-like">;
+def BoolLike : TypeOrValueSemanticsContainer<I1, "bool-like">;
 
-def BoolLikeOfAnyRank : TypeOrContainerOfAnyRank<I1, "bool-like">;
-
-// Type constraint for signless-integer-like types: signless integers, 
-// vectors of signless integers or tensors of signless integers.
+// Type constraint for signless-integer-like types: signless integers or
+// value-semantics containers of signless integers.
 def SignlessIntegerLike : TypeOrValueSemanticsContainer<
     AnySignlessInteger, "signless-integer">;
 
 // Type constraint for signless-integer-like types: signless integers, indices,
-// vectors of signless integers or indices, tensors of signless integers.
+// or value-semantics containers of signless integers or indices.
 def SignlessIntegerOrIndexLike : TypeOrValueSemanticsContainer<
     AnySignlessIntegerOrIndex, "signless-integer-like">;
 
-def SignlessIntegerOrIndexLikeOfAnyRank : TypeOrContainerOfAnyRank<
-    AnySignlessIntegerOrIndex,
-    "signless-integer-like">;
-
-// Type constraint for float-like types: floats, vectors or tensors thereof.
-def FloatLike : TypeOrContainer<AnyFloat, "floating-point-like">;
+// Type constraint for float-like types: floats or value-semantics containers
+// of floats.
+def FloatLike : TypeOrValueSemanticsContainer<AnyFloat, "floating-point-like">;
 
 // Type constraint for signless-integer-or-index-like or float-like types.
 def SignlessIntegerOrFloatLike : TypeConstraint<Or<[
diff --git a/mlir/include/mlir/IR/Location.h b/mlir/include/mlir/IR/Location.h
index e206501f5ee6a..8ce36ed415ac1 100644
--- a/mlir/include/mlir/IR/Location.h
+++ b/mlir/include/mlir/IR/Location.h
@@ -177,7 +177,7 @@ class FusedLocWith : public FusedLoc {
 /// column number. This is similar to the type of location that you get from
 /// most source languages.
 ///
-/// FileLineColLoc is a FileLineColRange with exactly one line and column.
+/// FileLineColLoc is a view to FileLineColRange with one line and column.
 class FileLineColLoc : public FileLineColRange {
 public:
   using FileLineColRange::FileLineColRange;
@@ -190,11 +190,12 @@ class FileLineColLoc : public FileLineColRange {
   StringAttr getFilename() const;
   unsigned getLine() const;
   unsigned getColumn() const;
-
-  /// Methods for support type inquiry through isa, cast, and dyn_cast.
-  static bool classof(Attribute attr);
 };
 
+/// Returns true iff the given location is a FileLineColRange with exactly one
+/// line and column.
+bool isStrictFileLineColLoc(Location loc);
+
 //===----------------------------------------------------------------------===//
 // OpaqueLoc
 //===----------------------------------------------------------------------===//
diff --git a/mlir/include/mlir/IR/OpAsmInterface.td b/mlir/include/mlir/IR/OpAsmInterface.td
index 98b5095ff2d66..34c830a12856f 100644
--- a/mlir/include/mlir/IR/OpAsmInterface.td
+++ b/mlir/include/mlir/IR/OpAsmInterface.td
@@ -109,6 +109,27 @@ def OpAsmOpInterface : OpInterface<"OpAsmOpInterface"> {
   ];
 }
 
+//===----------------------------------------------------------------------===//
+// OpAsmTypeInterface
+//===----------------------------------------------------------------------===//
+
+def OpAsmTypeInterface : TypeInterface<"OpAsmTypeInterface"> {
+  let description = [{
+    This interface provides hooks to interact with the AsmPrinter and AsmParser
+    classes.
+  }];
+  let cppNamespace = "::mlir";
+
+  let methods = [
+    InterfaceMethod<[{
+        Get a name to use when printing a value of this type.
+      }],
+      "void", "getAsmName",
+      (ins "::mlir::OpAsmSetNameFn":$setNameFn), "", ";"
+    >,
+  ];
+}
+
 //===----------------------------------------------------------------------===//
 // ResourceHandleParameter
 //===----------------------------------------------------------------------===//
diff --git a/mlir/include/mlir/IR/OpImplementation.h b/mlir/include/mlir/IR/OpImplementation.h
index 6c1ff4d0e5e6b..5eb8b4a5cff5b 100644
--- a/mlir/include/mlir/IR/OpImplementation.h
+++ b/mlir/include/mlir/IR/OpImplementation.h
@@ -202,7 +202,8 @@ class AsmPrinter {
   /// special or non-printable characters in it.
   virtual void printSymbolName(StringRef symbolRef);
 
-  /// Print a handle to the given dialect resource.
+  /// Print a handle to the given dialect resource. The handle key is quoted and
+  /// escaped if it has any special or non-printable characters in it.
   virtual void printResourceHandle(const AsmDialectResourceHandle &resource);
 
   /// Print an optional arrow followed by a type list.
@@ -734,7 +735,7 @@ class AsmParser {
   virtual OptionalParseResult parseOptionalInteger(APInt &result) = 0;
   virtual OptionalParseResult parseOptionalDecimalInteger(APInt &result) = 0;
 
- private:
+private:
   template <typename IntT, typename ParseFn>
   OptionalParseResult parseOptionalIntegerAndCheck(IntT &result,
                                                    ParseFn &&parseFn) {
@@ -756,7 +757,7 @@ class AsmParser {
     return success();
   }
 
- public:
+public:
   template <typename IntT>
   OptionalParseResult parseOptionalInteger(IntT &result) {
     return parseOptionalIntegerAndCheck(
@@ -1727,6 +1728,10 @@ class OpAsmParser : public AsmParser {
 // Dialect OpAsm interface.
 //===--------------------------------------------------------------------===//
 
+/// A functor used to set the name of the result. See 'getAsmResultNames' below
+/// for more details.
+using OpAsmSetNameFn = function_ref<void(StringRef)>;
+
 /// A functor used to set the name of the start of a result group of an
 /// operation. See 'getAsmResultNames' below for more details.
 using OpAsmSetValueNameFn = function_ref<void(Value, StringRef)>;
@@ -1820,7 +1825,8 @@ ParseResult parseDimensionList(OpAsmParser &parser,
 //===--------------------------------------------------------------------===//
 
 /// The OpAsmOpInterface, see OpAsmInterface.td for more details.
-#include "mlir/IR/OpAsmInterface.h.inc"
+#include "mlir/IR/OpAsmOpInterface.h.inc"
+#include "mlir/IR/OpAsmTypeInterface.h.inc"
 
 namespace llvm {
 template <>
diff --git a/mlir/include/mlir/IR/OperationSupport.h b/mlir/include/mlir/IR/OperationSupport.h
index d4035d14ab746..b29b2bb1b9b52 100644
--- a/mlir/include/mlir/IR/OperationSupport.h
+++ b/mlir/include/mlir/IR/OperationSupport.h
@@ -997,13 +997,21 @@ struct OperationState {
     if (!properties) {
       T *p = new T{};
       properties = p;
+#if defined(__clang__) && __has_warning("-Wdangling-assignment-gsl")
+#pragma clang diagnostic push
+// https://github.com/llvm/llvm-project/issues/126600
+#pragma clang diagnostic ignored "-Wdangling-assignment-gsl"
+#endif
       propertiesDeleter = [](OpaqueProperties prop) {
         delete prop.as<const T *>();
       };
-      propertiesSetter = [](OpaqueProperties new_prop,
+      propertiesSetter = [](OpaqueProperties newProp,
                             const OpaqueProperties prop) {
-        *new_prop.as<T *>() = *prop.as<const T *>();
+        *newProp.as<T *>() = *prop.as<const T *>();
       };
+#if defined(__clang__) && __has_warning("-Wdangling-assignment-gsl")
+#pragma clang diagnostic pop
+#endif
       propertiesId = TypeID::get<T>();
     }
     assert(propertiesId == TypeID::get<T>() && "Inconsistent properties");
diff --git a/mlir/include/mlir/InitAllExtensions.h b/mlir/include/mlir/InitAllExtensions.h
index 14a6a2787b3a5..887db344ed88b 100644
--- a/mlir/include/mlir/InitAllExtensions.h
+++ b/mlir/include/mlir/InitAllExtensions.h
@@ -26,6 +26,7 @@
 #include "mlir/Conversion/NVVMToLLVM/NVVMToLLVM.h"
 #include "mlir/Conversion/OpenMPToLLVM/ConvertOpenMPToLLVM.h"
 #include "mlir/Conversion/UBToLLVM/UBToLLVM.h"
+#include "mlir/Conversion/VectorToLLVM/ConvertVectorToLLVM.h"
 #include "mlir/Dialect/AMX/Transforms.h"
 #include "mlir/Dialect/Affine/TransformOps/AffineTransformOps.h"
 #include "mlir/Dialect/Bufferization/TransformOps/BufferizationTransformOps.h"
@@ -76,6 +77,7 @@ inline void registerAllExtensions(DialectRegistry &registry) {
   registerConvertAMXToLLVMInterface(registry);
   gpu::registerConvertGpuToLLVMInterface(registry);
   NVVM::registerConvertGpuToNVVMInterface(registry);
+  vector::registerConvertVectorToLLVMInterface(registry);
 
   // Register all transform dialect extensions.
   affine::registerTransformDialectExtension(registry);
diff --git a/mlir/include/mlir/Interfaces/CallInterfaces.h b/mlir/include/mlir/Interfaces/CallInterfaces.h
index 0020c19333d10..2bf3a3ca5f8a8 100644
--- a/mlir/include/mlir/Interfaces/CallInterfaces.h
+++ b/mlir/include/mlir/Interfaces/CallInterfaces.h
@@ -14,6 +14,7 @@
 #ifndef MLIR_INTERFACES_CALLINTERFACES_H
 #define MLIR_INTERFACES_CALLINTERFACES_H
 
+#include "mlir/IR/OpImplementation.h"
 #include "mlir/IR/SymbolTable.h"
 #include "llvm/ADT/PointerUnion.h"
 
@@ -35,6 +36,66 @@ namespace call_interface_impl {
 Operation *resolveCallable(CallOpInterface call,
                            SymbolTableCollection *symbolTable = nullptr);
 
+/// Parse a function or call result list.
+///
+///   function-result-list ::= function-result-list-parens
+///                          | non-function-type
+///   function-result-list-parens ::= `(` `)`
+///                                 | `(` function-result-list-no-parens `)`
+///   function-result-list-no-parens ::= function-result (`,` function-result)*
+///   function-result ::= type attribute-dict?
+///
+ParseResult
+parseFunctionResultList(OpAsmParser &parser, SmallVectorImpl<Type> &resultTypes,
+                        SmallVectorImpl<DictionaryAttr> &resultAttrs);
+
+/// Parses a function signature using `parser`. This does not deal with function
+/// signatures containing SSA region arguments (to parse these signatures, use
+/// function_interface_impl::parseFunctionSignature). When
+/// `mustParseEmptyResult`, `-> ()` is expected when there is no result type.
+///
+///   no-ssa-function-signature ::= `(` no-ssa-function-arg-list `)`
+///                               -> function-result-list
+///   no-ssa-function-arg-list  ::= no-ssa-function-arg
+///                               (`,` no-ssa-function-arg)*
+///   no-ssa-function-arg       ::= type attribute-dict?
+ParseResult parseFunctionSignature(OpAsmParser &parser,
+                                   SmallVectorImpl<Type> &argTypes,
+                                   SmallVectorImpl<DictionaryAttr> &argAttrs,
+                                   SmallVectorImpl<Type> &resultTypes,
+                                   SmallVectorImpl<DictionaryAttr> &resultAttrs,
+                                   bool mustParseEmptyResult = true);
+
+/// Print a function signature for a call or callable operation. If a body
+/// region is provided, the SSA arguments are printed in the signature. When
+/// `printEmptyResult` is false, `-> function-result-list` is omitted when
+/// `resultTypes` is empty.
+///
+///   function-signature     ::= ssa-function-signature
+///                            | no-ssa-function-signature
+///   ssa-function-signature ::= `(` ssa-function-arg-list `)`
+///                            -> function-result-list
+///   ssa-function-arg-list  ::= ssa-function-arg (`,` ssa-function-arg)*
+///   ssa-function-arg       ::= `%`name `:` type attribute-dict?
+void printFunctionSignature(OpAsmPrinter &p, TypeRange argTypes,
+                            ArrayAttr argAttrs, bool isVariadic,
+                            TypeRange resultTypes, ArrayAttr resultAttrs,
+                            Region *body = nullptr,
+                            bool printEmptyResult = true);
+
+/// Adds argument and result attributes, provided as `argAttrs` and
+/// `resultAttrs` arguments, to the list of operation attributes in `result`.
+/// Internally, argument and result attributes are stored as dict attributes
+/// with special names given by getResultAttrName, getArgumentAttrName.
+void addArgAndResultAttrs(Builder &builder, OperationState &result,
+                          ArrayRef<DictionaryAttr> argAttrs,
+                          ArrayRef<DictionaryAttr> resultAttrs,
+                          StringAttr argAttrsName, StringAttr resAttrsName);
+void addArgAndResultAttrs(Builder &builder, OperationState &result,
+                          ArrayRef<OpAsmParser::Argument> args,
+                          ArrayRef<DictionaryAttr> resultAttrs,
+                          StringAttr argAttrsName, StringAttr resAttrsName);
+
 } // namespace call_interface_impl
 
 } // namespace mlir
diff --git a/mlir/include/mlir/Interfaces/CallInterfaces.td b/mlir/include/mlir/Interfaces/CallInterfaces.td
index c6002da0d491c..e3c2aec401741 100644
--- a/mlir/include/mlir/Interfaces/CallInterfaces.td
+++ b/mlir/include/mlir/Interfaces/CallInterfaces.td
@@ -17,6 +17,48 @@
 
 include "mlir/IR/OpBase.td"
 
+
+/// Interface for operations with arguments attributes (both call-like
+/// and callable operations).
+def ArgumentAttributesMethods {
+  list<InterfaceMethod> methods = [
+    InterfaceMethod<[{
+        Get the array of argument attribute dictionaries. The method should
+        return an array attribute containing only dictionary attributes equal in
+        number to the number of arguments. Alternatively, the method can
+        return null to indicate that there are no argument attributes.
+      }],
+      "::mlir::ArrayAttr", "getArgAttrsAttr">,
+    InterfaceMethod<[{
+        Get the array of result attribute dictionaries. The method should return
+        an array attribute containing only dictionary attributes equal in number
+        to the number of results. Alternatively, the method can return
+        null to indicate that there are no result attributes.
+      }],
+      "::mlir::ArrayAttr", "getResAttrsAttr">,
+    InterfaceMethod<[{
+      Set the array of argument attribute dictionaries.
+    }],
+    "void", "setArgAttrsAttr", (ins "::mlir::ArrayAttr":$attrs)>,
+    InterfaceMethod<[{
+      Set the array of result attribute dictionaries.
+    }],
+    "void", "setResAttrsAttr", (ins "::mlir::ArrayAttr":$attrs)>,
+    InterfaceMethod<[{
+      Remove the array of argument attribute dictionaries. This is the same as
+      setting all argument attributes to an empty dictionary. The method should
+      return the removed attribute.
+    }],
+    "::mlir::Attribute", "removeArgAttrsAttr">,
+    InterfaceMethod<[{
+      Remove the array of result attribute dictionaries. This is the same as
+      setting all result attributes to an empty dictionary. The method should
+      return the removed attribute.
+    }],
+    "::mlir::Attribute", "removeResAttrsAttr">
+  ];
+}
+
 // `CallInterfaceCallable`: This is a type used to represent a single callable
 // region. A callable is either a symbol, or an SSA value, that is referenced by
 // a call-like operation. This represents the destination of the call.
@@ -81,7 +123,7 @@ def CallOpInterface : OpInterface<"CallOpInterface"> {
         return ::mlir::call_interface_impl::resolveCallable($_op);
       }]
     >
-  ];
+  ] # ArgumentAttributesMethods.methods;
 }
 
 /// Interface for callable operations.
@@ -113,48 +155,7 @@ def CallableOpInterface : OpInterface<"CallableOpInterface"> {
       allow for this method may be called on function declarations).
     }],
     "::llvm::ArrayRef<::mlir::Type>", "getResultTypes">,
-
-    InterfaceMethod<[{
-        Get the array of argument attribute dictionaries. The method should
-        return an array attribute containing only dictionary attributes equal in
-        number to the number of region arguments. Alternatively, the method can
-        return null to indicate that the region has no argument attributes.
-      }],
-      "::mlir::ArrayAttr", "getArgAttrsAttr", (ins),
-      /*methodBody=*/[{}], /*defaultImplementation=*/[{ return nullptr; }]>,
-    InterfaceMethod<[{
-        Get the array of result attribute dictionaries. The method should return
-        an array attribute containing only dictionary attributes equal in number
-        to the number of region results. Alternatively, the method can return
-        null to indicate that the region has no result attributes.
-      }],
-      "::mlir::ArrayAttr", "getResAttrsAttr", (ins),
-      /*methodBody=*/[{}], /*defaultImplementation=*/[{ return nullptr; }]>,
-    InterfaceMethod<[{
-      Set the array of argument attribute dictionaries.
-    }],
-    "void", "setArgAttrsAttr", (ins "::mlir::ArrayAttr":$attrs),
-      /*methodBody=*/[{}], /*defaultImplementation=*/[{ return; }]>,
-    InterfaceMethod<[{
-      Set the array of result attribute dictionaries.
-    }],
-    "void", "setResAttrsAttr", (ins "::mlir::ArrayAttr":$attrs),
-      /*methodBody=*/[{}], /*defaultImplementation=*/[{ return; }]>,
-    InterfaceMethod<[{
-      Remove the array of argument attribute dictionaries. This is the same as
-      setting all argument attributes to an empty dictionary. The method should
-      return the removed attribute.
-    }],
-    "::mlir::Attribute", "removeArgAttrsAttr", (ins),
-      /*methodBody=*/[{}], /*defaultImplementation=*/[{ return nullptr; }]>,
-    InterfaceMethod<[{
-      Remove the array of result attribute dictionaries. This is the same as
-      setting all result attributes to an empty dictionary. The method should
-      return the removed attribute.
-    }],
-    "::mlir::Attribute", "removeResAttrsAttr", (ins),
-      /*methodBody=*/[{}], /*defaultImplementation=*/[{ return nullptr; }]>,
-  ];
+  ] # ArgumentAttributesMethods.methods;
 }
 
 #endif // MLIR_INTERFACES_CALLINTERFACES
diff --git a/mlir/include/mlir/Interfaces/FunctionImplementation.h b/mlir/include/mlir/Interfaces/FunctionImplementation.h
index a5e6963e4e666..374c2c534f87d 100644
--- a/mlir/include/mlir/Interfaces/FunctionImplementation.h
+++ b/mlir/include/mlir/Interfaces/FunctionImplementation.h
@@ -33,19 +33,6 @@ class VariadicFlag {
   bool variadic;
 };
 
-/// Adds argument and result attributes, provided as `argAttrs` and
-/// `resultAttrs` arguments, to the list of operation attributes in `result`.
-/// Internally, argument and result attributes are stored as dict attributes
-/// with special names given by getResultAttrName, getArgumentAttrName.
-void addArgAndResultAttrs(Builder &builder, OperationState &result,
-                          ArrayRef<DictionaryAttr> argAttrs,
-                          ArrayRef<DictionaryAttr> resultAttrs,
-                          StringAttr argAttrsName, StringAttr resAttrsName);
-void addArgAndResultAttrs(Builder &builder, OperationState &result,
-                          ArrayRef<OpAsmParser::Argument> args,
-                          ArrayRef<DictionaryAttr> resultAttrs,
-                          StringAttr argAttrsName, StringAttr resAttrsName);
-
 /// Callback type for `parseFunctionOp`, the callback should produce the
 /// type that will be associated with a function-like operation from lists of
 /// function arguments and results, VariadicFlag indicates whether the function
@@ -58,11 +45,11 @@ using FuncTypeBuilder = function_ref<Type(
 /// indicates whether functions with variadic arguments are supported. The
 /// trailing arguments are populated by this function with names, types,
 /// attributes and locations of the arguments and those of the results.
-ParseResult
-parseFunctionSignature(OpAsmParser &parser, bool allowVariadic,
-                       SmallVectorImpl<OpAsmParser::Argument> &arguments,
-                       bool &isVariadic, SmallVectorImpl<Type> &resultTypes,
-                       SmallVectorImpl<DictionaryAttr> &resultAttrs);
+ParseResult parseFunctionSignatureWithArguments(
+    OpAsmParser &parser, bool allowVariadic,
+    SmallVectorImpl<OpAsmParser::Argument> &arguments, bool &isVariadic,
+    SmallVectorImpl<Type> &resultTypes,
+    SmallVectorImpl<DictionaryAttr> &resultAttrs);
 
 /// Parser implementation for function-like operations.  Uses
 /// `funcTypeBuilder` to construct the custom function type given lists of
@@ -84,9 +71,14 @@ void printFunctionOp(OpAsmPrinter &p, FunctionOpInterface op, bool isVariadic,
 
 /// Prints the signature of the function-like operation `op`. Assumes `op` has
 /// is a FunctionOpInterface and has passed verification.
-void printFunctionSignature(OpAsmPrinter &p, FunctionOpInterface op,
-                            ArrayRef<Type> argTypes, bool isVariadic,
-                            ArrayRef<Type> resultTypes);
+inline void printFunctionSignature(OpAsmPrinter &p, FunctionOpInterface op,
+                                   ArrayRef<Type> argTypes, bool isVariadic,
+                                   ArrayRef<Type> resultTypes) {
+  call_interface_impl::printFunctionSignature(
+      p, argTypes, op.getArgAttrsAttr(), isVariadic, resultTypes,
+      op.getResAttrsAttr(), &op->getRegion(0),
+      /*printEmptyResult=*/false);
+}
 
 /// Prints the list of function prefixed with the "attributes" keyword. The
 /// attributes with names listed in "elided" as well as those used by the
diff --git a/mlir/include/mlir/Interfaces/Utils/InferIntRangeCommon.h b/mlir/include/mlir/Interfaces/Utils/InferIntRangeCommon.h
index 3988a8826498a..e46358ccfc46f 100644
--- a/mlir/include/mlir/Interfaces/Utils/InferIntRangeCommon.h
+++ b/mlir/include/mlir/Interfaces/Utils/InferIntRangeCommon.h
@@ -20,6 +20,8 @@
 #include <optional>
 
 namespace mlir {
+class ShapedDimOpInterface;
+
 namespace intrange {
 /// Function that performs inference on an array of `ConstantIntRanges`,
 /// abstracted away here to permit writing the function that handles both
@@ -143,6 +145,12 @@ std::optional<bool> evaluatePred(CmpPredicate pred,
                                  const ConstantIntRanges &lhs,
                                  const ConstantIntRanges &rhs);
 
+/// Returns the integer range for the result of a `ShapedDimOpInterface` given
+/// the optional inferred ranges for the `dimension` index `maybeDim`. When a
+/// dynamic dimension is encountered, returns [0, signed_max(type(result))].
+ConstantIntRanges inferShapedDimOpInterface(ShapedDimOpInterface op,
+                                            const IntegerValueRange &maybeDim);
+
 } // namespace intrange
 } // namespace mlir
 
diff --git a/mlir/include/mlir/Target/LLVM/NVVM/Utils.h b/mlir/include/mlir/Target/LLVM/NVVM/Utils.h
index 2d6157b1e5a60..ca4b6424a292c 100644
--- a/mlir/include/mlir/Target/LLVM/NVVM/Utils.h
+++ b/mlir/include/mlir/Target/LLVM/NVVM/Utils.h
@@ -55,7 +55,7 @@ class SerializeGPUModuleBase : public LLVM::ModuleToObject {
   LogicalResult appendStandardLibs();
 
   /// Loads the bitcode files in `librariesToLink`.
-  virtual std::optional<SmallVector<std::unique_ptr<llvm::Module>>>
+  std::optional<SmallVector<std::unique_ptr<llvm::Module>>>
   loadBitcodeFiles(llvm::Module &module) override;
 
 protected:
diff --git a/mlir/include/mlir/Target/LLVMIR/ModuleImport.h b/mlir/include/mlir/Target/LLVMIR/ModuleImport.h
index 33c9af7c6335a..4642d58760ca8 100644
--- a/mlir/include/mlir/Target/LLVMIR/ModuleImport.h
+++ b/mlir/include/mlir/Target/LLVMIR/ModuleImport.h
@@ -67,10 +67,17 @@ class ModuleImport {
   /// Converts all global variables of the LLVM module to MLIR global variables.
   LogicalResult convertGlobals();
 
+  /// Converts all aliases of the LLVM module to MLIR variables.
+  LogicalResult convertAliases();
+
   /// Converts the data layout of the LLVM module to an MLIR data layout
   /// specification.
   LogicalResult convertDataLayout();
 
+  /// Converts target triple of the LLVM module to an MLIR target triple
+  /// specification.
+  void convertTargetTriple();
+
   /// Stores the mapping between an LLVM value and its MLIR counterpart.
   void mapValue(llvm::Value *llvm, Value mlir) { mapValue(llvm) = mlir; }
 
@@ -284,6 +291,9 @@ class ModuleImport {
   LogicalResult convertGlobal(llvm::GlobalVariable *globalVar);
   /// Imports the magic globals "global_ctors" and "global_dtors".
   LogicalResult convertGlobalCtorsAndDtors(llvm::GlobalVariable *globalVar);
+  /// Converts an LLVM global alias variable into an MLIR LLVM dialect alias
+  /// operation if a conversion exists. Otherwise, returns failure.
+  LogicalResult convertAlias(llvm::GlobalAlias *alias);
   /// Returns personality of `func` as a FlatSymbolRefAttr.
   FlatSymbolRefAttr getPersonalityAsAttr(llvm::Function *func);
   /// Imports `bb` into `block`, which must be initially empty.
@@ -316,24 +326,37 @@ class ModuleImport {
   LogicalResult convertBranchArgs(llvm::Instruction *branch,
                                   llvm::BasicBlock *target,
                                   SmallVectorImpl<Value> &blockArguments);
-  /// Appends the converted result type and operands of `callInst` to the
-  /// `types` and `operands` arrays. For indirect calls, the method additionally
-  /// inserts the called function at the beginning of the `operands` array.
-  /// If `allowInlineAsm` is set to false (the default), it will return failure
-  /// if the called operand is an inline asm which isn't convertible to MLIR as
-  /// a value.
-  LogicalResult convertCallTypeAndOperands(llvm::CallBase *callInst,
-                                           SmallVectorImpl<Type> &types,
-                                           SmallVectorImpl<Value> &operands,
-                                           bool allowInlineAsm = false);
-  /// Converts the parameter attributes attached to `func` and adds them to the
-  /// `funcOp`.
+  /// Convert `callInst` operands. For indirect calls, the method additionally
+  /// inserts the called function at the beginning of the returned `operands`
+  /// array.  If `allowInlineAsm` is set to false (the default), it will return
+  /// failure if the called operand is an inline asm which isn't convertible to
+  /// MLIR as a value.
+  FailureOr<SmallVector<Value>>
+  convertCallOperands(llvm::CallBase *callInst, bool allowInlineAsm = false);
+  /// Converts the callee's function type. For direct calls, it converts the
+  /// actual function type, which may differ from the called operand type in
+  /// variadic functions. For indirect calls, it converts the function type
+  /// associated with the call instruction. Returns failure when the call and
+  /// the callee are not compatible or when nested type conversions failed.
+  FailureOr<LLVMFunctionType> convertFunctionType(llvm::CallBase *callInst);
+  /// Returns the callee name, or an empty symbol if the call is not direct.
+  FlatSymbolRefAttr convertCalleeName(llvm::CallBase *callInst);
+  /// Converts the parameter and result attributes attached to `func` and adds
+  /// them to the `funcOp`.
   void convertParameterAttributes(llvm::Function *func, LLVMFuncOp funcOp,
                                   OpBuilder &builder);
   /// Converts the AttributeSet of one parameter in LLVM IR to a corresponding
   /// DictionaryAttr for the LLVM dialect.
   DictionaryAttr convertParameterAttribute(llvm::AttributeSet llvmParamAttrs,
                                            OpBuilder &builder);
+  /// Converts the parameter and result attributes attached to `call` and adds
+  /// them to the `callOp`.
+  void convertParameterAttributes(llvm::CallBase *call, CallOpInterface callOp,
+                                  OpBuilder &builder);
+  /// Converts the attributes attached to `inst` and adds them to the `op`.
+  LogicalResult convertCallAttributes(llvm::CallInst *inst, CallOp op);
+  /// Converts the attributes attached to `inst` and adds them to the `op`.
+  LogicalResult convertInvokeAttributes(llvm::InvokeInst *inst, InvokeOp op);
   /// Returns the builtin type equivalent to the given LLVM dialect type or
   /// nullptr if there is no equivalent. The returned type can be used to create
   /// an attribute for a GlobalOp or a ConstantOp.
@@ -393,6 +416,8 @@ class ModuleImport {
   Operation *constantInsertionOp = nullptr;
   /// Operation to insert the next global after.
   Operation *globalInsertionOp = nullptr;
+  /// Operation to insert the next alias after.
+  Operation *aliasInsertionOp = nullptr;
   /// Operation to insert comdat selector operations into.
   ComdatOp globalComdatOp = nullptr;
   /// The current context.
diff --git a/mlir/include/mlir/Target/LLVMIR/ModuleTranslation.h b/mlir/include/mlir/Target/LLVMIR/ModuleTranslation.h
index 1b62437761ed9..e86d576bdb241 100644
--- a/mlir/include/mlir/Target/LLVMIR/ModuleTranslation.h
+++ b/mlir/include/mlir/Target/LLVMIR/ModuleTranslation.h
@@ -23,12 +23,13 @@
 #include "mlir/Target/LLVMIR/TypeToLLVM.h"
 
 #include "llvm/ADT/SetVector.h"
-#include "llvm/Frontend/OpenMP/OMPIRBuilder.h"
+#include "llvm/IR/FPEnv.h"
 
 namespace llvm {
 class BasicBlock;
-class IRBuilderBase;
 class Function;
+class IRBuilderBase;
+class OpenMPIRBuilder;
 class Value;
 } // namespace llvm
 
@@ -188,6 +189,12 @@ class ModuleTranslation {
     return globalsMapping.lookup(op);
   }
 
+  /// Finds an LLVM IR global value that corresponds to the given MLIR operation
+  /// defining a global alias value.
+  llvm::GlobalValue *lookupAlias(Operation *op) {
+    return aliasesMapping.lookup(op);
+  }
+
   /// Returns the OpenMP IR builder associated with the LLVM IR module being
   /// constructed.
   llvm::OpenMPIRBuilder *getOpenMPBuilder();
@@ -228,6 +235,11 @@ class ModuleTranslation {
                             /*recordInsertions=*/false);
   }
 
+  /// Translates parameter attributes of a call and adds them to the returned
+  /// AttrBuilder. Returns failure if any of the translations failed.
+  FailureOr<llvm::AttrBuilder> convertParameterAttrs(CallOpInterface callOp,
+                                                     DictionaryAttr paramAttrs);
+
   /// Gets the named metadata in the LLVM IR module being constructed, creating
   /// it if it does not exist.
   llvm::NamedMDNode *getOrInsertNamedModuleMetadata(StringRef name);
@@ -321,7 +333,13 @@ class ModuleTranslation {
   LogicalResult convertFunctionSignatures();
   LogicalResult convertFunctions();
   LogicalResult convertComdats();
-  LogicalResult convertGlobals();
+
+  /// Handle conversion for both globals and global aliases.
+  ///
+  /// - Create named global variables that correspond to llvm.mlir.global
+  /// definitions, similarly Convert llvm.global_ctors and global_dtors ops.
+  /// - Create global alias that correspond to llvm.mlir.alias.
+  LogicalResult convertGlobalsAndAliases();
   LogicalResult convertOneFunction(LLVMFuncOp func);
   LogicalResult convertBlockImpl(Block &bb, bool ignoreArguments,
                                  llvm::IRBuilderBase &builder,
@@ -346,8 +364,8 @@ class ModuleTranslation {
   convertDialectAttributes(Operation *op,
                            ArrayRef<llvm::Instruction *> instructions);
 
-  /// Translates parameter attributes and adds them to the returned AttrBuilder.
-  /// Returns failure if any of the translations failed.
+  /// Translates parameter attributes of a function and adds them to the
+  /// returned AttrBuilder. Returns failure if any of the translations failed.
   FailureOr<llvm::AttrBuilder>
   convertParameterAttrs(LLVMFuncOp func, int argIdx, DictionaryAttr paramAttrs);
 
@@ -366,6 +384,10 @@ class ModuleTranslation {
   /// Mappings between llvm.mlir.global definitions and corresponding globals.
   DenseMap<Operation *, llvm::GlobalValue *> globalsMapping;
 
+  /// Mappings between llvm.mlir.alias definitions and corresponding global
+  /// aliases.
+  DenseMap<Operation *, llvm::GlobalValue *> aliasesMapping;
+
   /// A stateful object used to translate types.
   TypeToLLVMIRTranslator typeTranslator;
 
diff --git a/mlir/include/mlir/Transforms/Passes.td b/mlir/include/mlir/Transforms/Passes.td
index c4a8e7a81fa48..a39ab77fc8fb3 100644
--- a/mlir/include/mlir/Transforms/Passes.td
+++ b/mlir/include/mlir/Transforms/Passes.td
@@ -28,6 +28,7 @@ def Canonicalizer : Pass<"canonicalize"> {
     details.
   }];
   let constructor = "mlir::createCanonicalizerPass()";
+  let dependentDialects = ["ub::UBDialect"];
   let options = [
     Option<"topDownProcessingEnabled", "top-down", "bool",
            /*default=*/"true",
diff --git a/mlir/lib/Analysis/DataFlowFramework.cpp b/mlir/lib/Analysis/DataFlowFramework.cpp
index d2742c6e4b966..028decbae31c3 100644
--- a/mlir/lib/Analysis/DataFlowFramework.cpp
+++ b/mlir/lib/Analysis/DataFlowFramework.cpp
@@ -10,6 +10,7 @@
 #include "mlir/IR/Location.h"
 #include "mlir/IR/Operation.h"
 #include "mlir/IR/Value.h"
+#include "llvm/ADT/ScopeExit.h"
 #include "llvm/ADT/iterator.h"
 #include "llvm/Config/abi-breaking.h"
 #include "llvm/Support/Casting.h"
@@ -104,6 +105,10 @@ Location LatticeAnchor::getLoc() const {
 //===----------------------------------------------------------------------===//
 
 LogicalResult DataFlowSolver::initializeAndRun(Operation *top) {
+  // Enable enqueue to the worklist.
+  isRunning = true;
+  auto guard = llvm::make_scope_exit([&]() { isRunning = false; });
+
   // Initialize the analyses.
   for (DataFlowAnalysis &analysis : llvm::make_pointee_range(childAnalyses)) {
     DATAFLOW_DEBUG(llvm::dbgs()
@@ -134,6 +139,8 @@ LogicalResult DataFlowSolver::initializeAndRun(Operation *top) {
 
 void DataFlowSolver::propagateIfChanged(AnalysisState *state,
                                         ChangeResult changed) {
+  assert(isRunning &&
+         "DataFlowSolver is not running, should not use propagateIfChanged");
   if (changed == ChangeResult::Change) {
     DATAFLOW_DEBUG(llvm::dbgs() << "Propagating update to " << state->debugName
                                 << " of " << state->anchor << "\n"
diff --git a/mlir/lib/AsmParser/AsmParserImpl.h b/mlir/lib/AsmParser/AsmParserImpl.h
index d5b72d63813a4..1f8fbfdd93568 100644
--- a/mlir/lib/AsmParser/AsmParserImpl.h
+++ b/mlir/lib/AsmParser/AsmParserImpl.h
@@ -248,13 +248,7 @@ class AsmParserImpl : public BaseT {
 
   /// Parses a quoted string token if present.
   ParseResult parseOptionalString(std::string *string) override {
-    if (!parser.getToken().is(Token::string))
-      return failure();
-
-    if (string)
-      *string = parser.getToken().getStringValue();
-    parser.consumeToken();
-    return success();
+    return parser.parseOptionalString(string);
   }
 
   /// Parses a Base64 encoded string of bytes.
@@ -355,13 +349,7 @@ class AsmParserImpl : public BaseT {
 
   /// Parse a keyword, if present, into 'keyword'.
   ParseResult parseOptionalKeyword(StringRef *keyword) override {
-    // Check that the current token is a keyword.
-    if (!parser.isCurrentTokenAKeyword())
-      return failure();
-
-    *keyword = parser.getTokenSpelling();
-    parser.consumeToken();
-    return success();
+    return parser.parseOptionalKeyword(keyword);
   }
 
   /// Parse a keyword if it is one of the 'allowedKeywords'.
@@ -387,13 +375,7 @@ class AsmParserImpl : public BaseT {
 
   /// Parse an optional keyword or string and set instance into 'result'.`
   ParseResult parseOptionalKeywordOrString(std::string *result) override {
-    StringRef keyword;
-    if (succeeded(parseOptionalKeyword(&keyword))) {
-      *result = keyword.str();
-      return success();
-    }
-
-    return parseOptionalString(result);
+    return parser.parseOptionalKeywordOrString(result);
   }
 
   //===--------------------------------------------------------------------===//
@@ -514,7 +496,7 @@ class AsmParserImpl : public BaseT {
       return parser.emitError() << "dialect '" << dialect->getNamespace()
                                 << "' does not expect resource handles";
     }
-    StringRef resourceName;
+    std::string resourceName;
     return parser.parseResourceHandle(interface, resourceName);
   }
 
diff --git a/mlir/lib/AsmParser/Parser.cpp b/mlir/lib/AsmParser/Parser.cpp
index eccb3241012a2..b5f1d2e27c9ba 100644
--- a/mlir/lib/AsmParser/Parser.cpp
+++ b/mlir/lib/AsmParser/Parser.cpp
@@ -271,6 +271,17 @@ ParseResult Parser::parseToken(Token::Kind expectedToken,
   return emitWrongTokenError(message);
 }
 
+/// Parses a quoted string token if present.
+ParseResult Parser::parseOptionalString(std::string *string) {
+  if (!getToken().is(Token::string))
+    return failure();
+
+  if (string)
+    *string = getToken().getStringValue();
+  consumeToken();
+  return success();
+}
+
 /// Parse an optional integer value from the stream.
 OptionalParseResult Parser::parseOptionalInteger(APInt &result) {
   // Parse `false` and `true` keywords as 0 and 1 respectively.
@@ -412,15 +423,25 @@ ParseResult Parser::parseOptionalKeyword(StringRef *keyword) {
   return success();
 }
 
+ParseResult Parser::parseOptionalKeywordOrString(std::string *result) {
+  StringRef keyword;
+  if (succeeded(parseOptionalKeyword(&keyword))) {
+    *result = keyword.str();
+    return success();
+  }
+
+  return parseOptionalString(result);
+}
+
 //===----------------------------------------------------------------------===//
 // Resource Parsing
 
 FailureOr<AsmDialectResourceHandle>
 Parser::parseResourceHandle(const OpAsmDialectInterface *dialect,
-                            StringRef &name) {
+                            std::string &name) {
   assert(dialect && "expected valid dialect interface");
   SMLoc nameLoc = getToken().getLoc();
-  if (failed(parseOptionalKeyword(&name)))
+  if (failed(parseOptionalKeywordOrString(&name)))
     return emitError("expected identifier key for 'resource' entry");
   auto &resources = getState().symbols.dialectResources;
 
@@ -451,7 +472,7 @@ Parser::parseResourceHandle(Dialect *dialect) {
     return emitError() << "dialect '" << dialect->getNamespace()
                        << "' does not expect resource handles";
   }
-  StringRef resourceName;
+  std::string resourceName;
   return parseResourceHandle(interface, resourceName);
 }
 
@@ -2530,8 +2551,8 @@ class TopLevelOperationParser : public Parser {
 /// textual format.
 class ParsedResourceEntry : public AsmParsedResourceEntry {
 public:
-  ParsedResourceEntry(StringRef key, SMLoc keyLoc, Token value, Parser &p)
-      : key(key), keyLoc(keyLoc), value(value), p(p) {}
+  ParsedResourceEntry(std::string key, SMLoc keyLoc, Token value, Parser &p)
+      : key(std::move(key)), keyLoc(keyLoc), value(value), p(p) {}
   ~ParsedResourceEntry() override = default;
 
   StringRef getKey() const final { return key; }
@@ -2607,7 +2628,7 @@ class ParsedResourceEntry : public AsmParsedResourceEntry {
   }
 
 private:
-  StringRef key;
+  std::string key;
   SMLoc keyLoc;
   Token value;
   Parser &p;
@@ -2736,7 +2757,7 @@ ParseResult TopLevelOperationParser::parseDialectResourceFileMetadata() {
     return parseCommaSeparatedListUntil(Token::r_brace, [&]() -> ParseResult {
       // Parse the name of the resource entry.
       SMLoc keyLoc = getToken().getLoc();
-      StringRef key;
+      std::string key;
       if (failed(parseResourceHandle(handler, key)) ||
           parseToken(Token::colon, "expected ':'"))
         return failure();
@@ -2763,8 +2784,8 @@ ParseResult TopLevelOperationParser::parseExternalResourceFileMetadata() {
     return parseCommaSeparatedListUntil(Token::r_brace, [&]() -> ParseResult {
       // Parse the name of the resource entry.
       SMLoc keyLoc = getToken().getLoc();
-      StringRef key;
-      if (failed(parseOptionalKeyword(&key)))
+      std::string key;
+      if (failed(parseOptionalKeywordOrString(&key)))
         return emitError(
             "expected identifier key for 'external_resources' entry");
       if (parseToken(Token::colon, "expected ':'"))
diff --git a/mlir/lib/AsmParser/Parser.h b/mlir/lib/AsmParser/Parser.h
index 37670bd789fec..1b8aa7c9dce6f 100644
--- a/mlir/lib/AsmParser/Parser.h
+++ b/mlir/lib/AsmParser/Parser.h
@@ -146,6 +146,9 @@ class Parser {
   /// output a diagnostic and return failure.
   ParseResult parseToken(Token::Kind expectedToken, const Twine &message);
 
+  /// Parses a quoted string token if present.
+  ParseResult parseOptionalString(std::string *string);
+
   /// Parse an optional integer value from the stream.
   OptionalParseResult parseOptionalInteger(APInt &result);
 
@@ -171,13 +174,16 @@ class Parser {
   /// Parse a keyword, if present, into 'keyword'.
   ParseResult parseOptionalKeyword(StringRef *keyword);
 
+  /// Parse an optional keyword or string and set instance into 'result'.`
+  ParseResult parseOptionalKeywordOrString(std::string *result);
+
   //===--------------------------------------------------------------------===//
   // Resource Parsing
   //===--------------------------------------------------------------------===//
 
   /// Parse a handle to a dialect resource within the assembly format.
   FailureOr<AsmDialectResourceHandle>
-  parseResourceHandle(const OpAsmDialectInterface *dialect, StringRef &name);
+  parseResourceHandle(const OpAsmDialectInterface *dialect, std::string &name);
   FailureOr<AsmDialectResourceHandle> parseResourceHandle(Dialect *dialect);
 
   //===--------------------------------------------------------------------===//
diff --git a/mlir/lib/Bindings/Python/IRAttributes.cpp b/mlir/lib/Bindings/Python/IRAttributes.cpp
index 7bc21a31c3c84..12725a0ed0939 100644
--- a/mlir/lib/Bindings/Python/IRAttributes.cpp
+++ b/mlir/lib/Bindings/Python/IRAttributes.cpp
@@ -1372,13 +1372,19 @@ class PyDenseIntElementsAttribute
 
     MlirType type = mlirAttributeGetType(*this);
     type = mlirShapedTypeGetElementType(type);
-    assert(mlirTypeIsAInteger(type) &&
-           "expected integer element type in dense int elements attribute");
+    // Index type can also appear as a DenseIntElementsAttr and therefore can be
+    // casted to integer.
+    assert(mlirTypeIsAInteger(type) ||
+           mlirTypeIsAIndex(type) && "expected integer/index element type in "
+                                     "dense int elements attribute");
     // Dispatch element extraction to an appropriate C function based on the
     // elemental type of the attribute. nb::int_ is implicitly constructible
     // from any C++ integral type and handles bitwidth correctly.
     // TODO: consider caching the type properties in the constructor to avoid
     // querying them on each element access.
+    if (mlirTypeIsAIndex(type)) {
+      return nb::int_(mlirDenseElementsAttrGetIndexValue(*this, pos));
+    }
     unsigned width = mlirIntegerTypeGetWidth(type);
     bool isUnsigned = mlirIntegerTypeIsUnsigned(type);
     if (isUnsigned) {
@@ -1468,7 +1474,10 @@ class PyDenseResourceElementsAttribute
     // The userData is a Py_buffer* that the deleter owns.
     auto deleter = [](void *userData, const void *data, size_t size,
                       size_t align) {
+      if (!Py_IsInitialized())
+        Py_Initialize();
       Py_buffer *ownedView = static_cast<Py_buffer *>(userData);
+      nb::gil_scoped_acquire gil;
       PyBuffer_Release(ownedView);
       delete ownedView;
     };
diff --git a/mlir/lib/Bindings/Python/IRCore.cpp b/mlir/lib/Bindings/Python/IRCore.cpp
index 8e351cb22eb94..47a85c2a486fd 100644
--- a/mlir/lib/Bindings/Python/IRCore.cpp
+++ b/mlir/lib/Bindings/Python/IRCore.cpp
@@ -1296,8 +1296,10 @@ void PyOperationBase::print(std::optional<int64_t> largeElementsLimit,
     fileObject = nb::module_::import_("sys").attr("stdout");
 
   MlirOpPrintingFlags flags = mlirOpPrintingFlagsCreate();
-  if (largeElementsLimit)
+  if (largeElementsLimit) {
     mlirOpPrintingFlagsElideLargeElementsAttrs(flags, *largeElementsLimit);
+    mlirOpPrintingFlagsElideLargeResourceString(flags, *largeElementsLimit);
+  }
   if (enableDebugInfo)
     mlirOpPrintingFlagsEnableDebugInfo(flags, /*enable=*/true,
                                        /*prettyForm=*/prettyDebugInfo);
diff --git a/mlir/lib/Bytecode/Writer/BytecodeWriter.cpp b/mlir/lib/Bytecode/Writer/BytecodeWriter.cpp
index 0e96aa97abeba..2b4697434717d 100644
--- a/mlir/lib/Bytecode/Writer/BytecodeWriter.cpp
+++ b/mlir/lib/Bytecode/Writer/BytecodeWriter.cpp
@@ -66,6 +66,9 @@ BytecodeWriterConfig::BytecodeWriterConfig(FallbackAsmResourceMap &map,
     : BytecodeWriterConfig(producer) {
   attachFallbackResourcePrinter(map);
 }
+BytecodeWriterConfig::BytecodeWriterConfig(BytecodeWriterConfig &&config)
+    : impl(std::move(config.impl)) {}
+
 BytecodeWriterConfig::~BytecodeWriterConfig() = default;
 
 ArrayRef<std::unique_ptr<AttrTypeBytecodeWriter<Attribute>>>
diff --git a/mlir/lib/CAPI/Dialect/Func.cpp b/mlir/lib/CAPI/Dialect/Func.cpp
index 942e090fdfc19..8265b61b98426 100644
--- a/mlir/lib/CAPI/Dialect/Func.cpp
+++ b/mlir/lib/CAPI/Dialect/Func.cpp
@@ -19,3 +19,9 @@ void mlirFuncSetArgAttr(MlirOperation op, intptr_t pos, MlirStringRef name,
   llvm::cast<mlir::func::FuncOp>(unwrap(op))
       .setArgAttr(pos, unwrap(name), unwrap(attr));
 }
+
+void mlirFuncSetResultAttr(MlirOperation op, intptr_t pos, MlirStringRef name,
+                           MlirAttribute attr) {
+  llvm::cast<mlir::func::FuncOp>(unwrap(op))
+      .setResultAttr(pos, unwrap(name), unwrap(attr));
+}
diff --git a/mlir/lib/CAPI/Dialect/LLVM.cpp b/mlir/lib/CAPI/Dialect/LLVM.cpp
index da450dd3fd8a3..69c804b7667f3 100644
--- a/mlir/lib/CAPI/Dialect/LLVM.cpp
+++ b/mlir/lib/CAPI/Dialect/LLVM.cpp
@@ -65,6 +65,10 @@ MlirType mlirLLVMFunctionTypeGetInput(MlirType type, intptr_t pos) {
                   .getParamType(static_cast<unsigned>(pos)));
 }
 
+MlirType mlirLLVMFunctionTypeGetReturnType(MlirType type) {
+  return wrap(llvm::cast<LLVM::LLVMFunctionType>(unwrap(type)).getReturnType());
+}
+
 bool mlirTypeIsALLVMStructType(MlirType type) {
   return isa<LLVM::LLVMStructType>(unwrap(type));
 }
diff --git a/mlir/lib/CAPI/IR/BuiltinAttributes.cpp b/mlir/lib/CAPI/IR/BuiltinAttributes.cpp
index 11d1ade552f5a..8d57ab6b59e79 100644
--- a/mlir/lib/CAPI/IR/BuiltinAttributes.cpp
+++ b/mlir/lib/CAPI/IR/BuiltinAttributes.cpp
@@ -758,6 +758,9 @@ int64_t mlirDenseElementsAttrGetInt64Value(MlirAttribute attr, intptr_t pos) {
 uint64_t mlirDenseElementsAttrGetUInt64Value(MlirAttribute attr, intptr_t pos) {
   return llvm::cast<DenseElementsAttr>(unwrap(attr)).getValues<uint64_t>()[pos];
 }
+uint64_t mlirDenseElementsAttrGetIndexValue(MlirAttribute attr, intptr_t pos) {
+  return llvm::cast<DenseElementsAttr>(unwrap(attr)).getValues<uint64_t>()[pos];
+}
 float mlirDenseElementsAttrGetFloatValue(MlirAttribute attr, intptr_t pos) {
   return llvm::cast<DenseElementsAttr>(unwrap(attr)).getValues<float>()[pos];
 }
diff --git a/mlir/lib/CAPI/Target/CMakeLists.txt b/mlir/lib/CAPI/Target/CMakeLists.txt
index ce86fd3def964..ea617da72261b 100644
--- a/mlir/lib/CAPI/Target/CMakeLists.txt
+++ b/mlir/lib/CAPI/Target/CMakeLists.txt
@@ -8,5 +8,6 @@ add_mlir_upstream_c_api_library(MLIRCAPITarget
   MLIRToLLVMIRTranslationRegistration
   MLIRCAPIIR
   MLIRLLVMToLLVMIRTranslation
+  MLIRLLVMIRToLLVMTranslation
   MLIRSupport
 )
diff --git a/mlir/lib/CAPI/Target/LLVMIR.cpp b/mlir/lib/CAPI/Target/LLVMIR.cpp
index dc798372be746..1c1912aec0f2f 100644
--- a/mlir/lib/CAPI/Target/LLVMIR.cpp
+++ b/mlir/lib/CAPI/Target/LLVMIR.cpp
@@ -8,16 +8,15 @@
 //===----------------------------------------------------------------------===//
 
 #include "mlir-c/Target/LLVMIR.h"
-#include "llvm-c/Support.h"
 
 #include "llvm/IR/LLVMContext.h"
 #include "llvm/IR/Module.h"
-#include <memory>
+#include "llvm/IR/Type.h"
 
 #include "mlir/CAPI/IR.h"
-#include "mlir/CAPI/Support.h"
 #include "mlir/CAPI/Wrap.h"
 #include "mlir/Target/LLVMIR/ModuleTranslation.h"
+#include "mlir/Target/LLVMIR/TypeFromLLVM.h"
 
 using namespace mlir;
 
@@ -34,3 +33,47 @@ LLVMModuleRef mlirTranslateModuleToLLVMIR(MlirOperation module,
 
   return moduleRef;
 }
+
+DEFINE_C_API_PTR_METHODS(MlirTypeFromLLVMIRTranslator,
+                         mlir::LLVM::TypeFromLLVMIRTranslator)
+
+MlirTypeFromLLVMIRTranslator
+mlirTypeFromLLVMIRTranslatorCreate(MlirContext ctx) {
+  MLIRContext *context = unwrap(ctx);
+  auto *translator = new LLVM::TypeFromLLVMIRTranslator(*context);
+  return wrap(translator);
+}
+
+void mlirTypeFromLLVMIRTranslatorDestroy(
+    MlirTypeFromLLVMIRTranslator translator) {
+  delete static_cast<LLVM::TypeFromLLVMIRTranslator *>(unwrap(translator));
+}
+
+MlirType mlirTypeFromLLVMIRTranslatorTranslateType(
+    MlirTypeFromLLVMIRTranslator translator, LLVMTypeRef llvmType) {
+  LLVM::TypeFromLLVMIRTranslator *translator_ = unwrap(translator);
+  mlir::Type type = translator_->translateType(llvm::unwrap(llvmType));
+  return wrap(type);
+}
+
+DEFINE_C_API_PTR_METHODS(MlirTypeToLLVMIRTranslator,
+                         mlir::LLVM::TypeToLLVMIRTranslator)
+
+MlirTypeToLLVMIRTranslator
+mlirTypeToLLVMIRTranslatorCreate(LLVMContextRef ctx) {
+  llvm::LLVMContext *context = llvm::unwrap(ctx);
+  auto *translator = new LLVM::TypeToLLVMIRTranslator(*context);
+  return wrap(translator);
+}
+
+void mlirTypeToLLVMIRTranslatorDestroy(MlirTypeToLLVMIRTranslator translator) {
+  delete static_cast<LLVM::TypeToLLVMIRTranslator *>(unwrap(translator));
+}
+
+LLVMTypeRef
+mlirTypeToLLVMIRTranslatorTranslateType(MlirTypeToLLVMIRTranslator translator,
+                                        MlirType mlirType) {
+  LLVM::TypeToLLVMIRTranslator *translator_ = unwrap(translator);
+  llvm::Type *type = translator_->translateType(unwrap(mlirType));
+  return llvm::wrap(type);
+}
diff --git a/mlir/lib/Conversion/AMDGPUToROCDL/AMDGPUToROCDL.cpp b/mlir/lib/Conversion/AMDGPUToROCDL/AMDGPUToROCDL.cpp
index 51f5d7a161b90..9fb51f0bc1f1e 100644
--- a/mlir/lib/Conversion/AMDGPUToROCDL/AMDGPUToROCDL.cpp
+++ b/mlir/lib/Conversion/AMDGPUToROCDL/AMDGPUToROCDL.cpp
@@ -343,7 +343,7 @@ struct LDSBarrierOpLowering : public ConvertOpToLLVMPattern<LDSBarrierOp> {
                << chipset.majorVersion;
 
       Location loc = op->getLoc();
-      rewriter.create<ROCDL::WaitcntOp>(loc, ldsOnlyBits);
+      rewriter.create<ROCDL::SWaitcntOp>(loc, ldsOnlyBits);
       rewriter.replaceOpWithNewOp<ROCDL::SBarrierOp>(op);
     } else {
       Location loc = op->getLoc();
diff --git a/mlir/lib/Conversion/ArithToAMDGPU/ArithToAMDGPU.cpp b/mlir/lib/Conversion/ArithToAMDGPU/ArithToAMDGPU.cpp
index 33370566996ee..60a002c41bfb2 100644
--- a/mlir/lib/Conversion/ArithToAMDGPU/ArithToAMDGPU.cpp
+++ b/mlir/lib/Conversion/ArithToAMDGPU/ArithToAMDGPU.cpp
@@ -102,20 +102,23 @@ void ExtFOnFloat8RewritePattern::rewrite(arith::ExtFOp op,
     return rewriter.replaceOp(op, result);
   }
   int64_t numElements = inType.getNumElements();
+
   Value zero = rewriter.create<arith::ConstantOp>(
       loc, outElemType, rewriter.getFloatAttr(outElemType, 0.0));
+  VectorType outType = cast<VectorType>(op.getOut().getType());
+
   if (inType.getShape().empty()) {
+    Value zerodSplat =
+        rewriter.createOrFold<vector::SplatOp>(loc, outType, zero);
     Value scalarIn =
         rewriter.create<vector::ExtractOp>(loc, in, ArrayRef<int64_t>{});
-    // Recurse to send the 0-D vector case to the 1-D vector case
     Value scalarExt =
         rewriter.create<arith::ExtFOp>(loc, outElemType, scalarIn);
-    Value result = rewriter.create<vector::InsertOp>(loc, scalarExt, zero,
+    Value result = rewriter.create<vector::InsertOp>(loc, scalarExt, zerodSplat,
                                                      ArrayRef<int64_t>{});
     return rewriter.replaceOp(op, result);
   }
 
-  VectorType outType = cast<VectorType>(op.getOut().getType());
   VectorType flatTy = VectorType::get(SmallVector<int64_t>{numElements},
                                       outType.getElementType());
   Value result = rewriter.createOrFold<vector::SplatOp>(loc, flatTy, zero);
diff --git a/mlir/lib/Conversion/ArmSMEToLLVM/ArmSMEToLLVM.cpp b/mlir/lib/Conversion/ArmSMEToLLVM/ArmSMEToLLVM.cpp
index 40a3489f7a4d7..417555792b44f 100644
--- a/mlir/lib/Conversion/ArmSMEToLLVM/ArmSMEToLLVM.cpp
+++ b/mlir/lib/Conversion/ArmSMEToLLVM/ArmSMEToLLVM.cpp
@@ -317,7 +317,7 @@ struct ConvertArmSMESpillsAndFillsToLLVM : public ConvertToLLVMPattern {
     auto allTruePredicate = rewriter.create<arith::ConstantOp>(
         loc, DenseElementsAttr::get(predicateType, true));
     // Create padding vector (never used due to all-true predicate).
-    auto padVector = rewriter.create<LLVM::UndefOp>(loc, sliceType);
+    auto padVector = rewriter.create<LLVM::PoisonOp>(loc, sliceType);
     // Get a pointer to the current slice.
     auto slicePtr =
         getInMemoryTileSlicePtr(rewriter, loc, tileAlloca, sliceIndex);
diff --git a/mlir/lib/Conversion/AsyncToLLVM/AsyncToLLVM.cpp b/mlir/lib/Conversion/AsyncToLLVM/AsyncToLLVM.cpp
index 9b5aeb3fef30b..47d4474a5c28d 100644
--- a/mlir/lib/Conversion/AsyncToLLVM/AsyncToLLVM.cpp
+++ b/mlir/lib/Conversion/AsyncToLLVM/AsyncToLLVM.cpp
@@ -396,8 +396,10 @@ class CoroBeginOpConversion : public AsyncOpConversionPattern<CoroBeginOp> {
     // Allocate memory for the coroutine frame.
     auto allocFuncOp = LLVM::lookupOrCreateAlignedAllocFn(
         op->getParentOfType<ModuleOp>(), rewriter.getI64Type());
+    if (failed(allocFuncOp))
+      return failure();
     auto coroAlloc = rewriter.create<LLVM::CallOp>(
-        loc, allocFuncOp, ValueRange{coroAlign, coroSize});
+        loc, allocFuncOp.value(), ValueRange{coroAlign, coroSize});
 
     // Begin a coroutine: @llvm.coro.begin.
     auto coroId = CoroBeginOpAdaptor(adaptor.getOperands()).getId();
@@ -431,7 +433,9 @@ class CoroFreeOpConversion : public AsyncOpConversionPattern<CoroFreeOp> {
     // Free the memory.
     auto freeFuncOp =
         LLVM::lookupOrCreateFreeFn(op->getParentOfType<ModuleOp>());
-    rewriter.replaceOpWithNewOp<LLVM::CallOp>(op, freeFuncOp,
+    if (failed(freeFuncOp))
+      return failure();
+    rewriter.replaceOpWithNewOp<LLVM::CallOp>(op, freeFuncOp.value(),
                                               ValueRange(coroMem.getResult()));
 
     return success();
diff --git a/mlir/lib/Conversion/CMakeLists.txt b/mlir/lib/Conversion/CMakeLists.txt
index a570978f03757..0bd08ec6333e6 100644
--- a/mlir/lib/Conversion/CMakeLists.txt
+++ b/mlir/lib/Conversion/CMakeLists.txt
@@ -19,7 +19,6 @@ add_subdirectory(ControlFlowToLLVM)
 add_subdirectory(ControlFlowToSCF)
 add_subdirectory(ControlFlowToSPIRV)
 add_subdirectory(ConvertToLLVM)
-add_subdirectory(ConvertToSPIRV)
 add_subdirectory(FuncToEmitC)
 add_subdirectory(FuncToLLVM)
 add_subdirectory(FuncToSPIRV)
diff --git a/mlir/lib/Conversion/ComplexToLLVM/ComplexToLLVM.cpp b/mlir/lib/Conversion/ComplexToLLVM/ComplexToLLVM.cpp
index fcc18e6eee998..ad86fe362076b 100644
--- a/mlir/lib/Conversion/ComplexToLLVM/ComplexToLLVM.cpp
+++ b/mlir/lib/Conversion/ComplexToLLVM/ComplexToLLVM.cpp
@@ -33,9 +33,9 @@ using namespace mlir::arith;
 static constexpr unsigned kRealPosInComplexNumberStruct = 0;
 static constexpr unsigned kImaginaryPosInComplexNumberStruct = 1;
 
-ComplexStructBuilder ComplexStructBuilder::undef(OpBuilder &builder,
-                                                 Location loc, Type type) {
-  Value val = builder.create<LLVM::UndefOp>(loc, type);
+ComplexStructBuilder ComplexStructBuilder::poison(OpBuilder &builder,
+                                                  Location loc, Type type) {
+  Value val = builder.create<LLVM::PoisonOp>(loc, type);
   return ComplexStructBuilder(val);
 }
 
@@ -109,7 +109,8 @@ struct CreateOpConversion : public ConvertOpToLLVMPattern<complex::CreateOp> {
     // Pack real and imaginary part in a complex number struct.
     auto loc = complexOp.getLoc();
     auto structType = typeConverter->convertType(complexOp.getType());
-    auto complexStruct = ComplexStructBuilder::undef(rewriter, loc, structType);
+    auto complexStruct =
+        ComplexStructBuilder::poison(rewriter, loc, structType);
     complexStruct.setReal(rewriter, loc, adaptor.getReal());
     complexStruct.setImaginary(rewriter, loc, adaptor.getImaginary());
 
@@ -183,7 +184,7 @@ struct AddOpConversion : public ConvertOpToLLVMPattern<complex::AddOp> {
 
     // Initialize complex number struct for result.
     auto structType = typeConverter->convertType(op.getType());
-    auto result = ComplexStructBuilder::undef(rewriter, loc, structType);
+    auto result = ComplexStructBuilder::poison(rewriter, loc, structType);
 
     // Emit IR to add complex numbers.
     arith::FastMathFlagsAttr complexFMFAttr = op.getFastMathFlagsAttr();
@@ -214,7 +215,7 @@ struct DivOpConversion : public ConvertOpToLLVMPattern<complex::DivOp> {
 
     // Initialize complex number struct for result.
     auto structType = typeConverter->convertType(op.getType());
-    auto result = ComplexStructBuilder::undef(rewriter, loc, structType);
+    auto result = ComplexStructBuilder::poison(rewriter, loc, structType);
 
     // Emit IR to add complex numbers.
     arith::FastMathFlagsAttr complexFMFAttr = op.getFastMathFlagsAttr();
@@ -262,7 +263,7 @@ struct MulOpConversion : public ConvertOpToLLVMPattern<complex::MulOp> {
 
     // Initialize complex number struct for result.
     auto structType = typeConverter->convertType(op.getType());
-    auto result = ComplexStructBuilder::undef(rewriter, loc, structType);
+    auto result = ComplexStructBuilder::poison(rewriter, loc, structType);
 
     // Emit IR to add complex numbers.
     arith::FastMathFlagsAttr complexFMFAttr = op.getFastMathFlagsAttr();
@@ -302,7 +303,7 @@ struct SubOpConversion : public ConvertOpToLLVMPattern<complex::SubOp> {
 
     // Initialize complex number struct for result.
     auto structType = typeConverter->convertType(op.getType());
-    auto result = ComplexStructBuilder::undef(rewriter, loc, structType);
+    auto result = ComplexStructBuilder::poison(rewriter, loc, structType);
 
     // Emit IR to substract complex numbers.
     arith::FastMathFlagsAttr complexFMFAttr = op.getFastMathFlagsAttr();
diff --git a/mlir/lib/Conversion/ControlFlowToLLVM/ControlFlowToLLVM.cpp b/mlir/lib/Conversion/ControlFlowToLLVM/ControlFlowToLLVM.cpp
index d0ffb94f3f96a..debfd003bd5b5 100644
--- a/mlir/lib/Conversion/ControlFlowToLLVM/ControlFlowToLLVM.cpp
+++ b/mlir/lib/Conversion/ControlFlowToLLVM/ControlFlowToLLVM.cpp
@@ -61,9 +61,13 @@ struct AssertOpLowering : public ConvertOpToLLVMPattern<cf::AssertOp> {
 
     // Failed block: Generate IR to print the message and call `abort`.
     Block *failureBlock = rewriter.createBlock(opBlock->getParent());
-    LLVM::createPrintStrCall(rewriter, loc, module, "assert_msg", op.getMsg(),
-                             *getTypeConverter(), /*addNewLine=*/false,
-                             /*runtimeFunctionName=*/"puts");
+    auto createResult = LLVM::createPrintStrCall(
+        rewriter, loc, module, "assert_msg", op.getMsg(), *getTypeConverter(),
+        /*addNewLine=*/false,
+        /*runtimeFunctionName=*/"puts");
+    if (createResult.failed())
+      return failure();
+
     if (abortOnFailedAssert) {
       // Insert the `abort` declaration if necessary.
       auto abortFunc = module.lookupSymbol<LLVM::LLVMFuncOp>("abort");
diff --git a/mlir/lib/Conversion/ConvertToSPIRV/CMakeLists.txt b/mlir/lib/Conversion/ConvertToSPIRV/CMakeLists.txt
deleted file mode 100644
index 124a4c453e75c..0000000000000
--- a/mlir/lib/Conversion/ConvertToSPIRV/CMakeLists.txt
+++ /dev/null
@@ -1,36 +0,0 @@
-set(LLVM_OPTIONAL_SOURCES
-  ConvertToSPIRVPass.cpp
-)
-
-add_mlir_conversion_library(MLIRConvertToSPIRVPass
-  ConvertToSPIRVPass.cpp
-
-  ADDITIONAL_HEADER_DIRS
-  ${MLIR_MAIN_INCLUDE_DIR}/mlir/Conversion/ConvertToSPIRV
-
-  DEPENDS
-  MLIRConversionPassIncGen
-
-  LINK_LIBS PUBLIC
-  MLIRArithToSPIRV
-  MLIRArithTransforms
-  MLIRFuncToSPIRV
-  MLIRGPUDialect
-  MLIRGPUToSPIRV
-  MLIRIndexToSPIRV
-  MLIRIR
-  MLIRMemRefToSPIRV
-  MLIRPass
-  MLIRRewrite
-  MLIRSCFToSPIRV
-  MLIRSPIRVConversion
-  MLIRSPIRVDialect
-  MLIRSPIRVTransforms
-  MLIRSupport
-  MLIRTransforms
-  MLIRTransformUtils
-  MLIRUBToSPIRV
-  MLIRVectorDialect
-  MLIRVectorToSPIRV
-  MLIRVectorTransforms
-  )
diff --git a/mlir/lib/Conversion/ConvertToSPIRV/ConvertToSPIRVPass.cpp b/mlir/lib/Conversion/ConvertToSPIRV/ConvertToSPIRVPass.cpp
deleted file mode 100644
index ab9c048f56106..0000000000000
--- a/mlir/lib/Conversion/ConvertToSPIRV/ConvertToSPIRVPass.cpp
+++ /dev/null
@@ -1,136 +0,0 @@
-//===- ConvertToSPIRVPass.cpp - MLIR SPIR-V Conversion --------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "mlir/Conversion/ConvertToSPIRV/ConvertToSPIRVPass.h"
-#include "mlir/Conversion/ArithToSPIRV/ArithToSPIRV.h"
-#include "mlir/Conversion/FuncToSPIRV/FuncToSPIRV.h"
-#include "mlir/Conversion/GPUToSPIRV/GPUToSPIRV.h"
-#include "mlir/Conversion/IndexToSPIRV/IndexToSPIRV.h"
-#include "mlir/Conversion/MemRefToSPIRV/MemRefToSPIRV.h"
-#include "mlir/Conversion/SCFToSPIRV/SCFToSPIRV.h"
-#include "mlir/Conversion/UBToSPIRV/UBToSPIRV.h"
-#include "mlir/Conversion/VectorToSPIRV/VectorToSPIRV.h"
-#include "mlir/Dialect/Arith/Transforms/Passes.h"
-#include "mlir/Dialect/GPU/IR/GPUDialect.h"
-#include "mlir/Dialect/SPIRV/IR/SPIRVAttributes.h"
-#include "mlir/Dialect/SPIRV/IR/SPIRVDialect.h"
-#include "mlir/Dialect/SPIRV/Transforms/SPIRVConversion.h"
-#include "mlir/Dialect/Vector/IR/VectorOps.h"
-#include "mlir/Dialect/Vector/Transforms/LoweringPatterns.h"
-#include "mlir/Dialect/Vector/Transforms/VectorRewritePatterns.h"
-#include "mlir/IR/PatternMatch.h"
-#include "mlir/Pass/Pass.h"
-#include "mlir/Rewrite/FrozenRewritePatternSet.h"
-#include "mlir/Transforms/DialectConversion.h"
-#include "mlir/Transforms/GreedyPatternRewriteDriver.h"
-#include <memory>
-
-#define DEBUG_TYPE "convert-to-spirv"
-
-namespace mlir {
-#define GEN_PASS_DEF_CONVERTTOSPIRVPASS
-#include "mlir/Conversion/Passes.h.inc"
-} // namespace mlir
-
-using namespace mlir;
-
-namespace {
-
-/// Map memRef memory space to SPIR-V storage class.
-void mapToMemRef(Operation *op, spirv::TargetEnvAttr &targetAttr) {
-  spirv::TargetEnv targetEnv(targetAttr);
-  bool targetEnvSupportsKernelCapability =
-      targetEnv.allows(spirv::Capability::Kernel);
-  spirv::MemorySpaceToStorageClassMap memorySpaceMap =
-      targetEnvSupportsKernelCapability
-          ? spirv::mapMemorySpaceToOpenCLStorageClass
-          : spirv::mapMemorySpaceToVulkanStorageClass;
-  spirv::MemorySpaceToStorageClassConverter converter(memorySpaceMap);
-  spirv::convertMemRefTypesAndAttrs(op, converter);
-}
-
-/// Populate patterns for each dialect.
-void populateConvertToSPIRVPatterns(const SPIRVTypeConverter &typeConverter,
-                                    ScfToSPIRVContext &scfToSPIRVContext,
-                                    RewritePatternSet &patterns) {
-  arith::populateCeilFloorDivExpandOpsPatterns(patterns);
-  arith::populateArithToSPIRVPatterns(typeConverter, patterns);
-  populateBuiltinFuncToSPIRVPatterns(typeConverter, patterns);
-  populateFuncToSPIRVPatterns(typeConverter, patterns);
-  populateGPUToSPIRVPatterns(typeConverter, patterns);
-  index::populateIndexToSPIRVPatterns(typeConverter, patterns);
-  populateMemRefToSPIRVPatterns(typeConverter, patterns);
-  populateVectorToSPIRVPatterns(typeConverter, patterns);
-  populateSCFToSPIRVPatterns(typeConverter, scfToSPIRVContext, patterns);
-  ub::populateUBToSPIRVConversionPatterns(typeConverter, patterns);
-}
-
-/// A pass to perform the SPIR-V conversion.
-struct ConvertToSPIRVPass final
-    : impl::ConvertToSPIRVPassBase<ConvertToSPIRVPass> {
-  using ConvertToSPIRVPassBase::ConvertToSPIRVPassBase;
-
-  void runOnOperation() override {
-    Operation *op = getOperation();
-    MLIRContext *context = &getContext();
-
-    // Unroll vectors in function signatures to native size.
-    if (runSignatureConversion && failed(spirv::unrollVectorsInSignatures(op)))
-      return signalPassFailure();
-
-    // Unroll vectors in function bodies to native size.
-    if (runVectorUnrolling && failed(spirv::unrollVectorsInFuncBodies(op)))
-      return signalPassFailure();
-
-    // Generic conversion.
-    if (!convertGPUModules) {
-      spirv::TargetEnvAttr targetAttr = spirv::lookupTargetEnvOrDefault(op);
-      std::unique_ptr<ConversionTarget> target =
-          SPIRVConversionTarget::get(targetAttr);
-      SPIRVTypeConverter typeConverter(targetAttr);
-      RewritePatternSet patterns(context);
-      ScfToSPIRVContext scfToSPIRVContext;
-      mapToMemRef(op, targetAttr);
-      populateConvertToSPIRVPatterns(typeConverter, scfToSPIRVContext,
-                                     patterns);
-      if (failed(applyPartialConversion(op, *target, std::move(patterns))))
-        return signalPassFailure();
-      return;
-    }
-
-    // Clone each GPU kernel module for conversion, given that the GPU
-    // launch op still needs the original GPU kernel module.
-    SmallVector<Operation *, 1> gpuModules;
-    OpBuilder builder(context);
-    op->walk([&](gpu::GPUModuleOp gpuModule) {
-      if (nestInGPUModule)
-        builder.setInsertionPointToStart(gpuModule.getBody());
-      else
-        builder.setInsertionPoint(gpuModule);
-      gpuModules.push_back(builder.clone(*gpuModule));
-    });
-    // Run conversion for each module independently as they can have
-    // different TargetEnv attributes.
-    for (Operation *gpuModule : gpuModules) {
-      spirv::TargetEnvAttr targetAttr =
-          spirv::lookupTargetEnvOrDefault(gpuModule);
-      std::unique_ptr<ConversionTarget> target =
-          SPIRVConversionTarget::get(targetAttr);
-      SPIRVTypeConverter typeConverter(targetAttr);
-      RewritePatternSet patterns(context);
-      ScfToSPIRVContext scfToSPIRVContext;
-      mapToMemRef(gpuModule, targetAttr);
-      populateConvertToSPIRVPatterns(typeConverter, scfToSPIRVContext,
-                                     patterns);
-      if (failed(applyFullConversion(gpuModule, *target, std::move(patterns))))
-        return signalPassFailure();
-    }
-  }
-};
-
-} // namespace
diff --git a/mlir/lib/Conversion/FuncToLLVM/FuncToLLVM.cpp b/mlir/lib/Conversion/FuncToLLVM/FuncToLLVM.cpp
index 790e18d2fcceb..55f0a9ac3bbb2 100644
--- a/mlir/lib/Conversion/FuncToLLVM/FuncToLLVM.cpp
+++ b/mlir/lib/Conversion/FuncToLLVM/FuncToLLVM.cpp
@@ -660,7 +660,7 @@ struct UnrealizedConversionCastOpLowering
 // `ReturnOp` interacts with the function signature and must have as many
 // operands as the function has return values.  Because in LLVM IR, functions
 // can only return 0 or 1 value, we pack multiple values into a structure type.
-// Emit `UndefOp` followed by `InsertValueOp`s to create such structure if
+// Emit `PoisonOp` followed by `InsertValueOp`s to create such structure if
 // necessary before returning it
 struct ReturnOpLowering : public ConvertOpToLLVMPattern<func::ReturnOp> {
   using ConvertOpToLLVMPattern<func::ReturnOp>::ConvertOpToLLVMPattern;
@@ -714,7 +714,7 @@ struct ReturnOpLowering : public ConvertOpToLLVMPattern<func::ReturnOp> {
       return rewriter.notifyMatchFailure(op, "could not convert result types");
     }
 
-    Value packed = rewriter.create<LLVM::UndefOp>(loc, packedType);
+    Value packed = rewriter.create<LLVM::PoisonOp>(loc, packedType);
     for (auto [idx, operand] : llvm::enumerate(updatedOperands)) {
       packed = rewriter.create<LLVM::InsertValueOp>(loc, packed, operand, idx);
     }
diff --git a/mlir/lib/Conversion/GPUCommon/GPUOpsLowering.cpp b/mlir/lib/Conversion/GPUCommon/GPUOpsLowering.cpp
index 544fc57949e24..cfa434699cdef 100644
--- a/mlir/lib/Conversion/GPUCommon/GPUOpsLowering.cpp
+++ b/mlir/lib/Conversion/GPUCommon/GPUOpsLowering.cpp
@@ -603,7 +603,7 @@ LogicalResult impl::scalarizeVectorOp(Operation *op, ValueRange operands,
     return rewriter.notifyMatchFailure(op, "expected vector result");
 
   Location loc = op->getLoc();
-  Value result = rewriter.create<LLVM::UndefOp>(loc, vectorType);
+  Value result = rewriter.create<LLVM::PoisonOp>(loc, vectorType);
   Type indexType = converter.convertType(rewriter.getIndexType());
   StringAttr name = op->getName().getIdentifier();
   Type elementType = vectorType.getElementType();
@@ -771,7 +771,7 @@ LogicalResult GPUReturnOpLowering::matchAndRewrite(
     return rewriter.notifyMatchFailure(op, "could not convert result types");
   }
 
-  Value packed = rewriter.create<LLVM::UndefOp>(loc, packedType);
+  Value packed = rewriter.create<LLVM::PoisonOp>(loc, packedType);
   for (auto [idx, operand] : llvm::enumerate(updatedOperands)) {
     packed = rewriter.create<LLVM::InsertValueOp>(loc, packed, operand, idx);
   }
diff --git a/mlir/lib/Conversion/GPUToNVVM/WmmaOpsToNvvm.cpp b/mlir/lib/Conversion/GPUToNVVM/WmmaOpsToNvvm.cpp
index 2b040fddac748..4b50b9187b25b 100644
--- a/mlir/lib/Conversion/GPUToNVVM/WmmaOpsToNvvm.cpp
+++ b/mlir/lib/Conversion/GPUToNVVM/WmmaOpsToNvvm.cpp
@@ -279,7 +279,7 @@ struct WmmaConstantOpToNVVMLowering
         cast<gpu::MMAMatrixType>(subgroupMmaConstantOp.getType()));
     // If the element type is a vector create a vector from the operand.
     if (auto vecType = dyn_cast<VectorType>(type.getBody()[0])) {
-      Value vecCst = rewriter.create<LLVM::UndefOp>(loc, vecType);
+      Value vecCst = rewriter.create<LLVM::PoisonOp>(loc, vecType);
       for (int64_t vecEl = 0; vecEl < vecType.getNumElements(); vecEl++) {
         Value idx = rewriter.create<LLVM::ConstantOp>(
             loc, rewriter.getI32Type(), vecEl);
@@ -288,7 +288,7 @@ struct WmmaConstantOpToNVVMLowering
       }
       cst = vecCst;
     }
-    Value matrixStruct = rewriter.create<LLVM::UndefOp>(loc, type);
+    Value matrixStruct = rewriter.create<LLVM::PoisonOp>(loc, type);
     for (size_t i : llvm::seq(size_t(0), type.getBody().size())) {
       matrixStruct =
           rewriter.create<LLVM::InsertValueOp>(loc, matrixStruct, cst, i);
@@ -355,7 +355,7 @@ struct WmmaElementwiseOpToNVVMLowering
     size_t numOperands = adaptor.getOperands().size();
     LLVM::LLVMStructType destType = convertMMAToLLVMType(
         cast<gpu::MMAMatrixType>(subgroupMmaElementwiseOp.getType()));
-    Value matrixStruct = rewriter.create<LLVM::UndefOp>(loc, destType);
+    Value matrixStruct = rewriter.create<LLVM::PoisonOp>(loc, destType);
     for (size_t i = 0, e = destType.getBody().size(); i < e; ++i) {
       SmallVector<Value> extractedOperands;
       for (size_t opIdx = 0; opIdx < numOperands; opIdx++) {
diff --git a/mlir/lib/Conversion/LLVMCommon/MemRefBuilder.cpp b/mlir/lib/Conversion/LLVMCommon/MemRefBuilder.cpp
index 63f99eb744a83..86d6643820376 100644
--- a/mlir/lib/Conversion/LLVMCommon/MemRefBuilder.cpp
+++ b/mlir/lib/Conversion/LLVMCommon/MemRefBuilder.cpp
@@ -29,10 +29,10 @@ MemRefDescriptor::MemRefDescriptor(Value descriptor)
 }
 
 /// Builds IR creating an `undef` value of the descriptor type.
-MemRefDescriptor MemRefDescriptor::undef(OpBuilder &builder, Location loc,
-                                         Type descriptorType) {
+MemRefDescriptor MemRefDescriptor::poison(OpBuilder &builder, Location loc,
+                                          Type descriptorType) {
 
-  Value descriptor = builder.create<LLVM::UndefOp>(loc, descriptorType);
+  Value descriptor = builder.create<LLVM::PoisonOp>(loc, descriptorType);
   return MemRefDescriptor(descriptor);
 }
 
@@ -60,7 +60,7 @@ MemRefDescriptor MemRefDescriptor::fromStaticShape(
   auto convertedType = typeConverter.convertType(type);
   assert(convertedType && "unexpected failure in memref type conversion");
 
-  auto descr = MemRefDescriptor::undef(builder, loc, convertedType);
+  auto descr = MemRefDescriptor::poison(builder, loc, convertedType);
   descr.setAllocatedPtr(builder, loc, memory);
   descr.setAlignedPtr(builder, loc, alignedMemory);
   descr.setConstantOffset(builder, loc, offset);
@@ -218,13 +218,13 @@ Value MemRefDescriptor::bufferPtr(OpBuilder &builder, Location loc,
 /// - aligned pointer;
 /// - offset;
 /// - <rank> sizes;
-/// - <rank> shapes;
+/// - <rank> strides;
 /// where <rank> is the MemRef rank as provided in `type`.
 Value MemRefDescriptor::pack(OpBuilder &builder, Location loc,
                              const LLVMTypeConverter &converter,
                              MemRefType type, ValueRange values) {
   Type llvmType = converter.convertType(type);
-  auto d = MemRefDescriptor::undef(builder, loc, llvmType);
+  auto d = MemRefDescriptor::poison(builder, loc, llvmType);
 
   d.setAllocatedPtr(builder, loc, values[kAllocatedPtrPosInMemRefDescriptor]);
   d.setAlignedPtr(builder, loc, values[kAlignedPtrPosInMemRefDescriptor]);
@@ -260,7 +260,7 @@ void MemRefDescriptor::unpack(OpBuilder &builder, Location loc, Value packed,
 /// Returns the number of non-aggregate values that would be produced by
 /// `unpack`.
 unsigned MemRefDescriptor::getNumUnpackedValues(MemRefType type) {
-  // Two pointers, offset, <rank> sizes, <rank> shapes.
+  // Two pointers, offset, <rank> sizes, <rank> strides.
   return 3 + 2 * type.getRank();
 }
 
@@ -300,10 +300,10 @@ UnrankedMemRefDescriptor::UnrankedMemRefDescriptor(Value descriptor)
     : StructBuilder(descriptor) {}
 
 /// Builds IR creating an `undef` value of the descriptor type.
-UnrankedMemRefDescriptor UnrankedMemRefDescriptor::undef(OpBuilder &builder,
-                                                         Location loc,
-                                                         Type descriptorType) {
-  Value descriptor = builder.create<LLVM::UndefOp>(loc, descriptorType);
+UnrankedMemRefDescriptor UnrankedMemRefDescriptor::poison(OpBuilder &builder,
+                                                          Location loc,
+                                                          Type descriptorType) {
+  Value descriptor = builder.create<LLVM::PoisonOp>(loc, descriptorType);
   return UnrankedMemRefDescriptor(descriptor);
 }
 Value UnrankedMemRefDescriptor::rank(OpBuilder &builder, Location loc) const {
@@ -331,7 +331,7 @@ Value UnrankedMemRefDescriptor::pack(OpBuilder &builder, Location loc,
                                      UnrankedMemRefType type,
                                      ValueRange values) {
   Type llvmType = converter.convertType(type);
-  auto d = UnrankedMemRefDescriptor::undef(builder, loc, llvmType);
+  auto d = UnrankedMemRefDescriptor::poison(builder, loc, llvmType);
 
   d.setRank(builder, loc, values[kRankInUnrankedMemRefDescriptor]);
   d.setMemRefDescPtr(builder, loc, values[kPtrInUnrankedMemRefDescriptor]);
diff --git a/mlir/lib/Conversion/LLVMCommon/Pattern.cpp b/mlir/lib/Conversion/LLVMCommon/Pattern.cpp
index a47a2872ceb07..71b68619cc793 100644
--- a/mlir/lib/Conversion/LLVMCommon/Pattern.cpp
+++ b/mlir/lib/Conversion/LLVMCommon/Pattern.cpp
@@ -218,7 +218,7 @@ MemRefDescriptor ConvertToLLVMPattern::createMemRefDescriptor(
     ArrayRef<Value> sizes, ArrayRef<Value> strides,
     ConversionPatternRewriter &rewriter) const {
   auto structType = typeConverter->convertType(memRefType);
-  auto memRefDescriptor = MemRefDescriptor::undef(rewriter, loc, structType);
+  auto memRefDescriptor = MemRefDescriptor::poison(rewriter, loc, structType);
 
   // Field 1: Allocated pointer, used for malloc/free.
   memRefDescriptor.setAllocatedPtr(rewriter, loc, allocatedPtr);
@@ -276,11 +276,17 @@ LogicalResult ConvertToLLVMPattern::copyUnrankedDescriptors(
 
   // Find the malloc and free, or declare them if necessary.
   auto module = builder.getInsertionPoint()->getParentOfType<ModuleOp>();
-  LLVM::LLVMFuncOp freeFunc, mallocFunc;
-  if (toDynamic)
+  FailureOr<LLVM::LLVMFuncOp> freeFunc, mallocFunc;
+  if (toDynamic) {
     mallocFunc = LLVM::lookupOrCreateMallocFn(module, indexType);
-  if (!toDynamic)
+    if (failed(mallocFunc))
+      return failure();
+  }
+  if (!toDynamic) {
     freeFunc = LLVM::lookupOrCreateFreeFn(module);
+    if (failed(freeFunc))
+      return failure();
+  }
 
   unsigned unrankedMemrefPos = 0;
   for (unsigned i = 0, e = operands.size(); i < e; ++i) {
@@ -293,7 +299,8 @@ LogicalResult ConvertToLLVMPattern::copyUnrankedDescriptors(
     // Allocate memory, copy, and free the source if necessary.
     Value memory =
         toDynamic
-            ? builder.create<LLVM::CallOp>(loc, mallocFunc, allocationSize)
+            ? builder
+                  .create<LLVM::CallOp>(loc, mallocFunc.value(), allocationSize)
                   .getResult()
             : builder.create<LLVM::AllocaOp>(loc, getVoidPtrType(),
                                              IntegerType::get(getContext(), 8),
@@ -302,7 +309,7 @@ LogicalResult ConvertToLLVMPattern::copyUnrankedDescriptors(
     Value source = desc.memRefDescPtr(builder, loc);
     builder.create<LLVM::MemcpyOp>(loc, memory, source, allocationSize, false);
     if (!toDynamic)
-      builder.create<LLVM::CallOp>(loc, freeFunc, source);
+      builder.create<LLVM::CallOp>(loc, freeFunc.value(), source);
 
     // Create a new descriptor. The same descriptor can be returned multiple
     // times, attempting to modify its pointer can lead to memory leaks
@@ -312,7 +319,7 @@ LogicalResult ConvertToLLVMPattern::copyUnrankedDescriptors(
     if (!descriptorType)
       return failure();
     auto updatedDesc =
-        UnrankedMemRefDescriptor::undef(builder, loc, descriptorType);
+        UnrankedMemRefDescriptor::poison(builder, loc, descriptorType);
     Value rank = desc.rank(builder, loc);
     updatedDesc.setRank(builder, loc, rank);
     updatedDesc.setMemRefDescPtr(builder, loc, memory);
diff --git a/mlir/lib/Conversion/LLVMCommon/PrintCallHelper.cpp b/mlir/lib/Conversion/LLVMCommon/PrintCallHelper.cpp
index bd7b401efec17..337c01f01a7cc 100644
--- a/mlir/lib/Conversion/LLVMCommon/PrintCallHelper.cpp
+++ b/mlir/lib/Conversion/LLVMCommon/PrintCallHelper.cpp
@@ -27,7 +27,7 @@ static std::string ensureSymbolNameIsUnique(ModuleOp moduleOp,
   return uniqueName;
 }
 
-void mlir::LLVM::createPrintStrCall(
+LogicalResult mlir::LLVM::createPrintStrCall(
     OpBuilder &builder, Location loc, ModuleOp moduleOp, StringRef symbolName,
     StringRef string, const LLVMTypeConverter &typeConverter, bool addNewline,
     std::optional<StringRef> runtimeFunctionName) {
@@ -59,8 +59,11 @@ void mlir::LLVM::createPrintStrCall(
   SmallVector<LLVM::GEPArg> indices(1, 0);
   Value gep =
       builder.create<LLVM::GEPOp>(loc, ptrTy, arrayTy, msgAddr, indices);
-  Operation *printer =
+  FailureOr<LLVM::LLVMFuncOp> printer =
       LLVM::lookupOrCreatePrintStringFn(moduleOp, runtimeFunctionName);
-  builder.create<LLVM::CallOp>(loc, TypeRange(), SymbolRefAttr::get(printer),
-                               gep);
+  if (failed(printer))
+    return failure();
+  builder.create<LLVM::CallOp>(loc, TypeRange(),
+                               SymbolRefAttr::get(printer.value()), gep);
+  return success();
 }
diff --git a/mlir/lib/Conversion/LLVMCommon/VectorPattern.cpp b/mlir/lib/Conversion/LLVMCommon/VectorPattern.cpp
index 626135c10a3e9..bf3f31729c3da 100644
--- a/mlir/lib/Conversion/LLVMCommon/VectorPattern.cpp
+++ b/mlir/lib/Conversion/LLVMCommon/VectorPattern.cpp
@@ -87,7 +87,7 @@ LogicalResult LLVM::detail::handleMultidimensionalVectors(
   auto result1DVectorTy = resultTypeInfo.llvm1DVectorTy;
   auto resultNDVectoryTy = resultTypeInfo.llvmNDVectorTy;
   auto loc = op->getLoc();
-  Value desc = rewriter.create<LLVM::UndefOp>(loc, resultNDVectoryTy);
+  Value desc = rewriter.create<LLVM::PoisonOp>(loc, resultNDVectoryTy);
   nDVectorIterate(resultTypeInfo, rewriter, [&](ArrayRef<int64_t> position) {
     // For this unrolled `position` corresponding to the `linearIndex`^th
     // element, extract operand vectors
diff --git a/mlir/lib/Conversion/MathToLLVM/MathToLLVM.cpp b/mlir/lib/Conversion/MathToLLVM/MathToLLVM.cpp
index 668f8385ac2dc..98680773e00d2 100644
--- a/mlir/lib/Conversion/MathToLLVM/MathToLLVM.cpp
+++ b/mlir/lib/Conversion/MathToLLVM/MathToLLVM.cpp
@@ -39,6 +39,7 @@ using CeilOpLowering = ConvertFMFMathToLLVMPattern<math::CeilOp, LLVM::FCeilOp>;
 using CopySignOpLowering =
     ConvertFMFMathToLLVMPattern<math::CopySignOp, LLVM::CopySignOp>;
 using CosOpLowering = ConvertFMFMathToLLVMPattern<math::CosOp, LLVM::CosOp>;
+using CoshOpLowering = ConvertFMFMathToLLVMPattern<math::CoshOp, LLVM::CoshOp>;
 using CtPopFOpLowering =
     VectorConvertToLLVMPattern<math::CtPopOp, LLVM::CtPopOp>;
 using Exp2OpLowering = ConvertFMFMathToLLVMPattern<math::Exp2Op, LLVM::Exp2Op>;
@@ -58,9 +59,12 @@ using RoundEvenOpLowering =
 using RoundOpLowering =
     ConvertFMFMathToLLVMPattern<math::RoundOp, LLVM::RoundOp>;
 using SinOpLowering = ConvertFMFMathToLLVMPattern<math::SinOp, LLVM::SinOp>;
+using SinhOpLowering = ConvertFMFMathToLLVMPattern<math::SinhOp, LLVM::SinhOp>;
 using SqrtOpLowering = ConvertFMFMathToLLVMPattern<math::SqrtOp, LLVM::SqrtOp>;
 using FTruncOpLowering =
     ConvertFMFMathToLLVMPattern<math::TruncOp, LLVM::FTruncOp>;
+using TanOpLowering = ConvertFMFMathToLLVMPattern<math::TanOp, LLVM::TanOp>;
+using TanhOpLowering = ConvertFMFMathToLLVMPattern<math::TanhOp, LLVM::TanhOp>;
 
 // A `CtLz/CtTz/absi(a)` is converted into `CtLz/CtTz/absi(a, false)`.
 template <typename MathOp, typename LLVMOp>
@@ -310,6 +314,7 @@ void mlir::populateMathToLLVMConversionPatterns(
     CeilOpLowering,
     CopySignOpLowering,
     CosOpLowering,
+    CoshOpLowering,
     CountLeadingZerosOpLowering,
     CountTrailingZerosOpLowering,
     CtPopFOpLowering,
@@ -327,8 +332,11 @@ void mlir::populateMathToLLVMConversionPatterns(
     RoundOpLowering,
     RsqrtOpLowering,
     SinOpLowering,
+    SinhOpLowering,
     SqrtOpLowering,
-    FTruncOpLowering
+    FTruncOpLowering,
+    TanOpLowering,
+    TanhOpLowering
   >(converter);
   // clang-format on
 }
diff --git a/mlir/lib/Conversion/MemRefToLLVM/AllocLikeConversion.cpp b/mlir/lib/Conversion/MemRefToLLVM/AllocLikeConversion.cpp
index a6408391b1330..c5b2e83df93dc 100644
--- a/mlir/lib/Conversion/MemRefToLLVM/AllocLikeConversion.cpp
+++ b/mlir/lib/Conversion/MemRefToLLVM/AllocLikeConversion.cpp
@@ -14,9 +14,9 @@
 
 using namespace mlir;
 
-namespace {
-LLVM::LLVMFuncOp getNotalignedAllocFn(const LLVMTypeConverter *typeConverter,
-                                      Operation *module, Type indexType) {
+static FailureOr<LLVM::LLVMFuncOp>
+getNotalignedAllocFn(const LLVMTypeConverter *typeConverter, Operation *module,
+                     Type indexType) {
   bool useGenericFn = typeConverter->getOptions().useGenericFunctions;
   if (useGenericFn)
     return LLVM::lookupOrCreateGenericAllocFn(module, indexType);
@@ -24,8 +24,9 @@ LLVM::LLVMFuncOp getNotalignedAllocFn(const LLVMTypeConverter *typeConverter,
   return LLVM::lookupOrCreateMallocFn(module, indexType);
 }
 
-LLVM::LLVMFuncOp getAlignedAllocFn(const LLVMTypeConverter *typeConverter,
-                                   Operation *module, Type indexType) {
+static FailureOr<LLVM::LLVMFuncOp>
+getAlignedAllocFn(const LLVMTypeConverter *typeConverter, Operation *module,
+                  Type indexType) {
   bool useGenericFn = typeConverter->getOptions().useGenericFunctions;
 
   if (useGenericFn)
@@ -34,8 +35,6 @@ LLVM::LLVMFuncOp getAlignedAllocFn(const LLVMTypeConverter *typeConverter,
   return LLVM::lookupOrCreateAlignedAllocFn(module, indexType);
 }
 
-} // end namespace
-
 Value AllocationOpLLVMLowering::createAligned(
     ConversionPatternRewriter &rewriter, Location loc, Value input,
     Value alignment) {
@@ -80,10 +79,13 @@ std::tuple<Value, Value> AllocationOpLLVMLowering::allocateBufferManuallyAlign(
         << " to integer address space "
            "failed. Consider adding memory space conversions.";
   }
-  LLVM::LLVMFuncOp allocFuncOp = getNotalignedAllocFn(
+  FailureOr<LLVM::LLVMFuncOp> allocFuncOp = getNotalignedAllocFn(
       getTypeConverter(), op->getParentWithTrait<OpTrait::SymbolTable>(),
       getIndexType());
-  auto results = rewriter.create<LLVM::CallOp>(loc, allocFuncOp, sizeBytes);
+  if (failed(allocFuncOp))
+    return std::make_tuple(Value(), Value());
+  auto results =
+      rewriter.create<LLVM::CallOp>(loc, allocFuncOp.value(), sizeBytes);
 
   Value allocatedPtr =
       castAllocFuncResult(rewriter, loc, results.getResult(), memRefType,
@@ -146,11 +148,13 @@ Value AllocationOpLLVMLowering::allocateBufferAutoAlign(
     sizeBytes = createAligned(rewriter, loc, sizeBytes, allocAlignment);
 
   Type elementPtrType = this->getElementPtrType(memRefType);
-  LLVM::LLVMFuncOp allocFuncOp = getAlignedAllocFn(
+  FailureOr<LLVM::LLVMFuncOp> allocFuncOp = getAlignedAllocFn(
       getTypeConverter(), op->getParentWithTrait<OpTrait::SymbolTable>(),
       getIndexType());
+  if (failed(allocFuncOp))
+    return Value();
   auto results = rewriter.create<LLVM::CallOp>(
-      loc, allocFuncOp, ValueRange({allocAlignment, sizeBytes}));
+      loc, allocFuncOp.value(), ValueRange({allocAlignment, sizeBytes}));
 
   return castAllocFuncResult(rewriter, loc, results.getResult(), memRefType,
                              elementPtrType, *getTypeConverter());
diff --git a/mlir/lib/Conversion/MemRefToLLVM/MemRefToLLVM.cpp b/mlir/lib/Conversion/MemRefToLLVM/MemRefToLLVM.cpp
index f7542b8b3bc5c..3646416def810 100644
--- a/mlir/lib/Conversion/MemRefToLLVM/MemRefToLLVM.cpp
+++ b/mlir/lib/Conversion/MemRefToLLVM/MemRefToLLVM.cpp
@@ -38,12 +38,12 @@ using namespace mlir;
 
 namespace {
 
-bool isStaticStrideOrOffset(int64_t strideOrOffset) {
+static bool isStaticStrideOrOffset(int64_t strideOrOffset) {
   return !ShapedType::isDynamic(strideOrOffset);
 }
 
-LLVM::LLVMFuncOp getFreeFn(const LLVMTypeConverter *typeConverter,
-                           ModuleOp module) {
+static FailureOr<LLVM::LLVMFuncOp>
+getFreeFn(const LLVMTypeConverter *typeConverter, ModuleOp module) {
   bool useGenericFn = typeConverter->getOptions().useGenericFunctions;
 
   if (useGenericFn)
@@ -220,8 +220,10 @@ struct DeallocOpLowering : public ConvertOpToLLVMPattern<memref::DeallocOp> {
   matchAndRewrite(memref::DeallocOp op, OpAdaptor adaptor,
                   ConversionPatternRewriter &rewriter) const override {
     // Insert the `free` declaration if it is not already present.
-    LLVM::LLVMFuncOp freeFunc =
+    FailureOr<LLVM::LLVMFuncOp> freeFunc =
         getFreeFn(getTypeConverter(), op->getParentOfType<ModuleOp>());
+    if (failed(freeFunc))
+      return failure();
     Value allocatedPtr;
     if (auto unrankedTy =
             llvm::dyn_cast<UnrankedMemRefType>(op.getMemref().getType())) {
@@ -236,7 +238,8 @@ struct DeallocOpLowering : public ConvertOpToLLVMPattern<memref::DeallocOp> {
       allocatedPtr = MemRefDescriptor(adaptor.getMemref())
                          .allocatedPtr(rewriter, op.getLoc());
     }
-    rewriter.replaceOpWithNewOp<LLVM::CallOp>(op, freeFunc, allocatedPtr);
+    rewriter.replaceOpWithNewOp<LLVM::CallOp>(op, freeFunc.value(),
+                                              allocatedPtr);
     return success();
   }
 };
@@ -711,10 +714,10 @@ struct MemRefCastOpLowering : public ConvertOpToLLVMPattern<memref::CastOp> {
       // rank = ConstantOp srcRank
       auto rankVal = rewriter.create<LLVM::ConstantOp>(
           loc, getIndexType(), rewriter.getIndexAttr(rank));
-      // undef = UndefOp
+      // poison = PoisonOp
       UnrankedMemRefDescriptor memRefDesc =
-          UnrankedMemRefDescriptor::undef(rewriter, loc, targetStructType);
-      // d1 = InsertValueOp undef, rank, 0
+          UnrankedMemRefDescriptor::poison(rewriter, loc, targetStructType);
+      // d1 = InsertValueOp poison, rank, 0
       memRefDesc.setRank(rewriter, loc, rankVal);
       // d2 = InsertValueOp d1, ptr, 1
       memRefDesc.setMemRefDescPtr(rewriter, loc, ptr);
@@ -838,7 +841,9 @@ struct MemRefCopyOpLowering : public ConvertOpToLLVMPattern<memref::CopyOp> {
     auto elemSize = getSizeInBytes(loc, srcType.getElementType(), rewriter);
     auto copyFn = LLVM::lookupOrCreateMemRefCopyFn(
         op->getParentOfType<ModuleOp>(), getIndexType(), sourcePtr.getType());
-    rewriter.create<LLVM::CallOp>(loc, copyFn,
+    if (failed(copyFn))
+      return failure();
+    rewriter.create<LLVM::CallOp>(loc, copyFn.value(),
                                   ValueRange{elemSize, sourcePtr, targetPtr});
 
     // Restore stack used for descriptors
@@ -923,7 +928,7 @@ struct MemorySpaceCastOpLowering
       Value sourceUnderlyingDesc = sourceDesc.memRefDescPtr(rewriter, loc);
 
       // Create and allocate storage for new memref descriptor.
-      auto result = UnrankedMemRefDescriptor::undef(
+      auto result = UnrankedMemRefDescriptor::poison(
           rewriter, loc, typeConverter->convertType(resultTypeU));
       result.setRank(rewriter, loc, rank);
       SmallVector<Value, 1> sizes;
@@ -1053,7 +1058,7 @@ struct MemRefReinterpretCastOpLowering
 
     // Create descriptor.
     Location loc = castOp.getLoc();
-    auto desc = MemRefDescriptor::undef(rewriter, loc, llvmTargetDescriptorTy);
+    auto desc = MemRefDescriptor::poison(rewriter, loc, llvmTargetDescriptorTy);
 
     // Set allocated and aligned pointers.
     Value allocatedPtr, alignedPtr;
@@ -1123,7 +1128,7 @@ struct MemRefReshapeOpLowering
       // Create descriptor.
       Location loc = reshapeOp.getLoc();
       auto desc =
-          MemRefDescriptor::undef(rewriter, loc, llvmTargetDescriptorTy);
+          MemRefDescriptor::poison(rewriter, loc, llvmTargetDescriptorTy);
 
       // Set allocated and aligned pointers.
       Value allocatedPtr, alignedPtr;
@@ -1205,7 +1210,7 @@ struct MemRefReshapeOpLowering
 
     // Create the unranked memref descriptor that holds the ranked one. The
     // inner descriptor is allocated on stack.
-    auto targetDesc = UnrankedMemRefDescriptor::undef(
+    auto targetDesc = UnrankedMemRefDescriptor::poison(
         rewriter, loc, typeConverter->convertType(targetType));
     targetDesc.setRank(rewriter, loc, resultRank);
     SmallVector<Value, 4> sizes;
@@ -1361,7 +1366,7 @@ class TransposeOpLowering : public ConvertOpToLLVMPattern<memref::TransposeOp> {
     if (transposeOp.getPermutation().isIdentity())
       return rewriter.replaceOp(transposeOp, {viewMemRef}), success();
 
-    auto targetMemRef = MemRefDescriptor::undef(
+    auto targetMemRef = MemRefDescriptor::poison(
         rewriter, loc,
         typeConverter->convertType(transposeOp.getIn().getType()));
 
@@ -1464,7 +1469,7 @@ struct ViewOpLowering : public ConvertOpToLLVMPattern<memref::ViewOp> {
 
     // Create the descriptor.
     MemRefDescriptor sourceMemRef(adaptor.getSource());
-    auto targetMemRef = MemRefDescriptor::undef(rewriter, loc, targetDescTy);
+    auto targetMemRef = MemRefDescriptor::poison(rewriter, loc, targetDescTy);
 
     // Field 1: Copy the allocated pointer, used for malloc/free.
     Value allocatedPtr = sourceMemRef.allocatedPtr(rewriter, loc);
diff --git a/mlir/lib/Conversion/NVGPUToNVVM/NVGPUToNVVM.cpp b/mlir/lib/Conversion/NVGPUToNVVM/NVGPUToNVVM.cpp
index 7e97fb84434f8..f53de416f2abd 100644
--- a/mlir/lib/Conversion/NVGPUToNVVM/NVGPUToNVVM.cpp
+++ b/mlir/lib/Conversion/NVGPUToNVVM/NVGPUToNVVM.cpp
@@ -143,7 +143,7 @@ static Value convertIntrinsicResult(Location loc, Type intrinsicResultType,
 
       for (unsigned i = 0, e = structType.getBody().size() / 2; i < e; i++) {
         Value vec =
-            rewriter.create<LLVM::UndefOp>(loc, arrayType.getElementType());
+            rewriter.create<LLVM::PoisonOp>(loc, arrayType.getElementType());
         Value x1 =
             rewriter.create<LLVM::ExtractValueOp>(loc, intrinsicResult, i * 2);
         Value x2 = rewriter.create<LLVM::ExtractValueOp>(loc, intrinsicResult,
@@ -157,7 +157,7 @@ static Value convertIntrinsicResult(Location loc, Type intrinsicResultType,
     }
 
     // Create the final vectorized result.
-    Value result = rewriter.create<LLVM::UndefOp>(loc, arrayType);
+    Value result = rewriter.create<LLVM::PoisonOp>(loc, arrayType);
     for (const auto &el : llvm::enumerate(elements)) {
       result = rewriter.create<LLVM::InsertValueOp>(loc, result, el.value(),
                                                     el.index());
@@ -296,7 +296,7 @@ struct MmaLdMatrixOpToNVVM : public ConvertOpToLLVMPattern<nvgpu::LdMatrixOp> {
     // actual vector type (still of width 32b) and repack them into a result
     // struct.
     Type finalResultType = typeConverter->convertType(vectorResultType);
-    Value result = b.create<LLVM::UndefOp>(finalResultType);
+    Value result = b.create<LLVM::PoisonOp>(finalResultType);
     for (int64_t i = 0, e = vectorResultType.getDimSize(0); i < e; i++) {
       Value i32Register =
           num32BitRegs > 1 ? b.create<LLVM::ExtractValueOp>(ldMatrixResult, i)
@@ -1421,7 +1421,7 @@ struct NVGPUWarpgroupMmaOpLowering
     /// Generates multiple wgmma instructions to complete the given GEMM shape
     Value generateWgmmaGroup() {
       Value wgmmaResult =
-          b.create<LLVM::UndefOp>(adaptor.getMatrixC().getType());
+          b.create<LLVM::PoisonOp>(adaptor.getMatrixC().getType());
 
       // Perform GEMM
       SmallVector<Value> wgmmaResults;
@@ -1631,7 +1631,7 @@ struct NVGPUWarpgroupMmaInitAccumulatorOpLowering
                         .getBody()
                         .front();
     Value zero = b.create<LLVM::ConstantOp>(elemType, b.getZeroAttr(elemType));
-    Value packStruct = b.create<LLVM::UndefOp>(packStructType);
+    Value packStruct = b.create<LLVM::PoisonOp>(packStructType);
     SmallVector<Value> innerStructs;
     // Unpack the structs and set all values to zero
     for (auto [idx, s] : llvm::enumerate(packStructType.getBody())) {
@@ -1676,7 +1676,7 @@ struct NVGPURcpOpLowering : public ConvertOpToLLVMPattern<nvgpu::RcpOp> {
     VectorType inTy = op.getIn().getType();
     // apply rcp.approx.ftz.f on each element in vector.
     auto convert1DVec = [&](Type llvm1DVectorTy, Value inVec) {
-      Value ret1DVec = b.create<LLVM::UndefOp>(llvm1DVectorTy);
+      Value ret1DVec = b.create<LLVM::PoisonOp>(llvm1DVectorTy);
       int numElems = llvm::cast<VectorType>(llvm1DVectorTy).getNumElements();
       for (int i = 0; i < numElems; i++) {
         Value idx = b.create<LLVM::ConstantOp>(i64Ty, b.getI64IntegerAttr(i));
diff --git a/mlir/lib/Conversion/OpenMPToLLVM/OpenMPToLLVM.cpp b/mlir/lib/Conversion/OpenMPToLLVM/OpenMPToLLVM.cpp
index 5d0003911bca8..12e3c07669839 100644
--- a/mlir/lib/Conversion/OpenMPToLLVM/OpenMPToLLVM.cpp
+++ b/mlir/lib/Conversion/OpenMPToLLVM/OpenMPToLLVM.cpp
@@ -226,7 +226,7 @@ void mlir::configureOpenMPToLLVMConversionLegality(
   target.addDynamicallyLegalOp<
       omp::AtomicReadOp, omp::AtomicWriteOp, omp::CancellationPointOp,
       omp::CancelOp, omp::CriticalDeclareOp, omp::FlushOp, omp::MapBoundsOp,
-      omp::MapInfoOp, omp::OrderedOp, omp::TargetEnterDataOp,
+      omp::MapInfoOp, omp::OrderedOp, omp::ScanOp, omp::TargetEnterDataOp,
       omp::TargetExitDataOp, omp::TargetUpdateOp, omp::ThreadprivateOp,
       omp::YieldOp>([&](Operation *op) {
     return typeConverter.isLegal(op->getOperandTypes()) &&
@@ -235,10 +235,10 @@ void mlir::configureOpenMPToLLVMConversionLegality(
   target.addDynamicallyLegalOp<
       omp::AtomicUpdateOp, omp::CriticalOp, omp::DeclareReductionOp,
       omp::DistributeOp, omp::LoopNestOp, omp::LoopOp, omp::MasterOp,
-      omp::OrderedRegionOp, omp::ParallelOp, omp::PrivateClauseOp,
-      omp::SectionOp, omp::SectionsOp, omp::SimdOp, omp::SingleOp,
-      omp::TargetDataOp, omp::TargetOp, omp::TaskgroupOp, omp::TaskloopOp,
-      omp::TaskOp, omp::TeamsOp, omp::WsloopOp>([&](Operation *op) {
+      omp::OrderedRegionOp, omp::ParallelOp, omp::SectionOp, omp::SectionsOp,
+      omp::SimdOp, omp::SingleOp, omp::TargetDataOp, omp::TargetOp,
+      omp::TaskgroupOp, omp::TaskloopOp, omp::TaskOp, omp::TeamsOp,
+      omp::WsloopOp>([&](Operation *op) {
     return std::all_of(op->getRegions().begin(), op->getRegions().end(),
                        [&](Region &region) {
                          return typeConverter.isLegal(&region);
@@ -246,6 +246,16 @@ void mlir::configureOpenMPToLLVMConversionLegality(
            typeConverter.isLegal(op->getOperandTypes()) &&
            typeConverter.isLegal(op->getResultTypes());
   });
+  target.addDynamicallyLegalOp<omp::PrivateClauseOp>(
+      [&](omp::PrivateClauseOp op) -> bool {
+        return std::all_of(op->getRegions().begin(), op->getRegions().end(),
+                           [&](Region &region) {
+                             return typeConverter.isLegal(&region);
+                           }) &&
+               typeConverter.isLegal(op->getOperandTypes()) &&
+               typeConverter.isLegal(op->getResultTypes()) &&
+               typeConverter.isLegal(op.getType());
+      });
 }
 
 void mlir::populateOpenMPToLLVMConversionPatterns(LLVMTypeConverter &converter,
@@ -264,6 +274,7 @@ void mlir::populateOpenMPToLLVMConversionPatterns(LLVMTypeConverter &converter,
       RegionLessOpConversion<omp::CancelOp>,
       RegionLessOpConversion<omp::CriticalDeclareOp>,
       RegionLessOpConversion<omp::OrderedOp>,
+      RegionLessOpConversion<omp::ScanOp>,
       RegionLessOpConversion<omp::TargetEnterDataOp>,
       RegionLessOpConversion<omp::TargetExitDataOp>,
       RegionLessOpConversion<omp::TargetUpdateOp>,
diff --git a/mlir/lib/Conversion/SPIRVToLLVM/SPIRVToLLVM.cpp b/mlir/lib/Conversion/SPIRVToLLVM/SPIRVToLLVM.cpp
index dbaac030aa0a2..6e0adfc1e0ff3 100644
--- a/mlir/lib/Conversion/SPIRVToLLVM/SPIRVToLLVM.cpp
+++ b/mlir/lib/Conversion/SPIRVToLLVM/SPIRVToLLVM.cpp
@@ -155,7 +155,7 @@ static Value broadcast(Location loc, Value toBroadcast, unsigned numElements,
   auto vectorType = VectorType::get(numElements, toBroadcast.getType());
   auto llvmVectorType = typeConverter.convertType(vectorType);
   auto llvmI32Type = typeConverter.convertType(rewriter.getIntegerType(32));
-  Value broadcasted = rewriter.create<LLVM::UndefOp>(loc, llvmVectorType);
+  Value broadcasted = rewriter.create<LLVM::PoisonOp>(loc, llvmVectorType);
   for (unsigned i = 0; i < numElements; ++i) {
     auto index = rewriter.create<LLVM::ConstantOp>(
         loc, llvmI32Type, rewriter.getI32IntegerAttr(i));
@@ -708,7 +708,7 @@ class ExecutionModePattern
 
     // Initialize the struct and set the execution mode value.
     rewriter.setInsertionPointToStart(block);
-    Value structValue = rewriter.create<LLVM::UndefOp>(loc, structType);
+    Value structValue = rewriter.create<LLVM::PoisonOp>(loc, structType);
     Value executionMode = rewriter.create<LLVM::ConstantOp>(
         loc, llvmI32Type,
         rewriter.getI32IntegerAttr(
@@ -1753,7 +1753,7 @@ class VectorShufflePattern
     auto componentsArray = components.getValue();
     auto *context = rewriter.getContext();
     auto llvmI32Type = IntegerType::get(context, 32);
-    Value targetOp = rewriter.create<LLVM::UndefOp>(loc, dstType);
+    Value targetOp = rewriter.create<LLVM::PoisonOp>(loc, dstType);
     for (unsigned i = 0; i < componentsArray.size(); i++) {
       if (!isa<IntegerAttr>(componentsArray[i]))
         return op.emitError("unable to support non-constant component");
diff --git a/mlir/lib/Conversion/TosaToLinalg/TosaToLinalg.cpp b/mlir/lib/Conversion/TosaToLinalg/TosaToLinalg.cpp
index f97e0ff1e30ea..d849c782bf08b 100644
--- a/mlir/lib/Conversion/TosaToLinalg/TosaToLinalg.cpp
+++ b/mlir/lib/Conversion/TosaToLinalg/TosaToLinalg.cpp
@@ -90,103 +90,123 @@ static Value createLinalgBodyCalculationForElementwiseOp(
   }
 
   // tosa::MulOp
-  if (isa<tosa::MulOp>(op) && isa<FloatType>(elementTy))
-    return rewriter.create<arith::MulFOp>(loc, resultTypes, args);
-
-  if (isa<tosa::MulOp>(op) && isa<IntegerType>(elementTy)) {
-    Value a = args[0];
-    Value b = args[1];
-    auto shift =
-        cast<IntegerAttr>(op->getAttr("shift")).getValue().getSExtValue();
-    if (shift > 0) {
-      auto shiftConst =
-          rewriter.create<arith::ConstantIntOp>(loc, shift, /*bitwidth=*/8);
-      if (!a.getType().isInteger(32))
-        a = rewriter.create<arith::ExtSIOp>(loc, rewriter.getI32Type(), a);
-
-      if (!b.getType().isInteger(32))
-        b = rewriter.create<arith::ExtSIOp>(loc, rewriter.getI32Type(), b);
-
-      auto result = rewriter.create<tosa::ApplyScaleOp>(
-          loc, rewriter.getI32Type(), a, b, shiftConst,
-          rewriter.getBoolAttr(false));
-
-      if (elementTy.isInteger(32))
-        return result;
-
-      return rewriter.create<arith::TruncIOp>(loc, elementTy, result);
+  if (isa<tosa::MulOp>(op)) {
+    auto shift_val = cast<tosa::MulOp>(op).getShift();
+    ElementsAttr shift_elem;
+    if (!shift_val.getImpl() ||
+        !matchPattern(shift_val, m_Constant(&shift_elem))) {
+      (void)rewriter.notifyMatchFailure(op, "shift value of mul not found");
     }
 
-    int aWidth = a.getType().getIntOrFloatBitWidth();
-    int bWidth = b.getType().getIntOrFloatBitWidth();
-    int cWidth = resultTypes[0].getIntOrFloatBitWidth();
+    int32_t shift = shift_elem.getValues<IntegerAttr>()[0].getInt();
 
-    if (aWidth < cWidth)
-      a = rewriter.create<arith::ExtSIOp>(loc, resultTypes[0], a);
-    if (bWidth < cWidth)
-      b = rewriter.create<arith::ExtSIOp>(loc, resultTypes[0], b);
+    if (isa<FloatType>(elementTy)) {
+      if (shift != 0) {
+        (void)rewriter.notifyMatchFailure(op,
+                                          "Cannot have shift value for float");
+        return nullptr;
+      }
+      return rewriter.create<arith::MulFOp>(loc, resultTypes, args[0], args[1]);
+    }
 
-    return rewriter.create<arith::MulIOp>(loc, resultTypes, a, b);
-  }
+    if (isa<IntegerType>(elementTy)) {
+      Value a = args[0];
+      Value b = args[1];
 
-  // tosa::NegateOp
-  if (isa<tosa::NegateOp>(op) && isa<FloatType>(elementTy))
-    return rewriter.create<arith::NegFOp>(loc, resultTypes, args);
+      if (shift > 0) {
+        auto shiftConst =
+            rewriter.create<arith::ConstantIntOp>(loc, shift, /*bitwidth=*/8);
+        if (!a.getType().isInteger(32))
+          a = rewriter.create<arith::ExtSIOp>(loc, rewriter.getI32Type(), a);
 
-  if (isa<tosa::NegateOp>(op) && isa<IntegerType>(elementTy)) {
-    int64_t inZp = 0, outZp = 0;
+        if (!b.getType().isInteger(32))
+          b = rewriter.create<arith::ExtSIOp>(loc, rewriter.getI32Type(), b);
 
-    if (cast<tosa::NegateOp>(op).getQuantizationInfo()) {
-      auto quantizationInfo = cast<tosa::NegateOp>(op).getQuantizationInfo();
-      inZp = quantizationInfo.value().getInputZp();
-      outZp = quantizationInfo.value().getOutputZp();
-    }
+        auto result = rewriter.create<tosa::ApplyScaleOp>(
+            loc, rewriter.getI32Type(), a, b, shiftConst,
+            rewriter.getBoolAttr(false));
 
-    int32_t inputBitWidth = elementTy.getIntOrFloatBitWidth();
-    if (!inZp && !outZp) {
-      auto constant = rewriter.create<arith::ConstantOp>(
-          loc, IntegerAttr::get(elementTy, 0));
-      return rewriter.create<arith::SubIOp>(loc, resultTypes, constant,
-                                            args[0]);
-    }
+        if (elementTy.isInteger(32))
+          return result;
+
+        return rewriter.create<arith::TruncIOp>(loc, elementTy, result);
+      }
+
+      int aWidth = a.getType().getIntOrFloatBitWidth();
+      int bWidth = b.getType().getIntOrFloatBitWidth();
+      int cWidth = resultTypes[0].getIntOrFloatBitWidth();
 
-    // Compute the maximum value that can occur in the intermediate buffer.
-    int64_t zpAdd = inZp + outZp;
-    int64_t maxValue = APInt::getSignedMaxValue(inputBitWidth).getSExtValue() +
-                       std::abs(zpAdd) + 1;
-
-    // Convert that maximum value into the maximum bitwidth needed to represent
-    // it. We assume 48-bit numbers may be supported further in the pipeline.
-    int intermediateBitWidth = 64;
-    if (maxValue <= APInt::getSignedMaxValue(16).getSExtValue()) {
-      intermediateBitWidth = 16;
-    } else if (maxValue <= APInt::getSignedMaxValue(32).getSExtValue()) {
-      intermediateBitWidth = 32;
-    } else if (maxValue <= APInt::getSignedMaxValue(48).getSExtValue()) {
-      intermediateBitWidth = 48;
+      if (aWidth < cWidth)
+        a = rewriter.create<arith::ExtSIOp>(loc, resultTypes[0], a);
+      if (bWidth < cWidth)
+        b = rewriter.create<arith::ExtSIOp>(loc, resultTypes[0], b);
+
+      return rewriter.create<arith::MulIOp>(loc, resultTypes, a, b);
     }
+  }
+
+  // tosa::NegateOp
+  if (isa<tosa::NegateOp>(op)) {
+    if (isa<FloatType>(elementTy))
+      return rewriter.create<arith::NegFOp>(loc, resultTypes, args);
+
+    if (isa<IntegerType>(elementTy)) {
+      auto inputZpAttr = cast<tosa::NegateOp>(op).getInput1ZpAttr();
+      auto outputZpAttr = cast<tosa::NegateOp>(op).getOutputZpAttr();
+
+      const int64_t inZp =
+          inputZpAttr ? inputZpAttr.getValue().getSExtValue() : 0;
+      const int64_t outZp =
+          outputZpAttr ? outputZpAttr.getValue().getSExtValue() : 0;
+
+      if (!inZp && !outZp) {
+        auto constant = rewriter.create<arith::ConstantOp>(
+            loc, IntegerAttr::get(elementTy, 0));
+        return rewriter.create<arith::SubIOp>(loc, resultTypes, constant,
+                                              args[0]);
+      }
 
-    Type intermediateType = rewriter.getIntegerType(intermediateBitWidth);
-    Value zpAddValue = rewriter.create<arith::ConstantOp>(
-        loc, rewriter.getIntegerAttr(intermediateType, zpAdd));
-
-    // The negation can be applied by doing:
-    //  outputValue = inZp + outZp - inputValue
-    auto ext = rewriter.create<arith::ExtSIOp>(loc, intermediateType, args[0]);
-    auto sub = rewriter.create<arith::SubIOp>(loc, zpAddValue, ext);
-
-    // Clamp to the negation range.
-    Value min = rewriter.create<arith::ConstantIntOp>(
-        loc, APInt::getSignedMinValue(inputBitWidth).getSExtValue(),
-        intermediateType);
-    Value max = rewriter.create<arith::ConstantIntOp>(
-        loc, APInt::getSignedMaxValue(inputBitWidth).getSExtValue(),
-        intermediateType);
-    auto clamp =
-        clampIntHelper(loc, sub, min, max, rewriter, /*isUnsigned=*/false);
-
-    // Truncate to the final value.
-    return rewriter.create<arith::TruncIOp>(loc, elementTy, clamp);
+      // Compute the maximum value that can occur in the intermediate buffer.
+      const int32_t inputBitWidth = elementTy.getIntOrFloatBitWidth();
+      const int64_t zpAdd = inZp + outZp;
+      const int64_t maxValue =
+          APInt::getSignedMaxValue(inputBitWidth).getSExtValue() +
+          std::abs(zpAdd) + 1;
+
+      // Convert that maximum value into the maximum bitwidth needed to
+      // represent it. We assume 48-bit numbers may be supported further in
+      // the pipeline.
+      int intermediateBitWidth = 64;
+      if (maxValue <= APInt::getSignedMaxValue(16).getSExtValue()) {
+        intermediateBitWidth = 16;
+      } else if (maxValue <= APInt::getSignedMaxValue(32).getSExtValue()) {
+        intermediateBitWidth = 32;
+      } else if (maxValue <= APInt::getSignedMaxValue(48).getSExtValue()) {
+        intermediateBitWidth = 48;
+      }
+
+      Type intermediateType = rewriter.getIntegerType(intermediateBitWidth);
+      Value zpAddValue = rewriter.create<arith::ConstantOp>(
+          loc, rewriter.getIntegerAttr(intermediateType, zpAdd));
+
+      // The negation can be applied by doing:
+      //  outputValue = inZp + outZp - inputValue
+      auto ext =
+          rewriter.create<arith::ExtSIOp>(loc, intermediateType, args[0]);
+      auto sub = rewriter.create<arith::SubIOp>(loc, zpAddValue, ext);
+
+      // Clamp to the negation range.
+      Value min = rewriter.create<arith::ConstantIntOp>(
+          loc, APInt::getSignedMinValue(inputBitWidth).getSExtValue(),
+          intermediateType);
+      Value max = rewriter.create<arith::ConstantIntOp>(
+          loc, APInt::getSignedMaxValue(inputBitWidth).getSExtValue(),
+          intermediateType);
+      auto clamp = clampIntHelper(loc, sub, min, max, rewriter, false);
+
+      // Truncate to the final value.
+      return rewriter.create<arith::TruncIOp>(loc, elementTy, clamp);
+    }
   }
 
   // tosa::BitwiseAndOp
@@ -374,8 +394,8 @@ static Value createLinalgBodyCalculationForElementwiseOp(
   // tosa::ClampOp
   if (isa<tosa::ClampOp>(op) && isa<FloatType>(elementTy)) {
     bool losesInfo = false;
-    APFloat minApf = cast<FloatAttr>(op->getAttr("min_fp")).getValue();
-    APFloat maxApf = cast<FloatAttr>(op->getAttr("max_fp")).getValue();
+    APFloat minApf = cast<FloatAttr>(op->getAttr("min_val")).getValue();
+    APFloat maxApf = cast<FloatAttr>(op->getAttr("max_val")).getValue();
     minApf.convert(cast<FloatType>(elementTy).getFloatSemantics(),
                    APFloat::rmNearestTiesToEven, &losesInfo);
     maxApf.convert(cast<FloatType>(elementTy).getFloatSemantics(),
@@ -390,9 +410,9 @@ static Value createLinalgBodyCalculationForElementwiseOp(
   if (isa<tosa::ClampOp>(op) && isa<IntegerType>(elementTy)) {
     auto intTy = cast<IntegerType>(elementTy);
     int64_t min =
-        cast<IntegerAttr>(op->getAttr("min_int")).getValue().getSExtValue();
+        cast<IntegerAttr>(op->getAttr("min_val")).getValue().getSExtValue();
     int64_t max =
-        cast<IntegerAttr>(op->getAttr("max_int")).getValue().getSExtValue();
+        cast<IntegerAttr>(op->getAttr("max_val")).getValue().getSExtValue();
 
     int64_t minRepresentable = std::numeric_limits<int64_t>::min();
     int64_t maxRepresentable = std::numeric_limits<int64_t>::max();
@@ -940,7 +960,13 @@ elementwiseMatchAndRewriteHelper(Operation *operation, ValueRange operands,
   auto loc = operation->getLoc();
   auto rank =
       cast<RankedTensorType>(operation->getResultTypes().front()).getRank();
-  auto expandedOperands = expandInputRanks(rewriter, loc, operands, rank);
+  // For the mul op we need to avoid expanding the rank of the optional shift
+  // input.
+  auto operandsToExpand =
+      isa<tosa::MulOp>(operation) ? operands.take_front(2) : operands;
+
+  auto expandedOperands =
+      expandInputRanks(rewriter, loc, operandsToExpand, rank);
   auto [targetShape, masterOperands] =
       computeTargetShape(rewriter, loc, indexPool, expandedOperands);
   auto broadcastOperands = broadcastDynamicDimensions(
@@ -1935,9 +1961,10 @@ struct TileConverter : public OpConversionPattern<tosa::TileOp> {
           nestedBuilder.create<linalg::YieldOp>(op.getLoc(), *args.begin());
         });
 
+    auto shapeValue = getTosaConstShape(
+        rewriter, loc, mlir::tosa::convertFromMlirShape(resultTy.getShape()));
     rewriter.replaceOpWithNewOp<tosa::ReshapeOp>(
-        op, resultTy, genericOp.getResult(0),
-        rewriter.getDenseI64ArrayAttr(resultTy.getShape()));
+        op, resultTy, genericOp.getResult(0), shapeValue);
     return success();
   }
 };
diff --git a/mlir/lib/Conversion/TosaToLinalg/TosaToLinalgNamed.cpp b/mlir/lib/Conversion/TosaToLinalg/TosaToLinalgNamed.cpp
index 57a5fe75a007b..6321cb6087394 100644
--- a/mlir/lib/Conversion/TosaToLinalg/TosaToLinalgNamed.cpp
+++ b/mlir/lib/Conversion/TosaToLinalg/TosaToLinalgNamed.cpp
@@ -258,7 +258,12 @@ class ConvConverter : public OpConversionPattern<TosaConvOp> {
     DenseI64ArrayAttr padAttr = op.getPadAttr();
     DenseI64ArrayAttr strideTosaAttr = op.getStrideAttr();
     DenseI64ArrayAttr dilationTosaAttr = op.getDilationAttr();
-    bool isQuantized = op.getQuantizationInfo().has_value();
+
+    auto failureOrMaybeZps = extractConvZpPair(op, rewriter);
+    if (llvm::failed(failureOrMaybeZps))
+      return failure();
+
+    auto maybeZps = failureOrMaybeZps.value();
 
     if (!weightTy.hasStaticShape() || !biasTy.hasStaticShape())
       return rewriter.notifyMatchFailure(
@@ -284,10 +289,7 @@ class ConvConverter : public OpConversionPattern<TosaConvOp> {
 
     // Apply padding as necessary.
     TypedAttr zeroAttr = rewriter.getZeroAttr(inputETy);
-    if (isQuantized) {
-      auto quantizationInfo = *op.getQuantizationInfo();
-      int64_t iZp = quantizationInfo.getInputZp();
-
+    if (maybeZps) {
       int64_t intMin =
           APInt::getSignedMinValue(inputETy.getIntOrFloatBitWidth())
               .getSExtValue();
@@ -295,11 +297,11 @@ class ConvConverter : public OpConversionPattern<TosaConvOp> {
           APInt::getSignedMaxValue(inputETy.getIntOrFloatBitWidth())
               .getSExtValue();
 
-      if (iZp < intMin || iZp > intMax)
+      if (maybeZps->inputZp < intMin || maybeZps->inputZp > intMax)
         return rewriter.notifyMatchFailure(
             op, "tosa.conv op quantization has zp outside of input range");
 
-      zeroAttr = rewriter.getIntegerAttr(inputETy, iZp);
+      zeroAttr = rewriter.getIntegerAttr(inputETy, maybeZps->inputZp);
     }
 
     llvm::SmallVector<int64_t> pad;
@@ -312,8 +314,8 @@ class ConvConverter : public OpConversionPattern<TosaConvOp> {
       // For 2D convolutions, we need to check if the target convolution op
       // wants a HWCF kernel layout.
       bool wantHwcf =
-          isQuantized ? std::is_same_v<LinalgConvQOp, linalg::Conv2DNhwcHwcfQOp>
-                      : std::is_same_v<LinalgConvOp, linalg::Conv2DNhwcHwcfOp>;
+          maybeZps ? std::is_same_v<LinalgConvQOp, linalg::Conv2DNhwcHwcfQOp>
+                   : std::is_same_v<LinalgConvOp, linalg::Conv2DNhwcHwcfOp>;
       if (wantHwcf) {
         // Transpose the kernel to match dimension ordering of the linalg
         // convolution operation.
@@ -374,10 +376,9 @@ class ConvConverter : public OpConversionPattern<TosaConvOp> {
     Value broadcastBias =
         linalgBroadcastAndMaybeExtSI(rewriter, loc, bias, biasEmptyTensor);
 
-    if (isQuantized) {
-      auto quantizationInfo = *op.getQuantizationInfo();
-      auto iZp = rewriter.getI32IntegerAttr(quantizationInfo.getInputZp());
-      auto kZp = rewriter.getI32IntegerAttr(quantizationInfo.getWeightZp());
+    if (maybeZps) {
+      auto iZp = rewriter.getI32IntegerAttr(maybeZps->inputZp);
+      auto kZp = rewriter.getI32IntegerAttr(maybeZps->weightZp);
 
       auto iZpVal = rewriter.create<arith::ConstantOp>(loc, iZp);
       auto kZpVal = rewriter.create<arith::ConstantOp>(loc, kZp);
@@ -440,26 +441,18 @@ class DepthwiseConvConverter
         /*inputSizeDims=*/{1, 2},
         /*kernelSizeDims=*/{0, 1}, rewriter);
 
-    bool isQuantized = op->hasAttr("quantization_info");
-    IntegerAttr iZp;
-    IntegerAttr kZp;
-    if (isQuantized) {
-      auto quantizationInfo =
-          cast<tosa::ConvOpQuantizationAttr>(op->getAttr("quantization_info"));
-      iZp = rewriter.getI32IntegerAttr(quantizationInfo.getInputZp());
-      kZp = rewriter.getI32IntegerAttr(quantizationInfo.getWeightZp());
-    }
+    auto failureOrMaybeZps = extractConvZpPair(op, rewriter);
+    if (llvm::failed(failureOrMaybeZps))
+      return failure();
+
+    auto maybeZps = failureOrMaybeZps.value();
 
     auto weightShape = weightTy.getShape();
     auto resultShape = resultTy.getShape();
 
     // Apply padding as necessary.
     TypedAttr zeroAttr = rewriter.getZeroAttr(inputETy);
-    if (isQuantized) {
-      auto quantizationInfo =
-          cast<tosa::ConvOpQuantizationAttr>(op->getAttr("quantization_info"));
-      int64_t iZp = quantizationInfo.getInputZp();
-
+    if (maybeZps) {
       int64_t intMin =
           APInt::getSignedMinValue(inputETy.getIntOrFloatBitWidth())
               .getSExtValue();
@@ -467,12 +460,12 @@ class DepthwiseConvConverter
           APInt::getSignedMaxValue(inputETy.getIntOrFloatBitWidth())
               .getSExtValue();
 
-      if (iZp < intMin || iZp > intMax)
+      if (maybeZps->inputZp < intMin || maybeZps->inputZp > intMax)
         return rewriter.notifyMatchFailure(
             op, "tosa.depthwise_conv op quantization has zp outside of input "
                 "range");
 
-      zeroAttr = rewriter.getIntegerAttr(inputETy, iZp);
+      zeroAttr = rewriter.getIntegerAttr(inputETy, maybeZps->inputZp);
     }
 
     llvm::SmallVector<int64_t> pad;
@@ -512,7 +505,7 @@ class DepthwiseConvConverter
     indexingMaps.push_back(rewriter.getMultiDimIdentityMap(resultRank));
     indexingMaps.push_back(rewriter.getMultiDimIdentityMap(resultRank));
 
-    if (!isQuantized) {
+    if (!maybeZps) {
       Value conv = rewriter
                        .create<linalg::DepthwiseConv2DNhwcHwcmOp>(
                            loc, linalgConvTy, ValueRange{input, weight},
@@ -539,8 +532,10 @@ class DepthwiseConvConverter
               .getResult(0);
       rewriter.replaceOp(op, result);
     } else {
+      IntegerAttr iZp = rewriter.getI32IntegerAttr(maybeZps->inputZp);
+      IntegerAttr wZp = rewriter.getI32IntegerAttr(maybeZps->weightZp);
       auto iZpVal = rewriter.create<arith::ConstantOp>(loc, iZp);
-      auto kZpVal = rewriter.create<arith::ConstantOp>(loc, kZp);
+      auto kZpVal = rewriter.create<arith::ConstantOp>(loc, wZp);
       Value conv =
           rewriter
               .create<linalg::DepthwiseConv2DNhwcHwcmQOp>(
@@ -595,18 +590,15 @@ class MatMulConverter : public OpConversionPattern<tosa::MatMulOp> {
                            .create<linalg::FillOp>(loc, ValueRange{zero},
                                                    ValueRange{emptyTensor})
                            .result();
-    if (!op.getQuantizationInfo()) {
+    if (!op.getAZp() && !op.getBZp()) {
       rewriter.replaceOpWithNewOp<linalg::BatchMatmulOp>(
           op, TypeRange{op.getType()},
           ValueRange{adaptor.getA(), adaptor.getB()}, ValueRange{zeroTensor});
       return success();
     }
 
-    auto quantizationInfo = *op.getQuantizationInfo();
-    auto aZp = rewriter.create<arith::ConstantOp>(
-        loc, rewriter.getI32IntegerAttr(quantizationInfo.getAZp()));
-    auto bZp = rewriter.create<arith::ConstantOp>(
-        loc, rewriter.getI32IntegerAttr(quantizationInfo.getBZp()));
+    auto aZp = rewriter.create<arith::ConstantOp>(loc, op.getAZpAttr());
+    auto bZp = rewriter.create<arith::ConstantOp>(loc, op.getBZpAttr());
     rewriter.replaceOpWithNewOp<linalg::QuantizedBatchMatmulOp>(
         op, TypeRange{op.getType()},
         ValueRange{adaptor.getA(), adaptor.getB(), aZp, bZp}, zeroTensor);
@@ -666,7 +658,7 @@ class FullyConnectedConverter
     Value broadcastBias =
         linalgBroadcastAndMaybeExtSI(rewriter, loc, bias, biasEmptyTensor);
 
-    if (!op.getQuantizationInfo()) {
+    if (!op.getInputZp() && !op.getWeightZp()) {
       Value matmul = rewriter
                          .create<linalg::MatmulOp>(
                              loc, TypeRange{op.getType()},
@@ -677,11 +669,9 @@ class FullyConnectedConverter
       return success();
     }
 
-    auto quantizationInfo = *op.getQuantizationInfo();
-    auto inputZp = rewriter.create<arith::ConstantOp>(
-        loc, rewriter.getI32IntegerAttr(quantizationInfo.getInputZp()));
-    auto outputZp = rewriter.create<arith::ConstantOp>(
-        loc, rewriter.getI32IntegerAttr(quantizationInfo.getWeightZp()));
+    auto inputZp = rewriter.create<arith::ConstantOp>(loc, op.getInputZpAttr());
+    auto outputZp =
+        rewriter.create<arith::ConstantOp>(loc, op.getWeightZpAttr());
     Value matmul =
         rewriter
             .create<linalg::QuantizedMatmulOp>(
@@ -963,10 +953,9 @@ class AvgPool2dConverter : public OpRewritePattern<tosa::AvgPool2dOp> {
 
             // If we have quantization information we need to apply an offset
             // for the input zp value.
-            if (op.getQuantizationInfo()) {
-              auto quantizationInfo = *op.getQuantizationInfo();
-              auto inputZp = rewriter.create<arith::ConstantOp>(
-                  loc, b.getIntegerAttr(accETy, quantizationInfo.getInputZp()));
+            if (op.getInputZp()) {
+              auto inputZp =
+                  rewriter.create<arith::ConstantOp>(loc, op.getInputZpAttr());
               Value offset =
                   rewriter.create<arith::MulIOp>(loc, accETy, count, inputZp);
               poolVal =
@@ -1018,11 +1007,9 @@ class AvgPool2dConverter : public OpRewritePattern<tosa::AvgPool2dOp> {
 
             // If we have quantization information we need to apply output
             // zeropoint.
-            if (op.getQuantizationInfo()) {
-              auto quantizationInfo = *op.getQuantizationInfo();
-              auto outputZp = rewriter.create<arith::ConstantOp>(
-                  loc, b.getIntegerAttr(scaled.getType(),
-                                        quantizationInfo.getOutputZp()));
+            if (op.getOutputZp()) {
+              auto outputZp =
+                  rewriter.create<arith::ConstantOp>(loc, op.getOutputZpAttr());
               scaled = rewriter.create<arith::AddIOp>(loc, scaled, outputZp)
                            .getResult();
             }
diff --git a/mlir/lib/Conversion/TosaToTensor/TosaToTensor.cpp b/mlir/lib/Conversion/TosaToTensor/TosaToTensor.cpp
index 5aa0269a675cb..7f029d56e2582 100644
--- a/mlir/lib/Conversion/TosaToTensor/TosaToTensor.cpp
+++ b/mlir/lib/Conversion/TosaToTensor/TosaToTensor.cpp
@@ -16,6 +16,7 @@
 #include "mlir/Dialect/Tensor/IR/Tensor.h"
 #include "mlir/Dialect/Tensor/Utils/Utils.h"
 #include "mlir/Dialect/Tosa/IR/TosaOps.h"
+#include "mlir/Dialect/Tosa/Utils/ConversionUtils.h"
 #include "mlir/IR/PatternMatch.h"
 #include "mlir/Transforms/DialectConversion.h"
 
@@ -235,7 +236,12 @@ class ReshapeConverter : public OpConversionPattern<tosa::ReshapeOp> {
       return rewriter.notifyMatchFailure(reshape.getLoc(),
                                          "expected input type to be tensor");
     }
-    auto newShape = reshape.getNewShape();
+
+    llvm::SmallVector<int64_t> newShape;
+    if (!tosa::getConstShapeValue(reshape.getShape().getDefiningOp(),
+                                  newShape)) {
+      return failure();
+    }
 
     // Infer all intermediate types
     auto inputType = inferReshapeInputType(input, newShape);
@@ -268,12 +274,28 @@ class SliceConverter : public OpConversionPattern<tosa::SliceOp> {
     ShapedType resultType = cast<ShapedType>(sliceOp.getType());
     if (llvm::isa<UnrankedTensorType>(resultType))
       return failure();
+
+    ElementsAttr startElems;
+    ElementsAttr sizeElems;
+
+    if (!matchPattern(sliceOp.getStart(), m_Constant(&startElems)))
+      return rewriter.notifyMatchFailure(
+          sliceOp, "start of slice must be a static ranked shape");
+
+    if (!matchPattern(sliceOp.getSize(), m_Constant(&sizeElems)))
+      return rewriter.notifyMatchFailure(
+          sliceOp, "size of slice must be a static ranked shape");
+
+    llvm::SmallVector<int64_t> sliceStarts =
+        llvm::to_vector(startElems.getValues<int64_t>());
+    llvm::SmallVector<int64_t> sliceSizes =
+        llvm::to_vector(sizeElems.getValues<int64_t>());
+
     SmallVector<int64_t> strides, sizes;
-    ArrayRef<int64_t> starts = sliceOp.getStart();
     strides.resize(cast<ShapedType>(sliceOp.getType()).getRank(), 1);
 
     SmallVector<Value> dynSizes;
-    for (const auto &i : llvm::enumerate(sliceOp.getSize())) {
+    for (const auto &i : llvm::enumerate(sliceSizes)) {
       int64_t size = i.value();
       size_t index = i.index();
       sizes.push_back(size == -1 ? ShapedType::kDynamic : size);
@@ -282,17 +304,27 @@ class SliceConverter : public OpConversionPattern<tosa::SliceOp> {
 
       auto dim = rewriter.create<tensor::DimOp>(loc, input, index);
       auto offset = rewriter.create<arith::ConstantOp>(
-          loc, rewriter.getIndexAttr(starts[index]));
+          loc, rewriter.getIndexAttr(sliceStarts[index]));
       dynSizes.push_back(rewriter.create<arith::SubIOp>(loc, dim, offset));
     }
 
     auto newSliceOp = rewriter.create<tensor::ExtractSliceOp>(
         sliceOp.getLoc(), sliceOp.getType(), input, ValueRange({}), dynSizes,
-        ValueRange({}), rewriter.getDenseI64ArrayAttr(starts),
+        ValueRange({}), rewriter.getDenseI64ArrayAttr(sliceStarts),
         rewriter.getDenseI64ArrayAttr(sizes),
         rewriter.getDenseI64ArrayAttr(strides));
 
     rewriter.replaceOp(sliceOp, newSliceOp.getResult());
+
+    // Remove const_shape ops when it no longer has use point.
+    Operation *startConstShape = sliceOp.getStart().getDefiningOp();
+    if (startConstShape->getResult(0).hasOneUse())
+      rewriter.eraseOp(startConstShape);
+
+    Operation *sizeConstShape = sliceOp.getSize().getDefiningOp();
+    if (sizeConstShape->getResult(0).hasOneUse())
+      rewriter.eraseOp(sizeConstShape);
+
     return success();
   }
 };
@@ -332,10 +364,10 @@ class PadConverter : public OpConversionPattern<tosa::PadOp> {
       TypedAttr constantAttr;
       if (isa<FloatType>(elementTy)) {
         constantAttr = rewriter.getFloatAttr(elementTy, 0.0);
-      } else if (isa<IntegerType>(elementTy) && !padOp.getQuantizationInfo()) {
+      } else if (isa<IntegerType>(elementTy) && !padOp.getInputZpAttr()) {
         constantAttr = rewriter.getIntegerAttr(elementTy, 0);
-      } else if (isa<IntegerType>(elementTy) && padOp.getQuantizationInfo()) {
-        int64_t value = padOp.getQuantizationInfo()->getInputZp();
+      } else if (isa<IntegerType>(elementTy) && padOp.getInputZpAttr()) {
+        int64_t value = padOp.getInputZpAttr().getInt();
         constantAttr = rewriter.getIntegerAttr(elementTy, value);
       }
       if (constantAttr)
diff --git a/mlir/lib/Conversion/UBToSPIRV/UBToSPIRV.cpp b/mlir/lib/Conversion/UBToSPIRV/UBToSPIRV.cpp
index a3806189e4060..01c35cba48c49 100644
--- a/mlir/lib/Conversion/UBToSPIRV/UBToSPIRV.cpp
+++ b/mlir/lib/Conversion/UBToSPIRV/UBToSPIRV.cpp
@@ -29,11 +29,6 @@ struct PoisonOpLowering final : OpConversionPattern<ub::PoisonOp> {
   matchAndRewrite(ub::PoisonOp op, OpAdaptor,
                   ConversionPatternRewriter &rewriter) const override {
     Type origType = op.getType();
-    if (!origType.isIntOrIndexOrFloat())
-      return rewriter.notifyMatchFailure(op, [&](Diagnostic &diag) {
-        diag << "unsupported type " << origType;
-      });
-
     Type resType = getTypeConverter()->convertType(origType);
     if (!resType)
       return rewriter.notifyMatchFailure(op, [&](Diagnostic &diag) {
diff --git a/mlir/lib/Conversion/VectorToArmSME/VectorToArmSME.cpp b/mlir/lib/Conversion/VectorToArmSME/VectorToArmSME.cpp
index 55965d9c2a531..4be0fffe8b728 100644
--- a/mlir/lib/Conversion/VectorToArmSME/VectorToArmSME.cpp
+++ b/mlir/lib/Conversion/VectorToArmSME/VectorToArmSME.cpp
@@ -332,7 +332,7 @@ struct SplatOpToArmSMELowering : public OpRewritePattern<vector::SplatOp> {
 ///   %transposed_src = arm_sme.tile_load %alloca[%c0, %c0]
 ///     layout<vertical> : memref<?x?xi32>, vector<[4]x[4]xi32>
 ///
-/// NOTE: Tranposing via memory is obviously expensive, the current intention
+/// NOTE: Transposing via memory is obviously expensive, the current intention
 /// is to avoid the transpose if possible, this is therefore intended as a
 /// fallback and to provide base support for Vector ops. If it turns out
 /// transposes can't be avoided then this should be replaced with a more optimal
diff --git a/mlir/lib/Conversion/VectorToLLVM/ConvertVectorToLLVM.cpp b/mlir/lib/Conversion/VectorToLLVM/ConvertVectorToLLVM.cpp
index a1e21cb524bd9..c9d637ce81f93 100644
--- a/mlir/lib/Conversion/VectorToLLVM/ConvertVectorToLLVM.cpp
+++ b/mlir/lib/Conversion/VectorToLLVM/ConvertVectorToLLVM.cpp
@@ -9,6 +9,7 @@
 #include "mlir/Conversion/VectorToLLVM/ConvertVectorToLLVM.h"
 
 #include "mlir/Conversion/ArithCommon/AttrToLLVMConverter.h"
+#include "mlir/Conversion/ConvertToLLVM/ToLLVMInterface.h"
 #include "mlir/Conversion/LLVMCommon/PrintCallHelper.h"
 #include "mlir/Conversion/LLVMCommon/TypeConverter.h"
 #include "mlir/Conversion/LLVMCommon/VectorPattern.h"
@@ -1027,7 +1028,7 @@ class VectorShuffleOpConversion
       eltType = arrayType.getElementType();
     else
       eltType = cast<VectorType>(llvmType).getElementType();
-    Value insert = rewriter.create<LLVM::UndefOp>(loc, llvmType);
+    Value insert = rewriter.create<LLVM::PoisonOp>(loc, llvmType);
     int64_t insPos = 0;
     for (int64_t extPos : mask) {
       Value value = adaptor.getV1();
@@ -1441,7 +1442,7 @@ class VectorTypeCastOpConversion
     auto int64Ty = IntegerType::get(rewriter.getContext(), 64);
 
     // Create descriptor.
-    auto desc = MemRefDescriptor::undef(rewriter, loc, llvmTargetDescriptorTy);
+    auto desc = MemRefDescriptor::poison(rewriter, loc, llvmTargetDescriptorTy);
     // Set allocated ptr.
     Value allocated = sourceMemRef.allocatedPtr(rewriter, loc);
     desc.setAllocatedPtr(rewriter, loc, allocated);
@@ -1546,11 +1547,15 @@ class VectorPrintOpConversion : public ConvertOpToLLVMPattern<vector::PrintOp> {
 
     auto punct = printOp.getPunctuation();
     if (auto stringLiteral = printOp.getStringLiteral()) {
-      LLVM::createPrintStrCall(rewriter, loc, parent, "vector_print_str",
-                               *stringLiteral, *getTypeConverter(),
-                               /*addNewline=*/false);
+      auto createResult =
+          LLVM::createPrintStrCall(rewriter, loc, parent, "vector_print_str",
+                                   *stringLiteral, *getTypeConverter(),
+                                   /*addNewline=*/false);
+      if (createResult.failed())
+        return failure();
+
     } else if (punct != PrintPunctuation::NoPunctuation) {
-      emitCall(rewriter, printOp->getLoc(), [&] {
+      FailureOr<LLVM::LLVMFuncOp> op = [&]() {
         switch (punct) {
         case PrintPunctuation::Close:
           return LLVM::lookupOrCreatePrintCloseFn(parent);
@@ -1563,7 +1568,10 @@ class VectorPrintOpConversion : public ConvertOpToLLVMPattern<vector::PrintOp> {
         default:
           llvm_unreachable("unexpected punctuation");
         }
-      }());
+      }();
+      if (failed(op))
+        return failure();
+      emitCall(rewriter, printOp->getLoc(), op.value());
     }
 
     rewriter.eraseOp(printOp);
@@ -1588,7 +1596,7 @@ class VectorPrintOpConversion : public ConvertOpToLLVMPattern<vector::PrintOp> {
 
     // Make sure element type has runtime support.
     PrintConversion conversion = PrintConversion::None;
-    Operation *printer;
+    FailureOr<Operation *> printer;
     if (printType.isF32()) {
       printer = LLVM::lookupOrCreatePrintF32Fn(parent);
     } else if (printType.isF64()) {
@@ -1631,6 +1639,8 @@ class VectorPrintOpConversion : public ConvertOpToLLVMPattern<vector::PrintOp> {
     } else {
       return failure();
     }
+    if (failed(printer))
+      return failure();
 
     switch (conversion) {
     case PrintConversion::ZeroExt64:
@@ -1648,7 +1658,7 @@ class VectorPrintOpConversion : public ConvertOpToLLVMPattern<vector::PrintOp> {
     case PrintConversion::None:
       break;
     }
-    emitCall(rewriter, loc, printer, value);
+    emitCall(rewriter, loc, printer.value(), value);
     return success();
   }
 
@@ -1672,9 +1682,10 @@ struct VectorSplatOpLowering : public ConvertOpToLLVMPattern<vector::SplatOp> {
     if (resultType.getRank() > 1)
       return failure();
 
-    // First insert it into an undef vector so we can shuffle it.
+    // First insert it into a poison vector so we can shuffle it.
     auto vectorType = typeConverter->convertType(splatOp.getType());
-    Value undef = rewriter.create<LLVM::UndefOp>(splatOp.getLoc(), vectorType);
+    Value poison =
+        rewriter.create<LLVM::PoisonOp>(splatOp.getLoc(), vectorType);
     auto zero = rewriter.create<LLVM::ConstantOp>(
         splatOp.getLoc(),
         typeConverter->convertType(rewriter.getIntegerType(32)),
@@ -1683,19 +1694,19 @@ struct VectorSplatOpLowering : public ConvertOpToLLVMPattern<vector::SplatOp> {
     // For 0-d vector, we simply do `insertelement`.
     if (resultType.getRank() == 0) {
       rewriter.replaceOpWithNewOp<LLVM::InsertElementOp>(
-          splatOp, vectorType, undef, adaptor.getInput(), zero);
+          splatOp, vectorType, poison, adaptor.getInput(), zero);
       return success();
     }
 
     // For 1-d vector, we additionally do a `vectorshuffle`.
     auto v = rewriter.create<LLVM::InsertElementOp>(
-        splatOp.getLoc(), vectorType, undef, adaptor.getInput(), zero);
+        splatOp.getLoc(), vectorType, poison, adaptor.getInput(), zero);
 
     int64_t width = cast<VectorType>(splatOp.getType()).getDimSize(0);
     SmallVector<int32_t> zeroValues(width, 0);
 
     // Shuffle the value across the desired number of elements.
-    rewriter.replaceOpWithNewOp<LLVM::ShuffleVectorOp>(splatOp, v, undef,
+    rewriter.replaceOpWithNewOp<LLVM::ShuffleVectorOp>(splatOp, v, poison,
                                                        zeroValues);
     return success();
   }
@@ -1724,11 +1735,11 @@ struct VectorSplatNdOpLowering : public ConvertOpToLLVMPattern<SplatOp> {
       return failure();
 
     // Construct returned value.
-    Value desc = rewriter.create<LLVM::UndefOp>(loc, llvmNDVectorTy);
+    Value desc = rewriter.create<LLVM::PoisonOp>(loc, llvmNDVectorTy);
 
     // Construct a 1-D vector with the splatted value that we insert in all the
     // places within the returned descriptor.
-    Value vdesc = rewriter.create<LLVM::UndefOp>(loc, llvm1DVectorTy);
+    Value vdesc = rewriter.create<LLVM::PoisonOp>(loc, llvm1DVectorTy);
     auto zero = rewriter.create<LLVM::ConstantOp>(
         loc, typeConverter->convertType(rewriter.getIntegerType(32)),
         rewriter.getZeroAttr(rewriter.getIntegerType(32)));
@@ -1869,7 +1880,7 @@ struct VectorFromElementsLowering
       return rewriter.notifyMatchFailure(fromElementsOp,
                                          "rank > 1 vectors are not supported");
     Type llvmType = typeConverter->convertType(vectorType);
-    Value result = rewriter.create<LLVM::UndefOp>(loc, llvmType);
+    Value result = rewriter.create<LLVM::PoisonOp>(loc, llvmType);
     for (auto [idx, val] : llvm::enumerate(adaptor.getElements()))
       result = rewriter.create<vector::InsertOp>(loc, val, result, idx);
     rewriter.replaceOp(fromElementsOp, result);
@@ -1933,3 +1944,27 @@ void mlir::populateVectorToLLVMMatrixConversionPatterns(
   patterns.add<VectorMatmulOpConversion>(converter);
   patterns.add<VectorFlatTransposeOpConversion>(converter);
 }
+
+namespace {
+struct VectorToLLVMDialectInterface : public ConvertToLLVMPatternInterface {
+  using ConvertToLLVMPatternInterface::ConvertToLLVMPatternInterface;
+  void loadDependentDialects(MLIRContext *context) const final {
+    context->loadDialect<LLVM::LLVMDialect>();
+  }
+
+  /// Hook for derived dialect interface to provide conversion patterns
+  /// and mark dialect legal for the conversion target.
+  void populateConvertToLLVMConversionPatterns(
+      ConversionTarget &target, LLVMTypeConverter &typeConverter,
+      RewritePatternSet &patterns) const final {
+    populateVectorToLLVMConversionPatterns(typeConverter, patterns);
+  }
+};
+} // namespace
+
+void mlir::vector::registerConvertVectorToLLVMInterface(
+    DialectRegistry &registry) {
+  registry.addExtension(+[](MLIRContext *ctx, vector::VectorDialect *dialect) {
+    dialect->addInterfaces<VectorToLLVMDialectInterface>();
+  });
+}
diff --git a/mlir/lib/Conversion/VectorToSPIRV/CMakeLists.txt b/mlir/lib/Conversion/VectorToSPIRV/CMakeLists.txt
index bb9f793d7fe0f..f4cdb2cf95a30 100644
--- a/mlir/lib/Conversion/VectorToSPIRV/CMakeLists.txt
+++ b/mlir/lib/Conversion/VectorToSPIRV/CMakeLists.txt
@@ -15,4 +15,5 @@ add_mlir_conversion_library(MLIRVectorToSPIRV
   MLIRSPIRVConversion
   MLIRVectorDialect
   MLIRTransforms
+  MLIRUBToSPIRV
   )
diff --git a/mlir/lib/Conversion/VectorToSPIRV/VectorToSPIRV.cpp b/mlir/lib/Conversion/VectorToSPIRV/VectorToSPIRV.cpp
index d3731db1ce55c..bca77ba68fbd1 100644
--- a/mlir/lib/Conversion/VectorToSPIRV/VectorToSPIRV.cpp
+++ b/mlir/lib/Conversion/VectorToSPIRV/VectorToSPIRV.cpp
@@ -13,7 +13,6 @@
 #include "mlir/Conversion/VectorToSPIRV/VectorToSPIRV.h"
 
 #include "mlir/Dialect/Arith/IR/Arith.h"
-#include "mlir/Dialect/SPIRV/IR/SPIRVDialect.h"
 #include "mlir/Dialect/SPIRV/IR/SPIRVOps.h"
 #include "mlir/Dialect/SPIRV/IR/SPIRVTypes.h"
 #include "mlir/Dialect/SPIRV/Transforms/SPIRVConversion.h"
@@ -138,6 +137,33 @@ struct VectorBroadcastConvert final
   }
 };
 
+// SPIR-V does not have a concept of a poison index for certain instructions,
+// which creates a UB hazard when lowering from otherwise equivalent Vector
+// dialect instructions, because this index will be considered out-of-bounds.
+// To avoid this, this function implements a dynamic sanitization that returns
+// some arbitrary safe index. For power-of-two vector sizes, this uses a bitmask
+// (presumably more efficient), and otherwise index 0 (always in-bounds).
+static Value sanitizeDynamicIndex(ConversionPatternRewriter &rewriter,
+                                  Location loc, Value dynamicIndex,
+                                  int64_t kPoisonIndex, unsigned vectorSize) {
+  if (llvm::isPowerOf2_32(vectorSize)) {
+    Value inBoundsMask = rewriter.create<spirv::ConstantOp>(
+        loc, dynamicIndex.getType(),
+        rewriter.getIntegerAttr(dynamicIndex.getType(), vectorSize - 1));
+    return rewriter.create<spirv::BitwiseAndOp>(loc, dynamicIndex,
+                                                inBoundsMask);
+  }
+  Value poisonIndex = rewriter.create<spirv::ConstantOp>(
+      loc, dynamicIndex.getType(),
+      rewriter.getIntegerAttr(dynamicIndex.getType(), kPoisonIndex));
+  Value cmpResult =
+      rewriter.create<spirv::IEqualOp>(loc, dynamicIndex, poisonIndex);
+  return rewriter.create<spirv::SelectOp>(
+      loc, cmpResult,
+      spirv::ConstantOp::getZero(dynamicIndex.getType(), loc, rewriter),
+      dynamicIndex);
+}
+
 struct VectorExtractOpConvert final
     : public OpConversionPattern<vector::ExtractOp> {
   using OpConversionPattern::OpConversionPattern;
@@ -155,14 +181,22 @@ struct VectorExtractOpConvert final
     }
 
     if (std::optional<int64_t> id =
-            getConstantIntValue(extractOp.getMixedPosition()[0]))
+            getConstantIntValue(extractOp.getMixedPosition()[0])) {
+      if (id == vector::ExtractOp::kPoisonIndex)
+        return rewriter.notifyMatchFailure(
+            extractOp,
+            "Static use of poison index handled elsewhere (folded to poison)");
       rewriter.replaceOpWithNewOp<spirv::CompositeExtractOp>(
           extractOp, dstType, adaptor.getVector(),
           rewriter.getI32ArrayAttr(id.value()));
-    else
+    } else {
+      Value sanitizedIndex = sanitizeDynamicIndex(
+          rewriter, extractOp.getLoc(), adaptor.getDynamicPosition()[0],
+          vector::ExtractOp::kPoisonIndex,
+          extractOp.getSourceVectorType().getNumElements());
       rewriter.replaceOpWithNewOp<spirv::VectorExtractDynamicOp>(
-          extractOp, dstType, adaptor.getVector(),
-          adaptor.getDynamicPosition()[0]);
+          extractOp, dstType, adaptor.getVector(), sanitizedIndex);
+    }
     return success();
   }
 };
@@ -267,13 +301,21 @@ struct VectorInsertOpConvert final
     }
 
     if (std::optional<int64_t> id =
-            getConstantIntValue(insertOp.getMixedPosition()[0]))
+            getConstantIntValue(insertOp.getMixedPosition()[0])) {
+      if (id == vector::InsertOp::kPoisonIndex)
+        return rewriter.notifyMatchFailure(
+            insertOp,
+            "Static use of poison index handled elsewhere (folded to poison)");
       rewriter.replaceOpWithNewOp<spirv::CompositeInsertOp>(
           insertOp, adaptor.getSource(), adaptor.getDest(), id.value());
-    else
+    } else {
+      Value sanitizedIndex = sanitizeDynamicIndex(
+          rewriter, insertOp.getLoc(), adaptor.getDynamicPosition()[0],
+          vector::InsertOp::kPoisonIndex,
+          insertOp.getDestVectorType().getNumElements());
       rewriter.replaceOpWithNewOp<spirv::VectorInsertDynamicOp>(
-          insertOp, insertOp.getDest(), adaptor.getSource(),
-          adaptor.getDynamicPosition()[0]);
+          insertOp, insertOp.getDest(), adaptor.getSource(), sanitizedIndex);
+    }
     return success();
   }
 };
@@ -728,10 +770,20 @@ struct VectorLoadOpConverter final
 
     spirv::StorageClass storageClass = attr.getValue();
     auto vectorType = loadOp.getVectorType();
-    auto vectorPtrType = spirv::PointerType::get(vectorType, storageClass);
-    Value castedAccessChain =
-        rewriter.create<spirv::BitcastOp>(loc, vectorPtrType, accessChain);
-    rewriter.replaceOpWithNewOp<spirv::LoadOp>(loadOp, vectorType,
+    // Use the converted vector type instead of original (single element vector
+    // would get converted to scalar).
+    auto spirvVectorType = typeConverter.convertType(vectorType);
+    auto vectorPtrType = spirv::PointerType::get(spirvVectorType, storageClass);
+
+    // For single element vectors, we don't need to bitcast the access chain to
+    // the original vector type. Both is going to be the same, a pointer
+    // to a scalar.
+    Value castedAccessChain = (vectorType.getNumElements() == 1)
+                                  ? accessChain
+                                  : rewriter.create<spirv::BitcastOp>(
+                                        loc, vectorPtrType, accessChain);
+
+    rewriter.replaceOpWithNewOp<spirv::LoadOp>(loadOp, spirvVectorType,
                                                castedAccessChain);
 
     return success();
@@ -764,8 +816,15 @@ struct VectorStoreOpConverter final
     spirv::StorageClass storageClass = attr.getValue();
     auto vectorType = storeOp.getVectorType();
     auto vectorPtrType = spirv::PointerType::get(vectorType, storageClass);
-    Value castedAccessChain =
-        rewriter.create<spirv::BitcastOp>(loc, vectorPtrType, accessChain);
+
+    // For single element vectors, we don't need to bitcast the access chain to
+    // the original vector type. Both is going to be the same, a pointer
+    // to a scalar.
+    Value castedAccessChain = (vectorType.getNumElements() == 1)
+                                  ? accessChain
+                                  : rewriter.create<spirv::BitcastOp>(
+                                        loc, vectorPtrType, accessChain);
+
     rewriter.replaceOpWithNewOp<spirv::StoreOp>(storeOp, castedAccessChain,
                                                 adaptor.getValueToStore());
 
diff --git a/mlir/lib/Conversion/VectorToSPIRV/VectorToSPIRVPass.cpp b/mlir/lib/Conversion/VectorToSPIRV/VectorToSPIRVPass.cpp
index 1932de1be603b..0735e1ee0c677 100644
--- a/mlir/lib/Conversion/VectorToSPIRV/VectorToSPIRVPass.cpp
+++ b/mlir/lib/Conversion/VectorToSPIRV/VectorToSPIRVPass.cpp
@@ -12,9 +12,11 @@
 
 #include "mlir/Conversion/VectorToSPIRV/VectorToSPIRVPass.h"
 
+#include "mlir/Conversion/UBToSPIRV/UBToSPIRV.h"
 #include "mlir/Conversion/VectorToSPIRV/VectorToSPIRV.h"
 #include "mlir/Dialect/SPIRV/IR/SPIRVDialect.h"
 #include "mlir/Dialect/SPIRV/Transforms/SPIRVConversion.h"
+#include "mlir/Dialect/UB/IR/UBOps.h"
 #include "mlir/Pass/Pass.h"
 #include "mlir/Transforms/DialectConversion.h"
 
@@ -48,6 +50,8 @@ void ConvertVectorToSPIRVPass::runOnOperation() {
 
   RewritePatternSet patterns(context);
   populateVectorToSPIRVPatterns(typeConverter, patterns);
+  // Used for folds, e.g. vector.extract[-1] -> ub.poison -> spirv.Undef.
+  ub::populateUBToSPIRVConversionPatterns(typeConverter, patterns);
 
   if (failed(applyPartialConversion(op, *target, std::move(patterns))))
     return signalPassFailure();
diff --git a/mlir/lib/Conversion/VectorToXeGPU/VectorToXeGPU.cpp b/mlir/lib/Conversion/VectorToXeGPU/VectorToXeGPU.cpp
index dc4ee4e926bb4..2d915b83f9a77 100644
--- a/mlir/lib/Conversion/VectorToXeGPU/VectorToXeGPU.cpp
+++ b/mlir/lib/Conversion/VectorToXeGPU/VectorToXeGPU.cpp
@@ -179,7 +179,7 @@ struct TransferReadLowering : public OpRewritePattern<vector::TransferReadOp> {
     if (isTransposeLoad &&
         elementType.getIntOrFloatBitWidth() < minTransposeBitWidth)
       return rewriter.notifyMatchFailure(
-          readOp, "Unsupported data type for tranposition");
+          readOp, "Unsupported data type for transposition");
 
     // If load is transposed, get the base shape for the tensor descriptor.
     SmallVector<int64_t> descShape(vecTy.getShape());
diff --git a/mlir/lib/Dialect/Affine/Analysis/Utils.cpp b/mlir/lib/Dialect/Affine/Analysis/Utils.cpp
index 29608647d8574..b829633252fdd 100644
--- a/mlir/lib/Dialect/Affine/Analysis/Utils.cpp
+++ b/mlir/lib/Dialect/Affine/Analysis/Utils.cpp
@@ -12,6 +12,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "mlir/Dialect/Affine/Analysis/Utils.h"
+
 #include "mlir/Analysis/Presburger/PresburgerRelation.h"
 #include "mlir/Dialect/Affine/Analysis/AffineAnalysis.h"
 #include "mlir/Dialect/Affine/Analysis/LoopAnalysis.h"
@@ -42,23 +43,49 @@ using Node = MemRefDependenceGraph::Node;
 // was encountered in the loop nest.
 void LoopNestStateCollector::collect(Operation *opToWalk) {
   opToWalk->walk([&](Operation *op) {
-    if (isa<AffineForOp>(op))
-      forOps.push_back(cast<AffineForOp>(op));
-    else if (op->getNumRegions() != 0 && !isa<AffineIfOp>(op))
-      hasNonAffineRegionOp = true;
-    else if (isa<AffineReadOpInterface>(op))
+    if (auto forOp = dyn_cast<AffineForOp>(op)) {
+      forOps.push_back(forOp);
+    } else if (isa<AffineReadOpInterface>(op)) {
       loadOpInsts.push_back(op);
-    else if (isa<AffineWriteOpInterface>(op))
+    } else if (isa<AffineWriteOpInterface>(op)) {
       storeOpInsts.push_back(op);
+    } else {
+      auto memInterface = dyn_cast<MemoryEffectOpInterface>(op);
+      if (!memInterface) {
+        if (op->hasTrait<OpTrait::HasRecursiveMemoryEffects>())
+          // This op itself is memory-effect free.
+          return;
+        // Check operands. Eg. ops like the `call` op are handled here.
+        for (Value v : op->getOperands()) {
+          if (!isa<MemRefType>(v.getType()))
+            continue;
+          // Conservatively, we assume the memref is read and written to.
+          memrefLoads.push_back(op);
+          memrefStores.push_back(op);
+        }
+      } else {
+        // Non-affine loads and stores.
+        if (hasEffect<MemoryEffects::Read>(op))
+          memrefLoads.push_back(op);
+        if (hasEffect<MemoryEffects::Write>(op))
+          memrefStores.push_back(op);
+        if (hasEffect<MemoryEffects::Free>(op))
+          memrefFrees.push_back(op);
+      }
+    }
   });
 }
 
-// Returns the load op count for 'memref'.
 unsigned Node::getLoadOpCount(Value memref) const {
   unsigned loadOpCount = 0;
   for (Operation *loadOp : loads) {
-    if (memref == cast<AffineReadOpInterface>(loadOp).getMemRef())
+    // Common case: affine reads.
+    if (auto affineLoad = dyn_cast<AffineReadOpInterface>(loadOp)) {
+      if (memref == affineLoad.getMemRef())
+        ++loadOpCount;
+    } else if (hasEffect<MemoryEffects::Read>(loadOp, memref)) {
       ++loadOpCount;
+    }
   }
   return loadOpCount;
 }
@@ -66,13 +93,40 @@ unsigned Node::getLoadOpCount(Value memref) const {
 // Returns the store op count for 'memref'.
 unsigned Node::getStoreOpCount(Value memref) const {
   unsigned storeOpCount = 0;
-  for (Operation *storeOp : stores) {
-    if (memref == cast<AffineWriteOpInterface>(storeOp).getMemRef())
+  for (auto *storeOp : llvm::concat<Operation *const>(stores, memrefStores)) {
+    // Common case: affine writes.
+    if (auto affineStore = dyn_cast<AffineWriteOpInterface>(storeOp)) {
+      if (memref == affineStore.getMemRef())
+        ++storeOpCount;
+    } else if (hasEffect<MemoryEffects::Write>(const_cast<Operation *>(storeOp),
+                                               memref)) {
       ++storeOpCount;
+    }
   }
   return storeOpCount;
 }
 
+// Returns the store op count for 'memref'.
+unsigned Node::hasStore(Value memref) const {
+  return llvm::any_of(
+      llvm::concat<Operation *const>(stores, memrefStores),
+      [&](Operation *storeOp) {
+        if (auto affineStore = dyn_cast<AffineWriteOpInterface>(storeOp)) {
+          if (memref == affineStore.getMemRef())
+            return true;
+        } else if (hasEffect<MemoryEffects::Write>(storeOp, memref)) {
+          return true;
+        }
+        return false;
+      });
+}
+
+unsigned Node::hasFree(Value memref) const {
+  return llvm::any_of(memrefFrees, [&](Operation *freeOp) {
+    return hasEffect<MemoryEffects::Free>(freeOp, memref);
+  });
+}
+
 // Returns all store ops in 'storeOps' which access 'memref'.
 void Node::getStoreOpsForMemref(Value memref,
                                 SmallVectorImpl<Operation *> *storeOps) const {
@@ -106,8 +160,89 @@ void Node::getLoadAndStoreMemrefSet(
   }
 }
 
-// Initializes the data dependence graph by walking operations in `block`.
-// Assigns each node in the graph a node id based on program order in 'f'.
+/// Returns the values that this op has a memref effect of type `EffectTys` on,
+/// not considering recursive effects.
+template <typename... EffectTys>
+static void getEffectedValues(Operation *op, SmallVectorImpl<Value> &values) {
+  auto memOp = dyn_cast<MemoryEffectOpInterface>(op);
+  if (!memOp) {
+    if (op->hasTrait<OpTrait::HasRecursiveMemoryEffects>())
+      // No effects.
+      return;
+    // Memref operands have to be considered as being affected.
+    for (Value operand : op->getOperands()) {
+      if (isa<MemRefType>(operand.getType()))
+        values.push_back(operand);
+    }
+    return;
+  }
+  SmallVector<SideEffects::EffectInstance<MemoryEffects::Effect>, 4> effects;
+  memOp.getEffects(effects);
+  for (auto &effect : effects) {
+    Value effectVal = effect.getValue();
+    if (isa<EffectTys...>(effect.getEffect()) && effectVal &&
+        isa<MemRefType>(effectVal.getType()))
+      values.push_back(effectVal);
+  };
+}
+
+/// Add `op` to MDG creating a new node and adding its memory accesses (affine
+/// or non-affine to memrefAccesses (memref -> list of nodes with accesses) map.
+static Node *
+addNodeToMDG(Operation *nodeOp, MemRefDependenceGraph &mdg,
+             DenseMap<Value, SetVector<unsigned>> &memrefAccesses) {
+  auto &nodes = mdg.nodes;
+  // Create graph node 'id' to represent top-level 'forOp' and record
+  // all loads and store accesses it contains.
+  LoopNestStateCollector collector;
+  collector.collect(nodeOp);
+  unsigned newNodeId = mdg.nextNodeId++;
+  Node &node = nodes.insert({newNodeId, Node(newNodeId, nodeOp)}).first->second;
+  for (Operation *op : collector.loadOpInsts) {
+    node.loads.push_back(op);
+    auto memref = cast<AffineReadOpInterface>(op).getMemRef();
+    memrefAccesses[memref].insert(node.id);
+  }
+  for (Operation *op : collector.storeOpInsts) {
+    node.stores.push_back(op);
+    auto memref = cast<AffineWriteOpInterface>(op).getMemRef();
+    memrefAccesses[memref].insert(node.id);
+  }
+  for (Operation *op : collector.memrefLoads) {
+    SmallVector<Value> effectedValues;
+    getEffectedValues<MemoryEffects::Read>(op, effectedValues);
+    if (llvm::any_of(((ValueRange)effectedValues).getTypes(),
+                     [](Type type) { return !isa<MemRefType>(type); }))
+      // We do not know the interaction here.
+      return nullptr;
+    for (Value memref : effectedValues)
+      memrefAccesses[memref].insert(node.id);
+    node.memrefLoads.push_back(op);
+  }
+  for (Operation *op : collector.memrefStores) {
+    SmallVector<Value> effectedValues;
+    getEffectedValues<MemoryEffects::Write>(op, effectedValues);
+    if (llvm::any_of((ValueRange(effectedValues)).getTypes(),
+                     [](Type type) { return !isa<MemRefType>(type); }))
+      return nullptr;
+    for (Value memref : effectedValues)
+      memrefAccesses[memref].insert(node.id);
+    node.memrefStores.push_back(op);
+  }
+  for (Operation *op : collector.memrefFrees) {
+    SmallVector<Value> effectedValues;
+    getEffectedValues<MemoryEffects::Free>(op, effectedValues);
+    if (llvm::any_of((ValueRange(effectedValues)).getTypes(),
+                     [](Type type) { return !isa<MemRefType>(type); }))
+      return nullptr;
+    for (Value memref : effectedValues)
+      memrefAccesses[memref].insert(node.id);
+    node.memrefFrees.push_back(op);
+  }
+
+  return &node;
+}
+
 bool MemRefDependenceGraph::init() {
   LLVM_DEBUG(llvm::dbgs() << "--- Initializing MDG ---\n");
   // Map from a memref to the set of ids of the nodes that have ops accessing
@@ -116,36 +251,19 @@ bool MemRefDependenceGraph::init() {
 
   DenseMap<Operation *, unsigned> forToNodeMap;
   for (Operation &op : block) {
-    if (dyn_cast<AffineForOp>(op)) {
-      // Create graph node 'id' to represent top-level 'forOp' and record
-      // all loads and store accesses it contains.
-      LoopNestStateCollector collector;
-      collector.collect(&op);
-      // Return false if a region holding op other than 'affine.for' and
-      // 'affine.if' was found (not currently supported).
-      if (collector.hasNonAffineRegionOp)
+    if (auto forOp = dyn_cast<AffineForOp>(op)) {
+      Node *node = addNodeToMDG(&op, *this, memrefAccesses);
+      if (!node)
         return false;
-      Node node(nextNodeId++, &op);
-      for (auto *opInst : collector.loadOpInsts) {
-        node.loads.push_back(opInst);
-        auto memref = cast<AffineReadOpInterface>(opInst).getMemRef();
-        memrefAccesses[memref].insert(node.id);
-      }
-      for (auto *opInst : collector.storeOpInsts) {
-        node.stores.push_back(opInst);
-        auto memref = cast<AffineWriteOpInterface>(opInst).getMemRef();
-        memrefAccesses[memref].insert(node.id);
-      }
-      forToNodeMap[&op] = node.id;
-      nodes.insert({node.id, node});
-    } else if (dyn_cast<AffineReadOpInterface>(op)) {
+      forToNodeMap[&op] = node->id;
+    } else if (isa<AffineReadOpInterface>(op)) {
       // Create graph node for top-level load op.
       Node node(nextNodeId++, &op);
       node.loads.push_back(&op);
       auto memref = cast<AffineReadOpInterface>(op).getMemRef();
       memrefAccesses[memref].insert(node.id);
       nodes.insert({node.id, node});
-    } else if (dyn_cast<AffineWriteOpInterface>(op)) {
+    } else if (isa<AffineWriteOpInterface>(op)) {
       // Create graph node for top-level store op.
       Node node(nextNodeId++, &op);
       node.stores.push_back(&op);
@@ -155,8 +273,9 @@ bool MemRefDependenceGraph::init() {
     } else if (op.getNumResults() > 0 && !op.use_empty()) {
       // Create graph node for top-level producer of SSA values, which
       // could be used by loop nest nodes.
-      Node node(nextNodeId++, &op);
-      nodes.insert({node.id, node});
+      Node *node = addNodeToMDG(&op, *this, memrefAccesses);
+      if (!node)
+        return false;
     } else if (!isMemoryEffectFree(&op) &&
                (op.getNumRegions() == 0 || isa<RegionBranchOpInterface>(op))) {
       // Create graph node for top-level op unless it is known to be
@@ -165,9 +284,10 @@ bool MemRefDependenceGraph::init() {
       // well-defined control flow. During the fusion validity checks, we look
       // for non-affine ops on the path from source to destination, at which
       // point we check which memrefs if any are used in the region.
-      Node node(nextNodeId++, &op);
-      nodes.insert({node.id, node});
-    } else if (op.getNumRegions() != 0) {
+      Node *node = addNodeToMDG(&op, *this, memrefAccesses);
+      if (!node)
+        return false;
+    } else if (op.getNumRegions() != 0 && !isa<RegionBranchOpInterface>(op)) {
       // Return false if non-handled/unknown region-holding ops are found. We
       // won't know what such ops do or what its regions mean; for e.g., it may
       // not be an imperative op.
@@ -175,6 +295,9 @@ bool MemRefDependenceGraph::init() {
                  << "MDG init failed; unknown region-holding op found!\n");
       return false;
     }
+    // We aren't creating nodes for memory-effect free ops either with no
+    // regions (unless it has results being used) or those with branch op
+    // interface.
   }
 
   for (auto &idAndNode : nodes) {
@@ -216,16 +339,20 @@ bool MemRefDependenceGraph::init() {
   // Walk memref access lists and add graph edges between dependent nodes.
   for (auto &memrefAndList : memrefAccesses) {
     unsigned n = memrefAndList.second.size();
+    Value srcMemRef = memrefAndList.first;
+    // Add edges between all dependent pairs among the node IDs on this memref.
     for (unsigned i = 0; i < n; ++i) {
       unsigned srcId = memrefAndList.second[i];
-      bool srcHasStore =
-          getNode(srcId)->getStoreOpCount(memrefAndList.first) > 0;
+      Node *srcNode = getNode(srcId);
+      bool srcHasStoreOrFree =
+          srcNode->hasStore(srcMemRef) || srcNode->hasFree(srcMemRef);
       for (unsigned j = i + 1; j < n; ++j) {
         unsigned dstId = memrefAndList.second[j];
-        bool dstHasStore =
-            getNode(dstId)->getStoreOpCount(memrefAndList.first) > 0;
-        if (srcHasStore || dstHasStore)
-          addEdge(srcId, dstId, memrefAndList.first);
+        Node *dstNode = getNode(dstId);
+        bool dstHasStoreOrFree =
+            dstNode->hasStore(srcMemRef) || dstNode->hasFree(srcMemRef);
+        if (srcHasStoreOrFree || dstHasStoreOrFree)
+          addEdge(srcId, dstId, srcMemRef);
       }
     }
   }
@@ -233,14 +360,14 @@ bool MemRefDependenceGraph::init() {
 }
 
 // Returns the graph node for 'id'.
-Node *MemRefDependenceGraph::getNode(unsigned id) {
+const Node *MemRefDependenceGraph::getNode(unsigned id) const {
   auto it = nodes.find(id);
   assert(it != nodes.end());
   return &it->second;
 }
 
 // Returns the graph node for 'forOp'.
-Node *MemRefDependenceGraph::getForOpNode(AffineForOp forOp) {
+const Node *MemRefDependenceGraph::getForOpNode(AffineForOp forOp) const {
   for (auto &idAndNode : nodes)
     if (idAndNode.second.op == forOp)
       return &idAndNode.second;
@@ -264,7 +391,7 @@ void MemRefDependenceGraph::removeNode(unsigned id) {
     }
   }
   // Remove each edge in 'outEdges[id]'.
-  if (outEdges.count(id) > 0) {
+  if (outEdges.contains(id)) {
     SmallVector<Edge, 2> oldOutEdges = outEdges[id];
     for (auto &outEdge : oldOutEdges) {
       removeEdge(id, outEdge.id, outEdge.value);
@@ -278,8 +405,8 @@ void MemRefDependenceGraph::removeNode(unsigned id) {
 
 // Returns true if node 'id' writes to any memref which escapes (or is an
 // argument to) the block. Returns false otherwise.
-bool MemRefDependenceGraph::writesToLiveInOrEscapingMemrefs(unsigned id) {
-  Node *node = getNode(id);
+bool MemRefDependenceGraph::writesToLiveInOrEscapingMemrefs(unsigned id) const {
+  const Node *node = getNode(id);
   for (auto *storeOpInst : node->stores) {
     auto memref = cast<AffineWriteOpInterface>(storeOpInst).getMemRef();
     auto *op = memref.getDefiningOp();
@@ -299,14 +426,14 @@ bool MemRefDependenceGraph::writesToLiveInOrEscapingMemrefs(unsigned id) {
 // is for 'value' if non-null, or for any value otherwise. Returns false
 // otherwise.
 bool MemRefDependenceGraph::hasEdge(unsigned srcId, unsigned dstId,
-                                    Value value) {
-  if (outEdges.count(srcId) == 0 || inEdges.count(dstId) == 0) {
+                                    Value value) const {
+  if (!outEdges.contains(srcId) || !inEdges.contains(dstId)) {
     return false;
   }
-  bool hasOutEdge = llvm::any_of(outEdges[srcId], [=](Edge &edge) {
+  bool hasOutEdge = llvm::any_of(outEdges.lookup(srcId), [=](const Edge &edge) {
     return edge.id == dstId && (!value || edge.value == value);
   });
-  bool hasInEdge = llvm::any_of(inEdges[dstId], [=](Edge &edge) {
+  bool hasInEdge = llvm::any_of(inEdges.lookup(dstId), [=](const Edge &edge) {
     return edge.id == srcId && (!value || edge.value == value);
   });
   return hasOutEdge && hasInEdge;
@@ -351,7 +478,8 @@ void MemRefDependenceGraph::removeEdge(unsigned srcId, unsigned dstId,
 // Returns true if there is a path in the dependence graph from node 'srcId'
 // to node 'dstId'. Returns false otherwise. `srcId`, `dstId`, and the
 // operations that the edges connected are expected to be from the same block.
-bool MemRefDependenceGraph::hasDependencePath(unsigned srcId, unsigned dstId) {
+bool MemRefDependenceGraph::hasDependencePath(unsigned srcId,
+                                              unsigned dstId) const {
   // Worklist state is: <node-id, next-output-edge-index-to-visit>
   SmallVector<std::pair<unsigned, unsigned>, 4> worklist;
   worklist.push_back({srcId, 0});
@@ -364,13 +492,13 @@ bool MemRefDependenceGraph::hasDependencePath(unsigned srcId, unsigned dstId) {
       return true;
     // Pop and continue if node has no out edges, or if all out edges have
     // already been visited.
-    if (outEdges.count(idAndIndex.first) == 0 ||
-        idAndIndex.second == outEdges[idAndIndex.first].size()) {
+    if (!outEdges.contains(idAndIndex.first) ||
+        idAndIndex.second == outEdges.lookup(idAndIndex.first).size()) {
       worklist.pop_back();
       continue;
     }
     // Get graph edge to traverse.
-    Edge edge = outEdges[idAndIndex.first][idAndIndex.second];
+    const Edge edge = outEdges.lookup(idAndIndex.first)[idAndIndex.second];
     // Increment next output edge index for 'idAndIndex'.
     ++idAndIndex.second;
     // Add node at 'edge.id' to the worklist. We don't need to consider
@@ -386,34 +514,34 @@ bool MemRefDependenceGraph::hasDependencePath(unsigned srcId, unsigned dstId) {
 // Returns the input edge count for node 'id' and 'memref' from src nodes
 // which access 'memref' with a store operation.
 unsigned MemRefDependenceGraph::getIncomingMemRefAccesses(unsigned id,
-                                                          Value memref) {
+                                                          Value memref) const {
   unsigned inEdgeCount = 0;
-  if (inEdges.count(id) > 0)
-    for (auto &inEdge : inEdges[id])
-      if (inEdge.value == memref) {
-        Node *srcNode = getNode(inEdge.id);
-        // Only count in edges from 'srcNode' if 'srcNode' accesses 'memref'
-        if (srcNode->getStoreOpCount(memref) > 0)
-          ++inEdgeCount;
-      }
+  for (const Edge &inEdge : inEdges.lookup(id)) {
+    if (inEdge.value == memref) {
+      const Node *srcNode = getNode(inEdge.id);
+      // Only count in edges from 'srcNode' if 'srcNode' accesses 'memref'
+      if (srcNode->getStoreOpCount(memref) > 0)
+        ++inEdgeCount;
+    }
+  }
   return inEdgeCount;
 }
 
 // Returns the output edge count for node 'id' and 'memref' (if non-null),
 // otherwise returns the total output edge count from node 'id'.
-unsigned MemRefDependenceGraph::getOutEdgeCount(unsigned id, Value memref) {
+unsigned MemRefDependenceGraph::getOutEdgeCount(unsigned id,
+                                                Value memref) const {
   unsigned outEdgeCount = 0;
-  if (outEdges.count(id) > 0)
-    for (auto &outEdge : outEdges[id])
-      if (!memref || outEdge.value == memref)
-        ++outEdgeCount;
+  for (const auto &outEdge : outEdges.lookup(id))
+    if (!memref || outEdge.value == memref)
+      ++outEdgeCount;
   return outEdgeCount;
 }
 
 /// Return all nodes which define SSA values used in node 'id'.
 void MemRefDependenceGraph::gatherDefiningNodes(
-    unsigned id, DenseSet<unsigned> &definingNodes) {
-  for (MemRefDependenceGraph::Edge edge : inEdges[id])
+    unsigned id, DenseSet<unsigned> &definingNodes) const {
+  for (const Edge &edge : inEdges.lookup(id))
     // By definition of edge, if the edge value is a non-memref value,
     // then the dependence is between a graph node which defines an SSA value
     // and another graph node which uses the SSA value.
@@ -426,8 +554,8 @@ void MemRefDependenceGraph::gatherDefiningNodes(
 // dependences. Returns nullptr if no such insertion point is found.
 Operation *
 MemRefDependenceGraph::getFusedLoopNestInsertionPoint(unsigned srcId,
-                                                      unsigned dstId) {
-  if (outEdges.count(srcId) == 0)
+                                                      unsigned dstId) const {
+  if (!outEdges.contains(srcId))
     return getNode(dstId)->op;
 
   // Skip if there is any defining node of 'dstId' that depends on 'srcId'.
@@ -443,13 +571,13 @@ MemRefDependenceGraph::getFusedLoopNestInsertionPoint(unsigned srcId,
 
   // Build set of insts in range (srcId, dstId) which depend on 'srcId'.
   SmallPtrSet<Operation *, 2> srcDepInsts;
-  for (auto &outEdge : outEdges[srcId])
+  for (auto &outEdge : outEdges.lookup(srcId))
     if (outEdge.id != dstId)
       srcDepInsts.insert(getNode(outEdge.id)->op);
 
   // Build set of insts in range (srcId, dstId) on which 'dstId' depends.
   SmallPtrSet<Operation *, 2> dstDepInsts;
-  for (auto &inEdge : inEdges[dstId])
+  for (auto &inEdge : inEdges.lookup(dstId))
     if (inEdge.id != srcId)
       dstDepInsts.insert(getNode(inEdge.id)->op);
 
@@ -509,7 +637,7 @@ void MemRefDependenceGraph::updateEdges(unsigned srcId, unsigned dstId,
     SmallVector<Edge, 2> oldInEdges = inEdges[srcId];
     for (auto &inEdge : oldInEdges) {
       // Add edge from 'inEdge.id' to 'dstId' if it's not a private memref.
-      if (privateMemRefs.count(inEdge.value) == 0)
+      if (!privateMemRefs.contains(inEdge.value))
         addEdge(inEdge.id, dstId, inEdge.value);
     }
   }
@@ -565,12 +693,17 @@ void MemRefDependenceGraph::updateEdges(unsigned sibId, unsigned dstId) {
 }
 
 // Adds ops in 'loads' and 'stores' to node at 'id'.
-void MemRefDependenceGraph::addToNode(
-    unsigned id, const SmallVectorImpl<Operation *> &loads,
-    const SmallVectorImpl<Operation *> &stores) {
+void MemRefDependenceGraph::addToNode(unsigned id, ArrayRef<Operation *> loads,
+                                      ArrayRef<Operation *> stores,
+                                      ArrayRef<Operation *> memrefLoads,
+                                      ArrayRef<Operation *> memrefStores,
+                                      ArrayRef<Operation *> memrefFrees) {
   Node *node = getNode(id);
   llvm::append_range(node->loads, loads);
   llvm::append_range(node->stores, stores);
+  llvm::append_range(node->memrefLoads, memrefLoads);
+  llvm::append_range(node->memrefStores, memrefStores);
+  llvm::append_range(node->memrefFrees, memrefFrees);
 }
 
 void MemRefDependenceGraph::clearNodeLoadAndStores(unsigned id) {
@@ -2167,3 +2300,41 @@ FailureOr<AffineValueMap> mlir::affine::simplifyConstrainedMinMaxOp(
   affine::canonicalizeMapAndOperands(&newMap, &newOperands);
   return AffineValueMap(newMap, newOperands);
 }
+
+Block *mlir::affine::findInnermostCommonBlockInScope(Operation *a,
+                                                     Operation *b) {
+  Region *aScope = mlir::affine::getAffineScope(a);
+  Region *bScope = mlir::affine::getAffineScope(b);
+  if (aScope != bScope)
+    return nullptr;
+
+  // Get the block ancestry of `op` while stopping at the affine scope `aScope`
+  // and store them in `ancestry`.
+  auto getBlockAncestry = [&](Operation *op,
+                              SmallVectorImpl<Block *> &ancestry) {
+    Operation *curOp = op;
+    do {
+      ancestry.push_back(curOp->getBlock());
+      if (curOp->getParentRegion() == aScope)
+        break;
+      curOp = curOp->getParentOp();
+    } while (curOp);
+    assert(curOp && "can't reach root op without passing through affine scope");
+    std::reverse(ancestry.begin(), ancestry.end());
+  };
+
+  SmallVector<Block *, 4> aAncestors, bAncestors;
+  getBlockAncestry(a, aAncestors);
+  getBlockAncestry(b, bAncestors);
+  assert(!aAncestors.empty() && !bAncestors.empty() &&
+         "at least one Block ancestor expected");
+
+  Block *innermostCommonBlock = nullptr;
+  for (unsigned a = 0, b = 0, e = aAncestors.size(), f = bAncestors.size();
+       a < e && b < f; ++a, ++b) {
+    if (aAncestors[a] != bAncestors[b])
+      break;
+    innermostCommonBlock = aAncestors[a];
+  }
+  return innermostCommonBlock;
+}
diff --git a/mlir/lib/Dialect/Affine/Transforms/AffineParallelize.cpp b/mlir/lib/Dialect/Affine/Transforms/AffineParallelize.cpp
index 64f2bc6e745c3..fa0676b206826 100644
--- a/mlir/lib/Dialect/Affine/Transforms/AffineParallelize.cpp
+++ b/mlir/lib/Dialect/Affine/Transforms/AffineParallelize.cpp
@@ -42,6 +42,8 @@ namespace {
 /// Convert all parallel affine.for op into 1-D affine.parallel op.
 struct AffineParallelize
     : public affine::impl::AffineParallelizeBase<AffineParallelize> {
+  using AffineParallelizeBase<AffineParallelize>::AffineParallelizeBase;
+
   void runOnOperation() override;
 };
 
@@ -90,8 +92,3 @@ void AffineParallelize::runOnOperation() {
     }
   }
 }
-
-std::unique_ptr<OperationPass<func::FuncOp>>
-mlir::affine::createAffineParallelizePass() {
-  return std::make_unique<AffineParallelize>();
-}
diff --git a/mlir/lib/Dialect/Affine/Transforms/LoopFusion.cpp b/mlir/lib/Dialect/Affine/Transforms/LoopFusion.cpp
index 6fefe4487ef59..b38dd8effe669 100644
--- a/mlir/lib/Dialect/Affine/Transforms/LoopFusion.cpp
+++ b/mlir/lib/Dialect/Affine/Transforms/LoopFusion.cpp
@@ -41,7 +41,7 @@ namespace affine {
 } // namespace affine
 } // namespace mlir
 
-#define DEBUG_TYPE "affine-loop-fusion"
+#define DEBUG_TYPE "affine-fusion"
 
 using namespace mlir;
 using namespace mlir::affine;
@@ -78,13 +78,13 @@ struct LoopFusion : public affine::impl::AffineLoopFusionBase<LoopFusion> {
 static bool canRemoveSrcNodeAfterFusion(
     unsigned srcId, unsigned dstId, const ComputationSliceState &fusionSlice,
     Operation *fusedLoopInsPoint, const DenseSet<Value> &escapingMemRefs,
-    MemRefDependenceGraph *mdg) {
+    const MemRefDependenceGraph &mdg) {
 
-  Operation *dstNodeOp = mdg->getNode(dstId)->op;
+  Operation *dstNodeOp = mdg.getNode(dstId)->op;
   bool hasOutDepsAfterFusion = false;
 
-  for (auto &outEdge : mdg->outEdges[srcId]) {
-    Operation *depNodeOp = mdg->getNode(outEdge.id)->op;
+  for (auto &outEdge : mdg.outEdges.lookup(srcId)) {
+    Operation *depNodeOp = mdg.getNode(outEdge.id)->op;
     // Skip dependence with dstOp since it will be removed after fusion.
     if (depNodeOp == dstNodeOp)
       continue;
@@ -134,22 +134,23 @@ static bool canRemoveSrcNodeAfterFusion(
 /// held if the 'mdg' is reused from a previous fusion step or if the node
 /// creation order changes in the future to support more advance cases.
 // TODO: Move this to a loop fusion utility once 'mdg' is also moved.
-static void getProducerCandidates(unsigned dstId, MemRefDependenceGraph *mdg,
+static void getProducerCandidates(unsigned dstId,
+                                  const MemRefDependenceGraph &mdg,
                                   SmallVectorImpl<unsigned> &srcIdCandidates) {
   // Skip if no input edges along which to fuse.
-  if (mdg->inEdges.count(dstId) == 0)
+  if (mdg.inEdges.count(dstId) == 0)
     return;
 
   // Gather memrefs from loads in 'dstId'.
-  auto *dstNode = mdg->getNode(dstId);
+  auto *dstNode = mdg.getNode(dstId);
   DenseSet<Value> consumedMemrefs;
   for (Operation *load : dstNode->loads)
     consumedMemrefs.insert(cast<AffineReadOpInterface>(load).getMemRef());
 
   // Traverse 'dstId' incoming edges and gather the nodes that contain a store
   // to one of the consumed memrefs.
-  for (auto &srcEdge : mdg->inEdges[dstId]) {
-    auto *srcNode = mdg->getNode(srcEdge.id);
+  for (const auto &srcEdge : mdg.inEdges.lookup(dstId)) {
+    const auto *srcNode = mdg.getNode(srcEdge.id);
     // Skip if 'srcNode' is not a loop nest.
     if (!isa<AffineForOp>(srcNode->op))
       continue;
@@ -169,10 +170,10 @@ static void getProducerCandidates(unsigned dstId, MemRefDependenceGraph *mdg,
 /// producer-consumer dependence between 'srcId' and 'dstId'.
 static void
 gatherProducerConsumerMemrefs(unsigned srcId, unsigned dstId,
-                              MemRefDependenceGraph *mdg,
+                              const MemRefDependenceGraph &mdg,
                               DenseSet<Value> &producerConsumerMemrefs) {
-  auto *dstNode = mdg->getNode(dstId);
-  auto *srcNode = mdg->getNode(srcId);
+  auto *dstNode = mdg.getNode(dstId);
+  auto *srcNode = mdg.getNode(srcId);
   gatherProducerConsumerMemrefs(srcNode->stores, dstNode->loads,
                                 producerConsumerMemrefs);
 }
@@ -214,14 +215,14 @@ static bool isEscapingMemref(Value memref, Block *block) {
 
 /// Returns in 'escapingMemRefs' the memrefs from affine store ops in node 'id'
 /// that escape the block or are accessed in a non-affine way.
-static void gatherEscapingMemrefs(unsigned id, MemRefDependenceGraph *mdg,
+static void gatherEscapingMemrefs(unsigned id, const MemRefDependenceGraph &mdg,
                                   DenseSet<Value> &escapingMemRefs) {
-  auto *node = mdg->getNode(id);
+  auto *node = mdg.getNode(id);
   for (Operation *storeOp : node->stores) {
     auto memref = cast<AffineWriteOpInterface>(storeOp).getMemRef();
     if (escapingMemRefs.count(memref))
       continue;
-    if (isEscapingMemref(memref, &mdg->block))
+    if (isEscapingMemref(memref, &mdg.block))
       escapingMemRefs.insert(memref);
   }
 }
@@ -237,29 +238,67 @@ static void sinkSequentialLoops(MemRefDependenceGraph::Node *node) {
   node->op = newRootForOp;
 }
 
-// Creates and returns a private (single-user) memref for fused loop rooted
-// at 'forOp', with (potentially reduced) memref size based on the
-// MemRefRegion written to by 'srcStoreOpInst' at depth 'dstLoopDepth'.
-// TODO: consider refactoring the common code from generateDma and
-// this one.
-static Value createPrivateMemRef(AffineForOp forOp, Operation *srcStoreOpInst,
+/// Get the operation that should act as a dominance filter while replacing
+/// memref uses with a private memref for which `producerStores` and
+/// `sliceInsertionBlock` are provided. This effectively determines in what
+/// part of the IR we should be performing the replacement.
+static Operation *
+getDominanceFilterForPrivateMemRefRepl(Block *sliceInsertionBlock,
+                                       ArrayRef<Operation *> producerStores) {
+  assert(!producerStores.empty() && "expected producer store");
+
+  // We first find the common block that contains the producer stores and
+  // the slice computation. The first ancestor among the ancestors of the
+  // producer stores in that common block is the dominance filter to use for
+  // replacement.
+  Block *commonBlock = nullptr;
+  // Find the common block of all relevant operations.
+  for (Operation *store : producerStores) {
+    Operation *otherOp =
+        !commonBlock ? &*sliceInsertionBlock->begin() : &*commonBlock->begin();
+    commonBlock = findInnermostCommonBlockInScope(store, otherOp);
+  }
+  assert(commonBlock &&
+         "common block of producer stores and slice should exist");
+
+  // Find the first ancestor among the ancestors of `producerStores` in
+  // `commonBlock`.
+  Operation *firstAncestor = nullptr;
+  for (Operation *store : producerStores) {
+    Operation *ancestor = commonBlock->findAncestorOpInBlock(*store);
+    assert(ancestor && "producer store should be contained in common block");
+    firstAncestor = !firstAncestor || ancestor->isBeforeInBlock(firstAncestor)
+                        ? ancestor
+                        : firstAncestor;
+  }
+  return firstAncestor;
+}
+
+// Creates and returns a private (single-user) memref for fused loop rooted at
+// 'forOp', with (potentially reduced) memref size based on the memref region
+// written to by `storeOps` at depth 'dstLoopDepth'. 'sliceInsertionBlock'
+// specifies the block in which the slice was/will be inserted.
+static Value createPrivateMemRef(AffineForOp forOp,
+                                 ArrayRef<Operation *> storeOps,
                                  unsigned dstLoopDepth,
                                  std::optional<unsigned> fastMemorySpace,
+                                 Block *sliceInsertionBlock,
                                  uint64_t localBufSizeThreshold) {
-  Operation *forInst = forOp.getOperation();
+  assert(!storeOps.empty() && "no source stores supplied");
+  Operation *srcStoreOp = storeOps[0];
 
   // Create builder to insert alloc op just before 'forOp'.
-  OpBuilder b(forInst);
+  OpBuilder b(forOp);
   // Builder to create constants at the top level.
-  OpBuilder top(forInst->getParentRegion());
+  OpBuilder top(forOp->getParentRegion());
   // Create new memref type based on slice bounds.
-  auto oldMemRef = cast<AffineWriteOpInterface>(srcStoreOpInst).getMemRef();
+  auto oldMemRef = cast<AffineWriteOpInterface>(srcStoreOp).getMemRef();
   auto oldMemRefType = cast<MemRefType>(oldMemRef.getType());
   unsigned rank = oldMemRefType.getRank();
 
   // Compute MemRefRegion for 'srcStoreOpInst' at depth 'dstLoopDepth'.
-  MemRefRegion region(srcStoreOpInst->getLoc());
-  bool validRegion = succeeded(region.compute(srcStoreOpInst, dstLoopDepth));
+  MemRefRegion region(srcStoreOp->getLoc());
+  bool validRegion = succeeded(region.compute(srcStoreOp, dstLoopDepth));
   (void)validRegion;
   assert(validRegion && "unexpected memref region failure");
   SmallVector<int64_t, 4> newShape;
@@ -332,62 +371,18 @@ static Value createPrivateMemRef(AffineForOp forOp, Operation *srcStoreOpInst,
       AffineMap::get(outerIVs.size() + rank, 0, remapExprs, forOp.getContext());
 
   // Replace all users of 'oldMemRef' with 'newMemRef'.
-  LogicalResult res =
-      replaceAllMemRefUsesWith(oldMemRef, newMemRef, {}, indexRemap,
-                               /*extraOperands=*/outerIVs,
-                               /*symbolOperands=*/{},
-                               /*domOpFilter=*/&*forOp.getBody()->begin());
+  Operation *domFilter =
+      getDominanceFilterForPrivateMemRefRepl(sliceInsertionBlock, storeOps);
+  LogicalResult res = replaceAllMemRefUsesWith(
+      oldMemRef, newMemRef, /*extraIndices=*/{}, indexRemap,
+      /*extraOperands=*/outerIVs,
+      /*symbolOperands=*/{}, domFilter);
   assert(succeeded(res) &&
          "replaceAllMemrefUsesWith should always succeed here");
   (void)res;
   return newMemRef;
 }
 
-/// Returns true if there are any non-affine uses of `memref` in any of
-/// the operations between `start` and `end` (both exclusive). Any other
-/// than affine read/write are treated as non-affine uses of `memref`.
-static bool hasNonAffineUsersOnPath(Operation *start, Operation *end,
-                                    Value memref) {
-  assert(start->getBlock() == end->getBlock());
-  assert(start->isBeforeInBlock(end) && "start expected to be before end");
-  Block *block = start->getBlock();
-  // Check if there is a non-affine memref user in any op between `start` and
-  // `end`.
-  return llvm::any_of(memref.getUsers(), [&](Operation *user) {
-    if (isa<AffineReadOpInterface, AffineWriteOpInterface>(user))
-      return false;
-    Operation *ancestor = block->findAncestorOpInBlock(*user);
-    return ancestor && start->isBeforeInBlock(ancestor) &&
-           ancestor->isBeforeInBlock(end);
-  });
-}
-
-/// Check whether a memref value used in any operation of 'src' has a
-/// non-affine operation that is between `src` and `end` (exclusive of `src`
-/// and `end`)  where `src` and `end` are expected to be in the same Block.
-/// Any other than affine read/write are treated as non-affine uses of memref.
-static bool hasNonAffineUsersOnPath(Operation *src, Operation *end) {
-  assert(src->getBlock() == end->getBlock() && "same block expected");
-
-  // Trivial case. `src` and `end` are exclusive.
-  if (src == end || end->isBeforeInBlock(src))
-    return false;
-
-  // Collect relevant memref values.
-  llvm::SmallDenseSet<Value, 2> memRefValues;
-  src->walk([&](Operation *op) {
-    for (Value v : op->getOperands())
-      // Collect memref values only.
-      if (isa<MemRefType>(v.getType()))
-        memRefValues.insert(v);
-    return WalkResult::advance();
-  });
-  // Look for non-affine users between `src` and `end`.
-  return llvm::any_of(memRefValues, [&](Value memref) {
-    return hasNonAffineUsersOnPath(src, end, memref);
-  });
-}
-
 // Checks the profitability of fusing a backwards slice of the loop nest
 // surrounding 'srcOpInst' into the loop nest surrounding 'dstLoadOpInsts'.
 // The argument 'srcStoreOpInst' is used to calculate the storage reduction on
@@ -832,7 +827,7 @@ struct GreedyFusion {
       // in 'srcIdCandidates'.
       dstNodeChanged = false;
       SmallVector<unsigned, 16> srcIdCandidates;
-      getProducerCandidates(dstId, mdg, srcIdCandidates);
+      getProducerCandidates(dstId, *mdg, srcIdCandidates);
 
       for (unsigned srcId : llvm::reverse(srcIdCandidates)) {
         // Get 'srcNode' from which to attempt fusion into 'dstNode'.
@@ -847,7 +842,7 @@ struct GreedyFusion {
           continue;
 
         DenseSet<Value> producerConsumerMemrefs;
-        gatherProducerConsumerMemrefs(srcId, dstId, mdg,
+        gatherProducerConsumerMemrefs(srcId, dstId, *mdg,
                                       producerConsumerMemrefs);
 
         // Skip if 'srcNode' out edge count on any memref is greater than
@@ -862,20 +857,7 @@ struct GreedyFusion {
         // block (e.g., memref block arguments, returned memrefs,
         // memrefs passed to function calls, etc.).
         DenseSet<Value> srcEscapingMemRefs;
-        gatherEscapingMemrefs(srcNode->id, mdg, srcEscapingMemRefs);
-
-        // Skip if there are non-affine operations in between the 'srcNode'
-        // and 'dstNode' using their memrefs. If so, we wouldn't be able to
-        // compute a legal insertion point for now. 'srcNode' and 'dstNode'
-        // memrefs with non-affine operation users would be considered
-        // escaping memrefs so we can limit this check to only scenarios with
-        // escaping memrefs.
-        if (!srcEscapingMemRefs.empty() &&
-            hasNonAffineUsersOnPath(srcNode->op, dstNode->op)) {
-          LLVM_DEBUG(llvm::dbgs()
-                     << "Can't fuse: non-affine users in between the loops\n");
-          continue;
-        }
+        gatherEscapingMemrefs(srcNode->id, *mdg, srcEscapingMemRefs);
 
         // Compute an operation list insertion point for the fused loop
         // nest which preserves dependences.
@@ -969,7 +951,7 @@ struct GreedyFusion {
         // insertion point.
         bool removeSrcNode = canRemoveSrcNodeAfterFusion(
             srcId, dstId, bestSlice, fusedLoopInsPoint, srcEscapingMemRefs,
-            mdg);
+            *mdg);
 
         DenseSet<Value> privateMemrefs;
         for (Value memref : producerConsumerMemrefs) {
@@ -1002,6 +984,10 @@ struct GreedyFusion {
 
         // Create private memrefs.
         if (!privateMemrefs.empty()) {
+          // Note the block into which fusion was performed. This can be used to
+          // place `alloc`s that create private memrefs.
+          Block *sliceInsertionBlock = bestSlice.insertPoint->getBlock();
+
           // Gather stores for all the private-to-be memrefs.
           DenseMap<Value, SmallVector<Operation *, 4>> privateMemRefToStores;
           dstAffineForOp.walk([&](AffineWriteOpInterface storeOp) {
@@ -1020,8 +1006,8 @@ struct GreedyFusion {
             SmallVector<Operation *, 4> &storesForMemref =
                 memrefToStoresPair.second;
             Value newMemRef = createPrivateMemRef(
-                dstAffineForOp, storesForMemref[0], bestDstLoopDepth,
-                fastMemorySpace, localBufSizeThreshold);
+                dstAffineForOp, storesForMemref, bestDstLoopDepth,
+                fastMemorySpace, sliceInsertionBlock, localBufSizeThreshold);
             // Create new node in dependence graph for 'newMemRef' alloc op.
             unsigned newMemRefNodeId = mdg->addNode(newMemRef.getDefiningOp());
             // Add edge from 'newMemRef' node to dstNode.
@@ -1039,8 +1025,10 @@ struct GreedyFusion {
 
         // Clear and add back loads and stores.
         mdg->clearNodeLoadAndStores(dstNode->id);
-        mdg->addToNode(dstId, dstLoopCollector.loadOpInsts,
-                       dstLoopCollector.storeOpInsts);
+        mdg->addToNode(
+            dstId, dstLoopCollector.loadOpInsts, dstLoopCollector.storeOpInsts,
+            dstLoopCollector.memrefLoads, dstLoopCollector.memrefStores,
+            dstLoopCollector.memrefFrees);
 
         if (removeSrcNode) {
           LLVM_DEBUG(llvm::dbgs()
@@ -1229,15 +1217,7 @@ struct GreedyFusion {
         storeMemrefs.insert(
             cast<AffineWriteOpInterface>(storeOpInst).getMemRef());
       }
-      if (storeMemrefs.size() > 1)
-        return false;
-
-      // Skip if a memref value in one node is used by a non-affine memref
-      // access that lies between 'dstNode' and 'sibNode'.
-      if (hasNonAffineUsersOnPath(dstNode->op, sibNode->op) ||
-          hasNonAffineUsersOnPath(sibNode->op, dstNode->op))
-        return false;
-      return true;
+      return storeMemrefs.size() <= 1;
     };
 
     // Search for siblings which load the same memref block argument.
@@ -1339,7 +1319,8 @@ struct GreedyFusion {
     // Clear and add back loads and stores
     mdg->clearNodeLoadAndStores(dstNode->id);
     mdg->addToNode(dstNode->id, dstLoopCollector.loadOpInsts,
-                   dstLoopCollector.storeOpInsts);
+                   dstLoopCollector.storeOpInsts, dstLoopCollector.memrefLoads,
+                   dstLoopCollector.memrefStores, dstLoopCollector.memrefFrees);
     // Remove old sibling loop nest if it no longer has outgoing dependence
     // edges, and it does not write to a memref which escapes the block.
     if (mdg->getOutEdgeCount(sibNode->id) == 0) {
diff --git a/mlir/lib/Dialect/Affine/Utils/Utils.cpp b/mlir/lib/Dialect/Affine/Utils/Utils.cpp
index 9e3257a62b12f..7ef016f88be37 100644
--- a/mlir/lib/Dialect/Affine/Utils/Utils.cpp
+++ b/mlir/lib/Dialect/Affine/Utils/Utils.cpp
@@ -1737,9 +1737,10 @@ static AffineExpr createDimSizeExprForTiledLayout(AffineExpr oldMapOutput,
 ///   %c4 = arith.constant 4 : index
 ///   %1 = affine.apply #map1(%c4, %0)
 ///   %2 = affine.apply #map2(%c4, %0)
+template <typename AllocLikeOp>
 static void createNewDynamicSizes(MemRefType oldMemRefType,
                                   MemRefType newMemRefType, AffineMap map,
-                                  memref::AllocOp *allocOp, OpBuilder b,
+                                  AllocLikeOp *allocOp, OpBuilder b,
                                   SmallVectorImpl<Value> &newDynamicSizes) {
   // Create new input for AffineApplyOp.
   SmallVector<Value, 4> inAffineApply;
@@ -1786,7 +1787,8 @@ static void createNewDynamicSizes(MemRefType oldMemRefType,
 }
 
 // TODO: Currently works for static memrefs with a single layout map.
-LogicalResult mlir::affine::normalizeMemRef(memref::AllocOp *allocOp) {
+template <typename AllocLikeOp>
+LogicalResult mlir::affine::normalizeMemRef(AllocLikeOp *allocOp) {
   MemRefType memrefType = allocOp->getType();
   OpBuilder b(*allocOp);
 
@@ -1802,7 +1804,7 @@ LogicalResult mlir::affine::normalizeMemRef(memref::AllocOp *allocOp) {
 
   SmallVector<Value, 4> symbolOperands(allocOp->getSymbolOperands());
   AffineMap layoutMap = memrefType.getLayout().getAffineMap();
-  memref::AllocOp newAlloc;
+  AllocLikeOp newAlloc;
   // Check if `layoutMap` is a tiled layout. Only single layout map is
   // supported for normalizing dynamic memrefs.
   SmallVector<std::tuple<AffineExpr, unsigned, unsigned>> tileSizePos;
@@ -1814,11 +1816,11 @@ LogicalResult mlir::affine::normalizeMemRef(memref::AllocOp *allocOp) {
                           newDynamicSizes);
     // Add the new dynamic sizes in new AllocOp.
     newAlloc =
-        b.create<memref::AllocOp>(allocOp->getLoc(), newMemRefType,
-                                  newDynamicSizes, allocOp->getAlignmentAttr());
+        b.create<AllocLikeOp>(allocOp->getLoc(), newMemRefType, newDynamicSizes,
+                              allocOp->getAlignmentAttr());
   } else {
-    newAlloc = b.create<memref::AllocOp>(allocOp->getLoc(), newMemRefType,
-                                         allocOp->getAlignmentAttr());
+    newAlloc = b.create<AllocLikeOp>(allocOp->getLoc(), newMemRefType,
+                                     allocOp->getAlignmentAttr());
   }
   // Replace all uses of the old memref.
   if (failed(replaceAllMemRefUsesWith(oldMemRef, /*newMemRef=*/newAlloc,
@@ -1843,6 +1845,11 @@ LogicalResult mlir::affine::normalizeMemRef(memref::AllocOp *allocOp) {
   return success();
 }
 
+template LogicalResult
+mlir::affine::normalizeMemRef<memref::AllocaOp>(memref::AllocaOp *op);
+template LogicalResult
+mlir::affine::normalizeMemRef<memref::AllocOp>(memref::AllocOp *op);
+
 MemRefType mlir::affine::normalizeMemRefType(MemRefType memrefType) {
   unsigned rank = memrefType.getRank();
   if (rank == 0)
diff --git a/mlir/lib/Dialect/Arith/IR/ArithOps.cpp b/mlir/lib/Dialect/Arith/IR/ArithOps.cpp
index 7ca104691e6df..8a9f223089794 100644
--- a/mlir/lib/Dialect/Arith/IR/ArithOps.cpp
+++ b/mlir/lib/Dialect/Arith/IR/ArithOps.cpp
@@ -1761,6 +1761,10 @@ OpFoldResult arith::BitcastOp::fold(FoldAdaptor adaptor) {
   if (llvm::isa<ShapedType>(resType))
     return {};
 
+  /// Bitcast poison.
+  if (llvm::isa<ub::PoisonAttr>(operand))
+    return ub::PoisonAttr::get(getContext());
+
   /// Bitcast integer or float to integer or float.
   APInt bits = llvm::isa<FloatAttr>(operand)
                    ? llvm::cast<FloatAttr>(operand).getValue().bitcastToAPInt()
@@ -1865,6 +1869,18 @@ OpFoldResult arith::CmpIOp::fold(FoldAdaptor adaptor) {
           getPredicate() == arith::CmpIPredicate::ne)
         return extOp.getOperand();
     }
+
+    // arith.cmpi ne, %val, %zero : i1 -> %val
+    if (getElementTypeOrSelf(getLhs().getType()).isInteger(1) &&
+        getPredicate() == arith::CmpIPredicate::ne)
+      return getLhs();
+  }
+
+  if (matchPattern(adaptor.getRhs(), m_One())) {
+    // arith.cmpi eq, %val, %one : i1 -> %val
+    if (getElementTypeOrSelf(getLhs().getType()).isInteger(1) &&
+        getPredicate() == arith::CmpIPredicate::eq)
+      return getLhs();
   }
 
   // Move constant to the right side.
diff --git a/mlir/lib/Dialect/Arith/Transforms/EmulateWideInt.cpp b/mlir/lib/Dialect/Arith/Transforms/EmulateWideInt.cpp
index 2d49854347d40..61f8d82a615d8 100644
--- a/mlir/lib/Dialect/Arith/Transforms/EmulateWideInt.cpp
+++ b/mlir/lib/Dialect/Arith/Transforms/EmulateWideInt.cpp
@@ -1044,9 +1044,9 @@ struct EmulateWideIntPass final
       return typeConverter.isLegal(op);
     };
     target.addDynamicallyLegalOp<func::CallOp, func::ReturnOp>(opLegalCallback);
-    target
-        .addDynamicallyLegalDialect<arith::ArithDialect, vector::VectorDialect>(
-            opLegalCallback);
+    target.addDynamicallyLegalOp<vector::PrintOp>(opLegalCallback);
+    target.addDynamicallyLegalDialect<arith::ArithDialect>(opLegalCallback);
+    target.addLegalDialect<vector::VectorDialect>();
 
     RewritePatternSet patterns(ctx);
     arith::populateArithWideIntEmulationPatterns(typeConverter, patterns);
diff --git a/mlir/lib/Dialect/Arith/Utils/Utils.cpp b/mlir/lib/Dialect/Arith/Utils/Utils.cpp
index 39c9005e449e3..8dde9866b22b3 100644
--- a/mlir/lib/Dialect/Arith/Utils/Utils.cpp
+++ b/mlir/lib/Dialect/Arith/Utils/Utils.cpp
@@ -69,7 +69,7 @@ mlir::inferExpandShapeOutputShape(OpBuilder &b, Location loc,
     Value indexGroupSize = cast<Value>(inputShape[inputIndex]);
     Value indexGroupStaticSizesProduct =
         b.create<arith::ConstantIndexOp>(loc, indexGroupStaticSizesProductInt);
-    Value dynamicDimSize = b.createOrFold<arith::DivUIOp>(
+    Value dynamicDimSize = b.createOrFold<arith::DivSIOp>(
         loc, indexGroupSize, indexGroupStaticSizesProduct);
     outputShapeValues.push_back(dynamicDimSize);
   }
diff --git a/mlir/lib/Dialect/ArmSME/Transforms/VectorLegalization.cpp b/mlir/lib/Dialect/ArmSME/Transforms/VectorLegalization.cpp
index 12c65a72babcb..dec3dca988ae9 100644
--- a/mlir/lib/Dialect/ArmSME/Transforms/VectorLegalization.cpp
+++ b/mlir/lib/Dialect/ArmSME/Transforms/VectorLegalization.cpp
@@ -304,7 +304,7 @@ struct LegalizeTransferReadOpsByDecomposition
                                          kMatchFailureNonPermutationMap);
 
     // Note: For 2D vector types the only non-identity permutation is a simple
-    // tranpose [1, 0].
+    // transpose [1, 0].
     bool transposed = !permutationMap.isIdentity();
 
     auto loc = readOp.getLoc();
@@ -352,7 +352,7 @@ struct LegalizeTransferWriteOpsByDecomposition
                                          kMatchFailureNonPermutationMap);
 
     // Note: For 2D vector types the only non-identity permutation is a simple
-    // tranpose [1, 0].
+    // transpose [1, 0].
     bool transposed = !permutationMap.isIdentity();
 
     auto loc = writeOp.getLoc();
diff --git a/mlir/lib/Dialect/Async/IR/Async.cpp b/mlir/lib/Dialect/Async/IR/Async.cpp
index a3e3f80954efc..d3bb250bb8ab9 100644
--- a/mlir/lib/Dialect/Async/IR/Async.cpp
+++ b/mlir/lib/Dialect/Async/IR/Async.cpp
@@ -308,7 +308,7 @@ void FuncOp::build(OpBuilder &builder, OperationState &state, StringRef name,
   if (argAttrs.empty())
     return;
   assert(type.getNumInputs() == argAttrs.size());
-  function_interface_impl::addArgAndResultAttrs(
+  call_interface_impl::addArgAndResultAttrs(
       builder, state, argAttrs, /*resultAttrs=*/std::nullopt,
       getArgAttrsAttrName(state.name), getResAttrsAttrName(state.name));
 }
diff --git a/mlir/lib/Dialect/Bufferization/IR/BufferizationOps.cpp b/mlir/lib/Dialect/Bufferization/IR/BufferizationOps.cpp
index 6be55a1d28224..4fce9be390bd6 100644
--- a/mlir/lib/Dialect/Bufferization/IR/BufferizationOps.cpp
+++ b/mlir/lib/Dialect/Bufferization/IR/BufferizationOps.cpp
@@ -28,11 +28,9 @@ FailureOr<Value> mlir::bufferization::castOrReallocMemRefValue(
     const BufferizationOptions &options) {
   auto srcType = llvm::cast<MemRefType>(value.getType());
 
-  // Element type, rank and memory space must match.
+  // Element type and rank must match.
   if (srcType.getElementType() != destType.getElementType())
     return failure();
-  if (srcType.getMemorySpace() != destType.getMemorySpace())
-    return failure();
   if (srcType.getRank() != destType.getRank())
     return failure();
 
diff --git a/mlir/lib/Dialect/Bufferization/Transforms/Bufferize.cpp b/mlir/lib/Dialect/Bufferization/Transforms/Bufferize.cpp
index 64d79f5b5d60c..313dbbd16984b 100644
--- a/mlir/lib/Dialect/Bufferization/Transforms/Bufferize.cpp
+++ b/mlir/lib/Dialect/Bufferization/Transforms/Bufferize.cpp
@@ -453,14 +453,15 @@ bufferization::bufferizeBlockSignature(Block *block, RewriterBase &rewriter,
     for (OpOperand &use : bbArg.getUses())
       bbArgUses.push_back(&use);
 
+    Type tensorType = bbArg.getType();
     // Change the bbArg type to memref.
     bbArg.setType(type);
 
     // Replace all uses of the original tensor bbArg.
     rewriter.setInsertionPointToStart(block);
     if (!bbArgUses.empty()) {
-      Value toTensorOp =
-          rewriter.create<bufferization::ToTensorOp>(bbArg.getLoc(), bbArg);
+      Value toTensorOp = rewriter.create<bufferization::ToTensorOp>(
+          bbArg.getLoc(), tensorType, bbArg);
       for (OpOperand *use : bbArgUses)
         use->set(toTensorOp);
     }
diff --git a/mlir/lib/Dialect/EmitC/IR/EmitC.cpp b/mlir/lib/Dialect/EmitC/IR/EmitC.cpp
index c818dd18a3d24..728a2d33f46e7 100644
--- a/mlir/lib/Dialect/EmitC/IR/EmitC.cpp
+++ b/mlir/lib/Dialect/EmitC/IR/EmitC.cpp
@@ -529,7 +529,7 @@ void FuncOp::build(OpBuilder &builder, OperationState &state, StringRef name,
   if (argAttrs.empty())
     return;
   assert(type.getNumInputs() == argAttrs.size());
-  function_interface_impl::addArgAndResultAttrs(
+  call_interface_impl::addArgAndResultAttrs(
       builder, state, argAttrs, /*resultAttrs=*/std::nullopt,
       getArgAttrsAttrName(state.name), getResAttrsAttrName(state.name));
 }
diff --git a/mlir/lib/Dialect/Func/IR/FuncOps.cpp b/mlir/lib/Dialect/Func/IR/FuncOps.cpp
index a490b4c3c4ab4..ba7b84f27d6a8 100644
--- a/mlir/lib/Dialect/Func/IR/FuncOps.cpp
+++ b/mlir/lib/Dialect/Func/IR/FuncOps.cpp
@@ -190,7 +190,7 @@ void FuncOp::build(OpBuilder &builder, OperationState &state, StringRef name,
   if (argAttrs.empty())
     return;
   assert(type.getNumInputs() == argAttrs.size());
-  function_interface_impl::addArgAndResultAttrs(
+  call_interface_impl::addArgAndResultAttrs(
       builder, state, argAttrs, /*resultAttrs=*/std::nullopt,
       getArgAttrsAttrName(state.name), getResAttrsAttrName(state.name));
 }
diff --git a/mlir/lib/Dialect/GPU/IR/GPUDialect.cpp b/mlir/lib/Dialect/GPU/IR/GPUDialect.cpp
index 301066e7d3e1f..d06f10d3137a1 100644
--- a/mlir/lib/Dialect/GPU/IR/GPUDialect.cpp
+++ b/mlir/lib/Dialect/GPU/IR/GPUDialect.cpp
@@ -1467,7 +1467,7 @@ ParseResult GPUFuncOp::parse(OpAsmParser &parser, OperationState &result) {
     return failure();
 
   auto signatureLocation = parser.getCurrentLocation();
-  if (failed(function_interface_impl::parseFunctionSignature(
+  if (failed(function_interface_impl::parseFunctionSignatureWithArguments(
           parser, /*allowVariadic=*/false, entryArgs, isVariadic, resultTypes,
           resultAttrs)))
     return failure();
@@ -1487,7 +1487,7 @@ ParseResult GPUFuncOp::parse(OpAsmParser &parser, OperationState &result) {
   result.addAttribute(getFunctionTypeAttrName(result.name),
                       TypeAttr::get(type));
 
-  function_interface_impl::addArgAndResultAttrs(
+  call_interface_impl::addArgAndResultAttrs(
       builder, result, entryArgs, resultAttrs, getArgAttrsAttrName(result.name),
       getResAttrsAttrName(result.name));
 
diff --git a/mlir/lib/Dialect/GPU/TransformOps/Utils.cpp b/mlir/lib/Dialect/GPU/TransformOps/Utils.cpp
index f4d36129bae77..f5a6d0893d9a9 100644
--- a/mlir/lib/Dialect/GPU/TransformOps/Utils.cpp
+++ b/mlir/lib/Dialect/GPU/TransformOps/Utils.cpp
@@ -14,6 +14,7 @@
 #include "mlir/Dialect/GPU/IR/GPUDialect.h"
 #include "mlir/Dialect/GPU/TransformOps/GPUTransformOps.h"
 #include "mlir/Dialect/MemRef/IR/MemRef.h"
+#include "mlir/Dialect/NVGPU/IR/NVGPUDialect.h"
 #include "mlir/Dialect/SCF/IR/DeviceMappingInterface.h"
 #include "mlir/Dialect/SCF/IR/SCF.h"
 #include "mlir/Dialect/Transform/IR/TransformDialect.h"
@@ -237,25 +238,17 @@ DiagnosedSilenceableFailure checkGpuLimits(TransformOpInterface transformOp,
                                            std::optional<int64_t> blockDimZ) {
 
   // TODO: pass a configuration object to set the limits properly.
-  static constexpr int maxTotalBlockdim = 1024;
-  static constexpr int maxBlockdimx = 1024;
-  static constexpr int maxBlockdimy = 1024;
-  static constexpr int maxBlockdimz = 64;
-  static constexpr int maxTotalGriddim = 2147483647;
-  static constexpr int maxGriddimx = 2147483647;
-  static constexpr int maxGriddimy = 65535;
-  static constexpr int maxGriddimz = 65535;
 
   if ((blockDimX.value_or(1) * blockDimY.value_or(1) * blockDimZ.value_or(1)) >
-          maxTotalBlockdim ||
+          kMaxTotalBlockdim ||
       (gridDimX.value_or(1) * gridDimY.value_or(1) * gridDimZ.value_or(1)) >
-          maxTotalGriddim ||
-      blockDimX.value_or(1) > maxBlockdimx ||
-      blockDimY.value_or(1) > maxBlockdimy ||
-      blockDimZ.value_or(1) > maxBlockdimz ||
-      gridDimY.value_or(1) > maxGriddimy ||
-      gridDimZ.value_or(1) > maxGriddimz ||
-      gridDimX.value_or(1) > maxGriddimx) {
+          kMaxTotalGriddim ||
+      blockDimX.value_or(1) > kMaxBlockdimx ||
+      blockDimY.value_or(1) > kMaxBlockdimy ||
+      blockDimZ.value_or(1) > kMaxBlockdimz ||
+      gridDimY.value_or(1) > kMaxGriddimy ||
+      gridDimZ.value_or(1) > kMaxGriddimz ||
+      gridDimX.value_or(1) > kMaxGriddimx) {
     return transformOp.emitSilenceableError()
            << "Trying to launch a GPU kernel with grid_dims = ("
            << gridDimX.value_or(1) << ", " << gridDimY.value_or(1) << ", "
diff --git a/mlir/lib/Dialect/LLVMIR/IR/FunctionCallUtils.cpp b/mlir/lib/Dialect/LLVMIR/IR/FunctionCallUtils.cpp
index 88421a16ccf9f..68d4426e65301 100644
--- a/mlir/lib/Dialect/LLVMIR/IR/FunctionCallUtils.cpp
+++ b/mlir/lib/Dialect/LLVMIR/IR/FunctionCallUtils.cpp
@@ -45,56 +45,85 @@ static constexpr llvm::StringRef kGenericFree = "_mlir_memref_to_llvm_free";
 static constexpr llvm::StringRef kMemRefCopy = "memrefCopy";
 
 /// Generic print function lookupOrCreate helper.
-LLVM::LLVMFuncOp mlir::LLVM::lookupOrCreateFn(Operation *moduleOp,
-                                              StringRef name,
-                                              ArrayRef<Type> paramTypes,
-                                              Type resultType, bool isVarArg) {
+FailureOr<LLVM::LLVMFuncOp>
+mlir::LLVM::lookupOrCreateFn(Operation *moduleOp, StringRef name,
+                             ArrayRef<Type> paramTypes, Type resultType,
+                             bool isVarArg, bool isReserved) {
   assert(moduleOp->hasTrait<OpTrait::SymbolTable>() &&
          "expected SymbolTable operation");
   auto func = llvm::dyn_cast_or_null<LLVM::LLVMFuncOp>(
       SymbolTable::lookupSymbolIn(moduleOp, name));
-  if (func)
+  auto funcT = LLVMFunctionType::get(resultType, paramTypes, isVarArg);
+  // Assert the signature of the found function is same as expected
+  if (func) {
+    if (funcT != func.getFunctionType()) {
+      if (isReserved) {
+        func.emitError("redefinition of reserved function '")
+            << name << "' of different type " << func.getFunctionType()
+            << " is prohibited";
+      } else {
+        func.emitError("redefinition of function '")
+            << name << "' of different type " << funcT << " is prohibited";
+      }
+      return failure();
+    }
     return func;
+  }
   OpBuilder b(moduleOp->getRegion(0));
   return b.create<LLVM::LLVMFuncOp>(
       moduleOp->getLoc(), name,
       LLVM::LLVMFunctionType::get(resultType, paramTypes, isVarArg));
 }
 
-LLVM::LLVMFuncOp mlir::LLVM::lookupOrCreatePrintI64Fn(Operation *moduleOp) {
-  return lookupOrCreateFn(moduleOp, kPrintI64,
-                          IntegerType::get(moduleOp->getContext(), 64),
-                          LLVM::LLVMVoidType::get(moduleOp->getContext()));
+static FailureOr<LLVM::LLVMFuncOp>
+lookupOrCreateReservedFn(Operation *moduleOp, StringRef name,
+                         ArrayRef<Type> paramTypes, Type resultType) {
+  return lookupOrCreateFn(moduleOp, name, paramTypes, resultType,
+                          /*isVarArg=*/false, /*isReserved=*/true);
 }
 
-LLVM::LLVMFuncOp mlir::LLVM::lookupOrCreatePrintU64Fn(Operation *moduleOp) {
-  return lookupOrCreateFn(moduleOp, kPrintU64,
-                          IntegerType::get(moduleOp->getContext(), 64),
-                          LLVM::LLVMVoidType::get(moduleOp->getContext()));
+FailureOr<LLVM::LLVMFuncOp>
+mlir::LLVM::lookupOrCreatePrintI64Fn(Operation *moduleOp) {
+  return lookupOrCreateReservedFn(
+      moduleOp, kPrintI64, IntegerType::get(moduleOp->getContext(), 64),
+      LLVM::LLVMVoidType::get(moduleOp->getContext()));
 }
 
-LLVM::LLVMFuncOp mlir::LLVM::lookupOrCreatePrintF16Fn(Operation *moduleOp) {
-  return lookupOrCreateFn(moduleOp, kPrintF16,
-                          IntegerType::get(moduleOp->getContext(), 16), // bits!
-                          LLVM::LLVMVoidType::get(moduleOp->getContext()));
+FailureOr<LLVM::LLVMFuncOp>
+mlir::LLVM::lookupOrCreatePrintU64Fn(Operation *moduleOp) {
+  return lookupOrCreateReservedFn(
+      moduleOp, kPrintU64, IntegerType::get(moduleOp->getContext(), 64),
+      LLVM::LLVMVoidType::get(moduleOp->getContext()));
 }
 
-LLVM::LLVMFuncOp mlir::LLVM::lookupOrCreatePrintBF16Fn(Operation *moduleOp) {
-  return lookupOrCreateFn(moduleOp, kPrintBF16,
-                          IntegerType::get(moduleOp->getContext(), 16), // bits!
-                          LLVM::LLVMVoidType::get(moduleOp->getContext()));
+FailureOr<LLVM::LLVMFuncOp>
+mlir::LLVM::lookupOrCreatePrintF16Fn(Operation *moduleOp) {
+  return lookupOrCreateReservedFn(
+      moduleOp, kPrintF16,
+      IntegerType::get(moduleOp->getContext(), 16), // bits!
+      LLVM::LLVMVoidType::get(moduleOp->getContext()));
 }
 
-LLVM::LLVMFuncOp mlir::LLVM::lookupOrCreatePrintF32Fn(Operation *moduleOp) {
-  return lookupOrCreateFn(moduleOp, kPrintF32,
-                          Float32Type::get(moduleOp->getContext()),
-                          LLVM::LLVMVoidType::get(moduleOp->getContext()));
+FailureOr<LLVM::LLVMFuncOp>
+mlir::LLVM::lookupOrCreatePrintBF16Fn(Operation *moduleOp) {
+  return lookupOrCreateReservedFn(
+      moduleOp, kPrintBF16,
+      IntegerType::get(moduleOp->getContext(), 16), // bits!
+      LLVM::LLVMVoidType::get(moduleOp->getContext()));
 }
 
-LLVM::LLVMFuncOp mlir::LLVM::lookupOrCreatePrintF64Fn(Operation *moduleOp) {
-  return lookupOrCreateFn(moduleOp, kPrintF64,
-                          Float64Type::get(moduleOp->getContext()),
-                          LLVM::LLVMVoidType::get(moduleOp->getContext()));
+FailureOr<LLVM::LLVMFuncOp>
+mlir::LLVM::lookupOrCreatePrintF32Fn(Operation *moduleOp) {
+  return lookupOrCreateReservedFn(
+      moduleOp, kPrintF32, Float32Type::get(moduleOp->getContext()),
+      LLVM::LLVMVoidType::get(moduleOp->getContext()));
+}
+
+FailureOr<LLVM::LLVMFuncOp>
+mlir::LLVM::lookupOrCreatePrintF64Fn(Operation *moduleOp) {
+  return lookupOrCreateReservedFn(
+      moduleOp, kPrintF64, Float64Type::get(moduleOp->getContext()),
+      LLVM::LLVMVoidType::get(moduleOp->getContext()));
 }
 
 static LLVM::LLVMPointerType getCharPtr(MLIRContext *context) {
@@ -106,75 +135,87 @@ static LLVM::LLVMPointerType getVoidPtr(MLIRContext *context) {
   return getCharPtr(context);
 }
 
-LLVM::LLVMFuncOp mlir::LLVM::lookupOrCreatePrintStringFn(
+FailureOr<LLVM::LLVMFuncOp> mlir::LLVM::lookupOrCreatePrintStringFn(
     Operation *moduleOp, std::optional<StringRef> runtimeFunctionName) {
-  return lookupOrCreateFn(moduleOp, runtimeFunctionName.value_or(kPrintString),
-                          getCharPtr(moduleOp->getContext()),
-                          LLVM::LLVMVoidType::get(moduleOp->getContext()));
+  return lookupOrCreateReservedFn(
+      moduleOp, runtimeFunctionName.value_or(kPrintString),
+      getCharPtr(moduleOp->getContext()),
+      LLVM::LLVMVoidType::get(moduleOp->getContext()));
 }
 
-LLVM::LLVMFuncOp mlir::LLVM::lookupOrCreatePrintOpenFn(Operation *moduleOp) {
-  return lookupOrCreateFn(moduleOp, kPrintOpen, {},
-                          LLVM::LLVMVoidType::get(moduleOp->getContext()));
+FailureOr<LLVM::LLVMFuncOp>
+mlir::LLVM::lookupOrCreatePrintOpenFn(Operation *moduleOp) {
+  return lookupOrCreateReservedFn(
+      moduleOp, kPrintOpen, {},
+      LLVM::LLVMVoidType::get(moduleOp->getContext()));
 }
 
-LLVM::LLVMFuncOp mlir::LLVM::lookupOrCreatePrintCloseFn(Operation *moduleOp) {
-  return lookupOrCreateFn(moduleOp, kPrintClose, {},
-                          LLVM::LLVMVoidType::get(moduleOp->getContext()));
+FailureOr<LLVM::LLVMFuncOp>
+mlir::LLVM::lookupOrCreatePrintCloseFn(Operation *moduleOp) {
+  return lookupOrCreateReservedFn(
+      moduleOp, kPrintClose, {},
+      LLVM::LLVMVoidType::get(moduleOp->getContext()));
 }
 
-LLVM::LLVMFuncOp mlir::LLVM::lookupOrCreatePrintCommaFn(Operation *moduleOp) {
-  return lookupOrCreateFn(moduleOp, kPrintComma, {},
-                          LLVM::LLVMVoidType::get(moduleOp->getContext()));
+FailureOr<LLVM::LLVMFuncOp>
+mlir::LLVM::lookupOrCreatePrintCommaFn(Operation *moduleOp) {
+  return lookupOrCreateReservedFn(
+      moduleOp, kPrintComma, {},
+      LLVM::LLVMVoidType::get(moduleOp->getContext()));
 }
 
-LLVM::LLVMFuncOp mlir::LLVM::lookupOrCreatePrintNewlineFn(Operation *moduleOp) {
-  return lookupOrCreateFn(moduleOp, kPrintNewline, {},
-                          LLVM::LLVMVoidType::get(moduleOp->getContext()));
+FailureOr<LLVM::LLVMFuncOp>
+mlir::LLVM::lookupOrCreatePrintNewlineFn(Operation *moduleOp) {
+  return lookupOrCreateReservedFn(
+      moduleOp, kPrintNewline, {},
+      LLVM::LLVMVoidType::get(moduleOp->getContext()));
 }
 
-LLVM::LLVMFuncOp mlir::LLVM::lookupOrCreateMallocFn(Operation *moduleOp,
-                                                    Type indexType) {
-  return LLVM::lookupOrCreateFn(moduleOp, kMalloc, indexType,
-                                getVoidPtr(moduleOp->getContext()));
+FailureOr<LLVM::LLVMFuncOp>
+mlir::LLVM::lookupOrCreateMallocFn(Operation *moduleOp, Type indexType) {
+  return lookupOrCreateReservedFn(moduleOp, kMalloc, indexType,
+                                  getVoidPtr(moduleOp->getContext()));
 }
 
-LLVM::LLVMFuncOp mlir::LLVM::lookupOrCreateAlignedAllocFn(Operation *moduleOp,
-                                                          Type indexType) {
-  return LLVM::lookupOrCreateFn(moduleOp, kAlignedAlloc, {indexType, indexType},
-                                getVoidPtr(moduleOp->getContext()));
+FailureOr<LLVM::LLVMFuncOp>
+mlir::LLVM::lookupOrCreateAlignedAllocFn(Operation *moduleOp, Type indexType) {
+  return lookupOrCreateReservedFn(moduleOp, kAlignedAlloc,
+                                  {indexType, indexType},
+                                  getVoidPtr(moduleOp->getContext()));
 }
 
-LLVM::LLVMFuncOp mlir::LLVM::lookupOrCreateFreeFn(Operation *moduleOp) {
-  return LLVM::lookupOrCreateFn(
+FailureOr<LLVM::LLVMFuncOp>
+mlir::LLVM::lookupOrCreateFreeFn(Operation *moduleOp) {
+  return lookupOrCreateReservedFn(
       moduleOp, kFree, getVoidPtr(moduleOp->getContext()),
       LLVM::LLVMVoidType::get(moduleOp->getContext()));
 }
 
-LLVM::LLVMFuncOp mlir::LLVM::lookupOrCreateGenericAllocFn(Operation *moduleOp,
-                                                          Type indexType) {
-  return LLVM::lookupOrCreateFn(moduleOp, kGenericAlloc, indexType,
-                                getVoidPtr(moduleOp->getContext()));
+FailureOr<LLVM::LLVMFuncOp>
+mlir::LLVM::lookupOrCreateGenericAllocFn(Operation *moduleOp, Type indexType) {
+  return lookupOrCreateReservedFn(moduleOp, kGenericAlloc, indexType,
+                                  getVoidPtr(moduleOp->getContext()));
 }
 
-LLVM::LLVMFuncOp
+FailureOr<LLVM::LLVMFuncOp>
 mlir::LLVM::lookupOrCreateGenericAlignedAllocFn(Operation *moduleOp,
                                                 Type indexType) {
-  return LLVM::lookupOrCreateFn(moduleOp, kGenericAlignedAlloc,
-                                {indexType, indexType},
-                                getVoidPtr(moduleOp->getContext()));
+  return lookupOrCreateReservedFn(moduleOp, kGenericAlignedAlloc,
+                                  {indexType, indexType},
+                                  getVoidPtr(moduleOp->getContext()));
 }
 
-LLVM::LLVMFuncOp mlir::LLVM::lookupOrCreateGenericFreeFn(Operation *moduleOp) {
-  return LLVM::lookupOrCreateFn(
+FailureOr<LLVM::LLVMFuncOp>
+mlir::LLVM::lookupOrCreateGenericFreeFn(Operation *moduleOp) {
+  return lookupOrCreateReservedFn(
       moduleOp, kGenericFree, getVoidPtr(moduleOp->getContext()),
       LLVM::LLVMVoidType::get(moduleOp->getContext()));
 }
 
-LLVM::LLVMFuncOp
+FailureOr<LLVM::LLVMFuncOp>
 mlir::LLVM::lookupOrCreateMemRefCopyFn(Operation *moduleOp, Type indexType,
                                        Type unrankedDescriptorType) {
-  return LLVM::lookupOrCreateFn(
+  return lookupOrCreateReservedFn(
       moduleOp, kMemRefCopy,
       ArrayRef<Type>{indexType, unrankedDescriptorType, unrankedDescriptorType},
       LLVM::LLVMVoidType::get(moduleOp->getContext()));
diff --git a/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp b/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp
index ef5f1b069b40a..bfcba40555a7c 100644
--- a/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp
+++ b/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp
@@ -1033,6 +1033,7 @@ void CallOp::build(OpBuilder &builder, OperationState &state, TypeRange results,
         /*memory_effects=*/nullptr,
         /*convergent=*/nullptr, /*no_unwind=*/nullptr, /*will_return=*/nullptr,
         /*op_bundle_operands=*/{}, /*op_bundle_tags=*/{},
+        /*arg_attrs=*/nullptr, /*res_attrs=*/nullptr,
         /*access_groups=*/nullptr, /*alias_scopes=*/nullptr,
         /*noalias_scopes=*/nullptr, /*tbaa=*/nullptr);
 }
@@ -1060,6 +1061,7 @@ void CallOp::build(OpBuilder &builder, OperationState &state,
         /*convergent=*/nullptr,
         /*no_unwind=*/nullptr, /*will_return=*/nullptr,
         /*op_bundle_operands=*/{}, /*op_bundle_tags=*/{},
+        /*arg_attrs=*/nullptr, /*res_attrs=*/nullptr,
         /*access_groups=*/nullptr,
         /*alias_scopes=*/nullptr, /*noalias_scopes=*/nullptr, /*tbaa=*/nullptr);
 }
@@ -1073,6 +1075,7 @@ void CallOp::build(OpBuilder &builder, OperationState &state,
         /*CConv=*/nullptr, /*TailCallKind=*/nullptr, /*memory_effects=*/nullptr,
         /*convergent=*/nullptr, /*no_unwind=*/nullptr, /*will_return=*/nullptr,
         /*op_bundle_operands=*/{}, /*op_bundle_tags=*/{},
+        /*arg_attrs=*/nullptr, /*res_attrs=*/nullptr,
         /*access_groups=*/nullptr, /*alias_scopes=*/nullptr,
         /*noalias_scopes=*/nullptr, /*tbaa=*/nullptr);
 }
@@ -1087,6 +1090,7 @@ void CallOp::build(OpBuilder &builder, OperationState &state, LLVMFuncOp func,
         /*convergent=*/nullptr, /*no_unwind=*/nullptr, /*will_return=*/nullptr,
         /*op_bundle_operands=*/{}, /*op_bundle_tags=*/{},
         /*access_groups=*/nullptr, /*alias_scopes=*/nullptr,
+        /*arg_attrs=*/nullptr, /*res_attrs=*/nullptr,
         /*noalias_scopes=*/nullptr, /*tbaa=*/nullptr);
 }
 
@@ -1331,42 +1335,53 @@ void CallOp::print(OpAsmPrinter &p) {
                            getVarCalleeTypeAttrName(), getCConvAttrName(),
                            getOperandSegmentSizesAttrName(),
                            getOpBundleSizesAttrName(),
-                           getOpBundleTagsAttrName()});
+                           getOpBundleTagsAttrName(), getArgAttrsAttrName(),
+                           getResAttrsAttrName()});
 
   p << " : ";
   if (!isDirect)
     p << getOperand(0).getType() << ", ";
 
-  // Reconstruct the function MLIR function type from operand and result types.
-  p.printFunctionalType(args.getTypes(), getResultTypes());
+  // Reconstruct the MLIR function type from operand and result types.
+  call_interface_impl::printFunctionSignature(
+      p, args.getTypes(), getArgAttrsAttr(),
+      /*isVariadic=*/false, getResultTypes(), getResAttrsAttr());
 }
 
 /// Parses the type of a call operation and resolves the operands if the parsing
 /// succeeds. Returns failure otherwise.
 static ParseResult parseCallTypeAndResolveOperands(
     OpAsmParser &parser, OperationState &result, bool isDirect,
-    ArrayRef<OpAsmParser::UnresolvedOperand> operands) {
+    ArrayRef<OpAsmParser::UnresolvedOperand> operands,
+    SmallVectorImpl<DictionaryAttr> &argAttrs,
+    SmallVectorImpl<DictionaryAttr> &resultAttrs) {
   SMLoc trailingTypesLoc = parser.getCurrentLocation();
   SmallVector<Type> types;
-  if (parser.parseColonTypeList(types))
+  if (parser.parseColon())
     return failure();
-
-  if (isDirect && types.size() != 1)
-    return parser.emitError(trailingTypesLoc,
-                            "expected direct call to have 1 trailing type");
-  if (!isDirect && types.size() != 2)
-    return parser.emitError(trailingTypesLoc,
-                            "expected indirect call to have 2 trailing types");
-
-  auto funcType = llvm::dyn_cast<FunctionType>(types.pop_back_val());
-  if (!funcType)
+  if (!isDirect) {
+    types.emplace_back();
+    if (parser.parseType(types.back()))
+      return failure();
+    if (parser.parseOptionalComma())
+      return parser.emitError(
+          trailingTypesLoc, "expected indirect call to have 2 trailing types");
+  }
+  SmallVector<Type> argTypes;
+  SmallVector<Type> resTypes;
+  if (call_interface_impl::parseFunctionSignature(parser, argTypes, argAttrs,
+                                                  resTypes, resultAttrs)) {
+    if (isDirect)
+      return parser.emitError(trailingTypesLoc,
+                              "expected direct call to have 1 trailing types");
     return parser.emitError(trailingTypesLoc,
                             "expected trailing function type");
-  if (funcType.getNumResults() > 1)
+  }
+
+  if (resTypes.size() > 1)
     return parser.emitError(trailingTypesLoc,
                             "expected function with 0 or 1 result");
-  if (funcType.getNumResults() == 1 &&
-      llvm::isa<LLVM::LLVMVoidType>(funcType.getResult(0)))
+  if (resTypes.size() == 1 && llvm::isa<LLVM::LLVMVoidType>(resTypes[0]))
     return parser.emitError(trailingTypesLoc,
                             "expected a non-void result type");
 
@@ -1374,12 +1389,12 @@ static ParseResult parseCallTypeAndResolveOperands(
   // indirect calls, while the types list is emtpy for direct calls.
   // Append the function input types to resolve the call operation
   // operands.
-  llvm::append_range(types, funcType.getInputs());
+  llvm::append_range(types, argTypes);
   if (parser.resolveOperands(operands, types, parser.getNameLoc(),
                              result.operands))
     return failure();
-  if (funcType.getNumResults() != 0)
-    result.addTypes(funcType.getResults());
+  if (resTypes.size() != 0)
+    result.addTypes(resTypes);
 
   return success();
 }
@@ -1493,8 +1508,14 @@ ParseResult CallOp::parse(OpAsmParser &parser, OperationState &result) {
     return failure();
 
   // Parse the trailing type list and resolve the operands.
-  if (parseCallTypeAndResolveOperands(parser, result, isDirect, operands))
+  SmallVector<DictionaryAttr> argAttrs;
+  SmallVector<DictionaryAttr> resultAttrs;
+  if (parseCallTypeAndResolveOperands(parser, result, isDirect, operands,
+                                      argAttrs, resultAttrs))
     return failure();
+  call_interface_impl::addArgAndResultAttrs(
+      parser.getBuilder(), result, argAttrs, resultAttrs,
+      getArgAttrsAttrName(result.name), getResAttrsAttrName(result.name));
   if (resolveOpBundleOperands(parser, opBundlesLoc, result, opBundleOperands,
                               opBundleOperandTypes,
                               getOpBundleSizesAttrName(result.name)))
@@ -1527,7 +1548,8 @@ void InvokeOp::build(OpBuilder &builder, OperationState &state, LLVMFuncOp func,
   auto calleeType = func.getFunctionType();
   build(builder, state, getCallOpResultTypes(calleeType),
         getCallOpVarCalleeType(calleeType), SymbolRefAttr::get(func), ops,
-        normalOps, unwindOps, nullptr, nullptr, {}, {}, normal, unwind);
+        /*arg_attrs=*/nullptr, /*res_attrs=*/nullptr, normalOps, unwindOps,
+        nullptr, nullptr, {}, {}, normal, unwind);
 }
 
 void InvokeOp::build(OpBuilder &builder, OperationState &state, TypeRange tys,
@@ -1535,8 +1557,9 @@ void InvokeOp::build(OpBuilder &builder, OperationState &state, TypeRange tys,
                      ValueRange normalOps, Block *unwind,
                      ValueRange unwindOps) {
   build(builder, state, tys,
-        /*var_callee_type=*/nullptr, callee, ops, normalOps, unwindOps, nullptr,
-        nullptr, {}, {}, normal, unwind);
+        /*var_callee_type=*/nullptr, callee, ops, /*arg_attrs=*/nullptr,
+        /*res_attrs=*/nullptr, normalOps, unwindOps, nullptr, nullptr, {}, {},
+        normal, unwind);
 }
 
 void InvokeOp::build(OpBuilder &builder, OperationState &state,
@@ -1544,7 +1567,8 @@ void InvokeOp::build(OpBuilder &builder, OperationState &state,
                      ValueRange ops, Block *normal, ValueRange normalOps,
                      Block *unwind, ValueRange unwindOps) {
   build(builder, state, getCallOpResultTypes(calleeType),
-        getCallOpVarCalleeType(calleeType), callee, ops, normalOps, unwindOps,
+        getCallOpVarCalleeType(calleeType), callee, ops,
+        /*arg_attrs=*/nullptr, /*res_attrs=*/nullptr, normalOps, unwindOps,
         nullptr, nullptr, {}, {}, normal, unwind);
 }
 
@@ -1636,14 +1660,16 @@ void InvokeOp::print(OpAsmPrinter &p) {
                           {getCalleeAttrName(), getOperandSegmentSizeAttr(),
                            getCConvAttrName(), getVarCalleeTypeAttrName(),
                            getOpBundleSizesAttrName(),
-                           getOpBundleTagsAttrName()});
+                           getOpBundleTagsAttrName(), getArgAttrsAttrName(),
+                           getResAttrsAttrName()});
 
   p << " : ";
   if (!isDirect)
     p << getOperand(0).getType() << ", ";
-  p.printFunctionalType(
-      llvm::drop_begin(getCalleeOperands().getTypes(), isDirect ? 0 : 1),
-      getResultTypes());
+  call_interface_impl::printFunctionSignature(
+      p, getCalleeOperands().drop_front(isDirect ? 0 : 1).getTypes(),
+      getArgAttrsAttr(),
+      /*isVariadic=*/false, getResultTypes(), getResAttrsAttr());
 }
 
 // <operation> ::= `llvm.invoke` (cconv)? (function-id | ssa-use)
@@ -1652,7 +1678,8 @@ void InvokeOp::print(OpAsmPrinter &p) {
 //                  `unwind` bb-id (`[` ssa-use-and-type-list `]`)?
 //                  ( `vararg(` var-callee-type `)` )?
 //                  ( `[` op-bundles-list `]` )?
-//                  attribute-dict? `:` (type `,`)? function-type
+//                  attribute-dict? `:` (type `,`)?
+//                  function-type-with-argument-attributes
 ParseResult InvokeOp::parse(OpAsmParser &parser, OperationState &result) {
   SmallVector<OpAsmParser::UnresolvedOperand, 8> operands;
   SymbolRefAttr funcAttr;
@@ -1714,8 +1741,15 @@ ParseResult InvokeOp::parse(OpAsmParser &parser, OperationState &result) {
     return failure();
 
   // Parse the trailing type list and resolve the function operands.
-  if (parseCallTypeAndResolveOperands(parser, result, isDirect, operands))
+  SmallVector<DictionaryAttr> argAttrs;
+  SmallVector<DictionaryAttr> resultAttrs;
+  if (parseCallTypeAndResolveOperands(parser, result, isDirect, operands,
+                                      argAttrs, resultAttrs))
     return failure();
+  call_interface_impl::addArgAndResultAttrs(
+      parser.getBuilder(), result, argAttrs, resultAttrs,
+      getArgAttrsAttrName(result.name), getResAttrsAttrName(result.name));
+
   if (resolveOpBundleOperands(parser, opBundlesLoc, result, opBundleOperands,
                               opBundleOperandTypes,
                               getOpBundleSizesAttrName(result.name)))
@@ -1891,6 +1925,13 @@ static Type getInsertExtractValueElementType(Type llvmType,
 }
 
 OpFoldResult LLVM::ExtractValueOp::fold(FoldAdaptor adaptor) {
+  if (auto extractValueOp = getContainer().getDefiningOp<ExtractValueOp>()) {
+    SmallVector<int64_t, 4> newPos(extractValueOp.getPosition());
+    newPos.append(getPosition().begin(), getPosition().end());
+    setPosition(newPos);
+    getContainerMutable().set(extractValueOp.getContainer());
+    return getResult();
+  }
   auto insertValueOp = getContainer().getDefiningOp<InsertValueOp>();
   OpFoldResult result = {};
   while (insertValueOp) {
@@ -2036,6 +2077,11 @@ LLVMFuncOp AddressOfOp::getFunction(SymbolTableCollection &symbolTable) {
       symbolTable.lookupSymbolIn(parentLLVMModule(*this), getGlobalNameAttr()));
 }
 
+AliasOp AddressOfOp::getAlias(SymbolTableCollection &symbolTable) {
+  return dyn_cast_or_null<AliasOp>(
+      symbolTable.lookupSymbolIn(parentLLVMModule(*this), getGlobalNameAttr()));
+}
+
 LogicalResult
 AddressOfOp::verifySymbolUses(SymbolTableCollection &symbolTable) {
   Operation *symbol =
@@ -2043,15 +2089,17 @@ AddressOfOp::verifySymbolUses(SymbolTableCollection &symbolTable) {
 
   auto global = dyn_cast_or_null<GlobalOp>(symbol);
   auto function = dyn_cast_or_null<LLVMFuncOp>(symbol);
+  auto alias = dyn_cast_or_null<AliasOp>(symbol);
 
-  if (!global && !function)
-    return emitOpError(
-        "must reference a global defined by 'llvm.mlir.global' or 'llvm.func'");
+  if (!global && !function && !alias)
+    return emitOpError("must reference a global defined by 'llvm.mlir.global', "
+                       "'llvm.mlir.alias' or 'llvm.func'");
 
   LLVMPointerType type = getType();
-  if (global && global.getAddrSpace() != type.getAddressSpace())
+  if ((global && global.getAddrSpace() != type.getAddressSpace()) ||
+      (alias && alias.getAddrSpace() != type.getAddressSpace()))
     return emitOpError("pointer address space must match address space of the "
-                       "referenced global");
+                       "referenced global or alias");
 
   return success();
 }
@@ -2188,38 +2236,50 @@ static LogicalResult verifyComdat(Operation *op,
   return success();
 }
 
-// operation ::= `llvm.mlir.global` linkage? visibility?
-//               (`unnamed_addr` | `local_unnamed_addr`)?
-//               `thread_local`? `constant`? `@` identifier
-//               `(` attribute? `)` (`comdat(` symbol-ref-id `)`)?
-//               attribute-list? (`:` type)? region?
-//
-// The type can be omitted for string attributes, in which case it will be
-// inferred from the value of the string as [strlen(value) x i8].
-ParseResult GlobalOp::parse(OpAsmParser &parser, OperationState &result) {
+/// Parse common attributes that might show up in the same order in both
+/// GlobalOp and AliasOp.
+template <typename OpType>
+static ParseResult parseCommonGlobalAndAlias(OpAsmParser &parser,
+                                             OperationState &result) {
   MLIRContext *ctx = parser.getContext();
   // Parse optional linkage, default to External.
-  result.addAttribute(getLinkageAttrName(result.name),
+  result.addAttribute(OpType::getLinkageAttrName(result.name),
                       LLVM::LinkageAttr::get(
                           ctx, parseOptionalLLVMKeyword<Linkage>(
                                    parser, result, LLVM::Linkage::External)));
 
   // Parse optional visibility, default to Default.
-  result.addAttribute(getVisibility_AttrName(result.name),
+  result.addAttribute(OpType::getVisibility_AttrName(result.name),
                       parser.getBuilder().getI64IntegerAttr(
                           parseOptionalLLVMKeyword<LLVM::Visibility, int64_t>(
                               parser, result, LLVM::Visibility::Default)));
 
   // Parse optional UnnamedAddr, default to None.
-  result.addAttribute(getUnnamedAddrAttrName(result.name),
+  result.addAttribute(OpType::getUnnamedAddrAttrName(result.name),
                       parser.getBuilder().getI64IntegerAttr(
                           parseOptionalLLVMKeyword<UnnamedAddr, int64_t>(
                               parser, result, LLVM::UnnamedAddr::None)));
 
   if (succeeded(parser.parseOptionalKeyword("thread_local")))
-    result.addAttribute(getThreadLocal_AttrName(result.name),
+    result.addAttribute(OpType::getThreadLocal_AttrName(result.name),
                         parser.getBuilder().getUnitAttr());
 
+  return success();
+}
+
+// operation ::= `llvm.mlir.global` linkage? visibility?
+//               (`unnamed_addr` | `local_unnamed_addr`)?
+//               `thread_local`? `constant`? `@` identifier
+//               `(` attribute? `)` (`comdat(` symbol-ref-id `)`)?
+//               attribute-list? (`:` type)? region?
+//
+// The type can be omitted for string attributes, in which case it will be
+// inferred from the value of the string as [strlen(value) x i8].
+ParseResult GlobalOp::parse(OpAsmParser &parser, OperationState &result) {
+  // Call into common parsing between GlobalOp and AliasOp.
+  if (parseCommonGlobalAndAlias<GlobalOp>(parser, result).failed())
+    return failure();
+
   if (succeeded(parser.parseOptionalKeyword("constant")))
     result.addAttribute(getConstantAttrName(result.name),
                         parser.getBuilder().getUnitAttr());
@@ -2422,6 +2482,145 @@ LogicalResult GlobalDtorsOp::verify() {
   return success();
 }
 
+//===----------------------------------------------------------------------===//
+// Builder, printer and verifier for LLVM::AliasOp.
+//===----------------------------------------------------------------------===//
+
+void AliasOp::build(OpBuilder &builder, OperationState &result, Type type,
+                    Linkage linkage, StringRef name, bool dsoLocal,
+                    bool threadLocal, ArrayRef<NamedAttribute> attrs) {
+  result.addAttribute(getSymNameAttrName(result.name),
+                      builder.getStringAttr(name));
+  result.addAttribute(getAliasTypeAttrName(result.name), TypeAttr::get(type));
+  if (dsoLocal)
+    result.addAttribute(getDsoLocalAttrName(result.name),
+                        builder.getUnitAttr());
+  if (threadLocal)
+    result.addAttribute(getThreadLocal_AttrName(result.name),
+                        builder.getUnitAttr());
+
+  result.addAttribute(getLinkageAttrName(result.name),
+                      LinkageAttr::get(builder.getContext(), linkage));
+  result.attributes.append(attrs.begin(), attrs.end());
+
+  result.addRegion();
+}
+
+void AliasOp::print(OpAsmPrinter &p) {
+  p << ' ' << stringifyLinkage(getLinkage()) << ' ';
+  StringRef visibility = stringifyVisibility(getVisibility_());
+  if (!visibility.empty())
+    p << visibility << ' ';
+
+  if (std::optional<mlir::LLVM::UnnamedAddr> unnamedAddr = getUnnamedAddr()) {
+    StringRef str = stringifyUnnamedAddr(*unnamedAddr);
+    if (!str.empty())
+      p << str << ' ';
+  }
+
+  if (getThreadLocal_())
+    p << "thread_local ";
+
+  p.printSymbolName(getSymName());
+  p.printOptionalAttrDict((*this)->getAttrs(),
+                          {SymbolTable::getSymbolAttrName(),
+                           getAliasTypeAttrName(), getLinkageAttrName(),
+                           getUnnamedAddrAttrName(), getThreadLocal_AttrName(),
+                           getVisibility_AttrName(), getUnnamedAddrAttrName()});
+
+  // Print the trailing type.
+  p << " : " << getType() << ' ';
+  // Print the initializer region.
+  p.printRegion(getInitializerRegion(), /*printEntryBlockArgs=*/false);
+}
+
+// operation ::= `llvm.mlir.alias` linkage? visibility?
+//               (`unnamed_addr` | `local_unnamed_addr`)?
+//               `thread_local`? `@` identifier
+//               `(` attribute? `)`
+//               attribute-list? `:` type region
+//
+ParseResult AliasOp::parse(OpAsmParser &parser, OperationState &result) {
+  // Call into common parsing between GlobalOp and AliasOp.
+  if (parseCommonGlobalAndAlias<AliasOp>(parser, result).failed())
+    return failure();
+
+  StringAttr name;
+  if (parser.parseSymbolName(name, getSymNameAttrName(result.name),
+                             result.attributes))
+    return failure();
+
+  SmallVector<Type, 1> types;
+  if (parser.parseOptionalAttrDict(result.attributes) ||
+      parser.parseOptionalColonTypeList(types))
+    return failure();
+
+  if (types.size() > 1)
+    return parser.emitError(parser.getNameLoc(), "expected zero or one type");
+
+  Region &initRegion = *result.addRegion();
+  if (parser.parseRegion(initRegion).failed())
+    return failure();
+
+  result.addAttribute(getAliasTypeAttrName(result.name),
+                      TypeAttr::get(types[0]));
+  return success();
+}
+
+LogicalResult AliasOp::verify() {
+  bool validType = isCompatibleOuterType(getType())
+                       ? !llvm::isa<LLVMVoidType, LLVMTokenType,
+                                    LLVMMetadataType, LLVMLabelType>(getType())
+                       : llvm::isa<PointerElementTypeInterface>(getType());
+  if (!validType)
+    return emitOpError(
+        "expects type to be a valid element type for an LLVM global alias");
+
+  // This matches LLVM IR verification logic, see llvm/lib/IR/Verifier.cpp
+  switch (getLinkage()) {
+  case Linkage::External:
+  case Linkage::Internal:
+  case Linkage::Private:
+  case Linkage::Weak:
+  case Linkage::Linkonce:
+  case Linkage::LinkonceODR:
+  case Linkage::AvailableExternally:
+    break;
+  default:
+    return emitOpError()
+           << "'" << stringifyLinkage(getLinkage())
+           << "' linkage not supported in aliases, available options: private, "
+              "internal, linkonce, weak, linkonce_odr, weak_odr, external or "
+              "available_externally";
+  }
+
+  return success();
+}
+
+LogicalResult AliasOp::verifyRegions() {
+  Block &b = getInitializerBlock();
+  auto ret = cast<ReturnOp>(b.getTerminator());
+  if (ret.getNumOperands() == 0 ||
+      !isa<LLVM::LLVMPointerType>(ret.getOperand(0).getType()))
+    return emitOpError("initializer region must always return a pointer");
+
+  for (Operation &op : b) {
+    auto iface = dyn_cast<MemoryEffectOpInterface>(op);
+    if (!iface || !iface.hasNoEffect())
+      return op.emitError()
+             << "ops with side effects are not allowed in alias initializers";
+  }
+
+  return success();
+}
+
+unsigned AliasOp::getAddrSpace() {
+  Block &initializer = getInitializerBlock();
+  auto ret = cast<ReturnOp>(initializer.getTerminator());
+  auto ptrTy = cast<LLVMPointerType>(ret.getOperand(0).getType());
+  return ptrTy.getAddressSpace();
+}
+
 //===----------------------------------------------------------------------===//
 // ShuffleVectorOp
 //===----------------------------------------------------------------------===//
@@ -2510,7 +2709,7 @@ void LLVMFuncOp::build(OpBuilder &builder, OperationState &result,
 
   assert(llvm::cast<LLVMFunctionType>(type).getNumParams() == argAttrs.size() &&
          "expected as many argument attribute lists as arguments");
-  function_interface_impl::addArgAndResultAttrs(
+  call_interface_impl::addArgAndResultAttrs(
       builder, result, argAttrs, /*resultAttrs=*/std::nullopt,
       getArgAttrsAttrName(result.name), getResAttrsAttrName(result.name));
 }
@@ -2595,7 +2794,7 @@ ParseResult LLVMFuncOp::parse(OpAsmParser &parser, OperationState &result) {
   auto signatureLocation = parser.getCurrentLocation();
   if (parser.parseSymbolName(nameAttr, SymbolTable::getSymbolAttrName(),
                              result.attributes) ||
-      function_interface_impl::parseFunctionSignature(
+      function_interface_impl::parseFunctionSignatureWithArguments(
           parser, /*allowVariadic=*/true, entryArgs, isVariadic, resultTypes,
           resultAttrs))
     return failure();
@@ -2636,7 +2835,7 @@ ParseResult LLVMFuncOp::parse(OpAsmParser &parser, OperationState &result) {
 
   if (failed(parser.parseOptionalAttrDictWithKeyword(result.attributes)))
     return failure();
-  function_interface_impl::addArgAndResultAttrs(
+  call_interface_impl::addArgAndResultAttrs(
       parser.getBuilder(), result, entryArgs, resultAttrs,
       getArgAttrsAttrName(result.name), getResAttrsAttrName(result.name));
 
diff --git a/mlir/lib/Dialect/LLVMIR/IR/NVVMDialect.cpp b/mlir/lib/Dialect/LLVMIR/IR/NVVMDialect.cpp
index dc7e724379ed0..62f0c21338111 100644
--- a/mlir/lib/Dialect/LLVMIR/IR/NVVMDialect.cpp
+++ b/mlir/lib/Dialect/LLVMIR/IR/NVVMDialect.cpp
@@ -147,9 +147,6 @@ LogicalResult CvtFloatToTF32Op::verify() {
     break;
   case RndMode::RN:
   case RndMode::RZ:
-    if (getSat() != NVVM::SaturationMode::NONE)
-      return emitError(
-          "Saturation mode not supported with rn/rz rounding modes.");
     break;
   default:
     return emitError(
@@ -935,7 +932,7 @@ LogicalResult NVVM::WgmmaMmaAsyncOp::verify() {
   // Check transpose (only available for f16/bf16)
   // Matrices A should be stored in row-major and B in column-major.
   // Only f16/bf16 matrices can be stored in either column-major or row-major
-  // by setting the tranpose value(imm-trans-a,imm-trans-b) in PTX code.
+  // by setting the transpose value(imm-trans-a,imm-trans-b) in PTX code.
   if ((typeA != WGMMATypes::f16 && typeA != WGMMATypes::bf16) &&
       (getLayoutA() == mlir::NVVM::MMALayout::col ||
        getLayoutB() == mlir::NVVM::MMALayout::row)) {
@@ -1221,26 +1218,102 @@ llvm::Intrinsic::ID CpAsyncBulkTensorReduceOp::getIntrinsicID(
   llvm_unreachable("Invalid Reduction Op for CpAsyncBulkTensorReduceOp");
 }
 
+#define CVT_F2TF32_ID_IMPL(rnd, relu, sf)                                      \
+  hasRelu ? llvm::Intrinsic::nvvm_f2tf32_##rnd##relu##sf                       \
+          : llvm::Intrinsic::nvvm_f2tf32_##rnd##sf
+
+#define GET_CVT_F2TF32_ID(rnd, relu, sf)                                       \
+  hasSatFinite ? CVT_F2TF32_ID_IMPL(rnd, relu, sf)                             \
+               : CVT_F2TF32_ID_IMPL(rnd, relu, )
+
 llvm::Intrinsic::ID CvtFloatToTF32Op::getIntrinsicID(NVVM::FPRoundingMode rnd,
                                                      NVVM::SaturationMode sat,
                                                      bool hasRelu) {
   using RndMode = NVVM::FPRoundingMode;
+  bool hasSatFinite = (sat == NVVM::SaturationMode::SATFINITE);
   switch (rnd) {
   case RndMode::RN:
-    return hasRelu ? llvm::Intrinsic::nvvm_f2tf32_rn_relu
-                   : llvm::Intrinsic::nvvm_f2tf32_rn;
+    return GET_CVT_F2TF32_ID(rn, _relu, _satfinite);
   case RndMode::RZ:
-    return hasRelu ? llvm::Intrinsic::nvvm_f2tf32_rz_relu
-                   : llvm::Intrinsic::nvvm_f2tf32_rz;
+    return GET_CVT_F2TF32_ID(rz, _relu, _satfinite);
   case RndMode::RNA:
-    return (sat == NVVM::SaturationMode::SATFINITE)
-               ? llvm::Intrinsic::nvvm_f2tf32_rna_satfinite
-               : llvm::Intrinsic::nvvm_f2tf32_rna;
+    return GET_CVT_F2TF32_ID(rna, , _satfinite);
   default:
     llvm_unreachable("Invalid RoundingMode for CvtFloatToTF32Op");
   }
 }
 
+llvm::Intrinsic::ID
+Tcgen05AllocOp::getIntrinsicIDAndArgs(Operation &op,
+                                      LLVM::ModuleTranslation &mt,
+                                      llvm::SmallVector<llvm::Value *> &args) {
+  auto curOp = cast<NVVM::Tcgen05AllocOp>(op);
+  unsigned AS = llvm::cast<LLVM::LLVMPointerType>(curOp.getAddr().getType())
+                    .getAddressSpace();
+  bool isShared = AS == NVVMMemorySpace::kSharedMemorySpace;
+  bool is2CTAMode = curOp.getGroup() == Tcgen05GroupKind::CTA_2;
+
+  llvm::Intrinsic::ID id;
+  if (isShared) {
+    id = is2CTAMode ? llvm::Intrinsic::nvvm_tcgen05_alloc_shared_cg2
+                    : llvm::Intrinsic::nvvm_tcgen05_alloc_shared_cg1;
+  } else {
+    id = is2CTAMode ? llvm::Intrinsic::nvvm_tcgen05_alloc_cg2
+                    : llvm::Intrinsic::nvvm_tcgen05_alloc_cg1;
+  }
+
+  // Fill the Intrinsic Args
+  args.push_back(mt.lookupValue(curOp.getAddr()));
+  args.push_back(mt.lookupValue(curOp.getNCols()));
+
+  return id;
+}
+
+llvm::Intrinsic::ID Tcgen05DeallocOp::getIntrinsicIDAndArgs(
+    Operation &op, LLVM::ModuleTranslation &mt,
+    llvm::SmallVector<llvm::Value *> &args) {
+  auto curOp = cast<NVVM::Tcgen05DeallocOp>(op);
+  auto id = (curOp.getGroup() == Tcgen05GroupKind::CTA_1)
+                ? llvm::Intrinsic::nvvm_tcgen05_dealloc_cg1
+                : llvm::Intrinsic::nvvm_tcgen05_dealloc_cg2;
+
+  // Fill the Intrinsic Args
+  args.push_back(mt.lookupValue(curOp.getTaddr()));
+  args.push_back(mt.lookupValue(curOp.getNCols()));
+
+  return id;
+}
+
+#define TCGEN05_COMMIT_IMPL(cg, is_shared, mc)                                 \
+  is_shared ? llvm::Intrinsic::nvvm_tcgen05_commit##mc##_shared##_##cg         \
+            : llvm::Intrinsic::nvvm_tcgen05_commit##mc##_##cg
+
+#define GET_TCGEN05_COMMIT_ID(cta_group, is_shared, has_mc)                    \
+  has_mc ? TCGEN05_COMMIT_IMPL(cta_group, is_shared, _mc)                      \
+         : TCGEN05_COMMIT_IMPL(cta_group, is_shared, )
+
+llvm::Intrinsic::ID
+Tcgen05CommitOp::getIntrinsicIDAndArgs(Operation &op,
+                                       LLVM::ModuleTranslation &mt,
+                                       llvm::SmallVector<llvm::Value *> &args) {
+  auto curOp = cast<NVVM::Tcgen05CommitOp>(op);
+  unsigned AS = llvm::cast<LLVM::LLVMPointerType>(curOp.getAddr().getType())
+                    .getAddressSpace();
+  bool isShared = AS == NVVMMemorySpace::kSharedMemorySpace;
+  bool hasMulticast = curOp.getMulticastMask() ? true : false;
+  bool is2CTAMode = curOp.getGroup() == Tcgen05GroupKind::CTA_2;
+
+  auto id = is2CTAMode ? GET_TCGEN05_COMMIT_ID(cg2, isShared, hasMulticast)
+                       : GET_TCGEN05_COMMIT_ID(cg1, isShared, hasMulticast);
+
+  // Fill the Intrinsic Args
+  args.push_back(mt.lookupValue(curOp.getAddr()));
+  if (hasMulticast)
+    args.push_back(mt.lookupValue(curOp.getMulticastMask()));
+
+  return id;
+}
+
 /// Infer the result ranges for the NVVM SpecialRangeableRegisterOp that might
 /// have ConstantRangeAttr.
 static void nvvmInferResultRanges(Operation *op, Value result,
diff --git a/mlir/lib/Dialect/Linalg/IR/LinalgInterfaces.cpp b/mlir/lib/Dialect/Linalg/IR/LinalgInterfaces.cpp
index caf9cdb3a3eb4..4185fcce393d5 100644
--- a/mlir/lib/Dialect/Linalg/IR/LinalgInterfaces.cpp
+++ b/mlir/lib/Dialect/Linalg/IR/LinalgInterfaces.cpp
@@ -147,7 +147,7 @@ linalg::isaBroadcastOpInterface(GenericOp op) {
 }
 
 //===----------------------------------------------------------------------===//
-// TranposeOpInterface implementation
+// TransposeOpInterface implementation
 //===----------------------------------------------------------------------===//
 std::optional<SmallVector<int64_t>>
 linalg::isaTransposeOpInterface(GenericOp op) {
diff --git a/mlir/lib/Dialect/Linalg/IR/LinalgOps.cpp b/mlir/lib/Dialect/Linalg/IR/LinalgOps.cpp
index c13b663dbf05b..d40cec02df633 100644
--- a/mlir/lib/Dialect/Linalg/IR/LinalgOps.cpp
+++ b/mlir/lib/Dialect/Linalg/IR/LinalgOps.cpp
@@ -121,14 +121,11 @@ using RegionBuilderFn = llvm::function_ref<void(ImplicitLocOpBuilder &, Block &,
 /// `regionBuilder`. The method is used by both named structured ops created by
 /// ods-gen and by manually defined C++ ops. It is called by both builders and
 /// parsers and creates a block with arguments corresponding to the elemental
-/// types of `inputTypes` and `outputTypes`. All output types are asserted to be
-/// ShapedType.
+/// types of `inputTypes` and `outputTypes`.
 static void fillStructuredOpRegion(OpBuilder &opBuilder, Region &region,
                                    TypeRange inputTypes, TypeRange outputTypes,
                                    ArrayRef<NamedAttribute> attrs,
                                    RegionBuilderFn regionBuilder) {
-  assert(llvm::all_of(outputTypes, llvm::IsaPred<ShapedType>));
-
   SmallVector<Type, 8> argTypes;
   SmallVector<Location, 8> argLocs;
   for (auto containers : {inputTypes, outputTypes}) {
@@ -203,6 +200,23 @@ static void buildMatmulOp(OpBuilder &b, OperationState &state,
                            attributes, regionBuilder);
 }
 
+static void buildBatchMatmulOp(OpBuilder &b, OperationState &state,
+                               std::optional<TypeRange> resultTensorTypes,
+                               ValueRange inputs, ValueRange outputs,
+                               ArrayRef<NamedAttribute> attributes,
+                               RegionBuilderFn regionBuilder,
+                               ArrayRef<AffineMap> indexingMaps) {
+  // Initialize indexingMaps attribute, for BatchMatmulOp.
+  SmallVector<Attribute, 4> indexingMapsAttrVal;
+  indexingMapsAttrVal =
+      llvm::map_to_vector(indexingMaps, [](AffineMap map) -> Attribute {
+        return AffineMapAttr::get(map);
+      });
+  state.addAttribute("indexing_maps", b.getArrayAttr(indexingMapsAttrVal));
+  return buildStructuredOp(b, state, resultTensorTypes, inputs, outputs,
+                           attributes, regionBuilder);
+}
+
 /// Common parsing used for both named structured ops created by ods-gen and by
 /// manually defined C++ ops. Does not handle regions.
 static ParseResult
@@ -3063,8 +3077,11 @@ LogicalResult WinogradInputTransformOp::verify() {
   int m = getM();
   int r = getR();
   int64_t tileSize = m + r - 1;
-  bool leftTransform = inputH != 1;
-  bool rightTransform = inputW != 1;
+
+  auto outputType = cast<ShapedType>(getOutput().getType());
+  ArrayRef<int64_t> outputShape = outputType.getShape();
+  bool leftTransform = outputShape[getOutputAlphaHDim()] != 1;
+  bool rightTransform = outputShape[getOutputAlphaWDim()] != 1;
 
   SmallVector<int64_t> expectedOutputShape(6, inputH);
   if (ShapedType::isDynamic(inputH)) {
@@ -3073,7 +3090,7 @@ LogicalResult WinogradInputTransformOp::verify() {
   } else {
     expectedOutputShape[getOutputAlphaHDim()] = leftTransform ? tileSize : 1;
     expectedOutputShape[getOutputTileHDim()] =
-        leftTransform ? (inputH - (r - 1)) / m : 1;
+        leftTransform ? (inputH - (r - 1)) / m : inputH;
   }
   if (ShapedType::isDynamic(inputW)) {
     expectedOutputShape[getOutputAlphaWDim()] = tileSize;
@@ -3081,13 +3098,11 @@ LogicalResult WinogradInputTransformOp::verify() {
   } else {
     expectedOutputShape[getOutputAlphaWDim()] = rightTransform ? tileSize : 1;
     expectedOutputShape[getOutputTileWDim()] =
-        rightTransform ? (inputW - (r - 1)) / m : 1;
+        rightTransform ? (inputW - (r - 1)) / m : inputW;
   }
   expectedOutputShape[getOutputNDim()] = inputShape[getInputNDim()];
   expectedOutputShape[getOutputCDim()] = inputShape[getInputCDim()];
 
-  auto outputType = cast<ShapedType>(getOutput().getType());
-  ArrayRef<int64_t> outputShape = outputType.getShape();
   if (failed(verifyCompatibleShape(expectedOutputShape, outputShape))) {
     return emitOpError("the output shape is not expected");
   }
@@ -3124,15 +3139,17 @@ LogicalResult WinogradInputTransformOp::getResultTilePosition(
     ArrayRef<OpFoldResult> sizes, SmallVector<OpFoldResult> &resultOffsets,
     SmallVector<OpFoldResult> &resultSizes) {
   IntegerAttr zeroAttr = builder.getI64IntegerAttr(0);
-  ShapedType inputType = getInputOperandType();
-  ArrayRef<int64_t> inputShape = inputType.getShape();
-  int64_t inputH = inputShape[getInputHDim()];
-  int64_t inputW = inputShape[getInputWDim()];
+  ShapedType outputType = getOutputOperandType();
+  ArrayRef<int64_t> outputShape = outputType.getShape();
+  int64_t outputAlphaH = outputShape[getOutputAlphaHDim()];
+  int64_t outputAlphaW = outputShape[getOutputAlphaWDim()];
+
   int64_t m = getM();
   int64_t r = getR();
   int64_t alpha = m + r - 1;
-  int64_t alphaH = inputH != 1 ? alpha : 1;
-  int64_t alphaW = inputW != 1 ? alpha : 1;
+  int64_t alphaH = outputAlphaH != 1 ? alpha : 1;
+  int64_t alphaW = outputAlphaW != 1 ? alpha : 1;
+
   IntegerAttr alphaHAttr = builder.getI64IntegerAttr(alphaH);
   IntegerAttr alphaWAttr = builder.getI64IntegerAttr(alphaW);
 
@@ -3157,22 +3174,26 @@ WinogradInputTransformOp::getTiledImplementation(OpBuilder &builder,
                                                  ArrayRef<OpFoldResult> offsets,
                                                  ArrayRef<OpFoldResult> sizes) {
   IntegerAttr oneAttr = builder.getI64IntegerAttr(1);
-  IntegerAttr zeroAttr = builder.getI64IntegerAttr(0);
-  ShapedType inputType = getInputOperandType();
-  ArrayRef<int64_t> inputShape = inputType.getShape();
-  int64_t inputH = inputShape[getInputHDim()];
-  int64_t inputW = inputShape[getInputWDim()];
   int64_t m = getM();
   int64_t r = getR();
 
+  ShapedType outputType = getOutputOperandType();
+  ArrayRef<int64_t> outputShape = outputType.getShape();
+  int64_t alphaH = outputShape[getOutputAlphaHDim()];
+  int64_t alphaW = outputShape[getOutputAlphaWDim()];
+
   Location loc = getLoc();
   MLIRContext *context = builder.getContext();
+  auto identityAffineMap =
+      AffineMap::get(1, 0, {builder.getAffineDimExpr(0)}, context);
   auto offsetAffineMap =
       AffineMap::get(1, 0, {builder.getAffineDimExpr(0) * m}, context);
   Value mappedOffsetH = affine::makeComposedAffineApply(
-      builder, loc, offsetAffineMap, offsets[getOutputTileHDim()]);
+      builder, loc, (alphaH != 1 ? offsetAffineMap : identityAffineMap),
+      offsets[getOutputTileHDim()]);
   Value mappedOffsetW = affine::makeComposedAffineApply(
-      builder, loc, offsetAffineMap, offsets[getOutputTileWDim()]);
+      builder, loc, (alphaW != 1 ? offsetAffineMap : identityAffineMap),
+      offsets[getOutputTileWDim()]);
   auto sizeAffineMap = AffineMap::get(
       1, 0, {builder.getAffineDimExpr(0) * m + (r - 1)}, context);
   Value mappedSizeH = affine::makeComposedAffineApply(
@@ -3183,16 +3204,14 @@ WinogradInputTransformOp::getTiledImplementation(OpBuilder &builder,
   SmallVector<Value> tiledOperands;
   SmallVector<OpFoldResult> sliceOffsets, sliceSizes;
 
-  OpFoldResult offsetH =
-      inputH != 1 ? OpFoldResult(mappedOffsetH) : OpFoldResult(zeroAttr);
-  OpFoldResult offsetW =
-      inputW != 1 ? OpFoldResult(mappedOffsetW) : OpFoldResult(zeroAttr);
+  OpFoldResult offsetH = OpFoldResult(mappedOffsetH);
+  OpFoldResult offsetW = OpFoldResult(mappedOffsetW);
   sliceOffsets.append(
       {offsets[getOutputNDim()], offsetH, offsetW, offsets[getOutputCDim()]});
   OpFoldResult sizeH =
-      inputH != 1 ? OpFoldResult(mappedSizeH) : OpFoldResult(oneAttr);
+      alphaH != 1 ? OpFoldResult(mappedSizeH) : OpFoldResult(oneAttr);
   OpFoldResult sizeW =
-      inputW != 1 ? OpFoldResult(mappedSizeW) : OpFoldResult(oneAttr);
+      alphaW != 1 ? OpFoldResult(mappedSizeW) : OpFoldResult(oneAttr);
   sliceSizes.append(
       {sizes[getOutputNDim()], sizeH, sizeW, sizes[getOutputCDim()]});
   int64_t inputRank = getInputOperandRank();
@@ -3300,28 +3319,29 @@ LogicalResult WinogradOutputTransformOp::getResultTilePosition(
 
   Location loc = getLoc();
   MLIRContext *context = builder.getContext();
+  auto identityAffineMap =
+      AffineMap::get(1, 0, {builder.getAffineDimExpr(0)}, context);
   auto affineMap =
       AffineMap::get(1, 0, {builder.getAffineDimExpr(0) * m}, context);
 
+  ShapedType valueType = getValueOperandType();
+  ArrayRef<int64_t> valueShape = valueType.getShape();
+  int64_t valueH = valueShape[0];
+  int64_t valueW = valueShape[1];
   Value mappedOffsetH = affine::makeComposedAffineApply(
-      builder, loc, affineMap, offsets[getValueTileHDim()]);
+      builder, loc, (valueH != 1 ? affineMap : identityAffineMap),
+      offsets[getValueTileHDim()]);
   Value mappedOffsetW = affine::makeComposedAffineApply(
-      builder, loc, affineMap, offsets[getValueTileWDim()]);
+      builder, loc, (valueW != 1 ? affineMap : identityAffineMap),
+      offsets[getValueTileWDim()]);
   Value mappedSizeH = affine::makeComposedAffineApply(
       builder, loc, affineMap, sizes[getValueTileHDim()]);
   Value mappedSizeW = affine::makeComposedAffineApply(
       builder, loc, affineMap, sizes[getValueTileWDim()]);
 
-  ShapedType valueType = getValueOperandType();
-  ArrayRef<int64_t> valueShape = valueType.getShape();
-  int64_t valueH = valueShape[0];
-  int64_t valueW = valueShape[1];
   IntegerAttr oneAttr = builder.getI64IntegerAttr(1);
-  IntegerAttr zeroAttr = builder.getI64IntegerAttr(0);
-  OpFoldResult offsetH =
-      valueH != 1 ? OpFoldResult(mappedOffsetH) : OpFoldResult(zeroAttr);
-  OpFoldResult offsetW =
-      valueW != 1 ? OpFoldResult(mappedOffsetW) : OpFoldResult(zeroAttr);
+  OpFoldResult offsetH = OpFoldResult(mappedOffsetH);
+  OpFoldResult offsetW = OpFoldResult(mappedOffsetW);
   OpFoldResult sizeH =
       valueH != 1 ? OpFoldResult(mappedSizeH) : OpFoldResult(oneAttr);
   OpFoldResult sizeW =
@@ -3406,19 +3426,22 @@ Operation *LinalgDialect::materializeConstant(OpBuilder &builder,
   return arith::ConstantOp::materialize(builder, value, type, loc);
 }
 
-/// Returns true if the result AffineExpr of the \p explicitMap is same as \p
-/// defaultMap.
-static bool isValidResultDimExprs(AffineMap explictMap, AffineMap defaultMap) {
-  auto explicitRange = explictMap.getResults();
-  auto defaultRange = defaultMap.getResults();
+// Returns true if the result expression of `subMap` are a subset of `fullMap`.
+static bool areResultExprsSubsetOf(AffineMap subMap, AffineMap fullMap) {
+  auto explicitRange = subMap.getResults();
+  auto defaultRange = fullMap.getResults();
   DenseSet<AffineExpr> explicitSet(explicitRange.begin(), explicitRange.end());
   DenseSet<AffineExpr> defaultSet(defaultRange.begin(), defaultRange.end());
   llvm::set_union(explicitSet, defaultSet);
   return explicitSet == defaultSet;
 }
 
-/// Returns true if the \p explictMap is broadcasted with respect to the
-/// \p defaultMap.
+/// Check if the user defined map is valid broadcast map. Here broadcast
+/// indexing maps are defined in context of corresponding default indexing maps
+/// for the given Op. This way the check becomes very simple i.e just check the
+/// number of result dims.
+/// Returns true if the explictMap is broadcasted with respect to the
+/// defaultMap.
 static bool isBroadcasted(AffineMap explictMap, AffineMap defaultMap) {
   return explictMap.getNumResults() < defaultMap.getNumResults();
 }
@@ -3435,11 +3458,10 @@ static LogicalResult verifyExtendedMatmulSemantic(MatmulOp matmulOp,
   auto opIndexingMap = opIndexingMaps[opIndex];
   auto defaultIndexingMap = defaultIndexingMaps[opIndex];
   // Check general validity of indexing map results.
-  if (!isValidResultDimExprs(opIndexingMap, defaultIndexingMap))
+  if (!areResultExprsSubsetOf(opIndexingMap, defaultIndexingMap))
     return matmulOp->emitOpError()
            << "Unexpected dim expression in map result.";
 
-  // Check if the requested broadcast is valid.
   if (isBroadcasted(opIndexingMap, defaultIndexingMap)) {
     if (!matmulOp.isValidLhsRhsBroadcastMap(opIndexingMap)) {
       return matmulOp->emitOpError()
@@ -3450,6 +3472,87 @@ static LogicalResult verifyExtendedMatmulSemantic(MatmulOp matmulOp,
   return success();
 }
 
+// Check general validity of input indexing map.
+static LogicalResult verifyInputMaps(BatchMatmulOp batchMatmulOp,
+                                     AffineMap opIndexingMap,
+                                     AffineMap defaultIndexingMap, bool isLHS) {
+  // Check the result dims are valid.
+  if (!areResultExprsSubsetOf(opIndexingMap, defaultIndexingMap))
+    return batchMatmulOp->emitOpError()
+           << "Unexpected result dim expression (outside the set of default "
+              "result dims).";
+
+  // Check for valid number of result dims of input maps.
+  if (opIndexingMap.getNumResults() > 3)
+    return batchMatmulOp->emitOpError()
+           << "no. of result dim expressions exceeds 3.";
+
+  auto hasValidBatchDim = [](AffineMap map) {
+    AffineExpr batchDim = map.getResult(0);
+    return batchDim.isFunctionOfDim(0);
+  };
+
+  // Check if the requested broadcast is valid.
+  if (isBroadcasted(opIndexingMap, defaultIndexingMap)) {
+    if (!batchMatmulOp.isValidLhsRhsBroadcastMap(opIndexingMap, isLHS))
+      return batchMatmulOp->emitOpError() << "Invalid broadcast requested.";
+  } else if (!hasValidBatchDim(opIndexingMap)) {
+    return batchMatmulOp->emitOpError()
+           << "Invalid batch dimension expression.";
+  }
+  return success();
+}
+
+/// This function checks if the given AffineMap for the output of a
+/// BatchMatmulOp has exactly 3 result dimensions and if the output map result
+/// dimensions are valid.
+static LogicalResult verifyOutputMap(BatchMatmulOp batchMatmulOp,
+                                     AffineMap opIndexingMap) {
+  if (opIndexingMap.getNumResults() != 3)
+    return batchMatmulOp->emitOpError()
+           << "expects 3 dims, but got (" << opIndexingMap.getNumResults()
+           << ").";
+
+  auto areValidOutputResultDim = [](AffineMap outputMap) {
+    return outputMap.getResult(0).isFunctionOfDim(0) &&
+           outputMap.getResult(1).isFunctionOfDim(1) &&
+           outputMap.getResult(2).isFunctionOfDim(2);
+  };
+
+  if (!areValidOutputResultDim(opIndexingMap))
+    return batchMatmulOp->emitOpError()
+           << "Invalid output map result dimension.";
+
+  return success();
+}
+
+/// Verifies the broadcast and transpose semantic specified by the explicit
+/// indexing map for the BatchMatmulOp op for each operand specified by opIndex.
+static LogicalResult
+verifyExtendedBatchMatmulSemantic(BatchMatmulOp batchMatmulOp,
+                                  unsigned opIndex) {
+  SmallVector<AffineMap, 3> opIndexingMaps =
+      batchMatmulOp.getIndexingMapsArray();
+  SmallVector<AffineMap, 3> defaultIndexingMaps =
+      batchMatmulOp.getDefaultIndexingMaps(batchMatmulOp->getContext());
+
+  if (opIndexingMaps.size() != 3)
+    return batchMatmulOp->emitOpError()
+           << "Indexing_map attribute must have 3 affine maps.";
+
+  auto opIndexingMap = opIndexingMaps[opIndex];
+  auto defaultIndexingMap = defaultIndexingMaps[opIndex];
+
+  if (opIndex == 2 && failed(verifyOutputMap(batchMatmulOp, opIndexingMap)))
+    return failure();
+
+  if (failed(verifyInputMaps(batchMatmulOp, opIndexingMap, defaultIndexingMap,
+                             opIndex == 0)))
+    return failure();
+
+  return success();
+}
+
 namespace mlir {
 namespace linalg {
 
@@ -3528,7 +3631,347 @@ bool MatmulOp::isValidLhsRhsBroadcastMap(AffineMap bcastMap) {
   return exp.isFunctionOfDim(bcastMap.getNumDims() - 1);
 }
 
+FailureOr<ArrayAttr> parseIndexingMapsAttr(OpAsmParser &parser) {
+  if (parser.parseOptionalKeyword("indexing_maps"))
+    return ArrayAttr{
+        nullptr}; // Success in case indexing_maps was not provided.
+
+  ArrayAttr arrayAttr;
+  if (parser.parseEqual() || parser.parseAttribute(arrayAttr))
+    return failure();
+
+  if (llvm::any_of(arrayAttr,
+                   [](auto elt) { return !dyn_cast<AffineMapAttr>(elt); }))
+    return parser.emitError(parser.getCurrentLocation())
+           << "element of indexing_maps array is not an affine_map";
+
+  return arrayAttr;
+}
+
 ParseResult MatmulOp::parse(OpAsmParser &parser, OperationState &result) {
+  FailureOr<ArrayAttr> indexingMapsAttr = parseIndexingMapsAttr(parser);
+  if (failed(indexingMapsAttr))
+    return failure();
+
+  if (*indexingMapsAttr == nullptr) {
+    auto indexingMapAttrs = llvm::map_to_vector(
+        MatmulOp::getDefaultIndexingMaps(parser.getContext()),
+        [](AffineMap map) -> Attribute { return AffineMapAttr::get(map); });
+    indexingMapsAttr = parser.getBuilder().getArrayAttr(indexingMapAttrs);
+  }
+
+  result.addAttribute("indexing_maps", *indexingMapsAttr);
+  return parseNamedStructuredOp(parser, result, MatmulOp::getNumRegionArgs(),
+                                MatmulOp::getRegionBuilder());
+}
+
+void MatmulOp::print(OpAsmPrinter &p) {
+  SmallVector<Attribute, 3> indexingMaps = llvm::map_to_vector(
+      MatmulOp::getDefaultIndexingMaps(getContext()),
+      [](AffineMap map) -> Attribute { return AffineMapAttr::get(map); });
+  if (!llvm::equal(getIndexingMaps(), indexingMaps)) {
+    p << " indexing_maps = [";
+    llvm::interleaveComma(getIndexingMaps(), p,
+                          [&](Attribute attr) { p.printAttribute(attr); });
+    p << "]";
+  }
+
+  SmallVector<StringRef, 3> elidedAttrs = {
+      "operandSegmentSizes", "linalg.memoized_indexing_maps", "indexing_maps"};
+  printNamedStructuredOp(p, getOperation(), getInputs(), getOutputs(),
+                         elidedAttrs);
+}
+
+/// Verify the user defined indexing maps.
+LogicalResult MatmulOp::verify() {
+  // Verification of pure matmul is handled by verifyStructuredOpInterface().
+  if (!hasUserDefinedMaps())
+    return success();
+
+  for (unsigned opIndex = 0; opIndex < 2; opIndex++) {
+    if (failed(verifyExtendedMatmulSemantic(*this, opIndex)))
+      return failure();
+  }
+  return success();
+}
+
+LogicalResult MatmulOp::fold(FoldAdaptor, SmallVectorImpl<OpFoldResult> &) {
+  return memref::foldMemRefCast(*this);
+}
+
+void MatmulOp::getEffects(
+    SmallVectorImpl<SideEffects::EffectInstance<MemoryEffects::Effect>>
+        &effects) {
+  if (hasPureTensorSemantics())
+    return;
+  getGenericEffectsImpl(effects, cast<LinalgOp>(getOperation()));
+}
+
+Speculation::Speculatability MatmulOp::getSpeculatability() {
+  return getGenericSpeculatabilityImpl(cast<LinalgOp>(getOperation()));
+}
+
+//===----------------------------------------------------------------------===//
+// ContractOp
+//===----------------------------------------------------------------------===//
+
+SmallVector<utils::IteratorType> ContractOp::getIteratorTypesArray() {
+  AffineMap outAffineMap = getIndexingMapsArray().pop_back_val();
+  // On well-formed IR, indexing_maps is non-empty, contained affine_maps'
+  // domains are all the same, and each implements a projected permutation.
+  // Each iteration space dim must occur for at least one operand and either
+  // takes part in a contraction/reduction or else has parallel iteration type.
+  // We have that a dim is a contraction/reduction dim if and only if the dim
+  // occurs for the output operand. We use this fact for fast inference:
+  // NB: In case we allow dims to occur solely for one input, the above still
+  //     holds: per the einsum semantics, these are reduction dims as well.
+  SmallVector<bool> dimsInOutput(outAffineMap.getNumDims(), false);
+  for (auto result : outAffineMap.getResults()) {
+    auto dimExpr = dyn_cast<AffineDimExpr>(result);
+    assert(dimExpr && "affine_map is a projected permutation");
+    dimsInOutput[dimExpr.getPosition()] = true;
+  }
+
+  SmallVector<utils::IteratorType> iteratorTypes;
+  for (auto dimOccursInOutput : dimsInOutput)
+    iteratorTypes.push_back(dimOccursInOutput ? utils::IteratorType::parallel
+                                              : utils::IteratorType::reduction);
+
+  return iteratorTypes;
+}
+
+unsigned ContractOp::getNumRegionArgs() { return 3; }
+
+/// Implement block region builder, which is called by 'fillStructuredOpRegion'.
+void ContractOp::regionBuilder(ImplicitLocOpBuilder &b, Block &block,
+                               ArrayRef<NamedAttribute> attrs) {
+  assert(block.getNumArguments() == 3 &&
+         "ContractOp regionBuilder expects 3 args");
+  RegionBuilderHelper helper(b, block);
+
+  TypeFn castSignedness = TypeFn::cast_signed;
+  auto castIter = llvm::find_if(attrs, [&](const NamedAttribute &attr) {
+    return attr.getName() == "cast";
+  });
+  if (castIter != attrs.end()) {
+    if (auto attr = llvm::dyn_cast<TypeFnAttr>(castIter->getValue()))
+      castSignedness = attr.getValue();
+  }
+
+  // TODO: Support fields with operators besides mult & add.
+  Type outType = block.getArgument(2).getType();
+  Value lhsAtOutType =
+      helper.buildTypeFn(castSignedness, outType, block.getArgument(0));
+  Value rhsAtOutType =
+      helper.buildTypeFn(castSignedness, outType, block.getArgument(1));
+  Value productAtOutType =
+      helper.buildBinaryFn(BinaryFn::mul, lhsAtOutType, rhsAtOutType);
+  Value result = helper.buildBinaryFn(BinaryFn::add, block.getArgument(2),
+                                      productAtOutType);
+  helper.yieldOutputs({result});
+}
+
+ParseResult ContractOp::parse(OpAsmParser &parser, OperationState &result) {
+  FailureOr<ArrayAttr> indexingMapsAttr = parseIndexingMapsAttr(parser);
+  if (failed(indexingMapsAttr) || *indexingMapsAttr == nullptr)
+    return parser.emitError(parser.getCurrentLocation(),
+                            "expected 'indexing_maps' attribute");
+  result.addAttribute("indexing_maps", *indexingMapsAttr);
+
+  return parseNamedStructuredOp(parser, result, getNumRegionArgs(),
+                                regionBuilder);
+}
+
+void ContractOp::print(OpAsmPrinter &p) {
+  p << " indexing_maps = [";
+  llvm::interleaveComma(getIndexingMaps(), p,
+                        [&](Attribute attr) { p.printAttribute(attr); });
+  p << "]";
+  printNamedStructuredOp(
+      p, getOperation(), getInputs(), getOutputs(),
+      /*elidedAttrs=*/{"indexing_maps", "operandSegmentSizes"});
+}
+
+LogicalResult ContractOp::verify() {
+  int iterationSpaceDims = -1;
+  // Map iter space dims to #occurrences in inputs' and output's affine_maps:
+  // e.g., inOccurrences[0] will hold #times that dim (with index) 0 is used to
+  // access an input operand (so occurrence count can be at most 2) and
+  // outOccurrences[1] will indicate whether dim 1 occurred in the output, etc.
+  SmallVector<size_t> inOccurrences;
+  SmallVector<size_t> outOccurrences;
+
+  // A helper so that for each operand's affine_map and type we check that ...
+  auto checkAffineMapAndType = [&](AffineMap affineMap, Type operandType,
+                                   bool isInput) -> LogicalResult {
+    // ... the affine_map is a projected permutation;
+    if (!affineMap.isProjectedPermutation())
+      return emitError("provided affine_map is not a projected permutation");
+
+    // ... the rank of the affine_map's results and corresponding type match;
+    if (auto shapedType = dyn_cast<ShapedType>(operandType)) {
+      if (affineMap.getNumResults() != shapedType.getRank())
+        return emitError("ranks of shaped operand and results of corresponding "
+                         "affine_map differ");
+    } else if (affineMap.getNumResults() != 0) {
+      return emitError("affine_map specifies shaped access while operand has "
+                       "non-shaped type");
+    }
+
+    // ... the rank of the affine_map's domain is the same as those seen prior;
+    if (iterationSpaceDims == -1) {
+      iterationSpaceDims = affineMap.getNumDims();
+      inOccurrences = SmallVector<size_t>(iterationSpaceDims, 0);
+      outOccurrences = SmallVector<size_t>(iterationSpaceDims, 0);
+    } else if (iterationSpaceDims != (int)affineMap.getNumDims()) {
+      return emitError("iteration spaces of provided affine_maps differ");
+    }
+
+    // ... update counts of dims used to access either an input or the output.
+    for (AffineExpr affineExpr : affineMap.getResults()) {
+      auto affineDimExpr = dyn_cast<AffineDimExpr>(affineExpr);
+      if (!affineDimExpr)
+        llvm_unreachable("affine_map is a projected permutation");
+
+      if (isInput)
+        inOccurrences[affineDimExpr.getPosition()] += 1;
+      else
+        outOccurrences[affineDimExpr.getPosition()] += 1;
+    }
+
+    return success();
+  };
+
+  for (auto &&[affineMap, operandType, isInput] :
+       llvm::zip(getIndexingMapsArray(), getOperandTypes(),
+                 SmallVector<bool>{true, true, false})) {
+    if (failed(checkAffineMapAndType(affineMap, operandType, isInput)))
+      return failure(); // NB: checkAffineMapAndType will emit relevant error.
+  }
+
+  bool hasContractingDim = false;
+  for (size_t dimIndex = 0; dimIndex < (size_t)iterationSpaceDims; dimIndex++) {
+    size_t inOccCount = inOccurrences[dimIndex];
+    size_t outOccCount = outOccurrences[dimIndex];
+
+    // We have a contracting dim if and only if ...
+    hasContractingDim |= inOccCount == 2 && outOccCount == 0;
+
+    if (inOccCount == 0 && outOccCount == 0)
+      return emitError() << "iteration space dim at index " << dimIndex
+                         << " not used to access any operand";
+
+    // NB: We disallow a dim which occurs for only one input operand and not
+    //     for the output. In terms of einsum semantics such dims have a
+    //     sensible meaning - namely an additional reduction per each such dim.
+    //     By contrast, the ContractionOpInterface does not know about this
+    //     iter type - cf. inferContractionDims' supported dim kinds. Similarly,
+    //     while vector.contract's verifier accepts dims of this kind many of
+    //     its lowerings give up on encountering these dims.
+    // TODO: Remove following once we have comprehensive support for input-only
+    //       reduction dims, at both the linalg- and vector-dialect levels.
+    if (inOccCount == 1 && outOccCount != 1)
+      return emitError()
+             << "iteration space dim at index " << dimIndex
+             << " is neither a contracting dim nor of parallel iteration type";
+  }
+
+  if (!hasContractingDim)
+    return emitError("'indexing_maps' do not specify a contracting dimension");
+
+  return success();
+}
+
+LogicalResult ContractOp::fold(FoldAdaptor, SmallVectorImpl<OpFoldResult> &) {
+  return memref::foldMemRefCast(*this);
+}
+
+void ContractOp::getEffects(
+    SmallVectorImpl<SideEffects::EffectInstance<MemoryEffects::Effect>>
+        &effects) {
+  if (hasPureTensorSemantics())
+    return;
+  getGenericEffectsImpl(effects, cast<LinalgOp>(getOperation()));
+}
+
+Speculation::Speculatability ContractOp::getSpeculatability() {
+  return getGenericSpeculatabilityImpl(cast<LinalgOp>(getOperation()));
+}
+
+//===----------------------------------------------------------------------===//
+// Implementation of BatchMatmulOp
+//===----------------------------------------------------------------------===//
+SmallVector<AffineMap>
+BatchMatmulOp::getDefaultIndexingMaps(MLIRContext *context) {
+  AffineExpr d0, d1, d2, d3;
+  SmallVector<AffineMap> indexingMaps;
+  bindDims(context, d0, d1, d2, d3);
+  indexingMaps.push_back(AffineMap::get(4, 0, {d0, d1, d3}, context));
+  indexingMaps.push_back(AffineMap::get(4, 0, {d0, d3, d2}, context));
+  indexingMaps.push_back(AffineMap::get(4, 0, {d0, d1, d2}, context));
+  return indexingMaps;
+}
+
+SmallVector<utils::IteratorType> BatchMatmulOp::getIteratorTypesArray() {
+  return SmallVector<utils::IteratorType>{
+      utils::IteratorType::parallel, utils::IteratorType::parallel,
+      utils::IteratorType::parallel, utils::IteratorType::reduction};
+}
+
+unsigned BatchMatmulOp::getNumRegionArgs() { return 3; }
+
+std::string BatchMatmulOp::getLibraryCallName() {
+  return generateLibraryCallName(getOperation());
+}
+
+/// Check if the op has broadcast and/or transpose semantic. Returns true if
+/// the user defined indexing maps are not equal to default map.
+bool BatchMatmulOp::hasUserDefinedMaps() {
+  SmallVector<AffineMap, 3> defaultMaps =
+      getDefaultIndexingMaps(this->getContext());
+  SmallVector<AffineMap, 3> explicitMaps = getIndexingMapsArray();
+  return defaultMaps != explicitMaps;
+}
+
+/// Returns true if the given broadcast map bcastMap is valid for this op.
+bool BatchMatmulOp::isValidLhsRhsBroadcastMap(AffineMap bcastMap, bool isLHS) {
+  assert(bcastMap.getNumResults() < 3 &&
+         "Expected less than 3 result dim expr.");
+  bool isValid = false;
+  enum Indices { batchPos, mPos, nPos, kPos };
+  if (bcastMap.getNumResults() == 1) {
+    AffineExpr exp = bcastMap.getResult(0);
+    isValid = exp.isFunctionOfDim(kPos);
+  } else if (bcastMap.getNumResults() == 2) {
+    AffineExpr exp0 = bcastMap.getResult(0);
+    AffineExpr exp1 = bcastMap.getResult(1);
+    isValid = isLHS
+                  ? (exp0.isFunctionOfDim(mPos) && exp1.isFunctionOfDim(kPos))
+                  : (exp0.isFunctionOfDim(kPos) && exp1.isFunctionOfDim(nPos));
+  }
+  return isValid;
+}
+
+void BatchMatmulOp::regionBuilder(ImplicitLocOpBuilder &b, Block &block,
+                                  ArrayRef<NamedAttribute> attrs) {
+  assert(block.getNumArguments() == 3 &&
+         "BatchMatmulOp regionBuilder expects 3 (>=0) args");
+  RegionBuilderHelper helper(b, block);
+  SmallVector<Value> yields;
+
+  auto toType = block.getArgument(2).getType();
+  Value castValA =
+      helper.buildTypeFn(TypeFn::cast_signed, toType, block.getArgument(0));
+  Value castValB =
+      helper.buildTypeFn(TypeFn::cast_signed, toType, block.getArgument(1));
+  Value mulVal = helper.buildBinaryFn(BinaryFn::mul, castValA, castValB);
+  Value addVal =
+      helper.buildBinaryFn(BinaryFn::add, block.getArgument(2), mulVal);
+  yields.push_back(addVal);
+  helper.yieldOutputs(yields);
+}
+
+ParseResult BatchMatmulOp::parse(OpAsmParser &parser, OperationState &result) {
   SmallVector<Attribute, 3> indexingMapsAttr;
   Attribute mapAttr;
   if (succeeded(parser.parseOptionalKeyword("indexing_maps"))) {
@@ -3557,23 +4000,25 @@ ParseResult MatmulOp::parse(OpAsmParser &parser, OperationState &result) {
   // Initialize indexingMaps, if not supplied explicitly.
   if (indexingMapsAttr.empty()) {
     indexingMapsAttr = llvm::map_to_vector(
-        MatmulOp::getDefaultIndexingMaps(parser.getContext()),
+        BatchMatmulOp::getDefaultIndexingMaps(parser.getContext()),
         [](AffineMap map) -> Attribute { return AffineMapAttr::get(map); });
   }
   result.addAttribute("indexing_maps",
                       parser.getBuilder().getArrayAttr(indexingMapsAttr));
 
-  return parseNamedStructuredOp(parser, result, MatmulOp::getNumRegionArgs(),
-                                MatmulOp::getRegionBuilder());
+  return ::parseNamedStructuredOp(parser, result,
+                                  BatchMatmulOp::getNumRegionArgs(),
+                                  BatchMatmulOp::getRegionBuilder());
 }
-void MatmulOp::print(OpAsmPrinter &p) {
+
+void BatchMatmulOp::print(OpAsmPrinter &p) {
   SmallVector<StringRef, 3> elidedAttrs = {
       "operandSegmentSizes", "linalg.memoized_indexing_maps", "indexing_maps"};
-  printNamedStructuredOp(p, getOperation(), getInputs(), getOutputs(),
-                         elidedAttrs);
+  ::printNamedStructuredOp(p, getOperation(), getInputs(), getOutputs(),
+                           elidedAttrs);
 
   SmallVector<Attribute, 3> indexingMaps = llvm::map_to_vector(
-      MatmulOp::getDefaultIndexingMaps(getContext()),
+      BatchMatmulOp::getDefaultIndexingMaps(getContext()),
       [](AffineMap map) -> Attribute { return AffineMapAttr::get(map); });
   if (!llvm::equal(getIndexingMaps(), indexingMaps)) {
     p << " indexing_maps = [";
@@ -3584,22 +4029,25 @@ void MatmulOp::print(OpAsmPrinter &p) {
 }
 
 /// Verify the user defined indexing maps.
-LogicalResult MatmulOp::verify() {
-  // Verification of pure matmul is handled by verifyStructuredOpInterface().
+LogicalResult BatchMatmulOp::verify() {
+  // Verification of pure batch_matmul is handled by
+  // verifyStructuredOpInterface().
   if (!hasUserDefinedMaps())
     return success();
 
-  for (unsigned opIndex = 0; opIndex < 2; opIndex++) {
-    if (failed(verifyExtendedMatmulSemantic(*this, opIndex)))
+  for (unsigned opIndex = 0; opIndex < 3; opIndex++) {
+    if (failed(verifyExtendedBatchMatmulSemantic(*this, opIndex)))
       return failure();
   }
   return success();
 }
 
-LogicalResult MatmulOp::fold(FoldAdaptor, SmallVectorImpl<OpFoldResult> &) {
+LogicalResult BatchMatmulOp::fold(FoldAdaptor,
+                                  SmallVectorImpl<OpFoldResult> &) {
   return memref::foldMemRefCast(*this);
 }
-void MatmulOp::getEffects(
+
+void BatchMatmulOp::getEffects(
     SmallVectorImpl<SideEffects::EffectInstance<MemoryEffects::Effect>>
         &effects) {
   if (hasPureTensorSemantics())
@@ -3607,7 +4055,7 @@ void MatmulOp::getEffects(
   getGenericEffectsImpl(effects, cast<LinalgOp>(getOperation()));
 }
 
-Speculation::Speculatability MatmulOp::getSpeculatability() {
+Speculation::Speculatability BatchMatmulOp::getSpeculatability() {
   return getGenericSpeculatabilityImpl(cast<LinalgOp>(getOperation()));
 }
 
diff --git a/mlir/lib/Dialect/Linalg/TransformOps/LinalgTransformOps.cpp b/mlir/lib/Dialect/Linalg/TransformOps/LinalgTransformOps.cpp
index 8f5b49e0c2130..51d1df52598c7 100644
--- a/mlir/lib/Dialect/Linalg/TransformOps/LinalgTransformOps.cpp
+++ b/mlir/lib/Dialect/Linalg/TransformOps/LinalgTransformOps.cpp
@@ -265,7 +265,6 @@ void transform::ApplyFoldAddIntoDestPatternsOp::populatePatterns(
 void transform::ApplyPadVectorizationPatternsOp::populatePatterns(
     RewritePatternSet &patterns) {
   linalg::populatePadOpVectorizationPatterns(patterns);
-  linalg::populateInsertSliceVectorizationPatterns(patterns);
 }
 
 //===----------------------------------------------------------------------===//
@@ -3504,9 +3503,6 @@ transform::VectorizeChildrenAndApplyPatternsOp::applyToOne(
 
   patterns.add<CopyVectorizationPattern>(ctx);
 
-  // Add misc. vectorization patterns (e.g. for tensor.insert_slice)
-  linalg::populateInsertSliceVectorizationPatterns(patterns);
-
   if (getVectorizePadding()) {
     linalg::populatePadOpVectorizationPatterns(patterns);
     // This creates an alternative path for lowering tensor.pad - by
diff --git a/mlir/lib/Dialect/Linalg/Transforms/BlockPackMatmul.cpp b/mlir/lib/Dialect/Linalg/Transforms/BlockPackMatmul.cpp
index ed1685a9cb9e6..7f9a0f7a6ca43 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/BlockPackMatmul.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/BlockPackMatmul.cpp
@@ -138,6 +138,16 @@ transposePackedMatmul(RewriterBase &rewriter, linalg::LinalgOp linalgOp,
 FailureOr<PackResult>
 linalg::blockPackMatmul(RewriterBase &rewriter, linalg::LinalgOp linalgOp,
                         const ControlBlockPackMatmulFn &controlPackMatmul) {
+  // Check to not let go the batch_matmul with extended semantic, through this
+  // transform.
+  if (auto *batchMatmulOp = dyn_cast<linalg::BatchMatmulOp>(&linalgOp)) {
+    if (batchMatmulOp->hasUserDefinedMaps()) {
+      return rewriter.notifyMatchFailure(
+          *batchMatmulOp,
+          "only batch_matmul ops with non-extended semantics are supported");
+    }
+  }
+
   if (linalgOp.hasPureBufferSemantics())
     return rewriter.notifyMatchFailure(linalgOp, "require tensor semantics");
 
diff --git a/mlir/lib/Dialect/Linalg/Transforms/DropUnitDims.cpp b/mlir/lib/Dialect/Linalg/Transforms/DropUnitDims.cpp
index 9b97865990bfd..bd4ffabfbb929 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/DropUnitDims.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/DropUnitDims.cpp
@@ -906,6 +906,10 @@ struct RankReduceContractionOps : OpRewritePattern<FromOpTy> {
 
   LogicalResult matchAndRewrite(FromOpTy contractionOp,
                                 PatternRewriter &rewriter) const override {
+    if (contractionOp.hasUserDefinedMaps()) {
+      return rewriter.notifyMatchFailure(
+          contractionOp, "ops with user-defined maps are not supported");
+    }
 
     auto loc = contractionOp.getLoc();
     auto inputs = contractionOp.getDpsInputs();
@@ -935,7 +939,8 @@ struct RankReduceContractionOps : OpRewritePattern<FromOpTy> {
         loc, collapsedResultTy, ValueRange{collapsedLhs, collapsedRhs},
         ValueRange{collapsedInit});
     for (auto attr : contractionOp->getAttrs()) {
-      if (attr.getName() == LinalgDialect::kMemoizedIndexingMapsAttrName)
+      if (attr.getName() == LinalgDialect::kMemoizedIndexingMapsAttrName ||
+          attr.getName() == "indexing_maps")
         continue;
       collapsedOp->setAttr(attr.getName(), attr.getValue());
     }
diff --git a/mlir/lib/Dialect/Linalg/Transforms/ElementwiseOpFusion.cpp b/mlir/lib/Dialect/Linalg/Transforms/ElementwiseOpFusion.cpp
index 3a57f368d4425..60cae77644291 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/ElementwiseOpFusion.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/ElementwiseOpFusion.cpp
@@ -13,6 +13,7 @@
 #include "mlir/Dialect/Linalg/Passes.h"
 
 #include "mlir/Dialect/Affine/IR/AffineOps.h"
+#include "mlir/Dialect/Arith/IR/Arith.h"
 #include "mlir/Dialect/Arith/Utils/Utils.h"
 #include "mlir/Dialect/Linalg/IR/Linalg.h"
 #include "mlir/Dialect/Linalg/Transforms/Transforms.h"
@@ -1572,9 +1573,9 @@ void generateCollapsedIndexingRegion(Location loc, Block *block,
         rewriter.create<linalg::IndexOp>(loc, foldedDims.index());
     for (auto dim : llvm::reverse(foldedDimsRef.drop_front())) {
       indexReplacementVals[dim] =
-          rewriter.create<arith::RemUIOp>(loc, newIndexVal, loopRange[dim]);
+          rewriter.create<arith::RemSIOp>(loc, newIndexVal, loopRange[dim]);
       newIndexVal =
-          rewriter.create<arith::DivUIOp>(loc, newIndexVal, loopRange[dim]);
+          rewriter.create<arith::DivSIOp>(loc, newIndexVal, loopRange[dim]);
     }
     indexReplacementVals[foldedDims.value().front()] = newIndexVal;
   }
diff --git a/mlir/lib/Dialect/Linalg/Transforms/EraseUnusedOperandsAndResults.cpp b/mlir/lib/Dialect/Linalg/Transforms/EraseUnusedOperandsAndResults.cpp
index 16ab45ea8bee6..d375878fb2c91 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/EraseUnusedOperandsAndResults.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/EraseUnusedOperandsAndResults.cpp
@@ -52,255 +52,265 @@ static bool isResultValueDead(linalg::GenericOp genericOp, OpResult result) {
   return true;
 }
 
-namespace {
-
-struct DeduplicateAndRemoveDeadOperandsAndResults
-    : public OpRewritePattern<GenericOp> {
-  DeduplicateAndRemoveDeadOperandsAndResults(MLIRContext *ctx,
-                                             bool removeOutputs)
-      : OpRewritePattern<GenericOp>(ctx), removeOutputs(removeOutputs) {}
-
-  LogicalResult matchAndRewrite(GenericOp genericOp,
-                                PatternRewriter &rewriter) const override {
-    // Create a map from argument position in the original op to the argument
-    // position in the new op. If the argument is dropped it wont have an entry.
-    SmallVector<OpOperand *> droppedOpOperands;
-
-    // Information needed to build the new op.
-    SmallVector<Value> newInputOperands, newOutputOperands;
-    SmallVector<AffineMap> newIndexingMaps;
-
-    // Gather information about duplicate input operands.
-    llvm::SmallDenseMap<unsigned, unsigned> origInsToNewInsPos =
-        deduplicateInputOperands(genericOp, droppedOpOperands, newInputOperands,
-                                 newIndexingMaps);
-
-    // Gather information about the dropped outputs.
-    llvm::SmallDenseMap<unsigned, unsigned> origOutsToNewOutsPos =
-        deduplicateOutputOperands(genericOp, droppedOpOperands,
-                                  newOutputOperands, newIndexingMaps);
-
-    // Check if there is any change to operands.
-    if (newInputOperands.size() + newOutputOperands.size() ==
-        genericOp->getNumOperands())
-      return failure();
-
-    // Create the new op with the body being empty.
-    Location loc = genericOp.getLoc();
-    SmallVector<Type> newResultTypes;
-    for (Value v : newOutputOperands)
-      if (isa<TensorType>(v.getType()))
-        newResultTypes.push_back(v.getType());
-    auto newOp = rewriter.create<GenericOp>(
-        loc, newResultTypes, newInputOperands, newOutputOperands,
-        rewriter.getAffineMapArrayAttr(newIndexingMaps),
-        genericOp.getIteratorTypes(), genericOp.getDocAttr(),
-        genericOp.getLibraryCallAttr(),
-        [](OpBuilder & /*builder*/, Location /*loc*/, ValueRange /*args*/) {
-          return;
-        });
-    // Copy over unknown attributes. They might be load bearing for some flow.
-    ArrayRef<StringRef> odsAttrs = genericOp.getAttributeNames();
-    for (NamedAttribute kv : genericOp->getAttrs())
-      if (!llvm::is_contained(odsAttrs, kv.getName().getValue()))
-        newOp->setAttr(kv.getName(), kv.getValue());
-
-    // Fix up the payload of the canonicalized operation.
-    populateOpPayload(genericOp, newOp, origInsToNewInsPos,
-                      origOutsToNewOutsPos, rewriter);
-
-    // Replace all live uses of the op.
-    SmallVector<Value> replacementsVals(genericOp->getNumResults(), nullptr);
-    for (const auto &result : llvm::enumerate(genericOp.getResults())) {
-      auto it = origOutsToNewOutsPos.find(result.index());
-      if (it == origOutsToNewOutsPos.end())
+//===---------------------------------------------------------------------===//
+// Helper methods for operand deduplication and dead results elimination
+//===---------------------------------------------------------------------===//
+
+// Deduplicate input operands, and return the
+// - Mapping from operand position in the original op, to operand position in
+// the canonicalized op.
+// - The preserved input operands list (by reference).
+llvm::SmallDenseMap<unsigned, unsigned> static deduplicateInputOperands(
+    GenericOp genericOp, SmallVector<OpOperand *> &droppedOpOperands,
+    SmallVector<Value> &newInputOperands,
+    SmallVector<AffineMap> &newIndexingMaps) {
+  llvm::SmallDenseMap<unsigned, unsigned> origToNewPos;
+  llvm::SmallDenseMap<std::pair<Value, AffineMap>, unsigned> dedupedInputs;
+  for (const auto &en : llvm::enumerate(genericOp.getDpsInputOperands())) {
+    OpOperand *inputOpOperand = en.value();
+    // Check if operand is dead and if dropping the indexing map makes the
+    // loops to shape computation invalid.
+    if (!genericOp.payloadUsesValueFromOperand(inputOpOperand)) {
+      // Add the current operands to the list of potentially droppable
+      // operands. If it cannot be dropped, this needs to be popped back.
+      droppedOpOperands.push_back(inputOpOperand);
+      if (genericOp.canOpOperandsBeDropped(droppedOpOperands))
         continue;
-      replacementsVals[result.index()] = newOp.getResult(it->second);
+      droppedOpOperands.pop_back();
     }
-    rewriter.replaceOp(genericOp, replacementsVals);
-    return success();
-  }
 
-private:
-  /// If unset, outputs are not modified by this pattern.
-  bool removeOutputs;
-
-  // Deduplicate input operands, and return the
-  // - Mapping from operand position in the original op, to operand position in
-  // the canonicalized op.
-  // - The preserved input operands list (by reference).
-  llvm::SmallDenseMap<unsigned, unsigned>
-  deduplicateInputOperands(GenericOp genericOp,
-                           SmallVector<OpOperand *> &droppedOpOperands,
-                           SmallVector<Value> &newInputOperands,
-                           SmallVector<AffineMap> &newIndexingMaps) const {
-    llvm::SmallDenseMap<unsigned, unsigned> origToNewPos;
-    llvm::SmallDenseMap<std::pair<Value, AffineMap>, unsigned> dedupedInputs;
-    for (const auto &en : llvm::enumerate(genericOp.getDpsInputOperands())) {
-      OpOperand *inputOpOperand = en.value();
-      // Check if operand is dead and if dropping the indexing map makes the
-      // loops to shape computation invalid.
-      if (!genericOp.payloadUsesValueFromOperand(inputOpOperand)) {
-        // Add the current operands to the list of potentially droppable
-        // operands. If it cannot be dropped, this needs to be popped back.
-        droppedOpOperands.push_back(inputOpOperand);
-        if (genericOp.canOpOperandsBeDropped(droppedOpOperands))
-          continue;
-        droppedOpOperands.pop_back();
-      }
+    // Check if this operand is a duplicate.
+    AffineMap indexingMap = genericOp.getMatchingIndexingMap(inputOpOperand);
+    auto it =
+        dedupedInputs.find(std::make_pair(inputOpOperand->get(), indexingMap));
+    if (it != dedupedInputs.end()) {
+      origToNewPos[en.index()] = it->second;
+      droppedOpOperands.push_back(inputOpOperand);
+      continue;
+    }
 
-      // Check if this operand is a duplicate.
-      AffineMap indexingMap = genericOp.getMatchingIndexingMap(inputOpOperand);
-      auto it = dedupedInputs.find(
-          std::make_pair(inputOpOperand->get(), indexingMap));
-      if (it != dedupedInputs.end()) {
-        origToNewPos[en.index()] = it->second;
-        droppedOpOperands.push_back(inputOpOperand);
-        continue;
-      }
+    // This is a preserved argument.
+    origToNewPos[en.index()] = newInputOperands.size();
+    dedupedInputs[{inputOpOperand->get(), indexingMap}] =
+        newInputOperands.size();
+    newInputOperands.push_back(inputOpOperand->get());
+    newIndexingMaps.push_back(indexingMap);
+  }
+  return origToNewPos;
+}
 
-      // This is a preserved argument.
-      origToNewPos[en.index()] = newInputOperands.size();
-      dedupedInputs[{inputOpOperand->get(), indexingMap}] =
-          newInputOperands.size();
-      newInputOperands.push_back(inputOpOperand->get());
-      newIndexingMaps.push_back(indexingMap);
+// Deduplicate output operands, and return the
+// - Mapping from operand position in the original op, to operand position in
+// the canonicalized op.
+// - The preserved output operands list (by reference).
+llvm::SmallDenseMap<unsigned, unsigned> static deduplicateOutputOperands(
+    GenericOp genericOp, SmallVector<OpOperand *> &droppedOpOperands,
+    SmallVector<Value> &newOutputOperands,
+    SmallVector<AffineMap> &newIndexingMaps, bool removeOutputs) {
+  llvm::SmallDenseMap<unsigned, unsigned> origToNewPos;
+  llvm::SmallDenseMap<std::tuple<Value, AffineMap, Value>, unsigned>
+      dedupedOutpts;
+  // If the op doesn't have tensor semantics or outputs should not be removed,
+  // keep all the outputs as preserved.
+  if (!genericOp.hasPureTensorSemantics() || !removeOutputs) {
+    for (const auto &en : llvm::enumerate(genericOp.getDpsInitsMutable())) {
+      origToNewPos[en.index()] = newOutputOperands.size();
+      newOutputOperands.push_back(en.value().get());
+      newIndexingMaps.push_back(genericOp.getMatchingIndexingMap(&en.value()));
     }
     return origToNewPos;
   }
-
-  // Deduplicate output operands, and return the
-  // - Mapping from operand position in the original op, to operand position in
-  // the canonicalized op.
-  // - The preserved output operands list (by reference).
-  llvm::SmallDenseMap<unsigned, unsigned>
-  deduplicateOutputOperands(GenericOp genericOp,
-                            SmallVector<OpOperand *> &droppedOpOperands,
-                            SmallVector<Value> &newOutputOperands,
-                            SmallVector<AffineMap> &newIndexingMaps) const {
-    llvm::SmallDenseMap<unsigned, unsigned> origToNewPos;
-    llvm::SmallDenseMap<std::tuple<Value, AffineMap, Value>, unsigned>
-        dedupedOutpts;
-    // If the op doesn't have tensor semantics or outputs should not be removed,
-    // keep all the outputs as preserved.
-    if (!genericOp.hasPureTensorSemantics() || !removeOutputs) {
-      for (const auto &en : llvm::enumerate(genericOp.getDpsInitsMutable())) {
-        origToNewPos[en.index()] = newOutputOperands.size();
-        newOutputOperands.push_back(en.value().get());
-        newIndexingMaps.push_back(
-            genericOp.getMatchingIndexingMap(&en.value()));
+  // Output argument can be dropped if the result has
+  // - no users, and
+  // - it is not used in the payload, and
+  // - the corresponding indexing maps are not needed for loop bound
+  //   computation.
+  auto yieldOp = cast<YieldOp>(genericOp.getBody()->getTerminator());
+  for (const auto &outputOpOperand :
+       llvm::enumerate(genericOp.getDpsInitsMutable())) {
+    OpResult result = genericOp.getTiedOpResult(&outputOpOperand.value());
+    AffineMap indexingMap =
+        genericOp.getMatchingIndexingMap(&outputOpOperand.value());
+    auto key = std::make_tuple(outputOpOperand.value().get(), indexingMap,
+                               yieldOp->getOperand(outputOpOperand.index()));
+    if (isResultValueDead(genericOp, result)) {
+      // Check if the opoperand can be dropped without affecting loop
+      // bound computation. Add the operand to the list of dropped op
+      // operand for checking. If it cannot be dropped, need to pop the
+      // value back.
+      droppedOpOperands.push_back(&outputOpOperand.value());
+      if (genericOp.canOpOperandsBeDropped(droppedOpOperands)) {
+        continue;
       }
-      return origToNewPos;
+      droppedOpOperands.pop_back();
     }
-    // Output argument can be dropped if the result has
-    // - no users, and
-    // - it is not used in the payload, and
-    // - the corresponding indexing maps are not needed for loop bound
-    //   computation.
-    auto yieldOp = cast<YieldOp>(genericOp.getBody()->getTerminator());
-    for (const auto &outputOpOperand :
-         llvm::enumerate(genericOp.getDpsInitsMutable())) {
-      OpResult result = genericOp.getTiedOpResult(&outputOpOperand.value());
-      AffineMap indexingMap =
-          genericOp.getMatchingIndexingMap(&outputOpOperand.value());
-      auto key = std::make_tuple(outputOpOperand.value().get(), indexingMap,
-                                 yieldOp->getOperand(outputOpOperand.index()));
-      if (isResultValueDead(genericOp, result)) {
-        // Check if the opoperand can be dropped without affecting loop
-        // bound computation. Add the operand to the list of dropped op
-        // operand for checking. If it cannot be dropped, need to pop the
-        // value back.
+
+    if (!genericOp.payloadUsesValueFromOperand(&outputOpOperand.value())) {
+      // The out operand can also be dropped if it is computed redundantly
+      // by another result, the conditions for that are
+      // - The same operand is used as the out operand
+      // - The same indexing map is used
+      // - The same yield value is used.
+      auto it = dedupedOutpts.find(key);
+      if (it != dedupedOutpts.end()) {
+        origToNewPos[outputOpOperand.index()] = it->second;
         droppedOpOperands.push_back(&outputOpOperand.value());
-        if (genericOp.canOpOperandsBeDropped(droppedOpOperands)) {
-          continue;
-        }
-        droppedOpOperands.pop_back();
+        continue;
       }
+    }
 
-      if (!genericOp.payloadUsesValueFromOperand(&outputOpOperand.value())) {
-        // The out operand can also be dropped if it is computed redundantly
-        // by another result, the conditions for that are
-        // - The same operand is used as the out operand
-        // - The same indexing map is used
-        // - The same yield value is used.
-        auto it = dedupedOutpts.find(key);
-        if (it != dedupedOutpts.end()) {
-          origToNewPos[outputOpOperand.index()] = it->second;
-          droppedOpOperands.push_back(&outputOpOperand.value());
-          continue;
-        }
-      }
+    origToNewPos[outputOpOperand.index()] = newOutputOperands.size();
+    dedupedOutpts[key] = newOutputOperands.size();
+    newOutputOperands.push_back(outputOpOperand.value().get());
+    newIndexingMaps.push_back(
+        genericOp.getMatchingIndexingMap(&outputOpOperand.value()));
+  }
+  return origToNewPos;
+}
 
-      origToNewPos[outputOpOperand.index()] = newOutputOperands.size();
-      dedupedOutpts[key] = newOutputOperands.size();
-      newOutputOperands.push_back(outputOpOperand.value().get());
-      newIndexingMaps.push_back(
-          genericOp.getMatchingIndexingMap(&outputOpOperand.value()));
+// Populate the body of the canonicalized operation.
+static void populateOpPayload(
+    GenericOp genericOp, GenericOp newOp,
+    const llvm::SmallDenseMap<unsigned, unsigned> &origInsToNewInsPos,
+    const llvm::SmallDenseMap<unsigned, unsigned> &origOutsToNewOutsPos,
+    RewriterBase &rewriter) {
+  // Merge the body of the original op with the new op.
+  Block *newOpBlock = &newOp.getRegion().front();
+  assert(newOpBlock->empty() && "expected new op to have an empty payload");
+  Block *origOpBlock = &genericOp.getRegion().front();
+  SmallVector<Value> replacements(origOpBlock->getNumArguments(), nullptr);
+
+  // Replace all arguments in the original op, with arguments from the
+  // canonicalized op.
+  auto updateReplacements =
+      [&](SmallVector<OpOperand *> &origOperands,
+          SmallVector<OpOperand *> &newOperands,
+          const llvm::SmallDenseMap<unsigned, unsigned> &map) {
+        for (const auto &origOperand : llvm::enumerate(origOperands)) {
+          auto it = map.find(origOperand.index());
+          if (it == map.end())
+            continue;
+          OpOperand *newOperand = newOperands[it->second];
+          replacements[origOperand.value()->getOperandNumber()] =
+              newOpBlock->getArgument(newOperand->getOperandNumber());
+        }
+      };
+
+  SmallVector<OpOperand *> origInputOperands = genericOp.getDpsInputOperands();
+  SmallVector<OpOperand *> newInputOperands = newOp.getDpsInputOperands();
+  updateReplacements(origInputOperands, newInputOperands, origInsToNewInsPos);
+
+  SmallVector<OpOperand *> origOutputOperands = llvm::to_vector(llvm::map_range(
+      genericOp.getDpsInitsMutable(), [](OpOperand &o) { return &o; }));
+  SmallVector<OpOperand *> newOutputOperands = llvm::to_vector(llvm::map_range(
+      newOp.getDpsInitsMutable(), [](OpOperand &o) { return &o; }));
+  updateReplacements(origOutputOperands, newOutputOperands,
+                     origOutsToNewOutsPos);
+
+  // Drop the unused yield args.
+  if (newOp.getNumDpsInits() != genericOp.getNumDpsInits()) {
+    OpBuilder::InsertionGuard g(rewriter);
+    YieldOp origYieldOp = cast<YieldOp>(origOpBlock->getTerminator());
+    rewriter.setInsertionPoint(origYieldOp);
+
+    SmallVector<Value> newYieldVals(newOp.getNumDpsInits(), nullptr);
+    for (const auto &yieldOpOperands :
+         llvm::enumerate(origYieldOp.getValues())) {
+      auto it = origOutsToNewOutsPos.find(yieldOpOperands.index());
+      if (it == origOutsToNewOutsPos.end())
+        continue;
+      newYieldVals[it->second] = yieldOpOperands.value();
     }
-    return origToNewPos;
+    rewriter.replaceOpWithNewOp<YieldOp>(origYieldOp, newYieldVals);
   }
 
-  // Populate the body of the canonicalized operation.
-  void populateOpPayload(
-      GenericOp genericOp, GenericOp newOp,
-      const llvm::SmallDenseMap<unsigned, unsigned> &origInsToNewInsPos,
-      const llvm::SmallDenseMap<unsigned, unsigned> &origOutsToNewOutsPos,
-      PatternRewriter &rewriter) const {
-    // Merge the body of the original op with the new op.
-    Block *newOpBlock = &newOp.getRegion().front();
-    assert(newOpBlock->empty() && "expected new op to have an empty payload");
-    Block *origOpBlock = &genericOp.getRegion().front();
-    SmallVector<Value> replacements(origOpBlock->getNumArguments(), nullptr);
-
-    // Replace all arguments in the original op, with arguments from the
-    // canonicalized op.
-    auto updateReplacements =
-        [&](SmallVector<OpOperand *> &origOperands,
-            SmallVector<OpOperand *> &newOperands,
-            const llvm::SmallDenseMap<unsigned, unsigned> &map) {
-          for (const auto &origOperand : llvm::enumerate(origOperands)) {
-            auto it = map.find(origOperand.index());
-            if (it == map.end())
-              continue;
-            OpOperand *newOperand = newOperands[it->second];
-            replacements[origOperand.value()->getOperandNumber()] =
-                newOpBlock->getArgument(newOperand->getOperandNumber());
-          }
-        };
-
-    SmallVector<OpOperand *> origInputOperands =
-        genericOp.getDpsInputOperands();
-    SmallVector<OpOperand *> newInputOperands = newOp.getDpsInputOperands();
-    updateReplacements(origInputOperands, newInputOperands, origInsToNewInsPos);
-
-    SmallVector<OpOperand *> origOutputOperands =
-        llvm::to_vector(llvm::map_range(genericOp.getDpsInitsMutable(),
-                                        [](OpOperand &o) { return &o; }));
-    SmallVector<OpOperand *> newOutputOperands =
-        llvm::to_vector(llvm::map_range(newOp.getDpsInitsMutable(),
-                                        [](OpOperand &o) { return &o; }));
-    updateReplacements(origOutputOperands, newOutputOperands,
-                       origOutsToNewOutsPos);
-
-    // Drop the unused yield args.
-    if (newOp.getNumDpsInits() != genericOp.getNumDpsInits()) {
-      OpBuilder::InsertionGuard g(rewriter);
-      YieldOp origYieldOp = cast<YieldOp>(origOpBlock->getTerminator());
-      rewriter.setInsertionPoint(origYieldOp);
-
-      SmallVector<Value> newYieldVals(newOp.getNumDpsInits(), nullptr);
-      for (const auto &yieldOpOperands :
-           llvm::enumerate(origYieldOp.getValues())) {
-        auto it = origOutsToNewOutsPos.find(yieldOpOperands.index());
-        if (it == origOutsToNewOutsPos.end())
-          continue;
-        newYieldVals[it->second] = yieldOpOperands.value();
-      }
-      rewriter.replaceOpWithNewOp<YieldOp>(origYieldOp, newYieldVals);
-    }
+  rewriter.mergeBlocks(origOpBlock, newOpBlock, replacements);
+}
 
-    rewriter.mergeBlocks(origOpBlock, newOpBlock, replacements);
+FailureOr<linalg::GenericOp>
+mlir::linalg::deduplicateOperandsAndRemoveDeadResults(
+    RewriterBase &rewriter, linalg::GenericOp genericOp, bool removeOutputs) {
+  // Create a map from argument position in the original op to the argument
+  // position in the new op. If the argument is dropped it wont have an entry.
+  SmallVector<OpOperand *> droppedOpOperands;
+
+  // Information needed to build the new op.
+  SmallVector<Value> newInputOperands, newOutputOperands;
+  SmallVector<AffineMap> newIndexingMaps;
+
+  // Gather information about duplicate input operands.
+  llvm::SmallDenseMap<unsigned, unsigned> origInsToNewInsPos =
+      deduplicateInputOperands(genericOp, droppedOpOperands, newInputOperands,
+                               newIndexingMaps);
+
+  // Gather information about the dropped outputs.
+  llvm::SmallDenseMap<unsigned, unsigned> origOutsToNewOutsPos =
+      deduplicateOutputOperands(genericOp, droppedOpOperands, newOutputOperands,
+                                newIndexingMaps, removeOutputs);
+
+  // Check if there is any change to operands.
+  if (newInputOperands.size() + newOutputOperands.size() ==
+      genericOp->getNumOperands())
+    return genericOp;
+
+  // Create the new op with the body being empty.
+  Location loc = genericOp.getLoc();
+  SmallVector<Type> newResultTypes;
+  for (Value v : newOutputOperands)
+    if (isa<TensorType>(v.getType()))
+      newResultTypes.push_back(v.getType());
+  auto newOp = rewriter.create<GenericOp>(
+      loc, newResultTypes, newInputOperands, newOutputOperands,
+      rewriter.getAffineMapArrayAttr(newIndexingMaps),
+      genericOp.getIteratorTypes(), genericOp.getDocAttr(),
+      genericOp.getLibraryCallAttr(),
+      [](OpBuilder & /*builder*/, Location /*loc*/, ValueRange /*args*/) {
+        return;
+      });
+  // Copy over unknown attributes. They might be load bearing for some flow.
+  ArrayRef<StringRef> odsAttrs = genericOp.getAttributeNames();
+  for (NamedAttribute kv : genericOp->getAttrs())
+    if (!llvm::is_contained(odsAttrs, kv.getName().getValue()))
+      newOp->setAttr(kv.getName(), kv.getValue());
+
+  // Fix up the payload of the canonicalized operation.
+  populateOpPayload(genericOp, newOp, origInsToNewInsPos, origOutsToNewOutsPos,
+                    rewriter);
+
+  // Replace all live uses of the op.
+  SmallVector<Value> replacementsVals(genericOp->getNumResults(), nullptr);
+  for (const auto &result : llvm::enumerate(genericOp.getResults())) {
+    auto it = origOutsToNewOutsPos.find(result.index());
+    if (it == origOutsToNewOutsPos.end())
+      continue;
+    replacementsVals[result.index()] = newOp.getResult(it->second);
   }
+  rewriter.replaceOp(genericOp, replacementsVals);
+  return newOp;
+}
+
+namespace {
+
+struct DeduplicateAndRemoveDeadOperandsAndResults
+    : public OpRewritePattern<GenericOp> {
+  DeduplicateAndRemoveDeadOperandsAndResults(MLIRContext *ctx,
+                                             bool removeOutputs)
+      : OpRewritePattern<GenericOp>(ctx), removeOutputs(removeOutputs) {}
+
+  LogicalResult matchAndRewrite(GenericOp genericOp,
+                                PatternRewriter &rewriter) const override {
+    FailureOr<GenericOp> newOp = deduplicateOperandsAndRemoveDeadResults(
+        rewriter, genericOp, removeOutputs);
+    if (failed(newOp) || newOp.value() == genericOp) {
+      return rewriter.notifyMatchFailure(
+          genericOp, "failed to dedup operands/remove dead results");
+    }
+    return success();
+  }
+
+private:
+  /// If unset, outputs are not modified by this pattern.
+  bool removeOutputs;
 };
 
 /// Remove unused cycles.
diff --git a/mlir/lib/Dialect/Linalg/Transforms/TransposeConv2D.cpp b/mlir/lib/Dialect/Linalg/Transforms/TransposeConv2D.cpp
index bdaf1f8666b92..436d485ab9368 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/TransposeConv2D.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/TransposeConv2D.cpp
@@ -138,7 +138,7 @@ FailureOr<Operation *> transposeConv2D(RewriterBase &rewriter,
                                linalg::Conv2DNhwcHwcfQOp>(rewriter, op);
 }
 
-void populateTranposeConv2DPatterns(RewritePatternSet &patterns) {
+void populateTransposeConv2DPatterns(RewritePatternSet &patterns) {
   MLIRContext *context = patterns.getContext();
   patterns.insert<
       ConvConverter<linalg::Conv2DNhwcFhwcOp, linalg::Conv2DNhwcHwcfOp>,
diff --git a/mlir/lib/Dialect/Linalg/Transforms/TransposeMatmul.cpp b/mlir/lib/Dialect/Linalg/Transforms/TransposeMatmul.cpp
index 6b934f7e8157d..e624f589917d1 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/TransposeMatmul.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/TransposeMatmul.cpp
@@ -88,6 +88,11 @@ FailureOr<Operation *>
 mlir::linalg::transposeBatchMatmul(RewriterBase &rewriter,
                                    linalg::BatchMatmulOp batchMatmulOp,
                                    bool transposeLHS) {
+  if (batchMatmulOp.hasUserDefinedMaps()) {
+    return rewriter.notifyMatchFailure(
+        batchMatmulOp, "ops with user-defined maps are not supported");
+  }
+
   if (!bufferization::hasTensorSemantics(batchMatmulOp))
     return rewriter.notifyMatchFailure(
         batchMatmulOp, "only matmul ops with tensors are supported");
diff --git a/mlir/lib/Dialect/Linalg/Transforms/Vectorization.cpp b/mlir/lib/Dialect/Linalg/Transforms/Vectorization.cpp
index 299bbc226dec8..f2c23c49a78e8 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/Vectorization.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/Vectorization.cpp
@@ -59,6 +59,37 @@ vectorizeConvolution(RewriterBase &rewriter, LinalgOp convOp,
                      ArrayRef<bool> inputVecScalableFlags = {},
                      bool flatten1DDepthwiseConv = false);
 
+/// Vectorize tensor::InsertSliceOp with:
+///   * vector::TransferReadOp + vector::TransferWriteOp
+/// The vector sizes are either:
+///   * user-provided in `inputVectorSizes`, or
+///   * inferred from the static dims in the input and output tensors.
+/// Bails out if:
+///   * vector sizes are not user-provided, and
+///   * at least one dim is dynamic (in both the input and output tensors).
+///
+/// Before:
+///     !t_in_type = tensor<1x2x3xf32>
+///     !t_out_type = tensor<9x8x7x1x2x3xf32>
+///     !v_type = vector<1x2x3xf32>
+///     %inserted_slice = tensor.insert_slice %src into %dest ... : !t_in_type
+///     into !t_out_type
+/// After:
+///     %read = vector.transfer_read %src[...], %pad ... : !t_in_type, !v_type
+///     %write = vector.transfer_write %read, %dest ... : !v_type, !t_out_type
+static LogicalResult
+vectorizeAsInsertSliceOp(RewriterBase &rewriter, tensor::InsertSliceOp sliceOp,
+                         ArrayRef<int64_t> inputVectorSizes,
+                         SmallVectorImpl<Value> &newResults);
+
+/// Returns the effective Pad value for the input op, provided it's a scalar.
+///
+/// Many Ops exhibit pad-like behaviour, but this isn't always explicit. If
+/// this Op performs padding, retrieve the padding value provided that it's
+/// a scalar and static/fixed for all the padded values. Returns an empty value
+/// otherwise.
+static Value getStaticPadVal(Operation *op);
+
 /// Return the unique instance of OpType in `block` if it is indeed unique.
 /// Return null if none or more than 1 instances exist.
 template <typename OpType>
@@ -1557,6 +1588,7 @@ static LogicalResult
 vectorizeAsTensorPackOp(RewriterBase &rewriter, tensor::PackOp packOp,
                         ArrayRef<int64_t> inputVectorSizes,
                         SmallVectorImpl<Value> &newResults) {
+  // TODO: Introduce a parent class that will handle the insertion point update.
   OpBuilder::InsertionGuard g(rewriter);
   rewriter.setInsertionPoint(packOp);
 
@@ -1633,6 +1665,7 @@ vectorizeAsTensorUnpackOp(RewriterBase &rewriter, tensor::UnPackOp unpackOp,
                           ArrayRef<int64_t> inputVectorSizes,
                           SmallVectorImpl<Value> &newResults) {
 
+  // TODO: Introduce a parent class that will handle the insertion point update.
   OpBuilder::InsertionGuard g(rewriter);
   rewriter.setInsertionPoint(unpackOp);
 
@@ -1763,7 +1796,7 @@ vectorizeAsTensorPadOp(RewriterBase &rewriter, tensor::PadOp padOp,
   auto padValue = padOp.getConstantPaddingValue();
   Location loc = padOp.getLoc();
 
-  // transfer_write_in_bounds(transfer_read_masked(pad_source, pad_value))
+  // TODO: Introduce a parent class that will handle the insertion point update.
   OpBuilder::InsertionGuard g(rewriter);
   rewriter.setInsertionPoint(padOp);
 
@@ -1874,6 +1907,38 @@ vectorizeUnPackOpPrecondition(tensor::UnPackOp unpackOp,
   return success();
 }
 
+static LogicalResult
+vectorizeInsertSliceOpPrecondition(tensor::InsertSliceOp sliceOp,
+                                   ArrayRef<int64_t> inputVectorSizes) {
+
+  TypedValue<RankedTensorType> source = sliceOp.getSource();
+  auto sourceType = source.getType();
+  if (!VectorType::isValidElementType(sourceType.getElementType()))
+    return failure();
+
+  // Get the pad value.
+  // TransferReadOp (which is used to vectorize InsertSliceOp), requires a
+  // scalar padding value. Note that:
+  //    * for in-bounds accesses,
+  // the value is actually irrelevant. There are 2 cases in which xfer.read
+  // accesses are known to be in-bounds:
+  //  1. The source shape is static (output vector sizes would be based on
+  //     the source shape and hence all memory accesses would be in-bounds),
+  //  2. Masking is used, i.e. the output vector sizes are user-provided. In
+  //     this case it is safe to assume that all memory accesses are in-bounds.
+  //
+  // When the value is not known and not needed, use 0. Otherwise, bail out.
+  Value padValue = getStaticPadVal(sliceOp);
+  bool isOutOfBoundsRead =
+      !sourceType.hasStaticShape() && inputVectorSizes.empty();
+
+  if (!padValue && isOutOfBoundsRead) {
+    LDBG("Failed to get a pad value for out-of-bounds read access\n");
+    return failure();
+  }
+  return success();
+}
+
 static LogicalResult vectorizeLinalgOpPrecondition(
     LinalgOp linalgOp, ArrayRef<int64_t> inputVectorSizes,
     bool vectorizeNDExtract, bool flatten1DDepthwiseConv) {
@@ -2144,6 +2209,9 @@ LogicalResult mlir::linalg::vectorizeOpPrecondition(
       .Case<tensor::UnPackOp>([&](auto unpackOp) {
         return vectorizeUnPackOpPrecondition(unpackOp, inputVectorSizes);
       })
+      .Case<tensor::InsertSliceOp>([&](auto sliceOp) {
+        return vectorizeInsertSliceOpPrecondition(sliceOp, inputVectorSizes);
+      })
       .Default([](auto) { return failure(); });
 }
 
@@ -2163,8 +2231,8 @@ static void convertAffineApply(RewriterBase &rewriter, LinalgOp linalgOp) {
 }
 
 bool mlir::linalg::hasVectorizationImpl(Operation *op) {
-  return isa<linalg::LinalgOp, tensor::PadOp, tensor::PackOp, tensor::UnPackOp>(
-      op);
+  return isa<linalg::LinalgOp, tensor::PadOp, tensor::PackOp, tensor::UnPackOp,
+             tensor::InsertSliceOp>(op);
 }
 
 /// Emit a suitable vector form for an operation. If provided,
@@ -2244,6 +2312,10 @@ LogicalResult mlir::linalg::vectorize(RewriterBase &rewriter, Operation *op,
             return vectorizeAsTensorPackOp(rewriter, packOp, inputVectorSizes,
                                            results);
           })
+          .Case<tensor::InsertSliceOp>([&](auto sliceOp) {
+            return vectorizeAsInsertSliceOp(rewriter, sliceOp, inputVectorSizes,
+                                            results);
+          })
           .Case<tensor::UnPackOp>([&](auto unpackOp) {
             return vectorizeAsTensorUnpackOp(rewriter, unpackOp,
                                              inputVectorSizes, results);
@@ -2540,6 +2612,9 @@ struct PadOpVectorizationWithTransferWritePattern
 /// this Op performs padding, retrieve the padding value provided that it's
 /// a scalar and static/fixed for all the padded values. Returns an empty value
 /// otherwise.
+///
+/// TODO: This is used twice (when checking vectorization pre-conditions and
+/// when vectorizing). Cache results instead of re-running.
 static Value getStaticPadVal(Operation *op) {
   if (!op)
     return {};
@@ -2583,113 +2658,118 @@ static Value getStaticPadVal(Operation *op) {
   return {};
 }
 
-/// Rewrite tensor.insert.slice as a vector.transfer_read +
-/// vector.transfer_write pair. The vector size is inferred from the static
-/// dims in the input and output tensors. If a dim is dynamic in both the input
-/// and output tensors, bails out.
-///
-/// Before:
-///     !t_in_type = tensor<1x2x3xf32>
-///     !t_out_type = tensor<9x8x7x1x2x3xf32>
-///     !v_type = vector<1x2x3xf32>
-///     %inserted_slice = tensor.insert_slice %src into %dest ... : !t_in_type
-///     into !t_out_type
-/// After:
-///     %read = vector.transfer_read %src[...], %pad ... : !t_in_type, !v_type
-///     %write = vector.transfer_write %read, %dest ... : !v_type, !t_out_type
-///
-/// TODO: Support masking
-struct InsertSliceVectorizePattern
-    : public OpRewritePattern<tensor::InsertSliceOp> {
-  using OpRewritePattern<tensor::InsertSliceOp>::OpRewritePattern;
+static LogicalResult
+vectorizeAsInsertSliceOp(RewriterBase &rewriter, tensor::InsertSliceOp sliceOp,
+                         ArrayRef<int64_t> inputVectorSizes,
+                         SmallVectorImpl<Value> &newResults) {
+  // TODO: Introduce a parent class that will handle the insertion point update.
+  OpBuilder::InsertionGuard g(rewriter);
+  rewriter.setInsertionPoint(sliceOp);
 
-  LogicalResult matchAndRewrite(tensor::InsertSliceOp sliceOp,
-                                PatternRewriter &rewriter) const final {
-    auto sourceType = sliceOp.getSource().getType();
-    if (!VectorType::isValidElementType(sourceType.getElementType()))
-      return failure();
+  TypedValue<RankedTensorType> source = sliceOp.getSource();
+  auto sourceType = source.getType();
+  auto resultType = sliceOp.getResultType();
+
+  Value padValue = getStaticPadVal(sliceOp);
 
-    auto resultType = sliceOp.getResultType();
-
-    // 1. Get the pad value.
-    // TransferReadOp requires a scalar padding value. Note that:
-    //    * for in-bounds access, the value is actually irrelevant.
-    //  There are 2 cases in which xfer.read accesses are known to be in-bounds:
-    //  1. The source shape is static (output vector sizes would be based on
-    //     the source shape and hence all memory accesses would be in-bounds),
-    //  2. Masking is used (output vector sizes would be user-provided, in which
-    //     case it is assumed that all memory accesses are in-bounds). This
-    //     remains a TODO.
-    //
-    // When the value is not known and not needed, use 0. Otherwise, bail out.
-    Value padValue = getStaticPadVal(sliceOp);
-    bool isOutOfBoundsRead = !sourceType.hasStaticShape();
-
-    if (!padValue && isOutOfBoundsRead) {
-      LDBG("Failed to get a pad value for out-of-bounds read access\n");
+  if (!padValue) {
+    auto elemType = sourceType.getElementType();
+    padValue = rewriter.create<arith::ConstantOp>(
+        sliceOp.getLoc(), elemType, rewriter.getZeroAttr(elemType));
+  }
+
+  // 2. Get the vector shape and in-bounds attributes
+  SmallVector<int64_t> vecShape;
+  SmallVector<bool> readInBounds;
+  SmallVector<bool> writeInBounds;
+  size_t rankDiff = resultType.getRank() - sourceType.getRank();
+  for (int64_t i = 0, end = sourceType.getRank(); i < end; ++i) {
+    if (!inputVectorSizes.empty()) {
+      vecShape.push_back(inputVectorSizes[i]);
+      readInBounds.push_back(false);
+      writeInBounds.push_back(false);
+    } else if (!sourceType.isDynamicDim(i)) {
+      vecShape.push_back(sourceType.getDimSize(i));
+      // Source shape is statically known: Neither read nor write are
+      // out-of-bounds.
+      readInBounds.push_back(true);
+      writeInBounds.push_back(true);
+    } else if (!resultType.isDynamicDim(i)) {
+      // Source shape is not statically known, but result shape is.
+      // Vectorize with size of result shape. This may be larger than the
+      // source size.
+      // FIXME: Using rankDiff implies that the source tensor is inserted at
+      // the end of the destination tensor. However, that's not required.
+      vecShape.push_back(resultType.getDimSize(rankDiff + i));
+      // Read may be out-of-bounds because the result size could be larger
+      // than the source size.
+      readInBounds.push_back(false);
+      // Write will be in-bounds provided that the corresponding write idx is 0.
+      // To keep this logic simple, conservatively mark as out-of-bounds.
+      writeInBounds.push_back(false);
+    } else {
+      // Neither source nor result dim of padOp is static. Cannot vectorize
+      // the copy.
+      // TODO: Add support for masking
       return failure();
     }
+  }
+  auto vecType = VectorType::get(vecShape, sourceType.getElementType());
+
+  // 3. Generate TransferReadOp + TransferWriteOp
+  ReifiedRankedShapedTypeDims reifiedSrcSizes;
+  Value maskOp;
 
-    if (!padValue) {
-      auto elemType = sourceType.getElementType();
-      padValue = rewriter.create<arith::ConstantOp>(
-          sliceOp.getLoc(), elemType, rewriter.getZeroAttr(elemType));
+  // If vector sizes are user provided, make sure to mask. First, generate the
+  // mask.
+  if (!inputVectorSizes.empty()) {
+    auto *srcDefOp = source.getDefiningOp();
+    if (!srcDefOp) {
+      LDBG("Unable to get the defining Op of " << sliceOp);
+      return failure();
     }
 
-    // 2. Get the vector shape and in-bounds attributes
-    SmallVector<int64_t> vecShape;
-    SmallVector<bool> readInBounds;
-    SmallVector<bool> writeInBounds;
-    size_t rankDiff = resultType.getRank() - sourceType.getRank();
-    for (unsigned i = 0; i < sourceType.getRank(); ++i) {
-      if (!sourceType.isDynamicDim(i)) {
-        vecShape.push_back(sourceType.getDimSize(i));
-        // Source shape is statically known: Neither read nor write are
-        // out-of-bounds.
-        readInBounds.push_back(true);
-        writeInBounds.push_back(true);
-      } else if (!resultType.isDynamicDim(i)) {
-        // Source shape is not statically known, but result shape is.
-        // Vectorize with size of result shape. This may be larger than the
-        // source size.
-        // FIXME: Using rankDiff implies that the source tensor is inserted at
-        // the end of the destination tensor. However, that's not required.
-        vecShape.push_back(resultType.getDimSize(rankDiff + i));
-        // Read may be out-of-bounds because the result size could be larger
-        // than the source size.
-        readInBounds.push_back(false);
-        // Write will in-bounds provided that the corresponding write idx is 0.
-        // To keep this logic simple, conservatively mark as out-of-bounds.
-        writeInBounds.push_back(false);
-      } else {
-        // Neither source nor result dim of padOp is static. Cannot vectorize
-        // the copy.
-        // TODO: Add support for masking
-        return failure();
-      }
+    LogicalResult status =
+        cast<ReifyRankedShapedTypeOpInterface>(srcDefOp).reifyResultShapes(
+            rewriter, reifiedSrcSizes);
+    if (status.failed()) {
+      LDBG("Unable to reify result shapes of " << srcDefOp);
+      return failure();
     }
-    auto vecType = VectorType::get(vecShape, sourceType.getElementType());
 
-    // 3. Generate TransferReadOp.
-    SmallVector<Value> readIndices(
-        vecType.getRank(),
-        rewriter.create<arith::ConstantIndexOp>(sliceOp.getLoc(), 0));
-    auto read = rewriter.create<vector::TransferReadOp>(
-        sliceOp.getLoc(), vecType, sliceOp.getSource(), readIndices, padValue,
-        ArrayRef<bool>{readInBounds});
+    // Create the mask
+    auto readMaskType = VectorType::get(inputVectorSizes, rewriter.getI1Type());
+    maskOp = rewriter.create<vector::CreateMaskOp>(
+        sliceOp.getLoc(), readMaskType, reifiedSrcSizes[0]);
+  }
 
-    // 4. Generate TransferWriteOp.
-    auto writeIndices = getValueOrCreateConstantIndexOp(
-        rewriter, sliceOp.getLoc(), sliceOp.getMixedOffsets());
+  SmallVector<Value> readIndices(
+      vecType.getRank(),
+      rewriter.create<arith::ConstantIndexOp>(sliceOp.getLoc(), 0));
+  Operation *read = rewriter.create<vector::TransferReadOp>(
+      sliceOp.getLoc(), vecType, source, readIndices, padValue,
+      ArrayRef<bool>{readInBounds});
 
-    // 5. Finalize
-    rewriter.replaceOpWithNewOp<vector::TransferWriteOp>(
-        sliceOp, read, sliceOp.getDest(), writeIndices,
-        ArrayRef<bool>{writeInBounds});
+  if (maskOp) {
+    read = mlir::vector::maskOperation(rewriter, read, maskOp);
+  }
 
-    return success();
+  auto writeIndices = getValueOrCreateConstantIndexOp(
+      rewriter, sliceOp.getLoc(), sliceOp.getMixedOffsets());
+
+  Operation *write = rewriter.create<vector::TransferWriteOp>(
+      sliceOp.getLoc(), read->getResult(0), sliceOp.getDest(), writeIndices,
+      ArrayRef<bool>{writeInBounds});
+
+  if (maskOp) {
+    write = mlir::vector::maskOperation(rewriter, write, maskOp);
   }
-};
+
+  // 4. Finalize
+  newResults.push_back(write->getResult(0));
+
+  return success();
+}
 
 /// Rewrite use of tensor::PadOp result in InsertSliceOp. E.g.:
 /// ```
@@ -2778,11 +2858,6 @@ struct PadOpVectorizationWithInsertSlicePattern
   }
 };
 
-void mlir::linalg::populateInsertSliceVectorizationPatterns(
-    RewritePatternSet &patterns) {
-  patterns.add<InsertSliceVectorizePattern>(patterns.getContext());
-}
-
 void mlir::linalg::populatePadOpVectorizationPatterns(
     RewritePatternSet &patterns, PatternBenefit baseBenefit) {
   patterns.add<PadOpVectorizationWithTransferReadPattern,
diff --git a/mlir/lib/Dialect/Linalg/Transforms/WinogradConv2D.cpp b/mlir/lib/Dialect/Linalg/Transforms/WinogradConv2D.cpp
index 79f77822116fd..f1059ddf5da2c 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/WinogradConv2D.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/WinogradConv2D.cpp
@@ -514,12 +514,14 @@ Value inputTransform(RewriterBase &rewriter, Location loc, Value input,
     Value CIter = ivs[3];
 
     auto context = builder.getContext();
+
+    auto identityAffineMap = rewriter.getMultiDimIdentityMap(1);
     auto affineMap =
         AffineMap::get(1, 0, {builder.getAffineDimExpr(0) * m}, context);
-    Value heightOffset =
-        builder.create<affine::AffineApplyOp>(loc, affineMap, tileHIter);
-    Value widthOffset =
-        builder.create<affine::AffineApplyOp>(loc, affineMap, tileWIter);
+    Value heightOffset = builder.create<affine::AffineApplyOp>(
+        loc, leftTransform ? affineMap : identityAffineMap, tileHIter);
+    Value widthOffset = builder.create<affine::AffineApplyOp>(
+        loc, rightTransform ? affineMap : identityAffineMap, tileWIter);
 
     // Extract (H, W) from (N, H, W, C).
     auto extractInput =
@@ -753,12 +755,13 @@ Value outputTransform(RewriterBase &rewriter, Location loc, Value value,
     Value zero = builder.create<arith::ConstantOp>(
         loc, rewriter.getZeroAttr(elementType));
 
+    auto identityAffineMap = rewriter.getMultiDimIdentityMap(1);
     auto affineMap =
         AffineMap::get(1, 0, {builder.getAffineDimExpr(0) * m}, context);
-    Value heightOffset =
-        builder.create<affine::AffineApplyOp>(loc, affineMap, tileHIter);
-    Value widthOffset =
-        builder.create<affine::AffineApplyOp>(loc, affineMap, tileWIter);
+    Value heightOffset = builder.create<affine::AffineApplyOp>(
+        loc, leftTransform ? affineMap : identityAffineMap, tileHIter);
+    Value widthOffset = builder.create<affine::AffineApplyOp>(
+        loc, rightTransform ? affineMap : identityAffineMap, tileWIter);
 
     Value outInitVal =
         extract2DDataFrom4D(builder, loc, args[0], NIter, FIter, heightOffset,
@@ -1075,16 +1078,17 @@ FailureOr<Operation *>
 decomposeWinogradInputTransformHelper(RewriterBase &rewriter,
                                       linalg::WinogradInputTransformOp op) {
   Location loc = op.getLoc();
-  Value input = op.getInput();
-  auto inputType = cast<ShapedType>(input.getType());
-  auto inputShape = inputType.getShape();
-  int64_t inputH = inputShape[1];
-  int64_t inputW = inputShape[2];
+  Value output = op.getOutput();
+  auto outputType = cast<ShapedType>(output.getType());
+  auto outputShape = outputType.getShape();
+
+  int64_t outputH = outputShape[0];
+  int64_t outputW = outputShape[1];
 
   // For F(m x 1, r x 1), we only need to do left side transform.
-  bool leftTransform = inputH != 1;
+  bool leftTransform = outputH != 1;
   // For F(1 x m, 1 x r), we only need to do right side transform.
-  bool rightTransform = inputW != 1;
+  bool rightTransform = outputW != 1;
   Value transformedInput =
       inputTransform(rewriter, loc, op.getInput(), op.getOutput(), op.getM(),
                      op.getR(), leftTransform, rightTransform);
diff --git a/mlir/lib/Dialect/MPI/IR/MPIOps.cpp b/mlir/lib/Dialect/MPI/IR/MPIOps.cpp
index dcb55d8921364..56d8edfbcc025 100644
--- a/mlir/lib/Dialect/MPI/IR/MPIOps.cpp
+++ b/mlir/lib/Dialect/MPI/IR/MPIOps.cpp
@@ -53,6 +53,16 @@ void mlir::mpi::RecvOp::getCanonicalizationPatterns(
   results.add<FoldCast<mlir::mpi::RecvOp>>(context);
 }
 
+void mlir::mpi::ISendOp::getCanonicalizationPatterns(
+    mlir::RewritePatternSet &results, mlir::MLIRContext *context) {
+  results.add<FoldCast<mlir::mpi::ISendOp>>(context);
+}
+
+void mlir::mpi::IRecvOp::getCanonicalizationPatterns(
+    mlir::RewritePatternSet &results, mlir::MLIRContext *context) {
+  results.add<FoldCast<mlir::mpi::IRecvOp>>(context);
+}
+
 //===----------------------------------------------------------------------===//
 // TableGen'd op method definitions
 //===----------------------------------------------------------------------===//
diff --git a/mlir/lib/Dialect/Math/Transforms/PolynomialApproximation.cpp b/mlir/lib/Dialect/Math/Transforms/PolynomialApproximation.cpp
index 24c892f68b503..777427de9465c 100644
--- a/mlir/lib/Dialect/Math/Transforms/PolynomialApproximation.cpp
+++ b/mlir/lib/Dialect/Math/Transforms/PolynomialApproximation.cpp
@@ -1667,28 +1667,125 @@ void mlir::populatePolynomialApproximateErfPattern(
   patterns.add<ErfPolynomialApproximation>(patterns.getContext());
 }
 
+template <typename OpType>
+static void
+populateMathF32ExpansionPattern(RewritePatternSet &patterns,
+                                llvm::function_ref<bool(StringRef)> predicate) {
+  if (predicate(OpType::getOperationName())) {
+    patterns.add<ReuseF32Expansion<OpType>>(patterns.getContext());
+  }
+}
+
+void mlir::populateMathF32ExpansionPatterns(
+    RewritePatternSet &patterns,
+    llvm::function_ref<bool(StringRef)> predicate) {
+  populateMathF32ExpansionPattern<math::AcosOp>(patterns, predicate);
+  populateMathF32ExpansionPattern<math::AcoshOp>(patterns, predicate);
+  populateMathF32ExpansionPattern<math::AsinOp>(patterns, predicate);
+  populateMathF32ExpansionPattern<math::AsinhOp>(patterns, predicate);
+  populateMathF32ExpansionPattern<math::AtanOp>(patterns, predicate);
+  populateMathF32ExpansionPattern<math::Atan2Op>(patterns, predicate);
+  populateMathF32ExpansionPattern<math::AtanhOp>(patterns, predicate);
+  populateMathF32ExpansionPattern<math::CbrtOp>(patterns, predicate);
+  populateMathF32ExpansionPattern<math::CosOp>(patterns, predicate);
+  populateMathF32ExpansionPattern<math::CoshOp>(patterns, predicate);
+  populateMathF32ExpansionPattern<math::ErfOp>(patterns, predicate);
+  populateMathF32ExpansionPattern<math::ExpOp>(patterns, predicate);
+  populateMathF32ExpansionPattern<math::Exp2Op>(patterns, predicate);
+  populateMathF32ExpansionPattern<math::ExpM1Op>(patterns, predicate);
+  populateMathF32ExpansionPattern<math::LogOp>(patterns, predicate);
+  populateMathF32ExpansionPattern<math::Log10Op>(patterns, predicate);
+  populateMathF32ExpansionPattern<math::Log1pOp>(patterns, predicate);
+  populateMathF32ExpansionPattern<math::Log2Op>(patterns, predicate);
+  populateMathF32ExpansionPattern<math::PowFOp>(patterns, predicate);
+  populateMathF32ExpansionPattern<math::RsqrtOp>(patterns, predicate);
+  populateMathF32ExpansionPattern<math::SinOp>(patterns, predicate);
+  populateMathF32ExpansionPattern<math::SinhOp>(patterns, predicate);
+  populateMathF32ExpansionPattern<math::SqrtOp>(patterns, predicate);
+  populateMathF32ExpansionPattern<math::TanOp>(patterns, predicate);
+  populateMathF32ExpansionPattern<math::TanhOp>(patterns, predicate);
+}
+
+template <typename OpType, typename PatternType>
+static void populateMathPolynomialApproximationPattern(
+    RewritePatternSet &patterns,
+    llvm::function_ref<bool(StringRef)> predicate) {
+  if (predicate(OpType::getOperationName())) {
+    patterns.add<PatternType>(patterns.getContext());
+  }
+}
+
+void mlir::populateMathPolynomialApproximationPatterns(
+    RewritePatternSet &patterns,
+    llvm::function_ref<bool(StringRef)> predicate) {
+  populateMathPolynomialApproximationPattern<AcosOp,
+                                             AcosPolynomialApproximation>(
+      patterns, predicate);
+  populateMathPolynomialApproximationPattern<AsinOp,
+                                             AsinPolynomialApproximation>(
+      patterns, predicate);
+  populateMathPolynomialApproximationPattern<AtanOp, AtanApproximation>(
+      patterns, predicate);
+  populateMathPolynomialApproximationPattern<Atan2Op, Atan2Approximation>(
+      patterns, predicate);
+  populateMathPolynomialApproximationPattern<CbrtOp, CbrtApproximation>(
+      patterns, predicate);
+  populateMathPolynomialApproximationPattern<
+      CosOp, SinAndCosApproximation<false, math::CosOp>>(patterns, predicate);
+  populateMathPolynomialApproximationPattern<ErfOp, ErfPolynomialApproximation>(
+      patterns, predicate);
+  populateMathPolynomialApproximationPattern<ExpOp, ExpApproximation>(
+      patterns, predicate);
+  populateMathPolynomialApproximationPattern<ExpM1Op, ExpM1Approximation>(
+      patterns, predicate);
+  populateMathPolynomialApproximationPattern<LogOp, LogApproximation>(
+      patterns, predicate);
+  populateMathPolynomialApproximationPattern<Log2Op, Log2Approximation>(
+      patterns, predicate);
+  populateMathPolynomialApproximationPattern<Log1pOp, Log1pApproximation>(
+      patterns, predicate);
+  populateMathPolynomialApproximationPattern<RsqrtOp, RsqrtApproximation>(
+      patterns, predicate);
+  populateMathPolynomialApproximationPattern<
+      SinOp, SinAndCosApproximation<true, math::SinOp>>(patterns, predicate);
+  populateMathPolynomialApproximationPattern<TanhOp, TanhApproximation>(
+      patterns, predicate);
+}
+
 void mlir::populateMathPolynomialApproximationPatterns(
     RewritePatternSet &patterns,
     const MathPolynomialApproximationOptions &options) {
-  // Patterns for leveraging existing f32 lowerings on other data types.
-  patterns
-      .add<ReuseF32Expansion<math::AtanOp>, ReuseF32Expansion<math::Atan2Op>,
-           ReuseF32Expansion<math::TanhOp>, ReuseF32Expansion<math::LogOp>,
-           ReuseF32Expansion<math::Log2Op>, ReuseF32Expansion<math::Log1pOp>,
-           ReuseF32Expansion<math::ErfOp>, ReuseF32Expansion<math::ExpOp>,
-           ReuseF32Expansion<math::ExpM1Op>, ReuseF32Expansion<math::CbrtOp>,
-           ReuseF32Expansion<math::SinOp>, ReuseF32Expansion<math::CosOp>>(
-          patterns.getContext());
-
-  patterns
-      .add<AtanApproximation, Atan2Approximation, TanhApproximation,
-           LogApproximation, Log2Approximation, Log1pApproximation,
-           ErfPolynomialApproximation, AsinPolynomialApproximation,
-           AcosPolynomialApproximation, ExpApproximation, ExpM1Approximation,
-           CbrtApproximation, SinAndCosApproximation<true, math::SinOp>,
-           SinAndCosApproximation<false, math::CosOp>>(patterns.getContext());
+  mlir::populateMathF32ExpansionPatterns(patterns, [](StringRef name) -> bool {
+    return llvm::is_contained(
+        {math::AtanOp::getOperationName(), math::Atan2Op::getOperationName(),
+         math::TanhOp::getOperationName(), math::LogOp::getOperationName(),
+         math::Log2Op::getOperationName(), math::Log1pOp::getOperationName(),
+         math::ErfOp::getOperationName(), math::ExpOp::getOperationName(),
+         math::ExpM1Op::getOperationName(), math::CbrtOp::getOperationName(),
+         math::SinOp::getOperationName(), math::CosOp::getOperationName()},
+        name);
+  });
+
+  populateMathPolynomialApproximationPatterns(
+      patterns, [](StringRef name) -> bool {
+        return llvm::is_contained(
+            {math::AtanOp::getOperationName(),
+             math::Atan2Op::getOperationName(),
+             math::TanhOp::getOperationName(), math::LogOp::getOperationName(),
+             math::Log2Op::getOperationName(),
+             math::Log1pOp::getOperationName(), math::ErfOp::getOperationName(),
+             math::AsinOp::getOperationName(), math::AcosOp::getOperationName(),
+             math::ExpOp::getOperationName(), math::ExpM1Op::getOperationName(),
+             math::CbrtOp::getOperationName(), math::SinOp::getOperationName(),
+             math::CosOp::getOperationName()},
+            name);
+      });
+
   if (options.enableAvx2) {
-    patterns.add<RsqrtApproximation, ReuseF32Expansion<math::RsqrtOp>>(
-        patterns.getContext());
+    auto predicateRsqrt = [](StringRef name) {
+      return name == math::RsqrtOp::getOperationName();
+    };
+    mlir::populateMathF32ExpansionPatterns(patterns, predicateRsqrt);
+    mlir::populateMathPolynomialApproximationPatterns(patterns, predicateRsqrt);
   }
 }
diff --git a/mlir/lib/Dialect/MemRef/IR/CMakeLists.txt b/mlir/lib/Dialect/MemRef/IR/CMakeLists.txt
index 845914ebd107a..734294bd014c6 100644
--- a/mlir/lib/Dialect/MemRef/IR/CMakeLists.txt
+++ b/mlir/lib/Dialect/MemRef/IR/CMakeLists.txt
@@ -16,6 +16,8 @@ add_mlir_dialect_library(MLIRMemRefDialect
   MLIRControlFlowInterfaces
   MLIRDialect
   MLIRDialectUtils
+  MLIRInferIntRangeCommon
+  MLIRInferIntRangeInterface
   MLIRInferTypeOpInterface
   MLIRIR
   MLIRMemorySlotInterfaces
diff --git a/mlir/lib/Dialect/MemRef/IR/MemRefOps.cpp b/mlir/lib/Dialect/MemRef/IR/MemRefOps.cpp
index 4f75b7618d636..e0930abc1887d 100644
--- a/mlir/lib/Dialect/MemRef/IR/MemRefOps.cpp
+++ b/mlir/lib/Dialect/MemRef/IR/MemRefOps.cpp
@@ -19,6 +19,7 @@
 #include "mlir/IR/TypeUtilities.h"
 #include "mlir/Interfaces/InferTypeOpInterface.h"
 #include "mlir/Interfaces/SideEffectInterfaces.h"
+#include "mlir/Interfaces/Utils/InferIntRangeCommon.h"
 #include "mlir/Interfaces/ViewLikeInterface.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/SmallBitVector.h"
@@ -915,6 +916,12 @@ Speculation::Speculatability DimOp::getSpeculatability() {
   return Speculation::Speculatable;
 }
 
+void DimOp::inferResultRangesFromOptional(ArrayRef<IntegerValueRange> argRanges,
+                                          SetIntLatticeFn setResultRange) {
+  setResultRange(getResult(),
+                 intrange::inferShapedDimOpInterface(*this, argRanges[1]));
+}
+
 /// Return a map with key being elements in `vals` and data being number of
 /// occurences of it. Use std::map, since the `vals` here are strides and the
 /// dynamic stride value is the same as the tombstone value for
diff --git a/mlir/lib/Dialect/MemRef/Transforms/NormalizeMemRefs.cpp b/mlir/lib/Dialect/MemRef/Transforms/NormalizeMemRefs.cpp
index 33772ccb7dd9d..08b853fe65b85 100644
--- a/mlir/lib/Dialect/MemRef/Transforms/NormalizeMemRefs.cpp
+++ b/mlir/lib/Dialect/MemRef/Transforms/NormalizeMemRefs.cpp
@@ -151,11 +151,11 @@ void NormalizeMemRefs::setCalleesAndCallersNonNormalizable(
   });
 }
 
-/// Check whether all the uses of AllocOps, CallOps and function arguments of a
-/// function are either of dereferencing type or are uses in: DeallocOp, CallOp
-/// or ReturnOp. Only if these constraints are satisfied will the function
-/// become a candidate for normalization. When the uses of a memref are
-/// non-normalizable and the memref map layout is trivial (identity), we can
+/// Check whether all the uses of AllocOps, AllocaOps, CallOps and function
+/// arguments of a function are either of dereferencing type or are uses in:
+/// DeallocOp, CallOp or ReturnOp. Only if these constraints are satisfied will
+/// the function become a candidate for normalization. When the uses of a memref
+/// are non-normalizable and the memref map layout is trivial (identity), we can
 /// still label the entire function as normalizable. We assume external
 /// functions to be normalizable.
 bool NormalizeMemRefs::areMemRefsNormalizable(func::FuncOp funcOp) {
@@ -174,6 +174,17 @@ bool NormalizeMemRefs::areMemRefsNormalizable(func::FuncOp funcOp) {
           .wasInterrupted())
     return false;
 
+  if (funcOp
+          .walk([&](memref::AllocaOp allocaOp) -> WalkResult {
+            Value oldMemRef = allocaOp.getResult();
+            if (!allocaOp.getType().getLayout().isIdentity() &&
+                !isMemRefNormalizable(oldMemRef.getUsers()))
+              return WalkResult::interrupt();
+            return WalkResult::advance();
+          })
+          .wasInterrupted())
+    return false;
+
   if (funcOp
           .walk([&](func::CallOp callOp) -> WalkResult {
             for (unsigned resIndex :
@@ -335,18 +346,23 @@ void NormalizeMemRefs::updateFunctionSignature(func::FuncOp funcOp,
 }
 
 /// Normalizes the memrefs within a function which includes those arising as a
-/// result of AllocOps, CallOps and function's argument. The ModuleOp argument
-/// is used to help update function's signature after normalization.
+/// result of AllocOps, AllocaOps, CallOps and function's argument. The ModuleOp
+/// argument is used to help update function's signature after normalization.
 void NormalizeMemRefs::normalizeFuncOpMemRefs(func::FuncOp funcOp,
                                               ModuleOp moduleOp) {
   // Turn memrefs' non-identity layouts maps into ones with identity. Collect
-  // alloc ops first and then process since normalizeMemRef replaces/erases ops
-  // during memref rewriting.
+  // alloc/alloca ops first and then process since normalizeMemRef
+  // replaces/erases ops during memref rewriting.
   SmallVector<memref::AllocOp, 4> allocOps;
   funcOp.walk([&](memref::AllocOp op) { allocOps.push_back(op); });
   for (memref::AllocOp allocOp : allocOps)
     (void)normalizeMemRef(&allocOp);
 
+  SmallVector<memref::AllocaOp> allocaOps;
+  funcOp.walk([&](memref::AllocaOp op) { allocaOps.push_back(op); });
+  for (memref::AllocaOp allocaOp : allocaOps)
+    (void)normalizeMemRef(&allocaOp);
+
   // We use this OpBuilder to create new memref layout later.
   OpBuilder b(funcOp);
 
diff --git a/mlir/lib/Dialect/NVGPU/Transforms/OptimizeSharedMemory.cpp b/mlir/lib/Dialect/NVGPU/Transforms/OptimizeSharedMemory.cpp
index 31ffacb29256f..72f7296a865f8 100644
--- a/mlir/lib/Dialect/NVGPU/Transforms/OptimizeSharedMemory.cpp
+++ b/mlir/lib/Dialect/NVGPU/Transforms/OptimizeSharedMemory.cpp
@@ -152,6 +152,10 @@ mlir::nvgpu::optimizeSharedMemoryReadsAndWrites(Operation *parentOp,
   if (!memRefType || !NVGPUDialect::hasSharedMemoryAddressSpace(memRefType))
     return failure();
 
+  // Not support 0D MemRefs.
+  if (memRefType.getRank() == 0)
+    return failure();
+
   // Abort if the given value has any sub-views; we do not do any alias
   // analysis.
   bool hasSubView = false;
diff --git a/mlir/lib/Dialect/OpenACC/IR/OpenACC.cpp b/mlir/lib/Dialect/OpenACC/IR/OpenACC.cpp
index 735dd4fd7b8bb..8f150cb33e6f3 100644
--- a/mlir/lib/Dialect/OpenACC/IR/OpenACC.cpp
+++ b/mlir/lib/Dialect/OpenACC/IR/OpenACC.cpp
@@ -32,11 +32,42 @@ using namespace acc;
 #include "mlir/Dialect/OpenACCMPCommon/Interfaces/OpenACCMPOpsInterfaces.cpp.inc"
 
 namespace {
+
+static bool isScalarLikeType(Type type) {
+  return type.isIntOrIndexOrFloat() || isa<ComplexType>(type);
+}
+
 struct MemRefPointerLikeModel
     : public PointerLikeType::ExternalModel<MemRefPointerLikeModel,
                                             MemRefType> {
   Type getElementType(Type pointer) const {
-    return llvm::cast<MemRefType>(pointer).getElementType();
+    return cast<MemRefType>(pointer).getElementType();
+  }
+  mlir::acc::VariableTypeCategory
+  getPointeeTypeCategory(Type pointer, TypedValue<PointerLikeType> varPtr,
+                         Type varType) const {
+    if (auto mappableTy = dyn_cast<MappableType>(varType)) {
+      return mappableTy.getTypeCategory(varPtr);
+    }
+    auto memrefTy = cast<MemRefType>(pointer);
+    if (!memrefTy.hasRank()) {
+      // This memref is unranked - aka it could have any rank, including a
+      // rank of 0 which could mean scalar. For now, return uncategorized.
+      return mlir::acc::VariableTypeCategory::uncategorized;
+    }
+
+    if (memrefTy.getRank() == 0) {
+      if (isScalarLikeType(memrefTy.getElementType())) {
+        return mlir::acc::VariableTypeCategory::scalar;
+      }
+      // Zero-rank non-scalar - need further analysis to determine the type
+      // category. For now, return uncategorized.
+      return mlir::acc::VariableTypeCategory::uncategorized;
+    }
+
+    // It has a rank - must be an array.
+    assert(memrefTy.getRank() > 0 && "rank expected to be positive");
+    return mlir::acc::VariableTypeCategory::array;
   }
 };
 
@@ -549,6 +580,7 @@ LogicalResult acc::DeleteOp::verify() {
       getDataClause() != acc::DataClause::acc_copyin &&
       getDataClause() != acc::DataClause::acc_copyin_readonly &&
       getDataClause() != acc::DataClause::acc_present &&
+      getDataClause() != acc::DataClause::acc_no_create &&
       getDataClause() != acc::DataClause::acc_declare_device_resident &&
       getDataClause() != acc::DataClause::acc_declare_link)
     return emitError(
@@ -2579,9 +2611,9 @@ checkDeclareOperands(Op &op, const mlir::ValueRange &operands,
           "expect valid declare data entry operation or acc.getdeviceptr "
           "as defining op");
 
-    mlir::Value varPtr{getVarPtr(operand.getDefiningOp())};
-    assert(varPtr && "declare operands can only be data entry operations which "
-                     "must have varPtr");
+    mlir::Value var{getVar(operand.getDefiningOp())};
+    assert(var && "declare operands can only be data entry operations which "
+                  "must have var");
     std::optional<mlir::acc::DataClause> dataClauseOptional{
         getDataClause(operand.getDefiningOp())};
     assert(dataClauseOptional.has_value() &&
@@ -2589,12 +2621,12 @@ checkDeclareOperands(Op &op, const mlir::ValueRange &operands,
            "dataClause");
 
     // If varPtr has no defining op - there is nothing to check further.
-    if (!varPtr.getDefiningOp())
+    if (!var.getDefiningOp())
       continue;
 
     // Check that the varPtr has a declare attribute.
     auto declareAttribute{
-        varPtr.getDefiningOp()->getAttr(mlir::acc::getDeclareAttrName())};
+        var.getDefiningOp()->getAttr(mlir::acc::getDeclareAttrName())};
     if (!declareAttribute)
       return op.emitError(
           "expect declare attribute on variable in declare operation");
diff --git a/mlir/lib/Dialect/OpenACC/Transforms/LegalizeDataValues.cpp b/mlir/lib/Dialect/OpenACC/Transforms/LegalizeDataValues.cpp
index 026b309ce4969..a553653c73479 100644
--- a/mlir/lib/Dialect/OpenACC/Transforms/LegalizeDataValues.cpp
+++ b/mlir/lib/Dialect/OpenACC/Transforms/LegalizeDataValues.cpp
@@ -36,17 +36,17 @@ static bool insideAccComputeRegion(mlir::Operation *op) {
   return false;
 }
 
-static void collectPtrs(mlir::ValueRange operands,
+static void collectVars(mlir::ValueRange operands,
                         llvm::SmallVector<std::pair<Value, Value>> &values,
                         bool hostToDevice) {
   for (auto operand : operands) {
-    Value varPtr = acc::getVarPtr(operand.getDefiningOp());
-    Value accPtr = acc::getAccPtr(operand.getDefiningOp());
-    if (varPtr && accPtr) {
+    Value var = acc::getVar(operand.getDefiningOp());
+    Value accVar = acc::getAccVar(operand.getDefiningOp());
+    if (var && accVar) {
       if (hostToDevice)
-        values.push_back({varPtr, accPtr});
+        values.push_back({var, accVar});
       else
-        values.push_back({accPtr, varPtr});
+        values.push_back({accVar, var});
     }
   }
 }
@@ -75,16 +75,16 @@ static void collectAndReplaceInRegion(Op &op, bool hostToDevice) {
   llvm::SmallVector<std::pair<Value, Value>> values;
 
   if constexpr (std::is_same_v<Op, acc::LoopOp>) {
-    collectPtrs(op.getReductionOperands(), values, hostToDevice);
-    collectPtrs(op.getPrivateOperands(), values, hostToDevice);
+    collectVars(op.getReductionOperands(), values, hostToDevice);
+    collectVars(op.getPrivateOperands(), values, hostToDevice);
   } else {
-    collectPtrs(op.getDataClauseOperands(), values, hostToDevice);
+    collectVars(op.getDataClauseOperands(), values, hostToDevice);
     if constexpr (!std::is_same_v<Op, acc::KernelsOp> &&
                   !std::is_same_v<Op, acc::DataOp> &&
                   !std::is_same_v<Op, acc::DeclareOp>) {
-      collectPtrs(op.getReductionOperands(), values, hostToDevice);
-      collectPtrs(op.getPrivateOperands(), values, hostToDevice);
-      collectPtrs(op.getFirstprivateOperands(), values, hostToDevice);
+      collectVars(op.getReductionOperands(), values, hostToDevice);
+      collectVars(op.getPrivateOperands(), values, hostToDevice);
+      collectVars(op.getFirstprivateOperands(), values, hostToDevice);
     }
   }
 
diff --git a/mlir/lib/Dialect/OpenMP/IR/OpenMPDialect.cpp b/mlir/lib/Dialect/OpenMP/IR/OpenMPDialect.cpp
index 88f56dc514422..5ec840e7fef81 100644
--- a/mlir/lib/Dialect/OpenMP/IR/OpenMPDialect.cpp
+++ b/mlir/lib/Dialect/OpenMP/IR/OpenMPDialect.cpp
@@ -2010,9 +2010,9 @@ static LogicalResult verifyPrivateVarList(OpType &op) {
       return op.emitError() << "failed to lookup privatizer op with symbol: '"
                             << privateSym << "'";
 
-    Type privatizerType = privatizerOp.getType();
+    Type privatizerType = privatizerOp.getArgType();
 
-    if (varType != privatizerType)
+    if (privatizerType && (varType != privatizerType))
       return op.emitError()
              << "type mismatch between a "
              << (privatizerOp.getDataSharingType() ==
@@ -3059,8 +3059,7 @@ void PrivateClauseOp::build(OpBuilder &odsBuilder, OperationState &odsState,
 }
 
 LogicalResult PrivateClauseOp::verifyRegions() {
-  Type symType = getType();
-
+  Type argType = getArgType();
   auto verifyTerminator = [&](Operation *terminator,
                               bool yieldsValue) -> LogicalResult {
     if (!terminator->getBlock()->getSuccessors().empty())
@@ -3081,11 +3080,11 @@ LogicalResult PrivateClauseOp::verifyRegions() {
              << "Did not expect any values to be yielded.";
     }
 
-    if (yieldedTypes.size() == 1 && yieldedTypes.front() == symType)
+    if (yieldedTypes.size() == 1 && yieldedTypes.front() == argType)
       return success();
 
     auto error = mlir::emitError(yieldOp.getLoc())
-                 << "Invalid yielded value. Expected type: " << symType
+                 << "Invalid yielded value. Expected type: " << argType
                  << ", got: ";
 
     if (yieldedTypes.empty())
@@ -3119,18 +3118,27 @@ LogicalResult PrivateClauseOp::verifyRegions() {
     return success();
   };
 
-  if (failed(verifyRegion(getAllocRegion(), /*expectedNumArgs=*/1, "alloc",
+  // Ensure all of the region arguments have the same type
+  for (Region *region : getRegions())
+    for (Type ty : region->getArgumentTypes())
+      if (ty != argType)
+        return emitError() << "Region argument type mismatch: got " << ty
+                           << " expected " << argType << ".";
+
+  mlir::Region &initRegion = getInitRegion();
+  if (!initRegion.empty() &&
+      failed(verifyRegion(getInitRegion(), /*expectedNumArgs=*/2, "init",
                           /*yieldsValue=*/true)))
     return failure();
 
   DataSharingClauseType dsType = getDataSharingType();
 
   if (dsType == DataSharingClauseType::Private && !getCopyRegion().empty())
-    return emitError("`private` clauses require only an `alloc` region.");
+    return emitError("`private` clauses do not require a `copy` region.");
 
   if (dsType == DataSharingClauseType::FirstPrivate && getCopyRegion().empty())
     return emitError(
-        "`firstprivate` clauses require both `alloc` and `copy` regions.");
+        "`firstprivate` clauses require at least a `copy` region.");
 
   if (dsType == DataSharingClauseType::FirstPrivate &&
       failed(verifyRegion(getCopyRegion(), /*expectedNumArgs=*/2, "copy",
diff --git a/mlir/lib/Dialect/SCF/IR/ValueBoundsOpInterfaceImpl.cpp b/mlir/lib/Dialect/SCF/IR/ValueBoundsOpInterfaceImpl.cpp
index 8a27bf186d1c2..410a6bffd345e 100644
--- a/mlir/lib/Dialect/SCF/IR/ValueBoundsOpInterfaceImpl.cpp
+++ b/mlir/lib/Dialect/SCF/IR/ValueBoundsOpInterfaceImpl.cpp
@@ -20,6 +20,16 @@ namespace {
 struct ForOpInterface
     : public ValueBoundsOpInterface::ExternalModel<ForOpInterface, ForOp> {
 
+  static AffineExpr getTripCountExpr(scf::ForOp forOp,
+                                     ValueBoundsConstraintSet &cstr) {
+    AffineExpr lbExpr = cstr.getExpr(forOp.getLowerBound());
+    AffineExpr ubExpr = cstr.getExpr(forOp.getUpperBound());
+    AffineExpr stepExpr = cstr.getExpr(forOp.getStep());
+    AffineExpr tripCountExpr =
+        AffineExpr(ubExpr - lbExpr).ceilDiv(stepExpr); // (ub - lb) / step
+    return tripCountExpr;
+  }
+
   /// Populate bounds of values/dimensions for iter_args/OpResults. If the
   /// value/dimension size does not change in an iteration, we can deduce that
   /// it the same as the initial value/dimension.
@@ -77,11 +87,7 @@ struct ForOpInterface
     // `value` is result of `forOp`, we can prove that:
     // %result == %init_arg + trip_count * (%yielded_value - %iter_arg).
     // Where trip_count is (ub - lb) / step.
-    AffineExpr lbExpr = cstr.getExpr(forOp.getLowerBound());
-    AffineExpr ubExpr = cstr.getExpr(forOp.getUpperBound());
-    AffineExpr stepExpr = cstr.getExpr(forOp.getStep());
-    AffineExpr tripCountExpr =
-        AffineExpr(ubExpr - lbExpr).ceilDiv(stepExpr); // (ub - lb) / step
+    AffineExpr tripCountExpr = getTripCountExpr(forOp, cstr);
     AffineExpr oneIterAdvanceExpr =
         cstr.getExpr(yieldedValue) - cstr.getExpr(iterArg);
     cstr.bound(value) ==
@@ -93,9 +99,18 @@ struct ForOpInterface
     auto forOp = cast<ForOp>(op);
 
     if (value == forOp.getInductionVar()) {
-      // TODO: Take into account step size.
       cstr.bound(value) >= forOp.getLowerBound();
       cstr.bound(value) < forOp.getUpperBound();
+      // iv <= lb + ((ub-lb)/step - 1) * step
+      // This bound does not replace the `iv < ub` constraint mentioned above,
+      // since constraints involving the multiplication of two constraint set
+      // dimensions are not supported.
+      AffineExpr tripCountMinusOne =
+          getTripCountExpr(forOp, cstr) - cstr.getExpr(1);
+      AffineExpr computedUpperBound =
+          cstr.getExpr(forOp.getLowerBound()) +
+          AffineExpr(tripCountMinusOne * cstr.getExpr(forOp.getStep()));
+      cstr.bound(value) <= computedUpperBound;
       return;
     }
 
diff --git a/mlir/lib/Dialect/SPIRV/IR/SPIRVOps.cpp b/mlir/lib/Dialect/SPIRV/IR/SPIRVOps.cpp
index 040bf6a34cea7..dc414339ae7b8 100644
--- a/mlir/lib/Dialect/SPIRV/IR/SPIRVOps.cpp
+++ b/mlir/lib/Dialect/SPIRV/IR/SPIRVOps.cpp
@@ -917,7 +917,7 @@ ParseResult spirv::FuncOp::parse(OpAsmParser &parser, OperationState &result) {
 
   // Parse the function signature.
   bool isVariadic = false;
-  if (function_interface_impl::parseFunctionSignature(
+  if (function_interface_impl::parseFunctionSignatureWithArguments(
           parser, /*allowVariadic=*/false, entryArgs, isVariadic, resultTypes,
           resultAttrs))
     return failure();
@@ -940,7 +940,7 @@ ParseResult spirv::FuncOp::parse(OpAsmParser &parser, OperationState &result) {
 
   // Add the attributes to the function arguments.
   assert(resultAttrs.size() == resultTypes.size());
-  function_interface_impl::addArgAndResultAttrs(
+  call_interface_impl::addArgAndResultAttrs(
       builder, result, entryArgs, resultAttrs, getArgAttrsAttrName(result.name),
       getResAttrsAttrName(result.name));
 
@@ -1717,10 +1717,32 @@ LogicalResult spirv::MatrixTimesVectorOp::verify() {
            << resultType.getNumElements() << ") must match the matrix rows ("
            << matrixType.getNumRows() << ")";
 
-  auto matrixElementType = matrixType.getElementType();
-  if (matrixElementType != vectorType.getElementType() ||
-      matrixElementType != resultType.getElementType())
-    return emitOpError("matrix, vector, and result element types must match");
+  if (matrixType.getElementType() != resultType.getElementType())
+    return emitOpError("matrix and result element types must match");
+
+  return success();
+}
+
+//===----------------------------------------------------------------------===//
+// spirv.VectorTimesMatrix
+//===----------------------------------------------------------------------===//
+
+LogicalResult spirv::VectorTimesMatrixOp::verify() {
+  auto vectorType = llvm::cast<VectorType>(getVector().getType());
+  auto matrixType = llvm::cast<spirv::MatrixType>(getMatrix().getType());
+  auto resultType = llvm::cast<VectorType>(getType());
+
+  if (matrixType.getNumRows() != vectorType.getNumElements())
+    return emitOpError("number of components in vector must equal the number "
+                       "of components in each column in matrix");
+
+  if (resultType.getNumElements() != matrixType.getNumColumns())
+    return emitOpError("number of columns in matrix must equal the number of "
+                       "components in result");
+
+  if (matrixType.getElementType() != resultType.getElementType())
+    return emitOpError("matrix must be a matrix with the same component type "
+                       "as the component type in result");
 
   return success();
 }
@@ -2020,6 +2042,45 @@ LogicalResult spirv::ImageDrefGatherOp::verify() {
   return verifyImageOperands(*this, attr, operandArguments);
 }
 
+//===----------------------------------------------------------------------===//
+// spirv.ImageWriteOp
+//===----------------------------------------------------------------------===//
+
+LogicalResult spirv::ImageWriteOp::verify() {
+  ImageType imageType = cast<ImageType>(getImage().getType());
+  Type sampledType = imageType.getElementType();
+  ImageSamplerUseInfo samplerInfo = imageType.getSamplerUseInfo();
+
+  if (!llvm::is_contained({spirv::ImageSamplerUseInfo::SamplerUnknown,
+                           spirv::ImageSamplerUseInfo::NoSampler},
+                          samplerInfo)) {
+    return emitOpError(
+        "the sampled operand of the underlying image must be 0 or 2");
+  }
+
+  // TODO: Do we need check for: "If the Arrayed operand is 1, then additional
+  // capabilities may be required; e.g., ImageCubeArray, or ImageMSArray."?
+
+  if (imageType.getDim() == spirv::Dim::SubpassData) {
+    return emitOpError(
+        "the Dim operand of the underlying image must not be SubpassData");
+  }
+
+  Type texelType = getElementTypeOrSelf(getTexel());
+  if (!isa<NoneType>(sampledType) && texelType != sampledType) {
+    return emitOpError(
+        "the texel component type must match the image sampled type");
+  }
+
+  // TODO: Ideally it should be somewhere verified that "The Image Format must
+  // not be Unknown, unless the StorageImageWriteWithoutFormat Capability was
+  // declared." This function however may not be the suitable place for such
+  // verification.
+
+  return verifyImageOperands(*this, getImageOperandsAttr(),
+                             getOperandArguments());
+}
+
 //===----------------------------------------------------------------------===//
 // spirv.ShiftLeftLogicalOp
 //===----------------------------------------------------------------------===//
diff --git a/mlir/lib/Dialect/Shape/IR/Shape.cpp b/mlir/lib/Dialect/Shape/IR/Shape.cpp
index 65efc88e9c403..2200af0f67a86 100644
--- a/mlir/lib/Dialect/Shape/IR/Shape.cpp
+++ b/mlir/lib/Dialect/Shape/IR/Shape.cpp
@@ -1297,7 +1297,7 @@ void FuncOp::build(OpBuilder &builder, OperationState &state, StringRef name,
   if (argAttrs.empty())
     return;
   assert(type.getNumInputs() == argAttrs.size());
-  function_interface_impl::addArgAndResultAttrs(
+  call_interface_impl::addArgAndResultAttrs(
       builder, state, argAttrs, /*resultAttrs=*/std::nullopt,
       getArgAttrsAttrName(state.name), getResAttrsAttrName(state.name));
 }
diff --git a/mlir/lib/Dialect/SparseTensor/Pipelines/CMakeLists.txt b/mlir/lib/Dialect/SparseTensor/Pipelines/CMakeLists.txt
index 427dbe91be472..c724b49762a1a 100644
--- a/mlir/lib/Dialect/SparseTensor/Pipelines/CMakeLists.txt
+++ b/mlir/lib/Dialect/SparseTensor/Pipelines/CMakeLists.txt
@@ -24,6 +24,7 @@ add_mlir_dialect_library(MLIRSparseTensorPipelines
   MLIRSparseTensorDialect
   MLIRSparseTensorTransforms
   MLIRTensorTransforms
+  MLIRUBToLLVM
   MLIRVectorToLLVMPass
   MLIRVectorTransforms
 )
diff --git a/mlir/lib/Dialect/SparseTensor/Pipelines/SparseTensorPipelines.cpp b/mlir/lib/Dialect/SparseTensor/Pipelines/SparseTensorPipelines.cpp
index 887ab1d40caba..018233dc767b4 100644
--- a/mlir/lib/Dialect/SparseTensor/Pipelines/SparseTensorPipelines.cpp
+++ b/mlir/lib/Dialect/SparseTensor/Pipelines/SparseTensorPipelines.cpp
@@ -6,23 +6,15 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "mlir/Dialect/SparseTensor/Pipelines/Passes.h"
-
-#include "mlir/Conversion/ArithToLLVM/ArithToLLVM.h"
-#include "mlir/Conversion/ControlFlowToLLVM/ControlFlowToLLVM.h"
-#include "mlir/Conversion/GPUToNVVM/GPUToNVVMPass.h"
 #include "mlir/Conversion/Passes.h"
 #include "mlir/Dialect/Arith/Transforms/Passes.h"
-#include "mlir/Dialect/Bufferization/Transforms/Bufferize.h"
-#include "mlir/Dialect/Bufferization/Transforms/OneShotAnalysis.h"
 #include "mlir/Dialect/Bufferization/Transforms/Passes.h"
 #include "mlir/Dialect/Func/IR/FuncOps.h"
 #include "mlir/Dialect/GPU/IR/GPUDialect.h"
 #include "mlir/Dialect/GPU/Transforms/Passes.h"
-#include "mlir/Dialect/LLVMIR/NVVMDialect.h"
 #include "mlir/Dialect/Linalg/Passes.h"
 #include "mlir/Dialect/MemRef/Transforms/Passes.h"
-#include "mlir/Dialect/SparseTensor/IR/SparseTensor.h"
+#include "mlir/Dialect/SparseTensor/Pipelines/Passes.h"
 #include "mlir/Dialect/SparseTensor/Transforms/Passes.h"
 #include "mlir/Pass/PassManager.h"
 #include "mlir/Transforms/Passes.h"
@@ -107,6 +99,9 @@ void mlir::sparse_tensor::buildSparsifier(OpPassManager &pm,
     pm.addPass(createGpuModuleToBinaryPass(gpuModuleToBinaryPassOptions));
   }
 
+  // Convert poison values.
+  pm.addPass(createUBToLLVMConversionPass());
+
   // Ensure all casts are realized.
   pm.addPass(createReconcileUnrealizedCastsPass());
 }
diff --git a/mlir/lib/Dialect/SparseTensor/Transforms/SparseGPUCodegen.cpp b/mlir/lib/Dialect/SparseTensor/Transforms/SparseGPUCodegen.cpp
index b7fac163ba5fe..a988b2f4f1f4a 100644
--- a/mlir/lib/Dialect/SparseTensor/Transforms/SparseGPUCodegen.cpp
+++ b/mlir/lib/Dialect/SparseTensor/Transforms/SparseGPUCodegen.cpp
@@ -1269,7 +1269,7 @@ struct LinalgOpRewriter : public OpRewritePattern<linalg::GenericOp> {
     AffineExpr i, j, k;
     bindDims(getContext(), i, j, k);
 
-    // TODO: more robust patterns, tranposed versions, more kernels,
+    // TODO: more robust patterns, transposed versions, more kernels,
     //       identify alpha and beta and pass them to the CUDA calls.
 
     // Recognize a SpMV kernel.
diff --git a/mlir/lib/Dialect/SparseTensor/Transforms/SparseStorageSpecifierToLLVM.cpp b/mlir/lib/Dialect/SparseTensor/Transforms/SparseStorageSpecifierToLLVM.cpp
index d4ddde8afec2a..01028f71c20bb 100644
--- a/mlir/lib/Dialect/SparseTensor/Transforms/SparseStorageSpecifierToLLVM.cpp
+++ b/mlir/lib/Dialect/SparseTensor/Transforms/SparseStorageSpecifierToLLVM.cpp
@@ -110,7 +110,7 @@ class SpecifierStructBuilder : public StructBuilder {
 
 Value SpecifierStructBuilder::getInitValue(OpBuilder &builder, Location loc,
                                            Type structType, Value source) {
-  Value metaData = builder.create<LLVM::UndefOp>(loc, structType);
+  Value metaData = builder.create<LLVM::PoisonOp>(loc, structType);
   SpecifierStructBuilder md(metaData);
   if (!source) {
     auto memSizeArrayType =
diff --git a/mlir/lib/Dialect/Tensor/IR/CMakeLists.txt b/mlir/lib/Dialect/Tensor/IR/CMakeLists.txt
index cfdd3847761a4..d9d09d6361a2f 100644
--- a/mlir/lib/Dialect/Tensor/IR/CMakeLists.txt
+++ b/mlir/lib/Dialect/Tensor/IR/CMakeLists.txt
@@ -16,6 +16,7 @@ add_mlir_dialect_library(MLIRTensorDialect
 
   DEPENDS
   MLIRTensorOpsIncGen
+  MLIRTensorInterfacesIncGen
 
   LINK_LIBS PUBLIC
   MLIRAffineDialect
@@ -26,6 +27,8 @@ add_mlir_dialect_library(MLIRTensorDialect
   MLIRDestinationStyleOpInterface
   MLIRDialectUtils
   MLIRIR
+  MLIRInferIntRangeCommon
+  MLIRInferIntRangeInterface
   MLIRInferTypeOpInterface
   MLIRParallelCombiningOpInterface
   MLIRShapedOpInterfaces
diff --git a/mlir/lib/Dialect/Tensor/IR/TensorInferTypeOpInterfaceImpl.cpp b/mlir/lib/Dialect/Tensor/IR/TensorInferTypeOpInterfaceImpl.cpp
index 7ff435a033985..f6fea08e2e717 100644
--- a/mlir/lib/Dialect/Tensor/IR/TensorInferTypeOpInterfaceImpl.cpp
+++ b/mlir/lib/Dialect/Tensor/IR/TensorInferTypeOpInterfaceImpl.cpp
@@ -16,24 +16,6 @@
 using namespace mlir;
 using namespace mlir::tensor;
 
-/// Compute a map that for a given dimension of the expanded type gives the
-/// dimension in the collapsed type it maps to. Essentially its the inverse of
-/// the `reassocation` maps.
-static llvm::DenseMap<int64_t, int64_t>
-getExpandedDimToCollapsedDimMap(ArrayRef<AffineMap> reassociation) {
-  llvm::DenseMap<int64_t, int64_t> expandedDimToCollapsedDim;
-  for (const auto &map : enumerate(reassociation)) {
-    unsigned startPos =
-        cast<AffineDimExpr>(map.value().getResults().front()).getPosition();
-    unsigned endPos =
-        cast<AffineDimExpr>(map.value().getResults().back()).getPosition();
-    for (auto dim : llvm::seq_inclusive(startPos, endPos)) {
-      expandedDimToCollapsedDim[dim] = map.index();
-    }
-  }
-  return expandedDimToCollapsedDim;
-}
-
 /// For reshape op compute the shape at dimension `dimIndex` of the output in
 /// terms of shape of the `src`, when the reshape op is a collapsing
 /// operation. It is the product of the shape of the collapsed dimensions of the
@@ -76,84 +58,15 @@ static SmallVector<OpFoldResult, 4> getCollapsedOutputShapeFromInputShape(
       }));
 }
 
-/// For an expanding reshape op, compute the value for a dimension of the output
-/// from the shape of the input.
-static OpFoldResult getExpandedOutputDimFromInputShape(
-    OpBuilder &builder, Location loc, int64_t dimIndex, Value src,
-    ArrayRef<int64_t> dstStaticShape, ArrayRef<AffineMap> reassociation,
-    llvm::DenseMap<int64_t, int64_t> &expandedDimToCollapsedDim) {
-  if (!ShapedType::isDynamic(dstStaticShape[dimIndex])) {
-    // Static dimension: return Attribute.
-    return builder.getIndexAttr(dstStaticShape[dimIndex]);
-  }
-  unsigned sourceDimPos = expandedDimToCollapsedDim[dimIndex];
-  unsigned startPos =
-      cast<AffineDimExpr>(reassociation[sourceDimPos].getResults().front())
-          .getPosition();
-  unsigned endPos =
-      cast<AffineDimExpr>(reassociation[sourceDimPos].getResults().back())
-          .getPosition();
-  int64_t linearizedStaticDim = 1;
-  for (auto d :
-       llvm::enumerate(dstStaticShape.slice(startPos, endPos - startPos + 1))) {
-    if (d.index() + startPos == static_cast<unsigned>(dimIndex))
-      continue;
-    assert(!ShapedType::isDynamic(d.value()) &&
-           "single dimension cannot be expanded into multiple dynamic "
-           "dimensions");
-    linearizedStaticDim *= d.value();
-  }
-  OpFoldResult sourceDim =
-      builder.create<tensor::DimOp>(loc, src, sourceDimPos).getResult();
-
-  // Dynamic dimension: return Value.
-  return affine::makeComposedAffineApply(
-             builder, loc,
-             AffineMap::get(
-                 0, 1,
-                 builder.getAffineSymbolExpr(0).floorDiv(linearizedStaticDim)),
-             sourceDim)
-      ->getResult(0);
-}
-
-/// Given the `src` of an expanding reshape op, the reassociation maps and the
-/// result type, compute the shape of the result of the reshape.
-static SmallVector<OpFoldResult, 4> getExpandedOutputShapeFromInputShape(
-    OpBuilder &builder, Location loc, Value src,
-    ArrayRef<int64_t> dstStaticShape, ArrayRef<AffineMap> reassociation) {
-  llvm::DenseMap<int64_t, int64_t> expandedDimToCollapsedDim =
-      getExpandedDimToCollapsedDimMap(reassociation);
-  return llvm::to_vector<4>(llvm::map_range(
-      llvm::seq<int64_t>(0, dstStaticShape.size()), [&](int64_t dim) {
-        return getExpandedOutputDimFromInputShape(builder, loc, dim, src,
-                                                  dstStaticShape, reassociation,
-                                                  expandedDimToCollapsedDim);
-      }));
-}
-
-static SmallVector<OpFoldResult, 4>
-getReshapeOutputShapeFromInputShape(OpBuilder &builder, Location loc, Value src,
-                                    ArrayRef<int64_t> dstStaticShape,
-                                    ArrayRef<AffineMap> reassocation) {
-  return dstStaticShape.size() >
-                 static_cast<size_t>(
-                     llvm::cast<ShapedType>(src.getType()).getRank())
-             ? getExpandedOutputShapeFromInputShape(
-                   builder, loc, src, dstStaticShape, reassocation)
-             : getCollapsedOutputShapeFromInputShape(
-                   builder, loc, src, dstStaticShape, reassocation);
-}
-
-template <typename OpTy>
-struct ReifyExpandOrCollapseShapeOp
+struct ReifyCollapseShapeOp
     : public ReifyRankedShapedTypeOpInterface::ExternalModel<
-          ReifyExpandOrCollapseShapeOp<OpTy>, OpTy> {
+          ReifyCollapseShapeOp, CollapseShapeOp> {
   LogicalResult
   reifyResultShapes(Operation *op, OpBuilder &b,
                     ReifiedRankedShapedTypeDims &reifiedReturnShapes) const {
     auto loc = op->getLoc();
-    auto reshapeOp = cast<OpTy>(op);
-    reifiedReturnShapes.push_back(getReshapeOutputShapeFromInputShape(
+    auto reshapeOp = cast<tensor::CollapseShapeOp>(op);
+    reifiedReturnShapes.push_back(getCollapsedOutputShapeFromInputShape(
         b, loc, reshapeOp.getSrc(), reshapeOp.getResultType().getShape(),
         reshapeOp.getReassociationMaps()));
     return success();
@@ -162,6 +75,20 @@ struct ReifyExpandOrCollapseShapeOp
 
 namespace {
 
+struct ReifyExpandShapeOp
+    : public ReifyRankedShapedTypeOpInterface::ExternalModel<ReifyExpandShapeOp,
+                                                             ExpandShapeOp> {
+  LogicalResult
+  reifyResultShapes(Operation *op, OpBuilder &b,
+                    ReifiedRankedShapedTypeDims &reifyResultShapes) const {
+    auto expandShapeOp = cast<tensor::ExpandShapeOp>(op);
+    SmallVector<OpFoldResult> resultShapes =
+        expandShapeOp.getMixedOutputShape();
+    reifyResultShapes.emplace_back(std::move(resultShapes));
+    return success();
+  }
+};
+
 struct ReifyPadOp
     : public ReifyRankedShapedTypeOpInterface::ExternalModel<ReifyPadOp,
                                                              PadOp> {
@@ -202,10 +129,8 @@ struct ReifyPadOp
 void mlir::tensor::registerInferTypeOpInterfaceExternalModels(
     DialectRegistry &registry) {
   registry.addExtension(+[](MLIRContext *ctx, TensorDialect *dialect) {
-    ExpandShapeOp::attachInterface<
-        ReifyExpandOrCollapseShapeOp<tensor::ExpandShapeOp>>(*ctx);
-    CollapseShapeOp::attachInterface<
-        ReifyExpandOrCollapseShapeOp<tensor::CollapseShapeOp>>(*ctx);
+    ExpandShapeOp::attachInterface<ReifyExpandShapeOp>(*ctx);
+    CollapseShapeOp::attachInterface<ReifyCollapseShapeOp>(*ctx);
     PadOp::attachInterface<ReifyPadOp>(*ctx);
   });
 }
diff --git a/mlir/lib/Dialect/Tensor/IR/TensorOps.cpp b/mlir/lib/Dialect/Tensor/IR/TensorOps.cpp
index 24a1d55315319..fda6246334e15 100644
--- a/mlir/lib/Dialect/Tensor/IR/TensorOps.cpp
+++ b/mlir/lib/Dialect/Tensor/IR/TensorOps.cpp
@@ -23,7 +23,9 @@
 #include "mlir/IR/OpDefinition.h"
 #include "mlir/IR/TypeUtilities.h"
 #include "mlir/Interfaces/DestinationStyleOpInterface.h"
+#include "mlir/Interfaces/InferIntRangeInterface.h"
 #include "mlir/Interfaces/LoopLikeInterface.h"
+#include "mlir/Interfaces/Utils/InferIntRangeCommon.h"
 #include "mlir/Support/LLVM.h"
 #include "llvm/ADT/DenseSet.h"
 #include "llvm/ADT/STLExtras.h"
@@ -782,6 +784,12 @@ Speculation::Speculatability DimOp::getSpeculatability() {
   return Speculation::Speculatable;
 }
 
+void DimOp::inferResultRangesFromOptional(ArrayRef<IntegerValueRange> argRanges,
+                                          SetIntLatticeFn setResultRange) {
+  setResultRange(getResult(),
+                 intrange::inferShapedDimOpInterface(*this, argRanges[1]));
+}
+
 OpFoldResult DimOp::fold(FoldAdaptor adaptor) {
   // All forms of folding require a known index.
   auto index = llvm::dyn_cast_if_present<IntegerAttr>(adaptor.getIndex());
@@ -1732,6 +1740,10 @@ ExpandShapeOp::inferOutputShape(OpBuilder &b, Location loc,
   return *outputShape;
 }
 
+SmallVector<OpFoldResult> ExpandShapeOp::getMixedOutputShape() {
+  return getMixedValues(getStaticOutputShape(), getOutputShape(), getContext());
+}
+
 void ExpandShapeOp::build(OpBuilder &builder, OperationState &result,
                           Type resultType, Value src,
                           ArrayRef<ReassociationIndices> reassociation,
@@ -4964,7 +4976,7 @@ struct FoldTensorCastProducerOp
 
     // Reject tensor::PackOp - there's dedicated pattern for that instead.
     if (!foldTensorCastPrecondition(op) ||
-        isa<tensor::PackOp, tensor::UnPackOp>(*op))
+        isa<tensor::RelayoutOpInterface>(*op))
       return failure();
 
     SmallVector<Type> newResultTypes(op->getResultTypes());
diff --git a/mlir/lib/Dialect/Tosa/IR/TosaCanonicalizations.cpp b/mlir/lib/Dialect/Tosa/IR/TosaCanonicalizations.cpp
index b8e0005dc1bc0..69b3f6d674167 100644
--- a/mlir/lib/Dialect/Tosa/IR/TosaCanonicalizations.cpp
+++ b/mlir/lib/Dialect/Tosa/IR/TosaCanonicalizations.cpp
@@ -111,8 +111,8 @@ struct ConsolidateTransposeOptimization
     auto permsTy =
         RankedTensorType::get(transposePerms.size(), rewriter.getI32Type());
     auto permsAttr = DenseIntElementsAttr::get(permsTy, perms);
-    Value permsValue =
-        rewriter.create<arith::ConstantOp>(transposeOp.getLoc(), permsAttr);
+    Value permsValue = rewriter.create<tosa::ConstOp>(transposeOp.getLoc(),
+                                                      permsTy, permsAttr);
 
     rewriter.replaceOpWithNewOp<tosa::TransposeOp>(
         transposeOp, transposeOp.getResult().getType(),
@@ -180,7 +180,7 @@ struct TransposeIsReshape : public OpRewritePattern<tosa::TransposeOp> {
 
     rewriter.replaceOpWithNewOp<tosa::ReshapeOp>(
         op, op.getType(), op.getInput1(),
-        rewriter.getDenseI64ArrayAttr(newShape));
+        getTosaConstShape(rewriter, op.getLoc(), newShape));
     return success();
   }
 };
@@ -207,10 +207,10 @@ struct MaterializePadValue : public OpRewritePattern<tosa::PadOp> {
     Attribute constantAttr;
     if (llvm::isa<FloatType>(elementTy)) {
       constantAttr = rewriter.getFloatAttr(elementTy, 0.0);
-    } else if (llvm::isa<IntegerType>(elementTy) && !op.getQuantizationInfo()) {
+    } else if (llvm::isa<IntegerType>(elementTy) && !op.getInputZpAttr()) {
       constantAttr = rewriter.getIntegerAttr(elementTy, 0);
-    } else if (llvm::isa<IntegerType>(elementTy) && op.getQuantizationInfo()) {
-      auto value = op.getQuantizationInfo()->getInputZp();
+    } else if (llvm::isa<IntegerType>(elementTy) && op.getInputZpAttr()) {
+      int64_t value = op.getInputZpAttr().getInt();
       constantAttr = rewriter.getIntegerAttr(elementTy, value);
     }
 
@@ -287,10 +287,12 @@ struct ClampIsNoOp : public OpRewritePattern<tosa::ClampOp> {
 
     if (isa<FloatType>(inputElementType)) {
       // Unlike integer types, floating point types can represent infinity.
-      auto minClamp = op.getMinFp();
-      auto maxClamp = op.getMaxFp();
-      bool isMin = minClamp.isInfinity() && minClamp.isNegative();
-      bool isMax = maxClamp.isInfinity() && !maxClamp.isNegative();
+      auto minClamp =
+          llvm::cast<mlir::FloatAttr>(op.getMinValAttr()).getValue();
+      auto maxClamp =
+          llvm::cast<mlir::FloatAttr>(op.getMaxValAttr()).getValue();
+      bool isMin = minClamp.isNegInfinity();
+      bool isMax = maxClamp.isInfinity();
 
       if (isMin && isMax) {
         rewriter.replaceOp(op, input);
@@ -300,8 +302,10 @@ struct ClampIsNoOp : public OpRewritePattern<tosa::ClampOp> {
     }
 
     if (inputElementType.isUnsignedInteger()) {
-      int64_t minClamp = op.getMinInt();
-      int64_t maxClamp = op.getMaxInt();
+      int64_t minClamp =
+          llvm::cast<mlir::IntegerAttr>(op.getMinValAttr()).getUInt();
+      int64_t maxClamp =
+          llvm::cast<mlir::IntegerAttr>(op.getMaxValAttr()).getUInt();
 
       int64_t intMin =
           APInt::getMinValue(inputElementType.getIntOrFloatBitWidth())
@@ -318,8 +322,10 @@ struct ClampIsNoOp : public OpRewritePattern<tosa::ClampOp> {
     }
 
     if (llvm::isa<IntegerType>(inputElementType)) {
-      int64_t minClamp = op.getMinInt();
-      int64_t maxClamp = op.getMaxInt();
+      int64_t minClamp =
+          llvm::cast<mlir::IntegerAttr>(op.getMinValAttr()).getInt();
+      int64_t maxClamp =
+          llvm::cast<mlir::IntegerAttr>(op.getMaxValAttr()).getInt();
 
       int64_t intMin =
           APInt::getSignedMinValue(inputElementType.getIntOrFloatBitWidth())
@@ -374,9 +380,10 @@ struct ClampClampOptimization : public OpRewritePattern<tosa::ClampOp> {
 
   LogicalResult matchAndRewrite(tosa::ClampOp op,
                                 PatternRewriter &rewriter) const override {
+    Value input = op.getInput();
+
     // Check the input to the CLAMP op is itself a CLAMP.
-    auto clampOp =
-        dyn_cast_if_present<tosa::ClampOp>(op.getInput().getDefiningOp());
+    auto clampOp = dyn_cast_if_present<tosa::ClampOp>(input.getDefiningOp());
     if (!clampOp)
       return failure();
 
@@ -386,34 +393,86 @@ struct ClampClampOptimization : public OpRewritePattern<tosa::ClampOp> {
     if (opNanMode == "IGNORE" && clampNanMode == "PROPAGATE")
       return failure();
 
-    // Check we have intersecting ranges.
-    const auto opMinInt = op.getMinInt();
-    const auto opMaxInt = op.getMaxInt();
-    const auto clampOpMinInt = clampOp.getMinInt();
-    const auto clampOpMaxInt = clampOp.getMaxInt();
-    ClampRange<std::int64_t> opRangeIntRange(opMinInt, opMaxInt);
-    ClampRange<std::int64_t> clampRangeIntRange(clampOpMinInt, clampOpMaxInt);
-    if (!opRangeIntRange.intersects(clampRangeIntRange))
-      return failure();
+    auto maxValAttr = op.getMaxValAttr();
+    auto minValAttr = op.getMinValAttr();
+    auto clampOpMaxValAttr = clampOp.getMaxValAttr();
+    auto clampOpMinValAttr = clampOp.getMinValAttr();
 
-    const auto opMinFloat = op.getMinFp();
-    const auto opMaxFloat = op.getMaxFp();
-    const auto clampOpMinFloat = clampOp.getMinFp();
-    const auto clampOpMaxFloat = clampOp.getMaxFp();
-    ClampRange<APFloat> opRangeFloatRange(opMinFloat, opMaxFloat);
-    ClampRange<APFloat> clampRangeFloatRange(clampOpMinFloat, clampOpMaxFloat);
-    if (!opRangeFloatRange.intersects(clampRangeFloatRange))
-      return failure();
+    auto inputEType = llvm::cast<ShapedType>(input.getType()).getElementType();
+    if (auto quantType =
+            llvm::dyn_cast<mlir::quant::UniformQuantizedType>(inputEType)) {
+      inputEType = quantType.getStorageType();
+    }
+
+    Attribute newMinValAttr, newMaxValAttr;
+    if (mlir::isa<FloatType>(inputEType)) {
+      auto floatMaxValAttr = cast<mlir::FloatAttr>(maxValAttr);
+      auto floatMinValAttr = cast<mlir::FloatAttr>(minValAttr);
+      auto clampOpFloatMaxValAttr = cast<mlir::FloatAttr>(clampOpMaxValAttr);
+      auto clampOpFloatMinValAttr = cast<mlir::FloatAttr>(clampOpMinValAttr);
+
+      // Check we have intersecting ranges.
+      const auto opMinFloat = floatMinValAttr.getValue();
+      const auto opMaxFloat = floatMaxValAttr.getValue();
+      const auto clampOpMinFloat = clampOpFloatMinValAttr.getValue();
+      const auto clampOpMaxFloat = clampOpFloatMaxValAttr.getValue();
+      ClampRange<APFloat> opRangeFloatRange(opMinFloat, opMaxFloat);
+      ClampRange<APFloat> clampRangeFloatRange(clampOpMinFloat,
+                                               clampOpMaxFloat);
+      if (!opRangeFloatRange.intersects(clampRangeFloatRange))
+        return failure();
+
+      // Run the transformation.
+      auto newMinVal = std::max(opMinFloat, clampOpMinFloat);
+      auto newMaxVal = std::min(opMaxFloat, clampOpMaxFloat);
+      newMinValAttr = rewriter.getFloatAttr(inputEType, newMinVal);
+      newMaxValAttr = rewriter.getFloatAttr(inputEType, newMaxVal);
+    } else {
+      assert(mlir::isa<IntegerType>(inputEType));
+      auto intMaxValAttr = cast<mlir::IntegerAttr>(maxValAttr);
+      auto intMinValAttr = cast<mlir::IntegerAttr>(minValAttr);
+      auto clampOpIntMaxValAttr = cast<mlir::IntegerAttr>(clampOpMaxValAttr);
+      auto clampOpIntMinValAttr = cast<mlir::IntegerAttr>(clampOpMinValAttr);
+
+      if (inputEType.isUnsignedInteger()) {
+        // Check we have intersecting ranges.
+        const auto opMinInt = intMinValAttr.getUInt();
+        const auto opMaxInt = intMaxValAttr.getUInt();
+        const auto clampOpMinInt = clampOpIntMinValAttr.getUInt();
+        const auto clampOpMaxInt = clampOpIntMaxValAttr.getUInt();
+        ClampRange<std::uint64_t> opRangeIntRange(opMinInt, opMaxInt);
+        ClampRange<std::uint64_t> clampRangeIntRange(clampOpMinInt,
+                                                     clampOpMaxInt);
+        if (!opRangeIntRange.intersects(clampRangeIntRange))
+          return failure();
+
+        // Run the transformation.
+        auto newMinVal = std::max(opMinInt, clampOpMinInt);
+        auto newMaxVal = std::min(opMaxInt, clampOpMaxInt);
+        newMinValAttr = rewriter.getIntegerAttr(inputEType, newMinVal);
+        newMaxValAttr = rewriter.getIntegerAttr(inputEType, newMaxVal);
+      } else {
+        // Check we have intersecting ranges.
+        const auto opMinInt = intMinValAttr.getInt();
+        const auto opMaxInt = intMaxValAttr.getInt();
+        const auto clampOpMinInt = clampOpIntMinValAttr.getInt();
+        const auto clampOpMaxInt = clampOpIntMaxValAttr.getInt();
+        ClampRange<std::int64_t> opRangeIntRange(opMinInt, opMaxInt);
+        ClampRange<std::int64_t> clampRangeIntRange(clampOpMinInt,
+                                                    clampOpMaxInt);
+        if (!opRangeIntRange.intersects(clampRangeIntRange))
+          return failure();
+
+        // Run the transformation.
+        auto newMinVal = std::max(opMinInt, clampOpMinInt);
+        auto newMaxVal = std::min(opMaxInt, clampOpMaxInt);
+        newMinValAttr = rewriter.getIntegerAttr(inputEType, newMinVal);
+        newMaxValAttr = rewriter.getIntegerAttr(inputEType, newMaxVal);
+      }
+    }
 
-    // Run the transformation.
-    const auto minFp = std::max(opMinFloat, clampOpMinFloat).convertToFloat();
-    const auto maxFp = std::min(opMaxFloat, clampOpMaxFloat).convertToFloat();
-    const auto minInt = std::max(opMinInt, clampOpMinInt);
-    const auto maxInt = std::min(opMaxInt, clampOpMaxInt);
     rewriter.replaceOpWithNewOp<tosa::ClampOp>(
-        op, op.getType(), clampOp.getInput(),
-        rewriter.getI64IntegerAttr(minInt), rewriter.getI64IntegerAttr(maxInt),
-        rewriter.getF32FloatAttr(minFp), rewriter.getF32FloatAttr(maxFp),
+        op, op.getType(), clampOp.getInput(), newMinValAttr, newMaxValAttr,
         rewriter.getStringAttr((opNanMode != clampNanMode) ? "IGNORE"
                                                            : opNanMode));
     return success();
@@ -444,8 +503,21 @@ struct ConcatSliceOptimization : public OpRewritePattern<tosa::SliceOp> {
           sliceOp, "slice input must be a static ranked tensor");
     int32_t axis = concatOp.getAxis();
 
-    llvm::SmallVector<int64_t> sliceStart(sliceOp.getStart());
-    llvm::ArrayRef<int64_t> sliceSize = sliceOp.getSize();
+    DenseElementsAttr startElems;
+    DenseElementsAttr sizeElems;
+
+    if (!matchPattern(sliceOp.getStart(), m_Constant(&startElems)))
+      return rewriter.notifyMatchFailure(
+          sliceOp, "start of slice must be a static ranked shape");
+
+    if (!matchPattern(sliceOp.getSize(), m_Constant(&sizeElems)))
+      return rewriter.notifyMatchFailure(
+          sliceOp, "size of slice must be a static ranked shape");
+
+    llvm::SmallVector<int64_t> sliceStarts =
+        llvm::to_vector(startElems.getValues<int64_t>());
+    llvm::SmallVector<int64_t> sliceSizes =
+        llvm::to_vector(sizeElems.getValues<int64_t>());
 
     // Validate slice on the concatenated axis. Slicing along this
     // axis should span only one of the inputs to the concatenate
@@ -457,17 +529,20 @@ struct ConcatSliceOptimization : public OpRewritePattern<tosa::SliceOp> {
         return rewriter.notifyMatchFailure(
             sliceOp, "concat input must be a static ranked tensor");
 
-      if (sliceStart[axis] >= 0 &&
-          (sliceStart[axis] + sliceSize[axis]) <= inputType.getDimSize(axis)) {
-        replaceWithSlice = rewriter
-                               .create<tosa::SliceOp>(
-                                   sliceOp.getLoc(), sliceOp.getType(), input,
-                                   rewriter.getDenseI64ArrayAttr(sliceStart),
-                                   rewriter.getDenseI64ArrayAttr(sliceSize))
-                               .getResult();
+      if (sliceStarts[axis] >= 0 && (sliceStarts[axis] + sliceSizes[axis]) <=
+                                        inputType.getDimSize(axis)) {
+        auto start_op =
+            getTosaConstShape(rewriter, sliceOp.getLoc(), sliceStarts);
+        auto size_op =
+            getTosaConstShape(rewriter, sliceOp.getLoc(), sliceSizes);
+        replaceWithSlice =
+            rewriter
+                .create<tosa::SliceOp>(sliceOp.getLoc(), sliceOp.getType(),
+                                       input, start_op, size_op)
+                .getResult();
         break;
       }
-      sliceStart[axis] -= inputType.getDimSize(axis);
+      sliceStarts[axis] -= inputType.getDimSize(axis);
     }
 
     if (!replaceWithSlice)
@@ -665,7 +740,18 @@ OpFoldResult MulOp::fold(FoldAdaptor adaptor) {
   auto rhsAttr =
       llvm::dyn_cast_if_present<DenseElementsAttr>(adaptor.getInput2());
 
-  const int64_t shift = llvm::isa<IntegerType>(resultETy) ? getShift() : 0;
+  // Result right shift on i32_t data type only. For simplification, synthesize
+  // a zero shift for other data type.
+  int32_t shift = 0;
+  if (resultETy.isInteger(32)) {
+    ElementsAttr shift_elem;
+    if (getShift().getImpl()) {
+      if (!matchPattern(getShift(), m_Constant(&shift_elem)))
+        // cannot be folded when the shift value is unknown.
+        return {};
+      shift = shift_elem.getValues<IntegerAttr>()[0].getInt();
+    }
+  }
 
   if (rhsTy == resultTy) {
     if (isSplatZero(resultETy, lhsAttr))
@@ -680,7 +766,7 @@ OpFoldResult MulOp::fold(FoldAdaptor adaptor) {
       return lhs;
   }
 
-  return mulBinaryFolder(lhsAttr, rhsAttr, resultTy, getShift());
+  return mulBinaryFolder(lhsAttr, rhsAttr, resultTy, shift);
 }
 
 OpFoldResult SubOp::fold(FoldAdaptor adaptor) {
@@ -921,8 +1007,12 @@ OpFoldResult ReshapeOp::fold(FoldAdaptor adaptor) {
     if (!getInput1().hasOneUse())
       return {};
 
+    llvm::SmallVector<int64_t> shapeVec;
+    if (!tosa::getConstShapeValue(getShape().getDefiningOp(), shapeVec))
+      return {};
+
     return operand.reshape(
-        llvm::cast<ShapedType>(operand.getType()).clone(getNewShape()));
+        llvm::cast<ShapedType>(operand.getType()).clone(shapeVec));
   }
 
   return {};
@@ -1014,7 +1104,12 @@ OpFoldResult SliceOp::fold(FoldAdaptor adaptor) {
 
   if (inputTy.hasStaticShape() && outputTy.hasStaticShape() &&
       outputTy.getNumElements() == 1) {
-    llvm::SmallVector<uint64_t> indices(getStart());
+    DenseElementsAttr startElems;
+    if (!matchPattern(getStart(), m_Constant(&startElems)))
+      return {};
+
+    llvm::SmallVector<uint64_t> indices =
+        llvm::to_vector(startElems.getValues<uint64_t>());
     auto value = operand.getValues<Attribute>()[indices];
     return SplatElementsAttr::get(outputTy, value);
   }
diff --git a/mlir/lib/Dialect/Tosa/IR/TosaOps.cpp b/mlir/lib/Dialect/Tosa/IR/TosaOps.cpp
index fdccce60fe1d8..4928be38476a9 100644
--- a/mlir/lib/Dialect/Tosa/IR/TosaOps.cpp
+++ b/mlir/lib/Dialect/Tosa/IR/TosaOps.cpp
@@ -217,53 +217,100 @@ void mlir::tosa::printTypeOrAttr(OpAsmPrinter &p, Operation *op, TypeAttr type,
 
 template <typename T>
 static LogicalResult verifyConvOp(T op) {
-  // All TOSA conv ops have an input() and weight().
+  // All TOSA conv ops have an input and weight arguments which must be ranked
+  // tensors.
   auto inputType = llvm::dyn_cast<RankedTensorType>(op.getInput().getType());
-
-  RankedTensorType weightType;
-  if constexpr (std::is_same_v<T, tosa::TransposeConv2DOp>)
-    weightType = llvm::dyn_cast<RankedTensorType>(op.getFilter().getType());
-  else
-    weightType = llvm::dyn_cast<RankedTensorType>(op.getWeight().getType());
-
-  // Must be ranked tensor types
   if (!inputType) {
     op.emitOpError("expect a ranked tensor for input, got ") << op.getInput();
     return failure();
   }
+
+  auto weightType = llvm::dyn_cast<RankedTensorType>(op.getWeight().getType());
   if (!weightType) {
-    if constexpr (std::is_same_v<T, tosa::TransposeConv2DOp>) {
-      op.emitOpError("expect a ranked tensor for filter, got ")
-          << op.getFilter();
-    } else {
-      op.emitOpError("expect a ranked tensor for weight, got ")
-          << op.getWeight();
-    }
+    op.emitOpError("expect a ranked tensor for weight, got ") << op.getWeight();
     return failure();
   }
 
   auto inputEType = inputType.getElementType();
   auto weightEType = weightType.getElementType();
+  auto biasEType =
+      llvm::cast<ShapedType>(op.getBias().getType()).getElementType();
+  auto resultEType =
+      llvm::cast<ShapedType>(op.getResult().getType()).getElementType();
+  bool biasIsFloat = llvm::isa<FloatType>(biasEType);
+  bool resultIsFloat = llvm::isa<FloatType>(resultEType);
+
+  if (auto quantType = llvm::dyn_cast<mlir::quant::QuantizedType>(inputEType))
+    inputEType = quantType.getStorageType();
 
-  bool inputIsQuant = !llvm::isa<FloatType>(inputEType);
-  bool weightIsQuant = !llvm::isa<FloatType>(weightEType);
+  if (auto quantType = llvm::dyn_cast<mlir::quant::QuantizedType>(biasEType))
+    biasEType = quantType.getStorageType();
 
-  // Either both must be quantized or both unquantized.
-  if (inputIsQuant != weightIsQuant) {
+  if (auto quantType = llvm::dyn_cast<mlir::quant::QuantizedType>(resultEType))
+    resultEType = quantType.getStorageType();
+
+  if (biasIsFloat && resultIsFloat && (biasEType != resultEType)) {
+    // for now, only enforce bias element type == result element type for
+    // float types.
+    op.emitOpError(
+        "expect both bias and result to have same element type, got ")
+        << biasEType << " and " << resultEType;
+    return failure();
+  }
+
+  if (isa<Float8E5M2Type>(inputEType) || isa<Float8E4M3FNType>(inputEType) ||
+      isa<Float8E5M2Type>(weightEType) || isa<Float8E4M3FNType>(weightEType)) {
+    if (inputEType != weightEType) {
+      op.emitOpError(
+          "expect both input and weight to have same element type, got ")
+          << inputEType << " and " << weightEType;
+      return failure();
+    }
+  }
+
+  bool inputIsFloat = llvm::isa<FloatType>(inputEType);
+  bool weightIsFloat = llvm::isa<FloatType>(weightEType);
+
+  // Either both must be float or both non-float.
+  if (inputIsFloat != weightIsFloat) {
     op.emitOpError(
         "expect both input and weight to be float or not together, got ")
         << inputEType << " and " << weightEType;
     return failure();
   }
 
-  // Quantized type must have constructed the quantizationattr, and unquantized
-  // types should not have a quantizationattr.
-  if ((inputIsQuant && !op.getQuantizationInfo()) ||
-      (!inputIsQuant && op.getQuantizationInfo())) {
-    op.emitOpError("quantizationattr is required for quantized type, and not "
-                   "allowed for float type");
+  // We require an explicit input zero point and weight zero point for i8
+  // convolution.
+  if (!op.getInputZp() && !op.getWeightZp())
+    return inputEType.isInteger(8) ? failure() : success();
+
+  ElementsAttr inputZpAttr;
+  ElementsAttr weightZpAttr;
+  if (!matchPattern(op.getInputZp(), m_Constant(&inputZpAttr)) ||
+      !matchPattern(op.getWeightZp(), m_Constant(&weightZpAttr))) {
+    op.emitOpError(
+        "bail out if the actual value of zero points cannot be determined");
     return failure();
   }
+
+  // Get and verify explicit zero points.
+  int64_t inputZpVal;
+  int64_t weightZpVal;
+
+  if (tosa::getZeroPoint(inputZpAttr, inputZpVal).failed() ||
+      tosa::verifyZeroPoint<T>(getElementTypeOrSelf(inputZpAttr), inputZpVal)
+          .failed()) {
+    op.emitOpError("input zero point must be zero for non-int8 integer types");
+    return failure();
+  }
+
+  if (tosa::getZeroPoint(weightZpAttr, weightZpVal).failed() ||
+      tosa::verifyZeroPoint<T>(getElementTypeOrSelf(weightZpAttr), weightZpVal)
+          .failed()) {
+    op.emitOpError("weight zero point must be zero for non-int8 integer types");
+    return failure();
+  }
+
   return success();
 }
 
@@ -296,8 +343,7 @@ static LogicalResult verifyConvOpModes(T op) {
   auto inputEType =
       llvm::cast<ShapedType>(op.getInput().getType()).getElementType();
 
-  if (auto quantType =
-          llvm::dyn_cast<mlir::quant::UniformQuantizedType>(inputEType))
+  if (auto quantType = llvm::dyn_cast<mlir::quant::QuantizedType>(inputEType))
     inputEType = quantType.getStorageType();
 
   auto accType = op.getAccType();
@@ -319,6 +365,55 @@ static LogicalResult verifyConvOpModes(T op) {
   if (inputEType.isF32() && !accType.isF32())
     return op.emitOpError("accumulator type for f32 tensor is not f32");
 
+  auto resultEType =
+      llvm::cast<ShapedType>(op.getResult().getType()).getElementType();
+
+  if (auto quantType = llvm::dyn_cast<mlir::quant::QuantizedType>(resultEType))
+    resultEType = quantType.getStorageType();
+
+  // check allowed input/result element types combinations
+  if ((inputEType.isInteger(8) && resultEType.isInteger(32)) ||
+      (inputEType.isInteger(16) && resultEType.isInteger(48)) ||
+      (isa<Float8E5M2Type>(inputEType) && resultEType.isF16()) ||
+      (isa<Float8E4M3FNType>(inputEType) && resultEType.isF16()) ||
+      (inputEType.isF16() && resultEType.isF16()) ||
+      (inputEType.isBF16() && resultEType.isBF16()) ||
+      (inputEType.isF32() && resultEType.isF32()))
+    return success();
+
+  return op.emitOpError("input/output element types are incompatible.");
+}
+
+// verify that inType and outType have same element types
+template <typename T>
+static LogicalResult verifySameElementTypes(T op, Type inType, Type outType) {
+  auto inputType = llvm::dyn_cast<TensorType>(inType);
+  auto outputType = llvm::dyn_cast<TensorType>(outType);
+  if (!inputType) {
+    op.emitOpError("expect shaped tensor for input, got ") << inType;
+    return failure();
+  }
+  if (!outputType) {
+    op.emitOpError("expect shaped tensor for output, got ") << outType;
+    return failure();
+  }
+  auto inputElementType = inputType.getElementType();
+  auto outputElementType = outputType.getElementType();
+  auto inputQuantType =
+      llvm::dyn_cast<mlir::quant::UniformQuantizedType>(inputElementType);
+  auto outputQuantType =
+      llvm::dyn_cast<mlir::quant::UniformQuantizedType>(outputElementType);
+  if ((inputElementType.isIntOrIndexOrFloat() || inputQuantType) &&
+      (outputElementType.isIntOrIndexOrFloat() || outputQuantType) &&
+      inputElementType != outputElementType) {
+    // only check if both element types are int/index/float/UniformQuantized
+    // eg, not sure how to check quant::QuantizedType
+    // this happens in test_conv2d_q_grouped_convolution in
+    // tfl-to-tosa-pipeline.mlir
+    op.emitOpError("expect input and output to have same element type, got ")
+        << inputElementType << " and " << outputElementType;
+    return failure();
+  }
   return success();
 }
 
@@ -381,26 +476,40 @@ LogicalResult tosa::ClampOp::verify() {
           llvm::dyn_cast<mlir::quant::UniformQuantizedType>(inputETy)) {
     inputETy = quantType.getStorageType();
   }
-  mlir::Type maxFpType = getMaxFpAttr().getType();
-  mlir::Type minFpType = getMinFpAttr().getType();
   mlir::Type outputETy =
       llvm::cast<ShapedType>(getOutput().getType()).getElementType();
   if (auto quantType =
           llvm::dyn_cast<mlir::quant::UniformQuantizedType>(outputETy)) {
     outputETy = quantType.getStorageType();
   }
-  unsigned dataTypeBitWidth = inputETy.getIntOrFloatBitWidth();
-
   if (inputETy != outputETy)
     return emitOpError("input/output element types are incompatible.");
 
-  // If input datatype is float, check that the two min/max_fp attributes
-  // share the same type and that their type is either the same of the input's
-  // datatype, or a float type whose bitwidth > input datatype bitwidth.
-  if (!inputETy.isInteger(dataTypeBitWidth)) {
-    if (((maxFpType != minFpType) ||
-         (maxFpType != inputETy && maxFpType.getIntOrFloatBitWidth() <=
-                                       inputETy.getIntOrFloatBitWidth())))
+  auto maxValAttr = getMaxValAttr();
+  auto minValAttr = getMinValAttr();
+
+  unsigned dataTypeBitWidth = inputETy.getIntOrFloatBitWidth();
+
+  if (inputETy.isInteger(dataTypeBitWidth)) {
+    // if input datatype is integer, check that the min_val/max_val attributes
+    // are integer attributes, and that their type is the same as the input's
+    // datatype
+    auto intMaxValAttr = mlir::dyn_cast<mlir::IntegerAttr>(maxValAttr);
+    auto intMinValAttr = mlir::dyn_cast<mlir::IntegerAttr>(minValAttr);
+    if (!intMaxValAttr || !intMinValAttr ||
+        (intMaxValAttr.getType() != intMinValAttr.getType()) ||
+        (intMaxValAttr.getType() != inputETy))
+      return emitOpError("min/max attributes types are incompatible with "
+                         "input/output element types.");
+  } else {
+    // otherwise, input datatype is float, check that the min_val/max_val
+    // attributes share the same type and that their type is the same as the
+    // input's datatype
+    auto floatMaxValAttr = mlir::dyn_cast<mlir::FloatAttr>(maxValAttr);
+    auto floatMinValAttr = mlir::dyn_cast<mlir::FloatAttr>(minValAttr);
+    if (!floatMaxValAttr || !floatMinValAttr ||
+        (floatMaxValAttr.getType() != floatMinValAttr.getType()) ||
+        (floatMaxValAttr.getType() != inputETy))
       return emitOpError("min/max attributes types are incompatible with "
                          "input/output element types.");
   }
@@ -421,21 +530,19 @@ static void buildConvOpWithQuantInfo(OpBuilder &builder, OperationState &result,
                                      DenseI64ArrayAttr stride,
                                      DenseI64ArrayAttr dilation,
                                      TypeAttr accType) {
-
-  result.addOperands({input, weight, bias});
+  auto zps = createZPsAsConst(builder, input, weight);
+  result.addOperands({input, weight, bias, zps.first, zps.second});
   result.addAttribute("pad", pad);
   result.addAttribute("stride", stride);
   result.addAttribute("dilation", dilation);
   result.addAttribute("acc_type", accType);
-
+  Type finalOutputType = outputType;
   auto quantAttr = buildConvOpQuantizationAttr(builder, input, weight);
   if (quantAttr) {
-    result.addAttribute("quantization_info", quantAttr);
-    result.addTypes(
-        buildConvOpResultTypeInfo(builder, outputType, input, weight));
-  } else {
-    result.addTypes(outputType);
+    finalOutputType =
+        buildConvOpResultTypeInfo(builder, outputType, input, weight);
   }
+  result.addTypes(finalOutputType);
 }
 
 /// Handles tosa.transpose_conv2d which has outpad and output shape
@@ -444,20 +551,19 @@ static void buildTransConvOpWithQuantInfo(
     OpBuilder &builder, OperationState &result, Type outputType, Value input,
     Value weight, Value bias, DenseI64ArrayAttr outpad,
     DenseI64ArrayAttr stride, DenseI64ArrayAttr outputShape, TypeAttr accType) {
-  result.addOperands({input, weight, bias});
+  auto zps = createZPsAsConst(builder, input, weight);
+  result.addOperands({input, weight, bias, zps.first, zps.second});
   result.addAttribute("out_pad", outpad);
   result.addAttribute("stride", stride);
   result.addAttribute("out_shape", outputShape);
   result.addAttribute("acc_type", accType);
-  auto quantAttr = ::buildConvOpQuantizationAttr(builder, input, weight);
-
+  Type finalOutputType = outputType;
+  auto quantAttr = buildConvOpQuantizationAttr(builder, input, weight);
   if (quantAttr) {
-    result.addAttribute("quantization_info", quantAttr);
-    result.addTypes(
-        buildConvOpResultTypeInfo(builder, outputType, input, weight));
-  } else {
-    result.addTypes(outputType);
+    finalOutputType =
+        buildConvOpResultTypeInfo(builder, outputType, input, weight);
   }
+  result.addTypes(finalOutputType);
 }
 
 /// The tosa.fully_connected op has its own builder as it does not have
@@ -488,7 +594,10 @@ static void buildMatMulOpWithQuantInfo(OpBuilder &builder,
   auto quantAttr = ::buildMatMulOpQuantizationAttr(builder, a, b);
 
   if (quantAttr) {
-    result.addAttribute("quantization_info", quantAttr);
+    result.addAttribute("a_zp", builder.getI32IntegerAttr(
+                                    static_cast<int32_t>(quantAttr.getAZp())));
+    result.addAttribute("b_zp", builder.getI32IntegerAttr(
+                                    static_cast<int32_t>(quantAttr.getBZp())));
 
     auto inputType = llvm::dyn_cast<ShapedType>(a.getType());
     assert(inputType && "Input must be a shaped tensor type!");
@@ -528,8 +637,14 @@ buildAvgPool2dOpWithQuantInfo(OpBuilder &builder, OperationState &result,
   result.addAttribute("pad", pad);
   result.addAttribute("acc_type", accType);
   auto quantAttr = buildUnaryOpQuantizationAttr(builder, input, outputType);
-  if (quantAttr)
-    result.addAttribute("quantization_info", quantAttr);
+  if (quantAttr) {
+    result.addAttribute("input_zp",
+                        builder.getI32IntegerAttr(
+                            static_cast<int32_t>(quantAttr.getInputZp())));
+    result.addAttribute("output_zp",
+                        builder.getI32IntegerAttr(
+                            static_cast<int32_t>(quantAttr.getOutputZp())));
+  }
   result.types.push_back(outputType);
 }
 
@@ -541,8 +656,15 @@ static void buildUnaryOpWithQuantInfo(OpBuilder &builder,
                                       Value input) {
   result.addOperands(input);
   auto quantAttr = buildUnaryOpQuantizationAttr(builder, input, outputType);
-  if (quantAttr)
-    result.addAttribute("quantization_info", quantAttr);
+  if (quantAttr) {
+    // note: negateOp has attributes input1_zp and output_zp
+    result.addAttribute("input1_zp",
+                        builder.getI32IntegerAttr(
+                            static_cast<int32_t>(quantAttr.getInputZp())));
+    result.addAttribute("output_zp",
+                        builder.getI32IntegerAttr(
+                            static_cast<int32_t>(quantAttr.getOutputZp())));
+  }
   result.types.push_back(outputType);
 }
 
@@ -554,8 +676,11 @@ static void buildPadOpWithQuantInfo(OpBuilder &builder, OperationState &result,
                                     Value paddings) {
   result.addOperands({input, paddings});
   auto quantAttr = buildPadOpQuantizationAttr(builder, input);
-  if (quantAttr)
-    result.addAttribute("quantization_info", quantAttr);
+  if (quantAttr) {
+    result.addAttribute("input_zp",
+                        builder.getI32IntegerAttr(
+                            static_cast<int32_t>(quantAttr.getInputZp())));
+  }
   result.types.push_back(outputType);
 }
 
@@ -568,8 +693,11 @@ static void buildExplicitValuePadOpWithQuantInfo(OpBuilder &builder,
                                                  Value padConst) {
   result.addOperands({input, paddings, padConst});
   auto quantAttr = buildPadOpQuantizationAttr(builder, input);
-  if (quantAttr)
-    result.addAttribute("quantization_info", quantAttr);
+  if (quantAttr) {
+    result.addAttribute("input_zp",
+                        builder.getI32IntegerAttr(
+                            static_cast<int32_t>(quantAttr.getInputZp())));
+  }
   result.types.push_back(outputType);
 }
 
@@ -790,7 +918,46 @@ LogicalResult tosa::FullyConnectedOp::inferReturnTypeComponents(
   return success();
 }
 
-LogicalResult FullyConnectedOp::verify() { return verifyConvOp(*this); }
+LogicalResult FullyConnectedOp::verify() {
+  // All TOSA conv ops have an input() and weight().
+  auto inputType = llvm::dyn_cast<RankedTensorType>(getInput().getType());
+
+  RankedTensorType weightType =
+      llvm::dyn_cast<RankedTensorType>(getWeight().getType());
+
+  // Must be ranked tensor types
+  if (!inputType) {
+    emitOpError("expect a ranked tensor for input, got ") << getInput();
+    return failure();
+  }
+  if (!weightType) {
+    emitOpError("expect a ranked tensor for weight, got ") << getWeight();
+    return failure();
+  }
+
+  auto inputEType = inputType.getElementType();
+  auto weightEType = weightType.getElementType();
+
+  bool inputIsQuant = !llvm::isa<FloatType>(inputEType);
+  bool weightIsQuant = !llvm::isa<FloatType>(weightEType);
+
+  // Either both must be quantized or both unquantized.
+  if (inputIsQuant != weightIsQuant) {
+    emitOpError(
+        "expect both input and weight to be float or not together, got ")
+        << inputEType << " and " << weightEType;
+    return failure();
+  }
+
+  // Quantized type must have constructed the quantizationattr, and unquantized
+  // types should not have a quantizationattr.
+  if ((inputIsQuant && !getInputZp()) || (!inputIsQuant && getInputZp())) {
+    emitOpError("input zero point is required for quantized type, and not "
+                "allowed for float type");
+    return failure();
+  }
+  return success();
+}
 
 LogicalResult tosa::MatMulOp::inferReturnTypeComponents(
     MLIRContext *context, ::std::optional<Location> location,
@@ -891,8 +1058,18 @@ LogicalResult tosa::SliceOp::inferReturnTypeComponents(
     MLIRContext *context, ::std::optional<Location> location,
     SliceOp::Adaptor adaptor,
     SmallVectorImpl<ShapedTypeComponents> &inferredReturnShapes) {
-  auto start = adaptor.getStart();
-  auto size = adaptor.getSize();
+
+  Type inputType = getElementTypeOrSelf(adaptor.getInput1().getType());
+  SmallVector<int64_t> start;
+  SmallVector<int64_t> size;
+
+  if (!tosa::getConstShapeValue(adaptor.getStart().getDefiningOp(), start) ||
+      !tosa::getConstShapeValue(adaptor.getSize().getDefiningOp(), size)) {
+    auto rank = cast<tosa::shapeType>(adaptor.getSize().getType()).getRank();
+    SmallVector<int64_t> fallback(rank, ShapedType::kDynamic);
+    inferredReturnShapes.push_back(ShapedTypeComponents(fallback, inputType));
+    return success();
+  }
 
   // if size[i] is -1, all remaining elements in dimension i are included
   // in the slice, similar to TF.
@@ -933,21 +1110,131 @@ LogicalResult tosa::SliceOp::verify() {
   if (!inputType)
     return success();
 
-  if (static_cast<size_t>(inputType.getRank()) != getStart().size())
+  auto startShapeRank =
+      llvm::cast<tosa::shapeType>(getStart().getType()).getRank();
+  if (inputType.getRank() != startShapeRank)
     return emitOpError(
         "length of start attribute is not equal rank of input shape");
 
-  if (static_cast<size_t>(inputType.getRank()) != getSize().size())
+  auto sizeShapeRank =
+      llvm::cast<tosa::shapeType>(getSize().getType()).getRank();
+  if (inputType.getRank() != sizeShapeRank)
     return emitOpError(
         "length of size attribute is not equal rank of input shape");
 
   return success();
 }
 
+LogicalResult tosa::MulOp::inferReturnTypeComponents(
+    MLIRContext *context, ::std::optional<Location> location,
+    ValueShapeRange operands, DictionaryAttr attributes,
+    OpaqueProperties properties, RegionRange regions,
+    SmallVectorImpl<ShapedTypeComponents> &inferredReturnShapes) {
+  // mul op's output shape only depend on input1 and input2, not on shift
+  ValueShapeRange twoInputs = operands.drop_back();
+  llvm::SmallVector<int64_t> outShape;
+  if (resolveBroadcastShape(twoInputs, outShape).failed()) {
+    inferredReturnShapes.push_back(ShapedTypeComponents());
+  } else {
+    inferredReturnShapes.push_back(ShapedTypeComponents(outShape));
+  }
+  return success();
+}
+
 LogicalResult tosa::MulOp::verify() {
-  Type elementTy = getInput1().getType().getElementType();
-  if (isa<FloatType>(elementTy) && getShift() != 0)
-    return emitOpError() << "require shift to be 0 for float type";
+  auto resElemType = getElementTypeOrSelf(getOutput());
+
+  // Verify if the element type among operands and result match tosa
+  // specification.
+  if (auto resIntType = dyn_cast<IntegerType>(resElemType)) {
+    IntegerType lhsIntType =
+        cast<IntegerType>(getElementTypeOrSelf(getInput1()));
+    IntegerType rhsIntType =
+        cast<IntegerType>(getElementTypeOrSelf(getInput2()));
+    if (lhsIntType != rhsIntType)
+      return emitOpError("requires the same element type for all operands");
+
+    // Though the spec requires the element type of result to be i32, a more
+    // relaxed way is provided at dialect level for easier cooperating with
+    // other dialects.
+    if (lhsIntType.getWidth() > resIntType.getWidth())
+      return emitOpError("invalid data type size for operands or result");
+
+  } else {
+    // For other supported type, the spec requires requires the same element
+    // type for all operands (excludes `shift` operand) and results.
+    for (int i = 0; i < 2; ++i) {
+      if (getElementTypeOrSelf(getOperand(i)) != resElemType)
+        return emitOpError(
+            "requires the same element type for all operands and results");
+    }
+
+    // verify shift has value 0 for non-integer types
+    ElementsAttr shift_elem;
+    if (matchPattern(getShift(), m_Constant(&shift_elem))) {
+      int32_t shift = shift_elem.getValues<IntegerAttr>()[0].getInt();
+      if (shift != 0) {
+        return emitOpError() << "require shift to be 0 for float type";
+      }
+    }
+  }
+
+  // Verify the op has same ranks for all main operands (excludes extra operands
+  // such as shift of mul op, so this is the only difference with the built-in
+  // `SameOperandsAndResultRank` trait) and results types, if known.
+
+  // delegate function that returns true if type is a shaped type with known
+  // rank
+  auto hasRank = [](const Type type) {
+    if (auto shaped_type = dyn_cast<ShapedType>(type))
+      return shaped_type.hasRank();
+
+    return false;
+  };
+
+  auto rankedOperandTypes =
+      llvm::to_vector(llvm::make_filter_range(getOperandTypes(), hasRank));
+
+  auto rankedResultTypes =
+      llvm::make_filter_range(getOperation()->getResultTypes(), hasRank);
+
+  // If all operands and results are unranked, then no further verification.
+  if (rankedOperandTypes.empty() && rankedResultTypes.empty())
+    return success();
+
+  // delegate function that returns rank of shaped type with known rank
+  auto getRank = [](const Type type) {
+    return cast<ShapedType>(type).getRank();
+  };
+
+  auto rank = !rankedOperandTypes.empty() ? getRank(*rankedOperandTypes.begin())
+                                          : getRank(*rankedResultTypes.begin());
+
+  for (size_t i = 0; i < 2; ++i) {
+    if (rank != getRank(rankedOperandTypes[i])) {
+      return emitOpError("operands don't have matching ranks");
+    }
+  }
+
+  for (const auto type : rankedResultTypes) {
+    if (rank != getRank(type)) {
+      return emitOpError("result type has different rank than operands");
+    }
+  }
+
+  // check for broadcast compatible shapes in first two operands (ignoring
+  // shift)
+
+  // delegate function that returns shape of shaped type
+  auto getShape = [](const Type type) {
+    return mlir::cast<ShapedType>(type).getShape();
+  };
+  SmallVector<int64_t> resultShape;
+  if (!mlir::OpTrait::util::getBroadcastedShape(getShape(rankedOperandTypes[0]),
+                                                getShape(rankedOperandTypes[1]),
+                                                resultShape)) {
+    return emitOpError("operands don't have broadcast-compatible shapes");
+  }
 
   return success();
 }
@@ -1077,8 +1364,16 @@ LogicalResult tosa::ReshapeOp::inferReturnTypeComponents(
     SmallVectorImpl<ShapedTypeComponents> &inferredReturnShapes) {
   ShapeAdaptor inputShape(adaptor.getInput1().getType());
   Type inputType = getElementTypeOrSelf(adaptor.getInput1().getType());
-  llvm::SmallVector<int64_t> newShapeValue =
-      convertToMlirShape(adaptor.getNewShape());
+  llvm::SmallVector<int64_t> newShapeValue;
+  if (!tosa::getConstShapeValue(adaptor.getShape().getDefiningOp(),
+                                newShapeValue)) {
+    auto rank = cast<tosa::shapeType>(adaptor.getShape().getType()).getRank();
+    SmallVector<int64_t> fallback(rank, ShapedType::kDynamic);
+    inferredReturnShapes.push_back(ShapedTypeComponents(fallback, inputType));
+    return success();
+  } else {
+    newShapeValue = convertToMlirShape(newShapeValue);
+  }
 
   // We cannot infer from the total number of elements so we must take the
   // shape attribute as exact.
@@ -1114,13 +1409,19 @@ llvm::LogicalResult tosa::ReshapeOp::verify() {
   TensorType inputType = getInput1().getType();
   RankedTensorType outputType = getType();
 
-  if ((int64_t)getNewShape().size() != outputType.getRank())
+  SmallVector<int64_t> shapeValues;
+  if (!tosa::getConstShapeValue(getShape().getDefiningOp(), shapeValues)) {
+    // skip following checks if shape is not constant
+    return mlir::success();
+  }
+
+  if ((int64_t)shapeValues.size() != outputType.getRank())
     return emitOpError() << "new shape does not match result rank";
 
   for (auto [newShapeDim, outputShapeDim] :
-       zip(getNewShape(), outputType.getShape())) {
-    if (newShapeDim != -1 && outputShapeDim != ShapedType::kDynamic &&
-        newShapeDim != outputShapeDim)
+       zip(shapeValues, outputType.getShape())) {
+    if (newShapeDim != -1 && newShapeDim != ShapedType::kDynamic &&
+        outputShapeDim != ShapedType::kDynamic && newShapeDim != outputShapeDim)
       return emitOpError() << "new shape is inconsistent with result shape";
 
     if (newShapeDim != ShapedType::kDynamic && newShapeDim < -1)
@@ -1139,10 +1440,10 @@ llvm::LogicalResult tosa::ReshapeOp::verify() {
     }
 
     int64_t newShapeElementsNum = std::accumulate(
-        getNewShape().begin(), getNewShape().end(), 1LL,
+        shapeValues.begin(), shapeValues.end(), 1LL,
         [](int64_t acc, int64_t dim) { return (dim > 0) ? acc * dim : acc; });
     bool isStaticNewShape =
-        llvm::all_of(getNewShape(), [](int64_t s) { return s > 0; });
+        llvm::all_of(shapeValues, [](int64_t s) { return s > 0; });
     if ((isStaticNewShape && inputElementsNum != newShapeElementsNum) ||
         (!isStaticNewShape && newShapeElementsNum > inputElementsNum)) {
       return emitOpError() << "cannot reshape " << inputElementsNum
@@ -1150,7 +1451,7 @@ llvm::LogicalResult tosa::ReshapeOp::verify() {
     }
   }
 
-  int missingDims = llvm::count(getNewShape(), -1);
+  int missingDims = llvm::count(shapeValues, -1);
   if (missingDims > 1)
     return emitOpError() << "expected at most one target dimension to be -1";
 
@@ -1589,7 +1890,6 @@ NARY_SHAPE_INFER(tosa::LogicalRightShiftOp)
 NARY_SHAPE_INFER(tosa::LogicalXorOp)
 NARY_SHAPE_INFER(tosa::MaximumOp)
 NARY_SHAPE_INFER(tosa::MinimumOp)
-NARY_SHAPE_INFER(tosa::MulOp)
 NARY_SHAPE_INFER(tosa::NegateOp)
 NARY_SHAPE_INFER(tosa::PowOp)
 NARY_SHAPE_INFER(tosa::ReciprocalOp)
@@ -1903,7 +2203,7 @@ LogicalResult TransposeConv2DOp::inferReturnTypeComponents(
   }
 
   // Weight shapes describes the filter width/height and the output channels.
-  ShapeAdaptor weightShape(adaptor.getFilter().getType());
+  ShapeAdaptor weightShape(adaptor.getWeight().getType());
   if (weightShape.hasRank()) {
     outputShape[3] = ShapedType::isDynamic(outputShape[3])
                          ? weightShape.getDimSize(0)
@@ -2199,6 +2499,51 @@ void WhileOp::print(OpAsmPrinter &parser) {
   parser.printOptionalAttrDictWithKeyword((*this)->getAttrs());
 }
 
+LogicalResult mlir::tosa::getZeroPoint(ElementsAttr zpAttr, int64_t &zp) {
+  Type zpElemType = zpAttr.getElementType();
+  if (auto quantType =
+          llvm::dyn_cast<mlir::quant::UniformQuantizedType>(zpElemType)) {
+    zp = quantType.getZeroPoint();
+    return success();
+  }
+  if (llvm::isa<FloatType>(zpElemType)) {
+    // non-zero zero point is not allowed for float types.
+    if (!zpAttr.getValues<APFloat>()[0].isZero())
+      return failure();
+    zp = 0;
+    return success();
+  }
+  if (llvm::isa<IntegerType>(zpElemType)) {
+    zp = zpAttr.getValues<APInt>()[0].getSExtValue();
+    return success();
+  }
+  // zero point is not allowed for unsupported types.
+  return failure();
+}
+
+// Create a rank-1 const tensor for zero point of the source tensor.
+std::optional<Value> mlir::tosa::createZeroPointTensor(OpBuilder &builder,
+                                                       Location loc,
+                                                       Type srcElemType,
+                                                       int64_t zp) {
+  srcElemType = getElementTypeOrSelf(srcElemType);
+  if (auto quantType = llvm::dyn_cast<mlir::quant::QuantizedType>(srcElemType))
+    srcElemType = quantType.getStorageType();
+  auto zpType = mlir::RankedTensorType::get({1}, srcElemType);
+  if (llvm::isa<FloatType>(srcElemType)) {
+    auto zpAttr = DenseElementsAttr::get(
+        zpType, builder.getFloatAttr(srcElemType, static_cast<double>(zp)));
+    return builder.create<tosa::ConstOp>(loc, zpType, zpAttr);
+  }
+  if (llvm::isa<IntegerType>(srcElemType)) {
+    auto zpAttr =
+        DenseElementsAttr::get(zpType, builder.getIntegerAttr(srcElemType, zp));
+    return builder.create<tosa::ConstOp>(loc, zpType, zpAttr);
+  }
+  llvm::errs() << "zero point is not allowed for unsupported data types\n";
+  return std::nullopt;
+}
+
 //===----------------------------------------------------------------------===//
 // TOSA Shape and Shape Operators Helper functions.
 //===----------------------------------------------------------------------===//
diff --git a/mlir/lib/Dialect/Tosa/Transforms/TosaDecomposeConv2D.cpp b/mlir/lib/Dialect/Tosa/Transforms/TosaDecomposeConv2D.cpp
index cb08360f90228..617a59bc87c9f 100644
--- a/mlir/lib/Dialect/Tosa/Transforms/TosaDecomposeConv2D.cpp
+++ b/mlir/lib/Dialect/Tosa/Transforms/TosaDecomposeConv2D.cpp
@@ -20,12 +20,6 @@ using namespace mlir::tosa;
 
 namespace {
 
-SmallVector<int64_t> convertFromMlirShape(ArrayRef<int64_t> shape) {
-  return to_vector(llvm::map_range(shape, [](int64_t dim) {
-    return ShapedType::isDynamic(dim) ? -1 : dim;
-  }));
-}
-
 struct Conv2DIsFullyConnected : public OpRewritePattern<tosa::Conv2DOp> {
   explicit Conv2DIsFullyConnected(MLIRContext *context)
       : OpRewritePattern(context) {}
@@ -59,19 +53,17 @@ struct Conv2DIsFullyConnected : public OpRewritePattern<tosa::Conv2DOp> {
     for (const auto &it : llvm::enumerate(padAttr))
       pad[it.index() + 2] = it.value();
 
+    Type inputETy = inputType.getElementType();
     if (llvm::any_of(pad, [](int64_t p) { return p != 0; })) {
-      Type inputETy = inputType.getElementType();
-      Attribute zeroAttr = rewriter.getZeroAttr(inputETy);
-      if (op.getQuantizationInfo()) {
-        auto quantizationInfo = op.getQuantizationInfo();
-        int64_t iZp = quantizationInfo->getInputZp();
+      auto failureOrMaybeZps = extractConvZpPair(op, rewriter);
+      if (failed(failureOrMaybeZps))
+        return failure();
 
-        if (!validIntegerRange(cast<IntegerType>(inputETy), iZp))
-          return rewriter.notifyMatchFailure(
-              op, "tosa.conv op quantization has zp outside of input range");
+      auto maybeZps = failureOrMaybeZps.value();
 
-        zeroAttr = rewriter.getIntegerAttr(inputETy, iZp);
-      }
+      Attribute zeroAttr =
+          maybeZps ? rewriter.getIntegerAttr(inputETy, maybeZps->inputZp)
+                   : rewriter.getZeroAttr(inputETy);
 
       llvm::SmallVector<int64_t> newShape(inputType.getShape());
 
@@ -100,12 +92,13 @@ struct Conv2DIsFullyConnected : public OpRewritePattern<tosa::Conv2DOp> {
     llvm::SmallVector<int64_t, 2> revisedInputShape{combined, inputShape[3]};
     auto revisedInputShapeType =
         RankedTensorType::get(revisedInputShape, inputType.getElementType());
-    auto reshapedInput = rewriter
-                             .create<tosa::ReshapeOp>(
-                                 op.getLoc(), revisedInputShapeType, input,
-                                 rewriter.getDenseI64ArrayAttr(
-                                     convertFromMlirShape(revisedInputShape)))
-                             .getResult();
+    auto revisedInputShapeValue = getTosaConstShape(
+        rewriter, op.getLoc(), convertFromMlirShape(revisedInputShape));
+    auto reshapedInput =
+        rewriter
+            .create<tosa::ReshapeOp>(op.getLoc(), revisedInputShapeType, input,
+                                     revisedInputShapeValue)
+            .getResult();
 
     // Reshape kernel to [OC,KH,KW,IC] -> [OC, IC].
     llvm::SmallVector<int64_t, 2> revisedWeightShape{weightShape[0],
@@ -113,25 +106,33 @@ struct Conv2DIsFullyConnected : public OpRewritePattern<tosa::Conv2DOp> {
     auto revisedWeightShapeType = RankedTensorType::get(
         revisedWeightShape,
         dyn_cast<RankedTensorType>(weight.getType()).getElementType());
-    auto reshapedWeight = rewriter
-                              .create<tosa::ReshapeOp>(
-                                  op.getLoc(), revisedWeightShapeType, weight,
-                                  rewriter.getDenseI64ArrayAttr(
-                                      convertFromMlirShape(revisedWeightShape)))
-                              .getResult();
+    auto revisedWeightShapeValue = getTosaConstShape(
+        rewriter, op.getLoc(), convertFromMlirShape(revisedWeightShape));
+    auto reshapedWeight =
+        rewriter
+            .create<tosa::ReshapeOp>(op.getLoc(), revisedWeightShapeType,
+                                     weight, revisedWeightShapeValue)
+            .getResult();
 
     // Perform a fully connected network over the reshaped input and weight.
     llvm::SmallVector<int64_t, 2> fullyConnectedShape{combined, weightShape[0]};
     auto fullyConnectedShapeType =
         RankedTensorType::get(fullyConnectedShape, resultType.getElementType());
 
+    auto failureOrMaybeZps = extractConvZpPair(op, rewriter);
+    if (failed(failureOrMaybeZps))
+      return failure();
+
+    auto maybeZps = failureOrMaybeZps.value();
     Value fullyConnectedValue;
-    if (op.getQuantizationInfo()) {
+    if (maybeZps) {
       fullyConnectedValue =
           rewriter
               .create<tosa::FullyConnectedOp>(
                   op.getLoc(), fullyConnectedShapeType, reshapedInput,
-                  reshapedWeight, op.getBias(), *op.getQuantizationInfo())
+                  reshapedWeight, op.getBias(),
+                  rewriter.getI32IntegerAttr(maybeZps->inputZp),
+                  rewriter.getI32IntegerAttr(maybeZps->weightZp))
               .getResult();
     } else {
       fullyConnectedValue = rewriter
@@ -144,9 +145,10 @@ struct Conv2DIsFullyConnected : public OpRewritePattern<tosa::Conv2DOp> {
     // Reshape output to [N, IH, IW, OC].
     llvm::SmallVector<int64_t, 4> outputShape{inputShape[0], inputShape[1],
                                               inputShape[2], weightShape[0]};
+    auto outputShapeValue = getTosaConstShape(
+        rewriter, op.getLoc(), convertFromMlirShape(outputShape));
     rewriter.replaceOpWithNewOp<tosa::ReshapeOp>(
-        op, resultType, fullyConnectedValue,
-        rewriter.getDenseI64ArrayAttr(convertFromMlirShape(outputShape)));
+        op, resultType, fullyConnectedValue, outputShapeValue);
     return success();
   }
 };
diff --git a/mlir/lib/Dialect/Tosa/Transforms/TosaDecomposeDepthwise.cpp b/mlir/lib/Dialect/Tosa/Transforms/TosaDecomposeDepthwise.cpp
index 45f4419875b48..b26397d0e3ed7 100644
--- a/mlir/lib/Dialect/Tosa/Transforms/TosaDecomposeDepthwise.cpp
+++ b/mlir/lib/Dialect/Tosa/Transforms/TosaDecomposeDepthwise.cpp
@@ -14,6 +14,7 @@
 #include "mlir/Dialect/Tosa/IR/TosaOps.h"
 #include "mlir/Dialect/Tosa/Transforms/Passes.h"
 #include "mlir/Dialect/Tosa/Utils/ConversionUtils.h"
+#include "mlir/IR/BuiltinTypes.h"
 #include "mlir/Pass/Pass.h"
 
 using namespace mlir;
@@ -54,26 +55,33 @@ struct DepthwiseConv2DIsMul : public OpRewritePattern<tosa::DepthwiseConv2DOp> {
     inputType = RankedTensorType::get(
         revisedInputShape,
         dyn_cast<RankedTensorType>(input.getType()).getElementType());
+    auto revisedInputShapeValue =
+        getTosaConstShape(rewriter, op.getLoc(), revisedInputShape);
     input = rewriter
-                .create<tosa::ReshapeOp>(
-                    op.getLoc(), inputType, input,
-                    rewriter.getDenseI64ArrayAttr(revisedInputShape))
+                .create<tosa::ReshapeOp>(op.getLoc(), inputType, input,
+                                         revisedInputShapeValue)
                 .getResult();
 
-    if (inputType.getElementType() != resultType.getElementType()) {
-      inputType = inputType.clone(resultType.getElementType());
+    Type inputETy = inputType.getElementType();
+    Type weightETy = weightType.getElementType();
+    Type resultETy = resultType.getElementType();
+
+    if (inputETy != resultETy) {
+      inputType = inputType.clone(resultETy);
       input = rewriter.create<tosa::CastOp>(op.getLoc(), inputType, input);
     }
 
-    if (weightType.getElementType() != resultType.getElementType()) {
-      weightType = weightType.clone(resultType.getElementType());
+    if (weightETy != resultETy) {
+      weightType = weightType.clone(resultETy);
       weight = rewriter.create<tosa::CastOp>(op.getLoc(), weightType, weight);
     }
 
-    if (auto quantizationInfo = op.getQuantizationInfo()) {
-      auto iZp = quantizationInfo->getInputZp();
-      auto wZp = quantizationInfo->getWeightZp();
+    auto failureOrMaybeZps = extractConvZpPair(op, rewriter);
+    if (failed(failureOrMaybeZps))
+      return failure();
 
+    auto maybeZps = failureOrMaybeZps.value();
+    if (maybeZps) {
       auto applyZp = [&](Value val, int64_t zp) -> Value {
         if (zp == 0)
           return val;
@@ -88,8 +96,8 @@ struct DepthwiseConv2DIsMul : public OpRewritePattern<tosa::DepthwiseConv2DOp> {
                                             zpVal);
       };
 
-      input = applyZp(input, iZp);
-      weight = applyZp(weight, wZp);
+      input = applyZp(input, maybeZps->inputZp);
+      weight = applyZp(weight, maybeZps->weightZp);
     }
 
     ArrayRef<int64_t> padAttr = op.getPad();
@@ -98,7 +106,6 @@ struct DepthwiseConv2DIsMul : public OpRewritePattern<tosa::DepthwiseConv2DOp> {
       pad[it.index() + 2] = it.value();
 
     if (llvm::any_of(pad, [](int64_t p) { return p != 0; })) {
-      Type inputETy = inputType.getElementType();
       Attribute zeroAttr = rewriter.getZeroAttr(inputETy);
 
       llvm::SmallVector<int64_t> newShape(inputType.getShape());
@@ -131,9 +138,15 @@ struct DepthwiseConv2DIsMul : public OpRewritePattern<tosa::DepthwiseConv2DOp> {
       return failure();
     }
 
+    auto shiftElementType = IntegerType::get(rewriter.getContext(), 8);
+    auto shiftType = RankedTensorType::get({1}, shiftElementType);
+    auto shiftZeroAttr = DenseElementsAttr::get(
+        shiftType, rewriter.getIntegerAttr(shiftElementType, 0));
+    Value constZero =
+        rewriter.create<tosa::ConstOp>(op.getLoc(), shiftType, shiftZeroAttr);
     Value mulValue = rewriter
                          .create<tosa::MulOp>(op.getLoc(), mulShapeType, input,
-                                              weight, /*shift=*/0)
+                                              weight, constZero)
                          .getResult();
 
     // Reshape output to [N, H, W, C * M].
@@ -141,9 +154,10 @@ struct DepthwiseConv2DIsMul : public OpRewritePattern<tosa::DepthwiseConv2DOp> {
     auto outputShapeType = RankedTensorType::get(
         outputShape,
         dyn_cast<RankedTensorType>(input.getType()).getElementType());
+    auto outputShapeValue =
+        getTosaConstShape(rewriter, op->getLoc(), outputShape);
     Value outputValue = rewriter.create<tosa::ReshapeOp>(
-        op.getLoc(), outputShapeType, mulValue,
-        rewriter.getDenseI64ArrayAttr(outputShape));
+        op.getLoc(), outputShapeType, mulValue, outputShapeValue);
 
     Value bias = op.getBias();
     if (EqualizeRanks(rewriter, op.getLoc(), outputValue, bias).failed()) {
diff --git a/mlir/lib/Dialect/Tosa/Transforms/TosaDecomposeTransposeConv.cpp b/mlir/lib/Dialect/Tosa/Transforms/TosaDecomposeTransposeConv.cpp
index 1b97f0b245d9b..26baddcf1dd15 100644
--- a/mlir/lib/Dialect/Tosa/Transforms/TosaDecomposeTransposeConv.cpp
+++ b/mlir/lib/Dialect/Tosa/Transforms/TosaDecomposeTransposeConv.cpp
@@ -69,22 +69,12 @@ class TransposeConvNonStridedConverter
     auto reverse2 = rewriter.create<tosa::ReverseOp>(
         loc, weightTy, reverse1, /* axis = */ rewriter.getI32IntegerAttr(2));
 
-    Value conv2d;
-    if (op.getQuantizationInfo()) {
-      conv2d = rewriter.create<tosa::Conv2DOp>(
-          loc, resultTy, input, reverse2, bias,
-          rewriter.getDenseI64ArrayAttr(convPad),
-          rewriter.getDenseI64ArrayAttr(stride),
-          rewriter.getDenseI64ArrayAttr({1, 1}),
-          /* acc_type = */ op.getAccType(), *op.getQuantizationInfo());
-    } else {
-      conv2d = rewriter.create<tosa::Conv2DOp>(
-          loc, resultTy, input, reverse2, bias,
-          rewriter.getDenseI64ArrayAttr(convPad),
-          rewriter.getDenseI64ArrayAttr(stride),
-          rewriter.getDenseI64ArrayAttr({1, 1}),
-          /* acc_type = */ op.getAccTypeAttr());
-    }
+    Value conv2d = rewriter.create<tosa::Conv2DOp>(
+        loc, resultTy, input, reverse2, bias, op.getInputZp(), op.getWeightZp(),
+        rewriter.getDenseI64ArrayAttr(convPad),
+        rewriter.getDenseI64ArrayAttr(stride),
+        rewriter.getDenseI64ArrayAttr({1, 1}),
+        /* acc_type = */ op.getAccType());
 
     rewriter.replaceOp(op, conv2d);
     return success();
@@ -144,13 +134,16 @@ class TransposeConvStridedConverter
     Value weightPaddingVal =
         getTosaConstShape(rewriter, op->getLoc(), weightPadding);
 
-    if (op.getQuantizationInfo().has_value()) {
-      auto quantInfo = op.getQuantizationInfo().value();
+    auto failureOrMaybeZps = extractConvZpPair(op, rewriter);
+    if (failed(failureOrMaybeZps))
+      return failure();
+
+    auto maybeZps = failureOrMaybeZps.value();
+    if (maybeZps) {
       weight = CreateOpAndInferShape<tosa::PadOp>(
           rewriter, loc, UnrankedTensorType::get(weightETy), weight,
           weightPaddingVal, nullptr,
-          rewriter.getAttr<PadOpQuantizationAttr>(quantInfo.getWeightZp()));
-
+          rewriter.getI32IntegerAttr(maybeZps->weightZp));
     } else {
       weight = CreateOpAndInferShape<tosa::PadOp>(
           rewriter, loc, UnrankedTensorType::get(weightETy), weight,
@@ -166,9 +159,11 @@ class TransposeConvStridedConverter
         outputChannels, weightHeight / stride[0],
         stride[0],      weightWidth / stride[1],
         stride[1],      inputChannels};
+
+    ImplicitLocOpBuilder builder(op->getLoc(), rewriter);
     weight = CreateOpAndInferShape<tosa::ReshapeOp>(
-        rewriter, loc, UnrankedTensorType::get(weightETy), weight,
-        rewriter.getDenseI64ArrayAttr(weightReshapeDims0));
+        builder, UnrankedTensorType::get(weightETy), weight,
+        getTosaConstShape(rewriter, loc, weightReshapeDims0));
 
     // Transpose the factored-out stride to the output channels.
     Value transposeWeightVal = rewriter.create<tosa::ConstOp>(
@@ -180,12 +175,13 @@ class TransposeConvStridedConverter
         transposeWeightVal);
 
     // Collapse the strides and output channels into a single dimension.
-    llvm::SmallVector<int64_t, 6> weightReshapeDims1 = {
+    llvm::SmallVector<int64_t, 4> weightReshapeDims1 = {
         outputChannels * stride[0] * stride[1], weightHeight / stride[0],
         weightWidth / stride[1], inputChannels};
+
     weight = CreateOpAndInferShape<tosa::ReshapeOp>(
         rewriter, loc, UnrankedTensorType::get(weightETy), weight,
-        rewriter.getDenseI64ArrayAttr(weightReshapeDims1));
+        getTosaConstShape(rewriter, loc, weightReshapeDims1));
     ShapedType restridedWeightTy = cast<ShapedType>(weight.getType());
 
     weight = CreateOpAndInferShape<tosa::ReverseOp>(
@@ -205,12 +201,11 @@ class TransposeConvStridedConverter
     Value inputPaddingVal =
         getTosaConstShape(rewriter, op->getLoc(), inputPadding);
 
-    if (op.getQuantizationInfo().has_value()) {
-      auto quantInfo = op.getQuantizationInfo().value();
+    if (maybeZps) {
       input = CreateOpAndInferShape<tosa::PadOp>(
           rewriter, loc, UnrankedTensorType::get(inputETy), input,
           inputPaddingVal, nullptr,
-          rewriter.getAttr<PadOpQuantizationAttr>(quantInfo.getInputZp()));
+          rewriter.getI32IntegerAttr(maybeZps->inputZp));
     } else {
       input = CreateOpAndInferShape<tosa::PadOp>(
           rewriter, loc, UnrankedTensorType::get(inputETy), input,
@@ -227,28 +222,34 @@ class TransposeConvStridedConverter
                                   biasETy),
             rewriter.getZeroAttr(biasETy)));
 
-    // Perform the convolution using the zero bias.
-    Value conv2d;
-    if (op.getQuantizationInfo()) {
-      conv2d = CreateOpAndInferShape<tosa::Conv2DOp>(
-                   rewriter, loc, UnrankedTensorType::get(resultETy), input,
-                   weight, zeroBias,
-                   /*pad=*/rewriter.getDenseI64ArrayAttr({0, 0, 0, 0}),
-                   /*stride=*/rewriter.getDenseI64ArrayAttr({1, 1}),
-                   /*dilation=*/rewriter.getDenseI64ArrayAttr({1, 1}),
-                   /* acc_type = */ op.getAccType(), *op.getQuantizationInfo())
-                   .getResult();
-    } else {
-      conv2d = CreateOpAndInferShape<tosa::Conv2DOp>(
-                   rewriter, loc, UnrankedTensorType::get(resultETy), input,
-                   weight, zeroBias,
-                   /*pad=*/rewriter.getDenseI64ArrayAttr({0, 0, 0, 0}),
-                   /*stride=*/rewriter.getDenseI64ArrayAttr({1, 1}),
-                   /*dilation=*/rewriter.getDenseI64ArrayAttr({1, 1}),
-                   /* acc_type = */ op.getAccTypeAttr())
-                   .getResult();
+    Value inputZp, weightZp;
+    if (maybeZps) {
+      auto maybeInputZp = createZeroPointTensor(
+          rewriter, loc, getElementTypeOrSelf(input.getType()),
+          maybeZps->inputZp);
+      auto maybeWeightZp = createZeroPointTensor(
+          rewriter, loc, getElementTypeOrSelf(weight.getType()),
+          maybeZps->weightZp);
+
+      if (!maybeInputZp.has_value() || !maybeWeightZp.has_value()) {
+        return rewriter.notifyMatchFailure(
+            op, "fail to create a const zero point tensor");
+      }
+
+      inputZp = *maybeInputZp;
+      weightZp = *maybeWeightZp;
     }
 
+    // Perform the convolution using the zero bias.
+    Value conv2d = CreateOpAndInferShape<tosa::Conv2DOp>(
+                       rewriter, loc, UnrankedTensorType::get(resultETy), input,
+                       weight, zeroBias, inputZp, weightZp,
+                       /*pad=*/rewriter.getDenseI64ArrayAttr({0, 0, 0, 0}),
+                       /*stride=*/rewriter.getDenseI64ArrayAttr({1, 1}),
+                       /*dilation=*/rewriter.getDenseI64ArrayAttr({1, 1}),
+                       /* acc_type = */ op.getAccType())
+                       .getResult();
+
     // Factor the resulting width / height.
     ShapedType convTy = cast<ShapedType>(conv2d.getType());
     Type convETy = convTy.getElementType();
@@ -259,9 +260,13 @@ class TransposeConvStridedConverter
     // Factor striding out of the convolution result.
     llvm::SmallVector<int64_t, 6> convReshapeDims0 = {
         batch, convHeight, convWidth, stride[0], stride[1], outputChannels};
+
+    auto convReshapeDims0Value =
+        getTosaConstShape(rewriter, loc, convReshapeDims0);
+
     conv2d = CreateOpAndInferShape<tosa::ReshapeOp>(
         rewriter, loc, UnrankedTensorType::get(resultETy), conv2d,
-        rewriter.getDenseI64ArrayAttr(convReshapeDims0));
+        convReshapeDims0Value);
 
     // Transpose the factored-out stride to the output channels.
     Value transposeConvVal = rewriter.create<tosa::ConstOp>(
@@ -275,9 +280,13 @@ class TransposeConvStridedConverter
     // Fuse striding behavior back into width / height.
     llvm::SmallVector<int64_t, 6> convReshapeDims1 = {
         batch, convHeight * stride[0], convWidth * stride[1], outputChannels};
+
+    auto convReshapeDims1Value =
+        getTosaConstShape(rewriter, loc, convReshapeDims1);
+
     conv2d = CreateOpAndInferShape<tosa::ReshapeOp>(
         rewriter, loc, UnrankedTensorType::get(resultETy), conv2d,
-        rewriter.getDenseI64ArrayAttr(convReshapeDims1));
+        convReshapeDims1Value);
 
     // Determine the amount to slice / pad from the result start.
     int64_t resultSliceTop = std::max<int64_t>(0, -pad[0]);
@@ -302,8 +311,8 @@ class TransposeConvStridedConverter
 
     auto slice = CreateOpAndInferShape<tosa::SliceOp>(
                      rewriter, loc, UnrankedTensorType::get(resultETy), conv2d,
-                     rewriter.getDenseI64ArrayAttr(sliceBegin),
-                     rewriter.getDenseI64ArrayAttr(sliceSize))
+                     getTosaConstShape(rewriter, loc, sliceBegin),
+                     getTosaConstShape(rewriter, loc, sliceSize))
                      .getResult();
 
     llvm::SmallVector<int64_t, 8> resultPadding = {0, 0, 0, 0, 0, 0, 0, 0};
diff --git a/mlir/lib/Dialect/Tosa/Transforms/TosaMakeBroadcastable.cpp b/mlir/lib/Dialect/Tosa/Transforms/TosaMakeBroadcastable.cpp
index 2a990eed3f681..79afc75fd6c8e 100644
--- a/mlir/lib/Dialect/Tosa/Transforms/TosaMakeBroadcastable.cpp
+++ b/mlir/lib/Dialect/Tosa/Transforms/TosaMakeBroadcastable.cpp
@@ -113,7 +113,7 @@ struct ConvertTosaOp<tosa::MulOp> : public OpRewritePattern<tosa::MulOp> {
 
     Value input1 = tosaBinaryOp.getInput1();
     Value input2 = tosaBinaryOp.getInput2();
-    int32_t shift = tosaBinaryOp.getShift();
+    Value shift = tosaBinaryOp.getShift();
     Value output = tosaBinaryOp.getResult();
     auto outputType = dyn_cast<RankedTensorType>(output.getType());
     if (!outputType)
diff --git a/mlir/lib/Dialect/Tosa/Transforms/TosaReduceTransposes.cpp b/mlir/lib/Dialect/Tosa/Transforms/TosaReduceTransposes.cpp
index 539f3e833b12d..64e5c31793f84 100644
--- a/mlir/lib/Dialect/Tosa/Transforms/TosaReduceTransposes.cpp
+++ b/mlir/lib/Dialect/Tosa/Transforms/TosaReduceTransposes.cpp
@@ -187,7 +187,7 @@ TosaReduceTransposes::transposeDenseAttribute(DenseElementsAttr input,
 
   // Asserted by TransposeOp verifier and TOSA disallowing tensor with dimension
   // 0. If not in place, something is very wrong.
-  if (rank <= 0 || oldType.getNumElements() <= 0 || perms.size() != rank) {
+  if (rank <= 0 || oldType.getNumElements() <= 0) {
     signalPassFailure();
     return std::nullopt;
   }
@@ -281,13 +281,19 @@ bool TosaReduceTransposes::collectFanIn(Operation *op,
   if (!llvm::isa<tosa::TransposeOp>(op) && !llvm::isa<tosa::ReshapeOp>(op) &&
       !llvm::isa<tosa::ConstOp>(op)) {
 
-    if (!op->hasTrait<OpTrait::tosa::TosaElementwiseOperator>())
+    if (!llvm::isa<tosa::MulOp>(op) &&
+        !op->hasTrait<OpTrait::tosa::TosaElementwiseOperator>())
       return false;
 
-    for (Value operand : op->getOperands())
+    for (Value operand : op->getOperands()) {
       // If this is a problem in future, think about alternatives to recursion.
+      if (llvm::isa<tosa::MulOp>(op) && operand == op->getOperand(2)) {
+        // do not recurse into MulOp's shift operand
+        continue;
+      }
       if (!collectFanIn(operand.getDefiningOp(), collected))
         return false;
+    }
   }
 
   // Insert in topological order.
@@ -316,7 +322,8 @@ std::optional<Value> TosaReduceTransposes::buildMappedToValue(
     Operation *op, const DenseMap<Value, Value> &valuesMap,
     IRRewriter &rewriter, ArrayRef<int32_t> hoistedPerms) {
   if (op->getNumResults() != 1 ||
-      !op->hasTrait<OpTrait::tosa::TosaElementwiseOperator>())
+      (!llvm::isa<tosa::MulOp>(op) &&
+       !op->hasTrait<OpTrait::tosa::TosaElementwiseOperator>()))
     return std::nullopt;
 
   auto resultType = llvm::cast<RankedTensorType>(op->getResult(0).getType());
@@ -324,6 +331,9 @@ std::optional<Value> TosaReduceTransposes::buildMappedToValue(
   for (Value v : op->getOperands()) {
     if (valuesMap.contains(v)) {
       operands.push_back(valuesMap.at(v));
+    } else if (llvm::isa<tosa::MulOp>(op) && v == op->getOperand(2)) {
+      // special case for MulOp's shift operand
+      operands.push_back(v);
     } else {
       return std::nullopt;
     }
@@ -390,13 +400,20 @@ std::optional<Value> TosaReduceTransposes::buildMappedToValue(
     return std::nullopt;
 
   // Do not insert a TransposeOp, instead we fold the reshape and its attribute.
+  llvm::SmallVector<int64_t> newShape;
+  if (!tosa::getConstShapeValue(reshapeOp.getShape().getDefiningOp(),
+                                newShape)) {
+    // this mean shape is not constant
+    return std::nullopt;
+  }
+  ImplicitLocOpBuilder builder(reshapeOp.getLoc(), rewriter);
   auto foldedReshape = rewriter.create<ReshapeOp>(
       reshapeOp.getLoc(),
       RankedTensorType::get(applyTOSAPermutation(shape, hoistedPerms),
                             reshapeOutputType.getElementType()),
       reshapeOp.getInput1(),
-      rewriter.getDenseI64ArrayAttr(
-          applyTOSAPermutation(reshapeOp.getNewShape(), hoistedPerms)));
+      getTosaConstShape(builder, applyTOSAPermutation(llvm::ArrayRef(newShape),
+                                                      hoistedPerms)));
   return foldedReshape->getResult(0);
 }
 
diff --git a/mlir/lib/Dialect/Tosa/Transforms/TosaValidation.cpp b/mlir/lib/Dialect/Tosa/Transforms/TosaValidation.cpp
index a49870687fdc6..678bb47935bd2 100644
--- a/mlir/lib/Dialect/Tosa/Transforms/TosaValidation.cpp
+++ b/mlir/lib/Dialect/Tosa/Transforms/TosaValidation.cpp
@@ -357,7 +357,7 @@ struct TosaValidation : public tosa::impl::TosaValidationBase<TosaValidation> {
   bool levelCheckTransposeConv2d(Operation *op) {
     if (auto transpose = dyn_cast<tosa::TransposeConv2DOp>(op)) {
       if (ShapedType filterType =
-              dyn_cast<ShapedType>(transpose.getFilter().getType())) {
+              dyn_cast<ShapedType>(transpose.getWeight().getType())) {
         auto shape = filterType.getShape();
         assert(shape.size() == 4);
         // level check kernel sizes for kH and KW
diff --git a/mlir/lib/Dialect/Tosa/Utils/ConversionUtils.cpp b/mlir/lib/Dialect/Tosa/Utils/ConversionUtils.cpp
index 62b0bc1857e39..8ab12d038849f 100644
--- a/mlir/lib/Dialect/Tosa/Utils/ConversionUtils.cpp
+++ b/mlir/lib/Dialect/Tosa/Utils/ConversionUtils.cpp
@@ -145,10 +145,10 @@ LogicalResult mlir::tosa::EqualizeRanks(ImplicitLocOpBuilder &builder,
       llvm::cast<RankedTensorType>(lowerTensorValue.getType());
   auto reshapeOutputType = RankedTensorType::get(
       ArrayRef<int64_t>(reshapeOutputShape), reshapeInputType.getElementType());
+  auto reshapeOutputShapeValue = getTosaConstShape(builder, reshapeOutputShape);
 
   auto reshapeLower = builder.create<tosa::ReshapeOp>(
-      reshapeOutputType, lowerTensorValue,
-      builder.getDenseI64ArrayAttr(reshapeOutputShape));
+      reshapeOutputType, lowerTensorValue, reshapeOutputShapeValue);
 
   if (input1Rank > input2Rank) {
     input1 = higherTensorValue;
@@ -161,15 +161,20 @@ LogicalResult mlir::tosa::EqualizeRanks(ImplicitLocOpBuilder &builder,
   return success();
 }
 
-Value mlir::tosa::getTosaConstShape(PatternRewriter &rewriter, Location loc,
+Value mlir::tosa::getTosaConstShape(ImplicitLocOpBuilder &builder,
                                     llvm::ArrayRef<int64_t> shape) {
-  auto attr = rewriter.getIndexTensorAttr(shape);
-  auto type = mlir::tosa::shapeType::get(rewriter.getContext(), shape.size());
-  mlir::Operation *mlir_op =
-      rewriter.create<tosa::ConstShapeOp>(loc, type, attr);
+  auto attr = builder.getIndexTensorAttr(convertFromMlirShape(shape));
+  auto type = mlir::tosa::shapeType::get(builder.getContext(), shape.size());
+  mlir::Operation *mlir_op = builder.create<tosa::ConstShapeOp>(type, attr);
   return mlir_op->getResult(0);
 }
 
+Value mlir::tosa::getTosaConstShape(PatternRewriter &rewriter, Location loc,
+                                    llvm::ArrayRef<int64_t> shape) {
+  ImplicitLocOpBuilder builder(loc, rewriter);
+  return getTosaConstShape(builder, shape);
+}
+
 SmallVector<int64_t> mlir::tosa::convertFromMlirShape(ArrayRef<int64_t> shape) {
   return to_vector(llvm::map_range(shape, [](int64_t dim) {
     return ShapedType::isDynamic(dim) ? -1 : dim;
diff --git a/mlir/lib/Dialect/Tosa/Utils/QuantUtils.cpp b/mlir/lib/Dialect/Tosa/Utils/QuantUtils.cpp
index 5c546f59cde41..0f7562767001c 100644
--- a/mlir/lib/Dialect/Tosa/Utils/QuantUtils.cpp
+++ b/mlir/lib/Dialect/Tosa/Utils/QuantUtils.cpp
@@ -112,19 +112,14 @@ void mlir::tosa::computeMultiplierAndShift(double scale, int32_t &multiplier,
 #define GET_QTYPE(inputType)                                                   \
   (llvm::dyn_cast<quant::QuantizedType>((inputType).getElementType()))
 
-/// Method to build ConvOpQuantizationAttr, called from
-/// ConvOpQuantInfoBuilder/TransConvOpQuantInfoBuilder:
-/// input_zp: input zeropoint
-/// weight_zp: weight zeropoint.
-ConvOpQuantizationAttr
-mlir::tosa::buildConvOpQuantizationAttr(OpBuilder &builder, Value input,
-                                        Value weight) {
+static std::optional<std::pair<std::int64_t, std::int64_t>>
+getConvZeroPoints(Value input, Value weight) {
 
   auto inputType = dyn_cast<ShapedType>(input.getType());
   auto weightType = dyn_cast<ShapedType>(weight.getType());
 
   if (!inputType || !weightType)
-    return nullptr;
+    return std::nullopt;
 
   auto inputQType = GET_UQTYPE(inputType);
   auto weightPerTensorQType = GET_UQTYPE(weightType);
@@ -150,10 +145,58 @@ mlir::tosa::buildConvOpQuantizationAttr(OpBuilder &builder, Value input,
       weightZp = weightPerAxisQType.getZeroPoints().front();
     }
 
-    return builder.getAttr<tosa::ConvOpQuantizationAttr>(inputZp, weightZp);
+    return std::make_pair(inputZp, weightZp);
   }
 
-  return nullptr;
+  return std::nullopt;
+}
+
+std::pair<Value, Value>
+mlir::tosa::createZPsAsConst(OpBuilder &builder, Value input, Value weight) {
+  std::int64_t inputZp, weightZp;
+
+  auto inputEType = getElementTypeOrSelf(input.getType());
+  auto weightEType = getElementTypeOrSelf(weight.getType());
+
+  if (mlir::isa<FloatType>(inputEType) && mlir::isa<FloatType>(weightEType)) {
+    inputZp = 0;
+    weightZp = 0;
+  } else {
+    auto maybeZps = getConvZeroPoints(input, weight);
+    if (!maybeZps.has_value())
+      return {};
+
+    inputZp = maybeZps->first;
+    weightZp = maybeZps->second;
+  }
+
+  auto maybeInputZpValue =
+      createZeroPointTensor(builder, input.getLoc(), inputEType, inputZp);
+  if (!maybeInputZpValue.has_value())
+    return {};
+
+  auto maybeWeightZpValue =
+      createZeroPointTensor(builder, weight.getLoc(), weightEType, weightZp);
+  if (!maybeWeightZpValue.has_value())
+    return {};
+
+  return std::make_pair(*maybeInputZpValue, *maybeWeightZpValue);
+}
+
+/// Method to build ConvOpQuantizationAttr, called from
+/// ConvOpQuantInfoBuilder/TransConvOpQuantInfoBuilder:
+/// input_zp: input zeropoint
+/// weight_zp: weight zeropoint.
+ConvOpQuantizationAttr
+mlir::tosa::buildConvOpQuantizationAttr(OpBuilder &builder, Value input,
+                                        Value weight) {
+
+  auto maybeZps = getConvZeroPoints(input, weight);
+  if (!maybeZps.has_value())
+    return nullptr;
+
+  return builder.getAttr<tosa::ConvOpQuantizationAttr>(maybeZps->first,
+                                                       maybeZps->second);
 }
 
 /// Builds MatMulOpQuantizationAttr, called from
diff --git a/mlir/lib/Dialect/Utils/StaticValueUtils.cpp b/mlir/lib/Dialect/Utils/StaticValueUtils.cpp
index 5c8f6ded39ba4..fcb736aa031f3 100644
--- a/mlir/lib/Dialect/Utils/StaticValueUtils.cpp
+++ b/mlir/lib/Dialect/Utils/StaticValueUtils.cpp
@@ -191,7 +191,8 @@ bool isEqualConstantIntOrValueArray(ArrayRef<OpFoldResult> ofrs1,
 /// elements for which ShapedType::isDynamic is true, will be replaced by
 /// dynamicValues.
 SmallVector<OpFoldResult> getMixedValues(ArrayRef<int64_t> staticValues,
-                                         ValueRange dynamicValues, Builder &b) {
+                                         ValueRange dynamicValues,
+                                         MLIRContext *context) {
   SmallVector<OpFoldResult> res;
   res.reserve(staticValues.size());
   unsigned numDynamic = 0;
@@ -200,10 +201,15 @@ SmallVector<OpFoldResult> getMixedValues(ArrayRef<int64_t> staticValues,
     int64_t value = staticValues[idx];
     res.push_back(ShapedType::isDynamic(value)
                       ? OpFoldResult{dynamicValues[numDynamic++]}
-                      : OpFoldResult{b.getI64IntegerAttr(staticValues[idx])});
+                      : OpFoldResult{IntegerAttr::get(
+                            IntegerType::get(context, 64), staticValues[idx])});
   }
   return res;
 }
+SmallVector<OpFoldResult> getMixedValues(ArrayRef<int64_t> staticValues,
+                                         ValueRange dynamicValues, Builder &b) {
+  return getMixedValues(staticValues, dynamicValues, b.getContext());
+}
 
 /// Decompose a vector of mixed static or dynamic values into the corresponding
 /// pair of arrays. This is the inverse function of `getMixedValues`.
diff --git a/mlir/lib/Dialect/Vector/IR/VectorOps.cpp b/mlir/lib/Dialect/Vector/IR/VectorOps.cpp
index 3fbfcb4979b49..d36bfddc13279 100644
--- a/mlir/lib/Dialect/Vector/IR/VectorOps.cpp
+++ b/mlir/lib/Dialect/Vector/IR/VectorOps.cpp
@@ -13,19 +13,20 @@
 
 #include "mlir/Dialect/Vector/IR/VectorOps.h"
 
+#include "mlir/Conversion/ConvertToLLVM/ToLLVMInterface.h"
 #include "mlir/Dialect/Affine/IR/ValueBoundsOpInterfaceImpl.h"
 #include "mlir/Dialect/Arith/IR/Arith.h"
 #include "mlir/Dialect/Arith/Utils/Utils.h"
 #include "mlir/Dialect/Bufferization/IR/BufferizableOpInterface.h"
 #include "mlir/Dialect/MemRef/IR/MemRef.h"
 #include "mlir/Dialect/Tensor/IR/Tensor.h"
+#include "mlir/Dialect/UB/IR/UBOps.h"
 #include "mlir/Dialect/Utils/IndexingUtils.h"
 #include "mlir/Dialect/Utils/StructuredOpsUtils.h"
 #include "mlir/IR/AffineExpr.h"
 #include "mlir/IR/AffineMap.h"
 #include "mlir/IR/Builders.h"
 #include "mlir/IR/BuiltinAttributes.h"
-#include "mlir/IR/BuiltinOps.h"
 #include "mlir/IR/BuiltinTypes.h"
 #include "mlir/IR/DialectImplementation.h"
 #include "mlir/IR/IRMapping.h"
@@ -41,7 +42,6 @@
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/StringSet.h"
 #include "llvm/ADT/TypeSwitch.h"
-#include "llvm/ADT/bit.h"
 
 #include <cassert>
 #include <cstdint>
@@ -429,6 +429,7 @@ void VectorDialect::initialize() {
                             TransferWriteOp>();
   declarePromisedInterface<SubsetExtractionOpInterface, TransferReadOp>();
   declarePromisedInterface<SubsetInsertionOpInterface, TransferWriteOp>();
+  declarePromisedInterface<ConvertToLLVMPatternInterface, VectorDialect>();
 }
 
 /// Materialize a single constant operation from a given attribute value with
@@ -436,6 +437,9 @@ void VectorDialect::initialize() {
 Operation *VectorDialect::materializeConstant(OpBuilder &builder,
                                               Attribute value, Type type,
                                               Location loc) {
+  if (isa<ub::PoisonAttrInterface>(value))
+    return value.getDialect().materializeConstant(builder, value, type, loc);
+
   return arith::ConstantOp::materialize(builder, value, type, loc);
 }
 
@@ -1274,6 +1278,13 @@ OpFoldResult vector::ExtractElementOp::fold(FoldAdaptor adaptor) {
   return srcElements[posIdx];
 }
 
+// Returns `true` if `index` is either within [0, maxIndex) or equal to
+// `poisonValue`.
+static bool isValidPositiveIndexOrPoison(int64_t index, int64_t poisonValue,
+                                         int64_t maxIndex) {
+  return index == poisonValue || (index >= 0 && index < maxIndex);
+}
+
 //===----------------------------------------------------------------------===//
 // ExtractOp
 //===----------------------------------------------------------------------===//
@@ -1355,11 +1366,12 @@ LogicalResult vector::ExtractOp::verify() {
   for (auto [idx, pos] : llvm::enumerate(position)) {
     if (auto attr = dyn_cast<Attribute>(pos)) {
       int64_t constIdx = cast<IntegerAttr>(attr).getInt();
-      if (constIdx < 0 || constIdx >= getSourceVectorType().getDimSize(idx)) {
+      if (!isValidPositiveIndexOrPoison(
+              constIdx, kPoisonIndex, getSourceVectorType().getDimSize(idx))) {
         return emitOpError("expected position attribute #")
                << (idx + 1)
                << " to be a non-negative integer smaller than the "
-                  "corresponding vector dimension";
+                  "corresponding vector dimension or poison (-1)";
       }
     }
   }
@@ -1479,7 +1491,7 @@ class ExtractFromInsertTransposeChainState {
 
   /// Try to fold in place to extract(source, extractPosition) and return the
   /// folded result. Return null if folding is not possible (e.g. due to an
-  /// internal tranposition in the result).
+  /// internal transposition in the result).
   Value tryToFoldExtractOpInPlace(Value source);
 
   ExtractOp extractOp;
@@ -1573,7 +1585,7 @@ ExtractFromInsertTransposeChainState::handleInsertOpWithPrefixPos(Value &res) {
 
 /// Try to fold in place to extract(source, extractPosition) and return the
 /// folded result. Return null if folding is not possible (e.g. due to an
-/// internal tranposition in the result).
+/// internal transposition in the result).
 Value ExtractFromInsertTransposeChainState::tryToFoldExtractOpInPlace(
     Value source) {
   // TODO: Canonicalization for dynamic position not implemented yet.
@@ -1977,12 +1989,75 @@ static Value foldScalarExtractFromFromElements(ExtractOp extractOp) {
   return fromElementsOp.getElements()[flatIndex];
 }
 
-OpFoldResult ExtractOp::fold(FoldAdaptor) {
+/// If the dynamic indices of `extractOp` or `insertOp` are in fact constants,
+/// then fold it.
+template <typename OpType, typename AdaptorType>
+static Value extractInsertFoldConstantOp(OpType op, AdaptorType adaptor,
+                                         SmallVectorImpl<Value> &operands) {
+  std::vector<int64_t> staticPosition = op.getStaticPosition().vec();
+  OperandRange dynamicPosition = op.getDynamicPosition();
+  ArrayRef<Attribute> dynamicPositionAttr = adaptor.getDynamicPosition();
+
+  // If the dynamic operands is empty, it is returned directly.
+  if (!dynamicPosition.size())
+    return {};
+
+  // `index` is used to iterate over the `dynamicPosition`.
+  unsigned index = 0;
+
+  // `opChange` is a flag. If it is true, it means to update `op` in place.
+  bool opChange = false;
+  for (unsigned i = 0, e = staticPosition.size(); i < e; ++i) {
+    if (!ShapedType::isDynamic(staticPosition[i]))
+      continue;
+    Attribute positionAttr = dynamicPositionAttr[index];
+    Value position = dynamicPosition[index++];
+    if (auto attr = mlir::dyn_cast_if_present<IntegerAttr>(positionAttr)) {
+      staticPosition[i] = attr.getInt();
+      opChange = true;
+      continue;
+    }
+    operands.push_back(position);
+  }
+
+  if (opChange) {
+    op.setStaticPosition(staticPosition);
+    op.getOperation()->setOperands(operands);
+    return op.getResult();
+  }
+  return {};
+}
+
+/// Fold an insert or extract operation into an poison value when a poison index
+/// is found at any dimension of the static position.
+static Attribute foldPoisonIndexInsertExtractOp(MLIRContext *context,
+                                                ArrayRef<int64_t> staticPos,
+                                                int64_t poisonVal) {
+  if (!llvm::is_contained(staticPos, poisonVal))
+    return {};
+
+  return ub::PoisonAttr::get(context);
+}
+
+/// Fold a vector extract from is a poison source.
+static Attribute foldPoisonSrcExtractOp(Attribute srcAttr) {
+  if (llvm::isa_and_nonnull<ub::PoisonAttr>(srcAttr))
+    return srcAttr;
+
+  return {};
+}
+
+OpFoldResult ExtractOp::fold(FoldAdaptor adaptor) {
   // Fold "vector.extract %v[] : vector<2x2xf32> from vector<2x2xf32>" to %v.
   // Note: Do not fold "vector.extract %v[] : f32 from vector<f32>" (type
   // mismatch).
   if (getNumIndices() == 0 && getVector().getType() == getResult().getType())
     return getVector();
+  if (auto res = foldPoisonIndexInsertExtractOp(
+          getContext(), adaptor.getStaticPosition(), kPoisonIndex))
+    return res;
+  if (auto res = foldPoisonSrcExtractOp(adaptor.getVector()))
+    return res;
   if (succeeded(foldExtractOpFromExtractChain(*this)))
     return getResult();
   if (auto res = ExtractFromInsertTransposeChainState(*this).fold())
@@ -1999,6 +2074,9 @@ OpFoldResult ExtractOp::fold(FoldAdaptor) {
     return val;
   if (auto val = foldScalarExtractFromFromElements(*this))
     return val;
+  SmallVector<Value> operands = {getVector()};
+  if (auto val = extractInsertFoldConstantOp(*this, adaptor, operands))
+    return val;
   return OpFoldResult();
 }
 
@@ -2249,6 +2327,7 @@ LogicalResult foldExtractFromFromElements(ExtractOp extractOp,
                                          resultType.getNumElements()));
   return success();
 }
+
 } // namespace
 
 void ExtractOp::getCanonicalizationPatterns(RewritePatternSet &results,
@@ -2316,7 +2395,7 @@ computeBroadcastedUnitDims(ArrayRef<int64_t> srcShape,
   for (auto [s1, s2] :
        llvm::zip_equal(srcShape, dstShape.drop_front(rankDiff))) {
     if (s1 != s2) {
-      assert(s1 == 1 && "expected dim-1 broadcasting");
+      assert(s1 == 1 && "expected \"dim-1\" broadcasting");
       res.insert(dstDim);
     }
     ++dstDim;
@@ -2335,7 +2414,7 @@ llvm::SetVector<int64_t> BroadcastOp::computeBroadcastedUnitDims() {
 
 /// Broadcast `value` to a vector of `dstShape`, knowing that exactly the
 /// `broadcastedDims` dimensions in the dstShape are broadcasted.
-/// This requires (and asserts) that the broadcast is free of dim-1
+/// This requires (and asserts) that the broadcast is free of "dim-1"
 /// broadcasting.
 /// Since vector.broadcast only allows expanding leading dimensions, an extra
 /// vector.transpose may be inserted to make the broadcast possible.
@@ -2421,10 +2500,10 @@ Value BroadcastOp::createOrFoldBroadcastOp(
   // 3.c. Append the srcShape.
   llvm::append_range(broadcastShape, srcVectorType.getShape());
 
-  // Ensure there are no dim-1 broadcasts.
+  // Ensure there are no "dim-1" broadcasts.
   assert(::computeBroadcastedUnitDims(srcVectorType.getShape(), broadcastShape)
              .empty() &&
-         "unexpected dim-1 broadcast");
+         "unexpected \"dim-1\" broadcast");
 
   VectorType broadcastType = VectorType::get(broadcastShape, elementType);
   assert(vector::isBroadcastableTo(value.getType(), broadcastType) ==
@@ -2600,7 +2679,7 @@ LogicalResult ShuffleOp::verify() {
   int64_t indexSize = (v1Type.getRank() == 0 ? 1 : v1Type.getDimSize(0)) +
                       (v2Type.getRank() == 0 ? 1 : v2Type.getDimSize(0));
   for (auto [idx, maskPos] : llvm::enumerate(mask)) {
-    if (maskPos != kMaskPoisonValue && (maskPos < 0 || maskPos >= indexSize))
+    if (!isValidPositiveIndexOrPoison(maskPos, kPoisonIndex, indexSize))
       return emitOpError("mask index #") << (idx + 1) << " out of range";
   }
   return success();
@@ -2634,43 +2713,71 @@ static bool isStepIndexArray(ArrayRef<T> idxArr, uint64_t begin, size_t width) {
 }
 
 OpFoldResult vector::ShuffleOp::fold(FoldAdaptor adaptor) {
-  VectorType v1Type = getV1VectorType();
+  auto v1Type = getV1VectorType();
+  auto v2Type = getV2VectorType();
+
+  assert(!v1Type.isScalable() && !v2Type.isScalable() &&
+         "Vector shuffle does not support scalable vectors");
+
   // For consistency: 0-D shuffle return type is 1-D, this cannot be a folding
   // but must be a canonicalization into a vector.broadcast.
   if (v1Type.getRank() == 0)
     return {};
 
-  // fold shuffle V1, V2, [0, 1, 2, 3] : <4xi32>, <2xi32> -> V1
-  if (!v1Type.isScalable() &&
-      isStepIndexArray(getMask(), 0, v1Type.getDimSize(0)))
+  // Fold shuffle V1, V2, [0, 1, 2, 3] : <4xi32>, <2xi32> -> V1.
+  auto mask = getMask();
+  if (isStepIndexArray(mask, 0, v1Type.getDimSize(0)))
     return getV1();
-  // fold shuffle V1, V2, [4, 5] : <4xi32>, <2xi32> -> V2
-  if (!getV1VectorType().isScalable() && !getV2VectorType().isScalable() &&
-      isStepIndexArray(getMask(), getV1VectorType().getDimSize(0),
-                       getV2VectorType().getDimSize(0)))
+  // Fold shuffle V1, V2, [4, 5] : <4xi32>, <2xi32> -> V2.
+  if (isStepIndexArray(mask, v1Type.getDimSize(0), v2Type.getDimSize(0)))
     return getV2();
 
-  Attribute lhs = adaptor.getV1(), rhs = adaptor.getV2();
-  if (!lhs || !rhs)
+  Attribute v1Attr = adaptor.getV1(), v2Attr = adaptor.getV2();
+  if (!v1Attr || !v2Attr)
     return {};
 
-  auto lhsType =
-      llvm::cast<VectorType>(llvm::cast<DenseElementsAttr>(lhs).getType());
+  // Fold shuffle poison, poison -> poison.
+  bool isV1Poison = isa<ub::PoisonAttr>(v1Attr);
+  bool isV2Poison = isa<ub::PoisonAttr>(v2Attr);
+  if (isV1Poison && isV2Poison)
+    return ub::PoisonAttr::get(getContext());
+
   // Only support 1-D for now to avoid complicated n-D DenseElementsAttr
   // manipulation.
-  if (lhsType.getRank() != 1)
+  if (v1Type.getRank() != 1)
     return {};
-  int64_t lhsSize = lhsType.getDimSize(0);
+
+  // Poison input attributes need special handling as they are not
+  // DenseElementsAttr. If an index is poison, we select the first element of
+  // the first non-poison input.
+  SmallVector<Attribute> v1Elements, v2Elements;
+  Attribute poisonElement;
+  if (!isV2Poison) {
+    v2Elements =
+        to_vector(cast<DenseElementsAttr>(v2Attr).getValues<Attribute>());
+    poisonElement = v2Elements[0];
+  }
+  if (!isV1Poison) {
+    v1Elements =
+        to_vector(cast<DenseElementsAttr>(v1Attr).getValues<Attribute>());
+    poisonElement = v1Elements[0];
+  }
 
   SmallVector<Attribute> results;
-  auto lhsElements = llvm::cast<DenseElementsAttr>(lhs).getValues<Attribute>();
-  auto rhsElements = llvm::cast<DenseElementsAttr>(rhs).getValues<Attribute>();
-  for (int64_t i : this->getMask()) {
-    if (i >= lhsSize) {
-      results.push_back(rhsElements[i - lhsSize]);
+  int64_t v1Size = v1Type.getDimSize(0);
+  for (int64_t maskIdx : mask) {
+    Attribute indexedElm;
+    // TODO: Return a partial poison vector when supported by the UB dialect.
+    if (maskIdx == ShuffleOp::kPoisonIndex) {
+      indexedElm = poisonElement;
     } else {
-      results.push_back(lhsElements[i]);
+      if (maskIdx < v1Size)
+        indexedElm = isV1Poison ? poisonElement : v1Elements[maskIdx];
+      else
+        indexedElm = isV2Poison ? poisonElement : v2Elements[maskIdx - v1Size];
     }
+
+    results.push_back(indexedElm);
   }
 
   return DenseElementsAttr::get(getResultVectorType(), results);
@@ -2882,7 +2989,8 @@ LogicalResult InsertOp::verify() {
   for (auto [idx, pos] : llvm::enumerate(position)) {
     if (auto attr = pos.dyn_cast<Attribute>()) {
       int64_t constIdx = cast<IntegerAttr>(attr).getInt();
-      if (constIdx < 0 || constIdx >= destVectorType.getDimSize(idx)) {
+      if (!isValidPositiveIndexOrPoison(constIdx, kPoisonIndex,
+                                        destVectorType.getDimSize(idx))) {
         return emitOpError("expected position attribute #")
                << (idx + 1)
                << " to be a non-negative integer smaller than the "
@@ -3028,6 +3136,13 @@ OpFoldResult vector::InsertOp::fold(FoldAdaptor adaptor) {
   // (type mismatch).
   if (getNumIndices() == 0 && getSourceType() == getType())
     return getSource();
+  SmallVector<Value> operands = {getSource(), getDest()};
+  if (auto val = extractInsertFoldConstantOp(*this, adaptor, operands))
+    return val;
+  if (auto res = foldPoisonIndexInsertExtractOp(
+          getContext(), adaptor.getStaticPosition(), kPoisonIndex))
+    return res;
+
   return {};
 }
 
@@ -3279,13 +3394,15 @@ class InsertStridedSliceConstantFolder final
         !destVector.hasOneUse())
       return failure();
 
-    auto denseDest = llvm::cast<DenseElementsAttr>(vectorDestCst);
-
     TypedValue<VectorType> sourceValue = op.getSource();
     Attribute sourceCst;
     if (!matchPattern(sourceValue, m_Constant(&sourceCst)))
       return failure();
 
+    // TODO: Support poison.
+    if (isa<ub::PoisonAttr>(vectorDestCst) || isa<ub::PoisonAttr>(sourceCst))
+      return failure();
+
     // TODO: Handle non-unit strides when they become available.
     if (op.hasNonUnitStrides())
       return failure();
@@ -3302,6 +3419,7 @@ class InsertStridedSliceConstantFolder final
     // increasing linearized position indices.
     // Because the destination may have higher dimensionality then the slice,
     // we keep track of two overlapping sets of positions and offsets.
+    auto denseDest = llvm::cast<DenseElementsAttr>(vectorDestCst);
     auto denseSlice = llvm::cast<DenseElementsAttr>(sourceCst);
     auto sliceValuesIt = denseSlice.value_begin<Attribute>();
     auto newValues = llvm::to_vector(denseDest.getValues<Attribute>());
diff --git a/mlir/lib/Dialect/Vector/Transforms/LowerVectorBitCast.cpp b/mlir/lib/Dialect/Vector/Transforms/LowerVectorBitCast.cpp
index d8c4939dc742a..89930a6bd35fa 100644
--- a/mlir/lib/Dialect/Vector/Transforms/LowerVectorBitCast.cpp
+++ b/mlir/lib/Dialect/Vector/Transforms/LowerVectorBitCast.cpp
@@ -11,6 +11,7 @@
 //
 //===----------------------------------------------------------------------===//
 
+#include "mlir/Dialect/UB/IR/UBOps.h"
 #include "mlir/Dialect/Vector/IR/VectorOps.h"
 #include "mlir/Dialect/Vector/Transforms/LoweringPatterns.h"
 #include "mlir/Dialect/Vector/Utils/VectorUtils.h"
@@ -32,7 +33,7 @@ namespace {
 ///
 /// Would be unrolled to:
 ///
-/// %result = arith.constant dense<0> : vector<1x2x3x8xi32>
+/// %result = ub.poison : vector<1x2x3x8xi32>
 /// %0 = vector.extract %a[0, 0, 0]                 ─┐
 ///        : vector<4xi64> from vector<1x2x3x4xi64>  |
 /// %1 = vector.bitcast %0                           | - Repeated 6x for
@@ -63,8 +64,7 @@ class UnrollBitCastOp final : public OpRewritePattern<vector::BitCastOp> {
         VectorType::get(shape, resultType.getElementType(), scalableDims);
 
     Location loc = op.getLoc();
-    Value result = rewriter.create<arith::ConstantOp>(
-        loc, resultType, rewriter.getZeroAttr(resultType));
+    Value result = rewriter.create<ub::PoisonOp>(loc, resultType);
     for (auto position : *unrollIterator) {
       Value extract =
           rewriter.create<vector::ExtractOp>(loc, op.getSource(), position);
diff --git a/mlir/lib/Dialect/Vector/Transforms/LowerVectorBroadcast.cpp b/mlir/lib/Dialect/Vector/Transforms/LowerVectorBroadcast.cpp
index 6c36bbaee8523..fec3c6c52e5e4 100644
--- a/mlir/lib/Dialect/Vector/Transforms/LowerVectorBroadcast.cpp
+++ b/mlir/lib/Dialect/Vector/Transforms/LowerVectorBroadcast.cpp
@@ -11,27 +11,16 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "mlir/Dialect/Affine/IR/AffineOps.h"
-#include "mlir/Dialect/Arith/IR/Arith.h"
-#include "mlir/Dialect/Arith/Utils/Utils.h"
-#include "mlir/Dialect/Linalg/IR/Linalg.h"
 #include "mlir/Dialect/MemRef/IR/MemRef.h"
-#include "mlir/Dialect/SCF/IR/SCF.h"
-#include "mlir/Dialect/Tensor/IR/Tensor.h"
-#include "mlir/Dialect/Utils/IndexingUtils.h"
-#include "mlir/Dialect/Utils/StructuredOpsUtils.h"
+#include "mlir/Dialect/UB/IR/UBOps.h"
 #include "mlir/Dialect/Vector/IR/VectorOps.h"
 #include "mlir/Dialect/Vector/Transforms/LoweringPatterns.h"
 #include "mlir/Dialect/Vector/Transforms/VectorRewritePatterns.h"
 #include "mlir/Dialect/Vector/Utils/VectorUtils.h"
-#include "mlir/IR/BuiltinAttributeInterfaces.h"
 #include "mlir/IR/BuiltinTypes.h"
-#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/IR/Location.h"
-#include "mlir/IR/Matchers.h"
 #include "mlir/IR/PatternMatch.h"
 #include "mlir/IR/TypeUtilities.h"
-#include "mlir/Interfaces/VectorInterfaces.h"
 
 #define DEBUG_TYPE "vector-broadcast-lowering"
 
@@ -86,8 +75,7 @@ class BroadcastOpLowering : public OpRewritePattern<vector::BroadcastOp> {
       VectorType resType = VectorType::Builder(dstType).dropDim(0);
       Value bcst =
           rewriter.create<vector::BroadcastOp>(loc, resType, op.getSource());
-      Value result = rewriter.create<arith::ConstantOp>(
-          loc, dstType, rewriter.getZeroAttr(dstType));
+      Value result = rewriter.create<ub::PoisonOp>(loc, dstType);
       for (int64_t d = 0, dim = dstType.getDimSize(0); d < dim; ++d)
         result = rewriter.create<vector::InsertOp>(loc, bcst, result, d);
       rewriter.replaceOp(op, result);
@@ -127,8 +115,7 @@ class BroadcastOpLowering : public OpRewritePattern<vector::BroadcastOp> {
     VectorType resType =
         VectorType::get(dstType.getShape().drop_front(), eltType,
                         dstType.getScalableDims().drop_front());
-    Value result = rewriter.create<arith::ConstantOp>(
-        loc, dstType, rewriter.getZeroAttr(dstType));
+    Value result = rewriter.create<ub::PoisonOp>(loc, dstType);
     if (m == 0) {
       // Stetch at start.
       Value ext = rewriter.create<vector::ExtractOp>(loc, op.getSource(), 0);
diff --git a/mlir/lib/Dialect/Vector/Transforms/LowerVectorShapeCast.cpp b/mlir/lib/Dialect/Vector/Transforms/LowerVectorShapeCast.cpp
index 95ebd4e9fe3d9..9c1e5fcee91de 100644
--- a/mlir/lib/Dialect/Vector/Transforms/LowerVectorShapeCast.cpp
+++ b/mlir/lib/Dialect/Vector/Transforms/LowerVectorShapeCast.cpp
@@ -11,40 +11,43 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "mlir/Dialect/Affine/IR/AffineOps.h"
-#include "mlir/Dialect/Arith/IR/Arith.h"
-#include "mlir/Dialect/Arith/Utils/Utils.h"
-#include "mlir/Dialect/Linalg/IR/Linalg.h"
 #include "mlir/Dialect/MemRef/IR/MemRef.h"
-#include "mlir/Dialect/SCF/IR/SCF.h"
-#include "mlir/Dialect/Tensor/IR/Tensor.h"
-#include "mlir/Dialect/Utils/IndexingUtils.h"
-#include "mlir/Dialect/Utils/StructuredOpsUtils.h"
+#include "mlir/Dialect/UB//IR/UBOps.h"
 #include "mlir/Dialect/Vector/IR/VectorOps.h"
 #include "mlir/Dialect/Vector/Transforms/LoweringPatterns.h"
 #include "mlir/Dialect/Vector/Transforms/VectorRewritePatterns.h"
 #include "mlir/Dialect/Vector/Utils/VectorUtils.h"
-#include "mlir/IR/BuiltinAttributeInterfaces.h"
 #include "mlir/IR/BuiltinTypes.h"
-#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/IR/Location.h"
-#include "mlir/IR/Matchers.h"
 #include "mlir/IR/PatternMatch.h"
 #include "mlir/IR/TypeUtilities.h"
-#include "mlir/Interfaces/VectorInterfaces.h"
 
 #define DEBUG_TYPE "vector-shape-cast-lowering"
 
 using namespace mlir;
 using namespace mlir::vector;
 
+/// Increments n-D `indices` by `step` starting from the innermost dimension.
+static void incIdx(SmallVectorImpl<int64_t> &indices, VectorType vecType,
+                   int step = 1) {
+  for (int dim : llvm::reverse(llvm::seq<int>(0, indices.size()))) {
+    assert(indices[dim] < vecType.getDimSize(dim) &&
+           "Indices are out of bound");
+    indices[dim] += step;
+    if (indices[dim] < vecType.getDimSize(dim))
+      break;
+
+    indices[dim] = 0;
+    step = 1;
+  }
+}
+
 namespace {
-/// ShapeOp 2D -> 1D downcast serves the purpose of flattening 2-D to 1-D
-/// vectors progressively on the way to target llvm.matrix intrinsics.
-/// This iterates over the most major dimension of the 2-D vector and performs
-/// rewrites into:
-///   vector.extract from 2-D + vector.insert_strided_slice offset into 1-D
-class ShapeCastOp2DDownCastRewritePattern
+/// ShapeOp n-D -> 1-D downcast serves the purpose of flattening N-D to 1-D
+/// vectors progressively. This iterates over the n-1 major dimensions of the
+/// n-D vector and performs rewrites into:
+///   vector.extract from n-D + vector.insert_strided_slice offset into 1-D
+class ShapeCastOpNDDownCastRewritePattern
     : public OpRewritePattern<vector::ShapeCastOp> {
 public:
   using OpRewritePattern::OpRewritePattern;
@@ -53,35 +56,51 @@ class ShapeCastOp2DDownCastRewritePattern
                                 PatternRewriter &rewriter) const override {
     auto sourceVectorType = op.getSourceVectorType();
     auto resultVectorType = op.getResultVectorType();
-
     if (sourceVectorType.isScalable() || resultVectorType.isScalable())
       return failure();
 
-    if (sourceVectorType.getRank() != 2 || resultVectorType.getRank() != 1)
+    int64_t srcRank = sourceVectorType.getRank();
+    int64_t resRank = resultVectorType.getRank();
+    if (srcRank < 2 || resRank != 1)
       return failure();
 
+    // Compute the number of 1-D vector elements involved in the reshape.
+    int64_t numElts = 1;
+    for (int64_t dim = 0; dim < srcRank - 1; ++dim)
+      numElts *= sourceVectorType.getDimSize(dim);
+
     auto loc = op.getLoc();
-    Value desc = rewriter.create<arith::ConstantOp>(
-        loc, resultVectorType, rewriter.getZeroAttr(resultVectorType));
-    unsigned mostMinorVectorSize = sourceVectorType.getShape()[1];
-    for (int64_t i = 0, e = sourceVectorType.getShape().front(); i != e; ++i) {
-      Value vec = rewriter.create<vector::ExtractOp>(loc, op.getSource(), i);
-      desc = rewriter.create<vector::InsertStridedSliceOp>(
-          loc, vec, desc,
-          /*offsets=*/i * mostMinorVectorSize, /*strides=*/1);
+    SmallVector<int64_t> srcIdx(srcRank - 1, 0);
+    SmallVector<int64_t> resIdx(resRank, 0);
+    int64_t extractSize = sourceVectorType.getShape().back();
+    Value result = rewriter.create<ub::PoisonOp>(loc, resultVectorType);
+
+    // Compute the indices of each 1-D vector element of the source extraction
+    // and destination slice insertion and generate such instructions.
+    for (int64_t i = 0; i < numElts; ++i) {
+      if (i != 0) {
+        incIdx(srcIdx, sourceVectorType, /*step=*/1);
+        incIdx(resIdx, resultVectorType, /*step=*/extractSize);
+      }
+
+      Value extract =
+          rewriter.create<vector::ExtractOp>(loc, op.getSource(), srcIdx);
+      result = rewriter.create<vector::InsertStridedSliceOp>(
+          loc, extract, result,
+          /*offsets=*/resIdx, /*strides=*/1);
     }
-    rewriter.replaceOp(op, desc);
+
+    rewriter.replaceOp(op, result);
     return success();
   }
 };
 
-/// ShapeOp 1D -> 2D upcast serves the purpose of unflattening 2-D from 1-D
-/// vectors progressively.
-/// This iterates over the most major dimension of the 2-D vector and performs
-/// rewrites into:
-///   vector.extract_strided_slice from 1-D + vector.insert into 2-D
+/// ShapeOp 1-D -> n-D upcast serves the purpose of unflattening n-D from 1-D
+/// vectors progressively. This iterates over the n-1 major dimension of the n-D
+/// vector and performs rewrites into:
+///   vector.extract_strided_slice from 1-D + vector.insert into n-D
 /// Note that 1-D extract_strided_slice are lowered to efficient vector.shuffle.
-class ShapeCastOp2DUpCastRewritePattern
+class ShapeCastOpNDUpCastRewritePattern
     : public OpRewritePattern<vector::ShapeCastOp> {
 public:
   using OpRewritePattern::OpRewritePattern;
@@ -90,43 +109,42 @@ class ShapeCastOp2DUpCastRewritePattern
                                 PatternRewriter &rewriter) const override {
     auto sourceVectorType = op.getSourceVectorType();
     auto resultVectorType = op.getResultVectorType();
-
     if (sourceVectorType.isScalable() || resultVectorType.isScalable())
       return failure();
 
-    if (sourceVectorType.getRank() != 1 || resultVectorType.getRank() != 2)
+    int64_t srcRank = sourceVectorType.getRank();
+    int64_t resRank = resultVectorType.getRank();
+    if (srcRank != 1 || resRank < 2)
       return failure();
 
+    // Compute the number of 1-D vector elements involved in the reshape.
+    int64_t numElts = 1;
+    for (int64_t dim = 0; dim < resRank - 1; ++dim)
+      numElts *= resultVectorType.getDimSize(dim);
+
+    // Compute the indices of each 1-D vector element of the source slice
+    // extraction and destination insertion and generate such instructions.
     auto loc = op.getLoc();
-    Value desc = rewriter.create<arith::ConstantOp>(
-        loc, resultVectorType, rewriter.getZeroAttr(resultVectorType));
-    unsigned mostMinorVectorSize = resultVectorType.getShape()[1];
-    for (int64_t i = 0, e = resultVectorType.getShape().front(); i != e; ++i) {
-      Value vec = rewriter.create<vector::ExtractStridedSliceOp>(
-          loc, op.getSource(), /*offsets=*/i * mostMinorVectorSize,
-          /*sizes=*/mostMinorVectorSize,
+    SmallVector<int64_t> srcIdx(srcRank, 0);
+    SmallVector<int64_t> resIdx(resRank - 1, 0);
+    int64_t extractSize = resultVectorType.getShape().back();
+    Value result = rewriter.create<ub::PoisonOp>(loc, resultVectorType);
+    for (int64_t i = 0; i < numElts; ++i) {
+      if (i != 0) {
+        incIdx(srcIdx, sourceVectorType, /*step=*/extractSize);
+        incIdx(resIdx, resultVectorType, /*step=*/1);
+      }
+
+      Value extract = rewriter.create<vector::ExtractStridedSliceOp>(
+          loc, op.getSource(), /*offsets=*/srcIdx, /*sizes=*/extractSize,
           /*strides=*/1);
-      desc = rewriter.create<vector::InsertOp>(loc, vec, desc, i);
+      result = rewriter.create<vector::InsertOp>(loc, extract, result, resIdx);
     }
-    rewriter.replaceOp(op, desc);
+    rewriter.replaceOp(op, result);
     return success();
   }
 };
 
-static void incIdx(llvm::MutableArrayRef<int64_t> idx, VectorType tp,
-                   int dimIdx, int initialStep = 1) {
-  int step = initialStep;
-  for (int d = dimIdx; d >= 0; d--) {
-    idx[d] += step;
-    if (idx[d] >= tp.getDimSize(d)) {
-      idx[d] = 0;
-      step = 1;
-    } else {
-      break;
-    }
-  }
-}
-
 // We typically should not lower general shape cast operations into data
 // movement instructions, since the assumption is that these casts are
 // optimized away during progressive lowering. For completeness, however,
@@ -145,18 +163,14 @@ class ShapeCastOpRewritePattern : public OpRewritePattern<vector::ShapeCastOp> {
     if (sourceVectorType.isScalable() || resultVectorType.isScalable())
       return failure();
 
-    // Special case 2D / 1D lowerings with better implementations.
-    // TODO: make is ND / 1D to allow generic ND -> 1D -> MD.
+    // Special case for n-D / 1-D lowerings with better implementations.
     int64_t srcRank = sourceVectorType.getRank();
     int64_t resRank = resultVectorType.getRank();
-    if ((srcRank == 2 && resRank == 1) || (srcRank == 1 && resRank == 2))
+    if ((srcRank > 1 && resRank == 1) || (srcRank == 1 && resRank > 1))
       return failure();
 
     // Generic ShapeCast lowering path goes all the way down to unrolled scalar
     // extract/insert chains.
-    // TODO: consider evolving the semantics to only allow 1D source or dest and
-    // drop this potentially very expensive lowering.
-    // Compute number of elements involved in the reshape.
     int64_t numElts = 1;
     for (int64_t r = 0; r < srcRank; r++)
       numElts *= sourceVectorType.getDimSize(r);
@@ -166,14 +180,13 @@ class ShapeCastOpRewritePattern : public OpRewritePattern<vector::ShapeCastOp> {
     //    x[0,1,0] = y[0,2]
     // etc., incrementing the two index vectors "row-major"
     // within the source and result shape.
-    SmallVector<int64_t> srcIdx(srcRank);
-    SmallVector<int64_t> resIdx(resRank);
-    Value result = rewriter.create<arith::ConstantOp>(
-        loc, resultVectorType, rewriter.getZeroAttr(resultVectorType));
+    SmallVector<int64_t> srcIdx(srcRank, 0);
+    SmallVector<int64_t> resIdx(resRank, 0);
+    Value result = rewriter.create<ub::PoisonOp>(loc, resultVectorType);
     for (int64_t i = 0; i < numElts; i++) {
       if (i != 0) {
-        incIdx(srcIdx, sourceVectorType, srcRank - 1);
-        incIdx(resIdx, resultVectorType, resRank - 1);
+        incIdx(srcIdx, sourceVectorType);
+        incIdx(resIdx, resultVectorType);
       }
 
       Value extract;
@@ -252,7 +265,7 @@ class ScalableShapeCastOpRewritePattern
     // have a single trailing scalable dimension. This is because there are no
     // legal representation of other scalable types in LLVM (and likely won't be
     // soon). There are also (currently) no operations that can index or extract
-    // from >= 2D scalable vectors or scalable vectors of fixed vectors.
+    // from >= 2-D scalable vectors or scalable vectors of fixed vectors.
     if (!isTrailingDimScalable(sourceVectorType) ||
         !isTrailingDimScalable(resultVectorType)) {
       return failure();
@@ -275,11 +288,9 @@ class ScalableShapeCastOpRewritePattern
     auto extractionVectorType = VectorType::get(
         {minExtractionSize}, sourceVectorType.getElementType(), {true});
 
-    Value result = rewriter.create<arith::ConstantOp>(
-        loc, resultVectorType, rewriter.getZeroAttr(resultVectorType));
-
-    SmallVector<int64_t> srcIdx(srcRank);
-    SmallVector<int64_t> resIdx(resRank);
+    Value result = rewriter.create<ub::PoisonOp>(loc, resultVectorType);
+    SmallVector<int64_t> srcIdx(srcRank, 0);
+    SmallVector<int64_t> resIdx(resRank, 0);
 
     // TODO: Try rewriting this with StaticTileOffsetRange (from IndexingUtils)
     // once D150000 lands.
@@ -334,8 +345,8 @@ class ScalableShapeCastOpRewritePattern
 
       // 4. Increment the insert/extract indices, stepping by minExtractionSize
       // for the trailing dimensions.
-      incIdx(srcIdx, sourceVectorType, srcRank - 1, minExtractionSize);
-      incIdx(resIdx, resultVectorType, resRank - 1, minExtractionSize);
+      incIdx(srcIdx, sourceVectorType, /*step=*/minExtractionSize);
+      incIdx(resIdx, resultVectorType, /*step=*/minExtractionSize);
     }
 
     rewriter.replaceOp(op, result);
@@ -352,8 +363,8 @@ class ScalableShapeCastOpRewritePattern
 
 void mlir::vector::populateVectorShapeCastLoweringPatterns(
     RewritePatternSet &patterns, PatternBenefit benefit) {
-  patterns.add<ShapeCastOp2DDownCastRewritePattern,
-               ShapeCastOp2DUpCastRewritePattern, ShapeCastOpRewritePattern,
+  patterns.add<ShapeCastOpNDDownCastRewritePattern,
+               ShapeCastOpNDUpCastRewritePattern, ShapeCastOpRewritePattern,
                ScalableShapeCastOpRewritePattern>(patterns.getContext(),
                                                   benefit);
 }
diff --git a/mlir/lib/Dialect/Vector/Transforms/LowerVectorTranspose.cpp b/mlir/lib/Dialect/Vector/Transforms/LowerVectorTranspose.cpp
index 3c92b222e6bc8..fb4dee33bc5f5 100644
--- a/mlir/lib/Dialect/Vector/Transforms/LowerVectorTranspose.cpp
+++ b/mlir/lib/Dialect/Vector/Transforms/LowerVectorTranspose.cpp
@@ -11,26 +11,19 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "mlir/Dialect/Affine/IR/AffineOps.h"
 #include "mlir/Dialect/Arith/IR/Arith.h"
-#include "mlir/Dialect/Arith/Utils/Utils.h"
-#include "mlir/Dialect/Linalg/IR/Linalg.h"
 #include "mlir/Dialect/MemRef/IR/MemRef.h"
-#include "mlir/Dialect/SCF/IR/SCF.h"
-#include "mlir/Dialect/Tensor/IR/Tensor.h"
+#include "mlir/Dialect/UB/IR/UBOps.h"
 #include "mlir/Dialect/Utils/IndexingUtils.h"
 #include "mlir/Dialect/Utils/StructuredOpsUtils.h"
 #include "mlir/Dialect/Vector/IR/VectorOps.h"
 #include "mlir/Dialect/Vector/Transforms/LoweringPatterns.h"
 #include "mlir/Dialect/Vector/Utils/VectorUtils.h"
-#include "mlir/IR/BuiltinAttributeInterfaces.h"
 #include "mlir/IR/BuiltinTypes.h"
 #include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/IR/Location.h"
-#include "mlir/IR/Matchers.h"
 #include "mlir/IR/PatternMatch.h"
 #include "mlir/IR/TypeUtilities.h"
-#include "mlir/Interfaces/VectorInterfaces.h"
 
 #define DEBUG_TYPE "lower-vector-transpose"
 
@@ -209,7 +202,7 @@ static Value transposeToShuffle16x16(OpBuilder &builder, Value source, int m,
   ImplicitLocOpBuilder b(source.getLoc(), builder);
   SmallVector<Value> vs;
   for (int64_t i = 0; i < m; ++i)
-    vs.push_back(b.create<vector::ExtractOp>(source, i));
+    vs.push_back(b.createOrFold<vector::ExtractOp>(source, i));
 
   // Interleave 32-bit lanes using
   //   8x _mm512_unpacklo_epi32
@@ -291,8 +284,7 @@ static Value transposeToShuffle16x16(OpBuilder &builder, Value source, int m,
 
   auto reshInputType = VectorType::get(
       {m, n}, cast<VectorType>(source.getType()).getElementType());
-  Value res =
-      b.create<arith::ConstantOp>(reshInputType, b.getZeroAttr(reshInputType));
+  Value res = b.create<ub::PoisonOp>(reshInputType);
   for (int64_t i = 0; i < m; ++i)
     res = b.create<vector::InsertOp>(vs[i], res, i);
   return res;
@@ -368,8 +360,7 @@ class TransposeOpLowering : public OpRewritePattern<vector::TransposeOp> {
     // of the leftmost transposed dimensions. We traverse every transpose
     // element using a linearized index that we delinearize to generate the
     // appropriate indices for the extract/insert operations.
-    Value result = rewriter.create<arith::ConstantOp>(
-        loc, resType, rewriter.getZeroAttr(resType));
+    Value result = rewriter.create<ub::PoisonOp>(loc, resType);
     int64_t numTransposedElements = ShapedType::getNumElements(prunedInShape);
 
     for (int64_t linearIdx = 0; linearIdx < numTransposedElements;
@@ -378,9 +369,9 @@ class TransposeOpLowering : public OpRewritePattern<vector::TransposeOp> {
       SmallVector<int64_t> insertIdxs(extractIdxs);
       applyPermutationToVector(insertIdxs, prunedTransp);
       Value extractOp =
-          rewriter.create<vector::ExtractOp>(loc, input, extractIdxs);
-      result =
-          rewriter.create<vector::InsertOp>(loc, extractOp, result, insertIdxs);
+          rewriter.createOrFold<vector::ExtractOp>(loc, input, extractIdxs);
+      result = rewriter.createOrFold<vector::InsertOp>(loc, extractOp, result,
+                                                       insertIdxs);
     }
 
     rewriter.replaceOp(op, result);
diff --git a/mlir/lib/Dialect/Vector/Transforms/VectorDropLeadUnitDim.cpp b/mlir/lib/Dialect/Vector/Transforms/VectorDropLeadUnitDim.cpp
index 3035c419a1b56..b53aa997c9014 100644
--- a/mlir/lib/Dialect/Vector/Transforms/VectorDropLeadUnitDim.cpp
+++ b/mlir/lib/Dialect/Vector/Transforms/VectorDropLeadUnitDim.cpp
@@ -377,18 +377,18 @@ mlir::vector::castAwayContractionLeadingOneDim(vector::ContractionOp contractOp,
     int64_t orginalZeroDim = it.value().getDimPosition(0);
     if (orginalZeroDim != dimToDrop) {
       // There are two reasons to be in this path, 1. We need to
-      // tranpose the operand to make the dim to be dropped
+      // transpose the operand to make the dim to be dropped
       // leading. 2. The dim to be dropped does not exist and in
-      // that case we dont want to add a unit tranpose but we must
+      // that case we dont want to add a unit transpose but we must
       // check all the indices to make sure this is the case.
-      bool tranposeNeeded = false;
+      bool transposeNeeded = false;
       SmallVector<int64_t> perm;
       SmallVector<AffineExpr> transposeResults;
 
       for (int64_t i = 0, e = map.getNumResults(); i < e; ++i) {
         int64_t currDim = map.getDimPosition(i);
         if (currDim == dimToDrop) {
-          tranposeNeeded = true;
+          transposeNeeded = true;
           perm.insert(perm.begin(), i);
           auto targetExpr = rewriter.getAffineDimExpr(currDim);
           transposeResults.insert(transposeResults.begin(), targetExpr);
@@ -413,9 +413,9 @@ mlir::vector::castAwayContractionLeadingOneDim(vector::ContractionOp contractOp,
         }
       }
 
-      // Do the tranpose now if needed so that we can drop the
+      // Do the transpose now if needed so that we can drop the
       // correct dim using extract later.
-      if (tranposeNeeded) {
+      if (transposeNeeded) {
         map = AffineMap::get(map.getNumDims(), 0, transposeResults,
                              contractOp.getContext());
         if (transposeNonOuterUnitDims) {
@@ -474,7 +474,7 @@ namespace {
 
 /// Turns vector.contract on vector with leading 1 dimensions into
 /// vector.extract followed by vector.contract on vector without leading
-/// 1 dimensions. Also performs tranpose of lhs and rhs operands if required
+/// 1 dimensions. Also performs transpose of lhs and rhs operands if required
 /// prior to extract.
 struct CastAwayContractionLeadingOneDim
     : public MaskableOpRewritePattern<vector::ContractionOp> {
diff --git a/mlir/lib/Dialect/Vector/Transforms/VectorEmulateNarrowType.cpp b/mlir/lib/Dialect/Vector/Transforms/VectorEmulateNarrowType.cpp
index 95064083b21d4..bf1ecd7d4559c 100644
--- a/mlir/lib/Dialect/Vector/Transforms/VectorEmulateNarrowType.cpp
+++ b/mlir/lib/Dialect/Vector/Transforms/VectorEmulateNarrowType.cpp
@@ -45,6 +45,13 @@ using namespace mlir;
 #define DBGSNL() (llvm::dbgs() << "\n")
 #define LDBG(X) LLVM_DEBUG(DBGS() << X << "\n")
 
+using VectorValue = TypedValue<VectorType>;
+using MemRefValue = TypedValue<MemRefType>;
+
+//===----------------------------------------------------------------------===//
+// Utils
+//===----------------------------------------------------------------------===//
+
 /// Returns a compressed mask for the emulated vector. For example, when
 /// emulating an eight-element `i8` vector with `i32` (i.e. when the source
 /// elements span two dest elements), this method compresses `vector<8xi1>`
@@ -194,13 +201,10 @@ static FailureOr<Operation *> getCompressedMaskOp(OpBuilder &rewriter,
 /// Extracts 1-D subvector from a 1-D vector. It is a wrapper function for
 /// emitting `vector.extract_strided_slice`.
 static Value staticallyExtractSubvector(OpBuilder &rewriter, Location loc,
-                                        VectorType extractType, Value source,
-                                        int64_t frontOffset,
+                                        Value source, int64_t frontOffset,
                                         int64_t subvecSize) {
   auto vectorType = cast<VectorType>(source.getType());
-  assert((vectorType.getRank() == 1 && extractType.getRank() == 1) &&
-         "expected 1-D source and destination types");
-  (void)vectorType;
+  assert(vectorType.getRank() == 1 && "expected 1-D source types");
   assert(frontOffset + subvecSize <= vectorType.getNumElements() &&
          "subvector out of bounds");
 
@@ -211,9 +215,12 @@ static Value staticallyExtractSubvector(OpBuilder &rewriter, Location loc,
   auto offsets = rewriter.getI64ArrayAttr({frontOffset});
   auto sizes = rewriter.getI64ArrayAttr({subvecSize});
   auto strides = rewriter.getI64ArrayAttr({1});
+
+  auto resultVectorType =
+      VectorType::get({subvecSize}, vectorType.getElementType());
   return rewriter
-      .create<vector::ExtractStridedSliceOp>(loc, extractType, source, offsets,
-                                             sizes, strides)
+      .create<vector::ExtractStridedSliceOp>(loc, resultVectorType, source,
+                                             offsets, sizes, strides)
       ->getResult(0);
 }
 
@@ -237,9 +244,10 @@ static Value staticallyInsertSubvector(OpBuilder &rewriter, Location loc,
 /// function emits multiple `vector.extract` and `vector.insert` ops, so only
 /// use it when `offset` cannot be folded into a constant value.
 static Value dynamicallyExtractSubVector(OpBuilder &rewriter, Location loc,
-                                         TypedValue<VectorType> source,
-                                         Value dest, OpFoldResult offset,
+                                         Value source, Value dest,
+                                         OpFoldResult offset,
                                          int64_t numElementsToExtract) {
+  assert(isa<VectorValue>(source) && "expected `source` to be a vector type");
   for (int i = 0; i < numElementsToExtract; ++i) {
     Value extractLoc =
         (i == 0) ? offset.dyn_cast<Value>()
@@ -255,9 +263,10 @@ static Value dynamicallyExtractSubVector(OpBuilder &rewriter, Location loc,
 
 /// Inserts a 1-D subvector into a 1-D `dest` vector at index `destOffsetVar`.
 static Value dynamicallyInsertSubVector(RewriterBase &rewriter, Location loc,
-                                        TypedValue<VectorType> source,
-                                        Value dest, OpFoldResult destOffsetVar,
+                                        Value source, Value dest,
+                                        OpFoldResult destOffsetVar,
                                         size_t length) {
+  assert(isa<VectorValue>(source) && "expected `source` to be a vector type");
   assert(length > 0 && "length must be greater than 0");
   Value destOffsetVal =
       getValueOrCreateConstantIndexOp(rewriter, loc, destOffsetVar);
@@ -273,34 +282,164 @@ static Value dynamicallyInsertSubVector(RewriterBase &rewriter, Location loc,
   return dest;
 }
 
-/// Returns the op sequence for an emulated sub-byte data type vector load.
-/// specifically, use `emulatedElemType` for loading a vector of `origElemType`.
-/// The load location is given by `base` and `linearizedIndices`, and the
-/// load size is given by `numEmulatedElementsToLoad`.
-static TypedValue<VectorType>
-emulatedVectorLoad(OpBuilder &rewriter, Location loc, Value base,
-                   OpFoldResult linearizedIndices,
-                   int64_t numEmultedElementsToLoad, Type origElemType,
-                   Type emulatedElemType) {
-  auto scale = emulatedElemType.getIntOrFloatBitWidth() /
-               origElemType.getIntOrFloatBitWidth();
+/// Emulate a vector load for `emulatedElemTy` using `containerElemTy`
+///
+/// Specifically, use `containerElemTy` for loading a vector of
+/// `emulatedElemTy`. The load location is given by `base` and
+/// `linearizedIndices`, and the load size is given by
+/// `numEmulatedElementsToLoad`.
+static VectorValue emulatedVectorLoad(OpBuilder &rewriter, Location loc,
+                                      Value base,
+                                      OpFoldResult linearizedIndices,
+                                      int64_t numContainerElemsToLoad,
+                                      Type emulatedElemTy,
+                                      Type containerElemTy) {
+  auto emulatedPerContainerElem = containerElemTy.getIntOrFloatBitWidth() /
+                                  emulatedElemTy.getIntOrFloatBitWidth();
   auto newLoad = rewriter.create<vector::LoadOp>(
-      loc, VectorType::get(numEmultedElementsToLoad, emulatedElemType), base,
+      loc, VectorType::get(numContainerElemsToLoad, containerElemTy), base,
       getValueOrCreateConstantIndexOp(rewriter, loc, linearizedIndices));
   return rewriter.create<vector::BitCastOp>(
-      loc, VectorType::get(numEmultedElementsToLoad * scale, origElemType),
+      loc,
+      VectorType::get(numContainerElemsToLoad * emulatedPerContainerElem,
+                      emulatedElemTy),
       newLoad);
 }
 
+/// Downcast two values to `downcastType`, then select values
+/// based on `mask`, and casts the result to `upcastType`.
+static Value downcastSelectAndUpcast(OpBuilder &builder, Location loc,
+                                     VectorType downcastType,
+                                     VectorType upcastType, Value mask,
+                                     Value trueValue, Value falseValue) {
+  assert(
+      downcastType.getNumElements() * downcastType.getElementTypeBitWidth() ==
+          upcastType.getNumElements() * upcastType.getElementTypeBitWidth() &&
+      "expected input and output number of bits to match");
+  if (trueValue.getType() != downcastType) {
+    trueValue = builder.create<vector::BitCastOp>(loc, downcastType, trueValue);
+  }
+  if (falseValue.getType() != downcastType) {
+    falseValue =
+        builder.create<vector::BitCastOp>(loc, downcastType, falseValue);
+  }
+  Value selectedType =
+      builder.create<arith::SelectOp>(loc, mask, trueValue, falseValue);
+  // Upcast the selected value to the new type.
+  return builder.create<vector::BitCastOp>(loc, upcastType, selectedType);
+}
+
+/// Emits `memref.generic_atomic_rmw` op to store a subbyte-sized value to a
+/// byte in `linearizedMemref`, with a mask. The `valueToStore` is a vector of
+/// subbyte-sized elements, with size of 8 bits, and the mask is used to select
+/// which elements to store.
+///
+/// Inputs:
+///   linearizedMemref = |2|2|2|2| : <4xi2> (<1xi8>)
+///   storeIdx = 2
+///   valueToStore = |3|3|3|3| : vector<4xi2>
+///   mask = |0|0|1|1| : vector<4xi1>
+///
+/// Result:
+///   linearizedMemref = |2|2|3|3| : <4xi2> (<1xi8>)
+static void atomicRMW(OpBuilder &builder, Location loc,
+                      MemRefValue linearizedMemref, Value storeIdx,
+                      VectorValue valueToStore, Value mask) {
+  assert(valueToStore.getType().getRank() == 1 && "expected 1-D vector");
+
+  // Create an atomic load-modify-write region using
+  // `memref.generic_atomic_rmw`.
+  auto atomicOp = builder.create<memref::GenericAtomicRMWOp>(
+      loc, linearizedMemref, ValueRange{storeIdx});
+  Value origValue = atomicOp.getCurrentValue();
+
+  OpBuilder::InsertionGuard guard(builder);
+  builder.setInsertionPointToStart(atomicOp.getBody());
+
+  // Load the original value from memory, and cast it to the original element
+  // type.
+  auto oneElemVecType = VectorType::get({1}, origValue.getType());
+  Value origVecValue = builder.create<vector::FromElementsOp>(
+      loc, oneElemVecType, ValueRange{origValue});
+
+  // Construct the final masked value and yield it.
+  Value maskedValue =
+      downcastSelectAndUpcast(builder, loc, valueToStore.getType(),
+                              oneElemVecType, mask, valueToStore, origVecValue);
+  auto scalarMaskedValue =
+      builder.create<vector::ExtractOp>(loc, maskedValue, 0);
+  builder.create<memref::AtomicYieldOp>(loc, scalarMaskedValue);
+}
+
+/// Generate a non-atomic read-modify-write sequence for storing to the emulated
+/// type. It has similar logic to `atomicRMWStore`, but without atomicity.
+static void nonAtomicRMW(OpBuilder &builder, Location loc,
+                         MemRefValue linearizedMemref, Value linearizedIndex,
+                         VectorValue valueToStore, Value mask) {
+  assert(valueToStore.getType().getRank() == 1 && "expected 1-D vector");
+
+  auto oneElemVecType =
+      VectorType::get({1}, linearizedMemref.getType().getElementType());
+  Value origVecValue = builder.create<vector::LoadOp>(
+      loc, oneElemVecType, linearizedMemref, ValueRange{linearizedIndex});
+  origVecValue = builder.create<vector::BitCastOp>(loc, valueToStore.getType(),
+                                                   origVecValue);
+
+  Value maskedValue =
+      downcastSelectAndUpcast(builder, loc, valueToStore.getType(),
+                              oneElemVecType, mask, valueToStore, origVecValue);
+  builder.create<vector::StoreOp>(loc, maskedValue, linearizedMemref,
+                                  linearizedIndex);
+}
+
+/// Extract `sliceNumElements` from source `vector` at `extractOffset`,
+/// and insert it into an empty vector at `insertOffset`.
+/// Inputs:
+///   vec_in  = |0|1|2|3| : vector<4xi2>
+///   extractOffset = 1
+///   sliceNumElements = 2
+///   insertOffset = 2
+/// Output:
+///   vec_out = |0|0|1|2| : vector<4xi2>
+static Value extractSliceIntoByte(ConversionPatternRewriter &rewriter,
+                                  Location loc, VectorValue vector,
+                                  int64_t extractOffset,
+                                  int64_t sliceNumElements,
+                                  int64_t insertOffset) {
+  assert(vector.getType().getRank() == 1 && "expected 1-D vector");
+  auto vectorElementType = vector.getType().getElementType();
+  // TODO: update and use `alignedConversionPrecondition` in the place of
+  // these asserts.
+  assert(
+      sliceNumElements * vectorElementType.getIntOrFloatBitWidth() <= 8 &&
+      "sliceNumElements * vector element size must be less than or equal to 8");
+  assert(8 % vectorElementType.getIntOrFloatBitWidth() == 0 &&
+         "vector element must be a valid sub-byte type");
+  auto emulatedPerContainerElem = 8 / vectorElementType.getIntOrFloatBitWidth();
+  auto emptyByteVector = rewriter.create<arith::ConstantOp>(
+      loc, VectorType::get({emulatedPerContainerElem}, vectorElementType),
+      rewriter.getZeroAttr(
+          VectorType::get({emulatedPerContainerElem}, vectorElementType)));
+  auto extracted = staticallyExtractSubvector(rewriter, loc, vector,
+                                              extractOffset, sliceNumElements);
+  return staticallyInsertSubvector(rewriter, loc, extracted, emptyByteVector,
+                                   insertOffset);
+}
+
 namespace {
 
 //===----------------------------------------------------------------------===//
 // ConvertVectorStore
 //===----------------------------------------------------------------------===//
 
+// TODO: Document-me
 struct ConvertVectorStore final : OpConversionPattern<vector::StoreOp> {
   using OpConversionPattern::OpConversionPattern;
 
+  ConvertVectorStore(MLIRContext *context, bool disableAtomicRMW)
+      : OpConversionPattern<vector::StoreOp>(context),
+        disableAtomicRMW(disableAtomicRMW) {}
+
   LogicalResult
   matchAndRewrite(vector::StoreOp op, OpAdaptor adaptor,
                   ConversionPatternRewriter &rewriter) const override {
@@ -311,17 +450,21 @@ struct ConvertVectorStore final : OpConversionPattern<vector::StoreOp> {
                                          "only 1-D vectors are supported ATM");
 
     auto loc = op.getLoc();
-    auto convertedType = cast<MemRefType>(adaptor.getBase().getType());
-    Type oldElementType = op.getValueToStore().getType().getElementType();
-    Type newElementType = convertedType.getElementType();
-    int srcBits = oldElementType.getIntOrFloatBitWidth();
-    int dstBits = newElementType.getIntOrFloatBitWidth();
 
-    if (dstBits % srcBits != 0) {
+    auto valueToStore = cast<VectorValue>(op.getValueToStore());
+    auto containerElemTy =
+        cast<MemRefType>(adaptor.getBase().getType()).getElementType();
+    Type emulatedElemTy = op.getValueToStore().getType().getElementType();
+    int emulatedBits = emulatedElemTy.getIntOrFloatBitWidth();
+    int containerBits = containerElemTy.getIntOrFloatBitWidth();
+
+    // Check per-element alignment.
+    if (containerBits % emulatedBits != 0) {
       return rewriter.notifyMatchFailure(
-          op, "only dstBits % srcBits == 0 supported");
+          op, "impossible to pack emulated elements into container elements "
+              "(bit-wise misalignment)");
     }
-    int scale = dstBits / srcBits;
+    int numSrcElemsPerDest = containerBits / emulatedBits;
 
     // Adjust the number of elements to store when emulating narrow types.
     // Here only the 1-D vector store is considered, and the N-D memref types
@@ -336,38 +479,195 @@ struct ConvertVectorStore final : OpConversionPattern<vector::StoreOp> {
     // vector.store %bitcast, %alloc[%linear_index] : memref<16xi8>,
     // vector<4xi8>
 
-    auto origElements = op.getValueToStore().getType().getNumElements();
-    if (origElements % scale != 0)
-      return failure();
+    auto origElements = valueToStore.getType().getNumElements();
+    bool isAlignedEmulation = origElements % numSrcElemsPerDest == 0;
 
     auto stridedMetadata =
         rewriter.create<memref::ExtractStridedMetadataOp>(loc, op.getBase());
 
     OpFoldResult linearizedIndices;
-    std::tie(std::ignore, linearizedIndices) =
+    memref::LinearizedMemRefInfo linearizedInfo;
+    std::tie(linearizedInfo, linearizedIndices) =
         memref::getLinearizedMemRefOffsetAndSize(
-            rewriter, loc, srcBits, dstBits,
+            rewriter, loc, emulatedBits, containerBits,
             stridedMetadata.getConstifiedMixedOffset(),
             stridedMetadata.getConstifiedMixedSizes(),
             stridedMetadata.getConstifiedMixedStrides(),
             getAsOpFoldResult(adaptor.getIndices()));
 
-    auto numElements = origElements / scale;
-    auto bitCast = rewriter.create<vector::BitCastOp>(
-        loc, VectorType::get(numElements, newElementType),
-        op.getValueToStore());
+    std::optional<int64_t> foldedNumFrontPadElems =
+        isAlignedEmulation
+            ? 0
+            : getConstantIntValue(linearizedInfo.intraDataOffset);
 
-    rewriter.replaceOpWithNewOp<vector::StoreOp>(
-        op, bitCast.getResult(), adaptor.getBase(),
-        getValueOrCreateConstantIndexOp(rewriter, loc, linearizedIndices));
+    if (!foldedNumFrontPadElems) {
+      return rewriter.notifyMatchFailure(
+          op, "subbyte store emulation: dynamic front padding size is "
+              "not yet implemented");
+    }
+
+    auto memrefBase = cast<MemRefValue>(adaptor.getBase());
+
+    // Conditions when atomic RMWs are not needed:
+    // 1. The source vector size (in bits) is a multiple of byte size.
+    // 2. The address of the store is aligned to the emulated width boundary.
+    //
+    // For example, to store a vector<4xi2> to <13xi2> at offset 4, does not
+    // need unaligned emulation because the store address is aligned and the
+    // source is a whole byte.
+    bool emulationRequiresPartialStores =
+        !isAlignedEmulation || *foldedNumFrontPadElems != 0;
+    if (!emulationRequiresPartialStores) {
+      // Basic case: storing full bytes.
+      auto numElements = origElements / numSrcElemsPerDest;
+      auto bitCast = rewriter.create<vector::BitCastOp>(
+          loc, VectorType::get(numElements, containerElemTy),
+          op.getValueToStore());
+      rewriter.replaceOpWithNewOp<vector::StoreOp>(
+          op, bitCast.getResult(), memrefBase,
+          getValueOrCreateConstantIndexOp(rewriter, loc, linearizedIndices));
+      return success();
+    }
+
+    // Next, handle the case when sub-byte read-modify-write
+    // sequences are needed to emulate a vector store.
+    // Here is an example:
+    //
+    // Vector to store: vector<7xi2>
+    // Value to store: 11 11 11 11 11 11 11 (all ones)
+    //
+    // Destination: memref<12xi2>
+    // Store offset: 2 (i.e. 4 bits into the 1st emulated byte).
+    //
+    // Input MLIR: vector.store %val, %dest[%c2] : memref<12xi2>, vector<7xi2>
+    //
+    // Destination memref before:
+    //
+    //    Byte 0     Byte 1     Byte 2
+    // +----------+----------+----------+
+    // | 00000000 | 00000000 | 00000000 |
+    // +----------+----------+----------+
+    //
+    // Destination memref after:
+    //
+    //    Byte 0     Byte 1     Byte 2
+    // +----------+----------+----------+
+    // | 00001111 | 11111111 | 11000000 |
+    // +----------+----------+----------+
+    //
+    // Note, stores to Byte 1 are "full-width" and hence don't require RMW (no
+    // need for atomicity). Stores to Bytes 0 and Byte 2 are "partial", hence
+    // requiring RMW access (atomicity is required).
+
+    // The index into the target memref we are storing to.
+    Value currentDestIndex =
+        getValueOrCreateConstantIndexOp(rewriter, loc, linearizedIndices);
+    // The index into the source vector we are currently processing.
+    auto currentSourceIndex = 0;
+
+    // Build a mask used for rmw.
+    auto subWidthStoreMaskType =
+        VectorType::get({numSrcElemsPerDest}, rewriter.getI1Type());
+
+    auto storeFunc = disableAtomicRMW ? nonAtomicRMW : atomicRMW;
+
+    // 1. Partial width store for the leading byte.
+    // When the store address is not aligned to emulated width boundary, deal
+    // with the unaligned part so that the rest elements are aligned to width
+    // boundary.
+    auto frontSubWidthStoreElem =
+        (numSrcElemsPerDest - *foldedNumFrontPadElems) % numSrcElemsPerDest;
+    if (frontSubWidthStoreElem > 0) {
+      SmallVector<bool> frontMaskValues(numSrcElemsPerDest, false);
+      if (*foldedNumFrontPadElems + origElements < numSrcElemsPerDest) {
+        std::fill_n(frontMaskValues.begin() + *foldedNumFrontPadElems,
+                    origElements, true);
+        frontSubWidthStoreElem = origElements;
+      } else {
+        std::fill_n(frontMaskValues.end() - frontSubWidthStoreElem,
+                    *foldedNumFrontPadElems, true);
+      }
+      auto frontMask = rewriter.create<arith::ConstantOp>(
+          loc, DenseElementsAttr::get(subWidthStoreMaskType, frontMaskValues));
+
+      currentSourceIndex = numSrcElemsPerDest - (*foldedNumFrontPadElems);
+      auto value =
+          extractSliceIntoByte(rewriter, loc, valueToStore, 0,
+                               frontSubWidthStoreElem, *foldedNumFrontPadElems);
+
+      storeFunc(rewriter, loc, memrefBase, currentDestIndex,
+                cast<VectorValue>(value), frontMask.getResult());
+    }
+
+    if (currentSourceIndex >= origElements) {
+      rewriter.eraseOp(op);
+      return success();
+    }
+
+    // Increment the destination index by 1 to align to the emulated width
+    // boundary.
+    auto constantOne = rewriter.create<arith::ConstantIndexOp>(loc, 1);
+    currentDestIndex = rewriter.create<arith::AddIOp>(
+        loc, rewriter.getIndexType(), currentDestIndex, constantOne);
+
+    // 2. Full width store for the inner output bytes.
+    // After the previous step, the store address is aligned to the emulated
+    // width boundary.
+    int64_t fullWidthStoreSize =
+        (origElements - currentSourceIndex) / numSrcElemsPerDest;
+    int64_t numNonFullWidthElements = fullWidthStoreSize * numSrcElemsPerDest;
+    if (fullWidthStoreSize > 0) {
+      auto fullWidthStorePart = staticallyExtractSubvector(
+          rewriter, loc, valueToStore, currentSourceIndex,
+          numNonFullWidthElements);
+
+      auto originType = cast<VectorType>(fullWidthStorePart.getType());
+      auto memrefElemType = getElementTypeOrSelf(memrefBase.getType());
+      auto storeType = VectorType::get(
+          {originType.getNumElements() / numSrcElemsPerDest}, memrefElemType);
+      auto bitCast = rewriter.create<vector::BitCastOp>(loc, storeType,
+                                                        fullWidthStorePart);
+      rewriter.create<vector::StoreOp>(loc, bitCast.getResult(), memrefBase,
+                                       currentDestIndex);
+
+      currentSourceIndex += numNonFullWidthElements;
+      currentDestIndex = rewriter.create<arith::AddIOp>(
+          loc, rewriter.getIndexType(), currentDestIndex,
+          rewriter.create<arith::ConstantIndexOp>(loc, fullWidthStoreSize));
+    }
+
+    // 3. Partial width store for the trailing output byte.
+    // It is needed when the residual length is smaller than the emulated width,
+    // which is not covered in step 2 above.
+    auto remainingElements = origElements - currentSourceIndex;
+    if (remainingElements != 0) {
+      auto subWidthStorePart =
+          extractSliceIntoByte(rewriter, loc, cast<VectorValue>(valueToStore),
+                               currentSourceIndex, remainingElements, 0);
+
+      // Generate back mask.
+      auto maskValues = SmallVector<bool>(numSrcElemsPerDest, 0);
+      std::fill_n(maskValues.begin(), remainingElements, 1);
+      auto backMask = rewriter.create<arith::ConstantOp>(
+          loc, DenseElementsAttr::get(subWidthStoreMaskType, maskValues));
+
+      storeFunc(rewriter, loc, memrefBase, currentDestIndex,
+                cast<VectorValue>(subWidthStorePart), backMask.getResult());
+    }
+
+    rewriter.eraseOp(op);
     return success();
   }
+
+private:
+  const bool disableAtomicRMW;
 };
 
 //===----------------------------------------------------------------------===//
 // ConvertVectorMaskedStore
 //===----------------------------------------------------------------------===//
 
+// TODO: Document-me
 struct ConvertVectorMaskedStore final
     : OpConversionPattern<vector::MaskedStoreOp> {
   using OpConversionPattern::OpConversionPattern;
@@ -382,20 +682,22 @@ struct ConvertVectorMaskedStore final
                                          "only 1-D vectors are supported ATM");
 
     auto loc = op.getLoc();
-    auto convertedType = cast<MemRefType>(adaptor.getBase().getType());
-    Type oldElementType = op.getValueToStore().getType().getElementType();
-    Type newElementType = convertedType.getElementType();
-    int srcBits = oldElementType.getIntOrFloatBitWidth();
-    int dstBits = newElementType.getIntOrFloatBitWidth();
-
-    if (dstBits % srcBits != 0) {
+    auto containerElemTy =
+        cast<MemRefType>(adaptor.getBase().getType()).getElementType();
+    Type emulatedElemTy = op.getValueToStore().getType().getElementType();
+    int emulatedBits = emulatedElemTy.getIntOrFloatBitWidth();
+    int containerBits = containerElemTy.getIntOrFloatBitWidth();
+
+    // Check per-element alignment.
+    if (containerBits % emulatedBits != 0) {
       return rewriter.notifyMatchFailure(
-          op, "only dstBits % srcBits == 0 supported");
+          op, "impossible to pack emulated elements into container elements "
+              "(bit-wise misalignment)");
     }
 
-    int scale = dstBits / srcBits;
+    int emulatedPerContainerElem = containerBits / emulatedBits;
     int origElements = op.getValueToStore().getType().getNumElements();
-    if (origElements % scale != 0)
+    if (origElements % emulatedPerContainerElem != 0)
       return failure();
 
     auto stridedMetadata =
@@ -404,7 +706,7 @@ struct ConvertVectorMaskedStore final
     memref::LinearizedMemRefInfo linearizedInfo;
     std::tie(linearizedInfo, linearizedIndicesOfr) =
         memref::getLinearizedMemRefOffsetAndSize(
-            rewriter, loc, srcBits, dstBits,
+            rewriter, loc, emulatedBits, containerBits,
             stridedMetadata.getConstifiedMixedOffset(),
             stridedMetadata.getConstifiedMixedSizes(),
             stridedMetadata.getConstifiedMixedStrides(),
@@ -444,13 +746,14 @@ struct ConvertVectorMaskedStore final
     //
     // FIXME: Make an example based on the comment above work (see #115460 for
     // reproducer).
-    FailureOr<Operation *> newMask =
-        getCompressedMaskOp(rewriter, loc, op.getMask(), origElements, scale);
+    FailureOr<Operation *> newMask = getCompressedMaskOp(
+        rewriter, loc, op.getMask(), origElements, emulatedPerContainerElem);
     if (failed(newMask))
       return failure();
 
-    auto numElements = (origElements + scale - 1) / scale;
-    auto newType = VectorType::get(numElements, newElementType);
+    auto numElements = (origElements + emulatedPerContainerElem - 1) /
+                       emulatedPerContainerElem;
+    auto newType = VectorType::get(numElements, containerElemTy);
     auto passThru = rewriter.create<arith::ConstantOp>(
         loc, newType, rewriter.getZeroAttr(newType));
 
@@ -458,7 +761,8 @@ struct ConvertVectorMaskedStore final
         loc, newType, adaptor.getBase(), linearizedIndices,
         newMask.value()->getResult(0), passThru);
 
-    auto newBitCastType = VectorType::get(numElements * scale, oldElementType);
+    auto newBitCastType =
+        VectorType::get(numElements * emulatedPerContainerElem, emulatedElemTy);
     Value valueToStore =
         rewriter.create<vector::BitCastOp>(loc, newBitCastType, newLoad);
     valueToStore = rewriter.create<arith::SelectOp>(
@@ -477,6 +781,7 @@ struct ConvertVectorMaskedStore final
 // ConvertVectorLoad
 //===----------------------------------------------------------------------===//
 
+// TODO: Document-me
 struct ConvertVectorLoad final : OpConversionPattern<vector::LoadOp> {
   using OpConversionPattern::OpConversionPattern;
 
@@ -490,17 +795,19 @@ struct ConvertVectorLoad final : OpConversionPattern<vector::LoadOp> {
                                          "only 1-D vectors are supported ATM");
 
     auto loc = op.getLoc();
-    auto convertedType = cast<MemRefType>(adaptor.getBase().getType());
-    Type oldElementType = op.getType().getElementType();
-    Type newElementType = convertedType.getElementType();
-    int srcBits = oldElementType.getIntOrFloatBitWidth();
-    int dstBits = newElementType.getIntOrFloatBitWidth();
-
-    if (dstBits % srcBits != 0) {
+    auto containerElemTy =
+        cast<MemRefType>(adaptor.getBase().getType()).getElementType();
+    Type emulatedElemTy = op.getType().getElementType();
+    int emulatedBits = emulatedElemTy.getIntOrFloatBitWidth();
+    int containerBits = containerElemTy.getIntOrFloatBitWidth();
+
+    // Check per-element alignment.
+    if (containerBits % emulatedBits != 0) {
       return rewriter.notifyMatchFailure(
-          op, "only dstBits % srcBits == 0 supported");
+          op, "impossible to pack emulated elements into container elements "
+              "(bit-wise misalignment)");
     }
-    int scale = dstBits / srcBits;
+    int emulatedPerContainerElem = containerBits / emulatedBits;
 
     // Adjust the number of elements to load when emulating narrow types,
     // and then cast back to the original type with vector.bitcast op.
@@ -532,7 +839,8 @@ struct ConvertVectorLoad final : OpConversionPattern<vector::LoadOp> {
     // compile time as they must be constants.
 
     auto origElements = op.getVectorType().getNumElements();
-    bool isUnalignedEmulation = origElements % scale != 0;
+    // Note, per-element-alignment was already verified above.
+    bool isFullyAligned = origElements % emulatedPerContainerElem == 0;
 
     auto stridedMetadata =
         rewriter.create<memref::ExtractStridedMetadataOp>(loc, op.getBase());
@@ -541,24 +849,24 @@ struct ConvertVectorLoad final : OpConversionPattern<vector::LoadOp> {
     memref::LinearizedMemRefInfo linearizedInfo;
     std::tie(linearizedInfo, linearizedIndices) =
         memref::getLinearizedMemRefOffsetAndSize(
-            rewriter, loc, srcBits, dstBits,
+            rewriter, loc, emulatedBits, containerBits,
             stridedMetadata.getConstifiedMixedOffset(),
             stridedMetadata.getConstifiedMixedSizes(),
             stridedMetadata.getConstifiedMixedStrides(),
             getAsOpFoldResult(adaptor.getIndices()));
 
     std::optional<int64_t> foldedIntraVectorOffset =
-        isUnalignedEmulation
-            ? getConstantIntValue(linearizedInfo.intraDataOffset)
-            : 0;
+        isFullyAligned ? 0
+                       : getConstantIntValue(linearizedInfo.intraDataOffset);
 
     // Always load enough elements which can cover the original elements.
-    int64_t maxintraDataOffset = foldedIntraVectorOffset.value_or(scale - 1);
-    auto numElements =
-        llvm::divideCeil(maxintraDataOffset + origElements, scale);
+    int64_t maxintraDataOffset =
+        foldedIntraVectorOffset.value_or(emulatedPerContainerElem - 1);
+    auto numElements = llvm::divideCeil(maxintraDataOffset + origElements,
+                                        emulatedPerContainerElem);
     Value result =
         emulatedVectorLoad(rewriter, loc, adaptor.getBase(), linearizedIndices,
-                           numElements, oldElementType, newElementType);
+                           numElements, emulatedElemTy, containerElemTy);
 
     if (!foldedIntraVectorOffset) {
       auto resultVector = rewriter.create<arith::ConstantOp>(
@@ -566,10 +874,9 @@ struct ConvertVectorLoad final : OpConversionPattern<vector::LoadOp> {
       result = dynamicallyExtractSubVector(
           rewriter, loc, dyn_cast<TypedValue<VectorType>>(result), resultVector,
           linearizedInfo.intraDataOffset, origElements);
-    } else if (isUnalignedEmulation) {
-      result =
-          staticallyExtractSubvector(rewriter, loc, op.getType(), result,
-                                     *foldedIntraVectorOffset, origElements);
+    } else if (!isFullyAligned) {
+      result = staticallyExtractSubvector(
+          rewriter, loc, result, *foldedIntraVectorOffset, origElements);
     }
     rewriter.replaceOp(op, result);
     return success();
@@ -580,6 +887,7 @@ struct ConvertVectorLoad final : OpConversionPattern<vector::LoadOp> {
 // ConvertVectorMaskedLoad
 //===----------------------------------------------------------------------===//
 
+// TODO: Document-me
 struct ConvertVectorMaskedLoad final
     : OpConversionPattern<vector::MaskedLoadOp> {
   using OpConversionPattern::OpConversionPattern;
@@ -593,17 +901,20 @@ struct ConvertVectorMaskedLoad final
                                          "only 1-D vectors are supported ATM");
 
     auto loc = op.getLoc();
-    auto convertedType = cast<MemRefType>(adaptor.getBase().getType());
-    Type oldElementType = op.getType().getElementType();
-    Type newElementType = convertedType.getElementType();
-    int srcBits = oldElementType.getIntOrFloatBitWidth();
-    int dstBits = newElementType.getIntOrFloatBitWidth();
 
-    if (dstBits % srcBits != 0) {
+    auto containerElemTy =
+        cast<MemRefType>(adaptor.getBase().getType()).getElementType();
+    Type emulatedElemTy = op.getType().getElementType();
+    int emulatedBits = emulatedElemTy.getIntOrFloatBitWidth();
+    int containerBits = containerElemTy.getIntOrFloatBitWidth();
+
+    // Check per-element alignment.
+    if (containerBits % emulatedBits != 0) {
       return rewriter.notifyMatchFailure(
-          op, "only dstBits % srcBits == 0 supported");
+          op, "impossible to pack emulated elements into container elements "
+              "(bit-wise misalignment)");
     }
-    int scale = dstBits / srcBits;
+    int emulatedPerContainerElem = containerBits / emulatedBits;
 
     // Adjust the number of elements to load when emulating narrow types,
     // and then cast back to the original type with vector.bitcast op.
@@ -649,7 +960,7 @@ struct ConvertVectorMaskedLoad final
     // subvector at the proper offset after bit-casting.
     auto origType = op.getVectorType();
     auto origElements = origType.getNumElements();
-    bool isUnalignedEmulation = origElements % scale != 0;
+    bool isAlignedEmulation = origElements % emulatedPerContainerElem == 0;
 
     auto stridedMetadata =
         rewriter.create<memref::ExtractStridedMetadataOp>(loc, op.getBase());
@@ -657,37 +968,40 @@ struct ConvertVectorMaskedLoad final
     memref::LinearizedMemRefInfo linearizedInfo;
     std::tie(linearizedInfo, linearizedIndices) =
         memref::getLinearizedMemRefOffsetAndSize(
-            rewriter, loc, srcBits, dstBits,
+            rewriter, loc, emulatedBits, containerBits,
             stridedMetadata.getConstifiedMixedOffset(),
             stridedMetadata.getConstifiedMixedSizes(),
             stridedMetadata.getConstifiedMixedStrides(),
             getAsOpFoldResult(adaptor.getIndices()));
 
     std::optional<int64_t> foldedIntraVectorOffset =
-        isUnalignedEmulation
-            ? getConstantIntValue(linearizedInfo.intraDataOffset)
-            : 0;
+        isAlignedEmulation
+            ? 0
+            : getConstantIntValue(linearizedInfo.intraDataOffset);
 
-    int64_t maxIntraDataOffset = foldedIntraVectorOffset.value_or(scale - 1);
-    FailureOr<Operation *> newMask = getCompressedMaskOp(
-        rewriter, loc, op.getMask(), origElements, scale, maxIntraDataOffset);
+    int64_t maxIntraDataOffset =
+        foldedIntraVectorOffset.value_or(emulatedPerContainerElem - 1);
+    FailureOr<Operation *> newMask =
+        getCompressedMaskOp(rewriter, loc, op.getMask(), origElements,
+                            emulatedPerContainerElem, maxIntraDataOffset);
     if (failed(newMask))
       return failure();
 
     Value passthru = op.getPassThru();
 
-    auto numElements =
-        llvm::divideCeil(maxIntraDataOffset + origElements, scale);
-    auto loadType = VectorType::get(numElements, newElementType);
-    auto newBitcastType = VectorType::get(numElements * scale, oldElementType);
+    auto numElements = llvm::divideCeil(maxIntraDataOffset + origElements,
+                                        emulatedPerContainerElem);
+    auto loadType = VectorType::get(numElements, containerElemTy);
+    auto newBitcastType =
+        VectorType::get(numElements * emulatedPerContainerElem, emulatedElemTy);
 
     auto emptyVector = rewriter.create<arith::ConstantOp>(
         loc, newBitcastType, rewriter.getZeroAttr(newBitcastType));
     if (!foldedIntraVectorOffset) {
       passthru = dynamicallyInsertSubVector(
-          rewriter, loc, dyn_cast<TypedValue<VectorType>>(passthru),
-          emptyVector, linearizedInfo.intraDataOffset, origElements);
-    } else if (isUnalignedEmulation) {
+          rewriter, loc, passthru, emptyVector, linearizedInfo.intraDataOffset,
+          origElements);
+    } else if (!isAlignedEmulation) {
       passthru = staticallyInsertSubvector(rewriter, loc, passthru, emptyVector,
                                            *foldedIntraVectorOffset);
     }
@@ -706,16 +1020,16 @@ struct ConvertVectorMaskedLoad final
         rewriter.create<vector::BitCastOp>(loc, newBitcastType, newLoad);
 
     Value mask = op.getMask();
-    auto newSelectMaskType =
-        VectorType::get(numElements * scale, rewriter.getI1Type());
+    auto newSelectMaskType = VectorType::get(
+        numElements * emulatedPerContainerElem, rewriter.getI1Type());
     // TODO: try to fold if op's mask is constant
     auto emptyMask = rewriter.create<arith::ConstantOp>(
         loc, newSelectMaskType, rewriter.getZeroAttr(newSelectMaskType));
     if (!foldedIntraVectorOffset) {
-      mask = dynamicallyInsertSubVector(
-          rewriter, loc, dyn_cast<TypedValue<VectorType>>(mask), emptyMask,
-          linearizedInfo.intraDataOffset, origElements);
-    } else if (isUnalignedEmulation) {
+      mask = dynamicallyInsertSubVector(rewriter, loc, mask, emptyMask,
+                                        linearizedInfo.intraDataOffset,
+                                        origElements);
+    } else if (!isAlignedEmulation) {
       mask = staticallyInsertSubvector(rewriter, loc, op.getMask(), emptyMask,
                                        *foldedIntraVectorOffset);
     }
@@ -724,12 +1038,11 @@ struct ConvertVectorMaskedLoad final
         rewriter.create<arith::SelectOp>(loc, mask, bitCast, passthru);
     if (!foldedIntraVectorOffset) {
       result = dynamicallyExtractSubVector(
-          rewriter, loc, dyn_cast<TypedValue<VectorType>>(result),
-          op.getPassThru(), linearizedInfo.intraDataOffset, origElements);
-    } else if (isUnalignedEmulation) {
-      result =
-          staticallyExtractSubvector(rewriter, loc, op.getType(), result,
-                                     *foldedIntraVectorOffset, origElements);
+          rewriter, loc, result, op.getPassThru(),
+          linearizedInfo.intraDataOffset, origElements);
+    } else if (!isAlignedEmulation) {
+      result = staticallyExtractSubvector(
+          rewriter, loc, result, *foldedIntraVectorOffset, origElements);
     }
     rewriter.replaceOp(op, result);
 
@@ -741,6 +1054,7 @@ struct ConvertVectorMaskedLoad final
 // ConvertVectorTransferRead
 //===----------------------------------------------------------------------===//
 
+// TODO: Document-me
 struct ConvertVectorTransferRead final
     : OpConversionPattern<vector::TransferReadOp> {
   using OpConversionPattern::OpConversionPattern;
@@ -755,23 +1069,26 @@ struct ConvertVectorTransferRead final
                                          "only 1-D vectors are supported ATM");
 
     auto loc = op.getLoc();
-    auto convertedType = cast<MemRefType>(adaptor.getSource().getType());
-    Type oldElementType = op.getType().getElementType();
-    Type newElementType = convertedType.getElementType();
-    int srcBits = oldElementType.getIntOrFloatBitWidth();
-    int dstBits = newElementType.getIntOrFloatBitWidth();
-
-    if (dstBits % srcBits != 0) {
+    auto containerElemTy =
+        cast<MemRefType>(adaptor.getSource().getType()).getElementType();
+    Type emulatedElemTy = op.getType().getElementType();
+    int emulatedBits = emulatedElemTy.getIntOrFloatBitWidth();
+    int containerBits = containerElemTy.getIntOrFloatBitWidth();
+
+    // Check per-element alignment.
+    if (containerBits % emulatedBits != 0) {
       return rewriter.notifyMatchFailure(
-          op, "only dstBits % srcBits == 0 supported");
+          op, "impossible to pack emulated elements into container elements "
+              "(bit-wise misalignment)");
     }
-    int scale = dstBits / srcBits;
+    int emulatedPerContainerElem = containerBits / emulatedBits;
 
     auto origElements = op.getVectorType().getNumElements();
 
-    bool isUnalignedEmulation = origElements % scale != 0;
+    // Note, per-element-alignment was already verified above.
+    bool isFullyAligned = origElements % emulatedPerContainerElem == 0;
 
-    auto newPadding = rewriter.create<arith::ExtUIOp>(loc, newElementType,
+    auto newPadding = rewriter.create<arith::ExtUIOp>(loc, containerElemTy,
                                                       adaptor.getPadding());
 
     auto stridedMetadata =
@@ -781,28 +1098,30 @@ struct ConvertVectorTransferRead final
     memref::LinearizedMemRefInfo linearizedInfo;
     std::tie(linearizedInfo, linearizedIndices) =
         memref::getLinearizedMemRefOffsetAndSize(
-            rewriter, loc, srcBits, dstBits,
+            rewriter, loc, emulatedBits, containerBits,
             stridedMetadata.getConstifiedMixedOffset(),
             stridedMetadata.getConstifiedMixedSizes(),
             stridedMetadata.getConstifiedMixedStrides(),
             getAsOpFoldResult(adaptor.getIndices()));
 
     std::optional<int64_t> foldedIntraVectorOffset =
-        isUnalignedEmulation
-            ? getConstantIntValue(linearizedInfo.intraDataOffset)
-            : 0;
+        isFullyAligned ? 0
+                       : getConstantIntValue(linearizedInfo.intraDataOffset);
 
-    int64_t maxIntraDataOffset = foldedIntraVectorOffset.value_or(scale - 1);
-    auto numElements =
-        llvm::divideCeil(maxIntraDataOffset + origElements, scale);
+    int64_t maxIntraDataOffset =
+        foldedIntraVectorOffset.value_or(emulatedPerContainerElem - 1);
+    auto numElements = llvm::divideCeil(maxIntraDataOffset + origElements,
+                                        emulatedPerContainerElem);
 
     auto newRead = rewriter.create<vector::TransferReadOp>(
-        loc, VectorType::get(numElements, newElementType), adaptor.getSource(),
+        loc, VectorType::get(numElements, containerElemTy), adaptor.getSource(),
         getValueOrCreateConstantIndexOp(rewriter, loc, linearizedIndices),
         newPadding);
 
     auto bitCast = rewriter.create<vector::BitCastOp>(
-        loc, VectorType::get(numElements * scale, oldElementType), newRead);
+        loc,
+        VectorType::get(numElements * emulatedPerContainerElem, emulatedElemTy),
+        newRead);
 
     Value result = bitCast->getResult(0);
     if (!foldedIntraVectorOffset) {
@@ -811,10 +1130,9 @@ struct ConvertVectorTransferRead final
       result = dynamicallyExtractSubVector(rewriter, loc, bitCast, zeros,
                                            linearizedInfo.intraDataOffset,
                                            origElements);
-    } else if (isUnalignedEmulation) {
-      result =
-          staticallyExtractSubvector(rewriter, loc, op.getType(), result,
-                                     *foldedIntraVectorOffset, origElements);
+    } else if (!isFullyAligned) {
+      result = staticallyExtractSubvector(
+          rewriter, loc, result, *foldedIntraVectorOffset, origElements);
     }
     rewriter.replaceOp(op, result);
 
@@ -1495,33 +1813,34 @@ struct RewriteExtOfBitCast : OpRewritePattern<ExtOpType> {
 /// LLVM to scramble with peephole optimizations. Templated to choose between
 /// signed and unsigned conversions.
 ///
-/// For example (signed):
+/// EXAMPLE 1 (signed):
 ///    arith.extsi %in : vector<8xi4> to vector<8xi32>
-///      is rewriten as
-///        %0 = vector.bitcast %in : vector<8xi4> to vector<4xi8>
-///        %1 = arith.shli %0, 4 : vector<4xi8>
-///        %2 = arith.shrsi %1, 4 : vector<4xi8>
-///        %3 = arith.shrsi %0, 4 : vector<4xi8>
-///        %4 = vector.interleave %2, %3 : vector<4xi8> -> vector<8xi8>
-///        %5 = arith.extsi %4 : vector<8xi8> to vector<8xi32>
+/// is rewriten as:
+///    %0 = vector.bitcast %in : vector<8xi4> to vector<4xi8>
+///    %1 = arith.shli %0, 4 : vector<4xi8>
+///    %2 = arith.shrsi %1, 4 : vector<4xi8>
+///    %3 = arith.shrsi %0, 4 : vector<4xi8>
+///    %4 = vector.interleave %2, %3 : vector<4xi8> -> vector<8xi8>
+///    %5 = arith.extsi %4 : vector<8xi8> to vector<8xi32>
 ///
+/// EXAMPLE 2 (fp):
 ///    arith.sitofp %in : vector<8xi4> to vector<8xf32>
-///      is rewriten as
-///        %0 = vector.bitcast %in : vector<8xi4> to vector<4xi8>
-///        %1 = arith.shli %0, 4 : vector<4xi8>
-///        %2 = arith.shrsi %1, 4 : vector<4xi8>
-///        %3 = arith.shrsi %0, 4 : vector<4xi8>
-///        %4 = vector.interleave %2, %3 : vector<4xi8> -> vector<8xi8>
-///        %5 = arith.sitofp %4 : vector<8xi8> to vector<8xf32>
+/// is rewriten as:
+///    %0 = vector.bitcast %in : vector<8xi4> to vector<4xi8>
+///    %1 = arith.shli %0, 4 : vector<4xi8>
+///    %2 = arith.shrsi %1, 4 : vector<4xi8>
+///    %3 = arith.shrsi %0, 4 : vector<4xi8>
+///    %4 = vector.interleave %2, %3 : vector<4xi8> -> vector<8xi8>
+///    %5 = arith.sitofp %4 : vector<8xi8> to vector<8xf32>
 ///
-/// Example (unsigned):
+/// EXAMPLE 3 (unsigned):
 ///    arith.extui %in : vector<8xi4> to vector<8xi32>
-///      is rewritten as
-///        %0 = vector.bitcast %in : vector<8xi4> to vector<4xi8>
-///        %1 = arith.andi %0, 15 : vector<4xi8>
-///        %2 = arith.shrui %0, 4 : vector<4xi8>
-///        %3 = vector.interleave %1, %2 : vector<4xi8> -> vector<8xi8>
-///        %4 = arith.extui %3 : vector<8xi8> to vector<8xi32>
+///  is rewritten as:
+///    %0 = vector.bitcast %in : vector<8xi4> to vector<4xi8>
+///    %1 = arith.andi %0, 15 : vector<4xi8>
+///    %2 = arith.shrui %0, 4 : vector<4xi8>
+///    %3 = vector.interleave %1, %2 : vector<4xi8> -> vector<8xi8>
+///    %4 = arith.extui %3 : vector<8xi8> to vector<8xi32>
 ///
 template <typename ConversionOpType, bool isSigned>
 struct RewriteAlignedSubByteIntExt : OpRewritePattern<ConversionOpType> {
@@ -1531,8 +1850,8 @@ struct RewriteAlignedSubByteIntExt : OpRewritePattern<ConversionOpType> {
                                 PatternRewriter &rewriter) const override {
     // Verify the preconditions.
     Value srcValue = conversionOp.getIn();
-    auto srcVecType = dyn_cast<VectorType>(srcValue.getType());
-    auto dstVecType = dyn_cast<VectorType>(conversionOp.getType());
+    VectorType srcVecType = dyn_cast<VectorType>(srcValue.getType());
+    VectorType dstVecType = dyn_cast<VectorType>(conversionOp.getType());
 
     if (failed(
             commonConversionPrecondition(rewriter, dstVecType, conversionOp)))
@@ -1572,15 +1891,16 @@ struct RewriteAlignedSubByteIntExt : OpRewritePattern<ConversionOpType> {
 ///
 /// For example:
 ///    arith.trunci %in : vector<8xi32> to vector<8xi4>
-///      is rewriten as
 ///
-///        %cst = arith.constant dense<15> : vector<4xi8>
-///        %cst_0 = arith.constant dense<4> : vector<4xi8>
-///        %0, %1 = vector.deinterleave %in : vector<8xi8>, vector<8xi8>
-///        %2 = arith.andi %0, %cst : vector<4xi8>
-///        %3 = arith.shli %1, %cst_0 : vector<4xi8>
-///        %4 = arith.ori %2, %3 : vector<4xi8>
-///        %5 = vector.bitcast %4 : vector<4xi8> to vector<8xi4>
+/// is rewriten as:
+///
+///   %cst = arith.constant dense<15> : vector<4xi8>
+///   %cst_0 = arith.constant dense<4> : vector<4xi8>
+///   %0, %1 = vector.deinterleave %in : vector<8xi8>, vector<8xi8>
+///   %2 = arith.andi %0, %cst : vector<4xi8>
+///   %3 = arith.shli %1, %cst_0 : vector<4xi8>
+///   %4 = arith.ori %2, %3 : vector<4xi8>
+///   %5 = vector.bitcast %4 : vector<4xi8> to vector<8xi4>
 ///
 struct RewriteAlignedSubByteIntTrunc : OpRewritePattern<arith::TruncIOp> {
   using OpRewritePattern<arith::TruncIOp>::OpRewritePattern;
@@ -1624,10 +1944,11 @@ struct RewriteAlignedSubByteIntTrunc : OpRewritePattern<arith::TruncIOp> {
 
 /// Rewrite a sub-byte vector transpose into a sequence of instructions that
 /// perform the transpose on wider (byte) element types.
-/// For example:
+///
+/// EXAMPLE:
 ///   %0 = vector.transpose %a, [1, 0] : vector<8x16xi4> to vector<16x8xi4>
 ///
-///   is rewritten as:
+/// is rewritten as:
 ///
 ///   %0 = arith.extsi %arg0 : vector<8x16xi4> to vector<8x16xi8>
 ///   %1 = vector.transpose %0, [1, 0] : vector<8x16xi8> to vector<16x8xi8>
@@ -1675,34 +1996,45 @@ struct RewriteVectorTranspose : OpRewritePattern<vector::TransposeOp> {
 // Public Interface Definition
 //===----------------------------------------------------------------------===//
 
+// The emulated type is inferred from the converted memref type.
 void vector::populateVectorNarrowTypeEmulationPatterns(
     const arith::NarrowTypeEmulationConverter &typeConverter,
-    RewritePatternSet &patterns) {
+    RewritePatternSet &patterns, bool disableAtomicRMW) {
 
   // Populate `vector.*` conversion patterns.
-  patterns.add<ConvertVectorLoad, ConvertVectorMaskedLoad, ConvertVectorStore,
+  // TODO: #119553 support atomicity
+  patterns.add<ConvertVectorLoad, ConvertVectorMaskedLoad,
                ConvertVectorMaskedStore, ConvertVectorTransferRead>(
       typeConverter, patterns.getContext());
+
+  // Populate `vector.*` store conversion patterns. The caller can choose
+  // to avoid emitting atomic operations and reduce it to read-modify-write
+  // sequence for stores if it is known there are no thread contentions.
+  patterns.insert<ConvertVectorStore>(patterns.getContext(), disableAtomicRMW);
 }
 
 void vector::populateVectorNarrowTypeRewritePatterns(
     RewritePatternSet &patterns, PatternBenefit benefit) {
+  // TODO: Document what the emulated type is.
   patterns.add<RewriteBitCastOfTruncI, RewriteExtOfBitCast<arith::ExtUIOp>,
                RewriteExtOfBitCast<arith::ExtSIOp>>(patterns.getContext(),
                                                     benefit);
 
   // Patterns for aligned cases. We set higher priority as they are expected to
   // generate better performance for aligned cases.
+  // The emulated type is always i8.
   patterns.add<RewriteAlignedSubByteIntExt<arith::ExtSIOp, /*isSigned=*/true>,
                RewriteAlignedSubByteIntExt<arith::SIToFPOp, /*isSigned=*/true>,
                RewriteAlignedSubByteIntTrunc>(patterns.getContext(),
                                               benefit.getBenefit() + 1);
+  // The emulated type is always i8.
   patterns
       .add<RewriteAlignedSubByteIntExt<arith::ExtUIOp, /*isSigned=*/false>,
            RewriteAlignedSubByteIntExt<arith::UIToFPOp, /*isSigned=*/false>>(
           patterns.getContext(), benefit.getBenefit() + 1);
 }
 
+// The emulated type is always i8.
 void vector::populateVectorTransposeNarrowTypeRewritePatterns(
     RewritePatternSet &patterns, PatternBenefit benefit) {
   patterns.add<RewriteVectorTranspose>(patterns.getContext(), benefit);
diff --git a/mlir/lib/Dialect/Vector/Transforms/VectorInsertExtractStridedSliceRewritePatterns.cpp b/mlir/lib/Dialect/Vector/Transforms/VectorInsertExtractStridedSliceRewritePatterns.cpp
index 72405fcfef00f..82a985c9e5824 100644
--- a/mlir/lib/Dialect/Vector/Transforms/VectorInsertExtractStridedSliceRewritePatterns.cpp
+++ b/mlir/lib/Dialect/Vector/Transforms/VectorInsertExtractStridedSliceRewritePatterns.cpp
@@ -96,11 +96,15 @@ class ConvertSameRankInsertStridedSliceIntoShuffle
                                 PatternRewriter &rewriter) const override {
     auto srcType = op.getSourceVectorType();
     auto dstType = op.getDestVectorType();
+    int64_t srcRank = srcType.getRank();
+
+    // Scalable vectors are not supported by vector shuffle.
+    if ((srcType.isScalable() || dstType.isScalable()) && srcRank == 1)
+      return failure();
 
     if (op.getOffsets().getValue().empty())
       return failure();
 
-    int64_t srcRank = srcType.getRank();
     int64_t dstRank = dstType.getRank();
     assert(dstRank >= srcRank);
     if (dstRank != srcRank)
@@ -184,6 +188,11 @@ class Convert1DExtractStridedSliceIntoShuffle
   LogicalResult matchAndRewrite(ExtractStridedSliceOp op,
                                 PatternRewriter &rewriter) const override {
     auto dstType = op.getType();
+    auto srcType = op.getSourceVectorType();
+
+    // Scalable vectors are not supported by vector shuffle.
+    if (dstType.isScalable() || srcType.isScalable())
+      return failure();
 
     assert(!op.getOffsets().getValue().empty() && "Unexpected empty offsets");
 
@@ -309,6 +318,8 @@ class DecomposeNDExtractStridedSlice
   }
 };
 
+// TODO: Make sure these `populate*` patterns are tested in isolation.
+
 void vector::populateVectorInsertExtractStridedSliceDecompositionPatterns(
     RewritePatternSet &patterns, PatternBenefit benefit) {
   patterns.add<DecomposeDifferentRankInsertStridedSlice,
@@ -331,4 +342,14 @@ void vector::populateVectorInsertExtractStridedSliceTransforms(
   patterns.add<ConvertSameRankInsertStridedSliceIntoShuffle,
                Convert1DExtractStridedSliceIntoShuffle>(patterns.getContext(),
                                                         benefit);
+  // Generate chains of extract/insert ops for scalable vectors only as they
+  // can't be lowered to vector shuffles.
+  populateVectorExtractStridedSliceToExtractInsertChainPatterns(
+      patterns,
+      /*controlFn=*/
+      [](ExtractStridedSliceOp op) {
+        return op.getType().isScalable() ||
+               op.getSourceVectorType().isScalable();
+      },
+      benefit);
 }
diff --git a/mlir/lib/Dialect/Vector/Transforms/VectorLinearize.cpp b/mlir/lib/Dialect/Vector/Transforms/VectorLinearize.cpp
index 68535ae5a7a5c..3ecd585c5a26d 100644
--- a/mlir/lib/Dialect/Vector/Transforms/VectorLinearize.cpp
+++ b/mlir/lib/Dialect/Vector/Transforms/VectorLinearize.cpp
@@ -72,13 +72,14 @@ struct LinearizeConstant final : OpConversionPattern<arith::ConstantOp> {
     auto resType =
         getTypeConverter()->convertType<VectorType>(constOp.getType());
 
+    if (!resType)
+      return rewriter.notifyMatchFailure(loc, "can't convert return type");
+
     if (resType.isScalable() && !isa<SplatElementsAttr>(constOp.getValue()))
       return rewriter.notifyMatchFailure(
           loc,
           "Cannot linearize a constant scalable vector that's not a splat");
 
-    if (!resType)
-      return rewriter.notifyMatchFailure(loc, "can't convert return type");
     if (!isLessThanTargetBitWidth(constOp, targetVectorBitWidth))
       return rewriter.notifyMatchFailure(
           loc, "Can't flatten since targetBitWidth <= OpSize");
@@ -459,6 +460,45 @@ struct LinearizeVectorInsert final
 private:
   unsigned targetVectorBitWidth;
 };
+
+/// This pattern converts the BitCastOp that works on nD (n > 1)
+/// vectors to a BitCastOp that works on linearized vectors.
+/// Following,
+///   vector.bitcast %v1: vector<4x2xf32> to vector<4x4xf16>
+/// is converted to :
+///   %v1_1d = vector.shape_cast %v1: vector<4x2xf32> to vector<8xf32>
+///   %out_1d = vector.bitcast %v1_1d: vector<8xf32> to vector<16xf16>
+///   %out_nd = vector.shape_cast %out_1d: vector<16xf16> to vector<4x4xf16>
+struct LinearizeVectorBitCast final
+    : public OpConversionPattern<vector::BitCastOp> {
+  using OpConversionPattern::OpConversionPattern;
+  LinearizeVectorBitCast(
+      const TypeConverter &typeConverter, MLIRContext *context,
+      unsigned targetVectBitWidth = std::numeric_limits<unsigned>::max(),
+      PatternBenefit benefit = 1)
+      : OpConversionPattern(typeConverter, context, benefit),
+        targetVectorBitWidth(targetVectBitWidth) {}
+  LogicalResult
+  matchAndRewrite(vector::BitCastOp castOp, OpAdaptor adaptor,
+                  ConversionPatternRewriter &rewriter) const override {
+    Location loc = castOp.getLoc();
+    auto resType = getTypeConverter()->convertType(castOp.getType());
+    if (!resType)
+      return rewriter.notifyMatchFailure(loc, "can't convert return type.");
+
+    if (!isLessThanTargetBitWidth(castOp, targetVectorBitWidth))
+      return rewriter.notifyMatchFailure(
+          loc, "Can't flatten since targetBitWidth <= OpSize");
+
+    rewriter.replaceOpWithNewOp<vector::BitCastOp>(castOp, resType,
+                                                   adaptor.getSource());
+    return mlir::success();
+  }
+
+private:
+  unsigned targetVectorBitWidth;
+};
+
 } // namespace
 
 void mlir::vector::populateVectorLinearizeTypeConversionsAndLegality(
@@ -485,7 +525,7 @@ void mlir::vector::populateVectorLinearizeTypeConversionsAndLegality(
   typeConverter.addTargetMaterialization(materializeCast);
   target.markUnknownOpDynamicallyLegal(
       [=](Operation *op) -> std::optional<bool> {
-        if ((isa<arith::ConstantOp>(op) ||
+        if ((isa<arith::ConstantOp>(op) || isa<vector::BitCastOp>(op) ||
              op->hasTrait<OpTrait::Vectorizable>())) {
           return (isLessThanTargetBitWidth(op, targetBitWidth)
                       ? typeConverter.isLegal(op)
@@ -494,8 +534,9 @@ void mlir::vector::populateVectorLinearizeTypeConversionsAndLegality(
         return std::nullopt;
       });
 
-  patterns.add<LinearizeConstant, LinearizeVectorizable>(
-      typeConverter, patterns.getContext(), targetBitWidth);
+  patterns
+      .add<LinearizeConstant, LinearizeVectorizable, LinearizeVectorBitCast>(
+          typeConverter, patterns.getContext(), targetBitWidth);
 }
 
 void mlir::vector::populateVectorLinearizeShuffleLikeOpsPatterns(
diff --git a/mlir/lib/Dialect/Vector/Transforms/VectorTransforms.cpp b/mlir/lib/Dialect/Vector/Transforms/VectorTransforms.cpp
index 275f11160487a..47fca8e72b573 100644
--- a/mlir/lib/Dialect/Vector/Transforms/VectorTransforms.cpp
+++ b/mlir/lib/Dialect/Vector/Transforms/VectorTransforms.cpp
@@ -1792,11 +1792,11 @@ struct DropUnitDimsFromTransposeOp final
     auto dropDimsShapeCast = rewriter.create<vector::ShapeCastOp>(
         loc, sourceTypeWithoutUnitDims, op.getVector());
     // Create the new transpose.
-    auto tranposeWithoutUnitDims =
+    auto transposeWithoutUnitDims =
         rewriter.create<vector::TransposeOp>(loc, dropDimsShapeCast, newPerm);
     // Restore the unit dims via shape cast.
     rewriter.replaceOpWithNewOp<vector::ShapeCastOp>(
-        op, op.getResultVectorType(), tranposeWithoutUnitDims);
+        op, op.getResultVectorType(), transposeWithoutUnitDims);
 
     return success();
   }
diff --git a/mlir/lib/Dialect/Vector/Transforms/VectorUnroll.cpp b/mlir/lib/Dialect/Vector/Transforms/VectorUnroll.cpp
index 800c1d9fb1dbf..c1e3850f05c5e 100644
--- a/mlir/lib/Dialect/Vector/Transforms/VectorUnroll.cpp
+++ b/mlir/lib/Dialect/Vector/Transforms/VectorUnroll.cpp
@@ -172,7 +172,7 @@ struct UnrollTransferReadPattern
           readOp.getPermutationMapAttr(), readOp.getPadding(), readOp.getMask(),
           readOp.getInBoundsAttr());
 
-      result = rewriter.create<vector::InsertStridedSliceOp>(
+      result = rewriter.createOrFold<vector::InsertStridedSliceOp>(
           loc, slicedRead, result, elementOffsets, strides);
     }
     rewriter.replaceOp(readOp, result);
@@ -213,7 +213,7 @@ struct UnrollTransferWritePattern
     Value resultTensor;
     for (SmallVector<int64_t> elementOffsets :
          StaticTileOffsetRange(originalSize, *targetShape, loopOrder)) {
-      Value slicedVector = rewriter.create<vector::ExtractStridedSliceOp>(
+      Value slicedVector = rewriter.createOrFold<vector::ExtractStridedSliceOp>(
           loc, writeOp.getVector(), elementOffsets, *targetShape, strides);
       SmallVector<Value> indices =
           sliceTransferIndices(elementOffsets, originalIndices,
@@ -289,8 +289,9 @@ struct UnrollContractionPattern
         SmallVector<int64_t> operandShape = applyPermutationMap(
             permutationMap, ArrayRef<int64_t>(*targetShape));
         SmallVector<int64_t> operandStrides(operandOffets.size(), 1);
-        slicesOperands[index] = rewriter.create<vector::ExtractStridedSliceOp>(
-            loc, operand, operandOffets, operandShape, operandStrides);
+        slicesOperands[index] =
+            rewriter.createOrFold<vector::ExtractStridedSliceOp>(
+                loc, operand, operandOffets, operandShape, operandStrides);
       };
 
       // Extract the new lhs operand.
@@ -333,7 +334,7 @@ struct UnrollContractionPattern
         loc, dstVecType, rewriter.getZeroAttr(dstVecType));
     for (const auto &it : accCache) {
       SmallVector<int64_t> dstStrides(it.first.size(), 1);
-      result = rewriter.create<vector::InsertStridedSliceOp>(
+      result = rewriter.createOrFold<vector::InsertStridedSliceOp>(
           loc, it.second, result, it.first, dstStrides);
     }
     rewriter.replaceOp(contractOp, result);
@@ -371,8 +372,10 @@ struct UnrollMultiReductionPattern
          StaticTileOffsetRange(originalSize, *targetShape)) {
       SmallVector<Value> operands;
       SmallVector<int64_t> operandStrides(offsets.size(), 1);
-      Value slicedOperand = rewriter.create<vector::ExtractStridedSliceOp>(
-          loc, reductionOp.getSource(), offsets, *targetShape, operandStrides);
+      Value slicedOperand =
+          rewriter.createOrFold<vector::ExtractStridedSliceOp>(
+              loc, reductionOp.getSource(), offsets, *targetShape,
+              operandStrides);
       operands.push_back(slicedOperand);
       SmallVector<int64_t> dstShape;
       SmallVector<int64_t> destOffset;
@@ -390,7 +393,7 @@ struct UnrollMultiReductionPattern
       if (accIt != accCache.end())
         acc = accIt->second;
       else
-        acc = rewriter.create<vector::ExtractStridedSliceOp>(
+        acc = rewriter.createOrFold<vector::ExtractStridedSliceOp>(
             loc, reductionOp.getAcc(), destOffset, dstShape, accStrides);
       operands.push_back(acc);
       auto targetType = VectorType::get(
@@ -406,7 +409,7 @@ struct UnrollMultiReductionPattern
         rewriter.getZeroAttr(reductionOp.getDestType()));
     for (const auto &it : accCache) {
       SmallVector<int64_t> dstStrides(it.first.size(), 1);
-      result = rewriter.create<vector::InsertStridedSliceOp>(
+      result = rewriter.createOrFold<vector::InsertStridedSliceOp>(
           loc, it.second, result, it.first, dstStrides);
     }
     rewriter.replaceOp(reductionOp, result);
@@ -453,12 +456,12 @@ struct UnrollElementwisePattern : public RewritePattern {
           continue;
         }
         extractOperands.push_back(
-            rewriter.create<vector::ExtractStridedSliceOp>(
+            rewriter.createOrFold<vector::ExtractStridedSliceOp>(
                 loc, operand.get(), offsets, *targetShape, strides));
       }
       Operation *newOp = cloneOpWithOperandsAndTypes(
           rewriter, loc, op, extractOperands, newVecType);
-      result = rewriter.create<vector::InsertStridedSliceOp>(
+      result = rewriter.createOrFold<vector::InsertStridedSliceOp>(
           loc, newOp->getResult(0), result, offsets, strides);
     }
     rewriter.replaceOp(op, result);
@@ -490,8 +493,9 @@ struct UnrollReductionPattern : public OpRewritePattern<vector::ReductionOp> {
     for (SmallVector<int64_t> offsets :
          StaticTileOffsetRange(originalSize, *targetShape)) {
       SmallVector<int64_t> strides(offsets.size(), 1);
-      Value slicedOperand = rewriter.create<vector::ExtractStridedSliceOp>(
-          loc, reductionOp.getVector(), offsets, *targetShape, strides);
+      Value slicedOperand =
+          rewriter.createOrFold<vector::ExtractStridedSliceOp>(
+              loc, reductionOp.getVector(), offsets, *targetShape, strides);
       Operation *newOp = cloneOpWithOperandsAndTypes(
           rewriter, loc, reductionOp, slicedOperand, reductionOp.getType());
       Value result = newOp->getResult(0);
@@ -548,12 +552,13 @@ struct UnrollTransposePattern : public OpRewritePattern<vector::TransposeOp> {
         permutedOffsets[indices.value()] = elementOffsets[indices.index()];
         permutedShape[indices.value()] = (*targetShape)[indices.index()];
       }
-      Value slicedOperand = rewriter.create<vector::ExtractStridedSliceOp>(
-          loc, transposeOp.getVector(), permutedOffsets, permutedShape,
-          strides);
-      Value transposedSlice =
-          rewriter.create<vector::TransposeOp>(loc, slicedOperand, permutation);
-      result = rewriter.create<vector::InsertStridedSliceOp>(
+      Value slicedOperand =
+          rewriter.createOrFold<vector::ExtractStridedSliceOp>(
+              loc, transposeOp.getVector(), permutedOffsets, permutedShape,
+              strides);
+      Value transposedSlice = rewriter.createOrFold<vector::TransposeOp>(
+          loc, slicedOperand, permutation);
+      result = rewriter.createOrFold<vector::InsertStridedSliceOp>(
           loc, transposedSlice, result, elementOffsets, strides);
     }
     rewriter.replaceOp(transposeOp, result);
@@ -596,17 +601,19 @@ struct UnrollGatherPattern : public OpRewritePattern<vector::GatherOp> {
       // To get the unrolled gather, extract the same slice based on the
       // decomposed shape from each of the index, mask, and pass-through
       // vectors.
-      Value indexSubVec = rewriter.create<vector::ExtractStridedSliceOp>(
+      Value indexSubVec = rewriter.createOrFold<vector::ExtractStridedSliceOp>(
           loc, gatherOp.getIndexVec(), elementOffsets, *targetShape, strides);
-      Value maskSubVec = rewriter.create<vector::ExtractStridedSliceOp>(
+      Value maskSubVec = rewriter.createOrFold<vector::ExtractStridedSliceOp>(
           loc, gatherOp.getMask(), elementOffsets, *targetShape, strides);
-      Value passThruSubVec = rewriter.create<vector::ExtractStridedSliceOp>(
-          loc, gatherOp.getPassThru(), elementOffsets, *targetShape, strides);
+      Value passThruSubVec =
+          rewriter.createOrFold<vector::ExtractStridedSliceOp>(
+              loc, gatherOp.getPassThru(), elementOffsets, *targetShape,
+              strides);
       auto slicedGather = rewriter.create<vector::GatherOp>(
           loc, targetType, gatherOp.getBase(), gatherOp.getIndices(),
           indexSubVec, maskSubVec, passThruSubVec);
 
-      result = rewriter.create<vector::InsertStridedSliceOp>(
+      result = rewriter.createOrFold<vector::InsertStridedSliceOp>(
           loc, slicedGather, result, elementOffsets, strides);
     }
     rewriter.replaceOp(gatherOp, result);
diff --git a/mlir/lib/Dialect/X86Vector/Transforms/LegalizeForLLVMExport.cpp b/mlir/lib/Dialect/X86Vector/Transforms/LegalizeForLLVMExport.cpp
index e918473cae9e3..260ac9ce589a3 100644
--- a/mlir/lib/Dialect/X86Vector/Transforms/LegalizeForLLVMExport.cpp
+++ b/mlir/lib/Dialect/X86Vector/Transforms/LegalizeForLLVMExport.cpp
@@ -90,6 +90,47 @@ struct MaskCompressOpConversion
   }
 };
 
+struct DotBF16OpConversion : public ConvertOpToLLVMPattern<DotBF16Op> {
+  using ConvertOpToLLVMPattern<DotBF16Op>::ConvertOpToLLVMPattern;
+
+  LogicalResult
+  matchAndRewrite(DotBF16Op op, OpAdaptor adaptor,
+                  ConversionPatternRewriter &rewriter) const override {
+    auto typeA = dyn_cast<VectorType>(op.getA().getType());
+    unsigned elemBitWidth = typeA.getElementTypeBitWidth();
+    unsigned opBitWidth = typeA.getShape()[0] * elemBitWidth;
+
+    auto opType = adaptor.getSrc().getType();
+    auto opSrc = adaptor.getSrc();
+    auto opA = adaptor.getA();
+    auto opB = adaptor.getB();
+
+    switch (opBitWidth) {
+    case 128: {
+      rewriter.replaceOpWithNewOp<DotBF16Ps128IntrOp>(op, opType, opSrc, opA,
+                                                      opB);
+      break;
+    }
+    case 256: {
+      rewriter.replaceOpWithNewOp<DotBF16Ps256IntrOp>(op, opType, opSrc, opA,
+                                                      opB);
+      break;
+    }
+    case 512: {
+      rewriter.replaceOpWithNewOp<DotBF16Ps512IntrOp>(op, opType, opSrc, opA,
+                                                      opB);
+      break;
+    }
+    default: {
+      return rewriter.notifyMatchFailure(op,
+                                         "unsupported AVX512-BF16 dot variant");
+    }
+    }
+
+    return success();
+  }
+};
+
 struct RsqrtOpConversion : public ConvertOpToLLVMPattern<RsqrtOp> {
   using ConvertOpToLLVMPattern<RsqrtOp>::ConvertOpToLLVMPattern;
 
@@ -161,8 +202,8 @@ using Registry = RegistryImpl<
 void mlir::populateX86VectorLegalizeForLLVMExportPatterns(
     const LLVMTypeConverter &converter, RewritePatternSet &patterns) {
   Registry::registerPatterns(converter, patterns);
-  patterns.add<MaskCompressOpConversion, RsqrtOpConversion, DotOpConversion>(
-      converter);
+  patterns.add<MaskCompressOpConversion, DotBF16OpConversion, RsqrtOpConversion,
+               DotOpConversion>(converter);
 }
 
 void mlir::configureX86VectorLegalizeForExportTarget(
@@ -170,6 +211,10 @@ void mlir::configureX86VectorLegalizeForExportTarget(
   Registry::configureTarget(target);
   target.addLegalOp<MaskCompressIntrOp>();
   target.addIllegalOp<MaskCompressOp>();
+  target.addLegalOp<DotBF16Ps128IntrOp>();
+  target.addLegalOp<DotBF16Ps256IntrOp>();
+  target.addLegalOp<DotBF16Ps512IntrOp>();
+  target.addIllegalOp<DotBF16Op>();
   target.addLegalOp<RsqrtIntrOp>();
   target.addIllegalOp<RsqrtOp>();
   target.addLegalOp<DotIntrOp>();
diff --git a/mlir/lib/Dialect/XeGPU/IR/XeGPUDialect.cpp b/mlir/lib/Dialect/XeGPU/IR/XeGPUDialect.cpp
index eb01b15de75c6..06fd03f3af3ad 100644
--- a/mlir/lib/Dialect/XeGPU/IR/XeGPUDialect.cpp
+++ b/mlir/lib/Dialect/XeGPU/IR/XeGPUDialect.cpp
@@ -55,6 +55,18 @@ ScatterTensorDescAttr::get(mlir::MLIRContext *context,
   return Base::get(context, scopeAttr, chunkSizeAttr);
 }
 
+LogicalResult ScatterTensorDescAttr::verify(
+    llvm::function_ref<mlir::InFlightDiagnostic()> emitError,
+    MemorySpaceAttr memory_space, IntegerAttr chunk_size) {
+  int64_t chunkSize = chunk_size.getInt();
+  SmallVector<int64_t> supportedChunkSizes = {1,  2,  3,  4,   8,
+                                              16, 32, 64, 128, 256};
+  if (!llvm::is_contained(supportedChunkSizes, chunkSize))
+    return emitError() << "invalid chunk size";
+
+  return success();
+}
+
 //===----------------------------------------------------------------------===//
 // XeGPU_SGMapAttr
 //===----------------------------------------------------------------------===//
@@ -166,8 +178,6 @@ mlir::Type TensorDescType::parse(::mlir::AsmParser &parser) {
         continue;
       }
     }
-    parser.emitError(parser.getCurrentLocation(),
-                     "Failed to parse the attribute.\n");
     return {};
   }
 
@@ -175,9 +185,10 @@ mlir::Type TensorDescType::parse(::mlir::AsmParser &parser) {
   if (parser.parseGreater())
     return {};
 
-  return TensorDescType::get(parser.getContext(), shape, elementType,
-                             encoding.value_or(mlir::Attribute()),
-                             sg_map.value_or(mlir::Attribute()));
+  return TensorDescType::getChecked(
+      [&]() { return parser.emitError(parser.getNameLoc()); },
+      parser.getContext(), shape, elementType,
+      encoding.value_or(mlir::Attribute()), sg_map.value_or(mlir::Attribute()));
 }
 
 void TensorDescType::print(::mlir::AsmPrinter &printer) const {
@@ -223,6 +234,79 @@ TensorDescType TensorDescType::get(llvm::ArrayRef<int64_t> shape,
   return Base::get(context, shape, elementType, attr, sg_map);
 }
 
+LogicalResult TensorDescType::verify(
+    llvm::function_ref<::mlir::InFlightDiagnostic()> emitError,
+    llvm::ArrayRef<int64_t> shape, mlir::Type elementType,
+    mlir::Attribute encoding, mlir::Attribute sg_map) {
+  size_t rank = shape.size();
+  if (rank != 1 && rank != 2)
+    return emitError() << "expected 1D or 2D tensor";
+
+  auto scatterAttr = mlir::dyn_cast_if_present<ScatterTensorDescAttr>(encoding);
+  if (scatterAttr) {
+    // Expected tensor ranks for scattered data:
+    //   - 1D tensor for fully non-contiguous elements (chunk size == 1)
+    //   - 2D tensor for scattered blocks (chunk size > 1)
+    unsigned chunkSize = scatterAttr.getChunkSize().getInt();
+    if (rank == 1 && chunkSize != 1)
+      return emitError() << "expected non-contiguous elements for 1D tensor";
+    if (rank == 2 && chunkSize < 2)
+      return emitError() << "expected chunk blocks for 2D tensor";
+  }
+
+  if (auto blockAttr =
+          mlir::dyn_cast_if_present<BlockTensorDescAttr>(encoding)) {
+    MemorySpaceAttr memorySpaceAttr = blockAttr.getMemorySpace();
+    if (rank == 2 && memorySpaceAttr &&
+        memorySpaceAttr.getValue() == MemorySpace::SLM)
+      return emitError() << "SLM is not supported for 2D block tensor";
+  }
+
+  if (auto sgMapAttr = llvm::dyn_cast_if_present<SGMapAttr>(sg_map)) {
+    ArrayRef<uint32_t> wiLayout = sgMapAttr.getWiLayout();
+    ArrayRef<uint32_t> wiData = sgMapAttr.getWiData();
+
+    if (rank == 1) {
+      if (wiLayout[0] != 1 || wiData[0] != 1)
+        return emitError()
+               << "outer layout distribution and data mapping must be 1 "
+                  "for 1D tensor";
+    }
+
+    if (scatterAttr) {
+      // Validate subgroup mapping rules for scattered tensors.
+      // A work-item's slice of the tensor with shape [sg_size] or
+      // [sg_size, chunk_size] will be [1] or [1, chunks_size] respectively,
+      // the mapping should reflect that.
+      if (wiData[0] != 1)
+        return emitError()
+               << "cannot map over non-contiguous scattered row elements";
+
+      unsigned chunkSize = scatterAttr.getChunkSize().getInt();
+      if (wiData[1] != chunkSize)
+        return emitError() << "work item data mapping must match the number of "
+                              "contiguous elements";
+    }
+
+    // For 1D tensor, pad the shape with an outer unit dimension to allow common
+    // validation logic.
+    SmallVector<int64_t> tensorShape(shape.begin(), shape.end());
+    if (rank == 1)
+      tensorShape = {1, tensorShape.back()};
+
+    size_t dims = tensorShape.size();
+    for (size_t i = 0; i < dims; ++i) {
+      uint32_t numElemPerWi = wiLayout[i] * wiData[i];
+      if (tensorShape[i] < numElemPerWi || tensorShape[i] % numElemPerWi != 0)
+        return emitError() << "cannot distribute " << tensorShape[i] << " over "
+                           << wiLayout[i] << " work items with " << wiData[i]
+                           << " elements each";
+    }
+  }
+
+  return success();
+}
+
 } // namespace xegpu
 } // namespace mlir
 
diff --git a/mlir/lib/Dialect/XeGPU/IR/XeGPUOps.cpp b/mlir/lib/Dialect/XeGPU/IR/XeGPUOps.cpp
index 81f46f941785a..25dc1f22f0432 100644
--- a/mlir/lib/Dialect/XeGPU/IR/XeGPUOps.cpp
+++ b/mlir/lib/Dialect/XeGPU/IR/XeGPUOps.cpp
@@ -81,24 +81,28 @@ static bool isWriteHintOrNone(const CachePolicyAttr &attr) {
 //   each dimension.
 static bool isArgShapesValid(ArrayRef<int64_t> descShape,
                              ArrayRef<int64_t> valShape, SGMapAttr sgMap) {
-  if (descShape == valShape) {
-    if (!sgMap)
-      return true;
-
-    // this can be relaxed if necessary by supporting non-2d shapes distribution
-    // until the constraints are defined this lives here instead of the tensor
-    // descriptor type.
-    return valShape.size() == sgMap.getWiLayout().size();
-  }
+  // Equal shapes with no distribution - no further verification needed.
+  if (descShape == valShape && !sgMap)
+    return true;
 
+  // Unknown distribution - cannot perform operation on partial shape.
   if (!sgMap)
     return false;
 
-  if (valShape.size() != descShape.size())
+  // Invalid rank or mixed rank usage.
+  size_t descRank = descShape.size();
+  if (descRank > 2 || valShape.size() != descRank)
     return false;
 
+  // For 1D, SG map is guaranteed to be unit size in the outer dimension.
+  // Only take the distribution over the innermost dimension for validation.
+  ArrayRef<uint32_t> wiLayout = sgMap.getWiLayout();
+  SmallVector<uint32_t> mapLayout(wiLayout.begin(), wiLayout.end());
+  if (descRank == 1)
+    mapLayout = {wiLayout.back()};
+
   for (const auto &[factor, dim, expected] :
-       llvm::zip_equal(sgMap.getWiLayout(), valShape, descShape)) {
+       llvm::zip_equal(mapLayout, valShape, descShape)) {
     if (factor * dim != expected)
       return false;
   }
@@ -227,10 +231,6 @@ LogicalResult CreateNdDescOp::verify() {
   if (getType().isScattered())
     return emitOpError("Expects a non-scattered TensorDesc.\n");
 
-  if (getType().getRank() == 2 &&
-      tdescMemorySpace == static_cast<unsigned>(MemorySpace::SLM))
-    return emitOpError("SLM is not supported for 2D Block TensorDesc.\n");
-
   return success();
 }
 
@@ -419,16 +419,8 @@ LogicalResult CreateDescOp::verify() {
            << " Source: " << srcMemorySpace
            << ", TensorDesc: " << tdescMemorySpace;
 
-  auto chunkSize = tdescTy.getChunkSize();
-
-  // check chunk_size
-  llvm::SmallVector<int64_t> supportedChunkSizes = {1,  2,  3,  4,   8,
-                                                    16, 32, 64, 128, 256};
-  if (!llvm::is_contained(supportedChunkSizes, chunkSize))
-    return emitOpError("Invalid chunk_size. Supported values are 1, 2, 3, 4, "
-                       "8, 16, 32, 64, 128, or 256.");
-
   // check total size
+  auto chunkSize = tdescTy.getChunkSize();
   auto elemBits = tdescTy.getElementType().getIntOrFloatBitWidth();
   auto bitsPerLane = elemBits * chunkSize;
   if (chunkSize > 1 && bitsPerLane % 32) {
@@ -513,10 +505,23 @@ LogicalResult LoadGatherOp::verify() {
 
   if (tdescTy.getRank() == 2) {
     if (!getTransposeAttr())
-      return emitOpError("load_gather has to be transposed.");
+      return emitOpError("load of rank-2 tensor has to be transposed.");
     transpose({1, 0}, tdescShape);
   }
 
+  if (auto sgMap = tdescTy.getSGMapAttr()) {
+    auto valueVecTy = cast<VectorType>(valueTy);
+    const int32_t wiData =
+        sgMap.getWiData()[0] > 1 ? sgMap.getWiData()[0] : sgMap.getWiData()[1];
+    // All represent the same concept: a number of row elements to store.
+    if (valueVecTy.getNumElements() != wiData ||
+        valueVecTy.getNumElements() != tdescTy.getChunkSize()) {
+      return emitOpError("Chunk size, vector size and wi_data must match.");
+    }
+    // Work-item's slice (i.e., vector shape to load) is [1] or [1, chunk_size].
+    tdescShape[tdescTy.getRank() - 1] = 1;
+  }
+
   if (valueShape != tdescShape)
     return emitOpError("Unexpected result shape")
            << "(Expected shape: " << makeString(tdescShape)
@@ -552,10 +557,23 @@ LogicalResult StoreScatterOp::verify() {
 
   if (tdescTy.getRank() == 2) {
     if (!getTransposeAttr())
-      return emitOpError("load_gather has to be transposed.");
+      return emitOpError("Store of a rank-2 tensor has to be transposed.");
     transpose({1, 0}, tdescShape);
   }
 
+  if (auto sgMap = tdescTy.getSGMapAttr()) {
+    auto valueVecTy = cast<VectorType>(valueTy);
+    const int32_t wiData =
+        sgMap.getWiData()[0] > 1 ? sgMap.getWiData()[0] : sgMap.getWiData()[1];
+    // All represent the same concept: a number of row elements to store.
+    if (valueVecTy.getNumElements() != wiData ||
+        valueVecTy.getNumElements() != tdescTy.getChunkSize()) {
+      return emitOpError("Chunk size, vector size and wi_data must match.");
+    }
+    // Work-item's slice (i.e., vector to store) is [1] or [1, chunk_size].
+    tdescShape[tdescTy.getRank() - 1] = 1;
+  }
+
   if (valueShape != tdescShape)
     return emitOpError("Unexpected value shape")
            << "(Expected shape: " << makeString(tdescShape)
diff --git a/mlir/lib/IR/AsmPrinter.cpp b/mlir/lib/IR/AsmPrinter.cpp
index c603db450cbdd..eea4f7fa5c4be 100644
--- a/mlir/lib/IR/AsmPrinter.cpp
+++ b/mlir/lib/IR/AsmPrinter.cpp
@@ -125,7 +125,8 @@ void OpAsmPrinter::printFunctionalType(Operation *op) {
 //===----------------------------------------------------------------------===//
 
 /// The OpAsmOpInterface, see OpAsmInterface.td for more details.
-#include "mlir/IR/OpAsmInterface.cpp.inc"
+#include "mlir/IR/OpAsmOpInterface.cpp.inc"
+#include "mlir/IR/OpAsmTypeInterface.cpp.inc"
 
 LogicalResult
 OpAsmDialectInterface::parseResource(AsmParsedResourceEntry &entry) const {
@@ -2187,13 +2188,6 @@ void AsmPrinter::Impl::printLocation(LocationAttr loc, bool allowAlias) {
   os << ')';
 }
 
-void AsmPrinter::Impl::printResourceHandle(
-    const AsmDialectResourceHandle &resource) {
-  auto *interface = cast<OpAsmDialectInterface>(resource.getDialect());
-  os << interface->getResourceKey(resource);
-  state.getDialectResources()[resource.getDialect()].insert(resource);
-}
-
 /// Returns true if the given dialect symbol data is simple enough to print in
 /// the pretty form. This is essentially when the symbol takes the form:
 ///   identifier (`<` body `>`)?
@@ -2278,6 +2272,13 @@ static void printElidedElementsAttr(raw_ostream &os) {
   os << R"(dense_resource<__elided__>)";
 }
 
+void AsmPrinter::Impl::printResourceHandle(
+    const AsmDialectResourceHandle &resource) {
+  auto *interface = cast<OpAsmDialectInterface>(resource.getDialect());
+  ::printKeywordOrString(interface->getResourceKey(resource), os);
+  state.getDialectResources()[resource.getDialect()].insert(resource);
+}
+
 LogicalResult AsmPrinter::Impl::printAlias(Attribute attr) {
   return state.getAliasState().getAlias(attr, os);
 }
@@ -3372,41 +3373,41 @@ void OperationPrinter::printResourceFileMetadata(
     auto printFn = [&](StringRef key, ResourceBuilder::ValueFn valueFn) {
       checkAddMetadataDict();
 
-      auto printFormatting = [&]() {
-        // Emit the top-level resource entry if we haven't yet.
-        if (!std::exchange(hadResource, true)) {
-          if (needResourceComma)
-            os << "," << newLine;
-          os << "  " << dictName << "_resources: {" << newLine;
-        }
-        // Emit the parent resource entry if we haven't yet.
-        if (!std::exchange(hadEntry, true)) {
-          if (needEntryComma)
-            os << "," << newLine;
-          os << "    " << name << ": {" << newLine;
-        } else {
-          os << "," << newLine;
-        }
-      };
-
+      std::string resourceStr;
+      auto printResourceStr = [&](raw_ostream &os) { os << resourceStr; };
       std::optional<uint64_t> charLimit =
           printerFlags.getLargeResourceStringLimit();
       if (charLimit.has_value()) {
-        std::string resourceStr;
         llvm::raw_string_ostream ss(resourceStr);
         valueFn(ss);
 
-        // Only print entry if it's string is small enough
+        // Only print entry if its string is small enough.
         if (resourceStr.size() > charLimit.value())
           return;
 
-        printFormatting();
-        os << "      " << key << ": " << resourceStr;
+        // Don't recompute resourceStr when valueFn is called below.
+        valueFn = printResourceStr;
+      }
+
+      // Emit the top-level resource entry if we haven't yet.
+      if (!std::exchange(hadResource, true)) {
+        if (needResourceComma)
+          os << "," << newLine;
+        os << "  " << dictName << "_resources: {" << newLine;
+      }
+      // Emit the parent resource entry if we haven't yet.
+      if (!std::exchange(hadEntry, true)) {
+        if (needEntryComma)
+          os << "," << newLine;
+        os << "    " << name << ": {" << newLine;
       } else {
-        printFormatting();
-        os << "      " << key << ": ";
-        valueFn(os);
+        os << "," << newLine;
       }
+      os << "      ";
+      ::printKeywordOrString(key, os);
+      os << ": ";
+      // Call printResourceStr or original valueFn, depending on charLimit.
+      valueFn(os);
     };
     ResourceBuilder entryBuilder(printFn);
     provider.buildResources(op, providerArgs..., entryBuilder);
diff --git a/mlir/lib/IR/Location.cpp b/mlir/lib/IR/Location.cpp
index ce78d30ee0a52..7a4df4fbd46d9 100644
--- a/mlir/lib/IR/Location.cpp
+++ b/mlir/lib/IR/Location.cpp
@@ -177,10 +177,8 @@ unsigned FileLineColLoc::getLine() const { return getStartLine(); }
 
 unsigned FileLineColLoc::getColumn() const { return getStartColumn(); }
 
-bool FileLineColLoc::classof(Attribute attr) {
-  // This could also have been for <= 2. But given this is matching previous
-  // behavior, it is left as is.
-  if (auto range = mlir::dyn_cast<FileLineColRange>(attr))
+bool mlir::isStrictFileLineColLoc(Location loc) {
+  if (auto range = mlir::dyn_cast<FileLineColRange>(loc))
     return range.getImpl()->size() == 2;
   return false;
 }
diff --git a/mlir/lib/Interfaces/CMakeLists.txt b/mlir/lib/Interfaces/CMakeLists.txt
index d3b7bf65ad3e7..a25694cfff5f2 100644
--- a/mlir/lib/Interfaces/CMakeLists.txt
+++ b/mlir/lib/Interfaces/CMakeLists.txt
@@ -58,6 +58,7 @@ add_mlir_library(MLIRFunctionInterfaces
   MLIRFunctionInterfacesIncGen
 
   LINK_LIBS PUBLIC
+  MLIRCallInterfaces
   MLIRIR
 )
 
diff --git a/mlir/lib/Interfaces/CallInterfaces.cpp b/mlir/lib/Interfaces/CallInterfaces.cpp
index da0ca0e24630f..e8ed4b339a0cb 100644
--- a/mlir/lib/Interfaces/CallInterfaces.cpp
+++ b/mlir/lib/Interfaces/CallInterfaces.cpp
@@ -7,9 +7,178 @@
 //===----------------------------------------------------------------------===//
 
 #include "mlir/Interfaces/CallInterfaces.h"
+#include "mlir/IR/Builders.h"
 
 using namespace mlir;
 
+//===----------------------------------------------------------------------===//
+// Argument and result attributes utilities
+//===----------------------------------------------------------------------===//
+
+static ParseResult
+parseTypeAndAttrList(OpAsmParser &parser, SmallVectorImpl<Type> &types,
+                     SmallVectorImpl<DictionaryAttr> &attrs) {
+  // Parse individual function results.
+  return parser.parseCommaSeparatedList([&]() -> ParseResult {
+    types.emplace_back();
+    attrs.emplace_back();
+    NamedAttrList attrList;
+    if (parser.parseType(types.back()) ||
+        parser.parseOptionalAttrDict(attrList))
+      return failure();
+    attrs.back() = attrList.getDictionary(parser.getContext());
+    return success();
+  });
+}
+
+ParseResult call_interface_impl::parseFunctionResultList(
+    OpAsmParser &parser, SmallVectorImpl<Type> &resultTypes,
+    SmallVectorImpl<DictionaryAttr> &resultAttrs) {
+  if (failed(parser.parseOptionalLParen())) {
+    // We already know that there is no `(`, so parse a type.
+    // Because there is no `(`, it cannot be a function type.
+    Type ty;
+    if (parser.parseType(ty))
+      return failure();
+    resultTypes.push_back(ty);
+    resultAttrs.emplace_back();
+    return success();
+  }
+
+  // Special case for an empty set of parens.
+  if (succeeded(parser.parseOptionalRParen()))
+    return success();
+  if (parseTypeAndAttrList(parser, resultTypes, resultAttrs))
+    return failure();
+  return parser.parseRParen();
+}
+
+ParseResult call_interface_impl::parseFunctionSignature(
+    OpAsmParser &parser, SmallVectorImpl<Type> &argTypes,
+    SmallVectorImpl<DictionaryAttr> &argAttrs,
+    SmallVectorImpl<Type> &resultTypes,
+    SmallVectorImpl<DictionaryAttr> &resultAttrs, bool mustParseEmptyResult) {
+  // Parse arguments.
+  if (parser.parseLParen())
+    return failure();
+  if (failed(parser.parseOptionalRParen())) {
+    if (parseTypeAndAttrList(parser, argTypes, argAttrs))
+      return failure();
+    if (parser.parseRParen())
+      return failure();
+  }
+  // Parse results.
+  if (succeeded(parser.parseOptionalArrow()))
+    return call_interface_impl::parseFunctionResultList(parser, resultTypes,
+                                                        resultAttrs);
+  if (mustParseEmptyResult)
+    return failure();
+  return success();
+}
+
+/// Print a function result list. The provided `attrs` must either be null, or
+/// contain a set of DictionaryAttrs of the same arity as `types`.
+static void printFunctionResultList(OpAsmPrinter &p, TypeRange types,
+                                    ArrayAttr attrs) {
+  assert(!types.empty() && "Should not be called for empty result list.");
+  assert((!attrs || attrs.size() == types.size()) &&
+         "Invalid number of attributes.");
+
+  auto &os = p.getStream();
+  bool needsParens = types.size() > 1 || llvm::isa<FunctionType>(types[0]) ||
+                     (attrs && !llvm::cast<DictionaryAttr>(attrs[0]).empty());
+  if (needsParens)
+    os << '(';
+  llvm::interleaveComma(llvm::seq<size_t>(0, types.size()), os, [&](size_t i) {
+    p.printType(types[i]);
+    if (attrs)
+      p.printOptionalAttrDict(llvm::cast<DictionaryAttr>(attrs[i]).getValue());
+  });
+  if (needsParens)
+    os << ')';
+}
+
+void call_interface_impl::printFunctionSignature(
+    OpAsmPrinter &p, TypeRange argTypes, ArrayAttr argAttrs, bool isVariadic,
+    TypeRange resultTypes, ArrayAttr resultAttrs, Region *body,
+    bool printEmptyResult) {
+  bool isExternal = !body || body->empty();
+  if (!isExternal && !isVariadic && !argAttrs && !resultAttrs &&
+      printEmptyResult) {
+    p.printFunctionalType(argTypes, resultTypes);
+    return;
+  }
+
+  p << '(';
+  for (unsigned i = 0, e = argTypes.size(); i < e; ++i) {
+    if (i > 0)
+      p << ", ";
+
+    if (!isExternal) {
+      ArrayRef<NamedAttribute> attrs;
+      if (argAttrs)
+        attrs = llvm::cast<DictionaryAttr>(argAttrs[i]).getValue();
+      p.printRegionArgument(body->getArgument(i), attrs);
+    } else {
+      p.printType(argTypes[i]);
+      if (argAttrs)
+        p.printOptionalAttrDict(
+            llvm::cast<DictionaryAttr>(argAttrs[i]).getValue());
+    }
+  }
+
+  if (isVariadic) {
+    if (!argTypes.empty())
+      p << ", ";
+    p << "...";
+  }
+
+  p << ')';
+
+  if (!resultTypes.empty()) {
+    p << " -> ";
+    printFunctionResultList(p, resultTypes, resultAttrs);
+  } else if (printEmptyResult) {
+    p << " -> ()";
+  }
+}
+
+void call_interface_impl::addArgAndResultAttrs(
+    Builder &builder, OperationState &result, ArrayRef<DictionaryAttr> argAttrs,
+    ArrayRef<DictionaryAttr> resultAttrs, StringAttr argAttrsName,
+    StringAttr resAttrsName) {
+  auto nonEmptyAttrsFn = [](DictionaryAttr attrs) {
+    return attrs && !attrs.empty();
+  };
+  // Convert the specified array of dictionary attrs (which may have null
+  // entries) to an ArrayAttr of dictionaries.
+  auto getArrayAttr = [&](ArrayRef<DictionaryAttr> dictAttrs) {
+    SmallVector<Attribute> attrs;
+    for (auto &dict : dictAttrs)
+      attrs.push_back(dict ? dict : builder.getDictionaryAttr({}));
+    return builder.getArrayAttr(attrs);
+  };
+
+  // Add the attributes to the operation arguments.
+  if (llvm::any_of(argAttrs, nonEmptyAttrsFn))
+    result.addAttribute(argAttrsName, getArrayAttr(argAttrs));
+
+  // Add the attributes to the operation results.
+  if (llvm::any_of(resultAttrs, nonEmptyAttrsFn))
+    result.addAttribute(resAttrsName, getArrayAttr(resultAttrs));
+}
+
+void call_interface_impl::addArgAndResultAttrs(
+    Builder &builder, OperationState &result,
+    ArrayRef<OpAsmParser::Argument> args, ArrayRef<DictionaryAttr> resultAttrs,
+    StringAttr argAttrsName, StringAttr resAttrsName) {
+  SmallVector<DictionaryAttr> argAttrs;
+  for (const auto &arg : args)
+    argAttrs.push_back(arg.attrs);
+  addArgAndResultAttrs(builder, result, argAttrs, resultAttrs, argAttrsName,
+                       resAttrsName);
+}
+
 //===----------------------------------------------------------------------===//
 // CallOpInterface
 //===----------------------------------------------------------------------===//
diff --git a/mlir/lib/Interfaces/FunctionImplementation.cpp b/mlir/lib/Interfaces/FunctionImplementation.cpp
index 988feee665fea..90f32896e8181 100644
--- a/mlir/lib/Interfaces/FunctionImplementation.cpp
+++ b/mlir/lib/Interfaces/FunctionImplementation.cpp
@@ -70,50 +70,7 @@ parseFunctionArgumentList(OpAsmParser &parser, bool allowVariadic,
       });
 }
 
-/// Parse a function result list.
-///
-///   function-result-list ::= function-result-list-parens
-///                          | non-function-type
-///   function-result-list-parens ::= `(` `)`
-///                                 | `(` function-result-list-no-parens `)`
-///   function-result-list-no-parens ::= function-result (`,` function-result)*
-///   function-result ::= type attribute-dict?
-///
-static ParseResult
-parseFunctionResultList(OpAsmParser &parser, SmallVectorImpl<Type> &resultTypes,
-                        SmallVectorImpl<DictionaryAttr> &resultAttrs) {
-  if (failed(parser.parseOptionalLParen())) {
-    // We already know that there is no `(`, so parse a type.
-    // Because there is no `(`, it cannot be a function type.
-    Type ty;
-    if (parser.parseType(ty))
-      return failure();
-    resultTypes.push_back(ty);
-    resultAttrs.emplace_back();
-    return success();
-  }
-
-  // Special case for an empty set of parens.
-  if (succeeded(parser.parseOptionalRParen()))
-    return success();
-
-  // Parse individual function results.
-  if (parser.parseCommaSeparatedList([&]() -> ParseResult {
-        resultTypes.emplace_back();
-        resultAttrs.emplace_back();
-        NamedAttrList attrs;
-        if (parser.parseType(resultTypes.back()) ||
-            parser.parseOptionalAttrDict(attrs))
-          return failure();
-        resultAttrs.back() = attrs.getDictionary(parser.getContext());
-        return success();
-      }))
-    return failure();
-
-  return parser.parseRParen();
-}
-
-ParseResult function_interface_impl::parseFunctionSignature(
+ParseResult function_interface_impl::parseFunctionSignatureWithArguments(
     OpAsmParser &parser, bool allowVariadic,
     SmallVectorImpl<OpAsmParser::Argument> &arguments, bool &isVariadic,
     SmallVectorImpl<Type> &resultTypes,
@@ -121,46 +78,11 @@ ParseResult function_interface_impl::parseFunctionSignature(
   if (parseFunctionArgumentList(parser, allowVariadic, arguments, isVariadic))
     return failure();
   if (succeeded(parser.parseOptionalArrow()))
-    return parseFunctionResultList(parser, resultTypes, resultAttrs);
+    return call_interface_impl::parseFunctionResultList(parser, resultTypes,
+                                                        resultAttrs);
   return success();
 }
 
-void function_interface_impl::addArgAndResultAttrs(
-    Builder &builder, OperationState &result, ArrayRef<DictionaryAttr> argAttrs,
-    ArrayRef<DictionaryAttr> resultAttrs, StringAttr argAttrsName,
-    StringAttr resAttrsName) {
-  auto nonEmptyAttrsFn = [](DictionaryAttr attrs) {
-    return attrs && !attrs.empty();
-  };
-  // Convert the specified array of dictionary attrs (which may have null
-  // entries) to an ArrayAttr of dictionaries.
-  auto getArrayAttr = [&](ArrayRef<DictionaryAttr> dictAttrs) {
-    SmallVector<Attribute> attrs;
-    for (auto &dict : dictAttrs)
-      attrs.push_back(dict ? dict : builder.getDictionaryAttr({}));
-    return builder.getArrayAttr(attrs);
-  };
-
-  // Add the attributes to the function arguments.
-  if (llvm::any_of(argAttrs, nonEmptyAttrsFn))
-    result.addAttribute(argAttrsName, getArrayAttr(argAttrs));
-
-  // Add the attributes to the function results.
-  if (llvm::any_of(resultAttrs, nonEmptyAttrsFn))
-    result.addAttribute(resAttrsName, getArrayAttr(resultAttrs));
-}
-
-void function_interface_impl::addArgAndResultAttrs(
-    Builder &builder, OperationState &result,
-    ArrayRef<OpAsmParser::Argument> args, ArrayRef<DictionaryAttr> resultAttrs,
-    StringAttr argAttrsName, StringAttr resAttrsName) {
-  SmallVector<DictionaryAttr> argAttrs;
-  for (const auto &arg : args)
-    argAttrs.push_back(arg.attrs);
-  addArgAndResultAttrs(builder, result, argAttrs, resultAttrs, argAttrsName,
-                       resAttrsName);
-}
-
 ParseResult function_interface_impl::parseFunctionOp(
     OpAsmParser &parser, OperationState &result, bool allowVariadic,
     StringAttr typeAttrName, FuncTypeBuilder funcTypeBuilder,
@@ -182,8 +104,8 @@ ParseResult function_interface_impl::parseFunctionOp(
   // Parse the function signature.
   SMLoc signatureLocation = parser.getCurrentLocation();
   bool isVariadic = false;
-  if (parseFunctionSignature(parser, allowVariadic, entryArgs, isVariadic,
-                             resultTypes, resultAttrs))
+  if (parseFunctionSignatureWithArguments(parser, allowVariadic, entryArgs,
+                                          isVariadic, resultTypes, resultAttrs))
     return failure();
 
   std::string errorMessage;
@@ -221,8 +143,8 @@ ParseResult function_interface_impl::parseFunctionOp(
 
   // Add the attributes to the function arguments.
   assert(resultAttrs.size() == resultTypes.size());
-  addArgAndResultAttrs(builder, result, entryArgs, resultAttrs, argAttrsName,
-                       resAttrsName);
+  call_interface_impl::addArgAndResultAttrs(
+      builder, result, entryArgs, resultAttrs, argAttrsName, resAttrsName);
 
   // Parse the optional function body. The printer will not print the body if
   // its empty, so disallow parsing of empty body in the parser.
@@ -241,68 +163,6 @@ ParseResult function_interface_impl::parseFunctionOp(
   return success();
 }
 
-/// Print a function result list. The provided `attrs` must either be null, or
-/// contain a set of DictionaryAttrs of the same arity as `types`.
-static void printFunctionResultList(OpAsmPrinter &p, ArrayRef<Type> types,
-                                    ArrayAttr attrs) {
-  assert(!types.empty() && "Should not be called for empty result list.");
-  assert((!attrs || attrs.size() == types.size()) &&
-         "Invalid number of attributes.");
-
-  auto &os = p.getStream();
-  bool needsParens = types.size() > 1 || llvm::isa<FunctionType>(types[0]) ||
-                     (attrs && !llvm::cast<DictionaryAttr>(attrs[0]).empty());
-  if (needsParens)
-    os << '(';
-  llvm::interleaveComma(llvm::seq<size_t>(0, types.size()), os, [&](size_t i) {
-    p.printType(types[i]);
-    if (attrs)
-      p.printOptionalAttrDict(llvm::cast<DictionaryAttr>(attrs[i]).getValue());
-  });
-  if (needsParens)
-    os << ')';
-}
-
-void function_interface_impl::printFunctionSignature(
-    OpAsmPrinter &p, FunctionOpInterface op, ArrayRef<Type> argTypes,
-    bool isVariadic, ArrayRef<Type> resultTypes) {
-  Region &body = op->getRegion(0);
-  bool isExternal = body.empty();
-
-  p << '(';
-  ArrayAttr argAttrs = op.getArgAttrsAttr();
-  for (unsigned i = 0, e = argTypes.size(); i < e; ++i) {
-    if (i > 0)
-      p << ", ";
-
-    if (!isExternal) {
-      ArrayRef<NamedAttribute> attrs;
-      if (argAttrs)
-        attrs = llvm::cast<DictionaryAttr>(argAttrs[i]).getValue();
-      p.printRegionArgument(body.getArgument(i), attrs);
-    } else {
-      p.printType(argTypes[i]);
-      if (argAttrs)
-        p.printOptionalAttrDict(
-            llvm::cast<DictionaryAttr>(argAttrs[i]).getValue());
-    }
-  }
-
-  if (isVariadic) {
-    if (!argTypes.empty())
-      p << ", ";
-    p << "...";
-  }
-
-  p << ')';
-
-  if (!resultTypes.empty()) {
-    p.getStream() << " -> ";
-    auto resultAttrs = op.getResAttrsAttr();
-    printFunctionResultList(p, resultTypes, resultAttrs);
-  }
-}
-
 void function_interface_impl::printFunctionAttributes(
     OpAsmPrinter &p, Operation *op, ArrayRef<StringRef> elided) {
   // Print out function attributes, if present.
diff --git a/mlir/lib/Interfaces/InferTypeOpInterface.cpp b/mlir/lib/Interfaces/InferTypeOpInterface.cpp
index 3eb401c449980..6b5e103cd36c2 100644
--- a/mlir/lib/Interfaces/InferTypeOpInterface.cpp
+++ b/mlir/lib/Interfaces/InferTypeOpInterface.cpp
@@ -48,14 +48,6 @@ mlir::reifyResultShapes(OpBuilder &b, Operation *op,
     assert(shapedType.getRank() ==
                static_cast<int64_t>(reifiedReturnShapes[resultIdx].size()) &&
            "incorrect implementation of ReifyRankedShapedTypeOpInterface");
-    for (int64_t dim = 0; dim < shapedType.getRank(); ++dim) {
-      // reifyResultShapes must return:
-      // * Attribute for static dimensions
-      // * Value for dynamic dimensions
-      assert(shapedType.isDynamicDim(dim) ==
-                 isa<Value>(reifiedReturnShapes[resultIdx][dim]) &&
-             "incorrect implementation of ReifyRankedShapedTypeOpInterface");
-    }
     ++resultIdx;
   }
   // Assert that every shaped value result was reified.
diff --git a/mlir/lib/Interfaces/SideEffectInterfaces.cpp b/mlir/lib/Interfaces/SideEffectInterfaces.cpp
index c9feb001a1984..59fd19310cea5 100644
--- a/mlir/lib/Interfaces/SideEffectInterfaces.cpp
+++ b/mlir/lib/Interfaces/SideEffectInterfaces.cpp
@@ -10,6 +10,7 @@
 
 #include "mlir/IR/SymbolTable.h"
 #include "llvm/ADT/SmallPtrSet.h"
+#include <utility>
 
 using namespace mlir;
 
@@ -41,10 +42,18 @@ bool mlir::isOpTriviallyDead(Operation *op) {
 /// allows for marking region operations as trivially dead without always being
 /// conservative of terminators.
 static bool wouldOpBeTriviallyDeadImpl(Operation *rootOp) {
-  // The set of operations to consider when checking for side effects.
-  SmallVector<Operation *, 1> effectingOps(1, rootOp);
+  // The set of operation intervals (end-exclusive) to consider when checking
+  // for side effects.
+  SmallVector<std::pair<Block::iterator, Block::iterator>, 1> effectingOps = {
+      std::make_pair(Block::iterator(rootOp), ++Block::iterator(rootOp))};
   while (!effectingOps.empty()) {
-    Operation *op = effectingOps.pop_back_val();
+    Block::iterator &it = effectingOps.back().first;
+    Block::iterator end = effectingOps.back().second;
+    if (it == end) {
+      effectingOps.pop_back();
+      continue;
+    }
+    mlir::Operation *op = &*(it++);
 
     // If the operation has recursive effects, push all of the nested operations
     // on to the stack to consider.
@@ -53,8 +62,7 @@ static bool wouldOpBeTriviallyDeadImpl(Operation *rootOp) {
     if (hasRecursiveEffects) {
       for (Region &region : op->getRegions()) {
         for (auto &block : region) {
-          for (auto &nestedOp : block)
-            effectingOps.push_back(&nestedOp);
+          effectingOps.push_back(std::make_pair(block.begin(), block.end()));
         }
       }
     }
@@ -86,10 +94,9 @@ static bool wouldOpBeTriviallyDeadImpl(Operation *rootOp) {
         return false;
       }
       continue;
-
-      // Otherwise, if the op has recursive side effects we can treat the
-      // operation itself as having no effects.
     }
+    // Otherwise, if the op only has recursive side effects we can treat the
+    // operation itself as having no effects. We will visit its children next.
     if (hasRecursiveEffects)
       continue;
 
diff --git a/mlir/lib/Interfaces/Utils/CMakeLists.txt b/mlir/lib/Interfaces/Utils/CMakeLists.txt
index ece6c8e46ffea..8c45f66997427 100644
--- a/mlir/lib/Interfaces/Utils/CMakeLists.txt
+++ b/mlir/lib/Interfaces/Utils/CMakeLists.txt
@@ -8,6 +8,7 @@ add_mlir_library(MLIRInferIntRangeCommon
     MLIRInferIntRangeInterfaceIncGen
 
     LINK_LIBS PUBLIC
+    MLIRShapedOpInterfaces
     MLIRInferIntRangeInterface
     MLIRIR
 )
diff --git a/mlir/lib/Interfaces/Utils/InferIntRangeCommon.cpp b/mlir/lib/Interfaces/Utils/InferIntRangeCommon.cpp
index 1eab4139488bd..2f47939df5a02 100644
--- a/mlir/lib/Interfaces/Utils/InferIntRangeCommon.cpp
+++ b/mlir/lib/Interfaces/Utils/InferIntRangeCommon.cpp
@@ -14,6 +14,7 @@
 #include "mlir/Interfaces/Utils/InferIntRangeCommon.h"
 
 #include "mlir/Interfaces/InferIntRangeInterface.h"
+#include "mlir/Interfaces/ShapedOpInterfaces.h"
 
 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/STLExtras.h"
@@ -725,3 +726,46 @@ std::optional<bool> mlir::intrange::evaluatePred(CmpPredicate pred,
     return false;
   return std::nullopt;
 }
+
+//===----------------------------------------------------------------------===//
+// Shaped type dimension accessors / ShapedDimOpInterface
+//===----------------------------------------------------------------------===//
+
+ConstantIntRanges
+mlir::intrange::inferShapedDimOpInterface(ShapedDimOpInterface op,
+                                          const IntegerValueRange &maybeDim) {
+  unsigned width =
+      ConstantIntRanges::getStorageBitwidth(op->getResult(0).getType());
+  APInt zero = APInt::getZero(width);
+  APInt typeMax = APInt::getSignedMaxValue(width);
+
+  auto shapedTy = cast<ShapedType>(op.getShapedValue().getType());
+  if (!shapedTy.hasRank())
+    return ConstantIntRanges::fromSigned(zero, typeMax);
+
+  int64_t rank = shapedTy.getRank();
+  int64_t minDim = 0;
+  int64_t maxDim = rank - 1;
+  if (!maybeDim.isUninitialized()) {
+    const ConstantIntRanges &dim = maybeDim.getValue();
+    minDim = std::max(minDim, dim.smin().getSExtValue());
+    maxDim = std::min(maxDim, dim.smax().getSExtValue());
+  }
+
+  std::optional<ConstantIntRanges> result;
+  auto joinResult = [&](const ConstantIntRanges &thisResult) {
+    if (!result.has_value())
+      result = thisResult;
+    else
+      result = result->rangeUnion(thisResult);
+  };
+  for (int64_t i = minDim; i <= maxDim; ++i) {
+    int64_t length = shapedTy.getDimSize(i);
+
+    if (ShapedType::isDynamic(length))
+      joinResult(ConstantIntRanges::fromSigned(zero, typeMax));
+    else
+      joinResult(ConstantIntRanges::constant(APInt(width, length)));
+  }
+  return result.value_or(ConstantIntRanges::fromSigned(zero, typeMax));
+}
diff --git a/mlir/lib/Target/Cpp/TranslateToCpp.cpp b/mlir/lib/Target/Cpp/TranslateToCpp.cpp
index 01de0e41f2035..3ba1244e637ff 100644
--- a/mlir/lib/Target/Cpp/TranslateToCpp.cpp
+++ b/mlir/lib/Target/Cpp/TranslateToCpp.cpp
@@ -1387,11 +1387,9 @@ LogicalResult CppEmitter::emitOperand(Value value) {
     // as they might be evaluated in the wrong order depending on the shape of
     // the expression tree.
     bool encloseInParenthesis = precedence.value() <= getExpressionPrecedence();
-    if (encloseInParenthesis) {
+    if (encloseInParenthesis)
       os << "(";
-      pushExpressionPrecedence(lowestPrecedence());
-    } else
-      pushExpressionPrecedence(precedence.value());
+    pushExpressionPrecedence(precedence.value());
 
     if (failed(emitOperation(*def, /*trailingSemicolon=*/false)))
       return failure();
diff --git a/mlir/lib/Target/LLVM/CMakeLists.txt b/mlir/lib/Target/LLVM/CMakeLists.txt
index 4be147d02d579..83fbf7a5fe5f3 100644
--- a/mlir/lib/Target/LLVM/CMakeLists.txt
+++ b/mlir/lib/Target/LLVM/CMakeLists.txt
@@ -125,7 +125,7 @@ function(embed_binary_to_src file output_file symbol)
     # Convert hex data for C compatibility
     string(REGEX REPLACE "([0-9a-f][0-9a-f])" "0x\\1," filedata ${filedata})
     # Write data to output file
-    file(WRITE ${output_file} "const char ${symbol}[] = {${filedata}};\nconst int ${symbol}_size = sizeof(${symbol});\n")
+    file(WRITE ${output_file} "const unsigned char ${symbol}[] = {${filedata}};\nconst int ${symbol}_size = sizeof(${symbol});\n")
 endfunction()
 
 set(MLIR_NVVM_EMBED_LIBDEVICE 0 CACHE BOOL "Embed CUDA libdevice.bc in the binary at build time instead of looking it up at runtime")
diff --git a/mlir/lib/Target/LLVM/NVVM/Target.cpp b/mlir/lib/Target/LLVM/NVVM/Target.cpp
index 86ff848d6c6c2..e240a7ae4917f 100644
--- a/mlir/lib/Target/LLVM/NVVM/Target.cpp
+++ b/mlir/lib/Target/LLVM/NVVM/Target.cpp
@@ -35,10 +35,12 @@
 #include "llvm/Support/Process.h"
 #include "llvm/Support/Program.h"
 #include "llvm/Support/TargetSelect.h"
+#include "llvm/Support/Timer.h"
 #include "llvm/Support/raw_ostream.h"
 
 #include <cstdint>
 #include <cstdlib>
+#include <optional>
 
 using namespace mlir;
 using namespace mlir::NVVM;
@@ -47,7 +49,7 @@ using namespace mlir::NVVM;
 #define __DEFAULT_CUDATOOLKIT_PATH__ ""
 #endif
 
-extern "C" const char _mlir_embedded_libdevice[];
+extern "C" const unsigned char _mlir_embedded_libdevice[];
 extern "C" const unsigned _mlir_embedded_libdevice_size;
 
 namespace {
@@ -160,7 +162,8 @@ LogicalResult SerializeGPUModuleBase::appendStandardLibs() {
   // Allocate a resource using one of the UnManagedResourceBlob method to wrap
   // the embedded data.
   auto unmanagedBlob = UnmanagedAsmResourceBlob::allocateInferAlign(
-      ArrayRef<char>{_mlir_embedded_libdevice, _mlir_embedded_libdevice_size});
+      ArrayRef<char>{(const char *)_mlir_embedded_libdevice,
+                     _mlir_embedded_libdevice_size});
   librariesToLink.push_back(DenseResourceElementsAttr::get(
       type, resourceManager.insert("_mlir_embedded_libdevice",
                                    std::move(unmanagedBlob))));
@@ -219,6 +222,16 @@ class NVPTXSerializer : public SerializeGPUModuleBase {
   std::optional<SmallVector<char, 0>>
   moduleToObject(llvm::Module &llvmModule) override;
 
+  /// Get LLVMIR->ISA performance result.
+  /// Return nullopt if moduleToObject has not been called or the target format
+  /// is LLVMIR.
+  std::optional<int64_t> getLLVMIRToISATimeInMs();
+
+  /// Get ISA->Binary performance result.
+  /// Return nullopt if moduleToObject has not been called or the target format
+  /// is LLVMIR or ISA.
+  std::optional<int64_t> getISAToBinaryTimeInMs();
+
 private:
   using TmpFile = std::pair<llvm::SmallString<128>, llvm::FileRemover>;
 
@@ -234,13 +247,20 @@ class NVPTXSerializer : public SerializeGPUModuleBase {
 
   /// Target options.
   gpu::TargetOptions targetOptions;
+
+  /// LLVMIR->ISA perf result.
+  std::optional<int64_t> llvmToISATimeInMs;
+
+  /// ISA->Binary perf result.
+  std::optional<int64_t> isaToBinaryTimeInMs;
 };
 } // namespace
 
 NVPTXSerializer::NVPTXSerializer(Operation &module, NVVMTargetAttr target,
                                  const gpu::TargetOptions &targetOptions)
     : SerializeGPUModuleBase(module, target, targetOptions),
-      targetOptions(targetOptions) {}
+      targetOptions(targetOptions), llvmToISATimeInMs(std::nullopt),
+      isaToBinaryTimeInMs(std::nullopt) {}
 
 std::optional<NVPTXSerializer::TmpFile>
 NVPTXSerializer::createTemp(StringRef name, StringRef suffix) {
@@ -255,6 +275,14 @@ NVPTXSerializer::createTemp(StringRef name, StringRef suffix) {
   return TmpFile(filename, llvm::FileRemover(filename.c_str()));
 }
 
+std::optional<int64_t> NVPTXSerializer::getLLVMIRToISATimeInMs() {
+  return llvmToISATimeInMs;
+}
+
+std::optional<int64_t> NVPTXSerializer::getISAToBinaryTimeInMs() {
+  return isaToBinaryTimeInMs;
+}
+
 gpu::GPUModuleOp NVPTXSerializer::getOperation() {
   return dyn_cast<gpu::GPUModuleOp>(&SerializeGPUModuleBase::getOperation());
 }
@@ -617,6 +645,10 @@ NVPTXSerializer::compileToBinaryNVPTX(const std::string &ptxCode) {
 
 std::optional<SmallVector<char, 0>>
 NVPTXSerializer::moduleToObject(llvm::Module &llvmModule) {
+  llvm::Timer moduleToObjectTimer(
+      "moduleToObjectTimer",
+      "Timer for perf llvm-ir -> isa and isa -> binary.");
+  moduleToObjectTimer.startTimer();
   // Return LLVM IR if the compilation target is `offload`.
 #define DEBUG_TYPE "serialize-to-llvm"
   LLVM_DEBUG({
@@ -649,6 +681,11 @@ NVPTXSerializer::moduleToObject(llvm::Module &llvmModule) {
     getOperation().emitError() << "Failed translating the module to ISA.";
     return std::nullopt;
   }
+
+  moduleToObjectTimer.stopTimer();
+  llvmToISATimeInMs = moduleToObjectTimer.getTotalTime().getWallTime() * 1000;
+  moduleToObjectTimer.clear();
+  moduleToObjectTimer.startTimer();
   if (isaCallback)
     isaCallback(serializedISA.value());
 
@@ -668,17 +705,24 @@ NVPTXSerializer::moduleToObject(llvm::Module &llvmModule) {
     return SmallVector<char, 0>(bin.begin(), bin.end());
   }
 
+  std::optional<SmallVector<char, 0>> result;
   // Compile to binary.
 #if MLIR_ENABLE_NVPTXCOMPILER
-  return compileToBinaryNVPTX(*serializedISA);
+  result = compileToBinaryNVPTX(*serializedISA);
 #else
-  return compileToBinary(*serializedISA);
+  result = compileToBinary(*serializedISA);
 #endif // MLIR_ENABLE_NVPTXCOMPILER
+
+  moduleToObjectTimer.stopTimer();
+  isaToBinaryTimeInMs = moduleToObjectTimer.getTotalTime().getWallTime() * 1000;
+  moduleToObjectTimer.clear();
+  return result;
 }
 
 std::optional<SmallVector<char, 0>>
 NVVMTargetAttrImpl::serializeToObject(Attribute attribute, Operation *module,
                                       const gpu::TargetOptions &options) const {
+  Builder builder(attribute.getContext());
   assert(module && "The module must be non null.");
   if (!module)
     return std::nullopt;
@@ -688,7 +732,16 @@ NVVMTargetAttrImpl::serializeToObject(Attribute attribute, Operation *module,
   }
   NVPTXSerializer serializer(*module, cast<NVVMTargetAttr>(attribute), options);
   serializer.init();
-  return serializer.run();
+  std::optional<SmallVector<char, 0>> result = serializer.run();
+  auto llvmToISATimeInMs = serializer.getLLVMIRToISATimeInMs();
+  if (llvmToISATimeInMs.has_value())
+    module->setAttr("LLVMIRToISATimeInMs",
+                    builder.getI64IntegerAttr(*llvmToISATimeInMs));
+  auto isaToBinaryTimeInMs = serializer.getISAToBinaryTimeInMs();
+  if (isaToBinaryTimeInMs.has_value())
+    module->setAttr("ISAToBinaryTimeInMs",
+                    builder.getI64IntegerAttr(*isaToBinaryTimeInMs));
+  return result;
 }
 
 Attribute
@@ -699,7 +752,7 @@ NVVMTargetAttrImpl::createObject(Attribute attribute, Operation *module,
   gpu::CompilationTarget format = options.getCompilationTarget();
   DictionaryAttr objectProps;
   Builder builder(attribute.getContext());
-  SmallVector<NamedAttribute, 2> properties;
+  SmallVector<NamedAttribute, 4> properties;
   if (format == gpu::CompilationTarget::Assembly)
     properties.push_back(
         builder.getNamedAttr("O", builder.getI32IntegerAttr(target.getO())));
@@ -708,6 +761,14 @@ NVVMTargetAttrImpl::createObject(Attribute attribute, Operation *module,
     properties.push_back(builder.getNamedAttr(gpu::elfSectionName,
                                               builder.getStringAttr(section)));
 
+  for (const auto *perfName : {"LLVMIRToISATimeInMs", "ISAToBinaryTimeInMs"}) {
+    if (module->hasAttr(perfName)) {
+      IntegerAttr attr = llvm::dyn_cast<IntegerAttr>(module->getAttr(perfName));
+      properties.push_back(builder.getNamedAttr(
+          perfName, builder.getI64IntegerAttr(attr.getInt())));
+    }
+  }
+
   if (!properties.empty())
     objectProps = builder.getDictionaryAttr(properties);
 
diff --git a/mlir/lib/Target/LLVMIR/AttrKindDetail.h b/mlir/lib/Target/LLVMIR/AttrKindDetail.h
index aa2e1d04e1b4a..19ffc02aba3c6 100644
--- a/mlir/lib/Target/LLVMIR/AttrKindDetail.h
+++ b/mlir/lib/Target/LLVMIR/AttrKindDetail.h
@@ -39,7 +39,7 @@ getAttrKindToNameMapping() {
       {llvm::Attribute::AttrKind::InReg, LLVMDialect::getInRegAttrName()},
       {llvm::Attribute::AttrKind::Nest, LLVMDialect::getNestAttrName()},
       {llvm::Attribute::AttrKind::NoAlias, LLVMDialect::getNoAliasAttrName()},
-      {llvm::Attribute::AttrKind::NoCapture,
+      {llvm::Attribute::AttrKind::Captures,
        LLVMDialect::getNoCaptureAttrName()},
       {llvm::Attribute::AttrKind::NoFree, LLVMDialect::getNoFreeAttrName()},
       {llvm::Attribute::AttrKind::NonNull, LLVMDialect::getNonNullAttrName()},
diff --git a/mlir/lib/Target/LLVMIR/DebugTranslation.cpp b/mlir/lib/Target/LLVMIR/DebugTranslation.cpp
index cf734de49acd6..1d3ed6f3262f9 100644
--- a/mlir/lib/Target/LLVMIR/DebugTranslation.cpp
+++ b/mlir/lib/Target/LLVMIR/DebugTranslation.cpp
@@ -151,7 +151,7 @@ DebugTranslation::translateTemporaryImpl(DICompositeTypeAttr attr) {
       attr.getAlignInBits(),
       /*OffsetInBits=*/0,
       /*Flags=*/static_cast<llvm::DINode::DIFlags>(attr.getFlags()),
-      /*Elements=*/nullptr, /*RuntimeLang=*/0,
+      /*Elements=*/nullptr, /*RuntimeLang=*/0, /*EnumKind=*/std::nullopt,
       /*VTableHolder=*/nullptr);
 }
 
@@ -187,7 +187,7 @@ DebugTranslation::translateImpl(DICompositeTypeAttr attr) {
       /*OffsetInBits=*/0,
       /*Flags=*/static_cast<llvm::DINode::DIFlags>(attr.getFlags()),
       getMDTupleOrNull(attr.getElements()),
-      /*RuntimeLang=*/0, /*VTableHolder=*/nullptr,
+      /*RuntimeLang=*/0, /*EnumKind*/ std::nullopt, /*VTableHolder=*/nullptr,
       /*TemplateParams=*/nullptr, /*Identifier=*/nullptr,
       /*Discriminator=*/nullptr,
       getExpressionAttrOrNull(attr.getDataLocation()),
diff --git a/mlir/lib/Target/LLVMIR/Dialect/LLVMIR/LLVMToLLVMIRTranslation.cpp b/mlir/lib/Target/LLVMIR/Dialect/LLVMIR/LLVMToLLVMIRTranslation.cpp
index 2084e527773ca..4f89ee703ebb9 100644
--- a/mlir/lib/Target/LLVMIR/Dialect/LLVMIR/LLVMToLLVMIRTranslation.cpp
+++ b/mlir/lib/Target/LLVMIR/Dialect/LLVMIR/LLVMToLLVMIRTranslation.cpp
@@ -224,6 +224,39 @@ static void convertLinkerOptionsOp(ArrayAttr options,
   linkerMDNode->addOperand(listMDNode);
 }
 
+static LogicalResult
+convertParameterAndResultAttrs(CallOpInterface callOp, llvm::CallBase *call,
+                               LLVM::ModuleTranslation &moduleTranslation) {
+  if (ArrayAttr argAttrsArray = callOp.getArgAttrsAttr()) {
+    for (auto [argIdx, argAttrsAttr] : llvm::enumerate(argAttrsArray)) {
+      if (auto argAttrs = cast<DictionaryAttr>(argAttrsAttr);
+          !argAttrs.empty()) {
+        FailureOr<llvm::AttrBuilder> attrBuilder =
+            moduleTranslation.convertParameterAttrs(callOp, argAttrs);
+        if (failed(attrBuilder))
+          return failure();
+        call->addParamAttrs(argIdx, *attrBuilder);
+      }
+    }
+  }
+
+  ArrayAttr resAttrsArray = callOp.getResAttrsAttr();
+  if (resAttrsArray && resAttrsArray.size() > 0) {
+    if (resAttrsArray.size() != 1)
+      return mlir::emitError(callOp.getLoc(),
+                             "llvm.func cannot have multiple results");
+    if (auto resAttrs = cast<DictionaryAttr>(resAttrsArray[0]);
+        !resAttrs.empty()) {
+      FailureOr<llvm::AttrBuilder> attrBuilder =
+          moduleTranslation.convertParameterAttrs(callOp, resAttrs);
+      if (failed(attrBuilder))
+        return failure();
+      call->addRetAttrs(*attrBuilder);
+    }
+  }
+  return success();
+}
+
 static LogicalResult
 convertOperationImpl(Operation &opInst, llvm::IRBuilderBase &builder,
                      LLVM::ModuleTranslation &moduleTranslation) {
@@ -265,6 +298,9 @@ convertOperationImpl(Operation &opInst, llvm::IRBuilderBase &builder,
     if (callOp.getWillReturnAttr())
       call->addFnAttr(llvm::Attribute::WillReturn);
 
+    if (failed(convertParameterAndResultAttrs(callOp, call, moduleTranslation)))
+      return failure();
+
     if (MemoryEffectsAttr memAttr = callOp.getMemoryEffectsAttr()) {
       llvm::MemoryEffects memEffects =
           llvm::MemoryEffects(llvm::MemoryEffects::Location::ArgMem,
@@ -372,6 +408,9 @@ convertOperationImpl(Operation &opInst, llvm::IRBuilderBase &builder,
           operandsRef.drop_front(), opBundles);
     }
     result->setCallingConv(convertCConvToLLVM(invOp.getCConv()));
+    if (failed(
+            convertParameterAndResultAttrs(invOp, result, moduleTranslation)))
+      return failure();
     moduleTranslation.mapBranch(invOp, result);
     // InvokeOp can only have 0 or 1 result
     if (invOp->getNumResults() != 0) {
@@ -447,15 +486,21 @@ convertOperationImpl(Operation &opInst, llvm::IRBuilderBase &builder,
         addressOfOp.getGlobal(moduleTranslation.symbolTable());
     LLVM::LLVMFuncOp function =
         addressOfOp.getFunction(moduleTranslation.symbolTable());
+    LLVM::AliasOp alias = addressOfOp.getAlias(moduleTranslation.symbolTable());
 
     // The verifier should not have allowed this.
-    assert((global || function) &&
-           "referencing an undefined global or function");
-
-    moduleTranslation.mapValue(
-        addressOfOp.getResult(),
-        global ? moduleTranslation.lookupGlobal(global)
-               : moduleTranslation.lookupFunction(function.getName()));
+    assert((global || function || alias) &&
+           "referencing an undefined global, function, or alias");
+
+    llvm::Value *llvmValue = nullptr;
+    if (global)
+      llvmValue = moduleTranslation.lookupGlobal(global);
+    else if (alias)
+      llvmValue = moduleTranslation.lookupAlias(alias);
+    else
+      llvmValue = moduleTranslation.lookupFunction(function.getName());
+
+    moduleTranslation.mapValue(addressOfOp.getResult(), llvmValue);
     return success();
   }
 
diff --git a/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp b/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
index 3fcdefa8a2f67..51a3cbdbb5e7f 100644
--- a/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
+++ b/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
@@ -75,21 +75,6 @@ class OpenMPAllocaStackFrame
   llvm::OpenMPIRBuilder::InsertPointTy allocaInsertPoint;
 };
 
-/// ModuleTranslation stack frame containing the partial mapping between MLIR
-/// values and their LLVM IR equivalents.
-class OpenMPVarMappingStackFrame
-    : public LLVM::ModuleTranslation::StackFrameBase<
-          OpenMPVarMappingStackFrame> {
-public:
-  MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(OpenMPVarMappingStackFrame)
-
-  explicit OpenMPVarMappingStackFrame(
-      const DenseMap<Value, llvm::Value *> &mapping)
-      : mapping(mapping) {}
-
-  DenseMap<Value, llvm::Value *> mapping;
-};
-
 /// Custom error class to signal translation errors that don't need reporting,
 /// since encountering them will have already triggered relevant error messages.
 ///
@@ -1040,6 +1025,18 @@ mapInitializationArgs(T loop, LLVM::ModuleTranslation &moduleTranslation,
   }
 }
 
+static void
+setInsertPointForPossiblyEmptyBlock(llvm::IRBuilderBase &builder,
+                                    llvm::BasicBlock *block = nullptr) {
+  if (block == nullptr)
+    block = builder.GetInsertBlock();
+
+  if (block->empty() || block->getTerminator() == nullptr)
+    builder.SetInsertPoint(block);
+  else
+    builder.SetInsertPoint(block->getTerminator());
+}
+
 /// Inline reductions' `init` regions. This functions assumes that the
 /// `builder`'s insertion point is where the user wants the `init` regions to be
 /// inlined; i.e. it does not try to find a proper insertion location for the
@@ -1078,10 +1075,7 @@ initReductionVars(OP op, ArrayRef<BlockArgument> reductionArgs,
     }
   }
 
-  if (initBlock->empty() || initBlock->getTerminator() == nullptr)
-    builder.SetInsertPoint(initBlock);
-  else
-    builder.SetInsertPoint(initBlock->getTerminator());
+  setInsertPointForPossiblyEmptyBlock(builder, initBlock);
 
   // store result of the alloc region to the allocated pointer to the real
   // reduction variable
@@ -1106,11 +1100,7 @@ initReductionVars(OP op, ArrayRef<BlockArgument> reductionArgs,
     assert(phis.size() == 1 && "expected one value to be yielded from the "
                                "reduction neutral element declaration region");
 
-    if (builder.GetInsertBlock()->empty() ||
-        builder.GetInsertBlock()->getTerminator() == nullptr)
-      builder.SetInsertPoint(builder.GetInsertBlock());
-    else
-      builder.SetInsertPoint(builder.GetInsertBlock()->getTerminator());
+    setInsertPointForPossiblyEmptyBlock(builder);
 
     if (isByRef[i]) {
       if (!reductionDecls[i].getAllocRegion().empty())
@@ -1344,9 +1334,82 @@ findAssociatedValue(Value privateVar, llvm::IRBuilderBase &builder,
   return moduleTranslation.lookupValue(privateVar);
 }
 
-/// Allocate delayed private variables. Returns the basic block which comes
-/// after all of these allocations. llvm::Value * for each of these private
-/// variables are populated in llvmPrivateVars.
+/// Initialize a single (first)private variable. You probably want to use
+/// allocateAndInitPrivateVars instead of this.
+static llvm::Error initPrivateVar(
+    llvm::IRBuilderBase &builder, LLVM::ModuleTranslation &moduleTranslation,
+    omp::PrivateClauseOp &privDecl, Value mlirPrivVar, BlockArgument &blockArg,
+    llvm::SmallVectorImpl<llvm::Value *>::iterator llvmPrivateVarIt,
+    llvm::BasicBlock *privInitBlock,
+    llvm::DenseMap<Value, Value> *mappedPrivateVars = nullptr) {
+  Region &initRegion = privDecl.getInitRegion();
+  if (initRegion.empty()) {
+    moduleTranslation.mapValue(blockArg, *llvmPrivateVarIt);
+    return llvm::Error::success();
+  }
+
+  // map initialization region block arguments
+  llvm::Value *nonPrivateVar = findAssociatedValue(
+      mlirPrivVar, builder, moduleTranslation, mappedPrivateVars);
+  assert(nonPrivateVar);
+  moduleTranslation.mapValue(privDecl.getInitMoldArg(), nonPrivateVar);
+  moduleTranslation.mapValue(privDecl.getInitPrivateArg(), *llvmPrivateVarIt);
+
+  // in-place convert the private initialization region
+  SmallVector<llvm::Value *, 1> phis;
+  if (failed(inlineConvertOmpRegions(initRegion, "omp.private.init", builder,
+                                     moduleTranslation, &phis)))
+    return llvm::createStringError(
+        "failed to inline `init` region of `omp.private`");
+
+  assert(phis.size() == 1 && "expected one allocation to be yielded");
+
+  // prefer the value yielded from the init region to the allocated private
+  // variable in case the region is operating on arguments by-value (e.g.
+  // Fortran character boxes).
+  moduleTranslation.mapValue(blockArg, phis[0]);
+  *llvmPrivateVarIt = phis[0];
+
+  // clear init region block argument mapping in case it needs to be
+  // re-created with a different source for another use of the same
+  // reduction decl
+  moduleTranslation.forgetMapping(initRegion);
+  return llvm::Error::success();
+}
+
+static llvm::Error
+initPrivateVars(llvm::IRBuilderBase &builder,
+                LLVM::ModuleTranslation &moduleTranslation,
+                MutableArrayRef<BlockArgument> privateBlockArgs,
+                MutableArrayRef<omp::PrivateClauseOp> privateDecls,
+                MutableArrayRef<mlir::Value> mlirPrivateVars,
+                llvm::SmallVectorImpl<llvm::Value *> &llvmPrivateVars,
+                llvm::DenseMap<Value, Value> *mappedPrivateVars = nullptr) {
+  if (privateBlockArgs.empty())
+    return llvm::Error::success();
+
+  llvm::BasicBlock *privInitBlock = splitBB(builder, true, "omp.private.init");
+  setInsertPointForPossiblyEmptyBlock(builder, privInitBlock);
+
+  for (auto [idx, zip] : llvm::enumerate(
+           llvm::zip_equal(privateDecls, mlirPrivateVars, privateBlockArgs))) {
+    auto [privDecl, mlirPrivVar, blockArg] = zip;
+    llvm::Error err = initPrivateVar(
+        builder, moduleTranslation, privDecl, mlirPrivVar, blockArg,
+        llvmPrivateVars.begin() + idx, privInitBlock, mappedPrivateVars);
+
+    if (err)
+      return err;
+
+    setInsertPointForPossiblyEmptyBlock(builder);
+  }
+
+  return llvm::Error::success();
+}
+
+/// Allocate and initialize delayed private variables. Returns the basic block
+/// which comes after all of these allocations. llvm::Value * for each of these
+/// private variables are populated in llvmPrivateVars.
 static llvm::Expected<llvm::BasicBlock *>
 allocatePrivateVars(llvm::IRBuilderBase &builder,
                     LLVM::ModuleTranslation &moduleTranslation,
@@ -1361,7 +1424,8 @@ allocatePrivateVars(llvm::IRBuilderBase &builder,
       llvm::cast<llvm::BranchInst>(allocaIP.getBlock()->getTerminator());
   splitBB(llvm::OpenMPIRBuilder::InsertPointTy(allocaIP.getBlock(),
                                                allocaTerminator->getIterator()),
-          true, "omp.region.after_alloca");
+          true, allocaTerminator->getStableDebugLoc(),
+          "omp.region.after_alloca");
 
   llvm::IRBuilderBase::InsertPointGuard guard(builder);
   // Update the allocaTerminator in case the alloca block was split above.
@@ -1373,69 +1437,25 @@ allocatePrivateVars(llvm::IRBuilderBase &builder,
 
   llvm::BasicBlock *afterAllocas = allocaTerminator->getSuccessor(0);
 
-  // FIXME: Some of the allocation regions do more than just allocating.
-  // They read from their block argument (amongst other non-alloca things).
-  // When OpenMPIRBuilder outlines the parallel region into a different
-  // function it places the loads for live in-values (such as these block
-  // arguments) at the end of the entry block (because the entry block is
-  // assumed to contain only allocas). Therefore, if we put these complicated
-  // alloc blocks in the entry block, these will not dominate the availability
-  // of the live-in values they are using. Fix this by adding a latealloc
-  // block after the entry block to put these in (this also helps to avoid
-  // mixing non-alloca code with allocas).
-  // Alloc regions which do not use the block argument can still be placed in
-  // the entry block (therefore keeping the allocas together).
-  llvm::BasicBlock *privAllocBlock = nullptr;
-  if (!privateBlockArgs.empty())
-    privAllocBlock = splitBB(builder, true, "omp.private.latealloc");
   for (auto [privDecl, mlirPrivVar, blockArg] :
        llvm::zip_equal(privateDecls, mlirPrivateVars, privateBlockArgs)) {
-    Region &allocRegion = privDecl.getAllocRegion();
-
-    // map allocation region block argument
-    llvm::Value *nonPrivateVar = findAssociatedValue(
-        mlirPrivVar, builder, moduleTranslation, mappedPrivateVars);
-    assert(nonPrivateVar);
-    moduleTranslation.mapValue(privDecl.getAllocMoldArg(), nonPrivateVar);
-
-    // in-place convert the private allocation region
-    SmallVector<llvm::Value *, 1> phis;
-    if (privDecl.getAllocMoldArg().getUses().empty()) {
-      // TODO this should use
-      // allocaIP.getBlock()->getFirstNonPHIOrDbgOrAlloca() so it goes before
-      // the code for fetching the thread id. Not doing this for now to avoid
-      // test churn.
-      builder.SetInsertPoint(allocaIP.getBlock()->getTerminator());
-    } else {
-      builder.SetInsertPoint(privAllocBlock->getTerminator());
-    }
-
-    if (failed(inlineConvertOmpRegions(allocRegion, "omp.private.alloc",
-                                       builder, moduleTranslation, &phis)))
-      return llvm::createStringError(
-          "failed to inline `alloc` region of `omp.private`");
-
-    assert(phis.size() == 1 && "expected one allocation to be yielded");
-
-    moduleTranslation.mapValue(blockArg, phis[0]);
-    llvmPrivateVars.push_back(phis[0]);
-
-    // clear alloc region block argument mapping in case it needs to be
-    // re-created with a different source for another use of the same
-    // reduction decl
-    moduleTranslation.forgetMapping(allocRegion);
+    llvm::Type *llvmAllocType =
+        moduleTranslation.convertType(privDecl.getType());
+    builder.SetInsertPoint(allocaIP.getBlock()->getTerminator());
+    llvm::Value *llvmPrivateVar = builder.CreateAlloca(
+        llvmAllocType, /*ArraySize=*/nullptr, "omp.private.alloc");
+    llvmPrivateVars.push_back(llvmPrivateVar);
   }
+
   return afterAllocas;
 }
 
 static LogicalResult
-initFirstPrivateVars(llvm::IRBuilderBase &builder,
+copyFirstPrivateVars(llvm::IRBuilderBase &builder,
                      LLVM::ModuleTranslation &moduleTranslation,
                      SmallVectorImpl<mlir::Value> &mlirPrivateVars,
                      SmallVectorImpl<llvm::Value *> &llvmPrivateVars,
-                     SmallVectorImpl<omp::PrivateClauseOp> &privateDecls,
-                     llvm::BasicBlock *afterAllocas) {
-  llvm::IRBuilderBase::InsertPointGuard guard(builder);
+                     SmallVectorImpl<omp::PrivateClauseOp> &privateDecls) {
   // Apply copy region for firstprivate.
   bool needsFirstprivate =
       llvm::any_of(privateDecls, [](omp::PrivateClauseOp &privOp) {
@@ -1446,13 +1466,9 @@ initFirstPrivateVars(llvm::IRBuilderBase &builder,
   if (!needsFirstprivate)
     return success();
 
-  assert(afterAllocas->getSinglePredecessor());
-
-  // Find the end of the allocation blocks
-  builder.SetInsertPoint(afterAllocas->getSinglePredecessor()->getTerminator());
   llvm::BasicBlock *copyBlock =
       splitBB(builder, /*CreateBranch=*/true, "omp.private.copy");
-  builder.SetInsertPoint(copyBlock->getFirstNonPHIOrDbgOrAlloca());
+  setInsertPointForPossiblyEmptyBlock(builder, copyBlock);
 
   for (auto [decl, mlirVar, llvmVar] :
        llvm::zip_equal(privateDecls, mlirPrivateVars, llvmPrivateVars)) {
@@ -1471,11 +1487,12 @@ initFirstPrivateVars(llvm::IRBuilderBase &builder,
     moduleTranslation.mapValue(decl.getCopyPrivateArg(), llvmVar);
 
     // in-place convert copy region
-    builder.SetInsertPoint(builder.GetInsertBlock()->getTerminator());
     if (failed(inlineConvertOmpRegions(copyRegion, "omp.private.copy", builder,
                                        moduleTranslation)))
       return decl.emitError("failed to inline `copy` region of `omp.private`");
 
+    setInsertPointForPossiblyEmptyBlock(builder);
+
     // ignore unused value yielded from copy region
 
     // clear copy region block argument mapping in case it needs to be
@@ -1541,12 +1558,6 @@ convertOmpSections(Operation &opInst, llvm::IRBuilderBase &builder,
           isByRef)))
     return failure();
 
-  // Store the mapping between reduction variables and their private copies on
-  // ModuleTranslation stack. It can be then recovered when translating
-  // omp.reduce operations in a separate call.
-  LLVM::ModuleTranslation::SaveStack<OpenMPVarMappingStackFrame> mappingGuard(
-      moduleTranslation, reductionVariableMap);
-
   SmallVector<StorableBodyGenCallbackTy> sectionCBs;
 
   for (Operation &op : *sectionsOp.getRegion().begin()) {
@@ -1773,13 +1784,19 @@ convertOmpTaskOp(omp::TaskOp taskOp, llvm::IRBuilderBase &builder,
     if (handleError(afterAllocas, *taskOp).failed())
       return llvm::make_error<PreviouslyReportedError>();
 
-    if (failed(initFirstPrivateVars(builder, moduleTranslation, mlirPrivateVars,
-                                    llvmPrivateVars, privateDecls,
-                                    afterAllocas.get())))
+    builder.restoreIP(codegenIP);
+    if (handleError(initPrivateVars(builder, moduleTranslation,
+                                    privateBlockArgs, privateDecls,
+                                    mlirPrivateVars, llvmPrivateVars),
+                    *taskOp)
+            .failed())
+      return llvm::make_error<PreviouslyReportedError>();
+
+    if (failed(copyFirstPrivateVars(builder, moduleTranslation, mlirPrivateVars,
+                                    llvmPrivateVars, privateDecls)))
       return llvm::make_error<PreviouslyReportedError>();
 
     // translate the body of the task:
-    builder.restoreIP(codegenIP);
     auto continuationBlockOrError = convertOmpOpRegions(
         taskOp.getRegion(), "omp.task.region", builder, moduleTranslation);
     if (failed(handleError(continuationBlockOrError, *taskOp)))
@@ -1920,9 +1937,15 @@ convertOmpWsloop(Operation &opInst, llvm::IRBuilderBase &builder,
                                 deferredStores, isByRef)))
     return failure();
 
-  if (failed(initFirstPrivateVars(builder, moduleTranslation, mlirPrivateVars,
-                                  llvmPrivateVars, privateDecls,
-                                  afterAllocas.get())))
+  if (handleError(initPrivateVars(builder, moduleTranslation, privateBlockArgs,
+                                  privateDecls, mlirPrivateVars,
+                                  llvmPrivateVars),
+                  opInst)
+          .failed())
+    return failure();
+
+  if (failed(copyFirstPrivateVars(builder, moduleTranslation, mlirPrivateVars,
+                                  llvmPrivateVars, privateDecls)))
     return failure();
 
   assert(afterAllocas.get()->getSinglePredecessor());
@@ -1940,12 +1963,6 @@ convertOmpWsloop(Operation &opInst, llvm::IRBuilderBase &builder,
   if (failed(convertIgnoredWrappers(loopOp, wsloopOp, moduleTranslation)))
     return failure();
 
-  // Store the mapping between reduction variables and their private copies on
-  // ModuleTranslation stack. It can be then recovered when translating
-  // omp.reduce operations in a separate call.
-  LLVM::ModuleTranslation::SaveStack<OpenMPVarMappingStackFrame> mappingGuard(
-      moduleTranslation, reductionVariableMap);
-
   // Set up the source location value for OpenMP runtime.
   llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder);
 
@@ -2101,14 +2118,20 @@ convertOmpParallel(omp::ParallelOp opInst, llvm::IRBuilderBase &builder,
             deferredStores, isByRef)))
       return llvm::make_error<PreviouslyReportedError>();
 
-    if (failed(initFirstPrivateVars(builder, moduleTranslation, mlirPrivateVars,
-                                    llvmPrivateVars, privateDecls,
-                                    afterAllocas.get())))
-      return llvm::make_error<PreviouslyReportedError>();
-
     assert(afterAllocas.get()->getSinglePredecessor());
     builder.restoreIP(codeGenIP);
 
+    if (handleError(initPrivateVars(builder, moduleTranslation,
+                                    privateBlockArgs, privateDecls,
+                                    mlirPrivateVars, llvmPrivateVars),
+                    *opInst)
+            .failed())
+      return llvm::make_error<PreviouslyReportedError>();
+
+    if (failed(copyFirstPrivateVars(builder, moduleTranslation, mlirPrivateVars,
+                                    llvmPrivateVars, privateDecls)))
+      return llvm::make_error<PreviouslyReportedError>();
+
     if (failed(
             initReductionVars(opInst, reductionArgs, builder, moduleTranslation,
                               afterAllocas.get()->getSinglePredecessor(),
@@ -2116,12 +2139,6 @@ convertOmpParallel(omp::ParallelOp opInst, llvm::IRBuilderBase &builder,
                               reductionVariableMap, isByRef, deferredStores)))
       return llvm::make_error<PreviouslyReportedError>();
 
-    // Store the mapping between reduction variables and their private copies on
-    // ModuleTranslation stack. It can be then recovered when translating
-    // omp.reduce operations in a separate call.
-    LLVM::ModuleTranslation::SaveStack<OpenMPVarMappingStackFrame> mappingGuard(
-        moduleTranslation, reductionVariableMap);
-
     // Save the alloca insertion point on ModuleTranslation stack for use in
     // nested regions.
     LLVM::ModuleTranslation::SaveStack<OpenMPAllocaStackFrame> frame(
@@ -2162,6 +2179,7 @@ convertOmpParallel(omp::ParallelOp opInst, llvm::IRBuilderBase &builder,
       tempTerminator->eraseFromParent();
       builder.restoreIP(*contInsertPoint);
     }
+
     return llvm::Error::success();
   };
 
@@ -2263,7 +2281,6 @@ convertOmpSimd(Operation &opInst, llvm::IRBuilderBase &builder,
 
   llvm::OpenMPIRBuilder::InsertPointTy allocaIP =
       findAllocaInsertPoint(builder, moduleTranslation);
-  llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder);
 
   llvm::Expected<llvm::BasicBlock *> afterAllocas = allocatePrivateVars(
       builder, moduleTranslation, privateBlockArgs, privateDecls,
@@ -2271,6 +2288,15 @@ convertOmpSimd(Operation &opInst, llvm::IRBuilderBase &builder,
   if (handleError(afterAllocas, opInst).failed())
     return failure();
 
+  if (handleError(initPrivateVars(builder, moduleTranslation, privateBlockArgs,
+                                  privateDecls, mlirPrivateVars,
+                                  llvmPrivateVars),
+                  opInst)
+          .failed())
+    return failure();
+
+  llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder);
+
   // Generator of the canonical loop body.
   SmallVector<llvm::CanonicalLoopInfo *> loopInfos;
   SmallVector<llvm::OpenMPIRBuilder::InsertPointTy> bodyInsertPoints;
@@ -3303,7 +3329,15 @@ static void processMapMembersWithParent(
     combinedInfo.BasePointers.emplace_back(
         mapData.BasePointers[basePointerIndex]);
     combinedInfo.Pointers.emplace_back(mapData.Pointers[memberDataIdx]);
-    combinedInfo.Sizes.emplace_back(mapData.Sizes[memberDataIdx]);
+
+    llvm::Value *size = mapData.Sizes[memberDataIdx];
+    if (checkIfPointerMap(memberClause)) {
+      size = builder.CreateSelect(
+          builder.CreateIsNull(mapData.Pointers[memberDataIdx]),
+          builder.getInt64(0), size);
+    }
+
+    combinedInfo.Sizes.emplace_back(size);
   }
 }
 
@@ -3683,6 +3717,9 @@ convertOmpTargetData(Operation *op, llvm::IRBuilderBase &builder,
       }
       break;
     case BodyGenTy::DupNoPriv:
+      // We must always restoreIP regardless of doing anything the caller
+      // does not restore it, leading to incorrect (no) branch generation.
+      builder.restoreIP(codeGenIP);
       break;
     case BodyGenTy::NoPriv:
       // If device info is available then region has already been generated
@@ -3828,7 +3865,7 @@ handleDeclareTargetMapVar(MapInfoData &mapData,
           if (insn->getFunction() == func) {
             auto *load = builder.CreateLoad(mapData.BasePointers[i]->getType(),
                                             mapData.BasePointers[i]);
-            load->moveBefore(insn);
+            load->moveBefore(insn->getIterator());
             user->replaceUsesOfWith(mapData.OriginalValue[i], load);
           }
         }
@@ -4238,6 +4275,8 @@ convertOmpTarget(Operation &opInst, llvm::IRBuilderBase &builder,
   using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
   auto bodyCB = [&](InsertPointTy allocaIP, InsertPointTy codeGenIP)
       -> llvm::OpenMPIRBuilder::InsertPointOrErrorTy {
+    llvm::IRBuilderBase::InsertPointGuard guard(builder);
+    builder.SetCurrentDebugLocation(llvm::DebugLoc());
     // Forward target-cpu and target-features function attributes from the
     // original function to the new outlined function.
     llvm::Function *llvmParentFn =
@@ -4281,13 +4320,21 @@ convertOmpTarget(Operation &opInst, llvm::IRBuilderBase &builder,
     if (failed(handleError(afterAllocas, *targetOp)))
       return llvm::make_error<PreviouslyReportedError>();
 
+    builder.restoreIP(codeGenIP);
+    if (handleError(initPrivateVars(builder, moduleTranslation,
+                                    privateBlockArgs, privateDecls,
+                                    mlirPrivateVars, llvmPrivateVars,
+                                    &mappedPrivateVars),
+                    *targetOp)
+            .failed())
+      return llvm::make_error<PreviouslyReportedError>();
+
     SmallVector<Region *> privateCleanupRegions;
     llvm::transform(privateDecls, std::back_inserter(privateCleanupRegions),
                     [](omp::PrivateClauseOp privatizer) {
                       return &privatizer.getDeallocRegion();
                     });
 
-    builder.restoreIP(codeGenIP);
     llvm::Expected<llvm::BasicBlock *> exitBlock = convertOmpOpRegions(
         targetRegion, "omp.target", builder, moduleTranslation);
 
@@ -4332,6 +4379,8 @@ convertOmpTarget(Operation &opInst, llvm::IRBuilderBase &builder,
                            llvm::Value *&retVal, InsertPointTy allocaIP,
                            InsertPointTy codeGenIP)
       -> llvm::OpenMPIRBuilder::InsertPointOrErrorTy {
+    llvm::IRBuilderBase::InsertPointGuard guard(builder);
+    builder.SetCurrentDebugLocation(llvm::DebugLoc());
     // We just return the unaltered argument for the host function
     // for now, some alterations may be required in the future to
     // keep host fallback functions working identically to the device
diff --git a/mlir/lib/Target/LLVMIR/ModuleImport.cpp b/mlir/lib/Target/LLVMIR/ModuleImport.cpp
index 40d86efe605ad..fd0283b856b6b 100644
--- a/mlir/lib/Target/LLVMIR/ModuleImport.cpp
+++ b/mlir/lib/Target/LLVMIR/ModuleImport.cpp
@@ -139,8 +139,8 @@ static LogicalResult convertInstructionImpl(OpBuilder &odsBuilder,
   if (iface.isConvertibleInstruction(inst->getOpcode()))
     return iface.convertInstruction(odsBuilder, inst, llvmOperands,
                                     moduleImport);
-    // TODO: Implement the `convertInstruction` hooks in the
-    // `LLVMDialectLLVMIRImportInterface` and move the following include there.
+  // TODO: Implement the `convertInstruction` hooks in the
+  // `LLVMDialectLLVMIRImportInterface` and move the following include there.
 #include "mlir/Dialect/LLVMIR/LLVMOpFromLLVMIRConversions.inc"
   return failure();
 }
@@ -647,6 +647,16 @@ LogicalResult ModuleImport::convertGlobals() {
   return success();
 }
 
+LogicalResult ModuleImport::convertAliases() {
+  for (llvm::GlobalAlias &alias : llvmModule->aliases()) {
+    if (failed(convertAlias(&alias))) {
+      return emitError(UnknownLoc::get(context))
+             << "unhandled global alias: " << diag(alias);
+    }
+  }
+  return success();
+}
+
 LogicalResult ModuleImport::convertDataLayout() {
   Location loc = mlirModule.getLoc();
   DataLayoutImporter dataLayoutImporter(context, llvmModule->getDataLayout());
@@ -662,6 +672,11 @@ LogicalResult ModuleImport::convertDataLayout() {
   return success();
 }
 
+void ModuleImport::convertTargetTriple() {
+  mlirModule->setAttr(LLVM::LLVMDialect::getTargetTripleAttrName(),
+                      builder.getStringAttr(llvmModule->getTargetTriple()));
+}
+
 LogicalResult ModuleImport::convertFunctions() {
   for (llvm::Function &func : llvmModule->functions())
     if (failed(processFunction(&func)))
@@ -947,6 +962,38 @@ ModuleImport::getOrCreateNamelessSymbolName(llvm::GlobalVariable *globalVar) {
   return symbolRef;
 }
 
+LogicalResult ModuleImport::convertAlias(llvm::GlobalAlias *alias) {
+  // Insert the global after the last one or at the start of the module.
+  OpBuilder::InsertionGuard guard(builder);
+  if (!aliasInsertionOp)
+    builder.setInsertionPointToStart(mlirModule.getBody());
+  else
+    builder.setInsertionPointAfter(aliasInsertionOp);
+
+  Type type = convertType(alias->getValueType());
+  AliasOp aliasOp = builder.create<AliasOp>(
+      mlirModule.getLoc(), type, convertLinkageFromLLVM(alias->getLinkage()),
+      alias->getName(),
+      /*dso_local=*/alias->isDSOLocal(),
+      /*thread_local=*/alias->isThreadLocal(),
+      /*attrs=*/ArrayRef<NamedAttribute>());
+  aliasInsertionOp = aliasOp;
+
+  clearRegionState();
+  Block *block = builder.createBlock(&aliasOp.getInitializerRegion());
+  setConstantInsertionPointToStart(block);
+  FailureOr<Value> initializer = convertConstantExpr(alias->getAliasee());
+  if (failed(initializer))
+    return failure();
+  builder.create<ReturnOp>(aliasOp.getLoc(), *initializer);
+
+  if (alias->hasAtLeastLocalUnnamedAddr())
+    aliasOp.setUnnamedAddr(convertUnnamedAddrFromLLVM(alias->getUnnamedAddr()));
+  aliasOp.setVisibility_(convertVisibilityFromLLVM(alias->getVisibility()));
+
+  return success();
+}
+
 LogicalResult ModuleImport::convertGlobal(llvm::GlobalVariable *globalVar) {
   // Insert the global after the last one or at the start of the module.
   OpBuilder::InsertionGuard guard(builder);
@@ -1084,7 +1131,7 @@ ModuleImport::getConstantsToConvert(llvm::Constant *constant) {
     llvm::Constant *current = workList.back();
     // References of global objects are just pointers to the object. Avoid
     // walking the elements of these here.
-    if (isa<llvm::GlobalObject>(current)) {
+    if (isa<llvm::GlobalObject>(current) || isa<llvm::GlobalAlias>(current)) {
       orderedSet.insert(current);
       workList.pop_back();
       continue;
@@ -1180,6 +1227,14 @@ FailureOr<Value> ModuleImport::convertConstant(llvm::Constant *constant) {
     return builder.create<AddressOfOp>(loc, type, symbolRef).getResult();
   }
 
+  // Convert global alias accesses.
+  if (auto *globalAliasObj = dyn_cast<llvm::GlobalAlias>(constant)) {
+    Type type = convertType(globalAliasObj->getType());
+    StringRef aliaseeName = globalAliasObj->getName();
+    FlatSymbolRefAttr symbolRef = FlatSymbolRefAttr::get(context, aliaseeName);
+    return builder.create<AddressOfOp>(loc, type, symbolRef).getResult();
+  }
+
   // Convert constant expressions.
   if (auto *constExpr = dyn_cast<llvm::ConstantExpr>(constant)) {
     // Convert the constant expression to a temporary LLVM instruction and
@@ -1489,16 +1544,15 @@ ModuleImport::convertBranchArgs(llvm::Instruction *branch,
   return success();
 }
 
-LogicalResult ModuleImport::convertCallTypeAndOperands(
-    llvm::CallBase *callInst, SmallVectorImpl<Type> &types,
-    SmallVectorImpl<Value> &operands, bool allowInlineAsm) {
-  if (!callInst->getType()->isVoidTy())
-    types.push_back(convertType(callInst->getType()));
-
+FailureOr<SmallVector<Value>>
+ModuleImport::convertCallOperands(llvm::CallBase *callInst,
+                                  bool allowInlineAsm) {
   bool isInlineAsm = callInst->isInlineAsm();
   if (isInlineAsm && !allowInlineAsm)
     return failure();
 
+  SmallVector<Value> operands;
+
   // Cannot use isIndirectCall() here because we need to handle Constant callees
   // that are not considered indirect calls by LLVM. However, in MLIR, they are
   // treated as indirect calls to constant operands that need to be converted.
@@ -1515,10 +1569,81 @@ LogicalResult ModuleImport::convertCallTypeAndOperands(
   FailureOr<SmallVector<Value>> arguments = convertValues(args);
   if (failed(arguments))
     return failure();
+
   llvm::append_range(operands, *arguments);
+  return operands;
+}
+
+/// Checks if `callType` and `calleeType` are compatible and can be represented
+/// in MLIR.
+static LogicalResult
+verifyFunctionTypeCompatibility(LLVMFunctionType callType,
+                                LLVMFunctionType calleeType) {
+  if (callType.getReturnType() != calleeType.getReturnType())
+    return failure();
+
+  if (calleeType.isVarArg()) {
+    // For variadic functions, the call can have more types than the callee
+    // specifies.
+    if (callType.getNumParams() < calleeType.getNumParams())
+      return failure();
+  } else {
+    // For non-variadic functions, the number of parameters needs to be the
+    // same.
+    if (callType.getNumParams() != calleeType.getNumParams())
+      return failure();
+  }
+
+  // Check that all operands match.
+  for (auto [operandType, argumentType] :
+       llvm::zip(callType.getParams(), calleeType.getParams()))
+    if (operandType != argumentType)
+      return failure();
+
   return success();
 }
 
+FailureOr<LLVMFunctionType>
+ModuleImport::convertFunctionType(llvm::CallBase *callInst) {
+  auto castOrFailure = [](Type convertedType) -> FailureOr<LLVMFunctionType> {
+    auto funcTy = dyn_cast_or_null<LLVMFunctionType>(convertedType);
+    if (!funcTy)
+      return failure();
+    return funcTy;
+  };
+
+  llvm::Value *calledOperand = callInst->getCalledOperand();
+  FailureOr<LLVMFunctionType> callType =
+      castOrFailure(convertType(callInst->getFunctionType()));
+  if (failed(callType))
+    return failure();
+  auto *callee = dyn_cast<llvm::Function>(calledOperand);
+  // For indirect calls, return the type of the call itself.
+  if (!callee)
+    return callType;
+
+  FailureOr<LLVMFunctionType> calleeType =
+      castOrFailure(convertType(callee->getFunctionType()));
+  if (failed(calleeType))
+    return failure();
+
+  // Compare the types to avoid constructing illegal call/invoke operations.
+  if (failed(verifyFunctionTypeCompatibility(*callType, *calleeType))) {
+    Location loc = translateLoc(callInst->getDebugLoc());
+    return emitError(loc) << "incompatible call and callee types: " << *callType
+                          << " and " << *calleeType;
+  }
+
+  return calleeType;
+}
+
+FlatSymbolRefAttr ModuleImport::convertCalleeName(llvm::CallBase *callInst) {
+  llvm::Value *calledOperand = callInst->getCalledOperand();
+  if (auto *callee = dyn_cast<llvm::Function>(calledOperand))
+    return SymbolRefAttr::get(context, callee->getName());
+  return {};
+}
+
 LogicalResult ModuleImport::convertIntrinsic(llvm::CallInst *inst) {
   if (succeeded(iface.convertIntrinsic(builder, inst, *this)))
     return success();
@@ -1600,78 +1725,49 @@ LogicalResult ModuleImport::convertInstruction(llvm::Instruction *inst) {
     return success();
   }
   if (inst->getOpcode() == llvm::Instruction::Call) {
-    auto callInst = cast<llvm::CallInst>(inst);
+    auto *callInst = cast<llvm::CallInst>(inst);
     llvm::Value *calledOperand = callInst->getCalledOperand();
 
-    SmallVector<Type> types;
-    SmallVector<Value> operands;
-    if (failed(convertCallTypeAndOperands(callInst, types, operands,
-                                          /*allowInlineAsm=*/true)))
+    FailureOr<SmallVector<Value>> operands =
+        convertCallOperands(callInst, /*allowInlineAsm=*/true);
+    if (failed(operands))
       return failure();
 
-    if (auto asmI = dyn_cast<llvm::InlineAsm>(calledOperand)) {
-      Type resultTy = convertType(callInst->getType());
-      if (!resultTy)
+    auto callOp = [&]() -> FailureOr<Operation *> {
+      if (auto *asmI = dyn_cast<llvm::InlineAsm>(calledOperand)) {
+        Type resultTy = convertType(callInst->getType());
+        if (!resultTy)
+          return failure();
+        return builder
+            .create<InlineAsmOp>(
+                loc, resultTy, *operands,
+                builder.getStringAttr(asmI->getAsmString()),
+                builder.getStringAttr(asmI->getConstraintString()),
+                /*has_side_effects=*/true,
+                /*is_align_stack=*/false, /*asm_dialect=*/nullptr,
+                /*operand_attrs=*/nullptr)
+            .getOperation();
+      }
+      FailureOr<LLVMFunctionType> funcTy = convertFunctionType(callInst);
+      if (failed(funcTy))
         return failure();
-      auto callOp = builder.create<InlineAsmOp>(
-          loc, resultTy, operands, builder.getStringAttr(asmI->getAsmString()),
-          builder.getStringAttr(asmI->getConstraintString()),
-          /*has_side_effects=*/true,
-          /*is_align_stack=*/false, /*asm_dialect=*/nullptr,
-          /*operand_attrs=*/nullptr);
-      if (!callInst->getType()->isVoidTy())
-        mapValue(inst, callOp.getResult(0));
-      else
-        mapNoResultOp(inst, callOp);
-    } else {
-      auto funcTy = dyn_cast<LLVMFunctionType>([&]() -> Type {
-        // Retrieve the real function type. For direct calls, use the callee's
-        // function type, as it may differ from the operand type in the case of
-        // variadic functions. For indirect calls, use the call function type.
-        if (auto callee = dyn_cast<llvm::Function>(calledOperand))
-          return convertType(callee->getFunctionType());
-        return convertType(callInst->getFunctionType());
-      }());
-
-      if (!funcTy)
+
+      FlatSymbolRefAttr callee = convertCalleeName(callInst);
+      auto callOp = builder.create<CallOp>(loc, *funcTy, callee, *operands);
+      if (failed(convertCallAttributes(callInst, callOp)))
         return failure();
+      // Handle parameter and result attributes.
+      convertParameterAttributes(callInst, callOp, builder);
+      return callOp.getOperation();
+    }();
 
-      auto callOp = [&]() -> CallOp {
-        if (auto callee = dyn_cast<llvm::Function>(calledOperand)) {
-          auto name = SymbolRefAttr::get(context, callee->getName());
-          return builder.create<CallOp>(loc, funcTy, name, operands);
-        }
-        return builder.create<CallOp>(loc, funcTy, operands);
-      }();
-
-      // Handle function attributes.
-      callOp.setCConv(convertCConvFromLLVM(callInst->getCallingConv()));
-      callOp.setTailCallKind(
-          convertTailCallKindFromLLVM(callInst->getTailCallKind()));
-      setFastmathFlagsAttr(inst, callOp);
-
-      callOp.setConvergent(callInst->isConvergent());
-      callOp.setNoUnwind(callInst->doesNotThrow());
-      callOp.setWillReturn(callInst->hasFnAttr(llvm::Attribute::WillReturn));
-
-      llvm::MemoryEffects memEffects = callInst->getMemoryEffects();
-      ModRefInfo othermem = convertModRefInfoFromLLVM(
-          memEffects.getModRef(llvm::MemoryEffects::Location::Other));
-      ModRefInfo argMem = convertModRefInfoFromLLVM(
-          memEffects.getModRef(llvm::MemoryEffects::Location::ArgMem));
-      ModRefInfo inaccessibleMem = convertModRefInfoFromLLVM(
-          memEffects.getModRef(llvm::MemoryEffects::Location::InaccessibleMem));
-      auto memAttr = MemoryEffectsAttr::get(callOp.getContext(), othermem,
-                                            argMem, inaccessibleMem);
-      // Only set the attribute when it does not match the default value.
-      if (!memAttr.isReadWrite())
-        callOp.setMemoryEffectsAttr(memAttr);
-
-      if (!callInst->getType()->isVoidTy())
-        mapValue(inst, callOp.getResult());
-      else
-        mapNoResultOp(inst, callOp);
-    }
+    if (failed(callOp))
+      return failure();
+
+    if (!callInst->getType()->isVoidTy())
+      mapValue(inst, (*callOp)->getResult(0));
+    else
+      mapNoResultOp(inst, *callOp);
     return success();
   }
   if (inst->getOpcode() == llvm::Instruction::LandingPad) {
@@ -1695,9 +1791,11 @@ LogicalResult ModuleImport::convertInstruction(llvm::Instruction *inst) {
   if (inst->getOpcode() == llvm::Instruction::Invoke) {
     auto *invokeInst = cast<llvm::InvokeInst>(inst);
 
-    SmallVector<Type> types;
-    SmallVector<Value> operands;
-    if (failed(convertCallTypeAndOperands(invokeInst, types, operands)))
+    if (invokeInst->isInlineAsm())
+      return emitError(loc) << "invoke of inline assembly is not supported";
+
+    FailureOr<SmallVector<Value>> operands = convertCallOperands(invokeInst);
+    if (failed(operands))
       return failure();
 
     // Check whether the invoke result is an argument to the normal destination
@@ -1724,27 +1822,25 @@ LogicalResult ModuleImport::convertInstruction(llvm::Instruction *inst) {
                                  unwindArgs)))
       return failure();
 
-    auto funcTy =
-        dyn_cast<LLVMFunctionType>(convertType(invokeInst->getFunctionType()));
-    if (!funcTy)
+    FailureOr<LLVMFunctionType> funcTy = convertFunctionType(invokeInst);
+    if (failed(funcTy))
       return failure();
 
+    FlatSymbolRefAttr calleeName = convertCalleeName(invokeInst);
+
     // Create the invoke operation. Normal destination block arguments will be
     // added later on to handle the case in which the operation result is
     // included in this list.
-    InvokeOp invokeOp;
-    if (llvm::Function *callee = invokeInst->getCalledFunction()) {
-      invokeOp = builder.create<InvokeOp>(
-          loc, funcTy,
-          SymbolRefAttr::get(builder.getContext(), callee->getName()), operands,
-          directNormalDest, ValueRange(),
-          lookupBlock(invokeInst->getUnwindDest()), unwindArgs);
-    } else {
-      invokeOp = builder.create<InvokeOp>(
-          loc, funcTy, /*callee=*/nullptr, operands, directNormalDest,
-          ValueRange(), lookupBlock(invokeInst->getUnwindDest()), unwindArgs);
-    }
-    invokeOp.setCConv(convertCConvFromLLVM(invokeInst->getCallingConv()));
+    auto invokeOp = builder.create<InvokeOp>(
+        loc, *funcTy, calleeName, *operands, directNormalDest, ValueRange(),
+        lookupBlock(invokeInst->getUnwindDest()), unwindArgs);
+
+    if (failed(convertInvokeAttributes(invokeInst, invokeOp)))
+      return failure();
+
+    // Handle parameter and result attributes.
+    convertParameterAttributes(invokeInst, invokeOp, builder);
+
     if (!invokeInst->getType()->isVoidTy())
       mapValue(inst, invokeOp.getResults().front());
     else
@@ -2061,6 +2157,17 @@ ModuleImport::convertParameterAttribute(llvm::AttributeSet llvmParamAttrs,
     // Skip attributes that are not attached.
     if (!llvmAttr.isValid())
       continue;
+
+    // TODO: Import captures(none) as a nocapture unit attribute until the
+    // LLVM dialect switches to the captures representation.
+    if (llvmAttr.hasKindAsEnum() &&
+        llvmAttr.getKindAsEnum() == llvm::Attribute::Captures) {
+      if (llvm::capturesNothing(llvmAttr.getCaptureInfo()))
+        paramAttrs.push_back(
+            builder.getNamedAttr(mlirName, builder.getUnitAttr()));
+      continue;
+    }
+
     Attribute mlirAttr;
     if (llvmAttr.isTypeAttribute())
       mlirAttr = TypeAttr::get(convertType(llvmAttr.getValueAsType()));
@@ -2097,6 +2204,72 @@ void ModuleImport::convertParameterAttributes(llvm::Function *func,
       builder.getArrayAttr(convertParameterAttribute(llvmResAttr, builder)));
 }
 
+void ModuleImport::convertParameterAttributes(llvm::CallBase *call,
+                                              CallOpInterface callOp,
+                                              OpBuilder &builder) {
+  llvm::AttributeList llvmAttrs = call->getAttributes();
+  SmallVector<llvm::AttributeSet> llvmArgAttrsSet;
+  bool anyArgAttrs = false;
+  for (size_t i = 0, e = call->arg_size(); i < e; ++i) {
+    llvmArgAttrsSet.emplace_back(llvmAttrs.getParamAttrs(i));
+    if (llvmArgAttrsSet.back().hasAttributes())
+      anyArgAttrs = true;
+  }
+  auto getArrayAttr = [&](ArrayRef<DictionaryAttr> dictAttrs) {
+    SmallVector<Attribute> attrs;
+    for (auto &dict : dictAttrs)
+      attrs.push_back(dict ? dict : builder.getDictionaryAttr({}));
+    return builder.getArrayAttr(attrs);
+  };
+  if (anyArgAttrs) {
+    SmallVector<DictionaryAttr> argAttrs;
+    for (auto &llvmArgAttrs : llvmArgAttrsSet)
+      argAttrs.emplace_back(convertParameterAttribute(llvmArgAttrs, builder));
+    callOp.setArgAttrsAttr(getArrayAttr(argAttrs));
+  }
+
+  llvm::AttributeSet llvmResAttr = llvmAttrs.getRetAttrs();
+  if (!llvmResAttr.hasAttributes())
+    return;
+  DictionaryAttr resAttrs = convertParameterAttribute(llvmResAttr, builder);
+  callOp.setResAttrsAttr(getArrayAttr({resAttrs}));
+}
+
+template <typename Op>
+static LogicalResult convertCallBaseAttributes(llvm::CallBase *inst, Op op) {
+  op.setCConv(convertCConvFromLLVM(inst->getCallingConv()));
+  return success();
+}
+
+LogicalResult ModuleImport::convertInvokeAttributes(llvm::InvokeInst *inst,
+                                                    InvokeOp op) {
+  return convertCallBaseAttributes(inst, op);
+}
+
+LogicalResult ModuleImport::convertCallAttributes(llvm::CallInst *inst,
+                                                  CallOp op) {
+  setFastmathFlagsAttr(inst, op.getOperation());
+  op.setTailCallKind(convertTailCallKindFromLLVM(inst->getTailCallKind()));
+  op.setConvergent(inst->isConvergent());
+  op.setNoUnwind(inst->doesNotThrow());
+  op.setWillReturn(inst->hasFnAttr(llvm::Attribute::WillReturn));
+
+  llvm::MemoryEffects memEffects = inst->getMemoryEffects();
+  ModRefInfo othermem = convertModRefInfoFromLLVM(
+      memEffects.getModRef(llvm::MemoryEffects::Location::Other));
+  ModRefInfo argMem = convertModRefInfoFromLLVM(
+      memEffects.getModRef(llvm::MemoryEffects::Location::ArgMem));
+  ModRefInfo inaccessibleMem = convertModRefInfoFromLLVM(
+      memEffects.getModRef(llvm::MemoryEffects::Location::InaccessibleMem));
+  auto memAttr = MemoryEffectsAttr::get(op.getContext(), othermem, argMem,
+                                        inaccessibleMem);
+  // Only set the attribute when it does not match the default value.
+  if (!memAttr.isReadWrite())
+    op.setMemoryEffectsAttr(memAttr);
+
+  return convertCallBaseAttributes(inst, op);
+}
+
 LogicalResult ModuleImport::processFunction(llvm::Function *func) {
   clearRegionState();
 
@@ -2106,7 +2279,7 @@ LogicalResult ModuleImport::processFunction(llvm::Function *func) {
       iface.isConvertibleIntrinsic(func->getIntrinsicID()))
     return success();
 
-  bool dsoLocal = func->hasLocalLinkage();
+  bool dsoLocal = func->isDSOLocal();
   CConv cconv = convertCConvFromLLVM(func->getCallingConv());
 
   // Insert the function at the end of the module.
@@ -2369,6 +2542,8 @@ mlir::translateLLVMIRToModule(std::unique_ptr<llvm::Module> llvmModule,
     return {};
   if (failed(moduleImport.convertFunctions()))
     return {};
-
+  if (failed(moduleImport.convertAliases()))
+    return {};
+  moduleImport.convertTargetTriple();
   return module;
 }
diff --git a/mlir/lib/Target/LLVMIR/ModuleTranslation.cpp b/mlir/lib/Target/LLVMIR/ModuleTranslation.cpp
index 4367100e3aca6..3da47de6ac24b 100644
--- a/mlir/lib/Target/LLVMIR/ModuleTranslation.cpp
+++ b/mlir/lib/Target/LLVMIR/ModuleTranslation.cpp
@@ -746,6 +746,8 @@ void ModuleTranslation::forgetMapping(Region &region) {
           branchMapping.erase(&op);
         if (isa<LLVM::GlobalOp>(op))
           globalsMapping.erase(&op);
+        if (isa<LLVM::AliasOp>(op))
+          aliasesMapping.erase(&op);
         if (isa<LLVM::CallOp>(op))
           callMapping.erase(&op);
         llvm::append_range(
@@ -1026,12 +1028,15 @@ static void addRuntimePreemptionSpecifier(bool dsoLocalRequested,
     gv->setDSOLocal(true);
 }
 
-/// Create named global variables that correspond to llvm.mlir.global
-/// definitions. Convert llvm.global_ctors and global_dtors ops.
-LogicalResult ModuleTranslation::convertGlobals() {
+LogicalResult ModuleTranslation::convertGlobalsAndAliases() {
   // Mapping from compile unit to its respective set of global variables.
   DenseMap<llvm::DICompileUnit *, SmallVector<llvm::Metadata *>> allGVars;
 
+  // First, create all global variables and global aliases in LLVM IR. A global
+  // or alias body may refer to another global/alias or itself, so all the
+  // mapping needs to happen prior to body conversion.
+
+  // Create all llvm::GlobalVariable
   for (auto op : getModuleBody(mlirModule).getOps<LLVM::GlobalOp>()) {
     llvm::Type *type = convertType(op.getType());
     llvm::Constant *cst = nullptr;
@@ -1135,9 +1140,35 @@ LogicalResult ModuleTranslation::convertGlobals() {
     }
   }
 
-  // Convert global variable bodies. This is done after all global variables
-  // have been created in LLVM IR because a global body may refer to another
-  // global or itself. So all global variables need to be mapped first.
+  // Create all llvm::GlobalAlias
+  for (auto op : getModuleBody(mlirModule).getOps<LLVM::AliasOp>()) {
+    llvm::Type *type = convertType(op.getType());
+    llvm::Constant *cst = nullptr;
+    llvm::GlobalValue::LinkageTypes linkage =
+        convertLinkageToLLVM(op.getLinkage());
+    llvm::Module &llvmMod = *llvmModule;
+
+    // Note address space and aliasee info isn't set just yet.
+    llvm::GlobalAlias *var = llvm::GlobalAlias::create(
+        type, op.getAddrSpace(), linkage, op.getSymName(), /*placeholder*/ cst,
+        &llvmMod);
+
+    var->setThreadLocalMode(op.getThreadLocal_()
+                                ? llvm::GlobalAlias::GeneralDynamicTLSModel
+                                : llvm::GlobalAlias::NotThreadLocal);
+
+    // Note there is no need to setup the comdat because GlobalAlias calls into
+    // the aliasee comdat information automatically.
+
+    if (op.getUnnamedAddr().has_value())
+      var->setUnnamedAddr(convertUnnamedAddrToLLVM(*op.getUnnamedAddr()));
+
+    var->setVisibility(convertVisibilityToLLVM(op.getVisibility_()));
+
+    aliasesMapping.try_emplace(op, var);
+  }
+
+  // Convert global variable bodies.
   for (auto op : getModuleBody(mlirModule).getOps<LLVM::GlobalOp>()) {
     if (Block *initializer = op.getInitializerBlock()) {
       llvm::IRBuilder<> builder(llvmModule->getContext());
@@ -1248,6 +1279,29 @@ LogicalResult ModuleTranslation::convertGlobals() {
         llvm::MDTuple::get(getLLVMContext(), globals));
   }
 
+  // Convert global alias bodies.
+  for (auto op : getModuleBody(mlirModule).getOps<LLVM::AliasOp>()) {
+    Block &initializer = op.getInitializerBlock();
+    llvm::IRBuilder<> builder(llvmModule->getContext());
+
+    for (mlir::Operation &op : initializer.without_terminator()) {
+      if (failed(convertOperation(op, builder)))
+        return emitError(op.getLoc(), "fail to convert alias initializer");
+      if (!isa<llvm::Constant>(lookupValue(op.getResult(0))))
+        return emitError(op.getLoc(), "unemittable constant value");
+    }
+
+    auto ret = cast<ReturnOp>(initializer.getTerminator());
+    auto *cst = cast<llvm::Constant>(lookupValue(ret.getOperand(0)));
+    assert(aliasesMapping.count(op));
+    auto *alias = cast<llvm::GlobalAlias>(aliasesMapping[op]);
+    alias->setAliasee(cst);
+  }
+
+  for (auto op : getModuleBody(mlirModule).getOps<LLVM::AliasOp>())
+    if (failed(convertDialectAttributes(op, {})))
+      return failure();
+
   return success();
 }
 
@@ -1563,30 +1617,50 @@ static void convertFunctionKernelAttributes(LLVMFuncOp func,
   }
 }
 
+static LogicalResult convertParameterAttr(llvm::AttrBuilder &attrBuilder,
+                                          llvm::Attribute::AttrKind llvmKind,
+                                          NamedAttribute namedAttr,
+                                          ModuleTranslation &moduleTranslation,
+                                          Location loc) {
+  return llvm::TypeSwitch<Attribute, LogicalResult>(namedAttr.getValue())
+      .Case<TypeAttr>([&](auto typeAttr) {
+        attrBuilder.addTypeAttr(
+            llvmKind, moduleTranslation.convertType(typeAttr.getValue()));
+        return success();
+      })
+      .Case<IntegerAttr>([&](auto intAttr) {
+        attrBuilder.addRawIntAttr(llvmKind, intAttr.getInt());
+        return success();
+      })
+      .Case<UnitAttr>([&](auto) {
+        attrBuilder.addAttribute(llvmKind);
+        return success();
+      })
+      .Case<LLVM::ConstantRangeAttr>([&](auto rangeAttr) {
+        attrBuilder.addConstantRangeAttr(
+            llvmKind,
+            llvm::ConstantRange(rangeAttr.getLower(), rangeAttr.getUpper()));
+        return success();
+      })
+      .Default([loc](auto) {
+        return emitError(loc, "unsupported parameter attribute type");
+      });
+}
+
 FailureOr<llvm::AttrBuilder>
 ModuleTranslation::convertParameterAttrs(LLVMFuncOp func, int argIdx,
                                          DictionaryAttr paramAttrs) {
   llvm::AttrBuilder attrBuilder(llvmModule->getContext());
   auto attrNameToKindMapping = getAttrNameToKindMapping();
+  Location loc = func.getLoc();
 
   for (auto namedAttr : paramAttrs) {
     auto it = attrNameToKindMapping.find(namedAttr.getName());
     if (it != attrNameToKindMapping.end()) {
       llvm::Attribute::AttrKind llvmKind = it->second;
-
-      llvm::TypeSwitch<Attribute>(namedAttr.getValue())
-          .Case<TypeAttr>([&](auto typeAttr) {
-            attrBuilder.addTypeAttr(llvmKind, convertType(typeAttr.getValue()));
-          })
-          .Case<IntegerAttr>([&](auto intAttr) {
-            attrBuilder.addRawIntAttr(llvmKind, intAttr.getInt());
-          })
-          .Case<UnitAttr>([&](auto) { attrBuilder.addAttribute(llvmKind); })
-          .Case<LLVM::ConstantRangeAttr>([&](auto rangeAttr) {
-            attrBuilder.addConstantRangeAttr(
-                llvmKind, llvm::ConstantRange(rangeAttr.getLower(),
-                                              rangeAttr.getUpper()));
-          });
+      if (failed(convertParameterAttr(attrBuilder, llvmKind, namedAttr, *this,
+                                      loc)))
+        return failure();
     } else if (namedAttr.getNameDialect()) {
       if (failed(iface.convertParameterAttr(func, argIdx, namedAttr, *this)))
         return failure();
@@ -1596,6 +1670,26 @@ ModuleTranslation::convertParameterAttrs(LLVMFuncOp func, int argIdx,
   return attrBuilder;
 }
 
+FailureOr<llvm::AttrBuilder>
+ModuleTranslation::convertParameterAttrs(CallOpInterface callOp,
+                                         DictionaryAttr paramAttrs) {
+  llvm::AttrBuilder attrBuilder(llvmModule->getContext());
+  Location loc = callOp.getLoc();
+  auto attrNameToKindMapping = getAttrNameToKindMapping();
+
+  for (auto namedAttr : paramAttrs) {
+    auto it = attrNameToKindMapping.find(namedAttr.getName());
+    if (it != attrNameToKindMapping.end()) {
+      llvm::Attribute::AttrKind llvmKind = it->second;
+      if (failed(convertParameterAttr(attrBuilder, llvmKind, namedAttr, *this,
+                                      loc)))
+        return failure();
+    }
+  }
+
+  return attrBuilder;
+}
+
 LogicalResult ModuleTranslation::convertFunctionSignatures() {
   // Declare all functions first because there may be function calls that form a
   // call graph with cycles, or global initializers that reference functions.
@@ -2055,7 +2149,7 @@ mlir::translateModuleToLLVMIR(Operation *module, llvm::LLVMContext &llvmContext,
     return nullptr;
   if (failed(translator.convertFunctionSignatures()))
     return nullptr;
-  if (failed(translator.convertGlobals()))
+  if (failed(translator.convertGlobalsAndAliases()))
     return nullptr;
   if (failed(translator.createTBAAMetadata()))
     return nullptr;
@@ -2066,8 +2160,8 @@ mlir::translateModuleToLLVMIR(Operation *module, llvm::LLVMContext &llvmContext,
 
   // Convert other top-level operations if possible.
   for (Operation &o : getModuleBody(module).getOperations()) {
-    if (!isa<LLVM::LLVMFuncOp, LLVM::GlobalOp, LLVM::GlobalCtorsOp,
-             LLVM::GlobalDtorsOp, LLVM::ComdatOp>(&o) &&
+    if (!isa<LLVM::LLVMFuncOp, LLVM::AliasOp, LLVM::GlobalOp,
+             LLVM::GlobalCtorsOp, LLVM::GlobalDtorsOp, LLVM::ComdatOp>(&o) &&
         !o.hasTrait<OpTrait::IsTerminator>() &&
         failed(translator.convertOperation(o, llvmBuilder))) {
       return nullptr;
diff --git a/mlir/lib/Transforms/CMakeLists.txt b/mlir/lib/Transforms/CMakeLists.txt
index 058039e47313e..3a8088bccf299 100644
--- a/mlir/lib/Transforms/CMakeLists.txt
+++ b/mlir/lib/Transforms/CMakeLists.txt
@@ -37,4 +37,5 @@ add_mlir_library(MLIRTransforms
   MLIRSideEffectInterfaces
   MLIRSupport
   MLIRTransformUtils
+  MLIRUBDialect
   )
diff --git a/mlir/lib/Transforms/Canonicalizer.cpp b/mlir/lib/Transforms/Canonicalizer.cpp
index 5f46960507036..7ccd503fb0288 100644
--- a/mlir/lib/Transforms/Canonicalizer.cpp
+++ b/mlir/lib/Transforms/Canonicalizer.cpp
@@ -13,6 +13,7 @@
 
 #include "mlir/Transforms/Passes.h"
 
+#include "mlir/Dialect/UB/IR/UBOps.h"
 #include "mlir/Pass/Pass.h"
 #include "mlir/Transforms/GreedyPatternRewriteDriver.h"
 
diff --git a/mlir/lib/Transforms/RemoveDeadValues.cpp b/mlir/lib/Transforms/RemoveDeadValues.cpp
index 3e7a0cca31c77..c20c54551cdf8 100644
--- a/mlir/lib/Transforms/RemoveDeadValues.cpp
+++ b/mlir/lib/Transforms/RemoveDeadValues.cpp
@@ -375,6 +375,8 @@ static void processRegionBranchOp(RegionBranchOpInterface regionBranchOp,
   // Mark live arguments in the regions of `regionBranchOp` in `liveArgs`.
   auto markLiveArgs = [&](DenseMap<Region *, BitVector> &liveArgs) {
     for (Region &region : regionBranchOp->getRegions()) {
+      if (region.empty())
+        continue;
       SmallVector<Value> arguments(region.front().getArguments());
       BitVector regionLiveArgs = markLives(arguments, nonLiveSet, la);
       liveArgs[&region] = regionLiveArgs;
@@ -420,6 +422,8 @@ static void processRegionBranchOp(RegionBranchOpInterface regionBranchOp,
   auto markNonForwardedReturnValues =
       [&](DenseMap<Operation *, BitVector> &nonForwardedRets) {
         for (Region &region : regionBranchOp->getRegions()) {
+          if (region.empty())
+            continue;
           Operation *terminator = region.front().getTerminator();
           nonForwardedRets[terminator] =
               BitVector(terminator->getNumOperands(), true);
@@ -499,6 +503,8 @@ static void processRegionBranchOp(RegionBranchOpInterface regionBranchOp,
         // Recompute `resultsToKeep` and `argsToKeep` based on
         // `terminatorOperandsToKeep`.
         for (Region &region : regionBranchOp->getRegions()) {
+          if (region.empty())
+            continue;
           Operation *terminator = region.front().getTerminator();
           for (const RegionSuccessor &successor : getSuccessors(&region)) {
             Region *successorRegion = successor.getSuccessor();
@@ -547,6 +553,8 @@ static void processRegionBranchOp(RegionBranchOpInterface regionBranchOp,
 
           // Update the terminator operands that need to be kept.
           for (Region &region : regionBranchOp->getRegions()) {
+            if (region.empty())
+              continue;
             updateOperandsOrTerminatorOperandsToKeep(
                 terminatorOperandsToKeep[region.back().getTerminator()],
                 resultsToKeep, argsToKeep, &region);
@@ -611,8 +619,8 @@ static void processRegionBranchOp(RegionBranchOpInterface regionBranchOp,
 
   // Do (2.a) and (2.b).
   for (Region &region : regionBranchOp->getRegions()) {
-    assert(!region.empty() && "expected a non-empty region in an op "
-                              "implementing `RegionBranchOpInterface`");
+    if (region.empty())
+      continue;
     BitVector argsToRemove = argsToKeep[&region].flip();
     cl.blocks.push_back({&region.front(), argsToRemove});
     collectNonLiveValues(nonLiveSet, region.front().getArguments(),
@@ -621,6 +629,8 @@ static void processRegionBranchOp(RegionBranchOpInterface regionBranchOp,
 
   // Do (2.c).
   for (Region &region : regionBranchOp->getRegions()) {
+    if (region.empty())
+      continue;
     Operation *terminator = region.front().getTerminator();
     cl.operands.push_back(
         {terminator, terminatorOperandsToKeep[terminator].flip()});
diff --git a/mlir/lib/Transforms/Utils/DialectConversion.cpp b/mlir/lib/Transforms/Utils/DialectConversion.cpp
index 403321d40d53c..7bfcd192c9aa9 100644
--- a/mlir/lib/Transforms/Utils/DialectConversion.cpp
+++ b/mlir/lib/Transforms/Utils/DialectConversion.cpp
@@ -1708,10 +1708,13 @@ FailureOr<Block *> ConversionPatternRewriter::convertRegionTypes(
 void ConversionPatternRewriter::replaceUsesOfBlockArgument(BlockArgument from,
                                                            Value to) {
   LLVM_DEBUG({
-    Operation *parentOp = from.getOwner()->getParentOp();
-    impl->logger.startLine() << "** Replace Argument : '" << from
-                             << "'(in region of '" << parentOp->getName()
-                             << "'(" << from.getOwner()->getParentOp() << ")\n";
+    impl->logger.startLine() << "** Replace Argument : '" << from << "'";
+    if (Operation *parentOp = from.getOwner()->getParentOp()) {
+      impl->logger.getOStream() << " (in region of '" << parentOp->getName()
+                                << "' (" << parentOp << ")\n";
+    } else {
+      impl->logger.getOStream() << " (unlinked block)\n";
+    }
   });
   impl->appendRewrite<ReplaceBlockArgRewrite>(from.getOwner(), from,
                                               impl->currentTypeConverter);
diff --git a/mlir/lib/Transforms/ViewOpGraph.cpp b/mlir/lib/Transforms/ViewOpGraph.cpp
index fa0af7665ba4c..75ee3ed74db5e 100644
--- a/mlir/lib/Transforms/ViewOpGraph.cpp
+++ b/mlir/lib/Transforms/ViewOpGraph.cpp
@@ -14,6 +14,7 @@
 #include "mlir/IR/Operation.h"
 #include "mlir/Pass/Pass.h"
 #include "mlir/Support/IndentedOstream.h"
+#include "llvm/ADT/STLExtras.h"
 #include "llvm/Support/Format.h"
 #include "llvm/Support/GraphWriter.h"
 #include <map>
@@ -29,7 +30,7 @@ using namespace mlir;
 
 static const StringRef kLineStyleControlFlow = "dashed";
 static const StringRef kLineStyleDataFlow = "solid";
-static const StringRef kShapeNode = "ellipse";
+static const StringRef kShapeNode = "Mrecord";
 static const StringRef kShapeNone = "plain";
 
 /// Return the size limits for eliding large attributes.
@@ -49,16 +50,25 @@ static std::string strFromOs(function_ref<void(raw_ostream &)> func) {
   return buf;
 }
 
-/// Escape special characters such as '\n' and quotation marks.
-static std::string escapeString(std::string str) {
-  return strFromOs([&](raw_ostream &os) { os.write_escaped(str); });
-}
-
 /// Put quotation marks around a given string.
 static std::string quoteString(const std::string &str) {
   return "\"" + str + "\"";
 }
 
+/// For Graphviz record nodes:
+/// " Braces, vertical bars and angle brackets must be escaped with a backslash
+/// character if you wish them to appear as a literal character "
+std::string escapeLabelString(const std::string &str) {
+  std::string buf;
+  llvm::raw_string_ostream os(buf);
+  for (char c : str) {
+    if (llvm::is_contained({'{', '|', '<', '}', '>', '\n', '"'}, c))
+      os << '\\';
+    os << c;
+  }
+  return buf;
+}
+
 using AttributeMap = std::map<std::string, std::string>;
 
 namespace {
@@ -79,6 +89,12 @@ struct Node {
   std::optional<int> clusterId;
 };
 
+struct DataFlowEdge {
+  Value value;
+  Node node;
+  std::string port;
+};
+
 /// This pass generates a Graphviz dataflow visualization of an MLIR operation.
 /// Note: See https://www.graphviz.org/doc/info/lang.html for more information
 /// about the Graphviz DOT language.
@@ -107,7 +123,7 @@ class PrintOpPass : public impl::ViewOpGraphBase<PrintOpPass> {
 private:
   /// Generate a color mapping that will color every operation with the same
   /// name the same way. It'll interpolate the hue in the HSV color-space,
-  /// attempting to keep the contrast suitable for black text.
+  /// using muted colors that provide good contrast for black text.
   template <typename T>
   void initColorMapping(T &irEntity) {
     backgroundColors.clear();
@@ -120,8 +136,10 @@ class PrintOpPass : public impl::ViewOpGraphBase<PrintOpPass> {
     });
     for (auto indexedOps : llvm::enumerate(ops)) {
       double hue = ((double)indexedOps.index()) / ops.size();
+      // Use lower saturation (0.3) and higher value (0.95) for better
+      // readability
       backgroundColors[indexedOps.value()->getName()].second =
-          std::to_string(hue) + " 1.0 1.0";
+          std::to_string(hue) + " 0.3 0.95";
     }
   }
 
@@ -129,8 +147,8 @@ class PrintOpPass : public impl::ViewOpGraphBase<PrintOpPass> {
   /// emitted.
   void emitAllEdgeStmts() {
     if (printDataFlowEdges) {
-      for (const auto &[value, node, label] : dataFlowEdges) {
-        emitEdgeStmt(valueToNode[value], node, label, kLineStyleDataFlow);
+      for (const auto &e : dataFlowEdges) {
+        emitEdgeStmt(valueToNode[e.value], e.node, e.port, kLineStyleDataFlow);
       }
     }
 
@@ -147,8 +165,7 @@ class PrintOpPass : public impl::ViewOpGraphBase<PrintOpPass> {
     os.indent();
     // Emit invisible anchor node from/to which arrows can be drawn.
     Node anchorNode = emitNodeStmt(" ", kShapeNone);
-    os << attrStmt("label", quoteString(escapeString(std::move(label))))
-       << ";\n";
+    os << attrStmt("label", quoteString(label)) << ";\n";
     builder();
     os.unindent();
     os << "}\n";
@@ -176,7 +193,8 @@ class PrintOpPass : public impl::ViewOpGraphBase<PrintOpPass> {
 
     // Always emit splat attributes.
     if (isa<SplatElementsAttr>(attr)) {
-      attr.print(os);
+      os << escapeLabelString(
+          strFromOs([&](raw_ostream &os) { attr.print(os); }));
       return;
     }
 
@@ -184,8 +202,8 @@ class PrintOpPass : public impl::ViewOpGraphBase<PrintOpPass> {
     auto elements = dyn_cast<ElementsAttr>(attr);
     if (elements && elements.getNumElements() > largeAttrLimit) {
       os << std::string(elements.getShapedType().getRank(), '[') << "..."
-         << std::string(elements.getShapedType().getRank(), ']') << " : "
-         << elements.getType();
+         << std::string(elements.getShapedType().getRank(), ']') << " : ";
+      emitMlirType(os, elements.getType());
       return;
     }
 
@@ -199,19 +217,27 @@ class PrintOpPass : public impl::ViewOpGraphBase<PrintOpPass> {
     std::string buf;
     llvm::raw_string_ostream ss(buf);
     attr.print(ss);
-    os << truncateString(buf);
+    os << escapeLabelString(truncateString(buf));
+  }
+
+  // Print a truncated and escaped MLIR type to `os`.
+  void emitMlirType(raw_ostream &os, Type type) {
+    std::string buf;
+    llvm::raw_string_ostream ss(buf);
+    type.print(ss);
+    os << escapeLabelString(truncateString(buf));
+  }
+
+  // Print a truncated and escaped MLIR operand to `os`.
+  void emitMlirOperand(raw_ostream &os, Value operand) {
+    operand.printAsOperand(os, OpPrintingFlags());
   }
 
   /// Append an edge to the list of edges.
   /// Note: Edges are written to the output stream via `emitAllEdgeStmts`.
-  void emitEdgeStmt(Node n1, Node n2, std::string label, StringRef style) {
+  void emitEdgeStmt(Node n1, Node n2, std::string port, StringRef style) {
     AttributeMap attrs;
     attrs["style"] = style.str();
-    // Do not label edges that start/end at a cluster boundary. Such edges are
-    // clipped at the boundary, but labels are not. This can lead to labels
-    // floating around without any edge next to them.
-    if (!n1.clusterId && !n2.clusterId)
-      attrs["label"] = quoteString(escapeString(std::move(label)));
     // Use `ltail` and `lhead` to draw edges between clusters.
     if (n1.clusterId)
       attrs["ltail"] = "cluster_" + std::to_string(*n1.clusterId);
@@ -219,7 +245,15 @@ class PrintOpPass : public impl::ViewOpGraphBase<PrintOpPass> {
       attrs["lhead"] = "cluster_" + std::to_string(*n2.clusterId);
 
     edges.push_back(strFromOs([&](raw_ostream &os) {
-      os << llvm::format("v%i -> v%i ", n1.id, n2.id);
+      os << "v" << n1.id;
+      if (!port.empty() && !n1.clusterId)
+        // Attach edge to south compass point of the result
+        os << ":res" << port << ":s";
+      os << " -> ";
+      os << "v" << n2.id;
+      if (!port.empty() && !n2.clusterId)
+        // Attach edge to north compass point of the operand
+        os << ":arg" << port << ":n";
       emitAttrList(os, attrs);
     }));
   }
@@ -240,11 +274,11 @@ class PrintOpPass : public impl::ViewOpGraphBase<PrintOpPass> {
                     StringRef background = "") {
     int nodeId = ++counter;
     AttributeMap attrs;
-    attrs["label"] = quoteString(escapeString(std::move(label)));
+    attrs["label"] = quoteString(label);
     attrs["shape"] = shape.str();
     if (!background.empty()) {
       attrs["style"] = "filled";
-      attrs["fillcolor"] = ("\"" + background + "\"").str();
+      attrs["fillcolor"] = quoteString(background.str());
     }
     os << llvm::format("v%i ", nodeId);
     emitAttrList(os, attrs);
@@ -252,8 +286,18 @@ class PrintOpPass : public impl::ViewOpGraphBase<PrintOpPass> {
     return Node(nodeId);
   }
 
-  /// Generate a label for an operation.
-  std::string getLabel(Operation *op) {
+  std::string getValuePortName(Value operand) {
+    // Print value as an operand and omit the leading '%' character.
+    auto str = strFromOs([&](raw_ostream &os) {
+      operand.printAsOperand(os, OpPrintingFlags());
+    });
+    // Replace % and # with _
+    std::replace(str.begin(), str.end(), '%', '_');
+    std::replace(str.begin(), str.end(), '#', '_');
+    return str;
+  }
+
+  std::string getClusterLabel(Operation *op) {
     return strFromOs([&](raw_ostream &os) {
       // Print operation name and type.
       os << op->getName();
@@ -267,18 +311,73 @@ class PrintOpPass : public impl::ViewOpGraphBase<PrintOpPass> {
 
       // Print attributes.
       if (printAttrs) {
-        os << "\n";
+        os << "\\l";
+        for (const NamedAttribute &attr : op->getAttrs()) {
+          os << escapeLabelString(attr.getName().getValue().str()) << ": ";
+          emitMlirAttr(os, attr.getValue());
+          os << "\\l";
+        }
+      }
+    });
+  }
+
+  /// Generate a label for an operation.
+  std::string getRecordLabel(Operation *op) {
+    return strFromOs([&](raw_ostream &os) {
+      os << "{";
+
+      // Print operation inputs.
+      if (op->getNumOperands() > 0) {
+        os << "{";
+        auto operandToPort = [&](Value operand) {
+          os << "<arg" << getValuePortName(operand) << "> ";
+          emitMlirOperand(os, operand);
+        };
+        interleave(op->getOperands(), os, operandToPort, "|");
+        os << "}|";
+      }
+      // Print operation name and type.
+      os << op->getName() << "\\l";
+
+      // Print attributes.
+      if (printAttrs && !op->getAttrs().empty()) {
+        // Extra line break to separate attributes from the operation name.
+        os << "\\l";
         for (const NamedAttribute &attr : op->getAttrs()) {
-          os << '\n' << attr.getName().getValue() << ": ";
+          os << attr.getName().getValue() << ": ";
           emitMlirAttr(os, attr.getValue());
+          os << "\\l";
         }
       }
+
+      if (op->getNumResults() > 0) {
+        os << "|{";
+        auto resultToPort = [&](Value result) {
+          os << "<res" << getValuePortName(result) << "> ";
+          emitMlirOperand(os, result);
+          if (printResultTypes) {
+            os << " ";
+            emitMlirType(os, result.getType());
+          }
+        };
+        interleave(op->getResults(), os, resultToPort, "|");
+        os << "}";
+      }
+
+      os << "}";
     });
   }
 
   /// Generate a label for a block argument.
   std::string getLabel(BlockArgument arg) {
-    return "arg" + std::to_string(arg.getArgNumber());
+    return strFromOs([&](raw_ostream &os) {
+      os << "<res" << getValuePortName(arg) << "> ";
+      arg.printAsOperand(os, OpPrintingFlags());
+      if (printResultTypes) {
+        os << " ";
+        emitMlirType(os, arg.getType());
+      }
+    });
   }
 
   /// Process a block. Emit a cluster and one node per block argument and
@@ -287,14 +386,12 @@ class PrintOpPass : public impl::ViewOpGraphBase<PrintOpPass> {
     emitClusterStmt([&]() {
       for (BlockArgument &blockArg : block.getArguments())
         valueToNode[blockArg] = emitNodeStmt(getLabel(blockArg));
-
       // Emit a node for each operation.
       std::optional<Node> prevNode;
       for (Operation &op : block) {
         Node nextNode = processOperation(&op);
         if (printControlFlowEdges && prevNode)
-          emitEdgeStmt(*prevNode, nextNode, /*label=*/"",
-                       kLineStyleControlFlow);
+          emitEdgeStmt(*prevNode, nextNode, /*port=*/"", kLineStyleControlFlow);
         prevNode = nextNode;
       }
     });
@@ -311,18 +408,19 @@ class PrintOpPass : public impl::ViewOpGraphBase<PrintOpPass> {
             for (Region &region : op->getRegions())
               processRegion(region);
           },
-          getLabel(op));
+          getClusterLabel(op));
     } else {
-      node = emitNodeStmt(getLabel(op), kShapeNode,
+      node = emitNodeStmt(getRecordLabel(op), kShapeNode,
                           backgroundColors[op->getName()].second);
     }
 
     // Insert data flow edges originating from each operand.
     if (printDataFlowEdges) {
       unsigned numOperands = op->getNumOperands();
-      for (unsigned i = 0; i < numOperands; i++)
-        dataFlowEdges.push_back({op->getOperand(i), node,
-                                 numOperands == 1 ? "" : std::to_string(i)});
+      for (unsigned i = 0; i < numOperands; i++) {
+        auto operand = op->getOperand(i);
+        dataFlowEdges.push_back({operand, node, getValuePortName(operand)});
+      }
     }
 
     for (Value result : op->getResults())
@@ -352,7 +450,7 @@ class PrintOpPass : public impl::ViewOpGraphBase<PrintOpPass> {
   /// Mapping of SSA values to Graphviz nodes/clusters.
   DenseMap<Value, Node> valueToNode;
   /// Output for data flow edges is delayed until the end to handle cycles
-  std::vector<std::tuple<Value, Node, std::string>> dataFlowEdges;
+  std::vector<DataFlowEdge> dataFlowEdges;
   /// Counter for generating unique node/subgraph identifiers.
   int counter = 0;
 
diff --git a/mlir/python/mlir/dialects/linalg/__init__.py b/mlir/python/mlir/dialects/linalg/__init__.py
index 8fb1227ee80ff..5cda4769d593f 100644
--- a/mlir/python/mlir/dialects/linalg/__init__.py
+++ b/mlir/python/mlir/dialects/linalg/__init__.py
@@ -10,6 +10,7 @@
 #   DSL -> YAML -> tblgen -> pytblgen -> build/.../_linalg_ops_gen.py.
 from .._linalg_ops_gen import *
 from .._linalg_enum_gen import *
+from .._linalg_enum_gen import _iteratortypeenum
 
 # These are the ground truth functions defined as:
 # ```
@@ -58,6 +59,7 @@
 
 from ...ir import *
 from .._ods_common import get_op_result_or_value as _get_op_result_or_value
+from ...extras.meta import region_op
 
 
 def transpose(
@@ -102,3 +104,92 @@ def broadcast(
     )
     fill_builtin_region(op.operation)
     return op
+
+
+@register_attribute_builder("IteratorTypeArrayAttr")
+def _IteratorTypeArrayAttr(x, context):
+    return ArrayAttr.get([_iteratortypeenum(v, context) for v in x])
+
+
+# The underscore is needed here so that there's no collision with opdsl generation.
+class GenericOp_(GenericOp):
+    def __init__(
+        self,
+        inputs,
+        outputs,
+        indexing_maps,
+        iterator_types,
+        *,
+        doc=None,
+        library_call=None,
+        loc=None,
+        ip=None,
+    ):
+        result_types = []
+        if isinstance(outputs[0].type, RankedTensorType):
+            result_types = [o.type for o in outputs]
+
+        super().__init__(
+            result_types,
+            inputs,
+            outputs,
+            indexing_maps,
+            iterator_types,
+            doc=doc,
+            library_call=library_call,
+            loc=loc,
+            ip=ip,
+        )
+        element_types = [i.type.element_type for i in inputs] + [
+            o.type.element_type for o in outputs
+        ]
+        self.regions[0].blocks.append(*element_types)
+
+
+generic = region_op(GenericOp_, terminator=YieldOp)
+
+
+def matmul(
+    *ins: Union[Operation, OpView, Value],
+    outs: Sequence[Union[Operation, OpView, Value]],
+    indexing_maps: Optional[Sequence[AffineMapAttr]] = None,
+    cast: Optional[Union[TypeFn, Attribute]] = None,
+):
+    ins = [_get_op_result_or_value(input) for input in ins]
+    if len(outs) > 1:
+        raise ValueError(f"{outs=} must have length 1.")
+    init = _get_op_result_or_value(outs[0])
+    result_types = [init.type] if isinstance(init.type, RankedTensorType) else []
+
+    op = MatmulOp(
+        result_tensors=result_types,
+        inputs=ins,
+        outputs=[init],
+        indexing_maps=indexing_maps,
+        cast=cast,
+    )
+    fill_builtin_region(op.operation)
+    return op
+
+
+def contract(
+    *ins: Union[Operation, OpView, Value],
+    outs: Sequence[Union[Operation, OpView, Value]],
+    indexing_maps: Sequence[AffineMapAttr],
+    cast: Optional[Union[TypeFn, Attribute]] = None,
+):
+    ins = [_get_op_result_or_value(input) for input in ins]
+    if len(outs) > 1:
+        raise ValueError(f"{outs=} must have length 1.")
+    init = _get_op_result_or_value(outs[0])
+    result_types = [init.type] if isinstance(init.type, RankedTensorType) else []
+
+    op = ContractOp(
+        result_tensors=result_types,
+        inputs=ins,
+        outputs=[init],
+        indexing_maps=indexing_maps,
+        cast=cast,
+    )
+    fill_builtin_region(op.operation)
+    return op
diff --git a/mlir/python/mlir/dialects/linalg/opdsl/ops/core_named_ops.py b/mlir/python/mlir/dialects/linalg/opdsl/ops/core_named_ops.py
index c95cd5eecfffc..040663c882a08 100644
--- a/mlir/python/mlir/dialects/linalg/opdsl/ops/core_named_ops.py
+++ b/mlir/python/mlir/dialects/linalg/opdsl/ops/core_named_ops.py
@@ -484,24 +484,6 @@ def batch_mmt4d(
     ) * TypeFn.cast_signed(TV.AccumType, rhs[D.b, D.n, D.k, D.n0, D.k0])
 
 
-@linalg_structured_op
-def batch_matmul(
-    A=TensorDef(T1, Batch, S.M, S.K),
-    B=TensorDef(T2, Batch, S.K, S.N),
-    C=TensorDef(U, Batch, S.M, S.N, output=True),
-):
-    """Performs a batched matrix multiplication of two 3D inputs.
-
-    Numeric casting is performed on the operands to the inner multiply, promoting
-    them to the same data type as the accumulator/output.
-    """
-    domain(D.b, D.m, D.n, D.k)
-    implements(ContractionOpInterface)
-    C[D.b, D.m, D.n] += TypeFn.cast_signed(U, A[D.b, D.m, D.k]) * TypeFn.cast_signed(
-        U, B[D.b, D.k, D.n]
-    )
-
-
 @linalg_structured_op
 def batch_matmul_transpose_a(
     A=TensorDef(T1, Batch, S.K, S.M),
diff --git a/mlir/test/Bytecode/resources.mlir b/mlir/test/Bytecode/resources.mlir
index 33ed01d20fa0c..3ef220e890042 100644
--- a/mlir/test/Bytecode/resources.mlir
+++ b/mlir/test/Bytecode/resources.mlir
@@ -4,21 +4,21 @@
 module @TestDialectResources attributes {
   // CHECK: bytecode.test = dense_resource<decl_resource> : tensor<2xui32>
   // CHECK: bytecode.test2 = dense_resource<resource> : tensor<4xf64>
-  // CHECK: bytecode.test3 = dense_resource<resource_2> : tensor<4xf64>
+  // CHECK: bytecode.test3 = dense_resource<"resource\09two"> : tensor<4xf64>
   bytecode.test = dense_resource<decl_resource> : tensor<2xui32>,
   bytecode.test2 = dense_resource<resource> : tensor<4xf64>,
-  bytecode.test3 = dense_resource<resource_2> : tensor<4xf64>
+  bytecode.test3 = dense_resource<"resource\09two"> : tensor<4xf64>
 } {}
 
 // CHECK: builtin: {
 // CHECK-NEXT: resource: "0x08000000010000000000000002000000000000000300000000000000"
-// CHECK-NEXT: resource_2: "0x08000000010000000000000002000000000000000300000000000000"
+// CHECK-NEXT: "resource\09two": "0x08000000010000000000000002000000000000000300000000000000"
 
 {-#
   dialect_resources: {
     builtin: {
       resource: "0x08000000010000000000000002000000000000000300000000000000",
-      resource_2: "0x08000000010000000000000002000000000000000300000000000000"
+      "resource\09two": "0x08000000010000000000000002000000000000000300000000000000"
     }
   }
 #-}
diff --git a/mlir/test/CAPI/translation.c b/mlir/test/CAPI/translation.c
index c9233d95fd512..8891c2a559b7e 100644
--- a/mlir/test/CAPI/translation.c
+++ b/mlir/test/CAPI/translation.c
@@ -58,11 +58,38 @@ static void testToLLVMIR(MlirContext ctx) {
   LLVMContextDispose(llvmCtx);
 }
 
+// CHECK-LABEL: testTypeToFromLLVMIRTranslator
+static void testTypeToFromLLVMIRTranslator(MlirContext ctx) {
+  fprintf(stderr, "testTypeToFromLLVMIRTranslator\n");
+  LLVMContextRef llvmCtx = LLVMContextCreate();
+
+  LLVMTypeRef llvmTy = LLVMInt32TypeInContext(llvmCtx);
+  MlirTypeFromLLVMIRTranslator fromLLVMTranslator =
+      mlirTypeFromLLVMIRTranslatorCreate(ctx);
+  MlirType mlirTy =
+      mlirTypeFromLLVMIRTranslatorTranslateType(fromLLVMTranslator, llvmTy);
+  // CHECK: i32
+  mlirTypeDump(mlirTy);
+
+  MlirTypeToLLVMIRTranslator toLLVMTranslator =
+      mlirTypeToLLVMIRTranslatorCreate(llvmCtx);
+  LLVMTypeRef llvmTy2 =
+      mlirTypeToLLVMIRTranslatorTranslateType(toLLVMTranslator, mlirTy);
+  // CHECK: i32
+  LLVMDumpType(llvmTy2);
+  fprintf(stderr, "\n");
+
+  mlirTypeFromLLVMIRTranslatorDestroy(fromLLVMTranslator);
+  mlirTypeToLLVMIRTranslatorDestroy(toLLVMTranslator);
+  LLVMContextDispose(llvmCtx);
+}
+
 int main(void) {
   MlirContext ctx = mlirContextCreate();
   mlirDialectHandleRegisterDialect(mlirGetDialectHandle__llvm__(), ctx);
   mlirContextGetOrLoadDialect(ctx, mlirStringRefCreateFromCString("llvm"));
   testToLLVMIR(ctx);
+  testTypeToFromLLVMIRTranslator(ctx);
   mlirContextDestroy(ctx);
   return 0;
 }
diff --git a/mlir/test/Conversion/AMDGPUToROCDL/amdgpu-to-rocdl.mlir b/mlir/test/Conversion/AMDGPUToROCDL/amdgpu-to-rocdl.mlir
index af6331646f0a5..062b63c076c3c 100644
--- a/mlir/test/Conversion/AMDGPUToROCDL/amdgpu-to-rocdl.mlir
+++ b/mlir/test/Conversion/AMDGPUToROCDL/amdgpu-to-rocdl.mlir
@@ -34,24 +34,24 @@ func.func @gpu_gcn_raw_buffer_load_i32(%buf: memref<64xi32>, %idx: i32) -> i32 {
 func.func @gpu_gcn_raw_buffer_load_i32_strided(%buf: memref<16x16xi32, strided<[?, ?], offset: ?>>, %i: i32, %j: i32) -> i32 {
     // CHECK: %[[descriptor:.*]] = builtin.unrealized_conversion_cast %{{.*}} : memref<16x16xi32, strided<[?, ?], offset: ?>> to !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)>
     // CHECK: %[[elem_size:.*]] = llvm.mlir.constant(4 : i32) : i32
-    // CHECK: %[[algn_ptr:.*]] = llvm.extractvalue %[[descriptor]][1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> 
-    // CHECK: %[[offset:.*]] = llvm.extractvalue %[[descriptor]][2] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> 
+    // CHECK: %[[algn_ptr:.*]] = llvm.extractvalue %[[descriptor]][1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)>
+    // CHECK: %[[offset:.*]] = llvm.extractvalue %[[descriptor]][2] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)>
     // CHECK: %[[ptr:.*]] = llvm.getelementptr %[[algn_ptr]][%[[offset]]] : (!llvm.ptr, i64) -> !llvm.ptr, i32
     // CHECK: %[[stride:.*]] = llvm.mlir.constant(0 : i16) : i16
-    // CHECK: %[[sz_i:.*]] = llvm.extractvalue %[[descriptor]][3, 0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> 
-    // CHECK: %[[stride_i:.*]] = llvm.extractvalue %[[descriptor]][4, 0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> 
+    // CHECK: %[[sz_i:.*]] = llvm.extractvalue %[[descriptor]][3, 0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)>
+    // CHECK: %[[stride_i:.*]] = llvm.extractvalue %[[descriptor]][4, 0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)>
     // CHECK: %[[ext_i:.*]] = llvm.mul %[[sz_i]], %[[stride_i]] : i64
-    // CHECK: %[[sz_j:.*]] = llvm.extractvalue %[[descriptor]][3, 1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> 
-    // CHECK: %[[stride_j:.*]] = llvm.extractvalue %[[descriptor]][4, 1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> 
+    // CHECK: %[[sz_j:.*]] = llvm.extractvalue %[[descriptor]][3, 1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)>
+    // CHECK: %[[stride_j:.*]] = llvm.extractvalue %[[descriptor]][4, 1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)>
     // CHECK: %[[ext_j:.*]] = llvm.mul %[[sz_j]], %[[stride_j]] : i64
     // CHECK: %[[num_records:.*]] = llvm.intr.umax(%[[ext_i]], %[[ext_j]]) : (i64, i64) -> i64
     // CHECK: %[[num_rec_i32:.*]] = llvm.trunc %[[num_records]] : i64 to i32
     // CHECK: %[[num_rec_bytes_i32:.*]] = llvm.mul %[[num_rec_i32]], %[[elem_size]] : i32
     // CHECK: %[[rsrc:.*]] = rocdl.make.buffer.rsrc %[[ptr]], %[[stride]], %[[num_rec_bytes_i32]], %{{.*}} : !llvm.ptr to <8>
-    // CHECK: %[[stride_i_1:.*]] = llvm.extractvalue %[[descriptor]][4, 0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> 
+    // CHECK: %[[stride_i_1:.*]] = llvm.extractvalue %[[descriptor]][4, 0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)>
     // CHECK: %[[stride_i_i32:.*]] = llvm.trunc %[[stride_i_1]] : i64 to i32
     // CHECK: %[[t_0:.*]] = llvm.mul %{{.*}}, %[[stride_i_i32]] : i32
-    // CHECK: %[[stride_j_1:.*]] = llvm.extractvalue %[[descriptor]][4, 1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> 
+    // CHECK: %[[stride_j_1:.*]] = llvm.extractvalue %[[descriptor]][4, 1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)>
     // CHECK: %[[stride_j_i32:.*]] = llvm.trunc %[[stride_j_1]] : i64 to i32
     // CHECK: %[[t_1:.*]] = llvm.mul %{{.*}}, %[[stride_j_i32]] : i32
     // CHECK: %[[index:.*]] = llvm.add %[[t_0]], %[[t_1]] : i32
@@ -287,9 +287,9 @@ func.func @amdgpu_raw_buffer_atomic_cmpswap_v2f16(%src : vector<2xf16>, %cmp : v
 func.func @lds_barrier() {
   // GFX908: llvm.inline_asm has_side_effects asm_dialect = att
   // GFX908-SAME: ";;;WARNING: BREAKS DEBUG WATCHES\0As_waitcnt lgkmcnt(0)\0As_barrier"
-  // GFX90A: rocdl.waitcnt -7937
+  // GFX90A: rocdl.s.waitcnt -7937
   // GFX90A-NEXT: rocdl.s.barrier
-  // GFX10:  rocdl.waitcnt -16129
+  // GFX10:  rocdl.s.waitcnt -16129
   // GFX10-NEXT: rocdl.s.barrier
   // GFX11:  llvm.inline_asm has_side_effects asm_dialect = att
   // GFX11-SAME: ";;;WARNING: BREAKS DEBUG WATCHES\0As_waitcnt lgkmcnt(0)\0As_barrier"
diff --git a/mlir/test/Conversion/ArithToAMDGPU/8-bit-floats.mlir b/mlir/test/Conversion/ArithToAMDGPU/8-bit-floats.mlir
index bd90facb61544..985fb532ea74a 100644
--- a/mlir/test/Conversion/ArithToAMDGPU/8-bit-floats.mlir
+++ b/mlir/test/Conversion/ArithToAMDGPU/8-bit-floats.mlir
@@ -10,7 +10,19 @@ func.func @scalar_ext(%v: f8E5M2FNUZ) -> f16 {
   return %w : f16
 }
 
-// No 0-D test because arith.extf hasn't been extended to support it.
+// -----
+
+// CHECK-LABEL: func.func @vector_zero_d(
+// CHECK-SAME:   %[[ARG0:[a-zA-Z0-9_]+]]: vector<f8E5M2FNUZ>) -> vector<f32>
+// CHECK: %[[CONST:.+]] = arith.constant dense<0.000000e+00> : vector<f32>
+// CHECK: %[[EXTRACT:.+]] = vector.extract %[[ARG0]][] : f8E5M2FNUZ from vector<f8E5M2FNUZ>
+// CHECK: %[[CONVERT:.+]] = amdgpu.ext_packed_fp8 %[[EXTRACT]][0] : f8E5M2FNUZ to f32
+// CHECK: %[[RESULT:.+]] = vector.insert %[[CONVERT]], %[[CONST]] [] : f32 into vector<f32>
+// CHECK: return %[[RESULT]] : vector<f32>
+func.func @vector_zero_d(%v: vector<f8E5M2FNUZ>) -> vector<f32> {
+  %w = arith.extf %v : vector<f8E5M2FNUZ> to vector<f32>
+  return %w : vector<f32>
+}
 
 // -----
 
diff --git a/mlir/test/Conversion/ArithToLLVM/convert-nd-vector-to-llvmir.mlir b/mlir/test/Conversion/ArithToLLVM/convert-nd-vector-to-llvmir.mlir
index b234cbbb35f32..86f0be81ce99e 100644
--- a/mlir/test/Conversion/ArithToLLVM/convert-nd-vector-to-llvmir.mlir
+++ b/mlir/test/Conversion/ArithToLLVM/convert-nd-vector-to-llvmir.mlir
@@ -2,7 +2,7 @@
 
 // CHECK-LABEL: @vec_bin
 func.func @vec_bin(%arg0: vector<2x2x2xf32>) -> vector<2x2x2xf32> {
-  // CHECK: llvm.mlir.undef : !llvm.array<2 x array<2 x vector<2xf32>>>
+  // CHECK: llvm.mlir.poison : !llvm.array<2 x array<2 x vector<2xf32>>>
 
   // This block appears 2x2 times
   // CHECK-NEXT: llvm.extractvalue %{{.*}}[0, 0] : !llvm.array<2 x array<2 x vector<2xf32>>>
@@ -23,7 +23,7 @@ func.func @vec_bin(%arg0: vector<2x2x2xf32>) -> vector<2x2x2xf32> {
 
 // CHECK-LABEL: @sexti
 func.func @sexti_vector(%arg0 : vector<1x2x3xi32>, %arg1 : vector<1x2x3xi64>) {
-  // CHECK: llvm.mlir.undef : !llvm.array<1 x array<2 x vector<3xi64>>>
+  // CHECK: llvm.mlir.poison : !llvm.array<1 x array<2 x vector<3xi64>>>
   // CHECK: llvm.extractvalue %{{.*}}[0, 0] : !llvm.array<1 x array<2 x vector<3xi32>>>
   // CHECK: llvm.sext %{{.*}} : vector<3xi32> to vector<3xi64>
   // CHECK: llvm.insertvalue %{{.*}}, %{{.*}}[0, 0] : !llvm.array<1 x array<2 x vector<3xi64>>>
@@ -36,7 +36,7 @@ func.func @sexti_vector(%arg0 : vector<1x2x3xi32>, %arg1 : vector<1x2x3xi64>) {
 
 // CHECK-LABEL: @zexti
 func.func @zexti_vector(%arg0 : vector<1x2x3xi32>, %arg1 : vector<1x2x3xi64>) {
-  // CHECK: llvm.mlir.undef : !llvm.array<1 x array<2 x vector<3xi64>>>
+  // CHECK: llvm.mlir.poison : !llvm.array<1 x array<2 x vector<3xi64>>>
   // CHECK: llvm.extractvalue %{{.*}}[0, 0] : !llvm.array<1 x array<2 x vector<3xi32>>>
   // CHECK: llvm.zext %{{.*}} : vector<3xi32> to vector<3xi64>
   // CHECK: llvm.insertvalue %{{.*}}, %{{.*}}[0, 0] : !llvm.array<1 x array<2 x vector<3xi64>>>
@@ -49,7 +49,7 @@ func.func @zexti_vector(%arg0 : vector<1x2x3xi32>, %arg1 : vector<1x2x3xi64>) {
 
 // CHECK-LABEL: @sitofp
 func.func @sitofp_vector(%arg0 : vector<1x2x3xi32>) -> vector<1x2x3xf32> {
-  // CHECK: llvm.mlir.undef : !llvm.array<1 x array<2 x vector<3xf32>>>
+  // CHECK: llvm.mlir.poison : !llvm.array<1 x array<2 x vector<3xf32>>>
   // CHECK: llvm.extractvalue %{{.*}}[0, 0] : !llvm.array<1 x array<2 x vector<3xi32>>>
   // CHECK: llvm.sitofp %{{.*}} : vector<3xi32> to vector<3xf32>
   // CHECK: llvm.insertvalue %{{.*}}, %{{.*}}[0, 0] : !llvm.array<1 x array<2 x vector<3xf32>>>
@@ -62,7 +62,7 @@ func.func @sitofp_vector(%arg0 : vector<1x2x3xi32>) -> vector<1x2x3xf32> {
 
 // CHECK-LABEL: @uitofp
 func.func @uitofp_vector(%arg0 : vector<1x2x3xi32>) -> vector<1x2x3xf32> {
-  // CHECK: llvm.mlir.undef : !llvm.array<1 x array<2 x vector<3xf32>>>
+  // CHECK: llvm.mlir.poison : !llvm.array<1 x array<2 x vector<3xf32>>>
   // CHECK: llvm.extractvalue %{{.*}}[0, 0] : !llvm.array<1 x array<2 x vector<3xi32>>>
   // CHECK: llvm.uitofp %{{.*}} : vector<3xi32> to vector<3xf32>
   // CHECK: llvm.insertvalue %{{.*}}, %{{.*}}[0, 0] : !llvm.array<1 x array<2 x vector<3xf32>>>
@@ -75,7 +75,7 @@ func.func @uitofp_vector(%arg0 : vector<1x2x3xi32>) -> vector<1x2x3xf32> {
 
 // CHECK-LABEL: @fptosi
 func.func @fptosi_vector(%arg0 : vector<1x2x3xf32>) -> vector<1x2x3xi32> {
-  // CHECK: llvm.mlir.undef : !llvm.array<1 x array<2 x vector<3xi32>>>
+  // CHECK: llvm.mlir.poison : !llvm.array<1 x array<2 x vector<3xi32>>>
   // CHECK: llvm.extractvalue %{{.*}}[0, 0] : !llvm.array<1 x array<2 x vector<3xf32>>>
   // CHECK: llvm.fptosi %{{.*}} : vector<3xf32> to vector<3xi32>
   // CHECK: llvm.insertvalue %{{.*}}, %{{.*}}[0, 0] : !llvm.array<1 x array<2 x vector<3xi32>>>
@@ -88,7 +88,7 @@ func.func @fptosi_vector(%arg0 : vector<1x2x3xf32>) -> vector<1x2x3xi32> {
 
 // CHECK-LABEL: @fptoui
 func.func @fptoui_vector(%arg0 : vector<1x2x3xf32>) -> vector<1x2x3xi32> {
-  // CHECK: llvm.mlir.undef : !llvm.array<1 x array<2 x vector<3xi32>>>
+  // CHECK: llvm.mlir.poison : !llvm.array<1 x array<2 x vector<3xi32>>>
   // CHECK: llvm.extractvalue %{{.*}}[0, 0] : !llvm.array<1 x array<2 x vector<3xf32>>>
   // CHECK: llvm.fptoui %{{.*}} : vector<3xf32> to vector<3xi32>
   // CHECK: llvm.insertvalue %{{.*}}, %{{.*}}[0, 0] : !llvm.array<1 x array<2 x vector<3xi32>>>
@@ -101,7 +101,7 @@ func.func @fptoui_vector(%arg0 : vector<1x2x3xf32>) -> vector<1x2x3xi32> {
 
 // CHECK-LABEL: @fpext
 func.func @fpext_vector(%arg0 : vector<1x2x3xf16>) -> vector<1x2x3xf64> {
-  // CHECK: llvm.mlir.undef : !llvm.array<1 x array<2 x vector<3xf64>>>
+  // CHECK: llvm.mlir.poison : !llvm.array<1 x array<2 x vector<3xf64>>>
   // CHECK: llvm.extractvalue %{{.*}}[0, 0] : !llvm.array<1 x array<2 x vector<3xf16>>>
   // CHECK: llvm.fpext %{{.*}} : vector<3xf16> to vector<3xf64>
   // CHECK: llvm.insertvalue %{{.*}}, %{{.*}}[0, 0] : !llvm.array<1 x array<2 x vector<3xf64>>>
@@ -114,7 +114,7 @@ func.func @fpext_vector(%arg0 : vector<1x2x3xf16>) -> vector<1x2x3xf64> {
 
 // CHECK-LABEL: @fptrunc
 func.func @fptrunc_vector(%arg0 : vector<1x2x3xf64>) -> vector<1x2x3xf16> {
-  // CHECK: llvm.mlir.undef : !llvm.array<1 x array<2 x vector<3xf16>>>
+  // CHECK: llvm.mlir.poison : !llvm.array<1 x array<2 x vector<3xf16>>>
   // CHECK: llvm.extractvalue %{{.*}}[0, 0] : !llvm.array<1 x array<2 x vector<3xf64>>>
   // CHECK: llvm.fptrunc %{{.*}} : vector<3xf64> to vector<3xf16>
   // CHECK: llvm.insertvalue %{{.*}}, %{{.*}}[0, 0] : !llvm.array<1 x array<2 x vector<3xf16>>>
@@ -127,7 +127,7 @@ func.func @fptrunc_vector(%arg0 : vector<1x2x3xf64>) -> vector<1x2x3xf16> {
 
 // CHECK-LABEL: @trunci
 func.func @trunci_vector(%arg0 : vector<1x2x3xi64>) -> vector<1x2x3xi16> {
-  // CHECK: llvm.mlir.undef : !llvm.array<1 x array<2 x vector<3xi16>>>
+  // CHECK: llvm.mlir.poison : !llvm.array<1 x array<2 x vector<3xi16>>>
   // CHECK: llvm.extractvalue %{{.*}}[0, 0] : !llvm.array<1 x array<2 x vector<3xi64>>>
   // CHECK: llvm.trunc %{{.*}} : vector<3xi64> to vector<3xi16>
   // CHECK: llvm.insertvalue %{{.*}}, %{{.*}}[0, 0] : !llvm.array<1 x array<2 x vector<3xi16>>>
@@ -140,7 +140,7 @@ func.func @trunci_vector(%arg0 : vector<1x2x3xi64>) -> vector<1x2x3xi16> {
 
 // CHECK-LABEL: @shl
 func.func @shl_vector(%arg0 : vector<1x2x3xi64>) -> vector<1x2x3xi64> {
-  // CHECK: llvm.mlir.undef : !llvm.array<1 x array<2 x vector<3xi64>>>
+  // CHECK: llvm.mlir.poison : !llvm.array<1 x array<2 x vector<3xi64>>>
   // CHECK: llvm.extractvalue %{{.*}}[0, 0] : !llvm.array<1 x array<2 x vector<3xi64>>>
   // CHECK: llvm.shl %{{.*}}, %{{.*}} : vector<3xi64>
   // CHECK: llvm.insertvalue %{{.*}}, %{{.*}}[0, 0] : !llvm.array<1 x array<2 x vector<3xi64>>>
@@ -154,7 +154,7 @@ func.func @shl_vector(%arg0 : vector<1x2x3xi64>) -> vector<1x2x3xi64> {
 
 // CHECK-LABEL: @shrs
 func.func @shrs_vector(%arg0 : vector<1x2x3xi64>) -> vector<1x2x3xi64> {
-  // CHECK: llvm.mlir.undef : !llvm.array<1 x array<2 x vector<3xi64>>>
+  // CHECK: llvm.mlir.poison : !llvm.array<1 x array<2 x vector<3xi64>>>
   // CHECK: llvm.extractvalue %{{.*}}[0, 0] : !llvm.array<1 x array<2 x vector<3xi64>>>
   // CHECK: llvm.ashr %{{.*}}, %{{.*}} : vector<3xi64>
   // CHECK: llvm.insertvalue %{{.*}}, %{{.*}}[0, 0] : !llvm.array<1 x array<2 x vector<3xi64>>>
@@ -168,7 +168,7 @@ func.func @shrs_vector(%arg0 : vector<1x2x3xi64>) -> vector<1x2x3xi64> {
 
 // CHECK-LABEL: @shru
 func.func @shru_vector(%arg0 : vector<1x2x3xi64>) -> vector<1x2x3xi64> {
-  // CHECK: llvm.mlir.undef : !llvm.array<1 x array<2 x vector<3xi64>>>
+  // CHECK: llvm.mlir.poison : !llvm.array<1 x array<2 x vector<3xi64>>>
   // CHECK: llvm.extractvalue %{{.*}}[0, 0] : !llvm.array<1 x array<2 x vector<3xi64>>>
   // CHECK: llvm.lshr %{{.*}}, %{{.*}} : vector<3xi64>
   // CHECK: llvm.insertvalue %{{.*}}, %{{.*}}[0, 0] : !llvm.array<1 x array<2 x vector<3xi64>>>
@@ -184,7 +184,7 @@ func.func @shru_vector(%arg0 : vector<1x2x3xi64>) -> vector<1x2x3xi64> {
 
 // CHECK-LABEL: @bitcast_2d
 func.func @bitcast_2d(%arg0: vector<2x4xf32>) {
-  // CHECK: llvm.mlir.undef
+  // CHECK: llvm.mlir.poison
   // CHECK: llvm.extractvalue %{{.*}}[0]
   // CHECK: llvm.bitcast %{{.*}} : vector<4xf32> to vector<4xi32>
   // CHECK: llvm.insertvalue %{{.*}}, %{{.*}}[0]
diff --git a/mlir/test/Conversion/ComplexToLLVM/convert-to-llvm.mlir b/mlir/test/Conversion/ComplexToLLVM/convert-to-llvm.mlir
index a60b974e374d3..40f8af7de44aa 100644
--- a/mlir/test/Conversion/ComplexToLLVM/convert-to-llvm.mlir
+++ b/mlir/test/Conversion/ComplexToLLVM/convert-to-llvm.mlir
@@ -6,7 +6,7 @@
 
 // CHECK-LABEL: func @complex_create
 // CHECK-SAME:    (%[[REAL0:.*]]: f32, %[[IMAG0:.*]]: f32)
-// CHECK-NEXT:    %[[CPLX0:.*]] = llvm.mlir.undef : !llvm.struct<(f32, f32)>
+// CHECK-NEXT:    %[[CPLX0:.*]] = llvm.mlir.poison : !llvm.struct<(f32, f32)>
 // CHECK-NEXT:    %[[CPLX1:.*]] = llvm.insertvalue %[[REAL0]], %[[CPLX0]][0] : !llvm.struct<(f32, f32)>
 // CHECK-NEXT:    %[[CPLX2:.*]] = llvm.insertvalue %[[IMAG0]], %[[CPLX1]][1] : !llvm.struct<(f32, f32)>
 func.func @complex_create(%real: f32, %imag: f32) -> complex<f32> {
@@ -37,7 +37,7 @@ func.func @complex_extract(%cplx: complex<f32>) {
 // CHECK-DAG:     %[[B_REAL:.*]] = llvm.extractvalue %[[B:.*]][0] : !llvm.struct<(f64, f64)>
 // CHECK-DAG:     %[[A_IMAG:.*]] = llvm.extractvalue %[[A]][1] : !llvm.struct<(f64, f64)>
 // CHECK-DAG:     %[[B_IMAG:.*]] = llvm.extractvalue %[[B]][1] : !llvm.struct<(f64, f64)>
-// CHECK:         %[[C0:.*]] = llvm.mlir.undef : !llvm.struct<(f64, f64)>
+// CHECK:         %[[C0:.*]] = llvm.mlir.poison : !llvm.struct<(f64, f64)>
 // CHECK-DAG:     %[[C_REAL:.*]] = llvm.fadd %[[A_REAL]], %[[B_REAL]] : f64
 // CHECK-DAG:     %[[C_IMAG:.*]] = llvm.fadd %[[A_IMAG]], %[[B_IMAG]] : f64
 // CHECK:         %[[C1:.*]] = llvm.insertvalue %[[C_REAL]], %[[C0]][0] : !llvm.struct<(f64, f64)>
@@ -58,7 +58,7 @@ func.func @complex_addition() {
 // CHECK-DAG:     %[[B_REAL:.*]] = llvm.extractvalue %[[B:.*]][0] : !llvm.struct<(f64, f64)>
 // CHECK-DAG:     %[[A_IMAG:.*]] = llvm.extractvalue %[[A]][1] : !llvm.struct<(f64, f64)>
 // CHECK-DAG:     %[[B_IMAG:.*]] = llvm.extractvalue %[[B]][1] : !llvm.struct<(f64, f64)>
-// CHECK:         %[[C0:.*]] = llvm.mlir.undef : !llvm.struct<(f64, f64)>
+// CHECK:         %[[C0:.*]] = llvm.mlir.poison : !llvm.struct<(f64, f64)>
 // CHECK-DAG:     %[[C_REAL:.*]] = llvm.fsub %[[A_REAL]], %[[B_REAL]] : f64
 // CHECK-DAG:     %[[C_IMAG:.*]] = llvm.fsub %[[A_IMAG]], %[[B_IMAG]] : f64
 // CHECK:         %[[C1:.*]] = llvm.insertvalue %[[C_REAL]], %[[C0]][0] : !llvm.struct<(f64, f64)>
@@ -88,7 +88,7 @@ func.func @complex_div(%lhs: complex<f32>, %rhs: complex<f32>) -> complex<f32> {
 // CHECK: %[[RHS_RE:.*]] = llvm.extractvalue %[[CASTED_RHS]][0] : ![[C_TY]]
 // CHECK: %[[RHS_IM:.*]] = llvm.extractvalue %[[CASTED_RHS]][1] : ![[C_TY]]
 
-// CHECK: %[[RESULT_0:.*]] = llvm.mlir.undef : ![[C_TY]]
+// CHECK: %[[RESULT_0:.*]] = llvm.mlir.poison : ![[C_TY]]
 
 // CHECK-DAG: %[[RHS_RE_SQ:.*]] = llvm.fmul %[[RHS_RE]], %[[RHS_RE]]  : f32
 // CHECK-DAG: %[[RHS_IM_SQ:.*]] = llvm.fmul %[[RHS_IM]], %[[RHS_IM]]  : f32
@@ -123,7 +123,7 @@ func.func @complex_mul(%lhs: complex<f32>, %rhs: complex<f32>) -> complex<f32> {
 // CHECK: %[[LHS_IM:.*]] = llvm.extractvalue %[[CASTED_LHS]][1] : ![[C_TY]]
 // CHECK: %[[RHS_RE:.*]] = llvm.extractvalue %[[CASTED_RHS]][0] : ![[C_TY]]
 // CHECK: %[[RHS_IM:.*]] = llvm.extractvalue %[[CASTED_RHS]][1] : ![[C_TY]]
-// CHECK: %[[RESULT_0:.*]] = llvm.mlir.undef : ![[C_TY]]
+// CHECK: %[[RESULT_0:.*]] = llvm.mlir.poison : ![[C_TY]]
 
 // CHECK-DAG: %[[REAL_TMP_0:.*]] = llvm.fmul %[[RHS_RE]], %[[LHS_RE]]  : f32
 // CHECK-DAG: %[[REAL_TMP_1:.*]] = llvm.fmul %[[RHS_IM]], %[[LHS_IM]]  : f32
@@ -159,7 +159,7 @@ func.func @complex_abs(%arg: complex<f32>) -> f32 {
 // CHECK-DAG:     %[[B_REAL:.*]] = llvm.extractvalue %[[B:.*]][0] : !llvm.struct<(f64, f64)>
 // CHECK-DAG:     %[[A_IMAG:.*]] = llvm.extractvalue %[[A]][1] : !llvm.struct<(f64, f64)>
 // CHECK-DAG:     %[[B_IMAG:.*]] = llvm.extractvalue %[[B]][1] : !llvm.struct<(f64, f64)>
-// CHECK:         %[[C0:.*]] = llvm.mlir.undef : !llvm.struct<(f64, f64)>
+// CHECK:         %[[C0:.*]] = llvm.mlir.poison : !llvm.struct<(f64, f64)>
 // CHECK-DAG:     %[[C_REAL:.*]] = llvm.fadd %[[A_REAL]], %[[B_REAL]] {fastmathFlags = #llvm.fastmath<reassoc>} : f64
 // CHECK-DAG:     %[[C_IMAG:.*]] = llvm.fadd %[[A_IMAG]], %[[B_IMAG]] {fastmathFlags = #llvm.fastmath<reassoc>} : f64
 // CHECK:         %[[C1:.*]] = llvm.insertvalue %[[C_REAL]], %[[C0]][0] : !llvm.struct<(f64, f64)>
@@ -180,7 +180,7 @@ func.func @complex_addition_with_fmf() {
 // CHECK-DAG:     %[[B_REAL:.*]] = llvm.extractvalue %[[B:.*]][0] : !llvm.struct<(f64, f64)>
 // CHECK-DAG:     %[[A_IMAG:.*]] = llvm.extractvalue %[[A]][1] : !llvm.struct<(f64, f64)>
 // CHECK-DAG:     %[[B_IMAG:.*]] = llvm.extractvalue %[[B]][1] : !llvm.struct<(f64, f64)>
-// CHECK:         %[[C0:.*]] = llvm.mlir.undef : !llvm.struct<(f64, f64)>
+// CHECK:         %[[C0:.*]] = llvm.mlir.poison : !llvm.struct<(f64, f64)>
 // CHECK-DAG:     %[[C_REAL:.*]] = llvm.fsub %[[A_REAL]], %[[B_REAL]] {fastmathFlags = #llvm.fastmath<nnan, ninf>} : f64
 // CHECK-DAG:     %[[C_IMAG:.*]] = llvm.fsub %[[A_IMAG]], %[[B_IMAG]] {fastmathFlags = #llvm.fastmath<nnan, ninf>} : f64
 // CHECK:         %[[C1:.*]] = llvm.insertvalue %[[C_REAL]], %[[C0]][0] : !llvm.struct<(f64, f64)>
@@ -206,7 +206,7 @@ func.func @complex_substraction_with_fmf() {
 // CHECK: %[[RHS_RE:.*]] = llvm.extractvalue %[[CASTED_RHS]][0] : ![[C_TY]]
 // CHECK: %[[RHS_IM:.*]] = llvm.extractvalue %[[CASTED_RHS]][1] : ![[C_TY]]
 
-// CHECK: %[[RESULT_0:.*]] = llvm.mlir.undef : ![[C_TY]]
+// CHECK: %[[RESULT_0:.*]] = llvm.mlir.poison : ![[C_TY]]
 
 // CHECK-DAG: %[[RHS_RE_SQ:.*]] = llvm.fmul %[[RHS_RE]], %[[RHS_RE]] {fastmathFlags = #llvm.fastmath<nsz, arcp>} : f32
 // CHECK-DAG: %[[RHS_IM_SQ:.*]] = llvm.fmul %[[RHS_IM]], %[[RHS_IM]] {fastmathFlags = #llvm.fastmath<nsz, arcp>} : f32
@@ -242,7 +242,7 @@ func.func @complex_div_with_fmf(%lhs: complex<f32>, %rhs: complex<f32>) -> compl
 // CHECK: %[[LHS_IM:.*]] = llvm.extractvalue %[[CASTED_LHS]][1] : ![[C_TY]]
 // CHECK: %[[RHS_RE:.*]] = llvm.extractvalue %[[CASTED_RHS]][0] : ![[C_TY]]
 // CHECK: %[[RHS_IM:.*]] = llvm.extractvalue %[[CASTED_RHS]][1] : ![[C_TY]]
-// CHECK: %[[RESULT_0:.*]] = llvm.mlir.undef : ![[C_TY]]
+// CHECK: %[[RESULT_0:.*]] = llvm.mlir.poison : ![[C_TY]]
 
 // CHECK-DAG: %[[REAL_TMP_0:.*]] = llvm.fmul %[[RHS_RE]], %[[LHS_RE]] {fastmathFlags = #llvm.fastmath<contract, afn>} : f32
 // CHECK-DAG: %[[REAL_TMP_1:.*]] = llvm.fmul %[[RHS_IM]], %[[LHS_IM]] {fastmathFlags = #llvm.fastmath<contract, afn>} : f32
diff --git a/mlir/test/Conversion/ComplexToLLVM/full-conversion.mlir b/mlir/test/Conversion/ComplexToLLVM/full-conversion.mlir
index b7756b3be543f..deae4f618f789 100644
--- a/mlir/test/Conversion/ComplexToLLVM/full-conversion.mlir
+++ b/mlir/test/Conversion/ComplexToLLVM/full-conversion.mlir
@@ -11,7 +11,7 @@ func.func @complex_div(%lhs: complex<f32>, %rhs: complex<f32>) -> complex<f32> {
 // CHECK: %[[RHS_RE:.*]] = llvm.extractvalue %[[RHS]][0] : ![[C_TY]]
 // CHECK: %[[RHS_IM:.*]] = llvm.extractvalue %[[RHS]][1] : ![[C_TY]]
 
-// CHECK: %[[RESULT_0:.*]] = llvm.mlir.undef : ![[C_TY]]
+// CHECK: %[[RESULT_0:.*]] = llvm.mlir.poison : ![[C_TY]]
 
 // CHECK-DAG: %[[RHS_RE_SQ:.*]] = llvm.fmul %[[RHS_RE]], %[[RHS_RE]]  : f32
 // CHECK-DAG: %[[RHS_IM_SQ:.*]] = llvm.fmul %[[RHS_IM]], %[[RHS_IM]]  : f32
@@ -41,7 +41,7 @@ func.func @complex_mul(%lhs: complex<f32>, %rhs: complex<f32>) -> complex<f32> {
 // CHECK: %[[LHS_IM:.*]] = llvm.extractvalue %[[LHS]][1] : ![[C_TY]]
 // CHECK: %[[RHS_RE:.*]] = llvm.extractvalue %[[RHS]][0] : ![[C_TY]]
 // CHECK: %[[RHS_IM:.*]] = llvm.extractvalue %[[RHS]][1] : ![[C_TY]]
-// CHECK: %[[RESULT_0:.*]] = llvm.mlir.undef : ![[C_TY]]
+// CHECK: %[[RESULT_0:.*]] = llvm.mlir.poison : ![[C_TY]]
 
 // CHECK-DAG: %[[REAL_TMP_0:.*]] = llvm.fmul %[[RHS_RE]], %[[LHS_RE]]  : f32
 // CHECK-DAG: %[[REAL_TMP_1:.*]] = llvm.fmul %[[RHS_IM]], %[[LHS_IM]]  : f32
diff --git a/mlir/test/Conversion/ConvertToSPIRV/argmax-kernel.mlir b/mlir/test/Conversion/ConvertToSPIRV/argmax-kernel.mlir
index 652f447228086..81ac2a04f3362 100644
--- a/mlir/test/Conversion/ConvertToSPIRV/argmax-kernel.mlir
+++ b/mlir/test/Conversion/ConvertToSPIRV/argmax-kernel.mlir
@@ -1,4 +1,4 @@
-// RUN: mlir-opt -convert-to-spirv -cse %s | FileCheck %s
+// RUN: mlir-opt -test-convert-to-spirv -cse %s | FileCheck %s
 
 module attributes {
   gpu.container_module,
diff --git a/mlir/test/Conversion/ConvertToSPIRV/arith.mlir b/mlir/test/Conversion/ConvertToSPIRV/arith.mlir
index 6418e931f7460..d9a33a1f7b6e8 100644
--- a/mlir/test/Conversion/ConvertToSPIRV/arith.mlir
+++ b/mlir/test/Conversion/ConvertToSPIRV/arith.mlir
@@ -1,4 +1,4 @@
-// RUN: mlir-opt -convert-to-spirv="run-signature-conversion=false run-vector-unrolling=false" -split-input-file %s | FileCheck %s
+// RUN: mlir-opt -test-convert-to-spirv="run-signature-conversion=false run-vector-unrolling=false" -split-input-file %s | FileCheck %s
 
 //===----------------------------------------------------------------------===//
 // arithmetic ops
diff --git a/mlir/test/Conversion/ConvertToSPIRV/combined.mlir b/mlir/test/Conversion/ConvertToSPIRV/combined.mlir
index 311174bef15ed..3039ed58463ea 100644
--- a/mlir/test/Conversion/ConvertToSPIRV/combined.mlir
+++ b/mlir/test/Conversion/ConvertToSPIRV/combined.mlir
@@ -1,4 +1,4 @@
-// RUN: mlir-opt -convert-to-spirv="run-signature-conversion=false run-vector-unrolling=false" -split-input-file %s | FileCheck %s
+// RUN: mlir-opt -test-convert-to-spirv="run-signature-conversion=false run-vector-unrolling=false" -split-input-file %s | FileCheck %s
 
 // CHECK-LABEL: @combined
 // CHECK: %[[C0_F32:.*]] = spirv.Constant 0.000000e+00 : f32
diff --git a/mlir/test/Conversion/ConvertToSPIRV/convert-gpu-modules-nested.mlir b/mlir/test/Conversion/ConvertToSPIRV/convert-gpu-modules-nested.mlir
index 7562de17c606e..5b12a656a2bf8 100644
--- a/mlir/test/Conversion/ConvertToSPIRV/convert-gpu-modules-nested.mlir
+++ b/mlir/test/Conversion/ConvertToSPIRV/convert-gpu-modules-nested.mlir
@@ -1,4 +1,4 @@
-// RUN: mlir-opt -convert-to-spirv="convert-gpu-modules=true nest-in-gpu-module=true run-signature-conversion=false run-vector-unrolling=false" %s | FileCheck %s
+// RUN: mlir-opt -test-convert-to-spirv="convert-gpu-modules=true nest-in-gpu-module=true run-signature-conversion=false run-vector-unrolling=false" %s | FileCheck %s
 
 module attributes {
   gpu.container_module,
diff --git a/mlir/test/Conversion/ConvertToSPIRV/convert-gpu-modules.mlir b/mlir/test/Conversion/ConvertToSPIRV/convert-gpu-modules.mlir
index 1fde6c34418fc..00bbd1c27116a 100644
--- a/mlir/test/Conversion/ConvertToSPIRV/convert-gpu-modules.mlir
+++ b/mlir/test/Conversion/ConvertToSPIRV/convert-gpu-modules.mlir
@@ -1,4 +1,4 @@
-// RUN: mlir-opt -convert-to-spirv="convert-gpu-modules=true run-signature-conversion=false run-vector-unrolling=false" -split-input-file %s | FileCheck %s
+// RUN: mlir-opt -test-convert-to-spirv="convert-gpu-modules=true run-signature-conversion=false run-vector-unrolling=false" -split-input-file %s | FileCheck %s
 
 module attributes {
   gpu.container_module,
diff --git a/mlir/test/Conversion/ConvertToSPIRV/gpu.mlir b/mlir/test/Conversion/ConvertToSPIRV/gpu.mlir
index 84f366e5874b0..3e197c076be6a 100644
--- a/mlir/test/Conversion/ConvertToSPIRV/gpu.mlir
+++ b/mlir/test/Conversion/ConvertToSPIRV/gpu.mlir
@@ -1,4 +1,4 @@
-// RUN: mlir-opt -convert-to-spirv -split-input-file %s | FileCheck %s
+// RUN: mlir-opt -test-convert-to-spirv -split-input-file %s | FileCheck %s
 
 module attributes {
   gpu.container_module,
diff --git a/mlir/test/Conversion/ConvertToSPIRV/index.mlir b/mlir/test/Conversion/ConvertToSPIRV/index.mlir
index f4b116849fa93..a7b892e358acb 100644
--- a/mlir/test/Conversion/ConvertToSPIRV/index.mlir
+++ b/mlir/test/Conversion/ConvertToSPIRV/index.mlir
@@ -1,4 +1,4 @@
-// RUN: mlir-opt -convert-to-spirv="run-signature-conversion=false run-vector-unrolling=false" -split-input-file %s | FileCheck %s
+// RUN: mlir-opt -test-convert-to-spirv="run-signature-conversion=false run-vector-unrolling=false" -split-input-file %s | FileCheck %s
 
 // CHECK-LABEL: @basic
 func.func @basic(%a: index, %b: index) {
diff --git a/mlir/test/Conversion/ConvertToSPIRV/memref.mlir b/mlir/test/Conversion/ConvertToSPIRV/memref.mlir
index 5af8bfc842ea1..f7d17d1ad16da 100644
--- a/mlir/test/Conversion/ConvertToSPIRV/memref.mlir
+++ b/mlir/test/Conversion/ConvertToSPIRV/memref.mlir
@@ -1,4 +1,4 @@
-// RUN: mlir-opt -convert-to-spirv="run-signature-conversion=false run-vector-unrolling=false" -cse %s | FileCheck %s
+// RUN: mlir-opt -test-convert-to-spirv="run-signature-conversion=false run-vector-unrolling=false" -cse %s | FileCheck %s
 
 module attributes {
   spirv.target_env = #spirv.target_env<
diff --git a/mlir/test/Conversion/ConvertToSPIRV/scf.mlir b/mlir/test/Conversion/ConvertToSPIRV/scf.mlir
index 350ebcfe57b15..fab45d4742850 100644
--- a/mlir/test/Conversion/ConvertToSPIRV/scf.mlir
+++ b/mlir/test/Conversion/ConvertToSPIRV/scf.mlir
@@ -1,4 +1,4 @@
-// RUN: mlir-opt -convert-to-spirv="run-signature-conversion=false run-vector-unrolling=false" -split-input-file %s | FileCheck %s
+// RUN: mlir-opt -test-convert-to-spirv="run-signature-conversion=false run-vector-unrolling=false" -split-input-file %s | FileCheck %s
 
 // CHECK-LABEL: @if_yield
 // CHECK: %[[VAR:.*]] = spirv.Variable : !spirv.ptr<f32, Function>
diff --git a/mlir/test/Conversion/ConvertToSPIRV/simple.mlir b/mlir/test/Conversion/ConvertToSPIRV/simple.mlir
index 00556140c3018..aa4a963d1d69a 100644
--- a/mlir/test/Conversion/ConvertToSPIRV/simple.mlir
+++ b/mlir/test/Conversion/ConvertToSPIRV/simple.mlir
@@ -1,4 +1,4 @@
-// RUN: mlir-opt -convert-to-spirv="run-signature-conversion=false run-vector-unrolling=false" -split-input-file %s | FileCheck %s
+// RUN: mlir-opt -test-convert-to-spirv="run-signature-conversion=false run-vector-unrolling=false" -split-input-file %s | FileCheck %s
 
 // CHECK-LABEL: @return_scalar
 // CHECK-SAME: %[[ARG0:.*]]: i32
diff --git a/mlir/test/Conversion/ConvertToSPIRV/ub.mlir b/mlir/test/Conversion/ConvertToSPIRV/ub.mlir
index f34ca01c94f00..e8ffcd4237ed2 100644
--- a/mlir/test/Conversion/ConvertToSPIRV/ub.mlir
+++ b/mlir/test/Conversion/ConvertToSPIRV/ub.mlir
@@ -1,4 +1,4 @@
-// RUN: mlir-opt -convert-to-spirv="run-signature-conversion=false run-vector-unrolling=false" -split-input-file %s | FileCheck %s
+// RUN: mlir-opt -test-convert-to-spirv="run-signature-conversion=false run-vector-unrolling=false" -split-input-file %s | FileCheck %s
 
 // CHECK-LABEL: @ub
 // CHECK: %[[UNDEF:.*]] = spirv.Undef : i32
diff --git a/mlir/test/Conversion/ConvertToSPIRV/vector-unroll.mlir b/mlir/test/Conversion/ConvertToSPIRV/vector-unroll.mlir
index 043f9422d8790..d68ba44ee8840 100644
--- a/mlir/test/Conversion/ConvertToSPIRV/vector-unroll.mlir
+++ b/mlir/test/Conversion/ConvertToSPIRV/vector-unroll.mlir
@@ -83,17 +83,17 @@ func.func @vaddi_reduction(%arg0 : vector<8xi32>, %arg1 : vector<8xi32>) -> (i32
 // CHECK-LABEL: @transpose
 // CHECK-SAME: (%[[ARG0:.+]]: vector<3xi32>, %[[ARG1:.+]]: vector<3xi32>)
 func.func @transpose(%arg0 : vector<2x3xi32>) -> (vector<3x2xi32>) {
-  // CHECK: %[[CST:.*]] = arith.constant dense<0> : vector<2xi32>
+  // CHECK: %[[UB:.*]] = ub.poison : vector<2xi32>
   // CHECK: %[[EXTRACT0:.*]] = vector.extract %[[ARG0]][0] : i32 from vector<3xi32>
-  // CHECK: %[[INSERT0:.*]]= vector.insert %[[EXTRACT0]], %[[CST]] [0] : i32 into vector<2xi32>
+  // CHECK: %[[INSERT0:.*]]= vector.insert %[[EXTRACT0]], %[[UB]] [0] : i32 into vector<2xi32>
   // CHECK: %[[EXTRACT1:.*]] = vector.extract %[[ARG1]][0] : i32 from vector<3xi32>
   // CHECK: %[[INSERT1:.*]] = vector.insert %[[EXTRACT1]], %[[INSERT0]][1] : i32 into vector<2xi32>
   // CHECK: %[[EXTRACT2:.*]] = vector.extract %[[ARG0]][1] : i32 from vector<3xi32>
-  // CHECK: %[[INSERT2:.*]] = vector.insert %[[EXTRACT2]], %[[CST]] [0] : i32 into vector<2xi32>
+  // CHECK: %[[INSERT2:.*]] = vector.insert %[[EXTRACT2]], %[[UB]] [0] : i32 into vector<2xi32>
   // CHECK: %[[EXTRACT3:.*]] = vector.extract %[[ARG1]][1] : i32 from vector<3xi32>
   // CHECK: %[[INSERT3:.*]] = vector.insert %[[EXTRACT3]], %[[INSERT2]] [1] : i32 into vector<2xi32>
   // CHECK: %[[EXTRACT4:.*]] = vector.extract %[[ARG0]][2] : i32 from vector<3xi32>
-  // CHECK: %[[INSERT4:.*]] = vector.insert %[[EXTRACT4]], %[[CST]] [0] : i32 into vector<2xi32>
+  // CHECK: %[[INSERT4:.*]] = vector.insert %[[EXTRACT4]], %[[UB]] [0] : i32 into vector<2xi32>
   // CHECK: %[[EXTRACT5:.*]] = vector.extract %[[ARG1]][2] : i32 from vector<3xi32>
   // CHECK: %[[INSERT5:.*]] = vector.insert %[[EXTRACT5]], %[[INSERT4]] [1] : i32 into vector<2xi32>
   // CHECK: return %[[INSERT1]], %[[INSERT3]], %[[INSERT5]] : vector<2xi32>, vector<2xi32>, vector<2xi32>
diff --git a/mlir/test/Conversion/ConvertToSPIRV/vector.mlir b/mlir/test/Conversion/ConvertToSPIRV/vector.mlir
index e369eadca5730..510f7a2d94c9e 100644
--- a/mlir/test/Conversion/ConvertToSPIRV/vector.mlir
+++ b/mlir/test/Conversion/ConvertToSPIRV/vector.mlir
@@ -1,4 +1,4 @@
-// RUN: mlir-opt -convert-to-spirv="run-signature-conversion=false run-vector-unrolling=false" -split-input-file %s | FileCheck %s
+// RUN: mlir-opt -test-convert-to-spirv="run-signature-conversion=false run-vector-unrolling=false" -split-input-file %s | FileCheck %s
 
 // CHECK-LABEL: @extract
 //  CHECK-SAME: %[[ARG:.+]]: vector<2xf32>
diff --git a/mlir/test/Conversion/FuncToLLVM/calling-convention.mlir b/mlir/test/Conversion/FuncToLLVM/calling-convention.mlir
index 18734d19e335c..058b69b8e3596 100644
--- a/mlir/test/Conversion/FuncToLLVM/calling-convention.mlir
+++ b/mlir/test/Conversion/FuncToLLVM/calling-convention.mlir
@@ -12,7 +12,7 @@
 // CHECK: %[[ALLOC1:.*]]: !llvm.ptr, %[[ALIGN1:.*]]: !llvm.ptr, %[[OFFSET1:.*]]: i64)
 func.func private @external(%arg0: memref<?x?xf32>, %arg1: memref<f32>)
   // Populate the descriptor for arg0.
-  // CHECK: %[[DESC00:.*]] = llvm.mlir.undef : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)>
+  // CHECK: %[[DESC00:.*]] = llvm.mlir.poison : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)>
   // CHECK: %[[DESC01:.*]] = llvm.insertvalue %arg0, %[[DESC00]][0]
   // CHECK: %[[DESC02:.*]] = llvm.insertvalue %arg1, %[[DESC01]][1]
   // CHECK: %[[DESC03:.*]] = llvm.insertvalue %arg2, %[[DESC02]][2]
@@ -27,7 +27,7 @@ func.func private @external(%arg0: memref<?x?xf32>, %arg1: memref<f32>)
   // CHECK: llvm.store %[[DESC07]], %[[DESC0_ALLOCA]]
 
   // Populate the descriptor for arg1.
-  // CHECK: %[[DESC10:.*]] = llvm.mlir.undef : !llvm.struct<(ptr, ptr, i64)>
+  // CHECK: %[[DESC10:.*]] = llvm.mlir.poison : !llvm.struct<(ptr, ptr, i64)>
   // CHECK: %[[DESC11:.*]] = llvm.insertvalue %arg7, %[[DESC10]][0] : !llvm.struct<(ptr, ptr, i64)>
   // CHECK: %[[DESC12:.*]] = llvm.insertvalue %arg8, %[[DESC11]][1] : !llvm.struct<(ptr, ptr, i64)>
   // CHECK: %[[DESC13:.*]] = llvm.insertvalue %arg9, %[[DESC12]][2] : !llvm.struct<(ptr, ptr, i64)>
@@ -135,7 +135,7 @@ func.func @return_var_memref_caller(%arg0: memref<4x3xf32>) {
   // CHECK: %[[SOURCE:.*]] = llvm.extractvalue %[[CALL_RES]][1]
   // CHECK: "llvm.intr.memcpy"(%[[ALLOCA]], %[[SOURCE]], %[[ALLOC_SIZE]]) <{isVolatile = false}>
   // CHECK: llvm.call @free(%[[SOURCE]])
-  // CHECK: %[[DESC:.*]] = llvm.mlir.undef : !llvm.struct<(i64, ptr)>
+  // CHECK: %[[DESC:.*]] = llvm.mlir.poison : !llvm.struct<(i64, ptr)>
   // CHECK: %[[RANK:.*]] = llvm.extractvalue %[[CALL_RES]][0] : !llvm.struct<(i64, ptr)>
   // CHECK: %[[DESC_1:.*]] = llvm.insertvalue %[[RANK]], %[[DESC]][0]
   // CHECK: llvm.insertvalue %[[ALLOCA]], %[[DESC_1]][1]
@@ -147,7 +147,7 @@ func.func @return_var_memref(%arg0: memref<4x3xf32>) -> memref<*xf32> attributes
   // Match the construction of the unranked descriptor.
   // CHECK: %[[ALLOCA:.*]] = llvm.alloca
   // CHECK: %[[RANK:.*]] = llvm.mlir.constant(2 : index)
-  // CHECK: %[[DESC_0:.*]] = llvm.mlir.undef : !llvm.struct<(i64, ptr)>
+  // CHECK: %[[DESC_0:.*]] = llvm.mlir.poison : !llvm.struct<(i64, ptr)>
   // CHECK: %[[DESC_1:.*]] = llvm.insertvalue %[[RANK]], %[[DESC_0]][0]
   // CHECK: %[[DESC_2:.*]] = llvm.insertvalue %[[ALLOCA]], %[[DESC_1]][1]
   %0 = memref.cast %arg0: memref<4x3xf32> to memref<*xf32>
@@ -167,7 +167,7 @@ func.func @return_var_memref(%arg0: memref<4x3xf32>) -> memref<*xf32> attributes
   // CHECK: %[[ALLOCATED:.*]] = llvm.call @malloc(%[[ALLOC_SIZE]])
   // CHECK: %[[ALLOCA_EXTRACTED:.*]] = llvm.extractvalue %[[DESC_2]][1]
   // CHECK: "llvm.intr.memcpy"(%[[ALLOCATED]], %[[ALLOCA_EXTRACTED]], %[[ALLOC_SIZE]]) <{isVolatile = false}>
-  // CHECK: %[[NEW_DESC:.*]] = llvm.mlir.undef : !llvm.struct<(i64, ptr)>
+  // CHECK: %[[NEW_DESC:.*]] = llvm.mlir.poison : !llvm.struct<(i64, ptr)>
   // CHECK: %[[RANK_EXTRACTED:.*]] = llvm.extractvalue %[[DESC_2]][0]
   // CHECK: %[[NEW_DESC_1:.*]] = llvm.insertvalue %[[RANK_EXTRACTED]], %[[NEW_DESC]][0]
   // CHECK: %[[NEW_DESC_2:.*]] = llvm.insertvalue %[[ALLOCATED]], %[[NEW_DESC_1]][1]
@@ -193,7 +193,7 @@ func.func @return_two_var_memref_caller(%arg0: memref<4x3xf32>) {
   // CHECK: %[[SOURCE_1:.*]] = llvm.extractvalue %[[RES_1:.*]][1] : ![[DESC_TYPE:.*>]]
   // CHECK: "llvm.intr.memcpy"(%[[ALLOCA_1]], %[[SOURCE_1]], %{{.*}}) <{isVolatile = false}>
   // CHECK: llvm.call @free(%[[SOURCE_1]])
-  // CHECK: %[[DESC_1:.*]] = llvm.mlir.undef : ![[DESC_TYPE]]
+  // CHECK: %[[DESC_1:.*]] = llvm.mlir.poison : ![[DESC_TYPE]]
   // CHECK: %[[DESC_11:.*]] = llvm.insertvalue %{{.*}}, %[[DESC_1]][0]
   // CHECK: llvm.insertvalue %[[ALLOCA_1]], %[[DESC_11]][1]
 
@@ -201,7 +201,7 @@ func.func @return_two_var_memref_caller(%arg0: memref<4x3xf32>) {
   // CHECK: %[[SOURCE_2:.*]] = llvm.extractvalue %[[RES_2:.*]][1]
   // CHECK: "llvm.intr.memcpy"(%[[ALLOCA_2]], %[[SOURCE_2]], %{{.*}}) <{isVolatile = false}>
   // CHECK: llvm.call @free(%[[SOURCE_2]])
-  // CHECK: %[[DESC_2:.*]] = llvm.mlir.undef : ![[DESC_TYPE]]
+  // CHECK: %[[DESC_2:.*]] = llvm.mlir.poison : ![[DESC_TYPE]]
   // CHECK: %[[DESC_21:.*]] = llvm.insertvalue %{{.*}}, %[[DESC_2]][0]
   // CHECK: llvm.insertvalue %[[ALLOCA_2]], %[[DESC_21]][1]
   return
@@ -211,7 +211,7 @@ func.func @return_two_var_memref_caller(%arg0: memref<4x3xf32>) {
 func.func @return_two_var_memref(%arg0: memref<4x3xf32>) -> (memref<*xf32>, memref<*xf32>) attributes { llvm.emit_c_interface } {
   // Match the construction of the unranked descriptor.
   // CHECK: %[[ALLOCA:.*]] = llvm.alloca
-  // CHECK: %[[DESC_0:.*]] = llvm.mlir.undef : !llvm.struct<(i64, ptr)>
+  // CHECK: %[[DESC_0:.*]] = llvm.mlir.poison : !llvm.struct<(i64, ptr)>
   // CHECK: %[[DESC_1:.*]] = llvm.insertvalue %{{.*}}, %[[DESC_0]][0]
   // CHECK: %[[DESC_2:.*]] = llvm.insertvalue %[[ALLOCA]], %[[DESC_1]][1]
   %0 = memref.cast %arg0 : memref<4x3xf32> to memref<*xf32>
@@ -223,18 +223,18 @@ func.func @return_two_var_memref(%arg0: memref<4x3xf32>) -> (memref<*xf32>, memr
   // CHECK: %[[ALLOCATED_1:.*]] = llvm.call @malloc(%{{.*}})
   // CHECK: %[[ALLOCA_EXTRACTED:.*]] = llvm.extractvalue %[[DESC_2]][1]
   // CHECK: "llvm.intr.memcpy"(%[[ALLOCATED_1]], %[[ALLOCA_EXTRACTED]], %{{.*}}) <{isVolatile = false}>
-  // CHECK: %[[RES_1:.*]] = llvm.mlir.undef
+  // CHECK: %[[RES_1:.*]] = llvm.mlir.poison
   // CHECK: %[[RES_11:.*]] = llvm.insertvalue %{{.*}}, %[[RES_1]][0]
   // CHECK: %[[RES_12:.*]] = llvm.insertvalue %[[ALLOCATED_1]], %[[RES_11]][1]
 
   // CHECK: %[[ALLOCATED_2:.*]] = llvm.call @malloc(%{{.*}})
   // CHECK: %[[ALLOCA_EXTRACTED:.*]] = llvm.extractvalue %[[DESC_2]][1]
   // CHECK: "llvm.intr.memcpy"(%[[ALLOCATED_2]], %[[ALLOCA_EXTRACTED]], %{{.*}}) <{isVolatile = false}>
-  // CHECK: %[[RES_2:.*]] = llvm.mlir.undef
+  // CHECK: %[[RES_2:.*]] = llvm.mlir.poison
   // CHECK: %[[RES_21:.*]] = llvm.insertvalue %{{.*}}, %[[RES_2]][0]
   // CHECK: %[[RES_22:.*]] = llvm.insertvalue %[[ALLOCATED_2]], %[[RES_21]][1]
 
-  // CHECK: %[[RESULTS:.*]] = llvm.mlir.undef : !llvm.struct<(struct<(i64, ptr)>, struct<(i64, ptr)>)>
+  // CHECK: %[[RESULTS:.*]] = llvm.mlir.poison : !llvm.struct<(struct<(i64, ptr)>, struct<(i64, ptr)>)>
   // CHECK: %[[RESULTS_1:.*]] = llvm.insertvalue %[[RES_12]], %[[RESULTS]]
   // CHECK: %[[RESULTS_2:.*]] = llvm.insertvalue %[[RES_22]], %[[RESULTS_1]]
   // CHECK: llvm.return %[[RESULTS_2]]
@@ -251,8 +251,8 @@ func.func @return_two_var_memref(%arg0: memref<4x3xf32>) -> (memref<*xf32>, memr
 // CHECK-SAME: -> !llvm.ptr
 func.func @bare_ptr_calling_conv(%arg0: memref<4x3xf32>, %arg1 : index, %arg2 : index, %arg3 : f32)
      -> (memref<4x3xf32>) attributes { llvm.bareptr } {
-  // CHECK: %[[UNDEF_DESC:.*]] = llvm.mlir.undef : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)>
-  // CHECK: %[[INSERT_ALLOCPTR:.*]] = llvm.insertvalue %[[ARG0]], %[[UNDEF_DESC]][0]
+  // CHECK: %[[POISON_DESC:.*]] = llvm.mlir.poison : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)>
+  // CHECK: %[[INSERT_ALLOCPTR:.*]] = llvm.insertvalue %[[ARG0]], %[[POISON_DESC]][0]
   // CHECK: %[[INSERT_ALIGNEDPTR:.*]] = llvm.insertvalue %[[ARG0]], %[[INSERT_ALLOCPTR]][1]
   // CHECK: %[[C0:.*]] = llvm.mlir.constant(0 : index) : i64
   // CHECK: %[[INSERT_OFFSET:.*]] = llvm.insertvalue %[[C0]], %[[INSERT_ALIGNEDPTR]][2]
@@ -280,8 +280,8 @@ func.func @bare_ptr_calling_conv(%arg0: memref<4x3xf32>, %arg1 : index, %arg2 :
 // CHECK-SAME: -> !llvm.struct<(f32, ptr)>
 func.func @bare_ptr_calling_conv_multiresult(%arg0: memref<4x3xf32>, %arg1 : index, %arg2 : index, %arg3 : f32)
      -> (f32, memref<4x3xf32>) attributes { llvm.bareptr } {
-  // CHECK: %[[UNDEF_DESC:.*]] = llvm.mlir.undef : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)>
-  // CHECK: %[[INSERT_ALLOCPTR:.*]] = llvm.insertvalue %[[ARG0]], %[[UNDEF_DESC]][0]
+  // CHECK: %[[POISON_DESC:.*]] = llvm.mlir.poison : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)>
+  // CHECK: %[[INSERT_ALLOCPTR:.*]] = llvm.insertvalue %[[ARG0]], %[[POISON_DESC]][0]
   // CHECK: %[[INSERT_ALIGNEDPTR:.*]] = llvm.insertvalue %[[ARG0]], %[[INSERT_ALLOCPTR]][1]
   // CHECK: %[[C0:.*]] = llvm.mlir.constant(0 : index) : i64
   // CHECK: %[[INSERT_OFFSET:.*]] = llvm.insertvalue %[[C0]], %[[INSERT_ALIGNEDPTR]][2]
@@ -305,7 +305,7 @@ func.func @bare_ptr_calling_conv_multiresult(%arg0: memref<4x3xf32>, %arg1 : ind
   %0 = memref.load %arg0[%arg1, %arg2] : memref<4x3xf32>
 
   // CHECK: %[[EXTRACT_MEMREF:.*]] = llvm.extractvalue %[[INSERT_STRIDE1]][0]
-  // CHECK: %[[RETURN_DESC:.*]] = llvm.mlir.undef : !llvm.struct<(f32, ptr)>
+  // CHECK: %[[RETURN_DESC:.*]] = llvm.mlir.poison : !llvm.struct<(f32, ptr)>
   // CHECK: %[[INSERT_RETURN0:.*]] = llvm.insertvalue %[[RETURN0]], %[[RETURN_DESC]][0]
   // CHECK: %[[INSERT_RETURN1:.*]] = llvm.insertvalue %[[EXTRACT_MEMREF]], %[[INSERT_RETURN0]][1]
   // CHECK: llvm.return %[[INSERT_RETURN1]]
diff --git a/mlir/test/Conversion/FuncToLLVM/func-memref-return.mlir b/mlir/test/Conversion/FuncToLLVM/func-memref-return.mlir
index 6b9df32fe02dd..bdb0092d155be 100644
--- a/mlir/test/Conversion/FuncToLLVM/func-memref-return.mlir
+++ b/mlir/test/Conversion/FuncToLLVM/func-memref-return.mlir
@@ -17,7 +17,7 @@
 func.func @check_static_return(%static : memref<32x18xf32>) -> memref<32x18xf32> {
 // CHECK:  llvm.return %{{.*}} : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)>
 
-// BAREPTR: %[[udf:.*]] = llvm.mlir.undef : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)>
+// BAREPTR: %[[udf:.*]] = llvm.mlir.poison : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)>
 // BAREPTR-NEXT: %[[base0:.*]] = llvm.insertvalue %[[arg]], %[[udf]][0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)>
 // BAREPTR-NEXT: %[[aligned:.*]] = llvm.insertvalue %[[arg]], %[[base0]][1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)>
 // BAREPTR-NEXT: %[[val0:.*]] = llvm.mlir.constant(0 : index) : i64
@@ -44,7 +44,7 @@ func.func @check_static_return(%static : memref<32x18xf32>) -> memref<32x18xf32>
 func.func @check_static_return_with_offset(%static : memref<32x18xf32, strided<[22,1], offset: 7>>) -> memref<32x18xf32, strided<[22,1], offset: 7>> {
 // CHECK:  llvm.return %{{.*}} : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)>
 
-// BAREPTR: %[[udf:.*]] = llvm.mlir.undef : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)>
+// BAREPTR: %[[udf:.*]] = llvm.mlir.poison : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)>
 // BAREPTR-NEXT: %[[base0:.*]] = llvm.insertvalue %[[arg]], %[[udf]][0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)>
 // BAREPTR-NEXT: %[[aligned:.*]] = llvm.insertvalue %[[arg]], %[[base0]][1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)>
 // BAREPTR-NEXT: %[[val0:.*]] = llvm.mlir.constant(7 : index) : i64
@@ -72,7 +72,7 @@ func.func @check_memref_func_call(%in : memref<10xi8>) -> memref<20xi8> {
   // BAREPTR:         %[[inDesc:.*]] = llvm.insertvalue %{{.*}}, %{{.*}}[4, 0]
   // BAREPTR-NEXT:    %[[barePtr:.*]] = llvm.extractvalue %[[inDesc]][1] : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)>
   // BAREPTR-NEXT:    %[[call:.*]] = llvm.call @foo(%[[barePtr]]) : (!llvm.ptr) -> !llvm.ptr
-  // BAREPTR-NEXT:    %[[desc0:.*]] = llvm.mlir.undef : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)>
+  // BAREPTR-NEXT:    %[[desc0:.*]] = llvm.mlir.poison : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)>
   // BAREPTR-NEXT:    %[[desc1:.*]] = llvm.insertvalue %[[call]], %[[desc0]][0] : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)>
   // BAREPTR-NEXT:    %[[desc2:.*]] = llvm.insertvalue %[[call]], %[[desc1]][1] : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)>
   // BAREPTR-NEXT:    %[[c0:.*]] = llvm.mlir.constant(0 : index) : i64
diff --git a/mlir/test/Conversion/FuncToLLVM/func-to-llvm.mlir b/mlir/test/Conversion/FuncToLLVM/func-to-llvm.mlir
index 22ac6eae73f53..2113557fbbb15 100644
--- a/mlir/test/Conversion/FuncToLLVM/func-to-llvm.mlir
+++ b/mlir/test/Conversion/FuncToLLVM/func-to-llvm.mlir
@@ -400,12 +400,12 @@ func.func @multireturn() -> (i64, f32, memref<42x?x10x?xf32>) {
   %0 = call @get_i64() : () -> (i64)
   %1 = call @get_f32() : () -> (f32)
   %2 = call @get_memref() : () -> (memref<42x?x10x?xf32>)
-// CHECK-NEXT:  {{.*}} = llvm.mlir.undef : !llvm.struct<(i64, f32, struct<(ptr, ptr, i64, array<4 x i64>, array<4 x i64>)>)>
+// CHECK-NEXT:  {{.*}} = llvm.mlir.poison : !llvm.struct<(i64, f32, struct<(ptr, ptr, i64, array<4 x i64>, array<4 x i64>)>)>
 // CHECK-NEXT:  {{.*}} = llvm.insertvalue {{.*}}, {{.*}}[0] : !llvm.struct<(i64, f32, struct<(ptr, ptr, i64, array<4 x i64>, array<4 x i64>)>)>
 // CHECK-NEXT:  {{.*}} = llvm.insertvalue {{.*}}, {{.*}}[1] : !llvm.struct<(i64, f32, struct<(ptr, ptr, i64, array<4 x i64>, array<4 x i64>)>)>
 // CHECK-NEXT:  {{.*}} = llvm.insertvalue {{.*}}, {{.*}}[2] : !llvm.struct<(i64, f32, struct<(ptr, ptr, i64, array<4 x i64>, array<4 x i64>)>)>
 // CHECK-NEXT:  llvm.return {{.*}} : !llvm.struct<(i64, f32, struct<(ptr, ptr, i64, array<4 x i64>, array<4 x i64>)>)>
-// CHECK32-NEXT:  {{.*}} = llvm.mlir.undef : !llvm.struct<(i64, f32, struct<(ptr, ptr, i32, array<4 x i32>, array<4 x i32>)>)>
+// CHECK32-NEXT:  {{.*}} = llvm.mlir.poison : !llvm.struct<(i64, f32, struct<(ptr, ptr, i32, array<4 x i32>, array<4 x i32>)>)>
 // CHECK32-NEXT:  {{.*}} = llvm.insertvalue {{.*}}, {{.*}}[0] : !llvm.struct<(i64, f32, struct<(ptr, ptr, i32, array<4 x i32>, array<4 x i32>)>)>
 // CHECK32-NEXT:  {{.*}} = llvm.insertvalue {{.*}}, {{.*}}[1] : !llvm.struct<(i64, f32, struct<(ptr, ptr, i32, array<4 x i32>, array<4 x i32>)>)>
 // CHECK32-NEXT:  {{.*}} = llvm.insertvalue {{.*}}, {{.*}}[2] : !llvm.struct<(i64, f32, struct<(ptr, ptr, i32, array<4 x i32>, array<4 x i32>)>)>
@@ -473,7 +473,7 @@ func.func @floorf(%arg0 : f32) {
   func.return
 }
 
-// Wrap the following tests in a module to control the place where 
+// Wrap the following tests in a module to control the place where
 // `llvm.func @abort()` is produced.
 module {
 // Lowers `cf.assert` to a function call to `abort` if the assertion is violated.
@@ -576,7 +576,7 @@ module attributes {transform.with_named_sequence} {
       transform.apply_conversion_patterns.memref.memref_to_llvm_type_converter
         {index_bitwidth = 32, use_opaque_pointers = true}
     } {
-      legal_dialects = ["llvm"], 
+      legal_dialects = ["llvm"],
       partial_conversion
     } : !transform.any_op
     transform.yield
diff --git a/mlir/test/Conversion/GPUCommon/memory-attrbution.mlir b/mlir/test/Conversion/GPUCommon/memory-attrbution.mlir
index 4fc19b8e93646..a3a6f53e5c0ad 100644
--- a/mlir/test/Conversion/GPUCommon/memory-attrbution.mlir
+++ b/mlir/test/Conversion/GPUCommon/memory-attrbution.mlir
@@ -12,7 +12,7 @@ gpu.module @kernel {
     // ROCDL: %[[raw:.*]] = llvm.alloca %[[size]] x f32 : (i64) -> !llvm.ptr<5>
 
     // Populate the memref descriptor.
-    // NVVM: %[[descr1:.*]] = llvm.mlir.undef : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)>
+    // NVVM: %[[descr1:.*]] = llvm.mlir.poison : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)>
     // NVVM: %[[descr2:.*]] = llvm.insertvalue %[[raw]], %[[descr1]][0]
     // NVVM: %[[descr3:.*]] = llvm.insertvalue %[[raw]], %[[descr2]][1]
     // NVVM: %[[c0:.*]] = llvm.mlir.constant(0 : index) : i64
@@ -22,7 +22,7 @@ gpu.module @kernel {
     // NVVM: %[[c1:.*]] = llvm.mlir.constant(1 : index) : i64
     // NVVM: %[[descr6:.*]] = llvm.insertvalue %[[c1]], %[[descr5]][4, 0]
 
-    // ROCDL: %[[descr1:.*]] = llvm.mlir.undef : !llvm.struct<(ptr<5>, ptr<5>, i64, array<1 x i64>, array<1 x i64>)>
+    // ROCDL: %[[descr1:.*]] = llvm.mlir.poison : !llvm.struct<(ptr<5>, ptr<5>, i64, array<1 x i64>, array<1 x i64>)>
     // ROCDL: %[[descr2:.*]] = llvm.insertvalue %[[raw]], %[[descr1]][0]
     // ROCDL: %[[descr3:.*]] = llvm.insertvalue %[[raw]], %[[descr2]][1]
     // ROCDL: %[[c0:.*]] = llvm.mlir.constant(0 : index) : i64
@@ -76,7 +76,7 @@ gpu.module @kernel {
     // ROCDL-SAME: !llvm.ptr<3>
 
     // Populate the memref descriptor.
-    // NVVM: %[[descr1:.*]] = llvm.mlir.undef : !llvm.struct<(ptr<3>, ptr<3>, i64, array<1 x i64>, array<1 x i64>)>
+    // NVVM: %[[descr1:.*]] = llvm.mlir.poison : !llvm.struct<(ptr<3>, ptr<3>, i64, array<1 x i64>, array<1 x i64>)>
     // NVVM: %[[descr2:.*]] = llvm.insertvalue %[[raw]], %[[descr1]][0]
     // NVVM: %[[descr3:.*]] = llvm.insertvalue %[[raw]], %[[descr2]][1]
     // NVVM: %[[c0:.*]] = llvm.mlir.constant(0 : index) : i64
@@ -86,7 +86,7 @@ gpu.module @kernel {
     // NVVM: %[[c1:.*]] = llvm.mlir.constant(1 : index) : i64
     // NVVM: %[[descr6:.*]] = llvm.insertvalue %[[c1]], %[[descr5]][4, 0]
 
-    // ROCDL: %[[descr1:.*]] = llvm.mlir.undef : !llvm.struct<(ptr<3>, ptr<3>, i64, array<1 x i64>, array<1 x i64>)>
+    // ROCDL: %[[descr1:.*]] = llvm.mlir.poison : !llvm.struct<(ptr<3>, ptr<3>, i64, array<1 x i64>, array<1 x i64>)>
     // ROCDL: %[[descr2:.*]] = llvm.insertvalue %[[raw]], %[[descr1]][0]
     // ROCDL: %[[descr3:.*]] = llvm.insertvalue %[[raw]], %[[descr2]][1]
     // ROCDL: %[[c0:.*]] = llvm.mlir.constant(0 : index) : i64
@@ -137,7 +137,7 @@ gpu.module @kernel {
     // ROCDL-SAME: !llvm.ptr<3>
 
     // Populate the memref descriptor.
-    // NVVM: %[[descr1:.*]] = llvm.mlir.undef : !llvm.struct<(ptr<3>, ptr<3>, i64, array<3 x i64>, array<3 x i64>)>
+    // NVVM: %[[descr1:.*]] = llvm.mlir.poison : !llvm.struct<(ptr<3>, ptr<3>, i64, array<3 x i64>, array<3 x i64>)>
     // NVVM: %[[descr2:.*]] = llvm.insertvalue %[[raw]], %[[descr1]][0]
     // NVVM: %[[descr3:.*]] = llvm.insertvalue %[[raw]], %[[descr2]][1]
     // NVVM: %[[c0:.*]] = llvm.mlir.constant(0 : index) : i64
@@ -155,7 +155,7 @@ gpu.module @kernel {
     // NVVM: %[[c1:.*]] = llvm.mlir.constant(1 : index) : i64
     // NVVM: %[[descr10:.*]] = llvm.insertvalue %[[c1]], %[[descr9]][4, 2]
 
-    // ROCDL: %[[descr1:.*]] = llvm.mlir.undef : !llvm.struct<(ptr<3>, ptr<3>, i64, array<3 x i64>, array<3 x i64>)>
+    // ROCDL: %[[descr1:.*]] = llvm.mlir.poison : !llvm.struct<(ptr<3>, ptr<3>, i64, array<3 x i64>, array<3 x i64>)>
     // ROCDL: %[[descr2:.*]] = llvm.insertvalue %[[raw]], %[[descr1]][0]
     // ROCDL: %[[descr3:.*]] = llvm.insertvalue %[[raw]], %[[descr2]][1]
     // ROCDL: %[[c0:.*]] = llvm.mlir.constant(0 : index) : i64
diff --git a/mlir/test/Conversion/GPUToLLVMSPV/gpu-to-llvm-spv.mlir b/mlir/test/Conversion/GPUToLLVMSPV/gpu-to-llvm-spv.mlir
index c2930971dbcf9..0c2c021b9d43c 100644
--- a/mlir/test/Conversion/GPUToLLVMSPV/gpu-to-llvm-spv.mlir
+++ b/mlir/test/Conversion/GPUToLLVMSPV/gpu-to-llvm-spv.mlir
@@ -415,8 +415,8 @@ gpu.module @kernels {
 
 // MemRef descriptor built from allocated pointer
 
-// CHECK-64:        %[[VAL_4:.*]] = llvm.mlir.undef : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)>
-// CHECK-32:        %[[VAL_4:.*]] = llvm.mlir.undef : !llvm.struct<(ptr, ptr, i32, array<1 x i32>, array<1 x i32>)>
+// CHECK-64:        %[[VAL_4:.*]] = llvm.mlir.poison : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)>
+// CHECK-32:        %[[VAL_4:.*]] = llvm.mlir.poison : !llvm.struct<(ptr, ptr, i32, array<1 x i32>, array<1 x i32>)>
 
 // CHECK:           %[[VAL_5:.*]] = llvm.insertvalue %[[VAL_3]], %[[VAL_4]][0]
 // CHECK:           llvm.insertvalue %[[VAL_3]], %[[VAL_5]][1]
@@ -426,8 +426,8 @@ gpu.module @kernels {
 // CHECK:           %[[VAL_14:.*]] = llvm.mlir.constant(16 : i64) : i64
 // CHECK:           %[[VAL_15:.*]] = llvm.alloca %[[VAL_14]] x i16 : (i64) -> !llvm.ptr
 
-// CHECK-64:        %[[VAL_16:.*]] = llvm.mlir.undef : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)>
-// CHECK-32:        %[[VAL_16:.*]] = llvm.mlir.undef : !llvm.struct<(ptr, ptr, i32, array<1 x i32>, array<1 x i32>)>
+// CHECK-64:        %[[VAL_16:.*]] = llvm.mlir.poison : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)>
+// CHECK-32:        %[[VAL_16:.*]] = llvm.mlir.poison : !llvm.struct<(ptr, ptr, i32, array<1 x i32>, array<1 x i32>)>
 
 // CHECK:           %[[VAL_17:.*]] = llvm.insertvalue %[[VAL_15]], %[[VAL_16]][0]
 // CHECK:           llvm.insertvalue %[[VAL_15]], %[[VAL_17]][1]
@@ -445,16 +445,16 @@ gpu.module @kernels {
 
 // MemRef descriptor built from new argument
 
-// CHECK-64:        %[[VAL_31:.*]] = llvm.mlir.undef : !llvm.struct<(ptr<3>, ptr<3>, i64, array<1 x i64>, array<1 x i64>)>
-// CHECK-32:        %[[VAL_31:.*]] = llvm.mlir.undef : !llvm.struct<(ptr<3>, ptr<3>, i32, array<1 x i32>, array<1 x i32>)>
+// CHECK-64:        %[[VAL_31:.*]] = llvm.mlir.poison : !llvm.struct<(ptr<3>, ptr<3>, i64, array<1 x i64>, array<1 x i64>)>
+// CHECK-32:        %[[VAL_31:.*]] = llvm.mlir.poison : !llvm.struct<(ptr<3>, ptr<3>, i32, array<1 x i32>, array<1 x i32>)>
 
 // CHECK:           %[[VAL_32:.*]] = llvm.insertvalue %[[VAL_29]], %[[VAL_31]][0]
 // CHECK:           llvm.insertvalue %[[VAL_29]], %[[VAL_32]][1]
 
 // Same as above
 
-// CHECK-64:        %[[VAL_41:.*]] = llvm.mlir.undef : !llvm.struct<(ptr<3>, ptr<3>, i64, array<1 x i64>, array<1 x i64>)>
-// CHECK-32:        %[[VAL_41:.*]] = llvm.mlir.undef : !llvm.struct<(ptr<3>, ptr<3>, i32, array<1 x i32>, array<1 x i32>)>
+// CHECK-64:        %[[VAL_41:.*]] = llvm.mlir.poison : !llvm.struct<(ptr<3>, ptr<3>, i64, array<1 x i64>, array<1 x i64>)>
+// CHECK-32:        %[[VAL_41:.*]] = llvm.mlir.poison : !llvm.struct<(ptr<3>, ptr<3>, i32, array<1 x i32>, array<1 x i32>)>
 
 // CHECK:           %[[VAL_42:.*]] = llvm.insertvalue %[[VAL_30]], %[[VAL_41]][0]
 // CHECK:           llvm.insertvalue %[[VAL_30]], %[[VAL_42]][1]
@@ -526,7 +526,7 @@ gpu.module @kernels {
     gpu.return
   }
 // CHECK-LABEL:   func.func @no_address_spaces_callee(
-// CHECK-SAME:                                             [[ARG0:%.*]]: memref<2x2xf32, 1> 
+// CHECK-SAME:                                             [[ARG0:%.*]]: memref<2x2xf32, 1>
 // CHECK-SAME:                                             [[ARG1:%.*]]: memref<4xf32, 1>
 // CHECK:         [[C0:%.*]] = llvm.mlir.constant(0 : i32) : i32
 // CHECK:         [[I0:%.*]] = llvm.call spir_funccc @_Z12get_group_idj([[C0]]) {
diff --git a/mlir/test/Conversion/GPUToNVVM/gpu-to-nvvm.mlir b/mlir/test/Conversion/GPUToNVVM/gpu-to-nvvm.mlir
index 94c0f9e34c29c..de2a4ff2079e2 100644
--- a/mlir/test/Conversion/GPUToNVVM/gpu-to-nvvm.mlir
+++ b/mlir/test/Conversion/GPUToNVVM/gpu-to-nvvm.mlir
@@ -569,7 +569,7 @@ gpu.module @test_module_27 {
   // CHECK-LABEL: func @gpu_unroll
   func.func @gpu_unroll(%arg0 : vector<4xf32>) -> vector<4xf32> {
     %result = math.exp %arg0 : vector<4xf32>
-    // CHECK: %[[V0:.+]] = llvm.mlir.undef : vector<4xf32>
+    // CHECK: %[[V0:.+]] = llvm.mlir.poison : vector<4xf32>
     // CHECK: %[[CL:.+]] = llvm.call @__nv_expf(%{{.*}}) : (f32) -> f32
     // CHECK: %[[V1:.+]] = llvm.insertelement %[[CL]], %[[V0]]
     // CHECK: %[[CL:.+]] = llvm.call @__nv_expf(%{{.*}}) : (f32) -> f32
@@ -729,13 +729,13 @@ gpu.module @test_module_33 {
 gpu.module @test_module_34 {
   // CHECK-LABEL: llvm.func @memref_signature(
   //  CHECK-SAME:     %{{.*}}: !llvm.ptr, %{{.*}}: !llvm.ptr, %{{.*}}: i64, %{{.*}}: i64, %{{.*}}: i64, %{{.*}}: f32) -> !llvm.struct<(struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)>, f32)>
-  //       CHECK:   llvm.mlir.undef
+  //       CHECK:   llvm.mlir.poison
   //       CHECK:   llvm.insertvalue
   //       CHECK:   llvm.insertvalue
   //       CHECK:   llvm.insertvalue
   //       CHECK:   llvm.insertvalue
   //       CHECK:   llvm.insertvalue
-  //       CHECK:   llvm.mlir.undef
+  //       CHECK:   llvm.mlir.poison
   //       CHECK:   llvm.insertvalue
   //       CHECK:   llvm.insertvalue
   //       CHECK:   llvm.return
diff --git a/mlir/test/Conversion/GPUToNVVM/wmma-ops-to-nvvm.mlir b/mlir/test/Conversion/GPUToNVVM/wmma-ops-to-nvvm.mlir
index 9dec666bf4b3d..b479467efc208 100644
--- a/mlir/test/Conversion/GPUToNVVM/wmma-ops-to-nvvm.mlir
+++ b/mlir/test/Conversion/GPUToNVVM/wmma-ops-to-nvvm.mlir
@@ -265,12 +265,12 @@ gpu.module @test_module {
 
 // CHECK-LABEL: func @gpu_wmma_constant_op
 //       CHECK: %[[CST:.+]] = llvm.mlir.constant(1.000000e+00 : f16) : f16
-//       CHECK: %[[V0:.+]] = llvm.mlir.undef : vector<2xf16>
+//       CHECK: %[[V0:.+]] = llvm.mlir.poison : vector<2xf16>
 //       CHECK: %[[C0:.+]] = llvm.mlir.constant(0 : i32) : i32
 //       CHECK: %[[V1:.+]] = llvm.insertelement %[[CST]], %[[V0]][%[[C0]] : i32] : vector<2xf16>
 //       CHECK: %[[C1:.+]] = llvm.mlir.constant(1 : i32) : i32
 //       CHECK: %[[V2:.+]] = llvm.insertelement %[[CST]], %[[V1]][%[[C1]] : i32] : vector<2xf16>
-//       CHECK: %[[M0:.+]] = llvm.mlir.undef : !llvm.struct<(vector<2xf16>, vector<2xf16>, vector<2xf16>, vector<2xf16>)>
+//       CHECK: %[[M0:.+]] = llvm.mlir.poison : !llvm.struct<(vector<2xf16>, vector<2xf16>, vector<2xf16>, vector<2xf16>)>
 //       CHECK: %[[M1:.+]] = llvm.insertvalue %[[V2]], %[[M0]][0] : !llvm.struct<(vector<2xf16>, vector<2xf16>, vector<2xf16>, vector<2xf16>)>
 //       CHECK: %[[M2:.+]] = llvm.insertvalue %[[V2]], %[[M1]][1] : !llvm.struct<(vector<2xf16>, vector<2xf16>, vector<2xf16>, vector<2xf16>)>
 //       CHECK: %[[M3:.+]] = llvm.insertvalue %[[V2]], %[[M2]][2] : !llvm.struct<(vector<2xf16>, vector<2xf16>, vector<2xf16>, vector<2xf16>)>
@@ -288,7 +288,7 @@ gpu.module @test_module {
 gpu.module @test_module {
 
 // CHECK-LABEL: func @gpu_wmma_elementwise
-//       CHECK: %[[M0:.*]] = llvm.mlir.undef : !llvm.struct<(vector<2xf16>, vector<2xf16>, vector<2xf16>, vector<2xf16>)>
+//       CHECK: %[[M0:.*]] = llvm.mlir.poison : !llvm.struct<(vector<2xf16>, vector<2xf16>, vector<2xf16>, vector<2xf16>)>
 //       CHECK: %[[A0:.*]] = llvm.extractvalue %{{.*}}[0] : !llvm.struct<(vector<2xf16>, vector<2xf16>, vector<2xf16>, vector<2xf16>)>
 //       CHECK: %[[B0:.*]] = llvm.extractvalue %{{.*}}[0] : !llvm.struct<(vector<2xf16>, vector<2xf16>, vector<2xf16>, vector<2xf16>)>
 //       CHECK: %[[C0:.*]] = llvm.fadd %[[A0]], %[[B0]]  : vector<2xf16>
@@ -306,7 +306,7 @@ gpu.module @test_module {
 //       CHECK: %[[C3:.*]] = llvm.fadd %[[A3]], %[[B3]]  : vector<2xf16>
 //       CHECK: %[[M4:.*]] = llvm.insertvalue %[[C3]], %[[M3]][3] : !llvm.struct<(vector<2xf16>, vector<2xf16>, vector<2xf16>, vector<2xf16>)>
 
-//       CHECK: %[[M0:.*]] = llvm.mlir.undef : !llvm.struct<(vector<2xf16>, vector<2xf16>, vector<2xf16>, vector<2xf16>)>
+//       CHECK: %[[M0:.*]] = llvm.mlir.poison : !llvm.struct<(vector<2xf16>, vector<2xf16>, vector<2xf16>, vector<2xf16>)>
 //       CHECK: %[[A0:.*]] = llvm.extractvalue %{{.*}}[0] : !llvm.struct<(vector<2xf16>, vector<2xf16>, vector<2xf16>, vector<2xf16>)>
 //       CHECK: %[[B0:.*]] = llvm.extractvalue %{{.*}}[0] : !llvm.struct<(vector<2xf16>, vector<2xf16>, vector<2xf16>, vector<2xf16>)>
 //       CHECK: %[[CMP0:.*]] = llvm.fcmp "ogt" %[[A0]], %[[B0]] : vector<2xf16>
diff --git a/mlir/test/Conversion/GPUToROCDL/gpu-to-rocdl.mlir b/mlir/test/Conversion/GPUToROCDL/gpu-to-rocdl.mlir
index 37a0c0067f7d3..11b9fa5e33f10 100644
--- a/mlir/test/Conversion/GPUToROCDL/gpu-to-rocdl.mlir
+++ b/mlir/test/Conversion/GPUToROCDL/gpu-to-rocdl.mlir
@@ -529,7 +529,7 @@ gpu.module @test_module {
   // CHECK-LABEL: func @gpu_unroll
   func.func @gpu_unroll(%arg0 : vector<4xf32>) -> vector<4xf32> {
     %result = math.sin %arg0 : vector<4xf32>
-    // CHECK: %[[V0:.+]] = llvm.mlir.undef : vector<4xf32>
+    // CHECK: %[[V0:.+]] = llvm.mlir.poison : vector<4xf32>
     // CHECK: %[[CL:.+]] = llvm.call @__ocml_sin_f32(%{{.*}}) : (f32) -> f32
     // CHECK: %[[V1:.+]] = llvm.insertelement %[[CL]], %[[V0]]
     // CHECK: %[[CL:.+]] = llvm.call @__ocml_sin_f32(%{{.*}}) : (f32) -> f32
diff --git a/mlir/test/Conversion/MathToLLVM/math-to-llvm.mlir b/mlir/test/Conversion/MathToLLVM/math-to-llvm.mlir
index 56129dbd27889..45a37af293890 100644
--- a/mlir/test/Conversion/MathToLLVM/math-to-llvm.mlir
+++ b/mlir/test/Conversion/MathToLLVM/math-to-llvm.mlir
@@ -161,11 +161,33 @@ func.func @rsqrt(%arg0 : f32) {
 
 // -----
 
-// CHECK-LABEL: func @sine(
-// CHECK-SAME: f32
-func.func @sine(%arg0 : f32) {
-  // CHECK: llvm.intr.sin(%arg0) : (f32) -> f32
+// CHECK-LABEL: func @trigonometrics
+// CHECK-SAME: [[ARG0:%.+]]: f32
+func.func @trigonometrics(%arg0: f32) {
+  // CHECK: llvm.intr.sin([[ARG0]]) : (f32) -> f32
   %0 = math.sin %arg0 : f32
+
+  // CHECK: llvm.intr.cos([[ARG0]]) : (f32) -> f32
+  %1 = math.cos %arg0 : f32
+
+  // CHECK: llvm.intr.tan([[ARG0]]) : (f32) -> f32
+  %2 = math.tan %arg0 : f32
+  func.return
+}
+
+// -----
+
+// CHECK-LABEL: func @hyperbolics
+// CHECK-SAME: [[ARG0:%.+]]: f32
+func.func @hyperbolics(%arg0: f32) {
+  // CHECK: llvm.intr.sinh([[ARG0]]) : (f32) -> f32
+  %0 = math.sinh %arg0 : f32
+
+  // CHECK: llvm.intr.cosh([[ARG0]]) : (f32) -> f32
+  %1 = math.cosh %arg0 : f32
+
+  // CHECK: llvm.intr.tanh([[ARG0]]) : (f32) -> f32
+  %2 = math.tanh %arg0 : f32
   func.return
 }
 
diff --git a/mlir/test/Conversion/MemRefToLLVM/convert-dynamic-memref-ops.mlir b/mlir/test/Conversion/MemRefToLLVM/convert-dynamic-memref-ops.mlir
index ebfc3b95d6eff..be3ddc20c17b7 100644
--- a/mlir/test/Conversion/MemRefToLLVM/convert-dynamic-memref-ops.mlir
+++ b/mlir/test/Conversion/MemRefToLLVM/convert-dynamic-memref-ops.mlir
@@ -15,7 +15,7 @@ func.func @mixed_alloc(%arg0: index, %arg1: index) -> memref<?x42x?xf32> {
 //  CHECK-NEXT:  %[[gep:.*]] = llvm.getelementptr %[[null]][%[[sz]]] : (!llvm.ptr, i64) -> !llvm.ptr, f32
 //  CHECK-NEXT:  %[[sz_bytes:.*]] = llvm.ptrtoint %[[gep]] : !llvm.ptr to i64
 //  CHECK-NEXT:  llvm.call @malloc(%[[sz_bytes]]) : (i64) -> !llvm.ptr
-//  CHECK-NEXT:  llvm.mlir.undef : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)>
+//  CHECK-NEXT:  llvm.mlir.poison : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)>
 //  CHECK-NEXT:  llvm.insertvalue %{{.*}}, %{{.*}}[0] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)>
 //  CHECK-NEXT:  llvm.insertvalue %{{.*}}, %{{.*}}[1] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)>
 //  CHECK-NEXT:  %[[off:.*]] = llvm.mlir.constant(0 : index) : i64
@@ -64,7 +64,7 @@ func.func @dynamic_alloc(%arg0: index, %arg1: index) -> memref<?x?xf32> {
 //  CHECK-NEXT:  %[[gep:.*]] = llvm.getelementptr %[[null]][%[[sz]]] : (!llvm.ptr, i64) -> !llvm.ptr, f32
 //  CHECK-NEXT:  %[[sz_bytes:.*]] = llvm.ptrtoint %[[gep]] : !llvm.ptr to i64
 //  CHECK-NEXT:  llvm.call @malloc(%[[sz_bytes]]) : (i64) -> !llvm.ptr
-//  CHECK-NEXT:  llvm.mlir.undef : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)>
+//  CHECK-NEXT:  llvm.mlir.poison : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)>
 //  CHECK-NEXT:  llvm.insertvalue %{{.*}}, %{{.*}}[0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)>
 //  CHECK-NEXT:  llvm.insertvalue %{{.*}}, %{{.*}}[1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)>
 //  CHECK-NEXT:  %[[off:.*]] = llvm.mlir.constant(0 : index) : i64
@@ -87,7 +87,7 @@ func.func @dynamic_alloca(%arg0: index, %arg1: index) -> memref<?x?xf32> {
 //  CHECK-NEXT:  %[[st1:.*]] = llvm.mlir.constant(1 : index) : i64
 //  CHECK-NEXT:  %[[num_elems:.*]] = llvm.mul %[[N]], %[[M]] : i64
 //  CHECK-NEXT:  %[[allocated:.*]] = llvm.alloca %[[num_elems]] x f32 : (i64) -> !llvm.ptr
-//  CHECK-NEXT:  llvm.mlir.undef : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)>
+//  CHECK-NEXT:  llvm.mlir.poison : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)>
 //  CHECK-NEXT:  llvm.insertvalue %[[allocated]], %{{.*}}[0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)>
 //  CHECK-NEXT:  llvm.insertvalue %[[allocated]], %{{.*}}[1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)>
 //  CHECK-NEXT:  %[[off:.*]] = llvm.mlir.constant(0 : index) : i64
@@ -102,7 +102,7 @@ func.func @dynamic_alloca(%arg0: index, %arg1: index) -> memref<?x?xf32> {
 // alignment. The same pointer is thus used for allocation and aligned
 // accesses.
 // CHECK: %[[alloca_aligned:.*]] = llvm.alloca %{{.*}} x f32 {alignment = 32 : i64} : (i64) -> !llvm.ptr
-// CHECK: %[[desc:.*]] = llvm.mlir.undef : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)>
+// CHECK: %[[desc:.*]] = llvm.mlir.poison : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)>
 // CHECK: %[[desc1:.*]] = llvm.insertvalue %[[alloca_aligned]], %[[desc]][0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)>
 // CHECK: llvm.insertvalue %[[alloca_aligned]], %[[desc1]][1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)>
   memref.alloca(%arg0, %arg1) {alignment = 32} : memref<?x?xf32>
@@ -272,7 +272,7 @@ module attributes { dlti.dl_spec = #dlti.dl_spec<
 // CHECK: [[INPUT:%.*]] = builtin.unrealized_conversion_cast %{{.*}} to !llvm.struct<(i64, ptr)>
 // CHECK: [[RANK:%.*]] = llvm.extractvalue [[INPUT]][0] : !llvm.struct<(i64, ptr)>
 // CHECK: [[SOURCE_DESC:%.*]] = llvm.extractvalue [[INPUT]][1]
-// CHECK: [[RESULT_0:%.*]] = llvm.mlir.undef : !llvm.struct<(i64, ptr)>
+// CHECK: [[RESULT_0:%.*]] = llvm.mlir.poison : !llvm.struct<(i64, ptr)>
 // CHECK: [[RESULT_1:%.*]] = llvm.insertvalue [[RANK]], [[RESULT_0]][0] : !llvm.struct<(i64, ptr)>
 
 // Compute size in bytes to allocate result ranked descriptor
@@ -378,14 +378,14 @@ func.func @memref_cast_ranked_to_unranked(%arg : memref<42x2x?xf32>) {
 // CHECK-DAG:  %[[p:.*]] = llvm.alloca %[[c]] x !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> : (i64) -> !llvm.ptr
 // CHECK-DAG:  llvm.store %{{.*}}, %[[p]] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)>, !llvm.ptr
 // CHECK-DAG:  %[[r:.*]] = llvm.mlir.constant(3 : index) : i64
-//     CHECK:  llvm.mlir.undef : !llvm.struct<(i64, ptr)>
+//     CHECK:  llvm.mlir.poison : !llvm.struct<(i64, ptr)>
 // CHECK-DAG:  llvm.insertvalue %[[r]], %{{.*}}[0] : !llvm.struct<(i64, ptr)>
 // CHECK-DAG:  llvm.insertvalue %[[p]], %{{.*}}[1] : !llvm.struct<(i64, ptr)>
 // CHECK32-DAG:  %[[c:.*]] = llvm.mlir.constant(1 : index) : i64
 // CHECK32-DAG:  %[[p:.*]] = llvm.alloca %[[c]] x !llvm.struct<(ptr, ptr, i32, array<3 x i32>, array<3 x i32>)> : (i64) -> !llvm.ptr
 // CHECK32-DAG:  llvm.store %{{.*}}, %[[p]] : !llvm.struct<(ptr, ptr, i32, array<3 x i32>, array<3 x i32>)>, !llvm.ptr
 // CHECK32-DAG:  %[[r:.*]] = llvm.mlir.constant(3 : index) : i32
-//     CHECK32:  llvm.mlir.undef : !llvm.struct<(i32, ptr)>
+//     CHECK32:  llvm.mlir.poison : !llvm.struct<(i32, ptr)>
 // CHECK32-DAG:  llvm.insertvalue %[[r]], %{{.*}}[0] : !llvm.struct<(i32, ptr)>
 // CHECK32-DAG:  llvm.insertvalue %[[p]], %{{.*}}[1] : !llvm.struct<(i32, ptr)>
   %0 = memref.cast %arg : memref<42x2x?xf32> to memref<*xf32>
@@ -450,7 +450,7 @@ func.func @memref_reinterpret_cast_ranked_to_static_shape(%input : memref<2x3xf3
 }
 // CHECK: [[INPUT:%.*]] = builtin.unrealized_conversion_cast %{{.*}} :
 // CHECK: to [[TY:!.*]]
-// CHECK: [[OUT_0:%.*]] = llvm.mlir.undef : [[TY]]
+// CHECK: [[OUT_0:%.*]] = llvm.mlir.poison : [[TY]]
 // CHECK: [[BASE_PTR:%.*]] = llvm.extractvalue [[INPUT]][0] : [[TY]]
 // CHECK: [[ALIGNED_PTR:%.*]] = llvm.extractvalue [[INPUT]][1] : [[TY]]
 // CHECK: [[OUT_1:%.*]] = llvm.insertvalue [[BASE_PTR]], [[OUT_0]][0] : [[TY]]
@@ -490,7 +490,7 @@ func.func @memref_reinterpret_cast_unranked_to_dynamic_shape(%offset: index,
 // CHECK-DAG: [[STRIDE_0:%.*]] = builtin.unrealized_conversion_cast [[STRIDE_0arg]]
 // CHECK-DAG: [[STRIDE_1:%.*]] = builtin.unrealized_conversion_cast [[STRIDE_1arg]]
 // CHECK-DAG: [[INPUT:%.*]] = builtin.unrealized_conversion_cast
-// CHECK: [[OUT_0:%.*]] = llvm.mlir.undef : [[TY:!.*]]
+// CHECK: [[OUT_0:%.*]] = llvm.mlir.poison : [[TY:!.*]]
 // CHECK: [[DESCRIPTOR:%.*]] = llvm.extractvalue [[INPUT]][1] : !llvm.struct<(i64, ptr)>
 // CHECK: [[BASE_PTR:%.*]] = llvm.load [[DESCRIPTOR]] : !llvm.ptr -> !llvm.ptr
 // CHECK: [[ALIGNED_PTR_PTR:%.*]] = llvm.getelementptr [[DESCRIPTOR]]{{\[}}1]
@@ -516,7 +516,7 @@ func.func @memref_reshape(%input : memref<2x3xf32>, %shape : memref<?xindex>) {
 // CHECK-DAG: [[INPUT:%.*]] = builtin.unrealized_conversion_cast %[[ARG0]] : {{.*}} to [[INPUT_TY:!.*]]
 // CHECK-DAG: [[SHAPE:%.*]] = builtin.unrealized_conversion_cast %[[ARG1]] : {{.*}} to [[SHAPE_TY:!.*]]
 // CHECK: [[RANK:%.*]] = llvm.extractvalue [[SHAPE]][3, 0] : [[SHAPE_TY]]
-// CHECK: [[UNRANKED_OUT_O:%.*]] = llvm.mlir.undef : !llvm.struct<(i64, ptr)>
+// CHECK: [[UNRANKED_OUT_O:%.*]] = llvm.mlir.poison : !llvm.struct<(i64, ptr)>
 // CHECK: [[UNRANKED_OUT_1:%.*]] = llvm.insertvalue [[RANK]], [[UNRANKED_OUT_O]][0] : !llvm.struct<(i64, ptr)>
 
 // Compute size in bytes to allocate result ranked descriptor
@@ -581,7 +581,7 @@ func.func @memref_of_memref() {
 
   // Check that the types are converted as expected.
   // ALIGNED-ALLOC: llvm.call @aligned_alloc
-  // ALIGNED-ALLOC: llvm.mlir.undef
+  // ALIGNED-ALLOC: llvm.mlir.poison
   // ALIGNED-ALLOC-SAME: !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)>
   %0 = memref.alloc() : memref<1xmemref<1xf32>>
   return
@@ -603,7 +603,7 @@ module attributes { dlti.dl_spec = #dlti.dl_spec<#dlti.dl_entry<index, 32>> } {
 
     // Check that the types are converted as expected.
     // ALIGNED-ALLOC: llvm.call @aligned_alloc
-    // ALIGNED-ALLOC: llvm.mlir.undef
+    // ALIGNED-ALLOC: llvm.mlir.poison
     // ALIGNED-ALLOC-SAME: !llvm.struct<(ptr, ptr, i32, array<1 x i32>, array<1 x i32>)>
     %0 = memref.alloc() : memref<1xmemref<1xf32>>
     return
diff --git a/mlir/test/Conversion/MemRefToLLVM/convert-static-memref-ops.mlir b/mlir/test/Conversion/MemRefToLLVM/convert-static-memref-ops.mlir
index 4f4ce0d0ed978..0a92c7cf7b216 100644
--- a/mlir/test/Conversion/MemRefToLLVM/convert-static-memref-ops.mlir
+++ b/mlir/test/Conversion/MemRefToLLVM/convert-static-memref-ops.mlir
@@ -7,7 +7,7 @@ func.func @zero_d_alloc() -> memref<f32> {
 // CHECK: %[[gep:.*]] = llvm.getelementptr %[[null]][%[[one]]] : (!llvm.ptr, i64) -> !llvm.ptr, f32
 // CHECK: %[[size_bytes:.*]] = llvm.ptrtoint %[[gep]] : !llvm.ptr to i64
 // CHECK: %[[ptr:.*]] = llvm.call @malloc(%[[size_bytes]]) : (i64) -> !llvm.ptr
-// CHECK: llvm.mlir.undef : !llvm.struct<(ptr, ptr, i64)>
+// CHECK: llvm.mlir.poison : !llvm.struct<(ptr, ptr, i64)>
 // CHECK: llvm.insertvalue %[[ptr]], %{{.*}}[0] : !llvm.struct<(ptr, ptr, i64)>
 // CHECK: llvm.insertvalue %[[ptr]], %{{.*}}[1] : !llvm.struct<(ptr, ptr, i64)>
 // CHECK: %[[c0:.*]] = llvm.mlir.constant(0 : index) : i64
@@ -49,7 +49,7 @@ func.func @aligned_1d_alloc() -> memref<42xf32> {
 // CHECK: %[[mod:.*]] = llvm.urem %[[bumped]], %[[alignment]] : i64
 // CHECK: %[[aligned:.*]] = llvm.sub %[[bumped]], %[[mod]] : i64
 // CHECK: %[[alignedBitCast:.*]] = llvm.inttoptr %[[aligned]] : i64 to !llvm.ptr
-// CHECK: llvm.mlir.undef : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)>
+// CHECK: llvm.mlir.poison : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)>
 // CHECK: llvm.insertvalue %[[ptr]], %{{.*}}[0] : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)>
 // CHECK: llvm.insertvalue %[[alignedBitCast]], %{{.*}}[1] : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)>
 // CHECK: %[[c0:.*]] = llvm.mlir.constant(0 : index) : i64
@@ -86,7 +86,7 @@ func.func @static_alloca() -> memref<32x18xf32> {
  // alignment. The same pointer is thus used for allocation and aligned
  // accesses.
  // CHECK: %[[alloca_aligned:.*]] = llvm.alloca %{{.*}} x f32 {alignment = 32 : i64} : (i64) -> !llvm.ptr
- // CHECK: %[[desc:.*]] = llvm.mlir.undef : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)>
+ // CHECK: %[[desc:.*]] = llvm.mlir.poison : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)>
  // CHECK: %[[desc1:.*]] = llvm.insertvalue %[[alloca_aligned]], %[[desc]][0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)>
  // CHECK: llvm.insertvalue %[[alloca_aligned]], %[[desc1]][1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)>
  memref.alloca() {alignment = 32} : memref<32x18xf32>
@@ -247,7 +247,7 @@ func.func @memref.reshape(%arg0: memref<4x5x6xf32>) -> memref<2x6x20xf32> {
   // CHECK: %[[cast0:.*]] = builtin.unrealized_conversion_cast %arg0 : memref<4x5x6xf32> to !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)>
   %0 = memref.get_global @__constant_3xi64 : memref<3xi64>
 
-  // CHECK: %[[undef:.*]] = llvm.mlir.undef : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)>
+  // CHECK: %[[undef:.*]] = llvm.mlir.poison : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)>
   // CHECK: %[[elem0:.*]] = llvm.extractvalue %[[cast0]][0] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)>
   // CHECK: %[[elem1:.*]] = llvm.extractvalue %[[cast0]][1] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)>
   // CHECK: %[[insert0:.*]] = llvm.insertvalue %[[elem0]], %[[undef]][0] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)>
@@ -285,7 +285,7 @@ func.func @memref.reshape(%arg0: memref<4x5x6xf32>) -> memref<2x6x20xf32> {
 func.func @memref.reshape.dynamic.dim(%arg: memref<?x?x?xf32>, %shape: memref<4xi64>) -> memref<?x?x12x32xf32> {
   // CHECK-DAG: %[[arg_cast:.*]] = builtin.unrealized_conversion_cast %[[arg]] : memref<?x?x?xf32> to !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)>
   // CHECK-DAG: %[[shape_cast:.*]] = builtin.unrealized_conversion_cast %[[shape]] : memref<4xi64> to !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)>
-  // CHECK: %[[undef:.*]] = llvm.mlir.undef : !llvm.struct<(ptr, ptr, i64, array<4 x i64>, array<4 x i64>)>
+  // CHECK: %[[undef:.*]] = llvm.mlir.poison : !llvm.struct<(ptr, ptr, i64, array<4 x i64>, array<4 x i64>)>
   // CHECK: %[[alloc_ptr:.*]] = llvm.extractvalue %[[arg_cast]][0] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)>
   // CHECK: %[[align_ptr:.*]] = llvm.extractvalue %[[arg_cast]][1] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)>
   // CHECK: %[[insert0:.*]] = llvm.insertvalue %[[alloc_ptr]], %[[undef]][0] : !llvm.struct<(ptr, ptr, i64, array<4 x i64>, array<4 x i64>)>
@@ -334,7 +334,7 @@ func.func @memref.reshape.dynamic.dim(%arg: memref<?x?x?xf32>, %shape: memref<4x
 func.func @memref.reshape_index(%arg0: memref<?x?xi32>, %shape: memref<1xindex>) ->  memref<?xi32> {
   // CHECK-DAG: %[[arg_cast:.*]] = builtin.unrealized_conversion_cast %[[arg]] : memref<?x?xi32> to !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)>
   // CHECK-DAG: %[[shape_cast:.*]] = builtin.unrealized_conversion_cast %[[shape]] : memref<1xindex> to !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)>
-  // CHECK: %[[undef:.*]] = llvm.mlir.undef : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)>
+  // CHECK: %[[undef:.*]] = llvm.mlir.poison : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)>
   // CHECK: %[[alloc_ptr:.*]] = llvm.extractvalue %[[arg_cast]][0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)>
   // CHECK: %[[align_ptr:.*]] = llvm.extractvalue %[[arg_cast]][1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)>
   // CHECK: %[[insert0:.*]] = llvm.insertvalue %[[alloc_ptr]], %[[undef:.*]][0] : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)>
@@ -372,7 +372,7 @@ func.func @memref_memory_space_cast(%input : memref<?xf32>) -> memref<?xf32, 1>
 // CHECK: [[STRIDE:%.*]] = llvm.extractvalue [[INPUT]][4, 0]
 // CHECK: [[CAST_ALLOC:%.*]] = llvm.addrspacecast [[ALLOC]] : !llvm.ptr to !llvm.ptr<1>
 // CHECK: [[CAST_ALIGN:%.*]] = llvm.addrspacecast [[ALIGN]] : !llvm.ptr to !llvm.ptr<1>
-// CHECK: [[RESULT_0:%.*]] = llvm.mlir.undef
+// CHECK: [[RESULT_0:%.*]] = llvm.mlir.poison
 // CHECK: [[RESULT_1:%.*]] = llvm.insertvalue [[CAST_ALLOC]], [[RESULT_0]][0]
 // CHECK: [[RESULT_2:%.*]] = llvm.insertvalue [[CAST_ALIGN]], [[RESULT_1]][1]
 // CHECK: [[RESULT_3:%.*]] = llvm.insertvalue [[OFFSET]], [[RESULT_2]][2]
diff --git a/mlir/test/Conversion/MemRefToLLVM/expand-then-convert-to-llvm.mlir b/mlir/test/Conversion/MemRefToLLVM/expand-then-convert-to-llvm.mlir
index 1fe4217cde982..5517eafb588e8 100644
--- a/mlir/test/Conversion/MemRefToLLVM/expand-then-convert-to-llvm.mlir
+++ b/mlir/test/Conversion/MemRefToLLVM/expand-then-convert-to-llvm.mlir
@@ -65,7 +65,7 @@ func.func @subview(%0 : memref<64x4xf32, strided<[4, 1], offset: 0>>, %arg0 : in
   // CHECK: %[[OFF2:.*]] = llvm.add %[[DESCSTRIDE0]], %[[ARG1]] : i64
   // CHECK: %[[TMP:.*]] = builtin.unrealized_conversion_cast %[[OFF2]] : i64 to index
   // CHECK: %[[OFF2:.*]] = builtin.unrealized_conversion_cast %[[TMP]] : index to i64
-  // CHECK: %[[DESC:.*]] = llvm.mlir.undef : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)>
+  // CHECK: %[[DESC:.*]] = llvm.mlir.poison : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)>
   // CHECK: %[[DESC0:.*]] = llvm.insertvalue %[[BASE]], %[[DESC]][0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)>
   // CHECK: %[[DESC1:.*]] = llvm.insertvalue %[[BASE_ALIGNED]], %[[DESC0]][1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)>
   // CHECK: %[[DESC2:.*]] = llvm.insertvalue %[[OFF2]], %[[DESC1]][2] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)>
@@ -101,7 +101,7 @@ func.func @subview_non_zero_addrspace(%0 : memref<64x4xf32, strided<[4, 1], offs
   // CHECK: %[[OFF2:.*]] = llvm.add %[[DESCSTRIDE0]], %[[ARG1]] : i64
   // CHECK: %[[TMP:.*]] = builtin.unrealized_conversion_cast %[[OFF2]] : i64 to index
   // CHECK: %[[OFF2:.*]] = builtin.unrealized_conversion_cast %[[TMP]] : index to i64
-  // CHECK: %[[DESC:.*]] = llvm.mlir.undef : !llvm.struct<(ptr<3>, ptr<3>, i64, array<2 x i64>, array<2 x i64>)>
+  // CHECK: %[[DESC:.*]] = llvm.mlir.poison : !llvm.struct<(ptr<3>, ptr<3>, i64, array<2 x i64>, array<2 x i64>)>
   // CHECK: %[[DESC0:.*]] = llvm.insertvalue %[[BASE]], %[[DESC]][0] : !llvm.struct<(ptr<3>, ptr<3>, i64, array<2 x i64>, array<2 x i64>)>
   // CHECK: %[[DESC1:.*]] = llvm.insertvalue %[[BASE_ALIGNED]], %[[DESC0]][1] : !llvm.struct<(ptr<3>, ptr<3>, i64, array<2 x i64>, array<2 x i64>)>
   // CHECK: %[[DESC2:.*]] = llvm.insertvalue %[[OFF2]], %[[DESC1]][2] : !llvm.struct<(ptr<3>, ptr<3>, i64, array<2 x i64>, array<2 x i64>)>
@@ -137,7 +137,7 @@ func.func @subview_const_size(%0 : memref<64x4xf32, strided<[4, 1], offset: 0>>,
   // CHECK: %[[OFF2:.*]] = llvm.add %[[DESCSTRIDE0]], %[[ARG1]] : i64
   // CHECK: %[[TMP:.*]] = builtin.unrealized_conversion_cast %[[OFF2]] : i64 to index
   // CHECK: %[[OFF2:.*]] = builtin.unrealized_conversion_cast %[[TMP]] : index to i64
-  // CHECK: %[[DESC:.*]] = llvm.mlir.undef : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)>
+  // CHECK: %[[DESC:.*]] = llvm.mlir.poison : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)>
   // CHECK: %[[DESC0:.*]] = llvm.insertvalue %[[BASE]], %[[DESC]][0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)>
   // CHECK: %[[DESC1:.*]] = llvm.insertvalue %[[BASE_ALIGNED]], %[[DESC0]][1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)>
   // CHECK: %[[DESC2:.*]] = llvm.insertvalue %[[OFF2]], %[[DESC1]][2] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)>
@@ -172,7 +172,7 @@ func.func @subview_const_stride(%0 : memref<64x4xf32, strided<[4, 1], offset: 0>
   // CHECK: %[[OFF2:.*]] = llvm.add %[[OFF0]], %[[ARG1]] : i64
   // CHECK: %[[TMP:.*]] = builtin.unrealized_conversion_cast %[[OFF2]] : i64 to index
   // CHECK: %[[OFF2:.*]] = builtin.unrealized_conversion_cast %[[TMP]] : index to i64
-  // CHECK: %[[DESC:.*]] = llvm.mlir.undef : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)>
+  // CHECK: %[[DESC:.*]] = llvm.mlir.poison : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)>
   // CHECK: %[[DESC0:.*]] = llvm.insertvalue %[[BASE]], %[[DESC]][0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)>
   // CHECK: %[[DESC1:.*]] = llvm.insertvalue %[[BASE_ALIGNED]], %[[DESC0]][1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)>
   // CHECK: %[[DESC2:.*]] = llvm.insertvalue %[[OFF2]], %[[DESC1]][2] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)>
@@ -198,7 +198,7 @@ func.func @subview_const_stride_and_offset(%0 : memref<64x4xf32, strided<[4, 1],
 
   // CHECK: %[[BASE:.*]] = llvm.extractvalue %[[MEMREF]][0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)>
   // CHECK: %[[BASE_ALIGNED:.*]] = llvm.extractvalue %[[MEMREF]][1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)>
-  // CHECK: %[[DESC:.*]] = llvm.mlir.undef : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)>
+  // CHECK: %[[DESC:.*]] = llvm.mlir.poison : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)>
   // CHECK: %[[DESC0:.*]] = llvm.insertvalue %[[BASE]], %[[DESC]][0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)>
   // CHECK: %[[DESC1:.*]] = llvm.insertvalue %[[BASE_ALIGNED]], %[[DESC0]][1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)>
   // CHECK: %[[CST_OFF:.*]] = llvm.mlir.constant(8 : index) : i64
@@ -242,7 +242,7 @@ func.func @subview_mixed_static_dynamic(%0 : memref<64x4xf32, strided<[4, 1], of
   // CHECK: %[[OFF2:.*]] = llvm.add %[[OFF0]], %[[BASE_OFF]] : i64
   // CHECK: %[[TMP:.*]] = builtin.unrealized_conversion_cast %[[OFF2]] : i64 to index
   // CHECK: %[[OFF2:.*]] = builtin.unrealized_conversion_cast %[[TMP]] : index to i64
-  // CHECK: %[[DESC:.*]] = llvm.mlir.undef : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)>
+  // CHECK: %[[DESC:.*]] = llvm.mlir.poison : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)>
   // CHECK: %[[DESC0:.*]] = llvm.insertvalue %[[BASE]], %[[DESC]][0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)>
   // CHECK: %[[DESC1:.*]] = llvm.insertvalue %[[BASE_ALIGNED]], %[[DESC0]][1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)>
   // CHECK: %[[DESC2:.*]] = llvm.insertvalue %[[OFF2]], %[[DESC1]][2] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)>
@@ -269,7 +269,7 @@ func.func @subview_leading_operands(%0 : memref<5x3xf32>, %1: memref<5x?xf32>) -
   // CHECK: %[[BASE:.*]] = llvm.extractvalue %[[MEMREF]][0] : !llvm.struct<(ptr, ptr, i64
   // Aligned ptr
   // CHECK: %[[BASE_ALIGNED:.*]] = llvm.extractvalue %[[MEMREF]][1] : !llvm.struct<(ptr, ptr, i64
-  // CHECK: %[[DESC:.*]] = llvm.mlir.undef : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)>
+  // CHECK: %[[DESC:.*]] = llvm.mlir.poison : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)>
   // CHECK: %[[DESC0:.*]] = llvm.insertvalue %[[BASE]], %[[DESC]][0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)>
   // CHECK: %[[DESC1:.*]] = llvm.insertvalue %[[BASE_ALIGNED]], %[[DESC0]][1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)>
   // Offset
@@ -304,7 +304,7 @@ func.func @subview_leading_operands_dynamic(%0 : memref<5x?xf32>) -> memref<3x?x
   // CHECK: %[[OFF0:.*]] = llvm.mul %[[STRIDE0]], %[[CST_OFF0]] overflow<nsw> : i64
   // CHECK: %[[TMP:.*]] = builtin.unrealized_conversion_cast %[[OFF0]] : i64 to index
   // CHECK: %[[OFF0:.*]] = builtin.unrealized_conversion_cast %[[TMP]] : index to i64
-  // CHECK: %[[DESC:.*]] = llvm.mlir.undef : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)>
+  // CHECK: %[[DESC:.*]] = llvm.mlir.poison : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)>
   // Alloc ptr
   // CHECK: %[[DESC0:.*]] = llvm.insertvalue %[[BASE]], %[[DESC]][0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)>
   // Aligned ptr
@@ -333,7 +333,7 @@ func.func @subview_rank_reducing_leading_operands(%0 : memref<5x3xf32>) -> memre
   // CHECK: %[[MEMREF:.*]] = builtin.unrealized_conversion_cast %[[MEM]]
   // CHECK: %[[BASE:.*]] = llvm.extractvalue %[[MEMREF]][0] : !llvm.struct<(ptr, ptr, i64
   // CHECK: %[[BASE_ALIGNED:.*]] = llvm.extractvalue %[[MEMREF]][1] : !llvm.struct<(ptr, ptr, i64
-  // CHECK: %[[DESC:.*]] = llvm.mlir.undef : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)>
+  // CHECK: %[[DESC:.*]] = llvm.mlir.poison : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)>
   // Alloc ptr
   // CHECK: %[[DESC0:.*]] = llvm.insertvalue %[[BASE]], %[[DESC]][0] : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)>
   // Aligned ptr
@@ -358,7 +358,7 @@ func.func @subview_negative_stride(%arg0 : memref<7xf32>) -> memref<7xf32, strid
   // CHECK: %[[MEMREF:.*]] = builtin.unrealized_conversion_cast %[[MEM]]
   // CHECK: %[[BASE:.*]] = llvm.extractvalue %[[MEMREF]][0] : !llvm.struct<(ptr, ptr, i64
   // CHECK: %[[BASE_ALIGNED:.*]] = llvm.extractvalue %[[MEMREF]][1] : !llvm.struct<(ptr, ptr, i64
-  // CHECK: %[[DESC:.*]] = llvm.mlir.undef : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)>
+  // CHECK: %[[DESC:.*]] = llvm.mlir.poison : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)>
   // CHECK: %[[DESC0:.*]] = llvm.insertvalue %[[BASE]], %[[DESC]][0] : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)>
   // CHECK: %[[DESC1:.*]] = llvm.insertvalue %[[BASE_ALIGNED]], %[[DESC0]][1] : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)>
   // CHECK: %[[CST_OFF0:.*]] = llvm.mlir.constant(6 : index) : i64
@@ -387,7 +387,7 @@ func.func @collapse_shape_static(%arg0: memref<1x3x4x1x5xf32>) -> memref<3x4x5xf
 // CHECK:           %[[BASE_BUFFER:.*]] = llvm.extractvalue %[[MEM]][0] : !llvm.struct<(ptr, ptr, i64, array<5 x i64>, array<5 x i64>)>
 // CHECK:           %[[ALIGNED_BUFFER:.*]] = llvm.extractvalue %[[MEM]][1] : !llvm.struct<(ptr, ptr, i64, array<5 x i64>, array<5 x i64>)>
 // CHECK:           %[[C0:.*]] = llvm.mlir.constant(0 : index) : i64
-// CHECK:           %[[DESC:.*]] = llvm.mlir.undef : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)>
+// CHECK:           %[[DESC:.*]] = llvm.mlir.poison : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)>
 // CHECK:           %[[DESC0:.*]] = llvm.insertvalue %[[BASE_BUFFER]], %[[DESC]][0] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)>
 // CHECK:           %[[DESC1:.*]] = llvm.insertvalue %[[ALIGNED_BUFFER]], %[[DESC0]][1] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)>
 // CHECK:           %[[DESC2:.*]] = llvm.insertvalue %[[C0]], %[[DESC1]][2] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)>
@@ -428,7 +428,7 @@ func.func @collapse_shape_dynamic_with_non_identity_layout(
 // CHECK:           %[[FINAL_SIZE1:.*]] = llvm.mul %[[SIZE1]], %[[SIZE2]] overflow<nsw> : i64
 // CHECK:           %[[SIZE1_TO_IDX:.*]] = builtin.unrealized_conversion_cast %[[FINAL_SIZE1]] : i64 to index
 // CHECK:           %[[FINAL_SIZE1:.*]] = builtin.unrealized_conversion_cast %[[SIZE1_TO_IDX]] : index to i64
-// CHECK:           %[[DESC:.*]] = llvm.mlir.undef : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)>
+// CHECK:           %[[DESC:.*]] = llvm.mlir.poison : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)>
 // CHECK:           %[[DESC0:.*]] = llvm.insertvalue %[[BASE_BUFFER]], %[[DESC]][0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)>
 // CHECK:           %[[DESC1:.*]] = llvm.insertvalue %[[ALIGNED_BUFFER]], %[[DESC0]][1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)>
 // CHECK:           %[[DESC2:.*]] = llvm.insertvalue %[[OFFSET]], %[[DESC1]][2] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)>
@@ -461,7 +461,7 @@ func.func @expand_shape_static(%arg0: memref<3x4x5xf32>) -> memref<1x3x4x1x5xf32
 // CHECK:           %[[BASE_BUFFER:.*]] = llvm.extractvalue %[[MEM]][0] : !llvm.struct<(ptr, ptr, i64,
 // CHECK:           %[[ALIGNED_BUFFER:.*]] = llvm.extractvalue %[[MEM]][1] : !llvm.struct<(ptr, ptr, i64,
 // CHECK:           %[[C0:.*]] = llvm.mlir.constant(0 : index) : i64
-// CHECK:           %[[DESC:.*]] = llvm.mlir.undef : !llvm.struct<(ptr, ptr, i64, array<5 x i64>, array<5 x i64>)>
+// CHECK:           %[[DESC:.*]] = llvm.mlir.poison : !llvm.struct<(ptr, ptr, i64, array<5 x i64>, array<5 x i64>)>
 // CHECK:           %[[DESC0:.*]] = llvm.insertvalue %[[BASE_BUFFER]], %[[DESC]][0] : !llvm.struct<(ptr, ptr, i64, array<5 x i64>, array<5 x i64>)>
 // CHECK:           %[[DESC1:.*]] = llvm.insertvalue %[[ALIGNED_BUFFER]], %[[DESC0]][1] : !llvm.struct<(ptr, ptr, i64, array<5 x i64>, array<5 x i64>)>
 // CHECK:           %[[DESC2:.*]] = llvm.insertvalue %[[C0]], %[[DESC1]][2] : !llvm.struct<(ptr, ptr, i64, array<5 x i64>, array<5 x i64>)>
@@ -496,7 +496,7 @@ func.func @collapse_shape_fold_zero_dim(%arg0 : memref<1x1xf32>) -> memref<f32>
 // CHECK:           %[[MEM:.*]] = builtin.unrealized_conversion_cast %[[ARG]] : memref<1x1xf32> to !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)>
 // CHECK:           %[[BASE_BUFFER:.*]] = llvm.extractvalue %[[MEM]][0] : !llvm.struct<(ptr, ptr, i64,
 // CHECK:           %[[ALIGNED_BUFFER:.*]] = llvm.extractvalue %[[MEM]][1] : !llvm.struct<(ptr, ptr, i64,
-// CHECK:           %[[DESC:.*]] = llvm.mlir.undef : !llvm.struct<(ptr, ptr, i64)>
+// CHECK:           %[[DESC:.*]] = llvm.mlir.poison : !llvm.struct<(ptr, ptr, i64)>
 // CHECK:           %[[DESC0:.*]] = llvm.insertvalue %[[BASE_BUFFER]], %[[DESC]][0] : !llvm.struct<(ptr, ptr, i64)>
 // CHECK:           %[[DESC1:.*]] = llvm.insertvalue %[[ALIGNED_BUFFER]], %[[DESC0]][1] : !llvm.struct<(ptr, ptr, i64)>
 // CHECK:           %[[C0:.*]] = llvm.mlir.constant(0 : index) : i64
@@ -518,7 +518,7 @@ func.func @expand_shape_zero_dim(%arg0 : memref<f32>) -> memref<1x1xf32> {
 // CHECK:           %[[BASE_BUFFER:.*]] = llvm.extractvalue %[[MEM]][0] : !llvm.struct<(ptr, ptr, i64)>
 // CHECK:           %[[ALIGNED_BUFFER:.*]] = llvm.extractvalue %[[MEM]][1] : !llvm.struct<(ptr, ptr, i64)>
 // CHECK:           %[[C0:.*]] = llvm.mlir.constant(0 : index) : i64
-// CHECK:           %[[DESC:.*]] = llvm.mlir.undef : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)>
+// CHECK:           %[[DESC:.*]] = llvm.mlir.poison : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)>
 // CHECK:           %[[DESC0:.*]] = llvm.insertvalue %[[BASE_BUFFER]], %[[DESC]][0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)>
 // CHECK:           %[[DESC1:.*]] = llvm.insertvalue %[[ALIGNED_BUFFER]], %[[DESC0]][1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)>
 // CHECK:           %[[DESC2:.*]] = llvm.insertvalue %[[C0]], %[[DESC1]][2] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)>
@@ -550,7 +550,7 @@ func.func @collapse_shape_dynamic(%arg0 : memref<1x2x?xf32>) -> memref<1x?xf32>
 // CHECK:           %[[FINAL_SIZE1:.*]] = llvm.mul %[[SIZE2]], %[[C2]] overflow<nsw> : i64
 // CHECK:           %[[SIZE1_TO_IDX:.*]] = builtin.unrealized_conversion_cast %[[FINAL_SIZE1]] : i64 to index
 // CHECK:           %[[FINAL_SIZE1:.*]] = builtin.unrealized_conversion_cast %[[SIZE1_TO_IDX]] : index to i64
-// CHECK:           %[[DESC:.*]] = llvm.mlir.undef : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)>
+// CHECK:           %[[DESC:.*]] = llvm.mlir.poison : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)>
 // CHECK:           %[[DESC0:.*]] = llvm.insertvalue %[[BASE_BUFFER]], %[[DESC]][0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)>
 // CHECK:           %[[DESC1:.*]] = llvm.insertvalue %[[ALIGNED_BUFFER]], %[[DESC0]][1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)>
 // CHECK:           %[[DESC2:.*]] = llvm.insertvalue %[[C0]], %[[DESC1]][2] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)>
@@ -588,7 +588,7 @@ func.func @expand_shape_dynamic(%arg0 : memref<1x?xf32>, %sz0: index) -> memref<
 // CHECK:           %[[FINAL_SIZE2:.*]] = llvm.select %[[IS_NEGATIVE_SIZE1]], %[[NEGATIVE_SIZE2]], %[[SIZE2]] : i1, i64
 // CHECK:           %[[SIZE2_TO_IDX:.*]] = builtin.unrealized_conversion_cast %[[FINAL_SIZE2]] : i64 to index
 // CHECK:           %[[FINAL_SIZE2:.*]] = builtin.unrealized_conversion_cast %[[SIZE2_TO_IDX]] : index to i64
-// CHECK:           %[[DESC:.*]] = llvm.mlir.undef : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)>
+// CHECK:           %[[DESC:.*]] = llvm.mlir.poison : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)>
 // CHECK:           %[[DESC0:.*]] = llvm.insertvalue %[[BASE_BUFFER]], %[[DESC]][0] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)>
 // CHECK:           %[[DESC1:.*]] = llvm.insertvalue %[[ALIGNED_BUFFER]], %[[DESC0]][1] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)>
 // CHECK:           %[[DESC2:.*]] = llvm.insertvalue %[[C0]], %[[DESC1]][2] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)>
@@ -638,7 +638,7 @@ func.func @expand_shape_dynamic_with_non_identity_layout(
 // CHECK:           %[[FINAL_STRIDE1:.*]] = llvm.mul %[[TMP_SIZE2]], %[[STRIDE1]]
 // CHECK:           %[[STRIDE1_TO_IDX:.*]] = builtin.unrealized_conversion_cast %[[FINAL_STRIDE1]] : i64 to index
 // CHECK:           %[[FINAL_STRIDE1:.*]] = builtin.unrealized_conversion_cast %[[STRIDE1_TO_IDX]] : index to i64
-// CHECK:           %[[DESC:.*]] = llvm.mlir.undef : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)>
+// CHECK:           %[[DESC:.*]] = llvm.mlir.poison : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)>
 // CHECK:           %[[DESC1:.*]] = llvm.insertvalue %[[BASE_BUFFER]], %[[DESC]][0] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)>
 // CHECK:           %[[DESC2:.*]] = llvm.insertvalue %[[ALIGNED_BUFFER]], %[[DESC1]][1] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)>
 // CHECK:           %[[DESC3:.*]] = llvm.insertvalue %[[OFFSET]], %[[DESC2]][2] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)>
diff --git a/mlir/test/Conversion/MemRefToLLVM/invalid.mlir b/mlir/test/Conversion/MemRefToLLVM/invalid.mlir
index 40dd75af1dd77..1e12b83a24b5a 100644
--- a/mlir/test/Conversion/MemRefToLLVM/invalid.mlir
+++ b/mlir/test/Conversion/MemRefToLLVM/invalid.mlir
@@ -2,6 +2,13 @@
 // Since the error is at an unknown location, we use FileCheck instead of
 // -veri-y-diagnostics here
 
+// CHECK: redefinition of reserved function 'malloc' of different type '!llvm.func<void (i64)>' is prohibited
+llvm.func @malloc(i64)
+func.func @redef_reserved() {
+    %alloc = memref.alloc() : memref<1024x64xf32, 1>
+    llvm.return
+}
+
 // CHECK: conversion of memref memory space "foo" to integer address space failed. Consider adding memory space conversions.
 // CHECK-LABEL: @bad_address_space
 func.func @bad_address_space(%a: memref<2xindex, "foo">) {
diff --git a/mlir/test/Conversion/MemRefToLLVM/memref-to-llvm.mlir b/mlir/test/Conversion/MemRefToLLVM/memref-to-llvm.mlir
index 67b68b7a1c044..523e894aaef8d 100644
--- a/mlir/test/Conversion/MemRefToLLVM/memref-to-llvm.mlir
+++ b/mlir/test/Conversion/MemRefToLLVM/memref-to-llvm.mlir
@@ -14,11 +14,11 @@ func.func @view(%arg0 : index, %arg1 : index, %arg2 : index) {
   // CHECK-DAG: %[[ARG0:.*]] = builtin.unrealized_conversion_cast %[[ARG0F]]
   // CHECK-DAG: %[[ARG1:.*]] = builtin.unrealized_conversion_cast %[[ARG1F]]
   // CHECK: llvm.mlir.constant(2048 : index) : i64
-  // CHECK: llvm.mlir.undef : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)>
+  // CHECK: llvm.mlir.poison : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)>
   %0 = memref.alloc() : memref<2048xi8>
 
   // Test two dynamic sizes.
-  // CHECK: llvm.mlir.undef : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)>
+  // CHECK: llvm.mlir.poison : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)>
   // CHECK: %[[BASE_PTR:.*]] = llvm.extractvalue %{{.*}}[1] : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)>
   // CHECK: %[[SHIFTED_BASE_PTR:.*]] = llvm.getelementptr %[[BASE_PTR]][%[[ARG2]]] : (!llvm.ptr, i64) -> !llvm.ptr, i8
   // CHECK: llvm.insertvalue %[[SHIFTED_BASE_PTR]], %{{.*}}[1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)>
@@ -33,7 +33,7 @@ func.func @view(%arg0 : index, %arg1 : index, %arg2 : index) {
   %1 = memref.view %0[%arg2][%arg0, %arg1] : memref<2048xi8> to memref<?x?xf32>
 
   // Test one dynamic size.
-  // CHECK: llvm.mlir.undef : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)>
+  // CHECK: llvm.mlir.poison : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)>
   // CHECK: %[[BASE_PTR_2:.*]] = llvm.extractvalue %{{.*}}[1] : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)>
   // CHECK: %[[SHIFTED_BASE_PTR_2:.*]] = llvm.getelementptr %[[BASE_PTR_2]][%[[ARG2]]] : (!llvm.ptr, i64) -> !llvm.ptr, i8
   // CHECK: llvm.insertvalue %[[SHIFTED_BASE_PTR_2]], %{{.*}}[1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)>
@@ -49,7 +49,7 @@ func.func @view(%arg0 : index, %arg1 : index, %arg2 : index) {
   %3 = memref.view %0[%arg2][%arg1] : memref<2048xi8> to memref<4x?xf32>
 
   // Test static sizes.
-  // CHECK: llvm.mlir.undef : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)>
+  // CHECK: llvm.mlir.poison : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)>
   // CHECK: %[[BASE_PTR_3:.*]] = llvm.extractvalue %{{.*}}[1] : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)>
   // CHECK: %[[SHIFTED_BASE_PTR_3:.*]] = llvm.getelementptr %[[BASE_PTR_3]][%[[ARG2]]] : (!llvm.ptr, i64) -> !llvm.ptr, i8
   // CHECK: llvm.insertvalue %[[SHIFTED_BASE_PTR_3]], %{{.*}}[1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)>
@@ -67,10 +67,10 @@ func.func @view(%arg0 : index, %arg1 : index, %arg2 : index) {
 
   // Test view memory space.
   // CHECK: llvm.mlir.constant(2048 : index) : i64
-  // CHECK: llvm.mlir.undef : !llvm.struct<(ptr<4>, ptr<4>, i64, array<1 x i64>, array<1 x i64>)>
+  // CHECK: llvm.mlir.poison : !llvm.struct<(ptr<4>, ptr<4>, i64, array<1 x i64>, array<1 x i64>)>
   %6 = memref.alloc() : memref<2048xi8, 4>
 
-  // CHECK: llvm.mlir.undef : !llvm.struct<(ptr<4>, ptr<4>, i64, array<2 x i64>, array<2 x i64>)>
+  // CHECK: llvm.mlir.poison : !llvm.struct<(ptr<4>, ptr<4>, i64, array<2 x i64>, array<2 x i64>)>
   // CHECK: %[[BASE_PTR_4:.*]] = llvm.extractvalue %{{.*}}[1] : !llvm.struct<(ptr<4>, ptr<4>, i64, array<1 x i64>, array<1 x i64>)>
   // CHECK: %[[SHIFTED_BASE_PTR_4:.*]] = llvm.getelementptr %[[BASE_PTR_4]][%[[ARG2]]] : (!llvm.ptr<4>, i64) -> !llvm.ptr<4>, i8
   // CHECK: llvm.insertvalue %[[SHIFTED_BASE_PTR_4]], %{{.*}}[1] : !llvm.struct<(ptr<4>, ptr<4>, i64, array<2 x i64>, array<2 x i64>)>
@@ -100,7 +100,7 @@ func.func @view(%arg0 : index, %arg1 : index, %arg2 : index) {
 // CHECK-INTERFACE:        %[[ARG1:.*]]: memref<0xi8>)
 func.func @view_empty_memref(%offset: index, %mem: memref<0xi8>) {
 
-  // CHECK: llvm.mlir.undef : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)>
+  // CHECK: llvm.mlir.poison : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)>
   // CHECK: llvm.mlir.constant(0 : index) : i64
   // CHECK: llvm.insertvalue %{{.*}}, %{{.*}}[2] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)>
   // CHECK: llvm.mlir.constant(4 : index) : i64
@@ -112,7 +112,7 @@ func.func @view_empty_memref(%offset: index, %mem: memref<0xi8>) {
   // CHECK: llvm.mlir.constant(4 : index) : i64
   // CHECK: = llvm.insertvalue %{{.*}}, %{{.*}}[4, 0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)>
 
-  // CHECK-INTERFACE: llvm.mlir.undef : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)>
+  // CHECK-INTERFACE: llvm.mlir.poison : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)>
   // CHECK-INTERFACE: llvm.mlir.constant(0 : index) : i64
   // CHECK-INTERFACE: llvm.insertvalue %{{.*}}, %{{.*}}[2] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)>
   // CHECK-INTERFACE: llvm.mlir.constant(4 : index) : i64
@@ -220,7 +220,7 @@ func.func @address_space(%arg0 : memref<32xf32, affine_map<(d0) -> (d0)>, 7>) {
 // -----
 
 // CHECK-LABEL: func @transpose
-//       CHECK:   llvm.mlir.undef : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)>
+//       CHECK:   llvm.mlir.poison : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)>
 //       CHECK:   llvm.insertvalue {{.*}}[0] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)>
 //       CHECK:    llvm.insertvalue {{.*}}[1] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)>
 //       CHECK:    llvm.insertvalue {{.*}}[2] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)>
@@ -265,7 +265,7 @@ func.func @get_gv0_memref() {
   // CHECK: %[[GEP:.*]] = llvm.getelementptr %[[ADDR]][0, 0] : (!llvm.ptr) -> !llvm.ptr, !llvm.array<2 x f32>
   // CHECK: %[[DEADBEEF:.*]] = llvm.mlir.constant(3735928559 : index) : i64
   // CHECK: %[[DEADBEEFPTR:.*]] = llvm.inttoptr %[[DEADBEEF]] : i64 to !llvm.ptr
-  // CHECK: llvm.mlir.undef : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)>
+  // CHECK: llvm.mlir.poison : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)>
   // CHECK: llvm.insertvalue %[[DEADBEEFPTR]], {{.*}}[0] : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)>
   // CHECK: llvm.insertvalue %[[GEP]], {{.*}}[1] : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)>
   // CHECK: %[[OFFSET:.*]] = llvm.mlir.constant(0 : index) : i64
@@ -285,7 +285,7 @@ func.func @get_gv2_memref() {
   // CHECK: %[[GEP:.*]] = llvm.getelementptr %[[ADDR]][0, 0, 0] : (!llvm.ptr) -> !llvm.ptr, !llvm.array<2 x array<3 x f32>>
   // CHECK: %[[DEADBEEF:.*]] = llvm.mlir.constant(3735928559 : index) : i64
   // CHECK: %[[DEADBEEFPTR:.*]] = llvm.inttoptr %[[DEADBEEF]] : i64 to !llvm.ptr
-  // CHECK: llvm.mlir.undef : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)>
+  // CHECK: llvm.mlir.poison : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)>
   // CHECK: llvm.insertvalue %[[DEADBEEFPTR]], {{.*}}[0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)>
   // CHECK: llvm.insertvalue %[[GEP]], {{.*}}[1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)>
   // CHECK: %[[OFFSET:.*]] = llvm.mlir.constant(0 : index) : i64
@@ -309,7 +309,7 @@ func.func @get_gv3_memref() {
   // CHECK: %[[GEP:.*]] = llvm.getelementptr %[[ADDR]][0] : (!llvm.ptr) -> !llvm.ptr, f32
   // CHECK: %[[DEADBEEF:.*]] = llvm.mlir.constant(3735928559 : index) : i64
   // CHECK: %[[DEADBEEFPTR:.*]] = llvm.inttoptr %[[DEADBEEF]] : i64 to !llvm.ptr
-  // CHECK: llvm.mlir.undef : !llvm.struct<(ptr, ptr, i64)>
+  // CHECK: llvm.mlir.poison : !llvm.struct<(ptr, ptr, i64)>
   // CHECK: llvm.insertvalue %[[DEADBEEFPTR]], {{.*}}[0] : !llvm.struct<(ptr, ptr, i64)>
   // CHECK: llvm.insertvalue %[[GEP]], {{.*}}[1] : !llvm.struct<(ptr, ptr, i64)>
   // CHECK: %[[OFFSET:.*]] = llvm.mlir.constant(0 : index) : i64
@@ -466,11 +466,11 @@ func.func @generic_atomic_rmw_in_alloca_scope(){
 func.func @memref_copy_ranked() {
   %0 = memref.alloc() : memref<2xf32>
   // CHECK: llvm.mlir.constant(2 : index) : i64
-  // CHECK: llvm.mlir.undef : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)>
+  // CHECK: llvm.mlir.poison : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)>
   %1 = memref.cast %0 : memref<2xf32> to memref<?xf32>
   %2 = memref.alloc() : memref<2xf32>
   // CHECK: llvm.mlir.constant(2 : index) : i64
-  // CHECK: llvm.mlir.undef : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)>
+  // CHECK: llvm.mlir.poison : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)>
   %3 = memref.cast %2 : memref<2xf32> to memref<?xf32>
   memref.copy %1, %3 : memref<?xf32> to memref<?xf32>
   // CHECK: [[ONE:%.*]] = llvm.mlir.constant(1 : index) : i64
@@ -550,18 +550,18 @@ func.func @memref_copy_noncontiguous(%in: memref<16x2xi32>, %offset: index) {
 func.func @memref_copy_unranked() {
   %0 = memref.alloc() : memref<2xi1>
   // CHECK: llvm.mlir.constant(2 : index) : i64
-  // CHECK: llvm.mlir.undef : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)>
+  // CHECK: llvm.mlir.poison : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)>
   %1 = memref.cast %0 : memref<2xi1> to memref<*xi1>
   %2 = memref.alloc() : memref<2xi1>
   // CHECK: llvm.mlir.constant(2 : index) : i64
-  // CHECK: llvm.mlir.undef : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)>
+  // CHECK: llvm.mlir.poison : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)>
   %3 = memref.cast %2 : memref<2xi1> to memref<*xi1>
   memref.copy %1, %3 : memref<*xi1> to memref<*xi1>
   // CHECK: [[ONE:%.*]] = llvm.mlir.constant(1 : index) : i64
   // CHECK: [[ALLOCA:%.*]] = llvm.alloca [[ONE]] x !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> : (i64) -> !llvm.ptr
   // CHECK: llvm.store {{%.*}}, [[ALLOCA]] : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)>, !llvm.ptr
   // CHECK: [[RANK:%.*]] = llvm.mlir.constant(1 : index) : i64
-  // CHECK: [[UNDEF:%.*]] = llvm.mlir.undef : !llvm.struct<(i64, ptr)>
+  // CHECK: [[UNDEF:%.*]] = llvm.mlir.poison : !llvm.struct<(i64, ptr)>
   // CHECK: [[INSERT:%.*]] = llvm.insertvalue [[RANK]], [[UNDEF]][0] : !llvm.struct<(i64, ptr)>
   // CHECK: [[INSERT2:%.*]] = llvm.insertvalue [[ALLOCA]], [[INSERT]][1] : !llvm.struct<(i64, ptr)>
   // CHECK: [[STACKSAVE:%.*]] = llvm.intr.stacksave : !llvm.ptr
@@ -612,7 +612,7 @@ func.func @extract_aligned_pointer_as_index_unranked(%m: memref<*xf32>) -> index
 // CHECK: %[[MEM_DESC:.*]] = builtin.unrealized_conversion_cast %[[ARG]] : memref<?x?xf32, strided<[?, ?], offset: ?>> to !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)>
 // CHECK: %[[BASE:.*]] = llvm.extractvalue %[[MEM_DESC]][0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)>
 // CHECK: %[[ALIGNED_BASE:.*]] = llvm.extractvalue %[[MEM_DESC]][1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)>
-// CHECK: %[[DESC:.*]] = llvm.mlir.undef : !llvm.struct<(ptr, ptr, i64)>
+// CHECK: %[[DESC:.*]] = llvm.mlir.poison : !llvm.struct<(ptr, ptr, i64)>
 // CHECK: %[[DESC0:.*]] = llvm.insertvalue %[[BASE]], %[[DESC]][0] : !llvm.struct<(ptr, ptr, i64)>
 // CHECK: %[[DESC1:.*]] = llvm.insertvalue %[[ALIGNED_BASE]], %[[DESC0]][1] : !llvm.struct<(ptr, ptr, i64)>
 // CHECK: %[[OFF0:.*]] = llvm.mlir.constant(0 : index) : i64
diff --git a/mlir/test/Conversion/NVGPUToNVVM/nvgpu-to-nvvm.mlir b/mlir/test/Conversion/NVGPUToNVVM/nvgpu-to-nvvm.mlir
index 75362378daaaa..6b59b5e4343b4 100644
--- a/mlir/test/Conversion/NVGPUToNVVM/nvgpu-to-nvvm.mlir
+++ b/mlir/test/Conversion/NVGPUToNVVM/nvgpu-to-nvvm.mlir
@@ -17,7 +17,7 @@ func.func @m16n8k16_fp16(%arg0: vector<4x2xf16>, %arg1: vector<2x2xf16>, %arg2:
   %d = nvgpu.mma.sync (%arg0, %arg1, %arg2) {mmaShape = [16, 8, 16]} : (vector<4x2xf16>, vector<2x2xf16>, vector<2x2xf16>) -> vector<2x2xf16>
   // CHECK-DAG: llvm.extractvalue [[d]][0] : !llvm.struct<(vector<2xf16>, vector<2xf16>)>
   // CHECK-DAG: llvm.extractvalue [[d]][1] : !llvm.struct<(vector<2xf16>, vector<2xf16>)>
-  // CHECK: llvm.mlir.undef : !llvm.array<2 x vector<2xf16>>
+  // CHECK: llvm.mlir.poison : !llvm.array<2 x vector<2xf16>>
   // CHECK-DAG: llvm.insertvalue {{%.+}}, {{%.+}}[0] : !llvm.array<2 x vector<2xf16>>
   // CHECK-DAG: llvm.insertvalue {{%.+}}, {{%.+}}[1] : !llvm.array<2 x vector<2xf16>>
   return %d : vector<2x2xf16>
@@ -32,13 +32,13 @@ func.func @m16n8k16_fp16_fp32(%arg0: vector<4x2xf16>, %arg1: vector<2x2xf16>, %a
   // CHECK-SAME: shape = #nvvm.shape<m = 16, n = 8, k = 16>
   // CHECK-SAME: (vector<2xf16>, vector<2xf16>, f32) -> !llvm.struct<(f32, f32, f32, f32)>
   %d = nvgpu.mma.sync (%arg0, %arg1, %arg2) {mmaShape = [16, 8, 16]} : (vector<4x2xf16>, vector<2x2xf16>, vector<2x2xf32>) -> vector<2x2xf32>
-  // CHECK: [[undef:%.+]] = llvm.mlir.undef : vector<2xf32>
+  // CHECK: [[undef:%.+]] = llvm.mlir.poison : vector<2xf32>
   // CHECK-DAG: llvm.extractvalue [[d]][0] : !llvm.struct<(f32, f32, f32, f32)>
   // CHECK-DAG: llvm.extractvalue [[d]][1] : !llvm.struct<(f32, f32, f32, f32)>
   // CHECK: [[d00:%.+]] = llvm.insertelement {{%.+}}, [[undef]][{{.*}}] : vector<2xf32>
   // CHECK: [[d01:%.+]] = llvm.insertelement {{%.+}}, [[d00]][{{.*}}] : vector<2xf32>
 
-  // CHECK: [[undef:%.+]] = llvm.mlir.undef : vector<2xf32>
+  // CHECK: [[undef:%.+]] = llvm.mlir.poison : vector<2xf32>
   // CHECK-DAG: llvm.extractvalue [[d]][2] : !llvm.struct<(f32, f32, f32, f32)>
   // CHECK-DAG: llvm.extractvalue [[d]][3] : !llvm.struct<(f32, f32, f32, f32)>
   // CHECK: [[d10:%.+]] = llvm.insertelement {{%.+}}, [[undef]][{{.*}}] : vector<2xf32>
@@ -62,7 +62,7 @@ func.func @m16n8k8_fp16(%arg0: vector<2x2xf16>, %arg1: vector<1x2xf16>, %arg2: v
   %d = nvgpu.mma.sync (%arg0, %arg1, %arg2) {mmaShape = [16, 8, 8]} : (vector<2x2xf16>, vector<1x2xf16>, vector<2x2xf16>) -> vector<2x2xf16>
   // CHECK-DAG: llvm.extractvalue [[d]][0] : !llvm.struct<(vector<2xf16>, vector<2xf16>)>
   // CHECK-DAG: llvm.extractvalue [[d]][1] : !llvm.struct<(vector<2xf16>, vector<2xf16>)>
-  // CHECK: llvm.mlir.undef : !llvm.array<2 x vector<2xf16>>
+  // CHECK: llvm.mlir.poison : !llvm.array<2 x vector<2xf16>>
   // CHECK-DAG: llvm.insertvalue {{%.+}}, {{%.+}}[0] : !llvm.array<2 x vector<2xf16>>
   // CHECK-DAG: llvm.insertvalue {{%.+}}, {{%.+}}[1] : !llvm.array<2 x vector<2xf16>>
   // CHECK: return
@@ -146,7 +146,7 @@ func.func @m8n8k4_f64(%arg0: vector<1x1xf64>, %arg1: vector<1x1xf64>, %arg2: vec
   // CHECK: [[d:%.+]] = nvvm.mma.sync A[{{%.+}}] B[{{%.+}}] C[{{%.+}}, {{%.+}}]
   // CHECK-SAME: shape = #nvvm.shape<m = 8, n = 8, k = 4>
   %d = nvgpu.mma.sync (%arg0, %arg1, %arg2) {mmaShape = [8, 8, 4]} : (vector<1x1xf64>, vector<1x1xf64>, vector<1x2xf64>) -> vector<1x2xf64>
-  // CHECK: llvm.mlir.undef : vector<2xf64>
+  // CHECK: llvm.mlir.poison : vector<2xf64>
   // CHECK-DAG: llvm.extractvalue [[d]][0] : !llvm.struct<(f64, f64)>
   // CHECK-DAG: llvm.extractvalue [[d]][1] : !llvm.struct<(f64, f64)>
   // CHECK-COUNT-2: llvm.insertelement {{.*}} : vector<2xf64>
@@ -202,13 +202,13 @@ func.func @m16n8k4_tf32(%arg0: vector<2x1xf32>, %arg1: vector<1x1xf32>, %arg2: v
   // CHECK-SAME: shape = #nvvm.shape<m = 16, n = 8, k = 4>
   // CHECK-SAME: -> !llvm.struct<(f32, f32, f32, f32)>
   %d = nvgpu.mma.sync (%arg0, %arg1, %arg2) {mmaShape = [16, 8, 4], tf32Enabled} : (vector<2x1xf32>, vector<1x1xf32>, vector<2x2xf32>) -> vector<2x2xf32>
-  // CHECK: [[undef:%.+]] = llvm.mlir.undef : vector<2xf32>
+  // CHECK: [[undef:%.+]] = llvm.mlir.poison : vector<2xf32>
   // CHECK-DAG: llvm.extractvalue [[d]][0] : !llvm.struct<(f32, f32, f32, f32)>
   // CHECK-DAG: llvm.extractvalue [[d]][1] : !llvm.struct<(f32, f32, f32, f32)>
   // CHECK: [[d00:%.+]] = llvm.insertelement {{%.+}}, [[undef]][{{.*}}] : vector<2xf32>
   // CHECK: [[d01:%.+]] = llvm.insertelement {{%.+}}, [[d00]][{{.*}}] : vector<2xf32>
 
-  // CHECK: [[undef:%.+]] = llvm.mlir.undef : vector<2xf32>
+  // CHECK: [[undef:%.+]] = llvm.mlir.poison : vector<2xf32>
   // CHECK-DAG: llvm.extractvalue [[d]][2] : !llvm.struct<(f32, f32, f32, f32)>
   // CHECK-DAG: llvm.extractvalue [[d]][3] : !llvm.struct<(f32, f32, f32, f32)>
   // CHECK: [[d10:%.+]] = llvm.insertelement {{%.+}}, [[undef]][{{.*}}] : vector<2xf32>
@@ -277,7 +277,7 @@ func.func @async_cp_zfill_f32_align4(
   %src: memref<128x128xf32>, %dst: memref<3x16x128xf32, 3>, %i : index, %srcElements : index) {
   // CHECK-DAG: %[[IDX1:.*]] = builtin.unrealized_conversion_cast %[[IDX]] : index to i64
   // CHECK-DAG: %[[SRC1:.*]] = builtin.unrealized_conversion_cast %[[SRCELEMENTS]] : index to i64
-  // CHECK-DAG: %[[BASEDST:.*]] = llvm.extractvalue %{{.*}}[1] : !llvm.struct<(ptr<3>, ptr<3>, i64, array<3 x i64>, array<3 x i64>)>                                   
+  // CHECK-DAG: %[[BASEDST:.*]] = llvm.extractvalue %{{.*}}[1] : !llvm.struct<(ptr<3>, ptr<3>, i64, array<3 x i64>, array<3 x i64>)>
   // CHECK-DAG: %[[S2048:.*]] = llvm.mlir.constant(2048 : index) : i64
   // CHECK-DAG: %[[LI1:.*]] = llvm.mul %[[IDX1]], %[[S2048]] : i64
   // CHECK-DAG: %[[S0:.*]] = llvm.mlir.constant(128 : index) : i64
@@ -285,7 +285,7 @@ func.func @async_cp_zfill_f32_align4(
   // CHECK-DAG: %[[FI1:.*]] = llvm.add %[[LI1]], %[[LI]] : i64
   // CHECK-DAG: %[[FI2:.*]] = llvm.add %[[FI1]], %[[IDX1]] : i64
   // CHECK-DAG: %[[ADDRESSDST:.*]] = llvm.getelementptr %[[BASEDST]][%[[FI2]]] : (!llvm.ptr<3>, i64) -> !llvm.ptr<3>, f32
-  // CHECK-DAG: %[[BASESRC:.*]] = llvm.extractvalue %{{.*}}[1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> 
+  // CHECK-DAG: %[[BASESRC:.*]] = llvm.extractvalue %{{.*}}[1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)>
   // CHECK-DAG: %[[S2:.*]] = llvm.mlir.constant(128 : index) : i64
   // CHECK-DAG: %[[FI2:.*]] = llvm.mul %[[IDX1]], %[[S2]]  : i64
   // CHECK-DAG: %[[FI3:.*]] = llvm.add %[[FI2]], %[[IDX1]]  : i64
@@ -293,7 +293,7 @@ func.func @async_cp_zfill_f32_align4(
   // CHECK-DAG: %[[CAST2:.*]] = llvm.addrspacecast %[[ADDRESSSRC]] : !llvm.ptr to !llvm.ptr<1>
   // CHECK-DAG: %[[c1:.*]] = llvm.mlir.constant(3 : i32) : i32
   // CHECK-DAG: %[[c2:.*]] = llvm.mlir.constant(32 : i32) : i32
-  // CHECK-DAG: %[[c3:.*]] = llvm.trunc %[[SRC1]] : i64 to i32  
+  // CHECK-DAG: %[[c3:.*]] = llvm.trunc %[[SRC1]] : i64 to i32
   // CHECK-DAG: %[[c4:.*]] = llvm.mul %[[c2]], %[[c3]] : i32
   // CHECK-DAG: %[[c5:.*]] = llvm.lshr %[[c4]], %[[c1]] : i32
   // CHECK-DAG: nvvm.cp.async.shared.global %[[ADDRESSDST]], %[[CAST2]], 16, cache = cg, %[[c5]]
@@ -312,7 +312,7 @@ func.func @async_cp_zfill_f32_align1(
   %src: memref<128x128xf32>, %dst: memref<3x16x128xf32, 3>, %i : index, %srcElements : index) {
   // CHECK-DAG: %[[IDX1:.*]] = builtin.unrealized_conversion_cast %[[IDX]] : index to i64
   // CHECK-DAG: %[[SRC1:.*]] = builtin.unrealized_conversion_cast %[[SRCELEMENTS]] : index to i64
-  // CHECK-DAG: %[[BASEDST:.*]] = llvm.extractvalue %{{.*}}[1] : !llvm.struct<(ptr<3>, ptr<3>, i64, array<3 x i64>, array<3 x i64>)>                                   
+  // CHECK-DAG: %[[BASEDST:.*]] = llvm.extractvalue %{{.*}}[1] : !llvm.struct<(ptr<3>, ptr<3>, i64, array<3 x i64>, array<3 x i64>)>
   // CHECK-DAG: %[[S2048:.*]] = llvm.mlir.constant(2048 : index) : i64
   // CHECK-DAG: %[[LI1:.*]] = llvm.mul %[[IDX1]], %[[S2048]] : i64
   // CHECK-DAG: %[[S0:.*]] = llvm.mlir.constant(128 : index) : i64
@@ -320,7 +320,7 @@ func.func @async_cp_zfill_f32_align1(
   // CHECK-DAG: %[[FI1:.*]] = llvm.add %[[LI1]], %[[LI]] : i64
   // CHECK-DAG: %[[FI2:.*]] = llvm.add %[[FI1]], %[[IDX1]] : i64
   // CHECK-DAG: %[[ADDRESSDST:.*]] = llvm.getelementptr %[[BASEDST]][%[[FI2]]] : (!llvm.ptr<3>, i64) -> !llvm.ptr<3>, f32
-  // CHECK-DAG: %[[BASESRC:.*]] = llvm.extractvalue %{{.*}}[1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> 
+  // CHECK-DAG: %[[BASESRC:.*]] = llvm.extractvalue %{{.*}}[1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)>
   // CHECK-DAG: %[[S2:.*]] = llvm.mlir.constant(128 : index) : i64
   // CHECK-DAG: %[[FI2:.*]] = llvm.mul %[[IDX1]], %[[S2]]  : i64
   // CHECK-DAG: %[[FI3:.*]] = llvm.add %[[FI2]], %[[IDX1]]  : i64
@@ -328,7 +328,7 @@ func.func @async_cp_zfill_f32_align1(
   // CHECK-DAG: %[[CAST2:.*]] = llvm.addrspacecast %[[ADDRESSSRC]] : !llvm.ptr to !llvm.ptr<1>
   // CHECK-DAG: %[[c1:.*]] = llvm.mlir.constant(3 : i32) : i32
   // CHECK-DAG: %[[c2:.*]] = llvm.mlir.constant(32 : i32) : i32
-  // CHECK-DAG: %[[c3:.*]] = llvm.trunc %[[SRC1]] : i64 to i32  
+  // CHECK-DAG: %[[c3:.*]] = llvm.trunc %[[SRC1]] : i64 to i32
   // CHECK-DAG: %[[c4:.*]] = llvm.mul %[[c2]], %[[c3]] : i32
   // CHECK-DAG: %[[c5:.*]] = llvm.lshr %[[c4]], %[[c1]] : i32
   // CHECK-DAG: nvvm.cp.async.shared.global %[[ADDRESSDST]], %[[CAST2]], 4, cache = ca, %[[c5]]
@@ -375,7 +375,7 @@ func.func @mma_sp_sync_f16_16832(%arg0: vector<4x2xf16>,
 
   // CHECK-DAG: llvm.extractvalue %[[d]][0] : !llvm.struct<(vector<2xf16>, vector<2xf16>)>
   // CHECK-DAG: llvm.extractvalue %[[d]][1] : !llvm.struct<(vector<2xf16>, vector<2xf16>)>
-  //     CHECK: llvm.mlir.undef : !llvm.array<2 x vector<2xf16>>
+  //     CHECK: llvm.mlir.poison : !llvm.array<2 x vector<2xf16>>
   //     CHECK: llvm.insertvalue %{{.+}}, %{{.+}}[0] : !llvm.array<2 x vector<2xf16>>
   //     CHECK: llvm.insertvalue %{{.+}}, %{{.+}}[1] : !llvm.array<2 x vector<2xf16>>
   return %d : vector<2x2xf16>
@@ -484,22 +484,22 @@ func.func @mbarrier() {
   %barrier = nvgpu.mbarrier.create -> !barrierType
 
   // CHECK: %[[barStr:.+]] =  builtin.unrealized_conversion_cast %[[barMemref]] : memref<1xi64, 3> to !llvm.struct<(ptr<3>, ptr<3>, i64, array<1 x i64>, array<1 x i64>)>
-  // CHECK: %[[base:.+]] = llvm.extractvalue %[[barStr]][1] : !llvm.struct<(ptr<3>, ptr<3>, i64, array<1 x i64>, array<1 x i64>)> 
+  // CHECK: %[[base:.+]] = llvm.extractvalue %[[barStr]][1] : !llvm.struct<(ptr<3>, ptr<3>, i64, array<1 x i64>, array<1 x i64>)>
   // CHECK: %[[barPtr:.+]] = llvm.getelementptr %[[base]][%[[mid]]] : (!llvm.ptr<3>, i64) -> !llvm.ptr<3>, i64
   // CHECK: nvvm.mbarrier.init.shared %[[barPtr]]
     nvgpu.mbarrier.init %barrier[%c0], %num_threads : !barrierType
-  
-  // CHECK: %[[base2:.+]] = llvm.extractvalue %[[barStr]][1] : !llvm.struct<(ptr<3>, ptr<3>, i64, array<1 x i64>, array<1 x i64>)> 
+
+  // CHECK: %[[base2:.+]] = llvm.extractvalue %[[barStr]][1] : !llvm.struct<(ptr<3>, ptr<3>, i64, array<1 x i64>, array<1 x i64>)>
   // CHECK: %[[barPtr2:.+]] = llvm.getelementptr %[[base2]][%[[mid]]] : (!llvm.ptr<3>, i64) -> !llvm.ptr<3>, i64
   // CHECK: %[[token:.+]] = nvvm.mbarrier.arrive.shared %[[barPtr2]]
   %token = nvgpu.mbarrier.arrive %barrier[%c0] : !barrierType -> !tokenType
-    
-  // CHECK: %[[base3:.+]] = llvm.extractvalue %[[barStr]][1] : !llvm.struct<(ptr<3>, ptr<3>, i64, array<1 x i64>, array<1 x i64>)> 
+
+  // CHECK: %[[base3:.+]] = llvm.extractvalue %[[barStr]][1] : !llvm.struct<(ptr<3>, ptr<3>, i64, array<1 x i64>, array<1 x i64>)>
   // CHECK: %[[barPtr3:.+]] = llvm.getelementptr %[[base3]][%[[mid]]] : (!llvm.ptr<3>, i64) -> !llvm.ptr<3>, i64
   // CHECK: nvvm.mbarrier.test.wait.shared %[[barPtr3]], %[[token]]
   %isDone = nvgpu.mbarrier.test.wait %barrier[%c0], %token : !barrierType, !tokenType
 
-  func.return 
+  func.return
 }
 
 // CHECK-LABEL: func @mbarrier_nocomplete
@@ -514,22 +514,22 @@ func.func @mbarrier_nocomplete() {
   %barrier = nvgpu.mbarrier.create -> !barrierType
 
   // CHECK: %[[barStr:.+]] =  builtin.unrealized_conversion_cast %[[barMemref]] : memref<1xi64, 3> to !llvm.struct<(ptr<3>, ptr<3>, i64, array<1 x i64>, array<1 x i64>)>
-  // CHECK: %[[base:.+]] = llvm.extractvalue %[[barStr]][1] : !llvm.struct<(ptr<3>, ptr<3>, i64, array<1 x i64>, array<1 x i64>)> 
+  // CHECK: %[[base:.+]] = llvm.extractvalue %[[barStr]][1] : !llvm.struct<(ptr<3>, ptr<3>, i64, array<1 x i64>, array<1 x i64>)>
   // CHECK: %[[barPtr:.+]] = llvm.getelementptr %[[base]][%[[mid]]] : (!llvm.ptr<3>, i64) -> !llvm.ptr<3>, i64
   // CHECK: nvvm.mbarrier.init.shared %[[barPtr]]
   nvgpu.mbarrier.init %barrier[%c0], %num_threads : !barrierType
-  
-  // CHECK: %[[base2:.+]] = llvm.extractvalue %[[barStr]][1] : !llvm.struct<(ptr<3>, ptr<3>, i64, array<1 x i64>, array<1 x i64>)> 
+
+  // CHECK: %[[base2:.+]] = llvm.extractvalue %[[barStr]][1] : !llvm.struct<(ptr<3>, ptr<3>, i64, array<1 x i64>, array<1 x i64>)>
   // CHECK: %[[barPtr2:.+]] = llvm.getelementptr %[[base2]][%[[mid]]] : (!llvm.ptr<3>, i64) -> !llvm.ptr<3>, i64
   // CHECK: %[[token:.+]] = nvvm.mbarrier.arrive.nocomplete.shared %[[barPtr2]]
   %token = nvgpu.mbarrier.arrive.nocomplete %barrier[%c0], %count : !barrierType -> !tokenType
-    
-  // CHECK: %[[base3:.+]] = llvm.extractvalue %[[barStr]][1] : !llvm.struct<(ptr<3>, ptr<3>, i64, array<1 x i64>, array<1 x i64>)> 
+
+  // CHECK: %[[base3:.+]] = llvm.extractvalue %[[barStr]][1] : !llvm.struct<(ptr<3>, ptr<3>, i64, array<1 x i64>, array<1 x i64>)>
   // CHECK: %[[barPtr3:.+]] = llvm.getelementptr %[[base3]][%[[mid]]] : (!llvm.ptr<3>, i64) -> !llvm.ptr<3>, i64
   // CHECK: nvvm.mbarrier.test.wait.shared %[[barPtr3]], %[[token]]
   %isDone = nvgpu.mbarrier.test.wait %barrier[%c0], %token : !barrierType, !tokenType
 
-  func.return 
+  func.return
 }
 
 // CHECK-LABEL: func @mbarrier_wait(
@@ -542,13 +542,13 @@ func.func @mbarrier_wait(%barriers : !nvgpu.mbarrier.group<memorySpace = #gpu.ad
   %n = arith.constant 100 : index
 
   %numBarriers = arith.constant 5 : index
-  
+
   scf.for %i = %c0 to %n step %c1 {
 // CHECK: %[[c5:.+]] = arith.constant 5 : index
 // CHECK: scf.for %[[i:.*]] =
 // CHECK: %[[S2:.+]] = arith.remui %[[i]], %[[c5]] : index
 // CHECK: %[[S3:.+]] = builtin.unrealized_conversion_cast %[[S2]] : index to i64
-// CHECK: %[[S4:.+]] = llvm.extractvalue %[[CARG0]][1] : !llvm.struct<(ptr<3>, ptr<3>, i64, array<1 x i64>, array<1 x i64>)> 
+// CHECK: %[[S4:.+]] = llvm.extractvalue %[[CARG0]][1] : !llvm.struct<(ptr<3>, ptr<3>, i64, array<1 x i64>, array<1 x i64>)>
 // CHECK: %[[S5:.+]] = llvm.getelementptr %[[S4]][%[[S3]]] : (!llvm.ptr<3>, i64) -> !llvm.ptr<3>, i64
 // CHECK: nvvm.mbarrier.test.wait.shared {{.*}}, %[[CARG1]]
     %mbarId = arith.remui %i, %numBarriers : index
@@ -562,46 +562,46 @@ func.func @mbarrier_txcount() {
     %num_threads = arith.constant 128 : index
     // CHECK: %[[c0:.+]] = arith.constant 0 : index
     // CHECK: %[[mid:.+]] = builtin.unrealized_conversion_cast %[[c0]] : index to i64
-    %c0 = arith.constant 0 : index  
+    %c0 = arith.constant 0 : index
 
     // CHECK: %[[barMemref:.+]] = memref.get_global @__mbarrier{{.*}} : memref<1xi64, 3>
     %barrier = nvgpu.mbarrier.create -> !barrierType
 
     // CHECK: %[[barStr:.+]] =  builtin.unrealized_conversion_cast %[[barMemref]] : memref<1xi64, 3> to !llvm.struct<(ptr<3>, ptr<3>, i64, array<1 x i64>, array<1 x i64>)>
-    // CHECK: %[[base:.+]] = llvm.extractvalue %[[barStr]][1] : !llvm.struct<(ptr<3>, ptr<3>, i64, array<1 x i64>, array<1 x i64>)> 
+    // CHECK: %[[base:.+]] = llvm.extractvalue %[[barStr]][1] : !llvm.struct<(ptr<3>, ptr<3>, i64, array<1 x i64>, array<1 x i64>)>
     // CHECK: %[[barPtr:.+]] = llvm.getelementptr %[[base]][%[[mid]]] : (!llvm.ptr<3>, i64) -> !llvm.ptr<3>, i64
     // CHECK: nvvm.mbarrier.init.shared %[[barPtr]]
     nvgpu.mbarrier.init %barrier[%c0], %num_threads : !barrierType
-    
+
     %tidxreg = nvvm.read.ptx.sreg.tid.x : i32
     %tidx = arith.index_cast %tidxreg : i32 to index
-    %cnd = arith.cmpi eq, %tidx, %c0 : index  
+    %cnd = arith.cmpi eq, %tidx, %c0 : index
 
     scf.if %cnd {
       %txcount = arith.constant 256 : index
-      // CHECK: %[[base2:.+]] = llvm.extractvalue %[[barStr]][1] : !llvm.struct<(ptr<3>, ptr<3>, i64, array<1 x i64>, array<1 x i64>)> 
+      // CHECK: %[[base2:.+]] = llvm.extractvalue %[[barStr]][1] : !llvm.struct<(ptr<3>, ptr<3>, i64, array<1 x i64>, array<1 x i64>)>
       // CHECK: %[[barPtr2:.+]] = llvm.getelementptr %[[base2]][%[[mid]]] : (!llvm.ptr<3>, i64) -> !llvm.ptr<3>, i64
       // CHECK: nvvm.mbarrier.arrive.expect_tx.shared %[[barPtr2]]
       nvgpu.mbarrier.arrive.expect_tx %barrier[%c0], %txcount : !barrierType
-      scf.yield 
+      scf.yield
     } else {
       %txcount = arith.constant 0 : index
-      // CHECK: %[[base2:.+]] = llvm.extractvalue %[[barStr]][1] : !llvm.struct<(ptr<3>, ptr<3>, i64, array<1 x i64>, array<1 x i64>)> 
+      // CHECK: %[[base2:.+]] = llvm.extractvalue %[[barStr]][1] : !llvm.struct<(ptr<3>, ptr<3>, i64, array<1 x i64>, array<1 x i64>)>
       // CHECK: %[[barPtr2:.+]] = llvm.getelementptr %[[base2]][%[[mid]]] : (!llvm.ptr<3>, i64) -> !llvm.ptr<3>, i64
       // CHECK: nvvm.mbarrier.arrive.expect_tx.shared %[[barPtr2]]
       nvgpu.mbarrier.arrive.expect_tx %barrier[%c0], %txcount : !barrierType
-      scf.yield 
+      scf.yield
     }
-      
+
 
     %phase_c0 = arith.constant 0 : i1
     %ticks = arith.constant 10000000 : index
-    // CHECK: %[[base3:.+]] = llvm.extractvalue %[[barStr]][1] : !llvm.struct<(ptr<3>, ptr<3>, i64, array<1 x i64>, array<1 x i64>)> 
+    // CHECK: %[[base3:.+]] = llvm.extractvalue %[[barStr]][1] : !llvm.struct<(ptr<3>, ptr<3>, i64, array<1 x i64>, array<1 x i64>)>
     // CHECK: %[[barPtr3:.+]] = llvm.getelementptr %[[base3]][%[[mid]]] : (!llvm.ptr<3>, i64) -> !llvm.ptr<3>, i64
     // CHECK: nvvm.mbarrier.try_wait.parity.shared %[[barPtr3]]
     nvgpu.mbarrier.try_wait.parity %barrier[%c0], %phase_c0, %ticks : !barrierType
 
-    func.return 
+    func.return
 }
 
 // CHECK-LABEL: func @mbarrier_txcount_pred
@@ -611,7 +611,7 @@ func.func @mbarrier_txcount_pred() {
     // CHECK: %[[mid:.+]] = builtin.unrealized_conversion_cast %[[c0]] : index to i64
     // CHECK: %[[S2:.+]] = gpu.thread_id  x
     // CHECK: %[[P:.+]] = arith.cmpi eq, %[[S2]], %[[c0]] : index
-    %c0 = arith.constant 0 : index  
+    %c0 = arith.constant 0 : index
     %tidx = gpu.thread_id x
     %pred = arith.cmpi eq, %tidx, %c0 : index
 
@@ -619,25 +619,25 @@ func.func @mbarrier_txcount_pred() {
     %barrier = nvgpu.mbarrier.create -> !barrierType
 
     // CHECK: %[[barStr:.+]] =  builtin.unrealized_conversion_cast %[[barMemref]] : memref<1xi64, 3> to !llvm.struct<(ptr<3>, ptr<3>, i64, array<1 x i64>, array<1 x i64>)>
-    // CHECK: %[[base:.+]] = llvm.extractvalue %[[barStr]][1] : !llvm.struct<(ptr<3>, ptr<3>, i64, array<1 x i64>, array<1 x i64>)> 
+    // CHECK: %[[base:.+]] = llvm.extractvalue %[[barStr]][1] : !llvm.struct<(ptr<3>, ptr<3>, i64, array<1 x i64>, array<1 x i64>)>
     // CHECK: %[[barPtr:.+]] = llvm.getelementptr %[[base]][%[[mid]]] : (!llvm.ptr<3>, i64) -> !llvm.ptr<3>, i64
     // CHECK: nvvm.mbarrier.init.shared %[[barPtr]], {{.*}}, predicate = %[[P]]
     nvgpu.mbarrier.init %barrier[%c0], %mine, predicate = %pred : !barrierType
-    
+
     %txcount = arith.constant 256 : index
-    // CHECK: %[[base2:.+]] = llvm.extractvalue %[[barStr]][1] : !llvm.struct<(ptr<3>, ptr<3>, i64, array<1 x i64>, array<1 x i64>)> 
+    // CHECK: %[[base2:.+]] = llvm.extractvalue %[[barStr]][1] : !llvm.struct<(ptr<3>, ptr<3>, i64, array<1 x i64>, array<1 x i64>)>
     // CHECK: %[[barPtr2:.+]] = llvm.getelementptr %[[base2]][%[[mid]]] : (!llvm.ptr<3>, i64) -> !llvm.ptr<3>, i64
     // CHECK: nvvm.mbarrier.arrive.expect_tx.shared %[[barPtr2]], {{.*}}, predicate = %[[P]]
     nvgpu.mbarrier.arrive.expect_tx %barrier[%c0], %txcount, predicate = %pred : !barrierType
 
     %phase_c0 = arith.constant 0 : i1
     %ticks = arith.constant 10000000 : index
-    // CHECK: %[[base3:.+]] = llvm.extractvalue %[[barStr]][1] : !llvm.struct<(ptr<3>, ptr<3>, i64, array<1 x i64>, array<1 x i64>)> 
+    // CHECK: %[[base3:.+]] = llvm.extractvalue %[[barStr]][1] : !llvm.struct<(ptr<3>, ptr<3>, i64, array<1 x i64>, array<1 x i64>)>
     // CHECK: %[[barPtr3:.+]] = llvm.getelementptr %[[base3]][%[[mid]]] : (!llvm.ptr<3>, i64) -> !llvm.ptr<3>, i64
     // CHECK: nvvm.mbarrier.try_wait.parity.shared %[[barPtr3]]
     nvgpu.mbarrier.try_wait.parity %barrier[%c0], %phase_c0, %ticks : !barrierType
 
-    func.return 
+    func.return
 }
 
 // CHECK-LABEL: func @async_tma_load
@@ -647,27 +647,27 @@ func.func @mbarrier_txcount_pred() {
 !tensorMap4d = !nvgpu.tensormap.descriptor<tensor = memref<2x2x32x32xf32,3>,   swizzle=swizzle_128b,l2promo = l2promo_128b,oob = zero, interleave = none>
 !tensorMap5d = !nvgpu.tensormap.descriptor<tensor = memref<2x2x2x32x32xf32,3>, swizzle=none,        l2promo = none,        oob = zero, interleave = none>
 !mbarrier = !nvgpu.mbarrier.group<memorySpace = #gpu.address_space<workgroup>>
-func.func @async_tma_load(%tensorMap1d: !tensorMap1d, %tensorMap2d: !tensorMap2d, %tensorMap3d: !tensorMap3d, %tensorMap4d: !tensorMap4d, %tensorMap5d: !tensorMap5d, 
-                              %buffer1d: memref<128xf32,3>,      
-                              %buffer2d: memref<32x32xf32,3>,    
-                              %buffer3d: memref<2x32x32xf32,3>,  
-                              %buffer4d: memref<2x2x32x32xf32,3>,  
+func.func @async_tma_load(%tensorMap1d: !tensorMap1d, %tensorMap2d: !tensorMap2d, %tensorMap3d: !tensorMap3d, %tensorMap4d: !tensorMap4d, %tensorMap5d: !tensorMap5d,
+                              %buffer1d: memref<128xf32,3>,
+                              %buffer2d: memref<32x32xf32,3>,
+                              %buffer3d: memref<2x32x32xf32,3>,
+                              %buffer4d: memref<2x2x32x32xf32,3>,
                               %buffer5d: memref<2x2x2x32x32xf32,3>,
                               %mbarrier: !mbarrier) {
   %c0 = arith.constant 0 : index
   %crd0 = arith.constant 0 : index
   %crd1 = arith.constant 0 : index
-  // CHECK: nvvm.cp.async.bulk.tensor.shared.cluster.global %{{.*}}, %{{.*}}, %{{.*}} box[%{{.*}}] 
+  // CHECK: nvvm.cp.async.bulk.tensor.shared.cluster.global %{{.*}}, %{{.*}}, %{{.*}} box[%{{.*}}]
   nvgpu.tma.async.load %tensorMap1d[%crd0], %mbarrier[%c0] to %buffer1d : !tensorMap1d, !mbarrier -> memref<128xf32,3>
-  // CHECK: nvvm.cp.async.bulk.tensor.shared.cluster.global %{{.*}}, %{{.*}}, %{{.*}} box[%{{.*}}, %{{.*}}] 
+  // CHECK: nvvm.cp.async.bulk.tensor.shared.cluster.global %{{.*}}, %{{.*}}, %{{.*}} box[%{{.*}}, %{{.*}}]
   nvgpu.tma.async.load %tensorMap2d[%crd0, %crd1], %mbarrier[%c0] to %buffer2d : !tensorMap2d, !mbarrier -> memref<32x32xf32,3>
-  // CHECK: nvvm.cp.async.bulk.tensor.shared.cluster.global %{{.*}}, %{{.*}}, %{{.*}} box[%{{.*}}, %{{.*}}, %{{.*}}] 
+  // CHECK: nvvm.cp.async.bulk.tensor.shared.cluster.global %{{.*}}, %{{.*}}, %{{.*}} box[%{{.*}}, %{{.*}}, %{{.*}}]
   nvgpu.tma.async.load %tensorMap3d[%crd0, %crd1, %crd0], %mbarrier[%c0] to %buffer3d : !tensorMap3d, !mbarrier -> memref<2x32x32xf32,3>
-  // CHECK: nvvm.cp.async.bulk.tensor.shared.cluster.global %{{.*}}, %{{.*}}, %{{.*}} box[%{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}] 
+  // CHECK: nvvm.cp.async.bulk.tensor.shared.cluster.global %{{.*}}, %{{.*}}, %{{.*}} box[%{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}]
   nvgpu.tma.async.load %tensorMap4d[%crd0, %crd1, %crd1, %crd0], %mbarrier[%c0] to %buffer4d : !tensorMap4d, !mbarrier -> memref<2x2x32x32xf32,3>
-  // CHECK: nvvm.cp.async.bulk.tensor.shared.cluster.global %{{.*}}, %{{.*}}, %{{.*}} box[%{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}] 
+  // CHECK: nvvm.cp.async.bulk.tensor.shared.cluster.global %{{.*}}, %{{.*}}, %{{.*}} box[%{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}]
   nvgpu.tma.async.load %tensorMap5d[%crd0, %crd1, %crd1, %crd0, %crd0], %mbarrier[%c0] to %buffer5d : !tensorMap5d, !mbarrier -> memref<2x2x2x32x32xf32,3>
-  func.return 
+  func.return
 }
 
 // CHECK-LABEL: func @async_tma_load
@@ -678,17 +678,17 @@ func.func @async_tma_load_gpu_address_space(%tensorMap1d: !tensorMap1dgpuspace,
   %c0 = arith.constant 0 : index
   %crd0 = arith.constant 0 : index
   %crd1 = arith.constant 0 : index
-  // CHECK: nvvm.cp.async.bulk.tensor.shared.cluster.global %{{.*}}, %{{.*}}, %{{.*}} box[%{{.*}}] 
+  // CHECK: nvvm.cp.async.bulk.tensor.shared.cluster.global %{{.*}}, %{{.*}}, %{{.*}} box[%{{.*}}]
   nvgpu.tma.async.load %tensorMap1d[%crd0], %mbarrier[%c0] to %buffer1d : !tensorMap1dgpuspace, !mbarrier -> memref<128xf32,#gpu.address_space<workgroup>>
-   func.return 
+   func.return
 }
 
 // CHECK-LABEL: func @async_tma_load_pred
-func.func @async_tma_load_pred(%tensorMap1d: !tensorMap1d, %tensorMap2d: !tensorMap2d, %tensorMap3d: !tensorMap3d, %tensorMap4d: !tensorMap4d, %tensorMap5d: !tensorMap5d, 
-                              %buffer1d: memref<128xf32,3>,      
-                              %buffer2d: memref<32x32xf32,3>,    
-                              %buffer3d: memref<2x32x32xf32,3>,  
-                              %buffer4d: memref<2x2x32x32xf32,3>,  
+func.func @async_tma_load_pred(%tensorMap1d: !tensorMap1d, %tensorMap2d: !tensorMap2d, %tensorMap3d: !tensorMap3d, %tensorMap4d: !tensorMap4d, %tensorMap5d: !tensorMap5d,
+                              %buffer1d: memref<128xf32,3>,
+                              %buffer2d: memref<32x32xf32,3>,
+                              %buffer3d: memref<2x32x32xf32,3>,
+                              %buffer4d: memref<2x2x32x32xf32,3>,
                               %buffer5d: memref<2x2x2x32x32xf32,3>,
                               %mbarrier: !mbarrier,
                               %p: i1) {
@@ -705,61 +705,61 @@ func.func @async_tma_load_pred(%tensorMap1d: !tensorMap1d, %tensorMap2d: !tensor
   nvgpu.tma.async.load %tensorMap4d[%crd0, %crd1, %crd1, %crd0], %mbarrier[%c0] to %buffer4d, predicate = %p : !tensorMap4d, !mbarrier -> memref<2x2x32x32xf32,3>
   // CHECK: nvvm.cp.async.bulk.tensor.shared.cluster.global %{{.*}}, %{{.*}}, %{{.*}} box[%{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}] predicate = %{{.*}}
   nvgpu.tma.async.load %tensorMap5d[%crd0, %crd1, %crd1, %crd0, %crd0], %mbarrier[%c0] to %buffer5d, predicate = %p : !tensorMap5d, !mbarrier -> memref<2x2x2x32x32xf32,3>
-  func.return 
+  func.return
 }
 
 func.func @async_tma_load_multicast(
-  %tensorMap1d: !tensorMap1d, %tensorMap2d: !tensorMap2d, 
-  %tensorMap3d: !tensorMap3d, %tensorMap4d: !tensorMap4d, 
-  %tensorMap5d: !tensorMap5d, %buffer1d: memref<128xf32,3>,      
-  %buffer2d: memref<32x32xf32,3>, %buffer3d: memref<2x32x32xf32,3>,  
+  %tensorMap1d: !tensorMap1d, %tensorMap2d: !tensorMap2d,
+  %tensorMap3d: !tensorMap3d, %tensorMap4d: !tensorMap4d,
+  %tensorMap5d: !tensorMap5d, %buffer1d: memref<128xf32,3>,
+  %buffer2d: memref<32x32xf32,3>, %buffer3d: memref<2x32x32xf32,3>,
   %buffer4d: memref<2x2x32x32xf32,3>, %buffer5d: memref<2x2x2x32x32xf32,3>,
   %mbarrier: !mbarrier,
   %multicastMask: i16) {
   %c0 = arith.constant 0 : index
   %crd0 = arith.constant 0 : index
   %crd1 = arith.constant 0 : index
-  // CHECK: nvvm.cp.async.bulk.tensor.shared.cluster.global %{{.*}}, %{{.*}}, %{{.*}} box[%{{.*}}] 
+  // CHECK: nvvm.cp.async.bulk.tensor.shared.cluster.global %{{.*}}, %{{.*}}, %{{.*}} box[%{{.*}}]
   nvgpu.tma.async.load %tensorMap1d[%crd0], %mbarrier[%c0] to %buffer1d multicast_mask = %multicastMask : !tensorMap1d, !mbarrier -> memref<128xf32,3>
-  // CHECK: nvvm.cp.async.bulk.tensor.shared.cluster.global %{{.*}}, %{{.*}}, %{{.*}} box[%{{.*}}, %{{.*}}] 
+  // CHECK: nvvm.cp.async.bulk.tensor.shared.cluster.global %{{.*}}, %{{.*}}, %{{.*}} box[%{{.*}}, %{{.*}}]
   nvgpu.tma.async.load %tensorMap2d[%crd0, %crd1], %mbarrier[%c0] to %buffer2d multicast_mask = %multicastMask : !tensorMap2d, !mbarrier -> memref<32x32xf32,3>
-  // CHECK: nvvm.cp.async.bulk.tensor.shared.cluster.global %{{.*}}, %{{.*}}, %{{.*}} box[%{{.*}}, %{{.*}}, %{{.*}}] 
+  // CHECK: nvvm.cp.async.bulk.tensor.shared.cluster.global %{{.*}}, %{{.*}}, %{{.*}} box[%{{.*}}, %{{.*}}, %{{.*}}]
   nvgpu.tma.async.load %tensorMap3d[%crd0, %crd1, %crd0], %mbarrier[%c0] to %buffer3d multicast_mask = %multicastMask : !tensorMap3d, !mbarrier -> memref<2x32x32xf32,3>
-  // CHECK: nvvm.cp.async.bulk.tensor.shared.cluster.global %{{.*}}, %{{.*}}, %{{.*}} box[%{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}] 
+  // CHECK: nvvm.cp.async.bulk.tensor.shared.cluster.global %{{.*}}, %{{.*}}, %{{.*}} box[%{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}]
   nvgpu.tma.async.load %tensorMap4d[%crd0, %crd1, %crd1, %crd0], %mbarrier[%c0] to %buffer4d multicast_mask = %multicastMask : !tensorMap4d, !mbarrier -> memref<2x2x32x32xf32,3>
-  // CHECK: nvvm.cp.async.bulk.tensor.shared.cluster.global %{{.*}}, %{{.*}}, %{{.*}} box[%{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}] 
+  // CHECK: nvvm.cp.async.bulk.tensor.shared.cluster.global %{{.*}}, %{{.*}}, %{{.*}} box[%{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}]
   nvgpu.tma.async.load %tensorMap5d[%crd0, %crd1, %crd1, %crd0, %crd0], %mbarrier[%c0] to %buffer5d multicast_mask = %multicastMask : !tensorMap5d, !mbarrier -> memref<2x2x2x32x32xf32,3>
-  func.return 
+  func.return
 }
 
-func.func @async_tma_store(%tensorMap1d: !tensorMap1d, %tensorMap2d: !tensorMap2d, %tensorMap3d: !tensorMap3d, %tensorMap4d: !tensorMap4d, %tensorMap5d: !tensorMap5d, 
-                           %buffer1d: memref<128xf32,3>,      
-                           %buffer2d: memref<32x32xf32,3>,    
-                           %buffer3d: memref<2x32x32xf32,3>,  
-                           %buffer4d: memref<2x2x32x32xf32,3>,  
+func.func @async_tma_store(%tensorMap1d: !tensorMap1d, %tensorMap2d: !tensorMap2d, %tensorMap3d: !tensorMap3d, %tensorMap4d: !tensorMap4d, %tensorMap5d: !tensorMap5d,
+                           %buffer1d: memref<128xf32,3>,
+                           %buffer2d: memref<32x32xf32,3>,
+                           %buffer3d: memref<2x32x32xf32,3>,
+                           %buffer4d: memref<2x2x32x32xf32,3>,
                            %buffer5d: memref<2x2x2x32x32xf32,3>) {
   %c0 = arith.constant 0 : index
   %crd0 = arith.constant 0 : index
   %crd1 = arith.constant 0 : index
-  // CHECK: nvvm.cp.async.bulk.tensor.global.shared.cta %{{.*}} %{{.*}}, box[%{{.*}}] 
-  nvgpu.tma.async.store %buffer1d to %tensorMap1d[%crd0] : memref<128xf32,3> -> !tensorMap1d 
-  // CHECK: nvvm.cp.async.bulk.tensor.global.shared.cta %{{.*}} %{{.*}}, box[%{{.*}}, %{{.*}}] 
+  // CHECK: nvvm.cp.async.bulk.tensor.global.shared.cta %{{.*}} %{{.*}}, box[%{{.*}}]
+  nvgpu.tma.async.store %buffer1d to %tensorMap1d[%crd0] : memref<128xf32,3> -> !tensorMap1d
+  // CHECK: nvvm.cp.async.bulk.tensor.global.shared.cta %{{.*}} %{{.*}}, box[%{{.*}}, %{{.*}}]
   nvgpu.tma.async.store %buffer2d to %tensorMap2d[%crd0, %crd1]  : memref<32x32xf32,3> -> !tensorMap2d
-  // CHECK: nvvm.cp.async.bulk.tensor.global.shared.cta %{{.*}} %{{.*}}, box[%{{.*}}, %{{.*}}, %{{.*}}] 
+  // CHECK: nvvm.cp.async.bulk.tensor.global.shared.cta %{{.*}} %{{.*}}, box[%{{.*}}, %{{.*}}, %{{.*}}]
   nvgpu.tma.async.store %buffer3d to %tensorMap3d[%crd0, %crd1, %crd0]  : memref<2x32x32xf32,3> -> !tensorMap3d
-  // CHECK: nvvm.cp.async.bulk.tensor.global.shared.cta %{{.*}} %{{.*}}, box[%{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}] 
+  // CHECK: nvvm.cp.async.bulk.tensor.global.shared.cta %{{.*}} %{{.*}}, box[%{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}]
   nvgpu.tma.async.store %buffer4d to %tensorMap4d[%crd0, %crd1, %crd1, %crd0]  : memref<2x2x32x32xf32,3> -> !tensorMap4d
-  // CHECK: nvvm.cp.async.bulk.tensor.global.shared.cta %{{.*}} %{{.*}}, box[%{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}] 
+  // CHECK: nvvm.cp.async.bulk.tensor.global.shared.cta %{{.*}} %{{.*}}, box[%{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}]
   nvgpu.tma.async.store %buffer5d to %tensorMap5d[%crd0, %crd1, %crd1, %crd0, %crd0]  : memref<2x2x2x32x32xf32,3> -> !tensorMap5d
-  func.return 
+  func.return
 }
 
 
-func.func @async_tma_store_predicate(%tensorMap1d: !tensorMap1d, %tensorMap2d: !tensorMap2d, %tensorMap3d: !tensorMap3d, %tensorMap4d: !tensorMap4d, %tensorMap5d: !tensorMap5d, 
-                           %buffer1d: memref<128xf32,3>,      
-                           %buffer2d: memref<32x32xf32,3>,    
-                           %buffer3d: memref<2x32x32xf32,3>,  
-                           %buffer4d: memref<2x2x32x32xf32,3>,  
+func.func @async_tma_store_predicate(%tensorMap1d: !tensorMap1d, %tensorMap2d: !tensorMap2d, %tensorMap3d: !tensorMap3d, %tensorMap4d: !tensorMap4d, %tensorMap5d: !tensorMap5d,
+                           %buffer1d: memref<128xf32,3>,
+                           %buffer2d: memref<32x32xf32,3>,
+                           %buffer3d: memref<2x32x32xf32,3>,
+                           %buffer4d: memref<2x2x32x32xf32,3>,
                            %buffer5d: memref<2x2x2x32x32xf32,3>,
                            %p: i1) {
   %c0 = arith.constant 0 : index
@@ -775,7 +775,7 @@ func.func @async_tma_store_predicate(%tensorMap1d: !tensorMap1d, %tensorMap2d: !
   nvgpu.tma.async.store %buffer4d to %tensorMap4d[%crd0, %crd1, %crd1, %crd0], predicate = %p  : memref<2x2x32x32xf32,3> -> !tensorMap4d
   // CHECK: nvvm.cp.async.bulk.tensor.global.shared.cta %{{.*}} %{{.*}}, box[%{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}], predicate = %{{.*}}
   nvgpu.tma.async.store %buffer5d to %tensorMap5d[%crd0, %crd1, %crd1, %crd0, %crd0], predicate = %p  : memref<2x2x2x32x32xf32,3> -> !tensorMap5d
-  func.return 
+  func.return
 }
 
 func.func @create_tensor_map(%devicePtr2d : memref<64x128xf32>, %devicePtr1d : memref<128xf32>) {
@@ -791,7 +791,7 @@ func.func @create_tensor_map(%devicePtr2d : memref<64x128xf32>, %devicePtr1d : m
   func.return
 }
 
-// CHECK-LABEL: @tma_prefetch(  
+// CHECK-LABEL: @tma_prefetch(
 // CHECK-SAME: %[[arg0:[a-zA-Z0-9_]+]]: !nvgpu.tensormap.descriptor<tensor = memref<128xf32, 3>, swizzle = none, l2promo = none, oob = nan, interleave = none>, %[[arg1:[a-zA-Z0-9_]+]]: i1
 func.func @tma_prefetch(%tensorMap1d: !tensorMap1d, %p : i1) {
   // CHECK: %[[S0:.+]] = builtin.unrealized_conversion_cast %[[arg0]] : !nvgpu.tensormap.descriptor<tensor = memref<128xf32, 3>, swizzle = none, l2promo = none, oob = nan, interleave = none> to !llvm.ptr
@@ -818,7 +818,7 @@ module @mymodule {
     %rhsShmem = memref.subview %rhsShmem3[0, 0, 0][1, 64, 64][1, 1, 1]  : memref<1x64x64xf16, strided<[4096, 64, 1], offset: 8192>, 3> to memref<64x64xf16, strided<[64, 1], offset: 8192>, 3>
     // CHECK: nvvm.cp.async.bulk.tensor.shared.cluster.global
     nvgpu.tma.async.load %lhsTensorMap[%c0, %c0], %mbarrier[%c0] to %lhsShmem : !lhsTensorMap, !barrierType -> memref<128x64xf16,3>
-    // CHECK: %[[desc:.+]] = llvm.extractvalue %{{.*}}[1] : !llvm.struct<(ptr<3>, ptr<3>, i64, array<2 x i64>, array<2 x i64>)> 
+    // CHECK: %[[desc:.+]] = llvm.extractvalue %{{.*}}[1] : !llvm.struct<(ptr<3>, ptr<3>, i64, array<2 x i64>, array<2 x i64>)>
     // CHECK: %[[c8192:.+]] = llvm.mlir.constant(8192 : index) : i64
     // CHECK: %[[shmemOfset:.+]] = llvm.getelementptr %[[desc]][%[[c8192]]] : (!llvm.ptr<3>, i64)
     // CHECK: nvvm.cp.async.bulk.tensor.shared.cluster.global %[[shmemOfset]], %{{.*}}, %{{.*}}, box[%{{.*}}, %{{.*}}]
@@ -838,7 +838,7 @@ func.func @create_wgmma_descriptor(%tensorMap : !tensorMap) -> !nvgpu.warpgroup.
     // CHECK: %[[S1:.+]] = builtin.unrealized_conversion_cast %[[Sre]] : memref<128x64xf16, 3> to !llvm.struct<(ptr<3>, ptr<3>, i64, array<2 x i64>, array<2 x i64>)>
     // CHECK: %[[c64:.+]] =  llvm.mlir.constant(64 : i64) : i64
     // CHECK: %[[c1024:.+]] = llvm.mlir.constant(1024 : i64) : i64
-    // CHECK: %[[S2:.+]] = llvm.extractvalue %[[S1]][1] : !llvm.struct<(ptr<3>, ptr<3>, i64, array<2 x i64>, array<2 x i64>)> 
+    // CHECK: %[[S2:.+]] = llvm.extractvalue %[[S1]][1] : !llvm.struct<(ptr<3>, ptr<3>, i64, array<2 x i64>, array<2 x i64>)>
     // CHECK: %[[S3:.+]] = llvm.ptrtoint %[[S2]] : !llvm.ptr<3> to i64
     // CHECK: %[[S4:.+]] = llvm.mlir.constant(46 : i64) : i64
     // CHECK: %[[S5:.+]] = llvm.shl %[[S3]], %[[S4]]  : i64
@@ -862,25 +862,25 @@ func.func @create_wgmma_descriptor(%tensorMap : !tensorMap) -> !nvgpu.warpgroup.
     // CHECK: %[[S25:.+]] = llvm.mlir.constant(0 : i64) : i64
     // CHECK: %[[S26:.+]] = llvm.shl %[[S7]], %[[S25]]  : i64
     // CHECK: %[[S27:.+]] = llvm.or %[[S24]], %[[S26]]  : i64
-    // CHECK: %[[ret:.+]] = builtin.unrealized_conversion_cast %[[S27]] : i64 to !nvgpu.warpgroup.descriptor<tensor = memref<128x64xf16, 3>> 
+    // CHECK: %[[ret:.+]] = builtin.unrealized_conversion_cast %[[S27]] : i64 to !nvgpu.warpgroup.descriptor<tensor = memref<128x64xf16, 3>>
     // CHECK: return %[[ret]]
   %descA = nvgpu.warpgroup.generate.descriptor %lhsShmem, %tensorMap : memref<128x64xf16,3>, !tensorMap -> !nvgpu.warpgroup.descriptor<tensor=memref<128x64xf16,3>>
   func.return %descA : !nvgpu.warpgroup.descriptor<tensor=memref<128x64xf16,3>>
 }
 
-// CHECK-LABEL: @warpgroup_mma_128_128_64(  
+// CHECK-LABEL: @warpgroup_mma_128_128_64(
 // CHECK-SAME: %[[arg0:[a-zA-Z0-9_]+]]: !nvgpu.warpgroup.descriptor<tensor = memref<128x64xf16, 3>>, %[[arg1:[a-zA-Z0-9_]+]]: !nvgpu.warpgroup.descriptor<tensor = memref<64x128xf16, 3>>, %[[arg2:[a-zA-Z0-9_]+]]: !nvgpu.warpgroup.accumulator<fragmented = vector<128x128xf32>>)
 func.func @warpgroup_mma_128_128_64(
-      %descA: !nvgpu.warpgroup.descriptor<tensor = memref<128x64xf16, 3>>, 
-      %descB: !nvgpu.warpgroup.descriptor<tensor = memref<64x128xf16, 3>>, 
-      %acc: !nvgpu.warpgroup.accumulator<fragmented = vector<128x128xf32>>) 
+      %descA: !nvgpu.warpgroup.descriptor<tensor = memref<128x64xf16, 3>>,
+      %descB: !nvgpu.warpgroup.descriptor<tensor = memref<64x128xf16, 3>>,
+      %acc: !nvgpu.warpgroup.accumulator<fragmented = vector<128x128xf32>>)
 {
 // CHECK-DAG: %[[S0:.+]] = builtin.unrealized_conversion_cast %[[arg0]] : !nvgpu.warpgroup.descriptor<tensor = memref<128x64xf16, 3>> to i64
 // CHECK-DAG: %[[S1:.+]] = builtin.unrealized_conversion_cast %[[arg1]] : !nvgpu.warpgroup.descriptor<tensor = memref<64x128xf16, 3>> to i64
 // CHECK-DAG: %[[ARG:.+]] = builtin.unrealized_conversion_cast %[[arg2]] : !nvgpu.warpgroup.accumulator<fragmented = vector<128x128xf32>> to !llvm.struct<(struct<(f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32)>, struct<(f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32)>)>
 // CHECK: nvvm.wgmma.fence.aligned
-// CHECK: %[[UD:.+]] =  llvm.mlir.undef : !llvm.struct<(struct<(f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32)>, struct<(f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32)>)>
-// CHECK: %[[S2:.+]] = llvm.extractvalue %[[ARG]][0] : !llvm.struct<(struct<(f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32)>, struct<(f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32)>)> 
+// CHECK: %[[UD:.+]] =  llvm.mlir.poison : !llvm.struct<(struct<(f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32)>, struct<(f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32)>)>
+// CHECK: %[[S2:.+]] = llvm.extractvalue %[[ARG]][0] : !llvm.struct<(struct<(f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32)>, struct<(f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32)>)>
 // CHECK: %[[S4:.+]] = nvvm.wgmma.mma_async %[[S0]], %[[S1]], %[[S2]], <m = 64, n = 128, k = 16>, D[<f32>, <one>, <wrapped>], A[<f16>, <one>, <row>], B[<f16>, <one>, <row>] : !llvm.struct<(f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32)> -> !llvm.struct<(f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32)>
 // CHECK: %[[S5:.+]] = llvm.mlir.constant(2 : i32) : i64
 // CHECK: %[[S6:.+]] = llvm.add %[[S0]], %[[S5]] : i64
@@ -897,7 +897,7 @@ func.func @warpgroup_mma_128_128_64(
 // CHECK: %[[S17:.+]] = llvm.mlir.constant(384 : i32) : i64
 // CHECK: %[[S18:.+]] = llvm.add %[[S1]], %[[S17]]  : i64
 // CHECK: %[[S19:.+]] = nvvm.wgmma.mma_async %[[S16]], %[[S18]], %[[S14]], <m = 64, n = 128, k = 16>, D[<f32>, <one>, <wrapped>], A[<f16>, <one>, <row>], B[<f16>, <one>, <row>] : !llvm.struct
-// CHECK: %[[S3:.+]] = llvm.extractvalue %[[ARG]][1] : !llvm.struct<(struct<(f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32)>, struct<(f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32)>)> 
+// CHECK: %[[S3:.+]] = llvm.extractvalue %[[ARG]][1] : !llvm.struct<(struct<(f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32)>, struct<(f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32)>)>
 // CHECK: %[[S21:.+]] = llvm.mlir.constant(512 : i32) : i64
 // CHECK: %[[S22:.+]] = llvm.add %[[S0]], %[[S21]]  : i64
 // CHECK: %[[S23:.+]] = nvvm.wgmma.mma_async %[[S22]], %[[S1]], %[[S3]], <m = 64, n = 128, k = 16>, D[<f32>, <one>, <wrapped>], A[<f16>, <one>, <row>], B[<f16>, <one>, <row>] : !llvm.struct
@@ -916,26 +916,26 @@ func.func @warpgroup_mma_128_128_64(
 // CHECK: %[[S36:.+]] = llvm.mlir.constant(384 : i32) : i64
 // CHECK: %[[S37:.+]] = llvm.add %[[S1]], %[[S36]]  : i64
 // CHECK: %[[S38:.+]] = nvvm.wgmma.mma_async %[[S35]], %[[S37]], %[[S33]], <m = 64, n = 128, k = 16>, D[<f32>, <one>, <wrapped>], A[<f16>, <one>, <row>], B[<f16>, <one>, <row>] : !llvm.struct
-// CHECK: %[[S40:.+]] = llvm.insertvalue %[[S19]], %[[UD]][0] : !llvm.struct<(struct<(f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32)>, struct<(f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32)>)> 
-// CHECK: %[[S41:.+]] = llvm.insertvalue %[[S38]], %[[S40]][1] : !llvm.struct<(struct<(f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32)>, struct<(f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32)>)> 
+// CHECK: %[[S40:.+]] = llvm.insertvalue %[[S19]], %[[UD]][0] : !llvm.struct<(struct<(f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32)>, struct<(f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32)>)>
+// CHECK: %[[S41:.+]] = llvm.insertvalue %[[S38]], %[[S40]][1] : !llvm.struct<(struct<(f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32)>, struct<(f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32)>)>
 // CHECK: nvvm.wgmma.commit.group.sync.aligned
-// CHECK: nvvm.wgmma.wait.group.sync.aligned 1  
-  %wgmmaResult = nvgpu.warpgroup.mma %descA, %descB, %acc {transposeB}: 
-      !nvgpu.warpgroup.descriptor<tensor = memref<128x64xf16, 3>>, 
-      !nvgpu.warpgroup.descriptor<tensor = memref<64x128xf16, 3>>, 
-      !nvgpu.warpgroup.accumulator<fragmented = vector<128x128xf32>> 
-      -> 
-      !nvgpu.warpgroup.accumulator<fragmented = vector<128x128xf32>>  
+// CHECK: nvvm.wgmma.wait.group.sync.aligned 1
+  %wgmmaResult = nvgpu.warpgroup.mma %descA, %descB, %acc {transposeB}:
+      !nvgpu.warpgroup.descriptor<tensor = memref<128x64xf16, 3>>,
+      !nvgpu.warpgroup.descriptor<tensor = memref<64x128xf16, 3>>,
+      !nvgpu.warpgroup.accumulator<fragmented = vector<128x128xf32>>
+      ->
+      !nvgpu.warpgroup.accumulator<fragmented = vector<128x128xf32>>
   return
 }
 
-// CHECK-LABEL: @warpgroup_mma_store(  
+// CHECK-LABEL: @warpgroup_mma_store(
 // CHECK-SAME: %[[arg0:[a-zA-Z0-9_]+]]: !nvgpu.warpgroup.accumulator<fragmented = vector<128x128xf32>>, %[[arg2:[a-zA-Z0-9_]+]]: memref<128x128xf32, 3>)
 func.func @warpgroup_mma_store(
-    %result : !nvgpu.warpgroup.accumulator<fragmented = vector<128x128xf32>>, 
+    %result : !nvgpu.warpgroup.accumulator<fragmented = vector<128x128xf32>>,
     %matrixD: memref<128x128xf32,3>) {
 // CHECK: %[[S0:.+]] = builtin.unrealized_conversion_cast %[[arg0]] : !nvgpu.warpgroup.accumulator<fragmented = vector<128x128xf32>> to !llvm.struct<(struct<(f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32)>, struct<(f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32)>)>
-// CHECK: %[[EX1:.+]] = llvm.extractvalue %[[S0]][0] : !llvm.struct<(struct<(f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32)>, struct<(f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32)>)> 
+// CHECK: %[[EX1:.+]] = llvm.extractvalue %[[S0]][0] : !llvm.struct<(struct<(f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32)>, struct<(f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32)>)>
 // CHECK: %[[S6:.+]] = llvm.mlir.constant(1 : i32) : i32
 // CHECK: %[[S5:.+]] = llvm.mlir.constant(2 : i32) : i32
 // CHECK: %[[S2:.+]] = llvm.mlir.constant(4 : i32) : i32
@@ -982,7 +982,7 @@ func.func @warpgroup_mma_store(
 // CHECK: memref.store %[[S35]], %[[arg2]][%[[S31]], %[[S32]]] : memref<128x128xf32, 3>
 // CHECK: memref.store %[[S36]], %[[arg2]][%[[S31]], %[[S34]]] : memref<128x128xf32, 3>
 
-// ### Store {d4, d5} of each thread ### 
+// ### Store {d4, d5} of each thread ###
 
 // CHECK: %[[S37:.+]] = llvm.mlir.constant(2 : i32) : i32
 // CHECK: %[[S38:.+]] = llvm.mul %[[S37]], %[[S4]]  : i32
@@ -996,7 +996,7 @@ func.func @warpgroup_mma_store(
 // CHECK: memref.store %[[S44]], %[[arg2]][%[[S40]], %[[S41]]] : memref<128x128xf32, 3>
 // CHECK: memref.store %[[S45]], %[[arg2]][%[[S40]], %[[S43]]] : memref<128x128xf32, 3>
 
-// ### Store {d6, d7} of each thread ### 
+// ### Store {d6, d7} of each thread ###
 
 // CHECK: %[[S46:.+]] = llvm.mlir.constant(3 : i32) : i32
 // CHECK: %[[S47:.+]] = llvm.mul %[[S46]], %[[S4]]  : i32
@@ -1015,8 +1015,8 @@ func.func @warpgroup_mma_store(
 // CHECK: %[[c1:.+]] = llvm.mlir.constant(1 : i32) : i32
 // CHECK: %[[c2:.+]] = llvm.mlir.constant(2 : i32) : i32
 
-// ### Store {d64, d65} of each thread ### 
-// CHECK: %[[EX2:.+]] = llvm.extractvalue %[[S0]][1] : !llvm.struct<(struct<(f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32)>, struct<(f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32)>)> 
+// ### Store {d64, d65} of each thread ###
+// CHECK: %[[EX2:.+]] = llvm.extractvalue %[[S0]][1] : !llvm.struct<(struct<(f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32)>, struct<(f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32)>)>
 // CHECK: %[[S315:.+]] = llvm.mlir.constant(1 : i32) : i32
 // CHECK: %[[S312:.+]] = llvm.mlir.constant(2 : i32) : i32
 // CHECK: %[[S311:.+]] = llvm.mlir.constant(4 : i32) : i32
@@ -1043,83 +1043,83 @@ func.func @warpgroup_mma_store(
 // CHECK: %[[S334:.+]] = arith.index_cast %[[S332]] : i32 to index
 // CHECK: %[[S335:.+]] = llvm.add %[[S332]], %[[S315]]  : i32
 // CHECK: %[[S336:.+]] = arith.index_cast %[[S335]] : i32 to index
-// CHECK: %[[S337:.+]] = llvm.extractvalue %[[EX2]][0] 
-// CHECK: %[[S338:.+]] = llvm.extractvalue %[[EX2]][1]  
+// CHECK: %[[S337:.+]] = llvm.extractvalue %[[EX2]][0]
+// CHECK: %[[S338:.+]] = llvm.extractvalue %[[EX2]][1]
 // CHECK: memref.store %[[S337]], %[[arg2]][%[[S333]], %[[S334]]] : memref<128x128xf32, 3>
 // CHECK: memref.store %[[S338]], %[[arg2]][%[[S333]], %[[S336]]] : memref<128x128xf32, 3>
 
 // Pattern continues similarly 31x times until {... d126, d127}
 
-  nvgpu.warpgroup.mma.store %result, %matrixD : 
-    !nvgpu.warpgroup.accumulator< fragmented = vector<128x128xf32>> 
+  nvgpu.warpgroup.mma.store %result, %matrixD :
+    !nvgpu.warpgroup.accumulator< fragmented = vector<128x128xf32>>
     to memref<128x128xf32,3>
-  return 
+  return
 }
 
 // CHECK-LABEL: @warpgroup_mma_store_multiple
 func.func @warpgroup_mma_store_multiple(
-    %shmem_m64n8k : memref<64x8xf32>, 
-    %shmem_m64n16k : memref<64x16xf32>, 
-    %shmem_m64n24k : memref<64x24xf32>, 
-    %shmem_m64n32k : memref<64x32xf32>, 
-    %shmem_m64n40k : memref<64x40xf32>, 
-    %shmem_m64n48k : memref<64x48xf32>, 
-    %shmem_m64n56k : memref<64x56xf32>, 
-    %shmem_m64n64k : memref<64x64xf32>, 
-    %shmem_m64n72k : memref<64x72xf32>, 
-    %shmem_m64n80k : memref<64x80xf32>, 
-    %shmem_m64n88k : memref<64x88xf32>, 
-    %shmem_m64n96k : memref<64x96xf32>, 
-    %shmem_m64n104k : memref<64x104xf32>, 
-    %shmem_m64n112k : memref<64x112xf32>, 
-    %shmem_m64n120k : memref<64x120xf32>, 
-    %shmem_m64n128k : memref<64x128xf32>, 
-    %shmem_m64n136k : memref<64x136xf32>, 
-    %shmem_m64n144k : memref<64x144xf32>, 
-    %shmem_m64n152k : memref<64x152xf32>, 
-    %shmem_m64n160k : memref<64x160xf32>, 
-    %shmem_m64n168k : memref<64x168xf32>, 
-    %shmem_m64n176k : memref<64x176xf32>, 
-    %shmem_m64n184k : memref<64x184xf32>, 
-    %shmem_m64n192k : memref<64x192xf32>, 
-    %shmem_m64n200k : memref<64x200xf32>, 
-    %shmem_m64n208k : memref<64x208xf32>, 
-    %shmem_m64n216k : memref<64x216xf32>, 
-    %shmem_m64n224k : memref<64x224xf32>, 
-    %shmem_m64n232k : memref<64x232xf32>, 
-    %shmem_m64n240k : memref<64x240xf32>, 
-    %shmem_m64n248k : memref<64x248xf32>, 
-    %shmem_m64n256k : memref<64x256xf32>, 
-    %res_m64n16k : !nvgpu.warpgroup.accumulator<fragmented = vector<64x16xf32>>, 
-    %res_m64n24k : !nvgpu.warpgroup.accumulator<fragmented = vector<64x24xf32>>, 
-    %res_m64n32k : !nvgpu.warpgroup.accumulator<fragmented = vector<64x32xf32>>, 
-    %res_m64n40k : !nvgpu.warpgroup.accumulator<fragmented = vector<64x40xf32>>, 
-    %res_m64n48k : !nvgpu.warpgroup.accumulator<fragmented = vector<64x48xf32>>, 
-    %res_m64n56k : !nvgpu.warpgroup.accumulator<fragmented = vector<64x56xf32>>, 
-    %res_m64n64k : !nvgpu.warpgroup.accumulator<fragmented = vector<64x64xf32>>, 
-    %res_m64n72k : !nvgpu.warpgroup.accumulator<fragmented = vector<64x72xf32>>, 
-    %res_m64n80k : !nvgpu.warpgroup.accumulator<fragmented = vector<64x80xf32>>, 
-    %res_m64n88k : !nvgpu.warpgroup.accumulator<fragmented = vector<64x88xf32>>, 
-    %res_m64n96k : !nvgpu.warpgroup.accumulator<fragmented = vector<64x96xf32>>, 
-    %res_m64n104k : !nvgpu.warpgroup.accumulator<fragmented = vector<64x104xf32>>, 
-    %res_m64n112k : !nvgpu.warpgroup.accumulator<fragmented = vector<64x112xf32>>, 
-    %res_m64n120k : !nvgpu.warpgroup.accumulator<fragmented = vector<64x120xf32>>, 
-    %res_m64n128k : !nvgpu.warpgroup.accumulator<fragmented = vector<64x128xf32>>, 
-    %res_m64n136k : !nvgpu.warpgroup.accumulator<fragmented = vector<64x136xf32>>, 
-    %res_m64n144k : !nvgpu.warpgroup.accumulator<fragmented = vector<64x144xf32>>, 
-    %res_m64n152k : !nvgpu.warpgroup.accumulator<fragmented = vector<64x152xf32>>, 
-    %res_m64n160k : !nvgpu.warpgroup.accumulator<fragmented = vector<64x160xf32>>, 
-    %res_m64n168k : !nvgpu.warpgroup.accumulator<fragmented = vector<64x168xf32>>, 
-    %res_m64n176k : !nvgpu.warpgroup.accumulator<fragmented = vector<64x176xf32>>, 
-    %res_m64n184k : !nvgpu.warpgroup.accumulator<fragmented = vector<64x184xf32>>, 
-    %res_m64n192k : !nvgpu.warpgroup.accumulator<fragmented = vector<64x192xf32>>, 
-    %res_m64n200k : !nvgpu.warpgroup.accumulator<fragmented = vector<64x200xf32>>, 
-    %res_m64n208k : !nvgpu.warpgroup.accumulator<fragmented = vector<64x208xf32>>, 
-    %res_m64n216k : !nvgpu.warpgroup.accumulator<fragmented = vector<64x216xf32>>, 
-    %res_m64n224k : !nvgpu.warpgroup.accumulator<fragmented = vector<64x224xf32>>, 
-    %res_m64n232k : !nvgpu.warpgroup.accumulator<fragmented = vector<64x232xf32>>, 
-    %res_m64n240k : !nvgpu.warpgroup.accumulator<fragmented = vector<64x240xf32>>, 
-    %res_m64n248k : !nvgpu.warpgroup.accumulator<fragmented = vector<64x248xf32>>, 
+    %shmem_m64n8k : memref<64x8xf32>,
+    %shmem_m64n16k : memref<64x16xf32>,
+    %shmem_m64n24k : memref<64x24xf32>,
+    %shmem_m64n32k : memref<64x32xf32>,
+    %shmem_m64n40k : memref<64x40xf32>,
+    %shmem_m64n48k : memref<64x48xf32>,
+    %shmem_m64n56k : memref<64x56xf32>,
+    %shmem_m64n64k : memref<64x64xf32>,
+    %shmem_m64n72k : memref<64x72xf32>,
+    %shmem_m64n80k : memref<64x80xf32>,
+    %shmem_m64n88k : memref<64x88xf32>,
+    %shmem_m64n96k : memref<64x96xf32>,
+    %shmem_m64n104k : memref<64x104xf32>,
+    %shmem_m64n112k : memref<64x112xf32>,
+    %shmem_m64n120k : memref<64x120xf32>,
+    %shmem_m64n128k : memref<64x128xf32>,
+    %shmem_m64n136k : memref<64x136xf32>,
+    %shmem_m64n144k : memref<64x144xf32>,
+    %shmem_m64n152k : memref<64x152xf32>,
+    %shmem_m64n160k : memref<64x160xf32>,
+    %shmem_m64n168k : memref<64x168xf32>,
+    %shmem_m64n176k : memref<64x176xf32>,
+    %shmem_m64n184k : memref<64x184xf32>,
+    %shmem_m64n192k : memref<64x192xf32>,
+    %shmem_m64n200k : memref<64x200xf32>,
+    %shmem_m64n208k : memref<64x208xf32>,
+    %shmem_m64n216k : memref<64x216xf32>,
+    %shmem_m64n224k : memref<64x224xf32>,
+    %shmem_m64n232k : memref<64x232xf32>,
+    %shmem_m64n240k : memref<64x240xf32>,
+    %shmem_m64n248k : memref<64x248xf32>,
+    %shmem_m64n256k : memref<64x256xf32>,
+    %res_m64n16k : !nvgpu.warpgroup.accumulator<fragmented = vector<64x16xf32>>,
+    %res_m64n24k : !nvgpu.warpgroup.accumulator<fragmented = vector<64x24xf32>>,
+    %res_m64n32k : !nvgpu.warpgroup.accumulator<fragmented = vector<64x32xf32>>,
+    %res_m64n40k : !nvgpu.warpgroup.accumulator<fragmented = vector<64x40xf32>>,
+    %res_m64n48k : !nvgpu.warpgroup.accumulator<fragmented = vector<64x48xf32>>,
+    %res_m64n56k : !nvgpu.warpgroup.accumulator<fragmented = vector<64x56xf32>>,
+    %res_m64n64k : !nvgpu.warpgroup.accumulator<fragmented = vector<64x64xf32>>,
+    %res_m64n72k : !nvgpu.warpgroup.accumulator<fragmented = vector<64x72xf32>>,
+    %res_m64n80k : !nvgpu.warpgroup.accumulator<fragmented = vector<64x80xf32>>,
+    %res_m64n88k : !nvgpu.warpgroup.accumulator<fragmented = vector<64x88xf32>>,
+    %res_m64n96k : !nvgpu.warpgroup.accumulator<fragmented = vector<64x96xf32>>,
+    %res_m64n104k : !nvgpu.warpgroup.accumulator<fragmented = vector<64x104xf32>>,
+    %res_m64n112k : !nvgpu.warpgroup.accumulator<fragmented = vector<64x112xf32>>,
+    %res_m64n120k : !nvgpu.warpgroup.accumulator<fragmented = vector<64x120xf32>>,
+    %res_m64n128k : !nvgpu.warpgroup.accumulator<fragmented = vector<64x128xf32>>,
+    %res_m64n136k : !nvgpu.warpgroup.accumulator<fragmented = vector<64x136xf32>>,
+    %res_m64n144k : !nvgpu.warpgroup.accumulator<fragmented = vector<64x144xf32>>,
+    %res_m64n152k : !nvgpu.warpgroup.accumulator<fragmented = vector<64x152xf32>>,
+    %res_m64n160k : !nvgpu.warpgroup.accumulator<fragmented = vector<64x160xf32>>,
+    %res_m64n168k : !nvgpu.warpgroup.accumulator<fragmented = vector<64x168xf32>>,
+    %res_m64n176k : !nvgpu.warpgroup.accumulator<fragmented = vector<64x176xf32>>,
+    %res_m64n184k : !nvgpu.warpgroup.accumulator<fragmented = vector<64x184xf32>>,
+    %res_m64n192k : !nvgpu.warpgroup.accumulator<fragmented = vector<64x192xf32>>,
+    %res_m64n200k : !nvgpu.warpgroup.accumulator<fragmented = vector<64x200xf32>>,
+    %res_m64n208k : !nvgpu.warpgroup.accumulator<fragmented = vector<64x208xf32>>,
+    %res_m64n216k : !nvgpu.warpgroup.accumulator<fragmented = vector<64x216xf32>>,
+    %res_m64n224k : !nvgpu.warpgroup.accumulator<fragmented = vector<64x224xf32>>,
+    %res_m64n232k : !nvgpu.warpgroup.accumulator<fragmented = vector<64x232xf32>>,
+    %res_m64n240k : !nvgpu.warpgroup.accumulator<fragmented = vector<64x240xf32>>,
+    %res_m64n248k : !nvgpu.warpgroup.accumulator<fragmented = vector<64x248xf32>>,
     %res_m64n256k : !nvgpu.warpgroup.accumulator<fragmented = vector<64x256xf32>>) {
     // CHECK-COUNT-8: memref.store %{{.*}}, %{{.*}}[%{{.*}}, %{{.*}}] : memref<64x16xf32>
     // CHECK-COUNT-12: memref.store %{{.*}}, %{{.*}}[%{{.*}}, %{{.*}}] : memref<64x24xf32>
@@ -1183,13 +1183,13 @@ func.func @warpgroup_mma_store_multiple(
     nvgpu.warpgroup.mma.store  %res_m64n240k, %shmem_m64n240k : !nvgpu.warpgroup.accumulator<fragmented = vector<64x240xf32>> to memref<64x240xf32>
     nvgpu.warpgroup.mma.store  %res_m64n248k, %shmem_m64n248k : !nvgpu.warpgroup.accumulator<fragmented = vector<64x248xf32>> to memref<64x248xf32>
     nvgpu.warpgroup.mma.store  %res_m64n256k, %shmem_m64n256k : !nvgpu.warpgroup.accumulator<fragmented = vector<64x256xf32>> to memref<64x256xf32>
-  return 
+  return
 }
 
 func.func @warpgroup_mma_init() {
   //CHECK: %[[S1:.+]] = llvm.mlir.constant(0.000000e+00 : f32) : f3
-  //CHECK: %[[S0:.+]] = llvm.mlir.undef : !llvm.struct<(struct<(f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32)>, struct<(f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32)>)>
-  //CHECK: %[[EX:.+]] = llvm.extractvalue %[[S0]][0] : !llvm.struct<(struct<(f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32)>, struct<(f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32)>)> 
+  //CHECK: %[[S0:.+]] = llvm.mlir.poison : !llvm.struct<(struct<(f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32)>, struct<(f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32)>)>
+  //CHECK: %[[EX:.+]] = llvm.extractvalue %[[S0]][0] : !llvm.struct<(struct<(f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32)>, struct<(f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32)>)>
   //CHECK: %[[S2:.+]] = llvm.insertvalue %[[S1]], %[[EX]][0] : !llvm.struct
   //CHECK: %[[S3:.+]] = llvm.insertvalue %[[S1]], %[[S2]][1] : !llvm.struct
   //CHECK: %[[S4:.+]] = llvm.insertvalue %[[S1]], %[[S3]][2] : !llvm.struct
@@ -1255,50 +1255,50 @@ func.func @warpgroup_mma_init() {
   //CHECK: %[[S64:.+]] = llvm.insertvalue %[[S1]], %[[S63]][62] : !llvm.struct
   //CHECK: %[[S65:.+]] = llvm.insertvalue %[[S1]], %[[S64]][63] : !llvm.struct
   %matrixC = nvgpu.warpgroup.mma.init.accumulator -> !nvgpu.warpgroup.accumulator< fragmented = vector<128x128xf32>>
-  return 
+  return
 }
 
-// CHECK-LABEL: @warpgroup_matrix_multiply_m128n128k64(  
+// CHECK-LABEL: @warpgroup_matrix_multiply_m128n128k64(
 // CHECK-SAME: %[[arg0:[a-zA-Z0-9_]+]]: !nvgpu.warpgroup.descriptor<tensor = memref<128x64xf16, 3>>, %[[arg1:[a-zA-Z0-9_]+]]: !nvgpu.warpgroup.descriptor<tensor = memref<64x128xf16, 3>>, %[[arg2:[a-zA-Z0-9_]+]]: memref<128x128xf32, 3>)
 func.func @warpgroup_matrix_multiply_m128n128k64(
-      %descA: !nvgpu.warpgroup.descriptor<tensor = memref<128x64xf16, 3>>, 
+      %descA: !nvgpu.warpgroup.descriptor<tensor = memref<128x64xf16, 3>>,
       %descB: !nvgpu.warpgroup.descriptor<tensor = memref<64x128xf16, 3>>,
-      %shmemD: memref<128x128xf32, 3>) 
+      %shmemD: memref<128x128xf32, 3>)
 {
   // Init
-  %matrixC = nvgpu.warpgroup.mma.init.accumulator ->  
+  %matrixC = nvgpu.warpgroup.mma.init.accumulator ->
                       !nvgpu.warpgroup.accumulator<fragmented = vector<128x128xf32>>
 
   // GEMM
-  %matrixD = nvgpu.warpgroup.mma %descA, %descB, %matrixC {transposeB}: 
-        !nvgpu.warpgroup.descriptor<tensor = memref<128x64xf16, 3>>, 
-        !nvgpu.warpgroup.descriptor<tensor = memref<64x128xf16, 3>>, 
-        !nvgpu.warpgroup.accumulator<fragmented = vector<128x128xf32>> 
-        -> 
-        !nvgpu.warpgroup.accumulator<fragmented = vector<128x128xf32>>  
+  %matrixD = nvgpu.warpgroup.mma %descA, %descB, %matrixC {transposeB}:
+        !nvgpu.warpgroup.descriptor<tensor = memref<128x64xf16, 3>>,
+        !nvgpu.warpgroup.descriptor<tensor = memref<64x128xf16, 3>>,
+        !nvgpu.warpgroup.accumulator<fragmented = vector<128x128xf32>>
+        ->
+        !nvgpu.warpgroup.accumulator<fragmented = vector<128x128xf32>>
 
 
-  // Epilogue 
-  nvgpu.warpgroup.mma.store %matrixD, %shmemD : 
-    !nvgpu.warpgroup.accumulator< fragmented = vector<128x128xf32>> 
+  // Epilogue
+  nvgpu.warpgroup.mma.store %matrixD, %shmemD :
+    !nvgpu.warpgroup.accumulator< fragmented = vector<128x128xf32>>
     to memref<128x128xf32,3>
 
 
 // CHECK-DAG: %[[S0:.+]] = builtin.unrealized_conversion_cast %[[arg0]] : !nvgpu.warpgroup.descriptor<tensor = memref<128x64xf16, 3>> to i64
 // CHECK-DAG: %[[S1:.+]] = builtin.unrealized_conversion_cast %[[arg1]] : !nvgpu.warpgroup.descriptor<tensor = memref<64x128xf16, 3>> to i64
 // CHECK: %[[S3:.+]] = llvm.mlir.constant(0.000000e+00 : f32) : f32
-// CHECK: %[[S4:.+]] = llvm.mlir.undef : !llvm.struct<(struct<(f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32)>, struct<(f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32)>)>
-// CHECK: %[[S5:.+]] = llvm.extractvalue %[[S4]][0] : !llvm.struct<(struct<(f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32)>, struct<(f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32)>)> 
-// CHECK: %[[S6:.+]] = llvm.insertvalue %[[S3]], %[[S5]][0] : !llvm.struct<(f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32)> 
-// CHECK: %[[S68:.+]] = llvm.insertvalue %[[S3]], %{{.*}}[63] : !llvm.struct<(f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32)> 
-// CHECK: %[[S69:.+]] = llvm.extractvalue %[[S4]][1] : !llvm.struct<(struct<(f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32)>, struct<(f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32)>)> 
-// CHECK: %[[S71:.+]] = llvm.insertvalue %[[S3]], %[[S69]][0] : !llvm.struct<(f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32)> 
-// CHECK: %[[S134:.+]] = llvm.insertvalue %[[S3]], %{{.*}}[63] : !llvm.struct<(f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32)> 
-// CHECK: %[[S135:.+]] = llvm.insertvalue %[[S68]], %[[S4]][0] : !llvm.struct<(struct<(f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32)>, struct<(f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32)>)> 
-// CHECK: %[[S136:.+]] = llvm.insertvalue %[[S134]], %[[S135]][1] : !llvm.struct<(struct<(f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32)>, struct<(f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32)>)> 
+// CHECK: %[[S4:.+]] = llvm.mlir.poison : !llvm.struct<(struct<(f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32)>, struct<(f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32)>)>
+// CHECK: %[[S5:.+]] = llvm.extractvalue %[[S4]][0] : !llvm.struct<(struct<(f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32)>, struct<(f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32)>)>
+// CHECK: %[[S6:.+]] = llvm.insertvalue %[[S3]], %[[S5]][0] : !llvm.struct<(f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32)>
+// CHECK: %[[S68:.+]] = llvm.insertvalue %[[S3]], %{{.*}}[63] : !llvm.struct<(f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32)>
+// CHECK: %[[S69:.+]] = llvm.extractvalue %[[S4]][1] : !llvm.struct<(struct<(f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32)>, struct<(f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32)>)>
+// CHECK: %[[S71:.+]] = llvm.insertvalue %[[S3]], %[[S69]][0] : !llvm.struct<(f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32)>
+// CHECK: %[[S134:.+]] = llvm.insertvalue %[[S3]], %{{.*}}[63] : !llvm.struct<(f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32)>
+// CHECK: %[[S135:.+]] = llvm.insertvalue %[[S68]], %[[S4]][0] : !llvm.struct<(struct<(f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32)>, struct<(f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32)>)>
+// CHECK: %[[S136:.+]] = llvm.insertvalue %[[S134]], %[[S135]][1] : !llvm.struct<(struct<(f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32)>, struct<(f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32)>)>
 // CHECK: nvvm.wgmma.fence.aligned
-// CHECK: %[[S137:.+]] = llvm.mlir.undef : !llvm.struct<(struct<(f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32)>, struct<(f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32)>)>
-// CHECK: %[[S138:.+]] = llvm.extractvalue %{{.*}}[0] : !llvm.struct<(struct<(f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32)>, struct<(f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32)>)> 
+// CHECK: %[[S137:.+]] = llvm.mlir.poison : !llvm.struct<(struct<(f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32)>, struct<(f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32)>)>
+// CHECK: %[[S138:.+]] = llvm.extractvalue %{{.*}}[0] : !llvm.struct<(struct<(f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32)>, struct<(f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32)>)>
 // CHECK: %[[S139:.+]] = nvvm.wgmma.mma_async %[[S0]], %[[S1]], %[[S138]], <m = 64, n = 128, k = 16>, D[<f32>, <one>, <wrapped>], A[<f16>, <one>, <row>], B[<f16>, <one>, <row>] : !llvm.struct<(f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32)> -> !llvm.struct<(f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32)>
 // CHECK: nvvm.wgmma.mma_async
 // CHECK: nvvm.wgmma.mma_async
@@ -1307,27 +1307,27 @@ func.func @warpgroup_matrix_multiply_m128n128k64(
 // CHECK: nvvm.wgmma.mma_async
 // CHECK: nvvm.wgmma.mma_async
 // CHECK: %[[S173:.+]] = nvvm.wgmma.mma_async
-// CHECK: %[[S174:.+]] = llvm.insertvalue %[[S154]], %[[S137]][0] : !llvm.struct<(struct<(f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32)>, struct<(f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32)>)> 
-// CHECK: %[[S175:.+]] = llvm.insertvalue %[[S173]], %[[S174]][1] : !llvm.struct<(struct<(f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32)>, struct<(f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32)>)> 
+// CHECK: %[[S174:.+]] = llvm.insertvalue %[[S154]], %[[S137]][0] : !llvm.struct<(struct<(f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32)>, struct<(f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32)>)>
+// CHECK: %[[S175:.+]] = llvm.insertvalue %[[S173]], %[[S174]][1] : !llvm.struct<(struct<(f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32)>, struct<(f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32)>)>
 // CHECK: nvvm.wgmma.commit.group.sync.aligned
 // CHECK: nvvm.wgmma.wait.group.sync.aligned 1
-// CHECK: %[[S176:.+]] = llvm.extractvalue %[[S175]][0] : !llvm.struct<(struct<(f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32)>, struct<(f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32)>)> 
-// CHECK: %[[S193:.+]] = llvm.extractvalue %[[S176]][0] : !llvm.struct<(f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32)> 
-// CHECK: %[[S194:.+]] = llvm.extractvalue %[[S176]][1] : !llvm.struct<(f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32)> 
+// CHECK: %[[S176:.+]] = llvm.extractvalue %[[S175]][0] : !llvm.struct<(struct<(f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32)>, struct<(f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32)>)>
+// CHECK: %[[S193:.+]] = llvm.extractvalue %[[S176]][0] : !llvm.struct<(f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32)>
+// CHECK: %[[S194:.+]] = llvm.extractvalue %[[S176]][1] : !llvm.struct<(f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32)>
 // CHECK: memref.store %[[S193]], %[[arg2]][%{{.*}}, %{{.*}}] : memref<128x128xf32, 3>
 // CHECK: memref.store %[[S194]], %[[arg2]][%{{.*}}, %{{.*}}] : memref<128x128xf32, 3>
-// CHECK: %[[S485:.+]] = llvm.extractvalue %[[S175]][1] : !llvm.struct<(struct<(f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32)>, struct<(f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32)>)> 
-// CHECK: %[[S503:.+]] = llvm.extractvalue %[[S485]][0] : !llvm.struct<(f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32)> 
-// CHECK: %[[S504:.+]] = llvm.extractvalue %[[S485]][1] : !llvm.struct<(f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32)> 
+// CHECK: %[[S485:.+]] = llvm.extractvalue %[[S175]][1] : !llvm.struct<(struct<(f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32)>, struct<(f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32)>)>
+// CHECK: %[[S503:.+]] = llvm.extractvalue %[[S485]][0] : !llvm.struct<(f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32)>
+// CHECK: %[[S504:.+]] = llvm.extractvalue %[[S485]][1] : !llvm.struct<(f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32)>
 // CHECK: memref.store %[[S503]], %[[arg2]][%{{.*}}, %{{.*}}] : memref<128x128xf32, 3>
 // CHECK: memref.store %[[S504]], %[[arg2]][%{{.*}}, %{{.*}}] : memref<128x128xf32, 3>
-  return 
+  return
 }
 
 
 module attributes {transform.with_named_sequence} {
   transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
-    %0 = transform.structured.match ops{["func.func"]} in %arg1 
+    %0 = transform.structured.match ops{["func.func"]} in %arg1
       : (!transform.any_op) -> !transform.any_op
     transform.apply_conversion_patterns to %0 {
       transform.apply_conversion_patterns.nvgpu.nvgpu_to_nvvm
@@ -1344,7 +1344,7 @@ module attributes {transform.with_named_sequence} {
 func.func @rcp_approx_ftz_f32(%in: vector<32x16xf32>) {
   // CHECK: %[[IN_LLVM:.*]] = builtin.unrealized_conversion_cast %[[IN]] : vector<32x16xf32> to !llvm.array<32 x vector<16xf32>>
   // CHECK: %[[IN1DVEC:.*]] = llvm.extractvalue %[[IN_LLVM]][0] : !llvm.array<32 x vector<16xf32>>
-  // CHECK: %[[OUT1DVEC:.*]] = llvm.mlir.undef : vector<16xf32>
+  // CHECK: %[[OUT1DVEC:.*]] = llvm.mlir.poison : vector<16xf32>
   // CHECK: %[[IDX_0:.+]] = llvm.mlir.constant(0 : i64) : i64
   // CHECK: %[[ELEM_0:.*]] = llvm.extractelement %[[IN1DVEC]][%[[IDX_0]] : i64]
   // CHECK: %[[ELEM_RCP0:.*]] = nvvm.rcp.approx.ftz.f %[[ELEM_0]] : f32
diff --git a/mlir/test/Conversion/OpenMPToLLVM/convert-to-llvmir.mlir b/mlir/test/Conversion/OpenMPToLLVM/convert-to-llvmir.mlir
index 4f37dd16b44dd..6f1ed73e778b4 100644
--- a/mlir/test/Conversion/OpenMPToLLVM/convert-to-llvmir.mlir
+++ b/mlir/test/Conversion/OpenMPToLLVM/convert-to-llvmir.mlir
@@ -507,28 +507,22 @@ llvm.func @_QPtarget_map_with_bounds(%arg0: !llvm.ptr, %arg1: !llvm.ptr, %arg2:
 
 // -----
 
-// CHECK: omp.private {type = private} @x.privatizer : !llvm.struct<{{.*}}> alloc {
-omp.private {type = private} @x.privatizer : memref<?xf32> alloc {
-// CHECK: ^bb0(%arg0: !llvm.struct<{{.*}}>):
-^bb0(%arg0: memref<?xf32>):
+// CHECK: omp.private {type = private} @x.privatizer : !llvm.struct<{{.*}}> init {
+omp.private {type = private} @x.privatizer : memref<?xf32> init {
+// CHECK: ^bb0(%arg0: !llvm.struct<{{.*}}>, %arg1: !llvm.struct<{{.*}}>):
+^bb0(%arg0: memref<?xf32>, %arg1: memref<?xf32>):
   // CHECK: omp.yield(%arg0 : !llvm.struct<{{.*}}>)
   omp.yield(%arg0 : memref<?xf32>)
 }
 
 // -----
 
-// CHECK: omp.private {type = firstprivate} @y.privatizer : i64 alloc {
-omp.private {type = firstprivate} @y.privatizer : index alloc {
-// CHECK: ^bb0(%arg0: i64):
-^bb0(%arg0: index):
-  // CHECK: omp.yield(%arg0 : i64)
-  omp.yield(%arg0 : index)
-// CHECK: } copy {
-} copy {
-// CHECK: ^bb0(%arg0: i64, %arg1: i64):
-^bb0(%arg0: index, %arg1: index):
-  // CHECK: omp.yield(%arg0 : i64)
-  omp.yield(%arg0 : index)
+// CHECK: omp.private {type = firstprivate} @y.privatizer : i64 copy {
+omp.private {type = firstprivate} @y.privatizer : index copy {
+// CHECK: ^bb0(%arg0: !llvm.ptr, %arg1: !llvm.ptr):
+^bb0(%arg0: !llvm.ptr, %arg1: !llvm.ptr):
+  // CHECK: omp.yield(%arg0 : !llvm.ptr)
+  omp.yield(%arg0 : !llvm.ptr)
 }
 
 // -----
diff --git a/mlir/test/Conversion/SPIRVToLLVM/bitwise-ops-to-llvm.mlir b/mlir/test/Conversion/SPIRVToLLVM/bitwise-ops-to-llvm.mlir
index a0afe0dafcaa2..0f25c84673ceb 100644
--- a/mlir/test/Conversion/SPIRVToLLVM/bitwise-ops-to-llvm.mlir
+++ b/mlir/test/Conversion/SPIRVToLLVM/bitwise-ops-to-llvm.mlir
@@ -92,12 +92,12 @@ spirv.func @bitfield_insert_scalar_greater_bit_width(%base: i16, %insert: i16, %
 // CHECK-LABEL: @bitfield_insert_vector
 //  CHECK-SAME: %[[BASE:.*]]: vector<2xi32>, %[[INSERT:.*]]: vector<2xi32>, %[[OFFSET:.*]]: i32, %[[COUNT:.*]]: i32
 spirv.func @bitfield_insert_vector(%base: vector<2xi32>, %insert: vector<2xi32>, %offset: i32, %count: i32) "None" {
-  // CHECK: %[[OFFSET_V0:.*]] = llvm.mlir.undef : vector<2xi32>
+  // CHECK: %[[OFFSET_V0:.*]] = llvm.mlir.poison : vector<2xi32>
   // CHECK: %[[ZERO:.*]] = llvm.mlir.constant(0 : i32) : i32
   // CHECK: %[[OFFSET_V1:.*]] = llvm.insertelement %[[OFFSET]], %[[OFFSET_V0]][%[[ZERO]] : i32] : vector<2xi32>
   // CHECK: %[[ONE:.*]] = llvm.mlir.constant(1 : i32) : i32
   // CHECK: %[[OFFSET_V2:.*]] = llvm.insertelement  %[[OFFSET]], %[[OFFSET_V1]][%[[ONE]] : i32] : vector<2xi32>
-  // CHECK: %[[COUNT_V0:.*]] = llvm.mlir.undef : vector<2xi32>
+  // CHECK: %[[COUNT_V0:.*]] = llvm.mlir.poison : vector<2xi32>
   // CHECK: %[[ZERO:.*]] = llvm.mlir.constant(0 : i32) : i32
   // CHECK: %[[COUNT_V1:.*]] = llvm.insertelement %[[COUNT]], %[[COUNT_V0]][%[[ZERO]] : i32] : vector<2xi32>
   // CHECK: %[[ONE:.*]] = llvm.mlir.constant(1 : i32) : i32
@@ -164,12 +164,12 @@ spirv.func @bitfield_sextract_scalar_greater_bit_width(%base: i32, %offset: i64,
 // CHECK-LABEL: @bitfield_sextract_vector
 //  CHECK-SAME: %[[BASE:.*]]: vector<2xi32>, %[[OFFSET:.*]]: i32, %[[COUNT:.*]]: i32
 spirv.func @bitfield_sextract_vector(%base: vector<2xi32>, %offset: i32, %count: i32) "None" {
-  // CHECK: %[[OFFSET_V0:.*]] = llvm.mlir.undef : vector<2xi32>
+  // CHECK: %[[OFFSET_V0:.*]] = llvm.mlir.poison : vector<2xi32>
   // CHECK: %[[ZERO:.*]] = llvm.mlir.constant(0 : i32) : i32
   // CHECK: %[[OFFSET_V1:.*]] = llvm.insertelement %[[OFFSET]], %[[OFFSET_V0]][%[[ZERO]] : i32] : vector<2xi32>
   // CHECK: %[[ONE:.*]] = llvm.mlir.constant(1 : i32) : i32
   // CHECK: %[[OFFSET_V2:.*]] = llvm.insertelement  %[[OFFSET]], %[[OFFSET_V1]][%[[ONE]] : i32] : vector<2xi32>
-  // CHECK: %[[COUNT_V0:.*]] = llvm.mlir.undef : vector<2xi32>
+  // CHECK: %[[COUNT_V0:.*]] = llvm.mlir.poison : vector<2xi32>
   // CHECK: %[[ZERO:.*]] = llvm.mlir.constant(0 : i32) : i32
   // CHECK: %[[COUNT_V1:.*]] = llvm.insertelement %[[COUNT]], %[[COUNT_V0]][%[[ZERO]] : i32] : vector<2xi32>
   // CHECK: %[[ONE:.*]] = llvm.mlir.constant(1 : i32) : i32
@@ -230,12 +230,12 @@ spirv.func @bitfield_uextract_scalar_greater_bit_width(%base: i8, %offset: i16,
 // CHECK-LABEL: @bitfield_uextract_vector
 //  CHECK-SAME: %[[BASE:.*]]: vector<2xi32>, %[[OFFSET:.*]]: i32, %[[COUNT:.*]]: i32
 spirv.func @bitfield_uextract_vector(%base: vector<2xi32>, %offset: i32, %count: i32) "None" {
-  // CHECK: %[[OFFSET_V0:.*]] = llvm.mlir.undef : vector<2xi32>
+  // CHECK: %[[OFFSET_V0:.*]] = llvm.mlir.poison : vector<2xi32>
   // CHECK: %[[ZERO:.*]] = llvm.mlir.constant(0 : i32) : i32
   // CHECK: %[[OFFSET_V1:.*]] = llvm.insertelement %[[OFFSET]], %[[OFFSET_V0]][%[[ZERO]] : i32] : vector<2xi32>
   // CHECK: %[[ONE:.*]] = llvm.mlir.constant(1 : i32) : i32
   // CHECK: %[[OFFSET_V2:.*]] = llvm.insertelement  %[[OFFSET]], %[[OFFSET_V1]][%[[ONE]] : i32] : vector<2xi32>
-  // CHECK: %[[COUNT_V0:.*]] = llvm.mlir.undef : vector<2xi32>
+  // CHECK: %[[COUNT_V0:.*]] = llvm.mlir.poison : vector<2xi32>
   // CHECK: %[[ZERO:.*]] = llvm.mlir.constant(0 : i32) : i32
   // CHECK: %[[COUNT_V1:.*]] = llvm.insertelement %[[COUNT]], %[[COUNT_V0]][%[[ZERO]] : i32] : vector<2xi32>
   // CHECK: %[[ONE:.*]] = llvm.mlir.constant(1 : i32) : i32
diff --git a/mlir/test/Conversion/SPIRVToLLVM/misc-ops-to-llvm.mlir b/mlir/test/Conversion/SPIRVToLLVM/misc-ops-to-llvm.mlir
index 31da59dcdc726..cb8d5ed3eff6f 100644
--- a/mlir/test/Conversion/SPIRVToLLVM/misc-ops-to-llvm.mlir
+++ b/mlir/test/Conversion/SPIRVToLLVM/misc-ops-to-llvm.mlir
@@ -70,7 +70,7 @@ spirv.func @vector_shuffle_same_size(%vector1: vector<2xf32>, %vector2: vector<2
 }
 
 spirv.func @vector_shuffle_different_size(%vector1: vector<3xf32>, %vector2: vector<2xf32>) -> vector<3xf32> "None" {
-  //      CHECK: %[[UNDEF:.*]] = llvm.mlir.undef : vector<3xf32>
+  //      CHECK: %[[UNDEF:.*]] = llvm.mlir.poison : vector<3xf32>
   // CHECK-NEXT: %[[C0_0:.*]] = llvm.mlir.constant(0 : i32) : i32
   // CHECK-NEXT: %[[C0_1:.*]] = llvm.mlir.constant(0 : i32) : i32
   // CHECK-NEXT: %[[EXT0:.*]] = llvm.extractelement %arg0[%[[C0_1]] : i32] : vector<3xf32>
@@ -90,7 +90,7 @@ spirv.func @vector_shuffle_different_size(%vector1: vector<3xf32>, %vector2: vec
 
 //      CHECK: module {
 // CHECK-NEXT:   llvm.mlir.global external constant @{{.*}}() {addr_space = 0 : i32} : !llvm.struct<(i32)> {
-// CHECK-NEXT:     %[[UNDEF:.*]] = llvm.mlir.undef : !llvm.struct<(i32)>
+// CHECK-NEXT:     %[[UNDEF:.*]] = llvm.mlir.poison : !llvm.struct<(i32)>
 // CHECK-NEXT:     %[[VAL:.*]] = llvm.mlir.constant(31 : i32) : i32
 // CHECK-NEXT:     %[[RET:.*]] = llvm.insertvalue %[[VAL]], %[[UNDEF]][0] : !llvm.struct<(i32)>
 // CHECK-NEXT:     llvm.return %[[RET]] : !llvm.struct<(i32)>
@@ -109,7 +109,7 @@ spirv.module Logical OpenCL {
 
 //      CHECK: module {
 // CHECK-NEXT:   llvm.mlir.global external constant @{{.*}}() {addr_space = 0 : i32} : !llvm.struct<(i32, array<3 x i32>)> {
-// CHECK-NEXT:     %[[UNDEF:.*]] = llvm.mlir.undef : !llvm.struct<(i32, array<3 x i32>)>
+// CHECK-NEXT:     %[[UNDEF:.*]] = llvm.mlir.poison : !llvm.struct<(i32, array<3 x i32>)>
 // CHECK-NEXT:     %[[EM:.*]] = llvm.mlir.constant(18 : i32) : i32
 // CHECK-NEXT:     %[[T0:.*]] = llvm.insertvalue %[[EM]], %[[UNDEF]][0] : !llvm.struct<(i32, array<3 x i32>)>
 // CHECK-NEXT:     %[[C0:.*]] = llvm.mlir.constant(32 : i32) : i32
diff --git a/mlir/test/Conversion/TosaToLinalg/tosa-to-linalg-invalid.mlir b/mlir/test/Conversion/TosaToLinalg/tosa-to-linalg-invalid.mlir
index 75b48f2b06d89..460e207d62de6 100644
--- a/mlir/test/Conversion/TosaToLinalg/tosa-to-linalg-invalid.mlir
+++ b/mlir/test/Conversion/TosaToLinalg/tosa-to-linalg-invalid.mlir
@@ -24,7 +24,8 @@ func.func @unranked_add(%arg0 : tensor<10x10xf32> , %arg1 : tensor<10x10xf32>, %
   %reduce = tosa.reduce_max %arg0 {axis = 1 : i32} : (tensor<10x10xf32>) -> tensor<10x1xf32>
   %1 = tosa.add %reduce, %arg1 : (tensor<10x1xf32>, tensor<10x10xf32>) -> tensor<10x10xf32>
   %0 = tosa.add %1, %arg2 : (tensor<10x10xf32>, tensor<*xf32>) -> tensor<*xf32>
-  %2 = tosa.reshape %0 {new_shape = array<i64: 10, 10>} : (tensor<*xf32>) -> tensor<10x10xf32>
+  %s = tosa.const_shape {value = dense<[10, 10]> : tensor<2xindex>} : () -> !tosa.shape<2>
+  %2 = tosa.reshape %0, %s : (tensor<*xf32>, !tosa.shape<2>) -> tensor<10x10xf32>
   return %2 : tensor<10x10xf32>
 }
 
diff --git a/mlir/test/Conversion/TosaToLinalg/tosa-to-linalg-named.mlir b/mlir/test/Conversion/TosaToLinalg/tosa-to-linalg-named.mlir
index 5eeaebb384e40..87c388b6f5ee3 100644
--- a/mlir/test/Conversion/TosaToLinalg/tosa-to-linalg-named.mlir
+++ b/mlir/test/Conversion/TosaToLinalg/tosa-to-linalg-named.mlir
@@ -23,7 +23,7 @@ func.func @matmul_quantized(%arg0: tensor<1x5x3xi8>, %arg1: tensor<1x3x6xi8>) ->
   // CHECK: [[ONE:%.+]] = arith.constant 1
   // CHECK: [[TWO:%.+]] = arith.constant 2
   // CHECK: linalg.quantized_batch_matmul ins(%arg0, %arg1, [[ONE]], [[TWO]] : tensor<1x5x3xi8>, tensor<1x3x6xi8>, i32, i32) outs([[FILLED]] : tensor<1x5x6xi32>) -> tensor<1x5x6xi32>
-  %0 = tosa.matmul %arg0, %arg1 {quantization_info = #tosa.matmul_quant<a_zp = 1, b_zp = 2>} : (tensor<1x5x3xi8>, tensor<1x3x6xi8>) -> tensor<1x5x6xi32>
+  %0 = tosa.matmul %arg0, %arg1 {a_zp = 1 : i32, b_zp = 2 : i32} : (tensor<1x5x3xi8>, tensor<1x3x6xi8>) -> tensor<1x5x6xi32>
   return %0 : tensor<1x5x6xi32>
 }
 
@@ -124,7 +124,7 @@ func.func @quantized_fully_connected(%arg0: tensor<5x3xi8>, %arg1: tensor<6x3xi8
   // CHECK: %[[C2:.+]] = arith.constant 2 : i32
   // CHECK: linalg.quantized_matmul ins(%arg0, %[[TRANSPOSE]], %[[C1]], %[[C2]] : tensor<5x3xi8>, tensor<3x6xi8>, i32, i32) outs(%[[BROADCAST]] : tensor<5x6xi32>) -> tensor<5x6xi32>
 
-  %0 = tosa.fully_connected %arg0, %arg1, %arg2 {quantization_info = #tosa.conv_quant<input_zp = 1, weight_zp = 2>} : (tensor<5x3xi8>, tensor<6x3xi8>, tensor<6xi32>) -> tensor<5x6xi32>
+  %0 = tosa.fully_connected %arg0, %arg1, %arg2 {input_zp = 1 : i32, weight_zp = 2 : i32} : (tensor<5x3xi8>, tensor<6x3xi8>, tensor<6xi32>) -> tensor<5x6xi32>
   return %0 : tensor<5x6xi32>
 }
 
@@ -544,7 +544,8 @@ func.func @conv2d_i8(%input: tensor<1x49x42x27xi8>, %weights: tensor<28x1x1x27xi
   // CHECK: linalg.conv_2d_nhwc_fhwc_q {dilations = dense<[2, 1]> : tensor<2xi64>, strides = dense<1> : tensor<2xi64>} ins(%arg0, %arg1, %c0_i32, %c0_i32_0 : tensor<1x49x42x27xi8>, tensor<28x1x1x27xi8>, i32, i32) outs(%[[BROADCAST]] : tensor<1x45x40x28xi32>) -> tensor<1x45x40x28xi32>
   // HWCF: linalg.conv_2d_nhwc_hwcf_q {dilations = dense<[2, 1]> : tensor<2xi64>, strides = dense<1> : tensor<2xi64>} ins(%arg0, %[[TRANSPOSE]], %c0_i32, %c0_i32_0 : tensor<1x49x42x27xi8>, tensor<1x1x27x28xi8>, i32, i32) outs(%{{[a-zA-Z0-9_]*}} : tensor<1x45x40x28xi32>) -> tensor<1x45x40x28xi32>
 
-  %0 = tosa.conv2d %input, %weights, %bias {acc_type = i32, dilation = array<i64: 2, 1>, pad = array<i64: 0, 0, 0, 0>, quantization_info = #tosa.conv_quant<input_zp = 0, weight_zp = 0>, stride = array<i64: 1, 1>} : (tensor<1x49x42x27xi8>, tensor<28x1x1x27xi8>, tensor<28xi8>) -> tensor<1x45x40x28xi32>
+  %zp = "tosa.const"() {value = dense<0> : tensor<1xi8>} : () -> tensor<1xi8>
+  %0 = tosa.conv2d %input, %weights, %bias, %zp, %zp {acc_type = i32, dilation = array<i64: 2, 1>, pad = array<i64: 0, 0, 0, 0>, stride = array<i64: 1, 1>} : (tensor<1x49x42x27xi8>, tensor<28x1x1x27xi8>, tensor<28xi8>, tensor<1xi8>, tensor<1xi8>) -> tensor<1x45x40x28xi32>
   return
 }
 
@@ -687,7 +688,9 @@ func.func @conv2d_quant(%arg0 : tensor<1x12x12x1xi8>, %arg1 : tensor<1024x3x3x1x
   // CHECK: tensor.pad %arg0 low[0, 1, 1, 0] high[0, 1, 1, 0]
   // CHECK:   tensor.yield %[[C22]]
   // CHECK: linalg.conv_2d_nhwc_fhwc_q
-  %0 = tosa.conv2d %arg0, %arg1, %arg2 {acc_type = i32, dilation = array<i64: 1, 1>, pad = array<i64: 1, 1, 1, 1>, quantization_info = #tosa.conv_quant<input_zp = -22, weight_zp = 42>, stride = array<i64: 1, 1>} : (tensor<1x12x12x1xi8>, tensor<1024x3x3x1xi8>, tensor<1024xi32>) -> tensor<1x12x12x1024xi32>
+  %input_zp = "tosa.const"() {value = dense<-22> : tensor<1xi8>} : () -> tensor<1xi8>
+  %weight_zp = "tosa.const"() {value = dense<42> : tensor<1xi8>} : () -> tensor<1xi8>
+  %0 = tosa.conv2d %arg0, %arg1, %arg2 , %input_zp, %weight_zp {acc_type = i32, dilation = array<i64: 1, 1>, pad = array<i64: 1, 1, 1, 1>, stride = array<i64: 1, 1>} : (tensor<1x12x12x1xi8>, tensor<1024x3x3x1xi8>, tensor<1024xi32>, tensor<1xi8>, tensor<1xi8>) -> tensor<1x12x12x1024xi32>
   return
 }
 
@@ -799,7 +802,9 @@ func.func @depthwise_conv_quant(%arg0 : tensor<1x12x12x4xi8>, %arg1 : tensor<3x3
   // CHECK:   [[ADD:%.+]] = arith.addi %[[ARG3]], %[[ARG4]] : i32
   // CHECK:   linalg.yield [[ADD]] : i32
   // CHECK: } -> tensor<1x12x12x512xi32>
-  %0 = tosa.depthwise_conv2d %arg0, %arg1, %arg2 {acc_type = i32, pad = array<i64: 1, 1, 1, 1>, quantization_info = #tosa.conv_quant<input_zp = -128, weight_zp = 42>, stride = array<i64: 1, 1>, dilation = array<i64: 1, 1> } : (tensor<1x12x12x4xi8>, tensor<3x3x4x128xi8>, tensor<512xi32>) -> tensor<1x12x12x512xi32>
+  %input_zp = "tosa.const"() {value = dense<-128> : tensor<1xi8>} : () -> tensor<1xi8>
+  %weight_zp = "tosa.const"() {value = dense<42> : tensor<1xi8>} : () -> tensor<1xi8>
+  %0 = tosa.depthwise_conv2d %arg0, %arg1, %arg2, %input_zp, %weight_zp {acc_type = i32, pad = array<i64: 1, 1, 1, 1>, stride = array<i64: 1, 1>, dilation = array<i64: 1, 1> } : (tensor<1x12x12x4xi8>, tensor<3x3x4x128xi8>, tensor<512xi32>, tensor<1xi8>, tensor<1xi8>) -> tensor<1x12x12x512xi32>
   return
 }
 
@@ -823,7 +828,9 @@ func.func @depthwise_conv_quant_dilations(%arg0 : tensor<1x14x14x4xi8>, %arg1 :
   // CHECK:   [[ADD:%.+]] = arith.addi %[[ARG3]], %[[ARG4]] : i32
   // CHECK:   linalg.yield [[ADD]] : i32
   // CHECK: } -> tensor<1x10x10x512xi32>
-  %0 = tosa.depthwise_conv2d %arg0, %arg1, %arg2 {acc_type = i32, pad = array<i64: 0, 0, 0, 0>, quantization_info = #tosa.conv_quant<input_zp = -128, weight_zp = 42>, stride = array<i64: 1, 1>, dilation = array<i64: 2, 2> } : (tensor<1x14x14x4xi8>, tensor<3x3x4x128xi8>, tensor<512xi32>)  -> tensor<1x10x10x512xi32>
+  %input_zp = "tosa.const"() {value = dense<-128> : tensor<1xi8>} : () -> tensor<1xi8>
+  %weight_zp = "tosa.const"() {value = dense<42> : tensor<1xi8>} : () -> tensor<1xi8>
+  %0 = tosa.depthwise_conv2d %arg0, %arg1, %arg2 , %input_zp, %weight_zp {acc_type = i32, pad = array<i64: 0, 0, 0, 0>, stride = array<i64: 1, 1>, dilation = array<i64: 2, 2> } : (tensor<1x14x14x4xi8>, tensor<3x3x4x128xi8>, tensor<512xi32>, tensor<1xi8>, tensor<1xi8>)  -> tensor<1x10x10x512xi32>
   return
 }
 
@@ -905,7 +912,9 @@ func.func @conv3d_i8(%input: tensor<1x49x48x47x27xi8>, %weights: tensor<28x3x4x5
   // CHECK-SAME: ins(%arg0, %[[TRANSPOSE]], %[[IZP]], %[[FZP]] : tensor<1x49x48x47x27xi8>, tensor<3x4x5x27x28xi8>, i32, i32)
   // CHECK-SAME: outs(%[[BROADCAST]] : tensor<1x47x45x43x28xi32>) -> tensor<1x47x45x43x28xi32>
 
-  %0 = tosa.conv3d %input, %weights, %bias {acc_type = i32, pad = array<i64: 0, 0, 0, 0, 0, 0>, quantization_info = #tosa.conv_quant<input_zp = -128, weight_zp = 42>, stride = array<i64: 1, 1, 1>, dilation = array<i64: 1, 1, 1>} : (tensor<1x49x48x47x27xi8>, tensor<28x3x4x5x27xi8>, tensor<28xi32>)  -> tensor<1x47x45x43x28xi32>
+  %input_zp = "tosa.const"() {value = dense<-128> : tensor<1xi8>} : () -> tensor<1xi8>
+  %weight_zp = "tosa.const"() {value = dense<42> : tensor<1xi8>} : () -> tensor<1xi8>
+  %0 = tosa.conv3d %input, %weights, %bias , %input_zp, %weight_zp {acc_type = i32, pad = array<i64: 0, 0, 0, 0, 0, 0>, stride = array<i64: 1, 1, 1>, dilation = array<i64: 1, 1, 1>} : (tensor<1x49x48x47x27xi8>, tensor<28x3x4x5x27xi8>, tensor<28xi32>, tensor<1xi8>, tensor<1xi8>)  -> tensor<1x47x45x43x28xi32>
   return
 }
 
diff --git a/mlir/test/Conversion/TosaToLinalg/tosa-to-linalg.mlir b/mlir/test/Conversion/TosaToLinalg/tosa-to-linalg.mlir
index f860dca85c9e9..17add2d41afe7 100644
--- a/mlir/test/Conversion/TosaToLinalg/tosa-to-linalg.mlir
+++ b/mlir/test/Conversion/TosaToLinalg/tosa-to-linalg.mlir
@@ -472,7 +472,8 @@ func.func @test_simple_f32(%arg0: tensor<1xf32>) -> () {
 
   // CHECK: linalg.generic
   // CHECK: arith.mulf
-  %4 = tosa.mul %0, %1 {shift = 0 : i8} : (tensor<1xf32>, tensor<1xf32>) -> tensor<1xf32>
+  %shift = "tosa.const"() <{value = dense<0> : tensor<1xi8>}> : () -> tensor<1xi8>
+  %4 = tosa.mul %0, %1, %shift : (tensor<1xf32>, tensor<1xf32>, tensor<1xi8>) -> tensor<1xf32>
 
   // CHECK: linalg.generic
   // CHECK: arith.negf
@@ -529,7 +530,7 @@ func.func @test_simple_f32(%arg0: tensor<1xf32>) -> () {
   // CHECK: linalg.generic
   // CHECK: arith.minimumf
   // CHECK: arith.maximumf
-  %18 = tosa.clamp %0 {min_int = 1 : i64, max_int = 5 : i64, min_fp = 1.0 : f32, max_fp = 5.0 : f32} : (tensor<1xf32>) -> tensor<1xf32>
+  %18 = tosa.clamp %0 {min_val = 1.0 : f32, max_val = 5.0 : f32} : (tensor<1xf32>) -> tensor<1xf32>
 
   // CHECK: linalg.generic
   // CHECK: arith.negf
@@ -618,7 +619,8 @@ func.func @test_simple_i16(%arg0: tensor<1xi16>) -> () {
   // CHECK: arith.extsi
   // CHECK: arith.extsi
   // CHECK: arith.muli
-  %0 = tosa.mul %arg0, %arg0 {shift = 0 : i8} : (tensor<1xi16>, tensor<1xi16>) -> tensor<1xi32>
+  %shift = "tosa.const"() <{value = dense<0> : tensor<1xi8>}> : () -> tensor<1xi8>
+  %0 = tosa.mul %arg0, %arg0, %shift : (tensor<1xi16>, tensor<1xi16>, tensor<1xi8>) -> tensor<1xi32>
 
   return
 }
@@ -646,12 +648,14 @@ func.func @test_simple_i32(%arg0: tensor<1xi32>, %unsigned: tensor<1xui32>, %uns
 
   // CHECK: linalg.generic
   // CHECK: arith.muli
-  %2 = tosa.mul %arg0, %arg0 {shift = 0 : i8} : (tensor<1xi32>, tensor<1xi32>) -> tensor<1xi32>
+  %shift1 = "tosa.const"() <{value = dense<0> : tensor<1xi8>}> : () -> tensor<1xi8>
+  %2 = tosa.mul %arg0, %arg0, %shift1 : (tensor<1xi32>, tensor<1xi32>, tensor<1xi8>) -> tensor<1xi32>
 
   // CHECK: linalg.generic
   // CHECK: arith.constant 2
   // CHECK: apply_scale
-  %3 = tosa.mul %arg0, %arg0 {shift = 2 : i8} : (tensor<1xi32>, tensor<1xi32>) -> tensor<1xi32>
+  %shift2 = "tosa.const"() <{value = dense<2> : tensor<1xi8>}> : () -> tensor<1xi8>
+  %3 = tosa.mul %arg0, %arg0, %shift2: (tensor<1xi32>, tensor<1xi32>, tensor<1xi8>) -> tensor<1xi32>
 
   // CHECK: linalg.generic
   // CHECK: arith.divsi
@@ -727,35 +731,14 @@ func.func @test_simple_i32(%arg0: tensor<1xi32>, %unsigned: tensor<1xui32>, %uns
   // CHECK: linalg.generic
   // CHECK-DAG: arith.maxsi
   // CHECK-DAG: arith.minsi
-  %19 = tosa.clamp %0 {min_int = 1 : i64, max_int = 5 : i64, min_fp = 1.0 : f32, max_fp = 5.0 : f32} : (tensor<1xi32>) -> tensor<1xi32>
+  %19 = tosa.clamp %0 {min_val = 1 : i32, max_val = 5 : i32} : (tensor<1xi32>) -> tensor<1xi32>
 
   // CHECK: linalg.generic
   // CHECK-DAG: %[[LB:.*]] = arith.constant 4 : i32
   // CHECK-DAG: %[[UB:.*]] = arith.constant 32 : i32
   // CHECK-DAG: arith.maxui %[[LB]],
   // CHECK-DAG: arith.minui %[[UB]],
-  %u0 = tosa.clamp %unsigned {min_int = 4 : i64, max_int = 32 : i64, min_fp = 1.0 : f32, max_fp = 5.0 : f32} : (tensor<1xui32>) -> tensor<1xui32>
-
-  // CHECK: linalg.generic
-  // CHECK-DAG: %[[LB:.*]] = arith.constant -1 : i32
-  // CHECK-DAG: %[[UB:.*]] = arith.constant -1 : i32
-  // CHECK-DAG: arith.maxui %[[LB]],
-  // CHECK-DAG: arith.minui %[[UB]],
-  %u1 = tosa.clamp %unsigned {min_int = 9223372036854775807 : i64, max_int = 9223372036854775807 : i64, min_fp = 1.0 : f32, max_fp = 5.0 : f32} : (tensor<1xui32>) -> tensor<1xui32>
-
-  // CHECK: linalg.generic
-  // CHECK-DAG: %[[LB:.*]] = arith.constant 0 : i32
-  // CHECK-DAG: %[[UB:.*]] = arith.constant 0 : i32
-  // CHECK-DAG: arith.maxui %[[LB]],
-  // CHECK-DAG: arith.minui %[[UB]],
-  %u2 = tosa.clamp %unsigned {min_int = -3 : i64, max_int = -2 : i64, min_fp = 1.0 : f32, max_fp = 5.0 : f32} : (tensor<1xui32>) -> tensor<1xui32>
-
-  // CHECK: linalg.generic
-  // CHECK-DAG: %[[LB:.*]] = arith.constant 0 : i64
-  // CHECK-DAG: %[[UB:.*]] = arith.constant 9223372036854775807 : i64
-  // CHECK-DAG: arith.maxui %[[LB]],
-  // CHECK-DAG: arith.minui %[[UB]],
-  %u3 = tosa.clamp %unsigned64 {min_int = -3 : i64, max_int = 9223372036854775807 : i64, min_fp = 1.0 : f32, max_fp = 5.0 : f32} : (tensor<1xui64>) -> tensor<1xui64>
+  %u0 = tosa.clamp %unsigned {min_val = 4 : ui32, max_val = 32 : ui32} : (tensor<1xui32>) -> tensor<1xui32>
 
   // CHECK: linalg.generic
   // CHECK: arith.trunci
@@ -805,15 +788,7 @@ func.func @test_i8(%arg0: tensor<1xi8>) -> () {
   // CHECK-DAG: %[[C126:.+]] = arith.constant 126
   // CHECK-DAG: %[[LOWER:.+]] = arith.maxsi %[[C127]], %[[ARG1]]
   // CHECK-DAG: %[[CLAMPED:.+]] = arith.minsi %[[C126]], %[[LOWER]]
-  %0 = tosa.clamp %arg0 {min_int = -127 : i64, max_int = 126 : i64, min_fp = 0.0 : f32, max_fp = 0.0 : f32} : (tensor<1xi8>) -> tensor<1xi8>
-
-  // CHECK: linalg.generic
-  // CHECK: ^bb0(%[[ARG1:.+]]: i8,
-  // CHECK-DAG: %[[C128:.+]] = arith.constant -128
-  // CHECK-DAG: %[[C127:.+]] = arith.constant 127
-  // CHECK-DAG: %[[LOWER:.+]] = arith.maxsi %[[C128]], %[[ARG1]]
-  // CHECK-DAG: %[[CLAMPED:.+]] = arith.minsi %[[C127]], %[[LOWER]]
-  %1 = tosa.clamp %arg0 {min_int = -130 : i64, max_int = 130 : i64, min_fp = 0.0 : f32, max_fp = 0.0 : f32} : (tensor<1xi8>) -> tensor<1xi8>
+  %0 = tosa.clamp %arg0 {min_val = -127 : i8, max_val = 126 : i8} : (tensor<1xi8>) -> tensor<1xi8>
 
   return
 }
@@ -828,7 +803,7 @@ func.func @test_i64(%arg0: tensor<1xi64>) -> () {
   // CHECK-DAG: %[[C126:.+]] = arith.constant 9223372036854775807
   // CHECK-DAG: %[[LOWER:.+]] = arith.maxsi %[[C127]], %[[ARG1]]
   // CHECK-DAG: %[[CLAMPED:.+]] = arith.minsi %[[C126]], %[[LOWER]]
-  %0 = tosa.clamp %arg0 {min_int = -9223372036854775808 : i64, max_int = 9223372036854775807 : i64, min_fp = 0.0 : f32, max_fp = 0.0 : f32} : (tensor<1xi64>) -> tensor<1xi64>
+  %0 = tosa.clamp %arg0 {min_val = -9223372036854775808 : i64, max_val = 9223372036854775807 : i64} : (tensor<1xi64>) -> tensor<1xi64>
 
   return
 }
@@ -843,7 +818,7 @@ func.func @test_clamp_f16(%arg0: tensor<1xf16>) -> () {
   // CHECK-DAG: %[[C6:.+]] = arith.constant 6.0
   // CHECK-DAG: %[[MIN:.+]] = arith.minimumf %[[ARG1]], %[[C6]]
   // CHECK-DAG: %[[MAX:.+]] = arith.maximumf %[[MIN]], %[[C0]]
-  %0 = tosa.clamp %arg0 {min_int = 0 : i64, max_int = 0 : i64, min_fp = 0.0 : f32, max_fp = 6.0 : f32} : (tensor<1xf16>) -> tensor<1xf16>
+  %0 = tosa.clamp %arg0 {min_val = 0.0 : f16, max_val = 6.0 : f16} : (tensor<1xf16>) -> tensor<1xf16>
 
   return
 }
@@ -878,33 +853,56 @@ func.func @test_bool(%arg0: tensor<1xi1>, %arg1: tensor<1xi1>) -> () {
 func.func @test_negate_quantized(%arg0: tensor<1xi8>) -> () {
   // CHECK: linalg.generic
   // CHECK: ^bb0(%[[BBARG0:.+]]: i8,
-  // CHECK: [[CNST:%.+]] = arith.constant 7
+  // CHECK: [[ZERO:%.+]] = arith.constant 0
+  // CHECK: [[SUB:%.+]] = arith.subi [[ZERO]], %[[BBARG0]]
+  // CHECK: linalg.yield [[SUB]]
+  %0 = tosa.negate %arg0 {input_zp1 = 0 : i32, output_zp = 0 : i32} : (tensor<1xi8>) -> tensor<1xi8>
+
+  // CHECK: linalg.generic
+  // CHECK: ^bb0(%[[BBARG0:.+]]: i8,
+  // CHECK: [[C32639:%.+]] = arith.constant 32639
   // CHECK: [[EXT:%.+]] = arith.extsi %[[BBARG0]] : i8 to i16
-  // CHECK: [[SUB:%.+]] = arith.subi [[CNST]], [[EXT]]
+  // CHECK: [[SUB:%.+]] = arith.subi [[C32639]], [[EXT]]
   // CHECK: [[MIN:%.+]] = arith.constant -128
   // CHECK: [[MAX:%.+]] = arith.constant 127
   // CHECK: [[LBOUND:%.+]] = arith.maxsi [[MIN]], [[SUB]]
   // CHECK: [[UBOUND:%.+]] = arith.minsi [[MAX]], [[LBOUND]]
   // CHECK: [[TRUNC:%.+]] = arith.trunci [[UBOUND]]
   // CHECK: linalg.yield [[TRUNC]]
-  %0 = tosa.negate %arg0 {quantization_info = #tosa.unary_quant<input_zp = 0, output_zp = 7>} : (tensor<1xi8>) -> tensor<1xi8>
+  %1 = tosa.negate %arg0 {input1_zp = 32639 : i32, output_zp = 0 : i32} : (tensor<1xi8>) -> tensor<1xi8>
 
   // CHECK: linalg.generic
   // CHECK: ^bb0(%[[BBARG0:.+]]: i8,
-  // CHECK: [[EXT:%.+]] = arith.extsi %[[BBARG0]] : i8 to i16
-  %1 = tosa.negate %arg0 {quantization_info = #tosa.unary_quant<input_zp = 32639, output_zp = 0>} : (tensor<1xi8>) -> tensor<1xi8>
+  // CHECK: [[C32640:%.+]] = arith.constant 32640
+  // CHECK: [[EXT:%.+]] = arith.extsi %[[BBARG0]] : i8 to i32
+  // CHECK: [[SUB:%.+]] = arith.subi [[C32640]], [[EXT]]
+  // CHECK: [[MIN:%.+]] = arith.constant -128
+  // CHECK: [[MAX:%.+]] = arith.constant 127
+  // CHECK: [[LBOUND:%.+]] = arith.maxsi [[MIN]], [[SUB]]
+  // CHECK: [[UBOUND:%.+]] = arith.minsi [[MAX]], [[LBOUND]]
+  // CHECK: [[TRUNC:%.+]] = arith.trunci [[UBOUND]]
+  // CHECK: linalg.yield [[TRUNC]]
+  %2 = tosa.negate %arg0 {input1_zp = 32640 : i32, output_zp = 0 : i32} : (tensor<1xi8>) -> tensor<1xi8>
 
   // CHECK: linalg.generic
   // CHECK: ^bb0(%[[BBARG0:.+]]: i8,
-  // CHECK: [[EXT:%.+]] = arith.extsi %[[BBARG0]] : i8 to i32
-  %2 = tosa.negate %arg0 {quantization_info = #tosa.unary_quant<input_zp = 32640, output_zp = 0>} : (tensor<1xi8>) -> tensor<1xi8>
+  // CHECK: [[C_128:%.+]] = arith.constant -128
+  // CHECK: [[EXT:%.+]] = arith.extsi %[[BBARG0]] : i8 to i16
+  // CHECK: [[SUB:%.+]] = arith.subi [[C_128]], [[EXT]]
+  // CHECK: [[MIN:%.+]] = arith.constant -128
+  // CHECK: [[MAX:%.+]] = arith.constant 127
+  // CHECK: [[LBOUND:%.+]] = arith.maxsi [[MIN]], [[SUB]]
+  // CHECK: [[UBOUND:%.+]] = arith.minsi [[MAX]], [[LBOUND]]
+  // CHECK: [[TRUNC:%.+]] = arith.trunci [[UBOUND]]
+  // CHECK: linalg.yield [[TRUNC]]
+  %3 = tosa.negate %arg0 {input1_zp = -128 : i32, output_zp = 0 : i32} : (tensor<1xi8>) -> tensor<1xi8>
 
   // CHECK: linalg.generic
   // CHECK: ^bb0(%[[BBARG0:.+]]: i8,
   // CHECK: [[ZERO:%.+]] = arith.constant 0
   // CHECK: [[SUB:%.+]] = arith.subi [[ZERO]],
   // CHECK: linalg.yield [[SUB]]
-  %3 = tosa.negate %arg0 {quantization_info = #tosa.unary_quant<input_zp = 0, output_zp = 0>} : (tensor<1xi8>) -> tensor<1xi8>
+  %4 = tosa.negate %arg0 {quantization_info = #tosa.unary_quant<input_zp = 0, output_zp = 0>} : (tensor<1xi8>) -> tensor<1xi8>
 
   return
 }
@@ -1375,7 +1373,8 @@ func.func @tile(%arg0 : tensor<2x3xi8>) -> () {
   // CHECK: [[GENERIC:%.+]] = linalg.generic {indexing_maps = [#[[$MAP0]], #[[$MAP1]]], iterator_types = ["parallel", "parallel", "parallel", "parallel"]} ins(%[[ARG0]] : tensor<2x3xi8>) outs([[INIT]] : tensor<2x2x1x3xi8>)
   // CHECK: ^bb0(%[[ARG1:[0-9a-zA-Z_]+]]: i8
   // CHECK:   linalg.yield %[[ARG1]] : i8
-  // CHECK: tosa.reshape [[GENERIC]] {new_shape = array<i64: 4, 3>}
+  // CHECK: [[CONST3:%.+]] = tosa.const_shape {value = dense<[4, 3]> : tensor<2xindex>} : () -> !tosa.shape<2>
+  // CHECK: tosa.reshape [[GENERIC]], [[CONST3]]
   %cst21 = tosa.const_shape { value = dense<[2, 1]> : tensor<2xindex> } : () -> !tosa.shape<2>
   %0 = tosa.tile %arg0, %cst21: (tensor<2x3xi8>, !tosa.shape<2>) -> tensor<4x3xi8>
 
@@ -1383,7 +1382,8 @@ func.func @tile(%arg0 : tensor<2x3xi8>) -> () {
   // CHECK: [[GENERIC:%.+]] = linalg.generic {indexing_maps = [#[[$MAP0]], #[[$MAP1]]], iterator_types = ["parallel", "parallel", "parallel", "parallel"]} ins(%[[ARG0]] : tensor<2x3xi8>) outs([[INIT]] : tensor<1x2x2x3xi8>)
   // CHECK: ^bb0(%[[ARG1:[0-9a-zA-Z_]+]]: i8
   // CHECK:   linalg.yield %[[ARG1]] : i8
-  // CHECK: tosa.reshape [[GENERIC]] {new_shape = array<i64: 2, 6>}
+  // CHECK: [[CONST8:%.+]] = tosa.const_shape {value = dense<[2, 6]> : tensor<2xindex>} : () -> !tosa.shape<2>
+  // tosa.reshape [[GENERIC]], [[CONST8]]
   %cst12 = tosa.const_shape { value = dense<[1, 2]> : tensor<2xindex> } : () -> !tosa.shape<2>
   %1 = tosa.tile %arg0, %cst12: (tensor<2x3xi8>, !tosa.shape<2>) -> tensor<2x6xi8>
 
@@ -1391,8 +1391,9 @@ func.func @tile(%arg0 : tensor<2x3xi8>) -> () {
   // CHECK: [[GENERIC:%.+]] = linalg.generic {indexing_maps = [#[[$MAP0]], #[[$MAP1]]], iterator_types = ["parallel", "parallel", "parallel", "parallel"]} ins(%[[ARG0]] : tensor<2x3xi8>) outs([[INIT]] : tensor<5x2x7x3xi8>)
   // CHECK: ^bb0(%[[ARG1:[0-9a-zA-Z_]+]]: i8
   // CHECK:   linalg.yield %[[ARG1]] : i8
-  // CHECK: tosa.reshape [[GENERIC]] {new_shape = array<i64: 10, 21>}
   %cst57 = tosa.const_shape { value = dense<[5, 7]> : tensor<2xindex> } : () -> !tosa.shape<2>
+  // CHECK: [[CONST13:%.+]] = tosa.const_shape {value = dense<[10, 21]> : tensor<2xindex>} : () -> !tosa.shape<2>
+  // CHECK: tosa.reshape [[GENERIC]], [[CONST13]]
   %2 = tosa.tile %arg0, %cst57: (tensor<2x3xi8>, !tosa.shape<2>)  -> tensor<10x21xi8>
 
   return
@@ -1412,7 +1413,8 @@ func.func @tile_dyn_input(%arg0 : tensor<?x3xi8>) -> () {
   // CHECK: %[[GENERIC:.+]] = linalg.generic {indexing_maps = [#[[$MAP0]], #[[$MAP1]]], iterator_types = ["parallel", "parallel", "parallel", "parallel"]} ins(%[[ARG0]] : tensor<?x3xi8>) outs(%[[INIT]] : tensor<2x?x1x3xi8>)
   // CHECK: ^bb0(%[[ARG1:.+]]: i8,
   // CHECK:   linalg.yield %[[ARG1]] : i8
-  // CHECK: tosa.reshape %[[GENERIC]] {new_shape = array<i64: -9223372036854775808, 3>}
+  // CHECK: %[[CONST3:.+]] = tosa.const_shape {value = dense<[-1, 3]> : tensor<2xindex>} : () -> !tosa.shape<2>
+  // CHECK: tosa.reshape %[[GENERIC]], %[[CONST3]]
   %cst21 = tosa.const_shape { value = dense<[2, 1]> : tensor<2xindex> } : () -> !tosa.shape<2>
   %0 = tosa.tile %arg0, %cst21: (tensor<?x3xi8>, !tosa.shape<2>)  -> tensor<?x3xi8>
 
@@ -1433,7 +1435,8 @@ func.func @tile_dyn_multiples(%arg0 : tensor<2x3xi8>) -> () {
   // CHECK: %[[GENERIC:.+]] = linalg.generic {indexing_maps = [#[[$MAP0]], #[[$MAP1]]], iterator_types = ["parallel", "parallel", "parallel", "parallel"]} ins(%[[ARG0]] : tensor<2x3xi8>) outs(%[[INIT]] : tensor<2x2x?x3xi8>)
   // CHECK: ^bb0(%[[ARG1:.+]]: i8,
   // CHECK:   linalg.yield %[[ARG1]] : i8
-  // CHECK: tosa.reshape %[[GENERIC]] {new_shape = array<i64: 2, -9223372036854775808>}
+  // CHECK: %[[CONST2:.+]] = tosa.const_shape {value = dense<[2, -1]> : tensor<2xindex>} : () -> !tosa.shape<2>
+  // CHECK: tosa.reshape %[[GENERIC]], %[[CONST2]]
   %cst = tosa.const_shape { value = dense<[2, -1]> : tensor<2xindex> } : () -> !tosa.shape<2>
   %0 = tosa.tile %arg0, %cst: (tensor<2x3xi8>, !tosa.shape<2>)  -> tensor<2x?xi8>
 
diff --git a/mlir/test/Conversion/TosaToTensor/tosa-to-tensor-invalid.mlir b/mlir/test/Conversion/TosaToTensor/tosa-to-tensor-invalid.mlir
index 36eb4d4669b07..a72d6b333f7ea 100644
--- a/mlir/test/Conversion/TosaToTensor/tosa-to-tensor-invalid.mlir
+++ b/mlir/test/Conversion/TosaToTensor/tosa-to-tensor-invalid.mlir
@@ -2,7 +2,9 @@
 
 // CHECK-LABEL:  @slice_resultType_unranked
 func.func @slice_resultType_unranked(%arg0: tensor<?xf32>) -> (tensor<*xf32>) {
+  %0 = tosa.const_shape  {value = dense<2> : tensor<1xindex>} : () -> !tosa.shape<1>
+  %1 = tosa.const_shape  {value = dense<0> : tensor<1xindex>} : () -> !tosa.shape<1>
   // expected-error@+1 {{failed to legalize operation 'tosa.slice'}}
-  %0 = "tosa.slice"(%arg0) {start = array<i64: 2>, size = array<i64: 0>} : (tensor<?xf32>)  -> (tensor<*xf32>)
-  return %0 : tensor<*xf32>
+  %2 = tosa.slice %arg0, %0, %1 : (tensor<?xf32>, !tosa.shape<1>, !tosa.shape<1>) -> tensor<*xf32>
+  return %2 : tensor<*xf32>
 }
diff --git a/mlir/test/Conversion/TosaToTensor/tosa-to-tensor.mlir b/mlir/test/Conversion/TosaToTensor/tosa-to-tensor.mlir
index 2f11b31aad230..c2eaba4c563d0 100644
--- a/mlir/test/Conversion/TosaToTensor/tosa-to-tensor.mlir
+++ b/mlir/test/Conversion/TosaToTensor/tosa-to-tensor.mlir
@@ -6,7 +6,8 @@
 // CHECK-SAME: %[[ARG_0:[a-zA-Z0-9_]+]]: tensor<f32>
 // CHECK: return %[[ARG_0]] : tensor<f32>
 func.func @test_reshape_0d_same_s2s_explicit(%arg0: tensor<f32>) -> tensor<f32> {
-  %0 = "tosa.reshape"(%arg0) {new_shape = array<i64>} : (tensor<f32>) -> tensor<f32>
+  %s = tosa.const_shape { value = dense<> : tensor<0xindex> } : () -> !tosa.shape<0>
+  %0 = "tosa.reshape"(%arg0, %s) : (tensor<f32>, !tosa.shape<0>) -> tensor<f32>
   return %0 : tensor<f32>
 }
 
@@ -18,7 +19,8 @@ func.func @test_reshape_0d_same_s2s_explicit(%arg0: tensor<f32>) -> tensor<f32>
 // CHECK: %[[VAL_1:.*]] = tensor.cast %[[VAL_0]] : tensor<1xf32> to tensor<?xf32>
 // CHECK: return %[[VAL_1]] : tensor<?xf32>
 func.func @test_reshape_0d_up_s2d_auto(%arg0: tensor<f32>) -> tensor<?xf32> {
-  %0 = "tosa.reshape"(%arg0) {new_shape = array<i64: -1>} : (tensor<f32>) -> tensor<?xf32>
+  %s = tosa.const_shape { value = dense<-1> : tensor<1xindex> } : () -> !tosa.shape<1>
+  %0 = "tosa.reshape"(%arg0, %s) : (tensor<f32>, !tosa.shape<1>) -> tensor<?xf32>
   return %0 : tensor<?xf32>
 }
 
@@ -30,7 +32,8 @@ func.func @test_reshape_0d_up_s2d_auto(%arg0: tensor<f32>) -> tensor<?xf32> {
 // CHECK: %[[VAL_1:.*]] = tensor.cast %[[VAL_0]] : tensor<1xf32> to tensor<?xf32>
 // CHECK: return %[[VAL_1]] : tensor<?xf32>
 func.func @test_reshape_0d_up_s2d_explicit(%arg0: tensor<f32>) -> tensor<?xf32> {
-  %0 = "tosa.reshape"(%arg0) {new_shape = array<i64: 1>} : (tensor<f32>) -> tensor<?xf32>
+  %s = tosa.const_shape { value = dense<1> : tensor<1xindex> } : () -> !tosa.shape<1>
+  %0 = "tosa.reshape"(%arg0, %s) : (tensor<f32>, !tosa.shape<1>) -> tensor<?xf32>
   return %0 : tensor<?xf32>
 }
 
@@ -41,7 +44,8 @@ func.func @test_reshape_0d_up_s2d_explicit(%arg0: tensor<f32>) -> tensor<?xf32>
 // CHECK: %[[VAL_0:.*]] = tensor.expand_shape %[[ARG_0]] [] output_shape [1] : tensor<f32> into tensor<1xf32>
 // CHECK: return %[[VAL_0]] : tensor<1xf32>
 func.func @test_reshape_0d_up_s2s_auto(%arg0: tensor<f32>) -> tensor<1xf32> {
-  %0 = "tosa.reshape"(%arg0) {new_shape = array<i64: -1>} : (tensor<f32>) -> tensor<1xf32>
+  %s = tosa.const_shape { value = dense<-1> : tensor<1xindex> } : () -> !tosa.shape<1>
+  %0 = "tosa.reshape"(%arg0, %s) : (tensor<f32>, !tosa.shape<1>) -> tensor<1xf32>
   return %0 : tensor<1xf32>
 }
 
@@ -52,7 +56,8 @@ func.func @test_reshape_0d_up_s2s_auto(%arg0: tensor<f32>) -> tensor<1xf32> {
 // CHECK: %[[VAL_0:.*]] = tensor.expand_shape %[[ARG_0]] [] output_shape [1] : tensor<f32> into tensor<1xf32>
 // CHECK: return %[[VAL_0]] : tensor<1xf32>
 func.func @test_reshape_0d_up_s2s_explicit(%arg0: tensor<f32>) -> tensor<1xf32> {
-  %0 = "tosa.reshape"(%arg0) {new_shape = array<i64: 1>} : (tensor<f32>) -> tensor<1xf32>
+  %s = tosa.const_shape { value = dense<1> : tensor<1xindex> } : () -> !tosa.shape<1>
+  %0 = "tosa.reshape"(%arg0, %s) : (tensor<f32>, !tosa.shape<1>) -> tensor<1xf32>
   return %0 : tensor<1xf32>
 }
 
@@ -64,7 +69,8 @@ func.func @test_reshape_0d_up_s2s_explicit(%arg0: tensor<f32>) -> tensor<1xf32>
 // CHECK: %[[VAL_1:.*]] = tensor.collapse_shape %[[VAL_0]] [] : tensor<1xf32> into tensor<f32>
 // CHECK: return %[[VAL_1]] : tensor<f32>
 func.func @test_reshape_1d_down_d2s_explicit(%arg0: tensor<?xf32>) -> tensor<f32> {
-  %0 = "tosa.reshape"(%arg0) {new_shape = array<i64>} : (tensor<?xf32>) -> tensor<f32>
+  %s = tosa.const_shape { value = dense<> : tensor<0xindex> } : () -> !tosa.shape<0>
+  %0 = "tosa.reshape"(%arg0, %s) : (tensor<?xf32>, !tosa.shape<0>) -> tensor<f32>
   return %0 : tensor<f32>
 }
 
@@ -75,7 +81,8 @@ func.func @test_reshape_1d_down_d2s_explicit(%arg0: tensor<?xf32>) -> tensor<f32
 // CHECK: %[[VAL_0:.*]] = tensor.collapse_shape %[[ARG_0]] [] : tensor<1xf32> into tensor<f32>
 // CHECK: return %[[VAL_0]] : tensor<f32>
 func.func @test_reshape_1d_down_s2s_explicit(%arg0: tensor<1xf32>) -> tensor<f32> {
-  %0 = "tosa.reshape"(%arg0) {new_shape = array<i64>} : (tensor<1xf32>) -> tensor<f32>
+  %s = tosa.const_shape { value = dense<> : tensor<0xindex> } : () -> !tosa.shape<0>
+  %0 = "tosa.reshape"(%arg0, %s) : (tensor<1xf32>, !tosa.shape<0>) -> tensor<f32>
   return %0 : tensor<f32>
 }
 
@@ -86,11 +93,12 @@ func.func @test_reshape_1d_down_s2s_explicit(%arg0: tensor<1xf32>) -> tensor<f32
 // CHECK: %[[C0:.*]] = arith.constant 0 : index
 // CHECK: %[[DIM:.*]] = tensor.dim %arg0, %[[C0]] : tensor<?xf32>
 // CHECK: %[[C2:.*]] = arith.constant 2 : index
-// CHECK: %[[VAL_0:.*]] = arith.divui %[[DIM]], %[[C2]] : index
+// CHECK: %[[VAL_0:.*]] = arith.divsi %[[DIM]], %[[C2]] : index
 // CHECK: %[[EXPANDED:.*]] = tensor.expand_shape %[[ARG_0]] {{\[\[}}0, 1]] output_shape [2, %[[VAL_0]]] : tensor<?xf32> into tensor<2x?xf32>
 // CHECK: return %[[EXPANDED]] : tensor<2x?xf32>
 func.func @test_reshape_1d_up_d2d_auto(%arg0: tensor<?xf32>) -> tensor<2x?xf32> {
-  %0 = "tosa.reshape"(%arg0) {new_shape = array<i64: 2, -1>} : (tensor<?xf32>) -> tensor<2x?xf32>
+  %s = tosa.const_shape { value = dense<[2, -1]> : tensor<2xindex> } : () -> !tosa.shape<2>
+  %0 = "tosa.reshape"(%arg0, %s) : (tensor<?xf32>, !tosa.shape<2>) -> tensor<2x?xf32>
   return %0 : tensor<2x?xf32>
 }
 
@@ -101,7 +109,8 @@ func.func @test_reshape_1d_up_d2d_auto(%arg0: tensor<?xf32>) -> tensor<2x?xf32>
 // CHECK: %[[VAL_0:.*]] = tensor.expand_shape %[[ARG_0]] {{\[\[}}0, 1]] output_shape [2, 3] : tensor<6xf32> into tensor<2x3xf32>
 // CHECK: return %[[VAL_0]] : tensor<2x3xf32>
 func.func @test_reshape_1d_up_s2s_explicit(%arg0: tensor<6xf32>) -> tensor<2x3xf32> {
-  %0 = "tosa.reshape"(%arg0) {new_shape = array<i64: 2, 3>} : (tensor<6xf32>) -> tensor<2x3xf32>
+  %s = tosa.const_shape { value = dense<[2, 3]> : tensor<2xindex> } : () -> !tosa.shape<2>
+  %0 = "tosa.reshape"(%arg0, %s) : (tensor<6xf32>, !tosa.shape<2>) -> tensor<2x3xf32>
   return %0 : tensor<2x3xf32>
 }
 
@@ -112,7 +121,8 @@ func.func @test_reshape_1d_up_s2s_explicit(%arg0: tensor<6xf32>) -> tensor<2x3xf
 // CHECK: %[[VAL_0:.*]] = tensor.collapse_shape %[[ARG_0]] {{\[\[}}0, 1]] : tensor<2x?xf32> into tensor<?xf32>
 // CHECK: return %[[VAL_0]] : tensor<?xf32>
 func.func @test_reshape_2d_down_d2d_auto(%arg0: tensor<2x?xf32>) -> tensor<?xf32> {
-  %0 = "tosa.reshape"(%arg0) {new_shape = array<i64: -1>} : (tensor<2x?xf32>) -> tensor<?xf32>
+  %s = tosa.const_shape { value = dense<-1> : tensor<1xindex> } : () -> !tosa.shape<1>
+  %0 = "tosa.reshape"(%arg0, %s) : (tensor<2x?xf32>, !tosa.shape<1>) -> tensor<?xf32>
   return %0 : tensor<?xf32>
 }
 
@@ -123,7 +133,8 @@ func.func @test_reshape_2d_down_d2d_auto(%arg0: tensor<2x?xf32>) -> tensor<?xf32
 // CHECK: %[[VAL_0:.*]] = tensor.collapse_shape %[[ARG_0]] {{\[\[}}0, 1]] : tensor<2x3xf32> into tensor<6xf32>
 // CHECK: return %[[VAL_0]] : tensor<6xf32>
 func.func @test_reshape_2d_down_s2s_explicit(%arg0: tensor<2x3xf32>) -> tensor<6xf32> {
-  %0 = "tosa.reshape"(%arg0) {new_shape = array<i64: 6>} : (tensor<2x3xf32>) -> tensor<6xf32>
+  %s = tosa.const_shape { value = dense<6> : tensor<1xindex> } : () -> !tosa.shape<1>
+  %0 = "tosa.reshape"(%arg0, %s) : (tensor<2x3xf32>, !tosa.shape<1>) -> tensor<6xf32>
   return %0 : tensor<6xf32>
 }
 
@@ -135,11 +146,12 @@ func.func @test_reshape_2d_down_s2s_explicit(%arg0: tensor<2x3xf32>) -> tensor<6
 // CHECK: %[[C0:.*]] = arith.constant 0 : index
 // CHECK: %[[DIM:.*]] = tensor.dim %[[VAL_0]], %[[C0]] : tensor<?xf32>
 // CHECK: %[[C2:.*]] = arith.constant 2 : index
-// CHECK: %[[DIV:.*]] = arith.divui %[[DIM]], %[[C2]] : index
+// CHECK: %[[DIV:.*]] = arith.divsi %[[DIM]], %[[C2]] : index
 // CHECK: %[[EXPANDED:.*]] = tensor.expand_shape %[[VAL_0]] {{\[\[}}0, 1]] output_shape [2, %[[DIV]]] : tensor<?xf32> into tensor<2x?xf32>
 // CHECK: return %[[EXPANDED]] : tensor<2x?xf32>
 func.func @test_reshape_2d_same_d2d_auto(%arg0: tensor<?x2xf32>) -> tensor<2x?xf32> {
-  %0 = "tosa.reshape"(%arg0) {new_shape = array<i64: 2, -1>} : (tensor<?x2xf32>) -> tensor<2x?xf32>
+  %s = tosa.const_shape { value = dense<[2, -1]> : tensor<2xindex> } : () -> !tosa.shape<2>
+  %0 = "tosa.reshape"(%arg0, %s) : (tensor<?x2xf32>, !tosa.shape<2>) -> tensor<2x?xf32>
   return %0 : tensor<2x?xf32>
 }
 
@@ -152,10 +164,12 @@ func.func @test_reshape_2d_same_d2d_auto(%arg0: tensor<?x2xf32>) -> tensor<2x?xf
 // CHECK: %[[VAL_2:.*]] = tensor.cast %[[VAL_1]] : tensor<4x2xf32> to tensor<?x2xf32>
 // CHECK: return %[[VAL_2]] : tensor<?x2xf32>
 func.func @test_reshape_2d_same_s2d_auto(%arg0: tensor<2x4xf32>) -> tensor<?x2xf32> {
-  %0 = "tosa.reshape"(%arg0) {new_shape = array<i64: -1, 2>} : (tensor<2x4xf32>) -> tensor<?x2xf32>
+  %s = tosa.const_shape { value = dense<[-1, 2]> : tensor<2xindex> } : () -> !tosa.shape<2>
+  %0 = "tosa.reshape"(%arg0, %s) : (tensor<2x4xf32>, !tosa.shape<2>) -> tensor<?x2xf32>
   return %0 : tensor<?x2xf32>
 }
 
+
 // -----
 
 // CHECK-LABEL: test_reshape_2d_same_s2d_explicit
@@ -165,7 +179,8 @@ func.func @test_reshape_2d_same_s2d_auto(%arg0: tensor<2x4xf32>) -> tensor<?x2xf
 // CHECK: %[[VAL_2:.*]] = tensor.cast %[[VAL_1]] : tensor<4x2xf32> to tensor<?x2xf32>
 // CHECK: return %[[VAL_2]] : tensor<?x2xf32>
 func.func @test_reshape_2d_same_s2d_explicit(%arg0: tensor<2x4xf32>) -> tensor<?x2xf32> {
-  %0 = "tosa.reshape"(%arg0) {new_shape = array<i64: 4, 2>} : (tensor<2x4xf32>) -> tensor<?x2xf32>
+  %s = tosa.const_shape { value = dense<[4, 2]> : tensor<2xindex> } : () -> !tosa.shape<2>
+  %0 = "tosa.reshape"(%arg0, %s) : (tensor<2x4xf32>, !tosa.shape<2>) -> tensor<?x2xf32>
   return %0 : tensor<?x2xf32>
 }
 
@@ -177,7 +192,8 @@ func.func @test_reshape_2d_same_s2d_explicit(%arg0: tensor<2x4xf32>) -> tensor<?
 // CHECK: %[[VAL_1:.*]] = tensor.expand_shape %[[VAL_0]] {{\[\[}}0, 1]] output_shape [2, 3] : tensor<6xf32> into tensor<2x3xf32>
 // CHECK: return %[[VAL_1]] : tensor<2x3xf32>
 func.func @test_reshape_2d_same_s2s_explicit(%arg0: tensor<3x2xf32>) -> tensor<2x3xf32> {
-  %0 = "tosa.reshape"(%arg0) {new_shape = array<i64: 2, 3>} : (tensor<3x2xf32>) -> tensor<2x3xf32>
+  %s = tosa.const_shape { value = dense<[2, 3]> : tensor<2xindex> } : () -> !tosa.shape<2>
+  %0 = "tosa.reshape"(%arg0, %s) : (tensor<3x2xf32>, !tosa.shape<2>) -> tensor<2x3xf32>
   return %0 : tensor<2x3xf32>
 }
 
@@ -189,12 +205,13 @@ func.func @test_reshape_2d_same_s2s_explicit(%arg0: tensor<3x2xf32>) -> tensor<2
 // CHECK: %[[C0:.*]] = arith.constant 0 : index
 // CHECK: %[[DIM:.*]] = tensor.dim %[[VAL_0]], %[[C0]] : tensor<?xf32>
 // CHECK: %[[C0_0:.*]] = arith.constant 0 : index
-// CHECK: %[[DIV:.*]] = arith.divui %[[DIM]], %[[C0_0]] : index
+// CHECK: %[[DIV:.*]] = arith.divsi %[[DIM]], %[[C0_0]] : index
 // CHECK: %[[VAL_1:.*]] = tensor.expand_shape %[[VAL_0]] {{\[\[}}0, 1, 2]] output_shape [0, 3, %[[DIV]]] : tensor<?xf32> into tensor<0x3x?xf32>
 // CHECK: %[[VAL_2:.*]] = tensor.cast %[[VAL_1]] : tensor<0x3x?xf32> to tensor<?x?x?xf32>
 // CHECK: return %[[VAL_2]] : tensor<?x?x?xf32>
 func.func @test_reshape_3d_same_d2d_auto_empty(%arg0: tensor<3x2x?xf32>) -> tensor<?x?x?xf32> {
-  %0 = "tosa.reshape"(%arg0) {new_shape = array<i64: 0, 3, -1>} : (tensor<3x2x?xf32>) -> tensor<?x?x?xf32>
+  %s = tosa.const_shape { value = dense<[0, 3, -1]> : tensor<3xindex> } : () -> !tosa.shape<3>
+  %0 = "tosa.reshape"(%arg0, %s) : (tensor<3x2x?xf32>, !tosa.shape<3>) -> tensor<?x?x?xf32>
   return %0 : tensor<?x?x?xf32>
 }
 
@@ -206,12 +223,13 @@ func.func @test_reshape_3d_same_d2d_auto_empty(%arg0: tensor<3x2x?xf32>) -> tens
 // CHECK: %[[C0:.*]] = arith.constant 0 : index
 // CHECK: %[[DIM:.*]] = tensor.dim %[[VAL_0]], %[[C0]] : tensor<?xf32>
 // CHECK: %[[C8:.*]] = arith.constant 8 : index
-// CHECK: %[[DIV:.*]] = arith.divui %[[DIM]], %[[C8]] : index
+// CHECK: %[[DIV:.*]] = arith.divsi %[[DIM]], %[[C8]] : index
 // CHECK: %[[VAL_1:.*]] = tensor.expand_shape %[[VAL_0]] {{\[\[}}0, 1, 2]] output_shape [2, %[[DIV]], 4] : tensor<?xf32> into tensor<2x?x4xf32>
 // CHECK: %[[VAL_2:.*]] = tensor.cast %[[VAL_1]] : tensor<2x?x4xf32> to tensor<?x?x?xf32>
 // CHECK: return %[[VAL_2]] : tensor<?x?x?xf32>
 func.func @test_reshape_3d_same_d2d_auto(%arg0: tensor<2x?x?xf32>) -> tensor<?x?x?xf32> {
-  %0 = "tosa.reshape"(%arg0) {new_shape = array<i64: 2, -1, 4>} : (tensor<2x?x?xf32>) -> tensor<?x?x?xf32>
+  %s = tosa.const_shape { value = dense<[2, -1, 4]> : tensor<3xindex> } : () -> !tosa.shape<3>
+  %0 = "tosa.reshape"(%arg0, %s) : (tensor<2x?x?xf32>, !tosa.shape<3>) -> tensor<?x?x?xf32>
   return %0 : tensor<?x?x?xf32>
 }
 
@@ -223,11 +241,12 @@ func.func @test_reshape_3d_same_d2d_auto(%arg0: tensor<2x?x?xf32>) -> tensor<?x?
 // CHECK: %[[C0:.*]] = arith.constant 0 : index
 // CHECK: %[[DIM:.*]] = tensor.dim %[[VAL_0]], %[[C0]] : tensor<?xf32>
 // CHECK: %[[C6:.*]] = arith.constant 6 : index
-// CHECK: %[[DIV:.*]] = arith.divui %[[DIM]], %[[C6]] : index
+// CHECK: %[[DIV:.*]] = arith.divsi %[[DIM]], %[[C6]] : index
 // CHECK: %[[VAL_1:.*]] = tensor.expand_shape %[[VAL_0]] {{\[\[}}0, 1, 2]] output_shape [2, 3, %[[DIV]]] : tensor<?xf32> into tensor<2x3x?xf32>
 // CHECK: return %[[VAL_1]] : tensor<2x3x?xf32>
 func.func @test_reshape_3d_same_d2d_auto_identity(%arg0: tensor<?x3x4xf32>) -> tensor<2x3x?xf32> {
-  %0 = "tosa.reshape"(%arg0) {new_shape = array<i64: 2, 3, -1>} : (tensor<?x3x4xf32>) -> tensor<2x3x?xf32>
+  %s = tosa.const_shape { value = dense<[2, 3, -1]> : tensor<3xindex> } : () -> !tosa.shape<3>
+  %0 = "tosa.reshape"(%arg0, %s) : (tensor<?x3x4xf32>, !tosa.shape<3>) -> tensor<2x3x?xf32>
   return %0 : tensor<2x3x?xf32>
 }
 
@@ -239,12 +258,13 @@ func.func @test_reshape_3d_same_d2d_auto_identity(%arg0: tensor<?x3x4xf32>) -> t
 // CHECK: %[[C0:.*]] = arith.constant 0 : index
 // CHECK: %[[DIM:.*]] = tensor.dim %[[VAL_0]], %[[C0]] : tensor<?xf32>
 // CHECK: %[[C6:.*]] = arith.constant 6 : index
-// CHECK: %[[DIV:.*]] = arith.divui %[[DIM]], %[[C6]] : index
+// CHECK: %[[DIV:.*]] = arith.divsi %[[DIM]], %[[C6]] : index
 // CHECK: %[[EXPANDED:.*]] = tensor.expand_shape %[[VAL_0]] {{\[\[}}0, 1, 2]] output_shape [%[[DIV]], 3, 2] : tensor<?xf32> into tensor<?x3x2xf32>
 // CHECK: %[[VAL_2:.*]] = tensor.cast %[[EXPANDED]] : tensor<?x3x2xf32> to tensor<?x?x?xf32>
 // CHECK: return %[[VAL_2]] : tensor<?x?x?xf32>
 func.func @test_reshape_3d_same_d2d_explicit_empty(%arg0: tensor<3x2x?xf32>) -> tensor<?x?x?xf32> {
-  %0 = "tosa.reshape"(%arg0) {new_shape = array<i64: 0, 3, 2>} : (tensor<3x2x?xf32>) -> tensor<?x?x?xf32>
+  %s = tosa.const_shape { value = dense<[0, 3, 2]> : tensor<3xindex> } : () -> !tosa.shape<3>
+  %0 = "tosa.reshape"(%arg0, %s) : (tensor<3x2x?xf32>, !tosa.shape<3>) -> tensor<?x?x?xf32>
   return %0 : tensor<?x?x?xf32>
 }
 
@@ -256,12 +276,13 @@ func.func @test_reshape_3d_same_d2d_explicit_empty(%arg0: tensor<3x2x?xf32>) ->
 // CHECK: %[[C0:.*]] = arith.constant 0 : index
 // CHECK: %[[DIM:.*]] = tensor.dim %[[VAL_0]], %[[C0]] : tensor<?xf32>
 // CHECK: %[[C12:.*]] = arith.constant 12 : index
-// CHECK: %[[DIV:.*]] = arith.divui %[[DIM]], %[[C12]] : index
+// CHECK: %[[DIV:.*]] = arith.divsi %[[DIM]], %[[C12]] : index
 // CHECK: %[[EXPANDED:.*]] = tensor.expand_shape %[[VAL_0]] {{\[\[}}0, 1, 2]] output_shape [%[[DIV]], 3, 4] : tensor<?xf32> into tensor<?x3x4xf32>
 // CHECK: %[[VAL_2:.*]] = tensor.cast %[[EXPANDED]] : tensor<?x3x4xf32> to tensor<?x?x?xf32>
 // CHECK: return %[[VAL_2]] : tensor<?x?x?xf32>
 func.func @test_reshape_3d_same_d2d_explicit(%arg0: tensor<?x?x?xf32>) -> tensor<?x?x?xf32> {
-  %0 = "tosa.reshape"(%arg0) {new_shape = array<i64: 2, 3, 4>} : (tensor<?x?x?xf32>) -> tensor<?x?x?xf32>
+  %s = tosa.const_shape { value = dense<[2, 3, 4]> : tensor<3xindex> } : () -> !tosa.shape<3>
+  %0 = "tosa.reshape"(%arg0, %s) : (tensor<?x?x?xf32>, !tosa.shape<3>) -> tensor<?x?x?xf32>
   return %0 : tensor<?x?x?xf32>
 }
 
@@ -272,7 +293,8 @@ func.func @test_reshape_3d_same_d2d_explicit(%arg0: tensor<?x?x?xf32>) -> tensor
 // CHECK: %[[VAL_0:.*]] = tensor.cast %[[ARG_0]] : tensor<?x3x4xf32> to tensor<2x3x?xf32>
 // CHECK: return %[[VAL_0]] : tensor<2x3x?xf32>
 func.func @test_reshape_3d_same_d2d_explicit_identity(%arg0: tensor<?x3x4xf32>) -> tensor<2x3x?xf32> {
-  %0 = "tosa.reshape"(%arg0) {new_shape = array<i64: 2, 3, 4>} : (tensor<?x3x4xf32>) -> tensor<2x3x?xf32>
+  %s = tosa.const_shape { value = dense<[2, 3, 4]> : tensor<3xindex> } : () -> !tosa.shape<3>
+  %0 = "tosa.reshape"(%arg0, %s) : (tensor<?x3x4xf32>, !tosa.shape<3>) -> tensor<2x3x?xf32>
   return %0 : tensor<2x3x?xf32>
 }
 
@@ -284,12 +306,13 @@ func.func @test_reshape_3d_same_d2d_explicit_identity(%arg0: tensor<?x3x4xf32>)
 // CHECK: %[[C0:.*]] = arith.constant 0 : index
 // CHECK: %[[DIM:.*]] = tensor.dim %[[VAL_0]], %[[C0]] : tensor<?xf32>
 // CHECK: %[[C8:.*]] = arith.constant 8 : index
-// CHECK: %[[DIV:.*]] = arith.divui %[[DIM]], %[[C8]] : index
+// CHECK: %[[DIV:.*]] = arith.divsi %[[DIM]], %[[C8]] : index
 // CHECK: %[[EXPANDED:.*]] = tensor.expand_shape %[[VAL_0]] {{\[\[}}0, 1, 2]] output_shape [2, %[[DIV]], 4] : tensor<?xf32> into tensor<2x?x4xf32>
 // CHECK: %[[VAL_2:.*]] = tensor.cast %[[EXPANDED]] : tensor<2x?x4xf32> to tensor<2x3x4xf32>
 // CHECK: return %[[VAL_2]] : tensor<2x3x4xf32>
 func.func @test_reshape_3d_same_d2s_auto(%arg0: tensor<?x?x?xf32>) -> tensor<2x3x4xf32> {
-  %0 = "tosa.reshape"(%arg0) {new_shape = array<i64: 2, -1, 4>} : (tensor<?x?x?xf32>) -> tensor<2x3x4xf32>
+  %s = tosa.const_shape { value = dense<[2, -1, 4]> : tensor<3xindex> } : () -> !tosa.shape<3>
+  %0 = "tosa.reshape"(%arg0, %s) : (tensor<?x?x?xf32>, !tosa.shape<3>) -> tensor<2x3x4xf32>
   return %0 : tensor<2x3x4xf32>
 }
 
@@ -301,12 +324,13 @@ func.func @test_reshape_3d_same_d2s_auto(%arg0: tensor<?x?x?xf32>) -> tensor<2x3
 // CHECK: %[[C0:.*]] = arith.constant 0 : index
 // CHECK: %[[DIM:.*]] = tensor.dim %[[VAL_0]], %[[C0]] : tensor<?xf32>
 // CHECK: %[[C12:.*]] = arith.constant 12 : index
-// CHECK: %[[DIV:.*]] = arith.divui %[[DIM]], %[[C12]] : index
+// CHECK: %[[DIV:.*]] = arith.divsi %[[DIM]], %[[C12]] : index
 // CHECK: %[[EXPANDED:.*]] = tensor.expand_shape %[[VAL_0]] {{\[\[}}0, 1, 2]] output_shape [%[[DIV]], 3, 4] : tensor<?xf32> into tensor<?x3x4xf32>
 // CHECK: %[[VAL_2:.*]] = tensor.cast %[[EXPANDED]] : tensor<?x3x4xf32> to tensor<2x3x4xf32>
 // CHECK: return %[[VAL_2]] : tensor<2x3x4xf32>
 func.func @test_reshape_3d_same_d2s_explicit(%arg0: tensor<?x?x?xf32>) -> tensor<2x3x4xf32> {
-  %0 = "tosa.reshape"(%arg0) {new_shape = array<i64: 2, 3, 4>} : (tensor<?x?x?xf32>) -> tensor<2x3x4xf32>
+  %s = tosa.const_shape { value = dense<[2, 3, 4]> : tensor<3xindex> } : () -> !tosa.shape<3>
+  %0 = "tosa.reshape"(%arg0, %s) : (tensor<?x?x?xf32>, !tosa.shape<3>) -> tensor<2x3x4xf32>
   return %0 : tensor<2x3x4xf32>
 }
 
@@ -316,7 +340,8 @@ func.func @test_reshape_3d_same_d2s_explicit(%arg0: tensor<?x?x?xf32>) -> tensor
 // CHECK-SAME: %[[ARG_0:[a-zA-Z0-9_]+]]: tensor<2x3x4xf32>
 // CHECK: return %[[ARG_0]] : tensor<2x3x4xf32>
 func.func @test_reshape_3d_same_s2s_explicit_identity(%arg0: tensor<2x3x4xf32>) -> tensor<2x3x4xf32> {
-  %0 = "tosa.reshape"(%arg0) {new_shape = array<i64: 2, 3, 4>} : (tensor<2x3x4xf32>) -> tensor<2x3x4xf32>
+  %s = tosa.const_shape { value = dense<[2, 3, 4]> : tensor<3xindex> } : () -> !tosa.shape<3>
+  %0 = "tosa.reshape"(%arg0, %s) : (tensor<2x3x4xf32>, !tosa.shape<3>) -> tensor<2x3x4xf32>
   return %0 : tensor<2x3x4xf32>
 }
 
@@ -328,12 +353,13 @@ func.func @test_reshape_3d_same_s2s_explicit_identity(%arg0: tensor<2x3x4xf32>)
 // CHECK: %[[C0:.*]] = arith.constant 0 : index
 // CHECK: %[[DIM:.*]] = tensor.dim %[[COLLAPSED]], %[[C0]] : tensor<?xf32>
 // CHECK: %[[C6:.*]] = arith.constant 6 : index
-// CHECK: %[[VAL_0:.*]] = arith.divui %[[DIM]], %[[C6]] : index
+// CHECK: %[[VAL_0:.*]] = arith.divsi %[[DIM]], %[[C6]] : index
 // CHECK: %[[EXPANDED:.*]] = tensor.expand_shape %[[COLLAPSED]] {{\[\[}}0, 1, 2, 3]] output_shape [%[[VAL_0]], 3, 2, 1] : tensor<?xf32> into tensor<?x3x2x1xf32>
 // CHECK: %[[CAST:.*]] = tensor.cast %[[EXPANDED]] : tensor<?x3x2x1xf32> to tensor<1x3x2x1xf32>
 // CHECK: return %[[CAST]] : tensor<1x3x2x1xf32>
 func.func @test_reshape_3d_up_d2s_explicit(%input: tensor<?x?x?xf32>) -> tensor<1x3x2x1xf32> {
-  %0 = tosa.reshape %input {new_shape = array<i64: 1, 3, 2, 1>} : (tensor<?x?x?xf32>) -> tensor<1x3x2x1xf32>
+  %s = tosa.const_shape { value = dense<[1, 3, 2, 1]> : tensor<4xindex> } : () -> !tosa.shape<4>
+  %0 = tosa.reshape %input, %s : (tensor<?x?x?xf32>, !tosa.shape<4>) -> tensor<1x3x2x1xf32>
   return %0 : tensor<1x3x2x1xf32>
 }
 
@@ -345,7 +371,8 @@ func.func @test_reshape_3d_up_d2s_explicit(%input: tensor<?x?x?xf32>) -> tensor<
 // CHECK: %[[VAL_1:.*]] = tensor.collapse_shape %[[VAL_0]] [] : tensor<1x1x1x1xf32> into tensor<f32>
 // CHECK: return %[[VAL_1]] : tensor<f32>
 func.func @test_reshape_4d_down_d2s_explicit(%arg0: tensor<?x?x?x?xf32>) -> tensor<f32> {
-  %0 = "tosa.reshape"(%arg0) {new_shape = array<i64>} : (tensor<?x?x?x?xf32>) -> tensor<f32>
+  %s = tosa.const_shape { value = dense<> : tensor<0xindex> } : () -> !tosa.shape<0>
+  %0 = "tosa.reshape"(%arg0, %s) : (tensor<?x?x?x?xf32>, !tosa.shape<0>) -> tensor<f32>
   return %0 : tensor<f32>
 }
 
@@ -357,11 +384,12 @@ func.func @test_reshape_4d_down_d2s_explicit(%arg0: tensor<?x?x?x?xf32>) -> tens
 // CHECK: %[[C0:.*]] = arith.constant 0 : index
 // CHECK: %[[DIM:.*]] = tensor.dim %[[COLLAPSED]], %[[C0]] : tensor<?xf32>
 // CHECK: %[[C6:.*]] = arith.constant 6 : index
-// CHECK: %[[VAL_0:.*]] = arith.divui %[[DIM]], %[[C6]] : index
+// CHECK: %[[VAL_0:.*]] = arith.divsi %[[DIM]], %[[C6]] : index
 // CHECK: %[[EXPANDED:.*]] = tensor.expand_shape %[[COLLAPSED]] {{\[\[}}0, 1, 2]] output_shape [%[[VAL_0]], 2, 3] : tensor<?xf32> into tensor<?x2x3xf32>
 // CHECK: return %[[EXPANDED]] : tensor<?x2x3xf32>
 func.func @test_reshape_5d_down_d2d_auto(%arg0: tensor<?x?x?x2x3xf32>) -> tensor<?x2x3xf32> {
-  %0 = "tosa.reshape"(%arg0) {new_shape = array<i64: -1, 2, 3>} : (tensor<?x?x?x2x3xf32>) -> tensor<?x2x3xf32>
+  %s = tosa.const_shape { value = dense<[-1, 2, 3]> : tensor<3xindex> } : () -> !tosa.shape<3>
+  %0 = "tosa.reshape"(%arg0, %s) : (tensor<?x?x?x2x3xf32>, !tosa.shape<3>) -> tensor<?x2x3xf32>
   return %0 : tensor<?x2x3xf32>
 }
 
@@ -373,11 +401,12 @@ func.func @test_reshape_5d_down_d2d_auto(%arg0: tensor<?x?x?x2x3xf32>) -> tensor
 // CHECK: %[[C0:.*]] = arith.constant 0 : index
 // CHECK: %[[DIM:.*]] = tensor.dim %[[COLLAPSED]], %[[C0]] : tensor<?xf32>
 // CHECK: %[[C385:.*]] = arith.constant 385 : index
-// CHECK: %[[VAL_0:.*]] = arith.divui %[[DIM]], %[[C385]] : index
+// CHECK: %[[VAL_0:.*]] = arith.divsi %[[DIM]], %[[C385]] : index
 // CHECK: %[[EXPANDED:.*]] = tensor.expand_shape %[[COLLAPSED]] {{\[\[}}0, 1, 2]] output_shape [%[[VAL_0]], 5, 77] : tensor<?xf32> into tensor<?x5x77xf32>
 // CHECK: return %[[EXPANDED]] : tensor<?x5x77xf32>
 func.func @test_reshape_6d_down_d2d_auto(%arg0: tensor<1x2x?x5x7x11xf32>) -> tensor<?x5x77xf32> {
-  %0 = "tosa.reshape"(%arg0) {new_shape = array<i64: -1, 5, 77>} : (tensor<1x2x?x5x7x11xf32>) -> tensor<?x5x77xf32>
+  %s = tosa.const_shape { value = dense<[-1, 5, 77]> : tensor<3xindex> } : () -> !tosa.shape<3>
+  %0 = "tosa.reshape"(%arg0, %s) : (tensor<1x2x?x5x7x11xf32>, !tosa.shape<3>) -> tensor<?x5x77xf32>
   return %0 : tensor<?x5x77xf32>
 }
 
@@ -388,7 +417,8 @@ func.func @test_reshape_6d_down_d2d_auto(%arg0: tensor<1x2x?x5x7x11xf32>) -> ten
 // CHECK: %[[VAL_0:.*]] = tensor.collapse_shape %[[ARG_0]] {{\[\[}}0, 1, 2], [3], [4, 5]] : tensor<1x2x3x5x7x11xf32> into tensor<6x5x77xf32>
 // CHECK: return %[[VAL_0]] : tensor<6x5x77xf32>
 func.func @test_reshape_6d_down_s2s_auto(%arg0: tensor<1x2x3x5x7x11xf32>) -> tensor<6x5x77xf32> {
-  %0 = "tosa.reshape"(%arg0) {new_shape = array<i64: 6, 5, -1>} : (tensor<1x2x3x5x7x11xf32>) -> tensor<6x5x77xf32>
+  %s = tosa.const_shape { value = dense<[6, 5, -1]> : tensor<3xindex> } : () -> !tosa.shape<3>
+  %0 = "tosa.reshape"(%arg0, %s) : (tensor<1x2x3x5x7x11xf32>, !tosa.shape<3>) -> tensor<6x5x77xf32>
   return %0 : tensor<6x5x77xf32>
 }
 
@@ -400,10 +430,13 @@ func.func @test_reshape_6d_down_s2s_auto(%arg0: tensor<1x2x3x5x7x11xf32>) -> ten
 //
 // See https://github.com/llvm/llvm-project/pull/91521 for a full description.
 
+// -----
+
 // CHECK-LABEL: reshape_bug_fix
 // CHECK: tensor.expand_shape
 func.func @reshape_bug_fix(%arg0: tensor<?xf32>) -> tensor<1x1x1x?xf32> {
-  %0 = tosa.reshape %arg0 {new_shape = array<i64: 1, 1, 1, -1>} : (tensor<?xf32>) -> tensor<1x1x1x?xf32>
+  %1 = "tosa.const_shape"() {value = dense<[1, 1, 1, -1]> : tensor<4xindex>} : () -> !tosa.shape<4>
+  %0 = "tosa.reshape"(%arg0, %1) : (tensor<?xf32>, !tosa.shape<4>) -> tensor<1x1x1x?xf32>
   return %0 : tensor<1x1x1x?xf32>
 }
 
@@ -414,21 +447,22 @@ func.func @reshape_bug_fix(%arg0: tensor<?xf32>) -> tensor<1x1x1x?xf32> {
 // CHECK: %[[VAL_0:.*]] = tensor.collapse_shape %[[ARG_0]] {{\[\[}}0, 1, 2], [3], [4, 5]] : tensor<1x2x3x5x7x11xf32> into tensor<6x5x77xf32>
 // CHECK: return %[[VAL_0]] : tensor<6x5x77xf32>
 func.func @test_reshape_6d_down_s2s_explicit(%arg0: tensor<1x2x3x5x7x11xf32>) -> tensor<6x5x77xf32> {
-  %0 = "tosa.reshape"(%arg0) {new_shape = array<i64: 6, 5, 77>} : (tensor<1x2x3x5x7x11xf32>) -> tensor<6x5x77xf32>
+  %s = tosa.const_shape { value = dense<[6, 5, 77]> : tensor<3xindex> } : () -> !tosa.shape<3>
+  %0 = "tosa.reshape"(%arg0, %s) : (tensor<1x2x3x5x7x11xf32>, !tosa.shape<3>) -> tensor<6x5x77xf32>
   return %0 : tensor<6x5x77xf32>
 }
 
 // -----
 
 // CHECK-LABEL: @test_reshape_samerank_unsigned
-//  CHECK-SAME: (%[[ARG0:.*]]: tensor<3x2xui8>)
+// CHECK-SAME: (%[[VAL_0:.*]]: tensor<3x2xui8>)
 func.func @test_reshape_samerank_unsigned(%arg0: tensor<3x2xui8>) -> tensor<2x3xui8> {
-  // CHECK-NEXT: %[[CAST1:.*]] = builtin.unrealized_conversion_cast %[[ARG0]] : tensor<3x2xui8> to tensor<3x2xi8>
-  // CHECK-NEXT: %[[RESHAPE1:.*]] = tensor.collapse_shape %[[CAST1]] {{\[}}[0, 1]] : tensor<3x2xi8> into tensor<6xi8>
-  // CHECK-NEXT: %[[RESHAPE2:.*]] = tensor.expand_shape %[[RESHAPE1]] {{\[}}[0, 1]] output_shape {{\[}}2, 3] : tensor<6xi8> into tensor<2x3xi8>
-  // CHECK-NEXT: %[[CAST2:.*]] = builtin.unrealized_conversion_cast %[[RESHAPE2]] : tensor<2x3xi8> to tensor<2x3xui8
-  %0 = "tosa.reshape"(%arg0) {new_shape = array<i64: 2, 3>} : (tensor<3x2xui8>) -> tensor<2x3xui8>
-  // CHECK-NEXT: return %[[CAST2]]
+  // CHECK: %[[CAST1:.*]] = builtin.unrealized_conversion_cast %arg0 : tensor<3x2xui8> to tensor<3x2xi8>
+  // CHECK: %[[RESHAPE1:.*]] = tensor.collapse_shape %[[CAST1]] {{\[}}[0, 1]] : tensor<3x2xi8> into tensor<6xi8>
+  // CHECK: %[[RESHAPE2:.*]] = tensor.expand_shape %[[RESHAPE1]] {{\[}}[0, 1]] output_shape {{\[}}2, 3] : tensor<6xi8> into tensor<2x3xi8>
+  // CHECK: %[[CAST2:.*]] = builtin.unrealized_conversion_cast %[[RESHAPE2]] : tensor<2x3xi8> to tensor<2x3xui8
+  %s = tosa.const_shape { value = dense<[2, 3]> : tensor<2xindex> } : () -> !tosa.shape<2>
+  %0 = "tosa.reshape"(%arg0, %s): (tensor<3x2xui8>, !tosa.shape<2>) -> tensor<2x3xui8>
   return %0 : tensor<2x3xui8>
 }
 
@@ -437,7 +471,9 @@ func.func @test_reshape_samerank_unsigned(%arg0: tensor<3x2xui8>) -> tensor<2x3x
 // CHECK-LABEL: func @slice
 func.func @slice(%arg0: tensor<6xf32>) ->() {
   // CHECK: [[SLICE:%.+]] = tensor.extract_slice %arg0[2] [1] [1]
-  %0 = "tosa.slice"(%arg0) {start = array<i64: 2>, size = array<i64: 1>} : (tensor<6xf32>)  -> (tensor<1xf32>)
+  %0 = tosa.const_shape  {value = dense<2> : tensor<1xindex>} : () -> !tosa.shape<1>
+  %1 = tosa.const_shape  {value = dense<1> : tensor<1xindex>} : () -> !tosa.shape<1>
+  %2 = tosa.slice %arg0, %0, %1 : (tensor<6xf32>, !tosa.shape<1>, !tosa.shape<1>) -> tensor<1xf32>
   return
 }
 
@@ -450,8 +486,10 @@ func.func @slice_dyn(%arg0: tensor<?xf32>) -> (tensor<?xf32>) {
   // CHECK: %[[C2:.+]] = arith.constant 2 : index
   // CHECK: %[[SUB:.+]] = arith.subi %[[DIM]], %[[C2]]
   // CHECK: tensor.extract_slice %arg0[2] [%[[SUB]]] [1]
-  %0 = "tosa.slice"(%arg0) {start = array<i64: 2>, size = array<i64: -1>} : (tensor<?xf32>)  -> (tensor<?xf32>)
-  return %0 : tensor<?xf32>
+  %0 = tosa.const_shape  {value = dense<2> : tensor<1xindex>} : () -> !tosa.shape<1>
+  %1 = tosa.const_shape  {value = dense<-1> : tensor<1xindex>} : () -> !tosa.shape<1>
+  %2 = tosa.slice %arg0, %0, %1 : (tensor<?xf32>, !tosa.shape<1>, !tosa.shape<1>) -> tensor<?xf32>
+  return %2 : tensor<?xf32>
 }
 
 // -----
@@ -488,7 +526,7 @@ func.func @pad_quant(%arg0 : tensor<1x2xi32>) -> (tensor<4x9xi32>) {
   // CHECK: [[CST:%.+]] = arith.constant 42 : i32
   // CHECK: tensor.pad
   // CHECK:   tensor.yield [[CST]]
-  %1 = "tosa.pad"(%arg0, %0) {quantization_info = #tosa.pad_quant<input_zp = 42>} : (tensor<1x2xi32>, !tosa.shape<4>)  -> (tensor<4x9xi32>)
+  %1 = "tosa.pad"(%arg0, %0) {input_zp = 42 : i32} : (tensor<1x2xi32>, !tosa.shape<4>)  -> (tensor<4x9xi32>)
   return %1 : tensor<4x9xi32>
 }
 
diff --git a/mlir/test/Conversion/UBToSPIRV/ub-to-spirv.mlir b/mlir/test/Conversion/UBToSPIRV/ub-to-spirv.mlir
index 771b53ad123b9..f497eb3bc552c 100644
--- a/mlir/test/Conversion/UBToSPIRV/ub-to-spirv.mlir
+++ b/mlir/test/Conversion/UBToSPIRV/ub-to-spirv.mlir
@@ -13,8 +13,7 @@ func.func @check_poison() {
   %1 = ub.poison : i16
 // CHECK: {{.*}} = spirv.Undef : f64
   %2 = ub.poison : f64
-// TODO: vector is not covered yet
-// CHECK: {{.*}} = ub.poison : vector<4xf32>
+// CHECK: {{.*}} = spirv.Undef : vector<4xf32>
   %3 = ub.poison : vector<4xf32>
   return
 }
diff --git a/mlir/test/Conversion/VectorToLLVM/vector-to-llvm-interface.mlir b/mlir/test/Conversion/VectorToLLVM/vector-to-llvm-interface.mlir
new file mode 100644
index 0000000000000..d261327ec005f
--- /dev/null
+++ b/mlir/test/Conversion/VectorToLLVM/vector-to-llvm-interface.mlir
@@ -0,0 +1,2456 @@
+// RUN: mlir-opt --convert-to-llvm="filter-dialects=vector" --split-input-file %s | FileCheck %s
+// RUN: mlir-opt %s -convert-vector-to-llvm -split-input-file | FileCheck %s
+
+//===========================================================================//
+// Basic tests for Vector-to-LLVM conversion
+//
+// These examples are meant to be convertible to LLVM with:
+//  * `populateVectorToLLVMConversionPatterns`,
+// i.e. no other patterns should be required.
+//===========================================================================//
+
+//===----------------------------------------------------------------------===//
+// vector.bitcast
+//===----------------------------------------------------------------------===//
+
+func.func @bitcast_f32_to_i32_vector_0d(%arg0: vector<f32>) -> vector<i32> {
+  %0 = vector.bitcast %arg0 : vector<f32> to vector<i32>
+  return %0 : vector<i32>
+}
+
+// CHECK-LABEL: @bitcast_f32_to_i32_vector_0d
+// CHECK-SAME:  %[[ARG_0:.*]]: vector<f32>
+// CHECK:       %[[VEC_F32_1D:.*]] = builtin.unrealized_conversion_cast %[[ARG_0]] : vector<f32> to vector<1xf32>
+// CHECK:       %[[VEC_I32_1D:.*]] = llvm.bitcast %[[VEC_F32_1D]] : vector<1xf32> to vector<1xi32>
+// CHECK:       %[[VEC_I32_0D:.*]] = builtin.unrealized_conversion_cast %[[VEC_I32_1D]] : vector<1xi32> to vector<i32>
+// CHECK:       return %[[VEC_I32_0D]] : vector<i32>
+
+// -----
+
+func.func @bitcast_f32_to_i32_vector(%arg0: vector<16xf32>) -> vector<16xi32> {
+  %0 = vector.bitcast %arg0 : vector<16xf32> to vector<16xi32>
+  return %0 : vector<16xi32>
+}
+
+
+// CHECK-LABEL: @bitcast_f32_to_i32_vector
+// CHECK-SAME:  %[[ARG_0:.*]]: vector<16xf32>
+// CHECK:       llvm.bitcast %[[ARG_0]] : vector<16xf32> to vector<16xi32>
+
+// -----
+
+func.func @bitcast_f32_to_i32_vector_scalable(%arg0: vector<[16]xf32>) -> vector<[16]xi32> {
+  %0 = vector.bitcast %arg0 : vector<[16]xf32> to vector<[16]xi32>
+  return %0 : vector<[16]xi32>
+}
+
+// CHECK-LABEL: @bitcast_f32_to_i32_vector_scalable
+// CHECK-SAME:  %[[ARG_0:.*]]: vector<[16]xf32>
+// CHECK:       llvm.bitcast %[[ARG_0]] : vector<[16]xf32> to vector<[16]xi32>
+
+// -----
+
+func.func @bitcast_i8_to_f32_vector(%arg0: vector<64xi8>) -> vector<16xf32> {
+  %0 = vector.bitcast %arg0 : vector<64xi8> to vector<16xf32>
+  return %0 : vector<16xf32>
+}
+
+// CHECK-LABEL: @bitcast_i8_to_f32_vector
+// CHECK-SAME:  %[[ARG_0:.*]]: vector<64xi8>
+// CHECK:       llvm.bitcast %[[ARG_0]] : vector<64xi8> to vector<16xf32>
+
+// -----
+
+func.func @bitcast_i8_to_f32_vector_scalable(%arg0: vector<[64]xi8>) -> vector<[16]xf32> {
+  %0 = vector.bitcast %arg0 : vector<[64]xi8> to vector<[16]xf32>
+  return %0 : vector<[16]xf32>
+}
+
+// CHECK-LABEL: @bitcast_i8_to_f32_vector_scalable
+// CHECK-SAME:  %[[ARG_0:.*]]: vector<[64]xi8>
+// CHECK:       llvm.bitcast %[[ARG_0]] : vector<[64]xi8> to vector<[16]xf32>
+
+// -----
+
+func.func @bitcast_index_to_i8_vector(%arg0: vector<16xindex>) -> vector<128xi8> {
+  %0 = vector.bitcast %arg0 : vector<16xindex> to vector<128xi8>
+  return %0 : vector<128xi8>
+}
+
+// CHECK-LABEL: @bitcast_index_to_i8_vector
+// CHECK-SAME:  %[[ARG_0:.*]]: vector<16xindex>
+// CHECK:       %[[T0:.*]] = builtin.unrealized_conversion_cast %[[ARG_0]] : vector<16xindex> to vector<16xi64>
+// CHECK:       llvm.bitcast %[[T0]] : vector<16xi64> to vector<128xi8>
+
+// -----
+
+func.func @bitcast_index_to_i8_vector_scalable(%arg0: vector<[16]xindex>) -> vector<[128]xi8> {
+  %0 = vector.bitcast %arg0 : vector<[16]xindex> to vector<[128]xi8>
+  return %0 : vector<[128]xi8>
+}
+
+// CHECK-LABEL: @bitcast_index_to_i8_vector_scalable
+// CHECK-SAME:  %[[ARG_0:.*]]: vector<[16]xindex>
+// CHECK:       %[[T0:.*]] = builtin.unrealized_conversion_cast %[[ARG_0]] : vector<[16]xindex> to vector<[16]xi64>
+// CHECK:       llvm.bitcast %[[T0]] : vector<[16]xi64> to vector<[128]xi8>
+
+// -----
+
+//===----------------------------------------------------------------------===//
+// vector.broadcast
+//===----------------------------------------------------------------------===//
+
+func.func @broadcast_vec0d_from_vec0d(%arg0: vector<f32>) -> vector<f32> {
+  %0 = vector.broadcast %arg0 : vector<f32> to vector<f32>
+  return %0 : vector<f32>
+}
+// CHECK-LABEL: @broadcast_vec0d_from_vec0d(
+// CHECK-SAME:  %[[A:.*]]: vector<f32>)
+// CHECK:       return %[[A]] : vector<f32>
+
+// -----
+
+func.func @broadcast_vec1d_from_vec1d(%arg0: vector<2xf32>) -> vector<2xf32> {
+  %0 = vector.broadcast %arg0 : vector<2xf32> to vector<2xf32>
+  return %0 : vector<2xf32>
+}
+// CHECK-LABEL: @broadcast_vec1d_from_vec1d(
+// CHECK-SAME:  %[[A:.*]]: vector<2xf32>)
+// CHECK:       return %[[A]] : vector<2xf32>
+
+// -----
+
+func.func @broadcast_vec1d_from_vec1d_scalable(%arg0: vector<[2]xf32>) -> vector<[2]xf32> {
+  %0 = vector.broadcast %arg0 : vector<[2]xf32> to vector<[2]xf32>
+  return %0 : vector<[2]xf32>
+}
+// CHECK-LABEL: @broadcast_vec1d_from_vec1d_scalable(
+// CHECK-SAME:  %[[A:.*]]: vector<[2]xf32>)
+// CHECK:       return %[[A]] : vector<[2]xf32>
+
+// -----
+
+//===----------------------------------------------------------------------===//
+// vector.shuffle
+//===----------------------------------------------------------------------===//
+
+func.func @shuffle_0D_direct(%arg0: vector<f32>) -> vector<3xf32> {
+  %1 = vector.shuffle %arg0, %arg0 [0, 1, 0] : vector<f32>, vector<f32>
+  return %1 : vector<3xf32>
+}
+// CHECK-LABEL: @shuffle_0D_direct(
+//  CHECK-SAME:     %[[A:.*]]: vector<f32>
+//       CHECK:   %[[c:.*]] = builtin.unrealized_conversion_cast %[[A]] : vector<f32> to vector<1xf32>
+//       CHECK:   %[[s:.*]] = llvm.shufflevector %[[c]], %[[c]] [0, 1, 0] : vector<1xf32>
+//       CHECK:   return %[[s]] : vector<3xf32>
+
+// -----
+
+func.func @shuffle_1D_direct(%arg0: vector<2xf32>, %arg1: vector<2xf32>) -> vector<2xf32> {
+  %1 = vector.shuffle %arg0, %arg1 [0, 1] : vector<2xf32>, vector<2xf32>
+  return %1 : vector<2xf32>
+}
+// CHECK-LABEL: @shuffle_1D_direct(
+// CHECK-SAME: %[[A:.*]]: vector<2xf32>,
+// CHECK-SAME: %[[B:.*]]: vector<2xf32>)
+//       CHECK:   return %[[A:.*]]: vector<2xf32>
+
+// -----
+
+func.func @shuffle_1D_index_direct(%arg0: vector<2xindex>, %arg1: vector<2xindex>) -> vector<2xindex> {
+  %1 = vector.shuffle %arg0, %arg1 [0, 1] : vector<2xindex>, vector<2xindex>
+  return %1 : vector<2xindex>
+}
+// CHECK-LABEL: @shuffle_1D_index_direct(
+// CHECK-SAME: %[[A:.*]]: vector<2xindex>,
+// CHECK-SAME: %[[B:.*]]: vector<2xindex>)
+//       CHECK:   return  %[[A:.*]]: vector<2xindex>
+
+// -----
+
+func.func @shuffle_poison_mask(%arg0: vector<2xf32>, %arg1: vector<2xf32>) -> vector<4xf32> {
+  %1 = vector.shuffle %arg0, %arg1 [0, -1, 3, -1] : vector<2xf32>, vector<2xf32>
+  return %1 : vector<4xf32>
+}
+// CHECK-LABEL: @shuffle_poison_mask(
+//  CHECK-SAME:   %[[A:.*]]: vector<2xf32>, %[[B:.*]]: vector<2xf32>)
+//       CHECK:     %[[s:.*]] = llvm.shufflevector %[[A]], %[[B]] [0, -1, 3, -1] : vector<2xf32>
+
+// -----
+
+func.func @shuffle_1D(%arg0: vector<2xf32>, %arg1: vector<3xf32>) -> vector<5xf32> {
+  %1 = vector.shuffle %arg0, %arg1 [4, 3, 2, 1, 0] : vector<2xf32>, vector<3xf32>
+  return %1 : vector<5xf32>
+}
+// CHECK-LABEL: @shuffle_1D(
+// CHECK-SAME: %[[A:.*]]: vector<2xf32>,
+// CHECK-SAME: %[[B:.*]]: vector<3xf32>)
+//       CHECK:   %[[U0:.*]] = llvm.mlir.poison : vector<5xf32>
+//       CHECK:   %[[C2:.*]] = llvm.mlir.constant(2 : index) : i64
+//       CHECK:   %[[E1:.*]] = llvm.extractelement %[[B]][%[[C2]] : i64] : vector<3xf32>
+//       CHECK:   %[[C0:.*]] = llvm.mlir.constant(0 : index) : i64
+//       CHECK:   %[[I1:.*]] = llvm.insertelement %[[E1]], %[[U0]][%[[C0]] : i64] : vector<5xf32>
+//       CHECK:   %[[C1:.*]] = llvm.mlir.constant(1 : index) : i64
+//       CHECK:   %[[E2:.*]] = llvm.extractelement %[[B]][%[[C1]] : i64] : vector<3xf32>
+//       CHECK:   %[[C1:.*]] = llvm.mlir.constant(1 : index) : i64
+//       CHECK:   %[[I2:.*]] = llvm.insertelement %[[E2]], %[[I1]][%[[C1]] : i64] : vector<5xf32>
+//       CHECK:   %[[C0:.*]] = llvm.mlir.constant(0 : index) : i64
+//       CHECK:   %[[E3:.*]] = llvm.extractelement %[[B]][%[[C0]] : i64] : vector<3xf32>
+//       CHECK:   %[[C2:.*]] = llvm.mlir.constant(2 : index) : i64
+//       CHECK:   %[[I3:.*]] = llvm.insertelement %[[E3]], %[[I2]][%[[C2]] : i64] : vector<5xf32>
+//       CHECK:   %[[C1:.*]] = llvm.mlir.constant(1 : index) : i64
+//       CHECK:   %[[E4:.*]] = llvm.extractelement %[[A]][%[[C1]] : i64] : vector<2xf32>
+//       CHECK:   %[[C3:.*]] = llvm.mlir.constant(3 : index) : i64
+//       CHECK:   %[[I4:.*]] = llvm.insertelement %[[E4]], %[[I3]][%[[C3]] : i64] : vector<5xf32>
+//       CHECK:   %[[C0:.*]] = llvm.mlir.constant(0 : index) : i64
+//       CHECK:   %[[E5:.*]] = llvm.extractelement %[[A]][%[[C0]] : i64] : vector<2xf32>
+//       CHECK:   %[[C4:.*]] = llvm.mlir.constant(4 : index) : i64
+//       CHECK:   %[[I5:.*]] = llvm.insertelement %[[E5]], %[[I4]][%[[C4]] : i64] : vector<5xf32>
+//       CHECK:   return %[[I5]] : vector<5xf32>
+
+// -----
+
+func.func @shuffle_2D(%a: vector<1x4xf32>, %b: vector<2x4xf32>) -> vector<3x4xf32> {
+  %1 = vector.shuffle %a, %b[1, 0, 2] : vector<1x4xf32>, vector<2x4xf32>
+  return %1 : vector<3x4xf32>
+}
+// CHECK-LABEL: @shuffle_2D(
+// CHECK-SAME: %[[A:.*]]: vector<1x4xf32>,
+// CHECK-SAME: %[[B:.*]]: vector<2x4xf32>)
+//       CHECK-DAG:   %[[VAL_0:.*]] = builtin.unrealized_conversion_cast %[[A]] : vector<1x4xf32> to !llvm.array<1 x vector<4xf32>>
+//       CHECK-DAG:   %[[VAL_1:.*]] = builtin.unrealized_conversion_cast %[[B]] : vector<2x4xf32> to !llvm.array<2 x vector<4xf32>>
+//       CHECK:   %[[U0:.*]] = llvm.mlir.poison : !llvm.array<3 x vector<4xf32>>
+//       CHECK:   %[[E1:.*]] = llvm.extractvalue %[[VAL_1]][0] : !llvm.array<2 x vector<4xf32>>
+//       CHECK:   %[[I1:.*]] = llvm.insertvalue %[[E1]], %[[U0]][0] : !llvm.array<3 x vector<4xf32>>
+//       CHECK:   %[[E2:.*]] = llvm.extractvalue %[[VAL_0]][0] : !llvm.array<1 x vector<4xf32>>
+//       CHECK:   %[[I2:.*]] = llvm.insertvalue %[[E2]], %[[I1]][1] : !llvm.array<3 x vector<4xf32>>
+//       CHECK:   %[[E3:.*]] = llvm.extractvalue %[[VAL_1]][1] : !llvm.array<2 x vector<4xf32>>
+//       CHECK:   %[[I3:.*]] = llvm.insertvalue %[[E3]], %[[I2]][2] : !llvm.array<3 x vector<4xf32>>
+//       CHECK:   %[[VAL_3:.*]] = builtin.unrealized_conversion_cast %[[I3]] : !llvm.array<3 x vector<4xf32>> to vector<3x4xf32>
+//       CHECK:   return %[[VAL_3]] : vector<3x4xf32>
+
+// -----
+
+//===----------------------------------------------------------------------===//
+// vector.extractelement
+//===----------------------------------------------------------------------===//
+
+func.func @extractelement_from_vec_0d_f32(%arg0: vector<f32>) -> f32 {
+  %1 = vector.extractelement %arg0[] : vector<f32>
+  return %1 : f32
+}
+// CHECK-LABEL: @extractelement_from_vec_0d_f32
+//       CHECK:   %[[C0:.*]] = llvm.mlir.constant(0 : index) : i64
+//       CHECK:   llvm.extractelement %{{.*}}[%[[C0]] : {{.*}}] : vector<1xf32>
+
+// -----
+
+func.func @extractelement_from_vec_1d_f32_idx_as_i32(%arg0: vector<16xf32>) -> f32 {
+  %0 = arith.constant 15 : i32
+  %1 = vector.extractelement %arg0[%0 : i32]: vector<16xf32>
+  return %1 : f32
+}
+// CHECK-LABEL: @extractelement_from_vec_1d_f32_idx_as_i32(
+//  CHECK-SAME:   %[[A:.*]]: vector<16xf32>)
+//       CHECK:   %[[C:.*]] = arith.constant 15 : i32
+//       CHECK:   %[[X:.*]] = llvm.extractelement %[[A]][%[[C]] : i32] : vector<16xf32>
+//       CHECK:   return %[[X]] : f32
+
+// -----
+
+func.func @extractelement_from_vec_1d_f32_idx_as_i32_scalable(%arg0: vector<[16]xf32>) -> f32 {
+  %0 = arith.constant 15 : i32
+  %1 = vector.extractelement %arg0[%0 : i32]: vector<[16]xf32>
+  return %1 : f32
+}
+// CHECK-LABEL: @extractelement_from_vec_1d_f32_idx_as_i32_scalable(
+//  CHECK-SAME:   %[[A:.*]]: vector<[16]xf32>)
+//       CHECK:   %[[C:.*]] = arith.constant 15 : i32
+//       CHECK:   %[[X:.*]] = llvm.extractelement %[[A]][%[[C]] : i32] : vector<[16]xf32>
+//       CHECK:   return %[[X]] : f32
+
+// -----
+func.func @extractelement_from_vec_1d_f32_idx_as_index(%arg0: vector<16xf32>) -> f32 {
+  %0 = arith.constant 15 : index
+  %1 = vector.extractelement %arg0[%0 : index]: vector<16xf32>
+  return %1 : f32
+}
+// CHECK-LABEL: @extractelement_from_vec_1d_f32_idx_as_index(
+//  CHECK-SAME:   %[[A:.*]]: vector<16xf32>)
+//       CHECK:   %[[C:.*]] = arith.constant 15 : index
+//       CHECK:   %[[I:.*]] = builtin.unrealized_conversion_cast %[[C]] : index to i64
+//       CHECK:   %[[X:.*]] = llvm.extractelement %[[A]][%[[I]] : i64] : vector<16xf32>
+//       CHECK:   return %[[X]] : f32
+
+// -----
+
+func.func @extractelement_from_vec_1d_f32_idx_as_index_scalable(%arg0: vector<[16]xf32>) -> f32 {
+  %0 = arith.constant 15 : index
+  %1 = vector.extractelement %arg0[%0 : index]: vector<[16]xf32>
+  return %1 : f32
+}
+// CHECK-LABEL: @extractelement_from_vec_1d_f32_idx_as_index_scalable(
+//  CHECK-SAME:   %[[A:.*]]: vector<[16]xf32>)
+//       CHECK:   %[[C:.*]] = arith.constant 15 : index
+//       CHECK:   %[[I:.*]] = builtin.unrealized_conversion_cast %[[C]] : index to i64
+//       CHECK:   %[[X:.*]] = llvm.extractelement %[[A]][%[[I]] : i64] : vector<[16]xf32>
+//       CHECK:   return %[[X]] : f32
+
+// -----
+
+//===----------------------------------------------------------------------===//
+// vector.extract
+//===----------------------------------------------------------------------===//
+
+func.func @extract_scalar_from_vec_1d_f32(%arg0: vector<16xf32>) -> f32 {
+  %0 = vector.extract %arg0[15]: f32 from vector<16xf32>
+  return %0 : f32
+}
+// CHECK-LABEL: @extract_scalar_from_vec_1d_f32
+//       CHECK:   llvm.mlir.constant(15 : i64) : i64
+//       CHECK:   llvm.extractelement {{.*}}[{{.*}} : i64] : vector<16xf32>
+//       CHECK:   return {{.*}} : f32
+
+
+// -----
+
+func.func @extract_scalar_from_vec_1d_f32_scalable(%arg0: vector<[16]xf32>) -> f32 {
+  %0 = vector.extract %arg0[15]: f32 from vector<[16]xf32>
+  return %0 : f32
+}
+// CHECK-LABEL: @extract_scalar_from_vec_1d_f32_scalable
+//       CHECK:   llvm.mlir.constant(15 : i64) : i64
+//       CHECK:   llvm.extractelement {{.*}}[{{.*}} : i64] : vector<[16]xf32>
+//       CHECK:   return {{.*}} : f32
+
+// -----
+
+func.func @extract_vec_1e_from_vec_1d_f32(%arg0: vector<16xf32>) -> vector<1xf32> {
+  %0 = vector.extract %arg0[15]: vector<1xf32> from vector<16xf32>
+  return %0 : vector<1xf32>
+}
+// CHECK-LABEL: @extract_vec_1e_from_vec_1d_f32(
+//  CHECK-SAME:   %[[A:.*]]: vector<16xf32>)
+//       CHECK:   %[[T0:.*]] = llvm.mlir.constant(15 : i64) : i64
+//       CHECK:   %[[T1:.*]] = llvm.extractelement %[[A]][%[[T0]] : i64] : vector<16xf32>
+//       CHECK:   %[[T2:.*]] = builtin.unrealized_conversion_cast %[[T1]] : f32 to vector<1xf32>
+//       CHECK:   return %[[T2]] : vector<1xf32>
+
+// -----
+
+func.func @extract_vec_1e_from_vec_1d_f32_scalable(%arg0: vector<[16]xf32>) -> vector<1xf32> {
+  %0 = vector.extract %arg0[15]: vector<1xf32> from vector<[16]xf32>
+  return %0 : vector<1xf32>
+}
+// CHECK-LABEL: @extract_vec_1e_from_vec_1d_f32_scalable(
+//  CHECK-SAME:   %[[A:.*]]: vector<[16]xf32>)
+//       CHECK:   %[[T0:.*]] = llvm.mlir.constant(15 : i64) : i64
+//       CHECK:   %[[T1:.*]] = llvm.extractelement %[[A]][%[[T0]] : i64] : vector<[16]xf32>
+//       CHECK:   %[[T2:.*]] = builtin.unrealized_conversion_cast %[[T1]] : f32 to vector<1xf32>
+//       CHECK:   return %[[T2]] : vector<1xf32>
+
+// -----
+
+func.func @extract_scalar_from_vec_1d_index(%arg0: vector<16xindex>) -> index {
+  %0 = vector.extract %arg0[15]: index from vector<16xindex>
+  return %0 : index
+}
+// CHECK-LABEL: @extract_scalar_from_vec_1d_index(
+//  CHECK-SAME:   %[[A:.*]]: vector<16xindex>)
+//       CHECK:   %[[T0:.*]] = builtin.unrealized_conversion_cast %[[A]] : vector<16xindex> to vector<16xi64>
+//       CHECK:   %[[T1:.*]] = llvm.mlir.constant(15 : i64) : i64
+//       CHECK:   %[[T2:.*]] = llvm.extractelement %[[T0]][%[[T1]] : i64] : vector<16xi64>
+//       CHECK:   %[[T3:.*]] = builtin.unrealized_conversion_cast %[[T2]] : i64 to index
+//       CHECK:   return %[[T3]] : index
+
+// -----
+
+func.func @extract_scalar_from_vec_1d_index_scalable(%arg0: vector<[16]xindex>) -> index {
+  %0 = vector.extract %arg0[15]: index from vector<[16]xindex>
+  return %0 : index
+}
+// CHECK-LABEL: @extract_scalar_from_vec_1d_index_scalable(
+//  CHECK-SAME:   %[[A:.*]]: vector<[16]xindex>)
+//       CHECK:   %[[T0:.*]] = builtin.unrealized_conversion_cast %[[A]] : vector<[16]xindex> to vector<[16]xi64>
+//       CHECK:   %[[T1:.*]] = llvm.mlir.constant(15 : i64) : i64
+//       CHECK:   %[[T2:.*]] = llvm.extractelement %[[T0]][%[[T1]] : i64] : vector<[16]xi64>
+//       CHECK:   %[[T3:.*]] = builtin.unrealized_conversion_cast %[[T2]] : i64 to index
+//       CHECK:   return %[[T3]] : index
+
+// -----
+
+func.func @extract_vec_2d_from_vec_3d_f32(%arg0: vector<4x3x16xf32>) -> vector<3x16xf32> {
+  %0 = vector.extract %arg0[0]: vector<3x16xf32> from vector<4x3x16xf32>
+  return %0 : vector<3x16xf32>
+}
+// CHECK-LABEL: @extract_vec_2d_from_vec_3d_f32
+//       CHECK:   llvm.extractvalue {{.*}}[0] : !llvm.array<4 x array<3 x vector<16xf32>>>
+//       CHECK:   return {{.*}} : vector<3x16xf32>
+
+
+// -----
+
+func.func @extract_vec_2d_from_vec_3d_f32_scalable(%arg0: vector<4x3x[16]xf32>) -> vector<3x[16]xf32> {
+  %0 = vector.extract %arg0[0]: vector<3x[16]xf32> from vector<4x3x[16]xf32>
+  return %0 : vector<3x[16]xf32>
+}
+// CHECK-LABEL: @extract_vec_2d_from_vec_3d_f32_scalable
+//       CHECK:   llvm.extractvalue {{.*}}[0] : !llvm.array<4 x array<3 x vector<[16]xf32>>>
+//       CHECK:   return {{.*}} : vector<3x[16]xf32>
+
+// -----
+
+func.func @extract_vec_1d_from_vec_3d_f32(%arg0: vector<4x3x16xf32>) -> vector<16xf32> {
+  %0 = vector.extract %arg0[0, 0]: vector<16xf32> from vector<4x3x16xf32>
+  return %0 : vector<16xf32>
+}
+// CHECK-LABEL: @extract_vec_1d_from_vec_3d_f32
+//       CHECK:   llvm.extractvalue {{.*}}[0, 0] : !llvm.array<4 x array<3 x vector<16xf32>>>
+//       CHECK:   return {{.*}} : vector<16xf32>
+
+// -----
+
+func.func @extract_vec_1d_from_vec_3d_f32_scalable(%arg0: vector<4x3x[16]xf32>) -> vector<[16]xf32> {
+  %0 = vector.extract %arg0[0, 0]: vector<[16]xf32> from vector<4x3x[16]xf32>
+  return %0 : vector<[16]xf32>
+}
+// CHECK-LABEL: @extract_vec_1d_from_vec_3d_f32_scalable
+//       CHECK:   llvm.extractvalue {{.*}}[0, 0] : !llvm.array<4 x array<3 x vector<[16]xf32>>>
+//       CHECK:   return {{.*}} : vector<[16]xf32>
+
+// -----
+
+func.func @extract_scalar_from_vec_3d_f32(%arg0: vector<4x3x16xf32>) -> f32 {
+  %0 = vector.extract %arg0[0, 0, 0]: f32 from vector<4x3x16xf32>
+  return %0 : f32
+}
+// CHECK-LABEL: @extract_scalar_from_vec_3d_f32
+//       CHECK:   llvm.extractvalue {{.*}}[0, 0] : !llvm.array<4 x array<3 x vector<16xf32>>>
+//       CHECK:   llvm.mlir.constant(0 : i64) : i64
+//       CHECK:   llvm.extractelement {{.*}}[{{.*}} : i64] : vector<16xf32>
+//       CHECK:   return {{.*}} : f32
+
+// -----
+
+func.func @extract_scalar_from_vec_3d_f32_scalable(%arg0: vector<4x3x[16]xf32>) -> f32 {
+  %0 = vector.extract %arg0[0, 0, 0]: f32 from vector<4x3x[16]xf32>
+  return %0 : f32
+}
+// CHECK-LABEL: @extract_scalar_from_vec_3d_f32_scalable
+//       CHECK:   llvm.extractvalue {{.*}}[0, 0] : !llvm.array<4 x array<3 x vector<[16]xf32>>>
+//       CHECK:   llvm.mlir.constant(0 : i64) : i64
+//       CHECK:   llvm.extractelement {{.*}}[{{.*}} : i64] : vector<[16]xf32>
+//       CHECK:   return {{.*}} : f32
+
+// -----
+
+func.func @extract_scalar_from_vec_1d_f32_dynamic_idx(%arg0: vector<16xf32>, %arg1: index) -> f32 {
+  %0 = vector.extract %arg0[%arg1]: f32 from vector<16xf32>
+  return %0 : f32
+}
+// CHECK-LABEL: @extract_scalar_from_vec_1d_f32_dynamic_idx
+//  CHECK-SAME:   %[[VEC:.+]]: vector<16xf32>, %[[INDEX:.+]]: index
+//       CHECK:   %[[UC:.+]] = builtin.unrealized_conversion_cast %[[INDEX]] : index to i64
+//       CHECK:   llvm.extractelement %[[VEC]][%[[UC]] : i64] : vector<16xf32>
+
+// -----
+
+func.func @extract_scalar_from_vec_1d_f32_dynamic_idx_scalable(%arg0: vector<[16]xf32>, %arg1: index) -> f32 {
+  %0 = vector.extract %arg0[%arg1]: f32 from vector<[16]xf32>
+  return %0 : f32
+}
+// CHECK-LABEL: @extract_scalar_from_vec_1d_f32_dynamic_idx_scalable
+//  CHECK-SAME:   %[[VEC:.+]]: vector<[16]xf32>, %[[INDEX:.+]]: index
+//       CHECK:   %[[UC:.+]] = builtin.unrealized_conversion_cast %[[INDEX]] : index to i64
+//       CHECK:   llvm.extractelement %[[VEC]][%[[UC]] : i64] : vector<[16]xf32>
+
+// -----
+
+func.func @extract_scalar_from_vec_2d_f32_inner_dynamic_idx(%arg0: vector<1x16xf32>, %arg1: index) -> f32 {
+  %0 = vector.extract %arg0[0, %arg1]: f32 from vector<1x16xf32>
+  return %0 : f32
+}
+
+// Lowering supports extracting from multi-dim vectors with dynamic indices
+// provided that only the trailing index is dynamic.
+
+// CHECK-LABEL: @extract_scalar_from_vec_2d_f32_inner_dynamic_idx(
+//       CHECK:   llvm.extractvalue
+//       CHECK:   llvm.extractelement
+
+func.func @extract_scalar_from_vec_2d_f32_inner_dynamic_idx_scalable(%arg0: vector<1x[16]xf32>, %arg1: index) -> f32 {
+  %0 = vector.extract %arg0[0, %arg1]: f32 from vector<1x[16]xf32>
+  return %0 : f32
+}
+
+// Lowering supports extracting from multi-dim vectors with dynamic indices
+// provided that only the trailing index is dynamic.
+
+// CHECK-LABEL: @extract_scalar_from_vec_2d_f32_inner_dynamic_idx_scalable(
+//       CHECK:   llvm.extractvalue
+//       CHECK:   llvm.extractelement
+
+// -----
+
+func.func @extract_scalar_from_vec_2d_f32_outer_dynamic_idx(%arg0: vector<1x16xf32>, %arg1: index) -> f32 {
+  %0 = vector.extract %arg0[%arg1, 0]: f32 from vector<1x16xf32>
+  return %0 : f32
+}
+
+// Lowering supports extracting from multi-dim vectors with dynamic indices
+// provided that only the trailing index is dynamic.
+
+// CHECK-LABEL: @extract_scalar_from_vec_2d_f32_outer_dynamic_idx(
+//       CHECK:   vector.extract
+
+func.func @extract_scalar_from_vec_2d_f32_outer_dynamic_idx_scalable(%arg0: vector<1x[16]xf32>, %arg1: index) -> f32 {
+  %0 = vector.extract %arg0[%arg1, 0]: f32 from vector<1x[16]xf32>
+  return %0 : f32
+}
+
+// Lowering does not support extracting from multi-dim vectors with non trailing
+// dynamic index, but it shouldn't crash.
+
+// CHECK-LABEL: @extract_scalar_from_vec_2d_f32_outer_dynamic_idx_scalable(
+//       CHECK:   vector.extract
+
+// -----
+
+func.func @extract_scalar_from_vec_0d_index(%arg0: vector<index>) -> index {
+  %0 = vector.extract %arg0[]: index from vector<index>
+  return %0 : index
+}
+// CHECK-LABEL: @extract_scalar_from_vec_0d_index(
+//  CHECK-SAME:   %[[A:.*]]: vector<index>)
+//       CHECK:   %[[T0:.*]] = builtin.unrealized_conversion_cast %[[A]] : vector<index> to vector<1xi64>
+//       CHECK:   %[[T1:.*]] = llvm.mlir.constant(0 : i64) : i64
+//       CHECK:   %[[T2:.*]] = llvm.extractelement %[[T0]][%[[T1]] : i64] : vector<1xi64>
+//       CHECK:   %[[T3:.*]] = builtin.unrealized_conversion_cast %[[T2]] : i64 to index
+//       CHECK:   return %[[T3]] : index
+
+// -----
+
+func.func @extract_scalar_from_vec_2d_f32_dynamic_idxs_compile_time_const(%arg : vector<32x1xi32>) -> i32 {
+  %0 = arith.constant 0 : index
+  %1 = vector.extract %arg[%0, %0] : i32 from vector<32x1xi32>
+  return %1 : i32
+}
+
+// At compile time, since the indices of extractOp are constants,
+// they will be collapsed and folded away; therefore, the lowering works.
+
+// CHECK-LABEL: @extract_scalar_from_vec_2d_f32_dynamic_idxs_compile_time_const
+//  CHECK-SAME:   %[[ARG:.*]]: vector<32x1xi32>) -> i32 {
+//       CHECK:   %[[CAST:.*]] = builtin.unrealized_conversion_cast %[[ARG]] : vector<32x1xi32> to !llvm.array<32 x vector<1xi32>>
+//       CHECK:   %[[VEC_0:.*]] = llvm.extractvalue %[[CAST]][0] : !llvm.array<32 x vector<1xi32>>
+//       CHECK:   %[[C0:.*]] = llvm.mlir.constant(0 : i64) : i64
+//       CHECK:   %[[RES:.*]] = llvm.extractelement %[[VEC_0]]{{\[}}%[[C0]] : i64] : vector<1xi32>
+//       CHECK:   return %[[RES]] : i32
+
+// -----
+
+//===----------------------------------------------------------------------===//
+// vector.insertelement
+//===----------------------------------------------------------------------===//
+
+func.func @insertelement_into_vec_0d_f32(%arg0: f32, %arg1: vector<f32>) -> vector<f32> {
+  %1 = vector.insertelement %arg0, %arg1[] : vector<f32>
+  return %1 : vector<f32>
+}
+// CHECK-LABEL: @insertelement_into_vec_0d_f32
+//  CHECK-SAME:   %[[A:.*]]: f32,
+//       CHECK:   %[[B:.*]] =  builtin.unrealized_conversion_cast %{{.*}} :
+//       CHECK:   vector<f32> to vector<1xf32>
+//       CHECK:   %[[C0:.*]] = llvm.mlir.constant(0 : index) : i64
+//       CHECK:   %[[X:.*]] = llvm.insertelement %[[A]], %[[B]][%[[C0]] : {{.*}}] : vector<1xf32>
+
+// -----
+
+func.func @insertelement_into_vec_1d_f32_idx_as_i32(%arg0: f32, %arg1: vector<4xf32>) -> vector<4xf32> {
+  %0 = arith.constant 3 : i32
+  %1 = vector.insertelement %arg0, %arg1[%0 : i32] : vector<4xf32>
+  return %1 : vector<4xf32>
+}
+// CHECK-LABEL: @insertelement_into_vec_1d_f32_idx_as_i32(
+//  CHECK-SAME:   %[[A:.*]]: f32,
+//  CHECK-SAME:   %[[B:.*]]: vector<4xf32>)
+//       CHECK:   %[[C:.*]] = arith.constant 3 : i32
+//       CHECK:   %[[X:.*]] = llvm.insertelement %[[A]], %[[B]][%[[C]] : i32] : vector<4xf32>
+//       CHECK:   return %[[X]] : vector<4xf32>
+
+// -----
+
+func.func @insertelement_into_vec_1d_f32_idx_as_i32_scalable(%arg0: f32, %arg1: vector<[4]xf32>) -> vector<[4]xf32> {
+  %0 = arith.constant 3 : i32
+  %1 = vector.insertelement %arg0, %arg1[%0 : i32] : vector<[4]xf32>
+  return %1 : vector<[4]xf32>
+}
+// CHECK-LABEL: @insertelement_into_vec_1d_f32_idx_as_i32_scalable(
+//  CHECK-SAME:   %[[A:.*]]: f32,
+//  CHECK-SAME:   %[[B:.*]]: vector<[4]xf32>)
+//       CHECK:   %[[C:.*]] = arith.constant 3 : i32
+//       CHECK:   %[[X:.*]] = llvm.insertelement %[[A]], %[[B]][%[[C]] : i32] : vector<[4]xf32>
+//       CHECK:   return %[[X]] : vector<[4]xf32>
+
+// -----
+
+func.func @insertelement_into_vec_1d_f32_scalable_idx_as_index(%arg0: f32, %arg1: vector<4xf32>) -> vector<4xf32> {
+  %0 = arith.constant 3 : index
+  %1 = vector.insertelement %arg0, %arg1[%0 : index] : vector<4xf32>
+  return %1 : vector<4xf32>
+}
+// CHECK-LABEL: @insertelement_into_vec_1d_f32_scalable_idx_as_index(
+//  CHECK-SAME:   %[[A:.*]]: f32,
+//  CHECK-SAME:   %[[B:.*]]: vector<4xf32>)
+//       CHECK:   %[[C:.*]] = arith.constant 3 : index
+//       CHECK:   %[[I:.*]] = builtin.unrealized_conversion_cast %[[C]] : index to i64
+//       CHECK:   %[[X:.*]] = llvm.insertelement %[[A]], %[[B]][%[[I]] : i64] : vector<4xf32>
+//       CHECK:   return %[[X]] : vector<4xf32>
+
+// -----
+
+func.func @insertelement_into_vec_1d_f32_scalable_idx_as_index_scalable(%arg0: f32, %arg1: vector<[4]xf32>) -> vector<[4]xf32> {
+  %0 = arith.constant 3 : index
+  %1 = vector.insertelement %arg0, %arg1[%0 : index] : vector<[4]xf32>
+  return %1 : vector<[4]xf32>
+}
+// CHECK-LABEL: @insertelement_into_vec_1d_f32_scalable_idx_as_index_scalable(
+//  CHECK-SAME:   %[[A:.*]]: f32,
+//  CHECK-SAME:   %[[B:.*]]: vector<[4]xf32>)
+//       CHECK:   %[[C:.*]] = arith.constant 3 : index
+//       CHECK:   %[[I:.*]] = builtin.unrealized_conversion_cast %[[C]] : index to i64
+//       CHECK:   %[[X:.*]] = llvm.insertelement %[[A]], %[[B]][%[[I]] : i64] : vector<[4]xf32>
+//       CHECK:   return %[[X]] : vector<[4]xf32>
+
+// -----
+
+//===----------------------------------------------------------------------===//
+// vector.insert
+//===----------------------------------------------------------------------===//
+
+func.func @insert_scalar_into_vec_1d_f32(%arg0: f32, %arg1: vector<4xf32>) -> vector<4xf32> {
+  %0 = vector.insert %arg0, %arg1[3] : f32 into vector<4xf32>
+  return %0 : vector<4xf32>
+}
+// CHECK-LABEL: @insert_scalar_into_vec_1d_f32
+//       CHECK:   llvm.mlir.constant(3 : i64) : i64
+//       CHECK:   llvm.insertelement {{.*}}, {{.*}}[{{.*}} : i64] : vector<4xf32>
+//       CHECK:   return {{.*}} : vector<4xf32>
+
+// -----
+
+func.func @insert_scalar_into_vec_1d_f32_scalable(%arg0: f32, %arg1: vector<[4]xf32>) -> vector<[4]xf32> {
+  %0 = vector.insert %arg0, %arg1[3] : f32 into vector<[4]xf32>
+  return %0 : vector<[4]xf32>
+}
+// CHECK-LABEL: @insert_scalar_into_vec_1d_f32_scalable
+//       CHECK:   llvm.mlir.constant(3 : i64) : i64
+//       CHECK:   llvm.insertelement {{.*}}, {{.*}}[{{.*}} : i64] : vector<[4]xf32>
+//       CHECK:   return {{.*}} : vector<[4]xf32>
+
+// -----
+
+func.func @insert_scalar_into_vec_1d_index(%arg0: index, %arg1: vector<4xindex>) -> vector<4xindex> {
+  %0 = vector.insert %arg0, %arg1[3] : index into vector<4xindex>
+  return %0 : vector<4xindex>
+}
+// CHECK-LABEL: @insert_scalar_into_vec_1d_index(
+//  CHECK-SAME:   %[[A:.*]]: index,
+//  CHECK-SAME:   %[[B:.*]]: vector<4xindex>)
+//   CHECK-DAG:   %[[T0:.*]] = builtin.unrealized_conversion_cast %[[A]] : index to i64
+//   CHECK-DAG:   %[[T1:.*]] = builtin.unrealized_conversion_cast %[[B]] : vector<4xindex> to vector<4xi64>
+//       CHECK:   %[[T3:.*]] = llvm.mlir.constant(3 : i64) : i64
+//       CHECK:   %[[T4:.*]] = llvm.insertelement %[[T0]], %[[T1]][%[[T3]] : i64] : vector<4xi64>
+//       CHECK:   %[[T5:.*]] = builtin.unrealized_conversion_cast %[[T4]] : vector<4xi64> to vector<4xindex>
+//       CHECK:   return %[[T5]] : vector<4xindex>
+
+// -----
+
+func.func @insert_scalar_into_vec_1d_index_scalable(%arg0: index, %arg1: vector<[4]xindex>) -> vector<[4]xindex> {
+  %0 = vector.insert %arg0, %arg1[3] : index into vector<[4]xindex>
+  return %0 : vector<[4]xindex>
+}
+// CHECK-LABEL: @insert_scalar_into_vec_1d_index_scalable(
+//  CHECK-SAME:   %[[A:.*]]: index,
+//  CHECK-SAME:   %[[B:.*]]: vector<[4]xindex>)
+//   CHECK-DAG:   %[[T0:.*]] = builtin.unrealized_conversion_cast %[[A]] : index to i64
+//   CHECK-DAG:   %[[T1:.*]] = builtin.unrealized_conversion_cast %[[B]] : vector<[4]xindex> to vector<[4]xi64>
+//       CHECK:   %[[T3:.*]] = llvm.mlir.constant(3 : i64) : i64
+//       CHECK:   %[[T4:.*]] = llvm.insertelement %[[T0]], %[[T1]][%[[T3]] : i64] : vector<[4]xi64>
+//       CHECK:   %[[T5:.*]] = builtin.unrealized_conversion_cast %[[T4]] : vector<[4]xi64> to vector<[4]xindex>
+//       CHECK:   return %[[T5]] : vector<[4]xindex>
+
+// -----
+
+func.func @insert_vec_2d_into_vec_3d_f32(%arg0: vector<8x16xf32>, %arg1: vector<4x8x16xf32>) -> vector<4x8x16xf32> {
+  %0 = vector.insert %arg0, %arg1[3] : vector<8x16xf32> into vector<4x8x16xf32>
+  return %0 : vector<4x8x16xf32>
+}
+// CHECK-LABEL: @insert_vec_2d_into_vec_3d_f32
+//       CHECK:   llvm.insertvalue {{.*}}, {{.*}}[3] : !llvm.array<4 x array<8 x vector<16xf32>>>
+//       CHECK:   return {{.*}} : vector<4x8x16xf32>
+
+// -----
+
+func.func @insert_vec_2d_into_vec_3d_f32_scalable(%arg0: vector<8x[16]xf32>, %arg1: vector<4x8x[16]xf32>) -> vector<4x8x[16]xf32> {
+  %0 = vector.insert %arg0, %arg1[3] : vector<8x[16]xf32> into vector<4x8x[16]xf32>
+  return %0 : vector<4x8x[16]xf32>
+}
+// CHECK-LABEL: @insert_vec_2d_into_vec_3d_f32_scalable
+//       CHECK:   llvm.insertvalue {{.*}}, {{.*}}[3] : !llvm.array<4 x array<8 x vector<[16]xf32>>>
+//       CHECK:   return {{.*}} : vector<4x8x[16]xf32>
+
+// -----
+
+func.func @insert_vec_1d_into_vec_3d_f32(%arg0: vector<16xf32>, %arg1: vector<4x8x16xf32>) -> vector<4x8x16xf32> {
+  %0 = vector.insert %arg0, %arg1[3, 7] : vector<16xf32> into vector<4x8x16xf32>
+  return %0 : vector<4x8x16xf32>
+}
+// CHECK-LABEL: @insert_vec_1d_into_vec_3d_f32
+//       CHECK:   llvm.insertvalue {{.*}}, {{.*}}[3, 7] : !llvm.array<4 x array<8 x vector<16xf32>>>
+//       CHECK:   return {{.*}} : vector<4x8x16xf32>
+
+// -----
+
+func.func @insert_vec_1d_into_vec_3d_f32_scalable(%arg0: vector<[16]xf32>, %arg1: vector<4x8x[16]xf32>) -> vector<4x8x[16]xf32> {
+  %0 = vector.insert %arg0, %arg1[3, 7] : vector<[16]xf32> into vector<4x8x[16]xf32>
+  return %0 : vector<4x8x[16]xf32>
+}
+// CHECK-LABEL: @insert_vec_1d_into_vec_3d_f32_scalable
+//       CHECK:   llvm.insertvalue {{.*}}, {{.*}}[3, 7] : !llvm.array<4 x array<8 x vector<[16]xf32>>>
+//       CHECK:   return {{.*}} : vector<4x8x[16]xf32>
+
+// -----
+
+func.func @insert_scalar_into_vec_3d_f32(%arg0: f32, %arg1: vector<4x8x16xf32>) -> vector<4x8x16xf32> {
+  %0 = vector.insert %arg0, %arg1[3, 7, 15] : f32 into vector<4x8x16xf32>
+  return %0 : vector<4x8x16xf32>
+}
+// CHECK-LABEL: @insert_scalar_into_vec_3d_f32
+//       CHECK:   llvm.extractvalue {{.*}}[3, 7] : !llvm.array<4 x array<8 x vector<16xf32>>>
+//       CHECK:   llvm.mlir.constant(15 : i64) : i64
+//       CHECK:   llvm.insertelement {{.*}}, {{.*}}[{{.*}} : i64] : vector<16xf32>
+//       CHECK:   llvm.insertvalue {{.*}}, {{.*}}[3, 7] : !llvm.array<4 x array<8 x vector<16xf32>>>
+//       CHECK:   return {{.*}} : vector<4x8x16xf32>
+
+// -----
+
+func.func @insert_scalar_into_vec_3d_f32_scalable(%arg0: f32, %arg1: vector<4x8x[16]xf32>) -> vector<4x8x[16]xf32> {
+  %0 = vector.insert %arg0, %arg1[3, 7, 15] : f32 into vector<4x8x[16]xf32>
+  return %0 : vector<4x8x[16]xf32>
+}
+// CHECK-LABEL: @insert_scalar_into_vec_3d_f32_scalable
+//       CHECK:   llvm.extractvalue {{.*}}[3, 7] : !llvm.array<4 x array<8 x vector<[16]xf32>>>
+//       CHECK:   llvm.mlir.constant(15 : i64) : i64
+//       CHECK:   llvm.insertelement {{.*}}, {{.*}}[{{.*}} : i64] : vector<[16]xf32>
+//       CHECK:   llvm.insertvalue {{.*}}, {{.*}}[3, 7] : !llvm.array<4 x array<8 x vector<[16]xf32>>>
+//       CHECK:   return {{.*}} : vector<4x8x[16]xf32>
+
+// -----
+
+func.func @insert_scalar_into_vec_1d_f32_dynamic_idx(%arg0: vector<16xf32>, %arg1: f32, %arg2: index)
+                                      -> vector<16xf32> {
+  %0 = vector.insert %arg1, %arg0[%arg2]: f32 into vector<16xf32>
+  return %0 : vector<16xf32>
+}
+
+// CHECK-LABEL: @insert_scalar_into_vec_1d_f32_dynamic_idx
+//  CHECK-SAME:   %[[DST:.+]]: vector<16xf32>, %[[SRC:.+]]: f32, %[[INDEX:.+]]: index
+//       CHECK:   %[[UC:.+]] = builtin.unrealized_conversion_cast %[[INDEX]] : index to i64
+//       CHECK:   llvm.insertelement %[[SRC]], %[[DST]][%[[UC]] : i64] : vector<16xf32>
+
+// -----
+
+func.func @insert_scalar_into_vec_1d_f32_dynamic_idx_scalable(%arg0: vector<[16]xf32>, %arg1: f32, %arg2: index)
+                                      -> vector<[16]xf32> {
+  %0 = vector.insert %arg1, %arg0[%arg2]: f32 into vector<[16]xf32>
+  return %0 : vector<[16]xf32>
+}
+
+// CHECK-LABEL: @insert_scalar_into_vec_1d_f32_dynamic_idx_scalable
+//  CHECK-SAME:   %[[DST:.+]]: vector<[16]xf32>, %[[SRC:.+]]: f32, %[[INDEX:.+]]: index
+//       CHECK:   %[[UC:.+]] = builtin.unrealized_conversion_cast %[[INDEX]] : index to i64
+//       CHECK:   llvm.insertelement %[[SRC]], %[[DST]][%[[UC]] : i64] : vector<[16]xf32>
+
+// -----
+
+func.func @insert_scalar_into_vec_2d_f32_dynamic_idx(%arg0: vector<1x16xf32>, %arg1: f32, %idx: index)
+                                        -> vector<1x16xf32> {
+  %0 = vector.insert %arg1, %arg0[0, %idx]: f32 into vector<1x16xf32>
+  return %0 : vector<1x16xf32>
+}
+
+// Multi-dim vectors are not supported but this test shouldn't crash.
+
+// CHECK-LABEL: @insert_scalar_into_vec_2d_f32_dynamic_idx(
+//       CHECK:   vector.insert
+
+// -----
+
+func.func @insert_scalar_into_vec_2d_f32_dynamic_idx_scalable(%arg0: vector<1x[16]xf32>, %arg1: f32, %idx: index)
+                                        -> vector<1x[16]xf32> {
+  %0 = vector.insert %arg1, %arg0[0, %idx]: f32 into vector<1x[16]xf32>
+  return %0 : vector<1x[16]xf32>
+}
+
+// Multi-dim vectors are not supported but this test shouldn't crash.
+
+// CHECK-LABEL: @insert_scalar_into_vec_2d_f32_dynamic_idx_scalable(
+//       CHECK:   vector.insert
+
+// -----
+
+func.func @insert_scalar_from_vec_2d_f32_dynamic_idxs_compile_time_const(%arg : vector<4x1xi32>) -> vector<4x1xi32> {
+  %0 = arith.constant 0 : index
+  %1 = arith.constant 1 : i32
+  %res = vector.insert %1, %arg[%0, %0] : i32 into vector<4x1xi32>
+  return %res : vector<4x1xi32>
+}
+
+// At compile time, since the indices of insertOp are constants,
+// they will be collapsed and folded away; therefore, the lowering works.
+
+// CHECK-LABEL: @insert_scalar_from_vec_2d_f32_dynamic_idxs_compile_time_const
+//  CHECK-SAME:   %[[ARG:.*]]: vector<4x1xi32>) -> vector<4x1xi32> {
+//       CHECK:   %[[CAST:.*]] = builtin.unrealized_conversion_cast %[[ARG]] : vector<4x1xi32> to !llvm.array<4 x vector<1xi32>>
+//       CHECK:   %[[C1:.*]] = arith.constant 1 : i32
+//       CHECK:   %[[VEC_0:.*]] = llvm.extractvalue %[[CAST]][0] : !llvm.array<4 x vector<1xi32>>
+//       CHECK:   %[[C0:.*]] = llvm.mlir.constant(0 : i64) : i64
+//       CHECK:   %[[VEC_1:.*]] = llvm.insertelement %[[C1]], %[[VEC_0]]{{\[}}%[[C0]] : i64] : vector<1xi32>
+//       CHECK:   %[[VEC_2:.*]] = llvm.insertvalue %[[VEC_1]], %[[CAST]][0] : !llvm.array<4 x vector<1xi32>>
+//       CHECK:   %[[RES:.*]] = builtin.unrealized_conversion_cast %[[VEC_2]] : !llvm.array<4 x vector<1xi32>> to vector<4x1xi32>
+//       CHECK:   return %[[RES]] : vector<4x1xi32>
+
+// -----
+
+//===----------------------------------------------------------------------===//
+// vector.type_cast
+//
+// TODO: Add tests for for vector.type_cast that would cover scalable vectors
+//===----------------------------------------------------------------------===//
+
+func.func @type_cast_f32(%arg0: memref<8x8x8xf32>) -> memref<vector<8x8x8xf32>> {
+  %0 = vector.type_cast %arg0: memref<8x8x8xf32> to memref<vector<8x8x8xf32>>
+  return %0 : memref<vector<8x8x8xf32>>
+}
+// CHECK-LABEL: @type_cast_f32
+//       CHECK:   llvm.mlir.poison : !llvm.struct<(ptr, ptr, i64)>
+//       CHECK:   %[[allocated:.*]] = llvm.extractvalue {{.*}}[0] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)>
+//       CHECK:   llvm.insertvalue %[[allocated]], {{.*}}[0] : !llvm.struct<(ptr, ptr, i64)>
+//       CHECK:   %[[aligned:.*]] = llvm.extractvalue {{.*}}[1] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)>
+//       CHECK:   llvm.insertvalue %[[aligned]], {{.*}}[1] : !llvm.struct<(ptr, ptr, i64)>
+//       CHECK:   llvm.mlir.constant(0 : index
+//       CHECK:   llvm.insertvalue {{.*}}[2] : !llvm.struct<(ptr, ptr, i64)>
+
+// NOTE: No test for scalable vectors - the input memref is fixed size.
+
+// -----
+
+func.func @type_cast_index(%arg0: memref<8x8x8xindex>) -> memref<vector<8x8x8xindex>> {
+  %0 = vector.type_cast %arg0: memref<8x8x8xindex> to memref<vector<8x8x8xindex>>
+  return %0 : memref<vector<8x8x8xindex>>
+}
+// CHECK-LABEL: @type_cast_index(
+// CHECK-SAME: %[[A:.*]]: memref<8x8x8xindex>)
+//       CHECK:   %{{.*}} = builtin.unrealized_conversion_cast %[[A]] : memref<8x8x8xindex> to !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)>
+
+//       CHECK:   %{{.*}} = builtin.unrealized_conversion_cast %{{.*}} : !llvm.struct<(ptr, ptr, i64)> to memref<vector<8x8x8xindex>>
+
+// NOTE: No test for scalable vectors - the input memref is fixed size.
+
+// -----
+
+func.func @type_cast_non_zero_addrspace(%arg0: memref<8x8x8xf32, 3>) -> memref<vector<8x8x8xf32>, 3> {
+  %0 = vector.type_cast %arg0: memref<8x8x8xf32, 3> to memref<vector<8x8x8xf32>, 3>
+  return %0 : memref<vector<8x8x8xf32>, 3>
+}
+// CHECK-LABEL: @type_cast_non_zero_addrspace
+//       CHECK:   llvm.mlir.poison : !llvm.struct<(ptr<3>, ptr<3>, i64)>
+//       CHECK:   %[[allocated:.*]] = llvm.extractvalue {{.*}}[0] : !llvm.struct<(ptr<3>, ptr<3>, i64, array<3 x i64>, array<3 x i64>)>
+//       CHECK:   llvm.insertvalue %[[allocated]], {{.*}}[0] : !llvm.struct<(ptr<3>, ptr<3>, i64)>
+//       CHECK:   %[[aligned:.*]] = llvm.extractvalue {{.*}}[1] : !llvm.struct<(ptr<3>, ptr<3>, i64, array<3 x i64>, array<3 x i64>)>
+//       CHECK:   llvm.insertvalue %[[aligned]], {{.*}}[1] : !llvm.struct<(ptr<3>, ptr<3>, i64)>
+//       CHECK:   llvm.mlir.constant(0 : index
+//       CHECK:   llvm.insertvalue {{.*}}[2] : !llvm.struct<(ptr<3>, ptr<3>, i64)>
+
+// NOTE: No test for scalable vectors - the input memref is fixed size.
+
+// -----
+
+//===----------------------------------------------------------------------===//
+// vector.print
+//===----------------------------------------------------------------------===//
+
+func.func @print_scalar_i64(%arg0: i64) {
+  vector.print %arg0 : i64
+  return
+}
+// CHECK-LABEL: @print_scalar_i64(
+// CHECK-SAME: %[[A:.*]]: i64)
+//       CHECK:    llvm.call @printI64(%[[A]]) : (i64) -> ()
+//       CHECK:    llvm.call @printNewline() : () -> ()
+
+// -----
+
+func.func @print_scalar_ui64(%arg0: ui64) {
+  vector.print %arg0 : ui64
+  return
+}
+// CHECK-LABEL: @print_scalar_ui64(
+// CHECK-SAME: %[[A:.*]]: ui64)
+//       CHECK:    %[[C:.*]] = builtin.unrealized_conversion_cast %[[A]] : ui64 to i64
+//       CHECK:    llvm.call @printU64(%[[C]]) : (i64) -> ()
+//       CHECK:    llvm.call @printNewline() : () -> ()
+
+// -----
+
+func.func @print_scalar_index(%arg0: index) {
+  vector.print %arg0 : index
+  return
+}
+// CHECK-LABEL: @print_scalar_index(
+// CHECK-SAME: %[[A:.*]]: index)
+//       CHECK:    %[[C:.*]] = builtin.unrealized_conversion_cast %[[A]] : index to i64
+//       CHECK:    llvm.call @printU64(%[[C]]) : (i64) -> ()
+//       CHECK:    llvm.call @printNewline() : () -> ()
+
+// -----
+
+func.func @print_scalar_f32(%arg0: f32) {
+  vector.print %arg0 : f32
+  return
+}
+// CHECK-LABEL: @print_scalar_f32(
+// CHECK-SAME: %[[A:.*]]: f32)
+//       CHECK:    llvm.call @printF32(%[[A]]) : (f32) -> ()
+//       CHECK:    llvm.call @printNewline() : () -> ()
+
+// -----
+
+func.func @print_scalar_f64(%arg0: f64) {
+  vector.print %arg0 : f64
+  return
+}
+// CHECK-LABEL: @print_scalar_f64(
+// CHECK-SAME: %[[A:.*]]: f64)
+//       CHECK:    llvm.call @printF64(%[[A]]) : (f64) -> ()
+//       CHECK:    llvm.call @printNewline() : () -> ()
+
+// -----
+
+// CHECK-LABEL: module {
+// CHECK: llvm.func @printString(!llvm.ptr)
+// CHECK: llvm.mlir.global private constant @[[GLOBAL_STR:.*]]({{.*}})
+// CHECK: @print_string
+//       CHECK-NEXT: %[[GLOBAL_ADDR:.*]] = llvm.mlir.addressof @[[GLOBAL_STR]] : !llvm.ptr
+//       CHECK-NEXT: %[[STR_PTR:.*]] = llvm.getelementptr %[[GLOBAL_ADDR]][0] : (!llvm.ptr) -> !llvm.ptr
+//       CHECK-NEXT: llvm.call @printString(%[[STR_PTR]]) : (!llvm.ptr) -> ()
+func.func @print_string() {
+  vector.print str "Hello, World!"
+  return
+}
+
+// -----
+
+//===----------------------------------------------------------------------===//
+// vector.reduction
+//===----------------------------------------------------------------------===//
+
+func.func @reduce_0d_f32(%arg0: vector<f32>) -> f32 {
+  %0 = vector.reduction <add>, %arg0 : vector<f32> into f32
+  return %0 : f32
+}
+// CHECK-LABEL: @reduce_0d_f32(
+// CHECK-SAME: %[[A:.*]]: vector<f32>)
+//      CHECK: %[[CA:.*]] = builtin.unrealized_conversion_cast %[[A]] : vector<f32> to vector<1xf32>
+//      CHECK: %[[C:.*]] = llvm.mlir.constant(0.000000e+00 : f32) : f32
+//      CHECK: %[[V:.*]] = "llvm.intr.vector.reduce.fadd"(%[[C]], %[[CA]])
+// CHECK-SAME: <{fastmathFlags = #llvm.fastmath<none>}> : (f32, vector<1xf32>) -> f32
+//      CHECK: return %[[V]] : f32
+
+// -----
+
+func.func @reduce_f16(%arg0: vector<16xf16>) -> f16 {
+  %0 = vector.reduction <add>, %arg0 : vector<16xf16> into f16
+  return %0 : f16
+}
+// CHECK-LABEL: @reduce_f16(
+// CHECK-SAME: %[[A:.*]]: vector<16xf16>)
+//      CHECK: %[[C:.*]] = llvm.mlir.constant(0.000000e+00 : f16) : f16
+//      CHECK: %[[V:.*]] = "llvm.intr.vector.reduce.fadd"(%[[C]], %[[A]])
+// CHECK-SAME: <{fastmathFlags = #llvm.fastmath<none>}> : (f16, vector<16xf16>) -> f16
+//      CHECK: return %[[V]] : f16
+
+// -----
+
+func.func @reduce_f16_scalable(%arg0: vector<[16]xf16>) -> f16 {
+  %0 = vector.reduction <add>, %arg0 : vector<[16]xf16> into f16
+  return %0 : f16
+}
+// CHECK-LABEL: @reduce_f16_scalable(
+// CHECK-SAME: %[[A:.*]]: vector<[16]xf16>)
+//      CHECK: %[[C:.*]] = llvm.mlir.constant(0.000000e+00 : f16) : f16
+//      CHECK: %[[V:.*]] = "llvm.intr.vector.reduce.fadd"(%[[C]], %[[A]])
+// CHECK-SAME: <{fastmathFlags = #llvm.fastmath<none>}> : (f16, vector<[16]xf16>) -> f16
+//      CHECK: return %[[V]] : f16
+
+// -----
+
+func.func @reduce_f32(%arg0: vector<16xf32>) -> f32 {
+  %0 = vector.reduction <add>, %arg0 : vector<16xf32> into f32
+  return %0 : f32
+}
+// CHECK-LABEL: @reduce_f32(
+// CHECK-SAME: %[[A:.*]]: vector<16xf32>)
+//      CHECK: %[[C:.*]] = llvm.mlir.constant(0.000000e+00 : f32) : f32
+//      CHECK: %[[V:.*]] = "llvm.intr.vector.reduce.fadd"(%[[C]], %[[A]])
+// CHECK-SAME: <{fastmathFlags = #llvm.fastmath<none>}> : (f32, vector<16xf32>) -> f32
+//      CHECK: return %[[V]] : f32
+
+// -----
+
+func.func @reduce_f32_scalable(%arg0: vector<[16]xf32>) -> f32 {
+  %0 = vector.reduction <add>, %arg0 : vector<[16]xf32> into f32
+  return %0 : f32
+}
+// CHECK-LABEL: @reduce_f32_scalable(
+// CHECK-SAME: %[[A:.*]]: vector<[16]xf32>)
+//      CHECK: %[[C:.*]] = llvm.mlir.constant(0.000000e+00 : f32) : f32
+//      CHECK: %[[V:.*]] = "llvm.intr.vector.reduce.fadd"(%[[C]], %[[A]])
+// CHECK-SAME: <{fastmathFlags = #llvm.fastmath<none>}> : (f32, vector<[16]xf32>) -> f32
+//      CHECK: return %[[V]] : f32
+
+// -----
+
+func.func @reduce_f64(%arg0: vector<16xf64>) -> f64 {
+  %0 = vector.reduction <add>, %arg0 : vector<16xf64> into f64
+  return %0 : f64
+}
+// CHECK-LABEL: @reduce_f64(
+// CHECK-SAME: %[[A:.*]]: vector<16xf64>)
+//      CHECK: %[[C:.*]] = llvm.mlir.constant(0.000000e+00 : f64) : f64
+//      CHECK: %[[V:.*]] = "llvm.intr.vector.reduce.fadd"(%[[C]], %[[A]])
+// CHECK-SAME: <{fastmathFlags = #llvm.fastmath<none>}> : (f64, vector<16xf64>) -> f64
+//      CHECK: return %[[V]] : f64
+
+// -----
+
+func.func @reduce_f64_scalable(%arg0: vector<[16]xf64>) -> f64 {
+  %0 = vector.reduction <add>, %arg0 : vector<[16]xf64> into f64
+  return %0 : f64
+}
+// CHECK-LABEL: @reduce_f64_scalable(
+// CHECK-SAME: %[[A:.*]]: vector<[16]xf64>)
+//      CHECK: %[[C:.*]] = llvm.mlir.constant(0.000000e+00 : f64) : f64
+//      CHECK: %[[V:.*]] = "llvm.intr.vector.reduce.fadd"(%[[C]], %[[A]])
+// CHECK-SAME: <{fastmathFlags = #llvm.fastmath<none>}> : (f64, vector<[16]xf64>) -> f64
+//      CHECK: return %[[V]] : f64
+
+// -----
+
+func.func @reduce_i8(%arg0: vector<16xi8>) -> i8 {
+  %0 = vector.reduction <add>, %arg0 : vector<16xi8> into i8
+  return %0 : i8
+}
+// CHECK-LABEL: @reduce_i8(
+// CHECK-SAME: %[[A:.*]]: vector<16xi8>)
+//      CHECK: %[[V:.*]] = "llvm.intr.vector.reduce.add"(%[[A]])
+//      CHECK: return %[[V]] : i8
+
+// -----
+
+func.func @reduce_i8_scalable(%arg0: vector<[16]xi8>) -> i8 {
+  %0 = vector.reduction <add>, %arg0 : vector<[16]xi8> into i8
+  return %0 : i8
+}
+// CHECK-LABEL: @reduce_i8_scalable(
+// CHECK-SAME: %[[A:.*]]: vector<[16]xi8>)
+//      CHECK: %[[V:.*]] = "llvm.intr.vector.reduce.add"(%[[A]])
+//      CHECK: return %[[V]] : i8
+
+// -----
+
+func.func @reduce_i32(%arg0: vector<16xi32>) -> i32 {
+  %0 = vector.reduction <add>, %arg0 : vector<16xi32> into i32
+  return %0 : i32
+}
+// CHECK-LABEL: @reduce_i32(
+// CHECK-SAME: %[[A:.*]]: vector<16xi32>)
+//      CHECK: %[[V:.*]] = "llvm.intr.vector.reduce.add"(%[[A]])
+//      CHECK: return %[[V]] : i32
+
+// -----
+
+func.func @reduce_i32_scalable(%arg0: vector<[16]xi32>) -> i32 {
+  %0 = vector.reduction <add>, %arg0 : vector<[16]xi32> into i32
+  return %0 : i32
+}
+// CHECK-LABEL: @reduce_i32_scalable(
+// CHECK-SAME: %[[A:.*]]: vector<[16]xi32>)
+//      CHECK: %[[V:.*]] = "llvm.intr.vector.reduce.add"(%[[A]])
+//      CHECK: return %[[V]] : i32
+
+// -----
+
+func.func @reduce_acc_i32(%arg0: vector<16xi32>, %arg1 : i32) -> i32 {
+  %0 = vector.reduction <add>, %arg0, %arg1 : vector<16xi32> into i32
+  return %0 : i32
+}
+// CHECK-LABEL: @reduce_acc_i32(
+//  CHECK-SAME: %[[A:.*]]: vector<16xi32>, %[[ACC:.*]]: i32)
+//       CHECK: %[[R:.*]] = "llvm.intr.vector.reduce.add"(%[[A]])
+//       CHECK: %[[V:.*]] = llvm.add %[[ACC]], %[[R]]
+//       CHECK: return %[[V]] : i32
+
+// -----
+
+func.func @reduce_acc_i32_scalable(%arg0: vector<[16]xi32>, %arg1 : i32) -> i32 {
+  %0 = vector.reduction <add>, %arg0, %arg1 : vector<[16]xi32> into i32
+  return %0 : i32
+}
+// CHECK-LABEL: @reduce_acc_i32_scalable(
+//  CHECK-SAME: %[[A:.*]]: vector<[16]xi32>, %[[ACC:.*]]: i32)
+//       CHECK: %[[R:.*]] = "llvm.intr.vector.reduce.add"(%[[A]])
+//       CHECK: %[[V:.*]] = llvm.add %[[ACC]], %[[R]]
+//       CHECK: return %[[V]] : i32
+
+// -----
+
+func.func @reduce_mul_i32(%arg0: vector<16xi32>) -> i32 {
+  %0 = vector.reduction <mul>, %arg0 : vector<16xi32> into i32
+  return %0 : i32
+}
+// CHECK-LABEL: @reduce_mul_i32(
+//  CHECK-SAME: %[[A:.*]]: vector<16xi32>)
+//       CHECK: %[[V:.*]] = "llvm.intr.vector.reduce.mul"(%[[A]])
+//       CHECK: return %[[V]] : i32
+
+// -----
+
+func.func @reduce_mul_i32_scalable(%arg0: vector<[16]xi32>) -> i32 {
+  %0 = vector.reduction <mul>, %arg0 : vector<[16]xi32> into i32
+  return %0 : i32
+}
+// CHECK-LABEL: @reduce_mul_i32_scalable(
+//  CHECK-SAME: %[[A:.*]]: vector<[16]xi32>)
+//       CHECK: %[[V:.*]] = "llvm.intr.vector.reduce.mul"(%[[A]])
+//       CHECK: return %[[V]] : i32
+
+// -----
+
+func.func @reduce_mul_acc_i32(%arg0: vector<16xi32>, %arg1 : i32) -> i32 {
+  %0 = vector.reduction <mul>, %arg0, %arg1 : vector<16xi32> into i32
+  return %0 : i32
+}
+// CHECK-LABEL: @reduce_mul_acc_i32(
+//  CHECK-SAME: %[[A:.*]]: vector<16xi32>, %[[ACC:.*]]: i32)
+//       CHECK: %[[R:.*]] = "llvm.intr.vector.reduce.mul"(%[[A]])
+//       CHECK: %[[V:.*]] = llvm.mul %[[ACC]], %[[R]]
+//       CHECK: return %[[V]] : i32
+
+// -----
+
+func.func @reduce_mul_acc_i32_scalable(%arg0: vector<[16]xi32>, %arg1 : i32) -> i32 {
+  %0 = vector.reduction <mul>, %arg0, %arg1 : vector<[16]xi32> into i32
+  return %0 : i32
+}
+// CHECK-LABEL: @reduce_mul_acc_i32_scalable(
+//  CHECK-SAME: %[[A:.*]]: vector<[16]xi32>, %[[ACC:.*]]: i32)
+//       CHECK: %[[R:.*]] = "llvm.intr.vector.reduce.mul"(%[[A]])
+//       CHECK: %[[V:.*]] = llvm.mul %[[ACC]], %[[R]]
+//       CHECK: return %[[V]] : i32
+
+// -----
+
+func.func @reduce_fmaximum_f32(%arg0: vector<16xf32>, %arg1: f32) -> f32 {
+  %0 = vector.reduction <maximumf>, %arg0, %arg1 : vector<16xf32> into f32
+  return %0 : f32
+}
+// CHECK-LABEL: @reduce_fmaximum_f32(
+// CHECK-SAME: %[[A:.*]]: vector<16xf32>, %[[B:.*]]: f32)
+//      CHECK: %[[V:.*]] = llvm.intr.vector.reduce.fmaximum(%[[A]]) : (vector<16xf32>) -> f32
+//      CHECK: %[[R:.*]] = llvm.intr.maximum(%[[V]], %[[B]]) : (f32, f32) -> f32
+//      CHECK: return %[[R]] : f32
+
+// -----
+
+func.func @reduce_fmaximum_f32_scalable(%arg0: vector<[16]xf32>, %arg1: f32) -> f32 {
+  %0 = vector.reduction <maximumf>, %arg0, %arg1 : vector<[16]xf32> into f32
+  return %0 : f32
+}
+// CHECK-LABEL: @reduce_fmaximum_f32_scalable(
+// CHECK-SAME: %[[A:.*]]: vector<[16]xf32>, %[[B:.*]]: f32)
+//      CHECK: %[[V:.*]] = llvm.intr.vector.reduce.fmaximum(%[[A]]) : (vector<[16]xf32>) -> f32
+//      CHECK: %[[R:.*]] = llvm.intr.maximum(%[[V]], %[[B]]) : (f32, f32) -> f32
+//      CHECK: return %[[R]] : f32
+
+// -----
+
+func.func @reduce_fminimum_f32(%arg0: vector<16xf32>, %arg1: f32) -> f32 {
+  %0 = vector.reduction <minimumf>, %arg0, %arg1 : vector<16xf32> into f32
+  return %0 : f32
+}
+// CHECK-LABEL: @reduce_fminimum_f32(
+// CHECK-SAME: %[[A:.*]]: vector<16xf32>, %[[B:.*]]: f32)
+//      CHECK: %[[V:.*]] = llvm.intr.vector.reduce.fminimum(%[[A]]) : (vector<16xf32>) -> f32
+//      CHECK: %[[R:.*]] = llvm.intr.minimum(%[[V]], %[[B]]) : (f32, f32) -> f32
+//      CHECK: return %[[R]] : f32
+
+// -----
+
+func.func @reduce_fminimum_f32_scalable(%arg0: vector<[16]xf32>, %arg1: f32) -> f32 {
+  %0 = vector.reduction <minimumf>, %arg0, %arg1 : vector<[16]xf32> into f32
+  return %0 : f32
+}
+// CHECK-LABEL: @reduce_fminimum_f32_scalable(
+// CHECK-SAME: %[[A:.*]]: vector<[16]xf32>, %[[B:.*]]: f32)
+//      CHECK: %[[V:.*]] = llvm.intr.vector.reduce.fminimum(%[[A]]) : (vector<[16]xf32>) -> f32
+//      CHECK: %[[R:.*]] = llvm.intr.minimum(%[[V]], %[[B]]) : (f32, f32) -> f32
+//      CHECK: return %[[R]] : f32
+
+// -----
+
+func.func @reduce_fmax_f32(%arg0: vector<16xf32>, %arg1: f32) -> f32 {
+  %0 = vector.reduction <maxnumf>, %arg0, %arg1 : vector<16xf32> into f32
+  return %0 : f32
+}
+// CHECK-LABEL: @reduce_fmax_f32(
+// CHECK-SAME: %[[A:.*]]: vector<16xf32>, %[[B:.*]]: f32)
+//      CHECK: %[[V:.*]] = llvm.intr.vector.reduce.fmax(%[[A]]) : (vector<16xf32>) -> f32
+//      CHECK: %[[R:.*]] = llvm.intr.maxnum(%[[V]], %[[B]]) : (f32, f32) -> f32
+//      CHECK: return %[[R]] : f32
+
+// -----
+
+func.func @reduce_fmax_f32_scalable(%arg0: vector<[16]xf32>, %arg1: f32) -> f32 {
+  %0 = vector.reduction <maxnumf>, %arg0, %arg1 : vector<[16]xf32> into f32
+  return %0 : f32
+}
+// CHECK-LABEL: @reduce_fmax_f32_scalable(
+// CHECK-SAME: %[[A:.*]]: vector<[16]xf32>, %[[B:.*]]: f32)
+//      CHECK: %[[V:.*]] = llvm.intr.vector.reduce.fmax(%[[A]]) : (vector<[16]xf32>) -> f32
+//      CHECK: %[[R:.*]] = llvm.intr.maxnum(%[[V]], %[[B]]) : (f32, f32) -> f32
+//      CHECK: return %[[R]] : f32
+
+// -----
+
+func.func @reduce_fmin_f32(%arg0: vector<16xf32>, %arg1: f32) -> f32 {
+  %0 = vector.reduction <minnumf>, %arg0, %arg1 : vector<16xf32> into f32
+  return %0 : f32
+}
+// CHECK-LABEL: @reduce_fmin_f32(
+// CHECK-SAME: %[[A:.*]]: vector<16xf32>, %[[B:.*]]: f32)
+//      CHECK: %[[V:.*]] = llvm.intr.vector.reduce.fmin(%[[A]]) : (vector<16xf32>) -> f32
+//      CHECK: %[[R:.*]] = llvm.intr.minnum(%[[V]], %[[B]]) : (f32, f32) -> f32
+//      CHECK: return %[[R]] : f32
+
+// -----
+
+func.func @reduce_fmin_f32_scalable(%arg0: vector<[16]xf32>, %arg1: f32) -> f32 {
+  %0 = vector.reduction <minnumf>, %arg0, %arg1 : vector<[16]xf32> into f32
+  return %0 : f32
+}
+// CHECK-LABEL: @reduce_fmin_f32_scalable(
+// CHECK-SAME: %[[A:.*]]: vector<[16]xf32>, %[[B:.*]]: f32)
+//      CHECK: %[[V:.*]] = llvm.intr.vector.reduce.fmin(%[[A]]) : (vector<[16]xf32>) -> f32
+//      CHECK: %[[R:.*]] = llvm.intr.minnum(%[[V]], %[[B]]) : (f32, f32) -> f32
+//      CHECK: return %[[R]] : f32
+
+// -----
+
+func.func @reduce_minui_i32(%arg0: vector<16xi32>) -> i32 {
+  %0 = vector.reduction <minui>, %arg0 : vector<16xi32> into i32
+  return %0 : i32
+}
+// CHECK-LABEL: @reduce_minui_i32(
+//  CHECK-SAME: %[[A:.*]]: vector<16xi32>)
+//       CHECK: %[[V:.*]] = "llvm.intr.vector.reduce.umin"(%[[A]])
+//       CHECK: return %[[V]] : i32
+
+// -----
+
+func.func @reduce_minui_i32_scalable(%arg0: vector<[16]xi32>) -> i32 {
+  %0 = vector.reduction <minui>, %arg0 : vector<[16]xi32> into i32
+  return %0 : i32
+}
+// CHECK-LABEL: @reduce_minui_i32_scalable(
+//  CHECK-SAME: %[[A:.*]]: vector<[16]xi32>)
+//       CHECK: %[[V:.*]] = "llvm.intr.vector.reduce.umin"(%[[A]])
+//       CHECK: return %[[V]] : i32
+
+// -----
+
+func.func @reduce_minui_acc_i32(%arg0: vector<16xi32>, %arg1 : i32) -> i32 {
+  %0 = vector.reduction <minui>, %arg0, %arg1 : vector<16xi32> into i32
+  return %0 : i32
+}
+// CHECK-LABEL: @reduce_minui_acc_i32(
+//  CHECK-SAME: %[[A:.*]]: vector<16xi32>, %[[ACC:.*]]: i32)
+//       CHECK: %[[R:.*]] = "llvm.intr.vector.reduce.umin"(%[[A]])
+//       CHECK: %[[S:.*]] = llvm.icmp "ule" %[[ACC]], %[[R]]
+//       CHECK: %[[V:.*]] = llvm.select %[[S]], %[[ACC]], %[[R]]
+//       CHECK: return %[[V]] : i32
+
+// -----
+
+func.func @reduce_minui_acc_i32_scalable(%arg0: vector<[16]xi32>, %arg1 : i32) -> i32 {
+  %0 = vector.reduction <minui>, %arg0, %arg1 : vector<[16]xi32> into i32
+  return %0 : i32
+}
+// CHECK-LABEL: @reduce_minui_acc_i32_scalable(
+//  CHECK-SAME: %[[A:.*]]: vector<[16]xi32>, %[[ACC:.*]]: i32)
+//       CHECK: %[[R:.*]] = "llvm.intr.vector.reduce.umin"(%[[A]])
+//       CHECK: %[[S:.*]] = llvm.icmp "ule" %[[ACC]], %[[R]]
+//       CHECK: %[[V:.*]] = llvm.select %[[S]], %[[ACC]], %[[R]]
+//       CHECK: return %[[V]] : i32
+
+// -----
+
+func.func @reduce_maxui_i32(%arg0: vector<16xi32>) -> i32 {
+  %0 = vector.reduction <maxui>, %arg0 : vector<16xi32> into i32
+  return %0 : i32
+}
+// CHECK-LABEL: @reduce_maxui_i32(
+//  CHECK-SAME: %[[A:.*]]: vector<16xi32>)
+//       CHECK: %[[V:.*]] = "llvm.intr.vector.reduce.umax"(%[[A]])
+//       CHECK: return %[[V]] : i32
+
+// -----
+
+func.func @reduce_maxui_i32_scalable(%arg0: vector<[16]xi32>) -> i32 {
+  %0 = vector.reduction <maxui>, %arg0 : vector<[16]xi32> into i32
+  return %0 : i32
+}
+// CHECK-LABEL: @reduce_maxui_i32_scalable(
+//  CHECK-SAME: %[[A:.*]]: vector<[16]xi32>)
+//       CHECK: %[[V:.*]] = "llvm.intr.vector.reduce.umax"(%[[A]])
+//       CHECK: return %[[V]] : i32
+
+// -----
+
+func.func @reduce_maxui_acc_i32(%arg0: vector<16xi32>, %arg1 : i32) -> i32 {
+  %0 = vector.reduction <maxui>, %arg0, %arg1 : vector<16xi32> into i32
+  return %0 : i32
+}
+// CHECK-LABEL: @reduce_maxui_acc_i32(
+//  CHECK-SAME: %[[A:.*]]: vector<16xi32>, %[[ACC:.*]]: i32)
+//       CHECK: %[[R:.*]] = "llvm.intr.vector.reduce.umax"(%[[A]])
+//       CHECK: %[[S:.*]] = llvm.icmp "uge" %[[ACC]], %[[R]]
+//       CHECK: %[[V:.*]] = llvm.select %[[S]], %[[ACC]], %[[R]]
+//       CHECK: return %[[V]] : i32
+
+// -----
+
+func.func @reduce_maxui_acc_i32_scalable(%arg0: vector<[16]xi32>, %arg1 : i32) -> i32 {
+  %0 = vector.reduction <maxui>, %arg0, %arg1 : vector<[16]xi32> into i32
+  return %0 : i32
+}
+// CHECK-LABEL: @reduce_maxui_acc_i32_scalable(
+//  CHECK-SAME: %[[A:.*]]: vector<[16]xi32>, %[[ACC:.*]]: i32)
+//       CHECK: %[[R:.*]] = "llvm.intr.vector.reduce.umax"(%[[A]])
+//       CHECK: %[[S:.*]] = llvm.icmp "uge" %[[ACC]], %[[R]]
+//       CHECK: %[[V:.*]] = llvm.select %[[S]], %[[ACC]], %[[R]]
+//       CHECK: return %[[V]] : i32
+
+// -----
+
+func.func @reduce_minsi_i32(%arg0: vector<16xi32>) -> i32 {
+  %0 = vector.reduction <minsi>, %arg0 : vector<16xi32> into i32
+  return %0 : i32
+}
+// CHECK-LABEL: @reduce_minsi_i32(
+//  CHECK-SAME: %[[A:.*]]: vector<16xi32>)
+//       CHECK: %[[V:.*]] = "llvm.intr.vector.reduce.smin"(%[[A]])
+//       CHECK: return %[[V]] : i32
+
+// -----
+
+func.func @reduce_minsi_i32_scalable(%arg0: vector<[16]xi32>) -> i32 {
+  %0 = vector.reduction <minsi>, %arg0 : vector<[16]xi32> into i32
+  return %0 : i32
+}
+// CHECK-LABEL: @reduce_minsi_i32_scalable(
+//  CHECK-SAME: %[[A:.*]]: vector<[16]xi32>)
+//       CHECK: %[[V:.*]] = "llvm.intr.vector.reduce.smin"(%[[A]])
+//       CHECK: return %[[V]] : i32
+
+// -----
+
+func.func @reduce_minsi_acc_i32(%arg0: vector<16xi32>, %arg1 : i32) -> i32 {
+  %0 = vector.reduction <minsi>, %arg0, %arg1 : vector<16xi32> into i32
+  return %0 : i32
+}
+// CHECK-LABEL: @reduce_minsi_acc_i32(
+//  CHECK-SAME: %[[A:.*]]: vector<16xi32>, %[[ACC:.*]]: i32)
+//       CHECK: %[[R:.*]] = "llvm.intr.vector.reduce.smin"(%[[A]])
+//       CHECK: %[[S:.*]] = llvm.icmp "sle" %[[ACC]], %[[R]]
+//       CHECK: %[[V:.*]] = llvm.select %[[S]], %[[ACC]], %[[R]]
+//       CHECK: return %[[V]] : i32
+
+// -----
+
+func.func @reduce_minsi_acc_i32_scalable(%arg0: vector<[16]xi32>, %arg1 : i32) -> i32 {
+  %0 = vector.reduction <minsi>, %arg0, %arg1 : vector<[16]xi32> into i32
+  return %0 : i32
+}
+// CHECK-LABEL: @reduce_minsi_acc_i32_scalable(
+//  CHECK-SAME: %[[A:.*]]: vector<[16]xi32>, %[[ACC:.*]]: i32)
+//       CHECK: %[[R:.*]] = "llvm.intr.vector.reduce.smin"(%[[A]])
+//       CHECK: %[[S:.*]] = llvm.icmp "sle" %[[ACC]], %[[R]]
+//       CHECK: %[[V:.*]] = llvm.select %[[S]], %[[ACC]], %[[R]]
+//       CHECK: return %[[V]] : i32
+
+// -----
+
+func.func @reduce_maxsi_i32(%arg0: vector<16xi32>) -> i32 {
+  %0 = vector.reduction <maxsi>, %arg0 : vector<16xi32> into i32
+  return %0 : i32
+}
+// CHECK-LABEL: @reduce_maxsi_i32(
+//  CHECK-SAME: %[[A:.*]]: vector<16xi32>)
+//       CHECK: %[[V:.*]] = "llvm.intr.vector.reduce.smax"(%[[A]])
+//       CHECK: return %[[V]] : i32
+
+// -----
+
+func.func @reduce_maxsi_i32_scalable(%arg0: vector<[16]xi32>) -> i32 {
+  %0 = vector.reduction <maxsi>, %arg0 : vector<[16]xi32> into i32
+  return %0 : i32
+}
+// CHECK-LABEL: @reduce_maxsi_i32_scalable(
+//  CHECK-SAME: %[[A:.*]]: vector<[16]xi32>)
+//       CHECK: %[[V:.*]] = "llvm.intr.vector.reduce.smax"(%[[A]])
+//       CHECK: return %[[V]] : i32
+
+// -----
+
+func.func @reduce_maxsi_acc_i32(%arg0: vector<16xi32>, %arg1 : i32) -> i32 {
+  %0 = vector.reduction <maxsi>, %arg0, %arg1 : vector<16xi32> into i32
+  return %0 : i32
+}
+// CHECK-LABEL: @reduce_maxsi_acc_i32(
+//  CHECK-SAME: %[[A:.*]]: vector<16xi32>, %[[ACC:.*]]: i32)
+//       CHECK: %[[R:.*]] = "llvm.intr.vector.reduce.smax"(%[[A]])
+//       CHECK: %[[S:.*]] = llvm.icmp "sge" %[[ACC]], %[[R]]
+//       CHECK: %[[V:.*]] = llvm.select %[[S]], %[[ACC]], %[[R]]
+//       CHECK: return %[[V]] : i32
+
+// -----
+
+func.func @reduce_maxsi_acc_i32_scalable(%arg0: vector<[16]xi32>, %arg1 : i32) -> i32 {
+  %0 = vector.reduction <maxsi>, %arg0, %arg1 : vector<[16]xi32> into i32
+  return %0 : i32
+}
+// CHECK-LABEL: @reduce_maxsi_acc_i32_scalable(
+//  CHECK-SAME: %[[A:.*]]: vector<[16]xi32>, %[[ACC:.*]]: i32)
+//       CHECK: %[[R:.*]] = "llvm.intr.vector.reduce.smax"(%[[A]])
+//       CHECK: %[[S:.*]] = llvm.icmp "sge" %[[ACC]], %[[R]]
+//       CHECK: %[[V:.*]] = llvm.select %[[S]], %[[ACC]], %[[R]]
+//       CHECK: return %[[V]] : i32
+
+// -----
+
+func.func @reduce_and_i32(%arg0: vector<16xi32>) -> i32 {
+  %0 = vector.reduction <and>, %arg0 : vector<16xi32> into i32
+  return %0 : i32
+}
+// CHECK-LABEL: @reduce_and_i32(
+//  CHECK-SAME: %[[A:.*]]: vector<16xi32>)
+//       CHECK: %[[V:.*]] = "llvm.intr.vector.reduce.and"(%[[A]])
+//       CHECK: return %[[V]] : i32
+
+// -----
+
+func.func @reduce_and_i32_scalable(%arg0: vector<[16]xi32>) -> i32 {
+  %0 = vector.reduction <and>, %arg0 : vector<[16]xi32> into i32
+  return %0 : i32
+}
+// CHECK-LABEL: @reduce_and_i32_scalable(
+//  CHECK-SAME: %[[A:.*]]: vector<[16]xi32>)
+//       CHECK: %[[V:.*]] = "llvm.intr.vector.reduce.and"(%[[A]])
+//       CHECK: return %[[V]] : i32
+
+// -----
+
+func.func @reduce_and_acc_i32(%arg0: vector<16xi32>, %arg1 : i32) -> i32 {
+  %0 = vector.reduction <and>, %arg0, %arg1 : vector<16xi32> into i32
+  return %0 : i32
+}
+// CHECK-LABEL: @reduce_and_acc_i32(
+//  CHECK-SAME: %[[A:.*]]: vector<16xi32>, %[[ACC:.*]]: i32)
+//       CHECK: %[[R:.*]] = "llvm.intr.vector.reduce.and"(%[[A]])
+//       CHECK: %[[V:.*]] = llvm.and %[[ACC]], %[[R]]
+//       CHECK: return %[[V]] : i32
+
+// -----
+
+func.func @reduce_and_acc_i32_scalable(%arg0: vector<[16]xi32>, %arg1 : i32) -> i32 {
+  %0 = vector.reduction <and>, %arg0, %arg1 : vector<[16]xi32> into i32
+  return %0 : i32
+}
+// CHECK-LABEL: @reduce_and_acc_i32_scalable(
+//  CHECK-SAME: %[[A:.*]]: vector<[16]xi32>, %[[ACC:.*]]: i32)
+//       CHECK: %[[R:.*]] = "llvm.intr.vector.reduce.and"(%[[A]])
+//       CHECK: %[[V:.*]] = llvm.and %[[ACC]], %[[R]]
+//       CHECK: return %[[V]] : i32
+
+// -----
+
+func.func @reduce_or_i32(%arg0: vector<16xi32>) -> i32 {
+  %0 = vector.reduction <or>, %arg0 : vector<16xi32> into i32
+  return %0 : i32
+}
+// CHECK-LABEL: @reduce_or_i32(
+//  CHECK-SAME: %[[A:.*]]: vector<16xi32>)
+//       CHECK: %[[V:.*]] = "llvm.intr.vector.reduce.or"(%[[A]])
+//       CHECK: return %[[V]] : i32
+
+// -----
+
+func.func @reduce_or_i32_scalable(%arg0: vector<[16]xi32>) -> i32 {
+  %0 = vector.reduction <or>, %arg0 : vector<[16]xi32> into i32
+  return %0 : i32
+}
+// CHECK-LABEL: @reduce_or_i32_scalable(
+//  CHECK-SAME: %[[A:.*]]: vector<[16]xi32>)
+//       CHECK: %[[V:.*]] = "llvm.intr.vector.reduce.or"(%[[A]])
+//       CHECK: return %[[V]] : i32
+
+// -----
+
+func.func @reduce_or_acc_i32(%arg0: vector<16xi32>, %arg1 : i32) -> i32 {
+  %0 = vector.reduction <or>, %arg0, %arg1 : vector<16xi32> into i32
+  return %0 : i32
+}
+// CHECK-LABEL: @reduce_or_acc_i32(
+//  CHECK-SAME: %[[A:.*]]: vector<16xi32>, %[[ACC:.*]]: i32)
+//       CHECK: %[[R:.*]] = "llvm.intr.vector.reduce.or"(%[[A]])
+//       CHECK: %[[V:.*]] = llvm.or %[[ACC]], %[[R]]
+//       CHECK: return %[[V]] : i32
+
+// -----
+
+func.func @reduce_or_acc_i32_scalable(%arg0: vector<[16]xi32>, %arg1 : i32) -> i32 {
+  %0 = vector.reduction <or>, %arg0, %arg1 : vector<[16]xi32> into i32
+  return %0 : i32
+}
+// CHECK-LABEL: @reduce_or_acc_i32_scalable(
+//  CHECK-SAME: %[[A:.*]]: vector<[16]xi32>, %[[ACC:.*]]: i32)
+//       CHECK: %[[R:.*]] = "llvm.intr.vector.reduce.or"(%[[A]])
+//       CHECK: %[[V:.*]] = llvm.or %[[ACC]], %[[R]]
+//       CHECK: return %[[V]] : i32
+
+// -----
+
+func.func @reduce_xor_i32(%arg0: vector<16xi32>) -> i32 {
+  %0 = vector.reduction <xor>, %arg0 : vector<16xi32> into i32
+  return %0 : i32
+}
+// CHECK-LABEL: @reduce_xor_i32(
+//  CHECK-SAME: %[[A:.*]]: vector<16xi32>)
+//       CHECK: %[[V:.*]] = "llvm.intr.vector.reduce.xor"(%[[A]])
+//       CHECK: return %[[V]] : i32
+
+// -----
+
+func.func @reduce_xor_i32_scalable(%arg0: vector<[16]xi32>) -> i32 {
+  %0 = vector.reduction <xor>, %arg0 : vector<[16]xi32> into i32
+  return %0 : i32
+}
+// CHECK-LABEL: @reduce_xor_i32_scalable(
+//  CHECK-SAME: %[[A:.*]]: vector<[16]xi32>)
+//       CHECK: %[[V:.*]] = "llvm.intr.vector.reduce.xor"(%[[A]])
+//       CHECK: return %[[V]] : i32
+
+// -----
+
+func.func @reduce_xor_acc_i32(%arg0: vector<16xi32>, %arg1 : i32) -> i32 {
+  %0 = vector.reduction <xor>, %arg0, %arg1 : vector<16xi32> into i32
+  return %0 : i32
+}
+// CHECK-LABEL: @reduce_xor_acc_i32(
+//  CHECK-SAME: %[[A:.*]]: vector<16xi32>, %[[ACC:.*]]: i32)
+//       CHECK: %[[R:.*]] = "llvm.intr.vector.reduce.xor"(%[[A]])
+//       CHECK: %[[V:.*]] = llvm.xor %[[ACC]], %[[R]]
+//       CHECK: return %[[V]] : i32
+
+// -----
+
+func.func @reduce_xor_acc_i32_scalable(%arg0: vector<[16]xi32>, %arg1 : i32) -> i32 {
+  %0 = vector.reduction <xor>, %arg0, %arg1 : vector<[16]xi32> into i32
+  return %0 : i32
+}
+// CHECK-LABEL: @reduce_xor_acc_i32_scalable(
+//  CHECK-SAME: %[[A:.*]]: vector<[16]xi32>, %[[ACC:.*]]: i32)
+//       CHECK: %[[R:.*]] = "llvm.intr.vector.reduce.xor"(%[[A]])
+//       CHECK: %[[V:.*]] = llvm.xor %[[ACC]], %[[R]]
+//       CHECK: return %[[V]] : i32
+
+// -----
+
+func.func @reduce_i64(%arg0: vector<16xi64>) -> i64 {
+  %0 = vector.reduction <add>, %arg0 : vector<16xi64> into i64
+  return %0 : i64
+}
+// CHECK-LABEL: @reduce_i64(
+// CHECK-SAME: %[[A:.*]]: vector<16xi64>)
+//      CHECK: %[[V:.*]] = "llvm.intr.vector.reduce.add"(%[[A]])
+//      CHECK: return %[[V]] : i64
+
+// -----
+
+func.func @reduce_i64_scalable(%arg0: vector<[16]xi64>) -> i64 {
+  %0 = vector.reduction <add>, %arg0 : vector<[16]xi64> into i64
+  return %0 : i64
+}
+// CHECK-LABEL: @reduce_i64_scalable(
+// CHECK-SAME: %[[A:.*]]: vector<[16]xi64>)
+//      CHECK: %[[V:.*]] = "llvm.intr.vector.reduce.add"(%[[A]])
+//      CHECK: return %[[V]] : i64
+
+// -----
+
+func.func @reduce_index(%arg0: vector<16xindex>) -> index {
+  %0 = vector.reduction <add>, %arg0 : vector<16xindex> into index
+  return %0 : index
+}
+// CHECK-LABEL: @reduce_index(
+// CHECK-SAME: %[[A:.*]]: vector<16xindex>)
+//      CHECK: %[[T0:.*]] = builtin.unrealized_conversion_cast %[[A]] : vector<16xindex> to vector<16xi64>
+//      CHECK: %[[T1:.*]] = "llvm.intr.vector.reduce.add"(%[[T0]])
+//      CHECK: %[[T2:.*]] = builtin.unrealized_conversion_cast %[[T1]] : i64 to index
+//      CHECK: return %[[T2]] : index
+
+// -----
+
+func.func @reduce_index_scalable(%arg0: vector<[16]xindex>) -> index {
+  %0 = vector.reduction <add>, %arg0 : vector<[16]xindex> into index
+  return %0 : index
+}
+// CHECK-LABEL: @reduce_index_scalable(
+// CHECK-SAME: %[[A:.*]]: vector<[16]xindex>)
+//      CHECK: %[[T0:.*]] = builtin.unrealized_conversion_cast %[[A]] : vector<[16]xindex> to vector<[16]xi64>
+//      CHECK: %[[T1:.*]] = "llvm.intr.vector.reduce.add"(%[[T0]])
+//      CHECK: %[[T2:.*]] = builtin.unrealized_conversion_cast %[[T1]] : i64 to index
+//      CHECK: return %[[T2]] : index
+
+// -----
+
+//===----------------------------------------------------------------------===//
+// vector.transpose
+//===----------------------------------------------------------------------===//
+
+func.func @transpose_0d(%arg0: vector<f32>) -> vector<f32> {
+  %0 = vector.transpose %arg0, [] : vector<f32> to vector<f32>
+  return %0 : vector<f32>
+}
+
+// CHECK-LABEL: func @transpose_0d
+// CHECK-SAME:  %[[A:.*]]: vector<f32>
+// CHECK:       return %[[A]] : vector<f32>
+
+// -----
+
+//===----------------------------------------------------------------------===//
+// vector.load
+//===----------------------------------------------------------------------===//
+
+func.func @load(%memref : memref<200x100xf32>, %i : index, %j : index) -> vector<8xf32> {
+  %0 = vector.load %memref[%i, %j] : memref<200x100xf32>, vector<8xf32>
+  return %0 : vector<8xf32>
+}
+
+// CHECK-LABEL: func @load
+// CHECK: %[[C100:.*]] = llvm.mlir.constant(100 : index) : i64
+// CHECK: %[[MUL:.*]] = llvm.mul %{{.*}}, %[[C100]]  : i64
+// CHECK: %[[ADD:.*]] = llvm.add %[[MUL]], %{{.*}}  : i64
+// CHECK: %[[GEP:.*]] = llvm.getelementptr %{{.*}}[%[[ADD]]] : (!llvm.ptr, i64) -> !llvm.ptr, f32
+// CHECK: llvm.load %[[GEP]] {alignment = 4 : i64} : !llvm.ptr -> vector<8xf32>
+
+// -----
+
+func.func @load_scalable(%memref : memref<200x100xf32>, %i : index, %j : index) -> vector<[8]xf32> {
+  %0 = vector.load %memref[%i, %j] : memref<200x100xf32>, vector<[8]xf32>
+  return %0 : vector<[8]xf32>
+}
+
+// CHECK-LABEL: func @load_scalable
+// CHECK: %[[C100:.*]] = llvm.mlir.constant(100 : index) : i64
+// CHECK: %[[MUL:.*]] = llvm.mul %{{.*}}, %[[C100]]  : i64
+// CHECK: %[[ADD:.*]] = llvm.add %[[MUL]], %{{.*}}  : i64
+// CHECK: %[[GEP:.*]] = llvm.getelementptr %{{.*}}[%[[ADD]]] : (!llvm.ptr, i64) -> !llvm.ptr, f32
+// CHECK: llvm.load %[[GEP]] {alignment = 4 : i64} : !llvm.ptr -> vector<[8]xf32>
+
+// -----
+
+func.func @load_nontemporal(%memref : memref<200x100xf32>, %i : index, %j : index) -> vector<8xf32> {
+  %0 = vector.load %memref[%i, %j] {nontemporal = true} : memref<200x100xf32>, vector<8xf32>
+  return %0 : vector<8xf32>
+}
+
+// CHECK-LABEL: func @load_nontemporal
+// CHECK: %[[C100:.*]] = llvm.mlir.constant(100 : index) : i64
+// CHECK: %[[MUL:.*]] = llvm.mul %{{.*}}, %[[C100]]  : i64
+// CHECK: %[[ADD:.*]] = llvm.add %[[MUL]], %{{.*}}  : i64
+// CHECK: %[[GEP:.*]] = llvm.getelementptr %{{.*}}[%[[ADD]]] : (!llvm.ptr, i64) -> !llvm.ptr, f32
+// CHECK: llvm.load %[[GEP]] {alignment = 4 : i64, nontemporal} : !llvm.ptr -> vector<8xf32>
+
+// -----
+
+func.func @load_nontemporal_scalable(%memref : memref<200x100xf32>, %i : index, %j : index) -> vector<[8]xf32> {
+  %0 = vector.load %memref[%i, %j] {nontemporal = true} : memref<200x100xf32>, vector<[8]xf32>
+  return %0 : vector<[8]xf32>
+}
+
+// CHECK-LABEL: func @load_nontemporal_scalable
+// CHECK: %[[C100:.*]] = llvm.mlir.constant(100 : index) : i64
+// CHECK: %[[MUL:.*]] = llvm.mul %{{.*}}, %[[C100]]  : i64
+// CHECK: %[[ADD:.*]] = llvm.add %[[MUL]], %{{.*}}  : i64
+// CHECK: %[[GEP:.*]] = llvm.getelementptr %{{.*}}[%[[ADD]]] : (!llvm.ptr, i64) -> !llvm.ptr, f32
+// CHECK: llvm.load %[[GEP]] {alignment = 4 : i64, nontemporal} : !llvm.ptr -> vector<[8]xf32>
+
+// -----
+
+func.func @load_index(%memref : memref<200x100xindex>, %i : index, %j : index) -> vector<8xindex> {
+  %0 = vector.load %memref[%i, %j] : memref<200x100xindex>, vector<8xindex>
+  return %0 : vector<8xindex>
+}
+// CHECK-LABEL: func @load_index
+// CHECK: %[[T0:.*]] = llvm.load %{{.*}} {alignment = 8 : i64} : !llvm.ptr -> vector<8xi64>
+// CHECK: %[[T1:.*]] = builtin.unrealized_conversion_cast %[[T0]] : vector<8xi64> to vector<8xindex>
+// CHECK: return %[[T1]] : vector<8xindex>
+
+// -----
+
+func.func @load_index_scalable(%memref : memref<200x100xindex>, %i : index, %j : index) -> vector<[8]xindex> {
+  %0 = vector.load %memref[%i, %j] : memref<200x100xindex>, vector<[8]xindex>
+  return %0 : vector<[8]xindex>
+}
+// CHECK-LABEL: func @load_index_scalable
+// CHECK: %[[T0:.*]] = llvm.load %{{.*}} {alignment = 8 : i64} : !llvm.ptr -> vector<[8]xi64>
+// CHECK: %[[T1:.*]] = builtin.unrealized_conversion_cast %[[T0]] : vector<[8]xi64> to vector<[8]xindex>
+// CHECK: return %[[T1]] : vector<[8]xindex>
+
+// -----
+
+func.func @load_0d(%memref : memref<200x100xf32>, %i : index, %j : index) -> vector<f32> {
+  %0 = vector.load %memref[%i, %j] : memref<200x100xf32>, vector<f32>
+  return %0 : vector<f32>
+}
+
+// CHECK-LABEL: func @load_0d
+// CHECK: %[[J:.*]] = builtin.unrealized_conversion_cast %{{.*}} : index to i64
+// CHECK: %[[I:.*]] = builtin.unrealized_conversion_cast %{{.*}} : index to i64
+// CHECK: %[[CAST_MEMREF:.*]] = builtin.unrealized_conversion_cast %{{.*}} : memref<200x100xf32> to !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)>
+// CHECK: %[[REF:.*]] = llvm.extractvalue %[[CAST_MEMREF]][1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)>
+// CHECK: %[[C100:.*]] = llvm.mlir.constant(100 : index) : i64
+// CHECK: %[[MUL:.*]] = llvm.mul %[[I]], %[[C100]] : i64
+// CHECK: %[[ADD:.*]] = llvm.add %[[MUL]], %[[J]] : i64
+// CHECK: %[[ADDR:.*]] = llvm.getelementptr %[[REF]][%[[ADD]]] : (!llvm.ptr, i64) -> !llvm.ptr, f32
+// CHECK: %[[LOAD:.*]] = llvm.load %[[ADDR]] {alignment = 4 : i64} : !llvm.ptr -> vector<1xf32>
+// CHECK: %[[RES:.*]] = builtin.unrealized_conversion_cast %[[LOAD]] : vector<1xf32> to vector<f32>
+// CHECK: return %[[RES]] : vector<f32>
+
+// -----
+
+//===----------------------------------------------------------------------===//
+// vector.store
+//===----------------------------------------------------------------------===//
+
+func.func @store(%memref : memref<200x100xf32>, %i : index, %j : index) {
+  %val = arith.constant dense<11.0> : vector<4xf32>
+  vector.store %val, %memref[%i, %j] : memref<200x100xf32>, vector<4xf32>
+  return
+}
+
+// CHECK-LABEL: func @store
+// CHECK: %[[C100:.*]] = llvm.mlir.constant(100 : index) : i64
+// CHECK: %[[MUL:.*]] = llvm.mul %{{.*}}, %[[C100]]  : i64
+// CHECK: %[[ADD:.*]] = llvm.add %[[MUL]], %{{.*}}  : i64
+// CHECK: %[[GEP:.*]] = llvm.getelementptr %{{.*}}[%[[ADD]]] : (!llvm.ptr, i64) -> !llvm.ptr, f32
+// CHECK: llvm.store %{{.*}}, %[[GEP]] {alignment = 4 : i64} :  vector<4xf32>, !llvm.ptr
+
+// -----
+
+func.func @store_scalable(%memref : memref<200x100xf32>, %i : index, %j : index) {
+  %val = arith.constant dense<11.0> : vector<[4]xf32>
+  vector.store %val, %memref[%i, %j] : memref<200x100xf32>, vector<[4]xf32>
+  return
+}
+
+// CHECK-LABEL: func @store_scalable
+// CHECK: %[[C100:.*]] = llvm.mlir.constant(100 : index) : i64
+// CHECK: %[[MUL:.*]] = llvm.mul %{{.*}}, %[[C100]]  : i64
+// CHECK: %[[ADD:.*]] = llvm.add %[[MUL]], %{{.*}}  : i64
+// CHECK: %[[GEP:.*]] = llvm.getelementptr %{{.*}}[%[[ADD]]] : (!llvm.ptr, i64) -> !llvm.ptr, f32
+// CHECK: llvm.store %{{.*}}, %[[GEP]] {alignment = 4 : i64} :  vector<[4]xf32>, !llvm.ptr
+
+// -----
+
+func.func @store_nontemporal(%memref : memref<200x100xf32>, %i : index, %j : index) {
+  %val = arith.constant dense<11.0> : vector<4xf32>
+  vector.store %val, %memref[%i, %j] {nontemporal = true} : memref<200x100xf32>, vector<4xf32>
+  return
+}
+
+// CHECK-LABEL: func @store_nontemporal
+// CHECK: %[[C100:.*]] = llvm.mlir.constant(100 : index) : i64
+// CHECK: %[[MUL:.*]] = llvm.mul %{{.*}}, %[[C100]]  : i64
+// CHECK: %[[ADD:.*]] = llvm.add %[[MUL]], %{{.*}}  : i64
+// CHECK: %[[GEP:.*]] = llvm.getelementptr %{{.*}}[%[[ADD]]] : (!llvm.ptr, i64) -> !llvm.ptr, f32
+// CHECK: llvm.store %{{.*}}, %[[GEP]] {alignment = 4 : i64, nontemporal} :  vector<4xf32>, !llvm.ptr
+
+// -----
+
+func.func @store_nontemporal_scalable(%memref : memref<200x100xf32>, %i : index, %j : index) {
+  %val = arith.constant dense<11.0> : vector<[4]xf32>
+  vector.store %val, %memref[%i, %j] {nontemporal = true} : memref<200x100xf32>, vector<[4]xf32>
+  return
+}
+
+// CHECK-LABEL: func @store_nontemporal_scalable
+// CHECK: %[[C100:.*]] = llvm.mlir.constant(100 : index) : i64
+// CHECK: %[[MUL:.*]] = llvm.mul %{{.*}}, %[[C100]]  : i64
+// CHECK: %[[ADD:.*]] = llvm.add %[[MUL]], %{{.*}}  : i64
+// CHECK: %[[GEP:.*]] = llvm.getelementptr %{{.*}}[%[[ADD]]] : (!llvm.ptr, i64) -> !llvm.ptr, f32
+// CHECK: llvm.store %{{.*}}, %[[GEP]] {alignment = 4 : i64, nontemporal} :  vector<[4]xf32>, !llvm.ptr
+
+// -----
+
+func.func @store_index(%memref : memref<200x100xindex>, %i : index, %j : index) {
+  %val = arith.constant dense<11> : vector<4xindex>
+  vector.store %val, %memref[%i, %j] : memref<200x100xindex>, vector<4xindex>
+  return
+}
+// CHECK-LABEL: func @store_index
+// CHECK: llvm.store %{{.*}}, %{{.*}} {alignment = 8 : i64} : vector<4xi64>, !llvm.ptr
+
+// -----
+
+func.func @store_index_scalable(%memref : memref<200x100xindex>, %i : index, %j : index) {
+  %val = arith.constant dense<11> : vector<[4]xindex>
+  vector.store %val, %memref[%i, %j] : memref<200x100xindex>, vector<[4]xindex>
+  return
+}
+// CHECK-LABEL: func @store_index_scalable
+// CHECK: llvm.store %{{.*}}, %{{.*}} {alignment = 8 : i64} : vector<[4]xi64>, !llvm.ptr
+
+// -----
+
+func.func @store_0d(%memref : memref<200x100xf32>, %i : index, %j : index) {
+  %val = arith.constant dense<11.0> : vector<f32>
+  vector.store %val, %memref[%i, %j] : memref<200x100xf32>, vector<f32>
+  return
+}
+
+// CHECK-LABEL: func @store_0d
+// CHECK: %[[J:.*]] = builtin.unrealized_conversion_cast %{{.*}} : index to i64
+// CHECK: %[[I:.*]] = builtin.unrealized_conversion_cast %{{.*}} : index to i64
+// CHECK: %[[CAST_MEMREF:.*]] = builtin.unrealized_conversion_cast %{{.*}} : memref<200x100xf32> to !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)>
+// CHECK: %[[CST:.*]] = arith.constant dense<1.100000e+01> : vector<f32>
+// CHECK: %[[VAL:.*]] = builtin.unrealized_conversion_cast %[[CST]] : vector<f32> to vector<1xf32>
+// CHECK: %[[REF:.*]] = llvm.extractvalue %[[CAST_MEMREF]][1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> 
+// CHECK: %[[C100:.*]] = llvm.mlir.constant(100 : index) : i64
+// CHECK: %[[MUL:.*]] = llvm.mul %[[I]], %[[C100]] : i64
+// CHECK: %[[ADD:.*]] = llvm.add %[[MUL]], %[[J]] : i64
+// CHECK: %[[ADDR:.*]] = llvm.getelementptr %[[REF]][%[[ADD]]] : (!llvm.ptr, i64) -> !llvm.ptr, f32
+// CHECK: llvm.store %[[VAL]], %[[ADDR]] {alignment = 4 : i64} : vector<1xf32>, !llvm.ptr
+// CHECK: return
+
+// -----
+
+//===----------------------------------------------------------------------===//
+// vector.maskedload
+//===----------------------------------------------------------------------===//
+
+func.func @masked_load(%arg0: memref<?xf32>, %arg1: vector<16xi1>, %arg2: vector<16xf32>) -> vector<16xf32> {
+  %c0 = arith.constant 0: index
+  %0 = vector.maskedload %arg0[%c0], %arg1, %arg2 : memref<?xf32>, vector<16xi1>, vector<16xf32> into vector<16xf32>
+  return %0 : vector<16xf32>
+}
+
+// CHECK-LABEL: func @masked_load
+// CHECK: %[[CO:.*]] = arith.constant 0 : index
+// CHECK: %[[C:.*]] = builtin.unrealized_conversion_cast %[[CO]] : index to i64
+// CHECK: %[[P:.*]] = llvm.getelementptr %{{.*}}[%[[C]]] : (!llvm.ptr, i64) -> !llvm.ptr, f32
+// CHECK: %[[L:.*]] = llvm.intr.masked.load %[[P]], %{{.*}}, %{{.*}} {alignment = 4 : i32} : (!llvm.ptr, vector<16xi1>, vector<16xf32>) -> vector<16xf32>
+// CHECK: return %[[L]] : vector<16xf32>
+
+// -----
+
+func.func @masked_load_scalable(%arg0: memref<?xf32>, %arg1: vector<[16]xi1>, %arg2: vector<[16]xf32>) -> vector<[16]xf32> {
+  %c0 = arith.constant 0: index
+  %0 = vector.maskedload %arg0[%c0], %arg1, %arg2 : memref<?xf32>, vector<[16]xi1>, vector<[16]xf32> into vector<[16]xf32>
+  return %0 : vector<[16]xf32>
+}
+
+// CHECK-LABEL: func @masked_load_scalable
+// CHECK: %[[CO:.*]] = arith.constant 0 : index
+// CHECK: %[[C:.*]] = builtin.unrealized_conversion_cast %[[CO]] : index to i64
+// CHECK: %[[P:.*]] = llvm.getelementptr %{{.*}}[%[[C]]] : (!llvm.ptr, i64) -> !llvm.ptr, f32
+// CHECK: %[[L:.*]] = llvm.intr.masked.load %[[P]], %{{.*}}, %{{.*}} {alignment = 4 : i32} : (!llvm.ptr, vector<[16]xi1>, vector<[16]xf32>) -> vector<[16]xf32>
+// CHECK: return %[[L]] : vector<[16]xf32>
+
+// -----
+
+func.func @masked_load_index(%arg0: memref<?xindex>, %arg1: vector<16xi1>, %arg2: vector<16xindex>) -> vector<16xindex> {
+  %c0 = arith.constant 0: index
+  %0 = vector.maskedload %arg0[%c0], %arg1, %arg2 : memref<?xindex>, vector<16xi1>, vector<16xindex> into vector<16xindex>
+  return %0 : vector<16xindex>
+}
+// CHECK-LABEL: func @masked_load_index
+// CHECK: %{{.*}} = llvm.intr.masked.load %{{.*}}, %{{.*}}, %{{.*}} {alignment = 8 : i32} : (!llvm.ptr, vector<16xi1>, vector<16xi64>) -> vector<16xi64>
+
+// -----
+
+func.func @masked_load_index_scalable(%arg0: memref<?xindex>, %arg1: vector<[16]xi1>, %arg2: vector<[16]xindex>) -> vector<[16]xindex> {
+  %c0 = arith.constant 0: index
+  %0 = vector.maskedload %arg0[%c0], %arg1, %arg2 : memref<?xindex>, vector<[16]xi1>, vector<[16]xindex> into vector<[16]xindex>
+  return %0 : vector<[16]xindex>
+}
+// CHECK-LABEL: func @masked_load_index_scalable
+// CHECK: %{{.*}} = llvm.intr.masked.load %{{.*}}, %{{.*}}, %{{.*}} {alignment = 8 : i32} : (!llvm.ptr, vector<[16]xi1>, vector<[16]xi64>) -> vector<[16]xi64>
+
+// -----
+
+//===----------------------------------------------------------------------===//
+// vector.maskedstore
+//===----------------------------------------------------------------------===//
+
+func.func @masked_store(%arg0: memref<?xf32>, %arg1: vector<16xi1>, %arg2: vector<16xf32>) {
+  %c0 = arith.constant 0: index
+  vector.maskedstore %arg0[%c0], %arg1, %arg2 : memref<?xf32>, vector<16xi1>, vector<16xf32>
+  return
+}
+
+// CHECK-LABEL: func @masked_store
+// CHECK: %[[CO:.*]] = arith.constant 0 : index
+// CHECK: %[[C:.*]] = builtin.unrealized_conversion_cast %[[CO]] : index to i64
+// CHECK: %[[P:.*]] = llvm.getelementptr %{{.*}}[%[[C]]] : (!llvm.ptr, i64) -> !llvm.ptr, f32
+// CHECK: llvm.intr.masked.store %{{.*}}, %[[P]], %{{.*}} {alignment = 4 : i32} : vector<16xf32>, vector<16xi1> into !llvm.ptr
+
+// -----
+
+func.func @masked_store_scalable(%arg0: memref<?xf32>, %arg1: vector<[16]xi1>, %arg2: vector<[16]xf32>) {
+  %c0 = arith.constant 0: index
+  vector.maskedstore %arg0[%c0], %arg1, %arg2 : memref<?xf32>, vector<[16]xi1>, vector<[16]xf32>
+  return
+}
+
+// CHECK-LABEL: func @masked_store_scalable
+// CHECK: %[[CO:.*]] = arith.constant 0 : index
+// CHECK: %[[C:.*]] = builtin.unrealized_conversion_cast %[[CO]] : index to i64
+// CHECK: %[[P:.*]] = llvm.getelementptr %{{.*}}[%[[C]]] : (!llvm.ptr, i64) -> !llvm.ptr, f32
+// CHECK: llvm.intr.masked.store %{{.*}}, %[[P]], %{{.*}} {alignment = 4 : i32} : vector<[16]xf32>, vector<[16]xi1> into !llvm.ptr
+
+// -----
+
+func.func @masked_store_index(%arg0: memref<?xindex>, %arg1: vector<16xi1>, %arg2: vector<16xindex>) {
+  %c0 = arith.constant 0: index
+  vector.maskedstore %arg0[%c0], %arg1, %arg2 : memref<?xindex>, vector<16xi1>, vector<16xindex>
+  return
+}
+// CHECK-LABEL: func @masked_store_index
+// CHECK: llvm.intr.masked.store %{{.*}}, %{{.*}}, %{{.*}} {alignment = 8 : i32} : vector<16xi64>, vector<16xi1> into !llvm.ptr
+
+// -----
+
+func.func @masked_store_index_scalable(%arg0: memref<?xindex>, %arg1: vector<[16]xi1>, %arg2: vector<[16]xindex>) {
+  %c0 = arith.constant 0: index
+  vector.maskedstore %arg0[%c0], %arg1, %arg2 : memref<?xindex>, vector<[16]xi1>, vector<[16]xindex>
+  return
+}
+// CHECK-LABEL: func @masked_store_index_scalable
+// CHECK: llvm.intr.masked.store %{{.*}}, %{{.*}}, %{{.*}} {alignment = 8 : i32} : vector<[16]xi64>, vector<[16]xi1> into !llvm.ptr
+
+// -----
+
+//===----------------------------------------------------------------------===//
+// vector.gather
+//===----------------------------------------------------------------------===//
+
+func.func @gather(%arg0: memref<?xf32>, %arg1: vector<3xi32>, %arg2: vector<3xi1>, %arg3: vector<3xf32>) -> vector<3xf32> {
+  %0 = arith.constant 0: index
+  %1 = vector.gather %arg0[%0][%arg1], %arg2, %arg3 : memref<?xf32>, vector<3xi32>, vector<3xi1>, vector<3xf32> into vector<3xf32>
+  return %1 : vector<3xf32>
+}
+
+// CHECK-LABEL: func @gather
+// CHECK: %[[P:.*]] = llvm.getelementptr %{{.*}}[%{{.*}}] : (!llvm.ptr, vector<3xi32>) -> !llvm.vec<3 x ptr>, f32
+// CHECK: %[[G:.*]] = llvm.intr.masked.gather %[[P]], %{{.*}}, %{{.*}} {alignment = 4 : i32} : (!llvm.vec<3 x ptr>, vector<3xi1>, vector<3xf32>) -> vector<3xf32>
+// CHECK: return %[[G]] : vector<3xf32>
+
+// -----
+
+func.func @gather_scalable(%arg0: memref<?xf32>, %arg1: vector<[3]xi32>, %arg2: vector<[3]xi1>, %arg3: vector<[3]xf32>) -> vector<[3]xf32> {
+  %0 = arith.constant 0: index
+  %1 = vector.gather %arg0[%0][%arg1], %arg2, %arg3 : memref<?xf32>, vector<[3]xi32>, vector<[3]xi1>, vector<[3]xf32> into vector<[3]xf32>
+  return %1 : vector<[3]xf32>
+}
+
+// CHECK-LABEL: func @gather_scalable
+// CHECK: %[[P:.*]] = llvm.getelementptr %{{.*}}[%{{.*}}] : (!llvm.ptr, vector<[3]xi32>) -> !llvm.vec<? x 3 x ptr>, f32
+// CHECK: %[[G:.*]] = llvm.intr.masked.gather %[[P]], %{{.*}}, %{{.*}} {alignment = 4 : i32} : (!llvm.vec<? x 3 x ptr>, vector<[3]xi1>, vector<[3]xf32>) -> vector<[3]xf32>
+// CHECK: return %[[G]] : vector<[3]xf32>
+
+// -----
+
+func.func @gather_global_memory(%arg0: memref<?xf32, 1>, %arg1: vector<3xi32>, %arg2: vector<3xi1>, %arg3: vector<3xf32>) -> vector<3xf32> {
+  %0 = arith.constant 0: index
+  %1 = vector.gather %arg0[%0][%arg1], %arg2, %arg3 : memref<?xf32, 1>, vector<3xi32>, vector<3xi1>, vector<3xf32> into vector<3xf32>
+  return %1 : vector<3xf32>
+}
+
+// CHECK-LABEL: func @gather_global_memory
+// CHECK: %[[P:.*]] = llvm.getelementptr %{{.*}}[%{{.*}}] : (!llvm.ptr<1>, vector<3xi32>) -> !llvm.vec<3 x ptr<1>>, f32
+// CHECK: %[[G:.*]] = llvm.intr.masked.gather %[[P]], %{{.*}}, %{{.*}} {alignment = 4 : i32} : (!llvm.vec<3 x ptr<1>>, vector<3xi1>, vector<3xf32>) -> vector<3xf32>
+// CHECK: return %[[G]] : vector<3xf32>
+
+// -----
+
+func.func @gather_global_memory_scalable(%arg0: memref<?xf32, 1>, %arg1: vector<[3]xi32>, %arg2: vector<[3]xi1>, %arg3: vector<[3]xf32>) -> vector<[3]xf32> {
+  %0 = arith.constant 0: index
+  %1 = vector.gather %arg0[%0][%arg1], %arg2, %arg3 : memref<?xf32, 1>, vector<[3]xi32>, vector<[3]xi1>, vector<[3]xf32> into vector<[3]xf32>
+  return %1 : vector<[3]xf32>
+}
+
+// CHECK-LABEL: func @gather_global_memory_scalable
+// CHECK: %[[P:.*]] = llvm.getelementptr %{{.*}}[%{{.*}}] : (!llvm.ptr<1>, vector<[3]xi32>) -> !llvm.vec<? x 3 x ptr<1>>, f32
+// CHECK: %[[G:.*]] = llvm.intr.masked.gather %[[P]], %{{.*}}, %{{.*}} {alignment = 4 : i32} : (!llvm.vec<? x 3 x ptr<1>>, vector<[3]xi1>, vector<[3]xf32>) -> vector<[3]xf32>
+// CHECK: return %[[G]] : vector<[3]xf32>
+
+// -----
+
+
+func.func @gather_index(%arg0: memref<?xindex>, %arg1: vector<3xindex>, %arg2: vector<3xi1>, %arg3: vector<3xindex>) -> vector<3xindex> {
+  %0 = arith.constant 0: index
+  %1 = vector.gather %arg0[%0][%arg1], %arg2, %arg3 : memref<?xindex>, vector<3xindex>, vector<3xi1>, vector<3xindex> into vector<3xindex>
+  return %1 : vector<3xindex>
+}
+
+// CHECK-LABEL: func @gather_index
+// CHECK: %[[P:.*]] = llvm.getelementptr %{{.*}}[%{{.*}}] : (!llvm.ptr, vector<3xi64>) -> !llvm.vec<3 x ptr>, i64
+// CHECK: %[[G:.*]] = llvm.intr.masked.gather %{{.*}}, %{{.*}}, %{{.*}} {alignment = 8 : i32} : (!llvm.vec<3 x ptr>, vector<3xi1>, vector<3xi64>) -> vector<3xi64>
+// CHECK: %{{.*}} = builtin.unrealized_conversion_cast %[[G]] : vector<3xi64> to vector<3xindex>
+
+// -----
+
+func.func @gather_index_scalable(%arg0: memref<?xindex>, %arg1: vector<[3]xindex>, %arg2: vector<[3]xi1>, %arg3: vector<[3]xindex>) -> vector<[3]xindex> {
+  %0 = arith.constant 0: index
+  %1 = vector.gather %arg0[%0][%arg1], %arg2, %arg3 : memref<?xindex>, vector<[3]xindex>, vector<[3]xi1>, vector<[3]xindex> into vector<[3]xindex>
+  return %1 : vector<[3]xindex>
+}
+
+// CHECK-LABEL: func @gather_index_scalable
+// CHECK: %[[P:.*]] = llvm.getelementptr %{{.*}}[%{{.*}}] : (!llvm.ptr, vector<[3]xi64>) -> !llvm.vec<? x 3 x ptr>, i64
+// CHECK: %[[G:.*]] = llvm.intr.masked.gather %{{.*}}, %{{.*}}, %{{.*}} {alignment = 8 : i32} : (!llvm.vec<? x 3 x ptr>, vector<[3]xi1>, vector<[3]xi64>) -> vector<[3]xi64>
+// CHECK: %{{.*}} = builtin.unrealized_conversion_cast %[[G]] : vector<[3]xi64> to vector<[3]xindex>
+
+// -----
+
+func.func @gather_2d_from_1d(%arg0: memref<?xf32>, %arg1: vector<2x3xi32>, %arg2: vector<2x3xi1>, %arg3: vector<2x3xf32>) -> vector<2x3xf32> {
+  %0 = arith.constant 0: index
+  %1 = vector.gather %arg0[%0][%arg1], %arg2, %arg3 : memref<?xf32>, vector<2x3xi32>, vector<2x3xi1>, vector<2x3xf32> into vector<2x3xf32>
+  return %1 : vector<2x3xf32>
+}
+
+// CHECK-LABEL: func @gather_2d_from_1d
+// CHECK: %[[B:.*]] = llvm.getelementptr %{{.*}} : (!llvm.ptr, i64) -> !llvm.ptr, f32
+// CHECK: %[[I0:.*]] = llvm.extractvalue %{{.*}}[0] : !llvm.array<2 x vector<3xi32>>
+// CHECK: %[[M0:.*]] = llvm.extractvalue %{{.*}}[0] : !llvm.array<2 x vector<3xi1>>
+// CHECK: %[[S0:.*]] = llvm.extractvalue %{{.*}}[0] : !llvm.array<2 x vector<3xf32>>
+// CHECK: %[[P0:.*]] = llvm.getelementptr %[[B]][%[[I0]]] : (!llvm.ptr, vector<3xi32>) -> !llvm.vec<3 x ptr>, f32
+// CHECK: %[[G0:.*]] = llvm.intr.masked.gather %[[P0]], %[[M0]], %[[S0]] {alignment = 4 : i32} : (!llvm.vec<3 x ptr>, vector<3xi1>, vector<3xf32>) -> vector<3xf32>
+// CHECK: %{{.*}} = llvm.insertvalue %[[G0]], %{{.*}}[0] : !llvm.array<2 x vector<3xf32>>
+// CHECK: %[[I1:.*]] = llvm.extractvalue %{{.*}}[1] : !llvm.array<2 x vector<3xi32>>
+// CHECK: %[[M1:.*]] = llvm.extractvalue %{{.*}}[1] : !llvm.array<2 x vector<3xi1>>
+// CHECK: %[[S1:.*]] = llvm.extractvalue %{{.*}}[1] : !llvm.array<2 x vector<3xf32>>
+// CHECK: %[[P1:.*]] = llvm.getelementptr %[[B]][%[[I1]]] : (!llvm.ptr, vector<3xi32>) -> !llvm.vec<3 x ptr>, f32
+// CHECK: %[[G1:.*]] = llvm.intr.masked.gather %[[P1]], %[[M1]], %[[S1]] {alignment = 4 : i32} : (!llvm.vec<3 x ptr>, vector<3xi1>, vector<3xf32>) -> vector<3xf32>
+// CHECK: %{{.*}} = llvm.insertvalue %[[G1]], %{{.*}}[1] : !llvm.array<2 x vector<3xf32>>
+
+// -----
+
+func.func @gather_2d_from_1d_scalable(%arg0: memref<?xf32>, %arg1: vector<2x[3]xi32>, %arg2: vector<2x[3]xi1>, %arg3: vector<2x[3]xf32>) -> vector<2x[3]xf32> {
+  %0 = arith.constant 0: index
+  %1 = vector.gather %arg0[%0][%arg1], %arg2, %arg3 : memref<?xf32>, vector<2x[3]xi32>, vector<2x[3]xi1>, vector<2x[3]xf32> into vector<2x[3]xf32>
+  return %1 : vector<2x[3]xf32>
+}
+
+// CHECK-LABEL: func @gather_2d_from_1d_scalable
+// CHECK: %[[B:.*]] = llvm.getelementptr %{{.*}} : (!llvm.ptr, i64) -> !llvm.ptr, f32
+// CHECK: %[[I0:.*]] = llvm.extractvalue %{{.*}}[0] : !llvm.array<2 x vector<[3]xi32>>
+// CHECK: %[[M0:.*]] = llvm.extractvalue %{{.*}}[0] : !llvm.array<2 x vector<[3]xi1>>
+// CHECK: %[[S0:.*]] = llvm.extractvalue %{{.*}}[0] : !llvm.array<2 x vector<[3]xf32>>
+// CHECK: %[[P0:.*]] = llvm.getelementptr %[[B]][%[[I0]]] : (!llvm.ptr, vector<[3]xi32>) -> !llvm.vec<? x 3 x ptr>, f32
+// CHECK: %[[G0:.*]] = llvm.intr.masked.gather %[[P0]], %[[M0]], %[[S0]] {alignment = 4 : i32} : (!llvm.vec<? x 3 x ptr>, vector<[3]xi1>, vector<[3]xf32>) -> vector<[3]xf32>
+// CHECK: %{{.*}} = llvm.insertvalue %[[G0]], %{{.*}}[0] : !llvm.array<2 x vector<[3]xf32>>
+// CHECK: %[[I1:.*]] = llvm.extractvalue %{{.*}}[1] : !llvm.array<2 x vector<[3]xi32>>
+// CHECK: %[[M1:.*]] = llvm.extractvalue %{{.*}}[1] : !llvm.array<2 x vector<[3]xi1>>
+// CHECK: %[[S1:.*]] = llvm.extractvalue %{{.*}}[1] : !llvm.array<2 x vector<[3]xf32>>
+// CHECK: %[[P1:.*]] = llvm.getelementptr %[[B]][%[[I1]]] : (!llvm.ptr, vector<[3]xi32>) -> !llvm.vec<? x 3 x ptr>, f32
+// CHECK: %[[G1:.*]] = llvm.intr.masked.gather %[[P1]], %[[M1]], %[[S1]] {alignment = 4 : i32} : (!llvm.vec<? x 3 x ptr>, vector<[3]xi1>, vector<[3]xf32>) -> vector<[3]xf32>
+// CHECK: %{{.*}} = llvm.insertvalue %[[G1]], %{{.*}}[1] : !llvm.array<2 x vector<[3]xf32>>
+
+// -----
+
+
+func.func @gather_1d_from_2d(%arg0: memref<4x4xf32>, %arg1: vector<4xi32>, %arg2: vector<4xi1>, %arg3: vector<4xf32>) -> vector<4xf32> {
+  %0 = arith.constant 3 : index
+  %1 = vector.gather %arg0[%0, %0][%arg1], %arg2, %arg3 : memref<4x4xf32>, vector<4xi32>, vector<4xi1>, vector<4xf32> into vector<4xf32>
+  return %1 : vector<4xf32>
+}
+
+// CHECK-LABEL: func @gather_1d_from_2d
+// CHECK: %[[B:.*]] = llvm.getelementptr %{{.*}}[%{{.*}}] : (!llvm.ptr, i64) -> !llvm.ptr, f32
+// CHECK: %[[P:.*]] = llvm.getelementptr %[[B]][%{{.*}}] : (!llvm.ptr, vector<4xi32>) -> !llvm.vec<4 x ptr>, f32
+// CHECK: %[[G:.*]] = llvm.intr.masked.gather %[[P]], %{{.*}}, %{{.*}} {alignment = 4 : i32} : (!llvm.vec<4 x ptr>, vector<4xi1>, vector<4xf32>) -> vector<4xf32>
+// CHECK: return %[[G]] : vector<4xf32>
+
+// -----
+
+func.func @gather_1d_from_2d_scalable(%arg0: memref<4x?xf32>, %arg1: vector<[4]xi32>, %arg2: vector<[4]xi1>, %arg3: vector<[4]xf32>) -> vector<[4]xf32> {
+  %0 = arith.constant 3 : index
+  %1 = vector.gather %arg0[%0, %0][%arg1], %arg2, %arg3 : memref<4x?xf32>, vector<[4]xi32>, vector<[4]xi1>, vector<[4]xf32> into vector<[4]xf32>
+  return %1 : vector<[4]xf32>
+}
+
+// CHECK-LABEL: func @gather_1d_from_2d_scalable
+// CHECK: %[[B:.*]] = llvm.getelementptr %{{.*}}[%{{.*}}] : (!llvm.ptr, i64) -> !llvm.ptr, f32
+// CHECK: %[[P:.*]] = llvm.getelementptr %[[B]][%{{.*}}] : (!llvm.ptr, vector<[4]xi32>) -> !llvm.vec<? x 4 x ptr>, f32
+// CHECK: %[[G:.*]] = llvm.intr.masked.gather %[[P]], %{{.*}}, %{{.*}} {alignment = 4 : i32} : (!llvm.vec<? x 4 x ptr>, vector<[4]xi1>, vector<[4]xf32>) -> vector<[4]xf32>
+// CHECK: return %[[G]] : vector<[4]xf32>
+
+// -----
+
+//===----------------------------------------------------------------------===//
+// vector.scatter
+//===----------------------------------------------------------------------===//
+
+func.func @scatter(%arg0: memref<?xf32>, %arg1: vector<3xi32>, %arg2: vector<3xi1>, %arg3: vector<3xf32>) {
+  %0 = arith.constant 0: index
+  vector.scatter %arg0[%0][%arg1], %arg2, %arg3 : memref<?xf32>, vector<3xi32>, vector<3xi1>, vector<3xf32>
+  return
+}
+
+// CHECK-LABEL: func @scatter
+// CHECK: %[[P:.*]] = llvm.getelementptr %{{.*}}[%{{.*}}] : (!llvm.ptr, vector<3xi32>) -> !llvm.vec<3 x ptr>, f32
+// CHECK: llvm.intr.masked.scatter %{{.*}}, %[[P]], %{{.*}} {alignment = 4 : i32} : vector<3xf32>, vector<3xi1> into !llvm.vec<3 x ptr>
+
+// -----
+
+func.func @scatter_scalable(%arg0: memref<?xf32>, %arg1: vector<[3]xi32>, %arg2: vector<[3]xi1>, %arg3: vector<[3]xf32>) {
+  %0 = arith.constant 0: index
+  vector.scatter %arg0[%0][%arg1], %arg2, %arg3 : memref<?xf32>, vector<[3]xi32>, vector<[3]xi1>, vector<[3]xf32>
+  return
+}
+
+// CHECK-LABEL: func @scatter_scalable
+// CHECK: %[[P:.*]] = llvm.getelementptr %{{.*}}[%{{.*}}] : (!llvm.ptr, vector<[3]xi32>) -> !llvm.vec<? x 3 x ptr>, f32
+// CHECK: llvm.intr.masked.scatter %{{.*}}, %[[P]], %{{.*}} {alignment = 4 : i32} : vector<[3]xf32>, vector<[3]xi1> into !llvm.vec<? x 3 x ptr>
+
+// -----
+
+func.func @scatter_index(%arg0: memref<?xindex>, %arg1: vector<3xindex>, %arg2: vector<3xi1>, %arg3: vector<3xindex>) {
+  %0 = arith.constant 0: index
+  vector.scatter %arg0[%0][%arg1], %arg2, %arg3 : memref<?xindex>, vector<3xindex>, vector<3xi1>, vector<3xindex>
+  return
+}
+
+// CHECK-LABEL: func @scatter_index
+// CHECK: %[[P:.*]] = llvm.getelementptr %{{.*}}[%{{.*}}] : (!llvm.ptr, vector<3xi64>) -> !llvm.vec<3 x ptr>, i64
+// CHECK: llvm.intr.masked.scatter %{{.*}}, %[[P]], %{{.*}} {alignment = 8 : i32} : vector<3xi64>, vector<3xi1> into !llvm.vec<3 x ptr>
+
+// -----
+
+func.func @scatter_index_scalable(%arg0: memref<?xindex>, %arg1: vector<[3]xindex>, %arg2: vector<[3]xi1>, %arg3: vector<[3]xindex>) {
+  %0 = arith.constant 0: index
+  vector.scatter %arg0[%0][%arg1], %arg2, %arg3 : memref<?xindex>, vector<[3]xindex>, vector<[3]xi1>, vector<[3]xindex>
+  return
+}
+
+// CHECK-LABEL: func @scatter_index_scalable
+// CHECK: %[[P:.*]] = llvm.getelementptr %{{.*}}[%{{.*}}] : (!llvm.ptr, vector<[3]xi64>) -> !llvm.vec<? x 3 x ptr>, i64
+// CHECK: llvm.intr.masked.scatter %{{.*}}, %[[P]], %{{.*}} {alignment = 8 : i32} : vector<[3]xi64>, vector<[3]xi1> into !llvm.vec<? x 3 x ptr>
+
+// -----
+
+func.func @scatter_1d_into_2d(%arg0: memref<4x4xf32>, %arg1: vector<4xi32>, %arg2: vector<4xi1>, %arg3: vector<4xf32>) {
+  %0 = arith.constant 3 : index
+  vector.scatter %arg0[%0, %0][%arg1], %arg2, %arg3 : memref<4x4xf32>, vector<4xi32>, vector<4xi1>, vector<4xf32>
+  return
+}
+
+// CHECK-LABEL: func @scatter_1d_into_2d
+// CHECK: %[[B:.*]] = llvm.getelementptr %{{.*}}[%{{.*}}] : (!llvm.ptr, i64) -> !llvm.ptr, f32
+// CHECK: %[[P:.*]] = llvm.getelementptr %[[B]][%{{.*}}] : (!llvm.ptr, vector<4xi32>) -> !llvm.vec<4 x ptr>, f32
+// CHECK: llvm.intr.masked.scatter %{{.*}}, %[[P]], %{{.*}} {alignment = 4 : i32} : vector<4xf32>, vector<4xi1> into !llvm.vec<4 x ptr>
+
+// -----
+
+func.func @scatter_1d_into_2d_scalable(%arg0: memref<4x?xf32>, %arg1: vector<[4]xi32>, %arg2: vector<[4]xi1>, %arg3: vector<[4]xf32>) {
+  %0 = arith.constant 3 : index
+  vector.scatter %arg0[%0, %0][%arg1], %arg2, %arg3 : memref<4x?xf32>, vector<[4]xi32>, vector<[4]xi1>, vector<[4]xf32>
+  return
+}
+
+// CHECK-LABEL: func @scatter_1d_into_2d_scalable
+// CHECK: %[[B:.*]] = llvm.getelementptr %{{.*}}[%{{.*}}] : (!llvm.ptr, i64) -> !llvm.ptr, f32
+// CHECK: %[[P:.*]] = llvm.getelementptr %[[B]][%{{.*}}] : (!llvm.ptr, vector<[4]xi32>) -> !llvm.vec<? x 4 x ptr>, f32
+// CHECK: llvm.intr.masked.scatter %{{.*}}, %[[P]], %{{.*}} {alignment = 4 : i32} : vector<[4]xf32>, vector<[4]xi1> into !llvm.vec<? x 4 x ptr>
+
+// -----
+
+//===----------------------------------------------------------------------===//
+// vector.expandload
+//===----------------------------------------------------------------------===//
+
+func.func @expand_load_op(%arg0: memref<?xf32>, %arg1: vector<11xi1>, %arg2: vector<11xf32>) -> vector<11xf32> {
+  %c0 = arith.constant 0: index
+  %0 = vector.expandload %arg0[%c0], %arg1, %arg2 : memref<?xf32>, vector<11xi1>, vector<11xf32> into vector<11xf32>
+  return %0 : vector<11xf32>
+}
+
+// CHECK-LABEL: func @expand_load_op
+// CHECK: %[[CO:.*]] = arith.constant 0 : index
+// CHECK: %[[C:.*]] = builtin.unrealized_conversion_cast %[[CO]] : index to i64
+// CHECK: %[[P:.*]] = llvm.getelementptr %{{.*}}[%[[C]]] : (!llvm.ptr, i64) -> !llvm.ptr, f32
+// CHECK: %[[E:.*]] = "llvm.intr.masked.expandload"(%[[P]], %{{.*}}, %{{.*}}) : (!llvm.ptr, vector<11xi1>, vector<11xf32>) -> vector<11xf32>
+// CHECK: return %[[E]] : vector<11xf32>
+
+// -----
+
+func.func @expand_load_op_index(%arg0: memref<?xindex>, %arg1: vector<11xi1>, %arg2: vector<11xindex>) -> vector<11xindex> {
+  %c0 = arith.constant 0: index
+  %0 = vector.expandload %arg0[%c0], %arg1, %arg2 : memref<?xindex>, vector<11xi1>, vector<11xindex> into vector<11xindex>
+  return %0 : vector<11xindex>
+}
+// CHECK-LABEL: func @expand_load_op_index
+// CHECK: %{{.*}} = "llvm.intr.masked.expandload"(%{{.*}}, %{{.*}}, %{{.*}}) : (!llvm.ptr, vector<11xi1>, vector<11xi64>) -> vector<11xi64>
+
+// -----
+
+//===----------------------------------------------------------------------===//
+// vector.compressstore
+//===----------------------------------------------------------------------===//
+
+func.func @compress_store_op(%arg0: memref<?xf32>, %arg1: vector<11xi1>, %arg2: vector<11xf32>) {
+  %c0 = arith.constant 0: index
+  vector.compressstore %arg0[%c0], %arg1, %arg2 : memref<?xf32>, vector<11xi1>, vector<11xf32>
+  return
+}
+
+// CHECK-LABEL: func @compress_store_op
+// CHECK: %[[CO:.*]] = arith.constant 0 : index
+// CHECK: %[[C:.*]] = builtin.unrealized_conversion_cast %[[CO]] : index to i64
+// CHECK: %[[P:.*]] = llvm.getelementptr %{{.*}}[%[[C]]] : (!llvm.ptr, i64) -> !llvm.ptr, f32
+// CHECK: "llvm.intr.masked.compressstore"(%{{.*}}, %[[P]], %{{.*}}) : (vector<11xf32>, !llvm.ptr, vector<11xi1>) -> ()
+
+// -----
+
+func.func @compress_store_op_index(%arg0: memref<?xindex>, %arg1: vector<11xi1>, %arg2: vector<11xindex>) {
+  %c0 = arith.constant 0: index
+  vector.compressstore %arg0[%c0], %arg1, %arg2 : memref<?xindex>, vector<11xi1>, vector<11xindex>
+  return
+}
+// CHECK-LABEL: func @compress_store_op_index
+// CHECK: "llvm.intr.masked.compressstore"(%{{.*}}, %{{.*}}, %{{.*}}) : (vector<11xi64>, !llvm.ptr, vector<11xi1>) -> ()
+
+// -----
+
+//===----------------------------------------------------------------------===//
+// vector.splat
+//===----------------------------------------------------------------------===//
+
+// CHECK-LABEL: @splat_0d
+// CHECK-SAME: %[[ELT:.*]]: f32
+func.func @splat_0d(%elt: f32) -> vector<f32> {
+  %v = vector.splat %elt : vector<f32>
+  return %v : vector<f32>
+}
+// CHECK-NEXT: %[[UNDEF:[0-9]+]] = llvm.mlir.poison : vector<1xf32>
+// CHECK-NEXT: %[[ZERO:[0-9]+]] = llvm.mlir.constant(0 : i32) : i32
+// CHECK-NEXT: %[[V:[0-9]+]] = llvm.insertelement %[[ELT]], %[[UNDEF]][%[[ZERO]] : i32] : vector<1xf32>
+// CHECK-NEXT: %[[VCAST:[0-9]+]] = builtin.unrealized_conversion_cast %[[V]] : vector<1xf32> to vector<f32>
+// CHECK-NEXT: return %[[VCAST]] : vector<f32>
+
+// -----
+
+// CHECK-LABEL: @splat
+// CHECK-SAME: %[[VEC:[0-9a-zA-Z]+]]: vector<4xf32>
+// CHECK-SAME: %[[ELT:[0-9a-zA-Z]+]]: f32
+func.func @splat(%vec: vector<4xf32>, %elt: f32) -> vector<4xf32> {
+  %vb = vector.splat %elt : vector<4xf32>
+  %r = arith.mulf %vec, %vb : vector<4xf32>
+  return %r : vector<4xf32>
+}
+// CHECK-NEXT: %[[UNDEF:[0-9]+]] = llvm.mlir.poison : vector<4xf32>
+// CHECK-NEXT: %[[ZERO:[0-9]+]] = llvm.mlir.constant(0 : i32) : i32
+// CHECK-NEXT: %[[V:[0-9]+]] = llvm.insertelement %[[ELT]], %[[UNDEF]][%[[ZERO]] : i32] : vector<4xf32>
+// CHECK-NEXT: %[[SPLAT:[0-9]+]] = llvm.shufflevector %[[V]], %[[UNDEF]] [0, 0, 0, 0]
+// CHECK-NEXT: %[[SCALE:[0-9]+]] = arith.mulf %[[VEC]], %[[SPLAT]] : vector<4xf32>
+// CHECK-NEXT: return %[[SCALE]] : vector<4xf32>
+
+// -----
+
+// CHECK-LABEL: @splat_scalable
+// CHECK-SAME: %[[VEC:[0-9a-zA-Z]+]]: vector<[4]xf32>
+// CHECK-SAME: %[[ELT:[0-9a-zA-Z]+]]: f32
+func.func @splat_scalable(%vec: vector<[4]xf32>, %elt: f32) -> vector<[4]xf32> {
+  %vb = vector.splat %elt : vector<[4]xf32>
+  %r = arith.mulf %vec, %vb : vector<[4]xf32>
+  return %r : vector<[4]xf32>
+}
+// CHECK-NEXT: %[[UNDEF:[0-9]+]] = llvm.mlir.poison : vector<[4]xf32>
+// CHECK-NEXT: %[[ZERO:[0-9]+]] = llvm.mlir.constant(0 : i32) : i32
+// CHECK-NEXT: %[[V:[0-9]+]] = llvm.insertelement %[[ELT]], %[[UNDEF]][%[[ZERO]] : i32] : vector<[4]xf32>
+// CHECK-NEXT: %[[SPLAT:[0-9]+]] = llvm.shufflevector %[[V]], %[[UNDEF]] [0, 0, 0, 0]
+// CHECK-NEXT: %[[SCALE:[0-9]+]] = arith.mulf %[[VEC]], %[[SPLAT]] : vector<[4]xf32>
+// CHECK-NEXT: return %[[SCALE]] : vector<[4]xf32>
+
+// -----
+
+//===----------------------------------------------------------------------===//
+// vector.scalable_insert
+//===----------------------------------------------------------------------===//
+
+// CHECK-LABEL: @scalable_insert
+// CHECK-SAME: %[[SUB:.*]]: vector<4xf32>, %[[SV:.*]]: vector<[4]xf32>
+func.func @scalable_insert(%sub: vector<4xf32>, %dsv: vector<[4]xf32>) -> vector<[4]xf32> {
+  // CHECK-NEXT: %[[TMP:.*]] = llvm.intr.vector.insert %[[SUB]], %[[SV]][0] : vector<4xf32> into vector<[4]xf32>
+  %0 = vector.scalable.insert %sub, %dsv[0] : vector<4xf32> into vector<[4]xf32>
+  // CHECK-NEXT: llvm.intr.vector.insert %[[SUB]], %[[TMP]][4] : vector<4xf32> into vector<[4]xf32>
+  %1 = vector.scalable.insert %sub, %0[4] : vector<4xf32> into vector<[4]xf32>
+  return %1 : vector<[4]xf32>
+}
+
+// -----
+
+//===----------------------------------------------------------------------===//
+// vector.scalable_extract
+//===----------------------------------------------------------------------===//
+
+// CHECK-LABEL: @scalable_extract
+// CHECK-SAME: %[[VEC:.*]]: vector<[4]xf32>
+func.func @scalable_extract(%vec: vector<[4]xf32>) -> vector<8xf32> {
+  // CHECK-NEXT: %{{.*}} = llvm.intr.vector.extract %[[VEC]][0] : vector<8xf32> from vector<[4]xf32>
+  %0 = vector.scalable.extract %vec[0] : vector<8xf32> from vector<[4]xf32>
+  return %0 : vector<8xf32>
+}
+
+// -----
+
+//===----------------------------------------------------------------------===//
+// vector.interleave
+//===----------------------------------------------------------------------===//
+
+// CHECK-LABEL: @interleave_0d
+//  CHECK-SAME:     %[[LHS:.*]]: vector<i8>, %[[RHS:.*]]: vector<i8>)
+func.func @interleave_0d(%a: vector<i8>, %b: vector<i8>) -> vector<2xi8> {
+  // CHECK-DAG: %[[LHS_RANK1:.*]] = builtin.unrealized_conversion_cast %[[LHS]] : vector<i8> to vector<1xi8>
+  // CHECK-DAG: %[[RHS_RANK1:.*]] = builtin.unrealized_conversion_cast %[[RHS]] : vector<i8> to vector<1xi8>
+  // CHECK: %[[ZIP:.*]] = llvm.shufflevector %[[LHS_RANK1]], %[[RHS_RANK1]] [0, 1] : vector<1xi8>
+  // CHECK: return %[[ZIP]]
+  %0 = vector.interleave %a, %b : vector<i8> -> vector<2xi8>
+  return %0 : vector<2xi8>
+}
+
+// -----
+
+// CHECK-LABEL: @interleave_1d
+//  CHECK-SAME:     %[[LHS:.*]]: vector<8xf32>, %[[RHS:.*]]: vector<8xf32>)
+func.func @interleave_1d(%a: vector<8xf32>, %b: vector<8xf32>) -> vector<16xf32> {
+  // CHECK: %[[ZIP:.*]] = llvm.shufflevector %[[LHS]], %[[RHS]] [0, 8, 1, 9, 2, 10, 3, 11, 4, 12, 5, 13, 6, 14, 7, 15] : vector<8xf32>
+  // CHECK: return %[[ZIP]]
+  %0 = vector.interleave %a, %b : vector<8xf32> -> vector<16xf32>
+  return %0 : vector<16xf32>
+}
+
+// -----
+
+// CHECK-LABEL: @interleave_1d_scalable
+//  CHECK-SAME:     %[[LHS:.*]]: vector<[4]xi32>, %[[RHS:.*]]: vector<[4]xi32>)
+func.func @interleave_1d_scalable(%a: vector<[4]xi32>, %b: vector<[4]xi32>) -> vector<[8]xi32> {
+  // CHECK: %[[ZIP:.*]] = "llvm.intr.vector.interleave2"(%[[LHS]], %[[RHS]]) : (vector<[4]xi32>, vector<[4]xi32>) -> vector<[8]xi32>
+  // CHECK: return %[[ZIP]]
+  %0 = vector.interleave %a, %b : vector<[4]xi32> -> vector<[8]xi32>
+  return %0 : vector<[8]xi32>
+}
+
+// -----
+
+//===----------------------------------------------------------------------===//
+// vector.deinterleave
+//===----------------------------------------------------------------------===//
+
+// CHECK-LABEL: @deinterleave_1d
+// CHECK-SAME:  (%[[ARG:.*]]: vector<4xi32>) -> (vector<2xi32>, vector<2xi32>)
+func.func @deinterleave_1d(%arg: vector<4xi32>) -> (vector<2xi32>, vector<2xi32>) {
+  // CHECK: %[[POISON:.*]] = llvm.mlir.poison : vector<4xi32>
+  // CHECK: llvm.shufflevector %[[ARG]], %[[POISON]] [0, 2] : vector<4xi32>
+  // CHECK: llvm.shufflevector %[[ARG]], %[[POISON]] [1, 3] : vector<4xi32>
+  %0, %1 = vector.deinterleave %arg : vector<4xi32> -> vector<2xi32>
+  return %0, %1 : vector<2xi32>, vector<2xi32>
+}
+
+// -----
+
+// CHECK-LABEL: @deinterleave_1d_scalable
+// CHECK-SAME:  %[[ARG:.*]]: vector<[4]xi32>) -> (vector<[2]xi32>, vector<[2]xi32>)
+func.func @deinterleave_1d_scalable(%arg: vector<[4]xi32>) -> (vector<[2]xi32>, vector<[2]xi32>) {
+    // CHECK: %[[RES:.*]] = "llvm.intr.vector.deinterleave2"(%[[ARG]]) : (vector<[4]xi32>) -> !llvm.struct<(vector<[2]xi32>, vector<[2]xi32>)>
+    // CHECK: llvm.extractvalue %[[RES]][0] : !llvm.struct<(vector<[2]xi32>, vector<[2]xi32>)>
+    // CHECK: llvm.extractvalue %[[RES]][1] : !llvm.struct<(vector<[2]xi32>, vector<[2]xi32>)>
+    %0, %1 = vector.deinterleave %arg : vector<[4]xi32> -> vector<[2]xi32>
+    return %0, %1 : vector<[2]xi32>, vector<[2]xi32>
+}
+
+// -----
+
+//===----------------------------------------------------------------------===//
+// vector.from_elements
+//===----------------------------------------------------------------------===//
+
+// CHECK-LABEL: func.func @from_elements_1d(
+//  CHECK-SAME:     %[[ARG_0:.*]]: f32, %[[ARG_1:.*]]: f32)
+//       CHECK:   %[[UNDEF:.*]] = llvm.mlir.poison : vector<3xf32>
+//       CHECK:   %[[C0:.*]] = llvm.mlir.constant(0 : i64) : i64
+//       CHECK:   %[[INSERT0:.*]] = llvm.insertelement %[[ARG_0]], %[[UNDEF]][%[[C0]] : i64] : vector<3xf32>
+//       CHECK:   %[[C1:.*]] = llvm.mlir.constant(1 : i64) : i64
+//       CHECK:   %[[INSERT1:.*]] = llvm.insertelement %[[ARG_1]], %[[INSERT0]][%[[C1]] : i64] : vector<3xf32>
+//       CHECK:   %[[C2:.*]] = llvm.mlir.constant(2 : i64) : i64
+//       CHECK:   %[[INSERT2:.*]] = llvm.insertelement %[[ARG_0]], %[[INSERT1]][%[[C2]] : i64] : vector<3xf32>
+//       CHECK:   return %[[INSERT2]]
+func.func @from_elements_1d(%arg0: f32, %arg1: f32) -> vector<3xf32> {
+  %0 = vector.from_elements %arg0, %arg1, %arg0 : vector<3xf32>
+  return %0 : vector<3xf32>
+}
+
+// -----
+
+// CHECK-LABEL: func.func @from_elements_0d(
+//  CHECK-SAME:     %[[ARG_0:.*]]: f32)
+//       CHECK:   %[[UNDEF:.*]] = llvm.mlir.poison : vector<1xf32>
+//       CHECK:   %[[C0:.*]] = llvm.mlir.constant(0 : i64) : i64
+//       CHECK:   %[[INSERT0:.*]] = llvm.insertelement %[[ARG_0]], %[[UNDEF]][%[[C0]] : i64] : vector<1xf32>
+//       CHECK:   %[[CAST:.*]] = builtin.unrealized_conversion_cast %[[INSERT0]] : vector<1xf32> to vector<f32>
+//       CHECK:   return %[[CAST]]
+func.func @from_elements_0d(%arg0: f32) -> vector<f32> {
+  %0 = vector.from_elements %arg0 : vector<f32>
+  return %0 : vector<f32>
+}
+
+// -----
+
+//===----------------------------------------------------------------------===//
+// vector.step
+//===----------------------------------------------------------------------===//
+
+// CHECK-LABEL: @step_scalable
+// CHECK: %[[STEPVECTOR:.*]] = llvm.intr.stepvector : vector<[4]xi64>
+// CHECK: %[[CAST:.*]] = builtin.unrealized_conversion_cast %[[STEPVECTOR]] : vector<[4]xi64> to vector<[4]xindex>
+// CHECK: return %[[CAST]] : vector<[4]xindex>
+func.func @step_scalable() -> vector<[4]xindex> {
+  %0 = vector.step : vector<[4]xindex>
+  return %0 : vector<[4]xindex>
+}
diff --git a/mlir/test/Conversion/VectorToLLVM/vector-to-llvm.mlir b/mlir/test/Conversion/VectorToLLVM/vector-to-llvm.mlir
index 29bed9aae5682..36b37a137ac1e 100644
--- a/mlir/test/Conversion/VectorToLLVM/vector-to-llvm.mlir
+++ b/mlir/test/Conversion/VectorToLLVM/vector-to-llvm.mlir
@@ -1,92 +1,21 @@
 // RUN: mlir-opt %s -convert-vector-to-llvm -split-input-file | FileCheck %s
 
+//===========================================================================//
+// Complex tests for Vector-to-LLVM conversion
+//
+// These examples, in order to convert to LLVM, require
+//  * `populateVectorToLLVMConversionPatterns`.
+// as well as various other patterns/conversion that are part of
+// `ConvertVectorToLLVMPass`.
+//
+// Please, in the first instance, always try adding tests in
+// vector-to-llvm-interface.mlir instead.
+//===========================================================================//
+
 //===----------------------------------------------------------------------===//
 // vector.bitcast
 //===----------------------------------------------------------------------===//
 
-func.func @bitcast_f32_to_i32_vector_0d(%arg0: vector<f32>) -> vector<i32> {
-  %0 = vector.bitcast %arg0 : vector<f32> to vector<i32>
-  return %0 : vector<i32>
-}
-
-// CHECK-LABEL: @bitcast_f32_to_i32_vector_0d
-// CHECK-SAME:  %[[ARG_0:.*]]: vector<f32>
-// CHECK:       %[[VEC_F32_1D:.*]] = builtin.unrealized_conversion_cast %[[ARG_0]] : vector<f32> to vector<1xf32>
-// CHECK:       %[[VEC_I32_1D:.*]] = llvm.bitcast %[[VEC_F32_1D]] : vector<1xf32> to vector<1xi32>
-// CHECK:       %[[VEC_I32_0D:.*]] = builtin.unrealized_conversion_cast %[[VEC_I32_1D]] : vector<1xi32> to vector<i32>
-// CHECK:       return %[[VEC_I32_0D]] : vector<i32>
-
-// -----
-
-func.func @bitcast_f32_to_i32_vector(%arg0: vector<16xf32>) -> vector<16xi32> {
-  %0 = vector.bitcast %arg0 : vector<16xf32> to vector<16xi32>
-  return %0 : vector<16xi32>
-}
-
-
-// CHECK-LABEL: @bitcast_f32_to_i32_vector
-// CHECK-SAME:  %[[ARG_0:.*]]: vector<16xf32>
-// CHECK:       llvm.bitcast %[[ARG_0]] : vector<16xf32> to vector<16xi32>
-
-// -----
-
-func.func @bitcast_f32_to_i32_vector_scalable(%arg0: vector<[16]xf32>) -> vector<[16]xi32> {
-  %0 = vector.bitcast %arg0 : vector<[16]xf32> to vector<[16]xi32>
-  return %0 : vector<[16]xi32>
-}
-
-// CHECK-LABEL: @bitcast_f32_to_i32_vector_scalable
-// CHECK-SAME:  %[[ARG_0:.*]]: vector<[16]xf32>
-// CHECK:       llvm.bitcast %[[ARG_0]] : vector<[16]xf32> to vector<[16]xi32>
-
-// -----
-
-func.func @bitcast_i8_to_f32_vector(%arg0: vector<64xi8>) -> vector<16xf32> {
-  %0 = vector.bitcast %arg0 : vector<64xi8> to vector<16xf32>
-  return %0 : vector<16xf32>
-}
-
-// CHECK-LABEL: @bitcast_i8_to_f32_vector
-// CHECK-SAME:  %[[ARG_0:.*]]: vector<64xi8>
-// CHECK:       llvm.bitcast %[[ARG_0]] : vector<64xi8> to vector<16xf32>
-
-// -----
-
-func.func @bitcast_i8_to_f32_vector_scalable(%arg0: vector<[64]xi8>) -> vector<[16]xf32> {
-  %0 = vector.bitcast %arg0 : vector<[64]xi8> to vector<[16]xf32>
-  return %0 : vector<[16]xf32>
-}
-
-// CHECK-LABEL: @bitcast_i8_to_f32_vector_scalable
-// CHECK-SAME:  %[[ARG_0:.*]]: vector<[64]xi8>
-// CHECK:       llvm.bitcast %[[ARG_0]] : vector<[64]xi8> to vector<[16]xf32>
-
-// -----
-
-func.func @bitcast_index_to_i8_vector(%arg0: vector<16xindex>) -> vector<128xi8> {
-  %0 = vector.bitcast %arg0 : vector<16xindex> to vector<128xi8>
-  return %0 : vector<128xi8>
-}
-
-// CHECK-LABEL: @bitcast_index_to_i8_vector
-// CHECK-SAME:  %[[ARG_0:.*]]: vector<16xindex>
-// CHECK:       %[[T0:.*]] = builtin.unrealized_conversion_cast %[[ARG_0]] : vector<16xindex> to vector<16xi64>
-// CHECK:       llvm.bitcast %[[T0]] : vector<16xi64> to vector<128xi8>
-
-// -----
-
-func.func @bitcast_index_to_i8_vector_scalable(%arg0: vector<[16]xindex>) -> vector<[128]xi8> {
-  %0 = vector.bitcast %arg0 : vector<[16]xindex> to vector<[128]xi8>
-  return %0 : vector<[128]xi8>
-}
-
-// CHECK-LABEL: @bitcast_index_to_i8_vector_scalable
-// CHECK-SAME:  %[[ARG_0:.*]]: vector<[16]xindex>
-// CHECK:       %[[T0:.*]] = builtin.unrealized_conversion_cast %[[ARG_0]] : vector<[16]xindex> to vector<[16]xi64>
-// CHECK:       llvm.bitcast %[[T0]] : vector<[16]xi64> to vector<[128]xi8>
-
-// -----
-
 // CHECK-LABEL:   func.func @bitcast_2d(
 // CHECK-SAME:      %[[ARG_0:.*]]: vector<2x4xi32>) -> vector<2x2xi64> {
 // CHECK:           %[[T0:.*]] = builtin.unrealized_conversion_cast %[[ARG_0]] : vector<2x4xi32> to !llvm.array<2 x vector<4xi32>>
@@ -135,16 +64,6 @@ func.func @broadcast_vec0d_from_f32(%arg0: f32) -> vector<f32> {
 
 // -----
 
-func.func @broadcast_vec0d_from_vec0d(%arg0: vector<f32>) -> vector<f32> {
-  %0 = vector.broadcast %arg0 : vector<f32> to vector<f32>
-  return %0 : vector<f32>
-}
-// CHECK-LABEL: @broadcast_vec0d_from_vec0d(
-// CHECK-SAME:  %[[A:.*]]: vector<f32>)
-// CHECK:       return %[[A]] : vector<f32>
-
-// -----
-
 func.func @broadcast_vec1d_from_f32(%arg0: f32) -> vector<2xf32> {
   %0 = vector.broadcast %arg0 : f32 to vector<2xf32>
   return %0 : vector<2xf32>
@@ -259,26 +178,6 @@ func.func @broadcast_vec3d_from_scalar_scalable(%arg0: f32) -> vector<2x3x[4]xf3
 
 // -----
 
-func.func @broadcast_vec1d_from_vec1d(%arg0: vector<2xf32>) -> vector<2xf32> {
-  %0 = vector.broadcast %arg0 : vector<2xf32> to vector<2xf32>
-  return %0 : vector<2xf32>
-}
-// CHECK-LABEL: @broadcast_vec1d_from_vec1d(
-// CHECK-SAME:  %[[A:.*]]: vector<2xf32>)
-// CHECK:       return %[[A]] : vector<2xf32>
-
-// -----
-
-func.func @broadcast_vec1d_from_vec1d_scalable(%arg0: vector<[2]xf32>) -> vector<[2]xf32> {
-  %0 = vector.broadcast %arg0 : vector<[2]xf32> to vector<[2]xf32>
-  return %0 : vector<[2]xf32>
-}
-// CHECK-LABEL: @broadcast_vec1d_from_vec1d_scalable(
-// CHECK-SAME:  %[[A:.*]]: vector<[2]xf32>)
-// CHECK:       return %[[A]] : vector<[2]xf32>
-
-// -----
-
 func.func @broadcast_vec2d_from_vec0d(%arg0: vector<f32>) -> vector<3x2xf32> {
   %0 = vector.broadcast %arg0 : vector<f32> to vector<3x2xf32>
   return %0 : vector<3x2xf32>
@@ -286,7 +185,7 @@ func.func @broadcast_vec2d_from_vec0d(%arg0: vector<f32>) -> vector<3x2xf32> {
 // CHECK-LABEL: @broadcast_vec2d_from_vec0d(
 // CHECK-SAME:  %[[A:.*]]: vector<f32>)
 //       CHECK: %[[T0:.*]] = builtin.unrealized_conversion_cast %[[A]] : vector<f32> to vector<1xf32>
-//       CHECK: %[[T1:.*]] = arith.constant dense<0.000000e+00> : vector<3x2xf32>
+//       CHECK: %[[T1:.*]] = ub.poison : vector<3x2xf32>
 //       CHECK: %[[T2:.*]] = builtin.unrealized_conversion_cast %[[T1]] : vector<3x2xf32> to !llvm.array<3 x vector<2xf32>>
 //       CHECK: %[[T4:.*]] = llvm.mlir.constant(0 : index) : i64
 //       CHECK: %[[T5:.*]] = llvm.extractelement %[[T0]][%[[T4]] : i64] : vector<1xf32>
@@ -306,7 +205,7 @@ func.func @broadcast_vec2d_from_vec1d(%arg0: vector<2xf32>) -> vector<3x2xf32> {
 }
 // CHECK-LABEL: @broadcast_vec2d_from_vec1d(
 // CHECK-SAME:  %[[A:.*]]: vector<2xf32>)
-// CHECK:       %[[T0:.*]] = arith.constant dense<0.000000e+00> : vector<3x2xf32>
+// CHECK:       %[[T0:.*]] = ub.poison : vector<3x2xf32>
 // CHECK:       %[[T1:.*]] = builtin.unrealized_conversion_cast %[[T0]] : vector<3x2xf32> to !llvm.array<3 x vector<2xf32>>
 // CHECK:       %[[T2:.*]] = llvm.insertvalue %[[A]], %[[T1]][0] : !llvm.array<3 x vector<2xf32>>
 // CHECK:       %[[T3:.*]] = llvm.insertvalue %[[A]], %[[T2]][1] : !llvm.array<3 x vector<2xf32>>
@@ -322,7 +221,7 @@ func.func @broadcast_vec2d_from_vec1d_scalable(%arg0: vector<[2]xf32>) -> vector
 }
 // CHECK-LABEL: @broadcast_vec2d_from_vec1d_scalable(
 // CHECK-SAME:  %[[A:.*]]: vector<[2]xf32>)
-// CHECK:       %[[T0:.*]] = arith.constant dense<0.000000e+00> : vector<3x[2]xf32>
+// CHECK:       %[[T0:.*]] = ub.poison : vector<3x[2]xf32>
 // CHECK:       %[[T1:.*]] = builtin.unrealized_conversion_cast %[[T0]] : vector<3x[2]xf32> to !llvm.array<3 x vector<[2]xf32>>
 // CHECK:       %[[T2:.*]] = llvm.insertvalue %[[A]], %[[T1]][0] : !llvm.array<3 x vector<[2]xf32>>
 // CHECK:       %[[T3:.*]] = llvm.insertvalue %[[A]], %[[T2]][1] : !llvm.array<3 x vector<[2]xf32>>
@@ -339,7 +238,7 @@ func.func @broadcast_vec2d_from_index_vec1d(%arg0: vector<2xindex>) -> vector<3x
 // CHECK-LABEL: @broadcast_vec2d_from_index_vec1d(
 // CHECK-SAME:  %[[A:.*]]: vector<2xindex>)
 // CHECK:       %[[T1:.*]] = builtin.unrealized_conversion_cast %[[A]] : vector<2xindex> to vector<2xi64>
-// CHECK:       %[[T0:.*]] = arith.constant dense<0> : vector<3x2xindex>
+// CHECK:       %[[T0:.*]] = ub.poison : vector<3x2xindex>
 // CHECK:       %[[T2:.*]] = builtin.unrealized_conversion_cast %[[T0]] : vector<3x2xindex> to !llvm.array<3 x vector<2xi64>>
 // CHECK:       %[[T3:.*]] = llvm.insertvalue %[[T1]], %[[T2]][0] : !llvm.array<3 x vector<2xi64>>
 
@@ -355,7 +254,7 @@ func.func @broadcast_vec2d_from_index_vec1d_scalable(%arg0: vector<[2]xindex>) -
 // CHECK-LABEL: @broadcast_vec2d_from_index_vec1d_scalable(
 // CHECK-SAME:  %[[A:.*]]: vector<[2]xindex>)
 // CHECK:       %[[T1:.*]] = builtin.unrealized_conversion_cast %[[A]] : vector<[2]xindex> to vector<[2]xi64>
-// CHECK:       %[[T0:.*]] = arith.constant dense<0> : vector<3x[2]xindex>
+// CHECK:       %[[T0:.*]] = ub.poison : vector<3x[2]xindex>
 // CHECK:       %[[T2:.*]] = builtin.unrealized_conversion_cast %[[T0]] : vector<3x[2]xindex> to !llvm.array<3 x vector<[2]xi64>>
 // CHECK:       %[[T3:.*]] = llvm.insertvalue %[[T1]], %[[T2]][0] : !llvm.array<3 x vector<[2]xi64>>
 
@@ -370,9 +269,9 @@ func.func @broadcast_vec3d_from_vec1d(%arg0: vector<2xf32>) -> vector<4x3x2xf32>
 }
 // CHECK-LABEL: @broadcast_vec3d_from_vec1d(
 // CHECK-SAME:  %[[A:.*]]: vector<2xf32>)
-// CHECK-DAG:   %[[T0:.*]] = arith.constant dense<0.000000e+00> : vector<3x2xf32>
+// CHECK-DAG:   %[[T0:.*]] = ub.poison : vector<3x2xf32>
 // CHECK-DAG:   %[[T2:.*]] = builtin.unrealized_conversion_cast %[[T0]] : vector<3x2xf32> to !llvm.array<3 x vector<2xf32>>
-// CHECK-DAG:   %[[T1:.*]] = arith.constant dense<0.000000e+00> : vector<4x3x2xf32>
+// CHECK-DAG:   %[[T1:.*]] = ub.poison : vector<4x3x2xf32>
 // CHECK-DAG:   %[[T6:.*]] = builtin.unrealized_conversion_cast %[[T1]] : vector<4x3x2xf32> to !llvm.array<4 x array<3 x vector<2xf32>>>
 
 // CHECK:       %[[T3:.*]] = llvm.insertvalue %[[A]], %[[T2]][0] : !llvm.array<3 x vector<2xf32>>
@@ -395,9 +294,9 @@ func.func @broadcast_vec3d_from_vec1d_scalable(%arg0: vector<[2]xf32>) -> vector
 }
 // CHECK-LABEL: @broadcast_vec3d_from_vec1d_scalable(
 // CHECK-SAME:  %[[A:.*]]: vector<[2]xf32>)
-// CHECK-DAG:   %[[T0:.*]] = arith.constant dense<0.000000e+00> : vector<3x[2]xf32>
+// CHECK-DAG:   %[[T0:.*]] = ub.poison : vector<3x[2]xf32>
 // CHECK-DAG:   %[[T2:.*]] = builtin.unrealized_conversion_cast %[[T0]] : vector<3x[2]xf32> to !llvm.array<3 x vector<[2]xf32>>
-// CHECK-DAG:   %[[T1:.*]] = arith.constant dense<0.000000e+00> : vector<4x3x[2]xf32>
+// CHECK-DAG:   %[[T1:.*]] = ub.poison : vector<4x3x[2]xf32>
 // CHECK-DAG:   %[[T6:.*]] = builtin.unrealized_conversion_cast %[[T1]] : vector<4x3x[2]xf32> to !llvm.array<4 x array<3 x vector<[2]xf32>>>
 
 // CHECK:       %[[T3:.*]] = llvm.insertvalue %[[A]], %[[T2]][0] : !llvm.array<3 x vector<[2]xf32>>
@@ -421,7 +320,7 @@ func.func @broadcast_vec3d_from_vec2d(%arg0: vector<3x2xf32>) -> vector<4x3x2xf3
 // CHECK-LABEL: @broadcast_vec3d_from_vec2d(
 // CHECK-SAME:  %[[A:.*]]: vector<3x2xf32>)
 // CHECK:       %[[T1:.*]] = builtin.unrealized_conversion_cast %[[A]] : vector<3x2xf32> to !llvm.array<3 x vector<2xf32>>
-// CHECK:       %[[T0:.*]] = arith.constant dense<0.000000e+00> : vector<4x3x2xf32>
+// CHECK:       %[[T0:.*]] = ub.poison : vector<4x3x2xf32>
 // CHECK:       %[[T2:.*]] = builtin.unrealized_conversion_cast %[[T0]] : vector<4x3x2xf32> to !llvm.array<4 x array<3 x vector<2xf32>>>
 // CHECK:       %[[T3:.*]] = llvm.insertvalue %[[T1]], %[[T2]][0] : !llvm.array<4 x array<3 x vector<2xf32>>>
 // CHECK:       %[[T5:.*]] = llvm.insertvalue %[[T1]], %[[T3]][1] : !llvm.array<4 x array<3 x vector<2xf32>>>
@@ -439,7 +338,7 @@ func.func @broadcast_vec3d_from_vec2d_scalable(%arg0: vector<3x[2]xf32>) -> vect
 // CHECK-LABEL: @broadcast_vec3d_from_vec2d_scalable(
 // CHECK-SAME:  %[[A:.*]]: vector<3x[2]xf32>)
 // CHECK:       %[[T1:.*]] = builtin.unrealized_conversion_cast %[[A]] : vector<3x[2]xf32> to !llvm.array<3 x vector<[2]xf32>>
-// CHECK:       %[[T0:.*]] = arith.constant dense<0.000000e+00> : vector<4x3x[2]xf32>
+// CHECK:       %[[T0:.*]] = ub.poison : vector<4x3x[2]xf32>
 // CHECK:       %[[T2:.*]] = builtin.unrealized_conversion_cast %[[T0]] : vector<4x3x[2]xf32> to !llvm.array<4 x array<3 x vector<[2]xf32>>>
 // CHECK:       %[[T3:.*]] = llvm.insertvalue %[[T1]], %[[T2]][0] : !llvm.array<4 x array<3 x vector<[2]xf32>>>
 // CHECK:       %[[T5:.*]] = llvm.insertvalue %[[T1]], %[[T3]][1] : !llvm.array<4 x array<3 x vector<[2]xf32>>>
@@ -486,7 +385,7 @@ func.func @broadcast_stretch_at_start(%arg0: vector<1x4xf32>) -> vector<3x4xf32>
 // CHECK-LABEL: @broadcast_stretch_at_start(
 // CHECK-SAME:  %[[A:.*]]: vector<1x4xf32>)
 // CHECK:       %[[T2:.*]] = builtin.unrealized_conversion_cast %[[A]] : vector<1x4xf32> to !llvm.array<1 x vector<4xf32>>
-// CHECK:       %[[T1:.*]] = arith.constant dense<0.000000e+00> : vector<3x4xf32>
+// CHECK:       %[[T1:.*]] = ub.poison : vector<3x4xf32>
 // CHECK:       %[[T4:.*]] = builtin.unrealized_conversion_cast %[[T1]] : vector<3x4xf32> to !llvm.array<3 x vector<4xf32>>
 // CHECK:       %[[T3:.*]] = llvm.extractvalue %[[T2]][0] : !llvm.array<1 x vector<4xf32>>
 // CHECK:       %[[T5:.*]] = llvm.insertvalue %[[T3]], %[[T4]][0] : !llvm.array<3 x vector<4xf32>>
@@ -504,7 +403,7 @@ func.func @broadcast_stretch_at_start_scalable(%arg0: vector<1x[4]xf32>) -> vect
 // CHECK-LABEL: @broadcast_stretch_at_start_scalable(
 // CHECK-SAME:  %[[A:.*]]: vector<1x[4]xf32>)
 // CHECK:       %[[T2:.*]] = builtin.unrealized_conversion_cast %[[A]] : vector<1x[4]xf32> to !llvm.array<1 x vector<[4]xf32>>
-// CHECK:       %[[T1:.*]] = arith.constant dense<0.000000e+00> : vector<3x[4]xf32>
+// CHECK:       %[[T1:.*]] = ub.poison : vector<3x[4]xf32>
 // CHECK:       %[[T4:.*]] = builtin.unrealized_conversion_cast %[[T1]] : vector<3x[4]xf32> to !llvm.array<3 x vector<[4]xf32>>
 // CHECK:       %[[T3:.*]] = llvm.extractvalue %[[T2]][0] : !llvm.array<1 x vector<[4]xf32>>
 // CHECK:       %[[T5:.*]] = llvm.insertvalue %[[T3]], %[[T4]][0] : !llvm.array<3 x vector<[4]xf32>>
@@ -522,7 +421,7 @@ func.func @broadcast_stretch_at_end(%arg0: vector<4x1xf32>) -> vector<4x3xf32> {
 // CHECK-LABEL: @broadcast_stretch_at_end(
 // CHECK-SAME:  %[[A:.*]]: vector<4x1xf32>)
 // CHECK:       %[[T2:.*]] = builtin.unrealized_conversion_cast %[[A]] : vector<4x1xf32> to !llvm.array<4 x vector<1xf32>>
-// CHECK:       %[[T1:.*]] = arith.constant dense<0.000000e+00> : vector<4x3xf32>
+// CHECK:       %[[T1:.*]] = ub.poison : vector<4x3xf32>
 // CHECK:       %[[T7:.*]] = builtin.unrealized_conversion_cast %[[T1]] : vector<4x3xf32> to !llvm.array<4 x vector<3xf32>>
 // CHECK:       %[[T3:.*]] = llvm.extractvalue %[[T2]][0] : !llvm.array<4 x vector<1xf32>>
 // CHECK:       %[[T4:.*]] = llvm.mlir.constant(0 : i64) : i64
@@ -570,9 +469,9 @@ func.func @broadcast_stretch_in_middle(%arg0: vector<4x1x2xf32>) -> vector<4x3x2
 // CHECK-LABEL: @broadcast_stretch_in_middle(
 // CHECK-SAME:  %[[A:.*]]: vector<4x1x2xf32>) -> vector<4x3x2xf32> {
 // CHECK:       %[[T3:.*]] = builtin.unrealized_conversion_cast %[[A]] : vector<4x1x2xf32> to !llvm.array<4 x array<1 x vector<2xf32>>>
-// CHECK:       %[[T1:.*]] = arith.constant dense<0.000000e+00> : vector<4x3x2xf32>
+// CHECK:       %[[T1:.*]] = ub.poison : vector<4x3x2xf32>
 // CHECK:       %[[T9:.*]] = builtin.unrealized_conversion_cast %[[T1]] : vector<4x3x2xf32> to !llvm.array<4 x array<3 x vector<2xf32>>>
-// CHECK:       %[[T2:.*]] = arith.constant dense<0.000000e+00> : vector<3x2xf32>
+// CHECK:       %[[T2:.*]] = ub.poison : vector<3x2xf32>
 // CHECK:       %[[T5:.*]] = builtin.unrealized_conversion_cast %[[T2]] : vector<3x2xf32> to !llvm.array<3 x vector<2xf32>>
 // CHECK:       %[[T4:.*]] = llvm.extractvalue %[[T3]][0, 0] : !llvm.array<4 x array<1 x vector<2xf32>>>
 // CHECK:       %[[T6:.*]] = llvm.insertvalue %[[T4]], %[[T5]][0] : !llvm.array<3 x vector<2xf32>>
@@ -606,9 +505,9 @@ func.func @broadcast_stretch_in_middle_scalable_v1(%arg0: vector<4x1x[2]xf32>) -
 // CHECK-LABEL: @broadcast_stretch_in_middle_scalable_v1(
 // CHECK-SAME:  %[[A:.*]]: vector<4x1x[2]xf32>) -> vector<4x3x[2]xf32> {
 // CHECK:       %[[T3:.*]] = builtin.unrealized_conversion_cast %[[A]] : vector<4x1x[2]xf32> to !llvm.array<4 x array<1 x vector<[2]xf32>>>
-// CHECK:       %[[T1:.*]] = arith.constant dense<0.000000e+00> : vector<4x3x[2]xf32>
+// CHECK:       %[[T1:.*]] = ub.poison : vector<4x3x[2]xf32>
 // CHECK:       %[[T9:.*]] = builtin.unrealized_conversion_cast %[[T1]] : vector<4x3x[2]xf32> to !llvm.array<4 x array<3 x vector<[2]xf32>>>
-// CHECK:       %[[T2:.*]] = arith.constant dense<0.000000e+00> : vector<3x[2]xf32>
+// CHECK:       %[[T2:.*]] = ub.poison : vector<3x[2]xf32>
 // CHECK:       %[[T5:.*]] = builtin.unrealized_conversion_cast %[[T2]] : vector<3x[2]xf32> to !llvm.array<3 x vector<[2]xf32>>
 // CHECK:       %[[T4:.*]] = llvm.extractvalue %[[T3]][0, 0] : !llvm.array<4 x array<1 x vector<[2]xf32>>>
 // CHECK:       %[[T6:.*]] = llvm.insertvalue %[[T4]], %[[T5]][0] : !llvm.array<3 x vector<[2]xf32>>
@@ -1068,1048 +967,301 @@ func.func @masked_int_or_outerprod_scalable(%arg0: vector<[2]xi32>, %arg1: i32,
 // -----
 
 //===----------------------------------------------------------------------===//
-// vector.shuffle
+// vector.extract
 //===----------------------------------------------------------------------===//
 
-func.func @shuffle_0D_direct(%arg0: vector<f32>) -> vector<3xf32> {
-  %1 = vector.shuffle %arg0, %arg0 [0, 1, 0] : vector<f32>, vector<f32>
-  return %1 : vector<3xf32>
+// FIXME: Segfaults for --convert-to-llvm="filter-dialects=vector"
+func.func @extract_scalar_from_vec_1d_f32_poison_idx(%arg0: vector<16xf32>) -> f32 {
+  %0 = vector.extract %arg0[-1]: f32 from vector<16xf32>
+  return %0 : f32
 }
-// CHECK-LABEL: @shuffle_0D_direct(
-//  CHECK-SAME:     %[[A:.*]]: vector<f32>
-//       CHECK:   %[[c:.*]] = builtin.unrealized_conversion_cast %[[A]] : vector<f32> to vector<1xf32>
-//       CHECK:   %[[s:.*]] = llvm.shufflevector %[[c]], %[[c]] [0, 1, 0] : vector<1xf32>
-//       CHECK:   return %[[s]] : vector<3xf32>
+// CHECK-LABEL: @extract_scalar_from_vec_1d_f32_poison_idx
+//       CHECK:   %[[UB:.*]] = ub.poison : f32
+//       CHECK:   return %[[UB]] : f32
 
 // -----
 
-func.func @shuffle_1D_direct(%arg0: vector<2xf32>, %arg1: vector<2xf32>) -> vector<2xf32> {
-  %1 = vector.shuffle %arg0, %arg1 [0, 1] : vector<2xf32>, vector<2xf32>
-  return %1 : vector<2xf32>
+// FIXME: Segfaults for --convert-to-llvm="filter-dialects=vector"
+func.func @extract_vec_2d_from_vec_3d_f32_poison_idx(%arg0: vector<4x3x16xf32>) -> vector<3x16xf32> {
+  %0 = vector.extract %arg0[-1]: vector<3x16xf32> from vector<4x3x16xf32>
+  return %0 : vector<3x16xf32>
 }
-// CHECK-LABEL: @shuffle_1D_direct(
-// CHECK-SAME: %[[A:.*]]: vector<2xf32>,
-// CHECK-SAME: %[[B:.*]]: vector<2xf32>)
-//       CHECK:   return %[[A:.*]]: vector<2xf32>
+// CHECK-LABEL: @extract_vec_2d_from_vec_3d_f32_poison_idx
+//       CHECK:   %[[UB:.*]] = ub.poison : vector<3x16xf32>
+//       CHECK:   return %[[UB]] : vector<3x16xf32>
 
 // -----
 
-func.func @shuffle_1D_index_direct(%arg0: vector<2xindex>, %arg1: vector<2xindex>) -> vector<2xindex> {
-  %1 = vector.shuffle %arg0, %arg1 [0, 1] : vector<2xindex>, vector<2xindex>
-  return %1 : vector<2xindex>
+//===----------------------------------------------------------------------===//
+// vector.print
+//===----------------------------------------------------------------------===//
+
+func.func @print_scalar_i1(%arg0: i1) {
+  vector.print %arg0 : i1
+  return
 }
-// CHECK-LABEL: @shuffle_1D_index_direct(
-// CHECK-SAME: %[[A:.*]]: vector<2xindex>,
-// CHECK-SAME: %[[B:.*]]: vector<2xindex>)
-//       CHECK:   return  %[[A:.*]]: vector<2xindex>
+//
+// Type "boolean" always uses zero extension.
+//
+// CHECK-LABEL: @print_scalar_i1(
+// CHECK-SAME: %[[A:.*]]: i1)
+//       CHECK: %[[S:.*]] = arith.extui %[[A]] : i1 to i64
+//       CHECK: llvm.call @printI64(%[[S]]) : (i64) -> ()
+//       CHECK: llvm.call @printNewline() : () -> ()
 
 // -----
 
-func.func @shuffle_poison_mask(%arg0: vector<2xf32>, %arg1: vector<2xf32>) -> vector<4xf32> {
-  %1 = vector.shuffle %arg0, %arg1 [0, -1, 3, -1] : vector<2xf32>, vector<2xf32>
-  return %1 : vector<4xf32>
+func.func @print_scalar_i4(%arg0: i4) {
+  vector.print %arg0 : i4
+  return
 }
-// CHECK-LABEL: @shuffle_poison_mask(
-//  CHECK-SAME:   %[[A:.*]]: vector<2xf32>, %[[B:.*]]: vector<2xf32>)
-//       CHECK:     %[[s:.*]] = llvm.shufflevector %[[A]], %[[B]] [0, -1, 3, -1] : vector<2xf32>
+// CHECK-LABEL: @print_scalar_i4(
+// CHECK-SAME: %[[A:.*]]: i4)
+//       CHECK: %[[S:.*]] = arith.extsi %[[A]] : i4 to i64
+//       CHECK: llvm.call @printI64(%[[S]]) : (i64) -> ()
+//       CHECK: llvm.call @printNewline() : () -> ()
 
 // -----
 
-func.func @shuffle_1D(%arg0: vector<2xf32>, %arg1: vector<3xf32>) -> vector<5xf32> {
-  %1 = vector.shuffle %arg0, %arg1 [4, 3, 2, 1, 0] : vector<2xf32>, vector<3xf32>
-  return %1 : vector<5xf32>
+func.func @print_scalar_si4(%arg0: si4) {
+  vector.print %arg0 : si4
+  return
 }
-// CHECK-LABEL: @shuffle_1D(
-// CHECK-SAME: %[[A:.*]]: vector<2xf32>,
-// CHECK-SAME: %[[B:.*]]: vector<3xf32>)
-//       CHECK:   %[[U0:.*]] = llvm.mlir.undef : vector<5xf32>
-//       CHECK:   %[[C2:.*]] = llvm.mlir.constant(2 : index) : i64
-//       CHECK:   %[[E1:.*]] = llvm.extractelement %[[B]][%[[C2]] : i64] : vector<3xf32>
-//       CHECK:   %[[C0:.*]] = llvm.mlir.constant(0 : index) : i64
-//       CHECK:   %[[I1:.*]] = llvm.insertelement %[[E1]], %[[U0]][%[[C0]] : i64] : vector<5xf32>
-//       CHECK:   %[[C1:.*]] = llvm.mlir.constant(1 : index) : i64
-//       CHECK:   %[[E2:.*]] = llvm.extractelement %[[B]][%[[C1]] : i64] : vector<3xf32>
-//       CHECK:   %[[C1:.*]] = llvm.mlir.constant(1 : index) : i64
-//       CHECK:   %[[I2:.*]] = llvm.insertelement %[[E2]], %[[I1]][%[[C1]] : i64] : vector<5xf32>
-//       CHECK:   %[[C0:.*]] = llvm.mlir.constant(0 : index) : i64
-//       CHECK:   %[[E3:.*]] = llvm.extractelement %[[B]][%[[C0]] : i64] : vector<3xf32>
-//       CHECK:   %[[C2:.*]] = llvm.mlir.constant(2 : index) : i64
-//       CHECK:   %[[I3:.*]] = llvm.insertelement %[[E3]], %[[I2]][%[[C2]] : i64] : vector<5xf32>
-//       CHECK:   %[[C1:.*]] = llvm.mlir.constant(1 : index) : i64
-//       CHECK:   %[[E4:.*]] = llvm.extractelement %[[A]][%[[C1]] : i64] : vector<2xf32>
-//       CHECK:   %[[C3:.*]] = llvm.mlir.constant(3 : index) : i64
-//       CHECK:   %[[I4:.*]] = llvm.insertelement %[[E4]], %[[I3]][%[[C3]] : i64] : vector<5xf32>
-//       CHECK:   %[[C0:.*]] = llvm.mlir.constant(0 : index) : i64
-//       CHECK:   %[[E5:.*]] = llvm.extractelement %[[A]][%[[C0]] : i64] : vector<2xf32>
-//       CHECK:   %[[C4:.*]] = llvm.mlir.constant(4 : index) : i64
-//       CHECK:   %[[I5:.*]] = llvm.insertelement %[[E5]], %[[I4]][%[[C4]] : i64] : vector<5xf32>
-//       CHECK:   return %[[I5]] : vector<5xf32>
+// CHECK-LABEL: @print_scalar_si4(
+// CHECK-SAME: %[[A:.*]]: si4)
+//       CHECK: %[[C:.*]] = builtin.unrealized_conversion_cast %[[A]] : si4 to i4
+//       CHECK: %[[S:.*]] = arith.extsi %[[C]] : i4 to i64
+//       CHECK: llvm.call @printI64(%[[S]]) : (i64) -> ()
+//       CHECK: llvm.call @printNewline() : () -> ()
 
 // -----
 
-func.func @shuffle_2D(%a: vector<1x4xf32>, %b: vector<2x4xf32>) -> vector<3x4xf32> {
-  %1 = vector.shuffle %a, %b[1, 0, 2] : vector<1x4xf32>, vector<2x4xf32>
-  return %1 : vector<3x4xf32>
+func.func @print_scalar_ui4(%arg0: ui4) {
+  vector.print %arg0 : ui4
+  return
 }
-// CHECK-LABEL: @shuffle_2D(
-// CHECK-SAME: %[[A:.*]]: vector<1x4xf32>,
-// CHECK-SAME: %[[B:.*]]: vector<2x4xf32>)
-//       CHECK-DAG:   %[[VAL_0:.*]] = builtin.unrealized_conversion_cast %[[A]] : vector<1x4xf32> to !llvm.array<1 x vector<4xf32>>
-//       CHECK-DAG:   %[[VAL_1:.*]] = builtin.unrealized_conversion_cast %[[B]] : vector<2x4xf32> to !llvm.array<2 x vector<4xf32>>
-//       CHECK:   %[[U0:.*]] = llvm.mlir.undef : !llvm.array<3 x vector<4xf32>>
-//       CHECK:   %[[E1:.*]] = llvm.extractvalue %[[VAL_1]][0] : !llvm.array<2 x vector<4xf32>>
-//       CHECK:   %[[I1:.*]] = llvm.insertvalue %[[E1]], %[[U0]][0] : !llvm.array<3 x vector<4xf32>>
-//       CHECK:   %[[E2:.*]] = llvm.extractvalue %[[VAL_0]][0] : !llvm.array<1 x vector<4xf32>>
-//       CHECK:   %[[I2:.*]] = llvm.insertvalue %[[E2]], %[[I1]][1] : !llvm.array<3 x vector<4xf32>>
-//       CHECK:   %[[E3:.*]] = llvm.extractvalue %[[VAL_1]][1] : !llvm.array<2 x vector<4xf32>>
-//       CHECK:   %[[I3:.*]] = llvm.insertvalue %[[E3]], %[[I2]][2] : !llvm.array<3 x vector<4xf32>>
-//       CHECK:   %[[VAL_3:.*]] = builtin.unrealized_conversion_cast %[[I3]] : !llvm.array<3 x vector<4xf32>> to vector<3x4xf32>
-//       CHECK:   return %[[VAL_3]] : vector<3x4xf32>
+// CHECK-LABEL: @print_scalar_ui4(
+// CHECK-SAME: %[[A:.*]]: ui4)
+//       CHECK: %[[C:.*]] = builtin.unrealized_conversion_cast %[[A]] : ui4 to i4
+//       CHECK: %[[S:.*]] = arith.extui %[[C]] : i4 to i64
+//       CHECK: llvm.call @printU64(%[[S]]) : (i64) -> ()
+//       CHECK: llvm.call @printNewline() : () -> ()
 
 // -----
 
-//===----------------------------------------------------------------------===//
-// vector.extractelement
-//===----------------------------------------------------------------------===//
-
-func.func @extractelement_from_vec_0d_f32(%arg0: vector<f32>) -> f32 {
-  %1 = vector.extractelement %arg0[] : vector<f32>
-  return %1 : f32
+func.func @print_scalar_i32(%arg0: i32) {
+  vector.print %arg0 : i32
+  return
 }
-// CHECK-LABEL: @extractelement_from_vec_0d_f32
-//       CHECK:   %[[C0:.*]] = llvm.mlir.constant(0 : index) : i64
-//       CHECK:   llvm.extractelement %{{.*}}[%[[C0]] : {{.*}}] : vector<1xf32>
+// CHECK-LABEL: @print_scalar_i32(
+// CHECK-SAME: %[[A:.*]]: i32)
+//       CHECK: %[[S:.*]] = arith.extsi %[[A]] : i32 to i64
+//       CHECK: llvm.call @printI64(%[[S]]) : (i64) -> ()
+//       CHECK: llvm.call @printNewline() : () -> ()
 
 // -----
 
-func.func @extractelement_from_vec_1d_f32_idx_as_i32(%arg0: vector<16xf32>) -> f32 {
-  %0 = arith.constant 15 : i32
-  %1 = vector.extractelement %arg0[%0 : i32]: vector<16xf32>
-  return %1 : f32
+func.func @print_scalar_ui32(%arg0: ui32) {
+  vector.print %arg0 : ui32
+  return
 }
-// CHECK-LABEL: @extractelement_from_vec_1d_f32_idx_as_i32(
-//  CHECK-SAME:   %[[A:.*]]: vector<16xf32>)
-//       CHECK:   %[[C:.*]] = arith.constant 15 : i32
-//       CHECK:   %[[X:.*]] = llvm.extractelement %[[A]][%[[C]] : i32] : vector<16xf32>
-//       CHECK:   return %[[X]] : f32
+// CHECK-LABEL: @print_scalar_ui32(
+// CHECK-SAME: %[[A:.*]]: ui32)
+//       CHECK: %[[C:.*]] = builtin.unrealized_conversion_cast %[[A]] : ui32 to i32
+//       CHECK: %[[S:.*]] = arith.extui %[[C]] : i32 to i64
+//       CHECK: llvm.call @printU64(%[[S]]) : (i64) -> ()
 
 // -----
 
-func.func @extractelement_from_vec_1d_f32_idx_as_i32_scalable(%arg0: vector<[16]xf32>) -> f32 {
-  %0 = arith.constant 15 : i32
-  %1 = vector.extractelement %arg0[%0 : i32]: vector<[16]xf32>
-  return %1 : f32
+func.func @print_scalar_i40(%arg0: i40) {
+  vector.print %arg0 : i40
+  return
 }
-// CHECK-LABEL: @extractelement_from_vec_1d_f32_idx_as_i32_scalable(
-//  CHECK-SAME:   %[[A:.*]]: vector<[16]xf32>)
-//       CHECK:   %[[C:.*]] = arith.constant 15 : i32
-//       CHECK:   %[[X:.*]] = llvm.extractelement %[[A]][%[[C]] : i32] : vector<[16]xf32>
-//       CHECK:   return %[[X]] : f32
+// CHECK-LABEL: @print_scalar_i40(
+// CHECK-SAME: %[[A:.*]]: i40)
+//       CHECK: %[[S:.*]] = arith.extsi %[[A]] : i40 to i64
+//       CHECK: llvm.call @printI64(%[[S]]) : (i64) -> ()
+//       CHECK: llvm.call @printNewline() : () -> ()
 
 // -----
-func.func @extractelement_from_vec_1d_f32_idx_as_index(%arg0: vector<16xf32>) -> f32 {
-  %0 = arith.constant 15 : index
-  %1 = vector.extractelement %arg0[%0 : index]: vector<16xf32>
-  return %1 : f32
+
+func.func @print_scalar_si40(%arg0: si40) {
+  vector.print %arg0 : si40
+  return
 }
-// CHECK-LABEL: @extractelement_from_vec_1d_f32_idx_as_index(
-//  CHECK-SAME:   %[[A:.*]]: vector<16xf32>)
-//       CHECK:   %[[C:.*]] = arith.constant 15 : index
-//       CHECK:   %[[I:.*]] = builtin.unrealized_conversion_cast %[[C]] : index to i64
-//       CHECK:   %[[X:.*]] = llvm.extractelement %[[A]][%[[I]] : i64] : vector<16xf32>
-//       CHECK:   return %[[X]] : f32
+// CHECK-LABEL: @print_scalar_si40(
+// CHECK-SAME: %[[A:.*]]: si40)
+//       CHECK: %[[C:.*]] = builtin.unrealized_conversion_cast %[[A]] : si40 to i40
+//       CHECK: %[[S:.*]] = arith.extsi %[[C]] : i40 to i64
+//       CHECK: llvm.call @printI64(%[[S]]) : (i64) -> ()
+//       CHECK: llvm.call @printNewline() : () -> ()
 
 // -----
 
-func.func @extractelement_from_vec_1d_f32_idx_as_index_scalable(%arg0: vector<[16]xf32>) -> f32 {
-  %0 = arith.constant 15 : index
-  %1 = vector.extractelement %arg0[%0 : index]: vector<[16]xf32>
-  return %1 : f32
+func.func @print_scalar_ui40(%arg0: ui40) {
+  vector.print %arg0 : ui40
+  return
 }
-// CHECK-LABEL: @extractelement_from_vec_1d_f32_idx_as_index_scalable(
-//  CHECK-SAME:   %[[A:.*]]: vector<[16]xf32>)
-//       CHECK:   %[[C:.*]] = arith.constant 15 : index
-//       CHECK:   %[[I:.*]] = builtin.unrealized_conversion_cast %[[C]] : index to i64
-//       CHECK:   %[[X:.*]] = llvm.extractelement %[[A]][%[[I]] : i64] : vector<[16]xf32>
-//       CHECK:   return %[[X]] : f32
+// CHECK-LABEL: @print_scalar_ui40(
+// CHECK-SAME: %[[A:.*]]: ui40)
+//       CHECK: %[[C:.*]] = builtin.unrealized_conversion_cast %[[A]] : ui40 to i40
+//       CHECK: %[[S:.*]] = arith.extui %[[C]] : i40 to i64
+//       CHECK: llvm.call @printU64(%[[S]]) : (i64) -> ()
+//       CHECK: llvm.call @printNewline() : () -> ()
 
 // -----
 
 //===----------------------------------------------------------------------===//
-// vector.extract
+// vector.extract_strided_slice
 //===----------------------------------------------------------------------===//
 
-func.func @extract_scalar_from_vec_1d_f32(%arg0: vector<16xf32>) -> f32 {
-  %0 = vector.extract %arg0[15]: f32 from vector<16xf32>
-  return %0 : f32
+func.func @extract_strided_slice_f32_1d_from_1d(%arg0: vector<4xf32>) -> vector<2xf32> {
+  %0 = vector.extract_strided_slice %arg0 {offsets = [2], sizes = [2], strides = [1]} : vector<4xf32> to vector<2xf32>
+  return %0 : vector<2xf32>
 }
-// CHECK-LABEL: @extract_scalar_from_vec_1d_f32
-//       CHECK:   llvm.mlir.constant(15 : i64) : i64
-//       CHECK:   llvm.extractelement {{.*}}[{{.*}} : i64] : vector<16xf32>
-//       CHECK:   return {{.*}} : f32
-
-// -----
+// CHECK-LABEL: @extract_strided_slice_f32_1d_from_1d
+//  CHECK-SAME:    %[[A:.*]]: vector<4xf32>)
+//       CHECK:    %[[T0:.*]] = llvm.shufflevector %[[A]], %[[A]] [2, 3] : vector<4xf32>
+//       CHECK:    return %[[T0]] : vector<2xf32>
 
-func.func @extract_scalar_from_vec_1d_f32_scalable(%arg0: vector<[16]xf32>) -> f32 {
-  %0 = vector.extract %arg0[15]: f32 from vector<[16]xf32>
-  return %0 : f32
-}
-// CHECK-LABEL: @extract_scalar_from_vec_1d_f32_scalable
-//       CHECK:   llvm.mlir.constant(15 : i64) : i64
-//       CHECK:   llvm.extractelement {{.*}}[{{.*}} : i64] : vector<[16]xf32>
-//       CHECK:   return {{.*}} : f32
+// NOTE: For scalable vectors we could only extract vector<[4]xf32> from vector<[4]xf32>, but that would be a NOP.
 
 // -----
 
-func.func @extract_vec_1e_from_vec_1d_f32(%arg0: vector<16xf32>) -> vector<1xf32> {
-  %0 = vector.extract %arg0[15]: vector<1xf32> from vector<16xf32>
-  return %0 : vector<1xf32>
+func.func @extract_strided_slice_index_1d_from_1d(%arg0: vector<4xindex>) -> vector<2xindex> {
+  %0 = vector.extract_strided_slice %arg0 {offsets = [2], sizes = [2], strides = [1]} : vector<4xindex> to vector<2xindex>
+  return %0 : vector<2xindex>
 }
-// CHECK-LABEL: @extract_vec_1e_from_vec_1d_f32(
-//  CHECK-SAME:   %[[A:.*]]: vector<16xf32>)
-//       CHECK:   %[[T0:.*]] = llvm.mlir.constant(15 : i64) : i64
-//       CHECK:   %[[T1:.*]] = llvm.extractelement %[[A]][%[[T0]] : i64] : vector<16xf32>
-//       CHECK:   %[[T2:.*]] = builtin.unrealized_conversion_cast %[[T1]] : f32 to vector<1xf32>
-//       CHECK:   return %[[T2]] : vector<1xf32>
+// CHECK-LABEL: @extract_strided_slice_index_1d_from_1d
+//  CHECK-SAME:    %[[A:.*]]: vector<4xindex>)
+//       CHECK:    %[[T0:.*]] = builtin.unrealized_conversion_cast %[[A]] : vector<4xindex> to vector<4xi64>
+//       CHECK:    %[[T2:.*]] = llvm.shufflevector %[[T0]], %[[T0]] [2, 3] : vector<4xi64>
+//       CHECK:    %[[T3:.*]] = builtin.unrealized_conversion_cast %[[T2]] : vector<2xi64> to vector<2xindex>
+//       CHECK:    return %[[T3]] : vector<2xindex>
+
+// NOTE: For scalable vectors we could only extract vector<[4]xindex> from vector<[4]xindex>, but that would be a NOP.
 
 // -----
 
-func.func @extract_vec_1e_from_vec_1d_f32_scalable(%arg0: vector<[16]xf32>) -> vector<1xf32> {
-  %0 = vector.extract %arg0[15]: vector<1xf32> from vector<[16]xf32>
-  return %0 : vector<1xf32>
+func.func @extract_strided_slice_f32_1d_from_2d(%arg0: vector<4x8xf32>) -> vector<2x8xf32> {
+  %0 = vector.extract_strided_slice %arg0 {offsets = [2], sizes = [2], strides = [1]} : vector<4x8xf32> to vector<2x8xf32>
+  return %0 : vector<2x8xf32>
 }
-// CHECK-LABEL: @extract_vec_1e_from_vec_1d_f32_scalable(
-//  CHECK-SAME:   %[[A:.*]]: vector<[16]xf32>)
-//       CHECK:   %[[T0:.*]] = llvm.mlir.constant(15 : i64) : i64
-//       CHECK:   %[[T1:.*]] = llvm.extractelement %[[A]][%[[T0]] : i64] : vector<[16]xf32>
-//       CHECK:   %[[T2:.*]] = builtin.unrealized_conversion_cast %[[T1]] : f32 to vector<1xf32>
-//       CHECK:   return %[[T2]] : vector<1xf32>
+// CHECK-LABEL: @extract_strided_slice_f32_1d_from_2d(
+//  CHECK-SAME:    %[[ARG:.*]]: vector<4x8xf32>)
+//       CHECK:    %[[A:.*]] = builtin.unrealized_conversion_cast %[[ARG]] : vector<4x8xf32> to !llvm.array<4 x vector<8xf32>>
+//       CHECK:    %[[T0:.*]] = llvm.mlir.poison : !llvm.array<2 x vector<8xf32>>
+//       CHECK:    %[[T1:.*]] = llvm.extractvalue %[[A]][2] : !llvm.array<4 x vector<8xf32>>
+//       CHECK:    %[[T2:.*]] = llvm.insertvalue %[[T1]], %[[T0]][0] : !llvm.array<2 x vector<8xf32>>
+//       CHECK:    %[[T3:.*]] = llvm.extractvalue %[[A]][3] : !llvm.array<4 x vector<8xf32>>
+//       CHECK:    %[[T4:.*]] = llvm.insertvalue %[[T3]], %[[T2]][1] : !llvm.array<2 x vector<8xf32>>
+//       CHECK:    %[[T5:.*]] = builtin.unrealized_conversion_cast %[[T4]] : !llvm.array<2 x vector<8xf32>> to vector<2x8xf32>
+//       CHECK:    return %[[T5]]
 
 // -----
 
-func.func @extract_scalar_from_vec_1d_index(%arg0: vector<16xindex>) -> index {
-  %0 = vector.extract %arg0[15]: index from vector<16xindex>
-  return %0 : index
+func.func @extract_strided_slice_f32_1d_from_2d_scalable(%arg0: vector<4x[8]xf32>) -> vector<2x[8]xf32> {
+  %0 = vector.extract_strided_slice %arg0 {offsets = [2], sizes = [2], strides = [1]} : vector<4x[8]xf32> to vector<2x[8]xf32>
+  return %0 : vector<2x[8]xf32>
 }
-// CHECK-LABEL: @extract_scalar_from_vec_1d_index(
-//  CHECK-SAME:   %[[A:.*]]: vector<16xindex>)
-//       CHECK:   %[[T0:.*]] = builtin.unrealized_conversion_cast %[[A]] : vector<16xindex> to vector<16xi64>
-//       CHECK:   %[[T1:.*]] = llvm.mlir.constant(15 : i64) : i64
-//       CHECK:   %[[T2:.*]] = llvm.extractelement %[[T0]][%[[T1]] : i64] : vector<16xi64>
-//       CHECK:   %[[T3:.*]] = builtin.unrealized_conversion_cast %[[T2]] : i64 to index
-//       CHECK:   return %[[T3]] : index
+// CHECK-LABEL:   func.func @extract_strided_slice_f32_1d_from_2d_scalable(
+//  CHECK-SAME:    %[[ARG:.*]]: vector<4x[8]xf32>)
+//       CHECK:    %[[A:.*]] = builtin.unrealized_conversion_cast %[[ARG]] : vector<4x[8]xf32> to !llvm.array<4 x vector<[8]xf32>>
+//       CHECK:    %[[CST:.*]] = arith.constant dense<0.000000e+00> : vector<2x[8]xf32>
+//       CHECK:    %[[DST:.*]] = builtin.unrealized_conversion_cast %[[CST]] : vector<2x[8]xf32> to !llvm.array<2 x vector<[8]xf32>>
+//       CHECK:    %[[E0:.*]] = llvm.extractvalue %[[A]][2] : !llvm.array<4 x vector<[8]xf32>>
+//       CHECK:    %[[E1:.*]] = llvm.extractvalue %[[A]][3] : !llvm.array<4 x vector<[8]xf32>>
+//       CHECK:    %[[I0:.*]] = llvm.insertvalue %[[E0]], %[[DST]][0] : !llvm.array<2 x vector<[8]xf32>>
+//       CHECK:    %[[I1:.*]] = llvm.insertvalue %[[E1]], %[[I0]][1] : !llvm.array<2 x vector<[8]xf32>>
+//       CHECK:    %[[RES:.*]] = builtin.unrealized_conversion_cast %[[I1]] : !llvm.array<2 x vector<[8]xf32>> to vector<2x[8]xf32>
+//       CHECK:    return %[[RES]]
 
 // -----
 
-func.func @extract_scalar_from_vec_1d_index_scalable(%arg0: vector<[16]xindex>) -> index {
-  %0 = vector.extract %arg0[15]: index from vector<[16]xindex>
-  return %0 : index
+func.func @extract_strided_slice_f32_2d_from_2d(%arg0: vector<4x8xf32>) -> vector<2x2xf32> {
+  %0 = vector.extract_strided_slice %arg0 {offsets = [2, 2], sizes = [2, 2], strides = [1, 1]} : vector<4x8xf32> to vector<2x2xf32>
+  return %0 : vector<2x2xf32>
 }
-// CHECK-LABEL: @extract_scalar_from_vec_1d_index_scalable(
-//  CHECK-SAME:   %[[A:.*]]: vector<[16]xindex>)
-//       CHECK:   %[[T0:.*]] = builtin.unrealized_conversion_cast %[[A]] : vector<[16]xindex> to vector<[16]xi64>
-//       CHECK:   %[[T1:.*]] = llvm.mlir.constant(15 : i64) : i64
-//       CHECK:   %[[T2:.*]] = llvm.extractelement %[[T0]][%[[T1]] : i64] : vector<[16]xi64>
-//       CHECK:   %[[T3:.*]] = builtin.unrealized_conversion_cast %[[T2]] : i64 to index
-//       CHECK:   return %[[T3]] : index
+// CHECK-LABEL: @extract_strided_slice_f32_2d_from_2d(
+//  CHECK-SAME:    %[[ARG:.*]]: vector<4x8xf32>)
+//       CHECK:    %[[A:.*]] = builtin.unrealized_conversion_cast %[[ARG]] : vector<4x8xf32> to !llvm.array<4 x vector<8xf32>>
+//       CHECK:    %[[VAL_2:.*]] = arith.constant dense<0.000000e+00> : vector<2x2xf32>
+//       CHECK:    %[[VAL_6:.*]] = builtin.unrealized_conversion_cast %[[VAL_2]] : vector<2x2xf32> to !llvm.array<2 x vector<2xf32>>
+//       CHECK:    %[[T2:.*]] = llvm.extractvalue %[[A]][2] : !llvm.array<4 x vector<8xf32>>
+//       CHECK:    %[[T3:.*]] = llvm.shufflevector %[[T2]], %[[T2]] [2, 3] : vector<8xf32>
+//       CHECK:    %[[T4:.*]] = llvm.insertvalue %[[T3]], %[[VAL_6]][0] : !llvm.array<2 x vector<2xf32>>
+//       CHECK:    %[[T5:.*]] = llvm.extractvalue %[[A]][3] : !llvm.array<4 x vector<8xf32>>
+//       CHECK:    %[[T6:.*]] = llvm.shufflevector %[[T5]], %[[T5]] [2, 3] : vector<8xf32>
+//       CHECK:    %[[T7:.*]] = llvm.insertvalue %[[T6]], %[[T4]][1] : !llvm.array<2 x vector<2xf32>>
+//       CHECK:    %[[VAL_12:.*]] = builtin.unrealized_conversion_cast %[[T7]] : !llvm.array<2 x vector<2xf32>> to vector<2x2xf32>
+//       CHECK:    return %[[VAL_12]] : vector<2x2xf32>
 
 // -----
 
-func.func @extract_vec_2d_from_vec_3d_f32(%arg0: vector<4x3x16xf32>) -> vector<3x16xf32> {
-  %0 = vector.extract %arg0[0]: vector<3x16xf32> from vector<4x3x16xf32>
-  return %0 : vector<3x16xf32>
+// NOTE: For scalable vectors, we can only extract "full" scalable dimensions
+// (e.g. [8] from [8], but not [4] from [8]).
+
+func.func @extract_strided_slice_f32_2d_from_2d_scalable(%arg0: vector<4x[8]xf32>) -> vector<2x[8]xf32> {
+  %0 = vector.extract_strided_slice %arg0 {offsets = [2, 0], sizes = [2, 8], strides = [1, 1]} : vector<4x[8]xf32> to vector<2x[8]xf32>
+  return %0 : vector<2x[8]xf32>
 }
-// CHECK-LABEL: @extract_vec_2d_from_vec_3d_f32
-//       CHECK:   llvm.extractvalue {{.*}}[0] : !llvm.array<4 x array<3 x vector<16xf32>>>
-//       CHECK:   return {{.*}} : vector<3x16xf32>
+// CHECK-LABEL: @extract_strided_slice_f32_2d_from_2d_scalable(
+//  CHECK-SAME:     %[[ARG:.*]]: vector<4x[8]xf32>)
+// CHECK:           %[[T1:.*]] = builtin.unrealized_conversion_cast %[[ARG]] : vector<4x[8]xf32> to !llvm.array<4 x vector<[8]xf32>>
+// CHECK:           %[[T3:.*]] = arith.constant dense<0.000000e+00> : vector<2x[8]xf32>
+// CHECK:           %[[T4:.*]] = builtin.unrealized_conversion_cast %[[T3]] : vector<2x[8]xf32> to !llvm.array<2 x vector<[8]xf32>>
+// CHECK:           %[[T5:.*]] = llvm.extractvalue %[[T1]][2] : !llvm.array<4 x vector<[8]xf32>>
+// CHECK:           %[[T6:.*]] = llvm.insertvalue %[[T5]], %[[T4]][0] : !llvm.array<2 x vector<[8]xf32>>
+// CHECK:           %[[T7:.*]] = llvm.extractvalue %[[T1]][3] : !llvm.array<4 x vector<[8]xf32>>
+// CHECK:           %[[T8:.*]] = llvm.insertvalue %[[T7]], %[[T6]][1] : !llvm.array<2 x vector<[8]xf32>>
+// CHECK:           %[[T9:.*]] = builtin.unrealized_conversion_cast %[[T8]] : !llvm.array<2 x vector<[8]xf32>> to vector<2x[8]xf32>
+// CHECK:           return %[[T9]] : vector<2x[8]xf32>
 
 // -----
 
-func.func @extract_vec_2d_from_vec_3d_f32_scalable(%arg0: vector<4x3x[16]xf32>) -> vector<3x[16]xf32> {
-  %0 = vector.extract %arg0[0]: vector<3x[16]xf32> from vector<4x3x[16]xf32>
-  return %0 : vector<3x[16]xf32>
+//===----------------------------------------------------------------------===//
+// vector.insert_strided_slice
+//===----------------------------------------------------------------------===//
+
+func.func @insert_strided_slice_f32_2d_into_3d(%b: vector<4x4xf32>, %c: vector<4x4x4xf32>) -> vector<4x4x4xf32> {
+  %0 = vector.insert_strided_slice %b, %c {offsets = [2, 0, 0], strides = [1, 1]} : vector<4x4xf32> into vector<4x4x4xf32>
+  return %0 : vector<4x4x4xf32>
 }
-// CHECK-LABEL: @extract_vec_2d_from_vec_3d_f32_scalable
-//       CHECK:   llvm.extractvalue {{.*}}[0] : !llvm.array<4 x array<3 x vector<[16]xf32>>>
-//       CHECK:   return {{.*}} : vector<3x[16]xf32>
+// CHECK-LABEL: @insert_strided_slice_f32_2d_into_3d
+//       CHECK:    llvm.insertvalue {{.*}}, {{.*}}[2] : !llvm.array<4 x array<4 x vector<4xf32>>>
 
 // -----
 
-func.func @extract_vec_1d_from_vec_3d_f32(%arg0: vector<4x3x16xf32>) -> vector<16xf32> {
-  %0 = vector.extract %arg0[0, 0]: vector<16xf32> from vector<4x3x16xf32>
-  return %0 : vector<16xf32>
+func.func @insert_strided_slice_f32_2d_into_3d_scalable(%b: vector<4x[4]xf32>, %c: vector<4x4x[4]xf32>) -> vector<4x4x[4]xf32> {
+  %0 = vector.insert_strided_slice %b, %c {offsets = [2, 0, 0], strides = [1, 1]} : vector<4x[4]xf32> into vector<4x4x[4]xf32>
+  return %0 : vector<4x4x[4]xf32>
 }
-// CHECK-LABEL: @extract_vec_1d_from_vec_3d_f32
-//       CHECK:   llvm.extractvalue {{.*}}[0, 0] : !llvm.array<4 x array<3 x vector<16xf32>>>
-//       CHECK:   return {{.*}} : vector<16xf32>
+// CHECK-LABEL: @insert_strided_slice_f32_2d_into_3d_scalable
+//       CHECK:    llvm.insertvalue {{.*}}, {{.*}}[2] : !llvm.array<4 x array<4 x vector<[4]xf32>>>
 
 // -----
 
-func.func @extract_vec_1d_from_vec_3d_f32_scalable(%arg0: vector<4x3x[16]xf32>) -> vector<[16]xf32> {
-  %0 = vector.extract %arg0[0, 0]: vector<[16]xf32> from vector<4x3x[16]xf32>
-  return %0 : vector<[16]xf32>
+func.func @insert_strided_index_slice_index_2d_into_3d(%b: vector<4x4xindex>, %c: vector<4x4x4xindex>) -> vector<4x4x4xindex> {
+  %0 = vector.insert_strided_slice %b, %c {offsets = [2, 0, 0], strides = [1, 1]} : vector<4x4xindex> into vector<4x4x4xindex>
+  return %0 : vector<4x4x4xindex>
 }
-// CHECK-LABEL: @extract_vec_1d_from_vec_3d_f32_scalable
-//       CHECK:   llvm.extractvalue {{.*}}[0, 0] : !llvm.array<4 x array<3 x vector<[16]xf32>>>
-//       CHECK:   return {{.*}} : vector<[16]xf32>
+// CHECK-LABEL: @insert_strided_index_slice_index_2d_into_3d
+//       CHECK:    llvm.insertvalue {{.*}}, {{.*}}[2] : !llvm.array<4 x array<4 x vector<4xi64>>>
 
 // -----
 
-func.func @extract_scalar_from_vec_3d_f32(%arg0: vector<4x3x16xf32>) -> f32 {
-  %0 = vector.extract %arg0[0, 0, 0]: f32 from vector<4x3x16xf32>
-  return %0 : f32
+func.func @insert_strided_index_slice_index_2d_into_3d_scalable(%b: vector<4x[4]xindex>, %c: vector<4x4x[4]xindex>) -> vector<4x4x[4]xindex> {
+  %0 = vector.insert_strided_slice %b, %c {offsets = [2, 0, 0], strides = [1, 1]} : vector<4x[4]xindex> into vector<4x4x[4]xindex>
+  return %0 : vector<4x4x[4]xindex>
 }
-// CHECK-LABEL: @extract_scalar_from_vec_3d_f32
-//       CHECK:   llvm.extractvalue {{.*}}[0, 0] : !llvm.array<4 x array<3 x vector<16xf32>>>
-//       CHECK:   llvm.mlir.constant(0 : i64) : i64
-//       CHECK:   llvm.extractelement {{.*}}[{{.*}} : i64] : vector<16xf32>
-//       CHECK:   return {{.*}} : f32
+// CHECK-LABEL: @insert_strided_index_slice_index_2d_into_3d_scalable
+//       CHECK:    llvm.insertvalue {{.*}}, {{.*}}[2] : !llvm.array<4 x array<4 x vector<[4]xi64>>>
 
 // -----
 
-func.func @extract_scalar_from_vec_3d_f32_scalable(%arg0: vector<4x3x[16]xf32>) -> f32 {
-  %0 = vector.extract %arg0[0, 0, 0]: f32 from vector<4x3x[16]xf32>
-  return %0 : f32
-}
-// CHECK-LABEL: @extract_scalar_from_vec_3d_f32_scalable
-//       CHECK:   llvm.extractvalue {{.*}}[0, 0] : !llvm.array<4 x array<3 x vector<[16]xf32>>>
-//       CHECK:   llvm.mlir.constant(0 : i64) : i64
-//       CHECK:   llvm.extractelement {{.*}}[{{.*}} : i64] : vector<[16]xf32>
-//       CHECK:   return {{.*}} : f32
-
-// -----
-
-func.func @extract_scalar_from_vec_1d_f32_dynamic_idx(%arg0: vector<16xf32>, %arg1: index) -> f32 {
-  %0 = vector.extract %arg0[%arg1]: f32 from vector<16xf32>
-  return %0 : f32
-}
-// CHECK-LABEL: @extract_scalar_from_vec_1d_f32_dynamic_idx
-//  CHECK-SAME:   %[[VEC:.+]]: vector<16xf32>, %[[INDEX:.+]]: index
-//       CHECK:   %[[UC:.+]] = builtin.unrealized_conversion_cast %[[INDEX]] : index to i64
-//       CHECK:   llvm.extractelement %[[VEC]][%[[UC]] : i64] : vector<16xf32>
-
-// -----
-
-func.func @extract_scalar_from_vec_1d_f32_dynamic_idx_scalable(%arg0: vector<[16]xf32>, %arg1: index) -> f32 {
-  %0 = vector.extract %arg0[%arg1]: f32 from vector<[16]xf32>
-  return %0 : f32
-}
-// CHECK-LABEL: @extract_scalar_from_vec_1d_f32_dynamic_idx_scalable
-//  CHECK-SAME:   %[[VEC:.+]]: vector<[16]xf32>, %[[INDEX:.+]]: index
-//       CHECK:   %[[UC:.+]] = builtin.unrealized_conversion_cast %[[INDEX]] : index to i64
-//       CHECK:   llvm.extractelement %[[VEC]][%[[UC]] : i64] : vector<[16]xf32>
-
-// -----
-
-func.func @extract_scalar_from_vec_2d_f32_inner_dynamic_idx(%arg0: vector<1x16xf32>, %arg1: index) -> f32 {
-  %0 = vector.extract %arg0[0, %arg1]: f32 from vector<1x16xf32>
-  return %0 : f32
-}
-
-// Lowering supports extracting from multi-dim vectors with dynamic indices
-// provided that only the trailing index is dynamic.
-
-// CHECK-LABEL: @extract_scalar_from_vec_2d_f32_inner_dynamic_idx(
-//       CHECK:   llvm.extractvalue
-//       CHECK:   llvm.extractelement
-
-func.func @extract_scalar_from_vec_2d_f32_inner_dynamic_idx_scalable(%arg0: vector<1x[16]xf32>, %arg1: index) -> f32 {
-  %0 = vector.extract %arg0[0, %arg1]: f32 from vector<1x[16]xf32>
-  return %0 : f32
-}
-
-// Lowering supports extracting from multi-dim vectors with dynamic indices
-// provided that only the trailing index is dynamic.
-
-// CHECK-LABEL: @extract_scalar_from_vec_2d_f32_inner_dynamic_idx_scalable(
-//       CHECK:   llvm.extractvalue
-//       CHECK:   llvm.extractelement
-
-// -----
-
-func.func @extract_scalar_from_vec_2d_f32_outer_dynamic_idx(%arg0: vector<1x16xf32>, %arg1: index) -> f32 {
-  %0 = vector.extract %arg0[%arg1, 0]: f32 from vector<1x16xf32>
-  return %0 : f32
-}
-
-// Lowering supports extracting from multi-dim vectors with dynamic indices
-// provided that only the trailing index is dynamic.
-
-// CHECK-LABEL: @extract_scalar_from_vec_2d_f32_outer_dynamic_idx(
-//       CHECK:   vector.extract
-
-func.func @extract_scalar_from_vec_2d_f32_outer_dynamic_idx_scalable(%arg0: vector<1x[16]xf32>, %arg1: index) -> f32 {
-  %0 = vector.extract %arg0[%arg1, 0]: f32 from vector<1x[16]xf32>
-  return %0 : f32
-}
-
-// Lowering does not support extracting from multi-dim vectors with non trailing
-// dynamic index, but it shouldn't crash.
-
-// CHECK-LABEL: @extract_scalar_from_vec_2d_f32_outer_dynamic_idx_scalable(
-//       CHECK:   vector.extract
-
-// -----
-
-func.func @extract_scalar_from_vec_0d_index(%arg0: vector<index>) -> index {
-  %0 = vector.extract %arg0[]: index from vector<index>
-  return %0 : index
-}
-// CHECK-LABEL: @extract_scalar_from_vec_0d_index(
-//  CHECK-SAME:   %[[A:.*]]: vector<index>)
-//       CHECK:   %[[T0:.*]] = builtin.unrealized_conversion_cast %[[A]] : vector<index> to vector<1xi64>
-//       CHECK:   %[[T1:.*]] = llvm.mlir.constant(0 : i64) : i64
-//       CHECK:   %[[T2:.*]] = llvm.extractelement %[[T0]][%[[T1]] : i64] : vector<1xi64>
-//       CHECK:   %[[T3:.*]] = builtin.unrealized_conversion_cast %[[T2]] : i64 to index
-//       CHECK:   return %[[T3]] : index
-
-// -----
-
-//===----------------------------------------------------------------------===//
-// vector.insertelement
-//===----------------------------------------------------------------------===//
-
-func.func @insertelement_into_vec_0d_f32(%arg0: f32, %arg1: vector<f32>) -> vector<f32> {
-  %1 = vector.insertelement %arg0, %arg1[] : vector<f32>
-  return %1 : vector<f32>
-}
-// CHECK-LABEL: @insertelement_into_vec_0d_f32
-//  CHECK-SAME:   %[[A:.*]]: f32,
-//       CHECK:   %[[B:.*]] =  builtin.unrealized_conversion_cast %{{.*}} :
-//       CHECK:   vector<f32> to vector<1xf32>
-//       CHECK:   %[[C0:.*]] = llvm.mlir.constant(0 : index) : i64
-//       CHECK:   %[[X:.*]] = llvm.insertelement %[[A]], %[[B]][%[[C0]] : {{.*}}] : vector<1xf32>
-
-// -----
-
-func.func @insertelement_into_vec_1d_f32_idx_as_i32(%arg0: f32, %arg1: vector<4xf32>) -> vector<4xf32> {
-  %0 = arith.constant 3 : i32
-  %1 = vector.insertelement %arg0, %arg1[%0 : i32] : vector<4xf32>
-  return %1 : vector<4xf32>
-}
-// CHECK-LABEL: @insertelement_into_vec_1d_f32_idx_as_i32(
-//  CHECK-SAME:   %[[A:.*]]: f32,
-//  CHECK-SAME:   %[[B:.*]]: vector<4xf32>)
-//       CHECK:   %[[C:.*]] = arith.constant 3 : i32
-//       CHECK:   %[[X:.*]] = llvm.insertelement %[[A]], %[[B]][%[[C]] : i32] : vector<4xf32>
-//       CHECK:   return %[[X]] : vector<4xf32>
-
-// -----
-
-func.func @insertelement_into_vec_1d_f32_idx_as_i32_scalable(%arg0: f32, %arg1: vector<[4]xf32>) -> vector<[4]xf32> {
-  %0 = arith.constant 3 : i32
-  %1 = vector.insertelement %arg0, %arg1[%0 : i32] : vector<[4]xf32>
-  return %1 : vector<[4]xf32>
-}
-// CHECK-LABEL: @insertelement_into_vec_1d_f32_idx_as_i32_scalable(
-//  CHECK-SAME:   %[[A:.*]]: f32,
-//  CHECK-SAME:   %[[B:.*]]: vector<[4]xf32>)
-//       CHECK:   %[[C:.*]] = arith.constant 3 : i32
-//       CHECK:   %[[X:.*]] = llvm.insertelement %[[A]], %[[B]][%[[C]] : i32] : vector<[4]xf32>
-//       CHECK:   return %[[X]] : vector<[4]xf32>
-
-// -----
-
-func.func @insertelement_into_vec_1d_f32_scalable_idx_as_index(%arg0: f32, %arg1: vector<4xf32>) -> vector<4xf32> {
-  %0 = arith.constant 3 : index
-  %1 = vector.insertelement %arg0, %arg1[%0 : index] : vector<4xf32>
-  return %1 : vector<4xf32>
-}
-// CHECK-LABEL: @insertelement_into_vec_1d_f32_scalable_idx_as_index(
-//  CHECK-SAME:   %[[A:.*]]: f32,
-//  CHECK-SAME:   %[[B:.*]]: vector<4xf32>)
-//       CHECK:   %[[C:.*]] = arith.constant 3 : index
-//       CHECK:   %[[I:.*]] = builtin.unrealized_conversion_cast %[[C]] : index to i64
-//       CHECK:   %[[X:.*]] = llvm.insertelement %[[A]], %[[B]][%[[I]] : i64] : vector<4xf32>
-//       CHECK:   return %[[X]] : vector<4xf32>
-
-// -----
-
-func.func @insertelement_into_vec_1d_f32_scalable_idx_as_index_scalable(%arg0: f32, %arg1: vector<[4]xf32>) -> vector<[4]xf32> {
-  %0 = arith.constant 3 : index
-  %1 = vector.insertelement %arg0, %arg1[%0 : index] : vector<[4]xf32>
-  return %1 : vector<[4]xf32>
-}
-// CHECK-LABEL: @insertelement_into_vec_1d_f32_scalable_idx_as_index_scalable(
-//  CHECK-SAME:   %[[A:.*]]: f32,
-//  CHECK-SAME:   %[[B:.*]]: vector<[4]xf32>)
-//       CHECK:   %[[C:.*]] = arith.constant 3 : index
-//       CHECK:   %[[I:.*]] = builtin.unrealized_conversion_cast %[[C]] : index to i64
-//       CHECK:   %[[X:.*]] = llvm.insertelement %[[A]], %[[B]][%[[I]] : i64] : vector<[4]xf32>
-//       CHECK:   return %[[X]] : vector<[4]xf32>
-
-// -----
-
-//===----------------------------------------------------------------------===//
-// vector.insert
-//===----------------------------------------------------------------------===//
-
-func.func @insert_scalar_into_vec_1d_f32(%arg0: f32, %arg1: vector<4xf32>) -> vector<4xf32> {
-  %0 = vector.insert %arg0, %arg1[3] : f32 into vector<4xf32>
-  return %0 : vector<4xf32>
-}
-// CHECK-LABEL: @insert_scalar_into_vec_1d_f32
-//       CHECK:   llvm.mlir.constant(3 : i64) : i64
-//       CHECK:   llvm.insertelement {{.*}}, {{.*}}[{{.*}} : i64] : vector<4xf32>
-//       CHECK:   return {{.*}} : vector<4xf32>
-
-// -----
-
-func.func @insert_scalar_into_vec_1d_f32_scalable(%arg0: f32, %arg1: vector<[4]xf32>) -> vector<[4]xf32> {
-  %0 = vector.insert %arg0, %arg1[3] : f32 into vector<[4]xf32>
-  return %0 : vector<[4]xf32>
-}
-// CHECK-LABEL: @insert_scalar_into_vec_1d_f32_scalable
-//       CHECK:   llvm.mlir.constant(3 : i64) : i64
-//       CHECK:   llvm.insertelement {{.*}}, {{.*}}[{{.*}} : i64] : vector<[4]xf32>
-//       CHECK:   return {{.*}} : vector<[4]xf32>
-
-// -----
-
-func.func @insert_scalar_into_vec_1d_index(%arg0: index, %arg1: vector<4xindex>) -> vector<4xindex> {
-  %0 = vector.insert %arg0, %arg1[3] : index into vector<4xindex>
-  return %0 : vector<4xindex>
-}
-// CHECK-LABEL: @insert_scalar_into_vec_1d_index(
-//  CHECK-SAME:   %[[A:.*]]: index,
-//  CHECK-SAME:   %[[B:.*]]: vector<4xindex>)
-//   CHECK-DAG:   %[[T0:.*]] = builtin.unrealized_conversion_cast %[[A]] : index to i64
-//   CHECK-DAG:   %[[T1:.*]] = builtin.unrealized_conversion_cast %[[B]] : vector<4xindex> to vector<4xi64>
-//       CHECK:   %[[T3:.*]] = llvm.mlir.constant(3 : i64) : i64
-//       CHECK:   %[[T4:.*]] = llvm.insertelement %[[T0]], %[[T1]][%[[T3]] : i64] : vector<4xi64>
-//       CHECK:   %[[T5:.*]] = builtin.unrealized_conversion_cast %[[T4]] : vector<4xi64> to vector<4xindex>
-//       CHECK:   return %[[T5]] : vector<4xindex>
-
-// -----
-
-func.func @insert_scalar_into_vec_1d_index_scalable(%arg0: index, %arg1: vector<[4]xindex>) -> vector<[4]xindex> {
-  %0 = vector.insert %arg0, %arg1[3] : index into vector<[4]xindex>
-  return %0 : vector<[4]xindex>
-}
-// CHECK-LABEL: @insert_scalar_into_vec_1d_index_scalable(
-//  CHECK-SAME:   %[[A:.*]]: index,
-//  CHECK-SAME:   %[[B:.*]]: vector<[4]xindex>)
-//   CHECK-DAG:   %[[T0:.*]] = builtin.unrealized_conversion_cast %[[A]] : index to i64
-//   CHECK-DAG:   %[[T1:.*]] = builtin.unrealized_conversion_cast %[[B]] : vector<[4]xindex> to vector<[4]xi64>
-//       CHECK:   %[[T3:.*]] = llvm.mlir.constant(3 : i64) : i64
-//       CHECK:   %[[T4:.*]] = llvm.insertelement %[[T0]], %[[T1]][%[[T3]] : i64] : vector<[4]xi64>
-//       CHECK:   %[[T5:.*]] = builtin.unrealized_conversion_cast %[[T4]] : vector<[4]xi64> to vector<[4]xindex>
-//       CHECK:   return %[[T5]] : vector<[4]xindex>
-
-// -----
-
-func.func @insert_vec_2d_into_vec_3d_f32(%arg0: vector<8x16xf32>, %arg1: vector<4x8x16xf32>) -> vector<4x8x16xf32> {
-  %0 = vector.insert %arg0, %arg1[3] : vector<8x16xf32> into vector<4x8x16xf32>
-  return %0 : vector<4x8x16xf32>
-}
-// CHECK-LABEL: @insert_vec_2d_into_vec_3d_f32
-//       CHECK:   llvm.insertvalue {{.*}}, {{.*}}[3] : !llvm.array<4 x array<8 x vector<16xf32>>>
-//       CHECK:   return {{.*}} : vector<4x8x16xf32>
-
-// -----
-
-func.func @insert_vec_2d_into_vec_3d_f32_scalable(%arg0: vector<8x[16]xf32>, %arg1: vector<4x8x[16]xf32>) -> vector<4x8x[16]xf32> {
-  %0 = vector.insert %arg0, %arg1[3] : vector<8x[16]xf32> into vector<4x8x[16]xf32>
-  return %0 : vector<4x8x[16]xf32>
-}
-// CHECK-LABEL: @insert_vec_2d_into_vec_3d_f32_scalable
-//       CHECK:   llvm.insertvalue {{.*}}, {{.*}}[3] : !llvm.array<4 x array<8 x vector<[16]xf32>>>
-//       CHECK:   return {{.*}} : vector<4x8x[16]xf32>
-
-// -----
-
-func.func @insert_vec_1d_into_vec_3d_f32(%arg0: vector<16xf32>, %arg1: vector<4x8x16xf32>) -> vector<4x8x16xf32> {
-  %0 = vector.insert %arg0, %arg1[3, 7] : vector<16xf32> into vector<4x8x16xf32>
-  return %0 : vector<4x8x16xf32>
-}
-// CHECK-LABEL: @insert_vec_1d_into_vec_3d_f32
-//       CHECK:   llvm.insertvalue {{.*}}, {{.*}}[3, 7] : !llvm.array<4 x array<8 x vector<16xf32>>>
-//       CHECK:   return {{.*}} : vector<4x8x16xf32>
-
-// -----
-
-func.func @insert_vec_1d_into_vec_3d_f32_scalable(%arg0: vector<[16]xf32>, %arg1: vector<4x8x[16]xf32>) -> vector<4x8x[16]xf32> {
-  %0 = vector.insert %arg0, %arg1[3, 7] : vector<[16]xf32> into vector<4x8x[16]xf32>
-  return %0 : vector<4x8x[16]xf32>
-}
-// CHECK-LABEL: @insert_vec_1d_into_vec_3d_f32_scalable
-//       CHECK:   llvm.insertvalue {{.*}}, {{.*}}[3, 7] : !llvm.array<4 x array<8 x vector<[16]xf32>>>
-//       CHECK:   return {{.*}} : vector<4x8x[16]xf32>
-
-// -----
-
-func.func @insert_scalar_into_vec_3d_f32(%arg0: f32, %arg1: vector<4x8x16xf32>) -> vector<4x8x16xf32> {
-  %0 = vector.insert %arg0, %arg1[3, 7, 15] : f32 into vector<4x8x16xf32>
-  return %0 : vector<4x8x16xf32>
-}
-// CHECK-LABEL: @insert_scalar_into_vec_3d_f32
-//       CHECK:   llvm.extractvalue {{.*}}[3, 7] : !llvm.array<4 x array<8 x vector<16xf32>>>
-//       CHECK:   llvm.mlir.constant(15 : i64) : i64
-//       CHECK:   llvm.insertelement {{.*}}, {{.*}}[{{.*}} : i64] : vector<16xf32>
-//       CHECK:   llvm.insertvalue {{.*}}, {{.*}}[3, 7] : !llvm.array<4 x array<8 x vector<16xf32>>>
-//       CHECK:   return {{.*}} : vector<4x8x16xf32>
-
-// -----
-
-func.func @insert_scalar_into_vec_3d_f32_scalable(%arg0: f32, %arg1: vector<4x8x[16]xf32>) -> vector<4x8x[16]xf32> {
-  %0 = vector.insert %arg0, %arg1[3, 7, 15] : f32 into vector<4x8x[16]xf32>
-  return %0 : vector<4x8x[16]xf32>
-}
-// CHECK-LABEL: @insert_scalar_into_vec_3d_f32_scalable
-//       CHECK:   llvm.extractvalue {{.*}}[3, 7] : !llvm.array<4 x array<8 x vector<[16]xf32>>>
-//       CHECK:   llvm.mlir.constant(15 : i64) : i64
-//       CHECK:   llvm.insertelement {{.*}}, {{.*}}[{{.*}} : i64] : vector<[16]xf32>
-//       CHECK:   llvm.insertvalue {{.*}}, {{.*}}[3, 7] : !llvm.array<4 x array<8 x vector<[16]xf32>>>
-//       CHECK:   return {{.*}} : vector<4x8x[16]xf32>
-
-// -----
-
-func.func @insert_scalar_into_vec_1d_f32_dynamic_idx(%arg0: vector<16xf32>, %arg1: f32, %arg2: index)
-                                      -> vector<16xf32> {
-  %0 = vector.insert %arg1, %arg0[%arg2]: f32 into vector<16xf32>
-  return %0 : vector<16xf32>
-}
-
-// CHECK-LABEL: @insert_scalar_into_vec_1d_f32_dynamic_idx
-//  CHECK-SAME:   %[[DST:.+]]: vector<16xf32>, %[[SRC:.+]]: f32, %[[INDEX:.+]]: index
-//       CHECK:   %[[UC:.+]] = builtin.unrealized_conversion_cast %[[INDEX]] : index to i64
-//       CHECK:   llvm.insertelement %[[SRC]], %[[DST]][%[[UC]] : i64] : vector<16xf32>
-
-// -----
-
-func.func @insert_scalar_into_vec_1d_f32_dynamic_idx_scalable(%arg0: vector<[16]xf32>, %arg1: f32, %arg2: index)
-                                      -> vector<[16]xf32> {
-  %0 = vector.insert %arg1, %arg0[%arg2]: f32 into vector<[16]xf32>
-  return %0 : vector<[16]xf32>
-}
-
-// CHECK-LABEL: @insert_scalar_into_vec_1d_f32_dynamic_idx_scalable
-//  CHECK-SAME:   %[[DST:.+]]: vector<[16]xf32>, %[[SRC:.+]]: f32, %[[INDEX:.+]]: index
-//       CHECK:   %[[UC:.+]] = builtin.unrealized_conversion_cast %[[INDEX]] : index to i64
-//       CHECK:   llvm.insertelement %[[SRC]], %[[DST]][%[[UC]] : i64] : vector<[16]xf32>
-
-// -----
-
-func.func @insert_scalar_into_vec_2d_f32_dynamic_idx(%arg0: vector<1x16xf32>, %arg1: f32, %idx: index)
-                                        -> vector<1x16xf32> {
-  %0 = vector.insert %arg1, %arg0[0, %idx]: f32 into vector<1x16xf32>
-  return %0 : vector<1x16xf32>
-}
-
-// Multi-dim vectors are not supported but this test shouldn't crash.
-
-// CHECK-LABEL: @insert_scalar_into_vec_2d_f32_dynamic_idx(
-//       CHECK:   vector.insert
-
-// -----
-
-func.func @insert_scalar_into_vec_2d_f32_dynamic_idx_scalable(%arg0: vector<1x[16]xf32>, %arg1: f32, %idx: index)
-                                        -> vector<1x[16]xf32> {
-  %0 = vector.insert %arg1, %arg0[0, %idx]: f32 into vector<1x[16]xf32>
-  return %0 : vector<1x[16]xf32>
-}
-
-// Multi-dim vectors are not supported but this test shouldn't crash.
-
-// CHECK-LABEL: @insert_scalar_into_vec_2d_f32_dynamic_idx_scalable(
-//       CHECK:   vector.insert
-
-// -----
-
-//===----------------------------------------------------------------------===//
-// vector.type_cast
-//
-// TODO: Add tests for for vector.type_cast that would cover scalable vectors
-//===----------------------------------------------------------------------===//
-
-func.func @type_cast_f32(%arg0: memref<8x8x8xf32>) -> memref<vector<8x8x8xf32>> {
-  %0 = vector.type_cast %arg0: memref<8x8x8xf32> to memref<vector<8x8x8xf32>>
-  return %0 : memref<vector<8x8x8xf32>>
-}
-// CHECK-LABEL: @type_cast_f32
-//       CHECK:   llvm.mlir.undef : !llvm.struct<(ptr, ptr, i64)>
-//       CHECK:   %[[allocated:.*]] = llvm.extractvalue {{.*}}[0] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)>
-//       CHECK:   llvm.insertvalue %[[allocated]], {{.*}}[0] : !llvm.struct<(ptr, ptr, i64)>
-//       CHECK:   %[[aligned:.*]] = llvm.extractvalue {{.*}}[1] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)>
-//       CHECK:   llvm.insertvalue %[[aligned]], {{.*}}[1] : !llvm.struct<(ptr, ptr, i64)>
-//       CHECK:   llvm.mlir.constant(0 : index
-//       CHECK:   llvm.insertvalue {{.*}}[2] : !llvm.struct<(ptr, ptr, i64)>
-
-// NOTE: No test for scalable vectors - the input memref is fixed size.
-
-// -----
-
-func.func @type_cast_index(%arg0: memref<8x8x8xindex>) -> memref<vector<8x8x8xindex>> {
-  %0 = vector.type_cast %arg0: memref<8x8x8xindex> to memref<vector<8x8x8xindex>>
-  return %0 : memref<vector<8x8x8xindex>>
-}
-// CHECK-LABEL: @type_cast_index(
-// CHECK-SAME: %[[A:.*]]: memref<8x8x8xindex>)
-//       CHECK:   %{{.*}} = builtin.unrealized_conversion_cast %[[A]] : memref<8x8x8xindex> to !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)>
-
-//       CHECK:   %{{.*}} = builtin.unrealized_conversion_cast %{{.*}} : !llvm.struct<(ptr, ptr, i64)> to memref<vector<8x8x8xindex>>
-
-// NOTE: No test for scalable vectors - the input memref is fixed size.
-
-// -----
-
-func.func @type_cast_non_zero_addrspace(%arg0: memref<8x8x8xf32, 3>) -> memref<vector<8x8x8xf32>, 3> {
-  %0 = vector.type_cast %arg0: memref<8x8x8xf32, 3> to memref<vector<8x8x8xf32>, 3>
-  return %0 : memref<vector<8x8x8xf32>, 3>
-}
-// CHECK-LABEL: @type_cast_non_zero_addrspace
-//       CHECK:   llvm.mlir.undef : !llvm.struct<(ptr<3>, ptr<3>, i64)>
-//       CHECK:   %[[allocated:.*]] = llvm.extractvalue {{.*}}[0] : !llvm.struct<(ptr<3>, ptr<3>, i64, array<3 x i64>, array<3 x i64>)>
-//       CHECK:   llvm.insertvalue %[[allocated]], {{.*}}[0] : !llvm.struct<(ptr<3>, ptr<3>, i64)>
-//       CHECK:   %[[aligned:.*]] = llvm.extractvalue {{.*}}[1] : !llvm.struct<(ptr<3>, ptr<3>, i64, array<3 x i64>, array<3 x i64>)>
-//       CHECK:   llvm.insertvalue %[[aligned]], {{.*}}[1] : !llvm.struct<(ptr<3>, ptr<3>, i64)>
-//       CHECK:   llvm.mlir.constant(0 : index
-//       CHECK:   llvm.insertvalue {{.*}}[2] : !llvm.struct<(ptr<3>, ptr<3>, i64)>
-
-// NOTE: No test for scalable vectors - the input memref is fixed size.
-
-// -----
-
-//===----------------------------------------------------------------------===//
-// vector.print
-//===----------------------------------------------------------------------===//
-
-func.func @print_scalar_i1(%arg0: i1) {
-  vector.print %arg0 : i1
-  return
-}
-//
-// Type "boolean" always uses zero extension.
-//
-// CHECK-LABEL: @print_scalar_i1(
-// CHECK-SAME: %[[A:.*]]: i1)
-//       CHECK: %[[S:.*]] = arith.extui %[[A]] : i1 to i64
-//       CHECK: llvm.call @printI64(%[[S]]) : (i64) -> ()
-//       CHECK: llvm.call @printNewline() : () -> ()
-
-// -----
-
-func.func @print_scalar_i4(%arg0: i4) {
-  vector.print %arg0 : i4
-  return
-}
-// CHECK-LABEL: @print_scalar_i4(
-// CHECK-SAME: %[[A:.*]]: i4)
-//       CHECK: %[[S:.*]] = arith.extsi %[[A]] : i4 to i64
-//       CHECK: llvm.call @printI64(%[[S]]) : (i64) -> ()
-//       CHECK: llvm.call @printNewline() : () -> ()
-
-// -----
-
-func.func @print_scalar_si4(%arg0: si4) {
-  vector.print %arg0 : si4
-  return
-}
-// CHECK-LABEL: @print_scalar_si4(
-// CHECK-SAME: %[[A:.*]]: si4)
-//       CHECK: %[[C:.*]] = builtin.unrealized_conversion_cast %[[A]] : si4 to i4
-//       CHECK: %[[S:.*]] = arith.extsi %[[C]] : i4 to i64
-//       CHECK: llvm.call @printI64(%[[S]]) : (i64) -> ()
-//       CHECK: llvm.call @printNewline() : () -> ()
-
-// -----
-
-func.func @print_scalar_ui4(%arg0: ui4) {
-  vector.print %arg0 : ui4
-  return
-}
-// CHECK-LABEL: @print_scalar_ui4(
-// CHECK-SAME: %[[A:.*]]: ui4)
-//       CHECK: %[[C:.*]] = builtin.unrealized_conversion_cast %[[A]] : ui4 to i4
-//       CHECK: %[[S:.*]] = arith.extui %[[C]] : i4 to i64
-//       CHECK: llvm.call @printU64(%[[S]]) : (i64) -> ()
-//       CHECK: llvm.call @printNewline() : () -> ()
-
-// -----
-
-func.func @print_scalar_i32(%arg0: i32) {
-  vector.print %arg0 : i32
-  return
-}
-// CHECK-LABEL: @print_scalar_i32(
-// CHECK-SAME: %[[A:.*]]: i32)
-//       CHECK: %[[S:.*]] = arith.extsi %[[A]] : i32 to i64
-//       CHECK: llvm.call @printI64(%[[S]]) : (i64) -> ()
-//       CHECK: llvm.call @printNewline() : () -> ()
-
-// -----
-
-func.func @print_scalar_ui32(%arg0: ui32) {
-  vector.print %arg0 : ui32
-  return
-}
-// CHECK-LABEL: @print_scalar_ui32(
-// CHECK-SAME: %[[A:.*]]: ui32)
-//       CHECK: %[[C:.*]] = builtin.unrealized_conversion_cast %[[A]] : ui32 to i32
-//       CHECK: %[[S:.*]] = arith.extui %[[C]] : i32 to i64
-//       CHECK: llvm.call @printU64(%[[S]]) : (i64) -> ()
-
-// -----
-
-func.func @print_scalar_i40(%arg0: i40) {
-  vector.print %arg0 : i40
-  return
-}
-// CHECK-LABEL: @print_scalar_i40(
-// CHECK-SAME: %[[A:.*]]: i40)
-//       CHECK: %[[S:.*]] = arith.extsi %[[A]] : i40 to i64
-//       CHECK: llvm.call @printI64(%[[S]]) : (i64) -> ()
-//       CHECK: llvm.call @printNewline() : () -> ()
-
-// -----
-
-func.func @print_scalar_si40(%arg0: si40) {
-  vector.print %arg0 : si40
-  return
-}
-// CHECK-LABEL: @print_scalar_si40(
-// CHECK-SAME: %[[A:.*]]: si40)
-//       CHECK: %[[C:.*]] = builtin.unrealized_conversion_cast %[[A]] : si40 to i40
-//       CHECK: %[[S:.*]] = arith.extsi %[[C]] : i40 to i64
-//       CHECK: llvm.call @printI64(%[[S]]) : (i64) -> ()
-//       CHECK: llvm.call @printNewline() : () -> ()
-
-// -----
-
-func.func @print_scalar_ui40(%arg0: ui40) {
-  vector.print %arg0 : ui40
-  return
-}
-// CHECK-LABEL: @print_scalar_ui40(
-// CHECK-SAME: %[[A:.*]]: ui40)
-//       CHECK: %[[C:.*]] = builtin.unrealized_conversion_cast %[[A]] : ui40 to i40
-//       CHECK: %[[S:.*]] = arith.extui %[[C]] : i40 to i64
-//       CHECK: llvm.call @printU64(%[[S]]) : (i64) -> ()
-//       CHECK: llvm.call @printNewline() : () -> ()
-
-// -----
-
-func.func @print_scalar_i64(%arg0: i64) {
-  vector.print %arg0 : i64
-  return
-}
-// CHECK-LABEL: @print_scalar_i64(
-// CHECK-SAME: %[[A:.*]]: i64)
-//       CHECK:    llvm.call @printI64(%[[A]]) : (i64) -> ()
-//       CHECK:    llvm.call @printNewline() : () -> ()
-
-// -----
-
-func.func @print_scalar_ui64(%arg0: ui64) {
-  vector.print %arg0 : ui64
-  return
-}
-// CHECK-LABEL: @print_scalar_ui64(
-// CHECK-SAME: %[[A:.*]]: ui64)
-//       CHECK:    %[[C:.*]] = builtin.unrealized_conversion_cast %[[A]] : ui64 to i64
-//       CHECK:    llvm.call @printU64(%[[C]]) : (i64) -> ()
-//       CHECK:    llvm.call @printNewline() : () -> ()
-
-// -----
-
-func.func @print_scalar_index(%arg0: index) {
-  vector.print %arg0 : index
-  return
-}
-// CHECK-LABEL: @print_scalar_index(
-// CHECK-SAME: %[[A:.*]]: index)
-//       CHECK:    %[[C:.*]] = builtin.unrealized_conversion_cast %[[A]] : index to i64
-//       CHECK:    llvm.call @printU64(%[[C]]) : (i64) -> ()
-//       CHECK:    llvm.call @printNewline() : () -> ()
-
-// -----
-
-func.func @print_scalar_f32(%arg0: f32) {
-  vector.print %arg0 : f32
-  return
-}
-// CHECK-LABEL: @print_scalar_f32(
-// CHECK-SAME: %[[A:.*]]: f32)
-//       CHECK:    llvm.call @printF32(%[[A]]) : (f32) -> ()
-//       CHECK:    llvm.call @printNewline() : () -> ()
-
-// -----
-
-func.func @print_scalar_f64(%arg0: f64) {
-  vector.print %arg0 : f64
-  return
-}
-// CHECK-LABEL: @print_scalar_f64(
-// CHECK-SAME: %[[A:.*]]: f64)
-//       CHECK:    llvm.call @printF64(%[[A]]) : (f64) -> ()
-//       CHECK:    llvm.call @printNewline() : () -> ()
-
-// -----
-
-// CHECK-LABEL: module {
-// CHECK: llvm.func @printString(!llvm.ptr)
-// CHECK: llvm.mlir.global private constant @[[GLOBAL_STR:.*]]({{.*}})
-// CHECK: @print_string
-//       CHECK-NEXT: %[[GLOBAL_ADDR:.*]] = llvm.mlir.addressof @[[GLOBAL_STR]] : !llvm.ptr
-//       CHECK-NEXT: %[[STR_PTR:.*]] = llvm.getelementptr %[[GLOBAL_ADDR]][0] : (!llvm.ptr) -> !llvm.ptr
-//       CHECK-NEXT: llvm.call @printString(%[[STR_PTR]]) : (!llvm.ptr) -> ()
-func.func @print_string() {
-  vector.print str "Hello, World!"
-  return
-}
-
-// -----
-
-//===----------------------------------------------------------------------===//
-// vector.extract_strided_slice
-//===----------------------------------------------------------------------===//
-
-func.func @extract_strided_slice_f32_1d_from_1d(%arg0: vector<4xf32>) -> vector<2xf32> {
-  %0 = vector.extract_strided_slice %arg0 {offsets = [2], sizes = [2], strides = [1]} : vector<4xf32> to vector<2xf32>
-  return %0 : vector<2xf32>
-}
-// CHECK-LABEL: @extract_strided_slice_f32_1d_from_1d
-//  CHECK-SAME:    %[[A:.*]]: vector<4xf32>)
-//       CHECK:    %[[T0:.*]] = llvm.shufflevector %[[A]], %[[A]] [2, 3] : vector<4xf32>
-//       CHECK:    return %[[T0]] : vector<2xf32>
-
-// NOTE: For scalable vectors we could only extract vector<[4]xf32> from vector<[4]xf32>, but that would be a NOP.
-
-// -----
-
-func.func @extract_strided_slice_index_1d_from_1d(%arg0: vector<4xindex>) -> vector<2xindex> {
-  %0 = vector.extract_strided_slice %arg0 {offsets = [2], sizes = [2], strides = [1]} : vector<4xindex> to vector<2xindex>
-  return %0 : vector<2xindex>
-}
-// CHECK-LABEL: @extract_strided_slice_index_1d_from_1d
-//  CHECK-SAME:    %[[A:.*]]: vector<4xindex>)
-//       CHECK:    %[[T0:.*]] = builtin.unrealized_conversion_cast %[[A]] : vector<4xindex> to vector<4xi64>
-//       CHECK:    %[[T2:.*]] = llvm.shufflevector %[[T0]], %[[T0]] [2, 3] : vector<4xi64>
-//       CHECK:    %[[T3:.*]] = builtin.unrealized_conversion_cast %[[T2]] : vector<2xi64> to vector<2xindex>
-//       CHECK:    return %[[T3]] : vector<2xindex>
-
-// NOTE: For scalable vectors we could only extract vector<[4]xindex> from vector<[4]xindex>, but that would be a NOP.
-
-// -----
-
-func.func @extract_strided_slice_f32_1d_from_2d(%arg0: vector<4x8xf32>) -> vector<2x8xf32> {
-  %0 = vector.extract_strided_slice %arg0 {offsets = [2], sizes = [2], strides = [1]} : vector<4x8xf32> to vector<2x8xf32>
-  return %0 : vector<2x8xf32>
-}
-// CHECK-LABEL: @extract_strided_slice_f32_1d_from_2d(
-//  CHECK-SAME:    %[[ARG:.*]]: vector<4x8xf32>)
-//       CHECK:    %[[A:.*]] = builtin.unrealized_conversion_cast %[[ARG]] : vector<4x8xf32> to !llvm.array<4 x vector<8xf32>>
-//       CHECK:    %[[T0:.*]] = llvm.mlir.undef : !llvm.array<2 x vector<8xf32>>
-//       CHECK:    %[[T1:.*]] = llvm.extractvalue %[[A]][2] : !llvm.array<4 x vector<8xf32>>
-//       CHECK:    %[[T2:.*]] = llvm.insertvalue %[[T1]], %[[T0]][0] : !llvm.array<2 x vector<8xf32>>
-//       CHECK:    %[[T3:.*]] = llvm.extractvalue %[[A]][3] : !llvm.array<4 x vector<8xf32>>
-//       CHECK:    %[[T4:.*]] = llvm.insertvalue %[[T3]], %[[T2]][1] : !llvm.array<2 x vector<8xf32>>
-//       CHECK:    %[[T5:.*]] = builtin.unrealized_conversion_cast %[[T4]] : !llvm.array<2 x vector<8xf32>> to vector<2x8xf32>
-//       CHECK:    return %[[T5]]
-
-// -----
-
-func.func @extract_strided_slice_f32_1d_from_2d_scalable(%arg0: vector<4x[8]xf32>) -> vector<2x[8]xf32> {
-  %0 = vector.extract_strided_slice %arg0 {offsets = [2], sizes = [2], strides = [1]} : vector<4x[8]xf32> to vector<2x[8]xf32>
-  return %0 : vector<2x[8]xf32>
-}
-// CHECK-LABEL:   func.func @extract_strided_slice_f32_1d_from_2d_scalable(
-//  CHECK-SAME:    %[[ARG:.*]]: vector<4x[8]xf32>)
-//       CHECK:    %[[A:.*]] = builtin.unrealized_conversion_cast %[[ARG]] : vector<4x[8]xf32> to !llvm.array<4 x vector<[8]xf32>>
-//       CHECK:    %[[T0:.*]] = llvm.mlir.undef : !llvm.array<2 x vector<[8]xf32>>
-//       CHECK:    %[[T1:.*]] = llvm.extractvalue %[[A]][2] : !llvm.array<4 x vector<[8]xf32>>
-//       CHECK:    %[[T2:.*]] = llvm.insertvalue %[[T1]], %[[T0]][0] : !llvm.array<2 x vector<[8]xf32>>
-//       CHECK:    %[[T3:.*]] = llvm.extractvalue %[[A]][3] : !llvm.array<4 x vector<[8]xf32>>
-//       CHECK:    %[[T4:.*]] = llvm.insertvalue %[[T3]], %[[T2]][1] : !llvm.array<2 x vector<[8]xf32>>
-//       CHECK:    %[[T5:.*]] = builtin.unrealized_conversion_cast %[[T4]] : !llvm.array<2 x vector<[8]xf32>> to vector<2x[8]xf32>
-//       CHECK:    return %[[T5]]
-
-// -----
-
-func.func @extract_strided_slice_f32_2d_from_2d(%arg0: vector<4x8xf32>) -> vector<2x2xf32> {
-  %0 = vector.extract_strided_slice %arg0 {offsets = [2, 2], sizes = [2, 2], strides = [1, 1]} : vector<4x8xf32> to vector<2x2xf32>
-  return %0 : vector<2x2xf32>
-}
-// CHECK-LABEL: @extract_strided_slice_f32_2d_from_2d(
-//  CHECK-SAME:    %[[ARG:.*]]: vector<4x8xf32>)
-//       CHECK:    %[[A:.*]] = builtin.unrealized_conversion_cast %[[ARG]] : vector<4x8xf32> to !llvm.array<4 x vector<8xf32>>
-//       CHECK:    %[[VAL_2:.*]] = arith.constant dense<0.000000e+00> : vector<2x2xf32>
-//       CHECK:    %[[VAL_6:.*]] = builtin.unrealized_conversion_cast %[[VAL_2]] : vector<2x2xf32> to !llvm.array<2 x vector<2xf32>>
-//       CHECK:    %[[T2:.*]] = llvm.extractvalue %[[A]][2] : !llvm.array<4 x vector<8xf32>>
-//       CHECK:    %[[T3:.*]] = llvm.shufflevector %[[T2]], %[[T2]] [2, 3] : vector<8xf32>
-//       CHECK:    %[[T4:.*]] = llvm.insertvalue %[[T3]], %[[VAL_6]][0] : !llvm.array<2 x vector<2xf32>>
-//       CHECK:    %[[T5:.*]] = llvm.extractvalue %[[A]][3] : !llvm.array<4 x vector<8xf32>>
-//       CHECK:    %[[T6:.*]] = llvm.shufflevector %[[T5]], %[[T5]] [2, 3] : vector<8xf32>
-//       CHECK:    %[[T7:.*]] = llvm.insertvalue %[[T6]], %[[T4]][1] : !llvm.array<2 x vector<2xf32>>
-//       CHECK:    %[[VAL_12:.*]] = builtin.unrealized_conversion_cast %[[T7]] : !llvm.array<2 x vector<2xf32>> to vector<2x2xf32>
-//       CHECK:    return %[[VAL_12]] : vector<2x2xf32>
-
-// -----
-
-// NOTE: For scalable vectors, we can only extract "full" scalable dimensions
-// (e.g. [8] from [8], but not [4] from [8]).
-
-func.func @extract_strided_slice_f32_2d_from_2d_scalable(%arg0: vector<4x[8]xf32>) -> vector<2x[8]xf32> {
-  %0 = vector.extract_strided_slice %arg0 {offsets = [2, 0], sizes = [2, 8], strides = [1, 1]} : vector<4x[8]xf32> to vector<2x[8]xf32>
-  return %0 : vector<2x[8]xf32>
-}
-// CHECK-LABEL: @extract_strided_slice_f32_2d_from_2d_scalable(
-//  CHECK-SAME:     %[[ARG:.*]]: vector<4x[8]xf32>)
-// CHECK:           %[[T1:.*]] = builtin.unrealized_conversion_cast %[[ARG]] : vector<4x[8]xf32> to !llvm.array<4 x vector<[8]xf32>>
-// CHECK:           %[[T3:.*]] = arith.constant dense<0.000000e+00> : vector<2x[8]xf32>
-// CHECK:           %[[T4:.*]] = builtin.unrealized_conversion_cast %[[T3]] : vector<2x[8]xf32> to !llvm.array<2 x vector<[8]xf32>>
-// CHECK:           %[[T5:.*]] = llvm.extractvalue %[[T1]][2] : !llvm.array<4 x vector<[8]xf32>>
-// CHECK:           %[[T6:.*]] = llvm.insertvalue %[[T5]], %[[T4]][0] : !llvm.array<2 x vector<[8]xf32>>
-// CHECK:           %[[T7:.*]] = llvm.extractvalue %[[T1]][3] : !llvm.array<4 x vector<[8]xf32>>
-// CHECK:           %[[T8:.*]] = llvm.insertvalue %[[T7]], %[[T6]][1] : !llvm.array<2 x vector<[8]xf32>>
-// CHECK:           %[[T9:.*]] = builtin.unrealized_conversion_cast %[[T8]] : !llvm.array<2 x vector<[8]xf32>> to vector<2x[8]xf32>
-// CHECK:           return %[[T9]] : vector<2x[8]xf32>
-
-// -----
-
-//===----------------------------------------------------------------------===//
-// vector.insert_strided_slice
-//===----------------------------------------------------------------------===//
-
-func.func @insert_strided_slice_f32_2d_into_3d(%b: vector<4x4xf32>, %c: vector<4x4x4xf32>) -> vector<4x4x4xf32> {
-  %0 = vector.insert_strided_slice %b, %c {offsets = [2, 0, 0], strides = [1, 1]} : vector<4x4xf32> into vector<4x4x4xf32>
-  return %0 : vector<4x4x4xf32>
-}
-// CHECK-LABEL: @insert_strided_slice_f32_2d_into_3d
-//       CHECK:    llvm.insertvalue {{.*}}, {{.*}}[2] : !llvm.array<4 x array<4 x vector<4xf32>>>
-
-// -----
-
-func.func @insert_strided_slice_f32_2d_into_3d_scalable(%b: vector<4x[4]xf32>, %c: vector<4x4x[4]xf32>) -> vector<4x4x[4]xf32> {
-  %0 = vector.insert_strided_slice %b, %c {offsets = [2, 0, 0], strides = [1, 1]} : vector<4x[4]xf32> into vector<4x4x[4]xf32>
-  return %0 : vector<4x4x[4]xf32>
-}
-// CHECK-LABEL: @insert_strided_slice_f32_2d_into_3d_scalable
-//       CHECK:    llvm.insertvalue {{.*}}, {{.*}}[2] : !llvm.array<4 x array<4 x vector<[4]xf32>>>
-
-// -----
-
-func.func @insert_strided_index_slice_index_2d_into_3d(%b: vector<4x4xindex>, %c: vector<4x4x4xindex>) -> vector<4x4x4xindex> {
-  %0 = vector.insert_strided_slice %b, %c {offsets = [2, 0, 0], strides = [1, 1]} : vector<4x4xindex> into vector<4x4x4xindex>
-  return %0 : vector<4x4x4xindex>
-}
-// CHECK-LABEL: @insert_strided_index_slice_index_2d_into_3d
-//       CHECK:    llvm.insertvalue {{.*}}, {{.*}}[2] : !llvm.array<4 x array<4 x vector<4xi64>>>
-
-// -----
-
-func.func @insert_strided_index_slice_index_2d_into_3d_scalable(%b: vector<4x[4]xindex>, %c: vector<4x4x[4]xindex>) -> vector<4x4x[4]xindex> {
-  %0 = vector.insert_strided_slice %b, %c {offsets = [2, 0, 0], strides = [1, 1]} : vector<4x[4]xindex> into vector<4x4x[4]xindex>
-  return %0 : vector<4x4x[4]xindex>
-}
-// CHECK-LABEL: @insert_strided_index_slice_index_2d_into_3d_scalable
-//       CHECK:    llvm.insertvalue {{.*}}, {{.*}}[2] : !llvm.array<4 x array<4 x vector<[4]xi64>>>
-
-// -----
-
-func.func @insert_strided_slice_f32_2d_into_2d(%a: vector<2x2xf32>, %b: vector<4x4xf32>) -> vector<4x4xf32> {
-  %0 = vector.insert_strided_slice %a, %b {offsets = [2, 2], strides = [1, 1]} : vector<2x2xf32> into vector<4x4xf32>
-  return %0 : vector<4x4xf32>
+func.func @insert_strided_slice_f32_2d_into_2d(%a: vector<2x2xf32>, %b: vector<4x4xf32>) -> vector<4x4xf32> {
+  %0 = vector.insert_strided_slice %a, %b {offsets = [2, 2], strides = [1, 1]} : vector<2x2xf32> into vector<4x4xf32>
+  return %0 : vector<4x4xf32>
 }
 
 // CHECK-LABEL: @insert_strided_slice_f32_2d_into_2d
@@ -2272,691 +1424,6 @@ func.func @fma_scalable(%vec_1d: vector<[8]xf32>, %vec_2d: vector<2x[4]xf32>, %v
 
   return %0, %1, %2: vector<[8]xf32>, vector<2x[4]xf32>, vector<1x1x[1]xf32>
 }
-
-// -----
-
-//===----------------------------------------------------------------------===//
-// vector.reduction
-//===----------------------------------------------------------------------===//
-
-func.func @reduce_0d_f32(%arg0: vector<f32>) -> f32 {
-  %0 = vector.reduction <add>, %arg0 : vector<f32> into f32
-  return %0 : f32
-}
-// CHECK-LABEL: @reduce_0d_f32(
-// CHECK-SAME: %[[A:.*]]: vector<f32>)
-//      CHECK: %[[CA:.*]] = builtin.unrealized_conversion_cast %[[A]] : vector<f32> to vector<1xf32>
-//      CHECK: %[[C:.*]] = llvm.mlir.constant(0.000000e+00 : f32) : f32
-//      CHECK: %[[V:.*]] = "llvm.intr.vector.reduce.fadd"(%[[C]], %[[CA]])
-// CHECK-SAME: <{fastmathFlags = #llvm.fastmath<none>}> : (f32, vector<1xf32>) -> f32
-//      CHECK: return %[[V]] : f32
-
-// -----
-
-func.func @reduce_f16(%arg0: vector<16xf16>) -> f16 {
-  %0 = vector.reduction <add>, %arg0 : vector<16xf16> into f16
-  return %0 : f16
-}
-// CHECK-LABEL: @reduce_f16(
-// CHECK-SAME: %[[A:.*]]: vector<16xf16>)
-//      CHECK: %[[C:.*]] = llvm.mlir.constant(0.000000e+00 : f16) : f16
-//      CHECK: %[[V:.*]] = "llvm.intr.vector.reduce.fadd"(%[[C]], %[[A]])
-// CHECK-SAME: <{fastmathFlags = #llvm.fastmath<none>}> : (f16, vector<16xf16>) -> f16
-//      CHECK: return %[[V]] : f16
-
-// -----
-
-func.func @reduce_f16_scalable(%arg0: vector<[16]xf16>) -> f16 {
-  %0 = vector.reduction <add>, %arg0 : vector<[16]xf16> into f16
-  return %0 : f16
-}
-// CHECK-LABEL: @reduce_f16_scalable(
-// CHECK-SAME: %[[A:.*]]: vector<[16]xf16>)
-//      CHECK: %[[C:.*]] = llvm.mlir.constant(0.000000e+00 : f16) : f16
-//      CHECK: %[[V:.*]] = "llvm.intr.vector.reduce.fadd"(%[[C]], %[[A]])
-// CHECK-SAME: <{fastmathFlags = #llvm.fastmath<none>}> : (f16, vector<[16]xf16>) -> f16
-//      CHECK: return %[[V]] : f16
-
-// -----
-
-func.func @reduce_f32(%arg0: vector<16xf32>) -> f32 {
-  %0 = vector.reduction <add>, %arg0 : vector<16xf32> into f32
-  return %0 : f32
-}
-// CHECK-LABEL: @reduce_f32(
-// CHECK-SAME: %[[A:.*]]: vector<16xf32>)
-//      CHECK: %[[C:.*]] = llvm.mlir.constant(0.000000e+00 : f32) : f32
-//      CHECK: %[[V:.*]] = "llvm.intr.vector.reduce.fadd"(%[[C]], %[[A]])
-// CHECK-SAME: <{fastmathFlags = #llvm.fastmath<none>}> : (f32, vector<16xf32>) -> f32
-//      CHECK: return %[[V]] : f32
-
-// -----
-
-func.func @reduce_f32_scalable(%arg0: vector<[16]xf32>) -> f32 {
-  %0 = vector.reduction <add>, %arg0 : vector<[16]xf32> into f32
-  return %0 : f32
-}
-// CHECK-LABEL: @reduce_f32_scalable(
-// CHECK-SAME: %[[A:.*]]: vector<[16]xf32>)
-//      CHECK: %[[C:.*]] = llvm.mlir.constant(0.000000e+00 : f32) : f32
-//      CHECK: %[[V:.*]] = "llvm.intr.vector.reduce.fadd"(%[[C]], %[[A]])
-// CHECK-SAME: <{fastmathFlags = #llvm.fastmath<none>}> : (f32, vector<[16]xf32>) -> f32
-//      CHECK: return %[[V]] : f32
-
-// -----
-
-func.func @reduce_f64(%arg0: vector<16xf64>) -> f64 {
-  %0 = vector.reduction <add>, %arg0 : vector<16xf64> into f64
-  return %0 : f64
-}
-// CHECK-LABEL: @reduce_f64(
-// CHECK-SAME: %[[A:.*]]: vector<16xf64>)
-//      CHECK: %[[C:.*]] = llvm.mlir.constant(0.000000e+00 : f64) : f64
-//      CHECK: %[[V:.*]] = "llvm.intr.vector.reduce.fadd"(%[[C]], %[[A]])
-// CHECK-SAME: <{fastmathFlags = #llvm.fastmath<none>}> : (f64, vector<16xf64>) -> f64
-//      CHECK: return %[[V]] : f64
-
-// -----
-
-func.func @reduce_f64_scalable(%arg0: vector<[16]xf64>) -> f64 {
-  %0 = vector.reduction <add>, %arg0 : vector<[16]xf64> into f64
-  return %0 : f64
-}
-// CHECK-LABEL: @reduce_f64_scalable(
-// CHECK-SAME: %[[A:.*]]: vector<[16]xf64>)
-//      CHECK: %[[C:.*]] = llvm.mlir.constant(0.000000e+00 : f64) : f64
-//      CHECK: %[[V:.*]] = "llvm.intr.vector.reduce.fadd"(%[[C]], %[[A]])
-// CHECK-SAME: <{fastmathFlags = #llvm.fastmath<none>}> : (f64, vector<[16]xf64>) -> f64
-//      CHECK: return %[[V]] : f64
-
-// -----
-
-func.func @reduce_i8(%arg0: vector<16xi8>) -> i8 {
-  %0 = vector.reduction <add>, %arg0 : vector<16xi8> into i8
-  return %0 : i8
-}
-// CHECK-LABEL: @reduce_i8(
-// CHECK-SAME: %[[A:.*]]: vector<16xi8>)
-//      CHECK: %[[V:.*]] = "llvm.intr.vector.reduce.add"(%[[A]])
-//      CHECK: return %[[V]] : i8
-
-// -----
-
-func.func @reduce_i8_scalable(%arg0: vector<[16]xi8>) -> i8 {
-  %0 = vector.reduction <add>, %arg0 : vector<[16]xi8> into i8
-  return %0 : i8
-}
-// CHECK-LABEL: @reduce_i8_scalable(
-// CHECK-SAME: %[[A:.*]]: vector<[16]xi8>)
-//      CHECK: %[[V:.*]] = "llvm.intr.vector.reduce.add"(%[[A]])
-//      CHECK: return %[[V]] : i8
-
-// -----
-
-func.func @reduce_i32(%arg0: vector<16xi32>) -> i32 {
-  %0 = vector.reduction <add>, %arg0 : vector<16xi32> into i32
-  return %0 : i32
-}
-// CHECK-LABEL: @reduce_i32(
-// CHECK-SAME: %[[A:.*]]: vector<16xi32>)
-//      CHECK: %[[V:.*]] = "llvm.intr.vector.reduce.add"(%[[A]])
-//      CHECK: return %[[V]] : i32
-
-// -----
-
-func.func @reduce_i32_scalable(%arg0: vector<[16]xi32>) -> i32 {
-  %0 = vector.reduction <add>, %arg0 : vector<[16]xi32> into i32
-  return %0 : i32
-}
-// CHECK-LABEL: @reduce_i32_scalable(
-// CHECK-SAME: %[[A:.*]]: vector<[16]xi32>)
-//      CHECK: %[[V:.*]] = "llvm.intr.vector.reduce.add"(%[[A]])
-//      CHECK: return %[[V]] : i32
-
-// -----
-
-func.func @reduce_acc_i32(%arg0: vector<16xi32>, %arg1 : i32) -> i32 {
-  %0 = vector.reduction <add>, %arg0, %arg1 : vector<16xi32> into i32
-  return %0 : i32
-}
-// CHECK-LABEL: @reduce_acc_i32(
-//  CHECK-SAME: %[[A:.*]]: vector<16xi32>, %[[ACC:.*]]: i32)
-//       CHECK: %[[R:.*]] = "llvm.intr.vector.reduce.add"(%[[A]])
-//       CHECK: %[[V:.*]] = llvm.add %[[ACC]], %[[R]]
-//       CHECK: return %[[V]] : i32
-
-// -----
-
-func.func @reduce_acc_i32_scalable(%arg0: vector<[16]xi32>, %arg1 : i32) -> i32 {
-  %0 = vector.reduction <add>, %arg0, %arg1 : vector<[16]xi32> into i32
-  return %0 : i32
-}
-// CHECK-LABEL: @reduce_acc_i32_scalable(
-//  CHECK-SAME: %[[A:.*]]: vector<[16]xi32>, %[[ACC:.*]]: i32)
-//       CHECK: %[[R:.*]] = "llvm.intr.vector.reduce.add"(%[[A]])
-//       CHECK: %[[V:.*]] = llvm.add %[[ACC]], %[[R]]
-//       CHECK: return %[[V]] : i32
-
-// -----
-
-func.func @reduce_mul_i32(%arg0: vector<16xi32>) -> i32 {
-  %0 = vector.reduction <mul>, %arg0 : vector<16xi32> into i32
-  return %0 : i32
-}
-// CHECK-LABEL: @reduce_mul_i32(
-//  CHECK-SAME: %[[A:.*]]: vector<16xi32>)
-//       CHECK: %[[V:.*]] = "llvm.intr.vector.reduce.mul"(%[[A]])
-//       CHECK: return %[[V]] : i32
-
-// -----
-
-func.func @reduce_mul_i32_scalable(%arg0: vector<[16]xi32>) -> i32 {
-  %0 = vector.reduction <mul>, %arg0 : vector<[16]xi32> into i32
-  return %0 : i32
-}
-// CHECK-LABEL: @reduce_mul_i32_scalable(
-//  CHECK-SAME: %[[A:.*]]: vector<[16]xi32>)
-//       CHECK: %[[V:.*]] = "llvm.intr.vector.reduce.mul"(%[[A]])
-//       CHECK: return %[[V]] : i32
-
-// -----
-
-func.func @reduce_mul_acc_i32(%arg0: vector<16xi32>, %arg1 : i32) -> i32 {
-  %0 = vector.reduction <mul>, %arg0, %arg1 : vector<16xi32> into i32
-  return %0 : i32
-}
-// CHECK-LABEL: @reduce_mul_acc_i32(
-//  CHECK-SAME: %[[A:.*]]: vector<16xi32>, %[[ACC:.*]]: i32)
-//       CHECK: %[[R:.*]] = "llvm.intr.vector.reduce.mul"(%[[A]])
-//       CHECK: %[[V:.*]] = llvm.mul %[[ACC]], %[[R]]
-//       CHECK: return %[[V]] : i32
-
-// -----
-
-func.func @reduce_mul_acc_i32_scalable(%arg0: vector<[16]xi32>, %arg1 : i32) -> i32 {
-  %0 = vector.reduction <mul>, %arg0, %arg1 : vector<[16]xi32> into i32
-  return %0 : i32
-}
-// CHECK-LABEL: @reduce_mul_acc_i32_scalable(
-//  CHECK-SAME: %[[A:.*]]: vector<[16]xi32>, %[[ACC:.*]]: i32)
-//       CHECK: %[[R:.*]] = "llvm.intr.vector.reduce.mul"(%[[A]])
-//       CHECK: %[[V:.*]] = llvm.mul %[[ACC]], %[[R]]
-//       CHECK: return %[[V]] : i32
-
-// -----
-
-func.func @reduce_fmaximum_f32(%arg0: vector<16xf32>, %arg1: f32) -> f32 {
-  %0 = vector.reduction <maximumf>, %arg0, %arg1 : vector<16xf32> into f32
-  return %0 : f32
-}
-// CHECK-LABEL: @reduce_fmaximum_f32(
-// CHECK-SAME: %[[A:.*]]: vector<16xf32>, %[[B:.*]]: f32)
-//      CHECK: %[[V:.*]] = llvm.intr.vector.reduce.fmaximum(%[[A]]) : (vector<16xf32>) -> f32
-//      CHECK: %[[R:.*]] = llvm.intr.maximum(%[[V]], %[[B]]) : (f32, f32) -> f32
-//      CHECK: return %[[R]] : f32
-
-// -----
-
-func.func @reduce_fmaximum_f32_scalable(%arg0: vector<[16]xf32>, %arg1: f32) -> f32 {
-  %0 = vector.reduction <maximumf>, %arg0, %arg1 : vector<[16]xf32> into f32
-  return %0 : f32
-}
-// CHECK-LABEL: @reduce_fmaximum_f32_scalable(
-// CHECK-SAME: %[[A:.*]]: vector<[16]xf32>, %[[B:.*]]: f32)
-//      CHECK: %[[V:.*]] = llvm.intr.vector.reduce.fmaximum(%[[A]]) : (vector<[16]xf32>) -> f32
-//      CHECK: %[[R:.*]] = llvm.intr.maximum(%[[V]], %[[B]]) : (f32, f32) -> f32
-//      CHECK: return %[[R]] : f32
-
-// -----
-
-func.func @reduce_fminimum_f32(%arg0: vector<16xf32>, %arg1: f32) -> f32 {
-  %0 = vector.reduction <minimumf>, %arg0, %arg1 : vector<16xf32> into f32
-  return %0 : f32
-}
-// CHECK-LABEL: @reduce_fminimum_f32(
-// CHECK-SAME: %[[A:.*]]: vector<16xf32>, %[[B:.*]]: f32)
-//      CHECK: %[[V:.*]] = llvm.intr.vector.reduce.fminimum(%[[A]]) : (vector<16xf32>) -> f32
-//      CHECK: %[[R:.*]] = llvm.intr.minimum(%[[V]], %[[B]]) : (f32, f32) -> f32
-//      CHECK: return %[[R]] : f32
-
-// -----
-
-func.func @reduce_fminimum_f32_scalable(%arg0: vector<[16]xf32>, %arg1: f32) -> f32 {
-  %0 = vector.reduction <minimumf>, %arg0, %arg1 : vector<[16]xf32> into f32
-  return %0 : f32
-}
-// CHECK-LABEL: @reduce_fminimum_f32_scalable(
-// CHECK-SAME: %[[A:.*]]: vector<[16]xf32>, %[[B:.*]]: f32)
-//      CHECK: %[[V:.*]] = llvm.intr.vector.reduce.fminimum(%[[A]]) : (vector<[16]xf32>) -> f32
-//      CHECK: %[[R:.*]] = llvm.intr.minimum(%[[V]], %[[B]]) : (f32, f32) -> f32
-//      CHECK: return %[[R]] : f32
-
-// -----
-
-func.func @reduce_fmax_f32(%arg0: vector<16xf32>, %arg1: f32) -> f32 {
-  %0 = vector.reduction <maxnumf>, %arg0, %arg1 : vector<16xf32> into f32
-  return %0 : f32
-}
-// CHECK-LABEL: @reduce_fmax_f32(
-// CHECK-SAME: %[[A:.*]]: vector<16xf32>, %[[B:.*]]: f32)
-//      CHECK: %[[V:.*]] = llvm.intr.vector.reduce.fmax(%[[A]]) : (vector<16xf32>) -> f32
-//      CHECK: %[[R:.*]] = llvm.intr.maxnum(%[[V]], %[[B]]) : (f32, f32) -> f32
-//      CHECK: return %[[R]] : f32
-
-// -----
-
-func.func @reduce_fmax_f32_scalable(%arg0: vector<[16]xf32>, %arg1: f32) -> f32 {
-  %0 = vector.reduction <maxnumf>, %arg0, %arg1 : vector<[16]xf32> into f32
-  return %0 : f32
-}
-// CHECK-LABEL: @reduce_fmax_f32_scalable(
-// CHECK-SAME: %[[A:.*]]: vector<[16]xf32>, %[[B:.*]]: f32)
-//      CHECK: %[[V:.*]] = llvm.intr.vector.reduce.fmax(%[[A]]) : (vector<[16]xf32>) -> f32
-//      CHECK: %[[R:.*]] = llvm.intr.maxnum(%[[V]], %[[B]]) : (f32, f32) -> f32
-//      CHECK: return %[[R]] : f32
-
-// -----
-
-func.func @reduce_fmin_f32(%arg0: vector<16xf32>, %arg1: f32) -> f32 {
-  %0 = vector.reduction <minnumf>, %arg0, %arg1 : vector<16xf32> into f32
-  return %0 : f32
-}
-// CHECK-LABEL: @reduce_fmin_f32(
-// CHECK-SAME: %[[A:.*]]: vector<16xf32>, %[[B:.*]]: f32)
-//      CHECK: %[[V:.*]] = llvm.intr.vector.reduce.fmin(%[[A]]) : (vector<16xf32>) -> f32
-//      CHECK: %[[R:.*]] = llvm.intr.minnum(%[[V]], %[[B]]) : (f32, f32) -> f32
-//      CHECK: return %[[R]] : f32
-
-// -----
-
-func.func @reduce_fmin_f32_scalable(%arg0: vector<[16]xf32>, %arg1: f32) -> f32 {
-  %0 = vector.reduction <minnumf>, %arg0, %arg1 : vector<[16]xf32> into f32
-  return %0 : f32
-}
-// CHECK-LABEL: @reduce_fmin_f32_scalable(
-// CHECK-SAME: %[[A:.*]]: vector<[16]xf32>, %[[B:.*]]: f32)
-//      CHECK: %[[V:.*]] = llvm.intr.vector.reduce.fmin(%[[A]]) : (vector<[16]xf32>) -> f32
-//      CHECK: %[[R:.*]] = llvm.intr.minnum(%[[V]], %[[B]]) : (f32, f32) -> f32
-//      CHECK: return %[[R]] : f32
-
-// -----
-
-func.func @reduce_minui_i32(%arg0: vector<16xi32>) -> i32 {
-  %0 = vector.reduction <minui>, %arg0 : vector<16xi32> into i32
-  return %0 : i32
-}
-// CHECK-LABEL: @reduce_minui_i32(
-//  CHECK-SAME: %[[A:.*]]: vector<16xi32>)
-//       CHECK: %[[V:.*]] = "llvm.intr.vector.reduce.umin"(%[[A]])
-//       CHECK: return %[[V]] : i32
-
-// -----
-
-func.func @reduce_minui_i32_scalable(%arg0: vector<[16]xi32>) -> i32 {
-  %0 = vector.reduction <minui>, %arg0 : vector<[16]xi32> into i32
-  return %0 : i32
-}
-// CHECK-LABEL: @reduce_minui_i32_scalable(
-//  CHECK-SAME: %[[A:.*]]: vector<[16]xi32>)
-//       CHECK: %[[V:.*]] = "llvm.intr.vector.reduce.umin"(%[[A]])
-//       CHECK: return %[[V]] : i32
-
-// -----
-
-func.func @reduce_minui_acc_i32(%arg0: vector<16xi32>, %arg1 : i32) -> i32 {
-  %0 = vector.reduction <minui>, %arg0, %arg1 : vector<16xi32> into i32
-  return %0 : i32
-}
-// CHECK-LABEL: @reduce_minui_acc_i32(
-//  CHECK-SAME: %[[A:.*]]: vector<16xi32>, %[[ACC:.*]]: i32)
-//       CHECK: %[[R:.*]] = "llvm.intr.vector.reduce.umin"(%[[A]])
-//       CHECK: %[[S:.*]] = llvm.icmp "ule" %[[ACC]], %[[R]]
-//       CHECK: %[[V:.*]] = llvm.select %[[S]], %[[ACC]], %[[R]]
-//       CHECK: return %[[V]] : i32
-
-// -----
-
-func.func @reduce_minui_acc_i32_scalable(%arg0: vector<[16]xi32>, %arg1 : i32) -> i32 {
-  %0 = vector.reduction <minui>, %arg0, %arg1 : vector<[16]xi32> into i32
-  return %0 : i32
-}
-// CHECK-LABEL: @reduce_minui_acc_i32_scalable(
-//  CHECK-SAME: %[[A:.*]]: vector<[16]xi32>, %[[ACC:.*]]: i32)
-//       CHECK: %[[R:.*]] = "llvm.intr.vector.reduce.umin"(%[[A]])
-//       CHECK: %[[S:.*]] = llvm.icmp "ule" %[[ACC]], %[[R]]
-//       CHECK: %[[V:.*]] = llvm.select %[[S]], %[[ACC]], %[[R]]
-//       CHECK: return %[[V]] : i32
-
-// -----
-
-func.func @reduce_maxui_i32(%arg0: vector<16xi32>) -> i32 {
-  %0 = vector.reduction <maxui>, %arg0 : vector<16xi32> into i32
-  return %0 : i32
-}
-// CHECK-LABEL: @reduce_maxui_i32(
-//  CHECK-SAME: %[[A:.*]]: vector<16xi32>)
-//       CHECK: %[[V:.*]] = "llvm.intr.vector.reduce.umax"(%[[A]])
-//       CHECK: return %[[V]] : i32
-
-// -----
-
-func.func @reduce_maxui_i32_scalable(%arg0: vector<[16]xi32>) -> i32 {
-  %0 = vector.reduction <maxui>, %arg0 : vector<[16]xi32> into i32
-  return %0 : i32
-}
-// CHECK-LABEL: @reduce_maxui_i32_scalable(
-//  CHECK-SAME: %[[A:.*]]: vector<[16]xi32>)
-//       CHECK: %[[V:.*]] = "llvm.intr.vector.reduce.umax"(%[[A]])
-//       CHECK: return %[[V]] : i32
-
-// -----
-
-func.func @reduce_maxui_acc_i32(%arg0: vector<16xi32>, %arg1 : i32) -> i32 {
-  %0 = vector.reduction <maxui>, %arg0, %arg1 : vector<16xi32> into i32
-  return %0 : i32
-}
-// CHECK-LABEL: @reduce_maxui_acc_i32(
-//  CHECK-SAME: %[[A:.*]]: vector<16xi32>, %[[ACC:.*]]: i32)
-//       CHECK: %[[R:.*]] = "llvm.intr.vector.reduce.umax"(%[[A]])
-//       CHECK: %[[S:.*]] = llvm.icmp "uge" %[[ACC]], %[[R]]
-//       CHECK: %[[V:.*]] = llvm.select %[[S]], %[[ACC]], %[[R]]
-//       CHECK: return %[[V]] : i32
-
-// -----
-
-func.func @reduce_maxui_acc_i32_scalable(%arg0: vector<[16]xi32>, %arg1 : i32) -> i32 {
-  %0 = vector.reduction <maxui>, %arg0, %arg1 : vector<[16]xi32> into i32
-  return %0 : i32
-}
-// CHECK-LABEL: @reduce_maxui_acc_i32_scalable(
-//  CHECK-SAME: %[[A:.*]]: vector<[16]xi32>, %[[ACC:.*]]: i32)
-//       CHECK: %[[R:.*]] = "llvm.intr.vector.reduce.umax"(%[[A]])
-//       CHECK: %[[S:.*]] = llvm.icmp "uge" %[[ACC]], %[[R]]
-//       CHECK: %[[V:.*]] = llvm.select %[[S]], %[[ACC]], %[[R]]
-//       CHECK: return %[[V]] : i32
-
-// -----
-
-func.func @reduce_minsi_i32(%arg0: vector<16xi32>) -> i32 {
-  %0 = vector.reduction <minsi>, %arg0 : vector<16xi32> into i32
-  return %0 : i32
-}
-// CHECK-LABEL: @reduce_minsi_i32(
-//  CHECK-SAME: %[[A:.*]]: vector<16xi32>)
-//       CHECK: %[[V:.*]] = "llvm.intr.vector.reduce.smin"(%[[A]])
-//       CHECK: return %[[V]] : i32
-
-// -----
-
-func.func @reduce_minsi_i32_scalable(%arg0: vector<[16]xi32>) -> i32 {
-  %0 = vector.reduction <minsi>, %arg0 : vector<[16]xi32> into i32
-  return %0 : i32
-}
-// CHECK-LABEL: @reduce_minsi_i32_scalable(
-//  CHECK-SAME: %[[A:.*]]: vector<[16]xi32>)
-//       CHECK: %[[V:.*]] = "llvm.intr.vector.reduce.smin"(%[[A]])
-//       CHECK: return %[[V]] : i32
-
-// -----
-
-func.func @reduce_minsi_acc_i32(%arg0: vector<16xi32>, %arg1 : i32) -> i32 {
-  %0 = vector.reduction <minsi>, %arg0, %arg1 : vector<16xi32> into i32
-  return %0 : i32
-}
-// CHECK-LABEL: @reduce_minsi_acc_i32(
-//  CHECK-SAME: %[[A:.*]]: vector<16xi32>, %[[ACC:.*]]: i32)
-//       CHECK: %[[R:.*]] = "llvm.intr.vector.reduce.smin"(%[[A]])
-//       CHECK: %[[S:.*]] = llvm.icmp "sle" %[[ACC]], %[[R]]
-//       CHECK: %[[V:.*]] = llvm.select %[[S]], %[[ACC]], %[[R]]
-//       CHECK: return %[[V]] : i32
-
-// -----
-
-func.func @reduce_minsi_acc_i32_scalable(%arg0: vector<[16]xi32>, %arg1 : i32) -> i32 {
-  %0 = vector.reduction <minsi>, %arg0, %arg1 : vector<[16]xi32> into i32
-  return %0 : i32
-}
-// CHECK-LABEL: @reduce_minsi_acc_i32_scalable(
-//  CHECK-SAME: %[[A:.*]]: vector<[16]xi32>, %[[ACC:.*]]: i32)
-//       CHECK: %[[R:.*]] = "llvm.intr.vector.reduce.smin"(%[[A]])
-//       CHECK: %[[S:.*]] = llvm.icmp "sle" %[[ACC]], %[[R]]
-//       CHECK: %[[V:.*]] = llvm.select %[[S]], %[[ACC]], %[[R]]
-//       CHECK: return %[[V]] : i32
-
-// -----
-
-func.func @reduce_maxsi_i32(%arg0: vector<16xi32>) -> i32 {
-  %0 = vector.reduction <maxsi>, %arg0 : vector<16xi32> into i32
-  return %0 : i32
-}
-// CHECK-LABEL: @reduce_maxsi_i32(
-//  CHECK-SAME: %[[A:.*]]: vector<16xi32>)
-//       CHECK: %[[V:.*]] = "llvm.intr.vector.reduce.smax"(%[[A]])
-//       CHECK: return %[[V]] : i32
-
-// -----
-
-func.func @reduce_maxsi_i32_scalable(%arg0: vector<[16]xi32>) -> i32 {
-  %0 = vector.reduction <maxsi>, %arg0 : vector<[16]xi32> into i32
-  return %0 : i32
-}
-// CHECK-LABEL: @reduce_maxsi_i32_scalable(
-//  CHECK-SAME: %[[A:.*]]: vector<[16]xi32>)
-//       CHECK: %[[V:.*]] = "llvm.intr.vector.reduce.smax"(%[[A]])
-//       CHECK: return %[[V]] : i32
-
-// -----
-
-func.func @reduce_maxsi_acc_i32(%arg0: vector<16xi32>, %arg1 : i32) -> i32 {
-  %0 = vector.reduction <maxsi>, %arg0, %arg1 : vector<16xi32> into i32
-  return %0 : i32
-}
-// CHECK-LABEL: @reduce_maxsi_acc_i32(
-//  CHECK-SAME: %[[A:.*]]: vector<16xi32>, %[[ACC:.*]]: i32)
-//       CHECK: %[[R:.*]] = "llvm.intr.vector.reduce.smax"(%[[A]])
-//       CHECK: %[[S:.*]] = llvm.icmp "sge" %[[ACC]], %[[R]]
-//       CHECK: %[[V:.*]] = llvm.select %[[S]], %[[ACC]], %[[R]]
-//       CHECK: return %[[V]] : i32
-
-// -----
-
-func.func @reduce_maxsi_acc_i32_scalable(%arg0: vector<[16]xi32>, %arg1 : i32) -> i32 {
-  %0 = vector.reduction <maxsi>, %arg0, %arg1 : vector<[16]xi32> into i32
-  return %0 : i32
-}
-// CHECK-LABEL: @reduce_maxsi_acc_i32_scalable(
-//  CHECK-SAME: %[[A:.*]]: vector<[16]xi32>, %[[ACC:.*]]: i32)
-//       CHECK: %[[R:.*]] = "llvm.intr.vector.reduce.smax"(%[[A]])
-//       CHECK: %[[S:.*]] = llvm.icmp "sge" %[[ACC]], %[[R]]
-//       CHECK: %[[V:.*]] = llvm.select %[[S]], %[[ACC]], %[[R]]
-//       CHECK: return %[[V]] : i32
-
-// -----
-
-func.func @reduce_and_i32(%arg0: vector<16xi32>) -> i32 {
-  %0 = vector.reduction <and>, %arg0 : vector<16xi32> into i32
-  return %0 : i32
-}
-// CHECK-LABEL: @reduce_and_i32(
-//  CHECK-SAME: %[[A:.*]]: vector<16xi32>)
-//       CHECK: %[[V:.*]] = "llvm.intr.vector.reduce.and"(%[[A]])
-//       CHECK: return %[[V]] : i32
-
-// -----
-
-func.func @reduce_and_i32_scalable(%arg0: vector<[16]xi32>) -> i32 {
-  %0 = vector.reduction <and>, %arg0 : vector<[16]xi32> into i32
-  return %0 : i32
-}
-// CHECK-LABEL: @reduce_and_i32_scalable(
-//  CHECK-SAME: %[[A:.*]]: vector<[16]xi32>)
-//       CHECK: %[[V:.*]] = "llvm.intr.vector.reduce.and"(%[[A]])
-//       CHECK: return %[[V]] : i32
-
-// -----
-
-func.func @reduce_and_acc_i32(%arg0: vector<16xi32>, %arg1 : i32) -> i32 {
-  %0 = vector.reduction <and>, %arg0, %arg1 : vector<16xi32> into i32
-  return %0 : i32
-}
-// CHECK-LABEL: @reduce_and_acc_i32(
-//  CHECK-SAME: %[[A:.*]]: vector<16xi32>, %[[ACC:.*]]: i32)
-//       CHECK: %[[R:.*]] = "llvm.intr.vector.reduce.and"(%[[A]])
-//       CHECK: %[[V:.*]] = llvm.and %[[ACC]], %[[R]]
-//       CHECK: return %[[V]] : i32
-
-// -----
-
-func.func @reduce_and_acc_i32_scalable(%arg0: vector<[16]xi32>, %arg1 : i32) -> i32 {
-  %0 = vector.reduction <and>, %arg0, %arg1 : vector<[16]xi32> into i32
-  return %0 : i32
-}
-// CHECK-LABEL: @reduce_and_acc_i32_scalable(
-//  CHECK-SAME: %[[A:.*]]: vector<[16]xi32>, %[[ACC:.*]]: i32)
-//       CHECK: %[[R:.*]] = "llvm.intr.vector.reduce.and"(%[[A]])
-//       CHECK: %[[V:.*]] = llvm.and %[[ACC]], %[[R]]
-//       CHECK: return %[[V]] : i32
-
-// -----
-
-func.func @reduce_or_i32(%arg0: vector<16xi32>) -> i32 {
-  %0 = vector.reduction <or>, %arg0 : vector<16xi32> into i32
-  return %0 : i32
-}
-// CHECK-LABEL: @reduce_or_i32(
-//  CHECK-SAME: %[[A:.*]]: vector<16xi32>)
-//       CHECK: %[[V:.*]] = "llvm.intr.vector.reduce.or"(%[[A]])
-//       CHECK: return %[[V]] : i32
-
-// -----
-
-func.func @reduce_or_i32_scalable(%arg0: vector<[16]xi32>) -> i32 {
-  %0 = vector.reduction <or>, %arg0 : vector<[16]xi32> into i32
-  return %0 : i32
-}
-// CHECK-LABEL: @reduce_or_i32_scalable(
-//  CHECK-SAME: %[[A:.*]]: vector<[16]xi32>)
-//       CHECK: %[[V:.*]] = "llvm.intr.vector.reduce.or"(%[[A]])
-//       CHECK: return %[[V]] : i32
-
-// -----
-
-func.func @reduce_or_acc_i32(%arg0: vector<16xi32>, %arg1 : i32) -> i32 {
-  %0 = vector.reduction <or>, %arg0, %arg1 : vector<16xi32> into i32
-  return %0 : i32
-}
-// CHECK-LABEL: @reduce_or_acc_i32(
-//  CHECK-SAME: %[[A:.*]]: vector<16xi32>, %[[ACC:.*]]: i32)
-//       CHECK: %[[R:.*]] = "llvm.intr.vector.reduce.or"(%[[A]])
-//       CHECK: %[[V:.*]] = llvm.or %[[ACC]], %[[R]]
-//       CHECK: return %[[V]] : i32
-
-// -----
-
-func.func @reduce_or_acc_i32_scalable(%arg0: vector<[16]xi32>, %arg1 : i32) -> i32 {
-  %0 = vector.reduction <or>, %arg0, %arg1 : vector<[16]xi32> into i32
-  return %0 : i32
-}
-// CHECK-LABEL: @reduce_or_acc_i32_scalable(
-//  CHECK-SAME: %[[A:.*]]: vector<[16]xi32>, %[[ACC:.*]]: i32)
-//       CHECK: %[[R:.*]] = "llvm.intr.vector.reduce.or"(%[[A]])
-//       CHECK: %[[V:.*]] = llvm.or %[[ACC]], %[[R]]
-//       CHECK: return %[[V]] : i32
-
-// -----
-
-func.func @reduce_xor_i32(%arg0: vector<16xi32>) -> i32 {
-  %0 = vector.reduction <xor>, %arg0 : vector<16xi32> into i32
-  return %0 : i32
-}
-// CHECK-LABEL: @reduce_xor_i32(
-//  CHECK-SAME: %[[A:.*]]: vector<16xi32>)
-//       CHECK: %[[V:.*]] = "llvm.intr.vector.reduce.xor"(%[[A]])
-//       CHECK: return %[[V]] : i32
-
-// -----
-
-func.func @reduce_xor_i32_scalable(%arg0: vector<[16]xi32>) -> i32 {
-  %0 = vector.reduction <xor>, %arg0 : vector<[16]xi32> into i32
-  return %0 : i32
-}
-// CHECK-LABEL: @reduce_xor_i32_scalable(
-//  CHECK-SAME: %[[A:.*]]: vector<[16]xi32>)
-//       CHECK: %[[V:.*]] = "llvm.intr.vector.reduce.xor"(%[[A]])
-//       CHECK: return %[[V]] : i32
-
-// -----
-
-func.func @reduce_xor_acc_i32(%arg0: vector<16xi32>, %arg1 : i32) -> i32 {
-  %0 = vector.reduction <xor>, %arg0, %arg1 : vector<16xi32> into i32
-  return %0 : i32
-}
-// CHECK-LABEL: @reduce_xor_acc_i32(
-//  CHECK-SAME: %[[A:.*]]: vector<16xi32>, %[[ACC:.*]]: i32)
-//       CHECK: %[[R:.*]] = "llvm.intr.vector.reduce.xor"(%[[A]])
-//       CHECK: %[[V:.*]] = llvm.xor %[[ACC]], %[[R]]
-//       CHECK: return %[[V]] : i32
-
-// -----
-
-func.func @reduce_xor_acc_i32_scalable(%arg0: vector<[16]xi32>, %arg1 : i32) -> i32 {
-  %0 = vector.reduction <xor>, %arg0, %arg1 : vector<[16]xi32> into i32
-  return %0 : i32
-}
-// CHECK-LABEL: @reduce_xor_acc_i32_scalable(
-//  CHECK-SAME: %[[A:.*]]: vector<[16]xi32>, %[[ACC:.*]]: i32)
-//       CHECK: %[[R:.*]] = "llvm.intr.vector.reduce.xor"(%[[A]])
-//       CHECK: %[[V:.*]] = llvm.xor %[[ACC]], %[[R]]
-//       CHECK: return %[[V]] : i32
-
-// -----
-
-func.func @reduce_i64(%arg0: vector<16xi64>) -> i64 {
-  %0 = vector.reduction <add>, %arg0 : vector<16xi64> into i64
-  return %0 : i64
-}
-// CHECK-LABEL: @reduce_i64(
-// CHECK-SAME: %[[A:.*]]: vector<16xi64>)
-//      CHECK: %[[V:.*]] = "llvm.intr.vector.reduce.add"(%[[A]])
-//      CHECK: return %[[V]] : i64
-
-// -----
-
-func.func @reduce_i64_scalable(%arg0: vector<[16]xi64>) -> i64 {
-  %0 = vector.reduction <add>, %arg0 : vector<[16]xi64> into i64
-  return %0 : i64
-}
-// CHECK-LABEL: @reduce_i64_scalable(
-// CHECK-SAME: %[[A:.*]]: vector<[16]xi64>)
-//      CHECK: %[[V:.*]] = "llvm.intr.vector.reduce.add"(%[[A]])
-//      CHECK: return %[[V]] : i64
-
-// -----
-
-func.func @reduce_index(%arg0: vector<16xindex>) -> index {
-  %0 = vector.reduction <add>, %arg0 : vector<16xindex> into index
-  return %0 : index
-}
-// CHECK-LABEL: @reduce_index(
-// CHECK-SAME: %[[A:.*]]: vector<16xindex>)
-//      CHECK: %[[T0:.*]] = builtin.unrealized_conversion_cast %[[A]] : vector<16xindex> to vector<16xi64>
-//      CHECK: %[[T1:.*]] = "llvm.intr.vector.reduce.add"(%[[T0]])
-//      CHECK: %[[T2:.*]] = builtin.unrealized_conversion_cast %[[T1]] : i64 to index
-//      CHECK: return %[[T2]] : index
-
-// -----
-
-func.func @reduce_index_scalable(%arg0: vector<[16]xindex>) -> index {
-  %0 = vector.reduction <add>, %arg0 : vector<[16]xindex> into index
-  return %0 : index
-}
-// CHECK-LABEL: @reduce_index_scalable(
-// CHECK-SAME: %[[A:.*]]: vector<[16]xindex>)
-//      CHECK: %[[T0:.*]] = builtin.unrealized_conversion_cast %[[A]] : vector<[16]xindex> to vector<[16]xi64>
-//      CHECK: %[[T1:.*]] = "llvm.intr.vector.reduce.add"(%[[T0]])
-//      CHECK: %[[T2:.*]] = builtin.unrealized_conversion_cast %[[T1]] : i64 to index
-//      CHECK: return %[[T2]] : index
-
 // -----
 
 //===----------------------------------------------------------------------===//
@@ -3060,591 +1527,143 @@ func.func @constant_mask_2d() -> vector<4x4xi1> {
 
 // -----
 
-func.func @constant_mask_2d_trailing_scalable() -> vector<4x[4]xi1> {
-  %0 = vector.constant_mask [2, 4] : vector<4x[4]xi1>
-  return %0 : vector<4x[4]xi1>
-}
-// CHECK-LABEL:   func.func @constant_mask_2d_trailing_scalable
-// CHECK:           %[[VAL_0:.*]] = arith.constant dense<true> : vector<[4]xi1>
-// CHECK:           %[[VAL_1:.*]] = arith.constant dense<false> : vector<4x[4]xi1>
-// CHECK:           %[[VAL_2:.*]] = builtin.unrealized_conversion_cast %[[VAL_1]] : vector<4x[4]xi1> to !llvm.array<4 x vector<[4]xi1>>
-// CHECK:           %[[VAL_3:.*]] = llvm.insertvalue %[[VAL_0]], %[[VAL_2]][0] : !llvm.array<4 x vector<[4]xi1>>
-// CHECK:           %[[VAL_4:.*]] = llvm.insertvalue %[[VAL_0]], %[[VAL_3]][1] : !llvm.array<4 x vector<[4]xi1>>
-// CHECK:           %[[VAL_5:.*]] = builtin.unrealized_conversion_cast %[[VAL_4]] : !llvm.array<4 x vector<[4]xi1>> to vector<4x[4]xi1>
-// CHECK:           return %[[VAL_5]] : vector<4x[4]xi1>
-
-// -----
-
-/// Currently, this is not supported as generating the mask would require
-/// unrolling the leading scalable dimension at compile time.
-func.func @negative_constant_mask_2d_leading_scalable() -> vector<[4]x4xi1> {
-  %0 = vector.constant_mask [4, 2] : vector<[4]x4xi1>
-  return %0 : vector<[4]x4xi1>
-}
-// CHECK-LABEL:   func.func @negative_constant_mask_2d_leading_scalable
-// CHECK:           %[[VAL_0:.*]] = vector.constant_mask [4, 2] : vector<[4]x4xi1>
-// CHECK:           return %[[VAL_0]] : vector<[4]x4xi1>
-
-// -----
-
-//===----------------------------------------------------------------------===//
-// vector.create_mask
-//===----------------------------------------------------------------------===//
-
-func.func @create_mask_0d(%num_elems : index) -> vector<i1> {
-  %v = vector.create_mask %num_elems : vector<i1>
-  return %v: vector<i1>
-}
-
-// CHECK-LABEL: func @create_mask_0d
-// CHECK-SAME: %[[NUM_ELEMS:.*]]: index
-// CHECK:  %[[INDICES:.*]] = arith.constant dense<0> : vector<i32>
-// CHECK:  %[[NUM_ELEMS_i32:.*]] = arith.index_cast %[[NUM_ELEMS]] : index to i32
-// CHECK:  %[[BOUNDS:.*]] = llvm.insertelement %[[NUM_ELEMS_i32]]
-// CHECK:  %[[BOUNDS_CAST:.*]] = builtin.unrealized_conversion_cast %[[BOUNDS]] : vector<1xi32> to vector<i32>
-// CHECK:  %[[RESULT:.*]] = arith.cmpi sgt, %[[BOUNDS_CAST]], %[[INDICES]] : vector<i32>
-// CHECK:  return %[[RESULT]] : vector<i1>
-
-// -----
-
-func.func @create_mask_1d(%num_elems : index) -> vector<4xi1> {
-  %v = vector.create_mask %num_elems : vector<4xi1>
-  return %v: vector<4xi1>
-}
-
-// CHECK-LABEL: func @create_mask_1d
-// CHECK-SAME: %[[NUM_ELEMS:.*]]: index
-// CHECK:  %[[INDICES:.*]] = arith.constant dense<[0, 1, 2, 3]> : vector<4xi32>
-// CHECK:  %[[NUM_ELEMS_i32:.*]] = arith.index_cast %[[NUM_ELEMS]] : index to i32
-// CHECK:  %[[BOUNDS_INSERT:.*]] = llvm.insertelement %[[NUM_ELEMS_i32]]
-// CHECK:  %[[BOUNDS:.*]] = llvm.shufflevector %[[BOUNDS_INSERT]]
-// CHECK:  %[[RESULT:.*]] = arith.cmpi sgt, %[[BOUNDS]], %[[INDICES]] : vector<4xi32>
-// CHECK:  return %[[RESULT]] : vector<4xi1>
-
-// -----
-
-func.func @create_mask_1d_scalable(%num_elems : index) -> vector<[4]xi1> {
-  %v = vector.create_mask %num_elems : vector<[4]xi1>
-  return %v: vector<[4]xi1>
-}
-
-// CHECK-LABEL: func @create_mask_1d_scalable
-// CHECK-SAME: %[[NUM_ELEMS:.*]]: index
-// CHECK:  %[[INDICES:.*]] = llvm.intr.stepvector : vector<[4]xi32>
-// CHECK:  %[[NUM_ELEMS_i32:.*]] = arith.index_cast %[[NUM_ELEMS]] : index to i32
-// CHECK:  %[[BOUNDS_INSERT:.*]] = llvm.insertelement %[[NUM_ELEMS_i32]], {{.*}} : vector<[4]xi32>
-// CHECK:  %[[BOUNDS:.*]] = llvm.shufflevector %[[BOUNDS_INSERT]], {{.*}} : vector<[4]xi32>
-// CHECK:  %[[RESULT:.*]] = arith.cmpi slt, %[[INDICES]], %[[BOUNDS]] : vector<[4]xi32>
-// CHECK: return %[[RESULT]] : vector<[4]xi1>
-
-// -----
-
-//===----------------------------------------------------------------------===//
-// vector.transpose
-//===----------------------------------------------------------------------===//
-
-func.func @transpose_0d(%arg0: vector<f32>) -> vector<f32> {
-  %0 = vector.transpose %arg0, [] : vector<f32> to vector<f32>
-  return %0 : vector<f32>
-}
-
-// CHECK-LABEL: func @transpose_0d
-// CHECK-SAME:  %[[A:.*]]: vector<f32>
-// CHECK:       return %[[A]] : vector<f32>
-
-// -----
-
-//===----------------------------------------------------------------------===//
-// vector.flat_transpose
-//===----------------------------------------------------------------------===//
-
-func.func @flat_transpose(%arg0: vector<16xf32>) -> vector<16xf32> {
-  %0 = vector.flat_transpose %arg0 { rows = 4: i32, columns = 4: i32 }
-     : vector<16xf32> -> vector<16xf32>
-  return %0 : vector<16xf32>
-}
-
-// CHECK-LABEL: func @flat_transpose
-// CHECK-SAME:  %[[A:.*]]: vector<16xf32>
-// CHECK:       %[[T:.*]] = llvm.intr.matrix.transpose %[[A]]
-// CHECK-SAME:      {columns = 4 : i32, rows = 4 : i32} :
-// CHECK-SAME:      vector<16xf32> into vector<16xf32>
-// CHECK:       return %[[T]] : vector<16xf32>
-
-// -----
-
-func.func @flat_transpose_index(%arg0: vector<16xindex>) -> vector<16xindex> {
-  %0 = vector.flat_transpose %arg0 { rows = 4: i32, columns = 4: i32 }
-     : vector<16xindex> -> vector<16xindex>
-  return %0 : vector<16xindex>
-}
-// CHECK-LABEL: func @flat_transpose_index
-// CHECK-SAME:  %[[A:.*]]: vector<16xindex>
-// CHECK:       %[[T0:.*]] = builtin.unrealized_conversion_cast %[[A]] : vector<16xindex> to vector<16xi64>
-// CHECK:       %[[T1:.*]] = llvm.intr.matrix.transpose %[[T0]]
-// CHECK-SAME:      {columns = 4 : i32, rows = 4 : i32} :
-// CHECK-SAME:      vector<16xi64> into vector<16xi64>
-// CHECK:       %[[T2:.*]] = builtin.unrealized_conversion_cast %[[T1]] : vector<16xi64> to vector<16xindex>
-// CHECK:       return %[[T2]] : vector<16xindex>
-
-// -----
-
-func.func @flat_transpose(%arg0: vector<16xf32>) -> vector<16xf32> {
-  %0 = vector.flat_transpose %arg0 { rows = 4: i32, columns = 4: i32 }
-     : vector<16xf32> -> vector<16xf32>
-  return %0 : vector<16xf32>
-}
-
-// CHECK-LABEL: func @flat_transpose
-// CHECK-SAME:  %[[A:.*]]: vector<16xf32>
-// CHECK:       %[[T:.*]] = llvm.intr.matrix.transpose %[[A]]
-// CHECK-SAME:      {columns = 4 : i32, rows = 4 : i32} :
-// CHECK-SAME:      vector<16xf32> into vector<16xf32>
-// CHECK:       return %[[T]] : vector<16xf32>
-
-// -----
-
-//===----------------------------------------------------------------------===//
-// vector.load
-//===----------------------------------------------------------------------===//
-
-func.func @load(%memref : memref<200x100xf32>, %i : index, %j : index) -> vector<8xf32> {
-  %0 = vector.load %memref[%i, %j] : memref<200x100xf32>, vector<8xf32>
-  return %0 : vector<8xf32>
-}
-
-// CHECK-LABEL: func @load
-// CHECK: %[[C100:.*]] = llvm.mlir.constant(100 : index) : i64
-// CHECK: %[[MUL:.*]] = llvm.mul %{{.*}}, %[[C100]]  : i64
-// CHECK: %[[ADD:.*]] = llvm.add %[[MUL]], %{{.*}}  : i64
-// CHECK: %[[GEP:.*]] = llvm.getelementptr %{{.*}}[%[[ADD]]] : (!llvm.ptr, i64) -> !llvm.ptr, f32
-// CHECK: llvm.load %[[GEP]] {alignment = 4 : i64} : !llvm.ptr -> vector<8xf32>
-
-// -----
-
-func.func @load_scalable(%memref : memref<200x100xf32>, %i : index, %j : index) -> vector<[8]xf32> {
-  %0 = vector.load %memref[%i, %j] : memref<200x100xf32>, vector<[8]xf32>
-  return %0 : vector<[8]xf32>
-}
-
-// CHECK-LABEL: func @load_scalable
-// CHECK: %[[C100:.*]] = llvm.mlir.constant(100 : index) : i64
-// CHECK: %[[MUL:.*]] = llvm.mul %{{.*}}, %[[C100]]  : i64
-// CHECK: %[[ADD:.*]] = llvm.add %[[MUL]], %{{.*}}  : i64
-// CHECK: %[[GEP:.*]] = llvm.getelementptr %{{.*}}[%[[ADD]]] : (!llvm.ptr, i64) -> !llvm.ptr, f32
-// CHECK: llvm.load %[[GEP]] {alignment = 4 : i64} : !llvm.ptr -> vector<[8]xf32>
-
-// -----
-
-func.func @load_nontemporal(%memref : memref<200x100xf32>, %i : index, %j : index) -> vector<8xf32> {
-  %0 = vector.load %memref[%i, %j] {nontemporal = true} : memref<200x100xf32>, vector<8xf32>
-  return %0 : vector<8xf32>
-}
-
-// CHECK-LABEL: func @load_nontemporal
-// CHECK: %[[C100:.*]] = llvm.mlir.constant(100 : index) : i64
-// CHECK: %[[MUL:.*]] = llvm.mul %{{.*}}, %[[C100]]  : i64
-// CHECK: %[[ADD:.*]] = llvm.add %[[MUL]], %{{.*}}  : i64
-// CHECK: %[[GEP:.*]] = llvm.getelementptr %{{.*}}[%[[ADD]]] : (!llvm.ptr, i64) -> !llvm.ptr, f32
-// CHECK: llvm.load %[[GEP]] {alignment = 4 : i64, nontemporal} : !llvm.ptr -> vector<8xf32>
-
-// -----
-
-func.func @load_nontemporal_scalable(%memref : memref<200x100xf32>, %i : index, %j : index) -> vector<[8]xf32> {
-  %0 = vector.load %memref[%i, %j] {nontemporal = true} : memref<200x100xf32>, vector<[8]xf32>
-  return %0 : vector<[8]xf32>
-}
-
-// CHECK-LABEL: func @load_nontemporal_scalable
-// CHECK: %[[C100:.*]] = llvm.mlir.constant(100 : index) : i64
-// CHECK: %[[MUL:.*]] = llvm.mul %{{.*}}, %[[C100]]  : i64
-// CHECK: %[[ADD:.*]] = llvm.add %[[MUL]], %{{.*}}  : i64
-// CHECK: %[[GEP:.*]] = llvm.getelementptr %{{.*}}[%[[ADD]]] : (!llvm.ptr, i64) -> !llvm.ptr, f32
-// CHECK: llvm.load %[[GEP]] {alignment = 4 : i64, nontemporal} : !llvm.ptr -> vector<[8]xf32>
-
-// -----
-
-func.func @load_index(%memref : memref<200x100xindex>, %i : index, %j : index) -> vector<8xindex> {
-  %0 = vector.load %memref[%i, %j] : memref<200x100xindex>, vector<8xindex>
-  return %0 : vector<8xindex>
-}
-// CHECK-LABEL: func @load_index
-// CHECK: %[[T0:.*]] = llvm.load %{{.*}} {alignment = 8 : i64} : !llvm.ptr -> vector<8xi64>
-// CHECK: %[[T1:.*]] = builtin.unrealized_conversion_cast %[[T0]] : vector<8xi64> to vector<8xindex>
-// CHECK: return %[[T1]] : vector<8xindex>
-
-// -----
-
-func.func @load_index_scalable(%memref : memref<200x100xindex>, %i : index, %j : index) -> vector<[8]xindex> {
-  %0 = vector.load %memref[%i, %j] : memref<200x100xindex>, vector<[8]xindex>
-  return %0 : vector<[8]xindex>
-}
-// CHECK-LABEL: func @load_index_scalable
-// CHECK: %[[T0:.*]] = llvm.load %{{.*}} {alignment = 8 : i64} : !llvm.ptr -> vector<[8]xi64>
-// CHECK: %[[T1:.*]] = builtin.unrealized_conversion_cast %[[T0]] : vector<[8]xi64> to vector<[8]xindex>
-// CHECK: return %[[T1]] : vector<[8]xindex>
-
-// -----
-
-func.func @load_0d(%memref : memref<200x100xf32>, %i : index, %j : index) -> vector<f32> {
-  %0 = vector.load %memref[%i, %j] : memref<200x100xf32>, vector<f32>
-  return %0 : vector<f32>
-}
-
-// CHECK-LABEL: func @load_0d
-// CHECK: %[[J:.*]] = builtin.unrealized_conversion_cast %{{.*}} : index to i64
-// CHECK: %[[I:.*]] = builtin.unrealized_conversion_cast %{{.*}} : index to i64
-// CHECK: %[[CAST_MEMREF:.*]] = builtin.unrealized_conversion_cast %{{.*}} : memref<200x100xf32> to !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)>
-// CHECK: %[[REF:.*]] = llvm.extractvalue %[[CAST_MEMREF]][1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)>
-// CHECK: %[[C100:.*]] = llvm.mlir.constant(100 : index) : i64
-// CHECK: %[[MUL:.*]] = llvm.mul %[[I]], %[[C100]] : i64
-// CHECK: %[[ADD:.*]] = llvm.add %[[MUL]], %[[J]] : i64
-// CHECK: %[[ADDR:.*]] = llvm.getelementptr %[[REF]][%[[ADD]]] : (!llvm.ptr, i64) -> !llvm.ptr, f32
-// CHECK: %[[LOAD:.*]] = llvm.load %[[ADDR]] {alignment = 4 : i64} : !llvm.ptr -> vector<1xf32>
-// CHECK: %[[RES:.*]] = builtin.unrealized_conversion_cast %[[LOAD]] : vector<1xf32> to vector<f32>
-// CHECK: return %[[RES]] : vector<f32>
-// -----
-
-//===----------------------------------------------------------------------===//
-// vector.store
-//===----------------------------------------------------------------------===//
-
-func.func @store(%memref : memref<200x100xf32>, %i : index, %j : index) {
-  %val = arith.constant dense<11.0> : vector<4xf32>
-  vector.store %val, %memref[%i, %j] : memref<200x100xf32>, vector<4xf32>
-  return
-}
-
-// CHECK-LABEL: func @store
-// CHECK: %[[C100:.*]] = llvm.mlir.constant(100 : index) : i64
-// CHECK: %[[MUL:.*]] = llvm.mul %{{.*}}, %[[C100]]  : i64
-// CHECK: %[[ADD:.*]] = llvm.add %[[MUL]], %{{.*}}  : i64
-// CHECK: %[[GEP:.*]] = llvm.getelementptr %{{.*}}[%[[ADD]]] : (!llvm.ptr, i64) -> !llvm.ptr, f32
-// CHECK: llvm.store %{{.*}}, %[[GEP]] {alignment = 4 : i64} :  vector<4xf32>, !llvm.ptr
-
-// -----
-
-func.func @store_scalable(%memref : memref<200x100xf32>, %i : index, %j : index) {
-  %val = arith.constant dense<11.0> : vector<[4]xf32>
-  vector.store %val, %memref[%i, %j] : memref<200x100xf32>, vector<[4]xf32>
-  return
-}
-
-// CHECK-LABEL: func @store_scalable
-// CHECK: %[[C100:.*]] = llvm.mlir.constant(100 : index) : i64
-// CHECK: %[[MUL:.*]] = llvm.mul %{{.*}}, %[[C100]]  : i64
-// CHECK: %[[ADD:.*]] = llvm.add %[[MUL]], %{{.*}}  : i64
-// CHECK: %[[GEP:.*]] = llvm.getelementptr %{{.*}}[%[[ADD]]] : (!llvm.ptr, i64) -> !llvm.ptr, f32
-// CHECK: llvm.store %{{.*}}, %[[GEP]] {alignment = 4 : i64} :  vector<[4]xf32>, !llvm.ptr
-
-// -----
-
-func.func @store_nontemporal(%memref : memref<200x100xf32>, %i : index, %j : index) {
-  %val = arith.constant dense<11.0> : vector<4xf32>
-  vector.store %val, %memref[%i, %j] {nontemporal = true} : memref<200x100xf32>, vector<4xf32>
-  return
-}
-
-// CHECK-LABEL: func @store_nontemporal
-// CHECK: %[[C100:.*]] = llvm.mlir.constant(100 : index) : i64
-// CHECK: %[[MUL:.*]] = llvm.mul %{{.*}}, %[[C100]]  : i64
-// CHECK: %[[ADD:.*]] = llvm.add %[[MUL]], %{{.*}}  : i64
-// CHECK: %[[GEP:.*]] = llvm.getelementptr %{{.*}}[%[[ADD]]] : (!llvm.ptr, i64) -> !llvm.ptr, f32
-// CHECK: llvm.store %{{.*}}, %[[GEP]] {alignment = 4 : i64, nontemporal} :  vector<4xf32>, !llvm.ptr
-
-// -----
-
-func.func @store_nontemporal_scalable(%memref : memref<200x100xf32>, %i : index, %j : index) {
-  %val = arith.constant dense<11.0> : vector<[4]xf32>
-  vector.store %val, %memref[%i, %j] {nontemporal = true} : memref<200x100xf32>, vector<[4]xf32>
-  return
-}
-
-// CHECK-LABEL: func @store_nontemporal_scalable
-// CHECK: %[[C100:.*]] = llvm.mlir.constant(100 : index) : i64
-// CHECK: %[[MUL:.*]] = llvm.mul %{{.*}}, %[[C100]]  : i64
-// CHECK: %[[ADD:.*]] = llvm.add %[[MUL]], %{{.*}}  : i64
-// CHECK: %[[GEP:.*]] = llvm.getelementptr %{{.*}}[%[[ADD]]] : (!llvm.ptr, i64) -> !llvm.ptr, f32
-// CHECK: llvm.store %{{.*}}, %[[GEP]] {alignment = 4 : i64, nontemporal} :  vector<[4]xf32>, !llvm.ptr
-
-// -----
-
-func.func @store_index(%memref : memref<200x100xindex>, %i : index, %j : index) {
-  %val = arith.constant dense<11> : vector<4xindex>
-  vector.store %val, %memref[%i, %j] : memref<200x100xindex>, vector<4xindex>
-  return
-}
-// CHECK-LABEL: func @store_index
-// CHECK: llvm.store %{{.*}}, %{{.*}} {alignment = 8 : i64} : vector<4xi64>, !llvm.ptr
-
-// -----
-
-func.func @store_index_scalable(%memref : memref<200x100xindex>, %i : index, %j : index) {
-  %val = arith.constant dense<11> : vector<[4]xindex>
-  vector.store %val, %memref[%i, %j] : memref<200x100xindex>, vector<[4]xindex>
-  return
-}
-// CHECK-LABEL: func @store_index_scalable
-// CHECK: llvm.store %{{.*}}, %{{.*}} {alignment = 8 : i64} : vector<[4]xi64>, !llvm.ptr
-
-// -----
-
-func.func @store_0d(%memref : memref<200x100xf32>, %i : index, %j : index) {
-  %val = arith.constant dense<11.0> : vector<f32>
-  vector.store %val, %memref[%i, %j] : memref<200x100xf32>, vector<f32>
-  return
-}
-
-// CHECK-LABEL: func @store_0d
-// CHECK: %[[J:.*]] = builtin.unrealized_conversion_cast %{{.*}} : index to i64
-// CHECK: %[[I:.*]] = builtin.unrealized_conversion_cast %{{.*}} : index to i64
-// CHECK: %[[CAST_MEMREF:.*]] = builtin.unrealized_conversion_cast %{{.*}} : memref<200x100xf32> to !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)>
-// CHECK: %[[CST:.*]] = arith.constant dense<1.100000e+01> : vector<f32>
-// CHECK: %[[VAL:.*]] = builtin.unrealized_conversion_cast %[[CST]] : vector<f32> to vector<1xf32>
-// CHECK: %[[REF:.*]] = llvm.extractvalue %[[CAST_MEMREF]][1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> 
-// CHECK: %[[C100:.*]] = llvm.mlir.constant(100 : index) : i64
-// CHECK: %[[MUL:.*]] = llvm.mul %[[I]], %[[C100]] : i64
-// CHECK: %[[ADD:.*]] = llvm.add %[[MUL]], %[[J]] : i64
-// CHECK: %[[ADDR:.*]] = llvm.getelementptr %[[REF]][%[[ADD]]] : (!llvm.ptr, i64) -> !llvm.ptr, f32
-// CHECK: llvm.store %[[VAL]], %[[ADDR]] {alignment = 4 : i64} : vector<1xf32>, !llvm.ptr
-// CHECK: return
-
-// -----
-
-//===----------------------------------------------------------------------===//
-// vector.maskedload
-//===----------------------------------------------------------------------===//
-
-func.func @masked_load(%arg0: memref<?xf32>, %arg1: vector<16xi1>, %arg2: vector<16xf32>) -> vector<16xf32> {
-  %c0 = arith.constant 0: index
-  %0 = vector.maskedload %arg0[%c0], %arg1, %arg2 : memref<?xf32>, vector<16xi1>, vector<16xf32> into vector<16xf32>
-  return %0 : vector<16xf32>
-}
-
-// CHECK-LABEL: func @masked_load
-// CHECK: %[[CO:.*]] = arith.constant 0 : index
-// CHECK: %[[C:.*]] = builtin.unrealized_conversion_cast %[[CO]] : index to i64
-// CHECK: %[[P:.*]] = llvm.getelementptr %{{.*}}[%[[C]]] : (!llvm.ptr, i64) -> !llvm.ptr, f32
-// CHECK: %[[L:.*]] = llvm.intr.masked.load %[[P]], %{{.*}}, %{{.*}} {alignment = 4 : i32} : (!llvm.ptr, vector<16xi1>, vector<16xf32>) -> vector<16xf32>
-// CHECK: return %[[L]] : vector<16xf32>
-
-// -----
-
-func.func @masked_load_scalable(%arg0: memref<?xf32>, %arg1: vector<[16]xi1>, %arg2: vector<[16]xf32>) -> vector<[16]xf32> {
-  %c0 = arith.constant 0: index
-  %0 = vector.maskedload %arg0[%c0], %arg1, %arg2 : memref<?xf32>, vector<[16]xi1>, vector<[16]xf32> into vector<[16]xf32>
-  return %0 : vector<[16]xf32>
-}
-
-// CHECK-LABEL: func @masked_load_scalable
-// CHECK: %[[CO:.*]] = arith.constant 0 : index
-// CHECK: %[[C:.*]] = builtin.unrealized_conversion_cast %[[CO]] : index to i64
-// CHECK: %[[P:.*]] = llvm.getelementptr %{{.*}}[%[[C]]] : (!llvm.ptr, i64) -> !llvm.ptr, f32
-// CHECK: %[[L:.*]] = llvm.intr.masked.load %[[P]], %{{.*}}, %{{.*}} {alignment = 4 : i32} : (!llvm.ptr, vector<[16]xi1>, vector<[16]xf32>) -> vector<[16]xf32>
-// CHECK: return %[[L]] : vector<[16]xf32>
-
-// -----
-
-func.func @masked_load_index(%arg0: memref<?xindex>, %arg1: vector<16xi1>, %arg2: vector<16xindex>) -> vector<16xindex> {
-  %c0 = arith.constant 0: index
-  %0 = vector.maskedload %arg0[%c0], %arg1, %arg2 : memref<?xindex>, vector<16xi1>, vector<16xindex> into vector<16xindex>
-  return %0 : vector<16xindex>
-}
-// CHECK-LABEL: func @masked_load_index
-// CHECK: %{{.*}} = llvm.intr.masked.load %{{.*}}, %{{.*}}, %{{.*}} {alignment = 8 : i32} : (!llvm.ptr, vector<16xi1>, vector<16xi64>) -> vector<16xi64>
-
-// -----
-
-func.func @masked_load_index_scalable(%arg0: memref<?xindex>, %arg1: vector<[16]xi1>, %arg2: vector<[16]xindex>) -> vector<[16]xindex> {
-  %c0 = arith.constant 0: index
-  %0 = vector.maskedload %arg0[%c0], %arg1, %arg2 : memref<?xindex>, vector<[16]xi1>, vector<[16]xindex> into vector<[16]xindex>
-  return %0 : vector<[16]xindex>
-}
-// CHECK-LABEL: func @masked_load_index_scalable
-// CHECK: %{{.*}} = llvm.intr.masked.load %{{.*}}, %{{.*}}, %{{.*}} {alignment = 8 : i32} : (!llvm.ptr, vector<[16]xi1>, vector<[16]xi64>) -> vector<[16]xi64>
-
-// -----
-
-//===----------------------------------------------------------------------===//
-// vector.maskedstore
-//===----------------------------------------------------------------------===//
-
-func.func @masked_store(%arg0: memref<?xf32>, %arg1: vector<16xi1>, %arg2: vector<16xf32>) {
-  %c0 = arith.constant 0: index
-  vector.maskedstore %arg0[%c0], %arg1, %arg2 : memref<?xf32>, vector<16xi1>, vector<16xf32>
-  return
-}
-
-// CHECK-LABEL: func @masked_store
-// CHECK: %[[CO:.*]] = arith.constant 0 : index
-// CHECK: %[[C:.*]] = builtin.unrealized_conversion_cast %[[CO]] : index to i64
-// CHECK: %[[P:.*]] = llvm.getelementptr %{{.*}}[%[[C]]] : (!llvm.ptr, i64) -> !llvm.ptr, f32
-// CHECK: llvm.intr.masked.store %{{.*}}, %[[P]], %{{.*}} {alignment = 4 : i32} : vector<16xf32>, vector<16xi1> into !llvm.ptr
-
-// -----
-
-func.func @masked_store_scalable(%arg0: memref<?xf32>, %arg1: vector<[16]xi1>, %arg2: vector<[16]xf32>) {
-  %c0 = arith.constant 0: index
-  vector.maskedstore %arg0[%c0], %arg1, %arg2 : memref<?xf32>, vector<[16]xi1>, vector<[16]xf32>
-  return
-}
-
-// CHECK-LABEL: func @masked_store_scalable
-// CHECK: %[[CO:.*]] = arith.constant 0 : index
-// CHECK: %[[C:.*]] = builtin.unrealized_conversion_cast %[[CO]] : index to i64
-// CHECK: %[[P:.*]] = llvm.getelementptr %{{.*}}[%[[C]]] : (!llvm.ptr, i64) -> !llvm.ptr, f32
-// CHECK: llvm.intr.masked.store %{{.*}}, %[[P]], %{{.*}} {alignment = 4 : i32} : vector<[16]xf32>, vector<[16]xi1> into !llvm.ptr
-
-// -----
-
-func.func @masked_store_index(%arg0: memref<?xindex>, %arg1: vector<16xi1>, %arg2: vector<16xindex>) {
-  %c0 = arith.constant 0: index
-  vector.maskedstore %arg0[%c0], %arg1, %arg2 : memref<?xindex>, vector<16xi1>, vector<16xindex>
-  return
+func.func @constant_mask_2d_trailing_scalable() -> vector<4x[4]xi1> {
+  %0 = vector.constant_mask [2, 4] : vector<4x[4]xi1>
+  return %0 : vector<4x[4]xi1>
 }
-// CHECK-LABEL: func @masked_store_index
-// CHECK: llvm.intr.masked.store %{{.*}}, %{{.*}}, %{{.*}} {alignment = 8 : i32} : vector<16xi64>, vector<16xi1> into !llvm.ptr
+// CHECK-LABEL:   func.func @constant_mask_2d_trailing_scalable
+// CHECK:           %[[VAL_0:.*]] = arith.constant dense<true> : vector<[4]xi1>
+// CHECK:           %[[VAL_1:.*]] = arith.constant dense<false> : vector<4x[4]xi1>
+// CHECK:           %[[VAL_2:.*]] = builtin.unrealized_conversion_cast %[[VAL_1]] : vector<4x[4]xi1> to !llvm.array<4 x vector<[4]xi1>>
+// CHECK:           %[[VAL_3:.*]] = llvm.insertvalue %[[VAL_0]], %[[VAL_2]][0] : !llvm.array<4 x vector<[4]xi1>>
+// CHECK:           %[[VAL_4:.*]] = llvm.insertvalue %[[VAL_0]], %[[VAL_3]][1] : !llvm.array<4 x vector<[4]xi1>>
+// CHECK:           %[[VAL_5:.*]] = builtin.unrealized_conversion_cast %[[VAL_4]] : !llvm.array<4 x vector<[4]xi1>> to vector<4x[4]xi1>
+// CHECK:           return %[[VAL_5]] : vector<4x[4]xi1>
 
 // -----
 
-func.func @masked_store_index_scalable(%arg0: memref<?xindex>, %arg1: vector<[16]xi1>, %arg2: vector<[16]xindex>) {
-  %c0 = arith.constant 0: index
-  vector.maskedstore %arg0[%c0], %arg1, %arg2 : memref<?xindex>, vector<[16]xi1>, vector<[16]xindex>
-  return
+/// Currently, this is not supported as generating the mask would require
+/// unrolling the leading scalable dimension at compile time.
+func.func @negative_constant_mask_2d_leading_scalable() -> vector<[4]x4xi1> {
+  %0 = vector.constant_mask [4, 2] : vector<[4]x4xi1>
+  return %0 : vector<[4]x4xi1>
 }
-// CHECK-LABEL: func @masked_store_index_scalable
-// CHECK: llvm.intr.masked.store %{{.*}}, %{{.*}}, %{{.*}} {alignment = 8 : i32} : vector<[16]xi64>, vector<[16]xi1> into !llvm.ptr
+// CHECK-LABEL:   func.func @negative_constant_mask_2d_leading_scalable
+// CHECK:           %[[VAL_0:.*]] = vector.constant_mask [4, 2] : vector<[4]x4xi1>
+// CHECK:           return %[[VAL_0]] : vector<[4]x4xi1>
 
 // -----
 
 //===----------------------------------------------------------------------===//
-// vector.gather
+// vector.create_mask
 //===----------------------------------------------------------------------===//
 
-func.func @gather(%arg0: memref<?xf32>, %arg1: vector<3xi32>, %arg2: vector<3xi1>, %arg3: vector<3xf32>) -> vector<3xf32> {
-  %0 = arith.constant 0: index
-  %1 = vector.gather %arg0[%0][%arg1], %arg2, %arg3 : memref<?xf32>, vector<3xi32>, vector<3xi1>, vector<3xf32> into vector<3xf32>
-  return %1 : vector<3xf32>
-}
-
-// CHECK-LABEL: func @gather
-// CHECK: %[[P:.*]] = llvm.getelementptr %{{.*}}[%{{.*}}] : (!llvm.ptr, vector<3xi32>) -> !llvm.vec<3 x ptr>, f32
-// CHECK: %[[G:.*]] = llvm.intr.masked.gather %[[P]], %{{.*}}, %{{.*}} {alignment = 4 : i32} : (!llvm.vec<3 x ptr>, vector<3xi1>, vector<3xf32>) -> vector<3xf32>
-// CHECK: return %[[G]] : vector<3xf32>
-
-// -----
-
-func.func @gather_scalable(%arg0: memref<?xf32>, %arg1: vector<[3]xi32>, %arg2: vector<[3]xi1>, %arg3: vector<[3]xf32>) -> vector<[3]xf32> {
-  %0 = arith.constant 0: index
-  %1 = vector.gather %arg0[%0][%arg1], %arg2, %arg3 : memref<?xf32>, vector<[3]xi32>, vector<[3]xi1>, vector<[3]xf32> into vector<[3]xf32>
-  return %1 : vector<[3]xf32>
+func.func @create_mask_0d(%num_elems : index) -> vector<i1> {
+  %v = vector.create_mask %num_elems : vector<i1>
+  return %v: vector<i1>
 }
 
-// CHECK-LABEL: func @gather_scalable
-// CHECK: %[[P:.*]] = llvm.getelementptr %{{.*}}[%{{.*}}] : (!llvm.ptr, vector<[3]xi32>) -> !llvm.vec<? x 3 x ptr>, f32
-// CHECK: %[[G:.*]] = llvm.intr.masked.gather %[[P]], %{{.*}}, %{{.*}} {alignment = 4 : i32} : (!llvm.vec<? x 3 x ptr>, vector<[3]xi1>, vector<[3]xf32>) -> vector<[3]xf32>
-// CHECK: return %[[G]] : vector<[3]xf32>
+// CHECK-LABEL: func @create_mask_0d
+// CHECK-SAME: %[[NUM_ELEMS:.*]]: index
+// CHECK:  %[[INDICES:.*]] = arith.constant dense<0> : vector<i32>
+// CHECK:  %[[NUM_ELEMS_i32:.*]] = arith.index_cast %[[NUM_ELEMS]] : index to i32
+// CHECK:  %[[BOUNDS:.*]] = llvm.insertelement %[[NUM_ELEMS_i32]]
+// CHECK:  %[[BOUNDS_CAST:.*]] = builtin.unrealized_conversion_cast %[[BOUNDS]] : vector<1xi32> to vector<i32>
+// CHECK:  %[[RESULT:.*]] = arith.cmpi sgt, %[[BOUNDS_CAST]], %[[INDICES]] : vector<i32>
+// CHECK:  return %[[RESULT]] : vector<i1>
 
 // -----
 
-func.func @gather_global_memory(%arg0: memref<?xf32, 1>, %arg1: vector<3xi32>, %arg2: vector<3xi1>, %arg3: vector<3xf32>) -> vector<3xf32> {
-  %0 = arith.constant 0: index
-  %1 = vector.gather %arg0[%0][%arg1], %arg2, %arg3 : memref<?xf32, 1>, vector<3xi32>, vector<3xi1>, vector<3xf32> into vector<3xf32>
-  return %1 : vector<3xf32>
+func.func @create_mask_1d(%num_elems : index) -> vector<4xi1> {
+  %v = vector.create_mask %num_elems : vector<4xi1>
+  return %v: vector<4xi1>
 }
 
-// CHECK-LABEL: func @gather_global_memory
-// CHECK: %[[P:.*]] = llvm.getelementptr %{{.*}}[%{{.*}}] : (!llvm.ptr<1>, vector<3xi32>) -> !llvm.vec<3 x ptr<1>>, f32
-// CHECK: %[[G:.*]] = llvm.intr.masked.gather %[[P]], %{{.*}}, %{{.*}} {alignment = 4 : i32} : (!llvm.vec<3 x ptr<1>>, vector<3xi1>, vector<3xf32>) -> vector<3xf32>
-// CHECK: return %[[G]] : vector<3xf32>
+// CHECK-LABEL: func @create_mask_1d
+// CHECK-SAME: %[[NUM_ELEMS:.*]]: index
+// CHECK:  %[[INDICES:.*]] = arith.constant dense<[0, 1, 2, 3]> : vector<4xi32>
+// CHECK:  %[[NUM_ELEMS_i32:.*]] = arith.index_cast %[[NUM_ELEMS]] : index to i32
+// CHECK:  %[[BOUNDS_INSERT:.*]] = llvm.insertelement %[[NUM_ELEMS_i32]]
+// CHECK:  %[[BOUNDS:.*]] = llvm.shufflevector %[[BOUNDS_INSERT]]
+// CHECK:  %[[RESULT:.*]] = arith.cmpi sgt, %[[BOUNDS]], %[[INDICES]] : vector<4xi32>
+// CHECK:  return %[[RESULT]] : vector<4xi1>
 
 // -----
 
-func.func @gather_global_memory_scalable(%arg0: memref<?xf32, 1>, %arg1: vector<[3]xi32>, %arg2: vector<[3]xi1>, %arg3: vector<[3]xf32>) -> vector<[3]xf32> {
-  %0 = arith.constant 0: index
-  %1 = vector.gather %arg0[%0][%arg1], %arg2, %arg3 : memref<?xf32, 1>, vector<[3]xi32>, vector<[3]xi1>, vector<[3]xf32> into vector<[3]xf32>
-  return %1 : vector<[3]xf32>
+func.func @create_mask_1d_scalable(%num_elems : index) -> vector<[4]xi1> {
+  %v = vector.create_mask %num_elems : vector<[4]xi1>
+  return %v: vector<[4]xi1>
 }
 
-// CHECK-LABEL: func @gather_global_memory_scalable
-// CHECK: %[[P:.*]] = llvm.getelementptr %{{.*}}[%{{.*}}] : (!llvm.ptr<1>, vector<[3]xi32>) -> !llvm.vec<? x 3 x ptr<1>>, f32
-// CHECK: %[[G:.*]] = llvm.intr.masked.gather %[[P]], %{{.*}}, %{{.*}} {alignment = 4 : i32} : (!llvm.vec<? x 3 x ptr<1>>, vector<[3]xi1>, vector<[3]xf32>) -> vector<[3]xf32>
-// CHECK: return %[[G]] : vector<[3]xf32>
+// CHECK-LABEL: func @create_mask_1d_scalable
+// CHECK-SAME: %[[NUM_ELEMS:.*]]: index
+// CHECK:  %[[INDICES:.*]] = llvm.intr.stepvector : vector<[4]xi32>
+// CHECK:  %[[NUM_ELEMS_i32:.*]] = arith.index_cast %[[NUM_ELEMS]] : index to i32
+// CHECK:  %[[BOUNDS_INSERT:.*]] = llvm.insertelement %[[NUM_ELEMS_i32]], {{.*}} : vector<[4]xi32>
+// CHECK:  %[[BOUNDS:.*]] = llvm.shufflevector %[[BOUNDS_INSERT]], {{.*}} : vector<[4]xi32>
+// CHECK:  %[[RESULT:.*]] = arith.cmpi slt, %[[INDICES]], %[[BOUNDS]] : vector<[4]xi32>
+// CHECK: return %[[RESULT]] : vector<[4]xi1>
 
 // -----
 
+//===----------------------------------------------------------------------===//
+// vector.flat_transpose
+//===----------------------------------------------------------------------===//
 
-func.func @gather_index(%arg0: memref<?xindex>, %arg1: vector<3xindex>, %arg2: vector<3xi1>, %arg3: vector<3xindex>) -> vector<3xindex> {
-  %0 = arith.constant 0: index
-  %1 = vector.gather %arg0[%0][%arg1], %arg2, %arg3 : memref<?xindex>, vector<3xindex>, vector<3xi1>, vector<3xindex> into vector<3xindex>
-  return %1 : vector<3xindex>
+func.func @flat_transpose(%arg0: vector<16xf32>) -> vector<16xf32> {
+  %0 = vector.flat_transpose %arg0 { rows = 4: i32, columns = 4: i32 }
+     : vector<16xf32> -> vector<16xf32>
+  return %0 : vector<16xf32>
 }
 
-// CHECK-LABEL: func @gather_index
-// CHECK: %[[P:.*]] = llvm.getelementptr %{{.*}}[%{{.*}}] : (!llvm.ptr, vector<3xi64>) -> !llvm.vec<3 x ptr>, i64
-// CHECK: %[[G:.*]] = llvm.intr.masked.gather %{{.*}}, %{{.*}}, %{{.*}} {alignment = 8 : i32} : (!llvm.vec<3 x ptr>, vector<3xi1>, vector<3xi64>) -> vector<3xi64>
-// CHECK: %{{.*}} = builtin.unrealized_conversion_cast %[[G]] : vector<3xi64> to vector<3xindex>
+// CHECK-LABEL: func @flat_transpose
+// CHECK-SAME:  %[[A:.*]]: vector<16xf32>
+// CHECK:       %[[T:.*]] = llvm.intr.matrix.transpose %[[A]]
+// CHECK-SAME:      {columns = 4 : i32, rows = 4 : i32} :
+// CHECK-SAME:      vector<16xf32> into vector<16xf32>
+// CHECK:       return %[[T]] : vector<16xf32>
 
 // -----
 
-func.func @gather_index_scalable(%arg0: memref<?xindex>, %arg1: vector<[3]xindex>, %arg2: vector<[3]xi1>, %arg3: vector<[3]xindex>) -> vector<[3]xindex> {
-  %0 = arith.constant 0: index
-  %1 = vector.gather %arg0[%0][%arg1], %arg2, %arg3 : memref<?xindex>, vector<[3]xindex>, vector<[3]xi1>, vector<[3]xindex> into vector<[3]xindex>
-  return %1 : vector<[3]xindex>
+func.func @flat_transpose_index(%arg0: vector<16xindex>) -> vector<16xindex> {
+  %0 = vector.flat_transpose %arg0 { rows = 4: i32, columns = 4: i32 }
+     : vector<16xindex> -> vector<16xindex>
+  return %0 : vector<16xindex>
 }
-
-// CHECK-LABEL: func @gather_index_scalable
-// CHECK: %[[P:.*]] = llvm.getelementptr %{{.*}}[%{{.*}}] : (!llvm.ptr, vector<[3]xi64>) -> !llvm.vec<? x 3 x ptr>, i64
-// CHECK: %[[G:.*]] = llvm.intr.masked.gather %{{.*}}, %{{.*}}, %{{.*}} {alignment = 8 : i32} : (!llvm.vec<? x 3 x ptr>, vector<[3]xi1>, vector<[3]xi64>) -> vector<[3]xi64>
-// CHECK: %{{.*}} = builtin.unrealized_conversion_cast %[[G]] : vector<[3]xi64> to vector<[3]xindex>
+// CHECK-LABEL: func @flat_transpose_index
+// CHECK-SAME:  %[[A:.*]]: vector<16xindex>
+// CHECK:       %[[T0:.*]] = builtin.unrealized_conversion_cast %[[A]] : vector<16xindex> to vector<16xi64>
+// CHECK:       %[[T1:.*]] = llvm.intr.matrix.transpose %[[T0]]
+// CHECK-SAME:      {columns = 4 : i32, rows = 4 : i32} :
+// CHECK-SAME:      vector<16xi64> into vector<16xi64>
+// CHECK:       %[[T2:.*]] = builtin.unrealized_conversion_cast %[[T1]] : vector<16xi64> to vector<16xindex>
+// CHECK:       return %[[T2]] : vector<16xindex>
 
 // -----
 
-func.func @gather_2d_from_1d(%arg0: memref<?xf32>, %arg1: vector<2x3xi32>, %arg2: vector<2x3xi1>, %arg3: vector<2x3xf32>) -> vector<2x3xf32> {
-  %0 = arith.constant 0: index
-  %1 = vector.gather %arg0[%0][%arg1], %arg2, %arg3 : memref<?xf32>, vector<2x3xi32>, vector<2x3xi1>, vector<2x3xf32> into vector<2x3xf32>
-  return %1 : vector<2x3xf32>
+func.func @flat_transpose(%arg0: vector<16xf32>) -> vector<16xf32> {
+  %0 = vector.flat_transpose %arg0 { rows = 4: i32, columns = 4: i32 }
+     : vector<16xf32> -> vector<16xf32>
+  return %0 : vector<16xf32>
 }
 
-// CHECK-LABEL: func @gather_2d_from_1d
-// CHECK: %[[B:.*]] = llvm.getelementptr %{{.*}} : (!llvm.ptr, i64) -> !llvm.ptr, f32
-// CHECK: %[[I0:.*]] = llvm.extractvalue %{{.*}}[0] : !llvm.array<2 x vector<3xi32>>
-// CHECK: %[[M0:.*]] = llvm.extractvalue %{{.*}}[0] : !llvm.array<2 x vector<3xi1>>
-// CHECK: %[[S0:.*]] = llvm.extractvalue %{{.*}}[0] : !llvm.array<2 x vector<3xf32>>
-// CHECK: %[[P0:.*]] = llvm.getelementptr %[[B]][%[[I0]]] : (!llvm.ptr, vector<3xi32>) -> !llvm.vec<3 x ptr>, f32
-// CHECK: %[[G0:.*]] = llvm.intr.masked.gather %[[P0]], %[[M0]], %[[S0]] {alignment = 4 : i32} : (!llvm.vec<3 x ptr>, vector<3xi1>, vector<3xf32>) -> vector<3xf32>
-// CHECK: %{{.*}} = llvm.insertvalue %[[G0]], %{{.*}}[0] : !llvm.array<2 x vector<3xf32>>
-// CHECK: %[[I1:.*]] = llvm.extractvalue %{{.*}}[1] : !llvm.array<2 x vector<3xi32>>
-// CHECK: %[[M1:.*]] = llvm.extractvalue %{{.*}}[1] : !llvm.array<2 x vector<3xi1>>
-// CHECK: %[[S1:.*]] = llvm.extractvalue %{{.*}}[1] : !llvm.array<2 x vector<3xf32>>
-// CHECK: %[[P1:.*]] = llvm.getelementptr %[[B]][%[[I1]]] : (!llvm.ptr, vector<3xi32>) -> !llvm.vec<3 x ptr>, f32
-// CHECK: %[[G1:.*]] = llvm.intr.masked.gather %[[P1]], %[[M1]], %[[S1]] {alignment = 4 : i32} : (!llvm.vec<3 x ptr>, vector<3xi1>, vector<3xf32>) -> vector<3xf32>
-// CHECK: %{{.*}} = llvm.insertvalue %[[G1]], %{{.*}}[1] : !llvm.array<2 x vector<3xf32>>
+// CHECK-LABEL: func @flat_transpose
+// CHECK-SAME:  %[[A:.*]]: vector<16xf32>
+// CHECK:       %[[T:.*]] = llvm.intr.matrix.transpose %[[A]]
+// CHECK-SAME:      {columns = 4 : i32, rows = 4 : i32} :
+// CHECK-SAME:      vector<16xf32> into vector<16xf32>
+// CHECK:       return %[[T]] : vector<16xf32>
 
 // -----
 
-func.func @gather_2d_from_1d_scalable(%arg0: memref<?xf32>, %arg1: vector<2x[3]xi32>, %arg2: vector<2x[3]xi1>, %arg3: vector<2x[3]xf32>) -> vector<2x[3]xf32> {
-  %0 = arith.constant 0: index
-  %1 = vector.gather %arg0[%0][%arg1], %arg2, %arg3 : memref<?xf32>, vector<2x[3]xi32>, vector<2x[3]xi1>, vector<2x[3]xf32> into vector<2x[3]xf32>
-  return %1 : vector<2x[3]xf32>
-}
-
-// CHECK-LABEL: func @gather_2d_from_1d_scalable
-// CHECK: %[[B:.*]] = llvm.getelementptr %{{.*}} : (!llvm.ptr, i64) -> !llvm.ptr, f32
-// CHECK: %[[I0:.*]] = llvm.extractvalue %{{.*}}[0] : !llvm.array<2 x vector<[3]xi32>>
-// CHECK: %[[M0:.*]] = llvm.extractvalue %{{.*}}[0] : !llvm.array<2 x vector<[3]xi1>>
-// CHECK: %[[S0:.*]] = llvm.extractvalue %{{.*}}[0] : !llvm.array<2 x vector<[3]xf32>>
-// CHECK: %[[P0:.*]] = llvm.getelementptr %[[B]][%[[I0]]] : (!llvm.ptr, vector<[3]xi32>) -> !llvm.vec<? x 3 x ptr>, f32
-// CHECK: %[[G0:.*]] = llvm.intr.masked.gather %[[P0]], %[[M0]], %[[S0]] {alignment = 4 : i32} : (!llvm.vec<? x 3 x ptr>, vector<[3]xi1>, vector<[3]xf32>) -> vector<[3]xf32>
-// CHECK: %{{.*}} = llvm.insertvalue %[[G0]], %{{.*}}[0] : !llvm.array<2 x vector<[3]xf32>>
-// CHECK: %[[I1:.*]] = llvm.extractvalue %{{.*}}[1] : !llvm.array<2 x vector<[3]xi32>>
-// CHECK: %[[M1:.*]] = llvm.extractvalue %{{.*}}[1] : !llvm.array<2 x vector<[3]xi1>>
-// CHECK: %[[S1:.*]] = llvm.extractvalue %{{.*}}[1] : !llvm.array<2 x vector<[3]xf32>>
-// CHECK: %[[P1:.*]] = llvm.getelementptr %[[B]][%[[I1]]] : (!llvm.ptr, vector<[3]xi32>) -> !llvm.vec<? x 3 x ptr>, f32
-// CHECK: %[[G1:.*]] = llvm.intr.masked.gather %[[P1]], %[[M1]], %[[S1]] {alignment = 4 : i32} : (!llvm.vec<? x 3 x ptr>, vector<[3]xi1>, vector<[3]xf32>) -> vector<[3]xf32>
-// CHECK: %{{.*}} = llvm.insertvalue %[[G1]], %{{.*}}[1] : !llvm.array<2 x vector<[3]xf32>>
+//===----------------------------------------------------------------------===//
+// vector.gather
+//
+// NOTE: vector.constant_mask won't lower with
+//  * --convert-to-llvm="filter-dialects=vector",
+// hence testing here.
+//===----------------------------------------------------------------------===//
 
-// -----
 
 func.func @gather_with_mask(%arg0: memref<?xf32>, %arg1: vector<2x3xi32>, %arg2: vector<2x3xf32>) -> vector<2x3xf32> {
   %0 = arith.constant 0: index
@@ -3704,290 +1723,10 @@ func.func @gather_with_zero_mask_scalable(%arg0: memref<?xf32>, %arg1: vector<2x
 
 // -----
 
-func.func @gather_1d_from_2d(%arg0: memref<4x4xf32>, %arg1: vector<4xi32>, %arg2: vector<4xi1>, %arg3: vector<4xf32>) -> vector<4xf32> {
-  %0 = arith.constant 3 : index
-  %1 = vector.gather %arg0[%0, %0][%arg1], %arg2, %arg3 : memref<4x4xf32>, vector<4xi32>, vector<4xi1>, vector<4xf32> into vector<4xf32>
-  return %1 : vector<4xf32>
-}
-
-// CHECK-LABEL: func @gather_1d_from_2d
-// CHECK: %[[B:.*]] = llvm.getelementptr %{{.*}}[%{{.*}}] : (!llvm.ptr, i64) -> !llvm.ptr, f32
-// CHECK: %[[P:.*]] = llvm.getelementptr %[[B]][%{{.*}}] : (!llvm.ptr, vector<4xi32>) -> !llvm.vec<4 x ptr>, f32
-// CHECK: %[[G:.*]] = llvm.intr.masked.gather %[[P]], %{{.*}}, %{{.*}} {alignment = 4 : i32} : (!llvm.vec<4 x ptr>, vector<4xi1>, vector<4xf32>) -> vector<4xf32>
-// CHECK: return %[[G]] : vector<4xf32>
-
-// -----
-
-func.func @gather_1d_from_2d_scalable(%arg0: memref<4x?xf32>, %arg1: vector<[4]xi32>, %arg2: vector<[4]xi1>, %arg3: vector<[4]xf32>) -> vector<[4]xf32> {
-  %0 = arith.constant 3 : index
-  %1 = vector.gather %arg0[%0, %0][%arg1], %arg2, %arg3 : memref<4x?xf32>, vector<[4]xi32>, vector<[4]xi1>, vector<[4]xf32> into vector<[4]xf32>
-  return %1 : vector<[4]xf32>
-}
-
-// CHECK-LABEL: func @gather_1d_from_2d_scalable
-// CHECK: %[[B:.*]] = llvm.getelementptr %{{.*}}[%{{.*}}] : (!llvm.ptr, i64) -> !llvm.ptr, f32
-// CHECK: %[[P:.*]] = llvm.getelementptr %[[B]][%{{.*}}] : (!llvm.ptr, vector<[4]xi32>) -> !llvm.vec<? x 4 x ptr>, f32
-// CHECK: %[[G:.*]] = llvm.intr.masked.gather %[[P]], %{{.*}}, %{{.*}} {alignment = 4 : i32} : (!llvm.vec<? x 4 x ptr>, vector<[4]xi1>, vector<[4]xf32>) -> vector<[4]xf32>
-// CHECK: return %[[G]] : vector<[4]xf32>
-
-// -----
-
-//===----------------------------------------------------------------------===//
-// vector.scatter
-//===----------------------------------------------------------------------===//
-
-func.func @scatter(%arg0: memref<?xf32>, %arg1: vector<3xi32>, %arg2: vector<3xi1>, %arg3: vector<3xf32>) {
-  %0 = arith.constant 0: index
-  vector.scatter %arg0[%0][%arg1], %arg2, %arg3 : memref<?xf32>, vector<3xi32>, vector<3xi1>, vector<3xf32>
-  return
-}
-
-// CHECK-LABEL: func @scatter
-// CHECK: %[[P:.*]] = llvm.getelementptr %{{.*}}[%{{.*}}] : (!llvm.ptr, vector<3xi32>) -> !llvm.vec<3 x ptr>, f32
-// CHECK: llvm.intr.masked.scatter %{{.*}}, %[[P]], %{{.*}} {alignment = 4 : i32} : vector<3xf32>, vector<3xi1> into !llvm.vec<3 x ptr>
-
-// -----
-
-func.func @scatter_scalable(%arg0: memref<?xf32>, %arg1: vector<[3]xi32>, %arg2: vector<[3]xi1>, %arg3: vector<[3]xf32>) {
-  %0 = arith.constant 0: index
-  vector.scatter %arg0[%0][%arg1], %arg2, %arg3 : memref<?xf32>, vector<[3]xi32>, vector<[3]xi1>, vector<[3]xf32>
-  return
-}
-
-// CHECK-LABEL: func @scatter_scalable
-// CHECK: %[[P:.*]] = llvm.getelementptr %{{.*}}[%{{.*}}] : (!llvm.ptr, vector<[3]xi32>) -> !llvm.vec<? x 3 x ptr>, f32
-// CHECK: llvm.intr.masked.scatter %{{.*}}, %[[P]], %{{.*}} {alignment = 4 : i32} : vector<[3]xf32>, vector<[3]xi1> into !llvm.vec<? x 3 x ptr>
-
-// -----
-
-func.func @scatter_index(%arg0: memref<?xindex>, %arg1: vector<3xindex>, %arg2: vector<3xi1>, %arg3: vector<3xindex>) {
-  %0 = arith.constant 0: index
-  vector.scatter %arg0[%0][%arg1], %arg2, %arg3 : memref<?xindex>, vector<3xindex>, vector<3xi1>, vector<3xindex>
-  return
-}
-
-// CHECK-LABEL: func @scatter_index
-// CHECK: %[[P:.*]] = llvm.getelementptr %{{.*}}[%{{.*}}] : (!llvm.ptr, vector<3xi64>) -> !llvm.vec<3 x ptr>, i64
-// CHECK: llvm.intr.masked.scatter %{{.*}}, %[[P]], %{{.*}} {alignment = 8 : i32} : vector<3xi64>, vector<3xi1> into !llvm.vec<3 x ptr>
-
-// -----
-
-func.func @scatter_index_scalable(%arg0: memref<?xindex>, %arg1: vector<[3]xindex>, %arg2: vector<[3]xi1>, %arg3: vector<[3]xindex>) {
-  %0 = arith.constant 0: index
-  vector.scatter %arg0[%0][%arg1], %arg2, %arg3 : memref<?xindex>, vector<[3]xindex>, vector<[3]xi1>, vector<[3]xindex>
-  return
-}
-
-// CHECK-LABEL: func @scatter_index_scalable
-// CHECK: %[[P:.*]] = llvm.getelementptr %{{.*}}[%{{.*}}] : (!llvm.ptr, vector<[3]xi64>) -> !llvm.vec<? x 3 x ptr>, i64
-// CHECK: llvm.intr.masked.scatter %{{.*}}, %[[P]], %{{.*}} {alignment = 8 : i32} : vector<[3]xi64>, vector<[3]xi1> into !llvm.vec<? x 3 x ptr>
-
-// -----
-
-func.func @scatter_1d_into_2d(%arg0: memref<4x4xf32>, %arg1: vector<4xi32>, %arg2: vector<4xi1>, %arg3: vector<4xf32>) {
-  %0 = arith.constant 3 : index
-  vector.scatter %arg0[%0, %0][%arg1], %arg2, %arg3 : memref<4x4xf32>, vector<4xi32>, vector<4xi1>, vector<4xf32>
-  return
-}
-
-// CHECK-LABEL: func @scatter_1d_into_2d
-// CHECK: %[[B:.*]] = llvm.getelementptr %{{.*}}[%{{.*}}] : (!llvm.ptr, i64) -> !llvm.ptr, f32
-// CHECK: %[[P:.*]] = llvm.getelementptr %[[B]][%{{.*}}] : (!llvm.ptr, vector<4xi32>) -> !llvm.vec<4 x ptr>, f32
-// CHECK: llvm.intr.masked.scatter %{{.*}}, %[[P]], %{{.*}} {alignment = 4 : i32} : vector<4xf32>, vector<4xi1> into !llvm.vec<4 x ptr>
-
-// -----
-
-func.func @scatter_1d_into_2d_scalable(%arg0: memref<4x?xf32>, %arg1: vector<[4]xi32>, %arg2: vector<[4]xi1>, %arg3: vector<[4]xf32>) {
-  %0 = arith.constant 3 : index
-  vector.scatter %arg0[%0, %0][%arg1], %arg2, %arg3 : memref<4x?xf32>, vector<[4]xi32>, vector<[4]xi1>, vector<[4]xf32>
-  return
-}
-
-// CHECK-LABEL: func @scatter_1d_into_2d_scalable
-// CHECK: %[[B:.*]] = llvm.getelementptr %{{.*}}[%{{.*}}] : (!llvm.ptr, i64) -> !llvm.ptr, f32
-// CHECK: %[[P:.*]] = llvm.getelementptr %[[B]][%{{.*}}] : (!llvm.ptr, vector<[4]xi32>) -> !llvm.vec<? x 4 x ptr>, f32
-// CHECK: llvm.intr.masked.scatter %{{.*}}, %[[P]], %{{.*}} {alignment = 4 : i32} : vector<[4]xf32>, vector<[4]xi1> into !llvm.vec<? x 4 x ptr>
-
-// -----
-
-//===----------------------------------------------------------------------===//
-// vector.expandload
-//===----------------------------------------------------------------------===//
-
-func.func @expand_load_op(%arg0: memref<?xf32>, %arg1: vector<11xi1>, %arg2: vector<11xf32>) -> vector<11xf32> {
-  %c0 = arith.constant 0: index
-  %0 = vector.expandload %arg0[%c0], %arg1, %arg2 : memref<?xf32>, vector<11xi1>, vector<11xf32> into vector<11xf32>
-  return %0 : vector<11xf32>
-}
-
-// CHECK-LABEL: func @expand_load_op
-// CHECK: %[[CO:.*]] = arith.constant 0 : index
-// CHECK: %[[C:.*]] = builtin.unrealized_conversion_cast %[[CO]] : index to i64
-// CHECK: %[[P:.*]] = llvm.getelementptr %{{.*}}[%[[C]]] : (!llvm.ptr, i64) -> !llvm.ptr, f32
-// CHECK: %[[E:.*]] = "llvm.intr.masked.expandload"(%[[P]], %{{.*}}, %{{.*}}) : (!llvm.ptr, vector<11xi1>, vector<11xf32>) -> vector<11xf32>
-// CHECK: return %[[E]] : vector<11xf32>
-
-// -----
-
-func.func @expand_load_op_index(%arg0: memref<?xindex>, %arg1: vector<11xi1>, %arg2: vector<11xindex>) -> vector<11xindex> {
-  %c0 = arith.constant 0: index
-  %0 = vector.expandload %arg0[%c0], %arg1, %arg2 : memref<?xindex>, vector<11xi1>, vector<11xindex> into vector<11xindex>
-  return %0 : vector<11xindex>
-}
-// CHECK-LABEL: func @expand_load_op_index
-// CHECK: %{{.*}} = "llvm.intr.masked.expandload"(%{{.*}}, %{{.*}}, %{{.*}}) : (!llvm.ptr, vector<11xi1>, vector<11xi64>) -> vector<11xi64>
-
-// -----
-
-//===----------------------------------------------------------------------===//
-// vector.compressstore
-//===----------------------------------------------------------------------===//
-
-func.func @compress_store_op(%arg0: memref<?xf32>, %arg1: vector<11xi1>, %arg2: vector<11xf32>) {
-  %c0 = arith.constant 0: index
-  vector.compressstore %arg0[%c0], %arg1, %arg2 : memref<?xf32>, vector<11xi1>, vector<11xf32>
-  return
-}
-
-// CHECK-LABEL: func @compress_store_op
-// CHECK: %[[CO:.*]] = arith.constant 0 : index
-// CHECK: %[[C:.*]] = builtin.unrealized_conversion_cast %[[CO]] : index to i64
-// CHECK: %[[P:.*]] = llvm.getelementptr %{{.*}}[%[[C]]] : (!llvm.ptr, i64) -> !llvm.ptr, f32
-// CHECK: "llvm.intr.masked.compressstore"(%{{.*}}, %[[P]], %{{.*}}) : (vector<11xf32>, !llvm.ptr, vector<11xi1>) -> ()
-
-// -----
-
-func.func @compress_store_op_index(%arg0: memref<?xindex>, %arg1: vector<11xi1>, %arg2: vector<11xindex>) {
-  %c0 = arith.constant 0: index
-  vector.compressstore %arg0[%c0], %arg1, %arg2 : memref<?xindex>, vector<11xi1>, vector<11xindex>
-  return
-}
-// CHECK-LABEL: func @compress_store_op_index
-// CHECK: "llvm.intr.masked.compressstore"(%{{.*}}, %{{.*}}, %{{.*}}) : (vector<11xi64>, !llvm.ptr, vector<11xi1>) -> ()
-
-// -----
-
-//===----------------------------------------------------------------------===//
-// vector.splat
-//===----------------------------------------------------------------------===//
-
-// CHECK-LABEL: @splat_0d
-// CHECK-SAME: %[[ELT:.*]]: f32
-func.func @splat_0d(%elt: f32) -> vector<f32> {
-  %v = vector.splat %elt : vector<f32>
-  return %v : vector<f32>
-}
-// CHECK-NEXT: %[[UNDEF:[0-9]+]] = llvm.mlir.undef : vector<1xf32>
-// CHECK-NEXT: %[[ZERO:[0-9]+]] = llvm.mlir.constant(0 : i32) : i32
-// CHECK-NEXT: %[[V:[0-9]+]] = llvm.insertelement %[[ELT]], %[[UNDEF]][%[[ZERO]] : i32] : vector<1xf32>
-// CHECK-NEXT: %[[VCAST:[0-9]+]] = builtin.unrealized_conversion_cast %[[V]] : vector<1xf32> to vector<f32>
-// CHECK-NEXT: return %[[VCAST]] : vector<f32>
-
-// -----
-
-// CHECK-LABEL: @splat
-// CHECK-SAME: %[[VEC:[0-9a-zA-Z]+]]: vector<4xf32>
-// CHECK-SAME: %[[ELT:[0-9a-zA-Z]+]]: f32
-func.func @splat(%vec: vector<4xf32>, %elt: f32) -> vector<4xf32> {
-  %vb = vector.splat %elt : vector<4xf32>
-  %r = arith.mulf %vec, %vb : vector<4xf32>
-  return %r : vector<4xf32>
-}
-// CHECK-NEXT: %[[UNDEF:[0-9]+]] = llvm.mlir.undef : vector<4xf32>
-// CHECK-NEXT: %[[ZERO:[0-9]+]] = llvm.mlir.constant(0 : i32) : i32
-// CHECK-NEXT: %[[V:[0-9]+]] = llvm.insertelement %[[ELT]], %[[UNDEF]][%[[ZERO]] : i32] : vector<4xf32>
-// CHECK-NEXT: %[[SPLAT:[0-9]+]] = llvm.shufflevector %[[V]], %[[UNDEF]] [0, 0, 0, 0]
-// CHECK-NEXT: %[[SCALE:[0-9]+]] = arith.mulf %[[VEC]], %[[SPLAT]] : vector<4xf32>
-// CHECK-NEXT: return %[[SCALE]] : vector<4xf32>
-
-// -----
-
-// CHECK-LABEL: @splat_scalable
-// CHECK-SAME: %[[VEC:[0-9a-zA-Z]+]]: vector<[4]xf32>
-// CHECK-SAME: %[[ELT:[0-9a-zA-Z]+]]: f32
-func.func @splat_scalable(%vec: vector<[4]xf32>, %elt: f32) -> vector<[4]xf32> {
-  %vb = vector.splat %elt : vector<[4]xf32>
-  %r = arith.mulf %vec, %vb : vector<[4]xf32>
-  return %r : vector<[4]xf32>
-}
-// CHECK-NEXT: %[[UNDEF:[0-9]+]] = llvm.mlir.undef : vector<[4]xf32>
-// CHECK-NEXT: %[[ZERO:[0-9]+]] = llvm.mlir.constant(0 : i32) : i32
-// CHECK-NEXT: %[[V:[0-9]+]] = llvm.insertelement %[[ELT]], %[[UNDEF]][%[[ZERO]] : i32] : vector<[4]xf32>
-// CHECK-NEXT: %[[SPLAT:[0-9]+]] = llvm.shufflevector %[[V]], %[[UNDEF]] [0, 0, 0, 0]
-// CHECK-NEXT: %[[SCALE:[0-9]+]] = arith.mulf %[[VEC]], %[[SPLAT]] : vector<[4]xf32>
-// CHECK-NEXT: return %[[SCALE]] : vector<[4]xf32>
-
-// -----
-
-//===----------------------------------------------------------------------===//
-// vector.scalable_insert
-//===----------------------------------------------------------------------===//
-
-// CHECK-LABEL: @scalable_insert
-// CHECK-SAME: %[[SUB:.*]]: vector<4xf32>, %[[SV:.*]]: vector<[4]xf32>
-func.func @scalable_insert(%sub: vector<4xf32>, %dsv: vector<[4]xf32>) -> vector<[4]xf32> {
-  // CHECK-NEXT: %[[TMP:.*]] = llvm.intr.vector.insert %[[SUB]], %[[SV]][0] : vector<4xf32> into vector<[4]xf32>
-  %0 = vector.scalable.insert %sub, %dsv[0] : vector<4xf32> into vector<[4]xf32>
-  // CHECK-NEXT: llvm.intr.vector.insert %[[SUB]], %[[TMP]][4] : vector<4xf32> into vector<[4]xf32>
-  %1 = vector.scalable.insert %sub, %0[4] : vector<4xf32> into vector<[4]xf32>
-  return %1 : vector<[4]xf32>
-}
-
-// -----
-
-//===----------------------------------------------------------------------===//
-// vector.scalable_extract
-//===----------------------------------------------------------------------===//
-
-// CHECK-LABEL: @scalable_extract
-// CHECK-SAME: %[[VEC:.*]]: vector<[4]xf32>
-func.func @scalable_extract(%vec: vector<[4]xf32>) -> vector<8xf32> {
-  // CHECK-NEXT: %{{.*}} = llvm.intr.vector.extract %[[VEC]][0] : vector<8xf32> from vector<[4]xf32>
-  %0 = vector.scalable.extract %vec[0] : vector<8xf32> from vector<[4]xf32>
-  return %0 : vector<8xf32>
-}
-
-// -----
-
 //===----------------------------------------------------------------------===//
 // vector.interleave
 //===----------------------------------------------------------------------===//
 
-// CHECK-LABEL: @interleave_0d
-//  CHECK-SAME:     %[[LHS:.*]]: vector<i8>, %[[RHS:.*]]: vector<i8>)
-func.func @interleave_0d(%a: vector<i8>, %b: vector<i8>) -> vector<2xi8> {
-  // CHECK-DAG: %[[LHS_RANK1:.*]] = builtin.unrealized_conversion_cast %[[LHS]] : vector<i8> to vector<1xi8>
-  // CHECK-DAG: %[[RHS_RANK1:.*]] = builtin.unrealized_conversion_cast %[[RHS]] : vector<i8> to vector<1xi8>
-  // CHECK: %[[ZIP:.*]] = llvm.shufflevector %[[LHS_RANK1]], %[[RHS_RANK1]] [0, 1] : vector<1xi8>
-  // CHECK: return %[[ZIP]]
-  %0 = vector.interleave %a, %b : vector<i8> -> vector<2xi8>
-  return %0 : vector<2xi8>
-}
-
-// -----
-
-// CHECK-LABEL: @interleave_1d
-//  CHECK-SAME:     %[[LHS:.*]]: vector<8xf32>, %[[RHS:.*]]: vector<8xf32>)
-func.func @interleave_1d(%a: vector<8xf32>, %b: vector<8xf32>) -> vector<16xf32> {
-  // CHECK: %[[ZIP:.*]] = llvm.shufflevector %[[LHS]], %[[RHS]] [0, 8, 1, 9, 2, 10, 3, 11, 4, 12, 5, 13, 6, 14, 7, 15] : vector<8xf32>
-  // CHECK: return %[[ZIP]]
-  %0 = vector.interleave %a, %b : vector<8xf32> -> vector<16xf32>
-  return %0 : vector<16xf32>
-}
-
-// -----
-
-// CHECK-LABEL: @interleave_1d_scalable
-//  CHECK-SAME:     %[[LHS:.*]]: vector<[4]xi32>, %[[RHS:.*]]: vector<[4]xi32>)
-func.func @interleave_1d_scalable(%a: vector<[4]xi32>, %b: vector<[4]xi32>) -> vector<[8]xi32> {
-  // CHECK: %[[ZIP:.*]] = "llvm.intr.vector.interleave2"(%[[LHS]], %[[RHS]]) : (vector<[4]xi32>, vector<[4]xi32>) -> vector<[8]xi32>
-  // CHECK: return %[[ZIP]]
-  %0 = vector.interleave %a, %b : vector<[4]xi32> -> vector<[8]xi32>
-  return %0 : vector<[8]xi32>
-}
-
-// -----
-
 // CHECK-LABEL: @interleave_2d
 //  CHECK-SAME:     %[[LHS:.*]]: vector<2x3xi8>, %[[RHS:.*]]: vector<2x3xi8>)
 func.func @interleave_2d(%a: vector<2x3xi8>, %b: vector<2x3xi8>) -> vector<2x6xi8> {
@@ -4014,30 +1753,6 @@ func.func @interleave_2d_scalable(%a: vector<2x[8]xi16>, %b: vector<2x[8]xi16>)
 // vector.deinterleave
 //===----------------------------------------------------------------------===//
 
-// CHECK-LABEL: @deinterleave_1d
-// CHECK-SAME:  (%[[ARG:.*]]: vector<4xi32>) -> (vector<2xi32>, vector<2xi32>)
-func.func @deinterleave_1d(%arg: vector<4xi32>) -> (vector<2xi32>, vector<2xi32>) {
-  // CHECK: %[[POISON:.*]] = llvm.mlir.poison : vector<4xi32>
-  // CHECK: llvm.shufflevector %[[ARG]], %[[POISON]] [0, 2] : vector<4xi32>
-  // CHECK: llvm.shufflevector %[[ARG]], %[[POISON]] [1, 3] : vector<4xi32>
-  %0, %1 = vector.deinterleave %arg : vector<4xi32> -> vector<2xi32>
-  return %0, %1 : vector<2xi32>, vector<2xi32>
-}
-
-// -----
-
-// CHECK-LABEL: @deinterleave_1d_scalable
-// CHECK-SAME:  %[[ARG:.*]]: vector<[4]xi32>) -> (vector<[2]xi32>, vector<[2]xi32>)
-func.func @deinterleave_1d_scalable(%arg: vector<[4]xi32>) -> (vector<[2]xi32>, vector<[2]xi32>) {
-    // CHECK: %[[RES:.*]] = "llvm.intr.vector.deinterleave2"(%[[ARG]]) : (vector<[4]xi32>) -> !llvm.struct<(vector<[2]xi32>, vector<[2]xi32>)>
-    // CHECK: llvm.extractvalue %[[RES]][0] : !llvm.struct<(vector<[2]xi32>, vector<[2]xi32>)>
-    // CHECK: llvm.extractvalue %[[RES]][1] : !llvm.struct<(vector<[2]xi32>, vector<[2]xi32>)>
-    %0, %1 = vector.deinterleave %arg : vector<[4]xi32> -> vector<[2]xi32>
-    return %0, %1 : vector<[2]xi32>, vector<[2]xi32>
-}
-
-// -----
-
 // CHECK-LABEL: @deinterleave_2d
 // CHECK-SAME: %[[ARG:.*]]: vector<2x8xf32>) -> (vector<2x4xf32>, vector<2x4xf32>)
 func.func @deinterleave_2d(%arg: vector<2x8xf32>) -> (vector<2x4xf32>, vector<2x4xf32>) {
@@ -4056,40 +1771,6 @@ func.func @deinterleave_2d_scalable(%arg: vector<2x[8]xf32>) -> (vector<2x[4]xf3
     return %0, %1 : vector<2x[4]xf32>, vector<2x[4]xf32>
 }
 
-// -----
-
-//===----------------------------------------------------------------------===//
-// vector.from_elements
-//===----------------------------------------------------------------------===//
-
-// CHECK-LABEL: func.func @from_elements_1d(
-//  CHECK-SAME:     %[[ARG_0:.*]]: f32, %[[ARG_1:.*]]: f32)
-//       CHECK:   %[[UNDEF:.*]] = llvm.mlir.undef : vector<3xf32>
-//       CHECK:   %[[C0:.*]] = llvm.mlir.constant(0 : i64) : i64
-//       CHECK:   %[[INSERT0:.*]] = llvm.insertelement %[[ARG_0]], %[[UNDEF]][%[[C0]] : i64] : vector<3xf32>
-//       CHECK:   %[[C1:.*]] = llvm.mlir.constant(1 : i64) : i64
-//       CHECK:   %[[INSERT1:.*]] = llvm.insertelement %[[ARG_1]], %[[INSERT0]][%[[C1]] : i64] : vector<3xf32>
-//       CHECK:   %[[C2:.*]] = llvm.mlir.constant(2 : i64) : i64
-//       CHECK:   %[[INSERT2:.*]] = llvm.insertelement %[[ARG_0]], %[[INSERT1]][%[[C2]] : i64] : vector<3xf32>
-//       CHECK:   return %[[INSERT2]]
-func.func @from_elements_1d(%arg0: f32, %arg1: f32) -> vector<3xf32> {
-  %0 = vector.from_elements %arg0, %arg1, %arg0 : vector<3xf32>
-  return %0 : vector<3xf32>
-}
-
-// -----
-
-// CHECK-LABEL: func.func @from_elements_0d(
-//  CHECK-SAME:     %[[ARG_0:.*]]: f32)
-//       CHECK:   %[[UNDEF:.*]] = llvm.mlir.undef : vector<1xf32>
-//       CHECK:   %[[C0:.*]] = llvm.mlir.constant(0 : i64) : i64
-//       CHECK:   %[[INSERT0:.*]] = llvm.insertelement %[[ARG_0]], %[[UNDEF]][%[[C0]] : i64] : vector<1xf32>
-//       CHECK:   %[[CAST:.*]] = builtin.unrealized_conversion_cast %[[INSERT0]] : vector<1xf32> to vector<f32>
-//       CHECK:   return %[[CAST]]
-func.func @from_elements_0d(%arg0: f32) -> vector<f32> {
-  %0 = vector.from_elements %arg0 : vector<f32>
-  return %0 : vector<f32>
-}
 
 // -----
 
@@ -4097,6 +1778,8 @@ func.func @from_elements_0d(%arg0: f32) -> vector<f32> {
 // vector.step
 //===----------------------------------------------------------------------===//
 
+// TODO: Investigate why this wouldn't lower with --convert-to-llvm="filter-dialects=vector"
+
 // CHECK-LABEL: @step
 // CHECK: %[[CST:.+]] = arith.constant dense<[0, 1, 2, 3]> : vector<4xindex>
 // CHECK: return %[[CST]] : vector<4xindex>
@@ -4104,14 +1787,3 @@ func.func @step() -> vector<4xindex> {
   %0 = vector.step : vector<4xindex>
   return %0 : vector<4xindex>
 }
-
-// -----
-
-// CHECK-LABEL: @step_scalable
-// CHECK: %[[STEPVECTOR:.*]] = llvm.intr.stepvector : vector<[4]xi64>
-// CHECK: %[[CAST:.*]] = builtin.unrealized_conversion_cast %[[STEPVECTOR]] : vector<[4]xi64> to vector<[4]xindex>
-// CHECK: return %[[CAST]] : vector<[4]xindex>
-func.func @step_scalable() -> vector<[4]xindex> {
-  %0 = vector.step : vector<[4]xindex>
-  return %0 : vector<[4]xindex>
-}
diff --git a/mlir/test/Conversion/VectorToSPIRV/vector-to-spirv.mlir b/mlir/test/Conversion/VectorToSPIRV/vector-to-spirv.mlir
index fd73cea5e4f30..4701ac5d96009 100644
--- a/mlir/test/Conversion/VectorToSPIRV/vector-to-spirv.mlir
+++ b/mlir/test/Conversion/VectorToSPIRV/vector-to-spirv.mlir
@@ -175,6 +175,16 @@ func.func @extract(%arg0 : vector<2xf32>) -> (vector<1xf32>, f32) {
 
 // -----
 
+// CHECK-LABEL: @extract_poison_idx
+//       CHECK:   %[[R:.+]] = spirv.Undef : f32
+//       CHECK:   return %[[R]]
+func.func @extract_poison_idx(%arg0 : vector<4xf32>) -> f32 {
+  %0 = vector.extract %arg0[-1] : f32 from vector<4xf32>
+  return %0: f32
+}
+
+// -----
+
 // CHECK-LABEL: @extract_size1_vector
 //  CHECK-SAME: %[[ARG0:.+]]: vector<1xf32>
 //       CHECK:   %[[R:.+]] = builtin.unrealized_conversion_cast %[[ARG0]]
@@ -200,12 +210,31 @@ func.func @extract_size1_vector_dynamic(%arg0 : vector<1xf32>, %id : index) -> f
 // CHECK-LABEL: @extract_dynamic
 //  CHECK-SAME: %[[V:.*]]: vector<4xf32>, %[[ARG1:.*]]: index
 //       CHECK:   %[[ID:.+]] = builtin.unrealized_conversion_cast %[[ARG1]] : index to i32
-//       CHECK:   spirv.VectorExtractDynamic %[[V]][%[[ID]]] : vector<4xf32>, i32
+//       CHECK:   %[[MASK:.+]] = spirv.Constant 3 :
+//       CHECK:   %[[MASKED:.+]] = spirv.BitwiseAnd %[[ID]], %[[MASK]] :
+//       CHECK:   spirv.VectorExtractDynamic %[[V]][%[[MASKED]]] : vector<4xf32>, i32
 func.func @extract_dynamic(%arg0 : vector<4xf32>, %id : index) -> f32 {
   %0 = vector.extract %arg0[%id] : f32 from vector<4xf32>
   return %0: f32
 }
 
+// -----
+
+// CHECK-LABEL: @extract_dynamic_non_pow2
+//  CHECK-SAME: %[[V:.*]]: vector<3xf32>, %[[ARG1:.*]]: index
+//       CHECK:   %[[ID:.+]] = builtin.unrealized_conversion_cast %[[ARG1]] : index to i32
+//       CHECK:   %[[POISON:.+]] = spirv.Constant -1 :
+//       CHECK:   %[[CMP:.+]] = spirv.IEqual %[[ID]], %[[POISON]]
+//       CHECK:   %[[ZERO:.+]] = spirv.Constant 0 :
+//       CHECK:   %[[SELECT:.+]] = spirv.Select %[[CMP]], %[[ZERO]], %[[ID]] :
+//       CHECK:   spirv.VectorExtractDynamic %[[V]][%[[SELECT]]] : vector<3xf32>, i32
+func.func @extract_dynamic_non_pow2(%arg0 : vector<3xf32>, %id : index) -> f32 {
+  %0 = vector.extract %arg0[%id] : f32 from vector<3xf32>
+  return %0: f32
+}
+
+// -----
+
 // CHECK-LABEL: @extract_dynamic_cst
 //  CHECK-SAME: %[[V:.*]]: vector<4xf32>
 //       CHECK:   spirv.CompositeExtract %[[V]][1 : i32] : vector<4xf32>
@@ -256,6 +285,16 @@ func.func @insert(%arg0 : vector<4xf32>, %arg1: f32) -> vector<4xf32> {
 
 // -----
 
+// CHECK-LABEL: @insert_poison_idx
+//       CHECK:   %[[R:.+]] = spirv.Undef : vector<4xf32>
+//       CHECK:   return %[[R]]
+func.func @insert_poison_idx(%arg0 : vector<4xf32>, %arg1: f32) -> vector<4xf32> {
+  %1 = vector.insert %arg1, %arg0[-1] : f32 into vector<4xf32>
+  return %1: vector<4xf32>
+}
+
+// -----
+
 // CHECK-LABEL: @insert_index_vector
 //       CHECK:   spirv.CompositeInsert %{{.+}}, %{{.+}}[2 : i32] : i32 into vector<4xi32>
 func.func @insert_index_vector(%arg0 : vector<4xindex>, %arg1: index) -> vector<4xindex> {
@@ -290,7 +329,9 @@ func.func @insert_size1_vector_dynamic(%arg0 : vector<1xf32>, %arg1: f32, %id :
 // CHECK-LABEL: @insert_dynamic
 //  CHECK-SAME: %[[VAL:.*]]: f32, %[[V:.*]]: vector<4xf32>, %[[ARG2:.*]]: index
 //       CHECK: %[[ID:.+]] = builtin.unrealized_conversion_cast %[[ARG2]] : index to i32
-//       CHECK:   spirv.VectorInsertDynamic %[[VAL]], %[[V]][%[[ID]]] : vector<4xf32>, i32
+//       CHECK:   %[[MASK:.+]] = spirv.Constant 3 :
+//       CHECK:   %[[MASKED:.+]] = spirv.BitwiseAnd %[[ID]], %[[MASK]] :
+//       CHECK:   spirv.VectorInsertDynamic %[[VAL]], %[[V]][%[[MASKED]]] : vector<4xf32>, i32
 func.func @insert_dynamic(%val: f32, %arg0 : vector<4xf32>, %id : index) -> vector<4xf32> {
   %0 = vector.insert %val, %arg0[%id] : f32 into vector<4xf32>
   return %0: vector<4xf32>
@@ -298,6 +339,21 @@ func.func @insert_dynamic(%val: f32, %arg0 : vector<4xf32>, %id : index) -> vect
 
 // -----
 
+// CHECK-LABEL: @insert_dynamic_non_pow2
+//  CHECK-SAME: %[[VAL:.*]]: f32, %[[V:.*]]: vector<3xf32>, %[[ARG2:.*]]: index
+//       CHECK: %[[ID:.+]] = builtin.unrealized_conversion_cast %[[ARG2]] : index to i32
+//       CHECK:   %[[POISON:.+]] = spirv.Constant -1 :
+//       CHECK:   %[[CMP:.+]] = spirv.IEqual %[[ID]], %[[POISON]]
+//       CHECK:   %[[ZERO:.+]] = spirv.Constant 0 :
+//       CHECK:   %[[SELECT:.+]] = spirv.Select %[[CMP]], %[[ZERO]], %[[ID]] :
+//       CHECK:   spirv.VectorInsertDynamic %[[VAL]], %[[V]][%[[SELECT]]] : vector<3xf32>, i32
+func.func @insert_dynamic_non_pow2(%val: f32, %arg0 : vector<3xf32>, %id : index) -> vector<3xf32> {
+  %0 = vector.insert %val, %arg0[%id] : f32 into vector<3xf32>
+  return %0: vector<3xf32>
+}
+
+// -----
+
 // CHECK-LABEL: @insert_dynamic_cst
 //  CHECK-SAME: %[[VAL:.*]]: f32, %[[V:.*]]: vector<4xf32>
 //       CHECK:   spirv.CompositeInsert %[[VAL]], %[[V]][2 : i32] : f32 into vector<4xf32>
@@ -948,6 +1004,27 @@ func.func @vector_load(%arg0 : memref<4xf32, #spirv.storage_class<StorageBuffer>
   return %0: vector<4xf32>
 }
 
+
+// CHECK-LABEL: @vector_load_single_elem
+//  CHECK-SAME: (%[[ARG0:.*]]: memref<4xf32, #spirv.storage_class<StorageBuffer>>)
+//       CHECK:   %[[S0:.+]] = builtin.unrealized_conversion_cast %[[ARG0]] : memref<4xf32, #spirv.storage_class<StorageBuffer>> to !spirv.ptr<!spirv.struct<(!spirv.array<4 x f32, stride=4> [0])>, StorageBuffer>
+//       CHECK:   %[[C0:.+]] = arith.constant 0 : index
+//       CHECK:   %[[S1:.+]] = builtin.unrealized_conversion_cast %[[C0]] : index to i32
+//       CHECK:   %[[CST1:.+]] = spirv.Constant 0 : i32
+//       CHECK:   %[[CST2:.+]] = spirv.Constant 0 : i32
+//       CHECK:   %[[CST3:.+]] = spirv.Constant 1 : i32
+//       CHECK:   %[[S4:.+]] = spirv.AccessChain %[[S0]][%[[CST1]], %[[S1]]] : !spirv.ptr<!spirv.struct<(!spirv.array<4 x f32, stride=4> [0])>, StorageBuffer>, i32, i32
+//       CHECK:   %[[S5:.+]] = spirv.Load "StorageBuffer" %[[S4]] : f32
+//       CHECK:   %[[R0:.+]] = builtin.unrealized_conversion_cast %[[S5]] : f32 to vector<1xf32>
+//       CHECK:   return %[[R0]] : vector<1xf32>
+func.func @vector_load_single_elem(%arg0 : memref<4xf32, #spirv.storage_class<StorageBuffer>>) -> vector<1xf32> {
+  %idx = arith.constant 0 : index
+  %cst_0 = arith.constant 0.000000e+00 : f32
+  %0 = vector.load %arg0[%idx] : memref<4xf32, #spirv.storage_class<StorageBuffer>>, vector<1xf32>
+  return %0: vector<1xf32>
+}
+
+
 // CHECK-LABEL: @vector_load_2d
 //  CHECK-SAME: (%[[ARG0:.*]]: memref<4x4xf32, #spirv.storage_class<StorageBuffer>>) -> vector<4xf32> {
 //       CHECK:   %[[S0:.+]] = builtin.unrealized_conversion_cast %[[ARG0]] : memref<4x4xf32, #spirv.storage_class<StorageBuffer>> to !spirv.ptr<!spirv.struct<(!spirv.array<16 x f32, stride=4> [0])>, StorageBuffer>
@@ -990,6 +1067,24 @@ func.func @vector_store(%arg0 : memref<4xf32, #spirv.storage_class<StorageBuffer
   return
 }
 
+// CHECK-LABEL: @vector_store_single_elem
+//  CHECK-SAME: (%[[ARG0:.*]]: memref<4xf32, #spirv.storage_class<StorageBuffer>>
+//  CHECK-SAME:  %[[ARG1:.*]]: vector<1xf32>
+//       CHECK:  %[[S0:.+]] = builtin.unrealized_conversion_cast %[[ARG0]] : memref<4xf32, #spirv.storage_class<StorageBuffer>> to !spirv.ptr<!spirv.struct<(!spirv.array<4 x f32, stride=4> [0])>, StorageBuffer>
+//       CHECK:  %[[S1:.+]] = builtin.unrealized_conversion_cast %[[ARG1]] : vector<1xf32> to f32
+//       CHECK:  %[[C0:.+]] = arith.constant 0 : index
+//       CHECK:  %[[S2:.+]] = builtin.unrealized_conversion_cast %[[C0]] : index to i32
+//       CHECK:  %[[CST1:.+]] = spirv.Constant 0 : i32
+//       CHECK:  %[[CST2:.+]] = spirv.Constant 0 : i32
+//       CHECK:  %[[CST3:.+]] = spirv.Constant 1 : i32
+//       CHECK:  %[[S4:.+]] = spirv.AccessChain %[[S0]][%[[CST1]], %[[S2]]] : !spirv.ptr<!spirv.struct<(!spirv.array<4 x f32, stride=4> [0])>, StorageBuffer>, i32, i32 -> !spirv.ptr<f32, StorageBuffer>
+//       CHECK:  spirv.Store "StorageBuffer" %[[S4]], %[[S1]] : f32
+func.func @vector_store_single_elem(%arg0 : memref<4xf32, #spirv.storage_class<StorageBuffer>>, %arg1 : vector<1xf32>) {
+  %idx = arith.constant 0 : index
+  vector.store %arg1, %arg0[%idx] : memref<4xf32, #spirv.storage_class<StorageBuffer>>, vector<1xf32>
+  return
+}
+
 // CHECK-LABEL: @vector_store_2d
 //  CHECK-SAME: (%[[ARG0:.*]]: memref<4x4xf32, #spirv.storage_class<StorageBuffer>>
 //  CHECK-SAME:  %[[ARG1:.*]]: vector<4xf32>
diff --git a/mlir/test/Dialect/Affine/loop-fusion-3.mlir b/mlir/test/Dialect/Affine/loop-fusion-3.mlir
index 6bc4feadb8c98..d291f46a519c3 100644
--- a/mlir/test/Dialect/Affine/loop-fusion-3.mlir
+++ b/mlir/test/Dialect/Affine/loop-fusion-3.mlir
@@ -351,8 +351,157 @@ func.func @should_not_fuse_since_top_level_non_affine_mem_write_users(
 // CHECK:  affine.for
 // CHECK:    arith.addf
 
+// Tests that fusion isn't prevented by the presence of a dealloc op in
+// between since we can move the fused nest.
+
+// CHECK-LABEL: func @fuse_non_affine_intervening_op
+func.func @fuse_non_affine_intervening_op() {
+  %cst = arith.constant 0.0 : f32
+
+  %a = memref.alloc() : memref<100xf32>
+  %b = memref.alloc() : memref<100xf32>
+  %c = memref.alloc() : memref<100xf32>
+
+  affine.for %i = 0 to 100 {
+    affine.store %cst, %a[%i] : memref<100xf32>
+    affine.store %cst, %c[%i] : memref<100xf32>
+  }
+
+  // The source is fused into the destination while being moved here.
+  // CHECK:      affine.for %{{.*}} = 0 to 100
+  // CHECK-NEXT:   affine.store %cst
+  // CHECK-NEXT:   affine.store %cst{{.*}}
+  // CHECK-NEXT:   affine.load
+  // CHECK-NEXT:   affine.store
+  // CHECK-NEXT: }
+  // CHECK-NEXT: memref.dealloc
+
+  memref.dealloc %c : memref<100xf32>
+
+  affine.for %i = 0 to 100 {
+    %v = affine.load %a[%i] : memref<100xf32>
+    affine.store %v, %b[%i] : memref<100xf32>
+  }
+
+  return
+}
+
+// Tests that fusion happens in the presence of intervening non-affine reads.
+
+// CHECK-LABEL: func @fuse_non_affine_intervening_read
+func.func @fuse_non_affine_intervening_read() {
+  %cst = arith.constant 0.0 : f32
+
+  %a = memref.alloc() : memref<100xf32>
+  %b = memref.alloc() : memref<100xf32>
+  %c = memref.alloc() : memref<100xf32>
+
+  affine.for %i = 0 to 100 {
+    affine.store %cst, %a[%i] : memref<100xf32>
+  }
+
+  // The source is fused into the destination while being moved here.
+  // CHECK:      affine.for %{{.*}} = 0 to 100
+  // CHECK-NEXT:   affine.store %cst
+  // CHECK-NEXT:   affine.load
+  // CHECK-NEXT:   affine.store
+  // CHECK-NEXT: }
+
+  // CHECK:     affine.for %{{.*}} = 0 to 100
+  // CHECK-NEXT:  memref.load
+  affine.for %i = 0 to 100 {
+    memref.load %a[%i] : memref<100xf32>
+  }
+
+  affine.for %i = 0 to 100 {
+    %v = affine.load %a[%i] : memref<100xf32>
+    affine.store %v, %b[%i] : memref<100xf32>
+  }
+
+  return
+}
+
+// Tests that fusion happens in the presence of intervening non-affine region
+// ops.
+
+// CHECK-LABEL: func @fuse_non_affine_intervening_read_nest
+func.func @fuse_non_affine_intervening_read_nest() {
+  %cst = arith.constant 0.0 : f32
+  %c0 = arith.constant 0 : index
+  %c1 = arith.constant 1 : index
+  %c100 = arith.constant 100 : index
+
+  %a = memref.alloc() : memref<100xf32>
+  %b = memref.alloc() : memref<100xf32>
+  %c = memref.alloc() : memref<100xf32>
+
+  affine.for %i = 0 to 100 {
+    affine.store %cst, %a[%i] : memref<100xf32>
+  }
+
+  // The source is fused into the destination while being moved here.
+  // CHECK:      affine.for %{{.*}} = 0 to 100
+  // CHECK-NEXT:   affine.store %cst
+  // CHECK-NEXT:   affine.load
+  // CHECK-NEXT:   affine.store
+  // CHECK-NEXT: }
+
+  // CHECK: scf.for
+  // CHECK-NEXT:  memref.load
+  scf.for %i = %c0 to %c100 step %c1 {
+    memref.load %a[%i] : memref<100xf32>
+  }
+
+  affine.for %i = 0 to 100 {
+    %v = affine.load %a[%i] : memref<100xf32>
+    affine.store %v, %b[%i] : memref<100xf32>
+  }
+
+  return
+}
+
+// Tests that fusion does not happen when there are non-affine sources
+// intervening.
+
+// CHECK-LABEL: func @no_fusion_scf_for_store
+func.func @no_fusion_scf_for_store() {
+  %cst = arith.constant 0.0 : f32
+  %cst1 = arith.constant 1.0 : f32
+  %c0 = arith.constant 0 : index
+  %c1 = arith.constant 1 : index
+  %c100 = arith.constant 100 : index
+
+  %a = memref.alloc() : memref<100xf32>
+  %b = memref.alloc() : memref<100xf32>
+  %c = memref.alloc() : memref<100xf32>
+
+  // CHECK:      affine.for %{{.*}} = 0 to 100
+  // CHECK-NEXT:   affine.store
+  // CHECK-NEXT: }
+  affine.for %i = 0 to 100 {
+    affine.store %cst, %a[%i] : memref<100xf32>
+  }
+
+  // CHECK: scf.for
+  scf.for %i = %c0 to %c100 step %c1 {
+    memref.store %cst1, %a[%i] : memref<100xf32>
+  }
+
+  // CHECK:      affine.for %{{.*}} = 0 to 100
+  // CHECK-NEXT:   affine.load
+  // CHECK-NEXT:   affine.store
+  affine.for %i = 0 to 100 {
+    // Non-affine source for this load.
+    %v = affine.load %a[%i] : memref<100xf32>
+    affine.store %v, %b[%i] : memref<100xf32>
+  }
+
+  return
+}
+
 // -----
 
+// CHECK-LABEL: func @fuse_minor_affine_map
 // MAXIMAL-LABEL: func @fuse_minor_affine_map
 func.func @fuse_minor_affine_map(%in: memref<128xf32>, %out: memref<20x512xf32>) {
   %tmp = memref.alloc() : memref<128xf32>
@@ -418,7 +567,7 @@ func.func @should_fuse_multi_store_producer_and_privatize_memfefs() {
   return
 }
 
-
+// CHECK-LABEL: func @should_fuse_multi_store_producer_with_escaping_memrefs_and_remove_src
 func.func @should_fuse_multi_store_producer_with_escaping_memrefs_and_remove_src(
     %a : memref<10xf32>, %b : memref<10xf32>) {
   %cst = arith.constant 0.000000e+00 : f32
@@ -467,7 +616,7 @@ func.func @should_fuse_multi_store_producer_with_escaping_memrefs_and_preserve_s
     %0 = affine.load %b[%i2] : memref<10xf32>
   }
 
-	// Loops '%i0' and '%i2' should be fused first and '%i0' should be removed
+  // Loops '%i0' and '%i2' should be fused first and '%i0' should be removed
   // since fusion is maximal. Then the fused loop and '%i1' should be fused
   // and the fused loop shouldn't be removed since fusion is not maximal.
   // CHECK:       affine.for %{{.*}} = 0 to 10 {
@@ -517,6 +666,31 @@ func.func @should_not_fuse_due_to_dealloc(%arg0: memref<16xf32>){
 // CHECK-NEXT:      arith.addf
 // CHECK-NEXT:      affine.store
 
+// CHECK-LABEL: func @cannot_fuse_intervening_deallocs
+func.func @cannot_fuse_intervening_deallocs(%arg0: memref<16xf32>){
+  %A = memref.alloc() : memref<16xf32>
+  %C = memref.alloc() : memref<16xf32>
+  %cst_1 = arith.constant 1.000000e+00 : f32
+  // CHECK: affine.for %{{.*}} = 0 to 16
+  affine.for %arg1 = 0 to 16 {
+    %a = affine.load %arg0[%arg1] : memref<16xf32>
+    affine.store %a, %A[%arg1] : memref<16xf32>
+    affine.store %a, %C[%arg1] : memref<16xf32>
+  }
+  // The presence of B's alloc prevents placement of the fused nest above C's
+  // dealloc. No fusion here.
+  memref.dealloc %C : memref<16xf32>
+  %B = memref.alloc() : memref<16xf32>
+  // CHECK: affine.for %{{.*}} = 0 to 16
+  affine.for %arg1 = 0 to 16 {
+    %a = affine.load %A[%arg1] : memref<16xf32>
+    %b = arith.addf %cst_1, %a : f32
+    affine.store %b, %B[%arg1] : memref<16xf32>
+  }
+  memref.dealloc %A : memref<16xf32>
+  return
+}
+
 // -----
 
 // CHECK-LABEL: func @should_fuse_defining_node_has_no_dependence_from_source_node
diff --git a/mlir/test/Dialect/Affine/loop-fusion-4.mlir b/mlir/test/Dialect/Affine/loop-fusion-4.mlir
index ea144f73bb21c..2830235431c76 100644
--- a/mlir/test/Dialect/Affine/loop-fusion-4.mlir
+++ b/mlir/test/Dialect/Affine/loop-fusion-4.mlir
@@ -285,3 +285,63 @@ module {
     spirv.ReturnValue %3 : !spirv.array<8192 x f32>
   }
 }
+
+// -----
+
+// PRODUCER-CONSUMER-LABEL: func @same_memref_load_store
+func.func @same_memref_load_store(%producer : memref<32xf32>, %consumer: memref<16xf32>){
+  %cst = arith.constant 2.000000e+00 : f32
+  // Source isn't removed.
+  // PRODUCER-CONSUMER: affine.for %{{.*}} = 0 to 32
+  affine.for %arg3 = 0 to 32 {
+    %0 = affine.load %producer[%arg3] : memref<32xf32>
+    %2 = arith.mulf %0, %cst : f32
+    affine.store %2, %producer[%arg3] : memref<32xf32>
+  }
+  affine.for %arg3 = 0 to 16 {
+    %0 = affine.load %producer[%arg3] : memref<32xf32>
+    %2 = arith.addf %0, %cst : f32
+    affine.store %2, %consumer[%arg3] : memref<16xf32>
+  }
+  // Fused nest.
+  // PRODUCER-CONSUMER:      affine.for %{{.*}} = 0 to 16
+  // PRODUCER-CONSUMER-NEXT:   affine.load %{{.*}}[%{{.*}}] : memref<32xf32>
+  // PRODUCER-CONSUMER-NEXT:   arith.mulf
+  // PRODUCER-CONSUMER-NEXT:   affine.store %{{.*}}, %{{.*}}[0] : memref<1xf32>
+  // PRODUCER-CONSUMER-NEXT:   affine.load %{{.*}}[0] : memref<1xf32>
+  // PRODUCER-CONSUMER-NEXT:   arith.addf
+  // PRODUCER-CONSUMER-NEXT:   affine.store
+  // PRODUCER-CONSUMER-NEXT: }
+  return
+}
+
+// PRODUCER-CONSUMER-LABEL: func @same_memref_load_multiple_stores
+func.func @same_memref_load_multiple_stores(%producer : memref<32xf32>, %producer_2 : memref<32xf32>, %consumer: memref<16xf32>){
+  %cst = arith.constant 2.000000e+00 : f32
+  // Source isn't removed.
+  // PRODUCER-CONSUMER: affine.for %{{.*}} = 0 to 32
+  affine.for %arg3 = 0 to 32 {
+    %0 = affine.load %producer[%arg3] : memref<32xf32>
+    %2 = arith.mulf %0, %cst : f32
+    affine.store %2, %producer[%arg3] : memref<32xf32>
+    affine.store %2, %producer_2[%arg3] : memref<32xf32>
+  }
+  affine.for %arg3 = 0 to 16 {
+    %0 = affine.load %producer[%arg3] : memref<32xf32>
+    %1 = affine.load %producer_2[%arg3] : memref<32xf32>
+    %2 = arith.addf %0, %1 : f32
+    affine.store %2, %consumer[%arg3] : memref<16xf32>
+  }
+  // Fused nest.
+  // PRODUCER-CONSUMER:      affine.for %{{.*}} = 0 to 16
+  // PRODUCER-CONSUMER-NEXT:   affine.load %{{.*}}[%{{.*}}] : memref<32xf32>
+  // PRODUCER-CONSUMER-NEXT:   arith.mulf
+  // PRODUCER-CONSUMER-NEXT:   affine.store %{{.*}}, %{{.*}}[0] : memref<1xf32>
+  // PRODUCER-CONSUMER-NEXT:   affine.store %{{.*}}, %{{.*}}[0] : memref<1xf32>
+  // PRODUCER-CONSUMER-NEXT:   affine.load %{{.*}}[0] : memref<1xf32>
+  // PRODUCER-CONSUMER-NEXT:   affine.load %{{.*}}[0] : memref<1xf32>
+  // PRODUCER-CONSUMER-NEXT:   arith.addf
+  // PRODUCER-CONSUMER-NEXT:   affine.store
+  // PRODUCER-CONSUMER-NEXT: }
+  return
+}
diff --git a/mlir/test/Dialect/Affine/loop-fusion-inner.mlir b/mlir/test/Dialect/Affine/loop-fusion-inner.mlir
index 61af9a4baf46d..e76441f2c3237 100644
--- a/mlir/test/Dialect/Affine/loop-fusion-inner.mlir
+++ b/mlir/test/Dialect/Affine/loop-fusion-inner.mlir
@@ -90,13 +90,13 @@ func.func @fusion_inner_multiple_nests() {
   // CHECK:      affine.for %{{.*}} = 0 to 4 {
   // Everything inside fused into two nests (the second will be DCE'd).
   // CHECK-NEXT:   memref.alloc() : memref<4xi8>
-  // CHECK-NEXT:   memref.alloc() : memref<1xi8>
-  // CHECK-NEXT:   memref.alloc() : memref<1xi8>
   // CHECK-NEXT:   memref.alloc() : memref<8x4xi8>
   // CHECK-NEXT:   memref.alloc() : memref<4xi8>
-  // CHECK-NEXT:   affine.for %{{.*}} = 0 to 2 {
+  // CHECK-NEXT:   affine.for %{{.*}} = 0 to 4 {
   // CHECK:        }
-  // CHECK:        affine.for %{{.*}} = 0 to 4 {
+  // CHECK-NEXT:   affine.for %{{.*}} = 0 to 2 {
+  // CHECK:          arith.muli
+  // CHECK-NEXT:     arith.extsi
   // CHECK:        }
   // CHECK-NEXT:   memref.dealloc
   // CHECK-NEXT: }
diff --git a/mlir/test/Dialect/Affine/loop-fusion.mlir b/mlir/test/Dialect/Affine/loop-fusion.mlir
index 045b1bec272e1..1c119e87c5336 100644
--- a/mlir/test/Dialect/Affine/loop-fusion.mlir
+++ b/mlir/test/Dialect/Affine/loop-fusion.mlir
@@ -448,8 +448,8 @@ func.func @should_fuse_no_top_level_access() {
 
 #set0 = affine_set<(d0) : (1 == 0)>
 
-// CHECK-LABEL: func @should_not_fuse_if_op_at_top_level() {
-func.func @should_not_fuse_if_op_at_top_level() {
+// CHECK-LABEL: func @should_fuse_despite_affine_if() {
+func.func @should_fuse_despite_affine_if() {
   %m = memref.alloc() : memref<10xf32>
   %cf7 = arith.constant 7.0 : f32
 
@@ -462,12 +462,10 @@ func.func @should_not_fuse_if_op_at_top_level() {
   %c0 = arith.constant 4 : index
   affine.if #set0(%c0) {
   }
-  // Top-level IfOp should prevent fusion.
+  // An unrelated affine.if op doesn't prevent fusion.
   // CHECK:      affine.for %{{.*}} = 0 to 10 {
-  // CHECK-NEXT:   affine.store %{{.*}}, %{{.*}}[%{{.*}}] : memref<10xf32>
-  // CHECK-NEXT: }
-  // CHECK:      affine.for %{{.*}} = 0 to 10 {
-  // CHECK-NEXT:   affine.load %{{.*}}[%{{.*}}] : memref<10xf32>
+  // CHECK-NEXT:   affine.store %{{.*}}, %{{.*}}[0] : memref<1xf32>
+  // CHECK-NEXT:   affine.load %{{.*}}[0] : memref<1xf32>
   // CHECK-NEXT: }
   return
 }
diff --git a/mlir/test/Dialect/Affine/ops.mlir b/mlir/test/Dialect/Affine/ops.mlir
index e3721806989bb..233c18c82831c 100644
--- a/mlir/test/Dialect/Affine/ops.mlir
+++ b/mlir/test/Dialect/Affine/ops.mlir
@@ -301,29 +301,29 @@ func.func @linearize_mixed(%index0: index, %index1: index, %index2: index, %basi
 
 // -----
 
-#map = affine_map<()[s0] -> (s0)>
+// CHECK-LABEL: @gpu_launch_affine
 
-// CHECK-LABEL: @gpu_affine_for
+// Test `thread_id` in AffineScope, the `thread_id` is in AffineScope's toplevel,
+// it is a valid symbol.
 
-module attributes {gpu.container_module} {
-  gpu.module @gpu {
-    gpu.func @gpu_affine_for(%arg0: memref<?x?xf32>) kernel {
-      %c3 = arith.constant 1 : index
-      %dim = memref.dim %arg0, %c3 : memref<?x?xf32>
-      %c0 = arith.constant 0 : index
-      affine.for %arg3 = %c0 to #map()[%dim] step 32 {
+module {
+  func.func @gpu_launch_affine() {
+    %c1 = arith.constant 1 : index
+    gpu.launch blocks(%arg0, %arg1, %arg2) in (%arg6 = %c1, %arg7 = %c1, %arg8 = %c1) 
+    threads(%arg3, %arg4, %arg5) in (%arg9 = %c1, %arg10 = %c1, %arg11 = %c1) {
+      %thread_id_x = gpu.thread_id  x
+      %c128 = arith.constant 128 : index
+      affine.for %arg12 = %thread_id_x to %c128 step 8 {
       }
-      gpu.return
+      gpu.terminator
     }
+    return
   }
 }
-// CHECK-SAME:        (%[[VAL_0:.*]]: memref<?x?xf32>) kernel {
-// CHECK:             %[[VAL_1:.*]] = arith.constant 1 : index
-// CHECK:             %[[VAL_2:.*]] = memref.dim %[[VAL_0]], %[[VAL_1]] : memref<?x?xf32>
-// CHECK:             %[[VAL_3:.*]] = arith.constant 0 : index
-// CHECK:             affine.for %[[VAL_4:.*]] = %[[VAL_3]] to %[[VAL_2]] step 32 {
-// CHECK:             }
-// CHECK:             gpu.return
+
+// CHECK: %[[THREAD_ID:.*]] = gpu.thread_id  x
+// CHECK: %[[VAL:.*]] = arith.constant 128 : index
+// CHECK: affine.for %{{.*}} = %[[THREAD_ID]] to %[[VAL]] step 8 {
 
 // -----
 
diff --git a/mlir/test/Dialect/Arith/canonicalize.mlir b/mlir/test/Dialect/Arith/canonicalize.mlir
index 522711b08f289..e3750bb020cad 100644
--- a/mlir/test/Dialect/Arith/canonicalize.mlir
+++ b/mlir/test/Dialect/Arith/canonicalize.mlir
@@ -160,6 +160,78 @@ func.func @selNotCond(%arg0: i1, %arg1 : i32, %arg2 : i32, %arg3 : i32, %arg4 :
   return %res1, %res2 : i32, i32
 }
 
+// CHECK-LABEL: @cmpiI1eq
+//  CHECK-SAME: (%[[ARG:.*]]: i1)
+//       CHECK: return %[[ARG]]
+func.func @cmpiI1eq(%arg0: i1) -> i1 {
+  %one = arith.constant 1 : i1
+  %res = arith.cmpi eq, %arg0, %one : i1
+  return %res : i1
+}
+
+// CHECK-LABEL: @cmpiI1eqVec
+//  CHECK-SAME: (%[[ARG:.*]]: vector<4xi1>)
+//       CHECK: return %[[ARG]]
+func.func @cmpiI1eqVec(%arg0: vector<4xi1>) -> vector<4xi1> {
+  %one = arith.constant dense<1> : vector<4xi1>
+  %res = arith.cmpi eq, %arg0, %one : vector<4xi1>
+  return %res : vector<4xi1>
+}
+
+// CHECK-LABEL: @cmpiI1ne
+//  CHECK-SAME: (%[[ARG:.*]]: i1)
+//       CHECK: return %[[ARG]]
+func.func @cmpiI1ne(%arg0: i1) -> i1 {
+  %zero = arith.constant 0 : i1
+  %res = arith.cmpi ne, %arg0, %zero : i1
+  return %res : i1
+}
+
+// CHECK-LABEL: @cmpiI1neVec
+//  CHECK-SAME: (%[[ARG:.*]]: vector<4xi1>)
+//       CHECK: return %[[ARG]]
+func.func @cmpiI1neVec(%arg0: vector<4xi1>) -> vector<4xi1> {
+  %zero = arith.constant dense<0> : vector<4xi1>
+  %res = arith.cmpi ne, %arg0, %zero : vector<4xi1>
+  return %res : vector<4xi1>
+}
+
+// CHECK-LABEL: @cmpiI1eqLhs
+//  CHECK-SAME: (%[[ARG:.*]]: i1)
+//       CHECK: return %[[ARG]]
+func.func @cmpiI1eqLhs(%arg0: i1) -> i1 {
+  %one = arith.constant 1 : i1
+  %res = arith.cmpi eq, %one, %arg0  : i1
+  return %res : i1
+}
+
+// CHECK-LABEL: @cmpiI1eqVecLhs
+//  CHECK-SAME: (%[[ARG:.*]]: vector<4xi1>)
+//       CHECK: return %[[ARG]]
+func.func @cmpiI1eqVecLhs(%arg0: vector<4xi1>) -> vector<4xi1> {
+  %one = arith.constant dense<1> : vector<4xi1>
+  %res = arith.cmpi eq, %one, %arg0 : vector<4xi1>
+  return %res : vector<4xi1>
+}
+
+// CHECK-LABEL: @cmpiI1neLhs
+//  CHECK-SAME: (%[[ARG:.*]]: i1)
+//       CHECK: return %[[ARG]]
+func.func @cmpiI1neLhs(%arg0: i1) -> i1 {
+  %zero = arith.constant 0 : i1
+  %res = arith.cmpi ne, %zero, %arg0 : i1
+  return %res : i1
+}
+
+// CHECK-LABEL: @cmpiI1neVecLhs
+//  CHECK-SAME: (%[[ARG:.*]]: vector<4xi1>)
+//       CHECK: return %[[ARG]]
+func.func @cmpiI1neVecLhs(%arg0: vector<4xi1>) -> vector<4xi1> {
+  %zero = arith.constant dense<0> : vector<4xi1>
+  %res = arith.cmpi ne, %zero, %arg0 : vector<4xi1>
+  return %res : vector<4xi1>
+}
+
 // Test case: Folding of comparisons with equal operands.
 // CHECK-LABEL: @cmpi_equal_operands
 //   CHECK-DAG:   %[[T:.*]] = arith.constant true
@@ -1771,6 +1843,28 @@ func.func @bitcastOfBitcast(%arg : i16) -> i16 {
 
 // -----
 
+// CHECK-LABEL: @bitcastPoisonItoFP(
+func.func @bitcastPoisonItoFP() -> f32 {
+  // CHECK: %[[P:.+]] = ub.poison : f32
+  // CHECK: return %[[P]] : f32
+  %p = ub.poison : i32
+  %res = arith.bitcast %p : i32 to f32
+  return %res : f32
+}
+
+// -----
+
+// CHECK-LABEL: @bitcastPoisonFPtoI(
+func.func @bitcastPoisonFPtoI() -> i32 {
+  // CHECK: %[[P:.+]] = ub.poison : i32
+  // CHECK: return %[[P]] : i32
+  %p = ub.poison : f32
+  %res = arith.bitcast %p : f32 to i32
+  return %res : i32
+}
+
+// -----
+
 // CHECK-LABEL: test_maxsi
 // CHECK-DAG: %[[C0:.+]] = arith.constant 42
 // CHECK-DAG: %[[MAX_INT_CST:.+]] = arith.constant 127
diff --git a/mlir/test/Dialect/Arith/emulate-wide-int-unsupported.mlir b/mlir/test/Dialect/Arith/emulate-wide-int-unsupported.mlir
index 091dd15dcd851..44f09b920ed4f 100644
--- a/mlir/test/Dialect/Arith/emulate-wide-int-unsupported.mlir
+++ b/mlir/test/Dialect/Arith/emulate-wide-int-unsupported.mlir
@@ -41,3 +41,13 @@ func.func @unsupported_argument_type(%arg0: vector<4xi128>) -> vector<4xi64> {
   return %0 : vector<4xi64>
 }
 
+// -----
+
+// Ensure this case not crash
+func.func @unsupported_vector(%arg0: vector<2xi1>) {
+  // expected-error@+1 {{failed to legalize unresolved materialization from ('vector<2x2xi32>') to ('vector<2xi64>') that remained live after conversion}}
+  %cst_0 = arith.constant dense<0> : vector<2xi64>
+  // expected-note@+1 {{see existing live user here}}
+  %0 = vector.mask %arg0 { vector.multi_reduction <xor>, %cst_0, %cst_0 [] : vector<2xi64> to vector<2xi64> } : vector<2xi1> -> vector<2xi64>
+  return
+}
diff --git a/mlir/test/Dialect/Bufferization/canonicalize.mlir b/mlir/test/Dialect/Bufferization/canonicalize.mlir
index 3ebc1e4fa8dea..b662e713e189c 100644
--- a/mlir/test/Dialect/Bufferization/canonicalize.mlir
+++ b/mlir/test/Dialect/Bufferization/canonicalize.mlir
@@ -28,17 +28,20 @@ func.func @buffer_cast_of_tensor_load(%arg0: memref<?xf32>) -> memref<?xf32> {
 // -----
 
 // If the memrefs are not the same type, don't fold them.
-// If the memrefs are not cast-compatible (e.g. different address space), don't
-// canonicalize them either.
-// CHECK-LABEL: func @no_fold_buffer_cast_of_tensor_load(
+// If the memrefs are not cast-compatible but one can be copied into the other
+// (e.g. different address space), canonicalize them to add + copy.
+// CHECK-LABEL: func @canonicalize_buffer_cast_of_tensor_load_different_address_space(
 //  CHECK-SAME:   %[[MEMREF_ADDRSPACE2:.*]]: memref<?xf32, 2>)
 //  CHECK-SAME:     -> memref<?xf32, 7> {
-//       CHECK: %[[TENSOR:.*]] = bufferization.to_tensor
-//  CHECK-SAME:   %[[MEMREF_ADDRSPACE2]] : memref<?xf32, 2> to tensor<?xf32, 7 : i64>
-//       CHECK: %[[MEMREF_ADDRSPACE7:.*]] = bufferization.to_memref
-//  CHECK-SAME:   %[[TENSOR]] : tensor<?xf32, 7 : i64> to memref<?xf32, 7>
-//       CHECK: return %[[MEMREF_ADDRSPACE7]]
-func.func @no_fold_buffer_cast_of_tensor_load(%arg0: memref<?xf32, 2>)
+//  CHECK-NOT: bufferization.to_tensor
+//  CHECK-NOT: bufferization.to_memref
+//      CHECK: %[[C0:.*]] = arith.constant 0 : index
+//      CHECK: %[[DIM:.*]] = memref.dim %[[MEMREF_ADDRSPACE2]], %[[C0]] : memref<?xf32, 2>
+//      CHECK: %[[MEMREF_ADDRSPACE7:.*]] = memref.alloc(%[[DIM]]) : memref<?xf32, 7>
+//      CHECK: memref.copy %[[MEMREF_ADDRSPACE2]], %[[MEMREF_ADDRSPACE7]]
+// CHECK-SAME:   memref<?xf32, 2> to memref<?xf32, 7>
+//      CHECK: return %[[MEMREF_ADDRSPACE7]]
+func.func @canonicalize_buffer_cast_of_tensor_load_different_address_space(%arg0: memref<?xf32, 2>)
     -> memref<?xf32, 7> {
   %0 = bufferization.to_tensor %arg0 : memref<?xf32, 2> to tensor<?xf32, 7>
   %1 = bufferization.to_memref %0 : tensor<?xf32, 7> to memref<?xf32, 7>
diff --git a/mlir/test/Dialect/GPU/dynamic-shared-memory.mlir b/mlir/test/Dialect/GPU/dynamic-shared-memory.mlir
index d73125fd763e6..810f7d2c3ebda 100644
--- a/mlir/test/Dialect/GPU/dynamic-shared-memory.mlir
+++ b/mlir/test/Dialect/GPU/dynamic-shared-memory.mlir
@@ -22,7 +22,7 @@ gpu.module @modules {
 
 // CHECK-DAG: %[[S0:.+]] = llvm.mlir.constant(32 : index) : i64
 // CHECK-DAG: %[[S1:.+]] = llvm.mlir.constant(64 : index) : i64
-// CHECK-DAG: %[[S2:.+]] = llvm.mlir.undef : !llvm.struct<(ptr<3>, ptr<3>, i64, array<2 x i64>, array<2 x i64>)>
+// CHECK-DAG: %[[S2:.+]] = llvm.mlir.poison : !llvm.struct<(ptr<3>, ptr<3>, i64, array<2 x i64>, array<2 x i64>)>
 // CHECK-DAG: %[[S3:.+]] = llvm.mlir.constant(1 : index) : i64
 // CHECK-DAG: %[[S4:.+]] = llvm.mlir.constant(0 : index) : i64
 // CHECK-DAG: %[[S5:.+]] = llvm.mlir.addressof @__dynamic_shmem__3 : !llvm.ptr<3>
@@ -56,7 +56,7 @@ gpu.module @modules {
     "test.use.shared.memory"(%0) : (memref<32x64xf32, #gpu.address_space<workgroup>>) -> ()
 // CHECK-DAG: %[[S0:.+]] = llvm.mlir.constant(32 : index) : i64
 // CHECK-DAG: %[[S1:.+]] = llvm.mlir.constant(64 : index) : i64
-// CHECK-DAG: %[[S2:.+]] = llvm.mlir.undef : !llvm.struct<(ptr<3>, ptr<3>, i64, array<2 x i64>, array<2 x i64>)>
+// CHECK-DAG: %[[S2:.+]] = llvm.mlir.poison : !llvm.struct<(ptr<3>, ptr<3>, i64, array<2 x i64>, array<2 x i64>)>
 // CHECK-DAG: %[[S3:.+]] = llvm.mlir.constant(1 : index) : i64
 // CHECK-DAG: %[[S4:.+]] = llvm.mlir.constant(0 : index) : i64
 // CHECK-DAG: %[[S5:.+]] = llvm.mlir.addressof @__dynamic_shmem__3 : !llvm.ptr<3>
@@ -82,7 +82,7 @@ gpu.module @modules {
     "test.use.shared.memory"(%0) : (memref<32x64xf32, #gpu.address_space<workgroup>>) -> ()
 // CHECK-DAG: %[[S0:.+]] = llvm.mlir.constant(32 : index) : i64
 // CHECK-DAG: %[[S1:.+]] = llvm.mlir.constant(64 : index) : i64
-// CHECK-DAG: %[[S2:.+]] = llvm.mlir.undef : !llvm.struct<(ptr<3>, ptr<3>, i64, array<2 x i64>, array<2 x i64>)>
+// CHECK-DAG: %[[S2:.+]] = llvm.mlir.poison : !llvm.struct<(ptr<3>, ptr<3>, i64, array<2 x i64>, array<2 x i64>)>
 // CHECK-DAG: %[[S3:.+]] = llvm.mlir.constant(1 : index) : i64
 // CHECK-DAG: %[[S4:.+]] = llvm.mlir.constant(0 : index) : i64
 // CHECK-DAG: %[[S5:.+]] = llvm.mlir.addressof @__dynamic_shmem__3 : !llvm.ptr<3>
diff --git a/mlir/test/Dialect/GPU/module-to-binary-nvvm.mlir b/mlir/test/Dialect/GPU/module-to-binary-nvvm.mlir
index e6284ccf94b50..dee680208bdf0 100644
--- a/mlir/test/Dialect/GPU/module-to-binary-nvvm.mlir
+++ b/mlir/test/Dialect/GPU/module-to-binary-nvvm.mlir
@@ -16,7 +16,7 @@ module attributes {gpu.container_module} {
   }
 
   // CHECK-LABEL:gpu.binary @kernel_module2
-  // CHECK-ISA:[#gpu.object<#nvvm.target<flags = {fast}>, properties = {O = 2 : i32}, assembly = "{{.*}}">, #gpu.object<#nvvm.target, properties = {O = 2 : i32}, assembly = "{{.*}}">]
+  // CHECK-ISA:[#gpu.object<#nvvm.target<flags = {fast}>, properties = {LLVMIRToISATimeInMs = {{[0-9]+}} : i64, O = 2 : i32}, assembly = "{{.*}}">, #gpu.object<#nvvm.target, properties = {LLVMIRToISATimeInMs = {{[0-9]+}} : i64, O = 2 : i32}, assembly = "{{.*}}">]
   gpu.module @kernel_module2 [#nvvm.target<flags = {fast}>, #nvvm.target] {
     llvm.func @kernel(%arg0: i32, %arg1: !llvm.ptr,
         %arg2: !llvm.ptr, %arg3: i64, %arg4: i64,
diff --git a/mlir/test/Dialect/GPU/transform-gpu.mlir b/mlir/test/Dialect/GPU/transform-gpu.mlir
index 0a5c85336831a..09ae0f4af686f 100644
--- a/mlir/test/Dialect/GPU/transform-gpu.mlir
+++ b/mlir/test/Dialect/GPU/transform-gpu.mlir
@@ -43,7 +43,7 @@ module attributes {transform.with_named_sequence} {
 !type = memref<2 x 32 x f32>
 !type1d = memref<32 x f32>
 
-// CHECK-DAG: #[[$MAP:.*]] = affine_map<()[s0] -> (s0 floordiv 128)> 
+// CHECK-DAG: #[[$MAP:.*]] = affine_map<()[s0] -> (s0 floordiv 128)>
 
 // CHECK-LABEL: func.func @warpgroup_3d(
 // CHECK-SAME:    %[[ARGX:[0-9a-z]+]]: memref<2x32xf32>
@@ -647,7 +647,7 @@ module attributes {transform.with_named_sequence} {
 #map = affine_map<(d0) -> (d0 *  128)>
 #map1 = affine_map<(d0) -> (d0 * 32)>
 
-// CHECK-DAG: #[[$MAPB:.*]] = affine_map<()[s0] -> (s0 * 128)> 
+// CHECK-DAG: #[[$MAPB:.*]] = affine_map<()[s0] -> (s0 * 128)>
 // CHECK-DAG: #[[$MAPW:.*]] = affine_map<()[s0, s1, s2] -> (s2 * 32 + ((s0 + s1 * 4) floordiv 32) * 32)>
 
 // CHECK-LABEL: func.func @simple_fill(
diff --git a/mlir/test/Dialect/LLVM/lower-to-llvm-e2e-with-target-tag.mlir b/mlir/test/Dialect/LLVM/lower-to-llvm-e2e-with-target-tag.mlir
index 2785b50886122..835ae01ffa8c1 100644
--- a/mlir/test/Dialect/LLVM/lower-to-llvm-e2e-with-target-tag.mlir
+++ b/mlir/test/Dialect/LLVM/lower-to-llvm-e2e-with-target-tag.mlir
@@ -30,7 +30,7 @@ func.func @subview(%0 : memref<64x4xf32, strided<[4, 1], offset: 0>>, %arg0 : in
   // CHECK-DAG: %[[STRIDE0:.*]] = llvm.mlir.constant(4 : index) : i64
   // CHECK-DAG: %[[DESCSTRIDE0:.*]] = llvm.mul %[[ARG0]], %[[STRIDE0]] overflow<nsw> : i64
   // CHECK-DAG: %[[OFF2:.*]] = llvm.add %[[DESCSTRIDE0]], %[[ARG1]] : i64
-  // CHECK-DAG: %[[DESC:.*]] = llvm.mlir.undef : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)>
+  // CHECK-DAG: %[[DESC:.*]] = llvm.mlir.poison : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)>
 
   // Base address and algined address.
   // CHECK-DAG: %[[DESC0:.*]] = llvm.insertvalue %[[BASE]], %[[DESC]][0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)>
@@ -59,7 +59,7 @@ module @named_inclusion_in_named attributes { transform.with_named_sequence } {
   transform.named_sequence private @lower_to_cpu(!transform.any_op {transform.consumed}) -> !transform.any_op
 
   transform.named_sequence @__transform_main(%toplevel_module: !transform.any_op {transform.consumed}) {
-    %m2 = transform.include @lower_to_cpu failures(suppress) (%toplevel_module) 
+    %m2 = transform.include @lower_to_cpu failures(suppress) (%toplevel_module)
       : (!transform.any_op) -> (!transform.any_op)
     transform.yield
   }
diff --git a/mlir/test/Dialect/LLVM/lower-to-llvm-e2e-with-top-level-named-sequence.mlir b/mlir/test/Dialect/LLVM/lower-to-llvm-e2e-with-top-level-named-sequence.mlir
index c1f30c7eaf643..864ebb2155740 100644
--- a/mlir/test/Dialect/LLVM/lower-to-llvm-e2e-with-top-level-named-sequence.mlir
+++ b/mlir/test/Dialect/LLVM/lower-to-llvm-e2e-with-top-level-named-sequence.mlir
@@ -29,7 +29,7 @@ func.func @subview(%0 : memref<64x4xf32, strided<[4, 1], offset: 0>>, %arg0 : in
   // CHECK-DAG: %[[STRIDE0:.*]] = llvm.mlir.constant(4 : index) : i64
   // CHECK-DAG: %[[DESCSTRIDE0:.*]] = llvm.mul %[[ARG0]], %[[STRIDE0]] overflow<nsw> : i64
   // CHECK-DAG: %[[OFF2:.*]] = llvm.add %[[DESCSTRIDE0]], %[[ARG1]] : i64
-  // CHECK-DAG: %[[DESC:.*]] = llvm.mlir.undef : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)>
+  // CHECK-DAG: %[[DESC:.*]] = llvm.mlir.poison : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)>
 
   // Base address and algined address.
   // CHECK-DAG: %[[DESC0:.*]] = llvm.insertvalue %[[BASE]], %[[DESC]][0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)>
@@ -57,7 +57,7 @@ module @named_inclusion_in_named attributes { transform.with_named_sequence } {
 
   transform.named_sequence @entry_point(
     %toplevel_module : !transform.any_op {transform.readonly}) {
-    transform.include @lower_to_llvm failures(suppress) (%toplevel_module) 
+    transform.include @lower_to_llvm failures(suppress) (%toplevel_module)
       : (!transform.any_op) -> (!transform.any_op)
     transform.yield
   }
diff --git a/mlir/test/Dialect/LLVMIR/alias.mlir b/mlir/test/Dialect/LLVMIR/alias.mlir
new file mode 100644
index 0000000000000..807843a27e6fa
--- /dev/null
+++ b/mlir/test/Dialect/LLVMIR/alias.mlir
@@ -0,0 +1,141 @@
+// RUN: mlir-opt %s -split-input-file --verify-roundtrip | FileCheck %s
+
+llvm.func internal @callee() -> !llvm.ptr attributes {dso_local} {
+  %0 = llvm.mlir.zero : !llvm.ptr
+  llvm.return %0 : !llvm.ptr
+}
+
+llvm.mlir.alias external @foo_alias : !llvm.ptr {
+  %0 = llvm.mlir.addressof @callee : !llvm.ptr
+  llvm.return %0 : !llvm.ptr
+}
+
+llvm.mlir.alias external @_ZTV1D : !llvm.struct<(array<3 x ptr>)> {
+  %0 = llvm.mlir.addressof @callee : !llvm.ptr
+  llvm.return %0 : !llvm.ptr
+}
+
+// CHECK: llvm.mlir.alias external @foo_alias : !llvm.ptr {
+// CHECK:   %[[ADDR:.*]] = llvm.mlir.addressof @callee : !llvm.ptr
+// CHECK:   llvm.return %[[ADDR]] : !llvm.ptr
+// CHECK: }
+// CHECK: llvm.mlir.alias external @_ZTV1D : !llvm.struct<(array<3 x ptr>)> {
+// CHECK:   %[[ADDR:.*]] = llvm.mlir.addressof @callee : !llvm.ptr
+// CHECK:   llvm.return %[[ADDR]] : !llvm.ptr
+// CHECK: }
+
+// -----
+
+llvm.mlir.global external @zed(42 : i32) : i32
+
+llvm.mlir.alias external @foo : i32 {
+  %0 = llvm.mlir.addressof @zed : !llvm.ptr
+  llvm.return %0 : !llvm.ptr
+}
+
+llvm.mlir.alias external @foo2 : i16 {
+  %0 = llvm.mlir.addressof @zed : !llvm.ptr
+  llvm.return %0 : !llvm.ptr
+}
+
+// CHECK: llvm.mlir.alias external @foo : i32 {
+// CHECK:   %[[ADDR:.*]] = llvm.mlir.addressof @zed : !llvm.ptr
+// CHECK:   llvm.return %[[ADDR]] : !llvm.ptr
+// CHECK: }
+// CHECK: llvm.mlir.alias external @foo2 : i16 {
+// CHECK:   %[[ADDR:.*]] = llvm.mlir.addressof @zed : !llvm.ptr
+// CHECK:   llvm.return %[[ADDR]] : !llvm.ptr
+// CHECK: }
+
+// -----
+
+llvm.mlir.global private constant @glob.private(dense<0> : tensor<32xi32>) : !llvm.array<32 x i32>
+
+llvm.mlir.alias linkonce_odr hidden @glob {dso_local} : !llvm.array<32 x i32> {
+  %0 = llvm.mlir.constant(1234 : i64) : i64
+  %1 = llvm.mlir.addressof @glob.private : !llvm.ptr
+  %2 = llvm.ptrtoint %1 : !llvm.ptr to i64
+  %3 = llvm.add %2, %0 : i64
+  %4 = llvm.inttoptr %3 : i64 to !llvm.ptr
+  llvm.return %4 : !llvm.ptr
+}
+
+// CHECK: llvm.mlir.global private constant @glob.private(dense<0> : tensor<32xi32>)
+// CHECK: llvm.mlir.alias linkonce_odr hidden @glob {dso_local} : !llvm.array<32 x i32> {
+// CHECK:   %[[CST:.*]] = llvm.mlir.constant(1234 : i64) : i64
+// CHECK:   %[[ADDR:.*]] = llvm.mlir.addressof @glob.private : !llvm.ptr
+// CHECK:   %[[INTADDR:.*]] = llvm.ptrtoint %[[ADDR]] : !llvm.ptr to i64
+// CHECK:   %[[BACKTOPTR:.*]] = llvm.add %[[INTADDR]], %[[CST]] : i64
+// CHECK:   %[[RET_ADDR:.*]]  = llvm.inttoptr %[[BACKTOPTR]] : i64 to !llvm.ptr
+// CHECK:   llvm.return %[[RET_ADDR]] : !llvm.ptr
+// CHECK: }
+
+// -----
+
+llvm.mlir.global external @v1(0 : i32) : i32
+llvm.mlir.alias external @a3 : i32 {
+  %0 = llvm.mlir.addressof @v1 : !llvm.ptr
+  %1 = llvm.addrspacecast %0 : !llvm.ptr to !llvm.ptr<2>
+  llvm.return %1 : !llvm.ptr<2>
+}
+
+// CHECK: llvm.mlir.alias external @a3 : i32 {
+// CHECK:   %[[ADDR:.*]] = llvm.mlir.addressof @v1 : !llvm.ptr
+// CHECK:   %1 = llvm.addrspacecast %[[ADDR]] : !llvm.ptr to !llvm.ptr<2>
+// CHECK:   llvm.return %1 : !llvm.ptr<2>
+// CHECK: }
+
+// -----
+
+llvm.mlir.global private @g1(0 : i32) {dso_local} : i32
+
+llvm.mlir.alias private @a1 {dso_local} : i32 {
+  %0 = llvm.mlir.addressof @g1 : !llvm.ptr
+  llvm.return %0 : !llvm.ptr
+}
+
+llvm.mlir.global internal constant @g2() : !llvm.ptr {
+  %0 = llvm.mlir.addressof @a1 : !llvm.ptr
+  llvm.return %0 : !llvm.ptr
+}
+
+llvm.mlir.alias private @a2 {dso_local} : !llvm.ptr {
+  %0 = llvm.mlir.addressof @a1 : !llvm.ptr
+  llvm.return %0 : !llvm.ptr
+}
+
+llvm.mlir.global internal constant @g3() : !llvm.ptr {
+  %0 = llvm.mlir.addressof @a2 : !llvm.ptr
+  llvm.return %0 : !llvm.ptr
+}
+
+// CHECK: llvm.mlir.alias private @a1 {dso_local} : i32 {
+// CHECK:   %[[ADDR:.*]] = llvm.mlir.addressof @g1 : !llvm.ptr
+// CHECK:   llvm.return %[[ADDR]] : !llvm.ptr
+// CHECK: }
+// CHECK: llvm.mlir.global internal constant @g2()
+// CHECK:   %[[ADDR:.*]] = llvm.mlir.addressof @a1 : !llvm.ptr
+// CHECK:   llvm.return %[[ADDR]] : !llvm.ptr
+// CHECK: }
+// CHECK: llvm.mlir.alias private @a2 {dso_local} : !llvm.ptr {
+// CHECK:   %[[ADDR:.*]] = llvm.mlir.addressof @a1 : !llvm.ptr
+// CHECK:   llvm.return %[[ADDR]] : !llvm.ptr
+// CHECK: }
+// CHECK: llvm.mlir.global internal constant @g3()
+// CHECK:   %[[ADDR:.*]] = llvm.mlir.addressof @a2 : !llvm.ptr
+// CHECK:   llvm.return %[[ADDR]] : !llvm.ptr
+// CHECK: }
+
+// -----
+
+llvm.mlir.global private @g30(0 : i32) {dso_local} : i32
+
+llvm.mlir.alias private unnamed_addr thread_local @a30 {dso_local} : i32 {
+  %0 = llvm.mlir.addressof @g30 : !llvm.ptr
+  llvm.return %0 : !llvm.ptr
+}
+
+// CHECK: llvm.mlir.alias private unnamed_addr thread_local @a30 {dso_local} : i32 {
+// CHECK:   %0 = llvm.mlir.addressof @g30 : !llvm.ptr
+// CHECK:   llvm.return %0 : !llvm.ptr
+// CHECK: }
diff --git a/mlir/test/Dialect/LLVMIR/canonicalize.mlir b/mlir/test/Dialect/LLVMIR/canonicalize.mlir
index 15f960167cb5f..c509cd82227c2 100644
--- a/mlir/test/Dialect/LLVMIR/canonicalize.mlir
+++ b/mlir/test/Dialect/LLVMIR/canonicalize.mlir
@@ -84,6 +84,16 @@ llvm.func @fold_unrelated_extractvalue(%arr: !llvm.array<4 x f32>) -> f32 {
   llvm.return %3 : f32
 }
 
+// -----
+// CHECK-LABEL: fold_extract_extractvalue
+llvm.func @fold_extract_extractvalue(%arr: !llvm.struct<(i64, array<1 x ptr<1>>)>) -> !llvm.ptr<1> {
+  // CHECK: llvm.extractvalue %{{.*}}[1, 0] 
+  // CHECK-NOT: extractvalue
+  %a = llvm.extractvalue %arr[1] : !llvm.struct<(i64, array<1 x ptr<1>>)> 
+  %b = llvm.extractvalue %a[0] : !llvm.array<1 x ptr<1>> 
+  llvm.return %b : !llvm.ptr<1>
+}
+
 // -----
 
 // CHECK-LABEL: fold_bitcast
diff --git a/mlir/test/Dialect/LLVMIR/invalid.mlir b/mlir/test/Dialect/LLVMIR/invalid.mlir
index 25806d9d0edd7..0415ab00bdb05 100644
--- a/mlir/test/Dialect/LLVMIR/invalid.mlir
+++ b/mlir/test/Dialect/LLVMIR/invalid.mlir
@@ -235,6 +235,7 @@ func.func @call_missing_ptr_type(%callee : !llvm.func<i8 (i8)>, %arg : i8) {
 func.func private @standard_func_callee()
 
 func.func @call_missing_ptr_type(%arg : i8) {
+  // expected-error@+2 {{expected '('}}
   // expected-error@+1 {{expected direct call to have 1 trailing type}}
   llvm.call @standard_func_callee(%arg) : !llvm.ptr, (i8) -> (i8)
   llvm.return
@@ -251,6 +252,7 @@ func.func @call_non_pointer_type(%callee : !llvm.func<i8 (i8)>, %arg : i8) {
 // -----
 
 func.func @call_non_function_type(%callee : !llvm.ptr, %arg : i8) {
+  // expected-error@+2 {{expected '('}}
   // expected-error@+1 {{expected trailing function type}}
   llvm.call %callee(%arg) : !llvm.ptr, !llvm.func<i8 (i8)>
   llvm.return
@@ -1189,6 +1191,14 @@ func.func @cp_async(%arg0: !llvm.ptr<3>, %arg1: !llvm.ptr<1>) {
 
 // -----
 
+func.func @mapa(%a: !llvm.ptr, %b : i32) {
+  // expected-error @below {{`res` and `a` should have the same type}}
+  %0 = nvvm.mapa %a, %b: !llvm.ptr -> !llvm.ptr<3>
+  return
+}
+
+// -----
+
 func.func @gep_struct_variable(%arg0: !llvm.ptr, %arg1: i32, %arg2: i32) {
   // expected-error @below {{op expected index 1 indexing a struct to be constant}}
   llvm.getelementptr %arg0[%arg1, %arg1] : (!llvm.ptr, i32, i32) -> !llvm.ptr, !llvm.struct<(i32)>
@@ -1695,3 +1705,53 @@ llvm.func @wrong_number_of_bundle_tags() {
   } : (i32, i32) -> ()
   llvm.return
 }
+
+// -----
+
+llvm.mlir.global external @x(42 : i32) : i32
+
+// expected-error@+1 {{expects type to be a valid element type for an LLVM global alias}}
+llvm.mlir.alias external @y : !llvm.label {
+  %0 = llvm.mlir.addressof @x : !llvm.ptr
+  llvm.return %0 : !llvm.ptr
+}
+
+// -----
+
+llvm.mlir.global external @x(42 : i32) : i32
+
+// expected-error@+1 {{linkage not supported in aliases, available options}}
+llvm.mlir.alias appending @y2 : i32 {
+  %0 = llvm.mlir.addressof @x : !llvm.ptr
+  llvm.return %0 : !llvm.ptr
+}
+
+// -----
+
+// expected-error@+1 {{initializer region must always return a pointer}}
+llvm.mlir.alias external @y3 : i32 {
+  %c = llvm.mlir.constant(42 : i64) : i64
+  llvm.return %c : i64
+}
+
+// -----
+
+llvm.mlir.global external @x(42 : i32) : i32
+
+llvm.mlir.alias external @y4 : i32 {
+  %0 = llvm.mlir.addressof @x : !llvm.ptr
+  // expected-error@+1 {{ops with side effects are not allowed in alias initializers}}
+  %2 = llvm.load %0 : !llvm.ptr -> i32
+  llvm.return %0 : !llvm.ptr
+}
+
+// -----
+
+llvm.mlir.global external @x(42 : i32) : i32
+
+llvm.mlir.alias external @y5 : i32 {
+  // expected-error@+1 {{pointer address space must match address space}}
+  %0 = llvm.mlir.addressof @x : !llvm.ptr<4>
+  llvm.return %0 : !llvm.ptr<4>
+}
+
diff --git a/mlir/test/Dialect/LLVMIR/nvvm.mlir b/mlir/test/Dialect/LLVMIR/nvvm.mlir
index 4c3b6648a41c0..dd54acd1e317e 100644
--- a/mlir/test/Dialect/LLVMIR/nvvm.mlir
+++ b/mlir/test/Dialect/LLVMIR/nvvm.mlir
@@ -509,6 +509,28 @@ func.func @wgmma_wait_group_sync_aligned() {
   return
 }
 
+func.func @griddepcontrol_wait() {
+  // CHECK: nvvm.griddepcontrol.wait
+  nvvm.griddepcontrol.wait
+  return
+}
+
+func.func @griddepcontrol_launch_dependents()
+{
+  // CHECK: nvvm.griddepcontrol.launch.dependents
+  nvvm.griddepcontrol.launch.dependents
+  return
+}
+
+// CHECK-LABEL: @mapa
+func.func @mapa(%a: !llvm.ptr, %a_shared: !llvm.ptr<3>, %b : i32) {
+  // CHECK:   nvvm.mapa %{{.*}}
+  %0 = nvvm.mapa %a, %b: !llvm.ptr -> !llvm.ptr
+  // CHECK:   nvvm.mapa %{{.*}}
+  %1 = nvvm.mapa %a_shared, %b: !llvm.ptr<3> -> !llvm.ptr<3>
+  return
+}
+
 // -----
 
 // Just check these don't emit errors.
diff --git a/mlir/test/Dialect/LLVMIR/rocdl.mlir b/mlir/test/Dialect/LLVMIR/rocdl.mlir
index 712f8c2a1caf6..73e844b5f3e13 100644
--- a/mlir/test/Dialect/LLVMIR/rocdl.mlir
+++ b/mlir/test/Dialect/LLVMIR/rocdl.mlir
@@ -258,6 +258,93 @@ func.func @rocdl.xdlops(%arg0 : f32, %arg1 : f32,
   llvm.return
 }
 
+
+llvm.func @rocdl.smfmac(%arg0 : i32,
+                   %arg1 : vector<4 x f16>,
+                   %arg2 : vector<8 x f16>,
+                   %arg3 : vector<4 x f32>,
+                   %arg4 : vector<16 x f32>,
+                   %arg5 : vector<4 x i16>,
+                   %arg6 : vector<8 x i16>,
+                   %arg7 : vector<2xi32>,
+                   %arg8 : vector<4xi32>,
+                   %arg9 : vector<16xi32>) -> vector<4 x f32> {
+  %csti32 = llvm.mlir.constant(42 : i32) : i32
+
+  // CHECK-LABEL: rocdl.smfmac
+  // CHECK: rocdl.smfmac.f32.16x16x32.f16 %{{.*}} : (vector<4xf16>, vector<8xf16>, vector<4xf32>, i32, i32, i32) -> vector<4xf32>
+  %r0 = rocdl.smfmac.f32.16x16x32.f16 %arg1, %arg2, %arg3, %csti32, %csti32, %csti32 :
+                                (vector<4xf16>, vector<8xf16>, vector<4xf32>,
+                                 i32, i32, i32) -> vector<4xf32>
+
+  // CHECK: rocdl.smfmac.f32.32x32x16.f16 %{{.*}} : (vector<4xf16>, vector<8xf16>, vector<16xf32>, i32, i32, i32) -> vector<16xf32>
+  %r1 = rocdl.smfmac.f32.32x32x16.f16 %arg1, %arg2, %arg4, %csti32, %csti32, %csti32 :
+                                (vector<4xf16>, vector<8xf16>, vector<16xf32>,
+                                 i32, i32, i32) -> vector<16xf32>
+
+  // CHECK: rocdl.smfmac.f32.16x16x32.bf16 %{{.*}} : (vector<4xi16>, vector<8xi16>, vector<4xf32>, i32, i32, i32) -> vector<4xf32>
+  %r2 = rocdl.smfmac.f32.16x16x32.bf16 %arg5, %arg6, %arg3, %csti32, %csti32, %csti32 :
+                                (vector<4xi16>, vector<8xi16>, vector<4xf32>,
+                                 i32, i32, i32) -> vector<4xf32>
+
+  // CHECK: rocdl.smfmac.f32.32x32x16.bf16 %{{.*}} : (vector<4xi16>, vector<8xi16>, vector<16xf32>, i32, i32, i32) -> vector<16xf32>
+  %r3 = rocdl.smfmac.f32.32x32x16.bf16 %arg5, %arg6, %arg4, %csti32, %csti32, %csti32 :
+                                (vector<4xi16>, vector<8xi16>, vector<16xf32>,
+                                 i32, i32, i32) -> vector<16xf32>
+
+  // CHECK: rocdl.smfmac.i32.16x16x64.i8 %{{.*}} : (vector<2xi32>, vector<4xi32>, vector<4xi32>, i32, i32, i32) -> vector<4xi32>
+  %r4 = rocdl.smfmac.i32.16x16x64.i8 %arg7, %arg8, %arg8, %csti32, %csti32, %csti32 :
+                                (vector<2xi32>, vector<4xi32>, vector<4xi32>,
+                                 i32, i32, i32) -> vector<4xi32>
+
+  // CHECK: rocdl.smfmac.i32.32x32x32.i8 %{{.*}} : (vector<2xi32>, vector<4xi32>, vector<16xi32>, i32, i32, i32) -> vector<16xi32>
+  %r5 = rocdl.smfmac.i32.32x32x32.i8 %arg7, %arg8, %arg9, %csti32, %csti32, %csti32 :
+                                (vector<2xi32>, vector<4xi32>, vector<16xi32>,
+                                 i32, i32, i32) -> vector<16xi32>
+
+  // CHECK: rocdl.smfmac.f32.16x16x64.bf8.bf8 %{{.*}} : (vector<2xi32>, vector<4xi32>, vector<4xf32>, i32, i32, i32) -> vector<4xf32>
+  %r6 = rocdl.smfmac.f32.16x16x64.bf8.bf8 %arg7, %arg8, %arg3, %csti32, %csti32, %csti32 :
+                                (vector<2xi32>, vector<4xi32>, vector<4xf32>,
+                                 i32, i32, i32) -> vector<4xf32>
+
+  // CHECK: rocdl.smfmac.f32.16x16x64.bf8.fp8 %{{.*}} : (vector<2xi32>, vector<4xi32>, vector<4xf32>, i32, i32, i32) -> vector<4xf32>
+  %r7 = rocdl.smfmac.f32.16x16x64.bf8.fp8 %arg7, %arg8, %arg3, %csti32, %csti32, %csti32 :
+                                (vector<2xi32>, vector<4xi32>, vector<4xf32>,
+                                 i32, i32, i32) -> vector<4xf32>
+
+  // CHECK: rocdl.smfmac.f32.16x16x64.fp8.bf8 %{{.*}} : (vector<2xi32>, vector<4xi32>, vector<4xf32>, i32, i32, i32) -> vector<4xf32>
+  %r8 = rocdl.smfmac.f32.16x16x64.fp8.bf8 %arg7, %arg8, %arg3, %csti32, %csti32, %csti32 :
+                                (vector<2xi32>, vector<4xi32>, vector<4xf32>,
+                                 i32, i32, i32) -> vector<4xf32>
+
+  // CHECK: rocdl.smfmac.f32.16x16x64.fp8.fp8 %{{.*}} : (vector<2xi32>, vector<4xi32>, vector<4xf32>, i32, i32, i32) -> vector<4xf32>
+  %r9 = rocdl.smfmac.f32.16x16x64.fp8.fp8 %arg7, %arg8, %arg3, %csti32, %csti32, %csti32 :
+                                (vector<2xi32>, vector<4xi32>, vector<4xf32>,
+                                 i32, i32, i32) -> vector<4xf32>
+
+  // CHECK: rocdl.smfmac.f32.32x32x32.bf8.bf8 %{{.*}} : (vector<2xi32>, vector<4xi32>, vector<16xf32>, i32, i32, i32) -> vector<16xf32>
+  %r10 = rocdl.smfmac.f32.32x32x32.bf8.bf8 %arg7, %arg8, %arg4, %csti32, %csti32, %csti32 :
+                                (vector<2xi32>, vector<4xi32>, vector<16xf32>,
+                                 i32, i32, i32) -> vector<16xf32>
+
+  // CHECK: rocdl.smfmac.f32.32x32x32.bf8.fp8 %{{.*}} : (vector<2xi32>, vector<4xi32>, vector<16xf32>, i32, i32, i32) -> vector<16xf32>
+  %r11 = rocdl.smfmac.f32.32x32x32.bf8.fp8 %arg7, %arg8, %arg4, %csti32, %csti32, %csti32 :
+                                (vector<2xi32>, vector<4xi32>, vector<16xf32>,
+                                 i32, i32, i32) -> vector<16xf32>
+
+  // CHECK: rocdl.smfmac.f32.32x32x32.fp8.bf8 %{{.*}} : (vector<2xi32>, vector<4xi32>, vector<16xf32>, i32, i32, i32) -> vector<16xf32>
+  %r12 = rocdl.smfmac.f32.32x32x32.fp8.bf8 %arg7, %arg8, %arg4, %csti32, %csti32, %csti32 :
+                                (vector<2xi32>, vector<4xi32>, vector<16xf32>,
+                                 i32, i32, i32) -> vector<16xf32>
+
+  // CHECK: rocdl.smfmac.f32.32x32x32.fp8.fp8 %{{.*}} : (vector<2xi32>, vector<4xi32>, vector<16xf32>, i32, i32, i32) -> vector<16xf32>
+  %r13 = rocdl.smfmac.f32.32x32x32.fp8.fp8 %arg7, %arg8, %arg4, %csti32, %csti32, %csti32 :
+                                (vector<2xi32>, vector<4xi32>, vector<16xf32>,
+                                 i32, i32, i32) -> vector<16xf32>
+
+  llvm.return %r0 : vector<4 x f32>
+}
+
 llvm.func @rocdl.mfma.scale.f32.32x32x64.f8f6f4(%arg0 : i32,
                    %arg1 : vector<16 x f32>, %arg2 : vector<8xi32>,
                    %arg3 : vector<6xi32>, %arg4 : vector<4xi32>) {
@@ -667,27 +754,52 @@ llvm.func @rocdl_8bit_floats(%source: i32, %stoch: i32) -> i32 {
 // CHECK-LABEL: @rocdl_8bit_floats
 // CHECK: rocdl.cvt.f32.bf8
 // CHECK: rocdl.cvt.f32.fp8
+// CHECK: rocdl.cvt.scalef32.f32.bf8
+// CHECK: rocdl.cvt.scalef32.f32.fp8
 // CHECK: rocdl.cvt.pk.bf8.f32
 // CHECK: rocdl.cvt.pk.fp8.f32
 // CHECK: rocdl.cvt.sr.bf8.f32
 // CHECK: rocdl.cvt.sr.fp8.f32
+// CHECK: rocdl.cvt.scalef32.sr.fp8.f32
+// CHECK: rocdl.cvt.sr.bf8.f32
+// CHECK: rocdl.cvt.scalef32.sr.bf8.f32
+// CHECK: rocdl.cvt.scalef32.pk.f32.fp8
+// CHECK: rocdl.cvt.scalef32.pk.f32.bf8
   %c0 = llvm.mlir.constant(0 : i32) : i32
   %c2 = llvm.mlir.constant(2 : i32) : i32
   %c3 = llvm.mlir.constant(3 : i32) : i32
+  %c4 = llvm.mlir.constant(1.0 : f32) : f32
   %false = llvm.mlir.constant(false) : i1
   %v1 = rocdl.cvt.f32.bf8 %source[%c0] : f32
   %v2 = rocdl.cvt.f32.fp8 %source[%c0] : f32
+  %v1_scaled = rocdl.cvt.scalef32.f32.bf8 %source[%c0], %c4 : f32
+  %v2_scaled = rocdl.cvt.scalef32.f32.fp8 %source[%c0], %c4 : f32
   %source2 = rocdl.cvt.pk.bf8.f32 %v1, %v2 -> %source[%false] : i32
   %source3 = rocdl.cvt.pk.fp8.f32 %v1, %v2 -> %source2[%false] : i32
   %source4 = rocdl.cvt.sr.bf8.f32 %v1, %stoch -> %source3[%c2] : i32
   %source5 = rocdl.cvt.sr.fp8.f32 %v2, %stoch -> %source4[%c3] : i32
+  %source5_scaled = rocdl.cvt.scalef32.sr.fp8.f32 %v2, %stoch, %c4 -> %source4[%c3] : i32
+  %source6 = rocdl.cvt.sr.bf8.f32 %v1, %stoch -> %source3[%c3] : i32
+  %source6_scaled  = rocdl.cvt.scalef32.sr.bf8.f32 %v2, %stoch, %c4 -> %source3[%c3] : i32
+  %source7_scaled = rocdl.cvt.scalef32.pk.f32.fp8 %source[%false], %c4 : f32
+  %source8_scaled = rocdl.cvt.scalef32.pk.f32.bf8 %source[%false], %c4 : f32
   llvm.return %source5 : i32
 }
 
-llvm.func @rocdl.waitcnt() {
-  // CHECK-LABEL: rocdl.waitcnt
-  // CHECK: rocdl.waitcnt 0
-  rocdl.waitcnt 0
+llvm.func @rocdl_8bit_packed_v2i16(%sourceA: f32, %sourceB: f32, %old: vector<2xi16>) -> vector<2xi16> {
+// CHECK-LABEL: @rocdl_8bit_packed_v2i16
+// CHECK: rocdl.cvt.scalef32.pk.fp8.f32
+  %c0 = llvm.mlir.constant(1.0 : f32) : f32
+  %false = llvm.mlir.constant(false) : i1
+  %source_scaled = rocdl.cvt.scalef32.pk.fp8.f32 %sourceA, %sourceB, %c0 -> %old[%false] : vector<2xi16>
+  %source2_scaled = rocdl.cvt.scalef32.pk.bf8.f32 %sourceA, %sourceB, %c0 -> %old[%false] : vector<2xi16>
+  llvm.return %source_scaled : vector<2xi16>
+}
+
+llvm.func @rocdl.s.waitcnt() {
+  // CHECK-LABEL: rocdl.s.waitcnt
+  // CHECK: rocdl.s.waitcnt 0
+  rocdl.s.waitcnt 0
   llvm.return
 }
 
diff --git a/mlir/test/Dialect/LLVMIR/roundtrip.mlir b/mlir/test/Dialect/LLVMIR/roundtrip.mlir
index 88660ce598f3c..09a0cd57e2675 100644
--- a/mlir/test/Dialect/LLVMIR/roundtrip.mlir
+++ b/mlir/test/Dialect/LLVMIR/roundtrip.mlir
@@ -941,3 +941,48 @@ llvm.func @test_assume_intr_with_opbundles(%arg0 : !llvm.ptr) {
   llvm.intr.assume %0 ["tag1"(%1, %2 : i32, i32), "tag2"(%3 : i32)] : i1
   llvm.return
 }
+
+llvm.func @somefunc(i32, !llvm.ptr)
+
+// CHECK-LABEL: llvm.func @test_call_arg_attrs_direct(
+// CHECK-SAME:    %[[VAL_0:.*]]: i32,
+// CHECK-SAME:    %[[VAL_1:.*]]: !llvm.ptr)
+llvm.func @test_call_arg_attrs_direct(%arg0: i32, %arg1: !llvm.ptr) {
+  // CHECK: llvm.call @somefunc(%[[VAL_0]], %[[VAL_1]]) : (i32, !llvm.ptr {llvm.byval = i64}) -> ()
+  llvm.call @somefunc(%arg0, %arg1) : (i32, !llvm.ptr {llvm.byval = i64}) -> ()
+  llvm.return
+}
+
+// CHECK-LABEL: llvm.func @test_call_arg_attrs_indirect(
+// CHECK-SAME:    %[[VAL_0:.*]]: i16,
+// CHECK-SAME:    %[[VAL_1:.*]]: !llvm.ptr
+llvm.func @test_call_arg_attrs_indirect(%arg0: i16, %arg1: !llvm.ptr) -> i16 {
+  // CHECK: llvm.call tail %[[VAL_1]](%[[VAL_0]]) : !llvm.ptr, (i16 {llvm.noundef, llvm.signext}) -> (i16 {llvm.signext})
+  %0 = llvm.call tail %arg1(%arg0) : !llvm.ptr, (i16 {llvm.noundef, llvm.signext}) -> (i16 {llvm.signext})
+  llvm.return %0 : i16
+}
+
+// CHECK-LABEL:   llvm.func @test_invoke_arg_attrs(
+// CHECK-SAME:      %[[VAL_0:.*]]: i16) attributes {personality = @__gxx_personality_v0} {
+llvm.func @test_invoke_arg_attrs(%arg0: i16) attributes { personality = @__gxx_personality_v0 } {
+  // CHECK:  llvm.invoke @somefunc(%[[VAL_0]]) to ^bb2 unwind ^bb1 : (i16 {llvm.noundef, llvm.signext}) -> ()
+  llvm.invoke @somefunc(%arg0) to ^bb2 unwind ^bb1 : (i16 {llvm.noundef, llvm.signext}) -> ()
+^bb1:
+  %1 = llvm.landingpad cleanup : !llvm.struct<(ptr, i32)>
+  llvm.return
+^bb2:
+  llvm.return
+}
+
+// CHECK-LABEL:   llvm.func @test_invoke_arg_attrs_indirect(
+// CHECK-SAME:      %[[VAL_0:.*]]: i16,
+// CHECK-SAME:      %[[VAL_1:.*]]: !llvm.ptr) -> i16 attributes {personality = @__gxx_personality_v0} {
+llvm.func @test_invoke_arg_attrs_indirect(%arg0: i16, %arg1: !llvm.ptr) -> i16 attributes { personality = @__gxx_personality_v0 } {
+  // CHECK: llvm.invoke %[[VAL_1]](%[[VAL_0]]) to ^bb2 unwind ^bb1 : !llvm.ptr, (i16 {llvm.noundef, llvm.signext}) -> (i16 {llvm.signext})
+  %0 = llvm.invoke %arg1(%arg0) to ^bb2 unwind ^bb1 : !llvm.ptr, (i16 {llvm.noundef, llvm.signext}) -> (i16 {llvm.signext})
+^bb1:
+  %1 = llvm.landingpad cleanup : !llvm.struct<(ptr, i32)>
+  llvm.return %0 : i16
+^bb2:
+  llvm.return %0 : i16
+}
diff --git a/mlir/test/Dialect/Linalg/data-layout-propagation.mlir b/mlir/test/Dialect/Linalg/data-layout-propagation.mlir
index 07708231a6e2f..cb8064411bbae 100644
--- a/mlir/test/Dialect/Linalg/data-layout-propagation.mlir
+++ b/mlir/test/Dialect/Linalg/data-layout-propagation.mlir
@@ -1301,7 +1301,7 @@ func.func @push_down_unpack_through_expand(%5: tensor<?x32x8x8xf32>, %dim: index
 // CHECK:         %[[C32:.+]] = arith.constant 32 : index
 // CHECK:         %[[C0:.+]] = arith.constant 0 : index
 // CHECK:         %[[DIM0:.+]] = tensor.dim %[[ARG0]], %[[C0]] : tensor<?x32x8x8xf32>
-// CHECK:         %[[SZ0:.+]] = arith.divui %[[DIM0]], %[[C32]] : index
+// CHECK:         %[[SZ0:.+]] = arith.divsi %[[DIM0]], %[[C32]] : index
 // CHECK:         %[[EXPANDED:.+]] = tensor.expand_shape %[[ARG0]] {{\[}}[0, 1], [2], [3], [4]] output_shape [%[[SZ0]], 32, 32, 8, 8] : tensor<?x32x8x8xf32> into tensor<?x32x32x8x8xf32>
 // CHECK:         %[[DIM:.+]] = tensor.dim %[[EXPANDED]], %[[C0]] : tensor<?x32x32x8x8xf32>
 // CHECK:         %[[EMPTY:.+]] = tensor.empty(%[[DIM]]) : tensor<?x256x256xf32>
@@ -1322,7 +1322,7 @@ func.func @push_down_unpack_through_expand_empty_outer_dims_perm(%5: tensor<?x32
 // CHECK:         %[[C32:.+]] = arith.constant 32 : index
 // CHECK:         %[[C0:.+]] = arith.constant 0 : index
 // CHECK:         %[[DIM0:.+]] = tensor.dim %[[ARG0]], %[[C0]] : tensor<?x32x8x8xf32>
-// CHECK:         %[[SZ0:.+]] = arith.divui %[[DIM0]], %[[C32]] : index
+// CHECK:         %[[SZ0:.+]] = arith.divsi %[[DIM0]], %[[C32]] : index
 // CHECK:         %[[EXPANDED:.+]] = tensor.expand_shape %[[ARG0]] {{\[}}[0, 1], [2], [3], [4]] output_shape [%[[SZ0]], 32, 32, 8, 8] : tensor<?x32x8x8xf32> into tensor<?x32x32x8x8xf32>
 // CHECK:         %[[DIM:.+]] = tensor.dim %[[EXPANDED]], %[[C0]] : tensor<?x32x32x8x8xf32>
 // CHECK:         %[[EMPTY:.+]] = tensor.empty(%[[DIM]]) : tensor<?x256x256xf32>
@@ -1373,7 +1373,7 @@ func.func @push_down_unpack_through_expand_on_outer_dims(%5: tensor<?x32x8xf32>,
 // CHECK:         %[[C256:.+]] = arith.constant 256 : index
 // CHECK:         %[[C0:.+]] = arith.constant 0 : index
 // CHECK:         %[[DIM0:.+]] = tensor.dim %[[ARG0]], %[[C0]] : tensor<?x32x8xf32>
-// CHECK:         %[[SZ0:.+]] = arith.divui %[[DIM0]], %[[C256]] : index
+// CHECK:         %[[SZ0:.+]] = arith.divsi %[[DIM0]], %[[C256]] : index
 // CHECK:         %[[EXPANDED:.+]] = tensor.expand_shape %[[ARG0]] {{\[}}[0, 1], [2], [3]] output_shape [%[[SZ0]], 256, 32, 8] : tensor<?x32x8xf32> into tensor<?x256x32x8xf32>
 // CHECK:         %[[DIM:.+]] = tensor.dim %[[EXPANDED]], %[[C0]] : tensor<?x256x32x8xf32>
 // CHECK:         %[[EMPTY:.+]] = tensor.empty(%[[DIM]]) : tensor<?x256x256xf32>
diff --git a/mlir/test/Dialect/Linalg/fuse-with-reshape-by-collapsing.mlir b/mlir/test/Dialect/Linalg/fuse-with-reshape-by-collapsing.mlir
index f17881d59a266..7db997cd4c0b5 100644
--- a/mlir/test/Dialect/Linalg/fuse-with-reshape-by-collapsing.mlir
+++ b/mlir/test/Dialect/Linalg/fuse-with-reshape-by-collapsing.mlir
@@ -99,14 +99,14 @@ func.func @fuse_by_collapsing_indexing_op(%arg0 : tensor<2x12x5x336x9xi32>,
 //   CHECK-DAG:   %[[C7:.+]] = arith.constant 7 : index
 //       CHECK:     %[[IV0:.+]] = linalg.index 0
 //       CHECK:     %[[IV1:.+]] = linalg.index 1
-//       CHECK:     %[[REM_IV1:.+]] = arith.remui %[[IV1]], %[[C4]]
-//       CHECK:     %[[DIV_IV1:.+]] = arith.divui %[[IV1]], %[[C4]]
+//       CHECK:     %[[REM_IV1:.+]] = arith.remsi %[[IV1]], %[[C4]]
+//       CHECK:     %[[DIV_IV1:.+]] = arith.divsi %[[IV1]], %[[C4]]
 //       CHECK:     %[[IV2:.+]] = linalg.index 2
 //       CHECK:     %[[IV3:.+]] = linalg.index 3
-//       CHECK:     %[[REM1_IV3:.+]] = arith.remui %[[IV3]], %[[C8]]
-//       CHECK:     %[[DIV1_IV3:.+]] = arith.divui %[[IV3]], %[[C8]]
-//       CHECK:     %[[REM2_IV3:.+]] = arith.remui %[[DIV1_IV3]], %[[C7]]
-//       CHECK:     %[[DIV2_IV3:.+]] = arith.divui %[[DIV1_IV3]], %[[C7]]
+//       CHECK:     %[[REM1_IV3:.+]] = arith.remsi %[[IV3]], %[[C8]]
+//       CHECK:     %[[DIV1_IV3:.+]] = arith.divsi %[[IV3]], %[[C8]]
+//       CHECK:     %[[REM2_IV3:.+]] = arith.remsi %[[DIV1_IV3]], %[[C7]]
+//       CHECK:     %[[DIV2_IV3:.+]] = arith.divsi %[[DIV1_IV3]], %[[C7]]
 //       CHECK:     %[[IV4:.+]] = linalg.index 4
 //       CHECK:     %[[T0:.+]] = arith.addi %[[IV0]], %[[DIV_IV1]]
 //       CHECK:     %[[T1:.+]] = arith.addi %[[T0]], %[[REM_IV1]]
@@ -215,13 +215,13 @@ func.func @fuse_by_collapsing_dynamic(%arg0 : tensor<?x?x?x?x?xi32>,
 //  CHECK-DAG:   %[[D1:.+]] = tensor.dim %[[EXPAND]], %[[C5]]
 //      CHECK:   linalg.generic
 //      CHECK:     %[[IV0:.+]] = linalg.index 1
-//      CHECK:     %[[REM1_IV0:.+]] = arith.remui %[[IV0]], %[[C5]]
-//      CHECK:     %[[DIV1_IV0:.+]] = arith.divui %[[IV0]], %[[C5]]
-//      CHECK:     %[[REM2_IV0:.+]] = arith.remui %[[DIV1_IV0]], %[[D1]]
-//      CHECK:     %[[DIV2_IV0:.+]] = arith.divui %[[DIV1_IV0]], %[[D1]]
+//      CHECK:     %[[REM1_IV0:.+]] = arith.remsi %[[IV0]], %[[C5]]
+//      CHECK:     %[[DIV1_IV0:.+]] = arith.divsi %[[IV0]], %[[C5]]
+//      CHECK:     %[[REM2_IV0:.+]] = arith.remsi %[[DIV1_IV0]], %[[D1]]
+//      CHECK:     %[[DIV2_IV0:.+]] = arith.divsi %[[DIV1_IV0]], %[[D1]]
 //      CHECK:     %[[IV1:.+]] = linalg.index 3
-//      CHECK:     %[[REM1_IV1:.+]] = arith.remui %[[IV1]], %[[D0]]
-//      CHECK:     %[[DIV1_IV1:.+]] = arith.divui %[[IV1]], %[[D0]]
+//      CHECK:     %[[REM1_IV1:.+]] = arith.remsi %[[IV1]], %[[D0]]
+//      CHECK:     %[[DIV1_IV1:.+]] = arith.divsi %[[IV1]], %[[D0]]
 
 // -----
 
@@ -439,7 +439,7 @@ func.func @fuse_only_one_reassociation(%arg0 : tensor<?x?xf32>, %arg1 : tensor<4
 // CHECK-SAME:       outs(%[[COLLAPSE_ARG1_1]] :
 //      CHECK:   %[[DIM:.+]] = tensor.dim %[[GENERIC]], %[[C1]] : tensor<4x?x?xf32>
 //      CHECK:   %[[DIM_2:.+]] = tensor.dim %[[GENERIC]], %[[C2]] : tensor<4x?x?xf32>
-//      CHECK:   %[[VAL_1:.+]] = arith.divui %[[DIM_2]], %[[C8]] : index
+//      CHECK:   %[[VAL_1:.+]] = arith.divsi %[[DIM_2]], %[[C8]] : index
 //      CHECK:   %[[EXPANDED_3:.+]] = tensor.expand_shape %[[GENERIC]] {{\[\[}}0], [1], [2, 3]] output_shape [4, %[[DIM]], %[[VAL_1]], 8] : tensor<4x?x?xf32> into tensor<4x?x?x8xf32>
 //      CHECK:   return %[[EXPANDED_3]]
 
@@ -492,11 +492,11 @@ func.func @fold_non_consecutive_dims(%arg0 : tensor<?x?xi32>, %sz0: index, %sz1:
 // CHECK-SAME:       outs(%[[COLLAPSE_INIT]] :
 // CHECK-NEXT:   ^bb{{[0-9]}}
 //      CHECK:       %[[ID0:.+]] = linalg.index 0
-//  CHECK-DAG:       %[[T0:.+]] = arith.remui %[[ID0]], %[[C4]]
-//  CHECK-DAG:       %[[T1:.+]] = arith.divui %[[ID0]], %[[C4]]
+//  CHECK-DAG:       %[[T0:.+]] = arith.remsi %[[ID0]], %[[C4]]
+//  CHECK-DAG:       %[[T1:.+]] = arith.divsi %[[ID0]], %[[C4]]
 //      CHECK:       %[[ID1:.+]] = linalg.index 1
-//  CHECK-DAG:       %[[T2:.+]] = arith.remui %[[ID1]], %[[C8]]
-//  CHECK-DAG:       %[[T3:.+]] = arith.divui %[[ID1]], %[[C8]]
+//  CHECK-DAG:       %[[T2:.+]] = arith.remsi %[[ID1]], %[[C8]]
+//  CHECK-DAG:       %[[T3:.+]] = arith.divsi %[[ID1]], %[[C8]]
 //  CHECK-DAG:       %[[T4:.+]] = arith.addi %[[T1]], %[[T2]]
 //  CHECK-DAG:       %[[T5:.+]] = arith.addi %[[T4]], %[[T0]]
 //  CHECK-DAG:       %[[T6:.+]] = arith.addi %[[T5]], %[[T3]]
@@ -504,8 +504,8 @@ func.func @fold_non_consecutive_dims(%arg0 : tensor<?x?xi32>, %sz0: index, %sz1:
 //      CHECK:       linalg.yield %[[T7]]
 //      CHECK:   %[[DIM_1:.+]] = tensor.dim %[[GENERIC]], %[[C0]] : tensor<?x?xi32>
 //      CHECK:   %[[DIM_2:.+]] = tensor.dim %[[GENERIC]], %[[C1]] : tensor<?x?xi32>
-//      CHECK:   %[[VAL_2:.+]] = arith.divui %[[DIM_1]], %[[C8]] : index
-//      CHECK:   %[[VAL_3:.+]] = arith.divui %[[DIM_2]], %[[C4]] : index
+//      CHECK:   %[[VAL_2:.+]] = arith.divsi %[[DIM_1]], %[[C8]] : index
+//      CHECK:   %[[VAL_3:.+]] = arith.divsi %[[DIM_2]], %[[C4]] : index
 //      CHECK:   %[[EXPANDED_3:.+]] = tensor.expand_shape %[[GENERIC]] {{\[\[}}0, 1], [2, 3]] output_shape [%[[VAL_2]], 8, %[[VAL_3]], 4] : tensor<?x?xi32> into tensor<?x8x?x4xi32>
 //      CHECK:   return %[[EXPANDED_3]]
 
diff --git a/mlir/test/Dialect/Linalg/fusion-push-reshape.mlir b/mlir/test/Dialect/Linalg/fusion-push-reshape.mlir
index 751ece37bc094..7acbd843cd1e7 100644
--- a/mlir/test/Dialect/Linalg/fusion-push-reshape.mlir
+++ b/mlir/test/Dialect/Linalg/fusion-push-reshape.mlir
@@ -12,7 +12,7 @@
 // CHECK-SAME: iterator_types = ["parallel", "parallel"]}
 // CHECK-SAME: ins(%[[A]], %[[B]] : tensor<?x16xf32>, tensor<16xf32>) outs(%[[RI]] : tensor<?x16xf32>)
 //      CHECK: %[[DIM:.*]] = tensor.dim %[[R]], %[[C0]] : tensor<?x16xf32>
-//      CHECK: %[[VAL_1:.*]] = arith.divui %[[DIM]], %[[C112]] : index
+//      CHECK: %[[VAL_1:.*]] = arith.divsi %[[DIM]], %[[C112]] : index
 //      CHECK: %[[RR:.*]] = tensor.expand_shape %[[R]] {{\[\[}}0, 1], [2]] output_shape [%[[VAL_1]], 112, 16] : tensor<?x16xf32> into tensor<?x112x16xf32>
 //      CHECK: return %[[RR]] : tensor<?x112x16xf32>
 func.func @reshape(%A: tensor<?x16xf32>, %B: tensor<16xf32>, %init: tensor<?x112x16xf32>, %sz0: index) -> tensor<?x112x16xf32> {
diff --git a/mlir/test/Dialect/Linalg/generalize-named-ops.mlir b/mlir/test/Dialect/Linalg/generalize-named-ops.mlir
index aba26c35931fd..0ec71c35497b1 100644
--- a/mlir/test/Dialect/Linalg/generalize-named-ops.mlir
+++ b/mlir/test/Dialect/Linalg/generalize-named-ops.mlir
@@ -943,7 +943,6 @@ func.func @matmul_transpose_a_explicit(%arg0: memref<5x3xf32>, %arg1: memref<5x7
                       ]
                       ins(%arg0, %arg1 : memref<5x3xf32>, memref<5x7xf32>)
                       outs(%arg2: memref<3x7xf32>)
-                      
   return
 }
 
@@ -969,7 +968,6 @@ func.func @matmul_transpose_b_explicit(%arg0: memref<3x5xf32>, %arg1: memref<7x5
                       ]
                       ins(%arg0, %arg1 : memref<3x5xf32>, memref<7x5xf32>)
                       outs(%arg2: memref<3x7xf32>)
-                      
   return
 }
 
@@ -996,9 +994,236 @@ func.func @matmul_transpose_a_b_explicit(%arg0: memref<5x3xf32>, %arg1: memref<7
                       ]
                       ins(%arg0, %arg1 : memref<5x3xf32>, memref<7x5xf32>)
                       outs(%arg2: memref<3x7xf32>)
-                      
   return
 }
 
 // -----
 
+// CHECK: #[[$ATTR_0:.+]] = affine_map<(d0, d1, d2, d3) -> (d0, d1, d3)>
+// CHECK: #[[$ATTR_1:.+]] = affine_map<(d0, d1, d2, d3) -> (d0, d3, d2)>
+// CHECK: #[[$ATTR_2:.+]] = affine_map<(d0, d1, d2, d3) -> (d0, d1, d2)>
+
+// CHECK-LABEL:   func.func @batch_matmul(
+// CHECK-SAME:                            %[[VAL_0:.*]]: tensor<2x3x5xf32>,
+// CHECK-SAME:                            %[[VAL_1:.*]]: tensor<2x5x7xf32>,
+// CHECK-SAME:                            %[[VAL_2:.*]]: tensor<2x3x7xf32>) -> tensor<2x3x7xf32> {
+// CHECK:           %[[VAL_3:.*]] = linalg.generic {indexing_maps = [#[[$ATTR_0]], #[[$ATTR_1]], #[[$ATTR_2]]], iterator_types = ["parallel", "parallel", "parallel", "reduction"]} ins(%[[VAL_0]], %[[VAL_1]] : tensor<2x3x5xf32>, tensor<2x5x7xf32>) outs(%[[VAL_2]] : tensor<2x3x7xf32>) {
+// CHECK:           arith.mulf
+// CHECK:           arith.addf
+
+func.func @batch_matmul(%arg0: tensor<2x3x5xf32>, %arg1: tensor<2x5x7xf32>, %arg2: tensor<2x3x7xf32>) -> tensor<2x3x7xf32> {
+  %0 = linalg.batch_matmul indexing_maps = [
+                            affine_map<(d0, d1, d2, d3) -> (d0, d1, d3)>,
+                            affine_map<(d0, d1, d2, d3) -> (d0, d3, d2)>,
+                            affine_map<(d0, d1, d2, d3) -> (d0, d1, d2)>
+                           ]
+    ins(%arg0, %arg1: tensor<2x3x5xf32>, tensor<2x5x7xf32>)
+    outs(%arg2: tensor<2x3x7xf32>) -> tensor<2x3x7xf32>
+  return %0 : tensor<2x3x7xf32>
+}
+
+// -----
+
+// CHECK: #[[$ACCESS_A:.+]] = affine_map<(d0, d1, d2) -> (d0, d2)>
+// CHECK: #[[$ACCESS_B:.+]] = affine_map<(d0, d1, d2) -> (d2, d1)>
+// CHECK: #[[$ACCESS_C:.+]] = affine_map<(d0, d1, d2) -> (d0, d1)>
+
+// CHECK-LABEL: func.func @contract_matmul(
+// CHECK-SAME:      %[[A:.*]]: memref<3x5xf32>,
+// CHECK-SAME:      %[[B:.*]]: memref<5x7xf32>,
+// CHECK-SAME:      %[[C:.*]]: memref<3x7xf32>) {
+
+// CHECK:         linalg.generic
+// CHECK-SAME:        indexing_maps = [#[[$ACCESS_A]], #[[$ACCESS_B]], #[[$ACCESS_C]]]
+// CHECK-SAME:        iterator_types = ["parallel", "parallel", "reduction"]
+// CHECK-NEXT:    ^{{.+}}(
+// CHECK-NEXT:      arith.mulf
+// CHECK-NEXT:      arith.addf
+// CHECK-NEXT:      linalg.yield
+
+func.func @contract_matmul(%A: memref<3x5xf32>, %B: memref<5x7xf32>, %C: memref<3x7xf32>) {
+  linalg.contract
+      indexing_maps = [affine_map<(d0, d1, d2) -> (d0, d2)>,
+                       affine_map<(d0, d1, d2) -> (d2, d1)>,
+                       affine_map<(d0, d1, d2) -> (d0, d1)>]
+      ins(%A, %B : memref<3x5xf32>, memref<5x7xf32>)
+      outs(%C: memref<3x7xf32>)
+
+  return
+}
+
+// -----
+
+// CHECK: #[[$ACCESS_A:.+]] = affine_map<(d0, d1, d2) -> (d2, d0)>
+// CHECK: #[[$ACCESS_B:.+]] = affine_map<(d0, d1, d2) -> (d1, d2)>
+// CHECK: #[[$ACCESS_C:.+]] = affine_map<(d0, d1, d2) -> (d0, d1)>
+
+// CHECK-LABEL: func.func @contract_matmul_transpose_a_b(
+// CHECK-SAME:      %[[A:.*]]: memref<5x3xf32>,
+// CHECK-SAME:      %[[B:.*]]: memref<7x5xf32>,
+// CHECK-SAME:      %[[C:.*]]: memref<3x7xf32>) {
+
+// CHECK:         linalg.generic
+// CHECK-SAME:        indexing_maps = [#[[$ACCESS_A]], #[[$ACCESS_B]], #[[$ACCESS_C]]]
+// CHECK-SAME:        iterator_types = ["parallel", "parallel", "reduction"]
+// CHECK-NEXT:    ^{{.+}}(
+// CHECK-NEXT:      arith.mulf
+// CHECK-NEXT:      arith.addf
+// CHECK-NEXT:      linalg.yield
+
+func.func @contract_matmul_transpose_a_b(%A: memref<5x3xf32>, %B: memref<7x5xf32>, %C: memref<3x7xf32>) {
+  linalg.contract
+      indexing_maps = [affine_map<(d0, d1, d2) -> (d2, d0)>,
+                       affine_map<(d0, d1, d2) -> (d1, d2)>,
+                       affine_map<(d0, d1, d2) -> (d0, d1)>]
+      ins(%A, %B : memref<5x3xf32>, memref<7x5xf32>)
+      outs(%C: memref<3x7xf32>)
+  return
+}
+
+// -----
+
+// CHECK: #[[$ACCESS_A:.+]] = affine_map<(d0, d1, d2, d3) -> (d0, d1, d3)>
+// CHECK: #[[$ACCESS_B:.+]] = affine_map<(d0, d1, d2, d3) -> (d0, d3, d2)>
+// CHECK: #[[$ACCESS_C:.+]] = affine_map<(d0, d1, d2, d3) -> (d0, d1, d2)>
+
+// CHECK-LABEL: func.func @contract_batch_matmul(
+// CHECK-SAME:      %[[A:.*]]: memref<9x3x5xf32>,
+// CHECK-SAME:      %[[B:.*]]: memref<9x5x7xf32>,
+// CHECK-SAME:      %[[C:.*]]: memref<9x3x7xf32>) {
+
+// CHECK:         linalg.generic
+// CHECK-SAME:        indexing_maps = [#[[$ACCESS_A]], #[[$ACCESS_B]], #[[$ACCESS_C]]]
+// CHECK-SAME:        iterator_types = ["parallel", "parallel", "parallel", "reduction"]
+// CHECK-NEXT:    ^{{.+}}(
+// CHECK-NEXT:      arith.mulf
+// CHECK-NEXT:      arith.addf
+// CHECK-NEXT:      linalg.yield
+
+func.func @contract_batch_matmul(%A: memref<9x3x5xf32>, %B: memref<9x5x7xf32>, %C: memref<9x3x7xf32>) {
+  linalg.contract
+      indexing_maps = [affine_map<(d0, d1, d2, d3) -> (d0, d1, d3)>,
+                       affine_map<(d0, d1, d2, d3) -> (d0, d3, d2)>,
+                       affine_map<(d0, d1, d2, d3) -> (d0, d1, d2)>]
+      ins(%A, %B : memref<9x3x5xf32>, memref<9x5x7xf32>)
+      outs(%C: memref<9x3x7xf32>)
+  return
+}
+
+// -----
+
+// CHECK: #[[$ACCESS_A:.+]] = affine_map<(d0, d1, d2, d3) -> (d0, d1, d3)>
+// CHECK: #[[$ACCESS_B:.+]] = affine_map<(d0, d1, d2, d3) -> (d0, d3, d2)>
+// CHECK: #[[$ACCESS_C:.+]] = affine_map<(d0, d1, d2, d3) -> (d1, d2)>
+
+// CHECK-LABEL: func.func @contract_batch_reduce_matmul(
+// CHECK-SAME:      %[[A:.*]]: memref<9x3x5xf32>,
+// CHECK-SAME:      %[[B:.*]]: memref<9x5x7xf32>,
+// CHECK-SAME:      %[[C:.*]]: memref<3x7xf32>) {
+
+// CHECK:         linalg.generic
+// CHECK-SAME:        indexing_maps = [#[[$ACCESS_A]], #[[$ACCESS_B]], #[[$ACCESS_C]]]
+// CHECK-SAME:        iterator_types = ["reduction", "parallel", "parallel", "reduction"]
+// CHECK-NEXT:    ^{{.+}}(
+// CHECK-NEXT:      arith.mulf
+// CHECK-NEXT:      arith.addf
+// CHECK-NEXT:      linalg.yield
+
+#accessA = affine_map<(d0, d1, d2, d3) -> (d0, d1, d3)>
+#accessB = affine_map<(d0, d1, d2, d3) -> (d0, d3, d2)>
+#accessC = affine_map<(d0, d1, d2, d3) -> (d1, d2)>
+func.func @contract_batch_reduce_matmul(
+    %A: memref<9x3x5xf32>, %B: memref<9x5x7xf32>, %C: memref<3x7xf32>) {
+  linalg.contract
+      indexing_maps = [#accessA, #accessB, #accessC]
+      ins(%A, %B : memref<9x3x5xf32>, memref<9x5x7xf32>)
+      outs(%C: memref<3x7xf32>)
+  return
+}
+
+// -----
+
+// CHECK: #[[$ACCESS_A:.+]] = affine_map<(d0, d1, d2, d3) -> (d0, d3, d1)>
+// CHECK: #[[$ACCESS_B:.+]] = affine_map<(d0, d1, d2, d3) -> (d0, d2, d3)>
+// CHECK: #[[$ACCESS_C:.+]] = affine_map<(d0, d1, d2, d3) -> (d1, d2)>
+
+// CHECK-LABEL: func.func @contract_batch_reduce_matmul_permute_m_with_k_and_k_with_n(
+// CHECK-SAME:      %[[A:.*]]: memref<9x5x3xf32>,
+// CHECK-SAME:      %[[B:.*]]: memref<9x7x5xf32>,
+// CHECK-SAME:      %[[C:.*]]: memref<3x7xf32>) {
+
+// CHECK:         linalg.generic
+// CHECK-SAME:        indexing_maps = [#[[$ACCESS_A]], #[[$ACCESS_B]], #[[$ACCESS_C]]]
+// CHECK-SAME:        iterator_types = ["reduction", "parallel", "parallel", "reduction"]
+// CHECK-NEXT:    ^{{.+}}(
+// CHECK-NEXT:      arith.mulf
+// CHECK-NEXT:      arith.addf
+// CHECK-NEXT:      linalg.yield
+
+#accessA = affine_map<(d0, d1, d2, d3) -> (d0, d3, d1)>
+#accessB = affine_map<(d0, d1, d2, d3) -> (d0, d2, d3)>
+#accessC = affine_map<(d0, d1, d2, d3) -> (d1, d2)>
+func.func @contract_batch_reduce_matmul_permute_m_with_k_and_k_with_n(
+    %A: memref<9x5x3xf32>, %B: memref<9x7x5xf32>, %C: memref<3x7xf32>) {
+  linalg.contract
+      indexing_maps = [#accessA, #accessB, #accessC]
+      ins(%A, %B : memref<9x5x3xf32>, memref<9x7x5xf32>)
+      outs(%C: memref<3x7xf32>)
+  return
+}
+
+// -----
+
+// CHECK: #[[$ACCESS_A_B:.+]] = affine_map<(d0) -> (d0)>
+// CHECK: #[[$ACCESS_C:.+]] = affine_map<(d0) -> ()>
+
+// CHECK-LABEL: func.func @contract_dot(
+// CHECK-SAME:      %[[A:.*]]: memref<9xf32>, %[[B:.*]]: memref<9xf32>,
+// CHECK-SAME:      %[[C:.*]]: memref<f32>) {
+
+// CHECK:         linalg.generic
+// CHECK-SAME:        indexing_maps = [#[[$ACCESS_A_B]], #[[$ACCESS_A_B]], #[[$ACCESS_C]]]
+// CHECK-SAME:        iterator_types = ["reduction"]
+// CHECK-NEXT:    ^{{.+}}(
+// CHECK-NEXT:      arith.mulf
+// CHECK-NEXT:      arith.addf
+// CHECK-NEXT:      linalg.yield
+
+#accessAB = affine_map<(d0) -> (d0)>
+#accessC = affine_map<(d0) -> ()>
+func.func @contract_dot(
+    %A: memref<9xf32>, %B: memref<9xf32>, %C: memref<f32>) {
+  linalg.contract
+      indexing_maps = [#accessAB, #accessAB, #accessC]
+      ins(%A, %B : memref<9xf32>, memref<9xf32>)
+      outs(%C: memref<f32>)
+  return
+}
+
+// -----
+
+// CHECK: #[[$ACCESS_A_B:.+]] = affine_map<(d0, d1, d2) -> (d2)>
+// CHECK: #[[$ACCESS_C:.+]] = affine_map<(d0, d1, d2) -> (d0, d1)>
+
+// CHECK-LABEL: func.func @contract_matmul_bcast_a_b(
+// CHECK-SAME:      %[[A:.*]]: memref<5xf32>, %[[B:.*]]: memref<5xf32>,
+// CHECK-SAME:      %[[C:.*]]: memref<3x7xf32>) {
+
+// CHECK:         linalg.generic
+// CHECK-SAME:        indexing_maps = [#[[$ACCESS_A_B]], #[[$ACCESS_A_B]], #[[$ACCESS_C]]]
+// CHECK-SAME:        iterator_types = ["parallel", "parallel", "reduction"]
+// CHECK-NEXT:    ^{{.+}}(
+// CHECK-NEXT:      arith.mulf
+// CHECK-NEXT:      arith.addf
+// CHECK-NEXT:      linalg.yield
+
+#accessAB = affine_map<(d0, d1, d2) -> (d2)>
+#accessC = affine_map<(d0, d1, d2) -> (d0, d1)>
+func.func @contract_matmul_bcast_a_b(
+    %A: memref<5xf32>, %B: memref<5xf32>, %C: memref<3x7xf32>) {
+  linalg.contract
+      indexing_maps = [#accessAB, #accessAB, #accessC]
+      ins(%A, %B : memref<5xf32>, memref<5xf32>)
+      outs(%C: memref<3x7xf32>)
+  return
+}
diff --git a/mlir/test/Dialect/Linalg/generalize-named-polymorphic-ops.mlir b/mlir/test/Dialect/Linalg/generalize-named-polymorphic-ops.mlir
index c170c5be4abff..bbd6e0fc8e2cc 100644
--- a/mlir/test/Dialect/Linalg/generalize-named-polymorphic-ops.mlir
+++ b/mlir/test/Dialect/Linalg/generalize-named-polymorphic-ops.mlir
@@ -120,6 +120,58 @@ func.func @generalize_matmul_unsigned_tensor_f16f64i32(%A : tensor<16x8xf16>, %B
 
 // -----
 
+func.func @generalize_matmul_as_contraction_tensor_f16f64f32(
+    %A: tensor<16x8xf16>,
+    %B: tensor<8x32xf64>,
+    %C: tensor<16x32xf32>) -> tensor<16x32xf32> {
+  %0 = linalg.contract
+      indexing_maps = [affine_map<(d0, d1, d2) -> (d0, d2)>,
+                       affine_map<(d0, d1, d2) -> (d2, d1)>,
+                      affine_map<(d0, d1, d2) -> (d0, d1)>]
+      ins(%A, %B: tensor<16x8xf16>, tensor<8x32xf64>)
+      outs(%C: tensor<16x32xf32>) -> tensor<16x32xf32>
+  return %0: tensor<16x32xf32>
+}
+
+// CHECK-LABEL: @generalize_matmul_as_contraction_tensor_f16f64f32
+// CHECK:         ^{{.*}}(%[[A_ARG:.+]]: f16, %[[B_ARG:.+]]: f64, %[[C_ARG:.+]]: f32)
+// Verify floating point extension and truncation.
+// CHECK-NEXT:      %[[A_CAST:.+]] = arith.extf %[[A_ARG]] : f16 to f32
+// CHECK-NEXT:      %[[B_CAST:.+]] = arith.truncf %[[B_ARG]] : f64 to f32
+// CHECK-NEXT:      %[[MUL:.+]] = arith.mulf %[[A_CAST]], %[[B_CAST]] : f32
+// CHECK-NEXT:      %[[ADD:.+]] = arith.addf %[[C_ARG]], %[[MUL]] : f32
+// CHECK-NEXT:      linalg.yield %[[ADD]] : f32
+// CHECK-NEXT:    -> tensor<16x32xf32>
+
+// -----
+
+func.func @generalize_matmul_as_contract_with_ext_and_trunc(
+    %A: tensor<24x12xf16>,
+    %B: tensor<12x25xf16>,
+    %C: tensor<24x25xf32>) -> tensor<24x25xf16> {
+  %0 = linalg.contract
+      indexing_maps = [affine_map<(m, n, k) -> (m, k)>,
+                       affine_map<(m, n, k) -> (k, n)>,
+                       affine_map<(m, n, k) -> (m, n)>]
+      ins(%A, %B : tensor<24x12xf16>, tensor<12x25xf16>)
+      outs(%C : tensor<24x25xf32>) -> tensor<24x25xf32>
+  %1 = arith.truncf %0 : tensor<24x25xf32> to tensor<24x25xf16>
+  func.return %1 : tensor<24x25xf16>
+}
+
+// CHECK-LABEL: @generalize_matmul_as_contract_with_ext_and_trunc
+// CHECK:         ^{{.*}}(%[[A_ARG:.+]]: f16, %[[B_ARG:.+]]: f16, %[[C_ARG:.+]]: f32)
+// Verify floating point extension and truncation.
+// CHECK-NEXT:      %[[A_CAST:.+]] = arith.extf %[[A_ARG]] : f16 to f32
+// CHECK-NEXT:      %[[B_CAST:.+]] = arith.extf %[[B_ARG]] : f16 to f32
+// CHECK-NEXT:      %[[MUL:.+]] = arith.mulf %[[A_CAST]], %[[B_CAST]] : f32
+// CHECK-NEXT:      %[[ADD:.+]] = arith.addf %[[C_ARG]], %[[MUL]] : f32
+// CHECK-NEXT:      linalg.yield %[[ADD]] : f32
+// CHECK-NEXT:    -> tensor<24x25xf32>
+// CHECK-NEXT:    %[[RES:.+]] = arith.truncf {{.*}} : tensor<24x25xf32> to tensor<24x25xf16>
+
+// -----
+
 func.func @generalize_pooling_nhwc_max_f32(%input : tensor<1x4x16x1xf32>, %shape: tensor<2x2xf32>, %output: tensor<1x2x4x1xf32>) -> tensor<1x2x4x1xf32> {
   %0 = linalg.pooling_nhwc_max {dilations = dense<[1, 2]> : tensor<2xi64>, strides = dense<[2, 4]> : tensor<2xi64>}
     ins(%input, %shape : tensor<1x4x16x1xf32>, tensor<2x2xf32>) outs(%output : tensor<1x2x4x1xf32>) -> tensor<1x2x4x1xf32>
diff --git a/mlir/test/Dialect/Linalg/invalid.mlir b/mlir/test/Dialect/Linalg/invalid.mlir
index a59472377a732..cff741e75077e 100644
--- a/mlir/test/Dialect/Linalg/invalid.mlir
+++ b/mlir/test/Dialect/Linalg/invalid.mlir
@@ -370,6 +370,24 @@ func.func @invalid_static_matmul(%arg0: memref<2x4xf32>, %arg1: memref<3x4xf32>,
 
 // -----
 
+func.func @invalid_scalar_input_matmul(%arg0: f32, %arg1: memref<3x4xf32>, %arg2: memref<2x4xf32>) {
+  // expected-error @+1 {{'linalg.matmul' op expected operand rank (0) to match the result rank of indexing_map #0 (2)}}
+  linalg.matmul ins(%arg0, %arg1 : f32, memref<3x4xf32>)
+                outs(%arg2 : memref<2x4xf32>)
+  return
+}
+
+// -----
+
+func.func @invalid_scalar_output_matmul(%arg0: memref<2x3xf32>, %arg1: memref<3x4xf32>, %arg2: f32) {
+  // expected-error @+1 {{'linalg.matmul' op operand #2 must be variadic of shaped of any type values, but got 'f32'}}
+  linalg.matmul ins(%arg0, %arg1 : memref<2x3xf32>, memref<3x4xf32>)
+                outs(%arg2 : f32)
+  return
+}
+
+// -----
+
 func.func @invalid_indexing_maps_matmul(%arg0: memref<2x4xf32>, %arg1: memref<3x4xf32>, %arg2: memref<2x4xf32>) {
   // expected-error @+1 {{expected attribute value}}
   linalg.matmul indexing_maps = [
@@ -529,6 +547,104 @@ func.func @invalid_indexing_maps_placement_matmul(%lhs: tensor<4x1xf32>, %rhs: t
 
 // -----
 
+func.func @invalid_indexing_maps_placement_contraction(
+    %lhs: tensor<4x1xf32>, %rhs: tensor<1x64xf32>, %init: tensor<4x64xf32>) {
+  // expected-error @+3 {{custom op 'linalg.contract' expected 'indexing_maps' attribute}}
+  // NB: indexing_maps should be provided before ins and outs
+  linalg.contract
+      ins(%lhs, %rhs : tensor<4x1xf32>, tensor<1x64xf32>)
+      outs(%init : tensor<4x64xf32>)
+      indexing_maps = [affine_map<(d0, d1, d2) -> (d0, d2)>,
+                       affine_map<(d0, d1, d2) -> (d2, d1)>,
+                       affine_map<(d0, d1, d2) -> (d0, d1)>]
+  return
+}
+
+// -----
+
+func.func @invalid_affine_map_in_indexing_maps_contraction(
+    %lhs: tensor<4x1xf32>, %rhs: tensor<1x64xf32>, %init: tensor<4x64xf32>) {
+  // expected-error @+1 {{provided affine_map is not a projected permutation}}
+  linalg.contract
+      indexing_maps = [affine_map<(d0, d1, d2) -> (d0 + d2, d2)>,
+                       affine_map<(d0, d1, d2) -> (d2, d1)>,
+                       affine_map<(d0, d1, d2) -> (d0, d1)>]
+      ins(%lhs, %rhs : tensor<4x1xf32>, tensor<1x64xf32>)
+      outs(%init : tensor<4x64xf32>) -> tensor<4x64xf32>
+  return
+}
+
+// -----
+
+func.func @differing_iteration_space_of_affine_maps_contraction(
+    %lhs: tensor<4x1xf32>, %rhs: tensor<1x64xf32>, %init: tensor<4x64xf32>) {
+  // expected-error @+1 {{iteration spaces of provided affine_maps differ}}
+  linalg.contract
+      indexing_maps = [affine_map<(d0, d1, d2) -> (d0, d2)>,
+                       affine_map<(d0, d1, d2, d3) -> (d2, d1)>,
+                       affine_map<(d0, d1, d2) -> (d0, d1)>]
+      ins(%lhs, %rhs : tensor<4x1xf32>, tensor<1x64xf32>)
+      outs(%init : tensor<4x64xf32>) -> tensor<4x64xf32>
+  return
+}
+
+// -----
+
+func.func @mismatched_ranks_affine_map_and_operand_contraction(
+    %lhs: tensor<4x1x2xf32>, %rhs: tensor<1x64xf32>, %init: tensor<4x64xf32>) {
+  // expected-error @+1 {{ranks of shaped operand and results of corresponding affine_map differ}}
+  linalg.contract
+      indexing_maps = [affine_map<(d0, d1, d2) -> (d0, d2)>,
+                       affine_map<(d0, d1, d2) -> (d2, d1)>,
+                       affine_map<(d0, d1, d2) -> (d0, d1)>]
+      ins(%lhs, %rhs : tensor<4x1x2xf32>, tensor<1x64xf32>)
+      outs(%init : tensor<4x64xf32>) -> tensor<4x64xf32>
+  return
+}
+// -----
+
+func.func @mismatch_type_affine_map_and_operand_contraction(
+    %lhs: f32, %rhs: tensor<4x64xf32>, %init: tensor<4x64xf32>) {
+  // expected-error @+1 {{affine_map specifies shaped access while operand has non-shaped type}}
+  linalg.contract
+      indexing_maps = [affine_map<(d0, d1) -> (d0)>,
+                       affine_map<(d0, d1) -> (d0, d1)>,
+                       affine_map<(d0, d1) -> (d0, d1)>]
+      ins(%lhs, %rhs : f32, tensor<4x64xf32>)
+      outs(%init : tensor<4x64xf32>) -> tensor<4x64xf32>
+  return
+}
+
+// -----
+
+func.func @unused_iteration_space_dim_contraction(
+    %lhs: tensor<4x1xf32>, %rhs: tensor<1x64xf32>, %init: tensor<4x64xf32>) {
+  // expected-error @+1 {{iteration space dim at index 3 not used to access any operand}}
+  linalg.contract
+      indexing_maps = [affine_map<(d0, d1, d2, d3) -> (d0, d2)>,
+                       affine_map<(d0, d1, d2, d3) -> (d2, d1)>,
+                       affine_map<(d0, d1, d2, d3) -> (d0, d1)>]
+      ins(%lhs, %rhs : tensor<4x1xf32>, tensor<1x64xf32>)
+      outs(%init : tensor<4x64xf32>) -> tensor<4x64xf32>
+  return
+}
+
+// -----
+
+func.func @unused_iteration_space_dim_contraction(
+    %lhs: tensor<8x4x1xf32>, %rhs: tensor<1x64xf32>, %init: tensor<4x64xf32>) {
+  // expected-error @+1 {{iteration space dim at index 3 is neither a contracting dim nor of parallel iteration type}}
+  linalg.contract
+      indexing_maps = [affine_map<(d0, d1, d2, d3) -> (d0, d2, d3)>,
+                       affine_map<(d0, d1, d2, d3) -> (d2, d1)>,
+                       affine_map<(d0, d1, d2, d3) -> (d0, d1)>]
+      ins(%lhs, %rhs : tensor<8x4x1xf32>, tensor<1x64xf32>)
+      outs(%init : tensor<4x64xf32>) -> tensor<4x64xf32>
+  return
+}
+
+// -----
+
 func.func @invalid_static_2d_conv(%input : memref<1x3x4x2xf32>, %filter: memref<3x2x2x1xf32>, %output: memref<1x2x3x1xf32>) {
   // expected-error @+1 {{inferred input/output operand #0 has shape's dimension #1 to be greater than or equal to 4, but found 3}}
   linalg.conv_2d_nhwc_hwcf
@@ -1142,3 +1258,204 @@ func.func @winograd_output_transform_output_width(%arg0: tensor<6x6x3x3x2x2xf32>
   %0 = linalg.winograd_output_transform m(4) r(3) ins(%arg0 : tensor<6x6x3x3x2x2xf32>) outs(%arg1 : tensor<2x12x11x2xf32>) -> tensor<2x12x11x2xf32>
   return %0 : tensor<2x12x11x2xf32>
 }
+
+// -----
+
+func.func @indexing_map_size_mismatch_batch_matmul(%arg0: memref<?x?x?xf32>,
+     %arg1: memref<?x?x?xf32>, %arg2: memref<?x?x?xf32>) {
+     // expected-error @+1 {{Indexing_map attribute must have 3 affine maps}}
+     linalg.batch_matmul indexing_maps = [
+      affine_map<(d0, d1, d2, d3) -> (d0, d1, d3)>,
+      affine_map<(d0, d1, d2, d3) -> (d0, d2, d3)>
+    ]
+    ins(%arg0, %arg1 : memref<?x?x?xf32>, memref<?x?x?xf32>)
+    outs(%arg2: memref<?x?x?xf32>)
+    return
+}
+
+// -----
+
+func.func @indexing_map_size_one_batch_matmul(%arg0: memref<?x?x?xf32>,
+     %arg1: memref<?x?x?xf32>, %arg2: memref<?x?x?xf32>) {
+     // expected-error @+1 {{Indexing_map attribute must have 3 affine maps}}
+     linalg.batch_matmul indexing_maps = [
+      affine_map<(d0, d1, d2, d3) -> (d0, d1, d3)>
+    ]
+    ins(%arg0, %arg1 : memref<?x?x?xf32>, memref<?x?x?xf32>)
+    outs(%arg2: memref<?x?x?xf32>)
+    return
+}
+
+// -----
+
+func.func @missing_indexing_map_batch_matmul(%arg0: memref<?x?x?xf32>, %arg1: memref<?x?x?xf32>, %arg2: memref<?x?x?xf32>) {
+  // expected-error @+1 {{expected attribute value}}
+  linalg.batch_matmul indexing_maps = [
+                       ,
+                       affine_map<(d0, d1, d2, d3) -> (d0, d3, d2)>,
+                       affine_map<(d0, d1, d2, d3) -> (d0, d1, d2)>
+                      ]
+                      ins(%arg0, %arg1 : memref<?x?x?xf32>, memref<?x?x?xf32>)
+                      outs(%arg2 :memref<?x?x?xf32>)
+  return
+}
+
+// -----
+
+func.func @invalid_dim_expr_batch_matmul_a(%arg0: memref<?x?x?xf32>, %arg1: memref<?x?x?xf32>, %arg2: memref<?x?x?xf32>) {
+  // expected-error @+1 {{Unexpected result dim expression (outside the set of default result dims)}}
+  linalg.batch_matmul indexing_maps = [
+                       affine_map<(d0, d1, d2, d3) -> (d0, d2, d3)>,
+                       affine_map<(d0, d1, d2, d3) -> (d0, d3, d2)>,
+                       affine_map<(d0, d1, d2, d3) -> (d0, d1, d2)>
+                     ]
+                     ins(%arg0, %arg1 : memref<?x?x?xf32>, memref<?x?x?xf32>) outs(%arg2 :memref<?x?x?xf32>)
+  return
+}
+
+// -----
+
+func.func @invalid_dim_expr_batch_matmul_b(%arg0: memref<?x?x?xf32>, %arg1: memref<?x?x?xf32>, %arg2: memref<?x?x?xf32>) {
+  // expected-error @+1 {{Unexpected result dim expression (outside the set of default result dims)}}
+  linalg.batch_matmul indexing_maps = [
+                       affine_map<(d0, d1, d2, d3) -> (d0, d1, d3)>,
+                       affine_map<(d0, d1, d2, d3) -> (d0, d3, d1)>,
+                       affine_map<(d0, d1, d2, d3) -> (d0, d1, d2)>
+                     ]
+                     ins(%arg0, %arg1 : memref<?x?x?xf32>, memref<?x?x?xf32>) outs(%arg2 :memref<?x?x?xf32>)
+  return
+}
+
+// -----
+
+func.func @invalid_bcast_batch_matmul_a(%arg0: memref<?xf32>, %arg1: memref<?x?x?xf32>, %arg2: memref<?x?x?xf32>) {
+  // expected-error @+1 {{'linalg.batch_matmul' op Invalid broadcast requested}}
+  linalg.batch_matmul indexing_maps = [
+                       affine_map<(d0, d1, d2, d3) -> (d0)>,
+                       affine_map<(d0, d1, d2, d3) -> (d0, d3, d2)>,
+                       affine_map<(d0, d1, d2, d3) -> (d0, d1, d2)>
+                     ]
+                     ins(%arg0, %arg1 : memref<?xf32>, memref<?x?x?xf32>) outs(%arg2: memref<?x?x?xf32>)
+  return
+}
+
+// -----
+
+func.func @invalid_multi_dim_bcast_expr_batch_matmul_a(%arg0: memref<?x?xf32>, %arg1: memref<?x?x?xf32>, %arg2: memref<?x?x?xf32>) {
+  // expected-error @+1 {{'linalg.batch_matmul' op Invalid broadcast requested}}
+  linalg.batch_matmul indexing_maps = [
+                       affine_map<(d0, d1, d2, d3) -> (d0, d3)>,
+                       affine_map<(d0, d1, d2, d3) -> (d0, d3, d2)>,
+                       affine_map<(d0, d1, d2, d3) -> (d0, d1, d2)>
+                     ]
+                     ins(%arg0, %arg1 : memref<?x?xf32>, memref<?x?x?xf32>) outs(%arg2: memref<?x?x?xf32>)
+  return
+}
+
+// -----
+
+func.func @invalid_multi_dim_bcast_expr_batch_matmul_b(%arg0: memref<?x?x?xf32>, %arg1: memref<?x?xf32>, %arg2: memref<?x?x?xf32>) {
+  // expected-error @+1 {{'linalg.batch_matmul' op Invalid broadcast requested}}
+  linalg.batch_matmul indexing_maps = [
+                       affine_map<(d0, d1, d2, d3) -> (d0, d1, d3)>,
+                       affine_map<(d0, d1, d2, d3) -> (d3, d0)>,
+                       affine_map<(d0, d1, d2, d3) -> (d0, d1, d2)>
+                     ]
+                     ins(%arg0, %arg1 : memref<?x?x?xf32>, memref<?x?xf32>) outs(%arg2: memref<?x?x?xf32>)
+  return
+}
+
+// -----
+
+func.func @invalid_bcast_batch_matmul_b(%arg0: memref<?x?x?xf32>, %arg1: memref<?xf32>, %arg2: memref<?x?x?xf32>) {
+  // expected-error @+1 {{'linalg.batch_matmul' op Invalid broadcast requested}}
+  linalg.batch_matmul indexing_maps = [
+                       affine_map<(d0, d1, d2, d3) -> (d0, d1, d3)>,
+                       affine_map<(d0, d1, d2, d3) -> (d2)>,
+                       affine_map<(d0, d1, d2, d3) -> (d0, d1, d2)>
+                     ]
+                     ins(%arg0, %arg1 : memref<?x?x?xf32>, memref<?xf32>) outs(%arg2: memref<?x?x?xf32>)
+  return
+}
+
+// -----
+
+func.func @invalid_batch_dim_batch_matmul_a(%arg0: memref<?x?x?xf32>, %arg1: memref<?x?x?xf32>, %arg2: memref<?x?x?xf32>) {
+  // expected-error @+1 {{'linalg.batch_matmul' op Invalid batch dimension expression}}
+  linalg.batch_matmul indexing_maps = [
+                       affine_map<(d0, d1, d2, d3) -> (d1, d0, d3)>,
+                       affine_map<(d0, d1, d2, d3) -> (d0, d3, d2)>,
+                       affine_map<(d0, d1, d2, d3) -> (d0, d1, d2)>
+                     ]
+                     ins(%arg0, %arg1 : memref<?x?x?xf32>, memref<?x?x?xf32>) outs(%arg2 :memref<?x?x?xf32>)
+  return
+}
+
+// -----
+
+func.func @invalid_batch_dim_batch_matmul_b(%arg0: memref<?x?x?xf32>, %arg1: memref<?x?x?xf32>, %arg2: memref<?x?x?xf32>) {
+  // expected-error @+1 {{'linalg.batch_matmul' op Invalid batch dimension expression}}
+  linalg.batch_matmul indexing_maps = [
+                       affine_map<(d0, d1, d2, d3) -> (d0, d1, d3)>,
+                       affine_map<(d0, d1, d2, d3) -> (d2, d3, d0)>,
+                       affine_map<(d0, d1, d2, d3) -> (d0, d1, d2)>
+                     ]
+                     ins(%arg0, %arg1 : memref<?x?x?xf32>, memref<?x?x?xf32>) outs(%arg2 :memref<?x?x?xf32>)
+  return
+}
+
+// -----
+
+func.func @invalid_A_map_result_num_batch_matmul(%arg0: memref<?x?x?xf32>, %arg1: memref<?x?x?xf32>, %arg2: memref<?x?xf32>) {
+  // expected-error @+1 {{'linalg.batch_matmul' op no. of result dim expressions exceeds 3.}}
+  linalg.batch_matmul indexing_maps = [
+                            affine_map<(d0, d1, d2, d3) -> (d0, d1, d3, d3)>,
+                            affine_map<(d0, d1, d2, d3) -> (d0, d3, d2)>,
+                            affine_map<(d0, d1, d2, d3) -> (d0, d1, d2)>
+                           ]
+    ins(%arg0, %arg1: memref<?x?x?xf32>, memref<?x?x?xf32>)
+    outs(%arg2: memref<?x?xf32>)
+    return
+}
+
+// -----
+
+func.func @invalid_B_map_result_num_batch_matmul(%arg0: memref<?x?x?xf32>, %arg1: memref<?x?x?xf32>, %arg2: memref<?x?xf32>) {
+  // expected-error @+1 {{'linalg.batch_matmul' op no. of result dim expressions exceeds 3.}}
+  linalg.batch_matmul indexing_maps = [
+                            affine_map<(d0, d1, d2, d3) -> (d0, d1, d3)>,
+                            affine_map<(d0, d1, d2, d3) -> (d0, d3, d2, d3)>,
+                            affine_map<(d0, d1, d2, d3) -> (d0, d1, d2)>
+                           ]
+    ins(%arg0, %arg1: memref<?x?x?xf32>, memref<?x?x?xf32>)
+    outs(%arg2: memref<?x?xf32>)
+    return
+}
+
+// -----
+
+func.func @invalid_C_map_result_num_batch_matmul(%arg0: memref<?x?x?xf32>, %arg1: memref<?x?x?xf32>, %arg2: memref<?x?xf32>) {
+  // expected-error @+1 {{'linalg.batch_matmul' op expects 3 dims, but got (2).}}
+  linalg.batch_matmul indexing_maps = [
+                            affine_map<(d0, d1, d2, d3) -> (d0, d1, d3)>,
+                            affine_map<(d0, d1, d2, d3) -> (d0, d3, d2)>,
+                            affine_map<(d0, d1, d2, d3) -> (d1, d2)>
+                           ]
+    ins(%arg0, %arg1: memref<?x?x?xf32>, memref<?x?x?xf32>)
+    outs(%arg2: memref<?x?xf32>)
+    return
+}
+
+// -----
+
+func.func @invalid_C_map_result_dim_batch_matmul(%arg0: memref<?x?x?xf32>, %arg1: memref<?x?x?xf32>, %arg2: memref<?x?x?xf32>) {
+  // expected-error @+1 {{'linalg.batch_matmul' op Invalid output map result dimension.}}
+  linalg.batch_matmul indexing_maps = [
+                            affine_map<(d0, d1, d2, d3) -> (d0, d1, d3)>,
+                            affine_map<(d0, d1, d2, d3) -> (d0, d3, d2)>,
+                            affine_map<(d0, d1, d2, d3) -> (d0, d1, d3)>
+                           ]
+    ins(%arg0, %arg1: memref<?x?x?xf32>, memref<?x?x?xf32>)
+    outs(%arg2: memref<?x?x?xf32>)
+    return
+}
diff --git a/mlir/test/Dialect/Linalg/loops.mlir b/mlir/test/Dialect/Linalg/loops.mlir
index 6286a11c11a21..efe8010cffc91 100644
--- a/mlir/test/Dialect/Linalg/loops.mlir
+++ b/mlir/test/Dialect/Linalg/loops.mlir
@@ -627,6 +627,53 @@ func.func @scalar_code(%arg0: memref<f32>, %arg1 : memref<f32>, %arg2 : memref<f
 //----------------------------------------------------------------------------//
 // Named ops to loops.
 //----------------------------------------------------------------------------//
+func.func @batch_reduce_matmul_as_contract(
+    %A: memref<?x?x?xf32>, %B: memref<?x?x?xf32>, %C: memref<?x?xf32>) {
+  linalg.contract
+      indexing_maps = [affine_map<(d0, d1, d2, d3) -> (d0, d1, d3)>,
+                       affine_map<(d0, d1, d2, d3) -> (d0, d3, d2)>,
+                       affine_map<(d0, d1, d2, d3) -> (d1, d2)>]
+      ins(%A, %B : memref<?x?x?xf32>, memref<?x?x?xf32>)
+      outs(%C : memref<?x?xf32>)
+  return
+}
+// CHECK-LABEL: @batch_reduce_matmul_as_contract
+//  CHECK-SAME: %[[mA:[a-zA-Z0-9]+]]: memref<?x?x?xf32>
+//  CHECK-SAME: %[[mB:[a-zA-Z0-9]+]]: memref<?x?x?xf32>
+//  CHECK-SAME: %[[mC:[a-zA-Z0-9]+]]: memref<?x?xf32>
+//       CHECK: %[[B:.*]] = memref.dim %[[mA]], %c0 : memref<?x?x?xf32>
+//       CHECK: %[[M:.*]] = memref.dim %[[mA]], %c1 : memref<?x?x?xf32>
+//       CHECK: %[[K:.*]] = memref.dim %[[mA]], %c2 : memref<?x?x?xf32>
+//       CHECK: %[[N:.*]] = memref.dim %[[mB]], %c2 : memref<?x?x?xf32>
+//       CHECK: scf.for %[[b:.*]] = %{{.*}} to %[[B]]
+//       CHECK:   scf.for %[[m:.*]] = %{{.*}} to %[[M]]
+//       CHECK:     scf.for %[[n:.*]] = %{{.*}} to %[[N]]
+//       CHECK:       scf.for %[[k:.*]] = %{{.*}} to %[[K]]
+//       CHECK:       %[[va:.*]] = memref.load %[[mA]][%[[b]], %[[m]], %[[k]]] : memref<?x?x?xf32>
+//       CHECK:       %[[vb:.*]] = memref.load %[[mB]][%[[b]], %[[k]], %[[n]]] : memref<?x?x?xf32>
+//       CHECK:       %[[vc:.*]] = memref.load %[[mC]][%[[m]], %[[n]]] : memref<?x?xf32>
+//       CHECK:       %[[inc:.*]] = arith.mulf %[[va]], %[[vb]] : f32
+//       CHECK:       %[[res:.*]] = arith.addf %[[vc]], %[[inc]] : f32
+//       CHECK:       store %[[res]], %[[mC]][%[[m]], %[[n]]] : memref<?x?xf32>
+
+// CHECKPARALLEL-LABEL: @batch_reduce_matmul_as_contract
+//  CHECKPARALLEL-SAME: %[[mA:[a-zA-Z0-9]+]]: memref<?x?x?xf32>
+//  CHECKPARALLEL-SAME: %[[mB:[a-zA-Z0-9]+]]: memref<?x?x?xf32>
+//  CHECKPARALLEL-SAME: %[[mC:[a-zA-Z0-9]+]]: memref<?x?xf32>
+//       CHECKPARALLEL: %[[B:.*]] = memref.dim %[[mA]], %c0 : memref<?x?x?xf32>
+//       CHECKPARALLEL: %[[M:.*]] = memref.dim %[[mA]], %c1 : memref<?x?x?xf32>
+//       CHECKPARALLEL: %[[K:.*]] = memref.dim %[[mA]], %c2 : memref<?x?x?xf32>
+//       CHECKPARALLEL: %[[N:.*]] = memref.dim %[[mB]], %c2 : memref<?x?x?xf32>
+//       CHECKPARALLEL: scf.for %[[b:.*]] = %{{.*}} to %[[B]]
+//       CHECKPARALLEL:   scf.parallel (%[[m:.*]], %[[n:.*]]) = ({{.*}}) to (%[[M]], %[[N]]) step ({{.*}}) {
+//       CHECKPARALLEL:     scf.for %[[k:.*]] = %{{.*}} to %[[K]]
+//       CHECKPARALLEL:         %[[va:.*]] = memref.load %[[mA]][%[[b]], %[[m]], %[[k]]] : memref<?x?x?xf32>
+//       CHECKPARALLEL:         %[[vb:.*]] = memref.load %[[mB]][%[[b]], %[[k]], %[[n]]] : memref<?x?x?xf32>
+//       CHECKPARALLEL:         %[[vc:.*]] = memref.load %[[mC]][%[[m]], %[[n]]] : memref<?x?xf32>
+//       CHECKPARALLEL:         %[[inc:.*]] = arith.mulf %[[va]], %[[vb]] : f32
+//       CHECKPARALLEL:         %[[res:.*]] = arith.addf %[[vc]], %[[inc]] : f32
+//       CHECKPARALLEL:         store %[[res]], %[[mC]][%[[m]], %[[n]]] : memref<?x?xf32>
+
 func.func @named_batch_matmul(%A: memref<?x?x?xf32>, %B: memref<?x?x?xf32>, %C: memref<?x?x?xf32>) {
   linalg.batch_matmul ins(%A, %B : memref<?x?x?xf32>, memref<?x?x?xf32>)
                      outs(%C : memref<?x?x?xf32>)
diff --git a/mlir/test/Dialect/Linalg/named-ops.mlir b/mlir/test/Dialect/Linalg/named-ops.mlir
index 68aa5a85b5e0e..68ea97be911a6 100644
--- a/mlir/test/Dialect/Linalg/named-ops.mlir
+++ b/mlir/test/Dialect/Linalg/named-ops.mlir
@@ -1245,7 +1245,6 @@ func.func @matmul_transpose_a_explicit(%arg0: memref<5x3xf32>, %arg1: memref<5x7
                       ]
                       ins(%arg0, %arg1 : memref<5x3xf32>, memref<5x7xf32>)
                       outs(%arg2: memref<3x7xf32>)
-                      
   return
 }
 
@@ -1259,7 +1258,6 @@ func.func @matmul_transpose_b_explicit(%arg0: memref<3x5xf32>, %arg1: memref<7x5
                       ]
                       ins(%arg0, %arg1 : memref<3x5xf32>, memref<7x5xf32>)
                       outs(%arg2: memref<3x7xf32>)
-                      
   return
 }
 
@@ -1271,7 +1269,7 @@ func.func @matmul_transpose_b_explicit(%arg0: memref<3x5xf32>, %arg1: memref<7x5
 // CHECK-SAME:                                           %[[VAL_0:.*]]: memref<3x5xf32>,
 // CHECK-SAME:                                           %[[VAL_1:.*]]: memref<7x5xf32>,
 // CHECK-SAME:                                           %[[VAL_2:.*]]: memref<3x7xf32>) {
-// CHECK:           linalg.matmul ins(%[[VAL_0]], %[[VAL_1]] : memref<3x5xf32>, memref<7x5xf32>) outs(%[[VAL_2]] : memref<3x7xf32>) indexing_maps = [#[[$ATTR_0]], #[[$ATTR_1]], #[[$ATTR_2]]]
+// CHECK:           linalg.matmul indexing_maps = [#[[$ATTR_0]], #[[$ATTR_1]], #[[$ATTR_2]]] ins(%[[VAL_0]], %[[VAL_1]] : memref<3x5xf32>, memref<7x5xf32>) outs(%[[VAL_2]] : memref<3x7xf32>)
 // CHECK:           return
 // CHECK:         }
 
@@ -1296,7 +1294,7 @@ func.func @matmul_transpose_a_b_explicit(%arg0: memref<5x3xf32>, %arg1: memref<7
 // CHECK-SAME:                                             %[[VAL_0:.*]]: memref<5x3xf32>,
 // CHECK-SAME:                                             %[[VAL_1:.*]]: memref<7x5xf32>,
 // CHECK-SAME:                                             %[[VAL_2:.*]]: memref<3x7xf32>) {
-// CHECK:           linalg.matmul ins(%[[VAL_0]], %[[VAL_1]] : memref<5x3xf32>, memref<7x5xf32>) outs(%[[VAL_2]] : memref<3x7xf32>) indexing_maps = [#[[$ATTR_0]], #[[$ATTR_1]], #[[$ATTR_2]]]
+// CHECK:           linalg.matmul indexing_maps = [#[[$ATTR_0]], #[[$ATTR_1]], #[[$ATTR_2]]] ins(%[[VAL_0]], %[[VAL_1]] : memref<5x3xf32>, memref<7x5xf32>) outs(%[[VAL_2]] : memref<3x7xf32>)
 // CHECK:           return
 // CHECK:         }
 
@@ -1317,6 +1315,7 @@ func.func @matmul_bcast_a(%arg0: memref<5xf32>, %arg1: memref<5x7xf32>, %arg2: m
 // CHECK: #[[$ATTR_2:.+]] = affine_map<(d0, d1, d2) -> (d0, d1)>
 // CHECK-LABEL: func @matmul_bcast_a
 //       CHECK:   linalg.matmul
+//  CHECK-SAME:     indexing_maps = [#[[$ATTR_0]], #[[$ATTR_1]], #[[$ATTR_2]]]
 //  CHECK-SAME:     ins(%{{.+}}, %{{.+}} : memref<5xf32>, memref<5x7xf32>)
 //  CHECK-SAME:     outs(%{{.+}} : memref<3x7xf32>)
 
@@ -1337,6 +1336,7 @@ func.func @matmul_bcast_a_dim1(%arg0: memref<5xf32>, %arg1: memref<5x7xf32>, %ar
 // CHECK: #[[$ATTR_2:.+]] = affine_map<(d0, d1, d2) -> (d0, d1)>
 // CHECK-LABEL: func @matmul_bcast_a_dim1
 //       CHECK:   linalg.matmul
+//  CHECK-SAME:     indexing_maps = [#[[$ATTR_0]], #[[$ATTR_1]], #[[$ATTR_2]]]
 //  CHECK-SAME:     ins(%{{.+}}, %{{.+}} : memref<5xf32>, memref<5x7xf32>)
 //  CHECK-SAME:     outs(%{{.+}} : memref<3x7xf32>)
 
@@ -1357,6 +1357,7 @@ func.func @matmul_bcast_b(%arg0: memref<3x5xf32>, %arg1: memref<5xf32>, %arg2: m
 // CHECK: #[[$ATTR_2:.+]] = affine_map<(d0, d1, d2) -> (d0, d1)>
 // CHECK-LABEL: func @matmul_bcast_b
 //       CHECK:   linalg.matmul
+//  CHECK-SAME:     indexing_maps = [#[[$ATTR_0]], #[[$ATTR_1]], #[[$ATTR_2]]]
 //  CHECK-SAME:     ins(%{{.+}}, %{{.+}} : memref<3x5xf32>, memref<5xf32>)
 //  CHECK-SAME:     outs(%{{.+}} : memref<3x7xf32>)
 
@@ -1378,7 +1379,7 @@ func.func @matmul_bcast_a_b(%arg0: memref<5xf32>, %arg1: memref<5xf32>, %arg2: m
 // CHECK-LABEL:   func.func @matmul_bcast_a_b(
 // CHECK-SAME:                                %[[VAL_0:.*]]: memref<5xf32>, %[[VAL_1:.*]]: memref<5xf32>,
 // CHECK-SAME:                                %[[VAL_2:.*]]: memref<3x7xf32>) {
-// CHECK:           linalg.matmul ins(%[[VAL_0]], %[[VAL_1]] : memref<5xf32>, memref<5xf32>) outs(%[[VAL_2]] : memref<3x7xf32>) indexing_maps = [#[[$ATTR_0]], #[[$ATTR_0]], #[[$ATTR_1]]]
+// CHECK:           linalg.matmul indexing_maps = [#[[$ATTR_0]], #[[$ATTR_0]], #[[$ATTR_1]]] ins(%[[VAL_0]], %[[VAL_1]] : memref<5xf32>, memref<5xf32>) outs(%[[VAL_2]] : memref<3x7xf32>)
 // CHECK:           return
 // CHECK:         }
 
@@ -1399,6 +1400,7 @@ func.func @matmul_bcast_b_dim1(%arg0: memref<3x5xf32>, %arg1: memref<5xf32>, %ar
 // CHECK: #[[$ATTR_2:.+]] = affine_map<(d0, d1, d2) -> (d0, d1)>
 // CHECK-LABEL: func @matmul_bcast_b_dim1
 //       CHECK:   linalg.matmul
+//  CHECK-SAME:     indexing_maps = [#[[$ATTR_0]], #[[$ATTR_1]], #[[$ATTR_2]]]
 //  CHECK-SAME:     ins(%{{.+}}, %{{.+}} : memref<3x5xf32>, memref<5xf32>)
 //  CHECK-SAME:     outs(%{{.+}} : memref<3x7xf32>)
 
@@ -1422,7 +1424,7 @@ func.func @dynamic_matmul_bcast_a(%arg0: memref<?xf32>, %arg1: memref<?x?xf32>,
 // CHECK-SAME:                                      %[[VAL_0:.*]]: memref<?xf32>,
 // CHECK-SAME:                                      %[[VAL_1:.*]]: memref<?x?xf32>,
 // CHECK-SAME:                                      %[[VAL_2:.*]]: memref<?x?xf32>) {
-// CHECK:           linalg.matmul ins(%[[VAL_0]], %[[VAL_1]] : memref<?xf32>, memref<?x?xf32>) outs(%[[VAL_2]] : memref<?x?xf32>) indexing_maps = [#[[$ATTR_0]], #[[$ATTR_1]], #[[$ATTR_2]]]
+// CHECK:           linalg.matmul indexing_maps = [#[[$ATTR_0]], #[[$ATTR_1]], #[[$ATTR_2]]] ins(%[[VAL_0]], %[[VAL_1]] : memref<?xf32>, memref<?x?xf32>) outs(%[[VAL_2]] : memref<?x?xf32>)
 // CHECK:           return
 // CHECK:         }
 
@@ -1446,7 +1448,7 @@ func.func @matmul_bcast_a_transpose_b(%arg0: memref<5xf32>, %arg1: memref<7x5xf3
 // CHECK-SAME:                                  %[[VAL_0:.*]]: memref<5xf32>,
 // CHECK-SAME:                                  %[[VAL_1:.*]]: memref<7x5xf32>,
 // CHECK-SAME:                                  %[[VAL_2:.*]]: memref<3x7xf32>) {
-// CHECK:           linalg.matmul ins(%[[VAL_0]], %[[VAL_1]] : memref<5xf32>, memref<7x5xf32>) outs(%[[VAL_2]] : memref<3x7xf32>) indexing_maps = [#[[$ATTR_0]], #[[$ATTR_1]], #[[$ATTR_2]]]
+// CHECK:           linalg.matmul indexing_maps = [#[[$ATTR_0]], #[[$ATTR_1]], #[[$ATTR_2]]] ins(%[[VAL_0]], %[[VAL_1]] : memref<5xf32>, memref<7x5xf32>) outs(%[[VAL_2]] : memref<3x7xf32>)
 // CHECK:           return
 // CHECK:         }
 
@@ -1470,7 +1472,7 @@ func.func @matmul_bcast_b_transpose_a(%arg0: memref<5x3xf32>, %arg1: memref<5xf3
 // CHECK-SAME:                                          %[[VAL_0:.*]]: memref<5x3xf32>,
 // CHECK-SAME:                                          %[[VAL_1:.*]]: memref<5xf32>,
 // CHECK-SAME:                                          %[[VAL_2:.*]]: memref<3x7xf32>) {
-// CHECK:           linalg.matmul ins(%[[VAL_0]], %[[VAL_1]] : memref<5x3xf32>, memref<5xf32>) outs(%[[VAL_2]] : memref<3x7xf32>) indexing_maps = [#[[$ATTR_0]], #[[$ATTR_1]], #[[$ATTR_2]]]
+// CHECK:           linalg.matmul indexing_maps = [#[[$ATTR_0]], #[[$ATTR_1]], #[[$ATTR_2]]] ins(%[[VAL_0]], %[[VAL_1]] : memref<5x3xf32>, memref<5xf32>) outs(%[[VAL_2]] : memref<3x7xf32>)
 // CHECK:           return
 // CHECK:         }
 
@@ -1487,6 +1489,154 @@ func.func @matmul_transpose_b(%arg0: memref<3x5xf32>, %arg1: memref<7x5xf32>, %a
 
 // -----
 
+// CHECK: #[[$ATTR_0:.+]] = affine_map<(d0, d1, d2, d3) -> (d3)>
+// CHECK: #[[$ATTR_1:.+]] = affine_map<(d0, d1, d2, d3) -> (d0, d3, d2)>
+// CHECK: #[[$ATTR_2:.+]] = affine_map<(d0, d1, d2, d3) -> (d0, d1, d2)>
+
+// CHECK-LABEL:   func.func @batch_matmul_bcast_k_to_fill_missing_dims_A(
+// CHECK-SAME:                                    %[[VAL_0:.*]]: memref<5xf32>,
+// CHECK-SAME:                                    %[[VAL_1:.*]]: memref<2x5x7xf32>,
+// CHECK-SAME:                                    %[[VAL_2:.*]]: memref<2x3x7xf32>) {
+// CHECK:           linalg.batch_matmul ins(%[[VAL_0]], %[[VAL_1]] : memref<5xf32>, memref<2x5x7xf32>) outs(%[[VAL_2]] : memref<2x3x7xf32>) indexing_maps = [#[[$ATTR_0]], #[[$ATTR_1]], #[[$ATTR_2]]]
+// CHECK:           return
+// CHECK:         }
+func.func @batch_matmul_bcast_k_to_fill_missing_dims_A(%arg0: memref<5xf32>, %arg1: memref<2x5x7xf32>, %arg2: memref<2x3x7xf32>) {
+  linalg.batch_matmul indexing_maps = [
+                       affine_map<(d0, d1, d2, d3) -> (d3)>,
+                       affine_map<(d0, d1, d2, d3) -> (d0, d3, d2)>,
+                       affine_map<(d0, d1, d2, d3) -> (d0, d1, d2)>
+                     ]
+                     ins(%arg0, %arg1 : memref<5xf32>, memref<2x5x7xf32>) outs(%arg2: memref<2x3x7xf32>)
+  return
+}
+
+// -----
+
+// CHECK: #[[$ATTR_0:.+]] = affine_map<(d0, d1, d2, d3) -> (d1, d3)>
+// CHECK: #[[$ATTR_1:.+]] = affine_map<(d0, d1, d2, d3) -> (d0, d3, d2)>
+// CHECK: #[[$ATTR_2:.+]] = affine_map<(d0, d1, d2, d3) -> (d0, d1, d2)>
+
+// CHECK-LABEL:   func.func @batch_matmul_bcast_batch_dim_A(
+// CHECK-SAME:                                              %[[VAL_0:.*]]: memref<3x5xf32>,
+// CHECK-SAME:                                              %[[VAL_1:.*]]: memref<2x5x7xf32>,
+// CHECK-SAME:                                              %[[VAL_2:.*]]: memref<2x3x7xf32>) {
+// CHECK:           linalg.batch_matmul ins(%[[VAL_0]], %[[VAL_1]] : memref<3x5xf32>, memref<2x5x7xf32>) outs(%[[VAL_2]] : memref<2x3x7xf32>) indexing_maps = [#[[$ATTR_0]], #[[$ATTR_1]], #[[$ATTR_2]]]
+// CHECK:           return
+// CHECK:         }
+func.func @batch_matmul_bcast_batch_dim_A(%arg0: memref<3x5xf32>, %arg1: memref<2x5x7xf32>, %arg2: memref<2x3x7xf32>) {
+  linalg.batch_matmul indexing_maps = [
+                       affine_map<(d0, d1, d2, d3) -> (d1, d3)>,
+                       affine_map<(d0, d1, d2, d3) -> (d0, d3, d2)>,
+                       affine_map<(d0, d1, d2, d3) -> (d0, d1, d2)>
+                     ]
+                     ins(%arg0, %arg1 : memref<3x5xf32>, memref<2x5x7xf32>) outs(%arg2: memref<2x3x7xf32>)
+  return
+}
+
+// -----
+
+// CHECK: #[[$ATTR_0:.+]] = affine_map<(d0, d1, d2, d3) -> (d0, d1, d3)>
+// CHECK: #[[$ATTR_1:.+]] = affine_map<(d0, d1, d2, d3) -> (d3)>
+// CHECK: #[[$ATTR_2:.+]] = affine_map<(d0, d1, d2, d3) -> (d0, d1, d2)>
+
+// CHECK-LABEL:   func.func @batch_matmul_bcast_batch_and_n_dim_B(
+// CHECK-SAME:                                                    %[[VAL_0:.*]]: memref<2x3x5xf32>,
+// CHECK-SAME:                                                    %[[VAL_1:.*]]: memref<5xf32>,
+// CHECK-SAME:                                                    %[[VAL_2:.*]]: memref<2x3x7xf32>) {
+// CHECK:           linalg.batch_matmul ins(%[[VAL_0]], %[[VAL_1]] : memref<2x3x5xf32>, memref<5xf32>) outs(%[[VAL_2]] : memref<2x3x7xf32>) indexing_maps = [#[[$ATTR_0]], #[[$ATTR_1]], #[[$ATTR_2]]]
+// CHECK:           return
+// CHECK:         }
+func.func @batch_matmul_bcast_batch_and_n_dim_B(%arg0: memref<2x3x5xf32>, %arg1: memref<5xf32>, %arg2: memref<2x3x7xf32>) {
+  linalg.batch_matmul indexing_maps = [
+                       affine_map<(d0, d1, d2, d3) -> (d0, d1, d3)>,
+                       affine_map<(d0, d1, d2, d3) -> (d3)>,
+                       affine_map<(d0, d1, d2, d3) -> (d0, d1, d2)>
+                     ]
+                     ins(%arg0, %arg1 : memref<2x3x5xf32>, memref<5xf32>) outs(%arg2: memref<2x3x7xf32>)
+  return
+}
+
+// -----
+
+// CHECK: #[[$ATTR_0:.+]] = affine_map<(d0, d1, d2, d3) -> (d0, d1, d3)>
+// CHECK: #[[$ATTR_1:.+]] = affine_map<(d0, d1, d2, d3) -> (d3, d2)>
+// CHECK: #[[$ATTR_2:.+]] = affine_map<(d0, d1, d2, d3) -> (d0, d1, d2)>
+
+// CHECK-LABEL:   func.func @batch_matmul_bcast_batch_dim_B(
+// CHECK-SAME:                                              %[[VAL_0:.*]]: memref<2x3x5xf32>,
+// CHECK-SAME:                                              %[[VAL_1:.*]]: memref<5x7xf32>,
+// CHECK-SAME:                                              %[[VAL_2:.*]]: memref<2x3x7xf32>) {
+// CHECK:           linalg.batch_matmul ins(%[[VAL_0]], %[[VAL_1]] : memref<2x3x5xf32>, memref<5x7xf32>) outs(%[[VAL_2]] : memref<2x3x7xf32>) indexing_maps = [#[[$ATTR_0]], #[[$ATTR_1]], #[[$ATTR_2]]]
+// CHECK:           return
+// CHECK:         }
+
+func.func @batch_matmul_bcast_batch_dim_B(%arg0: memref<2x3x5xf32>, %arg1: memref<5x7xf32>, %arg2: memref<2x3x7xf32>) {
+  linalg.batch_matmul indexing_maps = [
+                       affine_map<(d0, d1, d2, d3) -> (d0, d1, d3)>,
+                       affine_map<(d0, d1, d2, d3) -> (d3, d2)>,
+                       affine_map<(d0, d1, d2, d3) -> (d0, d1, d2)>
+                     ]
+                     ins(%arg0, %arg1 : memref<2x3x5xf32>, memref<5x7xf32>) outs(%arg2: memref<2x3x7xf32>)
+  return
+}
+
+// -----
+
+// CHECK-LABEL: func @batch_matmul_explicit_transpose_A
+//       CHECK:   linalg.batch_matmul
+//  CHECK-SAME:     ins(%{{.+}}, %{{.+}} : memref<2x5x3xf32>, memref<2x5x7xf32>)
+//  CHECK-SAME:     outs(%{{.+}} : memref<2x3x7xf32>)
+func.func @batch_matmul_explicit_transpose_A(%arg0: memref<2x5x3xf32>, %arg1: memref<2x5x7xf32>, %arg2: memref<2x3x7xf32>) {
+  linalg.batch_matmul indexing_maps = [
+                       affine_map<(d0, d1, d2, d3) -> (d0, d3, d1)>,
+                       affine_map<(d0, d1, d2, d3) -> (d0, d3, d2)>,
+                       affine_map<(d0, d1, d2, d3) -> (d0, d1, d2)>
+                     ]
+                     ins(%arg0, %arg1 : memref<2x5x3xf32>, memref<2x5x7xf32>) outs(%arg2: memref<2x3x7xf32>)
+  return
+}
+
+// -----
+
+// CHECK-LABEL: func @batch_matmul_explicit_transpose_B
+//       CHECK:   linalg.batch_matmul
+//  CHECK-SAME:     ins(%{{.+}}, %{{.+}} : memref<2x3x5xf32>, memref<2x7x5xf32>)
+//  CHECK-SAME:     outs(%{{.+}} : memref<2x3x7xf32>)
+func.func @batch_matmul_explicit_transpose_B(%arg0: memref<2x3x5xf32>, %arg1: memref<2x7x5xf32>, %arg2: memref<2x3x7xf32>) {
+  linalg.batch_matmul indexing_maps = [
+                       affine_map<(d0, d1, d2, d3) -> (d0, d1, d3)>,
+                       affine_map<(d0, d1, d2, d3) -> (d0, d2, d3)>,
+                       affine_map<(d0, d1, d2, d3) -> (d0, d1, d2)>
+                     ]
+                     ins(%arg0, %arg1 : memref<2x3x5xf32>, memref<2x7x5xf32>) outs(%arg2: memref<2x3x7xf32>)
+  return
+}
+
+// -----
+
+// CHECK: #[[$ATTR_0:.+]] = affine_map<(d0, d1, d2, d3) -> (d1, d3)>
+// CHECK: #[[$ATTR_1:.+]] = affine_map<(d0, d1, d2, d3) -> (d0, d2, d3)>
+// CHECK: #[[$ATTR_2:.+]] = affine_map<(d0, d1, d2, d3) -> (d0, d1, d2)>
+
+// CHECK-LABEL:   func.func @batch_matmul_bcast_A_transpose_B(
+// CHECK-SAME:                                                %[[VAL_0:.*]]: memref<3x5xf32>,
+// CHECK-SAME:                                                %[[VAL_1:.*]]: memref<2x7x5xf32>,
+// CHECK-SAME:                                                %[[VAL_2:.*]]: memref<2x3x7xf32>) {
+// CHECK:           linalg.batch_matmul ins(%[[VAL_0]], %[[VAL_1]] : memref<3x5xf32>, memref<2x7x5xf32>) outs(%[[VAL_2]] : memref<2x3x7xf32>) indexing_maps = [#[[$ATTR_0]], #[[$ATTR_1]], #[[$ATTR_2]]]
+// CHECK:           return
+// CHECK:         }
+func.func @batch_matmul_bcast_A_transpose_B(%arg0: memref<3x5xf32>, %arg1: memref<2x7x5xf32>, %arg2: memref<2x3x7xf32>) {
+  linalg.batch_matmul indexing_maps = [
+                       affine_map<(d0, d1, d2, d3) -> (d1, d3)>,
+                       affine_map<(d0, d1, d2, d3) -> (d0, d2, d3)>,
+                       affine_map<(d0, d1, d2, d3) -> (d0, d1, d2)>
+                     ]
+                     ins(%arg0, %arg1 : memref<3x5xf32>, memref<2x7x5xf32>) outs(%arg2: memref<2x3x7xf32>)
+  return
+}
+
+// -----
+
 // CHECK-LABEL: func @batchmatmul_transpose_a
 //       CHECK:   linalg.batch_matmul_transpose_a
 //  CHECK-SAME:     ins(%{{.+}}, %{{.+}} : memref<2x5x3xf32>, memref<2x5x7xf32>)
@@ -1509,6 +1659,128 @@ func.func @batchmatmul_transpose_b(%arg0: memref<2x3x5xf32>, %arg1: memref<2x7x5
 
 // -----
 
+// CHECK: #[[$ATTR_0:.+]] = affine_map<(d0, d1, d2, d3) -> (d0, d1, d3)>
+// CHECK: #[[$ATTR_1:.+]] = affine_map<(d0, d1, d2, d3) -> (d0, d3, d2)>
+// CHECK: #[[$ATTR_2:.+]] = affine_map<(d0, d1, d2, d3) -> (d0, d1, d2)>
+// CHECK-LABEL: func @contract
+//       CHECK:   linalg.contract
+//  CHECK-SAME:     indexing_maps = [#[[$ATTR_0]], #[[$ATTR_1]], #[[$ATTR_2]]]
+//  CHECK-SAME:     ins(%{{.+}}, %{{.+}} : memref<2x3x5xf32>, memref<2x5x7xf32>)
+//  CHECK-SAME:     outs(%{{.+}} : memref<2x3x7xf32>)
+func.func @contract(
+    %A: memref<2x3x5xf32>, %B: memref<2x5x7xf32>, %C: memref<2x3x7xf32>) {
+  linalg.contract
+      indexing_maps = [affine_map<(d0, d1, d2, d3) -> (d0, d1, d3)>,
+                       affine_map<(d0, d1, d2, d3) -> (d0, d3, d2)>,
+                       affine_map<(d0, d1, d2, d3) -> (d0, d1, d2)>]
+      ins(%A, %B : memref<2x3x5xf32>, memref<2x5x7xf32>)
+      outs(%C: memref<2x3x7xf32>)
+  return
+}
+
+// -----
+
+// CHECK: #[[$ACCESS_A:.+]] = affine_map<(d0, d1, d2) -> (d2)>
+// CHECK: #[[$ACCESS_B:.+]] = affine_map<(d0, d1, d2) -> (d2, d1)>
+// CHECK: #[[$ACCESS_C:.+]] = affine_map<(d0, d1, d2) -> (d0, d1)>
+// CHECK-LABEL: func @contract_matmul_bcast_a
+func.func @contract_matmul_bcast_a(%A: memref<5xf32>, %B: memref<5x7xf32>, %C: memref<3x7xf32>) {
+// CHECK:  linalg.contract
+// CHECK-SAME: indexing_maps = [#[[$ACCESS_A]], #[[$ACCESS_B]], #[[$ACCESS_C]]]
+// CHECK-SAME: ins(%{{.+}}, %{{.+}} : memref<5xf32>, memref<5x7xf32>)
+// CHECK-SAME: outs(%{{.+}} : memref<3x7xf32>)
+  linalg.contract
+      indexing_maps = [affine_map<(d0, d1, d2) -> (d2)>,
+                       affine_map<(d0, d1, d2) -> (d2, d1)>,
+                       affine_map<(d0, d1, d2) -> (d0, d1)>]
+      ins(%A, %B : memref<5xf32>, memref<5x7xf32>)
+      outs(%C: memref<3x7xf32>)
+  return
+}
+
+// -----
+
+// CHECK: #[[$ACCESS_A:.+]] = affine_map<(d0, d1, d2) -> (d0, d2)>
+// CHECK: #[[$ACCESS_B:.+]] = affine_map<(d0, d1, d2) -> (d2)>
+// CHECK: #[[$ACCESS_C:.+]] = affine_map<(d0, d1, d2) -> (d0, d1)>
+// CHECK-LABEL: func @contract_matmul_bcast_b
+func.func @contract_matmul_bcast_b(%A: memref<3x5xf32>, %B: memref<5xf32>, %C: memref<3x7xf32>) {
+// CHECK:  linalg.contract
+// CHECK-SAME: indexing_maps = [#[[$ACCESS_A]], #[[$ACCESS_B]], #[[$ACCESS_C]]]
+// CHECK-SAME: ins(%{{.+}}, %{{.+}} : memref<3x5xf32>, memref<5xf32>)
+// CHECK-SAME: outs(%{{.+}} : memref<3x7xf32>)
+  linalg.contract
+      indexing_maps = [affine_map<(d0, d1, d2) -> (d0, d2)>,
+                       affine_map<(d0, d1, d2) -> (d2)>,
+                       affine_map<(d0, d1, d2) -> (d0, d1)>]
+      ins(%A, %B : memref<3x5xf32>, memref<5xf32>)
+      outs(%C: memref<3x7xf32>)
+  return
+}
+
+// -----
+
+// CHECK: #[[$ACCESS_A:.+]] = affine_map<(d0, d1, d2) -> (d2)>
+// CHECK: #[[$ACCESS_B:.+]] = affine_map<(d0, d1, d2) -> (d0, d1)>
+// CHECK-LABEL: func.func @contract_matmul_bcast_a_b
+func.func @contract_matmul_bcast_a_b(
+    %A: memref<5xf32>, %B: memref<5xf32>, %C: memref<3x7xf32>) {
+// CHECK:  linalg.contract
+// CHECK-SAME: indexing_maps = [#[[$ACCESS_A]], #[[$ACCESS_A]], #[[$ACCESS_B]]]
+// CHECK-SAME: ins(%{{.+}}, %{{.+}} : memref<5xf32>, memref<5xf32>)
+// CHECK-SAME: outs(%{{.+}} : memref<3x7xf32>)
+  linalg.contract
+      indexing_maps = [affine_map<(d0, d1, d2) -> (d2)>,
+                       affine_map<(d0, d1, d2) -> (d2)>,
+                       affine_map<(d0, d1, d2) -> (d0, d1)>]
+      ins(%A, %B : memref<5xf32>, memref<5xf32>)
+      outs(%C: memref<3x7xf32>)
+  return
+}
+
+// -----
+
+// CHECK: #[[$ACCESS_A:.+]] = affine_map<(d0, d1, d2) -> (d2)>
+// CHECK: #[[$ACCESS_B:.+]] = affine_map<(d0, d1, d2) -> (d1, d2)>
+// CHECK: #[[$ACCESS_C:.+]] = affine_map<(d0, d1, d2) -> (d0, d1)>
+// CHECK-LABEL: func.func @contract_matmul_bcast_a_transpose_b
+func.func @contract_matmul_bcast_a_transpose_b(
+    %A: memref<5xf32>, %B: memref<7x5xf32>, %C: memref<3x7xf32>) {
+// CHECK:  linalg.contract
+// CHECK-SAME: indexing_maps = [#[[$ACCESS_A]], #[[$ACCESS_B]], #[[$ACCESS_C]]]
+// CHECK-SAME: ins(%{{.+}}, %{{.+}} : memref<5xf32>, memref<7x5xf32>)
+// CHECK-SAME: outs(%{{.+}} : memref<3x7xf32>)
+  linalg.contract
+      indexing_maps = [affine_map<(d0, d1, d2) -> (d2)>,
+                       affine_map<(d0, d1, d2) -> (d1, d2)>,
+                       affine_map<(d0, d1, d2) -> (d0, d1)>]
+      ins(%A, %B : memref<5xf32>, memref<7x5xf32>)
+      outs(%C: memref<3x7xf32>)
+  return
+}
+
+// -----
+
+// CHECK: #[[$ACCESS_A:.+]] = affine_map<(d0, d1, d2) -> (d2, d0)>
+// CHECK: #[[$ACCESS_B:.+]] = affine_map<(d0, d1, d2) -> (d2)>
+// CHECK: #[[$ACCESS_C:.+]] = affine_map<(d0, d1, d2) -> (d0, d1)>
+// CHECK-LABEL:   func.func @contract_matmul_bcast_b_transpose_a
+func.func @contract_matmul_bcast_b_transpose_a(%A: memref<5x3xf32>, %B: memref<5xf32>, %C: memref<3x7xf32>) {
+// CHECK:      linalg.contract
+// CHECK-SAME:     indexing_maps = [#[[$ACCESS_A]], #[[$ACCESS_B]], #[[$ACCESS_C]]]
+// CHECK-SAME:     ins(%{{.+}}, %{{.+}} : memref<5x3xf32>, memref<5xf32>)
+// CHECK-SAME:     outs(%{{.+}} : memref<3x7xf32>)
+  linalg.contract
+      indexing_maps = [affine_map<(d0, d1, d2) -> (d2, d0)>,
+                       affine_map<(d0, d1, d2) -> (d2)>,
+                       affine_map<(d0, d1, d2) -> (d0, d1)>]
+      ins(%A, %B : memref<5x3xf32>, memref<5xf32>)
+      outs(%C: memref<3x7xf32>)
+  return
+}
+
+// -----
+
 // CHECK-LABEL: func @mmt4d
 func.func @mmt4d(%A: tensor<10x32x8x1xf32>, %B: tensor<80x32x4x1xf32>, %C: tensor<10x80x8x4xf32>) -> tensor<10x80x8x4xf32> {
   // CHECK: %{{.+}} = linalg.mmt4d
diff --git a/mlir/test/Dialect/Linalg/rank-reduce-contraction-ops.mlir b/mlir/test/Dialect/Linalg/rank-reduce-contraction-ops.mlir
index ebdbe70ff46eb..c68a6362f52c5 100644
--- a/mlir/test/Dialect/Linalg/rank-reduce-contraction-ops.mlir
+++ b/mlir/test/Dialect/Linalg/rank-reduce-contraction-ops.mlir
@@ -35,6 +35,23 @@ func.func @singleton_batch_matmul_memref(%arg0 : memref<1x?x?xf32>, %arg1 : memr
 
 // -----
 
+func.func @negative_singleton_batch_matmul_to_matmul_memref(%arg0 : memref<1x?x?xf32>, %arg1 : memref<1x?x?xf32>, %arg2: memref<1x?x?xf32>) {
+  // CHECK-LABEL: @negative_singleton_batch_matmul_to_matmul_memref
+  // CHECK-NOT:   collapse_shape
+  // CHECK-NOT:   linalg.matmul
+  // CHECK-NOT:   expand_shape
+  linalg.batch_matmul indexing_maps = [
+                          affine_map<(d0, d1, d2, d3) -> (d0, d1, d3)>,
+                          affine_map<(d0, d1, d2, d3) -> (d0, d2, d3)>,
+                          affine_map<(d0, d1, d2, d3) -> (d0, d1, d2)>
+                          ]
+      ins(%arg0, %arg1 : memref<1x?x?xf32>, memref<1x?x?xf32>)
+      outs(%arg2 : memref<1x?x?xf32>)
+  return
+}
+
+// -----
+
 func.func @singleton_batch_matvec(%arg0 : tensor<1x128x512xf32>, %arg1 : tensor<1x512xf32>, %arg2: tensor<1x128xf32>) -> tensor<1x128xf32> {
   // CHECK-LABEL: @singleton_batch_matvec
   //  CHECK-SAME:     %[[LHS:[a-zA-Z0-9]+]]: tensor<1x128x512xf32>
@@ -135,6 +152,20 @@ func.func @matmul_to_matvec(%arg0: memref<?x?xf32>, %arg1: memref<?x1xf32>, %arg
 
 // -----
 
+func.func @negative_matmul_to_matvec(%arg0: memref<?xf32>, %arg1: memref<?x1xf32>, %arg2: memref<?x1xf32>) {
+  // CHECK-LABEL: @negative_matmul_to_matvec
+  // CHECK-NOT: linalg.matvec
+    linalg.matmul indexing_maps = [
+                          affine_map<(d0, d1, d2) -> (d2)>,
+                          affine_map<(d0, d1, d2) -> (d2, d1)>,
+                          affine_map<(d0, d1, d2) -> (d0, d1)>
+                          ]
+                          ins(%arg0, %arg1: memref<?xf32>, memref<?x1xf32>) outs(%arg2: memref<?x1xf32>)
+    return
+}
+
+// -----
+
 func.func @matmul_to_vecmat_tensor(%arg0: tensor<1x?xf32>, %arg1: tensor<?x?xf32>, %arg2: tensor<1x?xf32>) -> tensor<1x?xf32> {
   // CHECK-LABEL: @matmul_to_vecmat
   //  CHECK-SAME:     %[[LHS:[a-zA-Z0-9]+]]: tensor<1x?xf32>
diff --git a/mlir/test/Dialect/Linalg/reshape_fusion.mlir b/mlir/test/Dialect/Linalg/reshape_fusion.mlir
index b8df5fc88e199..ef853e4d662a7 100644
--- a/mlir/test/Dialect/Linalg/reshape_fusion.mlir
+++ b/mlir/test/Dialect/Linalg/reshape_fusion.mlir
@@ -37,12 +37,12 @@ func.func @generic_op_reshape_producer_fusion(%arg0 : tensor<?x?x4x?xf32>,
 //      CHECK:   %[[DIM:.+]] = tensor.dim %[[ARG1]], %[[C0]] : tensor<?x?x?xf32>
 //      CHECK:   %[[DIM_0:.+]] = tensor.dim %[[ARG1]], %[[C1]] : tensor<?x?x?xf32>
 //      CHECK:   %[[DIM_1:.+]] = tensor.dim %[[ARG1]], %[[C2]] : tensor<?x?x?xf32>
-//      CHECK:   %[[VAL_0:.+]] = arith.divui %[[DIM_1]], %[[C4]] : index
+//      CHECK:   %[[VAL_0:.+]] = arith.divsi %[[DIM_1]], %[[C4]] : index
 //      CHECK:   %[[T1:.+]] = tensor.expand_shape %[[ARG1]] {{\[\[}}0], [1], [2, 3]] output_shape [%[[DIM]], %[[DIM_0]], %[[VAL_0]], 4] : tensor<?x?x?xf32> into tensor<?x?x?x4xf32>
 //      CHECK:   %[[DIM_2:.+]] = tensor.dim %[[ARG1]], %[[C0]] : tensor<?x?x?xf32>
 //      CHECK:   %[[DIM_3:.+]] = tensor.dim %[[ARG1]], %[[C1]] : tensor<?x?x?xf32>
 //      CHECK:   %[[DIM_4:.+]] = tensor.dim %[[ARG1]], %[[C2]] : tensor<?x?x?xf32>
-//      CHECK:   %[[VAL_1:.+]] = arith.divui %[[DIM_4]], %[[C4]] : index
+//      CHECK:   %[[VAL_1:.+]] = arith.divsi %[[DIM_4]], %[[C4]] : index
 //      CHECK:   %[[T2:.+]] = tensor.expand_shape %[[ARG1]] {{\[\[}}0], [1], [2, 3]] output_shape [%[[DIM_2]], %[[DIM_3]], %[[VAL_1]], 4] : tensor<?x?x?xf32> into tensor<?x?x?x4xf32>
 //      CHECK:   %[[T3:.+]] = linalg.generic
 // CHECK-SAME:     indexing_maps = [#[[MAP5]], #[[MAP6]], #[[MAP7]], #[[MAP6]]]
@@ -93,15 +93,15 @@ func.func @generic_op_reshape_consumer_fusion(%arg0 : tensor<?x?xf32>,
 //      CHECK:   %[[C0:.+]] = arith.constant 0 : index
 //      CHECK:   %[[DIM:.+]] = tensor.dim %[[ARG0]], %[[C0]] : tensor<?x?xf32>
 //      CHECK:   %[[DIM_0:.+]] = tensor.dim %[[ARG0]], %[[C1]] : tensor<?x?xf32>
-//      CHECK:   %[[VAL_0:.+]] = arith.divui %[[DIM_0]], %[[C20]] : index
+//      CHECK:   %[[VAL_0:.+]] = arith.divsi %[[DIM_0]], %[[C20]] : index
 //      CHECK:   %[[T0:.+]] = tensor.expand_shape %[[ARG0]] {{\[\[}}0], [1, 2, 3]] output_shape [%[[DIM]], 4, %[[VAL_0]], 5] : tensor<?x?xf32> into tensor<?x4x?x5xf32>
 //      CHECK:   %[[DIM_1:.+]] = tensor.dim %[[ARG1]], %[[C0]] : tensor<?x?xf32>
 //      CHECK:   %[[DIM_2:.+]] = tensor.dim %[[ARG1]], %[[C1]] : tensor<?x?xf32>
-//      CHECK:   %[[VAL_1:.+]] = arith.divui %[[DIM_2]], %[[C20]] : index
+//      CHECK:   %[[VAL_1:.+]] = arith.divsi %[[DIM_2]], %[[C20]] : index
 //      CHECK:   %[[T1:.+]] = tensor.expand_shape %[[ARG1]] {{\[\[}}0], [1, 2, 3]] output_shape [%[[DIM_1]], 4, %[[VAL_1]], 5] : tensor<?x?xf32> into tensor<?x4x?x5xf32>
 //      CHECK:   %[[DIM_4:.+]] = tensor.dim %[[ARG0]], %[[C0]] : tensor<?x?xf32>
 //      CHECK:   %[[DIM_5:.+]] = tensor.dim %[[ARG0]], %[[C1]] : tensor<?x?xf32>
-//      CHECK:   %[[VAL_2:.+]] = arith.divui %[[DIM_5]], %[[C20]] : index
+//      CHECK:   %[[VAL_2:.+]] = arith.divsi %[[DIM_5]], %[[C20]] : index
 //      CHECK:   %[[T2:.+]] = tensor.expand_shape %[[ARG0]] {{\[\[}}0], [1, 2, 3]] output_shape [%[[DIM_4]], 4, %[[VAL_2]], 5] : tensor<?x?xf32> into tensor<?x4x?x5xf32>
 //      CHECK:   %[[T3:.+]] = linalg.generic
 // CHECK-SAME:     indexing_maps = [#[[MAP2]], #[[MAP2]], #[[MAP3]], #[[MAP2]]]
@@ -144,18 +144,18 @@ func.func @reshape_as_consumer_permutation
 //      CHECK:   %[[DIM:.+]] = tensor.dim %[[ARG0]], %[[C0]] : tensor<?x?x?xf32>
 //      CHECK:   %[[DIM_0:.+]] = tensor.dim %[[ARG0]], %[[C1]] : tensor<?x?x?xf32>
 //      CHECK:   %[[DIM_1:.+]] = tensor.dim %[[ARG0]], %[[C2]] : tensor<?x?x?xf32>
-//      CHECK:   %[[VAL_0:.+]] = arith.divui %[[DIM]], %[[C12]] : index
-//      CHECK:   %[[VAL_1:.+]] = arith.divui %[[DIM_0]], %[[C2]] : index
+//      CHECK:   %[[VAL_0:.+]] = arith.divsi %[[DIM]], %[[C12]] : index
+//      CHECK:   %[[VAL_1:.+]] = arith.divsi %[[DIM_0]], %[[C2]] : index
 //      CHECK:   %[[T0:.+]] = tensor.expand_shape %[[ARG0]] {{\[\[}}0, 1, 2], [3, 4], [5]] output_shape [3, 4, %[[VAL_0]], %[[VAL_1]], 2, %[[DIM_1]]] : tensor<?x?x?xf32> into tensor<3x4x?x?x2x?xf32>
 //      CHECK:   %[[DIM_2:.+]] = tensor.dim %[[ARG1]], %[[C0]] : tensor<?x?xf32>
 //      CHECK:   %[[DIM_3:.+]] = tensor.dim %[[ARG1]], %[[C1]] : tensor<?x?xf32>
-//      CHECK:   %[[VAL_2:.+]] = arith.divui %[[DIM_2]], %[[C12]] : index
+//      CHECK:   %[[VAL_2:.+]] = arith.divsi %[[DIM_2]], %[[C12]] : index
 //      CHECK:   %[[T1:.+]] = tensor.expand_shape %[[ARG1]] {{\[\[}}0, 1, 2], [3]] output_shape [3, 4, %[[VAL_2]], %[[DIM_3]]] : tensor<?x?xf32> into tensor<3x4x?x?xf32>
 //      CHECK:   %[[DIM_5:.+]] = tensor.dim %[[ARG0]], %[[C0]] : tensor<?x?x?xf32>
 //      CHECK:   %[[DIM_6:.+]] = tensor.dim %[[ARG0]], %[[C1]] : tensor<?x?x?xf32>
 //      CHECK:   %[[DIM_7:.+]] = tensor.dim %[[ARG0]], %[[C2]] : tensor<?x?x?xf32>
-//      CHECK:   %[[VAL_3:.+]] = arith.divui %[[DIM_5]], %[[C2]] : index
-//      CHECK:   %[[VAL_4:.+]] = arith.divui %[[DIM_7]], %[[C12]] : index
+//      CHECK:   %[[VAL_3:.+]] = arith.divsi %[[DIM_5]], %[[C2]] : index
+//      CHECK:   %[[VAL_4:.+]] = arith.divsi %[[DIM_7]], %[[C12]] : index
 //      CHECK:   %[[T2:.+]] = tensor.expand_shape %[[ARG0]] {{\[\[}}0, 1], [2], [3, 4, 5]] output_shape [%[[VAL_3]], 2, %[[DIM_6]], 3, 4, %[[VAL_4]]] : tensor<?x?x?xf32> into tensor<?x2x?x3x4x?xf32>
 //      CHECK:   %[[T3:.+]] = linalg.generic
 // CHECK-SAME:     indexing_maps = [#[[MAP8]], #[[MAP9]], #[[MAP10]]]
@@ -463,15 +463,15 @@ func.func @generic_op_reshape_consumer_fusion_projected(%arg0 : tensor<?x?xf32>,
 //      CHECK:   %[[C0:.+]] = arith.constant 0 : index
 //      CHECK:   %[[DIM:.+]] = tensor.dim %[[ARG0]], %[[C0]] : tensor<?x?xf32>
 //      CHECK:   %[[DIM_0:.+]] = tensor.dim %[[ARG0]], %[[C1]] : tensor<?x?xf32>
-//      CHECK:   %[[VAL_0:.+]] = arith.divui %[[DIM]], %[[C20]] : index
+//      CHECK:   %[[VAL_0:.+]] = arith.divsi %[[DIM]], %[[C20]] : index
 //      CHECK:   %[[T0:.+]] = tensor.expand_shape %[[ARG0]] {{\[\[}}0, 1, 2], [3]] output_shape [%[[VAL_0]], 4, 5, %[[DIM_0]]] : tensor<?x?xf32> into tensor<?x4x5x?xf32>
 //      CHECK:   %[[DIM_1:.+]] = tensor.dim %[[ARG1]], %[[C0]] : tensor<?x?xf32>
 //      CHECK:   %[[DIM_2:.+]] = tensor.dim %[[ARG1]], %[[C1]] : tensor<?x?xf32>
-//      CHECK:   %[[VAL_1:.+]] = arith.divui %[[DIM_1]], %[[C20]] : index
+//      CHECK:   %[[VAL_1:.+]] = arith.divsi %[[DIM_1]], %[[C20]] : index
 //      CHECK:   %[[T1:.+]] = tensor.expand_shape %[[ARG1]] {{\[\[}}0, 1, 2], [3]] output_shape [%[[VAL_1]], 4, 5, %[[DIM_2]]] : tensor<?x?xf32> into tensor<?x4x5x?xf32>
 //      CHECK:   %[[DIM_4:.+]] = tensor.dim %[[ARG0]], %[[C0]] : tensor<?x?xf32>
 //      CHECK:   %[[DIM_5:.+]] = tensor.dim %[[ARG0]], %[[C1]] : tensor<?x?xf32>
-//      CHECK:   %[[VAL_2:.+]] = arith.divui %[[DIM_5]], %[[C20]] : index
+//      CHECK:   %[[VAL_2:.+]] = arith.divsi %[[DIM_5]], %[[C20]] : index
 //      CHECK:   %[[T2:.+]] = tensor.expand_shape %[[ARG0]] {{\[\[}}0], [1, 2, 3]] output_shape [%[[DIM_4]], %[[VAL_2]], 4, 5] : tensor<?x?xf32> into tensor<?x?x4x5xf32>
 //      CHECK:   %[[T3:.+]] = linalg.generic
 // CHECK-SAME:     indexing_maps = [#[[MAP4]], #[[MAP4]], #[[MAP5]]]
@@ -569,24 +569,24 @@ func.func @reshape_as_consumer_permutation_with_multiple_results
 //       CHECK:   %[[DIM:.+]] = tensor.dim %[[ARG0]], %[[C0]] : tensor<?x?x?xf32>
 //       CHECK:   %[[DIM_0:.+]] = tensor.dim %[[ARG0]], %[[C1]] : tensor<?x?x?xf32>
 //       CHECK:   %[[DIM_1:.+]] = tensor.dim %[[ARG0]], %[[C2]] : tensor<?x?x?xf32>
-//       CHECK:   %[[VAL_0:.+]] = arith.divui %[[DIM]], %[[C12]] : index
-//       CHECK:   %[[VAL_1:.+]] = arith.divui %[[DIM_0]], %[[C2]] : index
+//       CHECK:   %[[VAL_0:.+]] = arith.divsi %[[DIM]], %[[C12]] : index
+//       CHECK:   %[[VAL_1:.+]] = arith.divsi %[[DIM_0]], %[[C2]] : index
 //       CHECK:   %[[RESHAPE0:.+]] = tensor.expand_shape %[[ARG0]] {{\[\[}}0, 1, 2], [3, 4], [5]] output_shape [3, 4, %[[VAL_0]], %[[VAL_1]], 2, %[[DIM_1]]] : tensor<?x?x?xf32> into tensor<3x4x?x?x2x?xf32>
 //       CHECK:   %[[DIM_2:.+]] = tensor.dim %[[ARG1]], %[[C0]] : tensor<?x?xf32>
 //       CHECK:   %[[DIM_3:.+]] = tensor.dim %[[ARG1]], %[[C1]] : tensor<?x?xf32>
-//       CHECK:   %[[VAL_2:.+]] = arith.divui %[[DIM_2]], %[[C12]] : index
+//       CHECK:   %[[VAL_2:.+]] = arith.divsi %[[DIM_2]], %[[C12]] : index
 //       CHECK:   %[[RESHAPE1:.+]] = tensor.expand_shape %[[ARG1]] {{\[\[}}0, 1, 2], [3]] output_shape [3, 4, %[[VAL_2]], %[[DIM_3]]] : tensor<?x?xf32> into tensor<3x4x?x?xf32>
 //       CHECK:   %[[DIM_5:.+]] = tensor.dim %[[ARG0]], %[[C0]] : tensor<?x?x?xf32>
 //       CHECK:   %[[DIM_6:.+]] = tensor.dim %[[ARG0]], %[[C1]] : tensor<?x?x?xf32>
 //       CHECK:   %[[DIM_7:.+]] = tensor.dim %[[ARG0]], %[[C2]] : tensor<?x?x?xf32>
-//       CHECK:   %[[VAL_3:.+]] = arith.divui %[[DIM_5]], %[[C2]] : index
-//       CHECK:   %[[VAL_4:.+]] = arith.divui %[[DIM_7]], %[[C12]] : index
+//       CHECK:   %[[VAL_3:.+]] = arith.divsi %[[DIM_5]], %[[C2]] : index
+//       CHECK:   %[[VAL_4:.+]] = arith.divsi %[[DIM_7]], %[[C12]] : index
 //       CHECK:   %[[RESHAPE2:.+]] = tensor.expand_shape %[[ARG0]] {{\[\[}}0, 1], [2], [3, 4, 5]] output_shape [%[[VAL_3]], 2, %[[DIM_6]], 3, 4, %[[VAL_4]]] : tensor<?x?x?xf32> into tensor<?x2x?x3x4x?xf32>
 //       CHECK:   %[[DIM_9:.+]] = tensor.dim %[[ARG0]], %[[C0]] : tensor<?x?x?xf32>
 //       CHECK:   %[[DIM_10:.+]] = tensor.dim %[[ARG0]], %[[C1]] : tensor<?x?x?xf32>
 //       CHECK:   %[[DIM_11:.+]] = tensor.dim %[[ARG0]], %[[C2]] : tensor<?x?x?xf32>
-//       CHECK:   %[[VAL_5:.+]] = arith.divui %[[DIM_10]], %[[C2]] : index
-//       CHECK:   %[[VAL_6:.+]] = arith.divui %[[DIM_11]], %[[C12]] : index
+//       CHECK:   %[[VAL_5:.+]] = arith.divsi %[[DIM_10]], %[[C2]] : index
+//       CHECK:   %[[VAL_6:.+]] = arith.divsi %[[DIM_11]], %[[C12]] : index
 //       CHECK:   %[[RESHAPE3:.+]] = tensor.expand_shape %[[ARG0]] {{\[\[}}0], [1, 2], [3, 4, 5]] output_shape [%[[DIM_9]], %[[VAL_5]], 2, 3, 4, %[[VAL_6]]] : tensor<?x?x?xf32> into tensor<?x?x2x3x4x?xf32>
 //       CHECK:   %[[GENERIC:.+]]:2 = linalg.generic
 //  CHECK-SAME:      indexing_maps = [#[[MAP0]], #[[MAP1]], #[[MAP2]], #[[MAP3]]]
@@ -667,11 +667,11 @@ func.func @generic_op_reshape_consumer_fusion_reduction(%arg0 : tensor<?x?xf32>,
 //      CHECK:   %[[C0:.+]] = arith.constant 0 : index
 //      CHECK:   %[[DIM:.+]] = tensor.dim %[[ARG1]], %[[C0]] : tensor<?x?xf32>
 //      CHECK:   %[[DIM_0:.+]] = tensor.dim %[[ARG1]], %[[C1]] : tensor<?x?xf32>
-//      CHECK:   %[[VAL_0:.+]] = arith.divui %[[DIM]], %[[C20]] : index
+//      CHECK:   %[[VAL_0:.+]] = arith.divsi %[[DIM]], %[[C20]] : index
 //      CHECK:   %[[T1:.+]] = tensor.expand_shape %[[ARG1]] {{\[\[}}0, 1, 2], [3]] output_shape [%[[VAL_0]], 4, 5, %[[DIM_0]]] : tensor<?x?xf32> into tensor<?x4x5x?xf32>
 //      CHECK:   %[[DIM_1:.+]] = tensor.dim %[[ARG2]], %[[C0]] : tensor<?x?xf32>
 //      CHECK:   %[[DIM_2:.+]] = tensor.dim %[[ARG2]], %[[C1]] : tensor<?x?xf32>
-//      CHECK:   %[[VAL_1:.+]] = arith.divui %[[DIM_2]], %[[C20]] : index
+//      CHECK:   %[[VAL_1:.+]] = arith.divsi %[[DIM_2]], %[[C20]] : index
 //      CHECK:   %[[T2:.+]] = tensor.expand_shape %[[ARG2]] {{\[\[}}0], [1, 2, 3]] output_shape [%[[DIM_1]], %[[VAL_1]], 4, 5] : tensor<?x?xf32> into tensor<?x?x4x5xf32>
 //      CHECK:   %[[T3:.+]] = linalg.generic
 // CHECK-SAME:     indexing_maps = [#[[MAP0]], #[[MAP1]], #[[MAP2]]]
@@ -719,13 +719,13 @@ func.func @generic_op_reshape_producer_fusion_with_reduction(%arg0 : tensor<?x7x
 //      CHECK:   %[[C0:.+]] = arith.constant 0 : index
 //      CHECK:   %[[DIM:.+]] = tensor.dim %[[ARG1]], %[[C0]] : tensor<?x4x?xf32>
 //      CHECK:   %[[DIM_0:.+]] = tensor.dim %[[ARG1]], %[[C2]] : tensor<?x4x?xf32>
-//      CHECK:   %[[VAL_0:.+]] = arith.divui %[[DIM]], %[[C8]] : index
-//      CHECK:   %[[VAL_1:.+]] = arith.divui %[[DIM_0]], %[[C7]] : index
+//      CHECK:   %[[VAL_0:.+]] = arith.divsi %[[DIM]], %[[C8]] : index
+//      CHECK:   %[[VAL_1:.+]] = arith.divsi %[[DIM_0]], %[[C7]] : index
 //      CHECK:   %[[T1:.+]] = tensor.expand_shape %[[ARG1]] {{\[\[}}0, 1], [2], [3, 4]] output_shape [%[[VAL_0]], 8, 4, %[[VAL_1]], 7] : tensor<?x4x?xf32> into tensor<?x8x4x?x7xf32>
 //      CHECK:   %[[DIM_1:.+]] = tensor.dim %[[ARG2]], %[[C0]] : tensor<?x?xf32>
 //      CHECK:   %[[DIM_2:.+]] = tensor.dim %[[ARG2]], %[[C1]] : tensor<?x?xf32>
-//      CHECK:   %[[VAL_2:.+]] = arith.divui %[[DIM_1]], %[[C8]] : index
-//      CHECK:   %[[VAL_3:.+]] = arith.divui %[[DIM_2]], %[[C7]] : index
+//      CHECK:   %[[VAL_2:.+]] = arith.divsi %[[DIM_1]], %[[C8]] : index
+//      CHECK:   %[[VAL_3:.+]] = arith.divsi %[[DIM_2]], %[[C7]] : index
 //      CHECK:   %[[T2:.+]] = tensor.expand_shape %[[ARG2]] {{\[\[}}0, 1], [2, 3]] output_shape [%[[VAL_2]], 8, %[[VAL_3]], 7] : tensor<?x?xf32> into tensor<?x8x?x7xf32>
 //      CHECK:   %[[T3:.+]] = linalg.generic
 // CHECK-SAME:     indexing_maps = [#[[$MAP0]], #[[$MAP1]], #[[$MAP2]]]
@@ -764,15 +764,15 @@ func.func @linalg_add_reshape_consumer_fusion(%arg0 : tensor<?x?xf32>,
 //      CHECK:   %[[C0:.+]] = arith.constant 0 : index
 //      CHECK:   %[[DIM:.+]] = tensor.dim %[[ARG0]], %[[C0]] : tensor<?x?xf32>
 //      CHECK:   %[[DIM_0:.+]] = tensor.dim %[[ARG0]], %[[C1]] : tensor<?x?xf32>
-//      CHECK:   %[[VAL_0:.+]] = arith.divui %[[DIM_0]], %[[C20]] : index
+//      CHECK:   %[[VAL_0:.+]] = arith.divsi %[[DIM_0]], %[[C20]] : index
 //      CHECK:   %[[T1:.+]] = tensor.expand_shape %[[ARG0]] {{\[\[}}0], [1, 2, 3]] output_shape [%[[DIM]], %[[VAL_0]], 4, 5] : tensor<?x?xf32> into tensor<?x?x4x5xf32>
 //      CHECK:   %[[DIM_1:.+]] = tensor.dim %[[ARG1]], %[[C0]] : tensor<?x?xf32>
 //      CHECK:   %[[DIM_2:.+]] = tensor.dim %[[ARG1]], %[[C1]] : tensor<?x?xf32>
-//      CHECK:   %[[VAL_1:.+]] = arith.divui %[[DIM_2]], %[[C20]] : index
+//      CHECK:   %[[VAL_1:.+]] = arith.divsi %[[DIM_2]], %[[C20]] : index
 //      CHECK:   %[[T2:.+]] = tensor.expand_shape %[[ARG1]] {{\[\[}}0], [1, 2, 3]] output_shape [%[[DIM_1]], %[[VAL_1]], 4, 5] : tensor<?x?xf32> into tensor<?x?x4x5xf32>
 //      CHECK:   %[[DIM_4:.+]] = tensor.dim %[[ARG2]], %[[C0]] : tensor<?x?xf32>
 //      CHECK:   %[[DIM_5:.+]] = tensor.dim %[[ARG2]], %[[C1]] : tensor<?x?xf32>
-//      CHECK:   %[[VAL_2:.+]] = arith.divui %[[DIM_5]], %[[C20]] : index
+//      CHECK:   %[[VAL_2:.+]] = arith.divsi %[[DIM_5]], %[[C20]] : index
 //      CHECK:   %[[T3:.+]] = tensor.expand_shape %[[ARG2]] {{\[\[}}0], [1, 2, 3]] output_shape [%[[DIM_4]], %[[VAL_2]], 4, 5] : tensor<?x?xf32> into tensor<?x?x4x5xf32>
 //      CHECK:   %[[T4:.+]] = linalg.generic
 // CHECK-SAME:     indexing_maps = [#[[MAP]], #[[MAP]], #[[MAP]]]
@@ -809,13 +809,13 @@ func.func @linalg_add_reshape_producer_fusion(%arg0 : tensor<?x7x?x8xf32>,
 //      CHECK:   %[[C0:.+]] = arith.constant 0 : index
 //      CHECK:   %[[DIM:.+]] = tensor.dim %[[ARG1]], %[[C0]] : tensor<?x?xf32>
 //      CHECK:   %[[DIM_0:.+]] = tensor.dim %[[ARG1]], %[[C1]] : tensor<?x?xf32>
-//      CHECK:   %[[VAL_0:.+]] = arith.divui %[[DIM]], %[[C7]] : index
-//      CHECK:   %[[VAL_1:.+]] = arith.divui %[[DIM_0]], %[[C8]] : index
+//      CHECK:   %[[VAL_0:.+]] = arith.divsi %[[DIM]], %[[C7]] : index
+//      CHECK:   %[[VAL_1:.+]] = arith.divsi %[[DIM_0]], %[[C8]] : index
 //      CHECK:   %[[T1:.+]] = tensor.expand_shape %[[ARG1]] {{\[\[}}0, 1], [2, 3]] output_shape [%[[VAL_0]], 7, %[[VAL_1]], 8] : tensor<?x?xf32> into tensor<?x7x?x8xf32>
 //      CHECK:   %[[DIM_1:.+]] = tensor.dim %[[ARG2]], %[[C0]] : tensor<?x?xf32>
 //      CHECK:   %[[DIM_2:.+]] = tensor.dim %[[ARG2]], %[[C1]] : tensor<?x?xf32>
-//      CHECK:   %[[VAL_2:.+]] = arith.divui %[[DIM_1]], %[[C7]] : index
-//      CHECK:   %[[VAL_3:.+]] = arith.divui %[[DIM_2]], %[[C8]] : index
+//      CHECK:   %[[VAL_2:.+]] = arith.divsi %[[DIM_1]], %[[C7]] : index
+//      CHECK:   %[[VAL_3:.+]] = arith.divsi %[[DIM_2]], %[[C8]] : index
 //      CHECK:   %[[T2:.+]] = tensor.expand_shape %[[ARG2]] {{\[\[}}0, 1], [2, 3]] output_shape [%[[VAL_2]], 7, %[[VAL_3]], 8] : tensor<?x?xf32> into tensor<?x7x?x8xf32>
 //      CHECK:   %[[T3:.+]] = linalg.generic
 // CHECK-SAME:     indexing_maps = [#[[$MAP]], #[[$MAP]], #[[$MAP]]]
diff --git a/mlir/test/Dialect/Linalg/resolve-shaped-type-result-dims.mlir b/mlir/test/Dialect/Linalg/resolve-shaped-type-result-dims.mlir
index 8fb84248c9613..3bc1f56d816d7 100644
--- a/mlir/test/Dialect/Linalg/resolve-shaped-type-result-dims.mlir
+++ b/mlir/test/Dialect/Linalg/resolve-shaped-type-result-dims.mlir
@@ -210,15 +210,12 @@ func.func @dim_reshape_expansion(%arg0 : tensor<6x5x?xf32>, %sz0: index) -> (ind
   %3 = tensor.dim %0, %c4 : tensor<2x3x5x4x?x7xf32>
   return %1, %2, %3 : index, index, index
 }
-//      CHECK: #[[MAP:.+]] = affine_map<()[s0] -> (s0 floordiv 28)>
 //      CHECK: func @dim_reshape_expansion
 // CHECK-SAME:   %[[ARG0:[a-zA-Z0-9_]+]]: tensor<6x5x?xf32>
-//  CHECK-DAG:   %[[C2:.+]] = arith.constant 2 : index
+// CHECK-SAME:   %[[ARG1:.+]]: index
 //  CHECK-DAG:   %[[C3:.+]] = arith.constant 3 : index
 //  CHECK-DAG:   %[[C4:.+]] = arith.constant 4 : index
-//      CHECK:   %[[D0:.+]] = tensor.dim %[[ARG0]], %[[C2]]
-//      CHECK:   %[[D1:.+]] = affine.apply #[[MAP]]()[%[[D0]]]
-//      CHECK:   return %[[C3]], %[[C4]], %[[D1]]
+//      CHECK:   return %[[C3]], %[[C4]], %[[ARG1]]
 
 // -----
 
diff --git a/mlir/test/Dialect/Linalg/roundtrip.mlir b/mlir/test/Dialect/Linalg/roundtrip.mlir
index 1b8969bd11559..dc556761b09e5 100644
--- a/mlir/test/Dialect/Linalg/roundtrip.mlir
+++ b/mlir/test/Dialect/Linalg/roundtrip.mlir
@@ -277,22 +277,34 @@ func.func @generic_region(%arg0: memref<?x?xvector<3x4xi4>, strided<[?, 1], offs
 
 // -----
 
-
+#accessA = affine_map<(batch, m, n, k) -> (batch, m, k)>
+#accessB = affine_map<(batch, m, n, k) -> (batch, k, n)>
+#accessC = affine_map<(batch, m, n, k) -> (batch, m, n)>
 func.func @named_ops(%a3: memref<?x?x?xf32>, %b3: memref<?x?x?xf32>, %c3: memref<?x?x?xf32>,
                 %ta3: tensor<?x?x?xf32>, %tb3: tensor<?x?x?xf32>, %tc3: tensor<?x?x?xf32>)
-  -> (tensor<?x?x?xf32>)
+  -> (tensor<?x?x?xf32>, tensor<?x?x?xf32>)
 {
   linalg.batch_matmul ins(%a3, %b3: memref<?x?x?xf32>, memref<?x?x?xf32>)
                      outs(%c3: memref<?x?x?xf32>)
+  linalg.contract
+      indexing_maps = [#accessA, #accessB, #accessC]
+      ins(%a3, %b3: memref<?x?x?xf32>, memref<?x?x?xf32>)
+      outs(%c3: memref<?x?x?xf32>)
   %res1 = linalg.batch_matmul
                       ins(%ta3, %tb3: tensor<?x?x?xf32>, tensor<?x?x?xf32>)
                      outs(%tc3: tensor<?x?x?xf32>)
                   -> tensor<?x?x?xf32>
-  return %res1 : tensor<?x?x?xf32>
+  %res2 = linalg.contract
+      indexing_maps = [#accessA, #accessB, #accessC]
+      ins(%ta3, %tb3: tensor<?x?x?xf32>, tensor<?x?x?xf32>)
+      outs(%tc3: tensor<?x?x?xf32>) -> tensor<?x?x?xf32>
+  return %res1, %res2 : tensor<?x?x?xf32>, tensor<?x?x?xf32>
 }
 // CHECK-LABEL: func @named_ops
 //       CHECK:   linalg.batch_matmul
+//       CHECK:   linalg.contract
 //       CHECK:   linalg.batch_matmul
+//       CHECK:   linalg.contract
 
 // -----
 
diff --git a/mlir/test/Dialect/Linalg/tile-tensors.mlir b/mlir/test/Dialect/Linalg/tile-tensors.mlir
index 8f13c69070457..557233d8aa3ec 100644
--- a/mlir/test/Dialect/Linalg/tile-tensors.mlir
+++ b/mlir/test/Dialect/Linalg/tile-tensors.mlir
@@ -37,6 +37,53 @@ module attributes {transform.with_named_sequence} {
 
 // -----
 
+// CHECK:       #[[$MAP0:.*]] = affine_map<(d0, d1, d2) -> (d0, d2)>
+// CHECK-NEXT:  #[[$MAP1:.*]] = affine_map<(d0, d1, d2) -> (d2, d1)>
+// CHECK-NEXT:  #[[$MAP2:.*]] = affine_map<(d0, d1, d2) -> (d0, d1)>
+#access_maps = [affine_map<(d0, d1, d2) -> (d0, d2)>,
+                affine_map<(d0, d1, d2) -> (d2, d1)>,
+                affine_map<(d0, d1, d2) -> (d0, d1)>]
+
+// CHECK-LABEL: func @matmul_as_contract_tensors(
+// CHECK-SAME:    %[[TA:[0-9a-z]+]]: tensor<?x?xf32>
+// CHECK-SAME:    %[[TB:[0-9a-z]+]]: tensor<?x?xf32>
+// CHECK-SAME:    %[[TC:[0-9a-z]+]]: tensor<?x?xf32>) -> tensor<?x?xf32> {
+func.func @matmul_as_contract_tensors(
+  %A: tensor<?x?xf32>, %B: tensor<?x?xf32>, %C: tensor<?x?xf32>)
+    -> tensor<?x?xf32> {
+//      CHECK: %[[TD0:.*]] = scf.for {{.*}} to {{.*}} step {{.*}} iter_args(%[[TC0:.*]] = %[[TC]]) -> (tensor<?x?xf32>) {
+//      CHECK:   %[[TD1:.*]] = scf.for {{.*}} to {{.*}} step {{.*}} iter_args(%[[TC1:.*]] = %[[TC0]]) -> (tensor<?x?xf32>) {
+//      CHECK:     %[[TD2:.*]] = scf.for {{.*}} to {{.*}} step {{.*}} iter_args(%[[TC2:.*]] = %[[TC1]]) -> (tensor<?x?xf32>) {
+//      CHECK:       %[[sTA:.*]] = tensor.extract_slice %[[TA]][{{.*}}] : tensor<?x?xf32> to tensor<?x?xf32>
+//      CHECK:       %[[sTB:.*]] = tensor.extract_slice %[[TB]][{{.*}}] : tensor<?x?xf32> to tensor<?x?xf32>
+//      CHECK:       %[[sTC:.*]] = tensor.extract_slice %[[TC2]][{{.*}}] : tensor<?x?xf32> to tensor<?x?xf32>
+//      CHECK:       %[[sTD:.*]] = linalg.contract
+// CHECK-SAME:          indexing_maps = [#[[$MAP0]], #[[$MAP1]], #[[$MAP2]]]
+// CHECK-SAME:          ins(%[[sTA]], %[[sTB]] : tensor<?x?xf32>, tensor<?x?xf32>)
+// CHECK-SAME:          outs(%[[sTC]] : tensor<?x?xf32>)  -> tensor<?x?xf32>
+//      CHECK:       %[[TD:.*]] = tensor.insert_slice %[[sTD]] into %[[TC2]][{{.*}}]  : tensor<?x?xf32> into tensor<?x?xf32>
+//      CHECK:       scf.yield %[[TD]] : tensor<?x?xf32>
+//      CHECK:     scf.yield %[[TD2]] : tensor<?x?xf32>
+//      CHECK:   scf.yield %[[TD1]] : tensor<?x?xf32>
+  %0 = linalg.contract indexing_maps = #access_maps
+                       ins(%A, %B: tensor<?x?xf32>, tensor<?x?xf32>)
+                       outs(%C: tensor<?x?xf32>)
+    -> tensor<?x?xf32>
+
+//      CHECK: return %[[TD0]] : tensor<?x?xf32>
+  return %0 : tensor<?x?xf32>
+}
+
+module attributes {transform.with_named_sequence} {
+  transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
+    %0 = transform.structured.match ops{["linalg.contract"]} in %arg1 : (!transform.any_op) -> !transform.any_op
+    %1, %loops:3 = transform.structured.tile_using_for %0 tile_sizes [2, 3, 4] : (!transform.any_op) -> (!transform.any_op, !transform.any_op, !transform.any_op, !transform.any_op)
+    transform.yield
+  }
+}
+
+// -----
+
 // CHECK-LABEL: func @matmul_tensors_with_size_zeros(
 // CHECK-SAME:    %[[TA:[0-9a-z]+]]: tensor<?x?xf32>
 // CHECK-SAME:    %[[TB:[0-9a-z]+]]: tensor<?x?xf32>
diff --git a/mlir/test/Dialect/Linalg/transform-tile-and-winograd-rewrite.mlir b/mlir/test/Dialect/Linalg/transform-tile-and-winograd-rewrite.mlir
index 776dc5b748c84..cdc4b8a72a276 100644
--- a/mlir/test/Dialect/Linalg/transform-tile-and-winograd-rewrite.mlir
+++ b/mlir/test/Dialect/Linalg/transform-tile-and-winograd-rewrite.mlir
@@ -61,13 +61,12 @@ module attributes {transform.with_named_sequence} {
 // CHECK:      scf.yield %[[INSERTED_SLICE]]
 // CHECK:    scf.yield %[[S9]]
 // CHECK:  %[[S2:.*]] = tensor.empty() : tensor<6x6x2x2x2x5xf32>
-// CHECK:  %[[S3:.*]] = tensor.empty() : tensor<6x6x2x2x2x5xf32>
-// CHECK:  %[[S4:.*]] = scf.for %[[ARG3:.*]] = %[[C0]] to %[[C2]] step %[[C1]] iter_args(%[[ARG4:.*]] = %[[S3]])
+// CHECK:  %[[S4:.*]] = scf.for %[[ARG3:.*]] = %[[C0]] to %[[C2]] step %[[C1]] iter_args(%[[ARG4:.*]] = %[[S2]])
 // CHECK:    %[[S9:.*]] = scf.for %[[ARG5:.*]] = %[[C0]] to %[[C2]] step %[[C1]] iter_args(%[[ARG6:.*]] = %[[ARG4]])
 // CHECK:      %[[S10:.*]] = affine.apply #[[$MAP0]](%[[ARG3]])
 // CHECK:      %[[S11:.*]] = affine.apply #[[$MAP0]](%[[ARG5]])
 // CHECK:      %[[EXTRACTED_SLICE:.*]] = tensor.extract_slice %[[ARG0]][0, %[[S10]], %[[S11]], 0] [2, 6, 6, 5] [1, 1, 1, 1]
-// CHECK:      %[[EXTRACTED_SLICE_7:.*]] = tensor.extract_slice %[[S2]][0, 0, %[[ARG3]], %[[ARG5]], 0, 0] [6, 6, 1, 1, 2, 5] [1, 1, 1, 1, 1, 1]
+// CHECK:      %[[EXTRACTED_SLICE_7:.*]] = tensor.extract_slice %[[ARG6]][0, 0, %[[ARG3]], %[[ARG5]], 0, 0] [6, 6, 1, 1, 2, 5] [1, 1, 1, 1, 1, 1]
 // CHECK:      %[[S12:.*]] = scf.for %[[ARG7:.*]] = %[[C0]] to %[[C2]] step %[[C1]] iter_args(%[[ARG8:.*]] = %[[EXTRACTED_SLICE_7]])
 // CHECK:        %[[S13:.*]] = scf.for %[[ARG9:.*]] = %[[C0]] to %[[C5]] step %[[C1]] iter_args(%[[ARG10:.*]] = %[[ARG8]])
 // CHECK:          %[[EXTRACTED_SLICE_8:.*]] = tensor.extract_slice %[[EXTRACTED_SLICE]][%[[ARG7]], 0, 0, %[[ARG9]]] [1, 6, 6, 1] [1, 1, 1, 1]
@@ -195,13 +194,12 @@ module attributes {transform.with_named_sequence} {
 // CHECK:    scf.yield %[[S9]] : tensor<6x6x5x2xf32>
 // CHECK:  %[[PADDED:.*]] = tensor.pad %[[ARG0]] low[0, 0, 0, 0] high[0, 3, 3, 0]
 // CHECK:  %[[S2:.*]] = tensor.empty() : tensor<6x6x3x3x2x5xf32>
-// CHECK:  %[[S3:.*]] = tensor.empty() : tensor<6x6x3x3x2x5xf32>
-// CHECK:  %[[S4:.*]] = scf.for %[[ARG4:.*]] = %[[C0]] to %[[C3]] step %[[C1]] iter_args(%[[ARG5:.*]] = %[[S3]])
+// CHECK:  %[[S4:.*]] = scf.for %[[ARG4:.*]] = %[[C0]] to %[[C3]] step %[[C1]] iter_args(%[[ARG5:.*]] = %[[S2]])
 // CHECK:    %[[S9:.*]] = scf.for %[[ARG6:.*]] = %[[C0]] to %[[C3]] step %[[C1]] iter_args(%[[ARG7:.*]] = %[[ARG5]])
 // CHECK:      %[[S10:.*]] = affine.apply #[[$MAP0]](%[[ARG4]])
 // CHECK:      %[[S11:.*]] = affine.apply #[[$MAP0]](%[[ARG6]])
 // CHECK:      %[[EXTRACTED_SLICE_9:.*]] = tensor.extract_slice %[[PADDED]][0, %[[S10]], %[[S11]], 0] [2, 6, 6, 5] [1, 1, 1, 1]
-// CHECK:      %[[EXTRACTED_SLICE_10:.*]] = tensor.extract_slice %[[S2]][0, 0, %[[ARG4]], %[[ARG6]], 0, 0] [6, 6, 1, 1, 2, 5] [1, 1, 1, 1, 1, 1]
+// CHECK:      %[[EXTRACTED_SLICE_10:.*]] = tensor.extract_slice %[[ARG7]][0, 0, %[[ARG4]], %[[ARG6]], 0, 0] [6, 6, 1, 1, 2, 5] [1, 1, 1, 1, 1, 1]
 // CHECK:      %[[S12:.*]] = scf.for %[[ARG8:.*]] = %[[C0]] to %[[C2]] step %[[C1]] iter_args(%[[ARG9:.*]] = %[[EXTRACTED_SLICE_10]])
 // CHECK:        %[[S13:.*]] = scf.for %[[ARG10:.*]] = %[[C0]] to %[[C5]] step %[[C1]] iter_args(%[[ARG11:.*]] = %[[ARG9]])
 // CHECK:          %[[EXTRACTED_SLICE_11:.*]] = tensor.extract_slice %[[EXTRACTED_SLICE_9]][%[[ARG8]], 0, 0, %[[ARG10]]] [1, 6, 6, 1] [1, 1, 1, 1]
@@ -346,3 +344,104 @@ module attributes {transform.with_named_sequence} {
 // CHECK:       scf.yield %[[INSERTED_SLICE]]
 // CHECK:     scf.yield %[[S7]]
 // CHECK:   return %[[S6]]
+
+// -----
+
+func.func @conv2d_mx1_rx1_2(%arg0: tensor<2x6x2x5xf32>, %arg1: tensor<2x3x1x5xf32>, %arg2: tensor<2x4x2x2xf32>) -> tensor<2x4x2x2xf32> {
+  %cst = arith.constant 0.000000e+00 : f32
+  %0 = tensor.empty() : tensor<6x1x5x2xf32>
+  %1 = linalg.winograd_filter_transform m(4) r(3) ins(%arg1 : tensor<2x3x1x5xf32>) outs(%0 : tensor<6x1x5x2xf32>) -> tensor<6x1x5x2xf32>
+  %2 = tensor.empty() : tensor<6x1x1x2x2x5xf32>
+  %3 = linalg.winograd_input_transform m(4) r(3) ins(%arg0 : tensor<2x6x2x5xf32>) outs(%2 : tensor<6x1x1x2x2x5xf32>) -> tensor<6x1x1x2x2x5xf32>
+  %collapsed = tensor.collapse_shape %1 [[0, 1], [2], [3]] : tensor<6x1x5x2xf32> into tensor<6x5x2xf32>
+  %collapsed_0 = tensor.collapse_shape %3 [[0, 1], [2, 3, 4], [5]] : tensor<6x1x1x2x2x5xf32> into tensor<6x4x5xf32>
+  %4 = tensor.empty() : tensor<6x4x2xf32>
+  %5 = linalg.fill ins(%cst : f32) outs(%4 : tensor<6x4x2xf32>) -> tensor<6x4x2xf32>
+  %6 = linalg.batch_matmul ins(%collapsed_0, %collapsed : tensor<6x4x5xf32>, tensor<6x5x2xf32>) outs(%5 : tensor<6x4x2xf32>) -> tensor<6x4x2xf32>
+  %expanded = tensor.expand_shape %6 [[0, 1], [2, 3, 4], [5]] output_shape [6, 1, 1, 2, 2, 2] : tensor<6x4x2xf32> into tensor<6x1x1x2x2x2xf32>
+  %7 = linalg.winograd_output_transform m(4) r(3) ins(%expanded : tensor<6x1x1x2x2x2xf32>) outs(%arg2 : tensor<2x4x2x2xf32>) -> tensor<2x4x2x2xf32>
+  return %7 : tensor<2x4x2x2xf32>
+}
+
+module attributes {transform.with_named_sequence} {
+  transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
+    %0 = transform.structured.match ops{["linalg.winograd_filter_transform"]} in %arg1 : (!transform.any_op) -> !transform.any_op
+    %2 = transform.structured.match ops{["linalg.winograd_input_transform"]} in %arg1 : (!transform.any_op) -> !transform.any_op
+    %3, %loop3:2 = transform.structured.tile_using_for %2 tile_sizes [0, 0, 1, 1, 0, 0] : (!transform.any_op) -> (!transform.any_op, !transform.any_op, !transform.any_op)
+    %4 = transform.structured.match ops{["linalg.winograd_output_transform"]} in %arg1 : (!transform.any_op) -> !transform.any_op
+    %5, %loop5:2 = transform.structured.tile_using_for %4 tile_sizes [0, 0, 1, 1, 0, 0] : (!transform.any_op) -> (!transform.any_op, !transform.any_op, !transform.any_op)
+    %7 = transform.structured.decompose_winograd_op %0 : (!transform.any_op) -> (!transform.any_op)
+    %8 = transform.structured.match ops{["linalg.winograd_input_transform"]} in %3 : (!transform.any_op) -> !transform.any_op
+    %9 = transform.structured.decompose_winograd_op %8 : (!transform.any_op) -> (!transform.any_op)
+    %10 = transform.structured.match ops{["linalg.winograd_output_transform"]} in %5 : (!transform.any_op) -> !transform.any_op
+    %11 = transform.structured.decompose_winograd_op %10 : (!transform.any_op) -> (!transform.any_op)
+    transform.yield
+  }
+}
+
+// CHECK: #[[$MAP:.+]] = affine_map<(d0, d1) -> ()>
+// CHECK: #[[$MAP1:.+]] = affine_map<(d0, d1) -> (d0, d1)>
+// CHECK-LABEL: func.func @conv2d_mx1_rx1_2
+// CHECK-SAME:  (%[[ARG0:.*]]: tensor<2x6x2x5xf32>, %[[ARG1:.*]]: tensor<2x3x1x5xf32>, %[[ARG2:.*]]: tensor<2x4x2x2xf32>) -> tensor<2x4x2x2xf32> {
+// CHECK:   %[[CST:.*]] = arith.constant 3.200000e+01 : f32
+// CHECK:  %[[CST_0:.*]] = arith.constant dense<{{.*}}> : tensor<4x6xf32>
+// CHECK:  %[[CST_1:.*]] = arith.constant dense<{{.*}}> : tensor<6x6xf32>
+// CHECK:  %[[CST_2:.*]] = arith.constant dense<{{.*}}> : tensor<6x3xf32>
+// CHECK:   %[[C1:.*]] = arith.constant 1 : index
+// CHECK:   %[[C5:.*]] = arith.constant 5 : index
+// CHECK:   %[[C2:.*]] = arith.constant 2 : index
+// CHECK:   %[[C0:.*]] = arith.constant 0 : index
+// CHECK:   %[[CST_3:.*]] = arith.constant 0.000000e+00 : f32
+// CHECK:   %[[S0:.*]] = tensor.empty() : tensor<6x1x5x2xf32>
+// CHECK:   %[[S1:.*]] = scf.for %[[ARG3:.*]] = %[[C0]] to %[[C2]] step %[[C1]] iter_args(%[[ARG4:.*]] = %[[S0]])
+// CHECK:     %[[S7:.*]] = scf.for %[[ARG5:.*]] = %[[C0]] to %[[C5]] step %[[C1]] iter_args(%[[ARG6:.*]] = %[[ARG4]])
+// CHECK:       %[[EXTRACTED_SLICE:.*]] = tensor.extract_slice %[[ARG1]][%[[ARG3]], 0, 0, %[[ARG5]]] [1, 3, 1, 1] [1, 1, 1, 1]
+// CHECK:       %[[S8:.*]] = tensor.empty() : tensor<6x1xf32>
+// CHECK:       %[[S9:.*]] = linalg.fill ins(%[[CST_3]] : f32) outs(%[[S8]] : tensor<6x1xf32>) -> tensor<6x1xf32>
+// CHECK:       %[[S10:.*]] = linalg.matmul ins(%[[CST_2]], %[[EXTRACTED_SLICE]] : tensor<6x3xf32>, tensor<3x1xf32>) outs(%[[S9]] : tensor<6x1xf32>) -> tensor<6x1xf32>
+// CHECK:       %[[INSERTED_SLICE:.*]] = tensor.insert_slice %[[S10]] into %[[ARG6]][0, 0, %[[ARG5]], %[[ARG3]]] [6, 1, 1, 1] [1, 1, 1, 1]
+// CHECK:       scf.yield %[[INSERTED_SLICE]]
+// CHECK:     scf.yield %[[S7]]
+// CHECK:   %[[S2:.*]] = tensor.empty() : tensor<6x1x1x2x2x5xf32>
+// CHECK:   %[[S3:.*]] = scf.for %[[ARG3:.*]] = %[[C0]] to %[[C2]] step %[[C1]] iter_args(%[[ARG4:.*]] = %[[S2]])
+// CHECK:     %[[EXTRACTED_SLICE:.*]] = tensor.extract_slice %[[ARG0]][0, 0, %[[ARG3]], 0] [2, 6, 1, 5] [1, 1, 1, 1]
+// CHECK:     %[[EXTRACTED_SLICE_5:.*]] = tensor.extract_slice %[[ARG4]][0, 0, 0, %[[ARG3]], 0, 0] [6, 1, 1, 1, 2, 5] [1, 1, 1, 1, 1, 1]
+// CHECK:     %[[S9:.*]] = scf.for %[[ARG5:.*]] = %[[C0]] to %[[C2]] step %[[C1]] iter_args(%[[ARG6:.*]] = %[[EXTRACTED_SLICE_5]])
+// CHECK:     %[[S10:.*]] = scf.for %[[ARG7:.*]] = %[[C0]] to %[[C5]] step %[[C1]] iter_args(%[[ARG8:.*]] = %[[ARG6]])
+// CHECK:       %[[EXTRACTED_SLICE_6:.*]] = tensor.extract_slice %[[EXTRACTED_SLICE]][%[[ARG5]], 0, 0, %[[ARG7]]] [1, 6, 1, 1] [1, 1, 1, 1]
+// CHECK:       %[[S11:.*]] = tensor.empty() : tensor<6x1xf32>
+// CHECK:       %[[S12:.*]] = linalg.fill ins(%[[CST_3]] : f32) outs(%[[S11]] : tensor<6x1xf32>) -> tensor<6x1xf32>
+// CHECK:       %[[S13:.*]] = linalg.matmul ins(%[[CST_1]], %[[EXTRACTED_SLICE_6]] : tensor<6x6xf32>, tensor<6x1xf32>) outs(%[[S12]] : tensor<6x1xf32>) -> tensor<6x1xf32>
+// CHECK:       %[[INSERTED_SLICE_7:.*]] = tensor.insert_slice %[[S13]] into %[[ARG8]][0, 0, 0, 0, %[[ARG5]], %[[ARG7]]] [6, 1, 1, 1, 1, 1] [1, 1, 1, 1, 1, 1]
+// CHECK:       scf.yield %[[INSERTED_SLICE_7]]
+// CHECK:     scf.yield %[[S10]]
+// CHECK:    %[[INSERTED_SLICE:.*]] = tensor.insert_slice %[[S9]] into %[[ARG4]][0, 0, 0, %[[ARG3]], 0, 0] [6, 1, 1, 1, 2, 5] [1, 1, 1, 1, 1, 1]
+// CHECK:    scf.yield %[[INSERTED_SLICE]]
+// CHECK:   %[[COLLAPSED:.*]] = tensor.collapse_shape %[[S1]] {{\[}}[0, 1], [2], [3]]
+// CHECK:   %[[COLLAPSED_4:.*]] = tensor.collapse_shape %[[S3]] {{\[}}[0, 1], [2, 3, 4], [5]]
+// CHECK:   %[[S4:.*]] = tensor.empty() : tensor<6x4x2xf32>
+// CHECK:   %[[S5:.*]] = linalg.fill ins(%[[CST_3]] : f32) outs(%[[S4]] : tensor<6x4x2xf32>) -> tensor<6x4x2xf32>
+// CHECK:   %[[S6:.*]] = linalg.batch_matmul ins(%[[COLLAPSED_4]], %[[COLLAPSED]] : tensor<6x4x5xf32>, tensor<6x5x2xf32>) outs(%[[S5]] : tensor<6x4x2xf32>) -> tensor<6x4x2xf32>
+// CHECK:   %[[EXPANDED:.*]] = tensor.expand_shape %[[S6]] {{\[}}[0, 1], [2, 3, 4], [5]] output_shape [6, 1, 1, 2, 2, 2]
+// CHECK:   %[[S7:.*]] = scf.for %[[ARG3:.*]] = %[[C0]] to %[[C2]] step %[[C1]] iter_args(%[[ARG4:.*]] = %[[ARG2]])
+// CHECK:     %[[EXTRACTED_SLICE:.*]] = tensor.extract_slice %[[EXPANDED]][0, 0, 0, %[[ARG3]], 0, 0] [6, 1, 1, 1, 2, 2] [1, 1, 1, 1, 1, 1]
+// CHECK:     %[[EXTRACTED_SLICE_5:.*]] = tensor.extract_slice %[[ARG4]][0, 0, %[[ARG3]], 0] [2, 4, 1, 2] [1, 1, 1, 1]
+// CHECK:     %[[S8:.*]] = scf.for %[[ARG5:.*]] = %[[C0]] to %[[C2]] step %[[C1]] iter_args(%[[ARG6:.*]] = %[[EXTRACTED_SLICE_5]])
+// CHECK:       %[[S9:.*]] = scf.for %[[ARG7:.*]] = %[[C0]] to %[[C2]] step %[[C1]] iter_args(%[[ARG8:.*]] = %[[ARG6]])
+// CHECK:       %[[EXTRACTED_SLICE_6:.*]] = tensor.extract_slice %[[EXTRACTED_SLICE]][0, 0, 0, 0, %[[ARG5]], %[[ARG7]]] [6, 1, 1, 1, 1, 1] [1, 1, 1, 1, 1, 1]
+// CHECK:       %[[EXTRACTED_SLICE_7:.*]] = tensor.extract_slice %[[ARG8]][%[[ARG5]], 0, 0, %[[ARG7]]] [1, 4, 1, 1] [1, 1, 1, 1]
+// CHECK:       %[[S10:.*]] = tensor.empty() : tensor<4x1xf32>
+// CHECK:       %[[S11:.*]] = linalg.fill ins(%[[CST_3]] : f32) outs(%[[S10]] : tensor<4x1xf32>) -> tensor<4x1xf32>
+// CHECK:       %[[S12:.*]] = linalg.matmul ins(%[[CST_0]], %[[EXTRACTED_SLICE_6]] : tensor<4x6xf32>, tensor<6x1xf32>) outs(%[[S11]] : tensor<4x1xf32>) -> tensor<4x1xf32>
+// CHECK:       %[[S13:.*]] = linalg.generic {indexing_maps = [#[[$MAP]], #[[$MAP1]], #[[$MAP1]]], iterator_types = ["parallel", "parallel"]} ins(%[[CST]], %[[S12]] : f32, tensor<4x1xf32>) outs(%[[EXTRACTED_SLICE_7]] : tensor<4x1xf32>) {
+// CHECK:       ^bb0(%[[IN1:.*]]: f32, %[[IN2:.*]]: f32, %[[OUT:.*]]: f32):
+// CHECK:          %[[VAL_57:.*]] = arith.mulf %[[IN1]], %[[IN2]] : f32
+// CHECK:          %[[VAL_58:.*]] = arith.addf %[[VAL_57]], %[[OUT]] : f32
+// CHECK:          linalg.yield %[[VAL_58]] : f32
+// CHECK:       } -> tensor<4x1xf32>
+// CHECK:       %[[INSERTED_SLICE_8:.*]] = tensor.insert_slice %[[S13]] into %[[ARG8]][%[[ARG5]], 0, 0, %[[ARG7]]] [1, 4, 1, 1] [1, 1, 1, 1]
+// CHECK:       scf.yield %[[INSERTED_SLICE_8]]
+// CHECK:     scf.yield %[[S9]]
+// CHECK:     %[[INSERTED_SLICE:.*]] = tensor.insert_slice %[[S8]] into %[[ARG4]][0, 0, %[[ARG3]], 0] [2, 4, 1, 2] [1, 1, 1, 1]
+// CHECK:     scf.yield %[[INSERTED_SLICE]]
+// CHECK:   return %[[S7]]
diff --git a/mlir/test/Dialect/Linalg/transform-tile-winograd.mlir b/mlir/test/Dialect/Linalg/transform-tile-winograd.mlir
index 9598c434aadb8..fc6424fd4c812 100644
--- a/mlir/test/Dialect/Linalg/transform-tile-winograd.mlir
+++ b/mlir/test/Dialect/Linalg/transform-tile-winograd.mlir
@@ -21,11 +21,12 @@ module attributes {transform.with_named_sequence} {
 // CHECK-DAG:  %[[C5:.*]] = arith.constant 5 : index
 // CHECK-DAG:  %[[C1:.*]] = arith.constant 1 : index
 // CHECK-DAG:  %[[C1_1:.*]] = arith.constant 1 : index
-// CHECK:  %[[S1:.*]] = scf.for %[[ARG2:.*]] = %[[C0]] to %[[C2]] step %[[C1]]
-// CHECK:    %[[S2:.*]] = scf.for %[[ARG4:.*]] = %[[C0_0]] to %[[C5]] step %[[C1_1]]
+// CHECK:  %[[S1:.*]] = scf.for %[[ARG2:.*]] = %[[C0]] to %[[C2]] step %[[C1]] iter_args(%[[ARG3:.*]] = %[[ARG1]])
+// CHECK:    %[[S2:.*]] = scf.for %[[ARG4:.*]] = %[[C0_0]] to %[[C5]] step %[[C1_1]] iter_args(%[[ARG5:.*]] = %[[ARG3]])
 // CHECK:      %[[EXTRACTED_SLICE:.*]] = tensor.extract_slice %[[ARG0]][%[[ARG2]], 0, 0, %[[ARG4]]] [1, 3, 3, 1] [1, 1, 1, 1] : tensor<2x3x3x5xf32> to tensor<1x3x3x1xf32>
-// CHECK:      %[[EXTRACTED_SLICE_2:.*]] = tensor.extract_slice %[[ARG1]][0, 0, %[[ARG4]], %[[ARG2]]] [6, 6, 1, 1] [1, 1, 1, 1] : tensor<6x6x5x2xf32> to tensor<6x6x1x1xf32>
+// CHECK:      %[[EXTRACTED_SLICE_2:.*]] = tensor.extract_slice %[[ARG5]][0, 0, %[[ARG4]], %[[ARG2]]] [6, 6, 1, 1] [1, 1, 1, 1] : tensor<6x6x5x2xf32> to tensor<6x6x1x1xf32>
 // CHECK:      %[[S3:.*]] = linalg.winograd_filter_transform m(4) r(3) ins(%[[EXTRACTED_SLICE]] : tensor<1x3x3x1xf32>) outs(%[[EXTRACTED_SLICE_2]] : tensor<6x6x1x1xf32>) -> tensor<6x6x1x1xf32>
+// CHECK:      %[[INSERTED_SLICE:.*]] = tensor.insert_slice %[[S3]] into %[[ARG5]][0, 0, %[[ARG4]], %[[ARG2]]] [6, 6, 1, 1] [1, 1, 1, 1] : tensor<6x6x1x1xf32> into tensor<6x6x5x2xf32>
 
 // -----
 
@@ -51,14 +52,14 @@ module attributes {transform.with_named_sequence} {
 // CHECK-DAG:   %[[C5:.*]] = arith.constant 5 : index
 // CHECK-DAG:   %[[C1:.*]] = arith.constant 1 : index
 // CHECK-DAG:   %[[C2_1:.*]] = arith.constant 2 : index
-// CHECK:   %[[S1:.*]] = scf.for %[[ARG2:.*]] = %[[C0]] to %[[C2]] step %[[C1]]
-// CHECK:     %[[S2:.*]] = scf.for %[[ARG4:.*]] = %[[C0_0]] to %[[C5]] step %[[C2_1]]
+// CHECK:   %[[S1:.*]] = scf.for %[[ARG2:.*]] = %[[C0]] to %[[C2]] step %[[C1]] iter_args(%[[ARG3:.*]] = %[[ARG1]])
+// CHECK:     %[[S2:.*]] = scf.for %[[ARG4:.*]] = %[[C0_0]] to %[[C5]] step %[[C2_1]] iter_args(%[[ARG5:.*]] = %[[ARG3]])
 // CHECK:       %[[C5_2:.*]] = arith.constant 5 : index
 // CHECK:       %[[S3:.*]] = affine.min #[[$MAP0]](%[[ARG4]])
 // CHECK:       %[[EXTRACTED_SLICE:.*]] = tensor.extract_slice %[[ARG0]][%[[ARG2]], 0, 0, %[[ARG4]]] [1, 3, 3, %[[S3]]] [1, 1, 1, 1] : tensor<2x3x3x5xf32> to tensor<1x3x3x?xf32>
-// CHECK:       %[[EXTRACTED_SLICE_3:.*]] = tensor.extract_slice %[[ARG1]][0, 0, %[[ARG4]], %[[ARG2]]] [6, 6, %[[S3]], 1] [1, 1, 1, 1] : tensor<6x6x5x2xf32> to tensor<6x6x?x1xf32>
+// CHECK:       %[[EXTRACTED_SLICE_3:.*]] = tensor.extract_slice %[[ARG5]][0, 0, %[[ARG4]], %[[ARG2]]] [6, 6, %[[S3]], 1] [1, 1, 1, 1] : tensor<6x6x5x2xf32> to tensor<6x6x?x1xf32>
 // CHECK:       %[[S4:.*]] = linalg.winograd_filter_transform m(4) r(3) ins(%[[EXTRACTED_SLICE]] : tensor<1x3x3x?xf32>) outs(%[[EXTRACTED_SLICE_3]] : tensor<6x6x?x1xf32>) -> tensor<6x6x?x1xf32>
-
+// CHECK:      %[[INSERTED_SLICE:.*]] = tensor.insert_slice %[[S4]] into %[[ARG5]][0, 0, %[[ARG4]], %[[ARG2]]] [6, 6, %[[S3]], 1] [1, 1, 1, 1] : tensor<6x6x?x1xf32> into tensor<6x6x5x2xf32>
 // -----
 
 func.func @tile_winograd_filter(%arg0: tensor<2x3x1x5xf32>, %arg1: tensor<6x1x5x2xf32>) -> tensor<6x1x5x2xf32> {
@@ -82,11 +83,12 @@ module attributes {transform.with_named_sequence} {
 // CHECK-DAG:   %[[C5:.*]] = arith.constant 5 : index
 // CHECK-DAG:   %[[C1:.*]] = arith.constant 1 : index
 // CHECK-DAG:   %[[C1_1:.*]] = arith.constant 1 : index
-// CHECK:   %[[S1:.*]] = scf.for %[[ARG2:.*]] = %[[C0]] to %[[C2]] step %[[C1]]
-// CHECK:     %[[S2:.*]] = scf.for %[[ARG4:.*]] = %[[C0_0]] to %[[C5]] step %[[C1_1]]
+// CHECK:   %[[S1:.*]] = scf.for %[[ARG2:.*]] = %[[C0]] to %[[C2]] step %[[C1]] iter_args(%[[ARG3:.*]] = %[[ARG1]])
+// CHECK:     %[[S2:.*]] = scf.for %[[ARG4:.*]] = %[[C0_0]] to %[[C5]] step %[[C1_1]] iter_args(%[[ARG5:.*]] = %[[ARG3]])
 // CHECK:       %[[EXTRACTED_SLICE:.*]] = tensor.extract_slice %[[ARG0]][%[[ARG2]], 0, 0, %[[ARG4]]] [1, 3, 1, 1] [1, 1, 1, 1] : tensor<2x3x1x5xf32> to tensor<1x3x1x1xf32>
-// CHECK:       %[[EXTRACTED_SLICE_2:.*]] = tensor.extract_slice %[[ARG1]][0, 0, %[[ARG4]], %[[ARG2]]] [6, 1, 1, 1] [1, 1, 1, 1] : tensor<6x1x5x2xf32> to tensor<6x1x1x1xf32>
+// CHECK:       %[[EXTRACTED_SLICE_2:.*]] = tensor.extract_slice %[[ARG5]][0, 0, %[[ARG4]], %[[ARG2]]] [6, 1, 1, 1] [1, 1, 1, 1] : tensor<6x1x5x2xf32> to tensor<6x1x1x1xf32>
 // CHECK:       %[[S3:.*]] = linalg.winograd_filter_transform m(4) r(3) ins(%[[EXTRACTED_SLICE]] : tensor<1x3x1x1xf32>) outs(%[[EXTRACTED_SLICE_2]] : tensor<6x1x1x1xf32>) -> tensor<6x1x1x1xf32>
+// CHECK:       %[[INSERTED_SLICE:.*]] = tensor.insert_slice %[[S3]] into %[[ARG5]][0, 0, %[[ARG4]], %[[ARG2]]] [6, 1, 1, 1] [1, 1, 1, 1] : tensor<6x1x1x1xf32> into tensor<6x1x5x2xf32>
 
 // -----
 
@@ -113,15 +115,16 @@ module attributes {transform.with_named_sequence} {
 // CHECK-DAG: %[[C2_1:.*]] = arith.constant 2 : index
 // CHECK-DAG: %[[C1:.*]] = arith.constant 1 : index
 // CHECK-DAG: %[[C1_2:.*]] = arith.constant 1 : index
-// CHECK: %[[S1:.*]] = scf.for %[[ARG2:.*]] = %[[C0]] to %[[C2]] step %[[C1]]
-// CHECK: %[[S2:.*]] = scf.for %[[ARG4:.*]] = %[[C0_0]] to %[[C2_1]] step %[[C1_2]]
+// CHECK: %[[S1:.*]] = scf.for %[[ARG2:.*]] = %[[C0]] to %[[C2]] step %[[C1]] iter_args(%[[ARG3:.*]] = %[[ARG1]])
+// CHECK: %[[S2:.*]] = scf.for %[[ARG4:.*]] = %[[C0_0]] to %[[C2_1]] step %[[C1_2]] iter_args(%[[ARG5:.*]] = %[[ARG3]])
 // CHECK:   %[[S3:.*]] = affine.apply #[[$MAP0]](%[[ARG2]])
 // CHECK:   %[[S4:.*]] = affine.apply #[[$MAP0]](%[[ARG4]])
 // CHECK:   %[[S5:.*]] = affine.apply #[[$MAP1]]()
 // CHECK:   %[[S6:.*]] = affine.apply #[[$MAP1]]()
 // CHECK:   %[[EXTRACTED_SLICE:.*]] = tensor.extract_slice %[[ARG0]][0, %[[S3]], %[[S4]], 0] [2, %[[S5]], %[[S6]], 5] [1, 1, 1, 1] : tensor<2x10x10x5xf32> to tensor<2x?x?x5xf32>
-// CHECK:   %[[EXTRACTED_SLICE_5:.*]] = tensor.extract_slice %[[ARG1]][0, 0, %[[ARG2]], %[[ARG4]], 0, 0] [6, 6, 1, 1, 2, 5] [1, 1, 1, 1, 1, 1] : tensor<6x6x2x2x2x5xf32> to tensor<6x6x1x1x2x5xf32>
+// CHECK:   %[[EXTRACTED_SLICE_5:.*]] = tensor.extract_slice %[[ARG5]][0, 0, %[[ARG2]], %[[ARG4]], 0, 0] [6, 6, 1, 1, 2, 5] [1, 1, 1, 1, 1, 1] : tensor<6x6x2x2x2x5xf32> to tensor<6x6x1x1x2x5xf32>
 // CHECK:   %[[S7:.*]] = linalg.winograd_input_transform m(4) r(3) ins(%[[EXTRACTED_SLICE]] : tensor<2x?x?x5xf32>) outs(%[[EXTRACTED_SLICE_5]] : tensor<6x6x1x1x2x5xf32>) -> tensor<6x6x1x1x2x5xf32>
+// CHECK:   %[[INSERTED_SLICE:.*]] = tensor.insert_slice %[[S7]] into %[[ARG5]][0, 0, %[[ARG2]], %[[ARG4]], 0, 0] [6, 6, 1, 1, 2, 5] [1, 1, 1, 1, 1, 1] : tensor<6x6x1x1x2x5xf32> into tensor<6x6x2x2x2x5xf32>
 
 // -----
 
@@ -154,17 +157,18 @@ module attributes {transform.with_named_sequence} {
 // CHECK-DAG: %[[C1_2:.*]] = arith.constant 1 : index
 // CHECK-DAG: %[[C1_5:.*]] = arith.constant 1 : index
 // CHECK-DAG: %[[C1_7:.*]] = arith.constant 1 : index
-// CHECK: %[[S1:.*]] = scf.for %[[ARG2:.*]] = %[[C0]] to %[[C2]] step %[[C1]]
-// CHECK:   %[[S2:.*]] = scf.for %[[ARG4:.*]] = %[[C0_0]] to %[[C2_1]] step %[[C1_2]]
-// CHECK:     %[[S3:.*]] = scf.for %[[ARG6:.*]] = %[[C0_3]] to %[[C2_4]] step %[[C1_5]]
-// CHECK:       %[[S4:.*]] = scf.for %[[ARG8:.*]] = %[[C0_6]] to %[[C5]] step %[[C1_7]]
+// CHECK: %[[S1:.*]] = scf.for %[[ARG2:.*]] = %[[C0]] to %[[C2]] step %[[C1]] iter_args(%[[ARG3:.*]] = %[[ARG1]])
+// CHECK:   %[[S2:.*]] = scf.for %[[ARG4:.*]] = %[[C0_0]] to %[[C2_1]] step %[[C1_2]] iter_args(%[[ARG5:.*]] = %[[ARG3]])
+// CHECK:     %[[S3:.*]] = scf.for %[[ARG6:.*]] = %[[C0_3]] to %[[C2_4]] step %[[C1_5]] iter_args(%[[ARG7:.*]] = %[[ARG5]])
+// CHECK:       %[[S4:.*]] = scf.for %[[ARG8:.*]] = %[[C0_6]] to %[[C5]] step %[[C1_7]] iter_args(%[[ARG9:.*]] = %[[ARG7]])
 // CHECK:         %[[S5:.*]] = affine.apply #[[$MAP0]](%[[ARG2]])
 // CHECK:         %[[S6:.*]] = affine.apply #[[$MAP0]](%[[ARG4]])
 // CHECK:         %[[S7:.*]] = affine.apply #[[$MAP1]]()
 // CHECK:         %[[S8:.*]] = affine.apply #[[$MAP1]]()
 // CHECK:         %[[EXTRACTED_SLICE:.*]] = tensor.extract_slice %[[ARG0]][%[[ARG6]], %[[S5]], %[[S6]], %[[ARG8]]] [1, %[[S7]], %[[S8]], 1] [1, 1, 1, 1] : tensor<2x10x10x5xf32> to tensor<1x?x?x1xf32>
-// CHECK:         %[[EXTRACTED_SLICE_10:.*]] = tensor.extract_slice %[[ARG1]][0, 0, %[[ARG2]], %[[ARG4]], %[[ARG6]], %[[ARG8]]] [6, 6, 1, 1, 1, 1] [1, 1, 1, 1, 1, 1] : tensor<6x6x2x2x2x5xf32> to tensor<6x6x1x1x1x1xf32>
+// CHECK:         %[[EXTRACTED_SLICE_10:.*]] = tensor.extract_slice %[[ARG9]][0, 0, %[[ARG2]], %[[ARG4]], %[[ARG6]], %[[ARG8]]] [6, 6, 1, 1, 1, 1] [1, 1, 1, 1, 1, 1] : tensor<6x6x2x2x2x5xf32> to tensor<6x6x1x1x1x1xf32>
 // CHECK:         %[[S9:.*]] = linalg.winograd_input_transform m(4) r(3) ins(%[[EXTRACTED_SLICE]] : tensor<1x?x?x1xf32>) outs(%[[EXTRACTED_SLICE_10]] : tensor<6x6x1x1x1x1xf32>) -> tensor<6x6x1x1x1x1xf32>
+// CHECK:         %[[INSERTED_SLICE:.*]] = tensor.insert_slice %[[S9]] into %[[ARG9]][0, 0, %[[ARG2]], %[[ARG4]], %[[ARG6]], %[[ARG8]]] [6, 6, 1, 1, 1, 1] [1, 1, 1, 1, 1, 1] : tensor<6x6x1x1x1x1xf32> into tensor<6x6x2x2x2x5xf32>
 
 // -----
 
@@ -198,18 +202,19 @@ module attributes {transform.with_named_sequence} {
 // CHECK-DAG: %[[C2_3:.*]] = arith.constant 2 : index
 // CHECK-DAG: %[[C2_6:.*]] = arith.constant 2 : index
 // CHECK-DAG: %[[C2_8:.*]] = arith.constant 2 : index
-// CHECK: %[[S1:.*]] = scf.for %[[ARG2:.*]] = %[[C0]] to %[[C2]] step %[[C2_0]]
-// CHECK:   %[[S2:.*]] = scf.for %[[ARG4:.*]] = %[[C0_1]] to %[[C2_2]] step %[[C2_3]]
-// CHECK:     %[[S3:.*]] = scf.for %[[ARG6:.*]] = %[[C0_4]] to %[[C2_5]] step %[[C2_6]]
-// CHECK:       %[[S4:.*]] = scf.for %[[ARG8:.*]] = %[[C0_7]] to %[[C5]] step %[[C2_8]]
+// CHECK: %[[S1:.*]] = scf.for %[[ARG2:.*]] = %[[C0]] to %[[C2]] step %[[C2_0]] iter_args(%[[ARG3:.*]] = %[[ARG1]])
+// CHECK:   %[[S2:.*]] = scf.for %[[ARG4:.*]] = %[[C0_1]] to %[[C2_2]] step %[[C2_3]] iter_args(%[[ARG5:.*]] = %[[ARG3]])
+// CHECK:     %[[S3:.*]] = scf.for %[[ARG6:.*]] = %[[C0_4]] to %[[C2_5]] step %[[C2_6]] iter_args(%[[ARG7:.*]] = %[[ARG5]])
+// CHECK:       %[[S4:.*]] = scf.for %[[ARG8:.*]] = %[[C0_7]] to %[[C5]] step %[[C2_8]] iter_args(%[[ARG9:.*]] = %[[ARG7]])
 // CHECK:         %[[S5:.*]] = affine.min #[[$MAP0]](%[[ARG8]])
 // CHECK:         %[[S6:.*]] = affine.apply #[[$MAP1]](%[[ARG2]])
 // CHECK:         %[[S7:.*]] = affine.apply #[[$MAP1]](%[[ARG4]])
 // CHECK:         %[[S8:.*]] = affine.apply #[[$MAP2]]()
 // CHECK:         %[[S9:.*]] = affine.apply #[[$MAP2]]()
 // CHECK:         %[[EXTRACTED_SLICE:.*]] = tensor.extract_slice %[[ARG0]][%[[ARG6]], %[[S6]], %[[S7]], %[[ARG8]]] [2, %[[S8]], %[[S9]], %[[S5]]] [1, 1, 1, 1] : tensor<2x10x10x5xf32> to tensor<2x?x?x?xf32>
-// CHECK:         %[[EXTRACTED_SLICE_12:.*]] = tensor.extract_slice %[[ARG1]][0, 0, %[[ARG2]], %[[ARG4]], %[[ARG6]], %[[ARG8]]] [6, 6, 2, 2, 2, %[[S5]]] [1, 1, 1, 1, 1, 1] : tensor<6x6x2x2x2x5xf32> to tensor<6x6x2x2x2x?xf32>
+// CHECK:         %[[EXTRACTED_SLICE_12:.*]] = tensor.extract_slice %[[ARG9]][0, 0, %[[ARG2]], %[[ARG4]], %[[ARG6]], %[[ARG8]]] [6, 6, 2, 2, 2, %[[S5]]] [1, 1, 1, 1, 1, 1] : tensor<6x6x2x2x2x5xf32> to tensor<6x6x2x2x2x?xf32>
 // CHECK:         %[[S10:.*]] = linalg.winograd_input_transform m(4) r(3) ins(%[[EXTRACTED_SLICE]] : tensor<2x?x?x?xf32>) outs(%[[EXTRACTED_SLICE_12]] : tensor<6x6x2x2x2x?xf32>) -> tensor<6x6x2x2x2x?xf32>
+// CHECK:         %[[INSERTED_SLICE:.*]] = tensor.insert_slice %[[S10]] into %[[ARG9]][0, 0, %[[ARG2]], %[[ARG4]], %[[ARG6]], %[[ARG8]]] [6, 6, 2, 2, 2, %[[S5]]] [1, 1, 1, 1, 1, 1] : tensor<6x6x2x2x2x?xf32> into tensor<6x6x2x2x2x5xf32>
 
 // -----
 
@@ -226,8 +231,9 @@ module attributes {transform.with_named_sequence} {
   }
 }
 
-// CHECK: #[[$MAP0:.+]] = affine_map<(d0) -> (d0 * 4)>
-// CHECK: #[[$MAP1:.+]] = affine_map<() -> (6)>
+// CHECK: #[[$MAP:.+]] = affine_map<(d0) -> (d0)>
+// CHECK: #[[$MAP1:.+]] = affine_map<(d0) -> (d0 * 4)>
+// CHECK: #[[$MAP2:.+]] = affine_map<() -> (6)>
 // CHECK-LABEL: func.func @tile_winograd_input(
 // CHECK-SAME:  %[[ARG0:.*]]: tensor<2x1x10x5xf32>, %[[ARG1:.*]]: tensor<1x6x1x2x2x5xf32>) -> tensor<1x6x1x2x2x5xf32> {
 // CHECK-DAG:   %[[C0:.*]] = arith.constant 0 : index
@@ -242,17 +248,18 @@ module attributes {transform.with_named_sequence} {
 // CHECK-DAG:   %[[C1_2:.*]] = arith.constant 1 : index
 // CHECK-DAG:   %[[C1_5:.*]] = arith.constant 1 : index
 // CHECK-DAG:   %[[C1_7:.*]] = arith.constant 1 : index
-// CHECK:   %[[S1:.*]] = scf.for %[[ARG2:.*]] = %[[C0]] to %[[C1]] step %[[C1_0]]
-// CHECK:     %[[S2:.*]] = scf.for %[[ARG4:.*]] = %[[C0_1]] to %[[C2]] step %[[C1_2]]
-// CHECK:       %[[S3:.*]] = scf.for %[[ARG6:.*]] = %[[C0_3]] to %[[C2_4]] step %[[C1_5]]
-// CHECK:         %[[S4:.*]] = scf.for %[[ARG8:.*]] = %[[C0_6]] to %[[C5]] step %[[C1_7]]
-// CHECK:           %[[S5:.*]] = affine.apply #[[$MAP0]](%[[ARG2]])
-// CHECK:           %[[S6:.*]] = affine.apply #[[$MAP0]](%[[ARG4]])
-// CHECK:           %[[S7:.*]] = affine.apply #[[$MAP1]]()
-// CHECK:           %[[S8:.*]] = affine.apply #[[$MAP1]]()
-// CHECK:           %[[EXTRACTED_SLICE:.*]] = tensor.extract_slice %[[ARG0]][%[[ARG6]], 0, %[[S6]], %[[ARG8]]] [1, 1, %[[S8]], 1] [1, 1, 1, 1] : tensor<2x1x10x5xf32> to tensor<1x1x?x1xf32>
-// CHECK:           %[[EXTRACTED_SLICE_10:.*]] = tensor.extract_slice %[[ARG1]][0, 0, %[[ARG2]], %[[ARG4]], %[[ARG6]], %[[ARG8]]] [1, 6, 1, 1, 1, 1] [1, 1, 1, 1, 1, 1] : tensor<1x6x1x2x2x5xf32> to tensor<1x6x1x1x1x1xf32>
+// CHECK:   %[[S1:.*]] = scf.for %[[ARG2:.*]] = %[[C0]] to %[[C1]] step %[[C1_0]] iter_args(%[[ARG3:.*]] = %[[ARG1]])
+// CHECK:     %[[S2:.*]] = scf.for %[[ARG4:.*]] = %[[C0_1]] to %[[C2]] step %[[C1_2]] iter_args(%[[ARG5:.*]] = %[[ARG3]])
+// CHECK:       %[[S3:.*]] = scf.for %[[ARG6:.*]] = %[[C0_3]] to %[[C2_4]] step %[[C1_5]] iter_args(%[[ARG7:.*]] = %[[ARG5]])
+// CHECK:         %[[S4:.*]] = scf.for %[[ARG8:.*]] = %[[C0_6]] to %[[C5]] step %[[C1_7]] iter_args(%[[ARG9:.*]] = %[[ARG7]])
+// CHECK:           %[[S5:.*]] = affine.apply #[[$MAP]](%[[ARG2]])
+// CHECK:           %[[S6:.*]] = affine.apply #[[$MAP1]](%[[ARG4]])
+// CHECK:           %[[S7:.*]] = affine.apply #[[$MAP2]]()
+// CHECK:           %[[S8:.*]] = affine.apply #[[$MAP2]]()
+// CHECK:           %[[EXTRACTED_SLICE:.*]] = tensor.extract_slice %[[ARG0]][%[[ARG6]], %[[S5]], %[[S6]], %[[ARG8]]] [1, 1, %[[S8]], 1] [1, 1, 1, 1] : tensor<2x1x10x5xf32> to tensor<1x1x?x1xf32>
+// CHECK:           %[[EXTRACTED_SLICE_10:.*]] = tensor.extract_slice %[[ARG9]][0, 0, %[[ARG2]], %[[ARG4]], %[[ARG6]], %[[ARG8]]] [1, 6, 1, 1, 1, 1] [1, 1, 1, 1, 1, 1] : tensor<1x6x1x2x2x5xf32> to tensor<1x6x1x1x1x1xf32>
 // CHECK:           %[[S9:.*]] = linalg.winograd_input_transform m(4) r(3) ins(%[[EXTRACTED_SLICE]] : tensor<1x1x?x1xf32>) outs(%[[EXTRACTED_SLICE_10]] : tensor<1x6x1x1x1x1xf32>) -> tensor<1x6x1x1x1x1xf32>
+// CHECK:         %[[INSERTED_SLICE:.*]] = tensor.insert_slice %[[S9]] into %[[ARG9]][0, 0, %[[ARG2]], %[[ARG4]], %[[ARG6]], %[[ARG8]]] [1, 6, 1, 1, 1, 1] [1, 1, 1, 1, 1, 1] : tensor<1x6x1x1x1x1xf32> into tensor<1x6x1x2x2x5xf32>
 
 // -----
 
@@ -351,8 +358,9 @@ module attributes {transform.with_named_sequence} {
   }
 }
 
-// CHECK: #[[$MAP0:.+]] = affine_map<(d0) -> (d0 * 4)>
-// CHECK: #[[$MAP1:.+]] = affine_map<() -> (4)>
+// CHECK: #[[$MAP:.+]] = affine_map<(d0) -> (d0 * 4)>
+// CHECK: #[[$MAP1:.+]] = affine_map<(d0) -> (d0)>
+// CHECK: #[[$MAP2:.+]] = affine_map<() -> (4)>
 // CHECK-LABEL: func.func @tile_winograd_output(
 // CHECK-SAME:  %[[ARG0:.*]]: tensor<6x1x2x1x3x5xf32>, %[[ARG1:.*]]: tensor<3x8x1x5xf32>) -> tensor<3x8x1x5xf32> {
 // CHECK-DAG:   %[[C0:.*]] = arith.constant 0 : index
@@ -372,9 +380,9 @@ module attributes {transform.with_named_sequence} {
 // CHECK:       %[[S3:.*]] = scf.for %[[ARG6:.*]] = %[[C0_3]] to %[[C3]] step %[[C1_4]] iter_args(%[[ARG11:.*]] = %[[ARG10]]) -> (tensor<3x8x1x5xf32>)
 // CHECK:         %[[S4:.*]] = scf.for %[[ARG8:.*]] = %[[C0_5]] to %[[C5]] step %[[C1_6]] iter_args(%[[ARG12:.*]] = %[[ARG11]]) -> (tensor<3x8x1x5xf32>)
 // CHECK:           %[[EXTRACTED_SLICE:.*]] = tensor.extract_slice %[[ARG0]][0, 0, %[[ARG2]], %[[ARG4]], %[[ARG6]], %[[ARG8]]] [6, 1, 1, 1, 1, 1] [1, 1, 1, 1, 1, 1] : tensor<6x1x2x1x3x5xf32> to tensor<6x1x1x1x1x1xf32>
-// CHECK:           %[[S5:.*]] = affine.apply #[[$MAP0]](%[[ARG2]])
-// CHECK:           %[[S6:.*]] = affine.apply #[[$MAP0]](%[[ARG4]])
-// CHECK:           %[[S7:.*]] = affine.apply #[[$MAP1]]()
-// CHECK:           %[[S8:.*]] = affine.apply #[[$MAP1]]()
-// CHECK:           %[[EXTRACTED_SLICE_9:.*]] = tensor.extract_slice %[[ARG12]][%[[ARG6]], %[[S5]], 0, %[[ARG8]]] [1, %[[S7]], 1, 1] [1, 1, 1, 1] : tensor<3x8x1x5xf32> to tensor<1x?x1x1xf32>
+// CHECK:           %[[S5:.*]] = affine.apply #[[$MAP]](%[[ARG2]])
+// CHECK:           %[[S6:.*]] = affine.apply #[[$MAP1]](%[[ARG4]])
+// CHECK:           %[[S7:.*]] = affine.apply #[[$MAP2]]()
+// CHECK:           %[[S8:.*]] = affine.apply #[[$MAP2]]()
+// CHECK:           %[[EXTRACTED_SLICE_9:.*]] = tensor.extract_slice %[[ARG12]][%[[ARG6]], %[[S5]], %[[S6]], %[[ARG8]]] [1, %[[S7]], 1, 1] [1, 1, 1, 1] : tensor<3x8x1x5xf32> to tensor<1x?x1x1xf32>
 // CHECK:           %[[S9:.*]] = linalg.winograd_output_transform m(4) r(3) ins(%[[EXTRACTED_SLICE]] : tensor<6x1x1x1x1x1xf32>) outs(%[[EXTRACTED_SLICE_9]] : tensor<1x?x1x1xf32>) -> tensor<1x?x1x1xf32>
diff --git a/mlir/test/Dialect/Linalg/vectorization-pad-patterns.mlir b/mlir/test/Dialect/Linalg/vectorization-pad-patterns.mlir
index 08a3bbbb301c8..f5d0e58138eea 100644
--- a/mlir/test/Dialect/Linalg/vectorization-pad-patterns.mlir
+++ b/mlir/test/Dialect/Linalg/vectorization-pad-patterns.mlir
@@ -224,34 +224,16 @@ module attributes {transform.with_named_sequence} {
   }
 }
 
-
 // -----
 
-///----------------------------------------------------------------------------------------
-/// tensor::PadOp -> tensor::EmptyOp + linalg::FillOp/tensor::GenerateOp + tensor::InsertSliceOp
-/// [Pattern: GenericPadOpVectorizationPattern + InsertSliceVectorizePattern]
-/// TODO: Split the test into two, one for each pattern.
-///----------------------------------------------------------------------------------------
-
 func.func private @make_vector() -> tensor<12x13xf32>
 
-// Same as @pad_and_insert_slice_dest in vectorization-with-patterns.mlir, but
-// over here linalg::fill is not vectorized (patterns for linalg.fill are not
-// included here)
+// The destination of tensor.insert_slice matches the result of tensor.pad -
+// not supported.
+
 // CHECK-LABEL:   func.func @pad_and_insert_slice_dest(
-// CHECK-SAME:      %[[ARG_0:.*]]: tensor<1x5x6xf32>) -> tensor<1x12x13xf32> {
-//  CHECK-NOT:     tensor.pad
-//  CHECK-DAG:     %[[C0:.*]] = arith.constant 0 : index
-//  CHECK-DAG:     %[[PAD:.*]] = arith.constant 5.000000e+00 : f32
-//  CHECK-DAG:     %[[PAD_READ:.*]] = arith.constant 0.000000e+00 : f32
-//      CHECK:     %[[EMPTY:.*]] = tensor.empty() : tensor<1x12x13xf32>
-//      CHECK:     %[[FILL:.*]] = linalg.fill ins(%[[PAD]] : f32) outs(%[[EMPTY]] : tensor<1x12x13xf32>) -> tensor<1x12x13xf32>
-//      CHECK:     %[[READ_1:.*]] = vector.transfer_read %[[ARG_0]]{{\[}}%[[C0]], %[[C0]], %[[C0]]], %[[PAD]] {in_bounds = [true, true, true]} : tensor<1x5x6xf32>, vector<1x5x6xf32>
-//      CHECK:     %[[WRITE_1:.*]] = vector.transfer_write %[[READ_1]], %[[FILL]]{{\[}}%[[C0]], %[[C0]], %[[C0]]] {in_bounds = [true, true, true]} : vector<1x5x6xf32>, tensor<1x12x13xf32>
-//      CHECK:     %[[VEC:.*]] = call @make_vector() : () -> tensor<12x13xf32>
-//      CHECK:     %[[READ_2:.*]] = vector.transfer_read %[[VEC]]{{\[}}%[[C0]], %[[C0]]], %[[PAD_READ]] {in_bounds = [true, true]} : tensor<12x13xf32>, vector<12x13xf32>
-//      CHECK:     %[[RES:.*]] = vector.transfer_write %[[READ_2]], %[[WRITE_1]]{{\[}}%[[C0]], %[[C0]], %[[C0]]] {in_bounds = [true, true]} : vector<12x13xf32>, tensor<1x12x13xf32>
-//      CHECK:     return %[[RES]] : tensor<1x12x13xf32>
+// CHECK-NOT:     vector.transfer_read
+// CHECK-NOT:     vector.transfer_write
 
 func.func @pad_and_insert_slice_dest(
     %arg0: tensor<1x5x6xf32>) -> tensor<1x12x13xf32> {
@@ -270,8 +252,6 @@ module attributes {transform.with_named_sequence} {
     %func_op = transform.structured.match ops{["func.func"]} in %arg1 : (!transform.any_op) -> !transform.op<"func.func">
 
     transform.apply_patterns to %func_op {
-      // TODO: Split into two tests, one for each pattern
-      transform.apply_patterns.linalg.decompose_pad
       transform.apply_patterns.linalg.pad_vectorization
     } : !transform.op<"func.func">
     transform.yield
diff --git a/mlir/test/Dialect/Linalg/vectorization-with-patterns.mlir b/mlir/test/Dialect/Linalg/vectorization-with-patterns.mlir
index b688a677500c2..5ae3f893c2e73 100644
--- a/mlir/test/Dialect/Linalg/vectorization-with-patterns.mlir
+++ b/mlir/test/Dialect/Linalg/vectorization-with-patterns.mlir
@@ -82,6 +82,39 @@ module attributes {transform.with_named_sequence} {
 
 // -----
 
+// CHECK-LABEL: @matmul_as_contract
+// CHECK-SAME: %[[A:.*]]: tensor<24x12xf32>
+// CHECK-SAME: %[[B:.*]]: tensor<12x25xf32>
+// CHECK-SAME: %[[C:.*]]: tensor<24x25xf32>
+func.func @matmul_as_contract(%A: tensor<24x12xf32>,
+                              %B: tensor<12x25xf32>,
+                              %C: tensor<24x25xf32>) -> tensor<24x25xf32> {
+  // CHECK: %[[vA:.+]] = vector.transfer_read %[[A]]
+  // CHECK: %[[vB:.+]] = vector.transfer_read %[[B]]
+  // CHECK: %[[vC:.+]] = vector.transfer_read %[[C]]
+  // CHECK: %[[vR:.+]] = vector.contract {{.*}} %[[vA]], %[[vB]], %[[vC]]
+  // CHECK: vector.transfer_write %[[vR]], %[[C]]
+  %0 = linalg.contract
+      indexing_maps = [affine_map<(m, n, k) -> (m, k)>,
+                       affine_map<(m, n, k) -> (k, n)>,
+                       affine_map<(m, n, k) -> (m, n)>]
+      ins(%A, %B : tensor<24x12xf32>, tensor<12x25xf32>)
+      outs(%C : tensor<24x25xf32>) -> tensor<24x25xf32>
+  func.return %0 : tensor<24x25xf32>
+}
+
+module attributes {transform.with_named_sequence} {
+  transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
+    %0 = transform.structured.match ops{["linalg.contract"]} in %arg1 : (!transform.any_op) -> !transform.any_op
+    %1 = transform.get_parent_op %0 {isolated_from_above} : (!transform.any_op) -> !transform.any_op
+    %2 = transform.structured.vectorize_children_and_apply_patterns %1 : (!transform.any_op) -> !transform.any_op
+    // TODO: also tests the other available vectorization strategies
+    transform.yield
+  }
+}
+
+// -----
+
 #matmul_trait = {
   indexing_maps = [
     affine_map<(m, n, k) -> (m, k)>,
diff --git a/mlir/test/Dialect/Linalg/vectorization.mlir b/mlir/test/Dialect/Linalg/vectorization.mlir
index 0f2abe06569d6..6d39262945de5 100644
--- a/mlir/test/Dialect/Linalg/vectorization.mlir
+++ b/mlir/test/Dialect/Linalg/vectorization.mlir
@@ -66,7 +66,7 @@ func.func @vectorize_dynamic_identity_with_constant(%arg0: tensor<?xf32>,
 module attributes {transform.with_named_sequence} {
   transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
     %0 = transform.structured.match ops{["linalg.generic"]} in %arg1 : (!transform.any_op) -> !transform.any_op
-    %size = transform.structured.match ops{["arith.constant"]} in %arg1 : (!transform.any_op) -> !transform.any_op 
+    %size = transform.structured.match ops{["arith.constant"]} in %arg1 : (!transform.any_op) -> !transform.any_op
     transform.structured.vectorize %0 vector_sizes [%size] : !transform.any_op, !transform.any_op
     transform.yield
   }
@@ -690,7 +690,7 @@ module attributes {transform.with_named_sequence} {
   transform.named_sequence @__transform_main(%arg0: !transform.any_op {transform.readonly}) {
     %0 = transform.structured.match ops{["tensor.pack"]} in %arg0 : (!transform.any_op) -> !transform.any_op
     transform.structured.vectorize %0 vector_sizes [4, 1, 32] : !transform.any_op
-    transform.yield 
+    transform.yield
   }
 }
 
@@ -727,7 +727,7 @@ module attributes {transform.with_named_sequence} {
   transform.named_sequence @__transform_main(%arg0: !transform.any_op {transform.readonly}) {
     %0 = transform.structured.match ops{["tensor.pack"]} in %arg0 : (!transform.any_op) -> !transform.any_op
     transform.structured.vectorize %0 vector_sizes [32, 4, 1] : !transform.any_op
-    transform.yield 
+    transform.yield
   }
 }
 
@@ -768,7 +768,7 @@ module attributes {transform.with_named_sequence} {
   transform.named_sequence @__transform_main(%arg0: !transform.any_op {transform.readonly}) {
     %0 = transform.structured.match ops{["tensor.pack"]} in %arg0 : (!transform.any_op) -> !transform.any_op
     transform.structured.vectorize %0 vector_sizes [4, 1] : !transform.any_op
-    transform.yield 
+    transform.yield
   }
 }
 
@@ -933,7 +933,7 @@ func.func @test_vectorize_unpack(%source: tensor<8x8x32x16xf32>, %dest: tensor<2
     %0 = transform.structured.match ops{["tensor.unpack"]} in %arg0 : (!transform.any_op) -> !transform.any_op
    transform.structured.vectorize %0 vector_sizes [512, 128] : !transform.any_op
     transform.yield
-  } 
+  }
 }
 
 // -----
@@ -957,7 +957,7 @@ func.func @test_vectorize_unpack_no_masks(%source: tensor<8x8x32x16xf32>, %dest:
     %0 = transform.structured.match ops{["tensor.unpack"]} in %arg0 : (!transform.any_op) -> !transform.any_op
    transform.structured.vectorize %0 vector_sizes [256, 128] : !transform.any_op
     transform.yield
-  } 
+  }
  }
 
   // -----
@@ -981,7 +981,7 @@ func.func @test_vectorize_unpack_no_masks(%source: tensor<8x8x32x16xf32>, %dest:
     %0 = transform.structured.match ops{["tensor.unpack"]} in %arg0 : (!transform.any_op) -> !transform.any_op
    transform.structured.vectorize %0 vector_sizes [256, 128] : !transform.any_op
     transform.yield
-  } 
+  }
 }
 
   // -----
@@ -1022,7 +1022,7 @@ func.func @test_vectorize_padded_pack_no_vector_sizes(%arg0: tensor<32x7x15xf32>
 //  CHECK-DAG: %[[cst:.*]] = arith.constant 0.000000e+00 : f32
 //  CHECK-DAG: %[[c0:.*]] = arith.constant 0 : index
 //      CHECK: %[[transfer_read:.*]] =  vector.transfer_read %{{.*}}[%[[c0]], %[[c0]], %[[c0]]], %[[cst]]
-// CHECK-SAME:   {in_bounds = [true, false, false]} : tensor<32x7x15xf32>, vector<32x8x16xf32> 
+// CHECK-SAME:   {in_bounds = [true, false, false]} : tensor<32x7x15xf32>, vector<32x8x16xf32>
 //      CHECK: %[[shape_cast:.*]] = vector.shape_cast %[[transfer_read]] : vector<32x8x16xf32> to vector<32x4x2x1x16xf32>
 //      CHECK: %[[transpose:.*]] = vector.transpose %[[shape_cast]], [0, 1, 3, 4, 2] : vector<32x4x2x1x16xf32> to vector<32x4x1x16x2xf32>
 //  CHECK-DAG: %[[c0_1:.*]] = arith.constant 0 : index
@@ -1059,7 +1059,7 @@ func.func @test_vectorize_unpack_no_vector_sizes(%source: tensor<8x8x32x16xf32>,
     %0 = transform.structured.match ops{["tensor.unpack"]} in %arg0 : (!transform.any_op) -> !transform.any_op
    transform.structured.vectorize %0 : !transform.any_op
     transform.yield
-  } 
+  }
  }
 
   // -----
@@ -1083,10 +1083,10 @@ func.func @test_vectorize_unpack_no_vector_sizes_slice_output(%source: tensor<8x
     %0 = transform.structured.match ops{["tensor.unpack"]} in %arg0 : (!transform.any_op) -> !transform.any_op
    transform.structured.vectorize %0 : !transform.any_op
     transform.yield
-  } 
+  }
  }
 
-  // -----
+// -----
 
 func.func @test_vectorize_unpack_no_vector_sizes_permute(%source: tensor<4x7x4xf32>, %dest: tensor<7x16xf32>) -> tensor<7x16xf32> {
    %0 = tensor.unpack %source outer_dims_perm=[1, 0] inner_dims_pos = [1] inner_tiles = [4] into %dest : tensor<4x7x4xf32> -> tensor<7x16xf32>
@@ -1106,5 +1106,156 @@ func.func @test_vectorize_unpack_no_vector_sizes_permute(%source: tensor<4x7x4xf
     %0 = transform.structured.match ops{["tensor.unpack"]} in %arg0 : (!transform.any_op) -> !transform.any_op
    transform.structured.vectorize %0 : !transform.any_op
     transform.yield
-  } 
+  }
+ }
+
+// -----
+
+///----------------------------------------------------------------------------------------
+/// tensor.insert_slice
+///----------------------------------------------------------------------------------------
+
+func.func private @insert_slice_static_sizes(%source: tensor<?x3x?x1xi32>) -> tensor<5x3xi32> {
+  %c2 = arith.constant 2 : index
+  %init = tensor.empty() : tensor<5x3xi32>
+
+  %source_slice = tensor.extract_slice %source[0, %c2, 0, 0] [1, 1, 5, 1] [1, 1, 1, 1] : tensor<?x3x?x1xi32> to tensor<5x1xi32>
+  %res = tensor.insert_slice %source_slice into %init[0, %c2] [5, 1] [1, 1] : tensor<5x1xi32> into tensor<5x3xi32>
+
+  return %res : tensor<5x3xi32>
+}
+
+// CHECK-LABEL:   func.func private @insert_slice_static_sizes(
+// CHECK-SAME:      %[[SEC:.*]]: tensor<?x3x?x1xi32>) -> tensor<5x3xi32> {
+// CHECK:           %[[C_2:.*]] = arith.constant 2 : index
+// CHECK:           %[[INIT:.*]] = tensor.empty() : tensor<5x3xi32>
+// CHECK:           %[[SRC_SLICE:.*]] = tensor.extract_slice %[[SEC]][0, %[[C_2]], 0, 0] [1, 1, 5, 1] [1, 1, 1, 1] : tensor<?x3x?x1xi32> to tensor<5x1xi32>
+// CHECK-DAG:       %[[PAD:.*]] = arith.constant 0 : i32
+// CHECK-DAG:       %[[C_5:.*]] = arith.constant 5 : index
+// CHECK-DAG:       %[[C_1:.*]] = arith.constant 1 : index
+// CHECK:           %[[MASK:.*]] = vector.create_mask %[[C_5]], %[[C_1]] : vector<8x1xi1>
+// CHECK:           %[[C0:.*]] = arith.constant 0 : index
+// CHECK:           %[[READ:.*]] = vector.mask %[[MASK]] { vector.transfer_read %[[SRC_SLICE]][%[[C0]], %[[C0]]], %[[PAD]] : tensor<5x1xi32>, vector<8x1xi32> } : vector<8x1xi1> -> vector<8x1xi32>
+// CHECK:           %[[C_0:.*]] = arith.constant 0 : index
+// CHECK:           %[[RES:.*]] = vector.mask %[[MASK]] { vector.transfer_write %[[READ]], %[[INIT]][%[[C_0]], %[[C_2]]] : vector<8x1xi32>, tensor<5x3xi32> } : vector<8x1xi1> -> tensor<5x3xi32>
+// CHECK:           return %[[RES]] : tensor<5x3xi32>
+
+ module attributes {transform.with_named_sequence} {
+  transform.named_sequence @__transform_main(%arg0: !transform.any_op {transform.readonly}) {
+    %0 = transform.structured.match ops{["tensor.insert_slice"]} in %arg0 : (!transform.any_op) -> !transform.any_op
+    transform.structured.vectorize %0 vector_sizes [8, 1] : !transform.any_op
+    transform.yield
+  }
+ }
+
+// -----
+
+// One of the _source_ dimensions is dynamic (but _destination_ dimensions are static).
+
+func.func private @insert_slice_dynamic_src_dim(%source: tensor<?x3x?x1xi32>, %size: index) -> tensor<5x3xi32> {
+  %c2 = arith.constant 2 : index
+  %init = tensor.empty() : tensor<5x3xi32>
+
+  %source_slice = tensor.extract_slice %source[0, %c2, 0, 0] [1, 1, %size, 1] [1, 1, 1, 1] : tensor<?x3x?x1xi32> to tensor<?x1xi32>
+  %res = tensor.insert_slice %source_slice into %init[0, %c2] [%size, 1] [1, 1] : tensor<?x1xi32> into tensor<5x3xi32>
+
+  return %res : tensor<5x3xi32>
+}
+
+// CHECK-LABEL:   func.func private @insert_slice_dynamic_src_dim(
+// CHECK-SAME:      %[[SRC:.*]]: tensor<?x3x?x1xi32>,
+// CHECK-SAME:      %[[SIZE:.*]]: index) -> tensor<5x3xi32> {
+// CHECK:           %[[C_2:.*]] = arith.constant 2 : index
+// CHECK:           %[[INIT:.*]] = tensor.empty() : tensor<5x3xi32>
+// CHECK:           %[[SRC_SLICE:.*]] = tensor.extract_slice %[[SRC]][0, %[[C_2]], 0, 0] [1, 1, %[[SIZE]], 1] [1, 1, 1, 1] : tensor<?x3x?x1xi32> to tensor<?x1xi32>
+// CHECK-DAG:       %[[PAD:.*]] = arith.constant 0 : i32
+// CHECK-DAG:       %[[C_1:.*]] = arith.constant 1 : index
+// CHECK:           %[[MASK:.*]] = vector.create_mask %[[SIZE]], %[[C_1]] : vector<8x1xi1>
+// CHECK:           %[[C_0:.*]] = arith.constant 0 : index
+// CHECK:           %[[READ:.*]] = vector.mask %[[MASK]] { vector.transfer_read %[[SRC_SLICE]][%[[C_0]], %[[C_0]]], %[[PAD]] : tensor<?x1xi32>, vector<8x1xi32> } : vector<8x1xi1> -> vector<8x1xi32>
+// CHECK:           %[[C_0_1:.*]] = arith.constant 0 : index
+// CHECK:           %[[RES:.*]] = vector.mask %[[MASK]] { vector.transfer_write %[[READ]], %[[INIT]][%[[C_0_1]], %[[C_2]]] : vector<8x1xi32>, tensor<5x3xi32> } : vector<8x1xi1> -> tensor<5x3xi32>
+// CHECK:           return %[[RES]] : tensor<5x3xi32>
+
+ module attributes {transform.with_named_sequence} {
+  transform.named_sequence @__transform_main(%arg0: !transform.any_op {transform.readonly}) {
+    %0 = transform.structured.match ops{["tensor.insert_slice"]} in %arg0 : (!transform.any_op) -> !transform.any_op
+    transform.structured.vectorize %0 vector_sizes [8, 1] : !transform.any_op
+    transform.yield
+  }
+ }
+
+// -----
+
+// One of the _destination_ dimensions is dynamic (but _source_ dimensions are static).
+
+func.func private @insert_slice_dynamic_dest_dim(%source: tensor<?x3x?x1xi32>, %size: index) -> tensor<?x3xi32> {
+  %c2 = arith.constant 2 : index
+  %init = tensor.empty(%size) : tensor<?x3xi32>
+
+  %source_slice = tensor.extract_slice %source[0, %c2, 0, 0] [1, 1, 5, 1] [1, 1, 1, 1] : tensor<?x3x?x1xi32> to tensor<5x1xi32>
+  %res = tensor.insert_slice %source_slice into %init[0, %c2] [5, 1] [1, 1] : tensor<5x1xi32> into tensor<?x3xi32>
+
+  return %res : tensor<?x3xi32>
+}
+
+// CHECK-LABEL:   func.func private @insert_slice_dynamic_dest_dim(
+// CHECK-SAME:      %[[SRC:.*]]: tensor<?x3x?x1xi32>,
+// CHECK-SAME:      %[[SIZE:.*]]: index) -> tensor<?x3xi32> {
+// CHECK:           %[[C_2:.*]] = arith.constant 2 : index
+// CHECK:           %[[INIT:.*]] = tensor.empty(%[[SIZE]]) : tensor<?x3xi32>
+// CHECK:           %[[SRC_SLICE:.*]] = tensor.extract_slice %[[SRC]][0, %[[C_2]], 0, 0] [1, 1, 5, 1] [1, 1, 1, 1] : tensor<?x3x?x1xi32> to tensor<5x1xi32>
+// CHECK:           %[[PAD:.*]] = arith.constant 0 : i32
+// CHECK:           %[[C_5:.*]] = arith.constant 5 : index
+// CHECK:           %[[C_1:.*]] = arith.constant 1 : index
+// CHECK:           %[[MASK:.*]] = vector.create_mask %[[C_5]], %[[C_1]] : vector<8x1xi1>
+// CHECK:           %[[C_0:.*]] = arith.constant 0 : index
+// CHECK:           %[[READ:.*]] = vector.mask %[[MASK]] { vector.transfer_read %[[SRC_SLICE]][%[[C_0]], %[[C_0]]], %[[PAD]] : tensor<5x1xi32>, vector<8x1xi32> } : vector<8x1xi1> -> vector<8x1xi32>
+// CHECK:           %[[C_0_1:.*]] = arith.constant 0 : index
+// CHECK:           %[[WRITE:.*]] = vector.mask %[[MASK]] { vector.transfer_write %[[READ]], %[[INIT]][%[[C_0_1]], %[[C_2]]] : vector<8x1xi32>, tensor<?x3xi32> } : vector<8x1xi1> -> tensor<?x3xi32>
+// CHECK:           return %[[WRITE]] : tensor<?x3xi32>
+
+ module attributes {transform.with_named_sequence} {
+  transform.named_sequence @__transform_main(%arg0: !transform.any_op {transform.readonly}) {
+    %0 = transform.structured.match ops{["tensor.insert_slice"]} in %arg0 : (!transform.any_op) -> !transform.any_op
+    transform.structured.vectorize %0 vector_sizes [8, 1] : !transform.any_op
+    transform.yield
+  }
+ }
+
+// -----
+
+// At least one _source_ and one _destination_ dimensions are dynamic.
+
+func.func private @insert_slice_dynamic_source_and_dest_dim(%source: tensor<?x3x?x1xi32>, %size: index) -> tensor<?x3xi32> {
+  %c2 = arith.constant 2 : index
+  %init = tensor.empty(%size) : tensor<?x3xi32>
+
+  %source_slice = tensor.extract_slice %source[0, %c2, 0, 0] [1, 1, %size, 1] [1, 1, 1, 1] : tensor<?x3x?x1xi32> to tensor<?x1xi32>
+  %res = tensor.insert_slice %source_slice into %init[0, %c2] [%size, 1] [1, 1] : tensor<?x1xi32> into tensor<?x3xi32>
+
+  return %res : tensor<?x3xi32>
+}
+
+// CHECK-LABEL:   func.func private @insert_slice_dynamic_source_and_dest_dim(
+// CHECK-SAME:      %[[SRC:.*]]: tensor<?x3x?x1xi32>,
+// CHECK-SAME:      %[[SIZE:.*]]: index) -> tensor<?x3xi32> {
+// CHECK:           %[[C_2:.*]] = arith.constant 2 : index
+// CHECK:           %[[INIT:.*]] = tensor.empty(%[[SIZE]]) : tensor<?x3xi32>
+// CHECK:           %[[SRC_SIZE:.*]] = tensor.extract_slice %[[SRC]][0, %[[C_2]], 0, 0] [1, 1, %[[SIZE]], 1] [1, 1, 1, 1] : tensor<?x3x?x1xi32> to tensor<?x1xi32>
+// CHECK:           %[[PAD:.*]] = arith.constant 0 : i32
+// CHECK:           %[[C1:.*]] = arith.constant 1 : index
+// CHECK:           %[[MASK:.*]] = vector.create_mask %[[SIZE]], %[[C1]] : vector<8x1xi1>
+// CHECK:           %[[C0:.*]] = arith.constant 0 : index
+// CHECK:           %[[READ:.*]] = vector.mask %[[MASK]] { vector.transfer_read %[[SRC_SIZE]]{{\[}}%[[C0]], %[[C0]]], %[[PAD]] : tensor<?x1xi32>, vector<8x1xi32> } : vector<8x1xi1> -> vector<8x1xi32>
+// CHECK:           %[[C_0_1:.*]] = arith.constant 0 : index
+// CHECK:           %[[WRITE:.*]] = vector.mask %[[MASK]] { vector.transfer_write %[[READ]], %[[INIT]]{{\[}}%[[C_0_1]], %[[C_2]]] : vector<8x1xi32>, tensor<?x3xi32> } : vector<8x1xi1> -> tensor<?x3xi32>
+// CHECK:           return %[[WRITE]] : tensor<?x3xi32>
+
+ module attributes {transform.with_named_sequence} {
+  transform.named_sequence @__transform_main(%arg0: !transform.any_op {transform.readonly}) {
+    %0 = transform.structured.match ops{["tensor.insert_slice"]} in %arg0 : (!transform.any_op) -> !transform.any_op
+    transform.structured.vectorize %0 vector_sizes [8, 1] : !transform.any_op
+    transform.yield
+  }
  }
diff --git a/mlir/test/Dialect/MPI/ops.mlir b/mlir/test/Dialect/MPI/ops.mlir
index 8f2421a73396c..f23a7e18a2ee9 100644
--- a/mlir/test/Dialect/MPI/ops.mlir
+++ b/mlir/test/Dialect/MPI/ops.mlir
@@ -9,6 +9,9 @@ func.func @mpi_test(%ref : memref<100xf32>) -> () {
     // CHECK-NEXT: %retval, %rank = mpi.comm_rank : !mpi.retval, i32
     %retval, %rank = mpi.comm_rank : !mpi.retval, i32
 
+    // CHECK-NEXT: %retval_0, %size = mpi.comm_size : !mpi.retval, i32
+    %retval_0, %size = mpi.comm_size : !mpi.retval, i32
+
     // CHECK-NEXT: mpi.send(%arg0, %rank, %rank) : memref<100xf32>, i32, i32
     mpi.send(%ref, %rank, %rank) : memref<100xf32>, i32, i32
 
@@ -21,13 +24,43 @@ func.func @mpi_test(%ref : memref<100xf32>) -> () {
     // CHECK-NEXT: %2 = mpi.recv(%arg0, %rank, %rank) : memref<100xf32>, i32, i32 -> !mpi.retval
     %err3 = mpi.recv(%ref, %rank, %rank) : memref<100xf32>, i32, i32 -> !mpi.retval
 
-    // CHECK-NEXT: %3 = mpi.finalize : !mpi.retval
+    // CHECK-NEXT: %req = mpi.isend(%arg0, %rank, %rank) : memref<100xf32>, i32, i32 -> !mpi.request
+    %req = mpi.isend(%ref, %rank, %rank) : memref<100xf32>, i32, i32 -> !mpi.request
+
+    // CHECK-NEXT: %retval_1, %req_2 = mpi.isend(%arg0, %rank, %rank) : memref<100xf32>, i32, i32 -> !mpi.retval, !mpi.request
+    %err4, %req2 = mpi.isend(%ref, %rank, %rank) : memref<100xf32>, i32, i32 -> !mpi.retval, !mpi.request
+
+    // CHECK-NEXT: %req_3 = mpi.irecv(%arg0, %rank, %rank) : memref<100xf32>, i32, i32 -> !mpi.request
+    %req3 = mpi.irecv(%ref, %rank, %rank) : memref<100xf32>, i32, i32 -> !mpi.request
+
+    // CHECK-NEXT: %retval_4, %req_5 = mpi.irecv(%arg0, %rank, %rank) : memref<100xf32>, i32, i32 -> !mpi.retval, !mpi.request
+    %err5, %req4 = mpi.irecv(%ref, %rank, %rank) : memref<100xf32>, i32, i32 -> !mpi.retval, !mpi.request
+
+    // CHECK-NEXT: mpi.wait(%req) : !mpi.request
+    mpi.wait(%req) : !mpi.request
+
+    // CHECK-NEXT: %3 = mpi.wait(%req_2) : !mpi.request -> !mpi.retval
+    %err6 = mpi.wait(%req2) : !mpi.request -> !mpi.retval
+
+    // CHECK-NEXT: mpi.barrier : !mpi.retval
+    mpi.barrier : !mpi.retval
+
+    // CHECK-NEXT: %5 = mpi.barrier : !mpi.retval
+    %err7 = mpi.barrier : !mpi.retval
+
+    // CHECK-NEXT: mpi.allreduce(%arg0, %arg0, <MPI_SUM>) : memref<100xf32>, memref<100xf32>
+    mpi.allreduce(%ref, %ref, <MPI_SUM>) : memref<100xf32>, memref<100xf32>
+
+    // CHECK-NEXT: mpi.allreduce(%arg0, %arg0, <MPI_SUM>) : memref<100xf32>, memref<100xf32> -> !mpi.retval
+    %err8 = mpi.allreduce(%ref, %ref, <MPI_SUM>) : memref<100xf32>, memref<100xf32> -> !mpi.retval
+
+    // CHECK-NEXT: %7 = mpi.finalize : !mpi.retval
     %rval = mpi.finalize : !mpi.retval
 
-    // CHECK-NEXT: %4 = mpi.retval_check %retval = <MPI_SUCCESS> : i1
+    // CHECK-NEXT: %8 = mpi.retval_check %retval = <MPI_SUCCESS> : i1
     %res = mpi.retval_check %retval = <MPI_SUCCESS> : i1
 
-    // CHECK-NEXT: %5 = mpi.error_class %0 : !mpi.retval
+    // CHECK-NEXT: %9 = mpi.error_class %0 : !mpi.retval
     %errclass = mpi.error_class %err : !mpi.retval
 
     // CHECK-NEXT: return
diff --git a/mlir/test/Dialect/MemRef/int-range-inference.mlir b/mlir/test/Dialect/MemRef/int-range-inference.mlir
new file mode 100644
index 0000000000000..34568d1d1d520
--- /dev/null
+++ b/mlir/test/Dialect/MemRef/int-range-inference.mlir
@@ -0,0 +1,74 @@
+// RUN: mlir-opt -int-range-optimizations --split-input-file %s | FileCheck %s
+
+// CHECK-LABEL: @dim_const
+// CHECK: %[[ret:.+]] = arith.constant 3 : index
+// CHECK: return %[[ret]]
+func.func @dim_const(%m: memref<3x5xi32>) -> index {
+  %c0 = arith.constant 0 : index
+  %0 = memref.dim %m, %c0 : memref<3x5xi32>
+  return %0 : index
+}
+
+// -----
+
+// CHECK-LABEL: @dim_any_static
+// CHECK: %[[op:.+]] = memref.dim
+// CHECK: %[[ret:.+]] = test.reflect_bounds {smax = 5 : index, smin = 3 : index, umax = 5 : index, umin = 3 : index} %[[op]]
+// CHECK: return %[[ret]]
+func.func @dim_any_static(%m: memref<3x5xi32>, %x: index) -> index {
+  %0 = memref.dim %m, %x : memref<3x5xi32>
+  %1 = test.reflect_bounds %0 : index
+  return %1 : index
+}
+
+// -----
+
+// CHECK-LABEL: @dim_dynamic
+// CHECK: %[[op:.+]] = memref.dim
+// CHECK: %[[ret:.+]] = test.reflect_bounds {smax = 9223372036854775807 : index, smin = 0 : index, umax = 9223372036854775807 : index, umin = 0 : index} %[[op]]
+// CHECK: return %[[ret]]
+func.func @dim_dynamic(%m: memref<?x5xi32>) -> index {
+  %c0 = arith.constant 0 : index
+  %0 = memref.dim %m, %c0 : memref<?x5xi32>
+  %1 = test.reflect_bounds %0 : index
+  return %1 : index
+}
+
+// -----
+
+// CHECK-LABEL: @dim_any_dynamic
+// CHECK: %[[op:.+]] = memref.dim
+// CHECK: %[[ret:.+]] = test.reflect_bounds {smax = 9223372036854775807 : index, smin = 0 : index, umax = 9223372036854775807 : index, umin = 0 : index} %[[op]]
+// CHECK: return %[[ret]]
+func.func @dim_any_dynamic(%m: memref<?x5xi32>, %x: index) -> index {
+  %0 = memref.dim %m, %x : memref<?x5xi32>
+  %1 = test.reflect_bounds %0 : index
+  return %1 : index
+}
+
+// -----
+
+// CHECK-LABEL: @dim_some_omitting_dynamic
+// CHECK: %[[op:.+]] = memref.dim
+// CHECK: %[[ret:.+]] = test.reflect_bounds {smax = 5 : index, smin = 3 : index, umax = 5 : index, umin = 3 : index} %[[op]]
+// CHECK: return %[[ret]]
+func.func @dim_some_omitting_dynamic(%m: memref<?x3x5xi32>, %x: index) -> index {
+  %c1 = arith.constant 1 : index
+  %0 = arith.maxsi %x, %c1 : index
+  %1 = memref.dim %m, %0 : memref<?x3x5xi32>
+  %2 = test.reflect_bounds %1 : index
+  return %2 : index
+}
+
+// -----
+
+// CHECK-LABEL: @dim_unranked
+// CHECK: %[[op:.+]] = memref.dim
+// CHECK: %[[ret:.+]] = test.reflect_bounds {smax = 9223372036854775807 : index, smin = 0 : index, umax = 9223372036854775807 : index, umin = 0 : index} %[[op]]
+// CHECK: return %[[ret]]
+func.func @dim_unranked(%m: memref<*xi32>) -> index {
+  %c0 = arith.constant 0 : index
+  %0 = memref.dim %m, %c0 : memref<*xi32>
+  %1 = test.reflect_bounds %0 : index
+  return %1 : index
+}
diff --git a/mlir/test/Dialect/MemRef/normalize-memrefs.mlir b/mlir/test/Dialect/MemRef/normalize-memrefs.mlir
index 6d20ccbf2ca05..e93a1a4ebae53 100644
--- a/mlir/test/Dialect/MemRef/normalize-memrefs.mlir
+++ b/mlir/test/Dialect/MemRef/normalize-memrefs.mlir
@@ -31,6 +31,20 @@ func.func @permute() {
 // CHECK-NEXT: memref.dealloc [[MEM]]
 // CHECK-NEXT: return
 
+// CHECK-LABEL: func @alloca
+func.func @alloca(%idx : index) {
+  // CHECK-NEXT: memref.alloca() : memref<65xf32>
+  %A = memref.alloca() : memref<64xf32, affine_map<(d0) -> (d0 + 1)>>
+  // CHECK-NEXT: affine.load %{{.*}}[symbol(%arg0) + 1] : memref<65xf32>
+  affine.load %A[%idx] : memref<64xf32, affine_map<(d0) -> (d0 + 1)>>
+  affine.for %i = 0 to 64 {
+    %1 = affine.load %A[%i] : memref<64xf32, affine_map<(d0) -> (d0 + 1)>>
+    "prevent.dce"(%1) : (f32) -> ()
+    // CHECK: %{{.*}} = affine.load %{{.*}}[%arg{{.*}} + 1] : memref<65xf32>
+  }
+  return
+}
+
 // CHECK-LABEL: func @shift
 func.func @shift(%idx : index) {
   // CHECK-NEXT: memref.alloc() : memref<65xf32>
diff --git a/mlir/test/Dialect/NVGPU/optimize-shared-memory.mlir b/mlir/test/Dialect/NVGPU/optimize-shared-memory.mlir
index 5a212815ceb2a..7477e18728677 100644
--- a/mlir/test/Dialect/NVGPU/optimize-shared-memory.mlir
+++ b/mlir/test/Dialect/NVGPU/optimize-shared-memory.mlir
@@ -238,3 +238,13 @@ func.func @abort_if_subview(%arg0: memref<128x128xf16>,
 
   return %mat: vector<1x2xf16>
 }
+
+// -----
+
+// Ensure this case not crash
+
+// CHECK-LABEL: func @test_0_d
+func.func @test_0_d() -> memref<i32, #gpu.address_space<workgroup>> {
+  %alloc = memref.alloc() : memref<i32, #gpu.address_space<workgroup>>
+  return %alloc : memref<i32, #gpu.address_space<workgroup>>
+}
diff --git a/mlir/test/Dialect/OpenACC/ops.mlir b/mlir/test/Dialect/OpenACC/ops.mlir
index 2706792a263a8..28ab6f9fcfb4c 100644
--- a/mlir/test/Dialect/OpenACC/ops.mlir
+++ b/mlir/test/Dialect/OpenACC/ops.mlir
@@ -1892,3 +1892,44 @@ func.func @acc_combined() {
 // CHECK: acc.loop combined(kernels)
 // CHECK: acc.serial combined(loop)
 // CHECK: acc.loop combined(serial)
+
+acc.firstprivate.recipe @firstprivatization_memref_i32 : memref<i32> init {
+^bb0(%arg0: memref<i32>):
+  %alloca = memref.alloca() : memref<i32>
+  acc.yield %alloca : memref<i32>
+} copy {
+^bb0(%arg0: memref<i32>, %arg1: memref<i32>):
+  %0 = memref.load %arg1[] : memref<i32>
+  memref.store %0, %arg0[] : memref<i32>
+  acc.terminator
+}
+
+// CHECK-LABEL: acc.firstprivate.recipe @firstprivatization_memref_i32
+// CHECK:       memref.alloca
+
+acc.reduction.recipe @reduction_add_memref_i32 : memref<i32> reduction_operator <add> init {
+^bb0(%arg0: memref<i32>):
+  %c0_i32 = arith.constant 0 : i32
+  %alloca = memref.alloca() : memref<i32>
+  memref.store %c0_i32, %alloca[] : memref<i32>
+  acc.yield %alloca : memref<i32>
+} combiner {
+^bb0(%arg0: memref<i32>, %arg1: memref<i32>):
+  %0 = memref.load %arg0[] : memref<i32>
+  %1 = memref.load %arg1[] : memref<i32>
+  %2 = arith.addi %0, %1 : i32
+  memref.store %2, %arg0[] : memref<i32>
+  acc.yield %arg0 : memref<i32>
+}
+
+// CHECK-LABEL: acc.reduction.recipe @reduction_add_memref_i32
+// CHECK:       memref.alloca
+
+acc.private.recipe @privatization_memref_i32 : memref<i32> init {
+^bb0(%arg0: memref<i32>):
+  %alloca = memref.alloca() : memref<i32>
+  acc.yield %alloca : memref<i32>
+}
+
+// CHECK-LABEL: acc.private.recipe @privatization_memref_i32
+// CHECK:       memref.alloca
diff --git a/mlir/test/Dialect/OpenMP/invalid.mlir b/mlir/test/Dialect/OpenMP/invalid.mlir
index 06fcf90e34480..02b0af17564d4 100644
--- a/mlir/test/Dialect/OpenMP/invalid.mlir
+++ b/mlir/test/Dialect/OpenMP/invalid.mlir
@@ -2435,8 +2435,8 @@ func.func @omp_distribute_unconstrained_order() -> () {
   return
 }
 // -----
-omp.private {type = private} @x.privatizer : i32 alloc {
-^bb0(%arg0: i32):
+omp.private {type = private} @x.privatizer : i32 init {
+^bb0(%arg0: i32, %arg1: i32):
   %0 = arith.constant 0.0 : f32
   // expected-error @below {{Invalid yielded value. Expected type: 'i32', got: 'f32'}}
   omp.yield(%0 : f32)
@@ -2444,17 +2444,17 @@ omp.private {type = private} @x.privatizer : i32 alloc {
 
 // -----
 
-omp.private {type = private} @x.privatizer : i32 alloc {
-^bb0(%arg0: i32):
-  // expected-error @below {{Invalid yielded value. Expected type: 'i32', got: None}}
+// expected-error @below {{Region argument type mismatch: got 'f32' expected 'i32'.}}
+omp.private {type = private} @x.privatizer : i32 init {
+^bb0(%arg0: i32, %arg1: f32):
   omp.yield
 }
 
 // -----
 
-omp.private {type = private} @x.privatizer : f32 alloc {
-^bb0(%arg0: f32):
-  omp.yield(%arg0 : f32)
+omp.private {type = private} @x.privatizer : f32 init {
+^bb0(%arg0: f32, %arg1: f32):
+  omp.yield(%arg0: f32)
 } dealloc {
 ^bb0(%arg0: f32):
   // expected-error @below {{Did not expect any values to be yielded.}}
@@ -2463,27 +2463,24 @@ omp.private {type = private} @x.privatizer : f32 alloc {
 
 // -----
 
-omp.private {type = private} @x.privatizer : i32 alloc {
-^bb0(%arg0: i32):
+omp.private {type = private} @x.privatizer : i32 init {
+^bb0(%arg0: i32, %arg1: i32):
   // expected-error @below {{expected exit block terminator to be an `omp.yield` op.}}
   omp.terminator
 }
 
 // -----
 
-// expected-error @below {{`alloc`: expected 1 region arguments, got: 2}}
-omp.private {type = private} @x.privatizer : f32 alloc {
-^bb0(%arg0: f32, %arg1: f32):
+// expected-error @below {{`init`: expected 2 region arguments, got: 1}}
+omp.private {type = private} @x.privatizer : f32 init {
+^bb0(%arg0: f32):
   omp.yield(%arg0 : f32)
 }
 
 // -----
 
 // expected-error @below {{`copy`: expected 2 region arguments, got: 1}}
-omp.private {type = firstprivate} @x.privatizer : f32 alloc {
-^bb0(%arg0: f32):
-  omp.yield(%arg0 : f32)
-} copy {
+omp.private {type = firstprivate} @x.privatizer : f32 copy {
 ^bb0(%arg0: f32):
   omp.yield(%arg0 : f32)
 }
@@ -2491,30 +2488,24 @@ omp.private {type = firstprivate} @x.privatizer : f32 alloc {
 // -----
 
 // expected-error @below {{`dealloc`: expected 1 region arguments, got: 2}}
-omp.private {type = private} @x.privatizer : f32 alloc {
-^bb0(%arg0: f32):
-  omp.yield(%arg0 : f32)
-} dealloc {
+omp.private {type = private} @x.privatizer : f32 dealloc {
 ^bb0(%arg0: f32, %arg1: f32):
   omp.yield
 }
 
 // -----
 
-// expected-error @below {{`private` clauses require only an `alloc` region.}}
-omp.private {type = private} @x.privatizer : f32 alloc {
-^bb0(%arg0: f32):
-  omp.yield(%arg0 : f32)
-} copy {
+// expected-error @below {{`private` clauses do not require a `copy` region.}}
+omp.private {type = private} @x.privatizer : f32 copy {
 ^bb0(%arg0: f32, %arg1 : f32):
   omp.yield(%arg0 : f32)
 }
 
 // -----
 
-// expected-error @below {{`firstprivate` clauses require both `alloc` and `copy` regions.}}
-omp.private {type = firstprivate} @x.privatizer : f32 alloc {
-^bb0(%arg0: f32):
+// expected-error @below {{`firstprivate` clauses require at least a `copy` region.}}
+omp.private {type = firstprivate} @x.privatizer : f32 init {
+^bb0(%arg0: f32, %arg1: f32):
   omp.yield(%arg0 : f32)
 }
 
@@ -2529,8 +2520,8 @@ func.func @private_type_mismatch(%arg0: index) {
   return
 }
 
-omp.private {type = private} @var1.privatizer : !llvm.ptr alloc {
-^bb0(%arg0: !llvm.ptr):
+omp.private {type = private} @var1.privatizer : index init {
+^bb0(%arg0: !llvm.ptr, %arg1: !llvm.ptr):
   omp.yield(%arg0 : !llvm.ptr)
 }
 
@@ -2545,10 +2536,7 @@ func.func @firstprivate_type_mismatch(%arg0: index) {
   return
 }
 
-omp.private {type = firstprivate} @var1.privatizer : !llvm.ptr alloc {
-^bb0(%arg0: !llvm.ptr):
-  omp.yield(%arg0 : !llvm.ptr)
-} copy {
+omp.private {type = firstprivate} @var1.privatizer : index copy {
 ^bb0(%arg0: !llvm.ptr, %arg1: !llvm.ptr):
   omp.yield(%arg0 : !llvm.ptr)
 }
@@ -2576,10 +2564,7 @@ func.func @undefined_privatizer(%arg0: !llvm.ptr) {
 
 // -----
 
-omp.private {type = private} @var1.privatizer : !llvm.ptr alloc {
-^bb0(%arg0: !llvm.ptr):
-  omp.yield(%arg0 : !llvm.ptr)
-} copy {
+omp.private {type = private} @var1.privatizer : !llvm.ptr copy {
 ^bb0(%arg0: !llvm.ptr, %arg1: !llvm.ptr):
   omp.yield(%arg0 : !llvm.ptr)
 }
diff --git a/mlir/test/Dialect/OpenMP/ops.mlir b/mlir/test/Dialect/OpenMP/ops.mlir
index c1259fabe82fb..aca63600876aa 100644
--- a/mlir/test/Dialect/OpenMP/ops.mlir
+++ b/mlir/test/Dialect/OpenMP/ops.mlir
@@ -2685,34 +2685,31 @@ func.func @parallel_op_privatizers(%arg0: !llvm.ptr, %arg1: !llvm.ptr) {
   return
 }
 
-// CHECK-LABEL: omp.private {type = private} @a.privatizer : !llvm.ptr alloc {
-omp.private {type = private} @a.privatizer : !llvm.ptr alloc {
-// CHECK: ^bb0(%{{.*}}: {{.*}}):
-^bb0(%arg0: !llvm.ptr):
+// CHECK-LABEL: omp.private {type = private} @a.privatizer : !llvm.ptr init {
+omp.private {type = private} @a.privatizer : !llvm.ptr init {
+// CHECK: ^bb0(%{{.*}}: {{.*}}, %{{.*}}: {{.*}}):
+^bb0(%arg0: !llvm.ptr, %arg1: !llvm.ptr):
   omp.yield(%arg0 : !llvm.ptr)
 }
 
-// CHECK-LABEL: omp.private {type = private} @x.privatizer : !llvm.ptr alloc {
-omp.private {type = private} @x.privatizer : !llvm.ptr alloc {
-// CHECK: ^bb0(%{{.*}}: {{.*}}):
-^bb0(%arg0: !llvm.ptr):
+// CHECK-LABEL: omp.private {type = private} @x.privatizer : !llvm.ptr init {
+omp.private {type = private} @x.privatizer : !llvm.ptr init {
+// CHECK: ^bb0(%{{.*}}: {{.*}}, %{{.*}}: {{.*}}):
+^bb0(%arg0: !llvm.ptr, %arg1: !llvm.ptr):
   omp.yield(%arg0 : !llvm.ptr)
+// CHECK: } dealloc {
 } dealloc {
 // CHECK: ^bb0(%{{.*}}: {{.*}}):
 ^bb0(%arg0: !llvm.ptr):
   omp.yield
 }
 
-// CHECK-LABEL: omp.private {type = firstprivate} @y.privatizer : !llvm.ptr alloc {
-omp.private {type = firstprivate} @y.privatizer : !llvm.ptr alloc {
-// CHECK: ^bb0(%{{.*}}: {{.*}}):
-^bb0(%arg0: !llvm.ptr):
-  omp.yield(%arg0 : !llvm.ptr)
-// CHECK: } copy {
-} copy {
+// CHECK-LABEL: omp.private {type = firstprivate} @y.privatizer : !llvm.ptr copy {
+omp.private {type = firstprivate} @y.privatizer : !llvm.ptr copy {
 // CHECK: ^bb0(%{{.*}}: {{.*}}, %{{.*}}: {{.*}}):
 ^bb0(%arg0: !llvm.ptr, %arg1: !llvm.ptr):
   omp.yield(%arg0 : !llvm.ptr)
+// CHECK: } dealloc {
 } dealloc {
 // CHECK: ^bb0(%{{.*}}: {{.*}}):
 ^bb0(%arg0: !llvm.ptr):
diff --git a/mlir/test/Dialect/SCF/value-bounds-op-interface-impl.mlir b/mlir/test/Dialect/SCF/value-bounds-op-interface-impl.mlir
index b48f38f592dc9..339d97df001c5 100644
--- a/mlir/test/Dialect/SCF/value-bounds-op-interface-impl.mlir
+++ b/mlir/test/Dialect/SCF/value-bounds-op-interface-impl.mlir
@@ -270,6 +270,47 @@ func.func @compare_scf_for(%a: index, %b: index, %c: index) {
 
 // -----
 
+func.func @scf_for_induction_var_upper_bound() {
+  %c0 = arith.constant 0 : index
+  %c1 = arith.constant 1 : index
+  %c2 = arith.constant 2 : index
+  %c3 = arith.constant 3 : index
+  %c4 = arith.constant 4 : index
+  %c5 = arith.constant 5 : index
+  %c8 = arith.constant 8 : index
+  %c10 = arith.constant 10 : index
+  scf.for %iv = %c0 to %c10 step %c4 {
+    // expected-remark @below{{true}}
+    "test.compare"(%iv, %c8) {cmp = "LE"} : (index, index) -> ()
+  }
+  scf.for %iv = %c2 to %c8 step %c3 {
+    // expected-remark @below{{true}}
+    "test.compare"(%iv, %c5) {cmp = "LE"} : (index, index) -> ()
+  }
+  return
+}
+
+// -----
+
+#map_ceildiv_dynamic_divisor = affine_map<(i)[s] -> (i ceildiv s)>
+func.func @scf_for_induction_var_computed_upper_bound(%upperBound: index, %step: index) {
+  %c0 = arith.constant 0 : index
+  %c1 = arith.constant 1 : index
+  %tripCount = affine.apply #map_ceildiv_dynamic_divisor (%upperBound)[%step]
+  %tripCountMinusOne = arith.subi %tripCount, %c1 : index
+  %computedUpperBound = arith.muli %tripCountMinusOne, %step : index
+  scf.for %iv = %c0 to %upperBound step %step {
+    // TODO: Value bounds analysis will fail to compute upper bound
+    // because multiplication/division of unknown block arguments is
+    // not supported.
+    // expected-error @below{{unknown}}
+    "test.compare"(%iv, %computedUpperBound) {cmp = "LE"} : (index, index) -> ()
+  }
+  return
+}
+
+// -----
+
 func.func @scf_for_result_infer() {
   %c0 = arith.constant 0 : index
   %c1 = arith.constant 1 : index
diff --git a/mlir/test/Dialect/SPIRV/IR/group-ops.mlir b/mlir/test/Dialect/SPIRV/IR/group-ops.mlir
index c879b901311f2..d9957ad804161 100644
--- a/mlir/test/Dialect/SPIRV/IR/group-ops.mlir
+++ b/mlir/test/Dialect/SPIRV/IR/group-ops.mlir
@@ -196,3 +196,63 @@ func.func @group_fmul(%value: f32) -> f32 {
   %0 = spirv.KHR.GroupFMul <Workgroup> <Reduce> %value : f32
   return %0: f32
 }
+
+// -----
+
+//===----------------------------------------------------------------------===//
+// spirv.GroupNonUniformBallotBitCount 
+//===----------------------------------------------------------------------===//
+
+func.func @group_non_uniform_ballot_bit_count(%value: vector<4xi32>) -> i32 {
+  // CHECK: {{%.*}} = spirv.GroupNonUniformBallotBitCount <Subgroup> <Reduce> {{%.*}} : vector<4xi32> -> i32
+  %0 = spirv.GroupNonUniformBallotBitCount <Subgroup> <Reduce> %value : vector<4xi32> -> i32
+  return %0: i32
+}
+
+// -----
+
+func.func @group_non_uniform_ballot_bit_count_wrong_scope(%value: vector<4xi32>) -> i32 {
+  // expected-error @+1 {{execution_scope must be Scope of value Subgroup}}
+  %0 = spirv.GroupNonUniformBallotBitCount <Workgroup> <Reduce> %value : vector<4xi32> -> i32
+  return %0: i32
+}
+
+// -----
+
+func.func @group_non_uniform_ballot_bit_count_wrong_value_len(%value: vector<3xi32>) -> i32 {
+  // expected-error @+1 {{operand #0 must be vector of 32-bit signless/unsigned integer values of length 4, but got 'vector<3xi32>'}}
+  %0 = spirv.GroupNonUniformBallotBitCount <Subgroup> <InclusiveScan> %value : vector<3xi32> -> i32
+  return %0: i32
+}
+
+// -----
+
+func.func @group_non_uniform_ballot_bit_count_wrong_value_type(%value: vector<4xi8>) -> i32 {
+  // expected-error @+1 {{operand #0 must be vector of 32-bit signless/unsigned integer values of length 4, but got 'vector<4xi8>'}}
+  %0 = spirv.GroupNonUniformBallotBitCount <Subgroup> <InclusiveScan> %value : vector<4xi8> -> i32
+  return %0: i32
+}
+
+// -----
+
+func.func @group_non_uniform_ballot_bit_count_value_sign(%value: vector<4xsi32>) -> i32 {
+  // expected-error @+1 {{operand #0 must be vector of 32-bit signless/unsigned integer values of length 4, but got 'vector<4xsi32>'}}
+  %0 = spirv.GroupNonUniformBallotBitCount <Subgroup> <InclusiveScan> %value : vector<4xsi32> -> i32
+  return %0: i32
+}
+
+// -----
+
+func.func @group_non_uniform_ballot_bit_count_wrong_result_type(%value: vector<4xi32>) -> f32 {
+  // expected-error @+1 {{result #0 must be 8/16/32/64-bit signless/unsigned integer, but got 'f32'}}
+  %0 = spirv.GroupNonUniformBallotBitCount <Subgroup> <InclusiveScan> %value : vector<4xi32> -> f32
+  return %0: f32
+}
+
+// -----
+
+func.func @group_non_uniform_ballot_bit_count_wrong_result_sign(%value: vector<4xi32>) -> si32 {
+  // expected-error @+1 {{result #0 must be 8/16/32/64-bit signless/unsigned integer, but got 'si32'}}
+  %0 = spirv.GroupNonUniformBallotBitCount <Subgroup> <InclusiveScan> %value : vector<4xi32> -> si32
+  return %0: si32
+}
diff --git a/mlir/test/Dialect/SPIRV/IR/image-ops.mlir b/mlir/test/Dialect/SPIRV/IR/image-ops.mlir
index ab674ee0809ae..1161f85563ae6 100644
--- a/mlir/test/Dialect/SPIRV/IR/image-ops.mlir
+++ b/mlir/test/Dialect/SPIRV/IR/image-ops.mlir
@@ -115,3 +115,45 @@ func.func @image_query_size_error_result2(%arg0 : !spirv.image<f32, Buffer, NoDe
 }
 
 // -----
+
+//===----------------------------------------------------------------------===//
+// spirv.ImageWrite
+//===----------------------------------------------------------------------===//
+
+func.func @image_write(%arg0 : !spirv.image<f32, Dim2D, NoDepth, NonArrayed, SingleSampled, NoSampler, Rgba16>, %arg1 : vector<2xsi32>, %arg2 : vector<4xf32>) -> () {
+  // CHECK:  spirv.ImageWrite %arg0 : !spirv.image<f32, Dim2D, NoDepth, NonArrayed, SingleSampled, NoSampler, Rgba16>, %arg1 : vector<2xsi32>, %arg2 : vector<4xf32>
+  spirv.ImageWrite %arg0 : !spirv.image<f32, Dim2D, NoDepth, NonArrayed, SingleSampled, NoSampler, Rgba16>, %arg1 : vector<2xsi32>, %arg2 : vector<4xf32>
+  spirv.Return
+}
+
+// -----
+
+func.func @image_write_scalar_texel(%arg0 : !spirv.image<f32, Dim2D, NoDepth, NonArrayed, SingleSampled, NoSampler, Rgba16>, %arg1 : vector<2xsi32>, %arg2 : f32) -> () {
+  // CHECK:  spirv.ImageWrite %arg0 : !spirv.image<f32, Dim2D, NoDepth, NonArrayed, SingleSampled, NoSampler, Rgba16>, %arg1 : vector<2xsi32>, %arg2 : f32
+  spirv.ImageWrite %arg0 : !spirv.image<f32, Dim2D, NoDepth, NonArrayed, SingleSampled, NoSampler, Rgba16>, %arg1 : vector<2xsi32>, %arg2 : f32
+  spirv.Return
+}
+
+// -----
+
+func.func @image_write_need_sampler(%arg0 : !spirv.image<f32, Dim2D, NoDepth, NonArrayed, SingleSampled, NeedSampler, Rgba16>, %arg1 : vector<2xsi32>, %arg2 : vector<4xf32>) -> () {
+  // expected-error @+1 {{the sampled operand of the underlying image must be 0 or 2}}
+  spirv.ImageWrite %arg0 : !spirv.image<f32, Dim2D, NoDepth, NonArrayed, SingleSampled, NeedSampler, Rgba16>, %arg1 : vector<2xsi32>, %arg2 : vector<4xf32>
+  spirv.Return
+}
+
+// -----
+
+func.func @image_write_subpass_data(%arg0 : !spirv.image<f32, SubpassData, NoDepth, NonArrayed, SingleSampled, NoSampler, Rgba16>, %arg1 : vector<2xsi32>, %arg2 : vector<4xf32>) -> () {
+  // expected-error @+1 {{the Dim operand of the underlying image must not be SubpassData}}
+  spirv.ImageWrite %arg0 : !spirv.image<f32, SubpassData, NoDepth, NonArrayed, SingleSampled, NoSampler, Rgba16>, %arg1 : vector<2xsi32>, %arg2 : vector<4xf32>
+  spirv.Return
+}
+
+// -----
+
+func.func @image_write_texel_type_mismatch(%arg0 : !spirv.image<f32, Dim2D, NoDepth, NonArrayed, SingleSampled, NoSampler, Rgba16>, %arg1 : vector<2xsi32>, %arg2 : vector<4xi32>) -> () {
+  // expected-error @+1 {{the texel component type must match the image sampled type}}
+  spirv.ImageWrite %arg0 : !spirv.image<f32, Dim2D, NoDepth, NonArrayed, SingleSampled, NoSampler, Rgba16>, %arg1 : vector<2xsi32>, %arg2 : vector<4xi32>
+  spirv.Return
+}
diff --git a/mlir/test/Dialect/SPIRV/IR/matrix-ops.mlir b/mlir/test/Dialect/SPIRV/IR/matrix-ops.mlir
index 37e7514d664ef..ba95322dbf38b 100644
--- a/mlir/test/Dialect/SPIRV/IR/matrix-ops.mlir
+++ b/mlir/test/Dialect/SPIRV/IR/matrix-ops.mlir
@@ -36,6 +36,13 @@ spirv.module Logical GLSL450 requires #spirv.vce<v1.0, [Shader], []> {
     spirv.ReturnValue %result : vector<4xf32>
   }
 
+  // CHECK-LABEL: @vector_times_matrix_1
+  spirv.func @vector_times_matrix_1(%arg0: vector<3xf32>, %arg1: !spirv.matrix<4 x vector<3xf32>>) -> vector<4xf32> "None" {
+    // CHECK: {{%.*}} = spirv.VectorTimesMatrix {{%.*}}, {{%.*}} : vector<3xf32>, !spirv.matrix<4 x vector<3xf32>> -> vector<4xf32>
+    %result = spirv.VectorTimesMatrix %arg0, %arg1 : vector<3xf32>, !spirv.matrix<4 x vector<3xf32>> -> vector<4xf32>
+    spirv.ReturnValue %result : vector<4xf32>
+  }
+
   // CHECK-LABEL: @matrix_times_matrix_1
   spirv.func @matrix_times_matrix_1(%arg0: !spirv.matrix<3 x vector<3xf32>>, %arg1: !spirv.matrix<3 x vector<3xf32>>) -> !spirv.matrix<3 x vector<3xf32>> "None"{
     // CHECK: {{%.*}} = spirv.MatrixTimesMatrix {{%.*}}, {{%.*}} : !spirv.matrix<3 x vector<3xf32>>, !spirv.matrix<3 x vector<3xf32>> -> !spirv.matrix<3 x vector<3xf32>>
@@ -123,7 +130,6 @@ func.func @matrix_times_matrix_component_type_mismatch_1(%arg0 : !spirv.matrix<3
    return
 }
 
-
 // -----
 
 func.func @matrix_times_matrix_component_type_mismatch_2(%arg0 : !spirv.matrix<3 x vector<3xf64>>, %arg1 : !spirv.matrix<3x vector<3xf32>>){
@@ -135,7 +141,7 @@ func.func @matrix_times_matrix_component_type_mismatch_2(%arg0 : !spirv.matrix<3
 // -----
 
 func.func @matrix_times_vector_element_type_mismatch(%arg0: !spirv.matrix<4 x vector<3xf32>>, %arg1: vector<4xf16>) {
-  // expected-error @+1 {{matrix, vector, and result element types must match}}
+  // expected-error @+1 {{op failed to verify that all of {vector, result} have same element type}}
   %result = spirv.MatrixTimesVector %arg0, %arg1 : !spirv.matrix<4 x vector<3xf32>>, vector<4xf16> -> vector<3xf32>
   return
 }
@@ -155,3 +161,35 @@ func.func @matrix_times_vector_column_mismatch(%arg0: !spirv.matrix<4 x vector<3
   %result = spirv.MatrixTimesVector %arg0, %arg1 : !spirv.matrix<4 x vector<3xf32>>, vector<3xf32> -> vector<3xf32>
   return
 }
+
+// -----
+
+func.func @vector_times_matrix_vector_matrix_mismatch(%arg0: vector<4xf32>, %arg1: !spirv.matrix<4 x vector<3xf32>>) {
+  // expected-error @+1 {{number of components in vector must equal the number of components in each column in matrix}}
+  %result = spirv.VectorTimesMatrix %arg0, %arg1 : vector<4xf32>, !spirv.matrix<4 x vector<3xf32>> -> vector<3xf32>
+  return
+}
+
+// -----
+
+func.func @vector_times_matrix_result_matrix_mismatch(%arg0: vector<3xf32>, %arg1: !spirv.matrix<4 x vector<3xf32>>) {
+  // expected-error @+1 {{number of columns in matrix must equal the number of components in result}}
+  %result = spirv.VectorTimesMatrix %arg0, %arg1 : vector<3xf32>, !spirv.matrix<4 x vector<3xf32>> -> vector<3xf32>
+  return
+}
+
+// -----
+
+func.func @vector_times_matrix_vector_type_mismatch(%arg0: vector<3xf16>, %arg1: !spirv.matrix<4 x vector<3xf32>>) {
+  // expected-error @+1 {{op failed to verify that all of {vector, result} have same element type}}
+  %result = spirv.VectorTimesMatrix %arg0, %arg1 : vector<3xf16>, !spirv.matrix<4 x vector<3xf32>> -> vector<4xf32>
+  return
+}
+
+// -----
+
+func.func @vector_times_matrix_matrix_type_mismatch(%arg0: vector<3xf32>, %arg1: !spirv.matrix<4 x vector<3xf16>>) {
+  // expected-error @+1 {{matrix must be a matrix with the same component type as the component type in result}}
+  %result = spirv.VectorTimesMatrix %arg0, %arg1 : vector<3xf32>, !spirv.matrix<4 x vector<3xf16>> -> vector<4xf32>
+  return
+}
diff --git a/mlir/test/Dialect/SparseTensor/specifier_to_llvm.mlir b/mlir/test/Dialect/SparseTensor/specifier_to_llvm.mlir
index 00ff29125fb5f..8df8ac6b1675c 100644
--- a/mlir/test/Dialect/SparseTensor/specifier_to_llvm.mlir
+++ b/mlir/test/Dialect/SparseTensor/specifier_to_llvm.mlir
@@ -3,7 +3,7 @@
 #CSR = #sparse_tensor.encoding<{map = (d0, d1) -> (d0 : dense, d1 : compressed)}>
 
 // CHECK-LABEL:   func.func @sparse_metadata_init() -> !llvm.struct<(array<2 x i64>, array<3 x i64>)> {
-// CHECK-DAG:       %[[STRUCT:.*]] = llvm.mlir.undef : !llvm.struct<(array<2 x i64>, array<3 x i64>)>
+// CHECK-DAG:       %[[STRUCT:.*]] = llvm.mlir.poison : !llvm.struct<(array<2 x i64>, array<3 x i64>)>
 // CHECK-DAG:       %[[CST0:.*]] = arith.constant 0 : i64
 // CHECK:           %[[VAL_1:.*]] = llvm.insertvalue %[[CST0]], %[[STRUCT]][1, 0] : !llvm.struct<(array<2 x i64>, array<3 x i64>)>
 // CHECK:           %[[VAL_2:.*]] = llvm.insertvalue %[[CST0]], %[[VAL_1]][1, 1] : !llvm.struct<(array<2 x i64>, array<3 x i64>)>
diff --git a/mlir/test/Dialect/Tensor/bufferize.mlir b/mlir/test/Dialect/Tensor/bufferize.mlir
index ecd285be46194..9ea0a15f31185 100644
--- a/mlir/test/Dialect/Tensor/bufferize.mlir
+++ b/mlir/test/Dialect/Tensor/bufferize.mlir
@@ -372,7 +372,7 @@ func.func @tensor.expand_shape(%t1: tensor<?x10xf32>, %sz0: index) -> tensor<2x?
   // CHECK: %[[C0:.*]] = arith.constant 0 : index
   // CHECK: %[[DIM:.*]] = memref.dim %[[m1]], %[[C0]] : memref<?x10xf32>
   // CHECK: %[[C2:.*]] = arith.constant 2 : index
-  // CHECK: %[[VAL_1:.*]] = arith.divui %[[DIM]], %[[C2]] : index
+  // CHECK: %[[VAL_1:.*]] = arith.divsi %[[DIM]], %[[C2]] : index
   // CHECK: %[[expanded:.*]] = memref.expand_shape %[[m1]] {{\[\[}}0, 1], [2]] output_shape [2, %[[VAL_1]], 10] : memref<?x10xf32> into memref<2x?x10xf32>
   %0 = tensor.expand_shape %t1 [[0, 1], [2]] output_shape [2, %sz0, 10]
       : tensor<?x10xf32> into tensor<2x?x10xf32>
@@ -393,7 +393,7 @@ func.func @tensor.expand_shape_of_slice(
   %0 = tensor.extract_slice %t1[%o1, 5][%s1, 10][1, 1] :
       tensor<?x20xf32> to tensor<?x10xf32>
   // CHECK: %[[C7:.*]] = arith.constant 7 : index
-  // CHECK: %[[VAL_1:.*]] = arith.divui %{{.*}}, %[[C7]] : index
+  // CHECK: %[[VAL_1:.*]] = arith.divsi %{{.*}}, %[[C7]] : index
   // CHECK: %[[expanded:.*]] = memref.expand_shape %[[subview]] {{\[\[}}0, 1], [2, 3]] output_shape [%[[VAL_1]], 7, 2, 5] : memref<?x10xf32, strided<[20, 1], offset: ?>> into memref<?x7x2x5xf32, strided<[140, 20, 5, 1], offset: ?>>
   %1 = tensor.expand_shape %0 [[0, 1], [2, 3]] output_shape [%sz0, 7, 2, 5] :
       tensor<?x10xf32> into tensor<?x7x2x5xf32>
diff --git a/mlir/test/Dialect/Tensor/fold-empty-op.mlir b/mlir/test/Dialect/Tensor/fold-empty-op.mlir
index 65ceb4ff3e3df..850bbcee34020 100644
--- a/mlir/test/Dialect/Tensor/fold-empty-op.mlir
+++ b/mlir/test/Dialect/Tensor/fold-empty-op.mlir
@@ -10,7 +10,6 @@ module attributes {transform.with_named_sequence} {
   }
 }
 
-// CHECK: #[[$MAP:.+]] = affine_map<()[s0] -> (s0 floordiv 28)>
 // CHECK: #[[$MAP2:.+]] = affine_map<()[s0] -> (s0 * 28)>
 
 func.func @empty_reshape_expansion(%arg0 : index, %sz0: index) -> tensor<2x3x5x4x?x7xf32> {
@@ -19,11 +18,9 @@ func.func @empty_reshape_expansion(%arg0 : index, %sz0: index) -> tensor<2x3x5x4
   return %1 : tensor<2x3x5x4x?x7xf32>
 }
 // CHECK-LABEL: func @empty_reshape_expansion
-// CHECK-SAME:     %[[ARG0:.+]]: index
-// CHECK:        %[[OLD_INIT:.+]] = tensor.empty(%{{.*}}) : tensor<6x5x?xf32>
-// CHECK-NEXT:   %[[DIM:.*]] = tensor.dim %[[OLD_INIT]]
-// CHECK-NEXT:   %[[D:.+]] = affine.apply #[[$MAP]]()[%[[DIM]]]
-// CHECK-NEXT:   %[[INIT:.+]] = tensor.empty(%[[D]])
+// CHECK-SAME:     %[[ARG0:[a-zA-Z0-9]+]]: index,
+// CHECK-SAME:     %[[ARG1:[a-zA-Z0-9]+]]: index
+// CHECK-NEXT:   %[[INIT:.+]] = tensor.empty(%[[ARG1]])
 // CHECK-NEXT:   return %[[INIT]]
 
 func.func @empty_reshape_collapse(%arg0 : index) -> tensor<6x5x?xf32> {
diff --git a/mlir/test/Dialect/Tensor/int-range-inference.mlir b/mlir/test/Dialect/Tensor/int-range-inference.mlir
new file mode 100644
index 0000000000000..e90ebf5fccb8e
--- /dev/null
+++ b/mlir/test/Dialect/Tensor/int-range-inference.mlir
@@ -0,0 +1,74 @@
+// RUN: mlir-opt -int-range-optimizations --split-input-file %s | FileCheck %s
+
+// CHECK-LABEL: @dim_const
+// CHECK: %[[ret:.+]] = arith.constant 3 : index
+// CHECK: return %[[ret]]
+func.func @dim_const(%t: tensor<3x5xi32>) -> index {
+  %c0 = arith.constant 0 : index
+  %0 = tensor.dim %t, %c0 : tensor<3x5xi32>
+  return %0 : index
+}
+
+// -----
+
+// CHECK-LABEL: @dim_any_static
+// CHECK: %[[op:.+]] = tensor.dim
+// CHECK: %[[ret:.+]] = test.reflect_bounds {smax = 5 : index, smin = 3 : index, umax = 5 : index, umin = 3 : index} %[[op]]
+// CHECK: return %[[ret]]
+func.func @dim_any_static(%t: tensor<3x5xi32>, %x: index) -> index {
+  %0 = tensor.dim %t, %x : tensor<3x5xi32>
+  %1 = test.reflect_bounds %0 : index
+  return %1 : index
+}
+
+// -----
+
+// CHECK-LABEL: @dim_dynamic
+// CHECK: %[[op:.+]] = tensor.dim
+// CHECK: %[[ret:.+]] = test.reflect_bounds {smax = 9223372036854775807 : index, smin = 0 : index, umax = 9223372036854775807 : index, umin = 0 : index} %[[op]]
+// CHECK: return %[[ret]]
+func.func @dim_dynamic(%t: tensor<?x5xi32>) -> index {
+  %c0 = arith.constant 0 : index
+  %0 = tensor.dim %t, %c0 : tensor<?x5xi32>
+  %1 = test.reflect_bounds %0 : index
+  return %1 : index
+}
+
+// -----
+
+// CHECK-LABEL: @dim_any_dynamic
+// CHECK: %[[op:.+]] = tensor.dim
+// CHECK: %[[ret:.+]] = test.reflect_bounds {smax = 9223372036854775807 : index, smin = 0 : index, umax = 9223372036854775807 : index, umin = 0 : index} %[[op]]
+// CHECK: return %[[ret]]
+func.func @dim_any_dynamic(%t: tensor<?x5xi32>, %x: index) -> index {
+  %0 = tensor.dim %t, %x : tensor<?x5xi32>
+  %1 = test.reflect_bounds %0 : index
+  return %1 : index
+}
+
+// -----
+
+// CHECK-LABEL: @dim_some_omitting_dynamic
+// CHECK: %[[op:.+]] = tensor.dim
+// CHECK: %[[ret:.+]] = test.reflect_bounds {smax = 5 : index, smin = 3 : index, umax = 5 : index, umin = 3 : index} %[[op]]
+// CHECK: return %[[ret]]
+func.func @dim_some_omitting_dynamic(%t: tensor<?x3x5xi32>, %x: index) -> index {
+  %c1 = arith.constant 1 : index
+  %0 = arith.maxsi %x, %c1 : index
+  %1 = tensor.dim %t, %0 : tensor<?x3x5xi32>
+  %2 = test.reflect_bounds %1 : index
+  return %2 : index
+}
+
+// -----
+
+// CHECK-LABEL: @dim_unranked
+// CHECK: %[[op:.+]] = tensor.dim
+// CHECK: %[[ret:.+]] = test.reflect_bounds {smax = 9223372036854775807 : index, smin = 0 : index, umax = 9223372036854775807 : index, umin = 0 : index} %[[op]]
+// CHECK: return %[[ret]]
+func.func @dim_unranked(%t: tensor<*xi32>) -> index {
+  %c0 = arith.constant 0 : index
+  %0 = tensor.dim %t, %c0 : tensor<*xi32>
+  %1 = test.reflect_bounds %0 : index
+  return %1 : index
+}
diff --git a/mlir/test/Dialect/Tosa/canonicalize.mlir b/mlir/test/Dialect/Tosa/canonicalize.mlir
index 6f47f041b9199..24d572244a9b0 100644
--- a/mlir/test/Dialect/Tosa/canonicalize.mlir
+++ b/mlir/test/Dialect/Tosa/canonicalize.mlir
@@ -52,25 +52,16 @@ func.func @cast_nofold(%arg0: tensor<?x1xf32>) -> tensor<?x1xi32> {
 // CHECK-LABEL: @clamp_i32_not_noop
 func.func @clamp_i32_not_noop(%arg0: tensor<4xi32>) -> tensor<4xi32> {
   // CHECK: tosa.clamp
-  %0 = tosa.clamp %arg0 {min_int = 1 : i64, max_int = 4 : i64, min_fp = 1.0 : f32, max_fp = 4.0 : f32} : (tensor<4xi32>) -> tensor<4xi32>
+  %0 = tosa.clamp %arg0 {min_val = 1 : i32, max_val = 4 : i32} : (tensor<4xi32>) -> tensor<4xi32>
   return %0 : tensor<4xi32>
 }
 
 // -----
 
-// CHECK-LABEL: @clamp_f16_not_noop
-func.func @clamp_f16_not_noop(%arg0: tensor<4xf16>) -> tensor<4xf16> {
-  // CHECK: tosa.clamp
-  %0 = tosa.clamp %arg0 {min_int = -128 : i64, max_int = 127 : i64, min_fp = -3.40282347E+38 : f32, max_fp = 3.40282347E+38 : f32} : (tensor<4xf16>) -> tensor<4xf16>
-  return %0 : tensor<4xf16>
-}
-
-// -----
-
 // CHECK-LABEL: @clamp_f32_not_noop
 func.func @clamp_f32_not_noop(%arg0: tensor<4xf32>) -> tensor<4xf32> {
   // CHECK: tosa.clamp
-  %0 = tosa.clamp %arg0 {min_int = -128 : i64, max_int = 127 : i64, min_fp = -3.40282347E+38 : f32, max_fp = 3.40282347E+38 : f32} : (tensor<4xf32>) -> tensor<4xf32>
+  %0 = tosa.clamp %arg0 {min_val = -3.40282347E+38 : f32, max_val = 3.40282347E+38 : f32} : (tensor<4xf32>) -> tensor<4xf32>
   return %0 : tensor<4xf32>
 }
 
@@ -80,8 +71,8 @@ func.func @clamp_f32_not_noop(%arg0: tensor<4xf32>) -> tensor<4xf32> {
 func.func @clamp_f16_is_noop(%arg0: tensor<4xf16>) -> tensor<4xf16> {
   // CHECK: return %arg0
   // CHECK-NOT: "tosa.clamp"
-  // 0xFF800000 and 0x7F800000 are respectively negative and positive F32 infinity.
-  %0 = tosa.clamp %arg0 {min_int = -128 : i64, max_int = 127 : i64, min_fp = 0xFF800000 : f32, max_fp = 0x7F800000 : f32} : (tensor<4xf16>) -> tensor<4xf16>
+  // 0x7C00 and 0xFC00 are respectively positive and negative F32 infinity.
+  %0 = tosa.clamp %arg0 {max_val = 0x7C00 : f16, min_val = 0xFC00 : f16} : (tensor<4xf16>) -> tensor<4xf16>
   return %0 : tensor<4xf16>
 }
 
@@ -92,7 +83,7 @@ func.func @clamp_f32_is_noop(%arg0: tensor<4xf32>) -> tensor<4xf32> {
   // CHECK: return %arg0
   // CHECK-NOT: "tosa.clamp"
   // 0xFF800000 and 0x7F800000 are respectively negative and positive F32 infinity.
-  %0 = tosa.clamp %arg0 {min_int = -128 : i64, max_int = 127 : i64, min_fp = 0xFF800000 : f32, max_fp = 0x7F800000 : f32} : (tensor<4xf32>) -> tensor<4xf32>
+  %0 = tosa.clamp %arg0 {min_val = 0xFF800000 : f32, max_val = 0x7F800000 : f32} : (tensor<4xf32>) -> tensor<4xf32>
   return %0 : tensor<4xf32>
 }
 
@@ -102,7 +93,7 @@ func.func @clamp_f32_is_noop(%arg0: tensor<4xf32>) -> tensor<4xf32> {
 func.func @clamp_int8_is_noop(%arg0: tensor<4xi8>) -> tensor<4xi8> {
   // CHECK: return %arg0
   // CHECK-NOT: tosa.clamp
-  %0 = tosa.clamp %arg0 {min_int = -128 : i64, max_int = 127 : i64, min_fp = -3.40282347E+38 : f32, max_fp = 3.40282347E+38 : f32} :  (tensor<4xi8>) -> tensor<4xi8>
+  %0 = tosa.clamp %arg0 {min_val = -128 : i8, max_val = 127 : i8} :  (tensor<4xi8>) -> tensor<4xi8>
   return %0 : tensor<4xi8>
 }
 
@@ -112,7 +103,7 @@ func.func @clamp_int8_is_noop(%arg0: tensor<4xi8>) -> tensor<4xi8> {
 func.func @clamp_int16_is_noop(%arg0: tensor<4xi16>) -> tensor<4xi16> {
   // CHECK: return %arg0
   // CHECK-NOT: tosa.clamp
-  %0 = tosa.clamp %arg0 {min_int = -32768 : i64, max_int = 32767 : i64, min_fp = -3.40282347E+38 : f32, max_fp = 3.40282347E+38 : f32} :  (tensor<4xi16>) -> tensor<4xi16>
+  %0 = tosa.clamp %arg0 {min_val = -32768 : i16, max_val = 32767 : i16} :  (tensor<4xi16>) -> tensor<4xi16>
   return %0 : tensor<4xi16>
 }
 
@@ -122,7 +113,7 @@ func.func @clamp_int16_is_noop(%arg0: tensor<4xi16>) -> tensor<4xi16> {
 func.func @clamp_uint8_is_noop(%arg0: tensor<4xui8>) -> tensor<4xui8> {
   // CHECK: return %arg0
   // CHECK-NOT: tosa.clamp
-  %0 = tosa.clamp %arg0 {min_int = 0 : i64, max_int = 255 : i64, min_fp = -3.40282347E+38 : f32, max_fp = 3.40282347E+38 : f32} :  (tensor<4xui8>) -> tensor<4xui8>
+  %0 = tosa.clamp %arg0 {min_val = 0 : ui8, max_val = 255 : ui8} :  (tensor<4xui8>) -> tensor<4xui8>
   return %0 : tensor<4xui8>
 }
 
@@ -130,9 +121,9 @@ func.func @clamp_uint8_is_noop(%arg0: tensor<4xui8>) -> tensor<4xui8> {
 
 // CHECK-LABEL: @clamp_twice_is_single_clamp
 func.func @clamp_twice_is_single_clamp(%arg0: tensor<4xi8>) -> tensor<4xi8> {
-  // CHECK: tosa.clamp %arg0 {max_fp = 3.000000e+00 : f32, max_int = 2 : i64, min_fp = -3.000000e+00 : f32, min_int = -2 : i64}
-  %0 = tosa.clamp %arg0 {max_fp = 3.0 : f32, max_int = 4 : i64, min_fp = -5.0 : f32, min_int = -2 : i64} :  (tensor<4xi8>) -> tensor<4xi8>
-  %1 = tosa.clamp %0 {max_fp = 5.0 : f32, max_int = 2 : i64, min_fp = -3.0 : f32, min_int = -4 : i64} :  (tensor<4xi8>) -> tensor<4xi8>
+  // CHECK: tosa.clamp %arg0 {max_val = 2 : i8, min_val = -2 : i8}
+  %0 = tosa.clamp %arg0 {max_val = 4 : i8, min_val = -2 : i8} :  (tensor<4xi8>) -> tensor<4xi8>
+  %1 = tosa.clamp %0 {max_val = 2 : i8, min_val = -4 : i8} :  (tensor<4xi8>) -> tensor<4xi8>
   return %1 : tensor<4xi8>
 }
 
@@ -140,10 +131,10 @@ func.func @clamp_twice_is_single_clamp(%arg0: tensor<4xi8>) -> tensor<4xi8> {
 
 // CHECK: @disjoint_clamp_twice_is_not_single_clamp(%[[INPUT:.*]]: tensor<4xi8>)
 func.func @disjoint_clamp_twice_is_not_single_clamp(%arg0: tensor<4xi8>) -> tensor<4xi8> {
-  // CHECK: %[[CLAMP_1:.*]] = tosa.clamp %[[INPUT]] {max_fp = -5.000000e+00 : f32, max_int = -5 : i64, min_fp = -1.000000e+00 : f32, min_int = -10 : i64} :  (tensor<4xi8>) -> tensor<4xi8>
-  // CHECK-NEXT: tosa.clamp %[[CLAMP_1]] {max_fp = 5.000000e+00 : f32, max_int = 5 : i64, min_fp = 1.000000e+00 : f32, min_int = 1 : i64} :  (tensor<4xi8>) -> tensor<4xi8>
-  %0 = tosa.clamp %arg0 {max_fp = -5.0 : f32, max_int = -5 : i64, min_fp = -1.0 : f32, min_int = -10 : i64} :  (tensor<4xi8>) -> tensor<4xi8>
-  %1 = tosa.clamp %0 {max_fp = 5.0 : f32, max_int = 5 : i64, min_fp = 1.0 : f32, min_int = 1 : i64} :  (tensor<4xi8>) -> tensor<4xi8>
+  // CHECK: %[[CLAMP_1:.*]] = tosa.clamp %[[INPUT]] {max_val = -5 : i8, min_val = -10 : i8} :  (tensor<4xi8>) -> tensor<4xi8>
+  // CHECK-NEXT: tosa.clamp %[[CLAMP_1]] {max_val = 5 : i8, min_val = 1 : i8} :  (tensor<4xi8>) -> tensor<4xi8>
+  %0 = tosa.clamp %arg0 {max_val = -5 : i8, min_val = -10 : i8} :  (tensor<4xi8>) -> tensor<4xi8>
+  %1 = tosa.clamp %0 {max_val = 5 : i8, min_val = 1 : i8} :  (tensor<4xi8>) -> tensor<4xi8>
   return %1 : tensor<4xi8>
 }
 
@@ -151,9 +142,9 @@ func.func @disjoint_clamp_twice_is_not_single_clamp(%arg0: tensor<4xi8>) -> tens
 
 // CHECK-LABEL: @clamp_twice_with_nan_propagate_is_single_clamp
 func.func @clamp_twice_with_nan_propagate_is_single_clamp(%arg0: tensor<4xi8>) -> tensor<4xi8> {
-  // CHECK: tosa.clamp %arg0 {max_fp = 3.000000e+00 : f32, max_int = 2 : i64, min_fp = -3.000000e+00 : f32, min_int = -2 : i64}
-  %0 = tosa.clamp %arg0 {max_fp = 3.0 : f32, max_int = 4 : i64, min_fp = -5.0 : f32, min_int = -2 : i64, nan_mode = "PROPAGATE"} :  (tensor<4xi8>) -> tensor<4xi8>
-  %1 = tosa.clamp %0 {max_fp = 5.0 : f32, max_int = 2 : i64, min_fp = -3.0 : f32, min_int = -4 : i64, nan_mode = "PROPAGATE"} :  (tensor<4xi8>) -> tensor<4xi8>
+  // CHECK: tosa.clamp %arg0 {max_val = 2 : i8, min_val = -2 : i8}
+  %0 = tosa.clamp %arg0 {max_val = 4 : i8, min_val = -2 : i8, nan_mode = "PROPAGATE"} :  (tensor<4xi8>) -> tensor<4xi8>
+  %1 = tosa.clamp %0 {max_val = 2 : i8, min_val = -4 : i8, nan_mode = "PROPAGATE"} :  (tensor<4xi8>) -> tensor<4xi8>
   return %1 : tensor<4xi8>
 }
 
@@ -161,9 +152,9 @@ func.func @clamp_twice_with_nan_propagate_is_single_clamp(%arg0: tensor<4xi8>) -
 
 // CHECK-LABEL: @clamp_twice_with_nan_ignore_is_single_clamp
 func.func @clamp_twice_with_nan_ignore_is_single_clamp(%arg0: tensor<4xi8>) -> tensor<4xi8> {
-  // CHECK: tosa.clamp %arg0 {max_fp = 3.000000e+00 : f32, max_int = 2 : i64, min_fp = -3.000000e+00 : f32, min_int = -2 : i64, nan_mode = "IGNORE"}
-  %0 = tosa.clamp %arg0 {max_fp = 3.0 : f32, max_int = 4 : i64, min_fp = -5.0 : f32, min_int = -2 : i64, nan_mode = "IGNORE"} :  (tensor<4xi8>) -> tensor<4xi8>
-  %1 = tosa.clamp %0 {max_fp = 5.0 : f32, max_int = 2 : i64, min_fp = -3.0 : f32, min_int = -4 : i64, nan_mode = "IGNORE"} :  (tensor<4xi8>) -> tensor<4xi8>
+  // CHECK: tosa.clamp %arg0 {max_val = 2 : i8, min_val = -2 : i8, nan_mode = "IGNORE"}
+  %0 = tosa.clamp %arg0 {max_val = 4 : i8, min_val = -2 : i8, nan_mode = "IGNORE"} :  (tensor<4xi8>) -> tensor<4xi8>
+  %1 = tosa.clamp %0 {max_val = 2 : i8, min_val = -4 : i8, nan_mode = "IGNORE"} :  (tensor<4xi8>) -> tensor<4xi8>
   return %1 : tensor<4xi8>
 }
 
@@ -171,9 +162,9 @@ func.func @clamp_twice_with_nan_ignore_is_single_clamp(%arg0: tensor<4xi8>) -> t
 
 // CHECK-LABEL: @clamp_twice_with_nan_ignore_propagate_is_single_clamp
 func.func @clamp_twice_with_nan_ignore_propagate_is_single_clamp(%arg0: tensor<4xi8>) -> tensor<4xi8> {
-  // CHECK: tosa.clamp %arg0 {max_fp = 3.000000e+00 : f32, max_int = 2 : i64, min_fp = -3.000000e+00 : f32, min_int = -2 : i64, nan_mode = "IGNORE"}
-  %0 = tosa.clamp %arg0 {max_fp = 3.0 : f32, max_int = 4 : i64, min_fp = -5.0 : f32, min_int = -2 : i64, nan_mode = "IGNORE"} :  (tensor<4xi8>) -> tensor<4xi8>
-  %1 = tosa.clamp %0 {max_fp = 5.0 : f32, max_int = 2 : i64, min_fp = -3.0 : f32, min_int = -4 : i64, nan_mode = "PROPAGATE"} :  (tensor<4xi8>) -> tensor<4xi8>
+  // CHECK: tosa.clamp %arg0 {max_val = 2 : i8, min_val = -2 : i8, nan_mode = "IGNORE"}
+  %0 = tosa.clamp %arg0 {max_val = 4 : i8, min_val = -2 : i8, nan_mode = "IGNORE"} :  (tensor<4xi8>) -> tensor<4xi8>
+  %1 = tosa.clamp %0 {max_val = 2 : i8, min_val = -4 : i8, nan_mode = "PROPAGATE"} :  (tensor<4xi8>) -> tensor<4xi8>
   return %1 : tensor<4xi8>
 }
 
@@ -181,10 +172,10 @@ func.func @clamp_twice_with_nan_ignore_propagate_is_single_clamp(%arg0: tensor<4
 
 // CHECK: @clamp_twice_with_nan_propagate_ignore_is_not_single_clamp(%[[INPUT:.*]]: tensor<4xi8>)
 func.func @clamp_twice_with_nan_propagate_ignore_is_not_single_clamp(%arg0: tensor<4xi8>) -> tensor<4xi8> {
-  // CHECK: %[[CLAMP_1:.*]] = tosa.clamp %[[INPUT]] {max_fp = 3.000000e+00 : f32, max_int = 4 : i64, min_fp = -5.000000e+00 : f32, min_int = -2 : i64} :  (tensor<4xi8>) -> tensor<4xi8>
-  // CHECK-NEXT: tosa.clamp %[[CLAMP_1]] {max_fp = 5.000000e+00 : f32, max_int = 2 : i64, min_fp = -3.000000e+00 : f32, min_int = -4 : i64, nan_mode = "IGNORE"} :  (tensor<4xi8>) -> tensor<4xi8>
-  %0 = tosa.clamp %arg0 {max_fp = 3.0 : f32, max_int = 4 : i64, min_fp = -5.0 : f32, min_int = -2 : i64, nan_mode = "PROPAGATE"} :  (tensor<4xi8>) -> tensor<4xi8>
-  %1 = tosa.clamp %0 {max_fp = 5.0 : f32, max_int = 2 : i64, min_fp = -3.0 : f32, min_int = -4 : i64, nan_mode = "IGNORE"} :  (tensor<4xi8>) -> tensor<4xi8>
+  // CHECK: %[[CLAMP_1:.*]] = tosa.clamp %[[INPUT]] {max_val = 4 : i8, min_val = -2 : i8} :  (tensor<4xi8>) -> tensor<4xi8>
+  // CHECK-NEXT: tosa.clamp %[[CLAMP_1]] {max_val = 2 : i8, min_val = -4 : i8, nan_mode = "IGNORE"} :  (tensor<4xi8>) -> tensor<4xi8>
+  %0 = tosa.clamp %arg0 {max_val = 4 : i8, min_val = -2 : i8, nan_mode = "PROPAGATE"} :  (tensor<4xi8>) -> tensor<4xi8>
+  %1 = tosa.clamp %0 {max_val = 2 : i8, min_val = -4 : i8, nan_mode = "IGNORE"} :  (tensor<4xi8>) -> tensor<4xi8>
   return %1 : tensor<4xi8>
 }
 
@@ -317,7 +308,7 @@ func.func @pad_determine_val_f32(%arg0: tensor<?x?xf32>, %arg1 : tensor<2x2xi32>
 
 // CHECK-LABEL: @pad_determine_val_quant
 func.func @pad_determine_val_quant(%arg0: tensor<?x?xi32>, %arg1 : tensor<2x2xi32>) -> tensor<?x?xi32> {
-  // CHECK-DAG: %[[ZERO:.+]] = "tosa.const"() <{value = dense<0> : tensor<i32>}
+  // CHECK-DAG: %[[ZERO:.+]] = "tosa.const"() <{value = dense<42> : tensor<i32>}
   // CHECK-DAG: %[[PADDING:.+]] = tosa.const_shape {value = dense<[1, 0, 0, 1]> : tensor<4xindex>} : () -> !tosa.shape<4>
   // CHECK: tosa.pad %arg0, %[[PADDING]], %[[ZERO]]
   %0 = tosa.const_shape { value = dense<[1, 0, 0, 1]> : tensor<4xindex>} : () -> !tosa.shape<4>
@@ -331,8 +322,9 @@ func.func @pad_determine_val_quant(%arg0: tensor<?x?xi32>, %arg1 : tensor<2x2xi3
 func.func @mul_one_float(%arg0: tensor<2x3xf32>) -> tensor<2x3xf32> {
   // CHECK: return %arg0
   // CHECK-NOT: tosa.mul
+  %shift = "tosa.const"() <{value = dense<0> : tensor<1xi8>}> : () -> tensor<1xi8>
   %ones = "tosa.const"() {value = dense<1.0> : tensor<2x3xf32>} : () -> tensor<2x3xf32>
-  %1 = tosa.mul %arg0, %ones {shift = 0 : i8} : (tensor<2x3xf32>, tensor<2x3xf32>) -> tensor<2x3xf32>
+  %1 = tosa.mul %arg0, %ones, %shift : (tensor<2x3xf32>, tensor<2x3xf32>, tensor<1xi8>) -> tensor<2x3xf32>
   return %1 : tensor<2x3xf32>
 }
 
@@ -343,7 +335,8 @@ func.func @mul_bcast_one_float(%arg0: tensor<2x3xf32>) -> tensor<2x3xf32> {
   // CHECK: return %arg0
   // CHECK-NOT: tosa.mul
   %ones = "tosa.const"() {value = dense<1.0> : tensor<1x1xf32>} : () -> tensor<1x1xf32>
-  %1 = tosa.mul %ones, %arg0 {shift = 0 : i8} : (tensor<1x1xf32>, tensor<2x3xf32>) -> tensor<2x3xf32>
+  %shift = "tosa.const"() <{value = dense<0> : tensor<1xi8>}> : () -> tensor<1xi8>
+  %1 = tosa.mul %ones, %arg0, %shift : (tensor<1x1xf32>, tensor<2x3xf32>, tensor<1xi8>) -> tensor<2x3xf32>
   return %1 : tensor<2x3xf32>
 }
 
@@ -353,8 +346,22 @@ func.func @mul_bcast_one_float(%arg0: tensor<2x3xf32>) -> tensor<2x3xf32> {
 func.func @mul_one_int(%arg0: tensor<2x3xi32>) -> tensor<2x3xi32> {
   // CHECK: return %arg0
   // CHECK-NOT: tosa.mul
+  %shift = "tosa.const"() <{value = dense<0> : tensor<1xi8>}> : () -> tensor<1xi8>
   %ones = "tosa.const"() {value = dense<1> : tensor<2x3xi32>} : () -> tensor<2x3xi32>
-  %1 = tosa.mul %arg0, %ones {shift = 0 : i8} : (tensor<2x3xi32>, tensor<2x3xi32>) -> tensor<2x3xi32>
+  %1 = tosa.mul %arg0, %ones, %shift : (tensor<2x3xi32>, tensor<2x3xi32>, tensor<1xi8>) -> tensor<2x3xi32>
+  return %1 : tensor<2x3xi32>
+}
+
+// -----
+
+// CHECK-LABEL: @mul_one_int_and_shift
+func.func @mul_one_int_and_shift(%arg0: tensor<2x3xi32>) -> tensor<2x3xi32> {
+  // CHECK-DAG: %[[VAL_1:.*]] = "tosa.const"() <{value = dense<1> : tensor<2x3xi32>}>
+  // CHECK-DAG: %[[VAL_2:.*]] = "tosa.const"() <{value = dense<31> : tensor<1xi8>}>
+  // CHECK: %[[VAL_3:.*]] = tosa.mul %arg0, %[[VAL_1]], %[[VAL_2]] : (tensor<2x3xi32>, tensor<2x3xi32>, tensor<1xi8>)
+  %ones = "tosa.const"() {value = dense<1> : tensor<2x3xi32>} : () -> tensor<2x3xi32>
+  %shift = "tosa.const"() <{value = dense<31> : tensor<1xi8>}> : () -> tensor<1xi8>
+  %1 = tosa.mul %arg0, %ones, %shift : (tensor<2x3xi32>, tensor<2x3xi32>, tensor<1xi8>) -> tensor<2x3xi32>
   return %1 : tensor<2x3xi32>
 }
 
@@ -365,11 +372,12 @@ func.func @mul_zero_broadcast(%arg0: tensor<2x3xf32>) -> (tensor<2x3xf32>, tenso
   // CHECK: %[[ZERO:.*]] = "tosa.const"() <{value = dense<0.000000e+00> : tensor<2x3xf32>}
   // CHECK-NOT: tosa.mul
   %zeros = "tosa.const"() {value = dense<0.0> : tensor<1x1xf32>} : () -> tensor<1x1xf32>
-  %1 = tosa.mul %arg0, %zeros {shift = 0 : i8} : (tensor<2x3xf32>, tensor<1x1xf32>) -> tensor<2x3xf32>
+  %shift = "tosa.const"() <{value = dense<0> : tensor<1xi8>}> : () -> tensor<1xi8>
+  %1 = tosa.mul %arg0, %zeros, %shift : (tensor<2x3xf32>, tensor<1x1xf32>, tensor<1xi8>) -> tensor<2x3xf32>
 
   // CHECK-NOT: tosa.mul
   // CHECK: return %[[ZERO]], %[[ZERO]]
-  %2 = tosa.mul %zeros, %arg0 {shift = 0 : i8} : (tensor<1x1xf32>, tensor<2x3xf32>) -> tensor<2x3xf32>
+  %2 = tosa.mul %zeros, %arg0, %shift : (tensor<1x1xf32>, tensor<2x3xf32>, tensor<1xi8>) -> tensor<2x3xf32>
   return %1, %2 : tensor<2x3xf32>, tensor<2x3xf32>
 }
 
@@ -528,17 +536,20 @@ func.func @reduce_sum_nofold(%arg0: tensor<?x1xf32>) -> tensor<?x1xf32> {
 // CHECK-LABEL: @reshape_canonicalize
 func.func @reshape_canonicalize(%arg0: tensor<?x10xf32>) -> tensor<?x10xf32> {
   // CHECK: return %arg0
-  %0 = tosa.reshape %arg0 {new_shape = array<i64: -1, 10>}: (tensor<?x10xf32>) -> tensor<?x10xf32>
-  return %0 : tensor<?x10xf32>
+  %0 = "tosa.const_shape"() {value = dense<[-1, 10]> : tensor<2xindex>} : () -> !tosa.shape<2>
+  %1 = tosa.reshape %arg0, %0 : (tensor<?x10xf32>, !tosa.shape<2>) -> tensor<?x10xf32>
+  return %1 : tensor<?x10xf32>
 }
 
 // -----
 
 // CHECK-LABEL: @reshape_canonicalize_dyn_nofold
 func.func @reshape_canonicalize_dyn_nofold(%arg0: tensor<?x?x10xf32>) -> tensor<?x?x10xf32> {
-  // CHECK: %[[VAR0:.+]] = tosa.reshape %arg0 {new_shape = array<i64: -1, 2, 10>} : (tensor<?x?x10xf32>) -> tensor<?x?x10xf32>
+  // CHECK: %[[SHAPE:.+]] = tosa.const_shape {value = dense<[-1, 2, 10]> : tensor<3xindex>} : () -> !tosa.shape<3>
+  // CHECK: %[[VAR0:.+]] = tosa.reshape %arg0, %[[SHAPE]] : (tensor<?x?x10xf32>, !tosa.shape<3>) -> tensor<?x?x10xf32>
   // CHECK: return %[[VAR0]] : tensor<?x?x10xf32>
-  %0 = tosa.reshape %arg0 {new_shape = array<i64: -1, 2, 10>} : (tensor<?x?x10xf32>) -> tensor<?x?x10xf32>
+  %s = "tosa.const_shape"() {value = dense<[-1, 2, 10]> : tensor<3xindex>} : () -> !tosa.shape<3>
+  %0 = tosa.reshape %arg0, %s : (tensor<?x?x10xf32>, !tosa.shape<3>) -> tensor<?x?x10xf32>
   return %0 : tensor<?x?x10xf32>
 }
 
@@ -546,10 +557,13 @@ func.func @reshape_canonicalize_dyn_nofold(%arg0: tensor<?x?x10xf32>) -> tensor<
 
 // CHECK-LABEL: @reshape_canonicalize_double
 func.func @reshape_canonicalize_double(%arg0: tensor<?x10xf32>) -> tensor<?x5xf32> {
-  // CHECK: %[[VAL_1:.*]] = tosa.reshape %arg0 {new_shape = array<i64: -1, 5>}
+  // CHECK: %[[VAL_0:.*]] = tosa.const_shape {value = dense<[-1, 5]> : tensor<2xindex>} : () -> !tosa.shape<2>
+  // CHECK: %[[VAL_1:.*]] = tosa.reshape %arg0, %[[VAL_0]]
   // CHECK: return %[[VAL_1]]
-  %0 = tosa.reshape %arg0 {new_shape = array<i64: 5, -1>}: (tensor<?x10xf32>) -> tensor<5x?xf32>
-  %1 = tosa.reshape %0 {new_shape = array<i64: -1, 5>}: (tensor<5x?xf32>) -> tensor<?x5xf32>
+  %cst0 = "tosa.const_shape"() <{value = dense<[5, -1]> : tensor<2xindex>}> : () -> !tosa.shape<2>
+  %0 = tosa.reshape %arg0, %cst0 : (tensor<?x10xf32>, !tosa.shape<2>) -> tensor<5x?xf32>
+  %cst1 = "tosa.const_shape"() <{value = dense<[-1, 5]> : tensor<2xindex>}> : () -> !tosa.shape<2>
+  %1 = tosa.reshape %0, %cst1 : (tensor<5x?xf32>, !tosa.shape<2>) -> tensor<?x5xf32>
   return %1 : tensor<?x5xf32>
 }
 
@@ -560,8 +574,9 @@ func.func @reshape_canonicalize_const() -> tensor<1x5xi32> {
   // CHECK: %[[VAR0:.+]] = "tosa.const"() <{value = dense<{{\[\[}}0, 1, 2, 3, 4]]> : tensor<1x5xi32>}
   // CHECK: return %[[VAR0]]
   %0 = "tosa.const"() {value = dense<[0, 1, 2, 3, 4]> : tensor<5xi32>} : () -> tensor<5xi32>
-  %1 = tosa.reshape %0 {new_shape = array<i64: 1, 5>} : (tensor<5xi32>) -> tensor<1x5xi32>
-  return %1 : tensor<1x5xi32>
+  %1 = "tosa.const_shape"() {value = dense<[1, 5]> : tensor<2xindex>} : () -> !tosa.shape<2>
+  %2 = tosa.reshape %0, %1 : (tensor<5xi32>, !tosa.shape<2>) -> tensor<1x5xi32>
+  return %2 : tensor<1x5xi32>
 }
 
 // -----
@@ -570,7 +585,8 @@ func.func @reshape_canonicalize_const() -> tensor<1x5xi32> {
 func.func @reshape_canonicalize_const_dynamic() -> tensor<1x?xi32> {
   // CHECK: tosa.reshape
   %0 = "tosa.const"() {value = dense<[0, 1, 2, 3, 4]> : tensor<5xi32>} : () -> tensor<5xi32>
-  %1 = tosa.reshape %0 {new_shape = array<i64: 1, 5>} : (tensor<5xi32>) -> tensor<1x?xi32>
+  %2 = "tosa.const_shape"() {value = dense<[1, 5]> : tensor<2xindex>} : () -> !tosa.shape<2>
+  %1 = tosa.reshape %0, %2 : (tensor<5xi32>, !tosa.shape<2>) -> tensor<1x?xi32>
   return %1 : tensor<1x?xi32>
 }
 
@@ -582,7 +598,8 @@ func.func @reshape_canonicalize_const_splat() -> (tensor<10xi32>, tensor<1x10xi3
   // CHECK-DAG: %[[VAR1:.+]] = "tosa.const"() <{value = dense<0> : tensor<1x10xi32>}
   // CHECK: return %[[VAR0]], %[[VAR1]]
   %0 = "tosa.const"() {value = dense<0> : tensor<10xi32>} : () -> tensor<10xi32>
-  %1 = tosa.reshape %0 {new_shape = array<i64: 1, 10>} : (tensor<10xi32>) -> tensor<1x10xi32>
+  %2 = "tosa.const_shape"() {value = dense<[1, 10]> : tensor<2xindex>} : () -> !tosa.shape<2>
+  %1 = tosa.reshape %0, %2 : (tensor<10xi32>, !tosa.shape<2>) -> tensor<1x10xi32>
   return %0 , %1 : tensor<10xi32>, tensor<1x10xi32>
 }
 
@@ -592,7 +609,8 @@ func.func @reshape_canonicalize_const_splat() -> (tensor<10xi32>, tensor<1x10xi3
 func.func @reshape_canonicalize_const_sparse() -> (tensor<3xi32>, tensor<1x3xi32>) {
   // CHECK: tosa.reshape
   %0 = "tosa.const"() {value = dense<[1, 2, 3]> : tensor<3xi32>} : ()-> tensor<3xi32>
-  %1 = tosa.reshape %0 {new_shape = array<i64: 1, 3>} : (tensor<3xi32>) -> tensor<1x3xi32>
+  %2 = "tosa.const_shape"() {value = dense<[1, 3]> : tensor<2xindex>} : () -> !tosa.shape<2>
+  %1 = tosa.reshape %0, %2 : (tensor<3xi32>, !tosa.shape<2>) -> tensor<1x3xi32>
   return %0 , %1 : tensor<3xi32>, tensor<1x3xi32>
 }
 
@@ -602,9 +620,10 @@ func.func @reshape_canonicalize_const_sparse() -> (tensor<3xi32>, tensor<1x3xi32
 func.func @reshape_canonicalize_quant_nofold() -> (tensor<1x3x!quant.uniform<i8:f32, 1.000000e+00>>) {
   // disabled folding for quantized element types
   // CHECK{LITERAL}: "tosa.const"() <{value = dense<[1, 2, 3]> : tensor<3xi8>}> : () -> tensor<3x!quant.uniform<i8:f32, 1.000000e+00>>
-  // CHECK{LITERAL}: tosa.reshape %0 {new_shape = array<i64: 1, 3>} : (tensor<3x!quant.uniform<i8:f32, 1.000000e+00>>) -> tensor<1x3x!quant.uniform<i8:f32, 1.000000e+00>>
+  // CHECK{LITERAL}: tosa.reshape %0, %1 : (tensor<3x!quant.uniform<i8:f32, 1.000000e+00>>, !tosa.shape<2>) -> tensor<1x3x!quant.uniform<i8:f32, 1.000000e+00>>
   %0 = "tosa.const"() {value = dense<[1, 2, 3]> : tensor<3xi8>} : ()-> tensor<3x!quant.uniform<i8:f32, 1.000000e+00>>
-  %1 = tosa.reshape %0 {new_shape = array<i64: 1, 3>} : (tensor<3x!quant.uniform<i8:f32, 1.000000e+00>>) -> tensor<1x3x!quant.uniform<i8:f32, 1.000000e+00>>
+  %2 = "tosa.const_shape"() {value = dense<[1, 3]> : tensor<2xindex>} : () -> !tosa.shape<2>
+  %1 = tosa.reshape %0, %2 : (tensor<3x!quant.uniform<i8:f32, 1.000000e+00>>, !tosa.shape<2>) -> tensor<1x3x!quant.uniform<i8:f32, 1.000000e+00>>
   return %1 :  tensor<1x3x!quant.uniform<i8:f32, 1.000000e+00>>
 }
 
@@ -612,8 +631,9 @@ func.func @reshape_canonicalize_quant_nofold() -> (tensor<1x3x!quant.uniform<i8:
 
 // CHECK-LABEL: @transpose_canonicalize_strip_quant
 func.func @transpose_canonicalize_strip_quant() -> (tensor<2x1x3x!quant.uniform<i8:f32, 1.000000e+00>>) {
-  // CHECK: "tosa.const"() <{value = dense<0> : tensor<1x2x3xi8>}> : () -> tensor<1x2x3x!quant.uniform<i8:f32, 1.000000e+00>>
-  // CHECK: tosa.reshape %0 {new_shape = array<i64: 2, 1, 3>} : (tensor<1x2x3x!quant.uniform<i8:f32, 1.000000e+00>>) -> tensor<2x1x3x!quant.uniform<i8:f32, 1.000000e+00>>
+  // CHECK-DAG: tosa.const_shape {value = dense<[2, 1, 3]> : tensor<3xindex>} : () -> !tosa.shape<3>
+  // CHECK-DAG: "tosa.const"() <{value = dense<0> : tensor<1x2x3xi8>}> : () -> tensor<1x2x3x!quant.uniform<i8:f32, 1.000000e+00>>
+  // CHECK: tosa.reshape %0, %1 : (tensor<1x2x3x!quant.uniform<i8:f32, 1.000000e+00>>, !tosa.shape<3>) -> tensor<2x1x3x!quant.uniform<i8:f32, 1.000000e+00>>
   %perms = "tosa.const"() {value = dense<[1, 0, 2]> : tensor<3xi32>} : () -> tensor<3xi32>
   %0 = "tosa.const"() {value = dense<0> : tensor<1x2x3xi8>} : ()-> tensor<1x2x3x!quant.uniform<i8:f32, 1.000000e+00>>
   %1 = tosa.transpose %0, %perms : (tensor<1x2x3x!quant.uniform<i8:f32, 1.000000e+00>>, tensor<3xi32>) -> tensor<2x1x3x!quant.uniform<i8:f32, 1.000000e+00>>
@@ -624,18 +644,22 @@ func.func @transpose_canonicalize_strip_quant() -> (tensor<2x1x3x!quant.uniform<
 
 // CHECK-LABEL: @slice_fold
 func.func @slice_fold(%arg0: tensor<3x4xf32>) -> tensor<3x4xf32> {
+  %0 = tosa.const_shape {value = dense<[0, 0]> : tensor<2xindex>} : () -> !tosa.shape<2>
+  %1 = tosa.const_shape {value = dense<[3, 4]> : tensor<2xindex>} : () -> !tosa.shape<2>
   // CHECK: return %arg0
-  %0 = tosa.slice %arg0 { size = array<i64: 3, 4>, start = array<i64: 0, 0>}: (tensor<3x4xf32>) -> tensor<3x4xf32>
-  return %0 : tensor<3x4xf32>
+  %3 = tosa.slice %arg0, %0, %1 : (tensor<3x4xf32>, !tosa.shape<2>, !tosa.shape<2>) -> tensor<3x4xf32>
+  return %3 : tensor<3x4xf32>
 }
 
 // -----
 
 // CHECK-LABEL: @slice_nofold
 func.func @slice_nofold(%arg0: tensor<?x4xf32>) -> tensor<?x4xf32> {
+  %0 = tosa.const_shape {value = dense<[0, 0]> : tensor<2xindex>} : () -> !tosa.shape<2>
+  %1 = tosa.const_shape {value = dense<[3, 4]> : tensor<2xindex>} : () -> !tosa.shape<2>
   // CHECK: tosa.slice
-  %0 = tosa.slice %arg0 { size = array<i64: 3, 4>, start = array<i64: 0, 0>}: (tensor<?x4xf32>) -> tensor<?x4xf32>
-  return %0 : tensor<?x4xf32>
+  %3 = tosa.slice %arg0, %0, %1 : (tensor<?x4xf32>, !tosa.shape<2>, !tosa.shape<2>) -> tensor<?x4xf32>
+  return %3 : tensor<?x4xf32>
 }
 
 // -----
@@ -673,7 +697,8 @@ func.func @transpose_no_op(%arg0: tensor<3x4x5x6xf32>) -> tensor<3x4x5x6xf32> {
 
 // CHECK-LABEL: @transpose_is_reshape
 func.func @transpose_is_reshape(%arg0: tensor<1x4x5x1xf32>) -> tensor<1x4x1x5xf32> {
-  // CHECK: tosa.reshape %arg0 {new_shape = array<i64: 1, 4, 1, 5>} : (tensor<1x4x5x1xf32>) -> tensor<1x4x1x5xf32>
+  // CHECK: %[[CONST0:.+]] = tosa.const_shape {value = dense<[1, 4, 1, 5]> : tensor<4xindex>} : () -> !tosa.shape<4>
+  // CHECK: tosa.reshape %arg0, %[[CONST0]]
   %perms = "tosa.const"() <{value = dense<[3, 1, 0, 2]> : tensor<4xi32>}> : () -> tensor<4xi32>
   %0 = tosa.transpose %arg0, %perms : (tensor<1x4x5x1xf32>, tensor<4xi32>) -> tensor<1x4x1x5xf32>
   return %0 : tensor<1x4x1x5xf32>
@@ -686,7 +711,8 @@ func.func @transpose_is_reshape(%arg0: tensor<1x4x5x1xf32>) -> tensor<1x4x1x5xf3
 func.func @single_bit_reshape() -> tensor<1xi1> {
   // CHECK: "tosa.const"() <{value = dense<true> : tensor<1xi1>}
   %0 = arith.constant dense<true> : tensor<1x1xi1>
-  %1 = tosa.reshape %0 {new_shape = array<i64: 1>} : (tensor<1x1xi1>) -> tensor<1xi1>
+  %2 = "tosa.const_shape"() <{value = dense<1> : tensor<1xindex>}> : () -> !tosa.shape<1>
+  %1 = tosa.reshape %0, %2 : (tensor<1x1xi1>, !tosa.shape<1>) -> tensor<1xi1>
   return %1 : tensor<1xi1>
 }
 
@@ -715,9 +741,12 @@ func.func @fold_resize_bilinear(%arg0 : tensor<1x15x13x1xi8>) -> tensor<1x15x13x
 // CHECK: return %[[VAL_0]], %[[VAL_1]] : tensor<1x12x12x1xf32>, tensor<1x12x12x1xf32>
 func.func @canonicalize_concat_slice_final_axis(%arg0 : tensor<1x12x12x1xf32>, %arg1 : tensor<1x12x12x1xf32>) -> (tensor<1x12x12x1xf32>, tensor<1x12x12x1xf32>) {
   %0 = tosa.concat %arg0, %arg1 {axis = 3 : i32} : (tensor<1x12x12x1xf32>, tensor<1x12x12x1xf32>) -> tensor<1x12x12x2xf32>
-  %1 = tosa.slice %0 {size = array<i64: 1, 12, 12, 1>, start = array<i64: 0, 0, 0, 0>} : (tensor<1x12x12x2xf32>) -> tensor<1x12x12x1xf32>
-  %2 = tosa.slice %0 {size = array<i64: 1, 12, 12, 1>, start = array<i64: 0, 0, 0, 1>} : (tensor<1x12x12x2xf32>) -> tensor<1x12x12x1xf32>
-  return %1, %2 : tensor<1x12x12x1xf32>, tensor<1x12x12x1xf32>
+  %1 = tosa.const_shape {value = dense<[0, 0, 0, 0]> : tensor<4xindex>} : () -> !tosa.shape<4>
+  %2 = tosa.const_shape {value = dense<[0, 0, 0, 1]> : tensor<4xindex>} : () -> !tosa.shape<4>
+  %3 = tosa.const_shape {value = dense<[1, 12, 12, 1]> : tensor<4xindex>} : () -> !tosa.shape<4>
+  %4 = tosa.slice %0, %1, %3 : (tensor<1x12x12x2xf32>, !tosa.shape<4>, !tosa.shape<4>) -> tensor<1x12x12x1xf32>
+  %5 = tosa.slice %0, %2, %3 : (tensor<1x12x12x2xf32>, !tosa.shape<4>, !tosa.shape<4>) -> tensor<1x12x12x1xf32>
+  return %4, %5 : tensor<1x12x12x1xf32>, tensor<1x12x12x1xf32>
 }
 
 // -----
@@ -727,38 +756,56 @@ func.func @canonicalize_concat_slice_final_axis(%arg0 : tensor<1x12x12x1xf32>, %
 // CHECK: return %[[VAL_0]], %[[VAL_1]] : tensor<1x12x12xf32>, tensor<1x12x12xf32>
 func.func @canonicalize_concat_slice_middle_axis(%arg0 : tensor<1x12x12xf32>, %arg1 : tensor<1x12x12xf32>) -> (tensor<1x12x12xf32>, tensor<1x12x12xf32>) {
   %0 = tosa.concat %arg0, %arg1 {axis = 1 : i32} : (tensor<1x12x12xf32>, tensor<1x12x12xf32>) -> tensor<1x24x12xf32>
-  %1 = tosa.slice %0 {size = array<i64: 1, 12, 12>, start = array<i64: 0, 0, 0>} : (tensor<1x24x12xf32>) -> tensor<1x12x12xf32>
-  %2 = tosa.slice %0 {size = array<i64: 1, 12, 12>, start = array<i64: 0, 12, 0>} : (tensor<1x24x12xf32>) -> tensor<1x12x12xf32>
-  return %1, %2 : tensor<1x12x12xf32>, tensor<1x12x12xf32>
+  %1 = tosa.const_shape {value = dense<[0, 0, 0]> : tensor<3xindex>} : () -> !tosa.shape<3>
+  %2 = tosa.const_shape {value = dense<[0, 12, 0]> : tensor<3xindex>} : () -> !tosa.shape<3>
+  %3 = tosa.const_shape {value = dense<[1, 12, 12]> : tensor<3xindex>} : () -> !tosa.shape<3>
+  %4 = tosa.slice %0, %1, %3 : (tensor<1x24x12xf32>, !tosa.shape<3>, !tosa.shape<3>) -> tensor<1x12x12xf32>
+  %5 = tosa.slice %0, %2, %3 : (tensor<1x24x12xf32>, !tosa.shape<3>, !tosa.shape<3>) -> tensor<1x12x12xf32>
+  return %4, %5 : tensor<1x12x12xf32>, tensor<1x12x12xf32>
 }
 
 // -----
 
 // CHECK-LABEL: @canonicalize_cross_concat_inputs
 // CHECK-SAME: %[[VAL_0:.*]]: tensor<1x12x12xf32>, %[[VAL_1:.*]]: tensor<1x12x12xf32>
-// CHECK: %[[VAL_2:.*]] = tosa.concat %[[VAL_0]], %[[VAL_1]] {axis = 2 : i32} : (tensor<1x12x12xf32>, tensor<1x12x12xf32>) -> tensor<1x12x24xf32>
-// CHECK: %[[VAL_3:.*]] = tosa.slice %[[VAL_2]] {size = array<i64: 1, 12, 15>, start = array<i64: 0, 0, 0>} : (tensor<1x12x24xf32>) -> tensor<1x12x15xf32>
-// CHECK: %[[VAL_4:.*]] = tosa.slice %[[VAL_2]] {size = array<i64: 1, 12, 20>, start = array<i64: 0, 0, 4>} : (tensor<1x12x24xf32>) -> tensor<1x12x20xf32>
-// CHECK: return %[[VAL_3]], %[[VAL_4]] : tensor<1x12x15xf32>, tensor<1x12x20xf32>
+// CHECK-DAG: %[[VAL_2:.*]] = tosa.const_shape  {value = dense<[1, 12, 20]> : tensor<3xindex>}
+// CHECK-DAG: %[[VAL_3:.*]] = tosa.const_shape  {value = dense<[1, 12, 15]> : tensor<3xindex>}
+// CHECK-DAG: %[[VAL_4:.*]] = tosa.const_shape  {value = dense<[0, 0, 4]> : tensor<3xindex>}
+// CHECK-DAG: %[[VAL_5:.*]] = tosa.const_shape  {value = dense<0> : tensor<3xindex>}
+// CHECK: %[[VAL_6:.*]] = tosa.concat %[[VAL_0]], %[[VAL_1]] {axis = 2 : i32} : (tensor<1x12x12xf32>, tensor<1x12x12xf32>) -> tensor<1x12x24xf32>
+// CHECK: %[[VAL_7:.*]] = tosa.slice %[[VAL_6]], %[[VAL_5]], %[[VAL_3]]
+// CHECK: %[[VAL_8:.*]] = tosa.slice %[[VAL_6]], %[[VAL_4]], %[[VAL_2]]
+// CHECK: return %[[VAL_7]], %[[VAL_8]] : tensor<1x12x15xf32>, tensor<1x12x20xf32>
 func.func @canonicalize_cross_concat_inputs(%arg0 : tensor<1x12x12xf32>, %arg1 : tensor<1x12x12xf32>) -> (tensor<1x12x15xf32>, tensor<1x12x20xf32>) {
   %0 = tosa.concat %arg0, %arg1 {axis = 2 : i32} : (tensor<1x12x12xf32>, tensor<1x12x12xf32>) -> tensor<1x12x24xf32>
-  %1 = tosa.slice %0 {size = array<i64: 1, 12, 15>, start = array<i64: 0, 0, 0>} : (tensor<1x12x24xf32>) -> tensor<1x12x15xf32>
-  %2 = tosa.slice %0 {size = array<i64: 1, 12, 20>, start = array<i64: 0, 0, 4>} : (tensor<1x12x24xf32>) -> tensor<1x12x20xf32>
-  return %1, %2 : tensor<1x12x15xf32>, tensor<1x12x20xf32>
+  %1 = tosa.const_shape {value = dense<[0, 0, 0]> : tensor<3xindex>} : () -> !tosa.shape<3>
+  %2 = tosa.const_shape {value = dense<[0, 0, 4]> : tensor<3xindex>} : () -> !tosa.shape<3>
+  %3 = tosa.const_shape {value = dense<[1, 12, 15]> : tensor<3xindex>} : () -> !tosa.shape<3>
+  %4 = tosa.const_shape {value = dense<[1, 12, 20]> : tensor<3xindex>} : () -> !tosa.shape<3>
+  %5 = tosa.slice %0, %1, %3 : (tensor<1x12x24xf32>, !tosa.shape<3>, !tosa.shape<3>) -> tensor<1x12x15xf32>
+  %6 = tosa.slice %0, %2, %4 : (tensor<1x12x24xf32>, !tosa.shape<3>, !tosa.shape<3>) -> tensor<1x12x20xf32>
+  return %5, %6 : tensor<1x12x15xf32>, tensor<1x12x20xf32>
 }
 
 // -----
 
 // CHECK-LABEL: @canonicalize_concat_slice_on_non_concat_axis
 // CHECK-SAME: %[[VAL_0:.*]]: tensor<1x12x12xf32>, %[[VAL_1:.*]]: tensor<1x12x12xf32>
-// CHECK: %[[VAL_2:.*]] = tosa.slice %[[VAL_0]] {size = array<i64: 1, 6, 12>, start = array<i64: 0, 0, 0>} : (tensor<1x12x12xf32>) -> tensor<1x6x12xf32>
-// CHECK: %[[VAL_3:.*]] = tosa.slice %[[VAL_1]] {size = array<i64: 1, 3, 12>, start = array<i64: 1, 3, 0>} : (tensor<1x12x12xf32>) -> tensor<1x3x12xf32>
-// CHECK: return %[[VAL_2]], %[[VAL_3]] : tensor<1x6x12xf32>, tensor<1x3x12xf32>
+// CHECK-DAG: %[[VAL_2:.*]] = tosa.const_shape  {value = dense<[1, 3, 0]> : tensor<3xindex>}
+// CHECK-DAG: %[[VAL_3:.*]] = tosa.const_shape  {value = dense<[1, 3, 12]> : tensor<3xindex>}
+// CHECK-DAG: %[[VAL_4:.*]] = tosa.const_shape  {value = dense<0> : tensor<3xindex>}
+// CHECK-DAG: %[[VAL_5:.*]] = tosa.const_shape  {value = dense<[1, 6, 12]> : tensor<3xindex>}
+// CHECK: %[[VAL_6:.*]] = tosa.slice %[[VAL_0]], %[[VAL_4]], %[[VAL_5]]
+// CHECK: %[[VAL_7:.*]] = tosa.slice %[[VAL_1]], %[[VAL_2]], %[[VAL_3]]
+// CHECK: return %[[VAL_6]], %[[VAL_7]] : tensor<1x6x12xf32>, tensor<1x3x12xf32>
 func.func @canonicalize_concat_slice_on_non_concat_axis(%arg0 : tensor<1x12x12xf32>, %arg1 : tensor<1x12x12xf32>) -> (tensor<1x6x12xf32>, tensor<1x3x12xf32>) {
   %0 = tosa.concat %arg0, %arg1 {axis = 2 : i32} : (tensor<1x12x12xf32>, tensor<1x12x12xf32>) -> tensor<1x12x24xf32>
-  %1 = tosa.slice %0 {size = array<i64: 1, 6, 12>, start = array<i64: 0, 0, 0>} : (tensor<1x12x24xf32>) -> tensor<1x6x12xf32>
-  %2 = tosa.slice %0 {size = array<i64: 1, 3, 12>, start = array<i64: 1, 3, 12>} : (tensor<1x12x24xf32>) -> tensor<1x3x12xf32>
-  return %1, %2 : tensor<1x6x12xf32>, tensor<1x3x12xf32>
+  %1 = tosa.const_shape {value = dense<[0, 0, 0]> : tensor<3xindex>} : () -> !tosa.shape<3>
+  %2 = tosa.const_shape {value = dense<[1, 6, 12]> : tensor<3xindex>} : () -> !tosa.shape<3>
+  %3 = tosa.const_shape {value = dense<[1, 3, 12]> : tensor<3xindex>} : () -> !tosa.shape<3>
+  %4 = tosa.slice %0, %1, %2 : (tensor<1x12x24xf32>, !tosa.shape<3>, !tosa.shape<3>) -> tensor<1x6x12xf32>
+  %5 = tosa.slice %0, %3, %3 : (tensor<1x12x24xf32>, !tosa.shape<3>, !tosa.shape<3>) -> tensor<1x3x12xf32>
+  return %4, %5 : tensor<1x6x12xf32>, tensor<1x3x12xf32>
 }
 
 // -----
@@ -831,8 +878,11 @@ func.func nested @fold_tile_rank_zero() -> tensor<i32> {
 // check that segfault is fixed
 func.func @reshape_quant_nofold() -> tensor<1x1x1x1xi32> {
    %0 = "tosa.const"() {value = dense<127> : tensor<i8>} : () -> tensor<!quant.uniform<i8:f32, 3.0757404601899907E-5:-128>>
-   %1 = tosa.reshape %0 {new_shape = array<i64: 1, 1, 1, 1>} : (tensor<!quant.uniform<i8:f32, 3.0757404601899907E-5:-128>>) -> tensor<1x1x1x1x!quant.uniform<i8:f32, 3.0757404601899907E-5:-128>>
-   %2 = tosa.rescale %1 {double_round = true, input_zp = -128 : i32, multiplier = array<i32: 1073741824>, output_zp = 0 : i32, per_channel = false, scale32 = true, shift = array<i8: 30>} : (tensor<1x1x1x1x!quant.uniform<i8:f32, 3.0757404601899907E-5:-128>>) -> tensor<1x1x1x1xi32>
+   %cst0 = "tosa.const_shape"() {value = dense<[1, 1, 1, 1]> : tensor<4xindex>} : () -> !tosa.shape<4>
+   %1 = tosa.reshape %0, %cst0 : (tensor<!quant.uniform<i8:f32, 3.0757404601899907E-5:-128>>, !tosa.shape<4>) -> tensor<1x1x1x1x!quant.uniform<i8:f32, 3.0757404601899907E-5:-128>>
+   %multiplier = "tosa.const"() {value = dense<1073741824> : tensor<1xi32> } : () -> tensor<1xi32>
+   %shift = "tosa.const"() {value = dense<30> : tensor<1xi8> } : () -> tensor<1xi8>
+   %2 = tosa.rescale %1 {double_round = true, input_zp = -128 : i32, multiplier = array<i32: 1073741824>, output_zp = 0 : i32, per_channel = false, scale32 = true, shift = array<i8: 30>, input_unsigned = false, output_unsigned = false} : (tensor<1x1x1x1x!quant.uniform<i8:f32, 3.0757404601899907E-5:-128>>) -> tensor<1x1x1x1xi32>
    return %2 : tensor<1x1x1x1xi32>
 }
 
@@ -927,7 +977,8 @@ func.func @mul_quant_nofold() -> tensor<1x!quant.uniform<i8:f32, 3.0757404601899
    // CHECK: tosa.mul
    %0 = "tosa.const"() {value = dense<0> : tensor<1xi8>} : () -> tensor<1x!quant.uniform<i8:f32, 3.0757404601899907E-5:-128>>
    %1 = "tosa.const"() {value = dense<1> : tensor<1xi8>} : () -> tensor<1x!quant.uniform<i8:f32, 3.0757404601899907E-5:-128>>
-   %2 = tosa.mul %0, %1 { shift = 0 : i8} : (tensor<1x!quant.uniform<i8:f32, 3.0757404601899907E-5:-128>>, tensor<1x!quant.uniform<i8:f32, 3.0757404601899907E-5:-128>>) -> tensor<1x!quant.uniform<i8:f32, 3.0757404601899907E-5:-128>>
+   %shift = "tosa.const"() <{value = dense<0> : tensor<1xi8>}> : () -> tensor<1xi8>
+   %2 = tosa.mul %0, %1, %shift : (tensor<1x!quant.uniform<i8:f32, 3.0757404601899907E-5:-128>>, tensor<1x!quant.uniform<i8:f32, 3.0757404601899907E-5:-128>>, tensor<1xi8>) -> tensor<1x!quant.uniform<i8:f32, 3.0757404601899907E-5:-128>>
    return %2 : tensor<1x!quant.uniform<i8:f32, 3.0757404601899907E-5:-128>>
 }
 
diff --git a/mlir/test/Dialect/Tosa/constant-op-fold.mlir b/mlir/test/Dialect/Tosa/constant-op-fold.mlir
index 8198903b78ac0..e6fb741df9598 100644
--- a/mlir/test/Dialect/Tosa/constant-op-fold.mlir
+++ b/mlir/test/Dialect/Tosa/constant-op-fold.mlir
@@ -238,7 +238,8 @@ func.func @fold_div_splat_i32() -> tensor<i32> {
 func.func @fold_mul_zero_rhs_f32(%arg0: tensor<f32>) -> tensor<f32> {
   %zero = "tosa.const"() {value = dense<0.0> : tensor<f32>} : () -> tensor<f32>
   // CHECK: %[[ZERO:.+]] = "tosa.const"() <{value = dense<0.000000e+00>
-  %mul = tosa.mul %arg0, %zero {shift = 0 : i8} : (tensor<f32>, tensor<f32>) -> tensor<f32>
+  %shift = "tosa.const"() <{value = dense<0> : tensor<1xi8>}> : () -> tensor<1xi8>
+  %mul = tosa.mul %arg0, %zero, %shift : (tensor<f32>, tensor<f32>, tensor<1xi8>) -> tensor<f32>
   // CHECK: return %[[ZERO]]
   return %mul : tensor<f32>
 }
@@ -249,7 +250,8 @@ func.func @fold_mul_zero_rhs_f32(%arg0: tensor<f32>) -> tensor<f32> {
 func.func @fold_mul_zero_lhs_f32(%arg0: tensor<f32>) -> tensor<f32> {
   %zero = "tosa.const"() {value = dense<0.0> : tensor<f32>} : () -> tensor<f32>
   // CHECK: %[[ZERO:.+]] = "tosa.const"() <{value = dense<0.000000e+00>
-  %mul = tosa.mul %zero, %arg0 {shift = 0 : i8} : (tensor<f32>, tensor<f32>) -> tensor<f32>
+  %shift = "tosa.const"() <{value = dense<0> : tensor<1xi8>}> : () -> tensor<1xi8>
+  %mul = tosa.mul %zero, %arg0, %shift : (tensor<f32>, tensor<f32>, tensor<1xi8>) -> tensor<f32>
   // CHECK: return %[[ZERO]]
   return %mul : tensor<f32>
 }
@@ -259,8 +261,9 @@ func.func @fold_mul_zero_lhs_f32(%arg0: tensor<f32>) -> tensor<f32> {
 // CHECK-LABEL: @fold_mul_zero_rhs_i32
 func.func @fold_mul_zero_rhs_i32(%arg0: tensor<i32>) -> tensor<i32> {
   %zero = "tosa.const"() {value = dense<0> : tensor<i32>} : () -> tensor<i32>
+  %shift = "tosa.const"() <{value = dense<0> : tensor<1xi8>}> : () -> tensor<1xi8>
   // CHECK: %[[ZERO:.+]] = "tosa.const"() <{value = dense<0>
-  %mul = tosa.mul %arg0, %zero {shift = 0 : i8} : (tensor<i32>, tensor<i32>) -> tensor<i32>
+  %mul = tosa.mul %arg0, %zero, %shift : (tensor<i32>, tensor<i32>, tensor<1xi8>) -> tensor<i32>
   // CHECK: return %[[ZERO]]
   return %mul : tensor<i32>
 }
@@ -270,8 +273,9 @@ func.func @fold_mul_zero_rhs_i32(%arg0: tensor<i32>) -> tensor<i32> {
 // CHECK-LABEL: @fold_mul_zero_lhs_i32
 func.func @fold_mul_zero_lhs_i32(%arg0: tensor<i32>) -> tensor<i32> {
   %zero = "tosa.const"() {value = dense<0> : tensor<i32>} : () -> tensor<i32>
+  %shift = "tosa.const"() <{value = dense<0> : tensor<1xi8>}> : () -> tensor<1xi8>
   // CHECK: %[[ZERO:.+]] = "tosa.const"() <{value = dense<0>
-  %mul = tosa.mul %zero, %arg0 {shift = 0 : i8} : (tensor<i32>, tensor<i32>) -> tensor<i32>
+  %mul = tosa.mul %zero, %arg0, %shift : (tensor<i32>, tensor<i32>, tensor<1xi8>) -> tensor<i32>
   // CHECK: return %[[ZERO]]
   return %mul : tensor<i32>
 }
@@ -281,7 +285,8 @@ func.func @fold_mul_zero_lhs_i32(%arg0: tensor<i32>) -> tensor<i32> {
 // CHECK-LABEL: @fold_mul_one_rhs_f32
 func.func @fold_mul_one_rhs_f32(%arg0: tensor<f32>) -> tensor<f32> {
   %one = "tosa.const"() {value = dense<1.0> : tensor<f32>} : () -> tensor<f32>
-  %mul = tosa.mul %arg0, %one {shift = 0 : i8} : (tensor<f32>, tensor<f32>) -> tensor<f32>
+  %shift = "tosa.const"() <{value = dense<0> : tensor<1xi8>}> : () -> tensor<1xi8>
+  %mul = tosa.mul %arg0, %one, %shift : (tensor<f32>, tensor<f32>, tensor<1xi8>) -> tensor<f32>
   // CHECK: return %arg0
   return %mul : tensor<f32>
 }
@@ -291,7 +296,8 @@ func.func @fold_mul_one_rhs_f32(%arg0: tensor<f32>) -> tensor<f32> {
 // CHECK-LABEL: @fold_mul_one_lhs_f32
 func.func @fold_mul_one_lhs_f32(%arg0: tensor<f32>) -> tensor<f32> {
   %one = "tosa.const"() {value = dense<1.0> : tensor<f32>} : () -> tensor<f32>
-  %mul = tosa.mul %one, %arg0 {shift = 0 : i8} : (tensor<f32>, tensor<f32>) -> tensor<f32>
+  %shift = "tosa.const"() <{value = dense<0> : tensor<1xi8>}> : () -> tensor<1xi8>
+  %mul = tosa.mul %one, %arg0, %shift : (tensor<f32>, tensor<f32>, tensor<1xi8>) -> tensor<f32>
   // CHECK: return %arg0
   return %mul : tensor<f32>
 }
@@ -301,7 +307,8 @@ func.func @fold_mul_one_lhs_f32(%arg0: tensor<f32>) -> tensor<f32> {
 // CHECK-LABEL: @fold_mul_one_rhs_i32
 func.func @fold_mul_one_rhs_i32(%arg0: tensor<i32>) -> tensor<i32> {
   %one = "tosa.const"() {value = dense<64> : tensor<i32>} : () -> tensor<i32>
-  %mul = tosa.mul %arg0, %one {shift = 6 : i8} : (tensor<i32>, tensor<i32>) -> tensor<i32>
+  %shift = "tosa.const"() {value = dense<6> : tensor<1xi8>} : () -> tensor<1xi8>
+  %mul = tosa.mul %arg0, %one, %shift : (tensor<i32>, tensor<i32>, tensor<1xi8>) -> tensor<i32>
   // CHECK: return %arg0
   return %mul : tensor<i32>
 }
@@ -311,7 +318,8 @@ func.func @fold_mul_one_rhs_i32(%arg0: tensor<i32>) -> tensor<i32> {
 // CHECK-LABEL: @fold_mul_one_lhs_i32
 func.func @fold_mul_one_lhs_i32(%arg0: tensor<i32>) -> tensor<i32> {
   %one = "tosa.const"() {value = dense<64> : tensor<i32>} : () -> tensor<i32>
-  %mul = tosa.mul %one, %arg0 {shift = 6 : i8} : (tensor<i32>, tensor<i32>) -> tensor<i32>
+  %shift = "tosa.const"() {value = dense<6> : tensor<1xi8>} : () -> tensor<1xi8>
+  %mul = tosa.mul %one, %arg0, %shift : (tensor<i32>, tensor<i32>, tensor<1xi8>) -> tensor<i32>
   // CHECK: return %arg0
   return %mul : tensor<i32>
 }
@@ -322,7 +330,8 @@ func.func @fold_mul_one_lhs_i32(%arg0: tensor<i32>) -> tensor<i32> {
 func.func @fold_mul_splat_i8() -> tensor<10xi32> {
   %one = "tosa.const"() {value = dense<17> : tensor<10xi8>} : () -> tensor<10xi8>
   %two = "tosa.const"() {value = dense<32> : tensor<10xi8>} : () -> tensor<10xi8>
-  %mul = tosa.mul %one, %two {shift = 3 : i8} : (tensor<10xi8>, tensor<10xi8>) -> tensor<10xi32>
+  %shift = "tosa.const"() {value = dense<3> : tensor<1xi8>} : () -> tensor<1xi8>
+  %mul = tosa.mul %one, %two, %shift : (tensor<10xi8>, tensor<10xi8>, tensor<1xi8>) -> tensor<10xi32>
   // CHECK: %[[THREE:.+]] = "tosa.const"() <{value = dense<68> : tensor<10xi32>}
   // CHECK: return %[[THREE]]
   return %mul : tensor<10xi32>
@@ -334,7 +343,8 @@ func.func @fold_mul_splat_i8() -> tensor<10xi32> {
 func.func @fold_mul_splat_f32() -> tensor<10xf32> {
   %one = "tosa.const"() {value = dense<3.0> : tensor<10xf32>} : () -> tensor<10xf32>
   %two = "tosa.const"() {value = dense<2.0> : tensor<10xf32>} : () -> tensor<10xf32>
-  %mul = tosa.mul %one, %two {shift = 0 : i8} : (tensor<10xf32>, tensor<10xf32>) -> tensor<10xf32>
+  %shift = "tosa.const"() <{value = dense<0> : tensor<1xi8>}> : () -> tensor<1xi8>
+  %mul = tosa.mul %one, %two, %shift : (tensor<10xf32>, tensor<10xf32>, tensor<1xi8>) -> tensor<10xf32>
   // CHECK: %[[THREE:.+]] = "tosa.const"() <{value = dense<6.000000e+00> : tensor<10xf32>}
   // CHECK: return %[[THREE]]
   return %mul : tensor<10xf32>
@@ -495,7 +505,8 @@ func.func @fold_eq_i32(%arg0 : tensor<10xi32>) -> (tensor<10xi1>) {
 func.func @reshape_splat() -> tensor<6x5x4xi32> {
   // CHECK: %[[SPLAT:.+]] = "tosa.const"() <{value = dense<42> : tensor<6x5x4xi32>}
   %splat = "tosa.const"() {value = dense<42> : tensor<4x5x6xi32>} : () -> tensor<4x5x6xi32>
-  %reshape = tosa.reshape %splat { new_shape = array<i64: 6, 5, 4> } : (tensor<4x5x6xi32>) -> tensor<6x5x4xi32>
+  %const = tosa.const_shape {value = dense<[6, 5, 4]> : tensor<3xindex>} : () -> !tosa.shape<3>
+  %reshape = tosa.reshape %splat, %const : (tensor<4x5x6xi32>, !tosa.shape<3>) -> tensor<6x5x4xi32>
   // CHECK: return %[[SPLAT]]
   return %reshape : tensor<6x5x4xi32>
 }
@@ -506,7 +517,10 @@ func.func @reshape_splat() -> tensor<6x5x4xi32> {
 func.func @slice_splat() -> tensor<1x1x1xi32> {
   // CHECK: %[[SLICE:.+]] = "tosa.const"() <{value = dense<42> : tensor<1x1x1xi32>}
   %splat = "tosa.const"() {value = dense<42> : tensor<4x5x6xi32>} : () -> tensor<4x5x6xi32>
-  %slice = tosa.slice %splat { size = array<i64: 1, 1, 1>, start = array<i64: 1, 2, 3> } : (tensor<4x5x6xi32>) -> tensor<1x1x1xi32>
+  %start = tosa.const_shape {value = dense<[1, 2, 3]> : tensor<3xindex>} : () -> !tosa.shape<3>
+  %size = tosa.const_shape {value = dense<[1, 1, 1]> : tensor<3xindex>} : () -> !tosa.shape<3>
+  %slice= tosa.slice %splat, %start, %size : (tensor<4x5x6xi32>, !tosa.shape<3>, !tosa.shape<3>) -> tensor<1x1x1xi32>
+
   // CHECK: return %[[SLICE]]
   return %slice : tensor<1x1x1xi32>
 }
@@ -517,7 +531,9 @@ func.func @slice_splat() -> tensor<1x1x1xi32> {
 func.func @slice_singleton() -> tensor<1x1xi32> {
   %splat = "tosa.const"() {value = dense<[[0, 1, 2], [3, 4, 5], [6, 7 ,8]]> : tensor<3x3xi32>} : () -> tensor<3x3xi32>
   // CHECK: %[[SLICE:.+]] = "tosa.const"() <{value = dense<4> : tensor<1x1xi32>}
-  %slice = tosa.slice %splat { size = array<i64: 1, 1>, start = array<i64: 1, 1> } : (tensor<3x3xi32>) -> tensor<1x1xi32>
+  %start = tosa.const_shape {value = dense<[1, 1]> : tensor<2xindex>} : () -> !tosa.shape<2>
+  %size = tosa.const_shape {value = dense<[1, 1]> : tensor<2xindex>} : () -> !tosa.shape<2>
+  %slice= tosa.slice %splat, %start, %size : (tensor<3x3xi32>, !tosa.shape<2>, !tosa.shape<2>) -> tensor<1x1xi32>
   // CHECK: return %[[SLICE]]
   return %slice : tensor<1x1xi32>
 }
diff --git a/mlir/test/Dialect/Tosa/inlining.mlir b/mlir/test/Dialect/Tosa/inlining.mlir
index e892fdaa27750..2a3065e80d0ea 100644
--- a/mlir/test/Dialect/Tosa/inlining.mlir
+++ b/mlir/test/Dialect/Tosa/inlining.mlir
@@ -47,7 +47,8 @@ func.func @inlined_while_fn(%arg0: tensor<i32>, %arg1: tensor<i32>, %arg2: tenso
 }
 func.func private @while_body_50(%arg0: tensor<i32>, %arg1: tensor<i32>, %arg2: tensor<i32>, %arg3: tensor<10xi32>) -> (tensor<i32>, tensor<i32>, tensor<i32>, tensor<10xi32>) {
   %1 = "tosa.add"(%arg0, %arg1) : (tensor<i32>, tensor<i32>) -> tensor<i32>
-  %3 = "tosa.reshape"(%1) {new_shape = array<i64: 1>} : (tensor<i32>) -> tensor<1xi32>
+  %4 = "tosa.const_shape"() {value = dense<1> : tensor<1xindex>} : () -> !tosa.shape<1>
+  %3 = "tosa.reshape"(%1, %4) : (tensor<i32>, !tosa.shape<1>) -> tensor<1xi32>
   %2 = "tosa.add"(%arg3, %3) : (tensor<10xi32>, tensor<1xi32>) -> tensor<10xi32>
   return %1, %arg1, %arg2, %2: tensor<i32>, tensor<i32>, tensor<i32>, tensor<10xi32>
 }
diff --git a/mlir/test/Dialect/Tosa/invalid.mlir b/mlir/test/Dialect/Tosa/invalid.mlir
index 4808867b28bb9..913191be86f85 100644
--- a/mlir/test/Dialect/Tosa/invalid.mlir
+++ b/mlir/test/Dialect/Tosa/invalid.mlir
@@ -33,45 +33,41 @@ func.func @test_conv2d(%arg0: tensor<1x29x29x4xf32>, %arg1: tensor<16x3x3x4xi8>,
 // -----
 
 func.func @test_conv2d(%arg0: tensor<*xi8>, %arg1: tensor<16x3x3x4xi8>, %arg2: tensor<16xi8>) -> tensor<1x27x27x16xi8> {
+  %zp = "tosa.const"() {value = dense<0> : tensor<1xi8>} : () -> tensor<1xi8>
   // expected-error@+1 {{expect a ranked tensor for input, got <block argument> of type 'tensor<*xi8>' at index: 0}}
-  %0 = tosa.conv2d %arg0, %arg1, %arg2 {acc_type = i32, dilation = array<i64: 1, 1>, pad = array<i64: 0, 0, 0, 0>, stride = array<i64: 1, 1>}
-           : (tensor<*xi8>, tensor<16x3x3x4xi8>, tensor<16xi8>) -> tensor<1x27x27x16xi8>
+  %0 = tosa.conv2d %arg0, %arg1, %arg2, %zp, %zp {acc_type = i32, dilation = array<i64: 1, 1>, pad = array<i64: 0, 0, 0, 0>, stride = array<i64: 1, 1>}
+           : (tensor<*xi8>, tensor<16x3x3x4xi8>, tensor<16xi8>, tensor<1xi8>, tensor<1xi8>) -> tensor<1x27x27x16xi8>
   return %0 : tensor<1x27x27x16xi8>
 }
 
 // -----
 
 func.func @test_conv2d(%arg0: tensor<1x29x29x4xi8>, %arg1: tensor<*xi8>, %arg2: tensor<16xi8>) -> tensor<1x27x27x16xi8> {
+  %zp = "tosa.const"() {value = dense<0> : tensor<1xi8>} : () -> tensor<1xi8>
   // expected-error@+1 {{'tosa.conv2d' op operand #1 must be 4D tensor of 4-bit signless integer or 8-bit signless integer or Quint8 type or Qint4 type or Qint8 type or Qint16 type or Qint32 type or floating-point values, but got 'tensor<*xi8>'}}
-  %0 = tosa.conv2d %arg0, %arg1, %arg2 {acc_type = i32, dilation = array<i64: 1, 1>, pad = array<i64: 0, 0, 0, 0>, stride = array<i64: 1, 1>}
-           : (tensor<1x29x29x4xi8>, tensor<*xi8>, tensor<16xi8>) -> tensor<1x27x27x16xi8>
-  return %0 : tensor<1x27x27x16xi8>
-}
-
-// -----
-
-func.func @test_conv2d(%arg0: tensor<1x29x29x4xi8>, %arg1: tensor<16x3x3x4xi8>, %arg2: tensor<16xi8>) -> tensor<1x27x27x16xi8> {
-  // expected-error@+1 {{'tosa.conv2d' op quantizationattr is required for quantized type, and not allowed for float type}}
-  %0 = tosa.conv2d %arg0, %arg1, %arg2 {acc_type = f16, dilation = array<i64: 1, 1>, pad = array<i64: 0, 0, 0, 0>, stride = array<i64: 1, 1>}
-           : (tensor<1x29x29x4xi8>, tensor<16x3x3x4xi8>, tensor<16xi8>) -> tensor<1x27x27x16xi8>
+  %0 = tosa.conv2d %arg0, %arg1, %arg2, %zp, %zp {acc_type = i32, dilation = array<i64: 1, 1>, pad = array<i64: 0, 0, 0, 0>, stride = array<i64: 1, 1>}
+           : (tensor<1x29x29x4xi8>, tensor<*xi8>, tensor<16xi8>, tensor<1xi8>, tensor<1xi8>) -> tensor<1x27x27x16xi8>
   return %0 : tensor<1x27x27x16xi8>
 }
 
 // -----
 
 func.func @test_conv2d_acc_type(%arg0: tensor<1x29x29x4xi8>, %arg1: tensor<16x3x3x4xi8>, %arg2: tensor<16xi8>) -> tensor<1x27x27x16xi8> {
+  %zp = "tosa.const"() {value = dense<0> : tensor<1xi8>} : () -> tensor<1xi8>
   // expected-error@+1 {{'tosa.conv2d' op accumulator type for i8 tensor is not i32}}
-  %0 = tosa.conv2d %arg0, %arg1, %arg2 {acc_type = f16, dilation = array<i64: 1, 1>, pad = array<i64: 0, 0, 0, 0>, stride = array<i64: 1, 1>, quantization_info = #tosa.conv_quant<input_zp = 0, weight_zp = 0>}
-           : (tensor<1x29x29x4xi8>, tensor<16x3x3x4xi8>, tensor<16xi8>) -> tensor<1x27x27x16xi8>
+  %0 = tosa.conv2d %arg0, %arg1, %arg2, %zp, %zp {acc_type = f16, dilation = array<i64: 1, 1>, pad = array<i64: 0, 0, 0, 0>, stride = array<i64: 1, 1>}
+           : (tensor<1x29x29x4xi8>, tensor<16x3x3x4xi8>, tensor<16xi8>, tensor<1xi8>, tensor<1xi8>) -> tensor<1x27x27x16xi8>
   return %0 : tensor<1x27x27x16xi8>
 }
 
 // -----
 
 func.func @test_conv2d_acc_type(%arg0: tensor<1x29x29x4xi16>, %arg1: tensor<16x3x3x4xi8>, %arg2: tensor<16xi16>) -> tensor<1x27x27x16xi16> {
+  %input_zp = "tosa.const"() {value = dense<0> : tensor<1xi16>} : () -> tensor<1xi16>
+  %weight_zp = "tosa.const"() {value = dense<0> : tensor<1xi8>} : () -> tensor<1xi8>
   // expected-error@+1 {{'tosa.conv2d' op accumulator type for i16 tensor is not i48}}
-  %0 = tosa.conv2d %arg0, %arg1, %arg2 {acc_type = f16, dilation = array<i64: 1, 1>, pad = array<i64: 0, 0, 0, 0>, stride = array<i64: 1, 1>, quantization_info = #tosa.conv_quant<input_zp = 0, weight_zp = 0>}
-           : (tensor<1x29x29x4xi16>, tensor<16x3x3x4xi8>, tensor<16xi16>) -> tensor<1x27x27x16xi16>
+  %0 = tosa.conv2d %arg0, %arg1, %arg2, %input_zp, %weight_zp {acc_type = f16, dilation = array<i64: 1, 1>, pad = array<i64: 0, 0, 0, 0>, stride = array<i64: 1, 1>}
+           : (tensor<1x29x29x4xi16>, tensor<16x3x3x4xi8>, tensor<16xi16>, tensor<1xi16>, tensor<1xi8>) -> tensor<1x27x27x16xi16>
   return %0 : tensor<1x27x27x16xi16>
 }
 
@@ -123,28 +119,49 @@ func.func @test_conv2d_acc_type(%arg0: tensor<1x29x29x4xf32>, %arg1: tensor<16x3
 // -----
 
 func.func @test_conv3d_acc_type(%arg0: tensor<1x4x8x21x17xi8>, %arg1: tensor<34x1x1x1x17xi8>, %arg2: tensor<34xi8>) -> tensor<1x4x8x21x34xi8> {
+  %zp = "tosa.const"() {value = dense<0> : tensor<1xi8>} : () -> tensor<1xi8>
   // expected-error@+1 {{'tosa.conv3d' op accumulator type for i8 tensor is not i32}}
-  %0 = tosa.conv3d %arg0, %arg1, %arg2 {acc_type = f16, dilation = array<i64: 1, 1, 1>, pad = array<i64: 0, 0, 0, 0, 0, 0>, stride = array<i64: 1, 1, 1>, quantization_info = #tosa.conv_quant<input_zp = 0, weight_zp = 0>}
-           : (tensor<1x4x8x21x17xi8>, tensor<34x1x1x1x17xi8>, tensor<34xi8>) -> tensor<1x4x8x21x34xi8>
+  %0 = tosa.conv3d %arg0, %arg1, %arg2, %zp, %zp {acc_type = f16, dilation = array<i64: 1, 1, 1>, pad = array<i64: 0, 0, 0, 0, 0, 0>, stride = array<i64: 1, 1, 1>}
+           : (tensor<1x4x8x21x17xi8>, tensor<34x1x1x1x17xi8>, tensor<34xi8>, tensor<1xi8>, tensor<1xi8>) -> tensor<1x4x8x21x34xi8>
   return %0 : tensor<1x4x8x21x34xi8>
 }
 
 // -----
 
 func.func @test_depthwise_conv2d_acc_type(%arg0: tensor<1x4x4x4xi8>, %arg1: tensor<1x1x4x2xi8>, %arg2: tensor<8xi8>) -> tensor<1x4x4x8xi8> {
+  %zp = "tosa.const"() {value = dense<0> : tensor<1xi8>} : () -> tensor<1xi8>
   // expected-error@+1 {{'tosa.depthwise_conv2d' op accumulator type for i8 tensor is not i32}}
-  %0 = tosa.depthwise_conv2d %arg0, %arg1, %arg2 {acc_type = f16, dilation = array<i64: 1, 1>, pad = array<i64: 0, 0, 0, 0>, stride = array<i64: 1, 1>, quantization_info = #tosa.conv_quant<input_zp = 0, weight_zp = 0>} : (tensor<1x4x4x4xi8>, tensor<1x1x4x2xi8>, tensor<8xi8>) -> tensor<1x4x4x8xi8>
+  %0 = tosa.depthwise_conv2d %arg0, %arg1, %arg2, %zp, %zp {acc_type = f16, dilation = array<i64: 1, 1>, pad = array<i64: 0, 0, 0, 0>, stride = array<i64: 1, 1>} : (tensor<1x4x4x4xi8>, tensor<1x1x4x2xi8>, tensor<8xi8>, tensor<1xi8>, tensor<1xi8>) -> tensor<1x4x4x8xi8>
   return %0 : tensor<1x4x4x8xi8>
 }
 
 // -----
 
 func.func @test_transpose_conv2d(%arg0: tensor<1x32x32x8xi8>, %arg1: tensor<16x1x1x8xi8>, %arg2: tensor<16xi8>) -> tensor<1x32x32x16xi8> {
+  %zp = "tosa.const"() {value = dense<0> : tensor<1xi8>} : () -> tensor<1xi8>
   // expected-error@+1 {{'tosa.transpose_conv2d' op accumulator type for i8 tensor is not i32}}
-  %0 = tosa.transpose_conv2d %arg0, %arg1, %arg2 {acc_type = f16, out_pad = array<i64: 0, 0, 0, 0>, out_shape = array<i64: 1, 32, 32, 16>, stride = array<i64: 1, 1>, quantization_info = #tosa.conv_quant<input_zp = 0, weight_zp = 0>} : (tensor<1x32x32x8xi8>, tensor<16x1x1x8xi8>, tensor<16xi8>) -> tensor<1x32x32x16xi8>
+  %0 = tosa.transpose_conv2d %arg0, %arg1, %arg2, %zp, %zp {acc_type = f16, out_pad = array<i64: 0, 0, 0, 0>, out_shape = array<i64: 1, 32, 32, 16>, stride = array<i64: 1, 1>} : (tensor<1x32x32x8xi8>, tensor<16x1x1x8xi8>, tensor<16xi8>, tensor<1xi8>, tensor<1xi8>) -> tensor<1x32x32x16xi8>
   return %0 : tensor<1x32x32x16xi8>
 }
 
+// -----
+// CHECK-LABEL: conv2d_quant_any_acc
+func.func @test_conv2d_quant_any_acc(%arg0: tensor<1x4x4x4x!quant.any<i8<-8:7>>>, %arg1: tensor<8x1x1x4x!quant.any<i8<-8:7>>>, %arg2: tensor<8x!quant.any<i8<-8:7>>>) -> tensor<1x4x4x8x!quant.any<i8<-8:7>>> {
+  %zp = "tosa.const" () { value = dense<0> : tensor<1xi8> } : () -> tensor<1xi8>
+  // expected-error@+1 {{'tosa.conv2d' op accumulator type for i8 tensor is not i32}}
+  %0 = tosa.conv2d %arg0, %arg1, %arg2, %zp, %zp {acc_type = f32, dilation = array<i64: 1, 1>, pad = array<i64: 0, 0, 0, 0>, stride = array<i64: 1, 1>, local_bound = true} : (tensor<1x4x4x4x!quant.any<i8<-8:7>>>, tensor<8x1x1x4x!quant.any<i8<-8:7>>>, tensor<8x!quant.any<i8<-8:7>>>, tensor<1xi8>, tensor<1xi8>) -> tensor<1x4x4x8x!quant.any<i8<-8:7>>>
+  return %0 : tensor<1x4x4x8x!quant.any<i8<-8:7>>>
+}
+
+// -----
+// CHECK-LABEL: conv2d_quant_any_result
+func.func @test_conv2d_quant_any_result(%arg0: tensor<1x4x4x4x!quant.any<i8<-8:7>>>, %arg1: tensor<8x1x1x4x!quant.any<i8<-8:7>>>, %arg2: tensor<8x!quant.any<i8<-8:7>>>) -> tensor<1x4x4x8x!quant.any<i8<-8:7>>> {
+  %zp = "tosa.const" () { value = dense<0> : tensor<1xi8> } : () -> tensor<1xi8>
+  // expected-error@+1 {{'tosa.conv2d' op input/output element types are incompatible}}
+  %0 = tosa.conv2d %arg0, %arg1, %arg2, %zp, %zp {acc_type = i32, dilation = array<i64: 1, 1>, pad = array<i64: 0, 0, 0, 0>, stride = array<i64: 1, 1>, local_bound = true} : (tensor<1x4x4x4x!quant.any<i8<-8:7>>>, tensor<8x1x1x4x!quant.any<i8<-8:7>>>, tensor<8x!quant.any<i8<-8:7>>>, tensor<1xi8>, tensor<1xi8>) -> tensor<1x4x4x8x!quant.any<i8<-8:7>>>
+  return %0 : tensor<1x4x4x8x!quant.any<i8<-8:7>>>
+}
+
 // -----
 
 func.func @test_concat(%arg0 : tensor<2x1xf32>, %arg1 : tensor<2x2xf32>) -> tensor<?x?xf32> {
@@ -183,7 +200,7 @@ func.func @test_pad_non_const(%arg0: tensor<13x21x3xi8>, %arg1: tensor<i8>) -> t
 // -----
 
 func.func @test_pad_io_rank_mismatch(%arg0: tensor<13x21xf32>) {
-  %padding = tosa.const_shape {value = dense<0> : tensor<4xindex>} : () -> !tosa.shape<4>  
+  %padding = tosa.const_shape {value = dense<0> : tensor<4xindex>} : () -> !tosa.shape<4>
   // expected-error@+1 {{'tosa.pad' op expect same input and output tensor rank.}}
   %1 = tosa.pad %arg0, %padding : (tensor<13x21xf32>, !tosa.shape<4>) -> tensor<13x21x3xf32>
   return
@@ -211,7 +228,7 @@ func.func @test_pad_invalid_padConst_rank(%arg0: tensor<13x21xf32>, %arg1: tenso
 // -----
 
 func.func @test_pad_padding_shape_mismatch(%arg0: tensor<13x21x3xf32>) -> tensor<13x21x3xf32> {
-  %0 = tosa.const_shape {value = dense<1> : tensor<4xindex>} : () -> !tosa.shape<4>  
+  %0 = tosa.const_shape {value = dense<1> : tensor<4xindex>} : () -> !tosa.shape<4>
   // expected-error@+1 {{'tosa.pad' op expected padding tensor dim 0 to have size 6 (2*rank(shape1)) but got size 4}}
   %1 = tosa.pad %arg0, %0 : (tensor<13x21x3xf32>, !tosa.shape<4>) -> tensor<13x21x3xf32>
   return %1 : tensor<13x21x3xf32>
@@ -317,7 +334,8 @@ func.func @test_transpose_element_type_mismatch(%arg0: tensor<2x3xi32>) -> tenso
 
 func.func @test_fully_connected_non_const(%arg0: tensor<13x21x3xf32>, %arg1: tensor<2x3xf32>) -> tensor<273x2xf32> {
   %0 = "tosa.const"() {value = dense<0.000000e+00> : tensor<2xf32>} : () -> tensor<2xf32>
-  %1 = tosa.reshape %arg0 {new_shape = array<i64: 273, 3>} : (tensor<13x21x3xf32>) -> tensor<273x3xf32>
+  %3 = tosa.const_shape {value = dense<[273, 3]> : tensor<2xindex>} : () -> !tosa.shape<2>
+  %1 = tosa.reshape %arg0, %3 : (tensor<13x21x3xf32>, !tosa.shape<2>) -> tensor<273x3xf32>
   // expected-error@+1 {{'tosa.fully_connected' op weight of fully_connected is not constant}}
   %2 = tosa.fully_connected %1, %arg1, %0 : (tensor<273x3xf32>, tensor<2x3xf32>, tensor<2xf32>) -> tensor<273x2xf32>
   return %2 : tensor<273x2xf32>
@@ -327,7 +345,8 @@ func.func @test_fully_connected_non_const(%arg0: tensor<13x21x3xf32>, %arg1: ten
 
 func.func @test_fully_connected_non_const(%arg0: tensor<13x21x3xf32>, %arg1: tensor<2xf32>) -> tensor<273x2xf32> {
   %0 = "tosa.const"() {value = dense<[[-0.613216758, -0.63714242, -0.73500061], [0.180762768, 0.773053169, -0.933686495]]> : tensor<2x3xf32>} : () -> tensor<2x3xf32>
-  %1 = tosa.reshape %arg0 {new_shape = array<i64: 273, 3>} : (tensor<13x21x3xf32>) -> tensor<273x3xf32>
+  %3 = tosa.const_shape {value = dense<[273, 3]> : tensor<2xindex>} : () -> !tosa.shape<2>
+  %1 = tosa.reshape %arg0, %3 : (tensor<13x21x3xf32>, !tosa.shape<2>) -> tensor<273x3xf32>
   // expected-error@+1 {{'tosa.fully_connected' op bias of fully_connected is not constant}}
   %2 = tosa.fully_connected %1, %0, %arg1 : (tensor<273x3xf32>, tensor<2x3xf32>, tensor<2xf32>) -> tensor<273x2xf32>
   return %2 : tensor<273x2xf32>
@@ -427,81 +446,91 @@ func.func @test_reduce_min_invalid_output_rank(%arg0 : tensor<i32>) -> () {
 // -----
 
 func.func @test_reshape_type_mismatch(%arg0 : tensor<13x21x3xf32>) -> () {
+  %1 = tosa.const_shape {value = dense<[13, 21, 3, 1]> : tensor<4xindex>} : () -> !tosa.shape<4>
   // expected-error@+2 {{failed to infer returned types}}
   // expected-error@+1 {{'tosa.reshape' op inferred type(s) 'tensor<13x21x3x1xf32>' are incompatible with return type(s) of operation 'tensor<13x21x3x1xi32>'}}
-  %0 = tosa.reshape %arg0 {new_shape = array<i64: 13, 21, 3, 1>} : (tensor<13x21x3xf32>) -> tensor<13x21x3x1xi32>
+  %0 = tosa.reshape %arg0, %1 : (tensor<13x21x3xf32>, !tosa.shape<4>) -> tensor<13x21x3x1xi32>
   return
 }
 
 // -----
 
 func.func @test_reshape_static_zero_dim_input(%arg0 : tensor<13x0x3xf32>) -> () {
+  %s = tosa.const_shape {value = dense<[13, 21, 3]> : tensor<3xindex>} : () -> !tosa.shape<3>
   // expected-error@+1 {{'tosa.reshape' op operand #0 must be tosa-conformant tensor of number values, but got 'tensor<13x0x3xf32>'}}
-  %0 = "tosa.reshape"(%arg0) {new_shape = array<i64: 13, 21, 3>} : (tensor<13x0x3xf32>) -> tensor<13x0x3xf32>
+  %0 = "tosa.reshape"(%arg0, %s) : (tensor<13x0x3xf32>, !tosa.shape<3>) -> tensor<13x0x3xf32>
   return
 }
 
 // -----
 
 func.func @test_reshape_zero_dim_input(%arg0 : tensor<?x0x3xf32>) -> () {
+  %s = tosa.const_shape {value = dense<[13, 21, 3]> : tensor<3xindex>} : () -> !tosa.shape<3>
   // expected-error@+1 {{'tosa.reshape' op operand #0 must be tosa-conformant tensor of number values, but got 'tensor<?x0x3xf32>'}}
-  %0 = "tosa.reshape"(%arg0) {new_shape = array<i64: 13, 21, 3>} : (tensor<?x0x3xf32>) -> tensor<13x0x3xf32>
+  %0 = "tosa.reshape"(%arg0, %s) : (tensor<?x0x3xf32>, !tosa.shape<3>) -> tensor<13x0x3xf32>
   return
 }
 
 // -----
 
 func.func @test_reshape_rank_mismatch(%arg0 : tensor<?xf32>) -> () {
+  %s = tosa.const_shape {value = dense<[2, 4]> : tensor<2xindex>} : () -> !tosa.shape<2>
   // expected-error@+1 {{'tosa.reshape' op new shape does not match result rank}}
-  %0 = "tosa.reshape"(%arg0) {new_shape = array<i64: 2, 4>} : (tensor<?xf32>) -> tensor<?xf32>
+  %0 = "tosa.reshape"(%arg0, %s) : (tensor<?xf32>, !tosa.shape<2>) -> tensor<?xf32>
   return
 }
 
 // -----
 
 func.func @test_reshape_inconsistent_result_type(%arg0 : tensor<?xf32>) -> () {
+  %s = tosa.const_shape {value = dense<[2, 4, -1]> : tensor<3xindex>} : () -> !tosa.shape<3>
   // expected-error@+1 {{'tosa.reshape' op new shape is inconsistent with result shape}}
-  %0 = "tosa.reshape"(%arg0) {new_shape = array<i64: 2, 4, -1>} : (tensor<?xf32>) -> tensor<?x3x5xf32>
+  %0 = "tosa.reshape"(%arg0, %s) : (tensor<?xf32>, !tosa.shape<3>) -> tensor<?x3x5xf32>
   return
 }
 
 // -----
 
 func.func @test_reshape_invalid_size(%arg0 : tensor<2x4xf32>) -> () {
+  %s = tosa.const_shape {value = dense<[3, 5]> : tensor<2xindex>} : () -> !tosa.shape<2>
   // expected-error@+1 {{'tosa.reshape' op cannot reshape 8 elements into 15}}
-  %0 = "tosa.reshape"(%arg0) {new_shape = array<i64: 3, 5>} : (tensor<2x4xf32>) -> tensor<3x5xf32>
+  %0 = "tosa.reshape"(%arg0, %s) : (tensor<2x4xf32>, !tosa.shape<2>) -> tensor<3x5xf32>
   return
 }
 
 // -----
 
 func.func @test_reshape_invalid_newshape(%arg0 : tensor<1xf32>) -> () {
+  %s = tosa.const_shape {value = dense<[-1, 4]> : tensor<2xindex>} : () -> !tosa.shape<2>
   // expected-error@+1 {{'tosa.reshape' op cannot reshape 1 elements into 4}}
-  %0 = "tosa.reshape"(%arg0) {new_shape = array<i64: -1, 4>} : (tensor<1xf32>) -> tensor<?x4xf32>
+  %0 = "tosa.reshape"(%arg0, %s) : (tensor<1xf32>, !tosa.shape<2>) -> tensor<?x4xf32>
   return
 }
 
 // -----
 
 func.func @test_reshape_invalid_newshape(%arg0 : tensor<8xf32>) -> () {
+  %s = tosa.const_shape {value = dense<[1, 4]> : tensor<2xindex>} : () -> !tosa.shape<2>
   // expected-error@+1 {{'tosa.reshape' op cannot reshape 8 elements into 4}}
-  %0 = "tosa.reshape"(%arg0) {new_shape = array<i64: 1, 4>} : (tensor<8xf32>) -> tensor<?x4xf32>
+  %0 = "tosa.reshape"(%arg0, %s) : (tensor<8xf32>, !tosa.shape<2>) -> tensor<?x4xf32>
   return
 }
 
 // -----
 
 func.func @test_reshape_invalid_placeholders(%arg0 : tensor<?xf32>) -> () {
+  %s = tosa.const_shape {value = dense<[2, -1, -1]> : tensor<3xindex>} : () -> !tosa.shape<3>
   // expected-error@+1 {{'tosa.reshape' op expected at most one target dimension to be -1}}
-  %0 = "tosa.reshape"(%arg0) {new_shape = array<i64: 2, -1, -1>} : (tensor<?xf32>) -> tensor<2x?x?xf32>
+  %0 = "tosa.reshape"(%arg0, %s) : (tensor<?xf32>, !tosa.shape<3>) -> tensor<2x?x?xf32>
   return
 }
 
 // -----
 
 func.func @test_reshape_invalid_tensor_dim(%arg0 : tensor<4x?xf32>) -> () {
+  %s = tosa.const_shape {value = dense<[-2, -1]> : tensor<2xindex>} : () -> !tosa.shape<2>
   // expected-error@+1 {{'tosa.reshape' op new shape has invalid tensor dimension size -2}}
-  %0 = "tosa.reshape" (%arg0) {new_shape = array<i64: -2, -1>} : (tensor<4x?xf32>) -> tensor<?x4xf32>
+  %0 = "tosa.reshape" (%arg0, %s) : (tensor<4x?xf32>, !tosa.shape<2>) -> tensor<?x4xf32>
   return
 }
 
@@ -515,6 +544,15 @@ func.func @test_reverse_axis_out_of_range(%arg0 : tensor<13x21x3xf32>) -> () {
 
 // -----
 
+func.func @test_reshape_zero_dim_input(%arg0 : tensor<?x0x3xf32>) -> () {
+  %1 = tosa.const_shape {value = dense<[13, 21, 3]> : tensor<3xindex>} : () -> !tosa.shape<3>
+  // expected-error@+1 {{'tosa.reshape' op operand #0 must be tosa-conformant tensor of number values, but got 'tensor<?x0x3xf32>'}}
+  %0 = "tosa.reshape"(%arg0, %1) : (tensor<?x0x3xf32>, !tosa.shape<3>) -> tensor<13x0x3xf32>
+  return
+}
+
+// -----
+
 func.func @test_const_attribute_type_mismatch() -> tensor<100x100xf32> {
   // expected-error@+1 {{'tosa.const' op failed to verify that all of {value, output} have same shape}}
   %0 = "tosa.const"() {value = dense<0.000000e+00> : tensor<1x1xf32>} : () -> tensor<100x100xf32>
@@ -607,8 +645,10 @@ func.func @test_variable_write_shape(%arg0: tensor<1x4x8xi32>) -> () {
 
 func.func @test_slice_invalid_start() {
   %0 = tensor.empty() : tensor<4x31x31xf32>
+  %start = tosa.const_shape {value = dense<[1, 1]> : tensor<2xindex>} : () -> !tosa.shape<2>
+  %size = tosa.const_shape {value = dense<[1, 1, 1]> : tensor<3xindex>} : () -> !tosa.shape<3>
   // expected-error@+1 {{'tosa.slice' op length of start attribute is not equal rank of input shape}}
-  %1 = tosa.slice %0 {size = array<i64: 1, 1, 1>, start = array<i64: 1, 1>} : (tensor<4x31x31xf32>) -> tensor<*xf32>
+  %3 = tosa.slice %0, %start, %size : (tensor<4x31x31xf32>, !tosa.shape<2>, !tosa.shape<3>) -> tensor<*xf32>
   return
 }
 
@@ -616,8 +656,10 @@ func.func @test_slice_invalid_start() {
 
 func.func @test_slice_invalid_size() {
   %0 = tensor.empty() : tensor<4x31x31xf32>
+  %start = tosa.const_shape {value = dense<[1, 1, 1]> : tensor<3xindex>} : () -> !tosa.shape<3>
+  %size = tosa.const_shape {value = dense<[1]> : tensor<1xindex>} : () -> !tosa.shape<1>
   // expected-error@+1 {{'tosa.slice' op length of size attribute is not equal rank of input shape}}
-  %1 = tosa.slice %0 {size = array<i64: 1>, start = array<i64: 1, 1, 1>} : (tensor<4x31x31xf32>) -> tensor<*xf32>
+  %3 = tosa.slice %0, %start, %size : (tensor<4x31x31xf32>, !tosa.shape<3>, !tosa.shape<1>) -> tensor<*xf32>
   return
 }
 
@@ -724,15 +766,35 @@ func.func @test_transpose_conv2d_invalid_outshape(%arg0: tensor<1x32x32x8xf32>,
 
 // -----
 
+// CHECK-LABEL: test_mul_type_mismatch
+func.func @test_mul_type_mismatch(%arg0: tensor<13x21x3xf32>, %arg1: tensor<13x1x3xf16>) -> tensor<13x21x3xf32> {
+  %shift = "tosa.const"() {value = dense<0> : tensor<1xi8>} : () -> tensor<1xi8>
+  // expected-error@+1 {{'tosa.mul' op requires the same element type for all operands}}
+  %0 = tosa.mul %arg0, %arg1, %shift : (tensor<13x21x3xf32>, tensor<13x1x3xf16>, tensor<1xi8>) -> tensor<13x21x3xf32>
+  return %0 : tensor<13x21x3xf32>
+}
+
+// -----
+
 // CHECK-LABEL: test_mul_invalid_shift
 func.func @test_mul_invalid_shift(%arg0: tensor<13x21x3xf32>, %arg1: tensor<13x1x3xf32>) -> tensor<13x21x3xf32> {
+  %shift = "tosa.const"() {value = dense<1> : tensor<1xi8>} : () -> tensor<1xi8>
   // expected-error@+1 {{'tosa.mul' op require shift to be 0 for float type}}
-  %0 = tosa.mul %arg0, %arg1 {shift = 1 : i8} : (tensor<13x21x3xf32>, tensor<13x1x3xf32>) -> tensor<13x21x3xf32>
+  %0 = tosa.mul %arg0, %arg1, %shift : (tensor<13x21x3xf32>, tensor<13x1x3xf32>, tensor<1xi8>) -> tensor<13x21x3xf32>
   return %0 : tensor<13x21x3xf32>
 }
 
 // -----
 
+// CHECK-LABEL: test_mul_missing_shift
+func.func @test_mul_missing_shift(%arg0: tensor<13x21x3xi32>, %arg1: tensor<13x1x3xi32>) -> tensor<13x21x3xi32> {
+  // expected-error@+1 {{'tosa.mul' op expected 3 operands, but found 2}}
+  %0 = tosa.mul %arg0, %arg1 : (tensor<13x21x3xi32>, tensor<13x1x3xi32>) -> tensor<13x21x3xi32>
+  return %0 : tensor<13x21x3xi32>
+}
+
+// -----
+
 // CHECK-LABEL: test_unsupported_int64_data_type
 func.func @test_unsupported_int64_data_type(%arg0: tensor<1x13x13x5xf32>) -> tensor<1x13x13xi64> {
   // expected-error@+1 {{'tosa.argmax' op is not profile-aligned: element type 'i64' is not legal}}
@@ -745,7 +807,7 @@ func.func @test_unsupported_int64_data_type(%arg0: tensor<1x13x13x5xf32>) -> ten
 // CHECK-LABEL: test_mismatch_in_out_data_type_clamp
 func.func @test_mismatch_in_out_data_type_clamp(%arg0: tensor<13x21x3xf32>) -> tensor<13x21x3xf16> {
   // expected-error@+1 {{'tosa.clamp' op requires the same element type for all operands and results}}
-  %0 = tosa.clamp %arg0 {min_fp = 0.0 : f32, max_fp = 1.0: f32, min_int = 0 : i64, max_int = 1 : i64} : (tensor<13x21x3xf32>) -> tensor<13x21x3xf16>
+  %0 = tosa.clamp %arg0 {min_val = 0.0 : f32, max_val = 1.0: f32} : (tensor<13x21x3xf32>) -> tensor<13x21x3xf16>
   return %0 : tensor<13x21x3xf16>
 }
 
@@ -754,7 +816,7 @@ func.func @test_mismatch_in_out_data_type_clamp(%arg0: tensor<13x21x3xf32>) -> t
 // CHECK-LABEL: test_mismatch_in_out_shape_clamp
 func.func @test_mismatch_in_out_shape_clamp(%arg0: tensor<13x21x3xf32>) -> tensor<13x21x1xf32> {
   // expected-error@+1 {{'tosa.clamp' op requires the same shape for all operands and results}}
-  %0 = tosa.clamp %arg0 {min_fp = 0.0 : f32, max_fp = 1.0: f32, min_int = 0 : i64, max_int = 1 : i64} : (tensor<13x21x3xf32>) -> tensor<13x21x1xf32>
+  %0 = tosa.clamp %arg0 {min_val = 0.0 : f32, max_val = 1.0: f32} : (tensor<13x21x3xf32>) -> tensor<13x21x1xf32>
   return %0 : tensor<13x21x1xf32>
 }
 
@@ -1038,3 +1100,30 @@ func.func @test_sub_with_unequal_result_ranks(%arg0: tensor<1x21x3xf32>, %arg1:
   %0 = tosa.sub %arg0, %arg1 : (tensor<1x21x3xf32>, tensor<13x21x3xf32>) -> tensor<1x13x21x3xf32>
   return %0 : tensor<1x13x21x3xf32>
 }
+
+// -----
+// CHECK-LABEL: test_mul_non_scalar_shift_2d
+func.func @test_mul_non_scalar_shift_2d(%arg0: tensor<13x21x3xf32>, %arg1: tensor<13x1x3xf32>) -> tensor<13x21x3xf32> {
+  %shift = "tosa.const"() <{value = dense<0> : tensor<1x1xi8>}> : () -> tensor<1x1xi8>
+  // expected-error@+1 {{'tosa.mul' op operand #2 must be tosa-conformant scalar tensor of 8-bit signless integer values, but got 'tensor<1x1xi8>'}}
+  %0 = tosa.mul %arg0, %arg1, %shift : (tensor<13x21x3xf32>, tensor<13x1x3xf32>, tensor<1x1xi8>) -> tensor<13x21x3xf32>
+  return %0 : tensor<13x21x3xf32>
+}
+
+// -----
+// CHECK-LABEL: test_mul_non_scalar_shift_1d
+func.func @test_mul_non_scalar_shift_1d(%arg0: tensor<13x21x3xf32>, %arg1: tensor<13x1x3xf32>) -> tensor<13x21x3xf32> {
+  %shift = "tosa.const"() <{value = dense<0> : tensor<2xi8>}> : () -> tensor<2xi8>
+  // expected-error@+1 {{'tosa.mul' op operand #2 must be tosa-conformant scalar tensor of 8-bit signless integer values, but got 'tensor<2xi8>'}}
+  %0 = tosa.mul %arg0, %arg1, %shift : (tensor<13x21x3xf32>, tensor<13x1x3xf32>, tensor<2xi8>) -> tensor<13x21x3xf32>
+  return %0 : tensor<13x21x3xf32>
+}
+
+// -----
+// CHECK-LABEL: test_mul_non_broadcast
+func.func @test_mul_non_broadcast(%arg0: tensor<13x21x2xf32>, %arg1: tensor<3x1x3xf32>) -> tensor<13x21x3xf32> {
+  %shift = "tosa.const"() <{value = dense<0> : tensor<1xi8>}> : () -> tensor<1xi8>
+  // expected-error@+1 {{'tosa.mul' op operands don't have broadcast-compatible shapes}}
+  %0 = tosa.mul %arg0, %arg1, %shift : (tensor<13x21x2xf32>, tensor<3x1x3xf32>, tensor<1xi8>) -> tensor<13x21x3xf32>
+  return %0 : tensor<13x21x3xf32>
+}
diff --git a/mlir/test/Dialect/Tosa/level_check.mlir b/mlir/test/Dialect/Tosa/level_check.mlir
index 0fe35d88f0e73..a7f76f2d0fa64 100644
--- a/mlir/test/Dialect/Tosa/level_check.mlir
+++ b/mlir/test/Dialect/Tosa/level_check.mlir
@@ -70,8 +70,9 @@ func.func @test_concat(%arg0: tensor<1x1x1x13x21x3x8xf32>, %arg1: tensor<1x1x1x1
 // -----
 
 func.func @test_reshape(%arg0: tensor<13x21x3xf32>) -> tensor<1x1x1x1x1x1x819xf32> {
+  %1 = tosa.const_shape {value = dense<[1, 1, 1, 1, 1, 1, 819]> : tensor<7xindex>} : () -> !tosa.shape<7>
   // expected-error@+1 {{'tosa.reshape' op failed level check: result rank(shape) <= MAX_RANK}}
-  %0 = "tosa.reshape"(%arg0) {new_shape = array<i64: 1, 1, 1, 1, 1, 1, 819>} : (tensor<13x21x3xf32>) -> tensor<1x1x1x1x1x1x819xf32>
+  %0 = "tosa.reshape"(%arg0, %1) : (tensor<13x21x3xf32>, !tosa.shape<7>) -> tensor<1x1x1x1x1x1x819xf32>
   return %0 : tensor<1x1x1x1x1x1x819xf32>
 }
 
@@ -86,10 +87,11 @@ func.func @test_reverse(%arg0: tensor<1x1x1x1x13x21x3xf32>) -> tensor<1x1x1x1x13
 // -----
 // CHECK-LABEL: slice
 func.func @test_slice(%arg0: tensor<1x1x1x1x13x21x3xf32>) -> tensor<1x1x1x1x4x11x1xf32> {
+  %0 = tosa.const_shape {value = dense<[0, 0, 0, 0, 6, 8, 0]> : tensor<7xindex>} : () -> !tosa.shape<7>
+  %1 = tosa.const_shape {value = dense<[1, 1, 1, 1, 4, 11, 1]> : tensor<7xindex>} : () -> !tosa.shape<7>
   // expected-error@+1 {{'tosa.slice' op failed level check: operand rank(shape) <= MAX_RANK}}
-  %0 = "tosa.slice"(%arg0) {start = array<i64: 0, 0, 0, 0, 6, 8, 0>, size = array<i64: 1, 1, 1, 1, 4, 11, 1>} :
-          (tensor<1x1x1x1x13x21x3xf32>) -> tensor<1x1x1x1x4x11x1xf32>
-  return %0 : tensor<1x1x1x1x4x11x1xf32>
+  %2= tosa.slice %arg0, %0, %1 : (tensor<1x1x1x1x13x21x3xf32>, !tosa.shape<7>, !tosa.shape<7>) -> tensor<1x1x1x1x4x11x1xf32>
+  return %2 : tensor<1x1x1x1x4x11x1xf32>
 }
 
 // -----
@@ -736,8 +738,10 @@ func.func @test_custom(%arg0: tensor<1x1x1x1x1x1x10xi32>) -> tensor<1x1x1x1x1x1x
 
 // CHECK-LABEL: unranked_tensor
 func.func @test_unranked_tensor(%arg0: tensor<*xf32>) {
+  %0 = tosa.const_shape {value = dense<[0]> : tensor<1xindex>} : () -> !tosa.shape<1>
+  %1 = tosa.const_shape {value = dense<[1]> : tensor<1xindex>} : () -> !tosa.shape<1>
+
   // expected-error@+1 {{'tosa.slice' op failed level check: unranked tensor}}
-  %0 = "tosa.slice"(%arg0) {start = array<i64>, size = array<i64>} :
-          (tensor<*xf32>) -> tensor<*xf32>
+  %2= tosa.slice %arg0, %0, %1 : (tensor<*xf32>, !tosa.shape<1>, !tosa.shape<1>) -> tensor<*xf32>
   return
 }
diff --git a/mlir/test/Dialect/Tosa/ops.mlir b/mlir/test/Dialect/Tosa/ops.mlir
index 19b93d7611854..348849cfaa572 100644
--- a/mlir/test/Dialect/Tosa/ops.mlir
+++ b/mlir/test/Dialect/Tosa/ops.mlir
@@ -58,12 +58,29 @@ func.func @test_conv2d(%arg0: tensor<1x4x4x4xf32>, %arg1: tensor<8x1x1x4xf32>, %
   return %0 : tensor<1x4x4x8xf32>
 }
 
+// -----
+// CHECK-LABEL: conv2d_quant_uniform
+func.func @test_conv2d_quant_uniform(%arg0: tensor<1x4x4x4x!quant.uniform<i8:f32, 0.01>>, %arg1: tensor<8x1x1x4x!quant.uniform<i8:f32, 0.01>>, %arg2: tensor<8x!quant.uniform<i8:f32, 0.01>>) -> tensor<1x4x4x8x!quant.uniform<i32:f32, 0.01>> {
+  %zp = "tosa.const" () { value = dense<0> : tensor<1xi8> } : () -> tensor<1xi8>
+  %0 = tosa.conv2d %arg0, %arg1, %arg2, %zp, %zp {acc_type = i32, dilation = array<i64: 1, 1>, pad = array<i64: 0, 0, 0, 0>, stride = array<i64: 1, 1>, local_bound = true} : (tensor<1x4x4x4x!quant.uniform<i8:f32, 0.01>>, tensor<8x1x1x4x!quant.uniform<i8:f32, 0.01>>, tensor<8x!quant.uniform<i8:f32, 0.01>>, tensor<1xi8>, tensor<1xi8>) -> tensor<1x4x4x8x!quant.uniform<i32:f32, 0.01>>
+  return %0 : tensor<1x4x4x8x!quant.uniform<i32:f32, 0.01>>
+}
+
+// -----
+// CHECK-LABEL: conv2d_quant_any
+func.func @test_conv2d_quant_any(%arg0: tensor<1x4x4x4x!quant.any<i8<-8:7>>>, %arg1: tensor<8x1x1x4x!quant.any<i8<-8:7>>>, %arg2: tensor<8x!quant.any<i8<-8:7>>>) -> tensor<1x4x4x8x!quant.any<i32<-8:7>>> {
+  %zp = "tosa.const" () { value = dense<0> : tensor<1xi8> } : () -> tensor<1xi8>
+  %0 = tosa.conv2d %arg0, %arg1, %arg2, %zp, %zp {acc_type = i32, dilation = array<i64: 1, 1>, pad = array<i64: 0, 0, 0, 0>, stride = array<i64: 1, 1>, local_bound = true} : (tensor<1x4x4x4x!quant.any<i8<-8:7>>>, tensor<8x1x1x4x!quant.any<i8<-8:7>>>, tensor<8x!quant.any<i8<-8:7>>>, tensor<1xi8>, tensor<1xi8>) -> tensor<1x4x4x8x!quant.any<i32<-8:7>>>
+  return %0 : tensor<1x4x4x8x!quant.any<i32<-8:7>>>
+}
+
 // -----
 // CHECK-LABEL: conv2d_q8xi4
 func.func @test_conv2d_q8xi4(%arg0: tensor<1x11x11x3xi8>) -> tensor<1x1x1x3xi8> {
   %0 = "tosa.const"() {value = dense<0> : tensor<3x11x11x3xi4>} : () -> tensor<3x11x11x3xi4>
   %1 = "tosa.const"() {value = dense<[12, 23, 55]> : tensor<3xi32>} : () -> tensor<3xi32>
-  %2 = "tosa.conv2d"(%arg0, %0, %1) {acc_type = i32, dilation = array<i64: 1, 1>, pad = array<i64: 0, 0, 0, 0>, quantization_info = #tosa.conv_quant<input_zp = 0, weight_zp = 0>, stride = array<i64: 1, 1>} : (tensor<1x11x11x3xi8>, tensor<3x11x11x3xi4>, tensor<3xi32>) -> tensor<1x1x1x3xi32>
+  %zp = "tosa.const"() {value = dense<0> : tensor<1xi8>} : () -> tensor<1xi8>
+  %2 = "tosa.conv2d"(%arg0, %0, %1, %zp, %zp) {acc_type = i32, dilation = array<i64: 1, 1>, pad = array<i64: 0, 0, 0, 0>, stride = array<i64: 1, 1>} : (tensor<1x11x11x3xi8>, tensor<3x11x11x3xi4>, tensor<3xi32>, tensor<1xi8>, tensor<1xi8>) -> tensor<1x1x1x3xi32>
   %3 = "tosa.rescale"(%2) {double_round = true, input_zp = 0 : i32, multiplier = array<i32: 2026291432, 1079222024, 1693132724>, output_zp = 27 : i32, per_channel = true, scale32 = true, shift = array<i8: 37, 36, 37>} : (tensor<1x1x1x3xi32>) -> tensor<1x1x1x3xi8>
   return %3 : tensor<1x1x1x3xi8>
 }
@@ -176,42 +193,42 @@ func.func @test_transpose_conv2d_with_local_bound(%arg0: tensor<1x32x32x8xf32>,
 // -----
 // CHECK-LABEL: clamp
 func.func @test_clamp(%arg0: tensor<13x21x3xf32>) -> tensor<13x21x3xf32> {
-  %0 = tosa.clamp %arg0 {min_fp = 0.0 : f32, max_fp = 1.0: f32, min_int = 0 : i64, max_int = 1 : i64} : (tensor<13x21x3xf32>) -> tensor<13x21x3xf32>
+  %0 = tosa.clamp %arg0 {min_val = 0.0 : f32, max_val = 1.0: f32} : (tensor<13x21x3xf32>) -> tensor<13x21x3xf32>
   return %0 : tensor<13x21x3xf32>
 }
 
 // -----
 // CHECK-LABEL: clamp_propagate
 func.func @test_clamp_propagate(%arg0: tensor<13x21x3xf32>) -> tensor<13x21x3xf32> {
-  %0 = tosa.clamp %arg0 {min_fp = 0.0 : f32, max_fp = 1.0: f32, min_int = 0 : i64, max_int = 1 : i64, nan_mode = "PROPAGATE"} : (tensor<13x21x3xf32>) -> tensor<13x21x3xf32>
+  %0 = tosa.clamp %arg0 {min_val = 0.0 : f32, max_val = 1.0: f32, nan_mode = "PROPAGATE"} : (tensor<13x21x3xf32>) -> tensor<13x21x3xf32>
   return %0 : tensor<13x21x3xf32>
 }
 
 // -----
 // CHECK-LABEL: clamp_ignore
 func.func @test_clamp_ignore(%arg0: tensor<13x21x3xf32>) -> tensor<13x21x3xf32> {
-  %0 = tosa.clamp %arg0 {min_fp = 0.0 : f32, max_fp = 1.0: f32, min_int = 0 : i64, max_int = 1 : i64, nan_mode = "IGNORE"} : (tensor<13x21x3xf32>) -> tensor<13x21x3xf32>
+  %0 = tosa.clamp %arg0 {min_val = 0.0 : f32, max_val = 1.0: f32, nan_mode = "IGNORE"} : (tensor<13x21x3xf32>) -> tensor<13x21x3xf32>
   return %0 : tensor<13x21x3xf32>
 }
 
 // -----
 // CHECK-LABEL: clamp_f16
 func.func @test_clamp_f16(%arg0: tensor<13x21x3xf16>) -> tensor<13x21x3xf16> {
-  %0 = tosa.clamp %arg0 {min_fp = 0.0 : f16, max_fp = 1.0: f16, min_int = 0 : i64, max_int = 1 : i64} : (tensor<13x21x3xf16>) -> tensor<13x21x3xf16>
+  %0 = tosa.clamp %arg0 {min_val = 0.0 : f16, max_val = 1.0: f16} : (tensor<13x21x3xf16>) -> tensor<13x21x3xf16>
   return %0 : tensor<13x21x3xf16>
 }
 
 // -----
 // CHECK-LABEL: clamp_bf16
 func.func @test_clamp_bf16(%arg0: tensor<13x21x3xbf16>) -> tensor<13x21x3xbf16> {
-  %0 = tosa.clamp %arg0 {min_fp = 0.0 : bf16, max_fp = 1.0: bf16, min_int = 0 : i64, max_int = 1 : i64} : (tensor<13x21x3xbf16>) -> tensor<13x21x3xbf16>
+  %0 = tosa.clamp %arg0 {min_val = 0.0 : bf16, max_val = 1.0: bf16} : (tensor<13x21x3xbf16>) -> tensor<13x21x3xbf16>
   return %0 : tensor<13x21x3xbf16>
 }
 
 // -----
 // CHECK-LABEL: clamp_quantized
 func.func @test_clamp_quantized(%arg0: tensor<13x21x3x!quant.uniform<i8:f32, 1.000000e-01:-127>>) -> tensor<13x21x3x!quant.uniform<i8:f32, 1.000000e-01:-127>> {
-  %0 = tosa.clamp %arg0 {min_fp = 0.0 : f32, max_fp = 1.0: f32, min_int = 0 : i64, max_int = 1 : i64} : (tensor<13x21x3x!quant.uniform<i8:f32, 1.000000e-01:-127>>) -> tensor<13x21x3x!quant.uniform<i8:f32, 1.000000e-01:-127>>
+  %0 = tosa.clamp %arg0 {min_val = 0 : i8, max_val = 1 : i8} : (tensor<13x21x3x!quant.uniform<i8:f32, 1.000000e-01:-127>>) -> tensor<13x21x3x!quant.uniform<i8:f32, 1.000000e-01:-127>>
   return %0 : tensor<13x21x3x!quant.uniform<i8:f32, 1.000000e-01:-127>>
 }
 
@@ -327,17 +344,35 @@ func.func @test_min(%arg0: tensor<13x21x3xf32>, %arg1: tensor<1x21x3xf32>) -> te
   return %0 : tensor<13x21x3xf32>
 }
 
+// -----
+// CHECK-LABEL: test_mul_scalar_with_unranked_output
+func.func @test_mul_scalar_with_unranked_output(%arg0: tensor<f32>, %arg1: tensor<f32>) -> tensor<*xf32> {
+  %shift = "tosa.const"() <{value = dense<0> : tensor<1xi8>}> : () -> tensor<1xi8>
+  %0 = tosa.mul %arg0, %arg1, %shift : (tensor<f32>, tensor<f32>, tensor<1xi8>) -> tensor<*xf32>
+  return %0 : tensor<*xf32>
+}
+
 // -----
 // CHECK-LABEL: mul
 func.func @test_mul(%arg0: tensor<13x21x3xf32>, %arg1: tensor<13x1x3xf32>) -> tensor<13x21x3xf32> {
-  %0 = tosa.mul %arg0, %arg1 {shift = 0 : i8} : (tensor<13x21x3xf32>, tensor<13x1x3xf32>) -> tensor<13x21x3xf32>
+  %shift = "tosa.const"() <{value = dense<0> : tensor<1xi8>}> : () -> tensor<1xi8>
+  %0 = tosa.mul %arg0, %arg1, %shift : (tensor<13x21x3xf32>, tensor<13x1x3xf32>, tensor<1xi8>) -> tensor<13x21x3xf32>
   return %0 : tensor<13x21x3xf32>
 }
 
+// -----
+// CHECK-LABEL: i32_mul
+func.func @test_i32_mul(%arg0: tensor<13x21x3xi32>, %arg1: tensor<13x1x3xi32>) -> tensor<13x21x3xi32> {
+  %shift = "tosa.const"() <{value = dense<0> : tensor<1xi8>}> : () -> tensor<1xi8>
+  %0 = tosa.mul %arg0, %arg1, %shift : (tensor<13x21x3xi32>, tensor<13x1x3xi32>, tensor<1xi8>) -> tensor<13x21x3xi32>
+  return %0 : tensor<13x21x3xi32>
+}
+
 // -----
 // CHECK-LABEL: mul
 func.func @test_mul_relaxed_result_type(%arg0: tensor<13x21x3xi16>, %arg1: tensor<13x1x3xi16>) -> tensor<13x21x3xi16> {
-  %0 = "tosa.mul"(%arg0, %arg1)  { shift = 1 : i8 } : (tensor<13x21x3xi16>, tensor<13x1x3xi16>) -> tensor<13x21x3xi16>
+  %shift = "tosa.const"() <{value = dense<0> : tensor<1xi8>}> : () -> tensor<1xi8>
+  %0 = tosa.mul %arg0, %arg1, %shift : (tensor<13x21x3xi16>, tensor<13x1x3xi16>, tensor<1xi8>) -> tensor<13x21x3xi16>
   return %0 : tensor<13x21x3xi16>
 }
 
@@ -486,7 +521,8 @@ func.func @test_greater_equal(%arg0: tensor<13x1x3xf32>, %arg1: tensor<13x21x3xf
 // CHECK-LABEL: reduce_all
 func.func @test_reduce_all(%arg0: tensor<13x21x3xi1>) -> tensor<21x3xi1> {
   %0 = tosa.reduce_all %arg0 {axis = 0 : i32} : (tensor<13x21x3xi1>) -> tensor<1x21x3xi1>
-  %1 = tosa.reshape %0 {new_shape = array<i64: 21, 3>} : (tensor<1x21x3xi1>) -> tensor<21x3xi1>
+  %2 = tosa.const_shape {value = dense<[21, 3]> : tensor<2xindex>} : () -> !tosa.shape<2>
+  %1 = tosa.reshape %0, %2 : (tensor<1x21x3xi1>, !tosa.shape<2>) -> tensor<21x3xi1>
   return %1 : tensor<21x3xi1>
 }
 
@@ -494,7 +530,8 @@ func.func @test_reduce_all(%arg0: tensor<13x21x3xi1>) -> tensor<21x3xi1> {
 // CHECK-LABEL: reduce_any
 func.func @test_reduce_any(%arg0: tensor<13x21x3xi1>) -> tensor<21x3xi1> {
   %0 = tosa.reduce_any %arg0 {axis = 0 : i32} : (tensor<13x21x3xi1>) -> tensor<1x21x3xi1>
-  %1 = tosa.reshape %0 {new_shape = array<i64: 21, 3>} : (tensor<1x21x3xi1>) -> tensor<21x3xi1>
+  %2 = tosa.const_shape {value = dense<[21, 3]> : tensor<2xindex>} : () -> !tosa.shape<2>
+  %1 = tosa.reshape %0, %2 : (tensor<1x21x3xi1>, !tosa.shape<2>) -> tensor<21x3xi1>
   return %1 : tensor<21x3xi1>
 }
 
@@ -502,7 +539,8 @@ func.func @test_reduce_any(%arg0: tensor<13x21x3xi1>) -> tensor<21x3xi1> {
 // CHECK-LABEL: reduce_max
 func.func @test_reduce_max(%arg0: tensor<13x21x3xf32>) -> tensor<21x3xf32> {
   %0 = tosa.reduce_max %arg0 {axis = 0 : i32} : (tensor<13x21x3xf32>) -> tensor<1x21x3xf32>
-  %1 = tosa.reshape %0 {new_shape = array<i64: 21, 3>} : (tensor<1x21x3xf32>) -> tensor<21x3xf32>
+  %2 = tosa.const_shape {value = dense<[21, 3]> : tensor<2xindex>} : () -> !tosa.shape<2>
+  %1 = tosa.reshape %0, %2 : (tensor<1x21x3xf32>, !tosa.shape<2>) -> tensor<21x3xf32>
   return %1 : tensor<21x3xf32>
 }
 
@@ -510,7 +548,8 @@ func.func @test_reduce_max(%arg0: tensor<13x21x3xf32>) -> tensor<21x3xf32> {
 // CHECK-LABEL: reduce_min
 func.func @test_reduce_min(%arg0: tensor<13x21x3xf32>) -> tensor<21x3xf32> {
   %0 = tosa.reduce_min %arg0 {axis = 0 : i32} : (tensor<13x21x3xf32>) -> tensor<1x21x3xf32>
-  %1 = tosa.reshape %0 {new_shape = array<i64: 21, 3>} : (tensor<1x21x3xf32>) -> tensor<21x3xf32>
+  %2 = tosa.const_shape {value = dense<[21, 3]> : tensor<2xindex>} : () -> !tosa.shape<2>
+  %1 = tosa.reshape %0, %2 : (tensor<1x21x3xf32>, !tosa.shape<2>) -> tensor<21x3xf32>
   return %1 : tensor<21x3xf32>
 }
 
@@ -518,7 +557,8 @@ func.func @test_reduce_min(%arg0: tensor<13x21x3xf32>) -> tensor<21x3xf32> {
 // CHECK-LABEL: reduce_product
 func.func @test_reduce_product(%arg0: tensor<13x21x3xf32>) -> tensor<21x3xf32> {
   %0 = tosa.reduce_prod %arg0 {axis = 0 : i32} : (tensor<13x21x3xf32>) -> tensor<1x21x3xf32>
-  %1 = tosa.reshape %0 {new_shape = array<i64: 21, 3>} : (tensor<1x21x3xf32>) -> tensor<21x3xf32>
+  %2 = tosa.const_shape {value = dense<[21, 3]> : tensor<2xindex>} : () -> !tosa.shape<2>
+  %1 = tosa.reshape %0, %2 : (tensor<1x21x3xf32>, !tosa.shape<2>) -> tensor<21x3xf32>
   return %1 : tensor<21x3xf32>
 }
 
@@ -526,7 +566,8 @@ func.func @test_reduce_product(%arg0: tensor<13x21x3xf32>) -> tensor<21x3xf32> {
 // CHECK-LABEL: reduce_sum
 func.func @test_reduce_sum(%arg0: tensor<13x21x3xf32>) -> tensor<21x3xf32> {
   %0 = tosa.reduce_sum %arg0 {axis = 0 : i32} : (tensor<13x21x3xf32>) -> tensor<1x21x3xf32>
-  %1 = tosa.reshape %0 {new_shape = array<i64: 21, 3>} : (tensor<1x21x3xf32>) -> tensor<21x3xf32>
+  %2 = tosa.const_shape {value = dense<[21, 3]> : tensor<2xindex>} : () -> !tosa.shape<2>
+  %1 = tosa.reshape %0, %2 : (tensor<1x21x3xf32>, !tosa.shape<2>) -> tensor<21x3xf32>
   return %1 : tensor<21x3xf32>
 }
 
@@ -557,7 +598,8 @@ func.func @test_pad_explicit_value(%arg0: tensor<13x21x3xf32>) -> tensor<13x21x3
 // -----
 // CHECK-LABEL: reshape
 func.func @test_reshape(%arg0: tensor<13x21x3xf32>) -> tensor<1x819xf32> {
-  %0 = tosa.reshape %arg0 {new_shape = array<i64: 1, 819>} : (tensor<13x21x3xf32>) -> tensor<1x819xf32>
+  %1 = tosa.const_shape {value = dense<[1, 819]> : tensor<2xindex>} : () -> !tosa.shape<2>
+  %0 = tosa.reshape %arg0, %1 : (tensor<13x21x3xf32>, !tosa.shape<2>) -> tensor<1x819xf32>
   return %0 : tensor<1x819xf32>
 }
 
@@ -571,8 +613,19 @@ func.func @test_reverse(%arg0: tensor<13x21x3xf32>) -> tensor<13x21x3xf32> {
 // -----
 // CHECK-LABEL: slice
 func.func @test_slice(%arg0: tensor<13x21x3xf32>) -> tensor<4x11x1xf32> {
-  %0 = tosa.slice %arg0 {size = array<i64: 4, 11, 1>, start = array<i64: 6, 8, 0>} : (tensor<13x21x3xf32>) -> tensor<4x11x1xf32>
-  return %0 : tensor<4x11x1xf32>
+  %0 = tosa.const_shape {value = dense<[4, 11, 1]> : tensor<3xindex>} : () -> !tosa.shape<3>
+  %1 = tosa.const_shape {value = dense<[6, 8, 0]> : tensor<3xindex>} : () -> !tosa.shape<3>
+  %2 = tosa.slice %arg0, %0, %1 : (tensor<13x21x3xf32>, !tosa.shape<3>, !tosa.shape<3>) -> tensor<4x11x1xf32>
+  return %2 : tensor<4x11x1xf32>
+}
+
+// -----
+// CHECK-LABEL: slice_size
+func.func @test_slice_size(%arg0: tensor<13x21x3xf32>) -> tensor<7x11x1xf32> {
+  %0 = tosa.const_shape {value = dense<[-1, 11, 1]> : tensor<3xindex>} : () -> !tosa.shape<3>
+  %1 = tosa.const_shape {value = dense<[6, 8, 0]> : tensor<3xindex>} : () -> !tosa.shape<3>
+  %2 = tosa.slice %arg0, %0, %1 : (tensor<13x21x3xf32>, !tosa.shape<3>, !tosa.shape<3>) -> tensor<7x11x1xf32>
+  return %2 : tensor<7x11x1xf32>
 }
 
 // -----
@@ -695,7 +748,8 @@ func.func @test_while_loop(%arg0: tensor<10xi32>, %arg1: tensor<i32>) {
   ^bb0(%arg2: tensor<i32>, %arg3: tensor<i32>, %arg4: tensor<10xi32>):
     %2 = "tosa.const"() {value = dense<1> : tensor<i32>} : () -> tensor<i32>
     %3 = tosa.add %arg3, %2 : (tensor<i32>, tensor<i32>) -> tensor<i32>
-    %4 = tosa.reshape %2 {new_shape = array<i64: 1>} : (tensor<i32>) -> tensor<1xi32>
+    %7 = tosa.const_shape {value = dense<[1]> : tensor<1xindex>} : () -> !tosa.shape<1>
+    %4 = tosa.reshape %2, %7 : (tensor<i32>, !tosa.shape<1>) -> tensor<1xi32>
     %5 = tosa.add %arg4, %4 : (tensor<10xi32>, tensor<1xi32>) -> tensor<10xi32>
     %6 = tosa.add %arg2, %2 : (tensor<i32>, tensor<i32>) -> tensor<i32>
     tosa.yield %6, %3, %5 : tensor<i32>, tensor<i32>, tensor<10xi32>
diff --git a/mlir/test/Dialect/Tosa/quant-test.mlir b/mlir/test/Dialect/Tosa/quant-test.mlir
index 6437f12e3ff85..ee6caf285a248 100644
--- a/mlir/test/Dialect/Tosa/quant-test.mlir
+++ b/mlir/test/Dialect/Tosa/quant-test.mlir
@@ -12,7 +12,9 @@ func.func @test_build_qtype(%arg0 : tensor<16x1x1x8x!quant.uniform<u8<1:255>:f32
 // CHECK-LABEL: test_build_mult_and_shift
 func.func @test_build_mult_and_shift(%arg0: tensor<1x32x32x8x!quant.uniform<i8:f32, 0.015684768557548523>>, %arg1 : tensor<16x1x1x8x!quant.uniform<i8<-127:127>:f32, 0.015680249780416489>>, %arg2 : tensor<16xi32>) -> tensor<1x32x32x16x!quant.uniform<i32:f32, 0.078431375324726104>> {
   // CHECK: tosa.conv2d
-  %0 = "tosa.conv2d"(%arg0, %arg1, %arg2) {acc_type = i32, pad = array<i64: 1, 1, 2, 2>, dilation = array<i64: 2, 1>, stride = array<i64: 1, 1>, quantization_info = #tosa.conv_quant<input_zp = -1, weight_zp = 0>} : (tensor<1x32x32x8x!quant.uniform<i8:f32, 0.015684768557548523>>, tensor<16x1x1x8x!quant.uniform<i8<-127:127>:f32, 0.015680249780416489>>, tensor<16xi32>) -> tensor<1x32x32x16x!quant.uniform<i32:f32, 0.078431375324726104>>
+  %input_zp = "tosa.const"() {value = dense<-1> : tensor<1xi8>} : () -> tensor<1xi8>
+  %weight_zp = "tosa.const"() {value = dense<1> : tensor<1xi8>} : () -> tensor<1xi8>
+  %0 = "tosa.conv2d"(%arg0, %arg1, %arg2, %input_zp, %weight_zp) {acc_type = i32, pad = array<i64: 1, 1, 2, 2>, dilation = array<i64: 2, 1>, stride = array<i64: 1, 1>} : (tensor<1x32x32x8x!quant.uniform<i8:f32, 0.015684768557548523>>, tensor<16x1x1x8x!quant.uniform<i8<-127:127>:f32, 0.015680249780416489>>, tensor<16xi32>, tensor<1xi8>, tensor<1xi8>) -> tensor<1x32x32x16x!quant.uniform<i32:f32, 0.078431375324726104>>
   return %0 : tensor<1x32x32x16x!quant.uniform<i32:f32, 0.078431375324726104>>
 
 }
diff --git a/mlir/test/Dialect/Tosa/tosa-decompose-conv2d.mlir b/mlir/test/Dialect/Tosa/tosa-decompose-conv2d.mlir
index 95d9bb1b98ab7..9aade2fe45eb6 100644
--- a/mlir/test/Dialect/Tosa/tosa-decompose-conv2d.mlir
+++ b/mlir/test/Dialect/Tosa/tosa-decompose-conv2d.mlir
@@ -5,13 +5,16 @@
 // CHECK-LABEL: @conv2d_as_fully_connected
 func.func @conv2d_as_fully_connected(%arg0: tensor<4x10x10x2xf32>, %arg1: tensor<3x1x1x2xf32>, %arg2: tensor<3xf32>) -> tensor<4x10x10x3xf32> {
   // CHECK-NOT: tosa.conv2d
-  // CHECK: %[[VAR0:.*]] = tosa.reshape %arg0 {new_shape = array<i64: 400, 2>}
+  // CHECK-DAG: %[[CONST0:.*]] = tosa.const_shape {value = dense<[400, 2]> : tensor<2xindex>} : () -> !tosa.shape<2>
+  // CHECK-DAG: %[[CONST1:.*]] = tosa.const_shape {value = dense<[3, 2]> : tensor<2xindex>} : () -> !tosa.shape<2>
+  // CHECK-DAG: %[[CONST2:.*]] = tosa.const_shape {value = dense<[4, 10, 10, 3]> : tensor<4xindex>} : () -> !tosa.shape<4>
+  // CHECK: %[[VAR0:.*]] = tosa.reshape %arg0, %[[CONST0]]
   // CHECK-SAME: -> tensor<400x2xf32>
-  // CHECK: %[[VAR1:.*]] = tosa.reshape %arg1 {new_shape = array<i64: 3, 2>}
+  // CHECK: %[[VAR1:.*]] = tosa.reshape %arg1, %[[CONST1]]
   // CHECK-SAME: -> tensor<3x2xf32>
   // CHECK: %[[VAR2:.*]] = tosa.fully_connected %[[VAR0]], %[[VAR1]], %arg2
   // CHECK-SAME: -> tensor<400x3xf32>
-  // CHECK: %[[VAR3:.*]] = tosa.reshape %[[VAR2]] {new_shape = array<i64: 4, 10, 10, 3>}
+  // CHECK: %[[VAR3:.*]] = tosa.reshape %[[VAR2]], %[[CONST2]]
   // CHECK-SAME: -> tensor<4x10x10x3xf32>
   // CHECK: return %[[VAR3]]
   %0 = tosa.conv2d %arg0, %arg1, %arg2 {acc_type = f32, pad = array<i64: 0, 0, 0, 0>, stride = array<i64: 1, 1>, dilation = array<i64: 1, 1>} : (tensor<4x10x10x2xf32>, tensor<3x1x1x2xf32>, tensor<3xf32>) -> tensor<4x10x10x3xf32>
@@ -23,34 +26,41 @@ func.func @conv2d_as_fully_connected(%arg0: tensor<4x10x10x2xf32>, %arg1: tensor
 // CHECK-LABEL: @conv2d_as_fully_connected_quant
 func.func @conv2d_as_fully_connected_quant(%arg0: tensor<4x10x10x2xi8>, %arg1: tensor<3x1x1x2xi8>, %arg2: tensor<3xi32>) -> tensor<4x10x10x3xi32> {
   // CHECK-NOT: tosa.conv2d
-  // CHECK: %[[VAR0:.*]] = tosa.reshape %arg0 {new_shape = array<i64: 400, 2>}
+  // CHECK-DAG: %[[CONST0:.*]] = tosa.const_shape {value = dense<[400, 2]> : tensor<2xindex>} : () -> !tosa.shape<2>
+  // CHECK-DAG: %[[CONST1:.*]] = tosa.const_shape {value = dense<[3, 2]> : tensor<2xindex>} : () -> !tosa.shape<2>
+  // CHECK-DAG: %[[CONST2:.*]] = tosa.const_shape {value = dense<[4, 10, 10, 3]> : tensor<4xindex>} : () -> !tosa.shape<4>
+  // CHECK: %[[VAR0:.*]] = tosa.reshape %arg0, %[[CONST0]]
   // CHECK-SAME: -> tensor<400x2xi8>
-  // CHECK: %[[VAR1:.*]] = tosa.reshape %arg1 {new_shape = array<i64: 3, 2>}
+  // CHECK: %[[VAR1:.*]] = tosa.reshape %arg1, %[[CONST1]]
   // CHECK-SAME: -> tensor<3x2xi8>
   // CHECK: %[[VAR2:.*]] = tosa.fully_connected %[[VAR0]], %[[VAR1]], %arg2
-  // CHECK-SAME: quantization_info = #tosa.conv_quant<input_zp = 42, weight_zp = 24>
+  // CHECK-SAME: {input_zp = 42 : i32, weight_zp = 24 : i32}
   // CHECK-SAME: -> tensor<400x3xi32>
-  // CHECK: %[[VAR3:.*]] = tosa.reshape %[[VAR2]] {new_shape = array<i64: 4, 10, 10, 3>}
+  // CHECK: %[[VAR3:.*]] = tosa.reshape %[[VAR2]], %[[CONST2]]
   // CHECK-SAME: -> tensor<4x10x10x3xi32>
   // CHECK: return %[[VAR3]]
-  %0 = tosa.conv2d %arg0, %arg1, %arg2 {acc_type = i32, pad = array<i64: 0, 0, 0, 0>, stride = array<i64: 1, 1>, dilation = array<i64: 1, 1>, quantization_info = #tosa.conv_quant<input_zp = 42, weight_zp = 24>} : (tensor<4x10x10x2xi8>, tensor<3x1x1x2xi8>, tensor<3xi32>) -> tensor<4x10x10x3xi32>
+  %input_zp = "tosa.const"() {value = dense<42> : tensor<1xi8>} : () -> tensor<1xi8>
+  %weight_zp = "tosa.const"() {value = dense<24> : tensor<1xi8>} : () -> tensor<1xi8>
+  %0 = tosa.conv2d %arg0, %arg1, %arg2, %input_zp, %weight_zp {acc_type = i32, pad = array<i64: 0, 0, 0, 0>, stride = array<i64: 1, 1>, dilation = array<i64: 1, 1>} : (tensor<4x10x10x2xi8>, tensor<3x1x1x2xi8>, tensor<3xi32>, tensor<1xi8>, tensor<1xi8>) -> tensor<4x10x10x3xi32>
   return %0 : tensor<4x10x10x3xi32>
 }
 
 // -----
 
 // CHECK-LABEL:   func.func @conv_with_dynamic_dim(
-// CHECK-SAME:                                     %[[VAL_0:.*]]: tensor<?x14x14x64xi8>,
-// CHECK-SAME:                                     %[[VAL_1:.*]]: tensor<384x1x1x64xi8>,
-// CHECK-SAME:                                     %[[VAL_2:.*]]: tensor<384xi32>) -> tensor<?x14x14x384xi32> {
 func.func @conv_with_dynamic_dim(%arg0: tensor<?x14x14x64xi8>, %arg1: tensor<384x1x1x64xi8>, %arg2: tensor<384xi32>) -> tensor<?x14x14x384xi32> {
-// CHECK:           %[[VAL_3:.*]] = tosa.reshape %[[VAL_0]] {new_shape = array<i64: -1, 64>} : (tensor<?x14x14x64xi8>) -> tensor<?x64xi8>
-// CHECK:           %[[VAL_4:.*]] = tosa.reshape %[[VAL_1]] {new_shape = array<i64: 384, 64>} : (tensor<384x1x1x64xi8>) -> tensor<384x64xi8>
-// CHECK:           %[[VAL_5:.*]] = tosa.fully_connected %[[VAL_3]], %[[VAL_4]], %[[VAL_2]] {quantization_info = #tosa.conv_quant<input_zp = -6, weight_zp = 11>} : (tensor<?x64xi8>, tensor<384x64xi8>, tensor<384xi32>) -> tensor<?x384xi32>
-// CHECK:           %[[VAL_6:.*]] = tosa.reshape %[[VAL_5]] {new_shape = array<i64: -1, 14, 14, 384>} : (tensor<?x384xi32>) -> tensor<?x14x14x384xi32>
+// CHECK-DAG:           %[[CONST0:.*]] = tosa.const_shape {value = dense<[-1, 64]> : tensor<2xindex>} : () -> !tosa.shape<2>
+// CHECK-DAG:           %[[CONST1:.*]] = tosa.const_shape {value = dense<[384, 64]> : tensor<2xindex>} : () -> !tosa.shape<2>
+// CHECK-DAG:           %[[CONST2:.*]] = tosa.const_shape {value = dense<[-1, 14, 14, 384]> : tensor<4xindex>} : () -> !tosa.shape<4>
+// CHECK:           %[[VAL_3:.*]] = tosa.reshape %arg0, %[[CONST0]]
+// CHECK:           %[[VAL_4:.*]] = tosa.reshape %arg1, %[[CONST1]] : (tensor<384x1x1x64xi8>, !tosa.shape<2>) -> tensor<384x64xi8>
+// CHECK:           %[[VAL_5:.*]] = tosa.fully_connected %[[VAL_3]], %[[VAL_4]], %arg2 {input_zp = -6 : i32, weight_zp = 11 : i32} : (tensor<?x64xi8>, tensor<384x64xi8>, tensor<384xi32>) -> tensor<?x384xi32>
+// CHECK:           %[[VAL_6:.*]] = tosa.reshape %[[VAL_5]], %[[CONST2]] : (tensor<?x384xi32>, !tosa.shape<4>) -> tensor<?x14x14x384xi32>
 // CHECK:           return %[[VAL_6]] : tensor<?x14x14x384xi32>
 // CHECK:         }
-  %0 = tosa.conv2d %arg0, %arg1, %arg2 {acc_type = i32, dilation = array<i64: 1, 1>, pad = array<i64: 0, 0, 0, 0>, quantization_info = #tosa.conv_quant<input_zp = -6, weight_zp = 11>, stride = array<i64: 1, 1>} : (tensor<?x14x14x64xi8>, tensor<384x1x1x64xi8>, tensor<384xi32>) -> tensor<?x14x14x384xi32>
+  %input_zp = "tosa.const"() {value = dense<-6> : tensor<1xi8>} : () -> tensor<1xi8>
+  %weight_zp = "tosa.const"() {value = dense<11> : tensor<1xi8>} : () -> tensor<1xi8>
+  %0 = tosa.conv2d %arg0, %arg1, %arg2, %input_zp, %weight_zp {acc_type = i32, dilation = array<i64: 1, 1>, pad = array<i64: 0, 0, 0, 0>, stride = array<i64: 1, 1>} : (tensor<?x14x14x64xi8>, tensor<384x1x1x64xi8>, tensor<384xi32>, tensor<1xi8>, tensor<1xi8>) -> tensor<?x14x14x384xi32>
   return %0 : tensor<?x14x14x384xi32>
 }
 
@@ -58,13 +68,19 @@ func.func @conv_with_dynamic_dim(%arg0: tensor<?x14x14x64xi8>, %arg1: tensor<384
 
 // CHECK-LABEL: @conv2d_as_fully_connected_padded
 func.func @conv2d_as_fully_connected_padded(%arg0: tensor<4x10x10x2xi8>, %arg1: tensor<3x1x1x2xi8>, %arg2: tensor<3xi32>) -> tensor<4x12x12x3xi32> {
+  // CHECK-DAG: %[[FULLY_NEW_SHAPE:.+]] = tosa.const_shape  {value = dense<[4, 12, 12, 3]> : tensor<4xindex>}
+  // CHECK-DAG: %[[INPUT_NEW_SHAPE:.+]] = tosa.const_shape  {value = dense<[576, 2]> : tensor<2xindex>}
+  // CHECK-DAG: %[[FILTER_NEW_SHAPE:.+]] = tosa.const_shape  {value = dense<[3, 2]> : tensor<2xindex>}
   // CHECK-DAG: %[[PAD_SHAPE:.+]] = tosa.const_shape  {value = dense<[0, 0, 1, 1, 1, 1, 0, 0]> : tensor<8xindex>} : () -> !tosa.shape<8>
   // CHECK-DAG: %[[PAD_VAL:.+]] = "tosa.const"() <{value = dense<42> : tensor<i8>}
   // CHECK-DAG: %[[PAD:.+]] = tosa.pad %arg0, %[[PAD_SHAPE]], %[[PAD_VAL]] : (tensor<4x10x10x2xi8>, !tosa.shape<8>, tensor<i8>) -> tensor<4x12x12x2xi8>
-  // CHECK-DAG: %[[RESHAPE_INPUT:.+]] = tosa.reshape %[[PAD]] {new_shape = array<i64: 576, 2>}
-  // CHECK-DAG: %[[RESHAPE_FILTER:.+]] = tosa.reshape %arg1 {new_shape = array<i64: 3, 2>}
-  // CHECK-DAG: %[[FULLY:.+]] = tosa.fully_connected %[[RESHAPE_INPUT]], %[[RESHAPE_FILTER]], %arg2 {quantization_info = #tosa.conv_quant<input_zp = 42, weight_zp = 24>}
-  // CHECK: %[[RESHAPE:.+]] = tosa.reshape %[[FULLY]] {new_shape = array<i64: 4, 12, 12, 3>}
-  %0 = tosa.conv2d %arg0, %arg1, %arg2 {acc_type = i32, pad = array<i64: 1, 1, 1, 1>, stride = array<i64: 1, 1>, dilation = array<i64: 1, 1>, quantization_info = #tosa.conv_quant<input_zp = 42, weight_zp = 24>} : (tensor<4x10x10x2xi8>, tensor<3x1x1x2xi8>, tensor<3xi32>) -> tensor<4x12x12x3xi32>
+  // CHECK-DAG: %[[RESHAPE_INPUT:.+]] = tosa.reshape %[[PAD]], %[[INPUT_NEW_SHAPE]]
+  // CHECK-DAG: %[[RESHAPE_FILTER:.+]] = tosa.reshape %arg1, %[[FILTER_NEW_SHAPE]]
+  // CHECK-DAG: %[[FULLY:.+]] = tosa.fully_connected %[[RESHAPE_INPUT]], %[[RESHAPE_FILTER]], %arg2 {input_zp = 42 : i32, weight_zp = 24 : i32}
+  // CHECK: %[[RESHAPE:.+]] = tosa.reshape %[[FULLY]], %[[FULLY_NEW_SHAPE]]
+  %input_zp = "tosa.const"() {value = dense<42> : tensor<1xi8>} : () -> tensor<1xi8>
+  %weight_zp = "tosa.const"() {value = dense<24> : tensor<1xi8>} : () -> tensor<1xi8>
+  %0 = tosa.conv2d %arg0, %arg1, %arg2, %input_zp, %weight_zp {acc_type = i32, pad = array<i64: 1, 1, 1, 1>, stride = array<i64: 1, 1>, dilation = array<i64: 1, 1>} : (tensor<4x10x10x2xi8>, tensor<3x1x1x2xi8>, tensor<3xi32>, tensor<1xi8>, tensor<1xi8>) -> tensor<4x12x12x3xi32>
   return %0 : tensor<4x12x12x3xi32>
 }
+
diff --git a/mlir/test/Dialect/Tosa/tosa-decompose-depthwise.mlir b/mlir/test/Dialect/Tosa/tosa-decompose-depthwise.mlir
index bbcc206e1490c..6562a7c2ab55c 100644
--- a/mlir/test/Dialect/Tosa/tosa-decompose-depthwise.mlir
+++ b/mlir/test/Dialect/Tosa/tosa-decompose-depthwise.mlir
@@ -5,15 +5,19 @@
 // CHECK-LABEL: @depthwise_conv2d_as_mul
 func.func @depthwise_conv2d_as_mul(%arg0: tensor<4x10x10x2xf32>, %arg1: tensor<1x1x2x3xf32>, %arg2: tensor<6xf32>) -> tensor<4x10x10x6xf32> {
   // CHECK-NOT: tosa.depthwise_conv2d
-  // CHECK: %[[VAR0:.*]] = tosa.reshape %arg0 {new_shape = array<i64: 4, 10, 10, 2, 1>}
+  // CHECK-DAG: %[[CONST0:.+]] = tosa.const_shape {value = dense<[4, 10, 10, 2, 1]> : tensor<5xindex>
+  // CHECK-DAG: %[[CONST1:.+]] = tosa.const_shape {value = dense<[1, 1, 1, 2, 3]> : tensor<5xindex>
+  // CHECK-DAG: %[[CONST2:.+]] = tosa.const_shape {value = dense<[4, 10, 10, 6]> : tensor<4xindex>
+  // CHECK-DAG: %[[CONST3:.+]] = tosa.const_shape {value = dense<[1, 1, 1, 6]> : tensor<4xindex>
+  // CHECK-DAG: %[[VAR0:.*]] = tosa.reshape %arg0, %[[CONST0]]
   // CHECK-SAME: -> tensor<4x10x10x2x1xf32>
-  // CHECK: %[[VAR1:.*]] = tosa.reshape %arg1 {new_shape = array<i64: 1, 1, 1, 2, 3>}
+  // CHECK: %[[VAR1:.*]] = tosa.reshape %arg1, %[[CONST1]]
   // CHECK-SAME: -> tensor<1x1x1x2x3xf32>
   // CHECK: %[[VAR2:.*]] = tosa.mul %[[VAR0]], %[[VAR1]]
   // CHECK-SAME: -> tensor<4x10x10x2x3xf32>
-  // CHECK: %[[VAR3:.*]] = tosa.reshape %[[VAR2]] {new_shape = array<i64: 4, 10, 10, 6>}
+  // CHECK: %[[VAR3:.*]] = tosa.reshape %[[VAR2]], %[[CONST2]]
   // CHECK-SAME: -> tensor<4x10x10x6xf32>
-  // CHECK: %[[VAR4:.*]] = tosa.reshape %arg2 {new_shape = array<i64: 1, 1, 1, 6>}
+  // CHECK: %[[VAR4:.*]] = tosa.reshape %arg2, %[[CONST3]]
   // CHECK-SAME: -> tensor<1x1x1x6xf32>
   // CHECK: %[[VAR5:.*]] = tosa.add %[[VAR3]], %[[VAR4]]
   // CHECK-SAME: -> tensor<4x10x10x6xf32>
@@ -26,19 +30,26 @@ func.func @depthwise_conv2d_as_mul(%arg0: tensor<4x10x10x2xf32>, %arg1: tensor<1
 
 // CHECK-LABEL: @depthwise_conv2d_as_mul_q
 func.func @depthwise_conv2d_as_mul_q(%arg0: tensor<4x10x10x2xi8>, %arg1: tensor<1x1x2x3xi8>, %arg2: tensor<6xi32>) -> tensor<4x10x10x6xi32> {
+  // CHECK-DAG: %[[CONST0:.+]] = tosa.const_shape {value = dense<[4, 10, 10, 2, 1]> : tensor<5xindex>
   // CHECK-DAG: %[[iZp:.+]] = "tosa.const"() <{value = dense<7> : tensor<1x1x1x1x1xi32>}
   // CHECK-DAG: %[[wZp:.+]] = "tosa.const"() <{value = dense<11> : tensor<1x1x1x1xi32>}
-  // CHECK: %[[rIn:.+]] = tosa.reshape %arg0 {new_shape = array<i64: 4, 10, 10, 2, 1>}
+  // CHECK-DAG: %[[CONST3:.+]] = tosa.const_shape {value = dense<[1, 1, 1, 2, 3]> : tensor<5xindex>
+  // CHECK-DAG: %[[CONST4:.+]] = tosa.const_shape {value = dense<[4, 10, 10, 6]> : tensor<4xindex>
+  // CHECK-DAG: %[[CONST5:.+]] = tosa.const_shape {value = dense<[1, 1, 1, 6]> : tensor<4xindex>
+  // CHECK-DAG: %[[SHIFT:.*]] = "tosa.const"() <{value = dense<0> : tensor<1xi8>}> : () -> tensor<1xi8>
+  // CHECK: %[[rIn:.+]] = tosa.reshape %arg0, %[[CONST0]]
   // CHECK: %[[cIn:.+]] = tosa.cast %[[rIn]] : (tensor<4x10x10x2x1xi8>) -> tensor<4x10x10x2x1xi32>
   // CHECK: %[[cWe:.+]] = tosa.cast %arg1 : (tensor<1x1x2x3xi8>) -> tensor<1x1x2x3xi32>
   // CHECK: %[[sIn:.+]] = tosa.sub %[[cIn]], %[[iZp]]
   // CHECK: %[[sWe:.+]] = tosa.sub %[[cWe]], %[[wZp]]
-  // CHECK: %[[resWe:.+]] = tosa.reshape %[[sWe]] {new_shape = array<i64: 1, 1, 1, 2, 3>}
-  // CHECK: %[[mul:.+]] = tosa.mul %[[sIn]], %[[resWe]] {shift = 0 : i8}
-  // CHECK: %[[reO:.+]] = tosa.reshape %[[mul]] {new_shape = array<i64: 4, 10, 10, 6>}
-  // CHECK: %[[reArg2:.+]] = tosa.reshape %arg2 {new_shape = array<i64: 1, 1, 1, 6>}
+  // CHECK: %[[resWe:.+]] = tosa.reshape %[[sWe]], %[[CONST3]]
+  // CHECK: %[[mul:.+]] = tosa.mul %[[sIn]], %[[resWe]], %[[SHIFT]]
+  // CHECK: %[[reO:.+]] = tosa.reshape %[[mul]], %[[CONST4]]
+  // CHECK: %[[reArg2:.+]] = tosa.reshape %arg2, %[[CONST5]]
   // CHECK: %[[add:.+]] = tosa.add %[[reO]], %[[reArg2]]
-  %0 = tosa.depthwise_conv2d %arg0, %arg1, %arg2 {acc_type = i32, pad = array<i64: 0, 0, 0, 0>, stride = array<i64: 1, 1>, dilation = array<i64: 1, 1>, quantization_info = #tosa.conv_quant<input_zp = 7, weight_zp = 11>} : (tensor<4x10x10x2xi8>, tensor<1x1x2x3xi8>, tensor<6xi32>) -> tensor<4x10x10x6xi32>
+  %input_zp = "tosa.const"() {value = dense<7> : tensor<1xi8>} : () -> tensor<1xi8>
+  %weight_zp = "tosa.const"() {value = dense<11> : tensor<1xi8>} : () -> tensor<1xi8>
+  %0 = tosa.depthwise_conv2d %arg0, %arg1, %arg2, %input_zp, %weight_zp {acc_type = i32, pad = array<i64: 0, 0, 0, 0>, stride = array<i64: 1, 1>, dilation = array<i64: 1, 1> } : (tensor<4x10x10x2xi8>, tensor<1x1x2x3xi8>, tensor<6xi32>, tensor<1xi8>, tensor<1xi8>) -> tensor<4x10x10x6xi32>
   return %0 : tensor<4x10x10x6xi32>
 }
 
@@ -46,14 +57,19 @@ func.func @depthwise_conv2d_as_mul_q(%arg0: tensor<4x10x10x2xi8>, %arg1: tensor<
 
 // CHECK-LABEL: @depthwise_conv2d_as_mul_padded
 func.func @depthwise_conv2d_as_mul_padded(%arg0: tensor<4x10x10x2xf32>, %arg1: tensor<1x1x2x3xf32>, %arg2: tensor<6xf32>) -> tensor<4x12x12x6xf32> {
-  // CHECK-DAG: %[[pad:.+]] = tosa.const_shape {value = dense<[0, 0, 1, 1, 1, 1, 0, 0, 0, 0]> : tensor<10xindex>} : () -> !tosa.shape<10>
+  // CHECK-DAG: %[[CONST0:.+]] = tosa.const_shape {value = dense<[4, 10, 10, 2, 1]> : tensor<5xindex>}
+  // CHECK-DAG: %[[pad:.+]] = tosa.const_shape  {value = dense<[0, 0, 1, 1, 1, 1, 0, 0, 0, 0]> : tensor<10xindex>} : () -> !tosa.shape<10>
   // CHECK-DAG: %[[zero:.+]] = "tosa.const"() <{value = dense<0.000000e+00> : tensor<f32>}
-  // CHECK: %[[reIn:.+]] = tosa.reshape %arg0 {new_shape = array<i64: 4, 10, 10, 2, 1>}
+  // CHECK-DAG: %[[CONST3:.+]] = tosa.const_shape {value = dense<[1, 1, 1, 2, 3]> : tensor<5xindex>}
+  // CHECK-DAG: %[[CONST4:.+]] = tosa.const_shape {value = dense<[4, 12, 12, 6]> : tensor<4xindex>}
+  // CHECK-DAG: %[[CONST5:.+]] = tosa.const_shape {value = dense<[1, 1, 1, 6]> : tensor<4xindex>}
+  // CHECK-DAG: %[[SHIFT:.*]] = "tosa.const"() <{value = dense<0> : tensor<1xi8>}> : () -> tensor<1xi8>
+  // CHECK: %[[reIn:.+]] = tosa.reshape %arg0, %[[CONST0]]
   // CHECK: %[[padded:.+]] = tosa.pad %[[reIn]], %[[pad]], %[[zero]] : (tensor<4x10x10x2x1xf32>, !tosa.shape<10>, tensor<f32>) -> tensor<4x12x12x2x1xf32>
-  // CHECK: %[[reArg1:.+]] = tosa.reshape %arg1 {new_shape = array<i64: 1, 1, 1, 2, 3>}
-  // CHECK: %[[mul:.+]] = tosa.mul %3, %[[reArg1]] {shift = 0 : i8}
-  // CHECK: %[[reOut:.+]] = tosa.reshape %[[mul]] {new_shape = array<i64: 4, 12, 12, 6>}
-  // CHECK: %[[reArg2:.+]] = tosa.reshape %arg2 {new_shape = array<i64: 1, 1, 1, 6>}
+  // CHECK: %[[reArg1:.+]] = tosa.reshape %arg1, %[[CONST3]]
+  // CHECK: %[[mul:.+]] = tosa.mul %[[padded]], %[[reArg1]], %[[SHIFT]]
+  // CHECK: %[[reOut:.+]] = tosa.reshape %[[mul]], %[[CONST4]]
+  // CHECK: %[[reArg2:.+]] = tosa.reshape %arg2, %[[CONST5]]
   // CHECK: %[[add:.+]] = tosa.add %[[reOut]], %[[reArg2]]
   %0 = tosa.depthwise_conv2d %arg0, %arg1, %arg2 {acc_type = f32, pad = array<i64: 1, 1, 1, 1>, stride = array<i64: 1, 1>, dilation = array<i64: 1, 1>} : (tensor<4x10x10x2xf32>, tensor<1x1x2x3xf32>, tensor<6xf32>) -> tensor<4x12x12x6xf32>
   return %0 : tensor<4x12x12x6xf32>
diff --git a/mlir/test/Dialect/Tosa/tosa-decompose-transpose-conv.mlir b/mlir/test/Dialect/Tosa/tosa-decompose-transpose-conv.mlir
index 96f71c349938b..bd18b7ea0fdff 100644
--- a/mlir/test/Dialect/Tosa/tosa-decompose-transpose-conv.mlir
+++ b/mlir/test/Dialect/Tosa/tosa-decompose-transpose-conv.mlir
@@ -6,7 +6,7 @@ func.func @transpose_conv2d(%arg0: tensor<2x16x14x3xf32>, %arg1: tensor<5x3x6x3x
   // CHECK: %[[REV2:.+]] = tosa.reverse %[[REV1]] {axis = 2 : i32}
   // CHECK: tosa.conv2d %arg0, %[[REV2]], %arg2
   // CHECK-SAME: dilation = array<i64: 1, 1>, pad = array<i64: 2, 2, 5, 5>, stride = array<i64: 1, 1>
-  %0 = tosa.transpose_conv2d %arg0, %arg1, %arg2 {acc_type = f32, out_pad = array<i64: 0, 0, 0, 0>, out_shape = array<i64: -1, -1, -1, -1>, stride = array<i64: 1, 1>} : (tensor<2x16x14x3xf32>, tensor<5x3x6x3xf32>, tensor<5xf32>) -> tensor<2x18x19x5xf32>
+  %0 = tosa.transpose_conv2d %arg0, %arg1, %arg2{acc_type = f32, out_pad = array<i64: 0, 0, 0, 0>, out_shape = array<i64: -1, -1, -1, -1>, stride = array<i64: 1, 1>} : (tensor<2x16x14x3xf32>, tensor<5x3x6x3xf32>, tensor<5xf32>) -> tensor<2x18x19x5xf32>
   return %0 : tensor<2x18x19x5xf32>
 }
 
@@ -15,10 +15,14 @@ func.func @transpose_conv2d(%arg0: tensor<2x16x14x3xf32>, %arg1: tensor<5x3x6x3x
 // CHECK-LABEL: @transpose_conv2d_quantized
 
 func.func @transpose_conv2d_quantized(%arg0: tensor<2x16x14x3xi8>, %arg1: tensor<5x3x6x3xi8>, %arg2: tensor<5xi32>) -> (tensor<2x18x19x5xi32>) {
+  // CHECK-DAG: %[[INPUT_ZP:.+]]  = "tosa.const"() <{value = dense<-6> : tensor<1xi8>}
+  // CHECK-DAG: %[[WEIGHT_ZP:.+]]  = "tosa.const"() <{value = dense<11> : tensor<1xi8>}
   // CHECK: %[[REV1:.+]] = tosa.reverse %arg1 {axis = 1 : i32}
   // CHECK: %[[REV2:.+]] = tosa.reverse %[[REV1]] {axis = 2 : i32}
-  // CHECK: tosa.conv2d %arg0, %[[REV2]], %arg2 {acc_type = i32, dilation = array<i64: 1, 1>, pad = array<i64: 2, 2, 5, 5>, quantization_info = #tosa.conv_quant<input_zp = -22, weight_zp = 42>, stride = array<i64: 1, 1>}
-  %0 = tosa.transpose_conv2d %arg0, %arg1, %arg2 {acc_type = i32, out_pad = array<i64: 0, 0, 0, 0>, quantization_info = #tosa.conv_quant<input_zp = -22, weight_zp = 42>, out_shape = array<i64: -1, -1, -1, -1>, stride = array<i64: 1, 1>} : (tensor<2x16x14x3xi8>, tensor<5x3x6x3xi8>, tensor<5xi32>) -> tensor<2x18x19x5xi32>
+  // CHECK: tosa.conv2d %arg0, %[[REV2]], %arg2, %[[INPUT_ZP]], %[[WEIGHT_ZP]] {acc_type = i32, dilation = array<i64: 1, 1>, pad = array<i64: 2, 2, 5, 5>, stride = array<i64: 1, 1>}
+  %input_zp = "tosa.const"() {value = dense<-6> : tensor<1xi8>} : () -> tensor<1xi8>
+  %weight_zp = "tosa.const"() {value = dense<11> : tensor<1xi8>} : () -> tensor<1xi8>
+  %0 = tosa.transpose_conv2d %arg0, %arg1, %arg2, %input_zp, %weight_zp {acc_type = i32, out_pad = array<i64: 0, 0, 0, 0>, out_shape = array<i64: -1, -1, -1, -1>, stride = array<i64: 1, 1>} : (tensor<2x16x14x3xi8>, tensor<5x3x6x3xi8>, tensor<5xi32>, tensor<1xi8>, tensor<1xi8>) -> tensor<2x18x19x5xi32>
   return %0 : tensor<2x18x19x5xi32>
 }
 
@@ -26,33 +30,41 @@ func.func @transpose_conv2d_quantized(%arg0: tensor<2x16x14x3xi8>, %arg1: tensor
 
 // CHECK-LABEL: @transpose_conv2d_quantized_padded
 func.func @transpose_conv2d_quantized_padded(%arg0: tensor<2x16x14x3xi8>, %arg1: tensor<5x3x6x3xi8>, %arg2: tensor<5xi32>) -> (tensor<2x21x26x5xi32>) {
-  // CHECK-DAG: %[[REV0:.+]] = tosa.reverse %0 {axis = 2 : i32}
+  // CHECK-DAG: %[[INPUT_ZP:.+]]  = "tosa.const"() <{value = dense<-22> : tensor<1xi8>}
+  // CHECK-DAG: %[[WEIGHT_ZP:.+]]  = "tosa.const"() <{value = dense<42> : tensor<1xi8>}
+  // CHECK-DAG: %[[REV0:.+]] = tosa.reverse %2 {axis = 2 : i32}
   // CHECK-DAG: %[[REV1:.+]] = tosa.reverse %arg1 {axis = 1 : i32}
-  // CHECK: tosa.conv2d %arg0, %1, %arg2
+  // CHECK: tosa.conv2d %arg0, %3, %arg2, %[[INPUT_ZP]], %[[WEIGHT_ZP]]
   // CHECK-SAME: dilation = array<i64: 1, 1>, pad = array<i64: 3, 4, 8, 9>,
-  // CHECK-SAME: quantization_info = #tosa.conv_quant<input_zp = -22, weight_zp = 42>, stride = array<i64: 1, 1>}
-  %0 = tosa.transpose_conv2d %arg0, %arg1, %arg2 {
+  // CHECK-SAME: stride = array<i64: 1, 1>}
+  %input_zp = "tosa.const"() {value = dense<-22> : tensor<1xi8>} : () -> tensor<1xi8>
+  %weight_zp = "tosa.const"() {value = dense<42> : tensor<1xi8>} : () -> tensor<1xi8>
+  %0 = tosa.transpose_conv2d %arg0, %arg1, %arg2, %input_zp, %weight_zp {
     acc_type = i32,
     out_pad = array<i64: 1, 2, 3, 4>,
-    quantization_info = #tosa.conv_quant<input_zp = -22, weight_zp = 42>,
     out_shape = array<i64: -1, -1, -1, -1>,
-    stride = array<i64: 1, 1>} : (tensor<2x16x14x3xi8>, tensor<5x3x6x3xi8>, tensor<5xi32>) -> tensor<2x21x26x5xi32>
+    stride = array<i64: 1, 1>} : (tensor<2x16x14x3xi8>, tensor<5x3x6x3xi8>, tensor<5xi32>, tensor<1xi8>, tensor<1xi8>) -> tensor<2x21x26x5xi32>
   return %0 : tensor<2x21x26x5xi32>
 }
 
 // -----
 
 // CHECK-LABEL: @transpose_conv2d_strided
+
 func.func @transpose_conv2d_strided(%arg0: tensor<2x17x15x3xf32>, %arg1: tensor<5x3x5x3xf32>, %arg2: tensor<5xf32>) -> tensor<2x?x?x5xf32> {
   // Manipulate the weight matrix to handle striding.
   // CHECK-DAG: %[[PADV:.+]] = tosa.const_shape {value = dense<[0, 0, 0, 1, 0, 1, 0, 0]> : tensor<8xindex>} : () -> !tosa.shape<8>
   // CHECK-DAG: %[[TRANSV:.+]]  = "tosa.const"() <{value = dense<[2, 4, 0, 1, 3, 5]> : tensor<6xi32>}
   // CHECK-DAG: %[[PADW:.+]]  = tosa.pad %arg1, %[[PADV]]
-  // CHECK-DAG: %[[RESW1:.+]]  = tosa.reshape %[[PADW]] {new_shape = array<i64: 5, 2, 2, 2, 3, 3>}
+  // CHECK-DAG: %[[CONST1:.+]] = tosa.const_shape {value = dense<[5, 2, 2, 2, 3, 3]> : tensor<6xindex>}
+  // CHECK-DAG: %[[RESW1:.+]]  = tosa.reshape %[[PADW]], %[[CONST1]]
   // CHECK-DAG: %[[TRANS:.+]]  = tosa.transpose %[[RESW1]], %[[TRANSV]]
-  // CHECK-DAG: %[[RESW2:.+]]  = tosa.reshape %[[TRANS]] {new_shape = array<i64: 30, 2, 2, 3>}
+  // CHECK-DAG: %[[CONST3:.+]] = tosa.const_shape {value = dense<[30, 2, 2, 3]> : tensor<4xindex>}
+  // CHECK-DAG: %[[RESW2:.+]]  = tosa.reshape %[[TRANS]], %[[CONST3]]
   // CHECK-DAG: %[[REV1:.+]]  = tosa.reverse %[[RESW2]] {axis = 1 : i32}
   // CHECK-DAG: %[[NEWWEIGHT:.+]] = tosa.reverse %[[REV1]] {axis = 2 : i32}
+  // CHECK-DAG: %[[SIZE:.*]] = tosa.const_shape  {value = dense<[2, 35, 47, 5]> : tensor<4xindex>} : () -> !tosa.shape<4>
+  // CHECK-DAG: %[[START:.*]] = tosa.const_shape  {value = dense<0> : tensor<4xindex>} : () -> !tosa.shape<4>
 
   // Pad out the input matrix to handle the transpose conv.
   // CHECK-DAG: %[[PAD:.+]] = tosa.const_shape {value = dense<[0, 0, 1, 1, 1, 1, 0, 0]> : tensor<8xindex>} : () -> !tosa.shape<8>
@@ -62,13 +74,16 @@ func.func @transpose_conv2d_strided(%arg0: tensor<2x17x15x3xf32>, %arg1: tensor<
   // Manipulate the final shape.
   // CHECK-DAG: %[[BIAS:.+]]  = "tosa.const"() <{value = dense<0.000000e+00> : tensor<30xf32>}
   // CHECK-DAG: %[[CONV:.+]] = tosa.conv2d %[[NEWINPUT]], %[[NEWWEIGHT]], %[[BIAS]] {acc_type = f32, dilation = array<i64: 1, 1>, pad = array<i64: 0, 0, 0, 0>, stride = array<i64: 1, 1>}
-  // CHECK-DAG: %[[RESHAPE_OUT_1:.+]] = tosa.reshape %[[CONV]] {new_shape = array<i64: 2, 18, 16, 2, 3, 5>}
+  // CHECK-DAG: %[[CONST6:.+]] = tosa.const_shape {value = dense<[2, 18, 16, 2, 3, 5]> : tensor<6xindex>}
+  // CHECK-DAG: %[[RESHAPE_OUT_1:.+]] = tosa.reshape %[[CONV]], %[[CONST6]]
   // CHECK-DAG: %[[TRANS_OUT:.+]] = tosa.transpose %[[RESHAPE_OUT_1]], %[[TRANS2]]
-  // CHECK-DAG: %[[RESHAPE_OUT_2:.+]] = tosa.reshape %[[TRANS_OUT]] {new_shape = array<i64: 2, 36, 48, 5>}
-  // CHECK-DAG: %[[SLICE:.+]] = tosa.slice %[[RESHAPE_OUT_2]] {size = array<i64: 2, 35, 47, 5>, start = array<i64: 0, 0, 0, 0>}
-  // CHECK-DAG: %[[RESHAPE_ARG2:.+]] = tosa.reshape %arg2 {new_shape = array<i64: 1, 1, 1, 5>}
+  // CHECK-DAG: %[[CONST8:.+]] = tosa.const_shape {value = dense<[2, 36, 48, 5]> : tensor<4xindex>
+  // CHECK-DAG: %[[RESHAPE_OUT_2:.+]] = tosa.reshape %[[TRANS_OUT]], %[[CONST8]]
+  // CHECK-DAG: %[[SLICE:.+]] = tosa.slice %[[RESHAPE_OUT_2]], %[[START]], %[[SIZE]]
+  // CHECK-DAG: %[[CONST9:.+]] = tosa.const_shape {value = dense<[1, 1, 1, 5]> : tensor<4xindex>}
+  // CHECK-DAG: %[[RESHAPE_ARG2:.+]] = tosa.reshape %arg2, %[[CONST9]]
   // CHECK: %[[ADD:.+]] = tosa.add %[[SLICE]], %[[RESHAPE_ARG2]]
-  %0 = tosa.transpose_conv2d %arg0, %arg1, %arg2 {acc_type = f32, out_pad = array<i64: 0, 0, 0, 0>, out_shape = array<i64: -1, -1, -1, -1>, stride = array<i64: 2, 3>} : (tensor<2x17x15x3xf32>, tensor<5x3x5x3xf32>, tensor<5xf32>) -> tensor<2x35x47x5xf32>
+  %0 = tosa.transpose_conv2d %arg0, %arg1, %arg2{acc_type = f32, out_pad = array<i64: 0, 0, 0, 0>, out_shape = array<i64: -1, -1, -1, -1>, stride = array<i64: 2, 3>} : (tensor<2x17x15x3xf32>, tensor<5x3x5x3xf32>, tensor<5xf32>) -> tensor<2x35x47x5xf32>
   %1 = tensor.cast %0 : tensor<2x35x47x5xf32> to tensor<2x?x?x5xf32>
   return %1 : tensor<2x?x?x5xf32>
 }
@@ -76,32 +91,44 @@ func.func @transpose_conv2d_strided(%arg0: tensor<2x17x15x3xf32>, %arg1: tensor<
 // -----
 
 // CHECK-LABEL: @transpose_conv2d_strided_quantized
+
 func.func @transpose_conv2d_strided_quantized(%arg0: tensor<2x17x15x3xi8>, %arg1: tensor<5x3x5x3xi8>, %arg2: tensor<5xi32>) -> (tensor<2x35x47x5xi32>) {
   // Manipulate the weight matrix to handle striding.
   // CHECK-DAG: %[[PADV:.+]]  = tosa.const_shape {value = dense<[0, 0, 0, 1, 0, 1, 0, 0]> : tensor<8xindex>} : () -> !tosa.shape<8>
   // CHECK-DAG: %[[TRANSV:.+]]  = "tosa.const"() <{value = dense<[2, 4, 0, 1, 3, 5]> : tensor<6xi32>}
-  // CHECK-DAG: %[[PADW:.+]]  = tosa.pad %arg1, %[[PADV]] {quantization_info = #tosa.pad_quant<input_zp = 42>}
-  // CHECK-DAG: %[[RESW1:.+]]  = tosa.reshape %[[PADW]] {new_shape = array<i64: 5, 2, 2, 2, 3, 3>}
+  // CHECK-DAG: %[[PADW:.+]]  = tosa.pad %arg1, %[[PADV]] {input_zp = 42 : i32}
+  // CHECK-DAG: %[[CONST1:.+]] = tosa.const_shape {value = dense<[5, 2, 2, 2, 3, 3]> : tensor<6xindex>}
+  // CHECK-DAG: %[[RESW1:.+]]  = tosa.reshape %[[PADW]], %[[CONST1]]
   // CHECK-DAG: %[[TRANS:.+]]  = tosa.transpose %[[RESW1]], %[[TRANSV]]
-  // CHECK-DAG: %[[RESW2:.+]]  = tosa.reshape %[[TRANS]] {new_shape = array<i64: 30, 2, 2, 3>}
+  // CHECK-DAG: %[[CONST3:.+]] = tosa.const_shape {value = dense<[30, 2, 2, 3]> : tensor<4xindex>}
+  // CHECK-DAG: %[[RESW2:.+]]  = tosa.reshape %[[TRANS]], %[[CONST3]]
   // CHECK-DAG: %[[REV1:.+]]  = tosa.reverse %[[RESW2]] {axis = 1 : i32}
   // CHECK-DAG: %[[NEWWEIGHT:.+]] = tosa.reverse %[[REV1]] {axis = 2 : i32}
+  // CHECK-DAG: %[[SIZE:.*]] = tosa.const_shape  {value = dense<[2, 35, 47, 5]> : tensor<4xindex>} : () -> !tosa.shape<4>
+  // CHECK-DAG: %[[START:.*]] = tosa.const_shape  {value = dense<0> : tensor<4xindex>} : () -> !tosa.shape<4>
 
   // Pad out the input matrix to handle the transpose conv.
   // CHECK-DAG: %[[PAD:.+]]  = tosa.const_shape {value = dense<[0, 0, 1, 1, 1, 1, 0, 0]> : tensor<8xindex>} : () -> !tosa.shape<8>
   // CHECK-DAG: %[[TRANS2:.+]]  = "tosa.const"() <{value = dense<[0, 1, 3, 2, 4, 5]> : tensor<6xi32>}
-  // CHECK-DAG: %[[NEWINPUT:.+]] = tosa.pad %arg0, %[[PAD]] {quantization_info = #tosa.pad_quant<input_zp = -22>}
+  // CHECK-DAG: %[[NEWINPUT:.+]] = tosa.pad %arg0, %[[PAD]] {input_zp = -22 : i32}
 
   // Manipulate the final shape.
   // CHECK-DAG: %[[BIAS:.+]]  = "tosa.const"() <{value = dense<0> : tensor<30xi32>}
-  // CHECK-DAG: %[[CONV:.+]] = tosa.conv2d %[[NEWINPUT]], %[[NEWWEIGHT]], %[[BIAS]] {acc_type = i32, dilation = array<i64: 1, 1>, pad = array<i64: 0, 0, 0, 0>, quantization_info = #tosa.conv_quant<input_zp = -22, weight_zp = 42>, stride = array<i64: 1, 1>}
-  // CHECK-DAG: %[[RESHAPE_OUT_1:.+]] = tosa.reshape %[[CONV]] {new_shape = array<i64: 2, 18, 16, 2, 3, 5>}
+  // CHECK-DAG: %[[INPUT_ZP:.+]]  = "tosa.const"() <{value = dense<-22> : tensor<1xi8>}
+  // CHECK-DAG: %[[WEIGHT_ZP:.+]]  = "tosa.const"() <{value = dense<42> : tensor<1xi8>}
+  // CHECK-DAG: %[[CONV:.+]] = tosa.conv2d %[[NEWINPUT]], %[[NEWWEIGHT]], %[[BIAS]], %[[INPUT_ZP]], %[[WEIGHT_ZP]] {acc_type = i32, dilation = array<i64: 1, 1>, pad = array<i64: 0, 0, 0, 0>, stride = array<i64: 1, 1>}
+  // CHECK-DAG: %[[CONV_NEW_SHAPE:.*]] = tosa.const_shape  {value = dense<[2, 18, 16, 2, 3, 5]> : tensor<6xindex>}
+  // CHECK-DAG: %[[RESHAPE_OUT_1:.+]] = tosa.reshape %[[CONV]], %[[CONV_NEW_SHAPE]]
   // CHECK-DAG: %[[TRANS_OUT:.+]] = tosa.transpose %[[RESHAPE_OUT_1]], %[[TRANS2]]
-  // CHECK-DAG: %[[RESHAPE_OUT_2:.+]] = tosa.reshape %[[TRANS_OUT]] {new_shape = array<i64: 2, 36, 48, 5>}
-  // CHECK-DAG: %[[SLICE:.+]] = tosa.slice %[[RESHAPE_OUT_2]] {size = array<i64: 2, 35, 47, 5>, start = array<i64: 0, 0, 0, 0>}
-  // CHECK-DAG: %[[RESHAPE_ARG2:.+]] = tosa.reshape %arg2 {new_shape = array<i64: 1, 1, 1, 5>}
+  // CHECK-DAG: %[[TEANS_NEW_SHAPE:.+]] = tosa.const_shape {value = dense<[2, 36, 48, 5]> : tensor<4xindex>}
+  // CHECK-DAG: %[[RESHAPE_OUT_2:.+]] = tosa.reshape %[[TRANS_OUT]], %[[TEANS_NEW_SHAPE]]
+  // CHECK-DAG: %[[SLICE:.+]] = tosa.slice %[[RESHAPE_OUT_2]], %[[START]], %[[SIZE]]
+  // CHECK-DAG: %[[ARG2_NEW_SHAPE:.+]] = tosa.const_shape {value = dense<[1, 1, 1, 5]> : tensor<4xindex>}
+  // CHECK-DAG: %[[RESHAPE_ARG2:.+]] = tosa.reshape %arg2, %[[ARG2_NEW_SHAPE]]
   // CHECK: %[[ADD:.+]] = tosa.add %[[SLICE]], %[[RESHAPE_ARG2]]
-  %0 = tosa.transpose_conv2d %arg0, %arg1, %arg2 {acc_type = i32, out_pad = array<i64: 0, 0, 0, 0>, quantization_info = #tosa.conv_quant<input_zp = -22, weight_zp = 42>, out_shape = array<i64: -1, -1, -1, -1>, stride = array<i64: 2, 3>} : (tensor<2x17x15x3xi8>, tensor<5x3x5x3xi8>, tensor<5xi32>) -> tensor<2x35x47x5xi32>
+  %input_zp = "tosa.const"() {value = dense<-22> : tensor<1xi8>} : () -> tensor<1xi8>
+  %weight_zp = "tosa.const"() {value = dense<42> : tensor<1xi8>} : () -> tensor<1xi8>
+  %0 = tosa.transpose_conv2d %arg0, %arg1, %arg2, %input_zp, %weight_zp {acc_type = i32, out_pad = array<i64: 0, 0, 0, 0>, out_shape = array<i64: -1, -1, -1, -1>, stride = array<i64: 2, 3>} : (tensor<2x17x15x3xi8>, tensor<5x3x5x3xi8>, tensor<5xi32>, tensor<1xi8>, tensor<1xi8>) -> tensor<2x35x47x5xi32>
   return %0 : tensor<2x35x47x5xi32>
 }
 
@@ -109,32 +136,39 @@ func.func @transpose_conv2d_strided_quantized(%arg0: tensor<2x17x15x3xi8>, %arg1
 
 // CHECK-LABEL: @transpose_conv2d_strided_overpad
 func.func @transpose_conv2d_strided_overpad(%arg0 : tensor<1x16x1x1xi8>, %arg1 : tensor<1x2x1x1xi8>, %arg2 : tensor<1xi32>) -> (tensor<1x19x2x1xi32>) {
-  // CHECK-DAG: %[[WEIGHT_PAD:.+]] = tosa.const_shape  {value = dense<[0, 0, 0, 0, 0, 1, 0, 0]> : tensor<8xindex>} : () -> !tosa.shape<8>
+  // CHECK-DAG: %[[WEIGHT_PAD:.+]] = tosa.const_shape  {value = dense<[0, 0, 0, 0, 0, 1, 0, 0]> : tensor<8xindex>}
+  // CHECK-DAG: %[[CONST1:.+]] = tosa.const_shape {value = dense<[1, 2, 1, 1, 2, 1]> : tensor<6xindex>}
   // CHECK-DAG: %[[WEIGHT_PERMS:.+]] = "tosa.const"() <{value = dense<[2, 4, 0, 1, 3, 5]> : tensor<6xi32>}
-  // CHECK-DAG: %[[INPUT_PAD:.+]] = tosa.const_shape  {value = dense<[0, 0, 1, 1, 0, 0, 0, 0]> : tensor<8xindex>} : () -> !tosa.shape<8>
+  // CHECK-DAG: %[[CONST3:.+]] = tosa.const_shape {value = dense<[2, 2, 1, 1]> : tensor<4xindex>}
+  // CHECK-DAG: %[[INPUT_PAD:.+]] = tosa.const_shape  {value = dense<[0, 0, 1, 1, 0, 0, 0, 0]> : tensor<8xindex>}
   // CHECK-DAG: %[[ZERO:.+]] = "tosa.const"() <{value = dense<0> : tensor<2xi32>}
+  // CHECK-DAG: %[[CONST6:.+]] = tosa.const_shape {value = dense<[1, 17, 1, 1, 2, 1]> : tensor<6xindex>}
   // CHECK-DAG: %[[RESULT_PERMS:.+]] = "tosa.const"() <{value = dense<[0, 1, 3, 2, 4, 5]> : tensor<6xi32>}
-  // CHECK-DAG: %[[RESULT_PAD:.+]] = tosa.const_shape  {value = dense<[0, 0, 2, 0, 0, 0, 0, 0]> : tensor<8xindex>} : () -> !tosa.shape<8>
-  // CHECK: %[[PAD_WEIGHT:.+]] = tosa.pad %arg1, %[[WEIGHT_PAD]] {quantization_info = #tosa.pad_quant<input_zp = 93>}
-  // CHECK: %[[RESHAPE_WEIGHT_0:.+]] = tosa.reshape %[[PAD_WEIGHT]] {new_shape = array<i64: 1, 2, 1, 1, 2, 1>}
+  // CHECK-DAG: %[[CONST8:.+]] = tosa.const_shape {value = dense<[1, 17, 2, 1]> : tensor<4xindex>}
+  // CHECK-DAG: %[[RESULT_PAD:.+]] = tosa.const_shape  {value = dense<[0, 0, 2, 0, 0, 0, 0, 0]> : tensor<8xindex>}
+  // CHECK-DAG: %[[CONST10:.+]] = tosa.const_shape {value = dense<1> : tensor<4xindex>}
+  // CHECK-DAG: %[[INPUT_ZP:.*]] = "tosa.const"() <{value = dense<-103> : tensor<1xi8>}>
+  // CHECK-DAG: %[[WEIGHT_ZP:.*]] = "tosa.const"() <{value = dense<93> : tensor<1xi8>}>
+  // CHECK: %[[PAD_WEIGHT:.+]] = tosa.pad %arg1, %[[WEIGHT_PAD]] {input_zp = 93 : i32}
+  // CHECK: %[[RESHAPE_WEIGHT_0:.+]] = tosa.reshape %[[PAD_WEIGHT]], %[[CONST1]]
   // CHECK: %[[TRANSPOSE_WEIGHT:.+]] = tosa.transpose %[[RESHAPE_WEIGHT_0]], %[[WEIGHT_PERMS]]
-  // CHECK: %[[RESHAPE_WEIGHT_1:.+]] = tosa.reshape %[[TRANSPOSE_WEIGHT]] {new_shape = array<i64: 2, 2, 1, 1>}
+  // CHECK: %[[RESHAPE_WEIGHT_1:.+]] = tosa.reshape %[[TRANSPOSE_WEIGHT]], %[[CONST3]]
   // CHECK: %[[REVERSE:.+]] = tosa.reverse %[[RESHAPE_WEIGHT_1]] {axis = 1 : i32}
-  // CHECK: %[[PAD_INPUT:.+]] = tosa.pad %arg0, %[[INPUT_PAD]] {quantization_info = #tosa.pad_quant<input_zp = -103>}
-  // CHECK: %[[CONV:.+]] = tosa.conv2d %[[PAD_INPUT]], %[[REVERSE]], %[[ZERO]]
-  // CHECK-SAME{literal}: dilation = [1, 1], pad = [0, 0, 0, 0], quantization_info = #tosa.conv_quant<input_zp = -103, weight_zp = 93>, stride = [1, 1]}
-  // CHECK: %[[RESHAPE_RESULT_0:.+]] = tosa.reshape %[[CONV]] {new_shape = array<i64: 1, 17, 1, 1, 2, 1>}
+  // CHECK: %[[PAD_INPUT:.+]] = tosa.pad %arg0, %[[INPUT_PAD]] {input_zp = -103 : i32}
+  // CHECK: %[[CONV:.+]] = tosa.conv2d %[[PAD_INPUT]], %[[REVERSE]], %[[ZERO]], %[[INPUT_ZP]], %[[WEIGHT_ZP]] {acc_type = i32, dilation = array<i64: 1, 1>, pad = array<i64: 0, 0, 0, 0>, stride = array<i64: 1, 1>}
+  // CHECK: %[[RESHAPE_RESULT_0:.+]] = tosa.reshape %[[CONV]], %[[CONST6]]
   // CHECK: %[[TRANSPOSE_RESULT:.+]] = tosa.transpose %[[RESHAPE_RESULT_0]], %[[RESULT_PERMS]]
-  // CHECK: %[[RESHAPE_RESULT_1:.+]] = tosa.reshape %[[TRANSPOSE_RESULT]] {new_shape = array<i64: 1, 17, 2, 1>}
+  // CHECK: %[[RESHAPE_RESULT_1:.+]] = tosa.reshape %[[TRANSPOSE_RESULT]], %[[CONST8]]
   // CHECK: %[[PAD_RESULT:.+]] = tosa.pad %[[RESHAPE_RESULT_1]], %[[RESULT_PAD]]
-  // CHECK: %[[RESHAPE_ARG2:.+]] = tosa.reshape %arg2 {new_shape = array<i64: 1, 1, 1, 1>}
+  // CHECK: %[[RESHAPE_ARG2:.+]] = tosa.reshape %arg2, %[[CONST10]]
   // CHECK: %[[ADD:.+]] = tosa.add %[[PAD_RESULT]], %[[RESHAPE_ARG2]]
-  %2 =  tosa.transpose_conv2d %arg0, %arg1, %arg2 {
+  %input_zp = "tosa.const"() {value = dense<-103> : tensor<1xi8>} : () -> tensor<1xi8>
+  %weight_zp = "tosa.const"() {value = dense<93> : tensor<1xi8>} : () -> tensor<1xi8>
+  %2 =  tosa.transpose_conv2d %arg0, %arg1, %arg2, %input_zp, %weight_zp {
     acc_type = i32,
     out_pad = array<i64: 2, 0, 0, 1>,
     out_shape = array<i64: 1, -1, -1, 1>,
-    stride = array<i64: 1, 2>,
-    quantization_info = #tosa.conv_quant<input_zp = -103, weight_zp = 93>} :
-    (tensor<1x16x1x1xi8>, tensor<1x2x1x1xi8>, tensor<1xi32>) -> tensor<1x19x2x1xi32>
+    stride = array<i64: 1, 2>} :
+    (tensor<1x16x1x1xi8>, tensor<1x2x1x1xi8>, tensor<1xi32>, tensor<1xi8>, tensor<1xi8>) -> tensor<1x19x2x1xi32>
   "func.return" (%2) : (tensor<1x19x2x1xi32>) -> ()
 }
diff --git a/mlir/test/Dialect/Tosa/tosa-infer-shapes.mlir b/mlir/test/Dialect/Tosa/tosa-infer-shapes.mlir
index 6beb1ad629613..7dc9b048085fa 100644
--- a/mlir/test/Dialect/Tosa/tosa-infer-shapes.mlir
+++ b/mlir/test/Dialect/Tosa/tosa-infer-shapes.mlir
@@ -34,7 +34,7 @@ func.func @test_unary_f32(%arg0 : tensor<4xf32>) -> () {
   %1 = tosa.ceil %arg0 : (tensor<4xf32>) -> tensor<*xf32>
 
   // CHECK: tosa.clamp %arg0 {{.+}} : (tensor<4xf32>) -> tensor<4xf32>
-  %2 = tosa.clamp %arg0 { max_int = 10 : i64, min_int = 0 : i64, min_fp = 0.0 : f32, max_fp = 10.0 : f32 } : (tensor<4xf32>) -> tensor<*xf32>
+  %2 = tosa.clamp %arg0 { min_val = 0.0 : f32, max_val = 10.0 : f32 } : (tensor<4xf32>) -> tensor<*xf32>
 
   // CHECK: tosa.exp %arg0 : (tensor<4xf32>) -> tensor<4xf32>
   %3 = tosa.exp %arg0 : (tensor<4xf32>) -> tensor<*xf32>
@@ -82,7 +82,7 @@ func.func @test_unary_i32(%arg0 : tensor<4xi32>) -> () {
   %1 = tosa.bitwise_not %arg0 : (tensor<4xi32>) -> tensor<*xi32>
 
   // CHECK: tosa.clamp %arg0 {{.+}} : (tensor<4xi32>) -> tensor<4xi32>
-  %2 = tosa.clamp %arg0 { max_int = 10 : i64, min_int = 0 : i64, min_fp = 0.0 : f32, max_fp = 10.0 : f32 } : (tensor<4xi32>) -> tensor<*xi32>
+  %2 = tosa.clamp %arg0 { max_val = 10 : i32, min_val = 0 : i32} : (tensor<4xi32>) -> tensor<*xi32>
 
   // CHECK: tosa.clz %arg0 : (tensor<4xi32>) -> tensor<4xi32>
   %3 = tosa.clz %arg0 : (tensor<4xi32>) -> tensor<*xi32>
@@ -114,23 +114,24 @@ func.func @test_binary_scalar_f32(%arg0 : tensor<4xf32>, %arg1 : tensor<1xf32>)
   // CHECK: tosa.minimum %arg0, %arg1 : (tensor<4xf32>, tensor<1xf32>) -> tensor<4xf32>
   %2 = tosa.minimum %arg0, %arg1 : (tensor<4xf32>, tensor<1xf32>) -> tensor<*xf32>
 
-  // CHECK: tosa.mul %arg0, %arg1 {shift = 0 : i8} : (tensor<4xf32>, tensor<1xf32>) -> tensor<4xf32>
-  %3 = tosa.mul %arg0, %arg1 { shift = 0 : i8 } : (tensor<4xf32>, tensor<1xf32>) -> tensor<*xf32>
+  %3 = "tosa.const"() <{value = dense<0> : tensor<1xi8>}> : () -> tensor<1xi8>
+  // CHECK: tosa.mul %arg0, %arg1, %3 : (tensor<4xf32>, tensor<1xf32>, tensor<1xi8>) -> tensor<4xf32>
+  %4 = tosa.mul %arg0, %arg1, %3 : (tensor<4xf32>, tensor<1xf32>, tensor<1xi8>) -> tensor<*xf32>
 
   // CHECK: tosa.pow %arg0, %arg1 : (tensor<4xf32>, tensor<1xf32>) -> tensor<4xf32>
-  %4 = tosa.pow %arg0, %arg1 : (tensor<4xf32>, tensor<1xf32>) -> tensor<*xf32>
+  %5 = tosa.pow %arg0, %arg1 : (tensor<4xf32>, tensor<1xf32>) -> tensor<*xf32>
 
   // CHECK: tosa.sub %arg0, %arg1 : (tensor<4xf32>, tensor<1xf32>) -> tensor<4xf32>
-  %5 = tosa.sub %arg0, %arg1 : (tensor<4xf32>, tensor<1xf32>) -> tensor<*xf32>
+  %6 = tosa.sub %arg0, %arg1 : (tensor<4xf32>, tensor<1xf32>) -> tensor<*xf32>
 
   // CHECK: tosa.equal %arg0, %arg1 : (tensor<4xf32>, tensor<1xf32>) -> tensor<4xi1>
-  %6 = tosa.equal %arg0, %arg1 : (tensor<4xf32>, tensor<1xf32>) -> tensor<*xi1>
+  %7 = tosa.equal %arg0, %arg1 : (tensor<4xf32>, tensor<1xf32>) -> tensor<*xi1>
 
   // CHECK: tosa.greater %arg0, %arg1 : (tensor<4xf32>, tensor<1xf32>) -> tensor<4xi1>
-  %7 = tosa.greater %arg0, %arg1 : (tensor<4xf32>, tensor<1xf32>) -> tensor<*xi1>
+  %8 = tosa.greater %arg0, %arg1 : (tensor<4xf32>, tensor<1xf32>) -> tensor<*xi1>
 
   // CHECK: tosa.greater_equal %arg0, %arg1 : (tensor<4xf32>, tensor<1xf32>) -> tensor<4xi1>
-  %8 = tosa.greater_equal %arg0, %arg1 : (tensor<4xf32>, tensor<1xf32>) -> tensor<*xi1>
+  %9 = tosa.greater_equal %arg0, %arg1 : (tensor<4xf32>, tensor<1xf32>) -> tensor<*xi1>
 
   return
 }
@@ -148,23 +149,24 @@ func.func @test_binary_broadcast_f32(%arg0 : tensor<4xf32>, %arg1 : tensor<1xf32
   // CHECK: tosa.minimum %arg0, %arg1 : (tensor<4xf32>, tensor<1xf32>) -> tensor<4xf32>
   %2 = tosa.minimum %arg0, %arg1 : (tensor<4xf32>, tensor<1xf32>) -> tensor<*xf32>
 
-  // CHECK: tosa.mul %arg0, %arg1 {shift = 0 : i8} : (tensor<4xf32>, tensor<1xf32>) -> tensor<4xf32>
-  %3 = tosa.mul %arg0, %arg1 { shift = 0 : i8 } : (tensor<4xf32>, tensor<1xf32>) -> tensor<*xf32>
+  %3 = "tosa.const"() <{value = dense<0> : tensor<1xi8>}> : () -> tensor<1xi8>
+  // CHECK: tosa.mul %arg0, %arg1, %3 : (tensor<4xf32>, tensor<1xf32>, tensor<1xi8>) -> tensor<4xf32>
+  %4 = tosa.mul %arg0, %arg1, %3 : (tensor<4xf32>, tensor<1xf32>, tensor<1xi8>) -> tensor<*xf32>
 
   // CHECK: tosa.pow %arg0, %arg1 : (tensor<4xf32>, tensor<1xf32>) -> tensor<4xf32>
-  %4 = tosa.pow %arg0, %arg1 : (tensor<4xf32>, tensor<1xf32>) -> tensor<*xf32>
+  %5 = tosa.pow %arg0, %arg1 : (tensor<4xf32>, tensor<1xf32>) -> tensor<*xf32>
 
   // CHECK: tosa.sub %arg0, %arg1 : (tensor<4xf32>, tensor<1xf32>) -> tensor<4xf32>
-  %5 = tosa.sub %arg0, %arg1 : (tensor<4xf32>, tensor<1xf32>) -> tensor<*xf32>
+  %6 = tosa.sub %arg0, %arg1 : (tensor<4xf32>, tensor<1xf32>) -> tensor<*xf32>
 
   // CHECK: tosa.equal %arg0, %arg1 : (tensor<4xf32>, tensor<1xf32>) -> tensor<4xi1>
-  %6 = tosa.equal %arg0, %arg1 : (tensor<4xf32>, tensor<1xf32>) -> tensor<*xi1>
+  %7 = tosa.equal %arg0, %arg1 : (tensor<4xf32>, tensor<1xf32>) -> tensor<*xi1>
 
   // CHECK: tosa.greater %arg0, %arg1 : (tensor<4xf32>, tensor<1xf32>) -> tensor<4xi1>
-  %7 = tosa.greater %arg0, %arg1 : (tensor<4xf32>, tensor<1xf32>) -> tensor<*xi1>
+  %8 = tosa.greater %arg0, %arg1 : (tensor<4xf32>, tensor<1xf32>) -> tensor<*xi1>
 
   // CHECK: tosa.greater_equal %arg0, %arg1 : (tensor<4xf32>, tensor<1xf32>) -> tensor<4xi1>
-  %8 = tosa.greater_equal %arg0, %arg1 : (tensor<4xf32>, tensor<1xf32>) -> tensor<*xi1>
+  %9 = tosa.greater_equal %arg0, %arg1 : (tensor<4xf32>, tensor<1xf32>) -> tensor<*xi1>
 
   return
 }
@@ -206,14 +208,15 @@ func.func @test_binary_i32(%arg0 : tensor<4xi32>, %arg1 : tensor<1xi32>) -> () {
   // CHECK: tosa.minimum %arg0, %arg1 : (tensor<4xi32>, tensor<1xi32>) -> tensor<4xi32>
   %10 = tosa.minimum %arg0, %arg1 : (tensor<4xi32>, tensor<1xi32>) -> tensor<*xi32>
 
-  // CHECK: tosa.mul %arg0, %arg1 {shift = 0 : i8} : (tensor<4xi32>, tensor<1xi32>) -> tensor<4xi32>
-  %11 = tosa.mul %arg0, %arg1 { shift = 0 : i8 }: (tensor<4xi32>, tensor<1xi32>) -> tensor<*xi32>
+  // CHECK: tosa.mul %arg0, %arg1, %{{.*}} : (tensor<4xi32>, tensor<1xi32>, tensor<1xi8>) -> tensor<4xi32>
+  %shift = "tosa.const"() <{value = dense<0> : tensor<1xi8>}> : () -> tensor<1xi8>
+  %11 = tosa.mul %arg0, %arg1, %shift : (tensor<4xi32>, tensor<1xi32>, tensor<1xi8>) -> tensor<*xi32>
 
   // CHECK: tosa.pow %arg0, %arg1 : (tensor<4xi32>, tensor<1xi32>) -> tensor<4xi32>
-  %12 = tosa.pow %arg0, %arg1 : (tensor<4xi32>, tensor<1xi32>) -> tensor<*xi32>
+  %13 = tosa.pow %arg0, %arg1 : (tensor<4xi32>, tensor<1xi32>) -> tensor<*xi32>
 
   // CHECK:  tosa.sub %arg0, %arg1 : (tensor<4xi32>, tensor<1xi32>) -> tensor<4xi32>
-  %13 = tosa.sub %arg0, %arg1 : (tensor<4xi32>, tensor<1xi32>) -> tensor<*xi32>
+  %14 = tosa.sub %arg0, %arg1 : (tensor<4xi32>, tensor<1xi32>) -> tensor<*xi32>
 
   return
 }
@@ -375,29 +378,42 @@ func.func @test_table_dynamic(%arg0 : tensor<4x?xi16>, %arg1 : tensor<513xi16>)
 
 // CHECK-LABEL: @test_static_reshape
 func.func @test_static_reshape(%arg0 : tensor<4x4xi32>) -> () {
-  // CHECK: tosa.reshape %arg0 {new_shape = array<i64: 16>} : (tensor<4x4xi32>) -> tensor<16xi32>
-  %0 = tosa.reshape %arg0 {new_shape = array<i64: 16>} : (tensor<4x4xi32>) -> tensor<?xi32>
+  // CHECK: %[[CONST3:.+]] = tosa.const_shape {value = dense<16> : tensor<1xindex>} : () -> !tosa.shape<1>
+  %3 = tosa.const_shape {value = dense<16> : tensor<1xindex>} : () -> !tosa.shape<1>
+  // CHECK: tosa.reshape %arg0, %[[CONST3]] : (tensor<4x4xi32>, !tosa.shape<1>) -> tensor<16xi32>
+  %0 = tosa.reshape %arg0, %3 : (tensor<4x4xi32>, !tosa.shape<1>) -> tensor<16xi32>
 
-  // CHECK: tosa.reshape %arg0 {new_shape = array<i64: -1>} : (tensor<4x4xi32>) -> tensor<16xi32>
-  %1 = tosa.reshape %arg0 {new_shape = array<i64: -1>} : (tensor<4x4xi32>) -> tensor<?xi32>
+  // CHECK: %[[CONST4:.+]] = tosa.const_shape {value = dense<-1> : tensor<1xindex>} : () -> !tosa.shape<1>
+  // CHECK: tosa.reshape %arg0, %[[CONST4]] : (tensor<4x4xi32>, !tosa.shape<1>) -> tensor<16xi32>
+  %4 = tosa.const_shape {value = dense<-1> : tensor<1xindex>} : () -> !tosa.shape<1>
+  %1 = tosa.reshape %arg0, %4 : (tensor<4x4xi32>, !tosa.shape<1>) -> tensor<16xi32>
 
-  // CHECK: tosa.reshape %arg0 {new_shape = array<i64: 2, -1>} : (tensor<4x4xi32>) -> tensor<2x8xi32>
-  %2 = tosa.reshape %arg0 {new_shape = array<i64: 2, -1>} : (tensor<4x4xi32>) -> tensor<?x?xi32>
+  // CHECK: %[[CONST5:.+]] = tosa.const_shape {value = dense<[2, -1]> : tensor<2xindex>} : () -> !tosa.shape<2>
+  // CHECK: tosa.reshape %arg0, %[[CONST5]] : (tensor<4x4xi32>, !tosa.shape<2>) -> tensor<2x8xi32>
+  %5 = tosa.const_shape {value = dense<[2, -1]> : tensor<2xindex>} : () -> !tosa.shape<2>
+  %2 = tosa.reshape %arg0, %5 : (tensor<4x4xi32>, !tosa.shape<2>) -> tensor<2x8xi32>
 
   return
 }
+
 // -----
 
 // CHECK-LABEL: @test_dynamic_reshape
 func.func @test_dynamic_reshape(%arg0 : tensor<4x?xi32>) -> () {
-  // CHECK: %0 = tosa.reshape %arg0 {new_shape = array<i64: 16>} : (tensor<4x?xi32>) -> tensor<16xi32>
-  %0 = tosa.reshape %arg0 {new_shape = array<i64: 16>} : (tensor<4x?xi32>) -> tensor<?xi32>
+  // CHECK: %0 = tosa.const_shape {value = dense<16> : tensor<1xindex>} : () -> !tosa.shape<1>
+  %0 = tosa.const_shape {value = dense<16> : tensor<1xindex>} : () -> !tosa.shape<1>
+  // CHECK: %1 = tosa.reshape %arg0, %0 : (tensor<4x?xi32>, !tosa.shape<1>) -> tensor<16xi32>
+  %1 = tosa.reshape %arg0, %0 : (tensor<4x?xi32>, !tosa.shape<1>) -> tensor<?xi32>
 
-  // CHECK: %1 = tosa.reshape %arg0 {new_shape = array<i64: -1>} : (tensor<4x?xi32>) -> tensor<?xi32>
-  %1 = tosa.reshape %arg0 {new_shape = array<i64: -1>} : (tensor<4x?xi32>) -> tensor<?xi32>
+  // CHECK: %2 = tosa.const_shape {value = dense<-1> : tensor<1xindex>} : () -> !tosa.shape<1>
+  %2 = tosa.const_shape {value = dense<-1> : tensor<1xindex>} : () -> !tosa.shape<1>
+  // CHECK: %3 = tosa.reshape %arg0, %2 : (tensor<4x?xi32>, !tosa.shape<1>) -> tensor<?xi32>
+  %3 = tosa.reshape %arg0, %2 : (tensor<4x?xi32>, !tosa.shape<1>) -> tensor<?xi32>
 
-  // CHECK: %2 = tosa.reshape %arg0 {new_shape = array<i64: 2, -1>} : (tensor<4x?xi32>) -> tensor<2x?xi32>
-  %2 = tosa.reshape %arg0 {new_shape = array<i64: 2, -1>} : (tensor<4x?xi32>) -> tensor<?x?xi32>
+  // CHECK: %4 = tosa.const_shape {value = dense<[2, -1]> : tensor<2xindex>} : () -> !tosa.shape<2>
+  %4 = tosa.const_shape {value = dense<[2, -1]> : tensor<2xindex>} : () -> !tosa.shape<2>
+  // CHECK: %5 = tosa.reshape %arg0, %4 : (tensor<4x?xi32>, !tosa.shape<2>) -> tensor<2x?xi32>
+  %5 = tosa.reshape %arg0, %4 : (tensor<4x?xi32>, !tosa.shape<2>) -> tensor<?x?xi32>
 
   return
 }
@@ -517,8 +533,12 @@ func.func @test_padding_simple(%arg0 : tensor<1x2xf32>) -> () {
 
 // CHECK-LABEL: @test_slice
 func.func @test_slice(%arg0 : tensor<?xi32>) -> () {
-  // CHECK: tosa.slice %arg0 {size = array<i64: 2>, start = array<i64: 1>} : (tensor<?xi32>) -> tensor<2xi32>
-  %0 = tosa.slice %arg0 { size = array<i64: 2>, start = array<i64: 1> } : (tensor<?xi32>) -> tensor<?xi32>
+  // CHECK: %0 = tosa.const_shape  {value = dense<1> : tensor<1xindex>}
+  // CHECK: %1 = tosa.const_shape  {value = dense<2> : tensor<1xindex>}
+  // CHECK: %2 = tosa.slice %arg0, %0, %1 : (tensor<?xi32>, !tosa.shape<1>, !tosa.shape<1>) -> tensor<2xi32>
+  %0 = tosa.const_shape {value = dense<1> : tensor<1xindex>} : () -> !tosa.shape<1>
+  %1 = tosa.const_shape {value = dense<2> : tensor<1xindex>} : () -> !tosa.shape<1>
+  %2= tosa.slice %arg0, %0, %1 : (tensor<?xi32>, !tosa.shape<1>, !tosa.shape<1>) -> tensor<?xi32>
   return
 }
 
@@ -526,13 +546,17 @@ func.func @test_slice(%arg0 : tensor<?xi32>) -> () {
 
 // CHECK-LABEL: @test_slice_size_minus_one
 func.func @test_slice_size_minus_one(%arg0 : tensor<?x8x8x8xi32>) -> () {
-  // CHECK: tosa.slice %arg0 {size = array<i64: -1, -1, -1, -1>, start = array<i64: 0, 1, -1, 8>} : (tensor<?x8x8x8xi32>) -> tensor<?x7x?x?xi32>
+  // CHECK: %[[Start:.+]] = tosa.const_shape
+  // CHECK: %[[Size:.+]] = tosa.const_shape
+  // CHECK: %[[VAL:.+]] = tosa.slice %arg0, %[[Start]], %[[Size]] : (tensor<?x8x8x8xi32>, !tosa.shape<4>, !tosa.shape<4>) -> tensor<?x7x?x?xi32>
   // this checks following
   //  dim 0: size=-1, input dim=? => inferred output dim is ?
   //  dim 1: size=-1 => inferred output dim is input_dim - start
   //  dim 2: size=-1, start=-1 => inferred output dim is ?
   //  dim 3: size=-1, start=8 => inferred output dim is ? because start is out of bound
-  %2= tosa.slice %arg0 { start = array<i64: 0, 1, -1, 8>, size = array<i64: -1, -1, -1, -1> } : (tensor<?x8x8x8xi32>) -> tensor<?x?x?x?xi32>
+  %start = tosa.const_shape {value = dense<[0, 1, -1, 8]> : tensor<4xindex>} : () -> !tosa.shape<4>
+  %size = tosa.const_shape {value = dense<[-1, -1, -1, -1]> : tensor<4xindex>} : () -> !tosa.shape<4>
+  %2= tosa.slice %arg0, %start, %size : (tensor<?x8x8x8xi32>, !tosa.shape<4>, !tosa.shape<4>) -> tensor<?x?x?x?xi32>
   return
 }
 
@@ -540,13 +564,17 @@ func.func @test_slice_size_minus_one(%arg0 : tensor<?x8x8x8xi32>) -> () {
 
 // CHECK-LABEL: @test_slice_size_out_of_bound
 func.func @test_slice_size_out_of_bound(%arg0 : tensor<8x8x8x?xi32>) -> () {
-  // CHECK: tosa.slice %arg0 {size = array<i64: 0, -2, 9, 4>, start = array<i64: 0, 0, 0, 0>} : (tensor<8x8x8x?xi32>) -> tensor<?x?x?x4xi32>
+  // CHECK: %[[Start:.+]] = tosa.const_shape
+  // CHECK: %[[Size:.+]] = tosa.const_shape
+  // CHECK: %[[VAL:.+]] = tosa.slice %arg0, %[[Start]], %[[Size]] : (tensor<8x8x8x?xi32>, !tosa.shape<4>, !tosa.shape<4>) -> tensor<?x?x?x4xi32>
   // this checks following
   //  dim 0: size=0 => inferred output dim is ?
   //  dim 1: size=-2 => inferred output dim is ?
   //  dim 3: start+size out of bound because size too big: inferred output dim is ?
   //  dim 4: size=4, input dim=? => inferred output dim is 4
-  %2= tosa.slice %arg0 { start = array<i64: 0, 0, 0, 0>, size = array<i64: 0, -2, 9, 4> } : (tensor<8x8x8x?xi32>) -> tensor<?x?x?x?xi32>
+  %start = tosa.const_shape {value = dense<[0, 0, 0, 0]> : tensor<4xindex>} : () -> !tosa.shape<4>
+  %size = tosa.const_shape {value = dense<[0, -2, 9, 4]> : tensor<4xindex>} : () -> !tosa.shape<4>
+  %2= tosa.slice %arg0, %start, %size : (tensor<8x8x8x?xi32>, !tosa.shape<4>, !tosa.shape<4>) -> tensor<?x?x?x?xi32>
   return
 }
 
@@ -554,13 +582,17 @@ func.func @test_slice_size_out_of_bound(%arg0 : tensor<8x8x8x?xi32>) -> () {
 
 // CHECK-LABEL: @test_slice_start_out_of_bound
 func.func @test_slice_start_out_of_bound(%arg0 : tensor<8x8x8x?xi32>) -> () {
-  // CHECK: tosa.slice %arg0 {size = array<i64: 1, 1, 3, 4>, start = array<i64: -1, 8, 6, 8000000>} : (tensor<8x8x8x?xi32>) -> tensor<?x?x?x4xi32>
+  // CHECK: %[[Start:.+]] = tosa.const_shape
+  // CHECK: %[[Size:.+]] = tosa.const_shape
+  // CHECK: %[[VAL:.+]] = tosa.slice %arg0, %[[Start]], %[[Size]] : (tensor<8x8x8x?xi32>, !tosa.shape<4>, !tosa.shape<4>) -> tensor<?x?x?x4xi32>
   // this checks following
   //  dim 0: start=-1 => inferred output dim is ?
   //  dim 1: start=8 => inferred output dim is ?
   //  dim 2: start+size out of bound: inferred output dim is ?
   //  dim 3: start=8000000, size=4, input dim=? => inferred output dim is 4
-  %2= tosa.slice %arg0 { start = array<i64: -1, 8, 6, 8000000>, size = array<i64: 1, 1, 3, 4> } : (tensor<8x8x8x?xi32>) -> tensor<?x?x?x?xi32>
+  %start = tosa.const_shape {value = dense<[-1, 8, 6, 8000000]> : tensor<4xindex>} : () -> !tosa.shape<4>
+  %size = tosa.const_shape {value = dense<[1, 1, 3, 4]> : tensor<4xindex>} : () -> !tosa.shape<4>
+  %2= tosa.slice %arg0, %start, %size : (tensor<8x8x8x?xi32>, !tosa.shape<4>, !tosa.shape<4>) -> tensor<?x?x?x?xi32>
   return
 }
 
@@ -568,8 +600,12 @@ func.func @test_slice_start_out_of_bound(%arg0 : tensor<8x8x8x?xi32>) -> () {
 
 // CHECK-LABEL: @test_slice_dynamic
 func.func @test_slice_dynamic(%arg0 : tensor<10x?x2xf32>) -> () {
-  // CHECK: tosa.slice %arg0 {size = array<i64: 7, -1, 1>, start = array<i64: 1, 0, 0>} : (tensor<10x?x2xf32>) -> tensor<7x?x1xf32>
-  %0 = tosa.slice %arg0 {size = array<i64: 7, -1, 1>, start = array<i64: 1, 0, 0>} : (tensor<10x?x2xf32>) -> tensor<?x?x?xf32>
+  // CHECK: %0 = tosa.const_shape  {value = dense<[1, 0, 0]> : tensor<3xindex>}
+  // CHECK: %1 = tosa.const_shape  {value = dense<[7, -1, 1]> : tensor<3xindex>}
+  // CHECK: %2 = tosa.slice %arg0, %0, %1 : (tensor<10x?x2xf32>, !tosa.shape<3>, !tosa.shape<3>) -> tensor<7x?x1xf32>
+  %0 = tosa.const_shape {value = dense<[1, 0, 0]> : tensor<3xindex>} : () -> !tosa.shape<3>
+  %1 = tosa.const_shape {value = dense<[7, -1, 1]> : tensor<3xindex>} : () -> !tosa.shape<3>
+  %2= tosa.slice %arg0, %0, %1 : (tensor<10x?x2xf32>, !tosa.shape<3>, !tosa.shape<3>) -> tensor<?x?x?xf32>
   return
 }
 
@@ -1369,7 +1405,7 @@ func.func @test_non_tosa_consumer_shape(%arg0: tensor<4x4xf32>) -> !shape.shape
 
 // -----
 
-// CHECK-LABEL: test_non_tosa_consumer_shape2
+// CHECK-LABEL: test_non_tosa_consumer_shape
 func.func @test_non_tosa_consumer_shape2(%arg0: tensor<4x4xf32>) -> tensor<?xindex> {
   // CHECK: tosa.log %arg0 : (tensor<4x4xf32>) -> tensor<4x4xf32>
   %0 = tosa.log %arg0 : (tensor<4x4xf32>) -> tensor<*xf32>
diff --git a/mlir/test/Dialect/Tosa/tosa-reduce-transposes.mlir b/mlir/test/Dialect/Tosa/tosa-reduce-transposes.mlir
index 3f0d7544083a4..3a293009a5455 100644
--- a/mlir/test/Dialect/Tosa/tosa-reduce-transposes.mlir
+++ b/mlir/test/Dialect/Tosa/tosa-reduce-transposes.mlir
@@ -22,7 +22,7 @@ func.func @test_transpose_tracks_to_nullifying_single_step(%arg0: tensor<1x2x3x4
 func.func @test_transpose_tracks_to_nullifying_multi_unary_step(%arg0: tensor<1x2x3x4xi32>) -> tensor<1x2x3x4xi32> {
   %perms0 = "tosa.const"() {value = dense<[0, 2, 3, 1]> : tensor<4xi32>} : () -> tensor<4xi32>
   %0 = tosa.transpose %arg0, %perms0 : (tensor<1x2x3x4xi32>, tensor<4xi32>) -> tensor<1x3x4x2xi32>
-  %clamp = tosa.clamp %0 {max_fp = 1.0 : f32, min_fp = 0.0 : f32, max_int = 1 : i64, min_int = 0 : i64} : (tensor<1x3x4x2xi32>) -> tensor<1x3x4x2xi32>
+  %clamp = tosa.clamp %0 {max_val = 1 : i32, min_val = 0 : i32} : (tensor<1x3x4x2xi32>) -> tensor<1x3x4x2xi32>
   %abs = tosa.abs %clamp : (tensor<1x3x4x2xi32>) -> tensor<1x3x4x2xi32>
   %bitwise_not = tosa.bitwise_not %abs : (tensor<1x3x4x2xi32>) -> tensor<1x3x4x2xi32>
   %perms1 = "tosa.const"() {value = dense<[0, 3, 1, 2]> : tensor<4xi32>} : () -> tensor<4xi32>
@@ -41,7 +41,7 @@ func.func @test_transpose_tracks_to_nullifying_diverging_binary(%arg0: tensor<1x
   %perms0 = "tosa.const"() {value = dense<[0, 2, 3, 1]> : tensor<4xi32>} : () -> tensor<4xi32>
   %transpose0 = tosa.transpose %arg0, %perms0 : (tensor<1x2x3x4xi32>, tensor<4xi32>) -> tensor<1x3x4x2xi32>
   %transpose1 = tosa.transpose %arg1, %perms0 : (tensor<1x2x3x4xi32>, tensor<4xi32>) -> tensor<1x3x4x2xi32>
-  %clamp = tosa.clamp %transpose0 {max_fp = 1.0 : f32, min_fp = 0.0 : f32, max_int = 1 : i64, min_int = 0 : i64} : (tensor<1x3x4x2xi32>) -> tensor<1x3x4x2xi32>
+  %clamp = tosa.clamp %transpose0 {max_val = 1 : i32, min_val = 0 : i32} : (tensor<1x3x4x2xi32>) -> tensor<1x3x4x2xi32>
   %abs = tosa.abs %transpose1 : (tensor<1x3x4x2xi32>) -> tensor<1x3x4x2xi32>
   %add = tosa.add %clamp, %abs : (tensor<1x3x4x2xi32>, tensor<1x3x4x2xi32>) -> tensor<1x3x4x2xi32>
   %perms1 = "tosa.const"() {value = dense<[0, 3, 1, 2]> : tensor<4xi32>} : () -> tensor<4xi32>
@@ -61,7 +61,7 @@ func.func @test_transpose_tracks_to_nullifying_diverging_binary_with_broadcastin
   %perms0 = "tosa.const"() {value = dense<[0, 2, 3, 1]> : tensor<4xi32>} : () -> tensor<4xi32>
   %transpose0 = tosa.transpose %arg0, %perms0 : (tensor<1x2x3x4xi32>, tensor<4xi32>) -> tensor<1x3x4x2xi32>
   %transpose1 = tosa.transpose %arg1, %perms0 : (tensor<1x2x1x4xi32>, tensor<4xi32>) -> tensor<1x1x4x2xi32>
-  %clamp = tosa.clamp %transpose0 {max_fp = 1.0 : f32, min_fp = 0.0 : f32, max_int = 1 : i64, min_int = 0 : i64} : (tensor<1x3x4x2xi32>) -> tensor<1x3x4x2xi32>
+  %clamp = tosa.clamp %transpose0 {max_val = 1 : i32, min_val = 0 : i32} : (tensor<1x3x4x2xi32>) -> tensor<1x3x4x2xi32>
   %abs = tosa.abs %transpose1 : (tensor<1x1x4x2xi32>) -> tensor<1x1x4x2xi32>
   %add = tosa.add %clamp, %abs : (tensor<1x3x4x2xi32>, tensor<1x1x4x2xi32>) -> tensor<1x3x4x2xi32>
   %perms1 = "tosa.const"() {value = dense<[0, 3, 1, 2]> : tensor<4xi32>} : () -> tensor<4xi32>
@@ -122,13 +122,15 @@ func.func @test_torch_conv2d_with_elementwise_in_between(%arg0: tensor<3x3x10x10
 // -----
 
 // CHECK-LABEL: @test_mulop_conversion
-// CHECK-NEXT: %[[RES:.*]] = tosa.mul %arg0, %arg1
+// CHECK-NEXT: %[[SHIFT:.*]] = "tosa.const"() <{value = dense<0> : tensor<1xi8>}> : () -> tensor<1xi8>
+// CHECK-NEXT: %[[RES:.*]] = tosa.mul %arg0, %arg1, %[[SHIFT]]
 // CHECK-NEXT: return %[[RES]]
 func.func @test_mulop_conversion(%arg0: tensor<1x2x3x4xi32>, %arg1: tensor<1x2x3x4xi32>) -> tensor<1x2x3x4xi32> {
   %perms0 = "tosa.const"() {value = dense<[0, 2, 3, 1]> : tensor<4xi32>} : () -> tensor<4xi32>
   %transpose0 = tosa.transpose %arg0, %perms0 : (tensor<1x2x3x4xi32>, tensor<4xi32>) -> tensor<1x3x4x2xi32>
   %transpose1 = tosa.transpose %arg1, %perms0 : (tensor<1x2x3x4xi32>, tensor<4xi32>) -> tensor<1x3x4x2xi32>
-  %mul = tosa.mul %transpose0, %transpose1 {shift = 0 : i8} : (tensor<1x3x4x2xi32>, tensor<1x3x4x2xi32>) -> tensor<1x3x4x2xi32>
+  %shift = "tosa.const"() <{value = dense<0> : tensor<1xi8>}> : () -> tensor<1xi8>
+  %mul = tosa.mul %transpose0, %transpose1, %shift : (tensor<1x3x4x2xi32>, tensor<1x3x4x2xi32>, tensor<1xi8>) -> tensor<1x3x4x2xi32>
   %perms1 = "tosa.const"() {value = dense<[0, 3, 1, 2]> : tensor<4xi32>} : () -> tensor<4xi32>
   %result = tosa.transpose %mul, %perms1 : (tensor<1x3x4x2xi32>, tensor<4xi32>) -> tensor<1x2x3x4xi32>
   return %result : tensor<1x2x3x4xi32>
@@ -139,12 +141,14 @@ func.func @test_mulop_conversion(%arg0: tensor<1x2x3x4xi32>, %arg1: tensor<1x2x3
 // COM: this case is a reshape we don't convert, since can't fold the transpose into it.
 // COM: a transform actually occurs underneath the hood, but it results in identical IR.
 // CHECK-LABEL: @test_basic_non_broadcasting_reshape
-// CHECK: "tosa.const"() <{value = dense<[0, 2, 1]> : tensor<3xi32>}> : () -> tensor<3xi32>
-// CHECK: tosa.reshape %arg0 {new_shape = array<i64: 1, 3, 2>} : (tensor<2x3xi32>) -> tensor<1x3x2xi32>
-// CHECK: tosa.transpose %1, %0 : (tensor<1x3x2xi32>, tensor<3xi32>) -> tensor<1x2x3xi32>
+// CHECK-DAG: %[[VAL_1:.*]] = tosa.const_shape  {value = dense<[1, 3, 2]> : tensor<3xindex>}
+// CHECK-DAG: %[[VAL_2:.*]] = "tosa.const"() <{value = dense<[0, 2, 1]> : tensor<3xi32>}>
+// CHECK: %[[VAL_3:.*]] = tosa.reshape %arg0, %[[VAL_1]] : (tensor<2x3xi32>, !tosa.shape<3>) -> tensor<1x3x2xi32>
+// CHECK: %[[VAL_4:.*]] = tosa.transpose %[[VAL_3]], %[[VAL_2]] : (tensor<1x3x2xi32>, tensor<3xi32>) -> tensor<1x2x3xi32>
 func.func @test_basic_non_broadcasting_reshape(%arg0: tensor<2x3xi32>) -> tensor<1x2x3xi32> {
+  %shape = tosa.const_shape {value = dense<[1, 3, 2]> : tensor<3xindex>} : () -> !tosa.shape<3>
   %perms = "tosa.const"() {value = dense<[0, 2, 1]> : tensor<3xi32>} : () -> tensor<3xi32>
-  %1 = tosa.reshape %arg0 {new_shape = array<i64: 1, 3, 2>} : (tensor<2x3xi32>) -> tensor<1x3x2xi32>
+  %1 = tosa.reshape %arg0, %shape : (tensor<2x3xi32>, !tosa.shape<3>) -> tensor<1x3x2xi32>
   %2 = tosa.transpose %1, %perms : (tensor<1x3x2xi32>, tensor<3xi32>) -> tensor<1x2x3xi32>
   return %2 : tensor<1x2x3xi32>
 }
@@ -152,11 +156,13 @@ func.func @test_basic_non_broadcasting_reshape(%arg0: tensor<2x3xi32>) -> tensor
 // -----
 
 // CHECK-LABEL: @test_dynamic_broadcasting_reshape
-// CHECK: %[[RES:.*]] = tosa.reshape %arg0 {new_shape = array<i64: 1, 1, -1>} : (tensor<?xi32>) -> tensor<1x1x?xi32>
+// CHECK-DAG: %[[SHAPE:.*]] = tosa.const_shape  {value = dense<[1, 1, -1]> : tensor<3xindex>}
+// CHECK: %[[RES:.*]] = tosa.reshape %arg0, %[[SHAPE]] : (tensor<?xi32>, !tosa.shape<3>) -> tensor<1x1x?xi32>
 // CHECK: return %[[RES]]
 func.func @test_dynamic_broadcasting_reshape(%arg0: tensor<?xi32>) -> tensor<1x1x?xi32> {
+  %shape = tosa.const_shape {value = dense<[1, -1, 1]> : tensor<3xindex>} : () -> !tosa.shape<3>
   %perms = "tosa.const"() {value = dense<[0, 2, 1]> : tensor<3xi32>} : () -> tensor<3xi32>
-  %1 = tosa.reshape %arg0 {new_shape = array<i64: 1, -1, 1>} : (tensor<?xi32>) -> tensor<1x?x1xi32>
+  %1 = tosa.reshape %arg0, %shape : (tensor<?xi32>, !tosa.shape<3>) -> tensor<1x?x1xi32>
   %2 = tosa.transpose %1, %perms : (tensor<1x?x1xi32>, tensor<3xi32>) -> tensor<1x1x?xi32>
   return %2 : tensor<1x1x?xi32>
 }
@@ -165,12 +171,14 @@ func.func @test_dynamic_broadcasting_reshape(%arg0: tensor<?xi32>) -> tensor<1x1
 
 // CHECK-LABEL: @test_reshape_for_broadcast
 // CHECK-DAG: %[[RESHAPE_INPUT:.*]] = "tosa.const"() <{value = dense<[1, 2, 3, 4]>
-// CHECK-DAG: %[[RESHAPE:.*]] = tosa.reshape %[[RESHAPE_INPUT]] {new_shape = array<i64: 4, 1, 1>}
-// CHECK-DAG: %[[ADD:.*]] = tosa.add %arg0, %[[RESHAPE]]
+// CHECK-DAG: %[[SHAPE:.*]] = tosa.const_shape  {value = dense<[4, 1, 1]> : tensor<3xindex>}
+// CHECK: %[[RESHAPE:.*]] = tosa.reshape %[[RESHAPE_INPUT]], %[[SHAPE]] : (tensor<4xi32>, !tosa.shape<3>) -> tensor<4x1x1xi32>
+// CHECK: %[[ADD:.*]] = tosa.add %arg0, %[[RESHAPE]]
 // CHECK: return %[[ADD]]
 func.func @test_reshape_for_broadcast(%arg0: tensor<4x3x2xi32>) -> tensor<4x3x2xi32> {
   %0 = "tosa.const"() {value = dense<[1,2,3,4]> : tensor<4xi32>} : () -> tensor<4xi32>
-  %reshape = tosa.reshape %0 {new_shape = array<i64: 1, 1, 4>} : (tensor<4xi32>) -> tensor<1x1x4xi32>
+  %1 = tosa.const_shape {value = dense<[1, 1, 4]> : tensor<3xindex>} : () -> !tosa.shape<3>
+  %reshape = tosa.reshape %0, %1 : (tensor<4xi32>, !tosa.shape<3>) -> tensor<1x1x4xi32>
   %perms0 = "tosa.const"() {value = dense<[2, 1, 0]> : tensor<3xi32>} : () -> tensor<3xi32>
   %transpose0 = tosa.transpose %arg0, %perms0 : (tensor<4x3x2xi32>, tensor<3xi32>) -> tensor<2x3x4xi32>
   %add = tosa.add %transpose0, %reshape : (tensor<2x3x4xi32>, tensor<1x1x4xi32>) -> tensor<2x3x4xi32>
@@ -185,25 +193,30 @@ func.func @test_reshape_for_broadcast(%arg0: tensor<4x3x2xi32>) -> tensor<4x3x2x
 
 // CHECK-LABEL: @test_resnet18_common_case
 // COM: note that %74 is now represented by %arg2
-// CHECK-DAG: %[[VAL_2:.*]] = "tosa.const"() <{value = dense_resource<torch_tensor_64_torch.float32_1> : tensor<64xf32>}> : () -> tensor<64xf32>
-// CHECK-DAG: %[[VAL_3:.*]] = "tosa.const"() <{value = dense_resource<torch_tensor_64_torch.float32> : tensor<64xf32>}> : () -> tensor<64xf32>
-// CHECK-DAG: %[[VAL_4:.*]] = "tosa.const"() <{value = dense<9.99999974E-6> : tensor<1xf32>}> : () -> tensor<1xf32>
-// CHECK-DAG: %[[VAL_5:.*]] = "tosa.const"() <{value = dense<5.000000e-01> : tensor<1xf32>}> : () -> tensor<1xf32>
-// CHECK-DAG: %[[VAL_6:.*]] = tosa.add %arg1, %[[VAL_4]] : (tensor<64xf32>, tensor<1xf32>) -> tensor<64xf32>
-// CHECK-DAG: %[[VAL_7:.*]] = tosa.pow %[[VAL_6]], %[[VAL_5]] : (tensor<64xf32>, tensor<1xf32>) -> tensor<64xf32>
-// CHECK-DAG: %[[VAL_8:.*]] = tosa.reciprocal %[[VAL_7]] : (tensor<64xf32>) -> tensor<64xf32>
-// CHECK-DAG: %[[VAL_9:.*]] = tosa.reshape %arg0 {new_shape = array<i64: 1, 1, 1, 64>} : (tensor<64xf32>) -> tensor<1x1x1x64xf32>
-// CHECK-DAG: %[[VAL_10:.*]] = tosa.sub %arg2, %[[VAL_9]] : (tensor<1x112x112x64xf32>, tensor<1x1x1x64xf32>) -> tensor<1x112x112x64xf32>
-// CHECK-DAG: %[[VAL_11:.*]] = tosa.reshape %[[VAL_8]] {new_shape = array<i64: 1, 1, 1, 64>} : (tensor<64xf32>) -> tensor<1x1x1x64xf32>
-// CHECK-DAG: %[[VAL_12:.*]] = tosa.mul %[[VAL_10]], %[[VAL_11]] {shift = 0 : i8} : (tensor<1x112x112x64xf32>, tensor<1x1x1x64xf32>) -> tensor<1x112x112x64xf32>
-// CHECK-DAG: %[[VAL_13:.*]] = tosa.reshape %[[VAL_3]] {new_shape = array<i64: 1, 1, 1, 64>} : (tensor<64xf32>) -> tensor<1x1x1x64xf32>
-// CHECK-DAG: %[[VAL_14:.*]] = tosa.mul %[[VAL_12]], %[[VAL_13]] {shift = 0 : i8} : (tensor<1x112x112x64xf32>, tensor<1x1x1x64xf32>) -> tensor<1x112x112x64xf32>
-// CHECK-DAG: %[[VAL_15:.*]] = tosa.reshape %[[VAL_2]] {new_shape = array<i64: 1, 1, 1, 64>} : (tensor<64xf32>) -> tensor<1x1x1x64xf32>
-// CHECK-DAG: %[[VAL_16:.*]] = tosa.add %[[VAL_14]], %[[VAL_15]] : (tensor<1x112x112x64xf32>, tensor<1x1x1x64xf32>) -> tensor<1x112x112x64xf32>
-// CHECK-DAG: %[[VAL_17:.*]] = tosa.clamp %[[VAL_16]] {max_fp = 3.40282347E+38 : f32, max_int = 2147483647 : i64, min_fp = 0.000000e+00 : f32, min_int = 0 : i64} : (tensor<1x112x112x64xf32>) -> tensor<1x112x112x64xf32>
-// CHECK: return %[[VAL_17]] : tensor<1x112x112x64xf32>
-
+// CHECK-DAG: %[[CONST0:.+]] = "tosa.const"() <{value = dense<0> : tensor<1xi8>}> : () -> tensor<1xi8>
+// CHECK-DAG: %[[VAL_3:.*]] = "tosa.const"() <{value = dense_resource<torch_tensor_64_torch.float32_1> : tensor<64xf32>}> : () -> tensor<64xf32>
+// CHECK-DAG: %[[VAL_4:.*]] = "tosa.const"() <{value = dense_resource<torch_tensor_64_torch.float32> : tensor<64xf32>}> : () -> tensor<64xf32>
+// CHECK-DAG: %[[VAL_5:.*]] = "tosa.const"() <{value = dense<9.99999974E-6> : tensor<1xf32>}> : () -> tensor<1xf32>
+// CHECK-DAG: %[[VAL_6:.*]] = "tosa.const"() <{value = dense<5.000000e-01> : tensor<1xf32>}> : () -> tensor<1xf32>
+// CHECK-DAG: %[[VAL_7:.*]] = tosa.add %arg1, %[[VAL_5]] : (tensor<64xf32>, tensor<1xf32>) -> tensor<64xf32>
+// CHECK-DAG: %[[VAL_8:.*]] = tosa.pow %[[VAL_7]], %[[VAL_6]] : (tensor<64xf32>, tensor<1xf32>) -> tensor<64xf32>
+// CHECK-DAG: %[[VAL_9:.*]] = tosa.reciprocal %[[VAL_8]] : (tensor<64xf32>) -> tensor<64xf32>
+// CHECK-DAG: %[[VAL_10:.*]] = tosa.const_shape  {value = dense<[1, 1, 1, 64]> : tensor<4xindex>} : () -> !tosa.shape<4>
+// CHECK-DAG: %[[VAL_11:.*]] = tosa.reshape %arg0, %[[VAL_10]] : (tensor<64xf32>, !tosa.shape<4>) -> tensor<1x1x1x64xf32>
+// CHECK-DAG: %[[VAL_12:.*]] = tosa.sub %arg2, %[[VAL_11]] : (tensor<1x112x112x64xf32>, tensor<1x1x1x64xf32>) -> tensor<1x112x112x64xf32>
+// CHECK-DAG: %[[VAL_13:.*]] = tosa.const_shape  {value = dense<[1, 1, 1, 64]> : tensor<4xindex>} : () -> !tosa.shape<4>
+// CHECK-DAG: %[[VAL_14:.*]] = tosa.reshape %[[VAL_9]], %[[VAL_13]] : (tensor<64xf32>, !tosa.shape<4>) -> tensor<1x1x1x64xf32>
+// CHECK-DAG: %[[VAL_15:.*]] = tosa.mul %[[VAL_12]], %[[VAL_14]], %[[CONST0]] : (tensor<1x112x112x64xf32>, tensor<1x1x1x64xf32>, tensor<1xi8>) -> tensor<1x112x112x64xf32>
+// CHECK-DAG: %[[VAL_16:.*]] = tosa.const_shape  {value = dense<[1, 1, 1, 64]> : tensor<4xindex>} : () -> !tosa.shape<4>
+// CHECK-DAG: %[[VAL_17:.*]] = tosa.reshape %[[VAL_4]], %[[VAL_16]] : (tensor<64xf32>, !tosa.shape<4>) -> tensor<1x1x1x64xf32>
+// CHECK-DAG: %[[VAL_18:.*]] = tosa.mul %[[VAL_15]], %[[VAL_17]], %[[CONST0]] : (tensor<1x112x112x64xf32>, tensor<1x1x1x64xf32>, tensor<1xi8>) -> tensor<1x112x112x64xf32>
+// CHECK-DAG: %[[VAL_19:.*]] = tosa.const_shape  {value = dense<[1, 1, 1, 64]> : tensor<4xindex>} : () -> !tosa.shape<4>
+// CHECK-DAG: %[[VAL_20:.*]] = tosa.reshape %[[VAL_3]], %[[VAL_19]] : (tensor<64xf32>, !tosa.shape<4>) -> tensor<1x1x1x64xf32>
+// CHECK-DAG: %[[VAL_21:.*]] = tosa.add %[[VAL_18]], %[[VAL_20]] : (tensor<1x112x112x64xf32>, tensor<1x1x1x64xf32>) -> tensor<1x112x112x64xf32>
+// CHECK-DAG: %[[VAL_22:.*]] = tosa.clamp %[[VAL_21]] {max_val = 3.40282347E+38 : f32, min_val = 0.000000e+00 : f32} : (tensor<1x112x112x64xf32>) -> tensor<1x112x112x64xf32>
 func.func @test_resnet18_common_case(%arg0: tensor<64xf32>, %arg1: tensor<64xf32>, %74: tensor<1x112x112x64xf32>) -> tensor<1x112x112x64xf32> {
+    %shift = "tosa.const"() {value = dense<0> : tensor<1xi8>} : () -> tensor<1xi8>
+    %58 = tosa.const_shape {value = dense<[1, 64, 1, 1]> : tensor<4xindex>} : () -> !tosa.shape<4>
     %59 = "tosa.const"() <{value = dense_resource<torch_tensor_64_torch.float32_1> : tensor<64xf32>}> : () -> tensor<64xf32>
     %60 = "tosa.const"() <{value = dense_resource<torch_tensor_64_torch.float32> : tensor<64xf32>}> : () -> tensor<64xf32>
     %63 = "tosa.const"() <{value = dense<[0, 2, 3, 1]> : tensor<4xi32>}> : () -> tensor<4xi32>
@@ -214,20 +227,19 @@ func.func @test_resnet18_common_case(%arg0: tensor<64xf32>, %arg1: tensor<64xf32
     %76 = tosa.add %arg1, %69 : (tensor<64xf32>, tensor<1xf32>) -> tensor<64xf32>
     %77 = tosa.pow %76, %70 : (tensor<64xf32>, tensor<1xf32>) -> tensor<64xf32>
     %78 = tosa.reciprocal %77 : (tensor<64xf32>) -> tensor<64xf32>
-    %79 = tosa.reshape %arg0 {new_shape = array<i64: 1, 64, 1, 1>} : (tensor<64xf32>) -> tensor<1x64x1x1xf32>
+    %79 = tosa.reshape %arg0, %58 : (tensor<64xf32>, !tosa.shape<4>) -> tensor<1x64x1x1xf32>
     %80 = tosa.sub %75, %79 : (tensor<1x64x112x112xf32>, tensor<1x64x1x1xf32>) -> tensor<1x64x112x112xf32>
-    %81 = tosa.reshape %78 {new_shape = array<i64: 1, 64, 1, 1>} : (tensor<64xf32>) -> tensor<1x64x1x1xf32>
-    %82 = tosa.mul %80, %81 {shift = 0 : i8} : (tensor<1x64x112x112xf32>, tensor<1x64x1x1xf32>) -> tensor<1x64x112x112xf32>
-    %83 = tosa.reshape %60 {new_shape = array<i64: 1, 64, 1, 1>} : (tensor<64xf32>) -> tensor<1x64x1x1xf32>
-    %84 = tosa.mul %82, %83 {shift = 0 : i8} : (tensor<1x64x112x112xf32>, tensor<1x64x1x1xf32>) -> tensor<1x64x112x112xf32>
-    %85 = tosa.reshape %59 {new_shape = array<i64: 1, 64, 1, 1>} : (tensor<64xf32>) -> tensor<1x64x1x1xf32>
+    %81 = tosa.reshape %78, %58 : (tensor<64xf32>, !tosa.shape<4>) -> tensor<1x64x1x1xf32>
+    %82 = tosa.mul %80, %81, %shift : (tensor<1x64x112x112xf32>, tensor<1x64x1x1xf32>, tensor<1xi8>) -> tensor<1x64x112x112xf32>
+    %83 = tosa.reshape %60, %58 : (tensor<64xf32>, !tosa.shape<4>) -> tensor<1x64x1x1xf32>
+    %84 = tosa.mul %82, %83, %shift : (tensor<1x64x112x112xf32>, tensor<1x64x1x1xf32>, tensor<1xi8>) -> tensor<1x64x112x112xf32>
+    %85 = tosa.reshape %59, %58 : (tensor<64xf32>, !tosa.shape<4>) -> tensor<1x64x1x1xf32>
     %86 = tosa.add %84, %85 : (tensor<1x64x112x112xf32>, tensor<1x64x1x1xf32>) -> tensor<1x64x112x112xf32>
-    %87 = tosa.clamp %86 {max_fp = 3.40282347E+38 : f32, max_int = 2147483647 : i64, min_fp = 0.000000e+00 : f32, min_int = 0 : i64} : (tensor<1x64x112x112xf32>) -> tensor<1x64x112x112xf32>
+    %87 = tosa.clamp %86 {max_val = 3.40282347E+38 : f32, min_val = 0.000000e+00 : f32} : (tensor<1x64x112x112xf32>) -> tensor<1x64x112x112xf32>
     %88 = tosa.transpose %87, %63 : (tensor<1x64x112x112xf32>, tensor<4xi32>) -> tensor<1x112x112x64xf32>
     return %88 : tensor<1x112x112x64xf32>
 }
 
-
 // -----
 
 // CHECK-LABEL: @test_back_to_back_nullifiers
@@ -269,7 +281,7 @@ func.func @test_back_to_back_nullifiers_different_transposes(%arg0: tensor<2x3x4
 func.func @test_no_transform_if_outside_fan_in_cone(%arg0: tensor<3x3x3x3xi32>) -> (tensor<3x3x3x3xi32>, tensor<3x3x3x3xi32>) {
   %perms0 = "tosa.const"() {value = dense<[0, 2, 3, 1]> : tensor<4xi32>} : () -> tensor<4xi32>
   %0 = tosa.transpose %arg0, %perms0 : (tensor<3x3x3x3xi32>, tensor<4xi32>) -> tensor<3x3x3x3xi32>
-  %clamp = tosa.clamp %0  {max_fp = 3.40282347E+38 : f32, max_int = 2147483647 : i64, min_fp = 0.000000e+00 : f32, min_int = 0 : i64} : (tensor<3x3x3x3xi32>) -> tensor<3x3x3x3xi32>
+  %clamp = tosa.clamp %0  {max_val = 2147483647 : i32, min_val = 0 : i32} : (tensor<3x3x3x3xi32>) -> tensor<3x3x3x3xi32>
   %perms1 = "tosa.const"() {value = dense<[0, 3, 1, 2]> : tensor<4xi32>} : () -> tensor<4xi32>
   %1 = tosa.transpose %clamp, %perms1 : (tensor<3x3x3x3xi32>, tensor<4xi32>) -> tensor<3x3x3x3xi32>
   return %1, %clamp : tensor<3x3x3x3xi32>, tensor<3x3x3x3xi32>
@@ -283,8 +295,9 @@ func.func @test_no_transform_if_outside_fan_in_cone(%arg0: tensor<3x3x3x3xi32>)
 // CHECK: return %[[RESHAPE]], %[[CLAMP]]
 func.func @test_two_different_downstream_converge_to_reshape_same_perms(%arg0: tensor<64xf32>) -> (tensor<1x1x64xf32>, tensor<1x1x64xf32>) {
   %0 = "tosa.const"() <{value = dense<[0, 2, 1]> : tensor<3xi32>}> : () -> tensor<3xi32>
-  %1 = tosa.reshape %arg0 {new_shape = array<i64: 1, 64, 1>} : (tensor<64xf32>) -> tensor<1x64x1xf32>
-  %2 = tosa.clamp %1 {max_fp = 3.40282347E+38 : f32, max_int = 2147483647 : i64, min_fp = 0.000000e+00 : f32, min_int = 0 : i64} : (tensor<1x64x1xf32>) -> tensor<1x64x1xf32>
+  %shape = tosa.const_shape {value = dense<[1, 64, 1]> : tensor<3xindex>} : () -> !tosa.shape<3>
+  %1 = tosa.reshape %arg0, %shape : (tensor<64xf32>, !tosa.shape<3>) -> tensor<1x64x1xf32>
+  %2 = tosa.clamp %1 {max_val = 3.40282347E+38 : f32, min_val = 0.000000e+00 : f32} : (tensor<1x64x1xf32>) -> tensor<1x64x1xf32>
   %3 = tosa.transpose %1, %0 : (tensor<1x64x1xf32>, tensor<3xi32>) -> tensor<1x1x64xf32>
   %4 = tosa.transpose %2, %0 : (tensor<1x64x1xf32>, tensor<3xi32>) -> tensor<1x1x64xf32>
   return %3, %4 : tensor<1x1x64xf32>, tensor<1x1x64xf32>
@@ -303,8 +316,9 @@ func.func @test_two_different_downstream_converge_to_reshape_same_perms(%arg0: t
 func.func @test_two_different_downstream_converge_to_reshape_different_perms(%arg0: tensor<64xf32>) -> (tensor<1x1x64xf32>, tensor<64x1x1xf32>) {
   %0 = "tosa.const"() <{value = dense<[1, 2, 0]> : tensor<3xi32>}> : () -> tensor<3xi32>
   %1 = "tosa.const"() <{value = dense<[0, 2, 1]> : tensor<3xi32>}> : () -> tensor<3xi32>
-  %2 = tosa.reshape %arg0 {new_shape = array<i64: 1, 64, 1>} : (tensor<64xf32>) -> tensor<1x64x1xf32>
-  %3 = tosa.clamp %2 {max_fp = 3.40282347E+38 : f32, max_int = 2147483647 : i64, min_fp = 0.000000e+00 : f32, min_int = 0 : i64} : (tensor<1x64x1xf32>) -> tensor<1x64x1xf32>
+  %shape = tosa.const_shape {value = dense<[1, 64, 1]> : tensor<3xindex>} : () -> !tosa.shape<3>
+  %2 = tosa.reshape %arg0, %shape : (tensor<64xf32>, !tosa.shape<3>) -> tensor<1x64x1xf32>
+  %3 = tosa.clamp %2 {max_val = 3.40282347E+38 : f32, min_val = 0.000000e+00 : f32} : (tensor<1x64x1xf32>) -> tensor<1x64x1xf32>
   %4 = tosa.transpose %2, %1 : (tensor<1x64x1xf32>, tensor<3xi32>) -> tensor<1x1x64xf32>
   %5 = tosa.transpose %3, %0 : (tensor<1x64x1xf32>, tensor<3xi32>) -> tensor<64x1x1xf32>
   return %4, %5 : tensor<1x1x64xf32>, tensor<64x1x1xf32>
@@ -322,7 +336,7 @@ func.func @test_two_different_downstream_converge_to_reshape_different_perms(%ar
 // CHECK: return %[[RES1]], %[[RES2]]
 func.func @test_outside_perms_usage_of_fan_in(%arg0: tensor<2x3xf32>, %arg1: tensor<3x2xf32>) -> (tensor<2x3xf32>, tensor<3x2xf32>) {        %0 = "tosa.const"() <{value = dense<[1, 0]> : tensor<2xi32>}> : () -> tensor<2xi32>
   %1 = tosa.transpose %arg0, %0 : (tensor<2x3xf32>, tensor<2xi32>) -> tensor<3x2xf32>
-  %2 = tosa.clamp %1  {max_fp = 3.40282347E+38 : f32, max_int = 2147483647 : i64, min_fp = 0.000000e+00 : f32, min_int = 0 : i64} : (tensor<3x2xf32>) -> tensor<3x2xf32>
+  %2 = tosa.clamp %1  {max_val = 3.40282347E+38 : f32, min_val = 0.000000e+00 : f32} : (tensor<3x2xf32>) -> tensor<3x2xf32>
   %3 = tosa.transpose %2, %0 : (tensor<3x2xf32>, tensor<2xi32>) -> tensor<2x3xf32>
   %4 = tosa.add %arg1, %2 : (tensor<3x2xf32>, tensor<3x2xf32>) -> tensor<3x2xf32>
   return %3, %4: tensor<2x3xf32>, tensor<3x2xf32>
@@ -339,7 +353,7 @@ func.func @test_outside_perms_usage_of_fan_in(%arg0: tensor<2x3xf32>, %arg1: ten
 func.func @test_use_present_in_another_valid_perms_fan_in(%arg0: tensor<2x3xf32>, %arg1: tensor<2x3xf32>) -> (tensor<2x3xf32>, tensor<2x3xf32>) {
   %0 = "tosa.const"() <{value = dense<[1, 0]> : tensor<2xi32>}> : () -> tensor<2xi32>
   %1 = tosa.transpose %arg0, %0 : (tensor<2x3xf32>, tensor<2xi32>) -> tensor<3x2xf32>
-  %2 = tosa.clamp %1  {max_fp = 3.40282347E+38 : f32, max_int = 2147483647 : i64, min_fp = 0.000000e+00 : f32, min_int = 0 : i64} : (tensor<3x2xf32>) -> tensor<3x2xf32>
+  %2 = tosa.clamp %1  {max_val = 3.40282347E+38 : f32, min_val = 0.000000e+00 : f32} : (tensor<3x2xf32>) -> tensor<3x2xf32>
   %3 = tosa.transpose %2, %0 : (tensor<3x2xf32>, tensor<2xi32>) -> tensor<2x3xf32>
   %4 = tosa.transpose %arg1, %0 : (tensor<2x3xf32>, tensor<2xi32>) -> tensor<3x2xf32>
   %5 = tosa.add %4, %2 : (tensor<3x2xf32>, tensor<3x2xf32>) -> tensor<3x2xf32>
@@ -376,7 +390,7 @@ func.func @test_two_same_perms_fan_in_but_one_doesnt_convert_dependents(%arg0: t
 func.func @test_direct_use_in_other_transpose_with_same_perms(%arg0: tensor<3x3x3x3xi32>) -> (tensor<3x3x3x3xi32>, tensor<3x3x3x3xi32>) {
   %perms0 = "tosa.const"() {value = dense<[0, 2, 3, 1]> : tensor<4xi32>} : () -> tensor<4xi32>
   %0 = tosa.transpose %arg0, %perms0 : (tensor<3x3x3x3xi32>, tensor<4xi32>) -> tensor<3x3x3x3xi32>
-  %clamp = tosa.clamp %0 {max_fp = 3.40282347E+38 : f32, max_int = 2147483647 : i64, min_fp = 0.000000e+00 : f32, min_int = 0 : i64} : (tensor<3x3x3x3xi32>) -> tensor<3x3x3x3xi32>
+  %clamp = tosa.clamp %0 {max_val = 2147483647 : i32, min_val = 0 : i32} : (tensor<3x3x3x3xi32>) -> tensor<3x3x3x3xi32>
   %perms1 = "tosa.const"() {value = dense<[0, 3, 1, 2]> : tensor<4xi32>} : () -> tensor<4xi32>
   %1 = tosa.transpose %clamp, %perms1 : (tensor<3x3x3x3xi32>, tensor<4xi32>) -> tensor<3x3x3x3xi32>
   %2 = tosa.transpose %clamp, %perms1 : (tensor<3x3x3x3xi32>, tensor<4xi32>) -> tensor<3x3x3x3xi32>
@@ -400,12 +414,12 @@ func.func @test_const_transpose() -> tensor<2x3xi32> {
 
 // CHECK-LABEL: @test_transpose_tracks_to_const_single_step
 // CHECK: %[[NEW_CONST:.*]] = "tosa.const"() <{value = dense<0> : tensor<1x2x3x4xi32>}> : () -> tensor<1x2x3x4xi32>
-// CHECK: %[[NEW_CLAMP:.*]] = tosa.clamp %[[NEW_CONST]] {max_fp = 3.40282347E+38 : f32, max_int = 2147483647 : i64, min_fp = 0.000000e+00 : f32, min_int = 0 : i64} : (tensor<1x2x3x4xi32>) -> tensor<1x2x3x4xi32>
+// CHECK: %[[NEW_CLAMP:.*]] = tosa.clamp %[[NEW_CONST]] {max_val = 2147483647 : i32, min_val = 0 : i32} : (tensor<1x2x3x4xi32>) -> tensor<1x2x3x4xi32>
 // CHECK-NOT: tosa.transpose
 // CHECK: return %[[NEW_CLAMP]]
 func.func @test_transpose_tracks_to_const_single_step() -> tensor<1x2x3x4xi32> {
   %0 = "tosa.const"() {value = dense<0> : tensor<1x3x4x2xi32>} : () -> tensor<1x3x4x2xi32>
-  %clamp = tosa.clamp %0 {max_fp = 3.40282347E+38 : f32, max_int = 2147483647 : i64, min_fp = 0.000000e+00 : f32, min_int = 0 : i64} : (tensor<1x3x4x2xi32>) -> tensor<1x3x4x2xi32>
+  %clamp = tosa.clamp %0 {max_val = 2147483647 : i32, min_val = 0 : i32} : (tensor<1x3x4x2xi32>) -> tensor<1x3x4x2xi32>
   %perms1 = "tosa.const"() {value = dense<[0, 3, 1, 2]> : tensor<4xi32>} : () -> tensor<4xi32>
   %1 = tosa.transpose %clamp, %perms1 : (tensor<1x3x4x2xi32>, tensor<4xi32>) -> tensor<1x2x3x4xi32>
   return %1 : tensor<1x2x3x4xi32>
@@ -415,14 +429,14 @@ func.func @test_transpose_tracks_to_const_single_step() -> tensor<1x2x3x4xi32> {
 
 // CHECK-LABEL: @test_static_unary_path_to_const
 // CHECK: %[[NEW_CONST:.*]] = "tosa.const"() <{value = dense<1> : tensor<1x2x3x4xi32>}> : () -> tensor<1x2x3x4xi32>
-// CHECK: %[[NEW_CLAMP:.*]] = tosa.clamp %[[NEW_CONST]] {max_fp = 3.40282347E+38 : f32, max_int = 2147483647 : i64, min_fp = 0.000000e+00 : f32, min_int = 0 : i64} : (tensor<1x2x3x4xi32>) -> tensor<1x2x3x4xi32>
+// CHECK: %[[NEW_CLAMP:.*]] = tosa.clamp %[[NEW_CONST]] {max_val = 2147483647 : i32, min_val = 0 : i32} : (tensor<1x2x3x4xi32>) -> tensor<1x2x3x4xi32>
 // CHECK: %[[NEW_ABS:.*]] = tosa.abs %[[NEW_CLAMP]] : (tensor<1x2x3x4xi32>) -> tensor<1x2x3x4xi32>
 // CHECK: %[[NEW_NOT:.*]] = tosa.bitwise_not %[[NEW_ABS]] : (tensor<1x2x3x4xi32>) -> tensor<1x2x3x4xi32>
 // CHECK: return %[[NEW_NOT]]
 func.func @test_static_unary_path_to_const() -> tensor<1x2x3x4xi32> {
   %perms0 = "tosa.const"() {value = dense<[0, 2, 3, 1]> : tensor<4xi32>} : () -> tensor<4xi32>
   %0 = "tosa.const"() {value = dense<1> : tensor<1x3x4x2xi32>} : () -> tensor<1x3x4x2xi32>
-  %clamp = tosa.clamp %0 {max_fp = 3.40282347E+38 : f32, max_int = 2147483647 : i64, min_fp = 0.000000e+00 : f32, min_int = 0 : i64} : (tensor<1x3x4x2xi32>) -> tensor<1x3x4x2xi32>
+  %clamp = tosa.clamp %0 {max_val = 2147483647 : i32, min_val = 0 : i32} : (tensor<1x3x4x2xi32>) -> tensor<1x3x4x2xi32>
   %abs = tosa.abs %clamp : (tensor<1x3x4x2xi32>) -> tensor<1x3x4x2xi32>
   %bitwise_not = tosa.bitwise_not %abs : (tensor<1x3x4x2xi32>) -> tensor<1x3x4x2xi32>
   %perms1 = "tosa.const"() {value = dense<[0, 3, 1, 2]> : tensor<4xi32>} : () -> tensor<4xi32>
@@ -436,7 +450,7 @@ func.func @test_static_unary_path_to_const() -> tensor<1x2x3x4xi32> {
 // CHECK: %[[NEW_CONST:.*]] = "tosa.const"()
 // CHECK-SAME{LITERAL}: dense<[[[[1, 3, 5, 7], [9, 11, 13, 15], [17, 19, 21, 23]], [[2, 4, 6, 8], [10, 12, 14, 16], [18, 20, 22, 24]]]]>
 // CHECK: tensor<1x2x3x4xi32>}> : () -> tensor<1x2x3x4xi32>
-// CHECK: %[[NEW_CLAMP:.*]] = tosa.clamp %arg0 {max_fp = 3.40282347E+38 : f32, max_int = 2147483647 : i64, min_fp = 0.000000e+00 : f32, min_int = 0 : i64} : (tensor<1x2x3x4xi32>) -> tensor<1x2x3x4xi32>
+// CHECK: %[[NEW_CLAMP:.*]] = tosa.clamp %arg0 {max_val = 2147483647 : i32, min_val = 0 : i32} : (tensor<1x2x3x4xi32>) -> tensor<1x2x3x4xi32>
 // CHECK: %[[NEW_ABS:.*]] = tosa.abs %[[NEW_CONST]] : (tensor<1x2x3x4xi32>) -> tensor<1x2x3x4xi32>
 // CHECK: %[[NEW_ADD:.*]] = tosa.add %[[NEW_ABS]], %[[NEW_CLAMP]] : (tensor<1x2x3x4xi32>, tensor<1x2x3x4xi32>) -> tensor<1x2x3x4xi32>
 // CHECK: return %[[NEW_ADD]]
@@ -446,7 +460,7 @@ func.func @test_static_diverges_to_non_splat_const_and_nullifying(%arg0: tensor<
   %const = "tosa.const"() {value = dense<[[[[1, 2], [3, 4], [5, 6], [7, 8]],
    [[9, 10], [11, 12], [13, 14], [15, 16]],
    [[17, 18], [19, 20], [21, 22], [23, 24]]]]> : tensor<1x3x4x2xi32>} : () -> tensor<1x3x4x2xi32>
-  %clamp = tosa.clamp %transpose0 {max_fp = 3.40282347E+38 : f32, max_int = 2147483647 : i64, min_fp = 0.000000e+00 : f32, min_int = 0 : i64} : (tensor<1x3x4x2xi32>) -> tensor<1x3x4x2xi32>
+  %clamp = tosa.clamp %transpose0 {max_val = 2147483647 : i32, min_val = 0 : i32} : (tensor<1x3x4x2xi32>) -> tensor<1x3x4x2xi32>
   %abs = tosa.abs %const : (tensor<1x3x4x2xi32>) -> tensor<1x3x4x2xi32>
   %add = tosa.add %abs, %clamp : (tensor<1x3x4x2xi32>, tensor<1x3x4x2xi32>) -> tensor<1x3x4x2xi32>
   %perms2 = "tosa.const"() {value = dense<[0, 3, 1, 2]> : tensor<4xi32>} : () -> tensor<4xi32>
@@ -462,7 +476,7 @@ func.func @test_static_diverges_to_non_splat_const_and_nullifying(%arg0: tensor<
 func.func @test_multi_downstream_both_nullify(%arg0: tensor<3x3x3x3xi32>) -> (tensor<3x3x3x3xi32>, tensor<3x3x3x3xi32>) {
   %perms0 = "tosa.const"() {value = dense<[0, 2, 3, 1]> : tensor<4xi32>} : () -> tensor<4xi32>
   %0 = tosa.transpose %arg0, %perms0 : (tensor<3x3x3x3xi32>, tensor<4xi32>) -> tensor<3x3x3x3xi32>
-  %clamp = tosa.clamp %0 {max_fp = 3.40282347E+38 : f32, max_int = 2147483647 : i64, min_fp = 0.000000e+00 : f32, min_int = 0 : i64} : (tensor<3x3x3x3xi32>) -> tensor<3x3x3x3xi32>
+  %clamp = tosa.clamp %0 {max_val = 2147483647 : i32, min_val = 0 : i32} : (tensor<3x3x3x3xi32>) -> tensor<3x3x3x3xi32>
   %perms1 = "tosa.const"() {value = dense<[0, 3, 1, 2]> : tensor<4xi32>} : () -> tensor<4xi32>
   %1 = tosa.transpose %clamp, %perms1 : (tensor<3x3x3x3xi32>, tensor<4xi32>) -> tensor<3x3x3x3xi32>
   %2 = tosa.transpose %clamp, %perms1 : (tensor<3x3x3x3xi32>, tensor<4xi32>) -> tensor<3x3x3x3xi32>
@@ -482,7 +496,7 @@ func.func @test_multi_downstream_both_nullify(%arg0: tensor<3x3x3x3xi32>) -> (te
 func.func @test_multi_downstream_one_nullifies_upstream_other_does_not(%arg0: tensor<3x3x3x3xi32>) -> (tensor<3x3x3x3xi32>, tensor<3x3x3x3xi32>) {
   %perms0 = "tosa.const"() {value = dense<[0, 2, 3, 1]> : tensor<4xi32>} : () -> tensor<4xi32>
   %0 = tosa.transpose %arg0, %perms0 : (tensor<3x3x3x3xi32>, tensor<4xi32>) -> tensor<3x3x3x3xi32>
-  %clamp = tosa.clamp %0 {max_fp = 3.40282347E+38 : f32, max_int = 2147483647 : i64, min_fp = 0.000000e+00 : f32, min_int = 0 : i64} : (tensor<3x3x3x3xi32>) -> tensor<3x3x3x3xi32>
+  %clamp = tosa.clamp %0 {max_val = 2147483647 : i32, min_val = 0 : i32} : (tensor<3x3x3x3xi32>) -> tensor<3x3x3x3xi32>
   %perms1 = "tosa.const"() {value = dense<[0, 3, 1, 2]> : tensor<4xi32>} : () -> tensor<4xi32>
   %1 = tosa.transpose %clamp, %perms1 : (tensor<3x3x3x3xi32>, tensor<4xi32>) -> tensor<3x3x3x3xi32>
   %2 = tosa.transpose %clamp, %perms0 : (tensor<3x3x3x3xi32>, tensor<4xi32>) -> tensor<3x3x3x3xi32>
@@ -523,7 +537,7 @@ func.func @test_transpose_tracks_to_nullifying_diverging_binary_unknown_dim_repl
   %perms0 = "tosa.const"() {value = dense<[0, 2, 3, 1]> : tensor<4xi32>} : () -> tensor<4xi32>
   %transpose0 = tosa.transpose %arg0, %perms0 : (tensor<1x?x3x4xi32>, tensor<4xi32>) -> tensor<?x3x4x?xi32>
   %transpose1 = tosa.transpose %arg1, %perms0 : (tensor<1x2x?x4xi32>, tensor<4xi32>) -> tensor<1x?x?x2xi32>
-  %clamp = tosa.clamp %transpose0 {min_int = 0 : i64, max_int = 1 : i64, min_fp = 0.0 : f64, max_fp = 1.0 : f64} : (tensor<?x3x4x?xi32>) -> tensor<?x3x4x?xi32>
+  %clamp = tosa.clamp %transpose0 {min_val = 0 : i32, max_val = 1 : i32} : (tensor<?x3x4x?xi32>) -> tensor<?x3x4x?xi32>
   %abs = tosa.abs %transpose1 : (tensor<1x?x?x2xi32>) -> tensor<1x?x?x2xi32>
   %add = tosa.add %clamp, %abs : (tensor<?x3x4x?xi32>, tensor<1x?x?x2xi32>) -> tensor<1x3x4x2xi32>
   %perms1 = "tosa.const"() {value = dense<[0, 3, 1, 2]> : tensor<4xi32>} : () -> tensor<4xi32>
@@ -558,7 +572,7 @@ func.func @test_unimplemented_non_const_perms(%perms: tensor<2xi32>) -> tensor<?
 func.func @test_unimplemented_transpose_tracks_to_non_nullifying_transpose_single_step(%arg0: tensor<1x2x3x4xi32>) -> tensor<1x2x4x3xi32> {
   %perms0 = "tosa.const"() {value = dense<[0, 3, 2, 1]> : tensor<4xi32>} : () -> tensor<4xi32>
   %0 = tosa.transpose %arg0, %perms0 : (tensor<1x2x3x4xi32>, tensor<4xi32>) -> tensor<1x4x3x2xi32>
-  %clamp = tosa.clamp %0 {min_int = 0 : i64, max_int = 1 : i64, min_fp = 0.0 : f64, max_fp = 1.0 : f64} : (tensor<1x4x3x2xi32>) -> tensor<1x4x3x2xi32>
+  %clamp = tosa.clamp %0 {min_val = 0 : i32, max_val = 1 : i32} : (tensor<1x4x3x2xi32>) -> tensor<1x4x3x2xi32>
   %perms1 = "tosa.const"() {value = dense<[0, 3, 1, 2]> : tensor<4xi32>} : () -> tensor<4xi32>
   %1 = tosa.transpose %clamp, %perms1 : (tensor<1x4x3x2xi32>, tensor<4xi32>) -> tensor<1x2x4x3xi32>
   return %1 : tensor<1x2x4x3xi32>
@@ -640,7 +654,7 @@ func.func @test_unimplemented_static_diverges_to_one_nullifying_one_non_nullifyi
   %perms1 = "tosa.const"() {value = dense<[0, 3, 2, 1]> : tensor<4xi32>} : () -> tensor<4xi32>
   %transpose0 = tosa.transpose %arg0, %perms0 : (tensor<1x2x3x4xi32>, tensor<4xi32>) -> tensor<1x3x4x2xi32>
   %transpose1 = tosa.transpose %arg1, %perms1 : (tensor<1x2x4x3xi32>, tensor<4xi32>) -> tensor<1x3x4x2xi32>
-  %clamp = tosa.clamp %transpose0 {min_int = 0 : i64, max_int = 1 : i64, min_fp = 0.0 : f64, max_fp = 1.0 : f64} : (tensor<1x3x4x2xi32>) -> tensor<1x3x4x2xi32>
+  %clamp = tosa.clamp %transpose0 {min_val = 0 : i32, max_val = 1 : i32} : (tensor<1x3x4x2xi32>) -> tensor<1x3x4x2xi32>
   %abs = tosa.abs %transpose1 : (tensor<1x3x4x2xi32>) -> tensor<1x3x4x2xi32>
   %add = tosa.add %clamp, %abs : (tensor<1x3x4x2xi32>, tensor<1x3x4x2xi32>) -> tensor<1x3x4x2xi32>
   %perms2 = "tosa.const"() {value = dense<[0, 3, 1, 2]> : tensor<4xi32>} : () -> tensor<4xi32>
diff --git a/mlir/test/Dialect/Tosa/transpose-fold.mlir b/mlir/test/Dialect/Tosa/transpose-fold.mlir
index 3e57d7ff618f6..a7e9982c8db70 100644
--- a/mlir/test/Dialect/Tosa/transpose-fold.mlir
+++ b/mlir/test/Dialect/Tosa/transpose-fold.mlir
@@ -6,9 +6,9 @@
 // CHECK:         }
 
 func.func @test_cancel_transpose_transpose(%arg0: tensor<1x2x3xi32>) -> (tensor<1x2x3xi32>) {
-	%0 = arith.constant dense<[1, 2, 0]> : tensor<3xi32>
+	%0 = "tosa.const"() {value = dense<[1, 2, 0]> : tensor<3xi32>} : () -> tensor<3xi32>
 	%1 = tosa.transpose %arg0, %0 : (tensor<1x2x3xi32>, tensor<3xi32>) -> tensor<2x3x1xi32>
-	%2 = arith.constant dense<[2, 0, 1]> : tensor<3xi32>
+	%2 = "tosa.const"() {value = dense<[2, 0, 1]> : tensor<3xi32>} : () -> tensor<3xi32>
 	%3 = tosa.transpose %1, %2 : (tensor<2x3x1xi32>, tensor<3xi32>) -> tensor<1x2x3xi32>
   return %3 : tensor<1x2x3xi32>
 }
@@ -21,7 +21,7 @@ func.func @test_cancel_transpose_transpose(%arg0: tensor<1x2x3xi32>) -> (tensor<
 // CHECK:         }
 
 func.func @test_remove_identity_transpose(%arg0: tensor<1x2x3xi32>) -> (tensor<1x2x3xi32>) {
-	%0 = arith.constant dense<[0, 1, 2]> : tensor<3xi32>
+	%0 = "tosa.const"() {value = dense<[0, 1, 2]> : tensor<3xi32>} : () -> tensor<3xi32>
 	%1 = tosa.transpose %arg0, %0 : (tensor<1x2x3xi32>, tensor<3xi32>) -> tensor<1x2x3xi32>
   return %1 : tensor<1x2x3xi32>
 }
@@ -30,15 +30,15 @@ func.func @test_remove_identity_transpose(%arg0: tensor<1x2x3xi32>) -> (tensor<1
 
 // CHECK-LABEL:   func.func @test_do_not_cancel_different_transpose(
 // CHECK-SAME:                                                      %[[VAL_0:.*]]: tensor<2x3x4x5xi32>) -> tensor<5x4x3x2xi32> {
-// CHECK:           %[[VAL_1:.*]] = arith.constant dense<[3, 2, 1, 0]> : tensor<4xi32>
+// CHECK:           %[[VAL_1:.*]] = "tosa.const"() <{value = dense<[3, 2, 1, 0]> : tensor<4xi32>}> : () -> tensor<4xi32>
 // CHECK:           %[[VAL_2:.*]] = tosa.transpose %[[VAL_0]], %[[VAL_1]] : (tensor<2x3x4x5xi32>, tensor<4xi32>) -> tensor<5x4x3x2xi32>
 // CHECK:           return %[[VAL_2]] : tensor<5x4x3x2xi32>
 // CHECK:         }
 
 func.func @test_do_not_cancel_different_transpose(%arg0: tensor<2x3x4x5xi32>) -> (tensor<5x4x3x2xi32>) {
-	%0 = arith.constant dense<[1, 2, 0, 3]> : tensor<4xi32>
+	%0 = "tosa.const"() {value = dense<[1, 2, 0, 3]> : tensor<4xi32>} : () -> tensor<4xi32>
 	%1 = tosa.transpose %arg0, %0 : (tensor<2x3x4x5xi32>, tensor<4xi32>) -> tensor<3x4x2x5xi32>
-	%2 = arith.constant dense<[3, 1, 0, 2]> : tensor<4xi32>
+	%2 = "tosa.const"() {value = dense<[3, 1, 0, 2]> : tensor<4xi32>} : () -> tensor<4xi32>
 	%3 = tosa.transpose %1, %2 : (tensor<3x4x2x5xi32>, tensor<4xi32>) -> tensor<5x4x3x2xi32>
   return %3 : tensor<5x4x3x2xi32>
 }
@@ -47,15 +47,15 @@ func.func @test_do_not_cancel_different_transpose(%arg0: tensor<2x3x4x5xi32>) ->
 
 // CHECK-LABEL:   func.func @test_prefer_compose_transpose(
 // CHECK-SAME:                                                      %[[VAL_0:.*]]: tensor<1x2x3x4xi32>) -> tensor<4x3x2x1xi32> {
-// CHECK:           %[[VAL_1:.*]] = arith.constant dense<[3, 2, 1, 0]> : tensor<4xi32>
+// CHECK:           %[[VAL_1:.*]] = "tosa.const"() <{value = dense<[3, 2, 1, 0]> : tensor<4xi32>}> : () -> tensor<4xi32>
 // CHECK:           %[[VAL_2:.*]] = tosa.transpose %[[VAL_0]], %[[VAL_1]] : (tensor<1x2x3x4xi32>, tensor<4xi32>) -> tensor<4x3x2x1xi32>
 // CHECK:           return %[[VAL_2]] : tensor<4x3x2x1xi32>
 // CHECK:         }
 
 func.func @test_prefer_compose_transpose(%arg0: tensor<1x2x3x4xi32>) -> (tensor<4x3x2x1xi32>) {
-	%0 = arith.constant dense<[1, 2, 0, 3]> : tensor<4xi32>
+	%0 = "tosa.const"() {value = dense<[1, 2, 0, 3]> : tensor<4xi32>} : () -> tensor<4xi32>
 	%1 = tosa.transpose %arg0, %0 : (tensor<1x2x3x4xi32>, tensor<4xi32>) -> tensor<2x3x1x4xi32>
-	%2 = arith.constant dense<[3, 1, 0, 2]> : tensor<4xi32>
+	%2 = "tosa.const"() {value = dense<[3, 1, 0, 2]> : tensor<4xi32>} : () -> tensor<4xi32>
 	%3 = tosa.transpose %1, %2 : (tensor<2x3x1x4xi32>, tensor<4xi32>) -> tensor<4x3x2x1xi32>
   return %3 : tensor<4x3x2x1xi32>
 }
diff --git a/mlir/test/Dialect/Vector/canonicalize.mlir b/mlir/test/Dialect/Vector/canonicalize.mlir
index 0eebb6e8d612d..93581cbfbe5e4 100644
--- a/mlir/test/Dialect/Vector/canonicalize.mlir
+++ b/mlir/test/Dialect/Vector/canonicalize.mlir
@@ -132,6 +132,64 @@ func.func @extract_from_create_mask_dynamic_position(%dim0: index, %index: index
 
 // -----
 
+// CHECK-LABEL: @extract_scalar_poison
+func.func @extract_scalar_poison() -> f32 {
+  // CHECK-NEXT: %[[UB:.*]] = ub.poison : f32
+  //  CHECK-NOT: vector.extract
+  // CHECK-NEXT: return %[[UB]] : f32
+  %0 = ub.poison : vector<4x8xf32>
+  %1 = vector.extract %0[2, 4] : f32 from vector<4x8xf32>
+  return %1 : f32
+}
+
+// -----
+
+// CHECK-LABEL: @extract_vector_poison
+func.func @extract_vector_poison() -> vector<8xf32> {
+  // CHECK-NEXT: %[[UB:.*]] = ub.poison : vector<8xf32>
+  //  CHECK-NOT: vector.extract
+  // CHECK-NEXT: return %[[UB]] : vector<8xf32>
+  %0 = ub.poison : vector<4x8xf32>
+  %1 = vector.extract %0[2] : vector<8xf32> from vector<4x8xf32>
+  return %1 : vector<8xf32>
+}
+
+// -----
+
+// CHECK-LABEL: @extract_scalar_poison_idx
+func.func @extract_scalar_poison_idx(%a: vector<4x5xf32>) -> f32 {
+  // CHECK-NEXT: %[[UB:.*]] = ub.poison : f32
+  //  CHECK-NOT: vector.extract
+  // CHECK-NEXT: return %[[UB]] : f32
+  %0 = vector.extract %a[-1, 0] : f32 from vector<4x5xf32>
+  return %0 : f32
+}
+
+// -----
+
+// CHECK-LABEL: @extract_vector_poison_idx
+func.func @extract_vector_poison_idx(%a: vector<4x5xf32>) -> vector<5xf32> {
+  // CHECK-NEXT: %[[UB:.*]] = ub.poison : vector<5xf32>
+  //  CHECK-NOT: vector.extract
+  // CHECK-NEXT: return %[[UB]] : vector<5xf32>
+  %0 = vector.extract %a[-1] : vector<5xf32> from vector<4x5xf32>
+  return %0 : vector<5xf32>
+}
+
+// -----
+
+// CHECK-LABEL: @extract_multiple_poison_idx
+func.func @extract_multiple_poison_idx(%a: vector<4x5x8xf32>)
+    -> vector<8xf32> {
+  // CHECK-NEXT: %[[UB:.*]] = ub.poison : vector<8xf32>
+  //  CHECK-NOT: vector.extract
+  // CHECK-NEXT: return %[[UB]] : vector<8xf32>
+  %0 = vector.extract %a[-1, -1] : vector<8xf32> from vector<4x5x8xf32>
+  return %0 : vector<8xf32>
+}
+
+// -----
+
 // CHECK-LABEL: extract_from_create_mask_dynamic_position_all_false
 //  CHECK-SAME: %[[DIM0:.*]]: index, %[[INDEX:.*]]: index
 func.func @extract_from_create_mask_dynamic_position_all_false(%dim0: index, %index: index) -> vector<6xi1> {
@@ -1975,6 +2033,62 @@ func.func @shuffle_1d() -> vector<4xi32> {
   return %shuffle : vector<4xi32>
 }
 
+// -----
+
+// Check that poison indices pick the first element of the first non-poison
+// input vector. That is, %v[0] (i.e., 5) in this test.
+
+// CHECK-LABEL: func @shuffle_1d_poison_idx
+//       CHECK:   %[[V:.+]] = arith.constant dense<[13, 10, 15, 10]> : vector<4xi32>
+//       CHECK:   return %[[V]]
+func.func @shuffle_1d_poison_idx() -> vector<4xi32> {
+  %v0 = arith.constant dense<[10, 11, 12]> : vector<3xi32>
+  %v1 = arith.constant dense<[13, 14, 15]> : vector<3xi32>
+  %shuffle = vector.shuffle %v0, %v1 [3, -1, 5, -1] : vector<3xi32>, vector<3xi32>
+  return %shuffle : vector<4xi32>
+}
+
+// -----
+
+// CHECK-LABEL: func @shuffle_1d_rhs_lhs_poison
+//   CHECK-NOT:   vector.shuffle
+//       CHECK:   %[[V:.+]] = ub.poison : vector<4xi32>
+//       CHECK:   return %[[V]]
+func.func @shuffle_1d_rhs_lhs_poison() -> vector<4xi32> {
+  %v0 = ub.poison : vector<3xi32>
+  %v1 = ub.poison : vector<3xi32>
+  %shuffle = vector.shuffle %v0, %v1 [3, 1, 5, 4] : vector<3xi32>, vector<3xi32>
+  return %shuffle : vector<4xi32>
+}
+
+// -----
+
+// CHECK-LABEL: func @shuffle_1d_lhs_poison
+//   CHECK-NOT:   vector.shuffle
+//       CHECK:   %[[V:.+]] = arith.constant dense<[11, 12, 11, 11]> : vector<4xi32>
+//       CHECK:   return %[[V]]
+func.func @shuffle_1d_lhs_poison() -> vector<4xi32> {
+  %v0 = arith.constant dense<[11, 12, 13]> : vector<3xi32>
+  %v1 = ub.poison : vector<3xi32>
+  %shuffle = vector.shuffle %v0, %v1 [3, 1, 5, 4] : vector<3xi32>, vector<3xi32>
+  return %shuffle : vector<4xi32>
+}
+
+// -----
+
+// CHECK-LABEL: func @shuffle_1d_rhs_poison
+//   CHECK-NOT:   vector.shuffle
+//       CHECK:   %[[V:.+]] = arith.constant dense<[11, 11, 13, 12]> : vector<4xi32>
+//       CHECK:   return %[[V]]
+func.func @shuffle_1d_rhs_poison() -> vector<4xi32> {
+  %v0 = ub.poison : vector<3xi32>
+  %v1 = arith.constant dense<[11, 12, 13]> : vector<3xi32>
+  %shuffle = vector.shuffle %v0, %v1 [3, 1, 5, 4] : vector<3xi32>, vector<3xi32>
+  return %shuffle : vector<4xi32>
+}
+
+// -----
+
 // CHECK-LABEL: func @shuffle_canonicalize_0d
 func.func @shuffle_canonicalize_0d(%v0 : vector<i32>, %v1 : vector<i32>) -> vector<1xi32> {
   // CHECK: vector.broadcast %{{.*}} : vector<i32> to vector<1xi32>
@@ -1982,6 +2096,8 @@ func.func @shuffle_canonicalize_0d(%v0 : vector<i32>, %v1 : vector<i32>) -> vect
   return %shuffle : vector<1xi32>
 }
 
+// -----
+
 // CHECK-LABEL: func @shuffle_fold1
 //       CHECK:   %arg0 : vector<4xi32>
 func.func @shuffle_fold1(%v0 : vector<4xi32>, %v1 : vector<2xi32>) -> vector<4xi32> {
@@ -1989,6 +2105,8 @@ func.func @shuffle_fold1(%v0 : vector<4xi32>, %v1 : vector<2xi32>) -> vector<4xi
   return %shuffle : vector<4xi32>
 }
 
+// -----
+
 // CHECK-LABEL: func @shuffle_fold2
 //       CHECK:   %arg1 : vector<2xi32>
 func.func @shuffle_fold2(%v0 : vector<4xi32>, %v1 : vector<2xi32>) -> vector<2xi32> {
@@ -1996,6 +2114,8 @@ func.func @shuffle_fold2(%v0 : vector<4xi32>, %v1 : vector<2xi32>) -> vector<2xi
   return %shuffle : vector<2xi32>
 }
 
+// -----
+
 // CHECK-LABEL: func @shuffle_fold3
 //       CHECK:   return %arg0 : vector<4x5x6xi32>
 func.func @shuffle_fold3(%v0 : vector<4x5x6xi32>, %v1 : vector<2x5x6xi32>) -> vector<4x5x6xi32> {
@@ -2003,6 +2123,8 @@ func.func @shuffle_fold3(%v0 : vector<4x5x6xi32>, %v1 : vector<2x5x6xi32>) -> ve
   return %shuffle : vector<4x5x6xi32>
 }
 
+// -----
+
 // CHECK-LABEL: func @shuffle_fold4
 //       CHECK:   return %arg1 : vector<2x5x6xi32>
 func.func @shuffle_fold4(%v0 : vector<4x5x6xi32>, %v1 : vector<2x5x6xi32>) -> vector<2x5x6xi32> {
@@ -2010,6 +2132,8 @@ func.func @shuffle_fold4(%v0 : vector<4x5x6xi32>, %v1 : vector<2x5x6xi32>) -> ve
   return %shuffle : vector<2x5x6xi32>
 }
 
+// -----
+
 // CHECK-LABEL: func @shuffle_nofold1
 //       CHECK:   %[[V:.+]] = vector.shuffle %arg0, %arg1 [0, 1, 2, 3, 4] : vector<4xi32>, vector<2xi32>
 //       CHECK:   return %[[V]]
@@ -2778,7 +2902,6 @@ func.func @from_elements_to_splat(%a: f32, %b: f32) -> (vector<2x3xf32>, vector<
   return %0, %1, %2 : vector<2x3xf32>, vector<2x3xf32>, vector<f32>
 }
 
-
 // -----
 
 // CHECK-LABEL: func @vector_insert_const_regression(
@@ -2792,6 +2915,75 @@ func.func @vector_insert_const_regression(%arg0: i8) -> vector<4xi8> {
 
 // -----
 
+// Insert a poison value shouldn't be folded as the resulting vector is not
+// fully poison.
+
+// CHECK-LABEL: @insert_scalar_poison
+func.func @insert_scalar_poison(%a: vector<4x8xf32>)
+    -> vector<4x8xf32> {
+  //  CHECK-NEXT: %[[UB:.*]] = ub.poison : f32
+  //  CHECK-NEXT: %[[RES:.*]] = vector.insert %[[UB]]
+  //  CHECK-NEXT: return %[[RES]] : vector<4x8xf32>
+  %0 = ub.poison : f32
+  %1 = vector.insert %0, %a[2, 3] : f32 into vector<4x8xf32>
+  return %1 : vector<4x8xf32>
+}
+
+// -----
+
+// Insert a poison value shouldn't be folded as the resulting vector is not
+// fully poison.
+
+// CHECK-LABEL: @insert_vector_poison
+func.func @insert_vector_poison(%a: vector<4x8xf32>)
+    -> vector<4x8xf32> {
+  //  CHECK-NEXT: %[[UB:.*]] = ub.poison : vector<8xf32>
+  //  CHECK-NEXT: %[[RES:.*]] = vector.insert %[[UB]]
+  //  CHECK-NEXT: return %[[RES]] : vector<4x8xf32>
+  %0 = ub.poison : vector<8xf32>
+  %1 = vector.insert %0, %a[2] : vector<8xf32> into vector<4x8xf32>
+  return %1 : vector<4x8xf32>
+}
+
+
+// -----
+
+// CHECK-LABEL: @insert_scalar_poison_idx
+func.func @insert_scalar_poison_idx(%a: vector<4x5xf32>, %b: f32)
+    -> vector<4x5xf32> {
+  // CHECK-NEXT: %[[UB:.*]] = ub.poison : vector<4x5xf32>
+  //  CHECK-NOT: vector.insert
+  // CHECK-NEXT: return %[[UB]] : vector<4x5xf32>
+  %0 = vector.insert %b, %a[-1, 0] : f32 into vector<4x5xf32>
+  return %0 : vector<4x5xf32>
+}
+
+// -----
+
+// CHECK-LABEL: @insert_vector_poison_idx
+func.func @insert_vector_poison_idx(%a: vector<4x5xf32>, %b: vector<5xf32>)
+    -> vector<4x5xf32> {
+  // CHECK-NEXT: %[[UB:.*]] = ub.poison : vector<4x5xf32>
+  //  CHECK-NOT: vector.insert
+  // CHECK-NEXT: return %[[UB]] : vector<4x5xf32>
+  %0 = vector.insert %b, %a[-1] : vector<5xf32> into vector<4x5xf32>
+  return %0 : vector<4x5xf32>
+}
+
+// -----
+
+// CHECK-LABEL: @insert_multiple_poison_idx
+func.func @insert_multiple_poison_idx(%a: vector<4x5x8xf32>, %b: vector<8xf32>)
+    -> vector<4x5x8xf32> {
+  // CHECK-NEXT: %[[UB:.*]] = ub.poison : vector<4x5x8xf32>
+  //  CHECK-NOT: vector.insert
+  // CHECK-NEXT: return %[[UB]] : vector<4x5x8xf32>
+  %0 = vector.insert %b, %a[-1, -1] : vector<8xf32> into vector<4x5x8xf32>
+  return %0 : vector<4x5x8xf32>
+}
+
+// -----
+
 // CHECK-LABEL: @contiguous_extract_strided_slices_to_extract
 // CHECK:        %[[EXTRACT:.+]] = vector.extract {{.*}}[0, 0, 0, 0, 0] : vector<4xi32> from vector<8x1x2x1x1x4xi32>
 // CHECK-NEXT:   return %[[EXTRACT]] :  vector<4xi32>
@@ -2979,3 +3171,29 @@ func.func @contiguous_scatter_step(%base: memref<?xf32>,
     memref<?xf32>, vector<16xindex>, vector<16xi1>, vector<16xf32>
   return
 }
+
+// -----
+
+// CHECK-LABEL: @fold_extract_constant_indices
+//   CHECK-SAME:   %[[ARG:.*]]: vector<32x1xi32>) -> i32 {
+//        CHECK:   %[[RES:.*]] = vector.extract %[[ARG]][0, 0] : i32 from vector<32x1xi32>
+//        CHECK:   return %[[RES]] : i32
+func.func @fold_extract_constant_indices(%arg : vector<32x1xi32>) -> i32 {
+  %0 = arith.constant 0 : index
+  %1 = vector.extract %arg[%0, %0] : i32 from vector<32x1xi32>
+  return %1 : i32
+}
+
+// -----
+
+// CHECK-LABEL: @fold_insert_constant_indices
+//  CHECK-SAME:   %[[ARG:.*]]: vector<4x1xi32>) -> vector<4x1xi32> {
+//       CHECK:   %[[VAL:.*]] = arith.constant 1 : i32
+//       CHECK:   %[[RES:.*]] = vector.insert %[[VAL]], %[[ARG]] [0, 0] : i32 into vector<4x1xi32>
+//       CHECK:   return %[[RES]] : vector<4x1xi32>
+func.func @fold_insert_constant_indices(%arg : vector<4x1xi32>) -> vector<4x1xi32> {
+  %0 = arith.constant 0 : index
+  %1 = arith.constant 1 : i32
+  %res = vector.insert %1, %arg[%0, %0] : i32 into vector<4x1xi32>
+  return %res : vector<4x1xi32>
+}
diff --git a/mlir/test/Dialect/Vector/invalid.mlir b/mlir/test/Dialect/Vector/invalid.mlir
index 1a70791fae125..57e348c7d5991 100644
--- a/mlir/test/Dialect/Vector/invalid.mlir
+++ b/mlir/test/Dialect/Vector/invalid.mlir
@@ -186,8 +186,8 @@ func.func @extract_0d(%arg0: vector<f32>) {
 // -----
 
 func.func @extract_position_overflow(%arg0: vector<4x8x16xf32>) {
-  // expected-error@+1 {{expected position attribute #3 to be a non-negative integer smaller than the corresponding vector dimension}}
-  %1 = vector.extract %arg0[0, 0, -1] : f32 from vector<4x8x16xf32>
+  // expected-error@+1 {{expected position attribute #3 to be a non-negative integer smaller than the corresponding vector dimension or poison (-1)}}
+  %1 = vector.extract %arg0[0, 0, -5] : f32 from vector<4x8x16xf32>
 }
 
 // -----
@@ -247,7 +247,7 @@ func.func @insert_vector_type(%a: f32, %b: vector<4x8x16xf32>) {
 
 func.func @insert_position_overflow(%a: f32, %b: vector<4x8x16xf32>) {
   // expected-error@+1 {{expected position attribute #3 to be a non-negative integer smaller than the corresponding dest vector dimension}}
-  %1 = vector.insert %a, %b[0, 0, -1] : f32 into vector<4x8x16xf32>
+  %1 = vector.insert %a, %b[0, 0, -5] : f32 into vector<4x8x16xf32>
 }
 
 // -----
diff --git a/mlir/test/Dialect/Vector/linearize.mlir b/mlir/test/Dialect/Vector/linearize.mlir
index 543e76b5b26e0..99b1bbab1eede 100644
--- a/mlir/test/Dialect/Vector/linearize.mlir
+++ b/mlir/test/Dialect/Vector/linearize.mlir
@@ -179,7 +179,7 @@ func.func @test_extract_strided_slice_1(%arg0 : vector<4x8xf32>) -> vector<2x2xf
 
 // ALL-LABEL:   func.func @test_extract_strided_slice_1_scalable(
 // ALL-SAME:    %[[VAL_0:.*]]: vector<4x[8]xf32>) -> vector<2x[8]xf32> {
-func.func @test_extract_strided_slice_1_scalable(%arg0: vector<4x[8]xf32>) -> vector<2x[8]xf32> {  
+func.func @test_extract_strided_slice_1_scalable(%arg0: vector<4x[8]xf32>) -> vector<2x[8]xf32> {
   // ALL-NOT: vector.shuffle
   // ALL-NOT: vector.shape_cast
   // ALL: %[[RES:.*]] = vector.extract_strided_slice %[[VAL_0]] {offsets = [1, 0], sizes = [2, 8], strides = [1, 1]} : vector<4x[8]xf32> to vector<2x[8]xf32>
@@ -318,3 +318,68 @@ func.func @test_vector_extract_scalar() {
   %0 = vector.extract %cst[0] : i32 from vector<4xi32>
   return
 }
+
+// -----
+
+// ALL-LABEL: test_vector_bitcast
+// ALL-SAME: %[[ARG_0:.*]]: vector<4x4xf32>
+func.func @test_vector_bitcast(%arg0: vector<4x4xf32>) -> vector<4x8xf16> {
+  // DEFAULT: %[[DOWNCAST:.*]] = vector.shape_cast %[[ARG_0]] : vector<4x4xf32> to vector<16xf32>
+  // DEFAULT: %[[BITCAST:.*]] = vector.bitcast %[[DOWNCAST]] : vector<16xf32> to vector<32xf16>
+  // DEFAULT: %[[UPCAST:.*]] = vector.shape_cast %[[BITCAST]] : vector<32xf16> to vector<4x8xf16>
+
+  // BW-128: %[[UPCAST:.*]] = vector.bitcast %[[ARG_0]] : vector<4x4xf32> to vector<4x8xf16>
+  // BW-0: %[[BITCAST:.*]] = vector.bitcast %[[ARG_0]] : vector<4x4xf32> to vector<4x8xf16>
+  %1 = vector.bitcast %arg0 : vector<4x4xf32> to vector<4x8xf16>
+  return %1 : vector<4x8xf16>
+}
+
+// -----
+
+// ALL-LABEL: test_vector_bitcast
+// ALL-SAME: %[[ARG_0:.*]]: vector<4x2xf32>
+func.func @test_vector_bitcast(%arg0: vector<4x2xf32>) -> vector<4x4xf16> {
+  // DEFAULT: %[[DOWNCAST:.*]] = vector.shape_cast %[[ARG_0]] : vector<4x2xf32> to vector<8xf32>
+  // DEFAULT: %[[BITCAST:.*]] = vector.bitcast %[[DOWNCAST]] : vector<8xf32> to vector<16xf16>
+  // DEFAULT: %[[UPCAST:.*]] = vector.shape_cast %[[BITCAST]] : vector<16xf16> to vector<4x4xf16>
+  // BW-128: %[[DOWNCAST:.*]] = vector.shape_cast %[[ARG_0]] : vector<4x2xf32> to vector<8xf32>
+  // BW-128: %[[BITCAST:.*]] = vector.bitcast %[[DOWNCAST]] : vector<8xf32> to vector<16xf16>
+  // BW-128: %[[UPCAST:.*]] = vector.shape_cast %[[BITCAST]] : vector<16xf16> to vector<4x4xf16>
+
+  // BW-0: %[[BITCAST:.*]] = vector.bitcast %[[ARG_0]] : vector<4x2xf32> to vector<4x4xf16>
+  %1 = vector.bitcast %arg0 : vector<4x2xf32> to vector<4x4xf16>
+  return %1 : vector<4x4xf16>
+}
+
+// -----
+
+// ALL-LABEL: test_vector_bitcast
+// ALL-SAME: %[[ARG_0:.*]]: vector<4x[2]xf32>
+func.func @test_vector_bitcast(%arg0: vector<4x[2]xf32>) -> vector<4x[4]xf16> {
+  // DEFAULT: %[[DOWNCAST:.*]] = vector.shape_cast %[[ARG_0]] : vector<4x[2]xf32> to vector<[8]xf32>
+  // DEFAULT: %[[BITCAST:.*]] = vector.bitcast %[[DOWNCAST]] : vector<[8]xf32> to vector<[16]xf16>
+  // DEFAULT: %[[UPCAST:.*]] = vector.shape_cast %[[BITCAST]] : vector<[16]xf16> to vector<4x[4]xf16>
+  // BW-128: %[[DOWNCAST:.*]] = vector.shape_cast %[[ARG_0]] : vector<4x[2]xf32> to vector<[8]xf32>
+  // BW-128: %[[BITCAST:.*]] = vector.bitcast %[[DOWNCAST]] : vector<[8]xf32> to vector<[16]xf16>
+  // BW-128: %[[UPCAST:.*]] = vector.shape_cast %[[BITCAST]] : vector<[16]xf16> to vector<4x[4]xf16>
+
+  // BW-0: %[[BITCAST:.*]] = vector.bitcast %[[ARG_0]] : vector<4x[2]xf32> to vector<4x[4]xf16>
+  %1 = vector.bitcast %arg0 : vector<4x[2]xf32> to vector<4x[4]xf16>
+  return %1 : vector<4x[4]xf16>
+}
+
+// -----
+// ALL-LABEL: test_vector_bitcast
+// ALL-SAME: %[[ARG_0:.*]]: vector<[4]x2xf32>
+func.func @test_vector_bitcast(%arg0: vector<[4]x2xf32>) -> vector<[4]x4xf16> {
+  // DEFAULT: %[[DOWNCAST:.*]] = vector.shape_cast %[[ARG_0]] : vector<[4]x2xf32> to vector<[8]xf32>
+  // DEFAULT: %[[BITCAST:.*]] = vector.bitcast %[[DOWNCAST]] : vector<[8]xf32> to vector<[16]xf16>
+  // DEFAULT: %[[UPCAST:.*]] = vector.shape_cast %[[BITCAST]] : vector<[16]xf16> to vector<[4]x4xf16>
+  // BW-128: %[[DOWNCAST:.*]] = vector.shape_cast %[[ARG_0]] : vector<[4]x2xf32> to vector<[8]xf32>
+  // BW-128: %[[BITCAST:.*]] = vector.bitcast %[[DOWNCAST]] : vector<[8]xf32> to vector<[16]xf16>
+  // BW-128: %[[UPCAST:.*]] = vector.shape_cast %[[BITCAST]] : vector<[16]xf16> to vector<[4]x4xf16>
+
+  // BW-0: %[[BITCAST:.*]] = vector.bitcast %[[ARG_0]] : vector<[4]x2xf32> to vector<[4]x4xf16>
+  %1 = vector.bitcast %arg0 : vector<[4]x2xf32> to vector<[4]x4xf16>
+  return %1 : vector<[4]x4xf16>
+}
diff --git a/mlir/test/Dialect/Vector/ops.mlir b/mlir/test/Dialect/Vector/ops.mlir
index cd6f3f518a1c0..67484e06f456d 100644
--- a/mlir/test/Dialect/Vector/ops.mlir
+++ b/mlir/test/Dialect/Vector/ops.mlir
@@ -247,6 +247,13 @@ func.func @extract_0d(%a: vector<f32>) -> f32 {
   return %0 : f32
 }
 
+// CHECK-LABEL: @extract_poison_idx
+func.func @extract_poison_idx(%a: vector<4x5xf32>) -> f32 {
+  // CHECK-NEXT: vector.extract %{{.*}}[-1, 0] : f32 from vector<4x5xf32>
+  %0 = vector.extract %a[-1, 0] : f32 from vector<4x5xf32>
+  return %0 : f32
+}
+
 // CHECK-LABEL: @insert_element_0d
 func.func @insert_element_0d(%a: f32, %b: vector<f32>) -> vector<f32> {
   // CHECK-NEXT: vector.insertelement %{{.*}}, %{{.*}}[] : vector<f32>
@@ -299,6 +306,13 @@ func.func @insert_0d(%a: f32, %b: vector<f32>, %c: vector<2x3xf32>) -> (vector<f
   return %1, %2 : vector<f32>, vector<2x3xf32>
 }
 
+// CHECK-LABEL: @insert_poison_idx
+func.func @insert_poison_idx(%a: vector<4x5xf32>, %b: f32) {
+  // CHECK-NEXT: vector.insert %{{.*}}, %{{.*}}[-1, 0] : f32 into vector<4x5xf32>
+  vector.insert %b, %a[-1, 0] : f32 into vector<4x5xf32>
+  return
+}
+
 // CHECK-LABEL: @outerproduct
 func.func @outerproduct(%arg0: vector<4xf32>, %arg1: vector<8xf32>, %arg2: vector<4x8xf32>) -> vector<4x8xf32> {
   // CHECK: vector.outerproduct {{.*}} : vector<4xf32>, vector<8xf32>
diff --git a/mlir/test/Dialect/Vector/vector-bitcast-lowering-transforms.mlir b/mlir/test/Dialect/Vector/vector-bitcast-lowering-transforms.mlir
index 346291019451c..29e7007666e87 100644
--- a/mlir/test/Dialect/Vector/vector-bitcast-lowering-transforms.mlir
+++ b/mlir/test/Dialect/Vector/vector-bitcast-lowering-transforms.mlir
@@ -24,7 +24,7 @@ func.func @vector_bitcast_2d(%arg0: vector<2x4xi32>) -> vector<2x2xi64> {
 }
 // CHECK-LABEL: func.func @vector_bitcast_2d
 // CHECK-SAME:    %[[IN:[a-zA-Z0-9]+]]
-// CHECK:         %[[INIT:.+]] = arith.constant {{.+}} : vector<2x2xi64>
+// CHECK:         %[[INIT:.+]] = ub.poison : vector<2x2xi64>
 // CHECK:         %[[V1:.+]] = vector.extract %[[IN]][0] : vector<4xi32> from vector<2x4xi32>
 // CHECK:         %[[B1:.+]] = vector.bitcast %[[V1]] : vector<4xi32> to vector<2xi64>
 // CHECK:         %[[R1:.+]] = vector.insert %[[B1]], %[[INIT]] [0]
@@ -39,7 +39,7 @@ func.func @vector_bitcast_4d_with_scalable_dim(%arg0: vector<1x2x[3]x4xi64>) ->
 }
 // CHECK-LABEL: func.func @vector_bitcast_4d_with_scalable_dim
 // CHECK-SAME:    %[[IN:[a-zA-Z0-9]+]]
-// CHECK:         %[[INIT:.+]] = arith.constant dense<0> : vector<1x2x[3]x8xi32>
+// CHECK:         %[[INIT:.+]] = ub.poison : vector<1x2x[3]x8xi32>
 // CHECK:         %[[V1:.+]] = vector.extract %[[IN]][0, 0] : vector<[3]x4xi64> from vector<1x2x[3]x4xi64>
 // CHECK:         %[[B1:.+]] = vector.bitcast %[[V1]] : vector<[3]x4xi64> to vector<[3]x8xi32>
 // CHECK:         %[[R1:.+]] = vector.insert %[[B1]], %[[INIT]] [0, 0] : vector<[3]x8xi32> into vector<1x2x[3]x8xi32>
@@ -54,7 +54,7 @@ func.func @vector_bitcast_2d_trailing_scalable_dim(%arg0: vector<2x[2]xi64>) ->
 }
 // CHECK-LABEL: func.func @vector_bitcast_2d_trailing_scalable_dim
 // CHECK-SAME:    %[[IN:[a-zA-Z0-9]+]]
-// CHECK:         %[[INIT:.+]] = arith.constant dense<0> : vector<2x[4]xi32>
+// CHECK:         %[[INIT:.+]] = ub.poison : vector<2x[4]xi32>
 // CHECK:         %[[V1:.+]] = vector.extract %[[IN]][0] : vector<[2]xi64> from vector<2x[2]xi64>
 // CHECK:         %[[B1:.+]] = vector.bitcast %[[V1]] : vector<[2]xi64> to vector<[4]xi32>
 // CHECK:         %[[R1:.+]] = vector.insert %[[B1]], %[[INIT]] [0] : vector<[4]xi32> into vector<2x[4]xi32>
diff --git a/mlir/test/Dialect/Vector/vector-broadcast-lowering-transforms.mlir b/mlir/test/Dialect/Vector/vector-broadcast-lowering-transforms.mlir
index 4a5ea439134cf..8e167a520260f 100644
--- a/mlir/test/Dialect/Vector/vector-broadcast-lowering-transforms.mlir
+++ b/mlir/test/Dialect/Vector/vector-broadcast-lowering-transforms.mlir
@@ -41,8 +41,8 @@ func.func @broadcast_vec1d_from_vec1d(%arg0: vector<2xf32>) -> vector<2xf32> {
 
 // CHECK-LABEL: func @broadcast_vec2d_from_vec1d
 // CHECK-SAME: %[[A:.*0]]: vector<2xf32>
-// CHECK:      %[[C0:.*]] = arith.constant dense<0.000000e+00> : vector<3x2xf32>
-// CHECK:      %[[T0:.*]] = vector.insert %[[A]], %[[C0]] [0] : vector<2xf32> into vector<3x2xf32>
+// CHECK:      %[[U0:.*]] = ub.poison : vector<3x2xf32>
+// CHECK:      %[[T0:.*]] = vector.insert %[[A]], %[[U0]] [0] : vector<2xf32> into vector<3x2xf32>
 // CHECK:      %[[T1:.*]] = vector.insert %[[A]], %[[T0]] [1] : vector<2xf32> into vector<3x2xf32>
 // CHECK:      %[[T2:.*]] = vector.insert %[[A]], %[[T1]] [2] : vector<2xf32> into vector<3x2xf32>
 // CHECK:      return %[[T2]] : vector<3x2xf32>
@@ -54,12 +54,12 @@ func.func @broadcast_vec2d_from_vec1d(%arg0: vector<2xf32>) -> vector<3x2xf32> {
 
 // CHECK-LABEL: func @broadcast_vec3d_from_vec1d
 // CHECK-SAME: %[[A:.*0]]: vector<2xf32>
-// CHECK-DAG:  %[[C0:.*]] = arith.constant dense<0.000000e+00> : vector<3x2xf32>
-// CHECK-DAG:  %[[C1:.*]] = arith.constant dense<0.000000e+00> : vector<4x3x2xf32>
-// CHECK:      %[[T0:.*]] = vector.insert %[[A]], %[[C0]] [0] : vector<2xf32> into vector<3x2xf32>
+// CHECK-DAG:  %[[U0:.*]] = ub.poison : vector<3x2xf32>
+// CHECK-DAG:  %[[U1:.*]] = ub.poison : vector<4x3x2xf32>
+// CHECK:      %[[T0:.*]] = vector.insert %[[A]], %[[U0]] [0] : vector<2xf32> into vector<3x2xf32>
 // CHECK:      %[[T1:.*]] = vector.insert %[[A]], %[[T0]] [1] : vector<2xf32> into vector<3x2xf32>
 // CHECK:      %[[T2:.*]] = vector.insert %[[A]], %[[T1]] [2] : vector<2xf32> into vector<3x2xf32>
-// CHECK:      %[[T3:.*]] = vector.insert %[[T2]], %[[C1]] [0] : vector<3x2xf32> into vector<4x3x2xf32>
+// CHECK:      %[[T3:.*]] = vector.insert %[[T2]], %[[U1]] [0] : vector<3x2xf32> into vector<4x3x2xf32>
 // CHECK:      %[[T4:.*]] = vector.insert %[[T2]], %[[T3]] [1] : vector<3x2xf32> into vector<4x3x2xf32>
 // CHECK:      %[[T5:.*]] = vector.insert %[[T2]], %[[T4]] [2] : vector<3x2xf32> into vector<4x3x2xf32>
 // CHECK:      %[[T6:.*]] = vector.insert %[[T2]], %[[T5]] [3] : vector<3x2xf32> into vector<4x3x2xf32>
@@ -72,8 +72,8 @@ func.func @broadcast_vec3d_from_vec1d(%arg0: vector<2xf32>) -> vector<4x3x2xf32>
 
 // CHECK-LABEL: func @broadcast_vec3d_from_vec2d
 // CHECK-SAME: %[[A:.*0]]: vector<3x2xf32>
-// CHECK:      %[[C0:.*]] = arith.constant dense<0.000000e+00> : vector<4x3x2xf32>
-// CHECK:      %[[T0:.*]] = vector.insert %[[A]], %[[C0]] [0] : vector<3x2xf32> into vector<4x3x2xf32>
+// CHECK:      %[[U0:.*]] = ub.poison : vector<4x3x2xf32>
+// CHECK:      %[[T0:.*]] = vector.insert %[[A]], %[[U0]] [0] : vector<3x2xf32> into vector<4x3x2xf32>
 // CHECK:      %[[T1:.*]] = vector.insert %[[A]], %[[T0]] [1] : vector<3x2xf32> into vector<4x3x2xf32>
 // CHECK:      %[[T2:.*]] = vector.insert %[[A]], %[[T1]] [2] : vector<3x2xf32> into vector<4x3x2xf32>
 // CHECK:      %[[T3:.*]] = vector.insert %[[A]], %[[T2]] [3] : vector<3x2xf32> into vector<4x3x2xf32>
@@ -97,9 +97,9 @@ func.func @broadcast_stretch(%arg0: vector<1xf32>) -> vector<4xf32> {
 
 // CHECK-LABEL: func @broadcast_stretch_at_start
 // CHECK-SAME: %[[A:.*0]]: vector<1x4xf32>
-// CHECK:      %[[C0:.*]] = arith.constant dense<0.000000e+00> : vector<3x4xf32>
+// CHECK:      %[[U0:.*]] = ub.poison : vector<3x4xf32>
 // CHECK:      %[[T0:.*]] = vector.extract %[[A]][0] : vector<4xf32> from vector<1x4xf32>
-// CHECK:      %[[T1:.*]] = vector.insert %[[T0]], %[[C0]] [0] : vector<4xf32> into vector<3x4xf32>
+// CHECK:      %[[T1:.*]] = vector.insert %[[T0]], %[[U0]] [0] : vector<4xf32> into vector<3x4xf32>
 // CHECK:      %[[T2:.*]] = vector.insert %[[T0]], %[[T1]] [1] : vector<4xf32> into vector<3x4xf32>
 // CHECK:      %[[T3:.*]] = vector.insert %[[T0]], %[[T2]] [2] : vector<4xf32> into vector<3x4xf32>
 // CHECK:      return %[[T3]] : vector<3x4xf32>
@@ -111,10 +111,10 @@ func.func @broadcast_stretch_at_start(%arg0: vector<1x4xf32>) -> vector<3x4xf32>
 
 // CHECK-LABEL: func @broadcast_stretch_at_end
 // CHECK-SAME: %[[A:.*0]]: vector<4x1xf32>
-// CHECK:      %[[C0:.*]] = arith.constant dense<0.000000e+00> : vector<4x3xf32>
+// CHECK:      %[[U0:.*]] = ub.poison : vector<4x3xf32>
 // CHECK:      %[[T0:.*]] = vector.extract %[[A]][0, 0] : f32 from vector<4x1xf32>
 // CHECK:      %[[T2:.*]] = vector.splat %[[T0]] : vector<3xf32>
-// CHECK:      %[[T3:.*]] = vector.insert %[[T2]], %[[C0]] [0] : vector<3xf32> into vector<4x3xf32>
+// CHECK:      %[[T3:.*]] = vector.insert %[[T2]], %[[U0]] [0] : vector<3xf32> into vector<4x3xf32>
 // CHECK:      %[[T4:.*]] = vector.extract %[[A]][1, 0] : f32 from vector<4x1xf32>
 // CHECK:      %[[T6:.*]] = vector.splat %[[T4]] : vector<3xf32>
 // CHECK:      %[[T7:.*]] = vector.insert %[[T6]], %[[T3]] [1] : vector<3xf32> into vector<4x3xf32>
@@ -133,25 +133,25 @@ func.func @broadcast_stretch_at_end(%arg0: vector<4x1xf32>) -> vector<4x3xf32> {
 
 // CHECK-LABEL: func @broadcast_stretch_in_middle
 // CHECK-SAME: %[[A:.*0]]: vector<4x1x2xf32>
-// CHECK:      %[[C0:.*]] = arith.constant dense<0.000000e+00> : vector<4x3x2xf32>
-// CHECK:      %[[C1:.*]] = arith.constant dense<0.000000e+00> : vector<3x2xf32>
+// CHECK:      %[[U0:.*]] = ub.poison : vector<4x3x2xf32>
+// CHECK:      %[[U1:.*]] = ub.poison : vector<3x2xf32>
 // CHECK:      %[[T0:.*]] = vector.extract %[[A]][0, 0] : vector<2xf32> from vector<4x1x2xf32>
-// CHECK:      %[[T2:.*]] = vector.insert %[[T0]], %[[C1]] [0] : vector<2xf32> into vector<3x2xf32>
+// CHECK:      %[[T2:.*]] = vector.insert %[[T0]], %[[U1]] [0] : vector<2xf32> into vector<3x2xf32>
 // CHECK:      %[[T3:.*]] = vector.insert %[[T0]], %[[T2]] [1] : vector<2xf32> into vector<3x2xf32>
 // CHECK:      %[[T4:.*]] = vector.insert %[[T0]], %[[T3]] [2] : vector<2xf32> into vector<3x2xf32>
-// CHECK:      %[[T5:.*]] = vector.insert %[[T4]], %[[C0]] [0] : vector<3x2xf32> into vector<4x3x2xf32>
+// CHECK:      %[[T5:.*]] = vector.insert %[[T4]], %[[U0]] [0] : vector<3x2xf32> into vector<4x3x2xf32>
 // CHECK:      %[[T6:.*]] = vector.extract %[[A]][1, 0] : vector<2xf32> from vector<4x1x2xf32>
-// CHECK:      %[[T8:.*]] = vector.insert %[[T6]], %[[C1]] [0] : vector<2xf32> into vector<3x2xf32>
+// CHECK:      %[[T8:.*]] = vector.insert %[[T6]], %[[U1]] [0] : vector<2xf32> into vector<3x2xf32>
 // CHECK:      %[[T9:.*]] = vector.insert %[[T6]], %[[T8]] [1] : vector<2xf32> into vector<3x2xf32>
 // CHECK:      %[[T10:.*]] = vector.insert %[[T6]], %[[T9]] [2] : vector<2xf32> into vector<3x2xf32>
 // CHECK:      %[[T11:.*]] = vector.insert %[[T10]], %[[T5]] [1] : vector<3x2xf32> into vector<4x3x2xf32>
 // CHECK:      %[[T12:.*]] = vector.extract %[[A]][2, 0] : vector<2xf32> from vector<4x1x2xf32>
-// CHECK:      %[[T14:.*]] = vector.insert %[[T12]], %[[C1]] [0] : vector<2xf32> into vector<3x2xf32>
+// CHECK:      %[[T14:.*]] = vector.insert %[[T12]], %[[U1]] [0] : vector<2xf32> into vector<3x2xf32>
 // CHECK:      %[[T15:.*]] = vector.insert %[[T12]], %[[T14]] [1] : vector<2xf32> into vector<3x2xf32>
 // CHECK:      %[[T16:.*]] = vector.insert %[[T12]], %[[T15]] [2] : vector<2xf32> into vector<3x2xf32>
 // CHECK:      %[[T17:.*]] = vector.insert %[[T16]], %[[T11]] [2] : vector<3x2xf32> into vector<4x3x2xf32>
 // CHECK:      %[[T18:.*]] = vector.extract %[[A]][3, 0] : vector<2xf32> from vector<4x1x2xf32>
-// CHECK:      %[[T20:.*]] = vector.insert %[[T18]], %[[C1]] [0] : vector<2xf32> into vector<3x2xf32>
+// CHECK:      %[[T20:.*]] = vector.insert %[[T18]], %[[U1]] [0] : vector<2xf32> into vector<3x2xf32>
 // CHECK:      %[[T21:.*]] = vector.insert %[[T18]], %[[T20]] [1] : vector<2xf32> into vector<3x2xf32>
 // CHECK:      %[[T22:.*]] = vector.insert %[[T18]], %[[T21]] [2] : vector<2xf32> into vector<3x2xf32>
 // CHECK:      %[[T23:.*]] = vector.insert %[[T22]], %[[T17]] [3] : vector<3x2xf32> into vector<4x3x2xf32>
@@ -164,8 +164,8 @@ func.func @broadcast_stretch_in_middle(%arg0: vector<4x1x2xf32>) -> vector<4x3x2
 
 // CHECK-LABEL:   func.func @broadcast_scalable_duplication
 // CHECK-SAME:      %[[ARG0:.*]]: vector<[32]xf32>)
-// CHECK:           %[[CST:.*]] = arith.constant dense<0.000000e+00> : vector<1x[32]xf32>
-// CHECK:           %[[RES:.*]] = vector.insert %[[ARG0]], %[[CST]] [0] : vector<[32]xf32> into vector<1x[32]xf32>
+// CHECK:           %[[INIT:.*]] = ub.poison : vector<1x[32]xf32>
+// CHECK:           %[[RES:.*]] = vector.insert %[[ARG0]], %[[INIT]] [0] : vector<[32]xf32> into vector<1x[32]xf32>
 // CHECK:           return %[[RES]] : vector<1x[32]xf32>
 
 func.func @broadcast_scalable_duplication(%arg0: vector<[32]xf32>) -> vector<1x[32]xf32> {
diff --git a/mlir/test/Dialect/Vector/vector-contract-to-matrix-intrinsics-transforms.mlir b/mlir/test/Dialect/Vector/vector-contract-to-matrix-intrinsics-transforms.mlir
index 4867a416e5d14..08ac2ac5bb7d5 100644
--- a/mlir/test/Dialect/Vector/vector-contract-to-matrix-intrinsics-transforms.mlir
+++ b/mlir/test/Dialect/Vector/vector-contract-to-matrix-intrinsics-transforms.mlir
@@ -14,15 +14,15 @@
 // CHECK-SAME: %[[A:[a-zA-Z0-9]*]]: vector<2x4xf32>,
 // CHECK-SAME: %[[B:[a-zA-Z0-9]*]]: vector<4x3xf32>,
 // CHECK-SAME: %[[C:[a-zA-Z0-9]*]]: vector<2x3xf32>
-//  CHECK-DAG:  %[[vcst:.*]] = arith.constant dense<0.000000e+00> : vector<8xf32>
-//  CHECK-DAG:  %[[vcst_0:.*]] = arith.constant dense<0.000000e+00> : vector<12xf32>
-//  CHECK-DAG:  %[[vcst_1:.*]] = arith.constant dense<0.000000e+00> : vector<2x3xf32>
+//  CHECK-DAG:  %[[ub:.*]] = ub.poison : vector<8xf32>
+//  CHECK-DAG:  %[[ub_0:.*]] = ub.poison : vector<12xf32>
+//  CHECK-DAG:  %[[ub_1:.*]] = ub.poison : vector<2x3xf32>
 //      CHECK:  %[[a0:.*]] = vector.extract %[[A]][0] : vector<4xf32> from vector<2x4xf32>
-//      CHECK:  %[[a1:.*]] = vector.insert_strided_slice %[[a0]], %[[vcst]] {offsets = [0], strides = [1]} : vector<4xf32> into vector<8xf32>
+//      CHECK:  %[[a1:.*]] = vector.insert_strided_slice %[[a0]], %[[ub]] {offsets = [0], strides = [1]} : vector<4xf32> into vector<8xf32>
 //      CHECK:  %[[a2:.*]] = vector.extract %[[A]][1] : vector<4xf32> from vector<2x4xf32>
 //      CHECK:  %[[a3:.*]] = vector.insert_strided_slice %[[a2]], %[[a1]] {offsets = [4], strides = [1]} : vector<4xf32> into vector<8xf32>
 //      CHECK:  %[[b0:.*]] = vector.extract %[[B]][0] : vector<3xf32> from vector<4x3xf32>
-//      CHECK:  %[[b1:.*]] = vector.insert_strided_slice %[[b0]], %[[vcst_0]] {offsets = [0], strides = [1]} : vector<3xf32> into vector<12xf32>
+//      CHECK:  %[[b1:.*]] = vector.insert_strided_slice %[[b0]], %[[ub_0]] {offsets = [0], strides = [1]} : vector<3xf32> into vector<12xf32>
 //      CHECK:  %[[b2:.*]] = vector.extract %[[B]][1] : vector<3xf32> from vector<4x3xf32>
 //      CHECK:  %[[b3:.*]] = vector.insert_strided_slice %[[b2]], %[[b1]] {offsets = [3], strides = [1]} : vector<3xf32> into vector<12xf32>
 //      CHECK:  %[[b4:.*]] = vector.extract %[[B]][2] : vector<3xf32> from vector<4x3xf32>
@@ -31,7 +31,7 @@
 //      CHECK:  %[[b7:.*]] = vector.insert_strided_slice %[[b6]], %[[b5]] {offsets = [9], strides = [1]} : vector<3xf32> into vector<12xf32>
 //      CHECK:  %[[mm1:.*]] = vector.matrix_multiply %[[a3]], %[[b7]] {lhs_columns = 4 : i32, lhs_rows = 2 : i32, rhs_columns = 3 : i32} : (vector<8xf32>, vector<12xf32>) -> vector<6xf32>
 //      CHECK:  %[[mm2:.*]] = vector.extract_strided_slice %[[mm1]] {offsets = [0], sizes = [3], strides = [1]} : vector<6xf32> to vector<3xf32>
-//      CHECK:  %[[mm3:.*]] = vector.insert %[[mm2]], %[[vcst_1]] [0] : vector<3xf32> into vector<2x3xf32>
+//      CHECK:  %[[mm3:.*]] = vector.insert %[[mm2]], %[[ub_1]] [0] : vector<3xf32> into vector<2x3xf32>
 //      CHECK:  %[[mm4:.*]] = vector.extract_strided_slice %[[mm1]] {offsets = [3], sizes = [3], strides = [1]} : vector<6xf32> to vector<3xf32>
 //      CHECK:  %[[mm5:.*]] = vector.insert %[[mm4]], %[[mm3]] [1] : vector<3xf32> into vector<2x3xf32>
 //      CHECK:  %[[mm6:.*]] = arith.addf %[[C]], %[[mm5]] : vector<2x3xf32>
diff --git a/mlir/test/Dialect/Vector/vector-emulate-narrow-type-unaligned-non-atomic.mlir b/mlir/test/Dialect/Vector/vector-emulate-narrow-type-unaligned-non-atomic.mlir
new file mode 100644
index 0000000000000..1d6263535ae80
--- /dev/null
+++ b/mlir/test/Dialect/Vector/vector-emulate-narrow-type-unaligned-non-atomic.mlir
@@ -0,0 +1,128 @@
+// RUN: mlir-opt --test-emulate-narrow-int="arith-compute-bitwidth=1 memref-load-bitwidth=8 disable-atomic-rmw=true" --cse --split-input-file %s | FileCheck %s
+
+// TODO: remove memref.alloc() in the tests to eliminate noises.
+// memref.alloc exists here because sub-byte vector data types such as i2
+// are currently not supported as input arguments.
+
+///----------------------------------------------------------------------------------------
+/// vector.store
+///----------------------------------------------------------------------------------------
+
+func.func @vector_store_i2_const_index_two_partial_stores(%arg0: vector<3xi2>) {
+    %0 = memref.alloc() : memref<3x3xi2>
+    %c0 = arith.constant 0 : index
+    %c2 = arith.constant 2 : index
+    vector.store %arg0, %0[%c2, %c0] :memref<3x3xi2>, vector<3xi2>
+    return
+}
+
+// Emit two non-atomic RMW partial stores. Store 6 bits from the input vector (bits [12:18)),
+// into bytes [1:2] from a 3-byte output memref. Due to partial storing,
+// both bytes are accessed partially through masking.
+
+// CHECK: func @vector_store_i2_const_index_two_partial_stores(
+// CHECK-SAME: %[[ARG0:.+]]: vector<3xi2>)
+// CHECK: %[[ALLOC:.+]] = memref.alloc() : memref<3xi8>
+// CHECK: %[[C1:.+]] = arith.constant 1 : index
+
+// Part 1 RMW sequence
+// CHECK: %[[CST:.+]] = arith.constant dense<[false, false, true, true]>
+// CHECK: %[[CST0:.+]] = arith.constant dense<0> : vector<4xi2>
+// CHECK: %[[EXTRACT:.+]] = vector.extract_strided_slice %[[ARG0]]
+// CHECK-SAME: {offsets = [0], sizes = [2], strides = [1]} : vector<3xi2> to vector<2xi2>
+// CHECK: %[[INSERT:.+]] = vector.insert_strided_slice %[[EXTRACT]], %[[CST0]]
+// CHECK-SAME: {offsets = [2], strides = [1]} : vector<2xi2> into vector<4xi2>
+// CHECK: %[[LOAD:.+]] = vector.load
+// CHECK: %[[DOWNCAST:.+]] = vector.bitcast %[[LOAD]] : vector<1xi8> to vector<4xi2>
+// CHECK: %[[SELECT:.+]] = arith.select %[[CST]], %[[INSERT]], %[[DOWNCAST]]
+// CHECK: %[[UPCAST:.+]] = vector.bitcast %[[SELECT]]
+// CHECK: vector.store %[[UPCAST]], %[[ALLOC]][%[[C1]]]
+
+// Part 2 RMW sequence
+// CHECK: %[[OFFSET:.+]] = arith.addi %[[C1]], %[[C1]] : index
+// CHECK: %[[EXTRACT2:.+]] = vector.extract_strided_slice %[[ARG0]]
+// CHECK-SAME: {offsets = [2], sizes = [1], strides = [1]} : vector<3xi2> to vector<1xi2>
+// CHECK: %[[INSERT2:.+]] = vector.insert_strided_slice %[[EXTRACT2]], %[[CST0]]
+// CHECK-SAME: {offsets = [0], strides = [1]} : vector<1xi2> into vector<4xi2>
+// CHECK: %[[CST1:.+]] = arith.constant dense<[true, false, false, false]> : vector<4xi1>
+// CHECK: %[[LOAD2:.+]] = vector.load
+// CHECK: %[[UPCAST2:.+]] = vector.bitcast %[[LOAD2]] : vector<1xi8> to vector<4xi2>
+// CHECK: %[[SELECT2:.+]] = arith.select %[[CST1]], %[[INSERT2]], %[[UPCAST2]]
+// CHECK: %[[DOWNCAST2:.+]] = vector.bitcast %[[SELECT2]]
+// CHECK: vector.store %[[DOWNCAST2]], %[[ALLOC]][%[[OFFSET]]]
+
+
+// -----
+
+func.func @vector_store_i2_two_partial_one_full_stores(%arg0: vector<7xi2>) {
+    %0 = memref.alloc() : memref<3x7xi2>
+    %c0 = arith.constant 0 : index
+    %c1 = arith.constant 1 : index
+    vector.store %arg0, %0[%c1, %c0] :memref<3x7xi2>, vector<7xi2>
+    return
+}
+
+// In this example, emit two RMW stores and one full-width store.
+
+// CHECK: func @vector_store_i2_two_partial_one_full_stores(
+// CHECK-SAME: %[[ARG0:.+]]:
+// CHECK: %[[ALLOC:.+]] = memref.alloc() : memref<6xi8>
+// CHECK: %[[C1:.+]] = arith.constant 1 : index
+// CHECK: %[[CST:.+]] = arith.constant dense<[false, false, false, true]>
+// CHECK: %[[CST0:.+]] = arith.constant dense<0> : vector<4xi2>
+// CHECK: %[[EXTRACT:.+]] = vector.extract_strided_slice %[[ARG0]]
+// CHECK-SAME: {offsets = [0], sizes = [1], strides = [1]}
+// CHECK: %[[INSERT:.+]] = vector.insert_strided_slice %[[EXTRACT]], %[[CST0]]
+// CHECK-SAME: {offsets = [3], strides = [1]}
+// First sub-width RMW:
+// CHECK: %[[LOAD:.+]] = vector.load %[[ALLOC]][%[[C1]]]
+// CHECK: %[[UPCAST:.+]] = vector.bitcast %[[LOAD]] : vector<1xi8> to vector<4xi2>
+// CHECK: %[[SELECT:.+]] = arith.select %[[CST]], %[[INSERT]], %[[UPCAST]]
+// CHECK: %[[DOWNCAST:.+]] = vector.bitcast %[[SELECT]]
+// CHECK: vector.store %[[DOWNCAST]], %[[ALLOC]][%[[C1]]]
+
+// Full-width store:
+// CHECK: %[[INDEX:.+]] = arith.addi %[[C1]], %[[C1]]
+// CHECK: %[[EXTRACT1:.+]] = vector.extract_strided_slice %[[ARG0]]
+// CHECK-SAME: {offsets = [1], sizes = [4], strides = [1]}
+// CHECK: %[[BITCAST:.+]] = vector.bitcast %[[EXTRACT1]]
+// CHECK: vector.store %[[BITCAST]], %[[ALLOC]][%[[INDEX]]]
+
+// Second sub-width RMW:
+// CHECK: %[[INDEX2:.+]] = arith.addi %[[INDEX]], %[[C1]]
+// CHECK: %[[EXTRACT2:.+]] = vector.extract_strided_slice %[[ARG0]]
+// CHECK-SAME: {offsets = [5], sizes = [2], strides = [1]}
+// CHECK: %[[INSERT2:.+]] = vector.insert_strided_slice %[[EXTRACT2]]
+// CHECK-SAME: {offsets = [0], strides = [1]}
+// CHECK: %[[CST1:.+]] = arith.constant dense<[true, true, false, false]>
+// CHECK: %[[LOAD2:.+]] = vector.load %[[ALLOC]][%[[INDEX2]]]
+// CHECK: %[[UPCAST2:.+]] = vector.bitcast %[[LOAD2]]
+// CHECK: %[[SELECT2:.+]] = arith.select %[[CST1]], %[[INSERT2]], %[[UPCAST2]]
+// CHECK: %[[DOWNCAST2:.+]] = vector.bitcast %[[SELECT2]]
+// CHECK: vector.store %[[DOWNCAST2]], %[[ALLOC]][%[[INDEX2]]]
+
+// -----
+
+func.func @vector_store_i2_const_index_one_partial_store(%arg0: vector<1xi2>) {
+    %0 = memref.alloc() : memref<4x1xi2>
+    %c0 = arith.constant 0 : index
+    %c1 = arith.constant 1 : index
+    vector.store %arg0, %0[%c1, %c0] :memref<4x1xi2>, vector<1xi2>
+    return
+}
+
+// in this test, only emit partial RMW store as the store is within one byte.
+
+// CHECK: func @vector_store_i2_const_index_one_partial_store(
+// CHECK-SAME: %[[ARG0:.+]]: vector<1xi2>)
+// CHECK: %[[ALLOC:.+]] = memref.alloc() : memref<1xi8>
+// CHECK: %[[C0:.+]] = arith.constant 0 : index
+// CHECK: %[[CST:.+]] = arith.constant dense<[false, true, false, false]>
+// CHECK: %[[CST0:.+]] = arith.constant dense<0> : vector<4xi2>
+// CHECK: %[[INSERT:.+]] = vector.insert_strided_slice %[[ARG0]], %[[CST0]]
+// CHECK-SAME: {offsets = [1], strides = [1]} : vector<1xi2> into vector<4xi2>
+// CHECK: %[[LOAD:.+]] = vector.load %[[ALLOC]][%[[C0]]] : memref<1xi8>, vector<1xi8>
+// CHECK: %[[UPCAST:.+]] = vector.bitcast %[[LOAD]] : vector<1xi8> to vector<4xi2>
+// CHECK: %[[SELECT:.+]] = arith.select %[[CST]], %[[INSERT]], %[[UPCAST]]
+// CHECK: %[[DOWNCAST:.+]] = vector.bitcast %[[SELECT]]
+// CHECK: vector.store %[[DOWNCAST]], %[[ALLOC]][%[[C0]]]
diff --git a/mlir/test/Dialect/Vector/vector-emulate-narrow-type-unaligned.mlir b/mlir/test/Dialect/Vector/vector-emulate-narrow-type-unaligned.mlir
index 4332e80feed42..6fc974200c6f3 100644
--- a/mlir/test/Dialect/Vector/vector-emulate-narrow-type-unaligned.mlir
+++ b/mlir/test/Dialect/Vector/vector-emulate-narrow-type-unaligned.mlir
@@ -356,3 +356,141 @@ func.func @vector_maskedload_i2_constant_mask_unaligned(%passthru: vector<5xi2>)
 // CHECK: %[[RESULT:.+]] = vector.extract_strided_slice %[[SELECT]]
 // CHECK-SAME: {offsets = [1], sizes = [5], strides = [1]} : vector<8xi2> to vector<5xi2>
 // CHECK: return %[[RESULT]] : vector<5xi2>
+
+///----------------------------------------------------------------------------------------
+/// vector.store
+///----------------------------------------------------------------------------------------
+
+func.func @vector_store_i2_const_index_two_partial_stores(%arg0: vector<3xi2>) {
+    %src = memref.alloc() : memref<3x3xi2>
+    %c0 = arith.constant 0 : index
+    %c2 = arith.constant 2 : index
+    vector.store %arg0, %src[%c2, %c0] :memref<3x3xi2>, vector<3xi2>
+    return
+}
+
+// Emit two atomic RMW partial stores. Store 6 bits from the input vector (bits [12:18)),
+// into bytes [1:2] from a 3-byte output memref. Due to partial storing,
+// both bytes are accessed partially through masking.
+
+// CHECK-LABEL: func @vector_store_i2_const_index_two_partial_stores(
+// CHECK-SAME: %[[ARG0:.+]]: vector<3xi2>)
+// CHECK: %[[ALLOC:.+]] = memref.alloc() : memref<3xi8>
+// CHECK: %[[C1:.+]] = arith.constant 1 : index
+// CHECK: %[[CST:.+]] = arith.constant dense<[false, false, true, true]> : vector<4xi1>
+// CHECK: %[[CST_0:.+]] = arith.constant dense<0> : vector<4xi2>
+
+// Part 1 atomic RMW sequence (load bits [12, 16) from %src_as_bytes[1])
+// CHECK: %[[EXTRACT:.+]] = vector.extract_strided_slice %[[ARG0]]
+// CHECK-SAME: {offsets = [0], sizes = [2], strides = [1]} : vector<3xi2> to vector<2xi2>
+// CHECK: %[[INSERT:.+]] = vector.insert_strided_slice %[[EXTRACT]], %[[CST_0]]
+// CHECK-SAME: {offsets = [2], strides = [1]} : vector<2xi2> into vector<4xi2>
+// CHECK: %[[ATOMIC_RMW:.+]] = memref.generic_atomic_rmw %[[ALLOC]][%[[C1]]] : memref<3xi8> {
+// CHECK: %[[ARG:.+]]: i8):
+// CHECK: %[[FROM_ELEM:.+]] = vector.from_elements %[[ARG]] : vector<1xi8>
+// CHECK: %[[BITCAST:.+]] = vector.bitcast %[[FROM_ELEM]] : vector<1xi8> to vector<4xi2>
+// CHECK: %[[SELECT:.+]] = arith.select %[[CST]], %[[INSERT]], %[[BITCAST]] : vector<4xi1>, vector<4xi2>
+// CHECK: %[[BITCAST2:.+]] = vector.bitcast %[[SELECT]] : vector<4xi2> to vector<1xi8>
+// CHECK: %[[EXTRACT2:.+]] = vector.extract %[[BITCAST2]][0] : i8 from vector<1xi8>
+// CHECK: memref.atomic_yield %[[EXTRACT2]] : i8
+
+// Part 2 atomic RMW sequence (load bits [16, 18) from %src_as_bytes[2])
+// CHECK: %[[ADDR2:.+]] = arith.addi %[[C1]], %[[C1]] : index
+// CHECK: %[[EXTRACT3:.+]] = vector.extract_strided_slice %[[ARG0]]
+// CHECK-SAME: {offsets = [2], sizes = [1], strides = [1]} : vector<3xi2> to vector<1xi2>
+// CHECK: %[[INSERT2:.+]] = vector.insert_strided_slice %[[EXTRACT3]], %[[CST_0]]
+// CHECK-SAME: {offsets = [0], strides = [1]} : vector<1xi2> into vector<4xi2>
+// CHECK: %[[CST1:.+]] = arith.constant dense<[true, false, false, false]> : vector<4xi1>
+// CHECK: %[[ATOMIC_RMW2:.+]] = memref.generic_atomic_rmw %[[ALLOC]][%[[ADDR2]]] : memref<3xi8> {
+// CHECK: %[[ARG2:.+]]: i8):
+// CHECK: %[[FROM_ELEM2:.+]] = vector.from_elements %[[ARG2]] : vector<1xi8>
+// CHECK: %[[BITCAST4:.+]] = vector.bitcast %[[FROM_ELEM2]] : vector<1xi8> to vector<4xi2>
+// CHECK: %[[SELECT2:.+]] = arith.select %[[CST1]], %[[INSERT2]], %[[BITCAST4]] : vector<4xi1>, vector<4xi2>
+// CHECK: %[[BITCAST5:.+]] = vector.bitcast %[[SELECT2]] : vector<4xi2> to vector<1xi8>
+// CHECK: %[[EXTRACT4:.+]] = vector.extract %[[BITCAST5]][0] : i8 from vector<1xi8>
+// CHECK: memref.atomic_yield %[[EXTRACT4]] : i8
+
+// -----
+
+func.func @vector_store_i2_two_partial_one_full_stores(%arg0: vector<7xi2>) {
+    %0 = memref.alloc() : memref<3x7xi2>
+    %c0 = arith.constant 0 : index
+    %c1 = arith.constant 1 : index
+    vector.store %arg0, %0[%c1, %c0] :memref<3x7xi2>, vector<7xi2>
+    return
+}
+
+// In this example, emit 2 atomic RMWs and 1 non-atomic store:
+// CHECK-LABEL: func @vector_store_i2_two_partial_one_full_stores(
+// CHECK-SAME: %[[ARG0:.+]]: vector<7xi2>)
+// CHECK: %[[ALLOC:.+]] = memref.alloc() : memref<6xi8>
+// CHECK: %[[C1:.+]] = arith.constant 1 : index
+// CHECK: %[[CST:.+]] = arith.constant dense<[false, false, false, true]> : vector<4xi1>
+// CHECK: %[[CST0:.+]] = arith.constant dense<0> : vector<4xi2>
+
+// First atomic RMW:
+// CHECK: %[[EXTRACT:.+]] = vector.extract_strided_slice %[[ARG0]]
+// CHECK-SAME: {offsets = [0], sizes = [1], strides = [1]} : vector<7xi2> to vector<1xi2>
+// CHECK: %[[INSERT:.+]] = vector.insert_strided_slice %[[EXTRACT]], %[[CST0]]
+// CHECK-SAME: {offsets = [3], strides = [1]} : vector<1xi2> into vector<4xi2>
+// CHECK: %[[ATOMIC_RMW:.+]] = memref.generic_atomic_rmw %[[ALLOC]][%[[C1]]] : memref<6xi8> {
+// CHECK: %[[ARG:.+]]: i8):
+// CHECK: %[[FROM_ELEM:.+]] = vector.from_elements %[[ARG]] : vector<1xi8>
+// CHECK: %[[BITCAST:.+]] = vector.bitcast %[[FROM_ELEM]] : vector<1xi8> to vector<4xi2>
+// CHECK: %[[SELECT:.+]] = arith.select %[[CST]], %[[INSERT]], %[[BITCAST]] : vector<4xi1>, vector<4xi2>
+// CHECK: %[[BITCAST2:.+]] = vector.bitcast %[[SELECT]] : vector<4xi2> to vector<1xi8>
+// CHECK: %[[EXTRACT2:.+]] = vector.extract %[[BITCAST2]][0] : i8 from vector<1xi8>
+// CHECK: memref.atomic_yield %[[EXTRACT2]] : i8
+
+// Non-atomic store:
+// CHECK: %[[ADDR:.+]] = arith.addi %[[C1]], %[[C1]] : index
+// CHECK: %[[EXTRACT2:.+]] = vector.extract_strided_slice %[[ARG0]]
+// CHECK-SAME: {offsets = [1], sizes = [4], strides = [1]} : vector<7xi2> to vector<4xi2>
+// CHECK: %[[BITCAST3:.+]] = vector.bitcast %[[EXTRACT2]] : vector<4xi2> to vector<1xi8>
+// CHECK: vector.store %[[BITCAST3]], %[[ALLOC]][%[[ADDR]]] : memref<6xi8>, vector<1xi8>
+
+// Second atomic RMW:
+// CHECK: %[[ADDR2:.+]] = arith.addi %[[ADDR]], %[[C1]] : index
+// CHECK: %[[EXTRACT3:.+]] = vector.extract_strided_slice %[[ARG0]]
+// CHECK-SAME: {offsets = [5], sizes = [2], strides = [1]} : vector<7xi2> to vector<2xi2>
+// CHECK: %[[INSERT2:.+]] = vector.insert_strided_slice %[[EXTRACT3]], %[[CST0]]
+// CHECK-SAME: {offsets = [0], strides = [1]} : vector<2xi2> into vector<4xi2>
+// CHECK: %[[CST1:.+]] = arith.constant dense<[true, true, false, false]> : vector<4xi1> 
+// CHECK: %[[ATOMIC_RMW2:.+]] = memref.generic_atomic_rmw %[[ALLOC]][%[[ADDR2]]] : memref<6xi8> {
+// CHECK: %[[ARG2:.+]]: i8):
+// CHECK: %[[FROM_ELEM2:.+]] = vector.from_elements %[[ARG2]] : vector<1xi8>
+// CHECK: %[[BITCAST4:.+]] = vector.bitcast %[[FROM_ELEM2]] : vector<1xi8> to vector<4xi2>
+// CHECK: %[[SELECT2:.+]] = arith.select %[[CST1]], %[[INSERT2]], %[[BITCAST4]] :
+// CHECK-SAME: vector<4xi1>, vector<4xi2>
+// CHECK: %[[BITCAST5:.+]] = vector.bitcast %[[SELECT2]] : vector<4xi2> to vector<1xi8>
+// CHECK: %[[EXTRACT4:.+]] = vector.extract %[[BITCAST5]][0] : i8 from vector<1xi8>
+// CHECK: memref.atomic_yield %[[EXTRACT4]] : i8    
+
+// -----
+
+func.func @vector_store_i2_const_index_one_partial_store(%arg0: vector<1xi2>) {
+    %0 = memref.alloc() : memref<4x1xi2>
+    %c0 = arith.constant 0 : index
+    %c1 = arith.constant 1 : index
+    vector.store %arg0, %0[%c1, %c0] : memref<4x1xi2>, vector<1xi2>
+    return
+}
+
+// In this example, only emit 1 atomic store
+// CHECK-LABEL: func @vector_store_i2_const_index_one_partial_store(
+// CHECK-SAME: %[[ARG0:.+]]: vector<1xi2>)
+// CHECK: %[[ALLOC:.+]] = memref.alloc() : memref<1xi8>
+// CHECK: %[[C0:.+]] = arith.constant 0 : index
+// CHECK: %[[CST:.+]] = arith.constant dense<[false, true, false, false]> : vector<4xi1>
+// CHECK: %[[CST0:.+]] = arith.constant dense<0> : vector<4xi2>
+// CHECK: %[[INSERT:.+]] = vector.insert_strided_slice %[[ARG0]], %[[CST0]]
+// CHECK-SAME: {offsets = [1], strides = [1]} : vector<1xi2> into vector<4xi2>
+
+// CHECK: %[[ATOMIC_RMW:.+]] = memref.generic_atomic_rmw %[[ALLOC]][%[[C0]]] : memref<1xi8> {
+// CHECK: %[[ARG:.+]]: i8):
+// CHECK: %[[FROM_ELEM:.+]] = vector.from_elements %[[ARG]] : vector<1xi8>
+// CHECK: %[[BITCAST:.+]] = vector.bitcast %[[FROM_ELEM]] : vector<1xi8> to vector<4xi2>
+// CHECK: %[[SELECT:.+]] = arith.select %[[CST]], %[[INSERT]], %[[BITCAST]] : vector<4xi1>, vector<4xi2>
+// CHECK: %[[BITCAST2:.+]] = vector.bitcast %[[SELECT]] : vector<4xi2> to vector<1xi8>
+// CHECK: %[[EXTRACT2:.+]] = vector.extract %[[BITCAST2]][0] : i8 from vector<1xi8>
+// CHECK: memref.atomic_yield %[[EXTRACT2]] : i8
diff --git a/mlir/test/Dialect/Vector/vector-shape-cast-lowering-scalable-vectors.mlir b/mlir/test/Dialect/Vector/vector-shape-cast-lowering-scalable-vectors.mlir
index fde6ce9102446..f4becad3c79c1 100644
--- a/mlir/test/Dialect/Vector/vector-shape-cast-lowering-scalable-vectors.mlir
+++ b/mlir/test/Dialect/Vector/vector-shape-cast-lowering-scalable-vectors.mlir
@@ -7,9 +7,9 @@
 // CHECK-SAME: %[[arg0:.*]]: vector<2x1x[4]xi32>
 func.func @i32_3d_to_1d_last_dim_scalable(%arg0: vector<2x1x[4]xi32>) -> vector<[8]xi32>
 {
-  // CHECK-NEXT: %[[cst:.*]] = arith.constant dense<0> : vector<[8]xi32>
+  // CHECK-NEXT: %[[ub:.*]] = ub.poison : vector<[8]xi32>
   // CHECK-NEXT: %[[subvec0:.*]] = vector.extract %[[arg0]][0, 0] : vector<[4]xi32> from vector<2x1x[4]xi32>
-  // CHECK-NEXT: %[[res0:.*]] = vector.scalable.insert %[[subvec0]], %[[cst]][0] : vector<[4]xi32> into vector<[8]xi32>
+  // CHECK-NEXT: %[[res0:.*]] = vector.scalable.insert %[[subvec0]], %[[ub]][0] : vector<[4]xi32> into vector<[8]xi32>
   // CHECK-NEXT: %[[subvec1:.*]] = vector.extract %[[arg0]][1, 0] : vector<[4]xi32> from vector<2x1x[4]xi32>
   // CHECK-NEXT: %[[res1:.*]] = vector.scalable.insert %[[subvec1]], %[[res0]][4] : vector<[4]xi32> into vector<[8]xi32>
   %flat = vector.shape_cast %arg0 : vector<2x1x[4]xi32> to vector<[8]xi32>
@@ -22,9 +22,9 @@ func.func @i32_3d_to_1d_last_dim_scalable(%arg0: vector<2x1x[4]xi32>) -> vector<
 // CHECK-LABEL: i32_1d_to_3d_last_dim_scalable
 // CHECK-SAME: %[[arg0:.*]]: vector<[8]xi32>
 func.func @i32_1d_to_3d_last_dim_scalable(%arg0: vector<[8]xi32>) -> vector<2x1x[4]xi32> {
-  // CHECK-NEXT: %[[cst:.*]] = arith.constant dense<0> : vector<2x1x[4]xi32>
+  // CHECK-NEXT: %[[ub:.*]] = ub.poison : vector<2x1x[4]xi32>
   // CHECK-NEXT: %[[subvec0:.*]] = vector.scalable.extract %[[arg0]][0] : vector<[4]xi32> from vector<[8]xi32>
-  // CHECK-NEXT: %[[res0:.*]] = vector.insert %[[subvec0]], %[[cst]] [0, 0] : vector<[4]xi32> into vector<2x1x[4]xi32>
+  // CHECK-NEXT: %[[res0:.*]] = vector.insert %[[subvec0]], %[[ub]] [0, 0] : vector<[4]xi32> into vector<2x1x[4]xi32>
   // CHECK-NEXT: %[[subvec1:.*]] = vector.scalable.extract %[[arg0]][4] : vector<[4]xi32> from vector<[8]xi32>
   // CHECK-NEXT: %[[res1:.*]] = vector.insert %[[subvec1]], %[[res0]] [1, 0] : vector<[4]xi32> into vector<2x1x[4]xi32>
   %unflat = vector.shape_cast %arg0 : vector<[8]xi32> to vector<2x1x[4]xi32>
@@ -37,9 +37,9 @@ func.func @i32_1d_to_3d_last_dim_scalable(%arg0: vector<[8]xi32>) -> vector<2x1x
 // CHECK-LABEL: i8_2d_to_1d_last_dim_scalable
 // CHECK-SAME: %[[arg0:.*]]: vector<4x[8]xi8>
 func.func @i8_2d_to_1d_last_dim_scalable(%arg0: vector<4x[8]xi8>) -> vector<[32]xi8> {
-  // CHECK-NEXT: %[[cst:.*]] = arith.constant dense<0> : vector<[32]xi8>
+  // CHECK-NEXT: %[[ub:.*]] = ub.poison : vector<[32]xi8>
   // CHECK-NEXT: %[[subvec0:.*]] = vector.extract %[[arg0]][0] : vector<[8]xi8> from vector<4x[8]xi8>
-  // CHECK-NEXT: %[[res0:.*]] = vector.scalable.insert %[[subvec0]], %[[cst]][0] : vector<[8]xi8> into vector<[32]xi8>
+  // CHECK-NEXT: %[[res0:.*]] = vector.scalable.insert %[[subvec0]], %[[ub]][0] : vector<[8]xi8> into vector<[32]xi8>
   // CHECK-NEXT: %[[subvec1:.*]] = vector.extract %[[arg0]][1] : vector<[8]xi8> from vector<4x[8]xi8>
   // CHECK-NEXT: %[[res1:.*]] = vector.scalable.insert %[[subvec1]], %[[res0]][8] : vector<[8]xi8> into vector<[32]xi8>
   // CHECK-NEXT: %[[subvec2:.*]] = vector.extract %[[arg0]][2] : vector<[8]xi8> from vector<4x[8]xi8>
@@ -56,9 +56,9 @@ func.func @i8_2d_to_1d_last_dim_scalable(%arg0: vector<4x[8]xi8>) -> vector<[32]
 // CHECK-LABEL: i8_1d_to_2d_last_dim_scalable
 // CHECK-SAME: %[[arg0:.*]]: vector<[32]xi8>
 func.func @i8_1d_to_2d_last_dim_scalable(%arg0: vector<[32]xi8>) -> vector<4x[8]xi8> {
-  // CHECK-NEXT: %[[cst:.*]] = arith.constant dense<0> : vector<4x[8]xi8>
+  // CHECK-NEXT: %[[ub:.*]] = ub.poison : vector<4x[8]xi8>
   // CHECK-NEXT: %[[subvec0:.*]] = vector.scalable.extract %[[arg0]][0] : vector<[8]xi8> from vector<[32]xi8>
-  // CHECK-NEXT: %[[res0:.*]] = vector.insert %[[subvec0]], %[[cst]] [0] : vector<[8]xi8> into vector<4x[8]xi8>
+  // CHECK-NEXT: %[[res0:.*]] = vector.insert %[[subvec0]], %[[ub]] [0] : vector<[8]xi8> into vector<4x[8]xi8>
   // CHECK-NEXT: %[[subvec1:.*]] = vector.scalable.extract %[[arg0]][8] : vector<[8]xi8> from vector<[32]xi8>
   // CHECK-NEXT: %[[res1:.*]] = vector.insert %[[subvec1]], %[[res0]] [1] : vector<[8]xi8> into vector<4x[8]xi8>
   // CHECK-NEXT: %[[subvec2:.*]] = vector.scalable.extract %[[arg0]][16] : vector<[8]xi8> from vector<[32]xi8>
@@ -75,9 +75,9 @@ func.func @i8_1d_to_2d_last_dim_scalable(%arg0: vector<[32]xi8>) -> vector<4x[8]
 // CHECK-LABEL: f32_permute_leading_non_scalable_dims
 // CHECK-SAME: %[[arg0:.*]]: vector<2x3x[4]xf32>
 func.func @f32_permute_leading_non_scalable_dims(%arg0: vector<2x3x[4]xf32>) -> vector<3x2x[4]xf32> {
-  // CHECK-NEXT: %[[cst:.*]] = arith.constant dense<0.000000e+00> : vector<3x2x[4]xf32>
+  // CHECK-NEXT: %[[ub:.*]] = ub.poison : vector<3x2x[4]xf32>
   // CHECK-NEXT: %[[subvec0:.*]] = vector.extract %[[arg0]][0, 0] : vector<[4]xf32> from vector<2x3x[4]xf32>
-  // CHECK-NEXT: %[[res0:.*]] = vector.insert %[[subvec0]], %[[cst]] [0, 0] : vector<[4]xf32> into vector<3x2x[4]xf32>
+  // CHECK-NEXT: %[[res0:.*]] = vector.insert %[[subvec0]], %[[ub]] [0, 0] : vector<[4]xf32> into vector<3x2x[4]xf32>
   // CHECK-NEXT: %[[subvec1:.*]] = vector.extract %[[arg0]][0, 1] : vector<[4]xf32> from vector<2x3x[4]xf32>
   // CHECK-NEXT: %[[res1:.*]] = vector.insert %[[subvec1]], %[[res0]] [0, 1] : vector<[4]xf32> into vector<3x2x[4]xf32>
   // CHECK-NEXT: %[[subvec2:.*]] = vector.extract %[[arg0]][0, 2] : vector<[4]xf32> from vector<2x3x[4]xf32>
@@ -99,9 +99,9 @@ func.func @f32_permute_leading_non_scalable_dims(%arg0: vector<2x3x[4]xf32>) ->
 // CHECK-SAME: %[[arg0:.*]]: vector<2x2x[2]xf64>
 func.func @f64_flatten_leading_non_scalable_dims(%arg0: vector<2x2x[2]xf64>) -> vector<4x[2]xf64>
 {
-  // CHECK-NEXT: %[[cst:.*]] = arith.constant dense<0.000000e+00> : vector<4x[2]xf64>
+  // CHECK-NEXT: %[[ub:.*]] = ub.poison : vector<4x[2]xf64>
   // CHECK-NEXT: %[[subvec0:.*]] = vector.extract %[[arg0]][0, 0] : vector<[2]xf64> from vector<2x2x[2]xf64>
-  // CHECK-NEXT: %[[res0:.*]] = vector.insert %[[subvec0]], %[[cst]] [0] : vector<[2]xf64> into vector<4x[2]xf64>
+  // CHECK-NEXT: %[[res0:.*]] = vector.insert %[[subvec0]], %[[ub]] [0] : vector<[2]xf64> into vector<4x[2]xf64>
   // CHECK-NEXT: %[[subvec1:.*]] = vector.extract %[[arg0]][0, 1] : vector<[2]xf64> from vector<2x2x[2]xf64>
   // CHECK-NEXT: %[[res1:.*]] = vector.insert %[[subvec1]], %[[res0]] [1] : vector<[2]xf64> into vector<4x[2]xf64>
   // CHECK-NEXT: %[[subvec2:.*]] = vector.extract %[[arg0]][1, 0] : vector<[2]xf64> from vector<2x2x[2]xf64>
@@ -109,7 +109,7 @@ func.func @f64_flatten_leading_non_scalable_dims(%arg0: vector<2x2x[2]xf64>) ->
   // CHECK-NEXT: %[[subvec3:.*]] = vector.extract %[[arg0]][1, 1] : vector<[2]xf64> from vector<2x2x[2]xf64>
   // CHECK-NEXT: %[[res3:.*]] = vector.insert %[[subvec3]], %[[res2]] [3] : vector<[2]xf64> into vector<4x[2]xf64>
   %res = vector.shape_cast %arg0: vector<2x2x[2]xf64> to vector<4x[2]xf64>
-  // CHECK-NEXT: return %7 : vector<4x[2]xf64>
+  // CHECK-NEXT: return %[[res3:.*]] : vector<4x[2]xf64>
   return %res : vector<4x[2]xf64>
 }
 
@@ -119,10 +119,10 @@ func.func @f64_flatten_leading_non_scalable_dims(%arg0: vector<2x2x[2]xf64>) ->
 // CHECK-SAME: %[[arg0:.*]]: vector<3x[4]xf32>
 func.func @f32_reduce_trailing_scalable_dim(%arg0: vector<3x[4]xf32>) -> vector<6x[2]xf32>
 {
-  // CHECK-NEXT: %[[cst:.*]] = arith.constant dense<0.000000e+00> : vector<6x[2]xf32>
+  // CHECK-NEXT: %[[ub:.*]] = ub.poison : vector<6x[2]xf32>
   // CHECK-NEXT: %[[srcvec0:.*]] = vector.extract %[[arg0]][0] : vector<[4]xf32> from vector<3x[4]xf32>
   // CHECK-NEXT: %[[subvec0:.*]] = vector.scalable.extract %[[srcvec0]][0] : vector<[2]xf32> from vector<[4]xf32>
-  // CHECK-NEXT: %[[res0:.*]] = vector.insert %[[subvec0]], %[[cst]] [0] : vector<[2]xf32> into vector<6x[2]xf32>
+  // CHECK-NEXT: %[[res0:.*]] = vector.insert %[[subvec0]], %[[ub]] [0] : vector<[2]xf32> into vector<6x[2]xf32>
   // CHECK-NEXT: %[[subvec1:.*]] = vector.scalable.extract %[[srcvec0]][2] : vector<[2]xf32> from vector<[4]xf32>
   // CHECK-NEXT: %[[res1:.*]] = vector.insert %[[subvec1]], %[[res0]] [1] : vector<[2]xf32> into vector<6x[2]xf32>
   // CHECK-NEXT: %[[srcvec1:.*]] = vector.extract %[[arg0]][1] : vector<[4]xf32> from vector<3x[4]xf32>
@@ -146,16 +146,15 @@ func.func @f32_reduce_trailing_scalable_dim(%arg0: vector<3x[4]xf32>) -> vector<
 // CHECK-SAME: %[[arg0:.*]]: vector<4x[2]xf32>
 func.func @f32_increase_trailing_scalable_dim(%arg0: vector<4x[2]xf32>) -> vector<2x[4]xf32>
 {
-  // CHECK-NEXT: %[[cst:.*]] = arith.constant dense<0.000000e+00> : vector<2x[4]xf32>
+  //  CHECK-DAG: %[[ub0:.*]] = ub.poison : vector<2x[4]xf32>
+  //  CHECK-DAG: %[[ub1:.*]] = ub.poison : vector<[4]xf32>
   // CHECK-NEXT: %[[subvec0:.*]] = vector.extract %[[arg0]][0] : vector<[2]xf32> from vector<4x[2]xf32>
-  // CHECK-NEXT: %[[resvec0:.*]] = vector.extract %[[cst]][0] : vector<[4]xf32> from vector<2x[4]xf32>
-  // CHECK-NEXT: %[[resvec1:.*]] = vector.scalable.insert %[[subvec0]], %[[resvec0]][0] : vector<[2]xf32> into vector<[4]xf32>
+  // CHECK-NEXT: %[[resvec1:.*]] = vector.scalable.insert %[[subvec0]], %[[ub1]][0] : vector<[2]xf32> into vector<[4]xf32>
   // CHECK-NEXT: %[[subvec1:.*]] = vector.extract %[[arg0]][1] : vector<[2]xf32> from vector<4x[2]xf32>
   // CHECK-NEXT: %[[resvec2:.*]] = vector.scalable.insert %[[subvec1]], %[[resvec1]][2] : vector<[2]xf32> into vector<[4]xf32>
-  // CHECK-NEXT: %[[res0:.*]] = vector.insert %[[resvec2]], %[[cst]] [0] : vector<[4]xf32> into vector<2x[4]xf32>
+  // CHECK-NEXT: %[[res0:.*]] = vector.insert %[[resvec2]], %[[ub0]] [0] : vector<[4]xf32> into vector<2x[4]xf32>
   // CHECK-NEXT: %[[subvec3:.*]] = vector.extract %[[arg0]][2] : vector<[2]xf32> from vector<4x[2]xf32>
-  // CHECK-NEXT: %[[resvec3:.*]] = vector.extract %[[cst]][1] : vector<[4]xf32> from vector<2x[4]xf32>
-  // CHECK-NEXT: %[[resvec4:.*]] = vector.scalable.insert %[[subvec3]], %[[resvec3]][0] : vector<[2]xf32> into vector<[4]xf32>
+  // CHECK-NEXT: %[[resvec4:.*]] = vector.scalable.insert %[[subvec3]], %[[ub1]][0] : vector<[2]xf32> into vector<[4]xf32>
   // CHECK-NEXT: %[[subvec4:.*]] = vector.extract %[[arg0]][3] : vector<[2]xf32> from vector<4x[2]xf32>
   // CHECK-NEXT: %[[resvec5:.*]] = vector.scalable.insert %[[subvec4]], %[[resvec4]][2] : vector<[2]xf32> into vector<[4]xf32>
   // CHECK-NEXT: %[[res1:.*]] = vector.insert %[[resvec5]], %[[res0]] [1] : vector<[4]xf32> into vector<2x[4]xf32>
diff --git a/mlir/test/Dialect/Vector/vector-shape-cast-lowering-transforms.mlir b/mlir/test/Dialect/Vector/vector-shape-cast-lowering-transforms.mlir
index f2f1211fd70ee..ab30acf68b30b 100644
--- a/mlir/test/Dialect/Vector/vector-shape-cast-lowering-transforms.mlir
+++ b/mlir/test/Dialect/Vector/vector-shape-cast-lowering-transforms.mlir
@@ -1,4 +1,4 @@
-// RUN: mlir-opt %s --transform-interpreter --split-input-file | FileCheck %s
+// RUN: mlir-opt %s --transform-interpreter | FileCheck %s
 
 // CHECK-LABEL: func @nop_shape_cast
 // CHECK-SAME: %[[A:.*]]: vector<16xf32>
@@ -22,11 +22,11 @@ func.func @cancel_shape_cast(%arg0: vector<16xf32>) -> vector<16xf32> {
 // llvm.matrix operations
 // CHECK-LABEL: func @shape_casts
 func.func @shape_casts(%a: vector<2x2xf32>) -> (vector<4xf32>, vector<2x2xf32>) {
-  // CHECK-DAG: %[[cst22:.*]] = arith.constant dense<0.000000e+00> : vector<2x2xf32>
-  // CHECK-DAG: %[[cst:.*]] = arith.constant dense<0.000000e+00> : vector<4xf32>
+  // CHECK-DAG: %[[ub22:.*]] = ub.poison : vector<2x2xf32>
+  // CHECK-DAG: %[[ub:.*]] = ub.poison : vector<4xf32>
   // CHECK: %[[ex0:.*]] = vector.extract %{{.*}}[0] : vector<2xf32> from vector<2x2xf32>
   //
-  // CHECK: %[[in0:.*]] = vector.insert_strided_slice %[[ex0]], %[[cst]]
+  // CHECK: %[[in0:.*]] = vector.insert_strided_slice %[[ex0]], %[[ub]]
   // CHECK-SAME: {offsets = [0], strides = [1]} : vector<2xf32> into vector<4xf32>
   //
   // CHECK: %[[ex1:.*]] = vector.extract %{{.*}}[1] : vector<2xf32> from vector<2x2xf32>
@@ -42,7 +42,7 @@ func.func @shape_casts(%a: vector<2x2xf32>) -> (vector<4xf32>, vector<2x2xf32>)
   // CHECK-SAME: {offsets = [0], sizes = [2], strides = [1]} :
   // CHECK-SAME: vector<4xf32> to vector<2xf32>
   //
-  // CHECK: %[[res0:.*]] = vector.insert %[[ss0]], %[[cst22]] [0] :
+  // CHECK: %[[res0:.*]] = vector.insert %[[ss0]], %[[ub22]] [0] :
   // CHECK-SAME: vector<2xf32> into vector<2x2xf32>
   //
   // CHECK: %[[s2:.*]] = vector.extract_strided_slice %[[add]]
@@ -59,9 +59,9 @@ func.func @shape_casts(%a: vector<2x2xf32>) -> (vector<4xf32>, vector<2x2xf32>)
 
 // CHECK-LABEL: func @shape_cast_2d2d
 // CHECK-SAME: %[[A:.*]]: vector<3x2xf32>
-// CHECK: %[[C:.*]] = arith.constant dense<0.000000e+00> : vector<2x3xf32>
+// CHECK: %[[UB:.*]] = ub.poison : vector<2x3xf32>
 // CHECK: %[[T0:.*]] = vector.extract %[[A]][0, 0] : f32 from vector<3x2xf32>
-// CHECK: %[[T1:.*]] = vector.insert %[[T0]], %[[C]] [0, 0] : f32 into vector<2x3xf32>
+// CHECK: %[[T1:.*]] = vector.insert %[[T0]], %[[UB]] [0, 0] : f32 into vector<2x3xf32>
 // CHECK: %[[T2:.*]] = vector.extract %[[A]][0, 1] : f32 from vector<3x2xf32>
 // CHECK: %[[T3:.*]] = vector.insert %[[T2]], %[[T1]] [0, 1] : f32 into vector<2x3xf32>
 // CHECK: %[[T4:.*]] = vector.extract %[[A]][1, 0] : f32 from vector<3x2xf32>
@@ -81,20 +81,17 @@ func.func @shape_cast_2d2d(%arg0 : vector<3x2xf32>) -> vector<2x3xf32> {
 
 // CHECK-LABEL: func @shape_cast_3d1d
 // CHECK-SAME: %[[A:.*]]: vector<1x3x2xf32>
-// CHECK: %[[C:.*]] = arith.constant dense<0.000000e+00> : vector<6xf32>
-// CHECK: %[[T0:.*]] = vector.extract %[[A]][0, 0, 0] : f32 from vector<1x3x2xf32>
-// CHECK: %[[T1:.*]] = vector.insert %[[T0]], %[[C]] [0] : f32 into vector<6xf32>
-// CHECK: %[[T2:.*]] = vector.extract %[[A]][0, 0, 1] : f32 from vector<1x3x2xf32>
-// CHECK: %[[T3:.*]] = vector.insert %[[T2]], %[[T1]] [1] : f32 into vector<6xf32>
-// CHECK: %[[T4:.*]] = vector.extract %[[A]][0, 1, 0] : f32 from vector<1x3x2xf32>
-// CHECK: %[[T5:.*]] = vector.insert %[[T4]], %[[T3]] [2] : f32 into vector<6xf32>
-// CHECK: %[[T6:.*]] = vector.extract %[[A]][0, 1, 1] : f32 from vector<1x3x2xf32>
-// CHECK: %[[T7:.*]] = vector.insert %[[T6]], %[[T5]] [3] : f32 into vector<6xf32>
-// CHECK: %[[T8:.*]] = vector.extract %[[A]][0, 2, 0] : f32 from vector<1x3x2xf32>
-// CHECK: %[[T9:.*]] = vector.insert %[[T8]], %[[T7]] [4] : f32 into vector<6xf32>
-// CHECK: %[[T10:.*]] = vector.extract %[[A]][0, 2, 1] : f32 from vector<1x3x2xf32>
-// CHECK: %[[T11:.*]] = vector.insert %[[T10]], %[[T9]] [5] : f32 into vector<6xf32>
-// CHECK: return %[[T11]] : vector<6xf32>
+// CHECK: %[[UB:.*]] = ub.poison : vector<6xf32>
+// CHECK: %[[T0:.*]] = vector.extract %[[A]][0, 0] : vector<2xf32> from vector<1x3x2xf32>
+// CHECK: %[[T1:.*]] = vector.insert_strided_slice %[[T0]], %[[UB]]
+// CHECK-SAME:           {offsets = [0], strides = [1]} : vector<2xf32> into vector<6xf32>
+// CHECK: %[[T2:.*]] = vector.extract %[[A]][0, 1] : vector<2xf32> from vector<1x3x2xf32>
+// CHECK: %[[T3:.*]] = vector.insert_strided_slice %[[T2]], %[[T1]]
+// CHECK-SAME:           {offsets = [2], strides = [1]} : vector<2xf32> into vector<6xf32>
+// CHECK: %[[T4:.*]] = vector.extract %[[A]][0, 2] : vector<2xf32> from vector<1x3x2xf32>
+// CHECK: %[[T5:.*]] = vector.insert_strided_slice %[[T4]], %[[T3]]
+// CHECK-SAME:           {offsets = [4], strides = [1]} : vector<2xf32> into vector<6xf32>
+// CHECK: return %[[T5]] : vector<6xf32>
 
 func.func @shape_cast_3d1d(%arg0 : vector<1x3x2xf32>) -> vector<6xf32> {
   %s = vector.shape_cast %arg0 : vector<1x3x2xf32> to vector<6xf32>
@@ -103,20 +100,14 @@ func.func @shape_cast_3d1d(%arg0 : vector<1x3x2xf32>) -> vector<6xf32> {
 
 // CHECK-LABEL: func @shape_cast_1d3d
 // CHECK-SAME: %[[A:.*]]: vector<6xf32>
-// CHECK: %[[C:.*]] = arith.constant dense<0.000000e+00> : vector<2x1x3xf32>
-// CHECK: %[[T0:.*]] = vector.extract %[[A]][0] : f32 from vector<6xf32>
-// CHECK: %[[T1:.*]] = vector.insert %[[T0]], %[[C]] [0, 0, 0] : f32 into vector<2x1x3xf32>
-// CHECK: %[[T2:.*]] = vector.extract %[[A]][1] : f32 from vector<6xf32>
-// CHECK: %[[T3:.*]] = vector.insert %[[T2]], %[[T1]] [0, 0, 1] : f32 into vector<2x1x3xf32>
-// CHECK: %[[T4:.*]] = vector.extract %[[A]][2] : f32 from vector<6xf32>
-// CHECK: %[[T5:.*]] = vector.insert %[[T4]], %[[T3]] [0, 0, 2] : f32 into vector<2x1x3xf32>
-// CHECK: %[[T6:.*]] = vector.extract %[[A]][3] : f32 from vector<6xf32>
-// CHECK: %[[T7:.*]] = vector.insert %[[T6]], %[[T5]] [1, 0, 0] : f32 into vector<2x1x3xf32>
-// CHECK: %[[T8:.*]] = vector.extract %[[A]][4] : f32 from vector<6xf32>
-// CHECK: %[[T9:.*]] = vector.insert %[[T8]], %[[T7]] [1, 0, 1] : f32 into vector<2x1x3xf32>
-// CHECK: %[[T10:.*]] = vector.extract %[[A]][5] : f32 from vector<6xf32>
-// CHECK: %[[T11:.*]] = vector.insert %[[T10]], %[[T9]] [1, 0, 2] : f32 into vector<2x1x3xf32>
-// CHECK: return %[[T11]] : vector<2x1x3xf32>
+// CHECK: %[[UB:.*]] = ub.poison : vector<2x1x3xf32>
+// CHECK: %[[T0:.*]] = vector.extract_strided_slice %[[A]]
+// CHECK-SAME:           {offsets = [0], sizes = [3], strides = [1]} : vector<6xf32> to vector<3xf32>
+// CHECK: %[[T1:.*]] = vector.insert %[[T0]], %[[UB]] [0, 0] : vector<3xf32> into vector<2x1x3xf32>
+// CHECK: %[[T2:.*]] = vector.extract_strided_slice %[[A]]
+// CHECK:                {offsets = [3], sizes = [3], strides = [1]} : vector<6xf32> to vector<3xf32>
+// CHECK: %[[T3:.*]] = vector.insert %[[T2]], %[[T1]] [1, 0] : vector<3xf32> into vector<2x1x3xf32>
+// CHECK: return %[[T3]] : vector<2x1x3xf32>
 
 func.func @shape_cast_1d3d(%arg0 : vector<6xf32>) -> vector<2x1x3xf32> {
   %s = vector.shape_cast %arg0 : vector<6xf32> to vector<2x1x3xf32>
@@ -124,11 +115,11 @@ func.func @shape_cast_1d3d(%arg0 : vector<6xf32>) -> vector<2x1x3xf32> {
 }
 
 // CHECK-LABEL:   func.func @shape_cast_0d1d(
-// CHECK-SAME:                               %[[VAL_0:.*]]: vector<f32>) -> vector<1xf32> {
-// CHECK:           %[[VAL_1:.*]] = arith.constant dense<0.000000e+00> : vector<1xf32>
-// CHECK:           %[[VAL_2:.*]] = vector.extractelement %[[VAL_0]][] : vector<f32>
-// CHECK:           %[[VAL_3:.*]] = vector.insert %[[VAL_2]], %[[VAL_1]] [0] : f32 into vector<1xf32>
-// CHECK:           return %[[VAL_3]] : vector<1xf32>
+// CHECK-SAME:                               %[[ARG0:.*]]: vector<f32>) -> vector<1xf32> {
+// CHECK:           %[[UB:.*]] = ub.poison : vector<1xf32>
+// CHECK:           %[[EXTRACT0:.*]] = vector.extractelement %[[ARG0]][] : vector<f32>
+// CHECK:           %[[RES:.*]] = vector.insert %[[EXTRACT0]], %[[UB]] [0] : f32 into vector<1xf32>
+// CHECK:           return %[[RES]] : vector<1xf32>
 // CHECK:         }
 
 func.func @shape_cast_0d1d(%arg0 : vector<f32>) -> vector<1xf32> {
@@ -137,11 +128,11 @@ func.func @shape_cast_0d1d(%arg0 : vector<f32>) -> vector<1xf32> {
 }
 
 // CHECK-LABEL:   func.func @shape_cast_1d0d(
-// CHECK-SAME:                               %[[VAL_0:.*]]: vector<1xf32>) -> vector<f32> {
-// CHECK:           %[[VAL_1:.*]] = arith.constant dense<0.000000e+00> : vector<f32>
-// CHECK:           %[[VAL_2:.*]] = vector.extract %[[VAL_0]][0] : f32 from vector<1xf32>
-// CHECK:           %[[VAL_3:.*]] = vector.insertelement %[[VAL_2]], %[[VAL_1]][] : vector<f32>
-// CHECK:           return %[[VAL_3]] : vector<f32>
+// CHECK-SAME:                               %[[ARG0:.*]]: vector<1xf32>) -> vector<f32> {
+// CHECK:           %[[UB:.*]] = ub.poison : vector<f32>
+// CHECK:           %[[EXTRACT0:.*]] = vector.extract %[[ARG0]][0] : f32 from vector<1xf32>
+// CHECK:           %[[RES:.*]] = vector.insertelement %[[EXTRACT0]], %[[UB]][] : vector<f32>
+// CHECK:           return %[[RES]] : vector<f32>
 // CHECK:         }
 
 func.func @shape_cast_1d0d(%arg0 : vector<1xf32>) -> vector<f32> {
diff --git a/mlir/test/Dialect/Vector/vector-transpose-lowering.mlir b/mlir/test/Dialect/Vector/vector-transpose-lowering.mlir
index 219a72df52a19..83395504e8c74 100644
--- a/mlir/test/Dialect/Vector/vector-transpose-lowering.mlir
+++ b/mlir/test/Dialect/Vector/vector-transpose-lowering.mlir
@@ -2,9 +2,9 @@
 
 // CHECK-LABEL: func @transpose23
 // CHECK-SAME: %[[A:.*]]: vector<2x3xf32>
-// CHECK:      %[[Z:.*]] = arith.constant dense<0.000000e+00> : vector<3x2xf32>
+// CHECK:      %[[UB:.*]] = ub.poison : vector<3x2xf32>
 // CHECK:      %[[T0:.*]] = vector.extract %[[A]][0, 0] : f32 from vector<2x3xf32>
-// CHECK:      %[[T1:.*]] = vector.insert %[[T0]], %[[Z]] [0, 0] : f32 into vector<3x2xf32>
+// CHECK:      %[[T1:.*]] = vector.insert %[[T0]], %[[UB]] [0, 0] : f32 into vector<3x2xf32>
 // CHECK:      %[[T2:.*]] = vector.extract %[[A]][0, 1] : f32 from vector<2x3xf32>
 // CHECK:      %[[T3:.*]] = vector.insert %[[T2]], %[[T1]] [1, 0] : f32 into vector<3x2xf32>
 // CHECK:      %[[T4:.*]] = vector.extract %[[A]][0, 2] : f32 from vector<2x3xf32>
diff --git a/mlir/test/Dialect/Vector/vector-unroll-options.mlir b/mlir/test/Dialect/Vector/vector-unroll-options.mlir
index c51fc755dffa8..7e3fe56f6b124 100644
--- a/mlir/test/Dialect/Vector/vector-unroll-options.mlir
+++ b/mlir/test/Dialect/Vector/vector-unroll-options.mlir
@@ -232,11 +232,11 @@ func.func @vector_reduction(%v : vector<8xf32>) -> f32 {
 //       CHECK:   %[[add3:.*]] = arith.addf %[[add2]], %[[r3]]
 //       CHECK:   return %[[add3]]
 
-func.func @vector_tranpose(%v : vector<2x4x3x8xf32>) -> vector<2x3x8x4xf32> {
+func.func @vector_transpose(%v : vector<2x4x3x8xf32>) -> vector<2x3x8x4xf32> {
   %t = vector.transpose %v, [0, 2, 3, 1] : vector<2x4x3x8xf32> to vector<2x3x8x4xf32>
   return %t : vector<2x3x8x4xf32>
 }
-// CHECK-LABEL: func @vector_tranpose
+// CHECK-LABEL: func @vector_transpose
 //       CHECK:   %[[VI:.*]] = arith.constant dense<0.000000e+00> : vector<2x3x8x4xf32>
 //       CHECK:   %[[E0:.*]] = vector.extract_strided_slice %{{.*}} {offsets = [0, 0, 0, 0], sizes = [1, 2, 3, 4], strides = [1, 1, 1, 1]} : vector<2x4x3x8xf32> to vector<1x2x3x4xf32>
 //       CHECK:   %[[T0:.*]] = vector.transpose %[[E0]], [0, 2, 3, 1] : vector<1x2x3x4xf32> to vector<1x3x4x2xf32>
diff --git a/mlir/test/Dialect/Vector/vector-warp-distribute.mlir b/mlir/test/Dialect/Vector/vector-warp-distribute.mlir
index dbe0b39422369..38771f2593449 100644
--- a/mlir/test/Dialect/Vector/vector-warp-distribute.mlir
+++ b/mlir/test/Dialect/Vector/vector-warp-distribute.mlir
@@ -778,12 +778,11 @@ func.func @warp_constant(%laneid: index) -> (vector<1xf32>) {
 
 // CHECK-PROP-LABEL: func.func @vector_extract_1d(
 //   CHECK-PROP-DAG:   %[[C5_I32:.*]] = arith.constant 5 : i32
-//   CHECK-PROP-DAG:   %[[C1:.*]] = arith.constant 1 : index
 //       CHECK-PROP:   %[[R:.*]] = gpu.warp_execute_on_lane_0(%{{.*}})[32] -> (vector<2xf32>) {
 //       CHECK-PROP:     %[[V:.*]] = "some_def"() : () -> vector<64xf32>
 //       CHECK-PROP:     gpu.yield %[[V]] : vector<64xf32>
 //       CHECK-PROP:   }
-//       CHECK-PROP:   %[[E:.*]] = vector.extract %[[R]][%[[C1]]] : f32 from vector<2xf32>
+//       CHECK-PROP:   %[[E:.*]] = vector.extract %[[R]][1] : f32 from vector<2xf32>
 //       CHECK-PROP:   %[[SHUFFLED:.*]], %{{.*}} = gpu.shuffle  idx %[[E]], %[[C5_I32]]
 //       CHECK-PROP:   return %[[SHUFFLED]] : f32
 func.func @vector_extract_1d(%laneid: index) -> (f32) {
diff --git a/mlir/test/Dialect/X86Vector/dot-bf16.mlir b/mlir/test/Dialect/X86Vector/dot-bf16.mlir
new file mode 100644
index 0000000000000..b7ac46a66cebf
--- /dev/null
+++ b/mlir/test/Dialect/X86Vector/dot-bf16.mlir
@@ -0,0 +1,32 @@
+// REQUIRES: target=x86{{.*}}
+
+// RUN: mlir-opt %s \
+// RUN:   -convert-vector-to-llvm="enable-x86vector" -convert-to-llvm \
+// RUN:   -reconcile-unrealized-casts | \
+// RUN: mlir-translate --mlir-to-llvmir | \
+// RUN: llc -mcpu=sapphirerapids | \
+// RUN: FileCheck %s
+
+func.func @avx512bf16_dot_128(%src: vector<4xf32>, %a: vector<8xbf16>,
+    %b: vector<8xbf16>) -> vector<4xf32> {
+  %0 = x86vector.avx512.dot %src, %a, %b : vector<8xbf16> -> vector<4xf32>
+  return %0 : vector<4xf32>
+}
+// CHECK-LABEL: avx512bf16_dot_128:
+// CHECK: vdpbf16ps{{.*}}%xmm
+
+func.func @avx512bf16_dot_256(%src: vector<8xf32>, %a: vector<16xbf16>,
+    %b: vector<16xbf16>) -> vector<8xf32> {
+  %0 = x86vector.avx512.dot %src, %a, %b : vector<16xbf16> -> vector<8xf32>
+  return %0 : vector<8xf32>
+}
+// CHECK-LABEL: avx512bf16_dot_256:
+// CHECK: vdpbf16ps{{.*}}%ymm
+
+func.func @avx512bf16_dot_512(%src: vector<16xf32>, %a: vector<32xbf16>,
+    %b: vector<32xbf16>) -> vector<16xf32> {
+  %0 = x86vector.avx512.dot %src, %a, %b : vector<32xbf16> -> vector<16xf32>
+  return %0 : vector<16xf32>
+}
+// CHECK-LABEL: avx512bf16_dot_512:
+// CHECK: vdpbf16ps{{.*}}%zmm
diff --git a/mlir/test/Dialect/X86Vector/legalize-for-llvm.mlir b/mlir/test/Dialect/X86Vector/legalize-for-llvm.mlir
index 8b9006395fdfe..ed9177eaec9ce 100644
--- a/mlir/test/Dialect/X86Vector/legalize-for-llvm.mlir
+++ b/mlir/test/Dialect/X86Vector/legalize-for-llvm.mlir
@@ -43,6 +43,33 @@ func.func @avx512_vp2intersect(%a: vector<16xi32>, %b: vector<8xi64>)
   return %0, %1, %2, %3 : vector<16xi1>, vector<16xi1>, vector<8xi1>, vector<8xi1>
 }
 
+// CHECK-LABEL: func @avx512bf16_dot_128
+func.func @avx512bf16_dot_128(%src: vector<4xf32>, %a: vector<8xbf16>,
+  %b: vector<8xbf16>) -> (vector<4xf32>)
+{
+  // CHECK: x86vector.avx512.intr.dpbf16ps.128
+  %0 = x86vector.avx512.dot %src, %a, %b : vector<8xbf16> -> vector<4xf32>
+  return %0 : vector<4xf32>
+}
+
+// CHECK-LABEL: func @avx512bf16_dot_256
+func.func @avx512bf16_dot_256(%src: vector<8xf32>, %a: vector<16xbf16>,
+  %b: vector<16xbf16>) -> (vector<8xf32>)
+{
+  // CHECK: x86vector.avx512.intr.dpbf16ps.256
+  %0 = x86vector.avx512.dot %src, %a, %b : vector<16xbf16> -> vector<8xf32>
+  return %0 : vector<8xf32>
+}
+
+// CHECK-LABEL: func @avx512bf16_dot_512
+func.func @avx512bf16_dot_512(%src: vector<16xf32>, %a: vector<32xbf16>,
+  %b: vector<32xbf16>) -> (vector<16xf32>)
+{
+  // CHECK: x86vector.avx512.intr.dpbf16ps.512
+  %0 = x86vector.avx512.dot %src, %a, %b : vector<32xbf16> -> vector<16xf32>
+  return %0 : vector<16xf32>
+}
+
 // CHECK-LABEL: func @avx_rsqrt
 func.func @avx_rsqrt(%a: vector<8xf32>) -> (vector<8xf32>)
 {
diff --git a/mlir/test/Dialect/X86Vector/roundtrip.mlir b/mlir/test/Dialect/X86Vector/roundtrip.mlir
index 557978b51c512..cf74a7ee60255 100644
--- a/mlir/test/Dialect/X86Vector/roundtrip.mlir
+++ b/mlir/test/Dialect/X86Vector/roundtrip.mlir
@@ -47,6 +47,33 @@ func.func @avx512_mask_compress(%k1: vector<16xi1>, %a1: vector<16xf32>,
   return %0, %1, %2 : vector<16xf32>, vector<16xf32>, vector<8xi64>
 }
 
+// CHECK-LABEL: func @avx512bf16_dot_128
+func.func @avx512bf16_dot_128(%src: vector<4xf32>, %a: vector<8xbf16>,
+  %b: vector<8xbf16>) -> (vector<4xf32>)
+{
+  // CHECK: x86vector.avx512.dot {{.*}} : vector<8xbf16> -> vector<4xf32>
+  %0 = x86vector.avx512.dot %src, %a, %b : vector<8xbf16> -> vector<4xf32>
+  return %0 : vector<4xf32>
+}
+
+// CHECK-LABEL: func @avx512bf16_dot_256
+func.func @avx512bf16_dot_256(%src: vector<8xf32>, %a: vector<16xbf16>,
+  %b: vector<16xbf16>) -> (vector<8xf32>)
+{
+  // CHECK: x86vector.avx512.dot {{.*}} : vector<16xbf16> -> vector<8xf32>
+  %0 = x86vector.avx512.dot %src, %a, %b : vector<16xbf16> -> vector<8xf32>
+  return %0 : vector<8xf32>
+}
+
+// CHECK-LABEL: func @avx512bf16_dot_512
+func.func @avx512bf16_dot_512(%src: vector<16xf32>, %a: vector<32xbf16>,
+  %b: vector<32xbf16>) -> (vector<16xf32>)
+{
+  // CHECK: x86vector.avx512.dot {{.*}} : vector<32xbf16> -> vector<16xf32>
+  %0 = x86vector.avx512.dot %src, %a, %b : vector<32xbf16> -> vector<16xf32>
+  return %0 : vector<16xf32>
+}
+
 // CHECK-LABEL: func @avx_rsqrt
 func.func @avx_rsqrt(%a: vector<8xf32>) -> (vector<8xf32>)
 {
diff --git a/mlir/test/Dialect/XeGPU/XeGPUOps.mlir b/mlir/test/Dialect/XeGPU/XeGPUOps.mlir
index d7174a489888a..472176af72b19 100644
--- a/mlir/test/Dialect/XeGPU/XeGPUOps.mlir
+++ b/mlir/test/Dialect/XeGPU/XeGPUOps.mlir
@@ -97,6 +97,16 @@ gpu.func @test_load_nd_vc_3(%src: memref<24x32xf32>) {
   gpu.return
 }
 
+// CHECK: func @test_load_nd_vc_4(%[[arg0:.*]]: memref<24x32xf32>) {
+gpu.func @test_load_nd_vc_4(%src: memref<24x32xf32>) {
+  // CHECK: %[[R0:.*]] = xegpu.create_nd_tdesc %arg0[0, 0] : memref<24x32xf32> -> !xegpu.tensor_desc<32xf32, #xegpu.sg_map<wi_layout = [1, 16], wi_data = [1, 1]>>
+  %1 = xegpu.create_nd_tdesc %src[0, 0] : memref<24x32xf32> ->
+    !xegpu.tensor_desc<32xf32, #xegpu.sg_map<wi_layout = [1, 16], wi_data = [1, 1]>>
+  // CHECK: %[[R1:.*]] = xegpu.load_nd %[[R0]] <{l1_hint = #xegpu.cache_hint<cached>, l2_hint = #xegpu.cache_hint<uncached>}> : !xegpu.tensor_desc<32xf32, #xegpu.sg_map<wi_layout = [1, 16], wi_data = [1, 1]>> -> vector<2xf32>
+  %2 = xegpu.load_nd %1 <{l1_hint = #xegpu.cache_hint<cached>, l2_hint = #xegpu.cache_hint<uncached>}> : !xegpu.tensor_desc<32xf32, #xegpu.sg_map<wi_layout = [1, 16], wi_data = [1, 1]>> -> vector<2xf32>
+  gpu.return
+}
+
 // CHECK: func @test_store_nd_vc(%[[arg0:.*]]: memref<24x32xf16>) {
 gpu.func @test_store_nd_vc(%dst: memref<24x32xf16>) {
   // CHECK: %[[C:.*]] = arith.constant dense<1.000000e+00> : vector<24x32xf16>
@@ -132,6 +142,18 @@ gpu.func @test_store_nd_vc_3(%src: memref<24x32xf16>) {
   gpu.return
 }
 
+// CHECK: func @test_store_nd_vc_4(%[[arg0:.*]]: memref<24x32xf16>) {
+gpu.func @test_store_nd_vc_4(%src: memref<24x32xf16>) {
+   // CHECK: %[[C:.*]] = arith.constant dense<1.000000e+00> : vector<2xf16>
+  %1 = arith.constant dense<1.0>: vector<2xf16>
+  // CHECK: %[[R0:.*]] = xegpu.create_nd_tdesc %arg0[0, 0] : memref<24x32xf16> -> !xegpu.tensor_desc<32xf16, #xegpu.sg_map<wi_layout = [1, 16], wi_data = [1, 1]>>
+  %2 = xegpu.create_nd_tdesc %src[0, 0] : memref<24x32xf16> ->
+    !xegpu.tensor_desc<32xf16, #xegpu.sg_map<wi_layout = [1, 16], wi_data = [1, 1]>>
+  // CHECK: xegpu.store_nd %[[C]], %[[R0]] <{l1_hint = #xegpu.cache_hint<write_back>, l2_hint = #xegpu.cache_hint<uncached>}> : vector<2xf16>, !xegpu.tensor_desc<32xf16, #xegpu.sg_map<wi_layout = [1, 16], wi_data = [1, 1]>>
+  xegpu.store_nd %1, %2 <{l1_hint = #xegpu.cache_hint<write_back>, l2_hint = #xegpu.cache_hint<uncached>}>: vector<2xf16>, !xegpu.tensor_desc<32xf16, #xegpu.sg_map<wi_layout = [1, 16], wi_data = [1, 1]>>
+  gpu.return
+}
+
 // CHECK: gpu.func @test_create_update_nd_tdesc_vc(%[[arg0:.*]]: memref<24x32xf32>) {
 gpu.func @test_create_update_nd_tdesc_vc(%src: memref<24x32xf32>) {
   // CHECK: %[[REG:.*]] = xegpu.create_nd_tdesc %arg0[0, 0] : memref<24x32xf32> -> !xegpu.tensor_desc<8x16xf32>
@@ -159,15 +181,82 @@ gpu.func @test_create_tdesc_vc_1(%src: memref<?xf32, 3>) {
   gpu.return
 }
 
+// CHECK: gpu.func @test_create_tdesc_vc_2(%[[arg0:.*]]: memref<?xf32>) {
+gpu.func @test_create_tdesc_vc_2(%src: memref<?xf32>) {
+  //CHECK: %[[cst:.*]] = arith.constant dense<[0, 8, 16, 24]> : vector<4xindex>
+  %0 = arith.constant dense<[0, 8, 16, 24]> : vector<4xindex>
+  //CHECK: %[[R0:.*]] = xegpu.create_tdesc %[[arg0]], %[[cst]] : memref<?xf32>, vector<4xindex> -> !xegpu.tensor_desc<4xf32, #xegpu.scatter_tdesc_attr<>
+  %1 = xegpu.create_tdesc %src, %0 : memref<?xf32>, vector<4xindex>  -> !xegpu.tensor_desc<4xf32, #xegpu.scatter_tdesc_attr<chunk_size = 1>>
+  gpu.return
+}
+
 // CHECK: gpu.func @test_create_tdesc_vc_with_sg_map(%[[arg0:.*]]: ui64) {
 gpu.func @test_create_tdesc_vc_with_sg_map(%src: ui64) {
   //CHECK: %[[cst:.*]] = arith.constant dense<[0, 8, 16, 24]> : vector<4xindex>
   %0 = arith.constant dense<[0, 8, 16, 24]> : vector<4xindex>
-  //CHECK: %[[R0:.*]] = xegpu.create_tdesc %[[arg0]], %[[cst]] : ui64, vector<4xindex> -> !xegpu.tensor_desc<4x2xf32, #xegpu.scatter_tdesc_attr<chunk_size = 2 : i64>, #xegpu.sg_map<wi_layout = [4, 1], wi_data = [1, 1]>>
-  %1 = xegpu.create_tdesc %src, %0 : ui64, vector<4xindex> -> !xegpu.tensor_desc<4x2xf32, #xegpu.scatter_tdesc_attr<chunk_size = 2>, #xegpu.sg_map<wi_layout = [4, 1], wi_data = [1, 1]>>
+  //CHECK: %[[R0:.*]] = xegpu.create_tdesc %[[arg0]], %[[cst]] : ui64, vector<4xindex> -> !xegpu.tensor_desc<4x2xf32, #xegpu.scatter_tdesc_attr<chunk_size = 2 : i64>, #xegpu.sg_map<wi_layout = [4, 1], wi_data = [1, 2]>>
+  %1 = xegpu.create_tdesc %src, %0 : ui64, vector<4xindex> -> !xegpu.tensor_desc<4x2xf32, #xegpu.scatter_tdesc_attr<chunk_size = 2>, #xegpu.sg_map<wi_layout = [4, 1], wi_data = [1, 2]>>
+  gpu.return
+}
+
+// CHECK: gpu.func @test_load_with_sg_map(%[[arg0:.*]]: ui64) {
+gpu.func @test_load_with_sg_map(%src: ui64) {
+  //CHECK: %[[cst:.*]] = arith.constant dense<[0, 8, 16, 24]> : vector<4xindex>
+  %0 = arith.constant dense<[0, 8, 16, 24]> : vector<4xindex>
+  //CHECK: %[[cst1:.*]] = arith.constant dense<true> : vector<4xi1>
+  %1 = arith.constant dense<1>: vector<4xi1>
+  //CHECK: %[[R0:.*]] = xegpu.create_tdesc %[[arg0]], %[[cst]] : ui64, vector<4xindex> -> !xegpu.tensor_desc<4x2xf32, #xegpu.scatter_tdesc_attr<chunk_size = 2 : i64>, #xegpu.sg_map<wi_layout = [4, 1], wi_data = [1, 2]>>
+  %2 = xegpu.create_tdesc %src, %0 : ui64, vector<4xindex> -> !xegpu.tensor_desc<4x2xf32, #xegpu.scatter_tdesc_attr<chunk_size = 2>, #xegpu.sg_map<wi_layout = [4, 1], wi_data = [1, 2]>>
+  //CHECK: %[[R1:.*]] = xegpu.load %[[R0]], %[[cst1]] <{l1_hint = #xegpu.cache_hint<cached>, l2_hint = #xegpu.cache_hint<uncached>, transpose}> : !xegpu.tensor_desc<4x2xf32, #xegpu.scatter_tdesc_attr<chunk_size = 2 : i64>, #xegpu.sg_map<wi_layout = [4, 1], wi_data = [1, 2]>>, vector<4xi1> -> vector<2x1xf32> 
+  %3 = xegpu.load %2, %1 <{l1_hint = #xegpu.cache_hint<cached>, l2_hint = #xegpu.cache_hint<uncached>, transpose}> : !xegpu.tensor_desc<4x2xf32, #xegpu.scatter_tdesc_attr<chunk_size = 2>, #xegpu.sg_map<wi_layout = [4, 1], wi_data = [1, 2]>>, vector<4xi1> -> vector<2x1xf32>
   gpu.return
 }
 
+// CHECK: gpu.func @test_load_with_sg_map_2(%[[arg0:.*]]: ui64) {
+gpu.func @test_load_with_sg_map_2(%src: ui64) {
+  //CHECK: %[[cst:.*]] = arith.constant dense<[0, 8, 16, 24]> : vector<4xindex>
+  %0 = arith.constant dense<[0, 8, 16, 24]> : vector<4xindex>
+  //CHECK: %[[cst1:.*]] = arith.constant dense<true> : vector<4xi1>
+  %1 = arith.constant dense<1>: vector<4xi1>
+  //CHECK: %[[R0:.*]] = xegpu.create_tdesc %[[arg0]], %[[cst]] : ui64, vector<4xindex> -> !xegpu.tensor_desc<4xf32, #xegpu.scatter_tdesc_attr<>, #xegpu.sg_map<wi_layout = [1, 4], wi_data = [1, 1]>>
+  %2 = xegpu.create_tdesc %src, %0 : ui64, vector<4xindex> -> !xegpu.tensor_desc<4xf32, #xegpu.scatter_tdesc_attr<>, #xegpu.sg_map<wi_layout = [1, 4], wi_data = [1, 1]>>
+  //CHECK: %[[R1:.*]] = xegpu.load %[[R0]], %[[cst1]] <{l1_hint = #xegpu.cache_hint<cached>, l2_hint = #xegpu.cache_hint<uncached>}> : !xegpu.tensor_desc<4xf32, #xegpu.scatter_tdesc_attr<>, #xegpu.sg_map<wi_layout = [1, 4], wi_data = [1, 1]>>, vector<4xi1> -> vector<1xf32> 
+  %3 = xegpu.load %2, %1 <{l1_hint = #xegpu.cache_hint<cached>, l2_hint = #xegpu.cache_hint<uncached>}> : !xegpu.tensor_desc<4xf32, #xegpu.scatter_tdesc_attr<>, #xegpu.sg_map<wi_layout = [1, 4], wi_data = [1, 1]>>, vector<4xi1> -> vector<1xf32>
+  gpu.return
+}
+
+// CHECK: gpu.func @test_store_with_sg_map(%[[arg0:.*]]: ui64) {
+gpu.func @test_store_with_sg_map(%src: ui64) {
+  //CHECK: %[[cst:.*]] = arith.constant dense<[0, 8, 16, 24]> : vector<4xindex>
+  %0 = arith.constant dense<[0, 8, 16, 24]> : vector<4xindex>
+  //CHECK: %[[cst1:.*]] = arith.constant dense<true> : vector<4xi1>
+  %1 = arith.constant dense<1>: vector<4xi1>
+  //CHECK: %[[cst2:.*]] = arith.constant dense<2.900000e+00> : vector<2x1xf32>
+  %2 = arith.constant dense<2.9>: vector<2x1xf32>
+  //CHECK: %[[R0:.*]] = xegpu.create_tdesc %[[arg0]], %[[cst]] : ui64, vector<4xindex> -> !xegpu.tensor_desc<4x2xf32, #xegpu.scatter_tdesc_attr<chunk_size = 2 : i64>, #xegpu.sg_map<wi_layout = [4, 1], wi_data = [1, 2]>> 
+  %3 = xegpu.create_tdesc %src, %0 : ui64, vector<4xindex> -> !xegpu.tensor_desc<4x2xf32, #xegpu.scatter_tdesc_attr<chunk_size = 2>, #xegpu.sg_map<wi_layout = [4, 1], wi_data = [1, 2]>>
+  //CHECK: xegpu.store %[[cst2]], %[[R0]], %[[cst1]] <{l1_hint = #xegpu.cache_hint<write_back>, l2_hint = #xegpu.cache_hint<uncached>, transpose}> : vector<2x1xf32>, !xegpu.tensor_desc<4x2xf32, #xegpu.scatter_tdesc_attr<chunk_size = 2 : i64>, #xegpu.sg_map<wi_layout = [4, 1], wi_data = [1, 2]>>, vector<4xi1>
+  xegpu.store %2, %3, %1 <{l1_hint = #xegpu.cache_hint<write_back>, l2_hint = #xegpu.cache_hint<uncached>, transpose}> : vector<2x1xf32>, !xegpu.tensor_desc<4x2xf32, #xegpu.scatter_tdesc_attr<chunk_size = 2>, #xegpu.sg_map<wi_layout = [4, 1], wi_data = [1, 2]>>, vector<4xi1>
+  gpu.return
+}
+
+// CHECK: gpu.func @test_store_with_sg_map_2(%[[arg0:.*]]: ui64) {
+gpu.func @test_store_with_sg_map_2(%src: ui64) {
+  //CHECK: %[[cst:.*]] = arith.constant dense<[0, 8, 16, 24]> : vector<4xindex>
+  %0 = arith.constant dense<[0, 8, 16, 24]> : vector<4xindex>
+  //CHECK: %[[cst1:.*]] = arith.constant dense<true> : vector<4xi1>
+  %1 = arith.constant dense<1>: vector<4xi1>
+  //CHECK: %[[cst2:.*]] = arith.constant dense<2.900000e+00> : vector<1xf32>
+  %2 = arith.constant dense<2.9>: vector<1xf32>
+  //CHECK: %[[R0:.*]] = xegpu.create_tdesc %[[arg0]], %[[cst]] : ui64, vector<4xindex> -> !xegpu.tensor_desc<4xf32, #xegpu.scatter_tdesc_attr<>, #xegpu.sg_map<wi_layout = [1, 4], wi_data = [1, 1]>> 
+  %3 = xegpu.create_tdesc %src, %0 : ui64, vector<4xindex> -> !xegpu.tensor_desc<4xf32, #xegpu.scatter_tdesc_attr<>, #xegpu.sg_map<wi_layout = [1, 4], wi_data = [1, 1]>>
+  //CHECK: xegpu.store %[[cst2]], %[[R0]], %[[cst1]] <{l1_hint = #xegpu.cache_hint<write_back>, l2_hint = #xegpu.cache_hint<uncached>}> : vector<1xf32>, !xegpu.tensor_desc<4xf32, #xegpu.scatter_tdesc_attr<>, #xegpu.sg_map<wi_layout = [1, 4], wi_data = [1, 1]>>, vector<4xi1>
+  xegpu.store %2, %3, %1 <{l1_hint = #xegpu.cache_hint<write_back>, l2_hint = #xegpu.cache_hint<uncached>}> : vector<1xf32>, !xegpu.tensor_desc<4xf32, #xegpu.scatter_tdesc_attr<>, #xegpu.sg_map<wi_layout = [1, 4], wi_data = [1, 1]>>, vector<4xi1>
+  gpu.return
+}
+
+
+
 // CHECK: gpu.func @test_prefetch_vc(%[[arg0:.*]]: ui64) {
 gpu.func @test_prefetch_vc(%src: ui64) {
   //CHECK: %[[cst:.*]] = arith.constant dense<[0, 8, 16, 24]> : vector<4xindex>
diff --git a/mlir/test/Dialect/XeGPU/invalid.mlir b/mlir/test/Dialect/XeGPU/invalid.mlir
index 7816bff0582f8..86356e09de57c 100644
--- a/mlir/test/Dialect/XeGPU/invalid.mlir
+++ b/mlir/test/Dialect/XeGPU/invalid.mlir
@@ -17,7 +17,7 @@ func.func @test_create_nd_tdesc_vc_2(%src: memref<24x32xf32>) {
 
 // -----
 func.func @test_create_nd_tdesc_vc_3(%src: memref<2x24x32xf32, 3>) {
-  // expected-error@+1 {{SLM is not supported for 2D Block TensorDesc}}
+  // expected-error@+1 {{SLM is not supported for 2D block tensor}}
   %1 = xegpu.create_nd_tdesc %src[0, 0, 0] : memref<2x24x32xf32, 3> -> !xegpu.tensor_desc<8x16xf32, #xegpu.block_tdesc_attr<memory_space = slm>>
   return
 }
@@ -82,16 +82,33 @@ func.func @test_load_nd_vc_4(%src: memref<24x32xf32>) {
   %1 = xegpu.create_nd_tdesc %src[0, 0] : memref<24x32xf32> ->
     !xegpu.tensor_desc<8x16xf32, #xegpu.sg_map<wi_layout = [1, 16], wi_data = [1, 1]>>
   // expected-error@+1 {{Result shape doesn't match TensorDesc shape.}}
-  %2 = xegpu.load_nd %1 <{l1_hint = #xegpu.cache_hint<cached>, l2_hint = #xegpu.cache_hint<uncached>}> : !xegpu.tensor_desc<8x16xf32, #xegpu.sg_map<wi_layout = [1, 16], wi_data = [1, 1]>> -> vector<8x2xf32>
+  %2 = xegpu.load_nd %1 <{l1_hint = #xegpu.cache_hint<cached>,
+      l2_hint = #xegpu.cache_hint<uncached>}>
+    : !xegpu.tensor_desc<8x16xf32, #xegpu.sg_map<wi_layout = [1, 16], wi_data = [1, 1]>>
+    -> vector<8x2xf32>
   return
 }
 
 // -----
 func.func @test_load_nd_vc_5(%src: memref<24x32xf32>) {
   %1 = xegpu.create_nd_tdesc %src[0, 0] : memref<24x32xf32> ->
-      !xegpu.tensor_desc<16xf32, #xegpu.sg_map<wi_layout = [1, 16], wi_data = [1, 1]>>
+    !xegpu.tensor_desc<16xf32, #xegpu.sg_map<wi_layout = [1, 16], wi_data = [1, 1]>>
   // expected-error@+1 {{Result shape doesn't match TensorDesc shape.}}
-  %2 = xegpu.load_nd %1: !xegpu.tensor_desc<16xf32, #xegpu.sg_map<wi_layout = [1, 16], wi_data = [1, 1]>> -> vector<16xf32>
+  %2 = xegpu.load_nd %1 <{l1_hint = #xegpu.cache_hint<cached>,
+      l2_hint = #xegpu.cache_hint<uncached>}>
+    : !xegpu.tensor_desc<16xf32, #xegpu.sg_map<wi_layout = [1, 16], wi_data = [1, 1]>>
+    -> vector<8xf32>
+  return
+}
+
+// -----
+func.func @test_load_nd_vc_6(%src: memref<24x32xf32>) {
+  %1 = xegpu.create_nd_tdesc %src[0, 0] : memref<24x32xf32> ->
+    !xegpu.tensor_desc<8x16xf32>
+  // expected-error@+1 {{Result shape doesn't match TensorDesc shape.}}
+  %2 = xegpu.load_nd %1 <{l1_hint = #xegpu.cache_hint<cached>,
+      l2_hint = #xegpu.cache_hint<uncached>}>
+    : !xegpu.tensor_desc<8x16xf32> -> vector<8x1xf32>
   return
 }
 
@@ -116,6 +133,35 @@ func.func @test_store_nd_vc_2(%dst: memref<16xf16>) {
   return
 }
 
+// -----
+func.func @test_store_nd_vc_3(%dst: memref<24x32xf32>, %data: vector<8x2xf32>) {
+  %1 = xegpu.create_nd_tdesc %dst[0, 0] : memref<24x32xf32> ->
+    !xegpu.tensor_desc<8x16xf32, #xegpu.sg_map<wi_layout = [1, 16], wi_data = [1, 1]>>
+  // expected-error@+1 {{Result shape doesn't match TensorDesc shape.}}
+  xegpu.store_nd %data, %1
+    : vector<8x2xf32>, !xegpu.tensor_desc<8x16xf32, #xegpu.sg_map<wi_layout = [1, 16], wi_data = [1, 1]>>
+  return
+}
+
+// -----
+func.func @test_store_nd_vc_4(%dst: memref<24x32xf32>, %data: vector<2xf32>) {
+  %1 = xegpu.create_nd_tdesc %dst[0, 0] : memref<24x32xf32> ->
+    !xegpu.tensor_desc<16xf32, #xegpu.sg_map<wi_layout = [1, 16], wi_data = [1, 1]>>
+  // expected-error@+1 {{Result shape doesn't match TensorDesc shape.}}
+  xegpu.store_nd %data, %1
+    : vector<2xf32>, !xegpu.tensor_desc<16xf32, #xegpu.sg_map<wi_layout = [1, 16], wi_data = [1, 1]>>
+  return
+}
+
+// -----
+func.func @test_store_nd_vc_5(%dst: memref<24x32xf32>, %data: vector<8x1xf32>) {
+  %1 = xegpu.create_nd_tdesc %dst[0, 0] : memref<24x32xf32> ->
+    !xegpu.tensor_desc<8x16xf32>
+  // expected-error@+1 {{Result shape doesn't match TensorDesc shape.}}
+  xegpu.store_nd %data, %1 : vector<8x1xf32>, !xegpu.tensor_desc<8x16xf32>
+  return
+}
+
 // -----
 func.func @test_update_nd_offset_1(%dst: memref<16xf16>) {
   %0 = arith.constant dense<[0, 2, 4, 6, 8, 10, 12, 14]> : vector<8xindex>
@@ -137,14 +183,14 @@ func.func @test_create_tdesc_vc_1(%src: ui64) {
 // -----
 func.func @test_create_tdesc_vc_2(%src: ui64) {
   %0 = arith.constant dense<[0, 2, 4, 6, 8, 10, 12, 14]> : vector<8xindex>
-  // expected-error@+1 {{Incorrect TensorDesc shape}}
   %1 = xegpu.create_tdesc %src, %0 : ui64, vector<8xindex>
+  // expected-error@+1 {{expected chunk blocks for 2D tensor}}
           -> !xegpu.tensor_desc<8x4xf16, #xegpu.scatter_tdesc_attr<>>
   return
 }
 
 // -----
-func.func @test_create_tdesc_vc_1(%src: memref<?xf32>) {
+func.func @test_create_tdesc_vc_3(%src: memref<?xf32>) {
   %0 = arith.constant dense<[0, 8, 16, 24]> : vector<4xindex>
   // expected-error@+1 {{Memory space mismatch}}
   %1 = xegpu.create_tdesc %src, %0 : memref<?xf32>, vector<4xindex>
@@ -152,6 +198,15 @@ func.func @test_create_tdesc_vc_1(%src: memref<?xf32>) {
   return
 }
 
+// -----
+func.func @test_create_tdesc_vc_4(%src: memref<?xf32>) {
+  %0 = arith.constant dense<[0, 8, 16, 24]> : vector<4xindex>
+  %1 = xegpu.create_tdesc %src, %0 : memref<?xf32>, vector<4xindex>
+  // expected-error@+1 {{invalid chunk size}}
+          -> !xegpu.tensor_desc<4x5xf32, #xegpu.scatter_tdesc_attr<chunk_size = 5>>
+  return
+}
+
 // -----
 func.func @test_prefetch_vc_1(%src: memref<24x32xf16>) {
   %1 = xegpu.create_nd_tdesc %src[0, 0] : memref<24x32xf16> -> !xegpu.tensor_desc<24x32xf16>
@@ -170,6 +225,83 @@ func.func @test_prefetch_vc_2(%src: ui64) {
   return
 }
 
+// -----
+func.func @test_create_tdesc_sg_map_1(%src: ui64) {
+  %cst = arith.constant dense<[0, 8, 16, 24]> : vector<4xindex>
+  // expected-error@+1 {{outer layout distribution and data mapping must be 1 for 1D tensor}}
+  %1 = xegpu.create_tdesc %src, %cst : ui64, vector<4xindex> -> !xegpu.tensor_desc<4xf32, #xegpu.scatter_tdesc_attr<>, #xegpu.sg_map<wi_layout = [4, 1], wi_data = [1, 1]>>
+  return
+}
+
+// -----
+func.func @test_create_tdesc_sg_map_2(%src: ui64) {
+  %cst = arith.constant dense<[0, 8, 16, 24]> : vector<4xindex>
+  // expected-error@+1 {{cannot map over non-contiguous scattered row elements}}
+  %1 = xegpu.create_tdesc %src, %cst : ui64, vector<4xindex> -> !xegpu.tensor_desc<4x2xf32, #xegpu.scatter_tdesc_attr<chunk_size = 2>, #xegpu.sg_map<wi_layout = [1, 4], wi_data = [2, 1]>>
+  return
+}
+
+// -----
+func.func @test_create_tdesc_sg_map_3(%src: ui64) {
+  %cst = arith.constant dense<[0, 8, 16, 24]> : vector<4xindex>
+  // expected-error@+1 {{work item data mapping must match the number of contiguous elements}}
+  %1 = xegpu.create_tdesc %src, %cst : ui64, vector<4xindex> -> !xegpu.tensor_desc<4x3xf32, #xegpu.scatter_tdesc_attr<chunk_size = 3>, #xegpu.sg_map<wi_layout = [4, 1], wi_data = [1, 2]>>
+  return
+}
+
+// -----
+func.func @test_load_gather_sg_map_1(%src: ui64) {
+  %0 = arith.constant dense<1>: vector<4xi1>
+  %cst = arith.constant dense<[0, 8, 16, 24]> : vector<4xindex>
+  %1 = xegpu.create_tdesc %src, %cst : ui64, vector<4xindex> -> !xegpu.tensor_desc<4x2xf32, #xegpu.scatter_tdesc_attr<chunk_size = 2>, #xegpu.sg_map<wi_layout = [4, 1], wi_data = [1, 2]>>
+  // expected-error@+1 {{Unexpected result shape(Expected shape: [2, 1], Given shape: [1, 2])}}
+  %2 = xegpu.load %1, %0 <{l1_hint = #xegpu.cache_hint<cached>, transpose}> : !xegpu.tensor_desc<4x2xf32, #xegpu.scatter_tdesc_attr<chunk_size = 2>, #xegpu.sg_map<wi_layout = [4, 1], wi_data = [1, 2]>>, vector<4xi1> -> vector<1x2xf32>
+  return
+}
+
+// -----
+func.func @test_load_gather_sg_map_2(%src: ui64) {
+  %0 = arith.constant dense<1>: vector<4xi1>
+  %cst = arith.constant dense<[0, 8, 16, 24]> : vector<4xindex>
+  %1 = xegpu.create_tdesc %src, %cst : ui64, vector<4xindex> -> !xegpu.tensor_desc<4x2xf32, #xegpu.scatter_tdesc_attr<chunk_size = 2>, #xegpu.sg_map<wi_layout = [4, 1], wi_data = [1, 2]>>
+  // expected-error@+1 {{Unexpected result shape(Expected shape: [2, 1], Given shape: [2])}}
+  %2 = xegpu.load %1, %0 <{l1_hint = #xegpu.cache_hint<cached>, transpose}> : !xegpu.tensor_desc<4x2xf32, #xegpu.scatter_tdesc_attr<chunk_size = 2>, #xegpu.sg_map<wi_layout = [4, 1], wi_data = [1, 2]>>, vector<4xi1> -> vector<2xf32>
+  return
+}
+
+// -----
+func.func @test_load_gather_sg_map_3(%src: ui64) {
+  %0 = arith.constant dense<1>: vector<4xi1>
+  %cst = arith.constant dense<[0, 8, 16, 24]> : vector<4xindex>
+  %1 = xegpu.create_tdesc %src, %cst : ui64, vector<4xindex> -> !xegpu.tensor_desc<4x2xf32, #xegpu.scatter_tdesc_attr<chunk_size = 2>, #xegpu.sg_map<wi_layout = [4, 1], wi_data = [1, 2]>>
+  // expected-error@+1 {{Chunk size, vector size and wi_data must match}}
+  %2 = xegpu.load %1, %0 <{l1_hint = #xegpu.cache_hint<cached>, transpose}> : !xegpu.tensor_desc<4x2xf32, #xegpu.scatter_tdesc_attr<chunk_size = 2>, #xegpu.sg_map<wi_layout = [4, 1], wi_data = [1, 2]>>, vector<4xi1> -> vector<1xf32>
+  return
+}
+
+
+// -----
+func.func @test_store_scatter_sg_map_1(%src: ui64) {
+  %0 = arith.constant dense<1>: vector<4xi1>
+  %cst = arith.constant dense<[0, 8, 16, 24]> : vector<4xindex>
+  %val = arith.constant dense<2.9>: vector<1x2xf32>
+  %1 = xegpu.create_tdesc %src, %cst : ui64, vector<4xindex> -> !xegpu.tensor_desc<4x2xf32, #xegpu.scatter_tdesc_attr<chunk_size = 2>, #xegpu.sg_map<wi_layout = [4, 1], wi_data = [1, 2]>>
+  // expected-error@+1 {{Unexpected value shape(Expected shape: [2, 1], Given shape: [1, 2])}}
+  xegpu.store %val, %1, %0 <{l1_hint = #xegpu.cache_hint<cached>, transpose}> : vector<1x2xf32>, !xegpu.tensor_desc<4x2xf32, #xegpu.scatter_tdesc_attr<chunk_size = 2>, #xegpu.sg_map<wi_layout = [4, 1], wi_data = [1, 2]>>, vector<4xi1>
+  return
+}
+
+// -----
+func.func @test_store_scatter_sg_map_2(%src: ui64) {
+  %0 = arith.constant dense<1>: vector<4xi1>
+  %cst = arith.constant dense<[0, 8, 16, 24]> : vector<4xindex>
+  %val = arith.constant dense<2.9>: vector<2xf32>
+  %1 = xegpu.create_tdesc %src, %cst : ui64, vector<4xindex> -> !xegpu.tensor_desc<4x2xf32, #xegpu.scatter_tdesc_attr<chunk_size = 2>, #xegpu.sg_map<wi_layout = [4, 1], wi_data = [1, 2]>>
+  // expected-error@+1 {{Unexpected value shape(Expected shape: [2, 1], Given shape: [2])}}
+  xegpu.store %val, %1, %0 <{l1_hint = #xegpu.cache_hint<cached>, transpose}> : vector<2xf32>, !xegpu.tensor_desc<4x2xf32, #xegpu.scatter_tdesc_attr<chunk_size = 2>, #xegpu.sg_map<wi_layout = [4, 1], wi_data = [1, 2]>>, vector<4xi1>
+  return
+}
+
 // -----
 func.func @test_load_gather_vc_1(%src: memref<24x32xf16>) {
   %0 = arith.constant dense<1>: vector<4xi1>
@@ -238,4 +370,109 @@ func.func @test_atomic_rmw(%src: ui64, %value : vector<16x4xf32>, %mask : vector
   // expected-error@+1 {{failed to verify that all of {tensorDesc, value, result} have same shape}}
   xegpu.atomic_rmw addf %1, %mask, %value: !xegpu.tensor_desc<16x8xf32, #xegpu.scatter_tdesc_attr<chunk_size = 8>>, vector<16xi1>, vector<16x4xf32> -> vector<16x8xf32>
   return
-}
\ No newline at end of file
+}
+
+// -----
+func.func @tensor_desc_invalid_rank(%src: memref<24x32xf32>) {
+  %0 = xegpu.create_nd_tdesc %src[0, 0] : memref<24x32xf32> ->
+      // expected-error@+1 {{expected 1D or 2D tensor}}
+      !xegpu.tensor_desc<16x2x2xf32>
+  return
+}
+
+// -----
+func.func @tensor_desc_invalid_rank_1(%src: memref<24x32xf32>) {
+  %0 = xegpu.create_nd_tdesc %src[0, 0] : memref<24x32xf32> ->
+      // expected-error@+1 {{expected 1D or 2D tensor}}
+      !xegpu.tensor_desc<f32>
+  return
+}
+
+// -----
+func.func @tensor_desc_1D_invalid_map_layout(%src: memref<24x32xf32>) {
+  %0 = xegpu.create_nd_tdesc %src[0, 0] : memref<24x32xf32> ->
+      // expected-error@+1 {{outer layout distribution and data mapping must be 1 for 1D tensor}}
+      !xegpu.tensor_desc<16xf32, #xegpu.sg_map<wi_layout = [2, 16], wi_data = [1, 1]>>
+  return
+}
+
+// -----
+func.func @tensor_desc_1D_invalid_map_data(%src: memref<24x32xf32>) {
+  %0 = xegpu.create_nd_tdesc %src[0, 0] : memref<24x32xf32> ->
+      // expected-error@+1 {{outer layout distribution and data mapping must be 1 for 1D tensor}}
+      !xegpu.tensor_desc<16xf32, #xegpu.sg_map<wi_layout = [1, 16], wi_data = [2, 1]>>
+  return
+}
+
+// -----
+func.func @tensor_desc_invalid_map_layout(%src: memref<24x32xf32>) {
+  %0 = xegpu.create_nd_tdesc %src[0, 0] : memref<24x32xf32> ->
+      // expected-error@+1 {{cannot distribute 8 over 16 work items with 1 elements each}}
+      !xegpu.tensor_desc<4x8xf32, #xegpu.sg_map<wi_layout = [1, 16], wi_data = [1, 1]>>
+  return
+}
+
+// -----
+func.func @tensor_desc_invalid_map_layout_1(%src: memref<24x32xf32>) {
+  %0 = xegpu.create_nd_tdesc %src[0, 0] : memref<24x32xf32> ->
+      // expected-error@+1 {{cannot distribute 4 over 8 work items with 1 elements each}}
+      !xegpu.tensor_desc<4x8xf32, #xegpu.sg_map<wi_layout = [8, 2], wi_data = [1, 1]>>
+  return
+}
+
+// -----
+func.func @tensor_desc_invalid_map_data(%src: memref<24x32xf32>) {
+  %0 = xegpu.create_nd_tdesc %src[0, 0] : memref<24x32xf32> ->
+      // expected-error@+1 {{cannot distribute 4 over 2 work items with 4 elements each}}
+      !xegpu.tensor_desc<4x8xf32, #xegpu.sg_map<wi_layout = [2, 8], wi_data = [4, 1]>>
+  return
+}
+
+// -----
+func.func @tensor_desc_invalid_map_data_1(%src: memref<24x32xf32>) {
+  %0 = xegpu.create_nd_tdesc %src[0, 0] : memref<24x32xf32> ->
+      // expected-error@+1 {{cannot distribute 4 over 8 work items with 1 elements each}}
+      !xegpu.tensor_desc<4x8xf32, #xegpu.sg_map<wi_layout = [8, 2], wi_data = [1, 2]>>
+  return
+}
+
+// -----
+func.func @tensor_desc_scatter_invalid_map_data(%src: ui64) {
+  %0 = arith.constant dense<[0, 8, 16, 24]> : vector<4xindex>
+  %1 = xegpu.create_tdesc %src, %0 : ui64, vector<4xindex> ->
+      // expected-error@+1 {{cannot map over non-contiguous scattered row elements}}
+      !xegpu.tensor_desc<4x2xf32,
+        #xegpu.scatter_tdesc_attr<chunk_size = 2>,
+        #xegpu.sg_map<wi_layout = [1, 1], wi_data = [2, 1]>>
+  return
+}
+
+// -----
+func.func @tensor_desc_scatter_invalid_map_data_1(%src: ui64, %offsets: vector<16xindex>) {
+  %1 = xegpu.create_tdesc %src, %offsets : ui64, vector<16xindex> ->
+      // expected-error@+1 {{work item data mapping must match the number of contiguous elements}}
+      !xegpu.tensor_desc<16xf32,
+        #xegpu.scatter_tdesc_attr<chunk_size = 1>,
+        #xegpu.sg_map<wi_layout = [1, 8], wi_data = [1, 2]>>
+  return
+}
+
+// -----
+func.func @tensor_desc_scatter_invalid_chunk_size_1D(%src: ui64, %offsets: vector<16xindex>) {
+  %1 = xegpu.create_tdesc %src, %offsets : ui64, vector<16xindex> ->
+      // expected-error@+1 {{expected non-contiguous elements for 1D tensor}}
+      !xegpu.tensor_desc<16xf32,
+        #xegpu.scatter_tdesc_attr<chunk_size = 2>,
+        #xegpu.sg_map<wi_layout = [1, 8], wi_data = [1, 2]>>
+  return
+}
+
+// -----
+func.func @tensor_desc_scatter_invalid_chunk_size_2D(%src: ui64, %offsets: vector<16xindex>) {
+  %1 = xegpu.create_tdesc %src, %offsets : ui64, vector<16xindex> ->
+      // expected-error@+1 {{expected chunk blocks for 2D tensor}}
+      !xegpu.tensor_desc<16x2xf32,
+        #xegpu.scatter_tdesc_attr<chunk_size = 1>,
+        #xegpu.sg_map<wi_layout = [8, 1], wi_data = [1, 2]>>
+  return
+}
diff --git a/mlir/test/Examples/transform/ChH/full.mlir b/mlir/test/Examples/transform/ChH/full.mlir
index 259475ebdbf49..d49524b529a1d 100644
--- a/mlir/test/Examples/transform/ChH/full.mlir
+++ b/mlir/test/Examples/transform/ChH/full.mlir
@@ -379,7 +379,7 @@ module attributes { transform.with_named_sequence } {
 // The core computation, at the LLVM dialect level, must correspond to five
 // immediately adjacent fma on vector<64xf32>.
 
-// CHECK:      %[[R0:.+]] = llvm.mlir.undef : !llvm.array<5 x vector<64xf32>>
+// CHECK:      %[[R0:.+]] = llvm.mlir.poison : !llvm.array<5 x vector<64xf32>>
 
 // CHECK:      %[[V:.+]] = llvm.load %{{.*}} : !llvm.ptr -> !llvm.array<5 x vector<64xf32>>
 // CHECK-NEXT: %[[LINE0:.+]] = llvm.extractvalue %[[V]][0] : !llvm.array<5 x vector<64xf32>>
diff --git a/mlir/test/IR/attribute.mlir b/mlir/test/IR/attribute.mlir
index 0085d64ae82b6..5a005a393d8ac 100644
--- a/mlir/test/IR/attribute.mlir
+++ b/mlir/test/IR/attribute.mlir
@@ -416,10 +416,29 @@ func.func @non_type_in_type_array_attr_fail() {
 // Test StringAttr with custom type
 //===----------------------------------------------------------------------===//
 
-// CHECK-LABEL: func @string_attr_custom_type
-func.func @string_attr_custom_type() {
-  // CHECK: "string_data" : !foo.string
-  test.string_attr_with_type "string_data" : !foo.string
+// CHECK-LABEL: func @string_attr_custom_type_valid
+func.func @string_attr_custom_type_valid() {
+  // CHECK: "string_data" : i64
+  test.string_attr_with_type "string_data" : i64
+  return
+}
+
+// -----
+
+func.func @string_attr_custom_type_invalid() {
+  // expected-error @+1 {{'attr' failed to satisfy constraint: string attribute of integer}}
+  test.string_attr_with_type "string_data" : f32
+  return
+}
+
+// -----
+
+// CHECK-LABEL: func @string_attr_custom_mixed_type
+func.func @string_attr_custom_mixed_type() {
+  // CHECK: "string_data" : i64
+  test.string_attr_with_mixed_type "string_data" : i64
+  // CHECK: 42 : i64
+  test.string_attr_with_mixed_type 42 : i64
   return
 }
 
diff --git a/mlir/test/IR/dense-resource-elements-attr.mlir b/mlir/test/IR/dense-resource-elements-attr.mlir
index adba97994ff60..44cefc3aa1616 100644
--- a/mlir/test/IR/dense-resource-elements-attr.mlir
+++ b/mlir/test/IR/dense-resource-elements-attr.mlir
@@ -11,3 +11,18 @@
     }
   }
 #-}
+
+// -----
+
+// DenseResourceElementsHandle key blob\-"one" is quoted and escaped.
+// CHECK: attr = dense_resource<"blob\\-\22one\22"> : tensor<2xi16>
+"test.user_op"() {attr = dense_resource<"blob\\-\22one\22"> : tensor<2xi16>} : () -> ()
+
+{-#
+  dialect_resources: {
+    builtin: {
+      // CHECK: "blob\\-\22one\22": "0x0200000001000200"
+      "blob\\-\22one\22": "0x0200000001000200"
+    }
+  }
+#-}
diff --git a/mlir/test/IR/op-asm-interface.mlir b/mlir/test/IR/op-asm-interface.mlir
new file mode 100644
index 0000000000000..a9c199e3dc973
--- /dev/null
+++ b/mlir/test/IR/op-asm-interface.mlir
@@ -0,0 +1,24 @@
+// RUN: mlir-opt %s -split-input-file -verify-diagnostics | FileCheck %s
+
+//===----------------------------------------------------------------------===//
+// Test OpAsmOpInterface
+//===----------------------------------------------------------------------===//
+
+func.func @result_name_from_op_asm_type_interface() {
+  // CHECK-LABEL: @result_name_from_op_asm_type_interface
+  // CHECK: %op_asm_type_interface
+  %0 = "test.result_name_from_type"() : () -> !test.op_asm_type_interface
+  return
+}
+
+// -----
+
+func.func @block_argument_name_from_op_asm_type_interface() {
+  // CHECK-LABEL: @block_argument_name_from_op_asm_type_interface
+  // CHECK: ^bb0(%op_asm_type_interface
+  test.block_argument_name_from_type {
+    ^bb0(%arg0: !test.op_asm_type_interface):
+      "test.terminator"() : ()->()
+  }
+  return
+}
diff --git a/mlir/test/IR/pretty-resources-print.mlir b/mlir/test/IR/pretty-resources-print.mlir
index 625967fcb7603..297c83bbb1389 100644
--- a/mlir/test/IR/pretty-resources-print.mlir
+++ b/mlir/test/IR/pretty-resources-print.mlir
@@ -12,7 +12,7 @@
 // CHECK:      {-#
 // CHECK-NEXT:   external_resources: {
 // CHECK-NEXT:     external: {
-// CHECK-NEXT:       bool: true,
+// CHECK-NEXT:       "backslash\\tab\09": true,
 // CHECK-NEXT:       string: "\22string\22"
 // CHECK-NEXT:     },
 // CHECK-NEXT:     other_stuff: {
@@ -31,8 +31,8 @@
   external_resources: {
     external: {
       blob: "0x08000000010000000000000002000000000000000300000000000000",
-      bool: true,
-      string: "\"string\"" // with escape characters
+      "backslash\\tab\09": true, // quoted key with escape characters
+      string: "\"string\"" // string with escape characters
     },
     other_stuff: {
       bool: true
diff --git a/mlir/test/Integration/Dialect/Vector/CPU/shuffle16x16.mlir b/mlir/test/Integration/Dialect/Vector/CPU/shuffle16x16.mlir
index 03bc2b5123c48..33a76b13c245a 100644
--- a/mlir/test/Integration/Dialect/Vector/CPU/shuffle16x16.mlir
+++ b/mlir/test/Integration/Dialect/Vector/CPU/shuffle16x16.mlir
@@ -1,7 +1,9 @@
 // RUN: mlir-opt %s -convert-vector-to-scf -convert-scf-to-cf \
 // RUN:   -transform-interpreter \
 // RUN:   -test-transform-dialect-erase-schedule \
-// RUN:   -convert-vector-to-llvm -convert-func-to-llvm -convert-arith-to-llvm -convert-cf-to-llvm -reconcile-unrealized-casts | \
+// RUN:   -convert-vector-to-llvm -convert-func-to-llvm \
+// RUN:   -convert-arith-to-llvm -convert-cf-to-llvm \
+// RUN:   -convert-ub-to-llvm -reconcile-unrealized-casts | \
 // RUN: mlir-runner -e entry -entry-point-result=void  \
 // RUN:   -shared-libs=%mlir_c_runner_utils | \
 // RUN: FileCheck %s
diff --git a/mlir/test/Integration/Dialect/Vector/CPU/transfer-read-1d.mlir b/mlir/test/Integration/Dialect/Vector/CPU/transfer-read-1d.mlir
index c4222a744f0a9..2393bd1c79dea 100644
--- a/mlir/test/Integration/Dialect/Vector/CPU/transfer-read-1d.mlir
+++ b/mlir/test/Integration/Dialect/Vector/CPU/transfer-read-1d.mlir
@@ -1,9 +1,9 @@
-// RUN: mlir-opt %s -pass-pipeline="builtin.module(func.func(convert-vector-to-scf,expand-strided-metadata,lower-affine,convert-arith-to-llvm,convert-scf-to-cf),convert-vector-to-llvm,finalize-memref-to-llvm,convert-func-to-llvm,convert-arith-to-llvm,convert-cf-to-llvm,reconcile-unrealized-casts)" | \
+// RUN: mlir-opt %s -pass-pipeline="builtin.module(func.func(convert-vector-to-scf,expand-strided-metadata,lower-affine,convert-arith-to-llvm,convert-scf-to-cf),convert-vector-to-llvm,finalize-memref-to-llvm,convert-func-to-llvm,convert-arith-to-llvm,convert-ub-to-llvm,convert-cf-to-llvm,reconcile-unrealized-casts)" | \
 // RUN: mlir-runner -e entry -entry-point-result=void  \
 // RUN:   -shared-libs=%mlir_c_runner_utils | \
 // RUN: FileCheck %s
 
-// RUN: mlir-opt %s -pass-pipeline="builtin.module(func.func(convert-vector-to-scf{full-unroll=true},expand-strided-metadata,lower-affine,convert-arith-to-llvm,convert-scf-to-cf),convert-vector-to-llvm,finalize-memref-to-llvm,convert-func-to-llvm,convert-arith-to-llvm,convert-cf-to-llvm,reconcile-unrealized-casts)" | \
+// RUN: mlir-opt %s -pass-pipeline="builtin.module(func.func(convert-vector-to-scf{full-unroll=true},expand-strided-metadata,lower-affine,convert-arith-to-llvm,convert-scf-to-cf),convert-vector-to-llvm,finalize-memref-to-llvm,convert-func-to-llvm,convert-arith-to-llvm,convert-ub-to-llvm,convert-cf-to-llvm,reconcile-unrealized-casts)" | \
 // RUN: mlir-runner -e entry -entry-point-result=void  \
 // RUN:   -shared-libs=%mlir_c_runner_utils | \
 // RUN: FileCheck %s
diff --git a/mlir/test/Integration/Dialect/Vector/CPU/transfer-read-2d.mlir b/mlir/test/Integration/Dialect/Vector/CPU/transfer-read-2d.mlir
index 61e25c2dee732..009c1375beaab 100644
--- a/mlir/test/Integration/Dialect/Vector/CPU/transfer-read-2d.mlir
+++ b/mlir/test/Integration/Dialect/Vector/CPU/transfer-read-2d.mlir
@@ -1,9 +1,9 @@
-// RUN: mlir-opt %s -pass-pipeline="builtin.module(func.func(convert-vector-to-scf,lower-affine,convert-scf-to-cf),convert-vector-to-llvm,finalize-memref-to-llvm,convert-func-to-llvm,convert-arith-to-llvm,convert-cf-to-llvm,reconcile-unrealized-casts)" | \
+// RUN: mlir-opt %s -pass-pipeline="builtin.module(func.func(convert-vector-to-scf,lower-affine,convert-scf-to-cf),convert-vector-to-llvm,finalize-memref-to-llvm,convert-func-to-llvm,convert-arith-to-llvm,convert-cf-to-llvm,convert-ub-to-llvm,reconcile-unrealized-casts)" | \
 // RUN: mlir-runner -e entry -entry-point-result=void  \
 // RUN:   -shared-libs=%mlir_c_runner_utils | \
 // RUN: FileCheck %s
 
-// RUN: mlir-opt %s -pass-pipeline="builtin.module(func.func(convert-vector-to-scf{full-unroll=true},lower-affine,convert-scf-to-cf),convert-vector-to-llvm,finalize-memref-to-llvm,convert-func-to-llvm,convert-arith-to-llvm,convert-cf-to-llvm,reconcile-unrealized-casts)" | \
+// RUN: mlir-opt %s -pass-pipeline="builtin.module(func.func(convert-vector-to-scf{full-unroll=true},lower-affine,convert-scf-to-cf),convert-vector-to-llvm,finalize-memref-to-llvm,convert-func-to-llvm,convert-arith-to-llvm,convert-cf-to-llvm,convert-ub-to-llvm,reconcile-unrealized-casts)" | \
 // RUN: mlir-runner -e entry -entry-point-result=void  \
 // RUN:   -shared-libs=%mlir_c_runner_utils | \
 // RUN: FileCheck %s
diff --git a/mlir/test/Integration/Dialect/Vector/CPU/transfer-read-3d.mlir b/mlir/test/Integration/Dialect/Vector/CPU/transfer-read-3d.mlir
index 6c120d1b28c38..d41d9c93bc2b3 100644
--- a/mlir/test/Integration/Dialect/Vector/CPU/transfer-read-3d.mlir
+++ b/mlir/test/Integration/Dialect/Vector/CPU/transfer-read-3d.mlir
@@ -1,9 +1,9 @@
-// RUN: mlir-opt %s -pass-pipeline="builtin.module(func.func(convert-vector-to-scf,lower-affine,convert-scf-to-cf),convert-vector-to-llvm,finalize-memref-to-llvm,convert-func-to-llvm,convert-arith-to-llvm,convert-cf-to-llvm,reconcile-unrealized-casts)" | \
+// RUN: mlir-opt %s -pass-pipeline="builtin.module(func.func(convert-vector-to-scf,lower-affine,convert-scf-to-cf),convert-vector-to-llvm,finalize-memref-to-llvm,convert-func-to-llvm,convert-arith-to-llvm,convert-cf-to-llvm,convert-ub-to-llvm,reconcile-unrealized-casts)" | \
 // RUN: mlir-runner -e entry -entry-point-result=void  \
 // RUN:   -shared-libs=%mlir_c_runner_utils | \
 // RUN: FileCheck %s
 
-// RUN: mlir-opt %s -pass-pipeline="builtin.module(func.func(convert-vector-to-scf{full-unroll=true},lower-affine,convert-scf-to-cf),convert-vector-to-llvm,finalize-memref-to-llvm,convert-func-to-llvm,convert-arith-to-llvm,convert-cf-to-llvm,reconcile-unrealized-casts)" | \
+// RUN: mlir-opt %s -pass-pipeline="builtin.module(func.func(convert-vector-to-scf{full-unroll=true},lower-affine,convert-scf-to-cf),convert-vector-to-llvm,finalize-memref-to-llvm,convert-func-to-llvm,convert-arith-to-llvm,convert-cf-to-llvm,convert-ub-to-llvm,reconcile-unrealized-casts)" | \
 // RUN: mlir-runner -e entry -entry-point-result=void  \
 // RUN:   -shared-libs=%mlir_c_runner_utils | \
 // RUN: FileCheck %s
diff --git a/mlir/test/Integration/Dialect/Vector/CPU/transfer-to-loops.mlir b/mlir/test/Integration/Dialect/Vector/CPU/transfer-to-loops.mlir
index 1e74bc72e5b7f..a7fec69b69492 100644
--- a/mlir/test/Integration/Dialect/Vector/CPU/transfer-to-loops.mlir
+++ b/mlir/test/Integration/Dialect/Vector/CPU/transfer-to-loops.mlir
@@ -1,9 +1,9 @@
-// RUN: mlir-opt %s -pass-pipeline="builtin.module(func.func(convert-vector-to-scf,lower-affine,convert-scf-to-cf),convert-vector-to-llvm,finalize-memref-to-llvm,convert-func-to-llvm,convert-arith-to-llvm,convert-cf-to-llvm,reconcile-unrealized-casts)" | \
+// RUN: mlir-opt %s -pass-pipeline="builtin.module(func.func(convert-vector-to-scf,lower-affine,convert-scf-to-cf),convert-vector-to-llvm,finalize-memref-to-llvm,convert-func-to-llvm,convert-arith-to-llvm,convert-cf-to-llvm,convert-ub-to-llvm,reconcile-unrealized-casts)" | \
 // RUN: mlir-runner -e main -entry-point-result=void  \
 // RUN:   -shared-libs=%mlir_runner_utils,%mlir_c_runner_utils | \
 // RUN: FileCheck %s
 
-// RUN: mlir-opt %s -pass-pipeline="builtin.module(func.func(convert-vector-to-scf{full-unroll=true},lower-affine,convert-scf-to-cf),convert-vector-to-llvm,finalize-memref-to-llvm,convert-func-to-llvm,convert-arith-to-llvm,convert-cf-to-llvm,reconcile-unrealized-casts)" | \
+// RUN: mlir-opt %s -pass-pipeline="builtin.module(func.func(convert-vector-to-scf{full-unroll=true},lower-affine,convert-scf-to-cf),convert-vector-to-llvm,finalize-memref-to-llvm,convert-func-to-llvm,convert-arith-to-llvm,convert-cf-to-llvm,convert-ub-to-llvm,reconcile-unrealized-casts)" | \
 // RUN: mlir-runner -e main -entry-point-result=void  \
 // RUN:   -shared-libs=%mlir_runner_utils,%mlir_c_runner_utils | \
 // RUN: FileCheck %s
diff --git a/mlir/test/Interfaces/TilingInterface/tile-and-fuse-consumer.mlir b/mlir/test/Interfaces/TilingInterface/tile-and-fuse-consumer.mlir
index a2871b30698c5..2d35be403ef99 100644
--- a/mlir/test/Interfaces/TilingInterface/tile-and-fuse-consumer.mlir
+++ b/mlir/test/Interfaces/TilingInterface/tile-and-fuse-consumer.mlir
@@ -676,3 +676,65 @@ module attributes {transform.with_named_sequence} {
 //      CHECK:   }
 //      CHECK:   %[[RES_SLICE:.+]] = tensor.insert_slice
 //      CHECK:   return %[[LOOP_RESULT]]#1, %[[RES_SLICE]]
+
+// -----
+
+#map = affine_map<(d0, d1, d2) -> (d0, d1)>
+#map1 = affine_map<(d0, d1, d2) -> (d2)>
+#map2 = affine_map<(d0, d1, d2) -> (d0, d1, d2)>
+module {
+  func.func @fuse_with_tilable_consumer_with_projected_permutations(%arg0: tensor<256x256xf32>, %arg1: tensor<256x256xf32>, %arg2: tensor<24xf32>) -> tensor<256x256x24xf32> {
+    %c0 = arith.constant 0 : index
+    %c64 = arith.constant 64 : index
+    %c256 = arith.constant 256 : index
+    %0 = tensor.empty() : tensor<256x256xf32>
+    %1 = scf.for %arg3 = %c0 to %c256 step %c64 iter_args(%arg4 = %0) -> (tensor<256x256xf32>) {
+      %extracted_slice = tensor.extract_slice %arg4[%arg3, 0] [64, 256] [1, 1] : tensor<256x256xf32> to tensor<64x256xf32>
+      %extracted_slice_0 = tensor.extract_slice %arg0[%arg3, 0] [64, 256] [1, 1] : tensor<256x256xf32> to tensor<64x256xf32>
+      %extracted_slice_1 = tensor.extract_slice %arg1[%arg3, 0] [64, 256] [1, 1] : tensor<256x256xf32> to tensor<64x256xf32>
+      %4 = linalg.add ins(%extracted_slice_0, %extracted_slice_1 : tensor<64x256xf32>, tensor<64x256xf32>) outs(%extracted_slice : tensor<64x256xf32>) -> tensor<64x256xf32>
+      %inserted_slice = tensor.insert_slice %4 into %arg4[%arg3, 0] [64, 256] [1, 1] : tensor<64x256xf32> into tensor<256x256xf32>
+      scf.yield %inserted_slice : tensor<256x256xf32>
+    }
+    %2 = tensor.empty() : tensor<256x256x24xf32>
+    %3 = linalg.generic {indexing_maps = [#map, #map1, #map2], iterator_types = ["parallel", "parallel", "parallel"]} ins(%1, %arg2 : tensor<256x256xf32>, tensor<24xf32>) outs(%2 : tensor<256x256x24xf32>) {
+    ^bb0(%in: f32, %in_0: f32, %out: f32):
+      %4 = arith.addf %in, %in_0 : f32
+      linalg.yield %4 : f32
+    } -> tensor<256x256x24xf32>
+    return %3 : tensor<256x256x24xf32>
+  }
+}
+
+// CHECK: func.func @fuse_with_tilable_consumer_with_projected_permutations(%[[VAL_0:.*]]: tensor<256x256xf32>, %[[VAL_1:.*]]: tensor<256x256xf32>, %[[VAL_2:.*]]: tensor<24xf32>) -> tensor<256x256x24xf32> {
+// CHECK:             %[[VAL_3:.*]] = arith.constant 0 : index
+// CHECK:             %[[VAL_4:.*]] = arith.constant 64 : index
+// CHECK:             %[[VAL_5:.*]] = arith.constant 256 : index
+// CHECK:             %[[VAL_6:.*]] = tensor.empty() : tensor<256x256xf32>
+// CHECK:             %[[VAL_7:.*]] = tensor.empty() : tensor<256x256x24xf32>
+// CHECK:             %[[VAL_8:.*]]:2 = scf.for %[[VAL_9:.*]] = %[[VAL_3]] to %[[VAL_5]] step %[[VAL_4]] iter_args(%[[VAL_10:.*]] = %[[VAL_6]], %[[VAL_11:.*]] = %[[VAL_7]]) -> (tensor<256x256xf32>, tensor<256x256x24xf32>) {
+// CHECK:               %[[VAL_12:.*]] = tensor.extract_slice %[[VAL_10]]{{\[}}%[[VAL_9]], 0] [64, 256] [1, 1]
+// CHECK:               %[[VAL_13:.*]] = tensor.extract_slice %[[VAL_0]]{{\[}}%[[VAL_9]], 0] [64, 256] [1, 1]
+// CHECK:               %[[VAL_14:.*]] = tensor.extract_slice %[[VAL_1]]{{\[}}%[[VAL_9]], 0] [64, 256] [1, 1]
+// CHECK:               %[[VAL_15:.*]] = linalg.add ins(%[[VAL_13]], %[[VAL_14]] : tensor<64x256xf32>, tensor<64x256xf32>) outs(%[[VAL_12]] : tensor<64x256xf32>) -> tensor<64x256xf32>
+// CHECK:               %[[VAL_16:.*]] = tensor.insert_slice %[[VAL_15]] into %[[VAL_10]]{{\[}}%[[VAL_9]], 0] [64, 256] [1, 1]
+// CHECK:               %[[VAL_17:.*]] = tensor.extract_slice %[[VAL_2]][0] [24] [1] : tensor<24xf32> to tensor<24xf32>
+// CHECK:               %[[VAL_18:.*]] = tensor.extract_slice %[[VAL_11]]{{\[}}%[[VAL_9]], 0, 0] [64, 256, 24] [1, 1, 1]
+// CHECK:               %[[VAL_19:.*]] = linalg.generic {indexing_maps = [#map, #map1, #map2], iterator_types = ["parallel", "parallel", "parallel"]} ins(%[[VAL_15]], %[[VAL_17]] : tensor<64x256xf32>, tensor<24xf32>) outs(%[[VAL_18]] : tensor<64x256x24xf32>) {
+// CHECK:               ^bb0(%[[VAL_20:.*]]: f32, %[[VAL_21:.*]]: f32, %[[VAL_22:.*]]: f32):
+// CHECK:                 %[[VAL_23:.*]] = arith.addf %[[VAL_20]], %[[VAL_21]] : f32
+// CHECK:                 linalg.yield %[[VAL_23]] : f32
+// CHECK:               } -> tensor<64x256x24xf32>
+// CHECK:               %[[VAL_24:.*]] = tensor.insert_slice %[[VAL_25:.*]] into %[[VAL_11]]{{\[}}%[[VAL_9]], 0, 0] [64, 256, 24] [1, 1, 1]
+// CHECK:               scf.yield %[[VAL_16]], %[[VAL_24]] : tensor<256x256xf32>, tensor<256x256x24xf32>
+// CHECK:             }
+
+module attributes {transform.with_named_sequence} {
+  transform.named_sequence @__transform_main(%arg1 : !transform.any_op {transform.readonly}) {
+    %slice_op = transform.structured.match ops{["tensor.insert_slice"]} in %arg1
+      : (!transform.any_op) -> !transform.any_op
+    %a, %b = transform.test.fuse_consumer %slice_op num_consumer_to_fuse = 1
+      : (!transform.any_op) -> (!transform.any_op, !transform.any_op)
+    transform.yield
+  }
+}
diff --git a/mlir/test/Target/Cpp/expressions.mlir b/mlir/test/Target/Cpp/expressions.mlir
index 6b67065f5d1a1..3a1694e7d15dc 100644
--- a/mlir/test/Target/Cpp/expressions.mlir
+++ b/mlir/test/Target/Cpp/expressions.mlir
@@ -70,11 +70,11 @@ func.func @do_not_inline(%arg0: i32, %arg1: i32, %arg2 : i32) -> i32 {
 }
 
 // CPP-DEFAULT:      float parentheses_for_low_precedence(int32_t [[VAL_1:v[0-9]+]], int32_t [[VAL_2:v[0-9]+]], int32_t [[VAL_3:v[0-9]+]]) {
-// CPP-DEFAULT-NEXT:   return (float) ([[VAL_1]] + [[VAL_2]] * [[VAL_3]]);
+// CPP-DEFAULT-NEXT:   return (float) (([[VAL_1]] + [[VAL_2]]) * [[VAL_3]]);
 // CPP-DEFAULT-NEXT: }
 
 // CPP-DECLTOP:      float parentheses_for_low_precedence(int32_t [[VAL_1:v[0-9]+]], int32_t [[VAL_2:v[0-9]+]], int32_t [[VAL_3:v[0-9]+]]) {
-// CPP-DECLTOP-NEXT:   return (float) ([[VAL_1]] + [[VAL_2]] * [[VAL_3]]);
+// CPP-DECLTOP-NEXT:   return (float) (([[VAL_1]] + [[VAL_2]]) * [[VAL_3]]);
 // CPP-DECLTOP-NEXT: }
 
 func.func @parentheses_for_low_precedence(%arg0: i32, %arg1: i32, %arg2: i32) -> f32 {
diff --git a/mlir/test/Target/LLVMIR/Import/alias.ll b/mlir/test/Target/LLVMIR/Import/alias.ll
new file mode 100644
index 0000000000000..9f86da3ecc71c
--- /dev/null
+++ b/mlir/test/Target/LLVMIR/Import/alias.ll
@@ -0,0 +1,88 @@
+; RUN: mlir-translate --import-llvm %s -split-input-file | FileCheck %s
+
+@foo_alias = alias ptr, ptr @callee
+
+; CHECK: llvm.mlir.alias external @foo_alias : !llvm.ptr {
+; CHECK:   %[[ADDR:.*]] = llvm.mlir.addressof @callee : !llvm.ptr
+; CHECK:   llvm.return %[[ADDR]] : !llvm.ptr
+; CHECK: }
+
+define internal ptr @callee() {
+entry:
+  ret ptr null
+}
+
+; -----
+
+@zed = global i32 42
+@foo = alias i32, ptr @zed
+@foo2 = alias i16, ptr @zed
+
+; CHECK: llvm.mlir.alias external @foo : i32 {
+; CHECK:   %[[ADDR:.*]] = llvm.mlir.addressof @zed : !llvm.ptr
+; CHECK:   llvm.return %[[ADDR]] : !llvm.ptr
+; CHECK: }
+; CHECK: llvm.mlir.alias external @foo2 : i16 {
+; CHECK:   %[[ADDR:.*]] = llvm.mlir.addressof @zed : !llvm.ptr
+; CHECK:   llvm.return %[[ADDR]] : !llvm.ptr
+; CHECK: }
+
+; -----
+
+@v1 = global i32 0
+@a3 = alias i32, addrspacecast (ptr @v1 to ptr addrspace(2))
+; CHECK: llvm.mlir.alias external @a3 : i32 {
+; CHECK:   %[[ADDR:.*]] = llvm.mlir.addressof @v1 : !llvm.ptr
+; CHECK:   %[[CASTED_ADDR:.*]] = llvm.addrspacecast %[[ADDR]] : !llvm.ptr to !llvm.ptr<2>
+; CHECK:   llvm.return %[[CASTED_ADDR]] : !llvm.ptr<2>
+; CHECK: }
+
+; -----
+
+@some_name = constant { [3 x ptr] } { [3 x ptr] [ptr null, ptr null, ptr null] }
+@vtable = alias { [3 x ptr] }, ptr @some_name
+
+; CHECK: llvm.mlir.alias external @vtable : !llvm.struct<(array<3 x ptr>)> {
+; CHECK:   %[[ADDR:.*]] = llvm.mlir.addressof @some_name : !llvm.ptr
+; CHECK:   llvm.return %[[ADDR]] : !llvm.ptr
+; CHECK: }
+
+; -----
+
+@glob.private = private constant [32 x i32] zeroinitializer
+@glob = linkonce_odr hidden alias [32 x i32], inttoptr (i64 add (i64 ptrtoint (ptr @glob.private to i64), i64 1234) to ptr)
+
+; CHECK: llvm.mlir.alias linkonce_odr hidden @glob {dso_local} : !llvm.array<32 x i32> {
+; CHECK: %[[CST:.*]] = llvm.mlir.constant(1234 : i64) : i64
+; CHECK: %[[ADDR:.*]] = llvm.mlir.addressof @glob.private : !llvm.ptr
+; CHECK: %[[PTRTOINT:.*]] = llvm.ptrtoint %[[ADDR]] : !llvm.ptr to i64
+; CHECK: %[[INTTOPTR:.*]] = llvm.add %[[PTRTOINT]], %[[CST]] : i64
+; CHECK: %[[RET:.*]] = llvm.inttoptr %[[INTTOPTR]] : i64 to !llvm.ptr
+; CHECK: llvm.return %[[RET]] : !llvm.ptr
+
+; -----
+
+@g1 = private global i32 0
+@g2 = internal constant ptr @a1
+@g3 = internal constant ptr @a2
+@a1 = private alias i32, ptr @g1
+@a2 = private alias ptr, ptr @a1
+
+; CHECK: llvm.mlir.alias private @a1 {dso_local} : i32 {
+; CHECK:   %[[ADDR:.*]] = llvm.mlir.addressof @g1 : !llvm.ptr
+; CHECK:   llvm.return %[[ADDR]] : !llvm.ptr
+; CHECK: }
+; CHECK: llvm.mlir.alias private @a2 {dso_local} : !llvm.ptr {
+; CHECK-NEXT:   %[[ADDR:.*]] = llvm.mlir.addressof @a1 : !llvm.ptr
+; CHECK-NEXT:   llvm.return %[[ADDR]] : !llvm.ptr
+; CHECK-NEXT: }
+
+; CHECK: llvm.mlir.global internal constant @g2() {addr_space = 0 : i32, dso_local} : !llvm.ptr {
+; CHECK-NEXT:   %[[ADDR:.*]] = llvm.mlir.addressof @a1 : !llvm.ptr
+; CHECK-NEXT:   llvm.return %[[ADDR]] : !llvm.ptr
+; CHECK-NEXT: }
+
+; CHECK: llvm.mlir.global internal constant @g3() {addr_space = 0 : i32, dso_local} : !llvm.ptr {
+; CHECK-NEXT:   %[[ADDR:.*]] = llvm.mlir.addressof @a2 : !llvm.ptr
+; CHECK-NEXT:   llvm.return %[[ADDR]] : !llvm.ptr
+; CHECK-NEXT: }
diff --git a/mlir/test/Target/LLVMIR/Import/call-argument-attributes.ll b/mlir/test/Target/LLVMIR/Import/call-argument-attributes.ll
new file mode 100644
index 0000000000000..fa39c79bf0859
--- /dev/null
+++ b/mlir/test/Target/LLVMIR/Import/call-argument-attributes.ll
@@ -0,0 +1,22 @@
+; RUN: mlir-translate -import-llvm %s | FileCheck %s
+
+; CHECK-LABEL: llvm.func @somefunc(i32, !llvm.ptr)
+declare void @somefunc(i32, ptr)
+
+; CHECK-LABEL: llvm.func @test_call_arg_attrs_direct(
+; CHECK-SAME:    %[[VAL_0:.*]]: i32,
+; CHECK-SAME:    %[[VAL_1:.*]]: !llvm.ptr)
+define void @test_call_arg_attrs_direct(i32 %0, ptr %1) {
+  ; CHECK: llvm.call @somefunc(%[[VAL_0]], %[[VAL_1]]) : (i32, !llvm.ptr {llvm.byval = i64}) -> ()
+  call void @somefunc(i32 %0, ptr byval(i64) %1)
+  ret void
+}
+
+; CHECK-LABEL: llvm.func @test_call_arg_attrs_indirect(
+; CHECK-SAME:    %[[VAL_0:.*]]: i16,
+; CHECK-SAME:    %[[VAL_1:.*]]: !llvm.ptr
+define i16 @test_call_arg_attrs_indirect(i16 %0, ptr %1) {
+  ; CHECK: llvm.call tail %[[VAL_1]](%[[VAL_0]]) : !llvm.ptr, (i16 {llvm.noundef, llvm.signext}) -> (i16 {llvm.signext})
+  %3 = tail call signext i16 %1(i16 noundef signext %0)
+  ret i16 %3
+}
diff --git a/mlir/test/Target/LLVMIR/Import/function-attributes.ll b/mlir/test/Target/LLVMIR/Import/function-attributes.ll
index 079aa6f90bf11..beda421504648 100644
--- a/mlir/test/Target/LLVMIR/Import/function-attributes.ll
+++ b/mlir/test/Target/LLVMIR/Import/function-attributes.ll
@@ -12,6 +12,15 @@ define internal spir_func void @spir_func_internal() {
 
 ; // -----
 
+; Ensure that we have dso_local.
+; CHECK: llvm.func @dsolocal_func()
+; CHECK-SAME: attributes {dso_local}
+define dso_local void @dsolocal_func() {
+  ret void
+}
+
+; // -----
+
 ; CHECK-LABEL: @func_readnone
 ; CHECK-SAME:  attributes {memory_effects = #llvm.memory_effects<other = none, argMem = none, inaccessibleMem = none>}
 ; CHECK:   llvm.return
@@ -58,7 +67,7 @@ define ptr @func_arg_attrs(
     ptr dereferenceable(12) %arg10,
     ptr dereferenceable_or_null(42) %arg11,
     double inreg %arg12,
-    ptr nocapture %arg13,
+    ptr captures(none) %arg13,
     ptr nofree %arg14,
     ptr nonnull %arg15,
     ptr preallocated(double) %arg16,
diff --git a/mlir/test/Target/LLVMIR/Import/import-failure.ll b/mlir/test/Target/LLVMIR/Import/import-failure.ll
index b616cb81e0a8a..d929a59284762 100644
--- a/mlir/test/Target/LLVMIR/Import/import-failure.ll
+++ b/mlir/test/Target/LLVMIR/Import/import-failure.ll
@@ -1,4 +1,4 @@
-; RUN: not mlir-translate -import-llvm -emit-expensive-warnings -split-input-file %s 2>&1 | FileCheck %s
+; RUN: not mlir-translate -import-llvm -emit-expensive-warnings -split-input-file %s 2>&1 -o /dev/null | FileCheck %s
 
 ; CHECK:      <unknown>
 ; CHECK-SAME: error: unhandled instruction: indirectbr ptr %dst, [label %bb1, label %bb2]
@@ -353,3 +353,32 @@ declare void @llvm.experimental.noalias.scope.decl(metadata)
 ; CHECK:      import-failure.ll
 ; CHECK-SAME: warning: unhandled data layout token: ni:42
 target datalayout = "e-ni:42-i64:64"
+
+; // -----
+
+; CHECK:      <unknown>
+; CHECK-SAME: incompatible call and callee types: '!llvm.func<void (i64)>' and '!llvm.func<void (ptr)>'
+define void @incompatible_call_and_callee_types() {
+  call void @callee(i64 0)
+  ret void
+}
+
+declare void @callee(ptr)
+
+; // -----
+
+; CHECK:      <unknown>
+; CHECK-SAME: incompatible call and callee types: '!llvm.func<void ()>' and '!llvm.func<i32 ()>'
+define void @f() personality ptr @__gxx_personality_v0 {
+entry:
+  invoke void @g() to label %bb1 unwind label %bb2
+bb1:
+  ret void
+bb2:
+  %0 = landingpad i32 cleanup
+  unreachable
+}
+
+declare i32 @g()
+
+declare i32 @__gxx_personality_v0(...)
diff --git a/mlir/test/Target/LLVMIR/Import/instructions.ll b/mlir/test/Target/LLVMIR/Import/instructions.ll
index 77052ab6e41f6..c294e1b34f9bb 100644
--- a/mlir/test/Target/LLVMIR/Import/instructions.ll
+++ b/mlir/test/Target/LLVMIR/Import/instructions.ll
@@ -702,3 +702,21 @@ define void @fence() {
   fence syncscope("") seq_cst
   ret void
 }
+
+; // -----
+
+; CHECK-LABEL: @f
+define void @f() personality ptr @__gxx_personality_v0 {
+entry:
+; CHECK: llvm.invoke @g() to ^bb1 unwind ^bb2 vararg(!llvm.func<void (...)>) : () -> ()
+  invoke void @g() to label %bb1 unwind label %bb2
+bb1:
+  ret void
+bb2:
+  %0 = landingpad i32 cleanup
+  unreachable
+}
+
+declare void @g(...)
+
+declare i32 @__gxx_personality_v0(...)
diff --git a/mlir/test/Target/LLVMIR/Import/intrinsic.ll b/mlir/test/Target/LLVMIR/Import/intrinsic.ll
index bd335323a2e1c..249a0552c87f3 100644
--- a/mlir/test/Target/LLVMIR/Import/intrinsic.ll
+++ b/mlir/test/Target/LLVMIR/Import/intrinsic.ll
@@ -101,12 +101,23 @@ define void @floor_test(float %0, <8 x float> %1) {
   %4 = call <8 x float> @llvm.floor.v8f32(<8 x float> %1)
   ret void
 }
-; CHECK-LABEL:  llvm.func @cos_test
-define void @cos_test(float %0, <8 x float> %1) {
+; CHECK-LABEL:  llvm.func @trig_test
+define void @trig_test(float %0, <8 x float> %1) {
+  ; CHECK: llvm.intr.sin(%{{.*}}) : (f32) -> f32
+  %3 = call float @llvm.sin.f32(float %0)
+  ; CHECK: llvm.intr.sin(%{{.*}}) : (vector<8xf32>) -> vector<8xf32>
+  %4 = call <8 x float> @llvm.sin.v8f32(<8 x float> %1)
+
   ; CHECK: llvm.intr.cos(%{{.*}}) : (f32) -> f32
-  %3 = call float @llvm.cos.f32(float %0)
+  %5 = call float @llvm.cos.f32(float %0)
   ; CHECK: llvm.intr.cos(%{{.*}}) : (vector<8xf32>) -> vector<8xf32>
-  %4 = call <8 x float> @llvm.cos.v8f32(<8 x float> %1)
+  %6 = call <8 x float> @llvm.cos.v8f32(<8 x float> %1)
+
+  ; CHECK: llvm.intr.tan(%{{.*}}) : (f32) -> f32
+  %7 = call float @llvm.tan.f32(float %0)
+  ; CHECK: llvm.intr.tan(%{{.*}}) : (vector<8xf32>) -> vector<8xf32>
+  %8 = call <8 x float> @llvm.tan.v8f32(<8 x float> %1)
+
   ret void
 }
 ; CHECK-LABEL:  llvm.func @hyperbolic_trig_test
diff --git a/mlir/test/Target/LLVMIR/Import/invoke-argument-attributes.ll b/mlir/test/Target/LLVMIR/Import/invoke-argument-attributes.ll
new file mode 100644
index 0000000000000..42489832fd184
--- /dev/null
+++ b/mlir/test/Target/LLVMIR/Import/invoke-argument-attributes.ll
@@ -0,0 +1,26 @@
+; RUN: mlir-translate -import-llvm %s | FileCheck %s
+
+; CHECK-LABEL:   llvm.func @test(
+; CHECK-SAME:      %[[VAL_0:.*]]: i16 {llvm.noundef, llvm.signext}) -> (i16 {llvm.signext}) attributes {personality = @__gxx_personality_v0} {
+define signext i16 @test(i16 noundef signext %0) personality ptr @__gxx_personality_v0 {
+  ; CHECK:           %[[VAL_3:.*]] = llvm.invoke @somefunc(%[[VAL_0]]) to ^bb2 unwind ^bb1 : (i16 {llvm.noundef, llvm.signext}) -> (i16 {llvm.signext})
+  %2 = invoke signext i16 @somefunc(i16 noundef signext %0)
+          to label %7 unwind label %3
+
+3:                                                ; preds = %1
+  %4 = landingpad { ptr, i32 }
+          catch ptr null
+  %5 = extractvalue { ptr, i32 } %4, 0
+  %6 = tail call ptr @__cxa_begin_catch(ptr %5) #2
+  tail call void @__cxa_end_catch()
+  br label %7
+
+7:                                                ; preds = %1, %3
+  %8 = phi i16 [ 0, %3 ], [ %2, %1 ]
+  ret i16 %8
+}
+
+declare noundef signext i16 @somefunc(i16 noundef signext)
+declare i32 @__gxx_personality_v0(...)
+declare ptr @__cxa_begin_catch(ptr)
+declare void @__cxa_end_catch()
diff --git a/mlir/test/Target/LLVMIR/Import/target-triple.ll b/mlir/test/Target/LLVMIR/Import/target-triple.ll
new file mode 100644
index 0000000000000..a04b41a7aeb55
--- /dev/null
+++ b/mlir/test/Target/LLVMIR/Import/target-triple.ll
@@ -0,0 +1,5 @@
+; RUN: mlir-translate -import-llvm %s | FileCheck %s
+; CHECK: module attributes {
+; CHECK-SAME: llvm.target_triple = "aarch64-none-linux-android21"
+target triple = "aarch64-none-linux-android21"
+
diff --git a/mlir/test/Target/LLVMIR/alias.mlir b/mlir/test/Target/LLVMIR/alias.mlir
new file mode 100644
index 0000000000000..56832a4900746
--- /dev/null
+++ b/mlir/test/Target/LLVMIR/alias.mlir
@@ -0,0 +1,92 @@
+// RUN: mlir-translate -mlir-to-llvmir %s -split-input-file | FileCheck %s
+
+llvm.func internal @callee() -> !llvm.ptr attributes {dso_local} {
+  %0 = llvm.mlir.zero : !llvm.ptr
+  llvm.return %0 : !llvm.ptr
+}
+
+llvm.mlir.alias external @foo_alias : !llvm.ptr {
+  %0 = llvm.mlir.addressof @callee : !llvm.ptr
+  llvm.return %0 : !llvm.ptr
+}
+
+llvm.mlir.alias external @_ZTV1D : !llvm.struct<(array<3 x ptr>)> {
+  %0 = llvm.mlir.addressof @callee : !llvm.ptr
+  llvm.return %0 : !llvm.ptr
+}
+
+// CHECK: @foo_alias = alias ptr, ptr @callee
+// CHECK: @_ZTV1D = alias { [3 x ptr] }, ptr @callee
+
+// -----
+
+llvm.mlir.global external @zed(42 : i32) : i32
+
+llvm.mlir.alias external @foo : i32 {
+  %0 = llvm.mlir.addressof @zed : !llvm.ptr
+  llvm.return %0 : !llvm.ptr
+}
+
+llvm.mlir.alias external @foo2 : i16 {
+  %0 = llvm.mlir.addressof @zed : !llvm.ptr
+  llvm.return %0 : !llvm.ptr
+}
+
+// CHECK: @foo = alias i32, ptr @zed
+// CHECK: @foo2 = alias i16, ptr @zed
+
+// -----
+
+llvm.mlir.global private constant @glob.private(dense<0> : tensor<32xi32>) {dso_local} : !llvm.array<32 x i32>
+
+llvm.mlir.alias linkonce_odr hidden @glob {dso_local} : !llvm.array<32 x i32> {
+  %0 = llvm.mlir.constant(1234 : i64) : i64
+  %1 = llvm.mlir.addressof @glob.private : !llvm.ptr
+  %2 = llvm.ptrtoint %1 : !llvm.ptr to i64
+  %3 = llvm.add %2, %0 : i64
+  %4 = llvm.inttoptr %3 : i64 to !llvm.ptr
+  llvm.return %4 : !llvm.ptr
+}
+
+// CHECK: @glob = linkonce_odr hidden alias [32 x i32], inttoptr (i64 add (i64 ptrtoint (ptr @glob.private to i64), i64 1234) to ptr)
+
+// -----
+
+llvm.mlir.global external @v1(0 : i32) : i32
+llvm.mlir.alias external @a3 : i32 {
+  %0 = llvm.mlir.addressof @v1 : !llvm.ptr
+  %1 = llvm.addrspacecast %0 : !llvm.ptr to !llvm.ptr<2>
+  llvm.return %1 : !llvm.ptr<2>
+}
+
+// CHECK: @a3 = alias i32, addrspacecast (ptr @v1 to ptr addrspace(2))
+
+// -----
+
+llvm.mlir.global private @g1(0 : i32) {dso_local} : i32
+
+llvm.mlir.alias private @a1 {dso_local} : i32 {
+  %0 = llvm.mlir.addressof @g1 : !llvm.ptr
+  llvm.return %0 : !llvm.ptr
+}
+
+llvm.mlir.global internal constant @g2() {dso_local} : !llvm.ptr {
+  %0 = llvm.mlir.addressof @a1 : !llvm.ptr
+  llvm.return %0 : !llvm.ptr
+}
+
+llvm.mlir.alias private @a2 {dso_local} : !llvm.ptr {
+  %0 = llvm.mlir.addressof @a1 : !llvm.ptr
+  llvm.return %0 : !llvm.ptr
+}
+
+llvm.mlir.global internal constant @g3() {dso_local} : !llvm.ptr {
+  %0 = llvm.mlir.addressof @a2 : !llvm.ptr
+  llvm.return %0 : !llvm.ptr
+}
+
+// CHECK: @g1 = private global i32 0
+// CHECK: @g2 = internal constant ptr @a1
+// CHECK: @g3 = internal constant ptr @a2
+// CHECK: @a1 = private alias i32, ptr @g1
+// CHECK: @a2 = private alias ptr, ptr @a1
diff --git a/mlir/test/Target/LLVMIR/attribute-alias-scopes.mlir b/mlir/test/Target/LLVMIR/attribute-alias-scopes.mlir
index fb71a51512aee..ae399d4f7edef 100644
--- a/mlir/test/Target/LLVMIR/attribute-alias-scopes.mlir
+++ b/mlir/test/Target/LLVMIR/attribute-alias-scopes.mlir
@@ -34,8 +34,8 @@ llvm.func @alias_scopes(%arg1 : !llvm.ptr) {
 
 // Check the intrinsic declarations.
 // CHECK-DAG: declare void @llvm.experimental.noalias.scope.decl(metadata)
-// CHECK-DAG: declare void @llvm.memcpy.p0.p0.i32(ptr noalias nocapture writeonly, ptr noalias nocapture readonly, i32, i1 immarg)
-// CHECK-DAG: declare void @llvm.memset.p0.i32(ptr nocapture writeonly, i8, i32, i1 immarg)
+// CHECK-DAG: declare void @llvm.memcpy.p0.p0.i32(ptr noalias writeonly captures(none), ptr noalias readonly captures(none), i32, i1 immarg)
+// CHECK-DAG: declare void @llvm.memset.p0.i32(ptr writeonly captures(none), i8, i32, i1 immarg)
 
 // Check the translated metadata.
 // CHECK-DAG: ![[DOMAIN:[0-9]+]] = distinct !{![[DOMAIN]], !"The domain"}
@@ -141,8 +141,8 @@ llvm.func @alias_scopes(%arg1 : !llvm.ptr) {
 
 // Check the intrinsic declarations.
 // CHECK-DAG: declare void @llvm.experimental.noalias.scope.decl(metadata)
-// CHECK-DAG: declare void @llvm.memcpy.p0.p0.i32(ptr noalias nocapture writeonly, ptr noalias nocapture readonly, i32, i1 immarg)
-// CHECK-DAG: declare void @llvm.memset.p0.i32(ptr nocapture writeonly, i8, i32, i1 immarg)
+// CHECK-DAG: declare void @llvm.memcpy.p0.p0.i32(ptr noalias writeonly captures(none), ptr noalias readonly captures(none), i32, i1 immarg)
+// CHECK-DAG: declare void @llvm.memset.p0.i32(ptr writeonly captures(none), i8, i32, i1 immarg)
 
 // Check the translated metadata.
 // CHECK-DAG: ![[DOMAIN:[0-9]+]] = !{!"domain1", !"The domain"}
diff --git a/mlir/test/Target/LLVMIR/call-argument-attributes.mlir b/mlir/test/Target/LLVMIR/call-argument-attributes.mlir
new file mode 100644
index 0000000000000..b3d286dcda504
--- /dev/null
+++ b/mlir/test/Target/LLVMIR/call-argument-attributes.mlir
@@ -0,0 +1,17 @@
+// RUN: mlir-translate -mlir-to-llvmir %s | FileCheck %s
+
+llvm.func @somefunc(i32, !llvm.ptr)
+
+// CHECK-LABEL: define void @test_call_arg_attrs_direct
+llvm.func @test_call_arg_attrs_direct(%arg0: i32, %arg1: !llvm.ptr) {
+  // CHECK: call void @somefunc(i32 %{{.*}}, ptr byval(i64) %{{.*}})
+  llvm.call @somefunc(%arg0, %arg1) : (i32, !llvm.ptr {llvm.byval = i64}) -> ()
+  llvm.return
+}
+
+// CHECK-LABEL: define i16 @test_call_arg_attrs_indirect
+llvm.func @test_call_arg_attrs_indirect(%arg0: i16, %arg1: !llvm.ptr) -> i16 {
+  // CHECK: tail call signext i16 %{{.*}}(i16 noundef signext %{{.*}})
+  %0 = llvm.call tail %arg1(%arg0) : !llvm.ptr, (i16 {llvm.noundef, llvm.signext}) -> (i16 {llvm.signext})
+  llvm.return %0 : i16
+}
diff --git a/mlir/test/Target/LLVMIR/invoke-argument-attributes.mlir b/mlir/test/Target/LLVMIR/invoke-argument-attributes.mlir
new file mode 100644
index 0000000000000..ea8ed1d416435
--- /dev/null
+++ b/mlir/test/Target/LLVMIR/invoke-argument-attributes.mlir
@@ -0,0 +1,25 @@
+// RUN: mlir-translate -mlir-to-llvmir %s | FileCheck %s
+
+// CHECK-LABEL: @test
+llvm.func @test(%arg0: i16 {llvm.noundef, llvm.signext}) -> (i16 {llvm.signext}) attributes {personality = @__gxx_personality_v0} {
+  %0 = llvm.mlir.zero : !llvm.ptr
+  %1 = llvm.mlir.constant(0 : i16) : i16
+  // CHECK:      invoke signext i16 @somefunc(i16 noundef signext %{{.*}})
+  // CHECK-NEXT:   to label %{{.*}} unwind label %{{.*}}
+  %2 = llvm.invoke @somefunc(%arg0) to ^bb2 unwind ^bb1 : (i16 {llvm.noundef, llvm.signext}) -> (i16 {llvm.signext})
+^bb1:  // pred: ^bb0
+  %3 = llvm.landingpad (catch %0 : !llvm.ptr) : !llvm.struct<(ptr, i32)>
+  %4 = llvm.extractvalue %3[0] : !llvm.struct<(ptr, i32)>
+  %5 = llvm.call tail @__cxa_begin_catch(%4) : (!llvm.ptr) -> !llvm.ptr
+  llvm.call tail @__cxa_end_catch() : () -> ()
+  llvm.br ^bb3(%1 : i16)
+^bb2:  // pred: ^bb0
+  llvm.br ^bb3(%2 : i16)
+^bb3(%6: i16):  // 2 preds: ^bb1, ^bb2
+  llvm.return %6 : i16
+}
+
+llvm.func @somefunc(i16 {llvm.noundef, llvm.signext}) -> (i16 {llvm.noundef, llvm.signext})
+llvm.func @__gxx_personality_v0(...) -> i32
+llvm.func @__cxa_begin_catch(!llvm.ptr) -> !llvm.ptr
+llvm.func @__cxa_end_catch()
diff --git a/mlir/test/Target/LLVMIR/llvmir-debug.mlir b/mlir/test/Target/LLVMIR/llvmir-debug.mlir
index eac2c5090a5b5..d15274311d745 100644
--- a/mlir/test/Target/LLVMIR/llvmir-debug.mlir
+++ b/mlir/test/Target/LLVMIR/llvmir-debug.mlir
@@ -115,6 +115,9 @@ llvm.func @func_with_debug(%arg: i64) {
   // CHECK: call void @func_no_debug(), !dbg ![[FILE_LOC:[0-9]+]]
   llvm.call @func_no_debug() : () -> () loc("foo.mlir":1:2)
 
+  // CHECK: call void @func_no_debug(), !dbg ![[FILE_LOC:[0-9]+]]
+  llvm.call @func_no_debug() : () -> () loc("foo.mlir":1:2 to 5:6)
+
   // CHECK: call void @func_no_debug(), !dbg ![[NAMED_LOC:[0-9]+]]
   llvm.call @func_no_debug() : () -> () loc("named"("foo.mlir":10:10))
 
diff --git a/mlir/test/Target/LLVMIR/llvmir-intrinsics.mlir b/mlir/test/Target/LLVMIR/llvmir-intrinsics.mlir
index 72ea3fcb172d1..2c208789e36dd 100644
--- a/mlir/test/Target/LLVMIR/llvmir-intrinsics.mlir
+++ b/mlir/test/Target/LLVMIR/llvmir-intrinsics.mlir
@@ -103,12 +103,22 @@ llvm.func @floor_test(%arg0: f32, %arg1: vector<8xf32>) {
   llvm.return
 }
 
-// CHECK-LABEL: @cos_test
-llvm.func @cos_test(%arg0: f32, %arg1: vector<8xf32>) {
+// CHECK-LABEL: @trig_test
+llvm.func @trig_test(%arg0: f32, %arg1: vector<8xf32>) {
+  // CHECK: call float @llvm.sin.f32
+  llvm.intr.sin(%arg0) : (f32) -> f32
+  // CHECK: call <8 x float> @llvm.sin.v8f32
+  llvm.intr.sin(%arg1) : (vector<8xf32>) -> vector<8xf32>
+
   // CHECK: call float @llvm.cos.f32
-  "llvm.intr.cos"(%arg0) : (f32) -> f32
+  llvm.intr.cos(%arg0) : (f32) -> f32
   // CHECK: call <8 x float> @llvm.cos.v8f32
-  "llvm.intr.cos"(%arg1) : (vector<8xf32>) -> vector<8xf32>
+  llvm.intr.cos(%arg1) : (vector<8xf32>) -> vector<8xf32>
+
+  // CHECK: call float @llvm.tan.f32
+  llvm.intr.tan(%arg0) : (f32) -> f32
+  // CHECK: call <8 x float> @llvm.tan.v8f32
+  llvm.intr.tan(%arg1) : (vector<8xf32>) -> vector<8xf32>
   llvm.return
 }
 
@@ -1074,7 +1084,7 @@ llvm.func @experimental_constrained_fptrunc(%s: f64, %v: vector<4xf32>) {
 // CHECK-DAG: declare <8 x float> @llvm.fma.v8f32(<8 x float>, <8 x float>, <8 x float>) #0
 // CHECK-DAG: declare float @llvm.fmuladd.f32(float, float, float)
 // CHECK-DAG: declare <8 x float> @llvm.fmuladd.v8f32(<8 x float>, <8 x float>, <8 x float>) #0
-// CHECK-DAG: declare void @llvm.prefetch.p0(ptr nocapture readonly, i32 immarg, i32 immarg, i32 immarg)
+// CHECK-DAG: declare void @llvm.prefetch.p0(ptr readonly captures(none), i32 immarg, i32 immarg, i32 immarg)
 // CHECK-DAG: declare i1 @llvm.is.fpclass.f32(float, i32 immarg)
 // CHECK-DAG: declare float @llvm.exp.f32(float)
 // CHECK-DAG: declare <8 x float> @llvm.exp.v8f32(<8 x float>) #0
@@ -1115,24 +1125,24 @@ llvm.func @experimental_constrained_fptrunc(%s: f64, %v: vector<4xf32>) {
 // CHECK-DAG: declare i64 @llvm.llrint.i64.f64(double)
 // CHECK-DAG: declare <12 x float> @llvm.matrix.multiply.v12f32.v64f32.v48f32(<64 x float>, <48 x float>, i32 immarg, i32 immarg, i32 immarg)
 // CHECK-DAG: declare <48 x float> @llvm.matrix.transpose.v48f32(<48 x float>, i32 immarg, i32 immarg)
-// CHECK-DAG: declare <48 x float> @llvm.matrix.column.major.load.v48f32.i64(ptr nocapture, i64, i1 immarg, i32 immarg, i32 immarg)
-// CHECK-DAG: declare void @llvm.matrix.column.major.store.v48f32.i64(<48 x float>, ptr nocapture writeonly, i64, i1 immarg, i32 immarg, i32 immarg)
+// CHECK-DAG: declare <48 x float> @llvm.matrix.column.major.load.v48f32.i64(ptr captures(none), i64, i1 immarg, i32 immarg, i32 immarg)
+// CHECK-DAG: declare void @llvm.matrix.column.major.store.v48f32.i64(<48 x float>, ptr writeonly captures(none), i64, i1 immarg, i32 immarg, i32 immarg)
 // CHECK-DAG: declare <7 x i1> @llvm.get.active.lane.mask.v7i1.i64(i64, i64)
-// CHECK-DAG: declare <7 x float> @llvm.masked.load.v7f32.p0(ptr nocapture, i32 immarg, <7 x i1>, <7 x float>)
-// CHECK-DAG: declare void @llvm.masked.store.v7f32.p0(<7 x float>, ptr nocapture, i32 immarg, <7 x i1>)
+// CHECK-DAG: declare <7 x float> @llvm.masked.load.v7f32.p0(ptr captures(none), i32 immarg, <7 x i1>, <7 x float>)
+// CHECK-DAG: declare void @llvm.masked.store.v7f32.p0(<7 x float>, ptr captures(none), i32 immarg, <7 x i1>)
 // CHECK-DAG: declare <7 x float> @llvm.masked.gather.v7f32.v7p0(<7 x ptr>, i32 immarg, <7 x i1>, <7 x float>)
 // CHECK-DAG: declare void @llvm.masked.scatter.v7f32.v7p0(<7 x float>, <7 x ptr>, i32 immarg, <7 x i1>)
-// CHECK-DAG: declare <7 x float> @llvm.masked.expandload.v7f32(ptr nocapture, <7 x i1>, <7 x float>)
-// CHECK-DAG: declare void @llvm.masked.compressstore.v7f32(<7 x float>, ptr nocapture, <7 x i1>)
+// CHECK-DAG: declare <7 x float> @llvm.masked.expandload.v7f32(ptr captures(none), <7 x i1>, <7 x float>)
+// CHECK-DAG: declare void @llvm.masked.compressstore.v7f32(<7 x float>, ptr captures(none), <7 x i1>)
 // CHECK-DAG: declare void @llvm.var.annotation.p0.p0(ptr, ptr, ptr, i32, ptr)
 // CHECK-DAG: declare ptr @llvm.ptr.annotation.p0.p0(ptr, ptr, ptr, i32, ptr)
 // CHECK-DAG: declare i16 @llvm.annotation.i16.p0(i16, ptr, ptr, i32)
 // CHECK-DAG: declare void @llvm.trap()
 // CHECK-DAG: declare void @llvm.debugtrap()
 // CHECK-DAG: declare void @llvm.ubsantrap(i8 immarg)
-// CHECK-DAG: declare void @llvm.memcpy.p0.p0.i32(ptr noalias nocapture writeonly, ptr noalias nocapture readonly, i32, i1 immarg)
-// CHECK-DAG: declare void @llvm.memcpy.inline.p0.p0.i32(ptr noalias nocapture writeonly, ptr noalias nocapture readonly, i32, i1 immarg)
-// CHECK-DAG: declare void @llvm.memcpy.inline.p0.p0.i64(ptr noalias nocapture writeonly, ptr noalias nocapture readonly, i64, i1 immarg)
+// CHECK-DAG: declare void @llvm.memcpy.p0.p0.i32(ptr noalias writeonly captures(none), ptr noalias readonly captures(none), i32, i1 immarg)
+// CHECK-DAG: declare void @llvm.memcpy.inline.p0.p0.i32(ptr noalias writeonly captures(none), ptr noalias readonly captures(none), i32, i1 immarg)
+// CHECK-DAG: declare void @llvm.memcpy.inline.p0.p0.i64(ptr noalias writeonly captures(none), ptr noalias readonly captures(none), i64, i1 immarg)
 // CHECK-DAG: declare { i32, i1 } @llvm.sadd.with.overflow.i32(i32, i32)
 // CHECK-DAG: declare { <8 x i32>, <8 x i1> } @llvm.sadd.with.overflow.v8i32(<8 x i32>, <8 x i32>)
 // CHECK-DAG: declare { i32, i1 } @llvm.uadd.with.overflow.i32(i32, i32)
@@ -1159,16 +1169,16 @@ llvm.func @experimental_constrained_fptrunc(%s: f64, %v: vector<4xf32>) {
 // CHECK-DAG: declare i32 @llvm.expect.i32(i32, i32)
 // CHECK-DAG: declare i16 @llvm.expect.with.probability.i16(i16, i16, double immarg)
 // CHECK-DAG: declare nonnull ptr @llvm.threadlocal.address.p0(ptr nonnull)
-// CHECK-DAG: declare token @llvm.coro.id(i32, ptr readnone, ptr nocapture readonly, ptr)
+// CHECK-DAG: declare token @llvm.coro.id(i32, ptr readnone, ptr readonly captures(none), ptr)
 // CHECK-DAG: declare ptr @llvm.coro.begin(token, ptr writeonly)
 // CHECK-DAG: declare i64 @llvm.coro.size.i64()
 // CHECK-DAG: declare i32 @llvm.coro.size.i32()
 // CHECK-DAG: declare token @llvm.coro.save(ptr)
 // CHECK-DAG: declare i8 @llvm.coro.suspend(token, i1)
 // CHECK-DAG: declare i1 @llvm.coro.end(ptr, i1, token)
-// CHECK-DAG: declare ptr @llvm.coro.free(token, ptr nocapture readonly)
+// CHECK-DAG: declare ptr @llvm.coro.free(token, ptr readonly captures(none))
 // CHECK-DAG: declare void @llvm.coro.resume(ptr)
-// CHECK-DAG: declare ptr @llvm.coro.promise(ptr nocapture, i32, i1)
+// CHECK-DAG: declare ptr @llvm.coro.promise(ptr captures(none), i32, i1)
 // CHECK-DAG: declare <8 x i32> @llvm.vp.add.v8i32(<8 x i32>, <8 x i32>, <8 x i1>, i32)
 // CHECK-DAG: declare <8 x i32> @llvm.vp.sub.v8i32(<8 x i32>, <8 x i32>, <8 x i1>, i32)
 // CHECK-DAG: declare <8 x i32> @llvm.vp.mul.v8i32(<8 x i32>, <8 x i32>, <8 x i1>, i32)
@@ -1208,8 +1218,8 @@ llvm.func @experimental_constrained_fptrunc(%s: f64, %v: vector<4xf32>) {
 // CHECK-DAG: declare float @llvm.vp.reduce.fmin.v8f32(float, <8 x float>, <8 x i1>, i32)
 // CHECK-DAG: declare <8 x i32> @llvm.vp.select.v8i32(<8 x i1>, <8 x i32>, <8 x i32>, i32)
 // CHECK-DAG: declare <8 x i32> @llvm.vp.merge.v8i32(<8 x i1>, <8 x i32>, <8 x i32>, i32)
-// CHECK-DAG: declare void @llvm.experimental.vp.strided.store.v8i32.p0.i32(<8 x i32>, ptr nocapture, i32, <8 x i1>, i32)
-// CHECK-DAG: declare <8 x i32> @llvm.experimental.vp.strided.load.v8i32.p0.i32(ptr nocapture, i32, <8 x i1>, i32)
+// CHECK-DAG: declare void @llvm.experimental.vp.strided.store.v8i32.p0.i32(<8 x i32>, ptr captures(none), i32, <8 x i1>, i32)
+// CHECK-DAG: declare <8 x i32> @llvm.experimental.vp.strided.load.v8i32.p0.i32(ptr captures(none), i32, <8 x i1>, i32)
 // CHECK-DAG: declare <8 x i32> @llvm.vp.trunc.v8i32.v8i64(<8 x i64>, <8 x i1>, i32)
 // CHECK-DAG: declare <8 x i64> @llvm.vp.zext.v8i64.v8i32(<8 x i32>, <8 x i1>, i32)
 // CHECK-DAG: declare <8 x i64> @llvm.vp.sext.v8i64.v8i32(<8 x i32>, <8 x i1>, i32)
@@ -1227,10 +1237,10 @@ llvm.func @experimental_constrained_fptrunc(%s: f64, %v: vector<4xf32>) {
 // CHECK-DAG: declare <2 x i32> @llvm.vector.extract.v2i32.v8i32(<8 x i32>, i64 immarg)
 // CHECK-DAG: declare { <2 x double>, <2 x double> } @llvm.vector.deinterleave2.v4f64(<4 x double>)
 // CHECK-DAG: declare { <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.vector.deinterleave2.nxv8i32(<vscale x 8 x i32>)
-// CHECK-DAG: declare void @llvm.lifetime.start.p0(i64 immarg, ptr nocapture)
-// CHECK-DAG: declare void @llvm.lifetime.end.p0(i64 immarg, ptr nocapture)
-// CHECK-DAG: declare ptr @llvm.invariant.start.p0(i64 immarg, ptr nocapture)
-// CHECK-DAG: declare void @llvm.invariant.end.p0(ptr, i64 immarg, ptr nocapture)
+// CHECK-DAG: declare void @llvm.lifetime.start.p0(i64 immarg, ptr captures(none))
+// CHECK-DAG: declare void @llvm.lifetime.end.p0(i64 immarg, ptr captures(none))
+// CHECK-DAG: declare ptr @llvm.invariant.start.p0(i64 immarg, ptr captures(none))
+// CHECK-DAG: declare void @llvm.invariant.end.p0(ptr, i64 immarg, ptr captures(none))
 
 // CHECK-DAG: declare float @llvm.ssa.copy.f32(float returned)
 // CHECK-DAG: declare ptr @llvm.stacksave.p0()
diff --git a/mlir/test/Target/LLVMIR/llvmir.mlir b/mlir/test/Target/LLVMIR/llvmir.mlir
index 51086bc655af2..52aa69f4c481f 100644
--- a/mlir/test/Target/LLVMIR/llvmir.mlir
+++ b/mlir/test/Target/LLVMIR/llvmir.mlir
@@ -1195,7 +1195,7 @@ llvm.func @dereferenceableornullattr_decl(!llvm.ptr {llvm.dereferenceable_or_nul
 // CHECK-LABEL: declare void @inregattr_decl(ptr inreg)
 llvm.func @inregattr_decl(!llvm.ptr {llvm.inreg})
 
-// CHECK-LABEL: declare void @nocaptureattr_decl(ptr nocapture)
+// CHECK-LABEL: declare void @nocaptureattr_decl(ptr captures(none))
 llvm.func @nocaptureattr_decl(!llvm.ptr {llvm.nocapture})
 
 // CHECK-LABEL: declare void @nofreeattr_decl(ptr nofree)
diff --git a/mlir/test/Target/LLVMIR/nvvm/cvt_tf32.mlir b/mlir/test/Target/LLVMIR/nvvm/cvt_tf32.mlir
index 90a232e4baac6..ff7bad0149d4c 100644
--- a/mlir/test/Target/LLVMIR/nvvm/cvt_tf32.mlir
+++ b/mlir/test/Target/LLVMIR/nvvm/cvt_tf32.mlir
@@ -28,6 +28,20 @@ llvm.func @convert_float_to_tf32_rn_relu(%src : f32) -> i32 {
   llvm.return %res : i32
 }
 
+// CHECK-LABEL: @convert_float_to_tf32_rn_sf
+llvm.func @convert_float_to_tf32_rn_sf(%src : f32) -> i32 {
+  // CHECK: %{{.*}} = call i32 @llvm.nvvm.f2tf32.rn.satfinite(float %{{.*}})
+  %res = nvvm.cvt.float.to.tf32 %src {rnd = #nvvm.fp_rnd_mode<rn>, sat = #nvvm.sat_mode<satfinite>}
+  llvm.return %res : i32
+}
+
+// CHECK-LABEL: @convert_float_to_tf32_rn_relu_sf
+llvm.func @convert_float_to_tf32_rn_relu_sf(%src : f32) -> i32 {
+  // CHECK: %{{.*}} = call i32 @llvm.nvvm.f2tf32.rn.relu.satfinite(float %{{.*}})
+  %res = nvvm.cvt.float.to.tf32 %src {rnd = #nvvm.fp_rnd_mode<rn>, relu=true, sat = #nvvm.sat_mode<satfinite>}
+  llvm.return %res : i32
+}
+
 // CHECK-LABEL: @convert_float_to_tf32_rz
 llvm.func @convert_float_to_tf32_rz(%src : f32) -> i32 {
   // CHECK: %{{.*}} = call i32 @llvm.nvvm.f2tf32.rz(float %{{.*}})
@@ -41,3 +55,17 @@ llvm.func @convert_float_to_tf32_rz_relu(%src : f32) -> i32 {
   %res = nvvm.cvt.float.to.tf32 %src {rnd = #nvvm.fp_rnd_mode<rz>, relu=true}
   llvm.return %res : i32
 }
+
+// CHECK-LABEL: @convert_float_to_tf32_rz_sf
+llvm.func @convert_float_to_tf32_rz_sf(%src : f32) -> i32 {
+  // CHECK: %{{.*}} = call i32 @llvm.nvvm.f2tf32.rz.satfinite(float %{{.*}})
+  %res = nvvm.cvt.float.to.tf32 %src {rnd = #nvvm.fp_rnd_mode<rz>, sat = #nvvm.sat_mode<satfinite>}
+  llvm.return %res : i32
+}
+
+// CHECK-LABEL: @convert_float_to_tf32_rz_relu_sf
+llvm.func @convert_float_to_tf32_rz_relu_sf(%src : f32) -> i32 {
+  // CHECK: %{{.*}} = call i32 @llvm.nvvm.f2tf32.rz.relu.satfinite(float %{{.*}})
+  %res = nvvm.cvt.float.to.tf32 %src {rnd = #nvvm.fp_rnd_mode<rz>, relu=true, sat = #nvvm.sat_mode<satfinite>}
+  llvm.return %res : i32
+}
diff --git a/mlir/test/Target/LLVMIR/nvvm/tcgen05-alloc.mlir b/mlir/test/Target/LLVMIR/nvvm/tcgen05-alloc.mlir
new file mode 100644
index 0000000000000..781efa2567111
--- /dev/null
+++ b/mlir/test/Target/LLVMIR/nvvm/tcgen05-alloc.mlir
@@ -0,0 +1,42 @@
+// RUN: mlir-opt -split-input-file -verify-diagnostics %s
+// RUN: mlir-translate -mlir-to-llvmir -split-input-file -verify-diagnostics %s | FileCheck %s --check-prefix=CHECK-LLVM
+
+// CHECK-LABEL: @llvm_nvvm_tcgen05_alloc
+llvm.func @llvm_nvvm_tcgen05_alloc(%addr : !llvm.ptr, %ncols : i32) {
+  // CHECK-LLVM: call void @llvm.nvvm.tcgen05.alloc.cg1(ptr %{{.*}}, i32 %{{.*}})
+  nvvm.tcgen05.alloc %addr, %ncols : !llvm.ptr, i32
+
+  // CHECK-LLVM: call void @llvm.nvvm.tcgen05.alloc.cg2(ptr %{{.*}}, i32 %{{.*}})
+  nvvm.tcgen05.alloc %addr, %ncols {group = #nvvm.tcgen05_group<cta_2>} : !llvm.ptr, i32
+  llvm.return
+}
+
+// CHECK-LABEL: @llvm_nvvm_tcgen05_alloc_shared
+llvm.func @llvm_nvvm_tcgen05_alloc_shared(%addr : !llvm.ptr<3>, %ncols : i32) {
+  // CHECK-LLVM: call void @llvm.nvvm.tcgen05.alloc.shared.cg1(ptr addrspace(3) %{{.*}}, i32 %{{.*}})
+  nvvm.tcgen05.alloc %addr, %ncols : !llvm.ptr<3>, i32
+
+  // CHECK-LLVM: call void @llvm.nvvm.tcgen05.alloc.shared.cg2(ptr addrspace(3) %{{.*}}, i32 %{{.*}})
+  nvvm.tcgen05.alloc %addr, %ncols {group = #nvvm.tcgen05_group<cta_2>} : !llvm.ptr<3>, i32
+  llvm.return
+}
+
+// CHECK-LABEL: @llvm_nvvm_tcgen05_dealloc
+llvm.func @llvm_nvvm_tcgen05_dealloc(%addr : !llvm.ptr<6>, %ncols : i32) {
+  // CHECK-LLVM: call void @llvm.nvvm.tcgen05.dealloc.cg1(ptr addrspace(6) %{{.*}}, i32 %{{.*}})
+  nvvm.tcgen05.dealloc %addr, %ncols : !llvm.ptr<6>, i32
+
+  // CHECK-LLVM: call void @llvm.nvvm.tcgen05.dealloc.cg2(ptr addrspace(6) %{{.*}}, i32 %{{.*}})
+  nvvm.tcgen05.dealloc %addr, %ncols {group = #nvvm.tcgen05_group<cta_2>} : !llvm.ptr<6>, i32
+  llvm.return
+}
+
+// CHECK-LABEL: @llvm_nvvm_tcgen05_relinquish_alloc_permit
+llvm.func @llvm_nvvm_tcgen05_relinquish_alloc_permit() {
+  // CHECK-LLVM: call void @llvm.nvvm.tcgen05.relinq.alloc.permit.cg1()
+  nvvm.tcgen05.relinquish_alloc_permit
+
+  // CHECK-LLVM: call void @llvm.nvvm.tcgen05.relinq.alloc.permit.cg2()
+  nvvm.tcgen05.relinquish_alloc_permit {group = #nvvm.tcgen05_group<cta_2>}
+  llvm.return
+}
diff --git a/mlir/test/Target/LLVMIR/nvvm/tcgen05-barriers.mlir b/mlir/test/Target/LLVMIR/nvvm/tcgen05-barriers.mlir
new file mode 100644
index 0000000000000..7536a4567e34e
--- /dev/null
+++ b/mlir/test/Target/LLVMIR/nvvm/tcgen05-barriers.mlir
@@ -0,0 +1,56 @@
+// RUN: mlir-opt -split-input-file -verify-diagnostics %s
+// RUN: mlir-translate -mlir-to-llvmir -split-input-file -verify-diagnostics %s | FileCheck %s --check-prefix=CHECK-LLVM
+
+// CHECK-LABEL: @llvm_nvvm_tcgen05_fence
+llvm.func @llvm_nvvm_tcgen05_fence() {
+  // CHECK-LLVM: call void @llvm.nvvm.tcgen05.fence.before.thread.sync()
+  nvvm.tcgen05.fence #nvvm.tcgen05_fence<before>
+
+  // CHECK-LLVM: call void @llvm.nvvm.tcgen05.fence.after.thread.sync()
+  nvvm.tcgen05.fence #nvvm.tcgen05_fence<after>
+
+  llvm.return
+}
+
+// CHECK-LABEL: @llvm_nvvm_tcgen05_wait
+llvm.func @llvm_nvvm_tcgen05_wait() {
+  // CHECK-LLVM: call void @llvm.nvvm.tcgen05.wait.ld()
+  nvvm.tcgen05.wait #nvvm.tcgen05_wait<load>
+
+  // CHECK-LLVM: call void @llvm.nvvm.tcgen05.wait.st()
+  nvvm.tcgen05.wait #nvvm.tcgen05_wait<store>
+
+  llvm.return
+}
+
+// CHECK-LABEL: @llvm_nvvm_tcgen05_commit_generic
+llvm.func @llvm_nvvm_tcgen05_commit_generic(%barrier : !llvm.ptr, %cta_mask : i16) {
+  // CHECK-LLVM: call void @llvm.nvvm.tcgen05.commit.cg1(ptr %{{.*}})
+  nvvm.tcgen05.commit %barrier : !llvm.ptr
+
+  // CHECK-LLVM: call void @llvm.nvvm.tcgen05.commit.cg2(ptr %{{.*}})
+  nvvm.tcgen05.commit %barrier {group = #nvvm.tcgen05_group<cta_2>} : !llvm.ptr
+
+  // CHECK-LLVM: call void @llvm.nvvm.tcgen05.commit.mc.cg1(ptr %{{.*}}, i16 %{{.*}})
+  nvvm.tcgen05.commit %barrier, multicast_mask = %cta_mask : !llvm.ptr, i16
+
+  // CHECK-LLVM: call void @llvm.nvvm.tcgen05.commit.mc.cg2(ptr %{{.*}}, i16 %{{.*}})
+  nvvm.tcgen05.commit %barrier, multicast_mask = %cta_mask {group = #nvvm.tcgen05_group<cta_2>} : !llvm.ptr, i16
+  llvm.return
+}
+
+// CHECK-LABEL: @llvm_nvvm_tcgen05_commit_shared
+llvm.func @llvm_nvvm_tcgen05_commit_shared(%barrier : !llvm.ptr<3>, %cta_mask : i16) {
+  // CHECK-LLVM: call void @llvm.nvvm.tcgen05.commit.shared.cg1(ptr addrspace(3) %{{.*}})
+  nvvm.tcgen05.commit %barrier : !llvm.ptr<3>
+
+  // CHECK-LLVM: call void @llvm.nvvm.tcgen05.commit.shared.cg2(ptr addrspace(3) %{{.*}})
+  nvvm.tcgen05.commit %barrier {group = #nvvm.tcgen05_group<cta_2>} : !llvm.ptr<3>
+
+  // CHECK-LLVM: call void @llvm.nvvm.tcgen05.commit.mc.shared.cg1(ptr addrspace(3) %{{.*}}, i16 %{{.*}})
+  nvvm.tcgen05.commit %barrier, multicast_mask = %cta_mask : !llvm.ptr<3>, i16
+
+  // CHECK-LLVM: call void @llvm.nvvm.tcgen05.commit.mc.shared.cg2(ptr addrspace(3) %{{.*}}, i16 %{{.*}})
+  nvvm.tcgen05.commit %barrier, multicast_mask = %cta_mask {group = #nvvm.tcgen05_group<cta_2>} : !llvm.ptr<3>, i16
+  llvm.return
+}
diff --git a/mlir/test/Target/LLVMIR/nvvmir-invalid.mlir b/mlir/test/Target/LLVMIR/nvvmir-invalid.mlir
index cb08064590bc3..8957377607dad 100644
--- a/mlir/test/Target/LLVMIR/nvvmir-invalid.mlir
+++ b/mlir/test/Target/LLVMIR/nvvmir-invalid.mlir
@@ -117,22 +117,6 @@ llvm.func @convert_float_to_tf32_rna_relu(%src : f32) -> i32 {
 
 // -----
 
-llvm.func @convert_float_to_tf32_rn_sf(%src : f32) -> i32 {
-  // expected-error @below {{Saturation mode not supported with rn/rz rounding modes.}}
-  %res = nvvm.cvt.float.to.tf32 %src {rnd = #nvvm.fp_rnd_mode<rn>, sat = #nvvm.sat_mode<satfinite>}
-  llvm.return %res : i32
-}
-
-// -----
-
-llvm.func @convert_float_to_tf32_rz_sf(%src : f32) -> i32 {
-  // expected-error @below {{Saturation mode not supported with rn/rz rounding modes.}}
-  %res = nvvm.cvt.float.to.tf32 %src {rnd = #nvvm.fp_rnd_mode<rz>, sat = #nvvm.sat_mode<satfinite>}
-  llvm.return %res : i32
-}
-
-// -----
-
 llvm.func @convert_float_to_tf32_no_rnd_mode(%src : f32) -> i32 {
   // expected-error @below {{Only {rn,rz,rna} rounding modes supported for CvtFloatToTF32Op.}}
   %res = nvvm.cvt.float.to.tf32 %src
diff --git a/mlir/test/Target/LLVMIR/nvvmir.mlir b/mlir/test/Target/LLVMIR/nvvmir.mlir
index 7dad9a403def0..970cac707b058 100644
--- a/mlir/test/Target/LLVMIR/nvvmir.mlir
+++ b/mlir/test/Target/LLVMIR/nvvmir.mlir
@@ -757,3 +757,29 @@ llvm.func @nvvm_wgmma_wait_group_aligned() {
   nvvm.wgmma.wait.group.sync.aligned 20
   llvm.return
 }
+
+// -----
+// CHECK-LABEL: @nvvm_griddepcontrol_wait
+llvm.func @nvvm_griddepcontrol_wait() {
+  // CHECK: call void @llvm.nvvm.griddepcontrol.wait()
+  nvvm.griddepcontrol.wait
+  llvm.return
+}
+
+// -----
+// CHECK-LABEL: @nvvm_griddepcontrol_launch_dependents
+llvm.func @nvvm_griddepcontrol_launch_dependents() {
+  // CHECK: call void @llvm.nvvm.griddepcontrol.launch.dependents()
+  nvvm.griddepcontrol.launch.dependents
+  llvm.return
+}
+
+// -----
+// CHECK-LABEL: @nvvm_mapa
+llvm.func @nvvm_mapa(%a: !llvm.ptr, %a_shared: !llvm.ptr<3>, %b : i32) {
+  // CHECK-LLVM: call ptr @llvm.nvvm.mapa(ptr %{{.*}}, i32 %{{.*}})
+  %0 = nvvm.mapa %a, %b: !llvm.ptr -> !llvm.ptr
+  // CHECK-LLVM: call ptr addrspace(3) @llvm.nvvm.mapa.shared.cluster(ptr addrspace(3) %{{.*}}, i32 %{{.*}})
+  %1 = nvvm.mapa %a_shared, %b: !llvm.ptr<3> -> !llvm.ptr<3>
+  llvm.return
+}
diff --git a/mlir/test/Target/LLVMIR/omptarget-debug-nowait.mlir b/mlir/test/Target/LLVMIR/omptarget-debug-nowait.mlir
new file mode 100644
index 0000000000000..eaa88d9dd6053
--- /dev/null
+++ b/mlir/test/Target/LLVMIR/omptarget-debug-nowait.mlir
@@ -0,0 +1,40 @@
+// RUN: mlir-translate -mlir-to-llvmir %s
+
+module attributes {omp.is_target_device = false} {
+  llvm.func @main() {
+    %0 = llvm.mlir.constant(1 : i64) : i64
+    %1 = llvm.alloca %0 x f32 : (i64) -> !llvm.ptr
+    %3 = llvm.alloca %0 x i32 : (i64) -> !llvm.ptr
+    %6 = omp.map.info var_ptr(%1 : !llvm.ptr, f32) map_clauses(tofrom) capture(ByRef) -> !llvm.ptr
+    %7 = omp.map.info var_ptr(%3 : !llvm.ptr, i32) map_clauses(implicit, exit_release_or_enter_alloc) capture(ByCopy) -> !llvm.ptr
+    omp.target nowait map_entries(%6 -> %arg0, %7 -> %arg1 : !llvm.ptr, !llvm.ptr) {
+      %8 = llvm.mlir.constant(0 : i64) : i64
+      %9 = llvm.mlir.constant(100 : i32) : i32
+      llvm.br ^bb1(%9, %8 : i32, i64)
+    ^bb1(%13: i32, %14: i64):  // 2 preds: ^bb0, ^bb2
+      %15 = llvm.icmp "sgt" %14, %8 : i64
+      llvm.cond_br %15, ^bb2, ^bb3
+    ^bb2:  // pred: ^bb1
+      llvm.store %13, %arg1 : i32, !llvm.ptr
+      llvm.br ^bb1(%13, %14 : i32, i64)
+    ^bb3:  // pred: ^bb1
+      llvm.store %13, %arg1 : i32, !llvm.ptr
+      omp.terminator
+    }
+    llvm.return
+  } loc(#loc2)
+}
+
+#file = #llvm.di_file<"test.f90" in "">
+#di_null_type = #llvm.di_null_type
+#cu = #llvm.di_compile_unit<id = distinct[0]<>,
+ sourceLanguage = DW_LANG_Fortran95, file = #file, isOptimized = false,
+ emissionKind = Full>
+#sp_ty = #llvm.di_subroutine_type<callingConvention = DW_CC_program,
+ types = #di_null_type>
+#sp = #llvm.di_subprogram<compileUnit = #cu, name = "main", file=#file,
+ subprogramFlags = "Definition", type = #sp_ty>
+
+#loc1 = loc("test.f90":6:7)
+#loc2 = loc(fused<#sp>[#loc1])
+
diff --git a/mlir/test/Target/LLVMIR/omptarget-debug-var-1.mlir b/mlir/test/Target/LLVMIR/omptarget-debug-var-1.mlir
new file mode 100644
index 0000000000000..950f59f3e7ba5
--- /dev/null
+++ b/mlir/test/Target/LLVMIR/omptarget-debug-var-1.mlir
@@ -0,0 +1,63 @@
+// RUN: mlir-translate -mlir-to-llvmir %s | FileCheck %s
+
+#int_ty = #llvm.di_basic_type<tag = DW_TAG_base_type, name = "integer",
+  sizeInBits = 32, encoding = DW_ATE_signed>
+#real_ty = #llvm.di_basic_type<tag = DW_TAG_base_type, name = "real",
+  sizeInBits = 32, encoding = DW_ATE_float>
+#file = #llvm.di_file<"target.f90" in "">
+#di_null_type = #llvm.di_null_type
+#cu = #llvm.di_compile_unit<id = distinct[0]<>,
+  sourceLanguage = DW_LANG_Fortran95, file = #file, isOptimized = false,
+  emissionKind = Full>
+#array_ty = #llvm.di_composite_type<tag = DW_TAG_array_type,
+  baseType = #int_ty, elements = #llvm.di_subrange<count = 10 : i64>>
+#sp_ty = #llvm.di_subroutine_type<callingConvention = DW_CC_program,
+  types = #di_null_type>
+#g_var = #llvm.di_global_variable<scope = #cu, name = "arr",
+  linkageName = "_QFEarr", file = #file, line = 4,
+  type = #array_ty, isDefined = true>
+#g_var_expr = #llvm.di_global_variable_expression<var = #g_var>
+#sp = #llvm.di_subprogram<id = distinct[2]<>, compileUnit = #cu, scope = #file,
+  name = "test", file = #file, subprogramFlags = "Definition", type = #sp_ty>
+#var_arr = #llvm.di_local_variable<scope = #sp,
+  name = "arr", file = #file, line = 4, type = #array_ty>
+#var_i = #llvm.di_local_variable<scope = #sp,
+  name = "i", file = #file, line = 13, type = #int_ty>
+#var_x = #llvm.di_local_variable<scope = #sp,
+ name = "x", file = #file, line = 12, type = #real_ty>
+
+module attributes {llvm.target_triple = "amdgcn-amd-amdhsa", omp.is_target_device = true} {
+  llvm.func @test() {
+    %0 = llvm.mlir.constant(1 : i64) : i64
+    %1 = llvm.alloca %0 x f32 : (i64) -> !llvm.ptr
+    %4 = llvm.alloca %0 x i32 : (i64) -> !llvm.ptr
+    %6 = llvm.mlir.constant(9 : index) : i64
+    %7 = llvm.mlir.constant(0 : index) : i64
+    %8 = llvm.mlir.constant(1 : index) : i64
+    %10 = llvm.mlir.constant(10 : index) : i64
+    %11 = llvm.mlir.addressof @_QFEarr : !llvm.ptr
+    %14 = omp.map.info var_ptr(%1 : !llvm.ptr, f32) map_clauses(tofrom) capture(ByRef) -> !llvm.ptr
+    %15 = omp.map.bounds lower_bound(%7 : i64) upper_bound(%6 : i64) extent(%10 : i64) stride(%8 : i64) start_idx(%8 : i64)
+    %16 = omp.map.info var_ptr(%11 : !llvm.ptr, !llvm.array<10 x i32>) map_clauses(tofrom) capture(ByRef) bounds(%15) -> !llvm.ptr
+    %17 = omp.map.info var_ptr(%4 : !llvm.ptr, i32) map_clauses(implicit, exit_release_or_enter_alloc) capture(ByCopy) -> !llvm.ptr
+    omp.target map_entries(%14 -> %arg0, %16 -> %arg1, %17 -> %arg2 : !llvm.ptr, !llvm.ptr, !llvm.ptr) {
+      llvm.intr.dbg.declare #var_x = %arg0 : !llvm.ptr
+      llvm.intr.dbg.declare #var_arr = %arg1 : !llvm.ptr
+      llvm.intr.dbg.declare #var_i = %arg2 : !llvm.ptr
+      omp.terminator
+    }
+    llvm.return
+  } loc(#loc3)
+  llvm.mlir.global internal @_QFEarr() {addr_space = 0 : i32, dbg_exprs = [#g_var_expr]} : !llvm.array<10 x i32> {
+  } loc(#loc4)
+}
+#loc1 = loc("target.f90":4:7)
+#loc2 = loc("target.f90":11:7)
+#loc3 = loc(fused<#sp>[#loc2])
+#loc4 = loc(fused<#g_var>[#loc1])
+
+// CHECK: ![[SP:[0-9]+]] = distinct !DISubprogram(name: "__omp_offloading{{.*}}test{{.*}})
+// CHECK: !DILocalVariable(name: "dyn_ptr", arg: 1, scope: ![[SP]]{{.*}}flags: DIFlagArtificial)
+// CHECK: !DILocalVariable(name: "x", arg: 2, scope: ![[SP]]{{.*}})
+// CHECK: !DILocalVariable(name: "arr", arg: 3, scope: ![[SP]]{{.*}})
+// CHECK: !DILocalVariable(name: "i", arg: 4, scope: ![[SP]]{{.*}})
diff --git a/mlir/test/Target/LLVMIR/omptarget-debug-var-2.mlir b/mlir/test/Target/LLVMIR/omptarget-debug-var-2.mlir
new file mode 100644
index 0000000000000..22db86fd85e2c
--- /dev/null
+++ b/mlir/test/Target/LLVMIR/omptarget-debug-var-2.mlir
@@ -0,0 +1,62 @@
+// RUN: mlir-translate -mlir-to-llvmir %s | FileCheck %s
+
+#int_ty = #llvm.di_basic_type<tag = DW_TAG_base_type, name = "integer",
+  sizeInBits = 32, encoding = DW_ATE_signed>
+#real_ty = #llvm.di_basic_type<tag = DW_TAG_base_type, name = "real",
+  sizeInBits = 32, encoding = DW_ATE_float>
+#file = #llvm.di_file<"target.f90" in "">
+#di_null_type = #llvm.di_null_type
+#cu = #llvm.di_compile_unit<id = distinct[0]<>,
+  sourceLanguage = DW_LANG_Fortran95, file = #file, isOptimized = false,
+  emissionKind = Full>
+#array_ty = #llvm.di_composite_type<tag = DW_TAG_array_type,
+  baseType = #int_ty, elements = #llvm.di_subrange<count = 10 : i64>>
+#sp_ty = #llvm.di_subroutine_type<callingConvention = DW_CC_program,
+  types = #di_null_type>
+#g_var = #llvm.di_global_variable<scope = #cu, name = "arr",
+  linkageName = "_QFEarr", file = #file, line = 4,
+  type = #array_ty, isDefined = true>
+#g_var_expr = #llvm.di_global_variable_expression<var = #g_var>
+#sp = #llvm.di_subprogram<id = distinct[2]<>, compileUnit = #cu, scope = #file,
+  name = "test", file = #file, subprogramFlags = "Definition", type = #sp_ty>
+#var_arr = #llvm.di_local_variable<scope = #sp,
+  name = "arr", file = #file, line = 4, type = #array_ty>
+#var_i = #llvm.di_local_variable<scope = #sp,
+  name = "i", file = #file, line = 13, type = #int_ty>
+#var_x = #llvm.di_local_variable<scope = #sp,
+ name = "x", file = #file, line = 12, type = #real_ty>
+
+module attributes {omp.is_target_device = false} {
+  llvm.func @test() {
+    %0 = llvm.mlir.constant(1 : i64) : i64
+    %1 = llvm.alloca %0 x f32 : (i64) -> !llvm.ptr
+    %4 = llvm.alloca %0 x i32 : (i64) -> !llvm.ptr
+    %6 = llvm.mlir.constant(9 : index) : i64
+    %7 = llvm.mlir.constant(0 : index) : i64
+    %8 = llvm.mlir.constant(1 : index) : i64
+    %10 = llvm.mlir.constant(10 : index) : i64
+    %11 = llvm.mlir.addressof @_QFEarr : !llvm.ptr
+    %14 = omp.map.info var_ptr(%1 : !llvm.ptr, f32) map_clauses(tofrom) capture(ByRef) -> !llvm.ptr
+    %15 = omp.map.bounds lower_bound(%7 : i64) upper_bound(%6 : i64) extent(%10 : i64) stride(%8 : i64) start_idx(%8 : i64)
+    %16 = omp.map.info var_ptr(%11 : !llvm.ptr, !llvm.array<10 x i32>) map_clauses(tofrom) capture(ByRef) bounds(%15) -> !llvm.ptr
+    %17 = omp.map.info var_ptr(%4 : !llvm.ptr, i32) map_clauses(implicit, exit_release_or_enter_alloc) capture(ByCopy) -> !llvm.ptr
+    omp.target map_entries(%14 -> %arg0, %16 -> %arg1, %17 -> %arg2 : !llvm.ptr, !llvm.ptr, !llvm.ptr) {
+      llvm.intr.dbg.declare #var_x = %arg0 : !llvm.ptr
+      llvm.intr.dbg.declare #var_arr = %arg1 : !llvm.ptr
+      llvm.intr.dbg.declare #var_i = %arg2 : !llvm.ptr
+      omp.terminator
+    }
+    llvm.return
+  } loc(#loc3)
+  llvm.mlir.global internal @_QFEarr() {addr_space = 0 : i32, dbg_exprs = [#g_var_expr]} : !llvm.array<10 x i32> {
+  } loc(#loc4)
+}
+#loc1 = loc("target.f90":4:7)
+#loc2 = loc("target.f90":11:7)
+#loc3 = loc(fused<#sp>[#loc2])
+#loc4 = loc(fused<#g_var>[#loc1])
+
+// CHECK: ![[SP:[0-9]+]] = distinct !DISubprogram(name: "__omp_offloading{{.*}}test{{.*}})
+// CHECK: !DILocalVariable(name: "x", arg: 1, scope: ![[SP]]{{.*}})
+// CHECK: !DILocalVariable(name: "arr", arg: 2, scope: ![[SP]]{{.*}})
+// CHECK: !DILocalVariable(name: "i", arg: 3, scope: ![[SP]]{{.*}})
diff --git a/mlir/test/Target/LLVMIR/omptarget-declare-target-llvm-host.mlir b/mlir/test/Target/LLVMIR/omptarget-declare-target-llvm-host.mlir
index 244c0315c2dbc..0700c378b0056 100644
--- a/mlir/test/Target/LLVMIR/omptarget-declare-target-llvm-host.mlir
+++ b/mlir/test/Target/LLVMIR/omptarget-declare-target-llvm-host.mlir
@@ -1,20 +1,20 @@
 // RUN: mlir-translate -mlir-to-llvmir %s | FileCheck %s
 
-// CHECK-DAG: %struct.__tgt_offload_entry = type { ptr, ptr, i64, i32, i32 }
+// CHECK-DAG: %struct.__tgt_offload_entry = type { i64, i16, i16, i32, ptr, ptr, i64, i64, ptr }
 // CHECK-DAG: !omp_offload.info = !{!{{.*}}}
 module attributes {llvm.target_triple = "x86_64-unknown-linux-gnu", omp.is_target_device = false} {
 
   // CHECK-DAG: @_QMtest_0Earray_1d = global [3 x i32] [i32 1, i32 2, i32 3]
   // CHECK-DAG: @_QMtest_0Earray_1d_decl_tgt_ref_ptr = weak global ptr @_QMtest_0Earray_1d
   // CHECK-DAG: @.offloading.entry_name{{.*}} = internal unnamed_addr constant [36 x i8] c"_QMtest_0Earray_1d_decl_tgt_ref_ptr\00"
-  // CHECK-DAG: @.offloading.entry._QMtest_0Earray_1d_decl_tgt_ref_ptr = weak constant %struct.__tgt_offload_entry { ptr @_QMtest_0Earray_1d_decl_tgt_ref_ptr, ptr @.offloading.entry_name{{.*}}, i64 8, i32 1, i32 0 }, section "omp_offloading_entries", align 1
+  // CHECK-DAG: @.offloading.entry._QMtest_0Earray_1d_decl_tgt_ref_ptr = weak constant %struct.__tgt_offload_entry { i64 0, i16 1, i16 1, i32 1, ptr @_QMtest_0Earray_1d_decl_tgt_ref_ptr, ptr @.offloading.entry_name{{.*}}, i64 8, i64 0, ptr null }, section "llvm_offload_entries"
   // CHECK-DAG: !{{.*}} = !{i32 {{.*}}, !"_QMtest_0Earray_1d_decl_tgt_ref_ptr", i32 {{.*}}, i32 {{.*}}}
   llvm.mlir.global external @_QMtest_0Earray_1d(dense<[1, 2, 3]> : tensor<3xi32>) {addr_space = 0 : i32, omp.declare_target = #omp.declaretarget<device_type = (any), capture_clause = (link)>} : !llvm.array<3 x i32>
 
   // CHECK-DAG: @_QMtest_0Earray_2d = global [2 x [2 x i32]] {{.*}}
   // CHECK-DAG: @_QMtest_0Earray_2d_decl_tgt_ref_ptr = weak global ptr @_QMtest_0Earray_2d
   // CHECK-DAG: @.offloading.entry_name{{.*}} = internal unnamed_addr constant [36 x i8] c"_QMtest_0Earray_2d_decl_tgt_ref_ptr\00"
-  // CHECK-DAG: @.offloading.entry._QMtest_0Earray_2d_decl_tgt_ref_ptr = weak constant %struct.__tgt_offload_entry { ptr @_QMtest_0Earray_2d_decl_tgt_ref_ptr, ptr @.offloading.entry_name{{.*}}, i64 8, i32 1, i32 0 }, section "omp_offloading_entries", align 1
+  // CHECK-DAG: @.offloading.entry._QMtest_0Earray_2d_decl_tgt_ref_ptr = weak constant %struct.__tgt_offload_entry { i64 0, i16 1, i16 1, i32 1, ptr @_QMtest_0Earray_2d_decl_tgt_ref_ptr, ptr @.offloading.entry_name{{.*}}, i64 8, i64 0, ptr null }, section "llvm_offload_entries"
   // CHECK-DAG: !{{.*}} = !{i32 {{.*}}, !"_QMtest_0Earray_2d_decl_tgt_ref_ptr", i32 {{.*}}, i32 {{.*}}}
   llvm.mlir.global external @_QMtest_0Earray_2d() {addr_space = 0 : i32, omp.declare_target = #omp.declaretarget<device_type = (any), capture_clause = (link)>} : !llvm.array<2 x array<2 x i32>> {
     %0 = llvm.mlir.undef : !llvm.array<2 x array<2 x i32>>
@@ -34,7 +34,7 @@ module attributes {llvm.target_triple = "x86_64-unknown-linux-gnu", omp.is_targe
   // CHECK-DAG:  @_QMtest_0Edata_extended_link_1 = global float 2.000000e+00
   // CHECK-DAG:  @_QMtest_0Edata_extended_link_1_decl_tgt_ref_ptr = weak global ptr @_QMtest_0Edata_extended_link_1
   // CHECK-DAG:  @.offloading.entry_name{{.*}} = internal unnamed_addr constant [48 x i8] c"_QMtest_0Edata_extended_link_1_decl_tgt_ref_ptr\00"
-  // CHECK-DAG:  @.offloading.entry._QMtest_0Edata_extended_link_1_decl_tgt_ref_ptr = weak constant %struct.__tgt_offload_entry { ptr @_QMtest_0Edata_extended_link_1_decl_tgt_ref_ptr, ptr @.offloading.entry_name{{.*}}, i64 8, i32 1, i32 0 }, section "omp_offloading_entries", align 1
+  // CHECK-DAG:  @.offloading.entry._QMtest_0Edata_extended_link_1_decl_tgt_ref_ptr = weak constant %struct.__tgt_offload_entry { i64 0, i16 1, i16 1, i32 1, ptr @_QMtest_0Edata_extended_link_1_decl_tgt_ref_ptr, ptr @.offloading.entry_name{{.*}}, i64 8, i64 0, ptr null }, section "llvm_offload_entries"
   // CHECK-DAG:  !{{.*}} = !{i32 {{.*}}, !"_QMtest_0Edata_extended_link_1_decl_tgt_ref_ptr", i32 {{.*}}, i32 {{.*}}}
   llvm.mlir.global external @_QMtest_0Edata_extended_link_1() {addr_space = 0 : i32, omp.declare_target = #omp.declaretarget<device_type = (any), capture_clause = (link)>} : f32 {
     %0 = llvm.mlir.constant(2.000000e+00 : f32) : f32
@@ -44,7 +44,7 @@ module attributes {llvm.target_triple = "x86_64-unknown-linux-gnu", omp.is_targe
   // CHECK-DAG:  @_QMtest_0Edata_extended_link_2 = global float 3.000000e+00
   // CHECK-DAG:  @_QMtest_0Edata_extended_link_2_decl_tgt_ref_ptr = weak global ptr @_QMtest_0Edata_extended_link_2
   // CHECK-DAG:  @.offloading.entry_name{{.*}} = internal unnamed_addr constant [48 x i8] c"_QMtest_0Edata_extended_link_2_decl_tgt_ref_ptr\00"
-  // CHECK-DAG:  @.offloading.entry._QMtest_0Edata_extended_link_2_decl_tgt_ref_ptr = weak constant %struct.__tgt_offload_entry { ptr @_QMtest_0Edata_extended_link_2_decl_tgt_ref_ptr, ptr @.offloading.entry_name{{.*}}, i64 8, i32 1, i32 0 }, section "omp_offloading_entries", align 1
+  // CHECK-DAG:  @.offloading.entry._QMtest_0Edata_extended_link_2_decl_tgt_ref_ptr = weak constant %struct.__tgt_offload_entry { i64 0, i16 1, i16 1, i32 1, ptr @_QMtest_0Edata_extended_link_2_decl_tgt_ref_ptr, ptr @.offloading.entry_name{{.*}}, i64 8, i64 0, ptr null }, section "llvm_offload_entries"
   // CHECK-DAG:  !{{.*}} = !{i32 {{.*}}, !"_QMtest_0Edata_extended_link_2_decl_tgt_ref_ptr", i32 {{.*}}, i32 {{.*}}}
   llvm.mlir.global external @_QMtest_0Edata_extended_link_2() {addr_space = 0 : i32, omp.declare_target = #omp.declaretarget<device_type = (any), capture_clause = (link)>} : f32 {
     %0 = llvm.mlir.constant(3.000000e+00 : f32) : f32
@@ -53,7 +53,7 @@ module attributes {llvm.target_triple = "x86_64-unknown-linux-gnu", omp.is_targe
 
   // CHECK-DAG: @_QMtest_0Edata_extended_to_1 = global float 2.000000e+00
   // CHECK-DAG: @.offloading.entry_name{{.*}} = internal unnamed_addr constant [29 x i8] c"_QMtest_0Edata_extended_to_1\00"
-  // CHECK-DAG: @.offloading.entry._QMtest_0Edata_extended_to_1 = weak constant %struct.__tgt_offload_entry { ptr @_QMtest_0Edata_extended_to_1, ptr @.offloading.entry_name{{.*}}, i64 4, i32 0, i32 0 }, section "omp_offloading_entries", align 1
+  // CHECK-DAG: @.offloading.entry._QMtest_0Edata_extended_to_1 = weak constant %struct.__tgt_offload_entry { i64 0, i16 1, i16 1, i32 0, ptr @_QMtest_0Edata_extended_to_1, ptr @.offloading.entry_name{{.*}}, i64 4, i64 0, ptr null }, section "llvm_offload_entries"
   // CHECK-DAG: !{{.*}} = !{i32 {{.*}}, !"_QMtest_0Edata_extended_to_1", i32 {{.*}}, i32 {{.*}}}
   llvm.mlir.global external @_QMtest_0Edata_extended_to_1() {addr_space = 0 : i32, omp.declare_target = #omp.declaretarget<device_type = (any), capture_clause = (to)>} : f32 {
     %0 = llvm.mlir.constant(2.000000e+00 : f32) : f32
@@ -62,7 +62,7 @@ module attributes {llvm.target_triple = "x86_64-unknown-linux-gnu", omp.is_targe
 
   // CHECK-DAG: @_QMtest_0Edata_extended_enter_1 = global float 2.000000e+00
   // CHECK-DAG: @.offloading.entry_name{{.*}} = internal unnamed_addr constant [32 x i8] c"_QMtest_0Edata_extended_enter_1\00"
-  // CHECK-DAG: @.offloading.entry._QMtest_0Edata_extended_enter_1 = weak constant %struct.__tgt_offload_entry { ptr @_QMtest_0Edata_extended_enter_1, ptr @.offloading.entry_name{{.*}}, i64 4, i32 0, i32 0 }, section "omp_offloading_entries", align 1
+  // CHECK-DAG: @.offloading.entry._QMtest_0Edata_extended_enter_1 = weak constant %struct.__tgt_offload_entry { i64 0, i16 1, i16 1, i32 0, ptr @_QMtest_0Edata_extended_enter_1, ptr @.offloading.entry_name{{.*}}, i64 4, i64 0, ptr null }, section "llvm_offload_entries"
   // CHECK-DAG: !{{.*}} = !{i32 {{.*}}, !"_QMtest_0Edata_extended_enter_1", i32 {{.*}}, i32 {{.*}}}
   llvm.mlir.global external @_QMtest_0Edata_extended_enter_1() {addr_space = 0 : i32, omp.declare_target = #omp.declaretarget<device_type = (any), capture_clause = (enter)>} : f32 {
     %0 = llvm.mlir.constant(2.000000e+00 : f32) : f32
@@ -71,7 +71,7 @@ module attributes {llvm.target_triple = "x86_64-unknown-linux-gnu", omp.is_targe
 
   // CHECK-DAG: @_QMtest_0Edata_extended_to_2 = global float 3.000000e+00
   // CHECK-DAG: @.offloading.entry_name{{.*}} = internal unnamed_addr constant [29 x i8] c"_QMtest_0Edata_extended_to_2\00"
-  // CHECK-DAG: @.offloading.entry._QMtest_0Edata_extended_to_2 = weak constant %struct.__tgt_offload_entry { ptr @_QMtest_0Edata_extended_to_2, ptr @.offloading.entry_name{{.*}}, i64 4, i32 0, i32 0 }, section "omp_offloading_entries", align 1
+  // CHECK-DAG: @.offloading.entry._QMtest_0Edata_extended_to_2 = weak constant %struct.__tgt_offload_entry { i64 0, i16 1, i16 1, i32 0, ptr @_QMtest_0Edata_extended_to_2, ptr @.offloading.entry_name{{.*}}, i64 4, i64 0, ptr null }, section "llvm_offload_entries"
   // CHECK-DAG: !{{.*}} = !{i32 {{.*}}, !"_QMtest_0Edata_extended_to_2", i32 {{.*}}, i32 {{.*}}}
   llvm.mlir.global external @_QMtest_0Edata_extended_to_2() {addr_space = 0 : i32, omp.declare_target = #omp.declaretarget<device_type = (any), capture_clause = (to)>} : f32 {
     %0 = llvm.mlir.constant(3.000000e+00 : f32) : f32
@@ -80,7 +80,7 @@ module attributes {llvm.target_triple = "x86_64-unknown-linux-gnu", omp.is_targe
 
   // CHECK-DAG: @_QMtest_0Edata_extended_enter_2 = global float 3.000000e+00
   // CHECK-DAG: @.offloading.entry_name{{.*}} = internal unnamed_addr constant [32 x i8] c"_QMtest_0Edata_extended_enter_2\00"
-  // CHECK-DAG: @.offloading.entry._QMtest_0Edata_extended_enter_2 = weak constant %struct.__tgt_offload_entry { ptr @_QMtest_0Edata_extended_enter_2, ptr @.offloading.entry_name{{.*}}, i64 4, i32 0, i32 0 }, section "omp_offloading_entries", align 1
+  // CHECK-DAG: @.offloading.entry._QMtest_0Edata_extended_enter_2 = weak constant %struct.__tgt_offload_entry { i64 0, i16 1, i16 1, i32 0, ptr @_QMtest_0Edata_extended_enter_2, ptr @.offloading.entry_name{{.*}}, i64 4, i64 0, ptr null }, section "llvm_offload_entries"
   // CHECK-DAG: !{{.*}} = !{i32 {{.*}}, !"_QMtest_0Edata_extended_enter_2", i32 {{.*}}, i32 {{.*}}}
   llvm.mlir.global external @_QMtest_0Edata_extended_enter_2() {addr_space = 0 : i32, omp.declare_target = #omp.declaretarget<device_type = (any), capture_clause = (enter)>} : f32 {
     %0 = llvm.mlir.constant(3.000000e+00 : f32) : f32
@@ -90,7 +90,7 @@ module attributes {llvm.target_triple = "x86_64-unknown-linux-gnu", omp.is_targe
   // CHECK-DAG: @_QMtest_0Edata_int = global i32 1
   // CHECK-DAG: @_QMtest_0Edata_int_decl_tgt_ref_ptr = weak global ptr @_QMtest_0Edata_int
   // CHECK-DAG: @.offloading.entry_name{{.*}} = internal unnamed_addr constant [36 x i8] c"_QMtest_0Edata_int_decl_tgt_ref_ptr\00"
-  // CHECK-DAG: @.offloading.entry._QMtest_0Edata_int_decl_tgt_ref_ptr = weak constant %struct.__tgt_offload_entry { ptr @_QMtest_0Edata_int_decl_tgt_ref_ptr, ptr @.offloading.entry_name{{.*}}, i64 8, i32 1, i32 0 }, section "omp_offloading_entries", align 1
+  // CHECK-DAG: @.offloading.entry._QMtest_0Edata_int_decl_tgt_ref_ptr = weak constant %struct.__tgt_offload_entry { i64 0, i16 1, i16 1, i32 1, ptr @_QMtest_0Edata_int_decl_tgt_ref_ptr, ptr @.offloading.entry_name{{.*}}, i64 8, i64 0, ptr null }, section "llvm_offload_entries"
   // CHECK-DAG: !{{.*}} = !{i32 {{.*}}, !"_QMtest_0Edata_int_decl_tgt_ref_ptr", i32 {{.*}}, i32 {{.*}}}
   llvm.mlir.global external @_QMtest_0Edata_int() {addr_space = 0 : i32, omp.declare_target = #omp.declaretarget<device_type = (any), capture_clause = (link)>} : i32 {
     %0 = llvm.mlir.constant(10 : i32) : i32
@@ -99,7 +99,7 @@ module attributes {llvm.target_triple = "x86_64-unknown-linux-gnu", omp.is_targe
 
   // CHECK-DAG: @_QMtest_0Edata_int_clauseless_to = global i32 1
   // CHECK-DAG: @.offloading.entry_name{{.*}} = internal unnamed_addr constant [33 x i8] c"_QMtest_0Edata_int_clauseless_to\00"
-  // CHECK-DAG: @.offloading.entry._QMtest_0Edata_int_clauseless_to = weak constant %struct.__tgt_offload_entry { ptr @_QMtest_0Edata_int_clauseless_to, ptr @.offloading.entry_name{{.*}}, i64 4, i32 0, i32 0 }, section "omp_offloading_entries", align 1
+  // CHECK-DAG: @.offloading.entry._QMtest_0Edata_int_clauseless_to = weak constant %struct.__tgt_offload_entry { i64 0, i16 1, i16 1, i32 0, ptr @_QMtest_0Edata_int_clauseless_to, ptr @.offloading.entry_name{{.*}}, i64 4, i64 0, ptr null }, section "llvm_offload_entries"
   // CHECK-DAG: !{{.*}} = !{i32 {{.*}}, !"_QMtest_0Edata_int_clauseless_to", i32 {{.*}}, i32 {{.*}}}
   llvm.mlir.global external @_QMtest_0Edata_int_clauseless_to() {addr_space = 0 : i32, omp.declare_target = #omp.declaretarget<device_type = (any), capture_clause = (to)>} : i32 {
     %0 = llvm.mlir.constant(1 : i32) : i32
@@ -108,7 +108,7 @@ module attributes {llvm.target_triple = "x86_64-unknown-linux-gnu", omp.is_targe
 
   // CHECK-DAG: @_QMtest_0Edata_int_clauseless_enter = global i32 1
   // CHECK-DAG: @.offloading.entry_name{{.*}} = internal unnamed_addr constant [36 x i8] c"_QMtest_0Edata_int_clauseless_enter\00"
-  // CHECK-DAG: @.offloading.entry._QMtest_0Edata_int_clauseless_enter = weak constant %struct.__tgt_offload_entry { ptr @_QMtest_0Edata_int_clauseless_enter, ptr @.offloading.entry_name{{.*}}, i64 4, i32 0, i32 0 }, section "omp_offloading_entries", align 1
+  // CHECK-DAG: @.offloading.entry._QMtest_0Edata_int_clauseless_enter = weak constant %struct.__tgt_offload_entry { i64 0, i16 1, i16 1, i32 0, ptr @_QMtest_0Edata_int_clauseless_enter, ptr @.offloading.entry_name{{.*}}, i64 4, i64 0, ptr null }, section "llvm_offload_entries"
   // CHECK-DAG: !{{.*}} = !{i32 {{.*}}, !"_QMtest_0Edata_int_clauseless_enter", i32 {{.*}}, i32 {{.*}}}
   llvm.mlir.global external @_QMtest_0Edata_int_clauseless_enter() {addr_space = 0 : i32, omp.declare_target = #omp.declaretarget<device_type = (any), capture_clause = (enter)>} : i32 {
     %0 = llvm.mlir.constant(1 : i32) : i32
@@ -117,7 +117,7 @@ module attributes {llvm.target_triple = "x86_64-unknown-linux-gnu", omp.is_targe
 
   // CHECK-DAG: @_QMtest_0Edata_int_to = global i32 5
   // CHECK-DAG: @.offloading.entry_name{{.*}} = internal unnamed_addr constant [22 x i8] c"_QMtest_0Edata_int_to\00"
-  // CHECK-DAG: @.offloading.entry._QMtest_0Edata_int_to = weak constant %struct.__tgt_offload_entry { ptr @_QMtest_0Edata_int_to, ptr @.offloading.entry_name{{.*}}, i64 4, i32 0, i32 0 }, section "omp_offloading_entries", align 1
+  // CHECK-DAG: @.offloading.entry._QMtest_0Edata_int_to = weak constant %struct.__tgt_offload_entry { i64 0, i16 1, i16 1, i32 0, ptr @_QMtest_0Edata_int_to, ptr @.offloading.entry_name{{.*}}, i64 4, i64 0, ptr null }, section "llvm_offload_entries"
   // CHECK-DAG: !{{.*}} = !{i32 {{.*}}, !"_QMtest_0Edata_int_to", i32 {{.*}}, i32 {{.*}}}
   llvm.mlir.global external @_QMtest_0Edata_int_to() {addr_space = 0 : i32, omp.declare_target = #omp.declaretarget<device_type = (any), capture_clause = (to)>} : i32 {
     %0 = llvm.mlir.constant(5 : i32) : i32
@@ -126,7 +126,7 @@ module attributes {llvm.target_triple = "x86_64-unknown-linux-gnu", omp.is_targe
 
   // CHECK-DAG: @_QMtest_0Edata_int_enter = global i32 5
   // CHECK-DAG: @.offloading.entry_name{{.*}} = internal unnamed_addr constant [25 x i8] c"_QMtest_0Edata_int_enter\00"
-  // CHECK-DAG: @.offloading.entry._QMtest_0Edata_int_enter = weak constant %struct.__tgt_offload_entry { ptr @_QMtest_0Edata_int_enter, ptr @.offloading.entry_name{{.*}}, i64 4, i32 0, i32 0 }, section "omp_offloading_entries", align 1
+  // CHECK-DAG: @.offloading.entry._QMtest_0Edata_int_enter = weak constant %struct.__tgt_offload_entry { i64 0, i16 1, i16 1, i32 0, ptr @_QMtest_0Edata_int_enter, ptr @.offloading.entry_name{{.*}}, i64 4, i64 0, ptr null }, section "llvm_offload_entries"
   // CHECK-DAG: !{{.*}} = !{i32 {{.*}}, !"_QMtest_0Edata_int_enter", i32 {{.*}}, i32 {{.*}}}
   llvm.mlir.global external @_QMtest_0Edata_int_enter() {addr_space = 0 : i32, omp.declare_target = #omp.declaretarget<device_type = (any), capture_clause = (enter)>} : i32 {
     %0 = llvm.mlir.constant(5 : i32) : i32
@@ -136,7 +136,7 @@ module attributes {llvm.target_triple = "x86_64-unknown-linux-gnu", omp.is_targe
   // CHECK-DAG: @_QMtest_0Ept1 = global { ptr, i64, i32, i8, i8, i8, i8 } { ptr null, i64 ptrtoint (ptr getelementptr (i32, ptr null, i32 1) to i64), i32 20180515, i8 0, i8 9, i8 1, i8 0 }
   // CHECK-DAG: @_QMtest_0Ept1_decl_tgt_ref_ptr = weak global ptr @_QMtest_0Ept1
   // CHECK-DAG: @.offloading.entry_name{{.*}} = internal unnamed_addr constant [31 x i8] c"_QMtest_0Ept1_decl_tgt_ref_ptr\00"
-  // CHECK-DAG: @.offloading.entry._QMtest_0Ept1_decl_tgt_ref_ptr = weak constant %struct.__tgt_offload_entry { ptr @_QMtest_0Ept1_decl_tgt_ref_ptr, ptr @.offloading.entry_name{{.*}}, i64 8, i32 1, i32 0 }, section "omp_offloading_entries", align 1
+  // CHECK-DAG: @.offloading.entry._QMtest_0Ept1_decl_tgt_ref_ptr = weak constant %struct.__tgt_offload_entry { i64 0, i16 1, i16 1, i32 1, ptr @_QMtest_0Ept1_decl_tgt_ref_ptr, ptr @.offloading.entry_name{{.*}}, i64 8, i64 0, ptr null }, section "llvm_offload_entries"
   // CHECK-DAG: !{{.*}} = !{i32 {{.*}}, !"_QMtest_0Ept1_decl_tgt_ref_ptr", i32 {{.*}}, i32 {{.*}}}
   llvm.mlir.global external @_QMtest_0Ept1() {addr_space = 0 : i32, omp.declare_target = #omp.declaretarget<device_type = (any), capture_clause = (link)>} : !llvm.struct<(ptr, i64, i32, i8, i8, i8, i8)> {
     %0 = llvm.mlir.zero : !llvm.ptr
@@ -167,7 +167,7 @@ module attributes {llvm.target_triple = "x86_64-unknown-linux-gnu", omp.is_targe
   // CHECK-DAG: @_QMtest_0Ept2_tar = global i32 5
   // CHECK-DAG: @_QMtest_0Ept2_tar_decl_tgt_ref_ptr = weak global ptr @_QMtest_0Ept2_tar
   // CHECK-DAG: @.offloading.entry_name{{.*}} = internal unnamed_addr constant [35 x i8] c"_QMtest_0Ept2_tar_decl_tgt_ref_ptr\00"
-  // CHECK-DAG: @.offloading.entry._QMtest_0Ept2_tar_decl_tgt_ref_ptr = weak constant %struct.__tgt_offload_entry { ptr @_QMtest_0Ept2_tar_decl_tgt_ref_ptr, ptr @.offloading.entry_name{{.*}}, i64 8, i32 1, i32 0 }, section "omp_offloading_entries", align 1
+  // CHECK-DAG: @.offloading.entry._QMtest_0Ept2_tar_decl_tgt_ref_ptr = weak constant %struct.__tgt_offload_entry { i64 0, i16 1, i16 1, i32 1, ptr @_QMtest_0Ept2_tar_decl_tgt_ref_ptr, ptr @.offloading.entry_name{{.*}}, i64 8, i64 0, ptr null }, section "llvm_offload_entries"
   // CHECK-DAG: !{{.*}} = !{i32 {{.*}}, !"_QMtest_0Ept2_tar_decl_tgt_ref_ptr", i32 {{.*}}, i32 {{.*}}}
   llvm.mlir.global external @_QMtest_0Ept2_tar() {addr_space = 0 : i32, omp.declare_target = #omp.declaretarget<device_type = (any), capture_clause = (link)>} : i32 {
     %0 = llvm.mlir.constant(5 : i32) : i32
diff --git a/mlir/test/Target/LLVMIR/omptarget-if-nowait.mlir b/mlir/test/Target/LLVMIR/omptarget-if-nowait.mlir
new file mode 100644
index 0000000000000..6f8d938a4d5f2
--- /dev/null
+++ b/mlir/test/Target/LLVMIR/omptarget-if-nowait.mlir
@@ -0,0 +1,46 @@
+// RUN: mlir-translate -mlir-to-llvmir %s | FileCheck %s
+
+module attributes {omp.is_target_device = false, omp.target_triples = ["amdgcn-amd-amdhsa"]} {
+  llvm.func @target_if_nowait(%arg0: !llvm.ptr, %arg1: !llvm.ptr) {
+    %0 = llvm.mlir.constant(1 : i64) : i64
+    %3 = llvm.alloca %0 x i32 {bindc_name = "cond"} : (i64) -> !llvm.ptr
+    %6 = llvm.load %3 : !llvm.ptr -> i32
+    %7 = llvm.mlir.constant(0 : i64) : i32
+    %8 = llvm.icmp "ne" %6, %7 : i32
+    %9 = omp.map.info var_ptr(%3 : !llvm.ptr, i32) map_clauses(implicit, exit_release_or_enter_alloc) capture(ByCopy) -> !llvm.ptr {name = "cond"}
+    %10 = omp.map.info var_ptr(%arg0 : !llvm.ptr, f32) map_clauses(implicit, exit_release_or_enter_alloc) capture(ByCopy) -> !llvm.ptr {name = "var"}
+    %11 = omp.map.info var_ptr(%arg1 : !llvm.ptr, f32) map_clauses(implicit, exit_release_or_enter_alloc) capture(ByCopy) -> !llvm.ptr {name = "val"}
+    omp.target if(%8) nowait map_entries(%10 -> %arg3, %11 -> %arg4 : !llvm.ptr, !llvm.ptr) {
+      %12 = llvm.load %arg4 : !llvm.ptr -> f32
+      llvm.store %12, %arg3 : f32, !llvm.ptr
+      omp.terminator
+    }
+    llvm.return
+  }
+}
+
+// CHECK: define void @target_if_nowait{{.*}} {
+// CHECK: omp_if.then:
+// CHECK:   br label %[[TARGET_TASK_BB:.*]]
+
+// CHECK: [[TARGET_TASK_BB]]:
+// CHECK:   call ptr @__kmpc_omp_target_task_alloc
+// CHECK:   br label %[[OFFLOAD_CONT:.*]]
+
+// CHECK: [[OFFLOAD_CONT]]:
+// CHECK:   br label %omp_if.end
+
+// CHECK: omp_if.else:
+// CHECK:   br label %[[HOST_TASK_BB:.*]]
+
+// CHECK: [[HOST_TASK_BB]]:
+// CHECK:   call ptr @__kmpc_omp_task_alloc
+// CHECK:   br label %[[HOST_TASK_CONT:.*]]
+
+// CHECK: [[HOST_TASK_CONT]]:
+// CHECK:   br label %omp_if.end
+
+// CHECK: omp_if.end:
+// CHECK:   ret void
+// CHECK: }
+
diff --git a/mlir/test/Target/LLVMIR/omptarget-nested-ptr-record-type-mapping-host.mlir b/mlir/test/Target/LLVMIR/omptarget-nested-ptr-record-type-mapping-host.mlir
index 5d772a13fe578..e82db30f20b8b 100644
--- a/mlir/test/Target/LLVMIR/omptarget-nested-ptr-record-type-mapping-host.mlir
+++ b/mlir/test/Target/LLVMIR/omptarget-nested-ptr-record-type-mapping-host.mlir
@@ -27,7 +27,7 @@ module attributes {omp.is_target_device = false, omp.target_triples = ["amdgcn-a
   }
 }
 
-// CHECK: @.offload_sizes{{.*}} = private unnamed_addr constant [4 x i64] [i64 0, i64 48, i64 8, i64 20]
+// CHECK: @.offload_sizes{{.*}} = private unnamed_addr constant [4 x i64] [i64 0, i64 48, i64 8, i64 0]
 // CHECK: @.offload_maptypes{{.*}} = private unnamed_addr constant [4 x i64] [i64 32, i64 281474976710659, i64 281474976710659, i64 281474976710675]
 
 // CHECK: define void @omp_nested_derived_type_alloca_map(ptr %[[ARG:.*]]) {
diff --git a/mlir/test/Target/LLVMIR/omptarget-nowait-host-only.mlir b/mlir/test/Target/LLVMIR/omptarget-nowait-host-only.mlir
new file mode 100644
index 0000000000000..6b634226a3568
--- /dev/null
+++ b/mlir/test/Target/LLVMIR/omptarget-nowait-host-only.mlir
@@ -0,0 +1,29 @@
+// RUN: mlir-translate -mlir-to-llvmir %s | FileCheck %s
+
+// Tests `target ... nowait` when code gen targets the host rather than a
+// device.
+
+module attributes {omp.is_target_device = false} {
+  llvm.func @omp_target_nowait_() {
+    %0 = llvm.mlir.constant(1 : i64) : i64
+    %1 = llvm.alloca %0 x f32 {bindc_name = "x"} : (i64) -> !llvm.ptr
+    %3 = omp.map.info var_ptr(%1 : !llvm.ptr, f32) map_clauses(tofrom) capture(ByRef) -> !llvm.ptr {name = "x"}
+    omp.target nowait map_entries(%3 -> %arg0 : !llvm.ptr) {
+      %4 = llvm.mlir.constant(5.000000e+00 : f32) : f32
+      llvm.store %4, %arg0 : f32, !llvm.ptr
+      omp.terminator
+    }
+    llvm.return
+  }
+}
+
+// CHECK: define void @omp_target_nowait_()
+// CHECK-NOT: define {{.*}} @
+// CHECK-NOT: call ptr @__kmpc_omp_target_task_alloc({{.*}})
+// Verify that we directly emit a call to the "target" region's body from the
+// parent function of the the `omp.target` op.
+// CHECK: call void @__omp_offloading_[[DEV:.*]]_[[FIL:.*]]_omp_target_nowait__l[[LINE:.*]](ptr {{.*}})
+// CHECK-NEXT: ret void
+
+// CHECK: define internal void @__omp_offloading_[[DEV]]_[[FIL]]_omp_target_nowait__l[[LINE]](ptr %[[ADDR_X:.*]])
+// CHECK: store float 5{{.*}}, ptr %[[ADDR_X]], align 4
diff --git a/mlir/test/Target/LLVMIR/omptarget-nullary-record-ptr-member-map.mlir b/mlir/test/Target/LLVMIR/omptarget-nullary-record-ptr-member-map.mlir
new file mode 100644
index 0000000000000..c2d38f0461c26
--- /dev/null
+++ b/mlir/test/Target/LLVMIR/omptarget-nullary-record-ptr-member-map.mlir
@@ -0,0 +1,22 @@
+// RUN: mlir-translate -mlir-to-llvmir %s | FileCheck %s
+
+// This test checks that we generate a select statement in cases where we're
+// mapping a pointer, to select a size of 0 when the pointer is null and
+// select the size of the mapped type when it is not null. Preventing a runtime
+// mapping error in cases where we legally map null data to device.
+
+module attributes {omp.is_target_device = false, omp.target_triples = ["amdgcn-amd-amdhsa"]} {
+  llvm.func @test_select_gen(%arg0: !llvm.ptr, %arg1: !llvm.ptr) {
+    %0 = omp.map.info var_ptr(%arg0 : !llvm.ptr, i32) var_ptr_ptr(%arg1 : !llvm.ptr) map_clauses(tofrom) capture(ByRef) -> !llvm.ptr
+    %1 = omp.map.info var_ptr(%arg0 : !llvm.ptr, !llvm.struct<(ptr, i64, i32, i8, i8, i8, i8)>) map_clauses(to) capture(ByRef) members(%0 : [0] : !llvm.ptr) -> !llvm.ptr
+    omp.target map_entries(%0 -> %arg2, %1 -> %arg3 : !llvm.ptr, !llvm.ptr) {
+      omp.terminator
+    }
+    llvm.return
+  }
+}
+
+// CHECK: {{.*}}test_select_gen({{.*}}, ptr %[[ARG1:.*]]) {{.*}}
+// CHECK: %[[LOAD_ARG1:.*]] = load ptr, ptr %[[ARG1]], align 8
+// CHECK: %[[ICMP_ARG1:.*]] = icmp eq ptr %[[LOAD_ARG1]], null
+// CHECK: %[[SEL_ARG1:.*]] = select i1 %[[ICMP_ARG1]], i64 0, i64 4
diff --git a/mlir/test/Target/LLVMIR/omptarget-record-type-with-ptr-member-host.mlir b/mlir/test/Target/LLVMIR/omptarget-record-type-with-ptr-member-host.mlir
index d5ca33fb38f38..7421d8848ac90 100644
--- a/mlir/test/Target/LLVMIR/omptarget-record-type-with-ptr-member-host.mlir
+++ b/mlir/test/Target/LLVMIR/omptarget-record-type-with-ptr-member-host.mlir
@@ -59,7 +59,7 @@ module attributes {omp.is_target_device = false, omp.target_triples = ["amdgcn-a
 
 // CHECK: @[[FULL_ARR_GLOB:.*]] = internal global { ptr, i64, i32, i8, i8, i8, i8, [1 x [3 x i64]] } undef
 // CHECK: @[[ARR_SECT_GLOB:.*]] = internal global { ptr, i64, i32, i8, i8, i8, i8, [1 x [3 x i64]] } undef
-// CHECK: @.offload_sizes = private unnamed_addr constant [12 x i64] [i64 0, i64 48, i64 8, i64 0, i64 0, i64 48, i64 8, i64 0, i64 0, i64 24, i64 8, i64 4]
+// CHECK: @.offload_sizes = private unnamed_addr constant [12 x i64] [i64 0, i64 48, i64 8, i64 0, i64 0, i64 48, i64 8, i64 0, i64 0, i64 24, i64 8, i64 0]
 // CHECK: @.offload_maptypes = private unnamed_addr constant [12 x i64] [i64 32, i64 281474976710659, i64 281474976710659, i64 281474976710675, i64 32, i64 1407374883553283, i64 1407374883553283, i64 1407374883553299, i64 32, i64 2533274790395907, i64 2533274790395907, i64 2533274790395923]
 // CHECK: @.offload_mapnames = private constant [12 x ptr] [ptr @{{.*}}, ptr @{{.*}}, ptr @{{.*}}, ptr @{{.*}}, ptr @{{.*}}, ptr @{{.*}}, ptr @{{.*}}, ptr @{{.*}}, ptr @{{.*}}, ptr @{{.*}}, ptr @{{.*}}, ptr @{{.*}}]
 
@@ -86,7 +86,11 @@ module attributes {omp.is_target_device = false, omp.target_triples = ["amdgcn-a
 // CHECK: %[[ARR_SECT_PTR:.*]] = getelementptr inbounds i32, ptr %[[LARR_SECT]], i64 %[[ARR_SECT_OFFSET1]]
 // CHECK: %[[SCALAR_PTR_LOAD:.*]] = load ptr, ptr %[[SCALAR_BASE]], align 8
 // CHECK: %[[FULL_ARR_DESC_SIZE:.*]] = sdiv exact i64 sub (i64 ptrtoint (ptr getelementptr ({ ptr, i64, i32, i8, i8, i8, i8, [1 x [3 x i64]] }, ptr @full_arr, i32 1) to i64), i64 ptrtoint (ptr @full_arr to i64)), ptrtoint (ptr getelementptr (i8, ptr null, i32 1) to i64)
+// CHECK: %[[FULL_ARR_SIZE_CMP:.*]] = icmp eq ptr %[[FULL_ARR_PTR]], null
+// CHECK: %[[FULL_ARR_SIZE_SEL:.*]] = select i1 %[[FULL_ARR_SIZE_CMP]], i64 0, i64 %[[FULL_ARR_SIZE]]
 // CHECK: %[[ARR_SECT_DESC_SIZE:.*]] = sdiv exact i64 sub (i64 ptrtoint (ptr getelementptr ({ ptr, i64, i32, i8, i8, i8, i8, [1 x [3 x i64]] }, ptr @sect_arr, i32 1) to i64), i64 ptrtoint (ptr @sect_arr to i64)), ptrtoint (ptr getelementptr (i8, ptr null, i32 1) to i64)
+// CHECK: %[[ARR_SECT_SIZE_CMP:.*]] = icmp eq ptr %[[ARR_SECT_PTR]], null
+// CHECK: %[[ARR_SECT_SIZE_SEL:.*]] = select i1 %[[ARR_SECT_SIZE_CMP]], i64 0, i64 %[[ARR_SECT_SIZE]]
 // CHECK: %[[SCALAR_DESC_SZ4:.*]] = getelementptr { ptr, i64, i32, i8, i8, i8, i8 }, ptr %[[SCALAR_ALLOCA]], i32 1
 // CHECK: %[[SCALAR_DESC_SZ3:.*]] = ptrtoint ptr %[[SCALAR_DESC_SZ4]] to i64
 // CHECK: %[[SCALAR_DESC_SZ2:.*]] = ptrtoint ptr %[[SCALAR_ALLOCA]] to i64
@@ -112,7 +116,7 @@ module attributes {omp.is_target_device = false, omp.target_triples = ["amdgcn-a
 // CHECK: %[[OFFLOADPTRS:.*]] = getelementptr inbounds [12 x ptr], ptr %.offload_ptrs, i32 0, i32 3
 // CHECK: store ptr %[[FULL_ARR_PTR]], ptr %[[OFFLOADPTRS]], align 8
 // CHECK: %[[OFFLOADSIZES:.*]] = getelementptr inbounds [12 x i64], ptr %.offload_sizes, i32 0, i32 3
-// CHECK: store i64 %[[FULL_ARR_SIZE]], ptr %[[OFFLOADSIZES]], align 8
+// CHECK: store i64 %[[FULL_ARR_SIZE_SEL]], ptr %[[OFFLOADSIZES]], align 8
 
 // CHECK: %[[OFFLOADBASEPTRS:.*]] = getelementptr inbounds [12 x ptr], ptr %.offload_baseptrs, i32 0, i32 4
 // CHECK: store ptr @sect_arr, ptr %[[OFFLOADBASEPTRS]], align 8
@@ -133,7 +137,7 @@ module attributes {omp.is_target_device = false, omp.target_triples = ["amdgcn-a
 // CHECK: %[[OFFLOADPTRS:.*]] = getelementptr inbounds [12 x ptr], ptr %.offload_ptrs, i32 0, i32 7
 // CHECK: store ptr %[[ARR_SECT_PTR]], ptr %[[OFFLOADPTRS]], align 8
 // CHECK: %[[OFFLOADSIZES:.*]] = getelementptr inbounds [12 x i64], ptr %.offload_sizes, i32 0, i32 7
-// CHECK: store i64 %[[ARR_SECT_SIZE]], ptr %[[OFFLOADSIZES]], align 8
+// CHECK: store i64 %[[ARR_SECT_SIZE_SEL]], ptr %[[OFFLOADSIZES]], align 8
 
 // CHECK: %[[OFFLOADBASEPTRS:.*]] = getelementptr inbounds [12 x ptr], ptr %.offload_baseptrs, i32 0, i32 8
 // CHECK: store ptr %[[SCALAR_ALLOCA]], ptr %[[OFFLOADBASEPTRS]], align 8
diff --git a/mlir/test/Target/LLVMIR/openmp-firstprivate.mlir b/mlir/test/Target/LLVMIR/openmp-firstprivate.mlir
index 79412fb69f758..0358f713a3246 100644
--- a/mlir/test/Target/LLVMIR/openmp-firstprivate.mlir
+++ b/mlir/test/Target/LLVMIR/openmp-firstprivate.mlir
@@ -11,12 +11,7 @@ llvm.func @parallel_op_firstprivate(%arg0: !llvm.ptr) {
   llvm.return
 }
 
-omp.private {type = firstprivate} @x.privatizer : !llvm.ptr alloc {
-^bb0(%arg0: !llvm.ptr):
-  %c1 = llvm.mlir.constant(1 : i32) : i32
-  %0 = llvm.alloca %c1 x f32 : (i32) -> !llvm.ptr
-  omp.yield(%0 : !llvm.ptr)
-} copy {
+omp.private {type = firstprivate} @x.privatizer : f32 copy {
 ^bb0(%arg0: !llvm.ptr, %arg1: !llvm.ptr):
   %0 = llvm.load %arg0 : !llvm.ptr -> f32
   llvm.store %0, %arg1 : f32, !llvm.ptr
@@ -63,6 +58,14 @@ llvm.func @parallel_op_firstprivate_multi_block(%arg0: !llvm.ptr) {
 // CHECK: omp.par.entry:
 // CHECK:  %[[ORIG_PTR_PTR:.*]] = getelementptr { ptr }, ptr %{{.*}}, i32 0, i32 0
 // CHECK:  %[[ORIG_PTR:.*]] = load ptr, ptr %[[ORIG_PTR_PTR]], align 8
+// CHECK:  %[[PRIV_ALLOC:.*]] = alloca float, align 4
+// CHECK-NEXT: br label %omp.region.after_alloca
+
+// CHECK: omp.region.after_alloca:
+// CHECK-NEXT:   br label %[[PAR_REG:.*]]
+
+// Check that the body of the parallel region loads from the private clone.
+// CHECK: [[PAR_REG]]:
 // CHECK:   br label %[[PRIV_BB1:.*]]
 
 // CHECK: [[PRIV_BB1]]:
@@ -72,28 +75,26 @@ llvm.func @parallel_op_firstprivate_multi_block(%arg0: !llvm.ptr) {
 // CHECK-NEXT: br label %[[PRIV_BB2:.*]]
 
 // CHECK: [[PRIV_BB2]]:
-// CHECK-NEXT: %[[C1:.*]] = phi i32 [ 1, %[[PRIV_BB1]] ]
-// CHECK-NEXT: %[[PRIV_ALLOC:.*]] = alloca float, i32 %[[C1]], align 4
+// CHECK-NEXT: br label %[[PRIV_BB3:.*]]
+
+// CHECK: [[PRIV_BB3]]:
 // CHECK-NEXT: br label %omp.region.cont
 
 // CHECK: omp.region.cont:
-// CHECK-NEXT: %[[PRIV_ALLOC2:.*]] = phi ptr [ %[[PRIV_ALLOC]], %[[PRIV_BB2]] ]
-// CHECK-NEXT: br label %omp.private.latealloc
-
-// CHECK: omp.private.latealloc:
+// CHECK-NEXT: %[[PRIV_ALLOC2:.*]] = phi ptr [ %[[PRIV_ALLOC]], %[[PRIV_BB3]] ]
 // CHECK-NEXT: br label %omp.private.copy
 
 // CHECK: omp.private.copy:
-// CHECK-NEXT: br label %omp.private.copy3
+// CHECK-NEXT: br label %omp.private.copy4
 
-// CHECK: omp.private.copy3:
+// CHECK: omp.private.copy4:
 // CHECK-NEXT: %[[ORIG_VAL:.*]] = load float, ptr %[[ORIG_PTR]], align 4
 // CHECK-NEXT: br label %[[PRIV_BB3:.*]]
 
 // Check contents of the 2nd block in the `copy` region.
 // CHECK: [[PRIV_BB3]]:
-// CHECK-NEXT: %[[ORIG_VAL2:.*]] = phi float [ %[[ORIG_VAL]], %omp.private.copy3 ]
-// CHECK-NEXT: %[[PRIV_ALLOC3:.*]] = phi ptr [ %[[PRIV_ALLOC2]], %omp.private.copy3 ]
+// CHECK-NEXT: %[[ORIG_VAL2:.*]] = phi float [ %[[ORIG_VAL]], %omp.private.copy4 ]
+// CHECK-NEXT: %[[PRIV_ALLOC3:.*]] = phi ptr [ %[[PRIV_ALLOC2]], %omp.private.copy4 ]
 // CHECK-NEXT: store float %[[ORIG_VAL2]], ptr %[[PRIV_ALLOC3]], align 4
 // CHECK-NEXT: br label %[[PRIV_CONT:.*]]
 
@@ -101,20 +102,14 @@ llvm.func @parallel_op_firstprivate_multi_block(%arg0: !llvm.ptr) {
 // address.
 // CHECK: [[PRIV_CONT]]:
 // CHECK-NEXT:   %[[PRIV_ALLOC4:.*]] = phi ptr [ %[[PRIV_ALLOC3]], %[[PRIV_BB3]] ]
-// CHECK-NEXT:   br label %[[PAR_REG:.*]]
-
-// Check that the body of the parallel region loads from the private clone.
-// CHECK: [[PAR_REG]]:
 // CHECK:        %{{.*}} = load float, ptr %[[PRIV_ALLOC2]], align 4
 
-omp.private {type = firstprivate} @multi_block.privatizer : !llvm.ptr alloc {
-^bb0(%arg0: !llvm.ptr):
-  %c1 = llvm.mlir.constant(1 : i32) : i32
-  llvm.br ^bb1(%c1 : i32)
+omp.private {type = firstprivate} @multi_block.privatizer : f32 init {
+^bb0(%arg0: !llvm.ptr, %arg1: !llvm.ptr):
+  llvm.br ^bb1
 
-^bb1(%arg1: i32):
-  %0 = llvm.alloca %arg1 x f32 : (i32) -> !llvm.ptr
-  omp.yield(%0 : !llvm.ptr)
+^bb1:
+  omp.yield(%arg1 : !llvm.ptr)
 
 } copy {
 ^bb0(%arg0: !llvm.ptr, %arg1: !llvm.ptr):
@@ -145,8 +140,8 @@ llvm.func @non_const_len_char_test(%n: !llvm.ptr {fir.bindc_name = "n"}) {
   llvm.return
 }
 
-omp.private {type = firstprivate} @non_const_len_char : !llvm.struct<(ptr, i64)> alloc {
-^bb0(%orig_val: !llvm.struct<(ptr, i64)>):
+omp.private {type = firstprivate} @non_const_len_char : !llvm.struct<(ptr, i64)> init {
+^bb0(%orig_val: !llvm.struct<(ptr, i64)>, %arg1: !llvm.struct<(ptr, i64)>):
   %str_len = llvm.extractvalue %orig_val[1] : !llvm.struct<(ptr, i64)>
   %priv_alloc = llvm.alloca %str_len x i8 {bindc_name = "str", pinned} : (i64) -> !llvm.ptr
   %priv_val = llvm.mlir.undef : !llvm.struct<(ptr, i64)>
@@ -179,12 +174,7 @@ llvm.func @foo()
 // global), then we miss finding that input and we do not privatize the
 // variable.
 
-omp.private {type = firstprivate} @global_privatizer : !llvm.ptr alloc {
-^bb0(%arg0: !llvm.ptr):
-  %0 = llvm.mlir.constant(1 : i64) : i64
-  %1 = llvm.alloca %0 x f32 {bindc_name = "global", pinned} : (i64) -> !llvm.ptr
-  omp.yield(%1 : !llvm.ptr)
-} copy {
+omp.private {type = firstprivate} @global_privatizer : f32 copy {
 ^bb0(%arg0: !llvm.ptr, %arg1: !llvm.ptr):
   %0 = llvm.load %arg0 : !llvm.ptr -> f32
   llvm.store %0, %arg1 : f32, !llvm.ptr
@@ -210,6 +200,6 @@ llvm.mlir.global internal @global() {addr_space = 0 : i32} : f32 {
 // CHECK-NEXT:  omp.par.entry:
 // Verify that we found the privatizer by checking that we properly inlined the
 // bodies of the alloc and copy regions.
-// CHECK:         %[[PRIV_ALLOC:.*]] = alloca float, i64 1, align 4
+// CHECK:         %[[PRIV_ALLOC:.*]] = alloca float, align 4
 // CHECK:         %[[GLOB_VAL:.*]] = load float, ptr @global, align 4
 // CHECK:         store float %[[GLOB_VAL]], ptr %[[PRIV_ALLOC]], align 4
diff --git a/mlir/test/Target/LLVMIR/openmp-llvm.mlir b/mlir/test/Target/LLVMIR/openmp-llvm.mlir
index 9868ef227d49e..5f42240cf978e 100644
--- a/mlir/test/Target/LLVMIR/openmp-llvm.mlir
+++ b/mlir/test/Target/LLVMIR/openmp-llvm.mlir
@@ -2801,12 +2801,7 @@ llvm.func @par_task_(%arg0: !llvm.ptr {fir.bindc_name = "a"}) {
 llvm.func @foo(!llvm.ptr) -> ()
 llvm.func @destroy(!llvm.ptr) -> ()
 
-omp.private {type = firstprivate} @privatizer : !llvm.ptr alloc {
-^bb0(%arg0 : !llvm.ptr):
-  %0 = llvm.mlir.constant(1 : i64) : i64
-  %1 = llvm.alloca %0 x i32 : (i64) -> !llvm.ptr
-  omp.yield(%1 : !llvm.ptr)
-} copy {
+omp.private {type = firstprivate} @privatizer : i32 copy {
 ^bb0(%arg0: !llvm.ptr, %arg1: !llvm.ptr):
   %0 = llvm.load %arg0 : !llvm.ptr -> i32
   llvm.store %0, %arg1 : i32, !llvm.ptr
@@ -2829,19 +2824,24 @@ llvm.func @task(%arg0 : !llvm.ptr) {
 // CHECK:         %[[VAL_11:.*]] = load ptr, ptr %[[VAL_12:.*]], align 8
 // CHECK:         %[[VAL_13:.*]] = getelementptr { ptr }, ptr %[[VAL_11]], i32 0, i32 0
 // CHECK:         %[[VAL_14:.*]] = load ptr, ptr %[[VAL_13]], align 8
-// CHECK:         %[[VAL_15:.*]] = alloca i32, i64 1, align 4
-// CHECK:         br label %omp.private.latealloc
-// CHECK:       omp.private.latealloc:                            ; preds = %task.alloca
+// CHECK:         %[[VAL_15:.*]] = alloca i32, align 4
+// CHECK:         br label %omp.region.after_alloca
+
+// CHECK:       omp.region.after_alloca:
+// CHECK:         br label %task.body
+
+// CHECK:       task.body:                                        ; preds = %omp.region.after_alloca
+// CHECK:         br label %omp.private.init
+
+// CHECK:       omp.private.init:                                 ; preds = %task.body
 // CHECK:         br label %omp.private.copy
-// CHECK:       omp.private.copy:                                 ; preds = %omp.private.latealloc
+
+// CHECK:       omp.private.copy:                                 ; preds = %omp.private.init
 // CHECK:         %[[VAL_19:.*]] = load i32, ptr %[[VAL_14]], align 4
 // CHECK:         store i32 %[[VAL_19]], ptr %[[VAL_15]], align 4
-// CHECK:         br label %[[VAL_20:.*]]
-// CHECK:       [[VAL_20]]:
-// CHECK:         br label %task.body
-// CHECK:       task.body:                                        ; preds = %[[VAL_20]]
 // CHECK:         br label %omp.task.region
-// CHECK:       omp.task.region:                                  ; preds = %task.body
+
+// CHECK:       omp.task.region:                                  ; preds = %omp.private.copy
 // CHECK:         call void @foo(ptr %[[VAL_15]])
 // CHECK:         br label %omp.region.cont
 // CHECK:       omp.region.cont:                                  ; preds = %omp.task.region
diff --git a/mlir/test/Target/LLVMIR/openmp-omp.private-dealloc.mlir b/mlir/test/Target/LLVMIR/openmp-omp.private-dealloc.mlir
index 835caccb262c4..97f4980b1782f 100644
--- a/mlir/test/Target/LLVMIR/openmp-omp.private-dealloc.mlir
+++ b/mlir/test/Target/LLVMIR/openmp-omp.private-dealloc.mlir
@@ -10,12 +10,7 @@ llvm.func @parallel_op_dealloc(%arg0: !llvm.ptr) {
   llvm.return
 }
 
-omp.private {type = firstprivate} @x.privatizer : !llvm.ptr alloc {
-^bb0(%arg0: !llvm.ptr):
-  %c1 = llvm.mlir.constant(1 : i32) : i32
-  %0 = llvm.alloca %c1 x f32 : (i32) -> !llvm.ptr
-  omp.yield(%0 : !llvm.ptr)
-} copy {
+omp.private {type = firstprivate} @x.privatizer : f32 copy {
 ^bb0(%arg0: !llvm.ptr, %arg1: !llvm.ptr):
   %0 = llvm.load %arg0 : !llvm.ptr -> f32
   llvm.store %0, %arg1 : f32, !llvm.ptr
diff --git a/mlir/test/Target/LLVMIR/openmp-private.mlir b/mlir/test/Target/LLVMIR/openmp-private.mlir
index d2ca03a8fa027..704935fa12e97 100644
--- a/mlir/test/Target/LLVMIR/openmp-private.mlir
+++ b/mlir/test/Target/LLVMIR/openmp-private.mlir
@@ -60,9 +60,9 @@ llvm.func @parallel_op_2_privates(%arg0: !llvm.ptr, %arg1: !llvm.ptr) {
 
 // Check that the privatizer alloc region was inlined properly.
 // CHECK: %[[PRIV1_ALLOC:.*]] = alloca float, align 4
+// CHECK: %[[PRIV2_ALLOC:.*]] = alloca i32, align 4
 // CHECK: %[[ORIG1_VAL:.*]] = load float, ptr %[[ORIG1_PTR]], align 4
 // CHECK: store float %[[ORIG1_VAL]], ptr %[[PRIV1_ALLOC]], align 4
-// CHECK: %[[PRIV2_ALLOC:.*]] = alloca i32, align 4
 // CHECK: %[[ORIG2_VAL:.*]] = load i32, ptr %[[ORIG2_PTR]], align 4
 // CHECK: store i32 %[[ORIG2_VAL]], ptr %[[PRIV2_ALLOC]], align 4
 // CHECK-NEXT: br
@@ -72,22 +72,20 @@ llvm.func @parallel_op_2_privates(%arg0: !llvm.ptr, %arg1: !llvm.ptr) {
 // CHECK: load i32, ptr %[[PRIV2_ALLOC]], align 4
 // CHECK: }
 
-omp.private {type = private} @x.privatizer : !llvm.ptr alloc {
-^bb0(%arg0: !llvm.ptr):
+omp.private {type = private} @x.privatizer : f32 init {
+^bb0(%arg0: !llvm.ptr, %arg1: !llvm.ptr):
   %c1 = llvm.mlir.constant(1 : i32) : i32
-  %0 = llvm.alloca %c1 x f32 : (i32) -> !llvm.ptr
   %1 = llvm.load %arg0 : !llvm.ptr -> f32
-  llvm.store %1, %0 : f32, !llvm.ptr
-  omp.yield(%0 : !llvm.ptr)
+  llvm.store %1, %arg1 : f32, !llvm.ptr
+  omp.yield(%arg1 : !llvm.ptr)
 }
 
-omp.private {type = private} @y.privatizer : !llvm.ptr alloc {
-^bb0(%arg0: !llvm.ptr):
+omp.private {type = private} @y.privatizer : i32 init {
+^bb0(%arg0: !llvm.ptr, %arg1: !llvm.ptr):
   %c1 = llvm.mlir.constant(1 : i32) : i32
-  %0 = llvm.alloca %c1 x i32 : (i32) -> !llvm.ptr
   %1 = llvm.load %arg0 : !llvm.ptr -> i32
-  llvm.store %1, %0 : i32, !llvm.ptr
-  omp.yield(%0 : !llvm.ptr)
+  llvm.store %1, %arg1 : i32, !llvm.ptr
+  omp.yield(%arg1 : !llvm.ptr)
 }
 
 // -----
@@ -104,17 +102,20 @@ llvm.func @parallel_op_private_multi_block(%arg0: !llvm.ptr) {
 // CHECK: omp.par.entry:
 // CHECK:  %[[ORIG_PTR_PTR:.*]] = getelementptr { ptr }, ptr %{{.*}}, i32 0, i32 0
 // CHECK:  %[[ORIG_PTR:.*]] = load ptr, ptr %[[ORIG_PTR_PTR]], align 8
-// CHECK:  br label %omp.private.latealloc
+// CHECK:  %[[PRIV_ALLOC:.*]] = alloca float, align 4
+// CHECK-NEXT:   br label %[[PAR_REG:.*]]
 
-// CHECK: omp.private.latealloc:
+// CHECK: [[PAR_REG]]:
+// CHECK:  br label %omp.private.init
+
+// CHECK: omp.private.init:
 // CHECK:   br label %[[PRIV_BB1:.*]]
 
-// Check contents of the first block in the `alloc` region.
+// Check contents of the first block in the `init` region.
 // CHECK: [[PRIV_BB1]]:
-// CHECK-NEXT:   %[[PRIV_ALLOC:.*]] = alloca float, align 4
 // CHECK-NEXT:   br label %[[PRIV_BB2:.*]]
 
-// Check contents of the second block in the `alloc` region.
+// Check contents of the second block in the `init` region.
 // CHECK: [[PRIV_BB2]]:
 // CHECK-NEXT:   %[[ORIG_PTR2:.*]] = phi ptr [ %[[ORIG_PTR]], %[[PRIV_BB1]] ]
 // CHECK-NEXT:   %[[PRIV_ALLOC2:.*]] = phi ptr [ %[[PRIV_ALLOC]], %[[PRIV_BB1]] ]
@@ -126,22 +127,20 @@ llvm.func @parallel_op_private_multi_block(%arg0: !llvm.ptr) {
 // address.
 // CHECK: [[PRIV_CONT]]:
 // CHECK-NEXT:   %[[PRIV_ALLOC3:.*]] = phi ptr [ %[[PRIV_ALLOC2]], %[[PRIV_BB2]] ]
-// CHECK-NEXT:   br label %[[PAR_REG:.*]]
+// CHECK-NEXT:   br label %[[PAR_REG2:.*]]
 
 // Check that the body of the parallel region loads from the private clone.
-// CHECK: [[PAR_REG]]:
+// CHECK: [[PAR_REG2]]:
 // CHECK:        %{{.*}} = load float, ptr %[[PRIV_ALLOC3]], align 4
 
-omp.private {type = private} @multi_block.privatizer : !llvm.ptr alloc {
-^bb0(%arg0: !llvm.ptr):
-  %c1 = llvm.mlir.constant(1 : i32) : i32
-  %0 = llvm.alloca %c1 x f32 : (i32) -> !llvm.ptr
-  llvm.br ^bb1(%arg0, %0 : !llvm.ptr, !llvm.ptr)
+omp.private {type = private} @multi_block.privatizer : f32 init {
+^bb0(%arg0: !llvm.ptr, %arg1: !llvm.ptr):
+  llvm.br ^bb1(%arg0, %arg1 : !llvm.ptr, !llvm.ptr)
 
-^bb1(%arg1: !llvm.ptr, %arg2: !llvm.ptr):
-  %1 = llvm.load %arg1 : !llvm.ptr -> f32
-  llvm.store %1, %arg2 : f32, !llvm.ptr
-  omp.yield(%arg2 : !llvm.ptr)
+^bb1(%arg2: !llvm.ptr, %arg3: !llvm.ptr):
+  %1 = llvm.load %arg2 : !llvm.ptr -> f32
+  llvm.store %1, %arg3 : f32, !llvm.ptr
+  omp.yield(%arg3 : !llvm.ptr)
 }
 
 // Tests fix for Fujitsu test suite test: 0007_0019.f90: the
@@ -174,8 +173,8 @@ llvm.func @lower_region_with_addressof() {
   llvm.return
 }
 
-omp.private {type = private} @_QFlower_region_with_addressof_privatizer : !llvm.ptr alloc {
-^bb0(%arg0: !llvm.ptr):
+omp.private {type = private} @_QFlower_region_with_addressof_privatizer : !llvm.ptr init {
+^bb0(%arg0: !llvm.ptr, %arg1: !llvm.ptr):
   %0 = llvm.mlir.addressof @_QQfoo: !llvm.ptr
   omp.yield(%0 : !llvm.ptr)
 }
@@ -195,7 +194,7 @@ llvm.func @bar(!llvm.ptr)
 // that we access the different sets of args properly.
 
 // CHECK-LABEL: define internal void @private_and_reduction_..omp_par
-// CHECK-DAG:    %[[PRV_ALLOC:.*]] = alloca float, i64 1, align 4
+// CHECK-DAG:    %[[PRV_ALLOC:.*]] = alloca float, align 4
 // CHECK-DAG:     %[[RED_ALLOC:.*]] = alloca { ptr, i64, i32, i8, i8, i8, i8, [1 x [3 x i64]] }, i64 1, align 8
 
 // CHECK:         omp.par.region:
@@ -219,12 +218,7 @@ llvm.func @private_and_reduction_() attributes {fir.internal_name = "_QPprivate_
   llvm.return
 }
 
-omp.private {type = private} @privatizer.part : !llvm.ptr alloc {
-^bb0(%arg0: !llvm.ptr):
-  %0 = llvm.mlir.constant(1 : i64) : i64
-  %1 = llvm.alloca %0 x f32 {bindc_name = "to_priv", pinned} : (i64) -> !llvm.ptr
-  omp.yield(%1 : !llvm.ptr)
-}
+omp.private {type = private} @privatizer.part : f32
 
 omp.declare_reduction @reducer.part : !llvm.ptr alloc {
   %0 = llvm.mlir.constant(1 : i64) : i64
@@ -256,12 +250,7 @@ llvm.func @_QPequivalence() {
   llvm.return
 }
 
-omp.private {type = firstprivate} @_QFequivalenceEx_firstprivate_ptr_f32 : !llvm.ptr alloc {
-^bb0(%arg0: !llvm.ptr):
-  %0 = llvm.mlir.constant(1 : i64) : i64
-  %1 = llvm.alloca %0 x f32 {bindc_name = "x", pinned} : (i64) -> !llvm.ptr
-  omp.yield(%1 : !llvm.ptr)
-} copy {
+omp.private {type = firstprivate} @_QFequivalenceEx_firstprivate_ptr_f32 : f32 copy {
 ^bb0(%arg0: !llvm.ptr, %arg1: !llvm.ptr):
   %0 = llvm.load %arg0 : !llvm.ptr -> f32
   llvm.store %0, %arg1 : f32, !llvm.ptr
@@ -270,7 +259,7 @@ omp.private {type = firstprivate} @_QFequivalenceEx_firstprivate_ptr_f32 : !llvm
 
 // CHECK: define internal void @_QPequivalence..omp_par
 // CHECK-NOT: define {{.*}} @{{.*}}
-// CHECK:   %[[PRIV_ALLOC:.*]] = alloca float, i64 1, align 4
+// CHECK:   %[[PRIV_ALLOC:.*]] = alloca float, align 4
 // CHECK:   %[[HOST_VAL:.*]] = load float, ptr %{{.*}}, align 4
 // Test that we initialize the firstprivate variable.
 // CHECK:   store float %[[HOST_VAL]], ptr %[[PRIV_ALLOC]], align 4
diff --git a/mlir/test/Target/LLVMIR/openmp-simd-private.mlir b/mlir/test/Target/LLVMIR/openmp-simd-private.mlir
index 61542aa1aa4d7..40f46103a0ab4 100644
--- a/mlir/test/Target/LLVMIR/openmp-simd-private.mlir
+++ b/mlir/test/Target/LLVMIR/openmp-simd-private.mlir
@@ -1,17 +1,12 @@
 // RUN: mlir-translate -mlir-to-llvmir -split-input-file %s | FileCheck %s
 
-omp.private {type = private} @i_privatizer : !llvm.ptr alloc {
-^bb0(%arg0: !llvm.ptr):
-  %0 = llvm.mlir.constant(1 : i64) : i64
-  %1 = llvm.alloca %0 x i32 {bindc_name = "i", pinned} : (i64) -> !llvm.ptr
-  omp.yield(%1 : !llvm.ptr)
-}
+omp.private {type = private} @i_privatizer : i32
 
 // CHECK-LABEL: test_loop_var_privatization()
 //                Original (non-privatized) allocation for `i`.
 // CHECK:         %{{.*}} = alloca i32, i64 1, align 4
 // CHECK:         %[[DUMMY:.*]] = alloca float, i64 1, align 4
-// CHECK:         %[[PRIV_I:.*]] = alloca i32, i64 1, align 4
+// CHECK:         %[[PRIV_I:.*]] = alloca i32, align 4
 // CHECK:         br label %[[LATE_ALLOC:.*]]
 
 // CHECK:     [[LATE_ALLOC]]:
@@ -78,20 +73,15 @@ llvm.func @test_loop_var_privatization() attributes {fir.internal_name = "_QPtes
   llvm.return
 }
 
-omp.private {type = private} @dummy_privatizer : !llvm.ptr alloc {
-^bb0(%arg0: !llvm.ptr):
-  %0 = llvm.mlir.constant(1 : i64) : i64
-  %1 = llvm.alloca %0 x f32 {bindc_name = "dummy", pinned} : (i64) -> !llvm.ptr
-  omp.yield(%1 : !llvm.ptr)
-}
+omp.private {type = private} @dummy_privatizer : f32
 
 // CHECK-LABEL: test_private_clause()
 //                Original (non-privatized) allocation for `i`.
 // CHECK:         %{{.*}} = alloca i32, i64 1, align 4
 //                Original (non-privatized) allocation for `dummy`.
 // CHECK:         %{{.*}} = alloca float, i64 1, align 4
-// CHECK:         %[[PRIV_DUMMY:.*]] = alloca float, i64 1, align 4
-// CHECK:         %[[PRIV_I:.*]] = alloca i32, i64 1, align 4
+// CHECK:         %[[PRIV_DUMMY:.*]] = alloca float, align 4
+// CHECK:         %[[PRIV_I:.*]] = alloca i32, align 4
 
 // CHECK:       omp.simd.region:
 // CHECK-NOT:     br label
diff --git a/mlir/test/Target/LLVMIR/openmp-target-multiple-private.mlir b/mlir/test/Target/LLVMIR/openmp-target-multiple-private.mlir
index c632a0ee42f8a..a47955cc28e15 100644
--- a/mlir/test/Target/LLVMIR/openmp-target-multiple-private.mlir
+++ b/mlir/test/Target/LLVMIR/openmp-target-multiple-private.mlir
@@ -2,12 +2,7 @@
 
 llvm.func @dealloc_foo_0(!llvm.ptr)
 
-omp.private {type = private} @box.heap_privatizer0 : !llvm.ptr alloc {
-^bb0(%arg0: !llvm.ptr):
-  %0 = llvm.mlir.constant(1 : i32) : i32
-  %7 = llvm.alloca %0 x !llvm.struct<(ptr, i64, i32, i8, i8, i8, i8)>  : (i32) -> !llvm.ptr
-  omp.yield(%7 : !llvm.ptr)
-} dealloc {
+omp.private {type = private} @box.heap_privatizer0 : !llvm.struct<(ptr, i64, i32, i8, i8, i8, i8)> dealloc {
 ^bb0(%arg0: !llvm.ptr):
   llvm.call @dealloc_foo_0(%arg0) : (!llvm.ptr) -> ()
   omp.yield
@@ -16,12 +11,10 @@ omp.private {type = private} @box.heap_privatizer0 : !llvm.ptr alloc {
 llvm.func @alloc_foo_1(!llvm.ptr)
 llvm.func @dealloc_foo_1(!llvm.ptr)
 
-omp.private {type = private} @box.heap_privatizer1 : !llvm.ptr alloc {
-^bb0(%arg0: !llvm.ptr):
-  %0 = llvm.mlir.constant(1 : i32) : i32
-  %7 = llvm.alloca %0 x !llvm.struct<(ptr, i64, i32, i8, i8, i8, i8)>  : (i32) -> !llvm.ptr
+omp.private {type = private} @box.heap_privatizer1 : !llvm.struct<(ptr, i64, i32, i8, i8, i8, i8)> init {
+^bb0(%arg0: !llvm.ptr, %arg1: !llvm.ptr):
   llvm.call @alloc_foo_1(%arg0) : (!llvm.ptr) -> ()
-  omp.yield(%7 : !llvm.ptr)
+  omp.yield(%arg1 : !llvm.ptr)
 } dealloc {
 ^bb0(%arg0: !llvm.ptr):
   llvm.call @dealloc_foo_1(%arg0) : (!llvm.ptr) -> ()
diff --git a/mlir/test/Target/LLVMIR/openmp-target-private-allocatable.mlir b/mlir/test/Target/LLVMIR/openmp-target-private-allocatable.mlir
index 88b4a6a63c7eb..a25f920287fe3 100644
--- a/mlir/test/Target/LLVMIR/openmp-target-private-allocatable.mlir
+++ b/mlir/test/Target/LLVMIR/openmp-target-private-allocatable.mlir
@@ -3,12 +3,10 @@
 llvm.func @alloc_foo_1(!llvm.ptr)
 llvm.func @dealloc_foo_1(!llvm.ptr)
 
-omp.private {type = private} @box.heap_privatizer : !llvm.ptr alloc {
-^bb0(%arg0: !llvm.ptr):
-  %0 = llvm.mlir.constant(1 : i32) : i32
-  %7 = llvm.alloca %0 x !llvm.struct<(ptr, i64, i32, i8, i8, i8, i8)>  : (i32) -> !llvm.ptr
+omp.private {type = private} @box.heap_privatizer : !llvm.struct<(ptr, i64, i32, i8, i8, i8, i8)> init {
+^bb0(%arg0: !llvm.ptr, %arg1 : !llvm.ptr):
   llvm.call @alloc_foo_1(%arg0) : (!llvm.ptr) -> ()
-  omp.yield(%7 : !llvm.ptr)
+  omp.yield(%arg1 : !llvm.ptr)
 } dealloc {
 ^bb0(%arg0: !llvm.ptr):
   llvm.call @dealloc_foo_1(%arg0) : (!llvm.ptr) -> ()
diff --git a/mlir/test/Target/LLVMIR/openmp-target-private.mlir b/mlir/test/Target/LLVMIR/openmp-target-private.mlir
index c9d5f37384a0b..f97360e2c6e84 100644
--- a/mlir/test/Target/LLVMIR/openmp-target-private.mlir
+++ b/mlir/test/Target/LLVMIR/openmp-target-private.mlir
@@ -1,11 +1,7 @@
 // RUN: mlir-translate -mlir-to-llvmir %s | FileCheck %s
 
-omp.private {type = private} @simple_var.privatizer : !llvm.ptr alloc {
-^bb0(%arg0: !llvm.ptr):
-  %0 = llvm.mlir.constant(1 : i64) : i64
-  %1 = llvm.alloca %0 x i32 {bindc_name = "simple_var", pinned} : (i64) -> !llvm.ptr
-  omp.yield(%1 : !llvm.ptr)
-}
+omp.private {type = private} @simple_var.privatizer : i32
+
 llvm.func @target_map_single_private() attributes {fir.internal_name = "_QPtarget_map_single_private"} {
   %0 = llvm.mlir.constant(1 : i64) : i64
   %1 = llvm.alloca %0 x i32 {bindc_name = "simple_var"} : (i64) -> !llvm.ptr
@@ -24,16 +20,12 @@ llvm.func @target_map_single_private() attributes {fir.internal_name = "_QPtarge
 }
 // CHECK: define internal void @__omp_offloading_
 // CHECK-NOT: define {{.*}}
-// CHECK: %[[PRIV_ALLOC:.*]] = alloca i32, i64 1, align 4
+// CHECK: %[[PRIV_ALLOC:.*]] = alloca i32, align 4
 // CHECK: %[[ADD:.*]] = add i32 {{.*}}, 10
 // CHECK: store i32 %[[ADD]], ptr %[[PRIV_ALLOC]], align 4
 
-omp.private {type = private} @n.privatizer : !llvm.ptr alloc {
-^bb0(%arg0: !llvm.ptr):
-  %0 = llvm.mlir.constant(1 : i64) : i64
-  %1 = llvm.alloca %0 x f32 {bindc_name = "n", pinned} : (i64) -> !llvm.ptr
-  omp.yield(%1 : !llvm.ptr)
-}
+omp.private {type = private} @n.privatizer : f32
+
 llvm.func @target_map_2_privates() attributes {fir.internal_name = "_QPtarget_map_2_privates"} {
   %0 = llvm.mlir.constant(1 : i64) : i64
   %1 = llvm.alloca %0 x i32 {bindc_name = "simple_var"} : (i64) -> !llvm.ptr
@@ -59,8 +51,8 @@ llvm.func @target_map_2_privates() attributes {fir.internal_name = "_QPtarget_ma
 
 
 // CHECK: define internal void @__omp_offloading_
-// CHECK: %[[PRIV_I32_ALLOC:.*]] = alloca i32, i64 1, align 4
-// CHECK: %[[PRIV_FLOAT_ALLOC:.*]] = alloca float, i64 1, align 4
+// CHECK: %[[PRIV_I32_ALLOC:.*]] = alloca i32, align 4
+// CHECK: %[[PRIV_FLOAT_ALLOC:.*]] = alloca float, align 4
 // CHECK: %[[ADD_I32:.*]] = add i32 {{.*}}, 10
 // CHECK: store i32 %[[ADD_I32]], ptr %[[PRIV_I32_ALLOC]], align 4
 // CHECK: %[[LOAD_I32_AGAIN:.*]] = load i32, ptr %[[PRIV_I32_ALLOC]], align 4
@@ -72,14 +64,12 @@ llvm.func @target_map_2_privates() attributes {fir.internal_name = "_QPtarget_ma
 // privatizers. The idea here is to prove that we set the correct
 // insertion points for the builder when generating, first, LLVM IR for the
 // privatizer and then for the actual target region.
-omp.private {type = private} @multi_block.privatizer : !llvm.ptr alloc {
-^bb0(%arg0: !llvm.ptr):
-  %c1 = llvm.mlir.constant(1 : i32) : i32
-  llvm.br ^bb1(%c1 : i32)
+omp.private {type = private} @multi_block.privatizer : f32 init {
+^bb0(%arg0: !llvm.ptr, %arg1: !llvm.ptr):
+  llvm.br ^bb1
 
-^bb1(%arg1: i32):
-  %0 = llvm.alloca %arg1 x f32 : (i32) -> !llvm.ptr
-  omp.yield(%0 : !llvm.ptr)
+^bb1:
+  omp.yield(%arg1 : !llvm.ptr)
 }
 
 llvm.func @target_op_private_multi_block(%arg0: !llvm.ptr) {
@@ -90,8 +80,7 @@ llvm.func @target_op_private_multi_block(%arg0: !llvm.ptr) {
   llvm.return
 }
 // CHECK: define internal void @__omp_offloading_
-// CHECK: %[[ONE:.*]] = phi i32 [ 1, {{.*}} ]
-// CHECK: %[[PRIV_ALLOC:.*]] = alloca float, i32 %[[ONE]], align 4
+// CHECK: %[[PRIV_ALLOC:.*]] = alloca float, align 4
 // CHECK: %[[PHI_ALLOCA:.*]]  = phi ptr [ %[[PRIV_ALLOC]], {{.*}} ]
 // CHECK: %[[RESULT:.*]] = load float, ptr %[[PHI_ALLOCA]], align 4
 
@@ -105,8 +94,8 @@ llvm.func @target_op_private_multi_block(%arg0: !llvm.ptr) {
 // mapped by its pointer whereas the privatizer function expects the descriptor
 // by value. So, we have this test to ensure that the compiler correctly loads
 // from the mapped pointer before passing that to the privatizer function.
-omp.private {type = private} @_QFtarget_boxcharEchar_var_private_boxchar_c8xU : !llvm.struct<(ptr, i64)> alloc {
-^bb0(%arg0: !llvm.struct<(ptr, i64)>):
+omp.private {type = private} @_QFtarget_boxcharEchar_var_private_boxchar_c8xU : !llvm.struct<(ptr, i64)> init {
+^bb0(%arg0: !llvm.struct<(ptr, i64)>, %arg1: !llvm.struct<(ptr, i64)>):
   %0 = llvm.extractvalue %arg0[0] : !llvm.struct<(ptr, i64)>
   %1 = llvm.extractvalue %arg0[1] : !llvm.struct<(ptr, i64)>
   %2 = llvm.mlir.constant(1 : i64) : i64
diff --git a/mlir/test/Target/LLVMIR/openmp-target-simd-on_device.mlir b/mlir/test/Target/LLVMIR/openmp-target-simd-on_device.mlir
index 0ce90578ea9d6..5c971206731e4 100644
--- a/mlir/test/Target/LLVMIR/openmp-target-simd-on_device.mlir
+++ b/mlir/test/Target/LLVMIR/openmp-target-simd-on_device.mlir
@@ -1,8 +1,8 @@
 // RUN: mlir-translate -mlir-to-llvmir %s | FileCheck %s
 
 module attributes {omp.is_target_device = true} {
-  omp.private {type = private} @simd_privatizer : !llvm.ptr alloc {
-  ^bb0(%arg0: !llvm.ptr):
+  omp.private {type = private} @simd_privatizer : !llvm.ptr init {
+  ^bb0(%arg0: !llvm.ptr, %arg1: !llvm.ptr):
     omp.yield(%arg0 : !llvm.ptr)
   }
 
diff --git a/mlir/test/Target/LLVMIR/openmp-target-wsloop-private.mlir b/mlir/test/Target/LLVMIR/openmp-target-wsloop-private.mlir
new file mode 100644
index 0000000000000..4b5bb2c3c99af
--- /dev/null
+++ b/mlir/test/Target/LLVMIR/openmp-target-wsloop-private.mlir
@@ -0,0 +1,41 @@
+// RUN: mlir-translate -mlir-to-llvmir %s | FileCheck %s
+
+
+// Tests inlining behavior of `omp.private` ops for `omp.wsloop` ops nested in
+// `omp.target` ops.
+
+llvm.func @foo(%arg0: !llvm.ptr)
+
+omp.private {type = private} @impure_alloca_privatizer : !llvm.ptr init {
+^bb0(%arg0: !llvm.ptr, %arg1: !llvm.ptr):
+  llvm.call @foo(%arg0) : (!llvm.ptr) -> ()
+  omp.yield(%arg1 : !llvm.ptr)
+}
+
+llvm.func @test_alloca_ip_workaround() {
+  omp.target {
+    %65 = llvm.mlir.constant(1 : i32) : i32
+    %66 = llvm.alloca %65 x !llvm.struct<(ptr, i64, i32, i8, i8, i8, i8, array<1 x array<3 x i64>>)> {alignment = 8 : i64} : (i32) -> !llvm.ptr
+    %67 = llvm.mlir.constant(0 : index) : i64
+    %68 = llvm.mlir.constant(10 : i32) : i32
+    %69 = llvm.mlir.constant(1 : i32) : i32
+    omp.wsloop private(@impure_alloca_privatizer %66 -> %arg6 : !llvm.ptr) {
+      omp.loop_nest (%arg8) : i32 = (%69) to (%68) inclusive step (%69) {
+        omp.yield
+      }
+    }
+    omp.terminator
+  }
+  llvm.return
+}
+
+// CHECK-LABEL: define {{.*}} @__omp_offloading_{{.*}}_test_alloca_ip_workaround
+// CHECK:       entry:
+// CHECK:         %[[PRIV_ALLOC:.*]] = alloca ptr, align 8
+
+// CHECK:       omp.target:
+// CHECK:         %[[ALLOC_REG_ARG:.*]] = alloca { ptr, i64, i32, i8, i8, i8, i8, [1 x [3 x i64]] }, align 8
+// Verify that the `init` region of the privatizer is inlined within the `target`
+// region body not before that.
+// CHECK:         call void @foo(ptr %[[ALLOC_REG_ARG]])
+// CHECK:         br label %omp_loop.preheader
diff --git a/mlir/test/Target/LLVMIR/openmp-todo.mlir b/mlir/test/Target/LLVMIR/openmp-todo.mlir
index 457bb87fae27a..e97b5e54e6415 100644
--- a/mlir/test/Target/LLVMIR/openmp-todo.mlir
+++ b/mlir/test/Target/LLVMIR/openmp-todo.mlir
@@ -112,11 +112,11 @@ llvm.func @sections_allocate(%x : !llvm.ptr) {
 
 // -----
 
-omp.private {type = private} @x.privatizer : !llvm.ptr alloc {
-^bb0(%arg0: !llvm.ptr):
-  %0 = llvm.mlir.constant(1 : i32) : i32
-  %1 = llvm.alloca %0 x i32 : (i32) -> !llvm.ptr
-  omp.yield(%1 : !llvm.ptr)
+omp.private {type = private} @x.privatizer : i32 init {
+^bb0(%mold: !llvm.ptr, %private: !llvm.ptr):
+  %c0 = llvm.mlir.constant(0 : i32) : i32
+  llvm.store %c0, %private : i32, !llvm.ptr
+  omp.yield(%private: !llvm.ptr)
 }
 llvm.func @sections_private(%x : !llvm.ptr) {
   // expected-error@below {{not yet implemented: Unhandled clause privatization in omp.sections operation}}
@@ -228,11 +228,11 @@ llvm.func @single_allocate(%x : !llvm.ptr) {
 
 // -----
 
-omp.private {type = private} @x.privatizer : !llvm.ptr alloc {
-^bb0(%arg0: !llvm.ptr):
-  %0 = llvm.mlir.constant(1 : i32) : i32
-  %1 = llvm.alloca %0 x i32 : (i32) -> !llvm.ptr
-  omp.yield(%1 : !llvm.ptr)
+omp.private {type = private} @x.privatizer : i32 init {
+^bb0(%mold: !llvm.ptr, %private: !llvm.ptr):
+  %c0 = llvm.mlir.constant(0 : i32) : i32
+  llvm.store %c0, %private : i32, !llvm.ptr
+  omp.yield(%private: !llvm.ptr)
 }
 llvm.func @single_private(%x : !llvm.ptr) {
   // expected-error@below {{not yet implemented: Unhandled clause privatization in omp.single operation}}
@@ -341,12 +341,11 @@ llvm.func @target_is_device_ptr(%x : !llvm.ptr) {
 
 // -----
 
-omp.private {type = firstprivate} @x.privatizer : !llvm.ptr alloc {
-^bb0(%arg0: !llvm.ptr):
-  omp.yield(%arg0 : !llvm.ptr)
-} copy {
-^bb0(%arg0: !llvm.ptr, %arg1: !llvm.ptr):
-  omp.yield(%arg0 : !llvm.ptr)
+omp.private {type = firstprivate} @x.privatizer : i32 copy {
+^bb0(%mold: !llvm.ptr, %private: !llvm.ptr):
+  %0 = llvm.load %mold : !llvm.ptr -> i32
+  llvm.store %0, %private : i32, !llvm.ptr
+  omp.yield(%private: !llvm.ptr)
 }
 llvm.func @target_firstprivate(%x : !llvm.ptr) {
   // expected-error@below {{not yet implemented: Unhandled clause firstprivate in omp.target operation}}
@@ -529,11 +528,11 @@ llvm.func @teams_allocate(%x : !llvm.ptr) {
 
 // -----
 
-omp.private {type = private} @x.privatizer : !llvm.ptr alloc {
-^bb0(%arg0: !llvm.ptr):
-  %0 = llvm.mlir.constant(1 : i32) : i32
-  %1 = llvm.alloca %0 x i32 : (i32) -> !llvm.ptr
-  omp.yield(%1 : !llvm.ptr)
+omp.private {type = private} @x.privatizer : i32 init {
+^bb0(%mold: !llvm.ptr, %private: !llvm.ptr):
+  %c0 = llvm.mlir.constant(0 : i32) : i32
+  llvm.store %c0, %private : i32, !llvm.ptr
+  omp.yield(%private: !llvm.ptr)
 }
 llvm.func @teams_private(%x : !llvm.ptr) {
   // expected-error@below {{not yet implemented: Unhandled clause privatization in omp.teams operation}}
diff --git a/mlir/test/Target/LLVMIR/openmp-wsloop-private-cond_br.mlir b/mlir/test/Target/LLVMIR/openmp-wsloop-private-cond_br.mlir
index 4393fafb62efa..ea7706365b761 100644
--- a/mlir/test/Target/LLVMIR/openmp-wsloop-private-cond_br.mlir
+++ b/mlir/test/Target/LLVMIR/openmp-wsloop-private-cond_br.mlir
@@ -3,12 +3,7 @@
 // tests firx for test-suite test: pr69183.f90. Makes sure we can handle inling
 // private ops when the alloca block ends with a conditional branch.
 
-omp.private {type = private} @_QFwsloop_privateEi_private_ref_i32 : !llvm.ptr alloc {
-^bb0(%arg0: !llvm.ptr):
-  %0 = llvm.mlir.constant(1 : i64) : i64
-  %1 = llvm.alloca %0 x i32 {bindc_name = "i", pinned} : (i64) -> !llvm.ptr
-  omp.yield(%1 : !llvm.ptr)
-}
+omp.private {type = private} @_QFwsloop_privateEi_private_ref_i32 : i64
 
 llvm.func @wsloop_private_(%arg0: !llvm.ptr {fir.bindc_name = "y"}) attributes {fir.internal_name = "_QPwsloop_private", frame_pointer = #llvm.framePointerKind<all>, target_cpu = "x86-64"} {
   %0 = llvm.mlir.constant(1 : i64) : i64
@@ -35,13 +30,10 @@ llvm.func @wsloop_private_(%arg0: !llvm.ptr {fir.bindc_name = "y"}) attributes {
 }
 
 // CHECK:   %[[INT:.*]] = alloca i32, i64 1, align 4
-// CHECK:   br label %[[LATE_ALLOC_BB:.*]]
-
-// CHECK: [[LATE_ALLOC_BB]]:
 // CHECK:   br label %[[AFTER_ALLOC_BB:.*]]
 
 // CHECK: [[AFTER_ALLOC_BB]]:
-// CHECK:   br i1 false, label %[[BB1:.*]], label %5[[BB2:.*]]
+// CHECK:   br i1 false, label %[[BB1:.*]], label %[[BB2:.*]]
 
 // CHECK: [[BB1]]:
 // CHECK:   br label %[[BB3:.*]]
@@ -50,5 +42,8 @@ llvm.func @wsloop_private_(%arg0: !llvm.ptr {fir.bindc_name = "y"}) attributes {
 // CHECK:   br label %[[BB3:.*]]
 
 // CHECK: [[BB3]]:
+// CHECK:   br label %[[OMP_PRIVATE_INIT:.*]]
+
+// CHECK: [[OMP_PRIVATE_INIT]]:
 // CHECK:   br label %omp_loop.preheader
 
diff --git a/mlir/test/Target/LLVMIR/openmp-wsloop-private.mlir b/mlir/test/Target/LLVMIR/openmp-wsloop-private.mlir
index db67bb5fae58b..23a0ae5713aa2 100644
--- a/mlir/test/Target/LLVMIR/openmp-wsloop-private.mlir
+++ b/mlir/test/Target/LLVMIR/openmp-wsloop-private.mlir
@@ -3,21 +3,11 @@
 // tests a wsloop private + firstprivate + reduction to make sure block structure
 // is handled properly.
 
-omp.private {type = private} @_QFwsloop_privateEi_private_ref_i32 : !llvm.ptr alloc {
-^bb0(%arg0: !llvm.ptr):
-  %0 = llvm.mlir.constant(1 : i64) : i64
-  %1 = llvm.alloca %0 x i32 {bindc_name = "i", pinned} : (i64) -> !llvm.ptr
-  omp.yield(%1 : !llvm.ptr)
-}
+omp.private {type = private} @_QFwsloop_privateEi_private_ref_i32 : i32
 
 llvm.func @foo_free(!llvm.ptr)
 
-omp.private {type = firstprivate} @_QFwsloop_privateEc_firstprivate_ref_c8 : !llvm.ptr alloc {
-^bb0(%arg0: !llvm.ptr):
-  %0 = llvm.mlir.constant(1 : i64) : i64
-  %1 = llvm.alloca %0 x !llvm.array<1 x i8> {bindc_name = "c", pinned} : (i64) -> !llvm.ptr
-  omp.yield(%1 : !llvm.ptr)
-} copy {
+omp.private {type = firstprivate} @_QFwsloop_privateEc_firstprivate_ref_c8 : !llvm.array<1 x i8> copy {
 ^bb0(%arg0: !llvm.ptr, %arg1: !llvm.ptr):
   %0 = llvm.load %arg0 : !llvm.ptr -> !llvm.array<1 x i8>
   llvm.store %0, %arg1 : !llvm.array<1 x i8>, !llvm.ptr
@@ -63,8 +53,8 @@ llvm.func @wsloop_private_(%arg0: !llvm.ptr {fir.bindc_name = "y"}) attributes {
 
 // First, check that all memory for privates and reductions is allocated.
 // CHECK: omp.par.entry:
-// CHECK:   %[[CHR:.*]] = alloca [1 x i8], i64 1, align 1
-// CHECK:   %[[INT:.*]] = alloca i32, i64 1, align 4
+// CHECK:   %[[CHR:.*]] = alloca [1 x i8], align 1
+// CHECK:   %[[INT:.*]] = alloca i32, align 4
 // CHECK:   %[[FLT:.*]] = alloca float, align 4
 // CHECK:   %[[RED_ARR:.*]] = alloca [1 x ptr], align 8
 // CHECK:   br label %[[LATE_ALLOC_BB:.*]]
diff --git a/mlir/test/Target/LLVMIR/rocdl.mlir b/mlir/test/Target/LLVMIR/rocdl.mlir
index b74edb6210683..eac28c57e2ab4 100644
--- a/mlir/test/Target/LLVMIR/rocdl.mlir
+++ b/mlir/test/Target/LLVMIR/rocdl.mlir
@@ -137,10 +137,10 @@ llvm.func @rocdl.readlane(%src0 : f32, %src1: f64, %src2: i32, %src3: vector<2 x
   llvm.return %0 : f32
 }
 
-llvm.func @rocdl.waitcnt() {
-  // CHECK-LABEL: rocdl.waitcnt
+llvm.func @rocdl.s.waitcnt() {
+  // CHECK-LABEL: rocdl.s.waitcnt
   // CHECK-NEXT: call void @llvm.amdgcn.s.waitcnt(i32 0)
-  rocdl.waitcnt 0
+  rocdl.s.waitcnt 0
   llvm.return
 }
 
@@ -398,6 +398,95 @@ llvm.func @rocdl.xdlops(%arg0 : f32, %arg1 : f32,
   llvm.return %r0 : vector<32 x f32>
 }
 
+llvm.func @rocdl.smfmac(%arg0 : i32,
+                   %arg1 : vector<4 x f16>,
+                   %arg2 : vector<8 x f16>,
+                   %arg3 : vector<4 x f32>,
+                   %arg4 : vector<16 x f32>,
+                   %arg5 : vector<4 x i16>,
+                   %arg6 : vector<8 x i16>,
+                   %arg7 : vector<2xi32>,
+                   %arg8 : vector<4xi32>,
+                   %arg9 : vector<16xi32>) -> vector<4 x f32> {
+  %csti32 = llvm.mlir.constant(42 : i32) : i32
+
+  // CHECK-LABEL: rocdl.smfmac
+
+  // CHECK: call <4 x float> @llvm.amdgcn.smfmac.f32.16x16x32.f16(<4 x half> %{{.*}}, <8 x half> %{{.*}}, <4 x float> %{{.*}}, i32 42, i32 42, i32 42)
+  %r0 = rocdl.smfmac.f32.16x16x32.f16 %arg1, %arg2, %arg3, %csti32, %csti32, %csti32 :
+                                (vector<4xf16>, vector<8xf16>, vector<4xf32>,
+                                 i32, i32, i32) -> vector<4xf32>
+
+  // CHECK: call <16 x float> @llvm.amdgcn.smfmac.f32.32x32x16.f16(<4 x half> %{{.*}}, <8 x half> %{{.*}}, <16 x float> %{{.*}}, i32 42, i32 42, i32 42)
+  %r1 = rocdl.smfmac.f32.32x32x16.f16 %arg1, %arg2, %arg4, %csti32, %csti32, %csti32 :
+                                (vector<4xf16>, vector<8xf16>, vector<16xf32>,
+                                 i32, i32, i32) -> vector<16xf32>
+
+  // CHECK: call <4 x float> @llvm.amdgcn.smfmac.f32.16x16x32.bf16(<4 x i16> %{{.*}}, <8 x i16> %{{.*}}, <4 x float> %{{.*}}, i32 42, i32 42, i32 42)
+  %r2 = rocdl.smfmac.f32.16x16x32.bf16 %arg5, %arg6, %arg3, %csti32, %csti32, %csti32 :
+                                (vector<4xi16>, vector<8xi16>, vector<4xf32>,
+                                 i32, i32, i32) -> vector<4xf32>
+
+  // CHECK: call <16 x float> @llvm.amdgcn.smfmac.f32.32x32x16.bf16(<4 x i16> %{{.*}}, <8 x i16> %{{.*}}, <16 x float> %{{.*}}, i32 42, i32 42, i32 42)
+  %r3 = rocdl.smfmac.f32.32x32x16.bf16 %arg5, %arg6, %arg4, %csti32, %csti32, %csti32 :
+                                (vector<4xi16>, vector<8xi16>, vector<16xf32>,
+                                 i32, i32, i32) -> vector<16xf32>
+
+  // CHECK: call <4 x i32> @llvm.amdgcn.smfmac.i32.16x16x64.i8(<2 x i32> %{{.*}}, <4 x i32> %{{.*}}, <4 x i32> %{{.*}}, i32 42, i32 42, i32 42)
+  %r4 = rocdl.smfmac.i32.16x16x64.i8 %arg7, %arg8, %arg8, %csti32, %csti32, %csti32 :
+                                (vector<2xi32>, vector<4xi32>, vector<4xi32>,
+                                 i32, i32, i32) -> vector<4xi32>
+
+  // CHECK: call <16 x i32> @llvm.amdgcn.smfmac.i32.32x32x32.i8(<2 x i32> %{{.*}}, <4 x i32> %{{.*}}, <16 x i32> %{{.*}}, i32 42, i32 42, i32 42)
+  %r5 = rocdl.smfmac.i32.32x32x32.i8 %arg7, %arg8, %arg9, %csti32, %csti32, %csti32 :
+                                (vector<2xi32>, vector<4xi32>, vector<16xi32>,
+                                 i32, i32, i32) -> vector<16xi32>
+
+  // CHECK: call <4 x float> @llvm.amdgcn.smfmac.f32.16x16x64.bf8.bf8(<2 x i32> %{{.*}}, <4 x i32> %{{.*}}, <4 x float> %{{.*}}, i32 42, i32 42, i32 42)
+  %r6 = rocdl.smfmac.f32.16x16x64.bf8.bf8 %arg7, %arg8, %arg3, %csti32, %csti32, %csti32 :
+                                (vector<2xi32>, vector<4xi32>, vector<4xf32>,
+                                 i32, i32, i32) -> vector<4xf32>
+
+  // CHECK: call <4 x float> @llvm.amdgcn.smfmac.f32.16x16x64.bf8.fp8(<2 x i32> %{{.*}}, <4 x i32> %{{.*}}, <4 x float> %{{.*}}, i32 42, i32 42, i32 42)
+  %r7 = rocdl.smfmac.f32.16x16x64.bf8.fp8 %arg7, %arg8, %arg3, %csti32, %csti32, %csti32 :
+                                (vector<2xi32>, vector<4xi32>, vector<4xf32>,
+                                 i32, i32, i32) -> vector<4xf32>
+
+  // CHECK: call <4 x float> @llvm.amdgcn.smfmac.f32.16x16x64.fp8.bf8(<2 x i32> %{{.*}}, <4 x i32> %{{.*}}, <4 x float> %{{.*}}, i32 42, i32 42, i32 42)
+  %r8 = rocdl.smfmac.f32.16x16x64.fp8.bf8 %arg7, %arg8, %arg3, %csti32, %csti32, %csti32 :
+                                (vector<2xi32>, vector<4xi32>, vector<4xf32>,
+                                 i32, i32, i32) -> vector<4xf32>
+
+  // CHECK: call <4 x float> @llvm.amdgcn.smfmac.f32.16x16x64.fp8.fp8(<2 x i32> %{{.*}}, <4 x i32> %{{.*}}, <4 x float> %{{.*}}, i32 42, i32 42, i32 42)
+  %r9 = rocdl.smfmac.f32.16x16x64.fp8.fp8 %arg7, %arg8, %arg3, %csti32, %csti32, %csti32 :
+                                (vector<2xi32>, vector<4xi32>, vector<4xf32>,
+                                 i32, i32, i32) -> vector<4xf32>
+
+  // CHECK: call <16 x float> @llvm.amdgcn.smfmac.f32.32x32x32.bf8.bf8(<2 x i32> %{{.*}}, <4 x i32> %{{.*}}, <16 x float> %{{.*}}, i32 42, i32 42, i32 42)
+  %r10 = rocdl.smfmac.f32.32x32x32.bf8.bf8 %arg7, %arg8, %arg4, %csti32, %csti32, %csti32 :
+                                (vector<2xi32>, vector<4xi32>, vector<16xf32>,
+                                 i32, i32, i32) -> vector<16xf32>
+
+  // CHECK: call <16 x float> @llvm.amdgcn.smfmac.f32.32x32x32.bf8.fp8(<2 x i32> %{{.*}}, <4 x i32> %{{.*}}, <16 x float> %{{.*}}, i32 42, i32 42, i32 42)
+  %r11 = rocdl.smfmac.f32.32x32x32.bf8.fp8 %arg7, %arg8, %arg4, %csti32, %csti32, %csti32 :
+                                (vector<2xi32>, vector<4xi32>, vector<16xf32>,
+                                 i32, i32, i32) -> vector<16xf32>
+
+  // CHECK: call <16 x float> @llvm.amdgcn.smfmac.f32.32x32x32.fp8.bf8(<2 x i32> %{{.*}}, <4 x i32> %{{.*}}, <16 x float> %{{.*}}, i32 42, i32 42, i32 42)
+  %r12 = rocdl.smfmac.f32.32x32x32.fp8.bf8 %arg7, %arg8, %arg4, %csti32, %csti32, %csti32 :
+                                (vector<2xi32>, vector<4xi32>, vector<16xf32>,
+                                 i32, i32, i32) -> vector<16xf32>
+
+
+  // CHECK: call <16 x float> @llvm.amdgcn.smfmac.f32.32x32x32.fp8.fp8(<2 x i32> %{{.*}}, <4 x i32> %{{.*}}, <16 x float> %{{.*}}, i32 42, i32 42, i32 42)
+  %r13 = rocdl.smfmac.f32.32x32x32.fp8.fp8 %arg7, %arg8, %arg4, %csti32, %csti32, %csti32 :
+                                (vector<2xi32>, vector<4xi32>, vector<16xf32>,
+                                 i32, i32, i32) -> vector<16xf32>
+
+  llvm.return %r0 : vector<4 x f32>
+}
+
+
 llvm.func @rocdl.mfma.scale.f32.32x32x64.f8f6f4(%arg0 : i32,
                    %arg1 : vector<16 x f32>, %arg2 : vector<8xi32>,
                    %arg3 : vector<6xi32>, %arg4 : vector<4xi32>) -> vector<16 x f32> {
@@ -913,24 +1002,51 @@ llvm.func @rocdl.raw.buffer.atomic.cmpswap(%rsrc : vector<4xi32>,
 llvm.func @rocdl_8bit_floats(%source: i32, %stoch: i32) -> i32 {
 // CHECK-LABEL: @rocdl_8bit_floats
 // CHECK: call float @llvm.amdgcn.cvt.f32.bf8(i32 %{{.+}}, i32 0)
+// CHECK: call float @llvm.amdgcn.cvt.scalef32.f32.bf8(i32 %{{.+}}, float 1.000000e+00, i32 0)
 // CHECK: call float @llvm.amdgcn.cvt.f32.fp8(i32 %{{.+}}, i32 0)
+// CHECK: call float @llvm.amdgcn.cvt.scalef32.f32.fp8(i32 %{{.+}}, float 1.000000e+00, i32 0)
 // CHECK: call i32 @llvm.amdgcn.cvt.pk.bf8.f32(float %{{.+}}, float %{{.+}}, i32 %{{.+}}, i1 false)
 // CHECK: call i32 @llvm.amdgcn.cvt.pk.fp8.f32(float %{{.+}}, float %{{.+}}, i32 %{{.+}}, i1 false)
 // CHECK: call i32 @llvm.amdgcn.cvt.sr.bf8.f32(float %{{.+}}, i32 %{{.+}}, i32 %{{.+}}, i32 2)
 // CHECK: call i32 @llvm.amdgcn.cvt.sr.fp8.f32(float %{{.+}}, i32 %{{.+}}, i32 %{{.+}}, i32 3)
+// CHECK: call i32 @llvm.amdgcn.cvt.scalef32.sr.fp8.f32(i32 %{{.+}}, float %{{.+}}, i32 %{{.+}}, float 1.000000e+00, i32 3)
+// CHECK: call i32 @llvm.amdgcn.cvt.sr.bf8.f32(float %{{.+}}, i32 %{{.+}}, i32 %{{.+}}, i32 3)
+// CHECK: call i32 @llvm.amdgcn.cvt.scalef32.sr.bf8.f32(i32 %{{.+}}, float %{{.+}}, i32 %{{.+}}, float 1.000000e+00, i32 3)
+// CHECK: call <2 x float> @llvm.amdgcn.cvt.scalef32.pk.f32.fp8(i32 %{{.+}}, float 1.000000e+00, i1 false)
+// CHECK: call <2 x float> @llvm.amdgcn.cvt.scalef32.pk.f32.bf8(i32 %{{.+}}, float 1.000000e+00, i1 false)
+
   %c0 = llvm.mlir.constant(0 : i32) : i32
   %c2 = llvm.mlir.constant(2 : i32) : i32
   %c3 = llvm.mlir.constant(3 : i32) : i32
+  %c4 = llvm.mlir.constant(1.0 : f32) : f32
   %false = llvm.mlir.constant(false) : i1
   %v1 = rocdl.cvt.f32.bf8 %source[%c0] : f32
+  %v1_scaled = rocdl.cvt.scalef32.f32.bf8 %source[%c0], %c4 : f32
   %v2 = rocdl.cvt.f32.fp8 %source[%c0] : f32
+  %v2_scaled = rocdl.cvt.scalef32.f32.fp8 %source[%c0], %c4 : f32
   %source2 = rocdl.cvt.pk.bf8.f32 %v1, %v2 -> %source[%false] : i32
   %source3 = rocdl.cvt.pk.fp8.f32 %v1, %v2 -> %source2[%false] : i32
   %source4 = rocdl.cvt.sr.bf8.f32 %v1, %stoch -> %source3[%c2] : i32
   %source5 = rocdl.cvt.sr.fp8.f32 %v2, %stoch -> %source4[%c3] : i32
+  %source5_scaled = rocdl.cvt.scalef32.sr.fp8.f32 %v2, %stoch, %c4 -> %source4[%c3] : i32
+  %source6 = rocdl.cvt.sr.bf8.f32 %v1, %stoch -> %source3[%c3] : i32
+  %source6_scaled  = rocdl.cvt.scalef32.sr.bf8.f32 %v2, %stoch, %c4 -> %source3[%c3] : i32
+  %source7_scaled = rocdl.cvt.scalef32.pk.f32.fp8 %source[%false], %c4 : f32
+  %source8_scaled = rocdl.cvt.scalef32.pk.f32.bf8 %source[%false], %c4 : f32
   llvm.return %source5 : i32
 }
 
+llvm.func @rocdl_8bit_packed_v2i16(%sourceA: f32, %sourceB: f32, %old: vector<2xi16>) -> vector<2xi16> {
+// CHECK-LABEL:  @rocdl_8bit_packed_v2i16
+// CHECK: call <2 x i16> @llvm.amdgcn.cvt.scalef32.pk.fp8.f32(<2 x i16> %{{.+}}, float %{{.+}}, float %{{.+}}, float 1.000000e+00, i1 false)
+// CHECK: call <2 x i16> @llvm.amdgcn.cvt.scalef32.pk.bf8.f32(<2 x i16> %{{.+}}, float %{{.+}}, float %{{.+}}, float 1.000000e+00, i1 false)
+  %c0 = llvm.mlir.constant(1.0 : f32) : f32
+  %false = llvm.mlir.constant(false) : i1
+  %source_scaled = rocdl.cvt.scalef32.pk.fp8.f32 %sourceA, %sourceB, %c0 -> %old[%false] : vector<2xi16>
+  %source2_scaled = rocdl.cvt.scalef32.pk.bf8.f32 %sourceA, %sourceB, %c0 -> %old[%false] : vector<2xi16>
+  llvm.return %source_scaled : vector<2xi16>
+}
+
 llvm.func @rocdl_16bit_packed_floats(%sourceA: f32, %sourceB: f32) -> vector<2xf16> {
   // CHECK-LABEL: @rocdl_16bit_packed_floats
   // CHECK: call <2 x half> @llvm.amdgcn.cvt.pkrtz(float {{.*}}, float {{.*}})
diff --git a/mlir/test/Target/LLVMIR/x86vector.mlir b/mlir/test/Target/LLVMIR/x86vector.mlir
index 190732868cb7a..1df03f10c9321 100644
--- a/mlir/test/Target/LLVMIR/x86vector.mlir
+++ b/mlir/test/Target/LLVMIR/x86vector.mlir
@@ -60,6 +60,39 @@ llvm.func @LLVM_x86_vp2intersect_q_512(%a: vector<8xi64>, %b: vector<8xi64>)
   llvm.return %0 : !llvm.struct<(vector<8 x i1>, vector<8 x i1>)>
 }
 
+// CHECK-LABEL: define <4 x float> @LLVM_x86_avx512bf16_dpbf16ps_128
+llvm.func @LLVM_x86_avx512bf16_dpbf16ps_128(
+    %arg0: vector<4xf32>, %arg1: vector<8xbf16>, %arg2: vector<8xbf16>
+  ) -> vector<4xf32>
+{
+  // CHECK: call <4 x float> @llvm.x86.avx512bf16.dpbf16ps.128(
+  %0 = "x86vector.avx512.intr.dpbf16ps.128"(%arg0, %arg1, %arg2)
+    : (vector<4xf32>, vector<8xbf16>, vector<8xbf16>) -> vector<4xf32>
+  llvm.return %0 : vector<4xf32>
+}
+
+// CHECK-LABEL: define <8 x float> @LLVM_x86_avx512bf16_dpbf16ps_256
+llvm.func @LLVM_x86_avx512bf16_dpbf16ps_256(
+    %arg0: vector<8xf32>, %arg1: vector<16xbf16>, %arg2: vector<16xbf16>
+  ) -> vector<8xf32>
+{
+  // CHECK: call <8 x float> @llvm.x86.avx512bf16.dpbf16ps.256(
+  %0 = "x86vector.avx512.intr.dpbf16ps.256"(%arg0, %arg1, %arg2)
+    : (vector<8xf32>, vector<16xbf16>, vector<16xbf16>) -> vector<8xf32>
+  llvm.return %0 : vector<8xf32>
+}
+
+// CHECK-LABEL: define <16 x float> @LLVM_x86_avx512bf16_dpbf16ps_512
+llvm.func @LLVM_x86_avx512bf16_dpbf16ps_512(
+    %arg0: vector<16xf32>, %arg1: vector<32xbf16>, %arg2: vector<32xbf16>
+  ) -> vector<16xf32>
+{
+  // CHECK: call <16 x float> @llvm.x86.avx512bf16.dpbf16ps.512(
+  %0 = "x86vector.avx512.intr.dpbf16ps.512"(%arg0, %arg1, %arg2)
+    : (vector<16xf32>, vector<32xbf16>, vector<32xbf16>) -> vector<16xf32>
+  llvm.return %0 : vector<16xf32>
+}
+
 // CHECK-LABEL: define <8 x float> @LLVM_x86_avx_rsqrt_ps_256
 llvm.func @LLVM_x86_avx_rsqrt_ps_256(%a: vector <8xf32>) -> vector<8xf32>
 {
@@ -67,3 +100,14 @@ llvm.func @LLVM_x86_avx_rsqrt_ps_256(%a: vector <8xf32>) -> vector<8xf32>
   %0 = "x86vector.avx.intr.rsqrt.ps.256"(%a) : (vector<8xf32>) -> (vector<8xf32>)
   llvm.return %0 : vector<8xf32>
 }
+
+// CHECK-LABEL: define <8 x float> @LLVM_x86_avx_dp_ps_256
+llvm.func @LLVM_x86_avx_dp_ps_256(
+    %arg0: vector<8xf32>, %arg1: vector<8xf32>
+  ) -> vector<8xf32>
+{
+  // CHECK: call <8 x float> @llvm.x86.avx.dp.ps.256(
+  %0 = llvm.mlir.constant(-1 : i8) : i8
+  %1 = "x86vector.avx.intr.dp.ps.256"(%arg0, %arg1, %0) : (vector<8xf32>, vector<8xf32>, i8) -> vector<8xf32>
+  llvm.return %1 : vector<8xf32>
+}
diff --git a/mlir/test/Target/SPIRV/group-ops.mlir b/mlir/test/Target/SPIRV/group-ops.mlir
index 32da4d9c26bd1..2ba7f23258e7f 100644
--- a/mlir/test/Target/SPIRV/group-ops.mlir
+++ b/mlir/test/Target/SPIRV/group-ops.mlir
@@ -103,5 +103,15 @@ spirv.module Logical GLSL450 requires #spirv.vce<v1.0, [Shader], []> {
     %0 = spirv.KHR.GroupFMul <Workgroup> <Reduce> %value : f32
     spirv.ReturnValue %0: f32
   }
+}
+
+// -----
 
+spirv.module Logical GLSL450 requires #spirv.vce<v1.3, [Shader, GroupNonUniformBallot, Linkage], []> {
+  // CHECK-LABEL: @group_non_uniform_ballot_bit_count
+  spirv.func @group_non_uniform_ballot_bit_count(%value: vector<4xi32>) -> i32 "None" {
+    // CHECK: spirv.GroupNonUniformBallotBitCount <Subgroup> <Reduce> {{%.*}} : vector<4xi32> -> i32
+    %0 = spirv.GroupNonUniformBallotBitCount <Subgroup> <Reduce> %value : vector<4xi32> -> i32
+    spirv.ReturnValue %0 : i32
+  }
 }
diff --git a/mlir/test/Target/SPIRV/image-ops.mlir b/mlir/test/Target/SPIRV/image-ops.mlir
index 92429fc8023d2..6b52a84ba82f7 100644
--- a/mlir/test/Target/SPIRV/image-ops.mlir
+++ b/mlir/test/Target/SPIRV/image-ops.mlir
@@ -1,6 +1,6 @@
-// RUN: mlir-translate -no-implicit-module -test-spirv-roundtrip %s | FileCheck %s
+// RUN: mlir-translate --no-implicit-module --split-input-file --test-spirv-roundtrip %s | FileCheck %s
 
-spirv.module Logical GLSL450 requires #spirv.vce<v1.0, [Shader], []> {
+spirv.module Logical GLSL450 requires #spirv.vce<v1.0, [Shader, ImageQuery], []> {
   spirv.func @image(%arg0 : !spirv.sampled_image<!spirv.image<f32, Dim2D, NoDepth, NonArrayed, SingleSampled, NeedSampler, Unknown>>, %arg1 : vector<4xf32>, %arg2 : f32) "None" {
     // CHECK: {{%.*}} = spirv.Image {{%.*}} : !spirv.sampled_image<!spirv.image<f32, Dim2D, NoDepth, NonArrayed, SingleSampled, NeedSampler, Unknown>>
     %0 = spirv.Image %arg0 : !spirv.sampled_image<!spirv.image<f32, Dim2D, NoDepth, NonArrayed, SingleSampled, NeedSampler, Unknown>>
@@ -13,4 +13,27 @@ spirv.module Logical GLSL450 requires #spirv.vce<v1.0, [Shader], []> {
     %0 = spirv.ImageQuerySize %arg0 : !spirv.image<f32, Dim2D, NoDepth, NonArrayed, SingleSampled, NoSampler, Unknown> -> vector<2xi32>
     spirv.Return
   }
+  spirv.func @image_write(%arg0 : !spirv.image<f32, Dim2D, NoDepth, NonArrayed, SingleSampled, NoSampler, Rgba8>, %arg1 : vector<2xsi32>, %arg2 : vector<4xf32>) "None" {
+    // CHECK:  spirv.ImageWrite %arg0 : !spirv.image<f32, Dim2D, NoDepth, NonArrayed, SingleSampled, NoSampler, Rgba8>, %arg1 : vector<2xsi32>, %arg2 : vector<4xf32>
+    spirv.ImageWrite %arg0 : !spirv.image<f32, Dim2D, NoDepth, NonArrayed, SingleSampled, NoSampler, Rgba8>, %arg1 : vector<2xsi32>, %arg2 : vector<4xf32>
+    spirv.Return
+  }
+  spirv.func @main() "None" {
+    spirv.Return
+  }
+  spirv.EntryPoint "GLCompute" @main
+}
+
+// -----
+
+spirv.module Logical GLSL450 requires #spirv.vce<v1.0, [Shader, StorageImageWriteWithoutFormat], []> {
+  spirv.func @image_write(%arg0 : !spirv.image<f32, Dim2D, NoDepth, NonArrayed, SingleSampled, NoSampler, Unknown>, %arg1 : vector<2xsi32>, %arg2 : vector<4xf32>) "None" {
+    // CHECK:  spirv.ImageWrite %arg0 : !spirv.image<f32, Dim2D, NoDepth, NonArrayed, SingleSampled, NoSampler, Unknown>, %arg1 : vector<2xsi32>, %arg2 : vector<4xf32>
+    spirv.ImageWrite %arg0 : !spirv.image<f32, Dim2D, NoDepth, NonArrayed, SingleSampled, NoSampler, Unknown>, %arg1 : vector<2xsi32>, %arg2 : vector<4xf32>
+    spirv.Return
+  }
+  spirv.func @main() "None" {
+    spirv.Return
+  }
+  spirv.EntryPoint "GLCompute" @main
 }
diff --git a/mlir/test/Target/SPIRV/matrix.mlir b/mlir/test/Target/SPIRV/matrix.mlir
index 0ec1dc27e4e93..452f8fc16f258 100644
--- a/mlir/test/Target/SPIRV/matrix.mlir
+++ b/mlir/test/Target/SPIRV/matrix.mlir
@@ -42,6 +42,13 @@ spirv.module Logical GLSL450 requires #spirv.vce<v1.0, [Shader], []> {
     %result = spirv.MatrixTimesVector %arg0, %arg1 : !spirv.matrix<3 x vector<4xf32>>, vector<3xf32> -> vector<4xf32>
     spirv.ReturnValue %result : vector<4xf32>
   }
+
+  // CHECK-LABEL: @vector_times_matrix_1
+  spirv.func @vector_times_matrix_1(%arg0: vector<3xf32>, %arg1: !spirv.matrix<4 x vector<3xf32>>) -> vector<4xf32> "None" {
+    // CHECK: {{%.*}} = spirv.VectorTimesMatrix {{%.*}}, {{%.*}} : vector<3xf32>, !spirv.matrix<4 x vector<3xf32>> -> vector<4xf32>
+    %result = spirv.VectorTimesMatrix %arg0, %arg1 : vector<3xf32>, !spirv.matrix<4 x vector<3xf32>> -> vector<4xf32>
+    spirv.ReturnValue %result : vector<4xf32>
+  }
   
   // CHECK-LABEL: @matrix_times_matrix_1
   spirv.func @matrix_times_matrix_1(%arg0: !spirv.matrix<3 x vector<3xf32>>, %arg1: !spirv.matrix<3 x vector<3xf32>>) -> !spirv.matrix<3 x vector<3xf32>> "None"{
diff --git a/mlir/test/Transforms/print-op-graph-back-edges.mlir b/mlir/test/Transforms/print-op-graph-back-edges.mlir
index ed922dd7cb13b..7950125e2f735 100644
--- a/mlir/test/Transforms/print-op-graph-back-edges.mlir
+++ b/mlir/test/Transforms/print-op-graph-back-edges.mlir
@@ -1,21 +1,21 @@
 // RUN: mlir-opt -view-op-graph %s -o %t 2>&1 | FileCheck -check-prefix=DFG %s
 
 // DFG-LABEL: digraph G {
-//       DFG:   compound = true;
-//       DFG:   subgraph cluster_1 {
-//       DFG:     v2 [label = " ", shape = plain];
-//       DFG:     label = "builtin.module : ()\n";
-//       DFG:     subgraph cluster_3 {
-//       DFG:       v4 [label = " ", shape = plain];
-//       DFG:       label = "";
-//       DFG:       v5 [fillcolor = "0.000000 1.0 1.0", label = "arith.addi : (index)\n\noverflowFlags: #arith.overflow<none...", shape = ellipse, style = filled];
-//       DFG:       v6 [fillcolor = "0.333333 1.0 1.0", label = "arith.constant : (index)\n\nvalue: 0 : index", shape = ellipse, style = filled];
-//       DFG:       v7 [fillcolor = "0.333333 1.0 1.0", label = "arith.constant : (index)\n\nvalue: 1 : index", shape = ellipse, style = filled];
-//       DFG:     }
-//       DFG:   }
-//       DFG:   v6 -> v5 [label = "0", style = solid];
-//       DFG:   v7 -> v5 [label = "1", style = solid];
-//       DFG: }
+//  DFG-NEXT:   compound = true;
+//  DFG-NEXT:   subgraph cluster_1 {
+//  DFG-NEXT:     v2 [label = " ", shape = plain];
+//  DFG-NEXT:     label = "builtin.module : ()\l";
+//  DFG-NEXT:     subgraph cluster_3 {
+//  DFG-NEXT:       v4 [label = " ", shape = plain];
+//  DFG-NEXT:       label = "";
+//  DFG-NEXT:       v5 [fillcolor = "0.000000 0.3 0.95", label = "{{\{\{}}<arg_c0> %c0|<arg_c1> %c1}|arith.addi\l\loverflowFlags: #arith.overflow\<none...\l|{<res_0> %0 index}}", shape = Mrecord, style = filled];
+//  DFG-NEXT:       v6 [fillcolor = "0.333333 0.3 0.95", label = "{arith.constant\l\lvalue: 0 : index\l|{<res_c0> %c0 index}}", shape = Mrecord, style = filled];
+//  DFG-NEXT:       v7 [fillcolor = "0.333333 0.3 0.95", label = "{arith.constant\l\lvalue: 1 : index\l|{<res_c1> %c1 index}}", shape = Mrecord, style = filled];
+//  DFG-NEXT:     }
+//  DFG-NEXT:   }
+//  DFG-NEXT:   v6:res_c0:s -> v5:arg_c0:n[style = solid];
+//  DFG-NEXT:   v7:res_c1:s -> v5:arg_c1:n[style = solid];
+//  DFG-NEXT: }
 
 module {
   %add = arith.addi %c0, %c1 : index
diff --git a/mlir/test/Transforms/print-op-graph-cycles.mlir b/mlir/test/Transforms/print-op-graph-cycles.mlir
index 7e4eb5616a28b..ba989544419f3 100644
--- a/mlir/test/Transforms/print-op-graph-cycles.mlir
+++ b/mlir/test/Transforms/print-op-graph-cycles.mlir
@@ -1,45 +1,45 @@
 // RUN: mlir-opt -view-op-graph -allow-unregistered-dialect %s -o %t 2>&1 | FileCheck -check-prefix=DFG %s
 
 // DFG-LABEL: digraph G {
-//       DFG:   compound = true;
-//       DFG:   subgraph cluster_1 {
-//       DFG:     v2 [label = " ", shape = plain];
-//       DFG:     label = "builtin.module : ()\n";
-//       DFG:     subgraph cluster_3 {
-//       DFG:       v4 [label = " ", shape = plain];
-//       DFG:       label = "";
-//       DFG:       subgraph cluster_5 {
-//       DFG:         v6 [label = " ", shape = plain];
-//       DFG:         label = "test.graph_region : ()\n";
-//       DFG:         subgraph cluster_7 {
-//       DFG:           v8 [label = " ", shape = plain];
-//       DFG:           label = "";
-//       DFG:           v9 [fillcolor = "0.000000 1.0 1.0", label = "op1 : (i32)\n", shape = ellipse, style = filled];
-//       DFG:           subgraph cluster_10 {
-//       DFG:             v11 [label = " ", shape = plain];
-//       DFG:             label = "test.ssacfg_region : (i32)\n";
-//       DFG:             subgraph cluster_12 {
-//       DFG:               v13 [label = " ", shape = plain];
-//       DFG:               label = "";
-//       DFG:               v14 [fillcolor = "0.166667 1.0 1.0", label = "op2 : (i32)\n", shape = ellipse, style = filled];
-//       DFG:             }
-//       DFG:           }
-//       DFG:           v15 [fillcolor = "0.166667 1.0 1.0", label = "op2 : (i32)\n", shape = ellipse, style = filled];
-//       DFG:           v16 [fillcolor = "0.500000 1.0 1.0", label = "op3 : (i32)\n", shape = ellipse, style = filled];
-//       DFG:         }
-//       DFG:       }
-//       DFG:     }
-//       DFG:   }
-//       DFG:   v9 -> v9 [label = "0", style = solid];
-//       DFG:   v15 -> v9 [label = "1", style = solid];
-//       DFG:   v9 -> v14 [label = "0", style = solid];
-//       DFG:   v11 -> v14 [ltail = cluster_10, style = solid];
-//       DFG:   v15 -> v14 [label = "2", style = solid];
-//       DFG:   v16 -> v14 [label = "3", style = solid];
-//       DFG:   v9 -> v15 [label = "0", style = solid];
-//       DFG:   v16 -> v15 [label = "1", style = solid];
-//       DFG:   v9 -> v16 [label = "", style = solid];
-//       DFG: }
+//  DFG-NEXT:   compound = true;
+//  DFG-NEXT:   subgraph cluster_1 {
+//  DFG-NEXT:     v2 [label = " ", shape = plain];
+//  DFG-NEXT:     label = "builtin.module : ()\l";
+//  DFG-NEXT:     subgraph cluster_3 {
+//  DFG-NEXT:       v4 [label = " ", shape = plain];
+//  DFG-NEXT:       label = "";
+//  DFG-NEXT:       subgraph cluster_5 {
+//  DFG-NEXT:         v6 [label = " ", shape = plain];
+//  DFG-NEXT:         label = "test.graph_region : ()\l";
+//  DFG-NEXT:         subgraph cluster_7 {
+//  DFG-NEXT:           v8 [label = " ", shape = plain];
+//  DFG-NEXT:           label = "";
+//  DFG-NEXT:           v9 [fillcolor = "0.000000 0.3 0.95", label = "{{\{\{}}<arg_0> %0|<arg_2> %2}|op1\l|{<res_0> %0 i32}}", shape = Mrecord, style = filled];
+//  DFG-NEXT:           subgraph cluster_10 {
+//  DFG-NEXT:             v11 [label = " ", shape = plain];
+//  DFG-NEXT:             label = "test.ssacfg_region : (i32)\l";
+//  DFG-NEXT:             subgraph cluster_12 {
+//  DFG-NEXT:               v13 [label = " ", shape = plain];
+//  DFG-NEXT:               label = "";
+//  DFG-NEXT:               v14 [fillcolor = "0.166667 0.3 0.95", label = "{{\{\{}}<arg_0> %0|<arg_1> %1|<arg_2> %2|<arg_3> %3}|op2\l|{<res_4> %4 i32}}", shape = Mrecord, style = filled];
+//  DFG-NEXT:             }
+//  DFG-NEXT:           }
+//  DFG-NEXT:           v15 [fillcolor = "0.166667 0.3 0.95", label = "{{\{\{}}<arg_0> %0|<arg_3> %3}|op2\l|{<res_2> %2 i32}}", shape = Mrecord, style = filled];
+//  DFG-NEXT:           v16 [fillcolor = "0.500000 0.3 0.95", label = "{{\{\{}}<arg_0> %0}|op3\l|{<res_3> %3 i32}}", shape = Mrecord, style = filled];
+//  DFG-NEXT:         }
+//  DFG-NEXT:       }
+//  DFG-NEXT:     }
+//  DFG-NEXT:   }
+//  DFG-NEXT:   v9:res_0:s -> v9:arg_0:n[style = solid];
+//  DFG-NEXT:   v15:res_2:s -> v9:arg_2:n[style = solid];
+//  DFG-NEXT:   v9:res_0:s -> v14:arg_0:n[style = solid];
+//  DFG-NEXT:   v11 -> v14:arg_1:n[ltail = cluster_10, style = solid];
+//  DFG-NEXT:   v15:res_2:s -> v14:arg_2:n[style = solid];
+//  DFG-NEXT:   v16:res_3:s -> v14:arg_3:n[style = solid];
+//  DFG-NEXT:   v9:res_0:s -> v15:arg_0:n[style = solid];
+//  DFG-NEXT:   v16:res_3:s -> v15:arg_3:n[style = solid];
+//  DFG-NEXT:   v9:res_0:s -> v16:arg_0:n[style = solid];
+//  DFG-NEXT: }
 
 "test.graph_region"() ({ // A Graph region
   %1 = "op1"(%1, %3) : (i32, i32) -> (i32)  // OK: %1, %3 allowed here
diff --git a/mlir/test/Transforms/print-op-graph.mlir b/mlir/test/Transforms/print-op-graph.mlir
index df03194a663d9..440b037d78092 100644
--- a/mlir/test/Transforms/print-op-graph.mlir
+++ b/mlir/test/Transforms/print-op-graph.mlir
@@ -6,49 +6,49 @@
 //       DFG:     subgraph {{.*}}
 //       DFG:       label = "func.func{{.*}}merge_blocks
 //       DFG:       subgraph {{.*}} {
-//       DFG:         v[[ARG0:.*]] [label = "arg0"
+//       DFG:         v[[ARG0:.*]] [label = "<res_arg0> %arg0 i32"
 //       DFG:         v[[CONST10:.*]] [{{.*}}label ={{.*}}10 : i32
 //       DFG:         subgraph [[CLUSTER_MERGE_BLOCKS:.*]] {
 //       DFG:           v[[ANCHOR:.*]] [label = " ", shape = plain]
 //       DFG:           label = "test.merge_blocks
 //       DFG:           subgraph {{.*}} {
-//       DFG:             v[[TEST_BR:.*]] [{{.*}}label = "test.br
+//       DFG:             v[[TEST_BR:.*]] [{{.*}}label = "{{.*}}test.br
 //       DFG:           }
 //       DFG:           subgraph {{.*}} {
 //       DFG:           }
 //       DFG:         }
-//       DFG:         v[[TEST_RET:.*]] [{{.*}}label = "test.return
-//       DFG:   v[[ARG0]] -> v[[TEST_BR]]
-//       DFG:   v[[CONST10]] -> v[[TEST_BR]]
-//       DFG:   v[[ANCHOR]] -> v[[TEST_RET]] [ltail = [[CLUSTER_MERGE_BLOCKS]], style = solid];
-//       DFG:   v[[ANCHOR]] -> v[[TEST_RET]] [ltail = [[CLUSTER_MERGE_BLOCKS]], style = solid];
+//       DFG:         v[[TEST_RET:.*]] [{{.*}}label = "{{.*}}test.return
+//       DFG:   v[[ARG0]]:res_arg0:s -> v[[TEST_BR]]:arg_arg0:n
+//       DFG:   v[[CONST10]]:res_c10_i32:s -> v[[TEST_BR]]
+//       DFG:   v[[ANCHOR]] -> v[[TEST_RET]]:arg_1_0:n[ltail = [[CLUSTER_MERGE_BLOCKS]], style = solid];
+//       DFG:   v[[ANCHOR]] -> v[[TEST_RET]]:arg_1_1:n[ltail = [[CLUSTER_MERGE_BLOCKS]], style = solid];
 
 // CFG-LABEL: digraph G {
 //       CFG:   subgraph {{.*}} {
 //       CFG:     subgraph {{.*}}
 //       CFG:       label = "func.func{{.*}}merge_blocks
 //       CFG:       subgraph {{.*}} {
-//       CFG:         v[[C1:.*]] [{{.*}}label = "arith.constant
-//       CFG:         v[[C2:.*]] [{{.*}}label = "arith.constant
-//       CFG:         v[[C3:.*]] [{{.*}}label = "arith.constant
-//       CFG:         v[[C4:.*]] [{{.*}}label = "arith.constant
-//       CFG:         v[[TEST_FUNC:.*]] [{{.*}}label = "test.func
+//       CFG:         v[[C1:.*]] [{{.*}}label = "{arith.constant
+//       CFG:         v[[C2:.*]] [{{.*}}label = "{arith.constant
+//       CFG:         v[[C3:.*]] [{{.*}}label = "{arith.constant
+//       CFG:         v[[C4:.*]] [{{.*}}label = "{arith.constant
+//       CFG:         v[[TEST_FUNC:.*]] [{{.*}}label = "{test.func
 //       CFG:         subgraph [[CLUSTER_MERGE_BLOCKS:.*]] {
 //       CFG:           v[[ANCHOR:.*]] [label = " ", shape = plain]
 //       CFG:           label = "test.merge_blocks
 //       CFG:           subgraph {{.*}} {
-//       CFG:             v[[TEST_BR:.*]] [{{.*}}label = "test.br
+//       CFG:             v[[TEST_BR:.*]] [{{.*}}label = "{{.*}}test.br
 //       CFG:           }
 //       CFG:           subgraph {{.*}} {
 //       CFG:           }
 //       CFG:         }
-//       CFG:         v[[TEST_RET:.*]] [{{.*}}label = "test.return
+//       CFG:         v[[TEST_RET:.*]] [{{.*}}label = "{{.*}}test.return
 //       CFG:   v[[C1]] -> v[[C2]]
 //       CFG:   v[[C2]] -> v[[C3]]
 //       CFG:   v[[C3]] -> v[[C4]]
 //       CFG:   v[[C4]] -> v[[TEST_FUNC]]
-//       CFG:   v[[TEST_FUNC]] -> v[[ANCHOR]] [lhead = [[CLUSTER_MERGE_BLOCKS]], style = dashed];
-//       CFG:   v[[ANCHOR]] -> v[[TEST_RET]] [ltail = [[CLUSTER_MERGE_BLOCKS]], style = dashed];
+//       CFG:   v[[TEST_FUNC]] -> v[[ANCHOR]][lhead = [[CLUSTER_MERGE_BLOCKS]], style = dashed];
+//       CFG:   v[[ANCHOR]] -> v[[TEST_RET]][ltail = [[CLUSTER_MERGE_BLOCKS]], style = dashed];
 
 func.func @merge_blocks(%arg0: i32, %arg1 : i32) -> () {
   %0 = arith.constant dense<[[0, 1], [2, 3]]> : tensor<2x2xi32>
diff --git a/mlir/test/Transforms/remove-dead-values.mlir b/mlir/test/Transforms/remove-dead-values.mlir
index fe7bcbc7c490b..e549926b90456 100644
--- a/mlir/test/Transforms/remove-dead-values.mlir
+++ b/mlir/test/Transforms/remove-dead-values.mlir
@@ -408,6 +408,22 @@ func.func @main(%arg3 : i32, %arg4 : i1) {
 
 // -----
 
+// The scf.if operation represents an if-then-else construct for conditionally
+// executing two regions of code. The 'the' region has exactly 1 block, and
+// the 'else' region may have 0 or 1 block. This case is to ensure 'else' region
+// with 0 block not crash.
+
+// CHECK-LABEL: func.func @clean_region_branch_op_with_empty_region
+func.func @clean_region_branch_op_with_empty_region(%arg0: i1, %arg1: memref<f32>) {
+  %cst = arith.constant 1.000000e+00 : f32
+  scf.if %arg0 {
+    memref.store %cst, %arg1[] : memref<f32>
+  }
+  return
+}
+
+// -----
+
 #map = affine_map<(d0)[s0, s1] -> (d0 * s0 + s1)>
 func.func @kernel(%arg0: memref<18xf32>) {
   %c1 = arith.constant 1 : index
diff --git a/mlir/test/lib/Conversion/ConvertToSPIRV/TestSPIRVVectorUnrolling.cpp b/mlir/test/lib/Conversion/ConvertToSPIRV/TestSPIRVVectorUnrolling.cpp
index 0bad43d5214b1..85245ea50782d 100644
--- a/mlir/test/lib/Conversion/ConvertToSPIRV/TestSPIRVVectorUnrolling.cpp
+++ b/mlir/test/lib/Conversion/ConvertToSPIRV/TestSPIRVVectorUnrolling.cpp
@@ -6,15 +6,12 @@
 //
 //===-------------------------------------------------------------------===//
 
-#include "mlir/Dialect/Arith/IR/Arith.h"
-#include "mlir/Dialect/Func/IR/FuncOps.h"
 #include "mlir/Dialect/SPIRV/IR/SPIRVDialect.h"
 #include "mlir/Dialect/SPIRV/Transforms/SPIRVConversion.h"
+#include "mlir/Dialect/UB/IR/UBOps.h"
 #include "mlir/Dialect/Vector/IR/VectorOps.h"
-#include "mlir/Dialect/Vector/Transforms/LoweringPatterns.h"
 #include "mlir/Pass/Pass.h"
 #include "mlir/Pass/PassManager.h"
-#include "mlir/Transforms/GreedyPatternRewriteDriver.h"
 
 namespace mlir {
 namespace {
@@ -30,7 +27,8 @@ struct TestSPIRVVectorUnrolling final
   }
 
   void getDependentDialects(DialectRegistry &registry) const override {
-    registry.insert<spirv::SPIRVDialect, vector::VectorDialect>();
+    registry
+        .insert<spirv::SPIRVDialect, vector::VectorDialect, ub::UBDialect>();
   }
 
   void runOnOperation() override {
diff --git a/mlir/test/lib/Dialect/LLVM/CMakeLists.txt b/mlir/test/lib/Dialect/LLVM/CMakeLists.txt
index 616f00a94a56e..b4b0a71d12920 100644
--- a/mlir/test/lib/Dialect/LLVM/CMakeLists.txt
+++ b/mlir/test/lib/Dialect/LLVM/CMakeLists.txt
@@ -20,6 +20,7 @@ mlir_target_link_libraries(MLIRLLVMTestPasses PUBLIC
   MLIRReconcileUnrealizedCasts
   MLIRSCFToControlFlow
   MLIRTransforms
+  MLIRUBToLLVM
   MLIRVectorToLLVMPass
   MLIRVectorToSCF
   )
diff --git a/mlir/test/lib/Dialect/LLVM/TestLowerToLLVM.cpp b/mlir/test/lib/Dialect/LLVM/TestLowerToLLVM.cpp
index 253748eb79ae1..fca58b6a17062 100644
--- a/mlir/test/lib/Dialect/LLVM/TestLowerToLLVM.cpp
+++ b/mlir/test/lib/Dialect/LLVM/TestLowerToLLVM.cpp
@@ -20,14 +20,12 @@
 #include "mlir/Conversion/MemRefToLLVM/MemRefToLLVM.h"
 #include "mlir/Conversion/ReconcileUnrealizedCasts/ReconcileUnrealizedCasts.h"
 #include "mlir/Conversion/SCFToControlFlow/SCFToControlFlow.h"
+#include "mlir/Conversion/UBToLLVM/UBToLLVM.h"
 #include "mlir/Conversion/VectorToLLVM/ConvertVectorToLLVMPass.h"
 #include "mlir/Conversion/VectorToSCF/VectorToSCF.h"
 #include "mlir/Dialect/Func/IR/FuncOps.h"
-#include "mlir/Dialect/LLVMIR/LLVMDialect.h"
 #include "mlir/Dialect/Linalg/Passes.h"
 #include "mlir/Dialect/MemRef/Transforms/Passes.h"
-#include "mlir/IR/DialectRegistry.h"
-#include "mlir/Pass/Pass.h"
 #include "mlir/Pass/PassManager.h"
 #include "mlir/Pass/PassOptions.h"
 #include "mlir/Transforms/Passes.h"
@@ -81,6 +79,8 @@ void buildTestLowerToLLVM(OpPassManager &pm,
   pm.addPass(createConvertControlFlowToLLVMPass());
   // Convert Index to LLVM (always needed).
   pm.addPass(createConvertIndexToLLVMPass());
+  // Convert UB to LLVM (always needed).
+  pm.addPass(createUBToLLVMConversionPass());
   // Convert remaining unrealized_casts (always needed).
   pm.addPass(createReconcileUnrealizedCastsPass());
 }
diff --git a/mlir/test/lib/Dialect/MemRef/TestEmulateNarrowType.cpp b/mlir/test/lib/Dialect/MemRef/TestEmulateNarrowType.cpp
index 7401e470ed4f2..ba2ea40e83d96 100644
--- a/mlir/test/lib/Dialect/MemRef/TestEmulateNarrowType.cpp
+++ b/mlir/test/lib/Dialect/MemRef/TestEmulateNarrowType.cpp
@@ -99,7 +99,8 @@ struct TestEmulateNarrowTypePass
 
     arith::populateArithNarrowTypeEmulationPatterns(typeConverter, patterns);
     memref::populateMemRefNarrowTypeEmulationPatterns(typeConverter, patterns);
-    vector::populateVectorNarrowTypeEmulationPatterns(typeConverter, patterns);
+    vector::populateVectorNarrowTypeEmulationPatterns(typeConverter, patterns,
+                                                      disableAtomicRMW);
 
     if (failed(applyPartialConversion(op, target, std::move(patterns))))
       signalPassFailure();
@@ -118,6 +119,12 @@ struct TestEmulateNarrowTypePass
       *this, "skip-memref-type-conversion",
       llvm::cl::desc("disable memref type conversion (to test failures)"),
       llvm::cl::init(false)};
+
+  Option<bool> disableAtomicRMW{
+      *this, "disable-atomic-rmw",
+      llvm::cl::desc("disable atomic read-modify-write and prefer generating "
+                     "normal sequence"),
+      llvm::cl::init(false)};
 };
 } // namespace
 
diff --git a/mlir/test/lib/Dialect/Test/TestOpDefs.cpp b/mlir/test/lib/Dialect/Test/TestOpDefs.cpp
index c6be26d0a44d9..f6b8a0005f285 100644
--- a/mlir/test/lib/Dialect/Test/TestOpDefs.cpp
+++ b/mlir/test/lib/Dialect/Test/TestOpDefs.cpp
@@ -506,6 +506,38 @@ void CustomResultsNameOp::getAsmResultNames(
         setNameFn(getResult(i), str.getValue());
 }
 
+//===----------------------------------------------------------------------===//
+// ResultNameFromTypeOp
+//===----------------------------------------------------------------------===//
+
+void ResultNameFromTypeOp::getAsmResultNames(
+    function_ref<void(Value, StringRef)> setNameFn) {
+  auto result = getResult();
+  auto setResultNameFn = [&](::llvm::StringRef name) {
+    setNameFn(result, name);
+  };
+  auto opAsmTypeInterface =
+      ::mlir::cast<::mlir::OpAsmTypeInterface>(result.getType());
+  opAsmTypeInterface.getAsmName(setResultNameFn);
+}
+
+//===----------------------------------------------------------------------===//
+// BlockArgumentNameFromTypeOp
+//===----------------------------------------------------------------------===//
+
+void BlockArgumentNameFromTypeOp::getAsmBlockArgumentNames(
+    ::mlir::Region &region, ::mlir::OpAsmSetValueNameFn setNameFn) {
+  for (auto &block : region) {
+    for (auto arg : block.getArguments()) {
+      if (auto opAsmTypeInterface =
+              ::mlir::dyn_cast<::mlir::OpAsmTypeInterface>(arg.getType())) {
+        auto setArgNameFn = [&](StringRef name) { setNameFn(arg, name); };
+        opAsmTypeInterface.getAsmName(setArgNameFn);
+      }
+    }
+  }
+}
+
 //===----------------------------------------------------------------------===//
 // ResultTypeWithTraitOp
 //===----------------------------------------------------------------------===//
diff --git a/mlir/test/lib/Dialect/Test/TestOps.td b/mlir/test/lib/Dialect/Test/TestOps.td
index 0b1f22b3ee932..2aa0658ab0e5d 100644
--- a/mlir/test/lib/Dialect/Test/TestOps.td
+++ b/mlir/test/lib/Dialect/Test/TestOps.td
@@ -193,8 +193,15 @@ def TypeArrayAttrWithDefaultOp : TEST_Op<"type_array_attr_with_default"> {
   let arguments = (ins DefaultValuedAttr<TypeArrayAttr, "{}">:$attr);
 }
 
-def TypeStringAttrWithTypeOp : TEST_Op<"string_attr_with_type"> {
-  let arguments = (ins TypedStrAttr<AnyType>:$attr);
+def TypedStringAttrWithTypeOp : TEST_Op<"string_attr_with_type"> {
+  let arguments = (ins TypedStrAttr<AnyInteger>:$attr);
+  let assemblyFormat = "$attr attr-dict";
+}
+
+def TypedStringAttrWithMixedTypeOp : TEST_Op<"string_attr_with_mixed_type"> {
+  let arguments = (ins
+    AnyAttrOf<[TypedStrAttr<AnyInteger>, I64Attr]>:$attr
+  );
   let assemblyFormat = "$attr attr-dict";
 }
 
@@ -550,7 +557,12 @@ def TestCallOp : TEST_Op<"call", [DeclareOpInterfaceMethods<SymbolUserOpInterfac
 
 def ConversionCallOp : TEST_Op<"conversion_call_op",
     [CallOpInterface]> {
-  let arguments = (ins Variadic<AnyType>:$arg_operands, SymbolRefAttr:$callee);
+  let arguments = (ins
+    Variadic<AnyType>:$arg_operands,
+    SymbolRefAttr:$callee,
+    OptionalAttr<DictArrayAttr>:$arg_attrs,
+    OptionalAttr<DictArrayAttr>:$res_attrs
+  );
   let results = (outs Variadic<AnyType>);
 
   let extraClassDeclaration = [{
@@ -611,6 +623,10 @@ def ConversionFuncOp : TEST_Op<"conversion_func_op", [FunctionOpInterface]> {
 def FunctionalRegionOp : TEST_Op<"functional_region_op",
     [CallableOpInterface]> {
   let regions = (region AnyRegion:$body);
+  let arguments = (ins
+    OptionalAttr<DictArrayAttr>:$arg_attrs,
+    OptionalAttr<DictArrayAttr>:$res_attrs
+  );
   let results = (outs FunctionType);
 
   let extraClassDeclaration = [{
@@ -924,6 +940,21 @@ def CustomResultsNameOp
   let results = (outs Variadic<AnyInteger>:$r);
 }
 
+// This is used to test OpAsmTypeInterface::getAsmName for op result name,
+def ResultNameFromTypeOp
+ : TEST_Op<"result_name_from_type",
+           [DeclareOpInterfaceMethods<OpAsmOpInterface, ["getAsmResultNames"]>]> {
+  let results = (outs AnyType:$r);
+}
+
+// This is used to test OpAsmTypeInterface::getAsmName for block argument,
+def BlockArgumentNameFromTypeOp
+  : TEST_Op<"block_argument_name_from_type",
+      [DeclareOpInterfaceMethods<OpAsmOpInterface, ["getAsmBlockArgumentNames"]>]> {
+  let regions = (region AnyRegion:$body);
+  let assemblyFormat = "regions attr-dict-with-keyword";
+}
+
 // This is used to test the OpAsmOpInterface::getDefaultDialect() feature:
 // operations nested in a region under this op will drop the "test." dialect
 // prefix.
@@ -3287,7 +3318,9 @@ def TestCallAndStoreOp : TEST_Op<"call_and_store",
     SymbolRefAttr:$callee,
     Arg<AnyMemRef, "", [MemWrite]>:$address,
     Variadic<AnyType>:$callee_operands,
-    BoolAttr:$store_before_call
+    BoolAttr:$store_before_call,
+    OptionalAttr<DictArrayAttr>:$arg_attrs,
+    OptionalAttr<DictArrayAttr>:$res_attrs
   );
   let results = (outs
     Variadic<AnyType>:$results
@@ -3302,7 +3335,9 @@ def TestCallOnDeviceOp : TEST_Op<"call_on_device",
   let arguments = (ins
     SymbolRefAttr:$callee,
     Variadic<AnyType>:$forwarded_operands,
-    AnyType:$non_forwarded_device_operand
+    AnyType:$non_forwarded_device_operand,
+    OptionalAttr<DictArrayAttr>:$arg_attrs,
+    OptionalAttr<DictArrayAttr>:$res_attrs
   );
   let results = (outs
     Variadic<AnyType>:$results
diff --git a/mlir/test/lib/Dialect/Test/TestTypeDefs.td b/mlir/test/lib/Dialect/Test/TestTypeDefs.td
index 60108ac86d1ed..6335701786ecc 100644
--- a/mlir/test/lib/Dialect/Test/TestTypeDefs.td
+++ b/mlir/test/lib/Dialect/Test/TestTypeDefs.td
@@ -398,4 +398,9 @@ def TestTypeVerification : Test_Type<"TestTypeVerification"> {
   let assemblyFormat = "`<` $param `>`";
 }
 
+def TestTypeOpAsmTypeInterface : Test_Type<"TestTypeOpAsmTypeInterface",
+    [DeclareTypeInterfaceMethods<OpAsmTypeInterface, ["getAsmName"]>]> {
+  let mnemonic = "op_asm_type_interface";
+}
+
 #endif // TEST_TYPEDEFS
diff --git a/mlir/test/lib/Dialect/Test/TestTypes.cpp b/mlir/test/lib/Dialect/Test/TestTypes.cpp
index b822e019e09d2..1ae7ac472d989 100644
--- a/mlir/test/lib/Dialect/Test/TestTypes.cpp
+++ b/mlir/test/lib/Dialect/Test/TestTypes.cpp
@@ -532,3 +532,8 @@ void TestRecursiveAliasType::print(AsmPrinter &printer) const {
   }
   printer << ">";
 }
+
+void TestTypeOpAsmTypeInterfaceType::getAsmName(
+    OpAsmSetNameFn setNameFn) const {
+  setNameFn("op_asm_type_interface");
+}
diff --git a/mlir/test/lib/IR/CMakeLists.txt b/mlir/test/lib/IR/CMakeLists.txt
index e5416da70d500..1abcfc77d2d9b 100644
--- a/mlir/test/lib/IR/CMakeLists.txt
+++ b/mlir/test/lib/IR/CMakeLists.txt
@@ -27,13 +27,16 @@ add_mlir_library(MLIRTestIR
   TestVisitorsGeneric.cpp
 
   EXCLUDE_FROM_LIBMLIR
+
+  LINK_LIBS PUBLIC
+  MLIRTestDialect
   )
+
 mlir_target_link_libraries(MLIRTestIR PUBLIC
   MLIRPass
   MLIRBytecodeReader
   MLIRBytecodeWriter
   MLIRFunctionInterfaces
-  MLIRTestDialect
   )
 
 target_include_directories(MLIRTestIR
diff --git a/mlir/test/lib/Pass/CMakeLists.txt b/mlir/test/lib/Pass/CMakeLists.txt
index 6698af86b8ae6..ab52f621c517e 100644
--- a/mlir/test/lib/Pass/CMakeLists.txt
+++ b/mlir/test/lib/Pass/CMakeLists.txt
@@ -1,6 +1,7 @@
 get_property(conversion_libs GLOBAL PROPERTY MLIR_CONVERSION_LIBS)
 # Exclude tests from libMLIR.so
 add_mlir_library(MLIRTestPass
+  TestConvertToSPIRVPass.cpp
   TestDynamicPipeline.cpp
   TestPassManager.cpp
   TestSPIRVCPURunnerPipeline.cpp
@@ -10,12 +11,15 @@ add_mlir_library(MLIRTestPass
 
   ADDITIONAL_HEADER_DIRS
   ${MLIR_MAIN_INCLUDE_DIR}/mlir/Pass
+
+  LINK_LIBS PUBLIC
+  MLIRTestDialect
   )
 mlir_target_link_libraries(MLIRTestPass PUBLIC
   ${conversion_libs}
   MLIRIR
+  MLIRSPIRVTransforms
   MLIRPass
-  MLIRTestDialect
   )
 
 target_include_directories(MLIRTestPass
diff --git a/mlir/test/lib/Pass/TestConvertToSPIRVPass.cpp b/mlir/test/lib/Pass/TestConvertToSPIRVPass.cpp
new file mode 100644
index 0000000000000..3c99d3c5b60ce
--- /dev/null
+++ b/mlir/test/lib/Pass/TestConvertToSPIRVPass.cpp
@@ -0,0 +1,175 @@
+//===- ConvertToSPIRVPass.cpp - MLIR SPIR-V Conversion --------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "mlir/Conversion/ArithToSPIRV/ArithToSPIRV.h"
+#include "mlir/Conversion/FuncToSPIRV/FuncToSPIRV.h"
+#include "mlir/Conversion/GPUToSPIRV/GPUToSPIRV.h"
+#include "mlir/Conversion/IndexToSPIRV/IndexToSPIRV.h"
+#include "mlir/Conversion/MemRefToSPIRV/MemRefToSPIRV.h"
+#include "mlir/Conversion/SCFToSPIRV/SCFToSPIRV.h"
+#include "mlir/Conversion/UBToSPIRV/UBToSPIRV.h"
+#include "mlir/Conversion/VectorToSPIRV/VectorToSPIRV.h"
+#include "mlir/Dialect/Arith/Transforms/Passes.h"
+#include "mlir/Dialect/GPU/IR/GPUDialect.h"
+#include "mlir/Dialect/SPIRV/IR/SPIRVAttributes.h"
+#include "mlir/Dialect/SPIRV/Transforms/SPIRVConversion.h"
+#include "mlir/Dialect/Vector/IR/VectorOps.h"
+#include "mlir/Dialect/Vector/Transforms/LoweringPatterns.h"
+#include "mlir/Dialect/Vector/Transforms/VectorRewritePatterns.h"
+#include "mlir/IR/PatternMatch.h"
+#include "mlir/Pass/Pass.h"
+#include "mlir/Pass/PassOptions.h"
+#include "mlir/Rewrite/FrozenRewritePatternSet.h"
+#include "mlir/Transforms/DialectConversion.h"
+#include <memory>
+
+#define DEBUG_TYPE "test-convert-to-spirv"
+
+using namespace mlir;
+
+namespace {
+
+/// Map memRef memory space to SPIR-V storage class.
+void mapToMemRef(Operation *op, spirv::TargetEnvAttr &targetAttr) {
+  spirv::TargetEnv targetEnv(targetAttr);
+  bool targetEnvSupportsKernelCapability =
+      targetEnv.allows(spirv::Capability::Kernel);
+  spirv::MemorySpaceToStorageClassMap memorySpaceMap =
+      targetEnvSupportsKernelCapability
+          ? spirv::mapMemorySpaceToOpenCLStorageClass
+          : spirv::mapMemorySpaceToVulkanStorageClass;
+  spirv::MemorySpaceToStorageClassConverter converter(memorySpaceMap);
+  spirv::convertMemRefTypesAndAttrs(op, converter);
+}
+
+/// Populate patterns for each dialect.
+void populateConvertToSPIRVPatterns(const SPIRVTypeConverter &typeConverter,
+                                    ScfToSPIRVContext &scfToSPIRVContext,
+                                    RewritePatternSet &patterns) {
+  arith::populateCeilFloorDivExpandOpsPatterns(patterns);
+  arith::populateArithToSPIRVPatterns(typeConverter, patterns);
+  populateBuiltinFuncToSPIRVPatterns(typeConverter, patterns);
+  populateFuncToSPIRVPatterns(typeConverter, patterns);
+  populateGPUToSPIRVPatterns(typeConverter, patterns);
+  index::populateIndexToSPIRVPatterns(typeConverter, patterns);
+  populateMemRefToSPIRVPatterns(typeConverter, patterns);
+  populateVectorToSPIRVPatterns(typeConverter, patterns);
+  populateSCFToSPIRVPatterns(typeConverter, scfToSPIRVContext, patterns);
+  ub::populateUBToSPIRVConversionPatterns(typeConverter, patterns);
+}
+
+/// A pass to perform the SPIR-V conversion.
+struct TestConvertToSPIRVPass final
+    : PassWrapper<TestConvertToSPIRVPass, OperationPass<>> {
+  Option<bool> runSignatureConversion{
+      *this, "run-signature-conversion",
+      llvm::cl::desc(
+          "Run function signature conversion to convert vector types"),
+      llvm::cl::init(true)};
+  Option<bool> runVectorUnrolling{
+      *this, "run-vector-unrolling",
+      llvm::cl::desc(
+          "Run vector unrolling to convert vector types in function bodies"),
+      llvm::cl::init(true)};
+  Option<bool> convertGPUModules{
+      *this, "convert-gpu-modules",
+      llvm::cl::desc("Clone and convert GPU modules"), llvm::cl::init(false)};
+  Option<bool> nestInGPUModule{
+      *this, "nest-in-gpu-module",
+      llvm::cl::desc("Put converted SPIR-V module inside the gpu.module "
+                     "instead of alongside it."),
+      llvm::cl::init(false)};
+
+  MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(TestConvertToSPIRVPass)
+
+  StringRef getArgument() const final { return "test-convert-to-spirv"; }
+  StringRef getDescription() const final {
+    return "Conversion to SPIR-V pass only used for internal tests.";
+  }
+  void getDependentDialects(DialectRegistry &registry) const override {
+    registry.insert<spirv::SPIRVDialect>();
+    registry.insert<vector::VectorDialect>();
+  }
+
+  TestConvertToSPIRVPass() = default;
+  TestConvertToSPIRVPass(bool convertGPUModules, bool nestInGPUModule) {
+    this->convertGPUModules = convertGPUModules;
+    this->nestInGPUModule = nestInGPUModule;
+  };
+  TestConvertToSPIRVPass(const TestConvertToSPIRVPass &) {}
+
+  void runOnOperation() override {
+    Operation *op = getOperation();
+    MLIRContext *context = &getContext();
+
+    // Unroll vectors in function signatures to native size.
+    if (runSignatureConversion && failed(spirv::unrollVectorsInSignatures(op)))
+      return signalPassFailure();
+
+    // Unroll vectors in function bodies to native size.
+    if (runVectorUnrolling && failed(spirv::unrollVectorsInFuncBodies(op)))
+      return signalPassFailure();
+
+    // Generic conversion.
+    if (!convertGPUModules) {
+      spirv::TargetEnvAttr targetAttr = spirv::lookupTargetEnvOrDefault(op);
+      std::unique_ptr<ConversionTarget> target =
+          SPIRVConversionTarget::get(targetAttr);
+      SPIRVTypeConverter typeConverter(targetAttr);
+      RewritePatternSet patterns(context);
+      ScfToSPIRVContext scfToSPIRVContext;
+      mapToMemRef(op, targetAttr);
+      populateConvertToSPIRVPatterns(typeConverter, scfToSPIRVContext,
+                                     patterns);
+      if (failed(applyPartialConversion(op, *target, std::move(patterns))))
+        return signalPassFailure();
+      return;
+    }
+
+    // Clone each GPU kernel module for conversion, given that the GPU
+    // launch op still needs the original GPU kernel module.
+    SmallVector<Operation *, 1> gpuModules;
+    OpBuilder builder(context);
+    op->walk([&](gpu::GPUModuleOp gpuModule) {
+      if (nestInGPUModule)
+        builder.setInsertionPointToStart(gpuModule.getBody());
+      else
+        builder.setInsertionPoint(gpuModule);
+      gpuModules.push_back(builder.clone(*gpuModule));
+    });
+    // Run conversion for each module independently as they can have
+    // different TargetEnv attributes.
+    for (Operation *gpuModule : gpuModules) {
+      spirv::TargetEnvAttr targetAttr =
+          spirv::lookupTargetEnvOrDefault(gpuModule);
+      std::unique_ptr<ConversionTarget> target =
+          SPIRVConversionTarget::get(targetAttr);
+      SPIRVTypeConverter typeConverter(targetAttr);
+      RewritePatternSet patterns(context);
+      ScfToSPIRVContext scfToSPIRVContext;
+      mapToMemRef(gpuModule, targetAttr);
+      populateConvertToSPIRVPatterns(typeConverter, scfToSPIRVContext,
+                                     patterns);
+      if (failed(applyFullConversion(gpuModule, *target, std::move(patterns))))
+        return signalPassFailure();
+    }
+  }
+};
+
+} // namespace
+
+namespace mlir::test {
+void registerTestConvertToSPIRVPass() {
+  PassRegistration<TestConvertToSPIRVPass>();
+}
+std::unique_ptr<Pass> createTestConvertToSPIRVPass(bool convertGPUModules,
+                                                   bool nestInGPUModule) {
+  return std::make_unique<TestConvertToSPIRVPass>(convertGPUModules,
+                                                  nestInGPUModule);
+}
+} // namespace mlir::test
diff --git a/mlir/test/lib/Pass/TestVulkanRunnerPipeline.cpp b/mlir/test/lib/Pass/TestVulkanRunnerPipeline.cpp
index e4cbbeb1f99bc..3b011d2b2b0ac 100644
--- a/mlir/test/lib/Pass/TestVulkanRunnerPipeline.cpp
+++ b/mlir/test/lib/Pass/TestVulkanRunnerPipeline.cpp
@@ -10,7 +10,6 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "mlir/Conversion/ConvertToSPIRV/ConvertToSPIRVPass.h"
 #include "mlir/Conversion/GPUCommon/GPUCommonPass.h"
 #include "mlir/Conversion/GPUToSPIRV/GPUToSPIRVPass.h"
 #include "mlir/Conversion/MemRefToLLVM/MemRefToLLVM.h"
@@ -26,6 +25,12 @@
 
 using namespace mlir;
 
+// Defined in the test directory, no public header.
+namespace mlir::test {
+std::unique_ptr<Pass> createTestConvertToSPIRVPass(bool convertGPUModules,
+                                                   bool nestInGPUModule);
+}
+
 namespace {
 
 struct VulkanRunnerPipelineOptions
@@ -47,10 +52,8 @@ void buildTestVulkanRunnerPipeline(OpPassManager &passManager,
       "SPV_KHR_storage_buffer_storage_class");
   passManager.addPass(createGpuSPIRVAttachTarget(attachTargetOptions));
 
-  ConvertToSPIRVPassOptions convertToSPIRVOptions{};
-  convertToSPIRVOptions.convertGPUModules = true;
-  convertToSPIRVOptions.nestInGPUModule = true;
-  passManager.addPass(createConvertToSPIRVPass(convertToSPIRVOptions));
+  passManager.addPass(test::createTestConvertToSPIRVPass(
+      /*convertGPUModules=*/true, /*nestInGPUModule=*/true));
 
   OpPassManager &spirvModulePM =
       passManager.nest<gpu::GPUModuleOp>().nest<spirv::ModuleOp>();
diff --git a/mlir/test/lib/Transforms/CMakeLists.txt b/mlir/test/lib/Transforms/CMakeLists.txt
index b91265d20fb48..1b9b9bffa5279 100644
--- a/mlir/test/lib/Transforms/CMakeLists.txt
+++ b/mlir/test/lib/Transforms/CMakeLists.txt
@@ -34,12 +34,14 @@ add_mlir_library(MLIRTestTransforms
 
   DEPENDS
   ${MLIRTestTransformsPDLDep}
+
+  LINK_LIBS PUBLIC
+  MLIRTestDialect
   )
 mlir_target_link_libraries(MLIRTestTransforms PUBLIC
   MLIRAnalysis
   MLIRFuncDialect
   MLIRInferIntRangeInterface
-  MLIRTestDialect
   MLIRTransforms
   )
 
diff --git a/mlir/test/lit.cfg.py b/mlir/test/lit.cfg.py
index f493e5be574a0..32b2f8b53d5fa 100644
--- a/mlir/test/lit.cfg.py
+++ b/mlir/test/lit.cfg.py
@@ -82,17 +82,32 @@ def add_runtime(name):
 # available. This is darwin specific since it's currently only needed on darwin.
 # Stolen from llvm/test/lit.cfg.py with a few modifications
 def get_asan_rtlib():
-    if not "asan" in config.available_features or not "Darwin" in config.host_os:
+    if not "asan" in config.available_features:
         return ""
-    # Find the asan rt lib
-    resource_dir = (
-        subprocess.check_output([config.host_cc.strip(), "-print-resource-dir"])
-        .decode("utf-8")
-        .strip()
-    )
-    return os.path.join(
-        resource_dir, "lib", "darwin", "libclang_rt.asan_osx_dynamic.dylib"
-    )
+
+    if "Darwin" in config.host_os:
+        # Find the asan rt lib
+        resource_dir = (
+            subprocess.check_output([config.host_cc.strip(), "-print-resource-dir"])
+            .decode("utf-8")
+            .strip()
+        )
+        return os.path.join(
+            resource_dir, "lib", "darwin", "libclang_rt.asan_osx_dynamic.dylib"
+        )
+    if "Linux" in config.host_os:
+        return (
+            subprocess.check_output(
+                [
+                    config.host_cxx.strip(),
+                    f"-print-file-name=libclang_rt.asan-{config.host_arch}.so",
+                ]
+            )
+            .decode("utf-8")
+            .strip()
+        )
+
+    return ""
 
 
 # On macOS, we can't do the DYLD_INSERT_LIBRARIES trick with a shim python
@@ -247,7 +262,9 @@ def find_real_python_interpreter():
 # TODO: detect Windows situation (or mark these tests as unsupported on these platforms).
 if "asan" in config.available_features:
     if "Linux" in config.host_os:
-        python_executable = f"LD_PRELOAD=$({config.host_cxx} -print-file-name=libclang_rt.asan-{config.host_arch}.so) {config.python_executable}"
+        python_executable = (
+            f"env LD_PRELOAD={get_asan_rtlib()} {config.python_executable}"
+        )
     if "Darwin" in config.host_os:
         # Ensure we use a non-shim Python executable, for the `DYLD_INSERT_LIBRARIES`
         # env variable to take effect
diff --git a/mlir/test/mlir-tblgen/attr-or-type-builder-invalid.td b/mlir/test/mlir-tblgen/attr-or-type-builder-invalid.td
index 4db7162184550..d16fcfc4cd099 100644
--- a/mlir/test/mlir-tblgen/attr-or-type-builder-invalid.td
+++ b/mlir/test/mlir-tblgen/attr-or-type-builder-invalid.td
@@ -1,4 +1,4 @@
-// RUN: not mlir-tblgen -gen-typedef-defs -I %S/../../include %s 2>&1 | FileCheck %s
+// RUN: not mlir-tblgen -gen-typedef-defs -I %S/../../include %s 2>&1 | FileCheck %s -DFILE=%s
 
 include "mlir/IR/AttrTypeBase.td"
 include "mlir/IR/OpBase.td"
@@ -13,14 +13,13 @@ class InvalidType<string name> : TypeDef<Test_Dialect, name> {
 }
 
 // This definition should not generate an error due to the use in `InvalidTypeA`
-// CHECK-NOT: Record `TestParameter' does not have a field named `type'!
 def TestParameter : TypeParameter<"int", "int parameter">;
 
 // Test builder uses wrong record class.
+// CHECK: [[FILE]]:[[@LINE+1]]:5: error: Initializer of 'typeName' in 'InvalidTypeA' could not be fully resolved: !strconcat("TestDialect", !strconcat(".", mnemonic))
 def InvalidTypeA : InvalidType<"InvalidTypeA"> {
   let parameters = (ins "int":$v0);
   let builders = [
-    // CHECK: Builder DAG arguments must be either strings or defs which inherit from CArg
     TypeBuilder<(ins TestParameter:$arg0), [{
       return $_get($_ctxt, arg0);
     }]>
diff --git a/mlir/test/mlir-tblgen/gen-dialect-doc.td b/mlir/test/mlir-tblgen/gen-dialect-doc.td
index 79d755111e8f6..72916704369ee 100644
--- a/mlir/test/mlir-tblgen/gen-dialect-doc.td
+++ b/mlir/test/mlir-tblgen/gen-dialect-doc.td
@@ -36,7 +36,15 @@ def ACOp : Op<Test_Dialect, "c", [NoMemoryEffect, SingleBlockImplicitTerminator<
 def ABOp : Op<Test_Dialect, "b", [NoMemoryEffect, SingleBlockImplicitTerminator<"YieldOp">]>;
 }
 
-def AEOp : Op<Test_Dialect, "e", [NoMemoryEffect, SingleBlockImplicitTerminator<"YieldOp">]>;
+def AEOp : Op<Test_Dialect, "e", [NoMemoryEffect]> {
+  let summary = "Op with a summary";
+  let description = "Op with a description";
+  let arguments = (ins ConfinedType<AnyType, [CPred<"::llvm::isa<::mlir::TensorType>($_self)">]>:$tensor,
+                       I16Attr:$int_attr);
+  let results = (outs
+    ConfinedType<AnyType, [CPred<"::llvm::isa<::mlir::TensorType>($_self)">]>:$output
+  ); 
+}
 
 def TestAttr : DialectAttr<Test_Dialect, CPred<"true">> {
   let summary = "attribute summary";
@@ -81,11 +89,33 @@ def TestEnum :
 }
 
 // CHECK: Dialect without a [TOC] here.
-// CHECK: TOC added by tool.
-// CHECK: [TOC]
+// CHECK-NEXT:  TOC added by tool.
+// CHECK-EMPTY:
+// CHECK-NEXT:  [TOC]
+// CHECK-EMPTY:
 
 // CHECK-NOT: [TOC]
-// CHECK: test.e
+
+// CHECK:       test.e
+// CHECK-EMPTY:
+// CHECK-NEXT:  _Op with a summary_
+// CHECK-EMPTY:
+// CHECK-NEXT:  Op with a description
+// CHECK-EMPTY:
+
+// CHECK: Operands:
+// CHECK-EMPTY:
+// CHECK-NEXT:  | Operand | Description |
+// CHECK-NEXT:  | :-----: | ----------- |
+// CHECK-NEXT:  | `tensor` | |
+// CHECK-EMPTY:
+// CHECK-NEXT:  Results:
+// CHECK-EMPTY:
+// CHECK-NEXT:  | Result | Description |
+// CHECK-NEXT:  | :----: | ----------- |
+// CHECK-NEXT:  | `output` | |
+// CHECK-EMPTY:
+
 // CHECK: Group of ops
 // CHECK: test.a
 // CHECK: test.d
@@ -96,9 +126,11 @@ def TestEnum :
 // CHECK: Interfaces: `NoMemoryEffect (MemoryEffectOpInterface)`
 // CHECK: Effects: `MemoryEffects::Effect{}`
 
-// CHECK: ## Attribute constraints
-// CHECK: ### attribute summary
-// CHECK: attribute description
+// CHECK:       ## Attribute constraints
+// CHECK-EMPTY:
+// CHECK-NEXT:  ### attribute summary
+// CHECK-EMPTY:
+// CHECK:       attribute description
 
 // CHECK: TestAttrDefAttr
 // CHECK: Syntax:
@@ -120,15 +152,20 @@ def TestEnum :
 // CHECK: Syntax:
 // CHECK: !test.test_type_def_params
 
-// CHECK: ## Enums
-// CHECK: ### TestEnum
-// CHECK: enum summary
-// CHECK: #### Cases:
-// CHECK: | Symbol | Value | String |
-// CHECK: | :----: | :---: | ------ |
-// CHECK: | First | `0` | first |
-// CHECK: | Second | `1` | second |
-// CHECK: | Third | `2` | third |
+// CHECK:       ## Enums
+// CHECK-EMPTY:
+// CHECK-NEXT:  ### TestEnum
+// CHECK-EMPTY:
+// CHECK-NEXT:  _Enum summary_
+// CHECK-EMPTY:
+// CHECK-NEXT:  #### Cases:
+// CHECK-EMPTY:
+// CHECK-NEXT:  | Symbol | Value | String |
+// CHECK-NEXT:  | :----: | :---: | ------ |
+// CHECK-NEXT:  | First | `0` | first |
+// CHECK-NEXT:  | Second | `1` | second |
+// CHECK-NEXT:  | Third | `2` | third |
+// CHECK-EMPTY:
 
 def Toc_Dialect : Dialect {
   let name = "test_toc";
diff --git a/mlir/test/mlir-tblgen/gen-pass-doc.td b/mlir/test/mlir-tblgen/gen-pass-doc.td
new file mode 100644
index 0000000000000..fd8e9cccb5fb5
--- /dev/null
+++ b/mlir/test/mlir-tblgen/gen-pass-doc.td
@@ -0,0 +1,33 @@
+// RUN: mlir-tblgen -gen-pass-doc -I %S/../../include -dialect=test %s | FileCheck %s
+
+include "mlir/Pass/PassBase.td"
+
+def TestPassDocA : Pass<"test-pass-doc-a"> {
+  let summary = "pass summary";
+  let description = [{
+    Pass description
+  }];
+
+  let options = [
+    ListOption<"option", "option", "std::string", "pass option">
+  ];
+}
+
+def TestPassDocB : Pass<"test-pass-doc-b"> {
+}
+
+// Ensure there are empty lines between individual pass docs.
+
+// CHECK:      `-test-pass-doc-a`
+// CHECK-EMPTY:
+// CHECK-NEXT:  _Pass summary_
+// CHECK-EMPTY:
+// CHECK-NEXT:  Pass description
+// CHECK-EMPTY:
+// CHECK-NEXT:  Options
+// CHECK-EMPTY:
+// CHECK-NEXT:  ```
+// CHECK-NEXT:  -option : pass option
+// CHECK-NEXT:  ```
+// CHECK-EMPTY:
+// CHECK-NEXT:  `-test-pass-doc-b`
diff --git a/mlir/test/python/dialects/builtin.py b/mlir/test/python/dialects/builtin.py
index 18ebba61e7fea..973a0eaeca2cd 100644
--- a/mlir/test/python/dialects/builtin.py
+++ b/mlir/test/python/dialects/builtin.py
@@ -246,3 +246,7 @@ def testDenseElementsAttr():
         # CHECK{LITERAL}: dense<[[0, 1], [2, 3]]> : tensor<2x2xi32>
         print(DenseElementsAttr.get(values, type=VectorType.get((2, 2), i32)))
         # CHECK{LITERAL}: dense<[[0, 1], [2, 3]]> : vector<2x2xi32>
+        idx_values = np.arange(4, dtype=np.int64)
+        idx_type = IndexType.get()
+        print(DenseElementsAttr.get(idx_values, type=VectorType.get([4], idx_type)))
+        # CHECK{LITERAL}: dense<[0, 1, 2, 3]> : vector<4xindex>
diff --git a/mlir/test/python/dialects/gpu/module-to-binary-nvvm.py b/mlir/test/python/dialects/gpu/module-to-binary-nvvm.py
index e0225911b2144..96e3f6efcc5d3 100644
--- a/mlir/test/python/dialects/gpu/module-to-binary-nvvm.py
+++ b/mlir/test/python/dialects/gpu/module-to-binary-nvvm.py
@@ -82,5 +82,5 @@ def testGPUToASMBin():
     print(gpu.CompilationTarget(o.format))
     # CHECK: b'//\n// Generated by LLVM NVPTX Back-End{{.*}}'
     print(o.object)
-    # CHECK: {O = 2 : i32}
+    # CHECK: {LLVMIRToISATimeInMs = {{[0-9]+}} : i64, O = 2 : i32}
     print(o.properties)
diff --git a/mlir/test/python/dialects/linalg/ops.py b/mlir/test/python/dialects/linalg/ops.py
index 72045a07b2da8..94f8ea4faf4a8 100644
--- a/mlir/test/python/dialects/linalg/ops.py
+++ b/mlir/test/python/dialects/linalg/ops.py
@@ -1,6 +1,6 @@
 # RUN: %PYTHON %s | FileCheck %s
 
-from mlir.dialects import arith, builtin, func, linalg, tensor
+from mlir.dialects import arith, func, linalg, tensor, memref
 from mlir.dialects.linalg.opdsl.lang import *
 from mlir.ir import *
 
@@ -84,6 +84,7 @@ def named_form(lhs, rhs):
 
     print(module)
 
+
 # CHECK-LABEL: TEST: testIdentityRegionOps
 @run
 def testIdentityRegionOps():
@@ -161,3 +162,307 @@ def broadcast_op(op1, op2, op3):
                 op5 = linalg.broadcast(op3, outs=[op2], dimensions=[0])
 
     print(module)
+
+
+# CHECK-LABEL: TEST: testGenericOp
+@run
+def testGenericOp():
+    with Context(), Location.unknown():
+        module = Module.create()
+        f32 = F32Type.get()
+        memref_t = MemRefType.get([10, 10], f32)
+        with InsertionPoint(module.body):
+            id_map_1 = AffineMap.get_identity(2)
+            # CHECK: %[[VAL_0:.*]] = tensor.empty() : tensor<16x16xf32>
+            # CHECK: %[[VAL_1:.*]] = tensor.empty() : tensor<16x16xf32>
+            x = tensor.empty((16, 16), f32)
+            y = tensor.empty((16, 16), f32)
+
+            # CHECK: %[[VAL_2:.*]] = linalg.generic {indexing_maps = [#map, #map], iterator_types = ["parallel", "parallel"]} ins(%[[VAL_0]] : tensor<16x16xf32>) outs(%[[VAL_1]] : tensor<16x16xf32>) {
+            # CHECK: ^bb0(%in: f32, %out: f32):
+            # CHECK:   linalg.yield %in : f32
+            # CHECK: } -> tensor<16x16xf32>
+            @linalg.generic(
+                [x],
+                [y],
+                [id_map_1, id_map_1],
+                [linalg.IteratorType.parallel, linalg.IteratorType.parallel],
+            )
+            def f(a, b):
+                assert isinstance(a, Value)
+                assert isinstance(a.type, F32Type)
+                assert isinstance(b, Value)
+                assert isinstance(b.type, F32Type)
+                return a
+
+            assert isinstance(f, Value)
+            assert isinstance(f.type, RankedTensorType)
+
+            # CHECK: %[[VAL_3:.*]] = tensor.empty() : tensor<16x16x16xf32>
+            z = tensor.empty((16, 16, 16), f32)
+
+            minor_id = AffineMap.get_minor_identity(3, 2)
+            id_map_2 = AffineMap.get_identity(3)
+
+            # CHECK: %[[VAL_4:.+]]:2 = linalg.generic {indexing_maps = [#map1, #map2, #map2], iterator_types = ["parallel", "parallel", "parallel"]} ins(%[[VAL_0]] : tensor<16x16xf32>) outs(%[[VAL_3]], %[[VAL_3]] : tensor<16x16x16xf32>, tensor<16x16x16xf32>) {
+            # CHECK: ^bb0(%in: f32, %out: f32, %out_1: f32):
+            # CHECK:   linalg.yield %in, %out : f32, f32
+            # CHECK: } -> (tensor<16x16x16xf32>, tensor<16x16x16xf32>)
+            @linalg.generic(
+                [x],
+                [z, z],
+                [minor_id, id_map_2, id_map_2],
+                [
+                    linalg.IteratorType.parallel,
+                    linalg.IteratorType.parallel,
+                    linalg.IteratorType.parallel,
+                ],
+            )
+            def g(a, b, c):
+                assert isinstance(a, Value)
+                assert isinstance(a.type, F32Type)
+                assert isinstance(b, Value)
+                assert isinstance(b.type, F32Type)
+                assert isinstance(c, Value)
+                assert isinstance(c.type, F32Type)
+                return a, b
+
+            assert isinstance(g, OpResultList)
+            assert len(g) == 2
+            assert isinstance(g[0].type, RankedTensorType)
+            assert isinstance(g[1].type, RankedTensorType)
+
+            # CHECK: %[[VAL_5:.*]] = memref.alloc() : memref<10x10xf32>
+            # CHECK: %[[VAL_6:.*]] = memref.alloc() : memref<10x10xf32>
+            xx = memref.alloc(memref_t, [], [])
+            yy = memref.alloc(memref_t, [], [])
+
+            # CHECK: linalg.generic {indexing_maps = [#map, #map], iterator_types = ["parallel", "parallel"]} ins(%[[VAL_5]] : memref<10x10xf32>) outs(%[[VAL_6]] : memref<10x10xf32>) {
+            # CHECK: ^bb0(%in: f32, %out: f32):
+            # CHECK:   linalg.yield %in : f32
+            # CHECK: }
+            @linalg.generic(
+                [xx],
+                [yy],
+                [id_map_1, id_map_1],
+                [linalg.IteratorType.parallel, linalg.IteratorType.parallel],
+            )
+            def f(a, b):
+                assert isinstance(a, Value)
+                assert isinstance(a.type, F32Type)
+                assert isinstance(b, Value)
+                assert isinstance(b.type, F32Type)
+                return a
+
+    module.operation.verify()
+    print(module)
+
+
+# CHECK-LABEL: TEST: testMatmulOp
+@run
+def testMatmulOp():
+    with Context(), Location.unknown():
+        module = Module.create()
+        f32 = F32Type.get()
+        with InsertionPoint(module.body):
+            a_shape = (4, 8)
+            b_shape = (8, 12)
+            b_transposed_shape = (12, 8)
+            c_shape = (4, 12)
+
+            dimM = ir.AffineDimExpr.get(0)
+            dimN = ir.AffineDimExpr.get(1)
+            dimK = ir.AffineDimExpr.get(2)
+
+            # CHECK: #[[$A_MAP:.*]] = affine_map<(d0, d1, d2) -> (d0, d2)>
+            # CHECK: #[[$BTrans_MAP:.*]] = affine_map<(d0, d1, d2) -> (d1, d2)>
+            # CHECK: #[[$C_MAP:.*]] = affine_map<(d0, d1, d2) -> (d0, d1)>
+            a_map = ir.AffineMap.get(3, 0, [dimM, dimK])
+            b_map = ir.AffineMap.get(3, 0, [dimK, dimN])
+            c_map = ir.AffineMap.get(3, 0, [dimM, dimN])
+            b_transposed_map = ir.AffineMap.get(3, 0, [dimN, dimK])
+
+            # CHECK: func.func @matmul_op(
+            @func.FuncOp.from_py_func(
+                # CHECK-SAME:                         %[[A:.*]]: tensor<4x8xf32>,
+                RankedTensorType.get(a_shape, f32),
+                # CHECK-SAME:                         %[[Amem:.*]]: memref<4x8xf32>,
+                MemRefType.get(a_shape, f32),
+                # CHECK-SAME:                         %[[B:.*]]: tensor<8x12xf32>,
+                RankedTensorType.get(b_shape, f32),
+                # CHECK-SAME:                         %[[Bmem:.*]]: memref<8x12xf32>,
+                MemRefType.get(b_shape, f32),
+                # CHECK-SAME:                         %[[BTrans:.*]]: tensor<12x8xf32>,
+                RankedTensorType.get(b_transposed_shape, f32),
+                # CHECK-SAME:                         %[[BTransmem:.*]]: memref<12x8xf32>,
+                MemRefType.get(b_transposed_shape, f32),
+                # CHECK-SAME:                         %[[C:.*]]: tensor<4x12xf32>,
+                RankedTensorType.get(c_shape, f32),
+                # CHECK-SAME:                         %[[Cmem:.*]]: memref<4x12xf32>)
+                MemRefType.get(c_shape, f32),
+            )
+            def matmul_op(A, Amem, B, Bmem, Btransposed, Btransposedmem, C, Cmem):
+                # CHECK: linalg.matmul ins(%[[A]], %[[B]] : tensor<4x8xf32>, tensor<8x12xf32>) outs(%[[C]] : tensor<4x12xf32>)
+                res = linalg.MatmulOp(
+                    result_tensors=(C.type,),
+                    inputs=(A, B),
+                    outputs=(C,),
+                )
+                linalg.fill_builtin_region(res.operation)
+                # CHECK: linalg.matmul ins(%[[A]], %[[B]] : tensor<4x8xf32>, tensor<8x12xf32>) outs(%[[C]] : tensor<4x12xf32>)
+                res = linalg.matmul(A, B, outs=(C,))
+
+                # CHECK: linalg.matmul indexing_maps = [#[[$A_MAP]], #[[$BTrans_MAP]], #[[$C_MAP]]] ins(%[[A]], %[[BTrans]] : tensor<4x8xf32>, tensor<12x8xf32>) outs(%[[C]] : tensor<4x12xf32>)
+                res = linalg.MatmulOp(
+                    result_tensors=(C.type,),
+                    inputs=(A, Btransposed),
+                    outputs=(C,),
+                    indexing_maps=[a_map, b_transposed_map, c_map],
+                )
+                linalg.fill_builtin_region(res.operation)
+                # CHECK: linalg.matmul indexing_maps = [#[[$A_MAP]], #[[$BTrans_MAP]], #[[$C_MAP]]] ins(%[[A]], %[[BTrans]] : tensor<4x8xf32>, tensor<12x8xf32>) outs(%[[C]] : tensor<4x12xf32>)
+                res = linalg.matmul(
+                    A,
+                    Btransposed,
+                    outs=(C,),
+                    indexing_maps=[a_map, b_transposed_map, c_map],
+                )
+
+                # And now with memrefs...
+
+                # CHECK: linalg.matmul ins(%[[Amem]], %[[Bmem]] : memref<4x8xf32>, memref<8x12xf32>) outs(%[[Cmem]] : memref<4x12xf32>)
+                res = linalg.MatmulOp(
+                    result_tensors=[],
+                    inputs=(Amem, Bmem),
+                    outputs=(Cmem,),
+                )
+                linalg.fill_builtin_region(res.operation)
+                # CHECK: linalg.matmul ins(%[[Amem]], %[[Bmem]] : memref<4x8xf32>, memref<8x12xf32>) outs(%[[Cmem]] : memref<4x12xf32>)
+                linalg.matmul(Amem, Bmem, outs=(Cmem,))
+
+                # CHECK: linalg.matmul indexing_maps = [#[[$A_MAP]], #[[$BTrans_MAP]], #[[$C_MAP]]] ins(%[[Amem]], %[[BTransmem]] : memref<4x8xf32>, memref<12x8xf32>) outs(%[[Cmem]] : memref<4x12xf32>)
+                res = linalg.MatmulOp(
+                    result_tensors=[],
+                    inputs=(Amem, Btransposedmem),
+                    outputs=(Cmem,),
+                    indexing_maps=[a_map, b_transposed_map, c_map],
+                )
+                linalg.fill_builtin_region(res.operation)
+                # CHECK: linalg.matmul indexing_maps = [#[[$A_MAP]], #[[$BTrans_MAP]], #[[$C_MAP]]] ins(%[[Amem]], %[[BTransmem]] : memref<4x8xf32>, memref<12x8xf32>) outs(%[[Cmem]] : memref<4x12xf32>)
+                linalg.matmul(
+                    Amem,
+                    Btransposedmem,
+                    outs=(Cmem,),
+                    indexing_maps=[a_map, b_transposed_map, c_map],
+                )
+
+        print(module)
+
+
+# CHECK-LABEL: TEST: testContractOp
+@run
+def testContractOp():
+    with Context(), Location.unknown():
+        module = Module.create()
+        f32 = F32Type.get()
+        with InsertionPoint(module.body):
+            a_shape = (4, 8)
+            b_shape = (8, 12)
+            b_transposed_shape = (12, 8)
+            c_shape = (4, 12)
+
+            dimM = ir.AffineDimExpr.get(0)
+            dimN = ir.AffineDimExpr.get(1)
+            dimK = ir.AffineDimExpr.get(2)
+
+            # CHECK: #[[$A_MAP:.*]] = affine_map<(d0, d1, d2) -> (d0, d2)>
+            # CHECK: #[[$B_MAP:.*]] = affine_map<(d0, d1, d2) -> (d2, d1)>
+            # CHECK: #[[$C_MAP:.*]] = affine_map<(d0, d1, d2) -> (d0, d1)>
+            # CHECK: #[[$BTrans_MAP:.*]] = affine_map<(d0, d1, d2) -> (d1, d2)>
+            a_map = ir.AffineMap.get(3, 0, [dimM, dimK])
+            b_map = ir.AffineMap.get(3, 0, [dimK, dimN])
+            c_map = ir.AffineMap.get(3, 0, [dimM, dimN])
+            b_transposed_map = ir.AffineMap.get(3, 0, [dimN, dimK])
+
+            # CHECK: func.func @matmul_as_contract_op(
+            @func.FuncOp.from_py_func(
+                # CHECK-SAME:                         %[[A:.*]]: tensor<4x8xf32>,
+                RankedTensorType.get(a_shape, f32),
+                # CHECK-SAME:                         %[[Amem:.*]]: memref<4x8xf32>,
+                MemRefType.get(a_shape, f32),
+                # CHECK-SAME:                         %[[B:.*]]: tensor<8x12xf32>,
+                RankedTensorType.get(b_shape, f32),
+                # CHECK-SAME:                         %[[Bmem:.*]]: memref<8x12xf32>,
+                MemRefType.get(b_shape, f32),
+                # CHECK-SAME:                         %[[BTrans:.*]]: tensor<12x8xf32>,
+                RankedTensorType.get(b_transposed_shape, f32),
+                # CHECK-SAME:                         %[[BTransmem:.*]]: memref<12x8xf32>,
+                MemRefType.get(b_transposed_shape, f32),
+                # CHECK-SAME:                         %[[C:.*]]: tensor<4x12xf32>,
+                RankedTensorType.get(c_shape, f32),
+                # CHECK-SAME:                         %[[Cmem:.*]]: memref<4x12xf32>)
+                MemRefType.get(c_shape, f32),
+            )
+            def matmul_as_contract_op(
+                A, Amem, B, Bmem, Btransposed, Btransposedmem, C, Cmem
+            ):
+                # CHECK: linalg.contract indexing_maps = [#[[$A_MAP]], #[[$B_MAP]], #[[$C_MAP]]] ins(%[[A]], %[[B]] : tensor<4x8xf32>, tensor<8x12xf32>) outs(%[[C]] : tensor<4x12xf32>)
+                op4 = linalg.ContractOp(
+                    result_tensors=(C.type,),
+                    inputs=(A, B),
+                    outputs=(C,),
+                    indexing_maps=[a_map, b_map, c_map],
+                )
+                linalg.fill_builtin_region(op4.operation)
+                # CHECK: linalg.contract indexing_maps = [#[[$A_MAP]], #[[$B_MAP]], #[[$C_MAP]]] ins(%[[A]], %[[B]] : tensor<4x8xf32>, tensor<8x12xf32>) outs(%[[C]] : tensor<4x12xf32>)
+                op5 = linalg.contract(
+                    A, B, outs=(C,), indexing_maps=[a_map, b_map, c_map]
+                )
+
+                # CHECK: linalg.contract indexing_maps = [#[[$A_MAP]], #[[$BTrans_MAP]], #[[$C_MAP]]] ins(%[[A]], %[[BTrans]] : tensor<4x8xf32>, tensor<12x8xf32>) outs(%[[C]] : tensor<4x12xf32>)
+                op4 = linalg.ContractOp(
+                    result_tensors=(C.type,),
+                    inputs=(A, Btransposed),
+                    outputs=(C,),
+                    indexing_maps=[a_map, b_transposed_map, c_map],
+                )
+                linalg.fill_builtin_region(op4.operation)
+                # CHECK: linalg.contract indexing_maps = [#[[$A_MAP]], #[[$BTrans_MAP]], #[[$C_MAP]]] ins(%[[A]], %[[BTrans]] : tensor<4x8xf32>, tensor<12x8xf32>) outs(%[[C]] : tensor<4x12xf32>)
+                op5 = linalg.contract(
+                    A,
+                    Btransposed,
+                    outs=(C,),
+                    indexing_maps=[a_map, b_transposed_map, c_map],
+                )
+                # And now with memrefs...
+
+                # CHECK: linalg.contract indexing_maps = [#[[$A_MAP]], #[[$B_MAP]], #[[$C_MAP]]] ins(%[[Amem]], %[[Bmem]] : memref<4x8xf32>, memref<8x12xf32>) outs(%[[Cmem]] : memref<4x12xf32>)
+                op4 = linalg.ContractOp(
+                    result_tensors=[],
+                    inputs=(Amem, Bmem),
+                    outputs=(Cmem,),
+                    indexing_maps=[a_map, b_map, c_map],
+                )
+                linalg.fill_builtin_region(op4.operation)
+                # CHECK: linalg.contract indexing_maps = [#[[$A_MAP]], #[[$B_MAP]], #[[$C_MAP]]] ins(%[[Amem]], %[[Bmem]] : memref<4x8xf32>, memref<8x12xf32>) outs(%[[Cmem]] : memref<4x12xf32>)
+                linalg.contract(
+                    Amem, Bmem, outs=(Cmem,), indexing_maps=[a_map, b_map, c_map]
+                )
+
+                # CHECK: linalg.contract indexing_maps = [#[[$A_MAP]], #[[$BTrans_MAP]], #[[$C_MAP]]] ins(%[[Amem]], %[[BTransmem]] : memref<4x8xf32>, memref<12x8xf32>) outs(%[[Cmem]] : memref<4x12xf32>)
+                op4 = linalg.ContractOp(
+                    result_tensors=[],
+                    inputs=(Amem, Btransposedmem),
+                    outputs=(Cmem,),
+                    indexing_maps=[a_map, b_transposed_map, c_map],
+                )
+                linalg.fill_builtin_region(op4.operation)
+                # CHECK: linalg.contract indexing_maps = [#[[$A_MAP]], #[[$BTrans_MAP]], #[[$C_MAP]]] ins(%[[Amem]], %[[BTransmem]] : memref<4x8xf32>, memref<12x8xf32>) outs(%[[Cmem]] : memref<4x12xf32>)
+                linalg.contract(
+                    Amem,
+                    Btransposedmem,
+                    outs=(Cmem,),
+                    indexing_maps=[a_map, b_transposed_map, c_map],
+                )
+
+        print(module)
diff --git a/mlir/test/python/ir/array_attributes.py b/mlir/test/python/ir/array_attributes.py
index 256a69a939658..ef1d835fc6401 100644
--- a/mlir/test/python/ir/array_attributes.py
+++ b/mlir/test/python/ir/array_attributes.py
@@ -572,6 +572,10 @@ def testGetDenseElementsIndex():
         print(arr)
         # CHECK: True
         print(arr.dtype == np.int64)
+        array = np.array([1, 2, 3], dtype=np.int64)
+        attr = DenseIntElementsAttr.get(array, type=VectorType.get([3], idx_type))
+        # CHECK: [1, 2, 3]
+        print(list(DenseIntElementsAttr(attr)))
 
 
 # CHECK-LABEL: TEST: testGetDenseResourceElementsAttr
diff --git a/mlir/test/python/ir/attributes.py b/mlir/test/python/ir/attributes.py
index 00c3e1b4decdb..2f3c4460d3f59 100644
--- a/mlir/test/python/ir/attributes.py
+++ b/mlir/test/python/ir/attributes.py
@@ -366,6 +366,10 @@ def testDenseIntAttr():
         # CHECK: i1
         print(ShapedType(a.type).element_type)
 
+        shape = Attribute.parse("dense<[0, 1, 2, 3]> : vector<4xindex>")
+        # CHECK: attr: dense<[0, 1, 2, 3]>
+        print("attr:", shape)
+
 
 @run
 def testDenseArrayGetItem():
diff --git a/mlir/test/python/ir/operation.py b/mlir/test/python/ir/operation.py
index d5d72c98b66ad..c2d3aed8808b4 100644
--- a/mlir/test/python/ir/operation.py
+++ b/mlir/test/python/ir/operation.py
@@ -583,14 +583,24 @@ def testOperationPrint():
         r"""
     func.func @f1(%arg0: i32) -> i32 {
       %0 = arith.constant dense<[1, 2, 3, 4]> : tensor<4xi32> loc("nom")
+      %1 = arith.constant dense_resource<resource1> : tensor<3xi64>
       return %arg0 : i32
     }
+
+    {-#
+      dialect_resources: {
+          builtin: {
+            resource1: "0x08000000010000000000000002000000000000000300000000000000"
+          }
+        }
+      #-}
   """,
         ctx,
     )
 
     # Test print to stdout.
     # CHECK: return %arg0 : i32
+    # CHECK: resource1: "0x08
     module.operation.print()
 
     # Test print to text file.
@@ -607,7 +617,8 @@ def testOperationPrint():
     module.operation.write_bytecode(bytecode_stream, desired_version=1)
     bytecode = bytecode_stream.getvalue()
     assert bytecode.startswith(b"ML\xefR"), "Expected bytecode to start with MLïR"
-    module_roundtrip = Module.parse(bytecode, ctx)
+    ctx2 = Context()
+    module_roundtrip = Module.parse(bytecode, ctx2)
     f = io.StringIO()
     module_roundtrip.operation.print(file=f)
     roundtrip_value = f.getvalue()
@@ -633,7 +644,8 @@ def testOperationPrint():
 
     # Test print with options.
     # CHECK: value = dense_resource<__elided__> : tensor<4xi32>
-    # CHECK: "func.return"(%arg0) : (i32) -> () -:4:7
+    # CHECK: "func.return"(%arg0) : (i32) -> () -:5:7
+    # CHECK-NOT: resource1: "0x08
     module.operation.print(
         large_elements_limit=2,
         enable_debug_info=True,
diff --git a/mlir/tools/mlir-opt/mlir-opt.cpp b/mlir/tools/mlir-opt/mlir-opt.cpp
index 960f7037a1b61..74007d01347ae 100644
--- a/mlir/tools/mlir-opt/mlir-opt.cpp
+++ b/mlir/tools/mlir-opt/mlir-opt.cpp
@@ -87,6 +87,7 @@ void registerTestComposeSubView();
 void registerTestCompositePass();
 void registerTestConstantFold();
 void registerTestControlFlowSink();
+void registerTestConvertToSPIRVPass();
 void registerTestDataLayoutPropagation();
 void registerTestDataLayoutQuery();
 void registerTestDeadCodeAnalysisPass();
@@ -226,6 +227,7 @@ void registerTestPasses() {
   mlir::test::registerTestCompositePass();
   mlir::test::registerTestConstantFold();
   mlir::test::registerTestControlFlowSink();
+  mlir::test::registerTestConvertToSPIRVPass();
   mlir::test::registerTestDataLayoutPropagation();
   mlir::test::registerTestDataLayoutQuery();
   mlir::test::registerTestDeadCodeAnalysisPass();
diff --git a/mlir/tools/mlir-tblgen/OpDefinitionsGen.cpp b/mlir/tools/mlir-tblgen/OpDefinitionsGen.cpp
index a970cbc5caceb..629e863dac5e3 100644
--- a/mlir/tools/mlir-tblgen/OpDefinitionsGen.cpp
+++ b/mlir/tools/mlir-tblgen/OpDefinitionsGen.cpp
@@ -1334,8 +1334,9 @@ static void emitAttrGetterWithReturnType(FmtContext &fctx,
       PrintFatalError("DefaultValuedAttr of type " + attr.getAttrDefName() +
                       " must have a constBuilder");
     }
-    std::string defaultValue = std::string(
-        tgfmt(attr.getConstBuilderTemplate(), &fctx, attr.getDefaultValue()));
+    std::string defaultValue =
+        std::string(tgfmt(attr.getConstBuilderTemplate(), &fctx,
+                          tgfmt(attr.getDefaultValue(), &fctx)));
     body << "    if (!attr)\n      return "
          << tgfmt(attr.getConvertFromStorageCall(),
                   &fctx.withSelf(defaultValue))
@@ -1467,6 +1468,7 @@ void OpEmitter::genPropertiesSupport() {
         os << "   if (!attr) attr = dict.get(\"result_segment_sizes\");";
       }
 
+      fctx.withBuilder(odsBuilder);
       setPropMethod << "{\n"
                     << formatv(propFromAttrFmt,
                                tgfmt(prop.getConvertFromAttributeCall(),
@@ -1479,7 +1481,7 @@ void OpEmitter::genPropertiesSupport() {
                                  prop.getStorageTypeValueOverride());
       } else if (prop.hasDefaultValue()) {
         setPropMethod << formatv(attrGetDefaultFmt, name,
-                                 prop.getDefaultValue());
+                                 tgfmt(prop.getDefaultValue(), &fctx));
       } else {
         setPropMethod << formatv(attrGetNoDefaultFmt, name);
       }
@@ -2919,6 +2921,9 @@ getBuilderSignature(const Builder &builder) {
   arguments.emplace_back("::mlir::OpBuilder &", odsBuilder);
   arguments.emplace_back("::mlir::OperationState &", builderOpState);
 
+  FmtContext fctx;
+  fctx.withBuilder(odsBuilder);
+
   for (unsigned i = 0, e = params.size(); i < e; ++i) {
     // If no name is provided, generate one.
     std::optional<StringRef> paramName = params[i].getName();
@@ -2931,7 +2936,7 @@ getBuilderSignature(const Builder &builder) {
       defaultValue = *defaultParamValue;
 
     arguments.emplace_back(params[i].getCppType(), std::move(name),
-                           defaultValue);
+                           tgfmt(defaultValue, &fctx));
   }
 
   return arguments;
@@ -3189,6 +3194,9 @@ void OpEmitter::buildParamList(SmallVectorImpl<MethodParameter> &paramList,
     }
   }
 
+  FmtContext fctx;
+  fctx.withBuilder(odsBuilder);
+
   for (int i = 0, e = op.getNumArgs(), numOperands = 0; i < e; ++i) {
     Argument arg = op.getArg(i);
     if (const auto *operand =
@@ -3210,7 +3218,7 @@ void OpEmitter::buildParamList(SmallVectorImpl<MethodParameter> &paramList,
       StringRef type = prop.getInterfaceType();
       std::string defaultValue;
       if (prop.hasDefaultValue() && i >= defaultValuedAttrLikeStartIndex) {
-        defaultValue = prop.getDefaultValue();
+        defaultValue = tgfmt(prop.getDefaultValue(), &fctx);
       }
       bool isOptional = prop.hasDefaultValue();
       paramList.emplace_back(type, propArg->name, StringRef(defaultValue),
@@ -3242,7 +3250,7 @@ void OpEmitter::buildParamList(SmallVectorImpl<MethodParameter> &paramList,
     if (i >= defaultValuedAttrStartIndex) {
       if (attrParamKind == AttrParamKind::UnwrappedValue &&
           canUseUnwrappedRawValue(attr))
-        defaultValue += attr.getDefaultValue();
+        defaultValue += tgfmt(attr.getDefaultValue(), &fctx);
       else
         defaultValue += "nullptr";
     }
@@ -4172,6 +4180,9 @@ OpOperandAdaptorEmitter::OpOperandAdaptorEmitter(
       staticVerifierEmitter(staticVerifierEmitter),
       emitHelper(op, /*emitForOp=*/false) {
 
+  FmtContext fctx;
+  fctx.withBuilder(odsBuilder);
+
   genericAdaptorBase.declare<VisibilityDeclaration>(Visibility::Public);
   bool useProperties = emitHelper.hasProperties();
   if (useProperties) {
@@ -4212,7 +4223,7 @@ OpOperandAdaptorEmitter::OpOperandAdaptorEmitter(
         if (prop.hasStorageTypeValueOverride())
           os << " = " << prop.getStorageTypeValueOverride();
         else if (prop.hasDefaultValue())
-          os << " = " << prop.getDefaultValue();
+          os << " = " << tgfmt(prop.getDefaultValue(), &fctx);
         comparatorOs << "        rhs." << name << " == this->" << name
                      << " &&\n";
         // Emit accessors using the interface type.
@@ -4454,7 +4465,6 @@ OpOperandAdaptorEmitter::OpOperandAdaptorEmitter(
   if (auto *m = genericAdaptor.addMethod("RangeT", "getOperands"))
     m->body() << "  return odsOperands;";
 
-  FmtContext fctx;
   fctx.withBuilder("::mlir::Builder(odsAttrs.getContext())");
 
   // Generate named accessor with Attribute return type.
@@ -4481,8 +4491,9 @@ OpOperandAdaptorEmitter::OpOperandAdaptorEmitter(
       // Use the default value if attribute is not set.
       // TODO: this is inefficient, we are recreating the attribute for every
       // call. This should be set instead.
-      std::string defaultValue = std::string(
-          tgfmt(attr.getConstBuilderTemplate(), &fctx, attr.getDefaultValue()));
+      std::string defaultValue =
+          std::string(tgfmt(attr.getConstBuilderTemplate(), &fctx,
+                            tgfmt(attr.getDefaultValue(), &fctx)));
       body << "if (!attr)\n  attr = " << defaultValue << ";\n";
     }
     body << "return attr;\n";
diff --git a/mlir/tools/mlir-tblgen/OpDocGen.cpp b/mlir/tools/mlir-tblgen/OpDocGen.cpp
index 1c394f5680a5c..43d406e4340f7 100644
--- a/mlir/tools/mlir-tblgen/OpDocGen.cpp
+++ b/mlir/tools/mlir-tblgen/OpDocGen.cpp
@@ -54,12 +54,12 @@ cl::opt<bool> allowHugoSpecificFeatures(
     cl::cat(docCat));
 
 void mlir::tblgen::emitSummary(StringRef summary, raw_ostream &os) {
-  if (!summary.empty()) {
-    StringRef trimmed = summary.trim();
-    char first = std::toupper(trimmed.front());
-    StringRef rest = trimmed.drop_front();
-    os << "\n_" << first << rest << "_\n\n";
-  }
+  if (summary.empty())
+    return;
+  StringRef trimmed = summary.trim();
+  char first = std::toupper(trimmed.front());
+  StringRef rest = trimmed.drop_front();
+  os << "\n_" << first << rest << "_\n";
 }
 
 // Emit the description by aligning the text to the left per line (e.g.,
@@ -69,6 +69,9 @@ void mlir::tblgen::emitSummary(StringRef summary, raw_ostream &os) {
 // in a way the user wanted but has some additional indenting due to being
 // nested in the op definition.
 void mlir::tblgen::emitDescription(StringRef description, raw_ostream &os) {
+  if (description.empty())
+    return;
+  os << "\n";
   raw_indented_ostream ros(os);
   StringRef trimmed = description.rtrim(" \t");
   ros.printReindented(trimmed);
@@ -80,6 +83,7 @@ void mlir::tblgen::emitDescriptionComment(StringRef description,
                                           raw_ostream &os, StringRef prefix) {
   if (description.empty())
     return;
+  os << "\n";
   raw_indented_ostream ros(os);
   StringRef trimmed = description.rtrim(" \t");
   ros.printReindented(trimmed, (Twine(prefix) + "/// ").str());
@@ -87,22 +91,14 @@ void mlir::tblgen::emitDescriptionComment(StringRef description,
     ros << "\n";
 }
 
-// Emits `str` with trailing newline if not empty.
-static void emitIfNotEmpty(StringRef str, raw_ostream &os) {
-  if (!str.empty()) {
-    emitDescription(str, os);
-    os << "\n";
-  }
-}
-
 /// Emit the given named constraint.
 template <typename T>
 static void emitNamedConstraint(const T &it, raw_ostream &os) {
   if (!it.name.empty())
     os << "| `" << it.name << "`";
   else
-    os << "&laquo;unnamed&raquo;";
-  os << " | " << it.constraint.getSummary() << "\n";
+    os << "| &laquo;unnamed&raquo;";
+  os << " | " << it.constraint.getSummary() << " |\n";
 }
 
 //===----------------------------------------------------------------------===//
@@ -112,6 +108,8 @@ static void emitNamedConstraint(const T &it, raw_ostream &os) {
 /// Emit the assembly format of an operation.
 static void emitAssemblyFormat(StringRef opName, StringRef format,
                                raw_ostream &os) {
+  if (format.empty())
+    return;
   os << "\nSyntax:\n\n```\noperation ::= `" << opName << "` ";
 
   // Print the assembly format aligned.
@@ -124,7 +122,7 @@ static void emitAssemblyFormat(StringRef opName, StringRef format,
     if (!formatChunk.empty())
       os.indent(indent) << formatChunk << "\n";
   } while (!split.second.empty());
-  os << "```\n\n";
+  os << "```\n";
 }
 
 /// Place `text` between backticks so that the Markdown processor renders it as
@@ -199,7 +197,7 @@ static void emitOpDoc(const Operator &op, raw_ostream &os) {
   std::string classNameStr = op.getQualCppClassName();
   StringRef className = classNameStr;
   (void)className.consume_front(stripPrefix);
-  os << formatv("### `{0}` ({1})\n", op.getOperationName(), className);
+  os << formatv("\n### `{0}` ({1})\n", op.getOperationName(), className);
 
   // Emit the summary, syntax, and description if present.
   if (op.hasSummary())
@@ -281,8 +279,8 @@ static void emitSourceLink(StringRef inputFilename, raw_ostream &os) {
 
   StringRef inputFromMlirInclude = inputFilename.substr(pathBegin);
 
-  os << "[source](https://github.com/llvm/llvm-project/blob/main/"
-     << inputFromMlirInclude << ")\n\n";
+  os << "\n[source](https://github.com/llvm/llvm-project/blob/main/"
+     << inputFromMlirInclude << ")\n";
 }
 
 static void emitOpDoc(const RecordKeeper &records, raw_ostream &os) {
@@ -299,9 +297,9 @@ static void emitOpDoc(const RecordKeeper &records, raw_ostream &os) {
 //===----------------------------------------------------------------------===//
 
 static void emitAttrDoc(const Attribute &attr, raw_ostream &os) {
-  os << "### " << attr.getSummary() << "\n\n";
+  os << "\n### " << attr.getSummary() << "\n";
   emitDescription(attr.getDescription(), os);
-  os << "\n\n";
+  os << "\n";
 }
 
 //===----------------------------------------------------------------------===//
@@ -309,9 +307,9 @@ static void emitAttrDoc(const Attribute &attr, raw_ostream &os) {
 //===----------------------------------------------------------------------===//
 
 static void emitTypeDoc(const Type &type, raw_ostream &os) {
-  os << "### " << type.getSummary() << "\n\n";
+  os << "\n### " << type.getSummary() << "\n";
   emitDescription(type.getDescription(), os);
-  os << "\n\n";
+  os << "\n";
 }
 
 //===----------------------------------------------------------------------===//
@@ -342,11 +340,11 @@ static void emitAttrOrTypeDefAssemblyFormat(const AttrOrTypeDef &def,
 }
 
 static void emitAttrOrTypeDefDoc(const AttrOrTypeDef &def, raw_ostream &os) {
-  os << formatv("### {0}\n", def.getCppClassName());
+  os << formatv("\n### {0}\n", def.getCppClassName());
 
   // Emit the summary if present.
   if (def.hasSummary())
-    os << "\n" << def.getSummary() << "\n";
+    emitSummary(def.getSummary(), os);
 
   // Emit the syntax if present.
   if (def.getMnemonic() && !def.hasCustomAssemblyFormat())
@@ -354,7 +352,6 @@ static void emitAttrOrTypeDefDoc(const AttrOrTypeDef &def, raw_ostream &os) {
 
   // Emit the description if present.
   if (def.hasDescription()) {
-    os << "\n";
     mlir::tblgen::emitDescription(def.getDescription(), os);
   }
 
@@ -363,11 +360,11 @@ static void emitAttrOrTypeDefDoc(const AttrOrTypeDef &def, raw_ostream &os) {
   if (!parameters.empty()) {
     os << "\n#### Parameters:\n\n";
     os << "| Parameter | C++ type | Description |\n"
-       << "| :-------: | :-------: | ----------- |\n";
+       << "| :-------: | :-------: | ----------- |";
     for (const auto &it : parameters) {
       auto desc = it.getSummary();
-      os << "| " << it.getName() << " | `" << it.getCppType() << "` | "
-         << (desc ? *desc : "") << " |\n";
+      os << "\n| " << it.getName() << " | `" << it.getCppType() << "` | "
+         << (desc ? *desc : "") << " |";
     }
   }
 
@@ -388,20 +385,19 @@ static void emitAttrOrTypeDefDoc(const RecordKeeper &records, raw_ostream &os,
 //===----------------------------------------------------------------------===//
 
 static void emitEnumDoc(const EnumAttr &def, raw_ostream &os) {
-  os << formatv("### {0}\n", def.getEnumClassName());
+  os << formatv("\n### {0}\n", def.getEnumClassName());
 
   // Emit the summary if present.
-  if (!def.getSummary().empty())
-    os << "\n" << def.getSummary() << "\n";
+  emitSummary(def.getSummary(), os);
 
   // Emit case documentation.
   std::vector<EnumAttrCase> cases = def.getAllCases();
   os << "\n#### Cases:\n\n";
   os << "| Symbol | Value | String |\n"
-     << "| :----: | :---: | ------ |\n";
+     << "| :----: | :---: | ------ |";
   for (const auto &it : cases) {
-    os << "| " << it.getSymbol() << " | `" << it.getValue() << "` | "
-       << it.getStr() << " |\n";
+    os << "\n| " << it.getSymbol() << " | `" << it.getValue() << "` | "
+       << it.getStr() << " |";
   }
 
   os << "\n";
@@ -447,7 +443,7 @@ static void emitBlock(ArrayRef<Attribute> attributes, StringRef inputFilename,
                       ArrayRef<Type> types, ArrayRef<TypeDef> typeDefs,
                       ArrayRef<EnumAttr> enums, raw_ostream &os) {
   if (!ops.empty()) {
-    os << "## Operations\n\n";
+    os << "\n## Operations\n";
     emitSourceLink(inputFilename, os);
     for (const OpDocGroup &grouping : ops) {
       bool nested = !grouping.summary.empty();
@@ -455,9 +451,9 @@ static void emitBlock(ArrayRef<Attribute> attributes, StringRef inputFilename,
           nested,
           [&](raw_ostream &os) {
             if (nested) {
-              os << "## " << StringRef(grouping.summary).trim() << "\n\n";
+              os << "\n## " << StringRef(grouping.summary).trim() << "\n";
               emitDescription(grouping.description, os);
-              os << "\n\n";
+              os << "\n";
             }
             for (const Operator &op : grouping.ops) {
               emitOpDoc(op, os);
@@ -468,32 +464,32 @@ static void emitBlock(ArrayRef<Attribute> attributes, StringRef inputFilename,
   }
 
   if (!attributes.empty()) {
-    os << "## Attribute constraints\n\n";
+    os << "\n## Attribute constraints\n";
     for (const Attribute &attr : attributes)
       emitAttrDoc(attr, os);
   }
 
   if (!attrDefs.empty()) {
-    os << "## Attributes\n\n";
+    os << "\n## Attributes\n";
     for (const AttrDef &def : attrDefs)
       emitAttrOrTypeDefDoc(def, os);
   }
 
   // TODO: Add link between use and def for types
   if (!types.empty()) {
-    os << "## Type constraints\n\n";
+    os << "\n## Type constraints\n";
     for (const Type &type : types)
       emitTypeDoc(type, os);
   }
 
   if (!typeDefs.empty()) {
-    os << "## Types\n\n";
+    os << "\n## Types\n";
     for (const TypeDef &def : typeDefs)
       emitAttrOrTypeDefDoc(def, os);
   }
 
   if (!enums.empty()) {
-    os << "## Enums\n\n";
+    os << "\n## Enums\n";
     for (const EnumAttr &def : enums)
       emitEnumDoc(def, os);
   }
@@ -504,14 +500,14 @@ static void emitDialectDoc(const Dialect &dialect, StringRef inputFilename,
                            ArrayRef<AttrDef> attrDefs, ArrayRef<OpDocGroup> ops,
                            ArrayRef<Type> types, ArrayRef<TypeDef> typeDefs,
                            ArrayRef<EnumAttr> enums, raw_ostream &os) {
-  os << "# '" << dialect.getName() << "' Dialect\n\n";
-  emitIfNotEmpty(dialect.getSummary(), os);
-  emitIfNotEmpty(dialect.getDescription(), os);
+  os << "\n# '" << dialect.getName() << "' Dialect\n";
+  emitSummary(dialect.getSummary(), os);
+  emitDescription(dialect.getDescription(), os);
 
   // Generate a TOC marker except if description already contains one.
   Regex r("^[[:space:]]*\\[TOC\\]$", Regex::RegexFlags::Newline);
   if (!r.match(dialect.getDescription()))
-    os << "[TOC]\n\n";
+    os << "\n[TOC]\n";
 
   emitBlock(attributes, inputFilename, attrDefs, ops, types, typeDefs, enums,
             os);
diff --git a/mlir/tools/mlir-tblgen/OpFormatGen.cpp b/mlir/tools/mlir-tblgen/OpFormatGen.cpp
index f03a3bfd398ed..fe724e86d6707 100644
--- a/mlir/tools/mlir-tblgen/OpFormatGen.cpp
+++ b/mlir/tools/mlir-tblgen/OpFormatGen.cpp
@@ -1999,7 +1999,7 @@ static void genNonDefaultValueCheck(MethodBody &body, const Operator &op,
     fctx.withBuilder("::mlir::OpBuilder((*this)->getContext())");
     body << getter << "Attr() != "
          << tgfmt(attr.getConstBuilderTemplate(), &fctx,
-                  attr.getDefaultValue());
+                  tgfmt(attr.getDefaultValue(), &fctx));
   }
   if (optionalAndDefault)
     body << ")";
@@ -2007,8 +2007,10 @@ static void genNonDefaultValueCheck(MethodBody &body, const Operator &op,
 
 static void genNonDefaultValueCheck(MethodBody &body, const Operator &op,
                                     PropertyVariable &propElement) {
-  body << op.getGetterName(propElement.getVar()->name)
-       << "() != " << propElement.getVar()->prop.getDefaultValue();
+  FmtContext fctx;
+  fctx.withBuilder("::mlir::OpBuilder((*this)->getContext())");
+  body << op.getGetterName(propElement.getVar()->name) << "() != "
+       << tgfmt(propElement.getVar()->prop.getDefaultValue(), &fctx);
 }
 
 /// Elide the variadic segment size attributes if necessary.
@@ -2045,8 +2047,9 @@ static void genPropDictPrinter(OperationFormat &fmt, Operator &op,
       const StringRef &name = namedAttr.name;
       FmtContext fctx;
       fctx.withBuilder("odsBuilder");
-      std::string defaultValue = std::string(
-          tgfmt(attr.getConstBuilderTemplate(), &fctx, attr.getDefaultValue()));
+      std::string defaultValue =
+          std::string(tgfmt(attr.getConstBuilderTemplate(), &fctx,
+                            tgfmt(attr.getDefaultValue(), &fctx)));
       body << "  {\n";
       body << "     ::mlir::Builder odsBuilder(getContext());\n";
       body << "     ::mlir::Attribute attr = " << op.getGetterName(name)
@@ -2059,8 +2062,10 @@ static void genPropDictPrinter(OperationFormat &fmt, Operator &op,
   // Similarly, elide default-valued properties.
   for (const NamedProperty &prop : op.getProperties()) {
     if (prop.prop.hasDefaultValue()) {
+      FmtContext fctx;
+      fctx.withBuilder("odsBuilder");
       body << "  if (" << op.getGetterName(prop.name)
-           << "() == " << prop.prop.getDefaultValue() << ") {";
+           << "() == " << tgfmt(prop.prop.getDefaultValue(), &fctx) << ") {";
       body << "    elidedProps.push_back(\"" << prop.name << "\");\n";
       body << "  }\n";
     }
@@ -2094,8 +2099,9 @@ static void genAttrDictPrinter(OperationFormat &fmt, Operator &op,
       const StringRef &name = namedAttr.name;
       FmtContext fctx;
       fctx.withBuilder("odsBuilder");
-      std::string defaultValue = std::string(
-          tgfmt(attr.getConstBuilderTemplate(), &fctx, attr.getDefaultValue()));
+      std::string defaultValue =
+          std::string(tgfmt(attr.getConstBuilderTemplate(), &fctx,
+                            tgfmt(attr.getDefaultValue(), &fctx)));
       body << "  {\n";
       body << "     ::mlir::Builder odsBuilder(getContext());\n";
       body << "     ::mlir::Attribute attr = " << op.getGetterName(name)
diff --git a/mlir/tools/mlir-tblgen/OpInterfacesGen.cpp b/mlir/tools/mlir-tblgen/OpInterfacesGen.cpp
index 1f1b1d9a34039..dcd68e6c2d636 100644
--- a/mlir/tools/mlir-tblgen/OpInterfacesGen.cpp
+++ b/mlir/tools/mlir-tblgen/OpInterfacesGen.cpp
@@ -627,8 +627,8 @@ static void emitInterfaceDoc(const Record &interfaceDef, raw_ostream &os) {
   Interface interface(&interfaceDef);
 
   // Emit the interface name followed by the description.
-  os << "## " << interface.getName() << " (`" << interfaceDef.getName()
-     << "`)\n\n";
+  os << "\n## " << interface.getName() << " (`" << interfaceDef.getName()
+     << "`)\n";
   if (auto description = interface.getDescription())
     mlir::tblgen::emitDescription(*description, os);
 
@@ -636,7 +636,7 @@ static void emitInterfaceDoc(const Record &interfaceDef, raw_ostream &os) {
   os << "\n### Methods:\n";
   for (const auto &method : interface.getMethods()) {
     // Emit the method name.
-    os << "#### `" << method.getName() << "`\n\n```c++\n";
+    os << "\n#### `" << method.getName() << "`\n\n```c++\n";
 
     // Emit the method signature.
     if (method.isStatic())
@@ -656,13 +656,13 @@ static void emitInterfaceDoc(const Record &interfaceDef, raw_ostream &os) {
     if (!method.getBody())
       os << "\nNOTE: This method *must* be implemented by the user.";
 
-    os << "\n\n";
+    os << "\n";
   }
 }
 
 bool InterfaceGenerator::emitInterfaceDocs() {
   os << "<!-- Autogenerated by mlir-tblgen; don't manually edit -->\n";
-  os << "# " << interfaceBaseType << " definitions\n";
+  os << "\n# " << interfaceBaseType << " definitions\n";
 
   for (const auto *def : defs)
     emitInterfaceDoc(*def, os);
diff --git a/mlir/tools/mlir-tblgen/PassDocGen.cpp b/mlir/tools/mlir-tblgen/PassDocGen.cpp
index a2cb514ece3eb..456f9ceffeb9b 100644
--- a/mlir/tools/mlir-tblgen/PassDocGen.cpp
+++ b/mlir/tools/mlir-tblgen/PassDocGen.cpp
@@ -22,14 +22,14 @@ using llvm::RecordKeeper;
 
 /// Emit the documentation for the given pass.
 static void emitDoc(const Pass &pass, raw_ostream &os) {
-  os << llvm::formatv("### `-{0}`\n", pass.getArgument());
+  os << llvm::formatv("\n### `-{0}`\n", pass.getArgument());
   emitSummary(pass.getSummary(), os);
   emitDescription(pass.getDescription(), os);
 
   // Handle the options of the pass.
   ArrayRef<PassOption> options = pass.getOptions();
   if (!options.empty()) {
-    os << "\n#### Options\n```\n";
+    os << "\n#### Options\n\n```\n";
     size_t longestOption = 0;
     for (const PassOption &option : options)
       longestOption = std::max(option.getArgument().size(), longestOption);
@@ -44,7 +44,7 @@ static void emitDoc(const Pass &pass, raw_ostream &os) {
   // Handle the statistics of the pass.
   ArrayRef<PassStatistic> stats = pass.getStatistics();
   if (!stats.empty()) {
-    os << "\n#### Statistics\n```\n";
+    os << "\n#### Statistics\n\n```\n";
     size_t longestStat = 0;
     for (const PassStatistic &stat : stats)
       longestStat = std::max(stat.getName().size(), longestStat);
diff --git a/mlir/tools/mlir-tblgen/RewriterGen.cpp b/mlir/tools/mlir-tblgen/RewriterGen.cpp
index a041c4d327779..f6eb5bdfe568e 100644
--- a/mlir/tools/mlir-tblgen/RewriterGen.cpp
+++ b/mlir/tools/mlir-tblgen/RewriterGen.cpp
@@ -879,7 +879,7 @@ void PatternEmitter::emitAttributeMatch(DagNode tree, StringRef opName,
   if (attr.hasDefaultValue()) {
     os << "if (!tblgen_attr) tblgen_attr = "
        << std::string(tgfmt(attr.getConstBuilderTemplate(), &fmtCtx,
-                            attr.getDefaultValue()))
+                            tgfmt(attr.getDefaultValue(), &fmtCtx)))
        << ";\n";
   } else if (attr.isOptional()) {
     // For a missing attribute that is optional according to definition, we
diff --git a/mlir/utils/emacs/tblgen-lsp-client.el b/mlir/utils/emacs/tblgen-lsp-client.el
new file mode 100644
index 0000000000000..6830e328eb8de
--- /dev/null
+++ b/mlir/utils/emacs/tblgen-lsp-client.el
@@ -0,0 +1,45 @@
+;;; tblgen-lsp-client.el --- Description -*- lexical-binding: t; -*-
+;;
+;; Package-Requires: ((emacs "24.3"))
+;;
+;; This file is not part of GNU Emacs.
+;;
+;;; Commentary:
+;;  LSP client to use with `tablegen-mode' that uses `tblgen-lsp-server' or any
+;;  user made compatible server.
+;;
+;;
+;;; Code:
+(require 'lsp-mode)
+
+(defgroup lsp-tblgen nil
+  "LSP support for Tablegen."
+  :group 'lsp-mode
+  :link '(url-link "https://mlir.llvm.org/docs/Tools/MLIRLSP/"))
+
+(defcustom lsp-tblgen-server-executable "tblgen-lsp-server"
+  "Command to start the mlir language server."
+  :group 'lsp-tblgen
+  :risky t
+  :type 'file)
+
+
+(defcustom lsp-tblgen-server-args ""
+  "Args of LSP client for TableGen, for example '--tablegen-compilation-database=.../build/tablegen_compile_commands.yml'"
+  :group 'lsp-tblgen
+  :risky t
+  :type 'file)
+
+(defun lsp-tblgen-setup ()
+  "Setup the LSP client for TableGen."
+  (add-to-list 'lsp-language-id-configuration '(tablegen-mode . "tablegen"))
+
+  (lsp-register-client
+   (make-lsp-client
+    :new-connection (lsp-stdio-connection (lambda () (cons lsp-tblgen-server-executable (split-string-shell-command lsp-tblgen-server-args))))
+    :activation-fn (lsp-activate-on "tablegen")
+    :priority -1
+    :server-id 'tblgen-lsp)))
+
+(provide 'tblgen-lsp)
+;;; tblgen-lsp-client.el ends here
diff --git a/offload/CMakeLists.txt b/offload/CMakeLists.txt
index f6e894d399685..25c879710645c 100644
--- a/offload/CMakeLists.txt
+++ b/offload/CMakeLists.txt
@@ -377,7 +377,7 @@ add_subdirectory(DeviceRTL)
 add_subdirectory(tools)
 
 # Build target agnostic offloading library.
-add_subdirectory(src)
+add_subdirectory(libomptarget)
 
 add_subdirectory(tools/offload-tblgen)
 add_subdirectory(liboffload)
diff --git a/offload/DeviceRTL/CMakeLists.txt b/offload/DeviceRTL/CMakeLists.txt
index 099634e211e7a..8f2a1fd01fabc 100644
--- a/offload/DeviceRTL/CMakeLists.txt
+++ b/offload/DeviceRTL/CMakeLists.txt
@@ -95,11 +95,10 @@ set (LIBOMPTARGET_LLVM_INCLUDE_DIRS_DEVICERTL "${LIBOMPTARGET_LLVM_INCLUDE_DIRS}
 list(TRANSFORM LIBOMPTARGET_LLVM_INCLUDE_DIRS_DEVICERTL PREPEND "-I")
 
 # Set flags for LLVM Bitcode compilation.
-set(bc_flags -c -foffload-lto -std=c++17 -fvisibility=hidden
-              ${clang_opt_flags} --offload-device-only
-             -nocudalib -nogpulib -nogpuinc -nostdlibinc
-             -fopenmp -fopenmp-cuda-mode
-             -Wno-unknown-cuda-version -Wno-openmp-target
+set(bc_flags -c -flto -std=c++17 -fvisibility=hidden
+             ${clang_opt_flags} -nogpulib -nostdlibinc
+             -fno-rtti -fno-exceptions -fconvergent-functions
+             -Wno-unknown-cuda-version
              -DOMPTARGET_DEVICE_RUNTIME
              -I${include_directory}
              -I${devicertl_base_directory}/../include
@@ -123,8 +122,7 @@ function(compileDeviceRTLLibrary target_name target_triple)
     add_custom_command(OUTPUT ${outfile}
       COMMAND ${CLANG_TOOL}
       ${bc_flags}
-      -fopenmp-targets=${target_triple}
-      -Xopenmp-target=${target_triple} -march=
+      --target=${target_triple}
       ${target_bc_flags}
       -MD -MF ${depfile}
       ${infile} -o ${outfile}
@@ -242,10 +240,8 @@ function(compileDeviceRTLLibrary target_name target_triple)
     set(ide_target_name omptarget-ide-${target_name})
     add_library(${ide_target_name} STATIC EXCLUDE_FROM_ALL ${src_files})
     target_compile_options(${ide_target_name} PRIVATE
-      -fopenmp-targets=${target_triple} -Xopenmp-target=${target_triple} -march=
-      -fopenmp -fopenmp-cuda-mode -mllvm -openmp-opt-disable
-      -foffload-lto -fvisibility=hidden --offload-device-only
-      -nocudalib -nogpulib -nogpuinc -nostdlibinc -Wno-unknown-cuda-version
+      -fvisibility=hidden --target=${target_triple}
+      -nogpulib -nostdlibinc -Wno-unknown-cuda-version
     )
     target_compile_definitions(${ide_target_name} PRIVATE SHARED_SCRATCHPAD_SIZE=512)
     target_include_directories(${ide_target_name} PRIVATE
diff --git a/offload/DeviceRTL/include/Allocator.h b/offload/DeviceRTL/include/Allocator.h
index 475f6a21bb47e..79c69a2a96b4e 100644
--- a/offload/DeviceRTL/include/Allocator.h
+++ b/offload/DeviceRTL/include/Allocator.h
@@ -17,8 +17,6 @@
 // Forward declaration.
 struct KernelEnvironmentTy;
 
-#pragma omp begin declare target device_type(nohost)
-
 namespace ompx {
 
 namespace allocator {
@@ -44,6 +42,4 @@ extern "C" {
 [[gnu::weak]] void free(void *Ptr);
 }
 
-#pragma omp end declare target
-
 #endif
diff --git a/offload/DeviceRTL/include/Configuration.h b/offload/DeviceRTL/include/Configuration.h
index f8b7a6c3c6c9d..95408933dd865 100644
--- a/offload/DeviceRTL/include/Configuration.h
+++ b/offload/DeviceRTL/include/Configuration.h
@@ -27,7 +27,7 @@ uint32_t getNumDevices();
 /// Return the device number in the system for omp_get_device_num.
 uint32_t getDeviceNum();
 
-/// Return the user choosen debug level.
+/// Return the user chosen debug level.
 uint32_t getDebugKind();
 
 /// Return if teams oversubscription is assumed
diff --git a/offload/DeviceRTL/include/DeviceTypes.h b/offload/DeviceRTL/include/DeviceTypes.h
index 1cd044f432e56..2e5d92380f040 100644
--- a/offload/DeviceRTL/include/DeviceTypes.h
+++ b/offload/DeviceRTL/include/DeviceTypes.h
@@ -12,9 +12,15 @@
 #ifndef OMPTARGET_TYPES_H
 #define OMPTARGET_TYPES_H
 
+#include <gpuintrin.h>
 #include <stddef.h>
 #include <stdint.h>
 
+template <typename T> using Private = __gpu_private T;
+template <typename T> using Constant = __gpu_constant T;
+template <typename T> using Local = __gpu_local T;
+template <typename T> using Global = __gpu_local T;
+
 enum omp_proc_bind_t {
   omp_proc_bind_false = 0,
   omp_proc_bind_true = 1,
@@ -99,14 +105,7 @@ struct TaskDescriptorTy {
   TaskFnTy TaskFn;
 };
 
-#pragma omp begin declare variant match(device = {arch(amdgcn)})
-using LaneMaskTy = uint64_t;
-#pragma omp end declare variant
-
-#pragma omp begin declare variant match(                                       \
-        device = {arch(amdgcn)}, implementation = {extension(match_none)})
 using LaneMaskTy = uint64_t;
-#pragma omp end declare variant
 
 namespace lanes {
 enum : LaneMaskTy { All = ~(LaneMaskTy)0 };
@@ -162,20 +161,6 @@ typedef enum omp_allocator_handle_t {
 #define __PRAGMA(STR) _Pragma(#STR)
 #define OMP_PRAGMA(STR) __PRAGMA(omp STR)
 
-#define SHARED(NAME)                                                           \
-  NAME [[clang::loader_uninitialized]];                                        \
-  OMP_PRAGMA(allocate(NAME) allocator(omp_pteam_mem_alloc))
-
-// TODO: clang should use address space 5 for omp_thread_mem_alloc, but right
-//       now that's not the case.
-#define THREAD_LOCAL(NAME)                                                     \
-  [[clang::address_space(5)]] NAME [[clang::loader_uninitialized]]
-
-// TODO: clang should use address space 4 for omp_const_mem_alloc, maybe it
-//       does?
-#define CONSTANT(NAME)                                                         \
-  [[clang::address_space(4)]] NAME [[clang::loader_uninitialized]]
-
 ///}
 
 #endif
diff --git a/offload/DeviceRTL/include/DeviceUtils.h b/offload/DeviceRTL/include/DeviceUtils.h
index 2243673aef61c..b92514ee9838a 100644
--- a/offload/DeviceRTL/include/DeviceUtils.h
+++ b/offload/DeviceRTL/include/DeviceUtils.h
@@ -15,8 +15,6 @@
 #include "DeviceTypes.h"
 #include "Shared/Utils.h"
 
-#pragma omp begin declare target device_type(nohost)
-
 namespace utils {
 
 template <typename T> struct type_identity {
@@ -95,6 +93,4 @@ bool isThreadLocalMemPtr(void *Ptr);
 
 } // namespace utils
 
-#pragma omp end declare target
-
 #endif
diff --git a/offload/DeviceRTL/include/Mapping.h b/offload/DeviceRTL/include/Mapping.h
index 2fb87abe5418c..f892a025159d4 100644
--- a/offload/DeviceRTL/include/Mapping.h
+++ b/offload/DeviceRTL/include/Mapping.h
@@ -24,12 +24,8 @@ enum {
   DIM_Z = 2,
 };
 
-#pragma omp begin declare target device_type(nohost)
-
 inline constexpr uint32_t MaxThreadsPerTeam = 1024;
 
-#pragma omp end declare target
-
 /// Initialize the mapping machinery.
 void init(bool IsSPMD);
 
@@ -63,7 +59,7 @@ LaneMaskTy activemask();
 /// Return a mask describing all threads with a smaller Id in the warp.
 LaneMaskTy lanemaskLT();
 
-/// Return a mask describing all threads with a larget Id in the warp.
+/// Return a mask describing all threads with a larger Id in the warp.
 LaneMaskTy lanemaskGT();
 
 /// Return the thread Id in the warp, in [0, getWarpSize()).
diff --git a/offload/DeviceRTL/include/State.h b/offload/DeviceRTL/include/State.h
index c487ff29680fa..db396dae6e445 100644
--- a/offload/DeviceRTL/include/State.h
+++ b/offload/DeviceRTL/include/State.h
@@ -22,8 +22,6 @@
 // Forward declaration.
 struct KernelEnvironmentTy;
 
-#pragma omp begin declare target device_type(nohost)
-
 namespace ompx {
 
 namespace memory {
@@ -33,7 +31,7 @@ namespace memory {
 /// Note: See the restrictions on __kmpc_alloc_shared for proper usage.
 void *allocShared(uint64_t Size, const char *Reason);
 
-/// Free \p Ptr, alloated via allocShared, for \p Reason.
+/// Free \p Ptr, allocated via allocShared, for \p Reason.
 ///
 /// Note: See the restrictions on __kmpc_free_shared for proper usage.
 void freeShared(void *Ptr, uint64_t Bytes, const char *Reason);
@@ -44,7 +42,7 @@ void *allocGlobal(uint64_t Size, const char *Reason);
 /// Return a pointer to the dynamic shared memory buffer.
 void *getDynamicBuffer();
 
-/// Free \p Ptr, alloated via allocGlobal, for \p Reason.
+/// Free \p Ptr, allocated via allocGlobal, for \p Reason.
 void freeGlobal(void *Ptr, const char *Reason);
 
 } // namespace memory
@@ -88,8 +86,7 @@ struct TeamStateTy {
   ParallelRegionFnTy ParallelRegionFnVar;
 };
 
-extern TeamStateTy TeamState;
-#pragma omp allocate(TeamState) allocator(omp_pteam_mem_alloc)
+extern Local<TeamStateTy> TeamState;
 
 struct ThreadStateTy {
 
@@ -115,8 +112,7 @@ struct ThreadStateTy {
   }
 };
 
-extern ThreadStateTy **ThreadStates;
-#pragma omp allocate(ThreadStates) allocator(omp_pteam_mem_alloc)
+extern Local<ThreadStateTy **> ThreadStates;
 
 /// Initialize the state machinery. Must be called by all threads.
 void init(bool IsSPMD, KernelEnvironmentTy &KernelEnvironment,
@@ -365,7 +361,7 @@ inline state::Value<uint32_t, state::VK_Level> Level;
 /// The `active-level` describes which of the parallel level counted with the
 /// `level-var` is active. There can only be one.
 ///
-/// active-level-var is 1, if ActiveLevelVar is not 0, otherweise it is 0.
+/// active-level-var is 1, if ActiveLevelVar is not 0, otherwise it is 0.
 inline state::Value<uint32_t, state::VK_ActiveLevel> ActiveLevel;
 
 /// TODO
@@ -378,6 +374,4 @@ inline state::Value<uint32_t, state::VK_RunSched> RunSched;
 
 } // namespace ompx
 
-#pragma omp end declare target
-
 #endif
diff --git a/offload/DeviceRTL/include/Synchronization.h b/offload/DeviceRTL/include/Synchronization.h
index 5a789441b9d35..f9eb8d0d23198 100644
--- a/offload/DeviceRTL/include/Synchronization.h
+++ b/offload/DeviceRTL/include/Synchronization.h
@@ -15,14 +15,12 @@
 #include "DeviceTypes.h"
 #include "DeviceUtils.h"
 
-#pragma omp begin declare target device_type(nohost)
-
 namespace ompx {
 namespace atomic {
 
 enum OrderingTy {
   relaxed = __ATOMIC_RELAXED,
-  aquire = __ATOMIC_ACQUIRE,
+  acquire = __ATOMIC_ACQUIRE,
   release = __ATOMIC_RELEASE,
   acq_rel = __ATOMIC_ACQ_REL,
   seq_cst = __ATOMIC_SEQ_CST,
@@ -220,6 +218,4 @@ void system(atomic::OrderingTy Ordering);
 
 } // namespace ompx
 
-#pragma omp end declare target
-
 #endif
diff --git a/offload/DeviceRTL/include/Workshare.h b/offload/DeviceRTL/include/Workshare.h
index fa9b3b2430b8c..554c3271c334c 100644
--- a/offload/DeviceRTL/include/Workshare.h
+++ b/offload/DeviceRTL/include/Workshare.h
@@ -12,8 +12,6 @@
 #ifndef OMPTARGET_WORKSHARE_H
 #define OMPTARGET_WORKSHARE_H
 
-#pragma omp begin declare target device_type(nohost)
-
 namespace ompx {
 
 namespace workshare {
@@ -25,6 +23,4 @@ void init(bool IsSPMD);
 
 } // namespace ompx
 
-#pragma omp end declare target
-
 #endif
diff --git a/offload/DeviceRTL/src/Allocator.cpp b/offload/DeviceRTL/src/Allocator.cpp
index ac662c48d4f5f..aac2a6005158e 100644
--- a/offload/DeviceRTL/src/Allocator.cpp
+++ b/offload/DeviceRTL/src/Allocator.cpp
@@ -19,8 +19,6 @@
 
 using namespace ompx;
 
-#pragma omp begin declare target device_type(nohost)
-
 [[gnu::used, gnu::retain, gnu::weak,
   gnu::visibility(
       "protected")]] DeviceMemoryPoolTy __omp_rtl_device_memory_pool;
@@ -77,5 +75,3 @@ void *allocator::alloc(uint64_t Size) { return BumpAllocator.alloc(Size); }
 void allocator::free(void *Ptr) { BumpAllocator.free(Ptr); }
 
 ///}
-
-#pragma omp end declare target
diff --git a/offload/DeviceRTL/src/Configuration.cpp b/offload/DeviceRTL/src/Configuration.cpp
index 9e14c203d4a04..0c31c66ab2deb 100644
--- a/offload/DeviceRTL/src/Configuration.cpp
+++ b/offload/DeviceRTL/src/Configuration.cpp
@@ -17,8 +17,6 @@
 
 using namespace ompx;
 
-#pragma omp begin declare target device_type(nohost)
-
 // Weak definitions will be overridden by CGOpenmpRuntimeGPU if enabled.
 [[gnu::weak]] extern const uint32_t __omp_rtl_debug_kind = 0;
 [[gnu::weak]] extern const uint32_t __omp_rtl_assume_no_thread_state = 0;
@@ -27,11 +25,11 @@ using namespace ompx;
     0;
 [[gnu::weak]] extern const uint32_t __omp_rtl_assume_teams_oversubscription = 0;
 
-// This variable should be visibile to the plugin so we override the default
+// This variable should be visible to the plugin so we override the default
 // hidden visibility.
 [[gnu::used, gnu::retain, gnu::weak,
-  gnu::visibility("protected")]] DeviceEnvironmentTy
-    CONSTANT(__omp_rtl_device_environment);
+  gnu::visibility(
+      "protected")]] Constant<DeviceEnvironmentTy> __omp_rtl_device_environment;
 
 uint32_t config::getAssumeTeamsOversubscription() {
   return __omp_rtl_assume_teams_oversubscription;
@@ -85,5 +83,3 @@ bool config::mayUseNestedParallelism() {
     return false;
   return state::getKernelEnvironment().Configuration.MayUseNestedParallelism;
 }
-
-#pragma omp end declare target
diff --git a/offload/DeviceRTL/src/Debug.cpp b/offload/DeviceRTL/src/Debug.cpp
index 1d9c962885422..5b5482d766b1d 100644
--- a/offload/DeviceRTL/src/Debug.cpp
+++ b/offload/DeviceRTL/src/Debug.cpp
@@ -21,8 +21,6 @@
 
 using namespace ompx;
 
-#pragma omp begin declare target device_type(nohost)
-
 extern "C" {
 void __assert_assume(bool condition) { __builtin_assume(condition); }
 
@@ -44,5 +42,3 @@ void __assert_fail_internal(const char *expr, const char *msg, const char *file,
   __builtin_trap();
 }
 }
-
-#pragma omp end declare target
diff --git a/offload/DeviceRTL/src/DeviceUtils.cpp b/offload/DeviceRTL/src/DeviceUtils.cpp
index c204a7be73b1f..d8109537832e9 100644
--- a/offload/DeviceRTL/src/DeviceUtils.cpp
+++ b/offload/DeviceRTL/src/DeviceUtils.cpp
@@ -15,14 +15,10 @@
 #include "Interface.h"
 #include "Mapping.h"
 
-#pragma omp begin declare target device_type(nohost)
-
 using namespace ompx;
 
 namespace impl {
 
-bool isSharedMemPtr(const void *Ptr) { return false; }
-
 void Unpack(uint64_t Val, uint32_t *LowBits, uint32_t *HighBits) {
   static_assert(sizeof(unsigned long) == 8, "");
   *LowBits = static_cast<uint32_t>(Val & 0x00000000FFFFFFFFUL);
@@ -42,7 +38,7 @@ uint64_t ballotSync(uint64_t Mask, int32_t Pred);
 /// AMDGCN Implementation
 ///
 ///{
-#pragma omp begin declare variant match(device = {arch(amdgcn)})
+#ifdef __AMDGPU__
 
 int32_t shuffle(uint64_t Mask, int32_t Var, int32_t SrcLane, int32_t Width) {
   int Self = mapping::getThreadIdInWarp();
@@ -66,15 +62,13 @@ bool isSharedMemPtr(const void *Ptr) {
   return __builtin_amdgcn_is_shared(
       (const __attribute__((address_space(0))) void *)Ptr);
 }
-#pragma omp end declare variant
+#endif
 ///}
 
 /// NVPTX Implementation
 ///
 ///{
-#pragma omp begin declare variant match(                                       \
-        device = {arch(nvptx, nvptx64)},                                       \
-            implementation = {extension(match_any)})
+#ifdef __NVPTX__
 
 int32_t shuffle(uint64_t Mask, int32_t Var, int32_t SrcLane, int32_t Width) {
   return __nvvm_shfl_sync_idx_i32(Mask, Var, SrcLane, Width - 1);
@@ -91,7 +85,7 @@ uint64_t ballotSync(uint64_t Mask, int32_t Pred) {
 
 bool isSharedMemPtr(const void *Ptr) { return __nvvm_isspacep_shared(Ptr); }
 
-#pragma omp end declare variant
+#endif
 ///}
 } // namespace impl
 
@@ -137,5 +131,3 @@ int64_t __kmpc_shuffle_int64(int64_t Val, int16_t Delta, int16_t Width) {
   return utils::shuffleDown(lanes::All, Val, Delta, Width);
 }
 }
-
-#pragma omp end declare target
diff --git a/offload/DeviceRTL/src/Kernel.cpp b/offload/DeviceRTL/src/Kernel.cpp
index 8bb275eae776c..9bb89573dc0cb 100644
--- a/offload/DeviceRTL/src/Kernel.cpp
+++ b/offload/DeviceRTL/src/Kernel.cpp
@@ -25,8 +25,6 @@
 
 using namespace ompx;
 
-#pragma omp begin declare target device_type(nohost)
-
 static void
 inititializeRuntime(bool IsSPMD, KernelEnvironmentTy &KernelEnvironment,
                     KernelLaunchEnvironmentTy &KernelLaunchEnvironment) {
@@ -155,5 +153,3 @@ void __kmpc_target_deinit() {
 
 int8_t __kmpc_is_spmd_exec_mode() { return mapping::isSPMDMode(); }
 }
-
-#pragma omp end declare target
diff --git a/offload/DeviceRTL/src/LibC.cpp b/offload/DeviceRTL/src/LibC.cpp
index e55008f46269f..83f9233d94803 100644
--- a/offload/DeviceRTL/src/LibC.cpp
+++ b/offload/DeviceRTL/src/LibC.cpp
@@ -8,8 +8,6 @@
 
 #include "LibC.h"
 
-#pragma omp begin declare target device_type(nohost)
-
 #if defined(__AMDGPU__) && !defined(OMPTARGET_HAS_LIBC)
 extern "C" int vprintf(const char *format, __builtin_va_list) { return -1; }
 #else
@@ -48,5 +46,3 @@ namespace ompx {
   return ::vprintf(Format, vlist);
 }
 } // namespace ompx
-
-#pragma omp end declare target
diff --git a/offload/DeviceRTL/src/Mapping.cpp b/offload/DeviceRTL/src/Mapping.cpp
index 8583a539824c8..641be81cca3ed 100644
--- a/offload/DeviceRTL/src/Mapping.cpp
+++ b/offload/DeviceRTL/src/Mapping.cpp
@@ -15,8 +15,6 @@
 #include "Interface.h"
 #include "State.h"
 
-#pragma omp begin declare target device_type(nohost)
-
 #include "llvm/Frontend/OpenMP/OMPGridValues.h"
 
 using namespace ompx;
@@ -24,24 +22,10 @@ using namespace ompx;
 namespace ompx {
 namespace impl {
 
-// Forward declarations defined to be defined for AMDGCN and NVPTX.
-LaneMaskTy activemask();
-LaneMaskTy lanemaskLT();
-LaneMaskTy lanemaskGT();
-uint32_t getThreadIdInWarp();
-uint32_t getThreadIdInBlock(int32_t Dim);
-uint32_t getNumberOfThreadsInBlock(int32_t Dim);
-uint32_t getNumberOfThreadsInKernel();
-uint32_t getBlockIdInKernel(int32_t Dim);
-uint32_t getNumberOfBlocksInKernel(int32_t Dim);
-uint32_t getWarpIdInBlock();
-uint32_t getNumberOfWarpsInBlock();
-uint32_t getWarpSize();
-
 /// AMDGCN Implementation
 ///
 ///{
-#pragma omp begin declare variant match(device = {arch(amdgcn)})
+#ifdef __AMDGPU__
 
 uint32_t getWarpSize() { return __builtin_amdgcn_wavefrontsize(); }
 
@@ -128,15 +112,13 @@ uint32_t getNumberOfWarpsInBlock() {
   return mapping::getNumberOfThreadsInBlock() / mapping::getWarpSize();
 }
 
-#pragma omp end declare variant
+#endif
 ///}
 
 /// NVPTX Implementation
 ///
 ///{
-#pragma omp begin declare variant match(                                       \
-        device = {arch(nvptx, nvptx64)},                                       \
-            implementation = {extension(match_any)})
+#ifdef __NVPTX__
 
 uint32_t getNumberOfThreadsInBlock(int32_t Dim) {
   switch (Dim) {
@@ -214,7 +196,7 @@ uint32_t getNumberOfWarpsInBlock() {
          mapping::getWarpSize();
 }
 
-#pragma omp end declare variant
+#endif
 ///}
 
 } // namespace impl
@@ -326,7 +308,7 @@ uint32_t mapping::getNumberOfProcessorElements() {
 
 // TODO: This is a workaround for initialization coming from kernels outside of
 //       the TU. We will need to solve this more correctly in the future.
-[[gnu::weak]] int SHARED(IsSPMDMode);
+[[gnu::weak, clang::loader_uninitialized]] Local<int> IsSPMDMode;
 
 void mapping::init(bool IsSPMD) {
   if (mapping::isInitialThreadInLevel0(IsSPMD))
@@ -376,7 +358,7 @@ float ompx_shfl_down_sync_f(uint64_t mask, float var, unsigned delta,
 }
 
 long ompx_shfl_down_sync_l(uint64_t mask, long var, unsigned delta, int width) {
-  return utils::shuffleDown(mask, var, delta, width);
+  return utils::shuffleDown(mask, utils::bitCast<int64_t>(var), delta, width);
 }
 
 double ompx_shfl_down_sync_d(uint64_t mask, double var, unsigned delta,
@@ -385,5 +367,3 @@ double ompx_shfl_down_sync_d(uint64_t mask, double var, unsigned delta,
       utils::shuffleDown(mask, utils::bitCast<int64_t>(var), delta, width));
 }
 }
-
-#pragma omp end declare target
diff --git a/offload/DeviceRTL/src/Misc.cpp b/offload/DeviceRTL/src/Misc.cpp
index ba6fbf5d5c7e3..734e937f03920 100644
--- a/offload/DeviceRTL/src/Misc.cpp
+++ b/offload/DeviceRTL/src/Misc.cpp
@@ -17,23 +17,17 @@
 
 #include "Debug.h"
 
-#pragma omp begin declare target device_type(nohost)
-
 namespace ompx {
 namespace impl {
 
-double getWTick();
-
-double getWTime();
-
 /// AMDGCN Implementation
 ///
 ///{
-#pragma omp begin declare variant match(device = {arch(amdgcn)})
+#ifdef __AMDGPU__
 
 double getWTick() {
   // The number of ticks per second for the AMDGPU clock varies by card and can
-  // only be retrived by querying the driver. We rely on the device environment
+  // only be retrieved by querying the driver. We rely on the device environment
   // to inform us what the proper frequency is.
   return 1.0 / config::getClockFrequency();
 }
@@ -42,14 +36,12 @@ double getWTime() {
   return static_cast<double>(__builtin_readsteadycounter()) * getWTick();
 }
 
-#pragma omp end declare variant
+#endif
 
 /// NVPTX Implementation
 ///
 ///{
-#pragma omp begin declare variant match(                                       \
-        device = {arch(nvptx, nvptx64)},                                       \
-            implementation = {extension(match_any)})
+#ifdef __NVPTX__
 
 double getWTick() {
   // Timer precision is 1ns
@@ -61,7 +53,7 @@ double getWTime() {
   return static_cast<double>(nsecs) * getWTick();
 }
 
-#pragma omp end declare variant
+#endif
 
 /// Lookup a device-side function using a host pointer /p HstPtr using the table
 /// provided by the device plugin. The table is an ordered pair of host and
@@ -105,14 +97,8 @@ void *indirectCallLookup(void *HstPtr) {
 }
 
 /// The openmp client instance used to communicate with the server.
-/// FIXME: This is marked as 'retain' so that it is not removed via
-/// `-mlink-builtin-bitcode`
-#ifdef __NVPTX__
-[[gnu::visibility("protected"), gnu::weak,
-  gnu::retain]] rpc::Client Client asm("__llvm_rpc_client");
-#else
-[[gnu::visibility("protected"), gnu::weak]] rpc::Client Client asm("__llvm_rpc_client");
-#endif
+[[gnu::visibility("protected"),
+  gnu::weak]] rpc::Client Client asm("__llvm_rpc_client");
 
 } // namespace impl
 } // namespace ompx
@@ -177,4 +163,3 @@ unsigned long long __llvm_omp_host_call(void *fn, void *data, size_t size) {
 }
 
 ///}
-#pragma omp end declare target
diff --git a/offload/DeviceRTL/src/Parallelism.cpp b/offload/DeviceRTL/src/Parallelism.cpp
index a87e363349b1e..08ce616aee1c4 100644
--- a/offload/DeviceRTL/src/Parallelism.cpp
+++ b/offload/DeviceRTL/src/Parallelism.cpp
@@ -43,8 +43,6 @@
 
 using namespace ompx;
 
-#pragma omp begin declare target device_type(nohost)
-
 namespace {
 
 uint32_t determineNumberOfThreads(int32_t NumThreadsClause) {
@@ -311,5 +309,3 @@ void __kmpc_push_num_teams(IdentTy *loc, int32_t tid, int32_t num_teams,
 
 void __kmpc_push_proc_bind(IdentTy *loc, uint32_t tid, int proc_bind) {}
 }
-
-#pragma omp end declare target
diff --git a/offload/DeviceRTL/src/Profiling.cpp b/offload/DeviceRTL/src/Profiling.cpp
index bb3caaadcc03d..df141af5ebeea 100644
--- a/offload/DeviceRTL/src/Profiling.cpp
+++ b/offload/DeviceRTL/src/Profiling.cpp
@@ -8,8 +8,6 @@
 
 #include "Profiling.h"
 
-#pragma omp begin declare target device_type(nohost)
-
 extern "C" {
 
 // Provides empty implementations for certain functions in compiler-rt
@@ -18,5 +16,3 @@ void __llvm_profile_register_function(void *Ptr) {}
 void __llvm_profile_register_names_function(void *Ptr, long int I) {}
 void __llvm_profile_instrument_memop(long int I, void *Ptr, int I2) {}
 }
-
-#pragma omp end declare target
diff --git a/offload/DeviceRTL/src/Reduction.cpp b/offload/DeviceRTL/src/Reduction.cpp
index d3b4528401953..fffd0063940c6 100644
--- a/offload/DeviceRTL/src/Reduction.cpp
+++ b/offload/DeviceRTL/src/Reduction.cpp
@@ -22,8 +22,6 @@ using namespace ompx;
 
 namespace {
 
-#pragma omp begin declare target device_type(nohost)
-
 void gpu_regular_warp_reduce(void *reduce_data, ShuffleReductFnTy shflFct) {
   for (uint32_t mask = mapping::getWarpSize() / 2; mask > 0; mask /= 2) {
     shflFct(reduce_data, /*LaneId - not used= */ 0,
@@ -73,16 +71,16 @@ static int32_t nvptx_parallel_reduce_nowait(void *reduce_data,
   if (NumThreads == 1)
     return 1;
 
-    //
-    // This reduce function handles reduction within a team. It handles
-    // parallel regions in both L1 and L2 parallelism levels. It also
-    // supports Generic, SPMD, and NoOMP modes.
-    //
-    // 1. Reduce within a warp.
-    // 2. Warp master copies value to warp 0 via shared memory.
-    // 3. Warp 0 reduces to a single value.
-    // 4. The reduced value is available in the thread that returns 1.
-    //
+  //
+  // This reduce function handles reduction within a team. It handles
+  // parallel regions in both L1 and L2 parallelism levels. It also
+  // supports Generic, SPMD, and NoOMP modes.
+  //
+  // 1. Reduce within a warp.
+  // 2. Warp master copies value to warp 0 via shared memory.
+  // 3. Warp 0 reduces to a single value.
+  // 4. The reduced value is available in the thread that returns 1.
+  //
 
 #if __has_builtin(__nvvm_reflect)
   if (__nvvm_reflect("__CUDA_ARCH") >= 700) {
@@ -198,15 +196,15 @@ int32_t __kmpc_nvptx_teams_reduce_nowait_v2(
   uint32_t NumThreads = omp_get_num_threads();
   uint32_t TeamId = omp_get_team_num();
   uint32_t NumTeams = omp_get_num_teams();
-  static unsigned SHARED(Bound);
-  static unsigned SHARED(ChunkTeamCount);
+  [[clang::loader_uninitialized]] static Local<unsigned> Bound;
+  [[clang::loader_uninitialized]] static Local<unsigned> ChunkTeamCount;
 
   // Block progress for teams greater than the current upper
   // limit. We always only allow a number of teams less or equal
   // to the number of slots in the buffer.
   bool IsMaster = (ThreadId == 0);
   while (IsMaster) {
-    Bound = atomic::load(&IterCnt, atomic::aquire);
+    Bound = atomic::load(&IterCnt, atomic::acquire);
     if (TeamId < Bound + num_of_records)
       break;
   }
@@ -259,7 +257,7 @@ int32_t __kmpc_nvptx_teams_reduce_nowait_v2(
   unsigned NumRecs = kmpcMin(NumTeams, uint32_t(num_of_records));
   if (ChunkTeamCount == NumTeams - Bound - 1) {
     // Ensure we see the global memory writes by other teams
-    fence::kernel(atomic::aquire);
+    fence::kernel(atomic::acquire);
 
     //
     // Last team processing.
@@ -316,5 +314,3 @@ int32_t __kmpc_nvptx_teams_reduce_nowait_v2(
 void *__kmpc_reduction_get_fixed_buffer() {
   return state::getKernelLaunchEnvironment().ReductionBuffer;
 }
-
-#pragma omp end declare target
diff --git a/offload/DeviceRTL/src/State.cpp b/offload/DeviceRTL/src/State.cpp
index 100bc8ab47983..cbe9735145340 100644
--- a/offload/DeviceRTL/src/State.cpp
+++ b/offload/DeviceRTL/src/State.cpp
@@ -23,23 +23,22 @@
 
 using namespace ompx;
 
-#pragma omp begin declare target device_type(nohost)
-
 /// Memory implementation
 ///
 ///{
 
 /// External symbol to access dynamic shared memory.
 [[gnu::aligned(
-    allocator::ALIGNMENT)]] extern unsigned char DynamicSharedBuffer[];
-#pragma omp allocate(DynamicSharedBuffer) allocator(omp_pteam_mem_alloc)
+    allocator::ALIGNMENT)]] extern Local<unsigned char> DynamicSharedBuffer[];
 
 /// The kernel environment passed to the init method by the compiler.
-static KernelEnvironmentTy *SHARED(KernelEnvironmentPtr);
+[[clang::loader_uninitialized]] static Local<KernelEnvironmentTy *>
+    KernelEnvironmentPtr;
 
 /// The kernel launch environment passed as argument to the kernel by the
 /// runtime.
-static KernelLaunchEnvironmentTy *SHARED(KernelLaunchEnvironmentPtr);
+[[clang::loader_uninitialized]] static Local<KernelLaunchEnvironmentTy *>
+    KernelLaunchEnvironmentPtr;
 
 ///}
 
@@ -111,7 +110,8 @@ static_assert(state::SharedScratchpadSize / mapping::MaxThreadsPerTeam <= 256,
               "Shared scratchpad of this size not supported yet.");
 
 /// The allocation of a single shared memory scratchpad.
-static SharedMemorySmartStackTy SHARED(SharedMemorySmartStack);
+[[clang::loader_uninitialized]] static Local<SharedMemorySmartStackTy>
+    SharedMemorySmartStack;
 
 void SharedMemorySmartStackTy::init(bool IsSPMD) {
   Usage[mapping::getThreadIdInBlock()] = 0;
@@ -223,8 +223,10 @@ void state::TeamStateTy::assertEqual(TeamStateTy &Other) const {
   ASSERT(HasThreadState == Other.HasThreadState, nullptr);
 }
 
-state::TeamStateTy SHARED(ompx::state::TeamState);
-state::ThreadStateTy **SHARED(ompx::state::ThreadStates);
+[[clang::loader_uninitialized]] Local<state::TeamStateTy>
+    ompx::state::TeamState;
+[[clang::loader_uninitialized]] Local<state::ThreadStateTy **>
+    ompx::state::ThreadStates;
 
 namespace {
 
@@ -452,13 +454,10 @@ void *llvm_omp_get_dynamic_shared() { return __kmpc_get_dynamic_shared(); }
 /// NUM_SHARED_VARIABLES_IN_SHARED_MEM we will malloc space for communication.
 constexpr uint64_t NUM_SHARED_VARIABLES_IN_SHARED_MEM = 64;
 
-[[clang::loader_uninitialized]] static void
-    *SharedMemVariableSharingSpace[NUM_SHARED_VARIABLES_IN_SHARED_MEM];
-#pragma omp allocate(SharedMemVariableSharingSpace)                            \
-    allocator(omp_pteam_mem_alloc)
-[[clang::loader_uninitialized]] static void **SharedMemVariableSharingSpacePtr;
-#pragma omp allocate(SharedMemVariableSharingSpacePtr)                         \
-    allocator(omp_pteam_mem_alloc)
+[[clang::loader_uninitialized]] static Local<void *>
+    SharedMemVariableSharingSpace[NUM_SHARED_VARIABLES_IN_SHARED_MEM];
+[[clang::loader_uninitialized]] static Local<void **>
+    SharedMemVariableSharingSpacePtr;
 
 void __kmpc_begin_sharing_variables(void ***GlobalArgs, uint64_t nArgs) {
   if (nArgs <= NUM_SHARED_VARIABLES_IN_SHARED_MEM) {
@@ -481,4 +480,3 @@ void __kmpc_get_shared_variables(void ***GlobalArgs) {
   *GlobalArgs = SharedMemVariableSharingSpacePtr;
 }
 }
-#pragma omp end declare target
diff --git a/offload/DeviceRTL/src/Synchronization.cpp b/offload/DeviceRTL/src/Synchronization.cpp
index e0e277928fa91..0854c21ee152a 100644
--- a/offload/DeviceRTL/src/Synchronization.cpp
+++ b/offload/DeviceRTL/src/Synchronization.cpp
@@ -19,8 +19,6 @@
 #include "Mapping.h"
 #include "State.h"
 
-#pragma omp begin declare target device_type(nohost)
-
 using namespace ompx;
 
 namespace impl {
@@ -28,34 +26,12 @@ namespace impl {
 /// Atomics
 ///
 ///{
-/// NOTE: This function needs to be implemented by every target.
-uint32_t atomicInc(uint32_t *Address, uint32_t Val, atomic::OrderingTy Ordering,
-                   atomic::MemScopeTy MemScope);
 ///}
 
-// Forward declarations defined to be defined for AMDGCN and NVPTX.
-uint32_t atomicInc(uint32_t *A, uint32_t V, atomic::OrderingTy Ordering,
-                   atomic::MemScopeTy MemScope);
-void namedBarrierInit();
-void namedBarrier();
-void fenceTeam(atomic::OrderingTy Ordering);
-void fenceKernel(atomic::OrderingTy Ordering);
-void fenceSystem(atomic::OrderingTy Ordering);
-void syncWarp(__kmpc_impl_lanemask_t);
-void syncThreads(atomic::OrderingTy Ordering);
-void syncThreadsAligned(atomic::OrderingTy Ordering) { syncThreads(Ordering); }
-void unsetLock(omp_lock_t *);
-int testLock(omp_lock_t *);
-void initLock(omp_lock_t *);
-void destroyLock(omp_lock_t *);
-void setLock(omp_lock_t *);
-void unsetCriticalLock(omp_lock_t *);
-void setCriticalLock(omp_lock_t *);
-
 /// AMDGCN Implementation
 ///
 ///{
-#pragma omp begin declare variant match(device = {arch(amdgcn)})
+#ifdef __AMDGPU__
 
 uint32_t atomicInc(uint32_t *A, uint32_t V, atomic::OrderingTy Ordering,
                    atomic::MemScopeTy MemScope) {
@@ -84,7 +60,7 @@ uint32_t atomicInc(uint32_t *A, uint32_t V, atomic::OrderingTy Ordering,
   default:
     __builtin_unreachable();
     Case(atomic::relaxed);
-    Case(atomic::aquire);
+    Case(atomic::acquire);
     Case(atomic::release);
     Case(atomic::acq_rel);
     Case(atomic::seq_cst);
@@ -93,7 +69,7 @@ uint32_t atomicInc(uint32_t *A, uint32_t V, atomic::OrderingTy Ordering,
   }
 }
 
-uint32_t SHARED(namedBarrierTracker);
+[[clang::loader_uninitialized]] Local<uint32_t> namedBarrierTracker;
 
 void namedBarrierInit() {
   // Don't have global ctors, and shared memory is not zero init
@@ -107,7 +83,7 @@ void namedBarrier() {
   uint32_t WarpSize = mapping::getWarpSize();
   uint32_t NumWaves = NumThreads / WarpSize;
 
-  fence::team(atomic::aquire);
+  fence::team(atomic::acquire);
 
   // named barrier implementation for amdgcn.
   // Uses two 16 bit unsigned counters. One for the number of waves to have
@@ -172,7 +148,7 @@ void syncThreads(atomic::OrderingTy Ordering) {
   __builtin_amdgcn_s_barrier();
 
   if (Ordering != atomic::relaxed)
-    fenceTeam(Ordering == atomic::acq_rel ? atomic::aquire : atomic::seq_cst);
+    fenceTeam(Ordering == atomic::acq_rel ? atomic::acquire : atomic::seq_cst);
 }
 void syncThreadsAligned(atomic::OrderingTy Ordering) { syncThreads(Ordering); }
 
@@ -198,19 +174,17 @@ void setCriticalLock(omp_lock_t *Lock) {
         !cas((uint32_t *)Lock, UNSET, SET, atomic::relaxed, atomic::relaxed)) {
       __builtin_amdgcn_s_sleep(32);
     }
-    fenceKernel(atomic::aquire);
+    fenceKernel(atomic::acquire);
   }
 }
 
-#pragma omp end declare variant
+#endif
 ///}
 
 /// NVPTX Implementation
 ///
 ///{
-#pragma omp begin declare variant match(                                       \
-        device = {arch(nvptx, nvptx64)},                                       \
-            implementation = {extension(match_any)})
+#ifdef __NVPTX__
 
 uint32_t atomicInc(uint32_t *Address, uint32_t Val, atomic::OrderingTy Ordering,
                    atomic::MemScopeTy MemScope) {
@@ -283,7 +257,7 @@ void unsetCriticalLock(omp_lock_t *Lock) { unsetLock(Lock); }
 
 void setCriticalLock(omp_lock_t *Lock) { setLock(Lock); }
 
-#pragma omp end declare variant
+#endif
 ///}
 
 } // namespace impl
@@ -401,5 +375,3 @@ void ompx_sync_block_divergent(int Ordering) {
   impl::syncThreads(atomic::OrderingTy(Ordering));
 }
 } // extern "C"
-
-#pragma omp end declare target
diff --git a/offload/DeviceRTL/src/Tasking.cpp b/offload/DeviceRTL/src/Tasking.cpp
index 23a967c1a337e..d0be0ace50dff 100644
--- a/offload/DeviceRTL/src/Tasking.cpp
+++ b/offload/DeviceRTL/src/Tasking.cpp
@@ -20,8 +20,6 @@
 
 using namespace ompx;
 
-#pragma omp begin declare target device_type(nohost)
-
 extern "C" {
 
 TaskDescriptorTy *__kmpc_omp_task_alloc(IdentTy *, int32_t, int32_t,
@@ -29,7 +27,7 @@ TaskDescriptorTy *__kmpc_omp_task_alloc(IdentTy *, int32_t, int32_t,
                                         size_t SharedValuesSize,
                                         TaskFnTy TaskFn) {
   auto TaskSizeInclPrivateValuesPadded =
-      utils::roundUp(TaskSizeInclPrivateValues, uint64_t(sizeof(void *)));
+      utils::roundUp(TaskSizeInclPrivateValues, sizeof(void *));
   auto TaskSizeTotal = TaskSizeInclPrivateValuesPadded + SharedValuesSize;
   TaskDescriptorTy *TaskDescriptor = (TaskDescriptorTy *)memory::allocGlobal(
       TaskSizeTotal, "explicit task descriptor");
@@ -103,5 +101,3 @@ int omp_in_final(void) {
 
 int omp_get_max_task_priority(void) { return 0; }
 }
-
-#pragma omp end declare target
diff --git a/offload/DeviceRTL/src/Workshare.cpp b/offload/DeviceRTL/src/Workshare.cpp
index ad60e66548be9..de4ed2e2102a6 100644
--- a/offload/DeviceRTL/src/Workshare.cpp
+++ b/offload/DeviceRTL/src/Workshare.cpp
@@ -44,10 +44,8 @@ struct DynamicScheduleTracker {
 #define NOT_FINISHED 1
 #define LAST_CHUNK 2
 
-#pragma omp begin declare target device_type(nohost)
-
 // TODO: This variable is a hack inherited from the old runtime.
-static uint64_t SHARED(Cnt);
+[[clang::loader_uninitialized]] static Local<uint64_t> Cnt;
 
 template <typename T, typename ST> struct omptarget_nvptx_LoopSupport {
   ////////////////////////////////////////////////////////////////////////////////
@@ -79,7 +77,7 @@ template <typename T, typename ST> struct omptarget_nvptx_LoopSupport {
     lb = lb + entityId * chunk;
     T inputUb = ub;
     ub = lb + chunk - 1; // Clang uses i <= ub
-    // Say ub' is the begining of the last chunk. Then who ever has a
+    // Say ub' is the beginning of the last chunk. Then who ever has a
     // lower bound plus a multiple of the increment equal to ub' is
     // the last one.
     T beginingLastChunk = inputUb - (inputUb % chunk);
@@ -459,7 +457,8 @@ template <typename T, typename ST> struct omptarget_nvptx_LoopSupport {
 //
 //  __kmpc_dispatch_deinit
 //
-static DynamicScheduleTracker **SHARED(ThreadDST);
+[[clang::loader_uninitialized]] static Local<DynamicScheduleTracker **>
+    ThreadDST;
 
 // Create a new DST, link the current one, and define the new as current.
 static DynamicScheduleTracker *pushDST() {
@@ -806,7 +805,7 @@ template <typename Ty> class StaticLoopChunker {
                                 NumIters, OneIterationPerThread);
   }
 
-  /// Worksharing `distrbute`-loop.
+  /// Worksharing `distribute`-loop.
   static void Distribute(IdentTy *Loc, void (*LoopBody)(Ty, void *), void *Arg,
                          Ty NumIters, Ty BlockChunk) {
     ASSERT(icv::Level == 0, "Bad distribute");
@@ -853,7 +852,7 @@ template <typename Ty> class StaticLoopChunker {
     ASSERT(state::ParallelTeamSize == 1, "Bad distribute");
   }
 
-  /// Worksharing `distrbute parallel for`-loop.
+  /// Worksharing `distribute parallel for`-loop.
   static void DistributeFor(IdentTy *Loc, void (*LoopBody)(Ty, void *),
                             void *Arg, Ty NumIters, Ty NumThreads,
                             Ty BlockChunk, Ty ThreadChunk) {
@@ -935,5 +934,3 @@ OMP_LOOP_ENTRY(_4u, uint32_t)
 OMP_LOOP_ENTRY(_8, int64_t)
 OMP_LOOP_ENTRY(_8u, uint64_t)
 }
-
-#pragma omp end declare target
diff --git a/offload/cmake/caches/AMDGPUBot.cmake b/offload/cmake/caches/AMDGPUBot.cmake
index 405abcc28f98d..f04f3de2cb145 100644
--- a/offload/cmake/caches/AMDGPUBot.cmake
+++ b/offload/cmake/caches/AMDGPUBot.cmake
@@ -9,7 +9,7 @@ set(BUILD_SHARED_LIBS ON CACHE BOOL "")
 set(CMAKE_C_COMPILER_LAUNCHER ccache CACHE STRING "")
 set(CMAKE_CXX_COMPILER_LAUNCHER ccache CACHE STRING "")
 
-set(LLVM_ENABLE_PROJECTS "clang;lld" CACHE STRING "")
+set(LLVM_ENABLE_PROJECTS "clang;lld;mlir;flang" CACHE STRING "")
 set(LLVM_ENABLE_RUNTIMES "compiler-rt;openmp;offload" CACHE STRING "")
 
 set(LLVM_ENABLE_PER_TARGET_RUNTIME_DIR ON CACHE BOOL "")
diff --git a/offload/cmake/caches/Offload.cmake b/offload/cmake/caches/Offload.cmake
index 8749fcccb0ed1..5533a6508f5d5 100644
--- a/offload/cmake/caches/Offload.cmake
+++ b/offload/cmake/caches/Offload.cmake
@@ -2,9 +2,6 @@ set(LLVM_ENABLE_PROJECTS "clang;clang-tools-extra;lld" CACHE STRING "")
 set(LLVM_ENABLE_RUNTIMES "compiler-rt;libunwind;libcxx;libcxxabi;openmp;offload" CACHE STRING "")
 set(LLVM_ENABLE_PER_TARGET_RUNTIME_DIR ON CACHE BOOL "")
 
-set(CLANG_DEFAULT_CXX_STDLIB "libc++" CACHE STRING "")
-set(CLANG_DEFAULT_LINKER "lld" CACHE STRING "")
-
 set(LLVM_RUNTIME_TARGETS default;amdgcn-amd-amdhsa;nvptx64-nvidia-cuda CACHE STRING "") 
 set(RUNTIMES_nvptx64-nvidia-cuda_CACHE_FILES "${CMAKE_SOURCE_DIR}/../libcxx/cmake/caches/NVPTX.cmake" CACHE STRING "")
 set(RUNTIMES_amdgcn-amd-amdhsa_CACHE_FILES "${CMAKE_SOURCE_DIR}/../libcxx/cmake/caches/AMDGPU.cmake" CACHE STRING "")
diff --git a/offload/include/OpenMP/OMPT/Callback.h b/offload/include/OpenMP/OMPT/Callback.h
index 68cb43745eb1f..9d545c643223f 100644
--- a/offload/include/OpenMP/OMPT/Callback.h
+++ b/offload/include/OpenMP/OMPT/Callback.h
@@ -56,7 +56,7 @@ FOREACH_OMPT_EMI_EVENT(declareOmptCallback)
 
 /// This function will call an OpenMP API function. Which in turn will lookup a
 /// given enum value of type \p ompt_callbacks_t and copy the address of the
-/// corresponding callback funtion into the provided pointer.
+/// corresponding callback function into the provided pointer.
 /// The pointer to the runtime function is passed during 'initializeLibrary'.
 /// \p which the enum value of the requested callback function
 /// \p callback the destination pointer where the address shall be copied
diff --git a/offload/include/PluginManager.h b/offload/include/PluginManager.h
index 324dcac7787ea..ec3adadf0819b 100644
--- a/offload/include/PluginManager.h
+++ b/offload/include/PluginManager.h
@@ -77,7 +77,7 @@ struct PluginManager {
   /// Iterate over all device images registered with this plugin.
   auto deviceImages() { return llvm::make_pointee_range(DeviceImages); }
 
-  /// Translation table retreived from the binary
+  /// Translation table retrieved from the binary
   HostEntriesBeginToTransTableTy HostEntriesBeginToTransTable;
   std::mutex TrlTblMtx; ///< For Translation Table
   /// Host offload entries in order of image registration
@@ -170,6 +170,12 @@ struct PluginManager {
 
   /// Devices associated with plugins, accesses to the container are exclusive.
   ProtectedObj<DeviceContainerTy> Devices;
+
+  /// References to upgraded legacy offloading entries.
+  std::list<llvm::SmallVector<llvm::offloading::EntryTy, 0>> LegacyEntries;
+  std::list<llvm::SmallVector<__tgt_device_image, 0>> LegacyImages;
+  llvm::DenseMap<__tgt_bin_desc *, __tgt_bin_desc> UpgradedDescriptors;
+  __tgt_bin_desc *upgradeLegacyEntries(__tgt_bin_desc *Desc);
 };
 
 /// Initialize the plugin manager and OpenMP runtime.
diff --git a/offload/include/device.h b/offload/include/device.h
index 3132d35b7b38c..f4b10abbaa3fd 100644
--- a/offload/include/device.h
+++ b/offload/include/device.h
@@ -134,7 +134,7 @@ struct DeviceTy {
   int32_t recordEvent(void *Event, AsyncInfoTy &AsyncInfo);
 
   /// Wait for an event. This function can be blocking or non-blocking,
-  /// depending on the implmentation. It is expected to set a dependence on the
+  /// depending on the implementation. It is expected to set a dependence on the
   /// event such that corresponding operations shall only start once the event
   /// is fulfilled.
   int32_t waitEvent(void *Event, AsyncInfoTy &AsyncInfo);
diff --git a/offload/include/omptarget.h b/offload/include/omptarget.h
index 2b6445e9fbe55..6971780c7bdb5 100644
--- a/offload/include/omptarget.h
+++ b/offload/include/omptarget.h
@@ -42,7 +42,7 @@ enum __tgt_target_return_t : int {
   OMP_TGT_SUCCESS = 0,
   /// offload may not execute on the requested target device
   /// this scenario can be caused by the device not available or unsupported
-  /// as described in the Execution Model in the specifcation
+  /// as described in the Execution Model in the specification
   /// this status may not be used for target device execution failure
   /// which should be handled internally in libomptarget
   OMP_TGT_FAIL = ~0
diff --git a/offload/liboffload/API/APIDefs.td b/offload/liboffload/API/APIDefs.td
index 60c1b85d26911..cee4adea1d9f6 100644
--- a/offload/liboffload/API/APIDefs.td
+++ b/offload/liboffload/API/APIDefs.td
@@ -36,7 +36,7 @@ class IsPointerType<string Type> {
   bit ret = !ne(!find(Type, "*", !sub(!size(Type), 1)), -1);
 }
 
-// Describes the valid range of a pointer parameter that reperesents an array
+// Describes the valid range of a pointer parameter that represents an array
 class Range<string Begin, string End> {
   string begin = Begin;
   string end = End;
diff --git a/offload/liboffload/API/README.md b/offload/liboffload/API/README.md
index 38a055811b2d0..b59ac2782a2be 100644
--- a/offload/liboffload/API/README.md
+++ b/offload/liboffload/API/README.md
@@ -62,7 +62,7 @@ which preserves ABI compatibility with C.
 Represents a C-style enum. Contains a list of `etor` values, which have a name
 and description.
 
-A `TaggedEtor` record type also exists which addtionally takes a type. This type
+A `TaggedEtor` record type also exists which additionally takes a type. This type
 is used when the enum is used as a parameter to a function with a type-tagged
 function parameter (e.g. `olGetDeviceInfo`).
 
diff --git a/offload/liboffload/src/OffloadLib.cpp b/offload/liboffload/src/OffloadLib.cpp
index 37876713212c9..70e1ce1f84d83 100644
--- a/offload/liboffload/src/OffloadLib.cpp
+++ b/offload/liboffload/src/OffloadLib.cpp
@@ -36,7 +36,7 @@ OffloadConfig &offloadConfig() {
   return Config;
 }
 
-// Pull in the declarations for the implementation funtions. The actual entry
+// Pull in the declarations for the implementation functions. The actual entry
 // points in this file wrap these.
 #include "OffloadImplFuncDecls.inc"
 
diff --git a/offload/src/CMakeLists.txt b/offload/libomptarget/CMakeLists.txt
similarity index 100%
rename from offload/src/CMakeLists.txt
rename to offload/libomptarget/CMakeLists.txt
diff --git a/offload/src/DeviceImage.cpp b/offload/libomptarget/DeviceImage.cpp
similarity index 100%
rename from offload/src/DeviceImage.cpp
rename to offload/libomptarget/DeviceImage.cpp
diff --git a/offload/src/KernelLanguage/API.cpp b/offload/libomptarget/KernelLanguage/API.cpp
similarity index 100%
rename from offload/src/KernelLanguage/API.cpp
rename to offload/libomptarget/KernelLanguage/API.cpp
diff --git a/offload/src/LegacyAPI.cpp b/offload/libomptarget/LegacyAPI.cpp
similarity index 100%
rename from offload/src/LegacyAPI.cpp
rename to offload/libomptarget/LegacyAPI.cpp
diff --git a/offload/src/OffloadRTL.cpp b/offload/libomptarget/OffloadRTL.cpp
similarity index 100%
rename from offload/src/OffloadRTL.cpp
rename to offload/libomptarget/OffloadRTL.cpp
diff --git a/offload/src/OpenMP/API.cpp b/offload/libomptarget/OpenMP/API.cpp
similarity index 99%
rename from offload/src/OpenMP/API.cpp
rename to offload/libomptarget/OpenMP/API.cpp
index e59bdba8abf0e..4576f9bd06121 100644
--- a/offload/src/OpenMP/API.cpp
+++ b/offload/libomptarget/OpenMP/API.cpp
@@ -185,7 +185,7 @@ EXTERN int omp_target_is_present(const void *Ptr, int DeviceNum) {
   // omp_target_is_present tests whether a host pointer refers to storage that
   // is mapped to a given device. However, due to the lack of the storage size,
   // only check 1 byte. Cannot set size 0 which checks whether the pointer (zero
-  // lengh array) is mapped instead of the referred storage.
+  // length array) is mapped instead of the referred storage.
   TargetPointerResultTy TPR =
       DeviceOrErr->getMappingInfo().getTgtPtrBegin(const_cast<void *>(Ptr), 1,
                                                    /*UpdateRefCount=*/false,
@@ -256,7 +256,7 @@ EXTERN int omp_target_memcpy(void *Dst, const void *Src, size_t Length,
       FATAL_MESSAGE(DstDevice, "%s",
                     toString(DstDeviceOrErr.takeError()).c_str());
     // First try to use D2D memcpy which is more efficient. If fails, fall back
-    // to unefficient way.
+    // to inefficient way.
     if (SrcDeviceOrErr->isDataExchangable(*DstDeviceOrErr)) {
       AsyncInfoTy AsyncInfo(*SrcDeviceOrErr);
       Rc = SrcDeviceOrErr->dataExchange(SrcAddr, *DstDeviceOrErr, DstAddr,
@@ -655,8 +655,8 @@ EXTERN void *omp_get_mapped_ptr(const void *Ptr, int DeviceNum) {
 
   int NumDevices = omp_get_initial_device();
   if (DeviceNum == NumDevices) {
-    DP("Device %d is initial device, returning Ptr " DPxMOD ".\n",
-           DeviceNum, DPxPTR(Ptr));
+    DP("Device %d is initial device, returning Ptr " DPxMOD ".\n", DeviceNum,
+       DPxPTR(Ptr));
     return const_cast<void *>(Ptr);
   }
 
diff --git a/offload/src/OpenMP/InteropAPI.cpp b/offload/libomptarget/OpenMP/InteropAPI.cpp
similarity index 100%
rename from offload/src/OpenMP/InteropAPI.cpp
rename to offload/libomptarget/OpenMP/InteropAPI.cpp
diff --git a/offload/src/OpenMP/Mapping.cpp b/offload/libomptarget/OpenMP/Mapping.cpp
similarity index 99%
rename from offload/src/OpenMP/Mapping.cpp
rename to offload/libomptarget/OpenMP/Mapping.cpp
index 595e3456ab54c..4b78ed3360a26 100644
--- a/offload/src/OpenMP/Mapping.cpp
+++ b/offload/libomptarget/OpenMP/Mapping.cpp
@@ -149,7 +149,7 @@ LookupResult MappingInfoTy::lookupMapping(HDTTMapAccessorTy &HDTTMap,
     //   std::prev(upper)->HDTT.HstPtrBegin <= hp < upper->HDTT.HstPtrBegin
     if (Upper != HDTTMap->begin()) {
       LR.TPR.setEntry(std::prev(Upper)->HDTT, OwnedTPR);
-      // the left side of extended address range is satisified.
+      // the left side of extended address range is satisfied.
       // hp >= LR.TPR.getEntry()->HstPtrBegin || hp >=
       // LR.TPR.getEntry()->HstPtrBase
       LR.Flags.IsContained = HP < LR.TPR.getEntry()->HstPtrEnd ||
@@ -158,7 +158,7 @@ LookupResult MappingInfoTy::lookupMapping(HDTTMapAccessorTy &HDTTMap,
 
     if (!LR.Flags.IsContained && Upper != HDTTMap->end()) {
       LR.TPR.setEntry(Upper->HDTT, OwnedTPR);
-      // the right side of extended address range is satisified.
+      // the right side of extended address range is satisfied.
       // hp < LR.TPR.getEntry()->HstPtrEnd || hp < LR.TPR.getEntry()->HstPtrBase
       LR.Flags.IsContained = HP >= LR.TPR.getEntry()->HstPtrBase;
     }
diff --git a/offload/src/OpenMP/OMPT/Callback.cpp b/offload/libomptarget/OpenMP/OMPT/Callback.cpp
similarity index 100%
rename from offload/src/OpenMP/OMPT/Callback.cpp
rename to offload/libomptarget/OpenMP/OMPT/Callback.cpp
diff --git a/offload/src/PluginManager.cpp b/offload/libomptarget/PluginManager.cpp
similarity index 86%
rename from offload/src/PluginManager.cpp
rename to offload/libomptarget/PluginManager.cpp
index 96fa0bb170489..8b77852d13439 100644
--- a/offload/src/PluginManager.cpp
+++ b/offload/libomptarget/PluginManager.cpp
@@ -124,16 +124,74 @@ void PluginManager::initializeAllDevices() {
   }
 }
 
+// Returns a pointer to the binary descriptor, upgrading from a legacy format if
+// necessary.
+__tgt_bin_desc *PluginManager::upgradeLegacyEntries(__tgt_bin_desc *Desc) {
+  struct LegacyEntryTy {
+    void *Address;
+    char *SymbolName;
+    size_t Size;
+    int32_t Flags;
+    int32_t Data;
+  };
+
+  if (UpgradedDescriptors.contains(Desc))
+    return &UpgradedDescriptors[Desc];
+
+  if (Desc->HostEntriesBegin == Desc->HostEntriesEnd ||
+      Desc->HostEntriesBegin->Reserved == 0)
+    return Desc;
+
+  // The new format mandates that each entry starts with eight bytes of zeroes.
+  // This allows us to detect the old format as this is a null pointer.
+  llvm::SmallVector<llvm::offloading::EntryTy, 0> &NewEntries =
+      LegacyEntries.emplace_back();
+  for (LegacyEntryTy &Entry : llvm::make_range(
+           reinterpret_cast<LegacyEntryTy *>(Desc->HostEntriesBegin),
+           reinterpret_cast<LegacyEntryTy *>(Desc->HostEntriesEnd))) {
+    llvm::offloading::EntryTy &NewEntry = NewEntries.emplace_back();
+
+    NewEntry.Address = Entry.Address;
+    NewEntry.Flags = Entry.Flags;
+    NewEntry.Data = Entry.Data;
+    NewEntry.Size = Entry.Size;
+    NewEntry.SymbolName = Entry.SymbolName;
+    NewEntry.Kind = object::OffloadKind::OFK_OpenMP;
+  }
+
+  // Create a new image struct so we can update the entries list.
+  llvm::SmallVector<__tgt_device_image, 0> &NewImages =
+      LegacyImages.emplace_back();
+  for (int32_t Image = 0; Image < Desc->NumDeviceImages; ++Image)
+    NewImages.emplace_back(
+        __tgt_device_image{Desc->DeviceImages[Image].ImageStart,
+                           Desc->DeviceImages[Image].ImageEnd,
+                           NewEntries.begin(), NewEntries.end()});
+
+  // Create the new binary descriptor containing the newly created memory.
+  __tgt_bin_desc &NewDesc = UpgradedDescriptors[Desc];
+  NewDesc.DeviceImages = NewImages.begin();
+  NewDesc.NumDeviceImages = Desc->NumDeviceImages;
+  NewDesc.HostEntriesBegin = NewEntries.begin();
+  NewDesc.HostEntriesEnd = NewEntries.end();
+
+  return &NewDesc;
+}
+
 void PluginManager::registerLib(__tgt_bin_desc *Desc) {
   PM->RTLsMtx.lock();
 
+  // Upgrade the entries from the legacy implementation if necessary.
+  Desc = upgradeLegacyEntries(Desc);
+
   // Add in all the OpenMP requirements associated with this binary.
   for (llvm::offloading::EntryTy &Entry :
        llvm::make_range(Desc->HostEntriesBegin, Desc->HostEntriesEnd))
-    if (Entry.Flags == OMP_REGISTER_REQUIRES)
+    if (Entry.Kind == object::OffloadKind::OFK_OpenMP &&
+        Entry.Flags == OMP_REGISTER_REQUIRES)
       PM->addRequirements(Entry.Data);
 
-  // Extract the exectuable image and extra information if availible.
+  // Extract the executable image and extra information if available.
   for (int32_t i = 0; i < Desc->NumDeviceImages; ++i)
     PM->addDeviceImage(*Desc, Desc->DeviceImages[i]);
 
@@ -217,7 +275,7 @@ void PluginManager::registerLib(__tgt_bin_desc *Desc) {
 
   // Auto Zero-Copy can only be currently triggered when the system is an
   // homogeneous APU architecture without attached discrete GPUs.
-  // If all devices suggest to use it, change requirment flags to trigger
+  // If all devices suggest to use it, change requirement flags to trigger
   // zero-copy behavior when mapping memory.
   if (UseAutoZeroCopy)
     addRequirements(OMPX_REQ_AUTO_ZERO_COPY);
@@ -232,6 +290,8 @@ int target(ident_t *Loc, DeviceTy &Device, void *HostPtr,
 void PluginManager::unregisterLib(__tgt_bin_desc *Desc) {
   DP("Unloading target library!\n");
 
+  Desc = upgradeLegacyEntries(Desc);
+
   PM->RTLsMtx.lock();
   // Find which RTL understands each image, if any.
   for (DeviceImageTy &DI : PM->deviceImages()) {
@@ -270,7 +330,8 @@ void PluginManager::unregisterLib(__tgt_bin_desc *Desc) {
   PM->TblMapMtx.lock();
   for (llvm::offloading::EntryTy *Cur = Desc->HostEntriesBegin;
        Cur < Desc->HostEntriesEnd; ++Cur) {
-    PM->HostPtrToTableMap.erase(Cur->Address);
+    if (Cur->Kind == object::OffloadKind::OFK_OpenMP)
+      PM->HostPtrToTableMap.erase(Cur->Address);
   }
 
   // Remove translation table for this descriptor.
@@ -340,6 +401,9 @@ static int loadImagesOntoDevice(DeviceTy &Device) {
           TransTable->TargetsEntries[DeviceId];
       for (llvm::offloading::EntryTy &Entry :
            llvm::make_range(Img->EntriesBegin, Img->EntriesEnd)) {
+        if (Entry.Kind != object::OffloadKind::OFK_OpenMP)
+          continue;
+
         __tgt_device_binary &Binary = *BinaryOrErr;
 
         llvm::offloading::EntryTy DeviceEntry = Entry;
@@ -377,22 +441,6 @@ static int loadImagesOntoDevice(DeviceTy &Device) {
       __tgt_target_table *TargetTable = TransTable->TargetsTable[DeviceId] =
           &TransTable->DeviceTables[DeviceId];
 
-      // 4) Verify whether the two table sizes match.
-      size_t Hsize =
-          TransTable->HostTable.EntriesEnd - TransTable->HostTable.EntriesBegin;
-      size_t Tsize = TargetTable->EntriesEnd - TargetTable->EntriesBegin;
-
-      // Invalid image for these host entries!
-      if (Hsize != Tsize) {
-        REPORT(
-            "Host and Target tables mismatch for device id %d [%zx != %zx].\n",
-            DeviceId, Hsize, Tsize);
-        TransTable->TargetsImages[DeviceId] = 0;
-        TransTable->TargetsTable[DeviceId] = 0;
-        Rc = OFFLOAD_FAIL;
-        break;
-      }
-
       MappingInfoTy::HDTTMapAccessorTy HDTTMap =
           Device.getMappingInfo().HostDataToTargetMap.getExclusiveAccessor();
 
@@ -403,7 +451,8 @@ static int loadImagesOntoDevice(DeviceTy &Device) {
               *EntryDeviceEnd = TargetTable->EntriesEnd;
            CurrDeviceEntry != EntryDeviceEnd;
            CurrDeviceEntry++, CurrHostEntry++) {
-        if (CurrDeviceEntry->Size == 0)
+        if (CurrDeviceEntry->Size == 0 ||
+            CurrDeviceEntry->Kind != object::OffloadKind::OFK_OpenMP)
           continue;
 
         assert(CurrDeviceEntry->Size == CurrHostEntry->Size &&
diff --git a/offload/src/device.cpp b/offload/libomptarget/device.cpp
similarity index 98%
rename from offload/src/device.cpp
rename to offload/libomptarget/device.cpp
index 943c778278730..2beb4093572da 100644
--- a/offload/src/device.cpp
+++ b/offload/libomptarget/device.cpp
@@ -196,7 +196,7 @@ int32_t DeviceTy::notifyDataMapped(void *HstPtr, int64_t Size) {
      DPxPTR(HstPtr), Size);
 
   if (RTL->data_notify_mapped(RTLDeviceID, HstPtr, Size)) {
-    REPORT("Notifiying about data mapping failed.\n");
+    REPORT("Notifying about data mapping failed.\n");
     return OFFLOAD_FAIL;
   }
   return OFFLOAD_SUCCESS;
@@ -206,7 +206,7 @@ int32_t DeviceTy::notifyDataUnmapped(void *HstPtr) {
   DP("Notifying about an unmapping: HstPtr=" DPxMOD "\n", DPxPTR(HstPtr));
 
   if (RTL->data_notify_unmapped(RTLDeviceID, HstPtr)) {
-    REPORT("Notifiying about data unmapping failed.\n");
+    REPORT("Notifying about data unmapping failed.\n");
     return OFFLOAD_FAIL;
   }
   return OFFLOAD_SUCCESS;
diff --git a/offload/src/exports b/offload/libomptarget/exports
similarity index 100%
rename from offload/src/exports
rename to offload/libomptarget/exports
diff --git a/offload/src/interface.cpp b/offload/libomptarget/interface.cpp
similarity index 99%
rename from offload/src/interface.cpp
rename to offload/libomptarget/interface.cpp
index ad84a43cef8af..ea354400f2e99 100644
--- a/offload/src/interface.cpp
+++ b/offload/libomptarget/interface.cpp
@@ -39,7 +39,7 @@ using namespace llvm::omp::target::ompt;
 //
 // The return bool indicates if the offload is to the host device
 // There are three possible results:
-// - Return false if the taregt device is ready for offload
+// - Return false if the target device is ready for offload
 // - Return true without reporting a runtime error if offload is
 //   disabled, perhaps because the initial device was specified.
 // - Report a runtime error and return true.
@@ -294,7 +294,7 @@ static KernelArgsTy *upgradeKernelArgs(KernelArgsTy *KernelArgs,
 
   // FIXME: This is a WA to "calibrate" the bad work done in the front end.
   // Delete this ugly code after the front end emits proper values.
-  auto CorrectMultiDim = [](uint32_t(&Val)[3]) {
+  auto CorrectMultiDim = [](uint32_t (&Val)[3]) {
     if (Val[1] == 0)
       Val[1] = 1;
     if (Val[2] == 0)
@@ -366,8 +366,8 @@ static inline int targetKernel(ident_t *Loc, int64_t DeviceId, int32_t NumTeams,
 
   int Rc = OFFLOAD_SUCCESS;
   Rc = target(Loc, *DeviceOrErr, HostPtr, *KernelArgs, AsyncInfo);
-  { // required to show syncronization
-    TIMESCOPE_WITH_DETAILS_AND_IDENT("Runtime: syncronize", "", Loc);
+  { // required to show synchronization
+    TIMESCOPE_WITH_DETAILS_AND_IDENT("Runtime: synchronize", "", Loc);
     if (Rc == OFFLOAD_SUCCESS)
       Rc = AsyncInfo.synchronize();
 
diff --git a/offload/src/omptarget.cpp b/offload/libomptarget/omptarget.cpp
similarity index 99%
rename from offload/src/omptarget.cpp
rename to offload/libomptarget/omptarget.cpp
index 89fa63347babe..5b25d955dd320 100644
--- a/offload/src/omptarget.cpp
+++ b/offload/libomptarget/omptarget.cpp
@@ -1409,7 +1409,7 @@ int target(ident_t *Loc, DeviceTy &Device, void *HostPtr,
   // API, we need the begin address itself, i.e. &A[N], as the API operates on
   // begin addresses, not bases. That's why we pass args and offsets as two
   // separate entities so that each plugin can do what it needs. This behavior
-  // was introdued via https://reviews.llvm.org/D33028 and commit 1546d319244c.
+  // was introduced via https://reviews.llvm.org/D33028 and commit 1546d319244c.
   SmallVector<void *> TgtArgs;
   SmallVector<ptrdiff_t> TgtOffsets;
 
@@ -1431,7 +1431,7 @@ int target(ident_t *Loc, DeviceTy &Device, void *HostPtr,
 
     // Clang might pass more values via the ArgPtrs to the runtime that we pass
     // on to the kernel.
-    // TOOD: Next time we adjust the KernelArgsTy we should introduce a new
+    // TODO: Next time we adjust the KernelArgsTy we should introduce a new
     // NumKernelArgs field.
     KernelArgs.NumArgs = TgtArgs.size();
   }
diff --git a/offload/src/private.h b/offload/libomptarget/private.h
similarity index 100%
rename from offload/src/private.h
rename to offload/libomptarget/private.h
diff --git a/offload/plugins-nextgen/amdgpu/src/rtl.cpp b/offload/plugins-nextgen/amdgpu/src/rtl.cpp
index 6fc75ac154289..92184ba796dbd 100644
--- a/offload/plugins-nextgen/amdgpu/src/rtl.cpp
+++ b/offload/plugins-nextgen/amdgpu/src/rtl.cpp
@@ -436,7 +436,7 @@ struct AMDGPUMemoryManagerTy : public DeviceAllocatorTy {
   /// have more previously allocated buffers.
   void *allocate(size_t Size, void *HstPtr, TargetAllocTy Kind) override;
 
-  /// Deallocation callack that will be called by the memory manager.
+  /// Deallocation callback that will be called by the memory manager.
   int free(void *TgtPtr, TargetAllocTy Kind) override {
     if (auto Err = MemoryPool->deallocate(TgtPtr)) {
       consumeError(std::move(Err));
@@ -493,7 +493,7 @@ struct AMDGPUDeviceImageTy : public DeviceImageTy {
   }
 
 private:
-  /// The exectuable loaded on the agent.
+  /// The executable loaded on the agent.
   hsa_executable_t Executable;
   StringMap<offloading::amdgpu::AMDGPUKernelMetaData> KernelInfoMap;
   uint16_t ELFABIVersion;
@@ -876,7 +876,7 @@ struct AMDGPUQueueTy {
     hsa_signal_store_relaxed(Queue->doorbell_signal, PacketId);
   }
 
-  /// Callack that will be called when an error is detected on the HSA queue.
+  /// Callback that will be called when an error is detected on the HSA queue.
   static void callbackError(hsa_status_t Status, hsa_queue_t *Source,
                             void *Data);
 
@@ -932,7 +932,7 @@ struct AMDGPUStreamTy {
   /// operation's output signal is set to the consumed slot's signal. If there
   /// is a previous asynchronous operation on the previous slot, the HSA async
   /// operation's input signal is set to the signal of the previous slot. This
-  /// way, we obtain a chain of dependant async operations. The action is a
+  /// way, we obtain a chain of dependent async operations. The action is a
   /// function that will be executed eventually after the operation is
   /// completed, e.g., for releasing a buffer.
   struct StreamSlotTy {
@@ -1055,10 +1055,10 @@ struct AMDGPUStreamTy {
   /// Timeout hint for HSA actively waiting for signal value to change
   const uint64_t StreamBusyWaitMicroseconds;
 
-  /// Indicate to spread data transfers across all avilable SDMAs
+  /// Indicate to spread data transfers across all available SDMAs
   bool UseMultipleSdmaEngines;
 
-  /// Return the current number of asychronous operations on the stream.
+  /// Return the current number of asynchronous operations on the stream.
   uint32_t size() const { return NextSlot; }
 
   /// Return the last valid slot on the stream.
@@ -1155,12 +1155,12 @@ struct AMDGPUStreamTy {
     // changes on the slot.
     std::atomic_thread_fence(std::memory_order_acquire);
 
-    // Peform the operation.
+    // Perform the operation.
     if (auto Err = Slot->performAction())
-      FATAL_MESSAGE(1, "Error peforming post action: %s",
+      FATAL_MESSAGE(1, "Error performing post action: %s",
                     toString(std::move(Err)).data());
 
-    // Signal the output signal to notify the asycnhronous operation finalized.
+    // Signal the output signal to notify the asynchronous operation finalized.
     Slot->Signal->signal();
 
     // Unregister callback.
@@ -1183,9 +1183,9 @@ struct AMDGPUStreamTy {
   /// action. There are two kinds of memory buffers:
   ///   1. For kernel arguments. This buffer can be freed after receiving the
   ///   kernel completion signal.
-  ///   2. For H2D tranfers that need pinned memory space for staging. This
+  ///   2. For H2D transfers that need pinned memory space for staging. This
   ///   buffer can be freed after receiving the transfer completion signal.
-  ///   3. For D2H tranfers that need pinned memory space for staging. This
+  ///   3. For D2H transfers that need pinned memory space for staging. This
   ///   buffer cannot be freed after receiving the transfer completion signal
   ///   because of the following asynchronous H2H callback.
   ///      For this reason, This action can only be taken at
@@ -1222,7 +1222,7 @@ struct AMDGPUStreamTy {
   /// Create an empty stream associated with a specific device.
   AMDGPUStreamTy(AMDGPUDeviceTy &Device);
 
-  /// Intialize the stream's signals.
+  /// Initialize the stream's signals.
   Error init() { return Plugin::success(); }
 
   /// Deinitialize the stream's signals.
@@ -1312,7 +1312,7 @@ struct AMDGPUStreamTy {
   /// Push an asynchronous memory copy device-to-host involving an unpinned
   /// memory buffer. The operation consists of a two-step copy from the
   /// device buffer to an intermediate pinned host buffer, and then, to a
-  /// unpinned host buffer. Both operations are asynchronous and dependant.
+  /// unpinned host buffer. Both operations are asynchronous and dependent.
   /// The intermediate pinned buffer will be released to the specified memory
   /// manager once the operation completes.
   Error pushMemoryCopyD2HAsync(void *Dst, const void *Src, void *Inter,
@@ -1374,7 +1374,7 @@ struct AMDGPUStreamTy {
   /// Push an asynchronous memory copy host-to-device involving an unpinned
   /// memory buffer. The operation consists of a two-step copy from the
   /// unpinned host buffer to an intermediate pinned host buffer, and then, to
-  /// the pinned host buffer. Both operations are asynchronous and dependant.
+  /// the pinned host buffer. Both operations are asynchronous and dependent.
   /// The intermediate pinned buffer will be released to the specified memory
   /// manager once the operation completes.
   Error pushMemoryCopyH2DAsync(void *Dst, const void *Src, void *Inter,
@@ -1672,7 +1672,7 @@ struct AMDGPUStreamManagerTy final
   }
 
 private:
-  /// Search for and assign an prefereably idle queue to the given Stream. If
+  /// Search for and assign an preferably idle queue to the given Stream. If
   /// there is no queue without current users, choose the queue with the lowest
   /// user count. If utilization is ignored: use round robin selection.
   inline Error assignNextQueue(AMDGPUStreamTy *Stream) {
@@ -1856,13 +1856,13 @@ struct AMDHostDeviceTy : public AMDGenericDeviceTy {
   /// Get a memory pool for fine-grained allocations.
   AMDGPUMemoryPoolTy &getFineGrainedMemoryPool() {
     assert(!FineGrainedMemoryPools.empty() && "No fine-grained mempool");
-    // Retrive any memory pool.
+    // Retrieve any memory pool.
     return *FineGrainedMemoryPools[0];
   }
 
   AMDGPUMemoryPoolTy &getCoarseGrainedMemoryPool() {
     assert(!CoarseGrainedMemoryPools.empty() && "No coarse-grained mempool");
-    // Retrive any memory pool.
+    // Retrieve any memory pool.
     return *CoarseGrainedMemoryPools[0];
   }
 
@@ -1937,7 +1937,7 @@ struct AMDGPUDeviceTy : public GenericDeviceTy, AMDGenericDeviceTy {
                          ClockFrequency) != HSA_STATUS_SUCCESS)
       ClockFrequency = 0;
 
-    // Load the grid values dependending on the wavefront.
+    // Load the grid values depending on the wavefront.
     if (WavefrontSize == 32)
       GridValues = getAMDGPUGridValues<32>();
     else if (WavefrontSize == 64)
@@ -2097,7 +2097,7 @@ struct AMDGPUDeviceTy : public GenericDeviceTy, AMDGenericDeviceTy {
 
     std::string LLDPath = ErrorOrPath.get();
     INFO(OMP_INFOTYPE_PLUGIN_KERNEL, getDeviceId(),
-         "Using `%s` to link JITed amdgcn ouput.", LLDPath.c_str());
+         "Using `%s` to link JITed amdgcn output.", LLDPath.c_str());
 
     std::string MCPU = "-plugin-opt=mcpu=" + getComputeUnitKind();
     StringRef Args[] = {LLDPath,
@@ -2158,15 +2158,15 @@ struct AMDGPUDeviceTy : public GenericDeviceTy, AMDGenericDeviceTy {
   }
 
   /// We want to set up the RPC server for host services to the GPU if it is
-  /// availible.
+  /// available.
   bool shouldSetupRPCServer() const override { return true; }
 
-  /// The RPC interface should have enough space for all availible parallelism.
+  /// The RPC interface should have enough space for all available parallelism.
   uint64_t requestedRPCPortCount() const override {
     return getHardwareParallelism();
   }
 
-  /// Get the stream of the asynchronous info sructure or get a new one.
+  /// Get the stream of the asynchronous info structure or get a new one.
   Error getStream(AsyncInfoWrapperTy &AsyncInfoWrapper,
                   AMDGPUStreamTy *&Stream) {
     // Get the stream (if any) from the async info.
@@ -2716,7 +2716,7 @@ struct AMDGPUDeviceTy : public GenericDeviceTy, AMDGenericDeviceTy {
       Status =
           Pool->getAttrRaw(HSA_AMD_MEMORY_POOL_INFO_ACCESSIBLE_BY_ALL, TmpBool);
       if (Status == HSA_STATUS_SUCCESS)
-        Info.add<InfoLevel3>("Accessable by all", TmpBool);
+        Info.add<InfoLevel3>("Accessible by all", TmpBool);
     }
 
     Info.add("ISAs");
@@ -2895,7 +2895,7 @@ struct AMDGPUDeviceTy : public GenericDeviceTy, AMDGenericDeviceTy {
 
   /// Envar specifying the maximum size in bytes where the memory copies are
   /// asynchronous operations. Up to this transfer size, the memory copies are
-  /// asychronous operations pushed to the corresponding stream. For larger
+  /// asynchronous operations pushed to the corresponding stream. For larger
   /// transfers, they are synchronous transfers.
   UInt32Envar OMPX_MaxAsyncCopyBytes;
 
diff --git a/offload/plugins-nextgen/common/CMakeLists.txt b/offload/plugins-nextgen/common/CMakeLists.txt
index 14c48f6ace971..de219efc8f79c 100644
--- a/offload/plugins-nextgen/common/CMakeLists.txt
+++ b/offload/plugins-nextgen/common/CMakeLists.txt
@@ -21,7 +21,7 @@ if (NOT LLVM_LINK_LLVM_DYLIB)
   endforeach()
 endif()
 
-# Include the RPC server from the `libc` project if availible.
+# Include the RPC server from the `libc` project if available.
 include(FindLibcCommonUtils)
 target_link_libraries(PluginCommon PRIVATE llvm-libc-common-utilities)
 if(TARGET llvmlibc_rpc_server AND ${LIBOMPTARGET_GPU_LIBC_SUPPORT})
diff --git a/offload/plugins-nextgen/common/include/ErrorReporting.h b/offload/plugins-nextgen/common/include/ErrorReporting.h
index 8478977a8f86a..2ad0f2b7dd6c6 100644
--- a/offload/plugins-nextgen/common/include/ErrorReporting.h
+++ b/offload/plugins-nextgen/common/include/ErrorReporting.h
@@ -80,8 +80,10 @@ class ErrorReporter {
   /// Print \p Format, instantiated with \p Args to stderr.
   /// TODO: Allow redirection into a file stream.
   template <typename... ArgsTy>
-  [[gnu::format(__printf__, 1, 2)]] static void print(const char *Format,
-                                                      ArgsTy &&...Args) {
+#ifdef __clang__ // https://gcc.gnu.org/bugzilla/show_bug.cgi?id=77958
+  [[gnu::format(__printf__, 1, 2)]]
+#endif
+  static void print(const char *Format, ArgsTy &&...Args) {
     raw_fd_ostream OS(STDERR_FILENO, false);
     OS << llvm::format(Format, Args...);
   }
@@ -89,8 +91,10 @@ class ErrorReporter {
   /// Print \p Format, instantiated with \p Args to stderr, but colored.
   /// TODO: Allow redirection into a file stream.
   template <typename... ArgsTy>
-  [[gnu::format(__printf__, 2, 3)]] static void
-  print(ColorTy Color, const char *Format, ArgsTy &&...Args) {
+#ifdef __clang__ // https://gcc.gnu.org/bugzilla/show_bug.cgi?id=77958
+  [[gnu::format(__printf__, 2, 3)]]
+#endif
+  static void print(ColorTy Color, const char *Format, ArgsTy &&...Args) {
     raw_fd_ostream OS(STDERR_FILENO, false);
     WithColor(OS, HighlightColor(Color)) << llvm::format(Format, Args...);
   }
@@ -99,8 +103,10 @@ class ErrorReporter {
   /// a banner.
   /// TODO: Allow redirection into a file stream.
   template <typename... ArgsTy>
-  [[gnu::format(__printf__, 1, 2)]] static void reportError(const char *Format,
-                                                            ArgsTy &&...Args) {
+#ifdef __clang__ // https://gcc.gnu.org/bugzilla/show_bug.cgi?id=77958
+  [[gnu::format(__printf__, 1, 2)]]
+#endif
+  static void reportError(const char *Format, ArgsTy &&...Args) {
     print(BoldRed, "%s", ErrorBanner);
     print(BoldRed, Format, Args...);
     print("\n");
diff --git a/offload/plugins-nextgen/common/include/GlobalHandler.h b/offload/plugins-nextgen/common/include/GlobalHandler.h
index d2914e7cd0eb4..7c7e6de613c9f 100644
--- a/offload/plugins-nextgen/common/include/GlobalHandler.h
+++ b/offload/plugins-nextgen/common/include/GlobalHandler.h
@@ -1,4 +1,4 @@
-//===- GlobalHandler.h - Target independent global & enviroment handling --===//
+//===- GlobalHandler.h - Target independent global & environment handling -===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
@@ -100,7 +100,7 @@ template <typename Ty> class StaticGlobalTy : public GlobalTy {
 
 /// Helper class to do the heavy lifting when it comes to moving globals between
 /// host and device. Through the GenericDeviceTy we access memcpy DtoH and HtoD,
-/// which means the only things specialized by the subclass is the retrival of
+/// which means the only things specialized by the subclass is the retrieval of
 /// global metadata (size, addr) from the device.
 /// \see getGlobalMetadataFromDevice
 class GenericGlobalHandlerTy {
diff --git a/offload/plugins-nextgen/common/include/JIT.h b/offload/plugins-nextgen/common/include/JIT.h
index 4414926a6178f..8c530436a754b 100644
--- a/offload/plugins-nextgen/common/include/JIT.h
+++ b/offload/plugins-nextgen/common/include/JIT.h
@@ -59,7 +59,7 @@ struct JITEngine {
   /// Compile the bitcode image \p Image and generate the binary image that can
   /// be loaded to the target device of the triple \p Triple architecture \p
   /// MCpu. \p PostProcessing will be called after codegen to handle cases such
-  /// as assember as an external tool.
+  /// as assembler as an external tool.
   Expected<const __tgt_device_image *>
   compile(const __tgt_device_image &Image, const std::string &ComputeUnitKind,
           PostProcessingFn PostProcessing);
diff --git a/offload/plugins-nextgen/common/include/PluginInterface.h b/offload/plugins-nextgen/common/include/PluginInterface.h
index eb266e8d4d451..a30589e039468 100644
--- a/offload/plugins-nextgen/common/include/PluginInterface.h
+++ b/offload/plugins-nextgen/common/include/PluginInterface.h
@@ -92,14 +92,14 @@ struct AsyncInfoWrapperTy {
   }
 
   /// Synchronize with the __tgt_async_info's pending operations if it's the
-  /// internal async info. The error associated to the aysnchronous operations
+  /// internal async info. The error associated to the asynchronous operations
   /// issued in this queue must be provided in \p Err. This function will update
   /// the error parameter with the result of the synchronization if it was
   /// actually executed. This function must be called before destroying the
   /// object and only once.
   void finalize(Error &Err);
 
-  /// Register \p Ptr as an associated alloction that is freed after
+  /// Register \p Ptr as an associated allocation that is freed after
   /// finalization.
   void freeAllocationAfterSynchronization(void *Ptr) {
     AsyncInfoPtr->AssociatedAllocations.push_back(Ptr);
@@ -456,7 +456,7 @@ struct KernelTraceInfoRecordTy {
 };
 
 /// Class representing a map of host pinned allocations. We track these pinned
-/// allocations, so memory tranfers invloving these buffers can be optimized.
+/// allocations, so memory transfers involving these buffers can be optimized.
 class PinnedAllocationMapTy {
 
   /// Struct representing a map entry.
@@ -482,7 +482,7 @@ class PinnedAllocationMapTy {
     /// becomes zero.
     mutable size_t References;
 
-    /// Create an entry with the host and device acessible pointers, the buffer
+    /// Create an entry with the host and device accessible pointers, the buffer
     /// size, and a boolean indicating whether the buffer was locked externally.
     EntryTy(void *HstPtr, void *DevAccessiblePtr, size_t Size,
             bool ExternallyLocked)
@@ -517,7 +517,7 @@ class PinnedAllocationMapTy {
   /// Indicate whether mapped host buffers should be locked automatically.
   bool LockMappedBuffers;
 
-  /// Indicate whether failures when locking mapped buffers should be ingored.
+  /// Indicate whether failures when locking mapped buffers should be ignored.
   bool IgnoreLockMappedFailures;
 
   /// Find an allocation that intersects with \p HstPtr pointer. Assume the
@@ -1122,7 +1122,7 @@ struct GenericPluginTy {
   /// Get the reference to the device with a certain device id.
   GenericDeviceTy &getDevice(int32_t DeviceId) {
     assert(isValidDeviceId(DeviceId) && "Invalid device id");
-    assert(Devices[DeviceId] && "Device is unitialized");
+    assert(Devices[DeviceId] && "Device is uninitialized");
 
     return *Devices[DeviceId];
   }
@@ -1270,7 +1270,7 @@ struct GenericPluginTy {
   int32_t data_retrieve(int32_t DeviceId, void *HstPtr, void *TgtPtr,
                         int64_t Size);
 
-  /// Copy data from the given device asynchornously.
+  /// Copy data from the given device asynchronously.
   int32_t data_retrieve_async(int32_t DeviceId, void *HstPtr, void *TgtPtr,
                               int64_t Size, __tgt_async_info *AsyncInfoPtr);
 
@@ -1308,7 +1308,7 @@ struct GenericPluginTy {
   int32_t wait_event(int32_t DeviceId, void *EventPtr,
                      __tgt_async_info *AsyncInfoPtr);
 
-  /// Syncrhonize execution until an event is done.
+  /// Synchronize execution until an event is done.
   int32_t sync_event(int32_t DeviceId, void *EventPtr);
 
   /// Remove the event from the plugin.
@@ -1327,7 +1327,7 @@ struct GenericPluginTy {
   /// Sets the offset into the devices for use by OMPT.
   int32_t set_device_identifier(int32_t UserId, int32_t DeviceId);
 
-  /// Returns if the plugin can support auotmatic copy.
+  /// Returns if the plugin can support automatic copy.
   int32_t use_auto_zero_copy(int32_t DeviceId);
 
   /// Look up a global symbol in the given binary.
diff --git a/offload/plugins-nextgen/common/include/RPC.h b/offload/plugins-nextgen/common/include/RPC.h
index f3a8e7555020d..ab110dd88315a 100644
--- a/offload/plugins-nextgen/common/include/RPC.h
+++ b/offload/plugins-nextgen/common/include/RPC.h
@@ -34,7 +34,7 @@ class DeviceImageTy;
 } // namespace plugin
 
 /// A generic class implementing the interface between the RPC server provided
-/// by the 'libc' project and 'libomptarget'. If the RPC server is not availible
+/// by the 'libc' project and 'libomptarget'. If the RPC server is not available
 /// these routines will perform no action.
 struct RPCServerTy {
 public:
@@ -48,7 +48,7 @@ struct RPCServerTy {
   llvm::Error startThread();
 
   /// Check if this device image is using an RPC server. This checks for the
-  /// precense of an externally visible symbol in the device image that will
+  /// presence of an externally visible symbol in the device image that will
   /// be present whenever RPC code is called.
   llvm::Expected<bool> isDeviceUsingRPC(plugin::GenericDeviceTy &Device,
                                         plugin::GenericGlobalHandlerTy &Handler,
@@ -72,6 +72,9 @@ struct RPCServerTy {
   /// Array of associated devices. These must be alive as long as the server is.
   std::unique_ptr<plugin::GenericDeviceTy *[]> Devices;
 
+  /// Mutex that guards accesses to the buffers and device array.
+  std::mutex BufferMutex{};
+
   /// A helper class for running the user thread that handles the RPC interface.
   /// Because we only need to check the RPC server while any kernels are
   /// working, we track submission / completion events to allow the thread to
@@ -80,7 +83,7 @@ struct RPCServerTy {
     std::thread Worker;
 
     /// A boolean indicating whether or not the worker thread should continue.
-    std::atomic<bool> Running;
+    std::atomic<uint32_t> Running;
 
     /// The number of currently executing kernels across all devices that need
     /// the server thread to be running.
@@ -90,6 +93,9 @@ struct RPCServerTy {
     std::condition_variable CV;
     std::mutex Mutex;
 
+    /// A reference to the main server's mutex.
+    std::mutex &BufferMutex;
+
     /// A reference to all the RPC interfaces that the server is handling.
     llvm::ArrayRef<void *> Buffers;
 
@@ -98,9 +104,9 @@ struct RPCServerTy {
 
     /// Initialize the worker thread to run in the background.
     ServerThread(void *Buffers[], plugin::GenericDeviceTy *Devices[],
-                 size_t Length)
-        : Running(true), NumUsers(0), CV(), Mutex(), Buffers(Buffers, Length),
-          Devices(Devices, Length) {}
+                 size_t Length, std::mutex &BufferMutex)
+        : Running(false), NumUsers(0), CV(), Mutex(), BufferMutex(BufferMutex),
+          Buffers(Buffers, Length), Devices(Devices, Length) {}
 
     ~ServerThread() { assert(!Running && "Thread not shut down explicitly\n"); }
 
diff --git a/offload/plugins-nextgen/common/src/GlobalHandler.cpp b/offload/plugins-nextgen/common/src/GlobalHandler.cpp
index 8ad7f15990ac6..8854fc52205a7 100644
--- a/offload/plugins-nextgen/common/src/GlobalHandler.cpp
+++ b/offload/plugins-nextgen/common/src/GlobalHandler.cpp
@@ -67,7 +67,7 @@ Error GenericGlobalHandlerTy::moveGlobalBetweenDeviceAndHost(
       return Err;
   }
 
-  DP("Succesfully %s %u bytes associated with global symbol '%s' %s the "
+  DP("Successfully %s %u bytes associated with global symbol '%s' %s the "
      "device "
      "(%p -> %p).\n",
      Device2Host ? "read" : "write", HostGlobal.getSize(),
diff --git a/offload/plugins-nextgen/common/src/PluginInterface.cpp b/offload/plugins-nextgen/common/src/PluginInterface.cpp
index c9acabea6977d..48c9b671c1a91 100644
--- a/offload/plugins-nextgen/common/src/PluginInterface.cpp
+++ b/offload/plugins-nextgen/common/src/PluginInterface.cpp
@@ -380,7 +380,8 @@ setupIndirectCallTable(GenericPluginTy &Plugin, GenericDeviceTy &Device,
       Image.getTgtImage()->EntriesBegin, Image.getTgtImage()->EntriesEnd);
   llvm::SmallVector<std::pair<void *, void *>> IndirectCallTable;
   for (const auto &Entry : Entries) {
-    if (Entry.Size == 0 || !(Entry.Flags & OMP_DECLARE_TARGET_INDIRECT))
+    if (Entry.Kind != object::OffloadKind::OFK_OpenMP || Entry.Size == 0 ||
+        !(Entry.Flags & OMP_DECLARE_TARGET_INDIRECT))
       continue;
 
     assert(Entry.Size == sizeof(void *) && "Global not a function pointer?");
@@ -738,7 +739,7 @@ GenericDeviceTy::GenericDeviceTy(GenericPluginTy &Plugin, int32_t DeviceId,
       OMPX_SharedMemorySize("LIBOMPTARGET_SHARED_MEMORY_SIZE"),
       // Do not initialize the following two envars since they depend on the
       // device initialization. These cannot be consulted until the device is
-      // initialized correctly. We intialize them in GenericDeviceTy::init().
+      // initialized correctly. We initialize them in GenericDeviceTy::init().
       OMPX_TargetStackSize(), OMPX_TargetHeapSize(),
       // By default, the initial number of streams and events is 1.
       OMPX_InitialNumStreams("LIBOMPTARGET_NUM_INITIAL_STREAMS", 1),
@@ -1040,7 +1041,7 @@ Error GenericDeviceTy::setupDeviceMemoryPool(GenericPluginTy &Plugin,
 
 Error GenericDeviceTy::setupRPCServer(GenericPluginTy &Plugin,
                                       DeviceImageTy &Image) {
-  // The plugin either does not need an RPC server or it is unavailible.
+  // The plugin either does not need an RPC server or it is unavailable.
   if (!shouldSetupRPCServer())
     return Plugin::success();
 
@@ -1325,16 +1326,16 @@ Error GenericDeviceTy::queryAsync(__tgt_async_info *AsyncInfo) {
 }
 
 Error GenericDeviceTy::memoryVAMap(void **Addr, void *VAddr, size_t *RSize) {
-  return Plugin::error("Device does not suppport VA Management");
+  return Plugin::error("Device does not support VA Management");
 }
 
 Error GenericDeviceTy::memoryVAUnMap(void *VAddr, size_t Size) {
-  return Plugin::error("Device does not suppport VA Management");
+  return Plugin::error("Device does not support VA Management");
 }
 
 Error GenericDeviceTy::getDeviceMemorySize(uint64_t &DSize) {
   return Plugin::error(
-      "Mising getDeviceMemorySize impelmentation (required by RR-heuristic");
+      "Missing getDeviceMemorySize implementation (required by RR-heuristic");
 }
 
 Expected<void *> GenericDeviceTy::dataAlloc(int64_t Size, void *HostPtr,
@@ -1813,7 +1814,7 @@ int32_t GenericPluginTy::initialize_record_replay(int32_t DeviceId,
 
   if (auto Err = RecordReplay->init(&Device, MemorySize, VAddr, Status,
                                     SaveOutput, ReqPtrArgOffset)) {
-    REPORT("WARNING RR did not intialize RR-properly with %lu bytes"
+    REPORT("WARNING RR did not initialize RR-properly with %lu bytes"
            "(Error: %s)\n",
            MemorySize, toString(std::move(Err)).data());
     RecordReplay->setStatus(RecordReplayTy::RRStatusTy::RRDeactivated);
@@ -1960,7 +1961,7 @@ int32_t GenericPluginTy::data_retrieve_async(int32_t DeviceId, void *HstPtr,
   auto Err =
       getDevice(DeviceId).dataRetrieve(HstPtr, TgtPtr, Size, AsyncInfoPtr);
   if (Err) {
-    REPORT("Faliure to copy data from device to host. Pointers: host "
+    REPORT("Failure to copy data from device to host. Pointers: host "
            "= " DPxMOD ", device = " DPxMOD ", size = %" PRId64 ": %s\n",
            DPxPTR(HstPtr), DPxPTR(TgtPtr), Size,
            toString(std::move(Err)).data());
diff --git a/offload/plugins-nextgen/common/src/RPC.cpp b/offload/plugins-nextgen/common/src/RPC.cpp
index 81ad9ca66808a..70f572923d4b1 100644
--- a/offload/plugins-nextgen/common/src/RPC.cpp
+++ b/offload/plugins-nextgen/common/src/RPC.cpp
@@ -99,13 +99,15 @@ static rpc::Status runServer(plugin::GenericDeviceTy &Device, void *Buffer) {
 }
 
 void RPCServerTy::ServerThread::startThread() {
-  Worker = std::thread([this]() { run(); });
+  if (!Running.fetch_or(true, std::memory_order_acquire))
+    Worker = std::thread([this]() { run(); });
 }
 
 void RPCServerTy::ServerThread::shutDown() {
+  if (!Running.fetch_and(false, std::memory_order_release))
+    return;
   {
     std::lock_guard<decltype(Mutex)> Lock(Mutex);
-    Running.store(false, std::memory_order_release);
     CV.notify_all();
   }
   if (Worker.joinable())
@@ -126,6 +128,7 @@ void RPCServerTy::ServerThread::run() {
     Lock.unlock();
     while (NumUsers.load(std::memory_order_relaxed) > 0 &&
            Running.load(std::memory_order_relaxed)) {
+      std::lock_guard<decltype(Mutex)> Lock(BufferMutex);
       for (const auto &[Buffer, Device] : llvm::zip_equal(Buffers, Devices)) {
         if (!Buffer || !Device)
           continue;
@@ -144,7 +147,7 @@ RPCServerTy::RPCServerTy(plugin::GenericPluginTy &Plugin)
       Devices(std::make_unique<plugin::GenericDeviceTy *[]>(
           Plugin.getNumDevices())),
       Thread(new ServerThread(Buffers.get(), Devices.get(),
-                              Plugin.getNumDevices())) {}
+                              Plugin.getNumDevices(), BufferMutex)) {}
 
 llvm::Error RPCServerTy::startThread() {
   Thread->startThread();
@@ -185,6 +188,7 @@ Error RPCServerTy::initDevice(plugin::GenericDeviceTy &Device,
   if (auto Err = Device.dataSubmit(ClientGlobal.getPtr(), &client,
                                    sizeof(rpc::Client), nullptr))
     return Err;
+  std::lock_guard<decltype(BufferMutex)> Lock(BufferMutex);
   Buffers[Device.getDeviceId()] = RPCBuffer;
   Devices[Device.getDeviceId()] = &Device;
 
@@ -192,6 +196,7 @@ Error RPCServerTy::initDevice(plugin::GenericDeviceTy &Device,
 }
 
 Error RPCServerTy::deinitDevice(plugin::GenericDeviceTy &Device) {
+  std::lock_guard<decltype(BufferMutex)> Lock(BufferMutex);
   Device.free(Buffers[Device.getDeviceId()], TARGET_ALLOC_HOST);
   Buffers[Device.getDeviceId()] = nullptr;
   Devices[Device.getDeviceId()] = nullptr;
diff --git a/offload/plugins-nextgen/cuda/src/rtl.cpp b/offload/plugins-nextgen/cuda/src/rtl.cpp
index 52e8a100dc87b..0d0c4858aa7fa 100644
--- a/offload/plugins-nextgen/cuda/src/rtl.cpp
+++ b/offload/plugins-nextgen/cuda/src/rtl.cpp
@@ -395,7 +395,7 @@ struct CUDADeviceTy : public GenericDeviceTy {
 
   virtual Error callGlobalConstructors(GenericPluginTy &Plugin,
                                        DeviceImageTy &Image) override {
-    // Check for the presense of global destructors at initialization time. This
+    // Check for the presence of global destructors at initialization time. This
     // is required when the image may be deallocated before destructors are run.
     GenericGlobalHandlerTy &Handler = Plugin.getGlobalHandler();
     if (Handler.isSymbolInImage(*this, Image, "nvptx$device$fini"))
@@ -495,15 +495,15 @@ struct CUDADeviceTy : public GenericDeviceTy {
   }
 
   /// We want to set up the RPC server for host services to the GPU if it is
-  /// availible.
+  /// available.
   bool shouldSetupRPCServer() const override { return true; }
 
-  /// The RPC interface should have enough space for all availible parallelism.
+  /// The RPC interface should have enough space for all available parallelism.
   uint64_t requestedRPCPortCount() const override {
     return getHardwareParallelism();
   }
 
-  /// Get the stream of the asynchronous info sructure or get a new one.
+  /// Get the stream of the asynchronous info structure or get a new one.
   Error getStream(AsyncInfoWrapperTy &AsyncInfoWrapper, CUstream &Stream) {
     // Get the stream (if any) from the async info.
     Stream = AsyncInfoWrapper.getQueueAs<CUstream>();
@@ -675,7 +675,7 @@ struct CUDADeviceTy : public GenericDeviceTy {
     if (Size >= Free) {
       *Addr = nullptr;
       return Plugin::error(
-          "Canot map memory size larger than the available device memory");
+          "Cannot map memory size larger than the available device memory");
     }
 
     // currently NVidia only supports pinned device types
diff --git a/offload/plugins-nextgen/host/dynamic_ffi/ffi.h b/offload/plugins-nextgen/host/dynamic_ffi/ffi.h
index 4a9e88cc4dc9c..80aa512236d28 100644
--- a/offload/plugins-nextgen/host/dynamic_ffi/ffi.h
+++ b/offload/plugins-nextgen/host/dynamic_ffi/ffi.h
@@ -38,8 +38,8 @@ typedef enum {
   FFI_BAD_ARGTYPE
 } ffi_status;
 
-// These are target depenent so we set them manually for each ABI by referencing
-// the FFI source.
+// These are target dependent so we set them manually for each ABI by
+// referencing the FFI source.
 typedef enum ffi_abi {
 #if (defined(_M_X64) || defined(__x86_64__))
   FFI_DEFAULT_ABI = 2, // FFI_UNIX64.
diff --git a/offload/test/CMakeLists.txt b/offload/test/CMakeLists.txt
index 8a827e0a625ef..4768d9ccf223b 100644
--- a/offload/test/CMakeLists.txt
+++ b/offload/test/CMakeLists.txt
@@ -1,6 +1,6 @@
 # CMakeLists.txt file for unit testing OpenMP offloading runtime library.
-if(NOT CMAKE_CXX_COMPILER_ID STREQUAL "Clang" OR
-   CMAKE_CXX_COMPILER_VERSION VERSION_LESS 6.0.0)
+if(NOT OPENMP_TEST_COMPILER_ID STREQUAL "Clang" OR
+   OPENMP_TEST_COMPILER_VERSION VERSION_LESS 6.0.0)
   message(STATUS "Can only test with Clang compiler in version 6.0.0 or later.")
   message(WARNING "The check-offload target will not be available!")
   return()
diff --git a/offload/test/api/omp_target_memcpy_async1.c b/offload/test/api/omp_target_memcpy_async1.c
index 1abcfde83dbd8..defc7829fcdac 100644
--- a/offload/test/api/omp_target_memcpy_async1.c
+++ b/offload/test/api/omp_target_memcpy_async1.c
@@ -1,6 +1,6 @@
 // RUN: %libomptarget-compile-and-run-generic
 
-// Test case for omp_target_memcpy_async, oringally from GCC
+// Test case for omp_target_memcpy_async, originally from GCC
 
 #include "stdio.h"
 #include <omp.h>
diff --git a/offload/test/mapping/target_uses_allocator.c b/offload/test/mapping/target_uses_allocator.c
index eb20e965c30bc..c0d71b22733b3 100755
--- a/offload/test/mapping/target_uses_allocator.c
+++ b/offload/test/mapping/target_uses_allocator.c
@@ -54,7 +54,7 @@ int test_omp_aligned_alloc_on_device() {
 int main() {
   int errors = 0;
   if (test_omp_aligned_alloc_on_device())
-    printf("FAILE\n");
+    printf("FAILED\n");
   else
     // CHECK: PASSED
     printf("PASSED\n");
diff --git a/offload/test/offloading/fortran/dump_map_tables.f90 b/offload/test/offloading/fortran/dump_map_tables.f90
index efde4ee56ca1a..424dec2066562 100644
--- a/offload/test/offloading/fortran/dump_map_tables.f90
+++ b/offload/test/offloading/fortran/dump_map_tables.f90
@@ -1,4 +1,4 @@
-! Offloading test with runtine call to ompx_dump_mapping_tables Fortran array
+! Offloading test with runtime call to ompx_dump_mapping_tables Fortran array
 ! writing some values and printing the variable mapped to device correctly
 ! receives the updates made on the device.
 ! REQUIRES: flang
diff --git a/offload/test/offloading/fortran/implicit-record-field-mapping.f90 b/offload/test/offloading/fortran/implicit-record-field-mapping.f90
index 77b13bed707c7..29894941c424b 100644
--- a/offload/test/offloading/fortran/implicit-record-field-mapping.f90
+++ b/offload/test/offloading/fortran/implicit-record-field-mapping.f90
@@ -3,7 +3,7 @@
 ! REQUIRES: flang, amdgpu
 
 ! This fails only because it needs the Fortran runtime built for device. If this
-! is avaialbe, this test succeeds when run.
+! is available, this test succeeds when run.
 ! XFAIL: *
 
 ! RUN: %libomptarget-compile-fortran-generic
diff --git a/offload/test/offloading/fortran/local-descriptor-map-regress.f90 b/offload/test/offloading/fortran/local-descriptor-map-regress.f90
index e6afc4a6fb9b1..659ca1c692b0f 100644
--- a/offload/test/offloading/fortran/local-descriptor-map-regress.f90
+++ b/offload/test/offloading/fortran/local-descriptor-map-regress.f90
@@ -1,10 +1,10 @@
 ! Small regression test that checks that we do not cause a runtime map error in
 ! cases where we are required to allocate a local variable for the fortran
-! descriptor to store into and then load from it, done so by re-using the
+! descriptor to store into and then load from it, done so by reusing the
 ! temporary local variable across all maps related to the mapped variable and
 ! associated local variable to make sure that each map does as it is intended
 ! to do with the original data. This prevents blobs of local descriptor data
-! remaining attatched on device long after it's supposed to, which can cause
+! remaining attached on device long after it's supposed to, which can cause
 ! weird map issues later in susbequent function invocations. However, it
 ! doesn't avoid a user shooting themselves in the foot by mapping data via
 ! enter and then not providing a corresponding exit.
diff --git a/offload/test/offloading/fortran/target-data-map-if-present.f90 b/offload/test/offloading/fortran/target-data-map-if-present.f90
new file mode 100644
index 0000000000000..c181573cd7a1c
--- /dev/null
+++ b/offload/test/offloading/fortran/target-data-map-if-present.f90
@@ -0,0 +1,29 @@
+! Offloading test that tests that if(present(a)) compiles and executes without
+! causing any compilation errors, primarily a regression test that does not
+! yield interesting results.
+! REQUIRES: flang, amdgpu
+
+! RUN: %libomptarget-compile-fortran-run-and-check-generic
+module mod
+    implicit none
+contains
+    subroutine routine(a)
+        implicit none
+        real, dimension(:), optional :: a
+        integer :: i
+       !$omp target data if(present(a)) map(alloc:a)
+            do i = 1, 10
+                a(i) = i
+            end do
+       !$omp end target data
+    end subroutine routine
+end module mod
+
+program main
+    use mod
+    real :: a(10)
+    call routine(a)
+    print *, a
+end program main
+
+! CHECK: 1. 2. 3. 4. 5. 6. 7. 8. 9. 10.
diff --git a/offload/test/offloading/fortran/target-map-derived-type-full-1.f90 b/offload/test/offloading/fortran/target-map-derived-type-full-1.f90
index 9f2aeb4bf9fc0..09a847f2cef8c 100644
--- a/offload/test/offloading/fortran/target-map-derived-type-full-1.f90
+++ b/offload/test/offloading/fortran/target-map-derived-type-full-1.f90
@@ -1,5 +1,5 @@
 ! Offloading test checking interaction of an explicit derived type mapping when
-! mapped to target and assinging one derived type to another
+! mapped to target and assigning one derived type to another
 ! REQUIRES: flang, amdgpu
 
 ! RUN: %libomptarget-compile-fortran-run-and-check-generic
diff --git a/offload/test/offloading/fortran/target-map-derived-type-full-implicit-1.f90 b/offload/test/offloading/fortran/target-map-derived-type-full-implicit-1.f90
index 8632b951d6fbd..e3d82b7693865 100644
--- a/offload/test/offloading/fortran/target-map-derived-type-full-implicit-1.f90
+++ b/offload/test/offloading/fortran/target-map-derived-type-full-implicit-1.f90
@@ -1,5 +1,5 @@
 ! Offloading test checking interaction of an implicit derived type mapping when
-! mapped to target and assinging one derived type to another
+! mapped to target and assigning one derived type to another
 ! REQUIRES: flang, amdgpu
 
 ! RUN: %libomptarget-compile-fortran-run-and-check-generic
diff --git a/offload/test/offloading/fortran/target-map-derived-type-full-implicit-2.f90 b/offload/test/offloading/fortran/target-map-derived-type-full-implicit-2.f90
index 9331a48c3eb75..a6cc6a548960f 100644
--- a/offload/test/offloading/fortran/target-map-derived-type-full-implicit-2.f90
+++ b/offload/test/offloading/fortran/target-map-derived-type-full-implicit-2.f90
@@ -1,5 +1,5 @@
 ! Offloading test checking interaction of an explicit derived type mapping when
-! mapped to target and assinging one derived type to another
+! mapped to target and assigning one derived type to another
 ! REQUIRES: flang, amdgpu
 
 ! RUN: %libomptarget-compile-fortran-run-and-check-generic
diff --git a/offload/test/offloading/fortran/target-map-nullary-pointer.f90 b/offload/test/offloading/fortran/target-map-nullary-pointer.f90
new file mode 100644
index 0000000000000..276725d36fb61
--- /dev/null
+++ b/offload/test/offloading/fortran/target-map-nullary-pointer.f90
@@ -0,0 +1,24 @@
+! Offloading test with a target region mapping a null-ary (no target or
+! allocated data) to device, and then setting the target on device before
+! printing the changed target on host.
+! REQUIRES: flang, amdgpu
+
+! RUN: %libomptarget-compile-fortran-run-and-check-generic
+program main
+    implicit none
+    integer,    pointer :: Set
+    integer,    target, allocatable :: Set_Target
+
+    allocate(Set_Target)
+
+    Set_Target = 30
+
+!$omp target map(Set)
+    Set => Set_Target
+    Set = 45
+!$omp end target
+
+    print *, Set_Target
+end program main
+
+! CHECK: 45
diff --git a/offload/test/offloading/parallel_target_teams_reduction_max.cpp b/offload/test/offloading/parallel_target_teams_reduction_max.cpp
index b6e39daf718c8..ae0cf25a90178 100644
--- a/offload/test/offloading/parallel_target_teams_reduction_max.cpp
+++ b/offload/test/offloading/parallel_target_teams_reduction_max.cpp
@@ -5,7 +5,7 @@
 // REQUIRES: gpu
 
 // This test validates that the OpenMP target reductions to find a maximum work
-// as indended for a few common data types.
+// as intended for a few common data types.
 
 #include <algorithm>
 #include <cassert>
diff --git a/offload/test/offloading/parallel_target_teams_reduction_min.cpp b/offload/test/offloading/parallel_target_teams_reduction_min.cpp
index 1ab4e9868985e..8a1610357a506 100644
--- a/offload/test/offloading/parallel_target_teams_reduction_min.cpp
+++ b/offload/test/offloading/parallel_target_teams_reduction_min.cpp
@@ -5,7 +5,7 @@
 // REQUIRES: gpu
 
 // This test validates that the OpenMP target reductions to find a minimum work
-// as indended for a few common data types.
+// as intended for a few common data types.
 
 #include <algorithm>
 #include <cassert>
diff --git a/offload/test/offloading/requires.c b/offload/test/offloading/requires.c
index 2a129a7ae86dc..249c478215d42 100644
--- a/offload/test/offloading/requires.c
+++ b/offload/test/offloading/requires.c
@@ -17,11 +17,15 @@
 // Various definitions copied from OpenMP RTL
 
 typedef struct {
-  void *addr;
-  char *name;
-  size_t size;
-  int32_t flags;
-  int32_t data;
+  uint64_t Reserved;
+  uint16_t Version;
+  uint16_t Kind;
+  uint32_t Flags;
+  void *Address;
+  char *SymbolName;
+  uint64_t Size;
+  uint64_t Data;
+  void *AuxAddr;
 } __tgt_offload_entry;
 
 enum Flags {
@@ -55,8 +59,9 @@ void run_reg_requires() {
 
   // This is the 2nd time this function is called so it should print SUCCESS if
   // REQ is compatible with `1` and otherwise cause an error.
-  __tgt_offload_entry entries[] = {{NULL, "", 0, OMP_REGISTER_REQUIRES, 1},
-                                   {NULL, "", 0, OMP_REGISTER_REQUIRES, REQ}};
+  __tgt_offload_entry entries[] = {
+      {0, 0, 1, OMP_REGISTER_REQUIRES, NULL, "", 0, 1, NULL},
+      {0, 0, 1, OMP_REGISTER_REQUIRES, NULL, "", 0, REQ, NULL}};
   __tgt_device_image image = {NULL, NULL, &entries[0], &entries[1] + 1};
   __tgt_bin_desc bin = {1, &image, &entries[0], &entries[1] + 1};
 
@@ -78,7 +83,8 @@ int main() {
 
 // This also runs reg requires for the first time.
 #pragma omp target
-  {}
+  {
+  }
 
   return 0;
 }
diff --git a/offload/test/offloading/struct_mapping_with_pointers.cpp b/offload/test/offloading/struct_mapping_with_pointers.cpp
index f0fde50889dac..9c1d4c67c2ee7 100644
--- a/offload/test/offloading/struct_mapping_with_pointers.cpp
+++ b/offload/test/offloading/struct_mapping_with_pointers.cpp
@@ -32,7 +32,7 @@ int main() {
   dat.datum[dat.arr[0][0]] = 0;
 
   /// The struct is mapped with type 0x0 when the pointer fields are mapped.
-  /// The struct is also map explicitely by the user. The second mapping by
+  /// The struct is also map explicitly by the user. The second mapping by
   /// the user must not overwrite the mapping set up for the pointer fields
   /// when mapping the struct happens after the mapping of the pointers.
 
diff --git a/offload/test/sanitizer/kernel_crash_many.c b/offload/test/sanitizer/kernel_crash_many.c
index f1d17ca2b76e2..9fd8af48f51fe 100644
--- a/offload/test/sanitizer/kernel_crash_many.c
+++ b/offload/test/sanitizer/kernel_crash_many.c
@@ -37,36 +37,36 @@ int main(void) {
 // CHECK: Kernel 1: {{.*}} (__omp_offloading_{{.*}}_main_l22)
 // CHECK:     launchKernel
 // NDEBG:     main
-// DEBUG:     main {{.*}}kernel_crash_many.c:
+// DEBUG:     main {{.*}}kernel_crash_many.c
 //
 // CHECK: Kernel 2: {{.*}} (__omp_offloading_{{.*}}_main_l22)
 // CHECK:     launchKernel
 // NDEBG:     main
-// DEBUG:     main {{.*}}kernel_crash_many.c:
+// DEBUG:     main {{.*}}kernel_crash_many.c
 //
 // CHECK: Kernel 3: {{.*}} (__omp_offloading_{{.*}}_main_l22)
 // CHECK:     launchKernel
 // NDEBG:     main
-// DEBUG:     main {{.*}}kernel_crash_many.c:
+// DEBUG:     main {{.*}}kernel_crash_many.c
 //
 // CHECK: Kernel 4: {{.*}} (__omp_offloading_{{.*}}_main_l22)
 // CHECK:     launchKernel
 // NDEBG:     main
-// DEBUG:     main {{.*}}kernel_crash_many.c:
+// DEBUG:     main {{.*}}kernel_crash_many.c
 //
 // CHECK: Kernel 5: {{.*}} (__omp_offloading_{{.*}}_main_l22)
 // CHECK:     launchKernel
 // NDEBG:     main
-// DEBUG:     main {{.*}}kernel_crash_many.c:
+// DEBUG:     main {{.*}}kernel_crash_many.c
 //
 // CHECK: Kernel 6: {{.*}} (__omp_offloading_{{.*}}_main_l22)
 // CHECK:     launchKernel
 // NDEBG:     main
-// DEBUG:     main {{.*}}kernel_crash_many.c:
+// DEBUG:     main {{.*}}kernel_crash_many.c
 //
 // CHECK: Kernel 7: {{.*}} (__omp_offloading_{{.*}}_main_l22)
 // CHECK:     launchKernel
 // NDEBG:     main
-// DEBUG:     main {{.*}}kernel_crash_many.c:
+// DEBUG:     main {{.*}}kernel_crash_many.c
 //
 // CHECK-NOT: Kernel {{[[0-9]]+}}:
diff --git a/offload/test/sanitizer/kernel_trap.c b/offload/test/sanitizer/kernel_trap.c
index 91c4c7229159b..3a531bd74c980 100644
--- a/offload/test/sanitizer/kernel_trap.c
+++ b/offload/test/sanitizer/kernel_trap.c
@@ -1,10 +1,10 @@
 
 // clang-format off
 // RUN: %libomptarget-compile-generic
-// RUN: %not --crash env -u LLVM_DISABLE_SYMBOLIZATION OFFLOAD_TRACK_NUM_KERNEL_LAUNCH_TRACES=1 %libomptarget-run-generic 2>&1 | %fcheck-generic --check-prefixes=CHECK,TRACE,NDEBG 
+// RUN: %not --crash env -u LLVM_DISABLE_SYMBOLIZATION OFFLOAD_TRACK_NUM_KERNEL_LAUNCH_TRACES=1 %libomptarget-run-generic 2>&1 | %fcheck-generic --check-prefixes=CHECK,TRACE
 // RUN: %not --crash %libomptarget-run-generic 2>&1 | %fcheck-generic --check-prefixes=CHECK
 // RUN: %libomptarget-compile-generic -g
-// RUN: %not --crash env -u LLVM_DISABLE_SYMBOLIZATION OFFLOAD_TRACK_NUM_KERNEL_LAUNCH_TRACES=1 %libomptarget-run-generic 2>&1 | %fcheck-generic --check-prefixes=CHECK,TRACE,DEBUG
+// RUN: %not --crash env -u LLVM_DISABLE_SYMBOLIZATION OFFLOAD_TRACK_NUM_KERNEL_LAUNCH_TRACES=1 %libomptarget-run-generic 2>&1 | %fcheck-generic --check-prefixes=CHECK,TRACE
 // RUN: %not --crash %libomptarget-run-generic 2>&1 | %fcheck-generic --check-prefixes=CHECK
 // clang-format on
 
@@ -39,6 +39,4 @@ int main(void) {
 // CHECK: OFFLOAD ERROR: Kernel 'omp target in main @ 30 (__omp_offloading_{{.*}}_main_l30)'
 // CHECK: OFFLOAD ERROR: execution interrupted by hardware trap instruction
 // TRACE:     launchKernel
-// NDEBG:     main
-// DEBUG:     main {{.*}}kernel_trap.c:
 // clang-format on
diff --git a/offload/test/sanitizer/kernel_trap.cpp b/offload/test/sanitizer/kernel_trap.cpp
index c67b3857fabba..44858be6cd3f6 100644
--- a/offload/test/sanitizer/kernel_trap.cpp
+++ b/offload/test/sanitizer/kernel_trap.cpp
@@ -47,6 +47,6 @@ int main(void) {
 // TRACE:     launchKernel
 // NDEBG:     cxx_function_name<S>(int, S*)
 // NDEBG:     main
-// DEBUG:     cxx_function_name<S>(int, S*) {{.*}}kernel_trap.cpp:
-// DEBUG:     main {{.*}}kernel_trap.cpp:
+// DEBUG:     cxx_function_name<S>(int, S*) {{.*}}kernel_trap.cpp
+// DEBUG:     main {{.*}}kernel_trap.cpp
 // clang-format on
diff --git a/offload/test/sanitizer/kernel_trap_many.c b/offload/test/sanitizer/kernel_trap_many.c
index f2e63794168b2..061c0fe225d4b 100644
--- a/offload/test/sanitizer/kernel_trap_many.c
+++ b/offload/test/sanitizer/kernel_trap_many.c
@@ -32,4 +32,4 @@ int main(void) {
 // TRACE: OFFLOAD ERROR: execution interrupted by hardware trap instruction
 // TRACE:     launchKernel
 // NDEBG:     main
-// DEBUG:     main {{.*}}kernel_trap_many.c:
+// DEBUG:     main {{.*}}kernel_trap_many.c
diff --git a/offload/test/unified_shared_memory/associate_ptr.c b/offload/test/unified_shared_memory/associate_ptr.c
index 5f795dd42dbd3..0e9d25cd73546 100644
--- a/offload/test/unified_shared_memory/associate_ptr.c
+++ b/offload/test/unified_shared_memory/associate_ptr.c
@@ -17,7 +17,7 @@ int main(int argc, char *argv[]) {
   int rc = omp_target_associate_ptr(&x, x_dev, sizeof x, 0, dev);
   assert(!rc && "expected omp_target_associate_ptr to succeed");
 
-// To determine whether x needs to be transfered, the runtime cannot simply
+// To determine whether x needs to be transferred, the runtime cannot simply
 // check whether unified shared memory is enabled and the 'close' modifier is
 // specified.  It must check whether x was previously placed in device memory
 // by, for example, omp_target_associate_ptr.
diff --git a/offload/test/unified_shared_memory/close_member.c b/offload/test/unified_shared_memory/close_member.c
index e5a15e3d19ab6..10ec42e90b02b 100644
--- a/offload/test/unified_shared_memory/close_member.c
+++ b/offload/test/unified_shared_memory/close_member.c
@@ -23,7 +23,7 @@ int main(int argc, char *argv[]) {
     s.x = 11;
     s.y = 21;
   }
-// To determine whether x needs to be transfered or deleted, the runtime
+// To determine whether x needs to be transferred or deleted, the runtime
 // cannot simply check whether unified shared memory is enabled and the
 // 'close' modifier is specified.  It must check whether x was previously
 // placed in device memory by, for example, a 'close' modifier that isn't
diff --git a/offload/tools/kernelreplay/llvm-omp-kernel-replay.cpp b/offload/tools/kernelreplay/llvm-omp-kernel-replay.cpp
index ff77154033239..47a299f0ab557 100644
--- a/offload/tools/kernelreplay/llvm-omp-kernel-replay.cpp
+++ b/offload/tools/kernelreplay/llvm-omp-kernel-replay.cpp
@@ -74,7 +74,7 @@ int main(int argc, char **argv) {
   unsigned NumThreads =
       (NumThreadsOpt > 0 ? NumThreadsOpt : NumThreadsJson.value());
   // TODO: Print a warning if number of teams/threads is explicitly set in the
-  // kernel info but overriden through command line options.
+  // kernel info but overridden through command line options.
   auto LoopTripCount =
       JsonKernelInfo->getAsObject()->getInteger("LoopTripCount");
   auto KernelFunc = JsonKernelInfo->getAsObject()->getString("Name");
@@ -93,7 +93,8 @@ int main(int argc, char **argv) {
   void *BAllocStart = reinterpret_cast<void *>(
       JsonKernelInfo->getAsObject()->getInteger("BumpAllocVAStart").value());
 
-  llvm::offloading::EntryTy KernelEntry = {nullptr, nullptr, 0, 0, 0};
+  llvm::offloading::EntryTy KernelEntry = {~0U,     0, 0, 0,      nullptr,
+                                           nullptr, 0, 0, nullptr};
   std::string KernelEntryName = KernelFunc.value().str();
   KernelEntry.SymbolName = const_cast<char *>(KernelEntryName.c_str());
   // Anything non-zero works to uniquely identify the kernel.
diff --git a/openmp/docs/ReleaseNotes.rst b/openmp/docs/ReleaseNotes.rst
index 0089f1aa31d14..6c1a46caf1d81 100644
--- a/openmp/docs/ReleaseNotes.rst
+++ b/openmp/docs/ReleaseNotes.rst
@@ -1,5 +1,5 @@
 ===========================
-OpenMP 20.0.0 Release Notes
+OpenMP 21.0.0 Release Notes
 ===========================
 
 
@@ -25,3 +25,5 @@ Device Runtime
 - Changed the OpenMP DeviceRTL to use 'generic' IR. The
   ``LIBOMPTARGET_DEVICE_ARCHITECTURES`` CMake argument is now unused and will
   always build support for AMDGPU and NVPTX targets.
+- Updated the offloading entry format but retained backwards compatibility with
+  the old format.
diff --git a/openmp/runtime/src/include/omp.h.var b/openmp/runtime/src/include/omp.h.var
index dee46e51d6b34..82f9d07657ffb 100644
--- a/openmp/runtime/src/include/omp.h.var
+++ b/openmp/runtime/src/include/omp.h.var
@@ -15,8 +15,14 @@
 #ifndef __OMP_H
 #   define __OMP_H
 
+#   ifndef __has_include
+#   define __has_include(x) 0
+#   endif
+
 #   include <stddef.h>
-#   include <stdlib.h>
+#   if (__has_include(<stdlib.h>))
+#     include <stdlib.h>
+#   endif
 #   include <stdint.h>
 
 #   define KMP_VERSION_MAJOR    @LIBOMP_VERSION_MAJOR@
diff --git a/openmp/runtime/src/kmp_platform.h b/openmp/runtime/src/kmp_platform.h
index 9c2215140467d..2f47efd3e85ff 100644
--- a/openmp/runtime/src/kmp_platform.h
+++ b/openmp/runtime/src/kmp_platform.h
@@ -116,6 +116,7 @@
 #define KMP_ARCH_PPC64_ELFv2 0
 #define KMP_ARCH_PPC64_XCOFF 0
 #define KMP_ARCH_PPC_XCOFF 0
+#define KMP_ARCH_PPC 0
 #define KMP_ARCH_MIPS 0
 #define KMP_ARCH_MIPS64 0
 #define KMP_ARCH_RISCV64 0
@@ -164,6 +165,9 @@
 #define KMP_ARCH_PPC_XCOFF 1
 #undef KMP_ARCH_PPC
 #define KMP_ARCH_PPC 1
+#elif defined(__powerpc__) && !defined(__LP64__)
+#undef KMP_ARCH_PPC
+#define KMP_ARCH_PPC 1
 #elif defined __ARM64_ARCH_8_32__
 #undef KMP_ARCH_AARCH64_32
 #define KMP_ARCH_AARCH64_32 1
diff --git a/openmp/runtime/src/z_Linux_asm.S b/openmp/runtime/src/z_Linux_asm.S
index 0bf9f07a13f14..607bfd8e3cb0f 100644
--- a/openmp/runtime/src/z_Linux_asm.S
+++ b/openmp/runtime/src/z_Linux_asm.S
@@ -2470,7 +2470,7 @@ __kmp_invoke_microtask:
 
 #endif /* KMP_ARCH_S390X */
 
-#if KMP_ARCH_ARM || KMP_ARCH_MIPS || KMP_ARCH_AARCH64_32
+#if KMP_ARCH_ARM || KMP_ARCH_MIPS || KMP_ARCH_PPC || KMP_ARCH_AARCH64_32
 #ifndef KMP_PREFIX_UNDERSCORE
 # define KMP_PREFIX_UNDERSCORE(x) x
 #endif
diff --git a/openmp/runtime/tools/summarizeStats.py b/openmp/runtime/tools/summarizeStats.py
index 7daed2e1cd5cc..c1a59288fae7f 100644
--- a/openmp/runtime/tools/summarizeStats.py
+++ b/openmp/runtime/tools/summarizeStats.py
@@ -1,5 +1,6 @@
 #!/usr/bin/env python
 
+
 import pandas as pd
 import numpy as np
 import re
@@ -37,7 +38,7 @@ def draw_circle_frame(self, x0, y0, r):
 
     frame_dict = {'polygon': draw_poly_frame, 'circle': draw_circle_frame}
     if frame not in frame_dict:
-        raise ValueError, 'unknown value for `frame`: %s' % frame
+        raise ValueError("unknown value for `frame`: %s" % frame)
 
     class RadarAxes(PolarAxes):
         """
@@ -143,7 +144,7 @@ def readFile(fname):
             res["counters"] = readCounters(f)
             return res
     except (OSError, IOError):
-        print "Cannot open " + fname
+        print("Cannot open " + fname)
         return None
 
 def usefulValues(l):
@@ -235,7 +236,7 @@ def compPie(data):
             compKeys[key] = data[key]
         else:
             nonCompKeys[key] = data[key]
-    print "comp keys:", compKeys, "\n\n non comp keys:", nonCompKeys
+    print("comp keys:", compKeys, "\n\n non comp keys:", nonCompKeys)
     return [compKeys, nonCompKeys]
 
 def drawMainPie(data, filebase, colors):
@@ -299,10 +300,10 @@ def main():
                 chartType = "radar"
                 drawRadarChart(data, s, filebase, params, colors[n])
                 """radar Charts finish here"""
-                plt.savefig(filebase+"_"+s+"_"+chartType, bbox_inches='tight')
-            elif s == 'timers':
-                print "overheads in "+filebase
-                numThreads = tmp[s]['SampleCount']['Total_OMP_parallel']
+                plt.savefig(filebase + "_" + s + "_" + chartType, bbox_inches="tight")
+            elif s == "timers":
+                print("overheads in " + filebase)
+                numThreads = tmp[s]["SampleCount"]["Total_OMP_parallel"]
                 for key in data.keys():
                     if key[0:5] == 'Total':
                         del data[key]
diff --git a/polly/docs/Architecture.rst b/polly/docs/Architecture.rst
index 645d6522e8694..506e0ff8d26be 100644
--- a/polly/docs/Architecture.rst
+++ b/polly/docs/Architecture.rst
@@ -27,7 +27,7 @@ executed in the so-called **Inliner cycle**, This is again a set of **Scalar
 Simplification** passes, a set of **Simple Loop Optimizations**, and the
 **Inliner** itself. Even though these passes make up the majority of the LLVM
 pass pipeline, the primary goal of these passes is still canonicalization
-without loosing semantic information that complicates later analysis. As part of
+without losing semantic information that complicates later analysis. As part of
 the inliner cycle, the LLVM inliner step-by-step tries to inline functions, runs
 canonicalization passes to exploit newly exposed simplification opportunities,
 and then tries to inline the further simplified functions. Some simple loop
diff --git a/polly/docs/doxygen.cfg.in b/polly/docs/doxygen.cfg.in
index befe6e138ec27..a553fc8154fb1 100644
--- a/polly/docs/doxygen.cfg.in
+++ b/polly/docs/doxygen.cfg.in
@@ -1066,7 +1066,7 @@ HTML_STYLESHEET        =
 # defined cascading style sheet that is included after the standard style sheets
 # created by doxygen. Using this option one can overrule certain style aspects.
 # This is preferred over using HTML_STYLESHEET since it does not replace the
-# standard style sheet and is therefor more robust against future updates.
+# standard style sheet and is therefore more robust against future updates.
 # Doxygen will copy the style sheet file to the output directory. For an example
 # see the documentation.
 # This tag requires that the tag GENERATE_HTML is set to YES.
@@ -1950,7 +1950,7 @@ PREDEFINED             =
 EXPAND_AS_DEFINED      =
 
 # If the SKIP_FUNCTION_MACROS tag is set to YES then doxygen's preprocessor will
-# remove all refrences to function-like macros that are alone on a line, have an
+# remove all references to function-like macros that are alone on a line, have an
 # all uppercase name, and do not end with a semicolon. Such function macros are
 # typically used for boiler-plate code, and will confuse the parser if not
 # removed.
diff --git a/polly/include/polly/CodeGen/IslExprBuilder.h b/polly/include/polly/CodeGen/IslExprBuilder.h
index 25f61be5787c1..df8140e0b8757 100644
--- a/polly/include/polly/CodeGen/IslExprBuilder.h
+++ b/polly/include/polly/CodeGen/IslExprBuilder.h
@@ -135,6 +135,9 @@ class IslExprBuilder final {
   /// @return The llvm::Value* containing the result of the computation.
   llvm::Value *create(__isl_take isl_ast_expr *Expr);
 
+  /// Create LLVM-IR for an isl_ast_expr[ession] and cast it to i1.
+  llvm::Value *createBool(__isl_take isl_ast_expr *Expr);
+
   /// Return the largest of two types.
   ///
   /// @param T1 The first type.
diff --git a/polly/include/polly/CodeGen/RuntimeDebugBuilder.h b/polly/include/polly/CodeGen/RuntimeDebugBuilder.h
index 46193f9ccaf2c..2e5349d9edd6e 100644
--- a/polly/include/polly/CodeGen/RuntimeDebugBuilder.h
+++ b/polly/include/polly/CodeGen/RuntimeDebugBuilder.h
@@ -118,7 +118,7 @@ struct RuntimeDebugBuilder {
 
   /// Call fflush
   ///
-  /// @parma Builder The builder used to insert the code.
+  /// @param Builder The builder used to insert the code.
   static void createFlush(PollyIRBuilder &Builder);
 };
 } // namespace polly
diff --git a/polly/include/polly/DependenceInfo.h b/polly/include/polly/DependenceInfo.h
index 7526a294c6baf..d562ad80592f2 100644
--- a/polly/include/polly/DependenceInfo.h
+++ b/polly/include/polly/DependenceInfo.h
@@ -211,7 +211,7 @@ struct DependenceAnalysis final : public AnalysisInfoMixin<DependenceAnalysis> {
 
     /// Invalidate the dependence information and recompute it when needed
     /// again.
-    /// May be required when the underlaying Scop was changed in a way that
+    /// May be required when the underlying Scop was changed in a way that
     /// would add new dependencies (e.g. between new statement instances
     /// insierted into the SCoP) or intentionally breaks existing ones. It is
     /// not required when updating the schedule that conforms the existing
@@ -251,7 +251,7 @@ class DependenceInfo final : public ScopPass {
   const Dependences &recomputeDependences(Dependences::AnalysisLevel Level);
 
   /// Invalidate the dependence information and recompute it when needed again.
-  /// May be required when the underlaying Scop was changed in a way that would
+  /// May be required when the underlying Scop was changed in a way that would
   /// add new dependencies (e.g. between new statement instances insierted into
   /// the SCoP) or intentionally breaks existing ones. It is not required when
   /// updating the schedule that conforms the existing dependencies.
diff --git a/polly/include/polly/ScopInfo.h b/polly/include/polly/ScopInfo.h
index 974de817e72db..ab0f81dd2836d 100644
--- a/polly/include/polly/ScopInfo.h
+++ b/polly/include/polly/ScopInfo.h
@@ -1494,7 +1494,7 @@ class ScopStmt final {
   /// @param Access  The access to add.
   /// @param Prepend If true, will add @p Access before all other instructions
   ///                (instead of appending it).
-  void addAccess(MemoryAccess *Access, bool Preprend = false);
+  void addAccess(MemoryAccess *Access, bool Prepend = false);
 
   /// Remove a MemoryAccess from this statement.
   ///
diff --git a/polly/include/polly/Support/SCEVAffinator.h b/polly/include/polly/Support/SCEVAffinator.h
index 63a341a01309b..faacfd8ba0e69 100644
--- a/polly/include/polly/Support/SCEVAffinator.h
+++ b/polly/include/polly/Support/SCEVAffinator.h
@@ -50,7 +50,7 @@ class SCEVAffinator final : public llvm::SCEVVisitor<SCEVAffinator, PWACtx> {
   /// Check an <nsw> AddRec for the loop @p L is cached.
   bool hasNSWAddRecForLoop(llvm::Loop *L) const;
 
-  /// Return the LoopInfo used by thi object.
+  /// Return the LoopInfo used by the object.
   llvm::LoopInfo *getLI() const { return &LI; }
 
 private:
diff --git a/polly/include/polly/Support/ScopHelper.h b/polly/include/polly/Support/ScopHelper.h
index 13852ecb18ee7..7818f67b505fd 100644
--- a/polly/include/polly/Support/ScopHelper.h
+++ b/polly/include/polly/Support/ScopHelper.h
@@ -83,7 +83,7 @@ using RecordedAssumptionsTy = llvm::SmallVector<Assumption, 8>;
 ///
 /// This function will add the assumption to the RecordedAssumptions. This
 /// collection will be added (@see addAssumption) to the assumed context once
-/// all paramaters are known and the context is fully built.
+/// all parameters are known and the context is fully built.
 ///
 /// @param RecordedAssumption container which keeps all recorded assumptions.
 /// @param Kind The assumption kind describing the underlying cause.
@@ -132,7 +132,7 @@ using BoxedLoopsSetTy = llvm::SetVector<const llvm::Loop *>;
 /// isNull(), isInstruction(), isLoad(), isStore(), ..., isMemTransferInst(),
 /// operator bool(), operator!()
 ///
-/// The functions isa, cast, cast_or_null, dyn_cast are modeled te resemble
+/// The functions isa, cast, cast_or_null, dyn_cast are modeled to resemble
 /// those from llvm/Support/Casting.h. Partial template function specialization
 /// is currently not supported in C++ such that those cannot be used directly.
 /// (llvm::isa could, but then llvm:cast etc. would not have the expected
diff --git a/polly/lib/Analysis/ScopBuilder.cpp b/polly/lib/Analysis/ScopBuilder.cpp
index 82fa9e11550f2..76c9b4775784e 100644
--- a/polly/lib/Analysis/ScopBuilder.cpp
+++ b/polly/lib/Analysis/ScopBuilder.cpp
@@ -2522,7 +2522,7 @@ combineReductionType(MemoryAccess::ReductionType RT0,
   return MemoryAccess::RT_NONE;
 }
 
-///  True if @p AllAccs intersects with @p MemAccs execpt @p LoadMA and @p
+///  True if @p AllAccs intersects with @p MemAccs except @p LoadMA and @p
 ///  StoreMA
 bool hasIntersectingAccesses(isl::set AllAccs, MemoryAccess *LoadMA,
                              MemoryAccess *StoreMA, isl::set Domain,
diff --git a/polly/lib/Analysis/ScopDetectionDiagnostic.cpp b/polly/lib/Analysis/ScopDetectionDiagnostic.cpp
index 14a6f074454f7..f810d543f1ac0 100644
--- a/polly/lib/Analysis/ScopDetectionDiagnostic.cpp
+++ b/polly/lib/Analysis/ScopDetectionDiagnostic.cpp
@@ -71,7 +71,7 @@ static Statistic RejectStatistics[] = {
     SCOP_STAT(LoopOnlySomeLatches, "Not all loop latches in scop"),
     SCOP_STAT(FuncCall, "Function call with side effects"),
     SCOP_STAT(NonSimpleMemoryAccess,
-              "Compilated access semantics (volatile or atomic)"),
+              "Complicated access semantics (volatile or atomic)"),
     SCOP_STAT(Alias, "Base address aliasing"),
     SCOP_STAT(Other, ""),
     SCOP_STAT(IntToPtr, "Integer to pointer conversions"),
diff --git a/polly/lib/CodeGen/IslExprBuilder.cpp b/polly/lib/CodeGen/IslExprBuilder.cpp
index 1688c41c624b2..8c54436f295b3 100644
--- a/polly/lib/CodeGen/IslExprBuilder.cpp
+++ b/polly/lib/CodeGen/IslExprBuilder.cpp
@@ -790,3 +790,10 @@ Value *IslExprBuilder::create(__isl_take isl_ast_expr *Expr) {
 
   llvm_unreachable("Unexpected enum value");
 }
+
+llvm::Value *IslExprBuilder::createBool(__isl_take isl_ast_expr *Expr) {
+  Value *Result = create(Expr);
+  if (!Result->getType()->isIntegerTy(1))
+    Result = Builder.CreateICmpNE(Result, Builder.getInt1(false));
+  return Result;
+}
diff --git a/polly/lib/CodeGen/IslNodeBuilder.cpp b/polly/lib/CodeGen/IslNodeBuilder.cpp
index 739bd63a5eb80..40205215ea0b3 100644
--- a/polly/lib/CodeGen/IslNodeBuilder.cpp
+++ b/polly/lib/CodeGen/IslNodeBuilder.cpp
@@ -1103,7 +1103,7 @@ Value *IslNodeBuilder::preloadInvariantLoad(const MemoryAccess &MA,
   Domain = nullptr;
 
   ExprBuilder.setTrackOverflow(true);
-  Value *Cond = ExprBuilder.create(DomainCond);
+  Value *Cond = ExprBuilder.createBool(DomainCond);
   Value *OverflowHappened = Builder.CreateNot(ExprBuilder.getOverflowState(),
                                               "polly.preload.cond.overflown");
   Cond = Builder.CreateAnd(Cond, OverflowHappened, "polly.preload.cond.result");
@@ -1159,7 +1159,7 @@ bool IslNodeBuilder::preloadInvariantEquivClass(
   // For an equivalence class of invariant loads we pre-load the representing
   // element with the unified execution context. However, we have to map all
   // elements of the class to the one preloaded load as they are referenced
-  // during the code generation and therefor need to be mapped.
+  // during the code generation and therefore need to be mapped.
   const MemoryAccessList &MAs = IAClass.InvariantAccesses;
   if (MAs.empty())
     return true;
diff --git a/polly/lib/CodeGen/Utils.cpp b/polly/lib/CodeGen/Utils.cpp
index 3afb2e580889b..e95705616022f 100644
--- a/polly/lib/CodeGen/Utils.cpp
+++ b/polly/lib/CodeGen/Utils.cpp
@@ -46,7 +46,7 @@ static BasicBlock *splitEdge(BasicBlock *Prev, BasicBlock *Succ,
   // llvm::SplitCriticalEdge is more efficient than
   // llvm::SplitBlockPredecessors, which is more general. In the future we might
   // either modify llvm::SplitCriticalEdge to allow skipping the critical edge
-  // check; or Copy&Pase it here.
+  // check; or Copy&Paste it here.
   BasicBlock *MiddleBlock = SplitBlockPredecessors(
       Succ, ArrayRef<BasicBlock *>(Prev), Suffix, DT, LI);
 
diff --git a/polly/lib/Support/SCEVAffinator.cpp b/polly/lib/Support/SCEVAffinator.cpp
index ce4467f082ba4..87e0fc056ca4b 100644
--- a/polly/lib/Support/SCEVAffinator.cpp
+++ b/polly/lib/Support/SCEVAffinator.cpp
@@ -502,7 +502,7 @@ PWACtx SCEVAffinator::visitUDivExpr(const SCEVUDivExpr *Expr) {
   }
 
   // TODO: One can represent the dividend as piece-wise function to be more
-  //       precise but therefor a heuristic is needed.
+  //       precise but therefore a heuristic is needed.
 
   // Assume a non-negative dividend.
   takeNonNegativeAssumption(DividendPWAC, RecordedAssumptions);
diff --git a/polly/lib/Support/ScopHelper.cpp b/polly/lib/Support/ScopHelper.cpp
index bece390d31dc6..fbf891be0a1e7 100644
--- a/polly/lib/Support/ScopHelper.cpp
+++ b/polly/lib/Support/ScopHelper.cpp
@@ -456,7 +456,7 @@ struct ScopExpander final : SCEVVisitor<ScopExpander, const SCEV *> {
 
     // FIXME: This emits a SCEV for GenSE (since GenLRepl will refer to the
     // induction variable of a generated loop), so we should not use SCEVVisitor
-    // with it. Howver, it still contains references to the SCoP region.
+    // with it. However, it still contains references to the SCoP region.
     return visit(Evaluated);
   }
   ///}
diff --git a/polly/lib/Transform/DeLICM.cpp b/polly/lib/Transform/DeLICM.cpp
index b7e464e6739c6..9a9768afe113e 100644
--- a/polly/lib/Transform/DeLICM.cpp
+++ b/polly/lib/Transform/DeLICM.cpp
@@ -1290,7 +1290,7 @@ class DeLICMImpl final : public ZoneAlgorithm {
           continue;
         }
 
-        // Check for more than one element acces per statement instance.
+        // Check for more than one element access per statement instance.
         // Currently we expect write accesses to be functional, eg. disallow
         //
         //   { Stmt[0] -> [i] : 0 <= i < 2 }
diff --git a/polly/lib/Transform/ManualOptimizer.cpp b/polly/lib/Transform/ManualOptimizer.cpp
index 0e330f207fbc4..98ab9819e5b1f 100644
--- a/polly/lib/Transform/ManualOptimizer.cpp
+++ b/polly/lib/Transform/ManualOptimizer.cpp
@@ -149,7 +149,7 @@ class SearchTransformVisitor final
   // transformed in innermost-first order.
   isl::schedule Result;
 
-  /// Check wether a schedule after a  transformation is legal. Return the old
+  /// Check whether a schedule after a  transformation is legal. Return the old
   /// schedule without the transformation.
   isl::schedule
   checkDependencyViolation(llvm::MDNode *LoopMD, llvm::Value *CodeRegion,
diff --git a/polly/lib/Transform/MatmulOptimizer.cpp b/polly/lib/Transform/MatmulOptimizer.cpp
index ff1683b2d63c5..01d431a97e7db 100644
--- a/polly/lib/Transform/MatmulOptimizer.cpp
+++ b/polly/lib/Transform/MatmulOptimizer.cpp
@@ -1759,7 +1759,7 @@ static bool isTCPattern(isl::schedule_node Node, const Dependences *D,
   //
   // For example, this covers the matrix multiplication pattern after a full
   // run of -polly-optree and -polly-delicm, where the write access is not
-  // through the original memory access, but trough a PHI node that was
+  // through the original memory access, but through a PHI node that was
   // delicmed. Subsequently, such band nodes will be replaced by a single band
   // node.
   //
diff --git a/polly/lib/Transform/ScheduleOptimizer.cpp b/polly/lib/Transform/ScheduleOptimizer.cpp
index 55d51982d90e7..070700a64a168 100644
--- a/polly/lib/Transform/ScheduleOptimizer.cpp
+++ b/polly/lib/Transform/ScheduleOptimizer.cpp
@@ -771,7 +771,7 @@ static void runIslScheduleOptimizer(
     return;
   }
 
-  // Apply ISL's algorithm only if not overriden by the user. Note that
+  // Apply ISL's algorithm only if not overridden by the user. Note that
   // post-rescheduling optimizations (tiling, pattern-based, prevectorization)
   // rely on the coincidence/permutable annotations on schedule tree bands that
   // are added by the rescheduling analyzer. Therefore, disabling the
diff --git a/polly/lib/Transform/ScheduleTreeTransform.cpp b/polly/lib/Transform/ScheduleTreeTransform.cpp
index f0684de825d27..3f3630027e6e3 100644
--- a/polly/lib/Transform/ScheduleTreeTransform.cpp
+++ b/polly/lib/Transform/ScheduleTreeTransform.cpp
@@ -572,13 +572,13 @@ class BandCollapseRewriter final
     isl::schedule_node_band Band = RootBand;
     isl::ctx Ctx = Band.ctx();
 
-    // Do not merge permutable band to avoid loosing the permutability property.
+    // Do not merge permutable band to avoid losing the permutability property.
     // Cannot collapse even two permutable loops, they might be permutable
     // individually, but not necassarily across.
     if (unsignedFromIslSize(Band.n_member()) > 1u && Band.permutable())
       return getBase().visitBand(Band);
 
-    // Find collapsable bands.
+    // Find collapsible bands.
     SmallVector<isl::schedule_node_band> Nest;
     int NumTotalLoops = 0;
     isl::schedule_node Body;
@@ -884,10 +884,10 @@ class GreedyFusionRewriter final
       collectPotentiallyFusableBands(Child, Bands, Child);
     }
 
-    // Direct children that had at least one of its decendants fused.
+    // Direct children that had at least one of its descendants fused.
     SmallDenseSet<isl_schedule_node *, 4> ChangedDirectChildren;
 
-    // Fuse neigboring bands until reaching the end of candidates.
+    // Fuse neighboring bands until reaching the end of candidates.
     int i = 0;
     while (i + 1 < (int)Bands.size()) {
       isl::schedule Fused =
diff --git a/polly/test/CodeGen/LoopParallelMD/do_not_mutate_debug_info.ll b/polly/test/CodeGen/LoopParallelMD/do_not_mutate_debug_info.ll
index 28531244421d1..eb7de01ba862c 100644
--- a/polly/test/CodeGen/LoopParallelMD/do_not_mutate_debug_info.ll
+++ b/polly/test/CodeGen/LoopParallelMD/do_not_mutate_debug_info.ll
@@ -1,4 +1,4 @@
-; This test checks that we do not accidently mutate the debug info when
+; This test checks that we do not accidentally mutate the debug info when
 ; inserting loop parallel metadata.
 ; RUN: opt %loadNPMPolly < %s  -S -polly -passes=polly-codegen -polly-ast-detect-parallel | FileCheck %s
 ; CHECK-NOT: !7 = !{!7}
diff --git a/polly/test/CodeGen/multiple-codegens.ll b/polly/test/CodeGen/multiple-codegens.ll
index 2fa974e66df50..a63f8a615ff9e 100644
--- a/polly/test/CodeGen/multiple-codegens.ll
+++ b/polly/test/CodeGen/multiple-codegens.ll
@@ -6,7 +6,7 @@
 ; RegionPassManager. -polly-codegen must not reuse the -polly-ast analysis the
 ; was created for the first -polly-scops pass.
 ; The current solution is that only the first -polly-codegen is allowed to
-; generate code, the second detects it is re-using an IslAst that belongs to a
+; generate code, the second detects it is reusing an IslAst that belongs to a
 ; different ScopInfo.
 ;
 ; int a, b, c;
diff --git a/polly/test/CodeGen/multiple-scops-in-a-row.ll b/polly/test/CodeGen/multiple-scops-in-a-row.ll
index b81ba04e36463..effae223c152a 100644
--- a/polly/test/CodeGen/multiple-scops-in-a-row.ll
+++ b/polly/test/CodeGen/multiple-scops-in-a-row.ll
@@ -6,7 +6,7 @@
 
 ; We explicitly check here that the second scop is not code generated. Later
 ; improvements may make this possible (e.g., Polly gaining support for
-; parameteric conditional expressions or a changed code generation order).
+; parametric conditional expressions or a changed code generation order).
 ; However, in case this happens, we want to ensure this test case is been
 ; reasoned about and updated accordingly.
 
diff --git a/polly/test/CodeGen/reduction_2.ll b/polly/test/CodeGen/reduction_2.ll
index 4aa306775e781..060a1866870e4 100644
--- a/polly/test/CodeGen/reduction_2.ll
+++ b/polly/test/CodeGen/reduction_2.ll
@@ -88,7 +88,7 @@ if.end:                                           ; preds = %if.then, %for.end
 declare void @llvm.memset.p0.i64(ptr nocapture, i8, i64, i32, i1) nounwind
 
 ; This is a negative test. We can prove that RED[0] in the conditional after
-; the loop is dereferencable and consequently expand the SCoP from the
+; the loop is dereferenceable and consequently expand the SCoP from the
 ; loop to include the conditional. However, during SCoP generation we realize
 ; that, while RED[0] is invariant, it is written to as part of the same scop
 ; and can consequently not be hoisted. Hence, we invalidate the scop.
diff --git a/polly/test/CodeGen/scalar-store-from-same-bb.ll b/polly/test/CodeGen/scalar-store-from-same-bb.ll
index 3f232da37e4cd..0c1164b245a43 100644
--- a/polly/test/CodeGen/scalar-store-from-same-bb.ll
+++ b/polly/test/CodeGen/scalar-store-from-same-bb.ll
@@ -2,7 +2,7 @@
 ; RUN: -passes=polly-codegen -S < %s | FileCheck %s
 
 ; This test ensures that the expression N + 1 that is stored in the phi-node
-; alloca, is directly computed and not incorrectly transfered through memory.
+; alloca, is directly computed and not incorrectly transferred through memory.
 
 ; CHECK: store i64 [[REG:%.*]], ptr %res.phiops
 ; CHECK: [[REG]] = add i64 %N, 1
diff --git a/polly/test/CodeGen/test-invalid-operands-for-select.ll b/polly/test/CodeGen/test-invalid-operands-for-select.ll
index 9f5013cf1bb16..fdc98fbb4d9e7 100644
--- a/polly/test/CodeGen/test-invalid-operands-for-select.ll
+++ b/polly/test/CodeGen/test-invalid-operands-for-select.ll
@@ -2,7 +2,7 @@
 ;
 ; Check that we do not crash as described here: http://llvm.org/PR21167
 ;
-; In case the pieceweise affine function used to create an isl_ast_expr
+; In case the piecewise affine function used to create an isl_ast_expr
 ; had empty cases (e.g., with contradicting constraints on the
 ; parameters), it was possible that the condition of the isl_ast_expr
 ; select was not a comparison but a constant (thus of type i64).
diff --git a/polly/test/DeLICM/load-in-cond-inf-loop.ll b/polly/test/DeLICM/load-in-cond-inf-loop.ll
index f6e23110aa6f1..a78a4691bb0d5 100644
--- a/polly/test/DeLICM/load-in-cond-inf-loop.ll
+++ b/polly/test/DeLICM/load-in-cond-inf-loop.ll
@@ -1,6 +1,6 @@
 ; RUN: opt %loadNPMPolly '-passes=print<polly-delicm>' -disable-output < %s | FileCheck %s
 
-; When %b is 0, %for.body13 is an infite loop. In this case the loaded
+; When %b is 0, %for.body13 is an infinite loop. In this case the loaded
 ; value %1 is not used anywhere.
 ; This is a problem when DeLICM tries to map %1 to %arrayidx16 because
 ; %1 has no corresponding when %b == 0 and therefore hat no location
diff --git a/polly/test/DeLICM/pr41656.ll b/polly/test/DeLICM/pr41656.ll
index d7cfde35a6e80..2a92503809a24 100644
--- a/polly/test/DeLICM/pr41656.ll
+++ b/polly/test/DeLICM/pr41656.ll
@@ -4,7 +4,7 @@
 ;
 ; This test case has an InvalidContext such that part of the predecessors
 ; of for.body.us.i lie within the invalid context. This causes a
-; consistency check withing the invalid context of PR41656 to fail.
+; consistency check within the invalid context of PR41656 to fail.
 ;
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 
diff --git a/polly/test/DeLICM/pr48783.ll b/polly/test/DeLICM/pr48783.ll
index e3c3eb6a19ccf..deba8bfcc5daf 100644
--- a/polly/test/DeLICM/pr48783.ll
+++ b/polly/test/DeLICM/pr48783.ll
@@ -4,7 +4,7 @@
 ;
 ; PHI predecessors of statement instances can only be reliably derived in defined behaviour situations. In this case, the inner loop's counter would overflow when its upper bound (%call24) is lower than its lower bound (2). However, due to the nsw flag, this would be undefined behavior and therefore not added to any runtime-check context, but to the defined-behaviour context.
 ;
-; Dereived from test case pr41656.ll
+; Derived from test case pr41656.ll
 ;
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 
diff --git a/polly/test/DeLICM/reject_outofquota.ll b/polly/test/DeLICM/reject_outofquota.ll
index 820679a5349d2..9bc6bf1f23733 100644
--- a/polly/test/DeLICM/reject_outofquota.ll
+++ b/polly/test/DeLICM/reject_outofquota.ll
@@ -66,7 +66,7 @@ return:
 ; CHECK: maximal number of operations exceeded during zone analysis
 
 ; Check that even if the quota was exceeded in DeLICM, DependenceInfo is still
-; successfull since it uses a different operations counter.
+; successful since it uses a different operations counter.
 ;
 ; DEP:     RAW dependences:
 ; DEP-NOT:        n/a
diff --git a/polly/test/IstAstInfo/reduction_modulo_schedule_multiple_dimensions_2.ll b/polly/test/IstAstInfo/reduction_modulo_schedule_multiple_dimensions_2.ll
index 46b2559c6e0b1..d7f9029fd347a 100644
--- a/polly/test/IstAstInfo/reduction_modulo_schedule_multiple_dimensions_2.ll
+++ b/polly/test/IstAstInfo/reduction_modulo_schedule_multiple_dimensions_2.ll
@@ -1,6 +1,6 @@
 ; RUN: opt %loadNPMPolly '-passes=polly-import-jscop,print<polly-ast>' -polly-ast-detect-parallel -disable-output < %s | FileCheck %s
 ;
-; Verify that the outer dimension doesnt't carry reduction dependences
+; Verify that the outer dimension doesn't carry reduction dependences
 ;
 ; CHECK-NOT:#pragma known-parallel reduction
 ; CHECK:    #pragma known-parallel
diff --git a/polly/test/IstAstInfo/reduction_modulo_schedule_multiple_dimensions_3.ll b/polly/test/IstAstInfo/reduction_modulo_schedule_multiple_dimensions_3.ll
index 6f40ee90fef53..f18060a2e20a8 100644
--- a/polly/test/IstAstInfo/reduction_modulo_schedule_multiple_dimensions_3.ll
+++ b/polly/test/IstAstInfo/reduction_modulo_schedule_multiple_dimensions_3.ll
@@ -1,6 +1,6 @@
 ; RUN: opt %loadNPMPolly '-passes=polly-import-jscop,print<polly-ast>' -polly-ast-detect-parallel -disable-output < %s | FileCheck %s
 ;
-; Verify that the outer dimension doesnt't carry reduction dependences
+; Verify that the outer dimension doesn't carry reduction dependences
 ;
 ; CHECK-NOT:#pragma known-parallel reduction
 ; CHECK:    #pragma known-parallel
diff --git a/polly/test/IstAstInfo/reduction_modulo_schedule_multiple_dimensions_4.ll b/polly/test/IstAstInfo/reduction_modulo_schedule_multiple_dimensions_4.ll
index f82b9569a88b1..8e2a590c5f57c 100644
--- a/polly/test/IstAstInfo/reduction_modulo_schedule_multiple_dimensions_4.ll
+++ b/polly/test/IstAstInfo/reduction_modulo_schedule_multiple_dimensions_4.ll
@@ -1,6 +1,6 @@
 ; RUN: opt %loadNPMPolly '-passes=polly-import-jscop,print<polly-ast>' -polly-ast-detect-parallel -disable-output < %s | FileCheck %s
 ;
-; Verify that the outer dimension doesnt't carry reduction dependences
+; Verify that the outer dimension doesn't carry reduction dependences
 ;
 ; CHECK-NOT:#pragma known-parallel reduction
 ; CHECK:    #pragma known-parallel
diff --git a/polly/test/ScheduleOptimizer/pattern-matching-based-opts-after-delicm.ll b/polly/test/ScheduleOptimizer/pattern-matching-based-opts-after-delicm.ll
index 8228a5c08f598..6e9ade869ec6c 100644
--- a/polly/test/ScheduleOptimizer/pattern-matching-based-opts-after-delicm.ll
+++ b/polly/test/ScheduleOptimizer/pattern-matching-based-opts-after-delicm.ll
@@ -7,7 +7,7 @@
 
 ; Check that the pattern matching detects the matrix multiplication pattern
 ; after a full run of -polly-optree and -polly-delicm, where the write access
-; is not through the original memory access, but trough a PHI node that was
+; is not through the original memory access, but through a PHI node that was
 ; delicmed. This test covers the polybench 2mm and 3mm cases.
 ;
 ; This test case generates the following schedule, which contains filters:
diff --git a/polly/test/ScheduleOptimizer/pattern-matching-based-opts-after-delicm_2.ll b/polly/test/ScheduleOptimizer/pattern-matching-based-opts-after-delicm_2.ll
index 4bda7584f5962..4ef0605a0ba75 100644
--- a/polly/test/ScheduleOptimizer/pattern-matching-based-opts-after-delicm_2.ll
+++ b/polly/test/ScheduleOptimizer/pattern-matching-based-opts-after-delicm_2.ll
@@ -5,7 +5,7 @@
 ;
 ; Check that the pattern matching detects the tensor contraction pattern
 ; after a full run of -polly-delicm. This test case generates the following
-; schedule, which contans two band nodes. Without DeLICM two statement are
+; schedule, which contains two band nodes. Without DeLICM two statement are
 ; generated.
 ;
 ; domain: "{ Stmt5[i0, i1, i2, i3, i4, i5] : 0 <= i0 <= 31 and 0 <= i1 <= 31 and
diff --git a/polly/test/ScopDetect/scev_remove_max.ll b/polly/test/ScopDetect/scev_remove_max.ll
index caf55bf87a667..f76c832ff08f5 100644
--- a/polly/test/ScopDetect/scev_remove_max.ll
+++ b/polly/test/ScopDetect/scev_remove_max.ll
@@ -1,6 +1,6 @@
 ; RUN: opt %loadNPMPolly '-passes=print<polly-detect>' < %s
 
-; This test case helps to determine wether SCEVRemoveMax::remove produces
+; This test case helps to determine whether SCEVRemoveMax::remove produces
 ; an infinite loop and a segmentation fault, if it processes, for example,
 ; '((-1 + (-1 * %b1)) umax {(-1 + (-1 * %yStart)),+,-1}<%.preheader>)'.
 ;
diff --git a/polly/test/ScopDetectionDiagnostics/ReportLoopHasNoExit.ll b/polly/test/ScopDetectionDiagnostics/ReportLoopHasNoExit.ll
index 92028093f70bb..3743bfae9fcaf 100644
--- a/polly/test/ScopDetectionDiagnostics/ReportLoopHasNoExit.ll
+++ b/polly/test/ScopDetectionDiagnostics/ReportLoopHasNoExit.ll
@@ -1,7 +1,7 @@
 ; XFAIL: *
 
 ; The test case stopped making sense after r310940 that added infinite loops to
-; the PostDominatorTree. Infinite loops are postdominated ony by the virtual
+; the PostDominatorTree. Infinite loops are postdominated only by the virtual
 ; root, which causes them not to appear in regions in ScopDetection anymore.
 
 ; RUN: opt %loadNPMPolly -pass-remarks-missed="polly-detect" -polly-allow-nonaffine-loops '-passes=print<polly-detect>' -disable-output < %s 2>&1 | FileCheck %s
diff --git a/polly/test/ScopInfo/NonAffine/non_affine_loop_used_later.ll b/polly/test/ScopInfo/NonAffine/non_affine_loop_used_later.ll
index 79b61eca258f2..63ff354d7e5f7 100644
--- a/polly/test/ScopInfo/NonAffine/non_affine_loop_used_later.ll
+++ b/polly/test/ScopInfo/NonAffine/non_affine_loop_used_later.ll
@@ -1,7 +1,7 @@
 ; RUN: opt %loadNPMPolly -polly-allow-nonaffine -polly-allow-nonaffine-branches -polly-allow-nonaffine-loops                                                                        '-passes=print<polly-detect>,print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
 ; RUN: opt %loadNPMPolly -polly-allow-nonaffine -polly-allow-nonaffine-branches -polly-allow-nonaffine-loops -polly-unprofitable-scalar-accs=true -polly-process-unprofitable=false '-passes=print<polly-detect>,print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s --check-prefix=PROFIT
 ;
-; Verify that we over approximate the read acces of A[j] in the last statement as j is
+; Verify that we over approximate the read access of A[j] in the last statement as j is
 ; computed in a non-affine loop we do not model.
 ;
 ; CHECK:      Function: f
diff --git a/polly/test/ScopInfo/allow-all-parameters-dereferencable.ll b/polly/test/ScopInfo/allow-all-parameters-dereferencable.ll
index cb06e352da658..70c3c56fb3112 100644
--- a/polly/test/ScopInfo/allow-all-parameters-dereferencable.ll
+++ b/polly/test/ScopInfo/allow-all-parameters-dereferencable.ll
@@ -28,7 +28,7 @@
 ; CODE-RTC-NEXT: br i1 %{{[a-zA-Z0-9\.]*}}, label %polly.preload.exec, label %polly.preload.merge
 
 ; Check that we don't generate a runtime check because we treat all
-; parameters as dereferencable.
+; parameters as dereferenceable.
 ; CODE-NOT: polly.preload.cond:                               ; preds = %polly.preload.begin
 ; CODE-NOT: br i1 %{{r1:[a-zA-Z0-9]*}}, label %polly.preload.exec, label %polly.preload.merge
 
diff --git a/polly/test/ScopInfo/invariant_same_loop_bound_multiple_times-1.ll b/polly/test/ScopInfo/invariant_same_loop_bound_multiple_times-1.ll
index a473ef30376c1..e3292b4e4aefa 100644
--- a/polly/test/ScopInfo/invariant_same_loop_bound_multiple_times-1.ll
+++ b/polly/test/ScopInfo/invariant_same_loop_bound_multiple_times-1.ll
@@ -1,7 +1,7 @@
 ; RUN: opt %loadNPMPolly '-passes=print<polly-function-scops>' -polly-invariant-load-hoisting=true -disable-output < %s 2>&1 | FileCheck %s
 ;
 ; Verify that we only have one parameter and one invariant load for all
-; three loads that occure in the region but actually access the same
+; three loads that occur in the region but actually access the same
 ; location. Also check that the execution context is the most generic
 ; one, e.g., here the universal set.
 ;
diff --git a/polly/test/ScopInfo/invariant_same_loop_bound_multiple_times-2.ll b/polly/test/ScopInfo/invariant_same_loop_bound_multiple_times-2.ll
index 66a0bc631b1dc..d69438de5817f 100644
--- a/polly/test/ScopInfo/invariant_same_loop_bound_multiple_times-2.ll
+++ b/polly/test/ScopInfo/invariant_same_loop_bound_multiple_times-2.ll
@@ -1,7 +1,7 @@
 ; RUN: opt %loadNPMPolly '-passes=print<polly-function-scops>' -polly-invariant-load-hoisting=true -disable-output < %s 2>&1 | FileCheck %s
 ;
 ; Verify that we only have one parameter and one invariant load for all
-; three loads that occure in the region but actually access the same
+; three loads that occur in the region but actually access the same
 ; location. Also check that the execution context is the most generic
 ; one, e.g., here the universal set.
 ;
diff --git a/polly/test/ScopInfo/multidim_gep_pointercast2.ll b/polly/test/ScopInfo/multidim_gep_pointercast2.ll
index b31a0d0262db9..9daae4b1ce3db 100644
--- a/polly/test/ScopInfo/multidim_gep_pointercast2.ll
+++ b/polly/test/ScopInfo/multidim_gep_pointercast2.ll
@@ -1,6 +1,6 @@
 ; RUN: opt %loadNPMPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
 ;
-; Verfy that we do not use the GetElementPtr information to delinearize A
+; Verify that we do not use the GetElementPtr information to delinearize A
 ; because of the cast in-between. Use the single-dimensional modeling instead.
 ;
 ;    void f(short A[][2]) {
diff --git a/polly/test/ScopInfo/multidim_many_references.ll b/polly/test/ScopInfo/multidim_many_references.ll
index 3801fda4923c3..f0f1c2b1f39db 100644
--- a/polly/test/ScopInfo/multidim_many_references.ll
+++ b/polly/test/ScopInfo/multidim_many_references.ll
@@ -28,7 +28,7 @@ target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 ; This test case verifies that the construction of the assumed context finishes
 ; successfully. Depending on how constrained are accumulated in the assumed
 ; context, this test case can take even for a smaller number of arrays over a
-; minute to complete. With the unrolling choosen in this test, an inefficient
+; minute to complete. With the unrolling chosen in this test, an inefficient
 ; formulation of the assumption tracking cause LLVM to crash due to excessive
 ; memory usage due to an overly large number of disjuncts being formed.
 
diff --git a/polly/test/ScopInfo/scalar_to_array.ll b/polly/test/ScopInfo/scalar_to_array.ll
index d64f1696c30b6..3f61d0d723046 100644
--- a/polly/test/ScopInfo/scalar_to_array.ll
+++ b/polly/test/ScopInfo/scalar_to_array.ll
@@ -109,7 +109,7 @@ return:                                           ; preds = %for.cond
 ; It is not possible to have a scop which accesses a scalar element that is
 ; a global variable. All global variables are pointers containing possibly
 ; a single element. Hence they do not need to be handled anyways.
-; Please note that this is still required when scalar to array rewritting is
+; Please note that this is still required when scalar to array rewriting is
 ; disabled.
 
 ; CHECK-LABEL: Function: use_after_scop
diff --git a/polly/test/ScopInfo/zero_ext_of_truncate.ll b/polly/test/ScopInfo/zero_ext_of_truncate.ll
index bd3749b6aa74f..cbe4af05169f8 100644
--- a/polly/test/ScopInfo/zero_ext_of_truncate.ll
+++ b/polly/test/ScopInfo/zero_ext_of_truncate.ll
@@ -8,7 +8,7 @@
 ;      }
 ;    }
 ;
-; FIXME: The truncated value should be a paramter.
+; FIXME: The truncated value should be a parameter.
 ; CHECK:         Assumed Context:
 ; CHECK-NEXT:    [N, tmp, M] -> { : }
 ; CHECK-NEXT:    Invalid Context:
diff --git a/polly/test/create_ll.sh b/polly/test/create_ll.sh
index c44d8ed81f2b5..1c03d9a015e01 100755
--- a/polly/test/create_ll.sh
+++ b/polly/test/create_ll.sh
@@ -12,7 +12,7 @@ opt -correlated-propagation -mem2reg -instcombine -loop-simplify -indvars \
 -instnamer ${LLFILE} -S -o ${LLFILE_TMP}
 
 # Insert a header into the new testcase containing a sample RUN line a FIXME and
-# an XFAIL. Then insert the formated C code and finally the LLVM-IR without
+# an XFAIL. Then insert the formatted C code and finally the LLVM-IR without
 # attributes, the module ID or the target triple.
 echo '; RUN: opt %loadPolly -S < %s | FileCheck %s' > ${LLFILE}
 echo ';' >> ${LLFILE}
diff --git a/polly/utils/pyscop/isl.py b/polly/utils/pyscop/isl.py
index c06b7bca28042..c5d92ae9812de 100644
--- a/polly/utils/pyscop/isl.py
+++ b/polly/utils/pyscop/isl.py
@@ -72,7 +72,7 @@ def initialize_isl_methods(self):
         if hasattr(self.__class__, "initialized"):
             return
 
-        self.__class__.initalized = True
+        self.__class__.initialized = True
         self.get_isl_method("read_from_str").argtypes = [Context, c_char_p, c_int]
         self.get_isl_method("copy").argtypes = [self.__class__]
         self.get_isl_method("copy").restype = c_int
@@ -204,7 +204,7 @@ def initialize_isl_methods(self):
         if hasattr(self.__class__, "initialized"):
             return
 
-        self.__class__.initalized = True
+        self.__class__.initialized = True
         self.get_isl_method("copy").argtypes = [self.__class__]
         self.get_isl_method("copy").restype = c_int
         self.get_isl_method("free").argtypes = [self.__class__]
diff --git a/polly/www/changelog.html b/polly/www/changelog.html
index 8c789cd31f530..6ba9ba4723058 100644
--- a/polly/www/changelog.html
+++ b/polly/www/changelog.html
@@ -40,7 +40,7 @@ <h2> 3.6</h2>
 <li>Run-time alias checks</li>
 <li>Computation of no-alias information for later LLVM optimizations
 (vectorizer, LICM, ...)</li>
-<li>Support for multi-dimensional arrays of parameteric size (still tested)</li>
+<li>Support for multi-dimensional arrays of parametric size (still tested)</li>
 <li>New assumption tracking framework</li>
 <ul>
 <li>Accesses to multi-dimensional arrays of fixed size are within bounds</li>
diff --git a/polly/www/get_started.html b/polly/www/get_started.html
index 70e8a6beca65c..d5f2b0dff9b3e 100644
--- a/polly/www/get_started.html
+++ b/polly/www/get_started.html
@@ -15,7 +15,7 @@
 <div id="content">
 
 <h1>Building and Installing Polly</h1>
-You can build Polly with <a href="https://cmake.org/">cmake</a> and your preferred geneator (e.g. Ninja, make, Visual Studio, etc.).
+You can build Polly with <a href="https://cmake.org/">cmake</a> and your preferred generator (e.g. Ninja, make, Visual Studio, etc.).
 
 <h3 id="source">Get the code</h3>
 
diff --git a/polly/www/index.html b/polly/www/index.html
index 87642324eadb4..6adc43ec3c4aa 100644
--- a/polly/www/index.html
+++ b/polly/www/index.html
@@ -90,7 +90,7 @@ <h4>AST Generation Paper published in TOPLAS</h4>
       while issues like the generation of the correct loop structure and loop
       bounds will be taken care of by our AST generator.
       <li><b>Polyhedral unrolling:</b> We discuss techniques that allow the
-      unrolling of non-trivial loops in the context of parameteric loop bounds,
+      unrolling of non-trivial loops in the context of parametric loop bounds,
       complex tile shapes and conditionally executed statements. Such unrolling
       support enables the generation of predicated code e.g. in the context of
       GPGPU computing.
@@ -110,7 +110,7 @@ <h4>AST Generation Paper published in TOPLAS</h4>
    <a href="https://www.grosser.es#pub-polyhedral-AST-generation">
    <em>Polyhedral AST generation is more than scanning polyhedra</em></a><br />
     Tobias Grosser, Sven Verdoolaege, Albert Cohen<br />
-    ACM Transations on Programming Languages and Systems (TOPLAS), 37(4),
+    ACM Transactions on Programming Languages and Systems (TOPLAS), 37(4),
     July 2015
 
     <br>
@@ -356,13 +356,13 @@ <h4>Experimental support for the <b>new isl code generator</b></h4>
   </td>
   </tr>
   <tr>
-  <td><p>Februar</p></td>
+  <td><p>February</p></td>
   <td><p>pollycc - a script to automatically compile with
   polyhedral optimizations </p></td>
   </tr>
 
   <tr>
-  <td><p> Januar</p></td>
+  <td><p> January</p></td>
   <td><p> Basic OpenMP support, Alias analysis integration,
   Pluto/POCC support </p></td>
   </tr>
diff --git a/polly/www/projects.html b/polly/www/projects.html
index 915c2b93921a0..ea7e137f031ea 100644
--- a/polly/www/projects.html
+++ b/polly/www/projects.html
@@ -42,7 +42,7 @@ <h3>Integrate Polly with the LLVM vectorizers</h3>
   to bring features of Polly to standard -O3 optimizations.
 
   <h3>Register tiling to obtain fast BLAS kernels with Polly</h3>
-  Even though Polly is already able to speep up compute kernels significantly,
+  Even though Polly is already able to speed up compute kernels significantly,
   when comparing to the best BLAS routines we still are at least one order of
   magnitude off. In this project you will investigate what is needed to close
   this performance gap. Earlier investigations have shown that register tiling
diff --git a/polly/www/publications.html b/polly/www/publications.html
index c256c8d151475..4890568fe6d2c 100644
--- a/polly/www/publications.html
+++ b/polly/www/publications.html
@@ -60,7 +60,7 @@ <h3> 2015 </h3>
   <ul>
   <li><em>Polyhedral AST generation is more than scanning polyhedra</em><br />
   Tobias Grosser, Sven Verdoolaege, Albert Cohen<br />
-   ACM Transations on Programming Languages and Systems (TOPLAS), 37(4), July
+   ACM Transactions on Programming Languages and Systems (TOPLAS), 37(4), July
    2015<br />
   <a href="https://www.grosser.es#pub-polyhedral-AST-generation">Paper</a>
   </li>
diff --git a/pstl/docs/ReleaseNotes.rst b/pstl/docs/ReleaseNotes.rst
index 941b029ecd674..a0b375134f78d 100644
--- a/pstl/docs/ReleaseNotes.rst
+++ b/pstl/docs/ReleaseNotes.rst
@@ -1,5 +1,5 @@
 =======================================
-PSTL 20.0.0 (In-Progress) Release Notes
+PSTL 21.0.0 (In-Progress) Release Notes
 =======================================
 
 .. contents::
diff --git a/third-party/benchmark/src/sysinfo.cc b/third-party/benchmark/src/sysinfo.cc
index 2bed1663af2e9..3993ae17f7fc4 100644
--- a/third-party/benchmark/src/sysinfo.cc
+++ b/third-party/benchmark/src/sysinfo.cc
@@ -160,11 +160,11 @@ ValueUnion GetSysctlImp(std::string const& name) {
   int mib[2];
 
   mib[0] = CTL_HW;
-  if ((name == "hw.ncpu") || (name == "hw.cpuspeed")) {
+  if ((name == "hw.ncpuonline") || (name == "hw.cpuspeed")) {
     ValueUnion buff(sizeof(int));
 
-    if (name == "hw.ncpu") {
-      mib[1] = HW_NCPU;
+    if (name == "hw.ncpuonline") {
+      mib[1] = HW_NCPUONLINE;
     } else {
       mib[1] = HW_CPUSPEED;
     }
@@ -482,27 +482,14 @@ std::string GetSystemName() {
 }
 
 int GetNumCPUsImpl() {
-#ifdef BENCHMARK_HAS_SYSCTL
-  int num_cpu = -1;
-  if (GetSysctl("hw.ncpu", &num_cpu)) return num_cpu;
-  PrintErrorAndDie("Err: ", strerror(errno));
-#elif defined(BENCHMARK_OS_WINDOWS)
+#ifdef BENCHMARK_OS_WINDOWS
   SYSTEM_INFO sysinfo;
   // Use memset as opposed to = {} to avoid GCC missing initializer false
   // positives.
   std::memset(&sysinfo, 0, sizeof(SYSTEM_INFO));
   GetSystemInfo(&sysinfo);
-  return sysinfo.dwNumberOfProcessors;  // number of logical
-                                        // processors in the current
-                                        // group
-#elif defined(BENCHMARK_OS_SOLARIS)
-  // Returns -1 in case of a failure.
-  long num_cpu = sysconf(_SC_NPROCESSORS_ONLN);
-  if (num_cpu < 0) {
-    PrintErrorAndDie("sysconf(_SC_NPROCESSORS_ONLN) failed with error: ",
-                     strerror(errno));
-  }
-  return (int)num_cpu;
+  // number of logical processors in the current group
+  return static_cast<int>(sysinfo.dwNumberOfProcessors);
 #elif defined(BENCHMARK_OS_QNX)
   return static_cast<int>(_syspage_ptr->num_cpu);
 #elif defined(BENCHMARK_OS_QURT)
@@ -511,63 +498,36 @@ int GetNumCPUsImpl() {
     hardware_threads.max_hthreads = 1;
   }
   return hardware_threads.max_hthreads;
+#elif defined(BENCHMARK_HAS_SYSCTL)
+  int num_cpu = -1;
+  constexpr auto* hwncpu =
+#if defined BENCHMARK_OS_MACOSX
+      "hw.logicalcpu";
+#elif defined(HW_NCPUONLINE)
+      "hw.ncpuonline";
 #else
-  int num_cpus = 0;
-  int max_id = -1;
-  std::ifstream f("/proc/cpuinfo");
-  if (!f.is_open()) {
-    PrintErrorAndDie("Failed to open /proc/cpuinfo");
-  }
-#if defined(__alpha__)
-  const std::string Key = "cpus detected";
-#else
-  const std::string Key = "processor";
-#endif
-  std::string ln;
-  while (std::getline(f, ln)) {
-    if (ln.empty()) continue;
-    std::size_t split_idx = ln.find(':');
-    std::string value;
-#if defined(__s390__)
-    // s390 has another format in /proc/cpuinfo
-    // it needs to be parsed differently
-    if (split_idx != std::string::npos)
-      value = ln.substr(Key.size() + 1, split_idx - Key.size() - 1);
-#else
-    if (split_idx != std::string::npos) value = ln.substr(split_idx + 1);
+      "hw.ncpu";
 #endif
-    if (ln.size() >= Key.size() && ln.compare(0, Key.size(), Key) == 0) {
-      num_cpus++;
-      if (!value.empty()) {
-        const int cur_id = benchmark::stoi(value);
-        max_id = std::max(cur_id, max_id);
-      }
-    }
-  }
-  if (f.bad()) {
-    PrintErrorAndDie("Failure reading /proc/cpuinfo");
-  }
-  if (!f.eof()) {
-    PrintErrorAndDie("Failed to read to end of /proc/cpuinfo");
-  }
-  f.close();
-
-  if ((max_id + 1) != num_cpus) {
-    fprintf(stderr,
-            "CPU ID assignments in /proc/cpuinfo seem messed up."
-            " This is usually caused by a bad BIOS.\n");
+  if (GetSysctl(hwncpu, &num_cpu)) return num_cpu;
+  PrintErrorAndDie("Err: ", strerror(errno));
+#elif defined(_SC_NPROCESSORS_ONLN)
+  // Returns -1 in case of a failure.
+  int num_cpu = static_cast<int>(sysconf(_SC_NPROCESSORS_ONLN));
+  if (num_cpu < 0) {
+    PrintErrorAndDie("sysconf(_SC_NPROCESSORS_ONLN) failed with error: ",
+                     strerror(errno));
   }
-  return num_cpus;
+  return num_cpu;
 #endif
   BENCHMARK_UNREACHABLE();
 }
 
 int GetNumCPUs() {
-  const int num_cpus = GetNumCPUsImpl();
+  int num_cpus = GetNumCPUsImpl();
   if (num_cpus < 1) {
-    PrintErrorAndDie(
-        "Unable to extract number of CPUs.  If your platform uses "
-        "/proc/cpuinfo, custom support may need to be added.");
+    std::cerr << "Unable to extract number of CPUs.\n";
+    /* There is at least one CPU which we run on. */
+    num_cpus = 1;
   }
   return num_cpus;
 }
diff --git a/utils/bazel/llvm-project-overlay/clang/BUILD.bazel b/utils/bazel/llvm-project-overlay/clang/BUILD.bazel
index a328bfa95d6fd..e3b20e43dd220 100644
--- a/utils/bazel/llvm-project-overlay/clang/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/clang/BUILD.bazel
@@ -14,7 +14,7 @@ load("//:workspace_root.bzl", "workspace_root")
 load("//llvm:binary_alias.bzl", "binary_alias")
 load("//llvm:cc_plugin_library.bzl", "cc_plugin_library")
 load("//llvm:driver.bzl", "llvm_driver_cc_binary")
-load("//llvm:tblgen.bzl", "gentbl")
+load("//mlir:tblgen.bzl", "gentbl_cc_library", "td_library")
 
 package(
     default_visibility = ["//visibility:public"],
@@ -41,7 +41,7 @@ cc_binary(
     ],
 )
 
-gentbl(
+gentbl_cc_library(
     name = "diagnostic_defs_gen",
     tbl_outs = [out for c in [
         "AST",
@@ -59,559 +59,530 @@ gentbl(
         "Serialization",
     ] for out in [
         (
-            "-gen-clang-diags-defs -clang-component=%s" % c,
+            [
+                "-gen-clang-diags-defs",
+                "-clang-component=%s" % c,
+            ],
             "include/clang/Basic/Diagnostic%sKinds.inc" % c,
         ),
         (
-            "-gen-clang-diags-enums -clang-component=%s" % c,
+            [
+                "-gen-clang-diags-enums",
+                "-clang-component=%s" % c,
+            ],
             "include/clang/Basic/Diagnostic%sEnums.inc" % c,
         ),
     ]] + [
         (
-            "-gen-clang-diag-groups",
+            ["-gen-clang-diag-groups"],
             "include/clang/Basic/DiagnosticGroups.inc",
         ),
         (
-            "-gen-clang-diags-index-name",
+            ["-gen-clang-diags-index-name"],
             "include/clang/Basic/DiagnosticIndexName.inc",
         ),
     ],
     tblgen = ":clang-tblgen",
     td_file = "include/clang/Basic/Diagnostic.td",
-    td_srcs = glob(["include/clang/Basic/*.td"]),
+    deps = [":BasicTdFiles"],
 )
 
-gentbl(
+td_library(
+    name = "ArmTdFiles",
+    srcs = [
+        "include/clang/Basic/arm_immcheck_incl.td",
+        "include/clang/Basic/arm_mve_defs.td",
+        "include/clang/Basic/arm_neon_incl.td",
+        "include/clang/Basic/arm_sve_sme_incl.td",
+    ],
+    includes = ["include"],
+)
+
+gentbl_cc_library(
     name = "basic_arm_neon_inc_gen",
     tbl_outs = [(
-        "-gen-arm-neon-sema",
+        ["-gen-arm-neon-sema"],
         "include/clang/Basic/arm_neon.inc",
     )],
     tblgen = ":clang-tblgen",
     td_file = "include/clang/Basic/arm_neon.td",
-    td_srcs = [
-        "include/clang/Basic/arm_immcheck_incl.td",
-        "include/clang/Basic/arm_neon.td",
-        "include/clang/Basic/arm_neon_incl.td",
-    ],
+    deps = [":ArmTdFiles"],
 )
 
-gentbl(
+gentbl_cc_library(
     name = "basic_arm_fp16_inc_gen",
     tbl_outs = [(
-        "-gen-arm-neon-sema",
+        ["-gen-arm-neon-sema"],
         "include/clang/Basic/arm_fp16.inc",
     )],
     tblgen = ":clang-tblgen",
     td_file = "include/clang/Basic/arm_fp16.td",
-    td_srcs = [
-        "include/clang/Basic/arm_fp16.td",
-        "include/clang/Basic/arm_immcheck_incl.td",
-        "include/clang/Basic/arm_neon_incl.td",
-    ],
+    deps = [":ArmTdFiles"],
 )
 
-gentbl(
+gentbl_cc_library(
     name = "basic_arm_mve_aliases_gen",
     tbl_outs = [(
-        "-gen-arm-mve-builtin-aliases",
+        ["-gen-arm-mve-builtin-aliases"],
         "include/clang/Basic/arm_mve_builtin_aliases.inc",
     )],
     tblgen = ":clang-tblgen",
     td_file = "include/clang/Basic/arm_mve.td",
-    td_srcs = [
-        "include/clang/Basic/arm_immcheck_incl.td",
-        "include/clang/Basic/arm_mve.td",
-        "include/clang/Basic/arm_mve_defs.td",
-    ],
+    deps = [":ArmTdFiles"],
 )
 
-gentbl(
+gentbl_cc_library(
     name = "basic_arm_sve_builtins_gen",
     tbl_outs = [(
-        "-gen-arm-sve-builtins",
+        ["-gen-arm-sve-builtins"],
         "include/clang/Basic/arm_sve_builtins.inc",
     )],
     tblgen = ":clang-tblgen",
     td_file = "include/clang/Basic/arm_sve.td",
-    td_srcs = [
-        "include/clang/Basic/arm_immcheck_incl.td",
-        "include/clang/Basic/arm_sve.td",
-        "include/clang/Basic/arm_sve_sme_incl.td",
-    ],
+    deps = [":ArmTdFiles"],
 )
 
-gentbl(
+gentbl_cc_library(
     name = "basic_arm_sve_builtin_cg_gen",
     tbl_outs = [(
-        "-gen-arm-sve-builtin-codegen",
+        ["-gen-arm-sve-builtin-codegen"],
         "include/clang/Basic/arm_sve_builtin_cg.inc",
     )],
     tblgen = ":clang-tblgen",
     td_file = "include/clang/Basic/arm_sve.td",
-    td_srcs = [
-        "include/clang/Basic/arm_immcheck_incl.td",
-        "include/clang/Basic/arm_sve.td",
-        "include/clang/Basic/arm_sve_sme_incl.td",
-    ],
+    deps = [":ArmTdFiles"],
 )
 
-gentbl(
+gentbl_cc_library(
     name = "basic_arm_sve_immcheck_types_gen",
     tbl_outs = [(
-        "-gen-arm-immcheck-types",
+        ["-gen-arm-immcheck-types"],
         "include/clang/Basic/arm_immcheck_types.inc",
     )],
     tblgen = ":clang-tblgen",
     td_file = "include/clang/Basic/arm_sve.td",
-    td_srcs = [
-        "include/clang/Basic/arm_immcheck_incl.td",
-        "include/clang/Basic/arm_sve.td",
-        "include/clang/Basic/arm_sve_sme_incl.td",
-    ],
+    deps = [":ArmTdFiles"],
 )
 
-gentbl(
+gentbl_cc_library(
     name = "basic_arm_sve_typeflags_gen",
     tbl_outs = [(
-        "-gen-arm-sve-typeflags",
+        ["-gen-arm-sve-typeflags"],
         "include/clang/Basic/arm_sve_typeflags.inc",
     )],
     tblgen = ":clang-tblgen",
     td_file = "include/clang/Basic/arm_sve.td",
-    td_srcs = [
-        "include/clang/Basic/arm_immcheck_incl.td",
-        "include/clang/Basic/arm_sve.td",
-        "include/clang/Basic/arm_sve_sme_incl.td",
-    ],
+    deps = [":ArmTdFiles"],
 )
 
-gentbl(
+gentbl_cc_library(
     name = "basic_arm_sve_sema_rangechecks_gen",
     tbl_outs = [(
-        "-gen-arm-sve-sema-rangechecks",
+        ["-gen-arm-sve-sema-rangechecks"],
         "include/clang/Basic/arm_sve_sema_rangechecks.inc",
     )],
     tblgen = ":clang-tblgen",
     td_file = "include/clang/Basic/arm_sve.td",
-    td_srcs = [
-        "include/clang/Basic/arm_immcheck_incl.td",
-        "include/clang/Basic/arm_sve.td",
-        "include/clang/Basic/arm_sve_sme_incl.td",
-    ],
+    deps = [":ArmTdFiles"],
 )
 
-gentbl(
+gentbl_cc_library(
     name = "basic_arm_sve_streaming_attrs_gen",
     tbl_outs = [(
-        "-gen-arm-sve-streaming-attrs",
+        ["-gen-arm-sve-streaming-attrs"],
         "include/clang/Basic/arm_sve_streaming_attrs.inc",
     )],
     tblgen = ":clang-tblgen",
     td_file = "include/clang/Basic/arm_sve.td",
-    td_srcs = [
-        "include/clang/Basic/arm_immcheck_incl.td",
-        "include/clang/Basic/arm_sve.td",
-        "include/clang/Basic/arm_sve_sme_incl.td",
-    ],
+    deps = [":ArmTdFiles"],
 )
 
-gentbl(
+gentbl_cc_library(
     name = "basic_arm_sme_builtins_gen",
     tbl_outs = [(
-        "-gen-arm-sme-builtins",
+        ["-gen-arm-sme-builtins"],
         "include/clang/Basic/arm_sme_builtins.inc",
     )],
     tblgen = ":clang-tblgen",
     td_file = "include/clang/Basic/arm_sme.td",
-    td_srcs = [
-        "include/clang/Basic/arm_immcheck_incl.td",
-        "include/clang/Basic/arm_sme.td",
-        "include/clang/Basic/arm_sve_sme_incl.td",
-    ],
+    deps = [":ArmTdFiles"],
 )
 
-gentbl(
+gentbl_cc_library(
     name = "basic_arm_sme_builtin_cg_gen",
     tbl_outs = [(
-        "-gen-arm-sme-builtin-codegen",
+        ["-gen-arm-sme-builtin-codegen"],
         "include/clang/Basic/arm_sme_builtin_cg.inc",
     )],
     tblgen = ":clang-tblgen",
     td_file = "include/clang/Basic/arm_sme.td",
-    td_srcs = [
-        "include/clang/Basic/arm_sme.td",
-        "include/clang/Basic/arm_sve_sme_incl.td",
-        "include/clang/Basic/arm_immcheck_incl.td",
-    ],
+    deps = [":ArmTdFiles"],
 )
 
-gentbl(
+gentbl_cc_library(
     name = "basic_arm_sme_builtins_za_state_gen",
     tbl_outs = [(
-        "-gen-arm-sme-builtin-za-state",
+        ["-gen-arm-sme-builtin-za-state"],
         "include/clang/Basic/arm_sme_builtins_za_state.inc",
     )],
     tblgen = ":clang-tblgen",
     td_file = "include/clang/Basic/arm_sme.td",
-    td_srcs = [
-        "include/clang/Basic/arm_immcheck_incl.td",
-        "include/clang/Basic/arm_sme.td",
-        "include/clang/Basic/arm_sve_sme_incl.td",
-    ],
+    deps = [":ArmTdFiles"],
 )
 
-gentbl(
+gentbl_cc_library(
     name = "basic_arm_sme_sema_rangechecks_gen",
     tbl_outs = [(
-        "-gen-arm-sme-sema-rangechecks",
+        ["-gen-arm-sme-sema-rangechecks"],
         "include/clang/Basic/arm_sme_sema_rangechecks.inc",
     )],
     tblgen = ":clang-tblgen",
     td_file = "include/clang/Basic/arm_sme.td",
-    td_srcs = [
-        "include/clang/Basic/arm_immcheck_incl.td",
-        "include/clang/Basic/arm_sme.td",
-        "include/clang/Basic/arm_sve_sme_incl.td",
-    ],
+    deps = [":ArmTdFiles"],
 )
 
-gentbl(
+gentbl_cc_library(
     name = "basic_arm_sme_streaming_attrs_gen",
     tbl_outs = [(
-        "-gen-arm-sme-streaming-attrs",
+        ["-gen-arm-sme-streaming-attrs"],
         "include/clang/Basic/arm_sme_streaming_attrs.inc",
     )],
     tblgen = ":clang-tblgen",
     td_file = "include/clang/Basic/arm_sme.td",
-    td_srcs = [
-        "include/clang/Basic/arm_immcheck_incl.td",
-        "include/clang/Basic/arm_sme.td",
-        "include/clang/Basic/arm_sve_sme_incl.td",
-    ],
+    deps = [":ArmTdFiles"],
 )
 
-gentbl(
+gentbl_cc_library(
     name = "basic_arm_mve_cg_gen",
     tbl_outs = [(
-        "-gen-arm-mve-builtin-codegen",
+        ["-gen-arm-mve-builtin-codegen"],
         "include/clang/Basic/arm_mve_builtin_cg.inc",
     )],
     tblgen = ":clang-tblgen",
     td_file = "include/clang/Basic/arm_mve.td",
-    td_srcs = [
-        "include/clang/Basic/arm_mve.td",
-        "include/clang/Basic/arm_mve_defs.td",
-    ],
+    deps = [":ArmTdFiles"],
 )
 
-gentbl(
+gentbl_cc_library(
     name = "basic_arm_mve_inc_gen",
     tbl_outs = [(
-        "-gen-arm-mve-builtin-def",
+        ["-gen-arm-mve-builtin-def"],
         "include/clang/Basic/arm_mve_builtins.inc",
     )],
     tblgen = ":clang-tblgen",
     td_file = "include/clang/Basic/arm_mve.td",
-    td_srcs = [
-        "include/clang/Basic/arm_mve.td",
-        "include/clang/Basic/arm_mve_defs.td",
-    ],
+    deps = [":ArmTdFiles"],
 )
 
-gentbl(
+gentbl_cc_library(
     name = "basic_arm_mve_sema_gen",
     tbl_outs = [(
-        "-gen-arm-mve-builtin-sema",
+        ["-gen-arm-mve-builtin-sema"],
         "include/clang/Basic/arm_mve_builtin_sema.inc",
     )],
     tblgen = ":clang-tblgen",
     td_file = "include/clang/Basic/arm_mve.td",
-    td_srcs = [
-        "include/clang/Basic/arm_mve.td",
-        "include/clang/Basic/arm_mve_defs.td",
-    ],
+    deps = [":ArmTdFiles"],
+)
+
+td_library(
+    name = "BuiltinsBaseTdFiles",
+    srcs = ["include/clang/Basic/BuiltinsBase.td"],
+    includes = ["include"],
+)
+
+td_library(
+    name = "BuiltinsRISCVXCVTdFiles",
+    srcs = ["include/clang/Basic/BuiltinsRISCVXCV.td"],
+    includes = ["include"],
+)
+
+td_library(
+    name = "BuiltinsX86BaseTdFiles",
+    srcs = ["include/clang/Basic/BuiltinsX86Base.td"],
+    includes = ["include"],
 )
 
-gentbl(
+gentbl_cc_library(
     name = "basic_builtins_bpf_gen",
     tbl_outs = [(
-        "-gen-clang-builtins",
+        ["-gen-clang-builtins"],
         "include/clang/Basic/BuiltinsBPF.inc",
     )],
     tblgen = ":clang-tblgen",
     td_file = "include/clang/Basic/BuiltinsBPF.td",
-    td_srcs = [
-        "include/clang/Basic/BuiltinsBPF.td",
-        "include/clang/Basic/BuiltinsBase.td",
-    ],
+    deps = [":BuiltinsBaseTdFiles"],
+)
+
+gentbl_cc_library(
+    name = "basic_builtins_hexagon_gen",
+    tbl_outs = [(
+        ["-gen-clang-builtins"],
+        "include/clang/Basic/BuiltinsHexagon.inc",
+    )],
+    tblgen = ":clang-tblgen",
+    td_file = "include/clang/Basic/BuiltinsHexagon.td",
+    deps = [":BuiltinsBaseTdFiles"],
+)
+
+gentbl_cc_library(
+    name = "basic_builtins_nvptx_gen",
+    tbl_outs = [(
+        ["-gen-clang-builtins"],
+        "include/clang/Basic/BuiltinsNVPTX.inc",
+    )],
+    tblgen = ":clang-tblgen",
+    td_file = "include/clang/Basic/BuiltinsNVPTX.td",
+    deps = [":BuiltinsBaseTdFiles"],
 )
 
-gentbl(
+gentbl_cc_library(
     name = "basic_builtins_spirv_gen",
     tbl_outs = [(
-        "-gen-clang-builtins",
+        ["-gen-clang-builtins"],
         "include/clang/Basic/BuiltinsSPIRV.inc",
     )],
     tblgen = ":clang-tblgen",
     td_file = "include/clang/Basic/BuiltinsSPIRV.td",
-    td_srcs = [
-        "include/clang/Basic/Builtins.td",
-        "include/clang/Basic/BuiltinsBase.td",
-        "include/clang/Basic/BuiltinsSPIRV.td",
-    ],
+    deps = [":BuiltinsBaseTdFiles"],
 )
 
-gentbl(
+gentbl_cc_library(
     name = "basic_builtins_riscv_gen",
     tbl_outs = [(
-        "-gen-clang-builtins",
+        ["-gen-clang-builtins"],
         "include/clang/Basic/BuiltinsRISCV.inc",
     )],
     tblgen = ":clang-tblgen",
     td_file = "include/clang/Basic/BuiltinsRISCV.td",
-    td_srcs = [
-        "include/clang/Basic/BuiltinsRISCV.td",
-        "include/clang/Basic/BuiltinsRISCVXCV.td",
-        "include/clang/Basic/BuiltinsBase.td",
+    deps = [
+        ":BuiltinsBaseTdFiles",
+        ":BuiltinsRISCVXCVTdFiles",
     ],
 )
 
-gentbl(
+gentbl_cc_library(
     name = "basic_builtins_x86_gen",
     tbl_outs = [(
-        "-gen-clang-builtins",
+        ["-gen-clang-builtins"],
         "include/clang/Basic/BuiltinsX86.inc",
     )],
     tblgen = ":clang-tblgen",
     td_file = "include/clang/Basic/BuiltinsX86.td",
-    td_srcs = [
-        "include/clang/Basic/BuiltinsX86.td",
-        "include/clang/Basic/BuiltinsX86Base.td",
-        "include/clang/Basic/BuiltinsBase.td",
+    deps = [
+        ":BuiltinsBaseTdFiles",
+        ":BuiltinsX86BaseTdFiles",
     ],
 )
 
-gentbl(
+gentbl_cc_library(
     name = "basic_builtins_x86_64_gen",
     tbl_outs = [(
-        "-gen-clang-builtins",
+        ["-gen-clang-builtins"],
         "include/clang/Basic/BuiltinsX86_64.inc",
     )],
     tblgen = ":clang-tblgen",
     td_file = "include/clang/Basic/BuiltinsX86_64.td",
-    td_srcs = [
-        "include/clang/Basic/BuiltinsX86_64.td",
-        "include/clang/Basic/BuiltinsX86Base.td",
-        "include/clang/Basic/BuiltinsBase.td",
+    deps = [
+        ":BuiltinsBaseTdFiles",
+        ":BuiltinsX86BaseTdFiles",
     ],
 )
 
-gentbl(
+gentbl_cc_library(
     name = "basic_builtins_gen",
     tbl_outs = [(
-        "-gen-clang-builtins",
+        ["-gen-clang-builtins"],
         "include/clang/Basic/Builtins.inc",
     )],
     tblgen = ":clang-tblgen",
     td_file = "include/clang/Basic/Builtins.td",
-    td_srcs = [
-        "include/clang/Basic/Builtins.td",
-        "include/clang/Basic/BuiltinsBase.td",
+    deps = [":BuiltinsBaseTdFiles"],
+)
+
+td_library(
+    name = "RiscvTdFiles",
+    srcs = [
+        "include/clang/Basic/riscv_vector_common.td",
     ],
+    includes = ["include"],
 )
 
-gentbl(
+gentbl_cc_library(
     name = "basic_riscv_vector_builtins_gen",
     tbl_outs = [(
-        "-gen-riscv-vector-builtins",
+        ["-gen-riscv-vector-builtins"],
         "include/clang/Basic/riscv_vector_builtins.inc",
     )],
     tblgen = ":clang-tblgen",
     td_file = "include/clang/Basic/riscv_vector.td",
-    td_srcs = [
-        "include/clang/Basic/riscv_vector.td",
-        "include/clang/Basic/riscv_vector_common.td",
-    ],
+    deps = [":RiscvTdFiles"],
 )
 
-gentbl(
+gentbl_cc_library(
     name = "basic_riscv_vector_builtin_cg_gen",
     tbl_outs = [(
-        "-gen-riscv-vector-builtin-codegen",
+        ["-gen-riscv-vector-builtin-codegen"],
         "include/clang/Basic/riscv_vector_builtin_cg.inc",
     )],
     tblgen = ":clang-tblgen",
     td_file = "include/clang/Basic/riscv_vector.td",
-    td_srcs = [
-        "include/clang/Basic/riscv_vector.td",
-        "include/clang/Basic/riscv_vector_common.td",
-    ],
+    deps = [":RiscvTdFiles"],
 )
 
-gentbl(
+gentbl_cc_library(
     name = "basic_riscv_vector_builtin_sema_gen",
     tbl_outs = [(
-        "-gen-riscv-vector-builtin-sema",
+        ["-gen-riscv-vector-builtin-sema"],
         "include/clang/Basic/riscv_vector_builtin_sema.inc",
     )],
     tblgen = ":clang-tblgen",
     td_file = "include/clang/Basic/riscv_vector.td",
-    td_srcs = [
-        "include/clang/Basic/riscv_vector.td",
-        "include/clang/Basic/riscv_vector_common.td",
-    ],
+    deps = [":RiscvTdFiles"],
 )
 
-gentbl(
+gentbl_cc_library(
     name = "basic_riscv_sifive_vector_builtins_gen",
     tbl_outs = [(
-        "-gen-riscv-sifive-vector-builtins",
+        ["-gen-riscv-sifive-vector-builtins"],
         "include/clang/Basic/riscv_sifive_vector_builtins.inc",
     )],
     tblgen = ":clang-tblgen",
     td_file = "include/clang/Basic/riscv_sifive_vector.td",
-    td_srcs = [
-        "include/clang/Basic/riscv_sifive_vector.td",
-        "include/clang/Basic/riscv_vector_common.td",
-    ],
+    deps = [":RiscvTdFiles"],
 )
 
-gentbl(
+gentbl_cc_library(
     name = "basic_riscv_sifive_vector_builtin_cg_gen",
     tbl_outs = [(
-        "-gen-riscv-sifive-vector-builtin-codegen",
+        ["-gen-riscv-sifive-vector-builtin-codegen"],
         "include/clang/Basic/riscv_sifive_vector_builtin_cg.inc",
     )],
     tblgen = ":clang-tblgen",
     td_file = "include/clang/Basic/riscv_sifive_vector.td",
-    td_srcs = [
-        "include/clang/Basic/riscv_sifive_vector.td",
-        "include/clang/Basic/riscv_vector_common.td",
-    ],
+    deps = [":RiscvTdFiles"],
 )
 
-gentbl(
+gentbl_cc_library(
     name = "basic_riscv_sifive_vector_builtin_sema_gen",
     tbl_outs = [(
-        "-gen-riscv-sifive-vector-builtin-sema",
+        ["-gen-riscv-sifive-vector-builtin-sema"],
         "include/clang/Basic/riscv_sifive_vector_builtin_sema.inc",
     )],
     tblgen = ":clang-tblgen",
     td_file = "include/clang/Basic/riscv_sifive_vector.td",
-    td_srcs = [
-        "include/clang/Basic/riscv_sifive_vector.td",
-        "include/clang/Basic/riscv_vector_common.td",
-    ],
+    deps = [":RiscvTdFiles"],
 )
 
-gentbl(
+gentbl_cc_library(
     name = "basic_arm_cde_gen",
     tbl_outs = [(
-        "-gen-arm-cde-builtin-def",
+        ["-gen-arm-cde-builtin-def"],
         "include/clang/Basic/arm_cde_builtins.inc",
     )],
     tblgen = ":clang-tblgen",
     td_file = "include/clang/Basic/arm_cde.td",
-    td_srcs = [
-        "include/clang/Basic/arm_cde.td",
-        "include/clang/Basic/arm_mve_defs.td",
-    ],
+    deps = [":ArmTdFiles"],
 )
 
-gentbl(
+gentbl_cc_library(
     name = "basic_arm_cde_aliases_gen",
     tbl_outs = [(
-        "-gen-arm-cde-builtin-aliases",
+        ["-gen-arm-cde-builtin-aliases"],
         "include/clang/Basic/arm_cde_builtin_aliases.inc",
     )],
     tblgen = ":clang-tblgen",
     td_file = "include/clang/Basic/arm_cde.td",
-    td_srcs = [
-        "include/clang/Basic/arm_cde.td",
-        "include/clang/Basic/arm_mve_defs.td",
-    ],
+    deps = [":ArmTdFiles"],
 )
 
-gentbl(
+gentbl_cc_library(
     name = "basic_arm_cde_cg_gen",
     tbl_outs = [(
-        "-gen-arm-cde-builtin-codegen",
+        ["-gen-arm-cde-builtin-codegen"],
         "include/clang/Basic/arm_cde_builtin_cg.inc",
     )],
     tblgen = ":clang-tblgen",
     td_file = "include/clang/Basic/arm_cde.td",
-    td_srcs = [
-        "include/clang/Basic/arm_cde.td",
-        "include/clang/Basic/arm_mve_defs.td",
-    ],
+    deps = [":ArmTdFiles"],
 )
 
-gentbl(
+gentbl_cc_library(
     name = "basic_arm_cde_sema_gen",
     tbl_outs = [(
-        "-gen-arm-cde-builtin-sema",
+        ["-gen-arm-cde-builtin-sema"],
         "include/clang/Basic/arm_cde_builtin_sema.inc",
     )],
     tblgen = ":clang-tblgen",
     td_file = "include/clang/Basic/arm_cde.td",
-    td_srcs = [
-        "include/clang/Basic/arm_cde.td",
-        "include/clang/Basic/arm_mve_defs.td",
+    deps = [":ArmTdFiles"],
+)
+
+td_library(
+    name = "ASTNodeTdFiles",
+    srcs = ["include/clang/Basic/ASTNode.td"],
+    includes = ["include"],
+)
+
+td_library(
+    name = "BasicCoreTdFiles",
+    srcs = [
+        "include/clang/Basic/AttrDocs.td",
+        "include/clang/Basic/DeclNodes.td",
+        "include/clang/Basic/StmtNodes.td",
     ],
+    includes = ["include"],
+    deps = [":ASTNodeTdFiles"],
 )
 
-gentbl(
+td_library(
+    name = "BasicTdFiles",
+    srcs = glob(["include/clang/Basic/*.td"]),
+    includes = ["include"],
+)
+
+gentbl_cc_library(
     name = "basic_attr_gen",
     tbl_outs = [
         (
-            "-gen-clang-attr-has-attribute-impl",
+            ["-gen-clang-attr-has-attribute-impl"],
             "include/clang/Basic/AttrHasAttributeImpl.inc",
         ),
         (
-            "-gen-clang-attr-list",
+            ["-gen-clang-attr-list"],
             "include/clang/Basic/AttrList.inc",
         ),
         (
-            "-gen-clang-attr-parsed-attr-list",
+            ["-gen-clang-attr-parsed-attr-list"],
             "include/clang/Basic/AttrParsedAttrList.inc",
         ),
         (
-            "-gen-clang-attr-subject-match-rule-list",
+            ["-gen-clang-attr-subject-match-rule-list"],
             "include/clang/Basic/AttrSubMatchRulesList.inc",
         ),
         (
-            "-gen-clang-regular-keyword-attr-info",
+            ["-gen-clang-regular-keyword-attr-info"],
             "include/clang/Basic/RegularKeywordAttrInfo.inc",
         ),
         (
-            "-gen-cxx11-attribute-info",
+            ["-gen-cxx11-attribute-info"],
             "include/clang/Basic/CXX11AttributeInfo.inc",
         ),
     ],
     tblgen = ":clang-tblgen",
     td_file = "include/clang/Basic/Attr.td",
-    td_srcs = [
-        "include/clang/Basic/ASTNode.td",
-        "include/clang/Basic/Attr.td",
-        "include/clang/Basic/AttrDocs.td",
-        "include/clang/Basic/DeclNodes.td",
-        "include/clang/Basic/StmtNodes.td",
-    ],
+    deps = [":BasicCoreTdFiles"],
 )
 
-gentbl(
+gentbl_cc_library(
     name = "libsema_openclbuiltins_inc_gen",
     strip_include_prefix = "lib/Sema",
     tbl_outs = [(
-        "-gen-clang-opencl-builtins",
+        ["-gen-clang-opencl-builtins"],
         "lib/Sema/OpenCLBuiltins.inc",
     )],
     tblgen = ":clang-tblgen",
     td_file = "lib/Sema/OpenCLBuiltins.td",
-    td_srcs = [
-        "lib/Sema/OpenCLBuiltins.td",
-    ],
 )
 
 # Table definition files can be used for documentation:
@@ -633,6 +604,7 @@ genrule(
         "echo '#define CLANG_VERSION_MAJOR_STRING \"{major}\"' >> $@\n" +
         "echo '#define CLANG_VERSION_MINOR {minor}' >> $@\n" +
         "echo '#define CLANG_VERSION_PATCHLEVEL {patch}' >> $@\n" +
+        "echo '#define MAX_CLANG_ABI_COMPAT_VERSION {major}' >> $@\n" +
         "echo '#define CLANG_VERSION_STRING \"{vers}\"' >> $@\n"
     ).format(
         major = LLVM_VERSION_MAJOR,
@@ -743,6 +715,8 @@ cc_library(
         ":basic_attr_gen",
         ":basic_builtins_bpf_gen",
         ":basic_builtins_gen",
+        ":basic_builtins_hexagon_gen",
+        ":basic_builtins_nvptx_gen",
         ":basic_builtins_riscv_gen",
         ":basic_builtins_spirv_gen",
         ":basic_builtins_x86_64_gen",
@@ -786,185 +760,174 @@ cc_library(
     ],
 )
 
-gentbl(
+gentbl_cc_library(
     name = "ast_attr_gen",
     tbl_outs = [
         (
-            "-gen-clang-attr-ast-visitor",
+            ["-gen-clang-attr-ast-visitor"],
             "include/clang/AST/AttrVisitor.inc",
         ),
         (
-            "-gen-clang-attr-classes",
+            ["-gen-clang-attr-classes"],
             "include/clang/AST/Attrs.inc",
         ),
         (
-            "-gen-clang-attr-doc-table",
-            "lib/AST/AttrDocTable.inc",
-        ),
-        (
-            "-gen-clang-attr-text-node-dump",
+            ["-gen-clang-attr-text-node-dump"],
             "include/clang/AST/AttrTextNodeDump.inc",
         ),
         (
-            "-gen-clang-attr-node-traverse",
+            ["-gen-clang-attr-node-traverse"],
             "include/clang/AST/AttrNodeTraverse.inc",
         ),
         (
-            "-gen-clang-attr-impl",
+            ["-gen-clang-attr-impl"],
             "include/clang/AST/AttrImpl.inc",
         ),
     ],
     tblgen = ":clang-tblgen",
     td_file = "include/clang/Basic/Attr.td",
-    td_srcs = [
-        "include/clang/Basic/Attr.td",
-        "include/clang/Basic/AttrDocs.td",
-        "include/clang/Basic/ASTNode.td",
-        "include/clang/Basic/DeclNodes.td",
-        "include/clang/Basic/StmtNodes.td",
+    deps = [":BasicCoreTdFiles"],
+)
+
+gentbl_cc_library(
+    name = "ast_attr_doc_table_gen",
+    strip_include_prefix = "lib/AST",
+    tbl_outs = [
+        (
+            ["-gen-clang-attr-doc-table"],
+            "lib/AST/AttrDocTable.inc",
+        ),
     ],
+    tblgen = ":clang-tblgen",
+    td_file = "include/clang/Basic/Attr.td",
+    deps = [":BasicCoreTdFiles"],
 )
 
-gentbl(
+gentbl_cc_library(
     name = "ast_decl_nodes_gen",
     tbl_outs = [(
-        "-gen-clang-decl-nodes",
+        ["-gen-clang-decl-nodes"],
         "include/clang/AST/DeclNodes.inc",
     )],
     tblgen = ":clang-tblgen",
     td_file = "include/clang/Basic/DeclNodes.td",
-    td_srcs = [
-        "include/clang/Basic/ASTNode.td",
-        "include/clang/Basic/DeclNodes.td",
-    ],
+    deps = [":ASTNodeTdFiles"],
 )
 
-gentbl(
+gentbl_cc_library(
     name = "ast_stmt_nodes_gen",
     tbl_outs = [(
-        "-gen-clang-stmt-nodes",
+        ["-gen-clang-stmt-nodes"],
         "include/clang/AST/StmtNodes.inc",
     )],
     tblgen = ":clang-tblgen",
     td_file = "include/clang/Basic/StmtNodes.td",
-    td_srcs = [
-        "include/clang/Basic/ASTNode.td",
-        "include/clang/Basic/StmtNodes.td",
-    ],
+    deps = [":ASTNodeTdFiles"],
 )
 
-gentbl(
+gentbl_cc_library(
     name = "ast_comment_nodes_gen",
     tbl_outs = [(
-        "-gen-clang-comment-nodes",
+        ["-gen-clang-comment-nodes"],
         "include/clang/AST/CommentNodes.inc",
     )],
     tblgen = ":clang-tblgen",
     td_file = "include/clang/Basic/CommentNodes.td",
-    td_srcs = [
-        "include/clang/Basic/ASTNode.td",
-        "include/clang/Basic/CommentNodes.td",
-    ],
+    deps = [":ASTNodeTdFiles"],
 )
 
-gentbl(
+gentbl_cc_library(
     name = "ast_comment_command_info_gen",
     tbl_outs = [
         (
-            "-gen-clang-comment-command-info",
+            ["-gen-clang-comment-command-info"],
             "include/clang/AST/CommentCommandInfo.inc",
         ),
         (
-            "-gen-clang-comment-command-list",
+            ["-gen-clang-comment-command-list"],
             "include/clang/AST/CommentCommandList.inc",
         ),
     ],
     tblgen = ":clang-tblgen",
     td_file = "include/clang/AST/CommentCommands.td",
-    td_srcs = ["include/clang/AST/CommentCommands.td"],
 )
 
-gentbl(
+gentbl_cc_library(
     name = "ast_comment_html_tags_gen",
     tbl_outs = [(
-        "-gen-clang-comment-html-tags",
+        ["-gen-clang-comment-html-tags"],
         "include/clang/AST/CommentHTMLTags.inc",
     )],
     tblgen = ":clang-tblgen",
     td_file = "include/clang/AST/CommentHTMLTags.td",
-    td_srcs = ["include/clang/AST/CommentHTMLTags.td"],
 )
 
-gentbl(
+gentbl_cc_library(
     name = "ast_comment_html_tags_properties_gen",
     tbl_outs = [(
-        "-gen-clang-comment-html-tags-properties",
+        ["-gen-clang-comment-html-tags-properties"],
         "include/clang/AST/CommentHTMLTagsProperties.inc",
     )],
     tblgen = ":clang-tblgen",
     td_file = "include/clang/AST/CommentHTMLTags.td",
-    td_srcs = ["include/clang/AST/CommentHTMLTags.td"],
 )
 
-gentbl(
+gentbl_cc_library(
     name = "ast_comment_html_named_character_references_gen",
     tbl_outs = [(
-        "-gen-clang-comment-html-named-character-references",
+        ["-gen-clang-comment-html-named-character-references"],
         "include/clang/AST/CommentHTMLNamedCharacterReferences.inc",
     )],
     tblgen = ":clang-tblgen",
     td_file = "include/clang/AST/CommentHTMLNamedCharacterReferences.td",
-    td_srcs = ["include/clang/AST/CommentHTMLNamedCharacterReferences.td"],
 )
 
-gentbl(
+gentbl_cc_library(
     name = "ast_stmt_data_collectors_gen",
     tbl_outs = [(
-        "-gen-clang-data-collectors",
+        ["-gen-clang-data-collectors"],
         "include/clang/AST/StmtDataCollectors.inc",
     )],
     tblgen = ":clang-tblgen",
     td_file = "include/clang/AST/StmtDataCollectors.td",
-    td_srcs = ["include/clang/AST/StmtDataCollectors.td"],
 )
 
-gentbl(
+gentbl_cc_library(
     name = "ast_bytecode_opcodes_gen",
+    strip_include_prefix = "lib/AST/ByteCode",
     tbl_outs = [(
-        "-gen-clang-opcodes",
+        ["-gen-clang-opcodes"],
         "lib/AST/ByteCode/Opcodes.inc",
     )],
     tblgen = ":clang-tblgen",
     td_file = "lib/AST/ByteCode/Opcodes.td",
-    td_srcs = ["lib/AST/ByteCode/Opcodes.td"],
 )
 
-gentbl(
+gentbl_cc_library(
     name = "ast_properties_base_gen",
     tbl_outs = [
         (
-            "-gen-clang-basic-reader",
+            ["-gen-clang-basic-reader"],
             "include/clang/AST/AbstractBasicReader.inc",
         ),
         (
-            "-gen-clang-basic-writer",
+            ["-gen-clang-basic-writer"],
             "include/clang/AST/AbstractBasicWriter.inc",
         ),
     ],
     tblgen = ":clang-tblgen",
     td_file = "include/clang/AST/PropertiesBase.td",
-    td_srcs = ["include/clang/AST/PropertiesBase.td"],
 )
 
-gentbl(
+gentbl_cc_library(
     name = "ast_type_properties_gen",
     tbl_outs = [
         (
-            "-gen-clang-type-reader",
+            ["-gen-clang-type-reader"],
             "include/clang/AST/AbstractTypeReader.inc",
         ),
         (
-            "-gen-clang-type-writer",
+            ["-gen-clang-type-writer"],
             "include/clang/AST/AbstractTypeWriter.inc",
         ),
     ],
@@ -972,24 +935,20 @@ gentbl(
     td_file = "include/clang/AST/TypeProperties.td",
     td_srcs = [
         "include/clang/AST/PropertiesBase.td",
-        "include/clang/AST/TypeProperties.td",
-        "include/clang/Basic/ASTNode.td",
         "include/clang/Basic/TypeNodes.td",
     ],
+    deps = [":ASTNodeTdFiles"],
 )
 
-gentbl(
+gentbl_cc_library(
     name = "type_nodes_gen",
     tbl_outs = [(
-        "-gen-clang-type-nodes",
+        ["-gen-clang-type-nodes"],
         "include/clang/AST/TypeNodes.inc",
     )],
     tblgen = ":clang-tblgen",
     td_file = "include/clang/Basic/TypeNodes.td",
-    td_srcs = [
-        "include/clang/Basic/ASTNode.td",
-        "include/clang/Basic/TypeNodes.td",
-    ],
+    deps = [":ASTNodeTdFiles"],
 )
 
 workspace_root(name = "workspace_root")
@@ -1041,6 +1000,7 @@ cc_library(
     ]),
     toolchains = [":workspace_root"],
     deps = [
+        ":ast_attr_doc_table_gen",
         ":ast_attr_gen",
         ":ast_bytecode_opcodes_gen",
         ":ast_comment_command_info_gen",
@@ -1132,35 +1092,29 @@ cc_library(
     ],
 )
 
-gentbl(
+gentbl_cc_library(
     name = "sema_attr_gen",
     tbl_outs = [
         (
-            "-gen-clang-attr-parsed-attr-impl",
+            ["-gen-clang-attr-parsed-attr-impl"],
             "include/clang/Sema/AttrParsedAttrImpl.inc",
         ),
         (
-            "-gen-clang-attr-parsed-attr-kinds",
+            ["-gen-clang-attr-parsed-attr-kinds"],
             "include/clang/Sema/AttrParsedAttrKinds.inc",
         ),
         (
-            "-gen-clang-attr-spelling-index",
+            ["-gen-clang-attr-spelling-index"],
             "include/clang/Sema/AttrSpellingListIndex.inc",
         ),
         (
-            "-gen-clang-attr-template-instantiate",
+            ["-gen-clang-attr-template-instantiate"],
             "include/clang/Sema/AttrTemplateInstantiate.inc",
         ),
     ],
     tblgen = ":clang-tblgen",
     td_file = "include/clang/Basic/Attr.td",
-    td_srcs = [
-        "include/clang/Basic/ASTNode.td",
-        "include/clang/Basic/Attr.td",
-        "include/clang/Basic/AttrDocs.td",
-        "include/clang/Basic/DeclNodes.td",
-        "include/clang/Basic/StmtNodes.td",
-    ],
+    deps = [":BasicCoreTdFiles"],
 )
 
 cc_library(
@@ -1218,27 +1172,21 @@ cc_library(
     ],
 )
 
-gentbl(
+gentbl_cc_library(
     name = "parse_attr_gen",
     tbl_outs = [
         (
-            "-gen-clang-attr-parser-string-switches",
+            ["-gen-clang-attr-parser-string-switches"],
             "include/clang/Parse/AttrParserStringSwitches.inc",
         ),
         (
-            "-gen-clang-attr-subject-match-rules-parser-string-switches",
+            ["-gen-clang-attr-subject-match-rules-parser-string-switches"],
             "include/clang/Parse/AttrSubMatchRulesParserStringSwitches.inc",
         ),
     ],
     tblgen = ":clang-tblgen",
     td_file = "include/clang/Basic/Attr.td",
-    td_srcs = [
-        "include/clang/Basic/ASTNode.td",
-        "include/clang/Basic/Attr.td",
-        "include/clang/Basic/AttrDocs.td",
-        "include/clang/Basic/DeclNodes.td",
-        "include/clang/Basic/StmtNodes.td",
-    ],
+    deps = [":BasicCoreTdFiles"],
 )
 
 cc_library(
@@ -1418,18 +1366,27 @@ cc_library(
     ],
 )
 
-gentbl(
+td_library(
+    name = "ToolingSyntaxTdFiles",
+    srcs = ["include/clang/Tooling/Syntax/Syntax.td"],
+    includes = ["include"],
+)
+
+gentbl_cc_library(
     name = "tooling_syntax_gen",
     tbl_outs = [
-        ("-gen-clang-syntax-node-list", "include/clang/Tooling/Syntax/Nodes.inc"),
-        ("-gen-clang-syntax-node-classes", "include/clang/Tooling/Syntax/NodeClasses.inc"),
+        (
+            ["-gen-clang-syntax-node-list"],
+            "include/clang/Tooling/Syntax/Nodes.inc",
+        ),
+        (
+            ["-gen-clang-syntax-node-classes"],
+            "include/clang/Tooling/Syntax/NodeClasses.inc",
+        ),
     ],
     tblgen = ":clang-tblgen",
     td_file = "include/clang/Tooling/Syntax/Nodes.td",
-    td_srcs = [
-        "include/clang/Tooling/Syntax/Nodes.td",
-        "include/clang/Tooling/Syntax/Syntax.td",
-    ],
+    deps = [":ToolingSyntaxTdFiles"],
 )
 
 cc_library(
@@ -1586,18 +1543,23 @@ cc_library(
     ],
 )
 
-gentbl(
+td_library(
+    name = "CheckerBaseTdFiles",
+    srcs = [
+        "include/clang/StaticAnalyzer/Checkers/CheckerBase.td",
+    ],
+    includes = ["include"],
+)
+
+gentbl_cc_library(
     name = "static_analyzer_checkers_gen",
     tbl_outs = [(
-        "-gen-clang-sa-checkers",
+        ["-gen-clang-sa-checkers"],
         "include/clang/StaticAnalyzer/Checkers/Checkers.inc",
     )],
     tblgen = ":clang-tblgen",
     td_file = "include/clang/StaticAnalyzer/Checkers/Checkers.td",
-    td_srcs = [
-        "include/clang/StaticAnalyzer/Checkers/CheckerBase.td",
-        "include/clang/StaticAnalyzer/Checkers/Checkers.td",
-    ],
+    deps = [":CheckerBaseTdFiles"],
 )
 
 cc_library(
@@ -1626,17 +1588,15 @@ cc_library(
     ],
 )
 
-gentbl(
+gentbl_cc_library(
     name = "driver_options_inc_gen",
     tbl_outs = [(
-        "-gen-opt-parser-defs",
+        ["-gen-opt-parser-defs"],
         "include/clang/Driver/Options.inc",
     )],
     tblgen = "//llvm:llvm-tblgen",
     td_file = "include/clang/Driver/Options.td",
-    td_srcs = [
-        "//llvm:include/llvm/Option/OptParser.td",
-    ],
+    deps = ["//llvm:OptParserTdFiles"],
 )
 
 cc_library(
@@ -1692,136 +1652,103 @@ cc_library(
     ],
 )
 
-gentbl(
+gentbl_cc_library(
     name = "headers_arm_neon_gen",
     tbl_outs = [(
-        "-gen-arm-neon",
+        ["-gen-arm-neon"],
         "lib/Headers/arm_neon.h",
     )],
     tblgen = ":clang-tblgen",
     td_file = "include/clang/Basic/arm_neon.td",
-    td_srcs = [
-        "include/clang/Basic/arm_immcheck_incl.td",
-        "include/clang/Basic/arm_neon.td",
-        "include/clang/Basic/arm_neon_incl.td",
-    ],
+    deps = [":ArmTdFiles"],
 )
 
-gentbl(
+gentbl_cc_library(
     name = "headers_arm_fp16_gen",
     tbl_outs = [(
-        "-gen-arm-fp16",
+        ["-gen-arm-fp16"],
         "lib/Headers/arm_fp16.h",
     )],
     tblgen = ":clang-tblgen",
     td_file = "include/clang/Basic/arm_fp16.td",
-    td_srcs = [
-        "include/clang/Basic/arm_fp16.td",
-        "include/clang/Basic/arm_immcheck_incl.td",
-        "include/clang/Basic/arm_neon_incl.td",
-    ],
+    deps = [":ArmTdFiles"],
 )
 
-gentbl(
+gentbl_cc_library(
     name = "headers_arm_mve_gen",
     tbl_outs = [(
-        "-gen-arm-mve-header",
+        ["-gen-arm-mve-header"],
         "lib/Headers/arm_mve.h",
     )],
     tblgen = ":clang-tblgen",
     td_file = "include/clang/Basic/arm_mve.td",
-    td_srcs = [
-        "include/clang/Basic/arm_mve.td",
-        "include/clang/Basic/arm_mve_defs.td",
-    ],
+    deps = [":ArmTdFiles"],
 )
 
-gentbl(
+gentbl_cc_library(
     name = "headers_arm_cde_gen",
     tbl_outs = [(
-        "-gen-arm-cde-header",
+        ["-gen-arm-cde-header"],
         "lib/Headers/arm_cde.h",
     )],
     tblgen = ":clang-tblgen",
     td_file = "include/clang/Basic/arm_cde.td",
-    td_srcs = [
-        "include/clang/Basic/arm_cde.td",
-        "include/clang/Basic/arm_mve_defs.td",
-    ],
+    deps = [":ArmTdFiles"],
 )
 
-gentbl(
+gentbl_cc_library(
     name = "headers_arm_sve_gen",
     tbl_outs = [(
-        "-gen-arm-sve-header",
+        ["-gen-arm-sve-header"],
         "lib/Headers/arm_sve.h",
     )],
     tblgen = ":clang-tblgen",
     td_file = "include/clang/Basic/arm_sve.td",
-    td_srcs = [
-        "include/clang/Basic/arm_immcheck_incl.td",
-        "include/clang/Basic/arm_sve.td",
-        "include/clang/Basic/arm_sve_sme_incl.td",
-    ],
+    deps = [":ArmTdFiles"],
 )
 
-gentbl(
+gentbl_cc_library(
     name = "headers_arm_bf16_gen",
     tbl_outs = [(
-        "-gen-arm-bf16",
+        ["-gen-arm-bf16"],
         "lib/Headers/arm_bf16.h",
     )],
     tblgen = ":clang-tblgen",
     td_file = "include/clang/Basic/arm_bf16.td",
-    td_srcs = [
-        "include/clang/Basic/arm_bf16.td",
-        "include/clang/Basic/arm_immcheck_incl.td",
-        "include/clang/Basic/arm_neon_incl.td",
-    ],
+    deps = [":ArmTdFiles"],
 )
 
-gentbl(
+gentbl_cc_library(
     name = "headers_arm_sme_gen",
     tbl_outs = [(
-        "-gen-arm-sme-header",
+        ["-gen-arm-sme-header"],
         "lib/Headers/arm_sme.h",
     )],
     tblgen = ":clang-tblgen",
     td_file = "include/clang/Basic/arm_sme.td",
-    td_srcs = [
-        "include/clang/Basic/arm_immcheck_incl.td",
-        "include/clang/Basic/arm_sme.td",
-        "include/clang/Basic/arm_sve_sme_incl.td",
-    ],
+    deps = [":ArmTdFiles"],
 )
 
-gentbl(
+gentbl_cc_library(
     name = "headers_arm_vector_type_gen",
     tbl_outs = [(
-        "-gen-arm-vector-type",
+        ["-gen-arm-vector-type"],
         "lib/Headers/arm_vector_types.h",
     )],
     tblgen = ":clang-tblgen",
     td_file = "include/clang/Basic/arm_neon.td",
-    td_srcs = [
-        "include/clang/Basic/arm_immcheck_incl.td",
-        "include/clang/Basic/arm_neon.td",
-        "include/clang/Basic/arm_neon_incl.td",
-    ],
+    deps = [":ArmTdFiles"],
 )
 
-gentbl(
+gentbl_cc_library(
     name = "headers_riscv_vector",
     tbl_outs = [(
-        "-gen-riscv-vector-header",
+        ["-gen-riscv-vector-header"],
         "lib/Headers/riscv_vector.h",
     )],
     tblgen = ":clang-tblgen",
     td_file = "include/clang/Basic/riscv_vector.td",
-    td_srcs = [
-        "include/clang/Basic/riscv_vector.td",
-        "include/clang/Basic/riscv_vector_common.td",
-    ],
+    deps = [":RiscvTdFiles"],
 )
 
 # We generate the set of builtin headers under a special subdirectory in the
@@ -2071,27 +1998,21 @@ cc_library(
     ],
 )
 
-gentbl(
+gentbl_cc_library(
     name = "serialization_attr_gen",
     tbl_outs = [
         (
-            "-gen-clang-attr-pch-read",
+            ["-gen-clang-attr-pch-read"],
             "include/clang/Serialization/AttrPCHRead.inc",
         ),
         (
-            "-gen-clang-attr-pch-write",
+            ["-gen-clang-attr-pch-write"],
             "include/clang/Serialization/AttrPCHWrite.inc",
         ),
     ],
     tblgen = ":clang-tblgen",
     td_file = "include/clang/Basic/Attr.td",
-    td_srcs = [
-        "include/clang/Basic/ASTNode.td",
-        "include/clang/Basic/Attr.td",
-        "include/clang/Basic/AttrDocs.td",
-        "include/clang/Basic/DeclNodes.td",
-        "include/clang/Basic/StmtNodes.td",
-    ],
+    deps = [":BasicCoreTdFiles"],
 )
 
 cc_library(
@@ -2173,7 +2094,6 @@ cc_library(
     hdrs = glob(["include/clang/FrontendTool/*.h"]),
     includes = ["include"],
     deps = [
-        ":arc_migrate",
         ":codegen",
         ":config",
         ":driver",
@@ -2186,33 +2106,6 @@ cc_library(
     ],
 )
 
-cc_library(
-    name = "arc_migrate",
-    srcs = glob([
-        "lib/ARCMigrate/*.cpp",
-        "lib/ARCMigrate/*.h",
-    ]),
-    hdrs = glob(["include/clang/ARCMigrate/*.h"]),
-    includes = ["include"],
-    deps = [
-        ":analysis",
-        ":ast",
-        ":basic",
-        ":edit",
-        ":frontend",
-        ":frontend_rewrite",
-        ":lex",
-        ":parse",
-        ":rewrite",
-        ":sema",
-        ":serialization",
-        ":static_analyzer_checkers",
-        ":static_analyzer_core",
-        "//llvm:Support",
-        "//llvm:TargetParser",
-    ],
-)
-
 cc_library(
     name = "libclang_static",
     srcs = glob([
@@ -2222,7 +2115,6 @@ cc_library(
     hdrs = glob(["include/clang-c/*.h"]),
     defines = ["CINDEX_NO_EXPORTS"],
     deps = [
-        ":arc_migrate",
         ":ast",
         ":basic",
         ":codegen",
@@ -2255,7 +2147,6 @@ cc_plugin_library(
     }),
     strip_include_prefix = "include",
     deps = [
-        ":arc_migrate",
         ":ast",
         ":basic",
         ":codegen",
@@ -2344,39 +2235,6 @@ cc_binary(
     ],
 )
 
-cc_binary(
-    name = "arcmt-test",
-    testonly = 1,
-    srcs = ["tools/arcmt-test/arcmt-test.cpp"],
-    stamp = 0,
-    deps = [
-        ":arc_migrate",
-        ":ast",
-        ":basic",
-        ":frontend",
-        ":frontend_rewrite",
-        ":lex",
-        "//llvm:Support",
-    ],
-)
-
-cc_binary(
-    name = "c-arcmt-test",
-    testonly = 1,
-    srcs = ["tools/c-arcmt-test/c-arcmt-test.c"],
-    copts = select({
-        "@platforms//os:windows": [],
-        "//conditions:default": ["-std=gnu99"],
-    }),
-    stamp = 0,
-    deps = [
-        ":codegen",
-        ":libclang",
-        "//llvm:MC",
-        "//llvm:Support",
-    ],
-)
-
 cc_binary(
     name = "clang-import-test",
     testonly = 1,
@@ -2563,18 +2421,16 @@ cc_binary(
     ],
 )
 
-gentbl(
+gentbl_cc_library(
     name = "linker_wrapper_opts_gen",
+    strip_include_prefix = "tools/clang-linker-wrapper",
     tbl_outs = [(
-        "-gen-opt-parser-defs",
+        ["-gen-opt-parser-defs"],
         "tools/clang-linker-wrapper/LinkerWrapperOpts.inc",
     )],
     tblgen = "//llvm:llvm-tblgen",
     td_file = "tools/clang-linker-wrapper/LinkerWrapperOpts.td",
-    td_srcs = [
-        "tools/clang-linker-wrapper/LinkerWrapperOpts.td",
-        "//llvm:include/llvm/Option/OptParser.td",
-    ],
+    deps = ["//llvm:OptParserTdFiles"],
 )
 
 cc_binary(
@@ -2664,16 +2520,16 @@ cc_binary(
     ],
 )
 
-gentbl(
+gentbl_cc_library(
     name = "ScanDepsTableGen",
     strip_include_prefix = "tools/clang-scan-deps",
     tbl_outs = [(
-        "-gen-opt-parser-defs",
+        ["-gen-opt-parser-defs"],
         "tools/clang-scan-deps/Opts.inc",
     )],
     tblgen = "//llvm:llvm-tblgen",
     td_file = "tools/clang-scan-deps/Opts.td",
-    td_srcs = ["//llvm:include/llvm/Option/OptParser.td"],
+    deps = ["//llvm:OptParserTdFiles"],
 )
 
 cc_library(
diff --git a/utils/bazel/llvm-project-overlay/clang/include/clang/Config/config.h b/utils/bazel/llvm-project-overlay/clang/include/clang/Config/config.h
index ac0d9eb24931f..fd26489ffd5a8 100644
--- a/utils/bazel/llvm-project-overlay/clang/include/clang/Config/config.h
+++ b/utils/bazel/llvm-project-overlay/clang/include/clang/Config/config.h
@@ -93,8 +93,7 @@
 #define ENABLE_EXPERIMENTAL_NEW_PASS_MANAGER 0
 
 /* Enable each functionality of modules */
-#define CLANG_ENABLE_ARCMT 1
-#define CLANG_ENABLE_OBJC_REWRITER 1
+#define CLANG_ENABLE_OBJC_REWRITER 0
 #define CLANG_ENABLE_STATIC_ANALYZER 1
 
 /* Spawn a new process clang.exe for the CC1 tool invocation, when necessary */
diff --git a/utils/bazel/llvm-project-overlay/libc/utils/MPFRWrapper/BUILD.bazel b/utils/bazel/llvm-project-overlay/libc/utils/MPFRWrapper/BUILD.bazel
index ca21eaee504ff..cd70788df5508 100644
--- a/utils/bazel/llvm-project-overlay/libc/utils/MPFRWrapper/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/libc/utils/MPFRWrapper/BUILD.bazel
@@ -26,6 +26,32 @@ cc_library(
     ),
 )
 
+libc_support_library(
+    name = "mp_common",
+    srcs = ["MPCommon.cpp"],
+    hdrs = ["MPCommon.h"],
+    # Disable layering check when using mpfr_system.
+    features = select(
+        {
+            "//conditions:default": [],
+            "//libc:mpfr_system": ["-layering_check"],
+        },
+    ),
+    deps = [
+        "//libc:__support_common",
+        "//libc:__support_cpp_string",
+        "//libc:__support_cpp_string_view",
+        "//libc:__support_cpp_type_traits",
+        "//libc:__support_fputil_cast",
+        "//libc:__support_fputil_fp_bits",
+        "//libc:__support_macros_config",
+        "//libc:__support_macros_properties_types",
+        "//libc/test/UnitTest:fp_test_helpers",
+        "//libc/utils/MPFRWrapper:mpfr_impl",
+        "@gmp//:gmp_",
+    ],
+)
+
 libc_support_library(
     name = "mpfr_wrapper",
     srcs = ["MPFRUtils.cpp"],
@@ -38,6 +64,7 @@ libc_support_library(
         },
     ),
     deps = [
+        ":mp_common",
         "//libc:__support_common",
         "//libc:__support_cpp_array",
         "//libc:__support_cpp_bit",
diff --git a/utils/bazel/llvm-project-overlay/lld/BUILD.bazel b/utils/bazel/llvm-project-overlay/lld/BUILD.bazel
index 40154de2c7a02..94bcf68896615 100644
--- a/utils/bazel/llvm-project-overlay/lld/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/lld/BUILD.bazel
@@ -43,6 +43,7 @@ cc_library(
     srcs = [":config_version_gen"] + glob(["Common/*.cpp"]),
     hdrs = glob(["include/lld/Common/*.h"]),
     includes = ["include"],
+    textual_hdrs = glob(["include/lld/Common/*.inc"]),
     deps = [
         ":vcs_version",
         "//llvm:CodeGen",
diff --git a/utils/bazel/llvm-project-overlay/lldb/BUILD.bazel b/utils/bazel/llvm-project-overlay/lldb/BUILD.bazel
index 181fe3b7c3311..374c93600ccee 100644
--- a/utils/bazel/llvm-project-overlay/lldb/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/lldb/BUILD.bazel
@@ -92,7 +92,7 @@ cc_library(
     ],
     hdrs = ["include/lldb/Version/Version.h"],
     features = ["-layering_check"],  # Version.inc breaks this unintentionally
-    strip_include_prefix = "include",
+    includes = ["include"],
     deps = ["//clang:basic"],
 )
 
@@ -208,7 +208,7 @@ cc_library(
         "source/API/**/*.h",
     ]),
     hdrs = glob(["include/lldb/API/**/*.h"]) + [":lldb-sbapi-dwarf-enums"],
-    strip_include_prefix = "include",
+    includes = ["include"],
     deps = [
         ":Breakpoint",
         ":Commands",
@@ -254,7 +254,7 @@ cc_library(
     name = "Breakpoint",
     srcs = glob(["source/Breakpoint/**/*.cpp"]),
     hdrs = glob(["include/lldb/Breakpoint/**/*.h"]),
-    strip_include_prefix = "include",
+    includes = ["include"],
     deps = [
         ":Core",
         ":DataFormattersHeaders",
@@ -275,7 +275,7 @@ cc_library(
         "source/DataFormatters/**/*.h",
     ]),
     hdrs = glob(["include/lldb/DataFormatters/**/*.h"]),
-    strip_include_prefix = "include",
+    includes = ["include"],
     textual_hdrs = glob(["source/DataFormatters/**/*.def"]),
     deps = [
         ":CoreHeaders",
@@ -292,7 +292,7 @@ cc_library(
     name = "Expression",
     srcs = glob(["source/Expression/**/*.cpp"]),
     hdrs = glob(["include/lldb/Expression/**/*.h"]) + [":lldb-sbapi-dwarf-enums"],
-    strip_include_prefix = "include",
+    includes = ["include"],
     deps = [
         ":Core",
         ":Headers",
@@ -314,7 +314,7 @@ cc_library(
     name = "Initialization",
     srcs = glob(["source/Initialization/**/*.cpp"]),
     hdrs = glob(["include/lldb/Initialization/**/*.h"]),
-    strip_include_prefix = "include",
+    includes = ["include"],
     deps = [
         ":Core",
         ":Headers",
@@ -349,33 +349,33 @@ gentbl_cc_library(
 cc_library(
     name = "APIHeaders",
     hdrs = glob(["include/lldb/API/**/*.h"]),
-    strip_include_prefix = "include",
+    includes = ["include"],
 )
 
 cc_library(
     name = "InterpreterHeaders",
     hdrs = glob(["include/lldb/Interpreter/**/*.h"]),
-    strip_include_prefix = "include",
+    includes = ["include"],
     deps = [":APIHeaders"],
 )
 
 cc_library(
     name = "BreakpointHeaders",
     hdrs = glob(["include/lldb/Breakpoint/**/*.h"]),
-    strip_include_prefix = "include",
+    includes = ["include"],
 )
 
 cc_library(
     name = "ExpressionHeaders",
     hdrs = glob(["include/lldb/Expression/**/*.h"]) + [":lldb-sbapi-dwarf-enums"],
-    strip_include_prefix = "include",
+    includes = ["include"],
     deps = ["//llvm:ExecutionEngine"],
 )
 
 cc_library(
     name = "DataFormattersHeaders",
     hdrs = glob(["include/lldb/DataFormatters/**/*.h"]),
-    strip_include_prefix = "include",
+    includes = ["include"],
 )
 
 cc_library(
@@ -420,7 +420,7 @@ cc_library(
     name = "Commands",
     srcs = glob(["source/Commands/**/*.cpp"]),
     hdrs = glob(["source/Commands/**/*.h"]),
-    strip_include_prefix = "source",
+    includes = ["source"],
     deps = [
         ":Breakpoint",
         ":CommandOptions",
@@ -445,7 +445,7 @@ cc_library(
 cc_library(
     name = "SymbolHeaders",
     hdrs = glob(["include/lldb/Symbol/**/*.h"]),
-    strip_include_prefix = "include",
+    includes = ["include"],
 )
 
 cc_library(
@@ -473,7 +473,7 @@ cc_library(
         "include/lldb/Host/macosx/*.h",
         "include/lldb/Host/posix/*.h",
     ]),
-    strip_include_prefix = "include",
+    includes = ["include"],
     deps = [":UtilityHeaders"],
 )
 
@@ -483,7 +483,7 @@ cc_library(
         "source/Host/macosx/cfcpp/*.h",
         "source/Host/macosx/objcxx/*.h",
     ]),
-    strip_include_prefix = "source",
+    includes = ["source"],
     tags = ["nobuildkite"],
     target_compatible_with = select({
         "@platforms//os:macos": [],
@@ -545,6 +545,7 @@ cc_library(
             "include/lldb/Host/posix/*.h",
         ]),
     }),
+    includes = ["include"],
     # TODO: Move this to Config library when https://github.com/bazelbuild/bazel/issues/21884 is fixed
     linkopts = select({
         "@platforms//os:macos": [
@@ -566,7 +567,6 @@ cc_library(
         ],
         "//conditions:default": [],
     }),
-    strip_include_prefix = "include",
     deps = [
         ":Headers",
         ":UtilityHeaders",
@@ -615,7 +615,7 @@ cc_library(
         "include/lldb/Core/**/*.h",
         "include/lldb/ValueObject/**/*.h",  # This should be its own library.
     ]),
-    strip_include_prefix = "include",
+    includes = ["include"],
     deps = [
         ":BreakpointHeaders",
         ":CoreProperties",
@@ -634,15 +634,21 @@ cc_library(
 
 cc_library(
     name = "Core",
-    srcs = glob([
-        "source/Core/**/*.cpp",
-        "source/ValueObject/**/*.cpp",  # This should be its own library.
-    ]),
-    hdrs = glob([
-        "include/lldb/Core/**/*.h",
-        "include/lldb/ValueObject/**/*.h",  # This should be its own library.
-    ]),
-    strip_include_prefix = "include",
+    srcs = glob(
+        [
+            "source/Core/**/*.cpp",
+            "source/ValueObject/**/*.cpp",  # This should be its own library.
+        ],
+        exclude = ["source/Core/Telemetry.cpp"],
+    ),
+    hdrs = glob(
+        [
+            "include/lldb/Core/**/*.h",
+            "include/lldb/ValueObject/**/*.h",  # This should be its own library.
+        ],
+        exclude = ["include/lldb/Core/Telemetry.h"],
+    ),
+    includes = ["include"],
     deps = [
         ":BreakpointHeaders",
         ":CoreHeaders",
@@ -691,7 +697,7 @@ cc_library(
 cc_library(
     name = "TargetHeaders",
     hdrs = glob(["include/lldb/Target/**/*.h"]),
-    strip_include_prefix = "include",
+    includes = ["include"],
     deps = [
         ":AppleArm64ExceptionClass",
         ":BreakpointHeaders",
@@ -727,27 +733,27 @@ cc_library(
     hdrs = glob(["include/lldb/lldb-*.h"]) + [
         "include/lldb/Symbol/SaveCoreOptions.h",
     ],
-    strip_include_prefix = "include",
+    includes = ["include"],
 )
 
 cc_library(
     name = "UtilityPrivateHeaders",
     hdrs = glob(["source/Utility/**/*.h"]),
-    strip_include_prefix = "source",
+    includes = ["source"],
     deps = [":Headers"],
 )
 
 cc_library(
     name = "UtilityHeaders",
     hdrs = glob(["include/lldb/Utility/**/*.h"]),
-    strip_include_prefix = "include",
+    includes = ["include"],
 )
 
 cc_library(
     name = "Utility",
     srcs = glob(["source/Utility/**/*.cpp"]),
     hdrs = glob(["include/lldb/Utility/**/*.h"]),
-    strip_include_prefix = "include",
+    includes = ["include"],
     deps = [
         ":CoreHeaders",
         ":Headers",
diff --git a/utils/bazel/llvm-project-overlay/lldb/source/Plugins/BUILD.bazel b/utils/bazel/llvm-project-overlay/lldb/source/Plugins/BUILD.bazel
index ca997ed12be4c..c4775dffca260 100644
--- a/utils/bazel/llvm-project-overlay/lldb/source/Plugins/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/lldb/source/Plugins/BUILD.bazel
@@ -16,7 +16,7 @@ cc_library(
     name = "PluginClangCommon",
     srcs = glob(["Language/ClangCommon/*.cpp"]),
     hdrs = glob(["Language/ClangCommon/*.h"]),
-    include_prefix = "Plugins",
+    includes = [".."],
     deps = [
         "//clang:basic",
         "//clang:lex",
@@ -31,7 +31,7 @@ cc_library(
 cc_library(
     name = "PluginObjCLanguageHeaders",
     hdrs = glob(["Language/ObjC/*.h"]),
-    include_prefix = "Plugins",
+    includes = [".."],
     deps = [
         ":PluginClangCommon",
         ":PluginExpressionParserClangHeaders",
@@ -42,7 +42,7 @@ cc_library(
 cc_library(
     name = "PluginObjCLanguage",
     srcs = glob(["Language/ObjC/*.cpp"]),
-    include_prefix = "Plugins",
+    includes = [".."],
     deps = [
         ":PluginAppleObjCRuntime",
         ":PluginExpressionParserClangHeaders",
@@ -65,7 +65,7 @@ cc_library(
 cc_library(
     name = "PluginTypeSystemClangHeaders",
     hdrs = glob(["TypeSystem/Clang/*.h"]),
-    include_prefix = "Plugins",
+    includes = [".."],
     deps = [
         ":PluginExpressionParserClangHeaders",
         "//clang:frontend",
@@ -77,7 +77,7 @@ cc_library(
     name = "PluginCPPRuntime",
     srcs = glob(["LanguageRuntime/CPlusPlus/*.cpp"]),
     hdrs = glob(["LanguageRuntime/CPlusPlus/*.h"]),
-    include_prefix = "Plugins",
+    includes = [".."],
     deps = [
         "//lldb:CoreHeaders",
         "//lldb:Headers",
@@ -92,7 +92,7 @@ cc_library(
     name = "PluginObjCRuntime",
     srcs = glob(["LanguageRuntime/ObjC/*.cpp"]),
     hdrs = glob(["LanguageRuntime/ObjC/*.h"]),
-    include_prefix = "Plugins",
+    includes = [".."],
     deps = [
         ":PluginTypeSystemClangHeaders",
         "//clang:ast",
@@ -110,7 +110,7 @@ cc_library(
     name = "PluginAppleObjCRuntime",
     srcs = glob(["LanguageRuntime/ObjC/AppleObjCRuntime/*.cpp"]),
     hdrs = glob(["LanguageRuntime/ObjC/AppleObjCRuntime/*.h"]),
-    include_prefix = "Plugins",
+    includes = [".."],
     deps = [
         ":PluginCPPRuntime",
         ":PluginExpressionParserClangHeaders",
@@ -162,7 +162,7 @@ cc_library(
 cc_library(
     name = "PluginExpressionParserClangHeaders",
     hdrs = glob(["ExpressionParser/Clang/*.h"]),
-    include_prefix = "Plugins",
+    includes = [".."],
     deps = [
         "//lldb:CoreHeaders",
         "//lldb:DataFormattersHeaders",
@@ -172,7 +172,7 @@ cc_library(
 cc_library(
     name = "PluginExpressionParserClang",
     srcs = glob(["ExpressionParser/Clang/*.cpp"]),
-    include_prefix = "Plugins",
+    includes = [".."],
     deps = [
         ":PluginCPPRuntime",
         ":PluginCPlusPlusLanguageHeaders",
@@ -233,7 +233,7 @@ gentbl_cc_library(
 cc_library(
     name = "PluginPlatformMacOSXObjCXXHeaders",
     hdrs = glob(["Platform/MacOSX/objcxx/*.h"]),
-    include_prefix = "Plugins",
+    includes = [".."],
     tags = ["nobuildkite"],
     target_compatible_with = select({
         "@platforms//os:macos": [],
@@ -270,7 +270,7 @@ cc_library(
                "//conditions:default": [],
            }),
     hdrs = glob(["Platform/MacOSX/*.h"]),
-    include_prefix = "Plugins",
+    includes = [".."],
     tags = ["nobuildkite"],
     deps = [
         ":PlatformMacOSXProperties",
@@ -319,7 +319,7 @@ gentbl_cc_library(
 cc_library(
     name = "PluginSymbolFileDWARFHeaders",
     hdrs = glob(["SymbolFile/DWARF/*.h"]),
-    include_prefix = "Plugins",
+    includes = [".."],
     deps = [
         ":PluginTypeSystemClangHeaders",
         "//lldb:Core",
@@ -358,7 +358,7 @@ cc_library(
     name = "PluginProcessUtility",
     srcs = glob(["Process/Utility/*.cpp"]),
     hdrs = glob(["Process/Utility/*.h"]),
-    include_prefix = "Plugins",
+    includes = [".."],
     deps = [
         "//lldb:BreakpointHeaders",
         "//lldb:Core",
@@ -378,7 +378,7 @@ cc_library(
     name = "PluginObjectFilePDB",
     srcs = glob(["ObjectFile/PDB/*.cpp"]),
     hdrs = glob(["ObjectFile/PDB/*.h"]),
-    include_prefix = "Plugins",
+    includes = [".."],
     deps = [
         "//lldb:Core",
         "//lldb:SymbolHeaders",
@@ -392,7 +392,7 @@ cc_library(
 cc_library(
     name = "PluginSymbolFileNativePDBHeaders",
     hdrs = glob(["SymbolFile/NativePDB/*.h"]),
-    include_prefix = "Plugins",
+    includes = [".."],
     deps = ["//lldb:Core"],
 )
 
@@ -425,7 +425,7 @@ cc_library(
 cc_library(
     name = "PluginSymbolFilePDBHeaders",
     hdrs = glob(["SymbolFile/PDB/*.h"]),
-    include_prefix = "Plugins",
+    includes = [".."],
     deps = ["//lldb:Core"],
 )
 
@@ -475,7 +475,7 @@ cc_library(
     hdrs = glob(["Process/gdb-remote/*.h"]) + [
         "Process/gdb-remote/GDBRemoteErrno.def",
     ],
-    include_prefix = "Plugins",
+    includes = [".."],
     deps = [
         ":PluginProcessUtility",
         ":ProcessGDBRemoteProperties",
@@ -501,7 +501,7 @@ cc_library(
     name = "PluginObjectContainerMachOArchive",
     srcs = glob(["ObjectContainer/Universal-Mach-O/*.cpp"]),
     hdrs = glob(["ObjectContainer/Universal-Mach-O/*.h"]),
-    include_prefix = "Plugins",
+    includes = [".."],
     deps = [
         "//lldb:Core",
         "//lldb:Host",
@@ -517,7 +517,7 @@ cc_library(
     name = "PluginObjectContainerBSDArchive",
     srcs = glob(["ObjectContainer/BSD-Archive/*.cpp"]),
     hdrs = glob(["ObjectContainer/BSD-Archive/*.h"]),
-    include_prefix = "Plugins",
+    includes = [".."],
     deps = [
         "//lldb:Core",
         "//lldb:Host",
@@ -533,7 +533,7 @@ cc_library(
     name = "PluginObjectContainerMachOFileset",
     srcs = glob(["ObjectContainer/Mach-O-Fileset/*.cpp"]),
     hdrs = glob(["ObjectContainer/Mach-O-Fileset/*.h"]),
-    include_prefix = "Plugins",
+    includes = [".."],
     deps = [
         "//lldb:Core",
         "//lldb:Host",
@@ -567,7 +567,7 @@ cc_library(
     name = "PluginStructuredDataDarwinLog",
     srcs = glob(["StructuredData/DarwinLog/*.cpp"]),
     hdrs = glob(["StructuredData/DarwinLog/*.h"]),
-    include_prefix = "Plugins",
+    includes = [".."],
     deps = [
         ":StructuredDataDarwinLogProperties",
         "//lldb:Breakpoint",
@@ -585,7 +585,7 @@ cc_library(
     name = "PluginTraceCommon",
     srcs = glob(["Trace/common/*.cpp"]),
     hdrs = glob(["Trace/common/*.h"]),
-    include_prefix = "Plugins",
+    includes = [".."],
     deps = [
         ":PluginProcessUtility",
         "//lldb:Core",
@@ -598,7 +598,7 @@ cc_library(
     name = "PluginPlatformPOSIX",
     srcs = glob(["Platform/POSIX/*.cpp"]),
     hdrs = glob(["Platform/POSIX/*.h"]),
-    include_prefix = "Plugins",
+    includes = [".."],
     deps = [
         ":PluginPlatformGDB",
         ":PluginTypeSystemClang",
@@ -636,7 +636,7 @@ cc_library(
     name = "PluginPlatformQemuUser",
     srcs = glob(["Platform/QemuUser/*.cpp"]),
     hdrs = glob(["Platform/QemuUser/*.h"]),
-    include_prefix = "Plugins",
+    includes = [".."],
     deps = [
         ":PlatformQemuUserProperties",
         ":PluginProcessGDBRemote",
@@ -653,7 +653,7 @@ cc_library(
     name = "PluginPlatformGDB",
     srcs = glob(["Platform/gdb-server/*.cpp"]),
     hdrs = glob(["Platform/gdb-server/*.h"]),
-    include_prefix = "Plugins",
+    includes = [".."],
     deps = [
         ":PluginProcessGDBRemote",
         ":PluginProcessUtility",
@@ -671,7 +671,7 @@ cc_library(
     name = "PluginPlatformLinux",
     srcs = glob(["Platform/Linux/*.cpp"]),
     hdrs = glob(["Platform/Linux/*.h"]),
-    include_prefix = "Plugins",
+    includes = [".."],
     deps = [
         ":PluginPlatformPOSIX",
         ":PluginTypeSystemClangHeaders",
@@ -709,7 +709,7 @@ cc_library(
     name = "PluginPlatformAndroid",
     srcs = glob(["Platform/Android/*.cpp"]),
     hdrs = glob(["Platform/Android/*.h"]),
-    include_prefix = "Plugins",
+    includes = [".."],
     deps = [
         ":PlatformAndroidProperties",
         ":PluginPlatformGDB",
@@ -727,7 +727,7 @@ cc_library(
     name = "PluginPlatformWindows",
     srcs = glob(["Platform/Windows/*.cpp"]),
     hdrs = glob(["Platform/Windows/*.h"]),
-    include_prefix = "Plugins",
+    includes = [".."],
     deps = [
         ":PluginPlatformGDB",
         ":PluginTypeSystemClangHeaders",
@@ -746,7 +746,7 @@ cc_library(
     name = "PluginMemoryHistoryASan",
     srcs = glob(["MemoryHistory/asan/*.cpp"]),
     hdrs = glob(["MemoryHistory/asan/*.h"]),
-    include_prefix = "Plugins",
+    includes = [".."],
     deps = [
         ":PluginProcessUtility",
         "//lldb:Core",
@@ -761,7 +761,7 @@ cc_library(
     name = "PluginClangREPL",
     srcs = glob(["REPL/Clang/*.cpp"]),
     hdrs = glob(["REPL/Clang/*.h"]),
-    include_prefix = "Plugins",
+    includes = [".."],
     deps = [
         ":PluginCPPRuntime",
         ":PluginClangCommon",
@@ -780,7 +780,7 @@ cc_library(
     name = "PluginSymbolVendorWasm",
     srcs = glob(["SymbolVendor/wasm/*.cpp"]),
     hdrs = glob(["SymbolVendor/wasm/*.h"]),
-    include_prefix = "Plugins",
+    includes = [".."],
     deps = [
         ":PluginObjectFileWasm",
         "//lldb:Core",
@@ -797,7 +797,7 @@ cc_library(
     name = "PluginSymbolVendorMacOSX",
     srcs = glob(["SymbolVendor/MacOSX/*.cpp"]),
     hdrs = glob(["SymbolVendor/MacOSX/*.h"]),
-    include_prefix = "Plugins",
+    includes = [".."],
     deps = [
         ":PluginObjectFileMachO",
         "//lldb:Core",
@@ -814,7 +814,7 @@ cc_library(
     name = "PluginSymbolVendorPECOFF",
     srcs = glob(["SymbolVendor/PECOFF/*.cpp"]),
     hdrs = glob(["SymbolVendor/PECOFF/*.h"]),
-    include_prefix = "Plugins",
+    includes = [".."],
     deps = [
         ":PluginObjectFilePECOFF",
         "//lldb:Core",
@@ -831,7 +831,7 @@ cc_library(
     name = "PluginSymbolVendorELF",
     srcs = glob(["SymbolVendor/ELF/*.cpp"]),
     hdrs = glob(["SymbolVendor/ELF/*.h"]),
-    include_prefix = "Plugins",
+    includes = [".."],
     deps = [
         ":PluginObjectFileELF",
         "//lldb:Core",
@@ -848,7 +848,7 @@ cc_library(
     name = "PluginObjCPlusPlusLanguage",
     srcs = glob(["Language/ObjCPlusPlus/*.cpp"]),
     hdrs = glob(["Language/ObjCPlusPlus/*.h"]),
-    include_prefix = "Plugins",
+    includes = [".."],
     deps = [
         ":PluginClangCommon",
         "//lldb:Core",
@@ -862,7 +862,7 @@ cc_library(
 cc_library(
     name = "PluginCPlusPlusLanguageHeaders",
     hdrs = glob(["Language/CPlusPlus/*.h"]),
-    include_prefix = "Plugins",
+    includes = [".."],
     deps = [
         ":PluginClangCommon",
         ":PluginExpressionParserClangHeaders",
@@ -873,7 +873,7 @@ cc_library(
 cc_library(
     name = "PluginCPlusPlusLanguage",
     srcs = glob(["Language/CPlusPlus/*.cpp"]),
-    include_prefix = "Plugins",
+    includes = [".."],
     deps = [
         ":PluginCPPRuntime",
         ":PluginCPlusPlusLanguageHeaders",
@@ -915,7 +915,7 @@ cc_library(
     name = "PluginTraceExporterCTF",
     srcs = glob(["TraceExporter/ctf/*.cpp"]),
     hdrs = glob(["TraceExporter/ctf/*.h"]),
-    include_prefix = "Plugins",
+    includes = [".."],
     deps = [
         ":PluginTraceExporterCommon",
         ":TraceExporterCTFOptions",
@@ -932,7 +932,7 @@ cc_library(
     name = "PluginTraceExporterCommon",
     srcs = glob(["TraceExporter/common/*.cpp"]),
     hdrs = glob(["TraceExporter/common/*.h"]),
-    include_prefix = "Plugins",
+    includes = [".."],
     deps = [
         "//lldb:Core",
         "//lldb:SymbolHeaders",
@@ -946,7 +946,7 @@ cc_library(
     name = "PluginABIPowerPC",
     srcs = glob(["ABI/PowerPC/*.cpp"]),
     hdrs = glob(["ABI/PowerPC/*.h"]),
-    include_prefix = "Plugins",
+    includes = [".."],
     deps = [
         ":PluginProcessUtility",
         ":PluginTypeSystemClang",
@@ -969,7 +969,7 @@ cc_library(
     name = "PluginABIHexagon",
     srcs = glob(["ABI/Hexagon/*.cpp"]),
     hdrs = glob(["ABI/Hexagon/*.h"]),
-    include_prefix = "Plugins",
+    includes = [".."],
     deps = [
         "//lldb:Core",
         "//lldb:Headers",
@@ -987,7 +987,7 @@ cc_library(
     name = "PluginABIMips",
     srcs = glob(["ABI/Mips/*.cpp"]),
     hdrs = glob(["ABI/Mips/*.h"]),
-    include_prefix = "Plugins",
+    includes = [".."],
     deps = [
         "//lldb:Core",
         "//lldb:Headers",
@@ -1005,7 +1005,7 @@ cc_library(
     name = "PluginABIMSP430",
     srcs = glob(["ABI/MSP430/*.cpp"]),
     hdrs = glob(["ABI/MSP430/*.h"]),
-    include_prefix = "Plugins",
+    includes = [".."],
     deps = [
         "//lldb:Core",
         "//lldb:Headers",
@@ -1023,7 +1023,7 @@ cc_library(
     name = "PluginABIX86",
     srcs = glob(["ABI/X86/*.cpp"]),
     hdrs = glob(["ABI/X86/*.h"]),
-    include_prefix = "Plugins",
+    includes = [".."],
     deps = [
         "//lldb:Core",
         "//lldb:Headers",
@@ -1041,7 +1041,7 @@ cc_library(
     name = "PluginABIARM",
     srcs = glob(["ABI/ARM/*.cpp"]),
     hdrs = glob(["ABI/ARM/*.h"]),
-    include_prefix = "Plugins",
+    includes = [".."],
     deps = [
         ":PluginProcessUtility",
         "//lldb:Core",
@@ -1061,7 +1061,7 @@ cc_library(
     name = "PluginABIARC",
     srcs = glob(["ABI/ARC/*.cpp"]),
     hdrs = glob(["ABI/ARC/*.h"]),
-    include_prefix = "Plugins",
+    includes = [".."],
     deps = [
         ":PluginProcessUtility",
         "//lldb:Core",
@@ -1081,7 +1081,7 @@ cc_library(
     name = "PluginABIRISCV",
     srcs = glob(["ABI/RISCV/*.cpp"]),
     hdrs = glob(["ABI/RISCV/*.h"]),
-    include_prefix = "Plugins",
+    includes = [".."],
     deps = [
         ":PluginProcessUtility",
         "//lldb:Core",
@@ -1101,7 +1101,7 @@ cc_library(
     name = "PluginABISystemZ",
     srcs = glob(["ABI/SystemZ/*.cpp"]),
     hdrs = glob(["ABI/SystemZ/*.h"]),
-    include_prefix = "Plugins",
+    includes = [".."],
     deps = [
         "//lldb:Core",
         "//lldb:Headers",
@@ -1119,7 +1119,7 @@ cc_library(
     name = "PluginABIAArch64",
     srcs = glob(["ABI/AArch64/*.cpp"]),
     hdrs = glob(["ABI/AArch64/*.h"]),
-    include_prefix = "Plugins",
+    includes = [".."],
     deps = [
         "//lldb:Core",
         "//lldb:Headers",
@@ -1137,13 +1137,13 @@ cc_library(
 cc_library(
     name = "PluginDynamicLoaderPosixDYLDHeaders",
     hdrs = glob(["DynamicLoader/POSIX-DYLD/*.h"]),
-    include_prefix = "Plugins",
+    includes = [".."],
 )
 
 cc_library(
     name = "PluginDynamicLoaderPosixDYLD",
     srcs = glob(["DynamicLoader/POSIX-DYLD/*.cpp"]),
-    include_prefix = "Plugins",
+    includes = [".."],
     deps = [
         ":PluginDynamicLoaderPosixDYLDHeaders",
         ":PluginProcessElfCore",
@@ -1164,7 +1164,7 @@ cc_library(
     name = "PluginDynamicLoaderWindowsDYLD",
     srcs = glob(["DynamicLoader/Windows-DYLD/*.cpp"]),
     hdrs = glob(["DynamicLoader/Windows-DYLD/*.h"]),
-    include_prefix = "Plugins",
+    includes = [".."],
     deps = [
         "//lldb:Core",
         "//lldb:Headers",
@@ -1179,7 +1179,7 @@ cc_library(
     name = "PluginDynamicLoaderHexagonDYLD",
     srcs = glob(["DynamicLoader/Hexagon-DYLD/*.cpp"]),
     hdrs = glob(["DynamicLoader/Hexagon-DYLD/*.h"]),
-    include_prefix = "Plugins",
+    includes = [".."],
     deps = [
         "//lldb:Breakpoint",
         "//lldb:Core",
@@ -1196,7 +1196,7 @@ cc_library(
     name = "PluginDynamicLoaderWasmDYLD",
     srcs = glob(["DynamicLoader/wasm-DYLD/*.cpp"]),
     hdrs = glob(["DynamicLoader/wasm-DYLD/*.h"]),
-    include_prefix = "Plugins",
+    includes = [".."],
     deps = [
         ":PluginObjectFileWasm",
         "//lldb:Core",
@@ -1210,7 +1210,7 @@ cc_library(
     name = "PluginDynamicLoaderStatic",
     srcs = glob(["DynamicLoader/Static/*.cpp"]),
     hdrs = glob(["DynamicLoader/Static/*.h"]),
-    include_prefix = "Plugins",
+    includes = [".."],
     deps = [
         "//lldb:Core",
         "//lldb:Host",
@@ -1244,7 +1244,7 @@ cc_library(
     name = "PluginDynamicLoaderMacOSXDYLD",
     srcs = glob(["DynamicLoader/MacOSX-DYLD/*.cpp"]),
     hdrs = glob(["DynamicLoader/MacOSX-DYLD/*.h"]),
-    include_prefix = "Plugins",
+    includes = [".."],
     deps = [
         ":DynamicLoaderMacOSXDYLDProperties",
         ":PluginObjCRuntime",
@@ -1285,13 +1285,13 @@ gentbl_cc_library(
 cc_library(
     name = "PluginDynamicLoaderDarwinKernelHeaders",
     hdrs = glob(["DynamicLoader/Darwin-Kernel/*.h"]),
-    include_prefix = "Plugins",
+    includes = [".."],
 )
 
 cc_library(
     name = "PluginDynamicLoaderDarwinKernel",
     srcs = glob(["DynamicLoader/Darwin-Kernel/*.cpp"]),
-    include_prefix = "Plugins",
+    includes = [".."],
     tags = ["nobuildkite"],
     deps = [
         ":DynamicLoaderDarwinKernelProperties",
@@ -1314,7 +1314,7 @@ cc_library(
     name = "PluginArchitecturePPC64",
     srcs = glob(["Architecture/PPC64/*.cpp"]),
     hdrs = glob(["Architecture/PPC64/*.h"]),
-    include_prefix = "Plugins",
+    includes = [".."],
     deps = [
         ":PluginProcessUtility",
         "//lldb:Core",
@@ -1330,7 +1330,7 @@ cc_library(
     name = "PluginArchitectureMips",
     srcs = glob(["Architecture/Mips/*.cpp"]),
     hdrs = glob(["Architecture/Mips/*.h"]),
-    include_prefix = "Plugins",
+    includes = [".."],
     deps = [
         "//lldb:Core",
         "//lldb:SymbolHeaders",
@@ -1344,7 +1344,7 @@ cc_library(
     name = "PluginArchitectureArm",
     srcs = glob(["Architecture/Arm/*.cpp"]),
     hdrs = glob(["Architecture/Arm/*.h"]),
-    include_prefix = "Plugins",
+    includes = [".."],
     deps = [
         ":PluginProcessUtility",
         "//lldb:Core",
@@ -1358,7 +1358,7 @@ cc_library(
     name = "PluginArchitectureAArch64",
     srcs = glob(["Architecture/AArch64/*.cpp"]),
     hdrs = glob(["Architecture/AArch64/*.h"]),
-    include_prefix = "Plugins",
+    includes = [".."],
     deps = [
         ":PluginProcessUtility",
         "//lldb:Core",
@@ -1372,7 +1372,7 @@ cc_library(
     name = "PluginDisassemblerLLVMC",
     srcs = glob(["Disassembler/LLVMC/*.cpp"]),
     hdrs = glob(["Disassembler/LLVMC/*.h"]),
-    include_prefix = "Plugins",
+    includes = [".."],
     deps = [
         "//lldb:Core",
         "//lldb:Symbol",
@@ -1391,7 +1391,7 @@ cc_library(
     name = "PluginSymbolFileSymtab",
     srcs = glob(["SymbolFile/Symtab/*.cpp"]),
     hdrs = glob(["SymbolFile/Symtab/*.h"]),
-    include_prefix = "Plugins",
+    includes = [".."],
     deps = [
         "//lldb:Core",
         "//lldb:Symbol",
@@ -1404,7 +1404,7 @@ cc_library(
     name = "PluginSymbolFileCTF",
     srcs = glob(["SymbolFile/CTF/*.cpp"]),
     hdrs = glob(["SymbolFile/CTF/*.h"]),
-    include_prefix = "Plugins",
+    includes = [".."],
     deps = [
         ":PluginExpressionParserClangHeaders",
         ":PluginTypeSystemClangHeaders",
@@ -1424,7 +1424,7 @@ cc_library(
     name = "PluginSymbolFileJSON",
     srcs = glob(["SymbolFile/JSON/*.cpp"]),
     hdrs = glob(["SymbolFile/JSON/*.h"]),
-    include_prefix = "Plugins",
+    includes = [".."],
     deps = [
         ":PluginObjectFileJSON",
         "//lldb:Core",
@@ -1439,7 +1439,7 @@ cc_library(
     name = "PluginSymbolFileBreakpad",
     srcs = glob(["SymbolFile/Breakpad/*.cpp"]),
     hdrs = glob(["SymbolFile/Breakpad/*.h"]),
-    include_prefix = "Plugins",
+    includes = [".."],
     deps = [
         ":PluginObjectFileBreakpad",
         "//lldb:Core",
@@ -1455,7 +1455,7 @@ cc_library(
     name = "PluginInstructionPPC64",
     srcs = glob(["Instruction/PPC64/*.cpp"]),
     hdrs = glob(["Instruction/PPC64/*.h"]),
-    include_prefix = "Plugins",
+    includes = [".."],
     deps = [
         ":PluginProcessUtility",
         "//lldb:Core",
@@ -1470,7 +1470,7 @@ cc_library(
     name = "PluginInstructionLoongArch",
     srcs = glob(["Instruction/LoongArch/*.cpp"]),
     hdrs = glob(["Instruction/LoongArch/*.h"]),
-    include_prefix = "Plugins",
+    includes = [".."],
     deps = [
         ":PluginProcessUtility",
         "//lldb:Core",
@@ -1486,7 +1486,7 @@ cc_library(
     name = "PluginInstructionMIPS",
     srcs = glob(["Instruction/MIPS/*.cpp"]),
     hdrs = glob(["Instruction/MIPS/*.h"]),
-    include_prefix = "Plugins",
+    includes = [".."],
     deps = [
         ":PluginProcessUtility",
         "//lldb:Core",
@@ -1505,7 +1505,7 @@ cc_library(
     name = "PluginInstructionMIPS64",
     srcs = glob(["Instruction/MIPS64/*.cpp"]),
     hdrs = glob(["Instruction/MIPS64/*.h"]),
-    include_prefix = "Plugins",
+    includes = [".."],
     deps = [
         ":PluginProcessUtility",
         "//lldb:Core",
@@ -1524,7 +1524,7 @@ cc_library(
     name = "PluginInstructionARM",
     srcs = glob(["Instruction/ARM/*.cpp"]),
     hdrs = glob(["Instruction/ARM/*.h"]),
-    include_prefix = "Plugins",
+    includes = [".."],
     deps = [
         ":PluginProcessUtility",
         "//lldb:Core",
@@ -1544,7 +1544,7 @@ cc_library(
     name = "PluginInstructionRISCV",
     srcs = glob(["Instruction/RISCV/*.cpp"]),
     hdrs = glob(["Instruction/RISCV/*.h"]),
-    include_prefix = "Plugins",
+    includes = [".."],
     deps = [
         ":PluginProcessUtility",
         "//lldb:Core",
@@ -1560,7 +1560,7 @@ cc_library(
     name = "PluginInstructionARM64",
     srcs = glob(["Instruction/ARM64/*.cpp"]),
     hdrs = glob(["Instruction/ARM64/*.h"]),
-    include_prefix = "Plugins",
+    includes = [".."],
     deps = [
         ":PluginProcessUtility",
         "//lldb:Core",
@@ -1576,7 +1576,7 @@ cc_library(
     name = "PluginInstrumentationRuntimeASanLibsanitizers",
     srcs = glob(["InstrumentationRuntime/ASanLibsanitizers/*.cpp"]),
     hdrs = glob(["InstrumentationRuntime/ASanLibsanitizers/*.h"]),
-    include_prefix = "Plugins",
+    includes = [".."],
     deps = [
         ":PluginInstrumentationRuntimeUtility",
         "//lldb:Breakpoint",
@@ -1593,7 +1593,7 @@ cc_library(
     name = "PluginInstrumentationRuntimeTSan",
     srcs = glob(["InstrumentationRuntime/TSan/*.cpp"]),
     hdrs = glob(["InstrumentationRuntime/TSan/*.h"]),
-    include_prefix = "Plugins",
+    includes = [".."],
     deps = [
         ":PluginProcessUtility",
         "//lldb:Breakpoint",
@@ -1614,7 +1614,7 @@ cc_library(
     name = "PluginInstrumentationRuntimeASan",
     srcs = glob(["InstrumentationRuntime/ASan/*.cpp"]),
     hdrs = glob(["InstrumentationRuntime/ASan/*.h"]),
-    include_prefix = "Plugins",
+    includes = [".."],
     deps = [
         ":PluginInstrumentationRuntimeUtility",
         "//lldb:Breakpoint",
@@ -1631,7 +1631,7 @@ cc_library(
     name = "PluginInstrumentationRuntimeMainThreadChecker",
     srcs = glob(["InstrumentationRuntime/MainThreadChecker/*.cpp"]),
     hdrs = glob(["InstrumentationRuntime/MainThreadChecker/*.h"]),
-    include_prefix = "Plugins",
+    includes = [".."],
     deps = [
         ":PluginProcessUtility",
         "//lldb:Breakpoint",
@@ -1651,7 +1651,7 @@ cc_library(
     name = "PluginInstrumentationRuntimeUBSan",
     srcs = glob(["InstrumentationRuntime/UBSan/*.cpp"]),
     hdrs = glob(["InstrumentationRuntime/UBSan/*.h"]),
-    include_prefix = "Plugins",
+    includes = [".."],
     deps = [
         ":PluginProcessUtility",
         "//lldb:Breakpoint",
@@ -1672,7 +1672,7 @@ cc_library(
     name = "PluginInstrumentationRuntimeUtility",
     srcs = glob(["InstrumentationRuntime/Utility/*.cpp"]),
     hdrs = glob(["InstrumentationRuntime/Utility/*.h"]),
-    include_prefix = "Plugins",
+    includes = [".."],
     deps = [
         "//lldb:Breakpoint",
         "//lldb:Core",
@@ -1705,7 +1705,7 @@ cc_library(
     name = "PluginJITLoaderGDB",
     srcs = glob(["JITLoader/GDB/*.cpp"]),
     hdrs = glob(["JITLoader/GDB/*.h"]),
-    include_prefix = "Plugins",
+    includes = [".."],
     deps = [
         ":JITLoaderGDBProperties",
         ":PluginObjectFileMachO",
@@ -1725,7 +1725,7 @@ cc_library(
     name = "PluginSymbolLocatorDefault",
     srcs = glob(["SymbolLocator/Default/*.cpp"]),
     hdrs = glob(["SymbolLocator/Default/*.h"]),
-    include_prefix = "Plugins",
+    includes = [".."],
     deps = [
         ":PluginObjectFileWasm",
         "//lldb:Core",
@@ -1761,7 +1761,7 @@ cc_library(
     name = "PluginSymbolLocatorDebuginfod",
     srcs = glob(["SymbolLocator/Debuginfod/*.cpp"]),
     hdrs = glob(["SymbolLocator/Debuginfod/*.h"]),
-    include_prefix = "Plugins",
+    includes = [".."],
     deps = [
         ":SymbolLocatorDebuginfodProperties",
         "//lldb:Core",
@@ -1779,7 +1779,7 @@ cc_library(
     name = "PluginSymbolLocatorDebugSymbols",
     srcs = glob(["SymbolLocator/DebugSymbols/*.cpp"]),
     hdrs = glob(["SymbolLocator/DebugSymbols/*.h"]),
-    include_prefix = "Plugins",
+    includes = [".."],
     tags = ["nobuildkite"],
     deps = [
         ":PluginObjectFileWasm",
@@ -1799,7 +1799,7 @@ cc_library(
 #     name = "PluginOperatingSystemPython",
 #     srcs = glob(["OperatingSystem/Python/*.cpp"]),
 #     hdrs = glob(["OperatingSystem/Python/*.h"]),
-#     include_prefix = "Plugins",
+#     includes = [".."],
 #     deps = [
 #         "//lldb:Core",
 #         "//lldb:Interpreter",
@@ -1812,7 +1812,7 @@ cc_library(
 #     name = "PluginScriptInterpreterPythonInterfaces",
 #     srcs = glob(["ScriptInterpreter/Python/Interfaces/*.cpp"]),
 #     hdrs = glob(["ScriptInterpreter/Python/Interfaces/*.h"]),
-#     include_prefix = "Plugins",
+#     includes = [".."],
 #     deps = [
 #         "//lldb:Core",
 #         "//lldb:Host",
@@ -1825,7 +1825,7 @@ cc_library(
 # cc_library(
 #     name = "PluginScriptInterpreterPythonHeaders",
 #     hdrs = glob(["ScriptInterpreter/Python/*.h"]),
-#     include_prefix = "Plugins",
+#     includes = [".."],
 #     deps = [
 #         "//lldb:Host",
 #     ],
@@ -1838,7 +1838,7 @@ cc_library(
 #         # Must be kept in sync with WORKSPACE python version
 #         'LLDB_PYTHON_RELATIVE_LIBDIR=\\"lib/python3.11/site-packages\\"',
 #     ],
-#     include_prefix = "Plugins",
+#     includes = [".."],
 #     deps = [
 #         "//lldb:Breakpoint",
 #         "//lldb:Core",
@@ -1856,7 +1856,7 @@ cc_library(
 #     name = "PluginScriptInterpreterLua",
 #     srcs = glob(["ScriptInterpreter/Lua/*.cpp"]),
 #     hdrs = glob(["ScriptInterpreter/Lua/*.h"]),
-#     include_prefix = "Plugins",
+#     includes = [".."],
 #     deps = [
 #         "//lldb:Core",
 #         "//lldb:Interpreter",
@@ -1867,7 +1867,7 @@ cc_library(
     name = "PluginScriptInterpreterNone",
     srcs = glob(["ScriptInterpreter/None/*.cpp"]),
     hdrs = glob(["ScriptInterpreter/None/*.h"]),
-    include_prefix = "Plugins",
+    includes = [".."],
     deps = [
         "//lldb:Core",
         "//lldb:InterpreterHeaders",
@@ -1880,7 +1880,7 @@ cc_library(
     name = "PluginSystemRuntimeMacOSX",
     srcs = glob(["SystemRuntime/MacOSX/*.cpp"]),
     hdrs = glob(["SystemRuntime/MacOSX/*.h"]),
-    include_prefix = "Plugins",
+    includes = [".."],
     deps = [
         ":PluginProcessUtility",
         ":PluginTypeSystemClang",
@@ -1902,7 +1902,7 @@ cc_library(
     name = "PluginObjectFileCOFF",
     srcs = glob(["ObjectFile/COFF/*.cpp"]),
     hdrs = glob(["ObjectFile/COFF/*.h"]),
-    include_prefix = "Plugins",
+    includes = [".."],
     deps = [
         "//lldb:Core",
         "//lldb:Host",
@@ -1919,7 +1919,7 @@ cc_library(
     name = "PluginObjectFileWasm",
     srcs = glob(["ObjectFile/wasm/*.cpp"]),
     hdrs = glob(["ObjectFile/wasm/*.h"]),
-    include_prefix = "Plugins",
+    includes = [".."],
     deps = [
         "//lldb:Core",
         "//lldb:Host",
@@ -1936,7 +1936,7 @@ cc_library(
     name = "PluginObjectFileJSON",
     srcs = glob(["ObjectFile/JSON/*.cpp"]),
     hdrs = glob(["ObjectFile/JSON/*.h"]),
-    include_prefix = "Plugins",
+    includes = [".."],
     deps = [
         "//lldb:Core",
         "//lldb:Host",
@@ -1951,7 +1951,7 @@ cc_library(
     name = "PluginObjectFilePlaceholder",
     srcs = glob(["ObjectFile/Placeholder/*.cpp"]),
     hdrs = glob(["ObjectFile/Placeholder/*.h"]),
-    include_prefix = "Plugins",
+    includes = [".."],
     deps = [
         "//lldb:Core",
         "//lldb:Headers",
@@ -1968,7 +1968,7 @@ cc_library(
     name = "PluginObjectFileMachO",
     srcs = glob(["ObjectFile/Mach-O/*.cpp"]),
     hdrs = glob(["ObjectFile/Mach-O/*.h"]),
-    include_prefix = "Plugins",
+    includes = [".."],
     deps = [
         ":PluginProcessUtility",
         "//lldb:Core",
@@ -1986,7 +1986,7 @@ cc_library(
     name = "PluginObjectFileMinidump",
     srcs = glob(["ObjectFile/Minidump/*.cpp"]),
     hdrs = glob(["ObjectFile/Minidump/*.h"]),
-    include_prefix = "Plugins",
+    includes = [".."],
     deps = [
         ":PluginProcessMinidump",
         ":PluginProcessUtility",
@@ -2027,7 +2027,7 @@ cc_library(
     name = "PluginObjectFilePECOFF",
     srcs = glob(["ObjectFile/PECOFF/*.cpp"]),
     hdrs = glob(["ObjectFile/PECOFF/*.h"]),
-    include_prefix = "Plugins",
+    includes = [".."],
     deps = [
         ":ObjectFilePECOFFProperties",
         ":PluginProcessUtility",
@@ -2050,7 +2050,7 @@ cc_library(
     name = "PluginObjectFileBreakpad",
     srcs = glob(["ObjectFile/Breakpad/*.cpp"]),
     hdrs = glob(["ObjectFile/Breakpad/*.h"]),
-    include_prefix = "Plugins",
+    includes = [".."],
     deps = [
         "//lldb:Core",
         "//lldb:Headers",
@@ -2067,7 +2067,7 @@ cc_library(
     name = "PluginObjectFileELF",
     srcs = glob(["ObjectFile/ELF/*.cpp"]),
     hdrs = glob(["ObjectFile/ELF/*.h"]),
-    include_prefix = "Plugins",
+    includes = [".."],
     deps = [
         "//lldb:Core",
         "//lldb:Headers",
@@ -2087,7 +2087,7 @@ cc_library(
     name = "PluginUnwindAssemblyX86",
     srcs = glob(["UnwindAssembly/x86/*.cpp"]),
     hdrs = glob(["UnwindAssembly/x86/*.h"]),
-    include_prefix = "Plugins",
+    includes = [".."],
     deps = [
         "//lldb:Core",
         "//lldb:Headers",
@@ -2105,7 +2105,7 @@ cc_library(
     name = "PluginUnwindAssemblyInstEmulation",
     srcs = glob(["UnwindAssembly/InstEmulation/*.cpp"]),
     hdrs = glob(["UnwindAssembly/InstEmulation/*.h"]),
-    include_prefix = "Plugins",
+    includes = [".."],
     deps = [
         "//lldb:Core",
         "//lldb:Headers",
@@ -2121,7 +2121,7 @@ cc_library(
     name = "PluginProcessPOSIX",
     srcs = glob(["Process/POSIX/*.cpp"]),
     hdrs = glob(["Process/POSIX/*.h"]),
-    include_prefix = "Plugins",
+    includes = [".."],
     deps = [
         ":PluginProcessUtility",
         "//lldb:Headers",
@@ -2137,7 +2137,7 @@ cc_library(
     name = "PluginProcessLinux",
     srcs = glob(["Process/Linux/*.cpp"]),
     hdrs = glob(["Process/Linux/*.h"]),
-    include_prefix = "Plugins",
+    includes = [".."],
     target_compatible_with = select({
         "@platforms//os:linux": [],
         "//conditions:default": ["@platforms//:incompatible"],
@@ -2160,7 +2160,7 @@ cc_library(
     name = "PluginScriptedProcess",
     srcs = glob(["Process/scripted/*.cpp"]),
     hdrs = glob(["Process/scripted/*.h"]),
-    include_prefix = "Plugins",
+    includes = [".."],
     deps = [
         ":PluginProcessUtility",
         "//lldb:Core",
@@ -2176,7 +2176,7 @@ cc_library(
     name = "PluginProcessMachCore",
     srcs = glob(["Process/mach-core/*.cpp"]),
     hdrs = glob(["Process/mach-core/*.h"]),
-    include_prefix = "Plugins",
+    includes = [".."],
     tags = ["nobuildkite"],
     deps = [
         ":PluginDynamicLoaderDarwinKernelHeaders",
@@ -2201,7 +2201,7 @@ cc_library(
     name = "PluginProcessElfCore",
     srcs = glob(["Process/elf-core/*.cpp"]),
     hdrs = glob(["Process/elf-core/*.h"]),
-    include_prefix = "Plugins",
+    includes = [".."],
     deps = [
         ":PluginDynamicLoaderPosixDYLDHeaders",
         ":PluginObjectFileELF",
@@ -2237,7 +2237,7 @@ cc_library(
     name = "PluginProcessMacOSXKernel",
     srcs = glob(["Process/MacOSX-Kernel/*.cpp"]),
     hdrs = glob(["Process/MacOSX-Kernel/*.h"]),
-    include_prefix = "Plugins",
+    includes = [".."],
     tags = ["nobuildkite"],
     target_compatible_with = select({
         "@platforms//os:macos": [],
@@ -2267,7 +2267,7 @@ cc_library(
     name = "PluginProcessMinidump",
     srcs = glob(["Process/minidump/*.cpp"]),
     hdrs = glob(["Process/minidump/*.h"]),
-    include_prefix = "Plugins",
+    includes = [".."],
     deps = [
         ":PluginDynamicLoaderPosixDYLD",
         ":PluginDynamicLoaderPosixDYLDHeaders",
@@ -2291,7 +2291,7 @@ cc_library(
     name = "PluginCXXItaniumABI",
     srcs = glob(["LanguageRuntime/CPlusPlus/ItaniumABI/*.cpp"]),
     hdrs = glob(["LanguageRuntime/CPlusPlus/ItaniumABI/*.h"]),
-    include_prefix = "Plugins",
+    includes = [".."],
     deps = [
         ":PluginCPPRuntime",
         ":PluginTypeSystemClang",
@@ -2314,7 +2314,7 @@ cc_library(
     name = "PluginGNUstepObjCRuntime",
     srcs = glob(["LanguageRuntime/ObjC/GNUstepObjCRuntime/*.cpp"]),
     hdrs = glob(["LanguageRuntime/ObjC/GNUstepObjCRuntime/*.h"]),
-    include_prefix = "Plugins",
+    includes = [".."],
     deps = [
         ":PluginExpressionParserClang",
         ":PluginObjCRuntime",
@@ -2337,7 +2337,7 @@ cc_library(
     name = "PluginRegisterTypeBuilderClang",
     srcs = glob(["RegisterTypeBuilder/*.cpp"]),
     hdrs = glob(["RegisterTypeBuilder/*.h"]),
-    include_prefix = "Plugins",
+    includes = [".."],
     deps = [
         ":PluginTypeSystemClangHeaders",
         "//clang:ast",
@@ -2375,5 +2375,5 @@ LLDB_PLUGIN(SymbolVendorMacOSX)
 cc_library(
     name = "PluginsConfig",
     hdrs = [":plugins_config_gen"],
-    include_prefix = "Plugins",
+    includes = [".."],
 )
diff --git a/utils/bazel/llvm-project-overlay/llvm/BUILD.bazel b/utils/bazel/llvm-project-overlay/llvm/BUILD.bazel
index a54d464ac81d3..dfed324a9cb49 100644
--- a/utils/bazel/llvm-project-overlay/llvm/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/llvm/BUILD.bazel
@@ -5,13 +5,12 @@
 load("@bazel_skylib//rules:common_settings.bzl", "string_flag")
 load("@bazel_skylib//rules:expand_template.bzl", "expand_template")
 load("@rules_python//python:defs.bzl", "py_binary")
-load("//mlir:tblgen.bzl", "td_library")
+load("//mlir:tblgen.bzl", "gentbl_cc_library", "gentbl_filegroup", "td_library")
 load(":binary_alias.bzl", "binary_alias")
 load(":config.bzl", "llvm_config_defines")
 load(":driver.bzl", "generate_driver_selects", "generate_driver_tools_def", "llvm_driver_cc_binary", "select_driver_tools")
 load(":enum_targets_gen.bzl", "enum_targets_gen")
 load(":targets.bzl", "llvm_targets")
-load(":tblgen.bzl", "gentbl")
 
 package(
     default_visibility = ["//visibility:public"],
@@ -424,9 +423,8 @@ cc_library(
 
 cc_library(
     name = "DebugInfo",
-    hdrs = glob(["include/llvm/DebugInfo/**/*.h"]),
+    hdrs = glob(["include/llvm/DebugInfo/*.h"]),
     copts = llvm_copts,
-    textual_hdrs = glob(["include/llvm/DebugInfo/**/*.def"]),
     deps = [
         ":Object",
         ":Support",
@@ -607,6 +605,7 @@ cc_library(
     deps = [
         ":BinaryFormat",
         ":DebugInfo",
+        ":DebugInfoBTF",
         ":DebugInfoDWARF",
         ":DebugInfoPDB",
         ":Demangle",
@@ -649,14 +648,14 @@ cc_binary(
         "utils/TableGen/Basic/Attributes.cpp",
         "utils/TableGen/Basic/CodeGenIntrinsics.cpp",
         "utils/TableGen/Basic/CodeGenIntrinsics.h",
-        "utils/TableGen/Basic/SDNodeProperties.cpp",
-        "utils/TableGen/Basic/SDNodeProperties.h",
-        "utils/TableGen/Basic/TableGen.h",
-        "utils/TableGen/Basic/TableGen.cpp",
-        "utils/TableGen/Basic/SequenceToOffsetTable.h",
         "utils/TableGen/Basic/DirectiveEmitter.cpp",
         "utils/TableGen/Basic/IntrinsicEmitter.cpp",
         "utils/TableGen/Basic/RISCVTargetDefEmitter.cpp",
+        "utils/TableGen/Basic/SDNodeProperties.cpp",
+        "utils/TableGen/Basic/SDNodeProperties.h",
+        "utils/TableGen/Basic/SequenceToOffsetTable.h",
+        "utils/TableGen/Basic/TableGen.cpp",
+        "utils/TableGen/Basic/TableGen.h",
         "utils/TableGen/Basic/VTEmitter.cpp",
         "utils/TableGen/llvm-min-tblgen.cpp",
     ],
@@ -739,36 +738,36 @@ cc_binary(
     ],
 )
 
-gentbl(
+gentbl_cc_library(
     name = "intrinsic_enums_gen",
-    tbl_outs = [("-gen-intrinsic-enums", "include/llvm/IR/IntrinsicEnums.inc")],
+    tbl_outs = [(
+        ["-gen-intrinsic-enums"],
+        "include/llvm/IR/IntrinsicEnums.inc",
+    )],
     tblgen = ":llvm-min-tblgen",
     td_file = "include/llvm/IR/Intrinsics.td",
-    td_srcs = glob([
-        "include/llvm/CodeGen/*.td",
-        "include/llvm/IR/Intrinsics*.td",
-    ]),
+    deps = [":CommonTargetTdFiles"],
 )
 
-gentbl(
+gentbl_cc_library(
     name = "intrinsics_impl_gen",
-    tbl_outs = [("-gen-intrinsic-impl", "include/llvm/IR/IntrinsicImpl.inc")],
+    tbl_outs = [(
+        ["-gen-intrinsic-impl"],
+        "include/llvm/IR/IntrinsicImpl.inc",
+    )],
     tblgen = ":llvm-min-tblgen",
     td_file = "include/llvm/IR/Intrinsics.td",
-    td_srcs = glob([
-        "include/llvm/CodeGen/*.td",
-        "include/llvm/IR/Intrinsics*.td",
-    ]),
+    deps = [":CommonTargetTdFiles"],
 )
 
-gentbl(
+gentbl_cc_library(
     name = "vt_gen",
-    tbl_outs = [("-gen-vt", "include/llvm/CodeGen/GenVT.inc")],
+    tbl_outs = [(
+        ["-gen-vt"],
+        "include/llvm/CodeGen/GenVT.inc",
+    )],
     tblgen = ":llvm-min-tblgen",
     td_file = "include/llvm/CodeGen/ValueTypes.td",
-    td_srcs = [
-        "include/llvm/CodeGen/ValueTypes.td",
-    ],
 )
 
 # Note that the intrinsics are not currently set up so they can be pruned for
@@ -849,27 +848,33 @@ llvm_target_intrinsics_list = [
 ]
 
 [[
-    gentbl(
+    gentbl_cc_library(
         name = "intrinsic_" + target["name"] + "_gen",
+        includes = ["include"],
         tbl_outs = [(
-            "-gen-intrinsic-enums -intrinsic-prefix=" + target["intrinsic_prefix"],
+            [
+                "-gen-intrinsic-enums",
+                "-intrinsic-prefix=" + target["intrinsic_prefix"],
+            ],
             "include/llvm/IR/Intrinsics" + target["name"] + ".h",
         )],
         tblgen = ":llvm-min-tblgen",
         td_file = "include/llvm/IR/Intrinsics.td",
-        td_srcs = glob([
-            "include/llvm/CodeGen/*.td",
-            "include/llvm/IR/*.td",
-        ]),
+        deps = [
+            ":CodegenTdFiles",
+            ":IRTdFiles",
+        ],
     ),
 ] for target in llvm_target_intrinsics_list]
 
-gentbl(
+gentbl_cc_library(
     name = "attributes_gen",
-    tbl_outs = [("-gen-attrs", "include/llvm/IR/Attributes.inc")],
+    tbl_outs = [(
+        ["-gen-attrs"],
+        "include/llvm/IR/Attributes.inc",
+    )],
     tblgen = ":llvm-min-tblgen",
     td_file = "include/llvm/IR/Attributes.td",
-    td_srcs = ["include/llvm/IR/Attributes.td"],
 )
 
 cc_library(
@@ -1220,6 +1225,7 @@ cc_library(
     copts = llvm_copts,
     deps = [
         ":Analysis",
+        ":BinaryFormat",
         ":BitReader",
         ":BitstreamWriter",
         ":Core",
@@ -1267,46 +1273,85 @@ filegroup(
     ]),
 )
 
+td_library(
+    name = "CodegenTdFiles",
+    srcs = glob(["include/llvm/CodeGen/*.td"]),
+    includes = ["include"],
+)
+
+td_library(
+    name = "IRTdFiles",
+    srcs = glob(["include/llvm/IR/*.td"]),
+    includes = ["include"],
+)
+
 td_library(
     name = "CommonTargetTdFiles",
     srcs = [":common_target_td_sources"],
     includes = ["include"],
 )
 
-gentbl(
+td_library(
+    name = "ArmTargetTdFiles",
+    srcs = glob([
+        "lib/Target/ARM/**/*.td",
+    ]),
+)
+
+gentbl_cc_library(
     name = "ARMTargetParserDefGen",
-    tbl_outs = [("-gen-arm-target-def", "include/llvm/TargetParser/ARMTargetParserDef.inc")],
+    tbl_outs = [(
+        ["-gen-arm-target-def"],
+        "include/llvm/TargetParser/ARMTargetParserDef.inc",
+    )],
     tblgen = ":llvm-min-tblgen",
     td_file = "lib/Target/ARM/ARM.td",
-    td_srcs = [
-        ":common_target_td_sources",
-    ] + glob([
-        "lib/Target/ARM/**/*.td",
+    deps = [
+        ":ArmTargetTdFiles",
+        ":CommonTargetTdFiles",
+    ],
+)
+
+td_library(
+    name = "AArch64TargetTdFiles",
+    srcs = glob([
+        "lib/Target/AArch64/**/*.td",
     ]),
 )
 
-gentbl(
+gentbl_cc_library(
     name = "AArch64TargetParserDefGen",
-    tbl_outs = [("-gen-arm-target-def", "include/llvm/TargetParser/AArch64TargetParserDef.inc")],
+    tbl_outs = [(
+        ["-gen-arm-target-def"],
+        "include/llvm/TargetParser/AArch64TargetParserDef.inc",
+    )],
     tblgen = ":llvm-min-tblgen",
     td_file = "lib/Target/AArch64/AArch64.td",
-    td_srcs = [
-        ":common_target_td_sources",
-    ] + glob([
-        "lib/Target/AArch64/**/*.td",
+    deps = [
+        ":AArch64TargetTdFiles",
+        ":CommonTargetTdFiles",
+    ],
+)
+
+td_library(
+    name = "RISCVTargetTdFiles",
+    srcs = glob([
+        "lib/Target/RISCV/**/*.td",
     ]),
 )
 
-gentbl(
+gentbl_cc_library(
     name = "RISCVTargetParserDefGen",
-    tbl_outs = [("-gen-riscv-target-def", "include/llvm/TargetParser/RISCVTargetParserDef.inc")],
+    tbl_outs = [(
+        ["-gen-riscv-target-def"],
+        "include/llvm/TargetParser/RISCVTargetParserDef.inc",
+    )],
     tblgen = ":llvm-min-tblgen",
     td_file = "lib/Target/RISCV/RISCV.td",
-    td_srcs = [
-        ":common_target_td_sources",
-    ] + glob([
-        "lib/Target/RISCV/**/*.td",
-    ]),
+    deps = [
+        ":CommonTargetTdFiles",
+        ":RISCVTargetTdFiles",
+    ],
 )
 
 cc_library(
@@ -1379,19 +1424,25 @@ cc_library(
     ],
 )
 
-gentbl(
+td_library(
+    name = "AMDGPUTargetTdFiles",
+    srcs = glob([
+        "lib/Target/AMDGPU/**/*.td",
+    ]),
+)
+
+gentbl_cc_library(
     name = "InstCombineTableGen",
     strip_include_prefix = "lib/Target/AMDGPU",
     tbl_outs = [(
-        "-gen-searchable-tables",
+        ["-gen-searchable-tables"],
         "lib/Target/AMDGPU/InstCombineTables.inc",
     )],
     tblgen = ":llvm-tblgen",
     td_file = "lib/Target/AMDGPU/InstCombineTables.td",
-    td_srcs = glob([
-        "lib/Target/AMDGPU/*.td",
-    ]) + [
-        ":common_target_td_sources",
+    deps = [
+        ":AMDGPUTargetTdFiles",
+        ":CommonTargetTdFiles",
     ],
 )
 
@@ -1593,40 +1644,46 @@ cc_library(
         ":BinaryFormat",
         ":Core",
         ":Object",
+        ":ObjectYAML",
         ":Support",
         ":TargetParser",
         ":TransformUtils",
     ],
 )
 
-filegroup(
-    name = "omp_td_files",
+td_library(
+    name = "OmpTdFiles",
     srcs = glob([
         "include/llvm/Frontend/OpenMP/*.td",
         "include/llvm/Frontend/Directive/*.td",
     ]),
+    includes = ["include"],
 )
 
-gentbl(
+gentbl_filegroup(
     name = "omp_gen",
-    library = False,
     tbl_outs = [
-        ("--gen-directive-decl", "include/llvm/Frontend/OpenMP/OMP.h.inc"),
+        (
+            ["--gen-directive-decl"],
+            "include/llvm/Frontend/OpenMP/OMP.h.inc",
+        ),
     ],
     tblgen = ":llvm-min-tblgen",
     td_file = "include/llvm/Frontend/OpenMP/OMP.td",
-    td_srcs = [":omp_td_files"],
+    deps = [":OmpTdFiles"],
 )
 
-gentbl(
+gentbl_filegroup(
     name = "omp_gen_impl",
-    library = False,
     tbl_outs = [
-        ("--gen-directive-impl", "include/llvm/Frontend/OpenMP/OMP.inc"),
+        (
+            ["--gen-directive-impl"],
+            "include/llvm/Frontend/OpenMP/OMP.inc",
+        ),
     ],
     tblgen = ":llvm-min-tblgen",
     td_file = "include/llvm/Frontend/OpenMP/OMP.td",
-    td_srcs = [":omp_td_files"],
+    deps = [":OmpTdFiles"],
 )
 
 cc_library(
@@ -1675,34 +1732,39 @@ cc_library(
     ],
 )
 
-filegroup(
-    name = "acc_td_files",
+td_library(
+    name = "AccTdFiles",
     srcs = glob([
         "include/llvm/Frontend/OpenACC/*.td",
         "include/llvm/Frontend/Directive/*.td",
     ]),
+    includes = ["include"],
 )
 
-gentbl(
+gentbl_filegroup(
     name = "acc_gen",
-    library = False,
     tbl_outs = [
-        ("--gen-directive-decl", "include/llvm/Frontend/OpenACC/ACC.h.inc"),
+        (
+            ["--gen-directive-decl"],
+            "include/llvm/Frontend/OpenACC/ACC.h.inc",
+        ),
     ],
     tblgen = ":llvm-min-tblgen",
     td_file = "include/llvm/Frontend/OpenACC/ACC.td",
-    td_srcs = [":acc_td_files"],
+    deps = [":AccTdFiles"],
 )
 
-gentbl(
+gentbl_filegroup(
     name = "acc_gen_impl",
-    library = False,
     tbl_outs = [
-        ("--gen-directive-impl", "include/llvm/Frontend/OpenACC/ACC.inc"),
+        (
+            ["--gen-directive-impl"],
+            "include/llvm/Frontend/OpenACC/ACC.inc",
+        ),
     ],
     tblgen = ":llvm-min-tblgen",
     td_file = "include/llvm/Frontend/OpenACC/ACC.td",
-    td_srcs = [":acc_td_files"],
+    deps = [":AccTdFiles"],
 )
 
 cc_library(
@@ -2019,64 +2081,217 @@ llvm_target_lib_list = [lib for lib in [
         "name": "AArch64",
         "short_name": "AArch64",
         "tbl_outs": [
-            ("-gen-register-bank", "lib/Target/AArch64/AArch64GenRegisterBank.inc"),
-            ("-gen-register-info", "lib/Target/AArch64/AArch64GenRegisterInfo.inc"),
-            ("-gen-instr-info", "lib/Target/AArch64/AArch64GenInstrInfo.inc"),
-            ("-gen-emitter", "lib/Target/AArch64/AArch64GenMCCodeEmitter.inc"),
-            ("-gen-pseudo-lowering", "lib/Target/AArch64/AArch64GenMCPseudoLowering.inc"),
-            ("-gen-asm-writer", "lib/Target/AArch64/AArch64GenAsmWriter.inc"),
-            ("-gen-asm-writer -asmwriternum=1", "lib/Target/AArch64/AArch64GenAsmWriter1.inc"),
-            ("-gen-asm-matcher", "lib/Target/AArch64/AArch64GenAsmMatcher.inc"),
-            ("-gen-dag-isel", "lib/Target/AArch64/AArch64GenDAGISel.inc"),
-            ("-gen-fast-isel", "lib/Target/AArch64/AArch64GenFastISel.inc"),
-            ("-gen-global-isel", "lib/Target/AArch64/AArch64GenGlobalISel.inc"),
-            ("-gen-global-isel-combiner -combiners=AArch64O0PreLegalizerCombiner", "lib/Target/AArch64/AArch64GenO0PreLegalizeGICombiner.inc"),
-            ("-gen-global-isel-combiner -combiners=AArch64PreLegalizerCombiner", "lib/Target/AArch64/AArch64GenPreLegalizeGICombiner.inc"),
-            ("-gen-global-isel-combiner -combiners=AArch64PostLegalizerCombiner", "lib/Target/AArch64/AArch64GenPostLegalizeGICombiner.inc"),
-            ("-gen-global-isel-combiner -combiners=AArch64PostLegalizerLowering", "lib/Target/AArch64/AArch64GenPostLegalizeGILowering.inc"),
-            ("-gen-callingconv", "lib/Target/AArch64/AArch64GenCallingConv.inc"),
-            ("-gen-subtarget", "lib/Target/AArch64/AArch64GenSubtargetInfo.inc"),
-            ("-gen-disassembler", "lib/Target/AArch64/AArch64GenDisassemblerTables.inc"),
-            ("-gen-searchable-tables", "lib/Target/AArch64/AArch64GenSystemOperands.inc"),
-            ("-gen-exegesis", "lib/Target/AArch64/AArch64GenExegesis.inc"),
+            (
+                ["-gen-register-bank"],
+                "lib/Target/AArch64/AArch64GenRegisterBank.inc",
+            ),
+            (
+                ["-gen-register-info"],
+                "lib/Target/AArch64/AArch64GenRegisterInfo.inc",
+            ),
+            (
+                ["-gen-instr-info"],
+                "lib/Target/AArch64/AArch64GenInstrInfo.inc",
+            ),
+            (
+                ["-gen-emitter"],
+                "lib/Target/AArch64/AArch64GenMCCodeEmitter.inc",
+            ),
+            (
+                ["-gen-pseudo-lowering"],
+                "lib/Target/AArch64/AArch64GenMCPseudoLowering.inc",
+            ),
+            (
+                ["-gen-asm-writer"],
+                "lib/Target/AArch64/AArch64GenAsmWriter.inc",
+            ),
+            (
+                [
+                    "-gen-asm-writer",
+                    "-asmwriternum=1",
+                ],
+                "lib/Target/AArch64/AArch64GenAsmWriter1.inc",
+            ),
+            (
+                ["-gen-asm-matcher"],
+                "lib/Target/AArch64/AArch64GenAsmMatcher.inc",
+            ),
+            (
+                ["-gen-dag-isel"],
+                "lib/Target/AArch64/AArch64GenDAGISel.inc",
+            ),
+            (
+                ["-gen-fast-isel"],
+                "lib/Target/AArch64/AArch64GenFastISel.inc",
+            ),
+            (
+                ["-gen-global-isel"],
+                "lib/Target/AArch64/AArch64GenGlobalISel.inc",
+            ),
+            (
+                [
+                    "-gen-global-isel-combiner",
+                    "-combiners=AArch64O0PreLegalizerCombiner",
+                ],
+                "lib/Target/AArch64/AArch64GenO0PreLegalizeGICombiner.inc",
+            ),
+            (
+                [
+                    "-gen-global-isel-combiner",
+                    "-combiners=AArch64PreLegalizerCombiner",
+                ],
+                "lib/Target/AArch64/AArch64GenPreLegalizeGICombiner.inc",
+            ),
+            (
+                [
+                    "-gen-global-isel-combiner",
+                    "-combiners=AArch64PostLegalizerCombiner",
+                ],
+                "lib/Target/AArch64/AArch64GenPostLegalizeGICombiner.inc",
+            ),
+            (
+                [
+                    "-gen-global-isel-combiner",
+                    "-combiners=AArch64PostLegalizerLowering",
+                ],
+                "lib/Target/AArch64/AArch64GenPostLegalizeGILowering.inc",
+            ),
+            (
+                ["-gen-callingconv"],
+                "lib/Target/AArch64/AArch64GenCallingConv.inc",
+            ),
+            (
+                ["-gen-subtarget"],
+                "lib/Target/AArch64/AArch64GenSubtargetInfo.inc",
+            ),
+            (
+                ["-gen-disassembler"],
+                "lib/Target/AArch64/AArch64GenDisassemblerTables.inc",
+            ),
+            (
+                ["-gen-searchable-tables"],
+                "lib/Target/AArch64/AArch64GenSystemOperands.inc",
+            ),
+            (
+                ["-gen-exegesis"],
+                "lib/Target/AArch64/AArch64GenExegesis.inc",
+            ),
         ],
     },
     {
         "name": "ARM",
         "short_name": "ARM",
         "tbl_outs": [
-            ("-gen-register-bank", "lib/Target/ARM/ARMGenRegisterBank.inc"),
-            ("-gen-register-info", "lib/Target/ARM/ARMGenRegisterInfo.inc"),
-            ("-gen-searchable-tables", "lib/Target/ARM/ARMGenSystemRegister.inc"),
-            ("-gen-instr-info", "lib/Target/ARM/ARMGenInstrInfo.inc"),
-            ("-gen-emitter", "lib/Target/ARM/ARMGenMCCodeEmitter.inc"),
-            ("-gen-pseudo-lowering", "lib/Target/ARM/ARMGenMCPseudoLowering.inc"),
-            ("-gen-asm-writer", "lib/Target/ARM/ARMGenAsmWriter.inc"),
-            ("-gen-asm-matcher", "lib/Target/ARM/ARMGenAsmMatcher.inc"),
-            ("-gen-dag-isel", "lib/Target/ARM/ARMGenDAGISel.inc"),
-            ("-gen-fast-isel", "lib/Target/ARM/ARMGenFastISel.inc"),
-            ("-gen-global-isel", "lib/Target/ARM/ARMGenGlobalISel.inc"),
-            ("-gen-callingconv", "lib/Target/ARM/ARMGenCallingConv.inc"),
-            ("-gen-subtarget", "lib/Target/ARM/ARMGenSubtargetInfo.inc"),
-            ("-gen-disassembler", "lib/Target/ARM/ARMGenDisassemblerTables.inc"),
+            (
+                ["-gen-register-bank"],
+                "lib/Target/ARM/ARMGenRegisterBank.inc",
+            ),
+            (
+                ["-gen-register-info"],
+                "lib/Target/ARM/ARMGenRegisterInfo.inc",
+            ),
+            (
+                ["-gen-searchable-tables"],
+                "lib/Target/ARM/ARMGenSystemRegister.inc",
+            ),
+            (
+                ["-gen-instr-info"],
+                "lib/Target/ARM/ARMGenInstrInfo.inc",
+            ),
+            (
+                ["-gen-emitter"],
+                "lib/Target/ARM/ARMGenMCCodeEmitter.inc",
+            ),
+            (
+                ["-gen-pseudo-lowering"],
+                "lib/Target/ARM/ARMGenMCPseudoLowering.inc",
+            ),
+            (
+                ["-gen-asm-writer"],
+                "lib/Target/ARM/ARMGenAsmWriter.inc",
+            ),
+            (
+                ["-gen-asm-matcher"],
+                "lib/Target/ARM/ARMGenAsmMatcher.inc",
+            ),
+            (
+                ["-gen-dag-isel"],
+                "lib/Target/ARM/ARMGenDAGISel.inc",
+            ),
+            (
+                ["-gen-fast-isel"],
+                "lib/Target/ARM/ARMGenFastISel.inc",
+            ),
+            (
+                ["-gen-global-isel"],
+                "lib/Target/ARM/ARMGenGlobalISel.inc",
+            ),
+            (
+                ["-gen-callingconv"],
+                "lib/Target/ARM/ARMGenCallingConv.inc",
+            ),
+            (
+                ["-gen-subtarget"],
+                "lib/Target/ARM/ARMGenSubtargetInfo.inc",
+            ),
+            (
+                ["-gen-disassembler"],
+                "lib/Target/ARM/ARMGenDisassemblerTables.inc",
+            ),
         ],
     },
     {
         "name": "AMDGPU",
         "short_name": "AMDGPU",
         "tbl_outs": [
-            ("-gen-register-bank", "lib/Target/AMDGPU/AMDGPUGenRegisterBank.inc"),
-            ("-gen-register-info", "lib/Target/AMDGPU/AMDGPUGenRegisterInfo.inc"),
-            ("-gen-instr-info", "lib/Target/AMDGPU/AMDGPUGenInstrInfo.inc"),
-            ("-gen-emitter", "lib/Target/AMDGPU/AMDGPUGenMCCodeEmitter.inc"),
-            ("-gen-pseudo-lowering", "lib/Target/AMDGPU/AMDGPUGenMCPseudoLowering.inc"),
-            ("-gen-asm-writer", "lib/Target/AMDGPU/AMDGPUGenAsmWriter.inc"),
-            ("-gen-asm-matcher", "lib/Target/AMDGPU/AMDGPUGenAsmMatcher.inc"),
-            ("-gen-dag-isel", "lib/Target/AMDGPU/AMDGPUGenDAGISel.inc"),
-            ("-gen-callingconv", "lib/Target/AMDGPU/AMDGPUGenCallingConv.inc"),
-            ("-gen-subtarget", "lib/Target/AMDGPU/AMDGPUGenSubtargetInfo.inc"),
-            ("-gen-disassembler", "lib/Target/AMDGPU/AMDGPUGenDisassemblerTables.inc"),
-            ("-gen-searchable-tables", "lib/Target/AMDGPU/AMDGPUGenSearchableTables.inc"),
+            (
+                ["-gen-register-bank"],
+                "lib/Target/AMDGPU/AMDGPUGenRegisterBank.inc",
+            ),
+            (
+                ["-gen-register-info"],
+                "lib/Target/AMDGPU/AMDGPUGenRegisterInfo.inc",
+            ),
+            (
+                ["-gen-instr-info"],
+                "lib/Target/AMDGPU/AMDGPUGenInstrInfo.inc",
+            ),
+            (
+                ["-gen-emitter"],
+                "lib/Target/AMDGPU/AMDGPUGenMCCodeEmitter.inc",
+            ),
+            (
+                ["-gen-pseudo-lowering"],
+                "lib/Target/AMDGPU/AMDGPUGenMCPseudoLowering.inc",
+            ),
+            (
+                ["-gen-asm-writer"],
+                "lib/Target/AMDGPU/AMDGPUGenAsmWriter.inc",
+            ),
+            (
+                ["-gen-asm-matcher"],
+                "lib/Target/AMDGPU/AMDGPUGenAsmMatcher.inc",
+            ),
+            (
+                ["-gen-dag-isel"],
+                "lib/Target/AMDGPU/AMDGPUGenDAGISel.inc",
+            ),
+            (
+                ["-gen-callingconv"],
+                "lib/Target/AMDGPU/AMDGPUGenCallingConv.inc",
+            ),
+            (
+                ["-gen-subtarget"],
+                "lib/Target/AMDGPU/AMDGPUGenSubtargetInfo.inc",
+            ),
+            (
+                ["-gen-disassembler"],
+                "lib/Target/AMDGPU/AMDGPUGenDisassemblerTables.inc",
+            ),
+            (
+                ["-gen-searchable-tables"],
+                "lib/Target/AMDGPU/AMDGPUGenSearchableTables.inc",
+            ),
         ],
         "tbl_deps": [
             ":InstCombineTableGen",
@@ -2088,164 +2303,479 @@ llvm_target_lib_list = [lib for lib in [
         "name": "AVR",
         "short_name": "AVR",
         "tbl_outs": [
-            ("-gen-asm-matcher", "lib/Target/AVR/AVRGenAsmMatcher.inc"),
-            ("-gen-asm-writer", "lib/Target/AVR/AVRGenAsmWriter.inc"),
-            ("-gen-callingconv", "lib/Target/AVR/AVRGenCallingConv.inc"),
-            ("-gen-dag-isel", "lib/Target/AVR/AVRGenDAGISel.inc"),
-            ("-gen-disassembler", "lib/Target/AVR/AVRGenDisassemblerTables.inc"),
-            ("-gen-emitter", "lib/Target/AVR/AVRGenMCCodeEmitter.inc"),
-            ("-gen-instr-info", "lib/Target/AVR/AVRGenInstrInfo.inc"),
-            ("-gen-register-info", "lib/Target/AVR/AVRGenRegisterInfo.inc"),
-            ("-gen-subtarget", "lib/Target/AVR/AVRGenSubtargetInfo.inc"),
+            (
+                ["-gen-asm-matcher"],
+                "lib/Target/AVR/AVRGenAsmMatcher.inc",
+            ),
+            (
+                ["-gen-asm-writer"],
+                "lib/Target/AVR/AVRGenAsmWriter.inc",
+            ),
+            (
+                ["-gen-callingconv"],
+                "lib/Target/AVR/AVRGenCallingConv.inc",
+            ),
+            (
+                ["-gen-dag-isel"],
+                "lib/Target/AVR/AVRGenDAGISel.inc",
+            ),
+            (
+                ["-gen-disassembler"],
+                "lib/Target/AVR/AVRGenDisassemblerTables.inc",
+            ),
+            (
+                ["-gen-emitter"],
+                "lib/Target/AVR/AVRGenMCCodeEmitter.inc",
+            ),
+            (
+                ["-gen-instr-info"],
+                "lib/Target/AVR/AVRGenInstrInfo.inc",
+            ),
+            (
+                ["-gen-register-info"],
+                "lib/Target/AVR/AVRGenRegisterInfo.inc",
+            ),
+            (
+                ["-gen-subtarget"],
+                "lib/Target/AVR/AVRGenSubtargetInfo.inc",
+            ),
         ],
     },
     {
         "name": "BPF",
         "short_name": "BPF",
         "tbl_outs": [
-            ("-gen-register-bank", "lib/Target/BPF/BPFGenRegisterBank.inc"),
-            ("-gen-asm-writer", "lib/Target/BPF/BPFGenAsmWriter.inc"),
-            ("-gen-asm-matcher", "lib/Target/BPF/BPFGenAsmMatcher.inc"),
-            ("-gen-callingconv", "lib/Target/BPF/BPFGenCallingConv.inc"),
-            ("-gen-dag-isel", "lib/Target/BPF/BPFGenDAGISel.inc"),
-            ("-gen-global-isel", "lib/Target/BPF/BPFGenGlobalISel.inc"),
-            ("-gen-disassembler", "lib/Target/BPF/BPFGenDisassemblerTables.inc"),
-            ("-gen-emitter", "lib/Target/BPF/BPFGenMCCodeEmitter.inc"),
-            ("-gen-instr-info", "lib/Target/BPF/BPFGenInstrInfo.inc"),
-            ("-gen-register-info", "lib/Target/BPF/BPFGenRegisterInfo.inc"),
-            ("-gen-subtarget", "lib/Target/BPF/BPFGenSubtargetInfo.inc"),
+            (
+                ["-gen-register-bank"],
+                "lib/Target/BPF/BPFGenRegisterBank.inc",
+            ),
+            (
+                ["-gen-asm-writer"],
+                "lib/Target/BPF/BPFGenAsmWriter.inc",
+            ),
+            (
+                ["-gen-asm-matcher"],
+                "lib/Target/BPF/BPFGenAsmMatcher.inc",
+            ),
+            (
+                ["-gen-callingconv"],
+                "lib/Target/BPF/BPFGenCallingConv.inc",
+            ),
+            (
+                ["-gen-dag-isel"],
+                "lib/Target/BPF/BPFGenDAGISel.inc",
+            ),
+            (
+                ["-gen-global-isel"],
+                "lib/Target/BPF/BPFGenGlobalISel.inc",
+            ),
+            (
+                ["-gen-disassembler"],
+                "lib/Target/BPF/BPFGenDisassemblerTables.inc",
+            ),
+            (
+                ["-gen-emitter"],
+                "lib/Target/BPF/BPFGenMCCodeEmitter.inc",
+            ),
+            (
+                ["-gen-instr-info"],
+                "lib/Target/BPF/BPFGenInstrInfo.inc",
+            ),
+            (
+                ["-gen-register-info"],
+                "lib/Target/BPF/BPFGenRegisterInfo.inc",
+            ),
+            (
+                ["-gen-subtarget"],
+                "lib/Target/BPF/BPFGenSubtargetInfo.inc",
+            ),
         ],
     },
     {
         "name": "Hexagon",
         "short_name": "Hexagon",
         "tbl_outs": [
-            ("-gen-asm-matcher", "lib/Target/Hexagon/HexagonGenAsmMatcher.inc"),
-            ("-gen-asm-writer", "lib/Target/Hexagon/HexagonGenAsmWriter.inc"),
-            ("-gen-callingconv", "lib/Target/Hexagon/HexagonGenCallingConv.inc"),
-            ("-gen-dag-isel", "lib/Target/Hexagon/HexagonGenDAGISel.inc"),
-            ("-gen-dfa-packetizer", "lib/Target/Hexagon/HexagonGenDFAPacketizer.inc"),
-            ("-gen-disassembler", "lib/Target/Hexagon/HexagonGenDisassemblerTables.inc"),
-            ("-gen-instr-info", "lib/Target/Hexagon/HexagonGenInstrInfo.inc"),
-            ("-gen-emitter", "lib/Target/Hexagon/HexagonGenMCCodeEmitter.inc"),
-            ("-gen-register-info", "lib/Target/Hexagon/HexagonGenRegisterInfo.inc"),
-            ("-gen-subtarget", "lib/Target/Hexagon/HexagonGenSubtargetInfo.inc"),
+            (
+                ["-gen-asm-matcher"],
+                "lib/Target/Hexagon/HexagonGenAsmMatcher.inc",
+            ),
+            (
+                ["-gen-asm-writer"],
+                "lib/Target/Hexagon/HexagonGenAsmWriter.inc",
+            ),
+            (
+                ["-gen-callingconv"],
+                "lib/Target/Hexagon/HexagonGenCallingConv.inc",
+            ),
+            (
+                ["-gen-dag-isel"],
+                "lib/Target/Hexagon/HexagonGenDAGISel.inc",
+            ),
+            (
+                ["-gen-dfa-packetizer"],
+                "lib/Target/Hexagon/HexagonGenDFAPacketizer.inc",
+            ),
+            (
+                ["-gen-disassembler"],
+                "lib/Target/Hexagon/HexagonGenDisassemblerTables.inc",
+            ),
+            (
+                ["-gen-instr-info"],
+                "lib/Target/Hexagon/HexagonGenInstrInfo.inc",
+            ),
+            (
+                ["-gen-emitter"],
+                "lib/Target/Hexagon/HexagonGenMCCodeEmitter.inc",
+            ),
+            (
+                ["-gen-register-info"],
+                "lib/Target/Hexagon/HexagonGenRegisterInfo.inc",
+            ),
+            (
+                ["-gen-subtarget"],
+                "lib/Target/Hexagon/HexagonGenSubtargetInfo.inc",
+            ),
         ],
     },
     {
         "name": "Lanai",
         "short_name": "Lanai",
         "tbl_outs": [
-            ("-gen-asm-matcher", "lib/Target/Lanai/LanaiGenAsmMatcher.inc"),
-            ("-gen-asm-writer", "lib/Target/Lanai/LanaiGenAsmWriter.inc"),
-            ("-gen-callingconv", "lib/Target/Lanai/LanaiGenCallingConv.inc"),
-            ("-gen-dag-isel", "lib/Target/Lanai/LanaiGenDAGISel.inc"),
-            ("-gen-disassembler", "lib/Target/Lanai/LanaiGenDisassemblerTables.inc"),
-            ("-gen-emitter", "lib/Target/Lanai/LanaiGenMCCodeEmitter.inc"),
-            ("-gen-instr-info", "lib/Target/Lanai/LanaiGenInstrInfo.inc"),
-            ("-gen-register-info", "lib/Target/Lanai/LanaiGenRegisterInfo.inc"),
-            ("-gen-subtarget", "lib/Target/Lanai/LanaiGenSubtargetInfo.inc"),
+            (
+                ["-gen-asm-matcher"],
+                "lib/Target/Lanai/LanaiGenAsmMatcher.inc",
+            ),
+            (
+                ["-gen-asm-writer"],
+                "lib/Target/Lanai/LanaiGenAsmWriter.inc",
+            ),
+            (
+                ["-gen-callingconv"],
+                "lib/Target/Lanai/LanaiGenCallingConv.inc",
+            ),
+            (
+                ["-gen-dag-isel"],
+                "lib/Target/Lanai/LanaiGenDAGISel.inc",
+            ),
+            (
+                ["-gen-disassembler"],
+                "lib/Target/Lanai/LanaiGenDisassemblerTables.inc",
+            ),
+            (
+                ["-gen-emitter"],
+                "lib/Target/Lanai/LanaiGenMCCodeEmitter.inc",
+            ),
+            (
+                ["-gen-instr-info"],
+                "lib/Target/Lanai/LanaiGenInstrInfo.inc",
+            ),
+            (
+                ["-gen-register-info"],
+                "lib/Target/Lanai/LanaiGenRegisterInfo.inc",
+            ),
+            (
+                ["-gen-subtarget"],
+                "lib/Target/Lanai/LanaiGenSubtargetInfo.inc",
+            ),
         ],
     },
     {
         "name": "LoongArch",
         "short_name": "LoongArch",
         "tbl_outs": [
-            ("-gen-asm-matcher", "lib/Target/LoongArch/LoongArchGenAsmMatcher.inc"),
-            ("-gen-asm-writer", "lib/Target/LoongArch/LoongArchGenAsmWriter.inc"),
-            ("-gen-dag-isel", "lib/Target/LoongArch/LoongArchGenDAGISel.inc"),
-            ("-gen-disassembler", "lib/Target/LoongArch/LoongArchGenDisassemblerTables.inc"),
-            ("-gen-emitter", "lib/Target/LoongArch/LoongArchGenMCCodeEmitter.inc"),
-            ("-gen-instr-info", "lib/Target/LoongArch/LoongArchGenInstrInfo.inc"),
-            ("-gen-pseudo-lowering", "lib/Target/LoongArch/LoongArchGenMCPseudoLowering.inc"),
-            ("-gen-register-info", "lib/Target/LoongArch/LoongArchGenRegisterInfo.inc"),
-            ("-gen-subtarget", "lib/Target/LoongArch/LoongArchGenSubtargetInfo.inc"),
+            (
+                ["-gen-asm-matcher"],
+                "lib/Target/LoongArch/LoongArchGenAsmMatcher.inc",
+            ),
+            (
+                ["-gen-asm-writer"],
+                "lib/Target/LoongArch/LoongArchGenAsmWriter.inc",
+            ),
+            (
+                ["-gen-dag-isel"],
+                "lib/Target/LoongArch/LoongArchGenDAGISel.inc",
+            ),
+            (
+                ["-gen-disassembler"],
+                "lib/Target/LoongArch/LoongArchGenDisassemblerTables.inc",
+            ),
+            (
+                ["-gen-emitter"],
+                "lib/Target/LoongArch/LoongArchGenMCCodeEmitter.inc",
+            ),
+            (
+                ["-gen-instr-info"],
+                "lib/Target/LoongArch/LoongArchGenInstrInfo.inc",
+            ),
+            (
+                ["-gen-pseudo-lowering"],
+                "lib/Target/LoongArch/LoongArchGenMCPseudoLowering.inc",
+            ),
+            (
+                ["-gen-register-info"],
+                "lib/Target/LoongArch/LoongArchGenRegisterInfo.inc",
+            ),
+            (
+                ["-gen-subtarget"],
+                "lib/Target/LoongArch/LoongArchGenSubtargetInfo.inc",
+            ),
         ],
     },
     {
         "name": "Mips",
         "short_name": "Mips",
         "tbl_outs": [
-            ("-gen-asm-matcher", "lib/Target/Mips/MipsGenAsmMatcher.inc"),
-            ("-gen-asm-writer", "lib/Target/Mips/MipsGenAsmWriter.inc"),
-            ("-gen-callingconv", "lib/Target/Mips/MipsGenCallingConv.inc"),
-            ("-gen-dag-isel", "lib/Target/Mips/MipsGenDAGISel.inc"),
-            ("-gen-disassembler", "lib/Target/Mips/MipsGenDisassemblerTables.inc"),
-            ("-gen-emitter", "lib/Target/Mips/MipsGenMCCodeEmitter.inc"),
-            ("-gen-exegesis", "lib/Target/Mips/MipsGenExegesis.inc"),
-            ("-gen-fast-isel", "lib/Target/Mips/MipsGenFastISel.inc"),
-            ("-gen-global-isel", "lib/Target/Mips/MipsGenGlobalISel.inc"),
-            ("-gen-global-isel-combiner -combiners=MipsPostLegalizerCombiner", "lib/Target/Mips/MipsGenPostLegalizeGICombiner.inc"),
-            ("-gen-instr-info", "lib/Target/Mips/MipsGenInstrInfo.inc"),
-            ("-gen-pseudo-lowering", "lib/Target/Mips/MipsGenMCPseudoLowering.inc"),
-            ("-gen-register-bank", "lib/Target/Mips/MipsGenRegisterBank.inc"),
-            ("-gen-register-info", "lib/Target/Mips/MipsGenRegisterInfo.inc"),
-            ("-gen-subtarget", "lib/Target/Mips/MipsGenSubtargetInfo.inc"),
+            (
+                ["-gen-asm-matcher"],
+                "lib/Target/Mips/MipsGenAsmMatcher.inc",
+            ),
+            (
+                ["-gen-asm-writer"],
+                "lib/Target/Mips/MipsGenAsmWriter.inc",
+            ),
+            (
+                ["-gen-callingconv"],
+                "lib/Target/Mips/MipsGenCallingConv.inc",
+            ),
+            (
+                ["-gen-dag-isel"],
+                "lib/Target/Mips/MipsGenDAGISel.inc",
+            ),
+            (
+                ["-gen-disassembler"],
+                "lib/Target/Mips/MipsGenDisassemblerTables.inc",
+            ),
+            (
+                ["-gen-emitter"],
+                "lib/Target/Mips/MipsGenMCCodeEmitter.inc",
+            ),
+            (
+                ["-gen-exegesis"],
+                "lib/Target/Mips/MipsGenExegesis.inc",
+            ),
+            (
+                ["-gen-fast-isel"],
+                "lib/Target/Mips/MipsGenFastISel.inc",
+            ),
+            (
+                ["-gen-global-isel"],
+                "lib/Target/Mips/MipsGenGlobalISel.inc",
+            ),
+            (
+                [
+                    "-gen-global-isel-combiner",
+                    "-combiners=MipsPostLegalizerCombiner",
+                ],
+                "lib/Target/Mips/MipsGenPostLegalizeGICombiner.inc",
+            ),
+            (
+                ["-gen-instr-info"],
+                "lib/Target/Mips/MipsGenInstrInfo.inc",
+            ),
+            (
+                ["-gen-pseudo-lowering"],
+                "lib/Target/Mips/MipsGenMCPseudoLowering.inc",
+            ),
+            (
+                ["-gen-register-bank"],
+                "lib/Target/Mips/MipsGenRegisterBank.inc",
+            ),
+            (
+                ["-gen-register-info"],
+                "lib/Target/Mips/MipsGenRegisterInfo.inc",
+            ),
+            (
+                ["-gen-subtarget"],
+                "lib/Target/Mips/MipsGenSubtargetInfo.inc",
+            ),
         ],
     },
     {
         "name": "MSP430",
         "short_name": "MSP430",
         "tbl_outs": [
-            ("-gen-asm-matcher", "lib/Target/MSP430/MSP430GenAsmMatcher.inc"),
-            ("-gen-asm-writer", "lib/Target/MSP430/MSP430GenAsmWriter.inc"),
-            ("-gen-callingconv", "lib/Target/MSP430/MSP430GenCallingConv.inc"),
-            ("-gen-dag-isel", "lib/Target/MSP430/MSP430GenDAGISel.inc"),
-            ("-gen-disassembler", "lib/Target/MSP430/MSP430GenDisassemblerTables.inc"),
-            ("-gen-emitter", "lib/Target/MSP430/MSP430GenMCCodeEmitter.inc"),
-            ("-gen-instr-info", "lib/Target/MSP430/MSP430GenInstrInfo.inc"),
-            ("-gen-register-info", "lib/Target/MSP430/MSP430GenRegisterInfo.inc"),
-            ("-gen-subtarget", "lib/Target/MSP430/MSP430GenSubtargetInfo.inc"),
+            (
+                ["-gen-asm-matcher"],
+                "lib/Target/MSP430/MSP430GenAsmMatcher.inc",
+            ),
+            (
+                ["-gen-asm-writer"],
+                "lib/Target/MSP430/MSP430GenAsmWriter.inc",
+            ),
+            (
+                ["-gen-callingconv"],
+                "lib/Target/MSP430/MSP430GenCallingConv.inc",
+            ),
+            (
+                ["-gen-dag-isel"],
+                "lib/Target/MSP430/MSP430GenDAGISel.inc",
+            ),
+            (
+                ["-gen-disassembler"],
+                "lib/Target/MSP430/MSP430GenDisassemblerTables.inc",
+            ),
+            (
+                ["-gen-emitter"],
+                "lib/Target/MSP430/MSP430GenMCCodeEmitter.inc",
+            ),
+            (
+                ["-gen-instr-info"],
+                "lib/Target/MSP430/MSP430GenInstrInfo.inc",
+            ),
+            (
+                ["-gen-register-info"],
+                "lib/Target/MSP430/MSP430GenRegisterInfo.inc",
+            ),
+            (
+                ["-gen-subtarget"],
+                "lib/Target/MSP430/MSP430GenSubtargetInfo.inc",
+            ),
         ],
     },
     {
         "name": "NVPTX",
         "short_name": "NVPTX",
         "tbl_outs": [
-            ("-gen-register-info", "lib/Target/NVPTX/NVPTXGenRegisterInfo.inc"),
-            ("-gen-instr-info", "lib/Target/NVPTX/NVPTXGenInstrInfo.inc"),
-            ("-gen-asm-writer", "lib/Target/NVPTX/NVPTXGenAsmWriter.inc"),
-            ("-gen-dag-isel", "lib/Target/NVPTX/NVPTXGenDAGISel.inc"),
-            ("-gen-subtarget", "lib/Target/NVPTX/NVPTXGenSubtargetInfo.inc"),
+            (
+                ["-gen-register-info"],
+                "lib/Target/NVPTX/NVPTXGenRegisterInfo.inc",
+            ),
+            (
+                ["-gen-instr-info"],
+                "lib/Target/NVPTX/NVPTXGenInstrInfo.inc",
+            ),
+            (
+                ["-gen-asm-writer"],
+                "lib/Target/NVPTX/NVPTXGenAsmWriter.inc",
+            ),
+            (
+                ["-gen-dag-isel"],
+                "lib/Target/NVPTX/NVPTXGenDAGISel.inc",
+            ),
+            (
+                ["-gen-subtarget"],
+                "lib/Target/NVPTX/NVPTXGenSubtargetInfo.inc",
+            ),
         ],
     },
     {
         "name": "PowerPC",
         "short_name": "PPC",
         "tbl_outs": [
-            ("-gen-asm-writer", "lib/Target/PowerPC/PPCGenAsmWriter.inc"),
-            ("-gen-asm-matcher", "lib/Target/PowerPC/PPCGenAsmMatcher.inc"),
-            ("-gen-emitter", "lib/Target/PowerPC/PPCGenMCCodeEmitter.inc"),
-            ("-gen-register-info", "lib/Target/PowerPC/PPCGenRegisterInfo.inc"),
-            ("-gen-instr-info", "lib/Target/PowerPC/PPCGenInstrInfo.inc"),
-            ("-gen-dag-isel", "lib/Target/PowerPC/PPCGenDAGISel.inc"),
-            ("-gen-fast-isel", "lib/Target/PowerPC/PPCGenFastISel.inc"),
-            ("-gen-callingconv", "lib/Target/PowerPC/PPCGenCallingConv.inc"),
-            ("-gen-subtarget", "lib/Target/PowerPC/PPCGenSubtargetInfo.inc"),
-            ("-gen-disassembler", "lib/Target/PowerPC/PPCGenDisassemblerTables.inc"),
-            ("-gen-register-bank", "lib/Target/PowerPC/PPCGenRegisterBank.inc"),
-            ("-gen-global-isel", "lib/Target/PowerPC/PPCGenGlobalISel.inc"),
-            ("-gen-exegesis", "lib/Target/PowerPC/PPCGenExegesis.inc"),
+            (
+                ["-gen-asm-writer"],
+                "lib/Target/PowerPC/PPCGenAsmWriter.inc",
+            ),
+            (
+                ["-gen-asm-matcher"],
+                "lib/Target/PowerPC/PPCGenAsmMatcher.inc",
+            ),
+            (
+                ["-gen-emitter"],
+                "lib/Target/PowerPC/PPCGenMCCodeEmitter.inc",
+            ),
+            (
+                ["-gen-register-info"],
+                "lib/Target/PowerPC/PPCGenRegisterInfo.inc",
+            ),
+            (
+                ["-gen-instr-info"],
+                "lib/Target/PowerPC/PPCGenInstrInfo.inc",
+            ),
+            (
+                ["-gen-dag-isel"],
+                "lib/Target/PowerPC/PPCGenDAGISel.inc",
+            ),
+            (
+                ["-gen-fast-isel"],
+                "lib/Target/PowerPC/PPCGenFastISel.inc",
+            ),
+            (
+                ["-gen-callingconv"],
+                "lib/Target/PowerPC/PPCGenCallingConv.inc",
+            ),
+            (
+                ["-gen-subtarget"],
+                "lib/Target/PowerPC/PPCGenSubtargetInfo.inc",
+            ),
+            (
+                ["-gen-disassembler"],
+                "lib/Target/PowerPC/PPCGenDisassemblerTables.inc",
+            ),
+            (
+                ["-gen-register-bank"],
+                "lib/Target/PowerPC/PPCGenRegisterBank.inc",
+            ),
+            (
+                ["-gen-global-isel"],
+                "lib/Target/PowerPC/PPCGenGlobalISel.inc",
+            ),
+            (
+                ["-gen-exegesis"],
+                "lib/Target/PowerPC/PPCGenExegesis.inc",
+            ),
         ],
     },
     {
         "name": "RISCV",
         "short_name": "RISCV",
         "tbl_outs": [
-            ("-gen-asm-matcher", "lib/Target/RISCV/RISCVGenAsmMatcher.inc"),
-            ("-gen-asm-writer", "lib/Target/RISCV/RISCVGenAsmWriter.inc"),
-            ("-gen-compress-inst-emitter", "lib/Target/RISCV/RISCVGenCompressInstEmitter.inc"),
-            ("-gen-dag-isel", "lib/Target/RISCV/RISCVGenDAGISel.inc"),
-            ("-gen-disassembler", "lib/Target/RISCV/RISCVGenDisassemblerTables.inc"),
-            ("-gen-instr-info", "lib/Target/RISCV/RISCVGenInstrInfo.inc"),
-            ("-gen-macro-fusion-pred", "lib/Target/RISCV/RISCVGenMacroFusion.inc"),
-            ("-gen-emitter", "lib/Target/RISCV/RISCVGenMCCodeEmitter.inc"),
-            ("-gen-pseudo-lowering", "lib/Target/RISCV/RISCVGenMCPseudoLowering.inc"),
-            ("-gen-register-bank", "lib/Target/RISCV/RISCVGenRegisterBank.inc"),
-            ("-gen-register-info", "lib/Target/RISCV/RISCVGenRegisterInfo.inc"),
-            ("-gen-subtarget", "lib/Target/RISCV/RISCVGenSubtargetInfo.inc"),
-            ("-gen-searchable-tables", "lib/Target/RISCV/RISCVGenSearchableTables.inc"),
-            ("-gen-exegesis", "lib/Target/RISCV/RISCVGenExegesis.inc"),
+            (
+                ["-gen-asm-matcher"],
+                "lib/Target/RISCV/RISCVGenAsmMatcher.inc",
+            ),
+            (
+                ["-gen-asm-writer"],
+                "lib/Target/RISCV/RISCVGenAsmWriter.inc",
+            ),
+            (
+                ["-gen-compress-inst-emitter"],
+                "lib/Target/RISCV/RISCVGenCompressInstEmitter.inc",
+            ),
+            (
+                ["-gen-dag-isel"],
+                "lib/Target/RISCV/RISCVGenDAGISel.inc",
+            ),
+            (
+                ["-gen-disassembler"],
+                "lib/Target/RISCV/RISCVGenDisassemblerTables.inc",
+            ),
+            (
+                ["-gen-instr-info"],
+                "lib/Target/RISCV/RISCVGenInstrInfo.inc",
+            ),
+            (
+                ["-gen-macro-fusion-pred"],
+                "lib/Target/RISCV/RISCVGenMacroFusion.inc",
+            ),
+            (
+                ["-gen-emitter"],
+                "lib/Target/RISCV/RISCVGenMCCodeEmitter.inc",
+            ),
+            (
+                ["-gen-pseudo-lowering"],
+                "lib/Target/RISCV/RISCVGenMCPseudoLowering.inc",
+            ),
+            (
+                ["-gen-register-bank"],
+                "lib/Target/RISCV/RISCVGenRegisterBank.inc",
+            ),
+            (
+                ["-gen-register-info"],
+                "lib/Target/RISCV/RISCVGenRegisterInfo.inc",
+            ),
+            (
+                ["-gen-subtarget"],
+                "lib/Target/RISCV/RISCVGenSubtargetInfo.inc",
+            ),
+            (
+                ["-gen-searchable-tables"],
+                "lib/Target/RISCV/RISCVGenSearchableTables.inc",
+            ),
+            (
+                ["-gen-exegesis"],
+                "lib/Target/RISCV/RISCVGenExegesis.inc",
+            ),
         ],
         "tbl_deps": [
             ":riscv_isel_target_gen",
@@ -2255,112 +2785,337 @@ llvm_target_lib_list = [lib for lib in [
         "name": "Sparc",
         "short_name": "Sparc",
         "tbl_outs": [
-            ("-gen-asm-writer", "lib/Target/Sparc/SparcGenAsmWriter.inc"),
-            ("-gen-asm-matcher", "lib/Target/Sparc/SparcGenAsmMatcher.inc"),
-            ("-gen-emitter", "lib/Target/Sparc/SparcGenMCCodeEmitter.inc"),
-            ("-gen-register-info", "lib/Target/Sparc/SparcGenRegisterInfo.inc"),
-            ("-gen-instr-info", "lib/Target/Sparc/SparcGenInstrInfo.inc"),
-            ("-gen-dag-isel", "lib/Target/Sparc/SparcGenDAGISel.inc"),
-            ("-gen-callingconv", "lib/Target/Sparc/SparcGenCallingConv.inc"),
-            ("-gen-subtarget", "lib/Target/Sparc/SparcGenSubtargetInfo.inc"),
-            ("-gen-disassembler", "lib/Target/Sparc/SparcGenDisassemblerTables.inc"),
-            ("-gen-searchable-tables", "lib/Target/Sparc/SparcGenSearchableTables.inc"),
+            (
+                ["-gen-asm-writer"],
+                "lib/Target/Sparc/SparcGenAsmWriter.inc",
+            ),
+            (
+                ["-gen-asm-matcher"],
+                "lib/Target/Sparc/SparcGenAsmMatcher.inc",
+            ),
+            (
+                ["-gen-emitter"],
+                "lib/Target/Sparc/SparcGenMCCodeEmitter.inc",
+            ),
+            (
+                ["-gen-register-info"],
+                "lib/Target/Sparc/SparcGenRegisterInfo.inc",
+            ),
+            (
+                ["-gen-instr-info"],
+                "lib/Target/Sparc/SparcGenInstrInfo.inc",
+            ),
+            (
+                ["-gen-dag-isel"],
+                "lib/Target/Sparc/SparcGenDAGISel.inc",
+            ),
+            (
+                ["-gen-callingconv"],
+                "lib/Target/Sparc/SparcGenCallingConv.inc",
+            ),
+            (
+                ["-gen-subtarget"],
+                "lib/Target/Sparc/SparcGenSubtargetInfo.inc",
+            ),
+            (
+                ["-gen-disassembler"],
+                "lib/Target/Sparc/SparcGenDisassemblerTables.inc",
+            ),
+            (
+                ["-gen-searchable-tables"],
+                "lib/Target/Sparc/SparcGenSearchableTables.inc",
+            ),
         ],
     },
     {
         "name": "SPIRV",
         "short_name": "SPIRV",
         "tbl_outs": [
-            ("-gen-asm-writer", "lib/Target/SPIRV/SPIRVGenAsmWriter.inc"),
-            ("-gen-emitter", "lib/Target/SPIRV/SPIRVGenMCCodeEmitter.inc"),
-            ("-gen-global-isel", "lib/Target/SPIRV/SPIRVGenGlobalISel.inc"),
-            ("-gen-global-isel-combiner -combiners=SPIRVPreLegalizerCombiner", "lib/Target/SPIRV/SPIRVGenPreLegalizeGICombiner.inc"),
-            ("-gen-instr-info", "lib/Target/SPIRV/SPIRVGenInstrInfo.inc"),
-            ("-gen-register-bank", "lib/Target/SPIRV/SPIRVGenRegisterBank.inc"),
-            ("-gen-register-info", "lib/Target/SPIRV/SPIRVGenRegisterInfo.inc"),
-            ("-gen-searchable-tables", "lib/Target/SPIRV/SPIRVGenTables.inc"),
-            ("-gen-subtarget", "lib/Target/SPIRV/SPIRVGenSubtargetInfo.inc"),
+            (
+                ["-gen-asm-writer"],
+                "lib/Target/SPIRV/SPIRVGenAsmWriter.inc",
+            ),
+            (
+                ["-gen-emitter"],
+                "lib/Target/SPIRV/SPIRVGenMCCodeEmitter.inc",
+            ),
+            (
+                ["-gen-global-isel"],
+                "lib/Target/SPIRV/SPIRVGenGlobalISel.inc",
+            ),
+            (
+                [
+                    "-gen-global-isel-combiner",
+                    "-combiners=SPIRVPreLegalizerCombiner",
+                ],
+                "lib/Target/SPIRV/SPIRVGenPreLegalizeGICombiner.inc",
+            ),
+            (
+                ["-gen-instr-info"],
+                "lib/Target/SPIRV/SPIRVGenInstrInfo.inc",
+            ),
+            (
+                ["-gen-register-bank"],
+                "lib/Target/SPIRV/SPIRVGenRegisterBank.inc",
+            ),
+            (
+                ["-gen-register-info"],
+                "lib/Target/SPIRV/SPIRVGenRegisterInfo.inc",
+            ),
+            (
+                ["-gen-searchable-tables"],
+                "lib/Target/SPIRV/SPIRVGenTables.inc",
+            ),
+            (
+                ["-gen-subtarget"],
+                "lib/Target/SPIRV/SPIRVGenSubtargetInfo.inc",
+            ),
         ],
     },
     {
         "name": "SystemZ",
         "short_name": "SystemZ",
         "tbl_outs": [
-            ("-gen-asm-matcher", "lib/Target/SystemZ/SystemZGenAsmMatcher.inc"),
-            ("-gen-asm-writer", "lib/Target/SystemZ/SystemZGenGNUAsmWriter.inc"),
-            ("-gen-asm-writer -asmwriternum=1", "lib/Target/SystemZ/SystemZGenHLASMAsmWriter.inc"),
-            ("-gen-callingconv", "lib/Target/SystemZ/SystemZGenCallingConv.inc"),
-            ("-gen-dag-isel", "lib/Target/SystemZ/SystemZGenDAGISel.inc"),
-            ("-gen-disassembler", "lib/Target/SystemZ/SystemZGenDisassemblerTables.inc"),
-            ("-gen-emitter", "lib/Target/SystemZ/SystemZGenMCCodeEmitter.inc"),
-            ("-gen-instr-info", "lib/Target/SystemZ/SystemZGenInstrInfo.inc"),
-            ("-gen-register-info", "lib/Target/SystemZ/SystemZGenRegisterInfo.inc"),
-            ("-gen-subtarget", "lib/Target/SystemZ/SystemZGenSubtargetInfo.inc"),
+            (
+                ["-gen-asm-matcher"],
+                "lib/Target/SystemZ/SystemZGenAsmMatcher.inc",
+            ),
+            (
+                ["-gen-asm-writer"],
+                "lib/Target/SystemZ/SystemZGenGNUAsmWriter.inc",
+            ),
+            (
+                [
+                    "-gen-asm-writer",
+                    "-asmwriternum=1",
+                ],
+                "lib/Target/SystemZ/SystemZGenHLASMAsmWriter.inc",
+            ),
+            (
+                ["-gen-callingconv"],
+                "lib/Target/SystemZ/SystemZGenCallingConv.inc",
+            ),
+            (
+                ["-gen-dag-isel"],
+                "lib/Target/SystemZ/SystemZGenDAGISel.inc",
+            ),
+            (
+                ["-gen-disassembler"],
+                "lib/Target/SystemZ/SystemZGenDisassemblerTables.inc",
+            ),
+            (
+                ["-gen-emitter"],
+                "lib/Target/SystemZ/SystemZGenMCCodeEmitter.inc",
+            ),
+            (
+                ["-gen-instr-info"],
+                "lib/Target/SystemZ/SystemZGenInstrInfo.inc",
+            ),
+            (
+                ["-gen-register-info"],
+                "lib/Target/SystemZ/SystemZGenRegisterInfo.inc",
+            ),
+            (
+                ["-gen-subtarget"],
+                "lib/Target/SystemZ/SystemZGenSubtargetInfo.inc",
+            ),
         ],
     },
     {
         "name": "VE",
         "short_name": "VE",
         "tbl_outs": [
-            ("-gen-asm-matcher", "lib/Target/VE/VEGenAsmMatcher.inc"),
-            ("-gen-asm-writer", "lib/Target/VE/VEGenAsmWriter.inc"),
-            ("-gen-callingconv", "lib/Target/VE/VEGenCallingConv.inc"),
-            ("-gen-dag-isel", "lib/Target/VE/VEGenDAGISel.inc"),
-            ("-gen-disassembler", "lib/Target/VE/VEGenDisassemblerTables.inc"),
-            ("-gen-emitter", "lib/Target/VE/VEGenMCCodeEmitter.inc"),
-            ("-gen-instr-info", "lib/Target/VE/VEGenInstrInfo.inc"),
-            ("-gen-register-info", "lib/Target/VE/VEGenRegisterInfo.inc"),
-            ("-gen-subtarget", "lib/Target/VE/VEGenSubtargetInfo.inc"),
+            (
+                ["-gen-asm-matcher"],
+                "lib/Target/VE/VEGenAsmMatcher.inc",
+            ),
+            (
+                ["-gen-asm-writer"],
+                "lib/Target/VE/VEGenAsmWriter.inc",
+            ),
+            (
+                ["-gen-callingconv"],
+                "lib/Target/VE/VEGenCallingConv.inc",
+            ),
+            (
+                ["-gen-dag-isel"],
+                "lib/Target/VE/VEGenDAGISel.inc",
+            ),
+            (
+                ["-gen-disassembler"],
+                "lib/Target/VE/VEGenDisassemblerTables.inc",
+            ),
+            (
+                ["-gen-emitter"],
+                "lib/Target/VE/VEGenMCCodeEmitter.inc",
+            ),
+            (
+                ["-gen-instr-info"],
+                "lib/Target/VE/VEGenInstrInfo.inc",
+            ),
+            (
+                ["-gen-register-info"],
+                "lib/Target/VE/VEGenRegisterInfo.inc",
+            ),
+            (
+                ["-gen-subtarget"],
+                "lib/Target/VE/VEGenSubtargetInfo.inc",
+            ),
         ],
     },
     {
         "name": "WebAssembly",
         "short_name": "WebAssembly",
         "tbl_outs": [
-            ("-gen-disassembler", "lib/Target/WebAssembly/WebAssemblyGenDisassemblerTables.inc"),
-            ("-gen-asm-writer", "lib/Target/WebAssembly/WebAssemblyGenAsmWriter.inc"),
-            ("-gen-instr-info", "lib/Target/WebAssembly/WebAssemblyGenInstrInfo.inc"),
-            ("-gen-dag-isel", "lib/Target/WebAssembly/WebAssemblyGenDAGISel.inc"),
-            ("-gen-fast-isel", "lib/Target/WebAssembly/WebAssemblyGenFastISel.inc"),
-            ("-gen-emitter", "lib/Target/WebAssembly/WebAssemblyGenMCCodeEmitter.inc"),
-            ("-gen-register-info", "lib/Target/WebAssembly/WebAssemblyGenRegisterInfo.inc"),
-            ("-gen-subtarget", "lib/Target/WebAssembly/WebAssemblyGenSubtargetInfo.inc"),
-            ("-gen-asm-matcher", "lib/Target/WebAssembly/WebAssemblyGenAsmMatcher.inc"),
+            (
+                ["-gen-disassembler"],
+                "lib/Target/WebAssembly/WebAssemblyGenDisassemblerTables.inc",
+            ),
+            (
+                ["-gen-asm-writer"],
+                "lib/Target/WebAssembly/WebAssemblyGenAsmWriter.inc",
+            ),
+            (
+                ["-gen-instr-info"],
+                "lib/Target/WebAssembly/WebAssemblyGenInstrInfo.inc",
+            ),
+            (
+                ["-gen-dag-isel"],
+                "lib/Target/WebAssembly/WebAssemblyGenDAGISel.inc",
+            ),
+            (
+                ["-gen-fast-isel"],
+                "lib/Target/WebAssembly/WebAssemblyGenFastISel.inc",
+            ),
+            (
+                ["-gen-emitter"],
+                "lib/Target/WebAssembly/WebAssemblyGenMCCodeEmitter.inc",
+            ),
+            (
+                ["-gen-register-info"],
+                "lib/Target/WebAssembly/WebAssemblyGenRegisterInfo.inc",
+            ),
+            (
+                ["-gen-subtarget"],
+                "lib/Target/WebAssembly/WebAssemblyGenSubtargetInfo.inc",
+            ),
+            (
+                ["-gen-asm-matcher"],
+                "lib/Target/WebAssembly/WebAssemblyGenAsmMatcher.inc",
+            ),
         ],
     },
     {
         "name": "X86",
         "short_name": "X86",
         "tbl_outs": [
-            ("-gen-register-bank", "lib/Target/X86/X86GenRegisterBank.inc"),
-            ("-gen-register-info", "lib/Target/X86/X86GenRegisterInfo.inc"),
-            ("-gen-disassembler", "lib/Target/X86/X86GenDisassemblerTables.inc"),
-            ("-gen-instr-info", "lib/Target/X86/X86GenInstrInfo.inc"),
-            ("-gen-asm-writer", "lib/Target/X86/X86GenAsmWriter.inc"),
-            ("-gen-asm-writer -asmwriternum=1", "lib/Target/X86/X86GenAsmWriter1.inc"),
-            ("-gen-asm-matcher", "lib/Target/X86/X86GenAsmMatcher.inc"),
-            ("-gen-dag-isel", "lib/Target/X86/X86GenDAGISel.inc"),
-            ("-gen-fast-isel", "lib/Target/X86/X86GenFastISel.inc"),
-            ("-gen-global-isel", "lib/Target/X86/X86GenGlobalISel.inc"),
-            ("-gen-callingconv", "lib/Target/X86/X86GenCallingConv.inc"),
-            ("-gen-subtarget", "lib/Target/X86/X86GenSubtargetInfo.inc"),
-            ("-gen-x86-fold-tables -asmwriternum=1", "lib/Target/X86/X86GenFoldTables.inc"),
-            ("-gen-x86-instr-mapping", "lib/Target/X86/X86GenInstrMapping.inc"),
-            ("-gen-exegesis", "lib/Target/X86/X86GenExegesis.inc"),
-            ("-gen-x86-mnemonic-tables -asmwriternum=1", "lib/Target/X86/X86GenMnemonicTables.inc"),
+            (
+                ["-gen-register-bank"],
+                "lib/Target/X86/X86GenRegisterBank.inc",
+            ),
+            (
+                ["-gen-register-info"],
+                "lib/Target/X86/X86GenRegisterInfo.inc",
+            ),
+            (
+                ["-gen-disassembler"],
+                "lib/Target/X86/X86GenDisassemblerTables.inc",
+            ),
+            (
+                ["-gen-instr-info"],
+                "lib/Target/X86/X86GenInstrInfo.inc",
+            ),
+            (
+                ["-gen-asm-writer"],
+                "lib/Target/X86/X86GenAsmWriter.inc",
+            ),
+            (
+                [
+                    "-gen-asm-writer",
+                    "-asmwriternum=1",
+                ],
+                "lib/Target/X86/X86GenAsmWriter1.inc",
+            ),
+            (
+                ["-gen-asm-matcher"],
+                "lib/Target/X86/X86GenAsmMatcher.inc",
+            ),
+            (
+                ["-gen-dag-isel"],
+                "lib/Target/X86/X86GenDAGISel.inc",
+            ),
+            (
+                ["-gen-fast-isel"],
+                "lib/Target/X86/X86GenFastISel.inc",
+            ),
+            (
+                ["-gen-global-isel"],
+                "lib/Target/X86/X86GenGlobalISel.inc",
+            ),
+            (
+                ["-gen-callingconv"],
+                "lib/Target/X86/X86GenCallingConv.inc",
+            ),
+            (
+                ["-gen-subtarget"],
+                "lib/Target/X86/X86GenSubtargetInfo.inc",
+            ),
+            (
+                [
+                    "-gen-x86-fold-tables",
+                    "-asmwriternum=1",
+                ],
+                "lib/Target/X86/X86GenFoldTables.inc",
+            ),
+            (
+                ["-gen-x86-instr-mapping"],
+                "lib/Target/X86/X86GenInstrMapping.inc",
+            ),
+            (
+                ["-gen-exegesis"],
+                "lib/Target/X86/X86GenExegesis.inc",
+            ),
+            (
+                [
+                    "-gen-x86-mnemonic-tables",
+                    "-asmwriternum=1",
+                ],
+                "lib/Target/X86/X86GenMnemonicTables.inc",
+            ),
         ],
     },
     {
         "name": "XCore",
         "short_name": "XCore",
         "tbl_outs": [
-            ("-gen-asm-writer", "lib/Target/XCore/XCoreGenAsmWriter.inc"),
-            ("-gen-callingconv", "lib/Target/XCore/XCoreGenCallingConv.inc"),
-            ("-gen-dag-isel", "lib/Target/XCore/XCoreGenDAGISel.inc"),
-            ("-gen-disassembler", "lib/Target/XCore/XCoreGenDisassemblerTables.inc"),
-            ("-gen-instr-info", "lib/Target/XCore/XCoreGenInstrInfo.inc"),
-            ("-gen-register-info", "lib/Target/XCore/XCoreGenRegisterInfo.inc"),
-            ("-gen-subtarget", "lib/Target/XCore/XCoreGenSubtargetInfo.inc"),
+            (
+                ["-gen-asm-writer"],
+                "lib/Target/XCore/XCoreGenAsmWriter.inc",
+            ),
+            (
+                ["-gen-callingconv"],
+                "lib/Target/XCore/XCoreGenCallingConv.inc",
+            ),
+            (
+                ["-gen-dag-isel"],
+                "lib/Target/XCore/XCoreGenDAGISel.inc",
+            ),
+            (
+                ["-gen-disassembler"],
+                "lib/Target/XCore/XCoreGenDisassemblerTables.inc",
+            ),
+            (
+                ["-gen-instr-info"],
+                "lib/Target/XCore/XCoreGenInstrInfo.inc",
+            ),
+            (
+                ["-gen-register-info"],
+                "lib/Target/XCore/XCoreGenRegisterInfo.inc",
+            ),
+            (
+                ["-gen-subtarget"],
+                "lib/Target/XCore/XCoreGenSubtargetInfo.inc",
+            ),
         ],
     },
 ] if lib["name"] in llvm_targets]
@@ -2370,88 +3125,151 @@ cc_library(
     textual_hdrs = ["lib/Target/X86/X86InstrInfo.h"],
 )
 
-gentbl(
+gentbl_cc_library(
     name = "amdgpu_isel_target_gen",
     strip_include_prefix = "lib/Target/AMDGPU",
     tbl_outs = [
-        ("-gen-global-isel", "lib/Target/AMDGPU/AMDGPUGenGlobalISel.inc"),
-        ("-gen-global-isel-combiner -combiners=AMDGPUPreLegalizerCombiner", "lib/Target/AMDGPU/AMDGPUGenPreLegalizeGICombiner.inc"),
-        ("-gen-global-isel-combiner -combiners=AMDGPUPostLegalizerCombiner", "lib/Target/AMDGPU/AMDGPUGenPostLegalizeGICombiner.inc"),
-        ("-gen-global-isel-combiner -combiners=AMDGPURegBankCombiner", "lib/Target/AMDGPU/AMDGPUGenRegBankGICombiner.inc"),
+        (
+            ["-gen-global-isel"],
+            "lib/Target/AMDGPU/AMDGPUGenGlobalISel.inc",
+        ),
+        (
+            [
+                "-gen-global-isel-combiner",
+                "-combiners=AMDGPUPreLegalizerCombiner",
+            ],
+            "lib/Target/AMDGPU/AMDGPUGenPreLegalizeGICombiner.inc",
+        ),
+        (
+            [
+                "-gen-global-isel-combiner",
+                "-combiners=AMDGPUPostLegalizerCombiner",
+            ],
+            "lib/Target/AMDGPU/AMDGPUGenPostLegalizeGICombiner.inc",
+        ),
+        (
+            [
+                "-gen-global-isel-combiner",
+                "-combiners=AMDGPURegBankCombiner",
+            ],
+            "lib/Target/AMDGPU/AMDGPUGenRegBankGICombiner.inc",
+        ),
     ],
     tblgen = ":llvm-tblgen",
     td_file = "lib/Target/AMDGPU/AMDGPUGISel.td",
-    td_srcs = [
-        ":common_target_td_sources",
-    ] + glob([
-        "lib/Target/AMDGPU/*.td",
-    ]),
+    deps = [
+        ":AMDGPUTargetTdFiles",
+        ":CommonTargetTdFiles",
+    ],
 )
 
-gentbl(
+gentbl_cc_library(
     name = "r600_target_gen",
     strip_include_prefix = "lib/Target/AMDGPU",
     tbl_outs = [
-        ("-gen-asm-writer", "lib/Target/AMDGPU/R600GenAsmWriter.inc"),
-        ("-gen-callingconv", "lib/Target/AMDGPU/R600GenCallingConv.inc"),
-        ("-gen-dag-isel", "lib/Target/AMDGPU/R600GenDAGISel.inc"),
-        ("-gen-dfa-packetizer", "lib/Target/AMDGPU/R600GenDFAPacketizer.inc"),
-        ("-gen-instr-info", "lib/Target/AMDGPU/R600GenInstrInfo.inc"),
-        ("-gen-emitter", "lib/Target/AMDGPU/R600GenMCCodeEmitter.inc"),
-        ("-gen-register-info", "lib/Target/AMDGPU/R600GenRegisterInfo.inc"),
-        ("-gen-subtarget", "lib/Target/AMDGPU/R600GenSubtargetInfo.inc"),
+        (
+            ["-gen-asm-writer"],
+            "lib/Target/AMDGPU/R600GenAsmWriter.inc",
+        ),
+        (
+            ["-gen-callingconv"],
+            "lib/Target/AMDGPU/R600GenCallingConv.inc",
+        ),
+        (
+            ["-gen-dag-isel"],
+            "lib/Target/AMDGPU/R600GenDAGISel.inc",
+        ),
+        (
+            ["-gen-dfa-packetizer"],
+            "lib/Target/AMDGPU/R600GenDFAPacketizer.inc",
+        ),
+        (
+            ["-gen-instr-info"],
+            "lib/Target/AMDGPU/R600GenInstrInfo.inc",
+        ),
+        (
+            ["-gen-emitter"],
+            "lib/Target/AMDGPU/R600GenMCCodeEmitter.inc",
+        ),
+        (
+            ["-gen-register-info"],
+            "lib/Target/AMDGPU/R600GenRegisterInfo.inc",
+        ),
+        (
+            ["-gen-subtarget"],
+            "lib/Target/AMDGPU/R600GenSubtargetInfo.inc",
+        ),
     ],
     tblgen = ":llvm-tblgen",
     td_file = "lib/Target/AMDGPU/R600.td",
-    td_srcs = [
-        ":common_target_td_sources",
-    ] + glob([
-        "lib/Target/AMDGPU/*.td",
-    ]),
+    deps = [
+        ":AMDGPUTargetTdFiles",
+        ":CommonTargetTdFiles",
+    ],
 )
 
-gentbl(
+gentbl_cc_library(
     name = "riscv_isel_target_gen",
     strip_include_prefix = "lib/Target/RISCV",
     tbl_outs = [
-        ("-gen-global-isel", "lib/Target/RISCV/RISCVGenGlobalISel.inc"),
-        ("-gen-global-isel-combiner -combiners=RISCVO0PreLegalizerCombiner", "lib/Target/RISCV/RISCVGenO0PreLegalizeGICombiner.inc"),
-        ("-gen-global-isel-combiner -combiners=RISCVPostLegalizerCombiner", "lib/Target/RISCV/RISCVGenPostLegalizeGICombiner.inc"),
-        ("-gen-global-isel-combiner -combiners=RISCVPreLegalizerCombiner", "lib/Target/RISCV/RISCVGenPreLegalizeGICombiner.inc"),
+        (
+            ["-gen-global-isel"],
+            "lib/Target/RISCV/RISCVGenGlobalISel.inc",
+        ),
+        (
+            [
+                "-gen-global-isel-combiner",
+                "-combiners=RISCVO0PreLegalizerCombiner",
+            ],
+            "lib/Target/RISCV/RISCVGenO0PreLegalizeGICombiner.inc",
+        ),
+        (
+            [
+                "-gen-global-isel-combiner",
+                "-combiners=RISCVPostLegalizerCombiner",
+            ],
+            "lib/Target/RISCV/RISCVGenPostLegalizeGICombiner.inc",
+        ),
+        (
+            [
+                "-gen-global-isel-combiner",
+                "-combiners=RISCVPreLegalizerCombiner",
+            ],
+            "lib/Target/RISCV/RISCVGenPreLegalizeGICombiner.inc",
+        ),
     ],
     tblgen = ":llvm-tblgen",
     td_file = "lib/Target/RISCV/RISCVGISel.td",
-    td_srcs = [
-        ":common_target_td_sources",
-    ] + glob([
-        "lib/Target/RISCV/**/*.td",
-    ]),
+    deps = [
+        ":CommonTargetTdFiles",
+        ":RISCVTargetTdFiles",
+    ],
 )
 
 [[
-    [gentbl(
+    [gentbl_cc_library(
         name = target["name"] + "CommonTableGen",
         strip_include_prefix = "lib/Target/" + target["name"],
-        tbl_outs = target["tbl_outs"],
+        tbl_outs = [(
+            # MSVC isn't happy with long string literals, while other compilers
+            # which support them get significant compile time improvements with
+            # them enabled. Ideally this flag would only be enabled on Windows via
+            # a select() on `@platforms//os:windows,`, but that would
+            # require refactoring gentbl from a macro into a rule.
+            # TODO(#92): Refactor gentbl to support this use
+            args + ["--long-string-literals=0"],
+            out,
+        ) for (args, out) in target["tbl_outs"]],
         tblgen = ":llvm-tblgen",
-        # MSVC isn't happy with long string literals, while other compilers
-        # which support them get significant compile time improvements with
-        # them enabled. Ideally this flag would only be enabled on Windows via
-        # a select() on `@platforms//os:windows,`, but that would
-        # require refactoring gentbl from a macro into a rule.
-        # TODO(#92): Refactor gentbl to support this use
-        tblgen_args = "--long-string-literals=0",
         td_file = "lib/Target/" + target["name"] + "/" + target["short_name"] + ".td",
-        td_srcs = [
-            ":common_target_td_sources",
-        ] + glob(
+        td_srcs = glob(
             [
                 "lib/Target/" + target["name"] + "/*.td",
                 "lib/Target/" + target["name"] + "/GISel/*.td",
             ],
             allow_empty = True,
         ),
-        deps = target.get("tbl_deps", []),
+        deps = target.get("tbl_deps", []) + [":CommonTargetTdFiles"],
     )],
     [cc_library(
         name = target["name"] + "Info",
@@ -2537,7 +3355,7 @@ gentbl(
             ":config",
             ":" + target["name"] + "CommonTableGen",
             ":" + target["name"] + "Info",
-        ],
+        ] + target.get("tbl_deps", []),
     )],
     [cc_library(
         name = target["name"] + "CodeGen",
@@ -2582,7 +3400,7 @@ gentbl(
             ":" + target["name"] + "CommonTableGen",
             ":" + target["name"] + "Info",
             ":" + target["name"] + "UtilsAndDesc",
-        ],
+        ] + target.get("tbl_deps", []),
     )],
     [cc_library(
         name = target["name"] + "AsmParser",
@@ -2878,16 +3696,16 @@ cc_library(
     ],
 )
 
-gentbl(
+gentbl_cc_library(
     name = "JITLinkTableGen",
     strip_include_prefix = "lib/ExecutionEngine/JITLink",
     tbl_outs = [(
-        "-gen-opt-parser-defs",
+        ["-gen-opt-parser-defs"],
         "lib/ExecutionEngine/JITLink/COFFOptions.inc",
     )],
     tblgen = ":llvm-tblgen",
     td_file = "lib/ExecutionEngine/JITLink/COFFOptions.td",
-    td_srcs = ["include/llvm/Option/OptParser.td"],
+    deps = [":OptParserTdFiles"],
 )
 
 cc_library(
@@ -3019,6 +3837,7 @@ cc_library(
     deps = [
         ":BinaryFormat",
         ":DebugInfo",
+        ":DebugInfoDWARF",
         ":JITLink",
         ":OrcJIT",
         ":OrcShared",
@@ -3120,16 +3939,16 @@ cc_library(
     ],
 )
 
-gentbl(
+gentbl_cc_library(
     name = "DllOptionsTableGen",
     strip_include_prefix = "lib/ToolDrivers/llvm-dlltool",
     tbl_outs = [(
-        "-gen-opt-parser-defs",
+        ["-gen-opt-parser-defs"],
         "lib/ToolDrivers/llvm-dlltool/Options.inc",
     )],
     tblgen = ":llvm-tblgen",
     td_file = "lib/ToolDrivers/llvm-dlltool/Options.td",
-    td_srcs = ["include/llvm/Option/OptParser.td"],
+    deps = [":OptParserTdFiles"],
 )
 
 cc_library(
@@ -3146,16 +3965,16 @@ cc_library(
     ],
 )
 
-gentbl(
+gentbl_cc_library(
     name = "LibOptionsTableGen",
     strip_include_prefix = "lib/ToolDrivers/llvm-lib",
     tbl_outs = [(
-        "-gen-opt-parser-defs",
+        ["-gen-opt-parser-defs"],
         "lib/ToolDrivers/llvm-lib/Options.inc",
     )],
     tblgen = ":llvm-tblgen",
     td_file = "lib/ToolDrivers/llvm-lib/Options.td",
-    td_srcs = ["include/llvm/Option/OptParser.td"],
+    deps = [":OptParserTdFiles"],
 )
 
 cc_library(
@@ -3373,16 +4192,16 @@ cc_library(
 ################################################################################
 # LLVM toolchain and development binaries
 
-gentbl(
+gentbl_cc_library(
     name = "DsymutilTableGen",
     strip_include_prefix = "tools/dsymutil",
     tbl_outs = [(
-        "-gen-opt-parser-defs",
+        ["-gen-opt-parser-defs"],
         "tools/dsymutil/Options.inc",
     )],
     tblgen = ":llvm-tblgen",
     td_file = "tools/dsymutil/Options.td",
-    td_srcs = ["include/llvm/Option/OptParser.td"],
+    deps = [":OptParserTdFiles"],
 )
 
 cc_library(
@@ -3585,16 +4404,16 @@ cc_binary(
     ],
 )
 
-gentbl(
+gentbl_cc_library(
     name = "CGDataOptsTableGen",
     strip_include_prefix = "tools/llvm-cgdata",
     tbl_outs = [(
-        "-gen-opt-parser-defs",
+        ["-gen-opt-parser-defs"],
         "tools/llvm-cgdata/Opts.inc",
     )],
     tblgen = ":llvm-tblgen",
     td_file = "tools/llvm-cgdata/Opts.td",
-    td_srcs = ["include/llvm/Option/OptParser.td"],
+    deps = [":OptParserTdFiles"],
 )
 
 cc_library(
@@ -3662,16 +4481,16 @@ cc_binary(
     ],
 )
 
-gentbl(
+gentbl_cc_library(
     name = "CvtResTableGen",
     strip_include_prefix = "tools/llvm-cvtres",
     tbl_outs = [(
-        "-gen-opt-parser-defs",
+        ["-gen-opt-parser-defs"],
         "tools/llvm-cvtres/Opts.inc",
     )],
     tblgen = ":llvm-tblgen",
     td_file = "tools/llvm-cvtres/Opts.td",
-    td_srcs = ["include/llvm/Option/OptParser.td"],
+    deps = [":OptParserTdFiles"],
 )
 
 cc_binary(
@@ -3720,16 +4539,16 @@ cc_binary(
     ],
 )
 
-gentbl(
+gentbl_cc_library(
     name = "CxxfiltOptsTableGen",
     strip_include_prefix = "tools/llvm-cxxfilt",
     tbl_outs = [(
-        "-gen-opt-parser-defs",
+        ["-gen-opt-parser-defs"],
         "tools/llvm-cxxfilt/Opts.inc",
     )],
     tblgen = ":llvm-tblgen",
     td_file = "tools/llvm-cxxfilt/Opts.td",
-    td_srcs = ["include/llvm/Option/OptParser.td"],
+    deps = [":OptParserTdFiles"],
 )
 
 cc_library(
@@ -3767,16 +4586,16 @@ cc_binary(
     ],
 )
 
-gentbl(
+gentbl_cc_library(
     name = "DebugInfodFindOptsTableGen",
     strip_include_prefix = "tools/llvm-debuginfod-find",
     tbl_outs = [(
-        "-gen-opt-parser-defs",
+        ["-gen-opt-parser-defs"],
         "tools/llvm-debuginfod-find/Opts.inc",
     )],
     tblgen = ":llvm-tblgen",
     td_file = "tools/llvm-debuginfod-find/Opts.td",
-    td_srcs = ["include/llvm/Option/OptParser.td"],
+    deps = [":OptParserTdFiles"],
 )
 
 cc_library(
@@ -3837,16 +4656,16 @@ cc_binary(
     ],
 )
 
-gentbl(
+gentbl_cc_library(
     name = "DwarfutilOptionsTableGen",
     strip_include_prefix = "tools/llvm-dwarfutil",
     tbl_outs = [(
-        "-gen-opt-parser-defs",
+        ["-gen-opt-parser-defs"],
         "tools/llvm-dwarfutil/Options.inc",
     )],
     tblgen = ":llvm-tblgen",
     td_file = "tools/llvm-dwarfutil/Options.td",
-    td_srcs = ["include/llvm/Option/OptParser.td"],
+    deps = [":OptParserTdFiles"],
 )
 
 cc_binary(
@@ -3875,16 +4694,16 @@ cc_binary(
     ],
 )
 
-gentbl(
+gentbl_cc_library(
     name = "DwpOptionsTableGen",
     strip_include_prefix = "tools/llvm-dwp",
     tbl_outs = [(
-        "-gen-opt-parser-defs",
+        ["-gen-opt-parser-defs"],
         "tools/llvm-dwp/Opts.inc",
     )],
     tblgen = ":llvm-tblgen",
     td_file = "tools/llvm-dwp/Opts.td",
-    td_srcs = ["include/llvm/Option/OptParser.td"],
+    deps = [":OptParserTdFiles"],
 )
 
 cc_library(
@@ -3949,16 +4768,16 @@ cc_binary(
     ],
 )
 
-gentbl(
+gentbl_cc_library(
     name = "GSYMUtilOptionsTableGen",
     strip_include_prefix = "tools/llvm-gsymutil",
     tbl_outs = [(
-        "-gen-opt-parser-defs",
+        ["-gen-opt-parser-defs"],
         "tools/llvm-gsymutil/Opts.inc",
     )],
     tblgen = ":llvm-tblgen",
     td_file = "tools/llvm-gsymutil/Opts.td",
-    td_srcs = ["include/llvm/Option/OptParser.td"],
+    deps = [":OptParserTdFiles"],
 )
 
 cc_library(
@@ -3986,16 +4805,16 @@ llvm_driver_cc_binary(
     deps = [":llvm-gsymutil-lib"],
 )
 
-gentbl(
+gentbl_cc_library(
     name = "IfsOptionsTableGen",
     strip_include_prefix = "tools/llvm-ifs",
     tbl_outs = [(
-        "-gen-opt-parser-defs",
+        ["-gen-opt-parser-defs"],
         "tools/llvm-ifs/Opts.inc",
     )],
     tblgen = ":llvm-tblgen",
     td_file = "tools/llvm-ifs/Opts.td",
-    td_srcs = ["include/llvm/Option/OptParser.td"],
+    deps = [":OptParserTdFiles"],
 )
 
 cc_library(
@@ -4069,16 +4888,16 @@ cc_binary(
     ],
 )
 
-gentbl(
+gentbl_cc_library(
     name = "LibtoolDarwinOptionsTableGen",
     strip_include_prefix = "tools/llvm-libtool-darwin",
     tbl_outs = [(
-        "-gen-opt-parser-defs",
+        ["-gen-opt-parser-defs"],
         "tools/llvm-libtool-darwin/Opts.inc",
     )],
     tblgen = ":llvm-tblgen",
     td_file = "tools/llvm-libtool-darwin/Opts.td",
-    td_srcs = ["include/llvm/Option/OptParser.td"],
+    deps = [":OptParserTdFiles"],
 )
 
 cc_library(
@@ -4130,16 +4949,16 @@ cc_binary(
     ],
 )
 
-gentbl(
+gentbl_cc_library(
     name = "LipoOptsTableGen",
     strip_include_prefix = "tools/llvm-lipo",
     tbl_outs = [(
-        "-gen-opt-parser-defs",
+        ["-gen-opt-parser-defs"],
         "tools/llvm-lipo/LipoOpts.inc",
     )],
     tblgen = ":llvm-tblgen",
     td_file = "tools/llvm-lipo/LipoOpts.td",
-    td_srcs = ["include/llvm/Option/OptParser.td"],
+    deps = [":OptParserTdFiles"],
 )
 
 cc_library(
@@ -4249,16 +5068,16 @@ cc_binary(
     ],
 )
 
-gentbl(
+gentbl_cc_library(
     name = "MlTableGen",
     strip_include_prefix = "tools/llvm-ml",
     tbl_outs = [(
-        "-gen-opt-parser-defs",
+        ["-gen-opt-parser-defs"],
         "tools/llvm-ml/Opts.inc",
     )],
     tblgen = ":llvm-tblgen",
     td_file = "tools/llvm-ml/Opts.td",
-    td_srcs = ["include/llvm/Option/OptParser.td"],
+    deps = [":OptParserTdFiles"],
 )
 
 cc_library(
@@ -4304,16 +5123,16 @@ cc_binary(
     ],
 )
 
-gentbl(
+gentbl_cc_library(
     name = "MtTableGen",
     strip_include_prefix = "tools/llvm-mt",
     tbl_outs = [(
-        "-gen-opt-parser-defs",
+        ["-gen-opt-parser-defs"],
         "tools/llvm-mt/Opts.inc",
     )],
     tblgen = ":llvm-tblgen",
     td_file = "tools/llvm-mt/Opts.td",
-    td_srcs = ["include/llvm/Option/OptParser.td"],
+    deps = [":OptParserTdFiles"],
 )
 
 cc_library(
@@ -4335,16 +5154,16 @@ llvm_driver_cc_binary(
     deps = [":llvm-mt-lib"],
 )
 
-gentbl(
+gentbl_cc_library(
     name = "NmOptsTableGen",
     strip_include_prefix = "tools/llvm-nm",
     tbl_outs = [(
-        "-gen-opt-parser-defs",
+        ["-gen-opt-parser-defs"],
         "tools/llvm-nm/Opts.inc",
     )],
     tblgen = ":llvm-tblgen",
     td_file = "tools/llvm-nm/Opts.td",
-    td_srcs = ["include/llvm/Option/OptParser.td"],
+    deps = [":OptParserTdFiles"],
 )
 
 cc_library(
@@ -4373,63 +5192,68 @@ llvm_driver_cc_binary(
     deps = [":llvm-nm-lib"],
 )
 
-gentbl(
+td_library(
+    name = "LlvmObjcopyCommonOpts",
+    srcs = ["tools/llvm-objcopy/CommonOpts.td"],
+)
+
+gentbl_cc_library(
     name = "llvm-objcopy-opts",
     strip_include_prefix = "tools/llvm-objcopy",
     tbl_outs = [(
-        "-gen-opt-parser-defs",
+        ["-gen-opt-parser-defs"],
         "tools/llvm-objcopy/ObjcopyOpts.inc",
     )],
     tblgen = ":llvm-tblgen",
     td_file = "tools/llvm-objcopy/ObjcopyOpts.td",
-    td_srcs = [
-        "include/llvm/Option/OptParser.td",
-        "tools/llvm-objcopy/CommonOpts.td",
+    deps = [
+        ":LlvmObjcopyCommonOpts",
+        ":OptParserTdFiles",
     ],
 )
 
-gentbl(
+gentbl_cc_library(
     name = "llvm-installnametool-opts",
     strip_include_prefix = "tools/llvm-objcopy",
     tbl_outs = [(
-        "-gen-opt-parser-defs",
+        ["-gen-opt-parser-defs"],
         "tools/llvm-objcopy/InstallNameToolOpts.inc",
     )],
     tblgen = ":llvm-tblgen",
     td_file = "tools/llvm-objcopy/InstallNameToolOpts.td",
-    td_srcs = [
-        "include/llvm/Option/OptParser.td",
-        "tools/llvm-objcopy/CommonOpts.td",
+    deps = [
+        ":LlvmObjcopyCommonOpts",
+        ":OptParserTdFiles",
     ],
 )
 
-gentbl(
+gentbl_cc_library(
     name = "llvm-strip-opts",
     strip_include_prefix = "tools/llvm-objcopy",
     tbl_outs = [(
-        "-gen-opt-parser-defs",
+        ["-gen-opt-parser-defs"],
         "tools/llvm-objcopy/StripOpts.inc",
     )],
     tblgen = ":llvm-tblgen",
     td_file = "tools/llvm-objcopy/StripOpts.td",
-    td_srcs = [
-        "include/llvm/Option/OptParser.td",
-        "tools/llvm-objcopy/CommonOpts.td",
+    deps = [
+        ":LlvmObjcopyCommonOpts",
+        ":OptParserTdFiles",
     ],
 )
 
-gentbl(
+gentbl_cc_library(
     name = "llvm-bitcode-strip-opts",
     strip_include_prefix = "tools/llvm-objcopy",
     tbl_outs = [(
-        "-gen-opt-parser-defs",
+        ["-gen-opt-parser-defs"],
         "tools/llvm-objcopy/BitcodeStripOpts.inc",
     )],
     tblgen = ":llvm-tblgen",
     td_file = "tools/llvm-objcopy/BitcodeStripOpts.td",
-    td_srcs = [
-        "include/llvm/Option/OptParser.td",
-        "tools/llvm-objcopy/CommonOpts.td",
+    deps = [
+        ":LlvmObjcopyCommonOpts",
+        ":OptParserTdFiles",
     ],
 )
 
@@ -4505,6 +5329,7 @@ cc_library(
         ":BinaryFormat",
         ":CodeGen",
         ":DebugInfo",
+        ":DebugInfoBTF",
         ":DebugInfoDWARF",
         ":Debuginfod",
         ":Demangle",
@@ -4527,16 +5352,16 @@ llvm_driver_cc_binary(
     deps = [":llvm-objdump-lib"],
 )
 
-gentbl(
+gentbl_cc_library(
     name = "ObjdumpOptsTableGen",
     strip_include_prefix = "tools/llvm-objdump",
     tbl_outs = [(
-        "-gen-opt-parser-defs",
+        ["-gen-opt-parser-defs"],
         "tools/llvm-objdump/ObjdumpOpts.inc",
     )],
     tblgen = ":llvm-tblgen",
     td_file = "tools/llvm-objdump/ObjdumpOpts.td",
-    td_srcs = ["include/llvm/Option/OptParser.td"],
+    deps = [":OptParserTdFiles"],
 )
 
 binary_alias(
@@ -4544,16 +5369,16 @@ binary_alias(
     binary = ":llvm-objdump",
 )
 
-gentbl(
+gentbl_cc_library(
     name = "OtoolOptsTableGen",
     strip_include_prefix = "tools/llvm-objdump",
     tbl_outs = [(
-        "-gen-opt-parser-defs",
+        ["-gen-opt-parser-defs"],
         "tools/llvm-objdump/OtoolOpts.inc",
     )],
     tblgen = ":llvm-tblgen",
     td_file = "tools/llvm-objdump/OtoolOpts.td",
-    td_srcs = ["include/llvm/Option/OptParser.td"],
+    deps = [":OptParserTdFiles"],
 )
 
 cc_binary(
@@ -4636,28 +5461,28 @@ cc_binary(
     ],
 )
 
-gentbl(
+gentbl_cc_library(
     name = "RcTableGen",
     strip_include_prefix = "tools/llvm-rc",
     tbl_outs = [(
-        "-gen-opt-parser-defs",
+        ["-gen-opt-parser-defs"],
         "tools/llvm-rc/Opts.inc",
     )],
     tblgen = ":llvm-tblgen",
     td_file = "tools/llvm-rc/Opts.td",
-    td_srcs = ["include/llvm/Option/OptParser.td"],
+    deps = [":OptParserTdFiles"],
 )
 
-gentbl(
+gentbl_cc_library(
     name = "WindresTableGen",
     strip_include_prefix = "tools/llvm-rc",
     tbl_outs = [(
-        "-gen-opt-parser-defs",
+        ["-gen-opt-parser-defs"],
         "tools/llvm-rc/WindresOpts.inc",
     )],
     tblgen = ":llvm-tblgen",
     td_file = "tools/llvm-rc/WindresOpts.td",
-    td_srcs = ["include/llvm/Option/OptParser.td"],
+    deps = [":OptParserTdFiles"],
 )
 
 # Workaround inability to put `.def` files into `srcs` with a library.
@@ -4696,16 +5521,16 @@ binary_alias(
     binary = ":llvm-rc",
 )
 
-gentbl(
+gentbl_cc_library(
     name = "ReadobjOptsTableGen",
     strip_include_prefix = "tools/llvm-readobj",
     tbl_outs = [(
-        "-gen-opt-parser-defs",
+        ["-gen-opt-parser-defs"],
         "tools/llvm-readobj/Opts.inc",
     )],
     tblgen = ":llvm-tblgen",
     td_file = "tools/llvm-readobj/Opts.td",
-    td_srcs = ["include/llvm/Option/OptParser.td"],
+    deps = [":OptParserTdFiles"],
 )
 
 cc_library(
@@ -4792,16 +5617,16 @@ cc_binary(
     ],
 )
 
-gentbl(
+gentbl_cc_library(
     name = "SizeOptsTableGen",
     strip_include_prefix = "tools/llvm-size",
     tbl_outs = [(
-        "-gen-opt-parser-defs",
+        ["-gen-opt-parser-defs"],
         "tools/llvm-size/Opts.inc",
     )],
     tblgen = ":llvm-tblgen",
     td_file = "tools/llvm-size/Opts.td",
-    td_srcs = ["include/llvm/Option/OptParser.td"],
+    deps = [":OptParserTdFiles"],
 )
 
 cc_library(
@@ -4844,16 +5669,16 @@ cc_binary(
     ],
 )
 
-gentbl(
+gentbl_cc_library(
     name = "StringsOptsTableGen",
     strip_include_prefix = "tools/llvm-strings",
     tbl_outs = [(
-        "-gen-opt-parser-defs",
+        ["-gen-opt-parser-defs"],
         "tools/llvm-strings/Opts.inc",
     )],
     tblgen = ":llvm-tblgen",
     td_file = "tools/llvm-strings/Opts.td",
-    td_srcs = ["include/llvm/Option/OptParser.td"],
+    deps = [":OptParserTdFiles"],
 )
 
 cc_binary(
@@ -4871,16 +5696,16 @@ cc_binary(
     ],
 )
 
-gentbl(
+gentbl_cc_library(
     name = "SymbolizerOptsTableGen",
     strip_include_prefix = "tools/llvm-symbolizer",
     tbl_outs = [(
-        "-gen-opt-parser-defs",
+        ["-gen-opt-parser-defs"],
         "tools/llvm-symbolizer/Opts.inc",
     )],
     tblgen = ":llvm-tblgen",
     td_file = "tools/llvm-symbolizer/Opts.td",
-    td_srcs = ["include/llvm/Option/OptParser.td"],
+    deps = [":OptParserTdFiles"],
 )
 
 cc_library(
@@ -4985,16 +5810,16 @@ cc_binary(
     deps = [":opt-driver"],
 )
 
-gentbl(
+gentbl_cc_library(
     name = "SancovOptsTableGen",
     strip_include_prefix = "tools/sancov",
     tbl_outs = [(
-        "-gen-opt-parser-defs",
+        ["-gen-opt-parser-defs"],
         "tools/sancov/Opts.inc",
     )],
     tblgen = ":llvm-tblgen",
     td_file = "tools/sancov/Opts.td",
-    td_srcs = ["include/llvm/Option/OptParser.td"],
+    deps = [":OptParserTdFiles"],
 )
 
 cc_library(
@@ -5377,16 +6202,16 @@ cc_binary(
     ],
 )
 
-gentbl(
+gentbl_cc_library(
     name = "ReadTAPIOptsTableGen",
     strip_include_prefix = "tools/llvm-readtapi",
     tbl_outs = [(
-        "-gen-opt-parser-defs",
+        ["-gen-opt-parser-defs"],
         "tools/llvm-readtapi/TapiOpts.inc",
     )],
     tblgen = ":llvm-tblgen",
     td_file = "tools/llvm-readtapi/TapiOpts.td",
-    td_srcs = ["include/llvm/Option/OptParser.td"],
+    deps = [":OptParserTdFiles"],
 )
 
 cc_binary(
@@ -5409,16 +6234,16 @@ cc_binary(
     ],
 )
 
-gentbl(
+gentbl_cc_library(
     name = "TLICheckerOptsTableGen",
     strip_include_prefix = "tools/llvm-tli-checker",
     tbl_outs = [(
-        "-gen-opt-parser-defs",
+        ["-gen-opt-parser-defs"],
         "tools/llvm-tli-checker/Opts.inc",
     )],
     tblgen = ":llvm-tblgen",
     td_file = "tools/llvm-tli-checker/Opts.td",
-    td_srcs = ["include/llvm/Option/OptParser.td"],
+    deps = [":OptParserTdFiles"],
 )
 
 cc_binary(
diff --git a/utils/bazel/llvm-project-overlay/llvm/unittests/BUILD.bazel b/utils/bazel/llvm-project-overlay/llvm/unittests/BUILD.bazel
index e85c099cec6dd..61f4700b057ab 100644
--- a/utils/bazel/llvm-project-overlay/llvm/unittests/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/llvm/unittests/BUILD.bazel
@@ -2,7 +2,7 @@
 # See https://llvm.org/LICENSE.txt for license information.
 # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 
-load("//llvm:tblgen.bzl", "gentbl")
+load("//mlir:tblgen.bzl", "gentbl_cc_library")
 
 package(
     default_visibility = ["//visibility:public"],
@@ -224,6 +224,8 @@ cc_test(
         # Skip a test that relies on reading files in a way that doesn't easily
         # work with Bazel.
         "--gtest_filter=-NativeSymbolReuseTest.*",
+        # TODO: this test is failing on some configs, investigate and re-enable it.
+        "--gtest_filter=-DebugLineBasicFixture.LookupAddressRangeWithStmtSequenceOffset",
     ],
     features = ["-layering_check"],  # #include "../lib/CodeGen/AsmPrinter/DwarfStringPool.h"
     deps = [
@@ -536,6 +538,7 @@ cc_test(
         "//llvm:Support",
         "//llvm:TargetParser",
         "//llvm:TestingSupport",
+        "//third-party/unittest:gmock",
         "//third-party/unittest:gtest",
         "//third-party/unittest:gtest_main",
     ],
@@ -559,46 +562,34 @@ cc_test(
     ],
 )
 
-gentbl(
+gentbl_cc_library(
     name = "option_tests_gen",
     strip_include_prefix = "Option",
     tbl_outs = [(
-        "-gen-opt-parser-defs",
+        ["-gen-opt-parser-defs"],
         "Option/Opts.inc",
     )],
     tblgen = "//llvm:llvm-tblgen",
     td_file = "Option/Opts.td",
-    td_srcs = [
-        "//llvm:include/llvm/Option/OptParser.td",
-    ],
+    deps = ["//llvm:OptParserTdFiles"],
 )
 
-gentbl(
-    name = "automata_automata_gen",
+gentbl_cc_library(
+    name = "automata_gen",
     strip_include_prefix = "TableGen",
-    tbl_outs = [(
-        "-gen-automata",
-        "TableGen/AutomataAutomata.inc",
-    )],
-    tblgen = "//llvm:llvm-tblgen",
-    td_file = "TableGen/Automata.td",
-    td_srcs = ["//llvm:common_target_td_sources"] + [
-        "TableGen/Automata.td",
+    tbl_outs = [
+        (
+            ["-gen-automata"],
+            "TableGen/AutomataAutomata.inc",
+        ),
+        (
+            ["-gen-searchable-tables"],
+            "TableGen/AutomataTables.inc",
+        ),
     ],
-)
-
-gentbl(
-    name = "automata_tables_gen",
-    strip_include_prefix = "TableGen",
-    tbl_outs = [(
-        "-gen-searchable-tables",
-        "TableGen/AutomataTables.inc",
-    )],
     tblgen = "//llvm:llvm-tblgen",
     td_file = "TableGen/Automata.td",
-    td_srcs = ["//llvm:common_target_td_sources"] + [
-        "TableGen/Automata.td",
-    ],
+    deps = ["//llvm:CommonTargetTdFiles"],
 )
 
 cc_test(
@@ -742,8 +733,7 @@ cc_test(
         allow_empty = False,
     ),
     deps = [
-        ":automata_automata_gen",
-        ":automata_tables_gen",
+        ":automata_gen",
         "//llvm:Support",
         "//llvm:TableGen",
         "//llvm:TableGenGlobalISel",
diff --git a/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel b/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel
index 3336daf6773b2..e07891f004850 100644
--- a/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel
@@ -7,7 +7,6 @@
 
 load("@bazel_skylib//rules:common_settings.bzl", "bool_flag")
 load("@bazel_skylib//rules:expand_template.bzl", "expand_template")
-load("//llvm:binary_alias.bzl", "binary_alias")
 load(
     ":build_defs.bzl",
     "cc_headers_only",
@@ -89,12 +88,36 @@ exports_files(glob(["include/**/*.td"]))
         deps = [":OpBaseTdFiles"],
     )
     for name in [
-        "OpAsmInterface",
         "RegionKindInterface",
         "SymbolInterfaces",
     ]
 ]
 
+gentbl_cc_library(
+    name = "OpAsmInterfaceIncGen",
+    tbl_outs = [
+        (
+            ["-gen-op-interface-decls"],
+            "include/mlir/IR/OpAsmOpInterface.h.inc",
+        ),
+        (
+            ["-gen-op-interface-defs"],
+            "include/mlir/IR/OpAsmOpInterface.cpp.inc",
+        ),
+        (
+            ["-gen-type-interface-decls"],
+            "include/mlir/IR/OpAsmTypeInterface.h.inc",
+        ),
+        (
+            ["-gen-type-interface-defs"],
+            "include/mlir/IR/OpAsmTypeInterface.cpp.inc",
+        ),
+    ],
+    tblgen = ":mlir-tblgen",
+    td_file = "include/mlir/IR/OpAsmInterface.td",
+    deps = [":OpBaseTdFiles"],
+)
+
 gentbl_cc_library(
     name = "TensorEncodingIncGen",
     tbl_outs = [
@@ -357,6 +380,7 @@ cc_library(
     hdrs = glob([
         "include/mlir/IR/*.h",
     ]) + [
+        "include/mlir/IR/OpAsmOpInterface.h.inc",
         "include/mlir/Interfaces/FoldInterfaces.h",
     ],
     includes = ["include"],
@@ -654,6 +678,7 @@ mlir_c_api_cc_library(
     ],
     includes = ["include"],
     deps = [
+        ":FromLLVMIRTranslation",
         ":LLVMToLLVMIRTranslation",
         ":Support",
         ":ToLLVMIRTranslation",
@@ -856,7 +881,6 @@ mlir_c_api_cc_library(
     includes = ["include"],
     deps = [
         ":ConversionPasses",
-        ":ConvertToSPIRV",
         ":Pass",
     ],
 )
@@ -4308,7 +4332,6 @@ cc_library(
         ":ControlFlowToSPIRV",
         ":ConversionPassIncGen",
         ":ConvertToLLVM",
-        ":ConvertToSPIRV",
         ":FuncToEmitC",
         ":FuncToLLVM",
         ":FuncToSPIRV",
@@ -4484,6 +4507,7 @@ cc_library(
     includes = ["include"],
     deps = [
         ":InferIntRangeInterface",
+        ":ShapedOpInterfaces",
         "//llvm:Support",
     ],
 )
@@ -5013,6 +5037,7 @@ cc_library(
         ":TransformPDLExtension",
         ":UBToLLVM",
         ":VCIXToLLVMIRTranslation",
+        ":VectorToLLVM",
         ":VectorTransformOps",
     ],
 )
@@ -5073,6 +5098,7 @@ cc_library(
         ":BufferizationInterfaces",
         ":BytecodeOpInterface",
         ":ControlFlowInterfaces",
+        ":ConvertToLLVMInterface",
         ":DestinationStyleOpInterface",
         ":DialectUtils",
         ":IR",
@@ -5085,6 +5111,7 @@ cc_library(
         ":SubsetOpInterface",
         ":Support",
         ":TensorDialect",
+        ":UBDialect",
         ":ValueBoundsOpInterface",
         ":VectorAttributesIncGen",
         ":VectorDialectIncGen",
@@ -5197,6 +5224,7 @@ cc_library(
         ":Support",
         ":TensorDialect",
         ":TransformUtils",
+        ":UBDialect",
         ":VectorDialect",
         ":VectorEnumsIncGen",
         ":VectorInterfaces",
@@ -5593,8 +5621,8 @@ cc_library(
     deps = [
         ":IR",
         ":LLVMDialect",
-        ":LLVMPassIncGen",
         ":LLVMIRTransformsDIExpressionLegalization",
+        ":LLVMPassIncGen",
         ":Pass",
     ],
 )
@@ -5976,6 +6004,7 @@ cc_library(
         ":IR",
         ":LLVMCommonConversion",
         ":MemRefDialect",
+        ":NVGPUDialect",
         ":NVVMDialect",
         ":SCFDialect",
         ":Support",
@@ -6174,6 +6203,8 @@ cc_library(
         ":SPIRVDialect",
         ":Support",
         ":TransformUtils",
+        ":UBDialect",
+        ":UBToSPIRV",
         ":VectorDialect",
         "//llvm:Support",
     ],
@@ -7728,6 +7759,7 @@ td_library(
     name = "TensorOpsTdFiles",
     srcs = [
         "include/mlir/Dialect/Tensor/IR/TensorBase.td",
+        "include/mlir/Dialect/Tensor/IR/TensorInterfaces.td",
         "include/mlir/Dialect/Tensor/IR/TensorOps.td",
     ],
     includes = ["include"],
@@ -7735,6 +7767,7 @@ td_library(
         ":CastInterfacesTdFiles",
         ":ControlFlowInterfacesTdFiles",
         ":DestinationStyleOpInterfaceTdFiles",
+        ":InferIntRangeInterfaceTdFiles",
         ":InferTypeOpInterfaceTdFiles",
         ":OpBaseTdFiles",
         ":ParallelCombiningOpInterfaceTdFiles",
@@ -7776,6 +7809,23 @@ gentbl_cc_library(
     deps = [":TensorOpsTdFiles"],
 )
 
+gentbl_cc_library(
+    name = "TensorInterfacesIncGen",
+    tbl_outs = [
+        (
+            ["--gen-op-interface-decls"],
+            "include/mlir/Dialect/Tensor/IR/TensorInterfaces.h.inc",
+        ),
+        (
+            ["--gen-op-interface-defs"],
+            "include/mlir/Dialect/Tensor/IR/TensorInterfaces.cpp.inc",
+        ),
+    ],
+    tblgen = ":mlir-tblgen",
+    td_file = "include/mlir/Dialect/Tensor/IR/TensorInterfaces.td",
+    deps = [":TensorOpsTdFiles"],
+)
+
 cc_library(
     name = "TensorDialect",
     srcs = [
@@ -7801,6 +7851,8 @@ cc_library(
         ":DestinationStyleOpInterface",
         ":DialectUtils",
         ":IR",
+        ":InferIntRangeCommon",
+        ":InferIntRangeInterface",
         ":InferTypeOpInterface",
         ":InliningUtils",
         ":LoopLikeInterface",
@@ -7809,6 +7861,7 @@ cc_library(
         ":SideEffectInterfaces",
         ":SubsetOpInterface",
         ":Support",
+        ":TensorInterfacesIncGen",
         ":TensorOpsIncGen",
         ":TilingInterface",
         ":TransformDialectInterfaces",
@@ -8382,6 +8435,7 @@ cc_library(
         ":Support",
         ":TransformUtils",
         ":TransformsPassIncGen",
+        ":UBDialect",
         "//llvm:Support",
     ],
 )
@@ -8632,36 +8686,6 @@ cc_library(
     ],
 )
 
-cc_library(
-    name = "ConvertToSPIRV",
-    srcs = ["lib/Conversion/ConvertToSPIRV/ConvertToSPIRVPass.cpp"],
-    hdrs = ["include/mlir/Conversion/ConvertToSPIRV/ConvertToSPIRVPass.h"],
-    includes = ["include"],
-    deps = [
-        ":ArithToSPIRV",
-        ":ArithTransforms",
-        ":ConversionPassIncGen",
-        ":FuncToSPIRV",
-        ":GPUDialect",
-        ":GPUToSPIRV",
-        ":IR",
-        ":IndexToSPIRV",
-        ":MemRefToSPIRV",
-        ":Pass",
-        ":Rewrite",
-        ":SCFToSPIRV",
-        ":SPIRVConversion",
-        ":SPIRVDialect",
-        ":SPIRVTransforms",
-        ":TransformUtils",
-        ":Transforms",
-        ":UBToSPIRV",
-        ":VectorDialect",
-        ":VectorToSPIRV",
-        ":VectorTransforms",
-    ],
-)
-
 cc_library(
     name = "VectorToXeGPU",
     srcs = ["lib/Conversion/VectorToXeGPU/VectorToXeGPU.cpp"],
@@ -9268,9 +9292,9 @@ cc_library(
         ":IR",
         ":LLVMConversionIncGen",
         ":LLVMDialect",
-        ":LLVMIntrinsicConversionIncGen",
         ":LLVMIRTransformsDIExpressionLegalization",
         ":LLVMIRTransformsLegalizeForExport",
+        ":LLVMIntrinsicConversionIncGen",
         ":OpenMPDialect",
         ":Support",
         ":TransformUtils",
@@ -10196,12 +10220,6 @@ cc_binary(
     ],
 )
 
-# TODO: Remove this alias.
-binary_alias(
-    name = "mlir-cpu-runner",
-    binary = ":mlir-runner",
-)
-
 # This target provides the headers from LLVM's Support target without any of
 # the symbols. In particular, it does not contain the static registration code
 # which may be executed by at most one shared library loaded by ORCJit. Direct
@@ -10416,7 +10434,7 @@ gentbl_cc_library(
     ],
     tblgen = ":mlir-tblgen",
     td_file = "//llvm:include/llvm/Frontend/OpenACC/ACC.td",
-    deps = ["//llvm:acc_td_files"],
+    deps = ["//llvm:AccTdFiles"],
 )
 
 td_library(
@@ -10663,7 +10681,7 @@ gentbl_cc_library(
     td_file = "//llvm:include/llvm/Frontend/OpenMP/OMP.td",
     deps = [
         ":OpBaseTdFiles",
-        "//llvm:omp_td_files",
+        "//llvm:OmpTdFiles",
     ],
 )
 
@@ -12017,6 +12035,7 @@ cc_library(
         ":ArmSVEDialect",
         ":ArmSVETransforms",
         ":ConversionPassIncGen",
+        ":ConvertToLLVMInterface",
         ":FuncDialect",
         ":IR",
         ":LLVMCommonConversion",
@@ -13510,6 +13529,8 @@ cc_library(
         ":CopyOpInterface",
         ":DialectUtils",
         ":IR",
+        ":InferIntRangeCommon",
+        ":InferIntRangeInterface",
         ":InferTypeOpInterface",
         ":InliningUtils",
         ":MemRefBaseIncGen",
diff --git a/utils/bazel/llvm-project-overlay/mlir/test/BUILD.bazel b/utils/bazel/llvm-project-overlay/mlir/test/BUILD.bazel
index f2e32b1b8c6db..3e6114abfc078 100644
--- a/utils/bazel/llvm-project-overlay/mlir/test/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/mlir/test/BUILD.bazel
@@ -546,21 +546,33 @@ cc_library(
     deps = [
         ":TestDialect",
         "//llvm:Support",
-        "//mlir:ConvertToSPIRV",
+        "//mlir:ArithToSPIRV",
+        "//mlir:ArithTransforms",
         "//mlir:FuncDialect",
+        "//mlir:FuncToSPIRV",
         "//mlir:GPUDialect",
         "//mlir:GPUToGPURuntimeTransforms",
         "//mlir:GPUToSPIRV",
         "//mlir:GPUTransforms",
         "//mlir:IR",
+        "//mlir:IndexToSPIRV",
         "//mlir:LLVMIRTransforms",
         "//mlir:MemRefToLLVM",
+        "//mlir:MemRefToSPIRV",
         "//mlir:MemRefTransforms",
         "//mlir:Pass",
+        "//mlir:Rewrite",
+        "//mlir:SCFToSPIRV",
+        "//mlir:SPIRVConversion",
         "//mlir:SPIRVDialect",
         "//mlir:SPIRVToLLVM",
         "//mlir:SPIRVTransforms",
         "//mlir:Support",
+        "//mlir:TransformUtils",
+        "//mlir:UBToSPIRV",
+        "//mlir:VectorDialect",
+        "//mlir:VectorToSPIRV",
+        "//mlir:VectorTransforms",
     ],
 )
 
@@ -678,6 +690,7 @@ cc_library(
         "//mlir:SPIRVDialect",
         "//mlir:TransformUtils",
         "//mlir:Transforms",
+        "//mlir:UBDialect",
         "//mlir:VectorDialect",
         "//mlir:VectorTransforms",
     ],
@@ -827,6 +840,7 @@ cc_library(
         "//mlir:SCFToControlFlow",
         "//mlir:TransformUtils",
         "//mlir:Transforms",
+        "//mlir:UBToLLVM",
         "//mlir:VectorToLLVM",
         "//mlir:VectorToSCF",
     ],
diff --git a/utils/bazel/llvm-project-overlay/mlir/test/mlir-tblgen/BUILD.bazel b/utils/bazel/llvm-project-overlay/mlir/test/mlir-tblgen/BUILD.bazel
index 7fe1fdc8c6936..1dd418c75984e 100644
--- a/utils/bazel/llvm-project-overlay/mlir/test/mlir-tblgen/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/mlir/test/mlir-tblgen/BUILD.bazel
@@ -34,6 +34,7 @@ package(default_visibility = ["//visibility:public"])
             "//mlir:include/mlir/IR/OpBase.td",
             "//mlir:include/mlir/Interfaces/InferTypeOpInterface.td",
             "//mlir:include/mlir/Interfaces/SideEffectInterfaces.td",
+            "//mlir:include/mlir/Pass/PassBase.td",
             "//mlir:mlir-opt",
             "//mlir:mlir-tblgen",
             "//mlir/test:lit_data",